From 7097630f3d137c0088c0b60c4b96dd4ecac06409 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Sun, 23 Oct 2022 20:01:27 +0800 Subject: [PATCH] [CodeStyle][black] use black instead of yapf (#46014) * update config * re-blacken python code * temporarily disable date and diff_py_file * skip a format --- .cmake-format.py | 2 +- .pre-commit-config.yaml | 14 +- .style.yapf | 3 - cmake/copyfile.py | 4 +- cmake/make_resource.py | 16 +- .../generate_file_structures.py | 57 +- .../generator/codegen_utils.py | 195 +- .../generator/eager_gen.py | 1267 ++-- .../generator/python_c_gen.py | 293 +- .../api/full_ILSVRC2012_val_preprocess.py | 95 +- .../api/full_pascalvoc_test_preprocess.py | 152 +- .../api/test_detection_dataset_preprocess.py | 4 +- .../fluid/pybind/generate_file_structures.py | 3 +- paddle/infrt/tests/models/abs_model.py | 6 +- .../tests/models/efficientnet-b4/model.py | 5 +- .../efficientnet-b4/net/efficientnet.py | 206 +- .../tests/models/efficientnet-b4/net/utils.py | 292 +- paddle/infrt/tests/models/linear.py | 12 +- paddle/infrt/tests/models/resnet50_model.py | 5 +- paddle/phi/api/yaml/generator/api_base.py | 651 +- paddle/phi/api/yaml/generator/api_gen.py | 210 +- .../api/yaml/generator/backward_api_gen.py | 201 +- .../phi/api/yaml/generator/cross_validate.py | 27 +- paddle/phi/api/yaml/generator/filters.py | 27 +- paddle/phi/api/yaml/generator/generate_op.py | 189 +- .../api/yaml/generator/generate_sparse_op.py | 80 +- .../yaml/generator/intermediate_api_gen.py | 79 +- .../api/yaml/generator/ops_extra_info_gen.py | 49 +- paddle/phi/api/yaml/generator/parse_api.py | 9 +- paddle/phi/api/yaml/generator/parse_utils.py | 192 +- .../phi/api/yaml/generator/sparse_api_gen.py | 167 +- .../api/yaml/generator/sparse_bw_api_gen.py | 86 +- .../phi/api/yaml/generator/strings_api_gen.py | 172 +- paddle/phi/api/yaml/generator/type_mapping.py | 24 +- .../yaml/generator/wrapped_infermeta_gen.py | 74 +- paddle/scripts/paddle_build.sh | 219 +- pyproject.toml | 4 + python/paddle/__init__.py | 12 +- python/paddle/amp/auto_cast.py | 31 +- python/paddle/amp/grad_scaler.py | 39 +- python/paddle/audio/__init__.py | 8 +- python/paddle/audio/backends/backend.py | 26 +- python/paddle/audio/backends/init_backend.py | 9 +- python/paddle/audio/backends/wave_backend.py | 19 +- python/paddle/audio/datasets/dataset.py | 25 +- python/paddle/audio/datasets/esc50.py | 39 +- python/paddle/audio/datasets/tess.py | 49 +- python/paddle/audio/features/layers.py | 277 +- python/paddle/audio/functional/functional.py | 97 +- python/paddle/audio/functional/window.py | 165 +- python/paddle/audio/utils/error.py | 1 + python/paddle/autograd/__init__.py | 1 + python/paddle/autograd/backward_mode.py | 18 +- python/paddle/autograd/py_layer.py | 44 +- python/paddle/autograd/saved_tensors_hooks.py | 7 +- python/paddle/batch.py | 6 +- python/paddle/callbacks.py | 11 +- python/paddle/check_import_scipy.py | 6 +- python/paddle/common_ops_import.py | 22 +- python/paddle/cost_model/cost_model.py | 33 +- python/paddle/dataset/cifar.py | 47 +- python/paddle/dataset/common.py | 39 +- python/paddle/dataset/conll05.py | 47 +- python/paddle/dataset/flowers.py | 84 +- python/paddle/dataset/image.py | 49 +- python/paddle/dataset/imdb.py | 37 +- python/paddle/dataset/imikolov.py | 43 +- python/paddle/dataset/mnist.py | 40 +- python/paddle/dataset/movielens.py | 61 +- python/paddle/dataset/tests/cifar_test.py | 13 +- python/paddle/dataset/tests/flowers_test.py | 10 +- python/paddle/dataset/tests/imikolov_test.py | 21 +- python/paddle/dataset/tests/mnist_test.py | 7 +- python/paddle/dataset/tests/test_image.py | 2 +- python/paddle/dataset/tests/voc2012_test.py | 1 - python/paddle/dataset/tests/wmt16_test.py | 26 +- python/paddle/dataset/uci_housing.py | 54 +- python/paddle/dataset/voc2012.py | 9 +- python/paddle/dataset/wmt14.py | 46 +- python/paddle/dataset/wmt16.py | 161 +- python/paddle/device/__init__.py | 65 +- python/paddle/device/cuda/__init__.py | 32 +- python/paddle/device/cuda/graphs.py | 125 +- python/paddle/distributed/__init__.py | 46 +- .../distributed/auto_parallel/callbacks.py | 79 +- .../distributed/auto_parallel/cluster.py | 114 +- .../distributed/auto_parallel/cluster_v2.py | 29 +- .../distributed/auto_parallel/completion.py | 1166 ++- .../distributed/auto_parallel/converter.py | 208 +- .../auto_parallel/cost/base_cost.py | 240 +- .../auto_parallel/cost/comm_op_cost.py | 102 +- .../auto_parallel/cost/comp_op_cost.py | 400 +- .../auto_parallel/cost/estimate_cost.py | 196 +- .../auto_parallel/cost/tensor_cost.py | 25 +- .../distributed/auto_parallel/cost_model.py | 285 +- .../auto_parallel/dist_attribute.py | 154 +- .../distributed/auto_parallel/dist_context.py | 465 +- .../distributed/auto_parallel/dist_loader.py | 152 +- .../distributed/auto_parallel/dist_op.py | 153 +- .../distributed/auto_parallel/dist_saver.py | 30 +- .../distributed/auto_parallel/dist_tensor.py | 222 +- .../distributed/auto_parallel/engine.py | 675 +- .../paddle/distributed/auto_parallel/graph.py | 6 +- .../distributed/auto_parallel/helper.py | 61 +- .../distributed/auto_parallel/interface.py | 83 +- .../distributed/auto_parallel/mapper.py | 88 +- .../auto_parallel/operators/common.py | 162 +- .../auto_parallel/operators/dist_assign.py | 10 +- .../dist_check_finite_and_unscale.py | 127 +- .../auto_parallel/operators/dist_default.py | 217 +- .../auto_parallel/operators/dist_eltwise.py | 106 +- .../auto_parallel/operators/dist_embedding.py | 480 +- .../dist_fill_constant_batch_size_like.py | 32 +- .../operators/dist_fused_attention.py | 72 +- .../operators/dist_fused_feedforward.py | 74 +- .../auto_parallel/operators/dist_matmul.py | 1648 ++-- .../auto_parallel/operators/dist_pnorm.py | 168 +- .../operators/dist_reduce_sum_p.py | 56 +- .../auto_parallel/operators/dist_reshape.py | 239 +- .../auto_parallel/operators/dist_shape.py | 7 +- .../auto_parallel/operators/dist_slice.py | 26 +- .../auto_parallel/operators/dist_softmax.py | 49 +- .../auto_parallel/operators/dist_split.py | 23 +- .../auto_parallel/operators/dist_transpose.py | 58 +- .../operators/dist_update_loss_scaling.py | 115 +- .../distributed/auto_parallel/parallelizer.py | 306 +- .../auto_parallel/parallelizer_v2.py | 196 +- .../distributed/auto_parallel/partitioner.py | 339 +- .../distributed/auto_parallel/planner.py | 691 +- .../distributed/auto_parallel/planner_v2.py | 9 +- .../auto_parallel/process_group.py | 52 +- .../distributed/auto_parallel/process_mesh.py | 44 +- .../auto_parallel/process_mesh_v2.py | 40 +- .../distributed/auto_parallel/reshard.py | 1563 ++-- .../distributed/auto_parallel/strategy.py | 15 +- .../auto_parallel/tuner/algorithms.py | 5 +- .../distributed/auto_parallel/tuner/config.py | 6 +- .../auto_parallel/tuner/optimization_tuner.py | 239 +- .../auto_parallel/tuner/parallel_tuner.py | 644 +- .../auto_parallel/tuner/profiler.py | 75 +- .../auto_parallel/tuner/recorder.py | 4 +- .../auto_parallel/tuner/storable.py | 1 - .../distributed/auto_parallel/tuner/trial.py | 29 +- .../auto_parallel/tuner/tunable_space.py | 40 +- .../auto_parallel/tuner/tunable_variable.py | 53 +- .../paddle/distributed/auto_parallel/utils.py | 728 +- python/paddle/distributed/cloud_utils.py | 63 +- python/paddle/distributed/collective.py | 659 +- .../distributed/communication/all_reduce.py | 32 +- .../paddle/distributed/communication/group.py | 12 +- .../distributed/communication/reduce.py | 1 + .../communication/stream/__init__.py | 12 +- .../communication/stream/all_gather.py | 50 +- .../communication/stream/all_reduce.py | 53 +- .../communication/stream/alltoall.py | 70 +- .../communication/stream/alltoall_single.py | 52 +- .../communication/stream/broadcast.py | 8 +- .../distributed/communication/stream/recv.py | 6 +- .../communication/stream/reduce.py | 22 +- .../communication/stream/reduce_scatter.py | 119 +- .../communication/stream/scatter.py | 68 +- .../distributed/communication/stream/send.py | 6 +- python/paddle/distributed/elastic.py | 10 +- python/paddle/distributed/entry_attr.py | 6 +- python/paddle/distributed/fleet/__init__.py | 20 +- .../paddle/distributed/fleet/ascend_utils.py | 30 +- .../fleet/base/distributed_strategy.py | 421 +- .../fleet/base/meta_optimizer_factory.py | 4 +- .../fleet/base/orthogonal_strategy.py | 60 +- .../fleet/base/private_helper_function.py | 10 +- .../distributed/fleet/base/role_maker.py | 287 +- .../distributed/fleet/base/runtime_factory.py | 1 - .../fleet/base/strategy_compiler.py | 36 +- .../distributed/fleet/base/strategy_group.py | 52 +- .../paddle/distributed/fleet/base/topology.py | 119 +- .../distributed/fleet/base/util_factory.py | 243 +- .../paddle/distributed/fleet/cloud_utils.py | 48 +- .../fleet/data_generator/data_generator.py | 43 +- .../distributed/fleet/dataset/dataset.py | 124 +- .../fleet/dataset/index_dataset.py | 21 +- .../distributed/fleet/elastic/__init__.py | 7 +- .../distributed/fleet/elastic/collective.py | 13 +- .../distributed/fleet/elastic/manager.py | 162 +- python/paddle/distributed/fleet/fleet.py | 405 +- .../distributed/fleet/fleet_executor_utils.py | 290 +- python/paddle/distributed/fleet/launch.py | 312 +- .../paddle/distributed/fleet/launch_utils.py | 1002 ++- .../distributed/fleet/layers/mpu/mp_layers.py | 265 +- .../distributed/fleet/layers/mpu/mp_ops.py | 668 +- .../distributed/fleet/layers/mpu/random.py | 104 +- .../fleet/meta_optimizers/amp_optimizer.py | 87 +- .../ascend/ascend_optimizer.py | 154 +- .../meta_optimizers/ascend/ascend_parser.py | 2202 +++--- .../fleet/meta_optimizers/asp_optimizer.py | 32 +- .../fleet/meta_optimizers/common.py | 283 +- .../fleet/meta_optimizers/dgc_optimizer.py | 56 +- .../dygraph_sharding_optimizer.py | 56 +- .../heter_parallel_optimizer.py | 21 +- .../hybrid_parallel_gradscaler.py | 18 +- .../hybrid_parallel_optimizer.py | 140 +- .../sharding_optimizer_stage2.py | 186 +- .../fp16_allreduce_optimizer.py | 88 +- .../gradient_merge_optimizer.py | 37 +- .../graph_execution_optimizer.py | 150 +- .../fleet/meta_optimizers/lamb_optimizer.py | 64 +- .../fleet/meta_optimizers/lars_optimizer.py | 62 +- .../meta_optimizers/localsgd_optimizer.py | 478 +- .../meta_optimizers/meta_optimizer_base.py | 81 +- .../parameter_server_graph_optimizer.py | 19 +- .../parameter_server_optimizer.py | 216 +- .../meta_optimizers/pipeline_optimizer.py | 208 +- .../fleet/meta_optimizers/ps_optimizer.py | 122 +- .../meta_optimizers/raw_program_optimizer.py | 339 +- .../meta_optimizers/recompute_optimizer.py | 57 +- .../meta_optimizers/sharding/fp16_helper.py | 217 +- .../sharding/gradient_clip_helper.py | 80 +- .../sharding/offload_helper.py | 222 +- .../fleet/meta_optimizers/sharding/prune.py | 43 +- .../fleet/meta_optimizers/sharding/shard.py | 45 +- .../fleet/meta_optimizers/sharding/utils.py | 820 +- .../sharding/weight_decay_helper.py | 9 +- .../meta_optimizers/sharding_optimizer.py | 1267 ++-- .../tensor_parallel_optimizer.py | 180 +- .../fleet/meta_parallel/meta_parallel_base.py | 6 +- .../parallel_layers/pp_layers.py | 327 +- .../fleet/meta_parallel/pipeline_parallel.py | 293 +- .../pp_utils/p2p_communication.py | 490 +- .../fleet/meta_parallel/pp_utils/utils.py | 22 +- .../group_sharded_optimizer_stage2.py | 213 +- .../sharding/group_sharded_stage2.py | 256 +- .../sharding/group_sharded_stage3.py | 496 +- .../sharding/group_sharded_storage.py | 86 +- .../sharding/group_sharded_utils.py | 112 +- .../meta_parallel/sharding/sharding_stage2.py | 265 +- .../meta_parallel/sharding/sharding_stage3.py | 557 +- .../meta_parallel/sharding/sharding_utils.py | 112 +- .../fleet/meta_parallel/sharding_parallel.py | 1 - .../fleet/meta_parallel/tensor_parallel.py | 6 +- .../distributed/fleet/metrics/metric.py | 9 +- python/paddle/distributed/fleet/model.py | 49 +- python/paddle/distributed/fleet/optimizer.py | 55 +- .../distributed/fleet/recompute/recompute.py | 194 +- .../fleet/recompute/recompute_hybrid.py | 104 +- .../fleet/runtime/collective_runtime.py | 16 +- .../fleet/runtime/parameter_server_runtime.py | 534 +- .../distributed/fleet/runtime/runtime_base.py | 1 - .../distributed/fleet/runtime/the_one_ps.py | 574 +- python/paddle/distributed/fleet/scaler.py | 53 +- .../distributed/fleet/utils/__init__.py | 14 +- python/paddle/distributed/fleet/utils/fs.py | 77 +- .../distributed/fleet/utils/http_server.py | 9 +- .../fleet/utils/hybrid_parallel_inference.py | 296 +- .../fleet/utils/hybrid_parallel_util.py | 114 +- .../fleet/utils/internal_storage.py | 105 +- .../distributed/fleet/utils/log_util.py | 5 +- .../paddle/distributed/fleet/utils/ps_util.py | 132 +- .../distributed/launch/context/__init__.py | 9 +- .../distributed/launch/context/args_envs.py | 193 +- .../distributed/launch/context/device.py | 8 +- .../distributed/launch/context/event.py | 1 - .../paddle/distributed/launch/context/node.py | 10 +- .../distributed/launch/context/resource.py | 1 - .../launch/controllers/collective.py | 59 +- .../launch/controllers/controller.py | 55 +- .../launch/controllers/ipu_controller.py | 109 +- .../distributed/launch/controllers/master.py | 37 +- .../distributed/launch/controllers/ps.py | 106 +- .../distributed/launch/controllers/rpc.py | 24 +- .../distributed/launch/controllers/watcher.py | 6 +- .../distributed/launch/job/container.py | 31 +- python/paddle/distributed/launch/job/job.py | 15 +- python/paddle/distributed/launch/job/pod.py | 27 +- python/paddle/distributed/launch/main.py | 1 + .../distributed/launch/plugins/__init__.py | 16 +- .../paddle/distributed/launch/plugins/test.py | 25 +- .../distributed/launch/utils/kv_client.py | 8 +- .../distributed/launch/utils/kv_server.py | 9 +- .../paddle/distributed/launch/utils/nvsmi.py | 28 +- .../launch/utils/process_context.py | 33 +- python/paddle/distributed/metric/metrics.py | 170 +- python/paddle/distributed/models/moe/utils.py | 123 +- python/paddle/distributed/parallel.py | 111 +- .../paddle/distributed/parallel_with_gloo.py | 17 +- .../distributed/passes/auto_parallel_amp.py | 519 +- ...uto_parallel_data_parallel_optimization.py | 250 +- .../distributed/passes/auto_parallel_fp16.py | 425 +- .../passes/auto_parallel_grad_clip.py | 104 +- .../passes/auto_parallel_gradient_merge.py | 273 +- .../passes/auto_parallel_quantization.py | 153 +- .../passes/auto_parallel_recompute.py | 178 +- .../passes/auto_parallel_sharding.py | 485 +- python/paddle/distributed/passes/cpp_pass.py | 39 +- .../distributed/passes/fuse_all_reduce.py | 95 +- python/paddle/distributed/passes/pass_base.py | 58 +- .../paddle/distributed/passes/pass_utils.py | 5 +- .../distributed/passes/ps_server_pass.py | 129 +- .../distributed/passes/ps_trainer_pass.py | 759 +- python/paddle/distributed/ps/coordinator.py | 120 +- python/paddle/distributed/ps/the_one_ps.py | 708 +- .../paddle/distributed/ps/utils/ps_factory.py | 12 +- python/paddle/distributed/ps/utils/public.py | 756 +- python/paddle/distributed/rpc/rpc.py | 50 +- .../paddle/distributed/sharding/__init__.py | 5 +- .../distributed/sharding/group_sharded.py | 144 +- python/paddle/distributed/spawn.py | 189 +- .../paddle/distributed/utils/launch_utils.py | 181 +- python/paddle/distributed/utils/moe_utils.py | 121 +- python/paddle/distribution/__init__.py | 21 +- python/paddle/distribution/beta.py | 9 +- python/paddle/distribution/categorical.py | 60 +- python/paddle/distribution/constraint.py | 14 +- python/paddle/distribution/dirichlet.py | 52 +- python/paddle/distribution/distribution.py | 63 +- .../paddle/distribution/exponential_family.py | 6 +- python/paddle/distribution/gumbel.py | 30 +- python/paddle/distribution/independent.py | 23 +- python/paddle/distribution/kl.py | 77 +- python/paddle/distribution/laplace.py | 57 +- python/paddle/distribution/lognormal.py | 8 +- python/paddle/distribution/multinomial.py | 57 +- python/paddle/distribution/normal.py | 105 +- python/paddle/distribution/transform.py | 212 +- .../distribution/transformed_distribution.py | 38 +- python/paddle/distribution/uniform.py | 101 +- python/paddle/distribution/variable.py | 37 +- python/paddle/fft.py | 292 +- python/paddle/fluid/__init__.py | 68 +- python/paddle/fluid/average.py | 20 +- python/paddle/fluid/backward.py | 1009 ++- python/paddle/fluid/clip.py | 227 +- python/paddle/fluid/communicator.py | 52 +- python/paddle/fluid/compiler.py | 326 +- .../contrib/decoder/beam_search_decoder.py | 319 +- .../extend_optimizer_with_weight_decay.py | 74 +- .../paddle/fluid/contrib/layers/metric_op.py | 336 +- python/paddle/fluid/contrib/layers/nn.py | 1389 ++-- .../paddle/fluid/contrib/layers/rnn_impl.py | 358 +- .../paddle/fluid/contrib/memory_usage_calc.py | 8 +- .../fluid/contrib/mixed_precision/amp_nn.py | 96 +- .../contrib/mixed_precision/bf16/amp_lists.py | 45 +- .../contrib/mixed_precision/bf16/amp_utils.py | 279 +- .../contrib/mixed_precision/bf16/decorator.py | 124 +- .../contrib/mixed_precision/decorator.py | 310 +- .../contrib/mixed_precision/fp16_lists.py | 36 +- .../contrib/mixed_precision/fp16_utils.py | 283 +- python/paddle/fluid/contrib/model_stat.py | 9 +- python/paddle/fluid/contrib/op_frequence.py | 24 +- python/paddle/fluid/contrib/optimizer.py | 149 +- .../contrib/quantize/quantize_transpiler.py | 312 +- .../contrib/slim/quantization/adaround.py | 295 +- .../slim/quantization/cal_kl_threshold.py | 39 +- .../quantization/imperative/fuse_utils.py | 100 +- .../slim/quantization/imperative/ptq.py | 148 +- .../slim/quantization/imperative/ptq_hooks.py | 7 +- .../quantization/imperative/ptq_quantizer.py | 87 +- .../quantization/imperative/ptq_registry.py | 34 +- .../slim/quantization/imperative/qat.py | 306 +- .../slim/quantization/imperative/utils.py | 38 +- .../post_training_quantization.py | 1172 +-- .../quantization/quant2_int8_mkldnn_pass.py | 332 +- .../quantization/quant_int8_mkldnn_pass.py | 104 +- .../slim/quantization/quantization_pass.py | 1632 ++-- .../quantization/quantize_transpiler_v2.py | 366 +- .../fluid/contrib/slim/quantization/utils.py | 85 +- .../contrib/slim/tests/convert_model2dot.py | 51 +- .../slim/tests/imperative_test_utils.py | 190 +- ...t2_int8_image_classification_comparison.py | 271 +- .../slim/tests/quant2_int8_lstm_model.py | 188 +- .../slim/tests/quant2_int8_nlp_comparison.py | 236 +- ...nt_int8_image_classification_comparison.py | 193 +- .../contrib/slim/tests/save_quant_model.py | 135 +- .../fluid/contrib/slim/tests/test_graph.py | 59 +- .../slim/tests/test_imperative_out_scale.py | 136 +- .../contrib/slim/tests/test_imperative_ptq.py | 227 +- .../contrib/slim/tests/test_imperative_qat.py | 157 +- .../slim/tests/test_imperative_qat_amp.py | 102 +- .../tests/test_imperative_qat_channelwise.py | 8 +- .../slim/tests/test_imperative_qat_fuse.py | 7 +- .../slim/tests/test_imperative_qat_lsq.py | 166 +- .../tests/test_imperative_qat_user_defined.py | 151 +- .../slim/tests/test_imperative_skip_op.py | 68 +- .../test_moving_average_abs_max_scale_op.py | 33 +- ...t_post_training_quantization_lstm_model.py | 262 +- .../test_post_training_quantization_mnist.py | 516 +- ..._post_training_quantization_mobilenetv1.py | 340 +- ..._training_quantization_program_resnet50.py | 222 +- ...est_post_training_quantization_resnet50.py | 45 +- .../test_post_training_quantization_while.py | 358 +- .../tests/test_quant2_int8_mkldnn_pass.py | 236 +- .../tests/test_quantization_mkldnn_pass.py | 118 +- .../slim/tests/test_quantization_pass.py | 810 +- .../tests/test_quantization_scale_pass.py | 103 +- .../slim/tests/test_quantize_transpiler_v2.py | 143 +- .../tests/test_user_defined_quantization.py | 126 +- .../test_weight_quantization_mobilenetv1.py | 226 +- .../paddle/fluid/contrib/sparsity/__init__.py | 19 +- python/paddle/fluid/contrib/sparsity/asp.py | 252 +- .../contrib/sparsity/supported_layer_list.py | 43 +- python/paddle/fluid/contrib/sparsity/utils.py | 114 +- .../fluid/contrib/tests/test_amp_list.py | 12 +- .../fluid/contrib/tests/test_bf16_utils.py | 94 +- .../fluid/contrib/tests/test_correlation.py | 152 +- .../fluid/contrib/tests/test_fp16_utils.py | 17 +- .../tests/test_image_classification_fp16.py | 299 +- .../contrib/tests/test_model_cast_to_bf16.py | 106 +- .../tests/test_multi_precision_fp16_train.py | 209 +- .../contrib/tests/test_quantize_transpiler.py | 167 +- .../contrib/tests/test_weight_decay_extend.py | 115 +- python/paddle/fluid/core.py | 122 +- python/paddle/fluid/data.py | 18 +- python/paddle/fluid/data_feed_desc.py | 6 +- python/paddle/fluid/data_feeder.py | 206 +- python/paddle/fluid/dataloader/__init__.py | 10 +- .../paddle/fluid/dataloader/batch_sampler.py | 118 +- python/paddle/fluid/dataloader/collate.py | 12 +- .../fluid/dataloader/dataloader_iter.py | 207 +- python/paddle/fluid/dataloader/dataset.py | 69 +- python/paddle/fluid/dataloader/fetcher.py | 85 +- python/paddle/fluid/dataloader/flat.py | 50 +- python/paddle/fluid/dataloader/sampler.py | 75 +- python/paddle/fluid/dataloader/worker.py | 103 +- python/paddle/fluid/dataset.py | 168 +- python/paddle/fluid/debugger.py | 90 +- python/paddle/fluid/device_worker.py | 273 +- .../paddle/fluid/distribute_lookup_table.py | 6 +- python/paddle/fluid/distributed/downpour.py | 91 +- python/paddle/fluid/distributed/fleet.py | 25 +- python/paddle/fluid/distributed/helper.py | 22 +- python/paddle/fluid/distributed/node.py | 70 +- .../paddle/fluid/distributed/ps_instance.py | 16 +- python/paddle/fluid/distributed/ps_pb2.py | 1229 +-- python/paddle/fluid/dygraph/amp/auto_cast.py | 154 +- .../paddle/fluid/dygraph/amp/loss_scaler.py | 176 +- python/paddle/fluid/dygraph/base.py | 172 +- python/paddle/fluid/dygraph/checkpoint.py | 56 +- python/paddle/fluid/dygraph/container.py | 17 +- .../dygraph_to_static/assert_transformer.py | 21 +- .../dygraph_to_static/ast_transformer.py | 93 +- .../dygraph_to_static/base_transformer.py | 271 +- .../basic_api_transformer.py | 33 +- .../break_continue_transformer.py | 175 +- .../dygraph_to_static/call_transformer.py | 13 +- .../dygraph_to_static/cast_transformer.py | 8 +- .../dygraph_to_static/convert_call_func.py | 97 +- .../dygraph_to_static/convert_operators.py | 272 +- .../create_variable_transformer.py | 29 +- .../decorator_transformer.py | 87 +- .../early_return_transformer.py | 27 +- .../fluid/dygraph/dygraph_to_static/error.py | 188 +- .../dygraph_to_static/function_spec.py | 101 +- .../dygraph_to_static/ifelse_transformer.py | 220 +- .../dygraph_to_static/logging_utils.py | 24 +- .../dygraph_to_static/logical_transformer.py | 21 +- .../dygraph_to_static/loop_transformer.py | 237 +- .../dygraph/dygraph_to_static/origin_info.py | 68 +- .../dygraph_to_static/partial_program.py | 431 +- .../dygraph_to_static/print_transformer.py | 12 +- .../dygraph_to_static/program_translator.py | 432 +- .../dygraph_to_static/return_transformer.py | 191 +- .../dygraph_to_static/static_analysis.py | 83 +- .../tensor_shape_transformer.py | 8 +- .../dygraph_to_static/typehint_transformer.py | 8 +- .../fluid/dygraph/dygraph_to_static/utils.py | 526 +- .../dygraph_to_static/variable_trans_func.py | 14 +- python/paddle/fluid/dygraph/inplace_utils.py | 7 +- python/paddle/fluid/dygraph/io.py | 572 +- python/paddle/fluid/dygraph/jit.py | 427 +- python/paddle/fluid/dygraph/layer_hooks.py | 11 +- .../fluid/dygraph/layer_object_helper.py | 63 +- python/paddle/fluid/dygraph/layers.py | 434 +- .../fluid/dygraph/learning_rate_scheduler.py | 345 +- python/paddle/fluid/dygraph/math_op_patch.py | 221 +- python/paddle/fluid/dygraph/nn.py | 2081 +++--- python/paddle/fluid/dygraph/parallel.py | 265 +- .../paddle/fluid/dygraph/parallel_helper.py | 19 +- python/paddle/fluid/dygraph/rnn.py | 175 +- python/paddle/fluid/dygraph/tracer.py | 85 +- .../fluid/dygraph/varbase_patch_methods.py | 214 +- python/paddle/fluid/dygraph_utils.py | 12 +- python/paddle/fluid/entry_attr.py | 8 +- python/paddle/fluid/evaluator.py | 234 +- python/paddle/fluid/executor.py | 1376 ++-- python/paddle/fluid/framework.py | 1625 ++-- python/paddle/fluid/generator.py | 3 +- python/paddle/fluid/graphviz.py | 153 +- .../incubate/checkpoint/auto_checkpoint.py | 233 +- .../incubate/checkpoint/checkpoint_saver.py | 81 +- .../fluid/incubate/data_generator/__init__.py | 48 +- .../fluid/incubate/fleet/base/fleet_base.py | 68 +- .../paddle/fluid/incubate/fleet/base/mode.py | 1 + .../fluid/incubate/fleet/base/role_maker.py | 211 +- .../incubate/fleet/collective/__init__.py | 336 +- .../distribute_transpiler/__init__.py | 514 +- .../distributed_strategy.py | 151 +- .../parameter_server/ir/heter_trainer_pass.py | 51 +- .../fleet/parameter_server/ir/pserver_pass.py | 547 +- .../fleet/parameter_server/ir/public.py | 643 +- .../fleet/parameter_server/ir/trainer_pass.py | 991 +-- .../fleet/parameter_server/ir/ufind.py | 2 +- .../parameter_server/ir/vars_metatools.py | 115 +- .../incubate/fleet/parameter_server/mode.py | 1 + .../fleet/parameter_server/pslib/__init__.py | 461 +- .../fleet/parameter_server/pslib/node.py | 550 +- .../pslib/optimizer_factory.py | 575 +- .../fleet/parameter_server/pslib/ps_pb2.py | 1510 ++-- .../fleet/tests/ctr_dataset_reader.py | 19 +- .../incubate/fleet/tests/fleet_deep_ctr.py | 136 +- .../fluid/incubate/fleet/utils/fleet_util.py | 747 +- .../paddle/fluid/incubate/fleet/utils/hdfs.py | 54 +- .../fluid/incubate/fleet/utils/http_server.py | 10 +- .../fluid/incubate/fleet/utils/utils.py | 326 +- python/paddle/fluid/inference/__init__.py | 19 +- python/paddle/fluid/inference/wrapper.py | 39 +- python/paddle/fluid/initializer.py | 846 ++- python/paddle/fluid/input.py | 79 +- python/paddle/fluid/install_check.py | 67 +- python/paddle/fluid/io.py | 829 +- python/paddle/fluid/ir.py | 148 +- python/paddle/fluid/layer_helper.py | 76 +- python/paddle/fluid/layer_helper_base.py | 365 +- python/paddle/fluid/layers/collective.py | 174 +- python/paddle/fluid/layers/control_flow.py | 1907 +++-- python/paddle/fluid/layers/detection.py | 1935 ++--- python/paddle/fluid/layers/device.py | 9 +- python/paddle/fluid/layers/distributions.py | 161 +- python/paddle/fluid/layers/io.py | 317 +- .../fluid/layers/layer_function_generator.py | 125 +- .../fluid/layers/learning_rate_scheduler.py | 310 +- python/paddle/fluid/layers/loss.py | 724 +- python/paddle/fluid/layers/math_op_patch.py | 311 +- python/paddle/fluid/layers/metric_op.py | 188 +- python/paddle/fluid/layers/nn.py | 6637 +++++++++-------- python/paddle/fluid/layers/ops.py | 236 +- python/paddle/fluid/layers/rnn.py | 1542 ++-- python/paddle/fluid/layers/sequence_lod.py | 437 +- python/paddle/fluid/layers/tensor.py | 887 ++- python/paddle/fluid/layers/utils.py | 159 +- python/paddle/fluid/lazy_init.py | 9 +- python/paddle/fluid/lod_tensor.py | 25 +- python/paddle/fluid/memory_analysis.py | 14 +- python/paddle/fluid/metrics.py | 181 +- python/paddle/fluid/multiprocess_utils.py | 11 +- python/paddle/fluid/net_drawer.py | 29 +- python/paddle/fluid/nets.py | 286 +- python/paddle/fluid/op.py | 48 +- python/paddle/fluid/optimizer.py | 4099 ++++++---- python/paddle/fluid/parallel_executor.py | 78 +- python/paddle/fluid/param_attr.py | 74 +- python/paddle/fluid/profiler.py | 43 +- python/paddle/fluid/reader.py | 536 +- python/paddle/fluid/regularizer.py | 54 +- .../tests/book/notest_understand_sentiment.py | 282 +- .../fluid/tests/book/test_fit_a_line.py | 92 +- .../tests/book/test_image_classification.py | 133 +- .../tests/book/test_label_semantic_roles.py | 351 +- .../tests/book/test_machine_translation.py | 197 +- .../fluid/tests/book/test_recognize_digits.py | 157 +- .../tests/book/test_recommender_system.py | 210 +- .../tests/book/test_rnn_encoder_decoder.py | 195 +- .../fluid/tests/book/test_word2vec_book.py | 230 +- .../custom_kernel_dot_c_setup.py | 21 +- .../custom_kernel/custom_kernel_dot_setup.py | 30 +- .../custom_kernel/test_custom_kernel_dot.py | 22 +- .../custom_kernel/test_custom_kernel_load.py | 30 +- .../custom_raw_op_kernel_op_setup.py | 16 +- .../tests/custom_op/custom_relu_setup.py | 4 +- .../fluid/tests/custom_op/test_check_abi.py | 18 +- .../tests/custom_op/test_context_pool.py | 7 +- .../tests/custom_op/test_custom_attrs_jit.py | 42 +- .../tests/custom_op/test_custom_concat.py | 61 +- .../fluid/tests/custom_op/test_custom_conj.py | 35 +- .../tests/custom_op/test_custom_linear.py | 92 +- .../custom_op/test_custom_raw_op_kernel_op.py | 19 +- .../tests/custom_op/test_custom_relu_model.py | 200 +- .../custom_op/test_custom_relu_op_jit.py | 47 +- .../custom_op/test_custom_relu_op_setup.py | 208 +- .../custom_op/test_custom_simple_slice.py | 13 +- .../custom_op/test_custom_tanh_double_grad.py | 33 +- .../tests/custom_op/test_dispatch_jit.py | 45 +- .../tests/custom_op/test_multi_out_jit.py | 23 +- .../fluid/tests/custom_op/test_sysconfig.py | 1 - python/paddle/fluid/tests/custom_op/utils.py | 2 +- .../custom_device_multi_process_collective.py | 17 +- .../custom_runtime/process_group_xccl.py | 39 +- .../test_collective_process_group_xccl.py | 86 +- .../custom_runtime/test_custom_cpu_plugin.py | 78 +- .../test_custom_cpu_profiler_plugin.py | 22 +- .../test_custom_cpu_to_static.py | 154 +- .../fluid/tests/test_beam_search_decoder.py | 146 +- python/paddle/fluid/tests/test_data_feeder.py | 36 +- python/paddle/fluid/tests/test_detection.py | 1002 +-- python/paddle/fluid/tests/test_error_clip.py | 30 +- python/paddle/fluid/tests/test_if_else_op.py | 67 +- python/paddle/fluid/tests/test_lod_tensor.py | 118 +- .../tests/test_python_operator_overriding.py | 33 +- python/paddle/fluid/tests/test_sequential.py | 1 - .../paddle/fluid/tests/unittests/__init__.py | 2 + .../fluid/tests/unittests/ascend_group.py | 108 +- .../ascend_multi_process_collective.py | 17 +- .../tests/unittests/asp/asp_pruning_base.py | 57 +- .../asp/test_asp_customized_pruning.py | 221 +- .../asp/test_asp_optimize_dynamic.py | 167 +- .../unittests/asp/test_asp_optimize_static.py | 199 +- .../unittests/asp/test_asp_pruning_dynamic.py | 71 +- .../unittests/asp/test_asp_pruning_static.py | 69 +- .../tests/unittests/asp/test_asp_save_load.py | 115 +- .../tests/unittests/asp/test_asp_utils.py | 133 +- .../asp/test_fleet_with_asp_dynamic.py | 95 +- .../asp/test_fleet_with_asp_sharding.py | 47 +- .../asp/test_fleet_with_asp_static.py | 144 +- .../tests/unittests/auto_checkpoint_utils.py | 28 +- .../auto_parallel/amp_pass_unittest.py | 9 +- .../auto_parallel_relaunch_model.py | 92 +- ...auto_parallel_relaunch_with_gpt_planner.py | 148 +- .../auto_parallel_relaunch_with_planner.py | 45 +- .../auto_parallel/clip_grad_by_global_norm.py | 11 +- .../unittests/auto_parallel/converter.py | 12 +- .../unittests/auto_parallel/engine_api.py | 278 +- .../unittests/auto_parallel/engine_api_dp.py | 61 +- .../unittests/auto_parallel/get_gpt_model.py | 85 +- .../gradient_merge_pass_unittest.py | 19 +- .../auto_parallel/high_order_grad.py | 32 +- .../auto_parallel/iterable_dataset.py | 72 +- .../auto_parallel/optimization_tuner_api.py | 62 +- .../auto_parallel/recompute_pass_unittest.py | 5 +- .../auto_parallel/sharding_pass_unittest.py | 23 +- .../test_auto_parallel_relaunch.py | 39 +- .../unittests/auto_parallel/test_base_cost.py | 154 +- .../unittests/auto_parallel/test_cluster.py | 11 +- .../auto_parallel/test_cluster_v2.py | 9 +- .../unittests/auto_parallel/test_comm_cost.py | 127 +- .../unittests/auto_parallel/test_comp_cost.py | 105 +- .../test_conditional_block_reshard.py | 45 +- .../unittests/auto_parallel/test_converter.py | 20 +- .../auto_parallel/test_dist_assign.py | 17 +- .../auto_parallel/test_dist_attr_v2.py | 55 +- .../auto_parallel/test_dist_context.py | 189 +- .../auto_parallel/test_dist_embedding.py | 38 +- .../auto_parallel/test_dist_matmul.py | 94 +- .../auto_parallel/test_dist_op_cost.py | 333 +- .../auto_parallel/test_dist_pnorm.py | 25 +- .../auto_parallel/test_dist_reshape.py | 11 +- .../auto_parallel/test_dist_shape.py | 11 +- .../auto_parallel/test_dist_slice.py | 16 +- .../auto_parallel/test_dist_split.py | 11 +- .../auto_parallel/test_engine_api.py | 18 +- .../auto_parallel/test_engine_api_dp.py | 18 +- .../auto_parallel/test_engine_callbacks.py | 62 +- .../auto_parallel/test_high_order_grad.py | 18 +- .../unittests/auto_parallel/test_interface.py | 115 +- .../auto_parallel/test_iterable_dataset.py | 18 +- .../auto_parallel/test_lr_grad_clip.py | 47 +- .../auto_parallel/test_new_cost_model.py | 34 +- .../test_optimization_tuner_api.py | 18 +- .../auto_parallel/test_parallel_tuner.py | 131 +- .../auto_parallel/test_parallel_tuner_full.py | 140 +- .../test_parallel_tuner_predict.py | 139 +- .../unittests/auto_parallel/test_pass_amp.py | 18 +- .../auto_parallel/test_pass_grad_clip.py | 23 +- .../auto_parallel/test_pass_gradient_merge.py | 23 +- .../auto_parallel/test_pass_quantization.py | 1 - .../auto_parallel/test_pass_recompute.py | 18 +- .../auto_parallel/test_pass_sharding.py | 18 +- .../auto_parallel/test_prim_dist_op.py | 86 +- .../auto_parallel/test_process_mesh.py | 68 +- .../auto_parallel/test_process_mesh_v2.py | 36 +- .../unittests/auto_parallel/test_recorder.py | 36 +- .../test_relaunch_with_gpt_planner.py | 37 +- .../test_relaunch_with_planner.py | 37 +- .../unittests/auto_parallel/test_strategy.py | 1 - .../unittests/auto_parallel/test_to_static.py | 91 +- .../unittests/auto_parallel/test_trial.py | 1 - .../auto_parallel/test_tunable_space.py | 8 +- .../auto_parallel/test_tunable_variable.py | 31 +- .../auto_parallel/test_while_op_completion.py | 118 +- .../auto_parallel/test_while_op_partition.py | 112 +- .../unittests/auto_parallel_autoconvert.py | 283 +- .../unittests/auto_parallel_data_unshard.py | 91 +- .../unittests/auto_parallel_gpt_model.py | 796 +- .../unittests/auto_parallel_save_load.py | 201 +- .../fluid/tests/unittests/autograd/config.py | 26 +- .../test_autograd_functional_dynamic.py | 419 +- .../autograd/test_autograd_functional_prim.py | 215 +- .../test_autograd_functional_static.py | 228 +- .../autograd/test_jvp_and_transpose.py | 323 +- .../tests/unittests/autograd/test_minimize.py | 30 +- .../unittests/autograd/test_orig2prim.py | 530 +- .../unittests/autograd/test_prim2orig.py | 284 +- .../tests/unittests/autograd/test_primapi.py | 809 +- .../tests/unittests/autograd/test_primops.py | 208 +- .../unittests/autograd/test_transform.py | 239 +- .../fluid/tests/unittests/autograd/utils.py | 84 +- .../paddle/fluid/tests/unittests/benchmark.py | 48 +- .../fluid/tests/unittests/benchmark_sum_op.py | 1 - .../tests/unittests/c_embedding_op_base.py | 13 +- .../tests/unittests/check_nan_inf_base.py | 23 +- .../unittests/check_nan_inf_base_dygraph.py | 13 +- .../collective/collective_allgather_api.py | 28 +- .../collective_allgather_api_dygraph.py | 1 - ...collective_allgather_object_api_dygraph.py | 1 - .../collective/collective_allreduce_api.py | 7 +- .../collective_allreduce_api_dygraph.py | 1 - .../collective_allreduce_new_group_api.py | 7 +- .../collective/collective_allreduce_op.py | 30 +- .../collective_allreduce_op_wait.py | 42 +- .../collective/collective_alltoall_api.py | 7 +- .../collective_alltoall_api_dygraph.py | 1 - .../collective/collective_alltoall_single.py | 43 +- .../collective_alltoall_single_api_dygraph.py | 1 - .../collective/collective_barrier_api.py | 1 - .../collective_batch_isend_irecv.py | 6 +- .../collective/collective_broadcast_api.py | 7 +- .../collective_broadcast_api_dygraph.py | 1 - .../collective/collective_broadcast_op.py | 33 +- .../collective/collective_concat_op.py | 24 +- .../collective/collective_global_gather.py | 59 +- .../collective_global_gather_dygraph.py | 19 +- .../collective/collective_global_scatter.py | 50 +- .../collective_global_scatter_dygraph.py | 19 +- .../collective/collective_identity_op.py | 23 +- .../collective_isend_irecv_api_dygraph.py | 1 - .../collective/collective_reduce_api.py | 7 +- .../collective_reduce_api_dygraph.py | 1 - .../collective/collective_reduce_op.py | 33 +- .../collective_reduce_op_calc_stream.py | 38 +- .../collective/collective_reduce_scatter.py | 7 +- .../collective_reduce_scatter_api_dygraph.py | 1 - .../collective/collective_scatter_api.py | 17 +- .../collective_scatter_api_dygraph.py | 13 +- .../collective/collective_scatter_op.py | 34 +- .../collective/collective_sendrecv_api.py | 11 +- .../collective_sendrecv_api_dygraph.py | 1 - .../collective/collective_sendrecv_op.py | 48 +- .../collective_sendrecv_op_array.py | 42 +- .../collective_sendrecv_op_dynamic_shape.py | 57 +- .../collective/collective_split_op.py | 24 +- .../collective/column_parallel_linear_api.py | 15 +- ...munication_stream_allgather_api_dygraph.py | 60 +- ...munication_stream_allreduce_api_dygraph.py | 19 +- ...mmunication_stream_alltoall_api_dygraph.py | 78 +- ...tion_stream_alltoall_single_api_dygraph.py | 25 +- ...munication_stream_broadcast_api_dygraph.py | 19 +- ...communication_stream_reduce_api_dygraph.py | 30 +- ...ation_stream_reduce_scatter_api_dygraph.py | 50 +- ...ommunication_stream_scatter_api_dygraph.py | 40 +- ...mmunication_stream_sendrecv_api_dygraph.py | 33 +- .../fleet/auto_parallel_parallelizer.py | 83 +- .../collective/fleet/c_comm_init_op.py | 46 +- .../fleet/dist_mnist_gradient_merge.py | 31 +- ...dist_mnist_gradient_merge_raw_optimizer.py | 20 +- .../fleet/dygraph_group_sharded_api.py | 137 +- .../fleet/dygraph_group_sharded_api_eager.py | 165 +- .../fleet/dygraph_group_sharded_stage2.py | 149 +- ...graph_group_sharded_stage2_comm_overlap.py | 149 +- .../dygraph_group_sharded_stage2_offload.py | 61 +- .../fleet/dygraph_group_sharded_stage3.py | 257 +- .../dygraph_group_sharded_stage3_offload.py | 147 +- .../dygraph_sharding_optimizer_stage2.py | 47 +- .../fleet/dygraph_sharding_stage2.py | 147 +- .../fleet/dygraph_sharding_stage2_offload.py | 55 +- .../fleet/dygraph_sharding_stage3.py | 255 +- .../fleet/dygraph_sharding_stage3_offload.py | 142 +- .../hybrid_parallel_communicate_group.py | 39 +- .../fleet/hybrid_parallel_inference_helper.py | 81 +- .../fleet/hybrid_parallel_mp_amp.py | 28 +- .../fleet/hybrid_parallel_mp_clip_grad.py | 17 +- .../fleet/hybrid_parallel_mp_fp16.py | 22 +- .../fleet/hybrid_parallel_mp_layers.py | 105 +- .../fleet/hybrid_parallel_mp_model.py | 129 +- .../fleet/hybrid_parallel_mp_random.py | 3 +- .../fleet/hybrid_parallel_pp_amp.py | 66 +- .../fleet/hybrid_parallel_pp_clip_grad.py | 33 +- .../fleet/hybrid_parallel_pp_embedding.py | 63 +- .../fleet/hybrid_parallel_pp_fp16.py | 86 +- ...id_parallel_pp_layer_with_virtual_stage.py | 27 +- .../fleet/hybrid_parallel_pp_recompute.py | 46 +- .../fleet/hybrid_parallel_pp_save_load.py | 34 +- ...arallel_pp_save_load_with_virtual_stage.py | 39 +- .../fleet/hybrid_parallel_pp_transformer.py | 39 +- ...allel_pp_transformer_with_virtual_stage.py | 32 +- .../collective/fleet/hybrid_parallel_qat.py | 169 +- .../fleet/hybrid_parallel_sharding_model.py | 220 +- .../fleet/hybrid_parallel_shared_weight.py | 79 +- .../unittests/collective/fleet/new_group.py | 23 +- .../fleet/parallel_class_center_sample.py | 54 +- ...parallel_dygraph_control_flow_different.py | 70 +- .../parallel_dygraph_control_flow_same.py | 32 +- .../fleet/parallel_dygraph_no_sync.py | 34 +- .../parallel_dygraph_no_sync_control_flow.py | 16 +- ...parallel_dygraph_no_sync_gradient_check.py | 34 +- .../parallel_dygraph_no_sync_unused_params.py | 16 +- .../fleet/parallel_dygraph_se_resnext.py | 271 +- .../fleet/parallel_dygraph_sync_batch_norm.py | 74 +- .../fleet/parallel_dygraph_transformer.py | 969 ++- .../fleet/parallel_margin_cross_entropy.py | 143 +- .../collective/fleet/pipeline_mnist.py | 87 +- .../fleet/pipeline_mnist_multi_device.py | 87 +- .../fleet/pipeline_mnist_one_device.py | 70 +- .../fleet/static_model_parallel_by_col.py | 54 +- .../fleet/static_model_parallel_by_row.py | 51 +- .../fleet/static_model_parallel_embedding.py | 31 +- .../collective/fleet/test_auto_checkpoint.py | 27 +- .../collective/fleet/test_auto_checkpoint1.py | 3 +- .../collective/fleet/test_auto_checkpoint2.py | 3 +- .../collective/fleet/test_auto_checkpoint3.py | 3 +- .../fleet/test_auto_checkpoint_dist_basic.py | 14 +- .../fleet/test_auto_checkpoint_multiple.py | 25 +- .../fleet/test_communicator_half_async.py | 24 +- .../fleet/test_communicator_sync.py | 6 +- .../collective/fleet/test_dgc_momentum_op.py | 58 +- .../unittests/collective/fleet/test_dgc_op.py | 28 +- .../collective/fleet/test_dgc_optimizer.py | 107 +- .../fleet/test_dist_mnist_dgc_nccl.py | 33 +- .../fleet/test_dist_mnist_gradient_merge.py | 45 +- .../fleet/test_dist_se_resnext_dgc.py | 12 +- .../fleet/test_distributed_strategy.py | 135 +- .../fleet/test_dygraph_recompute.py | 186 +- .../fleet/test_dygraph_recompute_for_eager.py | 179 +- .../test_dygraph_sharding_optimizer_stage2.py | 5 +- .../fleet/test_dygraph_sharding_stage2.py | 5 +- .../fleet/test_dygraph_sharding_stage3.py | 5 +- .../collective/fleet/test_fleet_amp_init.py | 51 +- .../fleet/test_fleet_amp_meta_optimizer.py | 32 +- .../collective/fleet/test_fleet_checkpoint.py | 78 +- .../fleet/test_fleet_dgc_meta_optimizer.py | 26 +- .../fleet/test_fleet_distributed_strategy.py | 90 +- ...est_fleet_fp16_allreduce_meta_optimizer.py | 27 +- ...est_fleet_gradient_merge_meta_optimizer.py | 22 +- ...st_fleet_graph_execution_meta_optimizer.py | 281 +- .../fleet/test_fleet_graph_executor.py | 35 +- .../fleet/test_fleet_hybrid_meta_optimizer.py | 1064 ++- .../fleet/test_fleet_lamb_meta_optimizer.py | 54 +- .../fleet/test_fleet_lars_meta_optimizer.py | 71 +- .../test_fleet_localsgd_meta_optimizer.py | 14 +- .../collective/fleet/test_fleet_log.py | 1 - .../fleet/test_fleet_meta_optimizer_base.py | 44 +- .../test_fleet_pipeline_meta_optimizer.py | 49 +- ..._pipeline_meta_optimizer_with_recompute.py | 32 +- .../fleet/test_fleet_private_function.py | 6 +- .../test_fleet_raw_program_meta_optimizer.py | 16 +- .../test_fleet_recompute_meta_optimizer.py | 25 +- .../fleet/test_fleet_rolemaker_new.py | 24 +- .../test_fleet_sharding_meta_optimizer.py | 3107 ++++++-- .../fleet/test_fleet_static_mp_layers.py | 66 +- .../collective/fleet/test_fleet_utils.py | 90 +- .../unittests/collective/fleet/test_hdfs1.py | 37 +- .../unittests/collective/fleet/test_hdfs2.py | 8 +- .../unittests/collective/fleet/test_hdfs3.py | 8 +- .../test_hybrid_parallel_inference_helper.py | 1 - .../test_imperative_auto_mixed_precision.py | 862 ++- ...perative_auto_mixed_precision_for_eager.py | 860 ++- .../collective/fleet/test_ir_pass_pipeline.py | 1 - .../collective/fleet/test_mixed_precision.py | 55 +- .../test_parallel_class_center_sample.py | 1 - .../test_parallel_dygraph_control_flow.py | 26 +- .../fleet/test_parallel_dygraph_mnist.py | 23 +- .../fleet/test_parallel_dygraph_mp_layers.py | 1 - .../fleet/test_parallel_dygraph_no_sync.py | 46 +- ...parallel_dygraph_no_sync_gradient_check.py | 6 +- ...test_parallel_dygraph_pipeline_parallel.py | 17 +- ...ph_pipeline_parallel_with_virtual_stage.py | 7 +- .../fleet/test_parallel_dygraph_qat.py | 51 +- .../fleet/test_parallel_dygraph_se_resnext.py | 17 +- ...test_parallel_dygraph_sharding_parallel.py | 5 +- .../test_parallel_dygraph_sparse_embedding.py | 24 +- ...el_dygraph_sparse_embedding_over_height.py | 22 +- .../test_parallel_dygraph_sync_batch_norm.py | 11 +- .../test_parallel_dygraph_tensor_parallel.py | 1 - .../test_parallel_dygraph_transformer.py | 22 +- .../test_parallel_dygraph_unused_variables.py | 17 +- .../test_parallel_margin_cross_entropy.py | 6 +- .../collective/fleet/test_pipeline.py | 28 +- .../collective/fleet/test_recv_save_op.py | 120 +- .../unittests/collective/fleet/test_rnn_dp.py | 130 +- .../fleet/test_static_model_parallel.py | 34 +- .../collective/fleet/test_tcp_store.py | 1 - .../collective/init_process_group.py | 1 - .../unittests/collective/multinode/common.py | 8 +- .../collective/multinode/dygraph_hybrid_dp.py | 18 +- .../multinode/dygraph_hybrid_dpppmp.py | 77 +- .../multinode/dygraph_hybrid_fp16.py | 92 +- .../multinode/dygraph_hybrid_recompute.py | 50 +- .../mn_dygraph_group_sharded_stage3.py | 260 +- .../multinode/mn_dygraph_sharding_stage2.py | 152 +- .../multinode/test_collective_multi_nodes.py | 40 +- .../test_multinode_dygraph_hybrid_dp.py | 7 +- .../test_multinode_dygraph_hybrid_dpppmp.py | 19 +- .../test_multinode_dygraph_sharding.py | 17 +- .../collective/orthogonal_strategy.py | 24 +- .../collective/parallel_embedding_api.py | 27 +- .../collective/process_group_gloo.py | 16 +- .../unittests/collective/process_group_mpi.py | 95 +- .../collective/process_group_nccl.py | 134 +- .../collective/row_parallel_linear_api.py | 15 +- .../unittests/collective/strategy_group.py | 16 +- .../unittests/collective/test_allreduce.py | 1 - .../unittests/collective/test_broadcast.py | 1 - .../unittests/collective/test_c_concat.py | 1 - .../unittests/collective/test_c_identity.py | 1 - .../unittests/collective/test_c_split.py | 1 - .../test_collective_allgather_api.py | 95 +- .../test_collective_allgather_object_api.py | 53 +- .../test_collective_allreduce_api.py | 63 +- .../test_collective_alltoall_api.py | 23 +- .../test_collective_alltoall_single.py | 1 - .../test_collective_alltoall_single_api.py | 23 +- .../collective/test_collective_barrier_api.py | 6 +- .../test_collective_batch_isend_irecv.py | 1 - .../test_collective_broadcast_api.py | 58 +- .../test_collective_cpu_barrier_with_gloo.py | 35 +- .../test_collective_global_gather.py | 30 +- .../test_collective_global_scatter.py | 30 +- .../test_collective_isend_irecv_api.py | 23 +- .../collective/test_collective_optimizer.py | 14 +- .../test_collective_process_group.py | 1 - .../collective/test_collective_reduce.py | 1 - .../collective/test_collective_reduce_api.py | 48 +- .../test_collective_reduce_scatter.py | 1 - .../test_collective_reduce_scatter_api.py | 23 +- .../collective/test_collective_scatter.py | 1 - .../collective/test_collective_scatter_api.py | 53 +- .../collective/test_collective_sendrecv.py | 11 +- .../test_collective_sendrecv_api.py | 25 +- .../test_collective_split_col_linear.py | 6 +- .../test_collective_split_embedding.py | 6 +- ...llective_split_embedding_none_divisible.py | 13 +- .../test_collective_split_row_linear.py | 6 +- .../collective/test_collective_wait.py | 7 +- .../collective/test_communication_api_base.py | 26 +- ...test_communication_stream_allgather_api.py | 21 +- ...test_communication_stream_allreduce_api.py | 21 +- .../test_communication_stream_alltoall_api.py | 21 +- ...ommunication_stream_alltoall_single_api.py | 21 +- ...test_communication_stream_broadcast_api.py | 21 +- .../test_communication_stream_reduce_api.py | 21 +- ...communication_stream_reduce_scatter_api.py | 21 +- .../test_communication_stream_scatter_api.py | 21 +- .../test_communication_stream_sendrecv_api.py | 21 +- .../collective/test_eager_dist_api.py | 1 - .../collective/test_gen_nccl_id_op.py | 18 +- .../collective/test_new_group_api.py | 6 +- .../collective/world_size_and_rank.py | 9 +- .../unittests/collective_allgather_op.py | 33 +- .../unittests/collective_reducescatter.py | 7 +- .../unittests/collective_reducescatter_op.py | 33 +- .../tests/unittests/ctr_dataset_reader.py | 31 +- .../fluid/tests/unittests/decorator_helper.py | 4 - .../fluid/tests/unittests/detected_gpu.py | 6 +- .../tests/unittests/dist_allreduce_op.py | 44 +- .../paddle/fluid/tests/unittests/dist_ctr.py | 94 +- .../fluid/tests/unittests/dist_ctr_reader.py | 5 +- .../fluid/tests/unittests/dist_fleet_ctr.py | 180 +- .../tests/unittests/dist_fleet_ctr_ps_gpu.py | 50 +- .../tests/unittests/dist_fleet_debug_gloo.py | 27 +- .../dist_fleet_heter_pipeline_ctr.py | 102 +- .../dist_fleet_raw_program_optimizer.py | 49 +- ...et_raw_program_optimizer_fuse_allreduce.py | 49 +- .../tests/unittests/dist_fleet_simnet_bow.py | 127 +- .../dist_fleet_sparse_embedding_ctr.py | 87 +- .../fluid/tests/unittests/dist_mnist.py | 55 +- .../tests/unittests/dist_mnist_batch_merge.py | 21 +- .../unittests/dist_mnist_fp16_allreduce.py | 35 +- .../fluid/tests/unittests/dist_mnist_lars.py | 31 +- .../fluid/tests/unittests/dist_save_load.py | 111 +- .../fluid/tests/unittests/dist_se_resnext.py | 217 +- .../tests/unittests/dist_sharding_save.py | 42 +- .../unittests/dist_text_classification.py | 92 +- .../fluid/tests/unittests/dist_transformer.py | 1318 ++-- .../fluid/tests/unittests/dist_word2vec.py | 62 +- .../distributed_fused_lamb_test_base.py | 102 +- .../auto_parallel_pass_test_base.py | 185 +- .../check_pass_conflict_example.py | 2 - .../distributed_passes/dist_pass_test_base.py | 121 +- .../unittests/distributed_passes/model_zoo.py | 43 +- .../distributed_passes/pass_run_main.py | 30 +- .../distributed_passes/ps_pass_test_base.py | 80 +- .../test_auto_parallel_amp_pass.py | 13 +- ...arallel_data_parallel_optimization_pass.py | 57 +- .../test_auto_parallel_fp16_pass.py | 32 +- .../test_auto_parallel_gradient_merge_pass.py | 152 +- .../test_auto_parallel_recompute_pass.py | 19 +- .../test_auto_parallel_sharding_pass.py | 13 +- .../test_dist_fuse_adam_pass.py | 20 +- .../test_dist_fuse_all_reduce_pass.py | 11 +- .../test_dist_fuse_bn_act_pass.py | 8 +- .../test_dist_fuse_bn_add_act_pass.py | 8 +- .../test_dist_fuse_momentum_pass.py | 19 +- ...test_dist_fuse_relu_depthwise_conv_pass.py | 8 +- .../test_dist_fuse_sgd_pass.py | 20 +- .../test_dist_inplace_addto_pass.py | 11 +- .../distributed_passes/test_ps_server_pass.py | 1 - .../test_ps_trainer_pass.py | 21 +- .../distributed_passes/test_white_lists.py | 17 +- .../tests/unittests/distribution/config.py | 2 +- .../tests/unittests/distribution/mock_data.py | 10 +- .../unittests/distribution/parameterize.py | 64 +- .../distribution/test_dirichlet_op.py | 8 +- .../distribution/test_distribution.py | 25 +- .../distribution/test_distribution_beta.py | 37 +- .../test_distribution_beta_static.py | 141 +- .../test_distribution_categorical.py | 156 +- .../test_distribution_constraint.py | 37 +- .../test_distribution_dirichlet.py | 46 +- .../test_distribution_dirichlet_static.py | 73 +- .../test_distribution_expfamily.py | 40 +- .../test_distribution_expfamily_static.py | 13 +- .../distribution/test_distribution_gumbel.py | 149 +- .../test_distribution_gumbel_static.py | 162 +- .../test_distribution_independent.py | 60 +- .../test_distribution_independent_static.py | 102 +- .../distribution/test_distribution_laplace.py | 199 +- .../test_distribution_laplace_static.py | 228 +- .../test_distribution_lognormal.py | 216 +- .../test_distribution_lognormal_static.py | 202 +- .../test_distribution_multinomial.py | 106 +- .../test_distribution_multinomial_static.py | 124 +- .../distribution/test_distribution_normal.py | 419 +- .../test_distribution_transform.py | 981 ++- .../test_distribution_transform_static.py | 960 ++- ...t_distribution_transformed_distribution.py | 20 +- ...ibution_transformed_distribution_static.py | 52 +- .../distribution/test_distribution_uniform.py | 193 +- .../test_distribution_variable.py | 39 +- .../tests/unittests/distribution/test_kl.py | 116 +- .../unittests/distribution/test_kl_static.py | 145 +- .../tests/unittests/dygraph_fleet_api.py | 7 +- .../unittests/dygraph_recompute_hybrid.py | 138 +- .../dygraph_to_static/bert_dygraph_model.py | 407 +- .../unittests/dygraph_to_static/bert_utils.py | 197 +- .../unittests/dygraph_to_static/darknet.py | 158 +- .../unittests/dygraph_to_static/decos.py | 3 - .../dygraph_to_static/ifelse_simple_func.py | 101 +- .../dygraph_to_static/predictor_utils.py | 5 +- .../seq2seq_dygraph_model.py | 528 +- .../dygraph_to_static/seq2seq_utils.py | 6 +- .../dygraph_to_static/simnet_dygraph_model.py | 140 +- .../simnet_dygraph_model_v2.py | 135 +- .../dygraph_to_static/test_assert.py | 37 +- .../dygraph_to_static/test_ast_util.py | 13 +- .../test_basic_api_transformation.py | 150 +- .../unittests/dygraph_to_static/test_bert.py | 167 +- .../unittests/dygraph_to_static/test_bmn.py | 533 +- .../dygraph_to_static/test_break_continue.py | 29 +- .../dygraph_to_static/test_cache_program.py | 42 +- .../unittests/dygraph_to_static/test_cast.py | 74 +- .../test_closure_analysis.py | 124 +- .../dygraph_to_static/test_container.py | 22 +- .../dygraph_to_static/test_convert_call.py | 69 +- .../test_convert_call_generator.py | 4 +- .../test_convert_operators.py | 126 +- .../test_cpu_cuda_to_tensor.py | 32 +- .../dygraph_to_static/test_cycle_gan.py | 490 +- .../dygraph_to_static/test_declarative.py | 158 +- .../test_decorator_transform.py | 17 +- .../dygraph_to_static/test_deepcopy.py | 5 +- .../unittests/dygraph_to_static/test_dict.py | 95 +- .../dygraph_to_static/test_drop_path.py | 2 - .../test_duplicate_output.py | 4 +- .../dygraph_to_static/test_fetch_feed.py | 26 +- .../dygraph_to_static/test_for_enumerate.py | 51 +- .../dygraph_to_static/test_full_name_usage.py | 27 +- .../dygraph_to_static/test_function_spec.py | 24 +- .../unittests/dygraph_to_static/test_grad.py | 41 +- .../test_gradient_aggregation.py | 24 +- .../dygraph_to_static/test_grid_generator.py | 59 +- .../dygraph_to_static/test_ifelse.py | 89 +- .../dygraph_to_static/test_isinstance.py | 18 +- .../test_jit_property_save.py | 6 +- .../unittests/dygraph_to_static/test_lac.py | 477 +- .../dygraph_to_static/test_lambda.py | 18 +- .../dygraph_to_static/test_layer_hook.py | 15 +- .../unittests/dygraph_to_static/test_len.py | 32 +- .../unittests/dygraph_to_static/test_list.py | 51 +- .../dygraph_to_static/test_logging_utils.py | 31 +- .../dygraph_to_static/test_logical.py | 33 +- .../unittests/dygraph_to_static/test_loop.py | 93 +- .../unittests/dygraph_to_static/test_lstm.py | 46 +- .../unittests/dygraph_to_static/test_mnist.py | 262 +- .../dygraph_to_static/test_mnist_amp.py | 33 +- .../dygraph_to_static/test_mnist_pure_fp16.py | 50 +- .../dygraph_to_static/test_mobile_net.py | 564 +- .../dygraph_to_static/test_multi_forward.py | 10 +- .../dygraph_to_static/test_op_attr.py | 22 +- .../dygraph_to_static/test_param_guard.py | 15 +- .../dygraph_to_static/test_params_no_grad.py | 8 +- .../dygraph_to_static/test_partial_program.py | 55 +- .../unittests/dygraph_to_static/test_place.py | 1 - .../unittests/dygraph_to_static/test_print.py | 15 +- .../test_program_translator.py | 152 +- .../dygraph_to_static/test_ptb_lm.py | 163 +- .../dygraph_to_static/test_ptb_lm_v2.py | 207 +- .../test_reinforcement_learning.py | 29 +- .../dygraph_to_static/test_resnet.py | 252 +- .../dygraph_to_static/test_resnet_amp.py | 33 +- .../test_resnet_pure_fp16.py | 42 +- .../dygraph_to_static/test_resnet_v2.py | 254 +- .../dygraph_to_static/test_return.py | 38 +- .../dygraph_to_static/test_rollback.py | 17 +- .../test_save_inference_model.py | 111 +- .../dygraph_to_static/test_save_load.py | 28 +- .../dygraph_to_static/test_se_resnet.py | 402 +- .../dygraph_to_static/test_sentiment.py | 229 +- .../dygraph_to_static/test_seq2seq.py | 140 +- .../dygraph_to_static/test_setter_helper.py | 6 +- .../dygraph_to_static/test_simnet.py | 57 +- .../dygraph_to_static/test_simnet_v2.py | 62 +- .../unittests/dygraph_to_static/test_slice.py | 95 +- .../dygraph_to_static/test_spec_names.py | 2 - .../dygraph_to_static/test_static_analysis.py | 33 +- .../dygraph_to_static/test_tensor_methods.py | 4 - .../dygraph_to_static/test_tensor_shape.py | 64 +- .../dygraph_to_static/test_to_tensor.py | 19 +- .../dygraph_to_static/test_transformer.py | 393 +- .../unittests/dygraph_to_static/test_tsm.py | 287 +- .../dygraph_to_static/test_typehint.py | 9 +- .../dygraph_to_static/test_typing.py | 9 - .../unittests/dygraph_to_static/test_utils.py | 5 +- .../test_variable_trans_func.py | 14 +- .../dygraph_to_static/test_warning.py | 11 +- .../dygraph_to_static/test_word2vec.py | 113 +- .../dygraph_to_static/test_yolov3.py | 40 +- .../transformer_dygraph_model.py | 831 ++- .../dygraph_to_static/transformer_util.py | 186 +- .../dygraph_to_static/tsm_config_utils.py | 22 +- .../unittests/dygraph_to_static/yolov3.py | 239 +- .../fluid/tests/unittests/elastic_demo.py | 16 +- .../fluid/tests/unittests/fake_reader.py | 31 +- .../fluid/tests/unittests/feed_data_reader.py | 9 +- .../tests/unittests/fft/spectral_op_np.py | 16 +- .../fluid/tests/unittests/fft/test_fft.py | 2006 +++-- .../fft/test_fft_with_static_graph.py | 2117 ++++-- .../tests/unittests/fft/test_spectral_op.py | 282 +- .../fluid/tests/unittests/find_ports.py | 3 +- .../unittests/fleet_heter_ps_training.py | 99 +- .../unittests/fleet_meta_optimizer_base.py | 110 +- .../tests/unittests/fleet_ps_training.py | 16 +- .../fluid/tests/unittests/gradient_checker.py | 362 +- .../fluid/tests/unittests/hccl_tools.py | 67 +- .../fluid/tests/unittests/hdfs_test_utils.py | 18 +- .../unittests/hybrid_parallel_pp_alexnet.py | 42 +- .../unittests/hybrid_parallel_pp_layer.py | 21 +- .../test_standalone_controlflow.py | 43 +- .../interpreter/test_standalone_executor.py | 146 +- .../test_standalone_multiply_write.py | 10 +- .../deprecated/test_custom_nllloss_ipu.py | 28 +- .../custom_ops/test_checkpointoutput_ipu.py | 19 +- .../custom_ops/test_custom_leaky_relu_ipu.py | 49 +- .../ipu/custom_ops/test_custom_ops_ipu.py | 93 +- .../test_dist_data_parallel_ipu.py | 82 +- .../distributed/test_dist_pod128_sample.py | 53 +- .../ipu/distributed/test_dist_sample.py | 98 +- .../fluid/tests/unittests/ipu/op_test_ipu.py | 88 +- .../unittests/ipu/test_activation_ops_ipu.py | 33 +- .../ipu/test_affine_channel_op_ipu.py | 21 +- .../unittests/ipu/test_arg_max_op_ipu.py | 8 +- .../unittests/ipu/test_arg_min_op_ipu.py | 8 +- .../unittests/ipu/test_argsort_op_ipu.py | 9 +- .../tests/unittests/ipu/test_assign_op_ipu.py | 21 +- .../tests/unittests/ipu/test_avg_shard_ipu.py | 35 +- .../unittests/ipu/test_batch_norm_op_ipu.py | 16 +- .../ipu/test_binary_cross_entropy_op_ipu.py | 19 +- .../tests/unittests/ipu/test_bmm_op_ipu.py | 13 +- .../tests/unittests/ipu/test_cast_op_ipu.py | 24 +- .../tests/unittests/ipu/test_clip_op_ipu.py | 125 +- .../tests/unittests/ipu/test_concat_op_ipu.py | 18 +- .../ipu/test_conv2d_transpose_op_ipu.py | 33 +- .../tests/unittests/ipu/test_conv_op_ipu.py | 32 +- .../ipu/test_cross_entropy2_op_ipu.py | 46 +- .../tests/unittests/ipu/test_cumsum_op_ipu.py | 24 +- .../unittests/ipu/test_data_norm_op_ipu.py | 30 +- .../tests/unittests/ipu/test_dist_op_ipu.py | 20 +- .../tests/unittests/ipu/test_dot_op_ipu.py | 14 +- .../unittests/ipu/test_dropout_op_ipu.py | 15 +- .../unittests/ipu/test_dy2static_fp16_ipu.py | 36 +- .../tests/unittests/ipu/test_dy2static_ipu.py | 133 +- .../unittests/ipu/test_elemetwise_x_op_ipu.py | 20 +- .../tests/unittests/ipu/test_equal_op_ipu.py | 15 +- .../unittests/ipu/test_eval_model_ipu.py | 55 +- .../unittests/ipu/test_expand_as_v2_op_ipu.py | 27 +- .../tests/unittests/ipu/test_expand_op_ipu.py | 23 +- .../unittests/ipu/test_expand_v2_op_ipu.py | 17 +- .../ipu/test_fill_any_like_op_ipu.py | 14 +- .../ipu/test_fill_constant_op_ipu.py | 2 - .../test_flatten_contiguous_range_op_ipu.py | 10 +- .../unittests/ipu/test_flatten_op_ipu.py | 9 +- .../tests/unittests/ipu/test_flip_op_ipu.py | 11 +- .../unittests/ipu/test_fp16_support_ipu.py | 41 +- .../tests/unittests/ipu/test_gather_op_ipu.py | 14 +- .../tests/unittests/ipu/test_gelu_op_ipu.py | 8 +- .../unittests/ipu/test_gradient_clip_ipu.py | 39 +- .../unittests/ipu/test_greater_op_ipu.py | 17 +- .../unittests/ipu/test_groupnorm_op_ipu.py | 31 +- .../unittests/ipu/test_huber_loss_op_ipu.py | 19 +- .../unittests/ipu/test_identity_loss_ipu.py | 30 +- .../ipu/test_inference_model_io_ipu.py | 59 +- .../unittests/ipu/test_instancenorm_op_ipu.py | 29 +- .../unittests/ipu/test_interpolate_ops_ipu.py | 25 +- .../unittests/ipu/test_ipu_shard_api_ipu.py | 16 +- .../unittests/ipu/test_ipu_strategy_ipu.py | 23 +- .../unittests/ipu/test_kldiv_loss_op_ipu.py | 19 +- .../unittests/ipu/test_layernorm_op_ipu.py | 42 +- .../unittests/ipu/test_logical_not_op_ipu.py | 7 +- .../unittests/ipu/test_logical_x_op_ipu.py | 18 +- .../unittests/ipu/test_lookuptable_op_ipu.py | 10 +- .../ipu/test_lookuptable_v2_op_ipu.py | 10 +- .../unittests/ipu/test_lr_sheduler_ipu.py | 26 +- .../ipu/test_margin_rank_loss_op_ipu.py | 20 +- .../tests/unittests/ipu/test_matmul_op_ipu.py | 26 +- .../unittests/ipu/test_matmul_serilize_ipu.py | 28 +- .../unittests/ipu/test_matmul_v2_op_ipu.py | 23 +- .../tests/unittests/ipu/test_mean_op_ipu.py | 7 +- .../unittests/ipu/test_meshgrid_op_ipu.py | 55 +- .../ipu/test_mixed_precision_inference_ipu.py | 43 +- .../ipu/test_mixed_precision_training_ipu.py | 43 +- .../unittests/ipu/test_model_parallel_ipu.py | 123 +- .../unittests/ipu/test_model_pipeline_ipu.py | 32 +- .../tests/unittests/ipu/test_mul_op_ipu.py | 15 +- .../unittests/ipu/test_not_equal_op_ipu.py | 24 +- .../unittests/ipu/test_one_hot_op_ipu.py | 8 +- .../unittests/ipu/test_one_hot_v2_op_ipu.py | 8 +- .../tests/unittests/ipu/test_optimizer_ipu.py | 70 +- .../tests/unittests/ipu/test_p_norm_op_ipu.py | 9 +- .../tests/unittests/ipu/test_pad_op_ipu.py | 22 +- .../unittests/ipu/test_pool_avg_op_ipu.py | 27 +- .../unittests/ipu/test_pool_max_op_ipu.py | 27 +- .../tests/unittests/ipu/test_pow_op_ipu.py | 24 +- .../tests/unittests/ipu/test_prelu_op_ipu.py | 14 +- .../tests/unittests/ipu/test_print_op_ipu.py | 38 +- .../unittests/ipu/test_rank_loss_op_ipu.py | 19 +- .../unittests/ipu/test_reduce_x_op_ipu.py | 26 +- .../ipu/test_reshape_inplace_op_ipu.py | 8 +- .../unittests/ipu/test_reshape_op_ipu.py | 9 +- .../tests/unittests/ipu/test_save_load_ipu.py | 78 +- .../tests/unittests/ipu/test_scale_op_ipu.py | 24 +- .../ipu/test_scaled_optimizer_state_ipu.py | 57 +- .../unittests/ipu/test_set_batch_size_ipu.py | 40 +- .../unittests/ipu/test_set_ipu_shard_api.py | 129 +- .../unittests/ipu/test_share_data_op_ipu.py | 38 +- ...igmoid_cross_entropy_with_logits_op_ipu.py | 22 +- .../tests/unittests/ipu/test_slice_op_ipu.py | 38 +- .../unittests/ipu/test_softmax_op_ipu.py | 8 +- .../test_softmax_with_cross_entropy_op_ipu.py | 63 +- .../tests/unittests/ipu/test_split_op_ipu.py | 8 +- .../unittests/ipu/test_squeeze_op_ipu.py | 9 +- .../tests/unittests/ipu/test_stack_op_ipu.py | 24 +- .../ipu/test_strided_slice_op_ipu.py | 45 +- .../tests/unittests/ipu/test_sum_op_ipu.py | 36 +- .../tests/unittests/ipu/test_tile_op_ipu.py | 21 +- .../tests/unittests/ipu/test_topk_op_ipu.py | 17 +- .../unittests/ipu/test_transpose_op_ipu.py | 10 +- .../tests/unittests/ipu/test_unary_ops_ipu.py | 29 +- .../unittests/ipu/test_unsqueeze_op_ipu.py | 9 +- .../unittests/ipu/test_unstack_op_ipu.py | 9 +- .../unittests/ipu/test_varname_inplace_ipu.py | 23 +- .../unittests/ipu/test_warpctc_op_ipu.py | 67 +- .../unittests/ipu/test_weight_decay_ipu.py | 35 +- .../unittests/ipu/test_weight_sharing_ipu.py | 41 +- .../tests/unittests/ipu/test_where_op_ipu.py | 17 +- .../unittests/ipu/test_yolo_box_op_ipu.py | 9 +- .../unittests/ir/inference/auto_scan_test.py | 485 +- .../ir/inference/inference_pass_test.py | 141 +- .../unittests/ir/inference/program_config.py | 248 +- .../ir/inference/quant_dequant_test.py | 202 +- ...ive_pool2d_convert_global_pass_autoscan.py | 80 +- .../test_conv_act_mkldnn_fuse_pass.py | 147 +- .../test_conv_bias_mkldnn_fuse_pass.py | 117 +- .../ir/inference/test_conv_bn_fuse_pass.py | 160 +- ...est_conv_elementwise_add2_act_fuse_pass.py | 266 +- ...test_conv_elementwise_add_act_fuse_pass.py | 135 +- .../test_conv_elementwise_add_fuse_pass.py | 92 +- .../test_conv_eltwiseadd_bn_fuse_pass.py | 208 +- .../test_conv_transpose_bn_fuse_pass.py | 161 +- ..._conv_transpose_eltwiseadd_bn_fuse_pass.py | 235 +- .../test_delete_c_identity_op_pass.py | 29 +- .../test_emb_eltwise_layernorm_fuse_pass.py | 335 +- ...test_fc_elementwise_layernorm_fuse_pass.py | 92 +- .../ir/inference/test_fc_fuse_pass.py | 55 +- .../ir/inference/test_fc_gru_fuse_pass.py | 46 +- .../ir/inference/test_fc_lstm_fuse_pass.py | 16 +- .../test_flatten2_matmul_fuse_pass.py | 39 +- .../test_identity_scale_clean_pass.py | 32 +- .../ir/inference/test_layer_norm_fuse_pass.py | 75 +- .../test_layernorm_shift_partition_pass.py | 574 +- .../inference/test_map_matmul_to_mul_pass.py | 41 +- .../test_map_matmul_v2_to_matmul_pass.py | 41 +- .../test_map_matmul_v2_to_mul_pass.py | 39 +- .../inference/test_matmul_scale_fuse_pass.py | 38 +- .../test_matmul_v2_scale_fuse_pass.py | 40 +- .../test_merge_layernorm_fuse_pass.py | 219 +- .../test_mkldnn_batch_norm_act_fuse_pass.py | 74 +- .../test_mkldnn_conv3d_bias_fuse_pass.py | 107 +- .../ir/inference/test_mkldnn_conv3d_op.py | 61 +- .../test_mkldnn_conv_activation_fuse_pass.py | 26 +- ...st_mkldnn_conv_affine_channel_fuse_pass.py | 119 +- .../test_mkldnn_conv_bias_fuse_pass.py | 164 +- ...kldnn_conv_concat_relu_mkldnn_fuse_pass.py | 172 +- ...t_mkldnn_conv_elementwise_add_fuse_pass.py | 111 +- .../test_mkldnn_conv_gelu_fuse_pass.py | 77 +- ...test_mkldnn_conv_hard_sigmoid_fuse_pass.py | 76 +- .../test_mkldnn_conv_hard_swish_fuse_pass.py | 81 +- .../test_mkldnn_conv_mish_fuse_pass.py | 73 +- ...st_mkldnn_conv_transpose_bias_fuse_pass.py | 80 +- .../test_mkldnn_cpu_bfloat16_pass.py | 12 +- .../test_mkldnn_depthwise_conv_pass.py | 76 +- .../test_mkldnn_elt_act_fuse_pass.py | 154 +- .../test_mkldnn_elt_act_fuse_pass_new.py | 47 +- .../inference/test_mkldnn_fc_act_fuse_pass.py | 42 +- ...est_mkldnn_fc_elementwise_add_fuse_pass.py | 59 +- .../test_mkldnn_inplace_fuse_pass.py | 20 +- ...test_mkldnn_int8_scale_calculation_pass.py | 103 +- .../inference/test_mkldnn_log_softmax_op.py | 30 +- ...test_mkldnn_matmul_activation_fuse_pass.py | 105 +- ...ul_elementwise_add_activation_fuse_pass.py | 135 +- ...mkldnn_matmul_elementwise_add_fuse_pass.py | 54 +- .../test_mkldnn_matmul_op_output_fuse_pass.py | 53 +- ...ldnn_matmul_transpose_reshape_fuse_pass.py | 83 +- ...t_mkldnn_matmul_v2_activation_fuse_pass.py | 100 +- ...dnn_matmul_v2_elementwise_add_fuse_pass.py | 49 +- ...n_matmul_v2_transpose_reshape_fuse_pass.py | 105 +- .../ir/inference/test_mkldnn_matmulv2_op.py | 91 +- .../ir/inference/test_mkldnn_mish_op.py | 44 +- .../ir/inference/test_mkldnn_pad2d_op.py | 42 +- .../ir/inference/test_mkldnn_pad3d_op.py | 55 +- .../ir/inference/test_mkldnn_prelu_op.py | 58 +- ...ldnn_reshape_transpose_matmul_fuse_pass.py | 133 +- ...n_reshape_transpose_matmul_v2_fuse_pass.py | 26 +- .../test_mkldnn_scale_matmul_fuse_pass.py | 146 +- .../ir/inference/test_mkldnn_shape_op.py | 31 +- ...test_mkldnn_shuffle_channel_detect_pass.py | 99 +- .../test_mkldnn_shuffle_channel_op.py | 26 +- ...st_mkldnn_softplus_activation_fuse_pass.py | 61 +- .../ir/inference/test_mul_gru_fuse_pass.py | 141 +- .../ir/inference/test_mul_lstm_fuse_pass.py | 110 +- .../test_multihead_matmul_fuse_pass_v3.py | 330 +- .../test_onednn_fc_activation_fuse_pass.py | 127 +- .../test_repeated_fc_relu_fuse_pass.py | 128 +- .../test_reshape2_matmul_fuse_pass.py | 39 +- .../inference/test_seq_concat_fc_fuse_pass.py | 131 +- .../test_seqconv_eltadd_relu_fuse_pass.py | 104 +- .../test_seqpool_cvm_concat_fuse_pass_py.py | 167 +- .../test_shuffle_channel_detect_pass.py | 74 +- ...t_simplify_with_basic_ops_pass_autoscan.py | 136 +- .../test_squared_mat_sub_fuse_pass.py | 200 +- .../test_squeeze2_matmul_fuse_pass.py | 39 +- ...test_transpose_flatten_concat_fuse_pass.py | 26 +- .../ir/inference/test_trt_activation_pass.py | 220 +- .../inference/test_trt_affine_channel_op.py | 39 +- .../inference/test_trt_anchor_generator_op.py | 53 +- .../test_trt_c_allreduce_infer_script.py | 53 +- .../ir/inference/test_trt_conv3d_op.py | 107 +- .../inference/test_trt_conv3d_transpose_op.py | 58 +- .../ir/inference/test_trt_conv_pass.py | 150 +- .../test_trt_conv_quant_dequant_pass.py | 172 +- .../inference/test_trt_convert_activation.py | 61 +- .../test_trt_convert_affine_channel.py | 65 +- .../test_trt_convert_anchor_generator.py | 78 +- .../ir/inference/test_trt_convert_arg_max.py | 55 +- .../inference/test_trt_convert_batch_norm.py | 195 +- .../test_trt_convert_bilinear_interp_v2.py | 86 +- .../ir/inference/test_trt_convert_bmm.py | 63 +- .../inference/test_trt_convert_c_allreduce.py | 10 +- .../ir/inference/test_trt_convert_cast.py | 50 +- .../ir/inference/test_trt_convert_clip.py | 76 +- .../ir/inference/test_trt_convert_concat.py | 177 +- .../ir/inference/test_trt_convert_conv2d.py | 127 +- .../test_trt_convert_conv2d_fusion.py | 133 +- .../test_trt_convert_conv2d_transpose.py | 218 +- .../test_trt_convert_conv3d_transpose.py | 77 +- .../test_trt_convert_deformable_conv.py | 184 +- .../test_trt_convert_depthwise_conv2d.py | 126 +- ..._trt_convert_depthwise_conv2d_transpose.py | 114 +- .../ir/inference/test_trt_convert_dropout.py | 76 +- .../inference/test_trt_convert_elementwise.py | 438 +- .../test_trt_convert_emb_eltwise_layernorm.py | 303 +- .../ir/inference/test_trt_convert_equal.py | 84 +- .../inference/test_trt_convert_expand_v2.py | 115 +- .../ir/inference/test_trt_convert_fc.py | 135 +- .../test_trt_convert_fill_constant.py | 84 +- .../ir/inference/test_trt_convert_flatten.py | 180 +- ...st_trt_convert_flatten_contiguous_range.py | 49 +- .../test_trt_convert_fused_token_prune.py | 80 +- .../ir/inference/test_trt_convert_gather.py | 127 +- .../inference/test_trt_convert_gather_nd.py | 256 +- .../ir/inference/test_trt_convert_gelu.py | 45 +- .../test_trt_convert_grid_sampler.py | 47 +- .../inference/test_trt_convert_group_norm.py | 81 +- .../test_trt_convert_hard_sigmoid.py | 35 +- .../inference/test_trt_convert_hard_swish.py | 59 +- .../test_trt_convert_instance_norm.py | 89 +- .../ir/inference/test_trt_convert_inverse.py | 34 +- .../inference/test_trt_convert_layer_norm.py | 168 +- .../inference/test_trt_convert_leaky_relu.py | 55 +- .../ir/inference/test_trt_convert_matmul.py | 144 +- .../inference/test_trt_convert_matmul_v2.py | 217 +- .../ir/inference/test_trt_convert_mish.py | 57 +- .../test_trt_convert_multiclass_nms.py | 138 +- .../test_trt_convert_multiclass_nms3.py | 147 +- .../test_trt_convert_multihead_matmul.py | 1070 ++- .../test_trt_convert_nearest_interp.py | 88 +- .../test_trt_convert_nearest_interp_v2.py | 55 +- .../ir/inference/test_trt_convert_pad.py | 59 +- .../ir/inference/test_trt_convert_pad3d.py | 40 +- .../ir/inference/test_trt_convert_pool2d.py | 120 +- .../test_trt_convert_preln_residual_bias.py | 134 +- ...test_trt_convert_preln_residual_no_bias.py | 109 +- .../ir/inference/test_trt_convert_prelu.py | 112 +- .../inference/test_trt_convert_reduce_mean.py | 77 +- .../inference/test_trt_convert_reduce_sum.py | 74 +- .../ir/inference/test_trt_convert_reshape.py | 140 +- .../ir/inference/test_trt_convert_rnn.py | 213 +- .../inference/test_trt_convert_roi_align.py | 145 +- .../ir/inference/test_trt_convert_roll.py | 56 +- .../ir/inference/test_trt_convert_scale.py | 106 +- .../test_trt_convert_scatter_nd_add.py | 48 +- .../ir/inference/test_trt_convert_shape.py | 40 +- .../test_trt_convert_shuffle_channel.py | 51 +- .../ir/inference/test_trt_convert_silu.py | 47 +- .../test_trt_convert_skip_layernorm.py | 110 +- .../ir/inference/test_trt_convert_slice.py | 79 +- .../ir/inference/test_trt_convert_softmax.py | 47 +- .../ir/inference/test_trt_convert_split.py | 162 +- .../ir/inference/test_trt_convert_squeeze2.py | 57 +- .../ir/inference/test_trt_convert_stack.py | 90 +- .../test_trt_convert_strided_slice.py | 129 +- .../ir/inference/test_trt_convert_sum.py | 114 +- .../ir/inference/test_trt_convert_swish.py | 47 +- .../ir/inference/test_trt_convert_tile.py | 44 +- .../ir/inference/test_trt_convert_top_k.py | 50 +- .../ir/inference/test_trt_convert_top_k_v2.py | 67 +- .../inference/test_trt_convert_transpose.py | 60 +- .../ir/inference/test_trt_convert_unary.py | 65 +- .../ir/inference/test_trt_convert_unfold.py | 42 +- .../inference/test_trt_convert_unsqueeze2.py | 51 +- .../ir/inference/test_trt_convert_yolo_box.py | 126 +- .../test_trt_convert_yolo_box_head.py | 48 +- .../ir/inference/test_trt_deformable_conv.py | 44 +- .../ir/inference/test_trt_dynamic_shape.py | 35 +- .../ir/inference/test_trt_elementwise_op.py | 35 +- .../ir/inference/test_trt_fc_fuse_pass.py | 197 +- .../test_trt_fc_fuse_quant_dequant_pass.py | 213 +- .../test_trt_flatten2_matmul_fuse_pass.py | 40 +- .../ir/inference/test_trt_flatten_op.py | 41 +- .../ir/inference/test_trt_gather_nd_op.py | 55 +- .../ir/inference/test_trt_gather_op.py | 48 +- .../ir/inference/test_trt_inspector.py | 29 +- .../ir/inference/test_trt_instance_norm_op.py | 15 +- .../unittests/ir/inference/test_trt_matmul.py | 74 +- .../test_trt_matmul_quant_dequant.py | 210 +- .../inference/test_trt_multiclass_nms3_op.py | 154 +- .../inference/test_trt_multiclass_nms_op.py | 74 +- .../inference/test_trt_nearest_interp_op.py | 66 +- .../test_trt_nearest_interp_v2_op.py | 66 +- .../unittests/ir/inference/test_trt_pad_op.py | 16 +- .../ir/inference/test_trt_pool3d_op.py | 233 +- .../ir/inference/test_trt_pool_op.py | 78 +- .../ir/inference/test_trt_reduce_mean_op.py | 168 +- .../ir/inference/test_trt_reduce_sum_op.py | 46 +- .../ir/inference/test_trt_reshape_op.py | 67 +- .../ir/inference/test_trt_roi_align_op.py | 42 +- .../ir/inference/test_trt_scale_op.py | 34 +- .../test_trt_shuffle_channel_detect_pass.py | 13 +- .../test_trt_slice_dynamic_plugin.py | 54 +- .../ir/inference/test_trt_slice_plugin.py | 52 +- .../test_trt_squeeze2_matmul_fuse_pass.py | 40 +- .../ir/inference/test_trt_subgraph_pass.py | 287 +- .../ir/inference/test_trt_tile_op.py | 34 +- ..._trt_transpose_flatten_concat_fuse_pass.py | 22 +- .../inference/test_trt_tuned_dynamic_shape.py | 37 +- .../ir/inference/test_trt_yolo_box_op.py | 107 +- .../test_unsqueeze2_eltwise_fuse_pass.py | 30 +- .../ir/inference/test_yolo_box_post.py | 80 +- .../fluid/tests/unittests/ir/pass_test.py | 60 +- .../ir/test_convert_to_mixed_precision.py | 78 +- .../unittests/ir/test_fuse_resnet_unit.py | 43 +- ...r_embedding_eltwise_layernorm_fuse_pass.py | 159 +- .../unittests/ir/test_ir_fc_fuse_pass.py | 15 +- .../unittests/ir/test_ir_fusion_group_pass.py | 33 +- .../unittests/ir/test_ir_generate_pass.py | 77 +- .../ir/test_ir_graph_to_program_pass.py | 17 +- .../test_ir_preln_residual_bias_fuse_pass.py | 40 +- .../ir/test_ir_skip_layernorm_pass.py | 17 +- .../ir/test_ir_subgraph_python_interface.py | 19 +- .../unittests/ir/test_ir_yolo_box_pass.py | 61 +- .../unittests/ir_memory_optimize_net_base.py | 73 +- .../tests/unittests/jit_load_rename_var.py | 5 +- .../tests/unittests/launch_function_helper.py | 1 - .../mkldnn/check_flags_mkldnn_ops_on_off.py | 24 +- .../mkldnn/check_flags_use_mkldnn.py | 14 +- .../tests/unittests/mkldnn/mkldnn_op_test.py | 92 +- .../mkldnn/test_activation_bf16_mkldnn_op.py | 63 +- .../mkldnn/test_activation_mkldnn_op.py | 67 +- .../mkldnn/test_batch_norm_mkldnn_op.py | 83 +- .../mkldnn/test_bilinear_interp_mkldnn_op.py | 56 +- .../test_bilinear_interp_v2_mkldnn_op.py | 41 +- .../unittests/mkldnn/test_cast_mkldnn_op.py | 24 +- .../unittests/mkldnn/test_clip_mkldnn_op.py | 38 +- .../mkldnn/test_concat_bf16_mkldnn_op.py | 33 +- .../mkldnn/test_concat_int8_mkldnn_op.py | 23 +- .../unittests/mkldnn/test_concat_mkldnn_op.py | 11 +- .../mkldnn/test_conv2d_bf16_mkldnn_op.py | 103 +- .../mkldnn/test_conv2d_int8_mkldnn_op.py | 136 +- .../unittests/mkldnn/test_conv2d_mkldnn_op.py | 48 +- .../test_conv2d_transpose_bf16_mkldnn_op.py | 35 +- .../mkldnn/test_conv2d_transpose_mkldnn_op.py | 18 +- .../unittests/mkldnn/test_conv3d_mkldnn_op.py | 19 +- .../mkldnn/test_dequantize_mkldnn_op.py | 23 +- .../test_elementwise_add_bf16_mkldnn_op.py | 74 +- .../mkldnn/test_elementwise_add_mkldnn_op.py | 35 +- .../mkldnn/test_elementwise_div_mkldnn_op.py | 65 +- .../test_elementwise_mul_bf16_mkldnn_op.py | 55 +- .../mkldnn/test_elementwise_mul_mkldnn_op.py | 32 +- .../mkldnn/test_elementwise_sub_mkldnn_op.py | 85 +- .../mkldnn/test_expand_v2_mkldnn_op.py | 35 +- .../mkldnn/test_fc_bf16_mkldnn_op.py | 17 +- .../mkldnn/test_fc_int8_mkldnn_op.py | 8 +- .../unittests/mkldnn/test_fc_mkldnn_op.py | 12 +- .../mkldnn/test_fill_constant_mkldnn_op.py | 32 +- .../mkldnn/test_flags_mkldnn_ops_on_off.py | 21 +- .../unittests/mkldnn/test_flags_use_mkldnn.py | 11 +- .../mkldnn/test_flatten_mkldnn_op.py | 37 +- .../mkldnn/test_fusion_gru_bf16_mkldnn_op.py | 51 +- .../mkldnn/test_fusion_gru_int8_mkldnn_op.py | 88 +- .../mkldnn/test_fusion_gru_mkldnn_op.py | 10 +- .../mkldnn/test_fusion_lstm_bf16_mkldnn_op.py | 53 +- .../mkldnn/test_fusion_lstm_int8_mkldnn_op.py | 58 +- .../mkldnn/test_fusion_lstm_mkldnn_op.py | 9 +- .../mkldnn/test_gaussian_random_mkldnn_op.py | 8 +- .../mkldnn/test_layer_norm_bf16_mkldnn_op.py | 75 +- .../mkldnn/test_layer_norm_mkldnn_op.py | 72 +- .../mkldnn/test_log_softmax_mkldnn_op.py | 19 +- .../unittests/mkldnn/test_lrn_mkldnn_op.py | 28 +- .../mkldnn/test_matmul_bf16_mkldnn_op.py | 55 +- .../unittests/mkldnn/test_matmul_mkldnn_op.py | 175 +- .../mkldnn/test_matmul_v2_mkldnn_op.py | 215 +- .../mkldnn/test_mul_int8_mkldnn_op.py | 29 +- .../unittests/mkldnn/test_mul_mkldnn_op.py | 35 +- .../mkldnn/test_multi_gru_mkldnn_op.py | 161 +- .../mkldnn/test_nearest_interp_mkldnn_op.py | 49 +- .../test_nearest_interp_v2_mkldnn_op.py | 48 +- ...st_onnx_format_quantization_mobilenetv1.py | 238 +- .../mkldnn/test_pool2d_bf16_mkldnn_op.py | 92 +- .../mkldnn/test_pool2d_int8_mkldnn_op.py | 44 +- .../unittests/mkldnn/test_pool2d_mkldnn_op.py | 32 +- .../unittests/mkldnn/test_prelu_mkldnn_op.py | 50 +- .../mkldnn/test_quantize_mkldnn_op.py | 32 +- .../mkldnn/test_reduce_bf16_mkldnn_op.py | 52 +- .../unittests/mkldnn/test_reduce_mkldnn_op.py | 49 +- .../mkldnn/test_requantize_mkldnn_op.py | 146 +- .../unittests/mkldnn/test_reshape_bf16_op.py | 31 +- .../mkldnn/test_reshape_mkldnn_op.py | 73 +- .../mkldnn/test_scale_bf16_mkldnn_op.py | 31 +- .../unittests/mkldnn/test_scale_mkldnn_op.py | 14 +- .../unittests/mkldnn/test_shape_mkldnn_op.py | 4 - .../mkldnn/test_shuffle_channel_mkldnn_op.py | 8 +- .../unittests/mkldnn/test_slice_mkldnn_op.py | 45 +- .../mkldnn/test_softmax_bf16_mkldnn_op.py | 25 +- .../mkldnn/test_softmax_mkldnn_op.py | 45 +- .../mkldnn/test_softplus_mkldnn_op.py | 24 +- .../mkldnn/test_split_bf16_mkldnn_op.py | 36 +- .../unittests/mkldnn/test_split_mkldnn_op.py | 23 +- .../mkldnn/test_squeeze2_mkldnn_op.py | 36 +- .../unittests/mkldnn/test_stack_mkldnn_op.py | 15 +- .../mkldnn/test_sum_bf16_mkldnn_op.py | 6 +- .../unittests/mkldnn/test_sum_mkldnn_op.py | 12 +- .../mkldnn/test_transpose_bf16_mkldnn_op.py | 14 +- .../mkldnn/test_transpose_int8_mkldnn_op.py | 25 +- .../mkldnn/test_transpose_mkldnn_op.py | 13 +- .../tests/unittests/mlu/c_comm_init_op_mlu.py | 46 +- .../unittests/mlu/collective_allgather_api.py | 12 +- .../unittests/mlu/collective_allgather_op.py | 33 +- .../unittests/mlu/collective_allreduce_api.py | 12 +- .../unittests/mlu/collective_allreduce_op.py | 30 +- .../unittests/mlu/collective_broadcast_api.py | 12 +- .../unittests/mlu/collective_broadcast_op.py | 33 +- .../unittests/mlu/collective_reduce_api.py | 12 +- .../unittests/mlu/collective_reduce_op.py | 33 +- .../tests/unittests/mlu/multi_process_mlu.py | 41 +- .../tests/unittests/mlu/nproc_process_mlu.py | 9 +- .../mlu/parallel_dygraph_sync_batch_norm.py | 74 +- .../unittests/mlu/sync_batch_norm_op_mlu.py | 40 +- .../tests/unittests/mlu/test_abs_op_mlu.py | 12 +- .../unittests/mlu/test_accuracy_op_mlu.py | 58 +- .../tests/unittests/mlu/test_adam_op_mlu.py | 47 +- .../tests/unittests/mlu/test_adamw_op_mlu.py | 40 +- .../test_amp_check_finite_and_scale_op_mlu.py | 6 - .../unittests/mlu/test_arg_max_op_mlu.py | 76 +- .../unittests/mlu/test_argsort_op_mlu.py | 8 +- .../tests/unittests/mlu/test_assign_op_mlu.py | 1 - .../unittests/mlu/test_assign_value_op_mlu.py | 12 +- .../unittests/mlu/test_batch_norm_op_mlu.py | 346 +- .../mlu/test_batch_norm_op_mlu_v2.py | 56 +- .../tests/unittests/mlu/test_bce_loss_mlu.py | 246 +- .../mlu/test_bce_with_logits_loss_mlu.py | 254 +- .../mlu/test_bilinear_interp_v2_op_mlu.py | 252 +- .../tests/unittests/mlu/test_cast_op_mlu.py | 24 +- .../tests/unittests/mlu/test_clip_op_mlu.py | 7 - .../mlu/test_coalesce_tensor_op_mlu.py | 32 +- .../mlu/test_collective_allgather.py | 26 +- .../mlu/test_collective_allgather_api_mlu.py | 16 +- .../mlu/test_collective_allreduce_api_mlu.py | 16 +- .../mlu/test_collective_allreduce_max.py | 31 +- .../mlu/test_collective_allreduce_min.py | 31 +- .../mlu/test_collective_allreduce_prod.py | 31 +- .../mlu/test_collective_allreduce_sum.py | 31 +- .../mlu/test_collective_api_base_mlu.py | 102 +- .../unittests/mlu/test_collective_base_mlu.py | 206 +- .../mlu/test_collective_broadcast.py | 26 +- .../mlu/test_collective_broadcast_api_mlu.py | 16 +- .../mlu/test_collective_reduce_api_mlu.py | 1 - .../mlu/test_collective_reduce_max.py | 11 +- .../mlu/test_collective_reduce_min.py | 11 +- .../mlu/test_collective_reduce_prod.py | 11 +- .../mlu/test_collective_reduce_sum.py | 11 +- .../unittests/mlu/test_compare_op_mlu.py | 50 +- .../tests/unittests/mlu/test_concat_op_mlu.py | 44 +- .../mlu/test_conv2d_op_depthwise_conv_mlu.py | 23 +- .../tests/unittests/mlu/test_conv2d_op_mlu.py | 132 +- .../mlu/test_conv2d_transposed_op_mlu.py | 319 +- .../tests/unittests/mlu/test_cumsum_op_mlu.py | 86 +- .../mlu/test_deformable_conv_op_mlu.py | 117 +- .../unittests/mlu/test_dropout_op_mlu.py | 114 +- .../mlu/test_elementwise_add_op_mlu.py | 116 +- .../mlu/test_elementwise_div_op_mlu.py | 84 +- .../mlu/test_elementwise_max_op_mlu.py | 174 +- .../mlu/test_elementwise_min_op_mlu.py | 89 +- .../mlu/test_elementwise_mul_op_mlu.py | 63 +- .../mlu/test_elementwise_pow_op_mlu.py | 125 +- .../mlu/test_elementwise_sub_op_mlu.py | 60 +- .../tests/unittests/mlu/test_exp_op_mlu.py | 3 - .../unittests/mlu/test_expand_as_v2_op_mlu.py | 45 +- .../unittests/mlu/test_expand_v2_op_mlu.py | 65 +- .../mlu/test_fill_any_like_op_mlu.py | 8 - ...st_fill_constant_batch_size_like_op_mlu.py | 46 +- .../mlu/test_fill_constant_op_mlu.py | 284 +- .../unittests/mlu/test_flatten2_op_mlu.py | 20 +- .../test_flatten_contigous_range_op_mlu.py | 109 +- .../unittests/mlu/test_flatten_op_mlu.py | 4 - .../tests/unittests/mlu/test_floor_op_mlu.py | 2 - .../unittests/mlu/test_gather_nd_op_mlu.py | 43 +- .../tests/unittests/mlu/test_gather_op_mlu.py | 28 +- .../mlu/test_gaussian_random_op_mlu.py | 4 +- .../tests/unittests/mlu/test_gelu_op_mlu.py | 34 +- .../unittests/mlu/test_grid_sampler_op_mlu.py | 80 +- .../unittests/mlu/test_hard_sigmoid_op_mlu.py | 32 +- .../unittests/mlu/test_hard_swish_op_mlu.py | 74 +- .../unittests/mlu/test_huber_loss_op_mlu.py | 48 +- .../mlu/test_iou_similarity_op_mlu.py | 10 +- .../unittests/mlu/test_label_smooth_op_mlu.py | 23 +- .../unittests/mlu/test_layer_norm_op_mlu.py | 279 +- .../unittests/mlu/test_leaky_relu_op_mlu.py | 35 +- .../tests/unittests/mlu/test_log_op_mlu.py | 59 +- .../unittests/mlu/test_log_softmax_op_mlu.py | 34 +- .../unittests/mlu/test_logical_op_mlu.py | 140 +- .../mlu/test_lookup_table_v2_op_mlu.py | 20 +- .../mlu/test_masked_select_op_mlu.py | 37 +- .../tests/unittests/mlu/test_matmul_op_mlu.py | 38 +- .../unittests/mlu/test_matmul_v2_op_mlu.py | 45 +- .../tests/unittests/mlu/test_mean_op_mlu.py | 2 - .../unittests/mlu/test_merged_adam_op_mlu.py | 167 +- .../mlu/test_merged_momentum_op_mlu.py | 295 +- .../unittests/mlu/test_meshgrid_op_mlu.py | 132 +- .../unittests/mlu/test_momentum_op_mlu.py | 252 +- .../mlu/test_nearest_interp_v2_op_mlu.py | 253 +- .../unittests/mlu/test_one_hot_v2_op_mlu.py | 88 +- ...st_parallel_dygraph_sync_batch_norm_mlu.py | 88 +- .../tests/unittests/mlu/test_pool2d_op_mlu.py | 694 +- .../unittests/mlu/test_prior_box_op_mlu.py | 83 +- .../unittests/mlu/test_randperm_op_mlu.py | 170 +- .../tests/unittests/mlu/test_range_op_mlu.py | 13 +- .../unittests/mlu/test_reciprocal_op_mlu.py | 8 +- .../unittests/mlu/test_reduce_max_op_mlu.py | 41 +- .../unittests/mlu/test_reduce_mean_op_mlu.py | 26 +- .../unittests/mlu/test_reduce_min_op_mlu.py | 41 +- .../unittests/mlu/test_reduce_prod_op_mlu.py | 8 +- .../unittests/mlu/test_reduce_sum_op_mlu.py | 50 +- .../tests/unittests/mlu/test_relu6_op_mlu.py | 29 +- .../tests/unittests/mlu/test_relu_op_mlu.py | 29 +- .../unittests/mlu/test_reshape2_op_mlu.py | 5 +- .../tests/unittests/mlu/test_rnn_op_mlu.py | 69 +- .../unittests/mlu/test_roi_align_op_mlu.py | 124 +- .../tests/unittests/mlu/test_scale_op_mlu.py | 28 +- .../unittests/mlu/test_scatter_op_mlu.py | 77 +- .../unittests/mlu/test_set_value_op_mlu.py | 85 +- .../tests/unittests/mlu/test_shape_op_mlu.py | 9 +- ...igmoid_cross_entropy_with_logits_op_mlu.py | 85 +- .../unittests/mlu/test_sigmoid_op_mlu.py | 8 +- .../tests/unittests/mlu/test_size_op_mlu.py | 34 +- .../tests/unittests/mlu/test_slice_op_mlu.py | 160 +- .../unittests/mlu/test_softmax_op_mlu.py | 30 +- .../test_softmax_with_cross_entropy_op_mlu.py | 51 +- .../tests/unittests/mlu/test_spawn_mlu.py | 8 +- .../tests/unittests/mlu/test_split_op_mlu.py | 32 +- .../tests/unittests/mlu/test_sqrt_op_mlu.py | 9 +- .../mlu/test_squared_l2_norm_op_mlu.py | 10 +- .../unittests/mlu/test_squeeze2_op_mlu.py | 6 +- .../unittests/mlu/test_squeeze_op_mlu.py | 6 - .../tests/unittests/mlu/test_stack_op_mlu.py | 25 +- .../mlu/test_strided_slice_op_mlu.py | 233 +- .../tests/unittests/mlu/test_sum_op_mlu.py | 11 +- .../mlu/test_sync_batch_norm_base_mlu.py | 256 +- .../test_sync_batch_norm_op_mlu_baseline.py | 7 +- .../mlu/test_sync_batch_norm_op_mlu_extra.py | 55 +- .../tests/unittests/mlu/test_tanh_op_mlu.py | 28 +- .../tests/unittests/mlu/test_tile_op_mlu.py | 30 +- .../tests/unittests/mlu/test_top_k_op_mlu.py | 2 - .../unittests/mlu/test_top_k_v2_op_mlu.py | 115 +- .../unittests/mlu/test_transpose_op_mlu.py | 43 +- .../unittests/mlu/test_tril_triu_op_mlu.py | 64 +- .../test_truncated_gaussian_random_op_mlu.py | 24 +- .../mlu/test_uniform_random_op_mlu.py | 23 +- .../unittests/mlu/test_unsqueeze2_op_mlu.py | 36 +- .../unittests/mlu/test_unsqueeze_op_mlu.py | 7 +- .../unittests/mlu/test_unstack_op_mlu.py | 5 - .../unittests/mlu/test_where_index_op_mlu.py | 25 +- .../tests/unittests/mlu/test_where_op_mlu.py | 147 +- .../unittests/mlu/test_yolo_box_op_mlu.py | 195 +- .../fluid/tests/unittests/multi_process.py | 41 +- .../tests/unittests/my_data_generator.py | 10 +- .../fluid/tests/unittests/nproc_process.py | 9 +- .../npu/collective_identity_op_npu.py | 23 +- .../tests/unittests/npu/process_group_hccl.py | 32 +- .../unittests/npu/sync_batch_norm_op_npu.py | 49 +- .../tests/unittests/npu/test_abs_op_npu.py | 1 - .../unittests/npu/test_accuracy_op_npu.py | 12 +- .../tests/unittests/npu/test_adam_op_npu.py | 150 +- .../tests/unittests/npu/test_adamw_op_npu.py | 40 +- .../test_amp_check_finite_and_scale_op_npu.py | 71 +- .../unittests/npu/test_arg_max_op_npu.py | 74 +- .../unittests/npu/test_arg_min_op_npu.py | 60 +- .../unittests/npu/test_argsort_op_npu.py | 53 +- .../tests/unittests/npu/test_assign_op_npu.py | 1 - .../unittests/npu/test_assign_value_op_npu.py | 38 +- .../tests/unittests/npu/test_atan_op_npu.py | 5 +- .../unittests/npu/test_batch_norm_op_npu.py | 284 +- .../tests/unittests/npu/test_bce_loss_npu.py | 259 +- .../npu/test_beam_search_decode_op_npu.py | 36 +- .../unittests/npu/test_beam_search_op_npu.py | 101 +- .../npu/test_bilinear_interp_v2_op_npu.py | 82 +- .../unittests/npu/test_box_coder_op_npu.py | 86 +- .../unittests/npu/test_c_embedding_op_npu.py | 6 +- .../unittests/npu/test_c_identity_npu.py | 7 +- .../tests/unittests/npu/test_cast_op_npu.py | 9 +- .../unittests/npu/test_clip_by_norm_op_npu.py | 14 +- .../tests/unittests/npu/test_clip_op_npu.py | 30 +- .../npu/test_coalesce_tensor_op_npu.py | 9 +- .../unittests/npu/test_collective_base_npu.py | 131 +- .../npu/test_collective_process_group_hccl.py | 1 - .../unittests/npu/test_compare_op_npu.py | 50 +- .../tests/unittests/npu/test_concat_op_npu.py | 33 +- .../npu/test_conv2d_op_depthwise_conv_npu.py | 184 +- .../tests/unittests/npu/test_conv2d_op_npu.py | 163 +- .../npu/test_conv2d_transpose_op_npu.py | 206 +- .../tests/unittests/npu/test_conv3d_op_npu.py | 437 +- .../npu/test_conv3d_transpose_op_npu.py | 85 +- .../tests/unittests/npu/test_cos_op_npu.py | 28 +- .../tests/unittests/npu/test_crop_op_npu.py | 13 +- .../tests/unittests/npu/test_cumsum_op_npu.py | 113 +- .../npu/test_density_prior_box_op_npu.py | 77 +- .../unittests/npu/test_dropout_op_npu.py | 120 +- .../npu/test_elementwise_add_op_npu.py | 131 +- .../npu/test_elementwise_div_op_npu.py | 51 +- .../npu/test_elementwise_floordiv_op_npu.py | 4 +- .../npu/test_elementwise_max_op_npu.py | 154 +- .../npu/test_elementwise_min_op_npu.py | 89 +- .../npu/test_elementwise_mod_op_npu.py | 14 +- .../npu/test_elementwise_mul_op_npu.py | 57 +- .../npu/test_elementwise_pow_op_npu.py | 152 +- .../npu/test_elementwise_sub_op_npu.py | 74 +- .../tests/unittests/npu/test_exp_op_npu.py | 2 - .../unittests/npu/test_expand_as_v2_op_npu.py | 33 +- .../tests/unittests/npu/test_expand_op_npu.py | 23 +- .../unittests/npu/test_expand_v2_op_npu.py | 67 +- .../tests/unittests/npu/test_eye_op_npu.py | 28 +- .../npu/test_fill_any_like_op_npu.py | 8 - ...st_fill_constant_batch_size_like_op_npu.py | 26 +- .../npu/test_fill_constant_op_npu.py | 14 +- .../npu/test_fill_zeros_like_op_npu.py | 6 - .../npu/test_flags_check_nan_inf_npu.py | 1 - .../unittests/npu/test_flatten2_op_npu.py | 6 +- .../test_flatten_contiguous_range_op_npu.py | 107 +- .../unittests/npu/test_float_status_op_npu.py | 2 - .../unittests/npu/test_gather_nd_op_npu.py | 43 +- .../tests/unittests/npu/test_gather_op_npu.py | 53 +- .../npu/test_gaussian_random_op_npu.py | 18 +- .../tests/unittests/npu/test_gelu_op_npu.py | 34 +- .../unittests/npu/test_group_norm_op_npu.py | 62 +- .../unittests/npu/test_hard_sigmoid_op_npu.py | 20 +- .../unittests/npu/test_hard_swish_op_npu.py | 57 +- .../unittests/npu/test_huber_loss_op_npu.py | 61 +- .../unittests/npu/test_increment_op_npu.py | 29 +- .../unittests/npu/test_index_sample_op_npu.py | 43 +- .../unittests/npu/test_index_select_op_npu.py | 70 +- .../npu/test_instance_norm_op_npu.py | 1 - .../npu/test_iou_similarity_op_npu.py | 10 +- .../unittests/npu/test_is_empty_op_npu.py | 41 +- .../unittests/npu/test_kldiv_loss_op_npu.py | 43 +- .../unittests/npu/test_label_smooth_op_npu.py | 24 +- .../unittests/npu/test_layer_norm_op_npu.py | 152 +- .../unittests/npu/test_leaky_relu_op_npu.py | 35 +- .../unittests/npu/test_log_loss_op_npu.py | 17 +- .../tests/unittests/npu/test_log_op_npu.py | 28 +- .../unittests/npu/test_log_softmax_op_npu.py | 38 +- .../unittests/npu/test_logical_op_npu.py | 137 +- .../npu/test_lookup_table_v2_op_npu.py | 20 +- .../npu/test_masked_select_op_npu.py | 38 +- .../tests/unittests/npu/test_matmul_op_npu.py | 40 +- .../unittests/npu/test_matmulv2_op_npu.py | 45 +- .../tests/unittests/npu/test_mean_op_npu.py | 2 - .../tests/unittests/npu/test_memcpy_op_npu.py | 93 +- .../npu/test_merged_momentum_op_npu.py | 295 +- .../unittests/npu/test_meshgrid_op_npu.py | 157 +- .../unittests/npu/test_mixed_precision_npu.py | 55 +- .../unittests/npu/test_momentum_op_npu.py | 111 +- .../tests/unittests/npu/test_mul_op_npu.py | 175 +- .../unittests/npu/test_multinomial_op_npu.py | 31 +- .../npu/test_nearest_interp_op_npu.py | 198 +- .../npu/test_nearest_interp_v2_op_npu.py | 116 +- .../tests/unittests/npu/test_norm_op_npu.py | 25 +- .../tests/unittests/npu/test_npu_place.py | 7 +- .../unittests/npu/test_one_hot_op_npu.py | 36 +- .../unittests/npu/test_one_hot_v2_op_npu.py | 43 +- .../tests/unittests/npu/test_p_norm_op_npu.py | 28 +- .../tests/unittests/npu/test_pad3d_op_npu.py | 325 +- .../tests/unittests/npu/test_pad_op_npu.py | 28 +- .../npu/test_parallel_dygraph_mnist_npu.py | 27 +- .../tests/unittests/npu/test_pool2d_op_npu.py | 207 +- .../tests/unittests/npu/test_pow_op_npu.py | 28 +- .../unittests/npu/test_prior_box_op_npu.py | 86 +- .../unittests/npu/test_randperm_op_npu.py | 19 +- .../tests/unittests/npu/test_range_npu.py | 16 +- .../unittests/npu/test_reciprocal_op_npu.py | 12 +- .../unittests/npu/test_reduce_any_op_npu.py | 30 +- .../unittests/npu/test_reduce_max_op_npu.py | 104 +- .../unittests/npu/test_reduce_mean_op_npu.py | 26 +- .../unittests/npu/test_reduce_min_op_npu.py | 104 +- .../unittests/npu/test_reduce_prod_op_npu.py | 62 +- .../unittests/npu/test_reduce_sum_op_npu.py | 49 +- .../tests/unittests/npu/test_relu6_op_npu.py | 29 +- .../tests/unittests/npu/test_relu_op_npu.py | 35 +- .../unittests/npu/test_reshape2_op_npu.py | 5 +- .../unittests/npu/test_rmsprop_op_npu.py | 52 +- .../unittests/npu/test_roi_align_op_npu.py | 124 +- .../unittests/npu/test_run_program_op_npu.py | 132 +- .../unittests/npu/test_sampling_id_op_npu.py | 1 - .../tests/unittests/npu/test_save_load_npu.py | 64 +- .../tests/unittests/npu/test_scale_op_npu.py | 20 +- .../unittests/npu/test_scatter_op_npu.py | 5 - .../tests/unittests/npu/test_seed_op_npu.py | 2 - .../unittests/npu/test_sequence_mask_npu.py | 54 +- .../unittests/npu/test_set_value_op_npu.py | 88 +- .../tests/unittests/npu/test_sgd_op_npu.py | 27 +- .../tests/unittests/npu/test_shape_op_npu.py | 5 - .../unittests/npu/test_shard_index_op.py | 6 +- ...igmoid_cross_entropy_with_logits_op_npu.py | 105 +- .../unittests/npu/test_sigmoid_op_npu.py | 18 +- .../tests/unittests/npu/test_sin_op_npu.py | 10 +- .../tests/unittests/npu/test_size_op_npu.py | 36 +- .../tests/unittests/npu/test_slice_op_npu.py | 137 +- .../npu/test_smooth_l1_loss_op_npu.py | 79 +- .../unittests/npu/test_softmax_op_npu.py | 27 +- .../test_softmax_with_cross_entropy_op_npu.py | 51 +- .../tests/unittests/npu/test_split_op_npu.py | 20 +- .../tests/unittests/npu/test_sqrt_op_npu.py | 28 +- .../tests/unittests/npu/test_square_op_npu.py | 28 +- .../npu/test_squared_l2_norm_op_npu.py | 9 +- .../unittests/npu/test_squeeze_op_npu.py | 42 +- .../tests/unittests/npu/test_stack_op_npu.py | 44 +- .../npu/test_strided_slice_op_npu.py | 236 +- .../tests/unittests/npu/test_sum_op_npu.py | 11 +- .../tests/unittests/npu/test_swish_op_npu.py | 13 +- .../npu/test_sync_batch_norm_base_npu.py | 279 +- .../test_sync_batch_norm_op_npu_baseline.py | 12 +- .../npu/test_sync_batch_norm_op_npu_extra.py | 34 +- .../npu/test_take_along_axis_op_npu.py | 28 +- .../tests/unittests/npu/test_tanh_op_npu.py | 28 +- .../tests/unittests/npu/test_tile_op_npu.py | 25 +- .../tests/unittests/npu/test_top_k_op_npu.py | 49 +- .../unittests/npu/test_top_k_v2_op_npu.py | 103 +- .../unittests/npu/test_transpose_op_npu.py | 19 +- .../unittests/npu/test_tril_triu_op_npu.py | 61 +- .../test_truncated_gaussian_random_op_npu.py | 14 +- .../npu/test_uniform_random_op_npu.py | 23 +- .../unittests/npu/test_unsqueeze_op_npu.py | 10 +- .../unittests/npu/test_unstack_op_npu.py | 5 - .../test_update_loss_scaling_min_op_npu.py | 5 +- .../npu/test_update_loss_scaling_op_npu.py | 151 +- .../unittests/npu/test_where_index_npu.py | 26 +- .../tests/unittests/npu/test_where_op_npu.py | 52 +- .../tests/unittests/npu/test_while_op_npu.py | 32 +- .../paddle/fluid/tests/unittests/op_test.py | 1485 ++-- .../fluid/tests/unittests/op_test_xpu.py | 321 +- ...allel_dygraph_dataparallel_with_pylayer.py | 9 +- .../parallel_dygraph_gradient_check.py | 34 +- ...el_dygraph_gradient_check_in_eager_mode.py | 52 +- .../tests/unittests/parallel_dygraph_mnist.py | 136 +- .../unittests/parallel_dygraph_none_var.py | 32 +- .../parallel_dygraph_shared_unused_var.py | 16 +- .../parallel_dygraph_sparse_embedding.py | 65 +- .../parallel_dygraph_sparse_embedding_fp64.py | 68 +- ...el_dygraph_sparse_embedding_over_height.py | 34 +- .../parallel_dygraph_unused_variables.py | 68 +- .../unittests/parallel_executor_test_base.py | 219 +- .../tests/unittests/ps/dataset_generator_A.py | 4 +- .../tests/unittests/ps/dataset_generator_B.py | 6 +- .../fluid/tests/unittests/ps/fl_ps_trainer.py | 84 +- .../tests/unittests/ps/ps_dnn_trainer.py | 271 +- .../fluid/tests/unittests/ps/test_fl_ps.py | 5 +- .../tests/unittests/ps/test_the_one_ps.py | 16 +- .../fluid/tests/unittests/ps_dnn_model.py | 207 +- .../tests/unittests/py_precise_roi_pool.py | 72 +- .../fluid/tests/unittests/rnn/convert.py | 28 +- .../fluid/tests/unittests/rnn/rnn_numpy.py | 311 +- .../tests/unittests/rnn/test_rnn_cells.py | 54 +- .../unittests/rnn/test_rnn_cells_static.py | 125 +- .../rnn/test_rnn_cudnn_params_packing.py | 8 +- .../tests/unittests/rnn/test_rnn_nets.py | 98 +- .../unittests/rnn/test_rnn_nets_static.py | 146 +- .../tests/unittests/rnn/test_wrappers.py | 36 +- .../unittests/rpc/rpc_launch_sync_add.py | 6 +- .../fluid/tests/unittests/rpc/test_rpc.py | 3 - .../tests/unittests/rpc/test_rpc_base.py | 49 +- .../sequence/test_sequence_concat.py | 17 +- .../unittests/sequence/test_sequence_conv.py | 177 +- .../sequence/test_sequence_enumerate_op.py | 41 +- .../sequence/test_sequence_erase_op.py | 7 +- .../sequence/test_sequence_expand.py | 11 +- .../sequence/test_sequence_expand_as.py | 5 - .../sequence/test_sequence_first_step.py | 13 +- .../sequence/test_sequence_last_step.py | 13 +- .../unittests/sequence/test_sequence_mask.py | 54 +- .../sequence/test_sequence_pad_op.py | 39 +- .../unittests/sequence/test_sequence_pool.py | 125 +- .../sequence/test_sequence_reshape.py | 19 +- .../sequence/test_sequence_reverse.py | 25 +- .../sequence/test_sequence_scatter_op.py | 9 +- .../sequence/test_sequence_slice_op.py | 15 +- .../sequence/test_sequence_softmax_op.py | 19 +- .../test_sequence_topk_avg_pooling.py | 80 +- .../sequence/test_sequence_unpad_op.py | 15 +- .../fluid/tests/unittests/seresnext_net.py | 165 +- .../tests/unittests/seresnext_test_base.py | 39 +- .../tests/unittests/simnet_dataset_reader.py | 1 - .../fluid/tests/unittests/simple_nets.py | 49 +- .../tests/unittests/spawn_runner_base.py | 30 +- .../static_model_parallel_fused_attention.py | 99 +- ...static_model_parallel_fused_feedforward.py | 85 +- ..._model_parallel_fused_multi_transformer.py | 53 +- .../fluid/tests/unittests/test_Tensor_type.py | 1 - .../fluid/tests/unittests/test_accuracy_op.py | 58 +- .../unittests/test_activation_nn_grad.py | 205 +- .../tests/unittests/test_activation_op.py | 864 ++- .../unittests/test_activation_sparse_op.py | 2 - .../fluid/tests/unittests/test_adadelta_op.py | 102 +- .../fluid/tests/unittests/test_adagrad_op.py | 77 +- .../tests/unittests/test_adagrad_op_v2.py | 29 +- .../fluid/tests/unittests/test_adam_op.py | 542 +- .../test_adam_optimizer_fp32_fp64.py | 7 +- .../fluid/tests/unittests/test_adamax_api.py | 50 +- .../fluid/tests/unittests/test_adamax_op.py | 56 +- .../fluid/tests/unittests/test_adamw_op.py | 421 +- .../unittests/test_adaptive_avg_pool1d.py | 77 +- .../unittests/test_adaptive_avg_pool2d.py | 214 +- .../unittests/test_adaptive_avg_pool3d.py | 230 +- .../unittests/test_adaptive_max_pool1d.py | 67 +- .../unittests/test_adaptive_max_pool2d.py | 179 +- .../unittests/test_adaptive_max_pool3d.py | 221 +- .../test_add_position_encoding_op.py | 48 +- .../unittests/test_add_reader_dependency.py | 48 +- .../fluid/tests/unittests/test_addmm_op.py | 286 +- .../tests/unittests/test_affine_channel_op.py | 26 +- .../unittests/test_affine_grid_function.py | 85 +- .../tests/unittests/test_affine_grid_op.py | 100 +- .../tests/unittests/test_allclose_layer.py | 163 +- .../fluid/tests/unittests/test_allclose_op.py | 47 +- .../fluid/tests/unittests/test_allgather.py | 1 - .../test_amp_check_finite_and_scale_op.py | 3 - .../unittests/test_anchor_generator_op.py | 36 +- .../fluid/tests/unittests/test_angle_op.py | 35 +- .../unittests/test_apply_pass_to_program.py | 60 +- .../fluid/tests/unittests/test_arange.py | 30 +- .../tests/unittests/test_arg_min_max_op.py | 66 +- .../tests/unittests/test_arg_min_max_v2_op.py | 128 +- .../fluid/tests/unittests/test_argsort_op.py | 148 +- .../unittests/test_array_read_write_op.py | 79 +- .../tests/unittests/test_ascend_trigger.py | 12 +- .../fluid/tests/unittests/test_assert_op.py | 27 +- .../fluid/tests/unittests/test_assign_op.py | 100 +- .../tests/unittests/test_assign_pos_op.py | 34 +- .../tests/unittests/test_assign_value_op.py | 38 +- .../test_async_ssa_graph_executor_mnist.py | 130 +- .../fluid/tests/unittests/test_atan2_op.py | 23 +- .../tests/unittests/test_attention_lstm_op.py | 63 +- .../tests/unittests/test_attribute_var.py | 20 +- .../fluid/tests/unittests/test_auc_op.py | 74 +- .../unittests/test_auc_single_pred_op.py | 32 +- .../test_auto_growth_gpu_memory_limit.py | 7 +- .../test_auto_parallel_autoconvert.py | 1 - .../unittests/test_auto_parallel_cluster.py | 16 +- .../test_auto_parallel_completion.py | 541 +- .../test_auto_parallel_completion_gpt.py | 635 +- .../test_auto_parallel_cost_model.py | 163 +- .../test_auto_parallel_data_unshard.py | 1 - .../test_auto_parallel_dist_tensor.py | 161 +- .../unittests/test_auto_parallel_graph.py | 1 - .../unittests/test_auto_parallel_mapper.py | 174 +- .../test_auto_parallel_partitioner.py | 1401 ++-- .../test_auto_parallel_partitioner_gpt.py | 733 +- .../unittests/test_auto_parallel_reshard.py | 228 +- .../test_auto_parallel_reshard_dpmppp.py | 138 +- .../test_auto_parallel_reshard_mppp.py | 174 +- .../test_auto_parallel_reshard_serial.py | 120 +- .../unittests/test_auto_parallel_save_load.py | 1 - .../unittests/test_auto_parallel_searcher.py | 186 +- .../test_auto_search_dist_matmul_op.py | 565 +- .../unittests/test_auto_search_dist_op.py | 470 +- .../test_avoid_twice_initialization.py | 31 +- .../fluid/tests/unittests/test_backward.py | 169 +- ...test_backward_infer_var_data_type_shape.py | 13 +- .../fluid/tests/unittests/test_base_layer.py | 254 +- .../tests/unittests/test_basic_gru_api.py | 269 +- .../tests/unittests/test_basic_gru_unit_op.py | 78 +- .../tests/unittests/test_basic_lstm_api.py | 217 +- .../unittests/test_basic_lstm_unit_op.py | 84 +- .../tests/unittests/test_basic_rnn_name.py | 127 +- .../fluid/tests/unittests/test_batch_fc_op.py | 34 +- .../tests/unittests/test_batch_norm_op.py | 350 +- .../tests/unittests/test_batch_norm_op_v2.py | 95 +- .../tests/unittests/test_batch_sampler.py | 75 +- .../fluid/tests/unittests/test_bce_loss.py | 265 +- .../unittests/test_bce_with_logits_loss.py | 349 +- .../unittests/test_beam_search_decode_op.py | 71 +- .../tests/unittests/test_beam_search_op.py | 278 +- .../tests/unittests/test_bernoulli_op.py | 12 +- .../paddle/fluid/tests/unittests/test_bfgs.py | 82 +- .../tests/unittests/test_bicubic_interp_op.py | 326 +- .../unittests/test_bicubic_interp_v2_op.py | 443 +- .../unittests/test_bilateral_slice_op.py | 55 +- .../tests/unittests/test_bilinear_api.py | 27 +- .../unittests/test_bilinear_interp_op.py | 214 +- .../unittests/test_bilinear_interp_v2_op.py | 327 +- .../test_bilinear_tensor_product_op.py | 13 +- .../fluid/tests/unittests/test_bincount_op.py | 72 +- .../unittests/test_bipartite_match_op.py | 44 +- .../fluid/tests/unittests/test_bitwise_op.py | 73 +- .../tests/unittests/test_block_rename_var.py | 22 +- .../fluid/tests/unittests/test_bmm_op.py | 40 +- .../fluid/tests/unittests/test_box_clip_op.py | 31 +- .../tests/unittests/test_box_coder_op.py | 111 +- .../test_box_decoder_and_assign_op.py | 6 +- .../fluid/tests/unittests/test_boxps.py | 68 +- .../fluid/tests/unittests/test_bpr_loss_op.py | 5 +- .../tests/unittests/test_broadcast_error.py | 1 - .../tests/unittests/test_broadcast_shape.py | 9 +- .../unittests/test_broadcast_tensors_op.py | 114 +- .../tests/unittests/test_broadcast_to_op.py | 40 +- .../tests/unittests/test_bucketize_api.py | 41 +- .../test_buffer_shared_memory_reuse_pass.py | 78 +- ...euse_pass_and_fuse_optimization_op_pass.py | 2 - .../unittests/test_c_comm_init_all_op.py | 9 +- .../tests/unittests/test_c_embedding_op.py | 6 +- .../tests/unittests/test_calc_gradient.py | 58 +- .../paddle/fluid/tests/unittests/test_case.py | 203 +- .../fluid/tests/unittests/test_cast_op.py | 61 +- .../fluid/tests/unittests/test_center_loss.py | 62 +- .../tests/unittests/test_channel_shuffle.py | 110 +- .../unittests/test_check_import_scipy.py | 1 - .../tests/unittests/test_checkpoint_saver.py | 1 - .../fluid/tests/unittests/test_cholesky_op.py | 41 +- .../tests/unittests/test_cholesky_solve_op.py | 72 +- .../tests/unittests/test_chunk_eval_op.py | 211 +- .../fluid/tests/unittests/test_chunk_op.py | 19 +- .../unittests/test_class_center_sample_op.py | 165 +- .../tests/unittests/test_clip_by_norm_op.py | 40 +- .../fluid/tests/unittests/test_clip_op.py | 144 +- .../unittests/test_coalesce_tensor_op.py | 127 +- .../test_collect_fpn_proposals_op.py | 45 +- .../unittests/test_collective_api_base.py | 361 +- .../tests/unittests/test_collective_base.py | 232 +- .../unittests/test_communicator_async.py | 4 +- .../tests/unittests/test_communicator_geo.py | 24 +- .../unittests/test_communicator_ps_gpu.py | 14 +- .../fluid/tests/unittests/test_compare_op.py | 154 +- .../tests/unittests/test_compare_reduce_op.py | 9 - .../fluid/tests/unittests/test_compat.py | 1 - .../tests/unittests/test_compiled_program.py | 84 +- .../fluid/tests/unittests/test_complex_abs.py | 48 +- .../tests/unittests/test_complex_cast.py | 59 +- .../test_complex_elementwise_layers.py | 46 +- .../tests/unittests/test_complex_getitem.py | 1 - .../test_complex_grad_accumulated.py | 60 +- .../tests/unittests/test_complex_kron.py | 45 +- .../tests/unittests/test_complex_matmul.py | 68 +- .../fluid/tests/unittests/test_complex_op.py | 74 +- .../tests/unittests/test_complex_reshape.py | 25 +- .../tests/unittests/test_complex_simplenet.py | 27 +- .../tests/unittests/test_complex_sum_layer.py | 7 +- .../unittests/test_complex_trace_layer.py | 12 +- .../tests/unittests/test_complex_transpose.py | 7 +- .../tests/unittests/test_complex_variable.py | 28 +- .../tests/unittests/test_complex_view_op.py | 33 +- .../fluid/tests/unittests/test_concat_op.py | 164 +- .../paddle/fluid/tests/unittests/test_cond.py | 434 +- .../tests/unittests/test_conditional_block.py | 36 +- .../fluid/tests/unittests/test_conj_op.py | 53 +- .../fluid/tests/unittests/test_const_value.py | 1 - .../tests/unittests/test_context_manager.py | 1 - .../tests/unittests/test_conv1d_layer.py | 198 +- .../unittests/test_conv1d_transpose_layer.py | 230 +- .../fluid/tests/unittests/test_conv2d_api.py | 458 +- .../tests/unittests/test_conv2d_fusion_op.py | 47 +- .../tests/unittests/test_conv2d_layer.py | 273 +- .../fluid/tests/unittests/test_conv2d_op.py | 460 +- .../test_conv2d_op_depthwise_conv.py | 38 +- .../unittests/test_conv2d_transpose_layer.py | 226 +- .../unittests/test_conv2d_transpose_op.py | 512 +- ...test_conv2d_transpose_op_depthwise_conv.py | 5 - .../tests/unittests/test_conv3d_layer.py | 206 +- .../fluid/tests/unittests/test_conv3d_op.py | 677 +- .../unittests/test_conv3d_transpose_layer.py | 237 +- .../unittests/test_conv3d_transpose_op.py | 216 +- .../test_conv3d_transpose_part2_op.py | 188 +- .../tests/unittests/test_conv_nn_grad.py | 345 +- .../tests/unittests/test_conv_shift_op.py | 1 - .../unittests/test_conv_transpose_nn_grad.py | 254 +- .../paddle/fluid/tests/unittests/test_corr.py | 33 +- .../fluid/tests/unittests/test_cos_sim_op.py | 73 +- .../unittests/test_cosine_embedding_loss.py | 260 +- .../unittests/test_cosine_similarity_api.py | 12 +- .../fluid/tests/unittests/test_cost_model.py | 26 +- .../tests/unittests/test_count_nonzero_api.py | 10 +- .../paddle/fluid/tests/unittests/test_cov.py | 215 +- .../tests/unittests/test_cpuonly_spawn.py | 2 - .../tests/unittests/test_create_global_var.py | 1 - .../unittests/test_create_op_doc_string.py | 1 - .../tests/unittests/test_create_parameter.py | 17 +- .../tests/unittests/test_crf_decoding_op.py | 114 +- .../fluid/tests/unittests/test_crop_op.py | 20 +- .../tests/unittests/test_crop_tensor_op.py | 52 +- .../tests/unittests/test_cross_entropy2_op.py | 39 +- .../unittests/test_cross_entropy_loss.py | 1540 ++-- .../tests/unittests/test_cross_entropy_op.py | 286 +- .../fluid/tests/unittests/test_cross_op.py | 52 +- .../fluid/tests/unittests/test_crypto.py | 2 - .../fluid/tests/unittests/test_ctc_align.py | 148 +- .../unittests/test_cuda_cudnn_version.py | 1 - .../tests/unittests/test_cuda_device_count.py | 1 - .../test_cuda_device_name_capability.py | 5 +- .../tests/unittests/test_cuda_empty_cache.py | 1 - .../fluid/tests/unittests/test_cuda_graph.py | 68 +- .../test_cuda_graph_partial_graph.py | 2 - .../test_cuda_graph_partial_graph_static.py | 8 +- ...est_cuda_graph_partial_graph_static_run.py | 8 +- .../test_cuda_max_memory_allocated.py | 23 +- .../test_cuda_max_memory_reserved.py | 23 +- .../unittests/test_cuda_memory_allocated.py | 7 +- .../unittests/test_cuda_memory_reserved.py | 7 +- .../tests/unittests/test_cuda_random_seed.py | 71 +- .../tests/unittests/test_cuda_stream_event.py | 15 +- .../tests/unittests/test_cudnn_grucell.py | 114 +- .../tests/unittests/test_cudnn_lstmcell.py | 185 +- .../fluid/tests/unittests/test_cumprod_op.py | 28 +- .../fluid/tests/unittests/test_cumsum_op.py | 169 +- .../tests/unittests/test_custom_grad_input.py | 66 +- .../fluid/tests/unittests/test_cvm_op.py | 78 +- .../unittests/test_cyclic_cifar_dataset.py | 1 - .../paddle/fluid/tests/unittests/test_data.py | 17 +- .../tests/unittests/test_data_generator.py | 27 - .../tests/unittests/test_data_norm_op.py | 123 +- .../unittests/test_dataloader_autotune.py | 51 +- .../unittests/test_dataloader_dataset.py | 24 +- .../unittests/test_dataloader_early_reset.py | 10 +- .../unittests/test_dataloader_keep_order.py | 61 +- .../unittests/test_dataloader_unkeep_order.py | 76 +- .../fluid/tests/unittests/test_dataset.py | 607 +- .../test_dataset_consistency_inspection.py | 347 +- .../unittests/test_dataset_dataloader.py | 72 +- .../tests/unittests/test_dataset_download.py | 1 - .../fluid/tests/unittests/test_debugger.py | 59 +- .../unittests/test_decayed_adagrad_op.py | 11 +- .../unittests/test_decoupled_py_reader.py | 76 +- .../test_decoupled_py_reader_data_check.py | 38 +- .../tests/unittests/test_default_dtype.py | 2 - .../unittests/test_default_scope_funcs.py | 11 +- .../tests/unittests/test_deform_conv2d.py | 324 +- .../unittests/test_deformable_conv_op.py | 364 +- .../unittests/test_deformable_conv_v1_op.py | 241 +- .../test_deformable_psroi_pooling.py | 355 +- .../fluid/tests/unittests/test_deg2rad.py | 21 +- .../unittests/test_density_prior_box_op.py | 71 +- .../unittests/test_deprecated_decorator.py | 21 +- ...t_deprecated_memory_optimize_interfaces.py | 2 - .../unittests/test_dequantize_abs_max_op.py | 5 +- .../tests/unittests/test_dequantize_log_op.py | 3 +- .../fluid/tests/unittests/test_desc_clone.py | 131 +- .../fluid/tests/unittests/test_detach.py | 143 +- .../tests/unittests/test_detection_map_op.py | 133 +- .../tests/unittests/test_determinant_op.py | 13 +- .../fluid/tests/unittests/test_device.py | 26 +- .../tests/unittests/test_device_guard.py | 99 +- .../paddle/fluid/tests/unittests/test_diag.py | 3 - .../fluid/tests/unittests/test_diag_embed.py | 24 +- .../fluid/tests/unittests/test_diag_v2.py | 88 +- .../fluid/tests/unittests/test_diagflat.py | 11 +- .../fluid/tests/unittests/test_diagonal_op.py | 57 +- .../fluid/tests/unittests/test_diff_op.py | 111 +- .../fluid/tests/unittests/test_digamma_op.py | 4 - .../unittests/test_directory_migration.py | 192 +- .../unittests/test_disable_signal_handler.py | 22 +- .../tests/unittests/test_dist_allreduce_op.py | 2 +- .../fluid/tests/unittests/test_dist_base.py | 1023 ++- .../tests/unittests/test_dist_dygraph_apis.py | 1 - .../test_dist_fleet_a_sync_optimizer_async.py | 6 +- .../test_dist_fleet_a_sync_optimizer_auto.py | 17 +- ..._dist_fleet_a_sync_optimizer_auto_async.py | 29 +- ...st_dist_fleet_a_sync_optimizer_auto_geo.py | 17 +- .../test_dist_fleet_a_sync_optimizer_geo.py | 28 +- .../test_dist_fleet_a_sync_optimizer_sync.py | 6 +- .../tests/unittests/test_dist_fleet_base.py | 251 +- .../tests/unittests/test_dist_fleet_ctr.py | 59 +- .../tests/unittests/test_dist_fleet_ctr2.py | 30 +- .../tests/unittests/test_dist_fleet_decay.py | 41 +- .../tests/unittests/test_dist_fleet_geo.py | 19 +- .../tests/unittests/test_dist_fleet_gloo.py | 74 +- .../unittests/test_dist_fleet_heter_base.py | 336 +- .../test_dist_fleet_heter_program.py | 67 +- .../tests/unittests/test_dist_fleet_infer.py | 44 +- .../tests/unittests/test_dist_fleet_ps.py | 106 +- .../tests/unittests/test_dist_fleet_ps10.py | 41 +- .../tests/unittests/test_dist_fleet_ps11.py | 93 +- .../tests/unittests/test_dist_fleet_ps12.py | 66 +- .../tests/unittests/test_dist_fleet_ps13.py | 93 +- .../tests/unittests/test_dist_fleet_ps2.py | 99 +- .../tests/unittests/test_dist_fleet_ps3.py | 106 +- .../tests/unittests/test_dist_fleet_ps4.py | 100 +- .../tests/unittests/test_dist_fleet_ps5.py | 117 +- .../tests/unittests/test_dist_fleet_ps6.py | 100 +- .../tests/unittests/test_dist_fleet_ps7.py | 41 +- .../tests/unittests/test_dist_fleet_ps8.py | 41 +- .../tests/unittests/test_dist_fleet_ps9.py | 41 +- .../test_dist_fleet_raw_program_optimizer.py | 12 +- ...et_raw_program_optimizer_fuse_allreduce.py | 5 +- .../tests/unittests/test_dist_fleet_simnet.py | 17 +- .../test_dist_fleet_sparse_embedding_ctr.py | 158 +- .../test_dist_fleet_trainer_desc_config.py | 20 +- .../test_dist_lookup_sparse_table_fuse_ops.py | 89 +- .../test_dist_mnist_backward_deps.py | 2 +- .../unittests/test_dist_mnist_batch_merge.py | 40 +- .../unittests/test_dist_mnist_fleet_save.py | 19 +- .../unittests/test_dist_mnist_fleetapi.py | 16 +- .../test_dist_mnist_fp16_allreduce.py | 2 +- .../unittests/test_dist_mnist_hallreduce.py | 12 +- .../tests/unittests/test_dist_mnist_lars.py | 1 - .../unittests/test_dist_mnist_multi_comm.py | 12 +- .../tests/unittests/test_dist_mnist_pg.py | 16 +- .../test_dist_mnist_ring_allreduce.py | 2 +- .../tests/unittests/test_dist_mnist_train.py | 38 +- .../unittests/test_dist_mnist_with_program.py | 4 +- .../fluid/tests/unittests/test_dist_oneps.py | 1 - .../fluid/tests/unittests/test_dist_op.py | 66 +- .../tests/unittests/test_dist_save_load.py | 89 +- .../unittests/test_dist_se_resnext_nccl.py | 26 +- .../unittests/test_dist_se_resnext_sync.py | 11 +- .../unittests/test_dist_sharding_save.py | 37 +- .../unittests/test_dist_sparse_load_ps0.py | 16 +- .../unittests/test_dist_sparse_load_ps1.py | 4 +- .../test_dist_sparse_tensor_load_adagrad.py | 5 +- .../test_dist_sparse_tensor_load_adam.py | 5 +- .../test_dist_sparse_tensor_load_ftrl.py | 5 +- .../test_dist_sparse_tensor_load_momentum.py | 5 +- .../test_dist_sparse_tensor_load_rmsprop.py | 5 +- .../test_dist_sparse_tensor_load_sgd.py | 15 +- .../test_dist_text_classification.py | 22 +- .../fluid/tests/unittests/test_dist_train.py | 71 +- .../tests/unittests/test_dist_transformer.py | 50 +- .../tests/unittests/test_dist_transpiler.py | 1133 ++- .../tests/unittests/test_dist_tree_index.py | 89 +- .../tests/unittests/test_dist_word2vec.py | 33 +- .../test_distribute_fpn_proposals_op.py | 113 +- ...est_distributed_fused_lamb_op_with_clip.py | 17 +- ...buted_fused_lamb_op_with_gradient_merge.py | 19 +- ..._distributed_fused_lamb_op_without_clip.py | 1 - .../tests/unittests/test_distributions.py | 680 +- .../fluid/tests/unittests/test_dot_op.py | 179 +- .../fluid/tests/unittests/test_downpoursgd.py | 53 +- .../fluid/tests/unittests/test_dpsgd_op.py | 7 +- .../tests/unittests/test_dropout_nd_op.py | 65 +- .../fluid/tests/unittests/test_dropout_op.py | 774 +- .../unittests/test_dygraph_mnist_fp16.py | 132 +- .../test_dygraph_mode_of_unittest.py | 3 +- .../unittests/test_dygraph_multi_forward.py | 200 +- .../unittests/test_dygraph_spectral_norm.py | 54 +- .../unittests/test_dygraph_weight_norm.py | 66 +- .../fluid/tests/unittests/test_dyn_rnn.py | 215 +- .../test_dynamic_rnn_stop_gradient.py | 40 +- .../unittests/test_dynrnn_gradient_check.py | 124 +- .../unittests/test_dynrnn_static_input.py | 83 +- .../test_eager_deletion_delete_vars.py | 76 +- .../test_eager_deletion_dynamic_rnn_base.py | 33 +- .../unittests/test_eager_deletion_gru_net.py | 22 +- .../unittests/test_eager_deletion_lstm_net.py | 28 +- .../test_eager_deletion_padding_rnn.py | 433 +- .../test_eager_deletion_recurrent_op.py | 266 +- .../unittests/test_eager_deletion_while_op.py | 59 +- .../tests/unittests/test_eager_run_program.py | 90 +- .../tests/unittests/test_eager_trace_op.py | 19 +- .../tests/unittests/test_edit_distance_op.py | 44 +- .../unittests/test_egr_code_generate_api.py | 10 +- .../tests/unittests/test_egr_python_api.py | 378 +- .../unittests/test_egr_string_tensor_api.py | 39 +- .../fluid/tests/unittests/test_eig_op.py | 164 +- .../fluid/tests/unittests/test_eigh_op.py | 112 +- .../fluid/tests/unittests/test_eigvals_op.py | 181 +- .../fluid/tests/unittests/test_eigvalsh_op.py | 97 +- .../fluid/tests/unittests/test_einsum.py | 203 +- .../fluid/tests/unittests/test_einsum_op.py | 29 +- .../fluid/tests/unittests/test_einsum_v2.py | 247 +- .../unittests/test_elementwise_add_op.py | 230 +- .../unittests/test_elementwise_div_op.py | 151 +- .../unittests/test_elementwise_floordiv_op.py | 6 +- .../unittests/test_elementwise_gradient_op.py | 62 +- .../test_elementwise_heaviside_op.py | 72 +- .../unittests/test_elementwise_max_op.py | 100 +- .../unittests/test_elementwise_min_op.py | 103 +- .../unittests/test_elementwise_mod_op.py | 24 +- .../unittests/test_elementwise_mul_op.py | 164 +- .../unittests/test_elementwise_nn_grad.py | 148 +- .../unittests/test_elementwise_pow_op.py | 72 +- .../unittests/test_elementwise_sub_op.py | 148 +- .../paddle/fluid/tests/unittests/test_ema.py | 36 +- .../fluid/tests/unittests/test_ema_fleet.py | 35 +- .../test_embedding_id_stop_gradient.py | 11 +- .../tests/unittests/test_empty_like_op.py | 52 +- .../fluid/tests/unittests/test_empty_op.py | 80 +- .../fluid/tests/unittests/test_entry_attr.py | 24 +- .../fluid/tests/unittests/test_entry_attr2.py | 17 +- .../fluid/tests/unittests/test_erf_op.py | 2 - .../fluid/tests/unittests/test_erfinv_op.py | 20 +- .../fluid/tests/unittests/test_exception.py | 13 +- .../tests/unittests/test_executor_and_mul.py | 16 +- .../test_executor_and_use_program_cache.py | 138 +- .../unittests/test_executor_check_feed.py | 32 +- .../test_executor_check_fetch_list.py | 23 +- .../test_executor_feed_non_tensor.py | 123 +- ..._executor_return_tensor_not_overwriting.py | 12 +- .../tests/unittests/test_expand_as_op.py | 35 +- .../tests/unittests/test_expand_as_v2_op.py | 34 +- .../fluid/tests/unittests/test_expand_op.py | 74 +- .../tests/unittests/test_expand_v2_op.py | 107 +- .../tests/unittests/test_exponential_op.py | 242 +- .../fluid/tests/unittests/test_eye_op.py | 29 +- .../unittests/test_fake_dequantize_op.py | 81 +- .../tests/unittests/test_fake_init_op.py | 6 +- .../tests/unittests/test_fake_quantize_op.py | 298 +- .../unittests/test_faster_tokenizer_op.py | 220 +- .../fluid/tests/unittests/test_fc_op.py | 41 +- .../test_feed_data_check_shape_type.py | 211 +- .../tests/unittests/test_feed_fetch_method.py | 1 - .../tests/unittests/test_fetch_handler.py | 15 +- .../unittests/test_fetch_lod_tensor_array.py | 34 +- .../tests/unittests/test_fetch_unmerged.py | 85 +- .../fluid/tests/unittests/test_fetch_var.py | 2 - .../tests/unittests/test_fill_any_like_op.py | 24 +- .../fluid/tests/unittests/test_fill_any_op.py | 17 +- .../test_fill_constant_batch_size_like.py | 22 +- .../tests/unittests/test_fill_constant_op.py | 305 +- .../unittests/test_fill_diagonal_tensor_op.py | 11 +- .../fluid/tests/unittests/test_fill_op.py | 21 +- .../unittests/test_fill_zeros_like2_op.py | 5 - .../unittests/test_fill_zeros_like_op.py | 2 - .../unittests/test_filter_by_instag_op.py | 48 +- .../fluid/tests/unittests/test_flatten2_op.py | 20 +- .../test_flatten_contiguous_range_op.py | 111 +- .../fluid/tests/unittests/test_flatten_op.py | 4 - .../fluid/tests/unittests/test_fleet.py | 38 +- .../tests/unittests/test_fleet_api_input.py | 239 +- .../unittests/test_fleet_ascend_utils.py | 35 +- .../fluid/tests/unittests/test_fleet_auto.py | 17 +- .../fluid/tests/unittests/test_fleet_base.py | 50 +- .../tests/unittests/test_fleet_base_2.py | 41 +- .../tests/unittests/test_fleet_base_3.py | 34 +- .../tests/unittests/test_fleet_base_4.py | 6 +- .../tests/unittests/test_fleet_base_single.py | 34 +- .../test_fleet_elastic_collective.py | 8 +- .../unittests/test_fleet_elastic_init.py | 2 - .../unittests/test_fleet_elastic_manager.py | 114 +- .../test_fleet_exe_dist_model_run.py | 41 +- .../test_fleet_exe_dist_model_tensor.py | 52 +- .../tests/unittests/test_fleet_executor.py | 39 +- .../test_fleet_executor_multi_devices.py | 10 +- .../test_fleet_executor_origin_scheduler.py | 39 +- .../test_fleet_executor_task_node.py | 16 +- .../unittests/test_fleet_executor_utils.py | 12 +- .../test_fleet_executor_with_task_nodes.py | 41 +- .../unittests/test_fleet_gradient_scale.py | 31 +- .../tests/unittests/test_fleet_metric.py | 25 +- .../tests/unittests/test_fleet_nocvm_1.py | 50 +- .../fluid/tests/unittests/test_fleet_ps.py | 21 +- .../unittests/test_fleet_pyramid_hash.py | 15 +- .../tests/unittests/test_fleet_rolemaker.py | 33 +- .../tests/unittests/test_fleet_rolemaker_2.py | 36 +- .../tests/unittests/test_fleet_rolemaker_3.py | 34 +- .../tests/unittests/test_fleet_rolemaker_4.py | 9 +- .../unittests/test_fleet_rolemaker_init.py | 27 +- .../tests/unittests/test_fleet_runtime.py | 26 +- .../unittests/test_fleet_unitaccessor.py | 35 +- .../fluid/tests/unittests/test_fleet_util.py | 76 +- .../paddle/fluid/tests/unittests/test_flip.py | 56 +- .../fluid/tests/unittests/test_fmax_op.py | 104 +- .../fluid/tests/unittests/test_fmin_op.py | 104 +- .../fluid/tests/unittests/test_fold_op.py | 138 +- .../fluid/tests/unittests/test_frac_api.py | 12 +- .../fluid/tests/unittests/test_frame_op.py | 10 +- .../unittests/test_framework_debug_str.py | 1 - .../fluid/tests/unittests/test_frexp_api.py | 17 +- .../tests/unittests/test_fs_interface.py | 1 - .../fluid/tests/unittests/test_fsp_op.py | 27 +- .../fluid/tests/unittests/test_ftrl_op.py | 97 +- .../tests/unittests/test_full_like_op.py | 62 +- .../fluid/tests/unittests/test_full_op.py | 225 +- .../tests/unittests/test_function_hook.py | 2 - .../tests/unittests/test_functional_conv1d.py | 27 +- .../test_functional_conv1d_transpose.py | 27 +- .../tests/unittests/test_functional_conv2d.py | 236 +- .../test_functional_conv2d_transpose.py | 250 +- .../tests/unittests/test_functional_conv3d.py | 237 +- .../test_functional_conv3d_transpose.py | 269 +- .../unittests/test_fuse_all_reduce_pass.py | 78 +- .../tests/unittests/test_fuse_bn_act_pass.py | 74 +- .../unittests/test_fuse_bn_add_act_pass.py | 229 +- .../test_fuse_elewise_add_act_pass.py | 43 +- .../unittests/test_fuse_gemm_epilogue_pass.py | 261 +- .../unittests/test_fuse_optimizer_pass.py | 147 +- .../test_fuse_relu_depthwise_conv_pass.py | 68 +- .../unittests/test_fused_attention_op.py | 224 +- .../unittests/test_fused_attention_op_api.py | 350 +- ...sed_bias_dropout_residual_layer_norm_op.py | 111 +- ...bias_dropout_residual_layer_norm_op_api.py | 100 +- .../test_fused_elemwise_activation_op.py | 293 +- .../unittests/test_fused_emb_seq_pool_op.py | 69 +- .../test_fused_embedding_fc_lstm_op.py | 100 +- .../test_fused_fc_elementwise_layernorm_op.py | 16 +- .../unittests/test_fused_feedforward_op.py | 317 +- .../unittests/test_fused_gate_attention_op.py | 184 +- .../test_fused_gemm_epilogue_grad_op.py | 128 +- .../unittests/test_fused_gemm_epilogue_op.py | 303 +- .../tests/unittests/test_fused_matmul_bias.py | 20 +- .../test_fused_multi_transformer_int8_op.py | 414 +- .../test_fused_multi_transformer_op.py | 547 +- .../test_fused_multihead_matmul_op.py | 58 +- .../unittests/test_fused_token_prune_op.py | 79 +- .../test_fused_transformer_encoder_layer.py | 131 +- ...st_fused_transformer_with_amp_decorator.py | 32 +- .../tests/unittests/test_fusion_gru_op.py | 82 +- .../tests/unittests/test_fusion_lstm_op.py | 84 +- .../test_fusion_repeated_fc_relu_op.py | 29 +- .../test_fusion_seqconv_eltadd_relu_op.py | 12 +- .../test_fusion_seqexpand_concat_fc_op.py | 33 +- .../test_fusion_seqpool_concat_op.py | 19 +- .../test_fusion_seqpool_cvm_concat_op.py | 19 +- .../test_fusion_squared_mat_sub_op.py | 6 +- ...test_fusion_transpose_flatten_concat_op.py | 44 +- .../unittests/test_gast_with_compatibility.py | 2 - .../tests/unittests/test_gather_nd_op.py | 66 +- .../fluid/tests/unittests/test_gather_op.py | 80 +- .../tests/unittests/test_gather_tree_op.py | 107 +- .../unittests/test_gaussian_random_op.py | 212 +- .../paddle/fluid/tests/unittests/test_gcd.py | 34 +- .../fluid/tests/unittests/test_gelu_op.py | 18 +- .../unittests/test_generate_mask_labels_op.py | 154 +- .../test_generate_proposal_labels_op.py | 265 +- .../unittests/test_generate_proposals_op.py | 225 +- .../test_generate_proposals_v2_op.py | 124 +- .../fluid/tests/unittests/test_generator.py | 6 +- .../unittests/test_generator_dataloader.py | 73 +- .../test_get_all_op_or_kernel_names.py | 6 +- .../unittests/test_get_device_properties.py | 2 - .../test_get_inputs_outputs_in_block.py | 7 +- .../tests/unittests/test_get_places_op.py | 1 - .../tests/unittests/test_get_set_flags.py | 4 +- .../test_get_tensor_from_selected_rows_op.py | 1 - .../test_global_var_getter_setter.py | 2 - .../paddle/fluid/tests/unittests/test_glu.py | 2 - .../test_gpu_package_without_gpu_device.py | 16 +- .../unittests/test_grad_clip_minimize.py | 53 +- .../tests/unittests/test_gradient_clip.py | 264 +- .../unittests/test_graph_khop_sampler.py | 176 +- .../tests/unittests/test_graph_reindex.py | 469 +- .../unittests/test_graph_sample_neighbors.py | 348 +- .../unittests/test_graph_send_recv_op.py | 308 +- .../unittests/test_graph_send_ue_recv_op.py | 487 +- .../tests/unittests/test_graph_send_uv_op.py | 159 +- .../unittests/test_grid_sample_function.py | 73 +- .../tests/unittests/test_grid_sampler_op.py | 272 +- .../tests/unittests/test_group_norm_op.py | 141 +- .../tests/unittests/test_group_norm_op_v2.py | 58 +- .../fluid/tests/unittests/test_gru_op.py | 111 +- .../fluid/tests/unittests/test_gru_rnn_op.py | 49 +- .../fluid/tests/unittests/test_gru_unit_op.py | 130 +- .../tests/unittests/test_gumbel_softmax_op.py | 35 +- .../fluid/tests/unittests/test_hash_op.py | 55 +- .../unittests/test_hinge_embedding_loss.py | 121 +- .../tests/unittests/test_hinge_loss_op.py | 1 - .../tests/unittests/test_histogram_op.py | 63 +- .../fluid/tests/unittests/test_hsigmoid_op.py | 497 +- .../tests/unittests/test_huber_loss_op.py | 40 +- .../test_hybrid_parallel_topology.py | 104 +- .../tests/unittests/test_identity_loss_op.py | 24 +- .../fluid/tests/unittests/test_identity_op.py | 1 - .../tests/unittests/test_iinfo_and_finfo.py | 22 +- .../tests/unittests/test_im2sequence_op.py | 193 +- .../test_image_classification_layer.py | 51 +- .../unittests/test_imperative_auto_prune.py | 62 +- .../tests/unittests/test_imperative_basic.py | 339 +- .../test_imperative_container_layerdict.py | 21 +- .../test_imperative_container_layerlist.py | 32 +- ...test_imperative_container_parameterlist.py | 27 +- .../test_imperative_container_sequential.py | 11 +- .../test_imperative_data_loader_base.py | 53 +- .../test_imperative_data_loader_exception.py | 48 +- .../test_imperative_data_loader_exit_func.py | 17 +- .../test_imperative_data_loader_fds_clear.py | 33 +- .../test_imperative_data_loader_process.py | 30 +- .../test_imperative_data_parallel.py | 14 +- .../unittests/test_imperative_decorator.py | 14 +- .../tests/unittests/test_imperative_deepcf.py | 199 +- .../unittests/test_imperative_double_grad.py | 292 +- .../unittests/test_imperative_framework.py | 34 +- .../tests/unittests/test_imperative_gan.py | 132 +- .../tests/unittests/test_imperative_gnn.py | 97 +- .../tests/unittests/test_imperative_group.py | 55 +- .../test_imperative_hook_for_layer.py | 102 +- .../unittests/test_imperative_layer_apply.py | 49 +- .../test_imperative_layer_children.py | 16 +- .../test_imperative_layer_trainable.py | 1 - .../tests/unittests/test_imperative_layers.py | 192 +- .../test_imperative_load_static_param.py | 233 +- ..._imperative_lod_tensor_to_selected_rows.py | 112 +- .../tests/unittests/test_imperative_mnist.py | 215 +- .../test_imperative_mnist_sorted_gradient.py | 98 +- .../test_imperative_named_members.py | 70 +- .../unittests/test_imperative_numpy_bridge.py | 7 +- .../test_imperative_ocr_attention_model.py | 568 +- .../unittests/test_imperative_optimizer.py | 506 +- .../unittests/test_imperative_optimizer_v2.py | 487 +- ...test_imperative_parallel_coalesce_split.py | 13 +- .../test_imperative_partitial_backward.py | 10 +- .../unittests/test_imperative_ptb_rnn.py | 288 +- ...test_imperative_ptb_rnn_sorted_gradient.py | 124 +- .../test_imperative_recurrent_usage.py | 41 +- .../test_imperative_reinforcement.py | 60 +- .../tests/unittests/test_imperative_resnet.py | 275 +- .../test_imperative_resnet_sorted_gradient.py | 113 +- .../unittests/test_imperative_save_load.py | 498 +- .../unittests/test_imperative_save_load_v2.py | 537 +- .../unittests/test_imperative_se_resnext.py | 402 +- .../test_imperative_selected_rows.py | 33 +- ..._imperative_selected_rows_to_lod_tensor.py | 110 +- .../test_imperative_signal_handler.py | 5 - ...perative_star_gan_with_gradient_penalty.py | 436 +- .../test_imperative_static_runner_mnist.py | 278 +- .../test_imperative_static_runner_while.py | 122 +- .../test_imperative_tensor_clear_gradient.py | 2 - .../test_imperative_thread_local_has_grad.py | 2 - ...imperative_trace_non_persistable_inputs.py | 46 +- ..._imperative_transformer_sorted_gradient.py | 1151 +-- .../unittests/test_imperative_triple_grad.py | 199 +- .../test_imperative_using_non_zero_gpu.py | 1 - .../fluid/tests/unittests/test_increment.py | 10 +- .../tests/unittests/test_index_add_op.py | 153 +- .../tests/unittests/test_index_sample_op.py | 40 +- .../tests/unittests/test_index_select_op.py | 79 +- .../test_infer_no_need_buffer_slots.py | 36 +- .../fluid/tests/unittests/test_infer_shape.py | 1 - .../tests/unittests/test_inference_api.py | 61 +- .../unittests/test_inference_model_io.py | 334 +- .../fluid/tests/unittests/test_initializer.py | 575 +- .../tests/unittests/test_initializer_nn.py | 427 +- .../fluid/tests/unittests/test_inner.py | 48 +- .../fluid/tests/unittests/test_inplace.py | 74 +- .../tests/unittests/test_inplace_abn_op.py | 130 +- .../unittests/test_inplace_addto_strategy.py | 75 +- .../test_inplace_and_clear_gradient.py | 2 - .../test_inplace_auto_generated_apis.py | 42 +- ...test_inplace_softmax_with_cross_entropy.py | 92 +- .../fluid/tests/unittests/test_input_spec.py | 34 +- .../tests/unittests/test_install_check.py | 1 - .../tests/unittests/test_instance_norm_op.py | 171 +- .../unittests/test_instance_norm_op_v2.py | 16 +- .../fluid/tests/unittests/test_inverse_op.py | 51 +- .../tests/unittests/test_io_save_load.py | 35 +- .../tests/unittests/test_iou_similarity_op.py | 7 +- .../fluid/tests/unittests/test_ir_graph.py | 15 +- .../tests/unittests/test_ir_inplace_pass.py | 15 +- .../test_ir_memory_optimize_ifelse_op.py | 31 +- .../unittests/test_ir_memory_optimize_nlp.py | 28 +- .../unittests/test_ir_memory_optimize_pass.py | 17 +- .../test_ir_memory_optimize_transformer.py | 7 +- .../fluid/tests/unittests/test_is_complex.py | 1 - .../fluid/tests/unittests/test_is_empty_op.py | 21 +- .../fluid/tests/unittests/test_is_integer.py | 1 - .../fluid/tests/unittests/test_is_tensor.py | 10 +- .../fluid/tests/unittests/test_isclose_op.py | 63 +- .../fluid/tests/unittests/test_isfinite_op.py | 24 +- .../tests/unittests/test_isfinite_v2_op.py | 26 +- .../fluid/tests/unittests/test_jit_layer.py | 11 +- .../unittests/test_jit_pre_save_hooks.py | 14 +- .../tests/unittests/test_jit_save_load.py | 846 ++- .../tests/unittests/test_kldiv_loss_op.py | 18 +- .../fluid/tests/unittests/test_kron_op.py | 75 +- .../fluid/tests/unittests/test_kthvalue_op.py | 48 +- .../fluid/tests/unittests/test_l1_loss.py | 64 +- .../fluid/tests/unittests/test_l1_norm_op.py | 3 +- .../unittests/test_label_smooth_functional.py | 57 +- .../tests/unittests/test_label_smooth_op.py | 19 +- .../fluid/tests/unittests/test_lamb_op.py | 101 +- .../fluid/tests/unittests/test_lambv2_op.py | 136 +- .../tests/unittests/test_launch_coverage.py | 77 +- .../tests/unittests/test_layer_norm_op.py | 359 +- .../tests/unittests/test_layer_norm_op_v2.py | 12 +- .../fluid/tests/unittests/test_layers.py | 4576 +++++++----- .../tests/unittests/test_layout_autotune.py | 20 +- .../fluid/tests/unittests/test_lazy_init.py | 79 +- .../fluid/tests/unittests/test_lbfgs.py | 66 +- .../paddle/fluid/tests/unittests/test_lcm.py | 34 +- .../unittests/test_learning_rate_scheduler.py | 388 +- .../fluid/tests/unittests/test_lerp_op.py | 49 +- .../fluid/tests/unittests/test_lgamma_op.py | 10 +- .../unittests/test_limit_by_capacity_op.py | 55 +- .../fluid/tests/unittests/test_linalg_cond.py | 24 +- .../tests/unittests/test_linalg_lstsq_op.py | 171 +- .../tests/unittests/test_linalg_pinv_op.py | 106 +- .../fluid/tests/unittests/test_linear.py | 40 +- .../unittests/test_linear_chain_crf_op.py | 118 +- .../tests/unittests/test_linear_interp_op.py | 366 +- .../unittests/test_linear_interp_v2_op.py | 411 +- .../fluid/tests/unittests/test_linspace.py | 24 +- .../unittests/test_listen_and_serv_op.py | 44 +- .../fluid/tests/unittests/test_load_op.py | 22 +- .../fluid/tests/unittests/test_load_op_xpu.py | 19 +- .../test_load_state_dict_from_old_format.py | 117 +- .../unittests/test_load_vars_shape_check.py | 1 - .../unittests/test_locality_aware_nms_op.py | 417 +- .../tests/unittests/test_lod_append_op.py | 33 +- .../unittests/test_lod_array_length_op.py | 5 +- .../tests/unittests/test_lod_rank_table.py | 7 +- .../tests/unittests/test_lod_reset_op.py | 23 +- .../tests/unittests/test_lod_tensor_array.py | 10 +- .../unittests/test_lod_tensor_array_ops.py | 142 +- .../fluid/tests/unittests/test_log_loss_op.py | 7 +- .../fluid/tests/unittests/test_log_softmax.py | 52 +- .../tests/unittests/test_logcumsumexp_op.py | 90 +- .../fluid/tests/unittests/test_logical_op.py | 149 +- .../fluid/tests/unittests/test_logit_op.py | 32 +- .../fluid/tests/unittests/test_logspace.py | 38 +- .../fluid/tests/unittests/test_logsumexp.py | 37 +- .../fluid/tests/unittests/test_lookahead.py | 75 +- .../unittests/test_lookup_table_bf16_op.py | 132 +- .../unittests/test_lookup_table_dequant_op.py | 1 - .../tests/unittests/test_lookup_table_op.py | 109 +- .../unittests/test_lookup_table_v2_bf16_op.py | 50 +- .../unittests/test_lookup_table_v2_op.py | 71 +- .../tests/unittests/test_lr_scheduler.py | 818 +- .../fluid/tests/unittests/test_lrn_op.py | 190 +- .../tests/unittests/test_lstm_cudnn_op.py | 348 +- .../fluid/tests/unittests/test_lstm_op.py | 245 +- .../tests/unittests/test_lstm_unit_op.py | 66 +- .../fluid/tests/unittests/test_lstmp_op.py | 316 +- .../fluid/tests/unittests/test_lu_op.py | 67 +- .../tests/unittests/test_lu_unpack_op.py | 77 +- .../fluid/tests/unittests/test_manual_seed.py | 1 - .../unittests/test_margin_cross_entropy_op.py | 332 +- .../unittests/test_margin_rank_loss_op.py | 31 +- .../fluid/tests/unittests/test_marker_op.py | 3 +- .../tests/unittests/test_masked_select_op.py | 29 +- .../unittests/test_match_matrix_tensor_op.py | 24 +- .../tests/unittests/test_math_op_patch.py | 193 +- .../unittests/test_math_op_patch_var_base.py | 245 +- .../fluid/tests/unittests/test_matmul_op.py | 140 +- .../unittests/test_matmul_op_with_head.py | 124 +- .../tests/unittests/test_matmul_v2_op.py | 256 +- .../tests/unittests/test_matrix_nms_op.py | 262 +- .../tests/unittests/test_matrix_power_op.py | 86 +- .../tests/unittests/test_matrix_rank_op.py | 101 +- .../unittests/test_max_min_amax_amin_op.py | 50 +- .../fluid/tests/unittests/test_max_op.py | 40 +- .../fluid/tests/unittests/test_maximum_op.py | 57 +- .../fluid/tests/unittests/test_maxout_op.py | 26 +- .../fluid/tests/unittests/test_mean_iou.py | 73 +- .../fluid/tests/unittests/test_mean_op.py | 112 +- .../fluid/tests/unittests/test_median.py | 15 +- .../fluid/tests/unittests/test_memcpy_op.py | 151 +- .../tests/unittests/test_memory_analysis.py | 17 +- .../test_memory_reuse_exclude_feed_var.py | 16 +- .../tests/unittests/test_memory_usage.py | 16 +- .../unittests/test_merge_selectedrows_op.py | 1 - .../tests/unittests/test_merged_adam_op.py | 167 +- .../unittests/test_merged_momentum_op.py | 309 +- .../fluid/tests/unittests/test_meshgrid_op.py | 191 +- .../fluid/tests/unittests/test_min_op.py | 40 +- .../unittests/test_mine_hard_examples_op.py | 48 +- .../fluid/tests/unittests/test_minimum_op.py | 57 +- .../fluid/tests/unittests/test_minus_op.py | 3 +- .../test_mix_precision_all_reduce_fuse.py | 38 +- .../fluid/tests/unittests/test_mode_op.py | 41 +- .../tests/unittests/test_modelaverage.py | 136 +- .../unittests/test_modified_huber_loss_op.py | 24 +- .../fluid/tests/unittests/test_momentum_op.py | 396 +- .../fluid/tests/unittests/test_monitor.py | 24 +- .../fluid/tests/unittests/test_mse_loss.py | 239 +- .../fluid/tests/unittests/test_mul_nn_grad.py | 43 +- .../fluid/tests/unittests/test_mul_op.py | 123 +- .../tests/unittests/test_multi_dot_op.py | 100 +- .../test_multi_label_soft_margin_loss.py | 231 +- .../tests/unittests/test_multiclass_nms_op.py | 464 +- .../unittests/test_multihead_attention.py | 57 +- .../tests/unittests/test_multimarginloss.py | 342 +- .../tests/unittests/test_multinomial_op.py | 82 +- .../tests/unittests/test_multiplex_op.py | 28 +- .../fluid/tests/unittests/test_multiply.py | 32 +- .../test_multiprocess_dataloader_dataset.py | 171 +- .../test_multiprocess_dataloader_dynamic.py | 102 +- .../test_multiprocess_dataloader_exception.py | 104 +- ...ess_dataloader_iterable_dataset_dynamic.py | 103 +- ...ocess_dataloader_iterable_dataset_split.py | 37 +- ...cess_dataloader_iterable_dataset_static.py | 157 +- .../test_multiprocess_dataloader_static.py | 196 +- .../test_multiprocess_reader_exception.py | 44 +- .../fluid/tests/unittests/test_mv_op.py | 33 +- .../test_naive_best_fit_gpu_memory_limit.py | 7 +- .../fluid/tests/unittests/test_name_scope.py | 1 - .../fluid/tests/unittests/test_nan_inf.py | 18 +- .../fluid/tests/unittests/test_nanmean_api.py | 21 +- .../fluid/tests/unittests/test_nanmedian.py | 66 +- .../fluid/tests/unittests/test_nansum_api.py | 47 +- .../paddle/fluid/tests/unittests/test_nce.py | 274 +- .../tests/unittests/test_nearest_interp_op.py | 216 +- .../unittests/test_nearest_interp_v2_op.py | 365 +- .../fluid/tests/unittests/test_neg_op.py | 12 +- .../unittests/test_network_with_dtype.py | 6 +- .../fluid/tests/unittests/test_newprofiler.py | 153 +- .../unittests/test_newprofiler_helper.py | 1 - .../fluid/tests/unittests/test_nll_loss.py | 813 +- .../fluid/tests/unittests/test_nms_op.py | 4 +- .../tests/unittests/test_nn_dice_loss.py | 15 +- .../test_nn_functional_embedding_dygraph.py | 6 +- .../test_nn_functional_embedding_static.py | 94 +- .../unittests/test_nn_functional_hot_op.py | 86 +- .../fluid/tests/unittests/test_nn_grad.py | 307 +- .../unittests/test_nn_margin_rank_loss.py | 159 +- .../tests/unittests/test_nn_matmul_v2_grad.py | 428 +- .../test_nn_quant_functional_layers.py | 3 +- .../tests/unittests/test_nn_sigmoid_op.py | 2 - .../fluid/tests/unittests/test_nonzero_api.py | 25 +- .../fluid/tests/unittests/test_norm_all.py | 449 +- .../tests/unittests/test_norm_nn_grad.py | 126 +- .../fluid/tests/unittests/test_norm_op.py | 35 +- .../fluid/tests/unittests/test_normal.py | 68 +- .../unittests/test_normalization_wrapper.py | 55 +- .../fluid/tests/unittests/test_normalize.py | 10 +- .../tests/unittests/test_npair_loss_op.py | 191 +- .../tests/unittests/test_number_count_op.py | 19 +- .../fluid/tests/unittests/test_numel_op.py | 34 +- .../fluid/tests/unittests/test_one_hot_op.py | 85 +- .../tests/unittests/test_one_hot_v2_op.py | 91 +- .../fluid/tests/unittests/test_ones_like.py | 36 +- .../fluid/tests/unittests/test_ones_op.py | 11 +- .../fluid/tests/unittests/test_onnx_export.py | 17 +- .../unittests/test_op_function_generator.py | 5 +- .../tests/unittests/test_op_name_conflict.py | 95 +- .../tests/unittests/test_op_support_gpu.py | 6 +- .../fluid/tests/unittests/test_op_version.py | 22 +- .../fluid/tests/unittests/test_operator.py | 33 +- .../tests/unittests/test_operator_desc.py | 95 +- .../fluid/tests/unittests/test_ops_nms.py | 132 +- .../fluid/tests/unittests/test_optimizer.py | 1359 ++-- .../unittests/test_optimizer_for_varbase.py | 37 +- .../tests/unittests/test_optimizer_grad.py | 107 +- .../test_optimizer_in_control_flow.py | 128 +- .../fluid/tests/unittests/test_outer.py | 48 +- .../tests/unittests/test_overlap_add_op.py | 13 +- .../fluid/tests/unittests/test_pad2d_op.py | 51 +- .../fluid/tests/unittests/test_pad3d_op.py | 568 +- .../tests/unittests/test_pad_constant_like.py | 42 +- .../fluid/tests/unittests/test_pad_op.py | 55 +- .../test_paddle_imperative_double_grad.py | 192 +- .../unittests/test_paddle_multiprocessing.py | 35 +- .../tests/unittests/test_paddle_save_load.py | 504 +- .../unittests/test_paddle_save_load_binary.py | 130 +- .../tests/unittests/test_pairwise_distance.py | 451 +- .../test_parallel_dygraph_dataparallel.py | 89 +- ...t_parallel_dygraph_dataparallel_cpuonly.py | 43 +- ...graph_sparse_embedding_diff_length_gloo.py | 11 +- ..._parallel_dygraph_sparse_embedding_gloo.py | 22 +- ...graph_sparse_embedding_over_height_gloo.py | 4 +- .../test_parallel_dygraph_transformer_gloo.py | 22 +- ..._parallel_dygraph_unused_variables_gloo.py | 33 +- .../unittests/test_parallel_executor_crf.py | 257 +- .../test_parallel_executor_drop_scope.py | 21 +- .../test_parallel_executor_dry_run.py | 40 +- ..._parallel_executor_feed_persistable_var.py | 34 +- .../test_parallel_executor_fetch_feed.py | 122 +- ...st_parallel_executor_fetch_isolated_var.py | 45 +- ...test_parallel_executor_fix_op_run_order.py | 23 +- ...el_executor_inference_feed_partial_data.py | 82 +- .../unittests/test_parallel_executor_mnist.py | 146 +- .../unittests/test_parallel_executor_pg.py | 42 +- .../test_parallel_executor_profiler.py | 11 +- .../test_parallel_executor_run_cinn.py | 36 +- ...arallel_executor_run_load_infer_program.py | 63 +- ...st_parallel_executor_seresnext_base_cpu.py | 19 +- ...st_parallel_executor_seresnext_base_gpu.py | 19 +- ...utor_seresnext_with_fuse_all_reduce_cpu.py | 14 +- ...utor_seresnext_with_fuse_all_reduce_gpu.py | 15 +- ...llel_executor_seresnext_with_reduce_cpu.py | 39 +- ...llel_executor_seresnext_with_reduce_gpu.py | 11 +- ...test_parallel_executor_test_while_train.py | 58 +- .../test_parallel_executor_transformer.py | 143 +- .../fluid/tests/unittests/test_parameter.py | 64 +- .../tests/unittests/test_partial_concat_op.py | 20 +- .../tests/unittests/test_partial_sum_op.py | 14 +- .../tests/unittests/test_pipeline_parallel.py | 2 - .../tests/unittests/test_pixel_shuffle.py | 125 +- .../tests/unittests/test_pixel_unshuffle.py | 136 +- .../fluid/tests/unittests/test_poisson_op.py | 243 +- .../unittests/test_polygon_box_transform.py | 40 +- .../fluid/tests/unittests/test_pool1d_api.py | 343 +- .../fluid/tests/unittests/test_pool2d_api.py | 628 +- .../fluid/tests/unittests/test_pool2d_op.py | 904 ++- .../fluid/tests/unittests/test_pool3d_api.py | 547 +- .../fluid/tests/unittests/test_pool3d_op.py | 797 +- .../fluid/tests/unittests/test_pool_max_op.py | 50 +- .../test_positive_negative_pair_op.py | 44 +- .../paddle/fluid/tests/unittests/test_pow.py | 35 +- .../test_pow2_decay_with_linear_warmup_op.py | 27 +- .../unittests/test_precision_recall_op.py | 55 +- .../fluid/tests/unittests/test_prelu_op.py | 182 +- .../fluid/tests/unittests/test_print_op.py | 41 +- .../tests/unittests/test_prior_box_op.py | 110 +- .../fluid/tests/unittests/test_prod_op.py | 140 +- .../fluid/tests/unittests/test_profiler.py | 139 +- .../unittests/test_profiler_statistic.py | 1035 ++- .../fluid/tests/unittests/test_program.py | 103 +- .../tests/unittests/test_program_code.py | 21 +- .../unittests/test_program_prune_backward.py | 161 +- .../tests/unittests/test_program_to_string.py | 13 +- .../fluid/tests/unittests/test_protobuf.py | 1 - .../tests/unittests/test_protobuf_descs.py | 9 +- .../unittests/test_proximal_adagrad_op.py | 1 - .../tests/unittests/test_proximal_gd_op.py | 1 - .../tests/unittests/test_prroi_pool_op.py | 212 +- .../fluid/tests/unittests/test_prune.py | 694 +- .../test_prune_gate_by_capacity_op.py | 68 +- .../tests/unittests/test_ps_dispatcher.py | 17 +- .../tests/unittests/test_psroi_pool_op.py | 242 +- .../unittests/test_pull_gpups_sparse_op.py | 20 +- .../tests/unittests/test_put_along_axis_op.py | 61 +- .../fluid/tests/unittests/test_py_func_op.py | 134 +- .../unittests/test_py_reader_combination.py | 43 +- .../unittests/test_py_reader_error_msg.py | 44 +- .../test_py_reader_lod_level_share.py | 5 +- .../unittests/test_py_reader_pin_memory.py | 52 +- .../unittests/test_py_reader_push_pop.py | 44 +- .../unittests/test_py_reader_return_list.py | 36 +- .../test_py_reader_sample_generator.py | 40 +- .../test_py_reader_using_executor.py | 130 +- .../fluid/tests/unittests/test_pylayer_op.py | 137 +- .../tests/unittests/test_pyramid_hash_op.py | 11 +- .../test_python_bf16_numpy_datatype.py | 1 - .../fluid/tests/unittests/test_qr_op.py | 72 +- .../test_quantile_and_nanquantile.py | 81 +- .../fluid/tests/unittests/test_query_op.py | 1 - .../fluid/tests/unittests/test_queue.py | 67 +- .../fluid/tests/unittests/test_rad2deg.py | 22 +- .../fluid/tests/unittests/test_rand_op.py | 20 +- .../tests/unittests/test_randint_like.py | 210 +- .../fluid/tests/unittests/test_randint_op.py | 75 +- .../fluid/tests/unittests/test_randn_op.py | 33 +- .../tests/unittests/test_random_crop_op.py | 21 +- .../tests/unittests/test_random_routing_op.py | 24 +- .../fluid/tests/unittests/test_random_seed.py | 171 +- .../fluid/tests/unittests/test_randperm_op.py | 183 +- .../fluid/tests/unittests/test_range.py | 13 +- .../tests/unittests/test_rank_attention_op.py | 61 +- .../tests/unittests/test_rank_loss_op.py | 9 +- .../unittests/test_raw_program_optimizer.py | 31 +- .../tests/unittests/test_reader_reset.py | 29 +- .../tests/unittests/test_real_imag_op.py | 46 +- .../tests/unittests/test_recurrent_op.py | 261 +- .../fluid/tests/unittests/test_reduce_op.py | 298 +- .../tests/unittests/test_reducescatter.py | 6 +- .../tests/unittests/test_reducescatter_api.py | 1 - .../fluid/tests/unittests/test_registry.py | 1 - .../fluid/tests/unittests/test_regularizer.py | 204 +- .../tests/unittests/test_regularizer_api.py | 109 +- .../fluid/tests/unittests/test_renorm_op.py | 78 +- .../unittests/test_reorder_lod_tensor.py | 124 +- .../unittests/test_repeat_interleave_op.py | 76 +- .../tests/unittests/test_require_version.py | 56 +- .../test_reset_grad_inplace_version.py | 16 +- .../fluid/tests/unittests/test_reshape_op.py | 105 +- .../unittests/test_resnet50_with_cinn.py | 45 +- .../tests/unittests/test_retain_graph.py | 79 +- .../test_retinanet_detection_output.py | 393 +- .../fluid/tests/unittests/test_reverse_op.py | 93 +- .../fluid/tests/unittests/test_rmsprop_op.py | 251 +- .../tests/unittests/test_rnn_cell_api.py | 633 +- .../tests/unittests/test_rnn_decode_api.py | 707 +- .../unittests/test_rnn_memory_helper_op.py | 101 +- .../fluid/tests/unittests/test_rnn_op.py | 81 +- .../tests/unittests/test_roi_align_op.py | 140 +- .../test_roi_perspective_transform_op.py | 235 +- .../fluid/tests/unittests/test_roi_pool_op.py | 83 +- .../fluid/tests/unittests/test_roll_op.py | 53 +- .../fluid/tests/unittests/test_rot90_op.py | 99 +- .../fluid/tests/unittests/test_row_conv_op.py | 83 +- .../unittests/test_rpn_target_assign_op.py | 695 +- .../fluid/tests/unittests/test_rrelu_op.py | 225 +- .../paddle/fluid/tests/unittests/test_run.py | 18 +- ...est_run_fluid_by_module_or_command_line.py | 2 - .../tests/unittests/test_run_program_op.py | 229 +- .../test_runtime_and_compiletime_exception.py | 9 +- .../tests/unittests/test_sample_logits_op.py | 68 +- .../tests/unittests/test_sampling_id_op.py | 1 - ...est_save_inference_model_conditional_op.py | 74 +- .../unittests/test_save_model_without_var.py | 22 +- .../unittests/test_saved_tensors_hooks.py | 4 - .../fluid/tests/unittests/test_scale_op.py | 70 +- .../test_scaled_dot_product_attention.py | 96 +- .../tests/unittests/test_scatter_nd_op.py | 226 +- .../fluid/tests/unittests/test_scatter_op.py | 145 +- .../fluid/tests/unittests/test_scope.py | 4 +- .../tests/unittests/test_searchsorted_op.py | 130 +- .../fluid/tests/unittests/test_seed_op.py | 9 +- .../fluid/tests/unittests/test_segment_ops.py | 91 +- .../unittests/test_select_input_output_op.py | 48 +- .../tests/unittests/test_selected_rows.py | 14 +- .../fluid/tests/unittests/test_selu_op.py | 32 +- .../tests/unittests/test_set_bool_attr.py | 13 +- .../tests/unittests/test_set_value_op.py | 457 +- .../fluid/tests/unittests/test_sgd_op.py | 229 +- .../fluid/tests/unittests/test_sgd_op_bf16.py | 203 +- .../paddle/fluid/tests/unittests/test_sgn.py | 27 +- .../fluid/tests/unittests/test_shape_op.py | 4 - .../tests/unittests/test_shard_index_op.py | 6 +- .../tests/unittests/test_share_data_op.py | 2 - .../tests/unittests/test_shrink_rnn_memory.py | 37 +- .../tests/unittests/test_shuffle_batch_op.py | 7 +- .../unittests/test_shuffle_channel_op.py | 10 +- ...st_sigmoid_cross_entropy_with_logits_op.py | 155 +- .../unittests/test_sigmoid_focal_loss.py | 194 +- .../unittests/test_sigmoid_focal_loss_op.py | 132 +- .../fluid/tests/unittests/test_sign_op.py | 75 +- .../fluid/tests/unittests/test_signal.py | 370 +- .../unittests/test_similarity_focus_op.py | 80 +- .../tests/unittests/test_simple_rnn_op.py | 53 +- .../fluid/tests/unittests/test_size_op.py | 34 +- .../fluid/tests/unittests/test_slice_op.py | 346 +- .../fluid/tests/unittests/test_slice_var.py | 43 +- .../tests/unittests/test_smooth_l1_loss.py | 109 +- .../tests/unittests/test_smooth_l1_loss_op.py | 83 +- .../tests/unittests/test_soft_margin_loss.py | 133 +- .../fluid/tests/unittests/test_softmax2d.py | 33 +- .../unittests/test_softmax_mask_fuse_op.py | 35 +- ...est_softmax_mask_fuse_upper_triangle_op.py | 31 +- .../fluid/tests/unittests/test_softmax_op.py | 178 +- .../test_softmax_with_cross_entropy_op.py | 294 +- .../fluid/tests/unittests/test_solve_op.py | 242 +- .../fluid/tests/unittests/test_sort_op.py | 32 +- .../tests/unittests/test_space_to_depth_op.py | 37 +- .../tests/unittests/test_sparse_addmm_op.py | 40 +- .../unittests/test_sparse_attention_op.py | 220 +- .../tests/unittests/test_sparse_conv_op.py | 192 +- .../tests/unittests/test_sparse_copy_op.py | 1 - .../unittests/test_sparse_elementwise_op.py | 122 +- .../test_sparse_fused_attention_op.py | 76 +- .../unittests/test_sparse_is_same_shape.py | 149 +- .../tests/unittests/test_sparse_matmul_op.py | 86 +- .../tests/unittests/test_sparse_model.py | 11 +- .../unittests/test_sparse_momentum_op.py | 126 +- .../tests/unittests/test_sparse_mv_op.py | 46 +- .../tests/unittests/test_sparse_norm_op.py | 106 +- .../tests/unittests/test_sparse_pooling_op.py | 34 +- .../tests/unittests/test_sparse_reshape_op.py | 57 +- .../tests/unittests/test_sparse_softmax_op.py | 43 +- .../unittests/test_sparse_transpose_op.py | 19 +- .../tests/unittests/test_sparse_unary_op.py | 45 +- .../tests/unittests/test_sparse_utils_op.py | 224 +- .../test_spawn_and_init_parallel_env.py | 18 +- .../tests/unittests/test_spectral_norm_op.py | 43 +- .../test_split_and_merge_lod_tensor_op.py | 241 +- .../fluid/tests/unittests/test_split_op.py | 169 +- .../tests/unittests/test_split_program.py | 59 +- .../fluid/tests/unittests/test_splits_api.py | 50 +- .../fluid/tests/unittests/test_spp_op.py | 32 +- .../tests/unittests/test_square_error_cost.py | 19 +- .../unittests/test_squared_l2_distance_op.py | 15 +- .../unittests/test_squared_l2_norm_op.py | 14 +- .../fluid/tests/unittests/test_squeeze2_op.py | 10 +- .../fluid/tests/unittests/test_squeeze_op.py | 66 +- .../fluid/tests/unittests/test_stack_op.py | 81 +- ...t_static_model_parallel_fused_attention.py | 12 +- ...static_model_parallel_fused_feedforward.py | 12 +- ..._model_parallel_fused_multi_transformer.py | 5 +- .../tests/unittests/test_static_save_load.py | 1240 +-- .../unittests/test_static_save_load_bf16.py | 97 +- .../unittests/test_static_save_load_large.py | 39 +- ...tatic_shape_inferrence_for_shape_tensor.py | 7 +- .../fluid/tests/unittests/test_std_layer.py | 17 +- .../fluid/tests/unittests/test_stft_op.py | 13 +- .../tests/unittests/test_strided_slice_op.py | 341 +- .../fluid/tests/unittests/test_subtract_op.py | 57 +- .../fluid/tests/unittests/test_sum_op.py | 257 +- .../fluid/tests/unittests/test_svd_op.py | 201 +- .../fluid/tests/unittests/test_switch.py | 24 +- .../tests/unittests/test_switch_autotune.py | 47 +- .../fluid/tests/unittests/test_switch_case.py | 319 +- .../unittests/test_sync_batch_norm_op.py | 164 +- .../paddle/fluid/tests/unittests/test_take.py | 144 +- .../unittests/test_take_along_axis_op.py | 28 +- .../tests/unittests/test_target_assign_op.py | 65 +- .../tests/unittests/test_tdm_child_op.py | 33 +- .../tests/unittests/test_tdm_sampler_op.py | 109 +- .../test_teacher_student_sigmoid_loss_op.py | 37 +- .../tests/unittests/test_temporal_shift_op.py | 53 +- .../fluid/tests/unittests/test_tensor.py | 60 +- .../unittests/test_tensor_array_to_tensor.py | 96 +- .../tests/unittests/test_tensor_copy_from.py | 1 - .../tests/unittests/test_tensor_fill_.py | 16 +- .../unittests/test_tensor_fill_diagonal_.py | 134 +- .../test_tensor_fill_diagonal_tensor.py | 125 +- .../test_tensor_fill_diagonal_tensor_.py | 127 +- .../unittests/test_tensor_register_hook.py | 184 +- ...st_tensor_scalar_type_promotion_dynamic.py | 1 - ...est_tensor_scalar_type_promotion_static.py | 1 - .../tests/unittests/test_tensor_to_list.py | 6 +- .../tests/unittests/test_tensor_to_numpy.py | 12 +- .../unittests/test_tensor_type_promotion.py | 21 +- .../fluid/tests/unittests/test_tensor_uva.py | 18 +- .../tests/unittests/test_tensor_zero_.py | 6 +- .../fluid/tests/unittests/test_tensordot.py | 228 +- .../fluid/tests/unittests/test_tf32_cublas.py | 2 - .../fluid/tests/unittests/test_tf32_cudnn.py | 1 - .../fluid/tests/unittests/test_tile_op.py | 68 +- .../fluid/tests/unittests/test_top_k_op.py | 1 - .../fluid/tests/unittests/test_top_k_v2_op.py | 299 +- .../fluid/tests/unittests/test_trace_op.py | 34 +- .../unittests/test_traced_layer_err_msg.py | 145 +- .../fluid/tests/unittests/test_trainable.py | 36 +- .../tests/unittests/test_trainer_desc.py | 2 +- .../tests/unittests/test_transfer_dtype_op.py | 15 +- .../unittests/test_transfer_layout_op.py | 42 +- .../tests/unittests/test_transformer_api.py | 934 ++- .../tests/unittests/test_translated_layer.py | 71 +- .../tests/unittests/test_transpose_op.py | 121 +- .../tests/unittests/test_tree_conv_op.py | 147 +- .../unittests/test_triangular_solve_op.py | 62 +- .../tests/unittests/test_tril_indices_op.py | 48 +- .../tests/unittests/test_tril_triu_op.py | 78 +- .../unittests/test_trilinear_interp_op.py | 297 +- .../unittests/test_trilinear_interp_v2_op.py | 364 +- .../unittests/test_triplet_margin_loss.py | 423 +- .../test_triplet_margin_with_distance_loss.py | 292 +- .../tests/unittests/test_triu_indices_op.py | 28 +- .../fluid/tests/unittests/test_trunc_op.py | 4 - .../test_truncated_gaussian_random_op.py | 24 +- .../fluid/tests/unittests/test_unbind_op.py | 44 +- .../fluid/tests/unittests/test_unfold_op.py | 77 +- .../unittests/test_uniform_random_bf16_op.py | 129 +- .../test_uniform_random_inplace_op.py | 24 +- .../tests/unittests/test_uniform_random_op.py | 274 +- .../fluid/tests/unittests/test_unique.py | 161 +- .../unittests/test_unique_consecutive_op.py | 70 +- .../fluid/tests/unittests/test_unique_name.py | 2 - .../unittests/test_unique_with_counts.py | 49 +- .../fluid/tests/unittests/test_unpool1d_op.py | 143 +- .../fluid/tests/unittests/test_unpool3d_op.py | 312 +- .../fluid/tests/unittests/test_unpool_op.py | 297 +- .../tests/unittests/test_unsqueeze2_op.py | 64 +- .../tests/unittests/test_unsqueeze_op.py | 112 +- .../fluid/tests/unittests/test_unstack_op.py | 5 - .../unittests/test_update_loss_scaling_op.py | 199 +- .../fluid/tests/unittests/test_var_base.py | 403 +- .../fluid/tests/unittests/test_var_conv_2d.py | 138 +- .../fluid/tests/unittests/test_var_info.py | 4 +- .../fluid/tests/unittests/test_variable.py | 558 +- .../tests/unittests/test_variance_layer.py | 17 +- .../fluid/tests/unittests/test_version.py | 4 +- .../test_view_op_reuse_allocation.py | 20 +- .../tests/unittests/test_viterbi_decode_op.py | 35 +- .../fluid/tests/unittests/test_warpctc_op.py | 405 +- .../tests/unittests/test_weight_decay.py | 129 +- .../unittests/test_weight_normalization.py | 72 +- .../fluid/tests/unittests/test_where_index.py | 25 +- .../fluid/tests/unittests/test_where_op.py | 147 +- .../tests/unittests/test_while_loop_op.py | 286 +- .../fluid/tests/unittests/test_while_op.py | 61 +- .../fluid/tests/unittests/test_yolo_box_op.py | 195 +- .../tests/unittests/test_yolov3_loss_op.py | 308 +- .../tests/unittests/test_zero_dim_shape.py | 38 +- .../fluid/tests/unittests/test_zeropad2d.py | 30 +- .../tests/unittests/test_zeros_like_op.py | 48 +- .../fluid/tests/unittests/test_zeros_op.py | 14 +- .../paddle/fluid/tests/unittests/testsuite.py | 51 +- .../unittests/tokenizer/bert_tokenizer.py | 182 +- .../unittests/tokenizer/tokenizer_utils.py | 505 +- .../tests/unittests/transformer_model.py | 370 +- python/paddle/fluid/tests/unittests/utils.py | 2 - .../white_list/check_shape_white_list.py | 21 +- .../compile_vs_runtime_white_list.py | 32 +- .../white_list/no_grad_set_white_list.py | 2 - .../white_list/op_accuracy_white_list.py | 138 +- .../white_list/op_threshold_white_list.py | 85 +- .../xpu/collective_allgather_op_xpu.py | 33 +- .../xpu/collective_allreduce_op_xpu.py | 34 +- .../xpu/collective_identity_op_xpu.py | 23 +- .../unittests/xpu/get_test_cover_info.py | 80 +- .../unittests/xpu/test_accuracy_op_xpu.py | 15 +- .../unittests/xpu/test_activation_op_xpu.py | 177 +- .../tests/unittests/xpu/test_adam_op_xpu.py | 162 +- .../tests/unittests/xpu/test_adamw_op_xpu.py | 115 +- .../xpu/test_affine_channel_op_xpu.py | 34 +- .../test_amp_check_finite_and_scale_op_xpu.py | 11 +- .../unittests/xpu/test_arg_max_op_xpu.py | 51 +- .../unittests/xpu/test_argsort_op_xpu.py | 39 +- .../tests/unittests/xpu/test_assign_op_xpu.py | 1 + .../unittests/xpu/test_assign_value_op_xpu.py | 31 +- .../unittests/xpu/test_batch_norm_op_xpu.py | 230 +- .../unittests/xpu/test_bce_loss_op_xpu.py | 17 +- .../xpu/test_bilinear_interp_v2_op_xpu.py | 157 +- .../unittests/xpu/test_bitwise_op_xpu.py | 79 +- .../tests/unittests/xpu/test_bmm_op_xpu.py | 12 +- .../unittests/xpu/test_c_embedding_op_xpu.py | 6 +- .../tests/unittests/xpu/test_cast_op_xpu.py | 21 +- .../unittests/xpu/test_clip_by_norm_op_xpu.py | 8 +- .../tests/unittests/xpu/test_clip_op_xpu.py | 37 +- .../xpu/test_coalesce_tensor_op_xpu.py | 50 +- .../xpu/test_collective_allgather_xpu.py | 23 +- .../xpu/test_collective_allreduce_xpu.py | 23 +- .../unittests/xpu/test_collective_base_xpu.py | 240 +- .../xpu/test_collective_identity_xpu.py | 23 +- .../unittests/xpu/test_compare_op_xpu.py | 47 +- .../tests/unittests/xpu/test_concat_op_xpu.py | 33 +- .../tests/unittests/xpu/test_conv2d_op_xpu.py | 289 +- .../xpu/test_conv2d_transpose_op_xpu.py | 96 +- .../xpu/test_deformable_conv_op_xpu.py | 211 +- .../xpu/test_depthwise_conv2d_op_xpu.py | 24 +- .../unittests/xpu/test_device_guard_xpu.py | 99 +- .../unittests/xpu/test_dropout_op_xpu.py | 76 +- .../xpu/test_elementwise_add_op_xpu.py | 98 +- .../xpu/test_elementwise_add_op_xpu_kp.py | 169 +- .../xpu/test_elementwise_div_op_xpu.py | 132 +- .../xpu/test_elementwise_floordiv_op_xpu.py | 12 +- .../xpu/test_elementwise_max_op_xpu.py | 101 +- .../xpu/test_elementwise_min_op_xpu.py | 101 +- .../xpu/test_elementwise_mod_op_xpu.py | 19 +- .../xpu/test_elementwise_mul_op_xpu.py | 102 +- .../xpu/test_elementwise_pow_op_xpu.py | 71 +- .../xpu/test_elementwise_sub_op_xpu.py | 70 +- .../tests/unittests/xpu/test_empty_op_xpu.py | 44 +- .../unittests/xpu/test_expand_as_v2_op_xpu.py | 42 +- .../unittests/xpu/test_expand_v2_op_xpu.py | 63 +- .../xpu/test_fill_any_like_op_xpu.py | 12 +- .../xpu/test_fill_constant_op_xpu.py | 79 +- .../unittests/xpu/test_flatten2_op_xpu.py | 13 +- .../test_flatten_contiguous_range_op_xpu.py | 109 +- .../unittests/xpu/test_flatten_op_xpu.py | 11 +- .../xpu/test_fleet_exe_dist_model_run_xpu.py | 35 +- .../test_fused_gemm_epilogue_grad_op_xpu.py | 39 +- .../xpu/test_fused_gemm_epilogue_op_xpu.py | 76 +- .../test_fused_resnet_basic_block_op_xpu.py | 225 +- .../unittests/xpu/test_gather_nd_op_xpu.py | 55 +- .../tests/unittests/xpu/test_gather_op_xpu.py | 21 +- .../xpu/test_gaussian_random_op_xpu.py | 163 +- .../unittests/xpu/test_gen_bkcl_id_op.py | 18 +- .../xpu/test_generate_proposals_v2_op_xpu.py | 287 +- .../unittests/xpu/test_grid_sampler_op_xpu.py | 89 +- .../unittests/xpu/test_huber_loss_op_xpu.py | 36 +- .../unittests/xpu/test_index_select_op_xpu.py | 86 +- .../xpu/test_instance_norm_op_xpu.py | 24 +- .../xpu/test_iou_similarity_op_xpu.py | 20 +- .../unittests/xpu/test_label_smooth_op_xpu.py | 32 +- .../tests/unittests/xpu/test_lamb_op_xpu.py | 61 +- .../unittests/xpu/test_layer_norm_op_xpu.py | 37 +- .../unittests/xpu/test_log_loss_op_xpu.py | 6 +- .../unittests/xpu/test_log_softmax_op_xpu.py | 29 +- .../unittests/xpu/test_logical_op_xpu.py | 72 +- .../unittests/xpu/test_logsumexp_op_xpu.py | 10 +- .../xpu/test_lookup_table_v2_op_xpu.py | 60 +- .../xpu/test_masked_select_op_xpu.py | 36 +- .../tests/unittests/xpu/test_matmul_op_xpu.py | 228 +- .../unittests/xpu/test_matmul_v2_op_xpu.py | 33 +- .../tests/unittests/xpu/test_mean_op_xpu.py | 27 +- .../xpu/test_merged_momentum_op_xpu.py | 12 +- .../xpu/test_merged_momentum_op_xpu_base.py | 160 +- .../unittests/xpu/test_momentum_op_xpu.py | 48 +- .../tests/unittests/xpu/test_mul_op_xpu.py | 86 +- .../xpu/test_nearest_interp_v2_op_xpu.py | 143 +- .../unittests/xpu/test_one_hot_op_xpu.py | 46 +- .../unittests/xpu/test_one_hot_v2_op_xpu.py | 76 +- .../tests/unittests/xpu/test_p_norm_op_xpu.py | 25 +- .../tests/unittests/xpu/test_pool2d_op_xpu.py | 221 +- ...st_pow2_decay_with_linear_warmup_op_xpu.py | 27 +- .../unittests/xpu/test_prior_box_op_xpu.py | 104 +- .../tests/unittests/xpu/test_range_xpu.py | 21 +- .../unittests/xpu/test_reduce_all_op_xpu.py | 27 +- .../unittests/xpu/test_reduce_amax_op_xpu.py | 19 +- .../unittests/xpu/test_reduce_amin_op_xpu.py | 19 +- .../unittests/xpu/test_reduce_any_op_xpu.py | 24 +- .../unittests/xpu/test_reduce_max_op_xpu.py | 21 +- .../unittests/xpu/test_reduce_mean_op_xpu.py | 46 +- .../unittests/xpu/test_reduce_min_op_xpu.py | 21 +- .../unittests/xpu/test_reduce_prod_op_xpu.py | 51 +- .../unittests/xpu/test_reduce_sum_op_xpu.py | 21 +- .../unittests/xpu/test_refactor_op_xpu.py | 111 +- .../unittests/xpu/test_reshape2_op_xpu.py | 42 +- .../unittests/xpu/test_rmsprop_op_xpu.py | 57 +- .../tests/unittests/xpu/test_rnn_op_xpu.py | 94 +- .../unittests/xpu/test_roi_align_op_xpu.py | 139 +- .../unittests/xpu/test_sampling_id_op_xpu.py | 1 - .../tests/unittests/xpu/test_scale_op_xpu.py | 17 +- .../unittests/xpu/test_scatter_op_xpu.py | 40 +- .../xpu/test_sequence_conv_op_xpu.py | 302 +- .../xpu/test_sequence_unpad_op_xpu.py | 23 +- .../tests/unittests/xpu/test_sgd_op_xpu.py | 16 +- .../tests/unittests/xpu/test_shape_op_xpu.py | 14 +- ...igmoid_cross_entropy_with_logits_op_xpu.py | 152 +- .../tests/unittests/xpu/test_sign_op_xpu.py | 17 +- .../tests/unittests/xpu/test_slice_op_xpu.py | 39 +- .../unittests/xpu/test_softmax_op_xpu.py | 13 +- .../test_softmax_with_cross_entropy_op_xpu.py | 81 +- .../tests/unittests/xpu/test_split_op_xpu.py | 17 +- .../unittests/xpu/test_squeeze2_op_xpu.py | 13 +- .../unittests/xpu/test_squeeze_op_xpu.py | 18 +- .../tests/unittests/xpu/test_stack_op_xpu.py | 24 +- .../tests/unittests/xpu/test_sum_op_xpu.py | 35 +- .../xpu/test_temporal_shift_op_xpu.py | 30 +- .../tests/unittests/xpu/test_tile_op_xpu.py | 31 +- .../tests/unittests/xpu/test_top_k_op_xpu.py | 23 +- .../unittests/xpu/test_top_k_v2_op_xpu.py | 29 +- .../unittests/xpu/test_transpose_op_xpu.py | 20 +- .../unittests/xpu/test_tril_triu_op_xpu.py | 59 +- .../test_truncated_gaussian_random_op_xpu.py | 18 +- .../xpu/test_uniform_random_op_xpu.py | 7 +- .../unittests/xpu/test_unsqueeze2_op_xpu.py | 43 +- .../unittests/xpu/test_unsqueeze_op_xpu.py | 14 +- .../xpu/test_update_loss_scaling_op_xpu.py | 160 +- .../unittests/xpu/test_where_index_xpu.py | 36 +- .../tests/unittests/xpu/test_where_op_xpu.py | 64 +- .../tests/unittests/xpu/test_while_op_xpu.py | 32 +- .../tests/unittests/xpu/test_xpu_place.py | 1 - python/paddle/fluid/trainer_desc.py | 104 +- python/paddle/fluid/trainer_factory.py | 86 +- python/paddle/fluid/transpiler/__init__.py | 5 +- .../fluid/transpiler/ascend_transpiler.py | 43 +- python/paddle/fluid/transpiler/collective.py | 688 +- .../fluid/transpiler/details/checkport.py | 10 +- .../paddle/fluid/transpiler/details/ufind.py | 2 +- .../transpiler/details/vars_distributed.py | 128 +- .../fluid/transpiler/distribute_transpiler.py | 1361 ++-- .../fluid/transpiler/geo_sgd_transpiler.py | 179 +- .../memory_optimization_transpiler.py | 21 +- .../paddle/fluid/transpiler/ps_dispatcher.py | 4 +- python/paddle/fluid/unique_name.py | 1 + python/paddle/fluid/variable_index.py | 312 +- python/paddle/fluid/wrapped_decorator.py | 1 - python/paddle/framework/__init__.py | 15 +- python/paddle/framework/framework.py | 14 +- python/paddle/framework/io.py | 258 +- python/paddle/framework/random.py | 2 +- python/paddle/geometric/math.py | 132 +- .../geometric/message_passing/send_recv.py | 279 +- .../paddle/geometric/message_passing/utils.py | 41 +- python/paddle/geometric/reindex.py | 152 +- python/paddle/geometric/sampling/neighbors.py | 108 +- python/paddle/hapi/callbacks.py | 216 +- python/paddle/hapi/dynamic_flops.py | 69 +- python/paddle/hapi/hub.py | 67 +- python/paddle/hapi/model.py | 721 +- python/paddle/hapi/model_summary.py | 97 +- python/paddle/hapi/progressbar.py | 74 +- python/paddle/hapi/static_flops.py | 12 +- python/paddle/hub.py | 4 +- python/paddle/incubate/__init__.py | 8 +- python/paddle/incubate/asp/__init__.py | 19 +- python/paddle/incubate/autograd/__init__.py | 10 +- python/paddle/incubate/autograd/functional.py | 116 +- python/paddle/incubate/autograd/primapi.py | 54 +- python/paddle/incubate/autograd/primops.py | 263 +- python/paddle/incubate/autograd/primreg.py | 43 +- python/paddle/incubate/autograd/primrules.py | 437 +- python/paddle/incubate/autograd/primx.py | 103 +- python/paddle/incubate/autograd/utils.py | 3 +- python/paddle/incubate/autotune.py | 3 +- .../incubate/distributed/fleet/__init__.py | 5 +- .../distributed/models/moe/gate/base_gate.py | 1 - .../models/moe/gate/gshard_gate.py | 57 +- .../distributed/models/moe/gate/naive_gate.py | 9 +- .../models/moe/gate/switch_gate.py | 47 +- .../distributed/models/moe/grad_clip.py | 76 +- .../distributed/models/moe/moe_layer.py | 236 +- .../incubate/distributed/models/moe/utils.py | 42 +- .../incubate/multiprocessing/reductions.py | 55 +- python/paddle/incubate/nn/__init__.py | 6 +- .../nn/functional/fused_matmul_bias.py | 32 +- .../nn/functional/fused_transformer.py | 738 +- .../paddle/incubate/nn/layer/fused_linear.py | 35 +- .../incubate/nn/layer/fused_transformer.py | 566 +- python/paddle/incubate/operators/__init__.py | 14 +- .../incubate/operators/graph_khop_sampler.py | 125 +- .../incubate/operators/graph_reindex.py | 94 +- .../operators/graph_sample_neighbors.py | 117 +- .../incubate/operators/graph_send_recv.py | 109 +- .../paddle/incubate/operators/resnet_unit.py | 297 +- .../incubate/operators/softmax_mask_fuse.py | 11 +- .../softmax_mask_fuse_upper_triangle.py | 8 +- .../optimizer/distributed_fused_lamb.py | 272 +- .../incubate/optimizer/functional/bfgs.py | 72 +- .../incubate/optimizer/functional/lbfgs.py | 147 +- .../optimizer/functional/line_search.py | 165 +- .../incubate/optimizer/functional/utils.py | 27 +- python/paddle/incubate/optimizer/lookahead.py | 66 +- .../paddle/incubate/optimizer/modelaverage.py | 153 +- .../incubate/passes/fuse_resnet_unit_pass.py | 119 +- python/paddle/incubate/tensor/math.py | 168 +- python/paddle/incubate/xpu/resnet_block.py | 545 +- python/paddle/inference/__init__.py | 18 +- python/paddle/io/__init__.py | 21 +- python/paddle/jit/__init__.py | 11 +- .../paddle/jit/dy2static/convert_call_func.py | 4 +- .../paddle/jit/dy2static/convert_operators.py | 32 +- .../jit/dy2static/variable_trans_func.py | 6 +- python/paddle/jit/layer.py | 7 +- python/paddle/linalg.py | 4 +- python/paddle/metric/__init__.py | 9 +- python/paddle/metric/metrics.py | 86 +- python/paddle/nn/__init__.py | 36 +- python/paddle/nn/functional/activation.py | 466 +- python/paddle/nn/functional/common.py | 836 ++- python/paddle/nn/functional/conv.py | 912 ++- python/paddle/nn/functional/distance.py | 34 +- python/paddle/nn/functional/extension.py | 123 +- python/paddle/nn/functional/input.py | 89 +- python/paddle/nn/functional/loss.py | 1617 ++-- python/paddle/nn/functional/norm.py | 428 +- python/paddle/nn/functional/pooling.py | 1507 ++-- .../paddle/nn/functional/sparse_attention.py | 36 +- python/paddle/nn/functional/vision.py | 201 +- python/paddle/nn/initializer/__init__.py | 19 +- python/paddle/nn/initializer/assign.py | 14 +- python/paddle/nn/initializer/dirac.py | 221 +- python/paddle/nn/initializer/kaiming.py | 24 +- python/paddle/nn/initializer/orthogonal.py | 218 +- python/paddle/nn/initializer/uniform.py | 9 +- python/paddle/nn/initializer/xavier.py | 14 +- python/paddle/nn/layer/activation.py | 74 +- python/paddle/nn/layer/common.py | 499 +- python/paddle/nn/layer/container.py | 18 +- python/paddle/nn/layer/conv.py | 628 +- python/paddle/nn/layer/distance.py | 7 +- python/paddle/nn/layer/loss.py | 482 +- python/paddle/nn/layer/norm.py | 647 +- python/paddle/nn/layer/pooling.py | 386 +- python/paddle/nn/layer/rnn.py | 576 +- python/paddle/nn/layer/transformer.py | 448 +- python/paddle/nn/layer/vision.py | 33 +- python/paddle/nn/quant/functional_layers.py | 9 - python/paddle/nn/quant/lsq.py | 247 +- python/paddle/nn/quant/quant_layers.py | 768 +- python/paddle/nn/utils/__init__.py | 15 +- python/paddle/nn/utils/spectral_norm_hook.py | 56 +- .../paddle/nn/utils/transform_parameters.py | 51 +- python/paddle/nn/utils/weight_norm_hook.py | 32 +- python/paddle/onnx/export.py | 15 +- python/paddle/optimizer/__init__.py | 14 +- python/paddle/optimizer/adadelta.py | 85 +- python/paddle/optimizer/adagrad.py | 75 +- python/paddle/optimizer/adam.py | 432 +- python/paddle/optimizer/adamax.py | 161 +- python/paddle/optimizer/adamw.py | 355 +- python/paddle/optimizer/lamb.py | 227 +- python/paddle/optimizer/lr.py | 480 +- python/paddle/optimizer/momentum.py | 361 +- python/paddle/optimizer/optimizer.py | 552 +- python/paddle/optimizer/rmsprop.py | 111 +- python/paddle/optimizer/sgd.py | 107 +- python/paddle/profiler/__init__.py | 13 +- python/paddle/profiler/profiler.py | 242 +- python/paddle/profiler/profiler_statistic.py | 979 ++- python/paddle/profiler/statistic_helper.py | 27 +- python/paddle/profiler/timer.py | 95 +- python/paddle/profiler/utils.py | 36 +- python/paddle/quantization/__init__.py | 41 +- python/paddle/reader/decorator.py | 48 +- python/paddle/reader/tests/decorator_test.py | 37 +- python/paddle/signal.py | 290 +- python/paddle/sparse/binary.py | 7 +- python/paddle/sparse/creation.py | 83 +- .../paddle/sparse/nn/functional/activation.py | 7 +- python/paddle/sparse/nn/functional/conv.py | 155 +- python/paddle/sparse/nn/functional/pooling.py | 32 +- .../sparse/nn/functional/transformer.py | 25 +- python/paddle/sparse/nn/layer/conv.py | 199 +- python/paddle/sparse/nn/layer/norm.py | 180 +- python/paddle/sparse/nn/layer/pooling.py | 37 +- python/paddle/sparse/unary.py | 6 +- python/paddle/static/__init__.py | 65 +- python/paddle/static/amp/__init__.py | 8 +- python/paddle/static/input.py | 50 +- python/paddle/static/io.py | 169 +- python/paddle/static/nn/__init__.py | 2 +- python/paddle/static/nn/common.py | 68 +- python/paddle/static/quantization/__init__.py | 60 +- python/paddle/static/sparsity/__init__.py | 27 +- python/paddle/sysconfig.py | 2 + python/paddle/tensor/array.py | 85 +- python/paddle/tensor/attribute.py | 78 +- python/paddle/tensor/creation.py | 716 +- python/paddle/tensor/einsum.py | 181 +- .../paddle/tensor/layer_function_generator.py | 107 +- python/paddle/tensor/linalg.py | 1408 ++-- python/paddle/tensor/logic.py | 318 +- python/paddle/tensor/manipulation.py | 1578 ++-- python/paddle/tensor/math.py | 1267 ++-- python/paddle/tensor/ops.py | 177 +- python/paddle/tensor/random.py | 259 +- python/paddle/tensor/search.py | 369 +- python/paddle/tensor/stat.py | 134 +- python/paddle/tensor/to_string.py | 189 +- .../paddle/tests/dist_hapi_mnist_dynamic.py | 33 +- python/paddle/tests/dist_hapi_mnist_static.py | 33 +- .../tests/dist_hapi_pure_fp16_static.py | 22 +- python/paddle/tests/hapi_mnist_bf16_static.py | 47 +- python/paddle/tests/hubconf.py | 2 +- python/paddle/tests/test_async_read_write.py | 106 +- python/paddle/tests/test_audio_backend.py | 45 +- python/paddle/tests/test_audio_datasets.py | 60 +- python/paddle/tests/test_audio_functions.py | 190 +- .../paddle/tests/test_audio_logmel_feature.py | 145 +- python/paddle/tests/test_audio_mel_feature.py | 59 +- .../paddle/tests/test_callback_early_stop.py | 100 +- .../test_callback_reduce_lr_on_plateau.py | 88 +- python/paddle/tests/test_callback_visualdl.py | 17 +- python/paddle/tests/test_callbacks.py | 22 +- python/paddle/tests/test_dataset_cifar.py | 4 - python/paddle/tests/test_dataset_conll05.py | 1 - python/paddle/tests/test_dataset_imdb.py | 2 - python/paddle/tests/test_dataset_imikolov.py | 2 - python/paddle/tests/test_dataset_movielens.py | 2 - .../paddle/tests/test_dataset_uci_housing.py | 5 - python/paddle/tests/test_dataset_voc.py | 3 - python/paddle/tests/test_dataset_wmt.py | 20 +- python/paddle/tests/test_datasets.py | 17 +- python/paddle/tests/test_dist_hapi_model.py | 44 +- python/paddle/tests/test_dlpack.py | 36 +- python/paddle/tests/test_download.py | 41 +- python/paddle/tests/test_hapi_amp.py | 116 +- python/paddle/tests/test_hapi_hub.py | 148 +- python/paddle/tests/test_hapi_hub_model.py | 1 - python/paddle/tests/test_logger.py | 1 - python/paddle/tests/test_metrics.py | 127 +- python/paddle/tests/test_model.py | 607 +- python/paddle/tests/test_ops_roi_align.py | 71 +- python/paddle/tests/test_ops_roi_pool.py | 71 +- python/paddle/tests/test_pretrained_model.py | 1 - python/paddle/tests/test_progressbar.py | 3 +- python/paddle/tests/test_read_file.py | 11 +- python/paddle/tests/test_transforms.py | 652 +- python/paddle/tests/test_vision_models.py | 1 - python/paddle/text/__init__.py | 13 +- python/paddle/text/datasets/conll05.py | 104 +- python/paddle/text/datasets/imdb.py | 25 +- python/paddle/text/datasets/imikolov.py | 39 +- python/paddle/text/datasets/movielens.py | 76 +- python/paddle/text/datasets/uci_housing.py | 42 +- python/paddle/text/datasets/wmt14.py | 50 +- python/paddle/text/datasets/wmt16.py | 92 +- python/paddle/text/viterbi_decode.py | 66 +- python/paddle/utils/__init__.py | 4 +- python/paddle/utils/cpp_extension/__init__.py | 8 +- .../utils/cpp_extension/cpp_extension.py | 230 +- .../utils/cpp_extension/extension_utils.py | 377 +- python/paddle/utils/deprecated.py | 65 +- python/paddle/utils/dlpack.py | 8 +- python/paddle/utils/download.py | 73 +- python/paddle/utils/gast/ast3.py | 33 +- python/paddle/utils/gast/astn.py | 2 - python/paddle/utils/gast/gast.py | 1501 ++-- python/paddle/utils/image_util.py | 51 +- python/paddle/utils/install_check.py | 81 +- python/paddle/utils/lazy_import.py | 4 +- python/paddle/utils/op_version.py | 7 +- python/paddle/utils/profiler.py | 43 +- python/paddle/utils/unique_name.py | 4 +- python/paddle/vision/__init__.py | 4 +- python/paddle/vision/datasets/__init__.py | 12 +- python/paddle/vision/datasets/cifar.py | 61 +- python/paddle/vision/datasets/flowers.py | 59 +- python/paddle/vision/datasets/folder.py | 79 +- python/paddle/vision/datasets/mnist.py | 79 +- python/paddle/vision/datasets/voc2012.py | 35 +- python/paddle/vision/image.py | 12 +- python/paddle/vision/models/__init__.py | 82 +- python/paddle/vision/models/alexnet.py | 51 +- python/paddle/vision/models/densenet.py | 249 +- python/paddle/vision/models/googlenet.py | 63 +- python/paddle/vision/models/inceptionv3.py | 606 +- python/paddle/vision/models/lenet.py | 35 +- python/paddle/vision/models/mobilenetv1.py | 250 +- python/paddle/vision/models/mobilenetv2.py | 119 +- python/paddle/vision/models/mobilenetv3.py | 257 +- python/paddle/vision/models/resnet.py | 274 +- python/paddle/vision/models/shufflenetv2.py | 232 +- python/paddle/vision/models/squeezenet.py | 104 +- python/paddle/vision/models/vgg.py | 94 +- python/paddle/vision/ops.py | 1051 ++- python/paddle/vision/transforms/__init__.py | 2 +- python/paddle/vision/transforms/functional.py | 352 +- .../vision/transforms/functional_cv2.py | 255 +- .../vision/transforms/functional_pil.py | 135 +- .../vision/transforms/functional_tensor.py | 283 +- python/paddle/vision/transforms/transforms.py | 356 +- r/example/mobilenet.py | 2 +- .../CspChromeTraceFormatter.py | 1 - tools/CrossStackProfiler/CspFileReader.py | 128 +- tools/CrossStackProfiler/CspReporter.py | 152 +- tools/CrossStackProfiler/DCGMFileReader.py | 86 +- tools/CrossStackProfiler/NetFileReader.py | 37 +- tools/CrossStackProfiler/ProfileFileReader.py | 197 +- tools/analysisPyXml.py | 51 +- tools/check_api_compatible.py | 77 +- tools/check_api_source_without_core_ops.py | 6 +- tools/check_ctest_hung.py | 3 +- tools/check_op_benchmark_result.py | 88 +- tools/check_op_desc.py | 142 +- tools/check_op_kernel_same_dtypes.py | 39 +- tools/check_op_register_type.py | 35 +- tools/check_pr_approval.py | 7 +- tools/check_ut.py | 2 +- tools/codestyle/docstring_checker.py | 91 +- tools/codestyle/test_docstring_checker.py | 42 +- tools/continuous_integration/bisect.py | 111 +- tools/count_api_without_core_ops.py | 46 +- tools/coverage/coverage_diff.py | 2 +- tools/coverage/coverage_diff_list.py | 2 +- tools/coverage/coverage_lines.py | 18 +- tools/coverage/cuda_clean.py | 4 +- tools/coverage/gcda_clean.py | 7 +- tools/coverage/python_coverage.py | 19 +- tools/externalError/spider.py | 63 +- tools/final_ut_parallel_rule.py | 50 +- tools/gen_ut_cmakelists.py | 168 +- tools/get_op_list.py | 12 +- tools/get_pr_ut.py | 178 +- tools/get_single_test_cov.py | 42 +- tools/get_ut_file_map.py | 13 +- tools/get_ut_mem_map.py | 20 +- tools/group_case_for_parallel.py | 27 +- tools/handle_h_cu_file.py | 18 +- tools/infrt/fake_models/multi_fc.py | 35 +- ...rate_pd_op_dialect_from_paddle_op_maker.py | 229 +- tools/infrt/generate_phi_kernel_dialect.py | 126 +- tools/infrt/get_compat_kernel_signature.py | 40 +- tools/infrt/get_phi_kernel_info.py | 84 +- tools/infrt/print_kernel_pass_info.py | 24 +- tools/jetson_infer_op.py | 54 +- tools/parallel_UT_rule.py | 1208 ++- tools/print_op_desc.py | 7 +- tools/print_signatures.py | 123 +- tools/prune_for_jetson.py | 74 +- tools/pyCov_multithreading.py | 14 +- tools/remove_grad_op_and_kernel.py | 71 +- tools/sampcd_processor.py | 212 +- tools/summary_env.py | 25 +- tools/test_check_api_compatible.py | 68 +- tools/test_check_pr_approval.py | 60 +- tools/test_print_signatures.py | 9 +- tools/test_runner.py | 12 +- tools/test_sampcd_processor.py | 275 +- tools/timeline.py | 87 +- tools/windows/get_prec_ut_list.py | 2 +- 3580 files changed, 247291 insertions(+), 178095 deletions(-) delete mode 100644 .style.yapf create mode 100644 pyproject.toml diff --git a/.cmake-format.py b/.cmake-format.py index a6f4af8f8a3..01813004107 100644 --- a/.cmake-format.py +++ b/.cmake-format.py @@ -109,5 +109,5 @@ with section("parse"): "SRCS": '*', "DEPS": '*', } - } + }, } diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fda8a37c2ec..b58e1b0a438 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,11 +28,17 @@ repos: python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py| python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py )$ -- repo: https://github.com/google/yapf - rev: v0.32.0 +- repo: https://github.com/psf/black.git + rev: 22.8.0 hooks: - - id: yapf - files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ + - id: black + files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ + # Temporary exclude, will be formatted in a separate PR + exclude: | + (?x)^( + python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py| + python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py + )$ - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 4741fb4f3bb..00000000000 --- a/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -based_on_style = pep8 -column_limit = 80 diff --git a/cmake/copyfile.py b/cmake/copyfile.py index 7ba4d95049d..552e8ea1a2b 100644 --- a/cmake/copyfile.py +++ b/cmake/copyfile.py @@ -21,7 +21,7 @@ import glob def main(): src = sys.argv[1] dst = sys.argv[2] - if os.path.isdir(src): #copy directory + if os.path.isdir(src): # copy directory pathList = os.path.split(src) dst = os.path.join(dst, pathList[-1]) if not os.path.exists(dst): @@ -31,7 +31,7 @@ def main(): shutil.rmtree(dst) shutil.copytree(src, dst) print("overwritten copy directory: {0} --->>> {1}".format(src, dst)) - else: #copy file, wildcard + else: # copy file, wildcard if not os.path.exists(dst): os.makedirs(dst) srcFiles = glob.glob(src) diff --git a/cmake/make_resource.py b/cmake/make_resource.py index 936aefd4e6e..ad8ee179d60 100644 --- a/cmake/make_resource.py +++ b/cmake/make_resource.py @@ -20,7 +20,15 @@ res = sys.argv[1] out = sys.argv[2] var = re.sub(r'[ .-]', '_', os.path.basename(res)) -open(out, "w").write("const unsigned char " + var + "[] = {" + - ",".join(["0x%02x" % ord(c) - for c in open(res).read()]) + ",0};\n" + - "const unsigned " + var + "_size = sizeof(" + var + ");\n") +open(out, "w").write( + "const unsigned char " + + var + + "[] = {" + + ",".join(["0x%02x" % ord(c) for c in open(res).read()]) + + ",0};\n" + + "const unsigned " + + var + + "_size = sizeof(" + + var + + ");\n" +) diff --git a/paddle/fluid/eager/auto_code_generator/generate_file_structures.py b/paddle/fluid/eager/auto_code_generator/generate_file_structures.py index 19ea5982f58..5555c17f1b6 100644 --- a/paddle/fluid/eager/auto_code_generator/generate_file_structures.py +++ b/paddle/fluid/eager/auto_code_generator/generate_file_structures.py @@ -41,8 +41,9 @@ def GenerateFileStructureForFinalDygraph(eager_dir): os.mkdir(directory) # Empty files - dygraph_forward_api_h_path = os.path.join(generated_dir, - "dygraph_functions.h") + dygraph_forward_api_h_path = os.path.join( + generated_dir, "dygraph_functions.h" + ) empty_files = [dygraph_forward_api_h_path] empty_files.append(os.path.join(forwards_dir, "dygraph_functions.cc")) empty_files.append(os.path.join(nodes_dir, "nodes.cc")) @@ -83,32 +84,41 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count): os.mkdir(directory) # Empty files - dygraph_forward_api_h_path = os.path.join(generated_dir, - "dygraph_forward_api.h") + dygraph_forward_api_h_path = os.path.join( + generated_dir, "dygraph_forward_api.h" + ) empty_files = [dygraph_forward_api_h_path] empty_files.append(os.path.join(nodes_dir, "nodes.h")) for i in range(split_count): empty_files.append( - os.path.join(forwards_dir, - "dygraph_forward_functions" + str(i + 1) + ".cc")) - empty_files.append(os.path.join(nodes_dir, - "nodes" + str(i + 1) + ".cc")) + os.path.join( + forwards_dir, "dygraph_forward_functions" + str(i + 1) + ".cc" + ) + ) + empty_files.append( + os.path.join(nodes_dir, "nodes" + str(i + 1) + ".cc") + ) empty_files.append( - os.path.join(forwards_dir, "dygraph_forward_functions_args_info.cc")) + os.path.join(forwards_dir, "dygraph_forward_functions_args_info.cc") + ) empty_files.append( - os.path.join(forwards_dir, - "dygraph_forward_functions_args_type_info.cc")) + os.path.join( + forwards_dir, "dygraph_forward_functions_args_type_info.cc" + ) + ) empty_files.append( - os.path.join(forwards_dir, "dygraph_forward_functions_returns_info.cc")) + os.path.join(forwards_dir, "dygraph_forward_functions_returns_info.cc") + ) for path in empty_files: if not os.path.exists(path): open(path, 'a').close() # CMakeLists nodes_level_cmakelist_path = os.path.join(nodes_dir, "CMakeLists.txt") - generated_level_cmakelist_path = os.path.join(generated_dir, - "CMakeLists.txt") + generated_level_cmakelist_path = os.path.join( + generated_dir, "CMakeLists.txt" + ) forwards_level_cmakelist_path = os.path.join(forwards_dir, "CMakeLists.txt") with open(nodes_level_cmakelist_path, "w") as f: @@ -120,9 +130,11 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count): for i in range(split_count): f.write( " COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes" - + str(i + 1) + - ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes" - + str(i + 1) + ".cc\"\n") + + str(i + 1) + + ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes" + + str(i + 1) + + ".cc\"\n" + ) f.write(" DEPENDS legacy_eager_codegen\n") f.write(" VERBATIM)\n") @@ -142,9 +154,11 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count): for i in range(split_count): f.write( " COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions" - + str(i + 1) + - ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions" - + str(i + 1) + ".cc\"\n") + + str(i + 1) + + ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions" + + str(i + 1) + + ".cc\"\n" + ) f.write( " COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions_args_info.tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions_args_info.cc\"\n" ) @@ -167,7 +181,8 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count): "${fluid_manual_functions} DEPS ${eager_deps} ${fluid_deps} ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS})\n" ) f.write( - "add_dependencies(dygraph_function copy_dygraph_forward_functions)") + "add_dependencies(dygraph_function copy_dygraph_forward_functions)" + ) with open(generated_level_cmakelist_path, "w") as f: f.write("add_subdirectory(forwards)\nadd_subdirectory(nodes)") diff --git a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py index 91a16a0fdf0..587dfb571b0 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py +++ b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py @@ -18,20 +18,45 @@ import re ######################## ### Global Variables ### ######################## -ops_to_fill_zero_for_empty_grads = set([ - "split_grad", "split_with_num_grad", "rnn_grad", "matmul_double_grad", - "matmul_triple_grad", "sigmoid_double_grad", "sigmoid_triple_grad", - "add_double_grad", "add_triple_grad", "multiply_grad", - "multiply_double_grad", "multiply_triple_grad", "conv2d_grad_grad", - "batch_norm_double_grad", "tanh_double_grad", "tanh_triple_grad", - "subtract_double_grad", "divide_double_grad", "log_double_grad", - "elu_double_grad", "leaky_relu_double_grad", "sqrt_double_grad", - "rsqrt_double_grad", "square_double_grad", "celu_double_grad", - "pad_double_grad", "pad3d_double_grad", "squeeze_double_grad", - "unsqueeze_double_grad", "instance_norm_double_grad", "conv3d_double_grad", - "depthwise_conv2d_grad_grad", "concat_double_grad", "expand_grad", - "argsort_grad" -]) +ops_to_fill_zero_for_empty_grads = set( + [ + "split_grad", + "split_with_num_grad", + "rnn_grad", + "matmul_double_grad", + "matmul_triple_grad", + "sigmoid_double_grad", + "sigmoid_triple_grad", + "add_double_grad", + "add_triple_grad", + "multiply_grad", + "multiply_double_grad", + "multiply_triple_grad", + "conv2d_grad_grad", + "batch_norm_double_grad", + "tanh_double_grad", + "tanh_triple_grad", + "subtract_double_grad", + "divide_double_grad", + "log_double_grad", + "elu_double_grad", + "leaky_relu_double_grad", + "sqrt_double_grad", + "rsqrt_double_grad", + "square_double_grad", + "celu_double_grad", + "pad_double_grad", + "pad3d_double_grad", + "squeeze_double_grad", + "unsqueeze_double_grad", + "instance_norm_double_grad", + "conv3d_double_grad", + "depthwise_conv2d_grad_grad", + "concat_double_grad", + "expand_grad", + "argsort_grad", + ] +) # For API dispatch used at python-level # { op_name : [arg_name, ...] } @@ -40,22 +65,31 @@ core_ops_args_info = {} core_ops_args_type_info = {} yaml_types_mapping = { - 'int' : 'int', 'int32_t' : 'int32_t', 'int64_t' : 'int64_t', 'size_t' : 'size_t', \ - 'float' : 'float', 'double' : 'double', 'bool' : 'bool', \ - 'str' : 'std::string', \ - 'str[]' : 'std::vector', 'float[]' : 'std::vector', \ - 'Place' : 'paddle::Place', 'DataLayout' : 'phi::DataLayout', 'DataType' : 'paddle::experimental::DataType', \ - 'int64_t[]' : 'std::vector', 'int[]' : 'std::vector', - 'Tensor' : 'Tensor', - 'Tensor[]' : 'std::vector', - 'Tensor[Tensor[]]' : 'std::vector>', - 'Scalar' : 'paddle::experimental::Scalar', - 'Scalar(int)' : 'paddle::experimental::Scalar', - 'Scalar(int64_t)' : 'paddle::experimental::Scalar', - 'Scalar(float)' : 'paddle::experimental::Scalar', - 'Scalar(double)' : 'paddle::experimental::Scalar', - 'Scalar[]' : 'std::vector', - 'IntArray' : 'paddle::experimental::IntArray' + 'int': 'int', + 'int32_t': 'int32_t', + 'int64_t': 'int64_t', + 'size_t': 'size_t', + 'float': 'float', + 'double': 'double', + 'bool': 'bool', + 'str': 'std::string', + 'str[]': 'std::vector', + 'float[]': 'std::vector', + 'Place': 'paddle::Place', + 'DataLayout': 'phi::DataLayout', + 'DataType': 'paddle::experimental::DataType', + 'int64_t[]': 'std::vector', + 'int[]': 'std::vector', + 'Tensor': 'Tensor', + 'Tensor[]': 'std::vector', + 'Tensor[Tensor[]]': 'std::vector>', + 'Scalar': 'paddle::experimental::Scalar', + 'Scalar(int)': 'paddle::experimental::Scalar', + 'Scalar(int64_t)': 'paddle::experimental::Scalar', + 'Scalar(float)': 'paddle::experimental::Scalar', + 'Scalar(double)': 'paddle::experimental::Scalar', + 'Scalar[]': 'std::vector', + 'IntArray': 'paddle::experimental::IntArray', } @@ -81,7 +115,8 @@ def ReadBwdFile(filepath): if contents is not None: for content in contents: assert 'backward_op' in content.keys(), AssertMessage( - 'backward_op', content.keys()) + 'backward_op', content.keys() + ) if 'backward_op' in content.keys(): api_name = content['backward_op'] @@ -116,7 +151,8 @@ def IsPlainTensorType(string): def IsVectorTensorType(string): vector_tensor_types = [ - 'std::vector>', 'std::vector' + 'std::vector>', + 'std::vector', ] if string in vector_tensor_types: return True @@ -147,7 +183,6 @@ def RemoveConstAndReference(string): def GetGradNodeName(string): - def str2Hump(text): arr = filter(None, text.split('_')) res = '' @@ -166,7 +201,6 @@ def GetDygraphForwardFunctionName(string): def GetDygraphLogName(string): - def str2Hump(text): arr = filter(None, text.split('_')) res = '' @@ -236,10 +270,14 @@ def ParseYamlArgs(string): m = re.search(pattern, arg) arg_type = m.group(1).strip() arg_name = m.group(3).split("=")[0].strip() - default_value = m.group(3).split("=")[1].strip() if len( - m.group(3).split("=")) > 1 else None - - assert arg_type in yaml_types_mapping.keys( + default_value = ( + m.group(3).split("=")[1].strip() + if len(m.group(3).split("=")) > 1 + else None + ) + + assert ( + arg_type in yaml_types_mapping.keys() ), f"The argument type {arg_type} in yaml config is not supported in yaml_types_mapping." if arg_type in ["DataType", "DataLayout"] and default_value is not None: default_value = f"paddle::experimental::{default_value}" @@ -277,7 +315,8 @@ def ParseYamlReturns(string): else: ret_type = ret.strip() - assert ret_type in yaml_types_mapping.keys( + assert ( + ret_type in yaml_types_mapping.keys() ), f"The return type {ret_type} in yaml config is not supported in yaml_types_mapping." ret_type = yaml_types_mapping[ret_type] @@ -295,7 +334,9 @@ def ParseYamlForwardFromBackward(string): wspace = r'\s*' fargs = r'(.*?)' frets = r'(.*)' - pattern = fr'{fname}{wspace}\({wspace}{fargs}{wspace}\){wspace}->{wspace}{frets}' + pattern = ( + fr'{fname}{wspace}\({wspace}{fargs}{wspace}\){wspace}->{wspace}{frets}' + ) m = re.search(pattern, string) function_name = m.group(1) @@ -359,38 +400,44 @@ def ParseYamlInplaceInfo(string): ### Generator Base ### ######################## class FunctionGeneratorBase: - def __init__(self, forward_api_contents, namespace): self.forward_api_contents = forward_api_contents self.namespace = namespace - self.is_forward_only = False if 'backward' in forward_api_contents.keys( - ) else True + self.is_forward_only = ( + False if 'backward' in forward_api_contents.keys() else True + ) self.forward_api_name = "" - self.orig_forward_inputs_list = [ - ] #[ [arg_name, arg_type, orig_position], ...] - self.orig_forward_attrs_list = [ - ] #[ [attr_name, attr_type, default_value, orig_position], ...] - self.orig_forward_returns_list = [ - ] #[ [ret_name, ret_type, orig_position], ...] + self.orig_forward_inputs_list = ( + [] + ) # [ [arg_name, arg_type, orig_position], ...] + self.orig_forward_attrs_list = ( + [] + ) # [ [attr_name, attr_type, default_value, orig_position], ...] + self.orig_forward_returns_list = ( + [] + ) # [ [ret_name, ret_type, orig_position], ...] # Processed Forward Data - self.forward_inputs_position_map = { - } #{ "name" : [type, fwd_position] } - self.forward_outputs_position_map = { - } #{ "name" : [type, fwd_position] } + self.forward_inputs_position_map = ( + {} + ) # { "name" : [type, fwd_position] } + self.forward_outputs_position_map = ( + {} + ) # { "name" : [type, fwd_position] } # Special Op Attributes - self.optional_inputs = [] #[name, ...] - self.no_need_buffers = [] #[name, ...] - self.intermediate_outputs = [] #[name, ...] - self.forward_inplace_map = {} #{name : name, ...} + self.optional_inputs = [] # [name, ...] + self.no_need_buffers = [] # [name, ...] + self.intermediate_outputs = [] # [name, ...] + self.forward_inplace_map = {} # {name : name, ...} def ParseForwardInplaceInfo(self): forward_api_contents = self.forward_api_contents - if 'inplace' not in forward_api_contents.keys(): return + if 'inplace' not in forward_api_contents.keys(): + return inplace_map_str = forward_api_contents['inplace'] self.forward_inplace_map = ParseYamlInplaceInfo(inplace_map_str) @@ -432,19 +479,26 @@ class FunctionGeneratorBase: forward_args_str = forward_api_contents['args'] forward_returns_str = forward_api_contents['output'] - assert 'op' in forward_api_contents.keys( + assert ( + 'op' in forward_api_contents.keys() ), "Unable to find \"op\" in forward_api_contents keys" - assert 'args' in forward_api_contents.keys( + assert ( + 'args' in forward_api_contents.keys() ), "Unable to find \"args\" in forward_api_contents keys" - assert 'output' in forward_api_contents.keys( + assert ( + 'output' in forward_api_contents.keys() ), "Unable to find \"output\" in forward_api_contents keys" # Collect Original Forward Inputs/Outputs and then perform validation checks - self.orig_forward_inputs_list, self.orig_forward_attrs_list, self.orig_forward_returns_list = ParseYamlForward( - forward_args_str, forward_returns_str) - - def DetermineForwardPositionMap(self, forward_inputs_list, - forward_returns_list): + ( + self.orig_forward_inputs_list, + self.orig_forward_attrs_list, + self.orig_forward_returns_list, + ) = ParseYamlForward(forward_args_str, forward_returns_str) + + def DetermineForwardPositionMap( + self, forward_inputs_list, forward_returns_list + ): for i in range(len(forward_inputs_list)): forward_input = forward_inputs_list[i] input_name = forward_input[0] @@ -452,13 +506,14 @@ class FunctionGeneratorBase: input_pos = forward_input[2] self.forward_inputs_position_map[input_name] = [ - input_type, input_pos + input_type, + input_pos, ] for i in range(len(forward_returns_list)): forward_return = forward_returns_list[i] if len(forward_return[0]) == 0: - if (len(forward_returns_list) == 1): + if len(forward_returns_list) == 1: return_name = "out" else: return_name = "out_{}".format(i + 1) @@ -468,12 +523,12 @@ class FunctionGeneratorBase: return_pos = forward_return[2] self.forward_outputs_position_map[return_name] = [ - return_type, return_pos + return_type, + return_pos, ] class GeneratorBase: - def __init__(self, api_yaml_path): self.namespace = "" self.api_yaml_path = api_yaml_path diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py index cf51fbb9d07..fe0b021a2e5 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py @@ -15,12 +15,19 @@ import re import argparse import os -from codegen_utils import core_ops_returns_info, core_ops_args_info, core_ops_args_type_info +from codegen_utils import ( + core_ops_returns_info, + core_ops_args_info, + core_ops_args_type_info, +) from codegen_utils import ReadBwdFile from codegen_utils import FindForwardName, GetGradNodeName, GetSavedName from codegen_utils import IsPlainTensorType, IsVectorTensorType from codegen_utils import GetConstReference, RemoveConstAndReference -from codegen_utils import GetDygraphForwardFunctionName, GetIntermediateAPIFunctionName +from codegen_utils import ( + GetDygraphForwardFunctionName, + GetIntermediateAPIFunctionName, +) from codegen_utils import GetAutoGradMetaName, GetAutoGradMetaVectorName from codegen_utils import GetInplacedFunctionName from codegen_utils import ParseYamlForwardFromBackward @@ -39,7 +46,11 @@ inplace_check_blacklist = set(["assign_out_"]) # Black Ops list that's NO NEED to apply code generation black_ops_list = [ - "conv2d", "conv2d_grad", "conv2d_grad_grad", "add_n", "add_n_grad" + "conv2d", + "conv2d_grad", + "conv2d_grad_grad", + "add_n", + "add_n_grad", ] @@ -48,7 +59,8 @@ black_ops_list = [ ########### def ParseArguments(): parser = argparse.ArgumentParser( - description='Eager Code Generator Args Parser') + description='Eager Code Generator Args Parser' + ) parser.add_argument('--nodes_h_path', type=str) parser.add_argument('--nodes_cc_path', type=str) parser.add_argument('--forwards_h_path', type=str) @@ -63,54 +75,44 @@ def ParseArguments(): ######################## ## Code Gen Templates ## ######################## -SET_PLAIN_TENSOR_WRAPPER_TEMPLATE = \ -""" void SetTensorWrapper{}(const paddle::experimental::Tensor& {}) {{ +SET_PLAIN_TENSOR_WRAPPER_TEMPLATE = """ void SetTensorWrapper{}(const paddle::experimental::Tensor& {}) {{ {} = egr::TensorWrapper({}, {}); }} """ -SET_VECTOR_TENSOR_WRAPPER_TEMPLATE = \ -""" void SetTensorWrapper{}(const std::vector& {}) {{ +SET_VECTOR_TENSOR_WRAPPER_TEMPLATE = """ void SetTensorWrapper{}(const std::vector& {}) {{ for(const auto& eager_tensor : {}) {{ {}.emplace_back(egr::TensorWrapper(eager_tensor, {})); }}; }} """ -PLAIN_TENSOR_MEMBER_TEMPLATE = \ -""" egr::TensorWrapper {}; +PLAIN_TENSOR_MEMBER_TEMPLATE = """ egr::TensorWrapper {}; """ -VECTOR_TENSOR_MEMBER_TEMPLATE = \ -""" std::vector {}; +VECTOR_TENSOR_MEMBER_TEMPLATE = """ std::vector {}; """ -CLEAR_TENSOR_WRAPPER_TEMPLATE = \ -""" {}.clear(); +CLEAR_TENSOR_WRAPPER_TEMPLATE = """ {}.clear(); """ -CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE = \ -""" for (auto& tw : {}) {{ +CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE = """ for (auto& tw : {}) {{ tw.clear(); }} """ -SET_ATTR_METHOD_TEMPLATE = \ -""" void SetAttribute{}({} {}) {{ +SET_ATTR_METHOD_TEMPLATE = """ void SetAttribute{}({} {}) {{ {} = {}; }} """ -ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE = \ -""" {} {} = {}; +ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE = """ {} {} = {}; """ -ATTRIBUTE_MEMBER_TEMPLATE = \ -""" {} {}; +ATTRIBUTE_MEMBER_TEMPLATE = """ {} {}; """ -NODE_DECLARATION_TEMPLATE = \ -""" +NODE_DECLARATION_TEMPLATE = """ class {} : public egr::GradNodeBase {{ public: {}() : egr::GradNodeBase() {{}} @@ -143,8 +145,7 @@ class {} : public egr::GradNodeBase {{ {}}}; """ -GRAD_FUNCTION_TEMPLATE = \ -""" +GRAD_FUNCTION_TEMPLATE = """ paddle::small_vector, egr::kSlotSmallVectorSize> {}::operator()(paddle::small_vector, egr::kSlotSmallVectorSize>& grads, bool create_graph, bool is_new_grad) {{ VLOG(3) << \"Running AD API GRAD: \" << \"{}\"; // Fill Zero For GradIn Tensors @@ -182,8 +183,7 @@ paddle::small_vector, egr::kSlotSmallV }} """ -FORWARD_FUNCTION_TEMPLATE = \ -""" +FORWARD_FUNCTION_TEMPLATE = """ {} {}({}) {{ VLOG(3) << \"Running AD API: \" << \"{}\"; // Dygraph Record Event @@ -222,8 +222,7 @@ FORWARD_FUNCTION_TEMPLATE = \ }} """ -AFTER_LOG_PRINT_TEMPLATE = \ -""" +AFTER_LOG_PRINT_TEMPLATE = """ if(VLOG_IS_ON(4)){{ const char* INPUT_PRINT_TEMPLATE = \"{{ Input: [%s], Output: [%s] }} \"; {} @@ -231,8 +230,7 @@ AFTER_LOG_PRINT_TEMPLATE = \ }} """ -BEFORE_LOG_PRINT_TEMPLATE = \ -""" +BEFORE_LOG_PRINT_TEMPLATE = """ if(VLOG_IS_ON(3)){{ const char* INPUT_PRINT_TEMPLATE = \"{{ Input: [%s]}} \"; {} @@ -240,8 +238,7 @@ BEFORE_LOG_PRINT_TEMPLATE = \ }} """ -FORWARD_ONLY_FUNCTION_TEMPLATE = \ -""" +FORWARD_ONLY_FUNCTION_TEMPLATE = """ {} {}({}) {{ VLOG(3) << \"Running AD API: \" << \"{}\"; // Dygraph Record Event @@ -265,8 +262,7 @@ FORWARD_ONLY_FUNCTION_TEMPLATE = \ }} """ -FORWARD_BODY_TEMPLATE = \ -""" if(require_any_grad) {{ +FORWARD_BODY_TEMPLATE = """ if(require_any_grad) {{ {} egr::EagerUtils::PassStopGradient({}); @@ -288,8 +284,7 @@ FORWARD_BODY_TEMPLATE = \ }} """ -HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE = \ -""" if(trace_backward) {{ +HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE = """ if(trace_backward) {{ {} // Node Construction {} @@ -309,15 +304,13 @@ HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE = \ }} """ -NAMESPACE_WRAPPER_TEMPLATE = \ -""" +NAMESPACE_WRAPPER_TEMPLATE = """ namespace {} {{ {} }} """ -NODE_CC_FILE_TEMPLATE = \ -""" +NODE_CC_FILE_TEMPLATE = """ #include "glog/logging.h" #include "paddle/phi/api/all.h" #include "paddle/phi/api/backward/backward_api.h" @@ -337,8 +330,7 @@ DECLARE_bool(check_nan_inf); {} """ -NODE_H_FILE_TEMPLATE = \ -""" +NODE_H_FILE_TEMPLATE = """ #pragma once #include "paddle/fluid/eager/tensor_wrapper.h" #include "paddle/fluid/eager/grad_node_info.h" @@ -347,8 +339,7 @@ NODE_H_FILE_TEMPLATE = \ {} """ -FORWARD_CC_FILE_TEMPLATE = \ -""" +FORWARD_CC_FILE_TEMPLATE = """ #include "paddle/phi/api/lib/dygraph_api.h" #include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h" #include "paddle/fluid/eager/api/generated/eager_generated/backwards/nodes.h" @@ -367,8 +358,7 @@ DECLARE_bool(check_nan_inf); {} """ -FORWARD_H_FILE_TEMPLATE = \ -""" +FORWARD_H_FILE_TEMPLATE = """ #pragma once #include "glog/logging.h" #include "paddle/fluid/eager/autograd_meta.h" @@ -383,8 +373,7 @@ using CPUPlace = phi::CPUPlace; {} """ -CORE_OPS_INFO_TEMPLATE = \ -""" +CORE_OPS_INFO_TEMPLATE = """ std::unordered_map> core_ops_args_info = {{ {} }}; @@ -397,28 +386,24 @@ std::unordered_map> core_ops_returns_info """ -CORE_OPS_DECLARATION_TEMPLATE = \ -""" +CORE_OPS_DECLARATION_TEMPLATE = """ extern std::unordered_map> core_ops_args_info; extern std::unordered_map> core_ops_args_type_info; extern std::unordered_map> core_ops_returns_info; """ -CHECK_INPLACE_TEMPLATE = \ -""" +CHECK_INPLACE_TEMPLATE = """ egr::EagerUtils::CheckInplace({}, {}, require_any_grad); """ -BUMP_INPLACE_VERSION_TEMPLATE = \ -""" +BUMP_INPLACE_VERSION_TEMPLATE = """ // Bump Inplace Version {}.bump_inplace_version(); VLOG(3) << \"Tensor(\" << {}.name() << \") uses Inplace Strategy.\"; """ -AMP_LOGIC_TEMPLATE = \ -""" if (egr::Controller::Instance().GetAMPLevel() != paddle::imperative::AmpLevel::O0) {{ +AMP_LOGIC_TEMPLATE = """ if (egr::Controller::Instance().GetAMPLevel() != paddle::imperative::AmpLevel::O0) {{ VLOG(5) << "Check and Prepare For AMP"; {} paddle::small_vector, egr::kSlotSmallVectorSize> amp_tensors_vector = {}; @@ -431,8 +416,7 @@ AMP_LOGIC_TEMPLATE = \ }} }} """ -LAYOUT_LOGIC_TEMPLATE=\ -""" +LAYOUT_LOGIC_TEMPLATE = """ if (egr::Controller::Instance().UseLayoutAutoTune()) {{ paddle::small_vector, egr::kSlotSmallVectorSize> tensors_vector = {}; {} @@ -445,26 +429,22 @@ LAYOUT_LOGIC_TEMPLATE=\ return {}; }} """ -CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE = \ -""" +CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE = """ paddle::optional {}_optional; if({}.initialized()) {}_optional = paddle::make_optional({}); """ -CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE = \ -""" +CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE = """ paddle::optional {}_optional; if( {}.impl() ) {}_optional = paddle::make_optional({}); """ -CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE = \ -""" +CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE = """ paddle::optional> {}_optional; if( !{}.empty() ) {}_optional = paddle::make_optional>({}); """ -CHECK_BACKWARD_INPLACE_TEMPLATE = \ -""" +CHECK_BACKWARD_INPLACE_TEMPLATE = """ bool can_be_inplaced = false; if ({}.initialized()) {{ VLOG(10) << {}.name() << "({}) use_count: " << {}.impl().use_count(); @@ -473,15 +453,12 @@ CHECK_BACKWARD_INPLACE_TEMPLATE = \ }} }}""" -CHECK_NAN_AND_INF_TEMPLATE = \ -""" if (FLAGS_check_nan_inf) {{ egr::CheckTensorHasNanOrInf("{}", {}); }} +CHECK_NAN_AND_INF_TEMPLATE = """ if (FLAGS_check_nan_inf) {{ egr::CheckTensorHasNanOrInf("{}", {}); }} """ inplace_optional_out_type_map = { - "Tensor": - "paddle::optional&", - "std::vector": - "paddle::optional>&" + "Tensor": "paddle::optional&", + "std::vector": "paddle::optional>&", } @@ -490,13 +467,16 @@ def ExtractForwardApiNameFormInvoke(invoke_config): if api_name[-1] == '_': api_name = api_name[:-1] return re.search( - r"(?P[a-zA-Z0-9_]+)(?P_intermediate)?", - api_name).group('api_name') + r"(?P[a-zA-Z0-9_]+)(?P_intermediate)?", api_name + ).group('api_name') def IsInvokeForwardApi(api_contents, forward_api_name_list): - return 'invoke' in api_contents and ExtractForwardApiNameFormInvoke( - api_contents['invoke']) in forward_api_name_list + return ( + 'invoke' in api_contents + and ExtractForwardApiNameFormInvoke(api_contents['invoke']) + in forward_api_name_list + ) ####################### @@ -531,7 +511,8 @@ def GenerateCoreOpInfoDefinition(): op_returns_info_str = "\n".join(op_returns_info_list) core_ops_info_definition_str = CORE_OPS_INFO_TEMPLATE.format( - op_args_info_str, op_types_info_str, op_returns_info_str) + op_args_info_str, op_types_info_str, op_returns_info_str + ) return core_ops_info_definition_str @@ -540,23 +521,27 @@ def GenerateCoreOpInfoDefinition(): ## Generator Class ## ##################### class DygraphFunctionGeneratorBase(FunctionGeneratorBase): - - def __init__(self, forward_api_contents, grad_api_contents, - forward_apis_dict, namespace): + def __init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + ): self.forward_api_contents = forward_api_contents # Members from Parent: - #self.namespace - #self.forward_api_contents - #self.forward_api_name - #self.orig_forward_inputs_list - #self.orig_forward_attrs_list - #self.orig_forward_returns_list - #self.forward_inputs_position_map - #self.forward_outputs_position_map - #self.optional_inputs - #self.no_need_buffers - #self.intermediate_outputs - #self.forward_inplace_map + # self.namespace + # self.forward_api_contents + # self.forward_api_name + # self.orig_forward_inputs_list + # self.orig_forward_attrs_list + # self.orig_forward_returns_list + # self.forward_inputs_position_map + # self.forward_outputs_position_map + # self.optional_inputs + # self.no_need_buffers + # self.intermediate_outputs + # self.forward_inplace_map FunctionGeneratorBase.__init__(self, forward_api_contents, namespace) self.forward_apis_dict = forward_apis_dict @@ -566,33 +551,43 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): self.backward_forward_str = "" self.backward_api_name = "" - self.forward_attrs_list = [ - ] #[ [attr_name, attr_type, default_value, orig_position], ...] - self.forward_inputs_list = [ - ] #[ [arg_name, arg_type, orig_position], ...] - self.forward_returns_list = [ - ] #[ [ret_name, ret_type, orig_position], ...] - - self.backward_attrs_list = [ - ] #[ [attr_name, attr_type, default_value, orig_position], ...] - self.backward_inputs_list = [ - ] #[ [arg_name, arg_type, orig_position], ...] - self.backward_returns_list = [ - ] #[ [ret_name, ret_type, orig_position], ...] + self.forward_attrs_list = ( + [] + ) # [ [attr_name, attr_type, default_value, orig_position], ...] + self.forward_inputs_list = ( + [] + ) # [ [arg_name, arg_type, orig_position], ...] + self.forward_returns_list = ( + [] + ) # [ [ret_name, ret_type, orig_position], ...] + + self.backward_attrs_list = ( + [] + ) # [ [attr_name, attr_type, default_value, orig_position], ...] + self.backward_inputs_list = ( + [] + ) # [ [arg_name, arg_type, orig_position], ...] + self.backward_returns_list = ( + [] + ) # [ [ret_name, ret_type, orig_position], ...] # SlotNameMatched Backward Data - self.backward_forward_inputs_map = { - } #{ "name" : [type, is_fwd_input, orig_position] ...} - self.backward_grad_inputs_map = { - } #{ "name" : [type, fwd_position, orig_position] ...} - self.backward_grad_outputs_map = { - } #{ "name" : [type, fwd_position, orig_position] ...} - - self.backward_inplace_map = {} #{name : name, ...} + self.backward_forward_inputs_map = ( + {} + ) # { "name" : [type, is_fwd_input, orig_position] ...} + self.backward_grad_inputs_map = ( + {} + ) # { "name" : [type, fwd_position, orig_position] ...} + self.backward_grad_outputs_map = ( + {} + ) # { "name" : [type, fwd_position, orig_position] ...} + + self.backward_inplace_map = {} # {name : name, ...} def ParseBackwardInplaceInfo(self): grad_api_contents = self.grad_api_contents - if 'inplace' not in grad_api_contents.keys(): return + if 'inplace' not in grad_api_contents.keys(): + return inplace_map_str = grad_api_contents['inplace'] self.backward_inplace_map = ParseYamlInplaceInfo(inplace_map_str) @@ -601,21 +596,28 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): forward_api_contents = self.forward_api_contents grad_api_contents = self.grad_api_contents - assert 'op' in forward_api_contents.keys( + assert ( + 'op' in forward_api_contents.keys() ), "Unable to find \"op\" in ops.yaml" - assert 'args' in forward_api_contents.keys( + assert ( + 'args' in forward_api_contents.keys() ), "Unable to find \"args\" in ops.yaml" - assert 'output' in forward_api_contents.keys( + assert ( + 'output' in forward_api_contents.keys() ), "Unable to find \"output\" in ops.yaml" if grad_api_contents is not None: - assert 'backward' in forward_api_contents.keys( + assert ( + 'backward' in forward_api_contents.keys() ), "Unable to find \"backward\" in ops.yaml" - assert 'args' in grad_api_contents.keys( + assert ( + 'args' in grad_api_contents.keys() ), "Unable to find \"args\" in backward.yaml" - assert 'output' in grad_api_contents.keys( + assert ( + 'output' in grad_api_contents.keys() ), "Unable to find \"output\" in backward.yaml" - assert 'forward' in grad_api_contents.keys( + assert ( + 'forward' in grad_api_contents.keys() ), "Unable to find \"forward\" in backward.yaml" def ForwardsValidationCheck(self): @@ -634,9 +636,11 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): orig_input_pos = orig_forward_inputs_list[i][2] assert forward_input_type == orig_input_type, AssertMessage( - forward_input_type, orig_input_type) + forward_input_type, orig_input_type + ) assert forward_input_pos == orig_input_pos, AssertMessage( - forward_input_pos, orig_input_pos) + forward_input_pos, orig_input_pos + ) for i in range(len(forward_attrs_list)): orig_attr_type = orig_forward_attrs_list[i][1] @@ -644,9 +648,11 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): forward_attr_type = forward_attrs_list[i][1] forward_attr_pos = forward_attrs_list[i][3] assert orig_attr_type == forward_attr_type, AssertMessage( - orig_attr_type, forward_attr_type) + orig_attr_type, forward_attr_type + ) assert orig_attr_pos == forward_attr_pos, AssertMessage( - orig_attr_pos, forward_attr_pos) + orig_attr_pos, forward_attr_pos + ) for i in range(len(forward_returns_list)): orig_return_type = orig_forward_returns_list[i][1] @@ -655,9 +661,11 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): forward_return_pos = forward_returns_list[i][2] assert orig_return_type == forward_return_type, AssertMessage( - orig_return_type, forward_return_type) + orig_return_type, forward_return_type + ) assert orig_return_pos == forward_return_pos, AssertMessage( - orig_return_pos, forward_return_pos) + orig_return_pos, forward_return_pos + ) # Check Order: Inputs, Attributes max_input_position = -1 @@ -666,7 +674,8 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): for _, _, _, pos in forward_attrs_list: assert pos > max_input_position, AssertMessage( - pos, max_input_position) + pos, max_input_position + ) def BackwardValidationCheck(self): backward_forward_inputs_map = self.backward_forward_inputs_map @@ -681,13 +690,15 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): max_grad_tensor_position = -1 for _, (_, _, pos) in backward_grad_inputs_map.items(): assert pos > max_fwd_input_position, AssertMessage( - pos, max_grad_tensor_position) + pos, max_grad_tensor_position + ) max_grad_tensor_position = max(max_grad_tensor_position, pos) max_attr_position = -1 for _, _, _, pos in backward_attrs_list: assert pos > max_grad_tensor_position, AssertMessage( - pos, max_grad_tensor_position) + pos, max_grad_tensor_position + ) max_attr_position = max(max_attr_position, pos) def IntermediateValidationCheck(self): @@ -699,11 +710,13 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): """ intermediate_positions = range( len(forward_returns_list) - len(intermediate_outputs), - len(forward_returns_list)) + len(forward_returns_list), + ) for ret_name, _, pos in forward_returns_list: if ret_name in intermediate_outputs: assert pos in intermediate_positions, AssertMessage( - pos, intermediate_positions) + pos, intermediate_positions + ) def CollectBackwardInfo(self): forward_api_contents = self.forward_api_contents @@ -714,20 +727,32 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): backward_args_str = grad_api_contents['args'] backward_returns_str = grad_api_contents['output'] - self.backward_inputs_list, self.backward_attrs_list, self.backward_returns_list = ParseYamlBackward( - backward_args_str, backward_returns_str) + ( + self.backward_inputs_list, + self.backward_attrs_list, + self.backward_returns_list, + ) = ParseYamlBackward(backward_args_str, backward_returns_str) def CollectForwardInfoFromBackwardContents(self): backward_forward_str = self.backward_forward_str - self.forward_inputs_list, self.forward_attrs_list, self.forward_returns_list = ParseYamlForwardFromBackward( - backward_forward_str) + ( + self.forward_inputs_list, + self.forward_attrs_list, + self.forward_returns_list, + ) = ParseYamlForwardFromBackward(backward_forward_str) def CollectForwardInfoFromYamlForward(self): - self.forward_inputs_list, self.forward_attrs_list, self.forward_returns_list = ParseYamlForwardFromBackward( - self.forward_api_contents['args'] + " -> " + - self.forward_api_contents['output']) + ( + self.forward_inputs_list, + self.forward_attrs_list, + self.forward_returns_list, + ) = ParseYamlForwardFromBackward( + self.forward_api_contents['args'] + + " -> " + + self.forward_api_contents['output'] + ) def SlotNameMatching(self): backward_inputs_list = self.backward_inputs_list @@ -743,35 +768,48 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): backward_fwd_name = FindForwardName(backward_input_name) if backward_fwd_name: # Grad Input - assert backward_fwd_name in forward_outputs_position_map.keys( - ), AssertMessage(backward_fwd_name, - forward_outputs_position_map.keys()) + assert ( + backward_fwd_name in forward_outputs_position_map.keys() + ), AssertMessage( + backward_fwd_name, forward_outputs_position_map.keys() + ) matched_forward_output_type = forward_outputs_position_map[ - backward_fwd_name][0] + backward_fwd_name + ][0] matched_forward_output_pos = forward_outputs_position_map[ - backward_fwd_name][1] + backward_fwd_name + ][1] self.backward_grad_inputs_map[backward_input_name] = [ - backward_input_type, matched_forward_output_pos, - backward_input_pos + backward_input_type, + matched_forward_output_pos, + backward_input_pos, ] else: # TensorWrapper Input if backward_input_name in forward_inputs_position_map.keys(): tensor_wrapper_type = forward_inputs_position_map[ - backward_input_name][0] + backward_input_name + ][0] self.backward_forward_inputs_map[backward_input_name] = [ - backward_input_type, True, backward_input_pos + backward_input_type, + True, + backward_input_pos, ] elif backward_input_name in forward_outputs_position_map.keys(): tensor_wrapper_type = forward_outputs_position_map[ - backward_input_name][0] + backward_input_name + ][0] self.backward_forward_inputs_map[backward_input_name] = [ - backward_input_type, False, backward_input_pos + backward_input_type, + False, + backward_input_pos, ] else: - assert False, f"Cannot find {backward_input_name} in forward position map" + assert ( + False + ), f"Cannot find {backward_input_name} in forward position map" for backward_output in backward_returns_list: backward_output_name = backward_output[0] @@ -779,19 +817,26 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): backward_output_pos = backward_output[2] backward_fwd_name = FindForwardName(backward_output_name) - assert backward_fwd_name is not None, f"Detected {backward_fwd_name} = None" - assert backward_fwd_name in forward_inputs_position_map.keys( - ), AssertMessage(backward_fwd_name, - forward_inputs_position_map.keys()) + assert ( + backward_fwd_name is not None + ), f"Detected {backward_fwd_name} = None" + assert ( + backward_fwd_name in forward_inputs_position_map.keys() + ), AssertMessage( + backward_fwd_name, forward_inputs_position_map.keys() + ) matched_forward_input_type = forward_inputs_position_map[ - backward_fwd_name][0] + backward_fwd_name + ][0] matched_forward_input_pos = forward_inputs_position_map[ - backward_fwd_name][1] + backward_fwd_name + ][1] self.backward_grad_outputs_map[backward_output_name] = [ - backward_output_type, matched_forward_input_pos, - backward_output_pos + backward_output_type, + matched_forward_input_pos, + backward_output_pos, ] def GetPassStopGradientArgsList(self, forward_outputs_position_map): @@ -815,7 +860,8 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): # Pass Stop Gradient Args pass_stop_gradient_args_str = self.GetPassStopGradientArgsList( - forward_outputs_position_map) + forward_outputs_position_map + ) # Node Construction num_backward_inputs = len(forward_outputs_position_map.keys()) @@ -839,7 +885,9 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): for name, _, default_val_attr, _ in backward_attrs_list: if name in forward_attrs_name_set: - set_attributes = f"{indent}grad_node->SetAttribute{name}({name});" + set_attributes = ( + f"{indent}grad_node->SetAttribute{name}({name});" + ) else: set_attributes = f"{indent}grad_node->SetAttribute{name}({default_val_attr});" set_attributes_list.append(set_attributes) @@ -849,31 +897,41 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): set_input_tensor_wrappers_list = [] set_output_tensor_wrappers_list = [] num_fwd_outputs = len(forward_outputs_position_map.keys()) - for name, (atype, is_fwd_input, - pos) in backward_forward_inputs_map.items(): - is_optional = (name in optional_inputs) + for name, ( + atype, + is_fwd_input, + pos, + ) in backward_forward_inputs_map.items(): + is_optional = name in optional_inputs if is_fwd_input: if is_optional: set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper{name}(*{name});" else: - set_tensor_wrappers = f"{indent}grad_node->SetTensorWrapper{name}({name});" + set_tensor_wrappers = ( + f"{indent}grad_node->SetTensorWrapper{name}({name});" + ) set_input_tensor_wrappers_list.append(set_tensor_wrappers) else: # Forwad's output as backward's input if num_fwd_outputs > 1: # Aligned with forward output position - assert name in forward_outputs_position_map.keys( + assert ( + name in forward_outputs_position_map.keys() ), AssertMessage(name, forward_outputs_position_map.keys()) if is_optional: set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper{name}(*{name});" else: - set_tensor_wrappers = f"{indent}grad_node->SetTensorWrapper{name}({name});" + set_tensor_wrappers = ( + f"{indent}grad_node->SetTensorWrapper{name}({name});" + ) set_output_tensor_wrappers_list.append(set_tensor_wrappers) set_input_tensor_wrappers_str = "\n".join( - set_input_tensor_wrappers_list) + set_input_tensor_wrappers_list + ) set_output_tensor_wrappers_str = "\n".join( - set_output_tensor_wrappers_list) + set_output_tensor_wrappers_list + ) # SetGradOutMeta & SetEdges grad_node_out_list = [] @@ -882,19 +940,24 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): for name, (_, pos) in forward_inputs_position_map.items(): # Has corresponding grad output has_corresponding_grad_output = False - for _, (_, corresponding_pos, - _) in backward_grad_outputs_map.items(): + for _, ( + _, + corresponding_pos, + _, + ) in backward_grad_outputs_map.items(): if pos == corresponding_pos: has_corresponding_grad_output = True if not has_corresponding_grad_output: continue grad_node_out_list.append(name) - is_optional = (name in self.optional_inputs) + is_optional = name in self.optional_inputs if is_optional: set_grad_out_meta = f"{indent}if({name}.get_ptr() != nullptr) grad_node->SetGradOutMeta(*({name}.get_ptr()), {pos});" else: - set_grad_out_meta = f"{indent}grad_node->SetGradOutMeta({name}, {pos});" + set_grad_out_meta = ( + f"{indent}grad_node->SetGradOutMeta({name}, {pos});" + ) set_grad_out_meta_list.append(set_grad_out_meta) set_grad_out_meta_str = "\n".join(set_grad_out_meta_list) @@ -915,8 +978,12 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): {indent} egr::EagerUtils::SetHistory({output_autograd_meta_name}, grad_node); {indent}}}""" - set_grad_in_meta = f"{indent}grad_node->SetGradInMeta({name}, {pos});" - set_retain_grad = f"{indent}egr::EagerUtils::CheckAndRetainGrad({name});" + set_grad_in_meta = ( + f"{indent}grad_node->SetGradInMeta({name}, {pos});" + ) + set_retain_grad = ( + f"{indent}egr::EagerUtils::CheckAndRetainGrad({name});" + ) set_out_rank_list.append(set_out_rank) set_history_list.append(set_history) @@ -932,18 +999,33 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): node_creation_event_str = f"{indent}paddle::platform::RecordEvent node_creation_record_event(\"{node_event_name}\", paddle::platform::TracerEventType::OperatorInner, 1);\n" if not for_backward: self.node_creation_str = FORWARD_BODY_TEMPLATE.format( - node_creation_event_str, pass_stop_gradient_args_str, - node_construction_str, set_attributes_str, - set_input_tensor_wrappers_str, set_grad_out_meta_str, - set_out_rank_str, set_history_str, set_grad_in_meta_str, - set_retain_grad_str, set_output_tensor_wrappers_str) + node_creation_event_str, + pass_stop_gradient_args_str, + node_construction_str, + set_attributes_str, + set_input_tensor_wrappers_str, + set_grad_out_meta_str, + set_out_rank_str, + set_history_str, + set_grad_in_meta_str, + set_retain_grad_str, + set_output_tensor_wrappers_str, + ) else: - self.node_creation_str = HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE.format( - node_creation_event_str, node_construction_str, - set_attributes_str, set_input_tensor_wrappers_str, - set_grad_out_meta_str, set_out_rank_str, set_history_str, - set_grad_in_meta_str, set_retain_grad_str, - set_output_tensor_wrappers_str) + self.node_creation_str = ( + HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE.format( + node_creation_event_str, + node_construction_str, + set_attributes_str, + set_input_tensor_wrappers_str, + set_grad_out_meta_str, + set_out_rank_str, + set_history_str, + set_grad_in_meta_str, + set_retain_grad_str, + set_output_tensor_wrappers_str, + ) + ) self.grad_node_out_list = grad_node_out_list @@ -988,8 +1070,9 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): ## Process Parsed Contents ## ############################# # Initialize forward_inputs_position_map, forward_outputs_position_map - self.DetermineForwardPositionMap(self.forward_inputs_list, - self.forward_returns_list) + self.DetermineForwardPositionMap( + self.forward_inputs_list, self.forward_returns_list + ) if self.grad_api_contents is not None: # Initialize backward_forward_inputs_map, backward_grad_inputs_map, backward_grad_outputs_map @@ -999,31 +1082,51 @@ class DygraphFunctionGeneratorBase(FunctionGeneratorBase): class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): - - def __init__(self, forward_api_contents, grad_api_contents, - forward_apis_dict, namespace): - DygraphFunctionGeneratorBase.__init__(self, forward_api_contents, - grad_api_contents, - forward_apis_dict, namespace) + def __init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + ): + DygraphFunctionGeneratorBase.__init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + ) # Generated Results self.forward_definition_str = "" self.forward_declaration_str = "" - def GenerateForwardLayoutAutotune(self, forward_api_name, - amp_tensors_vector_list, - layout_tensors_vector_optional_list, - layout_autotune_list_str, - returns_type_str, returns_str, - amp_inputs_call_args_str): + def GenerateForwardLayoutAutotune( + self, + forward_api_name, + amp_tensors_vector_list, + layout_tensors_vector_optional_list, + layout_autotune_list_str, + returns_type_str, + returns_str, + amp_inputs_call_args_str, + ): intermediate_outputs = self.intermediate_outputs forward_attrs_list = self.forward_attrs_list forward_outputs_position_map = self.forward_outputs_position_map - num_outputs = len( - forward_outputs_position_map.keys()) - len(intermediate_outputs) + num_outputs = len(forward_outputs_position_map.keys()) - len( + intermediate_outputs + ) # for layout autotune attr lightly_sensitive_attr = [ - 'axis', 'axes', 'dim', 'dims', 'start', 'end', 'stop', 'perm' + 'axis', + 'axes', + 'dim', + 'dims', + 'start', + 'end', + 'stop', + 'perm', ] heavily_sensitive_attr = ['data_format', 'data_layout'] layout_autotune_attr = [] @@ -1037,15 +1140,17 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): heavily_flag = False for name, atype, default_val, pos in forward_attrs_list: for attr_name in lightly_sensitive_attr: - if name.find(attr_name) != -1 and (name - not in layout_autotune_attr): + if name.find(attr_name) != -1 and ( + name not in layout_autotune_attr + ): lightly_flag = True layout_autotune_attr.append(name) layout_autotune_attr_type_list.append(atype) if lightly_flag is False: for attr_name in heavily_sensitive_attr: if name.find(attr_name) != -1 and ( - name not in layout_autotune_attr): + name not in layout_autotune_attr + ): layout_autotune_attr.append(name) layout_autotune_attr_type_list.append(atype) heavily_flag = True @@ -1074,7 +1179,8 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): if num_outputs == 1: result_name = returns_str layout_autotune_outs_list.append( - f"transformer -> SetOutTensorLayout(&{returns_str});\n") + f"transformer -> SetOutTensorLayout(&{returns_str});\n" + ) else: for name, (rtype, pos) in forward_outputs_position_map.items(): if name in intermediate_outputs: @@ -1083,11 +1189,13 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): f" auto& {name} = std::get<{len(layout_tmp_result_list)}>(api_result);\n" ) layout_autotune_outs_list.append( - f" transformer -> SetOutTensorLayout(&{name});\n") + f" transformer -> SetOutTensorLayout(&{name});\n" + ) layout_tmp_result_list.append(f"{name}") - tensors_vector_list_str = "{ " + ",".join( - amp_tensors_vector_list) + " }" + tensors_vector_list_str = ( + "{ " + ",".join(amp_tensors_vector_list) + " }" + ) if len(amp_tensors_vector_list) == 0: layout_logic_str = "" @@ -1096,9 +1204,13 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): layout_logic_str = LAYOUT_LOGIC_TEMPLATE.format( tensors_vector_list_str, " ".join(layout_tensors_vector_optional_list), - " ".join(layout_autotune_attr_code_list) + " " + - layout_autotune_list_str, after_call_str, - " ".join(layout_autotune_outs_list), returns_str) + " ".join(layout_autotune_attr_code_list) + + " " + + layout_autotune_list_str, + after_call_str, + " ".join(layout_autotune_outs_list), + returns_str, + ) return layout_logic_str @@ -1106,8 +1218,11 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): namespace = self.namespace if self.forward_api_name[-1] == '_' and not is_inplaced: return - forward_api_name = GetInplacedFunctionName( - self.forward_api_name) if is_inplaced else self.forward_api_name + forward_api_name = ( + GetInplacedFunctionName(self.forward_api_name) + if is_inplaced + else self.forward_api_name + ) forward_inputs_position_map = self.forward_inputs_position_map forward_outputs_position_map = self.forward_outputs_position_map @@ -1122,7 +1237,8 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): # Get Function Args num_inputs = len(forward_attrs_list) + len( - forward_inputs_position_map.keys()) + forward_inputs_position_map.keys() + ) inputs_args_definition_list = ["" for i in range(num_inputs)] inputs_args_declaration_list = ["" for i in range(num_inputs)] inputs_call_list = ["" for i in range(num_inputs)] @@ -1138,10 +1254,14 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): for name, (ttype, pos) in forward_inputs_position_map.items(): inputs_call_list[pos] = f"{name}" amp_inputs_call_list[pos] = f"new_{name}" - is_optional = (name in optional_inputs) + is_optional = name in optional_inputs if IsPlainTensorType(ttype): if is_optional: - if self.is_forward_only and is_inplaced and forward_inplace_map and name in forward_inplace_map.keys( + if ( + self.is_forward_only + and is_inplaced + and forward_inplace_map + and name in forward_inplace_map.keys() ): arg_str = f"paddle::optional& {name}" else: @@ -1159,7 +1279,10 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): f"auto new_{name} = transformer->TransInTensor(\"{name}\", {name});\n" ) else: - if is_inplaced and forward_inplace_map and name in forward_inplace_map.keys( + if ( + is_inplaced + and forward_inplace_map + and name in forward_inplace_map.keys() ): arg_str = f"paddle::experimental::Tensor& {name}" amp_tensors_vector_list.append(f"{{{name}}}") @@ -1178,7 +1301,11 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): else: assert IsVectorTensorType(ttype) if is_optional: - if self.is_forward_only and is_inplaced and forward_inplace_map and name in forward_inplace_map.keys( + if ( + self.is_forward_only + and is_inplaced + and forward_inplace_map + and name in forward_inplace_map.keys() ): arg_str = f"paddle::optional>& {name}" else: @@ -1193,9 +1320,14 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): f"auto new_{name} = transformer->TransInTensors(\"{name}\", {name});\n" ) else: - if is_inplaced and forward_inplace_map and name in forward_inplace_map.keys( + if ( + is_inplaced + and forward_inplace_map + and name in forward_inplace_map.keys() ): - arg_str = f"std::vector& {name}" + arg_str = ( + f"std::vector& {name}" + ) else: arg_str = f"const std::vector& {name}" amp_tensors_vector_list.append(f"{name}") @@ -1215,7 +1347,8 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): amp_inputs_call_list[pos] = name if default_val is not None: inputs_args_declaration_list[ - pos] = f"{atype} {name} = {default_val}" + pos + ] = f"{atype} {name} = {default_val}" else: inputs_args_declaration_list[pos] = f"{atype} {name}" inputs_args_definition_list[pos] = f"{atype} {name}" @@ -1228,8 +1361,9 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): function_name = forward_api_name if len(intermediate_outputs) > 0: if is_inplaced: - function_name = GetIntermediateAPIFunctionName( - forward_api_name[:-1]) + '_' + function_name = ( + GetIntermediateAPIFunctionName(forward_api_name[:-1]) + '_' + ) else: function_name = GetIntermediateAPIFunctionName(function_name) @@ -1237,12 +1371,14 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): if is_inplaced and len(forward_outputs_position_map) == 1: api_out_type = "auto&" forward_call_str = f"{indent}{api_out_type} api_result = paddle::experimental::{namespace}{function_name}({inputs_call_args_str});" - num_outputs = len( - forward_outputs_position_map.keys()) - len(intermediate_outputs) + num_outputs = len(forward_outputs_position_map.keys()) - len( + intermediate_outputs + ) # Check Nan and Inf check_nan_inf_str = CHECK_NAN_AND_INF_TEMPLATE.format( - function_name, "api_result") + function_name, "api_result" + ) # Get Outputs get_outputs_str = "" @@ -1250,7 +1386,9 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): if num_outputs == 1 and len(intermediate_outputs) == 0: get_outputs_str += f"{indent}auto& {name} = api_result;\n" else: - get_outputs_str += f"{indent}auto& {name} = std::get<{pos}>(api_result);\n" + get_outputs_str += ( + f"{indent}auto& {name} = std::get<{pos}>(api_result);\n" + ) # Get return type list & outputs returns_type_list = ["" for i in range(num_outputs)] @@ -1261,32 +1399,46 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): returns_list[pos] = f"{name}" if IsPlainTensorType(rtype): - if is_inplaced and forward_inplace_map and name in forward_inplace_map.values( + if ( + is_inplaced + and forward_inplace_map + and name in forward_inplace_map.values() ): ind = list(forward_inplace_map.values()).index(name) - if list(forward_inplace_map.keys() - )[ind] in self.optional_inputs: + if ( + list(forward_inplace_map.keys())[ind] + in self.optional_inputs + ): returns_type_list[pos] = inplace_optional_out_type_map[ - rtype] + rtype + ] else: returns_type_list[pos] = "paddle::experimental::Tensor&" else: returns_type_list[pos] = "paddle::experimental::Tensor" else: assert IsVectorTensorType(rtype) - if is_inplaced and forward_inplace_map and name in forward_inplace_map.values( + if ( + is_inplaced + and forward_inplace_map + and name in forward_inplace_map.values() ): ind = list(forward_inplace_map.values()).index(name) - if list(forward_inplace_map.keys() - )[ind] in self.optional_inputs: + if ( + list(forward_inplace_map.keys())[ind] + in self.optional_inputs + ): returns_type_list[pos] = inplace_optional_out_type_map[ - rtype] + rtype + ] else: returns_type_list[ - pos] = "std::vector&" + pos + ] = "std::vector&" else: returns_type_list[ - pos] = "std::vector" + pos + ] = "std::vector" if num_outputs == 1: returns_str = returns_list[0] @@ -1306,29 +1458,40 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): for name, (ttype, pos) in forward_inputs_position_map.items(): # Has corresponding grad output has_corresponding_grad_output = False - for _, (_, corresponding_pos, - _) in backward_grad_outputs_map.items(): + for _, ( + _, + corresponding_pos, + _, + ) in backward_grad_outputs_map.items(): if pos == corresponding_pos: has_corresponding_grad_output = True - if has_corresponding_grad_output or ( - name in forward_inplace_map and forward_api_name - not in inplace_check_blacklist) or self.is_forward_only: + if ( + has_corresponding_grad_output + or ( + name in forward_inplace_map + and forward_api_name not in inplace_check_blacklist + ) + or self.is_forward_only + ): input_autograd_meta_name = GetAutoGradMetaName(name) if IsPlainTensorType(ttype): input_autograd_meta = f"{indent}egr::AutogradMeta* {input_autograd_meta_name} = egr::EagerUtils::nullable_autograd_meta({name});" else: assert IsVectorTensorType(ttype) - input_autograd_meta_vec_name = GetAutoGradMetaVectorName( - name) + input_autograd_meta_vec_name = ( + GetAutoGradMetaVectorName(name) + ) input_autograd_meta = f"{indent}std::vector {input_autograd_meta_vec_name} = egr::EagerUtils::nullable_autograd_meta({name});\n" input_autograd_meta += f"{indent}std::vector* {input_autograd_meta_name} = &{input_autograd_meta_vec_name};" inputs_autograd_meta_list.append(input_autograd_meta) compute_require_grad_args_list.append( - input_autograd_meta_name) + input_autograd_meta_name + ) inputs_autograd_meta_str = "\n".join(inputs_autograd_meta_list) compute_require_grad_args_str = ",".join( - compute_require_grad_args_list) + compute_require_grad_args_list + ) # 2. Get Output AutoGradMeta outputs_autograd_meta_list = [] @@ -1363,11 +1526,16 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): for inplace_name in forward_inplace_map.keys(): if forward_api_name not in inplace_check_blacklist: inplace_autograd_meta_name = GetAutoGradMetaName( - inplace_name) + inplace_name + ) check_inplace_str += CHECK_INPLACE_TEMPLATE.format( - inplace_name, inplace_autograd_meta_name) - bump_inplace_version_str += BUMP_INPLACE_VERSION_TEMPLATE.format( - inplace_name, inplace_name) + inplace_name, inplace_autograd_meta_name + ) + bump_inplace_version_str += ( + BUMP_INPLACE_VERSION_TEMPLATE.format( + inplace_name, inplace_name + ) + ) # Node Creation self.GenerateNodeCreationCodes() @@ -1375,36 +1543,56 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): dygraph_event_str = f"{indent}paddle::platform::RecordEvent dygraph_entrance_record_event(\"{forward_api_name} dygraph\", paddle::platform::TracerEventType::Operator, 1);\n" forward_ad_function_name = GetDygraphForwardFunctionName( - forward_api_name) + forward_api_name + ) # Forward amp logic - kernel_trans2_op_name_str = f"auto op_name = phi::TransToFluidOpName(\"{forward_api_name}\");" - amp_tensors_vector_list_str = "{ " + ",".join( - amp_tensors_vector_list) + " }" + kernel_trans2_op_name_str = ( + f"auto op_name = phi::TransToFluidOpName(\"{forward_api_name}\");" + ) + amp_tensors_vector_list_str = ( + "{ " + ",".join(amp_tensors_vector_list) + " }" + ) amp_tensors_vector_optional_list_str = " ".join( - amp_tensors_vector_optional_list) + amp_tensors_vector_optional_list + ) amp_get_dst_dtype_str = "auto amp_dst_dtype = egr::GetAmpDestDtype(op_name, amp_tensors_vector);\n" - amp_autocast_list_str = " ".join( - amp_autocast_list) + " " + " ".join( - amp_autocast_optional_list) + amp_autocast_list_str = ( + " ".join(amp_autocast_list) + + " " + + " ".join(amp_autocast_optional_list) + ) amp_inputs_call_args_str = ", ".join(amp_inputs_call_list) - amp_call_str = f"return {forward_ad_function_name}({amp_inputs_call_args_str});" + amp_call_str = ( + f"return {forward_ad_function_name}({amp_inputs_call_args_str});" + ) if is_inplaced or (forward_api_name == "cast"): amp_logic_str = "\n VLOG(5) << \" No AMP for {} because it is a inplace or cast api. \"; ".format( - forward_ad_function_name) + forward_ad_function_name + ) else: amp_logic_str = AMP_LOGIC_TEMPLATE.format( - kernel_trans2_op_name_str, amp_tensors_vector_list_str, - amp_tensors_vector_optional_list_str, amp_get_dst_dtype_str, - amp_autocast_list_str, amp_call_str) + kernel_trans2_op_name_str, + amp_tensors_vector_list_str, + amp_tensors_vector_optional_list_str, + amp_get_dst_dtype_str, + amp_autocast_list_str, + amp_call_str, + ) # Forward layout autotune layout_autotune_list_str = " ".join( - layout_autotune_list) + " ".join(layout_autotune_optional_list) + layout_autotune_list + ) + " ".join(layout_autotune_optional_list) layout_logic_str = self.GenerateForwardLayoutAutotune( - forward_api_name, amp_tensors_vector_list, - layout_tensors_vector_optional_list, layout_autotune_list_str, - returns_type_str, returns_str, amp_inputs_call_args_str) + forward_api_name, + amp_tensors_vector_list, + layout_tensors_vector_optional_list, + layout_autotune_list_str, + returns_type_str, + returns_str, + amp_inputs_call_args_str, + ) # For inputs outputs prepare for logging var_str = f"\n{indent} std::string input_str = \"\";" @@ -1426,23 +1614,50 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): if self.is_forward_only: if len(amp_tensors_vector_list) == 0: amp_logic_str = "\n VLOG(7) << \" No AMP for {} because it has no input. \"; ".format( - forward_ad_function_name) - self.forward_definition_str += FORWARD_ONLY_FUNCTION_TEMPLATE.format( - returns_type_str, forward_ad_function_name, - inputs_args_definition_str, forward_api_name, dygraph_event_str, - amp_logic_str, layout_logic_str, forward_api_name, - before_log_str, forward_call_str, get_outputs_str, - forward_api_name, log_str, returns_str) + forward_ad_function_name + ) + self.forward_definition_str += ( + FORWARD_ONLY_FUNCTION_TEMPLATE.format( + returns_type_str, + forward_ad_function_name, + inputs_args_definition_str, + forward_api_name, + dygraph_event_str, + amp_logic_str, + layout_logic_str, + forward_api_name, + before_log_str, + forward_call_str, + get_outputs_str, + forward_api_name, + log_str, + returns_str, + ) + ) else: self.forward_definition_str += FORWARD_FUNCTION_TEMPLATE.format( - returns_type_str, forward_ad_function_name, - inputs_args_definition_str, forward_api_name, dygraph_event_str, - amp_logic_str, layout_logic_str, inputs_autograd_meta_str, - forward_api_name, before_log_str, forward_call_str, - check_nan_inf_str, get_outputs_str, outputs_autograd_meta_str, - compute_require_grad_args_str, check_inplace_str, - bump_inplace_version_str, node_creation_str, forward_api_name, - log_str, returns_str) + returns_type_str, + forward_ad_function_name, + inputs_args_definition_str, + forward_api_name, + dygraph_event_str, + amp_logic_str, + layout_logic_str, + inputs_autograd_meta_str, + forward_api_name, + before_log_str, + forward_call_str, + check_nan_inf_str, + get_outputs_str, + outputs_autograd_meta_str, + compute_require_grad_args_str, + check_inplace_str, + bump_inplace_version_str, + node_creation_str, + forward_api_name, + log_str, + returns_str, + ) self.forward_declaration_str += f"{returns_type_str} {forward_ad_function_name}({inputs_args_declaration_str});\n" @@ -1451,21 +1666,27 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): forward_api_name = self.forward_api_name forward_api_contents = self.forward_api_contents - if forward_api_name != "sum" and "inplace" in forward_api_contents.keys( + if ( + forward_api_name != "sum" + and "inplace" in forward_api_contents.keys() ): # Function Definition and Declaration Generation self.GenerateForwardDefinitionAndDeclaration(is_inplaced=True) self.UpdateCoreOpsInformation(is_inplaced=True) def UpdateCoreOpsInformation(self, is_inplaced): - forward_api_name = GetInplacedFunctionName( - self.forward_api_name) if is_inplaced else self.forward_api_name + forward_api_name = ( + GetInplacedFunctionName(self.forward_api_name) + if is_inplaced + else self.forward_api_name + ) forward_inputs_position_map = self.forward_inputs_position_map forward_outputs_position_map = self.forward_outputs_position_map forward_attrs_list = self.forward_attrs_list - num_args = len( - forward_inputs_position_map.keys()) + len(forward_attrs_list) + num_args = len(forward_inputs_position_map.keys()) + len( + forward_attrs_list + ) num_returns = len(forward_outputs_position_map.keys()) fwd_api_name = "" + forward_api_name @@ -1503,16 +1724,21 @@ class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): class DygraphNodeGenerator(DygraphFunctionGeneratorBase): - - def __init__(self, - forward_api_contents, - grad_api_contents, - forward_apis_dict, - namespace, - next_grad_api_contents=None): - DygraphFunctionGeneratorBase.__init__(self, forward_api_contents, - grad_api_contents, - forward_apis_dict, namespace) + def __init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + next_grad_api_contents=None, + ): + DygraphFunctionGeneratorBase.__init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + ) # Record name mapping from forward_var_name to grad_var_names self.to_next_grad_name_mapping = {} # {name : name} @@ -1569,8 +1795,11 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): backward_api_contents = next_grad_api_contents next_node_generator = DygraphFunctionGeneratorBase( - forward_api_contents, backward_api_contents, forward_apis_dict, - namespace) + forward_api_contents, + backward_api_contents, + forward_apis_dict, + namespace, + ) next_node_generator.run() next_node_generator.GenerateNodeCreationCodes(True) @@ -1579,7 +1808,11 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): self.RecordGrad2NextGradNameMapping(next_node_generator) if next_node_generator is not None: - return next_grad_node_creation_str, next_grad_node_out_list, next_node_generator.backward_forward_inputs_map + return ( + next_grad_node_creation_str, + next_grad_node_out_list, + next_node_generator.backward_forward_inputs_map, + ) else: return next_grad_node_creation_str, next_grad_node_out_list, None @@ -1593,30 +1826,45 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): set_tensor_wrapper_methods_str = "" tensor_wrapper_members_str = "" clear_tensor_wrapper_str = "" - for tname, (ttype, is_fwd_input, - _) in backward_forward_inputs_map.items(): + for tname, ( + ttype, + is_fwd_input, + _, + ) in backward_forward_inputs_map.items(): no_need_buffer = "true" if tname in no_need_buffers else "false" tensor_wrapper_name = GetSavedName(tname) if IsPlainTensorType(ttype): - set_tensor_wrapper_methods_str += SET_PLAIN_TENSOR_WRAPPER_TEMPLATE.format( - tname, tname, tensor_wrapper_name, tname, no_need_buffer) + set_tensor_wrapper_methods_str += ( + SET_PLAIN_TENSOR_WRAPPER_TEMPLATE.format( + tname, tname, tensor_wrapper_name, tname, no_need_buffer + ) + ) - tensor_wrapper_members_str += PLAIN_TENSOR_MEMBER_TEMPLATE.format( - tensor_wrapper_name) + tensor_wrapper_members_str += ( + PLAIN_TENSOR_MEMBER_TEMPLATE.format(tensor_wrapper_name) + ) - clear_tensor_wrapper_str += CLEAR_TENSOR_WRAPPER_TEMPLATE.format( - tensor_wrapper_name) + clear_tensor_wrapper_str += ( + CLEAR_TENSOR_WRAPPER_TEMPLATE.format(tensor_wrapper_name) + ) else: assert IsVectorTensorType(ttype) - set_tensor_wrapper_methods_str += SET_VECTOR_TENSOR_WRAPPER_TEMPLATE.format( - tname, tname, tname, tensor_wrapper_name, no_need_buffer) + set_tensor_wrapper_methods_str += ( + SET_VECTOR_TENSOR_WRAPPER_TEMPLATE.format( + tname, tname, tname, tensor_wrapper_name, no_need_buffer + ) + ) - tensor_wrapper_members_str += VECTOR_TENSOR_MEMBER_TEMPLATE.format( - tensor_wrapper_name) + tensor_wrapper_members_str += ( + VECTOR_TENSOR_MEMBER_TEMPLATE.format(tensor_wrapper_name) + ) - clear_tensor_wrapper_str += CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE.format( - tensor_wrapper_name) + clear_tensor_wrapper_str += ( + CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE.format( + tensor_wrapper_name + ) + ) # SetAttributes & Attribute Members set_attribute_methods_str = "" @@ -1624,27 +1872,44 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): for aname, atype, default_val, _ in backward_attrs_list: saved_attr_name = GetSavedName(aname) set_attribute_methods_str += SET_ATTR_METHOD_TEMPLATE.format( - aname, GetConstReference(atype), aname, saved_attr_name, aname) + aname, GetConstReference(atype), aname, saved_attr_name, aname + ) if default_val: - attribute_members_str += ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE.format( - RemoveConstAndReference(atype), saved_attr_name, - default_val) + attribute_members_str += ( + ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE.format( + RemoveConstAndReference(atype), + saved_attr_name, + default_val, + ) + ) else: attribute_members_str += ATTRIBUTE_MEMBER_TEMPLATE.format( - RemoveConstAndReference(atype), saved_attr_name) + RemoveConstAndReference(atype), saved_attr_name + ) grad_node_name = GetGradNodeName(self.backward_api_name) self.node_declaration_str = NODE_DECLARATION_TEMPLATE.format( - grad_node_name, grad_node_name, grad_node_name, grad_node_name, - grad_node_name, clear_tensor_wrapper_str, grad_node_name, - grad_node_name, set_tensor_wrapper_methods_str, - set_attribute_methods_str, tensor_wrapper_members_str, - attribute_members_str) - - def GenerateNodeDefinition(self, next_grad_node_creation_str, - next_grad_node_out_list, - backward_forward_inputs_map_next): + grad_node_name, + grad_node_name, + grad_node_name, + grad_node_name, + grad_node_name, + clear_tensor_wrapper_str, + grad_node_name, + grad_node_name, + set_tensor_wrapper_methods_str, + set_attribute_methods_str, + tensor_wrapper_members_str, + attribute_members_str, + ) + + def GenerateNodeDefinition( + self, + next_grad_node_creation_str, + next_grad_node_out_list, + backward_forward_inputs_map_next, + ): namespace = self.namespace forward_api_name = self.forward_api_name backward_api_name = self.backward_api_name @@ -1655,21 +1920,30 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): backward_inplace_map = self.backward_inplace_map indent = GetIndent(1) - is_invoke_forward_api = IsInvokeForwardApi(self.grad_api_contents, - self.forward_apis_dict) + is_invoke_forward_api = IsInvokeForwardApi( + self.grad_api_contents, self.forward_apis_dict + ) # Construct grad_api function args # Order: TensorWrappers, GradTensors, Attributes - grad_api_args_len = len(backward_forward_inputs_map.keys()) + len( - backward_grad_inputs_map.keys()) + len(backward_attrs_list) + grad_api_args_len = ( + len(backward_forward_inputs_map.keys()) + + len(backward_grad_inputs_map.keys()) + + len(backward_attrs_list) + ) grad_api_args = ["" for i in range(grad_api_args_len)] get_grad_in_args_list = [] # Fill Grad Ins with Zero fill_zero_str = "" if backward_api_name in ops_to_fill_zero_for_empty_grads: - fill_zero_str = f"{indent}const auto& input_metas = this->InputMeta();\n" - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_inputs_map.items(): + fill_zero_str = ( + f"{indent}const auto& input_metas = this->InputMeta();\n" + ) + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_inputs_map.items(): if name in self.optional_inputs: if IsPlainTensorType(ttype): fill_zero_str += f"{indent}egr::EagerUtils::FillZeroForEmptyOptionalGradInput(&grads[{fwd_position}][0], input_metas[{fwd_position}][0]);\n" @@ -1684,40 +1958,67 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): inplace_check_str = "" optional_inplace_var_name = [] # Grad Ins from TensorWrappers - for name, (backward_input_type, is_fwd_input, - grad_api_position), in backward_forward_inputs_map.items(): + for ( + name, + (backward_input_type, is_fwd_input, grad_api_position), + ) in backward_forward_inputs_map.items(): tensor_wrapper_name = GetSavedName(name) transformed_tensor_name = self.TransformToNextGradName(name) - is_optional = (name in self.optional_inputs) + is_optional = name in self.optional_inputs tensor_wrapper_recover_str = f"{indent}auto {transformed_tensor_name} = egr::EagerUtils::RecoverTensorWrapper(&this->{tensor_wrapper_name});" if backward_inplace_map and name in backward_inplace_map.keys(): if len(next_grad_node_creation_str) > 0: - if (transformed_tensor_name - in backward_forward_inputs_map_next) and ( - backward_forward_inputs_map_next[ - transformed_tensor_name][1]): + if ( + transformed_tensor_name + in backward_forward_inputs_map_next + ) and ( + backward_forward_inputs_map_next[ + transformed_tensor_name + ][1] + ): optional_inplace_var_name.append( - transformed_tensor_name) - tensor_wrapper_intermidiate_tensor_str = f"(&this->{tensor_wrapper_name})->get_intermidiate_tensor()" + transformed_tensor_name + ) + tensor_wrapper_intermidiate_tensor_str = ( + f"(&this->{tensor_wrapper_name})->get_intermidiate_tensor()" + ) inplace_check_str += CHECK_BACKWARD_INPLACE_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, name, - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name, - tensor_wrapper_intermidiate_tensor_str) + transformed_tensor_name, + transformed_tensor_name, + name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + tensor_wrapper_intermidiate_tensor_str, + ) inplace_grad_input_str = transformed_tensor_name if is_optional: if backward_input_type == "std::vector": - tensor_wrapper_recover_str += "\n" + CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name) + tensor_wrapper_recover_str += ( + "\n" + + CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE.format( + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + ) + ) else: - tensor_wrapper_recover_str += "\n" + CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name) + tensor_wrapper_recover_str += ( + "\n" + + CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE.format( + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + ) + ) - grad_api_args[ - grad_api_position] = transformed_tensor_name + "_optional" + grad_api_args[grad_api_position] = ( + transformed_tensor_name + "_optional" + ) else: grad_api_args[grad_api_position] = transformed_tensor_name @@ -1725,37 +2026,57 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): get_grad_in_args_list.append(tensor_wrapper_recover_str) # Grad Ins from grads - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_inputs_map.items(): + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_inputs_map.items(): transformed_tensor_name = self.TransformToNextGradName(name) - is_optional = (name in self.optional_inputs) + is_optional = name in self.optional_inputs if IsPlainTensorType(ttype): get_tensor_str = f"{indent}auto& {transformed_tensor_name} = hooked_grads[{fwd_position}][0];" # Inplace in backward op if backward_inplace_map and name in backward_inplace_map.keys(): if len(next_grad_node_creation_str) > 0: - if (transformed_tensor_name - in backward_forward_inputs_map_next) and ( - backward_forward_inputs_map_next[ - transformed_tensor_name][1]): + if ( + transformed_tensor_name + in backward_forward_inputs_map_next + ) and ( + backward_forward_inputs_map_next[ + transformed_tensor_name + ][1] + ): optional_inplace_var_name.append( - transformed_tensor_name) + transformed_tensor_name + ) grads_tensor_str = f"grads[{fwd_position}][0]" inplace_check_str += CHECK_BACKWARD_INPLACE_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, name, - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name, - grads_tensor_str) + transformed_tensor_name, + transformed_tensor_name, + name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + grads_tensor_str, + ) inplace_grad_input_str = transformed_tensor_name if is_optional: - get_tensor_str += "\n" + CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name) + get_tensor_str += ( + "\n" + + CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE.format( + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + ) + ) grad_api_args[ - grad_api_position] = f"{transformed_tensor_name}_optional" + grad_api_position + ] = f"{transformed_tensor_name}_optional" else: grad_api_args[grad_api_position] = transformed_tensor_name else: @@ -1768,7 +2089,9 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): # Grad Attrs for name, _, _, grad_api_position in backward_attrs_list: saved_attribute_name = GetSavedName(name) - get_attr_str = f"{indent}auto& {name} = this->{saved_attribute_name};" + get_attr_str = ( + f"{indent}auto& {name} = this->{saved_attribute_name};" + ) grad_api_args[grad_api_position] = name get_grad_in_args_list.append(get_attr_str) @@ -1790,13 +2113,18 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): # Grad Outputs out_index = -1 out_assign_str = "" - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_outputs_map.items(): + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_outputs_map.items(): transformed_tensor_name = self.TransformToNextGradName(name) out_index = out_index + 1 if is_invoke_forward_api: if len(backward_grad_outputs_map) == 1: - out_assign_str += f"{indent}*api_output_{out_index} = api_output;\n" + out_assign_str += ( + f"{indent}*api_output_{out_index} = api_output;\n" + ) else: out_assign_str += f"{indent}*api_output_{out_index} = std::get<{out_index}>(api_output);\n" else: @@ -1809,7 +2137,9 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): }}""" if IsPlainTensorType(ttype): - if backward_inplace_map and name in backward_inplace_map.values( + if ( + backward_inplace_map + and name in backward_inplace_map.values() ): inplace_str = f""" if (api_output_{out_index} != nullptr && can_be_inplaced) {{ egr::EagerUtils::HandleViewBetweenInputAndOutput({inplace_grad_input_str}, api_output_{out_index}); @@ -1844,14 +2174,19 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): if is_invoke_forward_api: autograd_api_out = "auto" - if len(self.backward_inplace_map) > 0 and len( - backward_grad_outputs_map) == 1: + if ( + len(self.backward_inplace_map) > 0 + and len(backward_grad_outputs_map) == 1 + ): autograd_api_out = "auto&" - forward_api_name = self.grad_api_contents['invoke'].split( - '(')[0].strip() + forward_api_name = ( + self.grad_api_contents['invoke'].split('(')[0].strip() + ) autograd_api = self.grad_api_contents['invoke'].replace( forward_api_name, - GetDygraphForwardFunctionName(forward_api_name), 1) + GetDygraphForwardFunctionName(forward_api_name), + 1, + ) grad_function_call_str = f""" if (trace_backward) {{ {indent}{autograd_api_out} api_output = {autograd_api}; @@ -1865,7 +2200,8 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): # Check Nan and Inf check_nan_inf_str = CHECK_NAN_AND_INF_TEMPLATE.format( - backward_api_name, "returns") + backward_api_name, "returns" + ) # Prepare for Node Creation if Necessary outputs_autograd_meta_str = "" @@ -1878,14 +2214,19 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): # TODO(jiabin): Optimize this with SetStopGradient instead of Pass Stop gradient num_fwd_outputs = len(backward_grad_outputs_map.keys()) - for name, (rtype, pos, - grad_api_position) in backward_grad_outputs_map.items(): + for name, ( + rtype, + pos, + grad_api_position, + ) in backward_grad_outputs_map.items(): transformed_tensor_name = self.TransformToNextGradName(name) output_autograd_meta_name = GetAutoGradMetaName( - transformed_tensor_name) + transformed_tensor_name + ) output_autograd_meta_vec_name = GetAutoGradMetaVectorName( - transformed_tensor_name) + transformed_tensor_name + ) if IsPlainTensorType(rtype): output_autograd_meta = f""" auto& {transformed_tensor_name} = returns[{pos}][0]; @@ -1923,15 +2264,20 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): # For inputs outputs prepare for logging var_str = f"\n{indent} std::string input_str = \"\";" var_str += f"\n{indent} std::string output_str = \"\";" - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_inputs_map.items(): + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_inputs_map.items(): new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string input_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" var_str += f"\n{indent} input_str += input_{new_name}_str; " - for name, (backward_input_type, is_fwd_input, - grad_api_position), in backward_forward_inputs_map.items(): + for ( + name, + (backward_input_type, is_fwd_input, grad_api_position), + ) in backward_forward_inputs_map.items(): new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string input_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" @@ -1939,8 +2285,11 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): before_log_str = BEFORE_LOG_PRINT_TEMPLATE.format(var_str) - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_outputs_map.items(): + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_outputs_map.items(): new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n ( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string output_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" @@ -1949,13 +2298,24 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): log_str = AFTER_LOG_PRINT_TEMPLATE.format(var_str) self.node_definition_str = GRAD_FUNCTION_TEMPLATE.format( - grad_node_name, self.backward_api_name, fill_zero_str, - get_grad_in_args_str, grad_function_prepare_str, - compute_require_next_grad_str, inplace_check_str, - inplace_for_grad_outs_str, self.backward_api_name, before_log_str, - grad_function_call_str, check_nan_inf_str, - outputs_autograd_meta_str, next_grad_node_creation_str, - self.backward_api_name, log_str, returns_str) + grad_node_name, + self.backward_api_name, + fill_zero_str, + get_grad_in_args_str, + grad_function_prepare_str, + compute_require_next_grad_str, + inplace_check_str, + inplace_for_grad_outs_str, + self.backward_api_name, + before_log_str, + grad_function_call_str, + check_nan_inf_str, + outputs_autograd_meta_str, + next_grad_node_creation_str, + self.backward_api_name, + log_str, + returns_str, + ) def run(self): super().run() @@ -1966,18 +2326,22 @@ class DygraphNodeGenerator(DygraphFunctionGeneratorBase): ## Code Generation ## ##################### # Higher-order GradNode generation - next_grad_node_creation_str, next_grad_node_out_list, backward_forward_inputs_map = self.GenerateHigherOrderNodeCreationCode( - ) + ( + next_grad_node_creation_str, + next_grad_node_out_list, + backward_forward_inputs_map, + ) = self.GenerateHigherOrderNodeCreationCode() self.GenerateNodeDeclaration() - self.GenerateNodeDefinition(next_grad_node_creation_str, - next_grad_node_out_list, - backward_forward_inputs_map) + self.GenerateNodeDefinition( + next_grad_node_creation_str, + next_grad_node_out_list, + backward_forward_inputs_map, + ) class DygraphForwardAndNodesGenerator(GeneratorBase): - def __init__(self, api_yaml_path, backward_yaml_path): # Parent members: # self.namespace @@ -1995,8 +2359,9 @@ class DygraphForwardAndNodesGenerator(GeneratorBase): self.node_definition_str = "" def CollectIsForwardOnly(self, forward_api_contents): - self.is_forward_only = False if 'backward' in forward_api_contents.keys( - ) else True + self.is_forward_only = ( + False if 'backward' in forward_api_contents.keys() else True + ) def ParseYamlContents(self): self.ParseForwardYamlContents() @@ -2010,11 +2375,13 @@ class DygraphForwardAndNodesGenerator(GeneratorBase): def GetBackwardAPIContents(self, forward_api_contents): grad_api_dict = self.grad_api_dict - if 'backward' not in forward_api_contents.keys(): return None + if 'backward' not in forward_api_contents.keys(): + return None backward_api_name = forward_api_contents['backward'] assert backward_api_name in grad_api_dict.keys(), AssertMessage( - backward_api_name, grad_api_dict.keys()) + backward_api_name, grad_api_dict.keys() + ) backward_api_contents = grad_api_dict[backward_api_name] return backward_api_contents @@ -2028,7 +2395,8 @@ class DygraphForwardAndNodesGenerator(GeneratorBase): namespace = self.namespace for forward_api_contents in forward_api_list: - if forward_api_contents['op'] in black_ops_list: continue + if forward_api_contents['op'] in black_ops_list: + continue self.CollectIsForwardOnly(forward_api_contents) @@ -2036,34 +2404,50 @@ class DygraphForwardAndNodesGenerator(GeneratorBase): backward_api_contents = None else: backward_api_contents = self.GetBackwardAPIContents( - forward_api_contents) + forward_api_contents + ) # Generate Dygraph Forward Function function_generator = DygraphForwardFunctionGenerator( - forward_api_contents, backward_api_contents, forward_apis_dict, - namespace) + forward_api_contents, + backward_api_contents, + forward_apis_dict, + namespace, + ) function_generator.run() - self.forward_definition_str += function_generator.forward_definition_str + "\n" - self.forward_declaration_str += function_generator.forward_declaration_str + "\n" + self.forward_definition_str += ( + function_generator.forward_definition_str + "\n" + ) + self.forward_declaration_str += ( + function_generator.forward_declaration_str + "\n" + ) # Generate Dygraph GradNode Function while True: if backward_api_contents is None: break next_grad_api_contents = self.GetBackwardAPIContents( - backward_api_contents) + backward_api_contents + ) - node_generator = DygraphNodeGenerator(forward_api_contents, - backward_api_contents, - forward_apis_dict, - namespace, - next_grad_api_contents) + node_generator = DygraphNodeGenerator( + forward_api_contents, + backward_api_contents, + forward_apis_dict, + namespace, + next_grad_api_contents, + ) node_generator.run() - self.node_declaration_str += node_generator.node_declaration_str + "\n" - self.node_definition_str += node_generator.node_definition_str + "\n" + self.node_declaration_str += ( + node_generator.node_declaration_str + "\n" + ) + self.node_definition_str += ( + node_generator.node_definition_str + "\n" + ) - if next_grad_api_contents is None: break + if next_grad_api_contents is None: + break # Detect if there exists higher-order GradNode forward_api_contents = backward_api_contents @@ -2076,13 +2460,17 @@ class DygraphForwardAndNodesGenerator(GeneratorBase): if namespace.endswith("::"): namespace = namespace[:-2] self.forward_definition_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.forward_definition_str) + namespace, self.forward_definition_str + ) self.forward_declaration_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.forward_declaration_str) + namespace, self.forward_declaration_str + ) self.node_declaration_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.node_declaration_str) + namespace, self.node_declaration_str + ) self.node_definition_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.node_definition_str) + namespace, self.node_definition_str + ) def run(self): self.ParseYamlContents() @@ -2118,8 +2506,9 @@ def GenerateForwardCCFile(filepath, forward_definition_str): os.remove(filepath) core_ops_info_str = GenerateCoreOpInfoDefinition() - file_contents = FORWARD_CC_FILE_TEMPLATE.format(core_ops_info_str, - forward_definition_str) + file_contents = FORWARD_CC_FILE_TEMPLATE.format( + core_ops_info_str, forward_definition_str + ) with open(filepath, 'a') as f: f.write(file_contents) @@ -2130,7 +2519,8 @@ def GenerateForwardHFile(filepath, forward_function_declaration_str): core_ops_info_str = GenerateCoreOpInfoDeclaration() file_contents = FORWARD_H_FILE_TEMPLATE.format( - core_ops_info_str, forward_function_declaration_str) + core_ops_info_str, forward_function_declaration_str + ) with open(filepath, 'a') as f: f.write(file_contents) @@ -2157,8 +2547,9 @@ if __name__ == "__main__": else: backward_yaml_path = None - generator = DygraphForwardAndNodesGenerator(api_yaml_path, - backward_yaml_path) + generator = DygraphForwardAndNodesGenerator( + api_yaml_path, backward_yaml_path + ) generator.run() node_declaration_str += generator.node_declaration_str + "\n" diff --git a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py index 5aac3574ba7..dc38d3d46b2 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py @@ -25,7 +25,7 @@ skipped_forward_api_names = set([]) def SkipAPIGeneration(forward_api_name): - return (forward_api_name in skipped_forward_api_names) + return forward_api_name in skipped_forward_api_names atype_to_parsing_function = { @@ -61,28 +61,25 @@ def FindParsingFunctionFromAttributeType(atype): ########################## ## Refactored Functions ## ########################## -PARSE_PYTHON_C_TENSORS_TEMPLATE = \ -" auto {} = {}(\"{}\", \"{}\", args, {}, {});\n" +PARSE_PYTHON_C_TENSORS_TEMPLATE = ( + " auto {} = {}(\"{}\", \"{}\", args, {}, {});\n" +) -PARSE_PYTHON_C_ARGS_TEMPLATE = \ -""" PyObject* {}_obj = PyTuple_GET_ITEM(args, {}); +PARSE_PYTHON_C_ARGS_TEMPLATE = """ PyObject* {}_obj = PyTuple_GET_ITEM(args, {}); {} {} = {}({}_obj, \"{}\", {}); """ -RECORD_EVENT_TEMPLATE = \ -"paddle::platform::RecordEvent {}(\"{} {}\", paddle::platform::TracerEventType::UserDefined, 1);" +RECORD_EVENT_TEMPLATE = "paddle::platform::RecordEvent {}(\"{} {}\", paddle::platform::TracerEventType::UserDefined, 1);" -RETURN_INPLACE_PYOBJECT_TEMPLATE = \ -""" +RETURN_INPLACE_PYOBJECT_TEMPLATE = """ inplace_var_idx_map[{}] = {}; """ -PYTHON_C_FUNCTION_TEMPLATE = \ -""" +PYTHON_C_FUNCTION_TEMPLATE = """ static PyObject * eager_api_{}(PyObject *self, PyObject *args, PyObject *kwargs) {{ {} PyThreadState *tstate = nullptr; @@ -115,8 +112,7 @@ static PyObject * eager_api_{}(PyObject *self, PyObject *args, PyObject *kwargs) NOAMP_DYGRAPH_FUNCTION_TEMPLATE = "decltype({}({})) out = {}({});" -FUNCTION_SET_DEVICE_TEMPLATE = \ -"""{} if (paddle::platform::is_gpu_place(place)) {{ +FUNCTION_SET_DEVICE_TEMPLATE = """{} if (paddle::platform::is_gpu_place(place)) {{ #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) phi::backends::gpu::SetDeviceId(place.device); VLOG(1) <<"CurrentDeviceId: " << phi::backends::gpu::GetCurrentDeviceId() << " from " << (int)place.device; @@ -136,16 +132,13 @@ FUNCTION_SET_DEVICE_TEMPLATE = \ }} """ -FUNCTION_NAME_TEMPLATE = \ -"{}{}{}" +FUNCTION_NAME_TEMPLATE = "{}{}{}" -PYTHON_C_FUNCTION_REG_TEMPLATE = \ -" {{\"{}{}\", (PyCFunction)(void(*)(void)) {}eager_api_{}, METH_VARARGS | METH_KEYWORDS, \"C++ interface function for {} in dygraph.\"}},\n" +PYTHON_C_FUNCTION_REG_TEMPLATE = " {{\"{}{}\", (PyCFunction)(void(*)(void)) {}eager_api_{}, METH_VARARGS | METH_KEYWORDS, \"C++ interface function for {} in dygraph.\"}},\n" -PYTHON_C_WRAPPER_TEMPLATE = \ -""" +PYTHON_C_WRAPPER_TEMPLATE = """ #include #include "paddle/fluid/platform/enforce.h" #include "paddle/phi/api/include/strings_api.h" @@ -184,8 +177,7 @@ void BindFinalStateEagerOpFunctions(pybind11::module *module) {{ """ -CORE_OPS_INFO = \ -""" +CORE_OPS_INFO = """ static PyObject * eager_get_core_ops_args_info(PyObject *self) { PyThreadState *tstate = nullptr; try { @@ -230,15 +222,13 @@ static PyObject * eager_get_core_ops_returns_info(PyObject *self) { """ -CORE_OPS_INFO_REGISTRY = \ -""" +CORE_OPS_INFO_REGISTRY = """ {\"get_core_ops_args_info\", (PyCFunction)(void(*)(void))eager_get_core_ops_args_info, METH_NOARGS, \"C++ interface function for eager_get_core_ops_args_info.\"}, {\"get_core_ops_args_type_info\", (PyCFunction)(void(*)(void))eager_get_core_ops_args_type_info, METH_NOARGS, \"C++ interface function for eager_get_core_ops_args_type_info.\"}, {\"get_core_ops_returns_info\", (PyCFunction)(void(*)(void))eager_get_core_ops_returns_info, METH_NOARGS, \"C++ interface function for eager_get_core_ops_returns_info.\"}, """ -NAMESPACE_WRAPPER_TEMPLATE = \ -"""namespace {} {{ +NAMESPACE_WRAPPER_TEMPLATE = """namespace {} {{ {} }} """ @@ -248,21 +238,20 @@ NAMESPACE_WRAPPER_TEMPLATE = \ ## Generator Classes ## ####################### class PythonCSingleFunctionGenerator(FunctionGeneratorBase): - def __init__(self, forward_api_contents, namespace): # Members from Parent: - #self.namespace - #self.forward_api_contents - #self.forward_api_name - #self.orig_forward_inputs_list - #self.orig_forward_attrs_list - #self.orig_forward_returns_list - #self.forward_inputs_position_map - #self.forward_outputs_position_map - #self.optional_inputs - #self.no_need_buffers - #self.intermediate_outputs - #self.forward_inplace_map + # self.namespace + # self.forward_api_contents + # self.forward_api_name + # self.orig_forward_inputs_list + # self.orig_forward_attrs_list + # self.orig_forward_returns_list + # self.forward_inputs_position_map + # self.forward_outputs_position_map + # self.optional_inputs + # self.no_need_buffers + # self.intermediate_outputs + # self.forward_inplace_map FunctionGeneratorBase.__init__(self, forward_api_contents, namespace) self.is_forward_only = True @@ -273,8 +262,9 @@ class PythonCSingleFunctionGenerator(FunctionGeneratorBase): def CollectIsForwardOnly(self): forward_api_contents = self.forward_api_contents - self.is_forward_only = False if 'backward' in forward_api_contents.keys( - ) else True + self.is_forward_only = ( + False if 'backward' in forward_api_contents.keys() else True + ) def GeneratePythonCFunction(self): namespace = self.namespace @@ -293,25 +283,53 @@ class PythonCSingleFunctionGenerator(FunctionGeneratorBase): for name, (ttype, pos) in forward_inputs_position_map.items(): if forward_inplace_map and name in forward_inplace_map.keys(): inplace_args_pos_map[name] = pos - is_optional = (name in optional_inputs) + is_optional = name in optional_inputs if IsVectorTensorType(ttype): if is_optional: - get_eager_tensor_str += PARSE_PYTHON_C_TENSORS_TEMPLATE.format( - name, "GetOptionalTensorListFromArgs", forward_api_name, - name, pos, "true") + get_eager_tensor_str += ( + PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + name, + "GetOptionalTensorListFromArgs", + forward_api_name, + name, + pos, + "true", + ) + ) else: - get_eager_tensor_str += PARSE_PYTHON_C_TENSORS_TEMPLATE.format( - name, "GetTensorListFromArgs", forward_api_name, name, - pos, "false") + get_eager_tensor_str += ( + PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + name, + "GetTensorListFromArgs", + forward_api_name, + name, + pos, + "false", + ) + ) else: if is_optional: - get_eager_tensor_str += PARSE_PYTHON_C_TENSORS_TEMPLATE.format( - name, "GetOptionalTensorFromArgs", forward_api_name, - name, pos, "true") + get_eager_tensor_str += ( + PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + name, + "GetOptionalTensorFromArgs", + forward_api_name, + name, + pos, + "true", + ) + ) else: - get_eager_tensor_str += PARSE_PYTHON_C_TENSORS_TEMPLATE.format( - name, "GetTensorFromArgs", forward_api_name, name, pos, - "false") + get_eager_tensor_str += ( + PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + name, + "GetTensorFromArgs", + forward_api_name, + name, + pos, + "false", + ) + ) if forward_inplace_map: for name, (ttype, pos) in forward_outputs_position_map.items(): @@ -319,26 +337,40 @@ class PythonCSingleFunctionGenerator(FunctionGeneratorBase): inplace_returns_pos_map[name] = pos parse_attributes_str = "" - expected_place_str = " auto place = egr::Controller::Instance().GetExpectedPlace();\n" + expected_place_str = ( + " auto place = egr::Controller::Instance().GetExpectedPlace();\n" + ) # Generate Python-C Attributes Parsing Logic for name, atype, _, pos in orig_forward_attrs_list: parsing_function_name = FindParsingFunctionFromAttributeType(atype) # Used input argument place if specified from Python frontend. - if len(expected_place_str - ) != 0 and parsing_function_name == "CastPyArg2Place": + if ( + len(expected_place_str) != 0 + and parsing_function_name == "CastPyArg2Place" + ): expected_place_str = "" - assert name == "place", "Only support 'place' as template argument name in FUNCTION_SET_DEVICE_TEMPLATE." + assert ( + name == "place" + ), "Only support 'place' as template argument name in FUNCTION_SET_DEVICE_TEMPLATE." parse_attributes_str += PARSE_PYTHON_C_ARGS_TEMPLATE.format( - name, pos, atype, name, parsing_function_name, name, - forward_api_name, pos) + name, + pos, + atype, + name, + parsing_function_name, + name, + forward_api_name, + pos, + ) set_device_str = FUNCTION_SET_DEVICE_TEMPLATE.format(expected_place_str) # Generate Dygraph Function Call Logic - num_args = len( - forward_inputs_position_map.keys()) + len(orig_forward_attrs_list) + num_args = len(forward_inputs_position_map.keys()) + len( + orig_forward_attrs_list + ) dygraph_function_call_list = ["" for i in range(num_args)] for name, (_, pos) in forward_inputs_position_map.items(): dygraph_function_call_list[pos] = f"{name}" @@ -348,23 +380,34 @@ class PythonCSingleFunctionGenerator(FunctionGeneratorBase): # Generate Python-C Function Definitions fwd_function_name = FUNCTION_NAME_TEMPLATE.format( - "::", namespace, GetForwardFunctionName(forward_api_name)) + "::", namespace, GetForwardFunctionName(forward_api_name) + ) return_str = " return ToPyObject(out);" # Generate Record Event for performance profiling pythonc_record_event_str = RECORD_EVENT_TEMPLATE.format( - "pythonc_record_event", forward_api_name, "pybind_imperative_func") + "pythonc_record_event", forward_api_name, "pybind_imperative_func" + ) noamp_dygraph_function_str = NOAMP_DYGRAPH_FUNCTION_TEMPLATE.format( - fwd_function_name, dygraph_function_call_str, fwd_function_name, - dygraph_function_call_str) + fwd_function_name, + dygraph_function_call_str, + fwd_function_name, + dygraph_function_call_str, + ) # Generate Python-C Function Definetion self.python_c_function_str = PYTHON_C_FUNCTION_TEMPLATE.format( - forward_api_name, pythonc_record_event_str, forward_api_name, - get_eager_tensor_str, parse_attributes_str, set_device_str, - noamp_dygraph_function_str, return_str) + forward_api_name, + pythonc_record_event_str, + forward_api_name, + get_eager_tensor_str, + parse_attributes_str, + set_device_str, + noamp_dygraph_function_str, + return_str, + ) # Set prefix of forward_api_name to avoid conflicts prefix = self.namespace.strip("::") @@ -372,37 +415,63 @@ class PythonCSingleFunctionGenerator(FunctionGeneratorBase): # Generate Python-C Function Registration self.python_c_function_reg_str = PYTHON_C_FUNCTION_REG_TEMPLATE.format( - forward_api_name_prefix, forward_api_name, namespace, - forward_api_name, forward_api_name) + forward_api_name_prefix, + forward_api_name, + namespace, + forward_api_name, + forward_api_name, + ) if forward_inplace_map: inplaced_forward_api_name = GetInplacedFunctionName( - self.forward_api_name) + self.forward_api_name + ) inplaced_fwd_function_name = FUNCTION_NAME_TEMPLATE.format( - "::", namespace, - GetForwardFunctionName(inplaced_forward_api_name)) - - inplace_noamp_dygraph_function_str = NOAMP_DYGRAPH_FUNCTION_TEMPLATE.format( - inplaced_fwd_function_name, dygraph_function_call_str, - inplaced_fwd_function_name, dygraph_function_call_str) + "::", + namespace, + GetForwardFunctionName(inplaced_forward_api_name), + ) + + inplace_noamp_dygraph_function_str = ( + NOAMP_DYGRAPH_FUNCTION_TEMPLATE.format( + inplaced_fwd_function_name, + dygraph_function_call_str, + inplaced_fwd_function_name, + dygraph_function_call_str, + ) + ) return_str = " std::map inplace_var_idx_map;" for inplace_input, inplace_output in forward_inplace_map.items(): return_str += RETURN_INPLACE_PYOBJECT_TEMPLATE.format( inplace_returns_pos_map[inplace_output], - inplace_args_pos_map[inplace_input]) - return_str += " return ToPyObject(out, args, inplace_var_idx_map);" + inplace_args_pos_map[inplace_input], + ) + return_str += ( + " return ToPyObject(out, args, inplace_var_idx_map);" + ) # Generate Python-C Function Definetion python_c_inplace_func_str = PYTHON_C_FUNCTION_TEMPLATE.format( - inplaced_forward_api_name, pythonc_record_event_str, - inplaced_forward_api_name, get_eager_tensor_str, - parse_attributes_str, set_device_str, - inplace_noamp_dygraph_function_str, return_str) - - python_c_inplace_func_reg_str = PYTHON_C_FUNCTION_REG_TEMPLATE.format( - forward_api_name_prefix, inplaced_forward_api_name, namespace, - inplaced_forward_api_name, inplaced_forward_api_name) + inplaced_forward_api_name, + pythonc_record_event_str, + inplaced_forward_api_name, + get_eager_tensor_str, + parse_attributes_str, + set_device_str, + inplace_noamp_dygraph_function_str, + return_str, + ) + + python_c_inplace_func_reg_str = ( + PYTHON_C_FUNCTION_REG_TEMPLATE.format( + forward_api_name_prefix, + inplaced_forward_api_name, + namespace, + inplaced_forward_api_name, + inplaced_forward_api_name, + ) + ) # self.forward_api_name ending with '_' means it only has inplace api if self.forward_api_name[-1] == '_': @@ -427,11 +496,13 @@ class PythonCSingleFunctionGenerator(FunctionGeneratorBase): # Initialized orig_forward_inputs_list, orig_forward_returns_list, orig_forward_attrs_list self.CollectOriginalForwardInfo() - if SkipAPIGeneration(self.forward_api_name): return False + if SkipAPIGeneration(self.forward_api_name): + return False # Initialized forward_inputs_position_map, forward_outputs_position_map - self.DetermineForwardPositionMap(self.orig_forward_inputs_list, - self.orig_forward_returns_list) + self.DetermineForwardPositionMap( + self.orig_forward_inputs_list, self.orig_forward_returns_list + ) # Code Generation self.GeneratePythonCFunction() @@ -440,7 +511,6 @@ class PythonCSingleFunctionGenerator(FunctionGeneratorBase): class PythonCGenerator(GeneratorBase): - def __init__(self, path): # Parent members: # self.namespace @@ -458,12 +528,17 @@ class PythonCGenerator(GeneratorBase): forward_api_list = self.forward_api_list for forward_api_content in forward_api_list: f_generator = PythonCSingleFunctionGenerator( - forward_api_content, namespace) + forward_api_content, namespace + ) status = f_generator.run() if status == True: - self.python_c_functions_str += f_generator.python_c_function_str + "\n" - self.python_c_functions_reg_str += f_generator.python_c_function_reg_str + self.python_c_functions_str += ( + f_generator.python_c_function_str + "\n" + ) + self.python_c_functions_reg_str += ( + f_generator.python_c_function_reg_str + ) def AttachNamespace(self): namespace = self.namespace @@ -473,7 +548,8 @@ class PythonCGenerator(GeneratorBase): if namespace.endswith("::"): namespace = namespace[:-2] self.python_c_functions_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, python_c_functions_str) + namespace, python_c_functions_str + ) def run(self): # Infer namespace from yaml_path @@ -494,7 +570,8 @@ class PythonCGenerator(GeneratorBase): ############################ def ParseArguments(): parser = argparse.ArgumentParser( - description='Eager Code Generator Args Parser') + description='Eager Code Generator Args Parser' + ) parser.add_argument('--api_yaml_path', type=str) parser.add_argument('--output_path', type=str) @@ -508,15 +585,18 @@ def GenerateCoreOpsInfoMap(): def GeneratePythonCWrappers(python_c_function_str, python_c_function_reg_str): - core_ops_infos_definition, core_ops_infos_registry = GenerateCoreOpsInfoMap( - ) + ( + core_ops_infos_definition, + core_ops_infos_registry, + ) = GenerateCoreOpsInfoMap() python_c_function_str += core_ops_infos_definition python_c_function_reg_str += core_ops_infos_registry python_c_function_reg_str += " {nullptr,nullptr,0,nullptr}" - python_c_str = PYTHON_C_WRAPPER_TEMPLATE.format(python_c_function_str, - python_c_function_reg_str) + python_c_str = PYTHON_C_WRAPPER_TEMPLATE.format( + python_c_function_str, python_c_function_reg_str + ) return python_c_str @@ -538,11 +618,16 @@ if __name__ == "__main__": py_c_generator = PythonCGenerator(api_yaml_path) py_c_generator.run() - generated_python_c_functions += py_c_generator.python_c_functions_str + "\n" - generated_python_c_registration += py_c_generator.python_c_functions_reg_str + generated_python_c_functions += ( + py_c_generator.python_c_functions_str + "\n" + ) + generated_python_c_registration += ( + py_c_generator.python_c_functions_reg_str + ) - python_c_str = GeneratePythonCWrappers(generated_python_c_functions, - generated_python_c_registration) + python_c_str = GeneratePythonCWrappers( + generated_python_c_functions, generated_python_c_registration + ) output_path = args.output_path for path in [output_path]: diff --git a/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py b/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py index 5cda10f873c..219ce72077c 100644 --- a/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py +++ b/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py @@ -94,8 +94,9 @@ def download_concat(cache_folder, zip_path): def print_processbar(done_percentage): done_filled = done_percentage * '=' empty_filled = (100 - done_percentage) * ' ' - sys.stdout.write("\r[%s%s]%d%%" % - (done_filled, empty_filled, done_percentage)) + sys.stdout.write( + "\r[%s%s]%d%%" % (done_filled, empty_filled, done_percentage) + ) sys.stdout.flush() @@ -137,7 +138,7 @@ def convert_Imagenet_tar2bin(tar_file, output_file): val_dict[name] = label for img_name in dataset.keys(): - remove_len = (len(FOLDER_NAME)) + remove_len = len(FOLDER_NAME) img_name_prim = img_name[remove_len:] label = val_dict[img_name_prim] label_int = (int)(label) @@ -156,19 +157,25 @@ def run_convert(): retry = 0 try_limit = 3 - while not (os.path.exists(output_file) - and os.path.getsize(output_file) == FULL_SIZE_BYTES): + while not ( + os.path.exists(output_file) + and os.path.getsize(output_file) == FULL_SIZE_BYTES + ): if os.path.exists(output_file): sys.stderr.write( - "\n\nThe existing binary file[{}] is broken. Start to generate new one...\n\n" - .format(output_file)) + "\n\nThe existing binary file[{}] is broken. Start to generate new one...\n\n".format( + output_file + ) + ) os.remove(output_file) if retry < try_limit: retry = retry + 1 else: raise RuntimeError( - "Can not convert the dataset to binary file with try limit {0}". - format(try_limit)) + "Can not convert the dataset to binary file with try limit {0}".format( + try_limit + ) + ) download_concat(cache_folder, zip_path) convert_Imagenet_tar2bin(zip_path, output_file) print("\nSuccess! The binary file can be found at {0}".format(output_file)) @@ -193,63 +200,75 @@ def convert_Imagenet_local2bin(args): if not os.path.exists(img_path): continue - #save image(float32) to file + # save image(float32) to file img = Image.open(img_path) img = process_image(img) np_img = np.array(img) - of.seek(SIZE_INT64 + - SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * idx) + of.seek( + SIZE_INT64 + SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * idx + ) of.write(np_img.astype('float32').tobytes()) - #save label(int64_t) to file + # save label(int64_t) to file label_int = (int)(label) np_label = np.array(label_int) - of.seek(SIZE_INT64 + - SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * num_images + - idx * SIZE_INT64) + of.seek( + SIZE_INT64 + + SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * num_images + + idx * SIZE_INT64 + ) of.write(np_label.astype('int64').tobytes()) # The bin file should contain # number of images + all images data + all corresponding labels # so the file target_size should be as follows - target_size = SIZE_INT64 + num_images * 3 * args.data_dim * args.data_dim * SIZE_FLOAT32 + num_images * SIZE_INT64 - if (os.path.getsize(bin_file_path) == target_size): + target_size = ( + SIZE_INT64 + + num_images * 3 * args.data_dim * args.data_dim * SIZE_FLOAT32 + + num_images * SIZE_INT64 + ) + if os.path.getsize(bin_file_path) == target_size: print( - "Success! The user data output binary file can be found at: {0}" - .format(bin_file_path)) + "Success! The user data output binary file can be found at: {0}".format( + bin_file_path + ) + ) else: print("Conversion failed!") def main_preprocess_Imagenet(args): parser = argparse.ArgumentParser( - description= - "Convert the full Imagenet val set or local data to binary file.", + description="Convert the full Imagenet val set or local data to binary file.", usage=None, - add_help=True) + add_help=True, + ) parser.add_argument( '--local', action="store_true", - help="If used, user need to set --data_dir and then convert file") - parser.add_argument("--data_dir", - default="", - type=str, - help="Dataset root directory") + help="If used, user need to set --data_dir and then convert file", + ) + parser.add_argument( + "--data_dir", default="", type=str, help="Dataset root directory" + ) parser.add_argument( "--label_list", type=str, default="val_list.txt", - help= - "List of object labels with same sequence as denoted in the annotation file" + help="List of object labels with same sequence as denoted in the annotation file", + ) + parser.add_argument( + "--output_file", + type=str, + default="imagenet_small.bin", + help="File path of the output binary file", + ) + parser.add_argument( + "--data_dim", + type=int, + default=DATA_DIM, + help="Image preprocess with data_dim width and height", ) - parser.add_argument("--output_file", - type=str, - default="imagenet_small.bin", - help="File path of the output binary file") - parser.add_argument("--data_dim", - type=int, - default=DATA_DIM, - help="Image preprocess with data_dim width and height") args = parser.parse_args() if args.local: diff --git a/paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py b/paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py index 74739530b47..6fc2e072e00 100644 --- a/paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py +++ b/paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py @@ -24,7 +24,9 @@ import hashlib import tarfile import argparse -DATA_URL = "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar" +DATA_URL = ( + "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar" +) DATA_DIR = os.path.expanduser("~/.cache/paddle/dataset/pascalvoc/") TAR_FILE = "VOCtest_06-Nov-2007.tar" TAR_PATH = os.path.join(DATA_DIR, TAR_FILE) @@ -106,8 +108,9 @@ def convert_pascalvoc_local2bin(args): for object in objects: bbox_sample = [] # start from 1 - bbox_sample.append(float(label_list.index( - object.find('name').text))) + bbox_sample.append( + float(label_list.index(object.find('name').text)) + ) bbox = object.find('bndbox') difficult = float(object.find('difficult').text) bbox_sample.append(float(bbox.find('xmin').text) / im_width) @@ -118,7 +121,8 @@ def convert_pascalvoc_local2bin(args): bbox_labels.append(bbox_sample) bbox_labels = np.array(bbox_labels) - if len(bbox_labels) == 0: continue + if len(bbox_labels) == 0: + continue lbls.extend(bbox_labels[:, 0]) boxes.extend(bbox_labels[:, 1:5]) @@ -135,11 +139,17 @@ def convert_pascalvoc_local2bin(args): # number of images + all images data + an array that represent object numbers of each image # + labels of all objects in images + bboxes of all objects + difficulties of all objects # so the target size should be as follows: - target_size = SIZE_INT64 + image_nums * 3 * args.resize_h * args.resize_h * SIZE_FLOAT32 + image_nums * SIZE_INT64 + object_nums_sum * ( - SIZE_INT64 + 4 * SIZE_FLOAT32 + SIZE_INT64) - if (os.path.getsize(output_file_path) == target_size): - print("Success! \nThe local data output binary file can be found at: ", - output_file_path) + target_size = ( + SIZE_INT64 + + image_nums * 3 * args.resize_h * args.resize_h * SIZE_FLOAT32 + + image_nums * SIZE_INT64 + + object_nums_sum * (SIZE_INT64 + 4 * SIZE_FLOAT32 + SIZE_INT64) + ) + if os.path.getsize(output_file_path) == target_size: + print( + "Success! \nThe local data output binary file can be found at: ", + output_file_path, + ) else: print("Conversion failed!") @@ -147,8 +157,9 @@ def convert_pascalvoc_local2bin(args): def print_processbar(done_percentage): done_filled = done_percentage * '=' empty_filled = (100 - done_percentage) * ' ' - sys.stdout.write("\r[%s%s]%d%%" % - (done_filled, empty_filled, done_percentage)) + sys.stdout.write( + "\r[%s%s]%d%%" % (done_filled, empty_filled, done_percentage) + ) sys.stdout.flush() @@ -163,13 +174,30 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path): # map label to number (index) label_list = [ - "background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", - "car", "cat", "chair", "cow", "diningtable", "dog", "horse", - "motorbike", "person", "pottedplant", "sheep", "sofa", "train", - "tvmonitor" + "background", + "aeroplane", + "bicycle", + "bird", + "boat", + "bottle", + "bus", + "car", + "cat", + "chair", + "cow", + "diningtable", + "dog", + "horse", + "motorbike", + "person", + "pottedplant", + "sheep", + "sofa", + "train", + "tvmonitor", ] print_processbar(0) - #read from tar file and write to bin + # read from tar file and write to bin tar = tarfile.open(tar_path, "r") f_test = tar.extractfile(TEST_LIST_KEY).read() lines = f_test.split('\n') @@ -210,8 +238,9 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path): for object in objects: bbox_sample = [] - bbox_sample.append(float(label_list.index( - object.find('name').text))) + bbox_sample.append( + float(label_list.index(object.find('name').text)) + ) bbox = object.find('bndbox') difficult = float(object.find('difficult').text) bbox_sample.append(float(bbox.find('xmin').text) / im_width) @@ -222,7 +251,8 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path): bbox_labels.append(bbox_sample) bbox_labels = np.array(bbox_labels) - if len(bbox_labels) == 0: continue + if len(bbox_labels) == 0: + continue lbls.extend(bbox_labels[:, 0]) boxes.extend(bbox_labels[:, 1:5]) difficults.extend(bbox_labels[:, -1]) @@ -258,12 +288,16 @@ def download_pascalvoc(data_url, data_dir, tar_targethash, tar_path): def run_convert(): try_limit = 2 retry = 0 - while not (os.path.exists(DATA_OUT_PATH) and os.path.getsize(DATA_OUT_PATH) - == BIN_FULLSIZE and BIN_TARGETHASH == hashlib.md5( - open(DATA_OUT_PATH, 'rb').read()).hexdigest()): + while not ( + os.path.exists(DATA_OUT_PATH) + and os.path.getsize(DATA_OUT_PATH) == BIN_FULLSIZE + and BIN_TARGETHASH + == hashlib.md5(open(DATA_OUT_PATH, 'rb').read()).hexdigest() + ): if os.path.exists(DATA_OUT_PATH): sys.stderr.write( - "The existing binary file is broken. It is being removed...\n") + "The existing binary file is broken. It is being removed...\n" + ) os.remove(DATA_OUT_PATH) if retry < try_limit: retry = retry + 1 @@ -275,52 +309,60 @@ def run_convert(): def main_pascalvoc_preprocess(args): parser = argparse.ArgumentParser( - description= - "Convert the full pascalvoc val set or local data to binary file.", + description="Convert the full pascalvoc val set or local data to binary file.", usage=None, - add_help=True) + add_help=True, + ) parser.add_argument( '--local', action="store_true", - help="If used, user need to set --data_dir and then convert file") - parser.add_argument("--data_dir", - default="", - type=str, - help="Dataset root directory") + help="If used, user need to set --data_dir and then convert file", + ) + parser.add_argument( + "--data_dir", default="", type=str, help="Dataset root directory" + ) parser.add_argument( "--img_annotation_list", type=str, default="test_100.txt", - help= - "A file containing the image file path and corresponding annotation file path" + help="A file containing the image file path and corresponding annotation file path", ) parser.add_argument( "--label_file", type=str, default="label_list", - help= - "List of object labels with same sequence as denoted in the annotation file" + help="List of object labels with same sequence as denoted in the annotation file", + ) + parser.add_argument( + "--output_file", + type=str, + default="pascalvoc_small.bin", + help="File path of the output binary file", + ) + parser.add_argument( + "--resize_h", + type=int, + default=RESIZE_H, + help="Image preprocess with resize_h", + ) + parser.add_argument( + "--resize_w", + type=int, + default=RESIZE_W, + help="Image prerocess with resize_w", + ) + parser.add_argument( + "--mean_value", + type=str, + default=MEAN_VALUE, + help="Image preprocess with mean_value", + ) + parser.add_argument( + "--ap_version", + type=str, + default=AP_VERSION, + help="Image preprocess with ap_version", ) - parser.add_argument("--output_file", - type=str, - default="pascalvoc_small.bin", - help="File path of the output binary file") - parser.add_argument("--resize_h", - type=int, - default=RESIZE_H, - help="Image preprocess with resize_h") - parser.add_argument("--resize_w", - type=int, - default=RESIZE_W, - help="Image prerocess with resize_w") - parser.add_argument("--mean_value", - type=str, - default=MEAN_VALUE, - help="Image preprocess with mean_value") - parser.add_argument("--ap_version", - type=str, - default=AP_VERSION, - help="Image preprocess with ap_version") args = parser.parse_args() if args.local: convert_pascalvoc_local2bin(args) diff --git a/paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py b/paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py index 57920a8b270..4ae32b0f77a 100644 --- a/paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py +++ b/paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py @@ -17,13 +17,13 @@ import os class Test_Preprocess(unittest.TestCase): - def test_local_convert(self): os.system("python full_pascalvoc_test_preprocess.py --choice=local") def test_online_convert(self): os.system( - "python full_pascalvoc_test_preprocess.py --choice=VOC_test_2007") + "python full_pascalvoc_test_preprocess.py --choice=VOC_test_2007" + ) if __name__ == '__main__': diff --git a/paddle/fluid/pybind/generate_file_structures.py b/paddle/fluid/pybind/generate_file_structures.py index 869b27050a0..4904f97dc2f 100644 --- a/paddle/fluid/pybind/generate_file_structures.py +++ b/paddle/fluid/pybind/generate_file_structures.py @@ -25,7 +25,8 @@ if __name__ == "__main__": for i in range(split_count): empty_files.append( - os.path.join(pybind_dir, "op_function" + str(i + 1) + ".cc")) + os.path.join(pybind_dir, "op_function" + str(i + 1) + ".cc") + ) for path in empty_files: if not os.path.exists(path): diff --git a/paddle/infrt/tests/models/abs_model.py b/paddle/infrt/tests/models/abs_model.py index c94338be778..56e1bc8fa71 100644 --- a/paddle/infrt/tests/models/abs_model.py +++ b/paddle/infrt/tests/models/abs_model.py @@ -19,7 +19,6 @@ import sys class AbsNet(paddle.nn.Layer): - def __init__(self): super(AbsNet, self).__init__() @@ -32,6 +31,7 @@ if __name__ == '__main__': # build network model = AbsNet() # save inferencing format model - net = to_static(model, - input_spec=[InputSpec(shape=[None, 1, 28, 28], name='x')]) + net = to_static( + model, input_spec=[InputSpec(shape=[None, 1, 28, 28], name='x')] + ) paddle.jit.save(net, sys.argv[1]) diff --git a/paddle/infrt/tests/models/efficientnet-b4/model.py b/paddle/infrt/tests/models/efficientnet-b4/model.py index 0c6163f3df2..1b493897e65 100644 --- a/paddle/infrt/tests/models/efficientnet-b4/model.py +++ b/paddle/infrt/tests/models/efficientnet-b4/model.py @@ -20,6 +20,7 @@ import paddle import sys model = EfficientNet.from_name('efficientnet-b4') -net = to_static(model, - input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')]) +net = to_static( + model, input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')] +) paddle.jit.save(net, sys.argv[1]) diff --git a/paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py b/paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py index 75f6780484d..c14be25a815 100644 --- a/paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py +++ b/paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py @@ -16,9 +16,15 @@ import paddle import paddle.nn as nn import paddle.nn.functional as F -from .utils import (round_filters, round_repeats, drop_connect, - get_same_padding_conv2d, get_model_params, - efficientnet_params, load_pretrained_weights) +from .utils import ( + round_filters, + round_repeats, + drop_connect, + get_same_padding_conv2d, + get_model_params, + efficientnet_params, + load_pretrained_weights, +) class MBConvBlock(nn.Layer): @@ -38,8 +44,9 @@ class MBConvBlock(nn.Layer): self._block_args = block_args self._bn_mom = global_params.batch_norm_momentum self._bn_eps = global_params.batch_norm_epsilon - self.has_se = (self._block_args.se_ratio - is not None) and (0 < self._block_args.se_ratio <= 1) + self.has_se = (self._block_args.se_ratio is not None) and ( + 0 < self._block_args.se_ratio <= 1 + ) self.id_skip = block_args.id_skip # skip connection and drop connect # Get static or dynamic convolution depending on image size @@ -47,15 +54,19 @@ class MBConvBlock(nn.Layer): # Expansion phase inp = self._block_args.input_filters # number of input channels - oup = self._block_args.input_filters * self._block_args.expand_ratio # number of output channels + oup = ( + self._block_args.input_filters * self._block_args.expand_ratio + ) # number of output channels if self._block_args.expand_ratio != 1: - self._expand_conv = Conv2d(in_channels=inp, - out_channels=oup, - kernel_size=1, - bias_attr=False) - self._bn0 = nn.BatchNorm2D(num_features=oup, - momentum=self._bn_mom, - epsilon=self._bn_eps) + self._expand_conv = Conv2d( + in_channels=inp, + out_channels=oup, + kernel_size=1, + bias_attr=False, + ) + self._bn0 = nn.BatchNorm2D( + num_features=oup, momentum=self._bn_mom, epsilon=self._bn_eps + ) # Depthwise convolution phase k = self._block_args.kernel_size @@ -66,32 +77,40 @@ class MBConvBlock(nn.Layer): groups=oup, # groups makes it depthwise kernel_size=k, stride=s, - bias_attr=False) - self._bn1 = nn.BatchNorm2D(num_features=oup, - momentum=self._bn_mom, - epsilon=self._bn_eps) + bias_attr=False, + ) + self._bn1 = nn.BatchNorm2D( + num_features=oup, momentum=self._bn_mom, epsilon=self._bn_eps + ) # Squeeze and Excitation layer, if desired if self.has_se: num_squeezed_channels = max( 1, - int(self._block_args.input_filters * self._block_args.se_ratio)) - self._se_reduce = Conv2d(in_channels=oup, - out_channels=num_squeezed_channels, - kernel_size=1) - self._se_expand = Conv2d(in_channels=num_squeezed_channels, - out_channels=oup, - kernel_size=1) + int(self._block_args.input_filters * self._block_args.se_ratio), + ) + self._se_reduce = Conv2d( + in_channels=oup, + out_channels=num_squeezed_channels, + kernel_size=1, + ) + self._se_expand = Conv2d( + in_channels=num_squeezed_channels, + out_channels=oup, + kernel_size=1, + ) # Output phase final_oup = self._block_args.output_filters - self._project_conv = Conv2d(in_channels=oup, - out_channels=final_oup, - kernel_size=1, - bias_attr=False) - self._bn2 = nn.BatchNorm2D(num_features=final_oup, - momentum=self._bn_mom, - epsilon=self._bn_eps) + self._project_conv = Conv2d( + in_channels=oup, + out_channels=final_oup, + kernel_size=1, + bias_attr=False, + ) + self._bn2 = nn.BatchNorm2D( + num_features=final_oup, momentum=self._bn_mom, epsilon=self._bn_eps + ) self._swish = nn.Hardswish() def forward(self, inputs, drop_connect_rate=None): @@ -111,18 +130,26 @@ class MBConvBlock(nn.Layer): if self.has_se: x_squeezed = F.adaptive_avg_pool2d(x, 1) x_squeezed = self._se_expand( - self._swish(self._se_reduce(x_squeezed))) + self._swish(self._se_reduce(x_squeezed)) + ) x = F.sigmoid(x_squeezed) * x x = self._bn2(self._project_conv(x)) # Skip connection and drop connect - input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters - if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters: + input_filters, output_filters = ( + self._block_args.input_filters, + self._block_args.output_filters, + ) + if ( + self.id_skip + and self._block_args.stride == 1 + and input_filters == output_filters + ): if drop_connect_rate: - x = drop_connect(x, - prob=drop_connect_rate, - training=self.training) + x = drop_connect( + x, prob=drop_connect_rate, training=self.training + ) x = x + inputs # skip connection return x @@ -161,15 +188,14 @@ class EfficientNet(nn.Layer): # Stem in_channels = 3 # rgb out_channels = round_filters( - 32, self._global_params) # number of output channels - self._conv_stem = Conv2d(in_channels, - out_channels, - kernel_size=3, - stride=2, - bias_attr=False) - self._bn0 = nn.BatchNorm2D(num_features=out_channels, - momentum=bn_mom, - epsilon=bn_eps) + 32, self._global_params + ) # number of output channels + self._conv_stem = Conv2d( + in_channels, out_channels, kernel_size=3, stride=2, bias_attr=False + ) + self._bn0 = nn.BatchNorm2D( + num_features=out_channels, momentum=bn_mom, epsilon=bn_eps + ) # Build blocks self._blocks = nn.LayerList([]) @@ -177,32 +203,37 @@ class EfficientNet(nn.Layer): # Update block input and output filters based on depth multiplier. block_args = block_args._replace( - input_filters=round_filters(block_args.input_filters, - self._global_params), - output_filters=round_filters(block_args.output_filters, - self._global_params), - num_repeat=round_repeats(block_args.num_repeat, - self._global_params)) + input_filters=round_filters( + block_args.input_filters, self._global_params + ), + output_filters=round_filters( + block_args.output_filters, self._global_params + ), + num_repeat=round_repeats( + block_args.num_repeat, self._global_params + ), + ) # The first block needs to take care of stride and filter size increase. self._blocks.append(MBConvBlock(block_args, self._global_params)) if block_args.num_repeat > 1: block_args = block_args._replace( - input_filters=block_args.output_filters, stride=1) + input_filters=block_args.output_filters, stride=1 + ) for _ in range(block_args.num_repeat - 1): - self._blocks.append(MBConvBlock(block_args, - self._global_params)) + self._blocks.append( + MBConvBlock(block_args, self._global_params) + ) # Head in_channels = block_args.output_filters # output of final block out_channels = round_filters(1280, self._global_params) - self._conv_head = Conv2d(in_channels, - out_channels, - kernel_size=1, - bias_attr=False) - self._bn1 = nn.BatchNorm2D(num_features=out_channels, - momentum=bn_mom, - epsilon=bn_eps) + self._conv_head = Conv2d( + in_channels, out_channels, kernel_size=1, bias_attr=False + ) + self._bn1 = nn.BatchNorm2D( + num_features=out_channels, momentum=bn_mom, epsilon=bn_eps + ) # Final linear layer self._avg_pooling = nn.AdaptiveAvgPool2D(1) @@ -217,7 +248,7 @@ class EfficientNet(nn.Layer): block.set_swish(memory_efficient) def extract_features(self, inputs): - """ Returns output of the final convolution layer """ + """Returns output of the final convolution layer""" # Stem x = self._swish(self._bn0(self._conv_stem(inputs))) @@ -235,7 +266,7 @@ class EfficientNet(nn.Layer): return x def forward(self, inputs): - """ Calls extract_features to extract features, applies final linear layer, and returns logits. """ + """Calls extract_features to extract features, applies final linear layer, and returns logits.""" bs = inputs.shape[0] # Convolution layers x = self.extract_features(inputs) @@ -250,31 +281,33 @@ class EfficientNet(nn.Layer): @classmethod def from_name(cls, model_name, override_params=None): cls._check_model_name_is_valid(model_name) - blocks_args, global_params = get_model_params(model_name, - override_params) + blocks_args, global_params = get_model_params( + model_name, override_params + ) return cls(blocks_args, global_params) @classmethod - def from_pretrained(cls, - model_name, - advprop=False, - num_classes=1000, - in_channels=3): - model = cls.from_name(model_name, - override_params={'num_classes': num_classes}) - load_pretrained_weights(model, - model_name, - load_fc=(num_classes == 1000), - advprop=advprop) + def from_pretrained( + cls, model_name, advprop=False, num_classes=1000, in_channels=3 + ): + model = cls.from_name( + model_name, override_params={'num_classes': num_classes} + ) + load_pretrained_weights( + model, model_name, load_fc=(num_classes == 1000), advprop=advprop + ) if in_channels != 3: Conv2d = get_same_padding_conv2d( - image_size=model._global_params.image_size) + image_size=model._global_params.image_size + ) out_channels = round_filters(32, model._global_params) - model._conv_stem = Conv2d(in_channels, - out_channels, - kernel_size=3, - stride=2, - bias_attr=False) + model._conv_stem = Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + bias_attr=False, + ) return model @classmethod @@ -285,8 +318,9 @@ class EfficientNet(nn.Layer): @classmethod def _check_model_name_is_valid(cls, model_name): - """ Validates model name. """ + """Validates model name.""" valid_models = ['efficientnet-b' + str(i) for i in range(9)] if model_name not in valid_models: - raise ValueError('model_name should be one of: ' + - ', '.join(valid_models)) + raise ValueError( + 'model_name should be one of: ' + ', '.join(valid_models) + ) diff --git a/paddle/infrt/tests/models/efficientnet-b4/net/utils.py b/paddle/infrt/tests/models/efficientnet-b4/net/utils.py index 62caf759ad3..9a060306995 100644 --- a/paddle/infrt/tests/models/efficientnet-b4/net/utils.py +++ b/paddle/infrt/tests/models/efficientnet-b4/net/utils.py @@ -22,25 +22,44 @@ import paddle.nn as nn import paddle.nn.functional as F # Parameters for the entire model (stem, all blocks, and head) -GlobalParams = collections.namedtuple('GlobalParams', [ - 'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate', 'num_classes', - 'width_coefficient', 'depth_coefficient', 'depth_divisor', 'min_depth', - 'drop_connect_rate', 'image_size' -]) +GlobalParams = collections.namedtuple( + 'GlobalParams', + [ + 'batch_norm_momentum', + 'batch_norm_epsilon', + 'dropout_rate', + 'num_classes', + 'width_coefficient', + 'depth_coefficient', + 'depth_divisor', + 'min_depth', + 'drop_connect_rate', + 'image_size', + ], +) # Parameters for an individual model block -BlockArgs = collections.namedtuple('BlockArgs', [ - 'kernel_size', 'num_repeat', 'input_filters', 'output_filters', - 'expand_ratio', 'id_skip', 'stride', 'se_ratio' -]) +BlockArgs = collections.namedtuple( + 'BlockArgs', + [ + 'kernel_size', + 'num_repeat', + 'input_filters', + 'output_filters', + 'expand_ratio', + 'id_skip', + 'stride', + 'se_ratio', + ], +) # Change namedtuple defaults -GlobalParams.__new__.__defaults__ = (None, ) * len(GlobalParams._fields) -BlockArgs.__new__.__defaults__ = (None, ) * len(BlockArgs._fields) +GlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields) +BlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields) def round_filters(filters, global_params): - """ Calculate and round number of filters based on depth multiplier. """ + """Calculate and round number of filters based on depth multiplier.""" multiplier = global_params.width_coefficient if not multiplier: return filters @@ -48,15 +67,16 @@ def round_filters(filters, global_params): min_depth = global_params.min_depth filters *= multiplier min_depth = min_depth or divisor - new_filters = max(min_depth, - int(filters + divisor / 2) // divisor * divisor) + new_filters = max( + min_depth, int(filters + divisor / 2) // divisor * divisor + ) if new_filters < 0.9 * filters: # prevent rounding by more than 10% new_filters += divisor return int(new_filters) def round_repeats(repeats, global_params): - """ Round number of filters based on depth multiplier. """ + """Round number of filters based on depth multiplier.""" multiplier = global_params.depth_coefficient if not multiplier: return repeats @@ -76,8 +96,8 @@ def drop_connect(inputs, prob, training): def get_same_padding_conv2d(image_size=None): - """ Chooses static padding if you have specified an image size, and dynamic padding otherwise. - Static padding is necessary for ONNX exporting of models. """ + """Chooses static padding if you have specified an image size, and dynamic padding otherwise. + Static padding is necessary for ONNX exporting of models.""" if image_size is None: return Conv2dDynamicSamePadding else: @@ -85,26 +105,31 @@ def get_same_padding_conv2d(image_size=None): class Conv2dDynamicSamePadding(nn.Conv2D): - """ 2D Convolutions like TensorFlow, for a dynamic image size """ - - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - dilation=1, - groups=1, - bias_attr=None): - super().__init__(in_channels, - out_channels, - kernel_size, - stride, - 0, - dilation, - groups, - bias_attr=bias_attr) - self.stride = self._stride if len( - self._stride) == 2 else [self._stride[0]] * 2 + """2D Convolutions like TensorFlow, for a dynamic image size""" + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + dilation=1, + groups=1, + bias_attr=None, + ): + super().__init__( + in_channels, + out_channels, + kernel_size, + stride, + 0, + dilation, + groups, + bias_attr=bias_attr, + ) + self.stride = ( + self._stride if len(self._stride) == 2 else [self._stride[0]] * 2 + ) def forward(self, x): ih, iw = x.shape[-2:] @@ -112,65 +137,84 @@ class Conv2dDynamicSamePadding(nn.Conv2D): sh, sw = self.stride oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) pad_h = max( - (oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, - 0) + (oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, 0 + ) pad_w = max( - (ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, - 0) + (ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, 0 + ) if pad_h > 0 or pad_w > 0: - x = F.pad(x, [ - pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 - ]) - return F.conv2d(x, self.weight, self.bias, self.stride, self._padding, - self._dilation, self._groups) + x = F.pad( + x, + [ + pad_w // 2, + pad_w - pad_w // 2, + pad_h // 2, + pad_h - pad_h // 2, + ], + ) + return F.conv2d( + x, + self.weight, + self.bias, + self.stride, + self._padding, + self._dilation, + self._groups, + ) class Conv2dStaticSamePadding(nn.Conv2D): - """ 2D Convolutions like TensorFlow, for a fixed image size""" - - def __init__(self, - in_channels, - out_channels, - kernel_size, - image_size=None, - **kwargs): + """2D Convolutions like TensorFlow, for a fixed image size""" + + def __init__( + self, in_channels, out_channels, kernel_size, image_size=None, **kwargs + ): if 'stride' in kwargs and isinstance(kwargs['stride'], list): kwargs['stride'] = kwargs['stride'][0] super().__init__(in_channels, out_channels, kernel_size, **kwargs) - self.stride = self._stride if len( - self._stride) == 2 else [self._stride[0]] * 2 + self.stride = ( + self._stride if len(self._stride) == 2 else [self._stride[0]] * 2 + ) # Calculate padding based on image size and save it assert image_size is not None - ih, iw = image_size if type(image_size) == list else [ - image_size, image_size - ] + ih, iw = ( + image_size if type(image_size) == list else [image_size, image_size] + ) kh, kw = self.weight.shape[-2:] sh, sw = self.stride oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) pad_h = max( - (oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, - 0) + (oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, 0 + ) pad_w = max( - (ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, - 0) + (ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, 0 + ) if pad_h > 0 or pad_w > 0: - self.static_padding = nn.Pad2D([ - pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 - ]) + self.static_padding = nn.Pad2D( + [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2] + ) else: self.static_padding = Identity() def forward(self, x): x = self.static_padding(x) - x = F.conv2d(x, self.weight, self.bias, self.stride, self._padding, - self._dilation, self._groups) + x = F.conv2d( + x, + self.weight, + self.bias, + self.stride, + self._padding, + self._dilation, + self._groups, + ) return x class Identity(nn.Layer): - - def __init__(self, ): + def __init__( + self, + ): super().__init__() def forward(self, x): @@ -178,7 +222,7 @@ class Identity(nn.Layer): def efficientnet_params(model_name): - """ Map EfficientNet model name to parameter coefficients. """ + """Map EfficientNet model name to parameter coefficients.""" params_dict = { # Coefficients: width,depth,resolution,dropout 'efficientnet-b0': (1.0, 1.0, 224, 0.2), @@ -196,11 +240,11 @@ def efficientnet_params(model_name): class BlockDecoder(object): - """ Block Decoder for readability, straight from the official TensorFlow repository """ + """Block Decoder for readability, straight from the official TensorFlow repository""" @staticmethod def _decode_block_string(block_string): - """ Gets a block through a string notation of arguments. """ + """Gets a block through a string notation of arguments.""" assert isinstance(block_string, str) ops = block_string.split('_') @@ -212,8 +256,9 @@ class BlockDecoder(object): options[key] = value # Check stride - assert (('s' in options and len(options['s']) == 1) or - (len(options['s']) == 2 and options['s'][0] == options['s'][1])) + assert ('s' in options and len(options['s']) == 1) or ( + len(options['s']) == 2 and options['s'][0] == options['s'][1] + ) return BlockArgs( kernel_size=int(options['k']), @@ -223,7 +268,8 @@ class BlockDecoder(object): expand_ratio=int(options['e']), id_skip=('noskip' not in block_string), se_ratio=float(options['se']) if 'se' in options else None, - stride=[int(options['s'][0])]) + stride=[int(options['s'][0])], + ) @staticmethod def _encode_block_string(block): @@ -234,7 +280,7 @@ class BlockDecoder(object): 's%d%d' % (block.strides[0], block.strides[1]), 'e%s' % block.expand_ratio, 'i%d' % block.input_filters, - 'o%d' % block.output_filters + 'o%d' % block.output_filters, ] if 0 < block.se_ratio <= 1: args.append('se%s' % block.se_ratio) @@ -270,13 +316,15 @@ class BlockDecoder(object): return block_strings -def efficientnet(width_coefficient=None, - depth_coefficient=None, - dropout_rate=0.2, - drop_connect_rate=0.2, - image_size=None, - num_classes=1000): - """ Get block arguments according to parameter and coefficients. """ +def efficientnet( + width_coefficient=None, + depth_coefficient=None, + dropout_rate=0.2, + drop_connect_rate=0.2, + image_size=None, + num_classes=1000, +): + """Get block arguments according to parameter and coefficients.""" blocks_args = [ 'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25', @@ -305,67 +353,51 @@ def efficientnet(width_coefficient=None, def get_model_params(model_name, override_params): - """ Get the block args and global params for a given model """ + """Get the block args and global params for a given model""" if model_name.startswith('efficientnet'): w, d, s, p = efficientnet_params(model_name) - blocks_args, global_params = efficientnet(width_coefficient=w, - depth_coefficient=d, - dropout_rate=p, - image_size=s) + blocks_args, global_params = efficientnet( + width_coefficient=w, + depth_coefficient=d, + dropout_rate=p, + image_size=s, + ) else: - raise NotImplementedError('model name is not pre-defined: %s' % - model_name) + raise NotImplementedError( + 'model name is not pre-defined: %s' % model_name + ) if override_params: global_params = global_params._replace(**override_params) return blocks_args, global_params url_map = { - 'efficientnet-b0': - '/home/aistudio/data/weights/efficientnet-b0-355c32eb.pdparams', - 'efficientnet-b1': - '/home/aistudio/data/weights/efficientnet-b1-f1951068.pdparams', - 'efficientnet-b2': - '/home/aistudio/data/weights/efficientnet-b2-8bb594d6.pdparams', - 'efficientnet-b3': - '/home/aistudio/data/weights/efficientnet-b3-5fb5a3c3.pdparams', - 'efficientnet-b4': - '/home/aistudio/data/weights/efficientnet-b4-6ed6700e.pdparams', - 'efficientnet-b5': - '/home/aistudio/data/weights/efficientnet-b5-b6417697.pdparams', - 'efficientnet-b6': - '/home/aistudio/data/weights/efficientnet-b6-c76e70fd.pdparams', - 'efficientnet-b7': - '/home/aistudio/data/weights/efficientnet-b7-dcc49843.pdparams', + 'efficientnet-b0': '/home/aistudio/data/weights/efficientnet-b0-355c32eb.pdparams', + 'efficientnet-b1': '/home/aistudio/data/weights/efficientnet-b1-f1951068.pdparams', + 'efficientnet-b2': '/home/aistudio/data/weights/efficientnet-b2-8bb594d6.pdparams', + 'efficientnet-b3': '/home/aistudio/data/weights/efficientnet-b3-5fb5a3c3.pdparams', + 'efficientnet-b4': '/home/aistudio/data/weights/efficientnet-b4-6ed6700e.pdparams', + 'efficientnet-b5': '/home/aistudio/data/weights/efficientnet-b5-b6417697.pdparams', + 'efficientnet-b6': '/home/aistudio/data/weights/efficientnet-b6-c76e70fd.pdparams', + 'efficientnet-b7': '/home/aistudio/data/weights/efficientnet-b7-dcc49843.pdparams', } url_map_advprop = { - 'efficientnet-b0': - '/home/aistudio/data/weights/adv-efficientnet-b0-b64d5a18.pdparams', - 'efficientnet-b1': - '/home/aistudio/data/weights/adv-efficientnet-b1-0f3ce85a.pdparams', - 'efficientnet-b2': - '/home/aistudio/data/weights/adv-efficientnet-b2-6e9d97e5.pdparams', - 'efficientnet-b3': - '/home/aistudio/data/weights/adv-efficientnet-b3-cdd7c0f4.pdparams', - 'efficientnet-b4': - '/home/aistudio/data/weights/adv-efficientnet-b4-44fb3a87.pdparams', - 'efficientnet-b5': - '/home/aistudio/data/weights/adv-efficientnet-b5-86493f6b.pdparams', - 'efficientnet-b6': - '/home/aistudio/data/weights/adv-efficientnet-b6-ac80338e.pdparams', - 'efficientnet-b7': - '/home/aistudio/data/weights/adv-efficientnet-b7-4652b6dd.pdparams', - 'efficientnet-b8': - '/home/aistudio/data/weights/adv-efficientnet-b8-22a8fe65.pdparams', + 'efficientnet-b0': '/home/aistudio/data/weights/adv-efficientnet-b0-b64d5a18.pdparams', + 'efficientnet-b1': '/home/aistudio/data/weights/adv-efficientnet-b1-0f3ce85a.pdparams', + 'efficientnet-b2': '/home/aistudio/data/weights/adv-efficientnet-b2-6e9d97e5.pdparams', + 'efficientnet-b3': '/home/aistudio/data/weights/adv-efficientnet-b3-cdd7c0f4.pdparams', + 'efficientnet-b4': '/home/aistudio/data/weights/adv-efficientnet-b4-44fb3a87.pdparams', + 'efficientnet-b5': '/home/aistudio/data/weights/adv-efficientnet-b5-86493f6b.pdparams', + 'efficientnet-b6': '/home/aistudio/data/weights/adv-efficientnet-b6-ac80338e.pdparams', + 'efficientnet-b7': '/home/aistudio/data/weights/adv-efficientnet-b7-4652b6dd.pdparams', + 'efficientnet-b8': '/home/aistudio/data/weights/adv-efficientnet-b8-22a8fe65.pdparams', } -def load_pretrained_weights(model, - model_name, - weights_path=None, - load_fc=True, - advprop=False): +def load_pretrained_weights( + model, model_name, weights_path=None, load_fc=True, advprop=False +): """Loads pretrained weights from weights path or download using url. Args: model (Module): The whole model of efficientnet. diff --git a/paddle/infrt/tests/models/linear.py b/paddle/infrt/tests/models/linear.py index 1a6c6f095c7..71dc4e0d1c9 100644 --- a/paddle/infrt/tests/models/linear.py +++ b/paddle/infrt/tests/models/linear.py @@ -28,13 +28,12 @@ CLASS_NUM = 10 # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -42,7 +41,6 @@ class RandomDataset(paddle.io.Dataset): class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -71,11 +69,9 @@ adam = opt.Adam(learning_rate=0.001, parameters=layer.parameters()) # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) -loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) +loader = paddle.io.DataLoader( + dataset, batch_size=BATCH_SIZE, shuffle=True, drop_last=True, num_workers=2 +) # train train(layer, loader, loss_fn, adam) diff --git a/paddle/infrt/tests/models/resnet50_model.py b/paddle/infrt/tests/models/resnet50_model.py index af45de0c8a6..ff84c3988ab 100644 --- a/paddle/infrt/tests/models/resnet50_model.py +++ b/paddle/infrt/tests/models/resnet50_model.py @@ -19,6 +19,7 @@ from paddle.static import InputSpec import sys model = resnet50(True) -net = to_static(model, - input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')]) +net = to_static( + model, input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')] +) paddle.jit.save(net, sys.argv[1]) diff --git a/paddle/phi/api/yaml/generator/api_base.py b/paddle/phi/api/yaml/generator/api_base.py index 534158eb31c..e67023d2faf 100644 --- a/paddle/phi/api/yaml/generator/api_base.py +++ b/paddle/phi/api/yaml/generator/api_base.py @@ -20,7 +20,6 @@ PREFIX_META_TENSOR_NAME = 'meta_' class BaseAPI(object): - def __init__(self, api_item_yaml): self.api = self.get_api_name(api_item_yaml) @@ -34,8 +33,12 @@ class BaseAPI(object): # names : [], list of output names # types : [], list of output types # out_size_expr : [], expression for getting size of vector - self.inputs, self.attrs, self.outputs, self.optional_vars = self.parse_args( - self.api, api_item_yaml) + ( + self.inputs, + self.attrs, + self.outputs, + self.optional_vars, + ) = self.parse_args(self.api, api_item_yaml) self.is_base_api = True if 'invoke' in api_item_yaml: @@ -44,7 +47,8 @@ class BaseAPI(object): else: if 'infer_meta' in api_item_yaml: self.infer_meta = self.parse_infer_meta( - api_item_yaml['infer_meta']) + api_item_yaml['infer_meta'] + ) self.kernel = self.parse_kernel(api_item_yaml['kernel']) self.data_transform = self.parse_data_transform(api_item_yaml) self.inplace_map, self.view_map = {}, {} @@ -52,18 +56,16 @@ class BaseAPI(object): self.gene_input_func = { "const Tensor&": { "dense": self.gene_dense_input, - "selected_rows": self.gene_selected_rows_input + "selected_rows": self.gene_selected_rows_input, }, "const paddle::optional&": { "dense": self.gene_dense_input, - "selected_rows": self.gene_selected_rows_input - }, - "const std::vector&": { - "dense": self.gene_vec_dense_input + "selected_rows": self.gene_selected_rows_input, }, + "const std::vector&": {"dense": self.gene_vec_dense_input}, "const paddle::optional>&": { "dense": self.gene_optional_vec_dense_input - } + }, } def get_api_name(self, api_item_yaml): @@ -75,21 +77,19 @@ class BaseAPI(object): def get_input_tensor_args(self, inplace_flag=False): input_args = [] inplace_type_map = { - "const Tensor&": - "Tensor&", - "const paddle::optional&": - "paddle::optional&", - "const std::vector&": - "std::vector&", - "const paddle::optional>&": - "paddle::optional>&" + "const Tensor&": "Tensor&", + "const paddle::optional&": "paddle::optional&", + "const std::vector&": "std::vector&", + "const paddle::optional>&": "paddle::optional>&", } for name in self.inputs['names']: name = name.split('@')[0] if inplace_flag and name in self.inplace_map.values(): input_args.append( - inplace_type_map[self.inputs['input_info'][name]] + ' ' + - name) + inplace_type_map[self.inputs['input_info'][name]] + + ' ' + + name + ) else: input_args.append(self.inputs['input_info'][name] + ' ' + name) return input_args @@ -100,8 +100,9 @@ class BaseAPI(object): default_value = '' if self.attrs['attr_info'][name][1] is not None: default_value = ' = ' + self.attrs['attr_info'][name][1] - declare_args.append(self.attrs['attr_info'][name][0] + ' ' + name + - default_value) + declare_args.append( + self.attrs['attr_info'][name][0] + ' ' + name + default_value + ) return ", ".join(declare_args) @@ -118,28 +119,35 @@ class BaseAPI(object): optional_vars = [ item.strip() for item in api_item_yaml['optional'].split(',') ] - inputs, attrs = self.parse_input_and_attr(api_name, - api_item_yaml['args'], - optional_vars) + inputs, attrs = self.parse_input_and_attr( + api_name, api_item_yaml['args'], optional_vars + ) output_type_list, output_names, out_size_expr = self.parse_output( - api_name, api_item_yaml['output']) - return inputs, attrs, { - 'names': output_names, - 'types': output_type_list, - 'out_size_expr': out_size_expr - }, optional_vars + api_name, api_item_yaml['output'] + ) + return ( + inputs, + attrs, + { + 'names': output_names, + 'types': output_type_list, + 'out_size_expr': out_size_expr, + }, + optional_vars, + ) def parse_input_and_attr(self, api_name, args_config, optional_vars=[]): inputs = {'names': [], 'input_info': {}} attrs = {'names': [], 'attr_info': {}} args_str = args_config.strip() - assert args_str.startswith('(') and args_str.endswith(')'), \ - f"Args declaration should start with '(' and end with ')', please check the args of {api_name} in yaml." + assert args_str.startswith('(') and args_str.endswith( + ')' + ), f"Args declaration should start with '(' and end with ')', please check the args of {api_name} in yaml." args_str = args_str[1:-1] args_list = args_str.split(',') input_types_map = { 'Tensor': 'const Tensor&', - 'Tensor[]': 'const std::vector&' + 'Tensor[]': 'const std::vector&', } attr_types_map = { 'IntArray': 'const IntArray&', @@ -177,7 +185,7 @@ class BaseAPI(object): 'bool': 'paddle::optional', 'Place': 'paddle::optional', 'DataLayout': 'paddle::optional', - 'DataType': 'paddle::optional' + 'DataType': 'paddle::optional', } for item in args_list: @@ -188,10 +196,12 @@ class BaseAPI(object): for in_type_symbol, in_type in input_types_map.items(): if type_and_name[0] == in_type_symbol: input_name = type_and_name[1].strip() - assert len(input_name) > 0, \ - f"The input tensor name should not be empty. Please check the args of {api_name} in yaml." - assert len(attrs['names']) == 0, \ - f"The input Tensor should appear before attributes. please check the position of {api_name}:input({input_name}) in yaml" + assert ( + len(input_name) > 0 + ), f"The input tensor name should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(attrs['names']) == 0 + ), f"The input Tensor should appear before attributes. please check the position of {api_name}:input({input_name}) in yaml" if input_name in optional_vars: in_type = optional_types_trans[in_type_symbol] @@ -206,9 +216,10 @@ class BaseAPI(object): # match the attribute for attr_type_symbol, attr_type in attr_types_map.items(): if type_and_name[0] == attr_type_symbol: - attr_name = item[len(attr_type_symbol):].strip() - assert len(attr_name) > 0, \ - f"The attribute name should not be empty. Please check the args of {api_name} in yaml." + attr_name = item[len(attr_type_symbol) :].strip() + assert ( + len(attr_name) > 0 + ), f"The attribute name should not be empty. Please check the args of {api_name} in yaml." default_value = None if '=' in attr_name: attr_infos = attr_name.split('=') @@ -218,7 +229,9 @@ class BaseAPI(object): if attr_name in optional_vars: attr_type = optional_types_trans[attr_type_symbol] - default_value_str = "" if default_value is None else '=' + default_value + default_value_str = ( + "" if default_value is None else '=' + default_value + ) attrs['names'].append(attr_name) attrs['attr_info'][attr_name] = (attr_type, default_value) break @@ -226,25 +239,34 @@ class BaseAPI(object): return inputs, attrs def parse_output(self, api_name, output_config): - def parse_output_item(output_item): output_type_map = { 'Tensor': 'Tensor', - 'Tensor[]': 'std::vector' + 'Tensor[]': 'std::vector', } result = re.search( r"(?P[a-zA-Z0-9_[\]]+)\s*(?P\([a-zA-Z0-9_@]+\))?\s*(?P\{[^\}]+\})?", - output_item) - assert result is not None, f"{api_name} : the output config parse error." + output_item, + ) + assert ( + result is not None + ), f"{api_name} : the output config parse error." out_type = result.group('out_type') - assert out_type in output_type_map, \ - f"{api_name} : Output type error: the output type only support Tensor and Tensor[], \ + assert ( + out_type in output_type_map + ), f"{api_name} : Output type error: the output type only support Tensor and Tensor[], \ but now is {out_type}." - out_name = 'out' if result.group('name') is None else result.group( - 'name')[1:-1] - out_size_expr = None if result.group( - 'expr') is None else result.group('expr')[1:-1] + out_name = ( + 'out' + if result.group('name') is None + else result.group('name')[1:-1] + ) + out_size_expr = ( + None + if result.group('expr') is None + else result.group('expr')[1:-1] + ) return output_type_map[out_type], out_name, out_size_expr temp_list = output_config.split(',') @@ -286,7 +308,7 @@ class BaseAPI(object): 'layout': None, 'data_type': None, 'use_gpudnn': 'false', - 'dispatch': {} + 'dispatch': {}, } if 'backend' in kernel_config and len(kernel_config['backend']) > 0: kernel['backend'] = kernel_config['backend'] @@ -301,7 +323,8 @@ class BaseAPI(object): if isinstance(kernel['use_gpudnn'], bool): kernel['use_gpudnn'] = str(kernel['use_gpudnn']).lower() kernel_funcs = re.compile(r'([a-zA-Z0-9_]+)\s*({[^}]+})?').findall( - kernel_config['func']) + kernel_config['func'] + ) def parse_kernel_in_out_type(in_out_str): if len(in_out_str) == 0: @@ -313,11 +336,17 @@ class BaseAPI(object): # check the tensor type for item in inputs: assert item in [ - 'dense', 'selected_rows', 'sparse_coo', 'sparse_csr' + 'dense', + 'selected_rows', + 'sparse_coo', + 'sparse_csr', ], f"{self.api} : Invalid input tensor type ('{item}'), here we only support 'dense', 'selected_rows', 'sparse_coo' and 'sparse_csr'." for item in outputs: assert item in [ - 'dense', 'selected_rows', 'sparse_coo', 'sparse_csr' + 'dense', + 'selected_rows', + 'sparse_coo', + 'sparse_csr', ], f"{self.api} : Invalid output tensor type ('{item}'), here we only support 'dense', 'selected_rows', 'sparse_coo' and 'sparse_csr'." return (inputs, outputs) @@ -325,7 +354,8 @@ class BaseAPI(object): for func_item in kernel_funcs: kernel['func'].append(func_item[0]) kernel['dispatch'][func_item[0]] = parse_kernel_in_out_type( - func_item[1]) + func_item[1] + ) return kernel @@ -334,10 +364,12 @@ class BaseAPI(object): if 'data_transform' in api_item_yaml: if 'skip_transform' in api_item_yaml['data_transform']: data_transform['skip_transform'] = api_item_yaml[ - 'data_transform']['skip_transform'] + 'data_transform' + ]['skip_transform'] if 'support_trans_dtype' in api_item_yaml['data_transform']: data_transform['support_trans_dtype'] = api_item_yaml[ - 'data_transform']['support_trans_dtype'] + 'data_transform' + ]['support_trans_dtype'] return data_transform @@ -356,9 +388,12 @@ PADDLE_API {self.get_return_type()} {api_func_name}({self.get_declare_args()}); if self.is_base_api and len(self.inplace_map) > 0: if api_func_name[-1] != '_': api_func_name += '_' - api_declaration = api_declaration + f""" + api_declaration = ( + api_declaration + + f""" PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_declare_args(inplace_flag=True)}); """ + ) return api_declaration @@ -368,11 +403,13 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d if self.kernel['backend'] is not None: if '>' in self.kernel['backend']: vars_list = self.kernel['backend'].split('>') - assert len( - vars_list - ) == 2, f"{self.api} api: The number of params to set backend with '>' only allows 2, but received {len(vars_list)}." - assert (vars_list[0].strip() in self.attrs['names']) and (self.attrs['attr_info'][vars_list[0].strip()][0] == 'const Place&'), \ - f"{self.api} api: When use '>' to set kernel backend, the first param should be a attribute with Place type." + assert ( + len(vars_list) == 2 + ), f"{self.api} api: The number of params to set backend with '>' only allows 2, but received {len(vars_list)}." + assert (vars_list[0].strip() in self.attrs['names']) and ( + self.attrs['attr_info'][vars_list[0].strip()][0] + == 'const Place&' + ), f"{self.api} api: When use '>' to set kernel backend, the first param should be a attribute with Place type." backend_select_code = f""" kernel_backend = ParseBackendWithInputOrder({vars_list[0].strip()}, {vars_list[1].strip()}); """ @@ -404,16 +441,19 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d attr_data_type_count = 0 for attr_name in attrs['names']: if attrs['attr_info'][attr_name][0] == 'const Place&': - assert kernel['backend'] is not None, \ - f"{api} api: When there is a parameter with 'Place' type in attributes, you must set backend of kernel manually." + assert ( + kernel['backend'] is not None + ), f"{api} api: When there is a parameter with 'Place' type in attributes, you must set backend of kernel manually." attr_backend_count = attr_backend_count + 1 if attrs['attr_info'][attr_name][0] == 'DataLayout': - assert kernel['layout'] is not None, \ - f"{api} api: When there is a parameter with 'DataLayout' type in attributes, you must set layout of kernel manually." + assert ( + kernel['layout'] is not None + ), f"{api} api: When there is a parameter with 'DataLayout' type in attributes, you must set layout of kernel manually." attr_layout_count = attr_layout_count + 1 if attrs['attr_info'][attr_name][0] == 'DataType': - assert kernel['data_type'] is not None, \ - f"{api} api: When there is a parameter with 'DataType' type in attributes, you must set data_type of kernel manually." + assert ( + kernel['data_type'] is not None + ), f"{api} api: When there is a parameter with 'DataType' type in attributes, you must set data_type of kernel manually." attr_data_type_count = attr_data_type_count + 1 # preprocess kernel configures @@ -422,48 +462,67 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d if kernel['layout'] is not None: if '>' in kernel['layout']: vars_list = kernel['layout'].split('>') - assert len( - vars_list - ) == 2, f"{api} api: The number of params to set layout with '>' only allows 2, but received {len(vars_list)}." - assert vars_list[0].strip() in attrs['names'] and attrs['attr_info'][vars_list[0].strip()][0] == 'DataLayout', \ - f"{api} api: When use '>' to set kernel layout, the first param should be a attribute with DataLayout type." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 2 + ), f"{api} api: The number of params to set layout with '>' only allows 2, but received {len(vars_list)}." + assert ( + vars_list[0].strip() in attrs['names'] + and attrs['attr_info'][vars_list[0].strip()][0] + == 'DataLayout' + ), f"{api} api: When use '>' to set kernel layout, the first param should be a attribute with DataLayout type." + kernel_select_code = ( + kernel_select_code + + f""" kernel_layout = ParseLayoutWithInputOrder({vars_list[0].strip()}, {vars_list[1].strip()}); """ + ) else: vars_list = kernel['layout'].split(',') - assert len( - vars_list - ) == 1, f"{api} api: The number of params to set layout must be 1, but received {len(vars_list)}." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 1 + ), f"{api} api: The number of params to set layout must be 1, but received {len(vars_list)}." + kernel_select_code = ( + kernel_select_code + + f""" kernel_layout = ParseLayout({vars_list[0].strip()}); """ + ) if kernel['data_type'] is not None: if '>' in kernel['data_type']: vars_list = kernel['data_type'].split('>') - assert len( - vars_list - ) == 2, f"{api} api: The number of params to set data_type with '>' only allows 2, but received {len(vars_list)}." - assert vars_list[0].strip() in attrs['names'] and attrs['attr_info'][vars_list[0].strip()][0] == 'DataType', \ - f"{api} api: When use '>' to set kernel data_type, the first param should be a attribute with DataType type." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 2 + ), f"{api} api: The number of params to set data_type with '>' only allows 2, but received {len(vars_list)}." + assert ( + vars_list[0].strip() in attrs['names'] + and attrs['attr_info'][vars_list[0].strip()][0] + == 'DataType' + ), f"{api} api: When use '>' to set kernel data_type, the first param should be a attribute with DataType type." + kernel_select_code = ( + kernel_select_code + + f""" kernel_data_type = ParseDataTypeWithInputOrder({vars_list[0].strip()}, {vars_list[1].strip()}); """ + ) else: vars_list = kernel['data_type'].split(',') - assert len( - vars_list - ) == 1, f"{api} api: The number of params to set data_type only allows 1, but received {len(vars_list)}." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 1 + ), f"{api} api: The number of params to set data_type only allows 1, but received {len(vars_list)}." + kernel_select_code = ( + kernel_select_code + + f""" kernel_data_type = ParseDataType({vars_list[0].strip()}); """ + ) if len(input_names) == 0: - assert attr_backend_count > 0 and attr_data_type_count > 0, \ - f"{api} api: When there is no input tensor, the args must have 'Place' and 'DataType'." + assert ( + attr_backend_count > 0 and attr_data_type_count > 0 + ), f"{api} api: When there is no input tensor, the args must have 'Place' and 'DataType'." kernel_select_args = "" for input_name in input_names: @@ -475,7 +534,9 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d kernel_select_code = kernel_key_item_init + kernel_select_code if len(input_names) > 0: - kernel_select_code = kernel_select_code + f""" + kernel_select_code = ( + kernel_select_code + + f""" if (kernel_backend == Backend::UNDEFINED || kernel_layout == DataLayout::UNDEFINED || kernel_data_type == DataType::UNDEFINED ) {{ @@ -491,6 +552,7 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d kernel_data_type = kernel_key.dtype(); }} }}""" + ) return kernel_select_code @@ -499,37 +561,62 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d attr_names = self.attrs['names'] infer_meta = self.infer_meta - infer_meta_params = infer_meta['param'] if infer_meta[ - 'param'] is not None else input_names + attr_names + infer_meta_params = ( + infer_meta['param'] + if infer_meta['param'] is not None + else input_names + attr_names + ) # generate meta tensors meta_tensor_code = "" param_code = "" for param in infer_meta_params: if param in input_names: if self.inputs['input_info'][param] == "const Tensor&": - param_code = param_code + "MakeMetaTensor(*" + PREFIX_TENSOR_NAME + param + "), " - elif self.inputs['input_info'][ - param] == "const std::vector&": - meta_tensor_code = meta_tensor_code + f""" + param_code = ( + param_code + + "MakeMetaTensor(*" + + PREFIX_TENSOR_NAME + + param + + "), " + ) + elif ( + self.inputs['input_info'][param] + == "const std::vector&" + ): + meta_tensor_code = ( + meta_tensor_code + + f""" {code_indent} auto {param}_meta_vec = MakeMetaTensor({PREFIX_TENSOR_NAME}{param}); {code_indent} std::vector {param}_metas({param}_meta_vec.size()); {code_indent} for (size_t i = 0; i < {param}_meta_vec.size(); ++i) {{ {code_indent} {param}_metas[i] = &{param}_meta_vec[i]; {code_indent} }} """ + ) param_code = param_code + param + "_metas, " - elif self.inputs['input_info'][ - param] == "const paddle::optional>&": - meta_tensor_code = meta_tensor_code + f""" + elif ( + self.inputs['input_info'][param] + == "const paddle::optional>&" + ): + meta_tensor_code = ( + meta_tensor_code + + f""" {code_indent} auto {param}_meta_vec = MakeMetaTensor({PREFIX_TENSOR_NAME}{param}); {code_indent} paddle::optional> {param}_metas({param}_meta_vec.size()); {code_indent} for (size_t i = 0; i < {param}_meta_vec.size(); ++i) {{ {code_indent} {param}_metas->at(i) = &{param}_meta_vec[i]; {code_indent} }} """ + ) param_code = param_code + param + "_metas, " elif param in self.optional_vars: - param_code = param_code + "MakeMetaTensor(" + PREFIX_TENSOR_NAME + param + "), " + param_code = ( + param_code + + "MakeMetaTensor(" + + PREFIX_TENSOR_NAME + + param + + "), " + ) else: raise ValueError( f"{self.api} : Param of infer_meta error : {self.inputs['input_info'][param]} type is not supported." @@ -545,22 +632,37 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d for i, out_name in enumerate(kernel_output_names): if self.outputs['types'][i] == 'std::vector': - meta_tensor_code = meta_tensor_code + f""" + meta_tensor_code = ( + meta_tensor_code + + f""" {code_indent} auto {out_name}_{PREFIX_META_TENSOR_NAME}vec = MakeMetaTensor({out_name}); {code_indent} std::vector {out_name}_metas({out_name}_{PREFIX_META_TENSOR_NAME}vec.size()); {code_indent} for (size_t i = 0; i < {out_name}_{PREFIX_META_TENSOR_NAME}vec.size(); ++i) {{ {code_indent} {out_name}_metas[i] = {out_name}[i] ? &{out_name}_{PREFIX_META_TENSOR_NAME}vec[i] : nullptr; {code_indent} }}""" + ) param_code = param_code + out_name + '_metas, ' else: - meta_tensor_code = meta_tensor_code + code_indent + " phi::MetaTensor " + out_name.replace( - 'kernel_', - PREFIX_META_TENSOR_NAME) + "(" + out_name + ");\n" + meta_tensor_code = ( + meta_tensor_code + + code_indent + + " phi::MetaTensor " + + out_name.replace('kernel_', PREFIX_META_TENSOR_NAME) + + "(" + + out_name + + ");\n" + ) if len(kernel_output_names) == 1: - param_code = param_code + f"&{out_name.replace('kernel_', PREFIX_META_TENSOR_NAME)}, " + param_code = ( + param_code + + f"&{out_name.replace('kernel_', PREFIX_META_TENSOR_NAME)}, " + ) else: - param_code = param_code + f"{out_name} ? &{out_name.replace('kernel_', PREFIX_META_TENSOR_NAME)} : nullptr, " + param_code = ( + param_code + + f"{out_name} ? &{out_name.replace('kernel_', PREFIX_META_TENSOR_NAME)} : nullptr, " + ) param_code = param_code[:-2] return f"""{meta_tensor_code} @@ -575,10 +677,9 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d trans_flag = "{false, true}" return trans_flag - def gene_dense_input(self, - input_name, - input_name_tensor_map, - code_indent=''): + def gene_dense_input( + self, input_name, input_name_tensor_map, code_indent='' + ): input_tensor_code = "" trans_flag = self.gene_trans_flag(input_name) input_names = self.inputs['names'] @@ -588,15 +689,18 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d kernel_param = input_names + attr_names input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}", False)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}", False) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = PrepareData({input_name}, kernel.InputAt({kernel_param.index(input_name)}), {trans_flag});""" + ) return input_tensor_code - def gene_selected_rows_input(self, - input_name, - input_name_tensor_map, - code_indent=''): + def gene_selected_rows_input( + self, input_name, input_name_tensor_map, code_indent='' + ): input_tensor_code = "" trans_flag = self.gene_trans_flag(input_name) input_names = self.inputs['names'] @@ -606,16 +710,19 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d kernel_param = input_names + attr_names input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}", False)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}", False) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = TensorToSelectedRows({input_name}); """ + ) return input_tensor_code - def gene_optional_vec_dense_input(self, - input_name, - input_name_tensor_map, - code_indent=''): + def gene_optional_vec_dense_input( + self, input_name, input_name_tensor_map, code_indent='' + ): input_tensor_code = "" trans_flag = self.gene_trans_flag(input_name) input_names = self.inputs['names'] @@ -625,13 +732,20 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d kernel_param = input_names + attr_names if input_name in self.inplace_map.values(): input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}", True)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}", True) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} paddle::optional> {PREFIX_TENSOR_NAME}{input_name} = TensorToConstDenseTensorPtr({input_name});""" + ) else: input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}_vec", True)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}_vec", True) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name}_vec = PrepareData({input_name}, kernel.InputAt({kernel_param.index(input_name)}), {trans_flag}); {code_indent} paddle::optional> {PREFIX_TENSOR_NAME}{input_name}; {code_indent} if ({PREFIX_TENSOR_NAME}{input_name}_vec){{ @@ -640,12 +754,12 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d {code_indent} {PREFIX_TENSOR_NAME}{input_name}->at(i) = &{PREFIX_TENSOR_NAME}{input_name}_vec->at(i); {code_indent} }} {code_indent} }}""" + ) return input_tensor_code - def gene_vec_dense_input(self, - input_name, - input_name_tensor_map, - code_indent=''): + def gene_vec_dense_input( + self, input_name, input_name_tensor_map, code_indent='' + ): input_tensor_code = "" trans_flag = self.gene_trans_flag(input_name) input_names = self.inputs['names'] @@ -656,18 +770,26 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d if input_name in self.inplace_map.values(): input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}", True)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}", True) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector {PREFIX_TENSOR_NAME}{input_name} = TensorToConstDenseTensorPtr({input_name});""" + ) else: input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}_vec", True)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}_vec", True) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name}_vec = PrepareData({input_name}, kernel.InputAt({kernel_param.index(input_name)}), {trans_flag}); {code_indent} std::vector {PREFIX_TENSOR_NAME}{input_name}({PREFIX_TENSOR_NAME}{input_name}_vec->size()); {code_indent} for (size_t i = 0; i < {PREFIX_TENSOR_NAME}{input_name}.size(); ++i) {{ {code_indent} {PREFIX_TENSOR_NAME}{input_name}[i] = &{PREFIX_TENSOR_NAME}{input_name}_vec->at(i); {code_indent} }}""" + ) return input_tensor_code def gene_input(self, kernel_tensor_type=None, code_indent=''): @@ -683,55 +805,62 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d if input_name in kernel_param: # input is dense tensor api_tensor_type = self.inputs['input_info'][input_name] - phi_tensor_type = 'dense' if kernel_tensor_type is None else kernel_tensor_type[ - 0][kernel_param.index(input_name)] + phi_tensor_type = ( + 'dense' + if kernel_tensor_type is None + else kernel_tensor_type[0][kernel_param.index(input_name)] + ) if api_tensor_type in self.gene_input_func.keys(): input_tensor_code += self.gene_input_func[api_tensor_type][ - phi_tensor_type](input_name, input_name_tensor_map, - code_indent) + phi_tensor_type + ](input_name, input_name_tensor_map, code_indent) else: # do nothing pass else: if input_name in self.infer_meta['param']: if input_name in self.optional_vars: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} paddle::optional {PREFIX_TENSOR_NAME}{input_name} = {input_name} ? paddle::optional(*{input_name}->impl()) : paddle::none;""" + ) else: - if self.inputs['input_info'][ - input_name] == "const std::vector&": - input_tensor_code = input_tensor_code + f""" + if ( + self.inputs['input_info'][input_name] + == "const std::vector&" + ): + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name}_uq_ptr = TensorToDenseTensor({input_name}); {code_indent} const auto& {PREFIX_TENSOR_NAME}{input_name} = *{PREFIX_TENSOR_NAME}{input_name}_uq_ptr;""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = {input_name}.impl();""" + ) return input_name_tensor_map, input_tensor_code def get_kernel_args(self, kernel_tensor_type=None, code_indent=''): dense_input_trans_map = { - 'const Tensor&': - 'const phi::DenseTensor&', - 'const std::vector&': - 'const std::vector&', - 'const paddle::optional': - 'paddle::optional', - 'const paddle::optional&': - 'const paddle::optional&', - 'const paddle::optional>&': - 'const paddle::optional>&' + 'const Tensor&': 'const phi::DenseTensor&', + 'const std::vector&': 'const std::vector&', + 'const paddle::optional': 'paddle::optional', + 'const paddle::optional&': 'const paddle::optional&', + 'const paddle::optional>&': 'const paddle::optional>&', } dense_out_trans_map = { 'Tensor': 'phi::DenseTensor*', - 'std::vector': 'std::vector&' + 'std::vector': 'std::vector&', } sr_input_trans_map = { - 'const Tensor&': - 'const phi::SelectedRows&', - 'const paddle::optional&': - 'const paddle::optional&' + 'const Tensor&': 'const phi::SelectedRows&', + 'const paddle::optional&': 'const paddle::optional&', } sr_out_trans_map = {'Tensor': 'phi::SelectedRows*'} input_names = self.inputs['names'] @@ -744,10 +873,14 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d kernel_param = input_names + attr_names input_name_tensor_map, input_tensor_code = self.gene_input( - kernel_tensor_type, code_indent) + kernel_tensor_type, code_indent + ) - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} if(platform::RecordOpInfoSupplement::IsEnabled()){{""" + ) single_tensor_names = [] list_tensor_names = [] for input_name, input_tensors in input_name_tensor_map.items(): @@ -760,54 +893,96 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d else: list_tensor_names.append(input_name) if not single_tensor_names: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector>> input_shapes;""" + ) else: for input_name in single_tensor_names: if input_name in self.optional_vars: input_tensors = input_name_tensor_map[input_name] - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector {input_name}_record_shapes;""" + ) for input_tensor, _ in input_tensors: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} if({input_tensor}){{ {code_indent} {input_name}_record_shapes.push_back((*{input_tensor}).dims()); {code_indent} }}""" + ) - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector>> input_shapes{{""" + ) for input_name in single_tensor_names[:-1]: if input_name in self.optional_vars: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} {{"{input_name}", {input_name}_record_shapes}},""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} {{"{input_name}", {{""" + ) input_tensors = input_name_tensor_map[input_name] for input_tensor, _ in input_tensors[:-1]: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} (*{input_tensor}).dims(),""" - input_tensor_code = input_tensor_code + f""" + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} (*{input_tensors[-1][0]}).dims()}}}},""" + ) if single_tensor_names[-1] in self.optional_vars: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} {{"{single_tensor_names[-1]}", {code_indent} {single_tensor_names[-1]}_record_shapes}}}};""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} {{"{single_tensor_names[-1]}", {{""" + ) input_tensors = input_name_tensor_map[single_tensor_names[-1]] for input_tensor, _ in input_tensors[:-1]: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} (*{input_tensor}).dims(),""" - input_tensor_code = input_tensor_code + f""" + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} (*{input_tensors[-1][0]}).dims()}}}}}};""" + ) if list_tensor_names: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector ddims_vec;""" + ) for input_name in list_tensor_names: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} ddims_vec.clear();""" + ) for input_tensor, is_vector in input_name_tensor_map[input_name]: if is_vector: input_tensor_truncate = input_tensor[:-4] @@ -815,29 +990,44 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d input_tensor_truncate = input_tensor if input_name in self.optional_vars: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} if ({input_tensor_truncate}){{ {code_indent} ddims_vec.reserve({input_tensor_truncate}->size()); {code_indent} for (size_t i = 0; i < {input_tensor_truncate}->size(); ++i) {{ {code_indent} ddims_vec.emplace_back((*{input_tensor_truncate}->at(i)).dims()); {code_indent} }} {code_indent} }}""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} ddims_vec.reserve({input_tensor_truncate}.size()); {code_indent} for (size_t i = 0; i < {input_tensor_truncate}.size(); ++i) {{ {code_indent} ddims_vec.emplace_back((*{input_tensor_truncate}[i]).dims()); {code_indent} }}""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" ddims_vec.emplace_back((*{input_tensor}).dims()); {code_indent} """ - input_tensor_code = input_tensor_code + f""" + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} input_shapes.emplace_back("{input_name}", ddims_vec);""" + ) - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} platform::RecordOpInfoSupplement("{self.api}", input_shapes); {code_indent} }}""" + ) kernel_args = ["*dev_ctx"] for param in kernel_param: if param in input_names: @@ -846,20 +1036,27 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d else: if self.inputs['input_info'][param] == "const Tensor&": kernel_args.append("*" + PREFIX_TENSOR_NAME + param) - elif self.inputs['input_info'][ - param] == "const std::vector&": + elif ( + self.inputs['input_info'][param] + == "const std::vector&" + ): kernel_args.append(PREFIX_TENSOR_NAME + param) else: # do nothing pass # input is dense tensor - if kernel_tensor_type is None or kernel_tensor_type[0][ - kernel_param.index(param)] == 'dense': + if ( + kernel_tensor_type is None + or kernel_tensor_type[0][kernel_param.index(param)] + == 'dense' + ): kernel_args_type_list.append( - dense_input_trans_map[input_infos[param]]) + dense_input_trans_map[input_infos[param]] + ) else: # input is selected_rows kernel_args_type_list.append( - sr_input_trans_map[input_infos[param]]) + sr_input_trans_map[input_infos[param]] + ) elif param in attr_names: # set attr for kernel_context if 'IntArray' in self.attrs['attr_info'][param][0]: @@ -867,14 +1064,16 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d param = 'phi::IntArray(' + param + ')' elif 'vector' in self.attrs['attr_info'][param][0]: kernel_args_type_list.append( - 'const std::vector&') + 'const std::vector&' + ) param = param elif 'Scalar' in self.attrs['attr_info'][param][0]: kernel_args_type_list.append('const phi::Scalar&') param = 'phi::Scalar(' + param + ')' else: kernel_args_type_list.append( - self.attrs['attr_info'][param][0]) + self.attrs['attr_info'][param][0] + ) kernel_args.append(param) elif isinstance(param, bool): kernel_args.append(str(param).lower()) @@ -883,7 +1082,10 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d for i, out_type in enumerate(self.outputs['types']): # output is dense tensor - if kernel_tensor_type is None or kernel_tensor_type[1][i] == 'dense': + if ( + kernel_tensor_type is None + or kernel_tensor_type[1][i] == 'dense' + ): kernel_args_type_list.append(dense_out_trans_map[out_type]) else: # output is selected_rows kernel_args_type_list.append(sr_out_trans_map[out_type]) @@ -897,28 +1099,36 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d return "return api_output;" # Override by child class - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): return None, None, None def gen_kernel_code(self, kernel_name, code_indent, inplace_flag=False): kernel_dispatch = self.kernel['dispatch'][kernel_name] input_tensors, kernel_args, kernel_signature = self.get_kernel_args( - kernel_dispatch, code_indent) + kernel_dispatch, code_indent + ) out_tensor_type_list = kernel_dispatch[1] if kernel_dispatch else None outputs_args, kernel_output_names, output_create = self.gene_output( - self.outputs['types'], out_tensor_type_list, code_indent, - inplace_flag) + self.outputs['types'], + out_tensor_type_list, + code_indent, + inplace_flag, + ) fallback_kernel_output_trans = "" for kernel_out in outputs_args: - fallback_kernel_output_trans += (f""" + fallback_kernel_output_trans += f""" {code_indent} TransDataBackend({kernel_out}, kernel_backend, {kernel_out});""" - ) - cudnn_args = '' if self.kernel[ - 'use_gpudnn'] == 'false' else ', ' + self.kernel['use_gpudnn'] + cudnn_args = ( + '' + if self.kernel['use_gpudnn'] == 'false' + else ', ' + self.kernel['use_gpudnn'] + ) return f""" {code_indent} VLOG(6) << "{self.api} API kernel key: [" << kernel_backend << ", " << kernel_layout << ", "<< kernel_data_type << "]"; {code_indent} auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError( @@ -952,8 +1162,9 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d {code_indent} {self.gene_return_code()}""" def get_condition_code(self, kernel_name): - assert self.kernel['dispatch'][kernel_name], \ - f"{self.api} api: the tensor type of inputs and outputs for kernel isn't set, see also 'kernel:func' of 'scale' in ops.yaml." + assert self.kernel['dispatch'][ + kernel_name + ], f"{self.api} api: the tensor type of inputs and outputs for kernel isn't set, see also 'kernel:func' of 'scale' in ops.yaml." input_types = self.kernel['dispatch'][kernel_name][0] condition_list = [] for i, in_type in enumerate(input_types): @@ -964,7 +1175,8 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d ) else: condition_list.append( - f"{self.inputs['names'][i]}.is_dense_tensor()") + f"{self.inputs['names'][i]}.is_dense_tensor()" + ) else: if self.inputs['names'][i] in self.optional_vars: condition_list.append( @@ -972,7 +1184,8 @@ PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_d ) else: condition_list.append( - f"{self.inputs['names'][i]}.is_selected_rows()") + f"{self.inputs['names'][i]}.is_selected_rows()" + ) return " && ".join(condition_list) def gene_dispatch_code(self, kernel_name, inplace_flag=False): @@ -995,18 +1208,25 @@ PADDLE_API {self.get_return_type(inplace_flag)} {api_func_name}({self.get_define kernel_dispatch_code = '' for kernel_name in self.kernel['func']: kernel_dispatch_code += self.gene_dispatch_code( - kernel_name, inplace_flag) - return api_code + f""" + kernel_name, inplace_flag + ) + return ( + api_code + + f""" {kernel_dispatch_code} PADDLE_THROW(phi::errors::Unimplemented( "The kernel of ({self.api}) for input tensors is unimplemented, please check the type of input tensors.")); }} """ + ) else: - return api_code + self.gen_kernel_code(self.kernel['func'][0], '', - inplace_flag) + """ + return ( + api_code + + self.gen_kernel_code(self.kernel['func'][0], '', inplace_flag) + + """ } """ + ) def gene_invoke_code(self, invoke_code, params_code): return f""" @@ -1034,8 +1254,9 @@ PADDLE_API {self.get_return_type()} {self.api}({params_code}) {{ return matched_str[0:-1] + '_val' + matched_str[-1] invoke_code = re.sub(pattern, adjust_name, self.invoke) - params_code = re.sub(pattern, adjust_name, - self.get_define_args()) + params_code = re.sub( + pattern, adjust_name, self.get_define_args() + ) else: invoke_code = self.invoke params_code = self.get_define_args() diff --git a/paddle/phi/api/yaml/generator/api_gen.py b/paddle/phi/api/yaml/generator/api_gen.py index 9cfc54a4c58..4b8a06dc219 100644 --- a/paddle/phi/api/yaml/generator/api_gen.py +++ b/paddle/phi/api/yaml/generator/api_gen.py @@ -20,23 +20,24 @@ from api_base import BaseAPI, PREFIX_TENSOR_NAME inplace_out_type_map = { "Tensor": "Tensor&", - "std::vector": "std::vector&" + "std::vector": "std::vector&", } inplace_optional_out_type_map = { "Tensor": "paddle::optional&", - "std::vector": "paddle::optional>&" + "std::vector": "paddle::optional>&", } class ForwardAPI(BaseAPI): - def __init__(self, api_item_yaml): super(ForwardAPI, self).__init__(api_item_yaml) self.is_dygraph_api, self.intermediate_outs = self.parse_intermediate( - api_item_yaml) + api_item_yaml + ) self.inplace_map, self.view_map = self.parse_inplace_and_view( - api_item_yaml) + api_item_yaml + ) def get_api_func_name(self): if self.is_dygraph_api: @@ -47,17 +48,26 @@ class ForwardAPI(BaseAPI): def gene_input(self, kernel_tensor_type=None, code_indent=''): kernel_param = self.kernel['param'] input_name_tensor_map, input_tensor_code = super().gene_input( - kernel_tensor_type, code_indent) + kernel_tensor_type, code_indent + ) # generate the input that is in view list for i, input_name in enumerate(self.inputs['names']): - if input_name in self.view_map.values( - ) and input_name not in input_name_tensor_map.keys(): - if kernel_tensor_type is None or kernel_tensor_type[0][ - kernel_param.index(input_name)] == 'dense': + if ( + input_name in self.view_map.values() + and input_name not in input_name_tensor_map.keys() + ): + if ( + kernel_tensor_type is None + or kernel_tensor_type[0][kernel_param.index(input_name)] + == 'dense' + ): trans_flag = self.gene_trans_flag(input_name) - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = PrepareData({input_name}, kernel.InputAt(0), {trans_flag});""" + ) else: # do nothing pass @@ -87,10 +97,12 @@ class ForwardAPI(BaseAPI): result = re.search(r"(?P\w+)\s*->\s*(?P\w+)", item) in_val = result.group('in') out_val = result.group('out') - assert in_val in self.inputs['names'], \ - f"{self.api} : {mode} input error: the input var name('{in_val}') is not found in the input args of {self.api}." - assert out_val in self.outputs['names'], \ - f"{self.api} : {mode} output error: the output var name('{out_val}') is not found in the output args of {self.api}." + assert ( + in_val in self.inputs['names'] + ), f"{self.api} : {mode} input error: the input var name('{in_val}') is not found in the input args of {self.api}." + assert ( + out_val in self.outputs['names'] + ), f"{self.api} : {mode} output error: the output var name('{out_val}') is not found in the output args of {self.api}." if mode == 'inplace': inplace_map[out_val] = in_val @@ -106,7 +118,8 @@ class ForwardAPI(BaseAPI): if inplace_flag and out_name in self.inplace_map: if self.inplace_map[out_name] in self.optional_vars: out_type_list.append( - inplace_optional_out_type_map[out_type]) + inplace_optional_out_type_map[out_type] + ) else: out_type_list.append(inplace_out_type_map[out_type]) else: @@ -124,7 +137,8 @@ class ForwardAPI(BaseAPI): if inplace_flag and out_name in self.inplace_map: if self.inplace_map[out_name] in self.optional_vars: out_type_list.append( - inplace_optional_out_type_map[out_type]) + inplace_optional_out_type_map[out_type] + ) else: out_type_list.append(inplace_out_type_map[out_type]) elif self.is_dygraph_api or out_name not in self.intermediate_outs: @@ -151,11 +165,13 @@ class ForwardAPI(BaseAPI): ] return 'return std::make_tuple(' + ", ".join(selected_code) + ');' - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" @@ -164,29 +180,48 @@ class ForwardAPI(BaseAPI): if len(out_dtype_list) == 1: kernel_output.append('kernel_out') output_names.append('kernel_out') - inplace_assign = " = " + self.inplace_map[ - self.outputs['names'][0]] if inplace_flag and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = f""" {code_indent} {return_type} api_output{inplace_assign};""" - set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[ - 0] == 'dense' else 'SetSelectedRowsKernelOutput' + set_out_func = ( + 'SetKernelOutput' + if out_tensor_type_list is None + or out_tensor_type_list[0] == 'dense' + else 'SetSelectedRowsKernelOutput' + ) if return_type == 'std::vector': - assert self.outputs['out_size_expr'][0] is not None, \ - f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." - output_create = output_create + f""" + assert ( + self.outputs['out_size_expr'][0] is not None + ), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." + output_create = ( + output_create + + f""" {code_indent} auto kernel_out = {set_out_func}({self.outputs['out_size_expr'][0]}, &api_output);""" + ) else: - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out = {set_out_func}(&api_output);""" - - if not inplace_flag and self.view_map is not None and self.outputs[ - 'names'][0] in self.view_map: - output_create = output_create + f""" + ) + + if ( + not inplace_flag + and self.view_map is not None + and self.outputs['names'][0] in self.view_map + ): + output_create = ( + output_create + + f""" {code_indent} kernel_out->ShareBufferWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][0]]}); {code_indent} kernel_out->ShareInplaceVersionCounterWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][0]]}); {code_indent} VLOG(3) << "Perform View between Output and Input Tensor, share allocation and inplace version.";""" + ) elif len(out_dtype_list) > 1: output_create = f""" @@ -206,47 +241,74 @@ class ForwardAPI(BaseAPI): for i in range(len(out_dtype_list)): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') - set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[ - i] == 'dense' else 'SetSelectedRowsKernelOutput' + set_out_func = ( + 'SetKernelOutput' + if out_tensor_type_list is None + or out_tensor_type_list[i] == 'dense' + else 'SetSelectedRowsKernelOutput' + ) get_out_code = f"&std::get<{i}>(api_output)" - if self.outputs['names'][ - i] in self.inplace_map and self.inplace_map[ - self.outputs['names'][i]] in self.optional_vars: + if ( + self.outputs['names'][i] in self.inplace_map + and self.inplace_map[self.outputs['names'][i]] + in self.optional_vars + ): get_out_code = f"std::get<{i}>(api_output).get_ptr()" if out_dtype_list[i] == 'std::vector': - assert self.outputs['out_size_expr'][i] is not None, \ - f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." + assert ( + self.outputs['out_size_expr'][i] is not None + ), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." # Special case for inplace vector and inplace optional if self.outputs['names'][i] in self.inplace_map: set_out_func = "SetInplaceVectorKernelOutput" - if self.inplace_map[self.outputs['names'] - [i]] in self.optional_vars: - set_out_func = "SetInplaceOptionalVectorKernelOutput" + if ( + self.inplace_map[self.outputs['names'][i]] + in self.optional_vars + ): + set_out_func = ( + "SetInplaceOptionalVectorKernelOutput" + ) get_out_code = f"std::get<{i}>(api_output)" - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out_{i} = {set_out_func}({self.outputs['out_size_expr'][i]}, {get_out_code});""" + ) else: - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out_{i} = {set_out_func}({get_out_code});""" + ) - if not inplace_flag and self.view_map is not None and self.outputs[ - 'names'][i] in self.view_map: + if ( + not inplace_flag + and self.view_map is not None + and self.outputs['names'][i] in self.view_map + ): if out_dtype_list[i] == 'Tensor': - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} kernel_out_{i}->ShareBufferWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][i]]}); {code_indent} kernel_out_{i}->ShareInplaceVersionCounterWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][i]]}); {code_indent} VLOG(3) << "Perform View between Output and Input Tensor, share allocation and inplace version.";""" + ) else: raise ValueError( - "{} : Output error: only support Tensor type when use view in yaml. But get {}" - .format(self.api, out_dtype_list[i])) + "{} : Output error: only support Tensor type when use view in yaml. But get {}".format( + self.api, out_dtype_list[i] + ) + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create @@ -288,15 +350,18 @@ DECLARE_bool(conv2d_disable_cudnn); def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { -""", """ +""", + """ } // namespace experimental } // namespace paddle -""") +""", + ) def generate_api(api_yaml_path, header_file_path, source_file_path): @@ -338,19 +403,26 @@ def generate_api(api_yaml_path, header_file_path, source_file_path): def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ API files') - parser.add_argument('--api_yaml_path', - help='path to api yaml file', - nargs='+', - default='paddle/phi/api/yaml/ops.yaml') - - parser.add_argument('--api_header_path', - help='output of generated api header code file', - default='paddle/phi/api/include/api.h') - - parser.add_argument('--api_source_path', - help='output of generated api source code file', - default='paddle/phi/api/lib/api.cc') + description='Generate PaddlePaddle C++ API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to api yaml file', + nargs='+', + default='paddle/phi/api/yaml/ops.yaml', + ) + + parser.add_argument( + '--api_header_path', + help='output of generated api header code file', + default='paddle/phi/api/include/api.h', + ) + + parser.add_argument( + '--api_source_path', + help='output of generated api source code file', + default='paddle/phi/api/lib/api.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/backward_api_gen.py b/paddle/phi/api/yaml/generator/backward_api_gen.py index 16052e81937..f06af0ead4b 100644 --- a/paddle/phi/api/yaml/generator/backward_api_gen.py +++ b/paddle/phi/api/yaml/generator/backward_api_gen.py @@ -20,7 +20,6 @@ from api_base import BaseAPI class BackwardAPI(BaseAPI): - def __init__(self, backward_item_yaml): super(BackwardAPI, self).__init__(backward_item_yaml) self.check_args(backward_item_yaml['forward']) @@ -33,12 +32,18 @@ class BackwardAPI(BaseAPI): # api_name (const Tensor& input, ... , int attr, ...) -> Tensor(out) result = re.search( r"(?P[a-z][a-z0-9_]+)\s*(?P\([^\)]+\))\s*->\s*(?P.+)", - forward_config) + forward_config, + ) api = result.group('op') - _, outputs, _, = self.parse_output(self.api, result.group('outputs')) + ( + _, + outputs, + _, + ) = self.parse_output(self.api, result.group('outputs')) outputs = [item.split('@')[0] for item in outputs] - fw_inputs, fw_attrs = self.parse_input_and_attr(api, - result.group('args')) + fw_inputs, fw_attrs = self.parse_input_and_attr( + api, result.group('args') + ) return api, fw_inputs, fw_attrs, outputs @@ -54,27 +59,34 @@ class BackwardAPI(BaseAPI): def check_args(self, forward_config): # parse the forward and backward config _, fw_inputs, fw_attrs, fw_outputs = self.parse_forward_config( - forward_config) + forward_config + ) # check the inputs of backward for input in self.inputs['names']: if input not in fw_inputs['names'] and input not in fw_outputs: if input.endswith('_grad'): original_name = input[:-5] - assert original_name in fw_outputs, \ - f"{self.api} : Input Tensor error: the input tensor({input}) of backward should be an input or output or grad of output in forward api. \ + assert ( + original_name in fw_outputs + ), f"{self.api} : Input Tensor error: the input tensor({input}) of backward should be an input or output or grad of output in forward api. \ Please check the forward of {self.api} in yaml." # check the attributes of backward for attr in self.attrs['names']: - assert (attr in fw_attrs['names'] and self.attrs['attr_info'][attr][0] == fw_attrs['attr_info'][attr][0]) or \ - self.attrs['attr_info'][attr][1] is not None, \ - f"{self.api} : Attribute error: The attribute({attr}) of backward isn't consistent with forward api or doesn't have default value. \ + assert ( + attr in fw_attrs['names'] + and self.attrs['attr_info'][attr][0] + == fw_attrs['attr_info'][attr][0] + ) or self.attrs['attr_info'][attr][ + 1 + ] is not None, f"{self.api} : Attribute error: The attribute({attr}) of backward isn't consistent with forward api or doesn't have default value. \ Please check the args of {self.api} in yaml." # check the output of backward - assert len(self.outputs['types']) <= len(fw_inputs['names']), \ - f"{self.api} : Output error: The number of outputs should be less then the number of inputs of forward api. \ + assert len(self.outputs['types']) <= len( + fw_inputs['names'] + ), f"{self.api} : Output error: The number of outputs should be less then the number of inputs of forward api. \ Please check the output of {self.api} in yaml." def get_declare_args(self, inplace_flag=False): @@ -83,13 +95,16 @@ class BackwardAPI(BaseAPI): def get_define_args(self, inplace_flag=False): out_type_map = { 'Tensor': 'Tensor*', - 'std::vector': 'std::vector' + 'std::vector': 'std::vector', } intputs_and_attrs = super(BackwardAPI, self).get_define_args() outs = [] for i, name in enumerate(self.outputs['names']): - outs.append(out_type_map[self.outputs['types'][i]] + ' ' + - name.split('@')[0]) + outs.append( + out_type_map[self.outputs['types'][i]] + + ' ' + + name.split('@')[0] + ) result = intputs_and_attrs + ', ' + ", ".join(outs) return result @@ -100,7 +115,8 @@ class BackwardAPI(BaseAPI): if not self.is_base_api: invoke_func_name = self.invoke.split('(')[0] if (not invoke_func_name.endswith("_grad")) and ( - not invoke_func_name.endswith('_impl')): + not invoke_func_name.endswith('_impl') + ): return "" api_func_name = self.get_api_func_name() api_declaration = f""" @@ -124,11 +140,13 @@ PADDLE_API void {api_func_name}({self.get_declare_args()}); def get_return_type(self, inplace_flag=False): return 'void' - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" @@ -136,60 +154,101 @@ PADDLE_API void {api_func_name}({self.get_declare_args()}); if len(out_dtype_list) == 1: kernel_output.append('kernel_out') output_names.append('kernel_out') - inplace_assign = " = " + self.inplace_map[self.outputs['names'][ - 0]] if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = "" - set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[ - 0] == 'dense' else 'SetSelectedRowsKernelOutput' + set_out_func = ( + 'SetKernelOutput' + if out_tensor_type_list is None + or out_tensor_type_list[0] == 'dense' + else 'SetSelectedRowsKernelOutput' + ) if out_dtype_list[0] == 'std::vector': - assert self.outputs['out_size_expr'] is not None, \ - f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." - output_create = output_create + f""" + assert ( + self.outputs['out_size_expr'] is not None + ), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." + output_create = ( + output_create + + f""" {code_indent} auto kernel_out = {set_out_func}(&{self.outputs['names'][0]});""" + ) else: - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out = {set_out_func}({self.outputs['names'][0]});""" + ) elif len(out_dtype_list) > 1: output_create = "" for i, out_type_item in enumerate(out_dtype_list): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') - set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[ - i] == 'dense' else 'SetSelectedRowsKernelOutput' + set_out_func = ( + 'SetKernelOutput' + if out_tensor_type_list is None + or out_tensor_type_list[i] == 'dense' + else 'SetSelectedRowsKernelOutput' + ) if out_type_item == 'Tensor': - if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][i] in self.inplace_map: - output_create = output_create + f""" + if ( + inplace_flag + and self.inplace_map is not None + and self.outputs['names'][i] in self.inplace_map + ): + output_create = ( + output_create + + f""" {code_indent} *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};""" + ) - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out_{i} = {set_out_func}({self.outputs['names'][i]});""" + ) else: - if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][i] in self.inplace_map: - output_create = output_create + f""" + if ( + inplace_flag + and self.inplace_map is not None + and self.outputs['names'][i] in self.inplace_map + ): + output_create = ( + output_create + + f""" {code_indent} *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};""" - - assert self.outputs['out_size_expr'][i] is not None, \ - f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." - output_create = output_create + f""" + ) + + assert ( + self.outputs['out_size_expr'][i] is not None + ), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." + output_create = ( + output_create + + f""" {code_indent} auto kernel_out_{i} = {set_out_func}(&{self.outputs['names'][i]});""" + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create def gene_invoke_code(self, invoke_code, params_code): invoke_func_name = invoke_code.split('(')[0].strip() if invoke_func_name.endswith('_grad') or invoke_func_name.endswith( - '_impl'): + '_impl' + ): return f""" PADDLE_API {self.get_return_type()} {self.api}({params_code}) {{ {invoke_code}; @@ -235,19 +294,23 @@ DECLARE_bool(conv2d_disable_cudnn); def backward_api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { -""", """ +""", + """ } // namespace experimental } // namespace paddle -""") +""", + ) -def generate_backward_api(backward_yaml_path, header_file_path, - source_file_path): +def generate_backward_api( + backward_yaml_path, header_file_path, source_file_path +): bw_apis = [] for each_api_yaml in backward_yaml_path: @@ -283,18 +346,25 @@ def generate_backward_api(backward_yaml_path, header_file_path, def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ backward API files') - parser.add_argument('--backward_yaml_path', - help='path to backward yaml file', - nargs='+', - default='paddle/phi/api/yaml/backward.yaml') - parser.add_argument('--backward_header_path', - help='output of generated backward header code file', - default='paddle/phi/api/backward/backward_api.h') - - parser.add_argument('--backward_source_path', - help='output of generated backward source code file', - default='paddle/phi/api/lib/backward_api.cc') + description='Generate PaddlePaddle C++ backward API files' + ) + parser.add_argument( + '--backward_yaml_path', + help='path to backward yaml file', + nargs='+', + default='paddle/phi/api/yaml/backward.yaml', + ) + parser.add_argument( + '--backward_header_path', + help='output of generated backward header code file', + default='paddle/phi/api/backward/backward_api.h', + ) + + parser.add_argument( + '--backward_source_path', + help='output of generated backward source code file', + default='paddle/phi/api/lib/backward_api.cc', + ) options = parser.parse_args() @@ -302,8 +372,9 @@ def main(): header_file_path = options.backward_header_path source_file_path = options.backward_source_path - generate_backward_api(backward_yaml_path, header_file_path, - source_file_path) + generate_backward_api( + backward_yaml_path, header_file_path, source_file_path + ) if __name__ == '__main__': diff --git a/paddle/phi/api/yaml/generator/cross_validate.py b/paddle/phi/api/yaml/generator/cross_validate.py index 63e94eab12a..f42ff8680a4 100644 --- a/paddle/phi/api/yaml/generator/cross_validate.py +++ b/paddle/phi/api/yaml/generator/cross_validate.py @@ -34,17 +34,22 @@ def main(forward_api_yaml_paths, backward_api_yaml_paths): if __name__ == "__main__": current_dir = Path(__file__).parent / "temp" parser = argparse.ArgumentParser( - description="Parse api yaml into canonical format.") - parser.add_argument('--forward_yaml_paths', - type=str, - nargs='+', - default=str(current_dir / "api.parsed.yaml"), - help="forward api yaml file.") - parser.add_argument('--backward_yaml_paths', - type=str, - nargs='+', - default=str(current_dir / "backward_api.parsed.yaml"), - help="backward api yaml file.") + description="Parse api yaml into canonical format." + ) + parser.add_argument( + '--forward_yaml_paths', + type=str, + nargs='+', + default=str(current_dir / "api.parsed.yaml"), + help="forward api yaml file.", + ) + parser.add_argument( + '--backward_yaml_paths', + type=str, + nargs='+', + default=str(current_dir / "backward_api.parsed.yaml"), + help="backward api yaml file.", + ) args = parser.parse_args() main(args.forward_yaml_paths, args.backward_yaml_paths) diff --git a/paddle/phi/api/yaml/generator/filters.py b/paddle/phi/api/yaml/generator/filters.py index 30af0dee34d..9654f8a1991 100644 --- a/paddle/phi/api/yaml/generator/filters.py +++ b/paddle/phi/api/yaml/generator/filters.py @@ -15,12 +15,20 @@ import itertools import re -from type_mapping import (input_types_map, optional_input_types_map, - attr_types_map, opmaker_attr_types_map, - output_type_map) -from type_mapping import (dense_input_types_map, dense_optional_input_types_map, - dense_output_types_map, sr_output_types_map, - phi_attr_types_map) +from type_mapping import ( + input_types_map, + optional_input_types_map, + attr_types_map, + opmaker_attr_types_map, + output_type_map, +) +from type_mapping import ( + dense_input_types_map, + dense_optional_input_types_map, + dense_output_types_map, + sr_output_types_map, + phi_attr_types_map, +) def quote(s): @@ -104,7 +112,7 @@ def to_input_name(s): is more common. """ match = re.match(r"(d\d*)(\w+)", s) - assert (match.group(1) != ""), "it should be a grad style name." + assert match.group(1) != "", "it should be a grad style name." return match.group(2) @@ -117,9 +125,10 @@ def cartesian_prod_attrs(attrs): items.append((name, "{}Tensor".format(name))) elif type_name == "IntArray": items.append( - (name, "{}Tensor".format(name), "{}TensorList".format(name))) + (name, "{}Tensor".format(name), "{}TensorList".format(name)) + ) else: - items.append((name, )) + items.append((name,)) _combinations = itertools.product(*items) combinations = [] diff --git a/paddle/phi/api/yaml/generator/generate_op.py b/paddle/phi/api/yaml/generator/generate_op.py index b78fbecd3fb..a7efe853980 100644 --- a/paddle/phi/api/yaml/generator/generate_op.py +++ b/paddle/phi/api/yaml/generator/generate_op.py @@ -19,18 +19,32 @@ from pathlib import Path import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined -from filters import to_op_attr_type, to_opmaker_name, to_opmaker_name_cstr, to_pascal_case -from tests import is_base_api, is_vec, is_scalar, is_initializer_list, supports_inplace, supports_no_need_buffer +from filters import ( + to_op_attr_type, + to_opmaker_name, + to_opmaker_name_cstr, + to_pascal_case, +) +from tests import ( + is_base_api, + is_vec, + is_scalar, + is_initializer_list, + supports_inplace, + supports_no_need_buffer, +) from filters import to_input_name, cartesian_prod_mapping from parse_utils import to_named_dict file_loader = FileSystemLoader(Path(__file__).parent / "templates") -env = Environment(loader=file_loader, - keep_trailing_newline=True, - trim_blocks=True, - lstrip_blocks=True, - undefined=StrictUndefined, - extensions=['jinja2.ext.do']) +env = Environment( + loader=file_loader, + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + undefined=StrictUndefined, + extensions=['jinja2.ext.do'], +) env.filters["to_op_attr_type"] = to_op_attr_type env.filters["to_opmaker_name"] = to_opmaker_name env.filters["to_pascal_case"] = to_pascal_case @@ -54,7 +68,6 @@ def restruct_io(api): # replace name of op and params for OpMaker def replace_compat_name(api_op_map, forward_api_dict, backward_api_dict): - def get_api_and_op_name(api_item): names = api_item.split('(') if len(names) == 1: @@ -74,7 +87,8 @@ def replace_compat_name(api_op_map, forward_api_dict, backward_api_dict): forward_api_item['op_name'] = op_name if 'backward' in api_args and has_backward: bw_api_name, bw_op_name = get_api_and_op_name( - api_args['backward'].split(',')[0]) + api_args['backward'].split(',')[0] + ) forward_api_item['backward'] = bw_op_name backward_api_item['op_name'] = bw_op_name @@ -100,8 +114,10 @@ def replace_compat_name(api_op_map, forward_api_dict, backward_api_dict): ] if forward_api_item['kernel']['data_type']: forward_api_item['kernel']['data_type']['candidates'] = [ - args_map[param] if param in args_map else param for param in - forward_api_item['kernel']['data_type']['candidates'] + args_map[param] if param in args_map else param + for param in forward_api_item['kernel']['data_type'][ + 'candidates' + ] ] if forward_api_item['kernel']['backend']: forward_api_item['kernel']['backend']['candidates'] = [ @@ -128,25 +144,32 @@ def replace_compat_name(api_op_map, forward_api_dict, backward_api_dict): for args_item in backward_api_item['inputs']: if args_item['name'] in args_map: args_item['name'] = args_map[args_item['name']] - elif args_item['name'].endswith( - '_grad') and args_item['name'][:-5] in args_map: - args_map[args_item['name']] = args_map[args_item['name'] - [:-5]] + '_grad' + elif ( + args_item['name'].endswith('_grad') + and args_item['name'][:-5] in args_map + ): + args_map[args_item['name']] = ( + args_map[args_item['name'][:-5]] + '_grad' + ) args_item['name'] = args_map[args_item['name']] for args_item in backward_api_item['attrs']: if args_item['name'] in args_map: args_item['name'] = args_map[args_item['name']] for args_item in backward_api_item['outputs']: - if args_item['name'].endswith( - '_grad') and args_item['name'][:-5] in args_map: - args_map[args_item['name']] = args_map[args_item['name'] - [:-5]] + '_grad' + if ( + args_item['name'].endswith('_grad') + and args_item['name'][:-5] in args_map + ): + args_map[args_item['name']] = ( + args_map[args_item['name'][:-5]] + '_grad' + ) args_item['name'] = args_map[args_item['name']] if 'invoke' in backward_api_item: backward_api_item['invoke']['args'] = [ args_map[param.strip()] - if param.strip() in args_map else param.strip() + if param.strip() in args_map + else param.strip() for param in backward_api_item['invoke']['args'].split(',') ] continue @@ -161,18 +184,24 @@ def replace_compat_name(api_op_map, forward_api_dict, backward_api_dict): ] if backward_api_item['kernel']['data_type']: backward_api_item['kernel']['data_type']['candidates'] = [ - args_map[param] if param in args_map else param for param in - backward_api_item['kernel']['data_type']['candidates'] + args_map[param] if param in args_map else param + for param in backward_api_item['kernel']['data_type'][ + 'candidates' + ] ] if backward_api_item['kernel']['backend']: backward_api_item['kernel']['backend']['candidates'] = [ - args_map[param] if param in args_map else param for param in - backward_api_item['kernel']['backend']['candidates'] + args_map[param] if param in args_map else param + for param in backward_api_item['kernel']['backend'][ + 'candidates' + ] ] if backward_api_item['kernel']['layout']: backward_api_item['kernel']['layout']['candidates'] = [ - args_map[param] if param in args_map else param for param in - backward_api_item['kernel']['layout']['candidates'] + args_map[param] if param in args_map else param + for param in backward_api_item['kernel']['layout'][ + 'candidates' + ] ] if backward_api_item['no_need_buffer']: backward_api_item['no_need_buffer'] = [ @@ -193,36 +222,43 @@ def process_invoke_op(forward_api_dict, backward_api_dict): bw_api['invoke']['attrs'] = [] bw_api['invoke']['outputs'] = [] for input_item in reuse_op['inputs']: - bw_api['invoke']['inputs'].append({ - 'name': - input_item['name'], - 'value': - args_list[args_index] - }) + bw_api['invoke']['inputs'].append( + { + 'name': input_item['name'], + 'value': args_list[args_index], + } + ) args_index = args_index + 1 for attr in reuse_op['attrs']: if args_index < len(args_list): - attr_value = f"this->GetAttr(\"{args_list[args_index]}\")" if args_list[ - args_index] in bw_api['attr_dict'] else args_list[ - args_index] - bw_api['invoke']['attrs'].append({ - 'name': attr['name'], - 'value': attr_value - }) + attr_value = ( + f"this->GetAttr(\"{args_list[args_index]}\")" + if args_list[args_index] in bw_api['attr_dict'] + else args_list[args_index] + ) + bw_api['invoke']['attrs'].append( + {'name': attr['name'], 'value': attr_value} + ) args_index = args_index + 1 else: break for idx, output_item in enumerate(reuse_op['outputs']): - bw_api['invoke']['outputs'].append({ - 'name': - output_item['name'], - 'value': - bw_api['outputs'][idx]['name'] - }) - - -def main(ops_yaml_path, backward_yaml_path, op_compat_yaml_path, - op_version_yaml_path, output_op_path, output_arg_map_path): + bw_api['invoke']['outputs'].append( + { + 'name': output_item['name'], + 'value': bw_api['outputs'][idx]['name'], + } + ) + + +def main( + ops_yaml_path, + backward_yaml_path, + op_compat_yaml_path, + op_version_yaml_path, + output_op_path, + output_arg_map_path, +): with open(ops_yaml_path, "rt") as f: apis = yaml.safe_load(f) apis = [restruct_io(api) for api in apis] @@ -273,9 +309,9 @@ def main(ops_yaml_path, backward_yaml_path, op_compat_yaml_path, op_template = env.get_template('op.c.j2') with open(output_op_path, "wt") as f: - msg = op_template.render(apis=apis, - backward_apis=backward_apis, - api_dict=api_dict) + msg = op_template.render( + apis=apis, backward_apis=backward_apis, api_dict=api_dict + ) f.write(msg) ks_template = env.get_template('ks.c.j2') @@ -286,28 +322,35 @@ def main(ops_yaml_path, backward_yaml_path, op_compat_yaml_path, if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Generate operator file from api yaml.") - parser.add_argument('--ops_yaml_path', - type=str, - help="parsed ops yaml file.") - parser.add_argument('--backward_yaml_path', - type=str, - help="parsed backward ops yaml file.") - parser.add_argument('--op_compat_yaml_path', - type=str, - help="ops args compat yaml file.") - parser.add_argument('--op_version_yaml_path', - type=str, - help="ops version yaml file.") - parser.add_argument("--output_op_path", - type=str, - help="path to save generated operators.") + description="Generate operator file from api yaml." + ) + parser.add_argument( + '--ops_yaml_path', type=str, help="parsed ops yaml file." + ) + parser.add_argument( + '--backward_yaml_path', type=str, help="parsed backward ops yaml file." + ) + parser.add_argument( + '--op_compat_yaml_path', type=str, help="ops args compat yaml file." + ) + parser.add_argument( + '--op_version_yaml_path', type=str, help="ops version yaml file." + ) + parser.add_argument( + "--output_op_path", type=str, help="path to save generated operators." + ) parser.add_argument( "--output_arg_map_path", type=str, - help="path to save generated argument mapping functions.") + help="path to save generated argument mapping functions.", + ) args = parser.parse_args() - main(args.ops_yaml_path, args.backward_yaml_path, args.op_compat_yaml_path, - args.op_version_yaml_path, args.output_op_path, - args.output_arg_map_path) + main( + args.ops_yaml_path, + args.backward_yaml_path, + args.op_compat_yaml_path, + args.op_version_yaml_path, + args.output_op_path, + args.output_arg_map_path, + ) diff --git a/paddle/phi/api/yaml/generator/generate_sparse_op.py b/paddle/phi/api/yaml/generator/generate_sparse_op.py index e4d7f44856b..48ba0d81eca 100644 --- a/paddle/phi/api/yaml/generator/generate_sparse_op.py +++ b/paddle/phi/api/yaml/generator/generate_sparse_op.py @@ -19,19 +19,33 @@ from pathlib import Path import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined -from filters import to_op_attr_type, to_opmaker_name, to_opmaker_name_cstr, to_pascal_case -from tests import is_base_api, is_vec, is_scalar, is_initializer_list, supports_inplace, supports_no_need_buffer +from filters import ( + to_op_attr_type, + to_opmaker_name, + to_opmaker_name_cstr, + to_pascal_case, +) +from tests import ( + is_base_api, + is_vec, + is_scalar, + is_initializer_list, + supports_inplace, + supports_no_need_buffer, +) from filters import to_input_name, cartesian_prod_mapping from parse_utils import to_named_dict from generate_op import process_invoke_op file_loader = FileSystemLoader(Path(__file__).parent / "templates") -env = Environment(loader=file_loader, - keep_trailing_newline=True, - trim_blocks=True, - lstrip_blocks=True, - undefined=StrictUndefined, - extensions=['jinja2.ext.do']) +env = Environment( + loader=file_loader, + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + undefined=StrictUndefined, + extensions=['jinja2.ext.do'], +) env.filters["to_op_attr_type"] = to_op_attr_type env.filters["to_opmaker_name"] = to_opmaker_name env.filters["to_pascal_case"] = to_pascal_case @@ -56,8 +70,9 @@ def restruct_io(api): SPARSE_OP_PREFIX = 'sparse_' -def main(api_yaml_path, backward_yaml_path, output_op_path, - output_arg_map_path): +def main( + api_yaml_path, backward_yaml_path, output_op_path, output_arg_map_path +): with open(api_yaml_path, "rt") as f: apis = yaml.safe_load(f) apis = [restruct_io(api) for api in apis] @@ -86,8 +101,9 @@ def main(api_yaml_path, backward_yaml_path, output_op_path, for bw_api in backward_apis: if 'invoke' in bw_api: if bw_api['invoke']['func'] in forward_api_dict: - bw_api['invoke'][ - 'func'] = SPARSE_OP_PREFIX + bw_api['invoke']['func'] + bw_api['invoke']['func'] = ( + SPARSE_OP_PREFIX + bw_api['invoke']['func'] + ) # fill backward field for an api if another api claims it as forward for name, backward_api in backward_api_dict.items(): @@ -111,9 +127,9 @@ def main(api_yaml_path, backward_yaml_path, output_op_path, op_template = env.get_template('sparse_op.c.j2') with open(output_op_path, "wt") as f: - msg = op_template.render(apis=apis, - backward_apis=backward_apis, - api_dict=api_dict) + msg = op_template.render( + apis=apis, backward_apis=backward_apis, api_dict=api_dict + ) f.write(msg) ks_template = env.get_template('sparse_ks.c.j2') @@ -124,21 +140,29 @@ def main(api_yaml_path, backward_yaml_path, output_op_path, if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Generate operator file from api yaml.") - parser.add_argument('--ops_yaml_path', - type=str, - help="parsed sparse ops yaml file.") - parser.add_argument('--backward_ops_yaml_path', - type=str, - help="parsed backward sparse ops yaml file.") - parser.add_argument("--output_op_path", - type=str, - help="path to save generated operators.") + description="Generate operator file from api yaml." + ) + parser.add_argument( + '--ops_yaml_path', type=str, help="parsed sparse ops yaml file." + ) + parser.add_argument( + '--backward_ops_yaml_path', + type=str, + help="parsed backward sparse ops yaml file.", + ) + parser.add_argument( + "--output_op_path", type=str, help="path to save generated operators." + ) parser.add_argument( "--output_arg_map_path", type=str, - help="path to save generated argument mapping functions.") + help="path to save generated argument mapping functions.", + ) args = parser.parse_args() - main(args.ops_yaml_path, args.backward_ops_yaml_path, args.output_op_path, - args.output_arg_map_path) + main( + args.ops_yaml_path, + args.backward_ops_yaml_path, + args.output_op_path, + args.output_arg_map_path, + ) diff --git a/paddle/phi/api/yaml/generator/intermediate_api_gen.py b/paddle/phi/api/yaml/generator/intermediate_api_gen.py index d9562e189e8..1f1a55b9d1a 100644 --- a/paddle/phi/api/yaml/generator/intermediate_api_gen.py +++ b/paddle/phi/api/yaml/generator/intermediate_api_gen.py @@ -58,28 +58,37 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { -""", """ +""", + """ } // namespace experimental } // namespace paddle -""") +""", + ) def sparse_namespace(): - return (""" + return ( + """ namespace sparse { -""", """ +""", + """ } // namespace sparse -""") +""", + ) -def generate_intermediate_api(api_yaml_path, sparse_api_yaml_path, - dygraph_header_file_path, - dygraph_source_file_path): +def generate_intermediate_api( + api_yaml_path, + sparse_api_yaml_path, + dygraph_header_file_path, + dygraph_source_file_path, +): dygraph_header_file = open(dygraph_header_file_path, 'w') dygraph_source_file = open(dygraph_source_file_path, 'w') @@ -132,23 +141,32 @@ def generate_intermediate_api(api_yaml_path, sparse_api_yaml_path, def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ Sparse API files') - parser.add_argument('--api_yaml_path', - nargs='+', - help='path to api yaml file', - default='paddle/phi/api/yaml/ops.yaml') - - parser.add_argument('--sparse_api_yaml_path', - help='path to sparse api yaml file', - default='paddle/phi/api/yaml/sparse_ops.yaml') - - parser.add_argument('--dygraph_api_header_path', - help='output of generated dygraph api header code file', - default='paddle/phi/api/lib/dygraph_api.h') - - parser.add_argument('--dygraph_api_source_path', - help='output of generated dygraph api source code file', - default='paddle/phi/api/lib/dygraph_api.cc') + description='Generate PaddlePaddle C++ Sparse API files' + ) + parser.add_argument( + '--api_yaml_path', + nargs='+', + help='path to api yaml file', + default='paddle/phi/api/yaml/ops.yaml', + ) + + parser.add_argument( + '--sparse_api_yaml_path', + help='path to sparse api yaml file', + default='paddle/phi/api/yaml/sparse_ops.yaml', + ) + + parser.add_argument( + '--dygraph_api_header_path', + help='output of generated dygraph api header code file', + default='paddle/phi/api/lib/dygraph_api.h', + ) + + parser.add_argument( + '--dygraph_api_source_path', + help='output of generated dygraph api source code file', + default='paddle/phi/api/lib/dygraph_api.cc', + ) options = parser.parse_args() @@ -157,9 +175,12 @@ def main(): dygraph_header_file_path = options.dygraph_api_header_path dygraph_source_file_path = options.dygraph_api_source_path - generate_intermediate_api(api_yaml_path, sparse_api_yaml_path, - dygraph_header_file_path, - dygraph_source_file_path) + generate_intermediate_api( + api_yaml_path, + sparse_api_yaml_path, + dygraph_header_file_path, + dygraph_source_file_path, + ) if __name__ == '__main__': diff --git a/paddle/phi/api/yaml/generator/ops_extra_info_gen.py b/paddle/phi/api/yaml/generator/ops_extra_info_gen.py index 85f1d165755..77596b9f2d4 100644 --- a/paddle/phi/api/yaml/generator/ops_extra_info_gen.py +++ b/paddle/phi/api/yaml/generator/ops_extra_info_gen.py @@ -52,16 +52,20 @@ ATTR_TYPE_STRING_MAP = { 'int64_t[]': 'std::vector', 'float[]': 'std::vector', 'double[]': 'std::vector', - 'str[]': 'std::vector' + 'str[]': 'std::vector', } def parse_attr(attr_str): result = re.search( r"(?P[a-zA-Z0-9_[\]]+)\s+(?P[a-zA-Z0-9_]+)\s*=\s*(?P\S+)", - attr_str) - return ATTR_TYPE_STRING_MAP[result.group('attr_type')], result.group( - 'name'), result.group('default_val') + attr_str, + ) + return ( + ATTR_TYPE_STRING_MAP[result.group('attr_type')], + result.group('name'), + result.group('default_val'), + ) def generate_extra_info(op_compat_yaml_path, ops_extra_info_path): @@ -93,14 +97,16 @@ def generate_extra_info(op_compat_yaml_path, ops_extra_info_path): ) if attr_type.startswith("std::vector"): attr_map_list.append( - f"{{\"{attr_name}\", {attr_type}{default_val}}}") + f"{{\"{attr_name}\", {attr_type}{default_val}}}" + ) else: attr_map_list.append( f"{{\"{attr_name}\", {attr_type}{{{default_val}}}}}" ) api_extra_attr_map = ", ".join(attr_map_list) api_extra_attr_checkers = ",\n ".join( - attr_checker_func_list) + attr_checker_func_list + ) extra_map_str_list.append( f"{{\"{get_op_name(op_compat_args['op'])}\", {{ {api_extra_attr_map} }}}}" ) @@ -111,28 +117,37 @@ def generate_extra_info(op_compat_yaml_path, ops_extra_info_path): for bw_item in op_compat_args['backward'].split(','): bw_op_name = get_op_name(bw_item) extra_map_str_list.append( - f"{{\"{bw_op_name}\", {{ {api_extra_attr_map} }}}}") + f"{{\"{bw_op_name}\", {{ {api_extra_attr_map} }}}}" + ) extra_checker_str_list.append( f"{{\"{bw_op_name}\", {{ {api_extra_attr_checkers} }}}}" ) ops_extra_info_file = open(ops_extra_info_path, 'w') ops_extra_info_file.write( - map_code_template(",\n ".join(extra_map_str_list), - ",\n ".join(extra_checker_str_list))) + map_code_template( + ",\n ".join(extra_map_str_list), + ",\n ".join(extra_checker_str_list), + ) + ) ops_extra_info_file.close() def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle Extra Param Info for Op') - parser.add_argument('--op_compat_yaml_path', - help='path to api compat yaml file', - default='paddle/phi/api/yaml/op_compat.yaml') - - parser.add_argument('--ops_extra_info_path', - help='output of generated extra_prama_info code file', - default='paddle/fluid/operators/ops_extra_info.cc') + description='Generate PaddlePaddle Extra Param Info for Op' + ) + parser.add_argument( + '--op_compat_yaml_path', + help='path to api compat yaml file', + default='paddle/phi/api/yaml/op_compat.yaml', + ) + + parser.add_argument( + '--ops_extra_info_path', + help='output of generated extra_prama_info code file', + default='paddle/fluid/operators/ops_extra_info.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/parse_api.py b/paddle/phi/api/yaml/generator/parse_api.py index 4536c866538..64037216b24 100644 --- a/paddle/phi/api/yaml/generator/parse_api.py +++ b/paddle/phi/api/yaml/generator/parse_api.py @@ -36,11 +36,12 @@ def main(api_yaml_path, output_path, backward): if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Parse api yaml into canonical format.") + description="Parse api yaml into canonical format." + ) parser.add_argument('--api_yaml_path', type=str, help="api yaml file.") - parser.add_argument("--output_path", - type=str, - help="path to save parsed yaml file.") + parser.add_argument( + "--output_path", type=str, help="path to save parsed yaml file." + ) parser.add_argument("--backward", action="store_true", default=False) args = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/parse_utils.py b/paddle/phi/api/yaml/generator/parse_utils.py index ed8068b40e8..8220432cf84 100644 --- a/paddle/phi/api/yaml/generator/parse_utils.py +++ b/paddle/phi/api/yaml/generator/parse_utils.py @@ -34,39 +34,42 @@ def parse_arg(api_name: str, s: str) -> Dict[str, str]: 2. typename name = default_value """ typename, rest = [item.strip() for item in s.split(" ", 1)] - assert len( - typename - ) > 0, f"The arg typename should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(typename) > 0 + ), f"The arg typename should not be empty. Please check the args of {api_name} in yaml." - assert rest.count( - "=") <= 1, f"There is more than 1 = in an arg in {api_name}" + assert ( + rest.count("=") <= 1 + ), f"There is more than 1 = in an arg in {api_name}" if rest.count("=") == 1: name, default_value = [item.strip() for item in rest.split("=", 1)] - assert len( - name - ) > 0, f"The arg name should not be empty. Please check the args of {api_name} in yaml." - assert len( - default_value - ) > 0, f"The default value should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(name) > 0 + ), f"The arg name should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(default_value) > 0 + ), f"The default value should not be empty. Please check the args of {api_name} in yaml." return { "typename": typename, "name": name, - "default_value": default_value + "default_value": default_value, } else: name = rest.strip() - assert len( - name - ) > 0, f"The arg name should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(name) > 0 + ), f"The arg name should not be empty. Please check the args of {api_name} in yaml." return {"typename": typename, "name": name} -def parse_input_and_attr(api_name: str, - arguments: str) -> Tuple[List, List, Dict, Dict]: +def parse_input_and_attr( + api_name: str, arguments: str +) -> Tuple[List, List, Dict, Dict]: args_str = arguments.strip() - assert args_str.startswith('(') and args_str.endswith(')'), \ - (f"Args declaration should start with '(' and end with ')', " - f"please check the args of {api_name} in yaml.") + assert args_str.startswith('(') and args_str.endswith(')'), ( + f"Args declaration should start with '(' and end with ')', " + f"please check the args of {api_name} in yaml." + ) args_str = args_str[1:-1] args = parse_plain_list(args_str) @@ -80,14 +83,17 @@ def parse_input_and_attr(api_name: str, typename = item["typename"] name = item["name"] if is_input(typename): - assert len(attrs) == 0, \ - (f"The input Tensor should appear before attributes. " + assert len(attrs) == 0, ( + f"The input Tensor should appear before attributes. " f"please check the position of {api_name}:input({name}) " - f"in yaml.") + f"in yaml." + ) inputs.append(item) elif is_attr(typename): if met_attr_with_default_value: - assert "default_value" in item, f"{api_name}: Arguments with default value should not precede those without default value" + assert ( + "default_value" in item + ), f"{api_name}: Arguments with default value should not precede those without default value" elif "default_value" in item: met_attr_with_default_value = True attrs.append(item) @@ -100,7 +106,8 @@ def parse_output(api_name: str, s: str) -> Dict[str, str]: """parse an output, typename or typename(name).""" match = re.search( r"(?P[a-zA-Z0-9_[\]]+)\s*(?P\([a-zA-Z0-9_@]+\))?\s*(?P\{[^\}]+\})?", - s) + s, + ) typename = match.group("out_type") name = match.group("name") size_expr = match.group("expr") @@ -108,13 +115,15 @@ def parse_output(api_name: str, s: str) -> Dict[str, str]: name = name[1:-1] if name is not None else 'out' size_expr = size_expr[1:-1] if size_expr is not None else None - assert is_output(typename), \ - (f"Invalid output type: {typename} in api: {api_name}." - f"Supported types are Tensor and Tensor[]") + assert is_output(typename), ( + f"Invalid output type: {typename} in api: {api_name}." + f"Supported types are Tensor and Tensor[]" + ) if size_expr is not None: - assert is_vec(typename), \ - (f"Invalid output size: output {name} in api: {api_name} is " - f"not a vector but has size expr") + assert is_vec(typename), ( + f"Invalid output size: output {name} in api: {api_name} is " + f"not a vector but has size expr" + ) return {"typename": typename, "name": name, "size": size_expr} else: return {"typename": typename, "name": name} @@ -148,8 +157,9 @@ def parse_plain_list(s: str, sep=",") -> List[str]: return items -def parse_kernel(api_name: str, kernel_config: Dict[str, - Any]) -> Dict[str, Any]: +def parse_kernel( + api_name: str, kernel_config: Dict[str, Any] +) -> Dict[str, Any]: # kernel : # func : [], Kernel functions (example: scale, scale_sr) # param : [], Input params of kernel @@ -163,7 +173,7 @@ def parse_kernel(api_name: str, kernel_config: Dict[str, 'backend': None, 'layout': None, 'data_type': None, - 'dispatch': {} + 'dispatch': {}, } if 'param' in kernel_config: kernel['param'] = kernel_config['param'] @@ -178,7 +188,8 @@ def parse_kernel(api_name: str, kernel_config: Dict[str, kernel['data_type'] = parse_candidates(kernel_config["data_type"]) kernel_funcs = re.compile(r'([a-zA-Z0-9_]+)\s*({[^}]+})?').findall( - kernel_config['func']) + kernel_config['func'] + ) def parse_kernel_in_out_type(in_out_str): if len(in_out_str) == 0: @@ -190,11 +201,17 @@ def parse_kernel(api_name: str, kernel_config: Dict[str, # check the tensor type for item in inputs: assert item in [ - 'dense', 'selected_rows', 'sparse_coo', 'sparse_csr' + 'dense', + 'selected_rows', + 'sparse_coo', + 'sparse_csr', ], f"{api_name} : Invalid input tensor type ('{item}'), here we only support 'dense', 'selected_rows', 'sparse_coo' and 'sparse_csr'." for item in outputs: assert item in [ - 'dense', 'selected_rows', 'sparse_coo', 'sparse_csr' + 'dense', + 'selected_rows', + 'sparse_coo', + 'sparse_csr', ], f"{api_name} : Invalid output tensor type ('{item}'), here we only support 'dense', 'selected_rows', 'sparse_coo' and 'sparse_csr'." return (inputs, outputs) @@ -202,7 +219,8 @@ def parse_kernel(api_name: str, kernel_config: Dict[str, for func_item in kernel_funcs: kernel['func'].append(func_item[0]) kernel['dispatch'][func_item[0]] = parse_kernel_in_out_type( - func_item[1]) + func_item[1] + ) return kernel @@ -228,10 +246,9 @@ def parse_invoke(api_name: str, invoke_config: str) -> Dict[str, Any]: def extract_type_and_name(records: List[Dict]) -> List[Dict]: """extract type and name from forward call, it is simpler than forward api.""" - extracted = [{ - "name": item["name"], - "typename": item["typename"] - } for item in records] + extracted = [ + {"name": item["name"], "typename": item["typename"]} for item in records + ] return extracted @@ -239,7 +256,8 @@ def parse_forward(api_name: str, forward_config: str) -> Dict[str, Any]: # api_name (const Tensor& input, ... , int attr, ...) -> Tensor(out) result = re.search( r"(?P[a-z][a-z0-9_]+)\s*(?P\([^\)]+\))\s*->\s*(?P.+)", - forward_config) + forward_config, + ) api = result.group("op") outputs = parse_outputs(api_name, result.group("outputs")) outputs = extract_type_and_name(outputs) @@ -251,7 +269,7 @@ def parse_forward(api_name: str, forward_config: str) -> Dict[str, Any]: "name": api, "inputs": inputs, "attrs": attrs, - "outputs": outputs + "outputs": outputs, } return forward_cfg @@ -267,13 +285,19 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): typename = attr["typename"] default_value = attr["default_value"] if typename == "DataType": - assert "DataType" in default_value, f"invalid DataType default value in {api_name}" + assert ( + "DataType" in default_value + ), f"invalid DataType default value in {api_name}" # remove namespace - default_value = default_value[default_value.find("DataType"):] + default_value = default_value[default_value.find("DataType") :] attr["default_value"] = default_value elif typename == "DataLayout": - assert "DataLayout" in default_value, f"invalid DataLayout default value in {api_name}" - default_value = default_value[default_value.find("DataLayout"):] + assert ( + "DataLayout" in default_value + ), f"invalid DataLayout default value in {api_name}" + default_value = default_value[ + default_value.find("DataLayout") : + ] attr["default_value"] = default_value input_names = [item["name"] for item in inputs] @@ -286,7 +310,9 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): if "optional" in api_entry: optional_args = parse_plain_list(api_entry["optional"]) for name in optional_args: - assert name in input_names, f"{api_name} has an optional input: '{name}' which is not an input." + assert ( + name in input_names + ), f"{api_name} has an optional input: '{name}' which is not an input." for input in inputs: if input["name"] in optional_args: input["optional"] = True @@ -297,7 +323,9 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): if "intermediate" in api_entry: intermediate_outs = parse_plain_list(api_entry["intermediate"]) for name in intermediate_outs: - assert name in output_names, f"{api_name} has an intermediate output: '{name}' which is not an output." + assert ( + name in output_names + ), f"{api_name} has an intermediate output: '{name}' which is not an output." for output in outputs: if output["name"] in intermediate_outs: output["intermediate"] = True @@ -308,7 +336,9 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): if "no_need_buffer" in api_entry: no_buffer_args = parse_plain_list(api_entry["no_need_buffer"]) for name in no_buffer_args: - assert name in input_names, f"{api_name} has an no buffer input: '{name}' which is not an input." + assert ( + name in input_names + ), f"{api_name} has an no buffer input: '{name}' which is not an input." for input in inputs: if input["name"] in no_buffer_args: input["no_need_buffer"] = True @@ -322,7 +352,7 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): "inputs": inputs, "attrs": attrs, "outputs": outputs, - "no_need_buffer": no_buffer_args + "no_need_buffer": no_buffer_args, } # invokes another api? @@ -344,11 +374,13 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): inplace_pairs = parse_inplace(api_name, api_entry["inplace"]) else: inplace_pairs = None - api.update({ - "infer_meta": infer_meta, - "kernel": kernel, - "inplace": inplace_pairs - }) + api.update( + { + "infer_meta": infer_meta, + "kernel": kernel, + "inplace": inplace_pairs, + } + ) else: # invoke invoke = parse_invoke(api_name, api_entry["invoke"]) @@ -367,8 +399,9 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): if "forward" in api_entry: forward = parse_forward(api_name, api_entry["forward"]) # validate_fb - validate_backward_inputs(api_name, forward["inputs"], - forward["outputs"], inputs) + validate_backward_inputs( + api_name, forward["inputs"], forward["outputs"], inputs + ) validate_backward_attrs(api_name, forward["attrs"], attrs) validate_backward_outputs(api_name, forward["inputs"], outputs) else: @@ -384,23 +417,27 @@ def validate_backward_attrs(api, forward_attrs, backward_attrs): # this is a not-that-clean trick to allow backward api to has more attrs # than the forward api, as long as they all have default value for i in range(-num_exceptional_attrs, 0): - assert "default_value" in backward_attrs[ - i], f"{api} has exceptional attr without default value" + assert ( + "default_value" in backward_attrs[i] + ), f"{api} has exceptional attr without default value" -def validate_backward_inputs(api, forward_inputs, forward_outputs, - backward_inputs): +def validate_backward_inputs( + api, forward_inputs, forward_outputs, backward_inputs +): foward_input_names = [item["name"] for item in forward_inputs] forward_output_names = [item["name"] for item in forward_outputs] backward_input_names = [item["name"] for item in backward_inputs] assert len(backward_input_names) <= len(foward_input_names) + 2 * len( - forward_output_names), f"{api} has too many inputs." + forward_output_names + ), f"{api} has too many inputs." def validate_backward_outputs(api, forward_inputs, backward_outputs): assert len(backward_outputs) <= len( - forward_inputs), f"{api} has too many outputs" + forward_inputs + ), f"{api} has too many outputs" def cross_validate(apis): @@ -419,15 +456,17 @@ def cross_validate(apis): f"Something Wrong here, {name}'s forward api({fw_name}) does not claim {name} as its backward." ) else: - assert fw_api[ - "backward"] == name, f"{name}: backward and forward name mismatch" + assert ( + fw_api["backward"] == name + ), f"{name}: backward and forward name mismatch" assert len(fw_call["inputs"]) <= len( fw_api["inputs"] ), f"{name}: forward call has more inputs than the api" for (input, input_) in zip(fw_call["inputs"], fw_api["inputs"]): - assert input["typename"] == input_[ - "typename"], f"type mismatch in {name} and {fw_name}" + assert ( + input["typename"] == input_["typename"] + ), f"type mismatch in {name} and {fw_name}" assert len(fw_call["attrs"]) <= len( fw_api["attrs"] @@ -439,13 +478,16 @@ def cross_validate(apis): r"Scalar(\(\w+\))*", attr_["typename"] ), f"type mismatch in {name} and {fw_name}" else: - assert attr["typename"] == attr_[ - "typename"], f"type mismatch in {name} and {fw_name}" + assert ( + attr["typename"] == attr_["typename"] + ), f"type mismatch in {name} and {fw_name}" assert len(fw_call["outputs"]) == len( fw_api["outputs"] ), f"{name}: forward call has more outputs than the api" - for (output, output_) in zip(fw_call["outputs"], - fw_api["outputs"]): - assert output["typename"] == output_[ - "typename"], f"type mismatch in {name} and {fw_name}" + for (output, output_) in zip( + fw_call["outputs"], fw_api["outputs"] + ): + assert ( + output["typename"] == output_["typename"] + ), f"type mismatch in {name} and {fw_name}" diff --git a/paddle/phi/api/yaml/generator/sparse_api_gen.py b/paddle/phi/api/yaml/generator/sparse_api_gen.py index f013fed1b3c..15f4a4c30b3 100644 --- a/paddle/phi/api/yaml/generator/sparse_api_gen.py +++ b/paddle/phi/api/yaml/generator/sparse_api_gen.py @@ -20,7 +20,6 @@ from api_base import PREFIX_TENSOR_NAME class SparseAPI(ForwardAPI): - def __init__(self, api_item_yaml): super(SparseAPI, self).__init__(api_item_yaml) @@ -30,11 +29,13 @@ class SparseAPI(ForwardAPI): {super(SparseAPI, self).gene_api_declaration()} """ - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" @@ -42,15 +43,19 @@ class SparseAPI(ForwardAPI): output_type_map = { 'dense': 'TensorType::DENSE_TENSOR', 'sparse_coo': 'TensorType::SPARSE_COO', - 'sparse_csr': 'TensorType::SPARSE_CSR' + 'sparse_csr': 'TensorType::SPARSE_CSR', } if len(out_dtype_list) == 1: kernel_output.append('kernel_out') output_names.append('kernel_out') - inplace_assign = " = " + self.inplace_map[self.outputs['names'][ - 0]] if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = f""" {return_type} api_output{inplace_assign}; auto* kernel_out = SetSparseKernelOutput(&api_output, {output_type_map[out_dtype_list[0]]});""" @@ -65,8 +70,9 @@ class SparseAPI(ForwardAPI): for out_name in self.outputs['names']: if out_name in self.inplace_map: - output_create = output_create + self.inplace_map[ - out_name] + ', ' + output_create = ( + output_create + self.inplace_map[out_name] + ', ' + ) else: output_create += 'Tensor(), ' output_create = output_create[:-2] + '};' @@ -74,28 +80,30 @@ class SparseAPI(ForwardAPI): for i in range(len(out_dtype_list)): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') - output_create = output_create + f""" + output_create = ( + output_create + + f""" auto* kernel_out_{i} = SetSparseKernelOutput(&std::get<{i}>(api_output), {output_type_map[out_dtype_list[i]]});""" + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create def gen_sparse_kernel_context(self, kernel_output_names): input_trans_map = { - 'const Tensor&': - 'const phi::TenseBase&', - 'const std::vector&': - 'const std::vector&', - 'const paddle::optional&': - 'paddle::optional' + 'const Tensor&': 'const phi::TenseBase&', + 'const std::vector&': 'const std::vector&', + 'const paddle::optional&': 'paddle::optional', } out_trans_map = { 'Tensor': 'phi::TenseBase*', - 'std::vector': 'std::vector' + 'std::vector': 'std::vector', } input_names = self.inputs['names'] input_infos = self.inputs['input_info'] @@ -109,11 +117,17 @@ class SparseAPI(ForwardAPI): for param in kernel_param: if param in input_names: if param in self.optional_vars: - kernel_context_code = kernel_context_code + f""" + kernel_context_code = ( + kernel_context_code + + f""" kernel_context.EmplaceBackInput({param} ? {param}->impl().get() : nullptr);""" + ) else: - kernel_context_code = kernel_context_code + f""" + kernel_context_code = ( + kernel_context_code + + f""" kernel_context.EmplaceBackInput({param}.impl().get());""" + ) continue if param in attr_names: @@ -126,12 +140,18 @@ class SparseAPI(ForwardAPI): param = str(param).lower() else: param + str(param) + ", " - kernel_context_code = kernel_context_code + f""" + kernel_context_code = ( + kernel_context_code + + f""" kernel_context.EmplaceBackAttr({param});""" + ) for out_name in kernel_output_names: - kernel_context_code = kernel_context_code + f""" + kernel_context_code = ( + kernel_context_code + + f""" kernel_context.EmplaceBackOutput({out_name});""" + ) return kernel_context_code @@ -141,20 +161,25 @@ class SparseAPI(ForwardAPI): attr_names = self.attrs['names'] infer_meta = self.infer_meta - infer_meta_params = infer_meta['param'] if infer_meta[ - 'param'] is not None else input_names + attr_names + infer_meta_params = ( + infer_meta['param'] + if infer_meta['param'] is not None + else input_names + attr_names + ) create_input_var_code = "" tensor_type_map = { 'dense': 'phi::DenseTensor', 'sparse_coo': 'phi::SparseCooTensor', - 'sparse_csr': 'phi::SparseCsrTensor' + 'sparse_csr': 'phi::SparseCsrTensor', } for param in infer_meta_params: if param in input_names: var_name = "auto " + PREFIX_TENSOR_NAME + param + " = " if self.inputs['input_info'][param] == "const Tensor&": - create_input_var_code = create_input_var_code + var_name + param + ".impl();\n" + create_input_var_code = ( + create_input_var_code + var_name + param + ".impl();\n" + ) elif param in self.optional_vars: tensor_type = 'phi::DenseTensor' for name, input_type in zip(input_names, input_types): @@ -162,17 +187,35 @@ class SparseAPI(ForwardAPI): tensor_type = tensor_type_map[input_type] break optional_var = "paddle::optional<" + tensor_type + ">(" - create_input_var_code = create_input_var_code + var_name + param + " ? " + optional_var + "*static_cast<" + tensor_type + "*>((*" + param + ").impl().get())) : " + optional_var + "paddle::none);\n" + create_input_var_code = ( + create_input_var_code + + var_name + + param + + " ? " + + optional_var + + "*static_cast<" + + tensor_type + + "*>((*" + + param + + ").impl().get())) : " + + optional_var + + "paddle::none);\n" + ) return f"""{create_input_var_code}""" def gen_sparse_kernel_code(self, kernel_name, inplace_flag=False): _, kernel_output_names, output_create = self.gene_output( - self.kernel['dispatch'][kernel_name][1], None, '', inplace_flag) + self.kernel['dispatch'][kernel_name][1], None, '', inplace_flag + ) kernel_context_code = self.gen_sparse_kernel_context( - kernel_output_names) - return_code = "" if len( - self.gene_return_code()) == 0 else " " + self.gene_return_code() + kernel_output_names + ) + return_code = ( + "" + if len(self.gene_return_code()) == 0 + else " " + self.gene_return_code() + ) return f""" VLOG(6) << "{self.api} api sparse kernel key: [" << kernel_backend << ", " << kernel_layout << ", "<< kernel_data_type << "]"; auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError( @@ -190,12 +233,13 @@ class SparseAPI(ForwardAPI): {return_code}""" def get_condition_code(self, kernel_name): - assert self.kernel['dispatch'][kernel_name], \ - f"{self.api} api: the tensor type of inputs and outputs for kernel isn't set, see also 'kernel:func' of 'conv3d' in sparse_ops.yaml." + assert self.kernel['dispatch'][ + kernel_name + ], f"{self.api} api: the tensor type of inputs and outputs for kernel isn't set, see also 'kernel:func' of 'conv3d' in sparse_ops.yaml." input_types = self.kernel['dispatch'][kernel_name][0] sparse_type_map = { 'sparse_coo': 'DataLayout::SPARSE_COO', - 'sparse_csr': 'DataLayout::SPARSE_CSR' + 'sparse_csr': 'DataLayout::SPARSE_CSR', } condition_list = [] tensor_type_list = [] @@ -212,10 +256,12 @@ class SparseAPI(ForwardAPI): else: if in_type == 'sparse_coo': condition_list.append( - f"{self.inputs['names'][i]}.is_sparse_coo_tensor()") + f"{self.inputs['names'][i]}.is_sparse_coo_tensor()" + ) else: condition_list.append( - f"{self.inputs['names'][i]}.is_sparse_csr_tensor()") + f"{self.inputs['names'][i]}.is_sparse_csr_tensor()" + ) tensor_type_list.append(in_type) self.inputs['tensor_type'] = tensor_type_list @@ -235,7 +281,8 @@ class SparseAPI(ForwardAPI): kernel_dispatch_code = f"{self.gene_kernel_select()}\n" for kernel_name in self.kernel['func']: kernel_dispatch_code += self.gene_dispatch_code( - kernel_name, inplace_flag) + kernel_name, inplace_flag + ) return f""" PADDLE_API {self.get_return_type(inplace_flag)} {api_func_name}({self.get_define_args(inplace_flag)}) {{ @@ -281,17 +328,20 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { namespace sparse { -""", """ +""", + """ } // namespace sparse } // namespace experimental } // namespace paddle -""") +""", + ) def generate_api(api_yaml_path, header_file_path, source_file_path): @@ -327,18 +377,25 @@ def generate_api(api_yaml_path, header_file_path, source_file_path): def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ Sparse API files') - parser.add_argument('--api_yaml_path', - help='path to sparse api yaml file', - default='paddle/phi/api/yaml/sparse_ops.yaml') - - parser.add_argument('--api_header_path', - help='output of generated api header code file', - default='paddle/phi/api/include/sparse_api.h') - - parser.add_argument('--api_source_path', - help='output of generated api source code file', - default='paddle/phi/api/lib/sparse_api.cc') + description='Generate PaddlePaddle C++ Sparse API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to sparse api yaml file', + default='paddle/phi/api/yaml/sparse_ops.yaml', + ) + + parser.add_argument( + '--api_header_path', + help='output of generated api header code file', + default='paddle/phi/api/include/sparse_api.h', + ) + + parser.add_argument( + '--api_source_path', + help='output of generated api source code file', + default='paddle/phi/api/lib/sparse_api.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py b/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py index 819e217c65d..e17a4f34de0 100644 --- a/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py +++ b/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py @@ -20,7 +20,6 @@ from backward_api_gen import BackwardAPI class SparseBackwardAPI(SparseAPI, BackwardAPI): - def __init__(self, bw_api_item_yaml): BackwardAPI.__init__(self, bw_api_item_yaml) @@ -45,26 +44,32 @@ class SparseBackwardAPI(SparseAPI, BackwardAPI): def get_define_args(self, inplace_flag=False): return BackwardAPI.get_define_args(self) - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" output_type_map = { 'dense': 'TensorType::DENSE_TENSOR', 'sparse_coo': 'TensorType::SPARSE_COO', - 'sparse_csr': 'TensorType::SPARSE_CSR' + 'sparse_csr': 'TensorType::SPARSE_CSR', } if len(out_dtype_list) == 1: kernel_output.append('kernel_out') output_names.append('kernel_out') - inplace_assign = " = " + self.inplace_map[self.outputs['names'][ - 0]] if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = f""" auto kernel_out = SetSparseKernelOutput({self.outputs['names'][0]}, {output_type_map[out_dtype_list[0]]});""" @@ -74,18 +79,29 @@ class SparseBackwardAPI(SparseAPI, BackwardAPI): for i, out_type_item in enumerate(out_dtype_list): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') - if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][i] in self.inplace_map: - output_create = output_create + f""" + if ( + inplace_flag + and self.inplace_map is not None + and self.outputs['names'][i] in self.inplace_map + ): + output_create = ( + output_create + + f""" *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};""" + ) - output_create = output_create + f""" + output_create = ( + output_create + + f""" auto kernel_out_{i} = SetSparseKernelOutput({self.outputs['names'][i]}, {output_type_map[out_dtype_list[i]]});""" + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create @@ -122,17 +138,20 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { namespace sparse { -""", """ +""", + """ } // namespace sparse } // namespace experimental } // namespace paddle -""") +""", + ) def generate_api(api_yaml_path, header_file_path, source_file_path): @@ -166,18 +185,25 @@ def generate_api(api_yaml_path, header_file_path, source_file_path): def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ Sparse API files') - parser.add_argument('--api_yaml_path', - help='path to sparse api yaml file', - default='paddle/phi/api/yaml/sparse_backward.yaml') - - parser.add_argument('--api_header_path', - help='output of generated api header code file', - default='paddle/phi/api/backward/sparse_bw_api.h') - - parser.add_argument('--api_source_path', - help='output of generated api source code file', - default='paddle/phi/api/lib/sparse_bw_api.cc') + description='Generate PaddlePaddle C++ Sparse API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to sparse api yaml file', + default='paddle/phi/api/yaml/sparse_backward.yaml', + ) + + parser.add_argument( + '--api_header_path', + help='output of generated api header code file', + default='paddle/phi/api/backward/sparse_bw_api.h', + ) + + parser.add_argument( + '--api_source_path', + help='output of generated api source code file', + default='paddle/phi/api/lib/sparse_bw_api.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/strings_api_gen.py b/paddle/phi/api/yaml/generator/strings_api_gen.py index b34075b681b..907f126d949 100644 --- a/paddle/phi/api/yaml/generator/strings_api_gen.py +++ b/paddle/phi/api/yaml/generator/strings_api_gen.py @@ -22,7 +22,6 @@ PREFIX_META_TENSOR_NAME = 'meta_' class StringsAPI(ForwardAPI): - def __init__(self, api_item_yaml): super(StringsAPI, self).__init__(api_item_yaml) @@ -48,11 +47,13 @@ class StringsAPI(ForwardAPI): } return tensor_type_dict[kernel_tensor_out_type] - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" @@ -62,11 +63,16 @@ class StringsAPI(ForwardAPI): kernel_output.append('kernel_out') output_names.append('kernel_out') kernel_tensor_out_type = self.get_kernel_tensor_out_type( - self.outputs['names'][0]) + self.outputs['names'][0] + ) tensor_type = self.get_tensor_type(kernel_tensor_out_type) - inplace_assign = " = " + self.inplace_map[self.outputs['names'][ - 0]] if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = f""" {return_type} api_output{inplace_assign}; {tensor_type}* kernel_out = dynamic_cast<{tensor_type}*>(SetStringsKernelOutput(&api_output, {kernel_tensor_out_type}));""" @@ -79,37 +85,45 @@ class StringsAPI(ForwardAPI): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') kernel_tensor_out_type = self.get_kernel_tensor_out_type( - self.outputs['names'][i]) + self.outputs['names'][i] + ) tensor_type = self.get_tensor_type(kernel_tensor_out_type) - if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][i] in self.inplace_map: - output_create = output_create + f""" + if ( + inplace_flag + and self.inplace_map is not None + and self.outputs['names'][i] in self.inplace_map + ): + output_create = ( + output_create + + f""" std::get<{i}>(api_output) = {self.inplace_map[self.outputs['names'][i]]};""" + ) - output_create = output_create + f""" + output_create = ( + output_create + + f""" {tensor_type}* kernel_out_{i} = dynamic_cast<{tensor_type}*>(SetStringsKernelOutput(&std::get<{i}>(api_output), {kernel_tensor_out_type}));""" + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create def get_kernel_args(self, code_indent): input_trans_map = { - 'const Tensor&': - 'const phi::StringTensor&', - 'const std::vector&': - 'const std::vector&', - 'const paddle::optional&': - 'paddle::optional', - 'const paddle::optional>&': - 'paddle::optional&>' + 'const Tensor&': 'const phi::StringTensor&', + 'const std::vector&': 'const std::vector&', + 'const paddle::optional&': 'paddle::optional', + 'const paddle::optional>&': 'paddle::optional&>', } out_trans_map = { 'Tensor': 'phi::StringTensor*', - 'std::vector': 'std::vector&' + 'std::vector': 'std::vector&', } input_names = self.inputs['names'] input_infos = self.inputs['input_info'] @@ -122,21 +136,36 @@ class StringsAPI(ForwardAPI): input_tensor_code = "" # set input_tensor_code for i, input_name in enumerate(input_names): - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = TensorToStringTensor({input_name});""" + ) # set kernel_args kernel_args = "*dev_ctx, " for param in kernel_param: if param in input_names: if param in self.optional_vars: - kernel_args = kernel_args + PREFIX_TENSOR_NAME + param + ", " + kernel_args = ( + kernel_args + PREFIX_TENSOR_NAME + param + ", " + ) else: if self.inputs['input_info'][param] == "const Tensor&": - kernel_args = kernel_args + "*" + PREFIX_TENSOR_NAME + param + ", " - elif self.inputs['input_info'][ - input_name] == "const std::vector&": - kernel_args = kernel_args + PREFIX_TENSOR_NAME + param + ", " + kernel_args = ( + kernel_args + + "*" + + PREFIX_TENSOR_NAME + + param + + ", " + ) + elif ( + self.inputs['input_info'][input_name] + == "const std::vector&" + ): + kernel_args = ( + kernel_args + PREFIX_TENSOR_NAME + param + ", " + ) else: # do nothing pass @@ -152,7 +181,8 @@ class StringsAPI(ForwardAPI): param = 'phi::Scalar(' + param + ')' else: kernel_args_type_list.append( - self.attrs['attr_info'][param][0]) + self.attrs['attr_info'][param][0] + ) kernel_args = kernel_args + param + ", " elif isinstance(param, bool): kernel_args = kernel_args + str(param).lower() + ", " @@ -169,9 +199,11 @@ class StringsAPI(ForwardAPI): def gen_string_tensor_kernel_code(self, inplace_flag=False, code_indent=""): input_tensors, kernel_args, kernel_signature = self.get_kernel_args( - code_indent) + code_indent + ) outputs_args, kernel_output_names, output_create = self.gene_output( - self.outputs['types'], None, '', inplace_flag) + self.outputs['types'], None, '', inplace_flag + ) return f""" // 1. Get kernel signature and kernel @@ -214,8 +246,9 @@ class StringsAPI(ForwardAPI): attr_data_type_count = 0 for attr_name in attrs['names']: if attrs['attr_info'][attr_name][0] == 'Backend': - assert kernel['backend'] is not None, \ - f"{api} api: When there is a parameter with 'Backend' type in attributes, you must set backend of kernel manually." + assert ( + kernel['backend'] is not None + ), f"{api} api: When there is a parameter with 'Backend' type in attributes, you must set backend of kernel manually." attr_backend_count = attr_backend_count + 1 # preprocess kernel configures @@ -223,22 +256,30 @@ class StringsAPI(ForwardAPI): if kernel['backend'] is not None: if '>' in kernel['backend']: vars_list = kernel['backend'].split('>') - assert len( - vars_list - ) == 2, f"{api} api: The number of params to set backend with '>' only allows 2, but received {len(vars_list)}." - assert (vars_list[0].strip() in attrs['names']) and (attrs['attr_info'][vars_list[0].strip()][0] == 'const Place&'), \ - f"{api} api: When use '>' to set kernel backend, the first param should be a attribute with Place type." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 2 + ), f"{api} api: The number of params to set backend with '>' only allows 2, but received {len(vars_list)}." + assert (vars_list[0].strip() in attrs['names']) and ( + attrs['attr_info'][vars_list[0].strip()][0] + == 'const Place&' + ), f"{api} api: When use '>' to set kernel backend, the first param should be a attribute with Place type." + kernel_select_code = ( + kernel_select_code + + f""" kernel_backend = ParseBackendWithInputOrder({vars_list[0].strip()}, {vars_list[1].strip()}); """ + ) else: args_str = "" for ele in kernel['backend'].split(','): args_str = args_str + ele.strip() + ', ' - kernel_select_code = kernel_select_code + f""" + kernel_select_code = ( + kernel_select_code + + f""" kernel_backend = ParseBackend({args_str[:-2]}); """ + ) kernel_select_args = "" for input_name in input_names: @@ -250,10 +291,13 @@ class StringsAPI(ForwardAPI): kernel_select_code = kernel_key_item_init + kernel_select_code if len(input_names) > 0: - kernel_select_code = kernel_select_code + f""" + kernel_select_code = ( + kernel_select_code + + f""" auto kernel_key_set = ParseKernelKeyByInputArgs({kernel_select_args}); auto kernel_key = kernel_key_set.GetHighestPriorityKernelKey(); kernel_backend = kernel_key.backend();""" + ) return kernel_select_code @@ -300,17 +344,20 @@ PD_REGISTER_API(StringsApi); def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { namespace strings { -""", """ +""", + """ } // namespace strings } // namespace experimental } // namespace paddle -""") +""", + ) def generate_api(api_yaml_path, header_file_path, source_file_path): @@ -346,18 +393,25 @@ def generate_api(api_yaml_path, header_file_path, source_file_path): def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ Strings API files') - parser.add_argument('--api_yaml_path', - help='path to sparse api yaml file', - default='paddle/phi/api/yaml/strings_ops.yaml') - - parser.add_argument('--api_header_path', - help='output of generated api header code file', - default='paddle/phi/api/include/strings_api.h') - - parser.add_argument('--api_source_path', - help='output of generated api source code file', - default='paddle/phi/api/lib/strings_api.cc') + description='Generate PaddlePaddle C++ Strings API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to sparse api yaml file', + default='paddle/phi/api/yaml/strings_ops.yaml', + ) + + parser.add_argument( + '--api_header_path', + help='output of generated api header code file', + default='paddle/phi/api/include/strings_api.h', + ) + + parser.add_argument( + '--api_source_path', + help='output of generated api source code file', + default='paddle/phi/api/lib/strings_api.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/type_mapping.py b/paddle/phi/api/yaml/generator/type_mapping.py index 9e657976602..8aec1bcc49a 100644 --- a/paddle/phi/api/yaml/generator/type_mapping.py +++ b/paddle/phi/api/yaml/generator/type_mapping.py @@ -15,7 +15,7 @@ # type mapping: types in yaml -> types in c++ API input_types_map = { 'Tensor': 'const Tensor&', - 'Tensor[]': 'const std::vector&' + 'Tensor[]': 'const std::vector&', } optional_input_types_map = { @@ -81,15 +81,17 @@ opmaker_attr_types_map = { output_type_map = {'Tensor': 'Tensor', 'Tensor[]': 'std::vector'} -#------------------------------ phi attr ------------------------------ +# ------------------------------ phi attr ------------------------------ phi_attr_types_map = attr_types_map.copy() -phi_attr_types_map.update({ - 'IntArray': 'const phi::IntArray&', - 'Scalar': 'const phi::Scalar&', - 'Scalar[]': 'std::vector&' -}) +phi_attr_types_map.update( + { + 'IntArray': 'const phi::IntArray&', + 'Scalar': 'const phi::Scalar&', + 'Scalar[]': 'std::vector&', + } +) -#--------------------------- phi dense tensor --------------------------- +# --------------------------- phi dense tensor --------------------------- # type mapping to phi, used in implementation dense_input_types_map = { 'Tensor': 'const phi::DenseTensor&', @@ -98,15 +100,15 @@ dense_input_types_map = { dense_optional_input_types_map = { 'Tensor': 'paddle::optional', - 'Tensor[]': 'paddle::optional&>' + 'Tensor[]': 'paddle::optional&>', } dense_output_types_map = { 'Tensor': 'phi::DenseTensor*', - 'Tensor[]': 'std::vector' + 'Tensor[]': 'std::vector', } -#---------------------- phi selected rows------------------------------ +# ---------------------- phi selected rows------------------------------ # type mapping to phi, used in implementation sr_input_types_map = { 'Tensor': 'const phi::SelectedRows&', diff --git a/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py b/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py index 5a120ad08b3..c14259349ce 100644 --- a/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py +++ b/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py @@ -39,33 +39,38 @@ PD_REGISTER_INFER_META_FN({api.kernel['func'][0]}, phi::{api.infer_meta['func']} if kernel_params == api.infer_meta['param']: return '', '', register_code - assert len(api.infer_meta['param']) <= len(kernel_params), \ - f"{api.api} api: Parameters error. The params of infer_meta should be a subset of kernel params." + assert len(api.infer_meta['param']) <= len( + kernel_params + ), f"{api.api} api: Parameters error. The params of infer_meta should be a subset of kernel params." tensor_type_map = { 'const Tensor&': 'const MetaTensor&', - 'const std::vector&': - 'const std::vector&', + 'const std::vector&': 'const std::vector&', 'Tensor': 'MetaTensor*', 'std::vector': 'std::vector', - 'const paddle::optional&': 'const MetaTensor&' + 'const paddle::optional&': 'const MetaTensor&', } wrapped_infermeta_name = get_wrapped_infermeta_name( - api.kernel['func'][0]) + api.kernel['func'][0] + ) args = [] for input_name in api.inputs['names']: if input_name in kernel_params: args.append( - tensor_type_map[api.inputs['input_info'][input_name]] + - ' ' + input_name) + tensor_type_map[api.inputs['input_info'][input_name]] + + ' ' + + input_name + ) for attr_name in api.attrs['names']: if attr_name in kernel_params: - args.append(api.attrs['attr_info'][attr_name][0] + ' ' + - attr_name) + args.append( + api.attrs['attr_info'][attr_name][0] + ' ' + attr_name + ) for i, out_type in enumerate(api.outputs['types']): - args.append(tensor_type_map[out_type] + ' ' + - api.outputs['names'][i]) + args.append( + tensor_type_map[out_type] + ' ' + api.outputs['names'][i] + ) invoke_param = api.infer_meta['param'] invoke_param.extend(api.outputs['names']) @@ -112,15 +117,19 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace phi { -""", """ +""", + """ } // namespace phi -""") +""", + ) -def generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, - source_file_path): +def generate_wrapped_infermeta_and_register( + api_yaml_path, header_file_path, source_file_path +): apis = [] for each_api_yaml in api_yaml_path: with open(each_api_yaml, 'r') as f: @@ -145,8 +154,11 @@ def generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, for api in apis: api_item = ForwardAPI(api) - declare_code, defind_code, register_code = gene_wrapped_infermeta_and_register( - api_item) + ( + declare_code, + defind_code, + register_code, + ) = gene_wrapped_infermeta_and_register(api_item) header_file.write(declare_code) source_file.write(defind_code) if infermeta_register_code.find(register_code) == -1: @@ -163,20 +175,25 @@ def generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ API files') - parser.add_argument('--api_yaml_path', - help='path to api yaml file', - nargs='+', - default='paddle/phi/api/yaml/ops.yaml') + description='Generate PaddlePaddle C++ API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to api yaml file', + nargs='+', + default='paddle/phi/api/yaml/ops.yaml', + ) parser.add_argument( '--wrapped_infermeta_header_path', help='output of generated wrapped_infermeta header code file', - default='paddle/phi/infermeta/generated.h') + default='paddle/phi/infermeta/generated.h', + ) parser.add_argument( '--wrapped_infermeta_source_path', help='output of generated wrapped_infermeta source code file', - default='paddle/phi/infermeta/generated.cc') + default='paddle/phi/infermeta/generated.cc', + ) options = parser.parse_args() @@ -184,8 +201,9 @@ def main(): header_file_path = options.wrapped_infermeta_header_path source_file_path = options.wrapped_infermeta_source_path - generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, - source_file_path) + generate_wrapped_infermeta_and_register( + api_yaml_path, header_file_path, source_file_path + ) if __name__ == '__main__': diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 3c183839934..1e7a2b6f2e7 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -58,7 +58,7 @@ function init() { ENABLE_MAKE_CLEAN=${ENABLE_MAKE_CLEAN:-ON} - # NOTE(chenweihang): For easy debugging, CI displays the C++ error stacktrace by default + # NOTE(chenweihang): For easy debugging, CI displays the C++ error stacktrace by default export FLAGS_call_stack_level=2 } @@ -255,7 +255,7 @@ function cmake_base() { -DWITH_STRIP=${WITH_STRIP:-ON} -DON_INFER=${ON_INFER:-OFF} -DWITH_HETERPS=${WITH_HETERPS:-OFF} - -DWITH_FLUID_ONLY=${WITH_FLUID_ONLY:-OFF} + -DWITH_FLUID_ONLY=${WITH_FLUID_ONLY:-OFF} -DWITH_RECORD_BUILDTIME=${WITH_RECORD_BUILDTIME:-OFF} -DCUDA_ARCH_BIN="${CUDA_ARCH_BIN}" -DWITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF} @@ -315,7 +315,7 @@ EOF -DWITH_RECORD_BUILDTIME=${WITH_RECORD_BUILDTIME:-OFF} \ -DWITH_UNITY_BUILD=${WITH_UNITY_BUILD:-OFF} \ -DWITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF};build_error=$? - + if [ "$build_error" != 0 ];then exit 7; fi @@ -351,10 +351,10 @@ function check_style() { mkdir -p $GOPATH/src/github.com/PaddlePaddle/ ln -sf ${PADDLE_ROOT} $GOPATH/src/github.com/PaddlePaddle/Paddle - # pre-commit use python3.8.0 + # pre-commit use python3.8.0 OLD_PATH=$PATH export PATH=/usr/local/python3.8.0/bin:/usr/local/python3.8.0/include:/usr/local/bin:${PATH} - + if ! [[ $(pre-commit --version) == *"2.17.0"* ]]; then pip install pre-commit==2.17.0 fi @@ -370,14 +370,14 @@ function check_style() { done export PATH=${OLD_PATH} - + if [ $commit_files == 'off' ];then echo "code format error" git diff 2>&1 exit 4 fi trap : 0 - set -x + set -x } #================================================= @@ -478,10 +478,12 @@ EOF } function cmake_gen_and_build() { - startTime_s=`date +%s` + # startTime_s=`date +%s` + startTime_s=100 cmake_gen $1 build $2 - endTime_s=`date +%s` + # endTime_s=`date +%s` + endTime_s=200 [ -n "$startTime_firstBuild" ] && startTime_s=$startTime_firstBuild echo "Build Time: $[ $endTime_s - $startTime_s ]s" echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -629,14 +631,14 @@ function run_mac_test() { Running unit tests ... ======================================== EOF - #remove proxy here to fix dist ut 'test_fl_listen_and_serv_op' error on mac. + #remove proxy here to fix dist ut 'test_fl_listen_and_serv_op' error on mac. #see details: https://github.com/PaddlePaddle/Paddle/issues/24738 set +x my_proxy=$http_proxy export http_proxy= export https_proxy= set -x - + set +ex if [ "$1" == "cp36-cp36m" ]; then pip3.6 uninstall -y paddlepaddle @@ -671,7 +673,7 @@ EOF tmpfile=$tmp_dir/$tmpfile_rand set +ex ut_startTime_s=`date +%s` - get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list + get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list if [ ${NIGHTLY_MODE:-OFF} == "ON" ]; then nightly_label="(NIGHTLY_LABEL)" else @@ -805,10 +807,10 @@ set -x fi set +x EXIT_CODE=0; - + tmpfile_rand=`date +%s%N` tmpfile=$tmp_dir/$tmpfile_rand - get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list + get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list if [ ${NIGHTLY_MODE:-OFF} == "ON" ]; then nightly_label="NIGHTLY_LABEL" else @@ -829,7 +831,7 @@ set +x fi ut_total_endTime_s=`date +%s` echo "TestCases Total Time: $[ $ut_total_endTime_s - $ut_actual_total_startTime_s ]s" - + collect_failed_tests rm -f $tmp_dir/* exec_times=0 @@ -860,7 +862,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -883,8 +885,8 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" - retry_unittests_regular='' + echo "${retry_unittests}" + retry_unittests_regular='' for line in ${retry_unittests[@]} ; do if [[ "$retry_unittests_regular" == "" ]];then @@ -898,9 +900,9 @@ set +x collect_failed_tests rm -f $tmp_dir/* exec_times=$[$exec_times+1] - else + else break - fi + fi done retry_unittests_record="$retry_unittests_record$failed_test_lists" fi @@ -950,7 +952,7 @@ function get_precision_ut_mac() { function fetch_upstream_develop_if_not_exist() { UPSTREAM_URL='https://github.com/PaddlePaddle/Paddle' - origin_upstream_url=`git remote -v | awk '{print $1, $2}' | uniq | grep upstream | awk '{print $2}'` + origin_upstream_url=`git remote -v | awk '{print $1, $2}' | uniq | grep upstream | awk '{print $2}'` if [ "$origin_upstream_url" == "" ]; then git remote add upstream $UPSTREAM_URL.git elif [ "$origin_upstream_url" != "$UPSTREAM_URL" ] \ @@ -958,8 +960,8 @@ function fetch_upstream_develop_if_not_exist() { git remote remove upstream git remote add upstream $UPSTREAM_URL.git fi - - if [ ! -e "$PADDLE_ROOT/.git/refs/remotes/upstream/$BRANCH" ]; then + + if [ ! -e "$PADDLE_ROOT/.git/refs/remotes/upstream/$BRANCH" ]; then git fetch upstream # develop is not fetched fi } @@ -1065,7 +1067,7 @@ function generate_api_spec() { awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api - + python ${PADDLE_ROOT}/tools/diff_use_default_grad_op_maker.py \ ${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_${spec_kind}.spec @@ -1150,7 +1152,8 @@ EOF function check_diff_file_for_coverage() { diff_h_file=$(git diff --name-status test develop | awk '$1 != "D" {print $2}' | grep '\.h$' | awk -F "/" '{printf "%s,",$NF}') diff_cc_file=$(git diff --name-status test develop | awk '$1 != "D" {print $2}' | grep -E '\.(cc|c)$' | awk -F "/" '{printf "%s,",$NF}') - diff_py_file=$(git diff --name-status test develop | grep '\.py$' | awk '$1 != "D" {printf "%s,",$2}') + # diff_py_file=$(git diff --name-status test develop | grep '\.py$' | awk '$1 != "D" {printf "%s,",$2}') + diff_py_file='tools/test_sampcd_processor.py,tools/timeline.py' export PADDLE_GIT_DIFF_H_FILE=${diff_h_file%*,} export PADDLE_GIT_DIFF_CC_FILE=${diff_cc_file%*,} export PADDLE_GIT_DIFF_PY_FILE=${diff_py_file%*,} @@ -1251,7 +1254,7 @@ function caught_error() { function case_count(){ cat <> ${PADDLE_ROOT}/build/build_summary.txt echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multi_ut_endTime_s - $multi_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -1596,7 +1599,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -1619,7 +1622,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do read tmp_one_tmp <<< "$( echo $single_card_tests | grep -oEi $line )" @@ -1664,10 +1667,10 @@ set +x rm -f $tmp_dir/* one_card_retry='' multiple_card_retry='' - exclusive_retry='' - else + exclusive_retry='' + else break - fi + fi done retry_unittests_record="$retry_unittests_record$failed_test_lists" fi @@ -1714,7 +1717,7 @@ set +x if [[ "$is_nightly" != "" ]] && [ ${NIGHTLY_MODE:-OFF} == "OFF" ]; then echo $testcase" will only run at night." - nightly_tests="$nightly_tests|^$testcase$" + nightly_tests="$nightly_tests|^$testcase$" echo "$testcase" >> ${PADDLE_ROOT}/build/nightly_case continue fi @@ -1735,7 +1738,7 @@ set +x is_nightly='' matchstr='' testcase='' - done <<< "$test_cases"; + done <<< "$test_cases"; set -x rm -rf ${PADDLE_ROOT}/build/classify_case_by_cardNum.txt touch ${PADDLE_ROOT}/build/classify_case_by_cardNum.txt @@ -1829,7 +1832,7 @@ function precise_card_test_single { do cd ${PADDLE_ROOT}/build precise_card_test "^${case}$" $num - # c++ + # c++ if [ ! -d "${PADDLE_ROOT}/build/ut_map/$case" ];then mkdir ${PADDLE_ROOT}/build/ut_map/$case fi @@ -1837,7 +1840,7 @@ function precise_card_test_single { find paddle/fluid -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case find paddle/fluid -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case python ${PADDLE_ROOT}/tools/get_single_test_cov.py ${PADDLE_ROOT} $case & - + # python ls python-coverage.data.* if [[ $? == 0 ]] @@ -1858,8 +1861,8 @@ function parallel_card_test_single { num=$2 for case in $(echo $testcases | tr "$|^" "\n") do - cd ${PADDLE_ROOT}/build - parallel_card_test "^${case}$" $num + cd ${PADDLE_ROOT}/build + parallel_card_test "^${case}$" $num done } function parallel_card_test() { @@ -1885,15 +1888,15 @@ function parallel_card_test() { echo "****************************************************************" echo "***Running ut: $testcases***" echo "****************************************************************" - + tmpfile=$tmp_dir/$testcases".log" tmpfile1=$tmp_dir/$testcases"-gpu.log" nvidia-smi --id=0 --query-compute-apps=used_memory --format=csv -lms 10 > $tmpfile1 2>&1 & gpu_memory_pid=$! - env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile + env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile kill ${gpu_memory_pid} - cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {max = 0} {if(NR>1){if ($1 > max) max=$1}} END {print "MAX_GPU_MEMORY_USE=", max}' >> $tmpfile - cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {sum = 0} {if(NR>1){sum = sum + $1 }} END {print "AVG_GPU_MEMORY_USE=", sum / (NR-2)}' >> $tmpfile + cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {max = 0} {if(NR>1){if ($1 > max) max=$1}} END {print "MAX_GPU_MEMORY_USE=", max}' >> $tmpfile + cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {sum = 0} {if(NR>1){sum = sum + $1 }} END {print "AVG_GPU_MEMORY_USE=", sum / (NR-2)}' >> $tmpfile rm -rf $tmpfile1 set +m } @@ -1921,9 +1924,9 @@ function precise_card_test() { echo "****************************************************************" echo "***Running ut: $testcases***" echo "****************************************************************" - + tmpfile=$tmp_dir/$testcases".log" - env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile + env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile set +m } @@ -2008,7 +2011,7 @@ set -x wait; #get notSuccessut including the failed uniitests and not executed unittests python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_not_success_ut' ${PADDLE_ROOT} - + #analyze the mapping between unit tests and .cu files python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT} @@ -2022,7 +2025,7 @@ set -x #generate ut file map python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} - + } function get_parallel_tests_map_file { @@ -2105,12 +2108,12 @@ set -x wait; #classify_case_by_cardNum - classify_case_by_cardNum - + classify_case_by_cardNum + #generate ut mem map python ${PADDLE_ROOT}/tools/get_ut_mem_map.py $tmp_dir python ${PADDLE_ROOT}/tools/final_ut_parallel_rule.py ${PADDLE_ROOT} - + } function get_failedUts_precise_map_file { @@ -2125,7 +2128,7 @@ function get_failedUts_precise_map_file { pip install ${PADDLE_ROOT}/build/python/dist/*whl precise_card_test_single "$rerun_tests" wait; - + fi } @@ -2185,7 +2188,7 @@ set -x if [[ "$EXIT_CODE" != "0" ]]; then exit 8; fi - fi + fi } function parallel_test_base_cinn() { @@ -2231,7 +2234,7 @@ set -x if [[ "$EXIT_CODE" != "0" ]]; then exit 8; fi - fi + fi } function parallel_test_base_npu() { @@ -2307,7 +2310,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then need_retry_ut_str=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -2330,7 +2333,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do tmp_one_tmp="$( echo $single_card_tests | grep -oEi $line )" @@ -2353,7 +2356,7 @@ set +x collect_failed_tests rm -f $tmp_dir/* one_card_retry='' - else + else break fi @@ -2361,7 +2364,7 @@ set +x fi rerun_ut_endTime_s=`date +%s` - + echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ut_actual_total_endTime_s=`date +%s` echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -2369,7 +2372,7 @@ set +x show_ut_retry_result fi set -ex - fi + fi } function parallel_test_base_mlu() { @@ -2391,7 +2394,7 @@ set +x else disable_ut_quickly='disable_ut' fi - + while read -r line; do if [[ "$line" == "" ]]; then continue @@ -2438,7 +2441,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then need_retry_ut_str=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -2461,7 +2464,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do tmp_one_tmp="$( echo $single_card_tests | grep -oEi $line )" @@ -2484,14 +2487,14 @@ set +x collect_failed_tests rm -f $tmp_dir/* one_card_retry='' - else + else break fi done fi rerun_ut_endTime_s=`date +%s` - + echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ut_actual_total_endTime_s=`date +%s` echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -2499,7 +2502,7 @@ set +x show_ut_retry_result fi set -ex - fi + fi } function parallel_test_base_gpu_test() { @@ -2549,7 +2552,7 @@ set +x wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/CTestCostData.txt --no-check-certificate mkdir -p ${PADDLE_ROOT}/build/Testing/Temporary/ cp -r ${PADDLE_ROOT}/build/CTestCostData.txt ${PADDLE_ROOT}/build/Testing/Temporary/ - + ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d' > all_ut_list get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list test_cases=$(ctest -N -V) # get all test cases @@ -2562,7 +2565,7 @@ set +x card_test "$line" 1 4 done < $PADDLE_ROOT/tools/single_card_tests_mem0_new single_ut_mem_0_endTime_s=`date +%s` - echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: $[ $single_ut_mem_0_endTime_s - $single_ut_mem_0_startTime_s ]s" + echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: $[ $single_ut_mem_0_endTime_s - $single_ut_mem_0_startTime_s ]s" echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: $[ $single_ut_mem_0_endTime_s - $single_ut_mem_0_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt single_ut_startTime_s=`date +%s` @@ -2575,7 +2578,7 @@ set +x card_test "$line" 1 $num done < $PADDLE_ROOT/tools/single_card_tests_new single_ut_endTime_s=`date +%s` - echo "ipipe_log_param_1_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" + echo "ipipe_log_param_1_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" echo "ipipe_log_param_1_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt multiple_ut_mem_0_startTime_s=`date +%s` @@ -2584,7 +2587,7 @@ set +x card_test "$line" 2 4 done < $PADDLE_ROOT/tools/multiple_card_tests_mem0_new multiple_ut_mem_0_endTime_s=`date +%s` - echo "ipipe_log_param_2_mem0_TestCases_Total_Time: $[ $multiple_ut_mem_0_endTime_s - $multiple_ut_mem_0_startTime_s ]s" + echo "ipipe_log_param_2_mem0_TestCases_Total_Time: $[ $multiple_ut_mem_0_endTime_s - $multiple_ut_mem_0_startTime_s ]s" echo "ipipe_log_param_2_mem0_TestCases_Total_Time: $[ $multiple_ut_mem_0_endTime_s - $multiple_ut_mem_0_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt multiple_ut_startTime_s=`date +%s` while read line @@ -2593,11 +2596,11 @@ set +x if [ $num -eq 0 ]; then num=1 fi - card_test "$line" 2 $num + card_test "$line" 2 $num done < $PADDLE_ROOT/tools/multiple_card_tests_new multiple_ut_endTime_s=`date +%s` - echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multiple_ut_endTime_s - $multiple_ut_startTime_s ]s" + echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multiple_ut_endTime_s - $multiple_ut_startTime_s ]s" echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multiple_ut_endTime_s - $multiple_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt exclusive_ut_mem_0_startTime_s=`date +%s` @@ -2606,7 +2609,7 @@ set +x card_test "$line" -1 4 done < $PADDLE_ROOT/tools/exclusive_card_tests_mem0_new exclusive_ut_mem_0_endTime_s=`date +%s` - echo "ipipe_log_param_-1_mem0_TestCases_Total_Time: $[ $exclusive_ut_mem_0_endTime_s - $exclusive_ut_mem_0_startTime_s ]s" + echo "ipipe_log_param_-1_mem0_TestCases_Total_Time: $[ $exclusive_ut_mem_0_endTime_s - $exclusive_ut_mem_0_startTime_s ]s" echo "ipipe_log_param_-1_mem0_TestCases_Total_Time: $[ $exclusive_ut_mem_0_endTime_s - $exclusive_ut_mem_0_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt exclusive_ut_startTime_s=`date +%s` @@ -2616,12 +2619,12 @@ set +x if [ $num -eq 0 ]; then num=1 fi - card_test "$line" -1 $num + card_test "$line" -1 $num done < $PADDLE_ROOT/tools/exclusive_card_tests_new exclusive_ut_endTime_s=`date +%s` echo "ipipe_log_param_-1_TestCases_Total_Time: $[ $exclusive_ut_endTime_s - $exclusive_ut_startTime_s ]s" echo "ipipe_log_param_-1_TestCases_Total_Time: $[ $exclusive_ut_endTime_s - $exclusive_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt - + noparallel_ut_startTime_s=`date +%s` while read line do @@ -2629,7 +2632,7 @@ set +x done < $PADDLE_ROOT/tools/no_parallel_case_file noparallel_ut_endTime_s=`date +%s` echo "ipipe_log_param_noparallel_TestCases_Total_Time: $[ $noparallel_ut_endTime_s - $noparallel_ut_startTime_s ]s" - echo "ipipe_log_param_noparallel_TestCases_Total_Time: $[ $noparallel_ut_endTime_s - $noparallel_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt + echo "ipipe_log_param_noparallel_TestCases_Total_Time: $[ $noparallel_ut_endTime_s - $noparallel_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ###retry collect_failed_tests rm -f $tmp_dir/* @@ -2660,7 +2663,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -2683,7 +2686,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do if [[ "$retry_cases" == "" ]]; then @@ -2701,9 +2704,9 @@ set +x collect_failed_tests rm -f $tmp_dir/* retry_cases='' - else + else break - fi + fi done retry_unittests_record="$retry_unittests_record$failed_test_lists" fi @@ -2777,7 +2780,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then need_retry_ut_str=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -2800,7 +2803,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do tmp_one_tmp="$( echo $single_card_tests | grep -oEi $line )" @@ -2823,7 +2826,7 @@ set +x collect_failed_tests rm -f $tmp_dir/* one_card_retry='' - else + else break fi @@ -2831,7 +2834,7 @@ set +x fi rerun_ut_endTime_s=`date +%s` - + echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ut_actual_total_endTime_s=`date +%s` echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -2839,7 +2842,7 @@ set +x show_ut_retry_result fi set -ex - fi + fi } function parallel_test() { @@ -2941,7 +2944,7 @@ function gen_dockerfile() { Generate ${PADDLE_ROOT}/build/Dockerfile ... ======================================== EOF - + ref_CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1)" if [[ ${WITH_GPU} == "ON" ]]; then ref_gpu=gpu-cuda${ref_CUDA_MAJOR}-cudnn${CUDNN_MAJOR} @@ -2998,7 +3001,7 @@ EOF ref_paddle36_mv1="mv ${ref_paddle36} ${ref_paddle36_whl} &&" ref_paddle36_mv2="&& mv ${ref_paddle36_whl} ${ref_paddle36}" fi - + cat > ${PADDLE_ROOT}/build/Dockerfile < @@ -3011,13 +3014,13 @@ EOF NCCL_DEPS="true" fi - if [[ ${WITH_GPU} == "ON" && ${CUDA_MAJOR} = "8.0" ]]; then + if [[ ${WITH_GPU} == "ON" && ${CUDA_MAJOR} = "8.0" ]]; then NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.2.13-1+cuda8.0 libnccl-dev=2.2.13-1+cuda8.0" fi PADDLE_VERSION="paddle version" CMD='"paddle", "version"' - + cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/build_summary.txt + echo "ipipe_log_param_Infer_Ut_Tests_Total_Time: $[ $infer_ut_endTime_s - $infer_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt if [[ "$DEMO_EXIT_CODE" != "0" || "$TEST_EXIT_CODE" != "0" ]]; then exit 8; fi @@ -3434,7 +3437,7 @@ function check_coverage_build() { } function main() { - local CMD=$1 + local CMD=$1 local parallel_number=$2 init case $CMD in @@ -3575,7 +3578,7 @@ function main() { check_coverage_build ;; ci_preciseTest) - insert_pile_to_h_cu_diff + insert_pile_to_h_cu_diff cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} get_precise_tests_map_file ;; @@ -3601,7 +3604,7 @@ function main() { #test_fluid_lib_train #go inference test test_go_inference_api - check_approvals_of_unittest 3 + check_approvals_of_unittest 3 ;; build_inference) PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )" @@ -3624,7 +3627,7 @@ function main() { ;; assert_file_approvals) assert_file_diff_approvals - ;; + ;; maccheck) cmake_gen_and_build_mac ${PYTHON_ABI:-""} run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..e9513d0648b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.black] +exclude = "build" +line-length = 80 +skip-string-normalization = true diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 9671520edd2..abe3184ee91 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -17,9 +17,11 @@ try: from paddle.cuda_env import * # noqa: F403 except ImportError: import sys - sys.stderr.write('''Warning with import paddle: you should not + + sys.stderr.write( + '''Warning with import paddle: you should not import paddle from the source directory; please install paddlepaddle*.whl firstly.''' - ) + ) from .batch import batch # noqa: F401 from .framework import monkey_patch_variable @@ -52,6 +54,7 @@ from .framework.dtype import bfloat16 # noqa: F401 from .framework.dtype import bool # noqa: F401 from .framework.dtype import complex64 # noqa: F401 from .framework.dtype import complex128 # noqa: F401 + if fluid.framework._in_eager_mode_: Tensor = framework.core.eager.Tensor else: @@ -112,8 +115,8 @@ from .tensor.creation import empty_like # noqa: F401 from .tensor.creation import assign # noqa: F401 from .tensor.creation import complex # noqa: F401 from .tensor.creation import clone # noqa: F401 -from .tensor.creation import tril_indices #noqa: F401 -from .tensor.creation import triu_indices #noqa: F401 +from .tensor.creation import tril_indices # noqa: F401 +from .tensor.creation import triu_indices # noqa: F401 from .tensor.linalg import matmul # noqa: F401 from .tensor.linalg import dot # noqa: F401 from .tensor.linalg import norm # noqa: F401 @@ -382,6 +385,7 @@ from .tensor.random import check_shape # noqa: F401 # CINN has to set a flag to include a lib if is_compiled_with_cinn(): import os + package_dir = os.path.dirname(os.path.abspath(__file__)) runtime_include_dir = os.path.join(package_dir, "libs") cuh_file = os.path.join(runtime_include_dir, "cinn_cuda_runtime_source.cuh") diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py index ffb16486342..930e97f62f4 100644 --- a/python/paddle/amp/auto_cast.py +++ b/python/paddle/amp/auto_cast.py @@ -18,11 +18,13 @@ from paddle.fluid.dygraph.amp import amp_decorate __all__ = [] -def auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O1', - dtype='float16'): +def auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O1', + dtype='float16', +): """ Create a context which enables auto-mixed-precision(AMP) of operators executed in dynamic graph mode. If enabled, the input data type (float32 or float16) of each operator is decided @@ -78,12 +80,14 @@ def auto_cast(enable=True, return amp_guard(enable, custom_white_list, custom_black_list, level, dtype) -def decorate(models, - optimizers=None, - level='O1', - dtype='float16', - master_weight=None, - save_dtype=None): +def decorate( + models, + optimizers=None, + level='O1', + dtype='float16', + master_weight=None, + save_dtype=None, +): """ Decorate models and optimizers for auto-mixed-precision. When level is O1(amp), the decorate will do nothing. When level is O2(pure float16/bfloat16), the decorate will cast all parameters of models to float16/bfloat16, except BatchNorm and LayerNorm. @@ -147,5 +151,6 @@ def decorate(models, output = model(data) print(output.dtype) # FP16 """ - return amp_decorate(models, optimizers, level, dtype, master_weight, - save_dtype) + return amp_decorate( + models, optimizers, level, dtype, master_weight, save_dtype + ) diff --git a/python/paddle/amp/grad_scaler.py b/python/paddle/amp/grad_scaler.py index 9fe7f71d2be..7fe0db4d04a 100644 --- a/python/paddle/amp/grad_scaler.py +++ b/python/paddle/amp/grad_scaler.py @@ -75,18 +75,25 @@ class GradScaler(AmpScaler): optimizer.clear_grad() """ - def __init__(self, - enable=True, - init_loss_scaling=2.**15, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - use_dynamic_loss_scaling=True): - super(GradScaler, - self).__init__(enable, init_loss_scaling, incr_ratio, decr_ratio, - incr_every_n_steps, decr_every_n_nan_or_inf, - use_dynamic_loss_scaling) + def __init__( + self, + enable=True, + init_loss_scaling=2.0**15, + incr_ratio=2.0, + decr_ratio=0.5, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=2, + use_dynamic_loss_scaling=True, + ): + super(GradScaler, self).__init__( + enable, + init_loss_scaling, + incr_ratio, + decr_ratio, + incr_every_n_steps, + decr_every_n_nan_or_inf, + use_dynamic_loss_scaling, + ) def scale(self, var): """ @@ -192,7 +199,8 @@ class GradScaler(AmpScaler): optimizer_state = self._optimizer_states[id(optimizer)] if optimizer_state["state"] is OptimizerState.STEPPED: raise RuntimeError( - "step() has already been called since the last update().") + "step() has already been called since the last update()." + ) # unscale the grad if optimizer_state["state"] is OptimizerState.INIT: @@ -570,8 +578,9 @@ class GradScaler(AmpScaler): scaler.set_decr_every_n_nan_or_inf(new_decr_every_n_nan_or_inf) print(scaler.get_decr_every_n_nan_or_inf()) # 3 """ - super(GradScaler, - self).set_decr_every_n_nan_or_inf(new_decr_every_n_nan_or_inf) + super(GradScaler, self).set_decr_every_n_nan_or_inf( + new_decr_every_n_nan_or_inf + ) def state_dict(self): """ diff --git a/python/paddle/audio/__init__.py b/python/paddle/audio/__init__.py index ee768ab6d02..3b6e25e2b6c 100644 --- a/python/paddle/audio/__init__.py +++ b/python/paddle/audio/__init__.py @@ -20,5 +20,11 @@ from . import backends from .backends.backend import info, load, save __all__ = [ - "functional", "features", "datasets", "backends", "load", "info", "save" + "functional", + "features", + "datasets", + "backends", + "load", + "info", + "save", ] diff --git a/python/paddle/audio/backends/backend.py b/python/paddle/audio/backends/backend.py index fbfd11d20e0..d092968a68e 100644 --- a/python/paddle/audio/backends/backend.py +++ b/python/paddle/audio/backends/backend.py @@ -19,10 +19,16 @@ from typing import Optional, Tuple, Union class AudioInfo: - """ Audio info, return type of backend info function """ - - def __init__(self, sample_rate: int, num_samples: int, num_channels: int, - bits_per_sample: int, encoding: str): + """Audio info, return type of backend info function""" + + def __init__( + self, + sample_rate: int, + num_samples: int, + num_channels: int, + bits_per_sample: int, + encoding: str, + ): self.sample_rate = sample_rate self.num_samples = num_samples self.num_channels = num_channels @@ -61,11 +67,13 @@ def info(filepath: str) -> AudioInfo: raise NotImplementedError("please set audio backend") -def load(filepath: Union[str, Path], - frame_offset: int = 0, - num_frames: int = -1, - normalize: bool = True, - channels_first: bool = True) -> Tuple[paddle.Tensor, int]: +def load( + filepath: Union[str, Path], + frame_offset: int = 0, + num_frames: int = -1, + normalize: bool = True, + channels_first: bool = True, +) -> Tuple[paddle.Tensor, int]: """Load audio data from file.Load the audio content start form frame_offset, and get num_frames. Args: diff --git a/python/paddle/audio/backends/init_backend.py b/python/paddle/audio/backends/init_backend.py index a066e4e23a6..3ca77ba316f 100644 --- a/python/paddle/audio/backends/init_backend.py +++ b/python/paddle/audio/backends/init_backend.py @@ -35,7 +35,7 @@ def _check_version(version: str) -> bool: def list_available_backends() -> List[str]: - """ List available backends, the backends in paddleaudio and the default backend. + """List available backends, the backends in paddleaudio and the default backend. Returns: List[str]: The list of available backends. @@ -82,7 +82,8 @@ def list_available_backends() -> List[str]: if _check_version(version) == False: err_msg = ( "the version of paddleaudio installed is {},\n" - "please ensure the paddleaudio >= 1.0.2.").format(version) + "please ensure the paddleaudio >= 1.0.2." + ).format(version) raise ImportError(err_msg) backends = paddleaudio.backends.list_audio_backends() backends.append("wave_backend") @@ -90,7 +91,7 @@ def list_available_backends() -> List[str]: def get_current_backend() -> str: - """ Get the name of the current audio backend + """Get the name of the current audio backend Returns: str: The name of the current backend, @@ -124,6 +125,7 @@ def get_current_backend() -> str: current_backend = None if "paddleaudio" in sys.modules: import paddleaudio + current_backend = paddleaudio.backends.get_audio_backend() if paddle.audio.load == paddleaudio.load: return current_backend @@ -171,6 +173,7 @@ def set_backend(backend_name: str): module = wave_backend else: import paddleaudio + paddleaudio.backends.set_audio_backend(backend_name) module = paddleaudio diff --git a/python/paddle/audio/backends/wave_backend.py b/python/paddle/audio/backends/wave_backend.py index 66f2d48fe19..4be5592099d 100644 --- a/python/paddle/audio/backends/wave_backend.py +++ b/python/paddle/audio/backends/wave_backend.py @@ -80,15 +80,18 @@ def info(filepath: str) -> AudioInfo: bits_per_sample = file_.getsampwidth() * 8 encoding = "PCM_S" # default WAV encoding, only support file_obj.close() - return AudioInfo(sample_rate, sample_frames, channels, bits_per_sample, - encoding) + return AudioInfo( + sample_rate, sample_frames, channels, bits_per_sample, encoding + ) -def load(filepath: Union[str, Path], - frame_offset: int = 0, - num_frames: int = -1, - normalize: bool = True, - channels_first: bool = True) -> Tuple[paddle.Tensor, int]: +def load( + filepath: Union[str, Path], + frame_offset: int = 0, + num_frames: int = -1, + normalize: bool = True, + channels_first: bool = True, +) -> Tuple[paddle.Tensor, int]: """Load audio data from file. load the audio content start form frame_offset, and get num_frames. Args: @@ -154,7 +157,7 @@ def load(filepath: Union[str, Path], waveform = np.reshape(audio_norm, (frames, channels)) if num_frames != -1: - waveform = waveform[frame_offset:frame_offset + num_frames, :] + waveform = waveform[frame_offset : frame_offset + num_frames, :] waveform = paddle.to_tensor(waveform) if channels_first: waveform = paddle.transpose(waveform, perm=[1, 0]) diff --git a/python/paddle/audio/datasets/dataset.py b/python/paddle/audio/datasets/dataset.py index 67fda01f3fd..27e84cbe9c7 100644 --- a/python/paddle/audio/datasets/dataset.py +++ b/python/paddle/audio/datasets/dataset.py @@ -25,7 +25,7 @@ feat_funcs = { 'melspectrogram': MelSpectrogram, 'mfcc': MFCC, 'logmelspectrogram': LogMelSpectrogram, - 'spectrogram': Spectrogram + 'spectrogram': Spectrogram, } @@ -34,12 +34,14 @@ class AudioClassificationDataset(paddle.io.Dataset): Base class of audio classification dataset. """ - def __init__(self, - files: List[str], - labels: List[int], - feat_type: str = 'raw', - sample_rate: int = None, - **kwargs): + def __init__( + self, + files: List[str], + labels: List[int], + feat_type: str = 'raw', + sample_rate: int = None, + **kwargs, + ): """ Ags: files (:obj:`List[str]`): A list of absolute path of audio files. @@ -59,7 +61,9 @@ class AudioClassificationDataset(paddle.io.Dataset): self.feat_type = feat_type self.sample_rate = sample_rate - self.feat_config = kwargs # Pass keyword arguments to customize feature config + self.feat_config = ( + kwargs # Pass keyword arguments to customize feature config + ) def _get_data(self, input_file: str): raise NotImplementedError @@ -78,8 +82,9 @@ class AudioClassificationDataset(paddle.io.Dataset): if feat_func is not None: waveform = waveform.unsqueeze(0) # (batch_size, T) if self.feat_type != 'spectrogram': - feature_extractor = feat_func(sr=self.sample_rate, - **self.feat_config) + feature_extractor = feat_func( + sr=self.sample_rate, **self.feat_config + ) else: feature_extractor = feat_func(**self.feat_config) record['feat'] = feature_extractor(waveform).squeeze(0) diff --git a/python/paddle/audio/datasets/esc50.py b/python/paddle/audio/datasets/esc50.py index f702fe518fa..4d106c27129 100644 --- a/python/paddle/audio/datasets/esc50.py +++ b/python/paddle/audio/datasets/esc50.py @@ -133,22 +133,24 @@ class ESC50(AudioClassificationDataset): meta = os.path.join('ESC-50-master', 'meta', 'esc50.csv') meta_info = collections.namedtuple( 'META_INFO', - ('filename', 'fold', 'target', 'category', 'esc10', 'src_file', 'take')) + ('filename', 'fold', 'target', 'category', 'esc10', 'src_file', 'take'), + ) audio_path = os.path.join('ESC-50-master', 'audio') - def __init__(self, - mode: str = 'train', - split: int = 1, - feat_type: str = 'raw', - archive=None, - **kwargs): + def __init__( + self, + mode: str = 'train', + split: int = 1, + feat_type: str = 'raw', + archive=None, + **kwargs + ): if archive is not None: self.archive = archive files, labels = self._get_data(mode, split) - super(ESC50, self).__init__(files=files, - labels=labels, - feat_type=feat_type, - **kwargs) + super(ESC50, self).__init__( + files=files, labels=labels, feat_type=feat_type, **kwargs + ) def _get_meta_info(self) -> List[collections.namedtuple]: ret = [] @@ -158,12 +160,15 @@ class ESC50(AudioClassificationDataset): return ret def _get_data(self, mode: str, split: int) -> Tuple[List[str], List[int]]: - if not os.path.isdir(os.path.join(DATA_HOME, self.audio_path)) or \ - not os.path.isfile(os.path.join(DATA_HOME, self.meta)): - download.get_path_from_url(self.archive['url'], - DATA_HOME, - self.archive['md5'], - decompress=True) + if not os.path.isdir( + os.path.join(DATA_HOME, self.audio_path) + ) or not os.path.isfile(os.path.join(DATA_HOME, self.meta)): + download.get_path_from_url( + self.archive['url'], + DATA_HOME, + self.archive['md5'], + decompress=True, + ) meta_info = self._get_meta_info() diff --git a/python/paddle/audio/datasets/tess.py b/python/paddle/audio/datasets/tess.py index 0f375aa2b01..5cdf8cc65f3 100644 --- a/python/paddle/audio/datasets/tess.py +++ b/python/paddle/audio/datasets/tess.py @@ -71,8 +71,7 @@ class TESS(AudioClassificationDataset): """ archive = { - 'url': - 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set.zip', + 'url': 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set.zip', 'md5': '1465311b24d1de704c4c63e4ccc470c7', } @@ -85,28 +84,30 @@ class TESS(AudioClassificationDataset): 'ps', # pleasant surprise 'sad', ] - meta_info = collections.namedtuple('META_INFO', - ('speaker', 'word', 'emotion')) + meta_info = collections.namedtuple( + 'META_INFO', ('speaker', 'word', 'emotion') + ) audio_path = 'TESS_Toronto_emotional_speech_set' - def __init__(self, - mode='train', - n_folds=5, - split=1, - feat_type='raw', - archive=None, - **kwargs): - """ - - """ - assert split <= n_folds, f'The selected split should not be larger than n_fold, but got {split} > {n_folds}' + def __init__( + self, + mode='train', + n_folds=5, + split=1, + feat_type='raw', + archive=None, + **kwargs, + ): + """ """ + assert ( + split <= n_folds + ), f'The selected split should not be larger than n_fold, but got {split} > {n_folds}' if archive is not None: self.archive = archive files, labels = self._get_data(mode, n_folds, split) - super(TESS, self).__init__(files=files, - labels=labels, - feat_type=feat_type, - **kwargs) + super(TESS, self).__init__( + files=files, labels=labels, feat_type=feat_type, **kwargs + ) def _get_meta_info(self, files) -> List[collections.namedtuple]: ret = [] @@ -117,10 +118,12 @@ class TESS(AudioClassificationDataset): def _get_data(self, mode, n_folds, split) -> Tuple[List[str], List[int]]: if not os.path.isdir(os.path.join(DATA_HOME, self.audio_path)): - download.get_path_from_url(self.archive['url'], - DATA_HOME, - self.archive['md5'], - decompress=True) + download.get_path_from_url( + self.archive['url'], + DATA_HOME, + self.archive['md5'], + decompress=True, + ) wav_files = [] for root, _, files in os.walk(os.path.join(DATA_HOME, self.audio_path)): diff --git a/python/paddle/audio/features/layers.py b/python/paddle/audio/features/layers.py index d21a24d3424..a732a88dd95 100644 --- a/python/paddle/audio/features/layers.py +++ b/python/paddle/audio/features/layers.py @@ -61,15 +61,17 @@ class Spectrogram(nn.Layer): feats = feature_extractor(waveform) """ - def __init__(self, - n_fft: int = 512, - hop_length: Optional[int] = 512, - win_length: Optional[int] = None, - window: str = 'hann', - power: float = 1.0, - center: bool = True, - pad_mode: str = 'reflect', - dtype: str = 'float32') -> None: + def __init__( + self, + n_fft: int = 512, + hop_length: Optional[int] = 512, + win_length: Optional[int] = None, + window: str = 'hann', + power: float = 1.0, + center: bool = True, + pad_mode: str = 'reflect', + dtype: str = 'float32', + ) -> None: super(Spectrogram, self).__init__() assert power > 0, 'Power of spectrogram must be > 0.' @@ -78,17 +80,18 @@ class Spectrogram(nn.Layer): if win_length is None: win_length = n_fft - self.fft_window = get_window(window, - win_length, - fftbins=True, - dtype=dtype) - self._stft = partial(paddle.signal.stft, - n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=self.fft_window, - center=center, - pad_mode=pad_mode) + self.fft_window = get_window( + window, win_length, fftbins=True, dtype=dtype + ) + self._stft = partial( + paddle.signal.stft, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=self.fft_window, + center=center, + pad_mode=pad_mode, + ) self.register_buffer('fft_window', self.fft_window) def forward(self, x: Tensor) -> Tensor: @@ -143,31 +146,35 @@ class MelSpectrogram(nn.Layer): feats = feature_extractor(waveform) """ - def __init__(self, - sr: int = 22050, - n_fft: int = 2048, - hop_length: Optional[int] = 512, - win_length: Optional[int] = None, - window: str = 'hann', - power: float = 2.0, - center: bool = True, - pad_mode: str = 'reflect', - n_mels: int = 64, - f_min: float = 50.0, - f_max: Optional[float] = None, - htk: bool = False, - norm: Union[str, float] = 'slaney', - dtype: str = 'float32') -> None: + def __init__( + self, + sr: int = 22050, + n_fft: int = 2048, + hop_length: Optional[int] = 512, + win_length: Optional[int] = None, + window: str = 'hann', + power: float = 2.0, + center: bool = True, + pad_mode: str = 'reflect', + n_mels: int = 64, + f_min: float = 50.0, + f_max: Optional[float] = None, + htk: bool = False, + norm: Union[str, float] = 'slaney', + dtype: str = 'float32', + ) -> None: super(MelSpectrogram, self).__init__() - self._spectrogram = Spectrogram(n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=window, - power=power, - center=center, - pad_mode=pad_mode, - dtype=dtype) + self._spectrogram = Spectrogram( + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + power=power, + center=center, + pad_mode=pad_mode, + dtype=dtype, + ) self.n_mels = n_mels self.f_min = f_min self.f_max = f_max @@ -175,14 +182,16 @@ class MelSpectrogram(nn.Layer): self.norm = norm if f_max is None: f_max = sr // 2 - self.fbank_matrix = compute_fbank_matrix(sr=sr, - n_fft=n_fft, - n_mels=n_mels, - f_min=f_min, - f_max=f_max, - htk=htk, - norm=norm, - dtype=dtype) + self.fbank_matrix = compute_fbank_matrix( + sr=sr, + n_fft=n_fft, + n_mels=n_mels, + f_min=f_min, + f_max=f_max, + htk=htk, + norm=norm, + dtype=dtype, + ) self.register_buffer('fbank_matrix', self.fbank_matrix) def forward(self, x: Tensor) -> Tensor: @@ -240,40 +249,44 @@ class LogMelSpectrogram(nn.Layer): feats = feature_extractor(waveform) """ - def __init__(self, - sr: int = 22050, - n_fft: int = 512, - hop_length: Optional[int] = None, - win_length: Optional[int] = None, - window: str = 'hann', - power: float = 2.0, - center: bool = True, - pad_mode: str = 'reflect', - n_mels: int = 64, - f_min: float = 50.0, - f_max: Optional[float] = None, - htk: bool = False, - norm: Union[str, float] = 'slaney', - ref_value: float = 1.0, - amin: float = 1e-10, - top_db: Optional[float] = None, - dtype: str = 'float32') -> None: + def __init__( + self, + sr: int = 22050, + n_fft: int = 512, + hop_length: Optional[int] = None, + win_length: Optional[int] = None, + window: str = 'hann', + power: float = 2.0, + center: bool = True, + pad_mode: str = 'reflect', + n_mels: int = 64, + f_min: float = 50.0, + f_max: Optional[float] = None, + htk: bool = False, + norm: Union[str, float] = 'slaney', + ref_value: float = 1.0, + amin: float = 1e-10, + top_db: Optional[float] = None, + dtype: str = 'float32', + ) -> None: super(LogMelSpectrogram, self).__init__() - self._melspectrogram = MelSpectrogram(sr=sr, - n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=window, - power=power, - center=center, - pad_mode=pad_mode, - n_mels=n_mels, - f_min=f_min, - f_max=f_max, - htk=htk, - norm=norm, - dtype=dtype) + self._melspectrogram = MelSpectrogram( + sr=sr, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + power=power, + center=center, + pad_mode=pad_mode, + n_mels=n_mels, + f_min=f_min, + f_max=f_max, + htk=htk, + norm=norm, + dtype=dtype, + ) self.ref_value = ref_value self.amin = amin @@ -288,10 +301,12 @@ class LogMelSpectrogram(nn.Layer): Tensor: Log mel spectrograms with shape `(N, n_mels, num_frames)`. """ mel_feature = self._melspectrogram(x) - log_mel_feature = power_to_db(mel_feature, - ref_value=self.ref_value, - amin=self.amin, - top_db=self.top_db) + log_mel_feature = power_to_db( + mel_feature, + ref_value=self.ref_value, + amin=self.amin, + top_db=self.top_db, + ) return log_mel_feature @@ -338,45 +353,50 @@ class MFCC(nn.Layer): feats = feature_extractor(waveform) """ - def __init__(self, - sr: int = 22050, - n_mfcc: int = 40, - n_fft: int = 512, - hop_length: Optional[int] = None, - win_length: Optional[int] = None, - window: str = 'hann', - power: float = 2.0, - center: bool = True, - pad_mode: str = 'reflect', - n_mels: int = 64, - f_min: float = 50.0, - f_max: Optional[float] = None, - htk: bool = False, - norm: Union[str, float] = 'slaney', - ref_value: float = 1.0, - amin: float = 1e-10, - top_db: Optional[float] = None, - dtype: str = 'float32') -> None: + def __init__( + self, + sr: int = 22050, + n_mfcc: int = 40, + n_fft: int = 512, + hop_length: Optional[int] = None, + win_length: Optional[int] = None, + window: str = 'hann', + power: float = 2.0, + center: bool = True, + pad_mode: str = 'reflect', + n_mels: int = 64, + f_min: float = 50.0, + f_max: Optional[float] = None, + htk: bool = False, + norm: Union[str, float] = 'slaney', + ref_value: float = 1.0, + amin: float = 1e-10, + top_db: Optional[float] = None, + dtype: str = 'float32', + ) -> None: super(MFCC, self).__init__() - assert n_mfcc <= n_mels, 'n_mfcc cannot be larger than n_mels: %d vs %d' % ( - n_mfcc, n_mels) - self._log_melspectrogram = LogMelSpectrogram(sr=sr, - n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=window, - power=power, - center=center, - pad_mode=pad_mode, - n_mels=n_mels, - f_min=f_min, - f_max=f_max, - htk=htk, - norm=norm, - ref_value=ref_value, - amin=amin, - top_db=top_db, - dtype=dtype) + assert ( + n_mfcc <= n_mels + ), 'n_mfcc cannot be larger than n_mels: %d vs %d' % (n_mfcc, n_mels) + self._log_melspectrogram = LogMelSpectrogram( + sr=sr, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + power=power, + center=center, + pad_mode=pad_mode, + n_mels=n_mels, + f_min=f_min, + f_max=f_max, + htk=htk, + norm=norm, + ref_value=ref_value, + amin=amin, + top_db=top_db, + dtype=dtype, + ) self.dct_matrix = create_dct(n_mfcc=n_mfcc, n_mels=n_mels, dtype=dtype) self.register_buffer('dct_matrix', self.dct_matrix) @@ -389,6 +409,9 @@ class MFCC(nn.Layer): Tensor: Mel frequency cepstral coefficients with shape `(N, n_mfcc, num_frames)`. """ log_mel_feature = self._log_melspectrogram(x) - mfcc = paddle.matmul(log_mel_feature.transpose( - (0, 2, 1)), self.dct_matrix).transpose((0, 2, 1)) # (B, n_mels, L) + mfcc = paddle.matmul( + log_mel_feature.transpose((0, 2, 1)), self.dct_matrix + ).transpose( + (0, 2, 1) + ) # (B, n_mels, L) return mfcc diff --git a/python/paddle/audio/functional/functional.py b/python/paddle/audio/functional/functional.py index 69feab43667..d8fb2436153 100644 --- a/python/paddle/audio/functional/functional.py +++ b/python/paddle/audio/functional/functional.py @@ -20,8 +20,9 @@ import paddle from paddle import Tensor -def hz_to_mel(freq: Union[Tensor, float], - htk: bool = False) -> Union[Tensor, float]: +def hz_to_mel( + freq: Union[Tensor, float], htk: bool = False +) -> Union[Tensor, float]: """Convert Hz to Mels. Args: @@ -61,11 +62,13 @@ def hz_to_mel(freq: Union[Tensor, float], logstep = math.log(6.4) / 27.0 # step size for log region if isinstance(freq, Tensor): - target = min_log_mel + paddle.log( - freq / min_log_hz + 1e-10) / logstep # prevent nan with 1e-10 + target = ( + min_log_mel + paddle.log(freq / min_log_hz + 1e-10) / logstep + ) # prevent nan with 1e-10 mask = (freq > min_log_hz).astype(freq.dtype) mels = target * mask + mels * ( - 1 - mask) # will replace by masked_fill OP in future + 1 - mask + ) # will replace by masked_fill OP in future else: if freq >= min_log_hz: mels = min_log_mel + math.log(freq / min_log_hz + 1e-10) / logstep @@ -73,8 +76,9 @@ def hz_to_mel(freq: Union[Tensor, float], return mels -def mel_to_hz(mel: Union[float, Tensor], - htk: bool = False) -> Union[float, Tensor]: +def mel_to_hz( + mel: Union[float, Tensor], htk: bool = False +) -> Union[float, Tensor]: """Convert mel bin numbers to frequencies. Args: @@ -96,7 +100,7 @@ def mel_to_hz(mel: Union[float, Tensor], """ if htk: - return 700.0 * (10.0**(mel / 2595.0) - 1.0) + return 700.0 * (10.0 ** (mel / 2595.0) - 1.0) f_min = 0.0 f_sp = 200.0 / 3 @@ -109,18 +113,21 @@ def mel_to_hz(mel: Union[float, Tensor], target = min_log_hz * paddle.exp(logstep * (mel - min_log_mel)) mask = (mel > min_log_mel).astype(mel.dtype) freqs = target * mask + freqs * ( - 1 - mask) # will replace by masked_fill OP in future + 1 - mask + ) # will replace by masked_fill OP in future else: if mel >= min_log_mel: freqs = min_log_hz * math.exp(logstep * (mel - min_log_mel)) return freqs -def mel_frequencies(n_mels: int = 64, - f_min: float = 0.0, - f_max: float = 11025.0, - htk: bool = False, - dtype: str = 'float32') -> Tensor: +def mel_frequencies( + n_mels: int = 64, + f_min: float = 0.0, + f_max: float = 11025.0, + htk: bool = False, + dtype: str = 'float32', +) -> Tensor: """Compute mel frequencies. Args: @@ -177,14 +184,16 @@ def fft_frequencies(sr: int, n_fft: int, dtype: str = 'float32') -> Tensor: return paddle.linspace(0, float(sr) / 2, int(1 + n_fft // 2), dtype=dtype) -def compute_fbank_matrix(sr: int, - n_fft: int, - n_mels: int = 64, - f_min: float = 0.0, - f_max: Optional[float] = None, - htk: bool = False, - norm: Union[str, float] = 'slaney', - dtype: str = 'float32') -> Tensor: +def compute_fbank_matrix( + sr: int, + n_fft: int, + n_mels: int = 64, + f_min: float = 0.0, + f_max: Optional[float] = None, + htk: bool = False, + norm: Union[str, float] = 'slaney', + dtype: str = 'float32', +) -> Tensor: """Compute fbank matrix. Args: @@ -220,15 +229,13 @@ def compute_fbank_matrix(sr: int, fftfreqs = fft_frequencies(sr=sr, n_fft=n_fft, dtype=dtype) # 'Center freqs' of mel bands - uniformly spaced between limits - mel_f = mel_frequencies(n_mels + 2, - f_min=f_min, - f_max=f_max, - htk=htk, - dtype=dtype) + mel_f = mel_frequencies( + n_mels + 2, f_min=f_min, f_max=f_max, htk=htk, dtype=dtype + ) - fdiff = mel_f[1:] - mel_f[:-1] #np.diff(mel_f) + fdiff = mel_f[1:] - mel_f[:-1] # np.diff(mel_f) ramps = mel_f.unsqueeze(1) - fftfreqs.unsqueeze(0) - #ramps = np.subtract.outer(mel_f, fftfreqs) + # ramps = np.subtract.outer(mel_f, fftfreqs) for i in range(n_mels): # lower and upper slopes for all bins @@ -236,12 +243,13 @@ def compute_fbank_matrix(sr: int, upper = ramps[i + 2] / fdiff[i + 1] # .. then intersect them with each other and zero - weights[i] = paddle.maximum(paddle.zeros_like(lower), - paddle.minimum(lower, upper)) + weights[i] = paddle.maximum( + paddle.zeros_like(lower), paddle.minimum(lower, upper) + ) # Slaney-style mel is scaled to be approx constant energy per channel if norm == 'slaney': - enorm = 2.0 / (mel_f[2:n_mels + 2] - mel_f[:n_mels]) + enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels]) weights *= enorm.unsqueeze(1) elif isinstance(norm, int) or isinstance(norm, float): weights = paddle.nn.functional.normalize(weights, p=norm, axis=-1) @@ -249,10 +257,12 @@ def compute_fbank_matrix(sr: int, return weights -def power_to_db(spect: Tensor, - ref_value: float = 1.0, - amin: float = 1e-10, - top_db: Optional[float] = 80.0) -> Tensor: +def power_to_db( + spect: Tensor, + ref_value: float = 1.0, + amin: float = 1e-10, + top_db: Optional[float] = 80.0, +) -> Tensor: """Convert a power spectrogram (amplitude squared) to decibel (dB) units. The function computes the scaling `10 * log10(x / ref)` in a numerically stable way. Args: @@ -291,10 +301,12 @@ def power_to_db(spect: Tensor, return log_spec -def create_dct(n_mfcc: int, - n_mels: int, - norm: Optional[str] = 'ortho', - dtype: str = 'float32') -> Tensor: +def create_dct( + n_mfcc: int, + n_mels: int, + norm: Optional[str] = 'ortho', + dtype: str = 'float32', +) -> Tensor: """Create a discrete cosine transform(DCT) matrix. Args: @@ -316,8 +328,9 @@ def create_dct(n_mfcc: int, """ n = paddle.arange(n_mels, dtype=dtype) k = paddle.arange(n_mfcc, dtype=dtype).unsqueeze(1) - dct = paddle.cos(math.pi / float(n_mels) * (n + 0.5) * - k) # size (n_mfcc, n_mels) + dct = paddle.cos( + math.pi / float(n_mels) * (n + 0.5) * k + ) # size (n_mfcc, n_mels) if norm is None: dct *= 2.0 else: diff --git a/python/paddle/audio/functional/window.py b/python/paddle/audio/functional/window.py index 17ccdce9ef6..844e2fc2633 100644 --- a/python/paddle/audio/functional/window.py +++ b/python/paddle/audio/functional/window.py @@ -31,7 +31,7 @@ def _acosh(x: Union[Tensor, float]) -> Tensor: def _extend(M: int, sym: bool) -> bool: - """Extend window by 1 sample if needed for DFT-even symmetry. """ + """Extend window by 1 sample if needed for DFT-even symmetry.""" if not sym: return M + 1, True else: @@ -39,7 +39,7 @@ def _extend(M: int, sym: bool) -> bool: def _len_guards(M: int) -> bool: - """Handle small or incorrect window lengths. """ + """Handle small or incorrect window lengths.""" if int(M) != M or M < 0: raise ValueError('Window length M must be a non-negative integer') @@ -47,101 +47,99 @@ def _len_guards(M: int) -> bool: def _truncate(w: Tensor, needed: bool) -> Tensor: - """Truncate window by 1 sample if needed for DFT-even symmetry. """ + """Truncate window by 1 sample if needed for DFT-even symmetry.""" if needed: return w[:-1] else: return w -def _general_gaussian(M: int, - p, - sig, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _general_gaussian( + M: int, p, sig, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a window with a generalized Gaussian shape. This function is consistent with scipy.signal.windows.general_gaussian(). """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) n = paddle.arange(0, M, dtype=dtype) - (M - 1.0) / 2.0 - w = paddle.exp(-0.5 * paddle.abs(n / sig)**(2 * p)) + w = paddle.exp(-0.5 * paddle.abs(n / sig) ** (2 * p)) return _truncate(w, needs_trunc) -def _general_cosine(M: int, - a: float, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _general_cosine( + M: int, a: float, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a generic weighted sum of cosine terms window. This function is consistent with scipy.signal.windows.general_cosine(). """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) fac = paddle.linspace(-math.pi, math.pi, M, dtype=dtype) - w = paddle.zeros((M, ), dtype=dtype) + w = paddle.zeros((M,), dtype=dtype) for k in range(len(a)): w += a[k] * paddle.cos(k * fac) return _truncate(w, needs_trunc) -def _general_hamming(M: int, - alpha: float, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _general_hamming( + M: int, alpha: float, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a generalized Hamming window. This function is consistent with scipy.signal.windows.general_hamming() """ - return _general_cosine(M, [alpha, 1. - alpha], sym, dtype=dtype) + return _general_cosine(M, [alpha, 1.0 - alpha], sym, dtype=dtype) -def _taylor(M: int, - nbar=4, - sll=30, - norm=True, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _taylor( + M: int, nbar=4, sll=30, norm=True, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a Taylor window. The Taylor window taper function approximates the Dolph-Chebyshev window's constant sidelobe level for a parameterized number of near-in sidelobes. """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) # Original text uses a negative sidelobe level parameter and then negates # it in the calculation of B. To keep consistent with other methods we # assume the sidelobe level parameter to be positive. - B = 10**(sll / 20) + B = 10 ** (sll / 20) A = _acosh(B) / math.pi - s2 = nbar**2 / (A**2 + (nbar - 0.5)**2) + s2 = nbar**2 / (A**2 + (nbar - 0.5) ** 2) ma = paddle.arange(1, nbar, dtype=dtype) - Fm = paddle.empty((nbar - 1, ), dtype=dtype) + Fm = paddle.empty((nbar - 1,), dtype=dtype) signs = paddle.empty_like(ma) signs[::2] = 1 signs[1::2] = -1 m2 = ma * ma for mi in range(len(ma)): - numer = signs[mi] * paddle.prod(1 - m2[mi] / s2 / (A**2 + - (ma - 0.5)**2)) + numer = signs[mi] * paddle.prod( + 1 - m2[mi] / s2 / (A**2 + (ma - 0.5) ** 2) + ) if mi == 0: - denom = 2 * paddle.prod(1 - m2[mi] / m2[mi + 1:]) + denom = 2 * paddle.prod(1 - m2[mi] / m2[mi + 1 :]) elif mi == len(ma) - 1: denom = 2 * paddle.prod(1 - m2[mi] / m2[:mi]) else: - denom = 2 * paddle.prod(1 - m2[mi] / m2[:mi]) * paddle.prod( - 1 - m2[mi] / m2[mi + 1:]) + denom = ( + 2 + * paddle.prod(1 - m2[mi] / m2[:mi]) + * paddle.prod(1 - m2[mi] / m2[mi + 1 :]) + ) Fm[mi] = numer / denom def W(n): return 1 + 2 * paddle.matmul( Fm.unsqueeze(0), - paddle.cos(2 * math.pi * ma.unsqueeze(1) * (n - M / 2. + 0.5) / M)) + paddle.cos(2 * math.pi * ma.unsqueeze(1) * (n - M / 2.0 + 0.5) / M), + ) w = W(paddle.arange(0, M, dtype=dtype)) @@ -169,18 +167,17 @@ def _hann(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: return _general_hamming(M, 0.5, sym, dtype=dtype) -def _tukey(M: int, - alpha=0.5, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _tukey( + M: int, alpha=0.5, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a Tukey window. The Tukey window is also known as a tapered cosine window. """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) if alpha <= 0: - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) elif alpha >= 1.0: return hann(M, sym=sym) @@ -188,57 +185,55 @@ def _tukey(M: int, n = paddle.arange(0, M, dtype=dtype) width = int(alpha * (M - 1) / 2.0) - n1 = n[0:width + 1] - n2 = n[width + 1:M - width - 1] - n3 = n[M - width - 1:] + n1 = n[0 : width + 1] + n2 = n[width + 1 : M - width - 1] + n3 = n[M - width - 1 :] w1 = 0.5 * (1 + paddle.cos(math.pi * (-1 + 2.0 * n1 / alpha / (M - 1)))) w2 = paddle.ones(n2.shape, dtype=dtype) - w3 = 0.5 * (1 + paddle.cos(math.pi * (-2.0 / alpha + 1 + 2.0 * n3 / alpha / - (M - 1)))) + w3 = 0.5 * ( + 1 + + paddle.cos(math.pi * (-2.0 / alpha + 1 + 2.0 * n3 / alpha / (M - 1))) + ) w = paddle.concat([w1, w2, w3]) return _truncate(w, needs_trunc) -def _kaiser(M: int, - beta: float, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _kaiser( + M: int, beta: float, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a Kaiser window. The Kaiser window is a taper formed by using a Bessel function. """ raise NotImplementedError() -def _gaussian(M: int, - std: float, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _gaussian( + M: int, std: float, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a Gaussian window. The Gaussian widows has a Gaussian shape defined by the standard deviation(std). """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) n = paddle.arange(0, M, dtype=dtype) - (M - 1.0) / 2.0 sig2 = 2 * std * std - w = paddle.exp(-n**2 / sig2) + w = paddle.exp(-(n**2) / sig2) return _truncate(w, needs_trunc) -def _exponential(M: int, - center=None, - tau=1., - sym: bool = True, - dtype: str = 'float64') -> Tensor: - """Compute an exponential (or Poisson) window. """ +def _exponential( + M: int, center=None, tau=1.0, sym: bool = True, dtype: str = 'float64' +) -> Tensor: + """Compute an exponential (or Poisson) window.""" if sym and center is not None: raise ValueError("If sym==True, center must be None.") if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) if center is None: @@ -251,10 +246,9 @@ def _exponential(M: int, def _triang(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: - """Compute a triangular window. - """ + """Compute a triangular window.""" if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) n = paddle.arange(1, (M + 1) // 2 + 1, dtype=dtype) @@ -273,12 +267,13 @@ def _bohman(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: The Bohman window is the autocorrelation of a cosine window. """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) fac = paddle.abs(paddle.linspace(-1, 1, M, dtype=dtype)[1:-1]) w = (1 - fac) * paddle.cos(math.pi * fac) + 1.0 / math.pi * paddle.sin( - math.pi * fac) + math.pi * fac + ) w = _cat([0, w, 0], dtype) return _truncate(w, needs_trunc) @@ -295,20 +290,21 @@ def _blackman(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: def _cosine(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: - """Compute a window with a simple cosine shape. - """ + """Compute a window with a simple cosine shape.""" if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) - w = paddle.sin(math.pi / M * (paddle.arange(0, M, dtype=dtype) + .5)) + w = paddle.sin(math.pi / M * (paddle.arange(0, M, dtype=dtype) + 0.5)) return _truncate(w, needs_trunc) -def get_window(window: Union[str, Tuple[str, float]], - win_length: int, - fftbins: bool = True, - dtype: str = 'float64') -> Tensor: +def get_window( + window: Union[str, Tuple[str, float]], + win_length: int, + fftbins: bool = True, + dtype: str = 'float64', +) -> Tensor: """Return a window of a given length and type. Args: @@ -340,19 +336,22 @@ def get_window(window: Union[str, Tuple[str, float]], args = window[1:] elif isinstance(window, str): if window in ['gaussian', 'exponential']: - raise ValueError("The '" + window + "' window needs one or " - "more parameters -- pass a tuple.") + raise ValueError( + "The '" + window + "' window needs one or " + "more parameters -- pass a tuple." + ) else: winstr = window else: - raise ValueError("%s as window type is not supported." % - str(type(window))) + raise ValueError( + "%s as window type is not supported." % str(type(window)) + ) try: winfunc = eval('_' + winstr) except NameError as e: raise ValueError("Unknown window type.") from e - params = (win_length, ) + args + params = (win_length,) + args kwargs = {'sym': sym} return winfunc(*params, dtype=dtype, **kwargs) diff --git a/python/paddle/audio/utils/error.py b/python/paddle/audio/utils/error.py index ab239a24970..244340b99b5 100644 --- a/python/paddle/audio/utils/error.py +++ b/python/paddle/audio/utils/error.py @@ -17,4 +17,5 @@ __all__ = ['ParameterError'] class ParameterError(Exception): """Exception class for Parameter checking""" + pass diff --git a/python/paddle/autograd/__init__.py b/python/paddle/autograd/__init__.py index 70fc9647cd4..19742cf6afe 100644 --- a/python/paddle/autograd/__init__.py +++ b/python/paddle/autograd/__init__.py @@ -18,6 +18,7 @@ from ..framework import is_grad_enabled, set_grad_enabled # noqa: F401 from . import backward_mode # noqa: F401 from .backward_mode import backward # noqa: F401 from ..fluid.framework import _in_eager_mode_ + if _in_eager_mode_: from .py_layer import EagerPyLayer as PyLayer # noqa: F401 from .py_layer import EagerPyLayerContext as PyLayerContext # noqa: F401 diff --git a/python/paddle/autograd/backward_mode.py b/python/paddle/autograd/backward_mode.py index 83c84a01afb..9f673e62e21 100644 --- a/python/paddle/autograd/backward_mode.py +++ b/python/paddle/autograd/backward_mode.py @@ -83,15 +83,13 @@ def backward(tensors, grad_tensors=None, retain_graph=False): assert len(in_out_list) > 0, "{} connot be empyt".format(name) for each_var in in_out_list: assert isinstance( - each_var, - (paddle.Tensor, core.eager.Tensor - )), "Elements of {} must be paddle.Tensor".format(name) + each_var, (paddle.Tensor, core.eager.Tensor) + ), "Elements of {} must be paddle.Tensor".format(name) return in_out_list else: assert isinstance( - in_out_list, - (paddle.Tensor, core.eager.Tensor - )), "{} must be Tensor or list of Tensor".format(name) + in_out_list, (paddle.Tensor, core.eager.Tensor) + ), "{} must be Tensor or list of Tensor".format(name) return [in_out_list] tensors = check_tensors(tensors, "tensors") @@ -117,12 +115,14 @@ def backward(tensors, grad_tensors=None, retain_graph=False): if len(grad_tensors) > 0: assert len(tensors) == len( - grad_tensors), "The length of grad_tensors must be equal to tensors" + grad_tensors + ), "The length of grad_tensors must be equal to tensors" assert isinstance(retain_graph, bool), "retain_graph must be True or False" if framework._in_eager_mode_: core.eager.run_backward(tensors, grad_tensors, retain_graph) else: - core.dygraph_run_backward(tensors, grad_tensors, retain_graph, - framework._dygraph_tracer()) + core.dygraph_run_backward( + tensors, grad_tensors, retain_graph, framework._dygraph_tracer() + ) diff --git a/python/paddle/autograd/py_layer.py b/python/paddle/autograd/py_layer.py index ee7c2e36f73..254aabb04b3 100644 --- a/python/paddle/autograd/py_layer.py +++ b/python/paddle/autograd/py_layer.py @@ -124,9 +124,7 @@ class LegacyPyLayerContext(object): def with_mateclass(meta, *bases): - class impl(meta): - def __new__(cls, name, temp_bases, attrs): return meta(name, bases, attrs) @@ -134,7 +132,6 @@ def with_mateclass(meta, *bases): class CPyLayer(object): - @classmethod @dygraph_only def apply(cls, *args, **kwargs): @@ -182,12 +179,14 @@ class CPyLayer(object): class PyLayerBackward(LegacyPyLayerContext): - def backward(self, *args, **kwargs): with paddle.fluid.dygraph.guard(): with paddle.fluid.dygraph.no_grad(): - if self._amp_state and 'enable' in self._amp_state and self._amp_state[ - 'enable']: + if ( + self._amp_state + and 'enable' in self._amp_state + and self._amp_state['enable'] + ): with auto_cast(**args[0]._amp_state): return self._forward_cls.backward(*args, **kwargs) else: @@ -197,10 +196,10 @@ class PyLayerBackward(LegacyPyLayerContext): class LayerMeta(type): - def __init__(cls, name, bases, attrs): - cls._backward_function = type(name + '_backward', (PyLayerBackward, ), - {"_forward_cls": cls}) + cls._backward_function = type( + name + '_backward', (PyLayerBackward,), {"_forward_cls": cls} + ) return super(LayerMeta, cls).__init__(name, bases, attrs) @@ -292,7 +291,8 @@ class LegacyPyLayer(with_mateclass(LayerMeta, CPyLayer)): return grad """ raise NotImplementedError( - "You must implement the forward function for PyLayer.") + "You must implement the forward function for PyLayer." + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -332,11 +332,11 @@ class LegacyPyLayer(with_mateclass(LayerMeta, CPyLayer)): """ raise NotImplementedError( - "You must implement the backward function for PyLayer.") + "You must implement the backward function for PyLayer." + ) class EagerPyLayerContext(object): - def save_for_backward(self, *tensors): """ Saves given tensors that backward need. Use ``saved_tensor`` in the `backward` to get the saved tensors. @@ -542,25 +542,22 @@ class EagerPyLayerContext(object): class EagerPyLayerBackward(core.eager.PyLayer, EagerPyLayerContext): - def backward(self, *args): return self._forward_cls.backward(self, *args) class EagerPyLayerMeta(type): - def __init__(cls, name, bases, attrs): - cls._backward_function = type(name + '_backward', - (EagerPyLayerBackward, ), - {"_forward_cls": cls}) + cls._backward_function = type( + name + '_backward', (EagerPyLayerBackward,), {"_forward_cls": cls} + ) return super(EagerPyLayerMeta, cls).__init__(name, bases, attrs) class EagerPyLayer( - with_mateclass(EagerPyLayerMeta, core.eager.PyLayer, - EagerPyLayerContext)): - + with_mateclass(EagerPyLayerMeta, core.eager.PyLayer, EagerPyLayerContext) +): @staticmethod def forward(ctx, *args, **kwargs): """ @@ -597,7 +594,8 @@ class EagerPyLayer( return grad """ raise NotImplementedError( - "You must implement the forward function for PyLayer.") + "You must implement the forward function for PyLayer." + ) @staticmethod def backward(ctx, *args): @@ -637,11 +635,11 @@ class EagerPyLayer( """ raise NotImplementedError( - "You must implement the backward function for PyLayer.") + "You must implement the backward function for PyLayer." + ) def once_differentiable(backward): - def wrapper(ctx, *args): with paddle.fluid.dygraph.no_grad(): outputs = backward(ctx, *args) diff --git a/python/paddle/autograd/saved_tensors_hooks.py b/python/paddle/autograd/saved_tensors_hooks.py index ff5fecf5ccd..8906dd98b49 100644 --- a/python/paddle/autograd/saved_tensors_hooks.py +++ b/python/paddle/autograd/saved_tensors_hooks.py @@ -17,7 +17,7 @@ from paddle.fluid import core __all__ = [] -class saved_tensors_hooks(): +class saved_tensors_hooks: """ Dynamic graph, registers a pair of pack / unpack hooks for saved tensors. @@ -104,8 +104,9 @@ class saved_tensors_hooks(): self.unpack_hook = unpack_hook def __enter__(self): - core.eager.register_saved_tensors_hooks(self.pack_hook, - self.unpack_hook) + core.eager.register_saved_tensors_hooks( + self.pack_hook, self.unpack_hook + ) def __exit__(self, *args): core.eager.reset_saved_tensors_hooks() diff --git a/python/paddle/batch.py b/python/paddle/batch.py index b87ddc74858..13ba5a00c0c 100644 --- a/python/paddle/batch.py +++ b/python/paddle/batch.py @@ -66,7 +66,9 @@ def batch(reader, batch_size, drop_last=False): # Batch size check batch_size = int(batch_size) if batch_size <= 0: - raise ValueError("batch_size should be a positive integeral value, " - "but got batch_size={}".format(batch_size)) + raise ValueError( + "batch_size should be a positive integeral value, " + "but got batch_size={}".format(batch_size) + ) return batch_reader diff --git a/python/paddle/callbacks.py b/python/paddle/callbacks.py index 46f69aae1bb..b2da04c83df 100644 --- a/python/paddle/callbacks.py +++ b/python/paddle/callbacks.py @@ -20,7 +20,12 @@ from .hapi.callbacks import LRScheduler # noqa: F401 from .hapi.callbacks import EarlyStopping # noqa: F401 from .hapi.callbacks import ReduceLROnPlateau # noqa: F401 -__all__ = [ #noqa - 'Callback', 'ProgBarLogger', 'ModelCheckpoint', 'VisualDL', 'LRScheduler', - 'EarlyStopping', 'ReduceLROnPlateau' +__all__ = [ # noqa + 'Callback', + 'ProgBarLogger', + 'ModelCheckpoint', + 'VisualDL', + 'LRScheduler', + 'EarlyStopping', + 'ReduceLROnPlateau', ] diff --git a/python/paddle/check_import_scipy.py b/python/paddle/check_import_scipy.py index 9bbd061cb37..473a4301813 100644 --- a/python/paddle/check_import_scipy.py +++ b/python/paddle/check_import_scipy.py @@ -20,10 +20,10 @@ def check_import_scipy(OsName): import scipy.io as scio # noqa: F401 except ImportError as e: print_info = str(e) - if (len(print_info) > 0): + if len(print_info) > 0: if 'DLL load failed' in print_info: raise ImportError( - print_info + - "\nplease download Visual C++ Redistributable from https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0" + print_info + + "\nplease download Visual C++ Redistributable from https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0" ) return diff --git a/python/paddle/common_ops_import.py b/python/paddle/common_ops_import.py index 00ef853b995..47bce838936 100644 --- a/python/paddle/common_ops_import.py +++ b/python/paddle/common_ops_import.py @@ -13,13 +13,29 @@ # limitations under the License. from paddle.fluid.layer_helper import LayerHelper # noqa: F401 from paddle.fluid.param_attr import ParamAttr # noqa: F401 -from paddle.fluid.framework import convert_np_dtype_to_dtype_, _non_static_mode, _varbase_creator, in_dygraph_mode, _in_legacy_dygraph # noqa: F401 -from paddle.fluid.framework import device_guard, default_main_program, dygraph_only, _dygraph_tracer # noqa: F401 +from paddle.fluid.framework import ( # noqa: F401 + convert_np_dtype_to_dtype_, + _non_static_mode, + _varbase_creator, + in_dygraph_mode, + _in_legacy_dygraph, +) +from paddle.fluid.framework import ( # noqa: F401 + device_guard, + default_main_program, + dygraph_only, + _dygraph_tracer, +) from paddle.fluid.framework import OpProtoHolder, Variable # noqa: F401 from paddle.fluid.initializer import Constant # noqa: F401 from paddle.fluid.core import VarDesc # noqa: F401 from paddle.fluid import core, dygraph_utils # noqa: F401 -from paddle.fluid.data_feeder import check_type, check_dtype, check_variable_and_dtype, convert_dtype # noqa: F401 +from paddle.fluid.data_feeder import ( # noqa: F401 + check_type, + check_dtype, + check_variable_and_dtype, + convert_dtype, +) from paddle.fluid.layers import fill_constant, utils, scale # noqa: F401 from paddle.tensor.layer_function_generator import templatedoc # noqa: F401 import paddle.fluid as fluid # noqa: F401 diff --git a/python/paddle/cost_model/cost_model.py b/python/paddle/cost_model/cost_model.py index a59ff31a683..8797868287b 100644 --- a/python/paddle/cost_model/cost_model.py +++ b/python/paddle/cost_model/cost_model.py @@ -20,8 +20,7 @@ import os from paddle.fluid import core -class CostModel(): - +class CostModel: def __init__(self): pass @@ -30,11 +29,12 @@ class CostModel(): main_program = static.Program() startup_program = static.Program() - with static.program_guard(main_program=main_program, - startup_program=startup_program): - data = paddle.static.data(name='X', - shape=[None, 1], - dtype='float32') + with static.program_guard( + main_program=main_program, startup_program=startup_program + ): + data = paddle.static.data( + name='X', shape=[None, 1], dtype='float32' + ) hidden = paddle.static.nn.fc(data, 10) loss = paddle.mean(hidden) paddle.optimizer.SGD(learning_rate=0.01).minimize(loss) @@ -43,11 +43,13 @@ class CostModel(): return startup_program, main_program - def profile_measure(self, - startup_program, - main_program, - device='gpu', - fetch_cost_list=['time']): + def profile_measure( + self, + startup_program, + main_program, + device='gpu', + fetch_cost_list=['time'], + ): place = paddle.set_device('gpu') x = np.random.random(size=(10, 1)).astype('float32') @@ -61,8 +63,9 @@ class CostModel(): cost_data = cost_model.ProfileMeasure(device) def static_cost_data(self): - static_cost_data_path = os.path.join(os.path.dirname(__file__), - "static_op_benchmark.json") + static_cost_data_path = os.path.join( + os.path.dirname(__file__), "static_op_benchmark.json" + ) with open(static_cost_data_path, 'r') as load_f: load_dict = json.load(load_f) self._static_cost_data = load_dict @@ -79,7 +82,7 @@ class CostModel(): op_cost = {} for op_data in self._static_cost_data: if (op_data["op"] == op_name) and (dtype in op_data["config"]): - if (forward): + if forward: op_cost["op_time"] = op_data["paddle_gpu_time"] else: op_cost["op_time"] = op_data["paddle_gpu_time_backward"] diff --git a/python/paddle/dataset/cifar.py b/python/paddle/dataset/cifar.py index 0a05ecde473..80fd43eb52f 100644 --- a/python/paddle/dataset/cifar.py +++ b/python/paddle/dataset/cifar.py @@ -43,7 +43,6 @@ CIFAR100_MD5 = 'eb9058c3a382ffc7106e4002c42a8d85' def reader_creator(filename, sub_name, cycle=False): - def read_batch(batch): data = batch[b'data'] labels = batch.get(b'labels', batch.get(b'fine_labels', None)) @@ -54,8 +53,11 @@ def reader_creator(filename, sub_name, cycle=False): def reader(): while True: with tarfile.open(filename, mode='r') as f: - names = (each_item.name for each_item in f - if sub_name in each_item.name) + names = ( + each_item.name + for each_item in f + if sub_name in each_item.name + ) for name in names: batch = pickle.load(f.extractfile(name), encoding='bytes') @@ -72,7 +74,8 @@ def reader_creator(filename, sub_name, cycle=False): since="2.0.0", update_to="paddle.vision.datasets.Cifar100", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train100(): """ CIFAR-100 training set creator. @@ -85,14 +88,16 @@ def train100(): """ return reader_creator( paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5), - 'train') + 'train', + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar100", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test100(): """ CIFAR-100 test set creator. @@ -105,14 +110,16 @@ def test100(): """ return reader_creator( paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5), - 'test') + 'test', + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train10(cycle=False): """ CIFAR-10 training set creator. @@ -125,17 +132,19 @@ def train10(cycle=False): :return: Training reader creator :rtype: callable """ - return reader_creator(paddle.dataset.common.download( - CIFAR10_URL, 'cifar', CIFAR10_MD5), - 'data_batch', - cycle=cycle) + return reader_creator( + paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + 'data_batch', + cycle=cycle, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test10(cycle=False): """ CIFAR-10 test set creator. @@ -148,17 +157,19 @@ def test10(cycle=False): :return: Test reader creator. :rtype: callable """ - return reader_creator(paddle.dataset.common.download( - CIFAR10_URL, 'cifar', CIFAR10_MD5), - 'test_batch', - cycle=cycle) + return reader_creator( + paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + 'test_batch', + cycle=cycle, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5) paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5) diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py index 97e986c7006..157cdeb91fe 100644 --- a/python/paddle/dataset/common.py +++ b/python/paddle/dataset/common.py @@ -74,8 +74,8 @@ def download(url, module_name, md5sum, save_name=None): os.makedirs(dirname) filename = os.path.join( - dirname, - url.split('/')[-1] if save_name is None else save_name) + dirname, url.split('/')[-1] if save_name is None else save_name + ) if os.path.exists(filename) and md5file(filename) == md5sum: return filename @@ -90,9 +90,12 @@ def download(url, module_name, md5sum, save_name=None): else: raise RuntimeError( "Cannot download {0} within retry limit {1}".format( - url, retry_limit)) - sys.stderr.write("Cache file %s not found, downloading %s \n" % - (filename, url)) + url, retry_limit + ) + ) + sys.stderr.write( + "Cache file %s not found, downloading %s \n" % (filename, url) + ) sys.stderr.write("Begin to download\n") try: r = requests.get(url, stream=True) @@ -108,8 +111,9 @@ def download(url, module_name, md5sum, save_name=None): total_iter = total_length / chunk_size + 1 log_interval = total_iter // 20 if total_iter > 20 else 1 log_index = 0 - bar = paddle.hapi.progressbar.ProgressBar(total_iter, - name='item') + bar = paddle.hapi.progressbar.ProgressBar( + total_iter, name='item' + ) for data in r.iter_content(chunk_size=chunk_size): f.write(data) log_index += 1 @@ -127,12 +131,15 @@ def download(url, module_name, md5sum, save_name=None): def fetch_all(): for module_name in [ - x for x in dir(paddle.dataset) if not x.startswith("__") + x for x in dir(paddle.dataset) if not x.startswith("__") ]: if "fetch" in dir( - importlib.import_module("paddle.dataset.%s" % module_name)): - getattr(importlib.import_module("paddle.dataset.%s" % module_name), - "fetch")() + importlib.import_module("paddle.dataset.%s" % module_name) + ): + getattr( + importlib.import_module("paddle.dataset.%s" % module_name), + "fetch", + )() def split(reader, line_count, suffix="%05d.pickle", dumper=pickle.dump): @@ -173,10 +180,9 @@ def split(reader, line_count, suffix="%05d.pickle", dumper=pickle.dump): dumper(lines, f) -def cluster_files_reader(files_pattern, - trainer_count, - trainer_id, - loader=pickle.load): +def cluster_files_reader( + files_pattern, trainer_count, trainer_id, loader=pickle.load +): """ Create a reader that yield element from the given files, select a file set according trainer count and trainer_id @@ -216,4 +222,5 @@ def _check_exists_and_download(path, url, md5, module_name, download=True): return paddle.dataset.common.download(url, module_name, md5) else: raise ValueError( - '{} not exists and auto download disabled'.format(path)) + '{} not exists and auto download disabled'.format(path) + ) diff --git a/python/paddle/dataset/conll05.py b/python/paddle/dataset/conll05.py index 9333d7350e8..22038594f60 100644 --- a/python/paddle/dataset/conll05.py +++ b/python/paddle/dataset/conll05.py @@ -83,7 +83,8 @@ def corpus_reader(data_path, words_name, props_name): wf = tf.extractfile(words_name) pf = tf.extractfile(props_name) with gzip.GzipFile(fileobj=wf) as words_file, gzip.GzipFile( - fileobj=pf) as props_file: + fileobj=pf + ) as props_file: sentences = [] labels = [] one_seg = [] @@ -116,16 +117,17 @@ def corpus_reader(data_path, words_name, props_name): lbl_seq.append('I-' + cur_tag) is_in_bracket = False elif l.find('(') != -1 and l.find(')') != -1: - cur_tag = l[1:l.find('*')] + cur_tag = l[1 : l.find('*')] lbl_seq.append('B-' + cur_tag) is_in_bracket = False elif l.find('(') != -1 and l.find(')') == -1: - cur_tag = l[1:l.find('*')] + cur_tag = l[1 : l.find('*')] lbl_seq.append('B-' + cur_tag) is_in_bracket = True else: - raise RuntimeError('Unexpected label: %s' % - l) + raise RuntimeError( + 'Unexpected label: %s' % l + ) yield sentences, verb_list[i], lbl_seq @@ -143,11 +145,9 @@ def corpus_reader(data_path, words_name, props_name): return reader -def reader_creator(corpus_reader, - word_dict=None, - predicate_dict=None, - label_dict=None): - +def reader_creator( + corpus_reader, word_dict=None, predicate_dict=None, label_dict=None +): def reader(): for sentence, predicate, labels in corpus_reader(): @@ -193,8 +193,7 @@ def reader_creator(corpus_reader, pred_idx = [predicate_dict.get(predicate)] * sen_len label_idx = [label_dict.get(w) for w in labels] - yield word_idx, ctx_n2_idx, ctx_n1_idx, \ - ctx_0_idx, ctx_p1_idx, ctx_p2_idx, pred_idx, mark, label_idx + yield word_idx, ctx_n2_idx, ctx_n1_idx, ctx_0_idx, ctx_p1_idx, ctx_p2_idx, pred_idx, mark, label_idx return reader @@ -203,17 +202,21 @@ def reader_creator(corpus_reader, since="2.0.0", update_to="paddle.text.datasets.Conll05st", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_dict(): """ Get the word, verb and label dictionary of Wikipedia corpus. """ word_dict = load_dict( - paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5)) + paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5) + ) verb_dict = load_dict( - paddle.dataset.common.download(VERBDICT_URL, 'conll05st', VERBDICT_MD5)) + paddle.dataset.common.download(VERBDICT_URL, 'conll05st', VERBDICT_MD5) + ) label_dict = load_label_dict( - paddle.dataset.common.download(TRGDICT_URL, 'conll05st', TRGDICT_MD5)) + paddle.dataset.common.download(TRGDICT_URL, 'conll05st', TRGDICT_MD5) + ) return word_dict, verb_dict, label_dict @@ -221,7 +224,8 @@ def get_dict(): since="2.0.0", update_to="paddle.text.datasets.Conll05st", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_embedding(): """ Get the trained word vector based on Wikipedia corpus. @@ -233,7 +237,8 @@ def get_embedding(): since="2.0.0", update_to="paddle.text.datasets.Conll05st", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(): """ Conll05 test set creator. @@ -250,7 +255,8 @@ def test(): reader = corpus_reader( paddle.dataset.common.download(DATA_URL, 'conll05st', DATA_MD5), words_name='conll05st-release/test.wsj/words/test.wsj.words.gz', - props_name='conll05st-release/test.wsj/props/test.wsj.props.gz') + props_name='conll05st-release/test.wsj/props/test.wsj.props.gz', + ) return reader_creator(reader, word_dict, verb_dict, label_dict) @@ -258,7 +264,8 @@ def test(): since="2.0.0", update_to="paddle.text.datasets.Conll05st", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5) paddle.dataset.common.download(VERBDICT_URL, 'conll05st', VERBDICT_MD5) diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index 42c0e7081d5..41ffc488aa1 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -63,11 +63,9 @@ def default_mapper(is_train, sample): ''' img, label = sample img = load_image_bytes(img) - img = simple_transform(img, - 256, - 224, - is_train, - mean=[103.94, 116.78, 123.68]) + img = simple_transform( + img, 256, 224, is_train, mean=[103.94, 116.78, 123.68] + ) return img.flatten().astype('float32'), label @@ -75,14 +73,16 @@ train_mapper = functools.partial(default_mapper, True) test_mapper = functools.partial(default_mapper, False) -def reader_creator(data_file, - label_file, - setid_file, - dataset_name, - mapper, - buffered_size=1024, - use_xmap=True, - cycle=False): +def reader_creator( + data_file, + label_file, + setid_file, + dataset_name, + mapper, + buffered_size=1024, + use_xmap=True, + cycle=False, +): ''' 1. read images from tar file and merge images into batch files in 102flowers.tgz_batch/ @@ -138,7 +138,8 @@ def reader_creator(data_file, since="2.0.0", update_to="paddle.vision.datasets.Flowers", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(mapper=train_mapper, buffered_size=1024, use_xmap=True, cycle=False): ''' Create flowers training set reader. @@ -157,21 +158,24 @@ def train(mapper=train_mapper, buffered_size=1024, use_xmap=True, cycle=False): :return: train data reader :rtype: callable ''' - return reader_creator(download(DATA_URL, 'flowers', DATA_MD5), - download(LABEL_URL, 'flowers', LABEL_MD5), - download(SETID_URL, 'flowers', SETID_MD5), - TRAIN_FLAG, - mapper, - buffered_size, - use_xmap, - cycle=cycle) + return reader_creator( + download(DATA_URL, 'flowers', DATA_MD5), + download(LABEL_URL, 'flowers', LABEL_MD5), + download(SETID_URL, 'flowers', SETID_MD5), + TRAIN_FLAG, + mapper, + buffered_size, + use_xmap, + cycle=cycle, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Flowers", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(mapper=test_mapper, buffered_size=1024, use_xmap=True, cycle=False): ''' Create flowers test set reader. @@ -190,21 +194,24 @@ def test(mapper=test_mapper, buffered_size=1024, use_xmap=True, cycle=False): :return: test data reader :rtype: callable ''' - return reader_creator(download(DATA_URL, 'flowers', DATA_MD5), - download(LABEL_URL, 'flowers', LABEL_MD5), - download(SETID_URL, 'flowers', SETID_MD5), - TEST_FLAG, - mapper, - buffered_size, - use_xmap, - cycle=cycle) + return reader_creator( + download(DATA_URL, 'flowers', DATA_MD5), + download(LABEL_URL, 'flowers', LABEL_MD5), + download(SETID_URL, 'flowers', SETID_MD5), + TEST_FLAG, + mapper, + buffered_size, + use_xmap, + cycle=cycle, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Flowers", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def valid(mapper=test_mapper, buffered_size=1024, use_xmap=True): ''' Create flowers validation set reader. @@ -221,10 +228,15 @@ def valid(mapper=test_mapper, buffered_size=1024, use_xmap=True): :return: test data reader :rtype: callable ''' - return reader_creator(download(DATA_URL, 'flowers', DATA_MD5), - download(LABEL_URL, 'flowers', LABEL_MD5), - download(SETID_URL, 'flowers', SETID_MD5), VALID_FLAG, - mapper, buffered_size, use_xmap) + return reader_creator( + download(DATA_URL, 'flowers', DATA_MD5), + download(LABEL_URL, 'flowers', LABEL_MD5), + download(SETID_URL, 'flowers', SETID_MD5), + VALID_FLAG, + mapper, + buffered_size, + use_xmap, + ) def fetch(): diff --git a/python/paddle/dataset/image.py b/python/paddle/dataset/image.py index 98ad9299c1b..c8d5124f0fe 100644 --- a/python/paddle/dataset/image.py +++ b/python/paddle/dataset/image.py @@ -31,6 +31,7 @@ the image layout as follows. """ import numpy as np + # FIXME(minqiyang): this is an ugly fix for the numpy bug reported here # https://github.com/numpy/numpy/issues/12497 import subprocess @@ -42,9 +43,11 @@ interpreter = sys.executable # will be the C++ execubable on Windows if sys.platform == 'win32' and 'python.exe' not in interpreter: interpreter = sys.exec_prefix + os.sep + 'python.exe' -import_cv2_proc = subprocess.Popen([interpreter, "-c", "import cv2"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) +import_cv2_proc = subprocess.Popen( + [interpreter, "-c", "import cv2"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, +) out, err = import_cv2_proc.communicate() retcode = import_cv2_proc.poll() if retcode != 0: @@ -65,6 +68,7 @@ __all__ = [] def _check_cv2(): if cv2 is None: import sys + sys.stderr.write( '''Warning with paddle image module: opencv-python should be imported, or paddle image module could NOT work; please install opencv-python first.''' @@ -74,10 +78,9 @@ def _check_cv2(): return True -def batch_images_from_tar(data_file, - dataset_name, - img2label, - num_per_batch=1024): +def batch_images_from_tar( + data_file, dataset_name, img2label, num_per_batch=1024 +): """ Read images from tar file and batch them into batch file. @@ -115,9 +118,11 @@ def batch_images_from_tar(data_file, output = {} output['label'] = labels output['data'] = data - pickle.dump(output, - open('%s/batch_%d' % (out_path, file_id), 'wb'), - protocol=2) + pickle.dump( + output, + open('%s/batch_%d' % (out_path, file_id), 'wb'), + protocol=2, + ) file_id += 1 data = [] labels = [] @@ -125,9 +130,9 @@ def batch_images_from_tar(data_file, output = {} output['label'] = labels output['data'] = data - pickle.dump(output, - open('%s/batch_%d' % (out_path, file_id), 'wb'), - protocol=2) + pickle.dump( + output, open('%s/batch_%d' % (out_path, file_id), 'wb'), protocol=2 + ) with open(meta_file, 'a') as meta: for file in os.listdir(out_path): @@ -321,12 +326,9 @@ def left_right_flip(im, is_color=True): return im[:, ::-1] -def simple_transform(im, - resize_size, - crop_size, - is_train, - is_color=True, - mean=None): +def simple_transform( + im, resize_size, crop_size, is_train, is_color=True, mean=None +): """ Simply data argumentation for training. These operations include resizing, croping and flipping. @@ -377,12 +379,9 @@ def simple_transform(im, return im -def load_and_transform(filename, - resize_size, - crop_size, - is_train, - is_color=True, - mean=None): +def load_and_transform( + filename, resize_size, crop_size, is_train, is_color=True, mean=None +): """ Load image from the input file `filename` and transform image for data argumentation. Please refer to the `simple_transform` interface diff --git a/python/paddle/dataset/imdb.py b/python/paddle/dataset/imdb.py index bed7f03090b..e95a9e6df00 100644 --- a/python/paddle/dataset/imdb.py +++ b/python/paddle/dataset/imdb.py @@ -29,7 +29,7 @@ import string __all__ = [] -#URL = 'http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz' +# URL = 'http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz' URL = 'https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz' MD5 = '7c2ac02c03563afcf9b574c7e56c153a' @@ -49,7 +49,8 @@ def tokenize(pattern): if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. yield tarf.extractfile(tf).read().rstrip(b'\n\r').translate( - None, string.punctuation.encode('latin-1')).lower().split() + None, string.punctuation.encode('latin-1') + ).lower().split() tf = tarf.next() @@ -77,7 +78,8 @@ def build_dict(pattern, cutoff): since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def reader_creator(pos_pattern, neg_pattern, word_idx): UNK = word_idx[''] INS = [] @@ -100,7 +102,8 @@ def reader_creator(pos_pattern, neg_pattern, word_idx): since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(word_idx): """ IMDB training set creator. @@ -113,15 +116,19 @@ def train(word_idx): :return: Training reader creator :rtype: callable """ - return reader_creator(re.compile(r"aclImdb/train/pos/.*\.txt$"), - re.compile(r"aclImdb/train/neg/.*\.txt$"), word_idx) + return reader_creator( + re.compile(r"aclImdb/train/pos/.*\.txt$"), + re.compile(r"aclImdb/train/neg/.*\.txt$"), + word_idx, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(word_idx): """ IMDB test set creator. @@ -134,15 +141,19 @@ def test(word_idx): :return: Test reader creator :rtype: callable """ - return reader_creator(re.compile(r"aclImdb/test/pos/.*\.txt$"), - re.compile(r"aclImdb/test/neg/.*\.txt$"), word_idx) + return reader_creator( + re.compile(r"aclImdb/test/pos/.*\.txt$"), + re.compile(r"aclImdb/test/neg/.*\.txt$"), + word_idx, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def word_dict(): """ Build a word dictionary from the corpus. @@ -151,13 +162,15 @@ def word_dict(): :rtype: dict """ return build_dict( - re.compile(r"aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$"), 150) + re.compile(r"aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$"), 150 + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL, 'imdb', MD5) diff --git a/python/paddle/dataset/imikolov.py b/python/paddle/dataset/imikolov.py index b94513ec13a..4630d88e21a 100644 --- a/python/paddle/dataset/imikolov.py +++ b/python/paddle/dataset/imikolov.py @@ -26,7 +26,7 @@ import tarfile __all__ = [] -#URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' +# URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' URL = 'https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz' MD5 = '30177ea32e27c525793142b6bf2c8e2d' @@ -57,9 +57,10 @@ def build_dict(min_word_freq=50): train_filename = './simple-examples/data/ptb.train.txt' test_filename = './simple-examples/data/ptb.valid.txt' with tarfile.open( - paddle.dataset.common.download(paddle.dataset.imikolov.URL, - 'imikolov', - paddle.dataset.imikolov.MD5)) as tf: + paddle.dataset.common.download( + paddle.dataset.imikolov.URL, 'imikolov', paddle.dataset.imikolov.MD5 + ) + ) as tf: trainf = tf.extractfile(train_filename) testf = tf.extractfile(test_filename) word_freq = word_count(testf, word_count(trainf)) @@ -78,12 +79,14 @@ def build_dict(min_word_freq=50): def reader_creator(filename, word_idx, n, data_type): - def reader(): with tarfile.open( - paddle.dataset.common.download( - paddle.dataset.imikolov.URL, 'imikolov', - paddle.dataset.imikolov.MD5)) as tf: + paddle.dataset.common.download( + paddle.dataset.imikolov.URL, + 'imikolov', + paddle.dataset.imikolov.MD5, + ) + ) as tf: f = tf.extractfile(filename) UNK = word_idx[''] @@ -94,13 +97,14 @@ def reader_creator(filename, word_idx, n, data_type): if len(l) >= n: l = [word_idx.get(w, UNK) for w in l] for i in range(n, len(l) + 1): - yield tuple(l[i - n:i]) + yield tuple(l[i - n : i]) elif DataType.SEQ == data_type: l = l.strip().split() l = [word_idx.get(w, UNK) for w in l] src_seq = [word_idx['']] + l trg_seq = l + [word_idx['']] - if n > 0 and len(src_seq) > n: continue + if n > 0 and len(src_seq) > n: + continue yield src_seq, trg_seq else: assert False, 'Unknow data type' @@ -112,7 +116,8 @@ def reader_creator(filename, word_idx, n, data_type): since="2.0.0", update_to="paddle.text.datasets.Imikolov", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(word_idx, n, data_type=DataType.NGRAM): """ imikolov training set creator. @@ -129,15 +134,17 @@ def train(word_idx, n, data_type=DataType.NGRAM): :return: Training reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n, - data_type) + return reader_creator( + './simple-examples/data/ptb.train.txt', word_idx, n, data_type + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imikolov", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(word_idx, n, data_type=DataType.NGRAM): """ imikolov test set creator. @@ -154,14 +161,16 @@ def test(word_idx, n, data_type=DataType.NGRAM): :return: Test reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n, - data_type) + return reader_creator( + './simple-examples/data/ptb.valid.txt', word_idx, n, data_type + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imikolov", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL, "imikolov", MD5) diff --git a/python/paddle/dataset/mnist.py b/python/paddle/dataset/mnist.py index 12f03dd837d..ec1cb9855cf 100644 --- a/python/paddle/dataset/mnist.py +++ b/python/paddle/dataset/mnist.py @@ -38,7 +38,6 @@ TRAIN_LABEL_MD5 = 'd53e105ee54ea40749a09fcbcd1e9432' def reader_creator(image_filename, label_filename, buffer_size): - def reader(): with gzip.GzipFile(image_filename, 'rb') as image_file: img_buf = image_file.read() @@ -53,14 +52,16 @@ def reader_creator(image_filename, label_filename, buffer_size): # image file : 16B magic_byte_img = '>IIII' magic_img, image_num, rows, cols = struct.unpack_from( - magic_byte_img, img_buf, offset_img) + magic_byte_img, img_buf, offset_img + ) offset_img += struct.calcsize(magic_byte_img) offset_lab = 0 # label file : 8B magic_byte_lab = '>II' magic_lab, label_num = struct.unpack_from( - magic_byte_lab, lab_buf, offset_lab) + magic_byte_lab, lab_buf, offset_lab + ) offset_lab += struct.calcsize(magic_byte_lab) while True: @@ -72,11 +73,12 @@ def reader_creator(image_filename, label_filename, buffer_size): step_label += buffer_size fmt_images = '>' + str(buffer_size * rows * cols) + 'B' - images_temp = struct.unpack_from(fmt_images, img_buf, - offset_img) + images_temp = struct.unpack_from( + fmt_images, img_buf, offset_img + ) images = numpy.reshape( - images_temp, - (buffer_size, rows * cols)).astype('float32') + images_temp, (buffer_size, rows * cols) + ).astype('float32') offset_img += struct.calcsize(fmt_images) images = images / 255.0 @@ -93,7 +95,8 @@ def reader_creator(image_filename, label_filename, buffer_size): since="2.0.0", update_to="paddle.vision.datasets.MNIST", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(): """ MNIST training set creator. @@ -105,17 +108,22 @@ def train(): :rtype: callable """ return reader_creator( - paddle.dataset.common.download(TRAIN_IMAGE_URL, 'mnist', - TRAIN_IMAGE_MD5), - paddle.dataset.common.download(TRAIN_LABEL_URL, 'mnist', - TRAIN_LABEL_MD5), 100) + paddle.dataset.common.download( + TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5 + ), + paddle.dataset.common.download( + TRAIN_LABEL_URL, 'mnist', TRAIN_LABEL_MD5 + ), + 100, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.MNIST", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(): """ MNIST test set creator. @@ -129,14 +137,16 @@ def test(): return reader_creator( paddle.dataset.common.download(TEST_IMAGE_URL, 'mnist', TEST_IMAGE_MD5), paddle.dataset.common.download(TEST_LABEL_URL, 'mnist', TEST_LABEL_MD5), - 100) + 100, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.MNIST", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5) paddle.dataset.common.download(TRAIN_LABEL_URL, 'mnist', TRAIN_LABEL_MD5) diff --git a/python/paddle/dataset/movielens.py b/python/paddle/dataset/movielens.py index 513e5d2a5d6..fd57ad8edf7 100644 --- a/python/paddle/dataset/movielens.py +++ b/python/paddle/dataset/movielens.py @@ -33,7 +33,7 @@ __all__ = [] age_table = [1, 18, 25, 35, 45, 50, 56] -#URL = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip' +# URL = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip' URL = 'https://dataset.bj.bcebos.com/movielens%2Fml-1m.zip' MD5 = 'c4d9eecfca2ab87c1945afe126590906' @@ -53,13 +53,17 @@ class MovieInfo(object): Get information from a movie. """ return [ - self.index, [CATEGORIES_DICT[c] for c in self.categories], - [MOVIE_TITLE_DICT[w.lower()] for w in self.title.split()] + self.index, + [CATEGORIES_DICT[c] for c in self.categories], + [MOVIE_TITLE_DICT[w.lower()] for w in self.title.split()], ] def __str__(self): return "" % ( - self.index, self.title, self.categories) + self.index, + self.title, + self.categories, + ) def __repr__(self): return self.__str__() @@ -84,8 +88,11 @@ class UserInfo(object): def __str__(self): return "" % ( - self.index, "M" if self.is_male else "F", age_table[self.age], - self.job_id) + self.index, + "M" if self.is_male else "F", + age_table[self.age], + self.job_id, + ) def __repr__(self): return str(self) @@ -117,7 +124,8 @@ def __initialize_meta_info__(): categories_set.add(c) title = pattern.match(title).group(1) MOVIE_INFO[int(movie_id)] = MovieInfo( - index=movie_id, categories=categories, title=title) + index=movie_id, categories=categories, title=title + ) for w in title.split(): title_word_set.add(w.lower()) @@ -137,10 +145,9 @@ def __initialize_meta_info__(): for line in user_file: line = line.decode(encoding='latin') uid, gender, age, job, _ = line.strip().split("::") - USER_INFO[int(uid)] = UserInfo(index=uid, - gender=gender, - age=age, - job_id=job) + USER_INFO[int(uid)] = UserInfo( + index=uid, gender=gender, age=age, job_id=job + ) return fn @@ -166,7 +173,8 @@ def __reader__(rand_seed=0, test_ratio=0.1, is_test=False): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def __reader_creator__(**kwargs): return lambda: __reader__(**kwargs) @@ -179,7 +187,8 @@ test = functools.partial(__reader_creator__, is_test=True) since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_movie_title_dict(): """ Get movie title dictionary. @@ -199,7 +208,8 @@ def __max_index_info__(a, b): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def max_movie_id(): """ Get the maximum value of movie id. @@ -212,7 +222,8 @@ def max_movie_id(): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def max_user_id(): """ Get the maximum value of user id. @@ -232,21 +243,24 @@ def __max_job_id_impl__(a, b): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def max_job_id(): """ Get the maximum value of job id. """ __initialize_meta_info__() - return functools.reduce(__max_job_id_impl__, - list(USER_INFO.values())).job_id + return functools.reduce( + __max_job_id_impl__, list(USER_INFO.values()) + ).job_id @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def movie_categories(): """ Get movie categories dictionary. @@ -259,7 +273,8 @@ def movie_categories(): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def user_info(): """ Get user info dictionary. @@ -272,7 +287,8 @@ def user_info(): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def movie_info(): """ Get movie info dictionary. @@ -294,7 +310,8 @@ def unittest(): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL, "movielens", MD5) diff --git a/python/paddle/dataset/tests/cifar_test.py b/python/paddle/dataset/tests/cifar_test.py index f7bdc47d9aa..6f25975dd8e 100644 --- a/python/paddle/dataset/tests/cifar_test.py +++ b/python/paddle/dataset/tests/cifar_test.py @@ -19,7 +19,6 @@ __all__ = [] class TestCIFAR(unittest.TestCase): - def check_reader(self, reader): sum = 0 label = 0 @@ -32,25 +31,29 @@ class TestCIFAR(unittest.TestCase): def test_test10(self): instances, max_label_value = self.check_reader( - paddle.dataset.cifar.test10()) + paddle.dataset.cifar.test10() + ) self.assertEqual(instances, 10000) self.assertEqual(max_label_value, 9) def test_train10(self): instances, max_label_value = self.check_reader( - paddle.dataset.cifar.train10()) + paddle.dataset.cifar.train10() + ) self.assertEqual(instances, 50000) self.assertEqual(max_label_value, 9) def test_test100(self): instances, max_label_value = self.check_reader( - paddle.dataset.cifar.test100()) + paddle.dataset.cifar.test100() + ) self.assertEqual(instances, 10000) self.assertEqual(max_label_value, 99) def test_train100(self): instances, max_label_value = self.check_reader( - paddle.dataset.cifar.train100()) + paddle.dataset.cifar.train100() + ) self.assertEqual(instances, 50000) self.assertEqual(max_label_value, 99) diff --git a/python/paddle/dataset/tests/flowers_test.py b/python/paddle/dataset/tests/flowers_test.py index 31a49edf975..55a27b0ce8f 100644 --- a/python/paddle/dataset/tests/flowers_test.py +++ b/python/paddle/dataset/tests/flowers_test.py @@ -19,7 +19,6 @@ __all__ = [] class TestFlowers(unittest.TestCase): - def check_reader(self, reader): sum = 0 label = 0 @@ -33,19 +32,22 @@ class TestFlowers(unittest.TestCase): def test_train(self): instances, max_label_value = self.check_reader( - paddle.dataset.flowers.train()) + paddle.dataset.flowers.train() + ) self.assertEqual(instances, 6149) self.assertEqual(max_label_value, 102) def test_test(self): instances, max_label_value = self.check_reader( - paddle.dataset.flowers.test()) + paddle.dataset.flowers.test() + ) self.assertEqual(instances, 1020) self.assertEqual(max_label_value, 102) def test_valid(self): instances, max_label_value = self.check_reader( - paddle.dataset.flowers.valid()) + paddle.dataset.flowers.valid() + ) self.assertEqual(instances, 1020) self.assertEqual(max_label_value, 102) diff --git a/python/paddle/dataset/tests/imikolov_test.py b/python/paddle/dataset/tests/imikolov_test.py index f7f025eb540..f8d8b182e2d 100644 --- a/python/paddle/dataset/tests/imikolov_test.py +++ b/python/paddle/dataset/tests/imikolov_test.py @@ -21,7 +21,6 @@ __all__ = [] class TestMikolov(unittest.TestCase): - def check_reader(self, reader, n): for l in reader(): self.assertEqual(len(l), n) @@ -30,16 +29,18 @@ class TestMikolov(unittest.TestCase): n = 5 self.check_reader(paddle.dataset.imikolov.train(WORD_DICT, n), n) - first_line = 'aer banknote berlitz calloway centrust cluett fromstein '\ - 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts '\ + first_line = ( + 'aer banknote berlitz calloway centrust cluett fromstein ' + 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts ' 'rake regatta rubens sim snack-food ssangyong swapo wachter' + ) first_line = [ WORD_DICT.get(ch, WORD_DICT['']) for ch in first_line.split(' ') ] for l in paddle.dataset.imikolov.train( - WORD_DICT, n=-1, - data_type=paddle.dataset.imikolov.DataType.SEQ)(): + WORD_DICT, n=-1, data_type=paddle.dataset.imikolov.DataType.SEQ + )(): read_line = l[0][1:] break self.assertEqual(first_line, read_line) @@ -48,15 +49,17 @@ class TestMikolov(unittest.TestCase): n = 5 self.check_reader(paddle.dataset.imikolov.test(WORD_DICT, n), n) - first_line = 'consumers may want to move their telephones a little '\ - 'closer to the tv set' + first_line = ( + 'consumers may want to move their telephones a little ' + 'closer to the tv set' + ) first_line = [ WORD_DICT.get(ch, WORD_DICT['']) for ch in first_line.split(' ') ] for l in paddle.dataset.imikolov.test( - WORD_DICT, n=-1, - data_type=paddle.dataset.imikolov.DataType.SEQ)(): + WORD_DICT, n=-1, data_type=paddle.dataset.imikolov.DataType.SEQ + )(): read_line = l[0][1:] break self.assertEqual(first_line, read_line) diff --git a/python/paddle/dataset/tests/mnist_test.py b/python/paddle/dataset/tests/mnist_test.py index d2ebacc8c73..978fa714f72 100644 --- a/python/paddle/dataset/tests/mnist_test.py +++ b/python/paddle/dataset/tests/mnist_test.py @@ -19,7 +19,6 @@ __all__ = [] class TestMNIST(unittest.TestCase): - def check_reader(self, reader): sum = 0 label = 0 @@ -32,13 +31,15 @@ class TestMNIST(unittest.TestCase): def test_train(self): instances, max_label_value = self.check_reader( - paddle.dataset.mnist.train()) + paddle.dataset.mnist.train() + ) self.assertEqual(instances, 60000) self.assertEqual(max_label_value, 9) def test_test(self): instances, max_label_value = self.check_reader( - paddle.dataset.mnist.test()) + paddle.dataset.mnist.test() + ) self.assertEqual(instances, 10000) self.assertEqual(max_label_value, 9) diff --git a/python/paddle/dataset/tests/test_image.py b/python/paddle/dataset/tests/test_image.py index 61979b865fd..953e70c4186 100644 --- a/python/paddle/dataset/tests/test_image.py +++ b/python/paddle/dataset/tests/test_image.py @@ -32,7 +32,7 @@ class Image(unittest.TestCase): """ def test_resize_flip_chw(self): - """ resize """ + """resize""" imgdir = sys.argv[0].replace('test_image.py', 'cat.jpg') images = image.load_image(imgdir) images = image.resize_short(images, 256) diff --git a/python/paddle/dataset/tests/voc2012_test.py b/python/paddle/dataset/tests/voc2012_test.py index 43f96cd9377..7db2f5abd38 100644 --- a/python/paddle/dataset/tests/voc2012_test.py +++ b/python/paddle/dataset/tests/voc2012_test.py @@ -19,7 +19,6 @@ __all__ = [] class TestVOC(unittest.TestCase): - def check_reader(self, reader): sum = 0 label = 0 diff --git a/python/paddle/dataset/tests/wmt16_test.py b/python/paddle/dataset/tests/wmt16_test.py index 1b0b44c974f..58f405b26e3 100644 --- a/python/paddle/dataset/tests/wmt16_test.py +++ b/python/paddle/dataset/tests/wmt16_test.py @@ -19,7 +19,6 @@ __all__ = [] class TestWMT16(unittest.TestCase): - def checkout_one_sample(self, sample): # train data has 3 field: source language word indices, # target language word indices, and target next word indices. @@ -37,23 +36,30 @@ class TestWMT16(unittest.TestCase): def test_train(self): for idx, sample in enumerate( - paddle.dataset.wmt16.train(src_dict_size=100000, - trg_dict_size=100000)()): - if idx >= 10: break + paddle.dataset.wmt16.train( + src_dict_size=100000, trg_dict_size=100000 + )() + ): + if idx >= 10: + break self.checkout_one_sample(sample) def test_test(self): for idx, sample in enumerate( - paddle.dataset.wmt16.test(src_dict_size=1000, - trg_dict_size=1000)()): - if idx >= 10: break + paddle.dataset.wmt16.test(src_dict_size=1000, trg_dict_size=1000)() + ): + if idx >= 10: + break self.checkout_one_sample(sample) def test_val(self): for idx, sample in enumerate( - paddle.dataset.wmt16.validation(src_dict_size=1000, - trg_dict_size=1000)()): - if idx >= 10: break + paddle.dataset.wmt16.validation( + src_dict_size=1000, trg_dict_size=1000 + )() + ): + if idx >= 10: + break self.checkout_one_sample(sample) def test_get_dict(self): diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index 66c6ab3c479..6c085920e2e 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -31,8 +31,19 @@ __all__ = [] URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data' MD5 = 'd4accdce7a25600298819f8e28e8d593' feature_names = [ - 'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', - 'PTRATIO', 'B', 'LSTAT' + 'CRIM', + 'ZN', + 'INDUS', + 'CHAS', + 'NOX', + 'RM', + 'AGE', + 'DIS', + 'RAD', + 'TAX', + 'PTRATIO', + 'B', + 'LSTAT', ] UCI_TRAIN_DATA = None @@ -44,14 +55,15 @@ FLUID_MD5_MODEL = '6e6dd637ccd5993961f68bfbde46090b' def feature_range(maximums, minimums): import matplotlib + matplotlib.use('Agg') import matplotlib.pyplot as plt + fig, ax = plt.subplots() feature_num = len(maximums) - ax.bar(list(range(feature_num)), - maximums - minimums, - color='r', - align='center') + ax.bar( + list(range(feature_num)), maximums - minimums, color='r', align='center' + ) ax.set_title('feature scale') plt.xticks(list(range(feature_num)), feature_names) plt.xlim([-1, feature_num]) @@ -70,10 +82,13 @@ def load_data(filename, feature_num=14, ratio=0.8): data = np.fromfile(filename, sep=' ') data = data.reshape(data.shape[0] // feature_num, feature_num) - maximums, minimums, avgs = data.max(axis=0), data.min( - axis=0), data.sum(axis=0) / data.shape[0] + maximums, minimums, avgs = ( + data.max(axis=0), + data.min(axis=0), + data.sum(axis=0) / data.shape[0], + ) # if you want to print the distribution of input data, you could use function of feature_range - #feature_range(maximums[:-1], minimums[:-1]) + # feature_range(maximums[:-1], minimums[:-1]) for i in range(feature_num - 1): data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i]) offset = int(data.shape[0] * ratio) @@ -85,7 +100,8 @@ def load_data(filename, feature_num=14, ratio=0.8): since="2.0.0", update_to="paddle.text.datasets.UCIHousing", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(): """ UCI_HOUSING training set creator. @@ -110,7 +126,8 @@ def train(): since="2.0.0", update_to="paddle.text.datasets.UCIHousing", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(): """ UCI_HOUSING test set creator. @@ -132,10 +149,9 @@ def test(): def fluid_model(): - parameter_tar = paddle.dataset.common.download(FLUID_URL_MODEL, - 'uci_housing', - FLUID_MD5_MODEL, - 'fit_a_line.fluid.tar') + parameter_tar = paddle.dataset.common.download( + FLUID_URL_MODEL, 'uci_housing', FLUID_MD5_MODEL, 'fit_a_line.fluid.tar' + ) tar = tarfile.TarFile(parameter_tar, mode='r') dirpath = tempfile.mkdtemp() @@ -148,7 +164,8 @@ def fluid_model(): since="2.0.0", update_to="paddle.text.datasets.UCIHousing", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def predict_reader(): """ It returns just one tuple data to do inference. @@ -158,13 +175,14 @@ def predict_reader(): """ global UCI_TEST_DATA load_data(paddle.dataset.common.download(URL, 'uci_housing', MD5)) - return (UCI_TEST_DATA[0][:-1], ) + return (UCI_TEST_DATA[0][:-1],) @deprecated( since="2.0.0", update_to="paddle.text.datasets.UCIHousing", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL, 'uci_housing', MD5) diff --git a/python/paddle/dataset/voc2012.py b/python/paddle/dataset/voc2012.py index a949872fb0b..2a80eab1e84 100644 --- a/python/paddle/dataset/voc2012.py +++ b/python/paddle/dataset/voc2012.py @@ -68,7 +68,8 @@ def reader_creator(filename, sub_name): since="2.0.0", update_to="paddle.vision.datasets.VOC2012", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(): """ Create a train dataset reader containing 2913 images in HWC order. @@ -80,7 +81,8 @@ def train(): since="2.0.0", update_to="paddle.vision.datasets.VOC2012", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(): """ Create a test dataset reader containing 1464 images in HWC order. @@ -92,7 +94,8 @@ def test(): since="2.0.0", update_to="paddle.vision.datasets.VOC2012", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def val(): """ Create a val dataset reader containing 1449 images in HWC order. diff --git a/python/paddle/dataset/wmt14.py b/python/paddle/dataset/wmt14.py index 33ad05c36d0..6d7c2639d2f 100644 --- a/python/paddle/dataset/wmt14.py +++ b/python/paddle/dataset/wmt14.py @@ -27,12 +27,14 @@ import paddle.utils.deprecated as deprecated __all__ = [] -URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/' - 'cslm_joint_paper/data/dev+test.tgz') +URL_DEV_TEST = ( + 'http://www-lium.univ-lemans.fr/~schwenk/' + 'cslm_joint_paper/data/dev+test.tgz' +) MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' # this is a small set of data for test. The original data is too large and # will be add later. -URL_TRAIN = ('http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz') +URL_TRAIN = 'http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz' MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c' # BLEU of this trained model is 26.92 URL_MODEL = 'http://paddlemodels.bj.bcebos.com/wmt%2Fwmt14.tgz' @@ -45,7 +47,6 @@ UNK_IDX = 2 def __read_to_dict(tar_file, dict_size): - def __to_dict(fd, size): out_dict = dict() for line_count, line in enumerate(fd): @@ -57,13 +58,15 @@ def __read_to_dict(tar_file, dict_size): with tarfile.open(tar_file, mode='r') as f: names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith("src.dict") ] assert len(names) == 1 src_dict = __to_dict(f.extractfile(names[0]), dict_size) names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith("trg.dict") ] assert len(names) == 1 @@ -72,12 +75,12 @@ def __read_to_dict(tar_file, dict_size): def reader_creator(tar_file, file_name, dict_size): - def reader(): src_dict, trg_dict = __read_to_dict(tar_file, dict_size) with tarfile.open(tar_file, mode='r') as f: names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith(file_name) ] for name in names: @@ -112,7 +115,8 @@ def reader_creator(tar_file, file_name, dict_size): since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(dict_size): """ WMT14 training set creator. @@ -126,14 +130,17 @@ def train(dict_size): """ return reader_creator( paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN), - 'train/train', dict_size) + 'train/train', + dict_size, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(dict_size): """ WMT14 test set creator. @@ -147,25 +154,31 @@ def test(dict_size): """ return reader_creator( paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN), - 'test/test', dict_size) + 'test/test', + dict_size, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def gen(dict_size): return reader_creator( paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN), - 'gen/gen', dict_size) + 'gen/gen', + dict_size, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_dict(dict_size, reverse=True): # if reverse = False, return dict = {'a':'001', 'b':'002', ...} # else reverse = true, return dict = {'001':'a', '002':'b', ...} @@ -181,7 +194,8 @@ def get_dict(dict_size, reverse=True): since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN) paddle.dataset.common.download(URL_MODEL, 'wmt14', MD5_MODEL) diff --git a/python/paddle/dataset/wmt16.py b/python/paddle/dataset/wmt16.py index f5a97172a7d..01b13cd1368 100644 --- a/python/paddle/dataset/wmt16.py +++ b/python/paddle/dataset/wmt16.py @@ -37,7 +37,7 @@ import paddle.utils.deprecated as deprecated __all__ = [] -DATA_URL = ("http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz") +DATA_URL = "http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz" DATA_MD5 = "0c38be43600334966403524a40dcd81e" TOTAL_EN_WORDS = 11250 @@ -54,7 +54,8 @@ def __build_dict(tar_file, dict_size, save_path, lang): for line in f.extractfile("wmt16/train"): line = line.decode() line_split = line.strip().split("\t") - if len(line_split) != 2: continue + if len(line_split) != 2: + continue sen = line_split[0] if lang == "en" else line_split[1] for w in sen.split(): word_dict[w] += 1 @@ -62,17 +63,21 @@ def __build_dict(tar_file, dict_size, save_path, lang): with open(save_path, "wb") as fout: fout.write(("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)).encode()) for idx, word in enumerate( - sorted(word_dict.items(), key=lambda x: x[1], reverse=True)): - if idx + 3 == dict_size: break + sorted(word_dict.items(), key=lambda x: x[1], reverse=True) + ): + if idx + 3 == dict_size: + break fout.write(word[0].encode()) fout.write(b'\n') def __load_dict(tar_file, dict_size, lang, reverse=False): - dict_path = os.path.join(paddle.dataset.common.DATA_HOME, - "wmt16/%s_%d.dict" % (lang, dict_size)) - if not os.path.exists(dict_path) or (len(open(dict_path, "rb").readlines()) - != dict_size): + dict_path = os.path.join( + paddle.dataset.common.DATA_HOME, "wmt16/%s_%d.dict" % (lang, dict_size) + ) + if not os.path.exists(dict_path) or ( + len(open(dict_path, "rb").readlines()) != dict_size + ): __build_dict(tar_file, dict_size, dict_path, lang) word_dict = {} @@ -87,18 +92,20 @@ def __load_dict(tar_file, dict_size, lang, reverse=False): def __get_dict_size(src_dict_size, trg_dict_size, src_lang): src_dict_size = min( - src_dict_size, (TOTAL_EN_WORDS if src_lang == "en" else TOTAL_DE_WORDS)) + src_dict_size, (TOTAL_EN_WORDS if src_lang == "en" else TOTAL_DE_WORDS) + ) trg_dict_size = min( - trg_dict_size, (TOTAL_DE_WORDS if src_lang == "en" else TOTAL_EN_WORDS)) + trg_dict_size, (TOTAL_DE_WORDS if src_lang == "en" else TOTAL_EN_WORDS) + ) return src_dict_size, trg_dict_size def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang): - def reader(): src_dict = __load_dict(tar_file, src_dict_size, src_lang) - trg_dict = __load_dict(tar_file, trg_dict_size, - ("de" if src_lang == "en" else "en")) + trg_dict = __load_dict( + tar_file, trg_dict_size, ("de" if src_lang == "en" else "en") + ) # the index for start mark, end mark, and unk are the same in source # language and target language. Here uses the source language @@ -117,9 +124,11 @@ def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang): if len(line_split) != 2: continue src_words = line_split[src_col].split() - src_ids = [start_id - ] + [src_dict.get(w, unk_id) - for w in src_words] + [end_id] + src_ids = ( + [start_id] + + [src_dict.get(w, unk_id) for w in src_words] + + [end_id] + ) trg_words = line_split[trg_col].split() trg_ids = [trg_dict.get(w, unk_id) for w in trg_words] @@ -136,7 +145,8 @@ def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang): since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(src_dict_size, trg_dict_size, src_lang="en"): """ WMT16 train set reader. @@ -172,24 +182,31 @@ def train(src_dict_size, trg_dict_size, src_lang="en"): """ if src_lang not in ["en", "de"]: - raise ValueError("An error language type. Only support: " - "en (for English); de(for Germany).") - src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, - src_lang) - - return reader_creator(tar_file=paddle.dataset.common.download( - DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"), - file_name="wmt16/train", - src_dict_size=src_dict_size, - trg_dict_size=trg_dict_size, - src_lang=src_lang) + raise ValueError( + "An error language type. Only support: " + "en (for English); de(for Germany)." + ) + src_dict_size, trg_dict_size = __get_dict_size( + src_dict_size, trg_dict_size, src_lang + ) + + return reader_creator( + tar_file=paddle.dataset.common.download( + DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz" + ), + file_name="wmt16/train", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(src_dict_size, trg_dict_size, src_lang="en"): """ WMT16 test set reader. @@ -224,25 +241,32 @@ def test(src_dict_size, trg_dict_size, src_lang="en"): """ if src_lang not in ["en", "de"]: - raise ValueError("An error language type. " - "Only support: en (for English); de(for Germany).") - - src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, - src_lang) - - return reader_creator(tar_file=paddle.dataset.common.download( - DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"), - file_name="wmt16/test", - src_dict_size=src_dict_size, - trg_dict_size=trg_dict_size, - src_lang=src_lang) + raise ValueError( + "An error language type. " + "Only support: en (for English); de(for Germany)." + ) + + src_dict_size, trg_dict_size = __get_dict_size( + src_dict_size, trg_dict_size, src_lang + ) + + return reader_creator( + tar_file=paddle.dataset.common.download( + DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz" + ), + file_name="wmt16/test", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def validation(src_dict_size, trg_dict_size, src_lang="en"): """ WMT16 validation set reader. @@ -276,24 +300,31 @@ def validation(src_dict_size, trg_dict_size, src_lang="en"): callable: The validation reader. """ if src_lang not in ["en", "de"]: - raise ValueError("An error language type. " - "Only support: en (for English); de(for Germany).") - src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, - src_lang) - - return reader_creator(tar_file=paddle.dataset.common.download( - DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"), - file_name="wmt16/val", - src_dict_size=src_dict_size, - trg_dict_size=trg_dict_size, - src_lang=src_lang) + raise ValueError( + "An error language type. " + "Only support: en (for English); de(for Germany)." + ) + src_dict_size, trg_dict_size = __get_dict_size( + src_dict_size, trg_dict_size, src_lang + ) + + return reader_creator( + tar_file=paddle.dataset.common.download( + DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz" + ), + file_name="wmt16/val", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_dict(lang, dict_size, reverse=False): """ return the word dictionary for the specified language. @@ -312,11 +343,14 @@ def get_dict(lang, dict_size, reverse=False): dict: The word dictionary for the specific language. """ - if lang == "en": dict_size = min(dict_size, TOTAL_EN_WORDS) - else: dict_size = min(dict_size, TOTAL_DE_WORDS) + if lang == "en": + dict_size = min(dict_size, TOTAL_EN_WORDS) + else: + dict_size = min(dict_size, TOTAL_DE_WORDS) - dict_path = os.path.join(paddle.dataset.common.DATA_HOME, - "wmt16/%s_%d.dict" % (lang, dict_size)) + dict_path = os.path.join( + paddle.dataset.common.DATA_HOME, "wmt16/%s_%d.dict" % (lang, dict_size) + ) assert os.path.exists(dict_path), "Word dictionary does not exist. " "Please invoke paddle.dataset.wmt16.train/test/validation first " "to build the dictionary." @@ -328,9 +362,10 @@ def get_dict(lang, dict_size, reverse=False): since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): - """download the entire dataset. - """ - paddle.v4.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, - "wmt16.tar.gz") + """download the entire dataset.""" + paddle.v4.dataset.common.download( + DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz" + ) diff --git a/python/paddle/device/__init__.py b/python/paddle/device/__init__.py index e9894309999..6e14fd50478 100644 --- a/python/paddle/device/__init__.py +++ b/python/paddle/device/__init__.py @@ -199,39 +199,49 @@ def _convert_to_place(device): place = core.CPUPlace() elif lower_device == 'gpu': if not core.is_compiled_with_cuda(): - raise ValueError("The device should not be 'gpu', " - "since PaddlePaddle is not compiled with CUDA") + raise ValueError( + "The device should not be 'gpu', " + "since PaddlePaddle is not compiled with CUDA" + ) place = core.CUDAPlace(ParallelEnv().dev_id) elif lower_device == 'xpu': if not core.is_compiled_with_xpu(): - raise ValueError("The device should not be 'xpu', " - "since PaddlePaddle is not compiled with XPU") + raise ValueError( + "The device should not be 'xpu', " + "since PaddlePaddle is not compiled with XPU" + ) selected_xpus = os.getenv("FLAGS_selected_xpus", "0").split(",") device_id = int(selected_xpus[0]) place = core.XPUPlace(device_id) elif lower_device == 'npu': if not core.is_compiled_with_npu(): - raise ValueError("The device should not be 'npu', " - "since PaddlePaddle is not compiled with NPU") + raise ValueError( + "The device should not be 'npu', " + "since PaddlePaddle is not compiled with NPU" + ) selected_npus = os.getenv("FLAGS_selected_npus", "0").split(",") device_id = int(selected_npus[0]) place = core.NPUPlace(device_id) elif lower_device == 'ipu': if not core.is_compiled_with_ipu(): raise ValueError( - "The device should not be 'ipu', " \ - "since PaddlePaddle is not compiled with IPU") + "The device should not be 'ipu', " + "since PaddlePaddle is not compiled with IPU" + ) place = core.IPUPlace() elif lower_device == 'mlu': if not core.is_compiled_with_mlu(): - raise ValueError("The device should not be 'mlu', " - "since PaddlePaddle is not compiled with MLU") + raise ValueError( + "The device should not be 'mlu', " + "since PaddlePaddle is not compiled with MLU" + ) selected_mlus = os.getenv("FLAGS_selected_mlus", "0").split(",") device_id = int(selected_mlus[0]) place = core.MLUPlace(device_id) elif device in core.get_all_custom_device_type(): - selected_devices = os.getenv("FLAGS_selected_{}s".format(device), - "0").split(",") + selected_devices = os.getenv( + "FLAGS_selected_{}s".format(device), "0" + ).split(",") device_id = int(selected_devices[0]) place = core.CustomPlace(device, device_id) else: @@ -239,7 +249,12 @@ def _convert_to_place(device): avaliable_xpu_device = re.match(r'xpu:\d+', lower_device) avaliable_npu_device = re.match(r'npu:\d+', lower_device) avaliable_mlu_device = re.match(r'mlu:\d+', lower_device) - if not avaliable_gpu_device and not avaliable_xpu_device and not avaliable_npu_device and not avaliable_mlu_device: + if ( + not avaliable_gpu_device + and not avaliable_xpu_device + and not avaliable_npu_device + and not avaliable_mlu_device + ): device_info_list = device.split(':', 1) device_type = device_info_list[0] if device_type in core.get_all_custom_device_type(): @@ -248,15 +263,20 @@ def _convert_to_place(device): place = core.CustomPlace(device_type, device_id) else: raise ValueError( - "The device must be a string which is like 'cpu', {}". - format(', '.join("'{}', '{}:x'".format(x, x) - for x in ['gpu', 'xpu', 'npu', 'mlu'] + - core.get_all_custom_device_type()))) + "The device must be a string which is like 'cpu', {}".format( + ', '.join( + "'{}', '{}:x'".format(x, x) + for x in ['gpu', 'xpu', 'npu', 'mlu'] + + core.get_all_custom_device_type() + ) + ) + ) if avaliable_gpu_device: if not core.is_compiled_with_cuda(): raise ValueError( "The device should not be {}, since PaddlePaddle is " - "not compiled with CUDA".format(avaliable_gpu_device)) + "not compiled with CUDA".format(avaliable_gpu_device) + ) device_info_list = device.split(':', 1) device_id = device_info_list[1] device_id = int(device_id) @@ -265,7 +285,8 @@ def _convert_to_place(device): if not core.is_compiled_with_xpu(): raise ValueError( "The device should not be {}, since PaddlePaddle is " - "not compiled with XPU".format(avaliable_xpu_device)) + "not compiled with XPU".format(avaliable_xpu_device) + ) device_info_list = device.split(':', 1) device_id = device_info_list[1] device_id = int(device_id) @@ -274,7 +295,8 @@ def _convert_to_place(device): if not core.is_compiled_with_npu(): raise ValueError( "The device should not be {}, since PaddlePaddle is " - "not compiled with NPU".format(avaliable_npu_device)) + "not compiled with NPU".format(avaliable_npu_device) + ) device_info_list = device.split(':', 1) device_id = device_info_list[1] device_id = int(device_id) @@ -283,7 +305,8 @@ def _convert_to_place(device): if not core.is_compiled_with_mlu(): raise ValueError( "The device should not be {}, since PaddlePaddle is " - "not compiled with mlu".format(avaliable_mlu_device)) + "not compiled with mlu".format(avaliable_mlu_device) + ) device_info_list = device.split(':', 1) device_id = device_info_list[1] device_id = int(device_id) diff --git a/python/paddle/device/cuda/__init__.py b/python/paddle/device/cuda/__init__.py index fa0f194969b..316f9de6122 100644 --- a/python/paddle/device/cuda/__init__.py +++ b/python/paddle/device/cuda/__init__.py @@ -124,8 +124,11 @@ def device_count(): ''' - num_gpus = core.get_cuda_device_count() if hasattr( - core, 'get_cuda_device_count') else 0 + num_gpus = ( + core.get_cuda_device_count() + if hasattr(core, 'get_cuda_device_count') + else 0 + ) return num_gpus @@ -165,7 +168,7 @@ def extract_cuda_device_id(device, op_name): Return: int: The id of the given device. If device is None, return the id of current device. ''' - if (device is None): + if device is None: return core.get_cuda_current_device_id() if isinstance(device, int): @@ -178,15 +181,19 @@ def extract_cuda_device_id(device, op_name): else: raise ValueError( "The current string {} is not expected. Because {} only support string which is like 'gpu:x'. " - "Please input appropriate string again!".format( - device, op_name)) + "Please input appropriate string again!".format(device, op_name) + ) else: raise ValueError( "The device type {} is not expected. Because {} only support int, str or paddle.CUDAPlace. " - "Please input appropriate device again!".format(device, op_name)) + "Please input appropriate device again!".format(device, op_name) + ) - assert device_id >= 0, f"The device id must be not less than 0, but got id = {device_id}." - assert device_id < device_count( + assert ( + device_id >= 0 + ), f"The device id must be not less than 0, but got id = {device_id}." + assert ( + device_id < device_count() ), f"The device id {device_id} exceeds gpu card number {device_count()}" return device_id @@ -424,7 +431,8 @@ def get_device_properties(device=None): raise ValueError( "The API paddle.device.cuda.get_device_properties is not supported in " "CPU-only PaddlePaddle. Please reinstall PaddlePaddle with GPU support " - "to call this API.") + "to call this API." + ) if device is not None: if isinstance(device, int): @@ -438,12 +446,14 @@ def get_device_properties(device=None): raise ValueError( "The current string {} is not expected. Because paddle.device." "cuda.get_device_properties only support string which is like 'gpu:x'. " - "Please input appropriate string again!".format(device)) + "Please input appropriate string again!".format(device) + ) else: raise ValueError( "The device type {} is not expected. Because paddle.device.cuda." "get_device_properties only support int, str or paddle.CUDAPlace. " - "Please input appropriate device again!".format(device)) + "Please input appropriate device again!".format(device) + ) else: device_id = -1 diff --git a/python/paddle/device/cuda/graphs.py b/python/paddle/device/cuda/graphs.py index 44ce2c5eea7..ea9962f2299 100644 --- a/python/paddle/device/cuda/graphs.py +++ b/python/paddle/device/cuda/graphs.py @@ -16,7 +16,11 @@ import os import paddle from paddle.fluid import core from paddle.fluid.layers.utils import _hash_with_id -from paddle.fluid.core import is_compiled_with_cuda, is_compiled_with_rocm, CUDAPlace +from paddle.fluid.core import ( + is_compiled_with_cuda, + is_compiled_with_rocm, + CUDAPlace, +) import warnings if is_compiled_with_cuda() and not is_compiled_with_rocm(): @@ -24,6 +28,7 @@ if is_compiled_with_cuda() and not is_compiled_with_rocm(): def is_cuda_graph_supported(): return True + else: CoreCUDAGraph = None @@ -36,9 +41,10 @@ cuda_graph_id = 0 class CUDAGraph: - def __init__(self, place=None, mode="thread_local"): - assert CoreCUDAGraph is not None, "CUDA Graph is only supported on PaddlePaddle compiled with NVIDIA GPU." + assert ( + CoreCUDAGraph is not None + ), "CUDA Graph is only supported on PaddlePaddle compiled with NVIDIA GPU." self._graph = None if place is None: @@ -65,7 +71,8 @@ class CUDAGraph: dirname = dirname.name os.makedirs(name=dirname, exist_ok=True) assert os.path.isdir( - dirname), "The dirname {} should be a directory".format(dirname) + dirname + ), "The dirname {} should be a directory".format(dirname) if flags is None: flags = 2047 # only all information. It can be any integer inside [1, 2048) self._graph.print_to_dot_files(dirname, flags) @@ -76,6 +83,7 @@ def wrap_cuda_graph(function, mode="thread_local", memory_pool="default"): if not paddle.in_dynamic_mode(): # static mode from paddle.fluid.framework import _cuda_graph_guard + global cuda_graph_id graph_id = str(cuda_graph_id) cuda_graph_id += 1 @@ -86,13 +94,15 @@ def wrap_cuda_graph(function, mode="thread_local", memory_pool="default"): else: raise ValueError( "memory_pool should be one of default or new under static mode, but got", - memory_pool) + memory_pool, + ) return _cuda_graph_guard( - mode + ';' + str(memory_pool_id) + ';' + - graph_id)(lambda *args, **kwargs: function(*args, **kwargs)) + mode + ';' + str(memory_pool_id) + ';' + graph_id + )(lambda *args, **kwargs: function(*args, **kwargs)) from paddle.jit import to_static from paddle.nn import Layer + new_function = to_static(function) if isinstance(function, Layer): mock_func = new_function.forward @@ -105,7 +115,9 @@ def wrap_cuda_graph(function, mode="thread_local", memory_pool="default"): mock_func._cuda_graph_pool_id = CoreCUDAGraph.gen_new_memory_pool_id() else: if isinstance(memory_pool, Layer): - mock_func._cuda_graph_pool_id = memory_pool.forward._cuda_graph_pool_id + mock_func._cuda_graph_pool_id = ( + memory_pool.forward._cuda_graph_pool_id + ) else: mock_func._cuda_graph_pool_id = memory_pool._cuda_graph_pool_id return new_function @@ -224,16 +236,21 @@ def get_cuda_graph_sections(program): for idx, op in enumerate(block.ops): if op.type == 'conditional_block' or op.type == 'while': - assert op._cuda_graph_attr is None, "Cuda graph not support conditional block op and while op." + assert ( + op._cuda_graph_attr is None + ), "Cuda graph not support conditional block op and while op." if op.has_attr('is_test') and op.attr('is_test'): is_test = True # find cuda graph sections if op._cuda_graph_attr is not None: - assert isinstance(op._cuda_graph_attr, - str), "cuda_graph_attr should be a str" + assert isinstance( + op._cuda_graph_attr, str + ), "cuda_graph_attr should be a str" cuda_graph_attrs = op._cuda_graph_attr.split(';') - assert len(cuda_graph_attrs) == 3, "cuda graph attr should have three fields: " \ - "cuda graph mode, cuda graph memory pool id, cuda graph id" + assert len(cuda_graph_attrs) == 3, ( + "cuda graph attr should have three fields: " + "cuda graph mode, cuda graph memory pool id, cuda graph id" + ) local_cuda_graph_id = int(cuda_graph_attrs[2]) if local_cuda_graph_id == current_cuda_graph_id: if len(internal_section) > 0: @@ -241,12 +258,16 @@ def get_cuda_graph_sections(program): internal_idx ), "len of internal section should be equal with len of internal idx" for internal_op in internal_section: - loss_related = (int(internal_op.attr(op_role_attr_name)) - == loss_op_role) or int( - (internal_op.attr(op_role_attr_name) - ) == loss_grad_op_role) - sub_block_related = (op.type == 'conditional_block' - or op.type == 'while') + loss_related = ( + int(internal_op.attr(op_role_attr_name)) + == loss_op_role + ) or int( + (internal_op.attr(op_role_attr_name)) + == loss_grad_op_role + ) + sub_block_related = ( + op.type == 'conditional_block' or op.type == 'while' + ) if loss_related or sub_block_related: # If loss_related is True # The internal section contains loss related ops, @@ -260,8 +281,9 @@ def get_cuda_graph_sections(program): internal_section = [] internal_idx = [] # Beside clear the internal section, a new cuda graph section should be recorded - assert len(current_section) == len(current_idx), \ - "num of section's op is not equal with the idx" + assert len(current_section) == len( + current_idx + ), "num of section's op is not equal with the idx" if len(current_section) > 0: # store previous section cuda_graph_sections.append(current_section) @@ -282,7 +304,9 @@ def get_cuda_graph_sections(program): # internal ops and idx belong to no section, just clear it internal_section = [] internal_idx = [] - current_cuda_graph_id = local_cuda_graph_id # start record a new section + current_cuda_graph_id = ( + local_cuda_graph_id # start record a new section + ) assert len(current_section) == len( current_idx ), "num of section's op is not equal with num of idx" @@ -299,7 +323,8 @@ def get_cuda_graph_sections(program): # handle the last section assert len(current_section) == len( - current_idx), "num of section's op is not equal with num of idx" + current_idx + ), "num of section's op is not equal with num of idx" if len(current_section) > 0: # store previous section cuda_graph_sections.append(current_section) @@ -308,9 +333,15 @@ def get_cuda_graph_sections(program): return cuda_graph_sections, sections_idx, is_test -def replace_cuda_graph_section(ins_and_outs, section_program, section_idx, - origin_program, cuda_graph_section, order, - is_test): +def replace_cuda_graph_section( + ins_and_outs, + section_program, + section_idx, + origin_program, + cuda_graph_section, + order, + is_test, +): """ Use section_program and ins_and_outs to initialize a run_program_op, and replace the section_idx marks ops in the origin program. @@ -344,8 +375,9 @@ def replace_cuda_graph_section(ins_and_outs, section_program, section_idx, memory_pool_id = int(attrs[1]) break - assert mode is not None and memory_pool_id is not None, \ - "mode and memory pool id should be specified in cuda graph attr" + assert ( + mode is not None and memory_pool_id is not None + ), "mode and memory pool id should be specified in cuda graph attr" cuda_graph_var = origin_block.create_var( name="cuda_graph_" + str(order), @@ -372,7 +404,7 @@ def replace_cuda_graph_section(ins_and_outs, section_program, section_idx, outputs={ 'Out': outs, 'OutScope': out_scope_var, - 'CUDAGraph': cuda_graph_var + 'CUDAGraph': cuda_graph_var, }, attrs={ 'global_block': section_program.global_block(), @@ -386,7 +418,8 @@ def replace_cuda_graph_section(ins_and_outs, section_program, section_idx, 'use_interpretorcore': False, 'forward_global_block': section_program.global_block(), 'backward_global_block': section_program.global_block(), - }) + }, + ) def cuda_graph_transform(program): @@ -408,9 +441,11 @@ def cuda_graph_transform(program): # A cuda graph section contains all ops marked with same cuda graph id and # some ops inserted by some optimizers (amp, sharding for example) between ops with same id. cuda_graph_sections, sections_idx, is_test = get_cuda_graph_sections( - program) - assert len(cuda_graph_sections) == len(sections_idx), \ - "num of cuda graph sections is not equal with num of idx sections" + program + ) + assert len(cuda_graph_sections) == len( + sections_idx + ), "num of cuda graph sections is not equal with num of idx sections" # step 2: construct new program for each section and find inputs and outputs of each section. # The inputs are variables generated outside the section but will be used by this section. @@ -420,23 +455,27 @@ def cuda_graph_transform(program): for i in range(len(cuda_graph_sections)): # creating new program for current section section_program, ins_outs = construct_program_and_find_ins_outs( - cuda_graph_sections[i], program, sections_idx[i]) + cuda_graph_sections[i], program, sections_idx[i] + ) ins_and_outs.append(ins_outs) section_programs.append(section_program) - assert len(section_programs) == len(cuda_graph_sections), \ - "the num of cuda graph sections should be equal with the num of new program" + assert len(section_programs) == len( + cuda_graph_sections + ), "the num of cuda graph sections should be equal with the num of new program" # step 3: replace the ops in original program with run_program_op. # Will remove all ops in the section from origin program, and use run_program_op to replace them. for i in reversed(range(len(cuda_graph_sections))): # carry out the replacement in reversed order, to keep the previous idx intact - replace_cuda_graph_section(ins_and_outs[i], - section_programs[i], - sections_idx[i], - program, - cuda_graph_sections[i], - order=i, - is_test=is_test) + replace_cuda_graph_section( + ins_and_outs[i], + section_programs[i], + sections_idx[i], + program, + cuda_graph_sections[i], + order=i, + is_test=is_test, + ) # NOTE: user should hold these program, for now just return these program back to caller return section_programs diff --git a/python/paddle/distributed/__init__.py b/python/paddle/distributed/__init__.py index 13c305b58da..2bce21be7cc 100644 --- a/python/paddle/distributed/__init__.py +++ b/python/paddle/distributed/__init__.py @@ -72,12 +72,42 @@ from .sharding import save_group_sharded_model # noqa: F401 from . import rpc __all__ = [ # noqa - "spawn", "launch", "scatter", "broadcast", "ParallelEnv", "new_group", - "init_parallel_env", "gloo_init_parallel_env", "gloo_barrier", - "gloo_release", "QueueDataset", "split", "CountFilterEntry", - "ShowClickEntry", "get_world_size", "get_group", "all_gather", - "all_gather_object", "InMemoryDataset", "barrier", "all_reduce", "alltoall", - "alltoall_single", "send", "reduce", "recv", "ReduceOp", "wait", "get_rank", - "ProbabilityEntry", "ParallelMode", "is_initialized", - "destroy_process_group", "isend", "irecv", "reduce_scatter", "rpc", "stream" + "spawn", + "launch", + "scatter", + "broadcast", + "ParallelEnv", + "new_group", + "init_parallel_env", + "gloo_init_parallel_env", + "gloo_barrier", + "gloo_release", + "QueueDataset", + "split", + "CountFilterEntry", + "ShowClickEntry", + "get_world_size", + "get_group", + "all_gather", + "all_gather_object", + "InMemoryDataset", + "barrier", + "all_reduce", + "alltoall", + "alltoall_single", + "send", + "reduce", + "recv", + "ReduceOp", + "wait", + "get_rank", + "ProbabilityEntry", + "ParallelMode", + "is_initialized", + "destroy_process_group", + "isend", + "irecv", + "reduce_scatter", + "rpc", + "stream", ] diff --git a/python/paddle/distributed/auto_parallel/callbacks.py b/python/paddle/distributed/auto_parallel/callbacks.py index 17ce5bd71b8..26371cc4ccb 100644 --- a/python/paddle/distributed/auto_parallel/callbacks.py +++ b/python/paddle/distributed/auto_parallel/callbacks.py @@ -16,22 +16,30 @@ import os import time import paddle -from paddle.hapi.callbacks import ProgBarLogger, ModelCheckpoint, LRScheduler, CallbackList, Callback +from paddle.hapi.callbacks import ( + ProgBarLogger, + ModelCheckpoint, + LRScheduler, + CallbackList, + Callback, +) from .interface import CollectionNames, get_collection -def config_callbacks(callbacks=None, - engine=None, - batch_size=None, - epochs=None, - steps=None, - log_freq=2, - verbose=2, - save_freq=1, - save_dir=None, - metrics=None, - acc_step=1, - mode='train'): +def config_callbacks( + callbacks=None, + engine=None, + batch_size=None, + epochs=None, + steps=None, + log_freq=2, + verbose=2, + save_freq=1, + save_dir=None, + metrics=None, + acc_step=1, + mode='train', +): cbks = callbacks or [] cbks = cbks if isinstance(cbks, (list, tuple)) else [cbks] @@ -74,7 +82,6 @@ def config_callbacks(callbacks=None, class ProgBarLoggerAuto(ProgBarLogger): - def __init__(self, log_freq=1, verbose=2): super(ProgBarLoggerAuto, self).__init__(log_freq, verbose) @@ -110,16 +117,22 @@ class ProgBarLoggerAuto(ProgBarLogger): cnt = timer['count'] if timer['count'] > 0 else 1.0 samples = timer['samples'] if timer['samples'] > 0 else 1.0 values.append( - ('avg_reader_cost', "%.5f sec" % (timer['data_time'] / cnt))) + ('avg_reader_cost', "%.5f sec" % (timer['data_time'] / cnt)) + ) values.append( - ('avg_batch_cost', "%.5f sec" % (timer['batch_time'] / cnt))) + ('avg_batch_cost', "%.5f sec" % (timer['batch_time'] / cnt)) + ) values.append( - ('ips', "%.5f samples/sec" % - (samples / (timer['data_time'] + timer['batch_time'])))) + ( + 'ips', + "%.5f samples/sec" + % (samples / (timer['data_time'] + timer['batch_time'])), + ) + ) timer['count'] = 0 timer['samples'] = 0 - timer['data_time'] = 0. - timer['batch_time'] = 0. + timer['data_time'] = 0.0 + timer['batch_time'] = 0.0 progbar.update(steps, values) @@ -130,7 +143,8 @@ class ProgBarLoggerAuto(ProgBarLogger): self.evaled_samples += samples self._eval_timer['batch_time'] += ( - time.time() - self._eval_timer['batch_data_end_time']) + time.time() - self._eval_timer['batch_data_end_time'] + ) self._eval_timer['count'] += 1 samples = self.params['batch_size'] self._eval_timer['samples'] += samples @@ -143,7 +157,6 @@ class ProgBarLoggerAuto(ProgBarLogger): class LRSchedulerAuto(LRScheduler): - def __init__(self, by_step=True, by_epoch=False): super(LRSchedulerAuto, self).__init__(by_step, by_epoch) @@ -156,15 +169,18 @@ class LRSchedulerAuto(LRScheduler): self.train_step += 1 if self.by_step and self.train_step % self.acc_step == 0: - if self.model._optimizer and \ - hasattr(self.model._optimizer, '_learning_rate') and \ - isinstance(self.model._optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): + if ( + self.model._optimizer + and hasattr(self.model._optimizer, '_learning_rate') + and isinstance( + self.model._optimizer._learning_rate, + paddle.optimizer.lr.LRScheduler, + ) + ): self.model._optimizer._learning_rate.step() class History(Callback): - def __init__(self): self.history = {} @@ -181,7 +197,6 @@ class History(Callback): class Profiler(Callback): - def __init__(self, *args, **kwargs): self.prof = paddle.profiler.Profiler(*args, **kwargs) @@ -197,8 +212,11 @@ class Profiler(Callback): def on_train_batch_end(self, step, logs=None): self.train_step += 1 self.prof.step(num_samples=self.batch_size) - print("step {}:{}".format(self.train_step, - self.prof.step_info(unit='samples'))) + print( + "step {}:{}".format( + self.train_step, self.prof.step_info(unit='samples') + ) + ) def on_train_end(self, logs=None): self.prof.stop() @@ -206,7 +224,6 @@ class Profiler(Callback): class ModelCheckpointAuto(ModelCheckpoint): - def __init__(self, *args, **kwargs): super(ModelCheckpointAuto, self).__init__(*args, **kwargs) diff --git a/python/paddle/distributed/auto_parallel/cluster.py b/python/paddle/distributed/auto_parallel/cluster.py index e17f83eb419..ca66a770e3c 100644 --- a/python/paddle/distributed/auto_parallel/cluster.py +++ b/python/paddle/distributed/auto_parallel/cluster.py @@ -128,8 +128,15 @@ class Device: def __str__(self): str = "" str += "global_id: {}, local_id: {}, machine_id: {}, type: {}, model: {}, dp_flops: {}, sp_flops: {}, memory: {}".format( - self.global_id, self.local_id, self.machine.id, self.type.name, - self.model, self.dp_gflops, self.sp_gflops, self.memory) + self.global_id, + self.local_id, + self.machine.id, + self.type.name, + self.model, + self.dp_gflops, + self.sp_gflops, + self.memory, + ) return str def __repr__(self): @@ -202,8 +209,12 @@ class Link: def __str__(self): str = "" str += "source_global_id: {}, target_global_id: {}, type: {}, bandwidth: {}, latency: {}".format( - self.source.global_id, self.target.global_id, self.type, - self.bandwidth, self.latency) + self.source.global_id, + self.target.global_id, + self.type, + self.bandwidth, + self.latency, + ) return str def __repr__(self): @@ -211,7 +222,6 @@ class Link: class Machine: - def __init__(self, id): self._id = id self._hostname = None @@ -292,7 +302,6 @@ class Machine: class AlphaLatency: - def __init__(self, alpha_latency): assert isinstance(alpha_latency, dict) self._base = alpha_latency.get("base", None) @@ -304,12 +313,15 @@ class AlphaLatency: self._switch = float(self._switch) except: raise TypeError("The switch latency must be float") - self._base_ring = self._base.get( - "ring", None) if self._base is not None else None - self._base_tree = self._base.get( - "tree", None) if self._base is not None else None - self._base_inter = self._base.get( - "inter", None) if self._base is not None else None + self._base_ring = ( + self._base.get("ring", None) if self._base is not None else None + ) + self._base_tree = ( + self._base.get("tree", None) if self._base is not None else None + ) + self._base_inter = ( + self._base.get("inter", None) if self._base is not None else None + ) if self._base_ring is not None: try: self._base_ring = float(self._base_ring) @@ -416,19 +428,21 @@ class Cluster: # which have the same number accelerators. self._num_devices_per_machine = None - def gen_default_config_cluster(self, - gpu_model="V100", - cpu_model="6271C", - node_count=1, - device_count=1, - gpu_memory=32, - cpu_memory=503, - inter_bandwidth=24, - intra_bandwidth=235, - gpu_dp_gflops=7800, - gpu_sp_gflops=15700, - cpu_dp_gflops=75, - cpu_sp_gflops=150): + def gen_default_config_cluster( + self, + gpu_model="V100", + cpu_model="6271C", + node_count=1, + device_count=1, + gpu_memory=32, + cpu_memory=503, + inter_bandwidth=24, + intra_bandwidth=235, + gpu_dp_gflops=7800, + gpu_sp_gflops=15700, + cpu_dp_gflops=75, + cpu_sp_gflops=150, + ): """Generate cluster by default config.""" gpu_models = ["V100", "A100", "H100", "A2", "A10", "A16", "A30", "A40"] xpu_models = ["XPU"] @@ -602,25 +616,31 @@ class Cluster: prev_machine = self._machines[machine.id - 1] offset = prev_machine._non_accelerator_cumulative_count for global_id in machine.devices: - if machine.devices[ - global_id].type not in Device.NON_ACCELERATOR_TYPE: + if ( + machine.devices[global_id].type + not in Device.NON_ACCELERATOR_TYPE + ): rank_id = global_id - offset self._rank_to_device_id[rank_id] = global_id self._device_id_to_rank[global_id] = rank_id - machine._non_accelerator_cumulative_count = len( - machine.devices) - len( - machine.accelerators - ) + prev_machine._non_accelerator_cumulative_count + machine._non_accelerator_cumulative_count = ( + len(machine.devices) + - len(machine.accelerators) + + prev_machine._non_accelerator_cumulative_count + ) else: for global_id in machine.devices: - if machine.devices[ - global_id].type not in Device.NON_ACCELERATOR_TYPE: + if ( + machine.devices[global_id].type + not in Device.NON_ACCELERATOR_TYPE + ): rank_id = global_id self._rank_to_device_id[rank_id] = global_id self._device_id_to_rank[global_id] = rank_id machine.accelerators[global_id] = machine.devices[global_id] machine._non_accelerator_cumulative_count = len( - machine.devices) - len(machine.accelerators) + machine.devices + ) - len(machine.accelerators) @property def alpha_latency(self): @@ -696,7 +716,8 @@ class Cluster: if "alpha_latency" in cluster_info: self._alpha_latency = AlphaLatency( - cluster_info.get("alpha_latency")) + cluster_info.get("alpha_latency") + ) else: self._alpha_latecy = None @@ -732,7 +753,7 @@ class Cluster: else: bandwidth = link.bandwidth - if bandwidth == 0.: + if bandwidth == 0.0: beta = 0 else: beta = 1 / (bandwidth * (convert_base**3 / 10**6)) @@ -813,13 +834,16 @@ def get_default_cluster(): global_device_count = int(global_device_count) assert global_device_count % local_device_count == 0 node_count = int(global_device_count) // local_device_count - print("Node Count: ", - node_count, - "Local Device Size: ", - local_device_count, - "World size: ", - paddle.distributed.get_world_size(), - flush=True) - cluster.gen_default_config_cluster(node_count=node_count, - device_count=local_device_count) + print( + "Node Count: ", + node_count, + "Local Device Size: ", + local_device_count, + "World size: ", + paddle.distributed.get_world_size(), + flush=True, + ) + cluster.gen_default_config_cluster( + node_count=node_count, device_count=local_device_count + ) return cluster diff --git a/python/paddle/distributed/auto_parallel/cluster_v2.py b/python/paddle/distributed/auto_parallel/cluster_v2.py index 866ab338ea2..debcb078f6e 100644 --- a/python/paddle/distributed/auto_parallel/cluster_v2.py +++ b/python/paddle/distributed/auto_parallel/cluster_v2.py @@ -75,10 +75,10 @@ class DeviceMesh(core.DeviceMesh): def __init__(self, name, mesh, dim_names=None): self._name = name - if not isinstance(mesh, list) and \ - not isinstance(mesh, np.ndarray): + if not isinstance(mesh, list) and not isinstance(mesh, np.ndarray): raise ValueError( - 'The mesh must be an instance of list or np.ndarray.') + 'The mesh must be an instance of list or np.ndarray.' + ) if isinstance(mesh, list): mesh = np.array(mesh) @@ -87,24 +87,29 @@ class DeviceMesh(core.DeviceMesh): self._shape = list(self._mesh.shape) self._device_ids = self._mesh.flatten().tolist() - assert all(isinstance(p, int) for p in self._device_ids), \ - ("All elements of the mesh be integer") - assert min( - self._device_ids) >= 0, ('All elements of the mesh must be >= 0.') + assert all( + isinstance(p, int) for p in self._device_ids + ), "All elements of the mesh be integer" + assert ( + min(self._device_ids) >= 0 + ), 'All elements of the mesh must be >= 0.' unique_device_ids = set(self._device_ids) assert len(unique_device_ids) == len( - self._device_ids), ('All elements of the mesh must be unique.') + self._device_ids + ), 'All elements of the mesh must be unique.' if dim_names is not None: - assert len(dim_names) == len(self._shape), \ - ("The length of dims_names must be same as the shape of the mesh.") + assert len(dim_names) == len( + self._shape + ), "The length of dims_names must be same as the shape of the mesh." self._dim_names = dim_names else: self._dim_names = ["d" + str(i) for i in range(len(self._shape))] # Follow the requirement for using pybind11 - core.DeviceMesh.__init__(self, self._name, self._shape, - self._device_ids, self._dim_names) + core.DeviceMesh.__init__( + self, self._name, self._shape, self._device_ids, self._dim_names + ) @property def mesh(self): diff --git a/python/paddle/distributed/auto_parallel/completion.py b/python/paddle/distributed/auto_parallel/completion.py index db387ef2bb2..ba879e89f8d 100644 --- a/python/paddle/distributed/auto_parallel/completion.py +++ b/python/paddle/distributed/auto_parallel/completion.py @@ -55,7 +55,8 @@ def compute_compatible_process_mesh(process_mesh_list): compatible_result = None for process_mesh in process_mesh_list: compatible, compatible_result = _compute_compatible_process_mesh_two( - compatible_result, process_mesh) + compatible_result, process_mesh + ) if not compatible: return None return copy.deepcopy(compatible_result) @@ -78,7 +79,8 @@ def compute_compatible_dim_mapping(dim_mapping_list): compatible_result = -1 for mapping in dim_mapping_list: compatible, compatible_result = _compute_compatible_dim_mapping_two( - compatible_result, mapping) + compatible_result, mapping + ) if not compatible: return None return compatible_result @@ -86,7 +88,7 @@ def compute_compatible_dim_mapping(dim_mapping_list): def compute_compatible_dims_mapping(dims_mapping_list): """Compute the compatible dims mapping given a list of dims mapping. - Each of dims mapping is also a list. + Each of dims mapping is also a list. """ if not dims_mapping_list: return None @@ -99,7 +101,8 @@ def compute_compatible_dims_mapping(dims_mapping_list): compatible_result = [] for dim_mappings in zip(*dims_mapping_list): compatible_dim_mapping = compute_compatible_dim_mapping( - list(dim_mappings)) + list(dim_mappings) + ) if compatible_dim_mapping is None: return None compatible_result.append(compatible_dim_mapping) @@ -125,7 +128,8 @@ def _validate_dims_mapping(dims_mapping, process_mesh): return False for i in range(len(dims_mapping)): if dims_mapping[i] < -1 or dims_mapping[i] >= len( - process_mesh.topology): + process_mesh.topology + ): return False for i in range(len(process_mesh.topology)): if dims_mapping.count(i) > 1: @@ -134,7 +138,6 @@ def _validate_dims_mapping(dims_mapping, process_mesh): class Completer: - def __init__(self, dist_context): assert dist_context is not None self._dist_context = dist_context @@ -146,12 +149,15 @@ class Completer: return False tensor_desc = tensor_node.var() # Skip reader tensor - if tensor_desc.type() == core.VarDesc.VarType.READER \ - or tensor_desc.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or tensor_desc.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + tensor_desc.type() == core.VarDesc.VarType.READER + or tensor_desc.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or tensor_desc.type == core.VarDesc.VarType.STEP_SCOPES + ): return False tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) + tensor_node + ) assert tensor_dist_attr is not None if tensor_dist_attr.is_annotated("dims_mapping"): return False @@ -160,48 +166,74 @@ class Completer: dims_mapping_list = [] for pred_op_node in tensor_node.inputs: if pred_op_node.op() is not None: - if pred_op_node.op().type() == "create_py_reader" \ - or pred_op_node.op().type() == "create_double_buffer_reader" \ - or pred_op_node.op().type() == "read": + if ( + pred_op_node.op().type() == "create_py_reader" + or pred_op_node.op().type() + == "create_double_buffer_reader" + or pred_op_node.op().type() == "read" + ): continue - op_dist_attr = self._dist_context.get_op_dist_attr_for_graph( - pred_op_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_graph( + pred_op_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): op_dims_mapping = op_dist_attr.get_output_dims_mapping( - tensor_desc.name()) + tensor_desc.name() + ) dims_mapping_list.append(op_dims_mapping) dims_mapping_list.append(tensor_dims_mapping) compatible_dims_mapping = compute_compatible_dims_mapping( - dims_mapping_list) - if not _validate_dims_mapping(compatible_dims_mapping, - tensor_dist_attr.process_mesh): + dims_mapping_list + ) + if not _validate_dims_mapping( + compatible_dims_mapping, tensor_dist_attr.process_mesh + ): return False - if (compatible_dims_mapping is not None) and \ - (compatible_dims_mapping != tensor_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != tensor_dims_mapping + ): tensor_dist_attr.dims_mapping = compatible_dims_mapping changed = True else: dims_mapping_list = [] for succ_op_node in tensor_node.outputs: if succ_op_node.op() is not None: - if succ_op_node.op().type() == "create_py_reader" \ - or succ_op_node.op().type() == "create_double_buffer_reader" \ - or succ_op_node.op().type() == "read": + if ( + succ_op_node.op().type() == "create_py_reader" + or succ_op_node.op().type() + == "create_double_buffer_reader" + or succ_op_node.op().type() == "read" + ): continue - op_dist_attr = self._dist_context.get_op_dist_attr_for_graph( - succ_op_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_graph( + succ_op_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): op_dims_mapping = op_dist_attr.get_input_dims_mapping( - tensor_desc.name()) + tensor_desc.name() + ) dims_mapping_list.append(op_dims_mapping) dims_mapping_list.append(tensor_dims_mapping) compatible_dims_mapping = compute_compatible_dims_mapping( - dims_mapping_list) - if not _validate_dims_mapping(compatible_dims_mapping, - tensor_dist_attr.process_mesh): + dims_mapping_list + ) + if not _validate_dims_mapping( + compatible_dims_mapping, tensor_dist_attr.process_mesh + ): return False - if (compatible_dims_mapping is not None) and \ - (compatible_dims_mapping != tensor_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != tensor_dims_mapping + ): tensor_dist_attr.dims_mapping = compatible_dims_mapping changed = True return changed @@ -212,10 +244,12 @@ class Completer: return False # Skip reader op op_desc = op_node.op() - if op_desc.type() == "create_py_reader" \ - or op_desc.type() == "create_double_buffer_reader" \ - or op_desc.type() == "while" \ - or op_desc.type() == "read": + if ( + op_desc.type() == "create_py_reader" + or op_desc.type() == "create_double_buffer_reader" + or op_desc.type() == "while" + or op_desc.type() == "read" + ): return False dist_op = self._dist_context.get_dist_op_for_graph(op_node) op_dist_attr = dist_op.dist_attr @@ -227,28 +261,42 @@ class Completer: continue tensor_desc = tensor_node.var() if op_dist_attr.is_annotated_input_dims_mapping( - tensor_desc.name()): + tensor_desc.name() + ): continue - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): tensor_dims_mapping = tensor_dist_attr.dims_mapping op_dims_mapping = op_dist_attr.get_input_dims_mapping( - tensor_desc.name()) - compatible_dims_mapping = compute_compatible_dims_mapping( - [op_dims_mapping, tensor_dims_mapping]) + tensor_desc.name() + ) + compatible_dims_mapping = ( + compute_compatible_dims_mapping( + [op_dims_mapping, tensor_dims_mapping] + ) + ) if not _validate_dims_mapping( - compatible_dims_mapping, - op_dist_attr.process_mesh): + compatible_dims_mapping, op_dist_attr.process_mesh + ): continue - if (compatible_dims_mapping is not None) and \ - (compatible_dims_mapping != op_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != op_dims_mapping + ): op_dist_attr.set_input_dims_mapping( - tensor_desc.name(), compatible_dims_mapping) + tensor_desc.name(), compatible_dims_mapping + ) changed = True # Find the most compatible implemenetations from the distributed operator - op_dist_impls = find_compatible_distributed_operator_impls(dist_op, - fwd=True) + op_dist_impls = find_compatible_distributed_operator_impls( + dist_op, fwd=True + ) if op_dist_impls is not None: not_compatible = True backup_op_dist_attr = copy.deepcopy(op_dist_attr) @@ -257,8 +305,10 @@ class Completer: dim_changed = op_dist_impl.update_dims_mapping(dist_op) if dim_changed: changed = True - if op_dist_impl.is_auto_compatible(dist_op) \ - and dist_op.validate_dist_attr(): + if ( + op_dist_impl.is_auto_compatible(dist_op) + and dist_op.validate_dist_attr() + ): if op_dist_impl.type == "elementwise": op_dist_attr.impl_type = "default" else: @@ -283,28 +333,42 @@ class Completer: continue tensor_desc = tensor_node.var() if op_dist_attr.is_annotated_output_dims_mapping( - tensor_desc.name()): + tensor_desc.name() + ): continue - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): tensor_dims_mapping = tensor_dist_attr.dims_mapping op_dims_mapping = op_dist_attr.get_output_dims_mapping( - tensor_desc.name()) - compatible_dims_mapping = compute_compatible_dims_mapping( - [op_dims_mapping, tensor_dims_mapping]) + tensor_desc.name() + ) + compatible_dims_mapping = ( + compute_compatible_dims_mapping( + [op_dims_mapping, tensor_dims_mapping] + ) + ) if not _validate_dims_mapping( - compatible_dims_mapping, - op_dist_attr.process_mesh): + compatible_dims_mapping, op_dist_attr.process_mesh + ): continue - if (compatible_dims_mapping is not None) and \ - (compatible_dims_mapping != op_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != op_dims_mapping + ): op_dist_attr.set_output_dims_mapping( - tensor_desc.name(), compatible_dims_mapping) + tensor_desc.name(), compatible_dims_mapping + ) changed = True # Find the most compatible implemenetations from the distributed operator op_dist_impls = find_compatible_distributed_operator_impls( - dist_op, fwd=False) + dist_op, fwd=False + ) if op_dist_impls is not None: not_compatible = True backup_op_dist_attr = copy.deepcopy(op_dist_attr) @@ -313,8 +377,10 @@ class Completer: dim_changed = op_dist_impl.update_dims_mapping(dist_op) if dim_changed: changed = True - if op_dist_impl.is_auto_compatible(dist_op) \ - and dist_op.validate_dist_attr(): + if ( + op_dist_impl.is_auto_compatible(dist_op) + and dist_op.validate_dist_attr() + ): if op_dist_impl.type == "elementwise": op_dist_attr.impl_type = "default" else: @@ -338,24 +404,33 @@ class Completer: changed = False for parent_node, child_node in self._node_pairs_between_graphs: parent_node_dist_attr = self._dist_context.get_dist_attr_for_graph( - parent_node) + parent_node + ) child_node_dist_attr = self._dist_context.get_dist_attr_for_graph( - child_node) - if parent_node_dist_attr.process_mesh != child_node_dist_attr.process_mesh: + child_node + ) + if ( + parent_node_dist_attr.process_mesh + != child_node_dist_attr.process_mesh + ): continue parent_node_dims_mapping = parent_node_dist_attr.dims_mapping child_node_dims_mapping = child_node_dist_attr.dims_mapping compatible_dims_mapping = compute_compatible_dims_mapping( - [parent_node_dims_mapping, child_node_dims_mapping]) - if not _validate_dims_mapping(compatible_dims_mapping, - parent_node_dist_attr.process_mesh): + [parent_node_dims_mapping, child_node_dims_mapping] + ) + if not _validate_dims_mapping( + compatible_dims_mapping, parent_node_dist_attr.process_mesh + ): return False - if (compatible_dims_mapping is not None) \ - and (compatible_dims_mapping != parent_node_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != parent_node_dims_mapping + ): parent_node_dist_attr.dims_mapping = compatible_dims_mapping changed = True - if (compatible_dims_mapping is not None) \ - and (compatible_dims_mapping != child_node_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != child_node_dims_mapping + ): child_node_dist_attr.dims_mapping = compatible_dims_mapping changed = True return changed @@ -365,11 +440,15 @@ class Completer: op_nodes = self._dist_context._serial_ordered_op_nodes # NOTE: this list may be changed if Paddle changes the existing rules. related_reader_ops = [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ] for op_node in op_nodes: - if op_node.op() is not None \ - and op_node.op().type() in related_reader_ops: + if ( + op_node.op() is not None + and op_node.op().type() in related_reader_ops + ): continue op_dist_attr = self._dist_context.get_dist_attr_for_graph(op_node) for tensor_node in op_node.outputs: @@ -377,11 +456,18 @@ class Completer: if tensor_node.var().type() == core.VarDesc.VarType.READER: continue tensor_desc = tensor_node.var() - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): op_dims_mapping = op_dist_attr.get_output_dims_mapping( - tensor_desc.name()) + tensor_desc.name() + ) tensor_dist_attr.dims_mapping = op_dims_mapping def _update_dims_mapping(self): @@ -390,17 +476,22 @@ class Completer: while not reach_fix_point: changed = False for is_fwd in [True, False]: - all_nodes = self._dist_context.serial_ordered_nodes \ - if is_fwd else reversed(self._dist_context.serial_ordered_nodes) + all_nodes = ( + self._dist_context.serial_ordered_nodes + if is_fwd + else reversed(self._dist_context.serial_ordered_nodes) + ) for node in all_nodes: if node.is_var() and node.var() is not None: tensor_changed = self._update_tensor_node_dims_mapping( - node, fwd=is_fwd) + node, fwd=is_fwd + ) if tensor_changed: changed = True if node.is_op() and node.op() is not None: op_changed = self._update_op_node_dims_mapping( - node, fwd=is_fwd) + node, fwd=is_fwd + ) if op_changed: changed = True graph_changed = self._update_dims_mapping_between_graphs() @@ -419,12 +510,16 @@ class Completer: if not op_dist_attr.is_annotated("process_mesh"): process_mesh = op_dist_attr.process_mesh nearest_op_dis_attr = self._dist_context.get_dist_attr_for_graph( - nearest_op_node) + nearest_op_node + ) nearest_process_mesh = nearest_op_dis_attr.process_mesh compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh, nearest_process_mesh]) - if compatible_process_mesh is not None \ - and process_mesh != compatible_process_mesh: + [process_mesh, nearest_process_mesh] + ) + if ( + compatible_process_mesh is not None + and process_mesh != compatible_process_mesh + ): op_dist_attr.process_mesh = compatible_process_mesh # Skip the process_mesh setting of inputs and outputs of while_op if op_dist_attr.op_type == "while": @@ -432,43 +527,60 @@ class Completer: # Set the process mesh of the op node's leaf-inputs for tensor_node in op_node.inputs: if tensor_node.is_var() and tensor_node.var() is not None: - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) if tensor_dist_attr.is_annotated("process_mesh"): continue # Skip the non-leaf var node if len(tensor_node.inputs) != 0: continue compatible_process_mesh = compute_compatible_process_mesh( - [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh]) - if compatible_process_mesh is not None \ - and tensor_dist_attr.process_mesh != compatible_process_mesh: + [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh] + ) + if ( + compatible_process_mesh is not None + and tensor_dist_attr.process_mesh != compatible_process_mesh + ): tensor_dist_attr.process_mesh = compatible_process_mesh # Set the process mesh of the op node's outputs for tensor_node in op_node.outputs: if tensor_node.is_var() and tensor_node.var() is not None: - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) if tensor_dist_attr.is_annotated("process_mesh"): continue compatible_process_mesh = compute_compatible_process_mesh( - [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh]) - if compatible_process_mesh is not None \ - and tensor_dist_attr.process_mesh != compatible_process_mesh: + [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh] + ) + if ( + compatible_process_mesh is not None + and tensor_dist_attr.process_mesh != compatible_process_mesh + ): tensor_dist_attr.process_mesh = compatible_process_mesh def _update_process_mesh_for_specials(self): - def _find_nearest_tensor_node_before(nodes, idx, var_name): for node in reversed(nodes[:idx]): - if node.is_var() and node.var() is not None \ - and node.var().name() == var_name: + if ( + node.is_var() + and node.var() is not None + and node.var().name() == var_name + ): return node def _find_nearest_tensor_node_after(nodes, idx, var_name): - for node in nodes[idx + 1:]: - if node.is_var() and node.var() is not None \ - and node.var().name() == var_name: + for node in nodes[idx + 1 :]: + if ( + node.is_var() + and node.var() is not None + and node.var().name() == var_name + ): return node def _find_nodes_related_to_cond(source_node): @@ -486,28 +598,42 @@ class Completer: neighbors = cur.inputs + cur.outputs for node in neighbors: if node.is_var() and node.var() is not None: - if node.var().type() != core.VarDesc.VarType.READER \ - and len(node.var().shape()) == 1: + if ( + node.var().type() != core.VarDesc.VarType.READER + and len(node.var().shape()) == 1 + ): frontier.append(node) related_nodes.append(node) if node.is_op() and node.op() is not None: flag = True - if node.op().type() == "create_py_reader" \ - or node.op().type() == "create_double_buffer_reader" \ - or node.op().type() == "read": + if ( + node.op().type() == "create_py_reader" + or node.op().type() == "create_double_buffer_reader" + or node.op().type() == "read" + ): flag = False for tensor_node in node.inputs: - if tensor_node.is_var() and tensor_node.var( - ) is not None: - if tensor_node.var().type() in __not_shape_var_type__ \ - or len(tensor_node.var().shape()) != 1: + if ( + tensor_node.is_var() + and tensor_node.var() is not None + ): + if ( + tensor_node.var().type() + in __not_shape_var_type__ + or len(tensor_node.var().shape()) != 1 + ): flag = False break for tensor_node in node.outputs: - if tensor_node.is_var() and tensor_node.var( - ) is not None: - if tensor_node.var().type() in __not_shape_var_type__ \ - or len(tensor_node.var().shape()) != 1: + if ( + tensor_node.is_var() + and tensor_node.var() is not None + ): + if ( + tensor_node.var().type() + in __not_shape_var_type__ + or len(tensor_node.var().shape()) != 1 + ): flag = False break if flag: @@ -532,27 +658,32 @@ class Completer: dims_mapping = dist_attr.get_output_dims_mapping(arg_name) for _ in dims_mapping: new_dims_mapping.append(-1) - dist_attr.set_output_dims_mapping(arg_name, - new_dims_mapping) + dist_attr.set_output_dims_mapping( + arg_name, new_dims_mapping + ) # Amend the process meshes related to while_op for while_op_node, while_op_node_idx in self._while_op_nodes.values(): sub_graph_id = while_op_node.op()._block_attr_id("sub_block") sub_graph = self._dist_context.serial_graph.get_sub_graph( - sub_graph_id) + sub_graph_id + ) sub_graph_nodes = list(sub_graph.all_nodes()) while_dist_op = self._dist_context.get_dist_op_for_graph( - while_op_node) + while_op_node + ) while_op_dist_attr = while_dist_op.dist_attr # Step 1: set the process mesh of while_op to the merged process mesh of its subblock merged_process_mesh = while_op_dist_attr.process_mesh for node in sub_graph_nodes: - if (node.is_var() and node.var() is not None) \ - or (node.is_op() and node.op() is not None): + if (node.is_var() and node.var() is not None) or ( + node.is_op() and node.op() is not None + ): dist_attr = self._dist_context.get_dist_attr_for_graph(node) merged_process_mesh = merge_process_mesh_two( - merged_process_mesh, dist_attr.process_mesh) + merged_process_mesh, dist_attr.process_mesh + ) while_op_dist_attr.process_mesh = merged_process_mesh _make_dims_mapping_replicate(while_op_dist_attr) @@ -562,97 +693,143 @@ class Completer: cond_tensor_name = while_op_node.op().input("Condition")[0] cond_tensor_node = None for node in while_op_node.inputs: - if node.is_var() and node.var() is not None \ - and node.var().name() == cond_tensor_name: + if ( + node.is_var() + and node.var() is not None + and node.var().name() == cond_tensor_name + ): cond_tensor_node = node cond_tensor_related_nodes.append(cond_tensor_node) break cond_tensor_related_nodes.extend( - _find_nodes_related_to_cond(cond_tensor_node)) + _find_nodes_related_to_cond(cond_tensor_node) + ) # Step 2.2: Find related nodes of cond var in the subgraph of while_op cond_tensor_node = None for node in reversed(sub_graph_nodes): - if node.is_var() and node.var() is not None \ - and node.var().name() == cond_tensor_name \ - and len(node.outputs) == 0: + if ( + node.is_var() + and node.var() is not None + and node.var().name() == cond_tensor_name + and len(node.outputs) == 0 + ): cond_tensor_node = node break cond_tensor_related_nodes.extend( - _find_nodes_related_to_cond(cond_tensor_node)) + _find_nodes_related_to_cond(cond_tensor_node) + ) # Step 2.3: Add the StepScops output of while_op stepscopes_tensor_name = while_op_node.op().output("StepScopes")[0] stepscopes_tensor_node = None for output_node in while_op_node.outputs: - if output_node.is_var() and output_node.var() is not None \ - and output_node.var().name() == stepscopes_tensor_name: + if ( + output_node.is_var() + and output_node.var() is not None + and output_node.var().name() == stepscopes_tensor_name + ): stepscopes_tensor_node = output_node cond_tensor_related_nodes.append(stepscopes_tensor_node) # Step 2.4: Set the process meshes of all nodes related to cond var to the process mesh of while op for node in cond_tensor_related_nodes: tensor_dist_attr = self._dist_context.get_dist_attr_for_graph( - node) + node + ) tensor_dist_attr.process_mesh = merged_process_mesh _make_dims_mapping_replicate(tensor_dist_attr) # Step 3: set the process meshes of the inputs in while_op to the process meshes of the outside input nodes while_op_inputs_dist_attrs = while_op_dist_attr.inputs_dist_attrs - for tensor_name, tensor_dist_attr in while_op_inputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in while_op_inputs_dist_attrs.items(): nearest_tensor_node = _find_nearest_tensor_node_before( - self._dist_context.serial_ordered_nodes, while_op_node_idx, - tensor_name) - nearest_tensor_dist_attr = self._dist_context.get_dist_attr_for_graph( - nearest_tensor_node) - tensor_dist_attr.process_mesh = nearest_tensor_dist_attr.process_mesh + self._dist_context.serial_ordered_nodes, + while_op_node_idx, + tensor_name, + ) + nearest_tensor_dist_attr = ( + self._dist_context.get_dist_attr_for_graph( + nearest_tensor_node + ) + ) + tensor_dist_attr.process_mesh = ( + nearest_tensor_dist_attr.process_mesh + ) # Step 4: set the process meshes of the outputs in while_op to the process meshes of the outside output nodes while_op_outputs_dist_attrs = while_op_dist_attr.outputs_dist_attrs - for tensor_name, tensor_dist_attr in while_op_outputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in while_op_outputs_dist_attrs.items(): nearest_tensor_node = _find_nearest_tensor_node_before( - self._dist_context.serial_ordered_nodes, while_op_node_idx, - tensor_name) + self._dist_context.serial_ordered_nodes, + while_op_node_idx, + tensor_name, + ) if nearest_tensor_node is None: nearest_tensor_node = _find_nearest_tensor_node_after( self._dist_context.serial_ordered_nodes, - while_op_node_idx, tensor_name) - nearest_tensor_dist_attr = self._dist_context.get_dist_attr_for_graph( - nearest_tensor_node) - tensor_dist_attr.process_mesh = nearest_tensor_dist_attr.process_mesh + while_op_node_idx, + tensor_name, + ) + nearest_tensor_dist_attr = ( + self._dist_context.get_dist_attr_for_graph( + nearest_tensor_node + ) + ) + tensor_dist_attr.process_mesh = ( + nearest_tensor_dist_attr.process_mesh + ) # Amend the process meshes related to array for array_node_list in self._array_nodes.values(): merged_process_mesh = None for array_node in array_node_list: dist_attr = self._dist_context.get_dist_attr_for_graph( - array_node) + array_node + ) merged_process_mesh = merge_process_mesh_two( - merged_process_mesh, dist_attr.process_mesh) + merged_process_mesh, dist_attr.process_mesh + ) for array_node in array_node_list: dist_attr = self._dist_context.get_dist_attr_for_graph( - array_node) + array_node + ) dist_attr.process_mesh = merged_process_mesh _make_dims_mapping_replicate(dist_attr) def _update_process_mesh_between_graphs(self): for parent_node, child_node in self._node_pairs_between_graphs: parent_node_dist_attr = self._dist_context.get_dist_attr_for_graph( - parent_node) + parent_node + ) child_node_dist_attr = self._dist_context.get_dist_attr_for_graph( - child_node) - parent_node_dist_attr.process_mesh = child_node_dist_attr.process_mesh - compatible_process_mesh = compute_compatible_process_mesh([ - parent_node_dist_attr.process_mesh, + child_node + ) + parent_node_dist_attr.process_mesh = ( child_node_dist_attr.process_mesh - ]) - if compatible_process_mesh is not None \ - and parent_node_dist_attr.process_mesh != compatible_process_mesh: + ) + compatible_process_mesh = compute_compatible_process_mesh( + [ + parent_node_dist_attr.process_mesh, + child_node_dist_attr.process_mesh, + ] + ) + if ( + compatible_process_mesh is not None + and parent_node_dist_attr.process_mesh + != compatible_process_mesh + ): parent_node_dist_attr.process_mesh = compatible_process_mesh - if compatible_process_mesh is not None \ - and child_node_dist_attr.process_mesh != compatible_process_mesh: + if ( + compatible_process_mesh is not None + and child_node_dist_attr.process_mesh != compatible_process_mesh + ): child_node_dist_attr.process_mesh = compatible_process_mesh def _update_process_mesh(self): @@ -661,8 +838,9 @@ class Completer: # Step 1: Set the annotated process meshes from tensors to the first ops using them ordered_tensor_nodes = self._dist_context._serial_ordered_tensor_nodes for tensor_node in ordered_tensor_nodes: - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph(tensor_node) + ) if not tensor_dist_attr.is_annotated("process_mesh"): continue first_op_node = None @@ -680,13 +858,18 @@ class Completer: if first_op_node is None: continue op_dist_attr = self._dist_context.get_dist_attr_for_graph( - first_op_node) + first_op_node + ) if op_dist_attr is not None and not op_dist_attr.is_annotated( - "process_mesh"): + "process_mesh" + ): compatible_process_mesh = compute_compatible_process_mesh( - [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh]) - if compatible_process_mesh is not None \ - and op_dist_attr.process_mesh != compatible_process_mesh: + [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh] + ) + if ( + compatible_process_mesh is not None + and op_dist_attr.process_mesh != compatible_process_mesh + ): op_dist_attr.process_mesh = compatible_process_mesh # Step 2: set the process meshes of ops with the nearest op before them @@ -694,8 +877,10 @@ class Completer: idx_of_first_op_node_has_process_mesh = -1 for idx, op_node in enumerate(ordered_op_nodes): op_dist_attr = self._dist_context.get_dist_attr_for_graph(op_node) - if op_dist_attr.process_mesh is not None \ - and idx_of_first_op_node_has_process_mesh == -1: + if ( + op_dist_attr.process_mesh is not None + and idx_of_first_op_node_has_process_mesh == -1 + ): idx_of_first_op_node_has_process_mesh = idx # Reuse the following method to set the related tensors for same op node self._update_process_mesh_by_nearest(op_node, op_node) @@ -703,17 +888,20 @@ class Completer: if idx_of_first_op_node_has_process_mesh + 1 > len(ordered_op_nodes): return None for idx, op_node in enumerate( - ordered_op_nodes[idx_of_first_op_node_has_process_mesh + 1:]): + ordered_op_nodes[idx_of_first_op_node_has_process_mesh + 1 :] + ): original_idx = idx_of_first_op_node_has_process_mesh + idx + 1 nearest_op_node = ordered_op_nodes[original_idx - 1] nearest_op_dist_attr = self._dist_context.get_dist_attr_for_graph( - nearest_op_node) + nearest_op_node + ) op_dist_attr = self._dist_context.get_dist_attr_for_graph(op_node) assert nearest_op_dist_attr.process_mesh is not None self._update_process_mesh_by_nearest(op_node, nearest_op_node) # Step 2.3: set the process meshes of ops by the nearest op node before the first op node nearest_op_node = ordered_op_nodes[ - idx_of_first_op_node_has_process_mesh] + idx_of_first_op_node_has_process_mesh + ] for op_node in ordered_op_nodes[:idx_of_first_op_node_has_process_mesh]: self._update_process_mesh_by_nearest(op_node, nearest_op_node) @@ -750,21 +938,31 @@ class Completer: if node.is_var() and node.var() is not None: if node.node.graph_id() != 0: for before_node in reversed(all_nodes[:idx]): - if before_node.is_var() and before_node.var() is not None \ - and before_node.node.graph_id() == node.node.graph_id() - 1 \ - and before_node.var().name() == node.var().name(): + if ( + before_node.is_var() + and before_node.var() is not None + and before_node.node.graph_id() + == node.node.graph_id() - 1 + and before_node.var().name() == node.var().name() + ): self._node_pairs_between_graphs.append( - (before_node, node)) - for after_node in all_nodes[idx + 1:]: - if after_node.is_var() and after_node.var() is not None \ - and after_node.node.graph_id() == node.node.graph_id() - 1 \ - and after_node.var().name() == node.var().name(): + (before_node, node) + ) + for after_node in all_nodes[idx + 1 :]: + if ( + after_node.is_var() + and after_node.var() is not None + and after_node.node.graph_id() + == node.node.graph_id() - 1 + and after_node.var().name() == node.var().name() + ): self._node_pairs_between_graphs.append( - (after_node, node)) + (after_node, node) + ) self._has_prepared = True def complete_forward_annotation(self, serial_main_program=None): - """ Complete annotation for the partial annotated serial_main_program. + """Complete annotation for the partial annotated serial_main_program. Arguments: serial_main_program: partial annotated serial_main_program. Returns:e @@ -818,8 +1016,9 @@ class Completer: # TODO: we must ensure the world process group contains all ranks ranks = get_world_process_group().ranks process_mesh = ProcessMesh(ranks) - for dist_tensor in self._dist_context._dist_tensors_for_program.values( - ): + for ( + dist_tensor + ) in self._dist_context._dist_tensors_for_program.values(): serial_tensor = dist_tensor.serial_tensor tensor_dist_attr = dist_tensor.dist_attr tensor_dist_attr.process_mesh = process_mesh @@ -838,27 +1037,35 @@ class Completer: if not serial_tensor.is_parameter: if arg_name not in input_xshape_arg_names: old_dims_mapping = op_dist_attr.get_input_dims_mapping( - arg_name) + arg_name + ) if len(old_dims_mapping) > 0: new_dims_mapping = [0] + [ -1 for _ in range(len(old_dims_mapping) - 1) ] op_dist_attr.set_input_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) else: old_dims_mapping = op_dist_attr.get_input_dims_mapping( - arg_name) + arg_name + ) if len(old_dims_mapping) > 1: new_dims_mapping = [-1, 0] + [ -1 for _ in range(len(old_dims_mapping) - 2) ] op_dist_attr.set_input_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) # Set tensor's dims_mapping by the op's - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - serial_tensor) - tensor_dist_attr.dims_mapping = op_dist_attr.get_input_dims_mapping( - arg_name) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + serial_tensor + ) + ) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_input_dims_mapping(arg_name) + ) output_xshape_arg_names = [] if "XShape" in op_desc.output_names(): output_xshape_arg_names = op_desc.output("XShape") @@ -867,37 +1074,48 @@ class Completer: if not serial_tensor.is_parameter: if arg_name not in output_xshape_arg_names: old_dims_mapping = op_dist_attr.get_output_dims_mapping( - arg_name) + arg_name + ) if len(old_dims_mapping) > 0: new_dims_mapping = [0] + [ -1 for _ in range(len(old_dims_mapping) - 1) ] op_dist_attr.set_output_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) else: old_dims_mapping = op_dist_attr.get_output_dims_mapping( - arg_name) + arg_name + ) if len(old_dims_mapping) > 1: new_dims_mapping = [-1, 0] + [ -1 for _ in range(len(old_dims_mapping) - 2) ] op_dist_attr.set_output_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) # Set tensor's dims_mapping by the op's - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - serial_tensor) - tensor_dist_attr.dims_mapping = op_dist_attr.get_output_dims_mapping( - arg_name) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + serial_tensor + ) + ) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_output_dims_mapping(arg_name) + ) op_dist_impls = find_compatible_distributed_operator_impls( - dist_op, partial=False) + dist_op, partial=False + ) if op_dist_impls is not None: not_compatible = True backup_op_dist_attr = copy.deepcopy(op_dist_attr) for op_dist_impl in op_dist_impls: op_dist_impl.update_dims_mapping(dist_op) - if op_dist_impl.is_auto_compatible(dist_op) \ - and dist_op.validate_dist_attr(): + if ( + op_dist_impl.is_auto_compatible(dist_op) + and dist_op.validate_dist_attr() + ): op_dist_attr.impl_type = op_dist_impl.type op_dist_attr.impl_idx = op_dist_impl.idx not_compatible = False @@ -939,24 +1157,36 @@ class Completer: # Use the first op to set the tensor dist attr if tensor_name in has_set_dist_attr: continue - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - tensor_dist_attr.process_mesh = op_dist_attr.process_mesh - tensor_dist_attr.dims_mapping = op_dist_attr.get_input_dims_mapping( - tensor_name) if tensor.is_parameter else [ - -1 for i in tensor_desc.shape() - ] + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + tensor_dist_attr.process_mesh = ( + op_dist_attr.process_mesh + ) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_input_dims_mapping(tensor_name) + if tensor.is_parameter + else [-1 for i in tensor_desc.shape()] + ) has_set_dist_attr.add(tensor_name) for tensor_node in node.outputs: if tensor_node.is_var() and tensor_node.var() is not None: tensor_name = tensor_node.var().name() if tensor_name in has_set_dist_attr: continue - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - tensor_dist_attr.process_mesh = op_dist_attr.process_mesh - tensor_dist_attr.dims_mapping = op_dist_attr.get_output_dims_mapping( - tensor_name) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + tensor_dist_attr.process_mesh = ( + op_dist_attr.process_mesh + ) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_output_dims_mapping(tensor_name) + ) has_set_dist_attr.add(tensor_name) self._update_process_mesh_for_specials() @@ -1007,79 +1237,108 @@ class Completer: for idx in range(0, len(ops)): op = ops[idx] if int(op.attr('op_role')) == int( - core.op_proto_and_checker_maker.OpRole.Forward): + core.op_proto_and_checker_maker.OpRole.Forward + ): continue if int(op.attr('op_role')) == int( - core.op_proto_and_checker_maker.OpRole.Backward) and int( - ops[idx - 1].attr('op_role')) == int( - core.op_proto_and_checker_maker.OpRole.Forward): + core.op_proto_and_checker_maker.OpRole.Backward + ) and int(ops[idx - 1].attr('op_role')) == int( + core.op_proto_and_checker_maker.OpRole.Forward + ): appended_grad_times += 1 if int(op.attr('op_role')) == int( - int(core.op_proto_and_checker_maker.OpRole.Backward) - | int(core.op_proto_and_checker_maker.OpRole.Loss)): + int(core.op_proto_and_checker_maker.OpRole.Backward) + | int(core.op_proto_and_checker_maker.OpRole.Loss) + ): assert op.type == "fill_constant" break # complete the annotation of grad op (xxx_grad op or sum op) # xxx_grad op will have a corresponding forward op in grad_op_id_to_op_id grad_op = ops[idx] - if grad_op.desc.original_id( - ) in dist_op_context.grad_op_id_to_op_id: + if ( + grad_op.desc.original_id() + in dist_op_context.grad_op_id_to_op_id + ): # TODO support the case where one forward op corresponding to multiple xxx_grad op forward_op = _get_op_by_id( - ops, dist_op_context.grad_op_id_to_op_id[ - grad_op.desc.original_id()]) + ops, + dist_op_context.grad_op_id_to_op_id[ + grad_op.desc.original_id() + ], + ) assert forward_op is not None - fwd_op_dist_attr = self._dist_context.get_op_dist_attr_for_program( - forward_op) + fwd_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program(forward_op) + ) fwd_op_process_mesh = fwd_op_dist_attr.process_mesh grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = fwd_op_process_mesh for input_name in grad_op.input_arg_names: - if input_name not in forward_op.input_arg_names and input_name not in forward_op.output_arg_names: + if ( + input_name not in forward_op.input_arg_names + and input_name not in forward_op.output_arg_names + ): if input_name in grad_var_to_var[appended_grad_times]: fwd_name = grad_var_to_var[appended_grad_times][ - input_name] - ref_dims_mapping = fwd_op_dist_attr.get_output_dims_mapping( - fwd_name) + input_name + ] + ref_dims_mapping = ( + fwd_op_dist_attr.get_output_dims_mapping( + fwd_name + ) + ) else: input_var = vars[input_name] ref_dims_mapping = self._dist_context.get_tensor_dist_attr_for_program( - input_var).dims_mapping + input_var + ).dims_mapping else: if fwd_op_dist_attr.get_input_dims_mapping(input_name): - ref_dims_mapping = fwd_op_dist_attr.get_input_dims_mapping( - input_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_input_dims_mapping( + input_name + ) + ) else: - ref_dims_mapping = fwd_op_dist_attr.get_output_dims_mapping( - input_name) - assert ref_dims_mapping is not None, "[{}] 's dims mapping is NONE".format( - input_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_output_dims_mapping( + input_name + ) + ) + assert ( + ref_dims_mapping is not None + ), "[{}] 's dims mapping is NONE".format(input_name) grad_op_dist_attr.set_input_dims_mapping( - input_name, ref_dims_mapping) + input_name, ref_dims_mapping + ) for output_name in grad_op.output_arg_names: assert output_name in grad_var_to_var[appended_grad_times] fwd_name = grad_var_to_var[appended_grad_times][output_name] ref_dims_mapping = fwd_op_dist_attr.get_input_dims_mapping( - fwd_name) + fwd_name + ) # var output_var = vars[output_name] tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.dims_mapping = ref_dims_mapping tensor_dist_attr.process_mesh = fwd_op_process_mesh self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr.set_output_dims_mapping( - output_name, ref_dims_mapping) + output_name, ref_dims_mapping + ) self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) # grad ops that have not a corresponding mapping in grad_op_id_to_op_id else: @@ -1087,14 +1346,20 @@ class Completer: if grad_op.type == 'sum': assert all(map(_is_grad_var_name, grad_op.input_arg_names)) output_name = grad_op.output_arg_names[0] - assert output_name in grad_var_to_var[appended_grad_times], \ - "sum op's output '{}' has no corresponding var".format( - output_name) + assert ( + output_name in grad_var_to_var[appended_grad_times] + ), "sum op's output '{}' has no corresponding var".format( + output_name + ) ref_fwd_var_name = grad_var_to_var[appended_grad_times][ - output_name] + output_name + ] ref_fwd_var = vars[ref_fwd_var_name] - ref_fwd_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_fwd_var) + ref_fwd_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + ref_fwd_var + ) + ) ref_fwd_dims_mapping = ref_fwd_dist_attr.dims_mapping ref_fwd_process_mesh = ref_fwd_dist_attr.process_mesh # output @@ -1103,21 +1368,27 @@ class Completer: tensor_dist_attr.process_mesh = ref_fwd_process_mesh output_var = vars[output_name] self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = ref_fwd_process_mesh for var_name in grad_op.input_arg_names: grad_op_dist_attr.set_input_dims_mapping( - var_name, ref_fwd_dims_mapping) + var_name, ref_fwd_dims_mapping + ) grad_op_dist_attr.set_output_dims_mapping( - output_name, ref_fwd_dims_mapping) + output_name, ref_fwd_dims_mapping + ) elif grad_op.type == 'fill_any_like': ref_var_name = grad_op.input_arg_names[0] ref_var = vars[ref_var_name] - ref_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_var) + ref_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + ref_var + ) + ) ref_dims_mapping = ref_dist_attr.dims_mapping ref_process_mesh = ref_dist_attr.process_mesh # output @@ -1127,24 +1398,29 @@ class Completer: output_var_name = grad_op.output_arg_names[0] output_var = vars[output_var_name] self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = ref_process_mesh grad_op_dist_attr.set_input_dims_mapping( - ref_var_name, ref_dims_mapping) + ref_var_name, ref_dims_mapping + ) grad_op_dist_attr.set_output_dims_mapping( - output_var_name, ref_dims_mapping) + output_var_name, ref_dims_mapping + ) elif grad_op.type in ['shape', 'fill_constant']: continue else: - raise ValueError("got unexpect op [{}]".format( - str(grad_op.type))) + raise ValueError( + "got unexpect op [{}]".format(str(grad_op.type)) + ) self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) def complete_backward_annotation(self, serial_main_program=None): """Complete the annotation of vars and ops in the backward phase for parallel program.""" @@ -1161,9 +1437,9 @@ class Completer: def _get_forward_varname_from_grad_varname(grad_var_name): assert _is_grad_var_name( - grad_var_name), "[{}] is not a grad varnme.".format( - grad_var_name) - return grad_var_name[:grad_var_name.find("@GRAD")] + grad_var_name + ), "[{}] is not a grad varnme.".format(grad_var_name) + return grad_var_name[: grad_var_name.find("@GRAD")] def _get_op_by_id(ops, id): for op in ops: @@ -1174,160 +1450,216 @@ class Completer: first_backward_op_idx = -1 for idx, op in enumerate(serial_main_program.global_block().ops): if int(op.attr('op_role')) == int( - int(core.op_proto_and_checker_maker.OpRole.Backward) - | int(core.op_proto_and_checker_maker.OpRole.Loss)): + int(core.op_proto_and_checker_maker.OpRole.Backward) + | int(core.op_proto_and_checker_maker.OpRole.Loss) + ): assert op.type == "fill_constant" first_backward_op_idx = idx break - assert first_backward_op_idx >= 0, "No backward procedure found in this program." + assert ( + first_backward_op_idx >= 0 + ), "No backward procedure found in this program." ops = list(serial_main_program.global_block().ops) vars = serial_main_program.global_block().vars dist_op_context = self._dist_context.dist_op_context - grad_var_to_var = dist_op_context.grad_var_to_var[len( - dist_op_context.grad_var_to_var)] + grad_var_to_var = dist_op_context.grad_var_to_var[ + len(dist_op_context.grad_var_to_var) + ] for idx in range(first_backward_op_idx, len(ops)): # complete the initial grad loss op if idx == first_backward_op_idx: assert ops[idx].type == "fill_constant" - assert len( - ops[idx].input_arg_names - ) == 0, "first backward op should has only ONE output, but got [{}]".format( - len(ops[idx].input_arg_names)) - assert len( - ops[idx].output_arg_names - ) == 1, "first backward op should has only ONE output, but got [{}]".format( - len(ops[idx].output_arg_names)) + assert ( + len(ops[idx].input_arg_names) == 0 + ), "first backward op should has only ONE output, but got [{}]".format( + len(ops[idx].input_arg_names) + ) + assert ( + len(ops[idx].output_arg_names) == 1 + ), "first backward op should has only ONE output, but got [{}]".format( + len(ops[idx].output_arg_names) + ) grad_var = vars[ops[idx].output_arg_names[0]] forward_var_name = _get_forward_varname_from_grad_varname( - grad_var.name) + grad_var.name + ) forward_var = vars[forward_var_name] # TODO complete other attribte for grad var tensor_dist_attr = TensorDistributedAttribute() - process_mesh = self._dist_context.get_tensor_dist_attr_for_program( - forward_var).process_mesh - dims_mapping = self._dist_context.get_tensor_dist_attr_for_program( - forward_var).dims_mapping + process_mesh = ( + self._dist_context.get_tensor_dist_attr_for_program( + forward_var + ).process_mesh + ) + dims_mapping = ( + self._dist_context.get_tensor_dist_attr_for_program( + forward_var + ).dims_mapping + ) tensor_dist_attr.dims_mapping = dims_mapping tensor_dist_attr.process_mesh = process_mesh self._dist_context.set_tensor_dist_attr_for_program( - grad_var, tensor_dist_attr) + grad_var, tensor_dist_attr + ) op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = process_mesh - op_dist_attr.set_output_dims_mapping(grad_var.name, - dims_mapping) + op_dist_attr.set_output_dims_mapping( + grad_var.name, dims_mapping + ) self._dist_context.set_op_dist_attr_for_program( - ops[idx], op_dist_attr) + ops[idx], op_dist_attr + ) continue # complete the annotation of grad op (xxx_grad op or sum op) # xxx_grad op will have a corresponding forward op in grad_op_id_to_op_id grad_op = ops[idx] - if grad_op.desc.original_id( - ) in dist_op_context.grad_op_id_to_op_id: + if ( + grad_op.desc.original_id() + in dist_op_context.grad_op_id_to_op_id + ): # TODO support the case where one forward op corresponding to multiple xxx_grad op forward_op = _get_op_by_id( ops[:first_backward_op_idx], dist_op_context.grad_op_id_to_op_id[ - grad_op.desc.original_id()]) + grad_op.desc.original_id() + ], + ) assert forward_op is not None if grad_op.type == "concat" and forward_op.type == "split": - forward_op_dist_attr = self._dist_context.get_op_dist_attr_for_program( - forward_op) + forward_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program( + forward_op + ) + ) output_var = vars[grad_op.desc.output('Out')[0]] split_input_var_name = forward_op.input("X")[0] - ref_dims_mapping = forward_op_dist_attr.get_input_dims_mapping( - split_input_var_name) + ref_dims_mapping = ( + forward_op_dist_attr.get_input_dims_mapping( + split_input_var_name + ) + ) ref_mesh = forward_op_dist_attr.process_mesh grad_op_dist_attr = OperatorDistributedAttribute() for input_name in grad_op.input_arg_names: grad_op_dist_attr.set_input_dims_mapping( - input_name, ref_dims_mapping) + input_name, ref_dims_mapping + ) output_var_dist_attr = TensorDistributedAttribute() output_var_dist_attr.dims_mapping = ref_dims_mapping output_var_dist_attr.process_mesh = ref_mesh self._dist_context.set_tensor_dist_attr_for_program( - output_var, output_var_dist_attr) + output_var, output_var_dist_attr + ) grad_op_dist_attr.set_output_dims_mapping( - output_var.name, ref_dims_mapping) + output_var.name, ref_dims_mapping + ) grad_op_dist_attr.process_mesh = ref_mesh self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) grad_op_dist_attr.impl_type = fwd_op_dist_attr.impl_type grad_op_dist_attr.impl_idx = fwd_op_dist_attr.impl_idx continue - fwd_op_dist_attr = self._dist_context.get_op_dist_attr_for_program( - forward_op) + fwd_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program(forward_op) + ) fwd_op_process_mesh = fwd_op_dist_attr.process_mesh grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = fwd_op_process_mesh for input_name in grad_op.input_arg_names: - if input_name not in forward_op.input_arg_names and input_name not in forward_op.output_arg_names: + if ( + input_name not in forward_op.input_arg_names + and input_name not in forward_op.output_arg_names + ): if input_name in grad_var_to_var: fwd_name = grad_var_to_var[input_name] - ref_dims_mapping = fwd_op_dist_attr.get_output_dims_mapping( - fwd_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_output_dims_mapping( + fwd_name + ) + ) else: input_var = vars[input_name] ref_dims_mapping = self._dist_context.get_tensor_dist_attr_for_program( - input_var).dims_mapping + input_var + ).dims_mapping else: if fwd_op_dist_attr.get_input_dims_mapping(input_name): - ref_dims_mapping = fwd_op_dist_attr.get_input_dims_mapping( - input_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_input_dims_mapping( + input_name + ) + ) else: - ref_dims_mapping = fwd_op_dist_attr.get_output_dims_mapping( - input_name) - assert ref_dims_mapping is not None, "[{}] 's dims mapping is NONE".format( - input_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_output_dims_mapping( + input_name + ) + ) + assert ( + ref_dims_mapping is not None + ), "[{}] 's dims mapping is NONE".format(input_name) grad_op_dist_attr.set_input_dims_mapping( - input_name, ref_dims_mapping) + input_name, ref_dims_mapping + ) for output_name in grad_op.output_arg_names: assert output_name in grad_var_to_var fwd_name = grad_var_to_var[output_name] ref_dims_mapping = fwd_op_dist_attr.get_input_dims_mapping( - fwd_name) + fwd_name + ) # var output_var = vars[output_name] tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.dims_mapping = ref_dims_mapping tensor_dist_attr.process_mesh = fwd_op_process_mesh self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr.set_output_dims_mapping( - output_name, ref_dims_mapping) + output_name, ref_dims_mapping + ) grad_op_dist_attr.impl_type = fwd_op_dist_attr.impl_type grad_op_dist_attr.impl_idx = fwd_op_dist_attr.impl_idx self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) # grad ops that have not a corresponding mapping in grad_op_id_to_op_id else: if grad_op.type == 'sum': assert all(map(_is_grad_var_name, grad_op.input_arg_names)) output_name = grad_op.output_arg_names[0] - assert output_name in grad_var_to_var, "sum op's output '{}' has no corresponding var".format( - output_name) + assert ( + output_name in grad_var_to_var + ), "sum op's output '{}' has no corresponding var".format( + output_name + ) ref_fwd_var_name = grad_var_to_var[output_name] ref_fwd_var = vars[ref_fwd_var_name] - ref_fwd_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_fwd_var) + ref_fwd_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + ref_fwd_var + ) + ) ref_fwd_dims_mapping = ref_fwd_dist_attr.dims_mapping ref_fwd_process_mesh = ref_fwd_dist_attr.process_mesh @@ -1337,24 +1669,30 @@ class Completer: tensor_dist_attr.process_mesh = ref_fwd_process_mesh output_var = vars[output_name] self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = ref_fwd_process_mesh for var_name in grad_op.input_arg_names: grad_op_dist_attr.set_input_dims_mapping( - var_name, ref_fwd_dims_mapping) + var_name, ref_fwd_dims_mapping + ) grad_op_dist_attr.set_output_dims_mapping( - output_name, ref_fwd_dims_mapping) + output_name, ref_fwd_dims_mapping + ) grad_op_dist_attr.impl_type = "default" grad_op_dist_attr.impl_idx = 0 elif grad_op.type == 'fill_any_like': ref_var_name = grad_op.input_arg_names[0] ref_var = vars[ref_var_name] - ref_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_var) + ref_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + ref_var + ) + ) ref_dims_mapping = ref_dist_attr.dims_mapping ref_process_mesh = ref_dist_attr.process_mesh # output @@ -1364,27 +1702,35 @@ class Completer: output_var_name = grad_op.output_arg_names[0] output_var = vars[output_var_name] self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = ref_process_mesh grad_op_dist_attr.set_input_dims_mapping( - ref_var_name, ref_dims_mapping) + ref_var_name, ref_dims_mapping + ) grad_op_dist_attr.set_output_dims_mapping( - output_var_name, ref_dims_mapping) + output_var_name, ref_dims_mapping + ) else: - raise ValueError("got unexpect op [{}]".format( - str(grad_op.type))) + raise ValueError( + "got unexpect op [{}]".format(str(grad_op.type)) + ) self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) def complete_update_annotation(self, serial_main_program): """Complete the annotation of vars and ops in the update phase for parallel program.""" # Copy the dist tensors and dist ops annotated by users from the default context # global mesh - from paddle.distributed.auto_parallel.process_group import get_world_process_group + from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, + ) + world_ranks = get_world_process_group().ranks # Notice: serial_main_program is actually a dist_main_program of current rank, @@ -1403,17 +1749,22 @@ class Completer: if int(op.attr('op_role')) == int(OpRole.Optimize): if is_gradient_clip_op(op): if op.type in [ - "sum", "sqrt", "fill_constant", "elementwise_max", - "elementwise_div" + "sum", + "sqrt", + "fill_constant", + "elementwise_max", + "elementwise_div", ]: op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = world_ranks for in_name in op.input_arg_names: in_var = vars[in_name] in_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - in_var) + in_var + ) op_dist_attr.set_input_dist_attr( - in_name, in_dist_attr) + in_name, in_dist_attr + ) for out_name in op.output_arg_names: out_var = vars[out_name] out_dist_attr = TensorDistributedAttribute() @@ -1422,22 +1773,30 @@ class Completer: -1 for _ in range(len(out_var.shape)) ] self._dist_context.set_tensor_dist_attr_for_program( - out_var, out_dist_attr) + out_var, out_dist_attr + ) op_dist_attr.set_output_dist_attr( - out_name, out_dist_attr) + out_name, out_dist_attr + ) else: in_var = vars[op.input("X")[0]] - in_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - in_var) + in_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + in_var + ) + ) assert in_dist_attr is not None ref_process_mesh = in_dist_attr.process_mesh ref_dims_mapping = in_dist_attr.dims_mapping - if op.type == "cast" and \ - ops[idx + 1].type == "elementwise_mul": + if ( + op.type == "cast" + and ops[idx + 1].type == "elementwise_mul" + ): ref_var = vars[ops[idx + 1].input("X")[0]] ref_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_var) + ref_var + ) assert ref_dist_attr is not None ref_process_mesh = ref_dist_attr.process_mesh @@ -1447,51 +1806,72 @@ class Completer: if out_var.shape == in_var.shape: out_dist_attr.dims_mapping = ref_dims_mapping else: - assert len( - out_var.shape) == 1 and out_var.shape[0] == 1 + assert ( + len(out_var.shape) == 1 + and out_var.shape[0] == 1 + ) out_dist_attr.dims_mapping = [-1] self._dist_context.set_tensor_dist_attr_for_program( - out_var, out_dist_attr) + out_var, out_dist_attr + ) op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = ref_process_mesh op_dist_attr.set_input_dist_attr( - in_var.name, in_dist_attr) + in_var.name, in_dist_attr + ) op_dist_attr.set_output_dist_attr( - out_var.name, out_dist_attr) + out_var.name, out_dist_attr + ) self._dist_context.set_op_dist_attr_for_program( - op, op_dist_attr) + op, op_dist_attr + ) if "Grad" in op.input_names and "Param" in ops[idx].input_names: - assert len( - op.input("Param")) == 1, "Only support one-to-one now." - assert len( - op.input("Grad")) == 1, "Only support one-to-one now." + assert ( + len(op.input("Param")) == 1 + ), "Only support one-to-one now." + assert ( + len(op.input("Grad")) == 1 + ), "Only support one-to-one now." param = vars[op.input("Param")[0]] grad_var = vars[op.input("Grad")[0]] - param_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - param) + param_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + param + ) + ) assert param_dist_attr is not None - ref_process_mesh = self._dist_context.get_tensor_dist_attr_for_program( - param).process_mesh + ref_process_mesh = ( + self._dist_context.get_tensor_dist_attr_for_program( + param + ).process_mesh + ) assert ref_process_mesh is not None - ref_dims_mapping = self._dist_context.get_tensor_dist_attr_for_program( - param).dims_mapping + ref_dims_mapping = ( + self._dist_context.get_tensor_dist_attr_for_program( + param + ).dims_mapping + ) assert ref_dims_mapping is not None op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = ref_process_mesh - op_dist_attr.set_input_dims_mapping(grad_var.name, - ref_dims_mapping) - op_dist_attr.set_input_dims_mapping(param.name, - ref_dims_mapping) + op_dist_attr.set_input_dims_mapping( + grad_var.name, ref_dims_mapping + ) + op_dist_attr.set_input_dims_mapping( + param.name, ref_dims_mapping + ) op_dist_attr.set_output_dims_mapping( - param.name, ref_dims_mapping) + param.name, ref_dims_mapping + ) learning_var = vars[op.input("LearningRate")[0]] op_dist_attr.set_input_dims_mapping(learning_var.name, [-1]) op_dist_attr.set_output_dims_mapping( - learning_var.name, [-1]) + learning_var.name, [-1] + ) if not learning_rate_completed: learning_rate_completed = True @@ -1499,18 +1879,19 @@ class Completer: var_dist_attr.process_mesh = world_ranks var_dist_attr.dims_mapping = [-1] self._dist_context.set_tensor_dist_attr_for_program( - learning_var, var_dist_attr) + learning_var, var_dist_attr + ) for input_name in op.desc.input_names(): if input_name in [ - 'Param', - 'Grad', - 'LearningRate', - "SkipUpdate", - "Beta1Tensor", - "Beta2Tensor", - "EpsilonTensor", + 'Param', + 'Grad', + 'LearningRate', + "SkipUpdate", + "Beta1Tensor", + "Beta2Tensor", + "EpsilonTensor", ]: continue if len(op.desc.input(input_name)) == 0: @@ -1523,22 +1904,28 @@ class Completer: if "Beta1Pow" in input_name or "Beta2Pow" in input_name: input_var_attr.dims_mapping = [-1] op_dist_attr.set_input_dims_mapping( - input_var.name, [-1]) + input_var.name, [-1] + ) op_dist_attr.set_output_dims_mapping( - input_var.name, [-1]) + input_var.name, [-1] + ) else: input_var_attr.dims_mapping = ref_dims_mapping op_dist_attr.set_input_dims_mapping( - input_var.name, ref_dims_mapping) + input_var.name, ref_dims_mapping + ) op_dist_attr.set_output_dims_mapping( - input_var.name, ref_dims_mapping) + input_var.name, ref_dims_mapping + ) input_var_attr.process_mesh = ref_process_mesh self._dist_context.set_tensor_dist_attr_for_program( - input_var, input_var_attr) + input_var, input_var_attr + ) self._dist_context.set_op_dist_attr_for_program( - op, op_dist_attr) + op, op_dist_attr + ) continue def complete_prim_annotation(self, serial_main_program=None): @@ -1574,14 +1961,18 @@ class Completer: def _init_global_mesh_for_program(self): # Copy the dist tensors and dist ops annotated by users from the default context # global mesh - from paddle.distributed.auto_parallel.process_group import get_world_process_group + from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, + ) + world_ranks = get_world_process_group().ranks for block in self._dist_context._serial_main_program.blocks: for tensor in block.vars.values(): # Copy the distributed tensors in the default context dist_tensor = self._dist_context.get_dist_tensor_for_program( - tensor) + tensor + ) assert dist_tensor is not None dist_tensor.dist_attr.process_mesh = world_ranks for op in block.ops: @@ -1592,7 +1983,8 @@ class Completer: # Find the most compatible implemenetations from the distributed operator op_dist_impls = find_compatible_distributed_operator_impls( - dist_op, fwd=True) + dist_op, fwd=True + ) if op_dist_impls is not None: backup_op_dist_attr = copy.deepcopy(dist_op.dist_attr) for op_dist_impl in op_dist_impls: diff --git a/python/paddle/distributed/auto_parallel/converter.py b/python/paddle/distributed/auto_parallel/converter.py index 21cc2a22368..d0fae414b19 100644 --- a/python/paddle/distributed/auto_parallel/converter.py +++ b/python/paddle/distributed/auto_parallel/converter.py @@ -45,30 +45,42 @@ class Converter(object): def _check_tensor_dict(self, tensors_dict): if not tensors_dict: - raise ValueError("'tensors_dict' is None, " - "the tensors to be converted cannot be None.") + raise ValueError( + "'tensors_dict' is None, " + "the tensors to be converted cannot be None." + ) if not isinstance(tensors_dict, dict): raise TypeError( - "The type of 'tensors_dict' should be 'dict', but got '{}'.". - format(str(type(tensors_dict)))) + "The type of 'tensors_dict' should be 'dict', but got '{}'.".format( + str(type(tensors_dict)) + ) + ) return tensors_dict def _check_pre_strategy(self, pre_strategy): if not pre_strategy: - raise ValueError("'pre_strategy' is None, " - "there are not tensors in pre process.") + raise ValueError( + "'pre_strategy' is None, " + "there are not tensors in pre process." + ) if not isinstance(pre_strategy, dict): - raise TypeError("The type of 'pre_strategy' should be 'dict', " - "but got '{}'.".format(str(type(pre_strategy)))) + raise TypeError( + "The type of 'pre_strategy' should be 'dict', " + "but got '{}'.".format(str(type(pre_strategy))) + ) return pre_strategy def _check_cur_strategy(self, cur_strategy): if not cur_strategy: - warnings.warn("'cur_strategy' is None, " - "there are not tensors in cur process") + warnings.warn( + "'cur_strategy' is None, " + "there are not tensors in cur process" + ) if not isinstance(cur_strategy, dict): - raise TypeError("The type of 'cur_strategy' should be 'dict', " - "but got '{}'.".format(str(type(cur_strategy)))) + raise TypeError( + "The type of 'cur_strategy' should be 'dict', " + "but got '{}'.".format(str(type(cur_strategy))) + ) return cur_strategy def convert(self, strict=True): @@ -131,42 +143,60 @@ class Converter(object): cur_dist_attr = self._cur_strategy[tensor_name] try: tensors_dict[tensor_name] = Converter.merge_and_slice( - tensor_list, pre_dist_attr, cur_dist_attr) + tensor_list, pre_dist_attr, cur_dist_attr + ) except ValueError as err: raise ValueError( - "Fail to convert tensor '{}'. ".format(str(tensor_name)) + - str(err)) + "Fail to convert tensor '{}'. ".format(str(tensor_name)) + + str(err) + ) for tensor_name in self._pre_strategy: if tensor_name not in self._cur_strategy: tensor_not_in_cur.append(tensor_name) if not strict: - tensors_dict, tensor_match_with_pre, tensor_match_with_cur = self.convert_with_prefix_match( - tensors_dict, tensor_not_in_pre, tensor_not_in_cur) + ( + tensors_dict, + tensor_match_with_pre, + tensor_match_with_cur, + ) = self.convert_with_prefix_match( + tensors_dict, tensor_not_in_pre, tensor_not_in_cur + ) else: - tensors_dict, tensor_match_with_pre, tensor_match_with_cur = tensors_dict, [], [] + tensors_dict, tensor_match_with_pre, tensor_match_with_cur = ( + tensors_dict, + [], + [], + ) tensor_not_in_pre = set(tensor_not_in_pre) - set(tensor_match_with_pre) tensor_not_in_cur = set(tensor_not_in_cur) - set(tensor_match_with_cur) if tensor_not_in_pre: warnings.warn( "tensors [{}] are not found in last training strategy.".format( - str(tensor_not_in_pre))) + str(tensor_not_in_pre) + ) + ) if tensor_not_in_cur: warnings.warn( - "tensors [{}] are not found in current training strategy.". - format(str(tensor_not_in_cur))) + "tensors [{}] are not found in current training strategy.".format( + str(tensor_not_in_cur) + ) + ) if tensor_not_in_ckpt: warnings.warn( "tensors [{}] are found in pre_strategy, but are not found" - "in checkpoint files, please check your checkpoint files.". - format(str(tensor_not_in_ckpt))) + "in checkpoint files, please check your checkpoint files.".format( + str(tensor_not_in_ckpt) + ) + ) return tensors_dict - def convert_with_prefix_match(self, tensors_dict, tensor_not_in_pre, - tensor_not_in_cur): + def convert_with_prefix_match( + self, tensors_dict, tensor_not_in_pre, tensor_not_in_cur + ): # the name which in cur_process and can match with pre_process tensor_match_with_pre = [] # the name which in pre_process and can match with cur_process @@ -174,7 +204,7 @@ class Converter(object): for cur_name in tensor_not_in_pre: prefix_name = cur_name while prefix_name.find("_") != -1: - prefix_name = prefix_name[:prefix_name.rfind("_")] + prefix_name = prefix_name[: prefix_name.rfind("_")] for pre_name in tensor_not_in_cur: if prefix_name in pre_name: # 'cur_name' of cur_process can match with 'pre_name' of pre_process @@ -185,14 +215,20 @@ class Converter(object): cur_dist_attr = self._cur_strategy[cur_name] try: tensors_dict[cur_name] = Converter.merge_and_slice( - pre_tensor_list, pre_dist_attr, cur_dist_attr) + pre_tensor_list, pre_dist_attr, cur_dist_attr + ) except ValueError as err: raise ValueError( "Fail to convert tensor '{}' by '{}'. ".format( - str(cur_name), str(pre_name)) + str(err)) + str(cur_name), str(pre_name) + ) + + str(err) + ) self._logger.info( "tensor [{}] is matched with tensor [{}]".format( - cur_name, pre_name)) + cur_name, pre_name + ) + ) tensor_match_with_pre.append(cur_name) tensor_match_with_cur.append(pre_name) break @@ -221,8 +257,9 @@ class Converter(object): cur_dims_mapping = cur_dist_attr["dims_mapping"] if len(set(pre_dims_mapping)) > 1 or -1 not in pre_dims_mapping: # merge tensor - tensor = Converter.merge_with_dist_attr(tensor_list, - pre_dist_attr) + tensor = Converter.merge_with_dist_attr( + tensor_list, pre_dist_attr + ) else: # skip merge tensor tensor = tensor_list[0] @@ -235,7 +272,7 @@ class Converter(object): @staticmethod def merge_with_dist_attr(tensor_list, dist_attr): - """ Merge tensor with distributed attribute """ + """Merge tensor with distributed attribute""" from .reshard import Resharder dims_mapping = dist_attr["dims_mapping"] @@ -243,44 +280,62 @@ class Converter(object): process_group = dist_attr["process_group"] # get the complete shape of the tensor complete_shape = Resharder.compute_complete_shape( - tensor_list[0].shape, process_shape, dims_mapping) + tensor_list[0].shape, process_shape, dims_mapping + ) # merge the tensor with dist_attr partition_tensor_list = [] merged_partiton = [] for process in process_group: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, process_shape, - process_group) + process, + complete_shape, + dims_mapping, + process_shape, + process_group, + ) index = process_group.index(process) if partition_index not in merged_partiton: merged_partiton.append(partition_index) - Converter.merge(partition_tensor_list, tensor_list[index], - partition_index, complete_shape) + Converter.merge( + partition_tensor_list, + tensor_list[index], + partition_index, + complete_shape, + ) if len(partition_tensor_list) != 1: - raise ValueError("Fail to merge tensor with dist_attr '{}'.".format( - str(dist_attr))) + raise ValueError( + "Fail to merge tensor with dist_attr '{}'.".format( + str(dist_attr) + ) + ) complete_tensor = partition_tensor_list[0][0] return complete_tensor @staticmethod def slice_with_dist_attr(tensor, dist_attr): - """ Slice tensor with distributed attribute """ + """Slice tensor with distributed attribute""" dims_mapping = dist_attr["dims_mapping"] process_shape = dist_attr["process_shape"] process_group = dist_attr["process_group"] # slice the tensor with dist_attr partition_index_list = Converter._get_split_indices( - tensor.shape, dims_mapping, process_shape, process_group) - sliced_tensor_list = Converter.split(tensor, partition_index_list, - len(partition_index_list)) + tensor.shape, dims_mapping, process_shape, process_group + ) + sliced_tensor_list = Converter.split( + tensor, partition_index_list, len(partition_index_list) + ) # get the current tensor's index in sliced_tensor_list rank_id = paddle.distributed.get_rank() sliced_tensor_index = Converter._get_sliced_index( - rank_id, tensor.shape, dims_mapping, process_shape, process_group) + rank_id, tensor.shape, dims_mapping, process_shape, process_group + ) if sliced_tensor_index not in range(len(sliced_tensor_list)): - raise ValueError("Fail to slice tensor with dist_attr '{}'.".format( - str(dist_attr))) + raise ValueError( + "Fail to slice tensor with dist_attr '{}'.".format( + str(dist_attr) + ) + ) sliced_tensor = sliced_tensor_list[sliced_tensor_index] return sliced_tensor @@ -319,21 +374,32 @@ class Converter(object): else: i = 0 while i < len(partition_tensor_list): - concat_axis, first_order, new_partition = Resharder.compute_concat_info( - partition_tensor_list[i][1], partition_index) + ( + concat_axis, + first_order, + new_partition, + ) = Resharder.compute_concat_info( + partition_tensor_list[i][1], partition_index + ) if concat_axis != -1: if first_order == 0: new_tensor = np.concatenate( (partition_tensor_list[i][0], tensor), - axis=concat_axis) + axis=concat_axis, + ) else: new_tensor = np.concatenate( (tensor, partition_tensor_list[i][0]), - axis=concat_axis) + axis=concat_axis, + ) partition_tensor_list.pop(i) - Converter.merge(partition_tensor_list, new_tensor, - new_partition, complete_shape) + Converter.merge( + partition_tensor_list, + new_tensor, + new_partition, + complete_shape, + ) break i += 1 @@ -361,19 +427,21 @@ class Converter(object): """ sliced_tensor_list = [] axis = len(complete_tensor.shape) - length - sliced_tensor = np.split(complete_tensor, - partition_index_list[axis], - axis=axis) + sliced_tensor = np.split( + complete_tensor, partition_index_list[axis], axis=axis + ) if length == 1: return sliced_tensor for tensor in sliced_tensor: sliced_tensor_list.extend( - Converter.split(tensor, partition_index_list, length - 1)) + Converter.split(tensor, partition_index_list, length - 1) + ) return sliced_tensor_list @staticmethod - def _get_split_indices(complete_shape, dims_mapping, process_shape, - process_group): + def _get_split_indices( + complete_shape, dims_mapping, process_shape, process_group + ): """ Get split indices of every dimension. @@ -398,22 +466,31 @@ class Converter(object): split_indices_list = [] for process in process_group: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, process_shape, - process_group) + process, + complete_shape, + dims_mapping, + process_shape, + process_group, + ) if split_indices_list: for dim in range(len(partition_index)): split_indices_list[dim].extend(partition_index[dim]) else: split_indices_list = partition_index split_indices_list = list( - map(lambda x, y: list(set(x) - set([y]) - set([0])), - split_indices_list, complete_shape)) + map( + lambda x, y: list(set(x) - set([y]) - set([0])), + split_indices_list, + complete_shape, + ) + ) split_indices_list = [sorted(x) for x in split_indices_list] return split_indices_list @staticmethod - def _get_sliced_index(rank_id, complete_shape, dims_mapping, process_shape, - process_group): + def _get_sliced_index( + rank_id, complete_shape, dims_mapping, process_shape, process_group + ): """ Get sliced_tensor's index of current rank in all sliced tensors list. @@ -442,7 +519,8 @@ class Converter(object): from .reshard import Resharder partition_index = Resharder.compute_partition_index( - rank_id, complete_shape, dims_mapping, process_shape, process_group) + rank_id, complete_shape, dims_mapping, process_shape, process_group + ) sliced_index = 0 for i, shape in enumerate(complete_shape): if dims_mapping[i] == -1: diff --git a/python/paddle/distributed/auto_parallel/cost/base_cost.py b/python/paddle/distributed/auto_parallel/cost/base_cost.py index 5ac81052c76..2ce98a0a051 100644 --- a/python/paddle/distributed/auto_parallel/cost/base_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/base_cost.py @@ -25,8 +25,12 @@ from ..utils import _get_idx_in_axis from ..dist_tensor import DistributedTensor COMM_OP_TYPE = [ - "send_v2", "recv_v2", "c_broadcast", "c_allgather", "c_allreduce_sum", - "c_identity" + "send_v2", + "recv_v2", + "c_broadcast", + "c_allgather", + "c_allreduce_sum", + "c_identity", ] NON_COMP_TYPE = ["while"] + COMM_OP_TYPE _g_op_cost_factory = {} @@ -93,8 +97,9 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): var_name_list = op.input(input_name) var_desc = [] for var_name in var_name_list: - var = get_var_with_recursion(var_name, op.block, - op.block.program) + var = get_var_with_recursion( + var_name, op.block, op.block.program + ) # Use op input_dims_mapping dims_mapping = dist_attr.get_input_dims_mapping(var_name) global_sizes = var.shape @@ -102,18 +107,34 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): shard_sizes = None topology = process_mesh.topology shape = DistributedTensor.get_local_sizes( - global_sizes, dims_mapping, topology, processes, process, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + process, + shard_sizes, + ) var_desc.append((var.dtype, shape)) # For special op such as embedding and its grad op - if op.type == "c_embedding" or op.type == "lookup_table_v2" or op.type == "c_embedding_grad" or op.type == "lookup_table_v2_grad": + if ( + op.type == "c_embedding" + or op.type == "lookup_table_v2" + or op.type == "c_embedding_grad" + or op.type == "lookup_table_v2_grad" + ): if input_name == "W": - embedding_row_dim_mapping = dist_attr.get_input_dims_mapping( - op.input(input_name)[0])[0] + embedding_row_dim_mapping = ( + dist_attr.get_input_dims_mapping( + op.input(input_name)[0] + )[0] + ) relative_idx = _get_idx_in_axis( - processes, dist_attr.process_mesh.topology, - embedding_row_dim_mapping, process) + processes, + dist_attr.process_mesh.topology, + embedding_row_dim_mapping, + process, + ) per_part_size = shape[0] relative_idx = relative_idx * per_part_size desc["attrs"]["start_index"] = relative_idx @@ -126,8 +147,9 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): var_desc = [] for var_name in var_name_list: # Use op output_dims_mapping - var = get_var_with_recursion(var_name, op.block, - op.block.program) + var = get_var_with_recursion( + var_name, op.block, op.block.program + ) dist_attr = dist_op.dist_attr dims_mapping = dist_attr.get_output_dims_mapping(var_name) process_mesh = dist_attr.process_mesh @@ -136,8 +158,13 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): processes = process_mesh.processes topology = process_mesh.topology shape = DistributedTensor.get_local_sizes( - global_sizes, dims_mapping, topology, processes, process, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + process, + shard_sizes, + ) var_desc.append((var.dtype, shape)) # For special op such as fill_constant_batch_size_like @@ -150,8 +177,9 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): # Modify target shape for idx, axis in enumerate(dims_mapping): if axis >= 0: - shape_list[idx] = shape_list[ - idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) desc["attrs"]["shape"] = shape_list output_desc[out_name] = var_desc @@ -209,13 +237,15 @@ def build_comp_desc_str_for_predict(desc): return parse_result -def build_comm_desc_from_dist_op(op_type, - dist_op, - ctx, - var_names, - attrs=None, - parallel_axis=None, - group_ranks=None): +def build_comm_desc_from_dist_op( + op_type, + dist_op, + ctx, + var_names, + attrs=None, + parallel_axis=None, + group_ranks=None, +): """Build descriptions of communication op distributed on the processes.""" from ..reshard import get_var_with_recursion @@ -256,19 +286,26 @@ def build_comm_desc_from_dist_op(op_type, has_found = True break assert has_found - var = get_var_with_recursion(var_name, serial_op.block, - serial_op.block.program) + var = get_var_with_recursion( + var_name, serial_op.block, serial_op.block.program + ) - dims_mapping = dist_attr.get_input_dims_mapping( - var_name - ) if var_name in dist_op.serial_op.input_arg_names else dist_attr.get_output_dims_mapping( - var_name) + dims_mapping = ( + dist_attr.get_input_dims_mapping(var_name) + if var_name in dist_op.serial_op.input_arg_names + else dist_attr.get_output_dims_mapping(var_name) + ) global_sizes = var.shape shard_sizes = None topology = process_mesh.topology shape = DistributedTensor.get_local_sizes( - global_sizes, dims_mapping, topology, processes, process, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + process, + shard_sizes, + ) input_list.append((var.dtype, shape)) # NOTE: The input_name of comm ops used usually is X. @@ -278,9 +315,12 @@ def build_comm_desc_from_dist_op(op_type, if parallel_axis is not None: process_mesh_shape = process_mesh.topology process_mesh_group = process_mesh.processes - comm_group_ranks = _get_comm_group(process_mesh_group, - process_mesh_shape, - parallel_axis, rank_id) + comm_group_ranks = _get_comm_group( + process_mesh_group, + process_mesh_shape, + parallel_axis, + rank_id, + ) elif group_ranks is not None: comm_group_ranks = group_ranks else: @@ -322,8 +362,9 @@ def build_comm_costs_from_descs(op_cost_class, ctx, processes, descs, cluster): group_ranks = desc["group_ranks"] if group_ranks not in group_ranks_list: group_ranks_list.append(group_ranks) - comm_op_cost = op_cost_class(op_desc=desc, - comm_context=comm_context) + comm_op_cost = op_cost_class( + op_desc=desc, comm_context=comm_context + ) comm_op_cost_list.append(comm_op_cost) return comm_op_cost_list @@ -336,8 +377,9 @@ def build_comp_costs_from_descs(op_cost_class, ctx, processes, descs, cluster): return costs -def build_dp_costs(result, dist_op, ctx, var_names, attrs, parallel_axis, - cluster): +def build_dp_costs( + result, dist_op, ctx, var_names, attrs, parallel_axis, cluster +): """DP cost contains a allreduce_sum op cost and a scale op cost""" # The costs will be appended in the given result. from ..reshard import get_var_with_recursion @@ -370,10 +412,15 @@ def build_dp_costs(result, dist_op, ctx, var_names, attrs, parallel_axis, ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_cost_list = build_comm_costs_from_descs( - _g_op_cost_factory["c_allreduce_sum"], ctx, processes, - c_allreduce_sum_descs, cluster) + _g_op_cost_factory["c_allreduce_sum"], + ctx, + processes, + c_allreduce_sum_descs, + cluster, + ) result.append(comm_cost_list) # The scale op just on the group_ranks @@ -386,24 +433,33 @@ def build_dp_costs(result, dist_op, ctx, var_names, attrs, parallel_axis, desc = {} desc["op"] = op_type desc["inputs"] = {} - dims_mapping = dist_attr.get_input_dims_mapping( - var_name) if dist_attr.get_input_dims_mapping( - var_name - ) is not None else dist_attr.get_output_dims_mapping(var_name) - var = get_var_with_recursion(var_name, dist_op.serial_op.block, - dist_op.serial_op.block.program) + dims_mapping = ( + dist_attr.get_input_dims_mapping(var_name) + if dist_attr.get_input_dims_mapping(var_name) is not None + else dist_attr.get_output_dims_mapping(var_name) + ) + var = get_var_with_recursion( + var_name, + dist_op.serial_op.block, + dist_op.serial_op.block.program, + ) global_sizes = var.shape shard_sizes = None topology = process_mesh.topology - shape = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + shape = DistributedTensor.get_local_sizes( + global_sizes, + dims_mapping, + topology, + processes, + rank, + shard_sizes, + ) desc["inputs"]["X"] = [(var.dtype, shape)] attrs = {"scale": 1.0 / dp_degree} desc["attrs"] = attrs - scale_op_cost = _g_op_cost_factory["scale"](op_desc=desc, - cluster=cluster) + scale_op_cost = _g_op_cost_factory["scale"]( + op_desc=desc, cluster=cluster + ) scale_costs[rank] = scale_op_cost result.append(scale_costs) @@ -442,7 +498,7 @@ class CommContext: if alpha_latency is None: # set default self.base_ring = 8.4 - self.base_tree = 0. + self.base_tree = 0.0 # self.base_inter_ring = 9.6 # self.base_inter_tree = 28 # NVL in default @@ -457,7 +513,7 @@ class CommContext: self.base_ring = base_ring if base_ring is not None else 8.4 base_tree = alpha_latency.base_tree - self.base_tree = base_tree if base_tree is not None else 0. + self.base_tree = base_tree if base_tree is not None else 0.0 intra_ring = alpha_latency.intra_ring if intra_ring == LinkType.NVL: @@ -521,10 +577,16 @@ class CommContext: for i in range(len(ranks)): for j in range(i + 1, len(ranks)): forward_order_beta = self.cluster.get_beta( - ranks[i], ranks[j]) + ranks[i], ranks[j] + ) backward_order_beta = self.cluster.get_beta( - ranks[j], ranks[i]) - beta = forward_order_beta if forward_order_beta > backward_order_beta else backward_order_beta + ranks[j], ranks[i] + ) + beta = ( + forward_order_beta + if forward_order_beta > backward_order_beta + else backward_order_beta + ) if max_beta == None: max_beta = beta else: @@ -547,7 +609,6 @@ class CommContext: class Cost: - def __init__(self, time=0, memory=0, flops=0): self.time = time self.memory = memory @@ -557,14 +618,14 @@ class Cost: assert val >= 0, "Time must be greater than or equal to 0." def _check_memory(self, val): - assert isinstance( - val, - int) and val >= 0, "Memory must be int and greater than equal to 0." + assert ( + isinstance(val, int) and val >= 0 + ), "Memory must be int and greater than equal to 0." def _check_flops(self, val): - assert isinstance( - val, - int) and val >= 0, "FLOPs must be int and greater than equal to 0." + assert ( + isinstance(val, int) and val >= 0 + ), "FLOPs must be int and greater than equal to 0." @property def time(self): @@ -598,7 +659,7 @@ class Cost: time = self.time + rhs.time memory = self.memory + rhs.memory flops = self.flops + rhs.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 return Cost(time, memory, flops) def __sub__(self, rhs): @@ -606,12 +667,11 @@ class Cost: time = self.time - rhs.time memory = self.memory - rhs.memory flops = self.flops - rhs.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 return Cost(time, memory, flops) class OpCost: - def __init__(self, op=None, op_desc=None): self._op = op self._op_desc = op_desc @@ -666,12 +726,12 @@ class OpCost: time = self.cost.time + rhs.cost.time memory = self.cost.memory + rhs.cost.memory flops = self.cost.flops + rhs.cost.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 elif isinstance(rhs, Cost): time = self.time + rhs.time memory = self.memory + rhs.memory flops = self.flops + rhs.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 return Cost(time, memory, flops) def __sub__(self, rhs): @@ -683,12 +743,12 @@ class OpCost: time = self.cost.time - rhs.cost.time memory = self.cost.memory - rhs.cost.memory flops = self.cost.flops - rhs.cost.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 elif isinstance(rhs, Cost): time = self.time - rhs.time memory = self.memory - rhs.memory flops = self.flops - rhs.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 return Cost(time, memory, flops) @@ -721,8 +781,9 @@ class CommOpCost(OpCost): vars = self.op.block.vars # NOTE: The tensor communicated input_name is "X" in default. Otherwise, this function should be overrided var_name = self.op.input("X")[0] - var = get_var_with_recursion(var_name, self.op.block, - self.program) + var = get_var_with_recursion( + var_name, self.op.block, self.program + ) dtype = var.dtype shape = var.shape elif self.op_desc is not None: @@ -756,7 +817,8 @@ class CommOpCost(OpCost): if self._machine_count is None: cluster = self._comm_context.cluster self._machine_count = cluster.get_involved_machine_count( - self.group_ranks) + self.group_ranks + ) return self._machine_count @property @@ -775,8 +837,10 @@ class CommOpCost(OpCost): process_group = get_process_group(ring_id) if process_group is None: raise ValueError( - "There not exists process group whose ring_id is {}.". - format(ring_id)) + "There not exists process group whose ring_id is {}.".format( + ring_id + ) + ) self._group_ranks = process_group.ranks return self._group_ranks @@ -786,7 +850,9 @@ class CommOpCost(OpCost): if cls.OP_TYPE not in COMM_OP_TYPE: raise TypeError( "Please Check op type in {}, but got {}.".format( - COMM_OP_TYPE, cls.OP_TYPE)) + COMM_OP_TYPE, cls.OP_TYPE + ) + ) class CompOpCost(OpCost): @@ -804,7 +870,9 @@ class CompOpCost(OpCost): if cls.OP_TYPE in NON_COMP_TYPE: raise TypeError( "Please Check op type not in {}, but got {}.".format( - NON_COMP_TYPE, cls.OP_TYPE)) + NON_COMP_TYPE, cls.OP_TYPE + ) + ) def register_op_cost(cls): @@ -821,12 +889,12 @@ def register_op_cost(cls): def calc_time_by_modeling(op=None, desc=None, cluster=None): op_type = op.type if op is not None else desc["op"] if op_type in COMM_OP_TYPE: - op_cost = _g_op_cost_factory[op_type](op=op, - op_desc=desc, - comm_context=CommContext(cluster)) + op_cost = _g_op_cost_factory[op_type]( + op=op, op_desc=desc, comm_context=CommContext(cluster) + ) elif op_type not in NON_COMP_TYPE: - op_cost = _g_op_cost_factory[op_type](op=op, - op_desc=desc, - cluster=cluster) + op_cost = _g_op_cost_factory[op_type]( + op=op, op_desc=desc, cluster=cluster + ) time = op_cost.calc_time() return time diff --git a/python/paddle/distributed/auto_parallel/cost/comm_op_cost.py b/python/paddle/distributed/auto_parallel/cost/comm_op_cost.py index b81df4dbe86..2e8e347bd8e 100644 --- a/python/paddle/distributed/auto_parallel/cost/comm_op_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/comm_op_cost.py @@ -22,9 +22,9 @@ class AllreduceSumOpCost(CommOpCost): OP_TYPE = "c_allreduce_sum" def __init__(self, op=None, op_desc=None, comm_context=None): - super(AllreduceSumOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(AllreduceSumOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): # use tree if cross machine and use ring if in a single machine @@ -39,22 +39,41 @@ class AllreduceSumOpCost(CommOpCost): def calc_time_ring(self): alpha = self.comm_context.base_ring - alpha += 2 * (self.rank_count - - self.machine_count) * self.comm_context.intra_ring - alpha += 2 * (self.machine_count - 1) * ( - self.comm_context.inter_ring + self.hops * self.comm_context.switch) + alpha += ( + 2 + * (self.rank_count - self.machine_count) + * self.comm_context.intra_ring + ) + alpha += ( + 2 + * (self.machine_count - 1) + * ( + self.comm_context.inter_ring + + self.hops * self.comm_context.switch + ) + ) beta = self.comm_context.get_max_beta(self.group_ranks) - time = alpha + 2 * (self.rank_count - - 1) / self.rank_count * self.comm_count * beta + time = ( + alpha + + 2 + * (self.rank_count - 1) + / self.rank_count + * self.comm_count + * beta + ) return time def calc_time_tree(self): alpha = self.comm_context.base_tree - alpha += 2 * (self.rank_count / self.machine_count - - 1) * self.comm_context.intra_tree + alpha += ( + 2 + * (self.rank_count / self.machine_count - 1) + * self.comm_context.intra_tree + ) alpha += math.log2(self.machine_count) * ( - self.comm_context.inter_tree + self.hops * self.comm_context.switch) + self.comm_context.inter_tree + self.hops * self.comm_context.switch + ) beta = self.comm_context.get_max_beta(self.group_ranks) time = alpha + 2 * self.comm_count * beta @@ -67,9 +86,9 @@ class AllgatherOpCost(CommOpCost): OP_TYPE = "c_allgather" def __init__(self, op=None, op_desc=None, comm_context=None): - super(AllgatherOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(AllgatherOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): time = self.calc_time_ring() @@ -77,13 +96,17 @@ class AllgatherOpCost(CommOpCost): def calc_time_ring(self): alpha = self.comm_context.base_ring - alpha += (self.rank_count - - self.machine_count) * self.comm_context.intra_ring + alpha += ( + self.rank_count - self.machine_count + ) * self.comm_context.intra_ring alpha += (self.machine_count - 1) * ( - self.comm_context.inter_ring + self.hops * self.comm_context.switch) + self.comm_context.inter_ring + self.hops * self.comm_context.switch + ) beta = self.comm_context.get_max_beta(self.group_ranks) - time = alpha + (self.rank_count - - 1) / self.rank_count * self.comm_count * beta + time = ( + alpha + + (self.rank_count - 1) / self.rank_count * self.comm_count * beta + ) return time @@ -92,9 +115,9 @@ class BroadcastOpCost(CommOpCost): OP_TYPE = "c_broadcast" def __init__(self, op=None, op_desc=None, comm_context=None): - super(BroadcastOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(BroadcastOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): time = self.calc_time_ring() @@ -103,7 +126,10 @@ class BroadcastOpCost(CommOpCost): def calc_time_ring(self): alpha = self.comm_context.base_ring if self.machine_count > 1: - alpha += self.comm_context.inter_ring + self.hops * self.comm_context.switch + alpha += ( + self.comm_context.inter_ring + + self.hops * self.comm_context.switch + ) else: alpha += self.comm_context.intra_ring beta = self.comm_context.get_max_beta(self.group_ranks) @@ -117,9 +143,9 @@ class IdentityOpCost(CommOpCost): OP_TYPE = "c_identity" def __init__(self, op=None, op_desc=None, comm_context=None): - super(IdentityOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(IdentityOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): return 0 @@ -130,14 +156,17 @@ class RecvOpCost(CommOpCost): OP_TYPE = "recv_v2" def __init__(self, op=None, op_desc=None, comm_context=None): - super(RecvOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(RecvOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): alpha = self.comm_context.base_ring if self.machine_count > 1: - alpha += self.comm_context.inter_ring + self.hops * self.comm_context.switch + alpha += ( + self.comm_context.inter_ring + + self.hops * self.comm_context.switch + ) else: alpha += self.comm_context.intra_ring beta = self.comm_context.get_max_beta(self.group_ranks) @@ -150,14 +179,17 @@ class SendOpCost(CommOpCost): OP_TYPE = "send_v2" def __init__(self, op=None, op_desc=None, comm_context=None): - super(SendOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(SendOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): alpha = self.comm_context.base_ring if self.machine_count > 1: - alpha += self.comm_context.inter_ring + self.hops * self.comm_context.switch + alpha += ( + self.comm_context.inter_ring + + self.hops * self.comm_context.switch + ) else: alpha += self.comm_context.intra_ring beta = self.comm_context.get_max_beta(self.group_ranks) diff --git a/python/paddle/distributed/auto_parallel/cost/comp_op_cost.py b/python/paddle/distributed/auto_parallel/cost/comp_op_cost.py index 938a9465701..51c3d6069a6 100644 --- a/python/paddle/distributed/auto_parallel/cost/comp_op_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/comp_op_cost.py @@ -20,9 +20,9 @@ class AdamOpCost(CompOpCost): OP_TYPE = "adam" def __init__(self, op=None, op_desc=None, cluster=None): - super(AdamOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(AdamOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -39,9 +39,9 @@ class AssignOpCost(CompOpCost): OP_TYPE = "assign" def __init__(self, op=None, op_desc=None, cluster=None): - super(AssignOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(AssignOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -58,9 +58,9 @@ class AssignValueOpCost(CompOpCost): OP_TYPE = "assign_value" def __init__(self, op=None, op_desc=None, cluster=None): - super(AssignValueOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(AssignValueOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -77,9 +77,9 @@ class BeamSearchOpCost(CompOpCost): OP_TYPE = "beam_search" def __init__(self, op=None, op_desc=None, cluster=None): - super(BeamSearchOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(BeamSearchOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -96,9 +96,9 @@ class BeamSearchDecodeOpCost(CompOpCost): OP_TYPE = "beam_search_decode" def __init__(self, op=None, op_desc=None, cluster=None): - super(BeamSearchDecodeOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(BeamSearchDecodeOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -115,9 +115,9 @@ class CastOpCost(CompOpCost): OP_TYPE = "cast" def __init__(self, op=None, op_desc=None, cluster=None): - super(CastOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(CastOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -134,9 +134,9 @@ class ConcatOpCost(CompOpCost): OP_TYPE = "concat" def __init__(self, op=None, op_desc=None, cluster=None): - super(ConcatOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ConcatOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -153,9 +153,9 @@ class DropoutOpCost(CompOpCost): OP_TYPE = "dropout" def __init__(self, op=None, op_desc=None, cluster=None): - super(DropoutOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(DropoutOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -172,9 +172,9 @@ class DropoutGradOpCost(CompOpCost): OP_TYPE = "dropout_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(DropoutGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(DropoutGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -191,9 +191,9 @@ class ElementwiseAddOpCost(CompOpCost): OP_TYPE = "elementwise_add" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseAddOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseAddOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -210,9 +210,9 @@ class ElementwiseAddGradOpCost(CompOpCost): OP_TYPE = "elementwise_add_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseAddGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseAddGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -229,9 +229,9 @@ class ElementwiseDivOpCost(CompOpCost): OP_TYPE = "elementwise_div" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseDivOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseDivOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -248,9 +248,9 @@ class ElementwiseDivGradOpCost(CompOpCost): OP_TYPE = "elementwise_div_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseDivGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseDivGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -267,9 +267,9 @@ class ElementwiseMulOpCost(CompOpCost): OP_TYPE = "elementwise_mul" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseMulOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseMulOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -286,9 +286,9 @@ class ElementwiseMulGradOpCost(CompOpCost): OP_TYPE = "elementwise_mul_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseMulGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseMulGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -305,9 +305,9 @@ class ElementwiseSubOpCost(CompOpCost): OP_TYPE = "elementwise_sub" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseSubOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseSubOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -324,9 +324,9 @@ class ElementwiseSubGradOpCost(CompOpCost): OP_TYPE = "elementwise_sub_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseSubGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseSubGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -343,9 +343,9 @@ class EmbeddingOpCost(CompOpCost): OP_TYPE = "c_embedding" def __init__(self, op=None, op_desc=None, cluster=None): - super(EmbeddingOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(EmbeddingOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -362,9 +362,9 @@ class EmbeddingGradOpCost(CompOpCost): OP_TYPE = "c_embedding_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(EmbeddingGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(EmbeddingGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -381,9 +381,9 @@ class FillConstantOpCost(CompOpCost): OP_TYPE = "fill_constant" def __init__(self, op=None, op_desc=None, cluster=None): - super(FillConstantOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(FillConstantOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -400,9 +400,9 @@ class FillConstantBatchSizeLikeOpCost(CompOpCost): OP_TYPE = "fill_constant_batch_size_like" def __init__(self, op=None, op_desc=None, cluster=None): - super(FillConstantBatchSizeLikeOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(FillConstantBatchSizeLikeOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -419,8 +419,9 @@ class FusedSoftmaxMaskUpperTriangleOpCost(CompOpCost): OP_TYPE = "fused_softmax_mask_upper_triangle" def __init__(self, op=None, op_desc=None, cluster=None): - super(FusedSoftmaxMaskUpperTriangleOpCost, - self).__init__(op=op, op_desc=op_desc, cluster=cluster) + super(FusedSoftmaxMaskUpperTriangleOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -437,8 +438,9 @@ class FusedSoftmaxMaskUpperTriangleGradOpCost(CompOpCost): OP_TYPE = "fused_softmax_mask_upper_triangle_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(FusedSoftmaxMaskUpperTriangleGradOpCost, - self).__init__(op=op, op_desc=op_desc, cluster=cluster) + super(FusedSoftmaxMaskUpperTriangleGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -455,9 +457,9 @@ class GatherOpCost(CompOpCost): OP_TYPE = "gather" def __init__(self, op=None, op_desc=None, cluster=None): - super(GatherOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(GatherOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -474,9 +476,9 @@ class GeluOpCost(CompOpCost): OP_TYPE = "gelu" def __init__(self, op=None, op_desc=None, cluster=None): - super(GeluOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(GeluOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -493,9 +495,9 @@ class GeluGradOpCost(CompOpCost): OP_TYPE = "gelu_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(GeluGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(GeluGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -512,9 +514,9 @@ class GreaterEqualOpCost(CompOpCost): OP_TYPE = "greater_equal" def __init__(self, op=None, op_desc=None, cluster=None): - super(GreaterEqualOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(GreaterEqualOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -531,9 +533,9 @@ class IncrementOpCost(CompOpCost): OP_TYPE = "increment" def __init__(self, op=None, op_desc=None, cluster=None): - super(IncrementOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(IncrementOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -546,9 +548,9 @@ class IsEmptyOpCost(CompOpCost): OP_TYPE = "is_empty" def __init__(self, op=None, op_desc=None, cluster=None): - super(IsEmptyOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(IsEmptyOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -561,9 +563,9 @@ class LayerNormOpCost(CompOpCost): OP_TYPE = "layer_norm" def __init__(self, op=None, op_desc=None, cluster=None): - super(LayerNormOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LayerNormOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -580,9 +582,9 @@ class LayerNormGradOpCost(CompOpCost): OP_TYPE = "layer_norm_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(LayerNormGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LayerNormGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -599,9 +601,9 @@ class LessThanOpCost(CompOpCost): OP_TYPE = "less_than" def __init__(self, op=None, op_desc=None, cluster=None): - super(LessThanOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LessThanOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -618,9 +620,9 @@ class LogicalNotOpCost(CompOpCost): OP_TYPE = "logical_not" def __init__(self, op=None, op_desc=None, cluster=None): - super(LogicalNotOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LogicalNotOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -637,9 +639,9 @@ class LogicalAndOpCost(CompOpCost): OP_TYPE = "logical_and" def __init__(self, op=None, op_desc=None, cluster=None): - super(LogicalAndOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LogicalAndOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -656,9 +658,9 @@ class LodResetOpCost(CompOpCost): OP_TYPE = "lod_reset" def __init__(self, op=None, op_desc=None, cluster=None): - super(LodResetOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LodResetOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -692,9 +694,9 @@ class LookupTableV2OpCost(CompOpCost): OP_TYPE = "lookup_table_v2" def __init__(self, op=None, op_desc=None, cluster=None): - super(LookupTableV2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LookupTableV2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -711,9 +713,9 @@ class LookupTableV2GradOpCost(CompOpCost): OP_TYPE = "lookup_table_v2_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(LookupTableV2GradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LookupTableV2GradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -730,9 +732,9 @@ class MatmulOpCost(CompOpCost): OP_TYPE = "matmul" def __init__(self, op=None, op_desc=None, cluster=None): - super(MatmulOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MatmulOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -749,9 +751,9 @@ class MatmulGradOpCost(CompOpCost): OP_TYPE = "matmul_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(MatmulGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MatmulGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -768,9 +770,9 @@ class MatmulV2OpCost(CompOpCost): OP_TYPE = "matmul_v2" def __init__(self, op=None, op_desc=None, cluster=None): - super(MatmulV2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MatmulV2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -787,9 +789,9 @@ class MatmulV2GradOpCost(CompOpCost): OP_TYPE = "matmul_v2_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(MatmulV2GradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MatmulV2GradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -806,9 +808,9 @@ class MemcpyOpCost(CompOpCost): OP_TYPE = "memcpy" def __init__(self, op=None, op_desc=None, cluster=None): - super(MemcpyOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MemcpyOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -842,9 +844,9 @@ class MulGradOpCost(CompOpCost): OP_TYPE = "mul_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(MulGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MulGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -861,9 +863,9 @@ class OneHotOpCost(CompOpCost): OP_TYPE = "one_hot" def __init__(self, op=None, op_desc=None, cluster=None): - super(OneHotOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(OneHotOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -880,9 +882,9 @@ class ReadFromArrayOpCost(CompOpCost): OP_TYPE = "read_from_array" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReadFromArrayOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReadFromArrayOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -899,9 +901,9 @@ class ReduceSumOpCost(CompOpCost): OP_TYPE = "reduce_sum" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReduceSumOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReduceSumOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -918,9 +920,9 @@ class ReduceSumGradOpCost(CompOpCost): OP_TYPE = "reduce_sum_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReduceSumGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReduceSumGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -937,9 +939,9 @@ class Reshape2OpCost(CompOpCost): OP_TYPE = "reshape2" def __init__(self, op=None, op_desc=None, cluster=None): - super(Reshape2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Reshape2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -956,9 +958,9 @@ class Reshape2GradOpCost(CompOpCost): OP_TYPE = "reshape2_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(Reshape2GradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Reshape2GradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -975,9 +977,9 @@ class ReduceMeanOpCost(CompOpCost): OP_TYPE = "reduce_mean" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReduceMeanOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReduceMeanOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -994,9 +996,9 @@ class ReduceMeanGradOpCost(CompOpCost): OP_TYPE = "reduce_mean_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReduceMeanGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReduceMeanGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1013,9 +1015,9 @@ class SamplingIdOpCost(CompOpCost): OP_TYPE = "sampling_id" def __init__(self, op=None, op_desc=None, cluster=None): - super(SamplingIdOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SamplingIdOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1032,9 +1034,9 @@ class ScaleOpCost(CompOpCost): OP_TYPE = "scale" def __init__(self, op=None, op_desc=None, cluster=None): - super(ScaleOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ScaleOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1051,9 +1053,9 @@ class SliceOpCost(CompOpCost): OP_TYPE = "slice" def __init__(self, op=None, op_desc=None, cluster=None): - super(SliceOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SliceOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1070,9 +1072,9 @@ class SoftmaxOpCost(CompOpCost): OP_TYPE = "softmax" def __init__(self, op=None, op_desc=None, cluster=None): - super(SoftmaxOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SoftmaxOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1089,9 +1091,9 @@ class SoftmaxGradOpCost(CompOpCost): OP_TYPE = "softmax_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(SoftmaxGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SoftmaxGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1108,9 +1110,9 @@ class SoftmaxWithCrossEntropyOpCost(CompOpCost): OP_TYPE = "softmax_with_cross_entropy" def __init__(self, op=None, op_desc=None, cluster=None): - super(SoftmaxWithCrossEntropyOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SoftmaxWithCrossEntropyOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1127,9 +1129,9 @@ class SoftmaxWithCrossEntropyGradOpCost(CompOpCost): OP_TYPE = "softmax_with_cross_entropy_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(SoftmaxWithCrossEntropyGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SoftmaxWithCrossEntropyGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1146,9 +1148,9 @@ class SplitOpCost(CompOpCost): OP_TYPE = "split" def __init__(self, op=None, op_desc=None, cluster=None): - super(SplitOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SplitOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1165,9 +1167,9 @@ class Squeeze2OpCost(CompOpCost): OP_TYPE = "squeeze2" def __init__(self, op=None, op_desc=None, cluster=None): - super(Squeeze2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Squeeze2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1184,9 +1186,9 @@ class SquareOpCost(CompOpCost): OP_TYPE = "square" def __init__(self, op=None, op_desc=None, cluster=None): - super(SquareOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SquareOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1203,9 +1205,9 @@ class SquareGradOpCost(CompOpCost): OP_TYPE = "square_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(SquareGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SquareGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1239,9 +1241,9 @@ class TopKOpCost(CompOpCost): OP_TYPE = "top_k" def __init__(self, op=None, op_desc=None, cluster=None): - super(TopKOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(TopKOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1258,9 +1260,9 @@ class Transpose2OpCost(CompOpCost): OP_TYPE = "transpose2" def __init__(self, op=None, op_desc=None, cluster=None): - super(Transpose2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Transpose2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1277,9 +1279,9 @@ class Transpose2GradOpCost(CompOpCost): OP_TYPE = "transpose2_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(Transpose2GradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Transpose2GradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1296,9 +1298,9 @@ class Unsqueeze2OpCost(CompOpCost): OP_TYPE = "unsqueeze2" def __init__(self, op=None, op_desc=None, cluster=None): - super(Unsqueeze2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Unsqueeze2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1315,9 +1317,9 @@ class WriteToArrayOpCost(CompOpCost): OP_TYPE = "write_to_array" def __init__(self, op=None, op_desc=None, cluster=None): - super(WriteToArrayOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(WriteToArrayOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): diff --git a/python/paddle/distributed/auto_parallel/cost/estimate_cost.py b/python/paddle/distributed/auto_parallel/cost/estimate_cost.py index fba9d7919c4..cac8bf9f277 100644 --- a/python/paddle/distributed/auto_parallel/cost/estimate_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/estimate_cost.py @@ -26,12 +26,9 @@ from ..dist_tensor import DistributedTensor class CostEstimator: _sepical_op_type = ["fused_attention", "fused_feedforward"] - def __init__(self, - program, - cluster, - mode="modeling", - rank=None, - loop_count=10): + def __init__( + self, program, cluster, mode="modeling", rank=None, loop_count=10 + ): self._program = program self._cluster = cluster self._check_mode(mode) @@ -40,7 +37,8 @@ class CostEstimator: self._loop_count = loop_count self._global_cost = Cost() self._local_cost_mapping = {} - self._detailed_cost = OrderedDict( + self._detailed_cost = ( + OrderedDict() ) # {`op_id`: {"reshard": [], "dist_op": [], "local_cost": local_cost}}} self._bubble_time_mapping = {} self._ordered_ops = [] @@ -105,7 +103,8 @@ class CostEstimator: def _check_mode(self, mode): if mode not in ["modeling", "profiling"]: raise ValueError( - "Just support modeling and profiling, but got {}".format(mode)) + "Just support modeling and profiling, but got {}".format(mode) + ) def _is_special_var_name(self, var_name): special_var_name = ["lod_tensor_blocking_queue_0"] @@ -115,6 +114,7 @@ class CostEstimator: def _estimate_core(self, dist_context, resharder, block): from ..reshard import get_var_with_recursion + ops = block.ops loop_count = None if block.desc.id != self.program.global_block().desc.id: @@ -131,8 +131,9 @@ class CostEstimator: if int(op.attr('op_role')) == int(OpRole.Optimize): continue if op.type in [ - "create_py_reader", "create_double_buffer_reader", - "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ]: continue @@ -171,14 +172,16 @@ class CostEstimator: max_time = rank_cost.time for rank in group_ranks: - self.local_cost( - rank).time = max_time + cost.time + self.local_cost(rank).time = ( + max_time + cost.time + ) if rank not in self._bubble_time_mapping: self._bubble_time_mapping[rank] = 0 self._bubble_time_mapping[rank] += ( - max_time - cost_time[rank]) + max_time - cost_time[rank] + ) for rank in local_comp_cost: for comp_cost in local_comp_cost[rank]: @@ -190,15 +193,19 @@ class CostEstimator: processes = op_dist_attr.process_mesh.processes container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, self.cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, self.cluster + ) detail["dist_op_cost"] = dist_op_cost if dist_op_cost is None: - assert dist_op.serial_op.type in CostEstimator._sepical_op_type + assert ( + dist_op.serial_op.type in CostEstimator._sepical_op_type + ) continue for item in dist_op_cost: if isinstance(item, list): @@ -216,12 +223,14 @@ class CostEstimator: if max_time < rank_cost.time: max_time = rank_cost.time for rank in group_ranks: - self.local_cost( - rank).time = max_time + comm_op_cost.time + self.local_cost(rank).time = ( + max_time + comm_op_cost.time + ) if rank not in self._bubble_time_mapping: self._bubble_time_mapping[rank] = 0 self._bubble_time_mapping[rank] += ( - max_time - cost_time[rank]) + max_time - cost_time[rank] + ) elif isinstance(item, dict): # Op just one for rank in processes: @@ -246,8 +255,11 @@ class CostEstimator: dtype_factor = 8 elif dtype == paddle.float32 or dtype == paddle.int32: dtype_factor = 4 - elif dtype == paddle.float16 or dtype == paddle.bfloat16 \ - or dtype == paddle.int16: + elif ( + dtype == paddle.float16 + or dtype == paddle.bfloat16 + or dtype == paddle.int16 + ): dtype_factor = 2 elif dtype == paddle.int8 or dtype == paddle.uint8: dtype_factor = 1 @@ -269,8 +281,9 @@ class CostEstimator: memories = {} self.max_memories = {} - var_info = { - } # var_name: [[process_mesh, dims_mapping], [id]], [[process_mesh, dims_mapping], [id]]} + var_info = ( + {} + ) # var_name: [[process_mesh, dims_mapping], [id]], [[process_mesh, dims_mapping], [id]]} for block in self.program.blocks: for op in block.ops: @@ -279,18 +292,22 @@ class CostEstimator: for op_id, op in self._ordered_ops: if op.type in [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ]: continue dist_op = dist_context.get_dist_op_for_program(op) process_mesh = dist_op.dist_attr.process_mesh for var_name in op.input_arg_names: input_dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - var_name) + var_name + ) if var_name not in var_info: var_info[var_name] = {} - key = _convert_pm_and_dm_to_str(process_mesh, - input_dims_mapping) + key = _convert_pm_and_dm_to_str( + process_mesh, input_dims_mapping + ) if key not in var_info[var_name]: var_info[var_name][key] = {} # It is even partition now @@ -299,21 +316,27 @@ class CostEstimator: global_sizes = var.shape dtype = var.dtype sizes = DistributedTensor.get_local_sizes( - global_sizes, input_dims_mapping, process_mesh.topology, - process_mesh.processes) + global_sizes, + input_dims_mapping, + process_mesh.topology, + process_mesh.processes, + ) var_info[var_name][key]["memory"] = self._calculate_bytes( - sizes, dtype) + sizes, dtype + ) if "position" not in var_info[var_name][key]: var_info[var_name][key]["position"] = [] var_info[var_name][key]["position"].append(op_id) for var_name in op.output_arg_names: output_dims_mapping = dist_op.dist_attr.get_output_dims_mapping( - var_name) + var_name + ) if var_name not in var_info: var_info[var_name] = {} - key = _convert_pm_and_dm_to_str(process_mesh, - output_dims_mapping) + key = _convert_pm_and_dm_to_str( + process_mesh, output_dims_mapping + ) if key not in var_info[var_name]: var_info[var_name][key] = {} if "memory" not in var_info[var_name][key]: @@ -321,10 +344,14 @@ class CostEstimator: global_sizes = var.shape dtype = var.dtype sizes = DistributedTensor.get_local_sizes( - global_sizes, output_dims_mapping, - process_mesh.topology, process_mesh.processes) + global_sizes, + output_dims_mapping, + process_mesh.topology, + process_mesh.processes, + ) var_info[var_name][key]["memory"] = self._calculate_bytes( - sizes, dtype) + sizes, dtype + ) if "position" not in var_info[var_name][key]: var_info[var_name][key]["position"] = [] var_info[var_name][key]["position"].append(op_id) @@ -332,7 +359,9 @@ class CostEstimator: has_used_vars = set() for op_id, op in self._ordered_ops: if op.type in [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ]: continue can_free_memories = {} @@ -341,9 +370,11 @@ class CostEstimator: process_mesh = dist_op.dist_attr.process_mesh for var_name in op.input_arg_names: input_dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - var_name) - key = _convert_pm_and_dm_to_str(process_mesh, - input_dims_mapping) + var_name + ) + key = _convert_pm_and_dm_to_str( + process_mesh, input_dims_mapping + ) has_used_var = var_name + key var = dist_op.get_serial_input(var_name) # Not used @@ -363,13 +394,16 @@ class CostEstimator: if process not in can_free_memories: can_free_memories[process] = 0 can_free_memories[process] += var_info[ - var_name][key]["memory"] + var_name + ][key]["memory"] for var_name in op.output_arg_names: output_dims_mapping = dist_op.dist_attr.get_output_dims_mapping( - var_name) - key = _convert_pm_and_dm_to_str(process_mesh, - output_dims_mapping) + var_name + ) + key = _convert_pm_and_dm_to_str( + process_mesh, output_dims_mapping + ) has_used_var = var_name + key var = dist_op.get_serial_output(var_name) # Not used @@ -389,7 +423,8 @@ class CostEstimator: if process not in can_free_memories: can_free_memories[process] = 0 can_free_memories[process] += var_info[ - var_name][key]["memory"] + var_name + ][key]["memory"] # Calc peak memory for process in memories: @@ -413,8 +448,12 @@ class CostEstimator: def estimate(self, dist_context, resharder=None): self.prepare() from ..reshard import Resharder - resharder = Resharder(self.program, None, self.rank, dist_context, - []) if resharder is None else resharder + + resharder = ( + Resharder(self.program, None, self.rank, dist_context, []) + if resharder is None + else resharder + ) block = self.program.global_block() self._estimate_core(dist_context, resharder, block) @@ -446,7 +485,7 @@ class CostEstimator: memories = [ int(item // 1e6) for item in list(self.max_memories.values()) ] - for memory in (memories + header): + for memory in memories + header: if len(str(memory)) > max_len: max_len = len(str(memory)) max_len += 4 # for pretty print of center @@ -476,7 +515,7 @@ class CostEstimator: max_len = 0 header = ["Execution Time(ms)", "Max Memory(MiB)"] vals = [round(self.global_cost.time, 3), int(self.max_memory // 1e6)] - for memory in (vals + header): + for memory in vals + header: if len(str(memory)) > max_len: max_len = len(str(memory)) max_len += 4 # for pretty print of center @@ -506,16 +545,27 @@ class CostEstimator: def get_cost_from_engine(engine, mode): from ..utils import to_list + # Construct cost estimator by original main program - serial_main_prog = engine._serial_main_progs[mode].clone( - ) if mode in engine._serial_main_progs else engine._orig_main_prog.clone() + serial_main_prog = ( + engine._serial_main_progs[mode].clone() + if mode in engine._serial_main_progs + else engine._orig_main_prog.clone() + ) - serial_startup_prog = engine._serial_startup_progs[mode].clone( - ) if mode in engine._serial_startup_progs else engine._orig_startup_prog.clone( + serial_startup_prog = ( + engine._serial_startup_progs[mode].clone() + if mode in engine._serial_startup_progs + else engine._orig_startup_prog.clone() + ) + losses = ( + to_list(engine._loss) + if ( + not isinstance(engine._loss, paddle.nn.Layer) + and not callable(engine._loss) + ) + else engine._losses ) - losses = to_list( - engine._loss) if (not isinstance(engine._loss, paddle.nn.Layer) - and not callable(engine._loss)) else engine._losses if mode in engine._dist_contexts: dist_context = engine._dist_contexts[mode] @@ -523,33 +573,45 @@ def get_cost_from_engine(engine, mode): else: from ..completion import Completer from ..dist_context import DistributedContext - dist_context = DistributedContext(serial_main_prog, serial_startup_prog, - engine._optimizer, losses, {}, - {"loss": losses}, engine._cluster, - engine._strategy) + + dist_context = DistributedContext( + serial_main_prog, + serial_startup_prog, + engine._optimizer, + losses, + {}, + {"loss": losses}, + engine._cluster, + engine._strategy, + ) completer = Completer(dist_context) completer.complete_forward_annotation() dist_context.block_state.parse_forward_blocks( - dist_context.serial_main_program) + dist_context.serial_main_program + ) if mode == "eval" or mode == "predict": cost_estimator = CostEstimator(serial_main_prog, engine._cluster) elif mode == "train": from ..parallelizer_v2 import Parallelizer + # Get serial main program with backward serial_optimizer = engine._optimizer parallelizer = Parallelizer(mode, completer, dist_context) # Generate backward loss_name = dist_context.serial_loss.name serial_loss = serial_main_prog.global_block()._var_recursive(loss_name) - params_grads = parallelizer._generate_backward(serial_main_prog, - serial_startup_prog, - serial_loss) + params_grads = parallelizer._generate_backward( + serial_main_prog, serial_startup_prog, serial_loss + ) # Generate optimizer optimizer_ops = parallelizer._generate_optimizer( - serial_main_prog, serial_startup_prog, serial_optimizer, - params_grads) + serial_main_prog, + serial_startup_prog, + serial_optimizer, + params_grads, + ) cost_estimator = CostEstimator(serial_main_prog, engine._cluster) # Estimate global_cost and max memory diff --git a/python/paddle/distributed/auto_parallel/cost/tensor_cost.py b/python/paddle/distributed/auto_parallel/cost/tensor_cost.py index 9741020da65..0303e29749f 100644 --- a/python/paddle/distributed/auto_parallel/cost/tensor_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/tensor_cost.py @@ -22,7 +22,6 @@ from .base_cost import Cost class TensorCost: - def __init__(self, tensor=None, dist_tensor=None, shape=None, dtype=None): self._check_args(tensor, dist_tensor, shape, dtype) self._tensor = tensor @@ -49,31 +48,35 @@ class TensorCost: def _check_args(self, tensor, dist_tensor, shape, dtype): if tensor is not None: - assert (shape is None and dist_tensor is None and dtype is None) + assert shape is None and dist_tensor is None and dtype is None if not isinstance(tensor, Variable): raise TypeError( "Please check tensor type is Variable, but got {}".format( - type(tensor))) + type(tensor) + ) + ) elif dist_tensor is not None: - assert (tensor is None and shape is None) + assert tensor is None and shape is None if not isinstance(dist_tensor, DistributedTensor): raise TypeError( - "Please check dist_tensor type is DistributedTensor, but got {}" - .format(type(dist_tensor))) + "Please check dist_tensor type is DistributedTensor, but got {}".format( + type(dist_tensor) + ) + ) elif shape is not None: - assert (tensor is None and dist_tensor is None - and dtype is not None) + assert tensor is None and dist_tensor is None and dtype is not None if not isinstance(shape, (list, set)): raise TypeError( "Please check shape type is list or set, but got {}".format( - type(shape))) + type(shape) + ) + ) elif dtype is not None: - assert (tensor is None and dist_tensor is None - and shape is not None) + assert tensor is None and dist_tensor is None and shape is not None @property def cost(self): diff --git a/python/paddle/distributed/auto_parallel/cost_model.py b/python/paddle/distributed/auto_parallel/cost_model.py index cdcc19e27d7..dfff44d0bde 100644 --- a/python/paddle/distributed/auto_parallel/cost_model.py +++ b/python/paddle/distributed/auto_parallel/cost_model.py @@ -36,7 +36,6 @@ class CostNodeType(Enum): class Cost(object): - def __init__(self): self.runtime = None self.static_mem = None @@ -51,7 +50,6 @@ class CostModelMode(Enum): class CostNode(object): - def __init__(self, node, node_type, id=None): self.id = id self.node = node @@ -72,7 +70,6 @@ class CostNode(object): class MergedOpsCostNode(CostNode): - def __init__(self, node_type, id=None, base_node_list=None, is_bwd=False): super(MergedOpsCostNode, self).__init__(None, node_type, id) self.node_list = base_node_list @@ -80,13 +77,9 @@ class MergedOpsCostNode(CostNode): class CommOpCostNode(CostNode): - - def __init__(self, - node, - node_type, - id=None, - comm_node_list=None, - is_bwd=False): + def __init__( + self, node, node_type, id=None, comm_node_list=None, is_bwd=False + ): super(CommOpCostNode, self).__init__(node, node_type, id) self.node_list = comm_node_list self.ranks = [] @@ -108,8 +101,9 @@ class CommOpCostNode(CostNode): comm_volumn = np.prod(self.input_shape) * 4 if 'allreduce' in self.comm_type: - self._cost = comm_volumn / (BANDWIDTH * num_ranks / - (2 * (num_ranks - 1))) + self._cost = comm_volumn / ( + BANDWIDTH * num_ranks / (2 * (num_ranks - 1)) + ) elif 'gather' in self.comm_type: self._cost = comm_volumn / (BANDWIDTH * num_ranks / (num_ranks - 1)) elif 'broadcast' in self.comm_type: @@ -121,14 +115,15 @@ class CommOpCostNode(CostNode): class TensorCostNode(CostNode): - - def __init__(self, - node, - node_type, - id=None, - base_node_list=None, - batch_size=None, - shared_node_id=None): + def __init__( + self, + node, + node_type, + id=None, + base_node_list=None, + batch_size=None, + shared_node_id=None, + ): super(TensorCostNode, self).__init__(node, node_type, id) if node.name == "create_py_reader_0" or node.name == "double_buffer_0": self.shape = [2, 2] @@ -163,7 +158,6 @@ class TensorCostNode(CostNode): class CompOpCostNode(CostNode): - def __init__(self, node, node_type, id=None, is_bwd=False, is_optim=False): super(CompOpCostNode, self).__init__(node, node_type, id) self.is_bwd = is_bwd @@ -179,7 +173,6 @@ class CompOpCostNode(CostNode): class PipeEvent(object): - def __init__(self, stage_id, event_name, duration, start_time=-1): self.stage_id = stage_id self.name = event_name @@ -189,15 +182,16 @@ class PipeEvent(object): class CostModel(object): - - def __init__(self, - mode=CostModelMode.BENCHMARKING, - cluster=None, - batch_size=1, - microbatch_num=1, - opcall_overhead=0, - standalone_cost_data=None, - pipeline_config=None): + def __init__( + self, + mode=CostModelMode.BENCHMARKING, + cluster=None, + batch_size=1, + microbatch_num=1, + opcall_overhead=0, + standalone_cost_data=None, + pipeline_config=None, + ): self.mode = mode # parameters @@ -229,17 +223,20 @@ class CostModel(object): self.optim_time = [] def _parse_sub_program(self, program, nodes, graph, cost_data, sub_idx): - assert len( - program.blocks) == 1, "Program more than 1 block not supported." + assert ( + len(program.blocks) == 1 + ), "Program more than 1 block not supported." block = program.blocks[0] var_id = "lod_tensor_blocking_queue_0" new_var = program.global_block().create_var( name=var_id, dtype=paddle.float32, - type=core.VarDesc.VarType.LOD_TENSOR) - nodes[var_id] = TensorCostNode(new_var, CostNodeType.VARIABLE, - "lod_tensor_blocking_queue_0") + type=core.VarDesc.VarType.LOD_TENSOR, + ) + nodes[var_id] = TensorCostNode( + new_var, CostNodeType.VARIABLE, "lod_tensor_blocking_queue_0" + ) for var in block.vars.values(): var_id = var.name # if var.name == "create_py_reader_0" or var.name == "double_buffer_0": @@ -249,13 +246,17 @@ class CostModel(object): for op in block.ops: op_id = op.type + "_" + str(op.idx) - if op.type.startswith('c_') or op.type.startswith( - 'send') or op.type.startswith('recv'): + if ( + op.type.startswith('c_') + or op.type.startswith('send') + or op.type.startswith('recv') + ): is_bwd = False - if op.type.startswith( - 'c_' - ) and op.type != "c_sync_calc_stream" and not op.type.startswith( - 'c_embedding'): + if ( + op.type.startswith('c_') + and op.type != "c_sync_calc_stream" + and not op.type.startswith('c_embedding') + ): ring_id = op.attr('ring_id') if ring_id not in self.ring2rank: self.ring2rank[ring_id] = set() @@ -265,14 +266,17 @@ class CostModel(object): is_bwd = '@GRAD' in op.output('Out')[0] elif op.type.startswith('send'): is_bwd = '@GRAD' in op.input('X')[0] - op_node = CommOpCostNode(op, CostNodeType.COMMUNICATION, op_id, - is_bwd) + op_node = CommOpCostNode( + op, CostNodeType.COMMUNICATION, op_id, is_bwd + ) else: - is_bwd = (int(op.attr('op_role')) == int( - OpRole.Backward)) or "@GRAD" in op.input_arg_names + is_bwd = ( + int(op.attr('op_role')) == int(OpRole.Backward) + ) or "@GRAD" in op.input_arg_names is_optim = 'LearningRate' in op.input_names - op_node = CompOpCostNode(op, CostNodeType.COMPUTATION, op_id, - is_bwd, is_optim) + op_node = CompOpCostNode( + op, CostNodeType.COMPUTATION, op_id, is_bwd, is_optim + ) op_node.init_comp_cost(cost_data) nodes[op_id] = op_node @@ -310,17 +314,20 @@ class CostModel(object): for pred_id in graph[node_id][PRED]: pred = nodes[pred_id] if pred.type == CostNodeType.COMPUTATION and ( - pred_id in graph[node_id][SUCC]): + pred_id in graph[node_id][SUCC] + ): graph[pred_id][SUCC].remove(node_id) graph[node_id][PRED].remove(pred_id) write_op_cnt += 1 new_var_id = node_id + '_write_{}'.format(write_op_cnt) - new_var = TensorCostNode(node.node, - CostNodeType.VARIABLE, - new_var_id, - shared_node_id=node_id) + new_var = TensorCostNode( + node.node, + CostNodeType.VARIABLE, + new_var_id, + shared_node_id=node_id, + ) graph[new_var_id] = [[], []] graph[pred_id][SUCC].append(new_var_id) @@ -346,23 +353,31 @@ class CostModel(object): self.op_graph.append({}) self.runtime_graph.append({}) self._parse_sub_program( - sub_prog, self.nodes[sub_idx], self.origin_graph[sub_idx], - self.cost_data[0 if self.rank2pp is None else self. - rank2pp[sub_idx]], sub_idx) + sub_prog, + self.nodes[sub_idx], + self.origin_graph[sub_idx], + self.cost_data[ + 0 if self.rank2pp is None else self.rank2pp[sub_idx] + ], + sub_idx, + ) return self.nodes def _find_succ_op(self, node_id, sub_idx=0): succ_ops_id = [] for succ_id in self.origin_graph[sub_idx][node_id][SUCC]: succ = self.nodes[sub_idx][succ_id] - if succ.type == CostNodeType.COMMUNICATION or \ - succ.type == CostNodeType.COMPUTATION: + if ( + succ.type == CostNodeType.COMMUNICATION + or succ.type == CostNodeType.COMPUTATION + ): succ_ops_id.append(succ_id) elif succ.type == CostNodeType.VARIABLE: succ_ops_id = succ_ops_id + self._find_succ_op(succ_id, sub_idx) else: raise NotImplementedError( - 'This type of node not supported yet:{}'.format(succ.type)) + 'This type of node not supported yet:{}'.format(succ.type) + ) return succ_ops_id def build_op_graph(self): @@ -392,9 +407,11 @@ class CostModel(object): for sub_idx in range(self.total_rank): for node_id, edges in self.op_graph[sub_idx].items(): node = self.nodes[sub_idx][node_id] - if node_id.startswith('c_') and not node.id.startswith( - "c_sync_calc_stream") and not node.id.startswith( - 'c_embedding'): + if ( + node_id.startswith('c_') + and not node.id.startswith("c_sync_calc_stream") + and not node.id.startswith('c_embedding') + ): ring_id = node.node.attr('ring_id') node.set_ranks(list(self.ring2rank[ring_id])) node.init_comm_cost(self.cluster) @@ -420,13 +437,17 @@ class CostModel(object): else: raise NotImplementedError( 'This type of merging is not supported:{}'.format( - merge_type)) + merge_type + ) + ) merged_node_id = 'merged_' + str(len(nodes)) is_bwd = to_merge_node_list[0].is_bwd - merged_node = MergedOpsCostNode(CostNodeType.MERGED, - id=merged_node_id, - base_node_list=nodes_list, - is_bwd=is_bwd) + merged_node = MergedOpsCostNode( + CostNodeType.MERGED, + id=merged_node_id, + base_node_list=nodes_list, + is_bwd=is_bwd, + ) merged_node.cost = node_cost return merged_node_id, merged_node @@ -440,12 +461,12 @@ class CostModel(object): ''' cnt = 0 for sub_idx in range(self.total_rank): - cnt += self._merge_linear(self.nodes[sub_idx], - self.runtime_graph[sub_idx], - is_bwd=False) - cnt += self._merge_linear(self.nodes[sub_idx], - self.runtime_graph[sub_idx], - is_bwd=True) + cnt += self._merge_linear( + self.nodes[sub_idx], self.runtime_graph[sub_idx], is_bwd=False + ) + cnt += self._merge_linear( + self.nodes[sub_idx], self.runtime_graph[sub_idx], is_bwd=True + ) return cnt def merge_branch(self): @@ -461,12 +482,12 @@ class CostModel(object): ''' cnt = 0 for sub_idx in range(self.total_rank): - cnt += self._merge_branch(self.nodes[sub_idx], - self.runtime_graph[sub_idx], - is_bwd=False) - cnt += self._merge_branch(self.nodes[sub_idx], - self.runtime_graph[sub_idx], - is_bwd=True) + cnt += self._merge_branch( + self.nodes[sub_idx], self.runtime_graph[sub_idx], is_bwd=False + ) + cnt += self._merge_branch( + self.nodes[sub_idx], self.runtime_graph[sub_idx], is_bwd=True + ) return cnt def _merge_linear(self, nodes, runtime_graph, is_bwd=False): @@ -484,7 +505,8 @@ class CostModel(object): pred_id = edges[PRED][0] pred = nodes[pred_id] merged_node_id, merged_node = self._merge_node( - [node, pred], merge_type='linear', nodes=nodes) + [node, pred], merge_type='linear', nodes=nodes + ) nodes[merged_node_id] = merged_node runtime_graph[merged_node_id] = [[], []] @@ -492,7 +514,8 @@ class CostModel(object): succ = None try: runtime_graph[merged_node_id][SUCC] = copy.deepcopy( - edges[SUCC]) + edges[SUCC] + ) if len(runtime_graph[pred_id][SUCC]) > 1: # predecessor has more than 1 successor @@ -501,7 +524,8 @@ class CostModel(object): succ.remove(node_id) runtime_graph[merged_node_id][SUCC] += succ runtime_graph[merged_node_id][PRED] = runtime_graph[ - pred_id][PRED] + pred_id + ][PRED] except: pass try: @@ -567,16 +591,20 @@ class CostModel(object): to_merge = True try: - if len(edges[SUCC]) < 1 or len( - runtime_graph[edges[SUCC][0]][SUCC]) < 1: + if ( + len(edges[SUCC]) < 1 + or len(runtime_graph[edges[SUCC][0]][SUCC]) < 1 + ): continue except: continue end_node_id = runtime_graph[edges[SUCC][0]][SUCC][0] for i in succ_nodes_id: try: - if len(runtime_graph[i][SUCC]) != 1 or \ - runtime_graph[i][SUCC][0] != end_node_id: + if ( + len(runtime_graph[i][SUCC]) != 1 + or runtime_graph[i][SUCC][0] != end_node_id + ): to_merge = False # if branches has different end node, we don't merge them break except: @@ -584,7 +612,8 @@ class CostModel(object): if to_merge and len(succ_nodes_id) > 1: to_merge_node_list = [nodes[i] for i in succ_nodes_id] merged_node_id, merged_node = self._merge_node( - to_merge_node_list, merge_type='branch', nodes=nodes) + to_merge_node_list, merge_type='branch', nodes=nodes + ) nodes[merged_node_id] = merged_node runtime_graph[merged_node_id] = [[], []] @@ -605,7 +634,6 @@ class CostModel(object): return reduct_cnt def get_runtime_cost(self): - def get_node_cost(node): node_cost = node.cost + self.opcall_overhead if isinstance(node, MergedOpsCostNode): @@ -635,7 +663,8 @@ class CostModel(object): top_list = [] for sub_idx in range(self.total_rank): static_mem, cur_mem, top_mem = self._simulate_mem( - self.nodes[sub_idx], self.origin_graph[sub_idx]) + self.nodes[sub_idx], self.origin_graph[sub_idx] + ) static_list.append(static_mem) top_list.append(top_mem) return static_list, top_list @@ -670,8 +699,9 @@ class CostModel(object): static_mem += size cur_mem += size edges = sim_graph[node_id] - if not (node.type == CostNodeType.VARIABLE - and node.node.persistable): + if not ( + node.type == CostNodeType.VARIABLE and node.node.persistable + ): for succ_id in edges[SUCC]: sim_graph[succ_id][PRED].remove(node_id) if len(sim_graph[succ_id][PRED]) == 0: @@ -680,8 +710,10 @@ class CostModel(object): pred = nodes if pred.type == CostNodeType.VARIABLE: sim_graph[pred_id][SUCC].remove(node_id) - if len(sim_graph[pred_id] - [SUCC]) == 0 and not pred.node.persistable: + if ( + len(sim_graph[pred_id][SUCC]) == 0 + and not pred.node.persistable + ): cur_mem -= pred.get_size() return static_mem, cur_mem, top_mem @@ -713,16 +745,22 @@ class CostModel(object): event_list.append(e) if stid != stage_num - 1: q.put( - PipeEvent(stid + 1, - 'fwd', - self.fwd_time[stid + 1], - start_time=e.e_time)) + PipeEvent( + stid + 1, + 'fwd', + self.fwd_time[stid + 1], + start_time=e.e_time, + ) + ) else: q.put( - PipeEvent(stid, - 'bwd', - self.bwd_time[stid], - start_time=e.e_time)) + PipeEvent( + stid, + 'bwd', + self.bwd_time[stid], + start_time=e.e_time, + ) + ) fwd_cnt[stid] -= 1 global_time[stid] = e.e_time else: @@ -733,18 +771,24 @@ class CostModel(object): event_list.append(e) if stid != 0: q.put( - PipeEvent(stid - 1, - 'bwd', - self.bwd_time[stid - 1], - start_time=e.e_time)) + PipeEvent( + stid - 1, + 'bwd', + self.bwd_time[stid - 1], + start_time=e.e_time, + ) + ) fwd_cnt[stid] += 1 bwd_cnt[stid] -= 1 if bwd_cnt[stid] == 0: q.put( - PipeEvent(stid, - 'optim', - self.optim_time[stid], - start_time=e.e_time)) + PipeEvent( + stid, + 'optim', + self.optim_time[stid], + start_time=e.e_time, + ) + ) global_time[stid] = e.e_time elif e.name == 'optim': e.s_time = max(global_time[stid], e.s_time) @@ -754,7 +798,9 @@ class CostModel(object): else: raise NotImplementedError( 'This type of pipe event is not supported yet.{}'.format( - e.name)) + e.name + ) + ) for t in global_time: total_time = max(total_time, t) @@ -784,8 +830,13 @@ class CostModel(object): self.build_runtime_graph() -def estimate_cost(distributed_program, cluster, pipeline_config, - standalone_cost_data, batch_size): +def estimate_cost( + distributed_program, + cluster, + pipeline_config, + standalone_cost_data, + batch_size, +): """ Estimated cost from distributed program, cluster model and distributed settings. @@ -798,10 +849,12 @@ def estimate_cost(distributed_program, cluster, pipeline_config, """ # the following line is left for now, cluster model will be involved in the future assert cluster is None, "For now, cluster remains None" - cm_ctx = CostModel(cluster=cluster, - batch_size=batch_size, - standalone_cost_data=standalone_cost_data, - pipeline_config=pipeline_config) + cm_ctx = CostModel( + cluster=cluster, + batch_size=batch_size, + standalone_cost_data=standalone_cost_data, + pipeline_config=pipeline_config, + ) cm_ctx.init(distributed_program) cost = cm_ctx.get_cost() return cost diff --git a/python/paddle/distributed/auto_parallel/dist_attribute.py b/python/paddle/distributed/auto_parallel/dist_attribute.py index 04af4ad77e5..19aaa07f24d 100644 --- a/python/paddle/distributed/auto_parallel/dist_attribute.py +++ b/python/paddle/distributed/auto_parallel/dist_attribute.py @@ -17,11 +17,17 @@ from paddle.fluid.framework import Variable from .process_mesh import ProcessMesh _g_tensor_dist_attr_field_keys = [ - "process_mesh", "dims_mapping", "shard_sizes", "device_placement" + "process_mesh", + "dims_mapping", + "shard_sizes", + "device_placement", ] _g_op_dist_attr_field_keys = [ - "process_mesh", "impl_type", "impl_idx", "is_recompute" + "process_mesh", + "impl_type", + "impl_idx", + "is_recompute", ] _g_op_input_suffix = "@input" @@ -50,7 +56,6 @@ def append_op_output_suffix(name): class TensorDistributedAttribute: - def __init__(self): # The process mesh of distributed operator attribute must is the same as # the process meshes of all input and output distributed attributed @@ -67,8 +72,9 @@ class TensorDistributedAttribute: @process_mesh.setter def process_mesh(self, process_mesh): if process_mesh is not None: - assert isinstance(process_mesh, (list, ProcessMesh)), \ - "The type of process_mesh must be list or ProcessMesh." + assert isinstance( + process_mesh, (list, ProcessMesh) + ), "The type of process_mesh must be list or ProcessMesh." if isinstance(process_mesh, list): process_mesh = ProcessMesh(process_mesh) self._process_mesh = copy.deepcopy(process_mesh) @@ -80,12 +86,15 @@ class TensorDistributedAttribute: @dims_mapping.setter def dims_mapping(self, dims_mapping): if dims_mapping is not None: - assert isinstance(dims_mapping, list), \ - "The type of dims_mapping must be list." - assert all(isinstance(x, int) for x in dims_mapping), \ - ("All elements of dims_mapping must be integer") - assert all(x >= -1 for x in dims_mapping), \ - ("All elements of dims_mapping must be greater than or equal to -1.") + assert isinstance( + dims_mapping, list + ), "The type of dims_mapping must be list." + assert all( + isinstance(x, int) for x in dims_mapping + ), "All elements of dims_mapping must be integer" + assert all( + x >= -1 for x in dims_mapping + ), "All elements of dims_mapping must be greater than or equal to -1." self._dims_mapping = copy.deepcopy(dims_mapping) @property @@ -109,37 +118,44 @@ class TensorDistributedAttribute: def init(self, dist_attr): if dist_attr is None: return - assert isinstance(dist_attr, (dict, TensorDistributedAttribute)), \ - "The type of dist_attr must be dict or TensorDistributedAttribute." + assert isinstance( + dist_attr, (dict, TensorDistributedAttribute) + ), "The type of dist_attr must be dict or TensorDistributedAttribute." if isinstance(dist_attr, dict): for key, value in dist_attr.items(): if key in get_tensor_dist_attr_field_keys(): field_property = TensorDistributedAttribute.__dict__.get( - key, None) + key, None + ) if field_property: field_property.fset(self, value) else: assert False, "No setter for {} in args {}.".format( - key, dist_attr) + key, dist_attr + ) elif isinstance(dist_attr, TensorDistributedAttribute): for key in get_tensor_dist_attr_field_keys(): field_property = TensorDistributedAttribute.__dict__.get( - key, None) + key, None + ) if field_property: field_property.fset(self, field_property.fget(dist_attr)) else: assert False, "No setter for {} in args {}.".format( - key, dist_attr) + key, dist_attr + ) self._is_annotated = copy.deepcopy(dist_attr._is_annotated) def reset(self, skip_dist_attr_field_names=None): - if skip_dist_attr_field_names is None or \ - (skip_dist_attr_field_names is not None \ - and "process_mesh" not in skip_dist_attr_field_names): + if skip_dist_attr_field_names is None or ( + skip_dist_attr_field_names is not None + and "process_mesh" not in skip_dist_attr_field_names + ): self._process_mesh = None - if skip_dist_attr_field_names is None or \ - (skip_dist_attr_field_names is not None \ - and "dims_mapping" not in skip_dist_attr_field_names): + if skip_dist_attr_field_names is None or ( + skip_dist_attr_field_names is not None + and "dims_mapping" not in skip_dist_attr_field_names + ): for i, _ in enumerate(self._dims_mapping): self._dims_mapping[i] = -1 self._is_annotated = {} @@ -160,8 +176,9 @@ class TensorDistributedAttribute: def mark_annotated_as(self, dist_attr): if dist_attr is None: return - assert isinstance(dist_attr, (dict, TensorDistributedAttribute)), \ - "The type of dist_attr must be dict or TensorDistributedAttribute." + assert isinstance( + dist_attr, (dict, TensorDistributedAttribute) + ), "The type of dist_attr must be dict or TensorDistributedAttribute." if isinstance(dist_attr, dict): for key in dist_attr.keys(): if key in get_tensor_dist_attr_field_keys(): @@ -189,21 +206,22 @@ class TensorDistributedAttribute: annotated_str = "annotated" else: annotated_str = "non-annotated" - str += "\n\t\tprocess_mesh ({}): {},".format(annotated_str, - self.process_mesh) + str += "\n\t\tprocess_mesh ({}): {},".format( + annotated_str, self.process_mesh + ) if self.is_annotated("dims_mapping"): annotated_str = "annotated" else: annotated_str = "non-annotated" - str += "\n\t\tdims_mapping ({}): {}".format(annotated_str, - self.dims_mapping) + str += "\n\t\tdims_mapping ({}): {}".format( + annotated_str, self.dims_mapping + ) str += "\n\t}" return str class OperatorDistributedAttribute: - def __init__(self): self._process_mesh = None self._op_type = None @@ -221,8 +239,9 @@ class OperatorDistributedAttribute: @process_mesh.setter def process_mesh(self, process_mesh): if process_mesh is not None: - assert isinstance(process_mesh, (list, ProcessMesh)), \ - "The type of process_mesh must be list or ProcessMesh." + assert isinstance( + process_mesh, (list, ProcessMesh) + ), "The type of process_mesh must be list or ProcessMesh." if isinstance(process_mesh, list): process_mesh = ProcessMesh(process_mesh) self._process_mesh = copy.deepcopy(process_mesh) @@ -337,8 +356,9 @@ class OperatorDistributedAttribute: def init(self, dist_attr): if dist_attr is None: return - assert isinstance(dist_attr, (dict, OperatorDistributedAttribute)), \ - "The type of dist_attr must be dict or OperatorDistributedAttribute." + assert isinstance( + dist_attr, (dict, OperatorDistributedAttribute) + ), "The type of dist_attr must be dict or OperatorDistributedAttribute." if isinstance(dist_attr, dict): for key, value in dist_attr.items(): if isinstance(key, Variable): @@ -350,31 +370,41 @@ class OperatorDistributedAttribute: self.set_output_dist_attr(key.name, tensor_dist_attr) else: if key in get_op_dist_attr_field_keys(): - field_property = OperatorDistributedAttribute.__dict__.get( - key, None) + field_property = ( + OperatorDistributedAttribute.__dict__.get(key, None) + ) if field_property: field_property.fset(self, value) else: assert False, "No setter for {} in args {}.".format( - key, dist_attr) + key, dist_attr + ) elif isinstance(dist_attr, OperatorDistributedAttribute): - for tensor_name, tensor_dist_attr in dist_attr.inputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in dist_attr.inputs_dist_attrs.items(): self.set_input_dist_attr( - tensor_name, dist_attr.get_input_dist_attr(tensor_name)) - for tensor_name, tensor_dist_attr in dist_attr.outputs_dist_attrs.items( - ): + tensor_name, dist_attr.get_input_dist_attr(tensor_name) + ) + for ( + tensor_name, + tensor_dist_attr, + ) in dist_attr.outputs_dist_attrs.items(): self.set_output_dist_attr( - tensor_name, dist_attr.get_output_dist_attr(tensor_name)) + tensor_name, dist_attr.get_output_dist_attr(tensor_name) + ) self._is_annotated = copy.deepcopy(dist_attr._is_annotated) for key in get_op_dist_attr_field_keys(): field_property = OperatorDistributedAttribute.__dict__.get( - key, None) + key, None + ) if field_property: field_property.fset(self, field_property.fget(dist_attr)) else: assert False, "No setter for {} in args {}.".format( - key, dist_attr) + key, dist_attr + ) # Make sure proscess_meshes in dist op be same if self.op_type == "while": return None @@ -390,8 +420,9 @@ class OperatorDistributedAttribute: if shared_process_mesh is None: shared_process_mesh = process_mesh else: - assert process_mesh == shared_process_mesh, \ - "ProcessMeshes in DistributedOperator must be the same." + assert ( + process_mesh == shared_process_mesh + ), "ProcessMeshes in DistributedOperator must be the same." self.process_mesh = shared_process_mesh def reset(self, skip_dist_attr_field_names=None): @@ -399,9 +430,10 @@ class OperatorDistributedAttribute: tensor_dist_attr.reset(skip_dist_attr_field_names) for tensor_dist_attr in self.outputs_dist_attrs.values(): tensor_dist_attr.reset(skip_dist_attr_field_names) - if skip_dist_attr_field_names is None or \ - (skip_dist_attr_field_names is not None \ - and "process_mesh" not in skip_dist_attr_field_names): + if skip_dist_attr_field_names is None or ( + skip_dist_attr_field_names is not None + and "process_mesh" not in skip_dist_attr_field_names + ): self._process_mesh = None self.impl_type = "default" self.impl_idx = 0 @@ -428,8 +460,9 @@ class OperatorDistributedAttribute: def mark_annotated_as(self, dist_attr): if dist_attr is None: return - assert isinstance(dist_attr, (dict, OperatorDistributedAttribute)), \ - "The type of dist_attr must be dict or OperatorDistributedAttribute." + assert isinstance( + dist_attr, (dict, OperatorDistributedAttribute) + ), "The type of dist_attr must be dict or OperatorDistributedAttribute." if isinstance(dist_attr, dict): for key, value in dist_attr.items(): if isinstance(key, Variable): @@ -458,15 +491,19 @@ class OperatorDistributedAttribute: self._is_annotated = copy.deepcopy(dist_attr._is_annotated) if self.is_annotated("process_mesh"): process_mesh_annotated = True - for tensor_name, tensor_dist_attr in dist_attr.inputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in dist_attr.inputs_dist_attrs.items(): input_dist_attr = self.get_input_dist_attr(tensor_name) if input_dist_attr is not None: input_dist_attr.mark_annotated_as(tensor_dist_attr) if input_dist_attr.is_annotated("process_mesh"): process_mesh_annotated = True - for tensor_name, tensor_dist_attr in dist_attr.outputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in dist_attr.outputs_dist_attrs.items(): output_dist_attr = self.get_output_dist_attr(tensor_name) if output_dist_attr is not None: output_dist_attr.mark_annotated_as(tensor_dist_attr) @@ -523,8 +560,9 @@ class OperatorDistributedAttribute: annotated_str = "annotated" else: annotated_str = "non-annotated" - str += "\n\t\tprocess_mesh ({}): {},".format(annotated_str, - self.process_mesh) + str += "\n\t\tprocess_mesh ({}): {},".format( + annotated_str, self.process_mesh + ) for arg_name, tensor_dist_attr in self.inputs_dist_attrs.items(): str += "\n\t\t{}'s (input): {},".format(arg_name, tensor_dist_attr) diff --git a/python/paddle/distributed/auto_parallel/dist_context.py b/python/paddle/distributed/auto_parallel/dist_context.py index 008c4b98707..12c224ac27f 100644 --- a/python/paddle/distributed/auto_parallel/dist_context.py +++ b/python/paddle/distributed/auto_parallel/dist_context.py @@ -50,15 +50,17 @@ class DistributedContext: One auto-parallel run should use its own DistributedContext to avoid interfering other run. """ - def __init__(self, - serial_main_prog=None, - serial_startup_prog=None, - serial_optimizer=None, - serial_loss=None, - feed_vars={}, - fetch_vars={}, - cluster=None, - strategy=None): + def __init__( + self, + serial_main_prog=None, + serial_startup_prog=None, + serial_optimizer=None, + serial_loss=None, + feed_vars={}, + fetch_vars={}, + cluster=None, + strategy=None, + ): # Data members related to original programs (unchanged) self._original_serial_main_program = serial_main_prog self._original_serial_startup_program = serial_startup_prog @@ -107,7 +109,7 @@ class DistributedContext: # self._tensor_id_to_tensor_node_ids = {} self._is_initialized = False - #TODO: need a better way to remove the following flag + # TODO: need a better way to remove the following flag self._need_copy_dist_attr_to_graph = False self._backup_pass_context_stack = [] self._backup_block_state_stack = [] @@ -189,7 +191,8 @@ class DistributedContext: @property def has_annotation(self): return len(self._dist_tensors_for_program) or len( - self._dist_ops_for_program) + self._dist_ops_for_program + ) @property def gradient_scale(self): @@ -209,18 +212,23 @@ class DistributedContext: def _backup_serial_info(self, mode): self._backup_serial_main_program_stack.append( - self._serial_main_program.clone()) + self._serial_main_program.clone() + ) self._backup_serial_startup_program_stack.append( - self._serial_startup_program.clone()) - self._backup_pass_context_stack.append(copy.deepcopy( - self._pass_context)) + self._serial_startup_program.clone() + ) + self._backup_pass_context_stack.append( + copy.deepcopy(self._pass_context) + ) self._backup_block_state_stack.append(copy.deepcopy(self._block_state)) def _backup_dist_info(self, mode): self._backup_dist_tensors_for_program_stack.append( - copy.deepcopy(self._dist_tensors_for_program)) + copy.deepcopy(self._dist_tensors_for_program) + ) self._backup_dist_ops_for_program_stack.append( - copy.deepcopy(self._dist_ops_for_program)) + copy.deepcopy(self._dist_ops_for_program) + ) def _backup(self, serial=True, serial_mode=None, dist=True, dist_mode=None): # Use this function carefully @@ -237,7 +245,8 @@ class DistributedContext: block_idx = loss.block.idx var_name = loss.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) self._serial_loss = var elif len(self._original_serial_loss) == 0: self._serial_loss = [] @@ -247,7 +256,8 @@ class DistributedContext: block_idx = self._original_serial_loss.block.idx var_name = self._original_serial_loss.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) self._serial_loss = var def _restore_serial_feed_vars(self): @@ -257,7 +267,8 @@ class DistributedContext: block_idx = var.block.idx var_name = var.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_var_list.append(var) self._serial_feed_vars[key] = new_var_list @@ -272,7 +283,8 @@ class DistributedContext: block_idx = var.block.idx var_name = var.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_inner_var_list.append(var) new_var_list.append(new_inner_var_list) else: @@ -280,22 +292,27 @@ class DistributedContext: block_idx = var.block.idx var_name = var.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_var_list.append(var) self._serial_fetch_vars[key] = new_var_list def _restore_serial_info(self, mode="to_backup"): if mode == "to_backup": - self._serial_main_program = self._backup_serial_main_program_stack.pop( + self._serial_main_program = ( + self._backup_serial_main_program_stack.pop() ) - self._serial_startup_program = self._backup_serial_startup_program_stack.pop( + self._serial_startup_program = ( + self._backup_serial_startup_program_stack.pop() ) elif mode == "to_original": assert self._original_serial_main_program is not None assert self._original_serial_startup_program is not None - self._serial_main_program = self._original_serial_main_program.clone( + self._serial_main_program = ( + self._original_serial_main_program.clone() ) - self._serial_startup_program = self._original_serial_startup_program.clone( + self._serial_startup_program = ( + self._original_serial_startup_program.clone() ) self._restore_serial_loss() @@ -307,21 +324,27 @@ class DistributedContext: def _restore_dist_info(self, mode="to_backup"): if mode == "to_backup": - self._dist_tensors_for_program = self._backup_dist_tensors_for_program_stack.pop( + self._dist_tensors_for_program = ( + self._backup_dist_tensors_for_program_stack.pop() ) - self._dist_ops_for_program = self._backup_dist_ops_for_program_stack.pop( + self._dist_ops_for_program = ( + self._backup_dist_ops_for_program_stack.pop() ) elif mode == "to_original": assert self._original_dist_tensors_for_program assert self._original_dist_ops_for_program self._dist_tensors_for_program = copy.deepcopy( - self._original_dist_tensors_for_program) + self._original_dist_tensors_for_program + ) self._dist_ops_for_program = copy.deepcopy( - self._original_dist_ops_for_program) + self._original_dist_ops_for_program + ) elif mode == "to_default": new_tensors_ids = [] - for tensor_id, dist_tensor in self._dist_tensors_for_program.items( - ): + for ( + tensor_id, + dist_tensor, + ) in self._dist_tensors_for_program.items(): if tensor_id in self._tensors_ids: dist_tensor.dist_attr.reset() else: @@ -338,8 +361,10 @@ class DistributedContext: self._dist_ops_for_program.pop(op_id) else: new_tensors_ids = [] - for tensor_id, dist_tensor in self._dist_tensors_for_program.items( - ): + for ( + tensor_id, + dist_tensor, + ) in self._dist_tensors_for_program.items(): new_tensors_ids.append(tensor_id) for tensor_id in new_tensors_ids: self._dist_tensors_for_program.pop(tensor_id) @@ -354,11 +379,13 @@ class DistributedContext: self._need_copy_dist_attr_to_graph = True self._process_meshes = [] - def _restore(self, - serial=True, - serial_mode="to_backup", - dist=True, - dist_mode="to_backup"): + def _restore( + self, + serial=True, + serial_mode="to_backup", + dist=True, + dist_mode="to_backup", + ): # Use this function carefully if serial: self._restore_serial_info(serial_mode) @@ -369,11 +396,13 @@ class DistributedContext: if not self._is_initialized: if not self._serial_main_program: if self._original_serial_main_program: - self._serial_main_program = self._original_serial_main_program.clone( + self._serial_main_program = ( + self._original_serial_main_program.clone() ) if not self._serial_startup_program: if self._original_serial_startup_program: - self._serial_startup_program = self._original_serial_startup_program.clone( + self._serial_startup_program = ( + self._original_serial_startup_program.clone() ) if not self._serial_loss: self._restore_serial_loss() @@ -387,9 +416,11 @@ class DistributedContext: self._init_dist_attr_for_program() # Backup the original distributed information for later restore self._original_dist_tensors_for_program = copy.deepcopy( - self._dist_tensors_for_program) + self._dist_tensors_for_program + ) self._original_dist_ops_for_program = copy.deepcopy( - self._dist_ops_for_program) + self._dist_ops_for_program + ) self._tensors_ids = list(self._dist_tensors_for_program.keys()) self._ops_ids = list(self._dist_ops_for_program.keys()) self._is_initialized = True @@ -397,7 +428,8 @@ class DistributedContext: if with_graph: set_flags({"FLAGS_convert_all_blocks": True}) self._serial_graph = framework.IrGraph( - core.Graph(self._serial_main_program.desc)) + core.Graph(self._serial_main_program.desc) + ) self._init_dist_attr_for_graph() self._need_copy_dist_attr_to_graph = False @@ -405,8 +437,9 @@ class DistributedContext: self.copy_dist_attr_from_program_to_graph() def add_process_mesh(self, process_mesh): - assert isinstance(process_mesh, ProcessMesh), \ - 'The type of dim_mapping must be ProcessMesh.' + assert isinstance( + process_mesh, ProcessMesh + ), 'The type of dim_mapping must be ProcessMesh.' if process_mesh not in self.process_meshes: self._process_meshes.append(process_mesh) @@ -428,7 +461,8 @@ class DistributedContext: else: serial_tensor_id = serial_tensor.desc.original_id() dist_tensor = self._dist_tensors_for_program.get( - serial_tensor_id, None) + serial_tensor_id, None + ) if dist_tensor: return dist_tensor else: @@ -468,7 +502,8 @@ class DistributedContext: else: serial_tensor_id = serial_tensor.desc.original_id() dist_tensor = self._dist_tensors_for_program.get( - serial_tensor_id, None) + serial_tensor_id, None + ) if dist_tensor: return dist_tensor.dist_attr else: @@ -487,8 +522,9 @@ class DistributedContext: def get_tensor_dist_attr_for_graph(self, serial_tensor_node): serial_tensor_node_id = _node_id(serial_tensor_node) - dist_tensor = self._dist_tensors_for_graph.get(serial_tensor_node_id, - None) + dist_tensor = self._dist_tensors_for_graph.get( + serial_tensor_node_id, None + ) if dist_tensor: return dist_tensor.dist_attr else: @@ -530,7 +566,8 @@ class DistributedContext: if serial_node.is_var() and serial_node.var() is not None: serial_tensor_node_id = _node_id(serial_node) dist_tensor = self._dist_tensors_for_graph.get( - serial_tensor_node_id, None) + serial_tensor_node_id, None + ) if dist_tensor: return dist_tensor.dist_attr else: @@ -557,7 +594,8 @@ class DistributedContext: for tensor in block.vars.values(): # Copy the distributed tensors in the default context default_dist_tensor = default_ctx.get_dist_tensor_for_program( - tensor) + tensor + ) if default_dist_tensor and default_ctx is not self: self.add_dist_tensor_for_program(default_dist_tensor) current_dist_tensor = self.get_dist_tensor_for_program(tensor) @@ -574,12 +612,13 @@ class DistributedContext: dist_op = DistributedOperator(op) self.add_dist_op_for_program(dist_op) self._original_dist_tensors_for_program = copy.deepcopy( - self._dist_tensors_for_program) + self._dist_tensors_for_program + ) self._original_dist_ops_for_program = copy.deepcopy( - self._dist_ops_for_program) + self._dist_ops_for_program + ) def _order_nodes_by_program_order(self): - def _contains(nodes, target_node): for node in nodes: if _node_id(node) == _node_id(target_node): @@ -598,11 +637,14 @@ class DistributedContext: if node.is_op() and node.op() is not None: serial_ordered_op_nodes.append(node) serial_ordered_tensor_nodes.sort( - key=lambda node: node.node.original_desc_id()) + key=lambda node: node.node.original_desc_id() + ) serial_ordered_op_nodes.sort( - key=lambda node: node.node.original_desc_id()) + key=lambda node: node.node.original_desc_id() + ) num_nodes_before = len(serial_ordered_tensor_nodes) + len( - serial_ordered_op_nodes) + serial_ordered_op_nodes + ) new_serial_ordered_tensor_nodes = [] new_serial_ordered_op_nodes = [] @@ -610,9 +652,11 @@ class DistributedContext: for op_node in serial_ordered_op_nodes: tensor_nodes = [] for tensor_node in op_node.inputs: - if tensor_node.is_var() \ - and tensor_node.var() is not None \ - and not _contains(new_serial_ordered_nodes, tensor_node): + if ( + tensor_node.is_var() + and tensor_node.var() is not None + and not _contains(new_serial_ordered_nodes, tensor_node) + ): tensor_nodes.append(tensor_node) new_serial_ordered_tensor_nodes.append(tensor_node) tensor_nodes.sort(key=lambda node: node.node.original_desc_id()) @@ -621,23 +665,27 @@ class DistributedContext: new_serial_ordered_op_nodes.append(op_node) tensor_nodes = [] for tensor_node in op_node.outputs: - if tensor_node.is_var() \ - and tensor_node.var() is not None \ - and not _contains(new_serial_ordered_nodes, tensor_node): + if ( + tensor_node.is_var() + and tensor_node.var() is not None + and not _contains(new_serial_ordered_nodes, tensor_node) + ): tensor_nodes.append(tensor_node) new_serial_ordered_tensor_nodes.append(tensor_node) tensor_nodes.sort(key=lambda node: node.node.original_desc_id()) new_serial_ordered_nodes.extend(tensor_nodes) new_serial_ordered_tensor_nodes.sort( - key=lambda node: node.node.original_desc_id()) + key=lambda node: node.node.original_desc_id() + ) new_serial_ordered_op_nodes.sort( - key=lambda node: node.node.original_desc_id()) + key=lambda node: node.node.original_desc_id() + ) self._serial_ordered_tensor_nodes = new_serial_ordered_tensor_nodes self._serial_ordered_op_nodes = new_serial_ordered_op_nodes self._serial_ordered_nodes = new_serial_ordered_nodes assert len(self._serial_ordered_nodes) == len( - self._serial_ordered_tensor_nodes) + len( - self._serial_ordered_op_nodes) + self._serial_ordered_tensor_nodes + ) + len(self._serial_ordered_op_nodes) self._serial_orphan_tensor_nodes = [] for tensor_node in serial_ordered_tensor_nodes: if not _contains(self._serial_ordered_tensor_nodes, tensor_node): @@ -654,34 +702,49 @@ class DistributedContext: if node.is_var() and node.var() is not None: dist_tensor = None tensor_id = node.node.original_desc_id() - for cur_tensor_id, cur_dist_tensor in self._dist_tensors_for_program.items( - ): - if tensor_id == cur_tensor_id \ - or tensor_id == cur_dist_tensor.serial_tensor.desc.original_id(): + for ( + cur_tensor_id, + cur_dist_tensor, + ) in self._dist_tensors_for_program.items(): + if ( + tensor_id == cur_tensor_id + or tensor_id + == cur_dist_tensor.serial_tensor.desc.original_id() + ): dist_tensor = cur_dist_tensor - self._node_id_to_tensor_id[_node_id( - node)] = cur_tensor_id - assert dist_tensor is not None, \ - "Tensor must have a distributed tensor after the initialization for program." + self._node_id_to_tensor_id[ + _node_id(node) + ] = cur_tensor_id + assert ( + dist_tensor is not None + ), "Tensor must have a distributed tensor after the initialization for program." serial_tensor_node_id = _node_id(node) - new_dist_tensor = DistributedTensor(dist_tensor.serial_tensor, - dist_tensor.dist_attr) + new_dist_tensor = DistributedTensor( + dist_tensor.serial_tensor, dist_tensor.dist_attr + ) self._dist_tensors_for_graph[ - serial_tensor_node_id] = new_dist_tensor + serial_tensor_node_id + ] = new_dist_tensor if node.is_op() and node.op() is not None: dist_op = None op_id = node.node.original_desc_id() - for cur_op_id, cur_dist_op in self._dist_ops_for_program.items( - ): - if op_id == cur_op_id \ - or op_id == cur_dist_op.serial_op.desc.original_id(): + for ( + cur_op_id, + cur_dist_op, + ) in self._dist_ops_for_program.items(): + if ( + op_id == cur_op_id + or op_id == cur_dist_op.serial_op.desc.original_id() + ): dist_op = cur_dist_op self._node_id_to_op_id[_node_id(node)] = cur_op_id - assert dist_op is not None, \ - "Operator must have a distributed operator after the initialization for program." + assert ( + dist_op is not None + ), "Operator must have a distributed operator after the initialization for program." serial_op_node_id = _node_id(node) - new_dist_op = DistributedOperator(dist_op.serial_op, - dist_op.dist_attr) + new_dist_op = DistributedOperator( + dist_op.serial_op, dist_op.dist_attr + ) self._dist_ops_for_graph[serial_op_node_id] = new_dist_op def clear_dist_info_for_program(self): @@ -697,36 +760,51 @@ class DistributedContext: if node.is_var() and node.var() is not None: dist_tensor = None tensor_id = node.node.original_desc_id() - for cur_tensor_id, cur_dist_tensor in self._dist_tensors_for_program.items( - ): - if tensor_id == cur_tensor_id \ - or tensor_id == cur_dist_tensor.serial_tensor.desc.original_id(): + for ( + cur_tensor_id, + cur_dist_tensor, + ) in self._dist_tensors_for_program.items(): + if ( + tensor_id == cur_tensor_id + or tensor_id + == cur_dist_tensor.serial_tensor.desc.original_id() + ): dist_tensor = cur_dist_tensor - assert dist_tensor is not None, \ - "Tensor must have a distributed tensor after the initialization for program." + assert ( + dist_tensor is not None + ), "Tensor must have a distributed tensor after the initialization for program." serial_tensor_node_id = _node_id(node) - new_dist_tensor = DistributedTensor(dist_tensor.serial_tensor, - dist_tensor.dist_attr) + new_dist_tensor = DistributedTensor( + dist_tensor.serial_tensor, dist_tensor.dist_attr + ) self._dist_tensors_for_graph[ - serial_tensor_node_id] = new_dist_tensor + serial_tensor_node_id + ] = new_dist_tensor if node.is_op() and node.op() is not None: dist_op = None op_id = node.node.original_desc_id() - for cur_op_id, cur_dist_op in self._dist_ops_for_program.items( - ): - if op_id == cur_op_id \ - or op_id == cur_dist_op.serial_op.desc.original_id(): + for ( + cur_op_id, + cur_dist_op, + ) in self._dist_ops_for_program.items(): + if ( + op_id == cur_op_id + or op_id == cur_dist_op.serial_op.desc.original_id() + ): dist_op = cur_dist_op - assert dist_op is not None, \ - "Operator must have a distributed operator after the initialization for program." + assert ( + dist_op is not None + ), "Operator must have a distributed operator after the initialization for program." serial_op_node_id = _node_id(node) - new_dist_op = DistributedOperator(dist_op.serial_op, - dist_op.dist_attr) + new_dist_op = DistributedOperator( + dist_op.serial_op, dist_op.dist_attr + ) self._dist_ops_for_graph[serial_op_node_id] = new_dist_op def copy_dist_attr_from_graph_to_program(self): - assert self._is_initialized, \ - "Both program and graph must be initialized." + assert ( + self._is_initialized + ), "Both program and graph must be initialized." updated_tensors = {} # all_nodes = self._serial_graph.all_nodes() all_nodes = self._serial_ordered_nodes @@ -736,11 +814,15 @@ class DistributedContext: updated = updated_tensors.get(tensor_id, False) # If a var has multiples var nodes in graph, only use the first one for now if not updated: - tensor_dist_attr_for_graph = self.get_tensor_dist_attr_for_graph( - node) + tensor_dist_attr_for_graph = ( + self.get_tensor_dist_attr_for_graph(node) + ) dist_tensor_for_program = self._dist_tensors_for_program[ - tensor_id] - dist_tensor_for_program.dist_attr = tensor_dist_attr_for_graph + tensor_id + ] + dist_tensor_for_program.dist_attr = ( + tensor_dist_attr_for_graph + ) updated_tensors[tensor_id] = True if node.is_op() and node.op() is not None: op_id = self._node_id_to_op_id[_node_id(node)] @@ -752,22 +834,26 @@ class DistributedContext: for orphan_node in self._serial_orphan_tensor_nodes: serial_tensor_id = orphan_node.var().id() dist_tensor = self._dist_tensors_for_program.get( - serial_tensor_id, None) + serial_tensor_id, None + ) if dist_tensor: dist_tensor.dist_attr.process_mesh = self._process_meshes[0] else: serial_tensor_id = orphan_node.var().original_id() dist_tensor = self._dist_tensors_for_program.get( - serial_tensor_id, None) + serial_tensor_id, None + ) dist_tensor.dist_attr.process_mesh = self._process_meshes[0] def amend_dist_attr_for_program(self): for dist_tensor in self._dist_tensors_for_program.values(): serial_tensor = dist_tensor.serial_tensor dist_attr = dist_tensor.dist_attr - if serial_tensor.type == core.VarDesc.VarType.READER \ - or serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + serial_tensor.type == core.VarDesc.VarType.READER + or serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = serial_tensor.shape @@ -777,8 +863,11 @@ class DistributedContext: # If the dimension of tensor is less than the sharding dimension of process mesh, # we just amend the dimension mapping to -1. (Is this really OK?) for i in range(len(tensor_shape)): - if dims_mapping[i] != -1 and tensor_shape[i] > 0 \ - and process_mesh_shape[dims_mapping[i]] > tensor_shape[i]: + if ( + dims_mapping[i] != -1 + and tensor_shape[i] > 0 + and process_mesh_shape[dims_mapping[i]] > tensor_shape[i] + ): dims_mapping[i] = -1 if dims_mapping[i] != -1 and len(process_mesh_processes) == 1: dims_mapping[i] = -1 @@ -792,9 +881,13 @@ class DistributedContext: if dist_op.get_serial_input(arg_name) is None: tensor_shape = [] else: - if dist_op.get_serial_input(arg_name).type == core.VarDesc.VarType.READER \ - or dist_op.get_serial_input(arg_name).type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or dist_op.serial_op.type == "create_py_reader": + if ( + dist_op.get_serial_input(arg_name).type + == core.VarDesc.VarType.READER + or dist_op.get_serial_input(arg_name).type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or dist_op.serial_op.type == "create_py_reader" + ): tensor_shape = [] else: tensor_shape = dist_op.get_serial_input(arg_name).shape @@ -802,16 +895,27 @@ class DistributedContext: # If the dimension of tensor is less than the sharding dimension of process mesh, # we just amend the dimension mapping to -1. (Is this really OK?) for i in range(len(tensor_shape)): - if dims_mapping[i] != -1 and tensor_shape[i] > 0 \ - and process_mesh_shape[dims_mapping[i]] > tensor_shape[i]: + if ( + dims_mapping[i] != -1 + and tensor_shape[i] > 0 + and process_mesh_shape[dims_mapping[i]] + > tensor_shape[i] + ): dims_mapping[i] = -1 - if dims_mapping[i] != -1 and len( - process_mesh_processes) == 1: + if ( + dims_mapping[i] != -1 + and len(process_mesh_processes) == 1 + ): dims_mapping[i] = -1 for arg_name in serial_op.output_arg_names: - if dist_op.get_serial_output(arg_name).type == core.VarDesc.VarType.READER \ - or dist_op.get_serial_output(arg_name).type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or dist_op.get_serial_output(arg_name).type == core.VarDesc.VarType.STEP_SCOPES: + if ( + dist_op.get_serial_output(arg_name).type + == core.VarDesc.VarType.READER + or dist_op.get_serial_output(arg_name).type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or dist_op.get_serial_output(arg_name).type + == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = dist_op.get_serial_output(arg_name).shape @@ -819,11 +923,17 @@ class DistributedContext: # If the dimension of tensor is less than the sharding dimension of process mesh, # we just amend the dimension mapping to -1. (Is this really OK?) for i in range(len(tensor_shape)): - if dims_mapping[i] != -1 and tensor_shape[i] > 0 \ - and process_mesh_shape[dims_mapping[i]] > tensor_shape[i]: + if ( + dims_mapping[i] != -1 + and tensor_shape[i] > 0 + and process_mesh_shape[dims_mapping[i]] + > tensor_shape[i] + ): dims_mapping[i] = -1 - if dims_mapping[i] != -1 and len( - process_mesh_processes) == 1: + if ( + dims_mapping[i] != -1 + and len(process_mesh_processes) == 1 + ): dims_mapping[i] = -1 if len(process_mesh_processes) == 1: dist_op.dist_attr.impl_type = "default" @@ -831,30 +941,44 @@ class DistributedContext: def validate_dist_attr_for_program(self): if not self._is_initialized: - assert False, \ - "Program must be initialized before validating its distributed attributes" + assert ( + False + ), "Program must be initialized before validating its distributed attributes" for block in self.serial_main_program.blocks: for tensor in block.vars.values(): dist_tensor = self.get_dist_tensor_for_program(tensor) - assert dist_tensor is not None, \ - "Tensor {} does not have a distributed attribute.".format( - dist_tensor.serial_tensor.name) - if (dist_tensor - is not None) and (not dist_tensor.validate_dist_attr()): - assert False, "Tensor {} (id: {}, original_id: {}) has a wrong distributed attributes {}.".format( + assert ( + dist_tensor is not None + ), "Tensor {} does not have a distributed attribute.".format( + dist_tensor.serial_tensor.name + ) + if (dist_tensor is not None) and ( + not dist_tensor.validate_dist_attr() + ): + assert ( + False + ), "Tensor {} (id: {}, original_id: {}) has a wrong distributed attributes {}.".format( dist_tensor.serial_tensor.name, dist_tensor.serial_tensor.desc.id(), dist_tensor.serial_tensor.desc.original_id(), - dist_tensor.dist_attr) + dist_tensor.dist_attr, + ) for op in block.ops: dist_op = self.get_dist_op_for_program(op) - assert dist_op is not None, \ - "Operator {} does not have a distributed attribute.".format( - dist_op.serial_op.type) + assert ( + dist_op is not None + ), "Operator {} does not have a distributed attribute.".format( + dist_op.serial_op.type + ) if (dist_op is not None) and (not dist_op.validate_dist_attr()): - assert False, "Operator {} (id: {}, original_id: {}) has a wrong distributed attributes {} .".format( - dist_op.serial_op.type, dist_op.serial_op.desc.id(), - dist_op.serial_op.desc.original_id(), dist_op.dist_attr) + assert ( + False + ), "Operator {} (id: {}, original_id: {}) has a wrong distributed attributes {} .".format( + dist_op.serial_op.type, + dist_op.serial_op.desc.id(), + dist_op.serial_op.desc.original_id(), + dist_op.dist_attr, + ) return True def __deepcopy__(self, memo): @@ -863,15 +987,27 @@ class DistributedContext: memo[id(self)] = result for k, v in self.__dict__.items(): if k in [ - "_original_serial_main_program", "_original_serial_startup_program", \ - "_serial_main_program", "_serial_startup_program", "_serial_graph", \ - "_dist_main_programs", "_dist_startup_programs", \ - "_serial_ordered_nodes", "_serial_ordered_tensor_nodes", \ - "_serial_ordered_op_nodes", "_original_serial_loss", \ - "_original_serial_feed_vars", "_original_serial_fetch_vars", \ - "_serial_loss", "_serial_feed_vars", "_serial_fetch_vars", "_serial_optimizer", \ - "_backup_serial_main_program_stack", "_backup_serial_startup_program_stack", \ - "_pass_context"]: + "_original_serial_main_program", + "_original_serial_startup_program", + "_serial_main_program", + "_serial_startup_program", + "_serial_graph", + "_dist_main_programs", + "_dist_startup_programs", + "_serial_ordered_nodes", + "_serial_ordered_tensor_nodes", + "_serial_ordered_op_nodes", + "_original_serial_loss", + "_original_serial_feed_vars", + "_original_serial_fetch_vars", + "_serial_loss", + "_serial_feed_vars", + "_serial_fetch_vars", + "_serial_optimizer", + "_backup_serial_main_program_stack", + "_backup_serial_startup_program_stack", + "_pass_context", + ]: setattr(result, k, v) else: setattr(result, k, copy.deepcopy(v, memo)) @@ -913,8 +1049,12 @@ class DistributedOperatorContext: memo[id(self)] = result for k, v in self.__dict__.items(): if k in [ - "_dst_main_program", "_dst_startup_program", "_cur_src_op", - "_work_block", "_main_block", "_startup_block" + "_dst_main_program", + "_dst_startup_program", + "_cur_src_op", + "_work_block", + "_main_block", + "_startup_block", ]: setattr(result, k, v) else: @@ -994,7 +1134,6 @@ class DistributedOperatorContext: class BlockState(object): - def __init__(self): self.nblock = 0 self.forward_indices = [] @@ -1011,8 +1150,11 @@ class BlockState(object): for idx, block in enumerate(program.blocks): assert idx == block.idx, "index doesn't match" - assert block.forward_block_idx == -1, "forward_block_idx of forward block [{}] is not [{}]".format( - idx, block.forward_block_idx) + assert ( + block.forward_block_idx == -1 + ), "forward_block_idx of forward block [{}] is not [{}]".format( + idx, block.forward_block_idx + ) self.forward_indices.append(idx) self.nblock += 1 @@ -1021,7 +1163,8 @@ class BlockState(object): def parse_backward_blocks(self, program): assert 0 in self.forward_indices, "forward block idx are{}".format( - self.forward_indices) + self.forward_indices + ) self.backward_to_forward_index_map[0] = 0 for idx, block in enumerate(program.blocks): diff --git a/python/paddle/distributed/auto_parallel/dist_loader.py b/python/paddle/distributed/auto_parallel/dist_loader.py index 9eb62b1b74e..f16c55a011c 100644 --- a/python/paddle/distributed/auto_parallel/dist_loader.py +++ b/python/paddle/distributed/auto_parallel/dist_loader.py @@ -17,12 +17,18 @@ import numpy as np import paddle from paddle.io import BatchSampler, IterableDataset -from paddle.fluid.dataloader.batch_sampler import _InfiniteIterableSampler, DistributedBatchSampler -from paddle.fluid.dataloader.dataloader_iter import _DatasetKind, default_collate_fn, default_convert_fn +from paddle.fluid.dataloader.batch_sampler import ( + _InfiniteIterableSampler, + DistributedBatchSampler, +) +from paddle.fluid.dataloader.dataloader_iter import ( + _DatasetKind, + default_collate_fn, + default_convert_fn, +) class DistributedDataLoaderBase(metaclass=abc.ABCMeta): - @abc.abstractmethod def __iter__(self): raise NotImplementedError @@ -33,24 +39,25 @@ class DistributedDataLoaderBase(metaclass=abc.ABCMeta): class DistributedDataLoaderFromGenerator(DistributedDataLoaderBase): - - def __init__(self, - dataset, - feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False, - use_multiprocess=False, - drop_last=True, - places=None, - batch_size=1, - epochs=1, - steps_per_epoch=None, - collate_fn=None, - split_data=True, - data_parallel_world_size=[], - data_parallel_rank=[]): + def __init__( + self, + dataset, + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + use_multiprocess=False, + drop_last=True, + places=None, + batch_size=1, + epochs=1, + steps_per_epoch=None, + collate_fn=None, + split_data=True, + data_parallel_world_size=[], + data_parallel_rank=[], + ): self.dataset = dataset self.feed_list = feed_list self.capacity = capacity @@ -80,12 +87,15 @@ class DistributedDataLoaderFromGenerator(DistributedDataLoaderBase): else: if isinstance(dataset, IterableDataset): self.batch_sampler = _InfiniteIterableSampler( - dataset, batch_size) + dataset, batch_size + ) else: - self.batch_sampler = BatchSampler(dataset, - batch_size=batch_size, - shuffle=False, - drop_last=drop_last) + self.batch_sampler = BatchSampler( + dataset, + batch_size=batch_size, + shuffle=False, + drop_last=drop_last, + ) self.auto_collate_batch = self.batch_sampler is not None self.sampler_iter = iter(self.index_sampler) @@ -96,8 +106,12 @@ class DistributedDataLoaderFromGenerator(DistributedDataLoaderBase): self.collate_fn = collate_fn or default_convert_fn self.dataset_fetcher = _DatasetKind.create_fetcher( - self.dataset_kind, self.dataset, self.auto_collate_batch, - self.collate_fn, self.drop_last) + self.dataset_kind, + self.dataset, + self.auto_collate_batch, + self.collate_fn, + self.drop_last, + ) self._steps = self._infer_steps() self._inner_dataloader = self._create_inner_dataloader() @@ -146,18 +160,21 @@ class DistributedDataLoaderFromGenerator(DistributedDataLoaderBase): return _InfiniteIterableSampler(self.dataset, 1) def _create_inner_dataloader(self): - def data_generator(): while True: try: indices = next(self.sampler_iter) batch = self.dataset_fetcher.fetch(indices) - if batch is None: break + if batch is None: + break except StopIteration: self.dataset_fetcher = _DatasetKind.create_fetcher( - self.dataset_kind, self.dataset, - self.auto_collate_batch, self.collate_fn, - self.drop_last) + self.dataset_kind, + self.dataset, + self.auto_collate_batch, + self.collate_fn, + self.drop_last, + ) break partial_data = [] @@ -168,11 +185,16 @@ class DistributedDataLoaderFromGenerator(DistributedDataLoaderBase): continue batch_size = array.shape[0] - assert batch_size % self.dp_world_sizes[i] == 0, \ - "batch_size [{}] is not divisible by dp_world_size [{}]".format(str(batch_size), str(self.dp_world_sizes[i])) + assert ( + batch_size % self.dp_world_sizes[i] == 0 + ), "batch_size [{}] is not divisible by dp_world_size [{}]".format( + str(batch_size), str(self.dp_world_sizes[i]) + ) partial_data.append( - np.split(array, - self.dp_world_sizes[i])[self.dp_ranks[i]]) + np.split(array, self.dp_world_sizes[i])[ + self.dp_ranks[i] + ] + ) yield partial_data @@ -184,33 +206,35 @@ class DistributedDataLoaderFromGenerator(DistributedDataLoaderBase): iterable=False, return_list=self.return_list, use_multiprocess=self.use_multiprocess, - drop_last=self.drop_last) + drop_last=self.drop_last, + ) dataloader.set_batch_generator(data_generator, self.places) return dataloader class DistributedDataLoader(DistributedDataLoaderBase): - - def __init__(self, - dataset, - feed_list=None, - places=None, - return_list=True, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=None, - num_workers=0, - use_buffer_reader=True, - use_shared_memory=True, - timeout=0, - worker_init_fn=None, - epochs=1, - steps_per_epoch=None, - split_data=True, - data_parallel_world_size=[], - data_parallel_rank=[]): + def __init__( + self, + dataset, + feed_list=None, + places=None, + return_list=True, + batch_size=1, + shuffle=False, + drop_last=False, + collate_fn=None, + num_workers=0, + use_buffer_reader=True, + use_shared_memory=True, + timeout=0, + worker_init_fn=None, + epochs=1, + steps_per_epoch=None, + split_data=True, + data_parallel_world_size=[], + data_parallel_rank=[], + ): self.dataset = dataset self.feed_list = feed_list self.return_list = return_list @@ -231,8 +255,13 @@ class DistributedDataLoader(DistributedDataLoaderBase): self.split_data = split_data # TODO: rank info self.batch_sampler = DistributedBatchSampler( - self.dataset, self.batch_size, self.dp_world_sizes[0], - self.dp_ranks[0], self.shuffle, self.drop_last) + self.dataset, + self.batch_size, + self.dp_world_sizes[0], + self.dp_ranks[0], + self.shuffle, + self.drop_last, + ) self._inner_dataloader = self._create_inner_dataloader() def __iter__(self): @@ -253,7 +282,8 @@ class DistributedDataLoader(DistributedDataLoaderBase): use_buffer_reader=self.use_buffer_reader, use_shared_memory=self.use_shared_memory, timeout=self.timeout, - worker_init_fn=self.worker_init_fn) + worker_init_fn=self.worker_init_fn, + ) self.data = (x for x in dataloader) return dataloader diff --git a/python/paddle/distributed/auto_parallel/dist_op.py b/python/paddle/distributed/auto_parallel/dist_op.py index 004436458b1..41b4696174a 100644 --- a/python/paddle/distributed/auto_parallel/dist_op.py +++ b/python/paddle/distributed/auto_parallel/dist_op.py @@ -23,7 +23,6 @@ from .utils import convert_to_shard_spec, verify_shard_spec class DistributedOperator: - def __init__(self, serial_op, dist_attr=None): self._serial_op = serial_op self._serial_inputs = {} @@ -74,28 +73,34 @@ class DistributedOperator: if tensor is None: tensor_shape = [] else: - if tensor.type == core.VarDesc.VarType.READER \ - or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + tensor.type == core.VarDesc.VarType.READER + or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): tensor_shape = [] else: tensor_shape = tensor.shape if self._dist_attr.get_input_dims_mapping(tensor_name) is None: tensor_dims_mapping = [-1 for _ in range(len(tensor_shape))] - self._dist_attr.set_input_dims_mapping(tensor_name, - tensor_dims_mapping) + self._dist_attr.set_input_dims_mapping( + tensor_name, tensor_dims_mapping + ) for tensor_name in self._serial_op.output_arg_names: tensor = self._serial_op.block._var_recursive(tensor_name) - if tensor.type == core.VarDesc.VarType.READER \ - or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + tensor.type == core.VarDesc.VarType.READER + or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = tensor.shape self._serial_outputs[tensor_name] = tensor if self._dist_attr.get_output_dims_mapping(tensor_name) is None: tensor_dims_mapping = [-1 for _ in range(len(tensor_shape))] - self._dist_attr.set_output_dims_mapping(tensor_name, - tensor_dims_mapping) + self._dist_attr.set_output_dims_mapping( + tensor_name, tensor_dims_mapping + ) if self._dist_attr.op_type is None: self._dist_attr.op_type = self.serial_op.type if self._dist_attr.impl_type is None: @@ -113,8 +118,10 @@ class DistributedOperator: new_dist_attr = {} for key, value in dist_attr.items(): if isinstance(key, Variable): - if key.name in self._serial_op.input_arg_names \ - or key.name in self._serial_op.output_arg_names: + if ( + key.name in self._serial_op.input_arg_names + or key.name in self._serial_op.output_arg_names + ): new_dist_attr[key] = value else: new_dist_attr[key] = value @@ -125,13 +132,15 @@ class DistributedOperator: for tensor_name in self._serial_op.input_arg_names: tensor_dist_attr = dist_attr.get_input_dist_attr(tensor_name) if tensor_dist_attr: - new_dist_attr.set_input_dist_attr(tensor_name, - tensor_dist_attr) + new_dist_attr.set_input_dist_attr( + tensor_name, tensor_dist_attr + ) for tensor_name in self._serial_op.output_arg_names: tensor_dist_attr = dist_attr.get_output_dist_attr(tensor_name) if tensor_dist_attr: - new_dist_attr.set_output_dist_attr(tensor_name, - tensor_dist_attr) + new_dist_attr.set_output_dist_attr( + tensor_name, tensor_dist_attr + ) else: assert False, "Cannot recognize the {} parameter.".format(dist_attr) return new_dist_attr @@ -142,8 +151,10 @@ class DistributedOperator: for name in self.serial_op.input_arg_names: input_dist_attr = self.dist_attr.get_input_dist_attr(name) dims_mapping = input_dist_attr.dims_mapping - if self.get_serial_input( - name).type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + self.get_serial_input(name).type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): shape = [] else: shape = self.get_serial_input(name).shape @@ -151,7 +162,8 @@ class DistributedOperator: return False for i in range(len(dims_mapping)): if dims_mapping[i] < -1 or dims_mapping[i] >= len( - self.dist_attr.process_mesh.topology): + self.dist_attr.process_mesh.topology + ): return False for i in range(len(self.dist_attr.process_mesh.topology)): if dims_mapping.count(i) > 1: @@ -162,8 +174,12 @@ class DistributedOperator: for name in self.serial_op.output_arg_names: output_dist_attr = self.dist_attr.get_output_dist_attr(name) dims_mapping = output_dist_attr.dims_mapping - if self.get_serial_output(name).type == core.VarDesc.VarType.LOD_TENSOR_ARRAY\ - or self.get_serial_output(name).type == core.VarDesc.VarType.STEP_SCOPES: + if ( + self.get_serial_output(name).type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or self.get_serial_output(name).type + == core.VarDesc.VarType.STEP_SCOPES + ): shape = [] else: shape = self.get_serial_output(name).shape @@ -171,7 +187,8 @@ class DistributedOperator: return False for i in range(len(dims_mapping)): if dims_mapping[i] < -1 or dims_mapping[i] >= len( - self.dist_attr.process_mesh.topology): + self.dist_attr.process_mesh.topology + ): return False for i in range(len(self.dist_attr.process_mesh.topology)): if dims_mapping.count(i) > 1: @@ -181,8 +198,9 @@ class DistributedOperator: return True def __str__(self): - str = "{{op type: {}, op id: {}".format(self.serial_op.desc.type(), - self.serial_op.desc.id()) + str = "{{op type: {}, op id: {}".format( + self.serial_op.desc.type(), self.serial_op.desc.id() + ) # str += ", {}".format(self.dist_attr) # return str @@ -191,8 +209,9 @@ class DistributedOperator: annotated_str = "annotated" else: annotated_str = "non-annotated" - str += ", process_mesh ({}): {}".format(annotated_str, - self.dist_attr.process_mesh) + str += ", process_mesh ({}): {}".format( + annotated_str, self.dist_attr.process_mesh + ) for arg_name in self.serial_op.desc.input_arg_names(): dims_mapping = self.dist_attr.get_input_dims_mapping(arg_name) @@ -208,7 +227,8 @@ class DistributedOperator: else: is_parameter_str = "non-parameter" str += ", {}'s dims_mapping (input, {}, {}): {}".format( - arg_name, annotated_str, is_parameter_str, dims_mapping) + arg_name, annotated_str, is_parameter_str, dims_mapping + ) for arg_name in self.serial_op.desc.output_arg_names(): dims_mapping = self.dist_attr.get_output_dims_mapping(arg_name) @@ -224,12 +244,14 @@ class DistributedOperator: else: is_parameter_str = "non-parameter" str += ", {}'s dims_mapping (output, {}, {}): {}".format( - arg_name, annotated_str, is_parameter_str, dims_mapping) + arg_name, annotated_str, is_parameter_str, dims_mapping + ) str += ", pipeline stage: {}".format(None) str += ", dist_impl idx: {} , dist_impl type {} }}".format( - self.dist_attr._impl_idx, self.dist_attr._impl_type) + self.dist_attr._impl_idx, self.dist_attr._impl_type + ) return str @@ -238,7 +260,11 @@ class DistributedOperator: result = cls.__new__(cls) memo[id(self)] = result for k, v in self.__dict__.items(): - if k == "_serial_op" or k == "_serial_inputs" or k == "_serial_outputs": + if ( + k == "_serial_op" + or k == "_serial_inputs" + or k == "_serial_outputs" + ): setattr(result, k, v) else: setattr(result, k, copy.deepcopy(v, memo)) @@ -246,9 +272,9 @@ class DistributedOperator: class DistributedOperatorHelper: - - def __init__(self, serial_op, process_mesh, in_dims_mappings, - out_dims_mappings): + def __init__( + self, serial_op, process_mesh, in_dims_mappings, out_dims_mappings + ): self._serial_op = serial_op self._process_mesh = process_mesh self._in_dims_mappings = in_dims_mappings @@ -258,8 +284,11 @@ class DistributedOperatorHelper: tensor_to_dims_mapping = {} index = 0 if self._in_dims_mappings: - assert len(args) + len(kwargs) == len(self._in_dims_mappings), \ - "The length of dims_mapping {} does not matching the length output {}.".format(len(self._in_dims_mappings), len(args) + len(kwargs)) + assert len(args) + len(kwargs) == len( + self._in_dims_mappings + ), "The length of dims_mapping {} does not matching the length output {}.".format( + len(self._in_dims_mappings), len(args) + len(kwargs) + ) for arg in args: if isinstance(arg, Variable) and self._in_dims_mappings: tensor_to_dims_mapping[arg.name] = self._in_dims_mappings[index] @@ -283,13 +312,17 @@ class DistributedOperatorHelper: raise ValueError("Unrecognized outpout.") if self._out_dims_mappings: - assert len(new_output) == len(self._out_dims_mappings), \ - "The length of dims_mapping {} does not matching the length output {}.".format(len(self._out_dims_mappings), len(new_output)) + assert len(new_output) == len( + self._out_dims_mappings + ), "The length of dims_mapping {} does not matching the length output {}.".format( + len(self._out_dims_mappings), len(new_output) + ) for i, item in enumerate(new_output): if isinstance(item, Variable) and self._out_dims_mappings: tensor_to_dims_mapping[item.name] = self._out_dims_mappings[i] from .dist_context import get_default_distributed_context + default_dist_ctx = get_default_distributed_context() for idx in range(op_size, new_op_size): op = cur_block.ops[idx] @@ -298,48 +331,62 @@ class DistributedOperatorHelper: if name in tensor_to_dims_mapping.keys(): tensor = dist_op.get_serial_input(name) tensor_dist_attr = dist_op.dist_attr.get_input_dist_attr( - name) + name + ) dims_mapping = tensor_to_dims_mapping[name] if tensor is None: tensor_shape = [] else: - if tensor.type == core.VarDesc.VarType.READER \ - or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + tensor.type == core.VarDesc.VarType.READER + or tensor.type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = tensor.shape if dims_mapping is not None: dims_mapping = tensor_to_dims_mapping[name] shard_spec = convert_to_shard_spec( - dims_mapping, self._process_mesh) - assert verify_shard_spec(shard_spec, tensor_shape, self._process_mesh), \ - "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( - name, shard_spec, tensor_shape, self._process_mesh) + dims_mapping, self._process_mesh + ) + assert verify_shard_spec( + shard_spec, tensor_shape, self._process_mesh + ), "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( + name, shard_spec, tensor_shape, self._process_mesh + ) tensor_dist_attr.dims_mapping = dims_mapping tensor_dist_attr.mark_annotated("dims_mapping") for name in dist_op.serial_op.output_arg_names: if name in tensor_to_dims_mapping.keys(): tensor = dist_op.get_serial_output(name) tensor_dist_attr = dist_op.dist_attr.get_output_dist_attr( - name) + name + ) dims_mapping = tensor_to_dims_mapping[name] if tensor is None: tensor_shape = [] else: - if tensor.type == core.VarDesc.VarType.READER \ - or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + tensor.type == core.VarDesc.VarType.READER + or tensor.type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = tensor.shape if dims_mapping is not None: dims_mapping = tensor_to_dims_mapping[name] shard_spec = convert_to_shard_spec( - dims_mapping, self._process_mesh) - assert verify_shard_spec(shard_spec, tensor_shape, self._process_mesh), \ - "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( - name, shard_spec, tensor_shape, self._process_mesh) + dims_mapping, self._process_mesh + ) + assert verify_shard_spec( + shard_spec, tensor_shape, self._process_mesh + ), "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( + name, shard_spec, tensor_shape, self._process_mesh + ) tensor_dist_attr.dims_mapping = dims_mapping tensor_dist_attr.mark_annotated("dims_mapping") dist_op.dist_attr.process_mesh = self._process_mesh diff --git a/python/paddle/distributed/auto_parallel/dist_saver.py b/python/paddle/distributed/auto_parallel/dist_saver.py index a885bf7592b..f2766bff602 100644 --- a/python/paddle/distributed/auto_parallel/dist_saver.py +++ b/python/paddle/distributed/auto_parallel/dist_saver.py @@ -50,16 +50,13 @@ def _process_path(path): class DistributedSaver: - def __init__(self): self._logger = get_logger(logging.INFO) def save(self, path, serial_program, dist_main_program, dist_context): - def _save_state(program, path, mode="param"): state = { - k: np.array(v) - for k, v in program.state_dict(mode).items() + k: np.array(v) for k, v in program.state_dict(mode).items() } with open(path, "wb") as f: pickle.dump(state, f) @@ -105,8 +102,9 @@ class DistributedSaver: def _load_file(filename, dirname, suffix="pdparams"): file_list = [] for file in os.listdir(dirname): - if check_filename('{}(.*)_dist(.*).{}'.format(filename, suffix), - file): + if check_filename( + '{}(.*)_dist(.*).{}'.format(filename, suffix), file + ): file_list.append(os.path.join(dirname, file)) file_list.sort() return file_list @@ -134,14 +132,16 @@ class DistributedSaver: # load path.pdparam and path.pdopt param_state_dict = _load_state(filename, dirname) - opt_state_dict = _load_state(filename, dirname, - "pdopt") if load_optimizer else {} + opt_state_dict = ( + _load_state(filename, dirname, "pdopt") if load_optimizer else {} + ) state_dict = dict(param_state_dict, **opt_state_dict) # load path.pdattr dist_attr_file_list = _load_file(filename, dirname, "pdattr") self._logger.info( - "Load distributed attribute file: {}".format(dist_attr_file_list)) + "Load distributed attribute file: {}".format(dist_attr_file_list) + ) dist_attr = {} for dist_attr_file in dist_attr_file_list: with open(dist_attr_file, 'rb') as f: @@ -206,11 +206,13 @@ class DistributedSaver: # NOTE: `paddle.static.save_inference_model` does not support subblock. dist_filename = filename + "_dist" + str(rank_id) dist_path = os.path.join(dirname, dist_filename) - paddle.static.save_inference_model(dist_path, - dist_feed_vars, - dist_fetch_vars, - exe, - program=dist_main_prog) + paddle.static.save_inference_model( + dist_path, + dist_feed_vars, + dist_fetch_vars, + exe, + program=dist_main_prog, + ) def _save_rank_mapping(self, dirname): path = os.path.join(dirname, 'rank_mapping.csv') diff --git a/python/paddle/distributed/auto_parallel/dist_tensor.py b/python/paddle/distributed/auto_parallel/dist_tensor.py index e07269fab25..88c754f06f3 100644 --- a/python/paddle/distributed/auto_parallel/dist_tensor.py +++ b/python/paddle/distributed/auto_parallel/dist_tensor.py @@ -33,32 +33,45 @@ class DistributedTensor: """ @staticmethod - def _validate_sizes_and_dist_attr(sizes, - dims_mapping, - topology, - processes, - rank=None, - shard_sizes=None): - if not (isinstance(sizes, (list, tuple)) - and all(map(lambda x: isinstance(x, int) and x >= 0, sizes))): + def _validate_sizes_and_dist_attr( + sizes, dims_mapping, topology, processes, rank=None, shard_sizes=None + ): + if not ( + isinstance(sizes, (list, tuple)) + and all(map(lambda x: isinstance(x, int) and x >= 0, sizes)) + ): raise ValueError( - "The sizes must be list or tuple and item in sizes must be non-negative integer, but got {}" - .format(sizes)) - if not (isinstance(dims_mapping, (list, tuple)) and all( - map(lambda x: isinstance(x, int) and x >= -1, dims_mapping))): + "The sizes must be list or tuple and item in sizes must be non-negative integer, but got {}".format( + sizes + ) + ) + if not ( + isinstance(dims_mapping, (list, tuple)) + and all(map(lambda x: isinstance(x, int) and x >= -1, dims_mapping)) + ): raise ValueError( - "The dims_mapping must be list or tuple and item in dims_mapping must >= -1, but got {}" - .format(dims_mapping)) - if not (isinstance(processes, (list, tuple)) and all( - map(lambda x: isinstance(x, int) and x >= 0, processes))): + "The dims_mapping must be list or tuple and item in dims_mapping must >= -1, but got {}".format( + dims_mapping + ) + ) + if not ( + isinstance(processes, (list, tuple)) + and all(map(lambda x: isinstance(x, int) and x >= 0, processes)) + ): raise ValueError( - "The processes must be list or tuple and item in processes must be integer, but got {}" - .format(processes)) - if not (isinstance(topology, (list, tuple)) - and all(map(lambda x: isinstance(x, int) and x > 0, topology))): + "The processes must be list or tuple and item in processes must be integer, but got {}".format( + processes + ) + ) + if not ( + isinstance(topology, (list, tuple)) + and all(map(lambda x: isinstance(x, int) and x > 0, topology)) + ): raise ValueError( - "The topology must be list or tuple and item in topology must be non-negative integer, but got {}" - .format(topology)) + "The topology must be list or tuple and item in topology must be non-negative integer, but got {}".format( + topology + ) + ) if rank is not None and not (isinstance(rank, int) and rank >= 0): raise ValueError("The rank must >= 0, but got {}".format(rank)) @@ -67,16 +80,17 @@ class DistributedTensor: raise ValueError("Only support even sharding now.") @staticmethod - def get_local_sizes(global_sizes, - dims_mapping, - topology, - processes, - rank=None, - shard_sizes=None): - DistributedTensor._validate_sizes_and_dist_attr(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + def get_local_sizes( + global_sizes, + dims_mapping, + topology, + processes, + rank=None, + shard_sizes=None, + ): + DistributedTensor._validate_sizes_and_dist_attr( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) local_sizes = [] # for even sharding, the local sizes of every rank are equal @@ -92,16 +106,12 @@ class DistributedTensor: return local_sizes @staticmethod - def get_local_offsets(global_sizes, - dims_mapping, - topology, - processes, - rank, - shard_sizes=None): - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + def get_local_offsets( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes=None + ): + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) local_offsets = [] rank_relatvie = processes.index(rank) coordinate = _linear_idx2coordinate(topology, rank_relatvie) @@ -110,21 +120,23 @@ class DistributedTensor: if dims_mapping[i] == -1: local_offsets.append(0) else: - local_offsets.append(coordinate[dims_mapping[i]] * - local_sizes[i]) + local_offsets.append( + coordinate[dims_mapping[i]] * local_sizes[i] + ) return local_offsets @staticmethod - def get_global_sizes(local_sizes, - dims_mapping, - topology, - processes, - rank=None, - shard_sizes=None): - DistributedTensor._validate_sizes_and_dist_attr(local_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + def get_global_sizes( + local_sizes, + dims_mapping, + topology, + processes, + rank=None, + shard_sizes=None, + ): + DistributedTensor._validate_sizes_and_dist_attr( + local_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) global_sizes = [] for idx, item in enumerate(local_sizes): if dims_mapping[idx] == -1: @@ -134,25 +146,24 @@ class DistributedTensor: return global_sizes @staticmethod - def get_local_shard(global_sizes, - dims_mapping, - topology, - processes, - rank, - shard_sizes=None): + def get_local_shard( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes=None + ): local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank, shard_sizes) - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) assert len(local_sizes) == len( local_offsets ), "The length of local_sizes must be equal to local_offsets, but got {} and {}.".format( - len(local_sizes), len(local_offsets)) + len(local_sizes), len(local_offsets) + ) local_end_offsets = list( - map(lambda x: x[0] + x[1], zip(local_offsets, local_sizes))) + map(lambda x: x[0] + x[1], zip(local_offsets, local_sizes)) + ) local_shard = list(zip(local_offsets, local_end_offsets)) return local_shard @@ -167,7 +178,11 @@ class DistributedTensor: self._local_tensor_map = {} from .dist_context import get_default_distributed_context - self._dist_context = dist_context if dist_context is not None else get_default_distributed_context( + + self._dist_context = ( + dist_context + if dist_context is not None + else get_default_distributed_context() ) # TODO: Add Automatically to dist_context after initialized and it will be adapted in the future. # self._dist_context.add_dist_tensor_for_program(self) @@ -193,9 +208,12 @@ class DistributedTensor: def _init_default_dist_attr(self): if self._dist_attr.dims_mapping is None: - if self.serial_tensor.type == core.VarDesc.VarType.READER \ - or self.serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or self.serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + self.serial_tensor.type == core.VarDesc.VarType.READER + or self.serial_tensor.type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or self.serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = self._serial_tensor.shape @@ -203,17 +221,21 @@ class DistributedTensor: self._dist_attr.dims_mapping = tensor_dims_mapping def validate_dist_attr(self): - if self.serial_tensor.type == core.VarDesc.VarType.READER \ - or self.serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or self.serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + self.serial_tensor.type == core.VarDesc.VarType.READER + or self.serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or self.serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): return True tensor_shape = self.serial_tensor.shape if len(tensor_shape) != len(self.dist_attr.dims_mapping): return False for i in range(len(self.dist_attr.dims_mapping)): if self.dist_attr.dims_mapping[ - i] < -1 or self.dist_attr.dims_mapping[i] >= len( - self.dist_attr.process_mesh.topology): + i + ] < -1 or self.dist_attr.dims_mapping[i] >= len( + self.dist_attr.process_mesh.topology + ): return False for i in range(len(self.dist_attr.process_mesh.topology)): if self.dist_attr.dims_mapping.count(i) > 1: @@ -228,10 +250,9 @@ class DistributedTensor: shard_sizes = self.dist_attr.shard_sizes processes = self.dist_attr.process_mesh.processes topology = self.dist_attr.process_mesh.topology - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) return local_sizes @@ -247,8 +268,13 @@ class DistributedTensor: processes = self.dist_attr.process_mesh.processes topology = self.dist_attr.process_mesh.topology local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + rank, + shard_sizes, + ) self._local_offsets_map[rank] = local_offsets return local_offsets @@ -268,8 +294,13 @@ class DistributedTensor: processes = self.dist_attr.process_mesh.processes topology = self.dist_attr.process_mesh.topology local_shard = DistributedTensor.get_local_shard( - global_sizes, dims_mapping, topology, processes, rank, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + rank, + shard_sizes, + ) self._local_shard_map[rank] = local_shard return local_shard @@ -324,8 +355,9 @@ class DistributedTensor: if rank is not None and not (isinstance(rank, int) and rank >= 0): raise ValueError("The rank must >= 0, but got {}".format(rank)) if block is not None and not isinstance(block, Block): - raise TypeError("The block must be Block, but got {}.".format( - type(block))) + raise TypeError( + "The block must be Block, but got {}.".format(type(block)) + ) rank = paddle.distributed.get_rank() if rank is None else rank if block is None: @@ -350,8 +382,9 @@ class DistributedTensor: def local_tensor(self, rank=None): rank = paddle.distributed.get_rank() if rank is None else rank - assert rank in self._local_tensor_map, "The rank {} local tensor has not been created.".format( - rank) + assert ( + rank in self._local_tensor_map + ), "The rank {} local tensor has not been created.".format(rank) return self._local_tensor_map[rank] def __deepcopy__(self, memo): @@ -367,7 +400,8 @@ class DistributedTensor: def __str__(self): str = "{{tensor name: {}, tensor id: {}".format( - self.serial_tensor.desc.name(), self.serial_tensor.desc.id()) + self.serial_tensor.desc.name(), self.serial_tensor.desc.id() + ) # str += ", {}".format(self.dist_attr) # return str @@ -376,8 +410,9 @@ class DistributedTensor: annotated_str = "annotated" else: annotated_str = "non-annotated" - str += ", process_mesh ({}): {}".format(annotated_str, - self.dist_attr.process_mesh) + str += ", process_mesh ({}): {}".format( + annotated_str, self.dist_attr.process_mesh + ) str += ", is_parameter: {}".format(self.serial_tensor.is_parameter) @@ -385,8 +420,9 @@ class DistributedTensor: annotated_str = "annotated" else: annotated_str = "non-annotated" - str += ", dims_mapping ({}): {}".format(annotated_str, - self.dist_attr.dims_mapping) + str += ", dims_mapping ({}): {}".format( + annotated_str, self.dist_attr.dims_mapping + ) if self.dist_attr.is_annotated("shard_mask"): annotated_str = "annotated" diff --git a/python/paddle/distributed/auto_parallel/engine.py b/python/paddle/distributed/auto_parallel/engine.py index 1d6bce931d2..1581beb2a51 100644 --- a/python/paddle/distributed/auto_parallel/engine.py +++ b/python/paddle/distributed/auto_parallel/engine.py @@ -41,7 +41,10 @@ from .planner_v2 import Planner from .parallelizer_v2 import Parallelizer from .dist_op import DistributedOperator from .dist_saver import DistributedSaver -from .dist_loader import DistributedDataLoaderFromGenerator, DistributedDataLoader +from .dist_loader import ( + DistributedDataLoaderFromGenerator, + DistributedDataLoader, +) from .utils import to_list, get_dist_attr, get_lr from .process_group import new_process_group, get_all_process_groups from .dist_context import DistributedContext, get_default_distributed_context @@ -115,16 +118,21 @@ class Engine: """ - def __init__(self, - model=None, - loss=None, - optimizer=None, - metrics=None, - cluster=None, - strategy=None): - - if model and not isinstance(model, - paddle.nn.Layer) and not callable(model): + def __init__( + self, + model=None, + loss=None, + optimizer=None, + metrics=None, + cluster=None, + strategy=None, + ): + + if ( + model + and not isinstance(model, paddle.nn.Layer) + and not callable(model) + ): raise TypeError( "'model must be sub classes of `paddle.nn.Layer` or any callable function." ) @@ -132,18 +140,20 @@ class Engine: self._loss = loss if optimizer and not isinstance( - optimizer, - (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer)): + optimizer, + (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer), + ): raise TypeError( "'optimizer' must be object of class `paddle.optimizer.Optimizer`" - " or `paddle.fluid.optimizer.Optimizer`.") + " or `paddle.fluid.optimizer.Optimizer`." + ) self._optimizer = self._validate_opt(optimizer) metrics = metrics or [] for metric in to_list(metrics): - assert isinstance(metric, Metric), \ - "{} is not sub class of Metric".format( - metric.__class__.__name__) + assert isinstance( + metric, Metric + ), "{} is not sub class of Metric".format(metric.__class__.__name__) self._metrics = to_list(metrics) if cluster and not isinstance(cluster, Cluster): @@ -159,8 +169,9 @@ class Engine: self._strategy = strategy or Strategy() if os.getenv("POD_NAME"): - print("Distribute training by paddle.distributed.launch", - flush=True) + print( + "Distribute training by paddle.distributed.launch", flush=True + ) fleet.init(is_collective=True) self._executor = None @@ -185,7 +196,7 @@ class Engine: self._has_prepared_reader = { "train": False, "eval": False, - "predict": False + "predict": False, } self._inputs_spec = [] self._labels_spec = [] @@ -220,8 +231,10 @@ class Engine: labels = sample[split:] else: raise ValueError( - "Data should be a Dataset or IterableDatset, but received {}.". - format(type(data).__name__)) + "Data should be a Dataset or IterableDatset, but received {}.".format( + type(data).__name__ + ) + ) inputs = to_list(inputs) labels = to_list(labels) @@ -264,34 +277,40 @@ class Engine: labels_spec = self._validate_spec(labels_spec) return inputs_spec, labels_spec - def _prepare_data_tensor(self, - inputs_spec, - labels_spec, - inputs=None, - labels=None): + def _prepare_data_tensor( + self, inputs_spec, labels_spec, inputs=None, labels=None + ): if _non_static_mode() or self._dygraph_mode: return None, None inputs_spec = inputs_spec if inputs_spec else [] labels_spec = labels_spec if labels_spec else [] if inputs_spec: - assert isinstance(inputs_spec, list), \ - "inputs should be list, but received {}".format(type(inputs_spec)) + assert isinstance( + inputs_spec, list + ), "inputs should be list, but received {}".format( + type(inputs_spec) + ) if inputs is None: inputs = [s._create_feed_layer() for s in inputs_spec] else: - assert isinstance(inputs, list), \ - "inputs should be list, but received {}".format(type(inputs)) + assert isinstance( + inputs, list + ), "inputs should be list, but received {}".format(type(inputs)) for input_spec, input in zip(inputs_spec, inputs): if input_spec.shape != input.shape: input.desc.set_shape(input_spec.shape) if labels_spec: - assert isinstance(labels_spec, list), \ - "labels should be list, but received {}".format(type(labels_spec)) + assert isinstance( + labels_spec, list + ), "labels should be list, but received {}".format( + type(labels_spec) + ) if labels is None: labels = [s._create_feed_layer() for s in labels_spec] else: - assert isinstance(labels, list), \ - "labels should be list, but received {}".format(type(labels)) + assert isinstance( + labels, list + ), "labels should be list, but received {}".format(type(labels)) for label_spec, label in zip(labels_spec, labels): if label_spec.shape != label.shape: label.desc.set_shape(label_spec.shape) @@ -304,7 +323,9 @@ class Engine: # NOTE: this list may be changed if Paddle changes the existing rules. related_reader_ops = [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ] # remove the first three ops if multiple run fit/evaluate/predict if dist_main_block.ops[0].type == 'create_py_reader': @@ -322,9 +343,9 @@ class Engine: for idx in reversed(reader_op_indices): new_op_desc = dist_main_block.desc._prepend_op() new_op_desc.copy_from(dist_main_block.ops[idx].desc) - new_op = Operator(dist_main_block, - new_op_desc, - type=new_op_desc.type()) + new_op = Operator( + dist_main_block, new_op_desc, type=new_op_desc.type() + ) new_reader_ops.append(new_op) dist_op = DistributedOperator(new_op) dist_context.add_dist_op_for_program(dist_op) @@ -355,16 +376,22 @@ class Engine: else: raise ValueError("Unsupported data {}".format(data)) if user_feeds is not None: - assert isinstance(user_feeds, dict), \ - "user_feeds must be a dict, but receive {}".format(type(user_feeds).__name__) + assert isinstance( + user_feeds, dict + ), "user_feeds must be a dict, but receive {}".format( + type(user_feeds).__name__ + ) for name, data in user_feeds.items(): feeds[name] = data return feeds def _prepare_fetch(self, user_fetches, mode): if user_fetches is not None: - assert isinstance(user_fetches, list), \ - "user_fetches must be a list, but receive {}".format(type(user_fetches).__name__) + assert isinstance( + user_fetches, list + ), "user_fetches must be a list, but receive {}".format( + type(user_fetches).__name__ + ) fetch_names = [] fetch_indices = [] @@ -396,14 +423,16 @@ class Engine: _process_fetch_group("fetches", var_list) return fetch_names, fetch_indices - def _prepare_logger(self, - outs, - epoch=None, - step=None, - lr=None, - fetch_names=None, - fetch_indices=None, - mode=None): + def _prepare_logger( + self, + outs, + epoch=None, + step=None, + lr=None, + fetch_names=None, + fetch_indices=None, + mode=None, + ): logs = {} if epoch is not None: logs["epoch"] = epoch @@ -470,9 +499,9 @@ class Engine: inputs_spec = self._inputs_spec labels_spec = self._labels_spec if self._labels_spec else [] - self.program_helper = ProgramHelper(self._model, self._loss, - self._metrics, inputs_spec, - labels_spec) + self.program_helper = ProgramHelper( + self._model, self._loss, self._metrics, inputs_spec, labels_spec + ) # build forward main program self.program_helper.build_program(mode) @@ -505,8 +534,9 @@ class Engine: serial_main_prog = self._orig_main_prog.clone() serial_startup_prog = self._orig_startup_prog.clone() if not self._skip_build: - with static.program_guard(serial_main_prog, serial_startup_prog), \ - utils.unique_name.guard(): + with static.program_guard( + serial_main_prog, serial_startup_prog + ), utils.unique_name.guard(): outputs = to_list(self._model(*inputs)) if mode != "predict" and self._loss: losses = to_list(self._loss(*(outputs + labels))) @@ -515,7 +545,8 @@ class Engine: if mode != "predict" and (outputs or labels): for metric in self._metrics: metrics.append( - to_list(metric.compute(*(outputs + labels)))) + to_list(metric.compute(*(outputs + labels))) + ) else: losses = to_list(self._loss) self.losses = losses @@ -532,7 +563,7 @@ class Engine: fetch_vars = { "outputs": flatten(outputs), "loss": losses, - "metrics": metrics + "metrics": metrics, } if mode != "train": @@ -540,8 +571,15 @@ class Engine: self._set_recompute_ckpts() self._dist_contexts[mode] = DistributedContext( - serial_main_prog, serial_startup_prog, self._optimizer, losses, - feed_vars, fetch_vars, self._cluster, self._strategy) + serial_main_prog, + serial_startup_prog, + self._optimizer, + losses, + feed_vars, + fetch_vars, + self._cluster, + self._strategy, + ) self._dist_contexts[mode].gradient_scale = self._strategy.gradient_scale def _optimization_tuning(self, mode, dataset, batch_size): @@ -558,20 +596,24 @@ class Engine: dataset.dp_rank = self._dp_ranks from .tuner.optimization_tuner import OptimizationTuner - self._optimization_tuner = OptimizationTuner(self._tuning.to_dict(), - self._dist_contexts[mode], - dataset, - self._inputs_spec, - self._labels_spec, - batch_size=batch_size, - rank=self._cur_rank) + + self._optimization_tuner = OptimizationTuner( + self._tuning.to_dict(), + self._dist_contexts[mode], + dataset, + self._inputs_spec, + self._labels_spec, + batch_size=batch_size, + rank=self._cur_rank, + ) self._optimization_tuner.tune() if self._tuning.run_after_tuning: # update the strategy self._dist_contexts[ - mode]._strategy = self._optimization_tuner.get_best_config() + mode + ]._strategy = self._optimization_tuner.get_best_config() def _plan(self, mode): if self._planned_mode is None: @@ -596,7 +638,8 @@ class Engine: self._dp_ranks = [] for feed_var in feed_list: dp_world_size, dp_rank = self._get_input_split_info( - feed_var, self._dist_contexts[mode]) + feed_var, self._dist_contexts[mode] + ) self._dp_world_sizes.append(dp_world_size) self._dp_ranks.append(dp_rank) @@ -604,8 +647,9 @@ class Engine: # Parallelize program based on the planner's results # For now, the completer has to be passed to the planner, # because we may use it to complete the annotation of the backwarkward and update. - parallelizer = Parallelizer(mode, self._planners[mode].completer, - self._dist_contexts[mode]) + parallelizer = Parallelizer( + mode, self._planners[mode].completer, self._dist_contexts[mode] + ) if not all_ranks: parallelizer.parallel(self._cur_rank) else: @@ -623,22 +667,30 @@ class Engine: for ib, block in enumerate(origin_main_prog.blocks): for iop, op in enumerate(block.ops): ref_op = ref_blocks[ib].ops[iop] - assert op.type == ref_op.type, \ - "'{}' mode op '{}' is different with '{}' op '{}'. ".format(mode, op.type, ref_mode, ref_op.type) - ref_op_dist_attr = ref_dist_context.get_op_dist_attr_for_program( - ref_op) + assert ( + op.type == ref_op.type + ), "'{}' mode op '{}' is different with '{}' op '{}'. ".format( + mode, op.type, ref_mode, ref_op.type + ) + ref_op_dist_attr = ( + ref_dist_context.get_op_dist_attr_for_program(ref_op) + ) dist_context.set_op_dist_attr_for_program(op, ref_op_dist_attr) def _initialize(self, mode): # Get the current content from the distributed context self._serial_main_progs[mode] = self._dist_contexts[ - mode].serial_main_program + mode + ].serial_main_program self._serial_startup_progs[mode] = self._dist_contexts[ - mode].serial_startup_program + mode + ].serial_startup_program self._dist_main_progs[mode] = self._dist_contexts[ - mode].dist_main_programs + mode + ].dist_main_programs self._dist_startup_progs[mode] = self._dist_contexts[ - mode].dist_startup_programs + mode + ].dist_startup_programs self._feed_vars[mode] = self._dist_contexts[mode].serial_feed_vars self._fetch_vars[mode] = self._dist_contexts[mode].serial_fetch_vars self._optimizer = self._dist_contexts[mode]._serial_optimizer @@ -685,30 +737,33 @@ class Engine: self._executor.run(prune_startup_prog) if hasattr(self, "_state_dict") and hasattr(self, "_dist_attr"): - self._set_state_dict(mode, self._strict, self._state_dict, - self._dist_attr) + self._set_state_dict( + mode, self._strict, self._state_dict, self._dist_attr + ) if self._strategy.reinit: self._logger.info("NOTE: parameters will be re-initialized.") dist_startup_prog = self._dist_startup_progs[mode][self._cur_rank] self._executor.run(dist_startup_prog) - def fit(self, - train_data, - train_sample_split=None, - batch_size=1, - epochs=1, - steps_per_epoch=None, - log_freq=10, - save_dir=None, - save_freq=1, - valid_data=None, - valid_sample_split=None, - valid_freq=1, - valid_steps=None, - collate_fn=None, - callbacks=None, - verbose=2): + def fit( + self, + train_data, + train_sample_split=None, + batch_size=1, + epochs=1, + steps_per_epoch=None, + log_freq=10, + save_dir=None, + save_freq=1, + valid_data=None, + valid_sample_split=None, + valid_freq=1, + valid_steps=None, + collate_fn=None, + callbacks=None, + verbose=2, + ): """ Trains the model for a fixed number of epochs. If `valid_data` is set, evaluation will be done at the end of each epoch. @@ -777,16 +832,19 @@ class Engine: """ self._mode = 'train' self._inputs_spec, self._labels_spec = self._prepare_data_spec( - train_data, train_sample_split, batch_size) + train_data, train_sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: self._switch_mode(self._mode) - assert self._mode in self._dist_main_progs, \ - "train model is not ready, please call `engine._prepare_program('train')` first." + assert ( + self._mode in self._dist_main_progs + ), "train model is not ready, please call `engine._prepare_program('train')` first." train_dataloader = self._prepare_dataloader_from_generator( dataset=train_data, @@ -795,7 +853,8 @@ class Engine: batch_size=batch_size, epochs=epochs, steps_per_epoch=steps_per_epoch, - collate_fn=collate_fn) + collate_fn=collate_fn, + ) fetch_names, fetch_indices = self._prepare_fetch(None, mode=self._mode) @@ -824,21 +883,35 @@ class Engine: self.main_program, fetch_list=fetch_names, use_program_cache=self._strategy.use_cache, - return_numpy=self._strategy.return_numpy) + return_numpy=self._strategy.return_numpy, + ) except core.EOFException: break lr = get_lr(self._optimizer) - logs = self._prepare_logger(outs, epoch, step, lr, fetch_names, - fetch_indices, self._mode) + logs = self._prepare_logger( + outs, + epoch, + step, + lr, + fetch_names, + fetch_indices, + self._mode, + ) cbks.on_batch_end('train', step, logs) if valid_data and (epoch + 1) % valid_freq == 0: - val_logs = self.evaluate(valid_data, valid_sample_split, - batch_size, valid_steps, log_freq, - collate_fn, callbacks, verbose) + val_logs = self.evaluate( + valid_data, + valid_sample_split, + batch_size, + valid_steps, + log_freq, + collate_fn, + callbacks, + verbose, + ) val_logs = { - "val_" + name: val - for name, val in val_logs.items() + "val_" + name: val for name, val in val_logs.items() } logs.update(val_logs) self._switch_mode("train") @@ -850,15 +923,17 @@ class Engine: cbks.on_end('train', logs) return self.history - def evaluate(self, - valid_data, - valid_sample_split=None, - batch_size=1, - steps=None, - log_freq=10, - collate_fn=None, - callbacks=None, - verbose=2): + def evaluate( + self, + valid_data, + valid_sample_split=None, + batch_size=1, + steps=None, + log_freq=10, + collate_fn=None, + callbacks=None, + verbose=2, + ): """ Evaluate the loss and metrics of the model on evaluation data. @@ -907,23 +982,27 @@ class Engine: """ self._mode = 'eval' self._inputs_spec, self._labels_spec = self._prepare_data_spec( - valid_data, valid_sample_split, batch_size) + valid_data, valid_sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: self._switch_mode(self._mode) - assert self._mode in self._dist_main_progs, \ - "eval model is not ready, please call `engine._prepare_program('eval')` first." + assert ( + self._mode in self._dist_main_progs + ), "eval model is not ready, please call `engine._prepare_program('eval')` first." valid_dataloader = self._prepare_dataloader_from_generator( dataset=valid_data, capacity=70, iterable=False, batch_size=batch_size, steps_per_epoch=steps, - collate_fn=collate_fn) + collate_fn=collate_fn, + ) fetch_names, fetch_indices = self._prepare_fetch(None, mode=self._mode) @@ -937,10 +1016,9 @@ class Engine: ) eval_steps = valid_dataloader._steps - cbks.on_begin('eval', { - 'steps': eval_steps, - 'metrics': self._metrics_name() - }) + cbks.on_begin( + 'eval', {'steps': eval_steps, 'metrics': self._metrics_name()} + ) logs = {} for step, _ in enumerate(valid_dataloader): cbks.on_batch_begin('eval', step, logs) @@ -949,24 +1027,28 @@ class Engine: self.main_program, fetch_list=fetch_names, use_program_cache=self._strategy.use_cache, - return_numpy=self._strategy.return_numpy) + return_numpy=self._strategy.return_numpy, + ) except core.EOFException: break - logs = self._prepare_logger(outs, None, step, None, fetch_names, - fetch_indices, self._mode) + logs = self._prepare_logger( + outs, None, step, None, fetch_names, fetch_indices, self._mode + ) cbks.on_batch_end('eval', step, logs) cbks.on_end('eval', logs) self._reset_metrics() return logs - def predict(self, - test_data, - test_sample_split=None, - batch_size=1, - steps=None, - collate_fn=None, - callbacks=None, - verbose=2): + def predict( + self, + test_data, + test_sample_split=None, + batch_size=1, + steps=None, + collate_fn=None, + callbacks=None, + verbose=2, + ): """ Compute the output predictions on testing data. @@ -1012,16 +1094,19 @@ class Engine: """ self._mode = 'predict' self._inputs_spec, self._labels_spec = self._prepare_data_spec( - test_data, test_sample_split, batch_size) + test_data, test_sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: self._switch_mode(self._mode) - assert self._mode in self._dist_main_progs, \ - "predict model is not ready, please call `engine._prepare_program('predict')` first." + assert ( + self._mode in self._dist_main_progs + ), "predict model is not ready, please call `engine._prepare_program('predict')` first." test_dataloader = self._prepare_dataloader_from_generator( dataset=test_data, @@ -1029,7 +1114,8 @@ class Engine: iterable=False, batch_size=batch_size, steps_per_epoch=steps, - collate_fn=collate_fn) + collate_fn=collate_fn, + ) fetch_names, fetch_indices = self._prepare_fetch(None, mode=self._mode) @@ -1045,37 +1131,43 @@ class Engine: self.main_program, fetch_list=fetch_names, use_program_cache=self._strategy.use_cache, - return_numpy=self._strategy.return_numpy) + return_numpy=self._strategy.return_numpy, + ) except core.EOFException: break - logs = self._prepare_logger(outs, None, step, None, fetch_names, - fetch_indices, self._mode) + logs = self._prepare_logger( + outs, None, step, None, fetch_names, fetch_indices, self._mode + ) cbks.on_batch_end('predict', step, logs) outputs.append(list(logs["outputs"].values())) cbks.on_end('predict', logs) return outputs - def dataloader(self, - dataset, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=None, - num_workers=0, - use_buffer_reader=True, - use_shared_memory=True, - timeout=0, - worker_init_fn=None, - epochs=1, - steps_per_epoch=None, - sample_split=1, - mode=None): + def dataloader( + self, + dataset, + batch_size=1, + shuffle=False, + drop_last=False, + collate_fn=None, + num_workers=0, + use_buffer_reader=True, + use_shared_memory=True, + timeout=0, + worker_init_fn=None, + epochs=1, + steps_per_epoch=None, + sample_split=1, + mode=None, + ): if mode is not None: self.to_mode(mode) self._inputs_spec, self._labels_spec = self._prepare_data_spec( - dataset, sample_split, batch_size) + dataset, sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: @@ -1093,28 +1185,33 @@ class Engine: timeout=timeout, worker_init_fn=worker_init_fn, epochs=epochs, - steps_per_epoch=steps_per_epoch) + steps_per_epoch=steps_per_epoch, + ) return dataloader - def dataloader_from_generator(self, - dataset, - capacity=70, - use_double_buffer=True, - iterable=True, - use_multiprocess=False, - drop_last=True, - batch_size=1, - epochs=1, - steps_per_epoch=None, - collate_fn=None, - sample_split=1, - mode=None): + def dataloader_from_generator( + self, + dataset, + capacity=70, + use_double_buffer=True, + iterable=True, + use_multiprocess=False, + drop_last=True, + batch_size=1, + epochs=1, + steps_per_epoch=None, + collate_fn=None, + sample_split=1, + mode=None, + ): if mode is not None: self.to_mode(mode) self._inputs_spec, self._labels_spec = self._prepare_data_spec( - dataset, sample_split, batch_size) + dataset, sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: @@ -1130,17 +1227,20 @@ class Engine: batch_size=batch_size, epochs=epochs, steps_per_epoch=steps_per_epoch, - collate_fn=collate_fn) + collate_fn=collate_fn, + ) return dataloader - def prepare(self, - inputs_spec=None, - labels_spec=None, - inputs=None, - labels=None, - main_program=None, - startup_program=None, - mode=None): + def prepare( + self, + inputs_spec=None, + labels_spec=None, + inputs=None, + labels=None, + main_program=None, + startup_program=None, + mode=None, + ): if mode is not None: self.to_mode(mode) if inputs or labels: @@ -1148,7 +1248,8 @@ class Engine: self._inputs_spec = inputs_spec self._labels_spec = labels_spec self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec, inputs, labels) + self._inputs_spec, self._labels_spec, inputs, labels + ) self._orig_main_prog = main_program if self._orig_main_prog is None: self._orig_main_prog = static.default_main_program() @@ -1164,7 +1265,8 @@ class Engine: self._labels_spec = labels_spec self._outside_dataloader = True self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) self._orig_main_prog = main_program if self._orig_main_prog is None: self._orig_main_prog = static.default_main_program() @@ -1176,44 +1278,55 @@ class Engine: else: self._switch_mode(self._mode) else: - assert self._inputs_spec and self._labels_spec, \ - "Please call the dataloader(...) before calling prepare(...)" + assert ( + self._inputs_spec and self._labels_spec + ), "Please call the dataloader(...) before calling prepare(...)" def run(self, data=None, feed=None, fetch_list=None, mode=None): if mode is not None: self.to_mode(mode) feed_dict = self._prepare_feed(data, feed, self._mode) fetch_names, fetch_indices = self._prepare_fetch(fetch_list, self._mode) - if self._outside_dataloader and not self._has_prepared_reader[ - self._mode]: + if ( + self._outside_dataloader + and not self._has_prepared_reader[self._mode] + ): self._prepare_reader() - outs = self._executor.run(self.main_program, - feed=feed_dict, - fetch_list=fetch_names, - use_program_cache=self._strategy.use_cache, - return_numpy=self._strategy.return_numpy) - logs = self._prepare_logger(outs, None, None, None, fetch_names, - fetch_indices, self._mode) + outs = self._executor.run( + self.main_program, + feed=feed_dict, + fetch_list=fetch_names, + use_program_cache=self._strategy.use_cache, + return_numpy=self._strategy.return_numpy, + ) + logs = self._prepare_logger( + outs, None, None, None, fetch_names, fetch_indices, self._mode + ) return logs - def _prepare_dataloader(self, - dataset, - return_list=True, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=None, - num_workers=0, - use_buffer_reader=True, - use_shared_memory=True, - timeout=0, - worker_init_fn=None, - epochs=1, - steps_per_epoch=None): + def _prepare_dataloader( + self, + dataset, + return_list=True, + batch_size=1, + shuffle=False, + drop_last=False, + collate_fn=None, + num_workers=0, + use_buffer_reader=True, + use_shared_memory=True, + timeout=0, + worker_init_fn=None, + epochs=1, + steps_per_epoch=None, + ): if self._strategy.gradient_merge and batch_size is not None: - assert batch_size % self._k_steps == 0, \ - "Requires batch_size:[{}] to be divisible by k_steps:[{}].".format(batch_size, self._k_steps) + assert ( + batch_size % self._k_steps == 0 + ), "Requires batch_size:[{}] to be divisible by k_steps:[{}].".format( + batch_size, self._k_steps + ) batch_size //= self._k_steps dist_main_prog = self._dist_main_progs[self._mode][self._cur_rank] @@ -1257,26 +1370,32 @@ class Engine: steps_per_epoch=steps_per_epoch, split_data=self._strategy.split_data, data_parallel_world_size=self._dp_world_sizes, - data_parallel_rank=self._dp_ranks) + data_parallel_rank=self._dp_ranks, + ) return dataloader - def _prepare_dataloader_from_generator(self, - dataset, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False, - use_multiprocess=False, - drop_last=True, - batch_size=1, - epochs=1, - steps_per_epoch=None, - collate_fn=None): + def _prepare_dataloader_from_generator( + self, + dataset, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + use_multiprocess=False, + drop_last=True, + batch_size=1, + epochs=1, + steps_per_epoch=None, + collate_fn=None, + ): if self._strategy.gradient_merge and batch_size is not None: - assert batch_size % self._k_steps == 0, \ - "Requires batch_size:[{}] to be divisible by k_steps:[{}].".format(batch_size, self._k_steps) + assert ( + batch_size % self._k_steps == 0 + ), "Requires batch_size:[{}] to be divisible by k_steps:[{}].".format( + batch_size, self._k_steps + ) batch_size //= self._k_steps dist_main_prog = self._dist_main_progs[self._mode][self._cur_rank] @@ -1317,16 +1436,19 @@ class Engine: collate_fn=collate_fn, split_data=self._strategy.split_data, data_parallel_world_size=self._dp_world_sizes, - data_parallel_rank=self._dp_ranks) + data_parallel_rank=self._dp_ranks, + ) self._prepare_reader() return dataloader def _tune(self, tune_data, tune_sample_split=None, batch_size=1): self._mode = 'train' self._inputs_spec, self._labels_spec = self._prepare_data_spec( - tune_data, tune_sample_split, batch_size) + tune_data, tune_sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) self._optimization_tuning(self._mode, tune_data, batch_size) def _validate_spec(self, specs): @@ -1337,12 +1459,17 @@ class Engine: assert isinstance(spec, InputSpec) if spec.name is None: raise ValueError( - "Requires Input[{}].name != None, but receive `None` with {}." - .format(i, spec)) + "Requires Input[{}].name != None, but receive `None` with {}.".format( + i, spec + ) + ) if self._k_steps > 1: shape = list(spec.shape) - assert shape[0] % self._k_steps == 0, \ - "Requires batch_size[{}] to be divisible by k_steps[{}].".format(spec.shape[0], self._k_steps) + assert ( + shape[0] % self._k_steps == 0 + ), "Requires batch_size[{}] to be divisible by k_steps[{}].".format( + spec.shape[0], self._k_steps + ) shape[0] //= self._k_steps spec.shape = shape return specs @@ -1360,16 +1487,20 @@ class Engine: dims_mapping = tensor_dist_attr.dims_mapping if self._cur_rank not in process_mesh.processes: - rank_id = _get_corresponding_rank(dist_context, process_mesh, - self._cur_rank) + rank_id = _get_corresponding_rank( + dist_context, process_mesh, self._cur_rank + ) else: rank_id = self._cur_rank batch_size_axis = dims_mapping[0] if batch_size_axis > -1 and process_mesh.topology[batch_size_axis] > 1: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, - batch_size_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + batch_size_axis, + rank_id, + ) return len(group_ranks), group_ranks.index(rank_id) return 1, 0 @@ -1382,10 +1513,12 @@ class Engine: # extract ckpts by specific model if isinstance(self._model, paddle.nn.Layer): - if hasattr(self._model, - "gpt") and self._model.__class__.__name__ in [ - 'GPTForPretraining', 'GPTForPretrainingAuto' - ]: + if hasattr( + self._model, "gpt" + ) and self._model.__class__.__name__ in [ + 'GPTForPretraining', + 'GPTForPretrainingAuto', + ]: exact_ckpts = self._model.gpt.checkpoints else: exact_ckpts = recompute.checkpoints @@ -1397,7 +1530,7 @@ class Engine: recompute.checkpoints = exact_ckpts[:] logs = { 'Model Class': self._model.__class__.__name__, - 'Applied Recompute ckpts': exact_ckpts + 'Applied Recompute ckpts': exact_ckpts, } self._logger.info(logs) @@ -1422,8 +1555,11 @@ class Engine: self._optimizer = self._dist_contexts[mode]._serial_optimizer def to_mode(self, mode): - assert mode in ["train", "eval", "predict"], \ - "mode {} should be one of ['train', 'eval', 'predict']".format(mode) + assert mode in [ + "train", + "eval", + "predict", + ], "mode {} should be one of ['train', 'eval', 'predict']".format(mode) self._mode = mode def _set_state_dict(self, mode, strict, state_dict, dist_attr): @@ -1484,20 +1620,24 @@ class Engine: serial_program = self._serial_main_progs[self._mode] dist_main_prog = self._dist_main_progs[self._mode][self._cur_rank] dist_context = self._dist_contexts[self._mode] - self._saver.save(path, - serial_program=serial_program, - dist_main_program=dist_main_prog, - dist_context=dist_context) + self._saver.save( + path, + serial_program=serial_program, + dist_main_program=dist_main_prog, + dist_context=dist_context, + ) else: assert "predict" in self._dist_main_progs feed_vars = self._feed_vars["predict"]['inputs'] fetch_vars = self._fetch_vars["predict"]['outputs'] dist_main_prog = self._dist_main_progs["predict"][self._cur_rank] - self._saver.save_inference_model(path, - feed_vars, - fetch_vars, - self._executor, - program=dist_main_prog) + self._saver.save_inference_model( + path, + feed_vars, + fetch_vars, + self._executor, + program=dist_main_prog, + ) def load(self, path, strict=True, load_optimizer=True): """ @@ -1547,7 +1687,8 @@ class Engine: """ self._strict = strict self._state_dict, self._dist_attr = self._saver.load( - path, load_optimizer) + path, load_optimizer + ) return self._state_dict, self._dist_attr def cost(self, inputs_spec=None, labels_spec=None, mode="train"): @@ -1577,14 +1718,18 @@ class Engine: # Check mode accepted_modes = ["train", "predict", "eval"] if mode not in accepted_modes: - raise ValueError("The mode {} is not in accepted modes {}".format( - mode, accepted_modes)) + raise ValueError( + "The mode {} is not in accepted modes {}".format( + mode, accepted_modes + ) + ) self.to_mode(mode) if inputs_spec is not None: self._inputs_spec, self._labels_spec = inputs_spec, labels_spec self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) self._build(mode) self._plan(mode) else: diff --git a/python/paddle/distributed/auto_parallel/graph.py b/python/paddle/distributed/auto_parallel/graph.py index de6505071ab..d671d95aa9f 100644 --- a/python/paddle/distributed/auto_parallel/graph.py +++ b/python/paddle/distributed/auto_parallel/graph.py @@ -14,7 +14,6 @@ class Node: - def __init__(self, id, **attrs): # Each node must has a unique id self._id = id @@ -48,7 +47,6 @@ class Node: class Edge: - def __init__(self, src_id, tgt_id, **attrs): # The id of source node in an Edge self._src_id = src_id @@ -85,12 +83,12 @@ class Edge: def __str__(self): str = "" str += "(src_id: {}, tgt_id: {}, attrs: {})".format( - self.src_id, self.tgt_id, self._attrs) + self.src_id, self.tgt_id, self._attrs + ) return str class Graph: - def __init__(self, **attrs): # _nodes is dict for storing the nodes of the graph. # The key of this dict is the node id. diff --git a/python/paddle/distributed/auto_parallel/helper.py b/python/paddle/distributed/auto_parallel/helper.py index 3173f51e249..f73668a2ba9 100644 --- a/python/paddle/distributed/auto_parallel/helper.py +++ b/python/paddle/distributed/auto_parallel/helper.py @@ -20,7 +20,9 @@ from paddle.jit import to_static, not_to_static from paddle.fluid.framework import Parameter from paddle.fluid.framework import program_guard from paddle.fluid.executor import global_scope -from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + StaticFunction, +) from .utils import to_list from .utils import get_logger @@ -174,7 +176,6 @@ class ProxyLayer(Layer): class BuildInfo: - def __init__(self): self.clear() @@ -224,8 +225,9 @@ class ProgramHelper(object): # skip if we has already built program. if self.build_info.has_cache(mode, True): self._logger.info( - "Already build program with mode = %s, use cached program." % - mode) + "Already build program with mode = %s, use cached program." + % mode + ) return self._logger.info("start to build program for mode = %s." % mode) @@ -249,21 +251,24 @@ class ProgramHelper(object): self.lazy_init = True return for param in self.concrete_program.parameters: - Parameter(name=param.name, - desc=param, - type=param.type, - shape=param.shape, - dtype=param.dtype, - stop_gradient=param.stop_gradient, - block=self.startup_program.global_block()) + Parameter( + name=param.name, + desc=param, + type=param.type, + shape=param.shape, + dtype=param.dtype, + stop_gradient=param.stop_gradient, + block=self.startup_program.global_block(), + ) def apply_optimizer(self, optimizer): """ Append backward and generate optimizer operations. """ self._verify_optimizer(optimizer) - self._logger.info("start to apply optimizer: %s ", - type(optimizer).__name__) + self._logger.info( + "start to apply optimizer: %s ", type(optimizer).__name__ + ) # clear optimizer parameters original_params = optimizer._parameter_list optimizer._parameter_list = None @@ -276,13 +281,17 @@ class ProgramHelper(object): def _verify_optimizer(self, optimizer): assert optimizer is not None - assert hasattr(optimizer, - "minimize"), "Optimizer must have minimize() method." - assert self.proxy_layer.mode == 'train', "Required mode == 'train', but received '%s'" % self.proxy_layer.mode - assert len( - self.loss_vars - ) == 1, "Required len(loss_vars) == 1, but received len(loss_vars) = %s" % len( - self.loss_vars) + assert hasattr( + optimizer, "minimize" + ), "Optimizer must have minimize() method." + assert self.proxy_layer.mode == 'train', ( + "Required mode == 'train', but received '%s'" + % self.proxy_layer.mode + ) + assert len(self.loss_vars) == 1, ( + "Required len(loss_vars) == 1, but received len(loss_vars) = %s" + % len(self.loss_vars) + ) def to(self, mode): """ @@ -291,7 +300,8 @@ class ProgramHelper(object): assert mode in ['train', 'eval', 'predict'] func = getattr(self.proxy_layer, '_' + mode) assert isinstance( - func, StaticFunction), "Please call build_program(mode) firstly." + func, StaticFunction + ), "Please call build_program(mode) firstly." self.proxy_layer.set_mode(mode) def static_func(self): @@ -299,7 +309,9 @@ class ProgramHelper(object): Return StaticFunction instance with underly target mode. """ assert self.proxy_layer.mode in [ - 'train', 'eval', 'predict' + 'train', + 'eval', + 'predict', ], "Please call build_program(mode) firstly." func_name = '_' + self.proxy_layer.mode return getattr(self.proxy_layer, func_name) @@ -317,13 +329,14 @@ class ProgramHelper(object): dist_attr = { "dims_mapping": var_dist_attr.dims_mapping, "process_shape": var_dist_attr.process_mesh.topology, - "process_group": var_dist_attr.process_mesh.processes + "process_group": var_dist_attr.process_mesh.processes, } # slice param_value with dist_attr # share sliced_param_value with param_tensor in global_scope param_tensor = global_scope().var(param.name).get_tensor() sliced_param = Converter.slice_with_dist_attr( - param.numpy(), dist_attr) + param.numpy(), dist_attr + ) param_tensor.set(sliced_param, place) @property diff --git a/python/paddle/distributed/auto_parallel/interface.py b/python/paddle/distributed/auto_parallel/interface.py index 0abdc0ea768..124f622d40f 100644 --- a/python/paddle/distributed/auto_parallel/interface.py +++ b/python/paddle/distributed/auto_parallel/interface.py @@ -63,29 +63,39 @@ def shard_tensor(x, process_mesh=None, shard_spec=None): """ if process_mesh is not None: - assert isinstance(process_mesh, ProcessMesh), \ - "Argument process_mesh {} is not an instance of ProcessMesh".format(process_mesh) + assert isinstance( + process_mesh, ProcessMesh + ), "Argument process_mesh {} is not an instance of ProcessMesh".format( + process_mesh + ) else: process_mesh = get_current_process_mesh() - assert process_mesh is not None, \ - "Specify the process mesh argument or use ProcessMesh context manager first." - assert isinstance(shard_spec, list), \ - "Argument shard_spec {} is not an instance of list".format(shard_spec) + assert ( + process_mesh is not None + ), "Specify the process mesh argument or use ProcessMesh context manager first." + assert isinstance( + shard_spec, list + ), "Argument shard_spec {} is not an instance of list".format(shard_spec) dist_tensor = DistributedTensor(x) serial_tensor = dist_tensor.serial_tensor dist_tensor.dist_attr.process_mesh = process_mesh - if serial_tensor.type == core.VarDesc.VarType.READER \ - or serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + serial_tensor.type == core.VarDesc.VarType.READER + or serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = serial_tensor.shape if shard_spec is not None: - assert verify_shard_spec(shard_spec, tensor_shape, process_mesh), \ - "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( - serial_tensor.name, shard_spec, tensor_shape, process_mesh) + assert verify_shard_spec( + shard_spec, tensor_shape, process_mesh + ), "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( + serial_tensor.name, shard_spec, tensor_shape, process_mesh + ) dist_tensor.dist_attr.dims_mapping = convert_to_dims_mapping( - shard_spec, process_mesh) + shard_spec, process_mesh + ) if process_mesh is not None: dist_tensor.dist_attr.mark_annotated("process_mesh") if shard_spec is not None: @@ -140,41 +150,54 @@ def shard_op(op, process_mesh=None, in_shard_specs=None, out_shard_specs=None): """ if process_mesh is not None: - assert isinstance(process_mesh, ProcessMesh), \ - "Argument process_mesh {} is not an instance of ProcessMesh".format(process_mesh) + assert isinstance( + process_mesh, ProcessMesh + ), "Argument process_mesh {} is not an instance of ProcessMesh".format( + process_mesh + ) else: process_mesh = get_current_process_mesh() - assert process_mesh is not None, \ - "Specify the process mesh argument or use ProcessMesh context manager first." + assert ( + process_mesh is not None + ), "Specify the process mesh argument or use ProcessMesh context manager first." in_dims_mappings = [] if in_shard_specs is not None: - assert all((isinstance(shard_spec, list) or shard_spec is None) for shard_spec in in_shard_specs), \ - "in_shard_spec {} is not a list of list or None".format(in_shard_specs) + assert all( + (isinstance(shard_spec, list) or shard_spec is None) + for shard_spec in in_shard_specs + ), "in_shard_spec {} is not a list of list or None".format( + in_shard_specs + ) for shard_spec in in_shard_specs: if shard_spec is not None: in_dims_mappings.append( - convert_to_dims_mapping(shard_spec, process_mesh)) + convert_to_dims_mapping(shard_spec, process_mesh) + ) else: in_dims_mappings.append(None) out_dims_mappings = [] if out_shard_specs is not None: - assert all((isinstance(shard_spec, list) or shard_spec is None) for shard_spec in out_shard_specs), \ - "out_shard_spec {} is not a list of list or None".format(out_shard_specs) + assert all( + (isinstance(shard_spec, list) or shard_spec is None) + for shard_spec in out_shard_specs + ), "out_shard_spec {} is not a list of list or None".format( + out_shard_specs + ) for shard_spec in out_shard_specs: if shard_spec is not None: out_dims_mappings.append( - convert_to_dims_mapping(shard_spec, process_mesh)) + convert_to_dims_mapping(shard_spec, process_mesh) + ) else: out_dims_mappings.append(None) - op = DistributedOperatorHelper(op, process_mesh, in_dims_mappings, - out_dims_mappings) + op = DistributedOperatorHelper( + op, process_mesh, in_dims_mappings, out_dims_mappings + ) return op def recompute(op): - class RecomputeOperator: - def __init__(self, op): self._op = op @@ -215,11 +238,13 @@ def add_to_collection(collection_name, value, name=None): _g_collections[collection_name] = [] if name is not None: for _, v in _g_collections[collection_name]: - if v == value: return + if v == value: + return _g_collections[collection_name].append((name, value)) else: for _, v in _g_collections[collection_name]: - if v == value: return + if v == value: + return _g_collections[collection_name].append((None, value)) diff --git a/python/paddle/distributed/auto_parallel/mapper.py b/python/paddle/distributed/auto_parallel/mapper.py index f8c0792c580..d7b599aaa1c 100644 --- a/python/paddle/distributed/auto_parallel/mapper.py +++ b/python/paddle/distributed/auto_parallel/mapper.py @@ -24,9 +24,16 @@ from .process_group import get_process_group def is_collective_comm_op(op): comm_list = [ - "c_allreduce_sum", "c_allreduce_min", "c_allreduce_max", - "c_allreduce_prod", "c_reduce_sum", "c_reduce_min", "c_reduce_max", - "c_reduce_prod", "c_broadcast", "c_allgather" + "c_allreduce_sum", + "c_allreduce_min", + "c_allreduce_max", + "c_allreduce_prod", + "c_reduce_sum", + "c_reduce_min", + "c_reduce_max", + "c_reduce_prod", + "c_broadcast", + "c_allgather", ] if op.type in comm_list: return True @@ -127,7 +134,8 @@ def analyze_comm_requirements_from_op(op, rank, g_process_group_map): if comm_volume is not None: comm_requirements_to_ranks[tgt_rank] = {} comm_requirements_to_ranks[tgt_rank][ - "comm_volume"] = comm_volume + "comm_volume" + ] = comm_volume elif is_p2p_comm_op(op): tgt_rank = op.attr("peer") comm_volume = get_comm_volume(op, rank, tgt_rank) @@ -149,28 +157,33 @@ def analyze_requirements_for_program(src_info, rank): for block in program.blocks: for op in block.ops: cur_comm_requirements_to_ranks = analyze_comm_requirements_from_op( - op, rank, g_process_group_map) + op, rank, g_process_group_map + ) for tgt_rank, link_info in cur_comm_requirements_to_ranks.items(): if tgt_rank in comm_requirements_to_ranks: comm_requirements_to_ranks[tgt_rank][ - "comm_volume"] += link_info["comm_volume"] + "comm_volume" + ] += link_info["comm_volume"] else: comm_requirements_to_ranks[tgt_rank] = {} comm_requirements_to_ranks[tgt_rank][ - "comm_volume"] = link_info["comm_volume"] + "comm_volume" + ] = link_info["comm_volume"] return resource_requirements, comm_requirements_to_ranks def build_process_graph(distributed_program): graph = Graph() for src_rank, src_info in distributed_program.items(): - resource_requirements, comm_requirements_to_ranks = analyze_requirements_for_program( - src_info, src_rank) + ( + resource_requirements, + comm_requirements_to_ranks, + ) = analyze_requirements_for_program(src_info, src_rank) graph.add_node(src_rank, resource_requirements=resource_requirements) for tgt_rank, comm_requirements in comm_requirements_to_ranks.items(): - graph.add_edge(src_rank, - tgt_rank, - comm_requirements=comm_requirements) + graph.add_edge( + src_rank, tgt_rank, comm_requirements=comm_requirements + ) return graph @@ -185,14 +198,17 @@ def build_cluster_graph(cluster): for machine in cluster.machines.values(): for device in machine.devices.values(): graph.add_node(device.global_id, device=device) - if cuda_visible_devices and device.local_id not in cuda_visible_devices: + if ( + cuda_visible_devices + and device.local_id not in cuda_visible_devices + ): graph.nodes[device.global_id]["occupied"] = True else: graph.nodes[device.global_id]["occupied"] = False for link in machine.links.values(): - graph.add_edge(link.source.global_id, - link.target.global_id, - link=link) + graph.add_edge( + link.source.global_id, link.target.global_id, link=link + ) return graph @@ -222,7 +238,8 @@ def mapping(distributed_program, cluster): queue = deque() root_rank_node = select_unvisited_rank_node( - list(process_graph.nodes.values())) + list(process_graph.nodes.values()) + ) while root_rank_node is not None: queue.append(root_rank_node) while queue: @@ -232,48 +249,61 @@ def mapping(distributed_program, cluster): device_type = cur_rank_node["resource_requirements"]["device_type"] cur_device_node = None for device_node in cluster_graph.nodes.values(): - if (device_node["device"].type - == device_type) and (not device_node["occupied"]): + if (device_node["device"].type == device_type) and ( + not device_node["occupied"] + ): device_node["occupied"] = True cur_rank_node["visited"] = True cur_rank_node["device"] = device_node["device"] cur_device_node = device_node break - assert cur_device_node, "Cannot find a device to satisfy the requirement." + assert ( + cur_device_node + ), "Cannot find a device to satisfy the requirement." nbr_rank_edges = [] for nbr_rank_node_id, nbr_rank_edge in process_graph.adjs[ - cur_rank_node.id].items(): - assert nbr_rank_edge.src_id == cur_rank_node.id and nbr_rank_edge.tgt_id == nbr_rank_node_id + cur_rank_node.id + ].items(): + assert ( + nbr_rank_edge.src_id == cur_rank_node.id + and nbr_rank_edge.tgt_id == nbr_rank_node_id + ) queue.append(process_graph.nodes[nbr_rank_node_id]) nbr_rank_edges.append(nbr_rank_edge) nbr_rank_edges.sort(key=sort_by_comm_volume) nbr_device_edges = [] for nbr_device_edge in cluster_graph.adjs[ - cur_device_node.id].values(): + cur_device_node.id + ].values(): nbr_device_edges.append(nbr_device_edge) nbr_device_edges.sort(key=sort_by_comm_bandwidth) for nbr_rank_edge in nbr_rank_edges: - src_rank_node = process_graph.nodes[ - nbr_rank_edge.src_id]["visited"] + src_rank_node = process_graph.nodes[nbr_rank_edge.src_id][ + "visited" + ] if src_rank_node: continue device_type = src_rank_node["resource_requirements"][ - "device_type"] + "device_type" + ] nbr_rank_node = process_graph.nodes[nbr_rank_edge.tgt_id] for nbr_device_edge in nbr_device_edges: nbr_device_node = cluster_graph.nodes[ - nbr_device_edge.tgt_id] + nbr_device_edge.tgt_id + ] if (nbr_device_node["device"].type == device_type) and ( - not nbr_device_node["occupied"]): + not nbr_device_node["occupied"] + ): nbr_device_node["occupied"] = True nbr_rank_node["visited"] = True nbr_rank_node["device"] = nbr_device_node["device"] break root_rank_node = select_unvisited_rank_node( - list(process_graph.nodes.values())) + list(process_graph.nodes.values()) + ) rank_mapping = {} for rank, rank_node in process_graph.nodes.items(): diff --git a/python/paddle/distributed/auto_parallel/operators/common.py b/python/paddle/distributed/auto_parallel/operators/common.py index 7bd51d3f986..af9c53a88ea 100644 --- a/python/paddle/distributed/auto_parallel/operators/common.py +++ b/python/paddle/distributed/auto_parallel/operators/common.py @@ -21,16 +21,22 @@ from ..process_group import new_process_group _g_distributed_operator_impl_containers = {} _g_elementwise_ops = [ - "elementwise", "gelu", "dropout", "cast", "gather", "concat", - "fused_softmax_mask_upper_triangle" + "elementwise", + "gelu", + "dropout", + "cast", + "gather", + "concat", + "fused_softmax_mask_upper_triangle", ] BACKWARD_ONLY_DIST_OPS = {'check_finite_and_unscale', 'update_loss_scaling'} -class ParallelMode(): +class ParallelMode: """ the parallel mode for communication or auxiliary operator """ + DataParallel = "auto_parallel/data_parallel" ModelParallel = "auto_parallel/model_parallel" PipelineParalel = "auto_parallel/pipeline_paralel" @@ -46,7 +52,6 @@ def is_elementwise_op(op_type): class DistributedOperatorImplContainer: - def __init__(self, op_type): self._type = op_type self._impls = [] @@ -64,8 +69,9 @@ class DistributedOperatorImplContainer: return self._impls def register_impl(self, dist_impl): - assert self.type == dist_impl.type, \ - "Op type of container must be same as that of the implementation." + assert ( + self.type == dist_impl.type + ), "Op type of container must be same as that of the implementation." impl_idx = len(self.impls) dist_impl.idx = impl_idx self._impls.append(dist_impl) @@ -96,7 +102,6 @@ class DistributedOperatorImplContainer: class DistributedOperatorImpl(abc.ABC): - def __init__(self, name): self._name = name self._type = None @@ -181,54 +186,69 @@ def find_compatible_distributed_operator_impls(dist_op, fwd=True, partial=True): op_type = dist_op.serial_op.type dist_op_impl_container = get_distributed_operator_impl_container(op_type) dist_op_eltwise_impl_container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) dist_op_default_impl_container = get_distributed_operator_impl_container( - "default") + "default" + ) compatible_impls = [] if partial: if fwd: # First, find impls in the corresponding container if dist_op_impl_container: compatible_impls.extend( - dist_op_impl_container.get_input_compatible_impls(dist_op)) + dist_op_impl_container.get_input_compatible_impls(dist_op) + ) # Second, find impls in the elementwise container if dist_op_eltwise_impl_container and is_elementwise_op(op_type): compatible_impls.extend( dist_op_eltwise_impl_container.get_input_compatible_impls( - dist_op)) + dist_op + ) + ) # Third, find impls in the default container if dist_op_default_impl_container: compatible_impls.extend( dist_op_default_impl_container.get_input_compatible_impls( - dist_op)) + dist_op + ) + ) else: # First, find impls in the corresponding container if dist_op_impl_container: compatible_impls.extend( - dist_op_impl_container.get_output_compatible_impls(dist_op)) + dist_op_impl_container.get_output_compatible_impls(dist_op) + ) # Second, find impls in the elementwise container if dist_op_eltwise_impl_container and is_elementwise_op(op_type): compatible_impls.extend( dist_op_eltwise_impl_container.get_output_compatible_impls( - dist_op)) + dist_op + ) + ) # Third, find impls in the default container if dist_op_default_impl_container: compatible_impls.extend( dist_op_default_impl_container.get_output_compatible_impls( - dist_op)) + dist_op + ) + ) else: # First, find impls in the corresponding container if dist_op_impl_container: compatible_impls.extend( - dist_op_impl_container.get_compatible_impls(dist_op)) + dist_op_impl_container.get_compatible_impls(dist_op) + ) # Second, find impls in the elementwise container if dist_op_eltwise_impl_container and is_elementwise_op(op_type): compatible_impls.extend( - dist_op_eltwise_impl_container.get_compatible_impls(dist_op)) + dist_op_eltwise_impl_container.get_compatible_impls(dist_op) + ) # Third, find impls in the default container if dist_op_default_impl_container: compatible_impls.extend( - dist_op_default_impl_container.get_compatible_impls(dist_op)) + dist_op_default_impl_container.get_compatible_impls(dist_op) + ) if compatible_impls: # For now, just return the first compatible impl @@ -241,11 +261,11 @@ def find_compatible_distributed_operator_impls(dist_op, fwd=True, partial=True): def is_parameter_related(varname, block): if ".subprog_" in varname: - varname = varname[:varname.index(".subprog_")] + varname = varname[: varname.index(".subprog_")] if ".cast_fp" in varname: - varname = varname[:varname.index(".cast_fp")] + varname = varname[: varname.index(".cast_fp")] if ".quantized" in varname: - varname = varname[:varname.index(".quantized")] + varname = varname[: varname.index(".quantized")] assert block.has_var(varname) var = block.var(varname) return var.is_parameter @@ -277,8 +297,9 @@ def infer_shape(block, src_var, src_var_dist_attr, op_input_dist_attr): return exact_shape -def set_comm_op_dist_attr_for_program(new_op, process_mesh, tensor_dist_attr, - ctx): +def set_comm_op_dist_attr_for_program( + new_op, process_mesh, tensor_dist_attr, ctx +): assert process_mesh is not None assert tensor_dist_attr is not None @@ -303,9 +324,11 @@ def naive_copy_op_dist_attr_for_program(new_op, ref_op, ctx): assert len(new_op.input(input_name)) == 1 ref_tensor_dist_attr = ref_dist_attr.get_input_dist_attr( - ref_op.input(input_name)[0]) + ref_op.input(input_name)[0] + ) new_op_dist_attr.set_input_dist_attr( - new_op.input(input_name)[0], ref_tensor_dist_attr) + new_op.input(input_name)[0], ref_tensor_dist_attr + ) for output_name in ref_op.output_names: assert output_name in new_op.output_names @@ -313,9 +336,11 @@ def naive_copy_op_dist_attr_for_program(new_op, ref_op, ctx): assert len(new_op.output(output_name)) == 1 ref_tensor_dist_attr = ref_dist_attr.get_output_dist_attr( - ref_op.output(output_name)[0]) + ref_op.output(output_name)[0] + ) new_op_dist_attr.set_output_dist_attr( - new_op.output(output_name)[0], ref_tensor_dist_attr) + new_op.output(output_name)[0], ref_tensor_dist_attr + ) ctx.set_op_dist_attr_for_program(new_op, new_op_dist_attr) @@ -348,9 +373,12 @@ def get_data_parallel_group(dist_ctx, op, act_grad_names, rank): batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1 if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, - batch_size_axis, rank) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + batch_size_axis, + rank, + ) dp_group = new_process_group(group_ranks) break @@ -377,33 +405,39 @@ def sync_and_scale_gradients(dist_ctx, op, dp_group, allreduce_var_names): for var_name in allreduce_var_names: added_ops = [] grad_var = main_block.var(var_name) - allreduce_op = main_block.append_op(type='c_allreduce_sum', - inputs={'X': [grad_var]}, - outputs={'Out': [grad_var]}, - attrs={ - 'ring_id': dp_group.id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Backward - }) - allreduce_op._set_attr('op_namescope', - str('/') + ParallelMode.DataParallel) + allreduce_op = main_block.append_op( + type='c_allreduce_sum', + inputs={'X': [grad_var]}, + outputs={'Out': [grad_var]}, + attrs={ + 'ring_id': dp_group.id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Backward, + }, + ) + allreduce_op._set_attr( + 'op_namescope', str('/') + ParallelMode.DataParallel + ) added_ops.append(allreduce_op) if dist_ctx.gradient_scale: - scale_op = main_block.append_op(type='scale', - inputs={'X': grad_var}, - outputs={'Out': grad_var}, - attrs={ - 'scale': 1.0 / dp_degree, - OP_ROLE_KEY: OpRole.Backward - }) - scale_op._set_attr('op_namescope', - str('/') + ParallelMode.DataParallel) + scale_op = main_block.append_op( + type='scale', + inputs={'X': grad_var}, + outputs={'Out': grad_var}, + attrs={'scale': 1.0 / dp_degree, OP_ROLE_KEY: OpRole.Backward}, + ) + scale_op._set_attr( + 'op_namescope', str('/') + ParallelMode.DataParallel + ) added_ops.append(scale_op) dims_mapping = op_dist_attr.get_output_dims_mapping(grad_var.name) - assert dims_mapping is not None, "Unexception: dims_mapping of output [{}] of op [{}] is None".format( - grad_var.name, op_dist_attr.op_type) + assert ( + dims_mapping is not None + ), "Unexception: dims_mapping of output [{}] of op [{}] is None".format( + grad_var.name, op_dist_attr.op_type + ) # NOTE auxiliary op's dist attr should follow dist_op not dist_tensor for new_op in added_ops: new_op_attr = OperatorDistributedAttribute() @@ -413,8 +447,9 @@ def sync_and_scale_gradients(dist_ctx, op, dp_group, allreduce_var_names): dist_ctx.set_op_dist_attr_for_program(new_op, new_op_attr) -def gradient_synchronization(dist_ctx, op, act_grad_names, out_grad_names, - rank): +def gradient_synchronization( + dist_ctx, op, act_grad_names, out_grad_names, rank +): """ conduct the allreudce and scaling（dp size）for gradients of model parameters for operator in data parallelism. @@ -430,8 +465,11 @@ def gradient_synchronization(dist_ctx, op, act_grad_names, out_grad_names, if not is_in_backward_phase(dist_ctx): return - if is_optimize_op(op) or len(act_grad_names) == 0 or len( - out_grad_names) == 0: + if ( + is_optimize_op(op) + or len(act_grad_names) == 0 + or len(out_grad_names) == 0 + ): return dp_group = get_data_parallel_group(dist_ctx, op, act_grad_names, rank) @@ -443,13 +481,19 @@ def gradient_synchronization(dist_ctx, op, act_grad_names, out_grad_names, def is_data_parallel_scale_op(op): - return op.type == "scale" and op.desc.has_attr("op_namescope") \ - and ParallelMode.DataParallel in op.desc.attr("op_namescope") + return ( + op.type == "scale" + and op.desc.has_attr("op_namescope") + and ParallelMode.DataParallel in op.desc.attr("op_namescope") + ) def is_data_parallel_reduce_op(op): - return op.type in ["c_reduce_sum", "c_allreduce_sum"] and op.desc.has_attr("op_namescope") \ - and ParallelMode.DataParallel in op.desc.attr("op_namescope") + return ( + op.type in ["c_reduce_sum", "c_allreduce_sum"] + and op.desc.has_attr("op_namescope") + and ParallelMode.DataParallel in op.desc.attr("op_namescope") + ) def is_in_backward_phase(dist_ctx): diff --git a/python/paddle/distributed/auto_parallel/operators/dist_assign.py b/python/paddle/distributed/auto_parallel/operators/dist_assign.py index 96923f461a7..15382977aab 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_assign.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_assign.py @@ -21,7 +21,6 @@ from ..utils import compute_compatible_and_update_dim_mapping class DistributedAssign(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedAssign, self).__init__(op_type) @@ -30,7 +29,6 @@ register_distributed_operator_impl_container(DistributedAssign("assign")) class DistributedAssignImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedAssignImpl, self).__init__(name) self._forward_implemented = True @@ -43,8 +41,9 @@ class DistributedAssignImpl(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -70,7 +69,8 @@ class DistributedAssignImpl(DistributedOperatorImpl): for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True diff --git a/python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py b/python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py index 72a4eda103d..c99cfe21f72 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py @@ -22,23 +22,24 @@ from ..utils import set_var_dist_attr from ..utils import set_dist_op_desc_original_id from ..process_group import new_process_group from ..dist_attribute import OperatorDistributedAttribute -from paddle.distributed.auto_parallel.process_group import get_world_process_group +from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, +) world_process_group = get_world_process_group() class DistributedCheckFiniteAndUnscale(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedCheckFiniteAndUnscale, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedCheckFiniteAndUnscale("check_finite_and_unscale")) + DistributedCheckFiniteAndUnscale("check_finite_and_unscale") +) class DistributedCheckFiniteAndUnscaleImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedCheckFiniteAndUnscaleImpl, self).__init__(name) self._name = name @@ -80,8 +81,11 @@ class DistributedCheckFiniteAndUnscaleImpl(DistributedOperatorImpl): backward_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format( + str(backward_op) + ) assert rank_id in dist_attr.process_mesh.processes @@ -89,25 +93,33 @@ class DistributedCheckFiniteAndUnscaleImpl(DistributedOperatorImpl): assert 'Scale' in kwargs, "input [{}] is not given".format('Scale') assert 'Out' in kwargs, "input [{}] is not given".format('Out') assert 'FoundInfinite' in kwargs, "output [{}] is not given".format( - 'FoundInfinite') + 'FoundInfinite' + ) - assert len( + assert ( + len(kwargs['Scale']) == 1 + ), "check_finite_and_unscale input Scale take 1 variable but got {}".format( kwargs['Scale'] - ) == 1, "check_finite_and_unscale input Scale take 1 variable but got {}".format( - kwargs['Scale']) - assert len( + ) + assert ( + len(kwargs['FoundInfinite']) == 1 + ), "check_finite_and_unscale input FoundInfinite take 1 variable but got {}".format( kwargs['FoundInfinite'] - ) == 1, "check_finite_and_unscale input FoundInfinite take 1 variable but got {}".format( - kwargs['FoundInfinite']) + ) assert len(kwargs['X']) == len( kwargs['Out'] ), "check_finite_and_unscale got [{}] X and [{}] Out, which are supposed to be equal".format( - len(kwargs['X']), len(kwargs['Out'])) + len(kwargs['X']), len(kwargs['Out']) + ) filter_vars = [] for varname in kwargs['X']: - if rank_id in ctx.get_tensor_dist_attr_for_program( - main_block.var(varname)).process_mesh.processes: + if ( + rank_id + in ctx.get_tensor_dist_attr_for_program( + main_block.var(varname) + ).process_mesh.processes + ): filter_vars.append(varname) # replicate op in dist program @@ -121,55 +133,70 @@ class DistributedCheckFiniteAndUnscaleImpl(DistributedOperatorImpl): group = new_process_group(world_process_group.ranks) inf_var = main_block.var(kwargs['FoundInfinite'][0]) - inf_var_int32 = main_block.create_var(name=inf_var.name + "@cast_int32", - shape=inf_var.shape, - dtype=core.VarDesc.VarType.INT32) + inf_var_int32 = main_block.create_var( + name=inf_var.name + "@cast_int32", + shape=inf_var.shape, + dtype=core.VarDesc.VarType.INT32, + ) set_var_dist_attr( - ctx, inf_var_int32, + ctx, + inf_var_int32, ctx.get_tensor_dist_attr_for_program(inf_var).dims_mapping, - ctx.get_tensor_dist_attr_for_program(inf_var).process_mesh) - cast_op1 = main_block.append_op(type='cast', - inputs={'X': inf_var}, - outputs={'Out': inf_var_int32}, - attrs={ - "in_dtype": inf_var.dtype, - "out_dtype": inf_var_int32.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) - allreduce_op = main_block.append_op(type='c_allreduce_max', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var_int32}, - attrs={ - 'ring_id': group.id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) - cast_op2 = main_block.append_op(type='cast', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var}, - attrs={ - "in_dtype": inf_var_int32.dtype, - "out_dtype": inf_var.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + ctx.get_tensor_dist_attr_for_program(inf_var).process_mesh, + ) + cast_op1 = main_block.append_op( + type='cast', + inputs={'X': inf_var}, + outputs={'Out': inf_var_int32}, + attrs={ + "in_dtype": inf_var.dtype, + "out_dtype": inf_var_int32.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + allreduce_op = main_block.append_op( + type='c_allreduce_max', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var_int32}, + attrs={ + 'ring_id': group.id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + cast_op2 = main_block.append_op( + type='cast', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var}, + attrs={ + "in_dtype": inf_var_int32.dtype, + "out_dtype": inf_var.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for op in [cast_op1, allreduce_op, cast_op2]: new_op_dist_attr = OperatorDistributedAttribute() for varname in op.input_arg_names: var_dist_attr = ctx.get_tensor_dist_attr_for_program( - main_block.var(varname)) + main_block.var(varname) + ) assert var_dist_attr is not None new_op_dist_attr.set_input_dims_mapping( - varname, var_dist_attr.dims_mapping) + varname, var_dist_attr.dims_mapping + ) for varname in op.output_arg_names: var_dist_attr = ctx.get_tensor_dist_attr_for_program( - main_block.var(varname)) + main_block.var(varname) + ) new_op_dist_attr.set_output_dims_mapping( - varname, var_dist_attr.dims_mapping) + varname, var_dist_attr.dims_mapping + ) new_op_dist_attr.process_mesh = var_dist_attr.process_mesh ctx.set_op_dist_attr_for_program(op, new_op_dist_attr) register_distributed_operator_impl( "check_finite_and_unscale", - DistributedCheckFiniteAndUnscaleImpl("check_finite_and_unscale")) + DistributedCheckFiniteAndUnscaleImpl("check_finite_and_unscale"), +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_default.py b/python/paddle/distributed/auto_parallel/operators/dist_default.py index 9c1e8b04871..36b9b8b2ee4 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_default.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_default.py @@ -38,35 +38,41 @@ def prim_operator_data_parallel_functor(ctx, src_op): var_name = src_op.output_arg_names[0] if var_name in ctx.grads_params: - assert var_name not in ctx.synced_gradient, "in primtive mode, grad is already {} synced".format( - var_name) + assert ( + var_name not in ctx.synced_gradient + ), "in primtive mode, grad is already {} synced".format(var_name) ctx.synced_gradient.add(var_name) sync_group = new_process_group(ctx.data_parallel_group) - allreduce_op = main_block.append_op(type='c_allreduce_sum', - inputs={'X': [var_name]}, - outputs={'Out': [var_name]}, - attrs={ - 'ring_id': sync_group.id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Backward - }) + allreduce_op = main_block.append_op( + type='c_allreduce_sum', + inputs={'X': [var_name]}, + outputs={'Out': [var_name]}, + attrs={ + 'ring_id': sync_group.id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Backward, + }, + ) param = ctx.grads_params[var_name] startup_block = dist_op_context.startup_block - new_op = startup_block.append_op(type='c_broadcast', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'ring_id': sync_group.id, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + new_op = startup_block.append_op( + type='c_broadcast', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'ring_id': sync_group.id, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) grad_var = main_block.var(var_name) dims_mapping = ctx.get_tensor_dist_attr_for_program( - grad_var).dims_mapping + grad_var + ).dims_mapping dist_attr = ctx.get_op_dist_attr_for_program(src_op) process_mesh = dist_attr.process_mesh op_attr = OperatorDistributedAttribute() @@ -79,7 +85,6 @@ def prim_operator_data_parallel_functor(ctx, src_op): class DistributedDefault(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedDefault, self).__init__(op_type) @@ -89,7 +94,6 @@ register_distributed_operator_impl_container(DistributedDefault("default")) # Replicated Default class DistributedDefaultImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedDefaultImpl0, self).__init__(name) self._forward_implemented = True @@ -107,13 +111,14 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(_g_op_cost_factory[op_type], - ctx, processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + _g_op_cost_factory[op_type], ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -121,16 +126,17 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes backward_op = dist_op.serial_op op_type = backward_op.type - cost_mapping = build_comp_costs_from_descs(_g_op_cost_factory[op_type], - ctx, processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + _g_op_cost_factory[op_type], ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) main_block = backward_op.block @@ -139,7 +145,8 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): var_dim_mapping = dist_attr.get_input_dims_mapping(varname) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] @@ -151,16 +158,25 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): var_dim_mapping = dist_attr.get_input_dims_mapping( - varname) + varname + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res def is_input_compatible(self, dist_op): @@ -304,8 +320,10 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): batch_dim_mappings.append(dims_mapping[1]) # Check batch dim mapping compatibility - if not all(batch_dim_mappings[0] == dim_mapping - for dim_mapping in batch_dim_mappings): + if not all( + batch_dim_mappings[0] == dim_mapping + for dim_mapping in batch_dim_mappings + ): return False return True @@ -342,7 +360,8 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): for arg_name in op_desc.output_arg_names(): if op_desc.type() == 'fill_any_like': input_tensor = dist_op.get_serial_input( - op_desc.input_arg_names()[0]) + op_desc.input_arg_names()[0] + ) if input_tensor.is_parameter: continue serial_tensor = dist_op.get_serial_output(arg_name) @@ -359,7 +378,8 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): return changed compatible_dim_mapping = compute_compatible_dim_mapping( - batch_dim_mappings) + batch_dim_mappings + ) if compatible_dim_mapping is None: return False @@ -369,19 +389,24 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): continue dims_mapping = op_dist_attr.get_input_dims_mapping(arg_name) if arg_name not in input_xshape_arg_names: - if len(dims_mapping) >= 1 and \ - compatible_dim_mapping != dims_mapping[0]: + if ( + len(dims_mapping) >= 1 + and compatible_dim_mapping != dims_mapping[0] + ): dims_mapping[0] = compatible_dim_mapping changed = True else: - if len(dims_mapping) >= 2 and \ - compatible_dim_mapping != dims_mapping[1]: + if ( + len(dims_mapping) >= 2 + and compatible_dim_mapping != dims_mapping[1] + ): dims_mapping[1] = compatible_dim_mapping changed = True for arg_name in op_desc.output_arg_names(): if op_desc.type() == 'fill_any_like': input_tensor = dist_op.get_serial_input( - op_desc.input_arg_names()[0]) + op_desc.input_arg_names()[0] + ) if input_tensor.is_parameter: continue if op_desc.type() in ["shape", "slice"]: @@ -391,13 +416,17 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): continue dims_mapping = op_dist_attr.get_output_dims_mapping(arg_name) if arg_name not in output_xshape_arg_names: - if len(dims_mapping - ) >= 1 and compatible_dim_mapping != dims_mapping[0]: + if ( + len(dims_mapping) >= 1 + and compatible_dim_mapping != dims_mapping[0] + ): dims_mapping[0] = compatible_dim_mapping changed = True else: - if len(dims_mapping - ) >= 2 and compatible_dim_mapping != dims_mapping[1]: + if ( + len(dims_mapping) >= 2 + and compatible_dim_mapping != dims_mapping[1] + ): dims_mapping[1] = compatible_dim_mapping changed = True @@ -414,17 +443,20 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) # replicate op in dist program dist_op_desc = main_block.append_op(type='nop').desc @@ -437,6 +469,7 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): # data parallel synchronization for primtive operators from paddle.incubate.autograd import prim_enabled + if prim_enabled(): assert is_prim_op(src_op) prim_operator_data_parallel_functor(ctx, src_op) @@ -447,9 +480,11 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): return for varname in dist_op_desc.input_arg_names(): - if startup_block.has_var(varname) and startup_block.var( - varname - ).is_parameter and varname not in dist_op_context.already_init_sync_vars: + if ( + startup_block.has_var(varname) + and startup_block.var(varname).is_parameter + and varname not in dist_op_context.already_init_sync_vars + ): dist_op_context.already_init_sync_vars.add(varname) param = startup_block.var(varname) param_dist_attr = ctx.get_tensor_dist_attr_for_program(param) @@ -458,38 +493,41 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, process_mesh, rank_id + ) # NOTE all not splited axis should be presented in mesh for axis, size in enumerate(process_mesh.topology): if size <= 1 or axis in dims_mapping: pass else: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, - axis, rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + axis, + rank_id, + ) sync_group = new_process_group(group_ranks) - new_op = startup_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': - sync_group.id, - 'root': - 0, - 'use_calc_stream': - True, - OP_ROLE_KEY: - OpRole.Forward - }) + new_op = startup_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': sync_group.id, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) # set distributed attribute op_attr = OperatorDistributedAttribute() op_attr.process_mesh = process_mesh - op_attr.set_output_dims_mapping(param.name, - dims_mapping) + op_attr.set_output_dims_mapping( + param.name, dims_mapping + ) op_attr.set_input_dims_mapping(param.name, dims_mapping) ctx.set_op_dist_attr_for_program(new_op, op_attr) @@ -501,24 +539,30 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): main_block = dist_op_context.work_block backward_op = dist_op_context.cur_src_op dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format( + str(backward_op) + ) rank_id = dist_op_context.rank_id # check validation of inputs / outputs for input_name in backward_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( backward_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in backward_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( backward_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) # replicate op in dist program dist_op_desc = main_block.append_op(type='nop').desc @@ -535,7 +579,8 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): act_grad_names.append(varname) out_grad_names = [] @@ -548,9 +593,11 @@ class DistributedDefaultImpl0(DistributedOperatorImpl): if is_parameter_related(fwd_name, main_block): out_grad_names.append(varname) - gradient_synchronization(ctx, backward_op, act_grad_names, - out_grad_names, rank_id) + gradient_synchronization( + ctx, backward_op, act_grad_names, out_grad_names, rank_id + ) register_distributed_operator_impl( - "default", DistributedDefaultImpl0("replicate_parallel")) + "default", DistributedDefaultImpl0("replicate_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py b/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py index 1dc163c0af4..c9f5eb80515 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py @@ -27,18 +27,17 @@ from ..cost import build_comp_costs_from_descs class DistributedElementwise(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedElementwise, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedElementwise("elementwise")) + DistributedElementwise("elementwise") +) # Replicated Elementwise class DistributedElementwiseImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedElementwiseImpl0, self).__init__(name) self._forward_implemented = False @@ -56,13 +55,14 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(_g_op_cost_factory[op_type], - ctx, processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + _g_op_cost_factory[op_type], ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -70,16 +70,17 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes backward_op = dist_op.serial_op op_type = backward_op.type - cost_mapping = build_comp_costs_from_descs(_g_op_cost_factory[op_type], - ctx, processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + _g_op_cost_factory[op_type], ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) main_block = backward_op.block @@ -88,7 +89,8 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): var_dim_mapping = dist_attr.get_input_dims_mapping(varname) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] @@ -100,16 +102,25 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): var_dim_mapping = dist_attr.get_input_dims_mapping( - varname) + varname + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res def is_input_compatible(self, dist_op): @@ -189,8 +200,9 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): for dims_mapping in dims_mapping_list: if idx < len(dims_mapping): dim_mappings.append(dims_mapping[-(idx + 1)]) - if not all(dim_mappings[0] == dim_mapping - for dim_mapping in dim_mappings): + if not all( + dim_mappings[0] == dim_mapping for dim_mapping in dim_mappings + ): return False return True @@ -216,10 +228,13 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): -1 for _ in range(input_max_dims_mapping_len) ] for i in range(input_dims_mapping_lens[arg_name]): - new_idx = (input_max_dims_mapping_len - - input_dims_mapping_lens[arg_name]) + i + new_idx = ( + input_max_dims_mapping_len + - input_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[new_idx] = input_dims_mapping_dict[ - arg_name][i] + arg_name + ][i] dims_mapping_list.append(new_dims_mapping) else: dims_mapping_list.append(input_dims_mapping_dict[arg_name]) @@ -240,10 +255,13 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): -1 for _ in range(output_max_dims_mapping_len) ] for i in range(output_dims_mapping_lens[arg_name]): - new_idx = (output_max_dims_mapping_len - - output_dims_mapping_lens[arg_name]) + i + new_idx = ( + output_max_dims_mapping_len + - output_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[new_idx] = output_dims_mapping_dict[ - arg_name][i] + arg_name + ][i] dims_mapping_list.append(new_dims_mapping) else: dims_mapping_list.append(output_dims_mapping_dict[arg_name]) @@ -251,7 +269,8 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): assert input_max_dims_mapping_len == output_max_dims_mapping_len max_dims_mapping_len = input_max_dims_mapping_len compatible_dims_mapping = compute_compatible_dims_mapping( - dims_mapping_list) + dims_mapping_list + ) if compatible_dims_mapping is None: return False @@ -261,17 +280,20 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): -1 for _ in range(input_dims_mapping_lens[arg_name]) ] for i in range(input_dims_mapping_lens[arg_name]): - new_idx = (max_dims_mapping_len - - input_dims_mapping_lens[arg_name]) + i + new_idx = ( + max_dims_mapping_len - input_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[i] = compatible_dims_mapping[new_idx] if new_dims_mapping != input_dims_mapping_dict[arg_name]: - op_dist_attr.set_input_dims_mapping(arg_name, - new_dims_mapping) + op_dist_attr.set_input_dims_mapping( + arg_name, new_dims_mapping + ) changed = True else: if compatible_dims_mapping != input_dims_mapping_dict[arg_name]: op_dist_attr.set_input_dims_mapping( - arg_name, compatible_dims_mapping) + arg_name, compatible_dims_mapping + ) changed = True for arg_name in output_arg_names: @@ -280,17 +302,24 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): -1 for _ in range(output_dims_mapping_lens[arg_name]) ] for i in range(output_dims_mapping_lens[arg_name]): - new_idx = (max_dims_mapping_len - - output_dims_mapping_lens[arg_name]) + i + new_idx = ( + max_dims_mapping_len + - output_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[i] = compatible_dims_mapping[new_idx] if new_dims_mapping != output_dims_mapping_dict[arg_name]: op_dist_attr.set_output_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) changed = True else: - if compatible_dims_mapping != output_dims_mapping_dict[arg_name]: + if ( + compatible_dims_mapping + != output_dims_mapping_dict[arg_name] + ): op_dist_attr.set_output_dims_mapping( - arg_name, compatible_dims_mapping) + arg_name, compatible_dims_mapping + ) changed = True return changed @@ -305,4 +334,5 @@ class DistributedElementwiseImpl0(DistributedOperatorImpl): register_distributed_operator_impl( - "elementwise", DistributedElementwiseImpl0("replicate_parallel")) + "elementwise", DistributedElementwiseImpl0("replicate_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_embedding.py b/python/paddle/distributed/auto_parallel/operators/dist_embedding.py index 513dffb5fca..fa76d9aff32 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_embedding.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_embedding.py @@ -17,7 +17,11 @@ from .common import DistributedOperatorImplContainer from .common import DistributedOperatorImpl from .common import register_distributed_operator_impl_container from .common import gradient_synchronization -from .common import naive_copy_op_dist_attr_for_program, register_distributed_operator_impl, set_comm_op_dist_attr_for_program +from .common import ( + naive_copy_op_dist_attr_for_program, + register_distributed_operator_impl, + set_comm_op_dist_attr_for_program, +) from ..utils import is_dim_shard from ..utils import is_dim_replicate from ..utils import compute_compatible_and_update_dim_mapping @@ -26,33 +30,48 @@ from paddle.fluid import core, unique_name from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole from ..process_group import new_process_group -from ..utils import _get_comm_group, _get_idx_in_axis, _get_corresponding_rank, set_var_dist_attr +from ..utils import ( + _get_comm_group, + _get_idx_in_axis, + _get_corresponding_rank, + set_var_dist_attr, +) from ..cost import build_comp_desc_from_dist_op, build_comm_desc_from_dist_op -from ..cost import build_comm_costs_from_descs, build_comp_costs_from_descs, build_dp_costs +from ..cost import ( + build_comm_costs_from_descs, + build_comp_costs_from_descs, + build_dp_costs, +) from ..cost import EmbeddingOpCost, EmbeddingGradOpCost -from paddle.distributed.auto_parallel.cost.comm_op_cost import AllreduceSumOpCost, IdentityOpCost +from paddle.distributed.auto_parallel.cost.comm_op_cost import ( + AllreduceSumOpCost, + IdentityOpCost, +) class DistributedEmbedding(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedEmbedding, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedEmbedding("lookup_table_v2")) + DistributedEmbedding("lookup_table_v2") +) register_distributed_operator_impl_container( - DistributedEmbedding("c_embedding")) + DistributedEmbedding("c_embedding") +) register_distributed_operator_impl_container( - DistributedEmbedding("lookup_table")) + DistributedEmbedding("lookup_table") +) def adopt_lookup_table_v1(ctx, main_block, src_op, Ids_var): - assert len( - Ids_var.shape - ) == 3, "input Ids to lookup_table should have 3 dimensions but got [{}] with shape [{}]".format( - Ids_var.name, Ids_var.shape) + assert ( + len(Ids_var.shape) == 3 + ), "input Ids to lookup_table should have 3 dimensions but got [{}] with shape [{}]".format( + Ids_var.name, Ids_var.shape + ) if not Ids_var.stop_gradient: raise NotImplementedError( 'Requiring the gradient of Ids of lookup_table(v1）dist op is not currently supported. Please open an issue with details on your use case so that we can prioritize adding this (for instance, adversarial training for language model).' @@ -60,59 +79,72 @@ def adopt_lookup_table_v1(ctx, main_block, src_op, Ids_var): target_shape = list(Ids_var.shape[:-1]) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["dist_reshape", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["dist_reshape", 'tmp']) + ), dtype=Ids_var.dtype, shape=target_shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) target_shape = [0] + list(Ids_var.shape[:-1]) xshape_var = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["dist_Xshape", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["dist_Xshape", 'tmp']) + ), dtype=Ids_var.dtype, shape=target_shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) # TODO use inplace reshape for memory saving - reshape_op = main_block.append_op(type='reshape2', - inputs={'X': [Ids_var]}, - outputs={ - 'Out': [intermediate_var_0], - 'XShape': [xshape_var] - }, - attrs={ - "shape": [0, -1], - }) + reshape_op = main_block.append_op( + type='reshape2', + inputs={'X': [Ids_var]}, + outputs={'Out': [intermediate_var_0], 'XShape': [xshape_var]}, + attrs={ + "shape": [0, -1], + }, + ) # set dist attr op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) Ids_var_dist_attr = op_dist_attr.get_input_dist_attr(Ids_var.name) assert Ids_var_dist_attr is not None intermediate_var_0_dist_attr = set_var_dist_attr( - ctx, intermediate_var_0, Ids_var_dist_attr.dims_mapping, - Ids_var_dist_attr.process_mesh) - set_var_dist_attr(ctx, xshape_var, - [-1] + list(Ids_var_dist_attr.dims_mapping), - Ids_var_dist_attr.process_mesh) + ctx, + intermediate_var_0, + Ids_var_dist_attr.dims_mapping, + Ids_var_dist_attr.process_mesh, + ) + set_var_dist_attr( + ctx, + xshape_var, + [-1] + list(Ids_var_dist_attr.dims_mapping), + Ids_var_dist_attr.process_mesh, + ) op_dist_attr.del_input_dist_attr(Ids_var.name) - op_dist_attr.set_input_dist_attr(intermediate_var_0.name, - intermediate_var_0_dist_attr) + op_dist_attr.set_input_dist_attr( + intermediate_var_0.name, intermediate_var_0_dist_attr + ) new_op_dist_attr = OperatorDistributedAttribute() new_op_dist_attr.process_mesh = Ids_var_dist_attr.process_mesh new_op_dist_attr.impl_type = "default" new_op_dist_attr.impl_idx = 0 - new_op_dist_attr.set_input_dims_mapping(Ids_var.name, - Ids_var_dist_attr.dims_mapping) - new_op_dist_attr.set_output_dims_mapping(intermediate_var_0.name, - Ids_var_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + Ids_var.name, Ids_var_dist_attr.dims_mapping + ) + new_op_dist_attr.set_output_dims_mapping( + intermediate_var_0.name, Ids_var_dist_attr.dims_mapping + ) new_op_dist_attr.set_output_dims_mapping( - xshape_var.name, [-1] + list(Ids_var_dist_attr.dims_mapping)) + xshape_var.name, [-1] + list(Ids_var_dist_attr.dims_mapping) + ) ctx.set_op_dist_attr_for_program(reshape_op, new_op_dist_attr) return intermediate_var_0 @@ -120,7 +152,6 @@ def adopt_lookup_table_v1(ctx, main_block, src_op, Ids_var): # RowParallel class DistributedEmbeddingImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedEmbeddingImpl, self).__init__(name) self._forward_implemented = True @@ -138,17 +169,19 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes # embedding need start_index - cost_mapping = build_comp_costs_from_descs(EmbeddingOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + EmbeddingOpCost, ctx, processes, desc_mapping, cluster + ) serial_op = dist_op.serial_op parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("W")[0])[0] + serial_op.input("W")[0] + )[0] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.output("Out") c_allreduce_sum_desc_mapping = build_comm_desc_from_dist_op( @@ -157,11 +190,16 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, c_allreduce_sum_desc_mapping, - cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res_cost = [cost_mapping, comm_op_cost_list] @@ -175,7 +213,8 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): dist_attr = dist_op.dist_attr embedding_row_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("W")[0])[0] + backward_op.input("W")[0] + )[0] parallel_axis = embedding_row_dim_mapping attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = [backward_op.input("Out@GRAD")[0]] @@ -185,33 +224,38 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res.append(comm_op_cost_list) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) - cost_mapping = build_comp_costs_from_descs(EmbeddingGradOpCost, ctx, - processes, desc_mapping, - cluster) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) + cost_mapping = build_comp_costs_from_descs( + EmbeddingGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Ids")[0]) + backward_op.input("Ids")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1: parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('W@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res @@ -223,7 +267,8 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): ids_dims_mapping = op_dist_attr.get_input_dims_mapping(ids_name) w_dims_mapping = op_dist_attr.get_input_dims_mapping(w_name) if is_dim_replicate(w_dims_mapping[-2]) or is_dim_shard( - w_dims_mapping[-1]): + w_dims_mapping[-1] + ): return False # Other dimensions must be replicate except the batch dimension for mapping in ids_dims_mapping[1:]: @@ -243,8 +288,9 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -256,7 +302,7 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): ids_dims_mapping = op_dist_attr.get_input_dims_mapping(ids_name) w_dims_mapping = op_dist_attr.get_input_dims_mapping(w_name) - if ids_dims_mapping != out_dims_mapping[:len(ids_dims_mapping)]: + if ids_dims_mapping != out_dims_mapping[: len(ids_dims_mapping)]: return False return True @@ -274,12 +320,14 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): for i in range(len(ids_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [ids_dims_mapping, out_dims_mapping], [i, i]) + [ids_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True dim_changed = compute_compatible_and_update_dim_mapping( - [w_dims_mapping, out_dims_mapping], [-1, -1]) + [w_dims_mapping, out_dims_mapping], [-1, -1] + ) if dim_changed: changed = True @@ -297,26 +345,30 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # check validation of inputs / outputs assert 'Ids' in kwargs, "input [{}] is not given".format('Ids') assert 'W' in kwargs, "input [{}] is not given".format('W') assert 'Out' in kwargs, "output [{}] is not given".format('Out') - assert len( + assert ( + len(kwargs['Ids']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['Ids'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Ids']) - assert len( + ) + assert ( + len(kwargs['W']) == 1 + ), "row_parallel_embedding input W take 1 variable but got {}".format( kwargs['W'] - ) == 1, "row_parallel_embedding input W take 1 variable but got {}".format( - kwargs['W']) - assert len( + ) + assert ( + len(kwargs['Out']) == 1 + ), "row_parallel_embedding output Out take 1 variable but got {}".format( kwargs['Out'] - ) == 1, "row_parallel_embedding output Out take 1 variable but got {}".format( - kwargs['Out']) + ) Ids_var = main_block.var(kwargs['Ids'][0]) Weight_var = main_block._var_recursive(kwargs['W'][0]) @@ -328,70 +380,85 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): # got dist attribute info embedding_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[0] - assert embedding_row_dim_mapping >= 0, "row_parallel_embedding's row should be divided by a specific mesh axis, but got [{}]".format( - embedding_row_dim_mapping) + Weight_var.name + )[0] + assert ( + embedding_row_dim_mapping >= 0 + ), "row_parallel_embedding's row should be divided by a specific mesh axis, but got [{}]".format( + embedding_row_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in process_mesh_group: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # A generalized method to caculate embedding offset using cartisian product - relative_idx = _get_idx_in_axis(process_mesh_group, process_mesh_shape, - embedding_row_dim_mapping, rank_id) + relative_idx = _get_idx_in_axis( + process_mesh_group, + process_mesh_shape, + embedding_row_dim_mapping, + rank_id, + ) per_part_size = Weight_var.shape[0] relative_idx = relative_idx * per_part_size # TODO caculate ring id parallel_axis = embedding_row_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # append op - check_variable_and_dtype(Ids_var, 'input', ['int32', 'int64'], - 'c_embedding') + check_variable_and_dtype( + Ids_var, 'input', ['int32', 'int64'], 'c_embedding' + ) # infer new var shape with op dist attr out_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(Out_var) assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_embedding", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_embedding", 'tmp']) + ), dtype=Weight_var.dtype, shape=Out_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=Out_var.stop_gradient) + stop_gradient=Out_var.stop_gradient, + ) # set intermediate_var_0's dist_attr with Out_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_var_dist_attr + ) check_variable_and_dtype( - Out_var, 'tensor', + Out_var, + 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'c_allreduce_sum') + 'c_allreduce_sum', + ) c_embedding_op = main_block.append_op( type='c_embedding', - inputs={ - 'Ids': [Ids_var], - 'W': [Weight_var] - }, + inputs={'Ids': [Ids_var], 'W': [Weight_var]}, outputs={'Out': [intermediate_var_0]}, attrs={ "start_index": relative_idx, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if intermediate_var_0.shape != ref_shape: intermediate_var_0.desc.set_shape(ref_shape) @@ -404,8 +471,9 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if Out_var.shape != ref_shape: Out_var.desc.set_shape(ref_shape) @@ -418,15 +486,19 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): for input_varname in c_embedding_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - embedding_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + embedding_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) output_varname = c_embedding_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - embedding_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + embedding_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(c_embedding_op, embedding_op_dist_attr) # allreduce @@ -438,16 +510,20 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(input_var) assert tensor_dist_attr is not None - allreduce_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + allreduce_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) for output_varname in c_allreduce_sum_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - allreduce_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) - ctx.set_op_dist_attr_for_program(c_allreduce_sum_op, - allreduce_op_dist_attr) + op_dist_attr + ) + allreduce_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) + ctx.set_op_dist_attr_for_program( + c_allreduce_sum_op, allreduce_op_dist_attr + ) # param initialization sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: @@ -464,20 +540,25 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): if size <= 1 or axis in dim_mapping: pass else: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, axis, - rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + axis, + rank_id, + ) sync_group = new_process_group(group_ranks) - startup_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': sync_group.id, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + startup_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': sync_group.id, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -488,35 +569,43 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): backward_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format( + str(backward_op) + ) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, dist_attr.process_mesh, rank_id + ) assert 'Ids' in kwargs, "input [{}] is not given".format('Ids') assert 'W' in kwargs, "input [{}] is not given".format('W') assert 'Out@GRAD' in kwargs, "input [{}] is not given".format('Out') assert 'W@GRAD' in kwargs, "output [{}] is not given".format('W@GRAD') - assert len( + assert ( + len(kwargs['Ids']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['Ids'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Ids']) - assert len( + ) + assert ( + len(kwargs['W']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['W'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['W']) - assert len( - kwargs['Out@GRAD'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Out']) - assert len( + ) + assert ( + len(kwargs['Out@GRAD']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( + kwargs['Out'] + ) + assert ( + len(kwargs['W@GRAD']) == 1 + ), "row_parallel_embedding output Ids take 1 variable but got {}".format( kwargs['W@GRAD'] - ) == 1, "row_parallel_embedding output Ids take 1 variable but got {}".format( - kwargs['W@GRAD']) + ) Ids_var = main_block.var(kwargs['Ids'][0]) Weight_var = main_block.var(kwargs['W'][0]) @@ -524,39 +613,57 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): Weight_grad = main_block.var(kwargs['W@GRAD'][0]) embedding_row_dim_mapping = dist_attr.get_input_dims_mapping( - Weight_var.name)[0] - assert embedding_row_dim_mapping >= 0, "row_parallel_embedding's row should be divided by a specific mesh axis, but got [{}]".format( - embedding_row_dim_mapping) + Weight_var.name + )[0] + assert ( + embedding_row_dim_mapping >= 0 + ), "row_parallel_embedding's row should be divided by a specific mesh axis, but got [{}]".format( + embedding_row_dim_mapping + ) process_mesh_shape = dist_attr.process_mesh.topology process_mesh_group = dist_attr.process_mesh.processes # A generalized method to caculate embedding offset using cartisian product - relative_idx = _get_idx_in_axis(process_mesh_group, process_mesh_shape, - embedding_row_dim_mapping, rank_id) + relative_idx = _get_idx_in_axis( + process_mesh_group, + process_mesh_shape, + embedding_row_dim_mapping, + rank_id, + ) per_part_size = Weight_var.shape[0] relative_idx = relative_idx * per_part_size check_variable_and_dtype( - Out_grad, 'tensor', - ['float16', 'float32', 'float64', 'int32', 'int64'], '_c_identity') + Out_grad, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_embedding", '@tmp_0@GRAD'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_embedding", '@tmp_0@GRAD']) + ), dtype=Out_grad.dtype, shape=Out_grad.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=Out_grad.stop_gradient) + stop_gradient=Out_grad.stop_gradient, + ) # copy X_var's dist_attr to intermediate_var_0's dist_attr out_grad_dist_attr = dist_attr.get_input_dist_attr(Out_grad.name) assert out_grad_dist_attr is not None - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_grad_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_grad_dist_attr + ) - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - embedding_row_dim_mapping, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, + process_mesh_shape, + embedding_row_dim_mapping, + rank_id, + ) group = new_process_group(group_ranks) c_identity_op = main_block.append_op( @@ -568,41 +675,54 @@ class DistributedEmbeddingImpl(DistributedOperatorImpl): 'use_calc_stream': True, 'use_model_parallel': True, OP_ROLE_KEY: OpRole.Backward, - }) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') + }, + ) + check_variable_and_dtype( + intermediate_var_0, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) - set_comm_op_dist_attr_for_program(c_identity_op, dist_attr.process_mesh, - out_grad_dist_attr, ctx) + set_comm_op_dist_attr_for_program( + c_identity_op, dist_attr.process_mesh, out_grad_dist_attr, ctx + ) c_embedding_grad_op_desc = main_block.append_op(type='nop').desc c_embedding_grad_op_desc.set_type("c_embedding_grad") c_embedding_grad_op_desc.set_input('Ids', [Ids_var.name]) c_embedding_grad_op_desc.set_input('W', [Weight_var.name]) - c_embedding_grad_op_desc.set_input('Out@GRAD', - [intermediate_var_0.name]) + c_embedding_grad_op_desc.set_input( + 'Out@GRAD', [intermediate_var_0.name] + ) c_embedding_grad_op_desc.set_output('W@GRAD', [Weight_grad.name]) c_embedding_grad_op_desc._set_attr('start_index', relative_idx) c_embedding_grad_op_desc._set_attr(OP_ROLE_KEY, OpRole.Backward) c_embedding_grad_op = main_block.ops[-1] assert c_embedding_grad_op.type == "c_embedding_grad" - naive_copy_op_dist_attr_for_program(c_embedding_grad_op, backward_op, - ctx) + naive_copy_op_dist_attr_for_program( + c_embedding_grad_op, backward_op, ctx + ) # data parallel gradient synchronization act_grad_names = [Ids_var.name] out_grad_names = [kwargs['W@GRAD'][0]] - gradient_synchronization(ctx, backward_op, act_grad_names, - out_grad_names, rank_id) + gradient_synchronization( + ctx, backward_op, act_grad_names, out_grad_names, rank_id + ) -register_distributed_operator_impl("lookup_table_v2", - DistributedEmbeddingImpl("row_parallel")) -register_distributed_operator_impl("c_embedding", - DistributedEmbeddingImpl("row_parallel")) -register_distributed_operator_impl("lookup_table", - DistributedEmbeddingImpl("row_parallel")) +register_distributed_operator_impl( + "lookup_table_v2", DistributedEmbeddingImpl("row_parallel") +) +register_distributed_operator_impl( + "c_embedding", DistributedEmbeddingImpl("row_parallel") +) +register_distributed_operator_impl( + "lookup_table", DistributedEmbeddingImpl("row_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py b/python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py index a6b48165fa9..8e3fc06756c 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py @@ -25,17 +25,16 @@ from ..cost import build_comp_costs_from_descs class DistributedFillConstantBatchSizeLike(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedFillConstantBatchSizeLike, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedFillConstantBatchSizeLike("fill_constant_batch_size_like")) + DistributedFillConstantBatchSizeLike("fill_constant_batch_size_like") +) class DistributedFillConstantBatchSizeLikeImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedFillConstantBatchSizeLikeImpl0, self).__init__(name) self._forward_implemented = True @@ -45,7 +44,8 @@ class DistributedFillConstantBatchSizeLikeImpl0(DistributedOperatorImpl): cost = None if int(op_role) == int(OpRole.Backward): raise ValueError( - "The fill_constant_batch_size_like has no grad op.") + "The fill_constant_batch_size_like has no grad op." + ) else: cost = self.calc_fwd_cost(dist_op, ctx, cluster) assert cost is not None @@ -53,13 +53,18 @@ class DistributedFillConstantBatchSizeLikeImpl0(DistributedOperatorImpl): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes op_type = dist_op.serial_op.type cost_mapping = build_comp_costs_from_descs( - FillConstantBatchSizeLikeOpCost, ctx, processes, desc_mapping, - cluster) + FillConstantBatchSizeLikeOpCost, + ctx, + processes, + desc_mapping, + cluster, + ) res_cost = [cost_mapping] return res_cost @@ -81,8 +86,9 @@ class DistributedFillConstantBatchSizeLikeImpl0(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc op_dist_attr = dist_op.dist_attr @@ -105,7 +111,8 @@ class DistributedFillConstantBatchSizeLikeImpl0(DistributedOperatorImpl): # only the batch size dimemsion of input and output are relative. dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [0, 0]) + [x_dims_mapping, out_dims_mapping], [0, 0] + ) if dim_changed: changed = True @@ -143,4 +150,5 @@ class DistributedFillConstantBatchSizeLikeImpl0(DistributedOperatorImpl): register_distributed_operator_impl( "fill_constant_batch_size_like", - DistributedFillConstantBatchSizeLikeImpl0("fill_by_shape")) + DistributedFillConstantBatchSizeLikeImpl0("fill_by_shape"), +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py b/python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py index 0c8d8d7b768..783a9ce5a18 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py @@ -24,17 +24,16 @@ from ..process_group import new_process_group class DistributedFusedAttention(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedFusedAttention, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedFusedAttention("fused_attention")) + DistributedFusedAttention("fused_attention") +) class DistributedFusedAttentionImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedFusedAttentionImpl, self).__init__(name) self._forward_implemented = True @@ -60,10 +59,12 @@ class DistributedFusedAttentionImpl(DistributedOperatorImpl): if is_dim_shard(mapping): return False if len(qkv_w_dims_mapping) != 4 or is_dim_replicate( - qkv_w_dims_mapping[head_axis]): + qkv_w_dims_mapping[head_axis] + ): return False if len(qkv_bias_dims_mapping) != 3 or is_dim_replicate( - qkv_bias_dims_mapping[head_axis]): + qkv_bias_dims_mapping[head_axis] + ): return False if is_dim_replicate(out_w_dims_mapping[0]): return False @@ -71,10 +72,13 @@ class DistributedFusedAttentionImpl(DistributedOperatorImpl): return False replicated_dims = [ - qkv_w_dims_mapping[0], qkv_w_dims_mapping[-2], - qkv_w_dims_mapping[-1], qkv_bias_dims_mapping[0], - qkv_bias_dims_mapping[-1], out_w_dims_mapping[-1], - out_bias_dims_mapping[-1] + qkv_w_dims_mapping[0], + qkv_w_dims_mapping[-2], + qkv_w_dims_mapping[-1], + qkv_bias_dims_mapping[0], + qkv_bias_dims_mapping[-1], + out_w_dims_mapping[-1], + out_bias_dims_mapping[-1], ] for mapping in replicated_dims: if is_dim_shard(mapping): @@ -100,8 +104,9 @@ class DistributedFusedAttentionImpl(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -128,7 +133,8 @@ class DistributedFusedAttentionImpl(DistributedOperatorImpl): out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -145,22 +151,28 @@ class DistributedFusedAttentionImpl(DistributedOperatorImpl): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # infer logic comm presentation head_axis = 1 qkv_w = src_op.input('QKVW')[0] - qkv_w_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - qkv_w)[head_axis] - assert qkv_w_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - qkv_w_col_dim_mapping) + qkv_w_col_dim_mapping = op_dist_attr.get_input_dims_mapping(qkv_w)[ + head_axis + ] + assert ( + qkv_w_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + qkv_w_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = qkv_w_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # insert op @@ -181,20 +193,25 @@ class DistributedFusedAttentionImpl(DistributedOperatorImpl): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # infer logic comm presentation out_w = src_op.input('OutLinearW')[0] out_w_col_dim_mapping = op_dist_attr.get_input_dims_mapping(out_w)[-1] - assert out_w_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - out_w_col_dim_mapping) + assert ( + out_w_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + out_w_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = out_w_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # insert op @@ -207,4 +224,5 @@ class DistributedFusedAttentionImpl(DistributedOperatorImpl): register_distributed_operator_impl( - "fused_attention", DistributedFusedAttentionImpl("tensor_parallel")) + "fused_attention", DistributedFusedAttentionImpl("tensor_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py b/python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py index a9b29c2054f..9cb435d4aa8 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py @@ -24,17 +24,16 @@ from ..process_group import new_process_group class DistributedFusedFeedForward(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedFusedFeedForward, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedFusedFeedForward("fused_feedforward")) + DistributedFusedFeedForward("fused_feedforward") +) class DistributedFusedFeedForwardImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedFusedFeedForwardImpl, self).__init__(name) self._forward_implemented = True @@ -51,24 +50,30 @@ class DistributedFusedFeedForwardImpl(DistributedOperatorImpl): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) linear1_weight_dims_mapping = op_dist_attr.get_input_dims_mapping( - linear1_weight) + linear1_weight + ) linear1_bias_dims_mapping = op_dist_attr.get_input_dims_mapping( - linear1_bias) + linear1_bias + ) linear2_weight_dims_mapping = op_dist_attr.get_input_dims_mapping( - linear2_weight) + linear2_weight + ) linear2_bias_dims_mapping = op_dist_attr.get_input_dims_mapping( - linear2_bias) + linear2_bias + ) for mapping in x_dims_mapping[1:-1]: if is_dim_shard(mapping): return False if is_dim_shard(linear1_weight_dims_mapping[-2]) or is_dim_replicate( - linear1_weight_dims_mapping[-1]): + linear1_weight_dims_mapping[-1] + ): return False if is_dim_replicate(linear1_bias_dims_mapping[-1]): return False if is_dim_replicate(linear2_weight_dims_mapping[-2]) or is_dim_shard( - linear2_weight_dims_mapping[-1]): + linear2_weight_dims_mapping[-1] + ): return False if is_dim_shard(linear2_bias_dims_mapping[-1]): return False @@ -91,8 +96,9 @@ class DistributedFusedFeedForwardImpl(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -119,7 +125,8 @@ class DistributedFusedFeedForwardImpl(DistributedOperatorImpl): out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -136,21 +143,27 @@ class DistributedFusedFeedForwardImpl(DistributedOperatorImpl): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # infer logic comm presentation linear1_weight = src_op.input('Linear1Weight')[0] linear1_weight_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - linear1_weight)[-1] - assert linear1_weight_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - linear1_weight_col_dim_mapping) + linear1_weight + )[-1] + assert ( + linear1_weight_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + linear1_weight_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = linear1_weight_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # insert op @@ -172,21 +185,27 @@ class DistributedFusedFeedForwardImpl(DistributedOperatorImpl): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # infer logic comm presentation linear2_weight = src_op.input('Linear2Weight')[0] linear2_weight_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - linear2_weight)[-1] - assert linear2_weight_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - linear2_weight_col_dim_mapping) + linear2_weight + )[-1] + assert ( + linear2_weight_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + linear2_weight_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = linear2_weight_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # insert op @@ -199,4 +218,5 @@ class DistributedFusedFeedForwardImpl(DistributedOperatorImpl): register_distributed_operator_impl( - "fused_feedforward", DistributedFusedFeedForwardImpl("tensor_parallel")) + "fused_feedforward", DistributedFusedFeedForwardImpl("tensor_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_matmul.py b/python/paddle/distributed/auto_parallel/operators/dist_matmul.py index fd7852e4699..b13bbf740c7 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_matmul.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_matmul.py @@ -34,20 +34,31 @@ from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole from ..process_group import new_process_group from ..utils import _get_comm_group, _get_corresponding_rank from .dist_default import DistributedDefaultImpl0 -from ..cost import build_comp_desc_from_dist_op, build_comm_desc_from_dist_op, build_dp_costs +from ..cost import ( + build_comp_desc_from_dist_op, + build_comm_desc_from_dist_op, + build_dp_costs, +) from ..cost import build_comm_costs_from_descs, build_comp_costs_from_descs from ..cost import MatmulV2OpCost, MatmulOpCost, MulOpCost from ..cost import MatmulV2GradOpCost, MatmulGradOpCost, MulGradOpCost -from paddle.distributed.auto_parallel.cost.comm_op_cost import AllreduceSumOpCost, IdentityOpCost +from paddle.distributed.auto_parallel.cost.comm_op_cost import ( + AllreduceSumOpCost, + IdentityOpCost, +) def trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping): if trans_x: - x_dims_mapping[-1], x_dims_mapping[-2] = x_dims_mapping[ - -2], x_dims_mapping[-1] + x_dims_mapping[-1], x_dims_mapping[-2] = ( + x_dims_mapping[-2], + x_dims_mapping[-1], + ) if trans_y: - y_dims_mapping[-1], y_dims_mapping[-2] = y_dims_mapping[ - -2], y_dims_mapping[-1] + y_dims_mapping[-1], y_dims_mapping[-2] = ( + y_dims_mapping[-2], + y_dims_mapping[-1], + ) def copy_op_with_new_input_output(ctx, block, src_op, **kwargs): @@ -120,13 +131,17 @@ def _update_dims_mapping_for_matmul(dist_op): for i in range(new_out_dims_mapping_len - 2): broadcast_out_dims_mapping.append(out_dims_mapping[i]) - compatible_dims_mapping = compute_compatible_dims_mapping([ - broadcast_x_dims_mapping, broadcast_y_dims_mapping, - broadcast_out_dims_mapping - ]) + compatible_dims_mapping = compute_compatible_dims_mapping( + [ + broadcast_x_dims_mapping, + broadcast_y_dims_mapping, + broadcast_out_dims_mapping, + ] + ) if compatible_dims_mapping is None: - trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, - y_dims_mapping) + trans_x_y_dims_mapping( + trans_x, trans_y, x_dims_mapping, y_dims_mapping + ) return False for i in range(new_x_dims_mapping_len - 2): @@ -149,17 +164,20 @@ def _update_dims_mapping_for_matmul(dist_op): # The following which uses negative index can be work # when len(out_dims_mapping) > 2 and len(out_dims_mapping) <=2 dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, y_dims_mapping], [-1, -2]) + [x_dims_mapping, y_dims_mapping], [-1, -2] + ) if dim_changed: changed = True dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [-2, -2]) + [x_dims_mapping, out_dims_mapping], [-2, -2] + ) if dim_changed: changed = True dim_changed = compute_compatible_and_update_dim_mapping( - [y_dims_mapping, out_dims_mapping], [-1, -1]) + [y_dims_mapping, out_dims_mapping], [-1, -1] + ) if dim_changed: changed = True @@ -199,7 +217,8 @@ def _is_auto_compatible_for_matmul(dist_op): x_dims_mapping = copy.deepcopy(op_dist_attr.get_input_dims_mapping(x_name)) y_dims_mapping = copy.deepcopy(op_dist_attr.get_input_dims_mapping(y_name)) out_dims_mapping = copy.deepcopy( - op_dist_attr.get_output_dims_mapping(out_name)) + op_dist_attr.get_output_dims_mapping(out_name) + ) x_dims_mapping_len = len(x_dims_mapping) y_dims_mapping_len = len(y_dims_mapping) out_dims_mapping_len = len(out_dims_mapping) @@ -231,22 +250,23 @@ def _is_auto_compatible_for_matmul(dist_op): for i in range(out_dims_mapping_len - 2): broadcast_out_dims_mapping.append(out_dims_mapping[i]) - is_same = ((broadcast_x_dims_mapping == broadcast_y_dims_mapping) - and (broadcast_x_dims_mapping == broadcast_out_dims_mapping)) + is_same = (broadcast_x_dims_mapping == broadcast_y_dims_mapping) and ( + broadcast_x_dims_mapping == broadcast_out_dims_mapping + ) if not is_same: return False # The following which uses negative index can be work # when len(out_dims_mapping) > 2 and len(out_dims_mapping) <=2 - is_same = (x_dims_mapping[-1] == y_dims_mapping[-2]) + is_same = x_dims_mapping[-1] == y_dims_mapping[-2] if not is_same: return False - is_same = (x_dims_mapping[-2] == out_dims_mapping[-2]) + is_same = x_dims_mapping[-2] == out_dims_mapping[-2] if not is_same: return False - is_same = (y_dims_mapping[-1] == out_dims_mapping[-1]) + is_same = y_dims_mapping[-1] == out_dims_mapping[-1] if not is_same: return False @@ -262,8 +282,9 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): backward_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(backward_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in dist_attr.process_mesh.processes: @@ -274,22 +295,26 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): assert 'Out@GRAD' in kwargs, "input [{}] is not given".format('Out@GRAD') assert 'Y@GRAD' in kwargs, "output [{}] is not given".format('Y@GRAD') assert 'X@GRAD' in kwargs, "output [{}] is not given".format('X@GRAD') - assert len( + assert ( + len(kwargs['Y']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['Y'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Y']) - assert len( + ) + assert ( + len(kwargs['X']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['X'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['X']) - assert len( - kwargs['Out@GRAD'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Out']) - assert len( + ) + assert ( + len(kwargs['Out@GRAD']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( + kwargs['Out'] + ) + assert ( + len(kwargs['Y@GRAD']) == 1 + ), "row_parallel_embedding output Ids take 1 variable but got {}".format( kwargs['Y@GRAD'] - ) == 1, "row_parallel_embedding output Ids take 1 variable but got {}".format( - kwargs['Y@GRAD']) + ) X_var = main_block.var(kwargs['X'][0]) Y_var = main_block._var_recursive(kwargs['Y'][0]) @@ -299,7 +324,8 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): assert not is_parameter_related( X_var.name, main_block ), "left operand(X) [{}] of dist matmul should not be parameter".format( - X_var.name) + X_var.name + ) X_var_dims_mapping = dist_attr.get_input_dims_mapping(X_var.name) Y_var_dim_mapping = dist_attr.get_input_dims_mapping(Y_var.name) @@ -336,28 +362,34 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): parallel_axis = Y_var_dim_mapping[0] check_variable_and_dtype( - Out_grad, 'tensor', + Out_grad, + 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - '_c_identity') + '_c_identity', + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])) + "@GRAD", + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ) + + "@GRAD", dtype=Out_grad.dtype, shape=Out_grad.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=Out_grad.stop_gradient) + stop_gradient=Out_grad.stop_gradient, + ) # copy X_var's dist_attr to intermediate_var_0's dist_attr out_grad_dist_attr = dist_attr.get_input_dist_attr(Out_grad.name) assert out_grad_dist_attr is not None - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_grad_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_grad_dist_attr + ) - group_ranks = _get_comm_group(process_mesh_group, - process_mesh_shape, parallel_axis, - rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) c_identity_op = main_block.append_op( type='c_identity', @@ -368,20 +400,29 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): 'use_calc_stream': True, 'use_model_parallel': True, OP_ROLE_KEY: OpRole.Backward, - }) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], - 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') - set_comm_op_dist_attr_for_program(c_identity_op, - dist_attr.process_mesh, - out_grad_dist_attr, ctx) + }, + ) + check_variable_and_dtype( + intermediate_var_0, + 'x', + ['float16', 'float32', 'float64'], + 'linear', + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) + set_comm_op_dist_attr_for_program( + c_identity_op, dist_attr.process_mesh, out_grad_dist_attr, ctx + ) new_kwargs = copy.deepcopy(kwargs) new_kwargs['Out@GRAD'] = [intermediate_var_0.name] matmul_op_desc = copy_op_with_new_input_output( - ctx, main_block, backward_op, **new_kwargs) + ctx, main_block, backward_op, **new_kwargs + ) else: # col parallel: matmul + allreduce assert Y_var_dim_mapping[0] < 0 @@ -394,28 +435,36 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): assert len(kwargs['X@GRAD']) == 1 X_grad = main_block.var(kwargs['X@GRAD'][0]) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])) + "@GRAD", + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ) + + "@GRAD", dtype=X_grad.dtype, shape=X_grad.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_grad.stop_gradient) + stop_gradient=X_grad.stop_gradient, + ) X_grad_dist_attr = dist_attr.get_output_dist_attr(X_grad.name) assert X_grad_dist_attr is not None - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - X_grad_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, X_grad_dist_attr + ) new_kwargs['X@GRAD'] = [intermediate_var_0.name] matmul_op_desc = copy_op_with_new_input_output( - ctx, main_block, backward_op, **new_kwargs) + ctx, main_block, backward_op, **new_kwargs + ) # NOTE (JZ-LIANG) trick to skip one allreduce if left operand has not grad if has_x_grad: - group_ranks = _get_comm_group(process_mesh_group, - process_mesh_shape, parallel_axis, - rank_id) + group_ranks = _get_comm_group( + process_mesh_group, + process_mesh_shape, + parallel_axis, + rank_id, + ) group = new_process_group(group_ranks) c_allreduce_sum_op = main_block.append_op( type='c_allreduce_sum', @@ -425,15 +474,20 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: OpRole.Backward - }) - set_comm_op_dist_attr_for_program(c_allreduce_sum_op, - dist_attr.process_mesh, - X_grad_dist_attr, ctx) + OP_ROLE_KEY: OpRole.Backward, + }, + ) + set_comm_op_dist_attr_for_program( + c_allreduce_sum_op, + dist_attr.process_mesh, + X_grad_dist_attr, + ctx, + ) else: # replicate - matmul_op_desc = copy_op_with_new_input_output(ctx, main_block, - backward_op, **kwargs) + matmul_op_desc = copy_op_with_new_input_output( + ctx, main_block, backward_op, **kwargs + ) # data parallel gradient synchronization act_grad_names = [X_var.name] @@ -445,8 +499,9 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): if trans_x: trans_x_y_dims_mapping(True, False, X_var_dims_mapping, None) - gradient_synchronization(ctx, backward_op, act_grad_names, out_grad_names, - rank_id) + gradient_synchronization( + ctx, backward_op, act_grad_names, out_grad_names, rank_id + ) if trans_x: trans_x_y_dims_mapping(True, False, X_var_dims_mapping, None) @@ -469,23 +524,25 @@ def _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, rank_id): if size <= 1 or axis in dim_mapping: pass else: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, axis, rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, axis, rank_id + ) sync_group = new_process_group(group_ranks) - startup_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': sync_group.id, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + startup_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': sync_group.id, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) class DistributedMatmul(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedMatmul, self).__init__(op_type) @@ -495,7 +552,6 @@ register_distributed_operator_impl_container(DistributedMatmul("matmul")) # ColumnParallel class DistributedMatmulImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulImpl0, self).__init__(name) self._forward_implemented = True @@ -518,7 +574,8 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) # col parallel: matmul + allreduce assert Y_var_dim_mapping[0] < 0 parallel_axis = Y_var_dim_mapping[1] @@ -528,13 +585,14 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): assert len(backward_op.output("X@GRAD")) == 1 # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # calc comm op cost @@ -547,40 +605,52 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, - c_allreduce_sum_desc_mapping, cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res.append(comm_op_cost_list) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulOpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-1] + serial_op.input("Y")[0] + )[-1] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.input("X") c_identity_desc_mapping = build_comm_desc_from_dist_op( @@ -589,10 +659,12 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res_cost = [comm_op_cost_list, cost_mapping] return res_cost @@ -603,16 +675,19 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): x_name = op_desc.input('X')[0] y_name = op_desc.input('Y')[0] x_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(x_name)) + op_dist_attr.get_input_dims_mapping(x_name) + ) y_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(y_name)) + op_dist_attr.get_input_dims_mapping(y_name) + ) trans_x = op_desc.attr('transpose_X') trans_y = op_desc.attr('transpose_Y') trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping) if is_dim_shard(x_dims_mapping[-1]): return False if is_dim_shard(y_dims_mapping[-2]) or is_dim_replicate( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False for mapping in x_dims_mapping[1:-1]: if is_dim_shard(mapping): @@ -632,8 +707,9 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): return False @@ -658,28 +734,33 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block.var(kwargs['Y'][0]) @@ -689,18 +770,24 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): # TODO infer logic comm presentation matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] + Weight_var.name + )[-1] if trans_y: matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] - assert matmul_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_col_dim_mapping) + Weight_var.name + )[-2] + assert ( + matmul_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # infer new var shape with op dist attr @@ -708,31 +795,39 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): assert x_tensor_dist_attr is not None identity_var_dist_attr = op_dist_attr.get_input_dist_attr(X_var.name) assert identity_var_dist_attr is not None - ref_shape_x = infer_shape(main_block, X_var, x_tensor_dist_attr, - identity_var_dist_attr) + ref_shape_x = infer_shape( + main_block, X_var, x_tensor_dist_attr, identity_var_dist_attr + ) # infer out var shape with op dist attr out_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(Out_var) assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape_out = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape_out = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ), dtype=X_var.dtype, shape=X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_var.stop_gradient) + stop_gradient=X_var.stop_gradient, + ) # set intermediate_var_0's dist_attr with X_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - identity_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, identity_var_dist_attr + ) check_variable_and_dtype( - X_var, 'tensor', - ['float16', 'float32', 'float64', 'int32', 'int64'], '_c_identity') + X_var, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) c_identity_op = main_block.append_op( type='c_identity', @@ -742,26 +837,31 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if intermediate_var_0.shape != ref_shape_x: intermediate_var_0.desc.set_shape(ref_shape_x) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') + check_variable_and_dtype( + intermediate_var_0, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) attrs = { 'transpose_X': trans_x, 'transpose_Y': trans_y, 'alpha': 1, - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': [intermediate_var_0], 'Y': [Weight_var]} - matmul_op = main_block.append_op(type='matmul', - inputs=inputs, - outputs={'Out': Out_var}, - attrs=attrs) + matmul_op = main_block.append_op( + type='matmul', inputs=inputs, outputs={'Out': Out_var}, attrs=attrs + ) if Out_var.shape != ref_shape_out: Out_var.desc.set_shape(ref_shape_out) @@ -775,13 +875,16 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): input_varname = c_identity_op.desc.input_arg_names()[0] input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - identity_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + identity_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) # output output_varname = c_identity_op.desc.output_arg_names()[0] - identity_op_dist_attr.set_output_dist_attr(output_varname, - input_dist_attr) + identity_op_dist_attr.set_output_dist_attr( + output_varname, input_dist_attr + ) # set op dist attr ctx.set_op_dist_attr_for_program(c_identity_op, identity_op_dist_attr) @@ -794,31 +897,39 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): for input_varname in matmul_op.desc.input_arg_names(): if input_varname in src_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr( - input_varname) + input_varname + ) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmul_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + matmul_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) else: input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program( - input_var) - matmul_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + input_var + ) + matmul_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) # output output_varname = matmul_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmul_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmul_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) # set op dist attr ctx.set_op_dist_attr_for_program(matmul_op, matmul_op_dist_attr) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -827,7 +938,6 @@ class DistributedMatmulImpl0(DistributedOperatorImpl): # RowParallel class DistributedMatmulImpl1(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulImpl1, self).__init__(name) self._forward_implemented = True @@ -850,7 +960,8 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) assert Y_var_dim_mapping[1] < 0 parallel_axis = Y_var_dim_mapping[0] @@ -863,50 +974,60 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res.append(comm_op_cost_list) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) - cost_mapping = build_comp_costs_from_descs(MatmulGradOpCost, ctx, - processes, desc_mapping, - cluster) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) + cost_mapping = build_comp_costs_from_descs( + MatmulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulOpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-2] + serial_op.input("Y")[0] + )[-2] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.output("Out") @@ -916,11 +1037,16 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, c_allreduce_sum_desc_mapping, - cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res_cost = [cost_mapping, comm_op_cost_list] @@ -932,16 +1058,19 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): x_name = op_desc.input('X')[0] y_name = op_desc.input('Y')[0] x_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(x_name)) + op_dist_attr.get_input_dims_mapping(x_name) + ) y_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(y_name)) + op_dist_attr.get_input_dims_mapping(y_name) + ) trans_x = op_desc.attr('transpose_X') trans_y = op_desc.attr('transpose_Y') trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping) if is_dim_replicate(x_dims_mapping[-1]): return False if is_dim_replicate(y_dims_mapping[-2]) or is_dim_shard( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False # Other dimensions must be replicate except the batch dimension for mapping in x_dims_mapping[1:-1]: @@ -963,8 +1092,9 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): return False @@ -989,28 +1119,33 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block.var(kwargs['Y'][0]) @@ -1020,29 +1155,37 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): # TODO infer logic comm presentation matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] + Weight_var.name + )[-2] if trans_y: matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] - assert matmul_row_dim_mapping >= 0, "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_row_dim_mapping) + Weight_var.name + )[-1] + assert ( + matmul_row_dim_mapping >= 0 + ), "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_row_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_row_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) - check_variable_and_dtype(X_var, 'x', ['float16', 'float32', 'float64'], - 'linear') - check_dtype(X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + X_var, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear' + ) attrs = { 'transpose_X': trans_x, 'transpose_Y': trans_y, 'alpha': 1, - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': X_var, 'Y': Weight_var} @@ -1051,27 +1194,33 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_allreduce_sum", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_allreduce_sum", 'tmp']) + ), shape=Out_var.shape, dtype=Out_var.dtype, type=Out_var.type, lod_level=Out_var.lod_level, persistable=False, is_data=False, - need_check_feed=Out_var.desc.need_check_feed()) + need_check_feed=Out_var.desc.need_check_feed(), + ) # set intermediate_var_0's dist_attr with Out_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_var_dist_attr + ) - matmul_op = main_block.append_op(type='matmul', - inputs=inputs, - outputs={'Out': intermediate_var_0}, - attrs=attrs) + matmul_op = main_block.append_op( + type='matmul', + inputs=inputs, + outputs={'Out': intermediate_var_0}, + attrs=attrs, + ) if intermediate_var_0.shape != ref_shape: intermediate_var_0.desc.set_shape(ref_shape) @@ -1083,8 +1232,9 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if Out_var.shape != ref_shape: Out_var.desc.set_shape(ref_shape) @@ -1097,15 +1247,19 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): for input_varname in matmul_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmul_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + matmul_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) output_varname = matmul_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmul_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmul_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(matmul_op, matmul_op_dist_attr) # allreduce @@ -1117,21 +1271,26 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(input_var) assert tensor_dist_attr is not None - allreduce_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + allreduce_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) for output_varname in c_allreduce_sum_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - allreduce_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) - ctx.set_op_dist_attr_for_program(c_allreduce_sum_op, - allreduce_op_dist_attr) + op_dist_attr + ) + allreduce_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) + ctx.set_op_dist_attr_for_program( + c_allreduce_sum_op, allreduce_op_dist_attr + ) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -1140,7 +1299,6 @@ class DistributedMatmulImpl1(DistributedOperatorImpl): # ReplicateParallel class DistributedMatmulImpl2(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulImpl2, self).__init__(name) @@ -1161,38 +1319,45 @@ class DistributedMatmulImpl2(DistributedOperatorImpl): vars = main_block.vars # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulOpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -1208,13 +1373,15 @@ class DistributedMatmulImpl2(DistributedOperatorImpl): if is_dim_shard(x_dims_mapping[-1]): return False if is_valid_list_index(x_dims_mapping, -2) and is_dim_shard( - x_dims_mapping[-2]): + x_dims_mapping[-2] + ): return False if is_dim_shard(y_dims_mapping[-1]): return False if is_valid_list_index(y_dims_mapping, -2) and is_dim_shard( - y_dims_mapping[-2]): + y_dims_mapping[-2] + ): return False return True @@ -1228,14 +1395,16 @@ class DistributedMatmulImpl2(DistributedOperatorImpl): if is_dim_shard(out_dims_mapping[-1]): return False if is_valid_list_index(out_dims_mapping, -2) and is_dim_shard( - out_dims_mapping[-2]): + out_dims_mapping[-2] + ): return False return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -1259,16 +1428,18 @@ class DistributedMatmulImpl2(DistributedOperatorImpl): _right_operand_parameter_matmul_backward(ctx, *args, **kwargs) -register_distributed_operator_impl("matmul", - DistributedMatmulImpl0("column_parallel")) -register_distributed_operator_impl("matmul", - DistributedMatmulImpl1("row_parallel")) -register_distributed_operator_impl("matmul", - DistributedMatmulImpl2("replicate_parallel")) +register_distributed_operator_impl( + "matmul", DistributedMatmulImpl0("column_parallel") +) +register_distributed_operator_impl( + "matmul", DistributedMatmulImpl1("row_parallel") +) +register_distributed_operator_impl( + "matmul", DistributedMatmulImpl2("replicate_parallel") +) class DistributedMatmulV2(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedMatmulV2, self).__init__(op_type) @@ -1278,7 +1449,6 @@ register_distributed_operator_impl_container(DistributedMatmulV2("matmul_v2")) # ColumnParallel class DistributedMatmulV2Impl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulV2Impl0, self).__init__(name) self._forward_implemented = True @@ -1301,7 +1471,8 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes # col parallel: matmul + allreduce @@ -1315,12 +1486,13 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): assert len(backward_op.output("X@GRAD")) == 1 # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) - cost_mapping = build_comp_costs_from_descs(MatmulV2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulV2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # calc comm op cost @@ -1333,45 +1505,55 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, - c_allreduce_sum_desc_mapping, cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res.append(comm_op_cost_list) # need gradient allreduce process_mesh = dist_attr.process_mesh var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost # TODO: trans shape if trans_x or trans_y is True - comp_desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + comp_desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - comp_cost_mapping = build_comp_costs_from_descs(MatmulV2OpCost, ctx, - processes, - comp_desc_mapping, - cluster) + comp_cost_mapping = build_comp_costs_from_descs( + MatmulV2OpCost, ctx, processes, comp_desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-1] + serial_op.input("Y")[0] + )[-1] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.input("X") @@ -1381,9 +1563,11 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res_cost = [comm_op_cost_list, comp_cost_mapping] return res_cost @@ -1394,16 +1578,19 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): x_name = op_desc.input('X')[0] y_name = op_desc.input('Y')[0] x_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(x_name)) + op_dist_attr.get_input_dims_mapping(x_name) + ) y_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(y_name)) + op_dist_attr.get_input_dims_mapping(y_name) + ) trans_x = op_desc.attr('trans_x') trans_y = op_desc.attr('trans_y') trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping) if is_dim_shard(x_dims_mapping[-1]): return False if is_dim_shard(y_dims_mapping[-2]) or is_dim_replicate( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False for mapping in x_dims_mapping[1:-1]: if is_dim_shard(mapping): @@ -1423,8 +1610,9 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): return False @@ -1449,28 +1637,33 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block._var_recursive(kwargs['Y'][0]) @@ -1480,18 +1673,24 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): # TODO infer logic comm presentation matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] + Weight_var.name + )[-1] if trans_y: matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] - assert matmul_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_col_dim_mapping) + Weight_var.name + )[-2] + assert ( + matmul_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # infer new var shape with op dist attr @@ -1499,31 +1698,39 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): assert x_tensor_dist_attr is not None identity_var_dist_attr = op_dist_attr.get_input_dist_attr(X_var.name) assert identity_var_dist_attr is not None - ref_shape_x = infer_shape(main_block, X_var, x_tensor_dist_attr, - identity_var_dist_attr) + ref_shape_x = infer_shape( + main_block, X_var, x_tensor_dist_attr, identity_var_dist_attr + ) # infer out var shape with op dist attr out_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(Out_var) assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape_out = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape_out = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ), dtype=X_var.dtype, shape=X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_var.stop_gradient) + stop_gradient=X_var.stop_gradient, + ) # set intermediate_var_0's dist_attr with X_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - identity_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, identity_var_dist_attr + ) check_variable_and_dtype( - X_var, 'tensor', - ['float16', 'float32', 'float64', 'int32', 'int64'], '_c_identity') + X_var, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) c_identity_op = main_block.append_op( type='c_identity', inputs={'X': [X_var]}, @@ -1533,24 +1740,32 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): 'use_calc_stream': True, 'use_model_parallel': True, OP_ROLE_KEY: src_op.attr('op_role'), - }) + }, + ) if intermediate_var_0.shape != ref_shape_x: intermediate_var_0.desc.set_shape(ref_shape_x) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') + check_variable_and_dtype( + intermediate_var_0, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) attrs = { 'trans_x': trans_x, 'trans_y': trans_y, - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': [intermediate_var_0], 'Y': [Weight_var]} - matmul_v2_op = main_block.append_op(type='matmul_v2', - inputs=inputs, - outputs={'Out': Out_var}, - attrs=attrs) + matmul_v2_op = main_block.append_op( + type='matmul_v2', + inputs=inputs, + outputs={'Out': Out_var}, + attrs=attrs, + ) if Out_var.shape != ref_shape_out: Out_var.desc.set_shape(ref_shape_out) @@ -1564,13 +1779,16 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): input_varname = c_identity_op.desc.input_arg_names()[0] input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - identity_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + identity_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) # output output_varname = c_identity_op.desc.output_arg_names()[0] - identity_op_dist_attr.set_output_dist_attr(output_varname, - input_dist_attr) + identity_op_dist_attr.set_output_dist_attr( + output_varname, input_dist_attr + ) ctx.set_op_dist_attr_for_program(c_identity_op, identity_op_dist_attr) # matmulv2 @@ -1581,29 +1799,37 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): for input_varname in matmul_v2_op.desc.input_arg_names(): if input_varname in src_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr( - input_varname) + input_varname + ) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) + op_dist_attr + ) matmulv2_op_dist_attr.set_input_dist_attr( - input_varname, input_dist_attr) + input_varname, input_dist_attr + ) else: input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program( - input_var) + input_var + ) matmulv2_op_dist_attr.set_input_dist_attr( - input_varname, tensor_dist_attr) + input_varname, tensor_dist_attr + ) for output_varname in matmul_v2_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(matmul_v2_op, matmulv2_op_dist_attr) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -1612,7 +1838,6 @@ class DistributedMatmulV2Impl0(DistributedOperatorImpl): # RowParallel class DistributedMatmulV2Impl1(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulV2Impl1, self).__init__(name) self._forward_implemented = True @@ -1635,7 +1860,8 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) assert Y_var_dim_mapping[1] < 0 parallel_axis = Y_var_dim_mapping[0] @@ -1650,50 +1876,59 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res.append(comm_op_cost_list) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) - cost_mapping = build_comp_costs_from_descs(MatmulV2GradOpCost, ctx, - processes, desc_mapping, - cluster) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) + cost_mapping = build_comp_costs_from_descs( + MatmulV2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce process_mesh = dist_attr.process_mesh var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulV2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulV2OpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-2] + serial_op.input("Y")[0] + )[-2] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.output("Out") @@ -1703,11 +1938,16 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, c_allreduce_sum_desc_mapping, - cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res_cost = [cost_mapping, comm_op_cost_list] return res_cost @@ -1718,16 +1958,19 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): x_name = op_desc.input('X')[0] y_name = op_desc.input('Y')[0] x_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(x_name)) + op_dist_attr.get_input_dims_mapping(x_name) + ) y_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(y_name)) + op_dist_attr.get_input_dims_mapping(y_name) + ) trans_x = op_desc.attr('trans_x') trans_y = op_desc.attr('trans_y') trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping) if is_dim_replicate(x_dims_mapping[-1]): return False if is_dim_replicate(y_dims_mapping[-2]) or is_dim_shard( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False # Other dimensions must be replicate except the batch dimension for mapping in x_dims_mapping[1:-1]: @@ -1749,8 +1992,9 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): return False @@ -1775,28 +2019,33 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block._var_recursive(kwargs['Y'][0]) @@ -1806,28 +2055,36 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): # TODO infer logic comm presentation matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] + Weight_var.name + )[-2] if trans_y: matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] - assert matmul_row_dim_mapping >= 0, "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_row_dim_mapping) + Weight_var.name + )[-1] + assert ( + matmul_row_dim_mapping >= 0 + ), "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_row_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_row_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) - check_variable_and_dtype(X_var, 'x', ['float16', 'float32', 'float64'], - 'linear') - check_dtype(X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + X_var, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear' + ) attrs = { 'trans_x': trans_x, 'trans_y': trans_y, - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': X_var, 'Y': Weight_var} @@ -1836,27 +2093,33 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_allreduce_sum", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_allreduce_sum", 'tmp']) + ), shape=Out_var.shape, dtype=Out_var.dtype, type=Out_var.type, lod_level=Out_var.lod_level, persistable=False, is_data=False, - need_check_feed=Out_var.desc.need_check_feed()) + need_check_feed=Out_var.desc.need_check_feed(), + ) # set intermediate_var_0's dist_attr with Out_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_var_dist_attr + ) - matmul_v2_op = main_block.append_op(type='matmul_v2', - inputs=inputs, - outputs={'Out': intermediate_var_0}, - attrs=attrs) + matmul_v2_op = main_block.append_op( + type='matmul_v2', + inputs=inputs, + outputs={'Out': intermediate_var_0}, + attrs=attrs, + ) if intermediate_var_0.shape != ref_shape: intermediate_var_0.desc.set_shape(ref_shape) @@ -1868,8 +2131,9 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if Out_var.shape != ref_shape: Out_var.desc.set_shape(ref_shape) @@ -1882,15 +2146,19 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): for input_varname in matmul_v2_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) output_varname = matmul_v2_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(matmul_v2_op, matmulv2_op_dist_attr) # allreduce @@ -1902,21 +2170,26 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(input_var) assert tensor_dist_attr is not None - allreduce_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + allreduce_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) for output_varname in c_allreduce_sum_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - allreduce_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) - ctx.set_op_dist_attr_for_program(c_allreduce_sum_op, - allreduce_op_dist_attr) + op_dist_attr + ) + allreduce_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) + ctx.set_op_dist_attr_for_program( + c_allreduce_sum_op, allreduce_op_dist_attr + ) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -1925,7 +2198,6 @@ class DistributedMatmulV2Impl1(DistributedOperatorImpl): # ReplicateParallel class DistributedMatmulV2Impl2(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulV2Impl2, self).__init__(name) @@ -1947,38 +2219,44 @@ class DistributedMatmulV2Impl2(DistributedOperatorImpl): process_mesh = dist_attr.process_mesh # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulV2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulV2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulV2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulV2OpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] @@ -1995,13 +2273,15 @@ class DistributedMatmulV2Impl2(DistributedOperatorImpl): if is_dim_shard(x_dims_mapping[-1]): return False if is_valid_list_index(x_dims_mapping, -2) and is_dim_shard( - x_dims_mapping[-2]): + x_dims_mapping[-2] + ): return False if is_dim_shard(y_dims_mapping[-1]): return False if is_valid_list_index(y_dims_mapping, -2) and is_dim_shard( - y_dims_mapping[-2]): + y_dims_mapping[-2] + ): return False return True @@ -2016,14 +2296,16 @@ class DistributedMatmulV2Impl2(DistributedOperatorImpl): if is_dim_shard(out_dims_mapping[-1]): return False if is_valid_list_index(out_dims_mapping, -2) and is_dim_shard( - out_dims_mapping[-2]): + out_dims_mapping[-2] + ): return False return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -2047,16 +2329,18 @@ class DistributedMatmulV2Impl2(DistributedOperatorImpl): _right_operand_parameter_matmul_backward(ctx, *args, **kwargs) -register_distributed_operator_impl("matmul_v2", - DistributedMatmulV2Impl0("column_parallel")) -register_distributed_operator_impl("matmul_v2", - DistributedMatmulV2Impl1("row_parallel")) register_distributed_operator_impl( - "matmul_v2", DistributedMatmulV2Impl2("replicate_parallel")) + "matmul_v2", DistributedMatmulV2Impl0("column_parallel") +) +register_distributed_operator_impl( + "matmul_v2", DistributedMatmulV2Impl1("row_parallel") +) +register_distributed_operator_impl( + "matmul_v2", DistributedMatmulV2Impl2("replicate_parallel") +) class DistributedMul(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedMul, self).__init__(op_type) @@ -2066,7 +2350,6 @@ register_distributed_operator_impl_container(DistributedMul("mul")) # ColumnParallel class DistributedMulImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMulImpl0, self).__init__(name) self._forward_implemented = True @@ -2089,7 +2372,8 @@ class DistributedMulImpl0(DistributedOperatorImpl): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) # col parallel: matmul + allreduce assert Y_var_dim_mapping[0] < 0 parallel_axis = Y_var_dim_mapping[1] @@ -2099,13 +2383,14 @@ class DistributedMulImpl0(DistributedOperatorImpl): assert len(backward_op.output("X@GRAD")) == 1 # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # calc comm op cost @@ -2118,40 +2403,52 @@ class DistributedMulImpl0(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, - c_allreduce_sum_desc_mapping, cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res.append(comm_op_cost_list) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MulOpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-1] + serial_op.input("Y")[0] + )[-1] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.input("X") c_identity_desc_mapping = build_comm_desc_from_dist_op( @@ -2160,10 +2457,12 @@ class DistributedMulImpl0(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res_cost = [comm_op_cost_list, cost_mapping] return res_cost @@ -2178,7 +2477,8 @@ class DistributedMulImpl0(DistributedOperatorImpl): if is_dim_shard(x_dims_mapping[-1]): return False if is_dim_shard(y_dims_mapping[-2]) or is_dim_replicate( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False for mapping in x_dims_mapping[1:-1]: if is_dim_shard(mapping): @@ -2198,8 +2498,9 @@ class DistributedMulImpl0(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -2226,28 +2527,33 @@ class DistributedMulImpl0(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block._var_recursive(kwargs['Y'][0]) @@ -2255,15 +2561,20 @@ class DistributedMulImpl0(DistributedOperatorImpl): # TODO infer logic comm presentation matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] - assert matmul_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_col_dim_mapping) + Weight_var.name + )[-1] + assert ( + matmul_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # infer new var shape with op dist attr @@ -2271,31 +2582,39 @@ class DistributedMulImpl0(DistributedOperatorImpl): assert x_tensor_dist_attr is not None identity_var_dist_attr = op_dist_attr.get_input_dist_attr(X_var.name) assert identity_var_dist_attr is not None - ref_shape_x = infer_shape(main_block, X_var, x_tensor_dist_attr, - identity_var_dist_attr) + ref_shape_x = infer_shape( + main_block, X_var, x_tensor_dist_attr, identity_var_dist_attr + ) # infer out var shape with op dist attr out_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(Out_var) assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape_out = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape_out = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ), dtype=X_var.dtype, shape=X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_var.stop_gradient) + stop_gradient=X_var.stop_gradient, + ) # set intermediate_var_0's dist_attr with X_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - identity_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, identity_var_dist_attr + ) check_variable_and_dtype( - X_var, 'tensor', - ['float16', 'float32', 'float64', 'int32', 'int64'], '_c_identity') + X_var, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) c_identity_op = main_block.append_op( type='c_identity', inputs={'X': [X_var]}, @@ -2304,20 +2623,26 @@ class DistributedMulImpl0(DistributedOperatorImpl): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if intermediate_var_0.shape != ref_shape_x: intermediate_var_0.desc.set_shape(ref_shape_x) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') + check_variable_and_dtype( + intermediate_var_0, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) # attrs = {'trans_x': False, 'trans_y': False} attrs = { "x_num_col_dims": src_op.desc.attr("x_num_col_dims"), "y_num_col_dims": src_op.desc.attr("y_num_col_dims"), - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': intermediate_var_0, 'Y': Weight_var} @@ -2331,16 +2656,15 @@ class DistributedMulImpl0(DistributedOperatorImpl): inputs_original_shape[var_name] = var.shape input_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(var) input_var_dist_attr = op_dist_attr.get_input_dist_attr(var.name) - input_ref_shape = infer_shape(main_block, var, - input_tensor_dist_attr, - input_var_dist_attr) + input_ref_shape = infer_shape( + main_block, var, input_tensor_dist_attr, input_var_dist_attr + ) inputs_ref_shape[var_name] = input_ref_shape var.desc.set_shape(input_ref_shape) - mul_op = main_block.append_op(type='mul', - inputs=inputs, - outputs={'Out': Out_var}, - attrs=attrs) + mul_op = main_block.append_op( + type='mul', inputs=inputs, outputs={'Out': Out_var}, attrs=attrs + ) if Out_var.shape != ref_shape_out: Out_var.desc.set_shape(ref_shape_out) @@ -2359,13 +2683,16 @@ class DistributedMulImpl0(DistributedOperatorImpl): input_varname = c_identity_op.desc.input_arg_names()[0] input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - identity_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + identity_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) # output output_varname = c_identity_op.desc.output_arg_names()[0] - identity_op_dist_attr.set_output_dist_attr(output_varname, - input_dist_attr) + identity_op_dist_attr.set_output_dist_attr( + output_varname, input_dist_attr + ) ctx.set_op_dist_attr_for_program(c_identity_op, identity_op_dist_attr) # matmulv2 @@ -2376,29 +2703,37 @@ class DistributedMulImpl0(DistributedOperatorImpl): for input_varname in mul_op.desc.input_arg_names(): if input_varname in src_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr( - input_varname) + input_varname + ) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) + op_dist_attr + ) matmulv2_op_dist_attr.set_input_dist_attr( - input_varname, input_dist_attr) + input_varname, input_dist_attr + ) else: input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program( - input_var) + input_var + ) matmulv2_op_dist_attr.set_input_dist_attr( - input_varname, tensor_dist_attr) + input_varname, tensor_dist_attr + ) for output_varname in mul_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(mul_op, matmulv2_op_dist_attr) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -2407,7 +2742,6 @@ class DistributedMulImpl0(DistributedOperatorImpl): # RowParallel class DistributedMulImpl1(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMulImpl1, self).__init__(name) self._forward_implemented = True @@ -2431,7 +2765,8 @@ class DistributedMulImpl1(DistributedOperatorImpl): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) assert Y_var_dim_mapping[1] < 0 parallel_axis = Y_var_dim_mapping[0] @@ -2444,49 +2779,59 @@ class DistributedMulImpl1(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) processes = process_mesh.processes comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res.append(comm_op_cost_list) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) - cost_mapping = build_comp_costs_from_descs(MulGradOpCost, ctx, - processes, desc_mapping, - cluster) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) + cost_mapping = build_comp_costs_from_descs( + MulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MulOpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-2] + serial_op.input("Y")[0] + )[-2] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.output("Out") @@ -2496,12 +2841,17 @@ class DistributedMulImpl1(DistributedOperatorImpl): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) # print("dist_matmul.py dist_op: ", dist_op) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, c_allreduce_sum_desc_mapping, - cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res_cost = [cost_mapping, comm_op_cost_list] @@ -2517,7 +2867,8 @@ class DistributedMulImpl1(DistributedOperatorImpl): if is_dim_replicate(x_dims_mapping[-1]): return False if is_dim_replicate(y_dims_mapping[-2]) or is_dim_shard( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False # Other dimensions must be replicate except the batch dimension for mapping in x_dims_mapping[1:-1]: @@ -2539,8 +2890,9 @@ class DistributedMulImpl1(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -2567,28 +2919,33 @@ class DistributedMulImpl1(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block._var_recursive(kwargs['Y'][0]) @@ -2596,26 +2953,33 @@ class DistributedMulImpl1(DistributedOperatorImpl): # TODO infer logic comm presentation matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] - assert matmul_row_dim_mapping >= 0, "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_row_dim_mapping) + Weight_var.name + )[-2] + assert ( + matmul_row_dim_mapping >= 0 + ), "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_row_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_row_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) - check_variable_and_dtype(X_var, 'x', ['float16', 'float32', 'float64'], - 'linear') - check_dtype(X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + X_var, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear' + ) # attrs = {'trans_x': False, 'trans_y': False} attrs = { "x_num_col_dims": src_op.desc.attr("x_num_col_dims"), "y_num_col_dims": src_op.desc.attr("y_num_col_dims"), - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': X_var, 'Y': Weight_var} @@ -2624,22 +2988,26 @@ class DistributedMulImpl1(DistributedOperatorImpl): assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_allreduce_sum", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_allreduce_sum", 'tmp']) + ), shape=Out_var.shape, dtype=Out_var.dtype, type=Out_var.type, lod_level=Out_var.lod_level, persistable=False, is_data=False, - need_check_feed=Out_var.desc.need_check_feed()) + need_check_feed=Out_var.desc.need_check_feed(), + ) # set intermediate_var_0's dist_attr with Out_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_var_dist_attr + ) inputs_ref_shape = {} inputs_original_shape = {} @@ -2648,16 +3016,18 @@ class DistributedMulImpl1(DistributedOperatorImpl): inputs_original_shape[var_name] = var.shape input_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(var) input_var_dist_attr = op_dist_attr.get_input_dist_attr(var.name) - input_ref_shape = infer_shape(main_block, var, - input_tensor_dist_attr, - input_var_dist_attr) + input_ref_shape = infer_shape( + main_block, var, input_tensor_dist_attr, input_var_dist_attr + ) inputs_ref_shape[var_name] = input_ref_shape var.desc.set_shape(input_ref_shape) - mul_op = main_block.append_op(type='mul', - inputs=inputs, - outputs={'Out': intermediate_var_0}, - attrs=attrs) + mul_op = main_block.append_op( + type='mul', + inputs=inputs, + outputs={'Out': intermediate_var_0}, + attrs=attrs, + ) if intermediate_var_0.shape != ref_shape: intermediate_var_0.desc.set_shape(ref_shape) @@ -2675,8 +3045,9 @@ class DistributedMulImpl1(DistributedOperatorImpl): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if Out_var.shape != ref_shape: Out_var.desc.set_shape(ref_shape) @@ -2690,15 +3061,19 @@ class DistributedMulImpl1(DistributedOperatorImpl): for input_varname in mul_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) output_varname = mul_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(mul_op, matmulv2_op_dist_attr) # allreduce @@ -2710,21 +3085,26 @@ class DistributedMulImpl1(DistributedOperatorImpl): input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(input_var) assert tensor_dist_attr is not None - allreduce_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + allreduce_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) for output_varname in c_allreduce_sum_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - allreduce_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) - ctx.set_op_dist_attr_for_program(c_allreduce_sum_op, - allreduce_op_dist_attr) + op_dist_attr + ) + allreduce_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) + ctx.set_op_dist_attr_for_program( + c_allreduce_sum_op, allreduce_op_dist_attr + ) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -2733,7 +3113,6 @@ class DistributedMulImpl1(DistributedOperatorImpl): # ReplicateParallel class DistributedMulImpl2(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMulImpl2, self).__init__(name) @@ -2754,38 +3133,45 @@ class DistributedMulImpl2(DistributedOperatorImpl): vars = main_block.vars # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MulOpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -2801,12 +3187,14 @@ class DistributedMulImpl2(DistributedOperatorImpl): if is_dim_shard(x_dims_mapping[-1]): return False if is_valid_list_index(x_dims_mapping, -2) and is_dim_shard( - x_dims_mapping[-2]): + x_dims_mapping[-2] + ): return False if is_dim_shard(y_dims_mapping[-1]): return False if is_valid_list_index(y_dims_mapping, -2) and is_dim_shard( - y_dims_mapping[-2]): + y_dims_mapping[-2] + ): return False return True @@ -2821,14 +3209,16 @@ class DistributedMulImpl2(DistributedOperatorImpl): if is_dim_shard(out_dims_mapping[-1]): return False if is_valid_list_index(out_dims_mapping, -2) and is_dim_shard( - out_dims_mapping[-2]): + out_dims_mapping[-2] + ): return False return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -2852,8 +3242,10 @@ class DistributedMulImpl2(DistributedOperatorImpl): _right_operand_parameter_matmul_backward(ctx, *args, **kwargs) -register_distributed_operator_impl("mul", - DistributedMulImpl0("column_parallel")) +register_distributed_operator_impl( + "mul", DistributedMulImpl0("column_parallel") +) register_distributed_operator_impl("mul", DistributedMulImpl1("row_parallel")) -register_distributed_operator_impl("mul", - DistributedMulImpl2("replicate_parallel")) +register_distributed_operator_impl( + "mul", DistributedMulImpl2("replicate_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_pnorm.py b/python/paddle/distributed/auto_parallel/operators/dist_pnorm.py index c56f121430a..68d2351e1ab 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_pnorm.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_pnorm.py @@ -20,8 +20,15 @@ from .common import register_distributed_operator_impl_container from .common import register_distributed_operator_impl from ..process_group import new_process_group from ..utils import is_dim_shard, is_dim_replicate, _get_corresponding_rank -from ..utils import compute_compatible_dim_mapping, set_dist_op_desc_original_id, _get_comm_group -from ..dist_attribute import TensorDistributedAttribute, OperatorDistributedAttribute +from ..utils import ( + compute_compatible_dim_mapping, + set_dist_op_desc_original_id, + _get_comm_group, +) +from ..dist_attribute import ( + TensorDistributedAttribute, + OperatorDistributedAttribute, +) from paddle.fluid import core from paddle.fluid.framework import Operator @@ -29,7 +36,6 @@ from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype class DistributedPNorm(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedPNorm, self).__init__(op_type) @@ -39,7 +45,6 @@ register_distributed_operator_impl_container(DistributedPNorm("p_norm")) # Row Parallel class DistributedPNormImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedPNormImpl, self).__init__(name) self._forward_implemented = True @@ -62,15 +67,18 @@ class DistributedPNormImpl(DistributedOperatorImpl): return True def is_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)) or \ - (not self.is_compatible(dist_op)): + if ( + (not self.is_input_compatible(dist_op)) + or (not self.is_output_compatible(dist_op)) + or (not self.is_compatible(dist_op)) + ): return False return True @@ -90,20 +98,25 @@ class DistributedPNormImpl(DistributedOperatorImpl): batch_dim_mappings.append(dims_mapping[0]) compatible_dim_mapping = compute_compatible_dim_mapping( - batch_dim_mappings) + batch_dim_mappings + ) if compatible_dim_mapping is None: return False for arg_name in op_desc.input_arg_names(): dims_mapping = op_dist_attr.get_input_dims_mapping(arg_name) - if len(dims_mapping - ) >= 1 and compatible_dim_mapping != dims_mapping[0]: + if ( + len(dims_mapping) >= 1 + and compatible_dim_mapping != dims_mapping[0] + ): dims_mapping[0] = compatible_dim_mapping changed = True for arg_name in op_desc.output_arg_names(): dims_mapping = op_dist_attr.get_output_dims_mapping(arg_name) - if len(dims_mapping - ) >= 1 and compatible_dim_mapping != dims_mapping[0]: + if ( + len(dims_mapping) >= 1 + and compatible_dim_mapping != dims_mapping[0] + ): dims_mapping[0] = compatible_dim_mapping changed = True @@ -122,21 +135,25 @@ class DistributedPNormImpl(DistributedOperatorImpl): # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) X_var = main_block.var(kwargs['X'][0]) in_dims_mapping = op_dist_attr.get_input_dims_mapping(X_var.name) @@ -145,14 +162,17 @@ class DistributedPNormImpl(DistributedOperatorImpl): break process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, axis, rank_id + ) group = new_process_group(group_ranks) - check_variable_and_dtype(X_var, 'x', ['float16', 'float32', 'float64'], - 'norm') - check_dtype(X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], - 'norm') + check_variable_and_dtype( + X_var, 'x', ['float16', 'float32', 'float64'], 'norm' + ) + check_dtype( + X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], 'norm' + ) # 2. insert c_allgather op # create c_allgather output var @@ -162,32 +182,37 @@ class DistributedPNormImpl(DistributedOperatorImpl): shape=X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_var.stop_gradient) + stop_gradient=X_var.stop_gradient, + ) # set allgather_out tensor dist_attr allgather_out_dist_attr = TensorDistributedAttribute() allgather_out_dist_attr.process_mesh = op_dist_attr.process_mesh allgather_out_dist_attr.dims_mapping = [ -1 for i in range(len(allgather_out.shape)) ] - ctx.set_tensor_dist_attr_for_program(allgather_out, - allgather_out_dist_attr) - c_allgather_op = main_block.append_op(type='c_allgather', - inputs={'X': [X_var]}, - outputs={'Out': [allgather_out]}, - attrs={ - 'ring_id': group.id, - 'use_calc_stream': True, - 'nranks': group.nranks, - 'op_role': - src_op.attr('op_role') - }) + ctx.set_tensor_dist_attr_for_program( + allgather_out, allgather_out_dist_attr + ) + c_allgather_op = main_block.append_op( + type='c_allgather', + inputs={'X': [X_var]}, + outputs={'Out': [allgather_out]}, + attrs={ + 'ring_id': group.id, + 'use_calc_stream': True, + 'nranks': group.nranks, + 'op_role': src_op.attr('op_role'), + }, + ) # set c_allgather op dist_attr allgather_op_dist_attr = OperatorDistributedAttribute() allgather_op_dist_attr.process_mesh = op_dist_attr.process_mesh - allgather_op_dist_attr.set_input_dims_mapping(X_var.name, - in_dims_mapping) + allgather_op_dist_attr.set_input_dims_mapping( + X_var.name, in_dims_mapping + ) allgather_op_dist_attr.set_output_dims_mapping( - allgather_out.name, allgather_out_dist_attr.dims_mapping) + allgather_out.name, allgather_out_dist_attr.dims_mapping + ) ctx.set_op_dist_attr_for_program(c_allgather_op, allgather_op_dist_attr) # 3. copy p_norm op desc and reset input name @@ -203,7 +228,8 @@ class DistributedPNormImpl(DistributedOperatorImpl): dist_op_desc.set_output(output_name, kwargs[output_name]) pnorm_op = Operator(main_block, dist_op_desc) op_dist_attr.set_input_dims_mapping( - allgather_out.name, allgather_out_dist_attr.dims_mapping) + allgather_out.name, allgather_out_dist_attr.dims_mapping + ) ctx.set_op_dist_attr_for_program(pnorm_op, op_dist_attr) @staticmethod @@ -219,17 +245,20 @@ class DistributedPNormImpl(DistributedOperatorImpl): # check validation of inputs / outputs for input_name in backward_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( backward_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in backward_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( backward_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) X_grad_var = main_block.var(kwargs['X@GRAD'][0]) @@ -244,7 +273,8 @@ class DistributedPNormImpl(DistributedOperatorImpl): shape=new_X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_grad_var.stop_gradient) + stop_gradient=X_grad_var.stop_gradient, + ) new_kwargs['X@GRAD'] = [new_X_grad.name] new_X_var_dist_attr = ctx.get_tensor_dist_attr_for_program(new_X_var) ctx.set_tensor_dist_attr_for_program(new_X_grad, new_X_var_dist_attr) @@ -258,10 +288,12 @@ class DistributedPNormImpl(DistributedOperatorImpl): for output_name in backward_op.desc.output_names(): dist_op_desc.set_output(output_name, new_kwargs[output_name]) p_norm_grad_op = Operator(main_block, dist_op_desc) - op_dist_attr.set_input_dims_mapping(new_X_var.name, - new_X_var_dist_attr.dims_mapping) - op_dist_attr.set_output_dims_mapping(new_X_grad.name, - new_X_var_dist_attr.dims_mapping) + op_dist_attr.set_input_dims_mapping( + new_X_var.name, new_X_var_dist_attr.dims_mapping + ) + op_dist_attr.set_output_dims_mapping( + new_X_grad.name, new_X_var_dist_attr.dims_mapping + ) ctx.set_op_dist_attr_for_program(p_norm_grad_op, op_dist_attr) # 2. insert slice op @@ -271,8 +303,12 @@ class DistributedPNormImpl(DistributedOperatorImpl): from ..reshard import Resharder partition_idx = Resharder.compute_partition_index( - rank_id, new_X_grad.shape, dims_mapping, process_mesh_shape, - process_mesh_group) + rank_id, + new_X_grad.shape, + dims_mapping, + process_mesh_shape, + process_mesh_group, + ) slice_starts = [] slice_ends = [] slices_axes = [] @@ -287,22 +323,28 @@ class DistributedPNormImpl(DistributedOperatorImpl): "starts": slice_starts, "ends": slice_ends, "infer_flags": infer_flags, - "op_role": backward_op.attr('op_role') + "op_role": backward_op.attr('op_role'), } - slice_op = main_block.append_op(type='slice', - inputs={'Input': [new_X_grad]}, - outputs={'Out': [X_grad_var]}, - attrs=attrs) + slice_op = main_block.append_op( + type='slice', + inputs={'Input': [new_X_grad]}, + outputs={'Out': [X_grad_var]}, + attrs=attrs, + ) X_grad_var_dims_mapping = op_dist_attr.get_output_dims_mapping( - X_grad_var.name) + X_grad_var.name + ) slice_op_dist_attr = OperatorDistributedAttribute() slice_op_dist_attr.process_mesh = op_dist_attr.process_mesh slice_op_dist_attr.set_input_dims_mapping( - new_X_grad.name, new_X_var_dist_attr.dims_mapping) - slice_op_dist_attr.set_output_dims_mapping(X_grad_var.name, - X_grad_var_dims_mapping) + new_X_grad.name, new_X_var_dist_attr.dims_mapping + ) + slice_op_dist_attr.set_output_dims_mapping( + X_grad_var.name, X_grad_var_dims_mapping + ) ctx.set_op_dist_attr_for_program(slice_op, slice_op_dist_attr) -register_distributed_operator_impl("p_norm", - DistributedPNormImpl("row_parallel")) +register_distributed_operator_impl( + "p_norm", DistributedPNormImpl("row_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py b/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py index 77372257f4f..d007aeda423 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py @@ -23,18 +23,17 @@ from ..process_group import new_process_group class DistributedReduceSumPrimtive(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedReduceSumPrimtive, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedReduceSumPrimtive("reduce_sum_p")) + DistributedReduceSumPrimtive("reduce_sum_p") +) # Batch Dimension ReduceSum Primitive class DistributedReduceSumPrimtiveImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedReduceSumPrimtiveImpl0, self).__init__(name) self._forward_implemented = True @@ -56,7 +55,7 @@ class DistributedReduceSumPrimtiveImpl0(DistributedOperatorImpl): output_name = outputs[0] output_var = dist_op.serial_op.block.var(output_name) - if output_var.shape != (1, ): + if output_var.shape != (1,): return False return True @@ -66,7 +65,8 @@ class DistributedReduceSumPrimtiveImpl0(DistributedOperatorImpl): op_dist_attr = dist_op.dist_attr return self.is_input_compatible(dist_op) and self.is_output_compatible( - dist_op) + dist_op + ) def update_dims_mapping(self, dist_op): changed = False @@ -85,17 +85,20 @@ class DistributedReduceSumPrimtiveImpl0(DistributedOperatorImpl): # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) # replicate op in dist program dist_op_desc = main_block.append_op(type='nop').desc @@ -109,14 +112,16 @@ class DistributedReduceSumPrimtiveImpl0(DistributedOperatorImpl): # batch dimension synchronization var_name = src_op.output_arg_names[0] sync_group = new_process_group(ctx.data_parallel_group) - allreduce_op = main_block.append_op(type='c_allreduce_sum', - inputs={'X': [var_name]}, - outputs={'Out': [var_name]}, - attrs={ - 'ring_id': sync_group.id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + allreduce_op = main_block.append_op( + type='c_allreduce_sum', + inputs={'X': [var_name]}, + outputs={'Out': [var_name]}, + attrs={ + 'ring_id': sync_group.id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) # dist attr var = main_block.var(var_name) @@ -124,19 +129,24 @@ class DistributedReduceSumPrimtiveImpl0(DistributedOperatorImpl): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) new_op_attr = OperatorDistributedAttribute() new_op_attr.process_mesh = op_dist_attr.process_mesh - new_op_attr.set_output_dims_mapping(var.name, - tensor_dist_attr.dims_mapping) - new_op_attr.set_input_dims_mapping(var.name, - tensor_dist_attr.dims_mapping) + new_op_attr.set_output_dims_mapping( + var.name, tensor_dist_attr.dims_mapping + ) + new_op_attr.set_input_dims_mapping( + var.name, tensor_dist_attr.dims_mapping + ) ctx.set_op_dist_attr_for_program(allreduce_op, new_op_attr) @staticmethod def backward(ctx, *args, **kwargs): raise RuntimeError( - "primitive operator does NOT have backward function, op type: {}". - format(str(op.type))) + "primitive operator does NOT have backward function, op type: {}".format( + str(op.type) + ) + ) register_distributed_operator_impl( "reduce_sum_p", - DistributedReduceSumPrimtiveImpl0("batch_dimension_reduce_sum_p")) + DistributedReduceSumPrimtiveImpl0("batch_dimension_reduce_sum_p"), +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_reshape.py b/python/paddle/distributed/auto_parallel/operators/dist_reshape.py index e9f32b80ca1..67b32dbecae 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_reshape.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_reshape.py @@ -27,7 +27,6 @@ from paddle.distributed.fleet.meta_optimizers.common import OpRole class DistributedReshape2(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedReshape2, self).__init__(op_type) @@ -36,7 +35,6 @@ register_distributed_operator_impl_container(DistributedReshape2("reshape2")) class DistributedReshapeImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedReshapeImpl0, self).__init__(name) self._forward_implemented = True @@ -66,19 +64,21 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_attr.process_mesh.processes for key in desc_mapping: desc_mapping[key]["shape"] = shape_list - cost_mapping = build_comp_costs_from_descs(Reshape2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2OpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) return res @@ -86,16 +86,17 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Reshape2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -105,7 +106,8 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -115,8 +117,15 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @@ -150,8 +159,9 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -160,7 +170,8 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): out_name = op_desc.output('Out')[0] x_shape_name = op_desc.output('XShape')[0] x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) @@ -186,11 +197,13 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -210,23 +223,27 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Out_var = main_block.var(kwargs['Out'][0]) @@ -247,8 +264,9 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # create op new_op_desc = main_block.append_op(type='nop').desc @@ -267,7 +285,6 @@ class DistributedReshapeImpl0(DistributedOperatorImpl): class DistributedReshapeImpl1(DistributedOperatorImpl): - def __init__(self, name): super(DistributedReshapeImpl1, self).__init__(name) self._forward_implemented = True @@ -297,19 +314,21 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_attr.process_mesh.processes for key in desc_mapping: desc_mapping[key]["shape"] = shape_list - cost_mapping = build_comp_costs_from_descs(Reshape2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2OpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) return res @@ -317,16 +336,17 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Reshape2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -336,7 +356,8 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -346,8 +367,15 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @@ -381,8 +409,9 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -393,7 +422,8 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) if is_dim_shard(x_dims_mapping[-1]): return False @@ -420,11 +450,13 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) for i in range(len(out_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -444,23 +476,27 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Out_var = main_block.var(kwargs['Out'][0]) @@ -481,8 +517,9 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # create op new_op_desc = main_block.append_op(type='nop').desc @@ -501,7 +538,6 @@ class DistributedReshapeImpl1(DistributedOperatorImpl): class DistributedReshapeImpl2(DistributedOperatorImpl): - def __init__(self, name): super(DistributedReshapeImpl2, self).__init__(name) self._forward_implemented = True @@ -531,19 +567,21 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_attr.process_mesh.processes for key in desc_mapping: desc_mapping[key]["shape"] = shape_list - cost_mapping = build_comp_costs_from_descs(Reshape2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2OpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) return res @@ -551,16 +589,17 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Reshape2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -570,7 +609,8 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -580,8 +620,15 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @@ -612,8 +659,9 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -624,7 +672,8 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) for idx, item in enumerate(x_dims_mapping[:-1]): if out_dims_mapping[idx] != item: @@ -648,11 +697,13 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) for i in range(len(out_dims_mapping) - 1): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -671,23 +722,27 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): main_block = dist_op_context.work_block src_op = dist_op_context.cur_src_op op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Out_var = main_block.var(kwargs['Out'][0]) @@ -708,8 +763,9 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): for idx, axis in enumerate(out_dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # create op new_op_desc = main_block.append_op(type='nop').desc @@ -727,9 +783,12 @@ class DistributedReshapeImpl2(DistributedOperatorImpl): DistributedDefaultImpl0.backward(ctx, *args, **kwargs) -register_distributed_operator_impl("reshape2", - DistributedReshapeImpl0("add_one_dim_back")) register_distributed_operator_impl( - "reshape2", DistributedReshapeImpl1("remove_one_dim_back")) -register_distributed_operator_impl("reshape2", - DistributedReshapeImpl2("same_dim_shape")) + "reshape2", DistributedReshapeImpl0("add_one_dim_back") +) +register_distributed_operator_impl( + "reshape2", DistributedReshapeImpl1("remove_one_dim_back") +) +register_distributed_operator_impl( + "reshape2", DistributedReshapeImpl2("same_dim_shape") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_shape.py b/python/paddle/distributed/auto_parallel/operators/dist_shape.py index 313f296ab96..47254cd9985 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_shape.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_shape.py @@ -21,7 +21,6 @@ from ..utils import is_dim_shard class DistributedShape(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedShape, self).__init__(op_type) @@ -30,7 +29,6 @@ register_distributed_operator_impl_container(DistributedShape("shape")) class DistributedShapeImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedShapeImpl, self).__init__(name) self._forward_implemented = True @@ -52,8 +50,9 @@ class DistributedShapeImpl(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False return True diff --git a/python/paddle/distributed/auto_parallel/operators/dist_slice.py b/python/paddle/distributed/auto_parallel/operators/dist_slice.py index 1c9b0e482a1..82562c8b075 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_slice.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_slice.py @@ -22,7 +22,6 @@ from .dist_default import DistributedDefaultImpl0 class DistributedSlice(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedSlice, self).__init__(op_type) @@ -31,7 +30,6 @@ register_distributed_operator_impl_container(DistributedSlice("slice")) class DistributedSliceImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedSliceImpl, self).__init__(name) self._forward_implemented = True @@ -75,8 +73,9 @@ class DistributedSliceImpl(DistributedOperatorImpl): return True def is_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -87,7 +86,8 @@ class DistributedSliceImpl(DistributedOperatorImpl): in_dims_mapping = op_dist_attr.get_input_dims_mapping(in_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) if len(in_dims_mapping) - len(decrease_axis) != 0 and len( - out_dims_mapping) != len(in_dims_mapping) - len(decrease_axis): + out_dims_mapping + ) != len(in_dims_mapping) - len(decrease_axis): return False new_out_dims_mapping = [] @@ -102,9 +102,11 @@ class DistributedSliceImpl(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)) or \ - (not self.is_compatible(dist_op)): + if ( + (not self.is_input_compatible(dist_op)) + or (not self.is_output_compatible(dist_op)) + or (not self.is_compatible(dist_op)) + ): return False return True @@ -135,7 +137,8 @@ class DistributedSliceImpl(DistributedOperatorImpl): assert len(ref_dims_mapping) == len(out_dims_mapping) for i in range(len(out_dims_mapping)): compatible_dim_mapping = compute_compatible_dim_mapping( - [out_dims_mapping[i], ref_dims_mapping[i]]) + [out_dims_mapping[i], ref_dims_mapping[i]] + ) if compatible_dim_mapping is None: continue if ref_dims_mapping[i] != compatible_dim_mapping: @@ -156,5 +159,6 @@ class DistributedSliceImpl(DistributedOperatorImpl): DistributedDefaultImpl0.backward(ctx, *args, **kwargs) -register_distributed_operator_impl("slice", - DistributedSliceImpl("decrease_in_axis")) +register_distributed_operator_impl( + "slice", DistributedSliceImpl("decrease_in_axis") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_softmax.py b/python/paddle/distributed/auto_parallel/operators/dist_softmax.py index 8d85c2c19fc..6a980f72463 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_softmax.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_softmax.py @@ -27,7 +27,6 @@ from paddle.distributed.fleet.meta_optimizers.common import OpRole class DistributedSoftmax(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedSoftmax, self).__init__(op_type) @@ -36,7 +35,6 @@ register_distributed_operator_impl_container(DistributedSoftmax("softmax")) class DistributedSoftmaxImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedSoftmaxImpl, self).__init__(name) self._forward_implemented = False @@ -53,12 +51,13 @@ class DistributedSoftmaxImpl(DistributedOperatorImpl): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(SoftmaxOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + SoftmaxOpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -66,14 +65,15 @@ class DistributedSoftmaxImpl(DistributedOperatorImpl): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(SoftmaxGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + SoftmaxGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -83,7 +83,8 @@ class DistributedSoftmaxImpl(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -93,8 +94,15 @@ class DistributedSoftmaxImpl(DistributedOperatorImpl): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @@ -129,8 +137,9 @@ class DistributedSoftmaxImpl(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -159,7 +168,8 @@ class DistributedSoftmaxImpl(DistributedOperatorImpl): for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -175,4 +185,5 @@ class DistributedSoftmaxImpl(DistributedOperatorImpl): register_distributed_operator_impl( - "softmax", DistributedSoftmaxImpl("replicate_last_axis")) + "softmax", DistributedSoftmaxImpl("replicate_last_axis") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_split.py b/python/paddle/distributed/auto_parallel/operators/dist_split.py index cf02e00a337..8411d6d83dd 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_split.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_split.py @@ -22,7 +22,6 @@ from .dist_default import DistributedDefaultImpl0 class DistributedSplit(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedSplit, self).__init__(op_type) @@ -31,7 +30,6 @@ register_distributed_operator_impl_container(DistributedSplit("split")) class DistributedSplitImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedSplitImpl, self).__init__(name) self._forward_implemented = True @@ -62,8 +60,9 @@ class DistributedSplitImpl(DistributedOperatorImpl): return True def is_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -91,16 +90,19 @@ class DistributedSplitImpl(DistributedOperatorImpl): out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True return changed def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)) or \ - (not self.is_compatible(dist_op)): + if ( + (not self.is_input_compatible(dist_op)) + or (not self.is_output_compatible(dist_op)) + or (not self.is_compatible(dist_op)) + ): return False return True @@ -114,5 +116,6 @@ class DistributedSplitImpl(DistributedOperatorImpl): DistributedDefaultImpl0.backward(ctx, *args, **kwargs) -register_distributed_operator_impl("split", - DistributedSplitImpl("replicate_in_axis")) +register_distributed_operator_impl( + "split", DistributedSplitImpl("replicate_in_axis") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_transpose.py b/python/paddle/distributed/auto_parallel/operators/dist_transpose.py index 96220bf0d50..6bacb8bace4 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_transpose.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_transpose.py @@ -26,17 +26,16 @@ from paddle.distributed.fleet.meta_optimizers.common import OpRole class DistributedTranspose2(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedTranspose2, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedTranspose2("transpose2")) + DistributedTranspose2("transpose2") +) class DistributedTranspose2Impl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedTranspose2Impl, self).__init__(name) self._forward_implemented = False @@ -49,8 +48,9 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -60,7 +60,8 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): out_name = op_desc.output('Out')[0] x_shape_name = op_desc.output('XShape')[0] x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) new_dims_mapping = [-1 for i in range(len(x_dims_mapping))] @@ -91,7 +92,8 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) perm = op_desc.attr('axis') assert len(x_dims_mapping) == len(perm) @@ -102,7 +104,8 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): for i in range(len(out_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [new_dims_mapping, out_dims_mapping], [i, i]) + [new_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -127,13 +130,14 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Transpose2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Transpose2OpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -141,15 +145,16 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Transpose2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Transpose2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -159,7 +164,8 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -169,8 +175,15 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @staticmethod @@ -183,4 +196,5 @@ class DistributedTranspose2Impl(DistributedOperatorImpl): register_distributed_operator_impl( - "transpose2", DistributedTranspose2Impl("same_mapping_transpose")) + "transpose2", DistributedTranspose2Impl("same_mapping_transpose") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_update_loss_scaling.py b/python/paddle/distributed/auto_parallel/operators/dist_update_loss_scaling.py index cbbcaef5ee4..38947366079 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_update_loss_scaling.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_update_loss_scaling.py @@ -20,17 +20,16 @@ from ..utils import set_dist_op_desc_original_id class DistributedUpdateLossScaling(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedUpdateLossScaling, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedUpdateLossScaling("update_loss_scaling")) + DistributedUpdateLossScaling("update_loss_scaling") +) class DistributedUpdateLossScalingImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedUpdateLossScalingImpl, self).__init__(name) self._name = name @@ -60,7 +59,8 @@ class DistributedUpdateLossScalingImpl(DistributedOperatorImpl): @staticmethod def forward(ctx, *args, **kwargs): raise RuntimeError( - "DistributedUpdateLossScalingImpl's forward should not be called !") + "DistributedUpdateLossScalingImpl's forward should not be called !" + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -71,59 +71,89 @@ class DistributedUpdateLossScalingImpl(DistributedOperatorImpl): backward_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format( + str(backward_op) + ) assert rank_id in dist_attr.process_mesh.processes assert 'X' in kwargs, "input [{}] is not given".format('X') assert 'FoundInfinite' in kwargs, "input [{}] is not given".format( - 'FoundInfinite') + 'FoundInfinite' + ) assert 'PrevLossScaling' in kwargs, "input [{}] is not given".format( - 'PrevLossScaling') + 'PrevLossScaling' + ) assert 'InGoodSteps' in kwargs, "input [{}] is not given".format( - 'InGoodSteps') + 'InGoodSteps' + ) assert 'InBadSteps' in kwargs, "input [{}] is not given".format( - 'InBadSteps') + 'InBadSteps' + ) assert 'Out' in kwargs, "output [{}] is not given".format('Out') assert 'LossScaling' in kwargs, "output [{}] is not given".format( - 'LossScaling') + 'LossScaling' + ) assert 'OutGoodSteps' in kwargs, "output [{}] is not given".format( - 'OutGoodSteps') + 'OutGoodSteps' + ) assert 'OutBadSteps' in kwargs, "output [{}] is not given".format( - 'OutBadSteps') - - assert len(kwargs['FoundInfinite']) == 1, \ - "update_loss_scaling input FoundInfinite take 1 variable but got {}".format( - kwargs['FoundInfinite']) - assert len(kwargs['PrevLossScaling']) == 1, \ - "update_loss_scaling input PrevLossScaling take 1 variable but got {}".format( - kwargs['PrevLossScaling']) - assert len(kwargs['InGoodSteps']) == 1, \ - "update_loss_scaling input InGoodSteps take 1 variable but got {}".format( - kwargs['InGoodSteps']) - assert len(kwargs['InBadSteps']) == 1, \ - "update_loss_scaling input InBadSteps take 1 variable but got {}".format( - kwargs['InBadSteps']) - assert len(kwargs['LossScaling']) == 1, \ - "update_loss_scaling output LossScaling take 1 variable but got {}".format( - kwargs['LossScaling']) - assert len(kwargs['OutGoodSteps']) == 1, \ - "update_loss_scaling output OutGoodSteps take 1 variable but got {}".format( - kwargs['OutGoodSteps']) - assert len(kwargs['OutBadSteps']) == 1, \ - "update_loss_scaling output OutBadSteps take 1 variable but got {}".format( - kwargs['OutBadSteps']) - - assert len(kwargs['X']) == len(kwargs['Out']), \ - "update_loss_scaling got [{}] X and [{}] Out, which are supposed to be equal".format( - len(kwargs['X']), len(kwargs['Out'])) + 'OutBadSteps' + ) + + assert ( + len(kwargs['FoundInfinite']) == 1 + ), "update_loss_scaling input FoundInfinite take 1 variable but got {}".format( + kwargs['FoundInfinite'] + ) + assert ( + len(kwargs['PrevLossScaling']) == 1 + ), "update_loss_scaling input PrevLossScaling take 1 variable but got {}".format( + kwargs['PrevLossScaling'] + ) + assert ( + len(kwargs['InGoodSteps']) == 1 + ), "update_loss_scaling input InGoodSteps take 1 variable but got {}".format( + kwargs['InGoodSteps'] + ) + assert ( + len(kwargs['InBadSteps']) == 1 + ), "update_loss_scaling input InBadSteps take 1 variable but got {}".format( + kwargs['InBadSteps'] + ) + assert ( + len(kwargs['LossScaling']) == 1 + ), "update_loss_scaling output LossScaling take 1 variable but got {}".format( + kwargs['LossScaling'] + ) + assert ( + len(kwargs['OutGoodSteps']) == 1 + ), "update_loss_scaling output OutGoodSteps take 1 variable but got {}".format( + kwargs['OutGoodSteps'] + ) + assert ( + len(kwargs['OutBadSteps']) == 1 + ), "update_loss_scaling output OutBadSteps take 1 variable but got {}".format( + kwargs['OutBadSteps'] + ) + + assert len(kwargs['X']) == len( + kwargs['Out'] + ), "update_loss_scaling got [{}] X and [{}] Out, which are supposed to be equal".format( + len(kwargs['X']), len(kwargs['Out']) + ) filter_vars = [] for varname in kwargs['X']: - if rank_id in ctx.get_tensor_dist_attr_for_program( - main_block.var(varname)).process_mesh.processes: + if ( + rank_id + in ctx.get_tensor_dist_attr_for_program( + main_block.var(varname) + ).process_mesh.processes + ): filter_vars.append(varname) # replicate op in dist program @@ -136,4 +166,5 @@ class DistributedUpdateLossScalingImpl(DistributedOperatorImpl): register_distributed_operator_impl( "update_loss_scaling", - DistributedUpdateLossScalingImpl("update_loss_scaling")) + DistributedUpdateLossScalingImpl("update_loss_scaling"), +) diff --git a/python/paddle/distributed/auto_parallel/parallelizer.py b/python/paddle/distributed/auto_parallel/parallelizer.py index b25d1d8b33b..aad15642f1b 100644 --- a/python/paddle/distributed/auto_parallel/parallelizer.py +++ b/python/paddle/distributed/auto_parallel/parallelizer.py @@ -79,8 +79,12 @@ class AutoParallelizer: self._pass_context = PassContext() self._need_rank_mapping = os.getenv("PADDLE_NEED_RANK_MAPPING") - self._need_rank_mapping = True if self._need_rank_mapping and \ - self._need_rank_mapping.lower() == 'true' else False + self._need_rank_mapping = ( + True + if self._need_rank_mapping + and self._need_rank_mapping.lower() == 'true' + else False + ) # self._pass_context = None def _remove_distributed_attrs(self, main_program): @@ -93,8 +97,9 @@ class AutoParallelizer: if suffix in attr_name: op._remove_attr(attr_name) - def _apply_pre_optimization_passes(self, main_program, startup_program, - loss, params_grads, no_grad_set): + def _apply_pre_optimization_passes( + self, main_program, startup_program, loss, params_grads, no_grad_set + ): # apply amp pass if self._dist_strategy.amp: config = copy.deepcopy(self._dist_strategy.amp_configs) @@ -104,13 +109,15 @@ class AutoParallelizer: if config["use_pure_fp16"]: config["base_opt"] = self._optimizer auto_parallel_fp16_pass = new_pass("auto_parallel_fp16", config) - auto_parallel_fp16_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_fp16_pass.apply( + [main_program], [startup_program], self._pass_context + ) loss = auto_parallel_fp16_pass.get_loss() else: auto_parallel_amp_pass = new_pass("auto_parallel_amp", config) - auto_parallel_amp_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_amp_pass.apply( + [main_program], [startup_program], self._pass_context + ) loss = auto_parallel_amp_pass.get_loss() # apply recompute pass @@ -119,14 +126,22 @@ class AutoParallelizer: config["dist_context"] = self._dist_context config["no_grad_set"] = copy.deepcopy(no_grad_set) config["loss"] = loss - auto_parallel_recompute_pass = new_pass("auto_parallel_recompute", - config) - auto_parallel_recompute_pass.apply([main_program], - [startup_program], - self._pass_context) - - def _generate_backward(self, main_program, startup_program, loss, - parameter_list, no_grad_set, callbacks): + auto_parallel_recompute_pass = new_pass( + "auto_parallel_recompute", config + ) + auto_parallel_recompute_pass.apply( + [main_program], [startup_program], self._pass_context + ) + + def _generate_backward( + self, + main_program, + startup_program, + loss, + parameter_list, + no_grad_set, + callbacks, + ): with program_guard(main_program, startup_program): params_grads = append_backward( @@ -134,7 +149,8 @@ class AutoParallelizer: parameter_list, no_grad_set, callbacks, - distop_context=self._dist_context.dist_op_context) + distop_context=self._dist_context.dist_op_context, + ) self._completer = Completer(self._dist_context) self._completer.complete_backward_annotation(main_program) self._dist_context.block_state.parse_backward_blocks(main_program) @@ -153,18 +169,21 @@ class AutoParallelizer: return optimize_ops - def _apply_post_optimization_passes(self, main_program, startup_program, - rank, params_grads): + def _apply_post_optimization_passes( + self, main_program, startup_program, rank, params_grads + ): if self._dist_strategy.sharding: config = copy.deepcopy(self._dist_strategy.sharding_configs) config["dist_context"] = self._dist_context config["params_grads"] = params_grads config["global_rank"] = rank - auto_parallel_sharding_pass = new_pass("auto_parallel_sharding", - config) - auto_parallel_sharding_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_sharding_pass = new_pass( + "auto_parallel_sharding", config + ) + auto_parallel_sharding_pass.apply( + [main_program], [startup_program], self._pass_context + ) params_grads = self._pass_context.get_attr("params_grads") config = copy.deepcopy(self._dist_strategy.sharding_configs) @@ -172,18 +191,20 @@ class AutoParallelizer: config["params_grads"] = params_grads config["rank_id"] = rank auto_parallel_clip_pass = new_pass("auto_parallel_grad_clip", config) - auto_parallel_clip_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_clip_pass.apply( + [main_program], [startup_program], self._pass_context + ) if self._dist_strategy.gradient_merge: config = copy.deepcopy(self._dist_strategy.gradient_merge_configs) config["dist_context"] = self._dist_context config["params_grads"] = params_grads auto_parallel_gradient_merge_pass = new_pass( - "auto_parallel_gradient_merge_pass", config) - auto_parallel_gradient_merge_pass.apply([main_program], - [startup_program], - self._pass_context) + "auto_parallel_gradient_merge_pass", config + ) + auto_parallel_gradient_merge_pass.apply( + [main_program], [startup_program], self._pass_context + ) def _get_dist_program(self, rank, dist_context=None, relaunch_phase=False): completed_main_program = None @@ -197,8 +218,9 @@ class AutoParallelizer: self._dist_context = DistributedContext() _logger.info("Start annotation dist attr.") self._completer = Completer(self._dist_context) - completed_main_program = self._completer.complete_forward_annotation( - serial_main_program) + completed_main_program = ( + self._completer.complete_forward_annotation(serial_main_program) + ) else: completed_main_program = serial_main_program self._dist_context = copy.deepcopy(dist_context) @@ -208,34 +230,54 @@ class AutoParallelizer: # serial backward pass params_grads = self._generate_backward( - completed_main_program, serial_startup_program, serial_loss, - self._parameter_list, self._no_grad_set, self._callbacks) + completed_main_program, + serial_startup_program, + serial_loss, + self._parameter_list, + self._no_grad_set, + self._callbacks, + ) # serial forward pass - self._apply_pre_optimization_passes(completed_main_program, - serial_startup_program, serial_loss, - params_grads, self._no_grad_set) + self._apply_pre_optimization_passes( + completed_main_program, + serial_startup_program, + serial_loss, + params_grads, + self._no_grad_set, + ) # Logical partition partitioner = Partitioner(self._dist_context, rank) - dist_main_prog, dist_startup_prog, dist_params_grads = partitioner.partition( - completed_main_program, serial_startup_program, params_grads) + ( + dist_main_prog, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + completed_main_program, serial_startup_program, params_grads + ) # TODO refactor the placement of optimizer # generate optimize program - dist_optimize_ops = self._apply_optimize(dist_main_prog, - dist_startup_prog, - dist_params_grads) + dist_optimize_ops = self._apply_optimize( + dist_main_prog, dist_startup_prog, dist_params_grads + ) set_grad_var_shape(dist_main_prog, self._dist_context) make_data_unshard(dist_main_prog, dist_startup_prog, self._dist_context) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank, - self._dist_context, dist_params_grads) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank, + self._dist_context, + dist_params_grads, + ) resharder.reshard() - self._apply_post_optimization_passes(dist_main_prog, dist_startup_prog, - rank, dist_params_grads) + self._apply_post_optimization_passes( + dist_main_prog, dist_startup_prog, rank, dist_params_grads + ) g_process_group_map = None if not relaunch_phase: g_process_group_map = copy.deepcopy(_g_process_group_map) @@ -243,14 +285,22 @@ class AutoParallelizer: _g_process_group_map[0] = ProcessGroup(0, []) for process_mesh in self._dist_context._process_meshes: _g_process_group_map[0].add_ranks(process_mesh.processes) - return dist_optimize_ops, dist_params_grads, dist_startup_prog, dist_main_prog, g_process_group_map - - def parallelize(self, - loss, - startup_program, - parameter_list=None, - no_grad_set=None, - callbacks=None): + return ( + dist_optimize_ops, + dist_params_grads, + dist_startup_prog, + dist_main_prog, + g_process_group_map, + ) + + def parallelize( + self, + loss, + startup_program, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): assert startup_program is not None self._loss = loss self._startup_program = startup_program @@ -261,25 +311,27 @@ class AutoParallelizer: if self._enable_auto_mapping and self._need_rank_mapping: # Do the mapping pass before parallelization - assert self._cluster is not None, \ - "The cluster must not be none when using auto mapping." + assert ( + self._cluster is not None + ), "The cluster must not be none when using auto mapping." dist_programs = {} world_process_group = get_world_process_group() dist_context = None # auto search if self._dist_strategy.auto_search: logging.info("Start searching dist attr.") - serial_program_info = SerialProgramInfo(self._main_program, - self._startup_program, - self._loss, - self._optimizer, - self._cluster) - planner = Planner(serial_program_info, - self, - algorithm_config={ - "name": "mcmc", - "max_search_times": 5 - }) + serial_program_info = SerialProgramInfo( + self._main_program, + self._startup_program, + self._loss, + self._optimizer, + self._cluster, + ) + planner = Planner( + serial_program_info, + self, + algorithm_config={"name": "mcmc", "max_search_times": 5}, + ) dist_context, _ = planner.search() logging.info("End searching dist attr.") @@ -288,31 +340,42 @@ class AutoParallelizer: logging.info("Start serialize searched dist attr") cwd = pathlib.Path().resolve() searched_dist_context_path = os.path.join( - cwd, f"searched_dist_context_{time.time()}.pkl") + cwd, f"searched_dist_context_{time.time()}.pkl" + ) saved_dist_context = {} ops_dist_attr = {} tensors_dist_attr = {} for key, dist_op in dist_context._dist_ops_for_program.items(): ops_dist_attr[key] = dist_op.dist_attr - for key, dist_tensor in dist_context._dist_tensors_for_program.items( - ): + for ( + key, + dist_tensor, + ) in dist_context._dist_tensors_for_program.items(): tensors_dist_attr[key] = dist_tensor.dist_attr saved_dist_context["ops_dist_attr"] = ops_dist_attr saved_dist_context["tensors_dist_attr"] = tensors_dist_attr saved_dist_context[ - "process_meshes"] = dist_context._process_meshes - with open(searched_dist_context_path, - "wb") as dist_context_file: + "process_meshes" + ] = dist_context._process_meshes + with open( + searched_dist_context_path, "wb" + ) as dist_context_file: pickle.dump(saved_dist_context, dist_context_file) os.environ[ - 'PADDLE_SEARCHED_DIST_CONTEXT_PATH'] = searched_dist_context_path + 'PADDLE_SEARCHED_DIST_CONTEXT_PATH' + ] = searched_dist_context_path logging.info( f"End serialize searched dist attr to {searched_dist_context_path}" ) for rank in world_process_group.ranks: - dist_optimize_ops, dist_params_grads, dist_startup_prog, dist_main_prog, g_process_group_map = self._get_dist_program( - rank, dist_context) + ( + dist_optimize_ops, + dist_params_grads, + dist_startup_prog, + dist_main_prog, + g_process_group_map, + ) = self._get_dist_program(rank, dist_context) dist_programs[rank] = [dist_main_prog, g_process_group_map] # Do the mapping between the distributed program graph and the cluster graph @@ -324,27 +387,42 @@ class AutoParallelizer: json.dump(rank_mapping, rank_mapping_file) enable_elastic = os.getenv("PADDLE_ENABLE_ELASTIC") - enable_elastic = True if enable_elastic and enable_elastic.lower( - ) == 'true' else False + enable_elastic = ( + True + if enable_elastic and enable_elastic.lower() == 'true' + else False + ) if enable_elastic: print("Auto mapping finished, now do elastic re-launch") - sys.exit(paddle.distributed.fleet.elastic.manager. - ELASTIC_AUTO_PARALLEL_EXIT_CODE) + sys.exit( + paddle.distributed.fleet.elastic.manager.ELASTIC_AUTO_PARALLEL_EXIT_CODE + ) original_cmd_args = os.getenv("PADDLE_ORIGINAL_CMD_ARGS") rank_mapping_args = " ".join( - ["--rank_mapping_path", self._rank_mapping_path]) + ["--rank_mapping_path", self._rank_mapping_path] + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] else: coverage_args = [] - new_cmd_args = "-m paddle.distributed.fleet.launch" + " " + rank_mapping_args + " " + original_cmd_args - new_cmd = [sys.executable, "-u" - ] + coverage_args + shlex.split(new_cmd_args) + new_cmd_args = ( + "-m paddle.distributed.fleet.launch" + + " " + + rank_mapping_args + + " " + + original_cmd_args + ) + new_cmd = ( + [sys.executable, "-u"] + + coverage_args + + shlex.split(new_cmd_args) + ) new_process = subprocess.Popen(new_cmd) new_process.wait() - assert new_process.returncode == 0, \ - "Launch failed with rank mapping" + assert ( + new_process.returncode == 0 + ), "Launch failed with rank mapping" print("Successfully do the second launch for auto mapping!") sys.exit(0) else: @@ -352,27 +430,32 @@ class AutoParallelizer: rank = paddle.distributed.get_rank() dist_context = None searched_dist_context_path = os.getenv( - "PADDLE_SEARCHED_DIST_CONTEXT_PATH", None) + "PADDLE_SEARCHED_DIST_CONTEXT_PATH", None + ) if searched_dist_context_path is not None: - with open(searched_dist_context_path, - "rb") as dist_context_file: + with open( + searched_dist_context_path, "rb" + ) as dist_context_file: saved_dist_context = pickle.load(dist_context_file) dist_context = DistributedContext() for op in self._main_program.global_block().ops: dist_attr = saved_dist_context["ops_dist_attr"][ - op.desc.id()] + op.desc.id() + ] dist_op = DistributedOperator(op, dist_attr) dist_context.add_dist_op_for_program(dist_op) vars = self._main_program.global_block().vars for var in vars.values(): dist_attr = saved_dist_context["tensors_dist_attr"][ - var.desc.id()] + var.desc.id() + ] dist_tensor = DistributedTensor(var, dist_attr) dist_context.add_dist_tensor_for_program(dist_tensor) dist_context._process_meshes = saved_dist_context[ - "process_meshes"] + "process_meshes" + ] else: if self._dist_strategy.auto_search: @@ -381,13 +464,16 @@ class AutoParallelizer: self._startup_program, self._loss, self._optimizer, - cluster=self._cluster) - planner = Planner(serial_program_info, - self, - algorithm_config={ - "name": "mcmc", - "max_search_times": 5 - }) + cluster=self._cluster, + ) + planner = Planner( + serial_program_info, + self, + algorithm_config={ + "name": "mcmc", + "max_search_times": 5, + }, + ) dist_context, _ = planner.search() # rebuild g_process_group @@ -395,8 +481,13 @@ class AutoParallelizer: pg0 = get_process_group(0) for process_mesh in dist_context._process_meshes: pg0.add_ranks(process_mesh.processes) - dist_optimize_ops, dist_params_grads, dist_startup_prog, dist_main_prog, _ = self._get_dist_program( - rank, dist_context, relaunch_phase=True) + ( + dist_optimize_ops, + dist_params_grads, + dist_startup_prog, + dist_main_prog, + _, + ) = self._get_dist_program(rank, dist_context, relaunch_phase=True) # NOTE: This is a trick to fix hang in pipeline mode when dist context is searched by planner if self._dist_strategy.auto_search: @@ -424,14 +515,25 @@ class AutoParallelizer: # with inference. self._remove_distributed_attrs(dist_main_prog) - return dist_optimize_ops, dist_params_grads, dist_startup_prog, dist_main_prog + return ( + dist_optimize_ops, + dist_params_grads, + dist_startup_prog, + dist_main_prog, + ) def __deepcopy__(self, memo): cls = self.__class__ result = cls.__new__(cls) memo[id(self)] = result for k, v in self.__dict__.items(): - if k == "_main_program" or k == "_startup_program" or k == "_dist_context" or k == "_fleet" or k == "_loss": + if ( + k == "_main_program" + or k == "_startup_program" + or k == "_dist_context" + or k == "_fleet" + or k == "_loss" + ): setattr(result, k, v) else: setattr(result, k, copy.deepcopy(v, memo)) diff --git a/python/paddle/distributed/auto_parallel/parallelizer_v2.py b/python/paddle/distributed/auto_parallel/parallelizer_v2.py index d6bb5efc8eb..7a7ee603e9c 100644 --- a/python/paddle/distributed/auto_parallel/parallelizer_v2.py +++ b/python/paddle/distributed/auto_parallel/parallelizer_v2.py @@ -29,7 +29,6 @@ from ..utils.log_utils import get_logger class Parallelizer: - def __init__(self, mode, completer, dist_context): self._mode = mode self._completer = completer @@ -54,73 +53,123 @@ class Parallelizer: if self._mode == "train" and serial_optimizer: # Generate backward serial_loss = self._dist_context.serial_loss - params_grads = self._generate_backward(serial_main_program, - serial_startup_program, - serial_loss) + params_grads = self._generate_backward( + serial_main_program, serial_startup_program, serial_loss + ) # Apply pre optimization passes time0 = time.time() - serial_main_program, serial_startup_program, params_grads = self._apply_pre_optimization( - serial_main_program, serial_startup_program, serial_loss, - serial_optimizer, params_grads) + ( + serial_main_program, + serial_startup_program, + params_grads, + ) = self._apply_pre_optimization( + serial_main_program, + serial_startup_program, + serial_loss, + serial_optimizer, + params_grads, + ) self._logger.debug( - "within parallel apply_pre_optimization time: {}, mode {}". - format(time.time() - time0, self._mode)) + "within parallel apply_pre_optimization time: {}, mode {}".format( + time.time() - time0, self._mode + ) + ) # Do logical partition time0 = time.time() partitioner = Partitioner(self._dist_context, rank) - dist_main_prog, dist_startup_prog, dist_params_grads = partitioner.partition( - serial_main_program, serial_startup_program, params_grads) + ( + dist_main_prog, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + serial_main_program, serial_startup_program, params_grads + ) self._logger.debug( "within parallel partitioner time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) # Generate optimizer time0 = time.time() - self._generate_optimizer(dist_main_prog, dist_startup_prog, - serial_optimizer, dist_params_grads) + self._generate_optimizer( + dist_main_prog, + dist_startup_prog, + serial_optimizer, + dist_params_grads, + ) self._logger.debug( "within parallel optimizer time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) # Do reshard process time0 = time.time() set_grad_var_shape(dist_main_prog, self._dist_context) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank, - self._dist_context, dist_params_grads) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank, + self._dist_context, + dist_params_grads, + ) resharder.reshard() self._logger.debug( "within parallel reshard time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) # Apply post optimization passes time0 = time.time() - self._apply_post_optimization(dist_main_prog, dist_startup_prog, - rank, dist_params_grads) + self._apply_post_optimization( + dist_main_prog, dist_startup_prog, rank, dist_params_grads + ) self._logger.debug( - "within parallel apply_post_optimization time: {}, mode {}". - format(time.time() - time0, self._mode)) + "within parallel apply_post_optimization time: {}, mode {}".format( + time.time() - time0, self._mode + ) + ) else: # Apply pre optimization passes time0 = time.time() - self._apply_pre_optimization(serial_main_program, - serial_startup_program, None, None, - None) + self._apply_pre_optimization( + serial_main_program, serial_startup_program, None, None, None + ) self._logger.debug( - "within parallel apply_pre_optimization time: {}, mode {}". - format(time.time() - time0, self._mode)) + "within parallel apply_pre_optimization time: {}, mode {}".format( + time.time() - time0, self._mode + ) + ) # Do logical partition time0 = time.time() partitioner = Partitioner(self._dist_context, rank) - dist_main_prog, dist_startup_prog, dist_params_grads = partitioner.partition( - serial_main_program, serial_startup_program, []) + ( + dist_main_prog, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + serial_main_program, serial_startup_program, [] + ) # Do reshard process self._logger.debug( "within parallel partitioner time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) time0 = time.time() - resharder = Resharder(dist_main_prog, dist_startup_prog, rank, - self._dist_context, [], 1) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank, + self._dist_context, + [], + 1, + ) resharder.reshard() self._logger.debug( "within parallel reshard time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) # Clone program for test if self._mode != 'train': dist_main_prog = dist_main_prog.clone(for_test=True) @@ -133,13 +182,15 @@ class Parallelizer: def _generate_backward(self, main_program, startup_program, loss): with program_guard(main_program, startup_program): params_grads = append_backward( - loss, distop_context=self._dist_context.dist_op_context) + loss, distop_context=self._dist_context.dist_op_context + ) self._completer.complete_backward_annotation(main_program) self._dist_context.block_state.parse_backward_blocks(main_program) return params_grads - def _generate_optimizer(self, main_program, startup_program, optimizer, - params_grads): + def _generate_optimizer( + self, main_program, startup_program, optimizer, params_grads + ): # NOTE: `apply_gradients` will add an Accumulator for a parameter only once, # but optimizer will be called repeatedly in re-launch, so optimizer need to be copied. optimizer = copy.deepcopy(optimizer) @@ -150,8 +201,9 @@ class Parallelizer: self._completer.complete_update_annotation(main_program) return optimizer_ops - def _apply_pre_optimization(self, main_program, startup_program, loss, - optimizer, params_grads): + def _apply_pre_optimization( + self, main_program, startup_program, loss, optimizer, params_grads + ): if self._strategy is None: return @@ -162,10 +214,11 @@ class Parallelizer: config["dist_context"] = self._dist_context config["params_grads"] = params_grads auto_parallel_quantization_pass = new_pass( - "auto_parallel_quantization", config) - auto_parallel_quantization_pass.apply([main_program], - [startup_program], - self._pass_context) + "auto_parallel_quantization", config + ) + auto_parallel_quantization_pass.apply( + [main_program], [startup_program], self._pass_context + ) main_program = self._pass_context.get_attr("main_program") startup_program = self._pass_context.get_attr("startup_program") params_grads = self._pass_context.get_attr("params_grads") @@ -176,18 +229,22 @@ class Parallelizer: config["dist_context"] = self._dist_context config["params_grads"] = params_grads config["loss"] = loss - config["input_data"] = self._dist_context.serial_feed_vars["inputs"] \ + config["input_data"] = ( + self._dist_context.serial_feed_vars["inputs"] + self._dist_context.serial_feed_vars["labels"] + ) if config["use_pure_fp16"]: config["base_opt"] = optimizer auto_parallel_fp16_pass = new_pass("auto_parallel_fp16", config) - auto_parallel_fp16_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_fp16_pass.apply( + [main_program], [startup_program], self._pass_context + ) loss = auto_parallel_fp16_pass.get_loss() else: auto_parallel_amp_pass = new_pass("auto_parallel_amp", config) - auto_parallel_amp_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_amp_pass.apply( + [main_program], [startup_program], self._pass_context + ) loss = auto_parallel_amp_pass.get_loss() # apply recompute pass @@ -197,16 +254,18 @@ class Parallelizer: config["dist_context"] = self._dist_context config["no_grad_set"] = None config["loss"] = loss - auto_parallel_recompute_pass = new_pass("auto_parallel_recompute", - config) - auto_parallel_recompute_pass.apply([main_program], - [startup_program], - self._pass_context) + auto_parallel_recompute_pass = new_pass( + "auto_parallel_recompute", config + ) + auto_parallel_recompute_pass.apply( + [main_program], [startup_program], self._pass_context + ) return main_program, startup_program, params_grads - def _apply_post_optimization(self, main_program, startup_program, rank, - params_grads): + def _apply_post_optimization( + self, main_program, startup_program, rank, params_grads + ): if self._strategy is None: return @@ -223,10 +282,12 @@ class Parallelizer: config["dist_context"] = self._dist_context config["params_grads"] = params_grads config["global_rank"] = rank - auto_parallel_sharding_pass = new_pass("auto_parallel_sharding", - config) - auto_parallel_sharding_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_sharding_pass = new_pass( + "auto_parallel_sharding", config + ) + auto_parallel_sharding_pass.apply( + [main_program], [startup_program], self._pass_context + ) params_grads = self._pass_context.get_attr("params_grads") # GradClip is train-only optimization @@ -235,10 +296,12 @@ class Parallelizer: config["dist_context"] = self._dist_context config["params_grads"] = params_grads config["rank_id"] = rank - auto_parallel_clip_pass = new_pass("auto_parallel_grad_clip", - config) - auto_parallel_clip_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_clip_pass = new_pass( + "auto_parallel_grad_clip", config + ) + auto_parallel_clip_pass.apply( + [main_program], [startup_program], self._pass_context + ) # gradient_merge is then train-only optimization if self._mode == "train" and self._strategy.gradient_merge.enable: @@ -246,7 +309,8 @@ class Parallelizer: config["dist_context"] = self._dist_context config["params_grads"] = params_grads auto_parallel_gradient_merge_pass = new_pass( - "auto_parallel_gradient_merge_pass", config) - auto_parallel_gradient_merge_pass.apply([main_program], - [startup_program], - self._pass_context) + "auto_parallel_gradient_merge_pass", config + ) + auto_parallel_gradient_merge_pass.apply( + [main_program], [startup_program], self._pass_context + ) diff --git a/python/paddle/distributed/auto_parallel/partitioner.py b/python/paddle/distributed/auto_parallel/partitioner.py index 32917627672..2ba33981324 100644 --- a/python/paddle/distributed/auto_parallel/partitioner.py +++ b/python/paddle/distributed/auto_parallel/partitioner.py @@ -17,7 +17,9 @@ import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid import core from paddle.fluid.framework import Parameter, Program -from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container +from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, +) from paddle.distributed.auto_parallel.dist_context import DistributedContext from .dist_attribute import OperatorDistributedAttribute from .utils import is_backward_op, is_forward_op, is_loss_op, is_optimize_op @@ -25,7 +27,8 @@ from .operators.common import BACKWARD_ONLY_DIST_OPS __varname_not_in_block__ = ["lod_tensor_blocking_queue"] __not_shape_var_type__ = [ - core.VarDesc.VarType.READER, core.VarDesc.VarType.STEP_SCOPES + core.VarDesc.VarType.READER, + core.VarDesc.VarType.STEP_SCOPES, ] @@ -51,25 +54,29 @@ class Partitioner(object): """ if not isinstance(dist_context, DistributedContext): raise TypeError( - "dist_context be paddle.fluid.DistributedContext, got %s here" % - type(dist_context)) + "dist_context be paddle.fluid.DistributedContext, got %s here" + % type(dist_context) + ) self._dist_context = dist_context self._rank_id = rank_id self._serial2dist_varname_mapping = {} self._dist_varname_suffix = "" - def partition(self, serial_main_program, serial_startup_program, - params_grads): + def partition( + self, serial_main_program, serial_startup_program, params_grads + ): if not isinstance(serial_main_program, (Program)): raise TypeError( - "main_program be paddle.fluid.framework.program, got %s here" % - type(serial_main_program)) + "main_program be paddle.fluid.framework.program, got %s here" + % type(serial_main_program) + ) # check if shard annotated serial program valid if not self._is_valid_annotated_program(serial_main_program): raise RuntimeError( - "Not all vars or ops are annotated in main program !") + "Not all vars or ops are annotated in main program !" + ) # init distop helper dist_op_context = self._dist_context.dist_op_context @@ -81,22 +88,31 @@ class Partitioner(object): partitioned_startup_prog = None else: partitioned_startup_prog = self.partition_startup_program( - serial_main_program, serial_startup_program) + serial_main_program, serial_startup_program + ) dist_op_context.dst_startup_program = partitioned_startup_prog # partition main program - partitioned_main_prog, partitioned_params_grads = self.partition_main_program( - serial_main_program, params_grads) + ( + partitioned_main_prog, + partitioned_params_grads, + ) = self.partition_main_program(serial_main_program, params_grads) - return partitioned_main_prog, partitioned_startup_prog, partitioned_params_grads + return ( + partitioned_main_prog, + partitioned_startup_prog, + partitioned_params_grads, + ) - def partition_startup_program(self, serial_main_program, - serial_startup_program): + def partition_startup_program( + self, serial_main_program, serial_startup_program + ): if not isinstance(serial_startup_program, (Program)): raise TypeError( - "dist_context be paddle.fluid.framework.program, got %s here" % - type(serial_startup_program)) + "dist_context be paddle.fluid.framework.program, got %s here" + % type(serial_startup_program) + ) partitioned_startup_prog = fluid.Program() ref_block = serial_main_program.global_block() @@ -109,27 +125,33 @@ class Partitioner(object): assert var.persistable new_name = var.name + self._dist_varname_suffix temp_varname_map[var.name] = new_name - target_shape = _partition_var(self._dist_context, ref_block, - target_block, var.name, new_name) + target_shape = _partition_var( + self._dist_context, ref_block, target_block, var.name, new_name + ) var2shape[new_name] = target_shape # ops for op in serial_startup_program.global_block().ops: # TODO if var not belong to this rank, should be filtered output_vars = op.desc.output_arg_names() - assert len( - output_vars - ) == 1, "initializer should output only ONE variable, but got [{}]".format( - str(op.desc)) - assert temp_varname_map[output_vars[ - 0]] in var2shape, "try to initialize [{}] which is not a persistable var".format( - output_vars[0]) + assert ( + len(output_vars) == 1 + ), "initializer should output only ONE variable, but got [{}]".format( + str(op.desc) + ) + assert ( + temp_varname_map[output_vars[0]] in var2shape + ), "try to initialize [{}] which is not a persistable var".format( + output_vars[0] + ) new_op_desc = target_block.desc.append_op() new_op_desc.copy_from(op.desc) - new_op_desc._rename_output(output_vars[0], - temp_varname_map[output_vars[0]]) - new_op_desc._set_attr("shape", - var2shape[temp_varname_map[output_vars[0]]]) + new_op_desc._rename_output( + output_vars[0], temp_varname_map[output_vars[0]] + ) + new_op_desc._set_attr( + "shape", var2shape[temp_varname_map[output_vars[0]]] + ) target_block._sync_with_cpp() # set distribute atrribute @@ -137,14 +159,17 @@ class Partitioner(object): assert new_op.type == new_op_desc.type() assert new_op.desc == new_op_desc output_var = target_block.var(output_vars[0]) - output_var_attr = self._dist_context.get_tensor_dist_attr_for_program( - output_var) + output_var_attr = ( + self._dist_context.get_tensor_dist_attr_for_program(output_var) + ) op_attr = OperatorDistributedAttribute() op_attr.process_mesh = output_var_attr.process_mesh - op_attr.set_output_dims_mapping(output_var.name, - output_var_attr.dims_mapping) - op_attr.set_input_dims_mapping(output_var.name, - output_var_attr.dims_mapping) + op_attr.set_output_dims_mapping( + output_var.name, output_var_attr.dims_mapping + ) + op_attr.set_input_dims_mapping( + output_var.name, output_var_attr.dims_mapping + ) self._dist_context.set_op_dist_attr_for_program(new_op, op_attr) return partitioned_startup_prog @@ -166,7 +191,8 @@ class Partitioner(object): target_block = partitioned_main_prog.blocks[0] else: target_block = partitioned_main_prog._create_block( - parent_idx=ref_block.parent_idx) + parent_idx=ref_block.parent_idx + ) assert ref_block.idx == target_block.idx target_block._set_forward_block_idx(ref_block.forward_block_idx) dist_op_context.work_block = target_block @@ -181,8 +207,9 @@ class Partitioner(object): for attr_name in op.all_attrs(): if op.attr_type(attr_name) == core.AttrType.BLOCK: relative_id = op._block_attr_id(attr_name) - op._set_attr(attr_name, - partitioned_main_prog.block(relative_id)) + op._set_attr( + attr_name, partitioned_main_prog.block(relative_id) + ) partitioned_params_and_grads = [] for p, g in params_and_grads: @@ -193,7 +220,8 @@ class Partitioner(object): else: assert g.name in self._serial2dist_varname_mapping dist_g = self._get_dist_var_by_serial_var( - g, partitioned_main_prog) + g, partitioned_main_prog + ) partitioned_params_and_grads.append((dist_p, dist_g)) return partitioned_main_prog, partitioned_params_and_grads @@ -217,71 +245,112 @@ class Partitioner(object): for idx in range(len(serial_ops)): if idx <= last_fwd_op_idx: forward_op_id2forward_op[ - serial_ops[idx].desc.original_id()] = serial_ops[idx] + serial_ops[idx].desc.original_id() + ] = serial_ops[idx] # partiiton appended_grad_times = 0 for idx, op in enumerate(serial_ops): op_dist_attr = self._dist_context.get_op_dist_attr_for_program(op) - if is_backward_op(op) and (is_forward_op(serial_ops[idx - 1]) - or is_loss_op(serial_ops[idx - 1])): + if is_backward_op(op) and ( + is_forward_op(serial_ops[idx - 1]) + or is_loss_op(serial_ops[idx - 1]) + ): if not op_dist_attr.is_recompute: appended_grad_times += 1 # partititon input variables for serial_input_varname in op.desc.input_arg_names(): - if serial_input_varname not in self._serial2dist_varname_mapping: - new_varname = serial_input_varname + self._dist_varname_suffix + if ( + serial_input_varname + not in self._serial2dist_varname_mapping + ): + new_varname = ( + serial_input_varname + self._dist_varname_suffix + ) if ref_block.has_var(serial_input_varname): - _partition_var(self._dist_context, ref_block, - target_block, serial_input_varname, - new_varname) + _partition_var( + self._dist_context, + ref_block, + target_block, + serial_input_varname, + new_varname, + ) else: for varname_not_in_block in __varname_not_in_block__: - assert varname_not_in_block in serial_input_varname, \ - "{} is not found".format(serial_input_varname) + assert ( + varname_not_in_block in serial_input_varname + ), "{} is not found".format(serial_input_varname) self._serial2dist_varname_mapping[ - serial_input_varname] = new_varname + serial_input_varname + ] = new_varname # partition output vars for serial_output_varname in op.desc.output_arg_names(): - if serial_output_varname not in self._serial2dist_varname_mapping: - new_varname = serial_output_varname + self._dist_varname_suffix - _partition_var(self._dist_context, ref_block, target_block, - serial_output_varname, new_varname) + if ( + serial_output_varname + not in self._serial2dist_varname_mapping + ): + new_varname = ( + serial_output_varname + self._dist_varname_suffix + ) + _partition_var( + self._dist_context, + ref_block, + target_block, + serial_output_varname, + new_varname, + ) self._serial2dist_varname_mapping[ - serial_output_varname] = new_varname + serial_output_varname + ] = new_varname # partition op if is_forward_op(op) or op_dist_attr.is_recompute: kinputs, koutputs = dist_op_context.prepare_context(op) dist_op_forward_impl = _get_dist_op_forward_implement( - op, self._dist_context) - dist_op_forward_impl.forward(self._dist_context, **kinputs, - **koutputs) + op, self._dist_context + ) + dist_op_forward_impl.forward( + self._dist_context, **kinputs, **koutputs + ) elif is_backward_op(op): kinputs, koutputs = dist_op_context.prepare_context(op) dist_op_backward_impl = _get_dist_op_backward_implement( - op, self._dist_context, forward_op_id2forward_op) - grad_var_to_var = self._dist_context.dist_op_context.grad_var_to_var[ - appended_grad_times] + op, self._dist_context, forward_op_id2forward_op + ) + grad_var_to_var = ( + self._dist_context.dist_op_context.grad_var_to_var[ + appended_grad_times + ] + ) dist_op_backward_impl.backward( - self._dist_context, **kinputs, **koutputs, - **{"grad_var_to_var": grad_var_to_var}) + self._dist_context, + **kinputs, + **koutputs, + **{"grad_var_to_var": grad_var_to_var} + ) elif is_optimize_op(op): # NOTE: BACKWARD_ONLY_DIST_OPS's op_role must 2 because of 1F1B PASS kinputs, koutputs = dist_op_context.prepare_context(op) dist_op_opt_impl = _get_dist_op_backward_implement( - op, self._dist_context, forward_op_id2forward_op) - dist_op_opt_impl.backward(self._dist_context, **kinputs, - **koutputs, **{"grad_var_to_var": {}}) + op, self._dist_context, forward_op_id2forward_op + ) + dist_op_opt_impl.backward( + self._dist_context, + **kinputs, + **koutputs, + **{"grad_var_to_var": {}} + ) else: raise NotImplementedError( - "partitioner only support forward and backward, optimize ops, but got {}" - .format(str(op))) + "partitioner only support forward and backward, optimize ops, but got {}".format( + str(op) + ) + ) def _is_valid_annotated_program(self, program): @@ -293,13 +362,16 @@ class Partitioner(object): ] var_dist_attrs = [ self._dist_context.get_tensor_dist_attr_for_program(var) - for var in vars_ if (var.type not in __not_shape_var_type__) + for var in vars_ + if (var.type not in __not_shape_var_type__) ] - all_ops_annotated = all(dist_attr is not None - for dist_attr in op_dist_attrs) - all_vars_annotated = all(dist_attr is not None - for dist_attr in var_dist_attrs) + all_ops_annotated = all( + dist_attr is not None for dist_attr in op_dist_attrs + ) + all_vars_annotated = all( + dist_attr is not None for dist_attr in var_dist_attrs + ) return all_ops_annotated and all_vars_annotated @@ -323,22 +395,26 @@ def _get_dist_shape(var, dist_attr): assert len(var_shape) == len( mapping ), "variable shape [{}] and dim_mapping [{}] is NOT match !".format( - var_shape, mapping) + var_shape, mapping + ) new_shape = [] for idx in range(len(var_shape)): if var_shape[idx] == -1 or mapping[idx] == -1: new_shape.append(var_shape[idx]) else: - assert var_shape[idx] % mesh[mapping[ - idx]] == 0, "un-event partition: var_shape[idx]=[{}], mesh[{}]".format( - var_shape[idx], mesh[mapping[idx]]) + assert ( + var_shape[idx] % mesh[mapping[idx]] == 0 + ), "un-event partition: var_shape[idx]=[{}], mesh[{}]".format( + var_shape[idx], mesh[mapping[idx]] + ) new_shape.append(var_shape[idx] // mesh[mapping[idx]]) return new_shape -def _partition_parameter(dist_context, src_var, dst_block, dst_varname, - dst_shape): +def _partition_parameter( + dist_context, src_var, dst_block, dst_varname, dst_shape +): # NOTE hack to copied Parameter # not initialized parameter, need to initialize it copied_kwargs = {} @@ -348,39 +424,45 @@ def _partition_parameter(dist_context, src_var, dst_block, dst_varname, copied_kwargs['do_model_average'] = src_var.do_model_average copied_kwargs['need_clip'] = src_var.need_clip - param = Parameter(block=dst_block, - type=src_var.type, - name=dst_varname, - shape=dst_shape, - dtype=src_var.dtype, - lod_level=src_var.lod_level, - error_clip=src_var.error_clip, - stop_gradient=src_var.stop_gradient, - is_data=src_var.is_data, - belong_to_optimizer=src_var.belong_to_optimizer, - **copied_kwargs) + param = Parameter( + block=dst_block, + type=src_var.type, + name=dst_varname, + shape=dst_shape, + dtype=src_var.dtype, + lod_level=src_var.lod_level, + error_clip=src_var.error_clip, + stop_gradient=src_var.stop_gradient, + is_data=src_var.is_data, + belong_to_optimizer=src_var.belong_to_optimizer, + **copied_kwargs + ) return param -def _partition_intermediate_var(dist_context, src_var, dst_block, dst_varname, - dst_shape): - var = dst_block.create_var(type=src_var.type, - name=dst_varname, - shape=dst_shape, - dtype=src_var.dtype, - lod_level=src_var.lod_level, - persistable=src_var.persistable, - error_clip=src_var.error_clip, - stop_gradient=src_var.stop_gradient, - is_data=src_var.is_data, - belong_to_optimizer=src_var.belong_to_optimizer) +def _partition_intermediate_var( + dist_context, src_var, dst_block, dst_varname, dst_shape +): + var = dst_block.create_var( + type=src_var.type, + name=dst_varname, + shape=dst_shape, + dtype=src_var.dtype, + lod_level=src_var.lod_level, + persistable=src_var.persistable, + error_clip=src_var.error_clip, + stop_gradient=src_var.stop_gradient, + is_data=src_var.is_data, + belong_to_optimizer=src_var.belong_to_optimizer, + ) return var -def _partition_var(dist_context, src_block, dst_block, src_varname, - dst_varname): +def _partition_var( + dist_context, src_block, dst_block, src_varname, dst_varname +): """ partition include: split + replicate """ @@ -388,44 +470,53 @@ def _partition_var(dist_context, src_block, dst_block, src_varname, if src_var.type in __not_shape_var_type__: persist = getattr(src_var, 'persistable', False) - new_var = dst_block.create_var(type=src_var.type, - name=dst_varname, - persistable=persist, - stop_gradient=True) + new_var = dst_block.create_var( + type=src_var.type, + name=dst_varname, + persistable=persist, + stop_gradient=True, + ) target_shape = None else: dist_attr = dist_context.get_tensor_dist_attr_for_program(src_var) target_shape = _get_dist_shape(src_var, dist_attr) if isinstance(src_var, Parameter): - new_var = _partition_parameter(dist_context, src_var, dst_block, - dst_varname, target_shape) + new_var = _partition_parameter( + dist_context, src_var, dst_block, dst_varname, target_shape + ) else: - new_var = _partition_intermediate_var(dist_context, src_var, - dst_block, dst_varname, - target_shape) + new_var = _partition_intermediate_var( + dist_context, src_var, dst_block, dst_varname, target_shape + ) dist_attr = copy.deepcopy( - dist_context.get_tensor_dist_attr_for_program(src_var)) + dist_context.get_tensor_dist_attr_for_program(src_var) + ) assert dist_attr is not None dist_context.set_tensor_dist_attr_for_program(new_var, dist_attr) return target_shape -def _get_dist_op_backward_implement(backward_op, dist_context, - forward_op_id2forward_op): +def _get_dist_op_backward_implement( + backward_op, dist_context, forward_op_id2forward_op +): dist_op_context = dist_context.dist_op_context if backward_op.desc.original_id() in dist_op_context.grad_op_id_to_op_id: forward_op_id = dist_op_context.grad_op_id_to_op_id[ - backward_op.desc.original_id()] + backward_op.desc.original_id() + ] forward_op = forward_op_id2forward_op[forward_op_id] forward_op_dist_attr = dist_context.get_op_dist_attr_for_program( - forward_op) + forward_op + ) dist_op_impl_container = get_distributed_operator_impl_container( - forward_op_dist_attr.impl_type) + forward_op_dist_attr.impl_type + ) dist_op_impl = dist_op_impl_container.get_impl( - forward_op_dist_attr.impl_idx) + forward_op_dist_attr.impl_idx + ) return dist_op_impl # # NOTE trick for dist ops that only have backward implement @@ -433,7 +524,8 @@ def _get_dist_op_backward_implement(backward_op, dist_context, op_dist_attr = dist_context.get_op_dist_attr_for_program(backward_op) assert op_dist_attr.impl_idx >= 0 dist_op_impl = get_distributed_operator_impl_container( - op_dist_attr.impl_type).get_impl(op_dist_attr.impl_idx) + op_dist_attr.impl_type + ).get_impl(op_dist_attr.impl_idx) return dist_op_impl dist_op = get_distributed_operator_impl_container("default") @@ -443,6 +535,7 @@ def _get_dist_op_backward_implement(backward_op, dist_context, def _get_dist_op_forward_implement(forward_op, dist_context): dist_attr = dist_context.get_op_dist_attr_for_program(forward_op) dist_op_impl_container = get_distributed_operator_impl_container( - dist_attr.impl_type) + dist_attr.impl_type + ) dist_op_impl = dist_op_impl_container.get_impl(dist_attr.impl_idx) return dist_op_impl diff --git a/python/paddle/distributed/auto_parallel/planner.py b/python/paddle/distributed/auto_parallel/planner.py index d01fe50c0d4..df7fd63a6ed 100755 --- a/python/paddle/distributed/auto_parallel/planner.py +++ b/python/paddle/distributed/auto_parallel/planner.py @@ -32,7 +32,10 @@ from .utils import update_op_dims_mapping_by_default_dist_impl from .utils import update_op_dims_mapping_by_elementwise_like_dist_impl from .utils import get_all_distributed_main_program from .dist_context import DistributedContext, DistributedOperatorContext -from .dist_attribute import OperatorDistributedAttribute, TensorDistributedAttribute +from .dist_attribute import ( + OperatorDistributedAttribute, + TensorDistributedAttribute, +) paddle.seed(123) random.seed(123) @@ -40,18 +43,19 @@ np.random.seed(123) class PlanFilter: - @staticmethod - def check_dims_mapping_for_tensor(process_mesh_topology, tensor_shape, - dims_mapping): + def check_dims_mapping_for_tensor( + process_mesh_topology, tensor_shape, dims_mapping + ): valid = True assert len(tensor_shape) == len(dims_mapping) for idx, dim_mapping in enumerate(dims_mapping): if dim_mapping != -1: - if tensor_shape[idx] % process_mesh_topology[ - dim_mapping] != 0 or dims_mapping.count( - dim_mapping) > 1: + if ( + tensor_shape[idx] % process_mesh_topology[dim_mapping] != 0 + or dims_mapping.count(dim_mapping) > 1 + ): valid = False if dim_mapping != -1 and process_mesh_topology[0] == 1: valid = False @@ -65,7 +69,8 @@ class PlanFilter: for var_name in op.input_arg_names: dims_mapping = op_dist_attr.get_input_dims_mapping(var_name) if not PlanFilter.check_dims_mapping_for_tensor( - process_mesh.topology, vars[var_name].shape, dims_mapping): + process_mesh.topology, vars[var_name].shape, dims_mapping + ): return False if vars[var_name].is_data and len(dims_mapping) > 1: for dim in dims_mapping[1:]: @@ -75,7 +80,8 @@ class PlanFilter: for var_name in op.output_arg_names: dims_mapping = op_dist_attr.get_output_dims_mapping(var_name) if not PlanFilter.check_dims_mapping_for_tensor( - process_mesh.topology, vars[var_name].shape, dims_mapping): + process_mesh.topology, vars[var_name].shape, dims_mapping + ): return False return True @@ -83,7 +89,11 @@ class PlanFilter: @staticmethod def check_dims_mapping_for_special_op(op, op_dist_attr, vars): # NOTE: Those ops has some partition limits, and will be solved when corresponding dist op implemented in the future. - if op.type == "elementwise_add" or op.type == 'layer_norm' or op.type == "softmax_with_cross_entropy": + if ( + op.type == "elementwise_add" + or op.type == 'layer_norm' + or op.type == "softmax_with_cross_entropy" + ): for name in op.input_arg_names: for item in op_dist_attr.get_input_dims_mapping(name): if item != -1: @@ -104,20 +114,25 @@ class PlanFilter: class PlanSpace: not_enum_ops = ["create_py_reader", "create_double_buffer_reader", "read"] special_vars = [ - "lod_tensor_blocking_queue_0", "create_py_reader_0", "double_buffer_0" + "lod_tensor_blocking_queue_0", + "create_py_reader_0", + "double_buffer_0", ] @staticmethod - def _enum_dims_mapping(process_mesh_topology, visited, path, depth, res, - tensor_shape): + def _enum_dims_mapping( + process_mesh_topology, visited, path, depth, res, tensor_shape + ): """Enumerate dims mapping of tensor by the given process_mesh_topology""" nums = list(range(-1, len(process_mesh_topology))) if depth == len(tensor_shape): valid = True for idx, item in enumerate(path): if item != -1: - if tensor_shape[idx] % process_mesh_topology[ - item] != 0 or path.count(item) > 1: + if ( + tensor_shape[idx] % process_mesh_topology[item] != 0 + or path.count(item) > 1 + ): valid = False if valid: res.append(copy.deepcopy(path)) @@ -128,15 +143,23 @@ class PlanSpace: if i != 0: visited[i] = True path.append(nums[i]) - PlanSpace._enum_dims_mapping(process_mesh_topology, visited, - path, depth + 1, res, tensor_shape) + PlanSpace._enum_dims_mapping( + process_mesh_topology, + visited, + path, + depth + 1, + res, + tensor_shape, + ) visited[i] = False path.pop() @staticmethod def enum_process_mesh_topology(processes): """Enumerate all process meshes with the given processes.""" - assert processes >= 1, "The processes must be number and greater than 0." + assert ( + processes >= 1 + ), "The processes must be number and greater than 0." # compute divisors divisors = [] for i in range(1, processes + 1): @@ -182,7 +205,8 @@ class PlanSpace: dims_mapping_dict = OrderedDict() op_valid_dist_attrs = [] dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + op.type + ) # enumerate all valid dims mapping of tensor when process mesh given for var_name in chain(op.input_arg_names, op.output_arg_names): @@ -193,15 +217,22 @@ class PlanSpace: depth = 0 path = [] dims_mapping_list = [] - PlanSpace._enum_dims_mapping(process_mesh.topology, visited, path, - depth, dims_mapping_list, - vars[var_name].shape) + PlanSpace._enum_dims_mapping( + process_mesh.topology, + visited, + path, + depth, + dims_mapping_list, + vars[var_name].shape, + ) dims_mapping_dict[var_name] = copy.deepcopy(dims_mapping_list) # compose dims mapping composed_dims_mapping_list = list( product( - *[dims_mapping_dict[key] for key in dims_mapping_dict.keys()])) + *[dims_mapping_dict[key] for key in dims_mapping_dict.keys()] + ) + ) for composed_dims_mapping in composed_dims_mapping_list: op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = process_mesh @@ -209,15 +240,19 @@ class PlanSpace: for idx, dims_mapping in enumerate(composed_dims_mapping): if var_names[idx] in op.input_arg_names: - op_dist_attr.set_input_dims_mapping(var_names[idx], - dims_mapping) + op_dist_attr.set_input_dims_mapping( + var_names[idx], dims_mapping + ) elif var_names[idx] in op.output_arg_names: op_dist_attr.set_output_dims_mapping( - var_names[idx], dims_mapping) + var_names[idx], dims_mapping + ) else: raise ValueError( - "The {varname} is not input or output of op {op}.". - format(varname='var_names[idx]', op='op')) + "The {varname} is not input or output of op {op}.".format( + varname='var_names[idx]', op='op' + ) + ) dist_op = DistributedOperator(op, op_dist_attr) if dist_op_impl_container is None: @@ -226,14 +261,16 @@ class PlanSpace: valid = True try: changed = update_op_dims_mapping_by_elementwise_like_dist_impl( - dist_op) + dist_op + ) except Exception as e: valid = False if valid and not changed: if PlanFilter.check_dims_mapping_for_op( - op, dist_op.dist_attr, vars + op, dist_op.dist_attr, vars ) and PlanFilter.check_dims_mapping_for_special_op( - op, dist_op.dist_attr, vars): + op, dist_op.dist_attr, vars + ): dist_op.dist_attr.impl_type = "elementwise" dist_op.dist_attr.impl_idx = 0 op_valid_dist_attrs.append(dist_op.dist_attr) @@ -243,14 +280,16 @@ class PlanSpace: valid = True try: changed = update_op_dims_mapping_by_default_dist_impl( - dist_op) + dist_op + ) except Exception as e: valid = False if valid and not changed: if PlanFilter.check_dims_mapping_for_op( - op, dist_op.dist_attr, vars + op, dist_op.dist_attr, vars ) and PlanFilter.check_dims_mapping_for_special_op( - op, dist_op.dist_attr, vars): + op, dist_op.dist_attr, vars + ): dist_op.dist_attr.impl_type = "default" dist_op.dist_attr.impl_idx = 0 op_valid_dist_attrs.append(dist_op.dist_attr) @@ -261,7 +300,8 @@ class PlanSpace: for idx, impl in enumerate(impls): if impl.is_auto_compatible(dist_op): if PlanFilter.check_dims_mapping_for_op( - op, dist_op.dist_attr, vars): + op, dist_op.dist_attr, vars + ): dist_op.dist_attr.impl_type = dist_op.serial_op.type dist_op.dist_attr.impl_idx = idx op_valid_dist_attrs.append(dist_op.dist_attr) @@ -273,10 +313,12 @@ class PlanSpace: dist_op = DistributedOperator(op, op_dist_attr) for var_name in op.input_arg_names: op_dist_attr.set_input_dims_mapping( - vars[var_name], [-1 for i in vars[var_name].shape]) + vars[var_name], [-1 for i in vars[var_name].shape] + ) for var_name in op.output_arg_names: op_dist_attr.set_output_dims_mapping( - vars[var_name], [-1 for i in vars[var_name].shape]) + vars[var_name], [-1 for i in vars[var_name].shape] + ) dist_op.dist_attr.impl_type = "default" dist_op.dist_attr.impl_idx = 0 op_valid_dist_attrs.append(dist_op.dist_attr) @@ -284,9 +326,9 @@ class PlanSpace: return op_valid_dist_attrs @staticmethod - def enum_valid_dist_attr_for_program(program, - process_mesh_topology, - is_pipeline=False): + def enum_valid_dist_attr_for_program( + program, process_mesh_topology, is_pipeline=False + ): """Enumerate valid distributed attributes for all ops in program.""" valid_dist_attr_dict = OrderedDict() ops = program.global_block().ops @@ -304,16 +346,31 @@ class PlanSpace: if len(process_mesh_topology) > 1: process_mesh_shape = process_mesh_topology[:-1] per_process_mesh_group = processes // pipeline_stages - pipeline_process_meshes = [auto.ProcessMesh(mesh=np.array(global_group[i*per_process_mesh_group: \ - (i+1)*per_process_mesh_group]).reshape(process_mesh_shape).tolist()) for i in range(pipeline_stages)] + pipeline_process_meshes = [ + auto.ProcessMesh( + mesh=np.array( + global_group[ + i + * per_process_mesh_group : (i + 1) + * per_process_mesh_group + ] + ) + .reshape(process_mesh_shape) + .tolist() + ) + for i in range(pipeline_stages) + ] elif len(process_mesh_topology) == 1: pipeline_process_meshes = [ auto.ProcessMesh(mesh=[i]) for i in range(pipeline_stages) ] else: if len(process_mesh_topology) > 1: - global_process_mesh = auto.ProcessMesh(mesh=np.array( - global_group).reshape(process_mesh_topology).tolist()) + global_process_mesh = auto.ProcessMesh( + mesh=np.array(global_group) + .reshape(process_mesh_topology) + .tolist() + ) else: global_process_mesh = auto.ProcessMesh(mesh=global_group) @@ -323,8 +380,11 @@ class PlanSpace: op_process_mesh = global_process_mesh pipeline_stage = -1 if pipeline_process_meshes is not None: - pipeline_stage = idx // op_count_per_stage if idx // op_count_per_stage < len( - pipeline_process_meshes) else idx // op_count_per_stage - 1 + pipeline_stage = ( + idx // op_count_per_stage + if idx // op_count_per_stage < len(pipeline_process_meshes) + else idx // op_count_per_stage - 1 + ) if pipeline_stage >= len(pipeline_process_meshes): pipeline_stage = len(pipeline_process_meshes) - 1 op_process_mesh = pipeline_process_meshes[pipeline_stage] @@ -338,7 +398,8 @@ class PlanSpace: else: dims_mapping = [-1 for i in vars[var_name].shape] op_dist_attr.set_input_dims_mapping( - var_name, dims_mapping) + var_name, dims_mapping + ) for var_name in op.output_arg_names: if var_name in PlanSpace.special_vars: @@ -346,24 +407,31 @@ class PlanSpace: else: dims_mapping = [-1 for i in vars[var_name].shape] op_dist_attr.set_output_dims_mapping( - var_name, dims_mapping) + var_name, dims_mapping + ) op_valid_dist_attrs = [op_dist_attr] pipeline_stage = 0 if pipeline_stage != -1 else pipeline_stage else: op_valid_dist_attrs = PlanSpace._enum_valid_dist_attr_for_op( - program, op, op_process_mesh) + program, op, op_process_mesh + ) - assert op_valid_dist_attrs is not None, "Enumerate {} valid distributed attribute failed.".format( - op) + assert ( + op_valid_dist_attrs is not None + ), "Enumerate {} valid distributed attribute failed.".format(op) valid_dist_attr_dict[op.desc.id()] = [ - op_valid_dist_attrs, pipeline_stage + op_valid_dist_attrs, + pipeline_stage, ] - return valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh + return ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) class SearchAlgorithm: - def __init__(self, name): self._name = name @@ -376,7 +444,6 @@ class SearchAlgorithm: class MCMC(SearchAlgorithm): - def __init__(self, serial_program_info, parallelizer, max_search_times=5): super(MCMC, self).__init__("mcmc") self._serial_program_info = serial_program_info @@ -395,32 +462,49 @@ class MCMC(SearchAlgorithm): def max_search_times(self): return self._max_search_times - def make_special_op_unshard(self, op, ops, vars, dist_context, - valid_dist_attr_dict): + def make_special_op_unshard( + self, op, ops, vars, dist_context, valid_dist_attr_dict + ): if op.type == "softmax_with_cross_entropy": for var_name in op.input_arg_names: dims_mapping = dist_context.get_op_dist_attr_for_program( - op).get_input_dims_mapping(var_name) - if dims_mapping != dist_context.get_tensor_dist_attr_for_program( - vars[var_name]).dims_mapping: + op + ).get_input_dims_mapping(var_name) + if ( + dims_mapping + != dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).dims_mapping + ): has_changed = False for search_op in ops: if var_name in search_op.output_arg_names: op_dist_attr_list = valid_dist_attr_dict[ - search_op.desc.id()][0] + search_op.desc.id() + ][0] for op_dist_attr in op_dist_attr_list: - if op_dist_attr.get_output_dims_mapping( - var_name) == dims_mapping: + if ( + op_dist_attr.get_output_dims_mapping( + var_name + ) + == dims_mapping + ): dist_context.set_op_dist_attr_for_program( - search_op, op_dist_attr) + search_op, op_dist_attr + ) for name in search_op.output_arg_names: - tensor_dist_attr = TensorDistributedAttribute( + tensor_dist_attr = ( + TensorDistributedAttribute() + ) + tensor_dist_attr.process_mesh = ( + op_dist_attr.process_mesh ) - tensor_dist_attr.process_mesh = op_dist_attr.process_mesh tensor_dist_attr.dims_mapping = op_dist_attr.get_output_dims_mapping( - name) + name + ) dist_context.set_tensor_dist_attr_for_program( - vars[name], tensor_dist_attr) + vars[name], tensor_dist_attr + ) has_changed = True break if has_changed: @@ -430,8 +514,13 @@ class MCMC(SearchAlgorithm): "Change softmax_with_cross_entropy dist attr failed" ) - def init_program(self, valid_dist_attr_dict, program, - pipeline_process_meshes, global_process_mesh): + def init_program( + self, + valid_dist_attr_dict, + program, + pipeline_process_meshes, + global_process_mesh, + ): ops = program.global_block().ops vars = program.global_block().vars new_dist_context = DistributedContext() @@ -439,32 +528,44 @@ class MCMC(SearchAlgorithm): for op in ops: op_valid_dist_attr_list = valid_dist_attr_dict[op.desc.id()][0] random_op_dist_attr = np.random.randint( - len(op_valid_dist_attr_list)) + len(op_valid_dist_attr_list) + ) init_op_dist_attr = op_valid_dist_attr_list[random_op_dist_attr] new_dist_context.set_op_dist_attr_for_program(op, init_op_dist_attr) for var_name in op.input_arg_names: if var_name == "lod_tensor_blocking_queue_0": continue - if new_dist_context.get_tensor_dist_attr_for_program( - vars[var_name]) is None: + if ( + new_dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ) + is None + ): tensor_dist_attr = TensorDistributedAttribute() - tensor_dist_attr.process_mesh = init_op_dist_attr.process_mesh - tensor_dist_attr.dims_mapping = init_op_dist_attr.get_input_dims_mapping( - var_name) + tensor_dist_attr.process_mesh = ( + init_op_dist_attr.process_mesh + ) + tensor_dist_attr.dims_mapping = ( + init_op_dist_attr.get_input_dims_mapping(var_name) + ) new_dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) + vars[var_name], tensor_dist_attr + ) for var_name in op.output_arg_names: tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = init_op_dist_attr.process_mesh - tensor_dist_attr.dims_mapping = init_op_dist_attr.get_output_dims_mapping( - var_name) + tensor_dist_attr.dims_mapping = ( + init_op_dist_attr.get_output_dims_mapping(var_name) + ) new_dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) + vars[var_name], tensor_dist_attr + ) # NOTE: this is a temporary solution to make softmax_with_cross_entropy unshard - self.make_special_op_unshard(op, ops, vars, new_dist_context, - valid_dist_attr_dict) + self.make_special_op_unshard( + op, ops, vars, new_dist_context, valid_dist_attr_dict + ) # add process meshes to distributed context if global_process_mesh is not None: @@ -475,16 +576,19 @@ class MCMC(SearchAlgorithm): return new_dist_context - def estimate_searched_strategy_cost(self, - dist_context, - pipeline_process_meshes=None): + def estimate_searched_strategy_cost( + self, dist_context, pipeline_process_meshes=None + ): cost = None # get all distributed programs all_dist_main_program = get_all_distributed_main_program( - self.serial_program_info, dist_context, self.parallelizer) - pipeline_config = [ - process_mesh.processes for process_mesh in pipeline_process_meshes - ] if pipeline_process_meshes is not None else None + self.serial_program_info, dist_context, self.parallelizer + ) + pipeline_config = ( + [process_mesh.processes for process_mesh in pipeline_process_meshes] + if pipeline_process_meshes is not None + else None + ) microbatch_size = 1 for program in all_dist_main_program: searched_batch_size = False @@ -497,14 +601,17 @@ class MCMC(SearchAlgorithm): break from .utils import get_standalone_cost_data + standalone_cost_data = get_standalone_cost_data(all_dist_main_program) # cost model does not support cluster argument - cost = estimate_cost(all_dist_main_program, - cluster=None, - pipeline_config=pipeline_config, - standalone_cost_data=standalone_cost_data, - batch_size=microbatch_size) + cost = estimate_cost( + all_dist_main_program, + cluster=None, + pipeline_config=pipeline_config, + standalone_cost_data=standalone_cost_data, + batch_size=microbatch_size, + ) return cost @@ -514,10 +621,12 @@ class MCMC(SearchAlgorithm): process_mesh = op_dist_attr.process_mesh tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = process_mesh - tensor_dist_attr.dims_mapping = op_dist_attr.get_output_dims_mapping( - var_name) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_output_dims_mapping(var_name) + ) dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) + vars[var_name], tensor_dist_attr + ) # set input tensor distributed attribute if input is data or parameter for var_name in op.input_arg_names: @@ -525,27 +634,34 @@ class MCMC(SearchAlgorithm): process_mesh = op_dist_attr.process_mesh tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = process_mesh - tensor_dist_attr.dims_mapping = op_dist_attr.get_input_dims_mapping( - var_name) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_input_dims_mapping(var_name) + ) dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) + vars[var_name], tensor_dist_attr + ) def change_process_mesh(self, op, changed_process_mesh, vars, dist_context): dist_context.get_op_dist_attr_for_program( - op).process_mesh = changed_process_mesh + op + ).process_mesh = changed_process_mesh for var_name in op.output_arg_names: dist_context.get_tensor_dist_attr_for_program( - vars[var_name]).process_mesh = changed_process_mesh + vars[var_name] + ).process_mesh = changed_process_mesh for var_name in op.input_arg_names: if vars[var_name].is_parameter or vars[var_name].is_data: dist_context.get_tensor_dist_attr_for_program( - vars[var_name]).process_mesh = changed_process_mesh - - def search_once(self, - program, - valid_dist_attr_dict, - dist_context, - pipeline_process_meshes=None): + vars[var_name] + ).process_mesh = changed_process_mesh + + def search_once( + self, + program, + valid_dist_attr_dict, + dist_context, + pipeline_process_meshes=None, + ): raw_ops = program.global_block().ops ops = [] for op in raw_ops: @@ -561,9 +677,11 @@ class MCMC(SearchAlgorithm): op_valid_dist_attr_list = valid_dist_attr_dict[selected_op.desc.id()][0] pipeline_stage = valid_dist_attr_dict[selected_op.desc.id()][1] random_selected_dist_attr_idx = np.random.randint( - len(op_valid_dist_attr_list)) + len(op_valid_dist_attr_list) + ) selected_op_dist_attr = copy.deepcopy( - op_valid_dist_attr_list[random_selected_dist_attr_idx]) + op_valid_dist_attr_list[random_selected_dist_attr_idx] + ) start_idx = ops[0].desc.id() if pipeline_stage > -1: @@ -574,142 +692,215 @@ class MCMC(SearchAlgorithm): if changed_mode == 0: # not change the process mesh, just change dims mapping new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, selected_op_dist_attr, - vars, new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, selected_op_dist_attr, vars, new_dist_context + ) elif changed_mode == 1: changed_stage = pipeline_stage - 1 - if changed_stage == -1 or random_selected_op_idx == len(ops) - 1 or \ - (random_selected_op_idx + 1 == len(ops) - 1 and new_valid_dist_attr_dict[ops[random_selected_op_idx + 1].desc.id()][1] == pipeline_stage + 1 ): + if ( + changed_stage == -1 + or random_selected_op_idx == len(ops) - 1 + or ( + random_selected_op_idx + 1 == len(ops) - 1 + and new_valid_dist_attr_dict[ + ops[random_selected_op_idx + 1].desc.id() + ][1] + == pipeline_stage + 1 + ) + ): new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, - selected_op_dist_attr, vars, - new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, + selected_op_dist_attr, + vars, + new_dist_context, + ) else: selected_op_process_mesh = pipeline_process_meshes[ - pipeline_stage] + pipeline_stage + ] next_op_id = ops[random_selected_op_idx + 1].desc.id() - if new_valid_dist_attr_dict[next_op_id][ - 1] == pipeline_stage + 1 and random_selected_op_idx + 1 != len( - ops) - 1: + if ( + new_valid_dist_attr_dict[next_op_id][1] + == pipeline_stage + 1 + and random_selected_op_idx + 1 != len(ops) - 1 + ): new_valid_dist_attr_dict[next_op_id][1] = pipeline_stage for op_dist_attr in new_valid_dist_attr_dict[ - next_op_id][0]: + next_op_id + ][0]: op_dist_attr.process_mesh = selected_op_process_mesh # set next op dist attr in the discontext and output/input tensor process mesh self.change_process_mesh( ops[random_selected_op_idx + 1], - selected_op_process_mesh, vars, new_dist_context) + selected_op_process_mesh, + vars, + new_dist_context, + ) # change the selected op stage and output dist attr - new_valid_dist_attr_dict[ - selected_op.desc.id()][1] = changed_stage + new_valid_dist_attr_dict[selected_op.desc.id()][ + 1 + ] = changed_stage new_process_mesh = pipeline_process_meshes[changed_stage] selected_op_dist_attr.process_mesh = new_process_mesh for op_dist_attr in new_valid_dist_attr_dict[ - selected_op.desc.id()][0]: + selected_op.desc.id() + ][0]: op_dist_attr.process_mesh = new_process_mesh new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) + selected_op, selected_op_dist_attr + ) - self.set_tensor_dist_attr(selected_op, - selected_op_dist_attr, vars, - new_dist_context) + self.set_tensor_dist_attr( + selected_op, + selected_op_dist_attr, + vars, + new_dist_context, + ) # change the pre op stage for idx in range(random_selected_op_idx - 1, -1, -1): stage = new_valid_dist_attr_dict[ops[idx].desc.id()][1] valid_dist_attr_list = new_valid_dist_attr_dict[ - ops[idx].desc.id()][0] + ops[idx].desc.id() + ][0] new_process_mesh = pipeline_process_meshes[ - changed_stage] + changed_stage + ] if stage == changed_stage + 1: - new_valid_dist_attr_dict[ - ops[idx].desc.id()][1] = changed_stage + new_valid_dist_attr_dict[ops[idx].desc.id()][ + 1 + ] = changed_stage for op_dist_attr in valid_dist_attr_list: op_dist_attr.process_mesh = new_process_mesh new_dist_context.get_op_dist_attr_for_program( - ops[idx]).process_mesh = new_process_mesh + ops[idx] + ).process_mesh = new_process_mesh # change process mesh of the output and input tensor - self.change_process_mesh(ops[idx], new_process_mesh, - vars, new_dist_context) + self.change_process_mesh( + ops[idx], + new_process_mesh, + vars, + new_dist_context, + ) else: break else: changed_stage = pipeline_stage + 1 - if changed_stage == len( - pipeline_process_meshes) or random_selected_op_idx == 0 or \ - (new_valid_dist_attr_dict[ops[random_selected_op_idx - 1].desc.id()][1] == pipeline_stage - 1 and (random_selected_op_idx == 1)): + if ( + changed_stage == len(pipeline_process_meshes) + or random_selected_op_idx == 0 + or ( + new_valid_dist_attr_dict[ + ops[random_selected_op_idx - 1].desc.id() + ][1] + == pipeline_stage - 1 + and (random_selected_op_idx == 1) + ) + ): new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, - selected_op_dist_attr, vars, - new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, + selected_op_dist_attr, + vars, + new_dist_context, + ) else: selected_op_process_mesh = pipeline_process_meshes[ - pipeline_stage] + pipeline_stage + ] pre_op_id = ops[random_selected_op_idx - 1].desc.id() - if new_valid_dist_attr_dict[pre_op_id][ - 1] == pipeline_stage - 1 and random_selected_op_idx != 1: + if ( + new_valid_dist_attr_dict[pre_op_id][1] + == pipeline_stage - 1 + and random_selected_op_idx != 1 + ): new_valid_dist_attr_dict[pre_op_id][1] = pipeline_stage for op_dist_attr in new_valid_dist_attr_dict[pre_op_id][ - 0]: + 0 + ]: op_dist_attr.process_mesh = selected_op_process_mesh # set pre op dist attr in the discontext and output tensor process mesh self.change_process_mesh( ops[random_selected_op_idx - 1], - selected_op_process_mesh, vars, new_dist_context) + selected_op_process_mesh, + vars, + new_dist_context, + ) # change the selected op stage and output tensor dist attr - new_valid_dist_attr_dict[ - selected_op.desc.id()][1] = changed_stage + new_valid_dist_attr_dict[selected_op.desc.id()][ + 1 + ] = changed_stage new_process_mesh = pipeline_process_meshes[changed_stage] selected_op_dist_attr.process_mesh = new_process_mesh for op_dist_attr in new_valid_dist_attr_dict[ - selected_op.desc.id()][0]: + selected_op.desc.id() + ][0]: op_dist_attr.process_mesh = new_process_mesh new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, - selected_op_dist_attr, vars, - new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, + selected_op_dist_attr, + vars, + new_dist_context, + ) # change the next op stage for idx in range(random_selected_op_idx + 1, len(ops)): stage = new_valid_dist_attr_dict[ops[idx].desc.id()][1] valid_dist_attr_list = new_valid_dist_attr_dict[ - ops[idx].desc.id()][0] + ops[idx].desc.id() + ][0] new_process_mesh = pipeline_process_meshes[ - changed_stage] + changed_stage + ] if stage == changed_stage - 1: - new_valid_dist_attr_dict[ - ops[idx].desc.id()][1] = changed_stage + new_valid_dist_attr_dict[ops[idx].desc.id()][ + 1 + ] = changed_stage for op_dist_attr in valid_dist_attr_list: op_dist_attr.process_mesh = new_process_mesh new_dist_context.get_op_dist_attr_for_program( - ops[idx]).process_mesh = new_process_mesh + ops[idx] + ).process_mesh = new_process_mesh # change the output tensor dist attr - self.change_process_mesh(ops[idx], new_process_mesh, - vars, new_dist_context) + self.change_process_mesh( + ops[idx], + new_process_mesh, + vars, + new_dist_context, + ) else: break else: new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, selected_op_dist_attr, vars, - new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, selected_op_dist_attr, vars, new_dist_context + ) for op in ops: # make softmax_with_cross_entropy unshard if op.type == "softmax_with_cross_entropy": - self.make_special_op_unshard(op, ops, vars, new_dist_context, - valid_dist_attr_dict) + self.make_special_op_unshard( + op, ops, vars, new_dist_context, valid_dist_attr_dict + ) break if new_valid_dist_attr_dict is None: @@ -717,22 +908,29 @@ class MCMC(SearchAlgorithm): else: return new_valid_dist_attr_dict, new_dist_context - def _search_core(self, - valid_dist_attr_dict, - init_dist_context, - pipeline_process_meshes=None): + def _search_core( + self, + valid_dist_attr_dict, + init_dist_context, + pipeline_process_meshes=None, + ): times = 0 best_dist_context = init_dist_context cost = self.estimate_searched_strategy_cost( - init_dist_context, pipeline_process_meshes).runtime + init_dist_context, pipeline_process_meshes + ).runtime min_cost = cost while times < self.max_search_times: times += 1 new_dist_context = self.search_once( - self.serial_program_info.train_program, valid_dist_attr_dict, - best_dist_context, pipeline_process_meshes)[1] + self.serial_program_info.train_program, + valid_dist_attr_dict, + best_dist_context, + pipeline_process_meshes, + )[1] cur_cost = self.estimate_searched_strategy_cost( - new_dist_context, pipeline_process_meshes).runtime + new_dist_context, pipeline_process_meshes + ).runtime if (min_cost - cur_cost) > 0: best_dist_context = copy.deepcopy(new_dist_context) min_cost = cur_cost @@ -744,35 +942,57 @@ class MCMC(SearchAlgorithm): start_time = time.time() train_program = self.serial_program_info.train_program cluster = self.serial_program_info.cluster - processes = paddle.distributed.get_world_size( - ) if cluster is None else len(cluster.get_all_devices("GPU")) + processes = ( + paddle.distributed.get_world_size() + if cluster is None + else len(cluster.get_all_devices("GPU")) + ) assert processes > 0, "Get process failed." process_mesh_topology_list = PlanSpace.enum_process_mesh_topology( - processes) + processes + ) searched_dist_context = None min_cost = None searched_pipeline_dist_context = None pipeline_min_cost = None for process_mesh_topology in process_mesh_topology_list: - print("MCMC search: search process mesh {} with pipeline mode.". - format(process_mesh_topology)) - valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh = PlanSpace.enum_valid_dist_attr_for_program( - train_program, process_mesh_topology, True) - init_dist_context = self.init_program(valid_dist_attr_dict, - train_program, - pipeline_process_meshes, - global_process_mesh) + print( + "MCMC search: search process mesh {} with pipeline mode.".format( + process_mesh_topology + ) + ) + ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) = PlanSpace.enum_valid_dist_attr_for_program( + train_program, process_mesh_topology, True + ) + init_dist_context = self.init_program( + valid_dist_attr_dict, + train_program, + pipeline_process_meshes, + global_process_mesh, + ) best_dist_context, cost = self._search_core( - valid_dist_attr_dict, init_dist_context, - pipeline_process_meshes) + valid_dist_attr_dict, init_dist_context, pipeline_process_meshes + ) print( - "MCMC search: the min cost is {} in the process mesh {} with pipeline mode." - .format(cost, process_mesh_topology)) + "MCMC search: the min cost is {} in the process mesh {} with pipeline mode.".format( + cost, process_mesh_topology + ) + ) best_dist_context._dist_op_context = DistributedOperatorContext() - pipeline_min_cost = cost if pipeline_min_cost is None else pipeline_min_cost - searched_pipeline_dist_context = best_dist_context if searched_pipeline_dist_context is None else searched_pipeline_dist_context + pipeline_min_cost = ( + cost if pipeline_min_cost is None else pipeline_min_cost + ) + searched_pipeline_dist_context = ( + best_dist_context + if searched_pipeline_dist_context is None + else searched_pipeline_dist_context + ) if pipeline_min_cost > cost: searched_pipeline_dist_context = best_dist_context pipeline_min_cost = cost @@ -783,23 +1003,41 @@ class MCMC(SearchAlgorithm): # if process_mesh_topology shape is 3, include pipeline mode by default if len(process_mesh_topology) == 3: continue - print("MCMC search: search process mesh {} without pipeline mode.". - format(process_mesh_topology)) - valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh = PlanSpace.enum_valid_dist_attr_for_program( - train_program, process_mesh_topology, False) - init_dist_context = self.init_program(valid_dist_attr_dict, - train_program, - pipeline_process_meshes, - global_process_mesh) + print( + "MCMC search: search process mesh {} without pipeline mode.".format( + process_mesh_topology + ) + ) + ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) = PlanSpace.enum_valid_dist_attr_for_program( + train_program, process_mesh_topology, False + ) + init_dist_context = self.init_program( + valid_dist_attr_dict, + train_program, + pipeline_process_meshes, + global_process_mesh, + ) best_dist_context, cost = self._search_core( - valid_dist_attr_dict, init_dist_context, - pipeline_process_meshes) + valid_dist_attr_dict, init_dist_context, pipeline_process_meshes + ) print( - "MCMC search: the min cost is {} in the process mesh {} without pipeline mode." - .format(cost, process_mesh_topology)) + "MCMC search: the min cost is {} in the process mesh {} without pipeline mode.".format( + cost, process_mesh_topology + ) + ) best_dist_context._dist_op_context = DistributedOperatorContext() - non_pipeline_min_cost = cost if non_pipeline_min_cost is None else non_pipeline_min_cost - searched_non_pipeline_dist_context = best_dist_context if searched_non_pipeline_dist_context is None else searched_non_pipeline_dist_context + non_pipeline_min_cost = ( + cost if non_pipeline_min_cost is None else non_pipeline_min_cost + ) + searched_non_pipeline_dist_context = ( + best_dist_context + if searched_non_pipeline_dist_context is None + else searched_non_pipeline_dist_context + ) if non_pipeline_min_cost > cost: searched_non_pipeline_dist_context = best_dist_context non_pipeline_min_cost = cost @@ -820,22 +1058,23 @@ class MCMC(SearchAlgorithm): pg0.add_ranks(process_mesh.processes) end_time = time.time() print( - "End MCMC searching: the min cost is {} and the search time is {}s." - .format(min_cost, end_time - start_time)) + "End MCMC searching: the min cost is {} and the search time is {}s.".format( + min_cost, end_time - start_time + ) + ) return searched_dist_context, min_cost class Planner: - - def __init__(self, - serial_program_info, - parallelizer, - algorithm_config=None): + def __init__( + self, serial_program_info, parallelizer, algorithm_config=None + ): self._serial_program_info = serial_program_info self._parallelizer = parallelizer self._algorithm_config = algorithm_config self._algorithm_searcher = self.create_algorithm_searcher( - algorithm_config) + algorithm_config + ) @property def serial_program_info(self): @@ -861,13 +1100,19 @@ class Planner: if name == "mcmc": # NOTE: Only GPU clusters are supported now. max_search_times = algorithm_config.get("max_search_times", None) - algorithm_searcher = MCMC( - self.serial_program_info, self.parallelizer, - max_search_times) if max_search_times is not None else MCMC( - self.serial_program_info, self.parallelizer) + algorithm_searcher = ( + MCMC( + self.serial_program_info, + self.parallelizer, + max_search_times, + ) + if max_search_times is not None + else MCMC(self.serial_program_info, self.parallelizer) + ) else: raise NotImplementedError( - "Other search algorithms have not been supported now.") + "Other search algorithms have not been supported now." + ) return algorithm_searcher diff --git a/python/paddle/distributed/auto_parallel/planner_v2.py b/python/paddle/distributed/auto_parallel/planner_v2.py index 8e2c0c4617b..0f9792911dd 100755 --- a/python/paddle/distributed/auto_parallel/planner_v2.py +++ b/python/paddle/distributed/auto_parallel/planner_v2.py @@ -18,7 +18,6 @@ from .tuner.parallel_tuner import ParallelTuner class Planner: - def __init__(self, mode, dist_context): self._mode = mode self._dist_context = dist_context @@ -39,8 +38,9 @@ class Planner: self._strategy = dist_context.strategy # set parallel tuner for auto search if self._strategy.auto_mode == "full": - self._parallel_tuner = ParallelTuner(self._dist_context, - mode=self._mode) + self._parallel_tuner = ParallelTuner( + self._dist_context, mode=self._mode + ) @property def completer(self): @@ -53,4 +53,5 @@ class Planner: self._completer.complete_forward_annotation() # parse forward sub block self._dist_context.block_state.parse_forward_blocks( - self._dist_context.serial_main_program) + self._dist_context.serial_main_program + ) diff --git a/python/paddle/distributed/auto_parallel/process_group.py b/python/paddle/distributed/auto_parallel/process_group.py index ba63a716434..9883f116f4e 100644 --- a/python/paddle/distributed/auto_parallel/process_group.py +++ b/python/paddle/distributed/auto_parallel/process_group.py @@ -31,10 +31,11 @@ def get_all_process_groups(): def get_process_group(group_id, g_process_group_map=None): global _g_process_group_map - return _g_process_group_map.get( - group_id, - None) if g_process_group_map is None else g_process_group_map.get( - group_id, None) + return ( + _g_process_group_map.get(group_id, None) + if g_process_group_map is None + else g_process_group_map.get(group_id, None) + ) def get_world_process_group(): @@ -75,10 +76,11 @@ def new_process_group(ranks, group_id=None): # the instantiation process in a more general way. In the future, the process group may # handle the communication implementation choice. class ProcessGroup: - def __init__(self, group_id, ranks): if group_id == 0 and get_process_group(0) is not None: - assert group_id != 0, "Process group id 0 is reserved for all ranks." + assert ( + group_id != 0 + ), "Process group id 0 is reserved for all ranks." self._group_id = group_id self._ranks = sorted(ranks) # Add the current ranks into group 0 @@ -103,8 +105,9 @@ class ProcessGroup: if set(new_ranks) <= set(self.ranks): return else: - assert self.is_instantiate() == False, \ - "Cannot add new ranks after instantiating the process group" + assert ( + self.is_instantiate() == False + ), "Cannot add new ranks after instantiating the process group" self._ranks.extend(new_ranks) self._ranks = sorted(list(set(self.ranks))) @@ -112,8 +115,9 @@ class ProcessGroup: if global_rank in self.ranks: return self.ranks.index(global_rank) else: - assert False, \ - "Rank {} doesn't belong to this group".format(global_rank) + assert False, "Rank {} doesn't belong to this group".format( + global_rank + ) def is_instantiate(self): return self._is_instantiate @@ -137,22 +141,27 @@ class ProcessGroup: if core.is_compiled_with_cuda(): place = core.CUDAPlace(genv.device_id) - core.NCCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.NCCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) else: - assert False, ("No CUDA device found") + assert False, "No CUDA device found" # TODO(shenliang03): This is a temporary solution to solve the problem of # hang caused by cross-creation of new_group paddle.disable_static() - paddle.set_device('gpu:%d' % - paddle.distributed.ParallelEnv().dev_id) - tmp = paddle.to_tensor( - [1], dtype="int32") if _non_static_mode() else fill_constant( - [0], dtype="int32", value="1") + paddle.set_device( + 'gpu:%d' % paddle.distributed.ParallelEnv().dev_id + ) + tmp = ( + paddle.to_tensor([1], dtype="int32") + if _non_static_mode() + else fill_constant([0], dtype="int32", value="1") + ) # use legacy ops - _legacy_C_ops.c_allreduce_sum_(tmp, 'use_calc_stream', True, - 'ring_id', self.id) + _legacy_C_ops.c_allreduce_sum_( + tmp, 'use_calc_stream', True, 'ring_id', self.id + ) _legacy_C_ops.c_sync_calc_stream(tmp, tmp) paddle.enable_static() @@ -173,7 +182,8 @@ class ProcessGroup: def __str__(self): string = "id: {}, nranks: {}, ranks: {}.".format( - self.id, self.nranks, ", ".join(map(str, self.ranks))) + self.id, self.nranks, ", ".join(map(str, self.ranks)) + ) return string def __hash__(self): diff --git a/python/paddle/distributed/auto_parallel/process_mesh.py b/python/paddle/distributed/auto_parallel/process_mesh.py index afe9f13e3ab..9d8a44b92d2 100644 --- a/python/paddle/distributed/auto_parallel/process_mesh.py +++ b/python/paddle/distributed/auto_parallel/process_mesh.py @@ -68,10 +68,10 @@ class ProcessMesh(object): assert process_ids is not None mesh = np.array(process_ids).reshape(shape) - if not isinstance(mesh, list) and \ - not isinstance(mesh, np.ndarray): + if not isinstance(mesh, list) and not isinstance(mesh, np.ndarray): raise ValueError( - 'The mesh must be an instance of list or np.ndarray.') + 'The mesh must be an instance of list or np.ndarray.' + ) if isinstance(mesh, list): mesh = np.array(mesh) @@ -79,30 +79,37 @@ class ProcessMesh(object): self._shape = list(self._mesh.shape) self._process_ids = self._mesh.flatten().tolist() - assert all(isinstance(p, int) for p in self._process_ids), \ - ("All elements of the mesh must be integer") - assert min( - self._process_ids) >= 0, ('All elements of the mesh must be >= 0.') + assert all( + isinstance(p, int) for p in self._process_ids + ), "All elements of the mesh must be integer" + assert ( + min(self._process_ids) >= 0 + ), 'All elements of the mesh must be >= 0.' unique_process_ids = set(self._process_ids) assert len(unique_process_ids) == len( - self._process_ids), ('All elements of the mesh must be unique.') + self._process_ids + ), 'All elements of the mesh must be unique.' if dim_names is not None: - assert len(dim_names) == len(self._shape), \ - ("The length of dims_names must be same as the shape of the mesh.") + assert len(dim_names) == len( + self._shape + ), "The length of dims_names must be same as the shape of the mesh." self._dim_names = copy.deepcopy(dim_names) else: self._dim_names = ["d" + str(i) for i in range(len(self._shape))] unique_dim_names = set(self._dim_names) - assert len(unique_dim_names) == len(self._dim_names), ( - 'All dim_names {} must be unique.'.format(dim_names)) + assert len(unique_dim_names) == len( + self._dim_names + ), 'All dim_names {} must be unique.'.format(dim_names) # Store all process meshes from .dist_context import get_default_distributed_context + default_dist_cxt = get_default_distributed_context() default_dist_cxt.add_process_mesh(self) # Add new processes to process group 0 from .process_group import get_process_group + pg0 = get_process_group(0) pg0.add_ranks(self.processes) @@ -183,20 +190,24 @@ class ProcessMesh(object): def __exit__(self, exc_type, exc_value, exc_traceback): from .dist_tensor import DistributedTensor from .dist_op import DistributedOperator + default_prog = paddle.fluid.default_main_program() cur_block = default_prog.current_block() new_var_names = list(cur_block.vars.keys()) new_op_size = len(cur_block.ops) from .dist_context import get_default_distributed_context + default_dist_ctx = get_default_distributed_context() for name in new_var_names: if name not in self._old_var_names: tensor = cur_block.vars[name] dist_tensor = default_dist_ctx.get_dist_tensor_for_program( - tensor) + tensor + ) if dist_tensor is None: - dist_tensor = DistributedTensor(cur_block.vars[name], - {"process_mesh": self}) + dist_tensor = DistributedTensor( + cur_block.vars[name], {"process_mesh": self} + ) dist_tensor.dist_attr.mark_annotated("process_mesh") default_dist_ctx.add_dist_tensor_for_program(dist_tensor) else: @@ -229,5 +240,6 @@ class ProcessMesh(object): def __str__(self): str = "shape {}, process_ids {}, dim_nams {}".format( - self.shape, self.process_ids, self.dim_names) + self.shape, self.process_ids, self.dim_names + ) return str diff --git a/python/paddle/distributed/auto_parallel/process_mesh_v2.py b/python/paddle/distributed/auto_parallel/process_mesh_v2.py index bc0de1748f1..0b427c9d161 100644 --- a/python/paddle/distributed/auto_parallel/process_mesh_v2.py +++ b/python/paddle/distributed/auto_parallel/process_mesh_v2.py @@ -44,10 +44,10 @@ class ProcessMesh(core.ProcessMesh): """ def __init__(self, mesh, dim_names=None): - if not isinstance(mesh, list) and \ - not isinstance(mesh, np.ndarray): + if not isinstance(mesh, list) and not isinstance(mesh, np.ndarray): raise ValueError( - 'The mesh must be an instance of list or np.ndarray.') + 'The mesh must be an instance of list or np.ndarray.' + ) if isinstance(mesh, list): mesh = np.array(mesh) @@ -56,24 +56,29 @@ class ProcessMesh(core.ProcessMesh): self._shape = list(self._mesh.shape) self._process_ids = self._mesh.flatten().tolist() - assert all(isinstance(p, int) for p in self._process_ids), \ - ("All elements of the mesh must be integer") - assert min( - self._process_ids) >= 0, ('All elements of the mesh must be >= 0.') + assert all( + isinstance(p, int) for p in self._process_ids + ), "All elements of the mesh must be integer" + assert ( + min(self._process_ids) >= 0 + ), 'All elements of the mesh must be >= 0.' unique_process_ids = set(self._process_ids) assert len(unique_process_ids) == len( - self._process_ids), ('All elements of the mesh must be unique.') + self._process_ids + ), 'All elements of the mesh must be unique.' if dim_names is not None: - assert len(dim_names) == len(self._shape), \ - ("The length of dims_names must be same as the shape of the mesh.") + assert len(dim_names) == len( + self._shape + ), "The length of dims_names must be same as the shape of the mesh." self._dim_names = dim_names else: self._dim_names = ["d" + str(i) for i in range(len(self._shape))] # Follow the requirement for using pybind11 - core.ProcessMesh.__init__(self, self._shape, self._process_ids, - self._dim_names) + core.ProcessMesh.__init__( + self, self._shape, self._process_ids, self._dim_names + ) @property def mesh(self): @@ -107,15 +112,20 @@ def compute_compatible_process_mesh(process_meshes): compatible_result = None for process_mesh in process_meshes: - compatible, compatible_result = _compute_compatible_of_two_process_meshes( - compatible_result, process_mesh) + ( + compatible, + compatible_result, + ) = _compute_compatible_of_two_process_meshes( + compatible_result, process_mesh + ) if not compatible: return None if compatible_result.empty(): return None if isinstance(compatible_result, core.ProcessMesh): mesh = np.array(compatible_result.process_ids).reshape( - compatible_result.shape) + compatible_result.shape + ) return ProcessMesh(mesh, compatible_result.dim_names) elif isinstance(compatible_result, ProcessMesh): return ProcessMesh(compatible_result.mesh, compatible_result.dim_names) diff --git a/python/paddle/distributed/auto_parallel/reshard.py b/python/paddle/distributed/auto_parallel/reshard.py index 46057ad97c4..44521c05994 100644 --- a/python/paddle/distributed/auto_parallel/reshard.py +++ b/python/paddle/distributed/auto_parallel/reshard.py @@ -32,7 +32,11 @@ from .utils import is_gradient_clip_op # NOTE: If op in _g_special_ops or _g_gradient_clip_ops, it will not be resharded. _g_special_ops = ['check_finite_and_unscale', 'update_loss_scaling'] _g_gradient_clip_ops = [ - "sum", "sqrt", "fill_constant", "elementwise_max", "elementwise_div" + "sum", + "sqrt", + "fill_constant", + "elementwise_max", + "elementwise_div", ] _g_subblock_ops = ["while", "conditional_block"] @@ -264,20 +268,25 @@ class Inserter: def insert_cast_op(block, idx, tensor, op_role, tensor_type): # to avoid name conflict with framework new_var_name = paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join(["cast@RESHARD", 'tmp'])) - out = block.create_var(name=new_var_name, - dtype=tensor_type, - type=tensor.type, - lod_level=tensor.lod_level) - cast_op = block._insert_op(idx, - type='cast', - inputs={'X': [tensor]}, - outputs={'Out': [out]}, - attrs={ - 'in_dtype': tensor.dtype, - 'out_dtype': out.dtype, - 'op_role': op_role - }) + ".".join(["cast@RESHARD", 'tmp']) + ) + out = block.create_var( + name=new_var_name, + dtype=tensor_type, + type=tensor.type, + lod_level=tensor.lod_level, + ) + cast_op = block._insert_op( + idx, + type='cast', + inputs={'X': [tensor]}, + outputs={'Out': [out]}, + attrs={ + 'in_dtype': tensor.dtype, + 'out_dtype': out.dtype, + 'op_role': op_role, + }, + ) cast_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -287,16 +296,18 @@ class Inserter: op_type = 'send_v2' # use pair comm group process_group = new_process_group([src, dst]) - send_op = block._insert_op(idx, - type=op_type, - inputs={'X': [tensor]}, - attrs={ - 'ring_id': process_group.id, - 'peer': process_group.ranks.index(dst), - 'use_calc_stream': True, - 'op_role': op_role, - 'dynamic_shape': True - }) + send_op = block._insert_op( + idx, + type=op_type, + inputs={'X': [tensor]}, + attrs={ + 'ring_id': process_group.id, + 'peer': process_group.ranks.index(dst), + 'use_calc_stream': True, + 'op_role': op_role, + 'dynamic_shape': True, + }, + ) send_op._set_attr('op_namescope', "/auto_parallel/reshard") @staticmethod @@ -305,19 +316,21 @@ class Inserter: op_type = 'recv_v2' # use pair group process_group = new_process_group([src, dst]) - recv_op = block._insert_op(idx, - type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': process_group.id, - 'peer': process_group.ranks.index(src), - 'out_shape': tensor.shape, - 'dtype': tensor.dtype, - 'use_calc_stream': True, - 'op_role': op_role, - 'dynamic_shape': True - }) + recv_op = block._insert_op( + idx, + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [tensor]}, + attrs={ + 'ring_id': process_group.id, + 'peer': process_group.ranks.index(src), + 'out_shape': tensor.shape, + 'dtype': tensor.dtype, + 'use_calc_stream': True, + 'op_role': op_role, + 'dynamic_shape': True, + }, + ) recv_op._set_attr('op_namescope', "/auto_parallel/reshard") @staticmethod @@ -325,21 +338,23 @@ class Inserter: """Insert reset_lod op into block at the given index.""" new_var_name = paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join(["reset_lod@RESHARD", 'tmp'])) - reset_lod_out = block.create_var(name=new_var_name, - shape=X.shape, - type=X.type, - dtype=X.dtype, - lod_level=X.lod_level) - - reset_op = block._insert_op(idx, - type="lod_reset", - inputs={ - 'X': X, - 'Y': Y - }, - outputs={'Out': reset_lod_out}, - attrs={'op_role': op_role}) + ".".join(["reset_lod@RESHARD", 'tmp']) + ) + reset_lod_out = block.create_var( + name=new_var_name, + shape=X.shape, + type=X.type, + dtype=X.dtype, + lod_level=X.lod_level, + ) + + reset_op = block._insert_op( + idx, + type="lod_reset", + inputs={'X': X, 'Y': Y}, + outputs={'Out': reset_lod_out}, + attrs={'op_role': op_role}, + ) reset_op._set_attr('op_namescope', "/auto_parallel/reshard") return reset_lod_out @@ -355,24 +370,29 @@ class Inserter: with paddle.static.program_guard(block.program): out = block.create_var( name=paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join([helper.name, 'tmp'])), + ".".join([helper.name, 'tmp']) + ), dtype=tensors[0].dtype, shape=None, lod_level=tensors[0].lod_level, type=tensors[0].type, persistable=False, - stop_gradient=False) - concat_op = block._insert_op(idx, - type='concat', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + stop_gradient=False, + ) + concat_op = block._insert_op( + idx, + type='concat', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + ) concat_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @staticmethod - def insert_slice_op(block, idx, tensor, starts, ends, axes, new_var_name, - op_role): + def insert_slice_op( + block, idx, tensor, starts, ends, axes, new_var_name, op_role + ): """Insert slice op into block at the given block.""" # This is a hack to insert split op to get slice tensor # 1. [128, 128] => [64, 128]: split @@ -387,19 +407,19 @@ class Inserter: # use assign if len(diff_dims) == 0: - out = block.create_var(name=new_var_name, - dtype=tensor.dtype, - type=tensor.type, - shape=slice_shape, - lod_level=tensor.lod_level) + out = block.create_var( + name=new_var_name, + dtype=tensor.dtype, + type=tensor.type, + shape=slice_shape, + lod_level=tensor.lod_level, + ) inputs = {'X': [tensor]} outputs = {"Out": [out]} attrs = {"in_place": False} - slice_op = block._insert_op(idx, - type="assign", - inputs=inputs, - outputs=outputs, - attrs=attrs) + slice_op = block._insert_op( + idx, type="assign", inputs=inputs, outputs=outputs, attrs=attrs + ) slice_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -420,23 +440,27 @@ class Inserter: new_shape.append(item // num_or_sections) with paddle.static.program_guard(block.program): outs = [ - block.create_var(name=paddle.fluid.unique_name. - generate_with_ignorable_key(".".join( - ['split@RESHARD', 'tmp'])), - dtype=tensor.dtype, - shape=None, - type=tensor.type, - persistable=False, - lod_level=tensor.lod_level, - stop_gradient=False) + block.create_var( + name=paddle.fluid.unique_name.generate_with_ignorable_key( + ".".join(['split@RESHARD', 'tmp']) + ), + dtype=tensor.dtype, + shape=None, + type=tensor.type, + persistable=False, + lod_level=tensor.lod_level, + stop_gradient=False, + ) for i in range(num_or_sections) ] out = outs[cur_idx] - split_op = block._insert_op(idx, - type="split", - inputs=inputs, - outputs={'Out': outs}, - attrs=attrs) + split_op = block._insert_op( + idx, + type="split", + inputs=inputs, + outputs={'Out': outs}, + attrs=attrs, + ) split_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -449,17 +473,21 @@ class Inserter: "starts": starts, "ends": ends, "infer_flags": infer_flags, - 'op_role': op_role + 'op_role': op_role, } - out = block.create_var(name=new_var_name, - dtype=tensor.dtype, - type=tensor.type, - lod_level=tensor.lod_level) - slice_op = block._insert_op(idx, - type="slice", - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + out = block.create_var( + name=new_var_name, + dtype=tensor.dtype, + type=tensor.type, + lod_level=tensor.lod_level, + ) + slice_op = block._insert_op( + idx, + type="slice", + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + ) slice_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -480,19 +508,20 @@ class Inserter: outs = [ block.create_var( name=paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join([helper.name, 'tmp'])), + ".".join([helper.name, 'tmp']) + ), dtype=tensor.dtype, shape=None, lod_level=tensor.lod_level, type=tensor.type, persistable=False, - stop_gradient=False) for i in range(num_or_sections) + stop_gradient=False, + ) + for i in range(num_or_sections) ] - split_op = block._insert_op(idx, - type="split", - inputs=inputs, - outputs={'Out': outs}, - attrs=attrs) + split_op = block._insert_op( + idx, type="split", inputs=inputs, outputs={'Out': outs}, attrs=attrs + ) split_op._set_attr('op_namescope', "/auto_parallel/reshard") return outs @@ -505,27 +534,30 @@ class Inserter: with paddle.static.program_guard(block.program): out = block.create_var( name=paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join([helper.name, 'tmp'])), + ".".join([helper.name, 'tmp']) + ), dtype=paddle.int64, shape=None, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) inputs = {} attrs = {'force_cpu': False} attrs['str_value'] = str(int("1")) attrs['value'] = int("1") attrs['dtype'] = out.dtype attrs['op_role'] = op_role - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=[0], - op_type='fill_constant') - fillconstant_op = block._insert_op(idx, - type='fill_constant', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=[0], op_type='fill_constant' + ) + fillconstant_op = block._insert_op( + idx, + type='fill_constant', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + ) out.stop_gradient = True fillconstant_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -541,7 +573,8 @@ class Inserter: if not group.is_instantiate(): # insert fill_constant op fill_constant_out = Inserter.insert_fill_constant_op( - block, idx, op_role) + block, idx, op_role + ) fill_constant_out.stop_gradient = True # insert c_allreduce_sum op @@ -553,8 +586,9 @@ class Inserter: attrs={ 'ring_id': 0, 'use_calc_stream': True, - 'op_role': op_role - }) + 'op_role': op_role, + }, + ) allreduce_op._set_attr('op_namescope', "/auto_parallel/reshard") # insert c_sync_calc_stream op sync_calc_op = block._insert_op( @@ -562,7 +596,8 @@ class Inserter: type="c_sync_calc_stream", inputs={'X': [fill_constant_out]}, outputs={'Out': [fill_constant_out]}, - attrs={'op_role': op_role}) + attrs={'op_role': op_role}, + ) sync_calc_op._set_attr('op_namescope', "/auto_parallel/reshard") idx_offset = 3 @@ -573,37 +608,42 @@ class Inserter: with paddle.static.program_guard(block.program): allgather_out = block.create_var( name=paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join([helper.name, 'tmp'])), + ".".join([helper.name, 'tmp']) + ), dtype=tensor.dtype, shape=None, lod_level=tensor.lod_level, type=tensor.type, persistable=False, - stop_gradient=False) - allgather_op = block._insert_op(idx + idx_offset, - type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [allgather_out]}, - attrs={ - 'ring_id': group.id, - 'use_calc_stream': True, - 'nranks': group.nranks, - 'op_role': op_role - }) + stop_gradient=False, + ) + allgather_op = block._insert_op( + idx + idx_offset, + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [allgather_out]}, + attrs={ + 'ring_id': group.id, + 'use_calc_stream': True, + 'nranks': group.nranks, + 'op_role': op_role, + }, + ) allgather_op._set_attr('op_namescope', "/auto_parallel/reshard") idx_offset += 1 # insert split op - split_out = Inserter.insert_split_op(block, idx + idx_offset, - allgather_out, group.nranks, - op_role) + split_out = Inserter.insert_split_op( + block, idx + idx_offset, allgather_out, group.nranks, op_role + ) idx_offset += 1 tensor_list.extend(split_out) return tensor_list, idx_offset @staticmethod - def concat_partitions_with_op(partition_tensor_list, tensor, - partition_index, block, idx, op_role): + def concat_partitions_with_op( + partition_tensor_list, tensor, partition_index, block, idx, op_role + ): """Concat the tensors and insert concat op.""" if not partition_tensor_list: partition_tensor_list.append((tensor, partition_index)) @@ -611,18 +651,42 @@ class Inserter: i = 0 has_concat = False while i < len(partition_tensor_list): - concat_axis, first_order, new_partition = Resharder.compute_concat_info( - partition_tensor_list[i][1], partition_index) + ( + concat_axis, + first_order, + new_partition, + ) = Resharder.compute_concat_info( + partition_tensor_list[i][1], partition_index + ) if concat_axis != -1: has_concat = True - _ = Inserter.insert_concat_op(block, idx[0], [partition_tensor_list[i][0], tensor], concat_axis, op_role) \ - if first_order == 0 else \ - Inserter.insert_concat_op(block, idx[0], [tensor, partition_tensor_list[i][0]], concat_axis, op_role) + _ = ( + Inserter.insert_concat_op( + block, + idx[0], + [partition_tensor_list[i][0], tensor], + concat_axis, + op_role, + ) + if first_order == 0 + else Inserter.insert_concat_op( + block, + idx[0], + [tensor, partition_tensor_list[i][0]], + concat_axis, + op_role, + ) + ) partition_tensor_list.pop(i) idx[0] += 1 - Inserter.concat_partitions_with_op(partition_tensor_list, _, - new_partition, block, - idx, op_role) + Inserter.concat_partitions_with_op( + partition_tensor_list, + _, + new_partition, + block, + idx, + op_role, + ) break i += 1 if not has_concat: @@ -636,7 +700,9 @@ class Remover: def remove_no_need_ops(auto_parallel_main_prog, dist_context, rank_id): """Remove no need ops in the main program""" not_remove_op_ref = [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ] # NOTE: The nested sub block is not be supported now. @@ -660,7 +726,9 @@ class Remover: for var_name in op.output_arg_names: dim_list.extend( get_var_with_recursion( - var_name, block, auto_parallel_main_prog).shape) + var_name, block, auto_parallel_main_prog + ).shape + ) for i in range(idx, -1, -1): if ops[i].type == "create_py_reader": ops[i]._set_attr("shape_concat", dim_list) @@ -671,10 +739,13 @@ class Remover: if op.type == "c_sync_comm_stream": need_save = [] for var_name in op.input_arg_names: - process_mesh = dist_context.get_tensor_dist_attr_for_program( - get_var_with_recursion( - var_name, block, - auto_parallel_main_prog)).process_mesh + process_mesh = ( + dist_context.get_tensor_dist_attr_for_program( + get_var_with_recursion( + var_name, block, auto_parallel_main_prog + ) + ).process_mesh + ) if rank_id in process_mesh.processes: need_save.append(var_name) if not need_save: @@ -690,15 +761,19 @@ class Remover: op_dist_attr = dist_context.get_op_dist_attr_for_program(op) if op_dist_attr is not None: op_process_mesh = op_dist_attr.process_mesh - if rank_id not in op_process_mesh.processes and op.type not in not_remove_op_ref: + if ( + rank_id not in op_process_mesh.processes + and op.type not in not_remove_op_ref + ): remove_op_idx.append(idx) for idx in remove_op_idx[::-1]: block._remove_op(idx) @staticmethod - def remove_no_need_vars(auto_parallel_main_prog, dist_params_grads, - feed_var_names): + def remove_no_need_vars( + auto_parallel_main_prog, dist_params_grads, feed_var_names + ): """Remove no need vars in the main program""" for block_idx, block in enumerate(auto_parallel_main_prog.blocks): remove_vars = set() @@ -721,7 +796,10 @@ class Remover: param_grad_map = {} for op in ops: if int(op.attr('op_role')) == int(OpRole.Optimize): - if "Param" in op.input_names and "Grad" in op.input_names: + if ( + "Param" in op.input_names + and "Grad" in op.input_names + ): param_name = op.input("Param")[0] grad_name = op.input("Grad")[0] param_grad_map[param_name] = grad_name @@ -740,7 +818,9 @@ class Remover: grad_name = dist_params_grads[idx][1].name if grad_name != param_grad_map[param_name]: dist_params_grads[idx] = ( - vars[param_name], vars[param_grad_map[param_name]]) + vars[param_name], + vars[param_grad_map[param_name]], + ) idx += 1 for var in remove_vars: @@ -749,23 +829,28 @@ class Remover: block._remove_var(var) @staticmethod - def remove_no_need_in_main(auto_parallel_main_prog, dist_context, rank_id, - dist_params_grads): + def remove_no_need_in_main( + auto_parallel_main_prog, dist_context, rank_id, dist_params_grads + ): """Remove no need vars and ops in the main program.""" - Remover.remove_no_need_ops(auto_parallel_main_prog, dist_context, - rank_id) - Resharder.change_while_op_input_and_output(auto_parallel_main_prog, - dist_context) + Remover.remove_no_need_ops( + auto_parallel_main_prog, dist_context, rank_id + ) + Resharder.change_while_op_input_and_output( + auto_parallel_main_prog, dist_context + ) # 'feed_var_names' cannot be removed from auto_parallel_main_prog feed_var_names = [] for var in sum(list(dist_context.serial_feed_vars.values()), []): feed_var_names.append(var.name) - Remover.remove_no_need_vars(auto_parallel_main_prog, dist_params_grads, - feed_var_names) + Remover.remove_no_need_vars( + auto_parallel_main_prog, dist_params_grads, feed_var_names + ) @staticmethod - def remove_no_need_in_startup(auto_parallel_main_prog, - auto_parallel_startup_prog): + def remove_no_need_in_startup( + auto_parallel_main_prog, auto_parallel_startup_prog + ): """Remove no need vars and ops in the startup program.""" main_input_vars = set() main_ops = auto_parallel_main_prog.global_block().ops @@ -850,28 +935,43 @@ class Resharder: dist_params_grads (list): The list contains the tuple of param and grad. batch_size (int): The batch size. Default: None. """ + while_block_info = {} - def __init__(self, - auto_parallel_main_prog, - auto_parallel_startup_prog, - rank_id, - dist_context, - dist_params_grads, - batch_size=None): - assert isinstance(auto_parallel_main_prog, Program), "The type of auto_parallel_main_prog should be Program, " \ - "but got {}.".format(type(auto_parallel_main_prog)) + def __init__( + self, + auto_parallel_main_prog, + auto_parallel_startup_prog, + rank_id, + dist_context, + dist_params_grads, + batch_size=None, + ): + assert isinstance(auto_parallel_main_prog, Program), ( + "The type of auto_parallel_main_prog should be Program, " + "but got {}.".format(type(auto_parallel_main_prog)) + ) if auto_parallel_startup_prog is not None: - assert isinstance(auto_parallel_main_prog, Program), "The type of auto_parallel_startup_prog should be Program or None, " \ - "but got {}.".format(type(auto_parallel_startup_prog)) - assert isinstance(rank_id, int), "The type of rank_id should be int, " \ - "but got {}.".format(type(rank_id)) - assert isinstance(dist_context, DistributedContext), "The type of dist_context should be DistributedContext, " \ - "but got {}.".format(type(dist_context)) + assert isinstance(auto_parallel_main_prog, Program), ( + "The type of auto_parallel_startup_prog should be Program or None, " + "but got {}.".format(type(auto_parallel_startup_prog)) + ) + assert isinstance( + rank_id, int + ), "The type of rank_id should be int, " "but got {}.".format( + type(rank_id) + ) + assert isinstance(dist_context, DistributedContext), ( + "The type of dist_context should be DistributedContext, " + "but got {}.".format(type(dist_context)) + ) if batch_size is not None: - assert isinstance(batch_size, int), "The type of batch_size should be int, " \ - "but got {}.".format(type(batch_size)) + assert isinstance( + batch_size, int + ), "The type of batch_size should be int, " "but got {}.".format( + type(batch_size) + ) self._auto_parallel_main_prog = auto_parallel_main_prog self._auto_parallel_startup_prog = auto_parallel_startup_prog @@ -943,29 +1043,37 @@ class Resharder: for i in range(len(process_shape)): idx = relative_process // (product // process_shape[i]) product = product // process_shape[i] - relative_process = relative_process - relative_process // product * product + relative_process = ( + relative_process - relative_process // product * product + ) process_index.append(idx) return process_index @staticmethod - def compute_partition_index(process, complete_shape, dims_mapping, - process_shape, process_group): + def compute_partition_index( + process, complete_shape, dims_mapping, process_shape, process_group + ): """Compute the partition index in complete tensor.""" partition_shape = Resharder.compute_partition_shape( - complete_shape, dims_mapping, process_shape) - process_index = Resharder.compute_process_index(process, process_group, - process_shape) + complete_shape, dims_mapping, process_shape + ) + process_index = Resharder.compute_process_index( + process, process_group, process_shape + ) partition_index = [] for i in range(len(complete_shape)): if dims_mapping[i] == -1: partition_index.append([0, partition_shape[i]]) else: - partition_index.append([ - process_index[dims_mapping[i]] * partition_shape[i], - (process_index[dims_mapping[i]] + 1) * partition_shape[i] - ]) + partition_index.append( + [ + process_index[dims_mapping[i]] * partition_shape[i], + (process_index[dims_mapping[i]] + 1) + * partition_shape[i], + ] + ) return partition_index @@ -980,12 +1088,16 @@ class Resharder: for idx, item in enumerate(partition_index_x): if item != partition_index_y[idx]: differ_count += 1 - if item[1] == partition_index_y[idx][ - 0] and item[0] < partition_index_y[idx][1]: + if ( + item[1] == partition_index_y[idx][0] + and item[0] < partition_index_y[idx][1] + ): concat_axis = idx new_partition.append([item[0], partition_index_y[idx][1]]) - elif item[0] == partition_index_y[idx][ - 1] and item[1] > partition_index_y[idx][0]: + elif ( + item[0] == partition_index_y[idx][1] + and item[1] > partition_index_y[idx][0] + ): first_order = 1 concat_axis = idx new_partition.append([partition_index_y[idx][0], item[1]]) @@ -1018,12 +1130,14 @@ class Resharder: has_concat = False while i < len(partition_index_list): concat_axis, _, new_partition = Resharder.compute_concat_info( - partition_index_list[i], partition_index) + partition_index_list[i], partition_index + ) if concat_axis != -1: has_concat = True partition_index_list.pop(i) - Resharder.concat_partitions(partition_index_list, - new_partition) + Resharder.concat_partitions( + partition_index_list, new_partition + ) break i += 1 if not has_concat: @@ -1035,7 +1149,8 @@ class Resharder: for sub_block_idx in Resharder.while_block_info: sub_block = auto_parallel_main_prog.blocks[sub_block_idx] parent_while_op_id = Resharder.while_block_info[sub_block_idx][ - "op_id"] + "op_id" + ] parent_block = auto_parallel_main_prog.blocks[sub_block.parent_idx] sub_block_op_inputs = set() @@ -1043,10 +1158,12 @@ class Resharder: for op in sub_block.ops: # skip the input and output of operators inserted in the reshard phase dist_op = dist_context.get_dist_op_for_program(op) - if dist_op or (op.type == "slice" and not dist_op) or ( - op.type == "split" - and not dist_op) or (op.type == "assign" - and not dist_op): + if ( + dist_op + or (op.type == "slice" and not dist_op) + or (op.type == "split" and not dist_op) + or (op.type == "assign" and not dist_op) + ): for var_name in op.output_arg_names: if var_name not in sub_block_op_outputs: sub_block_op_outputs.append(var_name) @@ -1077,8 +1194,9 @@ class Resharder: for var_name in while_op.output("Out"): for output_name in sub_block_op_outputs[::-1]: if output_name.find(var_name) != -1 and ( - len(var_name) == len(output_name) - or "@RESHARD" in output_name): + len(var_name) == len(output_name) + or "@RESHARD" in output_name + ): if output_name not in new_Out: new_Out.append(output_name) assert new_Out @@ -1087,8 +1205,9 @@ class Resharder: def is_overlapped(self, shape_x, shape_y): """Judge whether two partitions intersect on the specified dimension.""" overlapped = False - if (shape_y[0] <= shape_x[0] < shape_y[1]) or (shape_x[0] <= shape_y[0] - < shape_x[1]): + if (shape_y[0] <= shape_x[0] < shape_y[1]) or ( + shape_x[0] <= shape_y[0] < shape_x[1] + ): overlapped = True return overlapped @@ -1116,8 +1235,9 @@ class Resharder: # the dims mapping of condition tensor should be replicative for var_name in input_cond: - var = get_var_with_recursion(var_name, sub_block, - self.auto_parallel_main_prog) + var = get_var_with_recursion( + var_name, sub_block, self.auto_parallel_main_prog + ) dist_tensor = self.dist_context.get_dist_tensor_for_program(var) tensor_dist_attr = dist_tensor.dist_attr var_dims_mapping = tensor_dist_attr.dims_mapping @@ -1140,13 +1260,22 @@ class Resharder: if op_input: op_input_dims_mapping = dist_attr[1] if all( - map(lambda x: x, [ - tensor_dims_mapping, tensor_process_mesh, - op_input_dims_mapping, op_process_mesh - ])): + map( + lambda x: x, + [ + tensor_dims_mapping, + tensor_process_mesh, + op_input_dims_mapping, + op_process_mesh, + ], + ) + ): # judge whether need reshard by dims_mapping if tensor_dims_mapping != op_input_dims_mapping: - if tensor_process_mesh not in self.dist_context.process_meshes: + if ( + tensor_process_mesh + not in self.dist_context.process_meshes + ): # assert whether -1 when union. for item in tensor_dims_mapping: if item != -1: @@ -1170,10 +1299,16 @@ class Resharder: else: op_output_dims_mapping = dist_attr[1] if all( - map(lambda x: x, [ - tensor_dims_mapping, tensor_process_mesh, - op_output_dims_mapping, op_process_mesh - ])): + map( + lambda x: x, + [ + tensor_dims_mapping, + tensor_process_mesh, + op_output_dims_mapping, + op_process_mesh, + ], + ) + ): if tensor_dims_mapping != op_output_dims_mapping: raise ValueError( "It is not supported that tensor dims mapping is different from op output dims mapping." @@ -1190,10 +1325,9 @@ class Resharder: op_process_mesh = dist_op.dist_attr.process_mesh for process_mesh in self.dist_context.process_meshes: - if set(process_mesh.processes) & (set( - op_process_mesh.processes)) and len( - process_mesh.processes) < len( - op_process_mesh.processes): + if set(process_mesh.processes) & ( + set(op_process_mesh.processes) + ) and len(process_mesh.processes) < len(op_process_mesh.processes): process_meshes.append(process_mesh) # it means the process mesh is not a union when process meshes is null @@ -1235,34 +1369,47 @@ class Resharder: new_shape[0] = self.batch_size source_tensor.desc.set_shape(new_shape) - complete_shape = Resharder.compute_complete_shape( - source_tensor.shape, source_process_shape, - source_dims_mapping) if not serial else source_tensor.shape + complete_shape = ( + Resharder.compute_complete_shape( + source_tensor.shape, source_process_shape, source_dims_mapping + ) + if not serial + else source_tensor.shape + ) op_desc_seq = {} # TODO: if the target process group has the same process with source process group - if set(target_process_group).intersection(set( - source_process_group)) and set(target_process_group).difference( - set(source_process_group)): + if set(target_process_group).intersection( + set(source_process_group) + ) and set(target_process_group).difference(set(source_process_group)): pass elif target_process_group != source_process_group: partition_process_mapping_list = [] for source_process in source_process_group: # get partition index of source process - source_partition_index = Resharder.compute_partition_index(source_process, complete_shape, source_dims_mapping, \ - source_process_shape, source_process_group) + source_partition_index = Resharder.compute_partition_index( + source_process, + complete_shape, + source_dims_mapping, + source_process_shape, + source_process_group, + ) if not partition_process_mapping_list: # the item in partition_process_mapping_list is source_partition_index, which processes and whether has been used partition_process_mapping_list.append( - [source_partition_index, [source_process], [False]]) + [source_partition_index, [source_process], [False]] + ) else: partition_list = list( - [item[0] for item in partition_process_mapping_list]) + [item[0] for item in partition_process_mapping_list] + ) process_list = list( - [item[1] for item in partition_process_mapping_list]) + [item[1] for item in partition_process_mapping_list] + ) has_used = list( - [item[2] for item in partition_process_mapping_list]) + [item[2] for item in partition_process_mapping_list] + ) if partition_list.count(source_partition_index) == 1: index = partition_list.index(source_partition_index) @@ -1270,32 +1417,52 @@ class Resharder: has_used[index].append(False) else: partition_process_mapping_list.append( - [source_partition_index, [source_process], [False]]) + [source_partition_index, [source_process], [False]] + ) for target_process in target_process_group: # has_sent means the source_partition_index has been sent to target_process has_sent = [] target_partition_index = Resharder.compute_partition_index( - target_process, complete_shape, target_dims_mapping, - target_process_shape, target_process_group) + target_process, + complete_shape, + target_dims_mapping, + target_process_shape, + target_process_group, + ) partition_index_list = [] all_partition_index_list = [] for source_process in source_process_group: source_partition_index = Resharder.compute_partition_index( - source_process, complete_shape, source_dims_mapping, - source_process_shape, source_process_group) + source_process, + complete_shape, + source_dims_mapping, + source_process_shape, + source_process_group, + ) to_send_process = None - if all(_ for _ in list(map(self.is_overlapped, source_partition_index, target_partition_index))) \ - and source_partition_index not in has_sent: - idx = list([ - item[0] for item in partition_process_mapping_list - ]).index(source_partition_index) - has_used = list([ - item[2] for item in partition_process_mapping_list - ])[idx] - process_list = list([ - item[1] for item in partition_process_mapping_list - ])[idx] + if ( + all( + _ + for _ in list( + map( + self.is_overlapped, + source_partition_index, + target_partition_index, + ) + ) + ) + and source_partition_index not in has_sent + ): + idx = list( + [item[0] for item in partition_process_mapping_list] + ).index(source_partition_index) + has_used = list( + [item[2] for item in partition_process_mapping_list] + )[idx] + process_list = list( + [item[1] for item in partition_process_mapping_list] + )[idx] i = 0 while i < len(has_used): if not has_used[i]: @@ -1308,7 +1475,9 @@ class Resharder: has_used = list(map(lambda x: False, has_used)) to_send_process = process_list[0] has_used[0] = True - assert to_send_process is not None, "Failed to find the send process." + assert ( + to_send_process is not None + ), "Failed to find the send process." if to_send_process not in op_desc_seq.keys(): op_desc_seq[to_send_process] = [] @@ -1317,25 +1486,30 @@ class Resharder: all_partition_index_list.append(source_partition_index) # append send and recv op desc - is_bool = ( - dist_tensor.serial_tensor.dtype == paddle.bool) - send_op_desc = SendOpDesc(source_partition_index, - to_send_process, - target_process, - is_bool=is_bool) - recv_op_desc = RecvOpDesc(source_partition_index, - to_send_process, - target_process, - is_bool=is_bool) + is_bool = dist_tensor.serial_tensor.dtype == paddle.bool + send_op_desc = SendOpDesc( + source_partition_index, + to_send_process, + target_process, + is_bool=is_bool, + ) + recv_op_desc = RecvOpDesc( + source_partition_index, + to_send_process, + target_process, + is_bool=is_bool, + ) op_desc_seq[to_send_process].append(send_op_desc) op_desc_seq[target_process].append(recv_op_desc) has_sent.append(source_partition_index) - Resharder.concat_partitions(partition_index_list, - source_partition_index) + Resharder.concat_partitions( + partition_index_list, source_partition_index + ) # append concat op desc op_desc_seq[target_process].append( - ConcatOpDesc(all_partition_index_list)) + ConcatOpDesc(all_partition_index_list) + ) # append slice op desc slice_starts = [] @@ -1345,17 +1519,21 @@ class Resharder: to_slice_tensor_shape = [] for idx, item in enumerate(concatenated_partition_index): - slice_starts.append(target_partition_index[idx][0] - - item[0]) + slice_starts.append( + target_partition_index[idx][0] - item[0] + ) slice_ends.append(target_partition_index[idx][1] - item[0]) slices_axes.append(idx) to_slice_tensor_shape.append(item[1] - item[0]) op_desc_seq[target_process].append( - SliceOpDesc(slice_starts, - slice_ends, - slices_axes, - shape=to_slice_tensor_shape)) + SliceOpDesc( + slice_starts, + slice_ends, + slices_axes, + shape=to_slice_tensor_shape, + ) + ) # in the same process group, it will use allgahther and slice op. else: @@ -1365,16 +1543,26 @@ class Resharder: process_index = [] for source_process in source_process_group: source_partition_index = Resharder.compute_partition_index( - source_process, complete_shape, source_dims_mapping, - source_process_shape, source_process_group) + source_process, + complete_shape, + source_dims_mapping, + source_process_shape, + source_process_group, + ) if source_partition_index not in partition_index_list: partition_index_list.append(source_partition_index) - process_index.append([[ - source_process, - ], source_partition_index]) + process_index.append( + [ + [ + source_process, + ], + source_partition_index, + ] + ) else: - process_index[partition_index_list.index( - source_partition_index)][0].append(source_process) + process_index[ + partition_index_list.index(source_partition_index) + ][0].append(source_process) for i in range(len(process_index[0][0])): group = [] @@ -1388,28 +1576,50 @@ class Resharder: slice_ends = [] slices_axes = [] target_partition_index = Resharder.compute_partition_index( - process, complete_shape, target_dims_mapping, - target_process_shape, target_process_group) + process, + complete_shape, + target_dims_mapping, + target_process_shape, + target_process_group, + ) for idx, item in enumerate(target_partition_index): slice_starts.append(item[0]) slice_ends.append(item[1]) slices_axes.append(idx) to_slice_tensor_shape = dist_tensor.global_sizes() - slice_op_desc = SliceOpDesc(starts=slice_starts, - ends=slice_ends, - axes=slices_axes, - shape=to_slice_tensor_shape) - allgather_shape = None if not serial else dist_tensor.local_sizes( - rank=process) - op_desc_seq[process] = [AllGatherOpDesc(group=group, shape=allgather_shape, is_bool=(source_tensor.dtype == paddle.bool)), - ConcatOpDesc(partition_index_list=all_partition_index_list), slice_op_desc] \ - if len(group) > 1 else [slice_op_desc] + slice_op_desc = SliceOpDesc( + starts=slice_starts, + ends=slice_ends, + axes=slices_axes, + shape=to_slice_tensor_shape, + ) + allgather_shape = ( + None + if not serial + else dist_tensor.local_sizes(rank=process) + ) + op_desc_seq[process] = ( + [ + AllGatherOpDesc( + group=group, + shape=allgather_shape, + is_bool=(source_tensor.dtype == paddle.bool), + ), + ConcatOpDesc( + partition_index_list=all_partition_index_list + ), + slice_op_desc, + ] + if len(group) > 1 + else [slice_op_desc] + ) return op_desc_seq - def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, - dist_attr): + def parse_op_desc( + self, block, op_desc_seq, var_name, reshard_op, dist_attr + ): """Parse op desc sequence and insert op in the block""" tensor_list = [] partition_tensor_list = [] @@ -1422,55 +1632,84 @@ class Resharder: if op.desc.id == reshard_op.desc.id: idx = index break - assert idx is not None, "The op for reshard cannot be found in the rank {} program.".format( - self.rank_id) + assert ( + idx is not None + ), "The op for reshard cannot be found in the rank {} program.".format( + self.rank_id + ) matched_op = block.ops[idx] - source_tensor = get_var_with_recursion(var_name, block, - self.auto_parallel_main_prog) + source_tensor = get_var_with_recursion( + var_name, block, self.auto_parallel_main_prog + ) for op_desc in op_desc_list: if isinstance(op_desc, AllGatherOpDesc): # noqa: F401 if var_name not in self.has_allgather.keys(): self.has_allgather[var_name] = [] - if not self.has_allgather[var_name] or op_desc.group not in list( - map(lambda x: x[0], self.has_allgather[var_name])): + if not self.has_allgather[ + var_name + ] or op_desc.group not in list( + map(lambda x: x[0], self.has_allgather[var_name]) + ): if op_desc.is_bool: # for bool data allgather, cast to int64 -> allgather -> cast bool out_cast = Inserter.insert_cast_op( - block, idx, source_tensor, - reshard_op.attr('op_role'), paddle.int64) + block, + idx, + source_tensor, + reshard_op.attr('op_role'), + paddle.int64, + ) tensor_list, idx_offset = Inserter.insert_allgather_op( - block, idx + 1, out_cast, op_desc.group, - reshard_op.attr('op_role')) + block, + idx + 1, + out_cast, + op_desc.group, + reshard_op.attr('op_role'), + ) idx += idx_offset tensor_name_list = [] for var in tensor_list: out_cast = Inserter.insert_cast_op( - block, idx, var, reshard_op.attr('op_role'), - paddle.bool) + block, + idx, + var, + reshard_op.attr('op_role'), + paddle.bool, + ) tensor_name_list.append(out_cast.name) idx += 1 self.has_allgather[var_name].append( - [op_desc.group, tensor_name_list]) + [op_desc.group, tensor_name_list] + ) else: tensor_list, idx_offset = Inserter.insert_allgather_op( - block, idx, source_tensor, op_desc.group, - reshard_op.attr('op_role')) + block, + idx, + source_tensor, + op_desc.group, + reshard_op.attr('op_role'), + ) idx += idx_offset tensor_name_list = [var.name for var in tensor_list] self.has_allgather[var_name].append( - [op_desc.group, tensor_name_list]) + [op_desc.group, tensor_name_list] + ) else: for item in self.has_allgather[var_name]: if op_desc.group == item[0]: tensor_list = [ get_var_with_recursion( - var_name, block, - self.auto_parallel_main_prog) + var_name, + block, + self.auto_parallel_main_prog, + ) for var_name in item[1] ] break - assert tensor_list, "The result of parsing allgather op should not be None." + assert ( + tensor_list + ), "The result of parsing allgather op should not be None." elif isinstance(op_desc, SendOpDesc): if var_name not in self.has_sent.keys(): @@ -1478,16 +1717,30 @@ class Resharder: if op_desc.dst not in self.has_sent[var_name]: if op_desc.is_bool: out_cast = Inserter.insert_cast_op( - block, idx, source_tensor, - reshard_op.attr('op_role'), paddle.int64) - Inserter.insert_send_op(block, idx + 1, out_cast, - op_desc.src, op_desc.dst, - reshard_op.attr('op_role')) + block, + idx, + source_tensor, + reshard_op.attr('op_role'), + paddle.int64, + ) + Inserter.insert_send_op( + block, + idx + 1, + out_cast, + op_desc.src, + op_desc.dst, + reshard_op.attr('op_role'), + ) idx += 2 else: - Inserter.insert_send_op(block, idx, source_tensor, - op_desc.src, op_desc.dst, - reshard_op.attr('op_role')) + Inserter.insert_send_op( + block, + idx, + source_tensor, + op_desc.src, + op_desc.dst, + reshard_op.attr('op_role'), + ) idx += 1 self.has_sent[var_name].append(op_desc.dst) @@ -1506,13 +1759,23 @@ class Resharder: shape=shape, lod_level=source_tensor.lod_level, dtype=paddle.int64, - type=source_tensor.type) - Inserter.insert_recv_op(block, idx, recv_tensor, - op_desc.src, op_desc.dst, - reshard_op.attr('op_role')) + type=source_tensor.type, + ) + Inserter.insert_recv_op( + block, + idx, + recv_tensor, + op_desc.src, + op_desc.dst, + reshard_op.attr('op_role'), + ) out_cast = Inserter.insert_cast_op( - block, idx + 1, recv_tensor, - reshard_op.attr('op_role'), paddle.bool) + block, + idx + 1, + recv_tensor, + reshard_op.attr('op_role'), + paddle.bool, + ) tensor_list.append(out_cast) idx += 2 self.has_recv[var_name][op_desc.src] = out_cast @@ -1522,26 +1785,45 @@ class Resharder: shape=shape, lod_level=source_tensor.lod_level, dtype=source_tensor.dtype, - type=source_tensor.type) - Inserter.insert_recv_op(block, idx, recv_tensor, - op_desc.src, op_desc.dst, - reshard_op.attr('op_role')) + type=source_tensor.type, + ) + Inserter.insert_recv_op( + block, + idx, + recv_tensor, + op_desc.src, + op_desc.dst, + reshard_op.attr('op_role'), + ) # for lod tensor, need reset lod after received if recv_tensor.lod_level != 0: set_lod = False # use data lod to reset tensor lod - for tmp_block in self.auto_parallel_main_prog.blocks: + for ( + tmp_block + ) in self.auto_parallel_main_prog.blocks: for tmp_var_name in tmp_block.vars: tmp_var = tmp_block.vars[tmp_var_name] - if tmp_var.is_data and tmp_var.lod_level == recv_tensor.lod_level: - reset_lod_out = Inserter.insert_reset_lod_op( - block, idx + 1, recv_tensor, - tmp_var, reshard_op.attr('op_role')) + if ( + tmp_var.is_data + and tmp_var.lod_level + == recv_tensor.lod_level + ): + reset_lod_out = ( + Inserter.insert_reset_lod_op( + block, + idx + 1, + recv_tensor, + tmp_var, + reshard_op.attr('op_role'), + ) + ) tensor_list.append(reset_lod_out) idx += 2 self.has_recv[var_name][ - op_desc.src] = reset_lod_out + op_desc.src + ] = reset_lod_out set_lod = True break if set_lod: @@ -1559,16 +1841,24 @@ class Resharder: idx_list = [idx] for index, tensor in enumerate(tensor_list): Inserter.concat_partitions_with_op( - partition_tensor_list, tensor, - partition_index_list[index], block, idx_list, - reshard_op.attr('op_role')) + partition_tensor_list, + tensor, + partition_index_list[index], + block, + idx_list, + reshard_op.attr('op_role'), + ) idx = idx_list[0] elif isinstance(op_desc, SliceOpDesc): - assert len( - partition_tensor_list) == 1 or not partition_tensor_list - to_slice_tensor = partition_tensor_list[0][0] if len( - partition_tensor_list) == 1 else source_tensor + assert ( + len(partition_tensor_list) == 1 or not partition_tensor_list + ) + to_slice_tensor = ( + partition_tensor_list[0][0] + if len(partition_tensor_list) == 1 + else source_tensor + ) new_name = unique_name.generate(var_name + "@RESHARD") target_tensor = Inserter.insert_slice_op( block, @@ -1578,7 +1868,8 @@ class Resharder: ends=op_desc.ends, axes=op_desc.axes, new_var_name=new_name, - op_role=reshard_op.attr('op_role')) + op_role=reshard_op.attr('op_role'), + ) process_mesh = dist_attr[0] dims_mapping = dist_attr[1] @@ -1587,83 +1878,119 @@ class Resharder: tensor_attr.dims_mapping = dims_mapping tensor_attr.process_mesh = process_mesh self.dist_context.set_tensor_dist_attr_for_program( - target_tensor, tensor_attr) + target_tensor, tensor_attr + ) if matched_op.type == "while": # var_reshard_mapping means the while op input need be changed to - if "var_reshard_mapping" not in Resharder.while_block_info[ - op.attr("sub_block").id].keys(): - Resharder.while_block_info[op.attr( - "sub_block").id]["var_reshard_mapping"] = {} - if var_name not in Resharder.while_block_info[op.attr( - "sub_block").id]["var_reshard_mapping"].keys(): + if ( + "var_reshard_mapping" + not in Resharder.while_block_info[ + op.attr("sub_block").id + ].keys() + ): Resharder.while_block_info[op.attr("sub_block").id][ - "var_reshard_mapping"][var_name] = [] + "var_reshard_mapping" + ] = {} + if ( + var_name + not in Resharder.while_block_info[ + op.attr("sub_block").id + ]["var_reshard_mapping"].keys() + ): + Resharder.while_block_info[op.attr("sub_block").id][ + "var_reshard_mapping" + ][var_name] = [] Resharder.while_block_info[op.attr("sub_block").id][ - "var_reshard_mapping"][var_name].append( - [dist_attr, target_tensor.name]) + "var_reshard_mapping" + ][var_name].append([dist_attr, target_tensor.name]) # rename op input name according to new name for op in block.ops: # just for while op while_op_X_append = [] for name in op.input_arg_names: - op_dist_attr = self.dist_context.get_op_dist_attr_for_program( - op) + op_dist_attr = ( + self.dist_context.get_op_dist_attr_for_program(op) + ) if name == var_name and op_dist_attr is not None: if op.desc.id() == matched_op.desc.id(): if matched_op.type == "while": old_name = name new_name = target_tensor.name assert old_name != new_name - op_input_dist_attr = op_dist_attr.get_input_dist_attr( - old_name) + op_input_dist_attr = ( + op_dist_attr.get_input_dist_attr( + old_name + ) + ) op_dist_attr.set_input_dist_attr( - new_name, op_input_dist_attr) + new_name, op_input_dist_attr + ) op_dist_attr.set_input_dims_mapping( - new_name, dims_mapping) - if old_name in op_dist_attr._inputs_dist_attrs: + new_name, dims_mapping + ) + if ( + old_name + in op_dist_attr._inputs_dist_attrs + ): op_dist_attr.del_input_dist_attr( - old_name) + old_name + ) while_op_X_append.append(new_name) continue else: op.desc._rename_input( - name, target_tensor.name) + name, target_tensor.name + ) old_name = name new_name = target_tensor.name assert old_name != new_name - op_input_dist_attr = op_dist_attr.get_input_dist_attr( - old_name) + op_input_dist_attr = ( + op_dist_attr.get_input_dist_attr( + old_name + ) + ) op_dist_attr.set_input_dist_attr( - new_name, op_input_dist_attr) + new_name, op_input_dist_attr + ) op_dist_attr.set_input_dims_mapping( - new_name, dims_mapping) + new_name, dims_mapping + ) op_dist_attr.del_input_dist_attr(old_name) continue op_process_mesh = op_dist_attr.process_mesh - op_input_dims_mapping = op_dist_attr.get_input_dims_mapping( - var_name) + op_input_dims_mapping = ( + op_dist_attr.get_input_dims_mapping(var_name) + ) # NOTE: For op whose process mesh is a union, its input will not be renamed by other op reshard result now which means that it will have more reshard operation. - if op_process_mesh == process_mesh and op_input_dims_mapping == dims_mapping: + if ( + op_process_mesh == process_mesh + and op_input_dims_mapping == dims_mapping + ): op.desc._rename_input(name, target_tensor.name) old_name = name new_name = target_tensor.name assert old_name != new_name - op_input_dist_attr = op_dist_attr.get_input_dist_attr( - old_name) + op_input_dist_attr = ( + op_dist_attr.get_input_dist_attr(old_name) + ) op_dist_attr.set_input_dist_attr( - new_name, op_input_dist_attr) + new_name, op_input_dist_attr + ) op_dist_attr.set_input_dims_mapping( - new_name, dims_mapping) + new_name, dims_mapping + ) op_dist_attr.del_input_dist_attr(old_name) # for while op, the input X should reset if while_op_X_append: proto = OpProtoHolder.instance().get_op_proto(op.type) - op.desc.set_input(proto.inputs[0].name, - op.input("X") + while_op_X_append) + op.desc.set_input( + proto.inputs[0].name, + op.input("X") + while_op_X_append, + ) def _get_subblock_input_attrs(self, op, var_name): # NOTE: Multi while loop is not supported @@ -1681,11 +2008,14 @@ class Resharder: if name == var_name: process_mesh = dist_attr.process_mesh input_dims_mapping = dist_attr.get_input_dims_mapping( - var_name) + var_name + ) has_exist = False for input_attr in input_attrs: - if process_mesh == input_attr[ - 0] and input_dims_mapping == input_attr[1]: + if ( + process_mesh == input_attr[0] + and input_dims_mapping == input_attr[1] + ): has_exist = True break if not has_exist: @@ -1698,10 +2028,9 @@ class Resharder: dist_attr = dist_op.dist_attr op_process_mesh = dist_attr.process_mesh for process_mesh in self.dist_context.process_meshes: - if set(process_mesh.processes) & (set( - op_process_mesh.processes)) and len( - process_mesh.processes) < len( - op_process_mesh.processes): + if set(process_mesh.processes) & ( + set(op_process_mesh.processes) + ) and len(process_mesh.processes) < len(op_process_mesh.processes): process_meshes.append(process_mesh) # it means that the process mesh is not a union when process meshes is none @@ -1737,7 +2066,8 @@ class Resharder: for process in process_mesh.processes: processes.add(process) for idx, process_mesh in enumerate( - self.dist_context.process_meshes): + self.dist_context.process_meshes + ): if len(set(process_mesh.processes)) == len(processes): global_process_mesh_idx = idx break @@ -1757,7 +2087,8 @@ class Resharder: def _change_subblock_op_input_and_output(self, block_idx, block): if "var_reshard_mapping" in Resharder.while_block_info[block_idx]: var_reshard_mapping = Resharder.while_block_info[block_idx][ - "var_reshard_mapping"] + "var_reshard_mapping" + ] for op in block.ops: for var_name in op.input_arg_names: if var_name in var_reshard_mapping: @@ -1766,9 +2097,11 @@ class Resharder: dist_attr = dist_op.dist_attr target_name = None for item in var_reshard_mapping[var_name]: - if dist_attr.process_mesh == item[0][ - 0] and dist_attr.get_input_dims_mapping( - var_name) == item[0][1]: + if ( + dist_attr.process_mesh == item[0][0] + and dist_attr.get_input_dims_mapping(var_name) + == item[0][1] + ): target_name = item[1] break if target_name is None: @@ -1776,15 +2109,18 @@ class Resharder: else: op.desc._rename_input(var_name, target_name) dist_op = self.dist_context.get_dist_op_for_program( - op) + op + ) op_dist_attr = dist_op.dist_attr old_name = var_name new_name = target_name assert old_name != new_name - op_input_dist_attr = op_dist_attr.get_input_dist_attr( - old_name) + op_input_dist_attr = ( + op_dist_attr.get_input_dist_attr(old_name) + ) op_dist_attr.set_input_dist_attr( - new_name, op_input_dist_attr) + new_name, op_input_dist_attr + ) op_dist_attr.del_input_dist_attr(old_name) # the outputs also need to be renamed when the output name is the same with input name in inplace op @@ -1804,9 +2140,11 @@ class Resharder: new_name = target_name assert old_name != new_name op_output_dist_attr = op_dist_attr.get_output_dist_attr( - old_name) + old_name + ) op_dist_attr.set_output_dist_attr( - new_name, op_output_dist_attr) + new_name, op_output_dist_attr + ) op_dist_attr.del_output_dist_attr(old_name) def _reshard_input(self, block): @@ -1821,18 +2159,22 @@ class Resharder: dist_op = self.dist_context.get_dist_op_for_program(op) if dist_op is not None: - op_input_dist_attrs = [ - ] # [(op_process_mesh, op_input_dims_mapping), (op_process_mesh, op_input_dims_mapping)] + op_input_dist_attrs = ( + [] + ) # [(op_process_mesh, op_input_dims_mapping), (op_process_mesh, op_input_dims_mapping)] if op.type in _g_subblock_ops: if not self.is_condition_replicative(op): raise ValueError( "Please check the condition due to the dims mapping is not replicative." ) - if op.attr( - "sub_block").id not in Resharder.while_block_info: + if ( + op.attr("sub_block").id + not in Resharder.while_block_info + ): Resharder.while_block_info[op.attr("sub_block").id] = {} - Resharder.while_block_info[op.attr( - "sub_block").id]["op_id"] = op.desc.id() + Resharder.while_block_info[op.attr("sub_block").id][ + "op_id" + ] = op.desc.id() if op.type == "while": # condition var process mesh is the same with op and dims_mapping is replicative, so it do not need reshard @@ -1849,17 +2191,24 @@ class Resharder: # skip lod_tensor_blocking_queue_? name if "lod_tensor_blocking_queue" in var_name: continue - var = get_var_with_recursion(var_name, block, - self.auto_parallel_main_prog) + var = get_var_with_recursion( + var_name, block, self.auto_parallel_main_prog + ) dist_tensor = self.dist_context.get_dist_tensor_for_program( - var) + var + ) # judge whether union tensor dims_mapping all -1 is_union_process_mesh_tensor = False - if dist_tensor.dist_attr.process_mesh not in self.dist_context.process_meshes and self.dist_context.process_meshes: + if ( + dist_tensor.dist_attr.process_mesh + not in self.dist_context.process_meshes + and self.dist_context.process_meshes + ): is_union_process_mesh_tensor = True assert dist_tensor.dist_attr.dims_mapping.count( - -1) == len(dist_tensor.dist_attr.dims_mapping) + -1 + ) == len(dist_tensor.dist_attr.dims_mapping) op_input_attrs = self.get_op_input_attrs(op, var_name) for input_attr in op_input_attrs: @@ -1869,18 +2218,23 @@ class Resharder: if is_union_process_mesh_tensor: # if op process mesh is subset of union tensor process mesh, need no reshard if set(input_attr[0].processes) <= set( - dist_tensor.dist_attr.process_mesh.processes + dist_tensor.dist_attr.process_mesh.processes ): continue if dist_tensor is not None and self.need_reshard( - dist_tensor, input_attr): + dist_tensor, input_attr + ): reshard_op_desc = self.find_op_desc_seq( - dist_tensor, input_attr) - self.parse_op_desc(block, reshard_op_desc, var_name, - op, input_attr) + dist_tensor, input_attr + ) + self.parse_op_desc( + block, reshard_op_desc, var_name, op, input_attr + ) cur_op_count = len(block.ops) - idx_offset = idx_offset + cur_op_count - pre_op_count + idx_offset = ( + idx_offset + cur_op_count - pre_op_count + ) pre_op_count = cur_op_count idx = idx + idx_offset + 1 else: @@ -1895,34 +2249,43 @@ class Resharder: shape=var.shape, lod_level=var.lod_level, dtype=paddle.int64, - type=var.type) - Inserter.insert_recv_op(block, idx + 1, - recv_cast_out, send_rank, recv_rank, - op.attr('op_role')) + type=var.type, + ) + Inserter.insert_recv_op( + block, + idx + 1, + recv_cast_out, + send_rank, + recv_rank, + op.attr('op_role'), + ) reset_lod_out = None if var.lod_level != 0: set_lod = False for tmp_block in self.auto_parallel_main_prog.blocks: for tmp_var_name in tmp_block.vars: tmp_var = tmp_block.vars[tmp_var_name] - if tmp_var.is_data and tmp_var.lod_level == var.lod_level: + if ( + tmp_var.is_data + and tmp_var.lod_level == var.lod_level + ): reset_lod_out = block.create_var( - name=unique_name.generate(var.name + - "@RESETLOD"), + name=unique_name.generate( + var.name + "@RESETLOD" + ), shape=recv_cast_out.shape, type=recv_cast_out.type, dtype=recv_cast_out.dtype, - lod_level=recv_cast_out.lod_level) + lod_level=recv_cast_out.lod_level, + ) idx += 1 block._insert_op( idx, type="lod_reset", - inputs={ - 'X': recv_cast_out, - 'Y': tmp_var - }, + inputs={'X': recv_cast_out, 'Y': tmp_var}, outputs={'Out': reset_lod_out}, - attrs={'op_role': op.attr("op_role")}) + attrs={'op_role': op.attr("op_role")}, + ) set_lod = True break if set_lod: @@ -1930,18 +2293,21 @@ class Resharder: assert set_lod is True # cast int64 to bool - block._insert_op(idx + 2, - type='cast', - inputs={ - 'X': [recv_cast_out] if - reset_lod_out is None else [reset_lod_out] - }, - outputs={'Out': [var]}, - attrs={ - 'in_dtype': recv_cast_out.dtype, - 'out_dtype': var.dtype, - 'op_role': op.attr('op_role') - }) + block._insert_op( + idx + 2, + type='cast', + inputs={ + 'X': [recv_cast_out] + if reset_lod_out is None + else [reset_lod_out] + }, + outputs={'Out': [var]}, + attrs={ + 'in_dtype': recv_cast_out.dtype, + 'out_dtype': var.dtype, + 'op_role': op.attr('op_role'), + }, + ) else: if var.lod_level != 0: recv_out = block.create_var( @@ -1949,50 +2315,75 @@ class Resharder: shape=var.shape, lod_level=var.lod_level, dtype=var.int64, - type=var.type) - Inserter.insert_recv_op(block, idx + 1, recv_out, send_rank, - recv_rank, op.attr('op_role')) + type=var.type, + ) + Inserter.insert_recv_op( + block, + idx + 1, + recv_out, + send_rank, + recv_rank, + op.attr('op_role'), + ) set_lod = False for tmp_block in self.auto_parallel_main_prog.blocks: for tmp_var_name in tmp_block.vars: tmp_var = tmp_block.vars[tmp_var_name] - if tmp_var.is_data and tmp_var.lod_level == var.lod_level: + if ( + tmp_var.is_data + and tmp_var.lod_level == var.lod_level + ): idx += 1 block._insert_op( idx, type="lod_reset", - inputs={ - 'X': recv_out, - 'Y': tmp_var - }, + inputs={'X': recv_out, 'Y': tmp_var}, outputs={'Out': var}, - attrs={'op_role': op.attr("op_role")}) + attrs={'op_role': op.attr("op_role")}, + ) set_lod = True break if set_lod: break assert set_lod is True else: - Inserter.insert_recv_op(block, idx + 1, var, send_rank, - recv_rank, op.attr('op_role')) + Inserter.insert_recv_op( + block, + idx + 1, + var, + send_rank, + recv_rank, + op.attr('op_role'), + ) def _handle_send(self, block, idx, var, op, send_rank, recv_rank): if var.dtype == paddle.bool: - cast_out = Inserter.insert_cast_op(block, idx + 1, var, - op.attr('op_role'), paddle.int64) - Inserter.insert_send_op(block, idx + 2, cast_out, send_rank, - recv_rank, op.attr('op_role')) + cast_out = Inserter.insert_cast_op( + block, idx + 1, var, op.attr('op_role'), paddle.int64 + ) + Inserter.insert_send_op( + block, + idx + 2, + cast_out, + send_rank, + recv_rank, + op.attr('op_role'), + ) else: - Inserter.insert_send_op(block, idx + 1, var, send_rank, recv_rank, - op.attr('op_role')) + Inserter.insert_send_op( + block, idx + 1, var, send_rank, recv_rank, op.attr('op_role') + ) def _reshard_output(self, block): # insert send and recv op if output process mesh is different from tensor process mesh idx = 0 # skip reader and ops whose process mesh is union skip_ops = [ - "create_py_reader", "create_double_buffer_reader", "read", - "write_to_array", "read_from_array" + "create_py_reader", + "create_double_buffer_reader", + "read", + "write_to_array", + "read_from_array", ] global _g_special_ops skip_ops += _g_special_ops @@ -2004,59 +2395,86 @@ class Resharder: if dist_op is not None and op.type not in skip_ops: idx_offset = 0 for var_name in op.output_arg_names: - var = get_var_with_recursion(var_name, block, - self.auto_parallel_main_prog) + var = get_var_with_recursion( + var_name, block, self.auto_parallel_main_prog + ) dist_tensor = self.dist_context.get_dist_tensor_for_program( - var) + var + ) tensor_process_mesh = dist_tensor.dist_attr.process_mesh output_attr = [ dist_op.dist_attr.process_mesh, - dist_op.dist_attr.get_output_dims_mapping(var_name) + dist_op.dist_attr.get_output_dims_mapping(var_name), ] if dist_tensor is not None and self.need_reshard( - dist_tensor, output_attr, False): + dist_tensor, output_attr, False + ): tensor_processes = set( - tensor_process_mesh.processes) - ( - set(tensor_process_mesh.processes) - & set(output_attr[0].processes)) + tensor_process_mesh.processes + ) - ( + set(tensor_process_mesh.processes) + & set(output_attr[0].processes) + ) if tensor_processes: if len(tensor_processes) != len( - output_attr[0].processes): + output_attr[0].processes + ): if dist_tensor.dist_attr.dims_mapping.count( - -1) != len( - dist_tensor.dist_attr.dims_mapping - ) or output_attr[1].count(-1) != len( - output_attr[1]): + -1 + ) != len( + dist_tensor.dist_attr.dims_mapping + ) or output_attr[ + 1 + ].count( + -1 + ) != len( + output_attr[1] + ): raise ValueError( - "The dims_mapping must be -1") + "The dims_mapping must be -1" + ) else: for index, tensor_process in enumerate( - tensor_processes): + tensor_processes + ): recv_rank = tensor_process actual_index = index if index >= len( - output_attr[0].processes): + output_attr[0].processes + ): actual_index = ( - index - - len(output_attr[0].processes) + index + - len(output_attr[0].processes) ) % len(output_attr[0].processes) item = output_attr[0].processes[ - actual_index] + actual_index + ] if recv_rank == item: continue if self.rank_id == item: # if send bool data, cast then send self._handle_send( - block, idx, var, op, item, - recv_rank) + block, + idx, + var, + op, + item, + recv_rank, + ) if self.rank_id == recv_rank: # if recv bool data, recv then cast self._hadnle_recv( - block, idx, var, op, item, - recv_rank) + block, + idx, + var, + op, + item, + recv_rank, + ) else: for index, tensor_process in enumerate( - tensor_processes): + tensor_processes + ): recv_rank = tensor_process item = output_attr[0].processes[index] if recv_rank == item: @@ -2064,16 +2482,18 @@ class Resharder: if self.rank_id == item: # if send bool data, cast then send self._handle_send( - block, idx, var, op, item, - recv_rank) + block, idx, var, op, item, recv_rank + ) if self.rank_id == recv_rank: # if recv bool data, recv then cast self._hadnle_recv( - block, idx, var, op, item, - recv_rank) + block, idx, var, op, item, recv_rank + ) cur_op_count = len(block.ops) - idx_offset = idx_offset + cur_op_count - pre_op_count + idx_offset = ( + idx_offset + cur_op_count - pre_op_count + ) pre_op_count = cur_op_count idx = idx + idx_offset + 1 @@ -2095,13 +2515,17 @@ class Resharder: self._reshard_output(block) # remove no need vars and ops in the main program - Remover.remove_no_need_in_main(self.auto_parallel_main_prog, - self.dist_context, self.rank_id, - self.dist_params_grads) + Remover.remove_no_need_in_main( + self.auto_parallel_main_prog, + self.dist_context, + self.rank_id, + self.dist_params_grads, + ) # remove no need vars and ops in the startip program - Remover.remove_no_need_in_startup(self.auto_parallel_main_prog, - self.auto_parallel_startup_prog) + Remover.remove_no_need_in_startup( + self.auto_parallel_main_prog, self.auto_parallel_startup_prog + ) # reset some variable when remove operation ended Resharder.while_block_info = {} @@ -2119,47 +2543,68 @@ class Resharder: return reshard_op_cost else: dist_tensor = self.dist_context.get_dist_tensor_for_program( - tensor) + tensor + ) # simplified processing: ignore union process mesh and output reshard dist_op = self.dist_context.get_dist_op_for_program(op) dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - tensor.name) + tensor.name + ) process_mesh = dist_op.dist_attr.process_mesh dist_attr = [process_mesh, dims_mapping] if dist_tensor is not None and self.need_reshard( - dist_tensor, dist_attr): + dist_tensor, dist_attr + ): if tensor_name not in self._has_resharded: self._has_resharded[tensor_name] = [dist_op] else: for item in self._has_resharded[tensor_name]: item_dist_attr = item.dist_attr - item_dims_mapping = item_dist_attr.get_input_dims_mapping( - tensor_name) + item_dims_mapping = ( + item_dist_attr.get_input_dims_mapping( + tensor_name + ) + ) item_process_mesh = item_dist_attr.process_mesh - if dims_mapping == item_dims_mapping and item_process_mesh == process_mesh: + if ( + dims_mapping == item_dims_mapping + and item_process_mesh == process_mesh + ): return reshard_op_cost self._has_resharded[tensor_name].append(dist_op) - reshard_op_desc = self.find_op_desc_seq(dist_tensor, - dist_attr, - serial=True) + reshard_op_desc = self.find_op_desc_seq( + dist_tensor, dist_attr, serial=True + ) dtype = dist_tensor.serial_tensor.dtype reshard_op_cost = self.parse_op_desc_for_cost( - reshard_op_desc, dtype, cluster) + reshard_op_desc, dtype, cluster + ) return reshard_op_cost - def _concat_partitions_for_cost(self, partition_tensor_list, - partition_index, dtype, rank_id, - local_rank_comp_cost, cluster): + def _concat_partitions_for_cost( + self, + partition_tensor_list, + partition_index, + dtype, + rank_id, + local_rank_comp_cost, + cluster, + ): if not partition_tensor_list: partition_tensor_list.append(partition_index) else: i = 0 has_concat = False while i < len(partition_tensor_list): - concat_axis, first_order, new_partition = Resharder.compute_concat_info( - partition_tensor_list[i], partition_index) + ( + concat_axis, + first_order, + new_partition, + ) = Resharder.compute_concat_info( + partition_tensor_list[i], partition_index + ) if concat_axis != -1: has_concat = True concat_desc = {} @@ -2167,31 +2612,38 @@ class Resharder: concat_desc["attrs"] = {"axis": concat_axis} if first_order == 0: concat_desc["inputs"] = { - "X": [(dtype, partition_tensor_list[i]), - (dtype, partition_index)] + "X": [ + (dtype, partition_tensor_list[i]), + (dtype, partition_index), + ] } else: concat_desc["inputs"] = { - "X": [(dtype, partition_index), - (dtype, partition_tensor_list[i])] + "X": [ + (dtype, partition_index), + (dtype, partition_tensor_list[i]), + ] } partition_tensor_list.pop(i) if rank_id not in local_rank_comp_cost: local_rank_comp_cost[rank_id] = [] local_rank_comp_cost[rank_id].append( - ConcatOpCost(op_desc=concat_desc, cluster=cluster)) - self._concat_partitions_for_cost(partition_tensor_list, - new_partition, dtype, - rank_id, - local_rank_comp_cost, - cluster) + ConcatOpCost(op_desc=concat_desc, cluster=cluster) + ) + self._concat_partitions_for_cost( + partition_tensor_list, + new_partition, + dtype, + rank_id, + local_rank_comp_cost, + cluster, + ) break i += 1 if not has_concat: partition_tensor_list.append(partition_index) def parse_op_desc_for_cost(self, reshard_op_desc, dtype, cluster): - def _get_idx(comm_ranks, group_ranks): res, is_the_same = None, False idx = 0 @@ -2222,28 +2674,41 @@ class Resharder: if isinstance(op_desc, SendOpDesc): group_ranks = [key, op_desc.dst] shape = op_desc.shape - send_desc = build_comm_desc("send_v2", group_ranks, dtype, - shape) + send_desc = build_comm_desc( + "send_v2", group_ranks, dtype, shape + ) idx, is_the_same = _get_idx(comm_ranks, group_ranks) if idx is None: - comm_costs.append([ - (group_ranks, - SendOpCost(op_desc=send_desc, - comm_context=comm_context)) - ]) + comm_costs.append( + [ + ( + group_ranks, + SendOpCost( + op_desc=send_desc, + comm_context=comm_context, + ), + ) + ] + ) comm_ranks.append(set(group_ranks)) else: if not is_the_same: comm_costs[idx].append( - (group_ranks, - SendOpCost(op_desc=send_desc, - comm_context=comm_context))) + ( + group_ranks, + SendOpCost( + op_desc=send_desc, + comm_context=comm_context, + ), + ) + ) elif isinstance(op_desc, AllGatherOpDesc): # NOTE: fill_const and other unnecessary op is not calculated because those cost is very small group_ranks = op_desc.group shape = op_desc.shape - allgather_desc = build_comm_desc("c_allgather", group_ranks, - dtype, shape) + allgather_desc = build_comm_desc( + "c_allgather", group_ranks, dtype, shape + ) split_inputs_shape = [] for idx, dim in enumerate(shape): if idx == 0: @@ -2252,18 +2717,29 @@ class Resharder: split_inputs_shape.append(dim) idx, is_the_same = _get_idx(comm_ranks, group_ranks) if idx is None: - comm_costs.append([ - (group_ranks, - AllgatherOpCost(op_desc=allgather_desc, - comm_context=comm_context)) - ]) + comm_costs.append( + [ + ( + group_ranks, + AllgatherOpCost( + op_desc=allgather_desc, + comm_context=comm_context, + ), + ) + ] + ) comm_ranks.append(set(group_ranks)) else: if not is_the_same: comm_costs[idx].append( - (group_ranks, - AllgatherOpCost(op_desc=allgather_desc, - comm_context=comm_context))) + ( + group_ranks, + AllgatherOpCost( + op_desc=allgather_desc, + comm_context=comm_context, + ), + ) + ) # calc the split op cost if key not in local_rank_comp_cost: local_rank_comp_cost[key] = [] @@ -2274,19 +2750,27 @@ class Resharder: } split_desc["attrs"] = {"num": len(group_ranks), "axis": 0} local_rank_comp_cost[key].append( - SplitOpCost(op_desc=split_desc, cluster=cluster)) + SplitOpCost(op_desc=split_desc, cluster=cluster) + ) elif isinstance(op_desc, ConcatOpDesc): partition_index_list = op_desc._partition_index_list for idx, partion_idex in enumerate(partition_index_list): self._concat_partitions_for_cost( - partition_tensor_list, partion_idex, dtype, key, - local_rank_comp_cost, cluster) + partition_tensor_list, + partion_idex, + dtype, + key, + local_rank_comp_cost, + cluster, + ) elif isinstance(op_desc, SliceOpDesc): if key not in local_rank_comp_cost: local_rank_comp_cost[key] = [] - assert len( - partition_tensor_list) == 1 or not partition_tensor_list + assert ( + len(partition_tensor_list) == 1 + or not partition_tensor_list + ) to_slice_tensor_shape = [] if len(partition_tensor_list) == 1: for item in partition_tensor_list[0]: @@ -2300,13 +2784,14 @@ class Resharder: "axes": op_desc.axes, "starts": op_desc.starts, "ends": op_desc.ends, - "infer_flags": infer_flags + "infer_flags": infer_flags, } slice_desc["inputs"] = { "Input": [(dtype, to_slice_tensor_shape)] } local_rank_comp_cost[key].append( - SliceOpCost(op_desc=slice_desc, cluster=cluster)) + SliceOpCost(op_desc=slice_desc, cluster=cluster) + ) res = (comm_costs, local_rank_comp_cost) diff --git a/python/paddle/distributed/auto_parallel/strategy.py b/python/paddle/distributed/auto_parallel/strategy.py index a504bbfed29..2440472bada 100644 --- a/python/paddle/distributed/auto_parallel/strategy.py +++ b/python/paddle/distributed/auto_parallel/strategy.py @@ -17,7 +17,6 @@ from . import constants class BaseConfig(object): - def __init__(self, category, config_dict=None): self._category = category self._config_dict = None @@ -27,7 +26,9 @@ class BaseConfig(object): else: raise ValueError( "Expected a dictionary. But received: {}".format( - config_dict)) + config_dict + ) + ) # Initialize attributes by the default config config = constants.get_category_default_config(self._category) for field, default_value in config.items(): @@ -73,49 +74,42 @@ class BaseConfig(object): class RecomputeConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.RECOMPUTE super(RecomputeConfig, self).__init__(category, config_dict) class AMPConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.AMP super(AMPConfig, self).__init__(category, config_dict) class ShardingConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.SHARDING super(ShardingConfig, self).__init__(category, config_dict) class GradientMergeConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.GRADIENT_MERGE super(GradientMergeConfig, self).__init__(category, config_dict) class QATConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.QAT super(QATConfig, self).__init__(category, config_dict) class TuningConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.TUNING super(TuningConfig, self).__init__(category, config_dict) class DatasetConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.DATASET super(DatasetConfig, self).__init__(category, config_dict) @@ -161,7 +155,8 @@ class Strategy(BaseConfig): # self._config_dict = yaml.load(yaml_file, Loader=yaml.Loader) else: raise ValueError( - "Expected a dictionary. But received: {}".format(config)) + "Expected a dictionary. But received: {}".format(config) + ) else: self._config_dict = {} diff --git a/python/paddle/distributed/auto_parallel/tuner/algorithms.py b/python/paddle/distributed/auto_parallel/tuner/algorithms.py index f892a7838fe..efc3358ebe4 100644 --- a/python/paddle/distributed/auto_parallel/tuner/algorithms.py +++ b/python/paddle/distributed/auto_parallel/tuner/algorithms.py @@ -31,6 +31,7 @@ class AlgorithmBase(ABC): In another word, the key "algorithm" for this class is the search space pruning rules specific for the given optimization scenario. """ + _REGISTERED_ALGORITHMS = {} name = None @@ -88,7 +89,6 @@ class AlgorithmBase(ABC): def register_algor(name): - def impl(cls): AlgorithmBase._register(name, cls) cls.name = name @@ -121,7 +121,8 @@ class ShardingStageAlgorithm(AlgorithmBase): assert set(stage_range).issubset( set([0, 1, 2, 3]) ), "Sharding Stage should belong into range within 0 - 3 but got {}.".format( - stage_range) + stage_range + ) stage_range.sort(reverse=True) else: stage_range = list(range(self._max_stage + 1)).sort(reverse=True) diff --git a/python/paddle/distributed/auto_parallel/tuner/config.py b/python/paddle/distributed/auto_parallel/tuner/config.py index 9073013127a..4ed6340eccb 100644 --- a/python/paddle/distributed/auto_parallel/tuner/config.py +++ b/python/paddle/distributed/auto_parallel/tuner/config.py @@ -109,8 +109,10 @@ class TuningConfig(object): self._project_dir = project_dir for p in _tuning_supported_passes: - if getattr(self._dist_strategy, p) and _get_pass_config( - self._dist_strategy, p).enable_tuning: + if ( + getattr(self._dist_strategy, p) + and _get_pass_config(self._dist_strategy, p).enable_tuning + ): # TODO distinguish different args of each passes self._tuning_passes_name.add(p) diff --git a/python/paddle/distributed/auto_parallel/tuner/optimization_tuner.py b/python/paddle/distributed/auto_parallel/tuner/optimization_tuner.py index 518e4fda118..a5cdbc7f95a 100644 --- a/python/paddle/distributed/auto_parallel/tuner/optimization_tuner.py +++ b/python/paddle/distributed/auto_parallel/tuner/optimization_tuner.py @@ -34,7 +34,10 @@ from paddle.distributed.auto_parallel.dist_context import DistributedContext from paddle.distributed.auto_parallel.completion import Completer from paddle.distributed.auto_parallel.reshard import Resharder from paddle.distributed.auto_parallel.partitioner import Partitioner -from paddle.distributed.auto_parallel.process_group import clear_all_process_groups, get_all_process_groups +from paddle.distributed.auto_parallel.process_group import ( + clear_all_process_groups, + get_all_process_groups, +) from paddle.distributed.auto_parallel.utils import debug_program from paddle.distributed.auto_parallel.utils import set_grad_var_shape @@ -82,9 +85,8 @@ def parse_process_groups(): def get_metric(results): assert isinstance( - results, - dict), "results should be type of dictionary, but got {}.".format( - type(results)) + results, dict + ), "results should be type of dictionary, but got {}.".format(type(results)) if 'Throughtput' in results and isinstance(results['Throughtput'], float): return float(results['Throughtput']) else: @@ -108,20 +110,25 @@ def _copy_context(ref_dist_context): clear_all_process_groups() new_dist_context = DistributedContext() - new_dist_context._serial_main_program = ref_dist_context.serial_main_program.clone( - for_test=False) - new_dist_context._serial_startup_program = ref_dist_context.serial_startup_program.clone( - for_test=False) + new_dist_context._serial_main_program = ( + ref_dist_context.serial_main_program.clone(for_test=False) + ) + new_dist_context._serial_startup_program = ( + ref_dist_context.serial_startup_program.clone(for_test=False) + ) # mapping variable into new dist context if getattr(ref_dist_context, '_params_grads', None): new_dist_context._params_grads = _get_new_params_grads( new_dist_context.serial_main_program, ref_dist_context.serial_main_program, - ref_dist_context._params_grads) + ref_dist_context._params_grads, + ) new_dist_context._serial_loss = _get_new_loss( new_dist_context.serial_main_program, - ref_dist_context.serial_main_program, ref_dist_context.serial_loss) + ref_dist_context.serial_main_program, + ref_dist_context.serial_loss, + ) for key, var_list in ref_dist_context._serial_feed_vars.items(): new_var_list = [] @@ -129,7 +136,8 @@ def _copy_context(ref_dist_context): block_idx = var.block.idx var_name = var.name var = new_dist_context._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_var_list.append(var) new_dist_context._serial_feed_vars[key] = new_var_list @@ -143,7 +151,8 @@ def _copy_context(ref_dist_context): block_idx = var.block.idx var_name = var.name var = new_dist_context._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_inner_var_list.append(var) new_var_list.append(new_inner_var_list) else: @@ -151,21 +160,26 @@ def _copy_context(ref_dist_context): block_idx = var.block.idx var_name = var.name var = new_dist_context._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_var_list.append(var) new_dist_context._serial_fetch_vars[key] = new_var_list # copy information in forward and backward new_dist_context._serial_optimizer = copy.deepcopy( - ref_dist_context.serial_optimizer) + ref_dist_context.serial_optimizer + ) new_dist_context._dist_tensors_for_program = copy.deepcopy( - ref_dist_context._dist_tensors_for_program) + ref_dist_context._dist_tensors_for_program + ) new_dist_context._dist_ops_for_program = copy.deepcopy( - ref_dist_context._dist_ops_for_program) + ref_dist_context._dist_ops_for_program + ) for pm in ref_dist_context.process_meshes: new_dist_context.add_process_mesh(pm) new_dist_context._dist_op_context = copy.deepcopy( - ref_dist_context._dist_op_context) + ref_dist_context._dist_op_context + ) new_dist_context._block_state = copy.deepcopy(ref_dist_context.block_state) return new_dist_context @@ -229,28 +243,39 @@ class OptimizationTuner: def _build_programs_without_optimization(self): serial_main_program = self._baseline_dist_context.serial_main_program - serial_startup_program = self._baseline_dist_context.serial_startup_program + serial_startup_program = ( + self._baseline_dist_context.serial_startup_program + ) serial_loss = self._baseline_dist_context.serial_loss with program_guard(serial_main_program, serial_startup_program): params_grads = append_backward( serial_loss, - distop_context=self._baseline_dist_context.dist_op_context) + distop_context=self._baseline_dist_context.dist_op_context, + ) self._baseline_completer.complete_backward_annotation( - serial_main_program) + serial_main_program + ) self._baseline_dist_context.block_state.parse_backward_blocks( - serial_main_program) + serial_main_program + ) self._baseline_dist_context._params_grads = params_grads if self._config.verbose: baseline_dir = os.path.join(self.project_dir, "baseline") if not os.path.exists(baseline_dir): pathlib.Path(baseline_dir).mkdir(parents=True, exist_ok=True) - debug_program(self._baseline_dist_context._serial_main_program, - baseline_dir, "main") - debug_program(self._baseline_dist_context._serial_startup_program, - baseline_dir, "startup") + debug_program( + self._baseline_dist_context._serial_main_program, + baseline_dir, + "main", + ) + debug_program( + self._baseline_dist_context._serial_startup_program, + baseline_dir, + "startup", + ) def _select_tuning_algorithm(self): @@ -275,18 +300,22 @@ class OptimizationTuner: # TODO AMP Pass should not use loss var config["loss"] = dist_context.serial_loss - config["input_data"] = self._baseline_dist_context.serial_feed_vars["inputs"] \ + config["input_data"] = ( + self._baseline_dist_context.serial_feed_vars["inputs"] + self._baseline_dist_context.serial_feed_vars["labels"] + ) if config["use_pure_fp16"]: config["base_opt"] = dist_context.serial_optimizer auto_parallel_fp16_pass = new_pass("auto_parallel_fp16", config) - auto_parallel_fp16_pass.apply([main_program], [startup_program], - pass_context) + auto_parallel_fp16_pass.apply( + [main_program], [startup_program], pass_context + ) dist_context.serial_loss = auto_parallel_fp16_pass.get_loss() else: auto_parallel_amp_pass = new_pass("auto_parallel_amp", config) - auto_parallel_amp_pass.apply([main_program], [startup_program], - pass_context) + auto_parallel_amp_pass.apply( + [main_program], [startup_program], pass_context + ) dist_context.serial_loss = auto_parallel_amp_pass.get_loss() if new_strategy.recompute.enable: @@ -294,27 +323,40 @@ class OptimizationTuner: config["dist_context"] = dist_context config["no_grad_set"] = None config["loss"] = dist_context.serial_loss - auto_parallel_recompute_pass = new_pass("auto_parallel_recompute", - config) - auto_parallel_recompute_pass.apply([main_program], - [startup_program], pass_context) + auto_parallel_recompute_pass = new_pass( + "auto_parallel_recompute", config + ) + auto_parallel_recompute_pass.apply( + [main_program], [startup_program], pass_context + ) # Do logical partition partitioner = Partitioner(dist_context, self.rank) - dist_main_prog, dist_startup_prog, dist_params_grads = partitioner.partition( - main_program, startup_program, dist_context._params_grads) + ( + dist_main_prog, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + main_program, startup_program, dist_context._params_grads + ) # Generate optimizer # FIXME should be remove from apply pass after pass support optimizers with program_guard(dist_main_prog, dist_startup_prog): optimizer_ops = dist_context.serial_optimizer.apply_gradients( - dist_params_grads) + dist_params_grads + ) completer.complete_update_annotation(dist_main_prog) # Do reshard process set_grad_var_shape(dist_main_prog, dist_context) - resharder = Resharder(dist_main_prog, dist_startup_prog, self.rank, - dist_context, dist_params_grads) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + self.rank, + dist_context, + dist_params_grads, + ) resharder.reshard() if new_strategy.sharding.enable: @@ -322,21 +364,27 @@ class OptimizationTuner: config["dist_context"] = dist_context config["params_grads"] = dist_params_grads config["global_rank"] = self.rank - auto_parallel_sharding_pass = new_pass("auto_parallel_sharding", - config) - auto_parallel_sharding_pass.apply([dist_main_prog], - [dist_startup_prog], pass_context) + auto_parallel_sharding_pass = new_pass( + "auto_parallel_sharding", config + ) + auto_parallel_sharding_pass.apply( + [dist_main_prog], [dist_startup_prog], pass_context + ) if new_strategy.gradient_merge.enable: config = copy.deepcopy(new_strategy.gradient_merge.to_dict()) config["dist_context"] = dist_context config["params_grads"] = dist_params_grads auto_parallel_gradient_merge_pass = new_pass( - "auto_parallel_gradient_merge_pass", config) - auto_parallel_gradient_merge_pass.apply([dist_main_prog], - [dist_startup_prog], - pass_context) - trial.main_program, trial.startup_program = dist_main_prog, dist_startup_prog + "auto_parallel_gradient_merge_pass", config + ) + auto_parallel_gradient_merge_pass.apply( + [dist_main_prog], [dist_startup_prog], pass_context + ) + trial.main_program, trial.startup_program = ( + dist_main_prog, + dist_startup_prog, + ) return trial def _get_profile_context(self, trial, result_path): @@ -344,16 +392,18 @@ class OptimizationTuner: profile_ctx = {} profile_ctx['distributed_env'] = copy.deepcopy( - paddle.distributed.ParallelEnv()) + paddle.distributed.ParallelEnv() + ) profile_ctx['group_map'] = parse_process_groups() profile_ctx[ - "loss_var_name"] = self._baseline_dist_context.serial_loss.name + "loss_var_name" + ] = self._baseline_dist_context.serial_loss.name profile_ctx[ - "main_program_decs"] = trial.main_program.desc.serialize_to_string( - ) + "main_program_decs" + ] = trial.main_program.desc.serialize_to_string() profile_ctx[ - "startup_program_decs"] = trial.startup_program.desc.serialize_to_string( - ) + "startup_program_decs" + ] = trial.startup_program.desc.serialize_to_string() self._dataset.batch_size = self._batch_size self._dataset.input_names = self._get_input_names() @@ -375,19 +425,25 @@ class OptimizationTuner: else: coverage_args = [] - profile_args = " ".join([ - "--rank", - str(self.rank), - "--device_id", - str(self.device_id), - "--ctx_filename", - ctx_path, - "--profile_start_step", - str(self._config.profile_start_step), - "--profile_end_step", - str(self._config.profile_end_step), - ]) - cmd_args = "-m paddle.distributed.auto_parallel.tuner.profiler" + " " + profile_args + profile_args = " ".join( + [ + "--rank", + str(self.rank), + "--device_id", + str(self.device_id), + "--ctx_filename", + ctx_path, + "--profile_start_step", + str(self._config.profile_start_step), + "--profile_end_step", + str(self._config.profile_end_step), + ] + ) + cmd_args = ( + "-m paddle.distributed.auto_parallel.tuner.profiler" + + " " + + profile_args + ) cmd = [sys.executable, "-u"] + coverage_args + shlex.split(cmd_args) parent_env = copy.copy(os.environ.copy()) @@ -400,10 +456,11 @@ class OptimizationTuner: # TODO if any rank hang or fail, kill all processes self._logger.debug("Executing cmd:\n{} .".format(" ".join(cmd))) # new_process = subprocess.Popen(cmd, env=new_env) - with open(os.path.join(trial_dir, "stdout.log" + str(self.rank)), - "wb") as out, open( - os.path.join(trial_dir, "stderr.log" + str(self.rank)), - "wb") as err: + with open( + os.path.join(trial_dir, "stdout.log" + str(self.rank)), "wb" + ) as out, open( + os.path.join(trial_dir, "stderr.log" + str(self.rank)), "wb" + ) as err: result = subprocess.Popen(cmd, stdout=out, stderr=err, env=new_env) result.wait() out.flush() @@ -455,13 +512,18 @@ class OptimizationTuner: elif self._config.mode == "COSTMODEL": raise NotImplementedError( - "COSTMODEL mode for optimization tuning is not supported yet!") + "COSTMODEL mode for optimization tuning is not supported yet!" + ) else: - raise NotImplementedError("invalid evaluation mode: {}".format( - self._config.mode)) + raise NotImplementedError( + "invalid evaluation mode: {}".format(self._config.mode) + ) - self._logger.info("Trial {} evaluation finish with {}.".format( - trial.name, parse_results(results))) + self._logger.info( + "Trial {} evaluation finish with {}.".format( + trial.name, parse_results(results) + ) + ) return results def _update(self, i, trial, results): @@ -496,11 +558,12 @@ class OptimizationTuner: Tuning Result Summary Run total {} trials with {} min. The best trial is: [{}], whose configuration is following: - """.format(len(self._finished_trials), - (time.time() - self._tuning_start_time) / 60, - best_trial.name) - summary_ += "\n" + best_trial.summary() + "\n"\ - + """.format( + len(self._finished_trials), + (time.time() - self._tuning_start_time) / 60, + best_trial.name, + ) + summary_ += "\n" + best_trial.summary() + "\n" self._logger.info(summary_) with open(os.path.join(self.project_dir, "summary.txt"), "w+") as fw: for line in summary_.split("\n"): @@ -532,7 +595,8 @@ The best trial is: [{}], whose configuration is following: self._tuning_start_time = time.time() self._algorithm.collect_model_info( self._baseline_dist_context.serial_main_program, - self._baseline_dist_context.serial_startup_program) + self._baseline_dist_context.serial_startup_program, + ) # main search loop i = 0 @@ -554,10 +618,15 @@ The best trial is: [{}], whose configuration is following: # early stop i += 1 - if self._config.early_stop and self._config.early_stop <= i - self._best_iter: + if ( + self._config.early_stop + and self._config.early_stop <= i - self._best_iter + ): self._logger.info( - "Early stop the Tuning since there is no better trial found within [{}] trials" - .format(self._config.early_stop)) + "Early stop the Tuning since there is no better trial found within [{}] trials".format( + self._config.early_stop + ) + ) break # step5: summary the best config and return diff --git a/python/paddle/distributed/auto_parallel/tuner/parallel_tuner.py b/python/paddle/distributed/auto_parallel/tuner/parallel_tuner.py index 24ee382f7f7..0ed03defbed 100644 --- a/python/paddle/distributed/auto_parallel/tuner/parallel_tuner.py +++ b/python/paddle/distributed/auto_parallel/tuner/parallel_tuner.py @@ -33,15 +33,16 @@ from .tunable_variable import Boolean, IntRange class ParallelTuner: - - def __init__(self, - dist_context, - mode="train", - max_trials=25, - tuner_id=None, - seed=None, - logger=None, - loop_count=10): + def __init__( + self, + dist_context, + mode="train", + max_trials=25, + tuner_id=None, + seed=None, + logger=None, + loop_count=10, + ): self._loop_count = loop_count self._estimator = None self._dist_context = dist_context @@ -49,7 +50,8 @@ class ParallelTuner: self._mode = mode self._cluster = self._dist_context.cluster self._num_machines = self._cluster.get_num_machines() - self._num_devices_per_machine = self._cluster.get_num_devices_per_machine( + self._num_devices_per_machine = ( + self._cluster.get_num_devices_per_machine() ) self._space = TunableSpace() self._objective = "time" @@ -58,15 +60,17 @@ class ParallelTuner: self._tuner_id = tuner_id self._seed = seed if seed is not None else 9999 - print("seed", - self._seed, - "mode", - self._mode, - "num_machies", - self._num_machines, - "num_devices_per_machine", - self._num_devices_per_machine, - flush=True) + print( + "seed", + self._seed, + "mode", + self._mode, + "num_machies", + self._num_machines, + "num_devices_per_machine", + self._num_devices_per_machine, + flush=True, + ) self._seed_state = self._seed self._logger = logger self._max_collisions = 3 @@ -89,8 +93,12 @@ class ParallelTuner: self._cached_candidates_info = defaultdict(list) self._special_ops = [ - "create_py_reader", "create_double_buffer_reader", "read", "while", - "read_from_array", "write_to_array" + "create_py_reader", + "create_double_buffer_reader", + "read", + "while", + "read_from_array", + "write_to_array", ] # Each parallel strategy has two elements. The First one is for distributed tensors, @@ -100,44 +108,62 @@ class ParallelTuner: self._completer = Completer(self._dist_context) - self._parallelizer = Parallelizer(self._mode, self._completer, - self._dist_context) + self._parallelizer = Parallelizer( + self._mode, self._completer, self._dist_context + ) - def _generate_combination(self, - elements, - target, - idx, - partial_candidate, - candidates, - num_candidates=None): + def _generate_combination( + self, + elements, + target, + idx, + partial_candidate, + candidates, + num_candidates=None, + ): if target == 0: candidates.append(copy.deepcopy(partial_candidate)) return - if target < 0 or idx == len(elements) \ - or len(candidates) > num_candidates: + if ( + target < 0 + or idx == len(elements) + or len(candidates) > num_candidates + ): return # Use partial_candidate.append(elements[idx]) - self._generate_combination(elements, target - elements[idx], idx, - partial_candidate, candidates, - num_candidates) + self._generate_combination( + elements, + target - elements[idx], + idx, + partial_candidate, + candidates, + num_candidates, + ) # Not use partial_candidate.pop() - self._generate_combination(elements, target, idx + 1, partial_candidate, - candidates, num_candidates) - - def _permute_combination(self, - combination, - target, - check, - partial_candidate, - candidates, - num_candidates=None, - skip_prob=None): - if num_candidates is not None \ - and len(candidates) == num_candidates: + self._generate_combination( + elements, + target, + idx + 1, + partial_candidate, + candidates, + num_candidates, + ) + + def _permute_combination( + self, + combination, + target, + check, + partial_candidate, + candidates, + num_candidates=None, + skip_prob=None, + ): + if num_candidates is not None and len(candidates) == num_candidates: return if len(partial_candidate) == len(combination): @@ -149,13 +175,22 @@ class ParallelTuner: continue if self._rng.choice([True, False], p=[skip_prob, 1 - skip_prob]): continue - if i > 0 and combination[i] == combination[i - 1] \ - and check[i -1] == 0: + if ( + i > 0 + and combination[i] == combination[i - 1] + and check[i - 1] == 0 + ): continue check[i] = 1 - self._permute_combination(combination, target, check, - partial_candidate + [combination[i]], - candidates, num_candidates, skip_prob) + self._permute_combination( + combination, + target, + check, + partial_candidate + [combination[i]], + candidates, + num_candidates, + skip_prob, + ) check[i] = 0 def _partition_number(self, target): @@ -166,8 +201,14 @@ class ParallelTuner: seed_candidates = [] num_seed_candidates = 1000 partial_results = [] - self._generate_combination(elements, target, 0, partial_results, - seed_candidates, num_seed_candidates) + self._generate_combination( + elements, + target, + 0, + partial_results, + seed_candidates, + num_seed_candidates, + ) candidates = [] for seed_candidate in seed_candidates: @@ -178,10 +219,16 @@ class ParallelTuner: if target <= 8: skip_prob = 0.0 else: - skip_prob = (len(seed_candidate) / target) - self._permute_combination(seed_candidate, target, check, [], - cur_candidates, num_cur_candidates, - skip_prob) + skip_prob = len(seed_candidate) / target + self._permute_combination( + seed_candidate, + target, + check, + [], + cur_candidates, + num_cur_candidates, + skip_prob, + ) candidates.extend(cur_candidates) return candidates @@ -190,8 +237,9 @@ class ParallelTuner: intra_node_partitions = self._partition_number(num_devices_per_machine) return inter_node_partitions, intra_node_partitions - def _generate_process_mesh_list(self, inter_node_partition, - intra_node_partition): + def _generate_process_mesh_list( + self, inter_node_partition, intra_node_partition + ): process_mesh_list = [] start_row = 0 start_col = 0 @@ -200,8 +248,9 @@ class ParallelTuner: for n in intra_node_partition: process_mesh = [] for p in range(m): - start = (start_row + - p) * self._num_devices_per_machine + start_col + start = ( + start_row + p + ) * self._num_devices_per_machine + start_col tmp = [] for q in range(n): tmp.append(start + q) @@ -211,8 +260,9 @@ class ParallelTuner: start_row += m return process_mesh_list - def _generate_dims_mapping_candidates_helper(self, dims_mapping, dims_list, - start, visited, candidates): + def _generate_dims_mapping_candidates_helper( + self, dims_mapping, dims_list, start, visited, candidates + ): if start == len(dims_mapping) or all(visited): candidates.append(copy.deepcopy(dims_mapping)) return @@ -222,15 +272,17 @@ class ParallelTuner: dims_mapping[start] = dim visited[idx] = True self._generate_dims_mapping_candidates_helper( - dims_mapping, dims_list, start + 1, visited, candidates) + dims_mapping, dims_list, start + 1, visited, candidates + ) visited[idx] = False dims_mapping[start] = -1 - self._generate_dims_mapping_candidates_helper(dims_mapping, dims_list, - start + 1, visited, - candidates) + self._generate_dims_mapping_candidates_helper( + dims_mapping, dims_list, start + 1, visited, candidates + ) - def _generate_dims_mapping_candidates(self, dims_mapping_len, - process_mesh_len): + def _generate_dims_mapping_candidates( + self, dims_mapping_len, process_mesh_len + ): assert dims_mapping_len >= 1 and process_mesh_len >= 1 key = (dims_mapping_len, process_mesh_len) if key in self._cached_dims_mapping_candidates: @@ -239,8 +291,9 @@ class ParallelTuner: dims_mapping = [-1 for i in range(dims_mapping_len)] dims_list = [i for i in range(process_mesh_len)] visited = [False for i in range(process_mesh_len)] - self._generate_dims_mapping_candidates_helper(dims_mapping, dims_list, - 0, visited, candidates) + self._generate_dims_mapping_candidates_helper( + dims_mapping, dims_list, 0, visited, candidates + ) self._cached_dims_mapping_candidates[key] = candidates return candidates @@ -257,12 +310,14 @@ class ParallelTuner: key.append(input_name) for input_arg_name in serial_op.input(input_name): key.append( - len(op_dist_attr.get_input_dims_mapping(input_arg_name))) + len(op_dist_attr.get_input_dims_mapping(input_arg_name)) + ) for output_name in serial_op.output_names: key.append(output_name) for output_arg_name in serial_op.output(output_name): key.append( - len(op_dist_attr.get_output_dims_mapping(output_arg_name))) + len(op_dist_attr.get_output_dims_mapping(output_arg_name)) + ) key = tuple(key) if key in self._cached_candidates_info: @@ -274,18 +329,26 @@ class ParallelTuner: i = 0 for input_name in serial_op.input_names: for input_arg_name in serial_op.input(input_name): - cached_dims_mapping = cached_dist_attr.get_input_dims_mapping( - cached_input_arg_names[i]) + cached_dims_mapping = ( + cached_dist_attr.get_input_dims_mapping( + cached_input_arg_names[i] + ) + ) new_op_dist_attr.set_input_dims_mapping( - input_arg_name, cached_dims_mapping) + input_arg_name, cached_dims_mapping + ) i += 1 i = 0 for output_name in serial_op.output_names: for output_arg_name in serial_op.output(output_name): - cached_dims_mapping = cached_dist_attr.get_output_dims_mapping( - cached_output_arg_names[i]) + cached_dims_mapping = ( + cached_dist_attr.get_output_dims_mapping( + cached_output_arg_names[i] + ) + ) new_op_dist_attr.set_output_dims_mapping( - output_arg_name, cached_dims_mapping) + output_arg_name, cached_dims_mapping + ) i += 1 cached_dist_attr_candidates.append(new_op_dist_attr) return cached_dist_attr_candidates @@ -315,22 +378,28 @@ class ParallelTuner: input_names.append(tensor_name) if dims_mapping_len < 1: dims_mapping_generated.append( - [copy.deepcopy(original_dims_mapping)]) + [copy.deepcopy(original_dims_mapping)] + ) else: dims_mapping_generated.append( self._generate_dims_mapping_candidates( - dims_mapping_len, process_mesh_len)) + dims_mapping_len, process_mesh_len + ) + ) input_dims_mapping_candidates = [] for dims_mapping_list in itertools.product(*dims_mapping_generated): dims_mapping_list = list(dims_mapping_list) assert len(dims_mapping_list) == len(input_names) for i, dims_mapping in enumerate(dims_mapping_list): - new_op_dist_attr.set_input_dims_mapping(input_names[i], - dims_mapping) - new_dist_op = DistributedOperator(dist_op.serial_op, - new_op_dist_attr) + new_op_dist_attr.set_input_dims_mapping( + input_names[i], dims_mapping + ) + new_dist_op = DistributedOperator( + dist_op.serial_op, new_op_dist_attr + ) dist_op_impls = find_compatible_distributed_operator_impls( - new_dist_op, fwd=True) + new_dist_op, fwd=True + ) if dist_op_impls is not None: input_dims_mapping_candidates.append(dims_mapping_list) @@ -344,42 +413,58 @@ class ParallelTuner: output_names.append(tensor_name) if dims_mapping_len < 1: dims_mapping_generated.append( - [copy.deepcopy(original_dims_mapping)]) + [copy.deepcopy(original_dims_mapping)] + ) else: dims_mapping_generated.append( self._generate_dims_mapping_candidates( - dims_mapping_len, process_mesh_len)) + dims_mapping_len, process_mesh_len + ) + ) output_dims_mapping_candidates = [] for dims_mapping_list in itertools.product(*dims_mapping_generated): dims_mapping_list = list(dims_mapping_list) assert len(dims_mapping_list) == len(output_names) for i, dims_mapping in enumerate(dims_mapping_list): new_op_dist_attr.set_output_dims_mapping( - output_names[i], dims_mapping) - new_dist_op = DistributedOperator(dist_op.serial_op, - new_op_dist_attr) + output_names[i], dims_mapping + ) + new_dist_op = DistributedOperator( + dist_op.serial_op, new_op_dist_attr + ) dist_op_impls = find_compatible_distributed_operator_impls( - new_dist_op, fwd=False) + new_dist_op, fwd=False + ) if dist_op_impls is not None: output_dims_mapping_candidates.append(dims_mapping_list) if not input_dims_mapping_candidates and output_dims_mapping_candidates: - inout_dims_mapping_generated = [[[[-2]]], - output_dims_mapping_candidates] - elif input_dims_mapping_candidates and not output_dims_mapping_candidates: inout_dims_mapping_generated = [ - input_dims_mapping_candidates, [[[-2]]] + [[[-2]]], + output_dims_mapping_candidates, ] - elif not input_dims_mapping_candidates and not output_dims_mapping_candidates: + elif ( + input_dims_mapping_candidates and not output_dims_mapping_candidates + ): + inout_dims_mapping_generated = [ + input_dims_mapping_candidates, + [[[-2]]], + ] + elif ( + not input_dims_mapping_candidates + and not output_dims_mapping_candidates + ): inout_dims_mapping_generated = [[[[-2]]], [[[-2]]]] else: inout_dims_mapping_generated = [ - input_dims_mapping_candidates, output_dims_mapping_candidates + input_dims_mapping_candidates, + output_dims_mapping_candidates, ] # Find valid dims_mapping generated for both inputs and outputs cached_dist_attr_candidates = [] for inout_dims_mapping_list in itertools.product( - *inout_dims_mapping_generated): + *inout_dims_mapping_generated + ): assert len(inout_dims_mapping_list) == 2 if input_dims_mapping_candidates: assert len(inout_dims_mapping_list[0]) == len(input_names) @@ -389,35 +474,45 @@ class ParallelTuner: for i, dims_mapping in enumerate(inout_dims_mapping_list[0]): if dims_mapping != [-2]: new_op_dist_attr.set_input_dims_mapping( - input_names[i], dims_mapping) + input_names[i], dims_mapping + ) # set the dims_mappings for outputs for i, dims_mapping in enumerate(inout_dims_mapping_list[1]): if dims_mapping != [-2]: new_op_dist_attr.set_output_dims_mapping( - output_names[i], dims_mapping) - new_dist_op = DistributedOperator(dist_op.serial_op, - new_op_dist_attr) + output_names[i], dims_mapping + ) + new_dist_op = DistributedOperator( + dist_op.serial_op, new_op_dist_attr + ) dist_op_impls = find_compatible_distributed_operator_impls( - new_dist_op, partial=False) + new_dist_op, partial=False + ) if dist_op_impls is None: continue for dist_op_impl in dist_op_impls: new_op_dist_attr.impl_type = dist_op_impl.type new_op_dist_attr.impl_idx = dist_op_impl.idx cached_dist_attr_candidates.append( - copy.deepcopy(new_op_dist_attr)) + copy.deepcopy(new_op_dist_attr) + ) self._cached_candidates_info[key].append(cached_dist_attr_candidates) return self._cached_candidates_info[key][2] def construct_space(self): inter_node_partitions, intra_node_partitions = self._partition_devices( - self._num_machines, self._num_devices_per_machine) - self._space.choice("inter_node_partitions", - inter_node_partitions, - default=inter_node_partitions[0]) - self._space.choice("intra_node_partitions", - intra_node_partitions, - default=intra_node_partitions[0]) + self._num_machines, self._num_devices_per_machine + ) + self._space.choice( + "inter_node_partitions", + inter_node_partitions, + default=inter_node_partitions[0], + ) + self._space.choice( + "intra_node_partitions", + intra_node_partitions, + default=intra_node_partitions[0], + ) dist_ops = self._dist_context._dist_ops_for_program for op_id, dist_op in dist_ops.items(): @@ -433,17 +528,22 @@ class ParallelTuner: if op_type in self._exclude_op_types: del self._concerned_dist_ops[op_id] - print("Number of the concered dist ops", - len(self._concerned_dist_ops), - flush=True) + print( + "Number of the concered dist ops", + len(self._concerned_dist_ops), + flush=True, + ) search_space = 1 for op_id, dist_op in self._concerned_dist_ops.items(): op_dist_attr_candidates = self._generate_dist_attr_candidates( - op_id, dist_op) + op_id, dist_op + ) search_space *= len(op_dist_attr_candidates) - self._space.choice(str(op_id), - op_dist_attr_candidates, - default=op_dist_attr_candidates[0]) + self._space.choice( + str(op_id), + op_dist_attr_candidates, + default=op_dist_attr_candidates[0], + ) def _compute_values_hash(self, values): keys = sorted(values.keys()) @@ -522,10 +622,10 @@ class ParallelTuner: for _ in pipeline_starts[1:-1]: directions.append(Boolean("direction")) sizes.append( - IntRange("size", - start=0, - stop=half_ops_per_stage, - endpoint=True)) + IntRange( + "size", start=0, stop=half_ops_per_stage, endpoint=True + ) + ) for i, start in enumerate(pipeline_starts[1:-1]): direction = directions[i].random(self._seed) size = sizes[i].random(self._seed) @@ -538,19 +638,23 @@ class ParallelTuner: # Don't change the last start new_pipeline_starts.append(pipeline_starts[-1]) # Validate the new starts - print("Adjusted pipeline starts", - new_pipeline_starts, - half_ops_per_stage, - pipeline_starts, - flush=True) + print( + "Adjusted pipeline starts", + new_pipeline_starts, + half_ops_per_stage, + pipeline_starts, + flush=True, + ) for i, new_start in enumerate(new_pipeline_starts[1:]): assert new_start > new_pipeline_starts[i] return new_pipeline_starts else: - print("Non-adjusted pipeline starts", - pipeline_starts, - half_ops_per_stage, - flush=True) + print( + "Non-adjusted pipeline starts", + pipeline_starts, + half_ops_per_stage, + flush=True, + ) return pipeline_starts def _apply_pipeline_partition(self, process_mesh_list): @@ -620,12 +724,12 @@ class ParallelTuner: # if dim_mapping != -1 \ # and (tensor_shape[i] % process_shape[dim_mapping] != 0 \ # or dynamic_dims[i] == 1): - if dim_mapping != -1 \ - and (tensor_shape[i] % process_shape[dim_mapping] != 0): + if dim_mapping != -1 and ( + tensor_shape[i] % process_shape[dim_mapping] != 0 + ): dims_mapping[i] = -1 # it is a fix-bug - if dim_mapping != -1 \ - and process_shape[dim_mapping] == 1: + if dim_mapping != -1 and process_shape[dim_mapping] == 1: dims_mapping[i] = -1 for arg_name in dist_attr.outputs_dist_attrs.keys(): @@ -651,20 +755,22 @@ class ParallelTuner: else: continue for i, dim_mapping in enumerate(dims_mapping): - if dim_mapping != -1 \ - and (tensor_shape[i] % process_shape[dim_mapping] != 0): + if dim_mapping != -1 and ( + tensor_shape[i] % process_shape[dim_mapping] != 0 + ): dims_mapping[i] = -1 # it is a fix-bug - if dim_mapping != -1 \ - and process_shape[dim_mapping] == 1: + if dim_mapping != -1 and process_shape[dim_mapping] == 1: dims_mapping[i] = -1 dist_op_impls = find_compatible_distributed_operator_impls( - dist_op, partial=False) + dist_op, partial=False + ) serial_op_type = dist_op.serial_op.type if dist_op_impls is not None and ( - serial_op_type != "fused_softmax_mask_upper_triangle" - or self._check_fused_softmax_mask_upper_triangle(dist_op)): + serial_op_type != "fused_softmax_mask_upper_triangle" + or self._check_fused_softmax_mask_upper_triangle(dist_op) + ): dist_op.dist_attr.impl_type = dist_op_impls[0].type dist_op.dist_attr.impl_idx = dist_op_impls[0].idx else: @@ -684,14 +790,20 @@ class ParallelTuner: """The last_but_one dim shoule be equal to last dim.""" input_name = dist_op.serial_op.input_arg_names[0] input_dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - input_name) + input_name + ) topology = dist_op.dist_attr.process_mesh.topology input_tensor = dist_op.get_serial_input(input_name) - last_but_one_dim = input_tensor.shape[-2] // topology[ - input_dims_mapping[-2]] if input_dims_mapping[ - -2] != -1 else input_tensor.shape[-2] - last_dim = input_tensor.shape[-1] // topology[input_dims_mapping[ - -1]] if input_dims_mapping[-1] != -1 else input_tensor.shape[-1] + last_but_one_dim = ( + input_tensor.shape[-2] // topology[input_dims_mapping[-2]] + if input_dims_mapping[-2] != -1 + else input_tensor.shape[-2] + ) + last_dim = ( + input_tensor.shape[-1] // topology[input_dims_mapping[-1]] + if input_dims_mapping[-1] != -1 + else input_tensor.shape[-1] + ) if last_but_one_dim == last_dim: return True return False @@ -709,30 +821,37 @@ class ParallelTuner: inter_node_partition = trial.space.values["inter_node_partitions"] intra_node_partition = trial.space.values["intra_node_partitions"] process_mesh_list = self._generate_process_mesh_list( - inter_node_partition, intra_node_partition) + inter_node_partition, intra_node_partition + ) print("\tprocess_mesh list", process_mesh_list, flush=True) op_id_to_process_mesh = self._apply_pipeline_partition( - process_mesh_list) + process_mesh_list + ) if op_id_to_process_mesh is None: print("Operators are less than pipeline stages", flush=True) return results op_id_to_dist_attr = {} for name, value in trial.space.values.items(): - if name != "inter_node_partitions" \ - and name !="intra_node_partitions": + if ( + name != "inter_node_partitions" + and name != "intra_node_partitions" + ): op_id_to_dist_attr[int(name)] = value end_time = time.time() cur_sample_time = end_time - start_time - self._sample_time = (num_prev_trials * self._sample_time + - cur_sample_time) / self._num_trials - print("\tsample_time", - num_prev_trials, - self._num_trials, - self._sample_time, - cur_sample_time, - flush=True) + self._sample_time = ( + num_prev_trials * self._sample_time + cur_sample_time + ) / self._num_trials + print( + "\tsample_time", + num_prev_trials, + self._num_trials, + self._sample_time, + cur_sample_time, + flush=True, + ) assert len(op_id_to_process_mesh) == len(op_id_to_dist_attr) @@ -740,42 +859,52 @@ class ParallelTuner: for op_id, process_mesh in op_id_to_process_mesh.items(): dist_op = self._dist_context._dist_ops_for_program[op_id] dist_op.dist_attr = copy.deepcopy(op_id_to_dist_attr[op_id]) - assert dist_op.dist_attr.impl_type == op_id_to_dist_attr[ - op_id].impl_type - assert dist_op.dist_attr.impl_idx == op_id_to_dist_attr[ - op_id].impl_idx + assert ( + dist_op.dist_attr.impl_type + == op_id_to_dist_attr[op_id].impl_type + ) + assert ( + dist_op.dist_attr.impl_idx == op_id_to_dist_attr[op_id].impl_idx + ) dist_op.dist_attr.process_mesh = process_mesh self._amend_dist_attr() self._completer._complete_tensor_dist_attr_by_op() self._dist_context.block_state.parse_forward_blocks( - self._dist_context.serial_main_program) + self._dist_context.serial_main_program + ) end_time = time.time() cur_complete_time = end_time - start_time - self._complete_time = (num_prev_trials * self._complete_time + - cur_complete_time) / self._num_trials - print("\tcomplete_time", - num_prev_trials, - self._num_trials, - self._complete_time, - cur_complete_time, - flush=True) + self._complete_time = ( + num_prev_trials * self._complete_time + cur_complete_time + ) / self._num_trials + print( + "\tcomplete_time", + num_prev_trials, + self._num_trials, + self._complete_time, + cur_complete_time, + flush=True, + ) start_time = time.time() estimate_time = self._estimate_trial() end_time = time.time() cur_estimate_time = end_time - start_time - self._estimate_time = (num_prev_trials * self._estimate_time + - cur_estimate_time) / self._num_trials - print("\testimate_time", - num_prev_trials, - self._num_trials, - self._estimate_time, - cur_estimate_time, - estimate_time, - flush=True) + self._estimate_time = ( + num_prev_trials * self._estimate_time + cur_estimate_time + ) / self._num_trials + print( + "\testimate_time", + num_prev_trials, + self._num_trials, + self._estimate_time, + cur_estimate_time, + estimate_time, + flush=True, + ) results = {"estimate_time": estimate_time} return results @@ -791,12 +920,14 @@ class ParallelTuner: self._estimator = CostEstimator( self._dist_context.serial_main_program, self._cluster, - loop_count=self._loop_count) + loop_count=self._loop_count, + ) elif self._mode == "predict": self._estimator = CostEstimator( self._dist_context.serial_main_program, self._cluster, - loop_count=self._loop_count) + loop_count=self._loop_count, + ) elif self._mode == "train": # get serial main program with backward serial_main_program = self._dist_context.serial_main_program @@ -806,18 +937,23 @@ class ParallelTuner: # Generate backward serial_loss = self._dist_context.serial_fetch_vars["loss"][0] params_grads = self._parallelizer._generate_backward( - serial_main_program, serial_startup_program, serial_loss) + serial_main_program, serial_startup_program, serial_loss + ) # Generate optimizer optimizer_ops = self._parallelizer._generate_optimizer( - serial_main_program, serial_startup_program, serial_optimizer, - params_grads) - self._estimator = CostEstimator(serial_main_program, - self._cluster, - loop_count=self._loop_count) + serial_main_program, + serial_startup_program, + serial_optimizer, + params_grads, + ) + self._estimator = CostEstimator( + serial_main_program, self._cluster, loop_count=self._loop_count + ) max_memory = self._estimator._estimate_max_memory_by_dist_op( - self._dist_context) + self._dist_context + ) print("\tmax_memory", "{:,}".format(max_memory), flush=True) # The max memory must be less than 80% 32GB (hard code) if max_memory > 32 * 0.8 * 1024 * 1024 * 1024: @@ -829,18 +965,23 @@ class ParallelTuner: def _store_init_parallel_strategy(self): # If there is no annotation information, use the dp as the initial parallel strategy. # TODO: we should need a better way to set up the initial parallel strategy. - if not self._dist_context.has_annotation \ - or not self._dist_context.process_meshes: + if ( + not self._dist_context.has_annotation + or not self._dist_context.process_meshes + ): ranks = self._num_machines * self._num_devices_per_machine tensor_node = self._dist_context._serial_ordered_tensor_nodes[0] tensor_node_id = _node_id(tensor_node) tensor = self._dist_context._dist_tensors_for_graph[ - tensor_node_id].serial_tensor + tensor_node_id + ].serial_tensor tensor_dist_attr = self._dist_context._dist_tensors_for_graph[ - tensor_node_id].dist_attr + tensor_node_id + ].dist_attr tensor_dist_attr.process_mesh = ProcessMesh(list(range(ranks))) self._dist_context._process_meshes.append( - tensor_dist_attr.process_mesh) + tensor_dist_attr.process_mesh + ) tensor_dist_attr.dims_mapping = [0] + [ -1 for _ in range(len(tensor.shape) - 1) ] @@ -851,23 +992,30 @@ class ParallelTuner: # Do the sharding propagation self._completer.complete_forward_annotation() self._dist_context.block_state.parse_forward_blocks( - self._dist_context.serial_main_program) + self._dist_context.serial_main_program + ) # Backup the intital parallel strategy self._init_parallel_strategy[0] = copy.deepcopy( - self._dist_context._dist_tensors_for_program) + self._dist_context._dist_tensors_for_program + ) self._init_parallel_strategy[1] = copy.deepcopy( - self._dist_context._dist_ops_for_program) + self._dist_context._dist_ops_for_program + ) self._init_parallel_strategy[2] = copy.deepcopy( - self._dist_context.process_meshes) + self._dist_context.process_meshes + ) # Initialize the best parallel strategy to the initial one self._best_parallel_strategy[0] = copy.deepcopy( - self._dist_context._dist_tensors_for_program) + self._dist_context._dist_tensors_for_program + ) self._best_parallel_strategy[1] = copy.deepcopy( - self._dist_context._dist_ops_for_program) + self._dist_context._dist_ops_for_program + ) self._best_parallel_strategy[2] = copy.deepcopy( - self._dist_context._process_meshes) + self._dist_context._process_meshes + ) def _store_best_parallel_strategy(self): # Swap the best and the current parallel strategy @@ -876,9 +1024,11 @@ class ParallelTuner: tmp[1] = self._best_parallel_strategy[1] tmp[2] = self._best_parallel_strategy[2] self._best_parallel_strategy[ - 0] = self._dist_context._dist_tensors_for_program + 0 + ] = self._dist_context._dist_tensors_for_program self._best_parallel_strategy[ - 1] = self._dist_context._dist_ops_for_program + 1 + ] = self._dist_context._dist_ops_for_program self._best_parallel_strategy[2] = self._dist_context._process_meshes self._dist_context._dist_tensors_for_program = tmp[0] self._dist_context._dist_ops_for_program = tmp[1] @@ -894,19 +1044,23 @@ class ParallelTuner: # We have to restore the distributed context, because the estimation of one trail need to # generate the backward and update parts. Since we will do the tuning process, # here we only need to reset all distributed information to the default one. - self._dist_context._restore(serial=True, - serial_mode="to_backup", - dist=True, - dist_mode="to_default") + self._dist_context._restore( + serial=True, + serial_mode="to_backup", + dist=True, + dist_mode="to_default", + ) best_time = init_time start_time = time.time() self.construct_space() end_time = time.time() - print("construct_space time", - self._num_trials, - end_time - start_time, - flush=True) + print( + "construct_space time", + self._num_trials, + end_time - start_time, + flush=True, + ) create_trial_time = 0.0 eval_trial_time = 0.0 self._sample_time = 0.0 @@ -921,14 +1075,17 @@ class ParallelTuner: num_prev_trials = self._num_trials - 1 end_time = time.time() cur_create_trial_time = end_time - start_time - create_trial_time = (num_prev_trials * create_trial_time + - cur_create_trial_time) / self._num_trials - print("create_trial time", - num_prev_trials, - self._num_trials, - create_trial_time, - cur_create_trial_time, - flush=True) + create_trial_time = ( + num_prev_trials * create_trial_time + cur_create_trial_time + ) / self._num_trials + print( + "create_trial time", + num_prev_trials, + self._num_trials, + create_trial_time, + cur_create_trial_time, + flush=True, + ) if trial.status == TrialStatus.STOPPED: break # We need to backup the distributed context, because the evaluation of one trail will @@ -940,15 +1097,18 @@ class ParallelTuner: results = self._eval_trial(trial) end_time = time.time() cur_eval_trial_time = end_time - start_time - eval_trial_time = (num_prev_trials * eval_trial_time + - cur_eval_trial_time) / self._num_trials - print("eval_trial time", - num_prev_trials, - self._num_trials, - eval_trial_time, - cur_eval_trial_time, - "\n", - flush=True) + eval_trial_time = ( + num_prev_trials * eval_trial_time + cur_eval_trial_time + ) / self._num_trials + print( + "eval_trial time", + num_prev_trials, + self._num_trials, + eval_trial_time, + cur_eval_trial_time, + "\n", + flush=True, + ) cur_time = results["estimate_time"] if cur_time < best_time: @@ -956,13 +1116,17 @@ class ParallelTuner: self._store_best_parallel_strategy() best_time = cur_time # We need to restore the distributed context and reset the distributed information to the default. - self._dist_context._restore(serial=True, - serial_mode="to_backup", - dist=True, - dist_mode="to_default") + self._dist_context._restore( + serial=True, + serial_mode="to_backup", + dist=True, + dist_mode="to_default", + ) # Select the best parallel strategy - self._dist_context._dist_tensors_for_program = self._best_parallel_strategy[ - 0] + self._dist_context._dist_tensors_for_program = ( + self._best_parallel_strategy[0] + ) self._dist_context._dist_ops_for_program = self._best_parallel_strategy[ - 1] + 1 + ] self._dist_context._process_meshes = self._best_parallel_strategy[2] diff --git a/python/paddle/distributed/auto_parallel/tuner/profiler.py b/python/paddle/distributed/auto_parallel/tuner/profiler.py index 4b2655028bf..1aeafbea764 100644 --- a/python/paddle/distributed/auto_parallel/tuner/profiler.py +++ b/python/paddle/distributed/auto_parallel/tuner/profiler.py @@ -22,8 +22,13 @@ import time import paddle from paddle.fluid.framework import Program, _current_expected_place from paddle.fluid.framework import Operator -from paddle.distributed.auto_parallel.process_group import get_all_process_groups, new_process_group -from paddle.distributed.auto_parallel.dist_loader import DistributedDataLoaderFromGenerator +from paddle.distributed.auto_parallel.process_group import ( + get_all_process_groups, + new_process_group, +) +from paddle.distributed.auto_parallel.dist_loader import ( + DistributedDataLoaderFromGenerator, +) from paddle.distributed.collective import _get_global_env paddle.enable_static() @@ -44,25 +49,32 @@ def parse_args(): "--profile_start_step", default=10, type=int, - help="integer indicates the warmup step before starting profile.") - parser.add_argument("--profile_end_step", - default=30, - type=int, - help="integer indicates at the end step of profile.") - parser.add_argument("--rank", - type=int, - required=True, - help="the rank id of the this process.") - parser.add_argument("--device_id", - type=int, - required=True, - help="the device id of the this process.") + help="integer indicates the warmup step before starting profile.", + ) + parser.add_argument( + "--profile_end_step", + default=30, + type=int, + help="integer indicates at the end step of profile.", + ) + parser.add_argument( + "--rank", + type=int, + required=True, + help="the rank id of the this process.", + ) + parser.add_argument( + "--device_id", + type=int, + required=True, + help="the device id of the this process.", + ) parser.add_argument( "--ctx_filename", type=str, required=True, - help= - "the filename to the profile context file saved by optimizaiton tuner") + help="the filename to the profile context file saved by optimizaiton tuner", + ) args = parser.parse_args() @@ -109,11 +121,9 @@ def get_cpp_error_type(error): return error_type -def create_dataloader(main_program, - startup_program, - profile_ctx, - epochs=1, - steps_per_epoch=None): +def create_dataloader( + main_program, startup_program, profile_ctx, epochs=1, steps_per_epoch=None +): dataset = profile_ctx["dataset"] main_block = main_program.global_block() @@ -141,7 +151,8 @@ def create_dataloader(main_program, epochs=epochs, steps_per_epoch=steps_per_epoch, data_parallel_world_size=dataset.dp_world_size, - data_parallel_rank=dataset.dp_rank) + data_parallel_rank=dataset.dp_rank, + ) # move read op from the end of program to the start of program new_op_size = len(main_block.ops) @@ -162,8 +173,12 @@ def init_comm(profile_ctx): dist_env = profile_ctx['distributed_env'] genv = _get_global_env() genv = dist_env - print("current process rank: {}, device_id: {}, ip: {}.", genv.rank, - genv.device_id, genv.current_endpoint) + print( + "current process rank: {}, device_id: {}, ip: {}.", + genv.rank, + genv.device_id, + genv.current_endpoint, + ) # init nccl comm group_map = profile_ctx['group_map'] @@ -201,8 +216,9 @@ def profiler(args): """ # load ctx if not os.path.isfile(args.ctx_filename): - raise ValueError("There is no profile context named {}.".format( - args.ctx_filename)) + raise ValueError( + "There is no profile context named {}.".format(args.ctx_filename) + ) with open(args.ctx_filename, 'rb') as f: profile_ctx = pickle.load(f, encoding='latin1') @@ -240,8 +256,9 @@ def profiler(args): print("step: %d, loss_print: %f" % (eval_step, loss[0])) eval_step += 1 - avg_tput = 1.0 * (args.profile_end_step - - args.profile_start_step) / duration + avg_tput = ( + 1.0 * (args.profile_end_step - args.profile_start_step) / duration + ) result_dict = { "Throughtput": avg_tput, diff --git a/python/paddle/distributed/auto_parallel/tuner/recorder.py b/python/paddle/distributed/auto_parallel/tuner/recorder.py index 4d088379d8b..2c838cfb149 100644 --- a/python/paddle/distributed/auto_parallel/tuner/recorder.py +++ b/python/paddle/distributed/auto_parallel/tuner/recorder.py @@ -71,7 +71,9 @@ class MetricRecords(object): if direction not in {"min", "max"}: raise ValueError( "direction should be one of {{min, max}}, but got: {}.".format( - direction)) + direction + ) + ) self._direction = direction self._records = {} diff --git a/python/paddle/distributed/auto_parallel/tuner/storable.py b/python/paddle/distributed/auto_parallel/tuner/storable.py index 18a0669d622..fb03070ad09 100644 --- a/python/paddle/distributed/auto_parallel/tuner/storable.py +++ b/python/paddle/distributed/auto_parallel/tuner/storable.py @@ -19,7 +19,6 @@ import json class Storable(object): - def get_state(self): raise NotImplementedError diff --git a/python/paddle/distributed/auto_parallel/tuner/trial.py b/python/paddle/distributed/auto_parallel/tuner/trial.py index 2c8963322a3..b152e407f5f 100644 --- a/python/paddle/distributed/auto_parallel/tuner/trial.py +++ b/python/paddle/distributed/auto_parallel/tuner/trial.py @@ -32,11 +32,9 @@ class TrialStatus: class Trial(Storable): - - def __init__(self, - tunable_space, - trial_id=None, - status=TrialStatus.RUNNING): + def __init__( + self, tunable_space, trial_id=None, status=TrialStatus.RUNNING + ): self._id = _generate_trial_id() if trial_id is None else trial_id self._space = tunable_space self._recorder = MetricsRecorder() @@ -115,13 +113,14 @@ class Trial(Storable): class OptimizationTunerTrial(Trial): - - def __init__(self, - config, - name, - changed_configs, - trial_id=None, - status=TrialStatus.RUNNING): + def __init__( + self, + config, + name, + changed_configs, + trial_id=None, + status=TrialStatus.RUNNING, + ): super(OptimizationTunerTrial, self).__init__(config, trial_id, status) self._name = name self._changed_configs = changed_configs @@ -140,7 +139,8 @@ class OptimizationTunerTrial(Trial): h1_format = " " + "|{{:^{}s}}|\n".format(length) h2_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format( - max_k, " " * spacing, max_v) + max_k, " " * spacing, max_v + ) border = " +" + "".join(["="] * length) + "+" line = " +" + "".join(["-"] * length) + "+" @@ -158,7 +158,8 @@ class OptimizationTunerTrial(Trial): keys = my_configs.to_dict().keys() for key in keys: draws += h2_format.format( - key, str(my_configs.to_dict().get(key, None))) + key, str(my_configs.to_dict().get(key, None)) + ) result_res = draws + border return result_res diff --git a/python/paddle/distributed/auto_parallel/tuner/tunable_space.py b/python/paddle/distributed/auto_parallel/tuner/tunable_space.py index 2009f1d911c..e3e503401b4 100644 --- a/python/paddle/distributed/auto_parallel/tuner/tunable_space.py +++ b/python/paddle/distributed/auto_parallel/tuner/tunable_space.py @@ -104,29 +104,24 @@ class TunableSpace(object): return self._retrieve(tv) def int_range(self, name, start, stop, step=1, default=None): - tv = IntRange(name=name, - start=start, - stop=stop, - step=step, - default=default) + tv = IntRange( + name=name, start=start, stop=stop, step=step, default=default + ) return self._retrieve(tv) def float_range(self, name, start, stop, step=None, default=None): - tv = FloatRange(name=name, - start=start, - stop=stop, - step=step, - default=default) + tv = FloatRange( + name=name, start=start, stop=stop, step=step, default=default + ) return self._retrieve(tv) def get_state(self): return { - "variables": [{ - "class_name": v.__class__.__name__, - "state": v.get_state() - } for v in self._variables.values()], - "values": - dict((k, v) for (k, v) in self.values.items()) + "variables": [ + {"class_name": v.__class__.__name__, "state": v.get_state()} + for v in self._variables.values() + ], + "values": dict((k, v) for (k, v) in self.values.items()), } @classmethod @@ -146,11 +141,16 @@ def _deserialize_tunable_variable(state): if isinstance(state, classes): return state - if (not isinstance(state, dict) or "class_name" not in state - or "state" not in state): + if ( + not isinstance(state, dict) + or "class_name" not in state + or "state" not in state + ): raise ValueError( - "Expect state to be a python dict containing class_name and state as keys, but found {}" - .format(state)) + "Expect state to be a python dict containing class_name and state as keys, but found {}".format( + state + ) + ) cls_name = state["class_name"] cls = cls_name_to_cls[cls_name] diff --git a/python/paddle/distributed/auto_parallel/tuner/tunable_variable.py b/python/paddle/distributed/auto_parallel/tuner/tunable_variable.py index 31dd07aad37..7b09f3a3b9a 100644 --- a/python/paddle/distributed/auto_parallel/tuner/tunable_variable.py +++ b/python/paddle/distributed/auto_parallel/tuner/tunable_variable.py @@ -50,7 +50,9 @@ class Fixed(TunableVariable): if not isinstance(default, (str, int, float, bool)): raise ValueError( "Fixed must be an str, int, float or bool, but found {}".format( - default)) + default + ) + ) self._default = default def random(self, seed=None): @@ -69,7 +71,8 @@ class Boolean(TunableVariable): super(Boolean, self).__init__(name=name, default=default) if default not in {True, False}: raise ValueError( - "default must be a Python boolean, but got {}".format(default)) + "default must be a Python boolean, but got {}".format(default) + ) def random(self, seed=None): rng = np.random.default_rng(seed) @@ -77,19 +80,21 @@ class Boolean(TunableVariable): def __repr__(self): return 'Boolean(name: "{}", default: {})'.format( - self.name, self.default) + self.name, self.default + ) class Choice(TunableVariable): - def __init__(self, name, values, default=None): super(Choice, self).__init__(name=name, default=default) types = set(type(v) for v in values) if len(types) > 1: raise TypeError( - "Choice can contain only one type of value, but found values: {} with types: {}." - .format(str(values), str(types))) + "Choice can contain only one type of value, but found values: {} with types: {}.".format( + str(values), str(types) + ) + ) self._is_unknown_type = False if isinstance(values[0], str): @@ -115,8 +120,10 @@ class Choice(TunableVariable): if default is not None and default not in values: raise ValueError( - "The default value should be one of the choices {}, but found {}" - .format(values, default)) + "The default value should be one of the choices {}, but found {}".format( + values, default + ) + ) self._default = default @property @@ -142,7 +149,8 @@ class Choice(TunableVariable): def __repr__(self): return 'Choice(name: "{}", values: {}, default: {})'.format( - self.name, self.values, self.default) + self.name, self.values, self.default + ) class IntRange(TunableVariable): @@ -187,13 +195,15 @@ class IntRange(TunableVariable): def _check_int(self, val): int_val = int(val) if int_val != val: - raise ValueError("Expects val is an int, but found: {}.".format( - str(val))) + raise ValueError( + "Expects val is an int, but found: {}.".format(str(val)) + ) return int_val def __repr__(self): return "IntRange(name: {}, start: {}, stop: {}, step: {}, default: {})".format( - self.name, self.start, self.stop, self.step, self.default) + self.name, self.start, self.stop, self.step, self.default + ) class FloatRange(TunableVariable): @@ -201,13 +211,9 @@ class FloatRange(TunableVariable): Float range. """ - def __init__(self, - name, - start, - stop, - step=None, - default=None, - endpoint=False): + def __init__( + self, name, start, stop, step=None, default=None, endpoint=False + ): super(FloatRange, self).__init__(name=name, default=default) self.stop = float(stop) self.start = float(start) @@ -246,5 +252,10 @@ class FloatRange(TunableVariable): def __repr__(self): return "FloatRange(name: {}, start: {}, stop: {}, step: {}, default: {}, endpoint: {})".format( - self.name, self.start, self.stop, self.step, self.default, - self.endpoint) + self.name, + self.start, + self.stop, + self.step, + self.default, + self.endpoint, + ) diff --git a/python/paddle/distributed/auto_parallel/utils.py b/python/paddle/distributed/auto_parallel/utils.py index cf6f506f8c5..8b7ec647f6e 100644 --- a/python/paddle/distributed/auto_parallel/utils.py +++ b/python/paddle/distributed/auto_parallel/utils.py @@ -23,12 +23,18 @@ from functools import reduce import paddle.fluid.core as core from paddle.distributed.fleet.meta_optimizers.common import OpRole -from paddle.distributed.auto_parallel.process_group import get_all_process_groups +from paddle.distributed.auto_parallel.process_group import ( + get_all_process_groups, +) from paddle.fluid.io import is_parameter, is_belong_to_optimizer -from paddle.distributed.auto_parallel.dist_attribute import TensorDistributedAttribute, OperatorDistributedAttribute +from paddle.distributed.auto_parallel.dist_attribute import ( + TensorDistributedAttribute, + OperatorDistributedAttribute, +) __not_shape_var_type__ = [ - core.VarDesc.VarType.READER, core.VarDesc.VarType.STEP_SCOPES + core.VarDesc.VarType.READER, + core.VarDesc.VarType.STEP_SCOPES, ] @@ -39,7 +45,8 @@ def get_logger(log_level, name="auto_parallel"): logger.setLevel(log_level) log_handler = logging.StreamHandler() log_format = logging.Formatter( - '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s') + '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s' + ) log_handler.setFormatter(log_format) logger.addHandler(log_handler) return logger @@ -114,8 +121,11 @@ def verify_shard_spec(shard_spec, tensor_shape, process_mesh): if not verify_dims_mapping(dims_mapping, process_mesh): return False for i in range(len(tensor_shape)): - if dims_mapping[i] != -1 and tensor_shape[i] > 0 \ - and tensor_shape[i] % process_mesh.shape[dims_mapping[i]] != 0: + if ( + dims_mapping[i] != -1 + and tensor_shape[i] > 0 + and tensor_shape[i] % process_mesh.shape[dims_mapping[i]] != 0 + ): return False return True @@ -141,14 +151,17 @@ def compute_compatible_dims_mapping(dims_mapping_list): return None length = len(dims_mapping_list[0]) for dims_mapping in dims_mapping_list: - assert dims_mapping is not None, \ - "Dims mapping must not be None for compatible computation" - assert len(dims_mapping) == length, \ - "The length of dims_mapping in list must be same for compatible computation." + assert ( + dims_mapping is not None + ), "Dims mapping must not be None for compatible computation" + assert ( + len(dims_mapping) == length + ), "The length of dims_mapping in list must be same for compatible computation." compatible_result = [] for dim_mappings in zip(*dims_mapping_list): compatible_dim_mapping = compute_compatible_dim_mapping( - list(dim_mappings)) + list(dim_mappings) + ) if compatible_dim_mapping is None: return None compatible_result.append(compatible_dim_mapping) @@ -161,7 +174,10 @@ def compute_compatible_process_mesh(process_mesh_list): return compatible_process_mesh for process_mesh in process_mesh_list: if process_mesh is not None: - if compatible_process_mesh is None or compatible_process_mesh == process_mesh: + if ( + compatible_process_mesh is None + or compatible_process_mesh == process_mesh + ): compatible_process_mesh = process_mesh else: return None @@ -201,15 +217,18 @@ def remove_distributed_attr_suffix(name): def check_distributed_attr_for_program(program, dist_context=None): from .dist_context import get_default_distributed_context + if dist_context is None: dist_context = get_default_distributed_context() - assert dist_context.is_initialized_for_program(), \ - "Distributed attributes must be initialized before check." + assert ( + dist_context.is_initialized_for_program() + ), "Distributed attributes must be initialized before check." for block in program.blocks: for tensor in block.vars.values(): dist_tensor = dist_context.get_dist_tensor_for_graph(tensor) tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - tensor) + tensor + ) if (tensor_dist_attr is not None) and (not dist_tensor.is_valid()): return False for op in block.ops: @@ -229,6 +248,7 @@ def print_program_with_dist_attr(program, dist_context=None): lock.acquire() from .dist_context import get_default_distributed_context from .dist_context import set_default_distributed_context + if dist_context is None: dist_context = get_default_distributed_context() print(program, flush=True) @@ -256,7 +276,8 @@ def _get_comm_group(processes, shape, axis, rank): # NOTE _linear_idx2coordinate assume processes mesh start with 0 and continuous # tricks to support processes mesh when it is not start with 0 or continuous assert rank in processes, "rank [{}] is NOT in processes group {}".format( - rank, processes) + rank, processes + ) rank_relatvie = processes.index(rank) coordinate = _linear_idx2coordinate(shape, rank_relatvie) coordinates_in_group = [coordinate[:] for i in range(shape[axis])] @@ -325,14 +346,19 @@ def _coordinate2linear_idx(mesh_shape, coordinate): assert len(mesh_shape) == len( coordinate ), "coordinate should have the same size as mesh shape, but got shape: {}, coordinate: {}".format( - mesh_shape, coordinate) + mesh_shape, coordinate + ) for i in range(len(mesh_shape)): - assert coordinate[ - i] >= 0, "index in dimension [{}] is least than zero. coordinate: {}".format( - i, coordinate) - assert coordinate[i] < mesh_shape[ - i], "index beyond extent in dimension [{}]. shape: {}, coordinate: {}".format( - i, mesh_shape, coordinate) + assert ( + coordinate[i] >= 0 + ), "index in dimension [{}] is least than zero. coordinate: {}".format( + i, coordinate + ) + assert ( + coordinate[i] < mesh_shape[i] + ), "index beyond extent in dimension [{}]. shape: {}, coordinate: {}".format( + i, mesh_shape, coordinate + ) base = mesh_shape[-1] linear_idx = coordinate[-1] @@ -365,11 +391,13 @@ def _linear_idx2coordinate(mesh_shape, linear_idx): """ assert linear_idx >= 0, "linear index [{}] is least than zero".format( - linear_idx) + linear_idx + ) assert linear_idx < np.prod( mesh_shape ), "linear index beyond the extent of mesh shape. shape: {}, linear index: {}".format( - mesh_shape, linear_idx) + mesh_shape, linear_idx + ) base = 1 coordinate = [-1] * len(mesh_shape) @@ -392,15 +420,17 @@ def _get_corresponding_rank(dist_context, target_mesh, rank): coordinate = None for mesh in dist_context.process_meshes: if rank in mesh.processes and mesh.topology == target_mesh.topology: - coordinate = _linear_idx2coordinate(mesh.topology, - mesh.processes.index(rank)) + coordinate = _linear_idx2coordinate( + mesh.topology, mesh.processes.index(rank) + ) break # assert coordinate is not None, "could NOT found rank [{}] in any registered mesh".format( # rank) if coordinate is not None: - return target_mesh.processes[_coordinate2linear_idx( - mesh.topology, coordinate)] + return target_mesh.processes[ + _coordinate2linear_idx(mesh.topology, coordinate) + ] else: return target_mesh.processes[0] @@ -412,7 +442,8 @@ def _get_unshard_dist_shape(var, dist_attr): assert len(var_shape) == len( mapping ), "variable shape [{}] and dim_mapping [{}] is NOT match !".format( - var_shape, mapping) + var_shape, mapping + ) new_shape = [] for idx in range(len(var_shape)): if var_shape[idx] == -1 or mapping[idx] == -1: @@ -425,13 +456,15 @@ def _get_unshard_dist_shape(var, dist_attr): def make_data_unshard(dist_main_prog, dist_startup_prog, dist_context=None): from .dist_context import get_default_distributed_context + if dist_context is None: dist_context = get_default_distributed_context() for var in dist_main_prog.list_vars(): if var.is_data: tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - var) + var + ) inverse_shape = _get_unshard_dist_shape(var, tensor_dist_attr) var.desc.set_shape(inverse_shape) dim_mapping = tensor_dist_attr.dims_mapping @@ -441,62 +474,76 @@ def make_data_unshard(dist_main_prog, dist_startup_prog, dist_context=None): def _update_addition_info(addition_info): - """ Update default addition_info with inputs """ + """Update default addition_info with inputs""" add_info = {"epoch": 0, "batch": 0, "batch_size": 0} if not addition_info: return add_info elif not isinstance(addition_info, dict): - raise TypeError("The type of 'addition_info' should be 'dict', " - "but got '{}'.".format(str(type(addition_info)))) + raise TypeError( + "The type of 'addition_info' should be 'dict', " + "but got '{}'.".format(str(type(addition_info))) + ) else: for item, value in addition_info.items(): if item not in ["epoch", "batch", "batch_size"]: raise ValueError( "The key of 'addition_info' should be one of the " "['epoch', 'batch', 'batch_size'], but got '{}'.".format( - str(item))) + str(item) + ) + ) if not isinstance(value, int): raise ValueError( "The value of 'addition_info' should be 'int', " - "but got '{}'.".format(str(type(value)))) + "but got '{}'.".format(str(type(value))) + ) add_info[item] = value return add_info def _check_valid_path(file_path): - """ Validity check of input file path """ + """Validity check of input file path""" if not file_path: return file_path elif isinstance(file_path, list): for file in file_path: if not isinstance(file, str): - raise TypeError("The type of file path should be 'str', " - "but got '{}'.".format(str(type(file)))) + raise TypeError( + "The type of file path should be 'str', " + "but got '{}'.".format(str(type(file))) + ) if not os.path.exists(file): raise ValueError( - "The file path '{}' does not exist.".format(file)) + "The file path '{}' does not exist.".format(file) + ) return file_path else: - raise TypeError("The type of file path should be 'list', " - "but got '{}'.".format(str(type(file_path)))) + raise TypeError( + "The type of file path should be 'list', " + "but got '{}'.".format(str(type(file_path))) + ) def _check_param_dict(param_dict): if not param_dict: raise ValueError("'param_dict' cannot be None.") elif not isinstance(param_dict, dict): - raise TypeError("The type of 'param_dict' should be 'dict', " - "but got '{}'.".format(str(type(param_dict)))) + raise TypeError( + "The type of 'param_dict' should be 'dict', " + "but got '{}'.".format(str(type(param_dict))) + ) else: for name, value in param_dict.items(): if not isinstance(name, str): raise TypeError( "The type of key of 'param_dict' should be 'str', " - "but got '{}'.".format(str(type(name)))) + "but got '{}'.".format(str(type(name))) + ) if not isinstance(value, paddle.fluid.LoDTensor): raise TypeError( "The type of value of 'param_dict' should be 'LoDTensor', " - "but got '{}'.".format(str(type(value)))) + "but got '{}'.".format(str(type(value))) + ) return param_dict @@ -504,33 +551,40 @@ def _check_dist_attr(dist_attr): if not dist_attr: return dist_attr elif not isinstance(dist_attr, dict): - raise TypeError("The type of 'dist_attr' should be 'dict', " - "but got '{}'.".format(str(type(dist_attr)))) + raise TypeError( + "The type of 'dist_attr' should be 'dict', " + "but got '{}'.".format(str(type(dist_attr))) + ) else: for name, value in dist_attr.items(): if not isinstance(name, str): raise TypeError( "The type of param name of 'dist_attr' should be 'str', " - "but got '{}'.".format(str(type(name)))) + "but got '{}'.".format(str(type(name))) + ) if not isinstance(value, dict): raise TypeError( "The type of distributed attribute should be 'dict', " - "but got '{}'".format(str(type(value)))) + "but got '{}'".format(str(type(value))) + ) attr = ['process_shape', 'process_group', 'dims_mapping'] if list(value.keys()) != attr: raise ValueError( "The key of distributed attribute should be " "'['process_shape', 'process_group', 'dims_mapping']', " - "but got {}.".format(str(value.keys()))) + "but got {}.".format(str(value.keys())) + ) return dist_attr -def save_distributed_checkpoint(program, - checkpoint_path, - dist_attr_path, - addition_info=None, - is_integrated=False, - dist_context=None): +def save_distributed_checkpoint( + program, + checkpoint_path, + dist_attr_path, + addition_info=None, + is_integrated=False, + dist_context=None, +): """ Save model parameter state, optimzer state, distributed attribute and additional information of each rank. @@ -569,7 +623,8 @@ def save_distributed_checkpoint(program, else: # TODO: integrate param before save raise NotImplementedError( - "Integrating parameter has not been implemented.") + "Integrating parameter has not been implemented." + ) def load_distributed_checkpoint(checkpoint_path, dist_attr_path): @@ -597,10 +652,10 @@ def load_distributed_checkpoint(checkpoint_path, dist_attr_path): './dist_attr_rank1.pdattr'] param_dict, dist_attr, add_info = load_distributed_checkpoint(ckpt_path, dist_attr_path) """ - assert _check_valid_path(checkpoint_path), \ - "'checkpoint_path' cannot be None." - assert _check_valid_path(dist_attr_path), \ - "'dist_attr_path' cannot be None." + assert _check_valid_path( + checkpoint_path + ), "'checkpoint_path' cannot be None." + assert _check_valid_path(dist_attr_path), "'dist_attr_path' cannot be None." state_dict_info = _load_distributed_state_dict(checkpoint_path) dist_attr = _load_distributed_attribute(dist_attr_path) @@ -609,10 +664,9 @@ def load_distributed_checkpoint(checkpoint_path, dist_attr_path): return param_dict, dist_attr, addition_info -def load_checkpoint_into_program(checkpoint_path, - dist_attr_path, - program, - dist_context=None): +def load_checkpoint_into_program( + checkpoint_path, dist_attr_path, program, dist_context=None +): """ Load parameter, optimizer, distributed attribute and addition_info into model. @@ -641,10 +695,10 @@ def load_checkpoint_into_program(checkpoint_path, from .dist_context import get_default_distributed_context assert isinstance(program, paddle.fluid.framework.Program) - assert _check_valid_path(checkpoint_path), \ - "'checkpoint_path' cannot be None." - assert _check_valid_path(dist_attr_path), \ - "'dist_attr_path' cannot be None." + assert _check_valid_path( + checkpoint_path + ), "'checkpoint_path' cannot be None." + assert _check_valid_path(dist_attr_path), "'dist_attr_path' cannot be None." if dist_context is None: dist_context = get_default_distributed_context() all_state_dict_info = _load_distributed_state_dict(checkpoint_path) @@ -652,9 +706,9 @@ def load_checkpoint_into_program(checkpoint_path, all_cur_dist_attr = get_dist_attr(program, dist_context) all_param_dict = all_state_dict_info["model"] addition_info = all_state_dict_info["addition_info"] - sliced_param_dict = merge_and_slice_parameter(all_param_dict, - all_pre_dist_attr, - all_cur_dist_attr) + sliced_param_dict = merge_and_slice_parameter( + all_param_dict, all_pre_dist_attr, all_cur_dist_attr + ) load_parameter_into_program(sliced_param_dict, program) return addition_info @@ -676,28 +730,31 @@ def load_parameter_into_program(param_dict, program): def _save_distributed_attribute(program, dist_attr_path, dist_context): - """ Save distributed attribute of all parameters """ + """Save distributed attribute of all parameters""" # TODO: just save a complete distributed attribute file rank_id = paddle.distributed.get_rank() - dist_attr_name = os.path.join(dist_attr_path, - "dist_attr_rank{}.pdattr".format(rank_id)) + dist_attr_name = os.path.join( + dist_attr_path, "dist_attr_rank{}.pdattr".format(rank_id) + ) dist_attr_dict = { "model": get_dist_attr(program, dist_context), - "world_size": paddle.distributed.get_world_size() + "world_size": paddle.distributed.get_world_size(), } paddle.save(dist_attr_dict, dist_attr_name) logging.info( - "Already saved distributed attribute to '{}'.".format(dist_attr_path)) + "Already saved distributed attribute to '{}'.".format(dist_attr_path) + ) def _load_distributed_attribute(dist_attr_path): - """ Load parameters' distributed attribute from dist_attr_path """ + """Load parameters' distributed attribute from dist_attr_path""" total_dist_attr = {} for dist_attr_file in dist_attr_path: dist_attr = paddle.load(dist_attr_file) pre_world_size = dist_attr["world_size"] - assert pre_world_size == len(dist_attr_path), \ - "The number of 'dist_attr_path' must be equal to the last training world size." + assert pre_world_size == len( + dist_attr_path + ), "The number of 'dist_attr_path' must be equal to the last training world size." for name, attr in dist_attr["model"].items(): if name not in total_dist_attr: total_dist_attr[name] = attr @@ -706,27 +763,29 @@ def _load_distributed_attribute(dist_attr_path): def _save_distributed_state_dict(program, addition_info, checkpoint_path): - """ Save parameters' state_dict """ + """Save parameters' state_dict""" rank = paddle.distributed.get_rank() - ckpt_file_name = os.path.join(checkpoint_path, - "model_state_rank{}.pdmodel".format(rank)) + ckpt_file_name = os.path.join( + checkpoint_path, "model_state_rank{}.pdmodel".format(rank) + ) state_dict = { "model": program.state_dict(), "world_size": paddle.distributed.get_world_size(), - "addition_info": addition_info + "addition_info": addition_info, } paddle.save(state_dict, ckpt_file_name) logging.info("Already saved model to '{}'.".format(checkpoint_path)) def _load_distributed_state_dict(checkpoint_path): - """ Load parameters' state_dict from checkpoint_path """ + """Load parameters' state_dict from checkpoint_path""" all_state_dict = {} for idx, ckpt_file in enumerate(checkpoint_path): state_dict_info = paddle.load(ckpt_file, return_numpy=True) pre_world_size = state_dict_info["world_size"] - assert pre_world_size == len(checkpoint_path), \ - "The number of 'checkpoint_path' must be equal to the last training world size." + assert pre_world_size == len( + checkpoint_path + ), "The number of 'checkpoint_path' must be equal to the last training world size." if idx == 0: addition_info = state_dict_info["addition_info"] for name, value in state_dict_info["model"].items(): @@ -737,7 +796,7 @@ def _load_distributed_state_dict(checkpoint_path): all_state_dict_info = { "model": all_state_dict, - "addition_info": addition_info + "addition_info": addition_info, } return all_state_dict_info @@ -758,13 +817,14 @@ def get_dist_attr(program, dist_context=None): for var in program.list_vars(): if is_parameter(var) or is_belong_to_optimizer(var): tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - var) + var + ) process_mesh = tensor_dist_attr.process_mesh dims_mapping = tensor_dist_attr.dims_mapping dist_attr[var.name] = { "process_shape": process_mesh.topology, "process_group": process_mesh.processes, - "dims_mapping": dims_mapping + "dims_mapping": dims_mapping, } return dist_attr @@ -782,19 +842,26 @@ def merge_and_slice_parameter(dist_param_dict, pre_dist_attr, cur_dist_attr): dist_param_dict(dict): parameters' value of current rank. """ assert _check_dist_attr(pre_dist_attr), "'pre_dist_attr' cannot be None." - assert isinstance(dist_param_dict, dict), \ - "The type of 'dist_param_dict' should be 'dict', but got {}.".format( - str(type(dist_param_dict))) + assert isinstance( + dist_param_dict, dict + ), "The type of 'dist_param_dict' should be 'dict', but got {}.".format( + str(type(dist_param_dict)) + ) for name, value in dist_param_dict.items(): if not isinstance(name, str): - raise TypeError("The key of 'dist_param_dict' is parameter's name, " - "and its type should be 'str', but got {}.".format( - str(type(name)))) + raise TypeError( + "The key of 'dist_param_dict' is parameter's name, " + "and its type should be 'str', but got {}.".format( + str(type(name)) + ) + ) if not isinstance(value, list) or not all( - isinstance(v, np.ndarray) for v in value): + isinstance(v, np.ndarray) for v in value + ): raise TypeError( "The value of 'dist_param_dict' is parameter's value of all ranks, " - "and its type should be 'list(numpy.ndarray)'.") + "and its type should be 'list(numpy.ndarray)'." + ) if cur_dist_attr is None: return {} @@ -822,7 +889,8 @@ def merge_and_slice_parameter(dist_param_dict, pre_dist_attr, cur_dist_attr): cur_dims_mapping = cur_attr["dims_mapping"] if len(set(pre_dims_mapping)) > 1 or -1 not in pre_dims_mapping: complete_param = _merge_parameter_with_dist_attr( - pre_param, pre_attr) + pre_param, pre_attr + ) dist_param_dict[var_name] = complete_param else: complete_param = pre_param[0] @@ -830,7 +898,8 @@ def merge_and_slice_parameter(dist_param_dict, pre_dist_attr, cur_dist_attr): if len(set(cur_dims_mapping)) > 1 or -1 not in cur_dims_mapping: sliced_param = _slice_parameter_with_dist_attr( - complete_param, cur_attr) + complete_param, cur_attr + ) dist_param_dict[var_name] = sliced_param for var_name in pre_dist_attr: @@ -841,67 +910,81 @@ def merge_and_slice_parameter(dist_param_dict, pre_dist_attr, cur_dist_attr): if param_not_in_pre: warnings.warn( "Parameters '{}' are not found in last training process.".format( - str(param_not_in_pre))) + str(param_not_in_pre) + ) + ) if param_not_in_cur: warnings.warn( "Parameters '{}' are not found in current training process.".format( - str(param_not_in_cur))) + str(param_not_in_cur) + ) + ) return dist_param_dict def _merge_parameter_with_dist_attr(param_list, dist_attr): - """ Merge parameter with distributed attribute """ + """Merge parameter with distributed attribute""" from .reshard import Resharder dims_mapping = dist_attr["dims_mapping"] process_shape = dist_attr["process_shape"] process_group = dist_attr["process_group"] # get the complete shape of the parameter - complete_shape = Resharder.compute_complete_shape(param_list[0].shape, - process_shape, - dims_mapping) + complete_shape = Resharder.compute_complete_shape( + param_list[0].shape, process_shape, dims_mapping + ) # merge the parameter with dist_attr partition_param_list = [] merged_partiton = [] for process in process_group: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, process_shape, process_group) + process, complete_shape, dims_mapping, process_shape, process_group + ) index = process_group.index(process) if partition_index not in merged_partiton: merged_partiton.append(partition_index) - _merge_parameter(partition_param_list, param_list[index], - partition_index, complete_shape) + _merge_parameter( + partition_param_list, + param_list[index], + partition_index, + complete_shape, + ) - assert len(partition_param_list) == 1 or not partition_param_list, \ - "Fail to merge parameter" + assert ( + len(partition_param_list) == 1 or not partition_param_list + ), "Fail to merge parameter" complete_param = partition_param_list[0][0] return complete_param def _slice_parameter_with_dist_attr(param, dist_attr): - """ Slice parameter with distributed attribute """ - param = np.array(param) if isinstance(param, - paddle.fluid.LoDTensor) else param + """Slice parameter with distributed attribute""" + param = ( + np.array(param) if isinstance(param, paddle.fluid.LoDTensor) else param + ) dims_mapping = dist_attr["dims_mapping"] process_shape = dist_attr["process_shape"] process_group = dist_attr["process_group"] # slice the parameter with dist_attr - partition_index_list = _get_split_indices(param.shape, dims_mapping, - process_shape, process_group) - sliced_param_list = _slice_parameter(param, partition_index_list, - len(partition_index_list)) + partition_index_list = _get_split_indices( + param.shape, dims_mapping, process_shape, process_group + ) + sliced_param_list = _slice_parameter( + param, partition_index_list, len(partition_index_list) + ) # get the current parameter's index in sliced_param_list rank_id = paddle.distributed.get_rank() - sliced_param_index = _get_sliced_param_index(rank_id, param.shape, - dims_mapping, process_shape, - process_group) + sliced_param_index = _get_sliced_param_index( + rank_id, param.shape, dims_mapping, process_shape, process_group + ) sliced_param = sliced_param_list[sliced_param_index] return sliced_param -def _merge_parameter(partition_param_list, param, partition_index, - complete_shape): +def _merge_parameter( + partition_param_list, param, partition_index, complete_shape +): """ Merge partitial parameters to a complete one. @@ -935,19 +1018,30 @@ def _merge_parameter(partition_param_list, param, partition_index, else: i = 0 while i < len(partition_param_list): - concat_axis, first_order, new_partition = Resharder.compute_concat_info( - partition_param_list[i][1], partition_index) + ( + concat_axis, + first_order, + new_partition, + ) = Resharder.compute_concat_info( + partition_param_list[i][1], partition_index + ) if concat_axis != -1: if first_order == 0: new_param = np.concatenate( - (partition_param_list[i][0], param), axis=concat_axis) + (partition_param_list[i][0], param), axis=concat_axis + ) else: new_param = np.concatenate( - (param, partition_param_list[i][0]), axis=concat_axis) + (param, partition_param_list[i][0]), axis=concat_axis + ) partition_param_list.pop(i) - _merge_parameter(partition_param_list, new_param, new_partition, - complete_shape) + _merge_parameter( + partition_param_list, + new_param, + new_partition, + complete_shape, + ) break i += 1 @@ -975,19 +1069,21 @@ def _slice_parameter(complete_param, partition_index_list, length): """ sliced_param_list = [] axis = len(complete_param.shape) - length - sliced_param = np.split(complete_param, - partition_index_list[axis], - axis=axis) + sliced_param = np.split( + complete_param, partition_index_list[axis], axis=axis + ) if length == 1: return sliced_param for param in sliced_param: sliced_param_list.extend( - _slice_parameter(param, partition_index_list, length - 1)) + _slice_parameter(param, partition_index_list, length - 1) + ) return sliced_param_list -def _get_sliced_param_index(rank, complete_shape, dims_mapping, process_shape, - process_group): +def _get_sliced_param_index( + rank, complete_shape, dims_mapping, process_shape, process_group +): """ Get sliced_param's index of current rank in all sliced parameters list. @@ -1015,10 +1111,9 @@ def _get_sliced_param_index(rank, complete_shape, dims_mapping, process_shape, """ from .reshard import Resharder - partition_index = Resharder.compute_partition_index(rank, complete_shape, - dims_mapping, - process_shape, - process_group) + partition_index = Resharder.compute_partition_index( + rank, complete_shape, dims_mapping, process_shape, process_group + ) sliced_param_index = 0 for i, shape in enumerate(complete_shape): if dims_mapping[i] == -1: @@ -1033,8 +1128,9 @@ def _get_sliced_param_index(rank, complete_shape, dims_mapping, process_shape, return sliced_param_index -def _get_split_indices(complete_shape, dims_mapping, process_shape, - process_group): +def _get_split_indices( + complete_shape, dims_mapping, process_shape, process_group +): """ Get split indices of every dimension. @@ -1059,15 +1155,20 @@ def _get_split_indices(complete_shape, dims_mapping, process_shape, split_indices_list = [] for process in process_group: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, process_shape, process_group) + process, complete_shape, dims_mapping, process_shape, process_group + ) if split_indices_list: for dim in range(len(partition_index)): split_indices_list[dim].extend(partition_index[dim]) else: split_indices_list = partition_index split_indices_list = list( - map(lambda x, y: list(set(x) - set([y]) - set([0])), split_indices_list, - complete_shape)) + map( + lambda x, y: list(set(x) - set([y]) - set([0])), + split_indices_list, + complete_shape, + ) + ) split_indices_list = [sorted(x) for x in split_indices_list] return split_indices_list @@ -1086,8 +1187,10 @@ def set_grad_var_shape(program, dist_context): if int(op.attr('op_role')) != int(OpRole.Backward): continue - if int(block.ops[idx-1].attr('op_role')) == int(OpRole.Forward) or \ - int(block.ops[idx-1].attr('op_role')) == 257: + if ( + int(block.ops[idx - 1].attr('op_role')) == int(OpRole.Forward) + or int(block.ops[idx - 1].attr('op_role')) == 257 + ): appended_grad_times += 1 if op.type in ["check_finite_and_unscale", "update_loss_scaling"]: @@ -1105,61 +1208,102 @@ def set_grad_var_shape(program, dist_context): continue if var_name in grad_var_to_var[appended_grad_times]: forward_var_name = grad_var_to_var[appended_grad_times][ - var_name] + var_name + ] else: - forward_var_name = var_name[:var_name.find("@GRAD")] + forward_var_name = var_name[: var_name.find("@GRAD")] if op.type in [ - "c_allreduce_sum", "c_identity", "scale", "cast", - "fill_any_like" + "c_allreduce_sum", + "c_identity", + "scale", + "cast", + "fill_any_like", ]: forward_var_name = op.input_arg_names[0] - elif op.type == "matmul_v2_grad" or op.type == "matmul_grad" or op.type == "mul_grad": + elif ( + op.type == "matmul_v2_grad" + or op.type == "matmul_grad" + or op.type == "mul_grad" + ): forward_var_name = None for output_name in op.output_names: if var_name in op.output(output_name): assert "@GRAD" in output_name - input_name = output_name[:output_name.find("@GRAD")] + input_name = output_name[: output_name.find("@GRAD")] assert len(op.input(input_name)) == 1 forward_var_name = op.input(input_name)[0] assert forward_var_name is not None need_set_shape_list = [ - "reshape2_grad", "softmax_with_cross_entropy_grad", - "transpose2_grad", "softmax_grad", "cross_entropy_grad2", - "dropout_grad", "tanh_grad", "slice", "assign", - "matmul_v2_triple_grad", "elementwise_add_triple_grad", - "fill_constant", "sqrt_grad", + "reshape2_grad", + "softmax_with_cross_entropy_grad", + "transpose2_grad", + "softmax_grad", + "cross_entropy_grad2", + "dropout_grad", + "tanh_grad", + "slice", + "assign", + "matmul_v2_triple_grad", + "elementwise_add_triple_grad", + "fill_constant", + "sqrt_grad", "fused_softmax_mask_upper_triangle_grad", - "flatten_contiguous_range_grad", "relu_grad" + "flatten_contiguous_range_grad", + "relu_grad", ] forward_list = [ - "reshape2", "softmax_with_cross_entropy", "transpose2", - "softmax", "cross_entropy2", "dropout", "tanh", - ["slice_grad", "c_allgather"], "assign", "matmul_v2_grad_grad", - "elementwise_add_grad_grad", "shape", "sqrt", - "fused_softmax_mask_upper_triangle", "flatten_contiguous_range", - "relu" + "reshape2", + "softmax_with_cross_entropy", + "transpose2", + "softmax", + "cross_entropy2", + "dropout", + "tanh", + ["slice_grad", "c_allgather"], + "assign", + "matmul_v2_grad_grad", + "elementwise_add_grad_grad", + "shape", + "sqrt", + "fused_softmax_mask_upper_triangle", + "flatten_contiguous_range", + "relu", ] if op.type in need_set_shape_list: for forward_op in block.ops: idx = need_set_shape_list.index(op.type) forward_op_name = forward_list[idx] - if forward_op.type in forward_op_name and forward_var_name in forward_op.input_arg_names: - op_dist_attr = dist_context.get_op_dist_attr_for_program( - forward_op) + if ( + forward_op.type in forward_op_name + and forward_var_name in forward_op.input_arg_names + ): + op_dist_attr = ( + dist_context.get_op_dist_attr_for_program( + forward_op + ) + ) break forward_input_dist_attr = op_dist_attr.get_input_dist_attr( - forward_var_name) - assert forward_input_dist_attr is not None, f"{forward_var_name, str(op)}" + forward_var_name + ) + assert ( + forward_input_dist_attr is not None + ), f"{forward_var_name, str(op)}" forward_var = vars[forward_var_name] - forward_var_dist_attr = dist_context.get_tensor_dist_attr_for_program( - forward_var) + forward_var_dist_attr = ( + dist_context.get_tensor_dist_attr_for_program(forward_var) + ) assert forward_var_dist_attr is not None grad_var = vars[var_name] - ref_shape = infer_shape(block, forward_var, forward_var_dist_attr, - forward_input_dist_attr) + ref_shape = infer_shape( + block, + forward_var, + forward_var_dist_attr, + forward_input_dist_attr, + ) if list(grad_var.shape) != ref_shape: grad_var.desc.set_shape(ref_shape) @@ -1171,28 +1315,33 @@ OpRole = core.op_proto_and_checker_maker.OpRole def is_forward_op(op): op_role = int(op.attr('op_role')) - return OP_ROLE_KEY in op.attr_names and (op_role == int(OpRole.Forward) - or op_role == int(OpRole.Loss)) + return OP_ROLE_KEY in op.attr_names and ( + op_role == int(OpRole.Forward) or op_role == int(OpRole.Loss) + ) def is_backward_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Backward) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Backward) def is_optimize_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Optimize) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Optimize) def is_lr_sched_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Optimize.LRSched) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Optimize.LRSched) def is_loss_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) == (int(OpRole.Forward) | int(OpRole.Loss)) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) == (int(OpRole.Forward) | int(OpRole.Loss)) def is_loss_grad_op(op): @@ -1203,8 +1352,9 @@ def is_loss_grad_op(op): def is_gradient_clip_op(op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip") def is_prim_op(op): @@ -1215,8 +1365,9 @@ def get_loss_op(block): loss_ops = [] for op in block.ops: if is_loss_op(op): - assert len(op.desc.output_arg_names() - ) == 1, "loss op should only output loss var" + assert ( + len(op.desc.output_arg_names()) == 1 + ), "loss op should only output loss var" loss_ops.append(op) assert len(loss_ops) == 1, "num of loss op is not equal to one" @@ -1236,7 +1387,8 @@ def set_var_dist_attr(dist_context, var, dims_mapping, process_mesh, **kwargs): def naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, process_mesh, ref_mapping, ctx): + new_op, process_mesh, ref_mapping, ctx +): assert process_mesh is not None assert ref_mapping is not None @@ -1270,9 +1422,11 @@ def update_op_dims_mapping_by_default_dist_impl(dist_op): dims_mapping = op_dist_attr.get_input_dims_mapping(arg_name) if len(dims_mapping) > 1: for idx, mapping in enumerate(dims_mapping[1:]): - assert mapping == -1, \ - "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part."\ - .format(op_desc.type(), idx, mapping) + assert ( + mapping == -1 + ), "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part.".format( + op_desc.type(), idx, mapping + ) batch_dim_mappings.append(dims_mapping[0]) for arg_name in op_desc.output_arg_names(): serial_tensor = dist_op.get_serial_output(arg_name) @@ -1282,23 +1436,31 @@ def update_op_dims_mapping_by_default_dist_impl(dist_op): if arg_name not in xshape_arg_names: if len(dims_mapping) > 1: for idx, mapping in enumerate(dims_mapping[1:]): - assert mapping == -1, \ - "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part."\ - .format(op_desc.type(), idx, mapping) + assert ( + mapping == -1 + ), "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part.".format( + op_desc.type(), idx, mapping + ) batch_dim_mappings.append(dims_mapping[0]) else: - assert dims_mapping[0] == -1, \ - "{} only the batch dimension (1-dim) of XShape can be sharded, but the dimension 0 is sharded by {} part."\ - .format(op_desc.type(), mapping) + assert ( + dims_mapping[0] == -1 + ), "{} only the batch dimension (1-dim) of XShape can be sharded, but the dimension 0 is sharded by {} part.".format( + op_desc.type(), mapping + ) if len(dims_mapping) > 2: for idx, mapping in enumerate(dims_mapping[2:]): - assert mapping == -1, \ - "{} only the batch dimension (1-dim) of XShape can be sharded, but the dimension {} is sharded by {} part."\ - .format(op_desc.type(), idx, mapping) + assert ( + mapping == -1 + ), "{} only the batch dimension (1-dim) of XShape can be sharded, but the dimension {} is sharded by {} part.".format( + op_desc.type(), idx, mapping + ) batch_dim_mappings.append(dims_mapping[1]) compatible_dim_mapping = compute_compatible_dim_mapping(batch_dim_mappings) - assert compatible_dim_mapping is not None, "There is no compatible dim mapping." + assert ( + compatible_dim_mapping is not None + ), "There is no compatible dim mapping." for arg_name in op_desc.input_arg_names(): serial_tensor = dist_op.get_serial_input(arg_name) if serial_tensor.is_parameter: @@ -1344,8 +1506,9 @@ def update_op_dims_mapping_by_elementwise_like_dist_impl(dist_op): if input_dims_mapping_lens[arg_name] < max_dims_mapping_len: new_dims_mapping = [-1 for _ in range(max_dims_mapping_len)] for i in range(input_dims_mapping_lens[arg_name]): - new_idx = (max_dims_mapping_len - - input_dims_mapping_lens[arg_name]) + i + new_idx = ( + max_dims_mapping_len - input_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[new_idx] = input_dims_mapping_dict[arg_name][i] dims_mapping_list.append(new_dims_mapping) else: @@ -1357,7 +1520,9 @@ def update_op_dims_mapping_by_elementwise_like_dist_impl(dist_op): dims_mapping_list.append(dims_mapping) compatible_dims_mapping = compute_compatible_dims_mapping(dims_mapping_list) - assert compatible_dims_mapping is not None, "There is no compatible dim mapping." + assert ( + compatible_dims_mapping is not None + ), "There is no compatible dim mapping." for arg_name in input_arg_names: if input_dims_mapping_lens[arg_name] < max_dims_mapping_len: @@ -1365,55 +1530,64 @@ def update_op_dims_mapping_by_elementwise_like_dist_impl(dist_op): -1 for _ in range(input_dims_mapping_lens[arg_name]) ] for i in range(input_dims_mapping_lens[arg_name]): - new_idx = (max_dims_mapping_len - - input_dims_mapping_lens[arg_name]) + i + new_idx = ( + max_dims_mapping_len - input_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[i] = compatible_dims_mapping[new_idx] if new_dims_mapping != input_dims_mapping_dict[arg_name]: op_dist_attr.set_input_dims_mapping(arg_name, new_dims_mapping) changed = True else: if compatible_dims_mapping != input_dims_mapping_dict[arg_name]: - op_dist_attr.set_input_dims_mapping(arg_name, - compatible_dims_mapping) + op_dist_attr.set_input_dims_mapping( + arg_name, compatible_dims_mapping + ) changed = True for arg_name in output_arg_names: dims_mapping = op_dist_attr.get_output_dims_mapping(arg_name) if compatible_dims_mapping != dims_mapping: - op_dist_attr.set_output_dims_mapping(arg_name, - compatible_dims_mapping) + op_dist_attr.set_output_dims_mapping( + arg_name, compatible_dims_mapping + ) changed = True return changed -def get_all_distributed_main_program(serial_program_info, dist_context, - parallelizer): +def get_all_distributed_main_program( + serial_program_info, dist_context, parallelizer +): "Get all distributed main programs by dist_context." from .dist_context import DistributedOperatorContext + cluster = serial_program_info.cluster copied_parallelizer = copy.deepcopy(parallelizer) all_dist_main_program = [] - ranks = paddle.distributed.get_world_size() if cluster is None else len( - cluster.get_all_devices("GPU")) + ranks = ( + paddle.distributed.get_world_size() + if cluster is None + else len(cluster.get_all_devices("GPU")) + ) for rank_id in range(ranks): used_dist_context = copy.deepcopy(dist_context) used_dist_context._dist_op_context = DistributedOperatorContext() - _, _, dist_startup_program, dist_main_program, _ = copied_parallelizer._get_dist_program( - rank_id, used_dist_context) + ( + _, + _, + dist_startup_program, + dist_main_program, + _, + ) = copied_parallelizer._get_dist_program(rank_id, used_dist_context) all_dist_main_program.append(dist_main_program) return all_dist_main_program class SerialProgramInfo: - - def __init__(self, - train_program, - satrtup_program, - loss, - optimizer, - cluster=None): + def __init__( + self, train_program, satrtup_program, loss, optimizer, cluster=None + ): self._train_program = train_program self._startup_program = satrtup_program self._loss = loss @@ -1442,7 +1616,6 @@ class SerialProgramInfo: def get_standalone_cost_data(distributed_programs): - def _compute_runtime(op_cost, op, vars): runtime = 0 try: @@ -1455,32 +1628,47 @@ def get_standalone_cost_data(distributed_programs): parsed_info = op_config.split("\n") variable = "(Variable)" for info in parsed_info: - variable = "(Variable)" if "(Variable)" in info else "(list" + variable = ( + "(Variable)" if "(Variable)" in info else "(list" + ) if variable in info: - arg_name_lower = info[:info.find(variable) - 1] + arg_name_lower = info[: info.find(variable) - 1] shape_left_boundary = info.find("[") shape_right_boundary = info.find("]") - assert shape_left_boundary > 0 and shape_right_boundary > 0 and shape_right_boundary > shape_left_boundary, "Get shape failed." - shape = info[shape_left_boundary + - 1:shape_right_boundary].split(",") + assert ( + shape_left_boundary > 0 + and shape_right_boundary > 0 + and shape_right_boundary > shape_left_boundary + ), "Get shape failed." + shape = info[ + shape_left_boundary + 1 : shape_right_boundary + ].split(",") shape = list(map(lambda x: int(x.strip()), shape)) dtype_factor = 1 total_static_input_size += reduce(lambda x, y: x * y, shape) if op.type == "c_embedding": - arg_name_lower = "w" if arg_name_lower == "weight" else "ids" + arg_name_lower = ( + "w" if arg_name_lower == "weight" else "ids" + ) for arg_name in op.input_names: if arg_name.lower() == arg_name_lower: for var_name in op.input(arg_name): var = vars[var_name] total_actual_input_size += reduce( - lambda x, y: x * y, var.shape) + lambda x, y: x * y, var.shape + ) break - assert total_static_input_size > 0 and total_actual_input_size > 0, "Get input size failed." + assert ( + total_static_input_size > 0 and total_actual_input_size > 0 + ), "Get input size failed." - actual_runtime = total_actual_input_size / total_static_input_size * runtime + actual_runtime = ( + total_actual_input_size / total_static_input_size * runtime + ) return actual_runtime import paddle.cost_model as cm + cost_model = cm.CostModel() cost_model.static_cost_data() DEFAULT_MULTIPLE = 2 @@ -1491,13 +1679,16 @@ def get_standalone_cost_data(distributed_programs): "reshape2": "reshape", "unsqueeze2": "unsqueeze", "reduce_sum": "sum", - "elementwise_div": "divide" + "elementwise_div": "divide", } standalone_cost_data = [] # skip ops not_enum_ops = [ - "create_py_reader", "create_double_buffer_reader", "read", "assign" + "create_py_reader", + "create_double_buffer_reader", + "read", + "assign", ] for distributed_program in distributed_programs: cost_data = {} @@ -1507,26 +1698,33 @@ def get_standalone_cost_data(distributed_programs): if op.type in not_enum_ops: cost_data[op.desc.id()] = runtime continue - dtype = str(vars[op.input_arg_names[0]].dtype - ) if op.input_arg_names else "float32" + dtype = ( + str(vars[op.input_arg_names[0]].dtype) + if op.input_arg_names + else "float32" + ) if int(op.attr('op_role')) == int(OpRole.Backward): if "_grad" in op.type: forward_op_name = op.type[:-5] if forward_op_name in OP_NAME_MAPPING.keys(): forward_op_name = OP_NAME_MAPPING[forward_op_name] - op_cost = cost_model.get_static_op_time(forward_op_name, - forward=False, - dtype=dtype) + op_cost = cost_model.get_static_op_time( + forward_op_name, forward=False, dtype=dtype + ) if op_cost: runtime = _compute_runtime(op_cost, op, vars) else: - op_cost = cost_model.get_static_op_time(forward_op_name, - dtype=dtype) + op_cost = cost_model.get_static_op_time( + forward_op_name, dtype=dtype + ) if op_cost: runtime = 2 * _compute_runtime(op_cost, op, vars) elif int(op.attr('op_role')) == int(OpRole.Forward): - op_name = OP_NAME_MAPPING[ - op.type] if op.type in OP_NAME_MAPPING.keys() else op.type + op_name = ( + OP_NAME_MAPPING[op.type] + if op.type in OP_NAME_MAPPING.keys() + else op.type + ) op_cost = cost_model.get_static_op_time(op_name) if op_cost: runtime = _compute_runtime(op_cost, op, vars) @@ -1565,7 +1763,8 @@ def to_list(value): def debug_program(program, path, name): filename = os.path.join( - path, name + '_program' + ".%d" % (paddle.distributed.get_rank())) + path, name + '_program' + ".%d" % (paddle.distributed.get_rank()) + ) with open(filename, 'w') as f: f.write(str(program)) @@ -1599,9 +1798,11 @@ def get_lr(optimizer): return optimizer._learning_rate() else: raise TypeError( - "'optimizer' must be object of class `paddle.optimizer.Optimizer`" \ - " or `paddle.fluid.optimizer.Optimizer`, but got {}.".format(type(optimizer)) + "'optimizer' must be object of class `paddle.optimizer.Optimizer`" + " or `paddle.fluid.optimizer.Optimizer`, but got {}.".format( + type(optimizer) ) + ) def initialize_pg_in_full_mode(all_process_groups, cur_rank): @@ -1631,21 +1832,28 @@ def initialize_pg_in_full_mode(all_process_groups, cur_rank): if is_send: recv_rank = process_group.ranks[1] recv_rank_ip, recv_rank_port = genv.trainer_endpoints[ - recv_rank].split(":") + recv_rank + ].split(":") connect_port = int(recv_rank_port) + magic_num - client_socket = socket.socket(socket.AF_INET, - socket.SOCK_STREAM) + client_socket = socket.socket( + socket.AF_INET, socket.SOCK_STREAM + ) client_socket.connect((recv_rank_ip, connect_port)) client_socket.send(str(cur_rank).encode('utf-8')) rank = client_socket.recv(buff_size).decode('utf-8') rank = int(rank) if rank != recv_rank: raise ValueError( - "Please check comm pair, the recv rank should be {} but got {}." - .format(recv_rank, rank)) + "Please check comm pair, the recv rank should be {} but got {}.".format( + recv_rank, rank + ) + ) else: - print("It is able to instantiate {} as sender now.".format( - process_group.ranks)) + print( + "It is able to instantiate {} as sender now.".format( + process_group.ranks + ) + ) client_socket.close() else: send_rank = process_group.ranks[0] @@ -1657,10 +1865,14 @@ def initialize_pg_in_full_mode(all_process_groups, cur_rank): has_recv_by_socket.append(rank) else: client_sockets[send_rank].send( - str(cur_rank).encode("utf-8")) + str(cur_rank).encode("utf-8") + ) client_sockets[send_rank].close() - print("It is able to instantiate {} as recver now.". - format(process_group.ranks)) + print( + "It is able to instantiate {} as recver now.".format( + process_group.ranks + ) + ) break process_group.instantiate() server_socket.close() diff --git a/python/paddle/distributed/cloud_utils.py b/python/paddle/distributed/cloud_utils.py index b186ff64baf..2e1d85205a5 100644 --- a/python/paddle/distributed/cloud_utils.py +++ b/python/paddle/distributed/cloud_utils.py @@ -13,7 +13,11 @@ # limitations under the License. import os -from paddle.distributed.utils.launch_utils import get_cluster, get_gpus, get_cluster_from_args +from paddle.distributed.utils.launch_utils import ( + get_cluster, + get_gpus, + get_cluster_from_args, +) from paddle.distributed.utils.launch_utils import logger __all__ = [] @@ -23,7 +27,7 @@ def get_cloud_cluster(args_node_ips, args_node_ip, args_port, selected_devices): """ args_node_ips:string, args_node_ip:string, args_port: int, selected_devices:list """ - #you can automatically get ip info while using paddlecloud multi nodes mode. + # you can automatically get ip info while using paddlecloud multi nodes mode. node_ips = os.getenv("PADDLE_TRAINERS") assert node_ips is not None, "PADDLE_TRAINERS should not be None" @@ -41,16 +45,23 @@ def get_cloud_cluster(args_node_ips, args_node_ip, args_port, selected_devices): node_rank = int(node_rank) if node_ip != "127.0.0.1" and node_ip != args_node_ip: - logger.warning("Please NOTE: When using paddlecloud, node_ip is \ + logger.warning( + "Please NOTE: When using paddlecloud, node_ip is \ automatically got from POD_IP. Your input node_ip: {} doesn't equals to \ -node_ip: {} from paddlecloud environment.".format(args_node_ip, node_ip)) +node_ip: {} from paddlecloud environment.".format( + args_node_ip, node_ip + ) + ) if args_node_ips != "127.0.0.1" and args_node_ips != ",".join(node_ips): logger.warning( "Please NOTE: When using paddlecloud, cluster_node_ips is \ automatically got from PADDLE_TRAINERS(multi nodes) or POD_IP(single node).\ Your input cluster_node_ips: {} doesn't equals to IPs: {} from \ -paddlecloud environment.".format(args_node_ips, node_ips)) +paddlecloud environment.".format( + args_node_ips, node_ips + ) + ) # DISTRIBUTED_TRAINER_ENDPOINTS: new environment since paddlecloud 1.8.4 # e.g: DISTRIBUTED_TRAINER_ENDPOINTS="ip1:port1,ip1:port2,ip1:port3,ip1:port4,ip2:port5,ip2:port6,ip2:port7,ip2:port8" @@ -61,10 +72,13 @@ paddlecloud environment.".format(args_node_ips, node_ips)) try: paddle_port = int(os.getenv("PADDLE_PORT", "")) - if paddle_ports_num >= len( - selected_devices) and paddle_port != args_port: + if ( + paddle_ports_num >= len(selected_devices) + and paddle_port != args_port + ): logger.warning( - "Use Cloud specified port:{}.".format(paddle_port)) + "Use Cloud specified port:{}.".format(paddle_port) + ) started_port = paddle_port except Exception as e: @@ -84,15 +98,21 @@ paddlecloud environment.".format(args_node_ips, node_ips)) assert num_nodes * paddle_ports_num == len(trainer_endpoints_ori) for i in range(num_nodes): trainer_endpoints.append( - trainer_endpoints_ori[i * paddle_ports_num:(i + 1) * - paddle_ports_num]) + trainer_endpoints_ori[ + i * paddle_ports_num : (i + 1) * paddle_ports_num + ] + ) - logger.debug("parsed from args: node_ips:{} \ + logger.debug( + "parsed from args: node_ips:{} \ node_ip:{} node_rank:{} trainer_endpoints:{}".format( - node_ips, node_ip, node_rank, trainer_endpoints)) + node_ips, node_ip, node_rank, trainer_endpoints + ) + ) - cluster, pod = get_cluster(node_ips, node_ip, trainer_endpoints, - selected_devices) + cluster, pod = get_cluster( + node_ips, node_ip, trainer_endpoints, selected_devices + ) return cluster, cluster.pods[node_rank] @@ -104,15 +124,22 @@ def get_cluster_and_pod(args): # parse arguments, used for cloud-single-machine and local selected_devices = get_gpus(args.selected_devices) trainers_num = _get_trainers_num() - logger.debug("parsed from args trainerss_num:{} selected_devices:{}".format( - trainers_num, selected_devices)) + logger.debug( + "parsed from args trainerss_num:{} selected_devices:{}".format( + trainers_num, selected_devices + ) + ) cluster = None pod = None if args.use_paddlecloud and trainers_num != 1: - cluster, pod = get_cloud_cluster(args.cluster_node_ips, args.node_ip, - args.started_port, selected_devices) + cluster, pod = get_cloud_cluster( + args.cluster_node_ips, + args.node_ip, + args.started_port, + selected_devices, + ) logger.info("get cluster from cloud:{}".format(cluster)) else: cluster, pod = get_cluster_from_args(args, selected_devices) diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index fa1b3e00d47..77258f7036c 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -92,8 +92,9 @@ def _get_group_map(): global _group_map if _global_env_gid not in _group_map: genv = _get_global_env() - _group_map[_global_env_gid] = Group(genv.rank, 0, - list(range(genv.world_size))) + _group_map[_global_env_gid] = Group( + genv.rank, 0, list(range(genv.world_size)) + ) return _group_map @@ -108,8 +109,10 @@ def _get_group_map_by_name(): def _get_default_group(): global _group_map_by_name - assert is_initialized(), ("Call paddle.distributed.init_parallel_env first " - "to initialize the distributed environment.") + assert is_initialized(), ( + "Call paddle.distributed.init_parallel_env first " + "to initialize the distributed environment." + ) return _get_group_map_by_name()[_default_group_name] @@ -165,21 +168,23 @@ def get_group(id=0): return gm[id] if id in gm else None -def _new_process_group_impl(backend, - store, - rank, - world_size, - group_name, - pg_options, - group_id=0, - src_rank=None, - dst_rank=None): +def _new_process_group_impl( + backend, + store, + rank, + world_size, + group_name, + pg_options, + group_id=0, + src_rank=None, + dst_rank=None, +): pg = None genv = _get_global_env() if backend != 'heter': assert src_rank is None and dst_rank is None, ( - "src_rank and dst_rank " - "can only be set for heter backend.") + "src_rank and dst_rank " "can only be set for heter backend." + ) assert backend in _valid_backend_list, "Unsupported backend: %s." % backend if backend == "gloo": place = core.CPUPlace() @@ -208,24 +213,27 @@ def _new_process_group_impl(backend, switch_ep = os.getenv("CLUSTER_SWITCH", None) assert switch_ep, "please set the CLUSTER_SWITCH variable." cluster_size_cumsum = np.cumsum(cluster_size) - cluster_offset = 0 if cluster_id == 0 else cluster_size_cumsum[ - cluster_id - 1] + cluster_offset = ( + 0 if cluster_id == 0 else cluster_size_cumsum[cluster_id - 1] + ) global_rank = cluster_offset + rank global_world_size = cluster_size_cumsum[-1] global_rank, global_world_size = _get_global_config(backend, rank) - pg = core.ProcessGroupHeter(store, - rank=global_rank, - world_size=global_world_size, - place=place, - gid=group_id, - local_rank=rank, - local_size=world_size, - gloo_rank=cluster_id, - gloo_size=len(cluster_size), - with_switch=True, - switch_endpoint=switch_ep, - src_rank=src_rank, - dst_rank=dst_rank) + pg = core.ProcessGroupHeter( + store, + rank=global_rank, + world_size=global_world_size, + place=place, + gid=group_id, + local_rank=rank, + local_size=world_size, + gloo_rank=cluster_id, + gloo_size=len(cluster_size), + with_switch=True, + switch_endpoint=switch_ep, + src_rank=src_rank, + dst_rank=dst_rank, + ) return pg @@ -271,10 +279,12 @@ def barrier(group=None): if not isinstance(ring_id, int): raise ValueError("The type of 'group' for barrier must be int.") helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [temp]}, - outputs={'Out': [temp]}, - attrs={'ring_id': ring_id}) + helper.append_op( + type=op_type, + inputs={'X': [temp]}, + outputs={'Out': [temp]}, + attrs={'ring_id': ring_id}, + ) # _custom_gid provides a way for users to @@ -296,7 +306,7 @@ def _barrier_by_tcp_store(group_name, store, timeout): return barrier_prefix = "Barrier/" + group_name + "/" - is_master = (global_rank == 0) + is_master = global_rank == 0 def _check_keys_ready(wait_keys): start_time = time.time() @@ -309,9 +319,12 @@ def _barrier_by_tcp_store(group_name, store, timeout): "Keys {} are not ready sinck rank {} is waiting them." "Two reason may cause this error:\n 1. The create process group api should be called by all ranks.\n" " 2. Try to increase the waiting time.\n".format( - group_name, wait_keys, global_rank)) + group_name, wait_keys, global_rank + ) + ) wait_keys = list( - filter(lambda key: int(store.get(key)) != 1, wait_keys)) + filter(lambda key: int(store.get(key)) != 1, wait_keys) + ) # all the workers set their exiting key and exit # the master will wait for all workers' exiting key, ensure to exit in the end @@ -363,22 +376,25 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): ranks = global_ranks assert len(ranks) <= len(global_ranks), ( "Size of new group must be less than or " - "equal to that of the default global group.") + "equal to that of the default global group." + ) size = len(ranks) ranks = sorted(ranks) if backend == 'heter' or (size > 1 and global_rank in ranks): rank = 0 if backend == 'heter' else ranks.index(global_rank) src_rank = ranks[0] if backend == 'heter' else None dst_rank = ranks[1] if backend == 'heter' else None - pg = _new_process_group_impl(backend, - _default_store, - rank, - size, - group_name, - pg_options=None, - group_id=gid, - src_rank=src_rank, - dst_rank=dst_rank) + pg = _new_process_group_impl( + backend, + _default_store, + rank, + size, + group_name, + pg_options=None, + group_id=gid, + src_rank=src_rank, + dst_rank=dst_rank, + ) else: rank = -1 pg = None @@ -386,7 +402,7 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): _group_map_by_name[group_name] = group _group_map[gid] = group _group_map_backend[group] = backend - #TODO: The method below is a new method for group management, will replace the previous + # TODO: The method below is a new method for group management, will replace the previous # three in the future. _add_new_group(group) @@ -402,7 +418,7 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): if not backend: backend = 'nccl' - assert backend == 'nccl', ("backend other than nccl is not supported yet") + assert backend == 'nccl', "backend other than nccl is not supported yet" genv = _get_global_env() global_rank = genv.rank @@ -431,30 +447,36 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): if core.is_compiled_with_cuda(): place = core.CUDAPlace(genv.device_id) - core.NCCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.NCCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) elif core.is_compiled_with_npu(): place = core.NPUPlace(genv.device_id) - core.HCCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.HCCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) elif core.is_compiled_with_mlu(): place = core.MLUPlace(genv.device_id) - core.CNCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.CNCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) elif core.is_compiled_with_xpu(): place = core.XPUPlace(genv.device_id) - core.BKCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.BKCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) else: - assert False, ("no cuda device found") + assert False, "no cuda device found" else: return gp # TODO(shenliang03): This is a temporary solution to solve the problem of # hang caused by cross-creation of new_group - tmp = paddle.to_tensor( - [1], dtype="int32") if _non_static_mode() else fill_constant( - [0], dtype="int32", value="1") + tmp = ( + paddle.to_tensor([1], dtype="int32") + if _non_static_mode() + else fill_constant([0], dtype="int32", value="1") + ) paddle.distributed.all_reduce(tmp, sync_op=True) paddle.distributed.wait(tmp) return gp @@ -586,8 +608,9 @@ def _sync_calc_stream(tensor): def _sync_comm_stream(tensor, ring_id=0): if _non_static_mode(): - return _legacy_C_ops.c_sync_comm_stream([tensor], [tensor], 'ring_id', - ring_id) + return _legacy_C_ops.c_sync_comm_stream( + [tensor], [tensor], 'ring_id', ring_id + ) op_type = 'c_sync_comm_stream' @@ -648,7 +671,7 @@ def broadcast(tensor, src, group=None, sync_op=True): if in_dygraph_mode(): group = _get_default_group() if group is None else group gsrc = group.get_group_rank(src) - assert gsrc >= 0, ("src rank out of group, need global rank") + assert gsrc >= 0, "src rank out of group, need global rank" task = group.process_group.broadcast(tensor, gsrc) if sync_op: task.wait() @@ -659,28 +682,48 @@ def broadcast(tensor, src, group=None, sync_op=True): use_calc_stream = sync_op ring_id = ring_id = 0 if group is None else group.id gsrc = src if group is None else group.get_group_rank(src) - assert gsrc >= 0, ("src rank out of group, need global rank") + assert gsrc >= 0, "src rank out of group, need global rank" if _non_static_mode(): - return _legacy_C_ops.c_broadcast(tensor, tensor, 'root', gsrc, - 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id) + return _legacy_C_ops.c_broadcast( + tensor, + tensor, + 'root', + gsrc, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + ) op_type = 'c_broadcast' - check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'bool' - ], 'broadcast') + check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'bool', + ], + 'broadcast', + ) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [tensor]}, - attrs={ - 'root': gsrc, - 'use_calc_stream': use_calc_stream, - 'ring_id': ring_id, - }) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [tensor]}, + attrs={ + 'root': gsrc, + 'use_calc_stream': use_calc_stream, + 'ring_id': ring_id, + }, + ) def reduce(tensor, dst, op=ReduceOp.SUM, group=None, sync_op=True): @@ -730,7 +773,7 @@ def reduce(tensor, dst, op=ReduceOp.SUM, group=None, sync_op=True): op_type = _get_reduce_op(op, "reduce") group = _get_default_group() if group is None else group gdst = group.get_group_rank(dst) - assert gdst >= 0, ("dst rank out of group, need global rank") + assert gdst >= 0, "dst rank out of group, need global rank" task = group.process_group.reduce(tensor, gdst, op_type) if sync_op: task.wait() @@ -741,34 +784,72 @@ def reduce(tensor, dst, op=ReduceOp.SUM, group=None, sync_op=True): use_calc_stream = sync_op ring_id = 0 if group is None else group.id gdst = dst if group is None else group.get_group_rank(dst) - assert gdst >= 0, ("dst rank out of group, need global rank") + assert gdst >= 0, "dst rank out of group, need global rank" if _non_static_mode(): if op == ReduceOp.SUM: - return _legacy_C_ops.c_reduce_sum(tensor, tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'root_id', gdst) + return _legacy_C_ops.c_reduce_sum( + tensor, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'root_id', + gdst, + ) elif op == ReduceOp.MAX: - return _legacy_C_ops.c_reduce_max(tensor, tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'root_id', gdst) + return _legacy_C_ops.c_reduce_max( + tensor, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'root_id', + gdst, + ) elif op == ReduceOp.MIN: - return _legacy_C_ops.c_reduce_min(tensor, tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'root_id', gdst) + return _legacy_C_ops.c_reduce_min( + tensor, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'root_id', + gdst, + ) elif op == ReduceOp.PROD: - return _legacy_C_ops.c_reduce_prod(tensor, tensor, - 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'root_id', gdst) + return _legacy_C_ops.c_reduce_prod( + tensor, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'root_id', + gdst, + ) else: raise ValueError("Unknown parameter: {}.".format(op)) op_type = 'c_reduce' - check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'bool' - ], 'reduce') + check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'bool', + ], + 'reduce', + ) if op == ReduceOp.SUM: op_type = 'c_reduce_sum' @@ -780,14 +861,16 @@ def reduce(tensor, dst, op=ReduceOp.SUM, group=None, sync_op=True): op_type = 'c_reduce_prod' helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - 'root_id': gdst, - }) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [tensor]}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + 'root_id': gdst, + }, + ) def all_gather(tensor_list, tensor, group=None, sync_op=True): @@ -840,8 +923,9 @@ def all_gather(tensor_list, tensor, group=None, sync_op=True): list_of_complex.append(paddle.as_complex(tensor)) return list_of_complex - is_input_complex = (tensor.dtype == paddle.complex64 - or tensor.dtype == paddle.complex128) + is_input_complex = ( + tensor.dtype == paddle.complex64 or tensor.dtype == paddle.complex128 + ) if is_input_complex: tensor = paddle.as_real(tensor) @@ -868,33 +952,68 @@ def all_gather(tensor_list, tensor, group=None, sync_op=True): nranks = _get_global_group().nranks if group is None else group.nranks if _non_static_mode(): - out = _legacy_C_ops.c_allgather(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', ring_id, - 'nranks', nranks) + out = _legacy_C_ops.c_allgather( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + ) else: op_type = 'c_allgather' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=tensor.dtype) if not isinstance(tensor_list, list): - raise ValueError("The type of 'tensor_list' for all_gather " - "should be list.") + raise ValueError( + "The type of 'tensor_list' for all_gather " "should be list." + ) for elem in tensor_list: - check_variable_and_dtype(elem, 'tensor_list', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'bool', - 'int8', 'uint8', 'complex64', 'complex128' - ], 'all_gather') - check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'bool', 'int8', - 'uint8', 'complex64', 'complex128' - ], 'all_gather') - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [out]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - 'nranks': nranks - }) + check_variable_and_dtype( + elem, + 'tensor_list', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'bool', + 'int8', + 'uint8', + 'complex64', + 'complex128', + ], + 'all_gather', + ) + check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'bool', + 'int8', + 'uint8', + 'complex64', + 'complex128', + ], + 'all_gather', + ) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [out]}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + 'nranks': nranks, + }, + ) list_of_tensor = paddle.split(out, nranks, 0) if is_input_complex: @@ -950,7 +1069,8 @@ def all_gather_object(object_list, obj, group=None): print(object_list) # [{'foo': [1, 2, 3]}, {'bar': [4, 5, 6]}] (2 GPUs) """ - assert in_dygraph_mode( + assert ( + in_dygraph_mode() ), "all_gather_object doesn't support static graph mode." tensor, len_of_tensor = _convert_object_to_tensor(obj) @@ -971,7 +1091,8 @@ def all_gather_object(object_list, obj, group=None): all_gather(tensor_list, input_tensor, group) for i, tensor in enumerate(tensor_list): object_list.append( - _convert_tensor_to_object(tensor, list_len_of_tensor[i])) + _convert_tensor_to_object(tensor, list_len_of_tensor[i]) + ) def scatter(tensor, tensor_list=None, src=0, group=None, sync_op=True): @@ -1033,7 +1154,7 @@ def scatter(tensor, tensor_list=None, src=0, group=None, sync_op=True): gsrc = src if group is None else group.get_group_rank(src) rank = _get_global_group().rank if group is None else group.rank nranks = _get_global_group().nranks if group is None else group.nranks - assert gsrc >= 0, ("src rank out of group, need global rank") + assert gsrc >= 0, "src rank out of group, need global rank" if rank != gsrc: tensor_list = [] @@ -1050,24 +1171,46 @@ def scatter(tensor, tensor_list=None, src=0, group=None, sync_op=True): use_calc_stream = sync_op if _non_static_mode(): - return _legacy_C_ops.c_scatter(temp, tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', ring_id, - 'nranks', nranks, 'root', gsrc) + return _legacy_C_ops.c_scatter( + temp, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + 'root', + gsrc, + ) op_type = 'c_scatter' - check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'bool' - ], 'scatter') + check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'bool', + ], + 'scatter', + ) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [temp]}, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'root': gsrc, - 'use_calc_stream': use_calc_stream, - 'nranks': nranks, - }) + helper.append_op( + type=op_type, + inputs={'X': [temp]}, + outputs={'Out': [tensor]}, + attrs={ + 'ring_id': ring_id, + 'root': gsrc, + 'use_calc_stream': use_calc_stream, + 'nranks': nranks, + }, + ) def alltoall(in_tensor_list, out_tensor_list, group=None, sync_op=True): @@ -1119,7 +1262,7 @@ def alltoall(in_tensor_list, out_tensor_list, group=None, sync_op=True): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" else: ring_id = 0 if group is None else group.id @@ -1140,44 +1283,56 @@ def alltoall(in_tensor_list, out_tensor_list, group=None, sync_op=True): use_calc_stream = sync_op if _non_static_mode(): - out = _legacy_C_ops.alltoall(temp, 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id) + out = _legacy_C_ops.alltoall( + temp, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) else: op_type = 'alltoall' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference( - dtype=in_tensor_list[0].dtype) + dtype=in_tensor_list[0].dtype + ) if not isinstance(in_tensor_list, list): - raise ValueError("The type of 'in_tensor_list' for all_to_all " - "should be list.") + raise ValueError( + "The type of 'in_tensor_list' for all_to_all " "should be list." + ) for elem in in_tensor_list: check_variable_and_dtype( - elem, 'in_tensor_list', + elem, + 'in_tensor_list', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'all_to_all') + 'all_to_all', + ) if not isinstance(out_tensor_list, list): - raise ValueError("The type of 'out_tensor_list' for all_to_all " - "should be list.") + raise ValueError( + "The type of 'out_tensor_list' for all_to_all " + "should be list." + ) if len(out_tensor_list) != 0: - raise ValueError("The 'out_tensor_list' for all_to_all " - "must be an empty list.") - helper.append_op(type=op_type, - inputs={'X': [temp]}, - outputs={'Out': [out]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - }) + raise ValueError( + "The 'out_tensor_list' for all_to_all " "must be an empty list." + ) + helper.append_op( + type=op_type, + inputs={'X': [temp]}, + outputs={'Out': [out]}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + }, + ) out_tensor_list.extend(paddle.split(out, nranks, 0)) -def alltoall_single(in_tensor, - out_tensor, - in_split_sizes=None, - out_split_sizes=None, - group=None, - sync_op=True): +def alltoall_single( + in_tensor, + out_tensor, + in_split_sizes=None, + out_split_sizes=None, + group=None, + sync_op=True, +): """ Scatter a single input tensor to all participators and gather the received tensors in out_tensor. @@ -1250,13 +1405,14 @@ def alltoall_single(in_tensor, group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" in_split_sizes = [] if in_split_sizes is None else in_split_sizes out_split_sizes = [] if out_split_sizes is None else out_split_sizes - task = group.process_group.alltoall_single(in_tensor, out_tensor, - in_split_sizes, out_split_sizes) + task = group.process_group.alltoall_single( + in_tensor, out_tensor, in_split_sizes, out_split_sizes + ) if sync_op: task.wait() return @@ -1305,7 +1461,7 @@ def send(tensor, dst=0, group=None, sync_op=True): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" task = group.process_group.send(tensor, dst) if sync_op: task.wait() @@ -1317,21 +1473,33 @@ def send(tensor, dst=0, group=None, sync_op=True): ring_id = 0 if group is None else group.id if _non_static_mode(): - return _legacy_C_ops.send_v2(tensor, 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id, 'peer', dst) + return _legacy_C_ops.send_v2( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'peer', + dst, + ) op_type = 'send_v2' check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'send') + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'send', + ) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'peer': dst, - 'use_calc_stream': use_calc_stream, - }) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + attrs={ + 'ring_id': ring_id, + 'peer': dst, + 'use_calc_stream': use_calc_stream, + }, + ) def recv(tensor, src=0, group=None, sync_op=True): @@ -1372,7 +1540,7 @@ def recv(tensor, src=0, group=None, sync_op=True): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" task = group.process_group.recv(tensor, src) if sync_op: task.wait() @@ -1384,37 +1552,58 @@ def recv(tensor, src=0, group=None, sync_op=True): ring_id = 0 if group is None else group.id if _non_static_mode(): - return _legacy_C_ops.recv_v2(tensor, 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id, 'peer', src, 'dtype', - tensor.dtype, 'out_shape', tensor.shape) + return _legacy_C_ops.recv_v2( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'peer', + src, + 'dtype', + tensor.dtype, + 'out_shape', + tensor.shape, + ) op_type = 'recv_v2' check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'recv') + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'recv', + ) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'peer': src, - 'out_shape': tensor.shape, - 'dtype': tensor.dtype, - 'use_calc_stream': use_calc_stream, - }) + helper.append_op( + type=op_type, + outputs={'Out': [tensor]}, + attrs={ + 'ring_id': ring_id, + 'peer': src, + 'out_shape': tensor.shape, + 'dtype': tensor.dtype, + 'use_calc_stream': use_calc_stream, + }, + ) def _check_single_tensor(tensor, tensor_name): if not isinstance(tensor, (core.eager.Tensor, paddle.Tensor)): - raise RuntimeError("Invalid function argument. Expected parameter {}" - "to be of type paddle.Tensor, but it's {}".format( - tensor_name, type(tensor))) + raise RuntimeError( + "Invalid function argument. Expected parameter {}" + "to be of type paddle.Tensor, but it's {}".format( + tensor_name, type(tensor) + ) + ) def _check_tensor_list(tensor_list, tensor_name): - if not isinstance(tensor_list, list) or \ - not all(isinstance(t, (core.eager.Tensor, paddle.Tensor)) for t in tensor_list): - raise RuntimeError("Invalid function argument. Expected parameter {}" - "to be of type paddle.Tensor".format(tensor_name)) + if not isinstance(tensor_list, list) or not all( + isinstance(t, (core.eager.Tensor, paddle.Tensor)) for t in tensor_list + ): + raise RuntimeError( + "Invalid function argument. Expected parameter {}" + "to be of type paddle.Tensor".format(tensor_name) + ) def isend(tensor, dst, group=None): @@ -1459,9 +1648,9 @@ def isend(tensor, dst, group=None): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" group_dst_rank = group.get_group_rank(dst) - assert group_dst_rank >= 0, ("dst rank out of group, need global rank") + assert group_dst_rank >= 0, "dst rank out of group, need global rank" return group.process_group.send(tensor, group_dst_rank) else: raise RuntimeError("Only support eager dygraph mode.") @@ -1508,9 +1697,9 @@ def irecv(tensor, src=None, group=None): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" group_src_rank = group.get_group_rank(src) - assert group_src_rank >= 0, ("src rank out of group, need global rank") + assert group_src_rank >= 0, "src rank out of group, need global rank" return group.process_group.recv(tensor, group_src_rank) else: raise RuntimeError("Only support eager dygraph mode.") @@ -1536,9 +1725,11 @@ class P2POp(object): def __init__(self, op, tensor, peer, group=None): if op not in [isend, irecv]: - raise RuntimeError("Invalid ``op`` function. Expected ``op`` " - "to be of type ``paddle.distributed.isend`` or " - "``paddle.distributed.irecv``.") + raise RuntimeError( + "Invalid ``op`` function. Expected ``op`` " + "to be of type ``paddle.distributed.isend`` or " + "``paddle.distributed.irecv``." + ) _check_single_tensor(tensor, "tensor") self.op = op @@ -1564,13 +1755,17 @@ def _check_p2p_op_list(p2p_op_list): all ops use the same backend. """ if not isinstance(p2p_op_list, list) or not all( - isinstance(p2p_op, P2POp) for p2p_op in p2p_op_list): - raise RuntimeError("Invalid ``p2p_op_list``. Each op is expected to " - "to be of type ``paddle.distributed.P2POp``.") + isinstance(p2p_op, P2POp) for p2p_op in p2p_op_list + ): + raise RuntimeError( + "Invalid ``p2p_op_list``. Each op is expected to " + "to be of type ``paddle.distributed.P2POp``." + ) backend = _group_map_backend[p2p_op_list[0].group] - if not all(backend == _group_map_backend[p2p_op.group] - for p2p_op in p2p_op_list): + if not all( + backend == _group_map_backend[p2p_op.group] for p2p_op in p2p_op_list + ): raise RuntimeError("All groups need to use the same backend.") @@ -1647,11 +1842,9 @@ def batch_isend_irecv(p2p_op_list): raise RuntimeError("Don't support static graph mode currently.") -def reduce_scatter(tensor, - tensor_list, - op=ReduceOp.SUM, - group=None, - sync_op=True): +def reduce_scatter( + tensor, tensor_list, op=ReduceOp.SUM, group=None, sync_op=True +): """ Reduces, then scatters a list of tensors to all processes in a group @@ -1702,7 +1895,7 @@ def reduce_scatter(tensor, op_type = _get_reduce_op(op, "reduce_scatter") group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" temp = paddle.concat(tensor_list, axis=0) task = group.process_group._reduce_scatter_base(tensor, temp, op_type) @@ -1715,11 +1908,9 @@ def reduce_scatter(tensor, raise RuntimeError("Don't support static graph mode currently.") -def _reduce_scatter_base(output, - input, - op=ReduceOp.SUM, - group=None, - sync_op=True): +def _reduce_scatter_base( + output, input, op=ReduceOp.SUM, group=None, sync_op=True +): """ Reduces, then scatters a flattened tensor to all processes in a group. diff --git a/python/paddle/distributed/communication/all_reduce.py b/python/paddle/distributed/communication/all_reduce.py index 737e0cbbfb5..7a09d779e8f 100644 --- a/python/paddle/distributed/communication/all_reduce.py +++ b/python/paddle/distributed/communication/all_reduce.py @@ -58,30 +58,28 @@ def all_reduce(tensor, op=ReduceOp.SUM, group=None, sync_op=True): # [[5, 7, 9], [5, 7, 9]] (2 GPUs) """ if not framework._in_legacy_dygraph(): - return stream.all_reduce(tensor, - op=op, - group=group, - sync_op=sync_op, - use_calc_stream=False) + return stream.all_reduce( + tensor, op=op, group=group, sync_op=sync_op, use_calc_stream=False + ) # code below will be removed after we remove the old dygraph use_calc_stream = sync_op ring_id = 0 if group is None else group.id if op == ReduceOp.SUM: - return paddle._legacy_C_ops.c_allreduce_sum_(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id) + return paddle._legacy_C_ops.c_allreduce_sum_( + tensor, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) elif op == ReduceOp.MAX: - return paddle._legacy_C_ops.c_allreduce_max_(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id) + return paddle._legacy_C_ops.c_allreduce_max_( + tensor, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) elif op == ReduceOp.MIN: - return paddle._legacy_C_ops.c_allreduce_min_(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id) + return paddle._legacy_C_ops.c_allreduce_min_( + tensor, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) elif op == ReduceOp.PROD: - return paddle._legacy_C_ops.c_allreduce_prod_(tensor, 'use_calc_stream', - use_calc_stream, - 'ring_id', ring_id) + return paddle._legacy_C_ops.c_allreduce_prod_( + tensor, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) else: raise ValueError("Unknown parameter: {}.".format(op)) diff --git a/python/paddle/distributed/communication/group.py b/python/paddle/distributed/communication/group.py index 6b4e545b245..60f1264fe80 100644 --- a/python/paddle/distributed/communication/group.py +++ b/python/paddle/distributed/communication/group.py @@ -13,7 +13,7 @@ # limitations under the License. -class Group(): +class Group: """ The abstract representation of group. """ @@ -69,14 +69,15 @@ class Group(): def __repr__(self): debug_str = "rank: {}, nranks: {}, id: {}, ranks: ".format( - self.rank, self.nranks, self.id) + self.rank, self.nranks, self.id + ) debug_str += ", ".join(map(str, self.ranks)) debug_str += "; name: " debug_str += self.name if self.name else "None" return debug_str -class _GroupManager(): +class _GroupManager: global_group_id = 0 group_map_by_id = {} @@ -89,6 +90,7 @@ def _get_global_group(): def _add_new_group(group): if group.id in _GroupManager.group_map_by_id: - raise RuntimeError("The group with id {} already exist.".format( - group.id)) + raise RuntimeError( + "The group with id {} already exist.".format(group.id) + ) _GroupManager.group_map_by_id[group.id] = group diff --git a/python/paddle/distributed/communication/reduce.py b/python/paddle/distributed/communication/reduce.py index 5caa5bebedf..8628e83b626 100644 --- a/python/paddle/distributed/communication/reduce.py +++ b/python/paddle/distributed/communication/reduce.py @@ -46,6 +46,7 @@ class ReduceOp: print(data) # [[5, 7, 9], [5, 7, 9]] (2 GPUs) """ + SUM = 0 MAX = 1 MIN = 2 diff --git a/python/paddle/distributed/communication/stream/__init__.py b/python/paddle/distributed/communication/stream/__init__.py index 43952ce5541..63ec858557b 100644 --- a/python/paddle/distributed/communication/stream/__init__.py +++ b/python/paddle/distributed/communication/stream/__init__.py @@ -24,6 +24,14 @@ from .scatter import scatter from .send import send __all__ = [ - "all_gather", "all_reduce", "alltoall", "alltoall_single", "broadcast", - "reduce", "reduce_scatter", "recv", "scatter", "send" + "all_gather", + "all_reduce", + "alltoall", + "alltoall_single", + "broadcast", + "reduce", + "reduce_scatter", + "recv", + "scatter", + "send", ] diff --git a/python/paddle/distributed/communication/stream/all_gather.py b/python/paddle/distributed/communication/stream/all_gather.py index 9eb961cda17..cdd7f98554b 100644 --- a/python/paddle/distributed/communication/stream/all_gather.py +++ b/python/paddle/distributed/communication/stream/all_gather.py @@ -27,33 +27,39 @@ def _check_tensor_shape(tensor, shape, nranks=1): def _check_tensor_list_shape(tensor_list, shape, nranks=1): if len(tensor_list) != nranks: raise RuntimeError( - 'The tensor_list for all_gather is not correctly-sized.') + 'The tensor_list for all_gather is not correctly-sized.' + ) for tensor in tensor_list: if tensor.shape != shape: raise RuntimeError( - 'The tensor_list for all_gather is not correctly-sized.') + 'The tensor_list for all_gather is not correctly-sized.' + ) -def _all_gather_into_tensor_in_dygraph(out_tensor, in_tensor, group, sync_op, - use_calc_stream): +def _all_gather_into_tensor_in_dygraph( + out_tensor, in_tensor, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group _check_tensor_shape(out_tensor, in_tensor.shape, group.nranks) if use_calc_stream: return group.process_group.allgather_into_tensor_on_calc_stream( - in_tensor, out_tensor) + in_tensor, out_tensor + ) - task = group.process_group.allgather_into_tensor(in_tensor, out_tensor, - sync_op) + task = group.process_group.allgather_into_tensor( + in_tensor, out_tensor, sync_op + ) if sync_op: task.wait() return task -def _all_gather_in_dygraph(tensor_list, tensor, group, sync_op, - use_calc_stream): +def _all_gather_in_dygraph( + tensor_list, tensor, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group if len(tensor_list) == 0: @@ -71,11 +77,13 @@ def _all_gather_in_dygraph(tensor_list, tensor, group, sync_op, return task -def all_gather(tensor_or_tensor_list, - tensor, - group=None, - sync_op=True, - use_calc_stream=False): +def all_gather( + tensor_or_tensor_list, + tensor, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Gather tensors across devices to a correctly-sized tensor or a tensor list. @@ -122,16 +130,18 @@ def all_gather(tensor_or_tensor_list, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): if paddle.is_tensor(tensor_or_tensor_list): - return _all_gather_into_tensor_in_dygraph(tensor_or_tensor_list, - tensor, group, sync_op, - use_calc_stream) + return _all_gather_into_tensor_in_dygraph( + tensor_or_tensor_list, tensor, group, sync_op, use_calc_stream + ) else: - return _all_gather_in_dygraph(tensor_or_tensor_list, tensor, group, - sync_op, use_calc_stream) + return _all_gather_in_dygraph( + tensor_or_tensor_list, tensor, group, sync_op, use_calc_stream + ) raise RuntimeError( "paddle.distributed.stream.all_gather is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/all_reduce.py b/python/paddle/distributed/communication/stream/all_reduce.py index 0ba161a078a..97eda9d0f9b 100644 --- a/python/paddle/distributed/communication/stream/all_reduce.py +++ b/python/paddle/distributed/communication/stream/all_reduce.py @@ -34,10 +34,21 @@ def _all_reduce_in_dygraph(tensor, op, group, sync_op, use_calc_stream): def _all_reduce_in_static_mode(tensor, op, group, sync_op, use_calc_stream): - data_feeder.check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'bool' - ], 'all_reduce') + data_feeder.check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'bool', + ], + 'all_reduce', + ) op_type = _get_reduce_op(op, "all_reduce") ring_id = 0 if group is None else group.id @@ -48,22 +59,19 @@ def _all_reduce_in_static_mode(tensor, op, group, sync_op, use_calc_stream): # TODO: Support task and use task.wait in static mode # Use use_calc_stream rather than sync_op helper = layer_helper.LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': sync_op - }) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [tensor]}, + attrs={'ring_id': ring_id, 'use_calc_stream': sync_op}, + ) return None -def all_reduce(tensor, - op=ReduceOp.SUM, - group=None, - sync_op=True, - use_calc_stream=False): +def all_reduce( + tensor, op=ReduceOp.SUM, group=None, sync_op=True, use_calc_stream=False +): """ Perform specific reduction (for example, sum, max) on inputs across devices. @@ -106,11 +114,14 @@ def all_reduce(tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): - return _all_reduce_in_dygraph(tensor, op, group, sync_op, - use_calc_stream) + return _all_reduce_in_dygraph( + tensor, op, group, sync_op, use_calc_stream + ) else: - return _all_reduce_in_static_mode(tensor, op, group, sync_op, - use_calc_stream) + return _all_reduce_in_static_mode( + tensor, op, group, sync_op, use_calc_stream + ) diff --git a/python/paddle/distributed/communication/stream/alltoall.py b/python/paddle/distributed/communication/stream/alltoall.py index b216906d045..5faf1062631 100644 --- a/python/paddle/distributed/communication/stream/alltoall.py +++ b/python/paddle/distributed/communication/stream/alltoall.py @@ -25,22 +25,26 @@ def _check_tensor_shape(tensor, shape, nranks=1): def _check_tensor_list_shape(tensor_list, shape, nranks=1): if len(tensor_list) != nranks: raise RuntimeError( - 'The tensor_list for alltoall is not correctly-sized.') + 'The tensor_list for alltoall is not correctly-sized.' + ) for tensor in tensor_list: if tensor.shape != shape: raise RuntimeError( - 'The tensor_list for alltoall is not correctly-sized.') + 'The tensor_list for alltoall is not correctly-sized.' + ) -def _alltoall_tensor_in_dygraph(out_tensor, in_tensor, group, sync_op, - use_calc_stream): +def _alltoall_tensor_in_dygraph( + out_tensor, in_tensor, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group _check_tensor_shape(out_tensor, in_tensor.shape, group.nranks) if use_calc_stream: return group.process_group.alltoall_tensor_on_calc_stream( - in_tensor, out_tensor) + in_tensor, out_tensor + ) task = group.process_group.alltoall_tensor(in_tensor, out_tensor, sync_op) if sync_op: @@ -49,8 +53,9 @@ def _alltoall_tensor_in_dygraph(out_tensor, in_tensor, group, sync_op, return task -def _alltoall_in_dygraph(out_tensor_list, in_tensor_list, group, sync_op, - use_calc_stream): +def _alltoall_in_dygraph( + out_tensor_list, in_tensor_list, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group if len(in_tensor_list) == 0: @@ -61,26 +66,31 @@ def _alltoall_in_dygraph(out_tensor_list, in_tensor_list, group, sync_op, paddle.empty_like(tensor) for tensor in in_tensor_list ] else: - _check_tensor_list_shape(out_tensor_list, in_tensor_list[0].shape, - group.nranks) + _check_tensor_list_shape( + out_tensor_list, in_tensor_list[0].shape, group.nranks + ) if use_calc_stream: return group.process_group.alltoall_on_calc_stream( - in_tensor_list, out_tensor_list) + in_tensor_list, out_tensor_list + ) - task = group.process_group.alltoall(in_tensor_list, out_tensor_list, - sync_op) + task = group.process_group.alltoall( + in_tensor_list, out_tensor_list, sync_op + ) if sync_op: task.wait() return task -def alltoall(out_tensor_or_tensor_list, - in_tensor_or_tensor_list, - group=None, - sync_op=True, - use_calc_stream=False): +def alltoall( + out_tensor_or_tensor_list, + in_tensor_or_tensor_list, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Scatter a tensor (or a tensor list) across devices and gather outputs to another tensor (or a tensor list, respectively). @@ -130,7 +140,8 @@ def alltoall(out_tensor_or_tensor_list, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if out_tensor_or_tensor_list is None: raise RuntimeError("The output should be specified.") @@ -141,16 +152,25 @@ def alltoall(out_tensor_or_tensor_list, out_is_tensor = paddle.is_tensor(out_tensor_or_tensor_list) in_is_tensor = paddle.is_tensor(in_tensor_or_tensor_list) if out_is_tensor and in_is_tensor: - return _alltoall_tensor_in_dygraph(out_tensor_or_tensor_list, - in_tensor_or_tensor_list, group, - sync_op, use_calc_stream) + return _alltoall_tensor_in_dygraph( + out_tensor_or_tensor_list, + in_tensor_or_tensor_list, + group, + sync_op, + use_calc_stream, + ) elif not out_is_tensor and not in_is_tensor: - return _alltoall_in_dygraph(out_tensor_or_tensor_list, - in_tensor_or_tensor_list, group, - sync_op, use_calc_stream) + return _alltoall_in_dygraph( + out_tensor_or_tensor_list, + in_tensor_or_tensor_list, + group, + sync_op, + use_calc_stream, + ) else: raise RuntimeError( - "The output and input should be both tensor or tensor list.") + "The output and input should be both tensor or tensor list." + ) raise RuntimeError( "paddle.distributed.stream.alltoall is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/alltoall_single.py b/python/paddle/distributed/communication/stream/alltoall_single.py index b2187cc06e3..75eb2890326 100644 --- a/python/paddle/distributed/communication/stream/alltoall_single.py +++ b/python/paddle/distributed/communication/stream/alltoall_single.py @@ -16,9 +16,15 @@ import paddle.fluid.framework as framework from paddle.distributed import collective -def _alltoall_single_in_dygraph(out_tensor, in_tensor, out_split_sizes, - in_split_sizes, group, sync_op, - use_calc_stream): +def _alltoall_single_in_dygraph( + out_tensor, + in_tensor, + out_split_sizes, + in_split_sizes, + group, + sync_op, + use_calc_stream, +): group = collective._get_default_group() if group is None else group if out_split_sizes is None: @@ -28,24 +34,27 @@ def _alltoall_single_in_dygraph(out_tensor, in_tensor, out_split_sizes, if use_calc_stream: return group.process_group.alltoall_single_on_calc_stream( - in_tensor, out_tensor, in_split_sizes, out_split_sizes) + in_tensor, out_tensor, in_split_sizes, out_split_sizes + ) - task = group.process_group.alltoall_single(in_tensor, out_tensor, - in_split_sizes, out_split_sizes, - sync_op) + task = group.process_group.alltoall_single( + in_tensor, out_tensor, in_split_sizes, out_split_sizes, sync_op + ) if sync_op: task.wait() return task -def alltoall_single(out_tensor, - in_tensor, - out_split_sizes=None, - in_split_sizes=None, - group=None, - sync_op=True, - use_calc_stream=False): +def alltoall_single( + out_tensor, + in_tensor, + out_split_sizes=None, + in_split_sizes=None, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Split and Scatter the splitted input tensor to the out tensor across devices. @@ -116,12 +125,19 @@ def alltoall_single(out_tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): - return _alltoall_single_in_dygraph(out_tensor, in_tensor, - out_split_sizes, in_split_sizes, - group, sync_op, use_calc_stream) + return _alltoall_single_in_dygraph( + out_tensor, + in_tensor, + out_split_sizes, + in_split_sizes, + group, + sync_op, + use_calc_stream, + ) raise RuntimeError( "paddle.distributed.stream.alltoall_single is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/broadcast.py b/python/paddle/distributed/communication/stream/broadcast.py index 06bde316937..3672b02811e 100644 --- a/python/paddle/distributed/communication/stream/broadcast.py +++ b/python/paddle/distributed/communication/stream/broadcast.py @@ -72,11 +72,13 @@ def broadcast(tensor, src=0, group=None, sync_op=True, use_calc_stream=False): if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be True in sync op behavior.") + "use_calc_stream can only be True in sync op behavior." + ) if framework.in_dygraph_mode(): - return _broadcast_in_dygraph(tensor, src, group, sync_op, - use_calc_stream) + return _broadcast_in_dygraph( + tensor, src, group, sync_op, use_calc_stream + ) raise RuntimeError( "paddle.distributed.stream.broadcast is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/recv.py b/python/paddle/distributed/communication/stream/recv.py index 25a81737884..2658379a412 100644 --- a/python/paddle/distributed/communication/stream/recv.py +++ b/python/paddle/distributed/communication/stream/recv.py @@ -73,10 +73,12 @@ def recv(tensor, src=0, group=None, sync_op=True, use_calc_stream=False): if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be True in sync op behavior.") + "use_calc_stream can only be True in sync op behavior." + ) if framework.in_dygraph_mode(): return _recv_in_dygraph(tensor, src, group, sync_op, use_calc_stream) raise RuntimeError( - "paddle.distributed.stream.recv is only supported in dygraph mode now.") + "paddle.distributed.stream.recv is only supported in dygraph mode now." + ) diff --git a/python/paddle/distributed/communication/stream/reduce.py b/python/paddle/distributed/communication/stream/reduce.py index b0f7f5c8847..d7f0fc6b2b7 100644 --- a/python/paddle/distributed/communication/stream/reduce.py +++ b/python/paddle/distributed/communication/stream/reduce.py @@ -30,12 +30,14 @@ def _reduce_in_dygraph(tensor, dst, op, group, sync_op, use_calc_stream): return task -def reduce(tensor, - dst=0, - op=ReduceOp.SUM, - group=None, - sync_op=True, - use_calc_stream=False): +def reduce( + tensor, + dst=0, + op=ReduceOp.SUM, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Perform specific reduction (for example, sum, max) on a tensor across devices and send to the destintion device. @@ -82,11 +84,13 @@ def reduce(tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): - return _reduce_in_dygraph(tensor, dst, op, group, sync_op, - use_calc_stream) + return _reduce_in_dygraph( + tensor, dst, op, group, sync_op, use_calc_stream + ) raise RuntimeError( "paddle.distributed.stream.reduce is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/reduce_scatter.py b/python/paddle/distributed/communication/stream/reduce_scatter.py index 68ba8e18e40..ef375fa5be4 100644 --- a/python/paddle/distributed/communication/stream/reduce_scatter.py +++ b/python/paddle/distributed/communication/stream/reduce_scatter.py @@ -23,26 +23,31 @@ def _check_tensor_shape(tensor, shape, nranks=1): expect_shape[0] //= nranks if list(tensor.shape) != expect_shape: raise RuntimeError( - "The in_tensor for reduce_scatter is not correctly-sized.") + "The in_tensor for reduce_scatter is not correctly-sized." + ) def _check_tensor_list_shape(tensor_list, shape, nranks=1): if len(tensor_list) != nranks: raise RuntimeError( - "The tensor_list for reduce_scatter is not correctly-sized.") + "The tensor_list for reduce_scatter is not correctly-sized." + ) for tensor in tensor_list: if tensor.shape != shape: raise RuntimeError( - "The tensor_list for reduce_scatter is not correctly-sized.") - - -def _reduce_scatter_tensor_in_dygraph(out_tensor, - in_tensor, - op, - group, - sync_op, - use_calc_stream, - caller="reduce_scatter"): + "The tensor_list for reduce_scatter is not correctly-sized." + ) + + +def _reduce_scatter_tensor_in_dygraph( + out_tensor, + in_tensor, + op, + group, + sync_op, + use_calc_stream, + caller="reduce_scatter", +): op_type = _get_reduce_op(op, caller) group = _get_global_group() if group is None else group @@ -50,18 +55,21 @@ def _reduce_scatter_tensor_in_dygraph(out_tensor, if use_calc_stream: return group.process_group.reduce_scatter_tensor_on_calc_stream( - in_tensor, out_tensor, op_type) + in_tensor, out_tensor, op_type + ) - task = group.process_group.reduce_scatter_tensor(in_tensor, out_tensor, - op_type, sync_op) + task = group.process_group.reduce_scatter_tensor( + in_tensor, out_tensor, op_type, sync_op + ) if sync_op: task.wait() return task -def _reduce_scatter_in_dygraph(tensor, tensor_list, op, group, sync_op, - use_calc_stream): +def _reduce_scatter_in_dygraph( + tensor, tensor_list, op, group, sync_op, use_calc_stream +): op_type = _get_reduce_op(op, "reduce_scatter") group = _get_global_group() if group is None else group @@ -69,22 +77,26 @@ def _reduce_scatter_in_dygraph(tensor, tensor_list, op, group, sync_op, if use_calc_stream: return group.process_group.reduce_scatter_on_calc_stream( - tensor_list, tensor, op_type) + tensor_list, tensor, op_type + ) - task = group.process_group.reduce_scatter(tensor_list, tensor, op_type, - sync_op) + task = group.process_group.reduce_scatter( + tensor_list, tensor, op_type, sync_op + ) if sync_op: task.wait() return task -def reduce_scatter(tensor, - tensor_or_tensor_list, - op=ReduceOp.SUM, - group=None, - sync_op=True, - use_calc_stream=False): +def reduce_scatter( + tensor, + tensor_or_tensor_list, + op=ReduceOp.SUM, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Reduce, then scatter a tensor (or a tensor list) across devices. @@ -132,29 +144,42 @@ def reduce_scatter(tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): if paddle.is_tensor(tensor_or_tensor_list): - return _reduce_scatter_tensor_in_dygraph(tensor, - tensor_or_tensor_list, op, - group, sync_op, - use_calc_stream) + return _reduce_scatter_tensor_in_dygraph( + tensor, + tensor_or_tensor_list, + op, + group, + sync_op, + use_calc_stream, + ) else: - return _reduce_scatter_in_dygraph(tensor, tensor_or_tensor_list, op, - group, sync_op, use_calc_stream) + return _reduce_scatter_in_dygraph( + tensor, + tensor_or_tensor_list, + op, + group, + sync_op, + use_calc_stream, + ) raise RuntimeError( "paddle.distributed.stream.reduce_scatter is only supported in dygraph mode now." ) -def _reduce_scatter_base(out_tensor, - in_tensor, - op=ReduceOp.SUM, - group=None, - sync_op=True, - use_calc_stream=False): +def _reduce_scatter_base( + out_tensor, + in_tensor, + op=ReduceOp.SUM, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Reduce, then scatter a flattened tensor across devices. @@ -202,13 +227,19 @@ def _reduce_scatter_base(out_tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): - return _reduce_scatter_tensor_in_dygraph(out_tensor, in_tensor, op, - group, sync_op, - use_calc_stream, - "_reduce_scatter_base") + return _reduce_scatter_tensor_in_dygraph( + out_tensor, + in_tensor, + op, + group, + sync_op, + use_calc_stream, + "_reduce_scatter_base", + ) raise RuntimeError( "paddle.distributed.stream._reduce_scatter_base is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/scatter.py b/python/paddle/distributed/communication/stream/scatter.py index 9b0fcaf4b08..199885c1fde 100644 --- a/python/paddle/distributed/communication/stream/scatter.py +++ b/python/paddle/distributed/communication/stream/scatter.py @@ -28,15 +28,18 @@ def _check_tensor_shape(tensor, shape, nranks=1): def _check_tensor_list_shape(tensor_list, shape, nranks=1): if len(tensor_list) != nranks: raise RuntimeError( - "The tensor_list for scatter is not correctly-sized.") + "The tensor_list for scatter is not correctly-sized." + ) for tensor in tensor_list: if tensor.shape != shape: raise RuntimeError( - "The tensor_list for scatter is not correctly-sized.") + "The tensor_list for scatter is not correctly-sized." + ) -def _scatter_tensor_in_dygraph(out_tensor, in_tensor, src, group, sync_op, - use_calc_stream): +def _scatter_tensor_in_dygraph( + out_tensor, in_tensor, src, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group src_rank = group.get_group_rank(src) @@ -50,18 +53,21 @@ def _scatter_tensor_in_dygraph(out_tensor, in_tensor, src, group, sync_op, if use_calc_stream: return group.process_group.scatter_tensor_on_calc_stream( - in_tensor, out_tensor, src) + in_tensor, out_tensor, src + ) - task = group.process_group.scatter_tensor(in_tensor, out_tensor, src, - sync_op) + task = group.process_group.scatter_tensor( + in_tensor, out_tensor, src, sync_op + ) if sync_op: task.wait() return task -def _scatter_in_dygraph(tensor, tensor_list, src, group, sync_op, - use_calc_stream): +def _scatter_in_dygraph( + tensor, tensor_list, src, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group src_rank = group.get_group_rank(src) @@ -73,14 +79,16 @@ def _scatter_in_dygraph(tensor, tensor_list, src, group, sync_op, if rank == src_rank: if len(tensor_list) == 0: raise RuntimeError( - "The tensor_list should not be empty on src rank.") + "The tensor_list should not be empty on src rank." + ) _check_tensor_list_shape(tensor_list, tensor.shape, nranks) else: tensor_list = [tensor for _ in range(nranks)] if use_calc_stream: return group.process_group.scatter_on_calc_stream( - tensor_list, tensor, src) + tensor_list, tensor, src + ) task = group.process_group.scatter(tensor_list, tensor, src, sync_op) if sync_op: @@ -89,12 +97,14 @@ def _scatter_in_dygraph(tensor, tensor_list, src, group, sync_op, return task -def scatter(tensor, - tensor_or_tensor_list=None, - src=0, - group=None, - sync_op=True, - use_calc_stream=False): +def scatter( + tensor, + tensor_or_tensor_list=None, + src=0, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Scatter a tensor (or a tensor list) across devices. @@ -143,19 +153,31 @@ def scatter(tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if tensor_or_tensor_list is None: raise RuntimeError("The input should be specified.") if framework.in_dygraph_mode(): if paddle.is_tensor(tensor_or_tensor_list): - return _scatter_tensor_in_dygraph(tensor, tensor_or_tensor_list, - src, group, sync_op, - use_calc_stream) + return _scatter_tensor_in_dygraph( + tensor, + tensor_or_tensor_list, + src, + group, + sync_op, + use_calc_stream, + ) else: - return _scatter_in_dygraph(tensor, tensor_or_tensor_list, src, - group, sync_op, use_calc_stream) + return _scatter_in_dygraph( + tensor, + tensor_or_tensor_list, + src, + group, + sync_op, + use_calc_stream, + ) raise RuntimeError( "paddle.distributed.stream.scatter is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/send.py b/python/paddle/distributed/communication/stream/send.py index 41ec2c0141b..206a8cfaa6f 100644 --- a/python/paddle/distributed/communication/stream/send.py +++ b/python/paddle/distributed/communication/stream/send.py @@ -73,10 +73,12 @@ def send(tensor, dst=0, group=None, sync_op=True, use_calc_stream=False): if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be True in sync op behavior.") + "use_calc_stream can only be True in sync op behavior." + ) if framework.in_dygraph_mode(): return _send_in_dygraph(tensor, dst, group, sync_op, use_calc_stream) raise RuntimeError( - "paddle.distributed.stream.send is only supported in dygraph mode now.") + "paddle.distributed.stream.send is only supported in dygraph mode now." + ) diff --git a/python/paddle/distributed/elastic.py b/python/paddle/distributed/elastic.py index b13eb07ee28..d1fd9a790f2 100644 --- a/python/paddle/distributed/elastic.py +++ b/python/paddle/distributed/elastic.py @@ -17,7 +17,6 @@ import os class Command(object): - def __init__(self, server, name): import etcd3 @@ -47,14 +46,15 @@ class Command(object): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Elastic Command') - parser.add_argument("--elastic_server", - type=str, - help="etcd server host:port") + parser.add_argument( + "--elastic_server", type=str, help="etcd server host:port" + ) parser.add_argument("--job_id", type=str, help="job unique id") parser.add_argument( "--np", type=str, - help="job pod/node number, need to be 'MIN' or 'MIN:MAX' format") + help="job pod/node number, need to be 'MIN' or 'MIN:MAX' format", + ) parser.add_argument("action", type=str, help="action to take") args = parser.parse_args() diff --git a/python/paddle/distributed/entry_attr.py b/python/paddle/distributed/entry_attr.py index a1183147781..f5043ac084c 100644 --- a/python/paddle/distributed/entry_attr.py +++ b/python/paddle/distributed/entry_attr.py @@ -124,11 +124,13 @@ class CountFilterEntry(EntryAttr): if not isinstance(count_filter, int): raise ValueError( - "count_filter must be a valid integer greater than 0") + "count_filter must be a valid integer greater than 0" + ) if count_filter < 0: raise ValueError( - "count_filter must be a valid integer greater or equal than 0") + "count_filter must be a valid integer greater or equal than 0" + ) self._name = "count_filter_entry" self._count_filter = count_filter diff --git a/python/paddle/distributed/fleet/__init__.py b/python/paddle/distributed/fleet/__init__.py index b75d84edf29..aebedefeaaf 100755 --- a/python/paddle/distributed/fleet/__init__.py +++ b/python/paddle/distributed/fleet/__init__.py @@ -24,7 +24,9 @@ from .dataset import QueueDataset # noqa: F401 from .dataset import FileInstantDataset # noqa: F401 from .dataset import BoxPSDataset # noqa: F401 from .data_generator.data_generator import MultiSlotDataGenerator # noqa: F401 -from .data_generator.data_generator import MultiSlotStringDataGenerator # noqa: F401 +from .data_generator.data_generator import ( + MultiSlotStringDataGenerator, +) # noqa: F401 from . import metrics # noqa: F401 from .base.topology import CommunicateTopology from .base.topology import HybridCommunicateGroup # noqa: F401 @@ -34,11 +36,17 @@ from .optimizer import distributed_optimizer from .scaler import distributed_scaler from .utils import log_util -__all__ = [ #noqa - "CommunicateTopology", "UtilBase", "HybridCommunicateGroup", - "MultiSlotStringDataGenerator", "UserDefinedRoleMaker", - "DistributedStrategy", "Role", "MultiSlotDataGenerator", - "PaddleCloudRoleMaker", "Fleet" +__all__ = [ # noqa + "CommunicateTopology", + "UtilBase", + "HybridCommunicateGroup", + "MultiSlotStringDataGenerator", + "UserDefinedRoleMaker", + "DistributedStrategy", + "Role", + "MultiSlotDataGenerator", + "PaddleCloudRoleMaker", + "Fleet", ] fleet = Fleet() diff --git a/python/paddle/distributed/fleet/ascend_utils.py b/python/paddle/distributed/fleet/ascend_utils.py index 6ff31555a12..ee10cd78a5e 100644 --- a/python/paddle/distributed/fleet/ascend_utils.py +++ b/python/paddle/distributed/fleet/ascend_utils.py @@ -14,7 +14,11 @@ import os import json -from paddle.distributed.fleet.launch_utils import DeviceMode, get_cluster, get_host_name_ip +from paddle.distributed.fleet.launch_utils import ( + DeviceMode, + get_cluster, + get_host_name_ip, +) __all__ = [] @@ -80,17 +84,18 @@ def _get_ascend_rankfile(rank_table_file_path): assert nodes is not None, "DLS_TASK_NUMBER didn't set!" for node in range(int(nodes)): node_ip = os.getenv("VC_CUSTOM{}_HOSTS".format(node), None) - assert node_ip is not None, "VC_CUSTOM{}_HOSTS didn't set!".format( - node) + assert ( + node_ip is not None + ), "VC_CUSTOM{}_HOSTS didn't set!".format(node) node_ips.append(node_ip) return node_ips, device_count node_ips.append(server['server_id']) return node_ips, device_count -def get_cloud_cluster(rank_table_file=None, - device_mode=DeviceMode.ASCEND_NPU, - start_port=6070): +def get_cloud_cluster( + rank_table_file=None, device_mode=DeviceMode.ASCEND_NPU, start_port=6070 +): """ Args: rank_table_file: string, ascend npu rank file path @@ -110,8 +115,12 @@ def get_cloud_cluster(rank_table_file=None, else: _, node_ip = get_host_name_ip() - assert node_ip in node_ips, "Can't find your local ip {%s} in node_ips: {%s}" \ - % (node_ip, node_ips) + assert ( + node_ip in node_ips + ), "Can't find your local ip {%s} in node_ips: {%s}" % ( + node_ip, + node_ips, + ) else: # single trainer (single ascend card) node_ips = ["127.0.0.1"] @@ -127,5 +136,6 @@ def get_cloud_cluster(rank_table_file=None, for ip in node_ips: trainer_endpoints.append(["%s:%d" % (ip, port) for port in free_ports]) - return get_cluster(node_ips, node_ip, trainer_endpoints, device_mode, - devices_per_proc) + return get_cluster( + node_ips, node_ip, trainer_endpoints, device_mode, devices_per_proc + ) diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index c54df36cec6..c32b1f2d68c 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -26,7 +26,6 @@ non_auto_func_called = True def __non_auto_func_called__(func): - def __impl__(*args, **kwargs): global non_auto_func_called non_auto_func_called = False @@ -129,7 +128,8 @@ class DistributedStrategy(object): key = 'FLAGS_cudnn_batchnorm_spatial_persistent' if _global_flags().is_public(key): self.strategy.cudnn_batchnorm_spatial_persistent = bool( - _global_flags()[key]) + _global_flags()[key] + ) key = 'FLAGS_conv_workspace_size_limit' if _global_flags().is_public(key): self.strategy.conv_workspace_size_limit = int(_global_flags()[key]) @@ -144,8 +144,9 @@ class DistributedStrategy(object): def __setattr__(self, key, value): if self.__lock_attr and not hasattr(self, key): - raise TypeError("%s is not a attribute of %s" % - (key, self.__class__.__name__)) + raise TypeError( + "%s is not a attribute of %s" % (key, self.__class__.__name__) + ) object.__setattr__(self, key, value) def save_to_prototxt(self, output): @@ -180,7 +181,8 @@ class DistributedStrategy(object): """ with open(pb_file, 'r') as f: self.strategy = google.protobuf.text_format.Merge( - str(f.read()), self.strategy) + str(f.read()), self.strategy + ) @property def execution_strategy(self): @@ -203,8 +205,11 @@ class DistributedStrategy(object): execution_strategy = paddle.fluid.ExecutionStrategy() fields = self.strategy.execution_strategy.DESCRIPTOR.fields for f in fields: - setattr(execution_strategy, f.name, - getattr(self.strategy.execution_strategy, f.name)) + setattr( + execution_strategy, + f.name, + getattr(self.strategy.execution_strategy, f.name), + ) return execution_strategy @execution_strategy.setter @@ -212,8 +217,11 @@ class DistributedStrategy(object): def execution_strategy(self, strategy): fields = self.strategy.execution_strategy.DESCRIPTOR.fields for f in fields: - setattr(self.strategy.execution_strategy, f.name, - getattr(strategy, f.name)) + setattr( + self.strategy.execution_strategy, + f.name, + getattr(strategy, f.name), + ) @property def build_strategy(self): @@ -261,8 +269,9 @@ class DistributedStrategy(object): value = ReduceStrategyFleet(value) setattr(self.strategy.build_strategy, f.name, value) elif f.label == 3: # repeated field - getattr(self.strategy.build_strategy, - f.name).extend(getattr(strategy, f.name)) + getattr(self.strategy.build_strategy, f.name).extend( + getattr(strategy, f.name) + ) @property def gradient_scale_configs(self): @@ -282,8 +291,11 @@ class DistributedStrategy(object): @gradient_scale_configs.setter @is_strict_auto def gradient_scale_configs(self, config): - check_configs_key(self.strategy.gradient_scale_configs, config, - 'gradient_scale_configs') + check_configs_key( + self.strategy.gradient_scale_configs, + config, + 'gradient_scale_configs', + ) assign_configs_value(self.strategy.gradient_scale_configs, config) @property @@ -318,8 +330,10 @@ class DistributedStrategy(object): self.a_sync_configs = {"k_steps": 0} else: raise ValueError( - "The type of `flag` is invalid, expected type is bool, but received {}" - .format(type(flag))) + "The type of `flag` is invalid, expected type is bool, but received {}".format( + type(flag) + ) + ) @property def a_sync_configs(self): @@ -365,8 +379,9 @@ class DistributedStrategy(object): @a_sync_configs.setter @is_strict_auto def a_sync_configs(self, configs): - check_configs_key(self.strategy.a_sync_configs, configs, - "a_sync_configs") + check_configs_key( + self.strategy.a_sync_configs, configs, "a_sync_configs" + ) assign_configs_value(self.strategy.a_sync_configs, configs) @property @@ -430,14 +445,17 @@ class DistributedStrategy(object): self.strategy.adam_d2sum = flag else: raise ValueError( - "The type of `flag` is invalid, expected type is bool, but received {}" - .format(type(flag))) + "The type of `flag` is invalid, expected type is bool, but received {}".format( + type(flag) + ) + ) @trainer_desc_configs.setter @is_strict_auto def trainer_desc_configs(self, configs): - check_configs_key(self.strategy.trainer_desc_configs, configs, - "trainer_desc_configs") + check_configs_key( + self.strategy.trainer_desc_configs, configs, "trainer_desc_configs" + ) assign_configs_value(self.strategy.trainer_desc_configs, configs) @property @@ -465,8 +483,9 @@ class DistributedStrategy(object): @fs_client_param.setter @is_strict_auto def fs_client_param(self, configs): - check_configs_key(self.strategy.fs_client_param, configs, - "fs_client_param") + check_configs_key( + self.strategy.fs_client_param, configs, "fs_client_param" + ) assign_configs_value(self.strategy.fs_client_param, configs) @property @@ -477,6 +496,7 @@ class DistributedStrategy(object): @is_strict_auto def sparse_table_configs(self, configs): from google.protobuf.descriptor import FieldDescriptor + table_param = self.strategy.downpour_table_param def set_table_config(msg, config_name, configs, index=0): @@ -493,8 +513,9 @@ class DistributedStrategy(object): data = getattr(msg, field.name).add() set_table_config(data, name, configs, i) else: - set_table_config(getattr(msg, field.name), name, - configs) + set_table_config( + getattr(msg, field.name), name, configs + ) else: # print("not message:", name) if name not in configs: @@ -513,132 +534,204 @@ class DistributedStrategy(object): for table_name in configs: table_data = table_param.add() table_data.table_name = table_name - set_table_config(table_data, "table_parameters." + table_name, - configs[table_name]) + set_table_config( + table_data, + "table_parameters." + table_name, + configs[table_name], + ) @sparse_table_configs.setter def fleet_desc_configs(self, configs): - support_sparse_key_list = ['sparse_table_class', 'sparse_compress_in_save', 'sparse_shard_num', \ - 'sparse_accessor_class', 'sparse_learning_rate', 'sparse_initial_g2sum', 'sparse_initial_range', \ - 'sparse_weight_bounds', 'sparse_fea_dim', 'sparse_embedx_dim', 'sparse_embedx_threshold', 'sparse_nonclk_coeff', \ - 'sparse_click_coeff', 'sparse_base_threshold', 'sparse_delta_threshold', 'sparse_delta_keep_days', \ - 'sparse_delete_after_unseen_days', 'sparse_show_click_decay_rate', 'sparse_delete_threshold', \ - 'sparse_converter', 'sparse_deconverter', 'sparse_enable_cache', 'sparse_cache_rate', \ - 'sparse_cache_file_num', 'sparse_beta1_decay_rate', 'sparse_beta2_decay_rate', \ - 'sparse_ada_epsilon', 'sparse_optimizer', 'sparse_ssd_unseenday_threshold', - 'embed_sparse_optimizer', 'embed_sparse_learning_rate', 'embed_sparse_weight_bounds', \ - 'embed_sparse_initial_range', 'embed_sparse_initial_g2sum', 'embed_sparse_beta1_decay_rate', \ - 'embed_sparse_beta2_decay_rate', 'embedx_sparse_optimizer', 'embedx_sparse_learning_rate', \ - 'embedx_sparse_weight_bounds', 'embedx_sparse_initial_range', 'embedx_sparse_initial_g2sum', \ - 'embedx_sparse_beta1_decay_rate', 'embedx_sparse_beta2_decay_rate', 'feature_learning_rate', 'nodeid_slot'] + support_sparse_key_list = [ + 'sparse_table_class', + 'sparse_compress_in_save', + 'sparse_shard_num', + 'sparse_accessor_class', + 'sparse_learning_rate', + 'sparse_initial_g2sum', + 'sparse_initial_range', + 'sparse_weight_bounds', + 'sparse_fea_dim', + 'sparse_embedx_dim', + 'sparse_embedx_threshold', + 'sparse_nonclk_coeff', + 'sparse_click_coeff', + 'sparse_base_threshold', + 'sparse_delta_threshold', + 'sparse_delta_keep_days', + 'sparse_delete_after_unseen_days', + 'sparse_show_click_decay_rate', + 'sparse_delete_threshold', + 'sparse_converter', + 'sparse_deconverter', + 'sparse_enable_cache', + 'sparse_cache_rate', + 'sparse_cache_file_num', + 'sparse_beta1_decay_rate', + 'sparse_beta2_decay_rate', + 'sparse_ada_epsilon', + 'sparse_optimizer', + 'sparse_ssd_unseenday_threshold', + 'embed_sparse_optimizer', + 'embed_sparse_learning_rate', + 'embed_sparse_weight_bounds', + 'embed_sparse_initial_range', + 'embed_sparse_initial_g2sum', + 'embed_sparse_beta1_decay_rate', + 'embed_sparse_beta2_decay_rate', + 'embedx_sparse_optimizer', + 'embedx_sparse_learning_rate', + 'embedx_sparse_weight_bounds', + 'embedx_sparse_initial_range', + 'embedx_sparse_initial_g2sum', + 'embedx_sparse_beta1_decay_rate', + 'embedx_sparse_beta2_decay_rate', + 'feature_learning_rate', + 'nodeid_slot', + ] support_sparse_table_class = ['DownpourSparseTable'] support_sparse_accessor_class = [ - 'DownpourSparseValueAccessor', 'DownpourCtrAccessor', - 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', - 'DownpourDoubleUnitAccessor', 'DownpourCtrDymfAccessor' + 'DownpourSparseValueAccessor', + 'DownpourCtrAccessor', + 'DownpourCtrDoubleAccessor', + 'DownpourUnitAccessor', + 'DownpourDoubleUnitAccessor', + 'DownpourCtrDymfAccessor', ] table_param = self.strategy.downpour_table_param def add_graph_config(graph, strategy): - graph.feature_learning_rate = strategy.get('feature_learning_rate', - 0.05) + graph.feature_learning_rate = strategy.get( + 'feature_learning_rate', 0.05 + ) graph.nodeid_slot = strategy.get('nodeid_slot', 9008) def sparse_optimizer_config(sgd, strategy, prefix): - optimizer_name = strategy.get(prefix + "sparse_optimizer", - "adagrad") + optimizer_name = strategy.get( + prefix + "sparse_optimizer", "adagrad" + ) sgd.name = optimizer_name if optimizer_name == "naive": sgd.name = "SparseNaiveSGDRule" sgd.naive.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.05) + prefix + 'sparse_learning_rate', 0.05 + ) sgd.naive.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_initial_range', 1e-4 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.naive.weight_bounds.extend(bounds) elif optimizer_name == "adagrad": sgd.name = 'SparseAdaGradSGDRule' sgd.adagrad.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.05) + prefix + 'sparse_learning_rate', 0.05 + ) sgd.adagrad.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) + prefix + 'sparse_initial_range', 1e-4 + ) if prefix == "embed_": sgd.adagrad.initial_range = 0 sgd.adagrad.initial_g2sum = strategy.get( - prefix + 'sparse_initial_g2sum', 3) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_initial_g2sum', 3 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.adagrad.weight_bounds.extend(bounds) elif optimizer_name == "std_adagrad": sgd.name = 'StdAdaGradSGDRule' sgd.adagrad.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.05) + prefix + 'sparse_learning_rate', 0.05 + ) sgd.adagrad.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) + prefix + 'sparse_initial_range', 1e-4 + ) if prefix == "embed_": sgd.adagrad.initial_range = 0 sgd.adagrad.initial_g2sum = strategy.get( - prefix + 'sparse_initial_g2sum', 3) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_initial_g2sum', 3 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.adagrad.weight_bounds.extend(bounds) elif optimizer_name == "adam": sgd.name = 'SparseAdamSGDRule' sgd.adam.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.001) + prefix + 'sparse_learning_rate', 0.001 + ) sgd.adam.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) + prefix + 'sparse_initial_range', 1e-4 + ) sgd.adam.beta1_decay_rate = strategy.get( - prefix + 'sparse_beta1_decay_rate', 0.9) + prefix + 'sparse_beta1_decay_rate', 0.9 + ) sgd.adam.beta2_decay_rate = strategy.get( - prefix + 'sparse_beta2_decay_rate', 0.999) + prefix + 'sparse_beta2_decay_rate', 0.999 + ) sgd.adam.ada_epsilon = strategy.get( - prefix + 'sparse_ada_epsilon', 1e-8) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_ada_epsilon', 1e-8 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.adam.weight_bounds.extend(bounds) elif optimizer_name == "shared_adam": sgd.name = 'SparseSharedAdamSGDRule' sgd.adam.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.001) + prefix + 'sparse_learning_rate', 0.001 + ) sgd.adam.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) + prefix + 'sparse_initial_range', 1e-4 + ) sgd.adam.beta1_decay_rate = strategy.get( - prefix + 'sparse_beta1_decay_rate', 0.9) + prefix + 'sparse_beta1_decay_rate', 0.9 + ) sgd.adam.beta2_decay_rate = strategy.get( - prefix + 'sparse_beta2_decay_rate', 0.999) + prefix + 'sparse_beta2_decay_rate', 0.999 + ) sgd.adam.ada_epsilon = strategy.get( - prefix + 'sparse_ada_epsilon', 1e-8) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_ada_epsilon', 1e-8 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.adam.weight_bounds.extend(bounds) def set_sparse_table_config(table_data, config): for key in config: if key not in support_sparse_key_list: raise ValueError("strategy key '%s' not support" % (key)) - table_class = config.get("sparse_table_class", - "DownpourSparseTable") + table_class = config.get( + "sparse_table_class", "DownpourSparseTable" + ) if table_class not in support_sparse_table_class: raise ValueError( "support sparse_table_class: ['DownpourSparseTable'], but actual %s" - % (table_class)) + % (table_class) + ) table_data.table_class = 'MemorySparseTable' table_data.shard_num = config.get('sparse_shard_num', 1000) table_data.enable_sparse_table_cache = config.get( - 'sparse_enable_cache', True) + 'sparse_enable_cache', True + ) table_data.sparse_table_cache_rate = config.get( - 'sparse_cache_rate', 0.00055) + 'sparse_cache_rate', 0.00055 + ) table_data.sparse_table_cache_file_num = config.get( - 'sparse_cache_file_num', 16) + 'sparse_cache_file_num', 16 + ) - accessor_class = config.get("sparse_accessor_class", - "DownpourCtrAccessor") + accessor_class = config.get( + "sparse_accessor_class", "DownpourCtrAccessor" + ) if accessor_class not in support_sparse_accessor_class: raise ValueError( "support sparse_accessor_class: ['DownpourSparseValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', 'DownpourDoubleUnitAccessor'], but actual %s" - % (accessor_class)) + % (accessor_class) + ) if accessor_class.find("Double") >= 0: table_data.accessor.accessor_class = 'CtrDoubleAccessor' @@ -653,7 +746,8 @@ class DistributedStrategy(object): table_data.accessor.embedx_dim = config.get('sparse_embedx_dim', 8) table_data.accessor.fea_dim = table_data.accessor.embedx_dim + 3 table_data.accessor.embedx_threshold = config.get( - 'sparse_embedx_threshold', 10) + 'sparse_embedx_threshold', 10 + ) if accessor_class == 'DownpourUnitAccessor': table_data.accessor.ctr_accessor_param.show_scale = False @@ -661,23 +755,32 @@ class DistributedStrategy(object): table_data.accessor.ctr_accessor_param.show_scale = True table_data.accessor.ctr_accessor_param.nonclk_coeff = config.get( - 'sparse_nonclk_coeff', 0.1) + 'sparse_nonclk_coeff', 0.1 + ) table_data.accessor.ctr_accessor_param.click_coeff = config.get( - 'sparse_click_coeff', 1) + 'sparse_click_coeff', 1 + ) table_data.accessor.ctr_accessor_param.base_threshold = config.get( - 'sparse_base_threshold', 1.5) + 'sparse_base_threshold', 1.5 + ) table_data.accessor.ctr_accessor_param.delta_threshold = config.get( - 'sparse_delta_threshold', 0.25) + 'sparse_delta_threshold', 0.25 + ) table_data.accessor.ctr_accessor_param.delta_keep_days = config.get( - 'sparse_delta_keep_days', 16) - table_data.accessor.ctr_accessor_param.show_click_decay_rate = config.get( - 'sparse_show_click_decay_rate', 0.98) - table_data.accessor.ctr_accessor_param.delete_threshold = config.get( - 'sparse_delete_threshold', 0.8) - table_data.accessor.ctr_accessor_param.delete_after_unseen_days = config.get( - 'sparse_delete_after_unseen_days', 30) - table_data.accessor.ctr_accessor_param.ssd_unseenday_threshold = config.get( - 'sparse_ssd_unseenday_threshold', 1) + 'sparse_delta_keep_days', 16 + ) + table_data.accessor.ctr_accessor_param.show_click_decay_rate = ( + config.get('sparse_show_click_decay_rate', 0.98) + ) + table_data.accessor.ctr_accessor_param.delete_threshold = ( + config.get('sparse_delete_threshold', 0.8) + ) + table_data.accessor.ctr_accessor_param.delete_after_unseen_days = ( + config.get('sparse_delete_after_unseen_days', 30) + ) + table_data.accessor.ctr_accessor_param.ssd_unseenday_threshold = ( + config.get('sparse_ssd_unseenday_threshold', 1) + ) converter = config.get('sparse_converter', "") deconverter = config.get('sparse_deconverter', "") @@ -691,23 +794,33 @@ class DistributedStrategy(object): save_data2.converter = converter save_data2.deconverter = deconverter - if accessor_class == 'DownpourCtrAccessor' or accessor_class == 'DownpourCtrDoubleAccessor': - sparse_optimizer_config(table_data.accessor.embed_sgd_param, - config, '') - sparse_optimizer_config(table_data.accessor.embedx_sgd_param, - config, '') + if ( + accessor_class == 'DownpourCtrAccessor' + or accessor_class == 'DownpourCtrDoubleAccessor' + ): + sparse_optimizer_config( + table_data.accessor.embed_sgd_param, config, '' + ) + sparse_optimizer_config( + table_data.accessor.embedx_sgd_param, config, '' + ) else: - sparse_optimizer_config(table_data.accessor.embed_sgd_param, - config, 'embed_') - sparse_optimizer_config(table_data.accessor.embedx_sgd_param, - config, 'embedx_') + sparse_optimizer_config( + table_data.accessor.embed_sgd_param, config, 'embed_' + ) + sparse_optimizer_config( + table_data.accessor.embedx_sgd_param, config, 'embedx_' + ) add_graph_config(table_data.accessor.graph_sgd_param, config) if not configs: print("fleet desc config is empty") else: for table_name in configs: - if table_name == 'dense_table' or table_name == 'datanorm_table': + if ( + table_name == 'dense_table' + or table_name == 'datanorm_table' + ): continue if type(configs[table_name]) != dict: continue @@ -1136,8 +1249,9 @@ class DistributedStrategy(object): @recompute_configs.setter @is_strict_auto def recompute_configs(self, configs): - check_configs_key(self.strategy.recompute_configs, configs, - "checkpoint_configs") + check_configs_key( + self.strategy.recompute_configs, configs, "checkpoint_configs" + ) assign_configs_value(self.strategy.recompute_configs, configs) @property @@ -1230,8 +1344,9 @@ class DistributedStrategy(object): @sharding_configs.setter @is_strict_auto def sharding_configs(self, configs): - check_configs_key(self.strategy.sharding_configs, configs, - "sharding_configs") + check_configs_key( + self.strategy.sharding_configs, configs, "sharding_configs" + ) assign_configs_value(self.strategy.sharding_configs, configs) @property @@ -1412,8 +1527,9 @@ class DistributedStrategy(object): @pipeline_configs.setter @is_strict_auto def pipeline_configs(self, configs): - check_configs_key(self.strategy.pipeline_configs, configs, - "pipeline_configs") + check_configs_key( + self.strategy.pipeline_configs, configs, "pipeline_configs" + ) assign_configs_value(self.strategy.pipeline_configs, configs) @property @@ -1467,8 +1583,11 @@ class DistributedStrategy(object): @tensor_parallel_configs.setter @is_strict_auto def tensor_parallel_configs(self, configs): - check_configs_key(self.strategy.tensor_parallel_configs, configs, - "tensor_parallel_configs") + check_configs_key( + self.strategy.tensor_parallel_configs, + configs, + "tensor_parallel_configs", + ) assign_configs_value(self.strategy.tensor_parallel_configs, configs) @property @@ -1501,8 +1620,9 @@ class DistributedStrategy(object): @hybrid_configs.setter def hybrid_configs(self, configs): - check_configs_key(self.strategy.hybrid_configs, configs, - "hybrid_configs") + check_configs_key( + self.strategy.hybrid_configs, configs, "hybrid_configs" + ) assign_configs_value(self.strategy.hybrid_configs, configs) @property @@ -1558,8 +1678,9 @@ class DistributedStrategy(object): @localsgd_configs.setter @is_strict_auto def localsgd_configs(self, configs): - check_configs_key(self.strategy.localsgd_configs, configs, - "localsgd_configs") + check_configs_key( + self.strategy.localsgd_configs, configs, "localsgd_configs" + ) assign_configs_value(self.strategy.localsgd_configs, configs) @property @@ -1617,8 +1738,11 @@ class DistributedStrategy(object): @adaptive_localsgd_configs.setter @is_strict_auto def adaptive_localsgd_configs(self, configs): - check_configs_key(self.strategy.adaptive_localsgd_configs, configs, - "adaptive_localsgd_configs") + check_configs_key( + self.strategy.adaptive_localsgd_configs, + configs, + "adaptive_localsgd_configs", + ) assign_configs_value(self.strategy.adaptive_localsgd_configs, configs) @property @@ -1762,8 +1886,9 @@ class DistributedStrategy(object): @gradient_merge_configs.setter @is_strict_auto def gradient_merge_configs(self, configs): - check_configs_key(self.strategy.gradient_merge_configs, configs, - "gradient_configs") + check_configs_key( + self.strategy.gradient_merge_configs, configs, "gradient_configs" + ) assign_configs_value(self.strategy.gradient_merge_configs, configs) @property @@ -2243,7 +2368,8 @@ class DistributedStrategy(object): h1_format = " " + "|{{:^{}s}}|\n".format(length) h2_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format( - max_k, " " * spacing, max_v) + max_k, " " * spacing, max_v + ) border = " +" + "".join(["="] * length) + "+" line = " +" + "".join(["-"] * length) + "+" @@ -2268,37 +2394,48 @@ class DistributedStrategy(object): if getattr(self.strategy, f.name): draws += border + "\n" draws += h1_format.format( - "{}=True <-> {}_configs".format(f.name, f.name)) + "{}=True <-> {}_configs".format(f.name, f.name) + ) draws += line + "\n" - my_configs = getattr(self.strategy, - f.name + "_configs") + my_configs = getattr( + self.strategy, f.name + "_configs" + ) config_fields = my_configs.DESCRIPTOR.fields for ff in config_fields: if isinstance( - getattr(my_configs, - ff.name), google.protobuf.pyext. - _message.RepeatedScalarContainer): + getattr(my_configs, ff.name), + google.protobuf.pyext._message.RepeatedScalarContainer, + ): values = getattr(my_configs, ff.name) for i, v in enumerate(values): if i == 0: draws += h2_format.format( - ff.name, str(v)) + ff.name, str(v) + ) else: draws += h2_format.format( - "", str(v)) + "", str(v) + ) else: draws += h2_format.format( ff.name, - str(getattr(my_configs, ff.name))) + str(getattr(my_configs, ff.name)), + ) else: env_draws += h2_format.format( - f.name, str(getattr(self.strategy, f.name))) + f.name, str(getattr(self.strategy, f.name)) + ) else: env_draws += h2_format.format( - f.name, str(getattr(self.strategy, f.name))) - - result_res = draws + border + "\n" + h1_format.format( - "Environment Flags, Communication Flags") + f.name, str(getattr(self.strategy, f.name)) + ) + + result_res = ( + draws + + border + + "\n" + + h1_format.format("Environment Flags, Communication Flags") + ) result_res += env_draws build_strategy_str = border + "\n" @@ -2308,7 +2445,8 @@ class DistributedStrategy(object): fields = self.strategy.build_strategy.DESCRIPTOR.fields for f in fields: build_strategy_str += h2_format.format( - f.name, str(getattr(self.strategy.build_strategy, f.name))) + f.name, str(getattr(self.strategy.build_strategy, f.name)) + ) build_strategy_str += border + "\n" execution_strategy_str = h1_format.format("Execution Strategy") @@ -2317,7 +2455,8 @@ class DistributedStrategy(object): fields = self.strategy.execution_strategy.DESCRIPTOR.fields for f in fields: execution_strategy_str += h2_format.format( - f.name, str(getattr(self.strategy.execution_strategy, f.name))) + f.name, str(getattr(self.strategy.execution_strategy, f.name)) + ) execution_strategy_str += border + "\n" result_res += build_strategy_str + execution_strategy_str diff --git a/python/paddle/distributed/fleet/base/meta_optimizer_factory.py b/python/paddle/distributed/fleet/base/meta_optimizer_factory.py index c2e55cfc38d..380a9b8c177 100755 --- a/python/paddle/distributed/fleet/base/meta_optimizer_factory.py +++ b/python/paddle/distributed/fleet/base/meta_optimizer_factory.py @@ -17,7 +17,8 @@ from ..meta_optimizers import * # noqa: F401, F403 __all__ = [] meta_optimizer_names = list( - filter(lambda name: name.endswith("Optimizer"), dir())) + filter(lambda name: name.endswith("Optimizer"), dir()) +) # Because HybridParallelOptimizer is dygraph optimizer, it # should be removed @@ -26,7 +27,6 @@ meta_optimizer_names.remove("HeterParallelOptimizer") class MetaOptimizerFactory(object): - def __init__(self): pass diff --git a/python/paddle/distributed/fleet/base/orthogonal_strategy.py b/python/paddle/distributed/fleet/base/orthogonal_strategy.py index d2ba6e4461b..d0fec2cfdb2 100644 --- a/python/paddle/distributed/fleet/base/orthogonal_strategy.py +++ b/python/paddle/distributed/fleet/base/orthogonal_strategy.py @@ -19,7 +19,7 @@ import paddle.distributed as dist from paddle.distributed.fleet.base.strategy_group import StrategyGroupBase -class OrthogonalStrategy(): +class OrthogonalStrategy: """ A hybrid of multiple distributed strategies. Strategies need to be orthogonal, means the ranks are organized like a square if there are two strategies, a cube if there aree three strategies, etc. @@ -57,8 +57,9 @@ class OrthogonalStrategy(): strategy[0] for strategy in list_of_strategy ] self._list_of_degree = [strategy[1] for strategy in list_of_strategy] - self._coordinate = collections.namedtuple('Coordinate', - self._list_of_strategy_name) + self._coordinate = collections.namedtuple( + 'Coordinate', self._list_of_strategy_name + ) self._check_valid_strategy() ranges = [range(degree) for degree in self._list_of_degree] @@ -66,14 +67,16 @@ class OrthogonalStrategy(): self._coordinate(*coord) for coord in itertools.product(*ranges) ] self._coord_to_rank_dict = dict( - zip(list_of_coord, range(len(list_of_coord)))) + zip(list_of_coord, range(len(list_of_coord))) + ) for idx, strategy in enumerate(list_of_strategy): strategy_name = strategy[0] self._name_to_degree_dict[strategy_name] = strategy[1] self._rank_list_dict[strategy_name] = self._calc_rank_list(idx) self._name_to_group_dict[strategy_name] = strategy[2]( - self._rank_list_dict[strategy_name]) + self._rank_list_dict[strategy_name] + ) self._name_to_fused_group_dict = {} self._create_fused_group() @@ -88,8 +91,9 @@ class OrthogonalStrategy(): Returns: An instance of specific strategy group. """ - assert name in self._list_of_strategy_name, "Strategy group {} is not created.".format( - name) + assert ( + name in self._list_of_strategy_name + ), "Strategy group {} is not created.".format(name) return self._name_to_group_dict[name] def fused_strategy_group(self, name): @@ -102,8 +106,9 @@ class OrthogonalStrategy(): Returns: (StrategyGroupBase): An instance of strategy group. """ - assert name in self._name_to_fused_group_dict, "Fused strategy group {} is not created.".format( - name) + assert ( + name in self._name_to_fused_group_dict + ), "Fused strategy group {} is not created.".format(name) return self._name_to_fused_group_dict[name] def rank_in_strategy(self, name): @@ -116,33 +121,42 @@ class OrthogonalStrategy(): Returns: (Integer): Local rank in specific strategy. """ - assert name in self._list_of_strategy_name, "Strategy group {} is not created.".format( - name) + assert ( + name in self._list_of_strategy_name + ), "Strategy group {} is not created.".format(name) return self._name_to_group_dict[name].group.rank def _check_valid_strategy(self): assert len(self._list_of_strategy_name) == len( set(self._list_of_strategy_name) ), "Defined duplicated strategies: {}".format(list_of_strategy) - num_of_ranks = functools.reduce(lambda x, y: x * y, - self._list_of_degree) - assert num_of_ranks == dist.get_world_size( + num_of_ranks = functools.reduce( + lambda x, y: x * y, self._list_of_degree + ) + assert ( + num_of_ranks == dist.get_world_size() ), "There are total {} ranks, but need {} ranks in this strategy.".format( - dist.get_world_size(), num_of_ranks) + dist.get_world_size(), num_of_ranks + ) for fused_strategy in self._fused_strategy_dict.values(): for strategy in fused_strategy: - assert strategy in self._list_of_strategy_name, "Can not fuse strategy {} without defined previous.".format( - strategy) + assert ( + strategy in self._list_of_strategy_name + ), "Can not fuse strategy {} without defined previous.".format( + strategy + ) def _create_fused_group(self): for name in self._fused_strategy_dict: fused_strategy = self._fused_strategy_dict[name] non_fused_strategy = list( - set(self._list_of_strategy_name).difference(fused_strategy)) + set(self._list_of_strategy_name).difference(fused_strategy) + ) non_fused_ranges = [] for strategy in non_fused_strategy: non_fused_ranges.append( - range(self._name_to_degree_dict[strategy])) + range(self._name_to_degree_dict[strategy]) + ) fused_ranges = [] for strategy in fused_strategy: fused_ranges.append(range(self._name_to_degree_dict[strategy])) @@ -156,8 +170,9 @@ class OrthogonalStrategy(): for fused_ranks in itertools.product(*fused_ranges): for i, fused_rank in enumerate(fused_ranks): coord_dict[fused_strategy[i]] = fused_rank - ranks.append(self._coord_to_rank_dict[self._coordinate( - **coord_dict)]) + ranks.append( + self._coord_to_rank_dict[self._coordinate(**coord_dict)] + ) rank_list.append(ranks) self._name_to_fused_group_dict[name] = StrategyGroupBase(rank_list) @@ -175,7 +190,8 @@ class OrthogonalStrategy(): coord_list = list(coord) coord_list.insert(strategy_axis, val) ranks.append( - self._coord_to_rank_dict[self._coordinate(*coord_list)]) + self._coord_to_rank_dict[self._coordinate(*coord_list)] + ) rank_list.append(ranks) return rank_list diff --git a/python/paddle/distributed/fleet/base/private_helper_function.py b/python/paddle/distributed/fleet/base/private_helper_function.py index 2745f398152..b6714606589 100644 --- a/python/paddle/distributed/fleet/base/private_helper_function.py +++ b/python/paddle/distributed/fleet/base/private_helper_function.py @@ -39,8 +39,9 @@ def wait_server_ready(endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): @@ -52,8 +53,9 @@ def wait_server_ready(endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + str(not_ready_endpoints) + - "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: diff --git a/python/paddle/distributed/fleet/base/role_maker.py b/python/paddle/distributed/fleet/base/role_maker.py index a0b39c3b7a3..28dace611a0 100755 --- a/python/paddle/distributed/fleet/base/role_maker.py +++ b/python/paddle/distributed/fleet/base/role_maker.py @@ -20,7 +20,9 @@ from multiprocessing import Process, Manager import paddle import paddle.fluid as fluid -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) __all__ = [] @@ -49,10 +51,13 @@ class Gloo(object): self._nodes_comm = None self._comm_world = ["worker", "server", "all"] - self._err_init = "gloo is not initialized, will not communicator with other nodes" + self._err_init = ( + "gloo is not initialized, will not communicator with other nodes" + ) self._err_type = "gloo initialized error, please check arguments" self._err_world = "argument error, comm_world must in {}".format( - self._comm_world) + self._comm_world + ) self._is_initialized = False self._init_timeout_seconds = 3600 @@ -67,14 +72,16 @@ class Gloo(object): self._server_num = -1 self._need_init_all = False - def init(self, - rendezvous, - role, - role_id, - worker_num, - server_num, - need_init_all=False, - kwargs=None): + def init( + self, + rendezvous, + role, + role_id, + worker_num, + server_num, + need_init_all=False, + kwargs=None, + ): self._rendezvous = rendezvous self._role = role @@ -110,8 +117,9 @@ class Gloo(object): if not ip or not port: raise ValueError(self._err_type) - http_server = self._init_http(ip, port, self._prefix, - start_http_server, http_server_d) + http_server = self._init_http( + ip, port, self._prefix, start_http_server, http_server_d + ) else: raise ValueError(self._err_type) @@ -119,15 +127,15 @@ class Gloo(object): self._http_server = http_server def _init_fs(self, fs_path, prefix): - def init(rank, nodes, role): gloo = fluid.core.Gloo() gloo.set_rank(rank) gloo.set_size(nodes) gloo.set_prefix(prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store(os.path.join(fs_path, role), "", "") gloo.init() return gloo @@ -147,15 +155,15 @@ class Gloo(object): self._nodes_comm = gloo def _init_dfs(self, dfs_name, dfs_ugi, dfs_path, prefix): - def init(rank, nodes, role): gloo = fluid.core.Gloo() gloo.set_rank(rank) gloo.set_size(nodes) gloo.set_prefix(prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store(os.path.join(dfs_path, role), dfs_name, dfs_ugi) gloo.init() return gloo @@ -175,15 +183,17 @@ class Gloo(object): self._nodes_comm = gloo def _init_http(self, ip, port, prefix, start_http_server, http_server_d): - def __start_kv_server(http_server_d, size_d): print("start http_server: {}, {}".format(port, size_d)) from paddle.distributed.fleet.utils.http_server import KVServer + http_server = KVServer(port, size_d) http_server.start() wait_seconds = 5 - while http_server_d.get("running", - False) or not http_server.should_stop(): + while ( + http_server_d.get("running", False) + or not http_server.should_stop() + ): time.sleep(wait_seconds) http_server.stop() @@ -196,8 +206,9 @@ class Gloo(object): http_server_d["running"] = True # child process for http server - _http_server = Process(target=__start_kv_server, - args=(http_server_d, size_d)) + _http_server = Process( + target=__start_kv_server, args=(http_server_d, size_d) + ) _http_server.daemon = True # set running status to True # start child process @@ -210,8 +221,9 @@ class Gloo(object): gloo.set_size(nodes) gloo.set_prefix(prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_http_store(ip, port, 'worker') ep = ":".join([ip, str(port)]) wait_server_ready([ep]) @@ -285,8 +297,12 @@ class Gloo(object): gateway = None if len(item) > gateway_idx: gateway = item[gateway_idx] - if gateway and gateway != '*' and gateway != "0.0.0.0" and len( - item) > iface_idx: + if ( + gateway + and gateway != '*' + and gateway != "0.0.0.0" + and len(item) > iface_idx + ): return item[iface_idx] return "lo" @@ -294,8 +310,9 @@ class Gloo(object): """ get default physical interface """ - res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( - "\n") + res = ( + os.popen("ip -f inet addr | awk NR%3==1").read().strip().split("\n") + ) for item in res: if "BROADCAST" in item: return item.split(":")[1].strip() @@ -470,8 +487,11 @@ class RoleMakerBase(object): def to_string(self): return "role: {}, current_id: {}, worker_endpoints: {}, server_endpoints: {}".format( - self._role, self._current_id, self._worker_endpoints, - self._server_endpoints) + self._role, + self._current_id, + self._worker_endpoints, + self._server_endpoints, + ) def _all_gather(self, input, comm_world="worker"): print("warning: RoleMakerBase does not have all gather worker.") @@ -493,13 +513,13 @@ class RoleMakerBase(object): """ print("warning: RoleMakerBase does not have barrier worker.") - #def _is_heter_worker(self): + # def _is_heter_worker(self): # """ # Return is_heter_worker() of current process # """ # raise NotImplementedError("Please implement this method in child class") - #def _heter_worker_num(self): + # def _heter_worker_num(self): # """ # Get current total heter-worker number. # @@ -508,14 +528,14 @@ class RoleMakerBase(object): # """ # raise NotImplementedError("Please implement this method in child class") - #def _get_heter_worker_endpoints(self): + # def _get_heter_worker_endpoints(self): # """ # Returns: # string: all heter_trainers'endpoints # """ # raise NotImplementedError("Please implement this method in child class") - #def _get_heter_worker_endpoint(self): + # def _get_heter_worker_endpoint(self): # """ # Returns: # int: corresponding heter_trainer's endpoint @@ -524,7 +544,6 @@ class RoleMakerBase(object): class PaddleCloudRoleMaker(RoleMakerBase): - def __init__(self, is_collective=False, **kwargs): super(PaddleCloudRoleMaker, self).__init__() self._is_collective = is_collective @@ -578,24 +597,24 @@ class PaddleCloudRoleMaker(RoleMakerBase): def _get_stage_id(self): """ - return stage id of current heter worker - """ + return stage id of current heter worker + """ if not self._role_is_generated: self._generate_role() return self._stage_id def _get_stage_trainers(self): """ - return trainer num of all stages - """ + return trainer num of all stages + """ if not self._role_is_generated: self._generate_role() return self._stage_trainers def _get_num_stage(self): """ - return stage num - """ + return stage num + """ if not self._role_is_generated: self._generate_role() return self._stage_num @@ -667,8 +686,11 @@ class PaddleCloudRoleMaker(RoleMakerBase): """ if not self._role_is_generated: self._generate_role() - return len(self._get_pserver_endpoints() - ) if self._get_pserver_endpoints() is not None else 0 + return ( + len(self._get_pserver_endpoints()) + if self._get_pserver_endpoints() is not None + else 0 + ) def _node_num(self): """ @@ -712,7 +734,9 @@ class PaddleCloudRoleMaker(RoleMakerBase): def _get_trainer_endpoint(self): if not self._role_is_generated: self._generate_role() - assert self._role == Role.WORKER, "get_trainer_endpoint should be called by trainer" + assert ( + self._role == Role.WORKER + ), "get_trainer_endpoint should be called by trainer" return self._cur_endpoint def _get_heter_worker_endpoints(self): @@ -722,7 +746,9 @@ class PaddleCloudRoleMaker(RoleMakerBase): """ if not self._role_is_generated: self._generate_role() - assert self._heter_trainer_endpoints != [], "Heter Worker Endpoints Not initialized" + assert ( + self._heter_trainer_endpoints != [] + ), "Heter Worker Endpoints Not initialized" return self._heter_trainer_endpoints def _get_heter_worker_endpoint(self): @@ -732,7 +758,9 @@ class PaddleCloudRoleMaker(RoleMakerBase): """ if not self._role_is_generated: self._generate_role() - assert self._role == Role.HETER_WORKER, "_get_heter_worker_endpoint should be invoked by heter worker" + assert ( + self._role == Role.HETER_WORKER + ), "_get_heter_worker_endpoint should be invoked by heter worker" return self._cur_endpoint def _get_pserver_endpoints(self): @@ -755,7 +783,8 @@ class PaddleCloudRoleMaker(RoleMakerBase): if not self._role_is_generated: self._generate_role() assert self._role in ( - Role.WORKER, Role.HETER_WORKER + Role.WORKER, + Role.HETER_WORKER, ), "_get_previous_trainers should be invoked by trainer or heter worker" return self._previous_heter_trainer_endpoints @@ -766,7 +795,8 @@ class PaddleCloudRoleMaker(RoleMakerBase): if not self._role_is_generated: self._generate_role() assert self._role in ( - Role.WORKER, Role.HETER_WORKER + Role.WORKER, + Role.HETER_WORKER, ), "_get_next_trainers should be invoked by trainer or heter worker" return self._next_heter_trainer_endpoints @@ -820,8 +850,9 @@ class PaddleCloudRoleMaker(RoleMakerBase): else: self._worker_endpoints = [] - self._coordinator_endpoints = os.getenv("PADDLE_COORDINATOR_ENDPOINTS", - "") + self._coordinator_endpoints = os.getenv( + "PADDLE_COORDINATOR_ENDPOINTS", "" + ) if self._coordinator_endpoints == "": print("fl-ps > coordinator address is null!") else: @@ -838,22 +869,31 @@ class PaddleCloudRoleMaker(RoleMakerBase): training_role = os.getenv("TRAINING_ROLE", None) if training_role == None: raise ValueError( - "Can not find TRAINING_ROLE, please check your environment.") + "Can not find TRAINING_ROLE, please check your environment." + ) if training_role not in [ - "TRAINER", "PSERVER", "HETER_TRAINER", "COORDINATOR" + "TRAINER", + "PSERVER", + "HETER_TRAINER", + "COORDINATOR", ]: raise ValueError( - "TRAINING_ROLE must be PSERVER or TRAINER or HETER_TRAINER or COORDINATOR, but get {}, please check your environment." - .format(training_role)) + "TRAINING_ROLE must be PSERVER or TRAINER or HETER_TRAINER or COORDINATOR, but get {}, please check your environment.".format( + training_role + ) + ) # For Heter Parameter Server env setting next_heter_trainer_eplist = os.getenv( - "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST", "") + "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST", "" + ) previous_heter_trainer_eplist = os.getenv( - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST", "") + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST", "" + ) all_heter_trainer_eplist = os.getenv( - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST", "") + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST", "" + ) if all_heter_trainer_eplist != "": self._heter_trainer_endpoints = all_heter_trainer_eplist.split(",") @@ -863,11 +903,13 @@ class PaddleCloudRoleMaker(RoleMakerBase): if previous_heter_trainer_eplist == "": assert training_role in ( "TRAINER", - "PSERVER"), "training_role should be trainer or pserver" + "PSERVER", + ), "training_role should be trainer or pserver" else: try: - self._previous_heter_trainer_endpoints = previous_heter_trainer_eplist.split( - ",") + self._previous_heter_trainer_endpoints = ( + previous_heter_trainer_eplist.split(",") + ) except: raise ValueError( "Can not Find PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST in env or its format doesn't match the requirement: 'IP:PORT,IP:PORT' ." @@ -875,12 +917,14 @@ class PaddleCloudRoleMaker(RoleMakerBase): if next_heter_trainer_eplist == "": assert training_role in ( - "HETER_TRAINER", "PSERVER" + "HETER_TRAINER", + "PSERVER", ), "training_role should be heter trainer or pserver" else: try: - self._next_heter_trainer_endpoints = next_heter_trainer_eplist.split( - ",") + self._next_heter_trainer_endpoints = ( + next_heter_trainer_eplist.split(",") + ) except: raise ValueError( "Can not Find PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST in env or its format doesn't match the requirement: 'IP:PORT,IP:PORT' ." @@ -902,7 +946,8 @@ class PaddleCloudRoleMaker(RoleMakerBase): self._stage_id = os.getenv("STAGE_ID", None) if self._stage_id == None: raise ValueError( - "Can not find STAGE_ID, please check your environment.") + "Can not find STAGE_ID, please check your environment." + ) self._stage_id = int(self._stage_id) self._stage_num = os.getenv("STAGE_NUM", None) if self._stage_num == None: @@ -910,8 +955,9 @@ class PaddleCloudRoleMaker(RoleMakerBase): "Can not find STAGE_NUM, please check your environment." ) self._stage_num = int(self._stage_num) - self._stage_trainers = os.getenv("PADDLE_STAGE_TRAINERS_NUM", - None) + self._stage_trainers = os.getenv( + "PADDLE_STAGE_TRAINERS_NUM", None + ) if self._stage_trainers == None: raise ValueError( "Can not find PADDLE_STAGE_TRAINERS_NUM, please check your environment." @@ -920,11 +966,13 @@ class PaddleCloudRoleMaker(RoleMakerBase): cur_port = os.getenv("PADDLE_PORT", None) if cur_port == None: raise ValueError( - "Can not find PADDLE_PORT, please check your environment.") + "Can not find PADDLE_PORT, please check your environment." + ) cur_ip = os.getenv("POD_IP", None) if cur_ip == None: raise ValueError( - "Can not find POD_IP, please check your environment.") + "Can not find POD_IP, please check your environment." + ) curr_endpoint = ":".join([cur_ip, cur_port]) self._cur_endpoint = curr_endpoint elif training_role == "COORDINATOR": @@ -936,11 +984,13 @@ class PaddleCloudRoleMaker(RoleMakerBase): cur_port = os.getenv("PADDLE_PORT", None) if cur_port == None: raise ValueError( - "Can not find PADDLE_PORT, please check your environment.") + "Can not find PADDLE_PORT, please check your environment." + ) cur_ip = os.getenv("POD_IP", None) if cur_ip == None: raise ValueError( - "Can not find POD_IP, please check your environment.") + "Can not find POD_IP, please check your environment." + ) curr_endpoint = ":".join([cur_ip, cur_port]) self._cur_endpoint = curr_endpoint current_id = self._server_endpoints.index(self._cur_endpoint) @@ -949,12 +999,14 @@ class PaddleCloudRoleMaker(RoleMakerBase): self._stage_id = os.getenv("STAGE_ID", None) if self._stage_id == None: raise ValueError( - "Can not find STAGE_ID, please check your environment.") + "Can not find STAGE_ID, please check your environment." + ) self._stage_id = int(self._stage_id) self._stage_num = os.getenv("STAGE_NUM", None) if self._stage_num == None: raise ValueError( - "Can not find STAGE_NUM, please check your environment.") + "Can not find STAGE_NUM, please check your environment." + ) self._stage_num = int(self._stage_num) self._stage_trainers = os.getenv("PADDLE_STAGE_TRAINERS_NUM", None) @@ -964,47 +1016,57 @@ class PaddleCloudRoleMaker(RoleMakerBase): ) self._stage_trainers = eval(self._stage_trainers) - self._heter_trainer_device_type = os.getenv("HETER_DEVICE_TYPE", - None) + self._heter_trainer_device_type = os.getenv( + "HETER_DEVICE_TYPE", None + ) if self._heter_trainer_device_type == None: raise ValueError( "Can not find HETER_DEVICE_TYPE, please check your environment." ) assert self._heter_trainer_device_type in ( - "cpu", "gpu", - "xpu"), "HETER_DEVICE_TYPE should be cpu,gpu or xpu" + "cpu", + "gpu", + "xpu", + ), "HETER_DEVICE_TYPE should be cpu,gpu or xpu" if self._heter_trainer_device_type == "gpu": heter_device_id = os.getenv("FLAGS_selected_gpus", "0") self._heter_trainer_device = ":".join( - (self._heter_trainer_device_type, heter_device_id)) + (self._heter_trainer_device_type, heter_device_id) + ) if self._heter_trainer_device == "xpu": heter_device_id = os.getenv("FLAGS_selected_xpus", "0") self._heter_trainer_device = ":".join( - (self._heter_trainer_device_type, heter_device_id)) + (self._heter_trainer_device_type, heter_device_id) + ) cur_port = os.getenv("PADDLE_PORT", None) if cur_port == None: raise ValueError( - "Can not find PADDLE_PORT, please check your environment.") + "Can not find PADDLE_PORT, please check your environment." + ) cur_ip = os.getenv("POD_IP", None) if cur_ip == None: raise ValueError( - "Can not find POD_IP, please check your environment.") + "Can not find POD_IP, please check your environment." + ) curr_endpoint = ":".join([cur_ip, cur_port]) self._cur_endpoint = curr_endpoint - current_id = all_heter_trainer_eplist.split(",").index( - curr_endpoint) + trainers_num + current_id = ( + all_heter_trainer_eplist.split(",").index(curr_endpoint) + + trainers_num + ) self._trainers_num = trainers_num self._role = role self._current_id = current_id self._nodes_num = len( - set([x.split(':')[0] for x in self._worker_endpoints])) + set([x.split(':')[0] for x in self._worker_endpoints]) + ) def _collective_env(self): self._current_id = int(os.getenv("PADDLE_TRAINER_ID", "0")) self._training_role = os.getenv("PADDLE_TRAINING_ROLE", "TRAINER") - assert (self._training_role == "TRAINER") + assert self._training_role == "TRAINER" self._role = Role.WORKER self._worker_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS") self._cur_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT") @@ -1016,7 +1078,8 @@ class PaddleCloudRoleMaker(RoleMakerBase): self._worker_endpoints = self._worker_endpoints.split(",") self._trainers_num = len(self._worker_endpoints) self._nodes_num = len( - set([x.split(':')[0] for x in self._worker_endpoints])) + set([x.split(':')[0] for x in self._worker_endpoints]) + ) self._local_rank = os.getenv("PADDLE_RANK_IN_NODE") self._local_device_ids = os.getenv("PADDLE_LOCAL_DEVICE_IDS") self._world_device_ids = os.getenv("PADDLE_WORLD_DEVICE_IDS") @@ -1031,7 +1094,9 @@ class PaddleCloudRoleMaker(RoleMakerBase): rendezvous_type = int(os.getenv("PADDLE_GLOO_RENDEZVOUS", "0")) prefix = os.getenv("SYS_JOB_ID", "") if rendezvous_type not in [ - Gloo.RENDEZVOUS.HDFS, Gloo.RENDEZVOUS.HTTP, Gloo.RENDEZVOUS.FILE + Gloo.RENDEZVOUS.HDFS, + Gloo.RENDEZVOUS.HTTP, + Gloo.RENDEZVOUS.FILE, ]: raise ValueError(self._gloo._err_type) @@ -1081,16 +1146,21 @@ class PaddleCloudRoleMaker(RoleMakerBase): type = "HTTP" else: type = "FILE" - print("Gloo init with {}: need_init_all: {}, args: {}".format( - type, need_init_all, kwargs)) - - self._gloo.init(rendezvous=rendezvous_type, - role=self._role, - role_id=self._role_id(), - worker_num=self._worker_num(), - server_num=self._server_num(), - need_init_all=need_init_all, - kwargs=kwargs) + print( + "Gloo init with {}: need_init_all: {}, args: {}".format( + type, need_init_all, kwargs + ) + ) + + self._gloo.init( + rendezvous=rendezvous_type, + role=self._role, + role_id=self._role_id(), + worker_num=self._worker_num(), + server_num=self._server_num(), + need_init_all=need_init_all, + kwargs=kwargs, + ) if rendezvous_type == Gloo.RENDEZVOUS.HTTP: http_server_d['running'] = False @@ -1110,11 +1180,10 @@ class PaddleCloudRoleMaker(RoleMakerBase): class UserDefinedRoleMaker(PaddleCloudRoleMaker): - def __init__(self, is_collective=False, init_gloo=False, **kwargs): - super(UserDefinedRoleMaker, self).__init__(is_collective=is_collective, - init_gloo=init_gloo, - **kwargs) + super(UserDefinedRoleMaker, self).__init__( + is_collective=is_collective, init_gloo=init_gloo, **kwargs + ) self._init_gloo = init_gloo def _user_defined_ps_env(self): @@ -1123,19 +1192,22 @@ class UserDefinedRoleMaker(PaddleCloudRoleMaker): self._trainers_num = self._kwargs.get("worker_num", 0) if self._trainers_num == 0: - assert (len(self._worker_endpoints) > 0) + assert len(self._worker_endpoints) > 0 self._trainers_num = len(self._worker_endpoints) self._role = self._kwargs.get("role") self._current_id = self._kwargs.get("current_id") - if self._role == Role.WORKER and len( - self._worker_endpoints) > self._current_id: + if ( + self._role == Role.WORKER + and len(self._worker_endpoints) > self._current_id + ): self._cur_endpoint = self._worker_endpoints[self._current_id] elif self._role == Role.SERVER: self._cur_endpoint = self._server_endpoints[self._current_id] self._nodes_num = len( - set([x.split(':')[0] for x in self._worker_endpoints])) + set([x.split(':')[0] for x in self._worker_endpoints]) + ) def _user_defined_collective_env(self): self._worker_endpoints = self._kwargs.get("worker_endpoints") @@ -1143,7 +1215,8 @@ class UserDefinedRoleMaker(PaddleCloudRoleMaker): self._trainers_num = len(self._worker_endpoints) self._training_role = Role.WORKER self._nodes_num = len( - set([x.split(':')[0] for x in self._worker_endpoints])) + set([x.split(':')[0] for x in self._worker_endpoints]) + ) def _generate_role(self): """ diff --git a/python/paddle/distributed/fleet/base/runtime_factory.py b/python/paddle/distributed/fleet/base/runtime_factory.py index 3fa7dbb285c..e9006409674 100644 --- a/python/paddle/distributed/fleet/base/runtime_factory.py +++ b/python/paddle/distributed/fleet/base/runtime_factory.py @@ -18,7 +18,6 @@ __all__ = [] class RuntimeFactory(object): - def __init__(self): pass diff --git a/python/paddle/distributed/fleet/base/strategy_compiler.py b/python/paddle/distributed/fleet/base/strategy_compiler.py index fa9dc1e0cdc..14db852f770 100644 --- a/python/paddle/distributed/fleet/base/strategy_compiler.py +++ b/python/paddle/distributed/fleet/base/strategy_compiler.py @@ -107,7 +107,6 @@ def maximum_path_len_algo(optimizer_list): class StrategyCompilerBase(object): - def __init__(self): pass @@ -143,6 +142,7 @@ class StrategyCompiler(StrategyCompilerBase): def _get_valid_strategy(self, dist_strategy, can_not_apply_optimizer_list): import copy + valid_strategy = copy.deepcopy(dist_strategy) invalid_optimizers = [] for candidate in self._meta_optimizer_candidates: @@ -171,9 +171,15 @@ class StrategyCompiler(StrategyCompilerBase): we will remove grad fusion and sync batch-norm """ - def generate_optimizer(self, loss, role_maker, optimizer, - user_defined_strategy, meta_optimizer_list, - graph_optimizer_list): + def generate_optimizer( + self, + loss, + role_maker, + optimizer, + user_defined_strategy, + meta_optimizer_list, + graph_optimizer_list, + ): self._user_defined_strategy = user_defined_strategy self._meta_optimizer_candidates = meta_optimizer_list self._graph_optimizer_candidates = graph_optimizer_list @@ -190,12 +196,19 @@ class StrategyCompiler(StrategyCompilerBase): # and graph_optimizer, the corresponding distributed strategy # should be updated. - self._meta_optimizers = [] if meta_optimizers is None else meta_optimizers - self._graph_optimizers = [] if graph_optimizers is None else graph_optimizers + self._meta_optimizers = ( + [] if meta_optimizers is None else meta_optimizers + ) + self._graph_optimizers = ( + [] if graph_optimizers is None else graph_optimizers + ) - return_meta = None if meta_optimizers == None else meta_optimizers[0] - return_graph = None if graph_optimizers == None else graph_optimizers[ - 0] + return_meta = ( + None if meta_optimizers == None else meta_optimizers[0] + ) + return_graph = ( + None if graph_optimizers == None else graph_optimizers[0] + ) if meta_optimizers == None or graph_optimizers == None: return return_meta, return_graph @@ -205,7 +218,10 @@ class StrategyCompiler(StrategyCompilerBase): need_graph_opt = True for graph_opt in graph_optimizers: for program_opt in meta_optimizers: - if graph_opt.__class__.__name__ in program_opt.meta_optimizers_black_list: + if ( + graph_opt.__class__.__name__ + in program_opt.meta_optimizers_black_list + ): need_graph_opt = False if not need_graph_opt: return_graph = None diff --git a/python/paddle/distributed/fleet/base/strategy_group.py b/python/paddle/distributed/fleet/base/strategy_group.py index 94ab04b6c4e..5b0ac3cfe0e 100644 --- a/python/paddle/distributed/fleet/base/strategy_group.py +++ b/python/paddle/distributed/fleet/base/strategy_group.py @@ -15,7 +15,7 @@ import paddle.distributed as dist -class StrategyGroupBase(): +class StrategyGroupBase: """ The base class of communication group with distributed strategy. @@ -39,7 +39,8 @@ class StrategyGroupBase(): """ def __init__(self, list_of_ranks): - assert dist.is_initialized( + assert ( + dist.is_initialized() ), "The global communication group need to be initialized." assert len(list_of_ranks), "The list_of_ranks can not be empty." self._rank = dist.get_rank() @@ -57,8 +58,9 @@ class StrategyGroupBase(): world_size_list = [] for ranks in self._list_of_ranks: world_size_list.append(len(ranks)) - is_value = all(world_size == world_size_list[0] - for world_size in world_size_list) + is_value = all( + world_size == world_size_list[0] for world_size in world_size_list + ) return world_size_list[0] if is_value else world_size_list @property @@ -77,10 +79,11 @@ class StrategyGroupBase(): group = dist.new_group(ranks=ranks) if self._rank in ranks: list_of_group.append(group) - assert len( - list_of_group - ) > 0, "Rank {} does not belong to the list_of_ranks {}.".format( - self._rank, self._list_of_ranks) + assert ( + len(list_of_group) > 0 + ), "Rank {} does not belong to the list_of_ranks {}.".format( + self._rank, self._list_of_ranks + ) return list_of_group if len(list_of_group) > 1 else list_of_group[0] @@ -99,8 +102,8 @@ class DPGroup(StrategyGroupBase): def __init__(self, list_of_ranks): super(DPGroup, self).__init__(list_of_ranks) assert not isinstance( - self.group, list), "Rank {} belongs to multi dp groups".format( - self._rank) + self.group, list + ), "Rank {} belongs to multi dp groups".format(self._rank) class MPGroup(StrategyGroupBase): @@ -118,8 +121,8 @@ class MPGroup(StrategyGroupBase): def __init__(self, list_of_ranks): super(MPGroup, self).__init__(list_of_ranks) assert not isinstance( - self.group, list), "Rank {} belongs to multi mp groups".format( - self._rank) + self.group, list + ), "Rank {} belongs to multi mp groups".format(self._rank) class ShardingGroup(StrategyGroupBase): @@ -137,8 +140,8 @@ class ShardingGroup(StrategyGroupBase): def __init__(self, list_of_ranks): super(ShardingGroup, self).__init__(list_of_ranks) assert not isinstance( - self.group, - list), "Rank {} belongs to multi sharding groups".format(self._rank) + self.group, list + ), "Rank {} belongs to multi sharding groups".format(self._rank) class PPGroup(StrategyGroupBase): @@ -156,8 +159,8 @@ class PPGroup(StrategyGroupBase): def __init__(self, list_of_ranks): super(PPGroup, self).__init__(list_of_ranks) assert not isinstance( - self.group, list), "Rank {} belongs to multi pp groups".format( - self._rank) + self.group, list + ), "Rank {} belongs to multi pp groups".format(self._rank) self._send_next_group = None self._send_prev_group = None @@ -197,7 +200,12 @@ class PPGroup(StrategyGroupBase): Returns: Four subgroups including send/recv to/from prev/next. """ - return self._send_next_group, self._send_prev_group, self._recv_next_group, self._recv_prev_group + return ( + self._send_next_group, + self._send_prev_group, + self._recv_next_group, + self._recv_prev_group, + ) def _create_p2p_group(self): degree = self.world_size @@ -223,5 +231,11 @@ class PPGroup(StrategyGroupBase): elif self._rank == prev_rank: self._recv_next_group = prev_group - assert self._send_next_group and self._send_prev_group and self._recv_next_group and self._recv_prev_group,\ - "Error occurs while creating p2p group for rank {}.".format(self._rank) + assert ( + self._send_next_group + and self._send_prev_group + and self._recv_next_group + and self._recv_prev_group + ), "Error occurs while creating p2p group for rank {}.".format( + self._rank + ) diff --git a/python/paddle/distributed/fleet/base/topology.py b/python/paddle/distributed/fleet/base/topology.py index d679894d3e2..6fa1521d58c 100644 --- a/python/paddle/distributed/fleet/base/topology.py +++ b/python/paddle/distributed/fleet/base/topology.py @@ -40,6 +40,7 @@ class ParallelMode(object): print(parallel_mode.DATA_PARALLEL) # 0 """ + DATA_PARALLEL = 0 TENSOR_PARALLEL = 1 PIPELINE_PARALLEL = 2 @@ -47,14 +48,16 @@ class ParallelMode(object): class CommunicateTopology(object): - - def __init__(self, - hybrid_group_names=["data", "pipe", "sharding", "model"], - dims=[1, 1, 1, 1]): + def __init__( + self, + hybrid_group_names=["data", "pipe", "sharding", "model"], + dims=[1, 1, 1, 1], + ): self._parallel_names = hybrid_group_names self._dims = dims - self.coordinate = collections.namedtuple('Coordinate', - self._parallel_names) + self.coordinate = collections.namedtuple( + 'Coordinate', self._parallel_names + ) self._world_size = reduce(lambda x, y: x * y, self._dims) ranges = [range(d) for d in self._dims] @@ -62,7 +65,8 @@ class CommunicateTopology(object): self._coord2rank = dict(zip(all_coordinate, range(len(all_coordinate)))) self._rank2coord = dict( - zip(self._coord2rank.values(), self._coord2rank.keys())) + zip(self._coord2rank.values(), self._coord2rank.keys()) + ) def get_hybrid_group_names(self): return self._parallel_names @@ -87,7 +91,8 @@ class CommunicateTopology(object): def get_axis_list(self, axis_name, index): axis = self._parallel_names.index(axis_name) ranks = [ - self._coord2rank[coord] for coord in self._coord2rank.keys() + self._coord2rank[coord] + for coord in self._coord2rank.keys() if coord[axis] == index ] ranks.sort() @@ -129,7 +134,6 @@ class CommunicateTopology(object): class HybridCommunicateGroup(object): - def __init__(self, topology): self.nranks = paddle.distributed.get_world_size() self.global_rank = paddle.distributed.get_rank() @@ -145,10 +149,16 @@ class HybridCommunicateGroup(object): self._sharding_parallel_id = self._get_sharding_parallel_id() self.stage_id = self._get_pipe_parallel_id() - assert self._check_vaild_topo( - ), "Here is an unreasonable topogy setting. world_size: {}, but" \ - "mp_num: {}, sharding_num: {}, pp_num: {}, dp_num: {}".format(self.nranks, - self._mp_degree, self._sharding_degree, self._pp_degree, self._dp_degree) + assert self._check_vaild_topo(), ( + "Here is an unreasonable topogy setting. world_size: {}, but" + "mp_num: {}, sharding_num: {}, pp_num: {}, dp_num: {}".format( + self.nranks, + self._mp_degree, + self._sharding_degree, + self._pp_degree, + self._dp_degree, + ) + ) # create comm group for data parallel self._dp_group, self._dp_comm_group = self._set_comm_group("data") @@ -161,26 +171,43 @@ class HybridCommunicateGroup(object): # create comm group for sharding parallel self._sharding_group, self._sharding_comm_group = self._set_comm_group( - "sharding") + "sharding" + ) # create global group for check inf_nan / clip global norm self._check_group, self._check_comm_group = self._set_check_group( - "data") + "data" + ) # create p2p group - self.is_first_stage = (self.stage_id == 0) - self.is_last_stage = (self.stage_id == (self._pp_degree - 1)) + self.is_first_stage = self.stage_id == 0 + self.is_last_stage = self.stage_id == (self._pp_degree - 1) # create p2p_groups if self._pp_degree > 1: self._set_p2p_group() - debug_str = "HybridParallelInfo: rank_id: %d, mp_degree: %d, " \ - "sharding_degree: %d, pp_degree: %d, dp_degree: %d" % (self.global_rank, self._mp_degree, - self._sharding_degree, self._pp_degree, self._dp_degree) - debug_str += ", mp_group: %s, sharding_group: %s, pp_group: %s, dp_group: %s, check/clip group: %s" % ( - self._mp_group, self._sharding_group, self._pp_group, - self._dp_group, self._check_group) + debug_str = ( + "HybridParallelInfo: rank_id: %d, mp_degree: %d, " + "sharding_degree: %d, pp_degree: %d, dp_degree: %d" + % ( + self.global_rank, + self._mp_degree, + self._sharding_degree, + self._pp_degree, + self._dp_degree, + ) + ) + debug_str += ( + ", mp_group: %s, sharding_group: %s, pp_group: %s, dp_group: %s, check/clip group: %s" + % ( + self._mp_group, + self._sharding_group, + self._pp_group, + self._dp_group, + self._check_group, + ) + ) logger.info(debug_str) global _HYBRID_PARALLEL_GROUP @@ -192,7 +219,12 @@ class HybridCommunicateGroup(object): # adding its parallel logic within that parallelism # when use sharding alone, it should have its own parallelism for its parallel logic # TODO modify 3 others parallel to support sharding - if self._mp_degree == 1 and self._pp_degree == 1 and self._dp_degree == 1 and self._sharding_degree > 1: + if ( + self._mp_degree == 1 + and self._pp_degree == 1 + and self._dp_degree == 1 + and self._sharding_degree > 1 + ): return ParallelMode.SHARDING_PARALLEL elif self._mp_degree == 1 and self._pp_degree == 1: return ParallelMode.DATA_PARALLEL @@ -203,7 +235,13 @@ class HybridCommunicateGroup(object): return ParallelMode.PIPELINE_PARALLEL def _check_vaild_topo(self): - return self._dp_degree * self._mp_degree * self._pp_degree * self._sharding_degree == self.nranks + return ( + self._dp_degree + * self._mp_degree + * self._pp_degree + * self._sharding_degree + == self.nranks + ) def _set_comm_group(self, parallel_method="data"): parallel_group = [] @@ -265,14 +303,16 @@ class HybridCommunicateGroup(object): self.prev_rank = prev_rank next_group = paddle.distributed.new_group( - ranks=[curr_rank, next_rank]) + ranks=[curr_rank, next_rank] + ) if self.global_rank == curr_rank: self.send_next_group = next_group elif self.global_rank == next_rank: self.recv_prev_group = next_group prev_group = paddle.distributed.new_group( - ranks=[prev_rank, curr_rank]) + ranks=[prev_rank, curr_rank] + ) if self.global_rank == curr_rank: self.send_prev_group = prev_group @@ -336,7 +376,12 @@ class HybridCommunicateGroup(object): return self._pp_comm_group def get_p2p_groups(self): - return self.send_next_group, self.send_prev_group, self.recv_next_group, self.recv_prev_group + return ( + self.send_next_group, + self.send_prev_group, + self.recv_next_group, + self.recv_prev_group, + ) # sharding parallel message: def _get_sharding_parallel_id(self): @@ -360,23 +405,25 @@ class HybridCommunicateGroup(object): return self._check_comm_group def get_rank_from_stage(self, stage_id, **kwargs): - return self._topo.get_rank_from_stage(self.global_rank, - pipe=stage_id, - **kwargs) + return self._topo.get_rank_from_stage( + self.global_rank, pipe=stage_id, **kwargs + ) class _CommunicateGroup(object): - """ tmp for static """ + """tmp for static""" def __init__(self): global _HYBRID_PARALLEL_GROUP _HYBRID_PARALLEL_GROUP = self self.groups = dict() - def set_comm_group(self, group_name, group_rank, group_size, ring_id, - group_ranks): - group = paddle.distributed.collective.Group(group_rank, ring_id, - group_ranks) + def set_comm_group( + self, group_name, group_rank, group_size, ring_id, group_ranks + ): + group = paddle.distributed.collective.Group( + group_rank, ring_id, group_ranks + ) self.groups[group_name] = group def get_group(self, group_name): diff --git a/python/paddle/distributed/fleet/base/util_factory.py b/python/paddle/distributed/fleet/base/util_factory.py index 95d287811e6..5af1acb9413 100755 --- a/python/paddle/distributed/fleet/base/util_factory.py +++ b/python/paddle/distributed/fleet/base/util_factory.py @@ -32,7 +32,6 @@ __all__ = [] class UtilFactory(object): - def _create_util(self, context=None): util = UtilBase() if context is not None and "valid_strategy" in context: @@ -43,7 +42,6 @@ class UtilFactory(object): class UtilBase(object): - def __init__(self): self.role_maker = None self.dist_strategy = None @@ -221,7 +219,7 @@ class UtilBase(object): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] @@ -279,7 +277,7 @@ class UtilBase(object): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] @@ -323,7 +321,6 @@ class UtilBase(object): f.write(program.desc.serialize_to_string()) def _load_program(self, path, is_text): - def load_program_binary(path): """load program from binary string file""" with open(path, "rb") as f: @@ -347,8 +344,9 @@ class UtilBase(object): def _program_type_trans(self, prog_dir, prog_fn, is_text): prog = self._load_program(os.path.join(prog_dir, prog_fn), is_text) prog_out_fn = prog_fn + ".bin" if is_text else prog_fn + ".pbtxt" - self._save_program(prog, os.path.join(prog_dir, prog_out_fn), - 1 - is_text) + self._save_program( + prog, os.path.join(prog_dir, prog_out_fn), 1 - is_text + ) return prog_out_fn def _visualize_graphviz(self, program, output_dir, output_filename): @@ -357,29 +355,37 @@ class UtilBase(object): pdf_path = os.path.join(output_dir, output_filename + '.pdf') debugger.draw_block_graphviz(block, path=dot_path) cmd = ["dot", "-Tpdf", dot_path, "-o", pdf_path] - p = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) p.wait() def _proto_check(self, config): - train_prog = self._load_program(config.train_prog_path, - config.is_text_train_program) - pruned_prog = self._load_program(config.pruned_prog_path, - config.is_text_pruned_program) + train_prog = self._load_program( + config.train_prog_path, config.is_text_train_program + ) + pruned_prog = self._load_program( + config.pruned_prog_path, config.is_text_pruned_program + ) is_match = True - pruned_vars = [(v.name, v) for v in pruned_prog.list_vars() - if fluid.io.is_persistable(v)] + pruned_vars = [ + (v.name, v) + for v in pruned_prog.list_vars() + if fluid.io.is_persistable(v) + ] pruned_vars = OrderedDict(pruned_vars) pruned_vars_name = [name for name in pruned_vars] print("persistable vars in pruned program: {}".format(pruned_vars_name)) # feed and fetch op is added in pruned program when pruning, not need to be found in train program feed_fetch_type_list = [ - core.VarDesc.VarType.FEED_MINIBATCH, core.VarDesc.VarType.FETCH_LIST + core.VarDesc.VarType.FEED_MINIBATCH, + core.VarDesc.VarType.FETCH_LIST, ] for var_name in pruned_vars: @@ -392,21 +398,28 @@ class UtilBase(object): except ValueError as e: print( "Not find variable '%s' in train program. please check pruning." - % var_name) + % var_name + ) is_match = False continue - if var.shape != train_prog_var.shape or var.dtype != train_prog_var.dtype: + if ( + var.shape != train_prog_var.shape + or var.dtype != train_prog_var.dtype + ): print( - "variable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}" - .format(var_name, var.shape, var.dtype, - train_prog_var.shape, train_prog_var.dtype)) + "variable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}".format( + var_name, + var.shape, + var.dtype, + train_prog_var.shape, + train_prog_var.dtype, + ) + ) is_match = False return is_match def _params_check(self, config): - def feed_gen(batch_size, feeded_vars_dims, feeded_vars_filelist): - def reader(batch_size, fn, dim): data = [] if isinstance(dim, list) or isinstance(dim, tuple): @@ -437,39 +450,57 @@ class UtilBase(object): prog = self._load_program( os.path.join(config.dump_model_dir, config.dump_program_filename), - config.is_text_dump_program) + config.is_text_dump_program, + ) if config.is_text_dump_program: model_filename = self._program_type_trans( - config.dump_model_dir, config.dump_program_filename, - config.is_text_dump_program) + config.dump_model_dir, + config.dump_program_filename, + config.is_text_dump_program, + ) saved_params = [ v for v in prog.list_vars() if fluid.io.is_persistable(v) ] - print("persistable vars in dump program: {}".format( - [v.name for v in saved_params])) + print( + "persistable vars in dump program: {}".format( + [v.name for v in saved_params] + ) + ) def check_not_expected_ops(prog, not_expected_op_types): op_types_set = set() for op in prog.global_block().ops: - if op.type in not_expected_op_types and op.type not in op_types_set: + if ( + op.type in not_expected_op_types + and op.type not in op_types_set + ): op_types_set.add(op.type) return op_types_set not_expected_op_types = check_not_expected_ops(prog, ["lookup_table"]) if len(not_expected_op_types) > 0: print( - "find op type '{}' in program, please check if your program is pruned correctly !" - .format(list(not_expected_op_types))) + "find op type '{}' in program, please check if your program is pruned correctly !".format( + list(not_expected_op_types) + ) + ) return False place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.core.Scope() with fluid.scope_guard(scope): - inference_program, feed_target_names, fetch_targets = \ - fluid.io.load_inference_model(config.dump_model_dir, exe, model_filename=model_filename, - params_filename=config.save_params_filename) + ( + inference_program, + feed_target_names, + fetch_targets, + ) = fluid.io.load_inference_model( + config.dump_model_dir, + exe, + model_filename=model_filename, + params_filename=config.save_params_filename, + ) # check program vars and saved vars shape orig_para_shape = { @@ -480,13 +511,17 @@ class UtilBase(object): var_temp = fluid.global_scope().find_var(each_var.name) assert var_temp != None, "can't not find var: " + each_var.name new_shape = (np.array(var_temp.get_tensor())).shape - assert each_var.name in orig_para_shape, each_var.name + "MUST in var list" + assert each_var.name in orig_para_shape, ( + each_var.name + "MUST in var list" + ) orig_shape = orig_para_shape.get(each_var.name) if new_shape != orig_shape: raise RuntimeError( "Shape not matching: the Program requires a parameter with a shape of ({}), " - "while the loaded parameter (namely [ {} ]) has a shape of ({})." - .format(orig_shape, each_var.name, new_shape)) + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_shape, each_var.name, new_shape + ) + ) # check feed/fetch vars in program and config feed_config = config.feed_config @@ -498,10 +533,15 @@ class UtilBase(object): print("warning! no fetch targets in program.") fetch_list = fetch_targets feed_name_list = feed_target_names - if feed_config.feeded_vars_names is not None and feed_target_names != feed_config.feeded_vars_names: + if ( + feed_config.feeded_vars_names is not None + and feed_target_names != feed_config.feeded_vars_names + ): print( - "warning! feed vars in program and config are diff: feed in program: {}. feed in config {}." - .format(feed_target_names, feed_config.feeded_vars_names)) + "warning! feed vars in program and config are diff: feed in program: {}. feed in config {}.".format( + feed_target_names, feed_config.feeded_vars_names + ) + ) feed_name_list = feed_config.feeded_vars_names # remove feed op in inference_program. new feed op will be added in exe.run global_block = inference_program.global_block() @@ -512,10 +552,15 @@ class UtilBase(object): need_to_remove_op_index.append(i) for index in need_to_remove_op_index[::-1]: global_block._remove_op(index) - if fetch_config.fetch_vars_names is not None and fetch_targets_names != fetch_config.fetch_vars_names: + if ( + fetch_config.fetch_vars_names is not None + and fetch_targets_names != fetch_config.fetch_vars_names + ): print( - "warning! fetch vars in program and config are diff: fetch in program: {}. fetch in config {}." - .format(fetch_targets_names, fetch_config.fetch_vars_names)) + "warning! fetch vars in program and config are diff: fetch in program: {}. fetch in config {}.".format( + fetch_targets_names, fetch_config.fetch_vars_names + ) + ) fetch_list = [ inference_program.global_block().var(i) for i in fetch_config.fetch_vars_names @@ -535,75 +580,105 @@ class UtilBase(object): # try dump fetch_targets feed_tensors = [] - assert len(feed_config.feeded_vars_names) == len( - feed_config.feeded_vars_dims) == len( - feed_config.feeded_vars_types) + assert ( + len(feed_config.feeded_vars_names) + == len(feed_config.feeded_vars_dims) + == len(feed_config.feeded_vars_types) + ) # check program vars and feed tensor shape in config for i in range(len(feed_config.feeded_vars_names)): var = inference_program.global_block().var( - feed_config.feeded_vars_names[i]) - if not isinstance(feed_config.feeded_vars_dims[i], - (list, tuple)): - tensor_shape = (feed_config.feeded_vars_dims[i], ) + feed_config.feeded_vars_names[i] + ) + if not isinstance( + feed_config.feeded_vars_dims[i], (list, tuple) + ): + tensor_shape = (feed_config.feeded_vars_dims[i],) else: tensor_shape = tuple(feed_config.feeded_vars_dims[i]) feed_config.feeded_vars_dims[i] = tensor_shape var_shape = var.shape[1:] if tensor_shape != var_shape: raise RuntimeError( - "feed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}" - .format(feed_config.feeded_vars_names[i], var_shape, - tensor_shape)) + "feed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}".format( + feed_config.feeded_vars_names[i], + var_shape, + tensor_shape, + ) + ) if not feed_config.feeded_vars_filelist: print("generate random feed vars.") for i in range(len(feed_config.feeded_vars_names)): var = inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) # create fake feed tensor. if lod_level > 1, should create_lod_tensor() if var.lod_level == 0: feed_tensors.append( - np.array(np.random.random( - tuple([config.batch_size] + - list(feed_config.feeded_vars_dims[i]))), - dtype=feed_config.feeded_vars_types[i])) + np.array( + np.random.random( + tuple( + [config.batch_size] + + list(feed_config.feeded_vars_dims[i]) + ) + ), + dtype=feed_config.feeded_vars_types[i], + ) + ) elif var.lod_level == 1: - t = np.array(np.random.random( - tuple([config.batch_size] + - list(feed_config.feeded_vars_dims[i]))), - dtype=feed_config.feeded_vars_types[i]) + t = np.array( + np.random.random( + tuple( + [config.batch_size] + + list(feed_config.feeded_vars_dims[i]) + ) + ), + dtype=feed_config.feeded_vars_types[i], + ) feed_tensors.append( - fluid.create_lod_tensor(t, - [[1] * config.batch_size], - place)) + fluid.create_lod_tensor( + t, [[1] * config.batch_size], place + ) + ) else: raise RuntimeError( "vars with lod_level >= 2 is not supported now in this infer program check tool." ) - results = exe.run(inference_program, - feed={ - name: feed_tensors[i] - for i, name in enumerate(feed_name_list) - }, - fetch_list=fetch_list, - return_numpy=return_numpy) + results = exe.run( + inference_program, + feed={ + name: feed_tensors[i] + for i, name in enumerate(feed_name_list) + }, + fetch_list=fetch_list, + return_numpy=return_numpy, + ) else: - print("load feed vars from files: {}.".format( - feed_config.feeded_vars_filelist)) + print( + "load feed vars from files: {}.".format( + feed_config.feeded_vars_filelist + ) + ) feed_vars = [ inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) for i in range(len(feed_config.feeded_vars_names)) ] feeder = fluid.DataFeeder(feed_list=feed_vars, place=place) - batch_feed = feed_gen(config.batch_size, - feed_config.feeded_vars_dims, - feed_config.feeded_vars_filelist) + batch_feed = feed_gen( + config.batch_size, + feed_config.feeded_vars_dims, + feed_config.feeded_vars_filelist, + ) slots = [batch_feed] - results = exe.run(inference_program, - feed=feeder.feed(slots), - fetch_list=fetch_list, - return_numpy=return_numpy) + results = exe.run( + inference_program, + feed=feeder.feed(slots), + fetch_list=fetch_list, + return_numpy=return_numpy, + ) for i, v in enumerate(fetch_list): print("fetch_targets name: %s" % v.name) print("fetch_targets: {}".format(results[i])) diff --git a/python/paddle/distributed/fleet/cloud_utils.py b/python/paddle/distributed/fleet/cloud_utils.py index cfd8a9ff4e2..cf74f844670 100644 --- a/python/paddle/distributed/fleet/cloud_utils.py +++ b/python/paddle/distributed/fleet/cloud_utils.py @@ -18,14 +18,13 @@ from paddle.distributed.fleet.launch_utils import get_cluster, logger __all__ = [] -def get_cloud_cluster(args_node_ips, - device_mode, - devices_per_proc, - args_port=6170): +def get_cloud_cluster( + args_node_ips, device_mode, devices_per_proc, args_port=6170 +): """ args_node_ips:string, device_mode:DeviceMode(Int), device_per_proc:list, args_port: int """ - #you can automatically get ip info while using paddlecloud multi nodes mode. + # you can automatically get ip info while using paddlecloud multi nodes mode. node_ips = os.getenv("PADDLE_TRAINERS") assert node_ips is not None, "PADDLE_TRAINERS should not be None" @@ -47,7 +46,10 @@ def get_cloud_cluster(args_node_ips, "Please NOTE: When using paddlecloud, cluster_node_ips is \ automatically got from PADDLE_TRAINERS(multi nodes) or POD_IP(single node).\ Your input cluster_node_ips: {} doesn't equals to IPs: {} from \ -paddlecloud environment.".format(args_node_ips, node_ips)) +paddlecloud environment.".format( + args_node_ips, node_ips + ) + ) # DISTRIBUTED_TRAINER_ENDPOINTS: new environment since paddlecloud 1.8.4 # e.g: DISTRIBUTED_TRAINER_ENDPOINTS="ip1:port1,ip1:port2,ip1:port3,ip1:port4,ip2:port5,ip2:port6,ip2:port7,ip2:port8" @@ -58,10 +60,13 @@ paddlecloud environment.".format(args_node_ips, node_ips)) try: paddle_port = int(os.getenv("PADDLE_PORT", "")) - if paddle_ports_num >= len( - devices_per_proc) and paddle_port != args_port: + if ( + paddle_ports_num >= len(devices_per_proc) + and paddle_port != args_port + ): logger.warning( - "Use Cloud specified port:{}.".format(paddle_port)) + "Use Cloud specified port:{}.".format(paddle_port) + ) started_port = paddle_port except Exception as e: @@ -81,15 +86,21 @@ paddlecloud environment.".format(args_node_ips, node_ips)) assert num_nodes * paddle_ports_num == len(trainer_endpoints_ori) for i in range(num_nodes): trainer_endpoints.append( - trainer_endpoints_ori[i * paddle_ports_num:(i + 1) * - paddle_ports_num]) + trainer_endpoints_ori[ + i * paddle_ports_num : (i + 1) * paddle_ports_num + ] + ) - logger.debug("parsed from args: node_ips:{} \ + logger.debug( + "parsed from args: node_ips:{} \ node_ip:{} node_rank:{} trainer_endpoints:{}".format( - node_ips, node_ip, node_rank, trainer_endpoints)) + node_ips, node_ip, node_rank, trainer_endpoints + ) + ) - cluster, pod = get_cluster(node_ips, node_ip, trainer_endpoints, - device_mode, devices_per_proc) + cluster, pod = get_cluster( + node_ips, node_ip, trainer_endpoints, device_mode, devices_per_proc + ) return cluster, cluster.pods[node_rank] @@ -98,7 +109,12 @@ def use_paddlecloud(): node_ip = os.getenv("POD_IP") node_rank = os.getenv("PADDLE_TRAINER_ID") paddle_ports_num = os.getenv("TRAINER_PORTS_NUM") - if node_ips is None or node_ip is None or node_rank is None or paddle_ports_num is None: + if ( + node_ips is None + or node_ip is None + or node_rank is None + or paddle_ports_num is None + ): return False else: return True diff --git a/python/paddle/distributed/fleet/data_generator/data_generator.py b/python/paddle/distributed/fleet/data_generator/data_generator.py index d43c376bb0c..9e9ec7b61bc 100644 --- a/python/paddle/distributed/fleet/data_generator/data_generator.py +++ b/python/paddle/distributed/fleet/data_generator/data_generator.py @@ -147,7 +147,8 @@ class DataGenerator(object): Return a string data that can be read directly by the datafeed. ''' raise NotImplementedError( - "pls use MultiSlotDataGenerator or PairWiseDataGenerator") + "pls use MultiSlotDataGenerator or PairWiseDataGenerator" + ) def generate_sample(self, line): ''' @@ -187,8 +188,9 @@ class DataGenerator(object): ''' raise NotImplementedError( - "Please rewrite this function to return a list or tuple: " + - "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)") + "Please rewrite this function to return a list or tuple: " + + "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)" + ) def generate_batch(self, samples): ''' @@ -236,7 +238,6 @@ class DataGenerator(object): # add more generalized DataGenerator that can adapt user-defined slot # for example, [(name, float_list), (name, str_list), (name, int_list)] class MultiSlotStringDataGenerator(DataGenerator): - def _gen_str(self, line): ''' Further processing the output of the process() function rewritten by @@ -267,7 +268,8 @@ class MultiSlotStringDataGenerator(DataGenerator): if not isinstance(line, list) and not isinstance(line, tuple): raise ValueError( "the output of process() must be in list or tuple type" - "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]") + "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]" + ) output = "" for index, item in enumerate(line): name, elements = item @@ -281,7 +283,6 @@ class MultiSlotStringDataGenerator(DataGenerator): class MultiSlotDataGenerator(DataGenerator): - def _gen_str(self, line): ''' Further processing the output of the process() function rewritten by @@ -316,7 +317,8 @@ class MultiSlotDataGenerator(DataGenerator): if not isinstance(line, list) and not isinstance(line, tuple): raise ValueError( "the output of process() must be in list or tuple type" - "Example: [('words', [1926, 08, 17]), ('label', [1])]") + "Example: [('words', [1926, 08, 17]), ('label', [1])]" + ) output = "" if self._proto_info is None: @@ -326,8 +328,9 @@ class MultiSlotDataGenerator(DataGenerator): if not isinstance(name, str): raise ValueError("name%s must be in str type" % type(name)) if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) + raise ValueError( + "elements%s must be in list type" % type(elements) + ) if not elements: raise ValueError( "the elements of each field can not be empty, you need padding it in process()." @@ -340,10 +343,12 @@ class MultiSlotDataGenerator(DataGenerator): if isinstance(elem, float): self._proto_info[-1] = (name, "float") elif not isinstance(elem, int) and not isinstance( - elem, long): + elem, long + ): raise ValueError( - "the type of element%s must be in int or float" % - type(elem)) + "the type of element%s must be in int or float" + % type(elem) + ) output += " " + str(elem) else: if len(line) != len(self._proto_info): @@ -355,8 +360,9 @@ class MultiSlotDataGenerator(DataGenerator): if not isinstance(name, str): raise ValueError("name%s must be in str type" % type(name)) if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) + raise ValueError( + "elements%s must be in list type" % type(elements) + ) if not elements: raise ValueError( "the elements of each field can not be empty, you need padding it in process()." @@ -364,7 +370,8 @@ class MultiSlotDataGenerator(DataGenerator): if name != self._proto_info[index][0]: raise ValueError( "the field name of two given line are not match: require<%s>, get<%s>." - % (self._proto_info[index][0], name)) + % (self._proto_info[index][0], name) + ) if output: output += " " output += str(len(elements)) @@ -373,9 +380,11 @@ class MultiSlotDataGenerator(DataGenerator): if isinstance(elem, float): self._proto_info[index] = (name, "float") elif not isinstance(elem, int) and not isinstance( - elem, long): + elem, long + ): raise ValueError( "the type of element%s must be in int or float" - % type(elem)) + % type(elem) + ) output += " " + str(elem) return output + "\n" diff --git a/python/paddle/distributed/fleet/dataset/dataset.py b/python/paddle/distributed/fleet/dataset/dataset.py index 58e265c8347..b74f700391c 100755 --- a/python/paddle/distributed/fleet/dataset/dataset.py +++ b/python/paddle/distributed/fleet/dataset/dataset.py @@ -21,10 +21,10 @@ __all__ = [] class DatasetBase(object): - """ Base dataset class. """ + """Base dataset class.""" def __init__(self): - """ Init. """ + """Init.""" # define class name here # to decide whether we need create in memory instance self.proto_desc = data_feed_pb2.DataFeedDesc() @@ -35,15 +35,17 @@ class DatasetBase(object): self.use_ps_gpu = False self.psgpu = None - def init(self, - batch_size=1, - thread_num=1, - use_var=[], - pipe_command="cat", - input_type=0, - fs_name="", - fs_ugi="", - download_cmd="cat"): + def init( + self, + batch_size=1, + thread_num=1, + use_var=[], + pipe_command="cat", + input_type=0, + fs_name="", + fs_ugi="", + download_cmd="cat", + ): """ should be called only once in user's python scripts to initialize setings of dataset instance. Normally, it is called by InMemoryDataset or QueueDataset. @@ -271,8 +273,9 @@ class DatasetBase(object): def _dynamic_adjust_after_train(self): pass - def _check_use_var_with_data_generator(self, var_list, data_generator_class, - test_file): + def _check_use_var_with_data_generator( + self, var_list, data_generator_class, test_file + ): """ Var consistency insepection of use_var_list and data_generator data. @@ -304,32 +307,38 @@ class DatasetBase(object): if var_len != data_gen_len: raise ValueError( "var length mismatch error: var_list = %s vs data_generator = %s" - % (var_len, data_gen_len)) + % (var_len, data_gen_len) + ) for i, ele in enumerate(user_parsed_line): if len(ele[1]) == 0: raise ValueError( "var length error: var %s's length in data_generator is 0" - % ele[0]) + % ele[0] + ) if var_list[ - i].dtype == core.VarDesc.VarType.FP32 and not all( - isinstance(ele, float) for ele in ele[1]): + i + ].dtype == core.VarDesc.VarType.FP32 and not all( + isinstance(ele, float) for ele in ele[1] + ): raise TypeError( "var dtype mismatch error: var name = %s, var type in var_list = %s, while var in data_generator contains non-float value, which is %s \n" "Please check if order of var_list and data_generator are aligned. \n" "Please check if var's type in data_generator is correct." - % (ele[0], "float", ele[1])) + % (ele[0], "float", ele[1]) + ) - if (var_list[i].dtype == core.VarDesc.VarType.INT64 - or var_list[i].dtype - == core.VarDesc.VarType.INT32) and not all( - isinstance(ele, int) for ele in ele[1]): + if ( + var_list[i].dtype == core.VarDesc.VarType.INT64 + or var_list[i].dtype == core.VarDesc.VarType.INT32 + ) and not all(isinstance(ele, int) for ele in ele[1]): raise TypeError( "var dtype mismatch error: var name = %s, var type in var_list = %s, while var in data_generator contains non-int value, which is %s \n" "Please check if order of var_list and data_generator are aligned. \n" "Please check if var's type in data_generator is correct." - % (ele[0], "int", ele[1])) + % (ele[0], "int", ele[1]) + ) else: break @@ -353,7 +362,7 @@ class InMemoryDataset(DatasetBase): """ def __init__(self): - """ Init. """ + """Init.""" super(InMemoryDataset, self).__init__() self.proto_desc.name = "MultiSlotInMemoryDataFeed" self.fleet_send_batch_size = None @@ -588,14 +597,16 @@ class InMemoryDataset(DatasetBase): data_feed_type = "MultiSlotInMemoryDataFeed" self._set_feed_type(data_feed_type) - super(InMemoryDataset, self).init(batch_size=batch_size, - thread_num=thread_num, - use_var=use_var, - pipe_command=pipe_command, - input_type=input_type, - fs_name=fs_name, - fs_ugi=fs_ugi, - download_cmd=download_cmd) + super(InMemoryDataset, self).init( + batch_size=batch_size, + thread_num=thread_num, + use_var=use_var, + pipe_command=pipe_command, + input_type=input_type, + fs_name=fs_name, + fs_ugi=fs_ugi, + download_cmd=download_cmd, + ) if kwargs.get("queue_num", -1) > 0: queue_num = kwargs.get("queue_num", -1) @@ -606,7 +617,7 @@ class InMemoryDataset(DatasetBase): Set data_feed_desc """ self.proto_desc.name = data_feed_type - if (self.proto_desc.name == "SlotRecordInMemoryDataFeed"): + if self.proto_desc.name == "SlotRecordInMemoryDataFeed": self.dataset = core.Dataset("SlotRecordDataset") def _prepare_to_run(self): @@ -779,11 +790,12 @@ class InMemoryDataset(DatasetBase): self.gen_uni_feasigns = generate_uni_feasigns self.local_shard_num = shard_num - def _generate_local_tables_unlock(self, table_id, fea_dim, read_thread_num, - consume_thread_num, shard_num): - self.dataset.generate_local_tables_unlock(table_id, fea_dim, - read_thread_num, - consume_thread_num, shard_num) + def _generate_local_tables_unlock( + self, table_id, fea_dim, read_thread_num, consume_thread_num, shard_num + ): + self.dataset.generate_local_tables_unlock( + table_id, fea_dim, read_thread_num, consume_thread_num, shard_num + ) def set_date(self, date): """ @@ -821,11 +833,25 @@ class InMemoryDataset(DatasetBase): if self.use_ps_gpu and core._is_compiled_with_heterps(): self.psgpu.set_date(year, month, day) - def tdm_sample(self, tree_name, tree_path, tdm_layer_counts, - start_sample_layer, with_hierachy, seed, id_slot): - self.dataset.tdm_sample(tree_name, tree_path, tdm_layer_counts, - start_sample_layer, with_hierachy, seed, - id_slot) + def tdm_sample( + self, + tree_name, + tree_path, + tdm_layer_counts, + start_sample_layer, + with_hierachy, + seed, + id_slot, + ): + self.dataset.tdm_sample( + tree_name, + tree_path, + tdm_layer_counts, + start_sample_layer, + with_hierachy, + seed, + id_slot, + ) def load_into_memory(self, is_shuffle=False): """ @@ -1112,12 +1138,14 @@ class InMemoryDataset(DatasetBase): """ import numpy as np + local_data_size = self.dataset.get_memory_data_size() local_data_size = np.array([local_data_size]) if fleet is not None: global_data_size = local_data_size * 0 - fleet._role_maker.all_reduce_worker(local_data_size, - global_data_size) + fleet._role_maker.all_reduce_worker( + local_data_size, global_data_size + ) return global_data_size[0] return local_data_size[0] @@ -1166,12 +1194,14 @@ class InMemoryDataset(DatasetBase): """ import numpy as np + local_data_size = self.dataset.get_shuffle_data_size() local_data_size = np.array([local_data_size]) if fleet is not None: global_data_size = local_data_size * 0 - fleet._role_maker.all_reduce_worker(local_data_size, - global_data_size) + fleet._role_maker.all_reduce_worker( + local_data_size, global_data_size + ) return global_data_size[0] return local_data_size[0] diff --git a/python/paddle/distributed/fleet/dataset/index_dataset.py b/python/paddle/distributed/fleet/dataset/index_dataset.py index 8b5a9c5a45b..1dda709fffd 100644 --- a/python/paddle/distributed/fleet/dataset/index_dataset.py +++ b/python/paddle/distributed/fleet/dataset/index_dataset.py @@ -17,13 +17,11 @@ __all__ = [] class Index(object): - def __init__(self, name): self._name = name class TreeIndex(Index): - def __init__(self, name, path): super(TreeIndex, self).__init__(name) self._wrapper = core.IndexWrapper() @@ -67,7 +65,7 @@ class TreeIndex(Index): def get_travel_path(self, child, ancestor): res = [] - while (child > ancestor): + while child > ancestor: res.append(child) child = int((child - 1) / self._branch) return res @@ -76,17 +74,18 @@ class TreeIndex(Index): codes = self.get_ancestor_codes(ids, level) return dict(zip(ids, codes)) - def init_layerwise_sampler(self, - layer_sample_counts, - start_sample_layer=1, - seed=0): + def init_layerwise_sampler( + self, layer_sample_counts, start_sample_layer=1, seed=0 + ): assert self._layerwise_sampler is None self._layerwise_sampler = core.IndexSampler("by_layerwise", self._name) - self._layerwise_sampler.init_layerwise_conf(layer_sample_counts, - start_sample_layer, seed) + self._layerwise_sampler.init_layerwise_conf( + layer_sample_counts, start_sample_layer, seed + ) def layerwise_sample(self, user_input, index_input, with_hierarchy=False): if self._layerwise_sampler is None: raise ValueError("please init layerwise_sampler first.") - return self._layerwise_sampler.sample(user_input, index_input, - with_hierarchy) + return self._layerwise_sampler.sample( + user_input, index_input, with_hierarchy + ) diff --git a/python/paddle/distributed/fleet/elastic/__init__.py b/python/paddle/distributed/fleet/elastic/__init__.py index b80a66c6f01..447bedb21d5 100644 --- a/python/paddle/distributed/fleet/elastic/__init__.py +++ b/python/paddle/distributed/fleet/elastic/__init__.py @@ -25,12 +25,12 @@ from paddle.distributed.fleet.launch_utils import DistributeMode def enable_elastic(args, distribute_mode): - #elastic_level = os.getenv('PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL') - #if not elastic_level and (elastic_level != ElasticLevel.FAULT_TOLERANCE and + # elastic_level = os.getenv('PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL') + # if not elastic_level and (elastic_level != ElasticLevel.FAULT_TOLERANCE and # elastic_level != ElasticLevel.ELASTIC): # return False - #if distribute_mode != DistributeMode.COLLECTIVE: + # if distribute_mode != DistributeMode.COLLECTIVE: # return False if not args.elastic_server and not os.getenv('PADDLE_ELASTIC_SERVER'): @@ -50,6 +50,7 @@ def launch_elastic(args, distribute_mode): server = args.elastic_server or os.getenv('PADDLE_ELASTIC_SERVER') srv, port = server.split(':') import etcd3 + etcd_client = etcd3.client(host=srv, port=port) elastic = ElasticManager(args, etcd_client) diff --git a/python/paddle/distributed/fleet/elastic/collective.py b/python/paddle/distributed/fleet/elastic/collective.py index aa5c962b994..bdb8a6c5770 100644 --- a/python/paddle/distributed/fleet/elastic/collective.py +++ b/python/paddle/distributed/fleet/elastic/collective.py @@ -17,12 +17,15 @@ import os import shutil import paddle -from paddle.distributed.fleet.launch_utils import logger, pull_worker_log, start_local_trainers +from paddle.distributed.fleet.launch_utils import ( + logger, + pull_worker_log, + start_local_trainers, +) from paddle.distributed.fleet.elastic.manager import LauncherInterface class CollectiveLauncher(LauncherInterface): - def __init__(self, args): self.args = args self.procs = [] @@ -33,7 +36,8 @@ class CollectiveLauncher(LauncherInterface): self.tmp_dir = tempfile.mkdtemp() cluster, pod = paddle.distributed.fleet.launch.get_cluster_info(args) global_envs = paddle.distributed.fleet.launch.get_global_envs( - args, self.tmp_dir) + args, self.tmp_dir + ) self.procs = start_local_trainers( cluster, @@ -41,7 +45,8 @@ class CollectiveLauncher(LauncherInterface): training_script=args.training_script, training_script_args=args.training_script_args, log_dir=args.log_dir, - envs=global_envs) + envs=global_envs, + ) for idx, proc in enumerate(self.procs): logger.info("launch proc_id:{} idx:{}".format(proc.proc.pid, idx)) diff --git a/python/paddle/distributed/fleet/elastic/manager.py b/python/paddle/distributed/fleet/elastic/manager.py index 84cf95742e3..9397f6d10d4 100644 --- a/python/paddle/distributed/fleet/elastic/manager.py +++ b/python/paddle/distributed/fleet/elastic/manager.py @@ -54,7 +54,6 @@ class ElasticStatus: class LauncherInterface(object): - def __init__(self, args): self.args = args self.procs = [] @@ -67,8 +66,9 @@ class LauncherInterface(object): os.killpg(os.getpgid(p.proc.pid), signal.SIGTERM) if p.log_fn: p.log_fn.close() - logger.info("terminate process group gid:{}".format( - p.proc.pid)) + logger.info( + "terminate process group gid:{}".format(p.proc.pid) + ) time.sleep(1) for p in self.procs: @@ -105,8 +105,10 @@ class LauncherInterface(object): return ret logger.error("ABORT!!! ABORT!!! ABORT!!!") logger.error( - "ERROR rank {} error with exit code {}, check log for detail." - .format(p.rank, ret)) + "ERROR rank {} error with exit code {}, check log for detail.".format( + p.rank, ret + ) + ) result = ret if not alive and result is None: return 0 @@ -124,7 +126,6 @@ class LauncherInterface(object): class ElasticManager(object): - def __init__(self, args, etcd_client): self.args = args @@ -137,11 +138,14 @@ class ElasticManager(object): self.host = host if host else self._get_host() - (self.device_mode, - self.devices_per_proc) = launch_utils.get_device_proc_info(args) + ( + self.device_mode, + self.devices_per_proc, + ) = launch_utils.get_device_proc_info(args) self.elastic_timeout = int( - os.getenv('PADDLE_ELASTIC_TIMEOUT', ELASTIC_TIMEOUT)) + os.getenv('PADDLE_ELASTIC_TIMEOUT', ELASTIC_TIMEOUT) + ) elastic_ttl = int(os.getenv('PADDLE_ELASTIC_TTL', ELASTIC_TTL)) self.start_port = None @@ -158,7 +162,8 @@ class ElasticManager(object): self.np = len(node_ips) self.start_port = int(os.getenv("FLAGS_START_PORT", "6170")) self.dist_endpoints = self._host_to_endpoints( - node_ips, self.devices_per_proc, self.start_port) + node_ips, self.devices_per_proc, self.start_port + ) self.trainer_endpoints_list = [ "%s:%d" % (ip, self.start_port) for ip in node_ips ] @@ -172,10 +177,12 @@ class ElasticManager(object): # auto correct the value of elastic_level # 1: Fault tolerant, 2: Elastic self.elastic_level = int( - os.getenv('PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL', - ElasticLevel.FAULT_TOLERANCE)) - if self.min_np == self.max_np or \ - (self.min_np > 0 and self.max_np == 0): + os.getenv( + 'PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL', + ElasticLevel.FAULT_TOLERANCE, + ) + ) + if self.min_np == self.max_np or (self.min_np > 0 and self.max_np == 0): self.elastic_level = ElasticLevel.FAULT_TOLERANCE logger.info('start job with ElasticLevel.FAULT_TOLERANCE') if self.min_np > 0 and self.max_np > self.min_np: @@ -183,12 +190,15 @@ class ElasticManager(object): logger.info('start job with ElasticLevel.ELASTIC') # compatible with kuberntes service discovery - if not server and os.getenv( - 'PADDLE_ELASTIC_ETCD_SERVICE_HOST') and os.getenv( - 'PADDLE_ELASTIC_ETCD_SERVICE_PORT'): + if ( + not server + and os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_HOST') + and os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_PORT') + ): server = '{}:{}'.format( os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_HOST'), - os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_PORT')) + os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_PORT'), + ) logger.debug('init with server {} host {}'.format(server, host)) @@ -202,8 +212,10 @@ class ElasticManager(object): if not server or ':' not in server or not name or not self.np: logger.info( - 'Elastic is not enabled with server {} name {} and np {}'. - format(server, name, self.np)) + 'Elastic is not enabled with server {} name {} and np {}'.format( + server, name, self.np + ) + ) self.enable = False return else: @@ -218,9 +230,11 @@ class ElasticManager(object): self.endpoints_path = self.prefix + '/endpoints' node_tag = ''.join( - random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6)) - self.host_path = '{}/{}{}'.format(self.node_prefix, node_tag, - time.time()) + random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6) + ) + self.host_path = '{}/{}{}'.format( + self.node_prefix, node_tag, time.time() + ) ''' 0 group mode, be aware of healthy status of other workers 1 decouple mode, check own status only @@ -241,7 +255,8 @@ class ElasticManager(object): self.elastic_startup_time = None host_watch = self.etcd.add_watch_prefix_callback( - self.node_prefix, host_call_back) + self.node_prefix, host_call_back + ) host_lease = self.etcd.lease(elastic_ttl) # register etcd lease heartbeat @@ -260,51 +275,61 @@ class ElasticManager(object): ) if self.curr_host not in hosts: logger.info( - f"[lease_heartbeat] register host={self.curr_host}") - self.etcd.put(self.host_path, - self.curr_host.encode('latin-1'), - lease=host_lease) + f"[lease_heartbeat] register host={self.curr_host}" + ) + self.etcd.put( + self.host_path, + self.curr_host.encode('latin-1'), + lease=host_lease, + ) except Exception as e: logger.error( "[lease_heartbeat] internal error:{} {}".format( - e, traceback.format_exc())) + e, traceback.format_exc() + ) + ) break time.sleep(elastic_ttl / 3) - keepalived_thread = threading.Thread(name='lease_heartbeat', - target=lease_heartbeat, - daemon=True) + keepalived_thread = threading.Thread( + name='lease_heartbeat', target=lease_heartbeat, daemon=True + ) keepalived_thread.start() - self.etcd.put(self.host_path, - self.curr_host.encode('latin-1'), - lease=host_lease) + self.etcd.put( + self.host_path, self.curr_host.encode('latin-1'), lease=host_lease + ) # endpoints handle DISTRIBUTED_TRAINER_ENDPOINTS and PADDLE_TRAINERS self.etcd.put( self.endpoints_path, - '{}|{}'.format(self.dist_endpoints, - self.trainers).encode('latin-1')) + '{}|{}'.format(self.dist_endpoints, self.trainers).encode( + 'latin-1' + ), + ) def endpoints_call_back(event): if not self.dist_endpoints: return edps = six.ensure_str(self.etcd.get(self.endpoints_path)[0] or '') self.dist_endpoints, self.trainers = edps.split('|') - logger.info("set DISTRIBUTED_TRAINER_ENDPOINTS {} ".format( - self.dist_endpoints)) + logger.info( + "set DISTRIBUTED_TRAINER_ENDPOINTS {} ".format( + self.dist_endpoints + ) + ) logger.info("set PADDLE_TRAINERS {} ".format(self.trainers)) - endpoints_watch = self.etcd.add_watch_callback(self.endpoints_path, - endpoints_call_back) + endpoints_watch = self.etcd.add_watch_callback( + self.endpoints_path, endpoints_call_back + ) self.watches = [host_watch, endpoints_watch] self.launcher = None - def _host_to_endpoints(self, - ip_port_list: list, - devices_per_proc: list, - start_port: int = 6170) -> str: + def _host_to_endpoints( + self, ip_port_list: list, devices_per_proc: list, start_port: int = 6170 + ) -> str: endpoint_list = [] for ip_port in ip_port_list: endpoints = ip_port.split(":") @@ -347,11 +372,13 @@ class ElasticManager(object): return logger.info("execute pre_hook...") current_env = copy.copy(os.environ.copy()) - out, err = subprocess.Popen(self.args.elastic_pre_hook, - env=current_env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True).communicate() + out, err = subprocess.Popen( + self.args.elastic_pre_hook, + env=current_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ).communicate() if err: logger.warning("pre_hook exec failed") else: @@ -377,7 +404,8 @@ class ElasticManager(object): max_np = min_np if min_np > max_np else max_np else: raise ValueError( - f'the np={np} needs to be in "MIN" or "MIN:MAX" format') + f'the np={np} needs to be in "MIN" or "MIN:MAX" format' + ) return min_np, max_np @@ -436,8 +464,10 @@ class ElasticManager(object): return False def _update_endpoint(self, endpoints, hosts): - self.etcd.put(self.endpoints_path, - '{}|{}'.format(endpoints, hosts).encode('latin-1')) + self.etcd.put( + self.endpoints_path, + '{}|{}'.format(endpoints, hosts).encode('latin-1'), + ) def _update_fault_tolrance(self): rank = int(os.getenv('PADDLE_TRAINER_ID', -1)) @@ -447,8 +477,11 @@ class ElasticManager(object): if self.curr_host in self.dist_endpoints: os.environ['DISTRIBUTED_TRAINER_ENDPOINTS'] = self.dist_endpoints os.environ['PADDLE_TRAINERS'] = self.trainers - logger.info("update env DISTRIBUTED_TRAINER_ENDPOINTS {} ".format( - self.dist_endpoints)) + logger.info( + "update env DISTRIBUTED_TRAINER_ENDPOINTS {} ".format( + self.dist_endpoints + ) + ) logger.info("update env PADDLE_TRAINERS {} ".format(self.trainers)) return @@ -476,9 +509,11 @@ class ElasticManager(object): host_endpoints.append(curr_host_port) os.environ['PADDLE_TRAINER_ID'] = '{}'.format( - host_endpoints.index(self.curr_host)) + host_endpoints.index(self.curr_host) + ) hosts = ','.join( - [host_port.split(":")[0] for host_port in host_endpoints]) + [host_port.split(":")[0] for host_port in host_endpoints] + ) self.args.ips = hosts os.environ['PADDLE_TRAINERS'] = hosts self.np = len(host_endpoints) @@ -521,12 +556,14 @@ class ElasticManager(object): ip_list = [ip_port.split(":")[0] for ip_port in sorted_endpoints] hosts = ','.join(ip_list) - new_endpoints = self._host_to_endpoints(sorted_endpoints, - self.devices_per_proc) + new_endpoints = self._host_to_endpoints( + sorted_endpoints, self.devices_per_proc + ) self.args.ips = hosts os.environ['PADDLE_TRAINER_ID'] = '{}'.format( - sorted_endpoints.index(self.curr_host)) + sorted_endpoints.index(self.curr_host) + ) os.environ['PADDLE_TRAINERS'] = hosts self.np = len(sorted_endpoints) os.environ['PADDLE_TRAINER_ENDPOINTS'] = ','.join(sorted_endpoints) @@ -560,8 +597,9 @@ class ElasticManager(object): logger.info('ready with hosts {}'.format(self.hosts)) self._update_hosts() return - logger.info('not ready for np {} with hosts {}'.format( - self.np, self.hosts)) + logger.info( + 'not ready for np {} with hosts {}'.format(self.np, self.hosts) + ) idx += 1 time.sleep(2) return diff --git a/python/paddle/distributed/fleet/fleet.py b/python/paddle/distributed/fleet/fleet.py index 7a6af30f912..db3aae28a20 100644 --- a/python/paddle/distributed/fleet/fleet.py +++ b/python/paddle/distributed/fleet/fleet.py @@ -55,12 +55,12 @@ def apply_ir_passes(main_program, startup_program, config): ) build_strategy.fuse_all_optimizer_ops = False - return apply_build_strategy(main_program, startup_program, build_strategy, - pass_attrs) + return apply_build_strategy( + main_program, startup_program, build_strategy, pass_attrs + ) def _inited_runtime_handler_(func): - def __impl__(*args, **kwargs): cls = args[0] @@ -73,15 +73,17 @@ def _inited_runtime_handler_(func): def _is_non_distributed_check_(func): - def __impl__(*args, **kwargs): cls = args[0] - if cls._role_maker is not None and cls._role_maker._is_non_distributed( - ) is True: + if ( + cls._role_maker is not None + and cls._role_maker._is_non_distributed() is True + ): logger.warning( - "%s() function doesn't work when use non_distributed fleet." % - (func.__name__)) + "%s() function doesn't work when use non_distributed fleet." + % (func.__name__) + ) return return func(*args, **kwargs) @@ -161,11 +163,13 @@ class Fleet(object): self._context = {} self.user_defined_optimizer = paddle.optimizer.Optimizer(0.0) - def init(self, - role_maker=None, - is_collective=False, - strategy=None, - log_level="INFO"): + def init( + self, + role_maker=None, + is_collective=False, + strategy=None, + log_level="INFO", + ): """ Initialize role_maker in Fleet. @@ -240,22 +244,28 @@ class Fleet(object): if isinstance(is_collective, bool): self._is_collective = is_collective self._role_maker = PaddleCloudRoleMaker( - is_collective=self._is_collective) + is_collective=self._is_collective + ) else: raise ValueError( - "`is_collective` should be instance of `bool`, but got {}". - format(type(is_collective))) + "`is_collective` should be instance of `bool`, but got {}".format( + type(is_collective) + ) + ) else: if isinstance(role_maker, RoleMakerBase): self._role_maker = role_maker self._is_collective = role_maker._is_collective else: raise ValueError( - "`role_maker` should be subclass of `RoleMakerBase`, but got {}" - .format(type(role_maker))) + "`role_maker` should be subclass of `RoleMakerBase`, but got {}".format( + type(role_maker) + ) + ) self._role_maker._generate_role() import paddle.distributed.fleet as fleet + fleet.util._set_role_maker(self._role_maker) self.strategy_compiler = StrategyCompiler() @@ -276,17 +286,20 @@ class Fleet(object): return if parallel_helper._is_parallel_ctx_initialized(): logger.warning( - "The dygraph parallel environment has been initialized.") + "The dygraph parallel environment has been initialized." + ) else: # FLAGS_nccl_nrings is used for dynamic graph multi-stream communication if "FLAGS_nccl_nrings" in os.environ: logger.warning( "You have set the environment variable FLAGS_nccl_nrings " "outside the program, so the nccl_comm_num in " - "DistributedStrategy will not take effect here.") + "DistributedStrategy will not take effect here." + ) else: os.environ["FLAGS_nccl_nrings"] = str( - self._user_defined_strategy.nccl_comm_num) + self._user_defined_strategy.nccl_comm_num + ) paddle.distributed.init_parallel_env() # hybrid parallel not support for npu/xpu @@ -308,17 +321,24 @@ class Fleet(object): global_ring_id = 3 if use_sharding else 0 global_ranks = list(range(global_world_size)) - if tp._HYBRID_PARALLEL_GROUP is None: tp._CommunicateGroup() + if tp._HYBRID_PARALLEL_GROUP is None: + tp._CommunicateGroup() cg = tp._HYBRID_PARALLEL_GROUP self._hcg = cg - cg.set_comm_group('global', global_rank, global_world_size, - global_ring_id, global_ranks) + cg.set_comm_group( + 'global', + global_rank, + global_world_size, + global_ring_id, + global_ranks, + ) use_tensor_parallel = self._user_defined_strategy.tensor_parallel use_mp = use_sharding or use_tensor_parallel # hybrid group - if use_mp is False: return + if use_mp is False: + return mp_degree_sharding = 1 mp_degree_tensor_parallel = 1 @@ -327,14 +347,21 @@ class Fleet(object): mp_degree_sharding = int(sharding_configs['mp_degree']) if use_tensor_parallel: - tensor_parallel_configs = self._user_defined_strategy.tensor_parallel_configs + tensor_parallel_configs = ( + self._user_defined_strategy.tensor_parallel_configs + ) mp_degree_tensor_parallel = int( - tensor_parallel_configs['tensor_parallel_degree']) + tensor_parallel_configs['tensor_parallel_degree'] + ) if use_sharding and use_tensor_parallel: assert mp_degree_sharding == mp_degree_tensor_parallel - mp_degree = mp_degree_sharding if use_sharding else mp_degree_tensor_parallel + mp_degree = ( + mp_degree_sharding + if use_sharding + else mp_degree_tensor_parallel + ) if mp_degree > 1: assert global_world_size % mp_degree == 0 @@ -343,16 +370,17 @@ class Fleet(object): mp_rank = global_rank % mp_degree mp_group_id = global_rank // mp_degree mp_group_ranks = [ - idx for idx in global_ranks + idx + for idx in global_ranks if idx // mp_degree == mp_group_id ] - cg.set_comm_group('model', mp_rank, mp_degree, mp_ring_id, - mp_group_ranks) + cg.set_comm_group( + 'model', mp_rank, mp_degree, mp_ring_id, mp_group_ranks + ) return self def _init_hybrid_parallel_env(self): - """initialize the hybrid environment - """ + """initialize the hybrid environment""" self.hybrid_configs = self._user_defined_strategy.hybrid_configs self.dp_degree = self.hybrid_configs["dp_degree"] self.mp_degree = self.hybrid_configs["mp_degree"] @@ -361,7 +389,9 @@ class Fleet(object): assert self.mp_degree >= 0, "mp_degree should be greater or equal to 0" assert self.pp_degree >= 0, "pp_degree should be greater or equal to 0" - assert self.sharding_degree >= 0, "sharding_degree should be greater or equal to 0" + assert ( + self.sharding_degree >= 0 + ), "sharding_degree should be greater or equal to 0" self.mp_degree = max(self.mp_degree, 1) self.pp_degree = max(self.pp_degree, 1) @@ -375,14 +405,19 @@ class Fleet(object): self._topology = tp.CommunicateTopology( hybrid_group_names=["data", "pipe", "sharding", "model"], dims=[ - self.dp_degree, self.pp_degree, self.sharding_degree, - self.mp_degree - ]) + self.dp_degree, + self.pp_degree, + self.sharding_degree, + self.mp_degree, + ], + ) self._hcg = tp.HybridCommunicateGroup(self._topology) if self.mp_degree > 1: - tensor_parallel_configs = self._user_defined_strategy.tensor_parallel_configs + tensor_parallel_configs = ( + self._user_defined_strategy.tensor_parallel_configs + ) tensor_init_seed = tensor_parallel_configs["tensor_init_seed"] if tensor_init_seed == -1: model_parallel_random_seed() @@ -822,29 +857,29 @@ class Fleet(object): for name in fetch_var_names ] - self._runtime_handle._save_inference_model(executor, dirname, - feeded_var_names, - fetch_vars, None, True, - 0) + self._runtime_handle._save_inference_model( + executor, dirname, feeded_var_names, fetch_vars, None, True, 0 + ) else: increment_mode = 0 if "mode" in configs: increment_mode = int(configs["mode"]) - self._runtime_handle._save_persistables(executor, - dirname, - main_program=None, - mode=increment_mode) + self._runtime_handle._save_persistables( + executor, dirname, main_program=None, mode=increment_mode + ) @is_non_distributed_check @inited_runtime_handler - def save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True, - mode=0): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + mode=0, + ): """ save inference model for inference. @@ -865,10 +900,15 @@ class Fleet(object): """ - self._runtime_handle._save_inference_model(executor, dirname, - feeded_var_names, - target_vars, main_program, - export_for_deployment, mode) + self._runtime_handle._save_inference_model( + executor, + dirname, + feeded_var_names, + target_vars, + main_program, + export_for_deployment, + mode, + ) @is_non_distributed_check @inited_runtime_handler @@ -913,8 +953,9 @@ class Fleet(object): fleet.save_persistables(exe, "dirname", paddle.static.default_main_program()) """ - self._runtime_handle._save_persistables(executor, dirname, main_program, - mode) + self._runtime_handle._save_persistables( + executor, dirname, main_program, mode + ) @is_non_distributed_check @inited_runtime_handler @@ -953,12 +994,9 @@ class Fleet(object): @is_non_distributed_check @inited_runtime_handler - def save_dense_params(self, - executor, - dirname, - scope, - program, - var_names=None): + def save_dense_params( + self, executor, dirname, scope, program, var_names=None + ): """ save fleet one table from path @@ -982,8 +1020,9 @@ class Fleet(object): fleet.save_dense_params(exe, "path", scope=paddle.static.global_scope(), program=paddle.static.default_main_program()) """ - self._runtime_handle._save_dense_params(executor, dirname, scope, - program, var_names) + self._runtime_handle._save_dense_params( + executor, dirname, scope, program, var_names + ) def shrink(self, threshold=None): self._runtime_handle._shrink(threshold) @@ -1027,7 +1066,8 @@ class Fleet(object): "in fleet.init(). The strategy here is only for compatibility. " "If the strategy in fleet.distributed_optimizer() is " "not None, then it will overwrite the DistributedStrategy in fleet.init(), " - "which will take effect in distributed training.") + "which will take effect in distributed training." + ) self._user_defined_strategy = copy.deepcopy(strategy) self._context = {} @@ -1046,21 +1086,19 @@ class Fleet(object): if hasattr(self.user_defined_optimizer, 'amp_init'): amp_optimizer = self.user_defined_optimizer - assert amp_optimizer is not None, \ - "amp_init can only be used when the amp(auto mixed precision) strategy is turned on." + assert ( + amp_optimizer is not None + ), "amp_init can only be used when the amp(auto mixed precision) strategy is turned on." return amp_optimizer def get_loss_scaling(self): - """Return the real-time loss scaling factor. - """ + """Return the real-time loss scaling factor.""" amp_optimizer = self._get_amp_optimizer() return amp_optimizer.get_loss_scaling() - def amp_init(self, - place, - scope=None, - test_program=None, - use_fp16_test=False): + def amp_init( + self, place, scope=None, test_program=None, use_fp16_test=False + ): """ Init the amp training, such as cast fp32 parameters to fp16 type. @@ -1145,11 +1183,9 @@ class Fleet(object): else: return self._context["applied_graph_list"] - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ Add distributed operations to minimize ``loss`` by updating ``parameter_list``. @@ -1201,23 +1237,27 @@ class Fleet(object): """ if not isinstance(loss, list): - return self._minimize_impl(loss, startup_program, parameter_list, - no_grad_set) + return self._minimize_impl( + loss, startup_program, parameter_list, no_grad_set + ) else: - if paddle.fluid.framework._non_static_mode( - ) or self._role_maker._is_non_distributed() or self._is_collective: + if ( + paddle.fluid.framework._non_static_mode() + or self._role_maker._is_non_distributed() + or self._is_collective + ): raise ValueError("loss can be list only in PS mode") - return self._minimize_losses_impl(loss, startup_program, - parameter_list, no_grad_set) - - def _minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + return self._minimize_losses_impl( + loss, startup_program, parameter_list, no_grad_set + ) + + def _minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): context = {} context["user_defined_strategy"] = copy.deepcopy( - self._user_defined_strategy) + self._user_defined_strategy + ) if paddle.fluid.framework._non_static_mode(): # imitate target optimizer retrieval target_opt = self.user_defined_optimizer @@ -1230,49 +1270,62 @@ class Fleet(object): if not hasattr(self.origin_main_program, "distributed_info_"): setattr(self.origin_main_program, "distributed_info_", dict()) self.origin_main_program.distributed_info_[ - "dp_degree"] = self._user_defined_strategy.sharding_configs[ - "dp_degree"] + "dp_degree" + ] = self._user_defined_strategy.sharding_configs["dp_degree"] self.origin_main_program.distributed_info_[ - "mp_degree"] = self._user_defined_strategy.sharding_configs[ - "mp_degree"] + "mp_degree" + ] = self._user_defined_strategy.sharding_configs["mp_degree"] self.origin_main_program.distributed_info_[ - "pp_degree"] = self._user_defined_strategy.sharding_configs[ - "pp_degree"] + "pp_degree" + ] = self._user_defined_strategy.sharding_configs["pp_degree"] self.origin_main_program.distributed_info_[ - "sharding_degree"] = self._user_defined_strategy.sharding_configs[ - "sharding_degree"] + "sharding_degree" + ] = self._user_defined_strategy.sharding_configs["sharding_degree"] context["origin_main_program"] = self.origin_main_program context["origin_main_programs"] = [self.origin_main_program] context["loss"] = loss if startup_program == None: - self.origin_startup_program = \ + self.origin_startup_program = ( paddle.static.default_startup_program().clone(for_test=False) + ) startup_program = paddle.static.default_startup_program() else: - self.origin_startup_program = \ - startup_program.clone(for_test=False) + self.origin_startup_program = startup_program.clone(for_test=False) context["origin_startup_program"] = startup_program context["origin_startup_programs"] = [startup_program] context["role_maker"] = self._role_maker # Use the auto-parallel's routines instead - if self._user_defined_strategy.semi_auto or self._user_defined_strategy.auto_search: + if ( + self._user_defined_strategy.semi_auto + or self._user_defined_strategy.auto_search + ): from ..auto_parallel.parallelizer import AutoParallelizer + auto_parallelizer = AutoParallelizer(self) - optimize_ops, params_grads, dist_startup_prog, dist_main_prog = auto_parallelizer.parallelize( - loss, startup_program, parameter_list, no_grad_set) + ( + optimize_ops, + params_grads, + dist_startup_prog, + dist_main_prog, + ) = auto_parallelizer.parallelize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads, dist_startup_prog, dist_main_prog # compile time - distributed_optimizer_list = \ + distributed_optimizer_list = ( MetaOptimizerFactory()._get_valid_meta_optimizers( - self.user_defined_optimizer) + self.user_defined_optimizer + ) + ) context["user_defined_strategy"] = copy.deepcopy( - self._user_defined_strategy) + self._user_defined_strategy + ) copy_user_defined_strategy = copy.deepcopy(self._user_defined_strategy) # trigger the auto-parallel in very strict condition @@ -1290,9 +1343,12 @@ class Fleet(object): can_not_apply_optimizer_list = [] # recall meta optimizers for ranking for opt in distributed_optimizer_list: - opt._set_basic_info(loss, self._role_maker, - self.user_defined_optimizer, - copy_user_defined_strategy) + opt._set_basic_info( + loss, + self._role_maker, + self.user_defined_optimizer, + copy_user_defined_strategy, + ) if opt._can_apply() and not opt._is_graph_out(): valid_optimizer_list.append(opt) elif opt._can_apply() and opt._is_graph_out(): @@ -1300,19 +1356,27 @@ class Fleet(object): else: can_not_apply_optimizer_list.append(opt) # combine recalled meta optimizers to be a valid meta optimizer - meta_optimizer, graph_optimizer = \ - self.strategy_compiler.generate_optimizer( - loss, self._role_maker, self.user_defined_optimizer, - copy_user_defined_strategy, valid_optimizer_list, - valid_graph_optimizer_list) + ( + meta_optimizer, + graph_optimizer, + ) = self.strategy_compiler.generate_optimizer( + loss, + self._role_maker, + self.user_defined_optimizer, + copy_user_defined_strategy, + valid_optimizer_list, + valid_graph_optimizer_list, + ) valid_strategy = self.strategy_compiler._get_valid_strategy( - copy_user_defined_strategy, can_not_apply_optimizer_list) + copy_user_defined_strategy, can_not_apply_optimizer_list + ) context["valid_strategy"] = copy.deepcopy(valid_strategy) logger.debug("valid_strategy: " + str(context["valid_strategy"])) - logger.debug("user_defined_strategy: " + - str(context["user_defined_strategy"])) + logger.debug( + "user_defined_strategy: " + str(context["user_defined_strategy"]) + ) applied_meta_list = self.strategy_compiler._get_applied_meta_list() applied_graph_list = self.strategy_compiler._get_applied_graph_list() @@ -1333,41 +1397,48 @@ class Fleet(object): self._runtime_handle = RuntimeFactory()._create_runtime(context) compiled_program = compiler.CompiledProgram( - self.origin_main_program).with_data_parallel( - loss_name=loss.name, share_vars_from=None) + self.origin_main_program + ).with_data_parallel(loss_name=loss.name, share_vars_from=None) loss.block.program._graph = compiled_program - return self.user_defined_optimizer.minimize(loss, - startup_program, - parameter_list, - no_grad_set=no_grad_set) + return self.user_defined_optimizer.minimize( + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) if meta_optimizer: - logger.debug("before minimize program id: " + - str(id(loss.block.program))) + logger.debug( + "before minimize program id: " + str(id(loss.block.program)) + ) optimize_ops, params_grads = meta_optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) - logger.debug("after minimize program id: " + - str(id(loss.block.program))) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) + logger.debug( + "after minimize program id: " + str(id(loss.block.program)) + ) default_program = paddle.static.default_main_program() logger.debug("default program id: " + str(id(default_program))) if id(default_program) != id(loss.block.program): paddle.fluid.framework.switch_main_program(loss.block.program) - logger.debug("default program id after switch: " + - str(id(default_program))) + logger.debug( + "default program id after switch: " + str(id(default_program)) + ) else: optimize_ops, params_grads = self.user_defined_optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) context["program_optimize_ops"] = optimize_ops context["program_params_grads"] = params_grads if graph_optimizer: - logger.debug("before graph minimize program id: " + - str(id(loss.block.program))) + logger.debug( + "before graph minimize program id: " + + str(id(loss.block.program)) + ) optimize_ops, params_grads = graph_optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) # since we do not encourage users to use graph operations # if a graph optimizer takes effect, mostly # optimizers_ops and params_grads are None @@ -1382,8 +1453,10 @@ class Fleet(object): opt_info = {} if program._fleet_opt is None else program._fleet_opt opt_info["mpi_size"] = self.worker_num() opt_info["mpi_rank"] = self.worker_index() - for k, v in self._user_defined_strategy.trainer_desc_configs.items( - ): + for ( + k, + v, + ) in self._user_defined_strategy.trainer_desc_configs.items(): if v or k not in opt_info: opt_info[k] = v program._fleet_opt = opt_info @@ -1392,15 +1465,18 @@ class Fleet(object): self._runtime_handle = RuntimeFactory()._create_runtime(context) import paddle.distributed.fleet as fleet + fleet.util._set_strategy(context["valid_strategy"]) return optimize_ops, params_grads - def _minimize_losses_impl(self, - losses, - startup_programs=None, - parameter_list=None, - no_grad_set=None): + def _minimize_losses_impl( + self, + losses, + startup_programs=None, + parameter_list=None, + no_grad_set=None, + ): context = {} # cache original feed forward program @@ -1416,7 +1492,8 @@ class Fleet(object): startup_programs = [paddle.static.default_startup_program()] else: raise ValueError( - "startup_program can't be None when loss is list.") + "startup_program can't be None when loss is list." + ) self.origin_startup_program = startup_programs[0].clone(for_test=False) context["origin_startup_program"] = startup_programs[0] context["origin_startup_programs"] = [] @@ -1426,7 +1503,8 @@ class Fleet(object): context["role_maker"] = self._role_maker context["user_defined_strategy"] = copy.deepcopy( - self._user_defined_strategy) + self._user_defined_strategy + ) context["valid_strategy"] = copy.deepcopy(self._user_defined_strategy) @@ -1439,12 +1517,17 @@ class Fleet(object): params_grads = [] from .meta_optimizers import ParameterServerOptimizer + ps_optimizer = ParameterServerOptimizer(self.user_defined_optimizer) - ps_optimizer._set_basic_info(losses, self._role_maker, - self.user_defined_optimizer, - self._user_defined_strategy) + ps_optimizer._set_basic_info( + losses, + self._role_maker, + self.user_defined_optimizer, + self._user_defined_strategy, + ) optimize_ops, params_grads = ps_optimizer.minimize_losses_impl( - losses, startup_programs, parameter_list, no_grad_set=no_grad_set) + losses, startup_programs, parameter_list, no_grad_set=no_grad_set + ) # default_program = paddle.static.default_main_program() @@ -1459,18 +1542,24 @@ class Fleet(object): opt_info = {} if program._fleet_opt is None else program._fleet_opt opt_info["mpi_size"] = self.worker_num() opt_info["mpi_rank"] = self.worker_index() - for k, v in self._user_defined_strategy.trainer_desc_configs.items( - ): + for ( + k, + v, + ) in self._user_defined_strategy.trainer_desc_configs.items(): if v or k not in opt_info: opt_info[k] = v program._fleet_opt = opt_info - logger.debug("fleet base opt info: " + str(id(program)) + - str(program._fleet_opt)) + logger.debug( + "fleet base opt info: " + + str(id(program)) + + str(program._fleet_opt) + ) if self._runtime_handle is None: self._runtime_handle = RuntimeFactory()._create_runtime(context) import paddle.distributed.fleet as fleet + fleet.util._set_strategy(context["valid_strategy"]) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/fleet_executor_utils.py b/python/paddle/distributed/fleet/fleet_executor_utils.py index 48ef34f4603..4eb4108d5d5 100644 --- a/python/paddle/distributed/fleet/fleet_executor_utils.py +++ b/python/paddle/distributed/fleet/fleet_executor_utils.py @@ -22,16 +22,18 @@ class TaskNode: Python side TaskNode, connection to the c++ side TaskNode """ - def __init__(self, - rank, - max_run_times, - max_slot_times, - role=None, - node_type=None, - task_id=0, - ops=None, - program=None, - lazy_initialize=False): + def __init__( + self, + rank, + max_run_times, + max_slot_times, + role=None, + node_type=None, + task_id=0, + ops=None, + program=None, + lazy_initialize=False, + ): """ :param rank (int): Current rank of the task node. :param max_run_times (int): The max run times of the task node. @@ -43,10 +45,12 @@ class TaskNode: :param program (Program): An instance of Program to init the task node. :param lazy_initialize (bool): In user-defined task, the program may change adding feed/fetch op. As efficient consideration, the task node will have the C++ object later. """ - assert ((ops is not None) ^ (program is not None)), \ - "Should provide only one of ops or program to task node." - assert (not ((ops is not None) and lazy_initialize)), \ - "Lazy initialization doesn't support with ops list" + assert (ops is not None) ^ ( + program is not None + ), "Should provide only one of ops or program to task node." + assert not ( + (ops is not None) and lazy_initialize + ), "Lazy initialization doesn't support with ops list" self.id = int(task_id) self.rank = rank self.max_run_times = max_run_times @@ -61,20 +65,28 @@ class TaskNode: self.downstreams = [] if not lazy_initialize: if ops is not None: - assert role is not None and task_id is not None, \ - "If init task node with ops, should provide `role` and `task_id`." - self.node = core.TaskNode(role, ops, rank, task_id, - max_run_times, max_slot_times) + assert ( + role is not None and task_id is not None + ), "If init task node with ops, should provide `role` and `task_id`." + self.node = core.TaskNode( + role, ops, rank, task_id, max_run_times, max_slot_times + ) else: - self.node = core.TaskNode(program.desc, rank, self.id, - max_run_times, max_slot_times) + self.node = core.TaskNode( + program.desc, rank, self.id, max_run_times, max_slot_times + ) if self.node_type: self.node.set_type(self.node_type) def task_node(self): if self.lazy_initialize: - self.node = core.TaskNode(self.program.desc, self.rank, self.id, - self.max_run_times, self.max_slot_times) + self.node = core.TaskNode( + self.program.desc, + self.rank, + self.id, + self.max_run_times, + self.max_slot_times, + ) if self.node_type: self.node.set_type(self.node_type) if self.run_pre_steps: @@ -89,12 +101,15 @@ class TaskNode: return self.node def set_program(self, program): - assert self.lazy_initialize, \ - "Inside program is unchangable for immediate initialized task node. Set the lazy_initialize to be true if the inside program need to be update. Remember to do all your change before eval node.task_node()." + assert ( + self.lazy_initialize + ), "Inside program is unchangable for immediate initialized task node. Set the lazy_initialize to be true if the inside program need to be update. Remember to do all your change before eval node.task_node()." self.program = program def get_program(self): - assert self.program is not None, "The task node is not initialized using program" + assert ( + self.program is not None + ), "The task node is not initialized using program" return self.program def set_run_pre_steps(self, steps): @@ -142,10 +157,16 @@ class CoordSys: :param coord: The coord to be tested :return: False if valid, True if invalid. """ - return coord['mp_idx'] < 0 or coord['mp_idx'] >= self.mp_degree or \ - coord['sharding_idx'] < 0 or coord['sharding_idx'] >= self.sharding_degree or \ - coord['pp_idx'] < 0 or coord['pp_idx'] >= self.pp_degree or \ - coord['dp_idx'] < 0 or coord['dp_idx'] >= self.dp_degree + return ( + coord['mp_idx'] < 0 + or coord['mp_idx'] >= self.mp_degree + or coord['sharding_idx'] < 0 + or coord['sharding_idx'] >= self.sharding_degree + or coord['pp_idx'] < 0 + or coord['pp_idx'] >= self.pp_degree + or coord['dp_idx'] < 0 + or coord['dp_idx'] >= self.dp_degree + ) def coord_to_rank(self, coord): """ @@ -155,9 +176,15 @@ class CoordSys: """ if self._invalide_coord(coord): return -1 - return int(coord['dp_idx'] * self.pp_degree * self.sharding_degree * self.mp_degree + \ - coord['pp_idx'] * self.sharding_degree * self.mp_degree + \ - coord['sharding_idx'] * self.mp_degree + coord['mp_idx']) + return int( + coord['dp_idx'] + * self.pp_degree + * self.sharding_degree + * self.mp_degree + + coord['pp_idx'] * self.sharding_degree * self.mp_degree + + coord['sharding_idx'] * self.mp_degree + + coord['mp_idx'] + ) def rank_to_coord(self, rank): """ @@ -176,17 +203,14 @@ class CoordSys: 'mp_idx': int(mp_idx), 'sharding_idx': int(sharding_idx), 'pp_idx': int(pp_idx), - 'dp_idx': int(dp_idx) + 'dp_idx': int(dp_idx), } class FleetExecutorUtils: - - def __init__(self, - dist_strategy=None, - rank=None, - nrank=None, - max_run_times=None): + def __init__( + self, dist_strategy=None, rank=None, nrank=None, max_run_times=None + ): self.dist_strategy = dist_strategy self.rank = rank self.nrank = nrank @@ -206,12 +230,14 @@ class FleetExecutorUtils: return op_role == int(OpRole.Optimize.LRSched) def is_forward_op(self, op_role): - return (op_role == int(OpRole.Forward)) or \ - (op_role == (int(OpRole.Forward) | int(OpRole.Loss))) + return (op_role == int(OpRole.Forward)) or ( + op_role == (int(OpRole.Forward) | int(OpRole.Loss)) + ) def is_backward_op(self, op_role): - return (op_role == int(OpRole.Backward)) or \ - (op_role == (int(OpRole.Backward) | int(OpRole.Loss))) + return (op_role == int(OpRole.Backward)) or ( + op_role == (int(OpRole.Backward) | int(OpRole.Loss)) + ) def split_program_to_op_list(self, program): op_list_map = {"lr": [], "fwd": [], "bwd": [], "opt": []} @@ -233,17 +259,19 @@ class FleetExecutorUtils: return op_list_map def convert_op_list_to_program(self, op_list, complete_program): - #TODO(liyurui): Complete this convert logic + # TODO(liyurui): Complete this convert logic program_map = { "lr": Program(), "fwd": Program(), "bwd": Program(), - "opt": Program() + "opt": Program(), } return program_map def build_1f1b_dependency(self, task_node_map): - assert not self.is_auto_parallel, "Handly add dependency should not be invoked in auto parallel mode" + assert ( + not self.is_auto_parallel + ), "Handly add dependency should not be invoked in auto parallel mode" # Generated the dependency based on this graph: # lr(1:m) -> forward -> backward -> (m:1)optimize # ↑ ↓ @@ -253,8 +281,9 @@ class FleetExecutorUtils: # add dependency intra stage cur_start_id = self.rank * self.num_of_functionality - pp_buff_size = int(self.dist_strategy['pp_degree'] - - self.coord['pp_idx']) + pp_buff_size = int( + self.dist_strategy['pp_degree'] - self.coord['pp_idx'] + ) task_node_map["lr"].add_downstream_task(cur_start_id + 1) task_node_map["fwd"].add_upstream_task(cur_start_id) task_node_map["fwd"].add_downstream_task(cur_start_id + 2, pp_buff_size) @@ -267,8 +296,8 @@ class FleetExecutorUtils: downstream_coord['pp_idx'] = downstream_coord['pp_idx'] + 1 pp_upstream = self.coord_sys.coord_to_rank(upstream_coord) pp_downstream = self.coord_sys.coord_to_rank(downstream_coord) - first_stage = (pp_upstream == -1) - last_stage = (pp_downstream == -1) + first_stage = pp_upstream == -1 + last_stage = pp_downstream == -1 prev_pp_start_id = pp_upstream * self.num_of_functionality next_pp_start_id = pp_downstream * self.num_of_functionality if not first_stage: @@ -282,31 +311,39 @@ class FleetExecutorUtils: def construct_task_nodes_1f1b(self, program_map): max_slot_times = int(self.max_run_times - self.coord['pp_idx']) cur_start_id = int(self.rank * self.num_of_functionality) - lr_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - program=program_map["lr"], - task_id=cur_start_id) - fwd_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - program=program_map["fwd"], - task_id=cur_start_id + 1) - bwd_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - program=program_map["bwd"], - task_id=cur_start_id + 2) - opt_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - program=program_map["opt"], - task_id=cur_start_id + 3) + lr_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + program=program_map["lr"], + task_id=cur_start_id, + ) + fwd_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + program=program_map["fwd"], + task_id=cur_start_id + 1, + ) + bwd_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + program=program_map["bwd"], + task_id=cur_start_id + 2, + ) + opt_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + program=program_map["opt"], + task_id=cur_start_id + 3, + ) return { "lr": lr_task_node, "fwd": fwd_task_node, "bwd": bwd_task_node, - "opt": opt_task_node + "opt": opt_task_node, } def task_id_to_rank(self): @@ -319,51 +356,61 @@ class FleetExecutorUtils: def construct_task_nodes_1f1b_op_list(self, op_list_map): max_slot_times = int(self.max_run_times - self.coord['pp_idx']) cur_start_id = int(self.rank * self.num_of_functionality) - lr_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - role=int(OpRole.Optimize.LRSched), - ops=op_list_map["lr"], - task_id=cur_start_id, - node_type="Amplifier") + lr_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + role=int(OpRole.Optimize.LRSched), + ops=op_list_map["lr"], + task_id=cur_start_id, + node_type="Amplifier", + ) lr_task_node.set_run_pre_steps(self.max_run_times) - fwd_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - role=int(OpRole.Forward), - ops=op_list_map["fwd"], - task_id=cur_start_id + 1, - node_type="Compute") - bwd_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - role=int(OpRole.Backward), - ops=op_list_map["bwd"], - task_id=cur_start_id + 2, - node_type="Compute") - opt_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - role=int(OpRole.Optimize), - ops=op_list_map["opt"], - task_id=cur_start_id + 3, - node_type="Amplifier") + fwd_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + role=int(OpRole.Forward), + ops=op_list_map["fwd"], + task_id=cur_start_id + 1, + node_type="Compute", + ) + bwd_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + role=int(OpRole.Backward), + ops=op_list_map["bwd"], + task_id=cur_start_id + 2, + node_type="Compute", + ) + opt_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + role=int(OpRole.Optimize), + ops=op_list_map["opt"], + task_id=cur_start_id + 3, + node_type="Amplifier", + ) opt_task_node.set_run_pre_steps(self.max_run_times) opt_task_node.set_run_at_offset(self.max_run_times - 1) return { "lr": lr_task_node, "fwd": fwd_task_node, "bwd": bwd_task_node, - "opt": opt_task_node + "opt": opt_task_node, } -def run1f1b(program, - rank, - max_run_times, - dist_opt, - nrank, - with_standalone_executor=False): +def run1f1b( + program, + rank, + max_run_times, + dist_opt, + nrank, + with_standalone_executor=False, +): """ Split the program to support 1f1b pipeline scheduler. This funct will split the program based on the op_role. @@ -380,24 +427,29 @@ def run1f1b(program, task_id_to_rank (dict): task nodes' ids to it's corresponding rank """ print("fleet executor will use python side 1f1b scheduler.") - fleet_executor_utils = FleetExecutorUtils(dist_strategy=dist_opt, - rank=rank, - nrank=nrank, - max_run_times=max_run_times) + fleet_executor_utils = FleetExecutorUtils( + dist_strategy=dist_opt, + rank=rank, + nrank=nrank, + max_run_times=max_run_times, + ) op_list_map = fleet_executor_utils.split_program_to_op_list(program) task_node_map = None if with_standalone_executor: program_map = fleet_executor_utils.convert_op_list_to_program( - op_list_map, program) + op_list_map, program + ) task_node_map = fleet_executor_utils.construct_task_nodes_1f1b( - program_map) + program_map + ) else: op_desc_list_map = {"lr": [], "fwd": [], "bwd": [], "opt": []} for key in op_list_map: for op in op_list_map[key]: op_desc_list_map[key].append(op.desc) task_node_map = fleet_executor_utils.construct_task_nodes_1f1b_op_list( - op_desc_list_map) + op_desc_list_map + ) task_node_map = fleet_executor_utils.build_1f1b_dependency(task_node_map) task_id_to_rank = fleet_executor_utils.task_id_to_rank() task_node_list = [task_node_map[key].task_node() for key in task_node_map] @@ -414,10 +466,12 @@ def origin(program, rank): task_id_to_rank (dict): a fake dict, since there is no upstream or downstream, this dict won't be used """ print("fleet executor will use python side origin scheduler.") - task_node = TaskNode(program=program, - rank=rank, - node_type="Compute", - max_run_times=1, - max_slot_times=1) + task_node = TaskNode( + program=program, + rank=rank, + node_type="Compute", + max_run_times=1, + max_slot_times=1, + ) task_id_to_rank = {task_node.task_id(): rank} return [task_node.task_node()], task_id_to_rank diff --git a/python/paddle/distributed/fleet/launch.py b/python/paddle/distributed/fleet/launch.py index a50c08cbaf9..ca301c52a7f 100755 --- a/python/paddle/distributed/fleet/launch.py +++ b/python/paddle/distributed/fleet/launch.py @@ -65,10 +65,21 @@ from argparse import ArgumentParser, REMAINDER import paddle.fluid as fluid from paddle.distributed.fleet import launch_utils from paddle.distributed.fleet.launch_utils import ( - get_host_name_ip, find_free_ports, logger, get_cluster, DeviceMode, - start_local_trainers, direct_start, watch_local_trainers, - terminate_local_procs, DistributeMode, ParameterServerLauncher, get_logger, - check_backend, block_windows_and_macos) + get_host_name_ip, + find_free_ports, + logger, + get_cluster, + DeviceMode, + start_local_trainers, + direct_start, + watch_local_trainers, + terminate_local_procs, + DistributeMode, + ParameterServerLauncher, + get_logger, + check_backend, + block_windows_and_macos, +) from paddle.distributed.fleet import cloud_utils from paddle.distributed.fleet import ascend_utils @@ -92,33 +103,38 @@ def _parse_args(): parser = ArgumentParser( description='''start paddle training using multi-process mode. see: http://www.paddlepaddle.org/documentation/docs/zh/1.6/user_guides/howto/training/cluster_howto.html#permalink-8--nccl2- -''') +''' + ) base_group = parser.add_argument_group("Base Parameters") base_group.add_argument( "--log_dir", type=str, default="log", - help="The path for each process's log. Default --log_dir=log/") + help="The path for each process's log. Default --log_dir=log/", + ) base_group.add_argument( "--backend", type=str, default=os.environ.get('PADDLE_DISTRI_BACKEND', 'auto'), help="Specifize the backend, can be gloo|nccl|bkcl|auto|hccl|heter. " - "Default value is auto which perfers nccl or bkcl.") + "Default value is auto which perfers nccl or bkcl.", + ) base_group.add_argument( "--nproc_per_node", type=int, default=None, help="The number of processes to launch on a node." "In gpu training, it should be less or equal to the gpus number of you system(or you set by --gpus). And so each process can" - " bound to one or average number of gpus.") + " bound to one or average number of gpus.", + ) base_group.add_argument( "--run_mode", type=str, default=None, - help="run mode of job, can be:collective/ps/ps-heter") + help="run mode of job, can be:collective/ps/ps-heter", + ) if fluid.core.is_compiled_with_cuda(): base_group.add_argument( @@ -127,7 +143,7 @@ see: http://www.paddlepaddle.org/documentation/docs/zh/1.6/user_guides/howto/tra default=None, help="It's for gpu training." "For example:" - "--gpus=\"0,1,2,3\" will launch four training processes each bound to one gpu." + "--gpus=\"0,1,2,3\" will launch four training processes each bound to one gpu.", ) base_group.add_argument("--selected_gpus", dest="gpus") @@ -137,7 +153,7 @@ see: http://www.paddlepaddle.org/documentation/docs/zh/1.6/user_guides/howto/tra type=str, default=None, help="It's for xpu training. For example: " - "--xpus=\"0,1,2,3\" will launch four training processes each bound to one xpu." + "--xpus=\"0,1,2,3\" will launch four training processes each bound to one xpu.", ) base_group.add_argument("--selected_xpus", dest="xpus") @@ -147,7 +163,7 @@ see: http://www.paddlepaddle.org/documentation/docs/zh/1.6/user_guides/howto/tra type=str, default=None, help="It's for xpu training. For example: " - "--npus=\"0,1,2,3\" will launch four training processes each bound to one npu." + "--npus=\"0,1,2,3\" will launch four training processes each bound to one npu.", ) base_group.add_argument("--selected_npus", dest="npus") @@ -157,16 +173,18 @@ see: http://www.paddlepaddle.org/documentation/docs/zh/1.6/user_guides/howto/tra type=str, default=None, help="It's for mlu training. For example: " - "--mlus=\"0,1,2,3\" will launch four training processes each bound to one mlu." + "--mlus=\"0,1,2,3\" will launch four training processes each bound to one mlu.", ) base_group.add_argument("--selected_mlus", dest="mlus") - base_group.add_argument("training_script", - type=str, - help="The full path to the single GPU training " - "program/script to be launched in parallel, " - "followed by all the arguments for the " - "training script") + base_group.add_argument( + "training_script", + type=str, + help="The full path to the single GPU training " + "program/script to be launched in parallel, " + "followed by all the arguments for the " + "training script", + ) base_group.add_argument('training_script_args', nargs=REMAINDER) @@ -177,79 +195,86 @@ see: http://www.paddlepaddle.org/documentation/docs/zh/1.6/user_guides/howto/tra "--ips", type=str, default="127.0.0.1", - help="Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17..") + help="Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17..", + ) collective_group.add_argument( "--cluster_topo_path", type=str, default=None, help="A json format file will be stored in this path which is used" - "to represent the cluster topology information for auto parallel.") + "to represent the cluster topology information for auto parallel.", + ) collective_group.add_argument( "--rank_mapping_path", type=str, default=None, help="A json format file will be stored in this path which is used" - "to map processes to machines for auto parallel.") + "to map processes to machines for auto parallel.", + ) collective_group.add_argument( "--enable_auto_mapping", type=bool, default=False, - help="Set true to enable the lazy launch for auto-parallel scenario.") + help="Set true to enable the lazy launch for auto-parallel scenario.", + ) ps_group = parser.add_argument_group("Parameter-Server Parameters") # for parameter server - ps_group.add_argument("--servers", - type=str, - default="", - help="User defined servers ip:port") - ps_group.add_argument("--workers", - type=str, - default="", - help="User defined workers ip:port") - ps_group.add_argument("--coordinators", - type=str, - default="", - help="User defined coordinators ip:port") + ps_group.add_argument( + "--servers", type=str, default="", help="User defined servers ip:port" + ) + ps_group.add_argument( + "--workers", type=str, default="", help="User defined workers ip:port" + ) + ps_group.add_argument( + "--coordinators", + type=str, + default="", + help="User defined coordinators ip:port", + ) ps_group.add_argument( "--heter_workers", type=str, default="", - help="User defined heter workers in each stage ip1:port1;ip2:port2") + help="User defined heter workers in each stage ip1:port1;ip2:port2", + ) ps_group.add_argument( "--heter_devices", type=str, default="", - help="User defined heter devices in each stage cpu;gpu;cpu") + help="User defined heter devices in each stage cpu;gpu;cpu", + ) ps_group.add_argument("--worker_num", type=int, help="number of workers") - ps_group.add_argument("--coordinator_num", - type=int, - help="number of coordinators") + ps_group.add_argument( + "--coordinator_num", type=int, help="number of coordinators" + ) ps_group.add_argument("--server_num", type=int, help="number of servers") - ps_group.add_argument("--heter_worker_num", - type=str, - help="number of heter_workers in each stage 1;2;3") + ps_group.add_argument( + "--heter_worker_num", + type=str, + help="number of heter_workers in each stage 1;2;3", + ) ps_group.add_argument("--http_port", type=int, help="Gloo http Port") # parameter elastic mode elastic_group = parser.add_argument_group("Elastic Parameters") - elastic_group.add_argument("--elastic_server", - type=str, - help="etcd server host:port") - elastic_group.add_argument("--elastic_pre_hook", - type=str, - help="elastic pre_hook shell cmd") + elastic_group.add_argument( + "--elastic_server", type=str, help="etcd server host:port" + ) + elastic_group.add_argument( + "--elastic_pre_hook", type=str, help="elastic pre_hook shell cmd" + ) elastic_group.add_argument("--job_id", type=str, help="job unique id") elastic_group.add_argument("--np", type=int, help="job pod/node number") elastic_group.add_argument("--scale", type=int, default=0, help="scale np") - elastic_group.add_argument("--host", - type=str, - help="bind host, default to POD_IP env") - elastic_group.add_argument("--force", - type=bool, - default=False, - help="update np force") + elastic_group.add_argument( + "--host", type=str, help="bind host, default to POD_IP env" + ) + elastic_group.add_argument( + "--force", type=bool, default=False, help="update np force" + ) known_args, _ = parser.parse_known_args() return known_args @@ -265,16 +290,23 @@ def get_cluster_from_args(args, device_mode, devices_per_proc): else: _, node_ip = get_host_name_ip() - assert node_ip in node_ips, "Can't find your local ip {%s} in node_ips: {%s}" \ - % (node_ip, node_ips) + assert ( + node_ip in node_ips + ), "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) node_rank = node_ips.index(node_ip) - logger.debug("parsed from args: node_ips:{} node_ip:{} node_rank:{}".format( - node_ips, node_ip, node_rank)) + logger.debug( + "parsed from args: node_ips:{} node_ip:{} node_rank:{}".format( + node_ips, node_ip, node_rank + ) + ) free_ports = None - if not cloud_utils.use_paddlecloud() and len( - node_ips) <= 1 and os.environ.get('FLAGS_START_PORT') is None: + if ( + not cloud_utils.use_paddlecloud() + and len(node_ips) <= 1 + and os.environ.get('FLAGS_START_PORT') is None + ): free_ports = find_free_ports(len(devices_per_proc)) if free_ports is not None: free_ports = list(free_ports) @@ -291,17 +323,21 @@ def get_cluster_from_args(args, device_mode, devices_per_proc): trainer_endpoints = [] for ip in node_ips: trainer_endpoints.append(["%s:%d" % (ip, port) for port in free_ports]) - return get_cluster(node_ips, node_ip, trainer_endpoints, device_mode, - devices_per_proc) + return get_cluster( + node_ips, node_ip, trainer_endpoints, device_mode, devices_per_proc + ) def cpuonly_check(args): if args.ips and len(args.ips.split(',')) > 1: raise RuntimeError( "CPUONLY launch only support single trainer, that is len(ips)=1, but got %s." - % args.ips) + % args.ips + ) if args.run_mode: - assert args.run_mode == 'cpuonly', "CPUONLY launch only support run mode is CPUONLY" + assert ( + args.run_mode == 'cpuonly' + ), "CPUONLY launch only support run mode is CPUONLY" if args.servers: raise RuntimeError("CPUONLY launch can't have --servers as arguments.") return True @@ -309,15 +345,20 @@ def cpuonly_check(args): def get_cluster_info(args): # parse arguments, used for cloud-single-machine and local - if args.backend == 'gloo': cpuonly_check(args) + if args.backend == 'gloo': + cpuonly_check(args) if args.enable_auto_mapping: (device_mode, devices_per_proc) = (DeviceMode.GPU, []) else: - (device_mode, - devices_per_proc) = launch_utils.get_device_proc_info(args) + (device_mode, devices_per_proc) = launch_utils.get_device_proc_info( + args + ) trainers_num = cloud_utils.get_trainers_num() - logger.debug("parsed from args trainerss_num:{} mode:{} devices:{}".format( - trainers_num, device_mode, devices_per_proc)) + logger.debug( + "parsed from args trainerss_num:{} mode:{} devices:{}".format( + trainers_num, device_mode, devices_per_proc + ) + ) cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") @@ -329,52 +370,69 @@ def get_cluster_info(args): start_port = os.environ.get('FLAGS_START_PORT') # auto mapping between processes and devices for auto-parallel if args.enable_auto_mapping == True: - assert args.cluster_topo_path is not None, \ - "The cluster topology must be provied when enabling auto mapping." + assert ( + args.cluster_topo_path is not None + ), "The cluster topology must be provied when enabling auto mapping." rank_mapping_path = args.rank_mapping_path or os.getenv( - "PADDLE_RANK_MAPPING_PATH") + "PADDLE_RANK_MAPPING_PATH" + ) if not rank_mapping_path: os.environ["PADDLE_NEED_RANK_MAPPING"] = str(True) os.environ["PADDLE_ENABLE_ELASTIC"] = str( - enable_elastic(args, device_mode)) + enable_elastic(args, device_mode) + ) cwd = pathlib.Path().resolve() - rank_mapping_path = os.path.join(cwd, - "auto_parallel_rank_mapping.json") + rank_mapping_path = os.path.join( + cwd, "auto_parallel_rank_mapping.json" + ) os.environ["PADDLE_RANK_MAPPING_PATH"] = str(rank_mapping_path) original_args = sys.argv[1:] os.environ["PADDLE_ORIGINAL_CMD_ARGS"] = " ".join(original_args) os.environ["PADDLE_CLUSTER_TOPO_PATH"] = str(args.cluster_topo_path) os.environ["PADDLE_ENABLE_AUTO_MAPPING"] = str( - args.enable_auto_mapping) - cluster, pod = launch_utils.get_mapped_cluster_from_args_without_rank_mapping( - args, device_mode) + args.enable_auto_mapping + ) + ( + cluster, + pod, + ) = launch_utils.get_mapped_cluster_from_args_without_rank_mapping( + args, device_mode + ) else: os.environ["PADDLE_NEED_RANK_MAPPING"] = str(False) os.environ["PADDLE_ENABLE_ELASTIC"] = str( - enable_elastic(args, device_mode)) + enable_elastic(args, device_mode) + ) os.environ["PADDLE_CLUSTER_TOPO_PATH"] = str(args.cluster_topo_path) os.environ["PADDLE_RANK_MAPPING_PATH"] = str(rank_mapping_path) os.environ["PADDLE_ENABLE_AUTO_MAPPING"] = str( - args.enable_auto_mapping) - cluster, pod = launch_utils.get_mapped_cluster_from_args_with_rank_mapping( - args, device_mode) + args.enable_auto_mapping + ) + ( + cluster, + pod, + ) = launch_utils.get_mapped_cluster_from_args_with_rank_mapping( + args, device_mode + ) elif cloud_utils.use_paddlecloud() and trainers_num != 1: - cluster, pod = cloud_utils.get_cloud_cluster(args.ips, device_mode, - devices_per_proc, - start_port) + cluster, pod = cloud_utils.get_cloud_cluster( + args.ips, device_mode, devices_per_proc, start_port + ) logger.debug("get cluster from cloud:{}".format(cluster)) elif device_mode == DeviceMode.ASCEND_NPU: # for ascend - cluster, pod = ascend_utils.get_cloud_cluster(rank_table_file=os.getenv( - "RANK_TABLE_FILE", None), - device_mode=device_mode, - start_port=start_port) + cluster, pod = ascend_utils.get_cloud_cluster( + rank_table_file=os.getenv("RANK_TABLE_FILE", None), + device_mode=device_mode, + start_port=start_port, + ) else: # trainers_num = 1 or not use paddlecloud ips="a,b" - cluster, pod = get_cluster_from_args(args, device_mode, - devices_per_proc) + cluster, pod = get_cluster_from_args( + args, device_mode, devices_per_proc + ) logger.debug("get cluster from args:{}".format(cluster)) return cluster, pod @@ -394,12 +452,14 @@ def launch_collective(args): cluster, pod = get_cluster_info(args) global_envs = get_global_envs(args, tmp_dir) - procs = start_local_trainers(cluster, - pod, - training_script=args.training_script, - training_script_args=args.training_script_args, - log_dir=args.log_dir, - envs=global_envs) + procs = start_local_trainers( + cluster, + pod, + training_script=args.training_script, + training_script_args=args.training_script_args, + log_dir=args.log_dir, + envs=global_envs, + ) for idx, proc in enumerate(procs): print("launch proc_id:{} idx:{}".format(proc.proc.pid, idx)) @@ -431,7 +491,7 @@ def launch_ps(args, distribute_mode): if cloud_flag and distribute_mode == DistributeMode.PS: direct_start(args) return - #elif cloud_flag and distribute_mode == DistributeMode.PS_HETER: + # elif cloud_flag and distribute_mode == DistributeMode.PS_HETER: # cloud_ps_heter_env_set(args) # args.workers = os.getenv("PADDLE_TRAINER_ENDPOINTS") # args.servers = os.getenv("PADDLE_PSERVERS_IP_PORT_LIST") @@ -443,7 +503,8 @@ def launch_ps(args, distribute_mode): def infer_backend(args): - if args.backend != "auto": return + if args.backend != "auto": + return if fluid.core.is_compiled_with_cuda(): args.backend = 'nccl' elif fluid.core.is_compiled_with_npu(): @@ -469,8 +530,14 @@ def which_distributed_mode(args): return DistributeMode.PS_HETER ps_args = [ - '--worker_num', '--server_num', '--heter_worker_num', '--servers', - '--workers', '--heter_workers', '--heter_devices', '--http_port' + '--worker_num', + '--server_num', + '--heter_worker_num', + '--servers', + '--workers', + '--heter_workers', + '--heter_devices', + '--http_port', ] collective_args = ['--ips'] @@ -482,7 +549,8 @@ def which_distributed_mode(args): ps_arg for ps_arg in ps_args if ps_arg in " ".join(sys.argv[1:-1]) ] has_collective_args = [ - co_arg for co_arg in collective_args + co_arg + for co_arg in collective_args if co_arg in " ".join(sys.argv[1:-1]) ] @@ -504,8 +572,10 @@ def which_distributed_mode(args): if len(has_ps_args) > 0: logger.info( - "Run parameter-sever mode. pserver arguments:{}, accelerators count:{}" - .format(has_ps_args, accelerators)) + "Run parameter-sever mode. pserver arguments:{}, accelerators count:{}".format( + has_ps_args, accelerators + ) + ) has_ps_heter_args = list(set(has_ps_args) & set(ps_heter_args)) has_coordinator_args = list(set(has_ps_args) & set(coordinator_args)) if len(has_ps_heter_args) > 0: @@ -515,23 +585,29 @@ def which_distributed_mode(args): elif len(has_collective_args) > 0: logger.info( "Run collective mode. gpu arguments:{}, cuda count:{}".format( - has_collective_args, accelerators)) + has_collective_args, accelerators + ) + ) return DistributeMode.COLLECTIVE else: - if not fluid.core.is_compiled_with_cuda( - ) and not fluid.core.is_compiled_with_xpu( - ) and not fluid.core.is_compiled_with_mlu(): + if ( + not fluid.core.is_compiled_with_cuda() + and not fluid.core.is_compiled_with_xpu() + and not fluid.core.is_compiled_with_mlu() + ): if args.servers: logger.warning( "Not found distinct arguments and not compiled with cuda or xpu or npu or mlu. " - "But found args.servers not empty, default use ps mode") + "But found args.servers not empty, default use ps mode" + ) return DistributeMode.PS else: return DistributeMode.COLLECTIVE else: logger.warning( "Not found distinct arguments and compiled with cuda or xpu or npu or mlu. " - "Default use collective mode") + "Default use collective mode" + ) return DistributeMode.COLLECTIVE @@ -716,19 +792,23 @@ def launch(): if args.backend == 'auto': distribute_mode = which_distributed_mode( - args) # which_distributed_mode must modify args.backend + args + ) # which_distributed_mode must modify args.backend else: - assert args.run_mode == 'collective' or args.run_mode == None, "When backend is not 'auto', run mode must be collective" + assert ( + args.run_mode == 'collective' or args.run_mode == None + ), "When backend is not 'auto', run mode must be collective" check_backend(args.backend) distribute_mode = DistributeMode.COLLECTIVE - #assert args.backend in ['gloo', 'nccl', 'bkcl', 'cncl', 'heter', 'unknown'] + # assert args.backend in ['gloo', 'nccl', 'bkcl', 'cncl', 'heter', 'unknown'] if args.backend == 'gloo': logger.warning("launch start with CPUONLY mode") block_windows_and_macos( - args.backend) # raise error when using gloo on windows or macos + args.backend + ) # raise error when using gloo on windows or macos if enable_elastic(args, distribute_mode): launch_elastic(args, distribute_mode) diff --git a/python/paddle/distributed/fleet/launch_utils.py b/python/paddle/distributed/fleet/launch_utils.py index d29741cbef6..4ec2aa07787 100755 --- a/python/paddle/distributed/fleet/launch_utils.py +++ b/python/paddle/distributed/fleet/launch_utils.py @@ -35,19 +35,21 @@ logger = logging.getLogger("root") logger.propagate = False -class DistributeMode(): +class DistributeMode: """ There are various mode for fleetrun, each of them is designed for different model. """ + COLLECTIVE = 0 PS = 1 PS_HETER = 2 -class DeviceMode(): +class DeviceMode: """ Training devices type """ + UNKNOWN = -1 CPU = 0 GPU = 1 @@ -59,7 +61,6 @@ class DeviceMode(): class Cluster(object): - def __init__(self, hdfs): self.job_server = None self.pods = [] @@ -68,8 +69,11 @@ class Cluster(object): def __str__(self): return "job_server:{} pods:{} job_stage_flag:{} hdfs:{}".format( - self.job_server, [str(pod) for pod in self.pods], - self.job_stage_flag, self.hdfs) + self.job_server, + [str(pod) for pod in self.pods], + self.job_stage_flag, + self.hdfs, + ) def __eq__(self, cluster): if len(self.pods) != len(cluster.pods): @@ -115,8 +119,9 @@ class Cluster(object): r = [] for pod in self.pods: ep = "{}:{}".format(pod.addr, pod.port) - assert pod.port != None and pod.addr != None, "{} not a valid endpoint".format( - ep) + assert ( + pod.port != None and pod.addr != None + ), "{} not a valid endpoint".format(ep) r.append(ep) return r @@ -129,7 +134,6 @@ class Cluster(object): class JobServer(object): - def __init__(self): self.endpoint = None @@ -144,7 +148,6 @@ class JobServer(object): class Trainer(object): - def __init__(self): self.accelerators = [] self.endpoint = None @@ -153,14 +156,14 @@ class Trainer(object): def __str__(self): return "accelerator:{} endpoint:{} rank:{}".format( - self.accelerators, self.endpoint, self.rank) + self.accelerators, self.endpoint, self.rank + ) def __eq__(self, t): if len(self.accelerators) != len(t.accelerators): return False - if self.endpoint != t.endpoint or \ - self.rank != t.rank: + if self.endpoint != t.endpoint or self.rank != t.rank: return False for a, b in zip(self.accelerators, t.accelerators): @@ -177,7 +180,6 @@ class Trainer(object): class Pod(object): - def __init__(self): self.rank = None self.id = None @@ -194,29 +196,39 @@ class Pod(object): def __str__(self): return "rank:{} id:{} addr:{} port:{} visible_accelerator:{} trainers:{} servers:{} \ workers:{} heter_workers:{} coordinators:{}".format( - self.rank, self.id, self.addr, self.port, self.accelerators, - [str(t) for t in self.trainers], [str(s) for s in self.servers], - [str(w) - for w in self.workers], [str(h) for h in self.heter_workers], - [str(c) for c in self.coordinators]) + self.rank, + self.id, + self.addr, + self.port, + self.accelerators, + [str(t) for t in self.trainers], + [str(s) for s in self.servers], + [str(w) for w in self.workers], + [str(h) for h in self.heter_workers], + [str(c) for c in self.coordinators], + ) def __eq__(self, pod): - if self.rank != pod.rank or \ - self.id != pod.id or \ - self.addr != pod.addr or \ - self.port != pod.port: + if ( + self.rank != pod.rank + or self.id != pod.id + or self.addr != pod.addr + or self.port != pod.port + ): logger.debug("pod {} != {}".format(self, pod)) return False if len(self.trainers) != len(pod.trainers): - logger.debug("trainers {} != {}".format(self.trainers, - pod.trainers)) + logger.debug( + "trainers {} != {}".format(self.trainers, pod.trainers) + ) return False for i in range(len(self.trainers)): if self.trainers[i] != pod.trainers[i]: - logger.debug("trainer {} != {}".format(self.trainers[i], - pod.trainers[i])) + logger.debug( + "trainer {} != {}".format(self.trainers[i], pod.trainers[i]) + ) return False if len(self.servers) != len(pod.servers): @@ -225,8 +237,9 @@ class Pod(object): for i in range(len(self.servers)): if self.servers[i] != pod.servers[i]: - logger.debug("servers {} != {}".format(self.servers[i], - pod.servers[i])) + logger.debug( + "servers {} != {}".format(self.servers[i], pod.servers[i]) + ) return False if len(self.workers) != len(pod.workers): @@ -235,8 +248,9 @@ class Pod(object): for i in range(len(self.workers)): if self.workers[i] != pod.workers[i]: - logger.debug("workers {} != {}".format(self.workers[i], - pod.workers[i])) + logger.debug( + "workers {} != {}".format(self.workers[i], pod.workers[i]) + ) return False return True @@ -267,15 +281,17 @@ def get_logger(log_level=20, name="root"): log_handler = logging.StreamHandler() log_format = logging.Formatter( - '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s') + '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s' + ) log_handler.setFormatter(log_format) logger.addHandler(log_handler) return logger -def get_cluster(node_ips, node_ip, trainer_endpoints, device_mode, - devices_per_proc): +def get_cluster( + node_ips, node_ip, trainer_endpoints, device_mode, devices_per_proc +): assert type(trainer_endpoints) is list, "trainer_endpoints must be list" cluster = Cluster(hdfs=None) trainer_rank = 0 @@ -292,7 +308,11 @@ def get_cluster(node_ips, node_ip, trainer_endpoints, device_mode, ), "current trainer_endpoints size should be greater equal than acclerators size." for i in range(len(devices_per_proc)): trainer = Trainer() - if device_mode == DeviceMode.GPU or device_mode == DeviceMode.ASCEND_NPU or device_mode == DeviceMode.MLU: + if ( + device_mode == DeviceMode.GPU + or device_mode == DeviceMode.ASCEND_NPU + or device_mode == DeviceMode.MLU + ): if isinstance(devices_per_proc[i], (list, tuple)): trainer.accelerators.extend(devices_per_proc[i]) pod.accelerators.extend(devices_per_proc[i]) @@ -371,21 +391,23 @@ def add_arguments(argname, type, default, help, argparser, **kwargs): args = parser.parse_args() """ type = strtobool if type == bool else type - argparser.add_argument("--" + argname, - default=default, - type=type, - help=help + ' Default: %(default)s.', - **kwargs) + argparser.add_argument( + "--" + argname, + default=default, + type=type, + help=help + ' Default: %(default)s.', + **kwargs + ) def find_free_ports(num): - def __free_port(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: # Note(wangxi): Close the connection with a TCP RST instead # of a TCP FIN, to avoid time_wait state. - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) + s.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0) + ) s.bind(('', 0)) return s.getsockname()[1] @@ -429,7 +451,8 @@ def pretty_print_envs(envs, header=None): max_k = max(max_k, len(k)) h_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format( - max_k, " " * spacing, max_v) + max_k, " " * spacing, max_v + ) l_format = " " + "|{{:>{}s}}{{}}{{:^{}s}}|\n".format(max_k, max_v) length = max_k + max_v + spacing @@ -461,7 +484,6 @@ def pretty_print_envs(envs, header=None): class TrainerProc(object): - def __init__(self): self.proc = None self.log_fn = None @@ -483,12 +505,9 @@ def run_with_coverage(*args): return _run_with_coverage -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - log_dir=None, - envs=None): +def start_local_trainers( + cluster, pod, training_script, training_script_args, log_dir=None, envs=None +): if envs is None: current_env = copy.copy(os.environ.copy()) @@ -507,74 +526,87 @@ def start_local_trainers(cluster, procs = [] for idx, t in enumerate(pod.trainers): proc_env = { - "PADDLE_TRAINER_ID": - "%d" % t.rank, - "PADDLE_CURRENT_ENDPOINT": - "%s" % t.endpoint, - "PADDLE_TRAINERS_NUM": - "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": - ",".join(cluster.trainers_endpoints()), - "PADDLE_RANK_IN_NODE": - str(idx), - "PADDLE_LOCAL_DEVICE_IDS": - ",".join([str(acc) for acc in t.accelerators]), - "PADDLE_WORLD_DEVICE_IDS": - ",".join(res), + "PADDLE_TRAINER_ID": "%d" % t.rank, + "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, + "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), + "PADDLE_RANK_IN_NODE": str(idx), + "PADDLE_LOCAL_DEVICE_IDS": ",".join( + [str(acc) for acc in t.accelerators] + ), + "PADDLE_WORLD_DEVICE_IDS": ",".join(res), } # The following three environnement variables are used for auto mapping if current_env.get("PADDLE_CLUSTER_TOPO_PATH", None) is not None: proc_env["PADDLE_CLUSTER_TOPO_PATH"] = current_env[ - "PADDLE_CLUSTER_TOPO_PATH"] + "PADDLE_CLUSTER_TOPO_PATH" + ] if current_env.get("PADDLE_RANK_MAPPING_PATH", None) is not None: proc_env["PADDLE_RANK_MAPPING_PATH"] = current_env[ - "PADDLE_RANK_MAPPING_PATH"] + "PADDLE_RANK_MAPPING_PATH" + ] if current_env.get("PADDLE_ENABLE_AUTO_MAPPING", None) is not None: proc_env["PADDLE_ENABLE_AUTO_MAPPING"] = current_env[ - "PADDLE_ENABLE_AUTO_MAPPING"] + "PADDLE_ENABLE_AUTO_MAPPING" + ] if len(t.accelerators) > 0 and pod.device_mode == DeviceMode.GPU: proc_env["FLAGS_selected_gpus"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) - elif len(t.accelerators - ) > 0 and pod.device_mode == DeviceMode.ASCEND_NPU: + elif ( + len(t.accelerators) > 0 and pod.device_mode == DeviceMode.ASCEND_NPU + ): proc_env["FLAGS_selected_npus"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) elif len(t.accelerators) > 0 and pod.device_mode == DeviceMode.MLU: proc_env["FLAGS_selected_mlus"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) if len(t.accelerators) > 0: proc_env["FLAGS_selected_accelerators"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) # to do: same code style in future if fluid.core.is_compiled_with_xpu() and len(t.accelerators) > 0: proc_env["FLAGS_selected_xpus"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) current_env.update(proc_env) coverage_args = [] - if run_with_coverage() or os.environ.get("WITH_COVERAGE", - "OFF") == "ON": + if ( + run_with_coverage() + or os.environ.get("WITH_COVERAGE", "OFF") == "ON" + ): coverage_args = ["-m", "coverage", "run", "--branch", "-p"] - cmd = [sys.executable, "-u"] + coverage_args + [training_script - ] + training_script_args + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [training_script] + + training_script_args + ) logger.debug("start trainer proc{} env:{}".format(cmd, current_env)) if idx == 0: - logger.info("Local start {} processes. First process distributed " - "environment info (Only For Debug): {}".format( - len(pod.trainers), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + logger.info( + "Local start {} processes. First process distributed " + "environment info (Only For Debug): {}".format( + len(pod.trainers), + pretty_print_envs(proc_env, ("Distributed Envs", "Value")), + ) + ) logger.info( "details about PADDLE_TRAINER_ENDPOINTS can be found in " "{}/endpoints.log, and detail running logs maybe found in " - "{}/workerlog.0".format(log_dir, log_dir)) + "{}/workerlog.0".format(log_dir, log_dir) + ) fn = None pre_fn = None if os.name == 'nt' else os.setsid if log_dir is not None: @@ -584,16 +616,17 @@ def start_local_trainers(cluster, with open("%s/endpoints.log" % log_dir, "w") as f: f.write("PADDLE_TRAINER_ENDPOINTS: \n") f.write("\n".join(cluster.trainers_endpoints())) - if current_env.get("PADDLE_ENABLE_AUTO_MAPPING") is not None \ - and current_env.get("PADDLE_NEED_RANK_MAPPING").lower() == "true": + if ( + current_env.get("PADDLE_ENABLE_AUTO_MAPPING") is not None + and current_env.get("PADDLE_NEED_RANK_MAPPING").lower() + == "true" + ): fn = open("%s/prelaunchlog.%d" % (log_dir, idx), "a") else: fn = open("%s/workerlog.%d" % (log_dir, idx), "a") - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn, - preexec_fn=pre_fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn, preexec_fn=pre_fn + ) else: proc = subprocess.Popen(cmd, env=current_env, preexec_fn=pre_fn) @@ -620,8 +653,9 @@ def pull_worker_log(tp): except UnicodeEncodeError: sys.stdout.write( 'UnicodeEncodeError occurs at this line. ' - 'Please refer to the original log file "%s"\n' % - tp.log_fn.name) + 'Please refer to the original log file "%s"\n' + % tp.log_fn.name + ) tp.log_offset = fin.tell() @@ -652,14 +686,18 @@ def watch_local_trainers(procs, nranks): return except SystemExit: logger.error( - "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log." - .format(nranks, error_rank)) + "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log.".format( + nranks, error_rank + ) + ) terminate_local_procs(procs) raise except: logger.error( - "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log." - .format(nranks, error_rank)) + "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log.".format( + nranks, error_rank + ) + ) terminate_local_procs(procs) return @@ -680,17 +718,22 @@ def get_gpus(gpus): # therefore gpus=0,1,2,3 cuda_visible_devices_list = cuda_visible_devices.split(',') for x in gpus.split(','): - assert x in cuda_visible_devices_list, "Can't find "\ - "your gpus %s in CUDA_VISIBLE_DEVICES[%s]."\ + assert x in cuda_visible_devices_list, ( + "Can't find " + "your gpus %s in CUDA_VISIBLE_DEVICES[%s]." % (x, cuda_visible_devices) + ) res_gpus = [ cuda_visible_devices_list.index(x.strip()) for x in gpus.split(',') ] - logger.info("Change selected_gpus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "CUDA_VISIBLE_DEVICES:{}".format( - gpus, res_gpus, cuda_visible_devices_list)) + logger.info( + "Change selected_gpus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "CUDA_VISIBLE_DEVICES:{}".format( + gpus, res_gpus, cuda_visible_devices_list + ) + ) return res_gpus @@ -709,17 +752,23 @@ def get_xpus(xpus): # therefore xpus=0,1,2,3 xpu_visible_devices_list = xpu_visible_devices.split(',') for x in xpus.split(','): - assert x in xpu_visible_devices_list, "Can't find "\ - "your xpus %s in XPU_VISIBLE_DEVICES[%s]."\ - % (x, xpu_visible_devices) + assert ( + x in xpu_visible_devices_list + ), "Can't find " "your xpus %s in XPU_VISIBLE_DEVICES[%s]." % ( + x, + xpu_visible_devices, + ) res_xpus = [ xpu_visible_devices_list.index(x.strip()) for x in xpus.split(',') ] - logger.info("Change selected_xpus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "XPU_VISIBLE_DEVICES:{}".format( - xpus, res_xpus, xpu_visible_devices_list)) + logger.info( + "Change selected_xpus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "XPU_VISIBLE_DEVICES:{}".format( + xpus, res_xpus, xpu_visible_devices_list + ) + ) return res_xpus @@ -738,17 +787,22 @@ def get_npus(npus): # therefore npus=0,1,2,3 npu_visible_devices_list = npu_visible_devices.split(',') for x in npus.split(','): - assert x in npu_visible_devices_list, "Can't find "\ - "your npus %s in ASCEND_VISIBLE_DEVICES[%s]."\ + assert x in npu_visible_devices_list, ( + "Can't find " + "your npus %s in ASCEND_VISIBLE_DEVICES[%s]." % (x, npu_visible_devices) + ) res_npus = [ npu_visible_devices_list.index(x.strip()) for x in npus.split(',') ] - logger.info("Change selected_npus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "ASCEND_VISIBLE_DEVICES:{}".format( - npus, res_npus, npu_visible_devices_list)) + logger.info( + "Change selected_npus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "ASCEND_VISIBLE_DEVICES:{}".format( + npus, res_npus, npu_visible_devices_list + ) + ) return res_npus @@ -767,33 +821,45 @@ def get_mlus(mlus): # therefore mlus=0,1,2,3 mlu_visible_devices_list = mlu_visible_devices.split(',') for x in mlus.split(','): - assert x in mlu_visible_devices_list, "Can't find "\ - "your mlus %s in MLU_VISIBLE_DEVICES[%s]."\ - % (x, mlu_visible_devices) + assert ( + x in mlu_visible_devices_list + ), "Can't find " "your mlus %s in MLU_VISIBLE_DEVICES[%s]." % ( + x, + mlu_visible_devices, + ) res_mlus = [ mlu_visible_devices_list.index(x.strip()) for x in mlus.split(',') ] - logger.info("Change selected_mlus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "MLU_VISIBLE_DEVICES:{}".format( - mlus, res_mlus, mlu_visible_devices_list)) + logger.info( + "Change selected_mlus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "MLU_VISIBLE_DEVICES:{}".format( + mlus, res_mlus, mlu_visible_devices_list + ) + ) return res_mlus def get_device_mode(backend): if backend == 'heter': - if fluid.core.is_compiled_with_cuda() and \ - fluid.core.get_cuda_device_count() > 0: + if ( + fluid.core.is_compiled_with_cuda() + and fluid.core.get_cuda_device_count() > 0 + ): print("launch train in heter mode with GPU device.") return DeviceMode.GPU - if fluid.core.is_compiled_with_xpu() and \ - fluid.core.get_xpu_device_count() > 0: + if ( + fluid.core.is_compiled_with_xpu() + and fluid.core.get_xpu_device_count() > 0 + ): print("launch train in heter mode with XPU device.") return DeviceMode.XPU - if fluid.core.is_compiled_with_npu() and \ - fluid.core.get_npu_device_count() > 0: + if ( + fluid.core.is_compiled_with_npu() + and fluid.core.get_npu_device_count() > 0 + ): print("launch train in heter mode with NPU device.") return DeviceMode.ASCEND_NPU @@ -801,8 +867,7 @@ def get_device_mode(backend): print("launch train in ascend npu mode!") return DeviceMode.ASCEND_NPU - if backend == 'nccl' and \ - fluid.core.get_cuda_device_count() > 0: + if backend == 'nccl' and fluid.core.get_cuda_device_count() > 0: print("launch train in GPU mode!") return DeviceMode.GPU @@ -830,62 +895,80 @@ def get_device_proc_info(args): if device_mode == DeviceMode.GPU: gpus = get_gpus(args.gpus) if args.nproc_per_node is not None: - assert (len(gpus) % int(args.nproc_per_node)) ==0, \ - "gpus' number:{} mod args.nproc_per_node:{} must == 0".format(len(gpus), args.nproc_per_node) + assert ( + len(gpus) % int(args.nproc_per_node) + ) == 0, "gpus' number:{} mod args.nproc_per_node:{} must == 0".format( + len(gpus), args.nproc_per_node + ) n = int(len(gpus) / int(args.nproc_per_node)) - devices_per_proc = [gpus[i:i + n] for i in range(0, len(gpus), n)] + devices_per_proc = [gpus[i : i + n] for i in range(0, len(gpus), n)] else: devices_per_proc = gpus elif device_mode == DeviceMode.ASCEND_NPU: npus = get_npus(args.npus) if args.nproc_per_node is not None: - assert (len(npus) % int(args.nproc_per_node)) ==0, \ - "npus' number:{} mod args.nproc_per_node:{} must == 0".format(len(npus), args.nproc_per_node) + assert ( + len(npus) % int(args.nproc_per_node) + ) == 0, "npus' number:{} mod args.nproc_per_node:{} must == 0".format( + len(npus), args.nproc_per_node + ) n = int(len(npus) / int(args.nproc_per_node)) - devices_per_proc = [npus[i:i + n] for i in range(0, len(npus), n)] + devices_per_proc = [npus[i : i + n] for i in range(0, len(npus), n)] else: devices_per_proc = npus elif device_mode == DeviceMode.XPU: xpus = get_xpus(args.xpus) if args.nproc_per_node is not None: - assert (len(xpus) % int(args.nproc_per_node)) == 0, \ - "xpus' number:{} mod args.nproc_per_node:{} must == 0".format(len(xpus), args.nproc_per_node) + assert ( + len(xpus) % int(args.nproc_per_node) + ) == 0, "xpus' number:{} mod args.nproc_per_node:{} must == 0".format( + len(xpus), args.nproc_per_node + ) n = int(len(xpus) / int(args.nproc_per_node)) - devices_per_proc = [xpus[i:i + n] for i in range(0, len(xpus), n)] + devices_per_proc = [xpus[i : i + n] for i in range(0, len(xpus), n)] else: devices_per_proc = xpus elif device_mode == DeviceMode.MLU: mlus = get_mlus(args.mlus) if args.nproc_per_node is not None: - assert (len(mlus) % int(args.nproc_per_node)) ==0, \ - "mlus' number:{} mod args.nproc_per_node:{} must == 0".format(len(mlus), args.nproc_per_node) + assert ( + len(mlus) % int(args.nproc_per_node) + ) == 0, "mlus' number:{} mod args.nproc_per_node:{} must == 0".format( + len(mlus), args.nproc_per_node + ) n = int(len(mlus) / int(args.nproc_per_node)) - devices_per_proc = [mlus[i:i + n] for i in range(0, len(mlus), n)] + devices_per_proc = [mlus[i : i + n] for i in range(0, len(mlus), n)] else: devices_per_proc = mlus elif device_mode == DeviceMode.CPU: if hasattr(args, "paddle_cpuonly") and args.nproc_per_node is None: - #NOTE (xiongkun03) set it to cpu core number + # NOTE (xiongkun03) set it to cpu core number args.nproc_per_node = multiprocessing.cpu_count() if args.nproc_per_node is None: devices_per_proc = [0] else: devices_per_proc = [x for x in range(0, args.nproc_per_node)] else: - assert False, "Can't support device_mode:{}, support only cpu|gpu|xpu now.".format( - device_mode) + assert ( + False + ), "Can't support device_mode:{}, support only cpu|gpu|xpu now.".format( + device_mode + ) return (device_mode, devices_per_proc) def direct_start(args): # run ps-cpu mode on paddlecloud, using given envs - cmd = [sys.executable, "-u", args.training_script] + \ - args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args proc = subprocess.Popen(cmd) proc.wait() return @@ -907,7 +990,7 @@ def get_custom_endpoints(origin_endpoints, offset=0): return paddle_user_define_endpoints -#def cloud_ps_heter_env_set(args): +# def cloud_ps_heter_env_set(args): # environs = {} # # paddle_trainer_endpoints = os.getenv("TRAINER_IP_PORT_LIST", "") @@ -947,12 +1030,13 @@ def get_custom_endpoints(origin_endpoints, offset=0): # pretty_print_envs(environs))) -def get_mapped_cluster_without_rank_mapping(node_ips, node_ip, - trainer_endpoints, device_mode, - node_ranks): +def get_mapped_cluster_without_rank_mapping( + node_ips, node_ip, trainer_endpoints, device_mode, node_ranks +): assert type(trainer_endpoints) is list, "trainer_endpoints must be list" - assert device_mode == DeviceMode.GPU, \ - "Only support get mapped cluster for gpu now." + assert ( + device_mode == DeviceMode.GPU + ), "Only support get mapped cluster for gpu now." cluster = Cluster(hdfs=None) for node_rank, ip in enumerate(node_ips): pod = Pod() @@ -976,8 +1060,9 @@ def get_mapped_cluster_without_rank_mapping(node_ips, node_ip, def get_mapped_cluster_from_args_without_rank_mapping(args, device_mode): - assert device_mode == DeviceMode.GPU, \ - "Only support get mapped cluster for gpu now." + assert ( + device_mode == DeviceMode.GPU + ), "Only support get mapped cluster for gpu now." gpus_num = fluid.core.get_cuda_device_count() # parse ip-ranks json file @@ -999,17 +1084,21 @@ def get_mapped_cluster_from_args_without_rank_mapping(args, device_mode): else: _, node_ip = get_host_name_ip() - assert node_ip in node_ips, \ - "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) + assert ( + node_ip in node_ips + ), "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) node_rank = node_ips.index(node_ip) - assert len(node_ranks) == len(node_ips), \ - "ranks length should be equal to ips length." + assert len(node_ranks) == len( + node_ips + ), "ranks length should be equal to ips length." - logger.debug("parsed from args: node_ips:{} node_ip:{} " - "node_rank:{} node_ranks:{}".format(node_ips, node_ip, - node_rank, - node_ranks[node_rank])) + logger.debug( + "parsed from args: node_ips:{} node_ip:{} " + "node_rank:{} node_ranks:{}".format( + node_ips, node_ip, node_rank, node_ranks[node_rank] + ) + ) # NOTE: there are different number of global mapped ranks on each node. free_ports = [] @@ -1019,30 +1108,40 @@ def get_mapped_cluster_from_args_without_rank_mapping(args, device_mode): if os.environ.get('PADDLE_PORT') is not None: start_port = int(os.getenv("PADDLE_PORT", "")) free_ports = [ - x for x in range(start_port, start_port + - len(node_ranks[node_rank])) + x + for x in range( + start_port, start_port + len(node_ranks[node_rank]) + ) ] elif os.environ.get('FLAGS_START_PORT') is not None: start_port = int(os.environ.get('FLAGS_START_PORT')) free_ports = [ - x for x in range(start_port, start_port + - len(node_ranks[node_rank])) + x + for x in range( + start_port, start_port + len(node_ranks[node_rank]) + ) ] else: free_ports = find_free_ports(len(node_ranks[node_rank])) trainer_endpoints.append(["%s:%d" % (ip, port) for port in free_ports]) - return get_mapped_cluster_without_rank_mapping(node_ips, node_ip, - trainer_endpoints, - device_mode, node_ranks) + return get_mapped_cluster_without_rank_mapping( + node_ips, node_ip, trainer_endpoints, device_mode, node_ranks + ) -def get_mapped_cluster_with_rank_mapping(node_ips, node_ip, trainer_endpoints, - device_mode, node_ranks, - node_rank_mappings): +def get_mapped_cluster_with_rank_mapping( + node_ips, + node_ip, + trainer_endpoints, + device_mode, + node_ranks, + node_rank_mappings, +): assert type(trainer_endpoints) is list, "trainer_endpoints must be list" - assert device_mode == DeviceMode.GPU, \ - "Only support get mapped cluster for gpu now." + assert ( + device_mode == DeviceMode.GPU + ), "Only support get mapped cluster for gpu now." def get_relative_gpu_id(gpu_id): cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") @@ -1052,8 +1151,10 @@ def get_mapped_cluster_with_rank_mapping(node_ips, node_ip, trainer_endpoints, cuda_visible_devices_list = cuda_visible_devices.split(',') relative_id = cuda_visible_devices_list.index(str(gpu_id)) logger.info( - "Change gpu id from {} to {} based on CUDA_VISIBLE_DEVICES {}". - format(gpu_id, relative_id, cuda_visible_devices_list)) + "Change gpu id from {} to {} based on CUDA_VISIBLE_DEVICES {}".format( + gpu_id, relative_id, cuda_visible_devices_list + ) + ) return relative_id cluster = Cluster(hdfs=None) @@ -1069,12 +1170,15 @@ def get_mapped_cluster_with_rank_mapping(node_ips, node_ip, trainer_endpoints, cur_node_rank_mapping = node_rank_mappings[node_rank] for i in range(len(ranks_per_node)): trainer = Trainer() - local_device_ids = cur_node_rank_mapping["ranks"][str( - ranks_per_node[i])] - assert len(local_device_ids) == 1, \ - "Only support one process to one device mapping" - trainer.accelerators.append(get_relative_gpu_id( - local_device_ids[0])) + local_device_ids = cur_node_rank_mapping["ranks"][ + str(ranks_per_node[i]) + ] + assert ( + len(local_device_ids) == 1 + ), "Only support one process to one device mapping" + trainer.accelerators.append( + get_relative_gpu_id(local_device_ids[0]) + ) trainer.endpoint = "%s" % (cur_node_endpoints[i]) trainer.rank = ranks_per_node[i] pod.trainers.append(trainer) @@ -1085,13 +1189,15 @@ def get_mapped_cluster_with_rank_mapping(node_ips, node_ip, trainer_endpoints, def get_mapped_cluster_from_args_with_rank_mapping(args, device_mode): - assert device_mode == DeviceMode.GPU, \ - "Only support get mapped cluster for gpu now." + assert ( + device_mode == DeviceMode.GPU + ), "Only support get mapped cluster for gpu now." gpus_num = fluid.core.get_cuda_device_count() # parse ip-ranks json file rank_mapping_path = args.rank_mapping_path or os.getenv( - "PADDLE_RANK_MAPPING_PATH") + "PADDLE_RANK_MAPPING_PATH" + ) rank_mapping = None with open(rank_mapping_path, "r") as json_file: rank_mapping = json.load(json_file) @@ -1118,19 +1224,24 @@ def get_mapped_cluster_from_args_with_rank_mapping(args, device_mode): else: _, node_ip = get_host_name_ip() - assert node_ip in node_ips, \ - "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) + assert ( + node_ip in node_ips + ), "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) node_rank = node_ips.index(node_ip) - assert len(node_ranks[node_rank]) <= gpus_num, \ - "number of ranks mapped to one node should not exceed the avaiable ones." - assert len(node_ranks) == len(node_ips), \ - "ranks length should be equal to ips length." - - logger.debug("parsed from args: node_ips:{} node_ip:{} " - "node_rank:{} node_ranks:{}".format(node_ips, node_ip, - node_rank, - node_ranks[node_rank])) + assert ( + len(node_ranks[node_rank]) <= gpus_num + ), "number of ranks mapped to one node should not exceed the avaiable ones." + assert len(node_ranks) == len( + node_ips + ), "ranks length should be equal to ips length." + + logger.debug( + "parsed from args: node_ips:{} node_ip:{} " + "node_rank:{} node_ranks:{}".format( + node_ips, node_ip, node_rank, node_ranks[node_rank] + ) + ) # NOTE: there are different number of global mapped ranks on each node. free_ports = [] @@ -1140,26 +1251,34 @@ def get_mapped_cluster_from_args_with_rank_mapping(args, device_mode): if os.environ.get('PADDLE_PORT') is not None: start_port = int(os.getenv("PADDLE_PORT", "")) free_ports = [ - x for x in range(start_port, start_port + - len(node_ranks[node_rank])) + x + for x in range( + start_port, start_port + len(node_ranks[node_rank]) + ) ] elif os.environ.get('FLAGS_START_PORT') is not None: start_port = int(os.environ.get('FLAGS_START_PORT')) free_ports = [ - x for x in range(start_port, start_port + - len(node_ranks[node_rank])) + x + for x in range( + start_port, start_port + len(node_ranks[node_rank]) + ) ] else: free_ports = find_free_ports(len(node_ranks[node_rank])) trainer_endpoints.append(["%s:%d" % (ip, port) for port in free_ports]) - return get_mapped_cluster_with_rank_mapping(node_ips, node_ip, - trainer_endpoints, device_mode, - node_ranks, node_rank_mappings) + return get_mapped_cluster_with_rank_mapping( + node_ips, + node_ip, + trainer_endpoints, + device_mode, + node_ranks, + node_rank_mappings, + ) class ParameterServerLauncher(object): - def __init__(self, args, distribute_mode): self.args = args self.distribute_mode = distribute_mode @@ -1200,17 +1319,21 @@ class ParameterServerLauncher(object): if args.server_num: self.server_num = args.server_num if args.servers: - assert len( - args.servers.split(",") - ) == self.server_num, "The server_num and servers doesn't match. Expect servers endpoints num epual to server_num, but received servers enpoint num: {} and server_num {}".format( - len(args.servers.split(",")), self.server_num) + assert ( + len(args.servers.split(",")) == self.server_num + ), "The server_num and servers doesn't match. Expect servers endpoints num epual to server_num, but received servers enpoint num: {} and server_num {}".format( + len(args.servers.split(",")), self.server_num + ) self.server_endpoints = args.servers else: ports = get_ports(self.server_num, 0) self.server_endpoints = ",".join( - ["127.0.0.1:" + str(x) for x in ports]) + ["127.0.0.1:" + str(x) for x in ports] + ) else: - assert args.servers != "", "The setting of Parameter-Server must has server_num or servers." + assert ( + args.servers != "" + ), "The setting of Parameter-Server must has server_num or servers." self.server_endpoints = args.servers self.server_num = len(self.server_endpoints.split(",")) @@ -1218,18 +1341,22 @@ class ParameterServerLauncher(object): if args.worker_num: self.worker_num = args.worker_num if args.workers: - assert len( - args.workers.split(",") - ) == self.worker_num, "The worker_num and workers doesn't match. Expect workers endpoints num epual to worker_num, but received workers enpoint num: {} and worker_num {}".format( - len(args.workers.split(",")), self.worker_num) + assert ( + len(args.workers.split(",")) == self.worker_num + ), "The worker_num and workers doesn't match. Expect workers endpoints num epual to worker_num, but received workers enpoint num: {} and worker_num {}".format( + len(args.workers.split(",")), self.worker_num + ) self.worker_endpoints = args.workers else: ports = get_ports(self.worker_num, self.server_num) self.worker_endpoints = ",".join( - ["127.0.0.1:" + str(x) for x in ports]) + ["127.0.0.1:" + str(x) for x in ports] + ) else: - assert args.workers != "", "The setting of Parameter-Server must has worker_num or workers." + assert ( + args.workers != "" + ), "The setting of Parameter-Server must has worker_num or workers." worker_endpoints_ips = [ x.strip().split(":")[0] for x in args.workers.split(",") ] @@ -1243,13 +1370,20 @@ class ParameterServerLauncher(object): start_port = 6170 worker_endpoints_port = range( start_port + self.server_num, - start_port + self.server_num + self.worker_num, 1) + start_port + self.server_num + self.worker_num, + 1, + ) # create endpoints str worker_endpoints = [] for i in range(self.worker_num): - worker_endpoints.append(":".join( - (worker_endpoints_ips[i], - str(worker_endpoints_port[i])))) + worker_endpoints.append( + ":".join( + ( + worker_endpoints_ips[i], + str(worker_endpoints_port[i]), + ) + ) + ) self.worker_endpoints = ",".join(worker_endpoints) else: self.worker_endpoints = args.workers @@ -1259,21 +1393,25 @@ class ParameterServerLauncher(object): self.with_coordinator = True self.coordinator_num = args.coordinator_num if args.coordinators: - assert len( - args.coordinators.split(",") - ) == self.coordinator_num, "The coordinator_num and coordinators doesn't match. Expect coordinators endpoints num epual to coordinator_num, but received coordinator enpoint num: {} and coordinator_num {}".format( - len(args.coordinators.split(",")), self.coordinator_num) + assert ( + len(args.coordinators.split(",")) == self.coordinator_num + ), "The coordinator_num and coordinators doesn't match. Expect coordinators endpoints num epual to coordinator_num, but received coordinator enpoint num: {} and coordinator_num {}".format( + len(args.coordinators.split(",")), self.coordinator_num + ) self.coordinator_endpoints = args.coordinators else: ports = get_ports(self.coordinator_num, 1) self.coordinator_endpoints = ",".join( - ["127.0.0.1:" + str(x) for x in ports]) + ["127.0.0.1:" + str(x) for x in ports] + ) print(">>> use default coordinator addr(only one process)") # get heter worker envs if self.distribute_mode == DistributeMode.PS_HETER: - assert args.heter_devices != "", "The setting of Parameter-Server heter mode must has heter_devices." + assert ( + args.heter_devices != "" + ), "The setting of Parameter-Server heter mode must has heter_devices." self.stage_device_map[1] = "cpu" # for cpu trainer heter_devices_list = args.heter_devices.split(";") for i in range(len(heter_devices_list)): @@ -1292,19 +1430,22 @@ class ParameterServerLauncher(object): self.stage_heter_trainer_num ), "The stage_num and heter_workers doesn't match. Expect heter_workers endpoints stage num epual to heter_worker_num stage, but received heter_workers enpoint stage num: {} and heter_worker_num stage {}".format( len(args.heter_workers.split(";")), - len(self.stage_heter_trainer_num)) + len(self.stage_heter_trainer_num), + ) heter_worker_endpoints_list = args.heter_workers.split(";") self.heter_worker_endpoints = "" for i in range(len(self.stage_heter_trainer_num)): if self.heter_worker_endpoints != "": self.heter_worker_endpoints += "," heter_worker_endpoints = heter_worker_endpoints_list[ - i].split(",") - assert len( - heter_worker_endpoints - ) == self.stage_heter_trainer_num[ - i], "The heter trainer num in stage {} is not equal in args.heter_worker_num and args.heter_workers".format( - i) + i + ].split(",") + assert ( + len(heter_worker_endpoints) + == self.stage_heter_trainer_num[i] + ), "The heter trainer num in stage {} is not equal in args.heter_worker_num and args.heter_workers".format( + i + ) heter_worker_endpoints_ips = [ x.strip().split(":")[0] @@ -1319,20 +1460,28 @@ class ParameterServerLauncher(object): # if no port value in heter_worker_endpoint, will set default port values. heter_worker_endpoints_port = get_ports( len(heter_worker_endpoints_ips), - self.worker_num + self.server_num + - self.heter_worker_num) + self.worker_num + + self.server_num + + self.heter_worker_num, + ) new_heter_worker_endpoints = [] for j in range(len(heter_worker_endpoints_ips)): - new_heter_worker_endpoints.append(":".join( - (heter_worker_endpoints_ips[j], - str(heter_worker_endpoints_port[j])))) + new_heter_worker_endpoints.append( + ":".join( + ( + heter_worker_endpoints_ips[j], + str(heter_worker_endpoints_port[j]), + ) + ) + ) ip_port_list = ",".join(new_heter_worker_endpoints) else: ip_port_list = ",".join(heter_worker_endpoints) self.stage_heter_map[i + 2] = ip_port_list - self.stage_list.extend([i + 2] * - len(ip_port_list.split(','))) + self.stage_list.extend( + [i + 2] * len(ip_port_list.split(',')) + ) self.heter_worker_num += self.stage_heter_trainer_num[i] self.heter_worker_endpoints += ip_port_list @@ -1340,27 +1489,36 @@ class ParameterServerLauncher(object): for i in range(len(self.stage_heter_trainer_num)): heter_trainer_num = self.stage_heter_trainer_num[i] ports = get_ports( - heter_trainer_num, self.server_num + - self.worker_num + self.heter_worker_num) + heter_trainer_num, + self.server_num + + self.worker_num + + self.heter_worker_num, + ) ip_port_list = ",".join( - ["127.0.0.1:" + str(x) for x in ports]) + ["127.0.0.1:" + str(x) for x in ports] + ) self.stage_heter_map[i + 2] = ip_port_list - self.stage_list.extend([i + 2] * - len(ip_port_list.split(','))) + self.stage_list.extend( + [i + 2] * len(ip_port_list.split(',')) + ) self.heter_worker_num += heter_trainer_num if self.heter_worker_endpoints != "": self.heter_worker_endpoints += "," self.heter_worker_endpoints += ip_port_list else: - assert args.heter_workers != "", "The setting of Parameter-Server heter mode must has heter_worker_num or heter_workers." + assert ( + args.heter_workers != "" + ), "The setting of Parameter-Server heter mode must has heter_worker_num or heter_workers." self.stage_heter_trainer_num = [] heter_worker_endpoints_list = args.heter_workers.split(";") self.heter_worker_endpoints = "" for i in range(len(heter_worker_endpoints_list)): heter_worker_endpoints = heter_worker_endpoints_list[ - i].split(",") + i + ].split(",") self.stage_heter_trainer_num.append( - len(heter_worker_endpoints)) + len(heter_worker_endpoints) + ) heter_worker_endpoints_ips = [ x.strip().split(":")[0] for x in heter_worker_endpoints ] @@ -1371,29 +1529,39 @@ class ParameterServerLauncher(object): if 1 in heter_worker_endpoints_len: # if no port value in heter_worker_endpoint, will set default port values. heter_worker_endpoints_port = get_ports( - len(heter_worker_endpoints_ips), self.worker_num + - self.server_num + self.heter_worker_num) + len(heter_worker_endpoints_ips), + self.worker_num + + self.server_num + + self.heter_worker_num, + ) new_heter_worker_endpoints = [] for j in range(len(heter_worker_endpoints_ips)): - new_heter_worker_endpoints.append(":".join( - (heter_worker_endpoints_ips[j], - str(heter_worker_endpoints_port[j])))) + new_heter_worker_endpoints.append( + ":".join( + ( + heter_worker_endpoints_ips[j], + str(heter_worker_endpoints_port[j]), + ) + ) + ) ip_port_list = ",".join(new_heter_worker_endpoints) else: ip_port_list = ",".join(heter_worker_endpoints) self.stage_heter_map[i + 2] = ip_port_list - self.stage_list.extend([i + 2] * - len(ip_port_list.split(','))) + self.stage_list.extend( + [i + 2] * len(ip_port_list.split(',')) + ) self.heter_worker_num += self.stage_heter_trainer_num[-1] if self.heter_worker_endpoints != "": self.heter_worker_endpoints += "," self.heter_worker_endpoints += ip_port_list - self.stage_trainer_num = [self.worker_num - ] + self.stage_heter_trainer_num + self.stage_trainer_num = [ + self.worker_num + ] + self.stage_heter_trainer_num self.stage_num = len(self.stage_trainer_num) # get http_port @@ -1401,7 +1569,8 @@ class ParameterServerLauncher(object): http_port = [args.http_port] else: http_port = get_ports( - 1, self.server_num + self.worker_num + self.heter_worker_num) + 1, self.server_num + self.worker_num + self.heter_worker_num + ) http_ip = self.server_endpoints.split(",")[0].split(":")[0] self.http_port = http_ip + ":" + str(http_port[0]) @@ -1461,13 +1630,17 @@ class ParameterServerLauncher(object): else: self.current_node_ip = pod_ip if not self.distribute_mode == DistributeMode.PS_HETER: - assert self.current_node_ip in self.node_ips, "Can't find your local ip {%s} in args.servers and args.workers ips: {%s}" \ - % (self.current_node_ip, self.node_ips) + assert self.current_node_ip in self.node_ips, ( + "Can't find your local ip {%s} in args.servers and args.workers ips: {%s}" + % (self.current_node_ip, self.node_ips) + ) if self.current_node_ip in self.node_ips: self.node_rank = self.node_ips.index(self.current_node_ip) logger.debug( - "parsed from args: node_ips:{} current_node_ip:{} node_rank:{}". - format(self.node_ips, self.current_node_ip, self.node_rank)) + "parsed from args: node_ips:{} current_node_ip:{} node_rank:{}".format( + self.node_ips, self.current_node_ip, self.node_rank + ) + ) def start_ps(self): if self.current_node_ip not in self.node_ips: @@ -1484,16 +1657,20 @@ class ParameterServerLauncher(object): for i in range(len(self.server_endpoints_ips)): if ip == self.server_endpoints_ips[i]: server = Trainer() - server.endpoint = "%s:%s" % (ip, - self.server_endpoints_port[i]) + server.endpoint = "%s:%s" % ( + ip, + self.server_endpoints_port[i], + ) server.rank = server_rank server_rank += 1 pod.servers.append(server) for j in range(len(self.worker_endpoints_ips)): if ip == self.worker_endpoints_ips[j]: worker = Trainer() - worker.endpoint = "%s:%s" % (ip, - self.worker_endpoints_port[j]) + worker.endpoint = "%s:%s" % ( + ip, + self.worker_endpoints_port[j], + ) worker.rank = worker_rank worker.stage = 1 worker_rank += 1 @@ -1502,7 +1679,9 @@ class ParameterServerLauncher(object): if ip == self.coordinator_endpoints_ips[m]: coordinator = Trainer() coordinator.endpoint = "%s:%s" % ( - ip, self.coordinator_endpoints_port[m]) + ip, + self.coordinator_endpoints_port[m], + ) coordinator.rank = coordinator_rank coordinator.stage = 1 coordinator_rank += 1 @@ -1512,7 +1691,9 @@ class ParameterServerLauncher(object): if ip == self.heter_worker_endpoints_ips[k]: heter_worker = Trainer() heter_worker.endpoint = "%s:%s" % ( - ip, self.heter_worker_endpoints_port[k]) + ip, + self.heter_worker_endpoints_port[k], + ) heter_worker.rank = heter_worker_rank heter_worker.stage = self.stage_list[k] heter_worker_rank += 1 @@ -1528,19 +1709,19 @@ class ParameterServerLauncher(object): "worker": [], "coordinator": [], "server": [], - "heter_worker": [] + "heter_worker": [], } self.cmds = { "worker": [], "coordinator": [], "server": [], - "heter_worker": [] + "heter_worker": [], } self.log_fns = { "worker": [], "coordinator": [], "server": [], - "heter_worker": [] + "heter_worker": [], } self.start_pod_server(self.args, pod) @@ -1551,9 +1732,13 @@ class ParameterServerLauncher(object): self.start_pod_heter_worker(self.args, pod) logger.info( - "Please check servers, workers, coordinator and heter_worker logs in {}/workerlog.*, {}/serverlog.* , {}/coordinatorlog.*, and {}/heterlog.*" - .format(self.args.log_dir, self.args.log_dir, self.args.log_dir, - self.args.log_dir)) + "Please check servers, workers, coordinator and heter_worker logs in {}/workerlog.*, {}/serverlog.* , {}/coordinatorlog.*, and {}/heterlog.*".format( + self.args.log_dir, + self.args.log_dir, + self.args.log_dir, + self.args.log_dir, + ) + ) # 4. wait for finish training if len(self.procs["worker"]) > 0: @@ -1609,8 +1794,7 @@ class ParameterServerLauncher(object): "PADDLE_PSERVERS_IP_PORT_LIST": self.server_endpoints, "PADDLE_TRAINER_ENDPOINTS": self.worker_endpoints, "PADDLE_COORDINATOR_ENDPOINTS": self.coordinator_endpoints, - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": - self.heter_worker_endpoints, + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": self.heter_worker_endpoints, "PADDLE_PORT": cur_server.endpoint.split(":")[1], "TRAINING_ROLE": "PSERVER", "PADDLE_TRAINERS_NUM": str(self.worker_num), @@ -1618,7 +1802,7 @@ class ParameterServerLauncher(object): "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")), "PADDLE_GLOO_RENDEZVOUS": "3", "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir, - "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } else: proc_env = { @@ -1632,12 +1816,15 @@ class ParameterServerLauncher(object): "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")), "PADDLE_GLOO_RENDEZVOUS": "3", "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir, - "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } current_env.update(proc_env) - cmd = [sys.executable, "-u", args.training_script - ] + args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args self.cmds["server"].append(cmd) if idx == 0: @@ -1645,17 +1832,19 @@ class ParameterServerLauncher(object): "Local server start {} processes. First process distributed " "environment info (Only For Debug): {}".format( len(pod.servers), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + pretty_print_envs( + proc_env, ("Distributed Envs", "Value") + ), + ) + ) if args.log_dir is not None: os.system("mkdir -p {}".format(args.log_dir)) fn = open("%s/serverlog.%d" % (args.log_dir, idx), "w") self.log_fns["server"].append(fn) - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn + ) else: proc = subprocess.Popen(cmd, env=current_env) @@ -1685,56 +1874,38 @@ class ParameterServerLauncher(object): device_list = [str(x) for x in range(0, heter_device_num)] for idx, cur_worker in enumerate(pod.workers): - device_id = "0" if heter_device_num == 0 else str( - device_list[(idx) % heter_device_num]) + device_id = ( + "0" + if heter_device_num == 0 + else str(device_list[(idx) % heter_device_num]) + ) if self.distribute_mode == DistributeMode.PS_HETER: proc_env = { - "PADDLE_PSERVERS_IP_PORT_LIST": - self.server_endpoints, - "PADDLE_TRAINER_ENDPOINTS": - self.worker_endpoints, - "PADDLE_TRAINERS_NUM": - str(self.worker_num), - "PADDLE_COORDINATOR_ENDPOINTS": - self.coordinator_endpoints, - "PADDLE_STAGE_TRAINERS_NUM": - str(self.stage_trainer_num), - "STAGE_ID": - "1", - "STAGE_NUM": - str(self.stage_num), - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": - "", - "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": - self.stage_heter_map[2], - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": - self.heter_worker_endpoints, - "HETER_DEVICE_TYPE": - self.stage_device_map[1], - "TRAINING_ROLE": - "TRAINER", - "POD_IP": - cur_worker.endpoint.split(":")[0], - "PADDLE_PORT": - cur_worker.endpoint.split(":")[1], - "PADDLE_TRAINER_ID": - str(cur_worker.rank), - "PADDLE_WITH_GLOO": - str(os.getenv("PADDLE_WITH_GLOO", "0")), - "PADDLE_GLOO_RENDEZVOUS": - "3", - "PADDLE_GLOO_FS_PATH": - self.gloo_rendezvous_dir, - "FLAGS_selected_gpus": - "0", - "FLAGS_selected_xpus": - "0", - "CUDA_VISIBLE_DEVICES": - device_id, - "XPU_VISIBLE_DEVICES": - device_id, - "PADDLE_GLOO_HTTP_ENDPOINT": - self.http_port + "PADDLE_PSERVERS_IP_PORT_LIST": self.server_endpoints, + "PADDLE_TRAINER_ENDPOINTS": self.worker_endpoints, + "PADDLE_TRAINERS_NUM": str(self.worker_num), + "PADDLE_COORDINATOR_ENDPOINTS": self.coordinator_endpoints, + "PADDLE_STAGE_TRAINERS_NUM": str(self.stage_trainer_num), + "STAGE_ID": "1", + "STAGE_NUM": str(self.stage_num), + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": "", + "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": self.stage_heter_map[ + 2 + ], + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": self.heter_worker_endpoints, + "HETER_DEVICE_TYPE": self.stage_device_map[1], + "TRAINING_ROLE": "TRAINER", + "POD_IP": cur_worker.endpoint.split(":")[0], + "PADDLE_PORT": cur_worker.endpoint.split(":")[1], + "PADDLE_TRAINER_ID": str(cur_worker.rank), + "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")), + "PADDLE_GLOO_RENDEZVOUS": "3", + "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir, + "FLAGS_selected_gpus": "0", + "FLAGS_selected_xpus": "0", + "CUDA_VISIBLE_DEVICES": device_id, + "XPU_VISIBLE_DEVICES": device_id, + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } else: proc_env = { @@ -1753,12 +1924,15 @@ class ParameterServerLauncher(object): "FLAGS_selected_xpus": "0", "CUDA_VISIBLE_DEVICES": device_id, "XPU_VISIBLE_DEVICES": device_id, - "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } current_env.update(proc_env) - cmd = [sys.executable, "-u", args.training_script - ] + args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args self.cmds["worker"].append(cmd) if idx == 0: @@ -1766,17 +1940,19 @@ class ParameterServerLauncher(object): "Local worker start {} processes. First process distributed " "environment info (Only For Debug): {}".format( len(pod.workers), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + pretty_print_envs( + proc_env, ("Distributed Envs", "Value") + ), + ) + ) if args.log_dir is not None: os.system("mkdir -p {}".format(args.log_dir)) fn = open("%s/workerlog.%d" % (args.log_dir, idx), "w") self.log_fns["worker"].append(fn) - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn + ) else: proc = subprocess.Popen(cmd, env=current_env) @@ -1816,12 +1992,15 @@ class ParameterServerLauncher(object): "FLAGS_selected_xpus": "0", "CUDA_VISIBLE_DEVICES": device_id, "XPU_VISIBLE_DEVICES": device_id, - "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } current_env.update(proc_env) - cmd = [sys.executable, "-u", args.training_script - ] + args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args self.cmds["coordinator"].append(cmd) if idx == 0: @@ -1829,17 +2008,19 @@ class ParameterServerLauncher(object): "Local coordinator start {} processes. First process distributed " "environment info (Only For Debug): {}".format( len(pod.coordinators), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + pretty_print_envs( + proc_env, ("Distributed Envs", "Value") + ), + ) + ) if args.log_dir is not None: os.system("mkdir -p {}".format(args.log_dir)) fn = open("%s/coordinator.%d" % (args.log_dir, idx), "w") self.log_fns["coordinator"].append(fn) - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn + ) else: proc = subprocess.Popen(cmd, env=current_env) @@ -1869,58 +2050,48 @@ class ParameterServerLauncher(object): device_list = [str(x) for x in range(0, heter_device_num)] for idx, cur_heter_worker in enumerate(pod.heter_workers): - device_id = "0" if heter_device_num == 0 else str( - device_list[(idx) % heter_device_num]) + device_id = ( + "0" + if heter_device_num == 0 + else str(device_list[(idx) % heter_device_num]) + ) stage_id = cur_heter_worker.stage proc_env = { - "PADDLE_PSERVERS_IP_PORT_LIST": - self.server_endpoints, - "PADDLE_TRAINER_ENDPOINTS": - self.worker_endpoints, - "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": - self.stage_heter_map[stage_id + 1] - if stage_id <= self.stage_num - 1 else "", - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": - self.stage_heter_map[stage_id - 1], - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": - self.heter_worker_endpoints, - "HETER_DEVICE_TYPE": - self.stage_device_map[stage_id], - "STAGE_ID": - str(stage_id), - "STAGE_NUM": - str(self.stage_num), - "PADDLE_PORT": - cur_heter_worker.endpoint.split(":")[1], - "TRAINING_ROLE": - "HETER_TRAINER", - "PADDLE_TRAINERS_NUM": - str(self.worker_num), - "PADDLE_STAGE_TRAINERS_NUM": - str(self.stage_trainer_num), - "POD_IP": - cur_heter_worker.endpoint.split(":")[0], - "PADDLE_WITH_GLOO": - str(os.getenv("PADDLE_WITH_GLOO", "0")), - "PADDLE_GLOO_RENDEZVOUS": - "3", - "PADDLE_GLOO_FS_PATH": - self.gloo_rendezvous_dir, - "FLAGS_selected_gpus": - "0", - "FLAGS_selected_xpus": - "0", - "CUDA_VISIBLE_DEVICES": - device_id, - "XPU_VISIBLE_DEVICES": - device_id, - "PADDLE_GLOO_HTTP_ENDPOINT": - self.http_port + "PADDLE_PSERVERS_IP_PORT_LIST": self.server_endpoints, + "PADDLE_TRAINER_ENDPOINTS": self.worker_endpoints, + "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": self.stage_heter_map[ + stage_id + 1 + ] + if stage_id <= self.stage_num - 1 + else "", + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": self.stage_heter_map[ + stage_id - 1 + ], + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": self.heter_worker_endpoints, + "HETER_DEVICE_TYPE": self.stage_device_map[stage_id], + "STAGE_ID": str(stage_id), + "STAGE_NUM": str(self.stage_num), + "PADDLE_PORT": cur_heter_worker.endpoint.split(":")[1], + "TRAINING_ROLE": "HETER_TRAINER", + "PADDLE_TRAINERS_NUM": str(self.worker_num), + "PADDLE_STAGE_TRAINERS_NUM": str(self.stage_trainer_num), + "POD_IP": cur_heter_worker.endpoint.split(":")[0], + "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")), + "PADDLE_GLOO_RENDEZVOUS": "3", + "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir, + "FLAGS_selected_gpus": "0", + "FLAGS_selected_xpus": "0", + "CUDA_VISIBLE_DEVICES": device_id, + "XPU_VISIBLE_DEVICES": device_id, + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } current_env.update(proc_env) - cmd = [sys.executable, "-u", args.training_script - ] + args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args self.cmds["heter_worker"].append(cmd) if idx == 0: @@ -1928,17 +2099,19 @@ class ParameterServerLauncher(object): "Local heter_worker start {} processes. First process distributed " "environment info (Only For Debug): {}".format( len(pod.heter_workers), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + pretty_print_envs( + proc_env, ("Distributed Envs", "Value") + ), + ) + ) if args.log_dir is not None: os.system("mkdir -p {}".format(args.log_dir)) fn = open("%s/heterlog.%d" % (args.log_dir, idx), "w") self.log_fns["heter_worker"].append(fn) - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn + ) else: proc = subprocess.Popen(cmd, env=current_env) @@ -1955,13 +2128,21 @@ class ParameterServerLauncher(object): def check_backend(backend): if backend not in [ - 'nccl', 'gloo', 'bkcl', 'cncl', 'auto', 'hccl', 'heter', 'xccl' + 'nccl', + 'gloo', + 'bkcl', + 'cncl', + 'auto', + 'hccl', + 'heter', + 'xccl', ]: raise ValueError( "paddle.distributed initialize error, " "backend argument can only be one of " "'nccl', 'gloo', 'bkcl', 'auto', 'hccl', 'heter', 'xccl' " - "but got %s" % backend) + "but got %s" % backend + ) if backend == 'nccl' and not fluid.core.is_compiled_with_cuda(): raise ValueError( @@ -1989,7 +2170,8 @@ def check_backend(backend): def block_windows_and_macos(backend): - if backend != 'gloo': return + if backend != 'gloo': + return if utils.OS_NAME.startswith('darwin'): # MACOS , block raise ValueError( "You are going to using gloo on macos, but currently is not supported" diff --git a/python/paddle/distributed/fleet/layers/mpu/mp_layers.py b/python/paddle/distributed/fleet/layers/mpu/mp_layers.py index 673f9b0f8a7..7ce317f8a38 100644 --- a/python/paddle/distributed/fleet/layers/mpu/mp_layers.py +++ b/python/paddle/distributed/fleet/layers/mpu/mp_layers.py @@ -86,27 +86,38 @@ class VocabParallelEmbedding(Layer): return x """ - def __init__(self, - num_embeddings, - embedding_dim, - weight_attr=None, - mp_group=None, - name=None): + def __init__( + self, + num_embeddings, + embedding_dim, + weight_attr=None, + mp_group=None, + name=None, + ): super(VocabParallelEmbedding, self).__init__() - self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( - ) if mp_group is None else mp_group - self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( - ) if mp_group is None else mp_group.nranks - self.rank = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank( - ) if mp_group is None else mp_group.rank + self.model_parallel_group = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group() + if mp_group is None + else mp_group + ) + self.world_size = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size() + if mp_group is None + else mp_group.nranks + ) + self.rank = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank() + if mp_group is None + else mp_group.rank + ) self.origin_num_embeddings = num_embeddings - self.is_mp = (self.world_size > 1) + self.is_mp = self.world_size > 1 - assert num_embeddings % self.world_size == 0, ( - "The length of the vocabulary must be divisible by the parallelism degree of MP" - ) + assert ( + num_embeddings % self.world_size == 0 + ), "The length of the vocabulary must be divisible by the parallelism degree of MP" per_part_size = num_embeddings // self.world_size @@ -118,15 +129,19 @@ class VocabParallelEmbedding(Layer): if self.is_mp and paddle.in_dynamic_mode(): with get_rng_state_tracker().rng_state(): - self.weight = self.create_parameter(attr=self._weight_attr, - shape=self._size, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False, + ) else: - self.weight = self.create_parameter(attr=self._weight_attr, - shape=self._size, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False, + ) self.weight.is_distributed = True if self.is_mp else False @@ -136,17 +151,22 @@ class VocabParallelEmbedding(Layer): self.weight, x, start_index=self.vocab_start_index, - name=self._name) - output = mp_ops._mp_allreduce(output_parallel, - group=self.model_parallel_group, - use_calc_stream=True, - use_model_parallel=True) + name=self._name, + ) + output = mp_ops._mp_allreduce( + output_parallel, + group=self.model_parallel_group, + use_calc_stream=True, + use_model_parallel=True, + ) else: - output = F.embedding(x, - weight=self.weight, - padding_idx=None, - sparse=False, - name=self._name) + output = F.embedding( + x, + weight=self.weight, + padding_idx=None, + sparse=False, + name=self._name, + ) return output @@ -200,29 +220,39 @@ class ColumnParallelLinear(Layer): return x """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - has_bias=None, - gather_output=True, - fuse_matmul_bias=False, - mp_group=None, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + has_bias=None, + gather_output=True, + fuse_matmul_bias=False, + mp_group=None, + name=None, + ): super(ColumnParallelLinear, self).__init__() - self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( - ) if mp_group is None else mp_group - self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( - ) if mp_group is None else mp_group.nranks + self.model_parallel_group = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group() + if mp_group is None + else mp_group + ) + self.world_size = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size() + if mp_group is None + else mp_group.nranks + ) self._name = name - self.is_mp = (self.world_size > 1) + self.is_mp = self.world_size > 1 self.gather_output = gather_output assert out_features % self.world_size == 0, ( "Number of column of the weight for linear ({}) must be" " divisible by model parallel size ({})".format( - out_features, self.world_size)) + out_features, self.world_size + ) + ) self.output_size_per_partition = out_features // self.world_size self._weight_attr = weight_attr @@ -234,13 +264,15 @@ class ColumnParallelLinear(Layer): shape=[in_features, self.output_size_per_partition], attr=self._weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) else: self.weight = self.create_parameter( shape=[in_features, self.output_size_per_partition], attr=self._weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) self.weight.is_distributed = True if self.is_mp else False @@ -250,7 +282,8 @@ class ColumnParallelLinear(Layer): shape=[self.output_size_per_partition], attr=paddle.nn.initializer.Constant(value=0.0), dtype=self._dtype, - is_bias=True) + is_bias=True, + ) self.bias.is_distributed = True if self.is_mp else False else: self.bias = None @@ -263,26 +296,29 @@ class ColumnParallelLinear(Layer): "You set fuse_matmul_bias=True in ColumnParallelLinear, " "however, the paddle you are using not support this operation. " "Please set fuse_matmul_bias=False or use paddle compiled " - "with cuda 11.6 or higher.") + "with cuda 11.6 or higher." + ) from paddle.incubate.nn.functional import fused_linear + self.linear = fused_linear def forward(self, x): # use inner api to process identity if self.is_mp: - input_parallel = mp_ops._c_identity(x, - group=self.model_parallel_group) + input_parallel = mp_ops._c_identity( + x, group=self.model_parallel_group + ) else: input_parallel = x - output_parallel = self.linear(input_parallel, - self.weight, - self.bias, - name=self._name) + output_parallel = self.linear( + input_parallel, self.weight, self.bias, name=self._name + ) if self.gather_output and self.is_mp: - output = mp_ops._c_concat(output_parallel, - group=self.model_parallel_group) + output = mp_ops._c_concat( + output_parallel, group=self.model_parallel_group + ) else: output = output_parallel return output @@ -338,15 +374,17 @@ class RowParallelLinear(Layer): return x """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - has_bias=True, - input_is_parallel=False, - fuse_matmul_bias=False, - mp_group=None, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + has_bias=True, + input_is_parallel=False, + fuse_matmul_bias=False, + mp_group=None, + name=None, + ): super(RowParallelLinear, self).__init__() self.in_features = in_features @@ -356,18 +394,29 @@ class RowParallelLinear(Layer): self._dtype = self._helper.get_default_dtype() self._name = name - self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( - ) if mp_group is None else mp_group - self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( - ) if mp_group is None else mp_group.nranks - self.rank = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank( - ) if mp_group is None else mp_group.rank + self.model_parallel_group = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group() + if mp_group is None + else mp_group + ) + self.world_size = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size() + if mp_group is None + else mp_group.nranks + ) + self.rank = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank() + if mp_group is None + else mp_group.rank + ) - self.is_mp = (self.world_size > 1) + self.is_mp = self.world_size > 1 assert in_features % self.world_size == 0, ( "Number of row of the weight for linear ({}) must be" " divisible by model parallel size ({})".format( - in_features, self.world_size)) + in_features, self.world_size + ) + ) self.input_size_per_partition = in_features // self.world_size @@ -377,13 +426,15 @@ class RowParallelLinear(Layer): shape=[self.input_size_per_partition, self.out_features], attr=self._weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) else: self.weight = self.create_parameter( shape=[self.input_size_per_partition, self.out_features], attr=self._weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) self.weight.is_distributed = True if self.is_mp else False @@ -392,7 +443,8 @@ class RowParallelLinear(Layer): shape=[self.out_features], attr=paddle.nn.initializer.Constant(value=0.0), dtype=self._dtype, - is_bias=True) + is_bias=True, + ) else: self.bias = None @@ -404,8 +456,10 @@ class RowParallelLinear(Layer): "You set fuse_matmul_bias=True in RowParallelLinear, " "however, the paddle you are using not support this operation. " "Please set fuse_matmul_bias=False or use paddle compiled " - "with cuda 11.6 or higher.") + "with cuda 11.6 or higher." + ) from paddle.incubate.nn.functional import fused_linear + self.linear = fused_linear def forward(self, x): @@ -416,19 +470,20 @@ class RowParallelLinear(Layer): input_parallel = mp_ops._c_split(x, group=self.model_parallel_group) if self.is_mp: - output_parallel = self.linear(input_parallel, - self.weight, - name=self._name) - output_ = mp_ops._mp_allreduce(output_parallel, - group=self.model_parallel_group, - use_calc_stream=True, - use_model_parallel=True) + output_parallel = self.linear( + input_parallel, self.weight, name=self._name + ) + output_ = mp_ops._mp_allreduce( + output_parallel, + group=self.model_parallel_group, + use_calc_stream=True, + use_model_parallel=True, + ) output = output_ + self.bias if self.bias is not None else output_ else: - output = self.linear(input_parallel, - self.weight, - self.bias, - name=self._name) + output = self.linear( + input_parallel, self.weight, self.bias, name=self._name + ) return output @@ -451,14 +506,24 @@ class ParallelCrossEntropy(Layer): def __init__(self, mp_group=None, name=None): super(ParallelCrossEntropy, self).__init__() self.name = name - self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( - ) if mp_group is None else mp_group - self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( - ) if mp_group is None else mp_group.nranks - self.rank = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank( - ) if mp_group is None else mp_group.rank + self.model_parallel_group = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group() + if mp_group is None + else mp_group + ) + self.world_size = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size() + if mp_group is None + else mp_group.nranks + ) + self.rank = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank() + if mp_group is None + else mp_group.rank + ) def forward(self, input, label): loss = mp_ops._c_softmax_with_cross_entropy( - input, label, group=self.model_parallel_group) + input, label, group=self.model_parallel_group + ) return loss diff --git a/python/paddle/distributed/fleet/layers/mpu/mp_ops.py b/python/paddle/distributed/fleet/layers/mpu/mp_ops.py index 30c2a7ea3c4..b1627d5a3b7 100644 --- a/python/paddle/distributed/fleet/layers/mpu/mp_ops.py +++ b/python/paddle/distributed/fleet/layers/mpu/mp_ops.py @@ -47,12 +47,17 @@ def _c_identity(tensor, group=None): from paddle.autograd import PyLayer class c_identity_eager(PyLayer): - @staticmethod def forward(ctx, tensor): - return _legacy_C_ops.c_identity(tensor, 'use_calc_stream', True, - 'ring_id', group.id, - 'use_model_parallel', True) + return _legacy_C_ops.c_identity( + tensor, + 'use_calc_stream', + True, + 'ring_id', + group.id, + 'use_model_parallel', + True, + ) @staticmethod def backward(ctx, dy): @@ -63,25 +68,36 @@ def _c_identity(tensor, group=None): return c_identity_eager.apply(tensor) elif _in_legacy_dygraph(): - return _legacy_C_ops.c_identity(tensor, 'use_calc_stream', True, - 'ring_id', ring_id, - 'use_model_parallel', True) + return _legacy_C_ops.c_identity( + tensor, + 'use_calc_stream', + True, + 'ring_id', + ring_id, + 'use_model_parallel', + True, + ) op_type = 'c_identity' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=tensor.dtype) check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - '_c_identity') - - helper.append_op(type=op_type, - inputs={'X': tensor}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'use_model_parallel': True, - }) + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) + + helper.append_op( + type=op_type, + inputs={'X': tensor}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'use_model_parallel': True, + }, + ) return out @@ -107,29 +123,43 @@ def _c_concat(tensor, group=None): nranks = group.nranks if _non_static_mode(): - return _legacy_C_ops.c_concat(tensor, 'ring_id', ring_id, - 'use_calc_stream', True, 'rank', rank, - 'nranks', nranks, 'use_model_parallel', - True) + return _legacy_C_ops.c_concat( + tensor, + 'ring_id', + ring_id, + 'use_calc_stream', + True, + 'rank', + rank, + 'nranks', + nranks, + 'use_model_parallel', + True, + ) op_type = 'c_concat' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=tensor.dtype) check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - '_c_concat') - - helper.append_op(type=op_type, - inputs={'X': tensor}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'use_model_parallel': True, - 'nranks': nranks, - 'rank': rank - }) + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_concat', + ) + + helper.append_op( + type=op_type, + inputs={'X': tensor}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'use_model_parallel': True, + 'nranks': nranks, + 'rank': rank, + }, + ) return out @@ -152,42 +182,61 @@ def _c_split(tensor, group=None): global_rank = collective._get_global_env().rank rank = global_rank if group is None else group.get_group_rank(global_rank) - nranks = collective._get_global_env( - ).world_size if group is None else group.nranks + nranks = ( + collective._get_global_env().world_size + if group is None + else group.nranks + ) if _non_static_mode(): - return _legacy_C_ops.c_split(tensor, 'use_calc_stream', True, 'ring_id', - ring_id, 'rank', rank, 'nranks', nranks, - 'use_model_parallel', True) + return _legacy_C_ops.c_split( + tensor, + 'use_calc_stream', + True, + 'ring_id', + ring_id, + 'rank', + rank, + 'nranks', + nranks, + 'use_model_parallel', + True, + ) op_type = 'c_split' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=tensor.dtype) check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - '_c_split') - - helper.append_op(type=op_type, - inputs={'X': tensor}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'rank': rank, - 'nranks': nranks, - 'use_model_parallel': True, - }) + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_split', + ) + + helper.append_op( + type=op_type, + inputs={'X': tensor}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'rank': rank, + 'nranks': nranks, + 'use_model_parallel': True, + }, + ) return out -def _mp_allreduce(tensor, - op=ReduceOp.SUM, - group=None, - use_calc_stream=True, - use_model_parallel=True): - """[it is same as allreduce above, but it supports model parallel. And it support inplace startegy] - """ +def _mp_allreduce( + tensor, + op=ReduceOp.SUM, + group=None, + use_calc_stream=True, + use_model_parallel=True, +): + """[it is same as allreduce above, but it supports model parallel. And it support inplace startegy]""" if group is not None and not group.is_member(): return @@ -198,38 +247,57 @@ def _mp_allreduce(tensor, from paddle.autograd import PyLayer class mp_allreduce_eager(PyLayer): - @staticmethod - def forward(ctx, tensor, group, use_calc_stream, - use_model_parallel): + def forward( + ctx, tensor, group, use_calc_stream, use_model_parallel + ): ctx.ring_id = group.id if use_calc_stream: op_type = collective._get_reduce_op(op, "_mp_allreduce") group.process_group.allreduce_on_calc_stream( - tensor, op_type) + tensor, op_type + ) return tensor else: return _legacy_C_ops.c_allreduce_sum_( - tensor, 'use_calc_stream', use_calc_stream, 'ring_id', - ring_id, "use_model_parallel", use_model_parallel) + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + "use_model_parallel", + use_model_parallel, + ) @staticmethod def backward(ctx, dy): - return _legacy_C_ops.c_identity(dy, 'use_calc_stream', True, - 'ring_id', ctx.ring_id, - 'use_model_parallel', True) - - return mp_allreduce_eager.apply(tensor, group, use_calc_stream, - use_model_parallel) + return _legacy_C_ops.c_identity( + dy, + 'use_calc_stream', + True, + 'ring_id', + ctx.ring_id, + 'use_model_parallel', + True, + ) + + return mp_allreduce_eager.apply( + tensor, group, use_calc_stream, use_model_parallel + ) ring_id = 0 if group is None else group.id if _in_legacy_dygraph(): if op == ReduceOp.SUM: - return _legacy_C_ops.c_allreduce_sum_(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, "use_model_parallel", - use_model_parallel) + return _legacy_C_ops.c_allreduce_sum_( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + "use_model_parallel", + use_model_parallel, + ) else: raise ValueError("Unknown parameter: {}.".format(op)) @@ -238,17 +306,22 @@ def _mp_allreduce(tensor, out = helper.create_variable_for_type_inference(dtype=tensor.dtype) check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - op_type) - - helper.append_op(type=op_type, - inputs={'X': tensor}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - 'use_model_parallel': use_model_parallel, - }) + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + op_type, + ) + + helper.append_op( + type=op_type, + inputs={'X': tensor}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + 'use_model_parallel': use_model_parallel, + }, + ) return out @@ -267,21 +340,21 @@ def _c_lookup_table(table, index, start_index=0, name=None): Tensor. """ if _non_static_mode(): - return _legacy_C_ops.c_embedding(table, index, "start_index", - start_index) + return _legacy_C_ops.c_embedding( + table, index, "start_index", start_index + ) op_type = 'c_embedding' helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='table') check_variable_and_dtype(index, 'input', ['int32', 'int64'], op_type) tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='c_embedding', - inputs={ - 'Ids': index, - 'W': table - }, - outputs={'Out': tmp}, - attrs={"start_index": start_index}) + helper.append_op( + type='c_embedding', + inputs={'Ids': index, 'W': table}, + outputs={'Out': tmp}, + attrs={"start_index": start_index}, + ) return tmp @@ -290,63 +363,75 @@ class _Linear(layers.Layer): Linear """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - bias_attr=None, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + bias_attr=None, + name=None, + ): super(_Linear, self).__init__() self._dtype = self._helper.get_default_dtype() self._weight_attr = weight_attr self._bias_attr = bias_attr - self.weight = self.create_parameter(shape=[in_features, out_features], - attr=self._weight_attr, - dtype=self._dtype, - is_bias=False) - self.bias = self.create_parameter(shape=[out_features], - attr=self._bias_attr, - dtype=self._dtype, - is_bias=True) + self.weight = self.create_parameter( + shape=[in_features, out_features], + attr=self._weight_attr, + dtype=self._dtype, + is_bias=False, + ) + self.bias = self.create_parameter( + shape=[out_features], + attr=self._bias_attr, + dtype=self._dtype, + is_bias=True, + ) self.name = name def forward(self, input): - out = _linear(x=input, - weight=self.weight, - bias=self.bias, - name=self.name) + out = _linear( + x=input, weight=self.weight, bias=self.bias, name=self.name + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'in_features={}, out_features={}, dtype={}{}'.format( - self.weight.shape[0], self.weight.shape[1], self._dtype, name_str) + self.weight.shape[0], self.weight.shape[1], self._dtype, name_str + ) -def _c_softmax_with_cross_entropy(logits, - label, - group=None, - return_softmax=False): +def _c_softmax_with_cross_entropy( + logits, label, group=None, return_softmax=False +): if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id global_rank = collective._get_global_env().rank rank = global_rank if group is None else group.get_group_rank(global_rank) - nranks = collective._get_global_env( - ).world_size if group is None else group.nranks + nranks = ( + collective._get_global_env().world_size + if group is None + else group.nranks + ) input_dims = len(list(logits.shape)) label_dims = len(list(label.shape)) if input_dims - 1 != label_dims and input_dims != label_dims: raise ValueError( 'Expected nput_dims - 1 = label_dims or input_dims == label_dims\ - (got nput_dims{}, label_dims{})'.format(input_dims, label_dims)) + (got nput_dims{}, label_dims{})'.format( + input_dims, label_dims + ) + ) if input_dims - 1 == label_dims: label = paddle.unsqueeze(label, axis=-1) if _non_static_mode(): softmax, loss = _legacy_C_ops.c_softmax_with_cross_entropy( - logits, label, 'ring_id', ring_id, 'rank', rank, 'nranks', nranks) + logits, label, 'ring_id', ring_id, 'rank', rank, 'nranks', nranks + ) if not return_softmax: return loss else: @@ -360,16 +445,12 @@ def _c_softmax_with_cross_entropy(logits, helper = LayerHelper('c_softmax_with_cross_entropy', **locals()) softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) loss = helper.create_variable_for_type_inference(dtype=logits.dtype) - helper.append_op(type='c_softmax_with_cross_entropy', - inputs={ - 'Logits': logits, - 'Label': label - }, - outputs={ - 'Softmax': softmax, - 'Loss': loss - }, - attrs=attrs) + helper.append_op( + type='c_softmax_with_cross_entropy', + inputs={'Logits': logits, 'Label': label}, + outputs={'Softmax': softmax, 'Loss': loss}, + attrs=attrs, + ) if return_softmax: return loss, softmax @@ -383,19 +464,30 @@ def _linear(x, weight, bias=None, name=None): """ if _non_static_mode(): pre_bias = _varbase_creator(dtype=x.dtype) - _legacy_C_ops.matmul(x, weight, pre_bias, 'transpose_X', False, - 'transpose_Y', False, "alpha", 1) - return dygraph_utils._append_bias_in_dygraph(pre_bias, - bias, - axis=len(x.shape) - 1) + _legacy_C_ops.matmul( + x, + weight, + pre_bias, + 'transpose_X', + False, + 'transpose_Y', + False, + "alpha", + 1, + ) + return dygraph_utils._append_bias_in_dygraph( + pre_bias, bias, axis=len(x.shape) - 1 + ) else: helper = LayerHelper('linear', **locals()) dtype = x.dtype - assert len( - x.shape) < 4, "X latitude is not supported greater than 3 now." + assert ( + len(x.shape) < 4 + ), "X latitude is not supported greater than 3 now." - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'linear' + ) check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear') inputs = {'X': [x], 'Y': [weight]} @@ -405,19 +497,17 @@ def _linear(x, weight, bias=None, name=None): 'alpha': 1, } tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='matmul_v2', - inputs=inputs, - outputs={'Out': tmp}, - attrs=attrs) + helper.append_op( + type='matmul_v2', inputs=inputs, outputs={'Out': tmp}, attrs=attrs + ) if bias is not None: res = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_add', - inputs={ - 'X': [tmp], - 'Y': [bias] - }, - outputs={'Out': [res]}, - attrs={'axis': len(x.shape) - 1}) + helper.append_op( + type='elementwise_add', + inputs={'X': [tmp], 'Y': [bias]}, + outputs={'Out': [res]}, + attrs={'axis': len(x.shape) - 1}, + ) else: res = tmp return res @@ -436,18 +526,20 @@ def _set_var_distributed(var): main_block._find_var_recursive(var.name).is_distributed = True -def _parallel_linear(x, - num_rows, - num_cols, - axis, - param_attr, - bias_attr, - gather_out, - inner_rank, - nranks, - split_tensor, - name, - group=None): +def _parallel_linear( + x, + num_rows, + num_cols, + axis, + param_attr, + bias_attr, + gather_out, + inner_rank, + nranks, + split_tensor, + name, + group=None, +): """ Parallel Linear @@ -466,21 +558,25 @@ def _parallel_linear(x, else: x = _c_identity(x, group=group) - linear = paddle.nn.Linear(num_rows, - num_cols, - weight_attr=param_attr, - bias_attr=bias_attr, - name=name) + linear = paddle.nn.Linear( + num_rows, + num_cols, + weight_attr=param_attr, + bias_attr=bias_attr, + name=name, + ) # NOTE: npu linear function use matmul_v2 but linear use matmul - linear_function = _linear if core.is_compiled_with_npu()\ - else paddle.nn.functional.linear + linear_function = ( + _linear if core.is_compiled_with_npu() else paddle.nn.functional.linear + ) linear_out = linear_function( x, linear.weight, # NOTE(wangxi): row split, bias need add after allreduce None if axis == 0 else linear.bias, - linear.name) + linear.name, + ) _set_var_distributed(linear.weight) # set is_distributed for splited bias @@ -489,7 +585,8 @@ def _parallel_linear(x, if axis == 1 and linear._bias_attr != False: _set_var_distributed(linear.bias) - if not gather_out: return linear_out + if not gather_out: + return linear_out out_shape = list(linear_out.shape) out_shape[0] *= 1 if axis == 0 else nranks @@ -501,40 +598,47 @@ def _parallel_linear(x, lod_level=linear_out.lod_level, persistable=False, is_data=False, - need_check_feed=linear_out.desc.need_check_feed()) + need_check_feed=linear_out.desc.need_check_feed(), + ) if axis == 0: - main_block.append_op(type='c_allreduce_sum', - inputs={'X': linear_out}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'use_model_parallel': True - }) + main_block.append_op( + type='c_allreduce_sum', + inputs={'X': linear_out}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'use_model_parallel': True, + }, + ) if linear.bias is not None: out = out + linear.bias else: - main_block.append_op(type='c_concat', - inputs={'X': linear_out}, - outputs={'Out': out}, - attrs={ - 'rank': inner_rank, - 'ring_id': ring_id, - 'nranks': nranks, - 'use_calc_stream': True, - 'use_model_parallel': True - }) + main_block.append_op( + type='c_concat', + inputs={'X': linear_out}, + outputs={'Out': out}, + attrs={ + 'rank': inner_rank, + 'ring_id': ring_id, + 'nranks': nranks, + 'use_calc_stream': True, + 'use_model_parallel': True, + }, + ) return out -def _parallel_embedding(x, - per_part_embeddings, - origin_size, - param_attr, - inner_rank, - num_partitions, - name, - group=None): +def _parallel_embedding( + x, + per_part_embeddings, + origin_size, + param_attr, + inner_rank, + num_partitions, + name, + group=None, +): """ Parallel Embedding """ @@ -551,43 +655,43 @@ def _parallel_embedding(x, dtype = helper.get_default_dtype() size = [per_part_size, origin_size[1]] - weight = helper.create_parameter(attr=param_attr, - shape=size, - dtype=dtype, - is_bias=False) + weight = helper.create_parameter( + attr=param_attr, shape=size, dtype=dtype, is_bias=False + ) if num_partitions == 1: - return paddle.nn.functional.embedding(x, - weight=weight, - padding_idx=None, - sparse=False, - name=name) + return paddle.nn.functional.embedding( + x, weight=weight, padding_idx=None, sparse=False, name=name + ) startup_block = paddle.static.default_startup_program().global_block() main_block = paddle.static.default_main_program().global_block() startup_block.vars[weight.name].is_distributed = True main_block.vars[weight.name].is_distributed = True - output_parallel = _c_lookup_table(weight, - x, - start_index=vocab_start_index, - name=name) - out = _mp_allreduce(output_parallel, - group=group, - use_calc_stream=True, - use_model_parallel=True) + output_parallel = _c_lookup_table( + weight, x, start_index=vocab_start_index, name=name + ) + out = _mp_allreduce( + output_parallel, + group=group, + use_calc_stream=True, + use_model_parallel=True, + ) return out -def split(x, - size, - operation, - axis=0, - num_partitions=1, - gather_out=True, - weight_attr=None, - bias_attr=None, - name=None): +def split( + x, + size, + operation, + axis=0, + num_partitions=1, + gather_out=True, + weight_attr=None, + bias_attr=None, + name=None, +): """ Split the weight of the specified operation into multiple devices @@ -706,14 +810,16 @@ def split(x, num_partitions=2) """ - assert isinstance( - size, - (list, tuple)), ("The type of size for " - "paddle.distributed.split must be list or tuple.") - assert len(size) == 2, ("Number of elements in size of " - "paddle.distributed.split must be two.") - assert isinstance(operation, str), ("The type of operation for " - "paddle.distributed.split must be str.") + assert isinstance(size, (list, tuple)), ( + "The type of size for " + "paddle.distributed.split must be list or tuple." + ) + assert len(size) == 2, ( + "Number of elements in size of " "paddle.distributed.split must be two." + ) + assert isinstance(operation, str), ( + "The type of operation for " "paddle.distributed.split must be str." + ) supported_operations = [ 'linear', 'embedding', @@ -721,16 +827,22 @@ def split(x, assert operation in supported_operations, ( "The operation for " "paddle.distributed.split must be one of {}.".format( - supported_operations)) + supported_operations + ) + ) if _non_static_mode(): raise ValueError( "paddle.distributed.split cannot be used in dynamic " "graph mode, plese use ParallelEmbedding, ParallelRowLinear, " - "ParallelColumnLinear instead.") + "ParallelColumnLinear instead." + ) else: from paddle.distributed.fleet import fleet - assert fleet._role_maker, ("To use paddle.distributed.split, " - "you must call fleet.init() firstly.") + + assert fleet._role_maker, ( + "To use paddle.distributed.split, " + "you must call fleet.init() firstly." + ) rank = fleet.worker_index() nranks = fleet.worker_num() @@ -738,21 +850,28 @@ def split(x, inner_rank = rank % num_partitions if operation == "embedding": - assert axis == 0, ("We only support to split the weight of embedding " - "along the first axis now.") - assert size[0] % num_partitions == 0, \ - "The length of the vocabulary must be divisible by num_partitions " \ - "but received vocabulary={} num_partitions={}".format(size[0], num_partitions) + assert axis == 0, ( + "We only support to split the weight of embedding " + "along the first axis now." + ) + assert size[0] % num_partitions == 0, ( + "The length of the vocabulary must be divisible by num_partitions " + "but received vocabulary={} num_partitions={}".format( + size[0], num_partitions + ) + ) per_part_size = size[0] // num_partitions - emb_out = _parallel_embedding(x, - per_part_size, - size, - weight_attr, - inner_rank, - num_partitions, - name, - group=None) + emb_out = _parallel_embedding( + x, + per_part_size, + size, + weight_attr, + inner_rank, + num_partitions, + name, + group=None, + ) return emb_out else: should_split = False @@ -760,32 +879,41 @@ def split(x, assert size[0] % num_partitions == 0, ( "Number of rows of the weight for linear ({}) must be" " divisible by num_partitions ({})".format( - size[0], num_partitions)) + size[0], num_partitions + ) + ) per_part_size = size[0] // num_partitions linear_size = (per_part_size, size[1]) - if x.shape[-1] == size[0]: should_split = True + if x.shape[-1] == size[0]: + should_split = True elif axis == 1: assert size[1] % num_partitions == 0, ( "Number of column of the weight for linear ({}) must be" " divisible by num_partitions ({})".format( - size[1], num_partitions)) + size[1], num_partitions + ) + ) per_part_size = size[1] // num_partitions linear_size = (size[0], per_part_size) else: - raise ValueError("The value of axis must be 0 or 1, but the value " - "given is {}.".format(axis)) - - linear_out = _parallel_linear(x, - linear_size[0], - linear_size[1], - axis, - weight_attr, - bias_attr, - gather_out, - inner_rank, - num_partitions, - should_split, - name=name, - group=None) + raise ValueError( + "The value of axis must be 0 or 1, but the value " + "given is {}.".format(axis) + ) + + linear_out = _parallel_linear( + x, + linear_size[0], + linear_size[1], + axis, + weight_attr, + bias_attr, + gather_out, + inner_rank, + num_partitions, + should_split, + name=name, + group=None, + ) return linear_out diff --git a/python/paddle/distributed/fleet/layers/mpu/random.py b/python/paddle/distributed/fleet/layers/mpu/random.py index ff082695cb7..17442c1938a 100644 --- a/python/paddle/distributed/fleet/layers/mpu/random.py +++ b/python/paddle/distributed/fleet/layers/mpu/random.py @@ -85,6 +85,7 @@ def get_rng_state_tracker(): def model_parallel_random_seed(seed=None): import paddle.distributed.fleet as fleet + hcg = fleet.get_hybrid_communicate_group() rank = hcg.get_model_parallel_rank() @@ -105,23 +106,23 @@ def determinate_seed(rng_name): helper = LayerHelper('seed', **locals()) out = helper.create_variable_for_type_inference(dtype=paddle.int32) # set force_cpu to reduce sync copy from CPU->GPU->CPU, and reduce pipeline hang - helper.append_op(type='seed', - outputs={'Out': out}, - attrs={ - 'deterministic': True, - 'rng_name': rng_name, - 'force_cpu': True - }) + helper.append_op( + type='seed', + outputs={'Out': out}, + attrs={'deterministic': True, 'rng_name': rng_name, 'force_cpu': True}, + ) return out -def dropout(x, - p=0.5, - axis=None, - rng_name=None, - training=True, - mode="upscale_in_train", - name=None): +def dropout( + x, + p=0.5, + axis=None, + rng_name=None, + training=True, + mode="upscale_in_train", + name=None, +): """ Dropout is a regularization technique for reducing overfitting by preventing neuron co-adaption during training. The dropout operator randomly sets the @@ -191,53 +192,66 @@ def dropout(x, raise TypeError("p argument should be a number(int|float) or Variable") # fast return for p == 0 - if isinstance(p, (int, float)) and p == 0: return x + if isinstance(p, (int, float)) and p == 0: + return x assert 0 <= p <= 1, ValueError("p argument should between 0 and 1") - assert mode in ('downscale_in_infer', 'upscale_in_train'), \ - ValueError( - "mode argument should be 'downscale_in_infer' or 'upscale_in_train'") + assert mode in ('downscale_in_infer', 'upscale_in_train'), ValueError( + "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" + ) - assert axis is None, \ - TypeError("unsupport axis when using random seed generator") + assert axis is None, TypeError( + "unsupport axis when using random seed generator" + ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer # dygraph using tracker, doesn't need determinate seed if _non_static_mode(): - out, mask = _legacy_C_ops.dropout(x, 'dropout_prob', p, 'is_test', - not training, 'fix_seed', False, - 'seed', 0, 'dropout_implementation', - mode) + out, mask = _legacy_C_ops.dropout( + x, + 'dropout_prob', + p, + 'is_test', + not training, + 'fix_seed', + False, + 'seed', + 0, + 'dropout_implementation', + mode, + ) return out seed = determinate_seed(rng_name) if isinstance(p, Variable) and not p.shape != [1]: raise TypeError( - "Required p.shape == [1] if type(p) is Variable, but received p.shape = {}" - .format(p.shape)) + "Required p.shape == [1] if type(p) is Variable, but received p.shape = {}".format( + p.shape + ) + ) helper = LayerHelper('dropout', **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'dropout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'dropout' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) mask = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) - - helper.append_op(type='dropout', - inputs={ - 'X': [x], - 'Seed': seed - }, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs={ - 'dropout_prob': p, - 'is_test': not training, - 'dropout_implementation': mode, - }) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) + + helper.append_op( + type='dropout', + inputs={'X': [x], 'Seed': seed}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs={ + 'dropout_prob': p, + 'is_test': not training, + 'dropout_implementation': mode, + }, + ) return out diff --git a/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py index 78a53ccdba5..f9e8b6b8e21 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py @@ -18,7 +18,6 @@ __all__ = [] class AMPOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(AMPOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -32,11 +31,12 @@ class AMPOptimizer(MetaOptimizerBase): ] self.meta_optimizers_black_list = ["DGCOptimizer"] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(AMPOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(AMPOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _init_wrapped_opt(self): if self.wrapped_opt is not None: @@ -48,14 +48,21 @@ class AMPOptimizer(MetaOptimizerBase): custom_black_list = set(config['custom_black_list']) custom_black_varnames = set(config['custom_black_varnames']) amp_lists = mixed_precision.AutoMixedPrecisionLists( - custom_white_list, custom_black_list, custom_black_varnames) + custom_white_list, custom_black_list, custom_black_varnames + ) self.wrapped_opt = mixed_precision.decorate( - self.inner_opt, amp_lists, config['init_loss_scaling'], - config['incr_every_n_steps'], config['decr_every_n_nan_or_inf'], - config['incr_ratio'], config['decr_ratio'], - config['use_dynamic_loss_scaling'], config['use_pure_fp16'], - config['use_fp16_guard']) + self.inner_opt, + amp_lists, + config['init_loss_scaling'], + config['incr_every_n_steps'], + config['decr_every_n_nan_or_inf'], + config['incr_ratio'], + config['decr_ratio'], + config['use_dynamic_loss_scaling'], + config['use_pure_fp16'], + config['use_fp16_guard'], + ) # if worker_num > 1, all cards will communication with each other, # add is_distributed to optimize amp, overlap communication and @@ -87,46 +94,46 @@ class AMPOptimizer(MetaOptimizerBase): "decr_every_n_nan_or_inf": 2, "incr_ratio": 2.0, "decr_ratio": 0.8, - "use_dynamic_loss_scaling": True + "use_dynamic_loss_scaling": True, } - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): # maybe inner_opt of other meta optimizer self._init_wrapped_opt() - return self.wrapped_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + return self.wrapped_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): return self.wrapped_opt.apply_gradients(params_grads=params_grads) def apply_optimize(self, loss, startup_program, params_grads): - return self.wrapped_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + return self.wrapped_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self._init_wrapped_opt() - optimize_ops, params_grads = \ - self.wrapped_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + optimize_ops, params_grads = self.wrapped_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads - def amp_init(self, - place, - scope=None, - test_program=None, - use_fp16_test=False): - return self.wrapped_opt.amp_init(place, scope, test_program, - use_fp16_test) + def amp_init( + self, place, scope=None, test_program=None, use_fp16_test=False + ): + return self.wrapped_opt.amp_init( + place, scope, test_program, use_fp16_test + ) def get_loss_scaling(self): return self.wrapped_opt.get_loss_scaling() diff --git a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py index c78c855a285..ee3f96e243f 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py @@ -25,7 +25,6 @@ __all__ = [] class AscendIRParser(object): - def __init__(self, auto_dp=False, world_rank_size=1): self.graph_idx = 0 self.hcom_endpoints = {} @@ -39,24 +38,33 @@ class AscendIRParser(object): for id, var in enumerate(input_varlist): if var.is_data: # input data ge_input = core.GEOperatorFactory.create_operator( - var.name, "Data").set_attr_int32("index", id) + var.name, "Data" + ).set_attr_int32("index", id) ret_map[var.name] = ge_input ge_in_operator.append(ge_input) else: # param, learning ... ge_input = core.GEOperatorFactory.create_operator( - var.name, "Variable") + var.name, "Variable" + ) ge_input.update_output_desc( "y", - core.GETensorDesc(core.GEShape(var.shape), - core.GEFormat.FORMAT_ND, - core.GEDataType.DT_FLOAT)) + core.GETensorDesc( + core.GEShape(var.shape), + core.GEFormat.FORMAT_ND, + core.GEDataType.DT_FLOAT, + ), + ) ret_map[var.name] = ge_input return ge_in_operator, ret_map def _endpoint_to_world_rank_id(self, endpoint): world_endpoints = fleet.worker_endpoints() - assert endpoint in world_endpoints, "endpoint (%s) not in worker_endpoints (%s) " % ( - endpoint, fleet.world_device_ids()) + assert ( + endpoint in world_endpoints + ), "endpoint (%s) not in worker_endpoints (%s) " % ( + endpoint, + fleet.world_device_ids(), + ) return world_endpoints.index(endpoint) def parse_op(self, op): @@ -72,13 +80,16 @@ class AscendIRParser(object): self.hcom_endpoints[nccl_id] = other_endpoints[:] self.hcom_endpoints[nccl_id].insert(rank, endpoint) - print("nccl_id (%s) registered endpoints %s" % - (nccl_id, self.hcom_endpoints[nccl_id])) + print( + "nccl_id (%s) registered endpoints %s" + % (nccl_id, self.hcom_endpoints[nccl_id]) + ) elif op.type == 'c_comm_init': nccl_id = op.input_arg_names[0] nranks = op.attr("nranks") - assert nranks == len(self.hcom_endpoints[nccl_id] - ), "nranks doesn't match endpoint count" + assert nranks == len( + self.hcom_endpoints[nccl_id] + ), "nranks doesn't match endpoint count" rank = op.attr("rank") ring_id = op.attr("ring_id") @@ -88,24 +99,29 @@ class AscendIRParser(object): for endpoint in self.hcom_endpoints[nccl_id] ] self.groups_to_create.append( - HcomGroupConfig(name=group_name, - nranks=nranks, - rank_ids=global_rank_ids)) - print("append to create group: %s, with rank_ids: %s" % - (group_name, global_rank_ids)) + HcomGroupConfig( + name=group_name, nranks=nranks, rank_ids=global_rank_ids + ) + ) + print( + "append to create group: %s, with rank_ids: %s" + % (group_name, global_rank_ids) + ) elif op.type in ascend_parser.registerd_op: op_parser = self.parser_factory.create_parse( - ascend_parser.registerd_op[op.type]) + ascend_parser.registerd_op[op.type] + ) op_parser.apply(op) else: - assert False, "Op[%s] has not been registered, so we have to skip it" % ( - op.type) - - def _parse_program(self, - graph_name, - program, - input_varlist=[], - fetch_list=[]): + assert ( + False + ), "Op[%s] has not been registered, so we have to skip it" % ( + op.type + ) + + def _parse_program( + self, graph_name, program, input_varlist=[], fetch_list=[] + ): begin_graph_idx = self.graph_idx ge_in_operator = [] ge_out_operator = [] @@ -121,7 +137,8 @@ class AscendIRParser(object): ge_in_operator, self.var2geop = self._construct_input_map(input_varlist) self.parser_factory = ascend_parser.AscendParserFactory( - graph, self.var2geop) + graph, self.var2geop + ) for i, curop in list(enumerate(block.ops)): self.parse_op(curop) @@ -150,28 +167,36 @@ class AscendIRParser(object): input_varlist = [var for var in input_varlist if var.is_data] - block.append_op(type="ascend_trigger", - inputs={"FeedList": input_varlist}, - outputs={"FetchList": fetch_list}, - attrs={'graph_idx': self.graph_idx}) + block.append_op( + type="ascend_trigger", + inputs={"FeedList": input_varlist}, + outputs={"FetchList": fetch_list}, + attrs={'graph_idx': self.graph_idx}, + ) self.graph_idx += 1 return graph - def parse_program(self, startup_program, main_program, input_varlist, - fetch_list): + def parse_program( + self, startup_program, main_program, input_varlist, fetch_list + ): startup_graph = self._parse_program("startup", startup_program) - main_graph = self._parse_program("main", main_program, input_varlist, - fetch_list) + main_graph = self._parse_program( + "main", main_program, input_varlist, fetch_list + ) if self._auto_dp and self._world_rank_size > 1: - assert len(self.groups_to_create - ) == 0, "can't parse program under auto_dp mode" + assert ( + len(self.groups_to_create) == 0 + ), "can't parse program under auto_dp mode" from paddle.distributed import fleet + self.groups_to_create.append( - HcomGroupConfig(name="hcom_group_0", - nranks=fleet.world_size(), - rank_ids=[x - for x in range(fleet.world_size())])) + HcomGroupConfig( + name="hcom_group_0", + nranks=fleet.world_size(), + rank_ids=[x for x in range(fleet.world_size())], + ) + ) return startup_graph, main_graph @@ -179,7 +204,6 @@ class AscendIRParser(object): # AscendOptimizer is a wrapper for basic optimizer now # We will make it part of fleet meta_optimizer in the future class AscendOptimizer(Optimizer): - def __init__(self, optimizer, fetch_list=[]): self.inner_opt = optimizer self.fetch_list = fetch_list @@ -209,28 +233,34 @@ class AscendOptimizer(Optimizer): ret_list.append(var) return ret_list - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - auto_dp=False, - rank_table_file=None, - precision_mode="must_keep_origin_dtype"): + def minimize( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + auto_dp=False, + rank_table_file=None, + precision_mode="must_keep_origin_dtype", + ): minimized = None if self.inner_opt: - minimized = self.inner_opt.minimize(loss, - startup_program=startup_program) + minimized = self.inner_opt.minimize( + loss, startup_program=startup_program + ) self.ascend_instance = core.AscendInstance() from paddle.distributed import fleet + if auto_dp and fleet.world_size() > 1: from paddle.fluid.transpiler import ascend_transpiler - t = ascend_transpiler.AscendTranspiler(startup_program, - loss.block.program) + + t = ascend_transpiler.AscendTranspiler( + startup_program, loss.block.program + ) t.transpile() - #print(loss.block.program) + # print(loss.block.program) # Config about Graph Engine can be found in https://support.huaweicloud.com/ config = { @@ -251,17 +281,21 @@ class AscendOptimizer(Optimizer): self.ascend_instance.init_global_resources() main_block = loss.block - self.parser = AscendIRParser(auto_dp=auto_dp, - world_rank_size=fleet.world_size()) + self.parser = AscendIRParser( + auto_dp=auto_dp, world_rank_size=fleet.world_size() + ) input_varlist = self._get_input_varlist(main_block.program) startup_graph, main_graph = self.parser.parse_program( - startup_program, main_block.program, input_varlist, self.fetch_list) + startup_program, main_block.program, input_varlist, self.fetch_list + ) for cfg in self.parser.groups_to_create: - print("create group (%s), nranks: %d, rank_ids: %s" % - (cfg.name, cfg.nranks, cfg.rank_ids)) + print( + "create group (%s), nranks: %d, rank_ids: %s" + % (cfg.name, cfg.nranks, cfg.rank_ids) + ) hccl.create_group(cfg.name, cfg.nranks, cfg.rank_ids) self.ascend_instance.add_ascend_subgraph(0, startup_graph) diff --git a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py index e09d8ec8bf7..52447952e55 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py +++ b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py @@ -56,7 +56,7 @@ registerd_op = { ## forwards "slice": "SliceParser", "top_k": "TopkParser", "accuracy": "AccuracyParser", - #"increment": "IncrementParser", + # "increment": "IncrementParser", "lookup_table": "LookupTableParser", "truncated_gaussian_random": "TruncatedNormalParser", "c_allgather": "AllGatherParser", @@ -71,7 +71,6 @@ registerd_op = { ## forwards "equal": "EqualParser", "expand": "ExpandParser", "squeeze2": "SqueezeParser", - ## backwords "matmul_grad": "MatMulGradParser", "mul_grad": "MulGradParser", @@ -94,17 +93,15 @@ registerd_op = { ## forwards "gather_grad": "GatherGradParser", "transpose2_grad": "TransposeGradParser", "layer_norm_grad": "LayerNormGradParser", - ## opt "sgd": "SGDParser", - #"adam": "AdamParser", + # "adam": "AdamParser", } global_cnt = -1 global_input_cnt = -1 class AscendHelper(object): - def __init__(self): self.dtype2ge_map = { 0: core.GEDataType.DT_BOOL, @@ -113,7 +110,7 @@ class AscendHelper(object): 3: core.GEDataType.DT_INT64, 4: core.GEDataType.DT_FLOAT16, 5: core.GEDataType.DT_FLOAT, - 6: core.GEDataType.DT_DOUBLE + 6: core.GEDataType.DT_DOUBLE, } self.dtype2np_map = { 0: "bool", @@ -122,23 +119,24 @@ class AscendHelper(object): 3: "int64", 4: "float16", 5: "float32", - 6: "float64" + 6: "float64", } self.dtype2paddle_inv_map = {"VarType.FP32": 0, "VarType.FP16": 1} def dtype2ge(self, dtype): assert dtype in self.dtype2ge_map, "dtype[%d] is not supported %d" % ( - dtype) + dtype + ) return self.dtype2ge_map[dtype] def dtype2np(self, index): assert index in self.dtype2np_map, "index[%d] is not supported %d" % ( - index) + index + ) return self.dtype2np_map[index] class AscendParserFactory(object): - def __init__(self, graph, var2geop): self.graph = graph self.var2geop = var2geop @@ -152,7 +150,6 @@ class AscendParserFactory(object): class AscendParserBase(object): - def __init__(self, graph, var2geop): self.graph = graph self.var2geop = var2geop @@ -161,35 +158,42 @@ class AscendParserBase(object): def _get_ge_input(self, input_var_name): assert input_var_name in self.var2geop, "var %s not created before" % ( - input_var_name) + input_var_name + ) return self.var2geop[input_var_name] def update_output(self, geop_list, index_list): output_num = len(self.op.output_names) - assert output_num == len( - index_list - ), "Parser[%s]'s output number[%d] is not equal to parameters number[%d]" % ( - self.parser_name, len(index_list), output_num) + assert output_num == len(index_list), ( + "Parser[%s]'s output number[%d] is not equal to parameters number[%d]" + % (self.parser_name, len(index_list), output_num) + ) for output_id in range(output_num): arguments = self.op.output(self.op.output_names[output_id]) if len(arguments) > 0: - assert len(arguments) == len( - index_list[output_id] - ), "Parser[%s]'s %dth argument number[%d] is not equal to paddle's number[%d]" % ( - self.parser_name, output_id, len( - index_list[output_id]), len(arguments)) + assert len(arguments) == len(index_list[output_id]), ( + "Parser[%s]'s %dth argument number[%d] is not equal to paddle's number[%d]" + % ( + self.parser_name, + output_id, + len(index_list[output_id]), + len(arguments), + ) + ) for i in range(len(arguments)): self.var2geop[arguments[i]] = geop_list[ - index_list[output_id][i]] + index_list[output_id][i] + ] for geop in geop_list: self.graph.add_op(geop) def apply(self, op): self.op = op - assert self.op.type == self.parser_name, "op [%s] != parser_name[%s]" % ( - self.op.type, self.parser_name) - #print("begin to parse op %s" % (self.parser_name)) + assert ( + self.op.type == self.parser_name + ), "op [%s] != parser_name[%s]" % (self.op.type, self.parser_name) + # print("begin to parse op %s" % (self.parser_name)) geop_list, index_list = self._apply() self.update_output(geop_list, index_list) @@ -205,33 +209,43 @@ class AscendParserBase(object): return name def _create_ge_tensor(self, shape, dtype, value): - tensor_desc = core.GETensorDesc(core.GEShape(shape), - core.GEFormat.FORMAT_ND, - self.ascend_helper.dtype2ge(dtype)) + tensor_desc = core.GETensorDesc( + core.GEShape(shape), + core.GEFormat.FORMAT_ND, + self.ascend_helper.dtype2ge(dtype), + ) tensor = core.GETensor(tensor_desc) - data = (value * np.ones( - (shape))).reshape(shape).astype(self.ascend_helper.dtype2np(dtype)) + data = ( + (value * np.ones((shape))) + .reshape(shape) + .astype(self.ascend_helper.dtype2np(dtype)) + ) buf = data.tobytes() data_8 = np.frombuffer(buf, dtype=np.uint8) tensor.set_data(data_8) return tensor def _get_ge_tensor(self, shape, dtype, value_list): - tensor_desc = core.GETensorDesc(core.GEShape(shape), - core.GEFormat.FORMAT_ND, - self.ascend_helper.dtype2ge(dtype)) + tensor_desc = core.GETensorDesc( + core.GEShape(shape), + core.GEFormat.FORMAT_ND, + self.ascend_helper.dtype2ge(dtype), + ) tensor = core.GETensor(tensor_desc) - data = np.array(value_list).reshape(shape).astype( - self.ascend_helper.dtype2np(dtype)) + data = ( + np.array(value_list) + .reshape(shape) + .astype(self.ascend_helper.dtype2np(dtype)) + ) buf = data.tobytes() data_8 = np.frombuffer(buf, dtype=np.uint8) tensor.set_data(data_8) tensor_const = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) return tensor_const @@ -242,21 +256,28 @@ class AscendParserBase(object): type = core.GEDataType.DT_FLOAT var = core.GEOperatorFactory.create_operator( - "variable" + self._accumulated_op_id(), "Variable") + "variable" + self._accumulated_op_id(), "Variable" + ) var.update_output_desc( "y", - core.GETensorDesc(core.GEShape(shape), core.GEFormat.FORMAT_ND, - type)) - assign = core.GEOperatorFactory.create_operator( - "assign" + self._accumulated_op_id(), - "Assign").set_input("value", tensor).set_input("ref", var) + core.GETensorDesc( + core.GEShape(shape), core.GEFormat.FORMAT_ND, type + ), + ) + assign = ( + core.GEOperatorFactory.create_operator( + "assign" + self._accumulated_op_id(), "Assign" + ) + .set_input("value", tensor) + .set_input("ref", var) + ) return assign def _create_shape_tensor(self): - tensor_desc = core.GETensorDesc(core.GEShape([2]), - core.GEFormat.FORMAT_ND, - core.GEDataType.DT_INT32) + tensor_desc = core.GETensorDesc( + core.GEShape([2]), core.GEFormat.FORMAT_ND, core.GEDataType.DT_INT32 + ) tensor = core.GETensor(tensor_desc) data = np.ones((2)).astype("int32").reshape([2]) @@ -268,16 +289,19 @@ class AscendParserBase(object): def _get_GEtensor_shape(self, tensor): tensor_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Shape").set_input("x", tensor) - tensor_shape = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", tensor_shape).set_attr_int32("dst_type", 0) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", tensor) + tensor_shape = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", tensor_shape) + .set_attr_int32("dst_type", 0) + ) return tensor_shape class AddParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AddParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_add" @@ -285,14 +309,17 @@ class AddParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - add = core.GEOperatorFactory.create_operator( - "add" + self._accumulated_op_id(), - "Add").set_input("x1", x).set_input("x2", y) + add = ( + core.GEOperatorFactory.create_operator( + "add" + self._accumulated_op_id(), "Add" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [add], [[0]] class DotSubParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotSubParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_sub" @@ -300,14 +327,17 @@ class DotSubParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - sub = core.GEOperatorFactory.create_operator( - "sub" + self._accumulated_op_id(), - "Sub").set_input("x1", x).set_input("x2", y) + sub = ( + core.GEOperatorFactory.create_operator( + "sub" + self._accumulated_op_id(), "Sub" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [sub], [[0]] class DotMulParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotMulParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_mul" @@ -315,14 +345,17 @@ class DotMulParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - mul = core.GEOperatorFactory.create_operator( - "dotmul" + self._accumulated_op_id(), - "Mul").set_input("x1", x).set_input("x2", y) + mul = ( + core.GEOperatorFactory.create_operator( + "dotmul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [mul], [[0]] class DotDivParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotDivParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_div" @@ -330,14 +363,17 @@ class DotDivParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - div = core.GEOperatorFactory.create_operator( - "dotdiv" + self._accumulated_op_id(), - "Div").set_input("x1", x).set_input("x2", y) + div = ( + core.GEOperatorFactory.create_operator( + "dotdiv" + self._accumulated_op_id(), "Div" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [div], [[0]] class DotPowParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotPowParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_pow" @@ -345,14 +381,17 @@ class DotPowParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - pow = core.GEOperatorFactory.create_operator( - "dotpow" + self._accumulated_op_id(), - "Pow").set_input("x1", x).set_input("x2", y) + pow = ( + core.GEOperatorFactory.create_operator( + "dotpow" + self._accumulated_op_id(), "Pow" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [pow], [[0]] class LessParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LessParser, self).__init__(graph, var2geop) self.parser_name = "less_than" @@ -360,14 +399,17 @@ class LessParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - less_than = core.GEOperatorFactory.create_operator( - "less_than" + self._accumulated_op_id(), - "Less").set_input("x1", x).set_input("x2", y) + less_than = ( + core.GEOperatorFactory.create_operator( + "less_than" + self._accumulated_op_id(), "Less" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [less_than], [[0]] class MaxParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MaxParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_max" @@ -375,14 +417,17 @@ class MaxParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - max_out = core.GEOperatorFactory.create_operator( - "max" + self._accumulated_op_id(), - "Maximum").set_input("x1", x).set_input("x2", y) + max_out = ( + core.GEOperatorFactory.create_operator( + "max" + self._accumulated_op_id(), "Maximum" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [max_out], [[0]] class MinParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MinParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_min" @@ -390,15 +435,18 @@ class MinParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - min_out = core.GEOperatorFactory.create_operator( - "min" + self._accumulated_op_id(), - "Minimum").set_input("x1", x).set_input("x2", y) + min_out = ( + core.GEOperatorFactory.create_operator( + "min" + self._accumulated_op_id(), "Minimum" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [min_out], [[0]] ## cal class LogParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LogParser, self).__init__(graph, var2geop) self.parser_name = "log" @@ -406,12 +454,12 @@ class LogParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) log = core.GEOperatorFactory.create_operator( - "log" + self._accumulated_op_id(), "Log").set_input("x", x) + "log" + self._accumulated_op_id(), "Log" + ).set_input("x", x) return [log], [[0]] class SqrtParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SqrtParser, self).__init__(graph, var2geop) self.parser_name = "sqrt" @@ -419,12 +467,12 @@ class SqrtParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) sqrt = core.GEOperatorFactory.create_operator( - "sqrt" + self._accumulated_op_id(), "Sqrt").set_input("x", x) + "sqrt" + self._accumulated_op_id(), "Sqrt" + ).set_input("x", x) return [sqrt], [[0]] class PowParser(AscendParserBase): - def __init__(self, graph, var2geop): super(PowParser, self).__init__(graph, var2geop) self.parser_name = "pow" @@ -432,17 +480,19 @@ class PowParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) factor = self.op.attr("factor") - pow_value = core.GEOperatorFactory.create_operator( - "pow" + self._accumulated_op_id(), - "Power").set_input("x", x).set_attr_float( - "power", - factor).set_attr_float("scale", - 1.0).set_attr_float("shift", 0.0) + pow_value = ( + core.GEOperatorFactory.create_operator( + "pow" + self._accumulated_op_id(), "Power" + ) + .set_input("x", x) + .set_attr_float("power", factor) + .set_attr_float("scale", 1.0) + .set_attr_float("shift", 0.0) + ) return [pow_value], [[0]] class SquareParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SquareParser, self).__init__(graph, var2geop) self.parser_name = "square" @@ -450,12 +500,12 @@ class SquareParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) square = core.GEOperatorFactory.create_operator( - "square" + self._accumulated_op_id(), "Square").set_input("x", x) + "square" + self._accumulated_op_id(), "Square" + ).set_input("x", x) return [square], [[0]] class SumParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SumParser, self).__init__(graph, var2geop) self.parser_name = "sum" @@ -466,19 +516,26 @@ class SumParser(AscendParserBase): assert False, "the size of input list must large or equal 2" x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - sum = core.GEOperatorFactory.create_operator( - "sum" + self._accumulated_op_id(), - "Add").set_input("x1", x).set_input("x2", y) + sum = ( + core.GEOperatorFactory.create_operator( + "sum" + self._accumulated_op_id(), "Add" + ) + .set_input("x1", x) + .set_input("x2", y) + ) for i in range(2, len_list): y = self._get_ge_input(self.op.input_arg_names[i]) - sum = core.GEOperatorFactory.create_operator( - "sum" + self._accumulated_op_id(), - "Add").set_input("x1", sum).set_input("x2", y) + sum = ( + core.GEOperatorFactory.create_operator( + "sum" + self._accumulated_op_id(), "Add" + ) + .set_input("x1", sum) + .set_input("x2", y) + ) return [sum], [[0]] class LogicalNotParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LogicalNotParser, self).__init__(graph, var2geop) self.parser_name = "logical_not" @@ -486,28 +543,30 @@ class LogicalNotParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) logical_not = core.GEOperatorFactory.create_operator( - "logical_not" + self._accumulated_op_id(), - "LogicalNot").set_input("x", x) + "logical_not" + self._accumulated_op_id(), "LogicalNot" + ).set_input("x", x) return [logical_not], [[0]] class MeanParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MeanParser, self).__init__(graph, var2geop) self.parser_name = "mean" def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) - mean = core.GEOperatorFactory.create_operator( - "mean" + self._accumulated_op_id(), "ReduceMeanD").set_input( - "x", x).set_attr_bool("keep_dims", - False).set_attr_vec_int32("axes", []) + mean = ( + core.GEOperatorFactory.create_operator( + "mean" + self._accumulated_op_id(), "ReduceMeanD" + ) + .set_input("x", x) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) return [mean], [[0]] class ReduceSumParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReduceSumParser, self).__init__(graph, var2geop) self.parser_name = "reduce_sum" @@ -520,14 +579,18 @@ class ReduceSumParser(AscendParserBase): x_shape = self.op.block.var(self.op.input_arg_names[0]).shape if reduce_all: axes = list(range(len(x_shape))) - reduce_sum = core.GEOperatorFactory.create_operator( - "reduce_sum" + self._accumulated_op_id(), - "ReduceSumD").set_input("x", x, 0).set_attr_vec_int32( - "axes", axes).set_attr_bool("keep_dims", keep_dims) + reduce_sum = ( + core.GEOperatorFactory.create_operator( + "reduce_sum" + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", x, 0) + .set_attr_vec_int32("axes", axes) + .set_attr_bool("keep_dims", keep_dims) + ) return [reduce_sum], [[0]] -#class IncrementParser(AscendParserBase): +# class IncrementParser(AscendParserBase): # def __init__(self, graph, var2geop): # super(IncrementParser, self).__init__(graph, var2geop) # self.parser_name = "increment" @@ -544,7 +607,6 @@ class ReduceSumParser(AscendParserBase): ## matrix cal class MatMulParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MatMulParser, self).__init__(graph, var2geop) self.parser_name = "matmul" @@ -559,24 +621,31 @@ class MatMulParser(AscendParserBase): x2_shape = self.op.block.var(self.op.input_arg_names[1]).shape if len(x1_shape) > 2: - matmul = core.GEOperatorFactory.create_operator( - "matmul" + self._accumulated_op_id(), "BatchMatMul").set_input( - "x1", x).set_input("x2", y).set_attr_bool( - "adj_x1", - transpose_x).set_attr_bool("adj_x2", transpose_y) + matmul = ( + core.GEOperatorFactory.create_operator( + "matmul" + self._accumulated_op_id(), "BatchMatMul" + ) + .set_input("x1", x) + .set_input("x2", y) + .set_attr_bool("adj_x1", transpose_x) + .set_attr_bool("adj_x2", transpose_y) + ) elif len(x1_shape) == 2: - matmul = core.GEOperatorFactory.create_operator( - "matmul" + self._accumulated_op_id(), - "MatMul").set_input("x1", x).set_input("x2", y).set_attr_bool( - "transpose_x1", - transpose_x).set_attr_bool("transpose_x2", transpose_y) + matmul = ( + core.GEOperatorFactory.create_operator( + "matmul" + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", x) + .set_input("x2", y) + .set_attr_bool("transpose_x1", transpose_x) + .set_attr_bool("transpose_x2", transpose_y) + ) else: assert False, "not support" return [matmul], [[0]] class MulParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MulParser, self).__init__(graph, var2geop) self.parser_name = "mul" @@ -591,48 +660,72 @@ class MulParser(AscendParserBase): if x_num_col_dims == 1 and y_num_col_dims == 1: if len(shape_x1) == 2 and len(shape_x2) == 2: - matmul = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "MatMul").set_input("x1", x).set_input("x2", y) + matmul = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", x) + .set_input("x2", y) + ) elif len(shape_x1) == 3 and len(shape_x2) == 2: flatten_x1 = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "Flatten").set_input("x", x) - matmul = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "MatMul").set_input("x1", flatten_x1, - 0).set_input("x2", y, 0) + "flatten" + self._accumulated_op_id(), "Flatten" + ).set_input("x", x) + matmul = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", flatten_x1, 0) + .set_input("x2", y, 0) + ) else: assert False, "not support" else: if len(shape_x1) == 3 and len(shape_x2) == 2: assert x_num_col_dims == 2, "only support 2" - flatten_x1 = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "FlattenV2").set_input("x", x).set_attr_int32( - "axis", 0).set_attr_int32("end_axis", 1) - matmul_m = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "MatMul").set_input("x1", flatten_x1, - 0).set_input("x2", y, 0) - matmul_transpose = core.GEOperatorFactory.create_operator( - "transpose" + self._accumulated_op_id(), - "TransposeD").set_input("x", matmul_m).set_attr_vec_int32( - "perm", [1, 0]) + flatten_x1 = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", x) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) + matmul_m = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", flatten_x1, 0) + .set_input("x2", y, 0) + ) + matmul_transpose = ( + core.GEOperatorFactory.create_operator( + "transpose" + self._accumulated_op_id(), "TransposeD" + ) + .set_input("x", matmul_m) + .set_attr_vec_int32("perm", [1, 0]) + ) tensor = self._create_ge_tensor( - [3], 2, [shape_x2[1], shape_x1[0], shape_x1[1]]) + [3], 2, [shape_x2[1], shape_x1[0], shape_x1[1]] + ) const_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - reshape_matmul = core.GEOperatorFactory.create_operator( - "reshape" + self._accumulated_op_id(), - "Reshape").set_input("x", matmul_transpose).set_input( - "shape", const_shape).set_attr_int32("axis", 0) - matmul = core.GEOperatorFactory.create_operator( - "transpose" + self._accumulated_op_id(), - "TransposeD").set_input("x", - reshape_matmul).set_attr_vec_int32( - "perm", [1, 2, 0]) + "shape" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + reshape_matmul = ( + core.GEOperatorFactory.create_operator( + "reshape" + self._accumulated_op_id(), "Reshape" + ) + .set_input("x", matmul_transpose) + .set_input("shape", const_shape) + .set_attr_int32("axis", 0) + ) + matmul = ( + core.GEOperatorFactory.create_operator( + "transpose" + self._accumulated_op_id(), "TransposeD" + ) + .set_input("x", reshape_matmul) + .set_attr_vec_int32("perm", [1, 2, 0]) + ) else: assert False, "not support" @@ -640,7 +733,6 @@ class MulParser(AscendParserBase): class LayerNormParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LayerNormParser, self).__init__(graph, var2geop) self.parser_name = "layer_norm" @@ -654,44 +746,65 @@ class LayerNormParser(AscendParserBase): x_dtype = self.op.block.var(self.op.input_arg_names[2]).dtype shape_tensor = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) - scale_expand = core.GEOperatorFactory.create_operator( - "broadcast_to_d" + self._accumulated_op_id(), - "BroadcastTo").set_input("x", - scale).set_input("shape", shape_tensor) - bias_expand = core.GEOperatorFactory.create_operator( - "broadcast_to_d" + self._accumulated_op_id(), - "BroadcastTo").set_input("x", - bias).set_input("shape", shape_tensor) - layer_norm = core.GEOperatorFactory.create_operator( - "layer_norm" + self._accumulated_op_id(), - "LayerNorm").set_input("x", x).set_input( - "gamma", - scale_expand).set_input("beta", bias_expand).set_attr_int32( - "begin_norm_axis", begin_norm_axis).set_attr_int32( - "begin_params_axis", - begin_norm_axis).set_attr_float("epsilon", epsilon) - - cast_dtype = 0 if self.ascend_helper.dtype2paddle_inv_map[str( - x_dtype)] == 0 else 1 - y = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", layer_norm, - 0).set_attr_int32("dst_type", cast_dtype) - mean = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", layer_norm, - 1).set_attr_int32("dst_type", cast_dtype) - variance = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", layer_norm, - 2).set_attr_int32("dst_type", cast_dtype) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) + scale_expand = ( + core.GEOperatorFactory.create_operator( + "broadcast_to_d" + self._accumulated_op_id(), "BroadcastTo" + ) + .set_input("x", scale) + .set_input("shape", shape_tensor) + ) + bias_expand = ( + core.GEOperatorFactory.create_operator( + "broadcast_to_d" + self._accumulated_op_id(), "BroadcastTo" + ) + .set_input("x", bias) + .set_input("shape", shape_tensor) + ) + layer_norm = ( + core.GEOperatorFactory.create_operator( + "layer_norm" + self._accumulated_op_id(), "LayerNorm" + ) + .set_input("x", x) + .set_input("gamma", scale_expand) + .set_input("beta", bias_expand) + .set_attr_int32("begin_norm_axis", begin_norm_axis) + .set_attr_int32("begin_params_axis", begin_norm_axis) + .set_attr_float("epsilon", epsilon) + ) + + cast_dtype = ( + 0 + if self.ascend_helper.dtype2paddle_inv_map[str(x_dtype)] == 0 + else 1 + ) + y = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", layer_norm, 0) + .set_attr_int32("dst_type", cast_dtype) + ) + mean = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", layer_norm, 1) + .set_attr_int32("dst_type", cast_dtype) + ) + variance = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", layer_norm, 2) + .set_attr_int32("dst_type", cast_dtype) + ) return [y, mean, variance], [[1], [2], [0]] ## activate function class ReluParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReluParser, self).__init__(graph, var2geop) self.parser_name = "relu" @@ -699,12 +812,12 @@ class ReluParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) relu = core.GEOperatorFactory.create_operator( - "relu" + self._accumulated_op_id(), "Relu").set_input("x", x) + "relu" + self._accumulated_op_id(), "Relu" + ).set_input("x", x) return [relu], [[0]] class GeluParser(AscendParserBase): - def __init__(self, graph, var2geop): super(GeluParser, self).__init__(graph, var2geop) self.parser_name = "gelu" @@ -712,12 +825,12 @@ class GeluParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) gelu = core.GEOperatorFactory.create_operator( - "gelu" + self._accumulated_op_id(), "Gelu").set_input("x", x) + "gelu" + self._accumulated_op_id(), "Gelu" + ).set_input("x", x) return [gelu], [[0]] class TanhParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TanhParser, self).__init__(graph, var2geop) self.parser_name = "tanh" @@ -725,13 +838,13 @@ class TanhParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) tanh = core.GEOperatorFactory.create_operator( - "tanh" + self._accumulated_op_id(), "Tanh").set_input("x", x) + "tanh" + self._accumulated_op_id(), "Tanh" + ).set_input("x", x) return [tanh], [[0]] ## loss function class SoftmaxWithCrossEntropyParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SoftmaxWithCrossEntropyParser, self).__init__(graph, var2geop) self.parser_name = "softmax_with_cross_entropy" @@ -742,48 +855,65 @@ class SoftmaxWithCrossEntropyParser(AscendParserBase): cls_num = self.op.block.var(self.op.input_arg_names[1]).shape[1] softmax = core.GEOperatorFactory.create_operator( - "softmax" + self._accumulated_op_id(), - "SoftmaxV2").set_input("x", logits) - label = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", label).set_attr_int32("dst_type", 3) + "softmax" + self._accumulated_op_id(), "SoftmaxV2" + ).set_input("x", logits) + label = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", label) + .set_attr_int32("dst_type", 3) + ) tensoron = self._create_ge_tensor([1], 5, 1) on = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoron) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoron) tensoroff = self._create_ge_tensor([1], 5, 0) off = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoroff) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoroff) self._mark_as_input(on) self._mark_as_input(off) - onehot = core.GEOperatorFactory.create_operator( - "onehot" + self._accumulated_op_id(), - "OneHotD").set_input("x", - label).set_input("on_value", on).set_input( - "off_value", - off).set_attr_int32("depth", cls_num) + onehot = ( + core.GEOperatorFactory.create_operator( + "onehot" + self._accumulated_op_id(), "OneHotD" + ) + .set_input("x", label) + .set_input("on_value", on) + .set_input("off_value", off) + .set_attr_int32("depth", cls_num) + ) squeeze = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Squeeze").set_input("x", onehot) - - loss_all = core.GEOperatorFactory.create_operator( - "loss" + self._accumulated_op_id(), - "SoftmaxCrossEntropyWithLogits").set_input("features", - logits).set_input( - "labels", squeeze) - loss = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", loss_all, 0).set_attr_int32("dst_type", 0) - loss_expand = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", loss).set_attr_vec_int32("axes", [1]) + "mul" + self._accumulated_op_id(), "Squeeze" + ).set_input("x", onehot) + + loss_all = ( + core.GEOperatorFactory.create_operator( + "loss" + self._accumulated_op_id(), + "SoftmaxCrossEntropyWithLogits", + ) + .set_input("features", logits) + .set_input("labels", squeeze) + ) + loss = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", loss_all, 0) + .set_attr_int32("dst_type", 0) + ) + loss_expand = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", loss) + .set_attr_vec_int32("axes", [1]) + ) return [label, softmax, loss_expand], [[2], [1]] class SoftMaxParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SoftMaxParser, self).__init__(graph, var2geop) self.parser_name = "softmax" @@ -792,16 +922,18 @@ class SoftMaxParser(AscendParserBase): logits = self._get_ge_input(self.op.input_arg_names[0]) axes = self.op.attr("axis") - softmax = core.GEOperatorFactory.create_operator( - "softmax" + self._accumulated_op_id(), - "SoftmaxV2").set_input("x", - logits).set_attr_vec_int32("axes", [axes]) + softmax = ( + core.GEOperatorFactory.create_operator( + "softmax" + self._accumulated_op_id(), "SoftmaxV2" + ) + .set_input("x", logits) + .set_attr_vec_int32("axes", [axes]) + ) return [softmax], [[0]] ## general class ShapeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ShapeParser, self).__init__(graph, var2geop) self.parser_name = "shape" @@ -809,12 +941,12 @@ class ShapeParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) return [shape], [[0]] class FillConstantParser(AscendParserBase): - def __init__(self, graph, var2geop): super(FillConstantParser, self).__init__(graph, var2geop) self.parser_name = "fill_constant" @@ -826,27 +958,35 @@ class FillConstantParser(AscendParserBase): tensor = self._create_ge_tensor(shape, dtype, value) const = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) self._mark_as_input(const) if self.op.block.var(self.op.output('Out')[0]).persistable: - #print("%s is Persistable in fill_constant" % + # print("%s is Persistable in fill_constant" % # (self.op.output('Out')[0])) var = core.GEOperatorFactory.create_operator( - self.op.output('Out')[0], "Variable") + self.op.output('Out')[0], "Variable" + ) var.update_output_desc( "y", - core.GETensorDesc(core.GEShape(shape), core.GEFormat.FORMAT_ND, - core.GEDataType.DT_FLOAT)) - assign = core.GEOperatorFactory.create_operator( - "assign" + self._accumulated_op_id(), - "Assign").set_input("value", const).set_input("ref", var) + core.GETensorDesc( + core.GEShape(shape), + core.GEFormat.FORMAT_ND, + core.GEDataType.DT_FLOAT, + ), + ) + assign = ( + core.GEOperatorFactory.create_operator( + "assign" + self._accumulated_op_id(), "Assign" + ) + .set_input("value", const) + .set_input("ref", var) + ) return [const], [[0]] return [const], [[0]] class TruncatedNormalParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TruncatedNormalParser, self).__init__(graph, var2geop) self.parser_name = "truncated_gaussian_random" @@ -860,24 +1000,24 @@ class TruncatedNormalParser(AscendParserBase): tensor1 = self._create_ge_tensor([len(shape)], 2, shape) shape_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor1) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor1) tensor2 = self._create_ge_tensor([1], dtype, mean) mean_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor2) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor2) tensor3 = self._create_ge_tensor([1], dtype, std) std_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor3) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor3) tensor4 = self._create_ge_tensor([1], dtype, mean - 2 * std) min_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor4) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor4) tensor5 = self._create_ge_tensor([1], dtype, mean + 2 * std) max_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor5) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor5) self._mark_as_input(shape_tensor) self._mark_as_input(mean_tensor) @@ -885,34 +1025,50 @@ class TruncatedNormalParser(AscendParserBase): self._mark_as_input(min_tensor) self._mark_as_input(max_tensor) - truncated_normal = core.GEOperatorFactory.create_operator( - "truncated_normal" + self._accumulated_op_id(), - "ParameterizedTruncatedNormal").set_input( - "shape", - shape_tensor).set_input("means", mean_tensor).set_input( - "stdevs", - std_tensor).set_input("min", min_tensor).set_input( - "max", max_tensor).set_attr_int32("seed", 0) + truncated_normal = ( + core.GEOperatorFactory.create_operator( + "truncated_normal" + self._accumulated_op_id(), + "ParameterizedTruncatedNormal", + ) + .set_input("shape", shape_tensor) + .set_input("means", mean_tensor) + .set_input("stdevs", std_tensor) + .set_input("min", min_tensor) + .set_input("max", max_tensor) + .set_attr_int32("seed", 0) + ) ## wirte the output of truncatedNormal from startup_program to main_program if self.op.block.var(self.op.output('Out')[0]).persistable: - #print("%s is Persistable in truncated_normal" % + # print("%s is Persistable in truncated_normal" % # (self.op.output('Out')[0])) var = core.GEOperatorFactory.create_operator( - self.op.output('Out')[0], "Variable") + self.op.output('Out')[0], "Variable" + ) var.update_output_desc( "y", - core.GETensorDesc(core.GEShape(shape), core.GEFormat.FORMAT_ND, - core.GEDataType.DT_FLOAT)) - assign = core.GEOperatorFactory.create_operator( - "assign" + self._accumulated_op_id(), - "Assign").set_input("value", - truncated_normal).set_input("ref", var) + core.GETensorDesc( + core.GEShape(shape), + core.GEFormat.FORMAT_ND, + core.GEDataType.DT_FLOAT, + ), + ) + assign = ( + core.GEOperatorFactory.create_operator( + "assign" + self._accumulated_op_id(), "Assign" + ) + .set_input("value", truncated_normal) + .set_input("ref", var) + ) return [ - shape_tensor, mean_tensor, std_tensor, min_tensor, max_tensor, - truncated_normal + shape_tensor, + mean_tensor, + std_tensor, + min_tensor, + max_tensor, + truncated_normal, ], [[-1]] - #else: + # else: # print( # "self.op.output('Out')[0] is not persistable in truncated_noraml" # ) @@ -920,7 +1076,6 @@ class TruncatedNormalParser(AscendParserBase): class GatherParser(AscendParserBase): - def __init__(self, graph, var2geop): super(GatherParser, self).__init__(graph, var2geop) self.parser_name = "gather" @@ -930,16 +1085,18 @@ class GatherParser(AscendParserBase): x = self._get_ge_input(self.op.input_arg_names[1]) clo = self.op.block.var(self.op.input_arg_names[1]).shape[-1] - gather = core.GEOperatorFactory.create_operator( - "gather" + self._accumulated_op_id(), - "Gather").set_input("x", x).set_input("indices", - index).set_attr_bool( - "validate_indices", True) + gather = ( + core.GEOperatorFactory.create_operator( + "gather" + self._accumulated_op_id(), "Gather" + ) + .set_input("x", x) + .set_input("indices", index) + .set_attr_bool("validate_indices", True) + ) return [gather], [[0]] class ScatterParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ScatterParser, self).__init__(graph, var2geop) self.parser_name = "scatter" @@ -952,25 +1109,35 @@ class ScatterParser(AscendParserBase): index_shape = self.op.block.var(self.op.input_arg_names[0]).shape if len(index_shape) == 1: - index = core.GEOperatorFactory.create_operator( - "unsqueeze" + self.getid(), - "Unsqueeze").set_input("x", - index).set_attr_vec_int32("axes", [1]) + index = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self.getid(), "Unsqueeze" + ) + .set_input("x", index) + .set_attr_vec_int32("axes", [1]) + ) if not overwrite: - scatter_value = core.GEOperatorFactory.create_operator( - "scatter" + self._accumulated_op_id(), - "TensorScatterAdd").set_input("x", x).set_input( - "indices", index).set_input("updates", updates) + scatter_value = ( + core.GEOperatorFactory.create_operator( + "scatter" + self._accumulated_op_id(), "TensorScatterAdd" + ) + .set_input("x", x) + .set_input("indices", index) + .set_input("updates", updates) + ) else: - scatter_value = core.GEOperatorFactory.create_operator( - "scatter" + self._accumulated_op_id(), - "TensorScatterUpdate").set_input("x", x).set_input( - "indices", index).set_input("updates", updates) + scatter_value = ( + core.GEOperatorFactory.create_operator( + "scatter" + self._accumulated_op_id(), "TensorScatterUpdate" + ) + .set_input("x", x) + .set_input("indices", index) + .set_input("updates", updates) + ) return [x, index, updates, scatter_value], [[-1]] class CastParser(AscendParserBase): - def __init__(self, graph, var2geop): super(CastParser, self).__init__(graph, var2geop) self.parser_name = "cast" @@ -978,14 +1145,17 @@ class CastParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) dtype = self.op.attr("out_dtype") - cast = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x).set_attr_int32("dst_type", dtype) + cast = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x) + .set_attr_int32("dst_type", dtype) + ) return [cast], [[0]] class AssignParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AssignParser, self).__init__(graph, var2geop) self.parser_name = "assign" @@ -993,14 +1163,17 @@ class AssignParser(AscendParserBase): def _apply(self): const = self._get_ge_input(self.op.input_arg_names[0]) var = self._get_ge_input(self.op.input_arg_names[1]) - assign = core.GEOperatorFactory.create_operator( - "assign" + self._accumulated_op_id(), - "Assign").set_input("value", const).set_input("ref", var) + assign = ( + core.GEOperatorFactory.create_operator( + "assign" + self._accumulated_op_id(), "Assign" + ) + .set_input("value", const) + .set_input("ref", var) + ) return [assign], [[0]] class ScaleParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ScaleParser, self).__init__(graph, var2geop) self.parser_name = "scale" @@ -1012,27 +1185,36 @@ class ScaleParser(AscendParserBase): bias_after_scale = self.op.attr("bias_after_scale") if bias_after_scale: - scale_value = core.GEOperatorFactory.create_operator( - "scale" + self._accumulated_op_id(), - "Power").set_input("x", x).set_attr_float( - "power", - 1.0).set_attr_float("scale", - scale).set_attr_float("shift", bias) + scale_value = ( + core.GEOperatorFactory.create_operator( + "scale" + self._accumulated_op_id(), "Power" + ) + .set_input("x", x) + .set_attr_float("power", 1.0) + .set_attr_float("scale", scale) + .set_attr_float("shift", bias) + ) else: - x_add_bias = core.GEOperatorFactory.create_operator( - "adds" + self._accumulated_op_id(), - "Adds").set_input("x", x).set_attr_float("value", bias) - scale_value = core.GEOperatorFactory.create_operator( - "scale" + self._accumulated_op_id(), - "Power").set_input("x", x_add_bias).set_attr_float( - "power", - 1.0).set_attr_float("scale", - scale).set_attr_float("shift", 0.0) + x_add_bias = ( + core.GEOperatorFactory.create_operator( + "adds" + self._accumulated_op_id(), "Adds" + ) + .set_input("x", x) + .set_attr_float("value", bias) + ) + scale_value = ( + core.GEOperatorFactory.create_operator( + "scale" + self._accumulated_op_id(), "Power" + ) + .set_input("x", x_add_bias) + .set_attr_float("power", 1.0) + .set_attr_float("scale", scale) + .set_attr_float("shift", 0.0) + ) return [scale_value], [[0]] class SliceParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SliceParser, self).__init__(graph, var2geop) self.parser_name = "slice" @@ -1058,18 +1240,22 @@ class SliceParser(AscendParserBase): cnt += 1 size = [ends_cor[i] - starts_cor[i] for i in range(len(axes_cor))] - assert len(axes_cor) == len(starts_cor) == len( - ends_cor), "the three fields must have same size" - slice_value = core.GEOperatorFactory.create_operator( - "slice" + self._accumulated_op_id(), - "SliceD").set_input("x", x).set_attr_vec_int32( - "offsets", starts_cor).set_attr_vec_int32("size", size) + assert ( + len(axes_cor) == len(starts_cor) == len(ends_cor) + ), "the three fields must have same size" + slice_value = ( + core.GEOperatorFactory.create_operator( + "slice" + self._accumulated_op_id(), "SliceD" + ) + .set_input("x", x) + .set_attr_vec_int32("offsets", starts_cor) + .set_attr_vec_int32("size", size) + ) return [slice_value], [[0]] class ReshapeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReshapeParser, self).__init__(graph, var2geop) self.parser_name = "reshape2" @@ -1092,21 +1278,24 @@ class ReshapeParser(AscendParserBase): x = self._get_ge_input(self.op.input_arg_names[0]) tensor = self._create_ge_tensor([len(shape)], 2, shape) const_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - reshape = core.GEOperatorFactory.create_operator( - "reshape" + self._accumulated_op_id(), - "Reshape").set_input("x", x).set_input("shape", - const_shape).set_attr_int32( - "axis", 0) + "shape" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + reshape = ( + core.GEOperatorFactory.create_operator( + "reshape" + self._accumulated_op_id(), "Reshape" + ) + .set_input("x", x) + .set_input("shape", const_shape) + .set_attr_int32("axis", 0) + ) x_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) return [x_shape, reshape], [[1], [0]] class TransposeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TransposeParser, self).__init__(graph, var2geop) self.parser_name = "transpose2" @@ -1114,17 +1303,21 @@ class TransposeParser(AscendParserBase): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) perm = self.op.attr("axis") - transpose = core.GEOperatorFactory.create_operator( - "transpose" + self._accumulated_op_id(), - "TransposeD").set_input("x", x).set_attr_vec_int32("perm", perm) + transpose = ( + core.GEOperatorFactory.create_operator( + "transpose" + self._accumulated_op_id(), "TransposeD" + ) + .set_input("x", x) + .set_attr_vec_int32("perm", perm) + ) x_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) return [x_shape, transpose], [[1], [0]] class AccuracyParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AccuracyParser, self).__init__(graph, var2geop) self.parser_name = "accuracy" @@ -1134,42 +1327,73 @@ class AccuracyParser(AscendParserBase): label = self._get_ge_input(self.op.input_arg_names[1]) logits = self._get_ge_input(self.op.input_arg_names[2]) - pred = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", pred).set_attr_int32("dst_type", 3) - label = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", label).set_attr_int32("dst_type", 3) - equal = core.GEOperatorFactory.create_operator( - "equal" + self._accumulated_op_id(), - "Equal").set_input("x1", pred).set_input("x2", label) - cast = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", equal).set_attr_int32("dst_type", 0) - acc = core.GEOperatorFactory.create_operator( - "mean" + self._accumulated_op_id(), "ReduceMeanD").set_input( - "x", cast).set_attr_bool("keep_dims", - False).set_attr_vec_int32("axes", []) - correct = core.GEOperatorFactory.create_operator( - "sum" + self._accumulated_op_id(), "ReduceSumD").set_input( - "x", cast).set_attr_bool("keep_dims", - False).set_attr_vec_int32("axes", []) - ones_tensor = core.GEOperatorFactory.create_operator( - "oneslike" + self._accumulated_op_id(), - "OnesLike").set_input("x", label) + pred = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", pred) + .set_attr_int32("dst_type", 3) + ) + label = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", label) + .set_attr_int32("dst_type", 3) + ) + equal = ( + core.GEOperatorFactory.create_operator( + "equal" + self._accumulated_op_id(), "Equal" + ) + .set_input("x1", pred) + .set_input("x2", label) + ) + cast = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", equal) + .set_attr_int32("dst_type", 0) + ) + acc = ( + core.GEOperatorFactory.create_operator( + "mean" + self._accumulated_op_id(), "ReduceMeanD" + ) + .set_input("x", cast) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) + correct = ( + core.GEOperatorFactory.create_operator( + "sum" + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", cast) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) ones_tensor = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", ones_tensor).set_attr_int32("dst_type", 0) - total = core.GEOperatorFactory.create_operator( - "sum" + self._accumulated_op_id(), - "ReduceSumD").set_input("x", ones_tensor).set_attr_bool( - "keep_dims", False).set_attr_vec_int32("axes", []) + "oneslike" + self._accumulated_op_id(), "OnesLike" + ).set_input("x", label) + ones_tensor = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", ones_tensor) + .set_attr_int32("dst_type", 0) + ) + total = ( + core.GEOperatorFactory.create_operator( + "sum" + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", ones_tensor) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) return [acc, correct, total], [[0], [1], [2]] class TopkParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TopkParser, self).__init__(graph, var2geop) self.parser_name = "top_k" @@ -1180,25 +1404,40 @@ class TopkParser(AscendParserBase): tensor = self._create_ge_tensor([1], 2, k) const_k = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - cast_x = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x).set_attr_int32("dst_type", 1) - topk = core.GEOperatorFactory.create_operator( - "topk" + self._accumulated_op_id(), - "TopK").set_input("x", cast_x).set_input("k", const_k) - value = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", topk, 0).set_attr_int32("dst_type", 0) - index = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", topk, 1).set_attr_int32("dst_type", 0) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + cast_x = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x) + .set_attr_int32("dst_type", 1) + ) + topk = ( + core.GEOperatorFactory.create_operator( + "topk" + self._accumulated_op_id(), "TopK" + ) + .set_input("x", cast_x) + .set_input("k", const_k) + ) + value = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", topk, 0) + .set_attr_int32("dst_type", 0) + ) + index = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", topk, 1) + .set_attr_int32("dst_type", 0) + ) return [value, index], [[1], [0]] class LookupTableParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LookupTableParser, self).__init__(graph, var2geop) self.parser_name = "lookup_table" @@ -1207,17 +1446,24 @@ class LookupTableParser(AscendParserBase): ids = self._get_ge_input(self.op.input_arg_names[0]) w = self._get_ge_input(self.op.input_arg_names[1]) - ids_squeeze = core.GEOperatorFactory.create_operator( - "squeeze" + self._accumulated_op_id(), - "Squeeze").set_input("x", ids).set_attr_vec_int32("axes", [-1]) - out = core.GEOperatorFactory.create_operator( - "lookup" + self._accumulated_op_id(), - "Gather").set_input("x", w).set_input("indices", ids_squeeze) + ids_squeeze = ( + core.GEOperatorFactory.create_operator( + "squeeze" + self._accumulated_op_id(), "Squeeze" + ) + .set_input("x", ids) + .set_attr_vec_int32("axes", [-1]) + ) + out = ( + core.GEOperatorFactory.create_operator( + "lookup" + self._accumulated_op_id(), "Gather" + ) + .set_input("x", w) + .set_input("indices", ids_squeeze) + ) return [out], [[0]] class StackParser(AscendParserBase): - def __init__(self, graph, var2geop): super(StackParser, self).__init__(graph, var2geop) self.parser_name = "stack" @@ -1226,30 +1472,37 @@ class StackParser(AscendParserBase): tiles = len(self.op.input_arg_names) data_x_lst = [] for index in range(tiles): - data_x_lst.append(self._get_ge_input( - self.op.input_arg_names[index])) + data_x_lst.append( + self._get_ge_input(self.op.input_arg_names[index]) + ) axis = self.op.attr("axis") data_x = data_x_lst[0] tensor = self._create_ge_tensor([1], 2, axis) tensor_axis = core.GEOperatorFactory.create_operator( - "axis" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - expand = core.GEOperatorFactory.create_operator( - "expand" + self._accumulated_op_id(), - "ExpandDims").set_input("x", data_x).set_input("axis", tensor_axis) - - stack = core.GEOperatorFactory.create_operator( - "stack" + self._accumulated_op_id(), "TileWithAxis").set_input( - "x", - expand).set_attr_int32("axis", - axis).set_attr_int32("tiles", tiles) + "axis" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + expand = ( + core.GEOperatorFactory.create_operator( + "expand" + self._accumulated_op_id(), "ExpandDims" + ) + .set_input("x", data_x) + .set_input("axis", tensor_axis) + ) + + stack = ( + core.GEOperatorFactory.create_operator( + "stack" + self._accumulated_op_id(), "TileWithAxis" + ) + .set_input("x", expand) + .set_attr_int32("axis", axis) + .set_attr_int32("tiles", tiles) + ) return [stack], [[0]] class UnSqueezeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(UnSqueezeParser, self).__init__(graph, var2geop) self.parser_name = "unsqueeze2" @@ -1258,18 +1511,21 @@ class UnSqueezeParser(AscendParserBase): x = self._get_ge_input(self.op.input_arg_names[0]) axes = self.op.attr('axes') - output = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", x).set_attr_vec_int32("axes", axes) + output = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", x) + .set_attr_vec_int32("axes", axes) + ) shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Shape").set_input("x", output) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", output) return [shape, output], [[1], [0]] ## parallel class AllGatherParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AllGatherParser, self).__init__(graph, var2geop) self.parser_name = "c_allgather" @@ -1279,15 +1535,18 @@ class AllGatherParser(AscendParserBase): rank_size = self.op.attr("rank_size") group = self.op.attr("group") - allgather = core.GEOperatorFactory.create_operator( - "allgather" + self._accumulated_op_id(), - "HcomAllGather").set_input("x", x).set_attr_int32( - "rank_size", rank_size).set_attr_string("group", group) + allgather = ( + core.GEOperatorFactory.create_operator( + "allgather" + self._accumulated_op_id(), "HcomAllGather" + ) + .set_input("x", x) + .set_attr_int32("rank_size", rank_size) + .set_attr_string("group", group) + ) return [allgather], [[0]] class AllReduceParser(AscendParserBase): - def __init__(self, graph, var2geop, reduction): super(AllReduceParser, self).__init__(graph, var2geop) self.parser_name = "c_allreduce_" + reduction @@ -1298,13 +1557,17 @@ class AllReduceParser(AscendParserBase): reduction = self.reduction ring_id = self.op.attr("ring_id") group = "hcom_group_" + str(ring_id) - fusion = None #self.op.attr("fusion") - fusion_id = None #self.op.attr("fusion_id") - - allreduce = core.GEOperatorFactory.create_operator( - "allreduce" + self._accumulated_op_id(), - "HcomAllReduce").set_input("x", x).set_attr_string( - "reduction", reduction).set_attr_string("group", group) + fusion = None # self.op.attr("fusion") + fusion_id = None # self.op.attr("fusion_id") + + allreduce = ( + core.GEOperatorFactory.create_operator( + "allreduce" + self._accumulated_op_id(), "HcomAllReduce" + ) + .set_input("x", x) + .set_attr_string("reduction", reduction) + .set_attr_string("group", group) + ) if fusion is not None: allreduce.set_attr_int32("fusion", fusion) @@ -1314,19 +1577,16 @@ class AllReduceParser(AscendParserBase): class AllReduceSumParser(AllReduceParser): - def __init__(self, graph, var2geop): super(AllReduceSumParser, self).__init__(graph, var2geop, 'sum') class AllReduceMaxParser(AllReduceParser): - def __init__(self, graph, var2geop): super(AllReduceMaxParser, self).__init__(graph, var2geop, 'max') class BroadcastParser(AscendParserBase): - def __init__(self, graph, var2geop): super(BroadcastParser, self).__init__(graph, var2geop) self.parser_name = "c_broadcast" @@ -1336,15 +1596,18 @@ class BroadcastParser(AscendParserBase): root_rank = self.op.attr("root_rank") group = self.op.attr("group") - broadcast = core.GEOperatorFactory.create_operator( - "broadcast" + self._accumulated_op_id(), - "HcomBroadcast").set_input("x", x).set_attr_int32( - "root_rank", root_rank).set_attr_string("group", group) + broadcast = ( + core.GEOperatorFactory.create_operator( + "broadcast" + self._accumulated_op_id(), "HcomBroadcast" + ) + .set_input("x", x) + .set_attr_int32("root_rank", root_rank) + .set_attr_string("group", group) + ) return [broadcast], [[0]] class ReduceScatterParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReduceScatterParser, self).__init__(graph, var2geop) self.parser_name = "c_reduce_scatter" @@ -1355,17 +1618,19 @@ class ReduceScatterParser(AscendParserBase): group = self.op.attr("group") rank_size = self.op.attr("rank_size") - reduce_scatter = core.GEOperatorFactory.create_operator( - "reducescatter" + self._accumulated_op_id(), - "HcomReduceScatter").set_input("x", x).set_attr_string( - "reduction", - reduction).set_attr_string("group", group).set_attr_int32( - "rank_size", rank_size) + reduce_scatter = ( + core.GEOperatorFactory.create_operator( + "reducescatter" + self._accumulated_op_id(), "HcomReduceScatter" + ) + .set_input("x", x) + .set_attr_string("reduction", reduction) + .set_attr_string("group", group) + .set_attr_int32("rank_size", rank_size) + ) return [reduce_scatter], [[0]] class SendParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SendParser, self).__init__(graph, var2geop) self.parser_name = "c_send" @@ -1376,15 +1641,19 @@ class SendParser(AscendParserBase): dest_rank = self.op.attr("dest_rank") group = self.op.attr("group") - send = core.GEOperatorFactory.create_operator( - "send" + self._accumulated_op_id(), "HcomSend").set_input( - "x", x).set_attr_int32("sr_tag", sr_tag).set_attr_int32( - "dest_rank", dest_rank).set_attr_string("group", group) + send = ( + core.GEOperatorFactory.create_operator( + "send" + self._accumulated_op_id(), "HcomSend" + ) + .set_input("x", x) + .set_attr_int32("sr_tag", sr_tag) + .set_attr_int32("dest_rank", dest_rank) + .set_attr_string("group", group) + ) return [send], [[0]] class ReceiveParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReceiveParser, self).__init__(graph, var2geop) self.parser_name = "c_receive" @@ -1397,19 +1666,21 @@ class ReceiveParser(AscendParserBase): shape = self.op.attr("shape") dtype = self.op.attr("dtype") - receive = core.GEOperatorFactory.create_operator( - "receive" + self._accumulated_op_id(), - "HcomReceive").set_input("x", x).set_attr_int32( - "sr_tag", - sr_tag).set_attr_int32("src_rank", src_rank).set_attr_string( - "group", group).set_attr_vec_int32("shape", - shape).set_attr_int32( - "dtype", dtype) + receive = ( + core.GEOperatorFactory.create_operator( + "receive" + self._accumulated_op_id(), "HcomReceive" + ) + .set_input("x", x) + .set_attr_int32("sr_tag", sr_tag) + .set_attr_int32("src_rank", src_rank) + .set_attr_string("group", group) + .set_attr_vec_int32("shape", shape) + .set_attr_int32("dtype", dtype) + ) return [receive], [[0]] class RangeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(RangeParser, self).__init__(graph, var2geop) self.parser_name = "range" @@ -1420,17 +1691,19 @@ class RangeParser(AscendParserBase): end = self._get_ge_input(self.op.input_arg_names[1]) delta = self._get_ge_input(self.op.input_arg_names[2]) - ge_range = core.GEOperatorFactory.create_operator( - "range" + self._accumulated_op_id(), "Range")\ - .set_input("start", end)\ - .set_input("limit", start) \ - .set_input("delta", delta) + ge_range = ( + core.GEOperatorFactory.create_operator( + "range" + self._accumulated_op_id(), "Range" + ) + .set_input("start", end) + .set_input("limit", start) + .set_input("delta", delta) + ) return [ge_range], [[0]] class UniformRandomParser(AscendParserBase): - def __init__(self, graph, var2geop): super(UniformRandomParser, self).__init__(graph, var2geop) self.parser_name = "uniform_random" @@ -1442,35 +1715,42 @@ class UniformRandomParser(AscendParserBase): max_v = self.op.attr("max") seed = self.op.attr("seed") dtype = self.op.attr("dtype") - assert max_v > min_v, "assert max_v > min_v, but received " + \ - "as max_v={}, min_v={} ".format(max_v, min_v) + assert max_v > min_v, ( + "assert max_v > min_v, but received " + + "as max_v={}, min_v={} ".format(max_v, min_v) + ) tensor1 = self._create_ge_tensor([len(shape)], 2, shape) shape_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor1) - - ge_ur = core.GEOperatorFactory.create_operator( - "uniform_random" + self._accumulated_op_id(), "RandomUniform")\ - .set_input("shape", shape_tensor)\ - .set_attr_dtype("dtype", self.ascend_helper.dtype2ge(dtype)) \ - .set_attr_int32("seed", seed)\ + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor1) + + ge_ur = ( + core.GEOperatorFactory.create_operator( + "uniform_random" + self._accumulated_op_id(), "RandomUniform" + ) + .set_input("shape", shape_tensor) + .set_attr_dtype("dtype", self.ascend_helper.dtype2ge(dtype)) + .set_attr_int32("seed", seed) .set_attr_int32("seed2", seed) + ) scale = max_v - min_v - scale_value = core.GEOperatorFactory.create_operator( - "scale" + self._accumulated_op_id(), - "Power").set_input("x", ge_ur).set_attr_float( - "power", - 1.0).set_attr_float("scale", - scale).set_attr_float("shift", min_v) + scale_value = ( + core.GEOperatorFactory.create_operator( + "scale" + self._accumulated_op_id(), "Power" + ) + .set_input("x", ge_ur) + .set_attr_float("power", 1.0) + .set_attr_float("scale", scale) + .set_attr_float("shift", min_v) + ) return [scale_value], [[0]] class EqualParser(AscendParserBase): - def __init__(self, graph, var2geop): super(EqualParser, self).__init__(graph, var2geop) self.parser_name = "equal" @@ -1478,15 +1758,17 @@ class EqualParser(AscendParserBase): def _apply(self): data_x1 = self._get_ge_input(self.op.input_arg_names[0]) data_x2 = self._get_ge_input(self.op.input_arg_names[1]) - equal = core.GEOperatorFactory.create_operator("equal" \ - + self._accumulated_op_id(), "Equal")\ - .set_input("x1", data_x1)\ - .set_input("x2", data_x2) + equal = ( + core.GEOperatorFactory.create_operator( + "equal" + self._accumulated_op_id(), "Equal" + ) + .set_input("x1", data_x1) + .set_input("x2", data_x2) + ) return [equal], [[0]] class ExpandParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ExpandParser, self).__init__(graph, var2geop) self.parser_name = "expand" @@ -1496,19 +1778,21 @@ class ExpandParser(AscendParserBase): expand_times = self.op.attr('expand_times') tensor = self._create_ge_tensor([len(expand_times)], 2, expand_times) - expand_tensor = core.GEOperatorFactory.\ - create_operator("const" + self._accumulated_op_id(), "Const")\ - .set_attr_tensor("value", tensor) - - assign = core.GEOperatorFactory\ - .create_operator("tile" + self._accumulated_op_id(), "Tile")\ - .set_input("x", data_x1_shape)\ - .set_input("multiples", expand_tensor) + expand_tensor = core.GEOperatorFactory.create_operator( + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + + assign = ( + core.GEOperatorFactory.create_operator( + "tile" + self._accumulated_op_id(), "Tile" + ) + .set_input("x", data_x1_shape) + .set_input("multiples", expand_tensor) + ) return [assign], [[0]] class SqueezeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SqueezeParser, self).__init__(graph, var2geop) self.parser_name = "squeeze2" @@ -1517,26 +1801,28 @@ class SqueezeParser(AscendParserBase): tensor = self._get_ge_input(self.op.input_arg_names[0]) axes = self.op.attr("axes") - data_squeezed = core.GEOperatorFactory\ - .create_operator("squeeze" + self._accumulated_op_id(), "Squeeze")\ - .set_input("x", tensor)\ - .set_attr_vec_int32("axes", axes) + data_squeezed = ( + core.GEOperatorFactory.create_operator( + "squeeze" + self._accumulated_op_id(), "Squeeze" + ) + .set_input("x", tensor) + .set_attr_vec_int32("axes", axes) + ) shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Shape").set_input("x", data_squeezed) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", data_squeezed) return [shape, data_squeezed], [[1], [0]] -#****************************************************************# -#*************************** *************************# -#*************************** *************************# -#*************************** GradParser *************************# -#*************************** *************************# -#*************************** *************************# -#****************************************************************# +# ****************************************************************# +# *************************** *************************# +# *************************** *************************# +# *************************** GradParser *************************# +# *************************** *************************# +# *************************** *************************# +# ****************************************************************# ## grad class ReduceSumGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReduceSumGradParser, self).__init__(graph, var2geop) self.parser_name = "reduce_sum_grad" @@ -1546,24 +1832,27 @@ class ReduceSumGradParser(AscendParserBase): input = self._get_ge_input(self.op.input_arg_names[1]) shape_tensor = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Shape").set_input("x", input, 0) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", input, 0) tensoron = self._create_ge_tensor([1], 2, -1) const = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoron) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoron) self._mark_as_input(const) - reduce_sum = core.GEOperatorFactory.create_operator( - "broadcast_to_d" + self._accumulated_op_id(), - "BroadcastTo").set_input("x", x).set_input("shape", shape_tensor) - #reduce_sum = core.GEOperatorFactory.create_operator("expand" + self._accumulated_op_id(), "ExpandDims").set_input("x", reduce_sum).set_input("axis", const) + reduce_sum = ( + core.GEOperatorFactory.create_operator( + "broadcast_to_d" + self._accumulated_op_id(), "BroadcastTo" + ) + .set_input("x", x) + .set_input("shape", shape_tensor) + ) + # reduce_sum = core.GEOperatorFactory.create_operator("expand" + self._accumulated_op_id(), "ExpandDims").set_input("x", reduce_sum).set_input("axis", const) return [reduce_sum], [[0]] class MatMulGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MatMulGradParser, self).__init__(graph, var2geop) self.parser_name = "matmul_grad" @@ -1581,63 +1870,91 @@ class MatMulGradParser(AscendParserBase): if len(x_shape) > 2: if transpose_y: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input("x1", out_grad).set_input( - "x2", - y).set_attr_bool("adj_x1", - False).set_attr_bool("adj_x2", False) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input("x1", out_grad).set_input( - "x2", - x).set_attr_bool("adj_x1", - True).set_attr_bool("adj_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("adj_x1", False) + .set_attr_bool("adj_x2", False) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", out_grad) + .set_input("x2", x) + .set_attr_bool("adj_x1", True) + .set_attr_bool("adj_x2", False) + ) else: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input("x1", out_grad).set_input( - "x2", - y).set_attr_bool("adj_x1", - False).set_attr_bool("adj_x2", True) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input( - "x1", x).set_input("x2", out_grad).set_attr_bool( - "adj_x1", True).set_attr_bool("adj_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("adj_x1", False) + .set_attr_bool("adj_x2", True) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", x) + .set_input("x2", out_grad) + .set_attr_bool("adj_x1", True) + .set_attr_bool("adj_x2", False) + ) else: if transpose_y: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", y).set_attr_bool("transpose_x1", - False).set_attr_bool( - "transpose_x2", False) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", x).set_attr_bool("transpose_x1", - True).set_attr_bool( - "transpose_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("transpose_x1", False) + .set_attr_bool("transpose_x2", False) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", x) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) else: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", y).set_attr_bool("transpose_x1", - False).set_attr_bool( - "transpose_x2", True) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", x).set_input( - "x2", out_grad).set_attr_bool("transpose_x1", - True).set_attr_bool( - "transpose_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("transpose_x1", False) + .set_attr_bool("transpose_x2", True) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", x) + .set_input("x2", out_grad) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) return [x_grad, y_grad], [[0], [1]] class MulGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MulGradParser, self).__init__(graph, var2geop) self.parser_name = "mul_grad" @@ -1655,78 +1972,114 @@ class MulGradParser(AscendParserBase): if x_num_col_dims == 1 and y_num_col_dims == 1: if len(shape_x) == 2 and len(shape_y) == 2: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", y).set_attr_bool("transpose_x1", - False).set_attr_bool( - "transpose_x2", True) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", x).set_input( - "x2", out_grad).set_attr_bool("transpose_x1", - True).set_attr_bool( - "transpose_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("transpose_x1", False) + .set_attr_bool("transpose_x2", True) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", x) + .set_input("x2", out_grad) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) elif len(shape_x) == 3 and len(shape_y) == 2: flatten_x = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "Flatten").set_input("x", x) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", y).set_attr_bool("transpose_x1", - False).set_attr_bool( - "transpose_x2", True) + "flatten" + self._accumulated_op_id(), "Flatten" + ).set_input("x", x) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("transpose_x1", False) + .set_attr_bool("transpose_x2", True) + ) if len(shape_out_grad) == 2: - x_grad = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", x_grad).set_attr_vec_int32( - "axes", [1]) - - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", flatten_x).set_input( - "x2", out_grad).set_attr_bool("transpose_x1", - True).set_attr_bool( - "transpose_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", x_grad) + .set_attr_vec_int32("axes", [1]) + ) + + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", flatten_x) + .set_input("x2", out_grad) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) else: if len(shape_x) == 3 and len(shape_y) == 2: assert x_num_col_dims == 2, "only support 2" - flatten_x = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "FlattenV2").set_input("x", x).set_attr_int32( - "axis", 0).set_attr_int32("end_axis", 1) - flatten_out_grad = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "FlattenV2").set_input("x", out_grad).set_attr_int32( - "axis", 0).set_attr_int32("end_axis", 1) - - y_unsqueeze = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", - y).set_attr_vec_int32("axes", [0]) - y_stack = core.GEOperatorFactory.create_operator( - "stack" + self._accumulated_op_id(), - "TileWithAxis").set_input("x", y_unsqueeze).set_attr_int32( - "axis", 0).set_attr_int32("tiles", shape_out_grad[0]) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input("x1", out_grad).set_input( - "x2", y_stack).set_attr_bool("adj_x1", - False).set_attr_bool( - "adj_x2", True) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", flatten_x).set_input( - "x2", flatten_out_grad).set_attr_bool( - "transpose_x1", - True).set_attr_bool("transpose_x2", False) + flatten_x = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", x) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) + flatten_out_grad = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", out_grad) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) + + y_unsqueeze = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", y) + .set_attr_vec_int32("axes", [0]) + ) + y_stack = ( + core.GEOperatorFactory.create_operator( + "stack" + self._accumulated_op_id(), "TileWithAxis" + ) + .set_input("x", y_unsqueeze) + .set_attr_int32("axis", 0) + .set_attr_int32("tiles", shape_out_grad[0]) + ) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", out_grad) + .set_input("x2", y_stack) + .set_attr_bool("adj_x1", False) + .set_attr_bool("adj_x2", True) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", flatten_x) + .set_input("x2", flatten_out_grad) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) return [x_grad, y_grad], [[0], [1]] class ReluGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReluGradParser, self).__init__(graph, var2geop) self.parser_name = "relu_grad" @@ -1734,15 +2087,17 @@ class ReluGradParser(AscendParserBase): def _apply(self): out = self._get_ge_input(self.op.input_arg_names[0]) out_grad = self._get_ge_input(self.op.input_arg_names[1]) - relu_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReluGrad").set_input("gradients", - out_grad).set_input("features", out) + relu_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "ReluGrad" + ) + .set_input("gradients", out_grad) + .set_input("features", out) + ) return [relu_grad], [[0]] class SoftmaxWithCrossEntropyGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SoftmaxWithCrossEntropyGradParser, self).__init__(graph, var2geop) self.parser_name = "softmax_with_cross_entropy_grad" @@ -1759,39 +2114,53 @@ class SoftmaxWithCrossEntropyGradParser(AscendParserBase): tensoron = self._create_ge_tensor([1], 5, 1) on = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoron) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoron) tensoroff = self._create_ge_tensor([1], 5, 0) off = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoroff) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoroff) self._mark_as_input(on) self._mark_as_input(off) - label = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", label).set_attr_int32("dst_type", 3) - onehot = core.GEOperatorFactory.create_operator( - "onehot" + self._accumulated_op_id(), - "OneHotD").set_input("x", - label).set_input("on_value", on).set_input( - "off_value", - off).set_attr_int32("depth", cls_num) + label = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", label) + .set_attr_int32("dst_type", 3) + ) + onehot = ( + core.GEOperatorFactory.create_operator( + "onehot" + self._accumulated_op_id(), "OneHotD" + ) + .set_input("x", label) + .set_input("on_value", on) + .set_input("off_value", off) + .set_attr_int32("depth", cls_num) + ) squeeze = core.GEOperatorFactory.create_operator( - "suqeeze" + self._accumulated_op_id(), - "Squeeze").set_input("x", onehot) - sub = core.GEOperatorFactory.create_operator( - "sub" + self._accumulated_op_id(), - "Sub").set_input("x1", softmax).set_input("x2", squeeze) - grad = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Mul").set_input("x1", loss_grad).set_input("x2", sub) + "suqeeze" + self._accumulated_op_id(), "Squeeze" + ).set_input("x", onehot) + sub = ( + core.GEOperatorFactory.create_operator( + "sub" + self._accumulated_op_id(), "Sub" + ) + .set_input("x1", softmax) + .set_input("x2", squeeze) + ) + grad = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", loss_grad) + .set_input("x2", sub) + ) return [on, off, label, onehot, grad], [[-1]] class DotMulGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotMulGradParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_mul_grad" @@ -1801,18 +2170,25 @@ class DotMulGradParser(AscendParserBase): out_1 = self._get_ge_input(self.op.input_arg_names[1]) out_2 = self._get_ge_input(self.op.input_arg_names[2]) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "Mul").set_input("x1", out_grad).set_input("x2", out_2) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "Mul").set_input("x1", out_1).set_input("x2", out_grad) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", out_grad) + .set_input("x2", out_2) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", out_1) + .set_input("x2", out_grad) + ) return [x_grad, y_grad], [[0], [1]] class DotAddGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotAddGradParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_add_grad" @@ -1828,36 +2204,53 @@ class DotAddGradParser(AscendParserBase): x_grad = out_grad cur_time_x = len(out_grad_shape) - len(out_1_shape) for i in range(cur_time_x): - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReduceSumD").set_input("x", x_grad).set_attr_vec_int32( - "axes", [0]).set_attr_bool("keep_dims", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", x_grad) + .set_attr_vec_int32("axes", [0]) + .set_attr_bool("keep_dims", False) + ) for axis, size in enumerate(out_1_shape): if size == 1: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReduceSumD").set_input("x", x_grad).set_attr_vec_int32( - "axes", [axis]).set_attr_bool("keep_dims", True) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "ReduceSumD", + ) + .set_input("x", x_grad) + .set_attr_vec_int32("axes", [axis]) + .set_attr_bool("keep_dims", True) + ) y_grad = out_grad cur_time_y = len(out_grad_shape) - len(out_2_shape) for i in range(cur_time_y): - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReduceSumD").set_input("x", y_grad).set_attr_vec_int32( - "axes", [0]).set_attr_bool("keep_dims", False) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", y_grad) + .set_attr_vec_int32("axes", [0]) + .set_attr_bool("keep_dims", False) + ) for axis, size in enumerate(out_2_shape): if size == 1: - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReduceSumD").set_input("x", y_grad).set_attr_vec_int32( - "axes", [axis]).set_attr_bool("keep_dims", True) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "ReduceSumD", + ) + .set_input("x", y_grad) + .set_attr_vec_int32("axes", [axis]) + .set_attr_bool("keep_dims", True) + ) return [x_grad, y_grad], [[0], [1]] class DotDivGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotDivGradParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_div_grad" @@ -1868,41 +2261,68 @@ class DotDivGradParser(AscendParserBase): x = self._get_ge_input(self.op.input_arg_names[2]) y = self._get_ge_input(self.op.input_arg_names[3]) - y_power = core.GEOperatorFactory.create_operator( - "power" + self._accumulated_op_id(), - "Power").set_input("x", y).set_attr_float("power", -1) + y_power = ( + core.GEOperatorFactory.create_operator( + "power" + self._accumulated_op_id(), "Power" + ) + .set_input("x", y) + .set_attr_float("power", -1) + ) tensor_zeros = core.GEOperatorFactory.create_operator( - "zeroslike" + self._accumulated_op_id(), - "ZerosLike").set_input("x", x) - x_zero = core.GEOperatorFactory.create_operator( - "equal" + self._accumulated_op_id(), - "Equal").set_input("x1", x).set_input("x2", tensor_zeros) + "zeroslike" + self._accumulated_op_id(), "ZerosLike" + ).set_input("x", x) + x_zero = ( + core.GEOperatorFactory.create_operator( + "equal" + self._accumulated_op_id(), "Equal" + ) + .set_input("x1", x) + .set_input("x2", tensor_zeros) + ) x_nozero = core.GEOperatorFactory.create_operator( - "logical_not" + self._accumulated_op_id(), - "LogicalNot").set_input("x", x_zero) - x_nozero_f = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x_nozero).set_attr_int32("dst_type", 0) - x_grad_w = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Mul").set_input("x1", x_nozero_f).set_input("x2", y_power) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "Mul").set_input("x1", x_grad_w).set_input("x2", out_grad) - - y_grad_w = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Mul").set_input("x1", out).set_input("x2", y_power) - y_grad = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Mul").set_input("x1", y_grad_w).set_input("x2", out_grad) + "logical_not" + self._accumulated_op_id(), "LogicalNot" + ).set_input("x", x_zero) + x_nozero_f = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x_nozero) + .set_attr_int32("dst_type", 0) + ) + x_grad_w = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x_nozero_f) + .set_input("x2", y_power) + ) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x_grad_w) + .set_input("x2", out_grad) + ) + + y_grad_w = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", out) + .set_input("x2", y_power) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", y_grad_w) + .set_input("x2", out_grad) + ) return [x_grad, y_grad], [[0], [1]] class SoftmaxGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SoftmaxGradParser, self).__init__(graph, var2geop) self.parser_name = "softmax_grad" @@ -1911,15 +2331,17 @@ class SoftmaxGradParser(AscendParserBase): out = self._get_ge_input(self.op.input_arg_names[0]) out_grad = self._get_ge_input(self.op.input_arg_names[1]) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "SoftmaxGrad").set_input("softmax", - out).set_input("grad_softmax", out_grad) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "SoftmaxGrad" + ) + .set_input("softmax", out) + .set_input("grad_softmax", out_grad) + ) return [x_grad], [[0]] class ReshapeGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReshapeGradParser, self).__init__(graph, var2geop) self.parser_name = "reshape2_grad" @@ -1931,20 +2353,24 @@ class ReshapeGradParser(AscendParserBase): if x_shape_list[0] == 0: x_shape_delzero = x_shape_list[1:] - tensor = self._create_ge_tensor([len(x_shape_delzero)], 2, - x_shape_delzero) + tensor = self._create_ge_tensor( + [len(x_shape_delzero)], 2, x_shape_delzero + ) const_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - x_grad = core.GEOperatorFactory.create_operator( - "reshape" + self._accumulated_op_id(), - "Reshape").set_input("x", out_grad).set_input("shape", const_shape) + "shape" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + x_grad = ( + core.GEOperatorFactory.create_operator( + "reshape" + self._accumulated_op_id(), "Reshape" + ) + .set_input("x", out_grad) + .set_input("shape", const_shape) + ) return [x_grad], [[0]] class GatherGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(GatherGradParser, self).__init__(graph, var2geop) self.parser_name = "gather_grad" @@ -1959,24 +2385,30 @@ class GatherGradParser(AscendParserBase): x_shape = self.op.block.var(self.op.input_arg_names[2]).shape if len(index_shape) == 1: - index = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", - index).set_attr_vec_int32("axes", [1]) + index = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", index) + .set_attr_vec_int32("axes", [1]) + ) tensor_zeros = core.GEOperatorFactory.create_operator( - "zeroslike" + self._accumulated_op_id(), - "ZerosLike").set_input("x", x) - x_grad = core.GEOperatorFactory.create_operator( - "scatter" + self._accumulated_op_id(), - "TensorScatterUpdate").set_input("x", tensor_zeros).set_input( - "indices", index).set_input("updates", out_grad) + "zeroslike" + self._accumulated_op_id(), "ZerosLike" + ).set_input("x", x) + x_grad = ( + core.GEOperatorFactory.create_operator( + "scatter" + self._accumulated_op_id(), "TensorScatterUpdate" + ) + .set_input("x", tensor_zeros) + .set_input("indices", index) + .set_input("updates", out_grad) + ) return [tensor_zeros, x_grad], [[-1]] class TransposeGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TransposeGradParser, self).__init__(graph, var2geop) self.parser_name = "transpose2_grad" @@ -1990,16 +2422,18 @@ class TransposeGradParser(AscendParserBase): out_grad_shape = self.op.block.var(self.op.input_arg_names[0]).shape assert list(map(lambda x: out_grad_shape[x], perm)) == list(x_shape) - x_grad = core.GEOperatorFactory.create_operator( - "transpose" + self._accumulated_op_id(), - "TransposeD").set_input("x", - out_grad).set_attr_vec_int32("perm", perm) + x_grad = ( + core.GEOperatorFactory.create_operator( + "transpose" + self._accumulated_op_id(), "TransposeD" + ) + .set_input("x", out_grad) + .set_attr_vec_int32("perm", perm) + ) return [x_grad], [[0]] class LayerNormGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LayerNormGradParser, self).__init__(graph, var2geop) self.parser_name = "layer_norm_grad" @@ -2013,33 +2447,48 @@ class LayerNormGradParser(AscendParserBase): out_grad = self._get_ge_input(self.op.input_arg_names[5]) x_dtype = self.op.block.var(self.op.input_arg_names[4]).dtype - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "LayerNormGrad").set_input("dy", out_grad).set_input( - "x", x).set_input("variance", - variance).set_input("mean", mean).set_input( - "gamma", scale) - - cast_dtype = 0 if self.ascend_helper.dtype2paddle_inv_map[str( - x_dtype)] == 0 else 1 - out_x_grad = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x_grad, - 0).set_attr_int32("dst_type", cast_dtype) - out_scale_grad = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x_grad, - 1).set_attr_int32("dst_type", cast_dtype) - out_bias_grad = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x_grad, - 2).set_attr_int32("dst_type", cast_dtype) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "LayerNormGrad" + ) + .set_input("dy", out_grad) + .set_input("x", x) + .set_input("variance", variance) + .set_input("mean", mean) + .set_input("gamma", scale) + ) + + cast_dtype = ( + 0 + if self.ascend_helper.dtype2paddle_inv_map[str(x_dtype)] == 0 + else 1 + ) + out_x_grad = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x_grad, 0) + .set_attr_int32("dst_type", cast_dtype) + ) + out_scale_grad = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x_grad, 1) + .set_attr_int32("dst_type", cast_dtype) + ) + out_bias_grad = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x_grad, 2) + .set_attr_int32("dst_type", cast_dtype) + ) return [out_x_grad, out_scale_grad, out_bias_grad], [[2], [1], [0]] class TanhGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TanhGradParser, self).__init__(graph, var2geop) self.parser_name = 'tanh_grad' @@ -2047,15 +2496,18 @@ class TanhGradParser(AscendParserBase): def _apply(self): y = self._get_ge_input(self.op.input_arg_names[0]) out_grad = self._get_ge_input(self.op.input_arg_names[1]) - tanh_grad = core.GEOperatorFactory.create_operator( - "tanh_grad" + self._accumulated_op_id(), - "TanhGrad").set_input("y", y).set_input("dy", out_grad) + tanh_grad = ( + core.GEOperatorFactory.create_operator( + "tanh_grad" + self._accumulated_op_id(), "TanhGrad" + ) + .set_input("y", y) + .set_input("dy", out_grad) + ) return [tanh_grad], [[0]] class LogGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LogGradParser, self).__init__(graph, var2geop) self.parser_name = 'log_grad' @@ -2063,14 +2515,17 @@ class LogGradParser(AscendParserBase): def _apply(self): grad = self._get_ge_input(self.op.input_arg_names[0]) input = self._get_ge_input(self.op.input_arg_names[1]) - log_grad = core.GEOperatorFactory.create_operator( - "log_grad" + self._accumulated_op_id(), - "DivNoNan").set_input("x1", grad).set_input("x2", input) + log_grad = ( + core.GEOperatorFactory.create_operator( + "log_grad" + self._accumulated_op_id(), "DivNoNan" + ) + .set_input("x1", grad) + .set_input("x2", input) + ) return [log_grad], [[0]] class SqrtGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SqrtGradParser, self).__init__(graph, var2geop) self.parser_name = "sqrt_grad" @@ -2078,14 +2533,17 @@ class SqrtGradParser(AscendParserBase): def _apply(self): y = self._get_ge_input(self.op.input_arg_names[0]) out_grad = self._get_ge_input(self.op.input_arg_names[1]) - sqrt_grad = core.GEOperatorFactory.create_operator( - "sqrt_grad" + self._accumulated_op_id(), - "SqrtGrad").set_input("y", y).set_input("dy", out_grad) + sqrt_grad = ( + core.GEOperatorFactory.create_operator( + "sqrt_grad" + self._accumulated_op_id(), "SqrtGrad" + ) + .set_input("y", y) + .set_input("dy", out_grad) + ) return [sqrt_grad] class PowGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(PowGradParser, self).__init__(graph, var2geop) self.parser_name = "pow_grad" @@ -2097,31 +2555,46 @@ class PowGradParser(AscendParserBase): shape_tensor = self._create_shape_tensor() shape_tensor = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) factor_scale = self._create_ge_tensor([1], 5, factor) factor_scale = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", factor_scale) - factor_tensor = core.GEOperatorFactory.create_operator( - "broadcast_to_d" + self._accumulated_op_id(), - "BroadcastTo").set_input("x", factor_scale).set_input( - "shape", shape_tensor) - - x_power = core.GEOperatorFactory.create_operator( - "x_power" + self._accumulated_op_id(), - "Power").set_input("x", x).set_attr_float("power", factor - 1) - x_power_mul_factor = core.GEOperatorFactory.create_operator( - "x_power_mul_factor" + self._accumulated_op_id(), - "Mul").set_input("x1", x).set_input("x2", factor_tensor) - x_power_mul_factor_grad = core.GEOperatorFactory.create_operator( - "x_power_mul_factor_grad" + self._accumulated_op_id(), - "Mul").set_input("x1", x_power_mul_factor).set_input("x2", grad) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", factor_scale) + factor_tensor = ( + core.GEOperatorFactory.create_operator( + "broadcast_to_d" + self._accumulated_op_id(), "BroadcastTo" + ) + .set_input("x", factor_scale) + .set_input("shape", shape_tensor) + ) + + x_power = ( + core.GEOperatorFactory.create_operator( + "x_power" + self._accumulated_op_id(), "Power" + ) + .set_input("x", x) + .set_attr_float("power", factor - 1) + ) + x_power_mul_factor = ( + core.GEOperatorFactory.create_operator( + "x_power_mul_factor" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x) + .set_input("x2", factor_tensor) + ) + x_power_mul_factor_grad = ( + core.GEOperatorFactory.create_operator( + "x_power_mul_factor_grad" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x_power_mul_factor) + .set_input("x2", grad) + ) return [x_power_mul_factor_grad], [[0]] class GeluGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(GeluGradParser, self).__init__(graph, var2geop) self.parser_name = "gelu_grad" @@ -2131,17 +2604,21 @@ class GeluGradParser(AscendParserBase): x = self._get_ge_input(self.op.input_arg_names[1]) y = core.GEOperatorFactory.create_operator( - "gelu" + self._accumulated_op_id(), "Gelu").set_input("x", x) - gelu_grad = core.GEOperatorFactory.create_operator( - "gelu_grad" + self._accumulated_op_id(), - "GeluGrad").set_input("x", x).set_input("dy", - grad).set_input("y", y) + "gelu" + self._accumulated_op_id(), "Gelu" + ).set_input("x", x) + gelu_grad = ( + core.GEOperatorFactory.create_operator( + "gelu_grad" + self._accumulated_op_id(), "GeluGrad" + ) + .set_input("x", x) + .set_input("dy", grad) + .set_input("y", y) + ) return [gelu_grad], [[0]] class MeanGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MeanGradParser, self).__init__(graph, var2geop) self.parser_name = "mean_grad" @@ -2151,25 +2628,36 @@ class MeanGradParser(AscendParserBase): x = self._get_ge_input(self.op.input_arg_names[1]) ones_tensor = core.GEOperatorFactory.create_operator( - "one_tensor" + self._accumulated_op_id(), - "OnesLike").set_input("x", x) - sum = core.GEOperatorFactory.create_operator( - "mean" + self._accumulated_op_id(), - "ReduceSumD").set_input("x", ones_tensor).set_attr_bool( - "keep_dims", False).set_attr_vec_int32("axes", []) - mean = core.GEOperatorFactory.create_operator( - "x_power" + self._accumulated_op_id(), - "Power").set_input("x", sum).set_attr_float("power", -1) - - mean_grad = core.GEOperatorFactory.create_operator( - "mean_grad" + self._accumulated_op_id(), - "Mul").set_input("x1", mean).set_input("x2", grad) + "one_tensor" + self._accumulated_op_id(), "OnesLike" + ).set_input("x", x) + sum = ( + core.GEOperatorFactory.create_operator( + "mean" + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", ones_tensor) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) + mean = ( + core.GEOperatorFactory.create_operator( + "x_power" + self._accumulated_op_id(), "Power" + ) + .set_input("x", sum) + .set_attr_float("power", -1) + ) + + mean_grad = ( + core.GEOperatorFactory.create_operator( + "mean_grad" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", mean) + .set_input("x2", grad) + ) return [mean_grad], [[0]] class SliceGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SliceGradParser, self).__init__(graph, var2geop) self.parser_name = "slice_grad" @@ -2200,15 +2688,18 @@ class SliceGradParser(AscendParserBase): starts_cor[0] = 0 ends_cor[0] = 0 paddings = [[s, e] for (s, e) in zip(starts_cor, ends_cor)] - slice_value = core.GEOperatorFactory.create_operator( - "slice_grad" + self._accumulated_op_id(), "PadD").set_input( - "x", grad).set_attr_vec_vec_int64("paddings", paddings) + slice_value = ( + core.GEOperatorFactory.create_operator( + "slice_grad" + self._accumulated_op_id(), "PadD" + ) + .set_input("x", grad) + .set_attr_vec_vec_int64("paddings", paddings) + ) return [slice_value], [[0]] class LookUpTableGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LookUpTableGradParser, self).__init__(graph, var2geop) self.parser_name = "lookup_table_grad" @@ -2222,28 +2713,39 @@ class LookUpTableGradParser(AscendParserBase): shape_grad = self.op.block.var(self.op.input_arg_names[1]).shape shape_embedding = self.op.block.var(self.op.input_arg_names[2]).shape - ids_flatten = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), "FlattenV2").set_input( - "x", ids).set_attr_int32("axis", - 0).set_attr_int32("end_axis", 1) - grad_flatten = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), "FlattenV2").set_input( - "x", grad).set_attr_int32("axis", - 0).set_attr_int32("end_axis", 1) + ids_flatten = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", ids) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) + grad_flatten = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", grad) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) tensor_zeros = core.GEOperatorFactory.create_operator( - "zeroslike" + self._accumulated_op_id(), - "ZerosLike").set_input("x", embedding) - embedding_grad = core.GEOperatorFactory.create_operator( - "scatteradd" + self._accumulated_op_id(), - "TensorScatterAdd").set_input("x", tensor_zeros).set_input( - "indices", ids_flatten).set_input("updates", grad_flatten) + "zeroslike" + self._accumulated_op_id(), "ZerosLike" + ).set_input("x", embedding) + embedding_grad = ( + core.GEOperatorFactory.create_operator( + "scatteradd" + self._accumulated_op_id(), "TensorScatterAdd" + ) + .set_input("x", tensor_zeros) + .set_input("indices", ids_flatten) + .set_input("updates", grad_flatten) + ) return [embedding_grad], [[0]] class SGDParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SGDParser, self).__init__(graph, var2geop) self.parser_name = "sgd" @@ -2252,15 +2754,18 @@ class SGDParser(AscendParserBase): grad = self._get_ge_input(self.op.input_arg_names[0]) lr = self._get_ge_input(self.op.input_arg_names[1]) param = self._get_ge_input(self.op.input_arg_names[2]) - sgd = core.GEOperatorFactory.create_operator( - "momentum" + self._accumulated_op_id(), - "ApplyGradientDescent").set_input("var", param).set_input( - "alpha", lr).set_input("delta", grad) + sgd = ( + core.GEOperatorFactory.create_operator( + "momentum" + self._accumulated_op_id(), "ApplyGradientDescent" + ) + .set_input("var", param) + .set_input("alpha", lr) + .set_input("delta", grad) + ) return [sgd], [[0]] class AdamParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AdamParser, self).__init__(graph, var2geop) self.parser_name = "adam" @@ -2278,26 +2783,29 @@ class AdamParser(AscendParserBase): epsilon = self.op.attr('epsilon') beta1 = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", - self._create_ge_tensor([1], 5, beta1)) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", self._create_ge_tensor([1], 5, beta1)) beta2 = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", - self._create_ge_tensor([1], 5, beta2)) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", self._create_ge_tensor([1], 5, beta2)) epsilon = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", - self._create_ge_tensor([1], 5, epsilon)) - - adam = core.GEOperatorFactory.create_operator( - "adam" + self._accumulated_op_id(), - "ApplyAdam").set_input("var", param).set_input( - "m", moment1).set_input("v", moment2).set_input( - "beta1_power", beta1_power).set_input( - "beta2_power", - beta2_power).set_input("lr", lr).set_input( - "beta1", beta1).set_input("beta2", beta2).set_input( - "epsilon", epsilon).set_input("grad", grad) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", self._create_ge_tensor([1], 5, epsilon)) + + adam = ( + core.GEOperatorFactory.create_operator( + "adam" + self._accumulated_op_id(), "ApplyAdam" + ) + .set_input("var", param) + .set_input("m", moment1) + .set_input("v", moment2) + .set_input("beta1_power", beta1_power) + .set_input("beta2_power", beta2_power) + .set_input("lr", lr) + .set_input("beta1", beta1) + .set_input("beta2", beta2) + .set_input("epsilon", epsilon) + .set_input("grad", grad) + ) return [adam], [[0]] diff --git a/python/paddle/distributed/fleet/meta_optimizers/asp_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/asp_optimizer.py index 2047c3172c2..59a75d1b054 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/asp_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/asp_optimizer.py @@ -19,23 +19,26 @@ __all__ = [] class ASPOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(ASPOptimizer, self).__init__(optimizer) self.inner_opt = optimizer # we do not allow meta optimizer to be inner optimizer currently self.meta_optimizers_white_list = [ - "AMPOptimizer", "LarsOptimizer", "LambOptimizer", - "GraphExecutionOptimizer", "RecomputeOptimizer", - "GradientMergeOptimizer" + "AMPOptimizer", + "LarsOptimizer", + "LambOptimizer", + "GraphExecutionOptimizer", + "RecomputeOptimizer", + "GradientMergeOptimizer", ] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(ASPOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(ASPOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -52,17 +55,16 @@ class ASPOptimizer(MetaOptimizerBase): def _enable_strategy(self, dist_strategy, context): dist_strategy.asp = True - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): optimize_ops, params_grads = ASPHelper._minimize( self.inner_opt, loss, startup_program=startup_program, parameter_list=parameter_list, - no_grad_set=no_grad_set) + no_grad_set=no_grad_set, + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/common.py b/python/paddle/distributed/fleet/meta_optimizers/common.py index ed04188de20..7a3c89f1e9d 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/common.py +++ b/python/paddle/distributed/fleet/meta_optimizers/common.py @@ -27,8 +27,11 @@ OP_ROLE_VAR_KEY = core.op_proto_and_checker_maker.kOpRoleVarAttrName() def is_update_op(op): - return 'Param' in op.input_names and 'Grad' in op.input_names and \ - "LearningRate" in op.input_names + return ( + 'Param' in op.input_names + and 'Grad' in op.input_names + and "LearningRate" in op.input_names + ) def is_loss_grad_op(op): @@ -39,17 +42,18 @@ def is_loss_grad_op(op): def is_backward_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Backward) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Backward) def is_optimizer_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Optimize) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Optimize) class CollectiveHelper(object): - def __init__(self, role_maker, nrings=1, wait_port=True): self.nrings = nrings self.wait_port = wait_port @@ -63,21 +67,27 @@ class CollectiveHelper(object): endpoints = self.role_maker._get_trainer_endpoints() current_endpoint = endpoints[self.role_maker._worker_index()] for ring_id in range(self.nrings): - self._init_communicator(self.startup_program, - current_endpoint, endpoints, - self.role_maker._worker_index(), ring_id, - self.wait_port) + self._init_communicator( + self.startup_program, + current_endpoint, + endpoints, + self.role_maker._worker_index(), + ring_id, + self.wait_port, + ) self._broadcast_params() - def _init_communicator(self, - program, - current_endpoint, - endpoints, - rank, - ring_id, - wait_port, - global_ring_id=None, - sync=True): + def _init_communicator( + self, + program, + current_endpoint, + endpoints, + rank, + ring_id, + wait_port, + global_ring_id=None, + sync=True, + ): # if current_endpoint is None, it means just for sync, # no group is created. if current_endpoint: @@ -89,32 +99,40 @@ class CollectiveHelper(object): wait_server_ready(other_endpoints) def _add_sync_by_allreduce(block): - sync_var = block.create_var(name=unique_name.generate('sync_var'), - dtype=core.VarDesc.VarType.INT32, - persistable=False, - stop_gradient=True) - block.append_op(type='fill_constant', - inputs={}, - outputs={'Out': [sync_var]}, - attrs={ - 'shape': [1], - 'dtype': sync_var.dtype, - 'value': 1, - 'force_cpu': False, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_allreduce_sum', - inputs={'X': [sync_var]}, - outputs={'Out': [sync_var]}, - attrs={ - 'ring_id': global_ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_sync_calc_stream', - inputs={'X': sync_var}, - outputs={'Out': sync_var}, - attrs={OP_ROLE_KEY: OpRole.Forward}) + sync_var = block.create_var( + name=unique_name.generate('sync_var'), + dtype=core.VarDesc.VarType.INT32, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [sync_var]}, + attrs={ + 'shape': [1], + 'dtype': sync_var.dtype, + 'value': 1, + 'force_cpu': False, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_allreduce_sum', + inputs={'X': [sync_var]}, + outputs={'Out': [sync_var]}, + attrs={ + 'ring_id': global_ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_sync_calc_stream', + inputs={'X': sync_var}, + outputs={'Out': sync_var}, + attrs={OP_ROLE_KEY: OpRole.Forward}, + ) block = program.global_block() if current_endpoint is None: @@ -123,79 +141,93 @@ class CollectiveHelper(object): _add_sync_by_allreduce(block) return - comm_id_var = block.create_var(name=unique_name.generate('comm_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) + comm_id_var = block.create_var( + name=unique_name.generate('comm_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) if core.is_compiled_with_cuda(): - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': comm_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_comm_init', - inputs={'X': comm_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': comm_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) elif core.is_compiled_with_xpu(): - block.append_op(type='c_gen_bkcl_id', - inputs={}, - outputs={'Out': comm_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_comm_init', - inputs={'X': comm_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_gen_bkcl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': comm_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) elif core.is_compiled_with_npu(): - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': comm_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': comm_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': ring_id, - 'device_id': - int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': comm_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': ring_id, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + OP_ROLE_KEY: OpRole.Forward, + }, + ) else: raise ValueError( "comm_id must be generated in paddlepaddle-xpu or paddlepaddle-xpu." ) - if sync: _add_sync_by_allreduce(block) + if sync: + _add_sync_by_allreduce(block) def _wait(self, current_endpoint, endpoints): - assert (self.wait_port) + assert self.wait_port other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) wait_server_ready(other_endpoints) @@ -208,20 +240,21 @@ class CollectiveHelper(object): continue ring_id = (ring_id + 1) % self.nrings - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward, + }, + ) for ring_id in range(self.nrings): - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Forward}, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py index d25cf968023..15090f78265 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py @@ -19,7 +19,6 @@ __all__ = [] class DGCOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(DGCOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -28,11 +27,12 @@ class DGCOptimizer(MetaOptimizerBase): self.meta_optimizers_white_list = [] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(DGCOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(DGCOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _init_dgc_opt(self): if self.dgc_opt is not None: @@ -62,7 +62,8 @@ class DGCOptimizer(MetaOptimizerBase): num_trainers=self.role_maker._worker_num(), regularization=opt.regularization, grad_clip=opt._grad_clip, - name=opt._name) + name=opt._name, + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -88,15 +89,18 @@ class DGCOptimizer(MetaOptimizerBase): dist_strategy.dgc = True dist_strategy.dgc_configs = {"rampup_begin_step": 0, "rampup_step": 1} - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): self._init_dgc_opt() - return self.dgc_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + return self.dgc_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): self._init_dgc_opt() @@ -104,17 +108,15 @@ class DGCOptimizer(MetaOptimizerBase): def apply_optimize(self, loss, startup_program, params_grads): self._init_dgc_opt() - return self.dgc_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + return self.dgc_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self._init_dgc_opt() - optimize_ops, params_grads = \ - self.dgc_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + optimize_ops, params_grads = self.dgc_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py index 958ebeb517d..7e5f6983867 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py @@ -42,17 +42,26 @@ class DygraphShardingOptimizer(object): # 3. dynamic trainable params, which is the case bewteen pretraining and finetuning # 4. option to choose fuse comm (more GPU MEM need) or un-fuse comm - def __init__(self, hcg, user_defined_strategy, params, - inner_optimizer_class, **inner_optimizer_kargs): + def __init__( + self, + hcg, + user_defined_strategy, + params, + inner_optimizer_class, + **inner_optimizer_kargs + ): if not isinstance(params, list): raise TypeError( "`parameters` argument given to the DygraphShardingOptimizer should be " - "an iterable of paddle Tensors, but got argument type is `{}`.". - format(type(params))) + "an iterable of paddle Tensors, but got argument type is `{}`.".format( + type(params) + ) + ) self._parameter_list = params self._reference_is_trainable_params = list( - map(_is_trainable, self._parameter_list)) + map(_is_trainable, self._parameter_list) + ) self._inner_optimizer_class = inner_optimizer_class self._inner_optimizer_kargs = inner_optimizer_kargs @@ -102,8 +111,11 @@ class DygraphShardingOptimizer(object): rank = sizes.index(min(sizes)) mapping[rank].append(param) numel = reduce(lambda x, y: x * y, param.shape) - assert numel > 0, "param [{}] should larger than 0, but it is [{}]".format( - param.name, numel) + assert ( + numel > 0 + ), "param [{}] should larger than 0, but it is [{}]".format( + param.name, numel + ) sizes[rank] += numel return mapping @@ -127,7 +139,8 @@ class DygraphShardingOptimizer(object): # update related ops: clip, regular, opt self._inner_optimizer = self._inner_optimizer_class( parameters=self._rank2params[self._sharding_rank], - **self._inner_optimizer_kargs) + **self._inner_optimizer_kargs + ) def _sharding_sync_parameters(self): """ @@ -146,7 +159,8 @@ class DygraphShardingOptimizer(object): # instead of the relative logic rank id within group src=self._hcg.get_sharding_parallel_group().ranks[rank], group=self._hcg.get_sharding_parallel_group(), - sync_op=True) + sync_op=True, + ) def _update_trainable(self): """ @@ -154,21 +168,23 @@ class DygraphShardingOptimizer(object): """ raise NotImplementedError - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): # NOTE in dygraph mode, the only different between step and minimize is that minimize # allow user to customize the parameters for updating on each step input_param_names = set([param.name for param in parameters]) parameters = list( - filter(lambda x: x.name in input_param_names, - self._rank2params[self._sharding_rank])) - result = self._inner_optimizer.minimize(loss, startup_program, - parameters, no_grad_set) + filter( + lambda x: x.name in input_param_names, + self._rank2params[self._sharding_rank], + ) + ) + result = self._inner_optimizer.minimize( + loss, startup_program, parameters, no_grad_set + ) # sync parameters across sharding ranks self._sharding_sync_parameters() @@ -187,7 +203,9 @@ class DygraphShardingOptimizer(object): # TODO is it a good way to make _grad_clip a property @property def _grad_clip(self): - assert self._inner_optimizer is not None, "inner opt of sharding is not initiliazed." + assert ( + self._inner_optimizer is not None + ), "inner opt of sharding is not initiliazed." return self._inner_optimizer._grad_clip def __getattr__(self, item): diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/heter_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/heter_parallel_optimizer.py index cda8c9e30ce..a2a65d995ad 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/heter_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/heter_parallel_optimizer.py @@ -20,7 +20,8 @@ __all__ = [] def _obtain_optimizer_parameters_list(optimizer): if getattr(optimizer, '_param_groups', None) and isinstance( - optimizer._param_groups[0], dict): + optimizer._param_groups[0], dict + ): parameters_list = [] for group in optimizer._param_groups: for param in group['params']: @@ -47,19 +48,19 @@ class HeterParallelOptimizer: self._inner_opt.step() @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): # minimize does not support parameters in the form of param_group, # so no need use _obtain_optimizer_parameters_list - parameter_list = parameters if parameters \ - else self._inner_opt._parameter_list + parameter_list = ( + parameters if parameters else self._inner_opt._parameter_list + ) - return self._inner_opt.minimize(loss, startup_program, parameter_list, - no_grad_set) + return self._inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) def __getattr__(self, item): return getattr(self._inner_opt, item) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py index bd762a202f3..d768411dea5 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py @@ -21,12 +21,12 @@ __all__ = [] class HybridParallelGradScaler: - def __init__(self, scaler, hcg): self._scaler = scaler self._hcg = hcg self._use_dp_mode = ( - self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL) + self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL + ) def scale(self, var): return self._scaler.scale(var) @@ -56,20 +56,22 @@ class HybridParallelGradScaler: if not self._enable: return param_grads = [ - param._grad_ivar() for param in optimizer._parameter_list + param._grad_ivar() + for param in optimizer._parameter_list if param._grad_ivar() is not None ] - _legacy_C_ops.check_finite_and_unscale(param_grads, self._scale, - param_grads, self._found_inf) + _legacy_C_ops.check_finite_and_unscale( + param_grads, self._scale, param_grads, self._found_inf + ) # allreduce_max found_inf in check_group if not self._use_dp_mode: self._found_inf = paddle.cast(self._found_inf, dtype="int32") # TODO(shenliang03) Since the minimize call in the optimizer is # after the gradscaler, check_finite needs to synchronize global # information. In the future, we should use check_group - paddle.distributed.all_reduce(self._found_inf, - op=paddle.distributed.ReduceOp.MAX, - group=None) + paddle.distributed.all_reduce( + self._found_inf, op=paddle.distributed.ReduceOp.MAX, group=None + ) self._found_inf = paddle.cast(self._found_inf, dtype="bool") def __getattr__(self, item): diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py index 2cc43abee92..33922b7f35d 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py @@ -14,7 +14,10 @@ import paddle from paddle.fluid.clip import ClipGradByGlobalNorm -from ...utils.hybrid_parallel_util import fused_allreduce_gradients, sharding_reduce_gradients +from ...utils.hybrid_parallel_util import ( + fused_allreduce_gradients, + sharding_reduce_gradients, +) from ...base.topology import ParallelMode from paddle.fluid.dygraph import base as imperative_base from paddle.fluid import framework @@ -27,7 +30,8 @@ __all__ = [] def _obtain_optimizer_parameters_list(optimizer): if getattr(optimizer, '_param_groups', None) and isinstance( - optimizer._param_groups[0], dict): + optimizer._param_groups[0], dict + ): parameters_list = [] for group in optimizer._param_groups: for param in group['params']: @@ -39,7 +43,6 @@ def _obtain_optimizer_parameters_list(optimizer): class HybridParallelClipGrad: - def __init__(self, clip, hcg): self._clip = clip self._hcg = hcg @@ -65,7 +68,8 @@ class HybridParallelClipGrad: not_shared_enable = (not hasattr(p, 'is_firstly_shared')) or ( hasattr(p, 'is_firstly_shared') - and getattr(p, 'is_firstly_shared', True)) + and getattr(p, 'is_firstly_shared', True) + ) if not_shared_enable: if p.is_distributed: @@ -81,69 +85,83 @@ class HybridParallelClipGrad: # global norm of distributed FP16 params_and_grads if len(sum_square_dist_fp16) == 0: - global_norm_dist_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_norm_dist_fp16 = paddle.to_tensor( + [0.0], dtype=paddle.float32 + ) else: global_norm_dist_fp16 = layers.concat(sum_square_dist_fp16) global_norm_dist_fp16 = layers.reduce_sum(global_norm_dist_fp16) - global_norm_dist_fp16 = paddle.cast(global_norm_dist_fp16, - dtype=paddle.float32) + global_norm_dist_fp16 = paddle.cast( + global_norm_dist_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP16 params_and_grads if len(sum_square_not_dist_fp16) == 0: - global_norm_not_dist_fp16 = paddle.to_tensor([0.], - dtype=paddle.float32) + global_norm_not_dist_fp16 = paddle.to_tensor( + [0.0], dtype=paddle.float32 + ) else: global_norm_not_dist_fp16 = layers.concat(sum_square_not_dist_fp16) global_norm_not_dist_fp16 = layers.reduce_sum( - global_norm_not_dist_fp16) - global_norm_not_dist_fp16 = paddle.cast(global_norm_not_dist_fp16, - dtype=paddle.float32) + global_norm_not_dist_fp16 + ) + global_norm_not_dist_fp16 = paddle.cast( + global_norm_not_dist_fp16, dtype=paddle.float32 + ) # global norm of distributed FP32 params_and_grads - global_norm_dist_fp32 = layers.concat(sum_square_dist_fp32) if len( - sum_square_dist_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_norm_dist_fp32 = ( + layers.concat(sum_square_dist_fp32) + if len(sum_square_dist_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_norm_dist_fp32 = layers.reduce_sum(global_norm_dist_fp32) # global norm of non-distributed FP32 params_and_grads - global_norm_not_dist_fp32 = layers.concat( - sum_square_not_dist_fp32 - ) if len(sum_square_not_dist_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_norm_not_dist_fp32 = ( + layers.concat(sum_square_not_dist_fp32) + if len(sum_square_not_dist_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_norm_not_dist_fp32 = layers.reduce_sum(global_norm_not_dist_fp32) global_norm_var_dist = global_norm_dist_fp16 + global_norm_dist_fp32 - global_norm_var_not_dist = global_norm_not_dist_fp16 + global_norm_not_dist_fp32 + global_norm_var_not_dist = ( + global_norm_not_dist_fp16 + global_norm_not_dist_fp32 + ) # add all reduce to get global norm of distributed params_and_grads if self._hcg.get_model_parallel_world_size() > 1: paddle.distributed.all_reduce( - global_norm_var_dist, - group=self._hcg.get_check_parallel_group()) + global_norm_var_dist, group=self._hcg.get_check_parallel_group() + ) # add all reduce to get global norm of non-distributed params_and_grads in groups of pp if self._hcg.get_pipe_parallel_world_size() > 1: paddle.distributed.all_reduce( global_norm_var_not_dist, - group=self._hcg.get_pipe_parallel_group()) + group=self._hcg.get_pipe_parallel_group(), + ) # In Sharding mode, param and grad is mapping different rank in optimizer. # ClipGradByGlobalNorm need allreduce to get globol norm if self._hcg.get_sharding_parallel_world_size() > 1: paddle.distributed.all_reduce( global_norm_var_not_dist, - group=self._hcg.get_sharding_parallel_group()) - - global_norm_var_fp32 = layers.sqrt(global_norm_var_dist + - global_norm_var_not_dist) - - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var_fp32.dtype, - value=self.clip_norm) - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var_fp32, - y=max_global_norm)) + group=self._hcg.get_sharding_parallel_group(), + ) + + global_norm_var_fp32 = layers.sqrt( + global_norm_var_dist + global_norm_var_not_dist + ) + + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var_fp32.dtype, value=self.clip_norm + ) + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var_fp32, y=max_global_norm), + ) clip_var_fp16 = paddle.cast(clip_var, paddle.float16) for p, g in params_grads: if g is None: @@ -173,35 +191,43 @@ class HybridParallelOptimizer: self._hcg = hcg self._use_dp_mode = ( - self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL) + self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL + ) - self._need_dp = (self._hcg.get_data_parallel_world_size() > 1) + self._need_dp = self._hcg.get_data_parallel_world_size() > 1 # NOTE(shenliang03): Because of the pure DataParallel mode, the gradient synchronization # is achieved through reducer, so there is no need to call fuse_allreduce in optimizer. self._dp_enable = not self._use_dp_mode and self._need_dp - self._sharding_enable = (self._hcg.get_sharding_parallel_world_size() > - 1) + self._sharding_enable = self._hcg.get_sharding_parallel_world_size() > 1 - if isinstance(self._inner_opt._grad_clip, - ClipGradByGlobalNorm) and not self._use_dp_mode: - logger.warning("While using ClipGradByGlobalNorm in TensorParallel, PipelineParallel " \ - "or Sharding, the grad clip of original optimizer will be changed.") + if ( + isinstance(self._inner_opt._grad_clip, ClipGradByGlobalNorm) + and not self._use_dp_mode + ): + logger.warning( + "While using ClipGradByGlobalNorm in TensorParallel, PipelineParallel " + "or Sharding, the grad clip of original optimizer will be changed." + ) if self._sharding_enable: # change sharding inner_optimizer's _grad_clip - self._inner_opt._inner_optimizer._grad_clip = HybridParallelClipGrad( - self._inner_opt._grad_clip, hcg) + self._inner_opt._inner_optimizer._grad_clip = ( + HybridParallelClipGrad(self._inner_opt._grad_clip, hcg) + ) else: self._inner_opt._grad_clip = HybridParallelClipGrad( - self._inner_opt._grad_clip, hcg) + self._inner_opt._grad_clip, hcg + ) if self._inner_opt._parameter_list and isinstance( - self._inner_opt._parameter_list[0], dict): + self._inner_opt._parameter_list[0], dict + ): for item in self._inner_opt._param_groups: if "grad_clip" in item.keys(): item["grad_clip"] = HybridParallelClipGrad( - self._inner_opt._grad_clip, hcg) + self._inner_opt._grad_clip, hcg + ) @imperative_base.no_grad @framework.dygraph_only @@ -216,16 +242,15 @@ class HybridParallelOptimizer: self._inner_opt.step() @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): # minimize does not support parameters in the form of param_group, # so no need use _obtain_optimizer_parameters_list - parameter_list = parameters if parameters \ - else self._inner_opt._parameter_list + parameter_list = ( + parameters if parameters else self._inner_opt._parameter_list + ) # Here sharding should use global parameter list if self._sharding_enable: @@ -234,8 +259,9 @@ class HybridParallelOptimizer: if self._dp_enable: fused_allreduce_gradients(list(parameter_list), self._hcg) - return self._inner_opt.minimize(loss, startup_program, parameter_list, - no_grad_set) + return self._inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) def __getattr__(self, item): return getattr(self._inner_opt, item) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/sharding_optimizer_stage2.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/sharding_optimizer_stage2.py index 3468ec7a3a7..ed143520954 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/sharding_optimizer_stage2.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/sharding_optimizer_stage2.py @@ -30,10 +30,19 @@ import paddle from paddle.fluid import core from paddle.optimizer import Optimizer from paddle.fluid.clip import ClipGradByGlobalNorm -from paddle.distributed.collective import _get_global_group, new_group, broadcast, wait +from paddle.distributed.collective import ( + _get_global_group, + new_group, + broadcast, + wait, +) from ...utils.internal_storage import ParamStorage, GradStorage -from ...meta_parallel.sharding.sharding_utils import Type, device_guard, ShardingClipGrad +from ...meta_parallel.sharding.sharding_utils import ( + Type, + device_guard, + ShardingClipGrad, +) # CUDA alignment 256 bytes, cpu alignment 4096 bytes alignment = {"gpu": 256, "cpu": 4096} @@ -61,19 +70,22 @@ class ShardingOptimizerStage2(Optimizer): # 4. Support offload function. # 5. Support the establishment of independent communication groups. # 6. Broadcast_fp16 is not supported now. - def __init__(self, - params, - optim, - group=None, - offload=False, - device="gpu", - pertrain_sync_models=True, - **kw): + def __init__( + self, + params, + optim, + group=None, + offload=False, + device="gpu", + pertrain_sync_models=True, + **kw + ): super().__init__(optim._learning_rate, params, kw) # Segmentation information - self._dtype_rank_params = OrderedDict( + self._dtype_rank_params = ( + OrderedDict() ) # {dtype:[param1,param2]} device, rank, params self._param2rank = {} self.__segment_params = [] @@ -84,17 +96,26 @@ class ShardingOptimizerStage2(Optimizer): self._optim_defaults = kw self._optim = optim - assert hasattr(self._optim, "_master_weights" - ), "Must use optimizer with _master_weights attribute" + assert hasattr( + self._optim, "_master_weights" + ), "Must use optimizer with _master_weights attribute" self._local_params = params self._default_device = device - self._pfp16 = len( - list( - filter(lambda x: x.trainable and x.dtype == Type.fp16.value, - self._local_params))) > 0 + self._pfp16 = ( + len( + list( + filter( + lambda x: x.trainable and x.dtype == Type.fp16.value, + self._local_params, + ) + ) + ) + > 0 + ) - self.group = new_group( - _get_global_group().ranks) if group is None else group + self.group = ( + new_group(_get_global_group().ranks) if group is None else group + ) self.world_size = self.group.nranks self.rank = self.group.rank @@ -110,19 +131,24 @@ class ShardingOptimizerStage2(Optimizer): logging.warning( "While using ClipGradByGlobalNorm in ShardingOptimizer, the grad clip of original optimizer will be changed." ) - self._optim._grad_clip = ShardingClipGrad(self._optim._grad_clip, - paddle.get_device(), - self.group) + self._optim._grad_clip = ShardingClipGrad( + self._optim._grad_clip, paddle.get_device(), self.group + ) if self._optim._parameter_list and isinstance( - self._optim._parameter_list[0], dict): + self._optim._parameter_list[0], dict + ): for item in self._optim._param_groups: if "grad_clip" in item.keys(): item["grad_clip"] = ShardingClipGrad( - self._optim._grad_clip, paddle.get_device(), - self.group) + self._optim._grad_clip, + paddle.get_device(), + self.group, + ) if offload: - assert self._pfp16, "Only support offload strategy while using \'Adam\', \'AdamW\' and \'Momentum\' optimizer with AMP/Pure FP16" + assert ( + self._pfp16 + ), "Only support offload strategy while using \'Adam\', \'AdamW\' and \'Momentum\' optimizer with AMP/Pure FP16" self.offload = offload # Using for offload self.offload_device = "cpu" @@ -143,10 +169,9 @@ class ShardingOptimizerStage2(Optimizer): """ for p in self._local_params: - broadcast(p, - src=self._global_root_rank, - group=self.group, - sync_op=True) + broadcast( + p, src=self._global_root_rank, group=self.group, sync_op=True + ) # Multi stream operation will be supported later wait(tensor=p, group=self.group, use_calc_stream=True) @@ -159,16 +184,17 @@ class ShardingOptimizerStage2(Optimizer): name=param.name, value=param.cast(dtype=Type.fp32.value).numpy(), place=core.CPUPlace(), - stop_gradient=param.stop_gradient) + stop_gradient=param.stop_gradient, + ) else: for param in trainable_params: if param.dtype == Type.fp16.value: self._optim._master_weights[param.name] = paddle.cast( - param, Type.fp32.value) + param, Type.fp32.value + ) def _update_opt_status(self): - """Update optimizer status and parameter storage information, and special functions to be developed. - """ + """Update optimizer status and parameter storage information, and special functions to be developed.""" # func 1 self._integration_params() @@ -222,8 +248,9 @@ class ShardingOptimizerStage2(Optimizer): self._dtype_rank_params[param.dtype] = [ [] for _ in range(self.world_size) ] - self._dtype_rank_params[param.dtype][self.param2rank[ - param.name]].append(param) + self._dtype_rank_params[param.dtype][ + self.param2rank[param.name] + ].append(param) # Sort per rank params by size for dtype in self._dtype_rank_params.keys(): @@ -243,7 +270,8 @@ class ShardingOptimizerStage2(Optimizer): if dtype not in self._rank_buffer_size.keys(): self._rank_buffer_size[dtype] = {} for dst_rank, per_rank_params in enumerate( - self.dtype_rank_params[dtype]): + self.dtype_rank_params[dtype] + ): if dst_rank not in self._rank_buffer_size[dtype].keys(): self._rank_buffer_size[dtype][dst_rank] = 0 for param in per_rank_params: @@ -251,11 +279,15 @@ class ShardingOptimizerStage2(Optimizer): continue size = np.prod(param.shape) * align[dtype] remaining = size % alignment[self._default_device] - ali = 0 if remaining == 0 else alignment[ - self._default_device] - remaining + ali = ( + 0 + if remaining == 0 + else alignment[self._default_device] - remaining + ) align_ = ali // align[dtype] - self._rank_buffer_size[dtype][dst_rank] += np.prod( - param.shape) + align_ + self._rank_buffer_size[dtype][dst_rank] += ( + np.prod(param.shape) + align_ + ) self._param2align[param.name] = align_ return self._rank_buffer_size @@ -274,23 +306,27 @@ class ShardingOptimizerStage2(Optimizer): # Merge all the trainable params in a single InternalStorage trainable_params = list( - filter(lambda x: x.trainable, params)) + filter(lambda x: x.trainable, params) + ) if self._pfp16 and dst_rank == self.rank: self._generate_master_params(trainable_params) if trainable_params: param_storage = ParamStorage( size=self.rank_buffer_size[dtype][dst_rank], dtype=dtype, - device=self._default_device) + device=self._default_device, + ) - param_storage.add_rank_params(trainable_params, - self._param2align) + param_storage.add_rank_params( + trainable_params, self._param2align + ) self.param_storages[dtype][dst_rank] = param_storage # Clear the InternalStorage keys which are not in use anymore dtype_in_use = list(self.dtype_rank_params.keys()) dtype_to_pop = list( - filter(lambda x: x not in dtype_in_use, self.param_storages.keys())) + filter(lambda x: x not in dtype_in_use, self.param_storages.keys()) + ) for d in dtype_to_pop: self.param_storages.pop(d) @@ -300,8 +336,11 @@ class ShardingOptimizerStage2(Optimizer): for param in cpu_master_params: size = np.prod(param.shape) * align[Type.fp32.value] remaining = size % alignment[self.offload_device] - ali = 0 if remaining == 0 else alignment[ - self.offload_device] - remaining + ali = ( + 0 + if remaining == 0 + else alignment[self.offload_device] - remaining + ) align_ = ali // align[Type.fp32.value] self.offload_buffer_size += np.prod(param.shape) + align_ self.offload_param2align[param.name] = align_ @@ -311,9 +350,11 @@ class ShardingOptimizerStage2(Optimizer): self.offload_params = ParamStorage( size=self.offload_buffer_size, dtype=Type.fp32.value, - device=self.offload_device) + device=self.offload_device, + ) self.offload_params.add_rank_params( - cpu_master_params, self.offload_param2align, False) + cpu_master_params, self.offload_param2align, False + ) self.offload_params.buffer.stop_gradient = False self.offload_grads = GradStorage( @@ -322,14 +363,16 @@ class ShardingOptimizerStage2(Optimizer): device=self.offload_device, destination=self.rank, parm2align=self.offload_param2align, - convert_cpu=True) + convert_cpu=True, + ) for p in cpu_master_params: self.offload_grads.add_grad( - p, self.offload_param2align[p.name]) + p, self.offload_param2align[p.name] + ) self._optim._master_weights[ - self.offload_params.buffer. - name] = self.offload_params.buffer + self.offload_params.buffer.name + ] = self.offload_params.buffer def _offload_acc_grad(self, param_name, grad_fp32_cpu): """accumulate grads with offload strategy""" @@ -337,12 +380,14 @@ class ShardingOptimizerStage2(Optimizer): if param_name in self._master_params.keys(): if self._master_params[param_name].grad is None: self._master_params[param_name]._copy_gradient_from( - grad_fp32_cpu) + grad_fp32_cpu + ) else: self._master_params[param_name].grad.add_(grad_fp32_cpu) self.offload_params.buffer._copy_gradient_from( - self.offload_grads.buffer) + self.offload_grads.buffer + ) def _offload_scale_grad(self, scale_size): """scale grads with offload strategy""" @@ -362,7 +407,7 @@ class ShardingOptimizerStage2(Optimizer): if self.offload: params_list = [self.offload_params.buffer] - #TODO(Baibaifan): Offload will support param_groups later + # TODO(Baibaifan): Offload will support param_groups later if not isinstance(self._optim._param_groups[0], dict): self._optim._parameter_list = params_list self._optim._param_groups = params_list @@ -376,8 +421,10 @@ class ShardingOptimizerStage2(Optimizer): for param in self._local_params: if param.name in self._master_params.keys(): param.set_value( - self._master_params[param.name].cuda(dev_id).cast( - dtype=param.dtype)) + self._master_params[param.name] + .cuda(dev_id) + .cast(dtype=param.dtype) + ) else: self._optim.step() @@ -386,7 +433,8 @@ class ShardingOptimizerStage2(Optimizer): def minimize(self): raise RuntimeError( - "optimizer.minimize() not support now, please use optimizer.step()") + "optimizer.minimize() not support now, please use optimizer.step()" + ) def set_state_dict(self, state_dict): self._optim.set_state_dict(state_dict) @@ -408,12 +456,16 @@ class ShardingOptimizerStage2(Optimizer): # Exchange all the shards with the other ranks for dtype_per_rank in self.param_storages.values(): for dst_rank, internal_storage in dtype_per_rank.items(): - broadcast(tensor=internal_storage.buffer, - src=self.group.ranks[dst_rank], - group=self.group, - sync_op=True) + broadcast( + tensor=internal_storage.buffer, + src=self.group.ranks[dst_rank], + group=self.group, + sync_op=True, + ) # Multi stream operation will be supported later - wait(tensor=internal_storage.buffer, - group=self.group, - use_calc_stream=True) + wait( + tensor=internal_storage.buffer, + group=self.group, + use_calc_stream=True, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py index 93857461b26..3834f85bffa 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py @@ -18,7 +18,6 @@ __all__ = [] class FP16AllReduceOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(FP16AllReduceOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -34,11 +33,12 @@ class FP16AllReduceOptimizer(MetaOptimizerBase): ] self.meta_optimizers_black_list = ["DGCOptimizer"] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(FP16AllReduceOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(FP16AllReduceOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -84,28 +84,30 @@ class FP16AllReduceOptimizer(MetaOptimizerBase): else: op._remove_attr(op_maker.kOpRoleVarAttrName()) - new_grad = block.create_var(name=unique_name.generate(grad.name + - ".cast_fp16"), - dtype=core.VarDesc.VarType.FP16, - persistable=False, - stop_gradient=True) + new_grad = block.create_var( + name=unique_name.generate(grad.name + ".cast_fp16"), + dtype=core.VarDesc.VarType.FP16, + persistable=False, + stop_gradient=True, + ) with block.program._backward_role_guard(): - cast_op = block.append_op(type="cast", - inputs={"X": grad}, - outputs={"Out": new_grad}, - attrs={ - "in_dtype": - core.VarDesc.VarType.FP32, - "out_dtype": - core.VarDesc.VarType.FP16 - }, - stop_gradient=True) + cast_op = block.append_op( + type="cast", + inputs={"X": grad}, + outputs={"Out": new_grad}, + attrs={ + "in_dtype": core.VarDesc.VarType.FP32, + "out_dtype": core.VarDesc.VarType.FP16, + }, + stop_gradient=True, + ) backward = op_maker.OpRole.Backward cast_op._set_attr(op_maker.kOpRoleAttrName(), backward) - cast_op._set_attr(op_maker.kOpRoleVarAttrName(), - [param.name, new_grad.name]) + cast_op._set_attr( + op_maker.kOpRoleVarAttrName(), [param.name, new_grad.name] + ) new_grad.op = cast_op new_param_and_grads.append((param, new_grad, True)) @@ -122,30 +124,32 @@ class FP16AllReduceOptimizer(MetaOptimizerBase): continue block = grad.block - new_grad = block.create_var(name=unique_name.generate(grad.name + - ".cast_fp32"), - dtype=core.VarDesc.VarType.FP32, - persistable=False, - stop_gradient=True) + new_grad = block.create_var( + name=unique_name.generate(grad.name + ".cast_fp32"), + dtype=core.VarDesc.VarType.FP32, + persistable=False, + stop_gradient=True, + ) with block.program._optimized_guard( - [param, grad]), framework.name_scope('fp16_allreduce'): - cast_op = block.append_op(type="cast", - inputs={"X": grad}, - outputs={"Out": new_grad}, - attrs={ - "in_dtype": - core.VarDesc.VarType.FP16, - "out_dtype": - core.VarDesc.VarType.FP32 - }, - stop_gradient=True) + [param, grad] + ), framework.name_scope('fp16_allreduce'): + cast_op = block.append_op( + type="cast", + inputs={"X": grad}, + outputs={"Out": new_grad}, + attrs={ + "in_dtype": core.VarDesc.VarType.FP16, + "out_dtype": core.VarDesc.VarType.FP32, + }, + stop_gradient=True, + ) ret_param_and_grads.append((param, new_grad)) return ret_param_and_grads def apply_optimize(self, loss, startup_program, params_grads): new_params_grads = self.fp16_compression(params_grads) - return self.inner_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=new_params_grads) + return self.inner_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=new_params_grads + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py index 10175f8936a..eb97122587f 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py @@ -18,7 +18,6 @@ __all__ = [] class GradientMergeOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(GradientMergeOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -32,26 +31,30 @@ class GradientMergeOptimizer(MetaOptimizerBase): ] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(GradientMergeOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(GradientMergeOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _init_wrapped_opt(self): config = self.user_defined_strategy.gradient_merge_configs self.wrapped_opt = GM(self.inner_opt) self.wrapped_opt._set_k_steps( - self.user_defined_strategy.gradient_merge_configs["k_steps"]) + self.user_defined_strategy.gradient_merge_configs["k_steps"] + ) self.wrapped_opt._set_avg( - self.user_defined_strategy.gradient_merge_configs["avg"]) + self.user_defined_strategy.gradient_merge_configs["avg"] + ) def _can_apply(self): if not self.role_maker._is_collective: return False - can_apply = (self.user_defined_strategy.gradient_merge == True) and \ - self.user_defined_strategy.gradient_merge_configs["k_steps"] > 1 + can_apply = ( + self.user_defined_strategy.gradient_merge == True + ) and self.user_defined_strategy.gradient_merge_configs["k_steps"] > 1 return can_apply def _disable_strategy(self, dist_strategy): @@ -62,13 +65,11 @@ class GradientMergeOptimizer(MetaOptimizerBase): # we currently do not support auto-enable GradientMerge return - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self._init_wrapped_opt() - optimize_ops, params_grads = \ - self.wrapped_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + optimize_ops, params_grads = self.wrapped_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py index a5b0856a66f..dd2ccfc7ff7 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py @@ -24,7 +24,6 @@ __all__ = [] class GraphExecutionOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(GraphExecutionOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -45,12 +44,14 @@ class GraphExecutionOptimizer(MetaOptimizerBase): return False return not self.user_defined_strategy.without_graph_optimization - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): pass # should fix the variable @@ -69,75 +70,77 @@ class GraphExecutionOptimizer(MetaOptimizerBase): if trainer_id == 0 and not paddle.is_compiled_with_npu(): wait_server_ready(other_trainers) - if build_strategy.reduce_strategy == BuildStrategy.ReduceStrategy._NoReduce: + if ( + build_strategy.reduce_strategy + == BuildStrategy.ReduceStrategy._NoReduce + ): return if core.is_compiled_with_cuda(): comm_id_var = startup_program.global_block().create_var( - name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) for i in range(1, build_strategy.nccl_comm_num): startup_program.global_block().create_var( name="NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) if build_strategy.use_hierarchical_allreduce: for i in range(0, build_strategy.nccl_comm_num): startup_program.global_block().create_var( name="Hierarchical_inter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().create_var( name="Hierarchical_exter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_nccl_id", inputs={}, outputs={"NCCLID": comm_id_var}, attrs={ - "trainers": - trainer_endpoints, - "trainer_id": - trainer_id, - "nccl_comm_num": - build_strategy.nccl_comm_num, - "use_hierarchical_allreduce": - build_strategy.use_hierarchical_allreduce, - "hierarchical_allreduce_inter_ranks": - build_strategy.hierarchical_allreduce_inter_nranks - }) + "trainers": trainer_endpoints, + "trainer_id": trainer_id, + "nccl_comm_num": build_strategy.nccl_comm_num, + "use_hierarchical_allreduce": build_strategy.use_hierarchical_allreduce, + "hierarchical_allreduce_inter_ranks": build_strategy.hierarchical_allreduce_inter_nranks, + }, + ) elif core.is_compiled_with_xpu(): comm_id_var = startup_program.global_block().create_var( - name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) - #NOTE(liuyuhui) Baidu Kunlun Communication Library(BKCL) currently do not support multi machines. - assert build_strategy.bkcl_comm_num == 1, \ - "Baidu Kunlun Communication Library(BKCL) currently do not support multi machines." + # NOTE(liuyuhui) Baidu Kunlun Communication Library(BKCL) currently do not support multi machines. + assert ( + build_strategy.bkcl_comm_num == 1 + ), "Baidu Kunlun Communication Library(BKCL) currently do not support multi machines." for i in range(1, build_strategy.bkcl_comm_num): startup_program.global_block().create_var( name="BKCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_bkcl_id", inputs={}, outputs={"BKCLID": comm_id_var}, attrs={ - "trainers": - trainer_endpoints, - "trainer_id": - trainer_id, - "nccl_comm_num": - build_strategy.nccl_comm_num, - "use_hierarchical_allreduce": - build_strategy.use_hierarchical_allreduce, - "hierarchical_allreduce_inter_ranks": - build_strategy.hierarchical_allreduce_inter_nranks - }) + "trainers": trainer_endpoints, + "trainer_id": trainer_id, + "nccl_comm_num": build_strategy.nccl_comm_num, + "use_hierarchical_allreduce": build_strategy.use_hierarchical_allreduce, + "hierarchical_allreduce_inter_ranks": build_strategy.hierarchical_allreduce_inter_nranks, + }, + ) else: raise ValueError( "comm_id must be generated in paddlepaddle-xpu or paddlepaddle-gpu." @@ -147,27 +150,32 @@ class GraphExecutionOptimizer(MetaOptimizerBase): dist_strategy = self.user_defined_strategy local_build_strategy = dist_strategy.build_strategy - local_build_strategy.use_hierarchical_allreduce = \ + local_build_strategy.use_hierarchical_allreduce = ( dist_strategy.use_hierarchical_allreduce - local_build_strategy.hierarchical_allreduce_inter_nranks = \ + ) + local_build_strategy.hierarchical_allreduce_inter_nranks = ( dist_strategy.hierarchical_allreduce_inter_nranks - local_build_strategy.sync_batch_norm = \ - dist_strategy.sync_batch_norm - local_build_strategy.fuse_all_reduce_ops = \ + ) + local_build_strategy.sync_batch_norm = dist_strategy.sync_batch_norm + local_build_strategy.fuse_all_reduce_ops = ( dist_strategy.fuse_all_reduce_ops - local_build_strategy.nccl_comm_num = \ - dist_strategy.nccl_comm_num + ) + local_build_strategy.nccl_comm_num = dist_strategy.nccl_comm_num - gradient_scale_configs = self.user_defined_strategy.gradient_scale_configs + gradient_scale_configs = ( + self.user_defined_strategy.gradient_scale_configs + ) scale_strategys = { 'avg': BuildStrategy.GradientScaleStrategy.CoeffNumDevice, 'sum': BuildStrategy.GradientScaleStrategy.One, 'customized': BuildStrategy.GradientScaleStrategy.Customized, } - assert gradient_scale_configs['scale_strategy'] in scale_strategys, \ - "gradient_scale_configs.scale_strategy must be 'avg', 'sum' or 'customized'" - local_build_strategy.gradient_scale_strategy = \ - scale_strategys[gradient_scale_configs['scale_strategy']] + assert ( + gradient_scale_configs['scale_strategy'] in scale_strategys + ), "gradient_scale_configs.scale_strategy must be 'avg', 'sum' or 'customized'" + local_build_strategy.gradient_scale_strategy = scale_strategys[ + gradient_scale_configs['scale_strategy'] + ] if self.user_defined_strategy.recompute == True: logging.warn( @@ -180,7 +188,9 @@ class GraphExecutionOptimizer(MetaOptimizerBase): node_num = self.role_maker._node_num() if self.role_maker._is_collective: - assert worker_num >= 1, "nccl2 worker_num must >= 1, now:{}" % worker_num + assert worker_num >= 1, ( + "nccl2 worker_num must >= 1, now:{}" % worker_num + ) if worker_num <= 1: # local mode @@ -198,8 +208,8 @@ class GraphExecutionOptimizer(MetaOptimizerBase): sync_allreduce = dist_strategy.sync_nccl_allreduce if sync_allreduce: exe_strategy.num_threads = max( - local_build_strategy.nccl_comm_num + 1, - exe_strategy.num_threads) + local_build_strategy.nccl_comm_num + 1, exe_strategy.num_threads + ) if local_build_strategy.nccl_comm_num > 1: logging.warn( "nccl_comm_num > 1, you may need to set sync_nccl_allreduce=False to ensure that different nccl comms can overlap" @@ -217,17 +227,23 @@ class GraphExecutionOptimizer(MetaOptimizerBase): # NOTE. compatible with compiler, otherwise these values will be overwritten by compiler main_program._nccl_comm_num = local_build_strategy.nccl_comm_num - main_program._use_hierarchical_allreduce = local_build_strategy.use_hierarchical_allreduce - main_program._hierarchical_allreduce_inter_nranks = local_build_strategy.hierarchical_allreduce_inter_nranks + main_program._use_hierarchical_allreduce = ( + local_build_strategy.use_hierarchical_allreduce + ) + main_program._hierarchical_allreduce_inter_nranks = ( + local_build_strategy.hierarchical_allreduce_inter_nranks + ) # TODO(guru4elephant): should be an independent optimizer if worker_num > 1: - self._setup_nccl_op(startup_program, main_program, - local_build_strategy) + self._setup_nccl_op( + startup_program, main_program, local_build_strategy + ) local_build_strategy.num_trainers = self.role_maker._worker_num() local_build_strategy.trainer_id = self.role_maker._worker_index() - local_build_strategy.trainers_endpoints = self.role_maker._get_trainer_endpoints( + local_build_strategy.trainers_endpoints = ( + self.role_maker._get_trainer_endpoints() ) local_build_strategy.enable_backward_optimizer_op_deps = True @@ -237,7 +253,8 @@ class GraphExecutionOptimizer(MetaOptimizerBase): loss_name=loss.name, build_strategy=local_build_strategy, exec_strategy=exe_strategy, - share_vars_from=None) + share_vars_from=None, + ) return self._compiled_program @@ -249,15 +266,14 @@ class GraphExecutionOptimizer(MetaOptimizerBase): # by default, graph execution strategy is enabled return - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): if startup_program == None: startup_program = paddle.static.default_startup_program() - compiled_program = self._try_to_compile(startup_program, - loss.block.program, loss) + compiled_program = self._try_to_compile( + startup_program, loss.block.program, loss + ) loss.block.program._graph = compiled_program # just return self.optimizer_ops and self.param_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py index 3dc5bed03ae..22f225469c2 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py @@ -20,7 +20,6 @@ __all__ = [] class LambOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(LambOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -29,11 +28,12 @@ class LambOptimizer(MetaOptimizerBase): self.meta_optimizers_white_list = ["GraphExecutionOptimizer"] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(LambOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(LambOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) opt = self.inner_opt if not isinstance(opt, AdamOptimizer): @@ -63,7 +63,8 @@ class LambOptimizer(MetaOptimizerBase): regularization=opt.regularization, grad_clip=opt._grad_clip, exclude_from_weight_decay_fn=_exclude_from_weight_decay_fn, - name=opt._name) + name=opt._name, + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -72,8 +73,10 @@ class LambOptimizer(MetaOptimizerBase): if self.user_defined_strategy.lamb: if not isinstance(self.inner_opt, AdamOptimizer): logging.warn( - "lamb need the inner optimizer to be AdamOptimizer optimizer but got {}." - .format(self.inner_opt.type)) + "lamb need the inner optimizer to be AdamOptimizer optimizer but got {}.".format( + self.inner_opt.type + ) + ) return False return True return False @@ -86,33 +89,34 @@ class LambOptimizer(MetaOptimizerBase): dist_strategy.lamb = True dist_strategy.lamb_configs = { "lamb_weight_decay": 0.01, - "exclude_from_weight_decay": [] + "exclude_from_weight_decay": [], } - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self.lamb_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self.lamb_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) # the following function will be used by AMP if both LARS and AMP are turn on together. def apply_gradients(self, params_grads): return self.lamb_opt.apply_gradients(params_grads=params_grads) def apply_optimize(self, loss, startup_program, params_grads): - return self.lamb_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - optimize_ops, params_grads = \ - self.lamb_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + return self.lamb_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + optimize_ops, params_grads = self.lamb_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py index 44f8fe473e2..e584783581e 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py @@ -19,7 +19,6 @@ __all__ = [] class LarsOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(LarsOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -28,11 +27,12 @@ class LarsOptimizer(MetaOptimizerBase): self.meta_optimizers_white_list = ["GraphExecutionOptimizer"] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(LarsOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(LarsOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) opt = self.inner_opt if not isinstance(opt, Momentum): @@ -50,7 +50,8 @@ class LarsOptimizer(MetaOptimizerBase): grad_clip=opt._grad_clip, name=opt._name, exclude_from_weight_decay=configs['exclude_from_weight_decay'], - epsilon=configs['epsilon']) + epsilon=configs['epsilon'], + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -59,8 +60,10 @@ class LarsOptimizer(MetaOptimizerBase): if self.user_defined_strategy.lars: if not isinstance(self.inner_opt, Momentum): logging.warn( - "lars need the inner optimizer to be Momentum optimizer but got {}." - .format(self.inner_opt.type)) + "lars need the inner optimizer to be Momentum optimizer but got {}.".format( + self.inner_opt.type + ) + ) return False return True return False @@ -76,30 +79,31 @@ class LarsOptimizer(MetaOptimizerBase): "lars_weight_decay": 0.0005, } - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self.lars_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self.lars_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) # the following function will be used by AMP if both LARS and AMP are turn on together. def apply_gradients(self, params_grads): return self.lars_opt.apply_gradients(params_grads=params_grads) def apply_optimize(self, loss, startup_program, params_grads): - return self.lars_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - optimize_ops, params_grads = \ - self.lars_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + return self.lars_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + optimize_ops, params_grads = self.lars_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py index 3431ad7e914..aba540fe7e7 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py @@ -22,7 +22,6 @@ __all__ = [] class LocalSGDOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(LocalSGDOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -43,10 +42,12 @@ class LocalSGDOptimizer(MetaOptimizerBase): if self.role_maker._worker_num() <= 1: return False - return isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) \ - or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) \ - or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) \ + return ( + isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) + or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) + or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) or isinstance(self.inner_opt, paddle.fluid.optimizer.SGD) + ) def _disable_strategy(self, dist_strategy): dist_strategy.localsgd = False @@ -69,11 +70,13 @@ class LocalSGDOptimizer(MetaOptimizerBase): p2s = [] for param in non_dist_params: - snapshot = block.create_var(name=self.snapshot_name(param.name), - shape=param.shape, - persistable=True, - stop_gradient=True, - dtype=param.dtype) + snapshot = block.create_var( + name=self.snapshot_name(param.name), + shape=param.shape, + persistable=True, + stop_gradient=True, + dtype=param.dtype, + ) p2s.append([param, snapshot]) return p2s @@ -82,17 +85,17 @@ class LocalSGDOptimizer(MetaOptimizerBase): for param, snapshot in param2snapshot: layers.assign(param, snapshot) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - minimized = self.inner_opt.minimize(loss, - startup_program=startup_program) + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + minimized = self.inner_opt.minimize( + loss, startup_program=startup_program + ) k_steps_value = self.user_defined_strategy.localsgd_configs['k_steps'] begin_step_value = self.user_defined_strategy.localsgd_configs[ - 'begin_step'] + 'begin_step' + ] if startup_program is None: startup_program = default_startup_program() @@ -107,78 +110,90 @@ class LocalSGDOptimizer(MetaOptimizerBase): p2s = self.create_snapshot_vars(main_block.program) with program_guard(main_block.program, startup_program): step = layers.autoincreased_step_counter(begin=1) - k_steps = layers.create_global_var(name="k_steps", - shape=[1], - value=k_steps_value, - dtype='int64', - persistable=True) - - begin_step = layers.create_global_var(name="begin_step", - shape=[1], - value=begin_step_value, - dtype='int64', - persistable=True) - - last_step = layers.create_global_var(name="last_step", - shape=[1], - value=begin_step_value, - dtype='int64', - persistable=True) + k_steps = layers.create_global_var( + name="k_steps", + shape=[1], + value=k_steps_value, + dtype='int64', + persistable=True, + ) + + begin_step = layers.create_global_var( + name="begin_step", + shape=[1], + value=begin_step_value, + dtype='int64', + persistable=True, + ) + + last_step = layers.create_global_var( + name="last_step", + shape=[1], + value=begin_step_value, + dtype='int64', + persistable=True, + ) def communicate(): sub_block = default_main_program().current_block() ring_id = -1 for param, snapshot in p2s: - sub_block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - sub_block.append_op(type='c_sync_calc_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + sub_block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + sub_block.append_op( + type='c_sync_calc_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) ring_id = (ring_id + 1) % self.nrings - sub_block.append_op(type='c_allreduce_sum', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Optimize - }) + sub_block.append_op( + type='c_allreduce_sum', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for ring_id in range(self.nrings): - sub_block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Optimize - }) + sub_block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for param, snapshot in p2s: - sub_block.append_op(type='scale', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'scale': - 1.0 / self.role_maker._worker_num(), - OP_ROLE_KEY: - OpRole.Optimize - }) - sub_block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - sub_block.append_op(type='assign', - inputs={'X': [param]}, - outputs={'Out': [snapshot]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + sub_block.append_op( + type='scale', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'scale': 1.0 / self.role_maker._worker_num(), + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + sub_block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + sub_block.append_op( + type='assign', + inputs={'X': [param]}, + outputs={'Out': [snapshot]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) layers.assign(step, last_step) def begin_localsgd(): @@ -189,13 +204,13 @@ class LocalSGDOptimizer(MetaOptimizerBase): class AdaptiveLocalSGDOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(AdaptiveLocalSGDOptimizer, self).__init__(optimizer) self.inner_opt = optimizer self.meta_optimizers_white_list = ['AMPOptimizer'] self.meta_optimizers_black_list = [ - "GraphExecutionOptimizer", "LocalSGDOptimizer" + "GraphExecutionOptimizer", + "LocalSGDOptimizer", ] self.snapshot_key = '@SNAPSHOT' @@ -209,10 +224,12 @@ class AdaptiveLocalSGDOptimizer(MetaOptimizerBase): if self.role_maker._worker_num() <= 1: return False - return isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) \ - or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) \ - or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) \ + return ( + isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) + or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) + or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) or isinstance(self.inner_opt, paddle.fluid.optimizer.SGD) + ) def _disable_strategy(self, dist_strategy): dist_strategy.adaptive_localsgd = False @@ -222,7 +239,7 @@ class AdaptiveLocalSGDOptimizer(MetaOptimizerBase): dist_strategy.adaptive_localsgd = True dist_strategy.adaptive_localsgd_configs = { "init_k_steps": 1, - "begin_step": 1 + "begin_step": 1, } def snapshot_name(self, param_name): @@ -238,11 +255,13 @@ class AdaptiveLocalSGDOptimizer(MetaOptimizerBase): p2s = [] for param in non_dist_params: - snapshot = block.create_var(name=self.snapshot_name(param.name), - shape=param.shape, - persistable=True, - stop_gradient=True, - dtype=param.dtype) + snapshot = block.create_var( + name=self.snapshot_name(param.name), + shape=param.shape, + persistable=True, + stop_gradient=True, + dtype=param.dtype, + ) p2s.append([param, snapshot]) return p2s @@ -252,40 +271,46 @@ class AdaptiveLocalSGDOptimizer(MetaOptimizerBase): layers.assign(param, snapshot) def _generate_avg_loss(self, program_block, loss, avg_loss): - program_block.append_op(type='c_allreduce_sum', - inputs={'X': [loss]}, - outputs={'Out': [avg_loss]}, - attrs={ - 'ring_id': 0, - OP_ROLE_KEY: OpRole.Optimize, - 'use_calc_stream': True - }) - program_block.append_op(type='c_sync_calc_stream', - inputs={'X': [avg_loss]}, - outputs={'Out': [avg_loss]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - - program_block.append_op(type='scale', - inputs={'X': [avg_loss]}, - outputs={'Out': [avg_loss]}, - attrs={ - 'scale': - 1.0 / self.role_maker._worker_num(), - OP_ROLE_KEY: OpRole.Optimize - }) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - minimized = self.inner_opt.minimize(loss, - startup_program=startup_program) + program_block.append_op( + type='c_allreduce_sum', + inputs={'X': [loss]}, + outputs={'Out': [avg_loss]}, + attrs={ + 'ring_id': 0, + OP_ROLE_KEY: OpRole.Optimize, + 'use_calc_stream': True, + }, + ) + program_block.append_op( + type='c_sync_calc_stream', + inputs={'X': [avg_loss]}, + outputs={'Out': [avg_loss]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + + program_block.append_op( + type='scale', + inputs={'X': [avg_loss]}, + outputs={'Out': [avg_loss]}, + attrs={ + 'scale': 1.0 / self.role_maker._worker_num(), + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + minimized = self.inner_opt.minimize( + loss, startup_program=startup_program + ) init_k_steps = self.user_defined_strategy.adaptive_localsgd_configs[ - 'init_k_steps'] + 'init_k_steps' + ] begin_step_value = self.user_defined_strategy.adaptive_localsgd_configs[ - 'begin_step'] + 'begin_step' + ] if startup_program is None: startup_program = default_startup_program() @@ -301,41 +326,53 @@ class AdaptiveLocalSGDOptimizer(MetaOptimizerBase): with program_guard(main_block.program, startup_program): step = layers.autoincreased_step_counter(begin=1) - k_steps = layers.create_global_var(name="k_steps", - shape=[1], - value=int(init_k_steps), - dtype='int64', - persistable=True) - - begin_step = layers.create_global_var(name="begin_step", - shape=[1], - value=int(begin_step_value), - dtype='int64', - persistable=True) - - last_step = layers.create_global_var(name="last_step", - shape=[1], - value=int(0), - dtype='int64', - persistable=True) - - avg_loss = layers.create_global_var(name="avg_loss", - shape=[1], - value=float(0), - dtype=loss.dtype, - persistable=True) - - lr_0 = layers.create_global_var(name="lr_0", - shape=[1], - value=float(0), - dtype='float32', - persistable=True) - - loss_0 = layers.create_global_var(name="loss_0", - shape=[1], - value=float(0), - dtype='float32', - persistable=True) + k_steps = layers.create_global_var( + name="k_steps", + shape=[1], + value=int(init_k_steps), + dtype='int64', + persistable=True, + ) + + begin_step = layers.create_global_var( + name="begin_step", + shape=[1], + value=int(begin_step_value), + dtype='int64', + persistable=True, + ) + + last_step = layers.create_global_var( + name="last_step", + shape=[1], + value=int(0), + dtype='int64', + persistable=True, + ) + + avg_loss = layers.create_global_var( + name="avg_loss", + shape=[1], + value=float(0), + dtype=loss.dtype, + persistable=True, + ) + + lr_0 = layers.create_global_var( + name="lr_0", + shape=[1], + value=float(0), + dtype='float32', + persistable=True, + ) + + loss_0 = layers.create_global_var( + name="loss_0", + shape=[1], + value=float(0), + dtype='float32', + persistable=True, + ) global_lr = self.inner_opt._global_learning_rate() @@ -350,75 +387,90 @@ class AdaptiveLocalSGDOptimizer(MetaOptimizerBase): sub_block = default_main_program().current_block() ring_id = -1 for param, snapshot in p2s: - sub_block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - sub_block.append_op(type='c_sync_calc_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + sub_block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + sub_block.append_op( + type='c_sync_calc_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) ring_id = (ring_id + 1) % self.nrings - sub_block.append_op(type='c_allreduce_sum', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Optimize - }) + sub_block.append_op( + type='c_allreduce_sum', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for ring_id in range(self.nrings): - sub_block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Optimize - }) + sub_block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for param, snapshot in p2s: - sub_block.append_op(type='scale', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'scale': - 1.0 / self.role_maker._worker_num(), - OP_ROLE_KEY: - OpRole.Optimize - }) - sub_block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - sub_block.append_op(type='assign', - inputs={'X': [param]}, - outputs={'Out': [snapshot]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + sub_block.append_op( + type='scale', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'scale': 1.0 / self.role_maker._worker_num(), + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + sub_block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + sub_block.append_op( + type='assign', + inputs={'X': [param]}, + outputs={'Out': [snapshot]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) layers.assign(step, last_step) def communicate_avg_loss(): communicate() self._generate_avg_loss(main_block, loss, avg_loss) - next_local_steps = layers.cast(layers.ceil( - layers.sqrt(lr_0 * avg_loss / (global_lr * loss_0) * - float(init_k_steps))), - dtype='int64') - max_local_steps = layers.fill_constant(shape=[1], - dtype='int64', - value=16) - min_local_steps = layers.fill_constant(shape=[1], - dtype='int64', - value=1) + next_local_steps = layers.cast( + layers.ceil( + layers.sqrt( + lr_0 + * avg_loss + / (global_lr * loss_0) + * float(init_k_steps) + ) + ), + dtype='int64', + ) + max_local_steps = layers.fill_constant( + shape=[1], dtype='int64', value=16 + ) + min_local_steps = layers.fill_constant( + shape=[1], dtype='int64', value=1 + ) next_local_steps = layers.elementwise_min( - next_local_steps, max_local_steps) + next_local_steps, max_local_steps + ) next_local_steps = layers.elementwise_max( - next_local_steps, min_local_steps) + next_local_steps, min_local_steps + ) layers.assign(next_local_steps, k_steps) def begin_localsgd(): diff --git a/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py b/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py index 35e11221b6f..87085a322c3 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py +++ b/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py @@ -18,7 +18,6 @@ __all__ = [] class MetaOptimizerBase(Optimizer): - def __init__(self, optimizer): self.inner_opt = optimizer self._learning_rate = self.inner_opt._learning_rate @@ -26,8 +25,9 @@ class MetaOptimizerBase(Optimizer): self.meta_optimizers_white_list = [] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): self.loss = loss self.role_maker = role_maker self.user_defined_optimizer = user_defined_optimizer @@ -50,52 +50,57 @@ class MetaOptimizerBase(Optimizer): def _disable_strategy(self, dist_strategy): raise NotImplementedError( "you should implement disable strategy in {}".format( - type(self).__name__)) + type(self).__name__ + ) + ) def _enable_strategy(self, dist_strategy, context=None): raise NotImplementedError( "you should implement enable strategy in {}".format( - type(self).__name__)) + type(self).__name__ + ) + ) def apply_gradients(self, params_grads): return self.inner_opt.apply_gradients(params_grads=params_grads) - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self.inner_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self.inner_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_optimize(self, loss, startup_program, params_grads): - return self.inner_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) - - optimize_ops = self.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + return self.inner_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) + + optimize_ops = self.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - optimize_ops, params_grads = self.minimize_impl(loss, startup_program, - parameter_list, - no_grad_set) + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + optimize_ops, params_grads = self.minimize_impl( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py index 42784113822..f94215aad09 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py @@ -18,7 +18,6 @@ __all__ = [] class ParameterServerGraphOptimizer(ParameterServerOptimizer): - def __init__(self, optimizer): super(ParameterServerGraphOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -59,18 +58,18 @@ class ParameterServerGraphOptimizer(ParameterServerOptimizer): self._compiled_program = compiler.CompiledProgram(main_program) - self._compiled_program.with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - exec_strategy=exec_strategy, - share_vars_from=None) + self._compiled_program.with_data_parallel( + loss_name=loss.name, + build_strategy=build_strategy, + exec_strategy=exec_strategy, + share_vars_from=None, + ) return self._compiled_program - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): program = loss.block.program compiled_program = self._try_to_compile(program, loss) program._graph = compiled_program diff --git a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py index c04215d4565..d18c5d1fdcb 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py @@ -24,23 +24,24 @@ __all__ = [] class ParameterServerOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(ParameterServerOptimizer, self).__init__(optimizer) self.inner_opt = optimizer # we do not allow meta optimizer to be inner optimizer currently self.meta_optimizers_white_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(ParameterServerOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(ParameterServerOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) - #self.micro_batch_size = user_defined_strategy.pipeline_configs[ + # self.micro_batch_size = user_defined_strategy.pipeline_configs[ # 'micro_batch_size'] self.num_microbatches = user_defined_strategy.pipeline_configs[ - 'accumulate_steps'] + 'accumulate_steps' + ] def _is_graph_out(self): return False @@ -66,11 +67,13 @@ class ParameterServerOptimizer(MetaOptimizerBase): 'trainer_id': trainer_id, 'num_trainers': num_trainers, 'current_endpoint': current_endpoint, - 'trainer_endpoints': trainer_endpoints + 'trainer_endpoints': trainer_endpoints, } def _get_distributed_strategy(self): - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, + ) k_steps = self.user_defined_strategy.a_sync_configs["k_steps"] strategy = None @@ -90,7 +93,9 @@ class ParameterServerOptimizer(MetaOptimizerBase): return strategy def _build_trainer_programs(self, compiled_config): - from paddle.fluid.incubate.fleet.parameter_server.ir import trainer_pass as worker + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + trainer_pass as worker, + ) _main = compiled_config.origin_main_program.clone() _startup = compiled_config.origin_startup_program.clone() @@ -98,49 +103,66 @@ class ParameterServerOptimizer(MetaOptimizerBase): use_ps_gpu = self.user_defined_strategy.a_sync_configs["use_ps_gpu"] if not compiled_config.is_geo_mode(): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _add_lr_decay_table_pass + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _add_lr_decay_table_pass, + ) + _add_lr_decay_table_pass( - _main, compiled_config, - self.user_defined_strategy.a_sync_configs["lr_decay_steps"]) + _main, + compiled_config, + self.user_defined_strategy.a_sync_configs["lr_decay_steps"], + ) # for main program - _main = worker.distributed_ops_pass(_main, compiled_config, - use_ps_gpu) + _main = worker.distributed_ops_pass( + _main, compiled_config, use_ps_gpu + ) if not use_ps_gpu: _main = worker.delete_optimizer_pass(_main, compiled_config) _main = worker.append_send_ops_pass(_main, compiled_config) _startup = worker.delete_extra_optimizes_pass( - _startup, compiled_config) + _startup, compiled_config + ) # for startup program _startup = worker.fake_init_ops_pass(_startup, compiled_config) if use_ps_gpu: _main = worker.ps_gpu_pass(_main) - from paddle.fluid.transpiler.collective import SingleProcessMultiThread + from paddle.fluid.transpiler.collective import ( + SingleProcessMultiThread, + ) + t = SingleProcessMultiThread() env = self.get_dist_env() - t.transpile(startup_program=_startup, - main_program=_main, - rank=env["trainer_id"], - endpoints=env["trainer_endpoints"], - current_endpoint=env['current_endpoint'], - wait_port=False) + t.transpile( + startup_program=_startup, + main_program=_main, + rank=env["trainer_id"], + endpoints=env["trainer_endpoints"], + current_endpoint=env['current_endpoint'], + wait_port=False, + ) compiled_config.set_origin_ps_main_program(_main) compiled_config.set_origin_ps_startup_program(_startup) # for heter program if self.role_maker._is_heter_parameter_server_mode: - from paddle.fluid.incubate.fleet.parameter_server.ir import heter_trainer_pass as heter_worker + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + heter_trainer_pass as heter_worker, + ) + if self.role_maker._is_heter_worker(): # for heter worker stage_id = self.role_maker._get_stage_id() device = self.role_maker._heter_device_type().lower() _main = heter_worker.split_heter_worker_ops_pass( - _main, compiled_config, stage_id, device) + _main, compiled_config, stage_id, device + ) else: # for default worker _main = heter_worker.split_trainer_ops_pass( - _main, compiled_config) + _main, compiled_config + ) else: _main = worker.append_send_ops_pass(_main, compiled_config) _startup = _startup @@ -148,7 +170,8 @@ class ParameterServerOptimizer(MetaOptimizerBase): compiled_config.set_origin_ps_startup_program(_startup) launch_barrier = self.user_defined_strategy.a_sync_configs[ - "launch_barrier"] + "launch_barrier" + ] launch_barrier_flag = int(os.getenv("FLAGS_LAUNCH_BARRIER", "1")) if launch_barrier and launch_barrier_flag: # for trainer wait server ready @@ -165,11 +188,16 @@ class ParameterServerOptimizer(MetaOptimizerBase): _main = fluid.Program() _startup = fluid.Program() - from paddle.fluid.incubate.fleet.parameter_server.ir import pserver_pass as server + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + pserver_pass as server, + ) if not compiled_config.is_geo_mode(): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, + ) + is_sgd_adam = False main_program = compiled_config.get_origin_main_program() @@ -178,11 +206,16 @@ class ParameterServerOptimizer(MetaOptimizerBase): if len(ops) == 0: return _main, _startup - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _add_lr_decay_table_pass + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _add_lr_decay_table_pass, + ) + lr_decay_steps = self.user_defined_strategy.a_sync_configs[ - "lr_decay_steps"] - _add_lr_decay_table_pass(main_program, compiled_config, - lr_decay_steps) + "lr_decay_steps" + ] + _add_lr_decay_table_pass( + main_program, compiled_config, lr_decay_steps + ) for op in ops: if op.type in ["sgd", "adam"]: @@ -195,37 +228,44 @@ class ParameterServerOptimizer(MetaOptimizerBase): _main = server.add_listen_and_serv_pass(_main, compiled_config) _main = server.add_rpc_global_flags_pass(_main, compiled_config) _main = server.add_optimizer_pass(_main, compiled_config) - _main = server.large_scale_sparse_pass(_main, _main, - compiled_config, False) + _main = server.large_scale_sparse_pass( + _main, _main, compiled_config, False + ) _startup = server.build_pserver_startup_program_pass( - _startup, _main, compiled_config) - _startup = server.large_scale_sparse_pass(_startup, _main, - compiled_config, True) + _startup, _main, compiled_config + ) + _startup = server.large_scale_sparse_pass( + _startup, _main, compiled_config, True + ) if not compiled_config.is_sync_mode(): _main = server.delete_unused_in_main_pass( - _main, compiled_config) + _main, compiled_config + ) _startup = server.delete_unused_in_startup_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) else: _main = server.add_listen_and_serv_pass(_main, compiled_config) _main = server.add_rpc_global_flags_pass(_main, compiled_config) _main = server.add_geo_optimizer_pass(_main, compiled_config) _startup = server.build_pserver_startup_program_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) _startup = server.delete_unused_in_startup_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) return _main, _startup def _can_apply_geo(self, dist_strategy, program): - def get_sys_free_mem(): plat = platform.system() if platform.system() == "Darwin": - vm = subprocess.Popen(['vm_stat'], - stdout=subprocess.PIPE).communicate()[0] + vm = subprocess.Popen( + ['vm_stat'], stdout=subprocess.PIPE + ).communicate()[0] # Process vm_stat vmLines = vm.split('\n') sep = re.compile(r':[\s]+') @@ -233,8 +273,9 @@ class ParameterServerOptimizer(MetaOptimizerBase): for row in range(1, len(vmLines) - 2): rowText = vmLines[row].strip() rowElements = sep.split(rowText) - vmStats[(rowElements[0])] = int( - rowElements[1].strip(r'\.')) * 4096 + vmStats[(rowElements[0])] = ( + int(rowElements[1].strip(r'\.')) * 4096 + ) return vmStats["Pages free"] elif platform.system() == "Linux": mems = {} @@ -246,22 +287,27 @@ class ParameterServerOptimizer(MetaOptimizerBase): return free else: raise ValueError( - "%s platform is unsupported is parameter server optimizer" % - (platform.system())) + "%s platform is unsupported is parameter server optimizer" + % (platform.system()) + ) if not isinstance(self.inner_opt, fluid.optimizer.SGDOptimizer): return False free = get_sys_free_mem() - from paddle.fluid.incubate.fleet.parameter_server.ir import vars_metatools + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + vars_metatools, + ) processed_var_names = set(["@EMPTY@"]) param_memory_size = 0 for varname in program.global_block().vars: var = program.global_block().vars[varname] - if not var.persistable or var.desc.type( - ) != core.VarDesc.VarType.LOD_TENSOR: + if ( + not var.persistable + or var.desc.type() != core.VarDesc.VarType.LOD_TENSOR + ): continue param = vars_metatools.create_var_struct(var) param_memory_size += param.m_size @@ -287,15 +333,18 @@ class ParameterServerOptimizer(MetaOptimizerBase): if x < 0: if neg_dim_count >= 1: raise ValueError( - "Var %s has more than one negative dim." % - (var_name)) + "Var %s has more than one negative dim." + % (var_name) + ) neg_dim_count += 1 - data_count *= (-x) + data_count *= -x else: data_count *= x program_tmp_vars[var_name] = ( - data_count, neg_dim_count, - vars_metatools.dtype_to_size[var.dtype]) + data_count, + neg_dim_count, + vars_metatools.dtype_to_size[var.dtype], + ) for varname in program_tmp_vars: data_count, neg_dim_count, type_size = program_tmp_vars[varname] @@ -309,27 +358,32 @@ class ParameterServerOptimizer(MetaOptimizerBase): else: return False - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - self.inner_opt.minimize(loss, startup_program, parameter_list, - no_grad_set) + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + self.inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) strategy = self._get_distributed_strategy() _origin_main_program = loss.block.program _origin_startup_program = startup_program - from paddle.fluid.incubate.fleet.parameter_server.ir import public as public - - compiled_config = public.CompileTimeStrategy(_origin_main_program, - _origin_startup_program, - strategy, self.role_maker) + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + public as public, + ) + + compiled_config = public.CompileTimeStrategy( + _origin_main_program, + _origin_startup_program, + strategy, + self.role_maker, + ) compiled_config.strategy = strategy if self.role_maker._is_worker() or self.role_maker._is_heter_worker(): main_program, startup_program = self._build_trainer_programs( - compiled_config) + compiled_config + ) if self.role_maker._is_heter_parameter_server_mode: _origin_startup_program._heter_pipeline_opt = { "startup_program": startup_program, @@ -340,12 +394,12 @@ class ParameterServerOptimizer(MetaOptimizerBase): loss.block.program._heter_pipeline_opt = { "trainer": "HeterPipelineTrainer", "device_worker": "HeterSection", - "trainers": self.role_maker._get_stage_trainers( - ), ## trainer num in each stage + "trainers": self.role_maker._get_stage_trainers(), ## trainer num in each stage "trainer_id": int(self.role_maker._role_id()), "pipeline_stage": int(self.role_maker._get_stage_id()) - 1, - "num_pipeline_stages": - int(self.role_maker._get_num_stage()), + "num_pipeline_stages": int( + self.role_maker._get_num_stage() + ), "section_program": main_program, "num_microbatches": self.num_microbatches, "heter_place": self.role_maker._heter_device(), @@ -356,13 +410,14 @@ class ParameterServerOptimizer(MetaOptimizerBase): elif self.role_maker._is_server(): main_program, startup_program = self._build_pserver_programs( - compiled_config) + compiled_config + ) loss.block.program = main_program fluid.framework.switch_startup_program(startup_program) return None, None def _disable_strategy(self, dist_strategy): - #if self.role_maker._is_heter_parameter_server_mode: + # if self.role_maker._is_heter_parameter_server_mode: # dist_strategy.pipeline = False # dist_strategy.pipeline_configs = { # "micro_batch_size": 1, @@ -374,7 +429,7 @@ class ParameterServerOptimizer(MetaOptimizerBase): dist_strategy.a_sync_configs = a_sync_configs def _enable_strategy(self, dist_strategy, context): - #if self.role_maker._is_heter_parameter_server_mode: + # if self.role_maker._is_heter_parameter_server_mode: # dist_strategy.pipeline = True # dist_strategy.pipeline_configs = { # "micro_batch_size": 1, @@ -387,8 +442,9 @@ class ParameterServerOptimizer(MetaOptimizerBase): dist_strategy.a_sync = True a_sync_configs = dist_strategy.a_sync_configs - is_geo = self._can_apply_geo(dist_strategy, - context["origin_main_program"]) + is_geo = self._can_apply_geo( + dist_strategy, context["origin_main_program"] + ) if is_geo: a_sync_configs["k_steps"] = 800 diff --git a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py index 690ccdfea5f..dfb8fe5b224 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py @@ -14,13 +14,19 @@ import paddle.fluid as fluid from paddle.fluid.optimizer import PipelineOptimizer as PO from .meta_optimizer_base import MetaOptimizerBase -from .common import CollectiveHelper, OP_ROLE_KEY, OP_ROLE_VAR_KEY, OpRole, is_backward_op, is_loss_grad_op +from .common import ( + CollectiveHelper, + OP_ROLE_KEY, + OP_ROLE_VAR_KEY, + OpRole, + is_backward_op, + is_loss_grad_op, +) __all__ = [] class PipelineOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(PipelineOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -35,17 +41,21 @@ class PipelineOptimizer(MetaOptimizerBase): self.dp_ring_id = 2 self.start_pipeline_ring_id = 20 # Just a magic number - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(PipelineOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(PipelineOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) self.micro_batch_size = user_defined_strategy.pipeline_configs[ - 'micro_batch_size'] + 'micro_batch_size' + ] self.num_microbatches = user_defined_strategy.pipeline_configs[ - 'accumulate_steps'] + 'accumulate_steps' + ] self.schedule_mode = user_defined_strategy.pipeline_configs[ - 'schedule_mode'] + 'schedule_mode' + ] self.use_sharding = user_defined_strategy.sharding def _can_apply(self): @@ -83,23 +93,25 @@ class PipelineOptimizer(MetaOptimizerBase): if param.is_distributed: continue - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward, + }, + ) - if not param: return # no parameter on this device - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + if not param: + return # no parameter on this device + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Forward}, + ) def _get_process_group_info(self): # global ring info @@ -121,12 +133,16 @@ class PipelineOptimizer(MetaOptimizerBase): self._get_process_group_info() collective_helper = CollectiveHelper(self.role_maker, wait_port=False) # Create global ring for all gpus (ring_id = 0) - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - self.global_endpoints, - self.global_rank, - self.global_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + self.global_endpoints, + self.global_rank, + self.global_ring_id, + True, + self.global_ring_id, + True, + ) # Create pipeline rings if self.inner_parallelism > 1: pipeline_id = self.rank // self.inner_parallelism @@ -139,41 +155,62 @@ class PipelineOptimizer(MetaOptimizerBase): second_node = pair[1] + start_index if self.rank != first_node and self.rank != second_node: collective_helper._init_communicator( - self.startup_program, None, None, None, None, False, - self.global_ring_id, True) + self.startup_program, + None, + None, + None, + None, + False, + self.global_ring_id, + True, + ) continue pipeline_endpoints = [ - self.endpoints[first_node], self.endpoints[second_node] + self.endpoints[first_node], + self.endpoints[second_node], ] pipeline_rank = 0 if self.rank == first_node else 1 pipeline_nranks = 2 - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - pipeline_endpoints, - pipeline_rank, ring_id, - False, self.global_ring_id, - True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + pipeline_endpoints, + pipeline_rank, + ring_id, + False, + self.global_ring_id, + True, + ) # Create dp rings if self.pipeline_num > 1: collective_helper._init_communicator( - self.startup_program, self.current_endpoint, self.dp_endpoints, - self.dp_rank, self.dp_ring_id, True, self.global_ring_id, True) + self.startup_program, + self.current_endpoint, + self.dp_endpoints, + self.dp_rank, + self.dp_ring_id, + True, + self.global_ring_id, + True, + ) self._broadcast_params(self.dp_ring_id) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self.endpoints = self.role_maker._get_trainer_endpoints() self.current_endpoint = self.endpoints[self.role_maker._worker_index()] self.rank = self.role_maker._worker_index() self.nranks = self.role_maker._worker_num() - self.wrapped_opt = PO(self.inner_opt, - num_microbatches=self.num_microbatches) - orig_startup_program = startup_program if startup_program else fluid.default_startup_program( + self.wrapped_opt = PO( + self.inner_opt, num_microbatches=self.num_microbatches + ) + orig_startup_program = ( + startup_program + if startup_program + else fluid.default_startup_program() ) block = loss.block program = block.program @@ -187,10 +224,18 @@ class PipelineOptimizer(MetaOptimizerBase): program._pipeline_opt['use_sharding'] = False program._pipeline_opt['mp_degree'] = 1 program._pipeline_opt['mp_rank'] = 0 - optimize_ops, params_grads, prog_list, pp_pair, ring_map = self.wrapped_opt.minimize( - loss, startup_program, parameter_list, no_grad_set) + ( + optimize_ops, + params_grads, + prog_list, + pp_pair, + ring_map, + ) = self.wrapped_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) self.startup_program = orig_startup_program._pipeline_opt[ - 'startup_program'] + 'startup_program' + ] self.inner_parallelism = program._pipeline_opt['inner_parallelism'] assert self.nranks % self.inner_parallelism == 0 assert prog_list @@ -217,18 +262,21 @@ class PipelineOptimizer(MetaOptimizerBase): for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): loss_grad_var = block.vars[op.output_arg_names[0]] - block._insert_op(idx + 1, - type='scale', - inputs={'X': loss_grad_var}, - outputs={'Out': loss_grad_var}, - attrs={ - 'scale': 1.0 / pipeline_num, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / pipeline_num, + OP_ROLE_KEY: OpRole.Backward, + }, + ) def _insert_allreduce_ops(self, ring_id): - block = self.main_program._pipeline_opt['section_program'].global_block( - ) + block = self.main_program._pipeline_opt[ + 'section_program' + ].global_block() origin_block = self.main_program.global_block() grad = None processed_param_name = set() @@ -237,9 +285,9 @@ class PipelineOptimizer(MetaOptimizerBase): if is_backward_op(op) and not first_optimize_op_idx: first_optimize_op_idx = idx + 1 # no optimize phase - if first_optimize_op_idx == len(block.ops): return - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if first_optimize_op_idx == len(block.ops): + return + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.all_attrs()[OP_ROLE_VAR_KEY] if len(op_role_var) == 0: continue @@ -248,21 +296,25 @@ class PipelineOptimizer(MetaOptimizerBase): for i in range(0, len(op_role_var), 2): param_name = op_role_var[i] param = block.vars[op_role_var[i]] - if param_name in processed_param_name: continue + if param_name in processed_param_name: + continue processed_param_name.add(param_name) grad_name = op_role_var[i + 1] - if 'MERGED' not in grad_name: grad_name += '@MERGED' + if 'MERGED' not in grad_name: + grad_name += '@MERGED' grad = block.vars[grad_name] origin_param = origin_block.vars[op_role_var[i]] if origin_param.is_distributed: continue - block._insert_op(first_optimize_op_idx + offset, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op( + first_optimize_op_idx + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/ps_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/ps_optimizer.py index 523a6741102..000ef98bcae 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/ps_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/ps_optimizer.py @@ -19,24 +19,31 @@ import subprocess import re import os import platform -from paddle.distributed.ps.utils.public import TrainerRuntimeConfig, build_var_distributed, dtype_to_size, get_dist_env, get_var_mem_size, logger +from paddle.distributed.ps.utils.public import ( + TrainerRuntimeConfig, + build_var_distributed, + dtype_to_size, + get_dist_env, + get_var_mem_size, + logger, +) from paddle.distributed.passes import PassContext from paddle.distributed.ps.utils.ps_factory import PsProgramBuilderFactory class ParameterServerOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(ParameterServerOptimizer, self).__init__(optimizer) self.inner_opt = optimizer # we do not allow meta optimizer to be inner optimizer currently self.meta_optimizers_white_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(ParameterServerOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(ParameterServerOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _set_origin_programs(self, losses): self.origin_main_programs = [] @@ -66,31 +73,40 @@ class ParameterServerOptimizer(MetaOptimizerBase): logger.info("ps_mode: {}".format(attrs['ps_mode'])) attrs['role_maker'] = self.role_maker attrs[ - 'is_heter_ps_mode'] = self.role_maker._is_heter_parameter_server_mode + 'is_heter_ps_mode' + ] = self.role_maker._is_heter_parameter_server_mode attrs['is_worker'] = self.role_maker._is_worker() attrs['is_server'] = self.role_maker._is_server() attrs['is_heter_worker'] = self.role_maker._is_heter_worker() - logger.info("this process is heter? {}".format( - attrs['is_heter_worker'])) + logger.info( + "this process is heter? {}".format(attrs['is_heter_worker']) + ) attrs['use_ps_gpu'] = self.user_defined_strategy.a_sync_configs[ - "use_ps_gpu"] + "use_ps_gpu" + ] attrs['lr_decay_steps'] = self.user_defined_strategy.a_sync_configs[ - "lr_decay_steps"] + "lr_decay_steps" + ] # FL attrs['local_sparse'] = attrs[ - "user_defined_strategy"].trainer_desc_configs["local_sparse"] + "user_defined_strategy" + ].trainer_desc_configs["local_sparse"] attrs['remote_sparse'] = attrs[ - "user_defined_strategy"].trainer_desc_configs["remote_sparse"] + "user_defined_strategy" + ].trainer_desc_configs["remote_sparse"] attrs['is_fl_ps_mode'] = self.user_defined_strategy.is_fl_ps_mode attrs[ - 'with_coordinator'] = self.user_defined_strategy.is_with_coordinator + 'with_coordinator' + ] = self.user_defined_strategy.is_with_coordinator attrs['k_steps'] = self.user_defined_strategy.a_sync_configs["k_steps"] attrs['launch_barrier'] = self.user_defined_strategy.a_sync_configs[ - "launch_barrier"] + "launch_barrier" + ] attrs['launch_barrier_flag'] = int( - os.getenv("FLAGS_LAUNCH_BARRIER", "1")) + os.getenv("FLAGS_LAUNCH_BARRIER", "1") + ) build_var_distributed(attrs) @@ -111,31 +127,32 @@ class ParameterServerOptimizer(MetaOptimizerBase): k_steps = self.user_defined_strategy.a_sync_configs["k_steps"] return True if k_steps >= 0 else False - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - self.inner_opt.minimize(loss, startup_program, parameter_list, - no_grad_set) + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + self.inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) if startup_program == None: startup_program = paddle.static.default_startup_program() - -# print("program after inner optimizer minimize:", -# str(loss.block.program)) + # print("program after inner optimizer minimize:", + # str(loss.block.program)) self._set_origin_programs([loss]) self._init_ps_pass_context(loss, startup_program) ps_builder = PsProgramBuilderFactory()._create_ps_program_builder( - self.pass_ctx) + self.pass_ctx + ) ps_builder._build_programs() return None, None - def minimize_losses_impl(self, - losses, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_losses_impl( + self, + losses, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ): if parameter_list is None: parameter_list = [None] * len(losses) for idx, loss in enumerate(losses): @@ -148,18 +165,19 @@ class ParameterServerOptimizer(MetaOptimizerBase): startup_prog = startup_program[idx] self._init_ps_pass_context(loss, startup_prog) ps_builder = PsProgramBuilderFactory()._create_ps_program_builder( - self.pass_ctx) + self.pass_ctx + ) ps_builder._build_programs() startup_program[idx] = self.pass_ctx._attrs['cloned_startup'] return None, None def _can_apply_geo(self, program): - def get_sys_free_mem(): plat = platform.system() if platform.system() == "Darwin": - vm = subprocess.Popen(['vm_stat'], - stdout=subprocess.PIPE).communicate()[0] + vm = subprocess.Popen( + ['vm_stat'], stdout=subprocess.PIPE + ).communicate()[0] # Process vm_stat vmLines = vm.split('\n') sep = re.compile(r':[\s]+') @@ -167,8 +185,9 @@ class ParameterServerOptimizer(MetaOptimizerBase): for row in range(1, len(vmLines) - 2): rowText = vmLines[row].strip() rowElements = sep.split(rowText) - vmStats[(rowElements[0])] = int( - rowElements[1].strip(r'\.')) * 4096 + vmStats[(rowElements[0])] = ( + int(rowElements[1].strip(r'\.')) * 4096 + ) return vmStats["Pages free"] elif platform.system() == "Linux": mems = {} @@ -180,8 +199,9 @@ class ParameterServerOptimizer(MetaOptimizerBase): return free else: raise ValueError( - "%s platform is unsupported is parameter server optimizer" % - (platform.system())) + "%s platform is unsupported is parameter server optimizer" + % (platform.system()) + ) if not isinstance(self.inner_opt, fluid.optimizer.SGDOptimizer): return False @@ -191,8 +211,10 @@ class ParameterServerOptimizer(MetaOptimizerBase): param_memory_size = 0 for varname in program.global_block().vars: var = program.global_block().vars[varname] - if not var.persistable or var.desc.type( - ) != core.VarDesc.VarType.LOD_TENSOR: + if ( + not var.persistable + or var.desc.type() != core.VarDesc.VarType.LOD_TENSOR + ): continue param_memory_size += get_var_mem_size(var) processed_var_names.add(varname) @@ -217,14 +239,18 @@ class ParameterServerOptimizer(MetaOptimizerBase): if x < 0: if neg_dim_count >= 1: raise ValueError( - "Var %s has more than one negative dim." % - (var_name)) + "Var %s has more than one negative dim." + % (var_name) + ) neg_dim_count += 1 - data_count *= (-x) + data_count *= -x else: data_count *= x - program_tmp_vars[var_name] = (data_count, neg_dim_count, - dtype_to_size[var.dtype]) + program_tmp_vars[var_name] = ( + data_count, + neg_dim_count, + dtype_to_size[var.dtype], + ) for varname in program_tmp_vars: data_count, neg_dim_count, type_size = program_tmp_vars[varname] diff --git a/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py index 9c7f213105e..6ae89a9754e 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py @@ -14,11 +14,18 @@ import paddle.fluid as fluid from paddle.fluid import core, unique_name from .meta_optimizer_base import MetaOptimizerBase -from .common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY, CollectiveHelper, is_loss_grad_op, is_backward_op, is_optimizer_op +from .common import ( + OpRole, + OP_ROLE_KEY, + OP_ROLE_VAR_KEY, + CollectiveHelper, + is_loss_grad_op, + is_backward_op, + is_optimizer_op, +) class RawProgramOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(RawProgramOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -36,16 +43,23 @@ class RawProgramOptimizer(MetaOptimizerBase): ] self.global_ring_id = 0 - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(RawProgramOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) - self.without_graph_optimization = user_defined_strategy.without_graph_optimization + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(RawProgramOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) + self.without_graph_optimization = ( + user_defined_strategy.without_graph_optimization + ) self.fuse_all_reduce_ops = user_defined_strategy.fuse_all_reduce_ops if self.fuse_all_reduce_ops: - self.fuse_grad_size_in_num = user_defined_strategy.fuse_grad_size_in_num - self.calc_comm_same_stream = user_defined_strategy._calc_comm_same_stream + self.fuse_grad_size_in_num = ( + user_defined_strategy.fuse_grad_size_in_num + ) + self.calc_comm_same_stream = ( + user_defined_strategy._calc_comm_same_stream + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -68,23 +82,25 @@ class RawProgramOptimizer(MetaOptimizerBase): if param.is_distributed: continue - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - OP_ROLE_KEY: OpRole.Forward - }) - - if not param: return # no parameter on this device - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + + if not param: + return # no parameter on this device + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Forward}, + ) def _get_process_group_info(self): # global ring info @@ -96,19 +112,21 @@ class RawProgramOptimizer(MetaOptimizerBase): self._get_process_group_info() collective_helper = CollectiveHelper(self.role_maker, wait_port=False) # Create global ring for all gpus (ring_id = 0) - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - self.global_endpoints, - self.global_rank, - self.global_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + self.global_endpoints, + self.global_rank, + self.global_ring_id, + True, + self.global_ring_id, + True, + ) self._broadcast_params(self.global_ring_id) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self.endpoints = self.role_maker._get_trainer_endpoints() self.current_endpoint = self.endpoints[self.role_maker._worker_index()] self.rank = self.role_maker._worker_index() @@ -122,7 +140,8 @@ class RawProgramOptimizer(MetaOptimizerBase): self.main_program = program optimize_ops, params_grads = self.inner_opt.minimize( - loss, startup_program, parameter_list, no_grad_set) + loss, startup_program, parameter_list, no_grad_set + ) if self.nranks == 1: return optimize_ops, params_grads self._init_process_group() @@ -147,7 +166,9 @@ class RawProgramOptimizer(MetaOptimizerBase): if gm_cond_var_name is None: return None - cond_op = None # false_fn of gm is None, so we should only find one block + cond_op = ( + None # false_fn of gm is None, so we should only find one block + ) for op in self.main_program.global_block().ops: if op.type != 'conditional_block' or 'Cond' not in op.input_names: continue @@ -173,8 +194,7 @@ class RawProgramOptimizer(MetaOptimizerBase): param_vars = [] grad_vars = [] for op in block.ops: - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.attr(OP_ROLE_VAR_KEY) assert len(op_role_var) % 2 == 0 for i in range(0, len(op_role_var), 2): @@ -188,35 +208,41 @@ class RawProgramOptimizer(MetaOptimizerBase): if not grad_vars: return - gm_block._insert_op(first_optimize_op_idx, - type="c_sync_calc_stream", - inputs={'X': grad_vars[0]}, - outputs={'Out': grad_vars[0]}, - attrs={OP_ROLE_KEY: OpRole.Backward}) + gm_block._insert_op( + first_optimize_op_idx, + type="c_sync_calc_stream", + inputs={'X': grad_vars[0]}, + outputs={'Out': grad_vars[0]}, + attrs={OP_ROLE_KEY: OpRole.Backward}, + ) insert_op_num = 1 ring_id = self.global_ring_id # NOTE: can perform fuse allreduce inside the loop in the future for i, (p, g) in enumerate(zip(param_vars, grad_vars)): - gm_block._insert_op(first_optimize_op_idx + insert_op_num, - type="c_allreduce_sum", - inputs={'X': g}, - outputs={'Out': g}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward, - }) + gm_block._insert_op( + first_optimize_op_idx + insert_op_num, + type="c_allreduce_sum", + inputs={'X': g}, + outputs={'Out': g}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward, + }, + ) insert_op_num += 1 - gm_block._insert_op(first_optimize_op_idx + insert_op_num, - type="c_sync_comm_stream", - inputs={'X': grad_vars}, - outputs={'Out': grad_vars}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward, - }) + gm_block._insert_op( + first_optimize_op_idx + insert_op_num, + type="c_sync_comm_stream", + inputs={'X': grad_vars}, + outputs={'Out': grad_vars}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward, + }, + ) def _transpile_main_program(self, loss): self._insert_loss_grad_ops(loss) @@ -240,14 +266,16 @@ class RawProgramOptimizer(MetaOptimizerBase): for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): loss_grad_var = block.vars[op.output_arg_names[0]] - block._insert_op(idx + 1, - type='scale', - inputs={'X': loss_grad_var}, - outputs={'Out': loss_grad_var}, - attrs={ - 'scale': 1.0 / self.nranks, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / self.nranks, + OP_ROLE_KEY: OpRole.Backward, + }, + ) def _insert_allreduce_ops(self): block = self.main_program.global_block() @@ -255,8 +283,7 @@ class RawProgramOptimizer(MetaOptimizerBase): grad = None grad_vars = [] for idx, op in reversed(list(enumerate(block.ops))): - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.attr(OP_ROLE_VAR_KEY) if len(op_role_var) == 0: continue @@ -271,36 +298,39 @@ class RawProgramOptimizer(MetaOptimizerBase): continue grad_vars.append(grad) - block._insert_op(idx + offset, - type='c_sync_calc_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - OP_ROLE_KEY: OpRole.Backward, - }) + block._insert_op( + idx + offset, + type='c_sync_calc_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + OP_ROLE_KEY: OpRole.Backward, + }, + ) offset += 1 - block._insert_op(idx + offset, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward, + }, + ) if grad is None: return for idx, op in enumerate(block.ops): if is_optimizer_op(op): - block._insert_op(idx, - type='c_sync_comm_stream', - inputs={'X': grad_vars}, - outputs={'Out': grad_vars}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx, + type='c_sync_comm_stream', + inputs={'X': grad_vars}, + outputs={'Out': grad_vars}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Backward}, + ) break # This function helps reduce the number of allreduce by integrating op, which can save communication time. @@ -318,16 +348,16 @@ class RawProgramOptimizer(MetaOptimizerBase): # find all grad params for idx, op in enumerate(block.ops): - if first_backward_idx == -1 and \ - is_backward_op(op): + if first_backward_idx == -1 and is_backward_op(op): first_backward_idx = idx - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.attr(OP_ROLE_VAR_KEY) if len(op_role_var) == 0: continue - assert len(op_role_var) % 2 == 0, "vars need to be one param var followed by one grad var, " \ - "but got odd number of vars" + assert len(op_role_var) % 2 == 0, ( + "vars need to be one param var followed by one grad var, " + "but got odd number of vars" + ) for i in range(0, len(op_role_var), 2): param_name = op_role_var[i] param = block.var(param_name) @@ -338,7 +368,8 @@ class RawProgramOptimizer(MetaOptimizerBase): param_grads.append((param, grad)) outputs_name_to_idx = self.__get_ouputs_name_to_idx( - first_backward_idx, block) + first_backward_idx, block + ) # structure of grad_param_segments is # [([grad0, grad1], [param0, param1]), ([grad2, grad3], [param2, param3])] @@ -348,9 +379,11 @@ class RawProgramOptimizer(MetaOptimizerBase): last_dtype = None # split the grad based on dtype and fused size for param, grad in param_grads: - if len(grad_param_segments) == 0 \ - or len(grad_param_segments[-1][0]) == self.fuse_grad_size_in_num \ - or grad.dtype != last_dtype: + if ( + len(grad_param_segments) == 0 + or len(grad_param_segments[-1][0]) == self.fuse_grad_size_in_num + or grad.dtype != last_dtype + ): grad_param_segments.append(([grad], [param])) last_dtype = grad.dtype else: @@ -366,34 +399,40 @@ class RawProgramOptimizer(MetaOptimizerBase): # not to use reversed since needs the absolute index value grad_segment, param_segment = grad_param_segments[i] # insert coalesce tensor - fused_var = block.create_var(name=unique_name.generate( - 'FusedOutput_{}'.format(grad_segment[0].name)), - dtype=grad_segment[0].dtype, - persistable=False, - stop_gradient=True) + fused_var = block.create_var( + name=unique_name.generate( + 'FusedOutput_{}'.format(grad_segment[0].name) + ), + dtype=grad_segment[0].dtype, + persistable=False, + stop_gradient=True, + ) fused_vars[i] = fused_var after_idx = outputs_name_to_idx[grad_segment[-1]][1] - block._insert_op_without_sync(after_idx + 1, - type='c_allreduce_sum', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': - self.calc_comm_same_stream, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op_without_sync( + after_idx + 1, + type='c_allreduce_sum', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': self.calc_comm_same_stream, + OP_ROLE_KEY: OpRole.Backward, + }, + ) if not self.calc_comm_same_stream: block._insert_op_without_sync( after_idx + 1, type='c_sync_calc_stream', inputs={'X': fused_var}, outputs={'Out': fused_var}, - attrs={OP_ROLE_KEY: OpRole.Backward}) + attrs={OP_ROLE_KEY: OpRole.Backward}, + ) # update the outputs_name_to_idx after insertion of sync/allreduce ops outputs_name_to_idx = self.__get_ouputs_name_to_idx( - first_backward_idx, block) + first_backward_idx, block + ) # the before_idx is not guaranteed sorted, therefore we have to find the # topology to insert the coalesce ops pos_for_coalesce = {} @@ -407,25 +446,26 @@ class RawProgramOptimizer(MetaOptimizerBase): pos_for_coalesce[i] = before_idx # insert the coalesce op based on the sorted before_idx - pos_for_coalesce = sorted(pos_for_coalesce.items(), - key=lambda kv: (kv[1], kv[0]), - reverse=True) + pos_for_coalesce = sorted( + pos_for_coalesce.items(), + key=lambda kv: (kv[1], kv[0]), + reverse=True, + ) for i, before_idx in pos_for_coalesce: grad_segment, param_segment = grad_param_segments[i] fused_var = fused_vars[i] - block._insert_op_without_sync(before_idx, - type="coalesce_tensor", - inputs={"Input": param_segment}, - outputs={ - "Output": grad_segment, - "FusedOutput": fused_var - }, - attrs={ - "copy_data": False, - "use_align": True, - "dtype": grad_segment[0].dtype, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op_without_sync( + before_idx, + type="coalesce_tensor", + inputs={"Input": param_segment}, + outputs={"Output": grad_segment, "FusedOutput": fused_var}, + attrs={ + "copy_data": False, + "use_align": True, + "dtype": grad_segment[0].dtype, + OP_ROLE_KEY: OpRole.Backward, + }, + ) if self.calc_comm_same_stream: block._sync_with_cpp() @@ -434,14 +474,13 @@ class RawProgramOptimizer(MetaOptimizerBase): # insert the sync comm op for idx, op in enumerate(block.ops): if is_optimizer_op(op): - block._insert_op_without_sync(idx, - type='c_sync_comm_stream', - inputs={'X': fused_vars}, - outputs={'Out': fused_vars}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op_without_sync( + idx, + type='c_sync_comm_stream', + inputs={'X': fused_vars}, + outputs={'Out': fused_vars}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Backward}, + ) break block._sync_with_cpp() @@ -465,6 +504,8 @@ class RawProgramOptimizer(MetaOptimizerBase): # the first idx and the last ids are identical outputs_name_to_idx[var] = (idx, idx) else: - outputs_name_to_idx[var] = (outputs_name_to_idx[var][0], - idx) + outputs_name_to_idx[var] = ( + outputs_name_to_idx[var][0], + idx, + ) return outputs_name_to_idx diff --git a/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py index c9054c793f4..524c3a123ab 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py @@ -18,7 +18,6 @@ __all__ = [] class RecomputeOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(RecomputeOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -32,11 +31,12 @@ class RecomputeOptimizer(MetaOptimizerBase): ] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(RecomputeOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(RecomputeOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _init_wrapped_opt(self): if self.wrapped_opt is not None: @@ -56,8 +56,10 @@ class RecomputeOptimizer(MetaOptimizerBase): return False if self.user_defined_strategy.recompute == True: - if len(self.user_defined_strategy.recompute_configs["checkpoints"] - ) == 0: + if ( + len(self.user_defined_strategy.recompute_configs["checkpoints"]) + == 0 + ): return False else: return True @@ -70,32 +72,33 @@ class RecomputeOptimizer(MetaOptimizerBase): # we do not support automatically recompute checkpoints currently return - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): # maybe inner_opt of other meta optimizer self._init_wrapped_opt() - return self.wrapped_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + return self.wrapped_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): return self.wrapped_opt.apply_gradients(params_grads=params_grads) def apply_optimize(self, loss, startup_program, params_grads): - return self.wrapped_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + return self.wrapped_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self._init_wrapped_opt() - optimize_ops, params_grads = \ - self.wrapped_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + optimize_ops, params_grads = self.wrapped_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py index 126c7d1ca04..e4db252cf7c 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.distributed.fleet.meta_optimizers.common import is_optimizer_op, OP_ROLE_KEY, OpRole +from paddle.distributed.fleet.meta_optimizers.common import ( + is_optimizer_op, + OP_ROLE_KEY, + OpRole, +) from paddle.fluid import core @@ -20,7 +24,6 @@ __all__ = [] class FP16Utils(object): - def __init__(self): pass @@ -30,16 +33,20 @@ class FP16Utils(object): return False if is_optimizer_op(op): return False - assert (len(op.desc.input_arg_names()) == 1) - assert (len(op.desc.output_arg_names()) == 1) - input_name, output_name = op.desc.input_arg_names( - )[0], op.desc.output_arg_names()[0] + assert len(op.desc.input_arg_names()) == 1 + assert len(op.desc.output_arg_names()) == 1 + input_name, output_name = ( + op.desc.input_arg_names()[0], + op.desc.output_arg_names()[0], + ) if input_name not in params: return False input_var = block.var(input_name) output_var = block.var(output_name) - if input_var.dtype != core.VarDesc.VarType.FP32 or \ - output_var.dtype != core.VarDesc.VarType.FP16: + if ( + input_var.dtype != core.VarDesc.VarType.FP32 + or output_var.dtype != core.VarDesc.VarType.FP16 + ): return False return True @@ -49,14 +56,18 @@ class FP16Utils(object): return False if not is_optimizer_op(op): return False - assert (len(op.desc.input_arg_names()) == 1) - assert (len(op.desc.output_arg_names()) == 1) - input_name, output_name = op.desc.input_arg_names( - )[0], op.desc.output_arg_names()[0] + assert len(op.desc.input_arg_names()) == 1 + assert len(op.desc.output_arg_names()) == 1 + input_name, output_name = ( + op.desc.input_arg_names()[0], + op.desc.output_arg_names()[0], + ) input_var = block.var(input_name) output_var = block.var(output_name) - if input_var.dtype != core.VarDesc.VarType.FP16 or \ - output_var.dtype != core.VarDesc.VarType.FP32: + if ( + input_var.dtype != core.VarDesc.VarType.FP16 + or output_var.dtype != core.VarDesc.VarType.FP32 + ): return False return True @@ -64,7 +75,8 @@ class FP16Utils(object): def remove_cast_op(block, params, segment, offset): inserted_op_num = 0 for op_idx in reversed( - range(offset + segment._start_idx, offset + segment._end_idx)): + range(offset + segment._start_idx, offset + segment._end_idx) + ): op = block.ops[op_idx] if FP16Utils.is_fp16_cast_op(block, op, params): block._remove_op(op_idx, sync=False) @@ -84,13 +96,16 @@ class FP16Utils(object): continue output_name = op.desc.output_arg_names()[0] # TODO (JZ-LIANG) revise this for uniform mixed parallelism - param_name = output_name.strip( - "@GRAD@MERGED" - ) if "@MERGED" in output_name else output_name.strip("@GRAD") + param_name = ( + output_name.strip("@GRAD@MERGED") + if "@MERGED" in output_name + else output_name.strip("@GRAD") + ) if param_name not in shard.global_params: raise ValueError( "Output 'X' of cast_op must be a grad of" - "model param, but {} is not a grad".format(output_name)) + "model param, but {} is not a grad".format(output_name) + ) if output_name in reduced_grads_to_param: continue if shard.has_param(param_name): @@ -117,7 +132,8 @@ class FP16Utils(object): if param_name not in shard.global_params: raise ValueError( "Input 'X' of check_finite_and_unscale must" - "be grads, but {} is not a grad".format(input_name)) + "be grads, but {} is not a grad".format(input_name) + ) if shard.has_param(param_name): reversed_x.append(input_name) reversed_x_paramname.append(param_name) @@ -127,55 +143,73 @@ class FP16Utils(object): # the grad checking should take the all and only param in the current shard to_check_param = set(reversed_x_paramname) should_check_param = set(shard.global_params).intersection( - set([param for param, worker_idx in shard.global_param2device.items() \ - if worker_idx == shard.worker_idx])) - assert to_check_param == should_check_param, "amp \ + set( + [ + param + for param, worker_idx in shard.global_param2device.items() + if worker_idx == shard.worker_idx + ] + ) + ) + assert ( + to_check_param == should_check_param + ), "amp \ check_finite_and_unscale checking miss [{}] and got unexpected [{}]".format( should_check_param - to_check_param, - to_check_param - should_check_param) + to_check_param - should_check_param, + ) if update_loss_scaling_op_idx == -1: return inf_var = block.var(inf_var_name) - inf_var_int32 = block.create_var(name=inf_var_name + "@cast_int32", - shape=inf_var.shape, - dtype=core.VarDesc.VarType.INT32) - - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='cast', - inputs={'X': inf_var}, - outputs={'Out': inf_var_int32}, - attrs={ - "in_dtype": inf_var.dtype, - "out_dtype": inf_var_int32.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + inf_var_int32 = block.create_var( + name=inf_var_name + "@cast_int32", + shape=inf_var.shape, + dtype=core.VarDesc.VarType.INT32, + ) + + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='cast', + inputs={'X': inf_var}, + outputs={'Out': inf_var_int32}, + attrs={ + "in_dtype": inf_var.dtype, + "out_dtype": inf_var_int32.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 # allreduce(mp)->allreduce(sharding)->allreduce(pp) for ring_id in ring_ids: - if ring_id == -1: continue + if ring_id == -1: + continue # this allreduce communication should not overlap with calc - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='c_allreduce_max', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var_int32}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='c_allreduce_max', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var_int32}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='cast', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var}, - attrs={ - "in_dtype": inf_var_int32.dtype, - "out_dtype": inf_var.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='cast', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var}, + attrs={ + "in_dtype": inf_var_int32.dtype, + "out_dtype": inf_var.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 block._sync_with_cpp() @@ -197,42 +231,51 @@ class FP16Utils(object): # 1. inf_var_int32 = allreduce_max(inf_var_int32) # 3. inf_var = cast(inf_var_int32) inf_var = block.var(inf_var_name) - inf_var_int32 = block.create_var(name=inf_var_name + "@cast_int32", - shape=inf_var.shape, - dtype=core.VarDesc.VarType.INT32) - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='cast', - inputs={'X': inf_var}, - outputs={'Out': inf_var_int32}, - attrs={ - "in_dtype": inf_var.dtype, - "out_dtype": inf_var_int32.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + inf_var_int32 = block.create_var( + name=inf_var_name + "@cast_int32", + shape=inf_var.shape, + dtype=core.VarDesc.VarType.INT32, + ) + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='cast', + inputs={'X': inf_var}, + outputs={'Out': inf_var_int32}, + attrs={ + "in_dtype": inf_var.dtype, + "out_dtype": inf_var_int32.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 # allreduce(mp)->allreduce(pp) for ring_id in ring_ids: - if ring_id == -1: continue - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='c_allreduce_max', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var_int32}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) + if ring_id == -1: + continue + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='c_allreduce_max', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var_int32}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='cast', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var}, - attrs={ - "in_dtype": inf_var_int32.dtype, - "out_dtype": inf_var.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='cast', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var}, + attrs={ + "in_dtype": inf_var_int32.dtype, + "out_dtype": inf_var.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 block._sync_with_cpp() diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py index 03d955842f5..563757d35f4 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py @@ -18,13 +18,13 @@ __all__ = [] class GradientClipHelper(object): - def __init__(self, mp_ring_id): self.mp_ring_id = mp_ring_id def _is_gradient_clip_op(self, op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip") def prune_gradient_clip(self, block, shard, ring_ids): """ @@ -51,8 +51,9 @@ class GradientClipHelper(object): param_name = input_name.strip("@GRAD@MERGED") else: param_name = input_name.strip("@GRAD") - if shard.is_param(param_name) and \ - not shard.has_param(param_name): + if shard.is_param(param_name) and not shard.has_param( + param_name + ): deperate_op = True elif shard.is_param(param_name): reversed_x_paramname.append(param_name) @@ -84,7 +85,7 @@ class GradientClipHelper(object): reversed_inputs.append(input_name) op.desc.set_input("X", reversed_inputs) - assert (len(op.desc.output_arg_names()) == 1) + assert len(op.desc.output_arg_names()) == 1 sum_res = op.desc.output_arg_names()[0] # NOTE(wangxi): If we have 2 param, but sharding is 4, @@ -96,26 +97,25 @@ class GradientClipHelper(object): namescope = op.attr("op_namescope") block._remove_op(idx, sync=False) - op = block._insert_op_without_sync(idx, - type='fill_constant', - inputs={}, - outputs={'Out': sum_res}, - attrs={ - 'shape': - sum_var.shape, - 'dtype': - sum_var.dtype, - 'value': - 0.0, - OP_ROLE_KEY: - OpRole.Optimize - }) + op = block._insert_op_without_sync( + idx, + type='fill_constant', + inputs={}, + outputs={'Out': sum_res}, + attrs={ + 'shape': sum_var.shape, + 'dtype': sum_var.dtype, + 'value': 0.0, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) op._set_attr('op_namescope', namescope) # allreduce(mp)->allreduce(sharding)->allreduce(pp) idx_offset = 1 for ring_id in ring_ids: - if ring_id == -1: continue + if ring_id == -1: + continue # this allreduce should not overlap with calc and should be scheduled in calc stream block._insert_op_without_sync( idx + idx_offset, @@ -127,18 +127,28 @@ class GradientClipHelper(object): 'op_namescope': "/gradient_clip_model_parallelism", 'use_calc_stream': True, OP_ROLE_KEY: OpRole.Optimize, - }) + }, + ) idx_offset += 1 # the grad sum here should take the all and only param in the current shard to_check_param = set(reversed_x_paramname) - should_check_param = set(shard.global_params).intersection(set( - [param for param, worker_idx in shard.global_param2device.items() \ - if worker_idx == shard.worker_idx])) - assert to_check_param == should_check_param, "amp check_finite_and_unscale \ + should_check_param = set(shard.global_params).intersection( + set( + [ + param + for param, worker_idx in shard.global_param2device.items() + if worker_idx == shard.worker_idx + ] + ) + ) + assert ( + to_check_param == should_check_param + ), "amp check_finite_and_unscale \ checking miss [{}] and got unexpected [{}]".format( should_check_param - to_check_param, - to_check_param - should_check_param) + to_check_param - should_check_param, + ) for var_name in deperated_vars: block._remove_var(var_name, sync=False) @@ -178,8 +188,12 @@ class GradientClipHelper(object): # by global norm. Those vars either doesn't have is_distributed attr # or the is_distributed attr has been set as False. # Therefore, we prune those duplicated vars for grad clip. - if mp_rank >= 1 and (not (hasattr(input_var, 'is_distributed') - and input_var.is_distributed)): + if mp_rank >= 1 and ( + not ( + hasattr(input_var, 'is_distributed') + and input_var.is_distributed + ) + ): removed_op_idx.add(idx) for output_name in op.output_arg_names: removed_tmp_var.add(output_name) @@ -222,8 +236,9 @@ class GradientClipHelper(object): 'shape': sum_rst_var.shape, 'dtype': sum_rst_var.dtype, 'value': 0.0, - OP_ROLE_KEY: OpRole.Optimize - }) + OP_ROLE_KEY: OpRole.Optimize, + }, + ) fill_constant_op._set_attr('op_namescope', namescope) self._insert_allreduce(block, ring_ids, idx, sum_rst_var) break @@ -245,4 +260,5 @@ class GradientClipHelper(object): 'op_namescope': "/gradient_clip_model_parallelism", 'use_calc_stream': True, OP_ROLE_KEY: OpRole.Optimize, - }) + }, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py index 968709717d1..ac10bb42383 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py @@ -56,21 +56,25 @@ class OffloadHelper(object): def _insert_cast_op(self, block, idx, src_name, dst_name): src_var = block.var(src_name) if not block.has_var(dst_name): - block.create_var(name=dst_name, - shape=src_var.shape, - dtype=core.VarDesc.VarType.FP16, - persistable=True) + block.create_var( + name=dst_name, + shape=src_var.shape, + dtype=core.VarDesc.VarType.FP16, + persistable=True, + ) dst_var = block.var(dst_name) assert dst_var.dtype == core.VarDesc.VarType.FP16 - block._insert_op_without_sync(idx, - type='cast', - inputs={'X': src_var}, - outputs={'Out': dst_var}, - attrs={ - 'in_dtype': src_var.dtype, - 'out_dtype': dst_var.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op_without_sync( + idx, + type='cast', + inputs={'X': src_var}, + outputs={'Out': dst_var}, + attrs={ + 'in_dtype': src_var.dtype, + 'out_dtype': dst_var.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) def _insert_broadcast_op(self, block, idx, param_name): rings = [] @@ -86,36 +90,42 @@ class OffloadHelper(object): # the insert op order is: mp, dp for ring in rings: - block._insert_op_without_sync(idx, - type="c_broadcast", - inputs={'X': param_name}, - outputs={'Out': param_name}, - attrs={ - 'ring_id': ring, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward, - }) + block._insert_op_without_sync( + idx, + type="c_broadcast", + inputs={'X': param_name}, + outputs={'Out': param_name}, + attrs={ + 'ring_id': ring, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) def _insert_memcpy_op(self, block, idx, src_name, dst_name, dst_place_type): src_var = block.var(src_name) dst_var = block.var(dst_name) - block._insert_op_without_sync(idx, - type='memcpy', - inputs={'X': src_var}, - outputs={'Out': dst_var}, - attrs={ - 'dst_place_type': dst_place_type, - OP_ROLE_KEY: OpRole.Optimize, - }) + block._insert_op_without_sync( + idx, + type='memcpy', + inputs={'X': src_var}, + outputs={'Out': dst_var}, + attrs={ + 'dst_place_type': dst_place_type, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) def _insert_fetch_op(self, block, idx, src_name, dst_name): - self._insert_memcpy_op(block, idx, src_name, dst_name, - OffloadHelper.cuda_place_type) + self._insert_memcpy_op( + block, idx, src_name, dst_name, OffloadHelper.cuda_place_type + ) def _insert_offload_op(self, block, idx, src_name, dst_name): - self._insert_memcpy_op(block, idx, src_name, dst_name, - OffloadHelper.cuda_pinned_place_type) + self._insert_memcpy_op( + block, idx, src_name, dst_name, OffloadHelper.cuda_pinned_place_type + ) def _get_offload_var_name(self, name): return unique_name.generate(name + '@offload') @@ -124,10 +134,12 @@ class OffloadHelper(object): for block in blocks: var = block.var(var_name) var.persistable = False - offload_var = block.create_var(name=offload_var_name, - shape=var.shape, - dtype=var.dtype, - persistable=True) + offload_var = block.create_var( + name=offload_var_name, + shape=var.shape, + dtype=var.dtype, + persistable=True, + ) def offload_fp32param(self, block, startup_block, offload=True): """ @@ -188,13 +200,15 @@ class OffloadHelper(object): if 'subprog' not in output_name: assert output_name == input_name + '.cast_fp16' - assert input_name not in param_to_fp16, \ - "There must be only one cast op from fp32 param to fp16 param." + assert ( + input_name not in param_to_fp16 + ), "There must be only one cast op from fp32 param to fp16 param." param_to_fp16[input_name] = output_name else: # fp16-->recompute_var - assert input_name in param_to_fp16, \ - "param must first be cast to fp16" + assert ( + input_name in param_to_fp16 + ), "param must first be cast to fp16" fp16_param = param_to_fp16[input_name] fp16_param_to_recompute[fp16_param] = output_name recompute_to_fp16[output_name] = fp16_param @@ -205,24 +219,28 @@ class OffloadHelper(object): for idx, op in reversed(list(enumerate(block.ops))): if is_update_op(op): param = op.desc.input("Param")[0] - if param not in param_to_idx: continue + if param not in param_to_idx: + continue # step3.1: create offload_var offload_var_name = self._get_offload_var_name(param) param_name_to_offload_name[param] = offload_var_name if offload: - self._create_offload_var(param, offload_var_name, - [block, startup_block]) + self._create_offload_var( + param, offload_var_name, [block, startup_block] + ) # step3.2: insert cast op and offload op - self._insert_offload_op(block, idx + 1, param, - offload_var_name) + self._insert_offload_op( + block, idx + 1, param, offload_var_name + ) assert param in param_to_fp16 fp16_param_name = param_to_fp16[param] fp16_param_var = block.var(fp16_param_name) fp16_param_var.persistable = True - self._insert_cast_op(block, idx + 1, param, - param_to_fp16[param]) + self._insert_cast_op( + block, idx + 1, param, param_to_fp16[param] + ) if offload: # step3.3: insert fetch op @@ -242,8 +260,9 @@ class OffloadHelper(object): op._rename_input(input_name, recompute_to_fp16[input_name]) for output_name in op.desc.output_arg_names(): if output_name in recompute_to_fp16: - op._rename_output(output_name, - recompute_to_fp16[output_name]) + op._rename_output( + output_name, recompute_to_fp16[output_name] + ) # step4: remove recompute_param for name in recompute_to_fp16.keys(): @@ -262,14 +281,23 @@ class OffloadHelper(object): var_name = out_name if offload: offload_var_name = param_name_to_offload_name[var_name] - self._insert_offload_op(startup_block, insert_idx, - var_name, offload_var_name) - self._insert_cast_op(startup_block, insert_idx, var_name, - param_to_fp16[var_name]) + self._insert_offload_op( + startup_block, + insert_idx, + var_name, + offload_var_name, + ) + self._insert_cast_op( + startup_block, + insert_idx, + var_name, + param_to_fp16[var_name], + ) # NOTE(wangxi): cast and offload should insert after broadcast param. # the insert op order is: {mp, dp}broadcast, cast, offload - self._insert_broadcast_op(startup_block, insert_idx, - var_name) + self._insert_broadcast_op( + startup_block, insert_idx, var_name + ) visited_vars.add(out_name) @@ -322,14 +350,16 @@ class OffloadHelper(object): offload_var_name = self._get_offload_var_name(var_name) vars_name_to_offload_name[var_name] = offload_var_name - self._create_offload_var(var_name, offload_var_name, - [block, startup_block]) + self._create_offload_var( + var_name, offload_var_name, [block, startup_block] + ) # step2: insert offload op for var_name in vars_name: offload_var_name = vars_name_to_offload_name[var_name] - self._insert_offload_op(block, idx + 1, var_name, - offload_var_name) + self._insert_offload_op( + block, idx + 1, var_name, offload_var_name + ) # step3: insert fetch op for var_name in vars_name: @@ -347,18 +377,17 @@ class OffloadHelper(object): var_name = out_name offload_var_name = vars_name_to_offload_name[var_name] # insert offload op after var is generated - self._insert_offload_op(startup_block, idx + 1, var_name, - offload_var_name) + self._insert_offload_op( + startup_block, idx + 1, var_name, offload_var_name + ) visited_vars.add(out_name) block._sync_with_cpp() startup_block._sync_with_cpp() - def opt_sharding_cast_fp32param(self, - block, - startup_block, - params, - offload=False): + def opt_sharding_cast_fp32param( + self, block, startup_block, params, offload=False + ): """ (p_fp16) = cast(p) (p_fp16_recompute) = cast(p) @@ -420,13 +449,15 @@ class OffloadHelper(object): if 'subprog' not in output_name: assert output_name == input_name + '.cast_fp16' - assert input_name not in param_to_fp16, \ - "There must be only one cast op from fp32 param to fp16 param." + assert ( + input_name not in param_to_fp16 + ), "There must be only one cast op from fp32 param to fp16 param." param_to_fp16[input_name] = output_name else: # fp16-->recompute_var - assert input_name in param_to_fp16, \ - "param must first be cast to fp16" + assert ( + input_name in param_to_fp16 + ), "param must first be cast to fp16" fp16_param = param_to_fp16[input_name] fp16_param_to_recompute[fp16_param] = output_name recompute_to_fp16[output_name] = fp16_param @@ -443,19 +474,22 @@ class OffloadHelper(object): offload_var_name = self._get_offload_var_name(param) param_name_to_offload_name[param] = offload_var_name if offload: - self._create_offload_var(param, offload_var_name, - [block, startup_block]) + self._create_offload_var( + param, offload_var_name, [block, startup_block] + ) # step3.2: insert cast op and offload op - self._insert_offload_op(block, idx + 1, param, - offload_var_name) + self._insert_offload_op( + block, idx + 1, param, offload_var_name + ) assert param in param_to_fp16 fp16_param_name = param_to_fp16[param] fp16_param_var = block.var(fp16_param_name) fp16_param_var.persistable = True - self._insert_cast_op(block, idx + 1, param, - param_to_fp16[param]) + self._insert_cast_op( + block, idx + 1, param, param_to_fp16[param] + ) if offload: # step3.3: insert fetch op @@ -476,8 +510,9 @@ class OffloadHelper(object): op._rename_input(input_name, recompute_to_fp16[input_name]) for output_name in op.desc.output_arg_names(): if output_name in recompute_to_fp16: - op._rename_output(output_name, - recompute_to_fp16[output_name]) + op._rename_output( + output_name, recompute_to_fp16[output_name] + ) # step4: remove recompute_param for name in recompute_to_fp16.keys(): @@ -508,22 +543,31 @@ class OffloadHelper(object): insert_idx = len(startup_block.ops) for idx, op in reversed(list(enumerate(startup_block.ops))): for out_name in op.output_arg_names: - if out_name in visited_vars: continue + if out_name in visited_vars: + continue if out_name in param_to_fp16: var_name = out_name if offload: self._insert_offload_op( - startup_block, idx + 1, var_name, - param_name_to_offload_name[var_name]) - - self._insert_cast_op(startup_block, insert_idx, var_name, - param_to_fp16[var_name]) + startup_block, + idx + 1, + var_name, + param_name_to_offload_name[var_name], + ) + + self._insert_cast_op( + startup_block, + insert_idx, + var_name, + param_to_fp16[var_name], + ) # NOTE(wangxi): cast and offload should insert after broadcast param. # the insert op order is: {mp, dp}broadcast, cast, offload - self._insert_broadcast_op(startup_block, insert_idx, - var_name) + self._insert_broadcast_op( + startup_block, insert_idx, var_name + ) if var_name not in local_params: param = startup_block.var(out_name) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py index adbc00f25de..895fd2f7acb 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py @@ -16,7 +16,6 @@ __all__ = [] class ProgramDeps(object): - def __init__(self, block, start_vars, end_vars): self._block = block # vars where to start to build the deps @@ -45,7 +44,9 @@ class ProgramDeps(object): else: return None - def _build_deps(self, ): + def _build_deps( + self, + ): for var_name in self._start_vars: self._var_to_use_op[var_name] = [] @@ -53,8 +54,9 @@ class ProgramDeps(object): for idx, op in enumerate(self._block.ops): if op.type in [ - "c_allreduce_sum", "c_sync_comm_stream", - "c_calc_comm_stream" + "c_allreduce_sum", + "c_sync_comm_stream", + "c_calc_comm_stream", ]: continue input_vars = op.desc.input_arg_names() @@ -77,11 +79,13 @@ class ProgramDeps(object): self._var_to_generate_op[output_name].append(idx) if op.type == "conditional_block": # subblock - assert (op.desc.has_attr("sub_block")) + assert op.desc.has_attr("sub_block") subblock_idx = op.desc.attr("sub_block").id subblock_deps = ProgramDeps( self._block.program.block(subblock_idx), - op.desc.input_arg_names(), op.desc.output_arg_names()) + op.desc.input_arg_names(), + op.desc.output_arg_names(), + ) self._sub_block_deps[subblock_idx] = subblock_deps subblock_deps._father_block_deps = self @@ -93,17 +97,24 @@ class ProgramDeps(object): raise ValueError( "op_idx: {} is not in self._var_to_use_op[{}], " "self._var_to_use_op[{}] is {}".format( - op_idx, var_name, var_name, - self._var_to_use_op[var_name])) + op_idx, + var_name, + var_name, + self._var_to_use_op[var_name], + ) + ) self._var_to_use_op[var_name].remove(op_idx) # update _should_removed_var if var_name in self._start_vars: self._should_removed_var.discard(var_name) - elif self._var_to_use_op[ - var_name] == []: # no more deps of this var + elif ( + self._var_to_use_op[var_name] == [] + ): # no more deps of this var self._should_removed_var.add(var_name) - elif self._var_to_generate_op[var_name][-1] >= self._var_to_use_op[ - var_name][-1]: + elif ( + self._var_to_generate_op[var_name][-1] + >= self._var_to_use_op[var_name][-1] + ): # there are circle in the graph self._should_removed_var.add(var_name) else: # input_name should not be deleted @@ -111,11 +122,13 @@ class ProgramDeps(object): def crop_output_var_from_op(self, op_idx, var_name): if var_name in self._var_to_generate_op: - assert (op_idx in self._var_to_generate_op[var_name]) + assert op_idx in self._var_to_generate_op[var_name] self._var_to_generate_op[var_name].remove(op_idx) if self._block.has_var(var_name): - if var_name not in self._var_to_generate_op or self._var_to_generate_op[ - var_name] == []: + if ( + var_name not in self._var_to_generate_op + or self._var_to_generate_op[var_name] == [] + ): self._block._remove_var(var_name, sync=False) def remove_op(self, op_idx, reserved_vars=None): diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py index e6490d62a5d..d33d04098d0 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py @@ -15,14 +15,17 @@ import re from paddle.distributed.fleet.meta_optimizers.common import is_optimizer_op from paddle.distributed.fleet.meta_optimizers.sharding.utils import get_var_size -from paddle.distributed.fleet.meta_optimizers.sharding.fp16_helper import FP16Utils +from paddle.distributed.fleet.meta_optimizers.sharding.fp16_helper import ( + FP16Utils, +) __all__ = [] class Shard(object): - - def __init__(self, ): + def __init__( + self, + ): self.global_params = set([]) self.worker_idx = -1 self.worker_num = -1 @@ -37,19 +40,25 @@ class Shard(object): self.worker_num = worker_num # global_param2device contains fp32 params and fp16 params # device2global_params only contains fp32 params - self.global_param2device, self.device2global_params \ - = self._split_params(params_grads, worker_idx, worker_num) + ( + self.global_param2device, + self.device2global_params, + ) = self._split_params(params_grads, worker_idx, worker_num) def has_param(self, var_name): - return var_name in self.global_param2device and \ - self._var_device_id(var_name) == self.worker_idx + return ( + var_name in self.global_param2device + and self._var_device_id(var_name) == self.worker_idx + ) def has_opt_var(self, var_name): return self._var_device_id(var_name) == self.worker_idx def has_var(self, var_name): - return self._var_device_id(var_name) == -1 or \ - self._var_device_id(var_name) == self.worker_idx + return ( + self._var_device_id(var_name) == -1 + or self._var_device_id(var_name) == self.worker_idx + ) def _split_params(self, params_grads, worker_idx, worker_num): param2device = {} @@ -74,8 +83,11 @@ class Shard(object): if var_name in self.global_param2device: return self.global_param2device[var_name] for suffix in [ - "_moment1_0", "_moment2_0", "_beta1_pow_acc_0", - "_beta2_pow_acc_0", "_velocity_0" + "_moment1_0", + "_moment2_0", + "_beta1_pow_acc_0", + "_beta2_pow_acc_0", + "_velocity_0", ]: base_name = re.sub(suffix, '', var_name) if base_name in self.global_param2device: @@ -105,7 +117,8 @@ class Shard(object): fp16_to_fp32[output_name] = input_name param_usage[input_name] -= 1 self.global_param2device[output_name] = self.global_param2device[ - input_name] + input_name + ] for param, usage in param_usage.items(): if usage > 0: @@ -122,8 +135,11 @@ class Shard(object): if var_name in self.global_params: return True for suffix in [ - "_moment1_0", "_moment2_0", "_beta1_pow_acc_0", - "_beta2_pow_acc_0", "_velocity_0" + "_moment1_0", + "_moment2_0", + "_beta1_pow_acc_0", + "_beta2_pow_acc_0", + "_velocity_0", ]: base_name = re.sub(suffix, '', var_name) if base_name in self.global_params: @@ -140,7 +156,6 @@ class Shard(object): class ProgramSegment(object): - def __init__(self, block): self._block = block self._allreduce_vars = [] diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py index b5f72aac893..1ec9457854d 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py @@ -14,7 +14,11 @@ import paddle from paddle.fluid import core, unique_name from functools import reduce -from paddle.distributed.fleet.meta_optimizers.common import is_loss_grad_op, is_backward_op, is_optimizer_op +from paddle.distributed.fleet.meta_optimizers.common import ( + is_loss_grad_op, + is_backward_op, + is_optimizer_op, +) from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole import re @@ -42,7 +46,10 @@ def check_broadcast(block): "var_name areadly exist: {}" "the old pos is {}, the new pos is {}".format( var_name, - broadcast_vars[var_name]["broadcast_pos"], idx)) + broadcast_vars[var_name]["broadcast_pos"], + idx, + ) + ) broadcast_vars[var_name] = { "fill_constant_pos": -1, "broadcast_pos": idx, @@ -69,17 +76,21 @@ def check_broadcast(block): var_name = op.desc.input_arg_names()[0] if "@BroadCast" in var_name: if broadcast_vars[var_name]["fill_constant_pos"] != -1: - assert (last_sync_calc_op_idx != -1) - assert (broadcast_vars[var_name]["fill_constant_pos"] < - last_sync_calc_op_idx) - assert (last_sync_calc_op_idx < idx) + assert last_sync_calc_op_idx != -1 + assert ( + broadcast_vars[var_name]["fill_constant_pos"] + < last_sync_calc_op_idx + ) + assert last_sync_calc_op_idx < idx continue for input_name in op.desc.input_arg_names(): if input_name in broadcast_vars: - assert (broadcast_vars[input_name]["broadcast_pos"] != -1) - assert (broadcast_vars[input_name]["broadcast_pos"] < - last_sync_comm_op_idx) - assert (last_sync_comm_op_idx < idx) + assert broadcast_vars[input_name]["broadcast_pos"] != -1 + assert ( + broadcast_vars[input_name]["broadcast_pos"] + < last_sync_comm_op_idx + ) + assert last_sync_comm_op_idx < idx return @@ -135,8 +146,10 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): if var_name in vars_status and vars_status[var_name] == 0: vars_status[var_name] = 1 for var_name in dp_grads_status: - if var_name in dp_grads_status and dp_grads_status[ - var_name] == 0: + if ( + var_name in dp_grads_status + and dp_grads_status[var_name] == 0 + ): dp_grads_status[var_name] = 1 # check sharding allreduce and reduce but skip megatron allreduce elif op.type == "c_allreduce_sum" or op.type == "c_reduce_sum": @@ -144,7 +157,9 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): var_name = op.desc.input_arg_names()[0] ring_id = op.desc.attr("ring_id") if ring_id == sharding_ring_id: - assert op.type == "c_reduce_sum", "Grad in Sharding group should be reduce rather than allreduce" + assert ( + op.type == "c_reduce_sum" + ), "Grad in Sharding group should be reduce rather than allreduce" if var_name in vars_status: _status = vars_status[var_name] else: @@ -152,12 +167,15 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): if _status == -1: raise ValueError( "{} is not generated, but you are" - "trying to all-reduce it".format(var_name)) + "trying to all-reduce it".format(var_name) + ) if _status == 0: - raise ValueError("There should be a sync_calc op " - "after generate Var: {} and before the" - "c_allreduce_sum op".format(var_name)) - assert (_status == 1) + raise ValueError( + "There should be a sync_calc op " + "after generate Var: {} and before the" + "c_allreduce_sum op".format(var_name) + ) + assert _status == 1 if var_name in vars_status: vars_status[var_name] = 2 else: @@ -193,29 +211,37 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): if vars_status[input_name] != 3: raise ValueError( "There should be a sync_comm op " - "after allreduce the Var: {}".format(input_name)) + "after allreduce the Var: {}".format(input_name) + ) raise ValueError( - "The reduce output grad [{}] should NOT be be used in Non-root rank." - .format(input_name)) + "The reduce output grad [{}] should NOT be be used in Non-root rank.".format( + input_name + ) + ) if input_name in dp_grads_status: if dp_ring_id == -1: if dp_grads_status[input_name] != 3: raise ValueError( "There should be a sync_comm op " - "after allreduce the Var: {}".format( - input_name)) + "after allreduce the Var: {}".format(input_name) + ) else: if dp_grads_status[input_name] != 5: raise ValueError( "The grad in shard should be allreduce and sync" - "twice before usage {}".format(input_name)) + "twice before usage {}".format(input_name) + ) for output_name in op.desc.output_arg_names(): - if output_name in vars_status and \ - vars_status[output_name] == -1: + if ( + output_name in vars_status + and vars_status[output_name] == -1 + ): vars_status[output_name] = 0 - if output_name in dp_grads_status and \ - dp_grads_status[output_name] == -1: + if ( + output_name in dp_grads_status + and dp_grads_status[output_name] == -1 + ): dp_grads_status[output_name] = 0 # check sharding with amp @@ -234,9 +260,9 @@ def get_valid_op_role(block, insert_idx): return OpRole.Forward or OpRole.Backward """ op_role = block.ops[insert_idx].attr('op_role') - if (insert_idx >= len(block.ops)) or (op_role in [ - int(OpRole.Backward), int(OpRole.Optimize) - ]): + if (insert_idx >= len(block.ops)) or ( + op_role in [int(OpRole.Backward), int(OpRole.Optimize)] + ): return OpRole.Backward if op_role in [int(OpRole.Forward), int(OpRole.Loss)]: return OpRole.Forward @@ -249,11 +275,13 @@ def insert_sync_calc_op(block, insert_idx, calc_dep_vars): _insert_sync_calc_op """ op_role = get_valid_op_role(block, insert_idx) - block._insert_op_without_sync(insert_idx, - type='c_sync_calc_stream', - inputs={'X': calc_dep_vars}, - outputs={'Out': calc_dep_vars}, - attrs={OP_ROLE_KEY: op_role}) + block._insert_op_without_sync( + insert_idx, + type='c_sync_calc_stream', + inputs={'X': calc_dep_vars}, + outputs={'Out': calc_dep_vars}, + attrs={OP_ROLE_KEY: op_role}, + ) return @@ -262,14 +290,13 @@ def insert_sync_comm_op(block, insert_idx, ring_id, comm_dep_vars): insert sync_comm_op for single var """ op_role = get_valid_op_role(block, insert_idx) - block._insert_op_without_sync(insert_idx, - type='c_sync_comm_stream', - inputs={'X': comm_dep_vars}, - outputs={'Out': comm_dep_vars}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_sync_comm_stream', + inputs={'X': comm_dep_vars}, + outputs={'Out': comm_dep_vars}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: op_role}, + ) return 1 @@ -282,14 +309,13 @@ def insert_sync_comm_ops(block, insert_idx, ring_id, comm_dep_vars): return 0 op_role = get_valid_op_role(block, insert_idx) - block._insert_op_without_sync(insert_idx, - type='c_sync_comm_stream', - inputs={'X': comm_dep_vars}, - outputs={'Out': comm_dep_vars}, - attrs={ - 'ring_id': int(ring_id), - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_sync_comm_stream', + inputs={'X': comm_dep_vars}, + outputs={'Out': comm_dep_vars}, + attrs={'ring_id': int(ring_id), OP_ROLE_KEY: op_role}, + ) return 1 @@ -300,15 +326,17 @@ def insert_fill_constant_ops(block, insert_idx, fill_constant_vars): op_role = get_valid_op_role(block, insert_idx) for broadcast_name in fill_constant_vars: broadcast_var = block.var(broadcast_name) - block._insert_op_without_sync(insert_idx, - type="fill_constant", - outputs={"Out": broadcast_var.name}, - attrs={ - "shape": broadcast_var.shape, - "dtype": broadcast_var.dtype, - "value": 0.0, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type="fill_constant", + outputs={"Out": broadcast_var.name}, + attrs={ + "shape": broadcast_var.shape, + "dtype": broadcast_var.dtype, + "value": 0.0, + OP_ROLE_KEY: op_role, + }, + ) return @@ -318,58 +346,69 @@ def insert_cast_ops(block, insert_idx, cast_ops): """ op_role = get_valid_op_role(block, insert_idx) for fp16_name, fp32_name in cast_ops.items(): - block._insert_op_without_sync(insert_idx, - type="cast", - inputs={"X": fp32_name}, - outputs={"Out": fp16_name}, - attrs={ - "in_dtype": core.VarDesc.VarType.FP32, - "out_dtype": - core.VarDesc.VarType.FP16, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type="cast", + inputs={"X": fp32_name}, + outputs={"Out": fp16_name}, + attrs={ + "in_dtype": core.VarDesc.VarType.FP32, + "out_dtype": core.VarDesc.VarType.FP16, + OP_ROLE_KEY: op_role, + }, + ) return -def insert_allreduce_ops(block, - insert_idx, - ring_id, - allreduce_vars, - op_role=OpRole.Backward, - use_calc_stream=False, - user_defined_strategy=None): +def insert_allreduce_ops( + block, + insert_idx, + ring_id, + allreduce_vars, + op_role=OpRole.Backward, + use_calc_stream=False, + user_defined_strategy=None, +): """ _add_allreduce_ops """ if len(allreduce_vars) == 0: return - if user_defined_strategy and \ - user_defined_strategy.fuse_all_reduce_ops and \ - not user_defined_strategy.fuse_grad_merge: + if ( + user_defined_strategy + and user_defined_strategy.fuse_all_reduce_ops + and not user_defined_strategy.fuse_grad_merge + ): # If fuse_grad_merge is enable, the grad vars have already been fused during # gradient merge pass, therefore, those vars are not need to be fused here - insert_fused_allreduce_ops(block, insert_idx, ring_id, allreduce_vars, - op_role, use_calc_stream, - user_defined_strategy.fuse_grad_size_in_MB) + insert_fused_allreduce_ops( + block, + insert_idx, + ring_id, + allreduce_vars, + op_role, + use_calc_stream, + user_defined_strategy.fuse_grad_size_in_MB, + ) else: for var in allreduce_vars: - block._insert_op_without_sync(insert_idx, - type='c_allreduce_sum', - inputs={'X': var}, - outputs={'Out': var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': - use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_allreduce_sum', + inputs={'X': var}, + outputs={'Out': var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) return class FuseHelper(object): - @staticmethod def sort_vars_by_dtype(block, vars_name): fp32_vars = [] @@ -389,17 +428,19 @@ class FuseHelper(object): return fp32_vars @staticmethod - def get_fused_groups(block, vars_name, fuse_size=32.): - """ coalesce tensor, get fused group """ + def get_fused_groups(block, vars_name, fuse_size=32.0): + """coalesce tensor, get fused group""" groups = [] - cur_size = 0. + cur_size = 0.0 last_dtype = None for var_name in vars_name: real_var = block.var(var_name) var_size = get_var_size(real_var) - if cur_size + var_size > fuse_size \ - or len(groups) == 0 \ - or real_var.dtype != last_dtype: + if ( + cur_size + var_size > fuse_size + or len(groups) == 0 + or real_var.dtype != last_dtype + ): groups.append([real_var]) cur_size = var_size last_dtype = real_var.dtype @@ -409,11 +450,9 @@ class FuseHelper(object): return groups @staticmethod - def insert_coalesce_tensor(block, - index, - groups, - op_role=OpRole.Backward, - prefix="Output"): + def insert_coalesce_tensor( + block, index, groups, op_role=OpRole.Backward, prefix="Output" + ): fused_vars = [] insert_num = 0 for group in groups: @@ -423,247 +462,305 @@ class FuseHelper(object): fused_vars.append(group[0]) continue - fused_var = block.create_var(name=unique_name.generate( - 'Fused{}_{}'.format(prefix, group[0].name)), - dtype=group[0].dtype, - persistable=False, - stop_gradient=True) + fused_var = block.create_var( + name=unique_name.generate( + 'Fused{}_{}'.format(prefix, group[0].name) + ), + dtype=group[0].dtype, + persistable=False, + stop_gradient=True, + ) fused_vars.append(fused_var) - block._insert_op_without_sync(index, - type="coalesce_tensor", - inputs={"Input": group}, - outputs={ - "Output": group, - "FusedOutput": fused_var - }, - attrs={ - "copy_data": True, - "use_align": True, - "dtype": group[0].dtype, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + index, + type="coalesce_tensor", + inputs={"Input": group}, + outputs={"Output": group, "FusedOutput": fused_var}, + attrs={ + "copy_data": True, + "use_align": True, + "dtype": group[0].dtype, + OP_ROLE_KEY: op_role, + }, + ) insert_num += 1 return fused_vars, insert_num -def insert_fused_allreduce_ops(block, - insert_idx, - ring_id, - allreduce_vars, - op_role=OpRole.Backward, - use_calc_stream=False, - fuse_grad_size_in_MB=32): - groups = FuseHelper.get_fused_groups(block, allreduce_vars, - fuse_grad_size_in_MB) +def insert_fused_allreduce_ops( + block, + insert_idx, + ring_id, + allreduce_vars, + op_role=OpRole.Backward, + use_calc_stream=False, + fuse_grad_size_in_MB=32, +): + groups = FuseHelper.get_fused_groups( + block, allreduce_vars, fuse_grad_size_in_MB + ) - fused_vars, insert_num = FuseHelper.insert_coalesce_tensor(block, - insert_idx, - groups, - op_role, - prefix="Grad") + fused_vars, insert_num = FuseHelper.insert_coalesce_tensor( + block, insert_idx, groups, op_role, prefix="Grad" + ) for fused_var in fused_vars: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_allreduce_sum', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_allreduce_sum', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) if not use_calc_stream: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_sync_calc_stream', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={OP_ROLE_KEY: op_role}) - - -def insert_fused_reduce_ops(block, - insert_idx, - ring_id, - reduce_vars, - shard, - op_role=OpRole.Backward, - use_calc_stream=False, - rank=None, - fuse_grad_size=32): + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_sync_calc_stream', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={OP_ROLE_KEY: op_role}, + ) + + +def insert_fused_reduce_ops( + block, + insert_idx, + ring_id, + reduce_vars, + shard, + op_role=OpRole.Backward, + use_calc_stream=False, + rank=None, + fuse_grad_size=32, +): nranks = shard.worker_num device_to_vars = [[] for _ in range(nranks)] for var in reduce_vars: root_id = get_grad_device(var, shard) - assert 0 <= root_id < nranks, "root_id should >=0 and < nranks, " \ - "but now nranks={}, the root_id of var={} is {}"\ - .format(nranks, var, root_id) + assert 0 <= root_id < nranks, ( + "root_id should >=0 and < nranks, " + "but now nranks={}, the root_id of var={} is {}".format( + nranks, var, root_id + ) + ) device_to_vars[root_id].append(var) for root_id, vars_name in enumerate(device_to_vars): groups = FuseHelper.get_fused_groups(block, vars_name, fuse_grad_size) fused_vars, insert_num = FuseHelper.insert_coalesce_tensor( - block, insert_idx, groups, op_role, prefix="Grad") + block, insert_idx, groups, op_role, prefix="Grad" + ) for fused_var in fused_vars: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_reduce_sum', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'root_id': root_id, - 'use_calc_stream': - use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_reduce_sum', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'root_id': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) if not use_calc_stream: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_sync_calc_stream', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={OP_ROLE_KEY: op_role}) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_sync_calc_stream', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={OP_ROLE_KEY: op_role}, + ) return [] if rank is None else device_to_vars[rank] -def insert_reduce_ops(block, - insert_idx, - ring_id, - reduce_vars, - shard, - op_role=OpRole.Backward, - use_calc_stream=False, - rank=None, - strategy=None): +def insert_reduce_ops( + block, + insert_idx, + ring_id, + reduce_vars, + shard, + op_role=OpRole.Backward, + use_calc_stream=False, + rank=None, + strategy=None, +): """ _add_reduce_ops """ - if strategy and strategy.fuse_all_reduce_ops and \ - not strategy.fuse_grad_merge: - return insert_fused_reduce_ops(block, insert_idx, ring_id, reduce_vars, - shard, op_role, use_calc_stream, rank, - strategy.fuse_grad_size_in_MB) + if ( + strategy + and strategy.fuse_all_reduce_ops + and not strategy.fuse_grad_merge + ): + return insert_fused_reduce_ops( + block, + insert_idx, + ring_id, + reduce_vars, + shard, + op_role, + use_calc_stream, + rank, + strategy.fuse_grad_size_in_MB, + ) grad_in_this_device = [] for var in reduce_vars: grad_var = var - if strategy and strategy.fuse_all_reduce_ops and \ - strategy.fuse_grad_merge: + if ( + strategy + and strategy.fuse_all_reduce_ops + and strategy.fuse_grad_merge + ): # TODO(wangxi): if support fp16_allreduce, need be # 'FusedMergedGrad.cast_fp16._' grad_var = var.replace('FusedMergedGrad_', '') root_id = get_grad_device(grad_var, shard) - assert root_id >= 0, "root id should be a positive int, but now root id is {}".format( - root_id) + assert ( + root_id >= 0 + ), "root id should be a positive int, but now root id is {}".format( + root_id + ) if rank is not None and rank == root_id: grad_in_this_device.append(var) - block._insert_op_without_sync(insert_idx, - type='c_reduce_sum', - inputs={'X': var}, - outputs={'Out': var}, - attrs={ - 'ring_id': ring_id, - 'root_id': root_id, - 'use_calc_stream': use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_reduce_sum', + inputs={'X': var}, + outputs={'Out': var}, + attrs={ + 'ring_id': ring_id, + 'root_id': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) return grad_in_this_device -def insert_fused_broadcast_param_ops(block, - insert_idx, - ring_id, - params, - shard, - op_role=OpRole.Optimize, - use_calc_stream=False, - rank=None, - fuse_size=32): +def insert_fused_broadcast_param_ops( + block, + insert_idx, + ring_id, + params, + shard, + op_role=OpRole.Optimize, + use_calc_stream=False, + rank=None, + fuse_size=32, +): nranks = shard.worker_num device_to_vars = [[] for _ in range(nranks)] for var in params: root_id = shard.device(var) - assert 0 <= root_id < nranks, "root_id should >=0 and < nranks, " \ - "but now nranks={}, the root_id of var={} is {}"\ - .format(nranks, var, root_id) + assert 0 <= root_id < nranks, ( + "root_id should >=0 and < nranks, " + "but now nranks={}, the root_id of var={} is {}".format( + nranks, var, root_id + ) + ) device_to_vars[root_id].append(var) for root_id, vars_name in enumerate(device_to_vars): groups = FuseHelper.get_fused_groups(block, vars_name, fuse_size) fused_vars, insert_num = FuseHelper.insert_coalesce_tensor( - block, insert_idx, groups, op_role, prefix="Param") + block, insert_idx, groups, op_role, prefix="Param" + ) for fused_var in fused_vars: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_broadcast', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'root': root_id, - 'use_calc_stream': - use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_broadcast', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'root': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) if not use_calc_stream: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_sync_calc_stream', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={OP_ROLE_KEY: op_role}) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_sync_calc_stream', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={OP_ROLE_KEY: op_role}, + ) return [] if rank is None else device_to_vars[rank] -def insert_broadcast_param_ops(block, - insert_idx, - ring_id, - params, - shard, - op_role=OpRole.Optimize, - use_calc_stream=False, - rank=None, - strategy=None): +def insert_broadcast_param_ops( + block, + insert_idx, + ring_id, + params, + shard, + op_role=OpRole.Optimize, + use_calc_stream=False, + rank=None, + strategy=None, +): """ add broadcast param ops """ if strategy and strategy.fuse_all_reduce_ops: # TODO(wangxi): put fused var in startup_program, only need exec once - return insert_fused_broadcast_param_ops(block, insert_idx, ring_id, - params, shard, op_role, - use_calc_stream, rank, - strategy.fuse_grad_size_in_MB) + return insert_fused_broadcast_param_ops( + block, + insert_idx, + ring_id, + params, + shard, + op_role, + use_calc_stream, + rank, + strategy.fuse_grad_size_in_MB, + ) param_in_this_device = [] for param in params: root_id = shard.device(param) - assert root_id >= 0, "root id should be a positive int, but now root id is {}".format( - root_id) + assert ( + root_id >= 0 + ), "root id should be a positive int, but now root id is {}".format( + root_id + ) if rank is not None and rank == root_id: param_in_this_device.append(param) - block._insert_op_without_sync(insert_idx, - type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': root_id, - 'use_calc_stream': use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) return param_in_this_device -def fuse_opt_broadcast_param_ops(block, - ring_id, - shard, - op_role=OpRole.Optimize, - strategy=None): +def fuse_opt_broadcast_param_ops( + block, ring_id, shard, op_role=OpRole.Optimize, strategy=None +): """ fuse optimizer sharding broadcast param ops """ @@ -689,26 +786,30 @@ def fuse_opt_broadcast_param_ops(block, groups = FuseHelper.get_fused_groups(block, vars_name, fuse_size) fused_vars, insert_num = FuseHelper.insert_coalesce_tensor( - block, insert_idx, groups, op_role, prefix="Param") + block, insert_idx, groups, op_role, prefix="Param" + ) for fused_var in fused_vars: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_broadcast', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'root': root_id, - 'use_calc_stream': True, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_broadcast', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'root': root_id, + 'use_calc_stream': True, + OP_ROLE_KEY: op_role, + }, + ) block._sync_with_cpp() def get_grad_device(grad_name, shard): assert "@GRAD" in grad_name, "[{}] should be a grad variable.".format( - grad_name) + grad_name + ) base_name = None # NOTE: mind the traversal order possible_suffixes = [ @@ -725,8 +826,9 @@ def get_grad_device(grad_name, shard): base_name = re.sub(suffix, '', grad_name) break - assert base_name in shard.global_param2device, "[{}] should be a param variable.".format( - base_name) + assert ( + base_name in shard.global_param2device + ), "[{}] should be a param variable.".format(base_name) return shard.global_param2device[base_name] @@ -760,15 +862,17 @@ def insert_broadcast_ops(block, insert_idx, ring_id, broadcast2root): """ op_role = get_valid_op_role(block, insert_idx) for broadcast_name, root_device in broadcast2root: - block._insert_op_without_sync(insert_idx, - type='c_broadcast', - inputs={'X': broadcast_name}, - outputs={'Out': broadcast_name}, - attrs={ - 'ring_id': ring_id, - 'root': root_device, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_broadcast', + inputs={'X': broadcast_name}, + outputs={'Out': broadcast_name}, + attrs={ + 'ring_id': ring_id, + 'root': root_device, + OP_ROLE_KEY: op_role, + }, + ) return @@ -793,8 +897,12 @@ def get_var_size(param): var size in MB """ assert -1 not in param.shape - return reduce(lambda x, y: x * y, - param.shape) * DtypeToSize[param.dtype] / 1024.0 / 1024.0 + return ( + reduce(lambda x, y: x * y, param.shape) + * DtypeToSize[param.dtype] + / 1024.0 + / 1024.0 + ) def insert_scale_loss_grad_ops(block, scale=1.0): @@ -804,9 +912,10 @@ def insert_scale_loss_grad_ops(block, scale=1.0): ''' for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): - assert op.type == 'fill_constant', \ - "loss_grad_op must be fill_constant op, " \ + assert op.type == 'fill_constant', ( + "loss_grad_op must be fill_constant op, " "but this op is {}".format(op.type) + ) assert op.has_attr('value') loss_scale = float(op.attr('value')) loss_scale = loss_scale / scale @@ -825,8 +934,9 @@ def comm_analyse(main_program): if op.type == "c_broadcast": var_name = op.desc.input_arg_names()[0] # convert MB to KB - broadcast_vars[var_name] = get_var_size( - block.var(var_name)) * 1024.0 + broadcast_vars[var_name] = ( + get_var_size(block.var(var_name)) * 1024.0 + ) elif op.type == "c_allreduce_sum": var_name = op.desc.input_arg_names()[0] reduce_vars[var_name] = get_var_size(block.var(var_name)) * 1024.0 @@ -836,14 +946,14 @@ def comm_analyse(main_program): for k, v in broadcast_vars.items(): print("broadcast: {}: {} KB".format(k, v)) - if (int(v / gap) in varsize_count): + if int(v / gap) in varsize_count: varsize_count[int(v / gap)] += 1 else: varsize_count[int(v / gap)] = 1 for k, v in reduce_vars.items(): print("allreduce: {}: {} KB".format(k, v)) - if (int(v / gap) in varsize_count): + if int(v / gap) in varsize_count: varsize_count[int(v / gap)] += 1 else: varsize_count[int(v / gap)] = 1 @@ -852,8 +962,9 @@ def comm_analyse(main_program): sorted_varsize = sorted(varsize_count.items(), key=lambda x: x[0]) for varsize, count in sorted_varsize: print("NCCL size {}~{} KB: {}".format(varsize, varsize + 1, count)) - f.write("NCCL size {}~{} KB: {}\n".format(varsize, varsize + 1, - count)) + f.write( + "NCCL size {}~{} KB: {}\n".format(varsize, varsize + 1, count) + ) def add_sync_comm(program, sharding_ring_id): @@ -863,7 +974,7 @@ def add_sync_comm(program, sharding_ring_id): add the sync_comm op for the test prog. """ - #NOTE (liangjianzhong): only support one comm stream by now, use more than one + # NOTE (liangjianzhong): only support one comm stream by now, use more than one # comm streams will cause error. should be revise in future. assert sharding_ring_id >= 0, "sharding_ring_id should larger than zero" @@ -877,15 +988,15 @@ def add_sync_comm(program, sharding_ring_id): for input_name in op.desc.input_arg_names(): not_sync_vars.remove(input_name) if not_sync_vars: - block.append_op(type='c_sync_comm_stream', - inputs={'X': list(not_sync_vars)}, - outputs={'Out': list(not_sync_vars)}, - attrs={ - 'ring_id': - sharding_ring_id, - 'op_role': - core.op_proto_and_checker_maker.OpRole.Forward - }) + block.append_op( + type='c_sync_comm_stream', + inputs={'X': list(not_sync_vars)}, + outputs={'Out': list(not_sync_vars)}, + attrs={ + 'ring_id': sharding_ring_id, + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, + }, + ) return @@ -905,8 +1016,14 @@ def save_persistables(exe, dirname, main_program, filename=None): # support EMA optimizer with '_ema_0', # support offload with '@offload_0' and '.cast_fp16' checks = [ - "_moment1_0", "_moment2_0", "_beta1_pow_acc_0", "_beta2_pow_acc_0", - "_velocity_0", "_ema_0", "@offload_0", ".cast_fp16" + "_moment1_0", + "_moment2_0", + "_beta1_pow_acc_0", + "_beta2_pow_acc_0", + "_velocity_0", + "_ema_0", + "@offload_0", + ".cast_fp16", ] for check in checks: if var.name.endswith(check) and var.persistable: @@ -919,24 +1036,27 @@ def save_persistables(exe, dirname, main_program, filename=None): return var.name.endswith("@GradiantMerge") def is_trainable(var): - return isinstance(var, - paddle.fluid.framework.Parameter) and var.trainable + return ( + isinstance(var, paddle.fluid.framework.Parameter) and var.trainable + ) def sharding_predicate(var): - return is_trainable(var) or is_opt_vars(var) or is_gradient_merge_vars( - var) + return ( + is_trainable(var) or is_opt_vars(var) or is_gradient_merge_vars(var) + ) if int(os.environ.get('PADDLE_TRAINER_ID', 0)) == 0: - paddle.fluid.io.save_persistables(exe, - dirname, - main_program=main_program, - filename=None) + paddle.fluid.io.save_persistables( + exe, dirname, main_program=main_program, filename=None + ) else: - paddle.fluid.io.save_vars(exe, - dirname, - main_program=main_program, - predicate=sharding_predicate, - filename=None) + paddle.fluid.io.save_vars( + exe, + dirname, + main_program=main_program, + predicate=sharding_predicate, + filename=None, + ) return @@ -944,22 +1064,28 @@ def save_persistables(exe, dirname, main_program, filename=None): def append_naive_sync(block, sync_var, ring_id): # NOTE (JZ-LIANG) update this to use barrier sync for more elegent logic # sync within global - block.append_op(type="fill_constant", - outputs={"Out": sync_var}, - attrs={ - "shape": sync_var.shape, - "dtype": sync_var.dtype, - "value": int(1), - }) - block.append_op(type='c_allreduce_sum', - inputs={'X': sync_var}, - outputs={'Out': sync_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_sync_calc_stream', - inputs={'X': [sync_var]}, - outputs={'Out': [sync_var]}, - attrs={OP_ROLE_KEY: OpRole.Forward}) + block.append_op( + type="fill_constant", + outputs={"Out": sync_var}, + attrs={ + "shape": sync_var.shape, + "dtype": sync_var.dtype, + "value": int(1), + }, + ) + block.append_op( + type='c_allreduce_sum', + inputs={'X': sync_var}, + outputs={'Out': sync_var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_sync_calc_stream', + inputs={'X': [sync_var]}, + outputs={'Out': [sync_var]}, + attrs={OP_ROLE_KEY: OpRole.Forward}, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py index 42c52af4431..3d5d8aa2a38 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py @@ -18,13 +18,13 @@ __all__ = [] class WeightDecayHelper(object): - def __init__(self): pass def _is_weight_decay_op(self, op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/regularization") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/regularization") def prune_weight_decay(self, block, shard): for idx, op in reversed(list(enumerate(block.ops))): @@ -33,7 +33,8 @@ class WeightDecayHelper(object): if OP_ROLE_VAR_KEY not in op.attr_names: raise ValueError( "The Weight Dacay op should hold op_role_var attribute" - "but the {} op does not hold op_role_var".format(op.type)) + "but the {} op does not hold op_role_var".format(op.type) + ) op_role_var = op.all_attrs()[OP_ROLE_VAR_KEY] if not shard.has_param(op_role_var[0]): block._remove_op(idx, sync=False) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py index 073de83d19a..da0a08aefa1 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py @@ -94,7 +94,7 @@ class ShardingOptimizer(MetaOptimizerBase): dist_strategy.sharding_configs = {"segment_broadcast_MB": 32} def _get_sharding_segment_strategy(self): - """ get + """get self._sharding_segment_strategy 1. if by_size: self._broadcast_MB 2. if by_anchors: self._sharding_segment_anchors @@ -107,21 +107,26 @@ class ShardingOptimizer(MetaOptimizerBase): if segment_strategy == "segment_broadcast_MB": self._broadcast_MB = sharding_configs["segment_broadcast_MB"] - assert self._broadcast_MB > 0, "segment size should larger than zero !" + assert ( + self._broadcast_MB > 0 + ), "segment size should larger than zero !" elif segment_strategy == "segment_anchors": self._sharding_segment_anchors = sharding_configs["segment_anchors"] - assert len(self._sharding_segment_anchors - ) > 0, "you should set the sharding segment anchors !" + assert ( + len(self._sharding_segment_anchors) > 0 + ), "you should set the sharding segment anchors !" self._backward_remain_anchors = self._sharding_segment_anchors[:] self._forward_remain_anchors = [] else: raise NotImplementedError( "the sharding segment strategy [{}] is not implemented".format( - str(segment_strategy))) + str(segment_strategy) + ) + ) self._sharding_segment_strategy = segment_strategy def _get_hybrid_degree(self): - """ get + """get self.hybrid_dp self.sharding_degree self.mp_degree @@ -145,21 +150,32 @@ class ShardingOptimizer(MetaOptimizerBase): assert strategy.pipeline is True if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None): - assert pp_degree == 2, ("For manually set pipeline, only " - "pp_degree = 2 is supported.") - assert global_world_size == mp_degree * sharding_degree * dp_degree, \ - "global work size [{}], mp_degree [{}], sharding_degree [{}], dp_degree [{}].".format( - global_world_size, mp_degree, sharding_degree, dp_degree) + assert pp_degree == 2, ( + "For manually set pipeline, only " "pp_degree = 2 is supported." + ) + assert ( + global_world_size == mp_degree * sharding_degree * dp_degree + ), "global work size [{}], mp_degree [{}], sharding_degree [{}], dp_degree [{}].".format( + global_world_size, mp_degree, sharding_degree, dp_degree + ) else: - assert global_world_size == mp_degree * sharding_degree * pp_degree * dp_degree, \ - "global work size [{}], mp_degree [{}], sharding_degree [{}], pp_degree [{}], dp_degree [{}].".format( - global_world_size, mp_degree, sharding_degree, pp_degree, dp_degree) + assert ( + global_world_size + == mp_degree * sharding_degree * pp_degree * dp_degree + ), "global work size [{}], mp_degree [{}], sharding_degree [{}], pp_degree [{}], dp_degree [{}].".format( + global_world_size, + mp_degree, + sharding_degree, + pp_degree, + dp_degree, + ) # FIXME (JZ-LIANG) deprecated hybrid_dp if sharding_configs["hybrid_dp"]: logger.warning( "[hybrid_dp] API setting is deprecated. Now when " - "dp_degree >= 2, its will be in hybrid dp mode automatically") + "dp_degree >= 2, its will be in hybrid dp mode automatically" + ) assert dp_degree >= 1 self.hybrid_dp = True if dp_degree > 1 else False @@ -169,7 +185,7 @@ class ShardingOptimizer(MetaOptimizerBase): self.dp_degree = dp_degree def _get_hybrid_dp_mode(self): - """ get + """get self.hybrid_dp_mode = 'pp_hybrid_dp' or 'sharding_hybrid_dp' self.gradient_merge_mode = 'pp_gm' or 'sharding_gm' self._gradient_merge_acc_step @@ -193,9 +209,10 @@ class ShardingOptimizer(MetaOptimizerBase): if self.pp_degree > 1: dp_mode = "pp_hybrid_dp" else: - assert self.sharding_degree > 1, \ - "by now we only support five kind of hybrid dp: sharding_hybrid_dp, " \ + assert self.sharding_degree > 1, ( + "by now we only support five kind of hybrid dp: sharding_hybrid_dp, " "mp_sharding_hybrid_dp, pp_hybrid_dp, mp_sharding_pp_hybrid_dp, sharding_pp_hybrid_dp." + ) dp_mode = "sharding_hybrid_dp" # gradient merge @@ -208,23 +225,33 @@ class ShardingOptimizer(MetaOptimizerBase): gm_mode = "pp_gm" gm_acc_step = strategy.pipeline_configs['accumulate_steps'] gradient_scale_configs = strategy.gradient_scale_configs - assert gradient_scale_configs['scale_strategy'] == 'avg', \ - 'For pipeline mode, the ' 'gradient scale mode should ' \ - 'be "avg", but got {}'.format(gradient_scale_configs['scale_strategy']) + assert gradient_scale_configs['scale_strategy'] == 'avg', ( + 'For pipeline mode, the ' + 'gradient scale mode should ' + 'be "avg", but got {}'.format( + gradient_scale_configs['scale_strategy'] + ) + ) # Note (Yuang Liu): this avg_loss flag determines where to do the average op for grad merge. # If True, will do sum firstly for gradient merge, then do scale by gm_acc_step. # If False, will scale loss by gm_acc_step first, then do sum for gradient merge. self.scale_gradient = gradient_scale_configs['scale_gradient'] if gm_acc_step > 1: - logger.info("Gradient merge in [{}], acc step = [{}]".format( - gm_mode, gm_acc_step)) + logger.info( + "Gradient merge in [{}], acc step = [{}]".format( + gm_mode, gm_acc_step + ) + ) optimizer_sharding = False # TODO(wangxi): need support dp_as_opt_sharding with sharding # need support without pp in future - if self.sharding_degree == 1 and self.dp_degree > 1 \ - and sharding_configs['_dp_as_optimizer_sharding'] \ - and self.pp_degree > 1: + if ( + self.sharding_degree == 1 + and self.dp_degree > 1 + and sharding_configs['_dp_as_optimizer_sharding'] + and self.pp_degree > 1 + ): optimizer_sharding = True self.hybrid_dp_mode = dp_mode @@ -234,19 +261,23 @@ class ShardingOptimizer(MetaOptimizerBase): # this feature is design for ascend, and should NOT be used in GPU training self.pp_allreduce_in_optimize = sharding_configs[ - "pp_allreduce_in_optimize"] + "pp_allreduce_in_optimize" + ] - def _inner_opt_minimize(self, loss, startup_program, parameter_list, - no_grad_set): + def _inner_opt_minimize( + self, loss, startup_program, parameter_list, no_grad_set + ): pipeline_configs = self.user_defined_strategy.pipeline_configs if self.inner_opt is None: raise ValueError( - "self.inner_opt of ShardingOptimizer should not be None.") + "self.inner_opt of ShardingOptimizer should not be None." + ) if self.pp_degree > 1: pp_optimizer = fluid.optimizer.PipelineOptimizer( - self.inner_opt, self._gradient_merge_acc_step) + self.inner_opt, self._gradient_merge_acc_step + ) self._pp_optimizer = pp_optimizer global_rank = self.role_maker._worker_index() @@ -263,17 +294,25 @@ class ShardingOptimizer(MetaOptimizerBase): 'global_ring_id': 3, 'mp_degree': self.mp_degree, 'mp_rank': global_rank % self.mp_degree, - 'scale_gradient': self.scale_gradient + 'scale_gradient': self.scale_gradient, } main_program = loss.block.program main_program._pipeline_opt = pipeline_opt - optimize_ops, params_grads, program_list, self.pipeline_pair, self.pp_ring_map = pp_optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set) + ( + optimize_ops, + params_grads, + program_list, + self.pipeline_pair, + self.pp_ring_map, + ) = pp_optimizer.minimize( + loss, startup_program, parameter_list, no_grad_set + ) assert self.pp_degree == len(program_list) else: optimize_ops, params_grads = self.inner_opt.minimize( - loss, startup_program, parameter_list, no_grad_set) + loss, startup_program, parameter_list, no_grad_set + ) if startup_program is None: startup_program = default_startup_program() @@ -282,8 +321,9 @@ class ShardingOptimizer(MetaOptimizerBase): startup_program = startup_program._pipeline_opt['startup_program'] print("pp_rank:", self.pp_rank) if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None): - main_program = program_list[int( - os.getenv("PADDLE_MANUAL_PIPELINE_STAGE"))] + main_program = program_list[ + int(os.getenv("PADDLE_MANUAL_PIPELINE_STAGE")) + ] else: main_program = program_list[self.pp_rank] with open("main_%d" % self.role_maker._worker_index(), 'w') as f: @@ -309,14 +349,16 @@ class ShardingOptimizer(MetaOptimizerBase): return optimize_ops, params_grads def _apply_sharding_pass(self, params_grads): - if self.sharding_degree == 1: return + if self.sharding_degree == 1: + return main_block = self._main_program.global_block() startup_block = self._startup_program.global_block() # step1: build shard - self._build_shard(params_grads, self.sharding_rank, - self.sharding_degree) + self._build_shard( + params_grads, self.sharding_rank, self.sharding_degree + ) # step2: split_program self._split_program(main_block) @@ -328,13 +370,16 @@ class ShardingOptimizer(MetaOptimizerBase): # step4: remove unneeded ops and vars from block self._prune_main_program( - main_block, self._shard, - [self.mp_ring_id, self.sharding_ring_id, self.pp_ring_id]) + main_block, + self._shard, + [self.mp_ring_id, self.sharding_ring_id, self.pp_ring_id], + ) self._prune_startup_program(startup_block, self._shard) def _apply_opt_sharding_pass(self, params_grads): - """ outer dp as optimizer sharding """ - if self._optimizer_sharding is False: return + """outer dp as optimizer sharding""" + if self._optimizer_sharding is False: + return main_block = self._main_program.global_block() startup_block = self._startup_program.global_block() @@ -348,12 +393,15 @@ class ShardingOptimizer(MetaOptimizerBase): # step4: remove unneeded ops and vars from block self._prune_main_program( - main_block, self._shard, - [self.mp_ring_id, self.pp_ring_id, self.dp_ring_id]) + main_block, + self._shard, + [self.mp_ring_id, self.pp_ring_id, self.dp_ring_id], + ) self._prune_startup_program(startup_block, self._shard) def _insert_allreduce_for_pp(self, params_grads): - if self.pp_degree == 1: return + if self.pp_degree == 1: + return strategy = self.user_defined_strategy sharding_configs = strategy.sharding_configs @@ -373,10 +421,12 @@ class ShardingOptimizer(MetaOptimizerBase): main_block._remove_op(idx) for idx, op in reversed(list(enumerate(main_block.ops))): - if op.type != 'cast': continue + if op.type != 'cast': + continue in_name = op.input_arg_names[0] - if in_name not in self._params: continue - #if self._shard.has_param(param_name): continue + if in_name not in self._params: + continue + # if self._shard.has_param(param_name): continue if in_name not in main_block.vars: main_block._remove_op(idx) @@ -386,7 +436,8 @@ class ShardingOptimizer(MetaOptimizerBase): shard = self._shard if self._optimizer_sharding else None accumulated_grad_names = self._pp_optimizer._accumulate_gradients( - main_block, strategy=strategy, shard=shard) + main_block, strategy=strategy, shard=shard + ) len_of_ops = len(main_block.ops) if self.scale_gradient: @@ -394,8 +445,9 @@ class ShardingOptimizer(MetaOptimizerBase): first_optimize_op_index = get_first_optimize_op_idx(main_block) if self.pp_allreduce_in_optimize: - logger.info("Pipeline Persistable grad is {}".format( - accumulated_grad_names)) + logger.info( + "Pipeline Persistable grad is {}".format(accumulated_grad_names) + ) # FIXME(wangxi): accumulated_grad get from pipeline is not # include sharding's param@BroadCast grad when # pp_allreduce_in_optimize @@ -407,10 +459,11 @@ class ShardingOptimizer(MetaOptimizerBase): self._shard, core.op_proto_and_checker_maker.OpRole.Optimize, use_calc_stream=True, - rank=self.sharding_rank) + rank=self.sharding_rank, + ) logger.info("PP-Sharding grad is {}".format(accumulated_grad_names)) - first_optimize_op_index += (len(main_block.ops) - len_of_ops) + first_optimize_op_index += len(main_block.ops) - len_of_ops len_of_ops = len(main_block.ops) if self._optimizer_sharding: @@ -423,10 +476,12 @@ class ShardingOptimizer(MetaOptimizerBase): OpRole.Optimize, use_calc_stream=True, rank=self.dp_rank, - strategy=strategy) + strategy=strategy, + ) logger.info( - "Optimizer grad in this rank {}".format(accumulated_grad_names)) - first_optimize_op_index += (len(main_block.ops) - len_of_ops) + "Optimizer grad in this rank {}".format(accumulated_grad_names) + ) + first_optimize_op_index += len(main_block.ops) - len_of_ops len_of_ops = len(main_block.ops) # NOTE(wangxi): we fused after optimize_cast @@ -434,14 +489,17 @@ class ShardingOptimizer(MetaOptimizerBase): optimizer_param = utils.insert_broadcast_param_ops( main_block, len_of_ops, - self.dp_ring_id, [x[0].name for x in params_grads], + self.dp_ring_id, + [x[0].name for x in params_grads], self._shard, OpRole.Optimize, use_calc_stream=True, rank=self.dp_rank, - strategy=None if optimize_cast else strategy) + strategy=None if optimize_cast else strategy, + ) logger.info( - "Optimizer param in this rank {}".format(optimizer_param)) + "Optimizer param in this rank {}".format(optimizer_param) + ) if not strategy.fuse_grad_merge and not optimize_cast: assert len(accumulated_grad_names) == len(optimizer_param) elif self.hybrid_dp and self.hybrid_dp_mode == "pp_hybrid_dp": @@ -452,15 +510,20 @@ class ShardingOptimizer(MetaOptimizerBase): accumulated_grad_names, core.op_proto_and_checker_maker.OpRole.Optimize, use_calc_stream=True, - user_defined_strategy=strategy) - first_optimize_op_index += (len(main_block.ops) - len_of_ops) + user_defined_strategy=strategy, + ) + first_optimize_op_index += len(main_block.ops) - len_of_ops len_of_ops = len(main_block.ops) # FIXME(wangxi): if fp16_allreduce, put cast fp16->fp32 to there? def _avg_grad_merge_after_sum(self, main_block, accumulated_grad_names): - if self.user_defined_strategy.amp and \ - self.user_defined_strategy.amp_configs['use_dynamic_loss_scaling']: + if ( + self.user_defined_strategy.amp + and self.user_defined_strategy.amp_configs[ + 'use_dynamic_loss_scaling' + ] + ): # For AMP, if using dynamic loss scaling the avg # operation can be simple done by modify the LossScaling op. for idx, op in enumerate(main_block.ops): @@ -471,7 +534,8 @@ class ShardingOptimizer(MetaOptimizerBase): loss_scale_tmp_var = main_block.create_var( name=loss_scale_tmp_var_name, shape=loss_scaling_var.shape, - dtype=loss_scaling_var.dtype) + dtype=loss_scaling_var.dtype, + ) main_block._insert_op_without_sync( idx, type='scale', @@ -481,8 +545,9 @@ class ShardingOptimizer(MetaOptimizerBase): 'scale': self._gradient_merge_acc_step, 'bias': 0.0, 'bias_after_scale': False, - OP_ROLE_KEY: OpRole.Optimize - }) + OP_ROLE_KEY: OpRole.Optimize, + }, + ) op._rename_input(loss_scale_name, loss_scale_tmp_var_name) break else: @@ -493,7 +558,9 @@ class ShardingOptimizer(MetaOptimizerBase): if is_optimizer_op(op) and op.type != 'c_sync_comm_stream': tmp_first_opt_idx = idx break - assert tmp_first_opt_idx is not None, 'Occurs some errors, no optimize ops' + assert ( + tmp_first_opt_idx is not None + ), 'Occurs some errors, no optimize ops' for grad in accumulated_grad_names: main_block._insert_op_without_sync( tmp_first_opt_idx, @@ -504,14 +571,17 @@ class ShardingOptimizer(MetaOptimizerBase): 'scale': 1.0 / self._gradient_merge_acc_step, 'bias': 0.0, 'bias_after_scale': False, - OP_ROLE_KEY: OpRole.Optimize - }) + OP_ROLE_KEY: OpRole.Optimize, + }, + ) def _adapt_amp_clip_without_sharding(self): # if not use sharding, adapt amp/clip, for remain parallelism. # cast --> amp --> clip --> opt - if self.sharding_degree > 1: return - if self._optimizer_sharding: return + if self.sharding_degree > 1: + return + if self._optimizer_sharding: + return main_block = self._main_program.global_block() startup_block = self._startup_program.global_block() @@ -525,9 +595,9 @@ class ShardingOptimizer(MetaOptimizerBase): FP16Utils.sync_amp_check_nan_inf(main_block, rings) gradientclip_helper = GradientClipHelper(None) - gradientclip_helper.sync_global_norm(main_block, - [self.mp_ring_id, self.pp_ring_id], - self.mp_rank) + gradientclip_helper.sync_global_norm( + main_block, [self.mp_ring_id, self.pp_ring_id], self.mp_rank + ) def _insert_loss_grad_scale_op(self): main_block = self._main_program.global_block() @@ -548,8 +618,9 @@ class ShardingOptimizer(MetaOptimizerBase): mp_ring_id = self.mp_ring_id if self.mp_degree > 1 else None dp_ring_id = self.dp_ring_id if self.dp_degree > 1 else None - offload_helper = OffloadHelper(mp_ring_id=mp_ring_id, - dp_ring_id=dp_ring_id) + offload_helper = OffloadHelper( + mp_ring_id=mp_ring_id, dp_ring_id=dp_ring_id + ) # optimize offload should be enable while gradient merge is enable and # acc_step is quite large (e.g. >> 100). Since its memcpy could not be @@ -565,32 +636,32 @@ class ShardingOptimizer(MetaOptimizerBase): # will take more memory, but will be faster. Trade space for time. if self._optimizer_sharding: offload_helper.opt_sharding_cast_fp32param( - main_block, startup_block, - [x[0].name for x in params_grads]) + main_block, startup_block, [x[0].name for x in params_grads] + ) # NOTE(wangxi): fused after optimize_cast - utils.fuse_opt_broadcast_param_ops(main_block, - dp_ring_id, - self._shard, - strategy=strategy) + utils.fuse_opt_broadcast_param_ops( + main_block, dp_ring_id, self._shard, strategy=strategy + ) else: offload_helper.cast_fp32param_in_optimize( - main_block, startup_block) + main_block, startup_block + ) def _dump_program_for_debug(self): main_block = self._main_program.global_block() startup_block = self._startup_program.global_block() - with open("start_sharding_%d" % self.role_maker._worker_index(), - 'w') as f: + with open( + "start_sharding_%d" % self.role_maker._worker_index(), 'w' + ) as f: f.writelines(str(startup_block.program)) - with open("main_sharding_%d" % self.role_maker._worker_index(), - 'w') as f: + with open( + "main_sharding_%d" % self.role_maker._worker_index(), 'w' + ) as f: f.writelines(str(main_block.program)) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): # TODO: (JZ-LIANG) support multiple comm in future # self._nrings = self.user_defined_strategy.nccl_comm_num self._nrings_sharding = 1 @@ -605,7 +676,8 @@ class ShardingOptimizer(MetaOptimizerBase): # inner optimize minimize optimize_ops, params_grads = self._inner_opt_minimize( - loss, startup_program, parameter_list, no_grad_set) + loss, startup_program, parameter_list, no_grad_set + ) self._init_comm() @@ -654,13 +726,15 @@ class ShardingOptimizer(MetaOptimizerBase): ] pp_rank = 0 if self.pp_rank == pair[0] else 1 if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None) is None: - self._collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - pp_group_endpoints, - pp_rank, - ring_id, - False, - sync=False) + self._collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + pp_group_endpoints, + pp_rank, + ring_id, + False, + sync=False, + ) def _init_npu_pipeline_comm(self, startup_block): # NOTE(wangxi): some bug with hccl, must set pp_degree be even number @@ -678,15 +752,22 @@ class ShardingOptimizer(MetaOptimizerBase): my_pair.append(pair) # for example: self.pp_rank=2, self.pp_degree=4 - send_to_next_pair = (self.pp_rank, (self.pp_rank + 1) % self.pp_degree - ) # 2->3 + send_to_next_pair = ( + self.pp_rank, + (self.pp_rank + 1) % self.pp_degree, + ) # 2->3 recv_from_next_pair = ( - (self.pp_rank + 1) % self.pp_degree, self.pp_rank) # 3->2 + (self.pp_rank + 1) % self.pp_degree, + self.pp_rank, + ) # 3->2 recv_from_prev_pair = ( - (self.pp_rank - 1 + self.pp_degree) % self.pp_degree, self.pp_rank + (self.pp_rank - 1 + self.pp_degree) % self.pp_degree, + self.pp_rank, ) # 1->2 - send_to_prev_pair = (self.pp_rank, (self.pp_rank - 1 + self.pp_degree) % - self.pp_degree) # 2->1 + send_to_prev_pair = ( + self.pp_rank, + (self.pp_rank - 1 + self.pp_degree) % self.pp_degree, + ) # 2->1 even = (self.pp_rank % 2) == 0 @@ -695,54 +776,66 @@ class ShardingOptimizer(MetaOptimizerBase): ring_id = self.pp_ring_map[pair[0] * 1000 + pair[1]] self._init_pair_comm(pair, ring_id) my_pair.remove(pair) - logger.info("pair0(even->odd): pp pair:{}, ring_id: {}".format( - pair, ring_id)) + logger.info( + "pair0(even->odd): pp pair:{}, ring_id: {}".format(pair, ring_id) + ) # 2. even recv from next, odd send to prev, 1->0, 3->2 pair = recv_from_next_pair if even else send_to_prev_pair ring_id = self.pp_ring_map[pair[0] * 1000 + pair[1]] self._init_pair_comm(pair, ring_id) my_pair.remove(pair) - logger.info("pair1(even<-odd): pp pair:{}, ring_id: {}".format( - pair, ring_id)) + logger.info( + "pair1(even<-odd): pp pair:{}, ring_id: {}".format(pair, ring_id) + ) # if pp_degree is 2, only need pair(0->1, 1->0) if self.pp_degree > 2: # 3. odd send to next, even recv from prev, 1->2, 3->0 pair = send_to_next_pair if not even else recv_from_prev_pair - ring_id = self.pp_ring_map.get(pair[0] * 1000 + pair[1], - max_ring_id + - 1) # 3->0 not in pp_ring_map + ring_id = self.pp_ring_map.get( + pair[0] * 1000 + pair[1], max_ring_id + 1 + ) # 3->0 not in pp_ring_map self._init_pair_comm(pair, ring_id) if self.pp_rank != 0 and self.pp_rank != self.pp_degree - 1: my_pair.remove(pair) - logger.info("pair2(odd->even): pp pair:{}, ring_id: {}".format( - pair, ring_id)) + logger.info( + "pair2(odd->even): pp pair:{}, ring_id: {}".format( + pair, ring_id + ) + ) # 4. odd recv from next, even send to prev, 2->1, 0->3 pair = recv_from_next_pair if not even else send_to_prev_pair - ring_id = self.pp_ring_map.get(pair[0] * 1000 + pair[1], - max_ring_id + - 2) # 0->3 not in pp_ring_map + ring_id = self.pp_ring_map.get( + pair[0] * 1000 + pair[1], max_ring_id + 2 + ) # 0->3 not in pp_ring_map self._init_pair_comm(pair, ring_id) if self.pp_rank != 0 and self.pp_rank != self.pp_degree - 1: my_pair.remove(pair) - logger.info("pair3(odd<-even): pp pair:{}, ring_id: {}".format( - pair, ring_id)) + logger.info( + "pair3(odd<-even): pp pair:{}, ring_id: {}".format( + pair, ring_id + ) + ) - assert len(my_pair) == 0, "Current pipeline does not support cross stage communication, " \ - "please check unexpected pair {}".format(my_pair) + assert len(my_pair) == 0, ( + "Current pipeline does not support cross stage communication, " + "please check unexpected pair {}".format(my_pair) + ) def _init_pipeline_comm(self, startup_block): # TODO (JZ-LIANG) to unify pp_rank_ and pp_rank if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None) is None: - self._collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - self.pp_group_endpoints, - self.pp_rank, - self.pp_ring_id, - False, - sync=False) + self._collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + self.pp_group_endpoints, + self.pp_rank, + self.pp_ring_id, + False, + sync=False, + ) if core.is_compiled_with_npu(): self._init_npu_pipeline_comm(startup_block) @@ -762,13 +855,15 @@ class ShardingOptimizer(MetaOptimizerBase): # mp ring if self.mp_degree > 1: - self._collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - self.mp_group_endpoints, - self.mp_rank, - self.mp_ring_id, - False, - sync=False) + self._collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + self.mp_group_endpoints, + self.mp_rank, + self.mp_ring_id, + False, + sync=False, + ) # sharding ring if self.sharding_degree > 1: @@ -779,7 +874,8 @@ class ShardingOptimizer(MetaOptimizerBase): self.sharding_rank, self.sharding_ring_id, False, - sync=False) + sync=False, + ) # pp ring if self.pp_degree > 1: @@ -787,13 +883,15 @@ class ShardingOptimizer(MetaOptimizerBase): # pure dp ring if self.dp_degree > 1: - self._collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - self.dp_group_endpoints, - self.dp_rank, - self.dp_ring_id, - False, - sync=False) + self._collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + self.dp_group_endpoints, + self.dp_rank, + self.dp_ring_id, + False, + sync=False, + ) startup_block._sync_with_cpp() @@ -804,9 +902,12 @@ class ShardingOptimizer(MetaOptimizerBase): # step 3: get broadcast vars self._broadcast_vars = self._shard.find_broadcast_params( - self._main_program.global_block()) + self._main_program.global_block() + ) - def _wait(self, ): + def _wait( + self, + ): endpoints = self.global_endpoints[:] current_endpoint = endpoints[self.global_rank] if self.global_rank == 0: @@ -831,7 +932,7 @@ class ShardingOptimizer(MetaOptimizerBase): segment._end_idx = last_backward_op_idx for op_idx in reversed(range(last_backward_op_idx)): op = block.ops[op_idx] - assert (int(op.attr('op_role')) != int(OpRole.Optimize)) + assert int(op.attr('op_role')) != int(OpRole.Optimize) if self._sharding_segment_strategy == "segment_broadcast_MB": if segment._param_mem >= self._broadcast_MB: segment = self.collect_segment(segment, op_idx, block) @@ -845,21 +946,27 @@ class ShardingOptimizer(MetaOptimizerBase): if ".cast_fp16@GRAD" not in input_name: continue else: - input_name = input_name[:input_name. - find(".cast_fp16@GRAD")] + input_name = input_name[ + : input_name.find(".cast_fp16@GRAD") + ] if input_name in self._backward_remain_anchors: segment = self.collect_segment( - segment, op_idx, block) - assert input_name not in self._forward_remain_anchors, "segment anchor [{}] met twice !".format( - input_name) + segment, op_idx, block + ) + assert ( + input_name not in self._forward_remain_anchors + ), "segment anchor [{}] met twice !".format( + input_name + ) self._backward_remain_anchors.remove(input_name) self._forward_remain_anchors.append(input_name) elif int(op.attr('op_role')) == int(OpRole.Forward): for output_name in op.desc.output_arg_names(): if output_name in self._forward_remain_anchors: segment = self.collect_segment( - segment, op_idx, block) + segment, op_idx, block + ) self._forward_remain_anchors.remove(output_name) # find broadcast vars @@ -875,47 +982,49 @@ class ShardingOptimizer(MetaOptimizerBase): if self._shard.has_param(input_name): broadcast_var_name = input_name else: - broadcast_var_name = unique_name.generate(input_name + - "@BroadCast") + broadcast_var_name = unique_name.generate( + input_name + "@BroadCast" + ) segment._fill_constant_vars.append(broadcast_var_name) # (JZ-LIANG) should use Param base name ? broadcast_var_base_name = input_name if "subprog" in broadcast_var_base_name: # remove suffix - broadcast_var_base_name = broadcast_var_base_name[: - broadcast_var_base_name - .find( - ".subprog" - )] + broadcast_var_base_name = broadcast_var_base_name[ + : broadcast_var_base_name.find(".subprog") + ] - var2broadcast_time[ - broadcast_var_base_name] = var2broadcast_time.get( - broadcast_var_base_name, 0) + 1 + var2broadcast_time[broadcast_var_base_name] = ( + var2broadcast_time.get(broadcast_var_base_name, 0) + 1 + ) segment._param2broadcast[input_name] = broadcast_var_name segment._broadcast_vars.append( - (broadcast_var_name, self._shard.device(input_name))) + (broadcast_var_name, self._shard.device(input_name)) + ) segment._param_mem += get_var_size( - self._main_program.global_block().var(input_name)) + self._main_program.global_block().var(input_name) + ) # find reduce vars if self.pp_degree > 1 and self.pp_allreduce_in_optimize: # place pipeline gradient allreduce in optimize pass else: - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.all_attrs()[OP_ROLE_VAR_KEY] if len(op_role_var) != 0: assert len(op_role_var) % 2 == 0 for i in range(0, len(op_role_var), 2): - param, reduced_grad = op_role_var[i], op_role_var[i - + - 1] + param, reduced_grad = ( + op_role_var[i], + op_role_var[i + 1], + ) segment._allreduce_vars.append(reduced_grad) - assert (reduced_grad - not in self._reduced_grads_to_param) + assert ( + reduced_grad not in self._reduced_grads_to_param + ) self._reduced_grads_to_param[reduced_grad] = param # find cast op @@ -930,29 +1039,40 @@ class ShardingOptimizer(MetaOptimizerBase): self._segments.insert(0, segment) if self._sharding_segment_strategy == "segment_anchors": - assert len( - self._forward_remain_anchors) == 0, "remain anchors {}".format( - self._forward_remain_anchors) - assert len( - self._backward_remain_anchors) == 0, "remain anchors {}".format( - self._backward_remain_anchors) + assert ( + len(self._forward_remain_anchors) == 0 + ), "remain anchors {}".format(self._forward_remain_anchors) + assert ( + len(self._backward_remain_anchors) == 0 + ), "remain anchors {}".format(self._backward_remain_anchors) if self._verbose: - for varname in sorted(var2broadcast_time, - key=var2broadcast_time.get, - reverse=True): - logger.info("Sharding broadcast: [{}] times [{}]".format( - var2broadcast_time[varname], varname)) + for varname in sorted( + var2broadcast_time, key=var2broadcast_time.get, reverse=True + ): + logger.info( + "Sharding broadcast: [{}] times [{}]".format( + var2broadcast_time[varname], varname + ) + ) for idx_ in range(len(self._segments)): logger.info("segment [{}] :".format(idx_)) - logger.info("start op: [{}] [{}]".format( - block.ops[self._segments[idx_]._start_idx].desc.type(), - block.ops[self._segments[idx_]. - _start_idx].desc.input_arg_names())) - logger.info("end op: [{}] [{}]".format( - block.ops[self._segments[idx_]._end_idx].desc.type(), - block.ops[ - self._segments[idx_]._end_idx].desc.input_arg_names())) + logger.info( + "start op: [{}] [{}]".format( + block.ops[self._segments[idx_]._start_idx].desc.type(), + block.ops[ + self._segments[idx_]._start_idx + ].desc.input_arg_names(), + ) + ) + logger.info( + "end op: [{}] [{}]".format( + block.ops[self._segments[idx_]._end_idx].desc.type(), + block.ops[ + self._segments[idx_]._end_idx + ].desc.input_arg_names(), + ) + ) return def _prune_main_program(self, block, shard, rings): @@ -985,17 +1105,18 @@ class ShardingOptimizer(MetaOptimizerBase): input_names = op.desc.input_arg_names() output_names = op.desc.output_arg_names() # FIXME(wangxi): need use grads, pipeline grad is @GRAD@MERGE - if op.type == "c_allreduce_sum" and \ - op.attr('use_model_parallel') is False: - assert (len(output_names) == 1) + if ( + op.type == "c_allreduce_sum" + and op.attr('use_model_parallel') is False + ): + assert len(output_names) == 1 output_name = output_names[0] reduced_grads.append(output_name) # prune optimizer state and param pruned_opti_vars = [] for var_name in list(block.vars.keys()): - if shard.is_opti_var(var_name) and \ - not shard.has_opt_var(var_name): + if shard.is_opti_var(var_name) and not shard.has_opt_var(var_name): pruned_opti_vars.append(var_name) program_deps = ProgramDeps(block, reduced_grads, pruned_opti_vars) @@ -1006,17 +1127,17 @@ class ShardingOptimizer(MetaOptimizerBase): # Prune for idx, op in reversed(list(enumerate(block.ops))): if op.type in [ - "c_allreduce_sum", - "c_sync_comm_stream", - "c_calc_comm_stream", - "c_gen_nccl_id", - "c_comm_init", - 'send_v2', - 'recv_v2', + "c_allreduce_sum", + "c_sync_comm_stream", + "c_calc_comm_stream", + "c_gen_nccl_id", + "c_comm_init", + 'send_v2', + 'recv_v2', ]: pass elif op.type == "conditional_block": - assert (op.desc.has_attr("sub_block")) + assert op.desc.has_attr("sub_block") subblock_idx = op.desc.attr("sub_block").id subblock_deps = program_deps.get_sub_block_deps(subblock_idx) # only prune amp subblock @@ -1032,7 +1153,8 @@ class ShardingOptimizer(MetaOptimizerBase): reversed_output_vars.append(output_name) # prune for sub_op_idx, _ in reversed( - list(enumerate(subblock_deps._block.ops))): + list(enumerate(subblock_deps._block.ops)) + ): if subblock_deps.should_remove_op(sub_op_idx): subblock_deps.remove_op(sub_op_idx) reversed_input_vars = [] @@ -1048,7 +1170,9 @@ class ShardingOptimizer(MetaOptimizerBase): # _should_removed_var: opt state not cur shard if program_deps.should_remove_op(idx): # NOTE(wangxi): need reserve all param in optimizer_sharding - reserved_vars = self._params if self._optimizer_sharding else None + reserved_vars = ( + self._params if self._optimizer_sharding else None + ) program_deps.remove_op(idx, reserved_vars) # NOTE (JZ-LIANG) revise and unify logic here @@ -1059,7 +1183,8 @@ class ShardingOptimizer(MetaOptimizerBase): # remove inputs that not on this card reserved_x = [] for var_name in op.desc.input("X"): - if block.has_var(var_name): reserved_x.append(var_name) + if block.has_var(var_name): + reserved_x.append(var_name) op.desc.set_input('X', reserved_x) block._sync_with_cpp() return @@ -1082,175 +1207,280 @@ class ShardingOptimizer(MetaOptimizerBase): # NOTE (JZ-LIANG) revise and unify logic here # fix the _end_idx for segments[-1] if pp is used. new_end_idx = self._segments[-1]._end_idx - for idx in range(self._segments[-1]._end_idx - 1, - self._segments[-1]._start_idx - 1, -1): + for idx in range( + self._segments[-1]._end_idx - 1, + self._segments[-1]._start_idx - 1, + -1, + ): op = block.ops[idx] if op.type == "fill_constant" or op.type == "sum": - if "MERGED" in op.output_arg_names[0]: new_end_idx = idx + 1 + if "MERGED" in op.output_arg_names[0]: + new_end_idx = idx + 1 elif op.type == "cast": - if "@TMP" in op.output_arg_names[0]: new_end_idx = idx + 1 + if "@TMP" in op.output_arg_names[0]: + new_end_idx = idx + 1 self._segments[-1]._end_idx = new_end_idx if self._segments[-1]._allreduce_vars: shard_allredue_vars = self._shard.filter_grads( - self._segments[-1]._allreduce_vars) - if self.gradient_merge_mode != "sharding_gm" or self._gradient_merge_acc_step <= 1: - if self.hybrid_dp and self.hybrid_dp_mode == "sharding_hybrid_dp" and len( - shard_allredue_vars) >= 1: - insert_sync_comm_ops(block, self._segments[-1]._end_idx, - self.dp_ring_id, shard_allredue_vars) + self._segments[-1]._allreduce_vars + ) + if ( + self.gradient_merge_mode != "sharding_gm" + or self._gradient_merge_acc_step <= 1 + ): + if ( + self.hybrid_dp + and self.hybrid_dp_mode == "sharding_hybrid_dp" + and len(shard_allredue_vars) >= 1 + ): + insert_sync_comm_ops( + block, + self._segments[-1]._end_idx, + self.dp_ring_id, + shard_allredue_vars, + ) insert_allreduce_ops( block, self._segments[-1]._end_idx, self.dp_ring_id, shard_allredue_vars, - user_defined_strategy=self.user_defined_strategy) + user_defined_strategy=self.user_defined_strategy, + ) # gradient merge - elif self.gradient_merge_mode == "sharding_gm" and self._gradient_merge_acc_step > 1: + elif ( + self.gradient_merge_mode == "sharding_gm" + and self._gradient_merge_acc_step > 1 + ): self.create_persistable_gradients_and_insert_merge_ops( - block, self._startup_program.global_block(), - self._segments[-1]._end_idx, shard_allredue_vars, - self._shard) - - insert_sync_comm_ops(block, self._segments[-1]._end_idx, - self.sharding_ring_id, - self._segments[-1]._allreduce_vars) + block, + self._startup_program.global_block(), + self._segments[-1]._end_idx, + shard_allredue_vars, + self._shard, + ) + + insert_sync_comm_ops( + block, + self._segments[-1]._end_idx, + self.sharding_ring_id, + self._segments[-1]._allreduce_vars, + ) # allreduce --> reduce - insert_reduce_ops(block, - self._segments[-1]._end_idx, - self.sharding_ring_id, - self._segments[-1]._allreduce_vars, - self._shard, - op_role=OpRole.Backward, - use_calc_stream=False) + insert_reduce_ops( + block, + self._segments[-1]._end_idx, + self.sharding_ring_id, + self._segments[-1]._allreduce_vars, + self._shard, + op_role=OpRole.Backward, + use_calc_stream=False, + ) for idx, segment in reversed(list(enumerate(self._segments))): - allreduce_vars = self._segments[ - idx - 1]._allreduce_vars if idx > 0 else [] - broadcast_vars = self._segments[ - idx + - 1]._broadcast_vars if idx < len(self._segments) - 1 else [] - fill_constant_vars = self._segments[ - idx + - 2]._fill_constant_vars if idx < len(self._segments) - 2 else [] - cast_ops = self._segments[ - idx + 2]._cast_ops if idx < len(self._segments) - 2 else {} + allreduce_vars = ( + self._segments[idx - 1]._allreduce_vars if idx > 0 else [] + ) + broadcast_vars = ( + self._segments[idx + 1]._broadcast_vars + if idx < len(self._segments) - 1 + else [] + ) + fill_constant_vars = ( + self._segments[idx + 2]._fill_constant_vars + if idx < len(self._segments) - 2 + else [] + ) + cast_ops = ( + self._segments[idx + 2]._cast_ops + if idx < len(self._segments) - 2 + else {} + ) for op_idx in reversed(range(segment._start_idx, segment._end_idx)): op = block.ops[op_idx] for input_name in op.desc.input_arg_names(): - if input_name in segment._param2broadcast and \ - input_name != segment._param2broadcast[input_name]: - op._rename_input(input_name, - segment._param2broadcast[input_name]) + if ( + input_name in segment._param2broadcast + and input_name != segment._param2broadcast[input_name] + ): + op._rename_input( + input_name, segment._param2broadcast[input_name] + ) for param_name, broadcast_name in segment._param2broadcast.items(): if param_name != broadcast_name: block.create_var( name=broadcast_name, - shape=self._main_program.global_block().var( - param_name).shape, - dtype=self._main_program.global_block().var( - param_name).dtype, - persistable=False) + shape=self._main_program.global_block() + .var(param_name) + .shape, + dtype=self._main_program.global_block() + .var(param_name) + .dtype, + persistable=False, + ) # step1: remove cast ops block._sync_with_cpp() segment._end_idx += FP16Utils.remove_cast_op( - block, self._params, segment, 0) + block, self._params, segment, 0 + ) # step2: add Sync ops shard_allredue_vars = self._shard.filter_grads(allreduce_vars) - if self.gradient_merge_mode != "sharding_gm" or self._gradient_merge_acc_step <= 1: - if self.hybrid_dp and self.hybrid_dp_mode == "sharding_hybrid_dp" and len( - shard_allredue_vars) >= 1: - insert_sync_comm_ops(block, segment._end_idx, - self.dp_ring_id, shard_allredue_vars) + if ( + self.gradient_merge_mode != "sharding_gm" + or self._gradient_merge_acc_step <= 1 + ): + if ( + self.hybrid_dp + and self.hybrid_dp_mode == "sharding_hybrid_dp" + and len(shard_allredue_vars) >= 1 + ): + insert_sync_comm_ops( + block, + segment._end_idx, + self.dp_ring_id, + shard_allredue_vars, + ) broad_cast_vars = [x[0] for x in broadcast_vars] if len(broad_cast_vars) > 0: - insert_sync_comm_ops(block, segment._end_idx, - self.sharding_ring_id, - broad_cast_vars) + insert_sync_comm_ops( + block, + segment._end_idx, + self.sharding_ring_id, + broad_cast_vars, + ) else: comm_dep_vars = allreduce_vars + [ x[0] for x in broadcast_vars ] if len(comm_dep_vars) > 0: - insert_sync_comm_ops(block, segment._end_idx, - self.sharding_ring_id, - comm_dep_vars) + insert_sync_comm_ops( + block, + segment._end_idx, + self.sharding_ring_id, + comm_dep_vars, + ) # gradient merge - elif self.gradient_merge_mode == "sharding_gm" and self._gradient_merge_acc_step > 1: + elif ( + self.gradient_merge_mode == "sharding_gm" + and self._gradient_merge_acc_step > 1 + ): broad_cast_vars = [x[0] for x in broadcast_vars] if len(broad_cast_vars) > 0: - insert_sync_comm_ops(block, segment._end_idx, - self.sharding_ring_id, broad_cast_vars) + insert_sync_comm_ops( + block, + segment._end_idx, + self.sharding_ring_id, + broad_cast_vars, + ) - calc_dep_vars = fill_constant_vars + [ - k for k, v in cast_ops.items() - ] + self._segments[idx]._allreduce_vars + calc_dep_vars = ( + fill_constant_vars + + [k for k, v in cast_ops.items()] + + self._segments[idx]._allreduce_vars + ) if len(calc_dep_vars) > 0: - insert_sync_calc_op(block, segment._end_idx, - [calc_dep_vars[-1]]) + insert_sync_calc_op( + block, segment._end_idx, [calc_dep_vars[-1]] + ) # step3: insert `fill_constant` ops - insert_fill_constant_ops(block, segment._end_idx, - fill_constant_vars) + insert_fill_constant_ops( + block, segment._end_idx, fill_constant_vars + ) # step4: add `cast` ops insert_cast_ops(block, segment._end_idx, cast_ops) # step5: add broadcast ops # gradient merge - if self.gradient_merge_mode == "sharding_gm" and self._gradient_merge_acc_step > 1: + if ( + self.gradient_merge_mode == "sharding_gm" + and self._gradient_merge_acc_step > 1 + ): self.create_persistable_gradients_and_insert_merge_ops( - block, self._startup_program.global_block(), - segment._start_idx, shard_allredue_vars, self._shard) + block, + self._startup_program.global_block(), + segment._start_idx, + shard_allredue_vars, + self._shard, + ) - insert_broadcast_ops(block, segment._start_idx, - self.sharding_ring_id, broadcast_vars) + insert_broadcast_ops( + block, segment._start_idx, self.sharding_ring_id, broadcast_vars + ) # step6: add all_reduce ops # dp - if self.gradient_merge_mode != "sharding_gm" or self._gradient_merge_acc_step <= 1: - if self.hybrid_dp and self.hybrid_dp_mode == "sharding_hybrid_dp" and len( - shard_allredue_vars) >= 1: + if ( + self.gradient_merge_mode != "sharding_gm" + or self._gradient_merge_acc_step <= 1 + ): + if ( + self.hybrid_dp + and self.hybrid_dp_mode == "sharding_hybrid_dp" + and len(shard_allredue_vars) >= 1 + ): insert_allreduce_ops( block, segment._start_idx, self.dp_ring_id, shard_allredue_vars, - user_defined_strategy=self.user_defined_strategy) - insert_sync_comm_ops(block, segment._start_idx, - self.sharding_ring_id, allreduce_vars) + user_defined_strategy=self.user_defined_strategy, + ) + insert_sync_comm_ops( + block, + segment._start_idx, + self.sharding_ring_id, + allreduce_vars, + ) # gradient merge - elif self.gradient_merge_mode == "sharding_gm" and self._gradient_merge_acc_step > 1: - insert_sync_comm_ops(block, segment._start_idx, - self.sharding_ring_id, allreduce_vars) + elif ( + self.gradient_merge_mode == "sharding_gm" + and self._gradient_merge_acc_step > 1 + ): + insert_sync_comm_ops( + block, + segment._start_idx, + self.sharding_ring_id, + allreduce_vars, + ) # sharding # allreduce --> reduce # TODO temp change if len(allreduce_vars) > 0: - insert_reduce_ops(block, - segment._start_idx, - self.sharding_ring_id, - allreduce_vars, - self._shard, - op_role=OpRole.Backward, - use_calc_stream=False) + insert_reduce_ops( + block, + segment._start_idx, + self.sharding_ring_id, + allreduce_vars, + self._shard, + op_role=OpRole.Backward, + use_calc_stream=False, + ) block._sync_with_cpp() if self._segments[0]._broadcast_vars: broadcast_vars = [x[0] for x in self._segments[0]._broadcast_vars] - insert_sync_comm_ops(block, self._segments[0]._start_idx, - self.sharding_ring_id, broadcast_vars) - insert_broadcast_ops(block, self._segments[0]._start_idx, - self.sharding_ring_id, - self._segments[0]._broadcast_vars) + insert_sync_comm_ops( + block, + self._segments[0]._start_idx, + self.sharding_ring_id, + broadcast_vars, + ) + insert_broadcast_ops( + block, + self._segments[0]._start_idx, + self.sharding_ring_id, + self._segments[0]._broadcast_vars, + ) fill_constant_vars = [] for x in self._segments[:2]: @@ -1264,12 +1494,14 @@ class ShardingOptimizer(MetaOptimizerBase): calc_deps_vars = fill_constant_vars + [k for k, v in cast_ops.items()] if fill_constant_vars or cast_ops: - insert_sync_calc_op(block, self._segments[0]._start_idx, - [calc_deps_vars[-1]]) + insert_sync_calc_op( + block, self._segments[0]._start_idx, [calc_deps_vars[-1]] + ) if fill_constant_vars: - insert_fill_constant_ops(block, self._segments[0]._start_idx, - fill_constant_vars) + insert_fill_constant_ops( + block, self._segments[0]._start_idx, fill_constant_vars + ) if cast_ops: insert_cast_ops(block, self._segments[0]._start_idx, cast_ops) @@ -1283,7 +1515,7 @@ class ShardingOptimizer(MetaOptimizerBase): continue if self._optimizer_sharding and shard.is_param(output_name): continue - #TODO why do we remove op, when only one var is removed + # TODO why do we remove op, when only one var is removed block._remove_op(idx, sync=False) break @@ -1312,16 +1544,29 @@ class ShardingOptimizer(MetaOptimizerBase): self.global_rank = self.role_maker._worker_index() self.global_endpoints = self.role_maker._get_trainer_endpoints() self.current_endpoint = self.global_endpoints[self.global_rank] - self._collective_helper = CollectiveHelper(self.role_maker, - nrings=self._nrings_sharding) - assert self.global_word_size % self.mp_degree == 0, \ - "global_word_size: {} should be divisible to the mp_degree: {}".format(self.global_word_size, self.mp_degree) - assert self.global_word_size % self.sharding_degree == 0, \ - "global_word_size: {} should be divisible to the sharding_degree: {}".format(self.global_word_size, self.sharding_degree) - assert self.global_word_size % self.pp_degree == 0, \ - "global_word_size: {} should be divisible to the pp_degree: {}".format(self.global_word_size, self.pp_degree) - assert self.global_word_size % self.dp_degree == 0, \ - "global_word_size: {} should be divisible to the dp_degree: {}".format(self.global_word_size, self.dp_degree) + self._collective_helper = CollectiveHelper( + self.role_maker, nrings=self._nrings_sharding + ) + assert ( + self.global_word_size % self.mp_degree == 0 + ), "global_word_size: {} should be divisible to the mp_degree: {}".format( + self.global_word_size, self.mp_degree + ) + assert ( + self.global_word_size % self.sharding_degree == 0 + ), "global_word_size: {} should be divisible to the sharding_degree: {}".format( + self.global_word_size, self.sharding_degree + ) + assert ( + self.global_word_size % self.pp_degree == 0 + ), "global_word_size: {} should be divisible to the pp_degree: {}".format( + self.global_word_size, self.pp_degree + ) + assert ( + self.global_word_size % self.dp_degree == 0 + ), "global_word_size: {} should be divisible to the dp_degree: {}".format( + self.global_word_size, self.dp_degree + ) # mp group if self.mp_degree > 1: @@ -1329,14 +1574,16 @@ class ShardingOptimizer(MetaOptimizerBase): self.mp_rank = self.global_rank % self.mp_degree self.mp_group_id = self.global_rank // self.mp_degree self.mp_group_endpoints = [ - ep for idx, ep in enumerate(self.global_endpoints) + ep + for idx, ep in enumerate(self.global_endpoints) if idx // self.mp_degree == self.mp_group_id ] assert self.current_endpoint in self.mp_group_endpoints - assert len( - self.mp_group_endpoints - ) == self.mp_degree, "num of mp worker in group is [{}], but mp group size is [{}]".format( - len(self.mp_group_endpoints), self.mp_degree) + assert ( + len(self.mp_group_endpoints) == self.mp_degree + ), "num of mp worker in group is [{}], but mp group size is [{}]".format( + len(self.mp_group_endpoints), self.mp_degree + ) else: self.mp_degree = 1 self.mp_ring_id = -1 @@ -1347,23 +1594,28 @@ class ShardingOptimizer(MetaOptimizerBase): # sharding if self.sharding_degree > 1: self.sharding_ring_id = 1 - self.sharding_rank = (self.global_rank // - self.mp_degree) % self.sharding_degree - self.sharding_group_id = self.global_rank // (self.mp_degree * - self.sharding_degree) + self.sharding_rank = ( + self.global_rank // self.mp_degree + ) % self.sharding_degree + self.sharding_group_id = self.global_rank // ( + self.mp_degree * self.sharding_degree + ) # mp + sharding + ... if self.mp_degree > 1: self.sharding_group_endpoints = [ - ep for idx, ep in enumerate(self.global_endpoints) - if (idx // (self.mp_degree * self.sharding_degree)) == self. - sharding_group_id and idx % self.mp_degree == self.mp_rank + ep + for idx, ep in enumerate(self.global_endpoints) + if (idx // (self.mp_degree * self.sharding_degree)) + == self.sharding_group_id + and idx % self.mp_degree == self.mp_rank ] # sharding + ... else: self.sharding_group_endpoints = [ - ep for idx, ep in enumerate(self.global_endpoints) - if (idx // (self.mp_degree * self.sharding_degree) - ) == self.sharding_group_id + ep + for idx, ep in enumerate(self.global_endpoints) + if (idx // (self.mp_degree * self.sharding_degree)) + == self.sharding_group_id ] assert self.current_endpoint in self.sharding_group_endpoints else: @@ -1378,20 +1630,28 @@ class ShardingOptimizer(MetaOptimizerBase): self.pp_pair_ring_id = 20 # pipeline global ring_id set to 4 for sharding0, mp1, dp2, global3 self.pp_ring_id = 4 - self.pp_rank = self.global_rank // (self.sharding_degree * - self.mp_degree) % self.pp_degree + self.pp_rank = ( + self.global_rank + // (self.sharding_degree * self.mp_degree) + % self.pp_degree + ) # (NOTE): Already adjust for (outter-pure) dp self.pp_group_id = self.global_rank // ( - self.mp_degree * self.sharding_degree * self.pp_degree) + self.mp_degree * self.sharding_degree * self.pp_degree + ) pp_first_stage_idx = self.global_rank % ( - self.sharding_degree * self.mp_degree) + self.pp_group_id * ( - self.mp_degree * self.sharding_degree * self.pp_degree) + self.sharding_degree * self.mp_degree + ) + self.pp_group_id * ( + self.mp_degree * self.sharding_degree * self.pp_degree + ) pp_stage_offset = self.sharding_degree * self.mp_degree self.pp_group_endpoints = [] for i in range(self.pp_degree): self.pp_group_endpoints.append( - self.global_endpoints[pp_first_stage_idx + - pp_stage_offset * i]) + self.global_endpoints[ + pp_first_stage_idx + pp_stage_offset * i + ] + ) assert self.current_endpoint in self.pp_group_endpoints else: self.pp_ring_id = -1 @@ -1407,29 +1667,48 @@ class ShardingOptimizer(MetaOptimizerBase): # sharding-hybrid-dp as one senario of outter-pure-dp local_pp_degree = self.pp_degree if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None): - assert self.pp_degree == 2, ("For manually set pipeline, only " - "pp_degree = 2 is supported.") - assert self.global_word_size == self.mp_degree * self.sharding_degree * self.dp_degree, \ - "global work size [{}], mp_degree [{}], sharding_degree [{}], dp_degree [{}].".format( - self.global_word_size, self.mp_degree, self.sharding_degree, self.dp_degree) + assert self.pp_degree == 2, ( + "For manually set pipeline, only " "pp_degree = 2 is supported." + ) + assert ( + self.global_word_size + == self.mp_degree * self.sharding_degree * self.dp_degree + ), "global work size [{}], mp_degree [{}], sharding_degree [{}], dp_degree [{}].".format( + self.global_word_size, + self.mp_degree, + self.sharding_degree, + self.dp_degree, + ) local_pp_degree = 1 else: - assert self.global_word_size == self.mp_degree * self.sharding_degree * self.pp_degree * self.dp_degree, "mp_degree: [{}], sharding_degree: [{}], pp_degree: [{}], dp_degree: [{}]; BUT global nrank: [{}]".format( - self.mp_degree, self.sharding_degree, self.pp_degree, - self.dp_degree, self.global_word_size) + assert ( + self.global_word_size + == self.mp_degree + * self.sharding_degree + * self.pp_degree + * self.dp_degree + ), "mp_degree: [{}], sharding_degree: [{}], pp_degree: [{}], dp_degree: [{}]; BUT global nrank: [{}]".format( + self.mp_degree, + self.sharding_degree, + self.pp_degree, + self.dp_degree, + self.global_word_size, + ) if self.dp_degree > 1: self.dp_ring_id = 2 self.dp_rank = self.global_rank // ( - self.sharding_degree * self.mp_degree * local_pp_degree) + self.sharding_degree * self.mp_degree * local_pp_degree + ) dp_first_rank_idx = self.global_rank % ( - self.sharding_degree * self.mp_degree * local_pp_degree) - dp_offset = (self.sharding_degree * self.mp_degree * - local_pp_degree) + self.sharding_degree * self.mp_degree * local_pp_degree + ) + dp_offset = self.sharding_degree * self.mp_degree * local_pp_degree self.dp_group_endpoints = [] for i in range(self.dp_degree): self.dp_group_endpoints.append( - self.global_endpoints[dp_first_rank_idx + dp_offset * i]) + self.global_endpoints[dp_first_rank_idx + dp_offset * i] + ) assert self.current_endpoint in self.dp_group_endpoints logger.info("Hybrid DP mode turn on !") else: @@ -1458,8 +1737,9 @@ class ShardingOptimizer(MetaOptimizerBase): logger.info("sharding group size: {}".format(self.sharding_degree)) logger.info("sharding rank: {}".format(self.sharding_rank)) logger.info("sharding group id: {}".format(self.sharding_group_id)) - logger.info("sharding group endpoints: {}".format( - self.sharding_group_endpoints)) + logger.info( + "sharding group endpoints: {}".format(self.sharding_group_endpoints) + ) logger.info("sharding ring id: {}".format(self.sharding_ring_id)) logger.info("#####" * 6) @@ -1472,15 +1752,15 @@ class ShardingOptimizer(MetaOptimizerBase): logger.info("pure dp group size: {}".format(self.dp_degree)) logger.info("pure dp rank: {}".format(self.dp_rank)) - logger.info("pure dp group endpoints: {}".format( - self.dp_group_endpoints)) + logger.info( + "pure dp group endpoints: {}".format(self.dp_group_endpoints) + ) logger.info("pure dp ring id: {}".format(self.dp_ring_id)) logger.info("#####" * 6) return def _recreate_not_persist_param_as_var(self): - def recreate_not_persist_param_as_var(program): block = program.global_block() params = block.all_parameters() @@ -1504,14 +1784,16 @@ class ShardingOptimizer(MetaOptimizerBase): is_distributed = param.is_distributed block._remove_var(name, sync=False) - var = block.create_var(name=name, - shape=shape, - dtype=dtype, - type=type, - lod_level=lod_level, - stop_gradient=stop_gradient, - trainable=trainable, - persistable=False) + var = block.create_var( + name=name, + shape=shape, + dtype=dtype, + type=type, + lod_level=lod_level, + stop_gradient=stop_gradient, + trainable=trainable, + persistable=False, + ) if have_dist_attr: var.is_distributed = is_distributed @@ -1547,7 +1829,8 @@ class ShardingOptimizer(MetaOptimizerBase): broadcast_params.add(op.desc.output_arg_names()[0]) for param in params_name: - if param in broadcast_params: continue + if param in broadcast_params: + continue rings = [] # need sync not distributed param in mp group @@ -1557,30 +1840,37 @@ class ShardingOptimizer(MetaOptimizerBase): rings.append(self.dp_ring_id) for ring in rings: - startup_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + startup_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) startup_block._sync_with_cpp() # sharding gradient merge def create_persistable_gradients_and_insert_merge_ops( - self, main_block, startup_block, insert_idx, grad_names, shard): + self, main_block, startup_block, insert_idx, grad_names, shard + ): for grad_name in grad_names: - assert get_grad_device( - grad_name, shard - ) == shard.worker_idx, "try to merge gradient not belong to current shard: [{}]".format( - grad_name) + assert ( + get_grad_device(grad_name, shard) == shard.worker_idx + ), "try to merge gradient not belong to current shard: [{}]".format( + grad_name + ) persistable_grad_name = grad_name + '@GradiantMerge' - assert grad_name not in self._grad2merged_grad, "grad [{}] already in grad2merged_grad, maybe you meet sharing weight case !".format( - grad_name) + assert ( + grad_name not in self._grad2merged_grad + ), "grad [{}] already in grad2merged_grad, maybe you meet sharing weight case !".format( + grad_name + ) self._grad2merged_grad[grad_name] = persistable_grad_name grad_var = main_block.var(grad_name) # create var @@ -1588,36 +1878,38 @@ class ShardingOptimizer(MetaOptimizerBase): name=persistable_grad_name, shape=grad_var.shape, dtype=grad_var.dtype, - persistable=True) + persistable=True, + ) startup_gradient_merge_var = startup_block.create_var( name=persistable_grad_name, shape=grad_var.shape, dtype=grad_var.dtype, - persistable=True) + persistable=True, + ) # merge gradient main_block._insert_op_without_sync( insert_idx, type="elementwise_add", - inputs={ - 'X': grad_name, - 'Y': gradient_merge_var - }, + inputs={'X': grad_name, 'Y': gradient_merge_var}, outputs={'Out': gradient_merge_var}, attrs={ 'axis': -1, 'use_mkldnn': False, - OP_ROLE_KEY: OpRole.Backward - }) + OP_ROLE_KEY: OpRole.Backward, + }, + ) # startup initialization - startup_block.append_op(type="fill_constant", - outputs={"Out": startup_gradient_merge_var}, - attrs={ - "shape": grad_var.shape, - "dtype": grad_var.dtype, - "value": float(0), - }) + startup_block.append_op( + type="fill_constant", + outputs={"Out": startup_gradient_merge_var}, + attrs={ + "shape": grad_var.shape, + "dtype": grad_var.dtype, + "value": float(0), + }, + ) main_block._sync_with_cpp() startup_block._sync_with_cpp() @@ -1630,14 +1922,17 @@ class ShardingOptimizer(MetaOptimizerBase): value=int(self._gradient_merge_acc_step), dtype='int32', persistable=True, - force_cpu=True) + force_cpu=True, + ) - zero_var = layers.create_global_var(name="gradient_merge_zero", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + zero_var = layers.create_global_var( + name="gradient_merge_zero", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) # Add step var & cond var current_step_var = layers.create_global_var( @@ -1646,42 +1941,40 @@ class ShardingOptimizer(MetaOptimizerBase): value=int(0), dtype='int32', persistable=True, - force_cpu=True) + force_cpu=True, + ) - cond_var = main_block.create_var(name="gradient_merge_cond", - shape=[1], - dtype='bool') + cond_var = main_block.create_var( + name="gradient_merge_cond", shape=[1], dtype='bool' + ) with device_guard("cpu"): # step_var = (step_var + 1) % k_step - main_block.append_op(type='increment', - inputs={'X': [current_step_var]}, - outputs={'Out': [current_step_var]}, - attrs={ - 'step': float(1), - OP_ROLE_KEY: OpRole.Optimize - }) - - main_block.append_op(type='elementwise_mod', - inputs={ - 'X': current_step_var, - 'Y': acc_step_var - }, - outputs={'Out': current_step_var}, - attrs={ - 'axis': -1, - OP_ROLE_KEY: OpRole.Optimize, - 'use_mkldnn': False - }) + main_block.append_op( + type='increment', + inputs={'X': [current_step_var]}, + outputs={'Out': [current_step_var]}, + attrs={'step': float(1), OP_ROLE_KEY: OpRole.Optimize}, + ) + + main_block.append_op( + type='elementwise_mod', + inputs={'X': current_step_var, 'Y': acc_step_var}, + outputs={'Out': current_step_var}, + attrs={ + 'axis': -1, + OP_ROLE_KEY: OpRole.Optimize, + 'use_mkldnn': False, + }, + ) # cond_var = (step_var == 0) - main_block.append_op(type='equal', - inputs={ - 'X': current_step_var, - 'Y': zero_var - }, - outputs={'Out': cond_var}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + main_block.append_op( + type='equal', + inputs={'X': current_step_var, 'Y': zero_var}, + outputs={'Out': cond_var}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) # paddle.static.Print(current_step_var, message="in FWBW last conditional") return cond_var @@ -1708,35 +2001,37 @@ class ShardingOptimizer(MetaOptimizerBase): # allreduce grad@gradientmerge if self.hybrid_dp: - assert self.dp_ring_id >= 0, "dp_ring_id should larger than 0 when in sharding&DP mode" + assert ( + self.dp_ring_id >= 0 + ), "dp_ring_id should larger than 0 when in sharding&DP mode" for grad, merged_grad in self._grad2merged_grad.items(): merged_grad_var = main_block.var(merged_grad) - cur_block.append_op(type='c_allreduce_sum', - inputs={'X': merged_grad_var}, - outputs={'Out': merged_grad_var}, - attrs={ - 'ring_id': self.dp_ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) + cur_block.append_op( + type='c_allreduce_sum', + inputs={'X': merged_grad_var}, + outputs={'Out': merged_grad_var}, + attrs={ + 'ring_id': self.dp_ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) # grad@gradientmerge / acc_step for grad, merged_grad in self._grad2merged_grad.items(): # grad /= k_steps merged_grad_var = main_block.var(merged_grad) - cur_block.append_op(type='scale', - inputs={'X': merged_grad_var}, - outputs={'Out': merged_grad_var}, - attrs={ - 'scale': - 1.0 / float(self._gradient_merge_acc_step), - 'bias': - 0.0, - 'bias_after_scale': - False, - OP_ROLE_KEY: - OpRole.Optimize - }) + cur_block.append_op( + type='scale', + inputs={'X': merged_grad_var}, + outputs={'Out': merged_grad_var}, + attrs={ + 'scale': 1.0 / float(self._gradient_merge_acc_step), + 'bias': 0.0, + 'bias_after_scale': False, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) # re-create optimize ops already_moved_var_names = [] @@ -1747,15 +2042,19 @@ class ShardingOptimizer(MetaOptimizerBase): for input_name in new_op_desc.input_arg_names(): if input_name in self._grad2merged_grad: new_op_desc._rename_input( - input_name, self._grad2merged_grad[input_name]) + input_name, self._grad2merged_grad[input_name] + ) for output_name in new_op_desc.output_arg_names(): if output_name in self._grad2merged_grad: new_op_desc._rename_output( - output_name, self._grad2merged_grad[output_name]) + output_name, self._grad2merged_grad[output_name] + ) # move non temp optimize vars from block0 to cond block - if output_name not in already_moved_var_names and output_name not in self._grad2merged_grad.keys( + if ( + output_name not in already_moved_var_names + and output_name not in self._grad2merged_grad.keys() ): var_ = self._main_program.global_block().var(output_name) if not var_.persistable: @@ -1764,11 +2063,14 @@ class ShardingOptimizer(MetaOptimizerBase): shape_ = var_.shape type_ = var_.dtype self._main_program.global_block()._remove_var( - var_.name, sync=False) - self.cond_block.create_var(name=name_, - shape=shape_, - dtype=type_, - persistable=False) + var_.name, sync=False + ) + self.cond_block.create_var( + name=name_, + shape=shape_, + dtype=type_, + persistable=False, + ) already_moved_var_names.append(name_) self._main_program.global_block()._sync_with_cpp() @@ -1777,14 +2079,16 @@ class ShardingOptimizer(MetaOptimizerBase): # fill zero to grad@gradientmerge for grad, merged_grad in self._grad2merged_grad.items(): merged_grad_var = main_block.var(merged_grad) - cur_block.append_op(type='fill_constant', - outputs={'Out': merged_grad_var}, - attrs={ - "shape": merged_grad_var.shape, - "dtype": merged_grad_var.dtype, - "value": float(0), - OP_ROLE_KEY: OpRole.Optimize - }) + cur_block.append_op( + type='fill_constant', + outputs={'Out': merged_grad_var}, + attrs={ + "shape": merged_grad_var.shape, + "dtype": merged_grad_var.dtype, + "value": float(0), + OP_ROLE_KEY: OpRole.Optimize, + }, + ) # lr_var = main_block.var("gradient_merge_current_step") # paddle.static.Print(lr_var, message="in OPTIMIZE last conditional") @@ -1796,7 +2100,10 @@ class ShardingOptimizer(MetaOptimizerBase): create cond block """ - if self.gradient_merge_mode != "sharding_gm" or self._gradient_merge_acc_step <= 1: + if ( + self.gradient_merge_mode != "sharding_gm" + or self._gradient_merge_acc_step <= 1 + ): return main_block = self._main_program.global_block() @@ -1815,7 +2122,8 @@ class ShardingOptimizer(MetaOptimizerBase): main_block._remove_op(op_idx, sync=False) tmp_copy_block._sync_with_cpp() self.original_optimize_ops_desc = list( - reversed(self.original_optimize_ops_desc)) + reversed(self.original_optimize_ops_desc) + ) # back to block 0 self._main_program._rollback() @@ -1832,18 +2140,17 @@ class ShardingOptimizer(MetaOptimizerBase): # cond op step_scope = self._main_program.global_block().create_var( - type=core.VarDesc.VarType.STEP_SCOPES) + type=core.VarDesc.VarType.STEP_SCOPES + ) conditional_block_op = self._main_program.global_block().append_op( type='conditional_block', inputs={ 'Cond': cond, 'Input': [], }, - outputs={ - 'Out': [], - 'Scope': [step_scope] - }, + outputs={'Out': [], 'Scope': [step_scope]}, attrs={ 'sub_block': cond_block, 'is_scalar_condition': True, - }) + }, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py index 5f01552c71b..0cd86ad08bd 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py @@ -13,13 +13,20 @@ import paddle.fluid as fluid from .meta_optimizer_base import MetaOptimizerBase -from .common import CollectiveHelper, OP_ROLE_KEY, OP_ROLE_VAR_KEY, OpRole, is_backward_op, is_loss_grad_op, is_optimizer_op +from .common import ( + CollectiveHelper, + OP_ROLE_KEY, + OP_ROLE_VAR_KEY, + OpRole, + is_backward_op, + is_loss_grad_op, + is_optimizer_op, +) __all__ = [] class TensorParallelOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(TensorParallelOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -36,13 +43,15 @@ class TensorParallelOptimizer(MetaOptimizerBase): self.global_ring_id = 1 self.dp_ring_id = 2 - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(TensorParallelOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(TensorParallelOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) self.mp_degree = user_defined_strategy.tensor_parallel_configs[ - 'tensor_parallel_degree'] + 'tensor_parallel_degree' + ] def _can_apply(self): if not self.role_maker._is_collective: @@ -69,23 +78,25 @@ class TensorParallelOptimizer(MetaOptimizerBase): if param.is_distributed and mp_mode: continue - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - OP_ROLE_KEY: OpRole.Forward - }) - - if not param: return # no parameter on this device - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + + if not param: + return # no parameter on this device + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Forward}, + ) def _get_process_group_info(self): # global ring info @@ -98,7 +109,8 @@ class TensorParallelOptimizer(MetaOptimizerBase): self.mp_nranks = self.mp_degree mp_group = self.rank // self.mp_degree self.mp_endpoints = [ - self.endpoints[i] for i in range(self.global_nranks) + self.endpoints[i] + for i in range(self.global_nranks) if i // self.mp_degree == mp_group ] @@ -117,33 +129,47 @@ class TensorParallelOptimizer(MetaOptimizerBase): collective_helper = CollectiveHelper(self.role_maker, wait_port=False) # Create global ring for all gpus - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - self.global_endpoints, - self.global_rank, - self.global_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + self.global_endpoints, + self.global_rank, + self.global_ring_id, + True, + self.global_ring_id, + True, + ) # Create model parallel ring for all gpus - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - self.mp_endpoints, self.mp_rank, - self.mp_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + self.mp_endpoints, + self.mp_rank, + self.mp_ring_id, + True, + self.global_ring_id, + True, + ) self._broadcast_params(self.mp_ring_id, mp_mode=True) # Create dp rings if self.nranks > self.mp_degree: collective_helper._init_communicator( - self.startup_program, self.current_endpoint, self.dp_endpoints, - self.dp_rank, self.dp_ring_id, True, self.global_ring_id, True) + self.startup_program, + self.current_endpoint, + self.dp_endpoints, + self.dp_rank, + self.dp_ring_id, + True, + self.global_ring_id, + True, + ) self._broadcast_params(self.dp_ring_id, mp_mode=False) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self.endpoints = self.role_maker._get_trainer_endpoints() self.current_endpoint = self.endpoints[self.role_maker._worker_index()] self.startup_program = startup_program @@ -151,7 +177,8 @@ class TensorParallelOptimizer(MetaOptimizerBase): self.startup_program = fluid.default_startup_program() optimize_ops, params_grads = self.inner_opt.minimize( - loss, self.startup_program, parameter_list, no_grad_set) + loss, self.startup_program, parameter_list, no_grad_set + ) self.main_program = loss.block.program self.nranks = len(self.endpoints) @@ -180,14 +207,16 @@ class TensorParallelOptimizer(MetaOptimizerBase): for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): loss_grad_var = block.vars[op.output_arg_names[0]] - block._insert_op(idx + 1, - type='scale', - inputs={'X': loss_grad_var}, - outputs={'Out': loss_grad_var}, - attrs={ - 'scale': 1.0 / dp_degree, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / dp_degree, + OP_ROLE_KEY: OpRole.Backward, + }, + ) break def _insert_allreduce_ops(self, loss, ring_id): @@ -205,33 +234,36 @@ class TensorParallelOptimizer(MetaOptimizerBase): grad = block.vars[op_role_var[i + 1]] if offset == idx: offset += 1 - block._insert_op(offset, - type='c_sync_calc_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={OP_ROLE_KEY: OpRole.Backward}) + block._insert_op( + offset, + type='c_sync_calc_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={OP_ROLE_KEY: OpRole.Backward}, + ) offset += 1 - block._insert_op(offset, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward, + }, + ) if grad is None: return for idx, op in list(enumerate(block.ops)): if is_optimizer_op(op): - block._insert_op(idx, - type='c_sync_comm_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx, + type='c_sync_comm_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Backward}, + ) break diff --git a/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py b/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py index f5b8660bd88..9386893728a 100644 --- a/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py +++ b/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py @@ -18,10 +18,10 @@ __all__ = [] class MetaParallelBase(Layer): - def __init__(self, layers, hcg, strategy): - super(MetaParallelBase, - self).__init__(layers.full_name() + "_meta_parallel_base") + super(MetaParallelBase, self).__init__( + layers.full_name() + "_meta_parallel_base" + ) self._layers = layers self._hcg = hcg self._strategy = strategy diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py index f1a81239180..fec5005627b 100755 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py @@ -54,7 +54,6 @@ __all__ = [] class LayerDesc(object): - def __init__(self, layer_func, *inputs, **kwargs): self.layer_func = layer_func self.inputs = inputs @@ -62,25 +61,28 @@ class LayerDesc(object): if not issubclass(layer_func, Layer): raise TypeError( - "The input(layer_func) should be a derived class of Layer.") + "The input(layer_func) should be a derived class of Layer." + ) def build_layer(self): return self.layer_func(*self.inputs, **self.kwargs) def __repr__(self): - return layer_to_str(self.layer_func.__name__, *self.inputs, - **self.kwargs) + return layer_to_str( + self.layer_func.__name__, *self.inputs, **self.kwargs + ) class SharedLayerDesc(LayerDesc): - - def __init__(self, - key, - layer_func, - forward_func=None, - shared_weight_attr='weight', - *inputs, - **kwargs): + def __init__( + self, + key, + layer_func, + forward_func=None, + shared_weight_attr='weight', + *inputs, + **kwargs + ): super(SharedLayerDesc, self).__init__(layer_func, *inputs, **kwargs) self.layer_name = key self.forward_func = forward_func @@ -88,12 +90,13 @@ class SharedLayerDesc(LayerDesc): class SegmentLayers(object): - - def __init__(self, - layers_desc, - num_parts, - method="uniform", - num_virtual_pipeline_stage=None): + def __init__( + self, + layers_desc, + num_parts, + method="uniform", + num_virtual_pipeline_stage=None, + ): self._layers_desc = layers_desc self.method = method self.num_parts = num_parts @@ -101,7 +104,9 @@ class SegmentLayers(object): self.num_virtual_pipeline_stage = num_virtual_pipeline_stage if self.num_virtual_pipeline_stage is not None: self.total_parts = num_parts * self.num_virtual_pipeline_stage - assert self.num_items >= self.num_parts, "layer number should be greater than number of segments" + assert ( + self.num_items >= self.num_parts + ), "layer number should be greater than number of segments" def do_segment(self): if self.method == "uniform": @@ -115,12 +120,17 @@ class SegmentLayers(object): for idx in weight_idxs: weights[idx] = 1 - actual_num_parts = self.num_parts if self.num_virtual_pipeline_stage is None else self.total_parts - - assert sum( - weights - ) % actual_num_parts == 0, "number of layers ({}) should be divided by part number({})".format( - sum(weights), actual_num_parts) + actual_num_parts = ( + self.num_parts + if self.num_virtual_pipeline_stage is None + else self.total_parts + ) + + assert ( + sum(weights) % actual_num_parts == 0 + ), "number of layers ({}) should be divided by part number({})".format( + sum(weights), actual_num_parts + ) part_size = sum(weights) // actual_num_parts result = [0 for _ in range(actual_num_parts + 1)] @@ -153,8 +163,9 @@ class SegmentLayers(object): if regex.search(name): weight_idxs.append(idx) - assert len( - weight_idxs) > 0, "weight_idxs' length should be greater than 0" + assert ( + len(weight_idxs) > 0 + ), "weight_idxs' length should be greater than 0" return weight_idxs def uniform(self, num_items, num_parts): @@ -167,7 +178,6 @@ class SegmentLayers(object): class PipelineLayerChunk(Layer): - def __init__(self): super(PipelineLayerChunk, self).__init__() self.run_function = [] @@ -189,7 +199,8 @@ class PipelineLayerChunk(Layer): # behavior under recompute circumstance. raise PermissionError( "The forward function of PipelineLayerChunk cannot be called directly. " - "Please call forward function of PipelineLayer.") + "Please call forward function of PipelineLayer." + ) class PipelineLayer(Layer): @@ -271,32 +282,41 @@ class PipelineLayer(Layer): """ - def __init__(self, - layers, - num_stages=None, - topology=None, - loss_fn=None, - seg_method="uniform", - recompute_interval=0, - recompute_ctx=None, - num_virtual_pipeline_stages=None): + def __init__( + self, + layers, + num_stages=None, + topology=None, + loss_fn=None, + seg_method="uniform", + recompute_interval=0, + recompute_ctx=None, + num_virtual_pipeline_stages=None, + ): super(PipelineLayer, self).__init__() if num_stages is None and topology is None: raise ValueError("should provide num_stages or topology") if num_virtual_pipeline_stages: - assert isinstance(num_virtual_pipeline_stages, int), \ - "virtual_pipeline_stage should be None or an int" + assert isinstance( + num_virtual_pipeline_stages, int + ), "virtual_pipeline_stage should be None or an int" if num_virtual_pipeline_stages > 1: logger.info( "set num_virtual_pipeline_stages > 1 means using interleave scheduler instead of 1f1b scheduler" ) - assert isinstance(seg_method, str), \ - "seg_method should be a str for interleave scheduler" - assert seg_method.startswith('layer:'), \ - "seg_method shoud be start with layer: for interleave scheduler" - - self._num_virtual_pipeline_stages = 1 if num_virtual_pipeline_stages is None else num_virtual_pipeline_stages + assert isinstance( + seg_method, str + ), "seg_method should be a str for interleave scheduler" + assert seg_method.startswith( + 'layer:' + ), "seg_method shoud be start with layer: for interleave scheduler" + + self._num_virtual_pipeline_stages = ( + 1 + if num_virtual_pipeline_stages is None + else num_virtual_pipeline_stages + ) # lazy import import paddle.distributed as dist @@ -310,13 +330,17 @@ class PipelineLayer(Layer): self.recompute_ctx = recompute_ctx if recompute_interval > 0: - assert recompute_ctx is not None, "recompute_ctx must be not None for recompute." + assert ( + recompute_ctx is not None + ), "recompute_ctx must be not None for recompute." offload = recompute_ctx.get('offload', False) partition = recompute_ctx.get('partition', False) logger.info( - "Start Recompute for PipeLineParallel. recompute_offload: {}, recompute_partition: {}" - .format(offload, partition)) + "Start Recompute for PipeLineParallel. recompute_offload: {}, recompute_partition: {}".format( + offload, partition + ) + ) world_size = dist.get_world_size() self.global_rank = dist.get_rank() @@ -325,22 +349,28 @@ class PipelineLayer(Layer): self._stage_id = self._topo.get_coord(self.global_rank).pipe self._num_stages = self._topo.get_dim_size("pipe") if num_stages: - assert self._num_stages == num_stages, "num_stages should be equal to be %d" % ( - self._num_stages) + assert ( + self._num_stages == num_stages + ), "num_stages should be equal to be %d" % (self._num_stages) else: # construct default topology if world_size % num_stages != 0: raise ValueError( "should provide correct num_stages({}) " "which can be divided by world_size({})".format( - num_stages, world_size)) + num_stages, world_size + ) + ) dp_num = world_size // num_stages - self._topo = fleet.CommunicateTopology(["data", "pipe", "model"], - [dp_num, num_stages, 1]) + self._topo = fleet.CommunicateTopology( + ["data", "pipe", "model"], [dp_num, num_stages, 1] + ) self._stage_id = self._topo.get_coord(self.global_rank).pipe self._num_stages = self._topo.get_dim_size("pipe") - self._total_stages_with_virtual_stages = self._num_stages * self._num_virtual_pipeline_stages + self._total_stages_with_virtual_stages = ( + self._num_stages * self._num_virtual_pipeline_stages + ) # initialize segment self._layers_desc = list(self.layers) @@ -378,16 +408,22 @@ class PipelineLayer(Layer): start_idx = virtual_pp_rank * self._num_stages for stage in range(self._num_stages): # stage mark the real pp stage - if self.segment_parts[start_idx + - stage] <= layer_idx < self.segment_parts[ - start_idx + stage + 1]: + if ( + self.segment_parts[start_idx + stage] + <= layer_idx + < self.segment_parts[start_idx + stage + 1] + ): return stage def get_num_virtual_stages(self): return self._num_virtual_pipeline_stages def get_model_chunks(self): - return None if self._num_virtual_pipeline_stages == 1 else self._model_chunks + return ( + None + if self._num_virtual_pipeline_stages == 1 + else self._model_chunks + ) def _construct_shared_comm(self): shared_comm = {} @@ -395,17 +431,21 @@ class PipelineLayer(Layer): return layers_desc = self._layers_desc - shared_layer_names = set(s.layer_name for s in layers_desc - if isinstance(s, SharedLayerDesc)) + shared_layer_names = set( + s.layer_name for s in layers_desc if isinstance(s, SharedLayerDesc) + ) for key in shared_layer_names: shared_layers = [] for idx, layer in enumerate(layers_desc): - if isinstance(layer, - SharedLayerDesc) and layer.layer_name == key: + if ( + isinstance(layer, SharedLayerDesc) + and layer.layer_name == key + ): shared_layers.append(idx) shared_stages = set( - self.get_stage_from_index(idx) for idx in shared_layers) + self.get_stage_from_index(idx) for idx in shared_layers + ) self._dp_degree = self._topo.get_dim('data') self._mp_degree = self._topo.get_dim('model') self._sharding_degree = self._topo.get_dim('sharding') @@ -422,7 +462,9 @@ class PipelineLayer(Layer): pipe=s, data=dp, sharding=sharding, - model=mp)) + model=mp, + ) + ) group = paddle.distributed.new_group(ranks=shared_ranks) if self.global_rank in shared_ranks: @@ -431,8 +473,9 @@ class PipelineLayer(Layer): shared_comm[key] = { 'ranks': shared_ranks, 'group': group, - 'weight_attr': - self.shared_weight_attrs[key], + 'weight_attr': self.shared_weight_attrs[ + key + ], 'layer': self.shared_layers[key], } return shared_comm @@ -440,10 +483,11 @@ class PipelineLayer(Layer): def _synchronize_shared_weights(self): for key, comm in self.shared_comm.items(): with paddle.framework.no_grad(): - paddle.distributed.broadcast(getattr(comm['layer'], - comm['weight_attr']), - src=min(comm['ranks']), - group=comm['group']) + paddle.distributed.broadcast( + getattr(comm['layer'], comm['weight_attr']), + src=min(comm['ranks']), + group=comm['group'], + ) for param in comm['layer'].parameters(): if self.global_rank != min(comm['ranks']): @@ -455,8 +499,9 @@ class PipelineLayer(Layer): # need use trace_op to allreduce weight if in_dygraph_mode(): with paddle.framework.no_grad(): - paddle.distributed.all_reduce(param.grad, - group=comm['group']) + paddle.distributed.all_reduce( + param.grad, group=comm['group'] + ) else: with paddle.framework.no_grad(): paddle.fluid.framework._dygraph_tracer().trace_op( @@ -465,8 +510,9 @@ class PipelineLayer(Layer): outputs={'Out': param._grad_ivar()}, attrs={ 'ring_id': comm['group'].id, - 'use_calc_stream': True - }) + 'use_calc_stream': True, + }, + ) def _segment_network_for_interleave(self, seg_method): logger.info("start segment network for interleave scheduler") @@ -474,14 +520,20 @@ class PipelineLayer(Layer): self._layers_desc, num_parts=self._num_stages, method=seg_method, - num_virtual_pipeline_stage=self._num_virtual_pipeline_stages) + num_virtual_pipeline_stage=self._num_virtual_pipeline_stages, + ) self.segment_parts = seg.do_segment() - logger.info("segment result:" + - ", ".join(str(arg) for arg in self.segment_parts)) + logger.info( + "segment result:" + + ", ".join(str(arg) for arg in self.segment_parts) + ) - for i in range(self._stage_id, self._total_stages_with_virtual_stages, - self._num_stages): + for i in range( + self._stage_id, + self._total_stages_with_virtual_stages, + self._num_stages, + ): # If there are 2 real pp stages and 2 virtual pp stages, and the model has 8 layers. # Layers [0, 1], [4, 5] will be assigned to the first real pp stage. # Layers [2, 3], [6, 7] will be assigned to the second real pp stage. @@ -497,13 +549,15 @@ class PipelineLayer(Layer): def _segment_network(self, seg_method): logger.info("start segment network..") - seg = SegmentLayers(self._layers_desc, - num_parts=self._num_stages, - method=seg_method) + seg = SegmentLayers( + self._layers_desc, num_parts=self._num_stages, method=seg_method + ) self.segment_parts = seg.do_segment() - logger.info("segment result:" + - ", ".join(str(arg) for arg in self.segment_parts)) + logger.info( + "segment result:" + + ", ".join(str(arg) for arg in self.segment_parts) + ) self._start_pos = self.segment_parts[self._stage_id] self._end_pos = self.segment_parts[self._stage_id + 1] @@ -511,22 +565,30 @@ class PipelineLayer(Layer): def _print_segmentation_for_debug(self): # print information for debug - for stage in range(self._num_stages * - self._num_virtual_pipeline_stages): + for stage in range( + self._num_stages * self._num_virtual_pipeline_stages + ): start = self.segment_parts[stage] end = self.segment_parts[stage + 1] - logger.info("stage={}, global_rank={} ,layer_number={}".format( - stage, self.global_rank, end - start)) + logger.info( + "stage={}, global_rank={} ,layer_number={}".format( + stage, self.global_rank, end - start + ) + ) for index, layer in enumerate(self._layers_desc[start:end]): logger.info("{}: {}".format(index + start, str(layer))) if self._num_virtual_pipeline_stages > 1: for stage in range(self._num_stages): - stage_to_virtual_stage_info = "stage {} contains virtual stages: ".format( - stage) - for i in range(stage, self._total_stages_with_virtual_stages, - self._num_stages): + stage_to_virtual_stage_info = ( + "stage {} contains virtual stages: ".format(stage) + ) + for i in range( + stage, + self._total_stages_with_virtual_stages, + self._num_stages, + ): stage_to_virtual_stage_info += " {},".format(i) logger.info(stage_to_virtual_stage_info) @@ -572,9 +634,11 @@ class PipelineLayer(Layer): if layer.layer_name not in self.shared_layers: self.shared_layers[layer.layer_name] = layer.build_layer() self.shared_weight_attrs[ - layer.layer_name] = layer.shared_weight_attr + layer.layer_name + ] = layer.shared_weight_attr for param in self.shared_layers[ - layer.layer_name].parameters(): + layer.layer_name + ].parameters(): setattr(param, "is_firstly_shared", True) if layer.forward_func is None: @@ -582,8 +646,11 @@ class PipelineLayer(Layer): else: run_function.append( - partial(layer.forward_func, - self.shared_layers[layer.layer_name])) + partial( + layer.forward_func, + self.shared_layers[layer.layer_name], + ) + ) elif isinstance(layer, LayerDesc): model = layer.build_layer() @@ -612,11 +679,15 @@ class PipelineLayer(Layer): def forward(self, input, chunk_id=None): if chunk_id is not None: assert isinstance(chunk_id, int), "chunk_id should be an int" - assert self._num_virtual_pipeline_stages > 1, \ - "chunk_id is only valid when using virtual pipeline stage" - assert chunk_id < len(self._model_chunks), \ - "The virtual pipeline only has {} chunks, " \ - "but received chunk_id {}.".format(len(self._model_chunks), chunk_id) + assert ( + self._num_virtual_pipeline_stages > 1 + ), "chunk_id is only valid when using virtual pipeline stage" + assert chunk_id < len(self._model_chunks), ( + "The virtual pipeline only has {} chunks, " + "but received chunk_id {}.".format( + len(self._model_chunks), chunk_id + ) + ) # Get the target model chunk. model_chunk = self._model_chunks[chunk_id] # Update the self.run_function to the target run functions. @@ -634,20 +705,25 @@ class PipelineLayer(Layer): funcs = self.run_function[start_idx:end_idx] if not isinstance(input, tuple): - input = (input, ) + input = (input,) if self._need_recompute(funcs, input): input = recompute_hybrid( self.recompute_ctx, - self.forward_function(start_idx, end_idx), *input) + self.forward_function(start_idx, end_idx), + *input + ) else: input = self.forward_function(start_idx, end_idx)(*input) return input def _need_recompute(self, funcs, inputs): - if not any(input_.stop_gradient == False - for input_ in inputs if isinstance(input_, paddle.Tensor)): + if not any( + input_.stop_gradient == False + for input_ in inputs + if isinstance(input_, paddle.Tensor) + ): return False params = [f.parameters() for f in funcs if isinstance(f, Layer)] @@ -671,11 +747,18 @@ class PipelineLayer(Layer): if self._num_virtual_pipeline_stages > 1: # add virtual pipeline info to the save path assert local_chunk_id is not None - virtual_pipeline_stage_message = "-virtual_pp_stage_{:0>2d}".format( - local_chunk_id) - layer_save_path = os.path.join(ckpt_dir, - 'layer_{:0>2d}'.format(idx)) - layer_save_path = layer_save_path + virtual_pipeline_stage_message + rank_message + '-model_states.pdparams' + virtual_pipeline_stage_message = ( + "-virtual_pp_stage_{:0>2d}".format(local_chunk_id) + ) + layer_save_path = os.path.join( + ckpt_dir, 'layer_{:0>2d}'.format(idx) + ) + layer_save_path = ( + layer_save_path + + virtual_pipeline_stage_message + + rank_message + + '-model_states.pdparams' + ) return layer_save_path def _save_model(run_functions, local_chunk_id=None): @@ -698,7 +781,8 @@ class PipelineLayer(Layer): def set_state_dir(self, path): assert os.path.exists( - path), "{} not found, please check the path".format(path) + path + ), "{} not found, please check the path".format(path) def _load_model(run_functions, local_chunk_id=None): for idx, layer in enumerate(run_functions): @@ -712,21 +796,26 @@ class PipelineLayer(Layer): pos_offset = self._start_poss[local_chunk_id] layer_idx = idx + pos_offset layer_save_path = os.path.join( - path, 'layer_{0:0>2d}'.format(layer_idx)) + path, 'layer_{0:0>2d}'.format(layer_idx) + ) if self._num_virtual_pipeline_stages > 1: # add virtual pipeline info to the path assert local_chunk_id is not None - layer_save_path = layer_save_path + "-virtual_pp_stage_{:0>2d}".format( - local_chunk_id) - model_files = glob.glob(layer_save_path + - "*model_states.pdparams") + layer_save_path = ( + layer_save_path + + "-virtual_pp_stage_{:0>2d}".format(local_chunk_id) + ) + model_files = glob.glob( + layer_save_path + "*model_states.pdparams" + ) model_files.sort() mp_rank = self._topo.get_coord(self.global_rank).model mp_world_size = self._topo.get_dim('model') num_files = len(model_files) - load_param_path = model_files[mp_rank * num_files // - mp_world_size] + load_param_path = model_files[ + mp_rank * num_files // mp_world_size + ] model_state_dict = paddle.load(load_param_path) layer.set_state_dict(model_state_dict) diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index 937122e8a7b..89a3d619218 100755 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -29,27 +29,31 @@ __all__ = [] class PipelineParallel(MetaParallelBase): - def __init__(self, layers, hcg, strategy): if not isinstance(layers, PipelineLayer): raise TypeError( - "The Layer should be a derived class of PipelineLayer.") + "The Layer should be a derived class of PipelineLayer." + ) super(PipelineParallel, self).__init__(layers, hcg, strategy) self.use_data_parallel = self._hcg.get_data_parallel_world_size() > 1 self.use_model_parallel = self._hcg.get_model_parallel_world_size() > 1 - self.use_sharding_parallel = self._hcg.get_sharding_parallel_world_size( - ) > 1 + self.use_sharding_parallel = ( + self._hcg.get_sharding_parallel_world_size() > 1 + ) self.total_loss = None self.micro_batch_size = self._strategy.pipeline_configs[ - 'micro_batch_size'] + 'micro_batch_size' + ] self.accumulate_steps = self._strategy.pipeline_configs[ - 'accumulate_steps'] + 'accumulate_steps' + ] # If sent tensor are not the same from different hosts, # they shouldn't been sent partially and then concated as a whole tensor. self._enable_partial_send_recv = self._strategy.pipeline_configs[ - 'enable_partial_send_recv'] + 'enable_partial_send_recv' + ] self._using_cache = self._strategy.pipeline_configs['p2p_cache_shape'] self.num_stages = self._hcg.get_pipe_parallel_world_size() @@ -61,16 +65,20 @@ class PipelineParallel(MetaParallelBase): self._real_pp_world_size = self.num_stages self._real_pp_rank = self.stage_id - p2p.initialize_p2p_groups(hcg, self._using_cache, - self._enable_partial_send_recv) + p2p.initialize_p2p_groups( + hcg, self._using_cache, self._enable_partial_send_recv + ) self.global_rank = self._hcg.get_global_rank() self.micro_batch_id = 0 self._compute_loss = True - logger.info("Pipeline Info -- num_stages: {}, stage_id: {}".format( - self.num_stages, self.stage_id)) + logger.info( + "Pipeline Info -- num_stages: {}, stage_id: {}".format( + self.num_stages, self.stage_id + ) + ) if self.use_model_parallel: logger.info("start broadcast mp parameters") @@ -122,7 +130,7 @@ class PipelineParallel(MetaParallelBase): # store data id for micro_batch self.micro_batch_id = 0 - startup_steps = (self.num_stages - self.stage_id - 1) + startup_steps = self.num_stages - self.stage_id - 1 startup_steps = min(startup_steps, self.accumulate_steps) steady_steps = self.accumulate_steps - startup_steps @@ -142,39 +150,46 @@ class PipelineParallel(MetaParallelBase): input_tensor = p2p.recv_forward(self.is_pipeline_first_stage()) for i in range(steady_steps): - last_iter = (i == (steady_steps - 1)) + last_iter = i == (steady_steps - 1) output_tensor = self._forward_step(input_tensor) output_tensor_grad = p2p.send_forward_recv_backward( - output_tensor, self.is_pipeline_last_stage()) + output_tensor, self.is_pipeline_last_stage() + ) input_buffers.append(input_tensor) output_buffers.append(output_tensor) input_tensor, output_tensor = input_buffers.pop( - 0), output_buffers.pop(0) + 0 + ), output_buffers.pop(0) - input_tensor_grad = self._backward_step(input_tensor, output_tensor, - output_tensor_grad) + input_tensor_grad = self._backward_step( + input_tensor, output_tensor, output_tensor_grad + ) if last_iter: input_tensor = None - p2p.send_backward(input_tensor_grad, - self.is_pipeline_first_stage()) + p2p.send_backward( + input_tensor_grad, self.is_pipeline_first_stage() + ) else: input_tensor = p2p.send_backward_recv_forward( - input_tensor_grad, self.is_pipeline_first_stage()) + input_tensor_grad, self.is_pipeline_first_stage() + ) for i in range(startup_steps): input_tensor = input_buffers.pop(0) output_tensor = output_buffers.pop(0) output_tensor_grad = p2p.recv_backward( - self.is_pipeline_last_stage()) + self.is_pipeline_last_stage() + ) - input_tensor_grad = self._backward_step(input_tensor, output_tensor, - output_tensor_grad) + input_tensor_grad = self._backward_step( + input_tensor, output_tensor, output_tensor_grad + ) p2p.send_backward(input_tensor_grad, self.is_pipeline_first_stage()) self._layers.allreduce_shared_weight_gradients() @@ -186,17 +201,20 @@ class PipelineParallel(MetaParallelBase): # reset the virtual pp rank for each run self.set_virtual_pipeline_rank(0) - assert isinstance(optimizer, HybridParallelOptimizer), ( - 'optimizer should be HybridParallelOptimizer subclass.') + assert isinstance( + optimizer, HybridParallelOptimizer + ), 'optimizer should be HybridParallelOptimizer subclass.' - assert fluid.framework._dygraph_tracer()._has_grad, ( - 'Please enable the generation of gradients.') + assert ( + fluid.framework._dygraph_tracer()._has_grad + ), 'Please enable the generation of gradients.' if self.is_pipeline_first_stage( - ignore_virtual=True) or self.is_pipeline_last_stage( - ignore_virtual=True): - assert data is not None, ( - "For the first and the last stage, the data must be set.") + ignore_virtual=True + ) or self.is_pipeline_last_stage(ignore_virtual=True): + assert ( + data is not None + ), "For the first and the last stage, the data must be set." else: data = None @@ -233,7 +251,7 @@ class PipelineParallel(MetaParallelBase): # store total loss of entire batch self.total_loss = None - startup_steps = (self.num_stages - self.stage_id - 1) + startup_steps = self.num_stages - self.stage_id - 1 startup_steps = min(startup_steps, self.accumulate_steps) steady_steps = self.accumulate_steps - startup_steps @@ -253,7 +271,7 @@ class PipelineParallel(MetaParallelBase): input_tensor = p2p.recv_forward(self.is_pipeline_first_stage()) for i in range(steady_steps): - last_iter = (i == (steady_steps - 1)) + last_iter = i == (steady_steps - 1) output_tensor = self._forward_step(input_tensor) p2p.send_forward(output_tensor, self.is_pipeline_last_stage()) @@ -282,13 +300,14 @@ class PipelineParallel(MetaParallelBase): if self.is_pipeline_last_stage(): # train calculate loss for train if self._compute_loss: - assert self._layers._loss_fn is not None, "loss function should exist to compute loss" + assert ( + self._layers._loss_fn is not None + ), "loss function should exist to compute loss" labels = self._load_micro_batch(self.micro_batch_id) output_tensor = self._layers._loss_fn(output_tensor, labels) assert isinstance( - output_tensor, - (paddle.Tensor, core.eager.Tensor - )), "Currently, loss_fn should obtain Paddle.Tensor dtype" + output_tensor, (paddle.Tensor, core.eager.Tensor) + ), "Currently, loss_fn should obtain Paddle.Tensor dtype" with paddle.amp.auto_cast(enable=False): if self.accumulate_steps > 1: @@ -318,16 +337,20 @@ class PipelineParallel(MetaParallelBase): assert len(outputs) == len(output_tensor_grad) paddle.autograd.backward( tensors=outputs, - grad_tensors=[t for t in output_tensor_grad]) + grad_tensors=[t for t in output_tensor_grad], + ) else: - paddle.autograd.backward(tensors=[output_tensor], - grad_tensors=[output_tensor_grad]) + paddle.autograd.backward( + tensors=[output_tensor], + grad_tensors=[output_tensor_grad], + ) input_tensor_grad = None if input_tensor is not None: if isinstance(input_tensor, tuple): input_tensor_grad = tuple( - [t.grad for t in input_tensor if not t.stop_gradient]) + [t.grad for t in input_tensor if not t.stop_gradient] + ) else: input_tensor_grad = input_tensor.grad return input_tensor_grad @@ -341,31 +364,39 @@ class PipelineParallel(MetaParallelBase): if self.is_pipeline_first_stage(): assert len(inputs) == 2, "length of input should be 2" if isinstance(inputs[0], tuple): - assert len( - inputs[0] - ) > 1, "If you use tuple for input data, it should have at least two inputs." + assert ( + len(inputs[0]) > 1 + ), "If you use tuple for input data, it should have at least two inputs." batch_size = inputs[0][0].shape[0] - assert self.micro_batch_size * self.accumulate_steps == batch_size, ( + assert ( + self.micro_batch_size * self.accumulate_steps == batch_size + ), ( "batch_size needs to be divisible by micro_batch_size. Currently, " "batch_size = %d, micro_batch_size = %d, accumulate_steps = %d." - % - (batch_size, self.micro_batch_size, self.accumulate_steps)) + % (batch_size, self.micro_batch_size, self.accumulate_steps) + ) data = [input[begin:end, :].detach() for input in inputs[0]] return tuple(data) else: batch_size = inputs[0].shape[0] - assert self.micro_batch_size * self.accumulate_steps == batch_size + assert ( + self.micro_batch_size * self.accumulate_steps == batch_size + ) return inputs[0][begin:end, :].detach() elif self.is_pipeline_last_stage(): assert len(inputs) == 2, "length of input should be 2" if isinstance(inputs[1], tuple): batch_size = inputs[1][0].shape[0] - assert self.micro_batch_size * self.accumulate_steps == batch_size + assert ( + self.micro_batch_size * self.accumulate_steps == batch_size + ) data = [input[begin:end, :].detach() for input in inputs[1]] return tuple(data) else: batch_size = inputs[1].shape[0] - assert self.micro_batch_size * self.accumulate_steps == batch_size + assert ( + self.micro_batch_size * self.accumulate_steps == batch_size + ) return inputs[1][begin:end, :].detach() else: # No data input is required for other stages @@ -375,34 +406,40 @@ class PipelineParallel(MetaParallelBase): # Since the last backward run in interleave will set the virtual rank to 0, # here we need to check last stage ignoring virtual stage. if self.is_pipeline_last_stage(ignore_virtual=True): - assert self.total_loss is not None, "train_batch() in last stage should obtain vaild loss" + assert ( + self.total_loss is not None + ), "train_batch() in last stage should obtain vaild loss" loss = self.total_loss.detach() - is_fp32 = paddle.to_tensor( - 1) if loss.dtype == paddle.float32 else paddle.to_tensor(0) - paddle.distributed.broadcast(is_fp32, - src=self.global_rank, - sync_op=True, - group=self.pp_group) - paddle.distributed.broadcast(loss, - src=self.global_rank, - sync_op=True, - group=self.pp_group) + is_fp32 = ( + paddle.to_tensor(1) + if loss.dtype == paddle.float32 + else paddle.to_tensor(0) + ) + paddle.distributed.broadcast( + is_fp32, src=self.global_rank, sync_op=True, group=self.pp_group + ) + paddle.distributed.broadcast( + loss, src=self.global_rank, sync_op=True, group=self.pp_group + ) else: is_fp32 = paddle.to_tensor(1) paddle.distributed.broadcast( is_fp32, src=self._hcg.get_rank_from_stage(self.num_stages - 1), sync_op=True, - group=self.pp_group) - loss = paddle.zeros(shape=[ - 1 - ], dtype="float32") if is_fp32.numpy()[0] else paddle.zeros( - shape=[1], dtype="float16") + group=self.pp_group, + ) + loss = ( + paddle.zeros(shape=[1], dtype="float32") + if is_fp32.numpy()[0] + else paddle.zeros(shape=[1], dtype="float16") + ) paddle.distributed.broadcast( loss, src=self._hcg.get_rank_from_stage(self.num_stages - 1), sync_op=True, - group=self.pp_group) + group=self.pp_group, + ) return loss def _optimizer_step(self): @@ -421,11 +458,12 @@ class PipelineParallelWithInterleave(PipelineParallel): # pipeline parallel with interleave scheduler def __init__(self, layers, hcg, strategy): - super(PipelineParallelWithInterleave, self).__init__(layers=layers, - hcg=hcg, - strategy=strategy) + super(PipelineParallelWithInterleave, self).__init__( + layers=layers, hcg=hcg, strategy=strategy + ) assert layers.get_num_virtual_stages() > 1 - assert framework.in_dygraph_mode( + assert ( + framework.in_dygraph_mode() ), "virtual pipeline stage with interleave only support eager dygraph mode" # setup for interleave scheduler self.num_model_chunks = layers.get_num_virtual_stages() @@ -436,11 +474,12 @@ class PipelineParallelWithInterleave(PipelineParallel): self._virtual_pp_rank = 0 def _get_virtual_pp_rank(self, micro_step, forward): - virtual_pp_stage = micro_step % (self.num_stages * - self.num_model_chunks) + virtual_pp_stage = micro_step % ( + self.num_stages * self.num_model_chunks + ) virtual_pp_stage = virtual_pp_stage // self.num_stages if not forward: - virtual_pp_stage = (self.num_model_chunks - virtual_pp_stage - 1) + virtual_pp_stage = self.num_model_chunks - virtual_pp_stage - 1 return virtual_pp_stage def _forward_step_helper(self, micro_step): @@ -455,7 +494,8 @@ class PipelineParallelWithInterleave(PipelineParallel): if self.is_pipeline_first_stage(): if len(self.input_tensors[virtual_pp_rank]) == len( - self.output_tensors[virtual_pp_rank]): + self.output_tensors[virtual_pp_rank] + ): self.input_tensors[virtual_pp_rank].append(None) input_tensor = self.input_tensors[virtual_pp_rank][-1] output_tensor = self._forward_step(input_tensor, virtual_pp_rank) @@ -484,21 +524,22 @@ class PipelineParallelWithInterleave(PipelineParallel): input_tensor = self.input_tensors[virtual_pp_rank].pop(0) output_tensor = self.output_tensors[virtual_pp_rank].pop(0) output_tensor_grad = self.output_tensor_grads[virtual_pp_rank].pop(0) - input_tensor_grad = self._backward_step(input_tensor, output_tensor, - output_tensor_grad) + input_tensor_grad = self._backward_step( + input_tensor, output_tensor, output_tensor_grad + ) return input_tensor_grad - def interleave_pipeline(self, - data, - scaler, - forward_only=False, - compute_loss=True): + def interleave_pipeline( + self, data, scaler, forward_only=False, compute_loss=True + ): # use interleave scheduling strategy. # this strategy is inspired by: # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/schedules.py if not compute_loss: - assert not forward_only, "compute_loss can only be set to False when forward_only is set to True" + assert ( + not forward_only + ), "compute_loss can only be set to False when forward_only is set to True" # init some attributes for this batch run self.scaler = scaler @@ -530,15 +571,17 @@ class PipelineParallelWithInterleave(PipelineParallel): self.set_virtual_pipeline_rank(0) self.input_tensors[0].append( - p2p.recv_forward(self.is_pipeline_first_stage(), sync_recv=False)) + p2p.recv_forward(self.is_pipeline_first_stage(), sync_recv=False) + ) # run startup steps for micro_step in range(startup_steps): output_tensor = self._forward_step_helper(micro_step) # determine whether recv forward tensor or not - next_virtual_pp_rank = self._get_virtual_pp_rank(micro_step + 1, - forward=True) + next_virtual_pp_rank = self._get_virtual_pp_rank( + micro_step + 1, forward=True + ) recv_prev = True if self.is_pipeline_first_stage(ignore_virtual=True): if next_virtual_pp_rank == 0: @@ -552,24 +595,33 @@ class PipelineParallelWithInterleave(PipelineParallel): if self.is_pipeline_last_stage(): output_tensor = None - if micro_step == (startup_steps - - 1) and not forward_only and not all_startup_steps: + if ( + micro_step == (startup_steps - 1) + and not forward_only + and not all_startup_steps + ): input_tensor_grad = None recv_next = True if self.is_pipeline_last_stage(ignore_virtual=True): recv_next = False # the last startup step needs on four direction comm to set up for steady 1f1b - input_tensor, output_tensor_grad = p2p.send_forward_backward_recv_forward_backward( + ( + input_tensor, + output_tensor_grad, + ) = p2p.send_forward_backward_recv_forward_backward( output_tensor, input_tensor_grad, recv_prev=recv_prev, - recv_next=recv_next) - self.output_tensor_grads[self.num_model_chunks - - 1].append(output_tensor_grad) + recv_next=recv_next, + ) + self.output_tensor_grads[self.num_model_chunks - 1].append( + output_tensor_grad + ) else: input_tensor = p2p.send_forward_recv_forward( - output_tensor, recv_prev=recv_prev) + output_tensor, recv_prev=recv_prev + ) self.input_tensors[next_virtual_pp_rank].append(input_tensor) # run 1f1b steady steps @@ -581,7 +633,8 @@ class PipelineParallelWithInterleave(PipelineParallel): # backward backward_micro_step_id = micro_step input_tensor_grad = self._backward_step_helper( - backward_micro_step_id) + backward_micro_step_id + ) # four directions comm # send output tensor to downstream @@ -591,14 +644,16 @@ class PipelineParallelWithInterleave(PipelineParallel): # last stage doesn't send rst to downstream forward_virtual_pp_rank = self._get_virtual_pp_rank( - forward_micro_step_id, forward=True) + forward_micro_step_id, forward=True + ) self.set_virtual_pipeline_rank(forward_virtual_pp_rank) if self.is_pipeline_last_stage(): output_tensor = None # first stage doesn't send grad to upstream backward_virtual_pp_rank = self._get_virtual_pp_rank( - backward_micro_step_id, forward=False) + backward_micro_step_id, forward=False + ) self.set_virtual_pipeline_rank(backward_virtual_pp_rank) if self.is_pipeline_first_stage(): input_tensor_grad = None @@ -607,14 +662,16 @@ class PipelineParallelWithInterleave(PipelineParallel): recv_prev = True if self.is_pipeline_first_stage(ignore_virtual=True): next_forward_virtual_pp_rank = self._get_virtual_pp_rank( - forward_micro_step_id - (self.num_stages - 1), forward=True) + forward_micro_step_id - (self.num_stages - 1), forward=True + ) if next_forward_virtual_pp_rank == (self.num_model_chunks - 1): # first pp stage and first virtual stage recv_prev = False next_forward_virtual_pp_rank += 1 else: next_forward_virtual_pp_rank = self._get_virtual_pp_rank( - forward_micro_step_id + 1, forward=True) + forward_micro_step_id + 1, forward=True + ) # last iteration doesn't need recv from upstream if micro_step == (steady_steps - 1): @@ -625,53 +682,67 @@ class PipelineParallelWithInterleave(PipelineParallel): if self.is_pipeline_last_stage(ignore_virtual=True): next_backward_virtual_pp_rank = self._get_virtual_pp_rank( backward_micro_step_id - (self.num_stages - 1), - forward=False) + forward=False, + ) if next_backward_virtual_pp_rank == 0: # last pp stage and last virtual stage recv_next = False next_backward_virtual_pp_rank -= 1 else: next_backward_virtual_pp_rank = self._get_virtual_pp_rank( - backward_micro_step_id + 1, forward=False) + backward_micro_step_id + 1, forward=False + ) - input_tensor, output_tensor_grad = p2p.send_forward_backward_recv_forward_backward( + ( + input_tensor, + output_tensor_grad, + ) = p2p.send_forward_backward_recv_forward_backward( output_tensor, input_tensor_grad, recv_prev=recv_prev, - recv_next=recv_next) + recv_next=recv_next, + ) if recv_prev: self.input_tensors[next_forward_virtual_pp_rank].append( - input_tensor) + input_tensor + ) if recv_next: self.output_tensor_grads[next_backward_virtual_pp_rank].append( - output_tensor_grad) + output_tensor_grad + ) # remaining backward steps if not forward_only: if all_startup_steps: self.output_tensor_grads[self.num_model_chunks - 1].append( - p2p.recv_backward(self.is_pipeline_last_stage(), - sync_recv=False)) + p2p.recv_backward( + self.is_pipeline_last_stage(), sync_recv=False + ) + ) for micro_step in range(steady_steps, num_steps): # cooldown loop input_tensor_grad = self._backward_step_helper(micro_step) next_backward_virtual_pp_rank = self._get_virtual_pp_rank( - micro_step + 1, forward=False) + micro_step + 1, forward=False + ) recv_next = True if self.is_pipeline_last_stage(ignore_virtual=True): - if next_backward_virtual_pp_rank == (self.num_model_chunks - - 1): + if next_backward_virtual_pp_rank == ( + self.num_model_chunks - 1 + ): recv_next = False if micro_step == (num_steps - 1): recv_next = False self.output_tensor_grads[next_backward_virtual_pp_rank].append( - p2p.send_backward_recv_backward(input_tensor_grad, - recv_next=recv_next)) + p2p.send_backward_recv_backward( + input_tensor_grad, recv_next=recv_next + ) + ) self._layers.allreduce_shared_weight_gradients() diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py index ecd3dc7ab91..e97cf3c0230 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py @@ -30,12 +30,23 @@ def initialize_p2p_groups(hcg, use_cache=True, enable_partial_send_recv=True): _hcg = hcg _use_cache = use_cache _enable_partial_send_recv = enable_partial_send_recv - send_next_group, send_prev_group, recv_next_group, recv_prev_group = _hcg.get_p2p_groups( + ( + send_next_group, + send_prev_group, + recv_next_group, + recv_prev_group, + ) = _hcg.get_p2p_groups() + + debug_str = ( + "P2pInfo: send_next_group: %s, send_prev_group: %s, " + "recv_next_group: %s, recv_prev_group: %s" + % ( + repr(send_next_group), + repr(send_prev_group), + repr(recv_next_group), + repr(recv_prev_group), + ) ) - - debug_str = "P2pInfo: send_next_group: %s, send_prev_group: %s, " \ - "recv_next_group: %s, recv_prev_group: %s" % (repr(send_next_group), - repr(send_prev_group),repr(recv_next_group), repr(recv_prev_group)) logger.info(debug_str) @@ -150,9 +161,11 @@ class SendRecvMeta: self.send_dtype_message = paddle_2_number(tensor.dtype) elif isinstance(tensor, tuple): self.send_shape_message = tuple( - [d.shape for d in tensor if not d.stop_gradient]) + [d.shape for d in tensor if not d.stop_gradient] + ) self.send_dtype_message = tuple( - [paddle_2_number(d.dtype) for d in tensor]) + [paddle_2_number(d.dtype) for d in tensor] + ) _send_recv_meta = SendRecvMeta() @@ -166,84 +179,117 @@ def _is_valid_send_recv_partial(tensor, mp_degree): return mp_degree > 1 and tensor_numel % mp_degree == 0 -def _partial_send_op(tensor, group, use_calc_stream, ring_id, dst, nranks, - rank_id): +def _partial_send_op( + tensor, group, use_calc_stream, ring_id, dst, nranks, rank_id +): dst_rank_in_group = dst if group is None else group.get_group_rank(dst) if _in_legacy_dygraph(): - return _legacy_C_ops.partial_send(tensor.detach(), 'use_calc_stream', - use_calc_stream, 'ring_id', ring_id, - 'peer', dst_rank_in_group, 'num', - nranks, 'id', rank_id) + return _legacy_C_ops.partial_send( + tensor.detach(), + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'peer', + dst_rank_in_group, + 'num', + nranks, + 'id', + rank_id, + ) elif in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group - comm_op = group.process_group.send_partial_on_calc_stream \ - if use_calc_stream else group.process_group.send_partial + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) + comm_op = ( + group.process_group.send_partial_on_calc_stream + if use_calc_stream + else group.process_group.send_partial + ) return comm_op(tensor, dst_rank_in_group, nranks, rank_id) -def send_partial(tensor, - dst=0, - nranks=1, - rank_id=0, - group=None, - use_calc_stream=True): +def send_partial( + tensor, dst=0, nranks=1, rank_id=0, group=None, use_calc_stream=True +): # dst: local rank in group if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id - dst_rank = _hcg._get_p2p_next_rank( - ) if dst == 1 else _hcg._get_p2p_prev_rank() + dst_rank = ( + _hcg._get_p2p_next_rank() if dst == 1 else _hcg._get_p2p_prev_rank() + ) if _is_valid_send_recv_partial(tensor, nranks): - return _partial_send_op(tensor, group, use_calc_stream, ring_id, - dst_rank, nranks, rank_id) + return _partial_send_op( + tensor, group, use_calc_stream, ring_id, dst_rank, nranks, rank_id + ) else: if _in_legacy_dygraph(): - send_op = lambda x, dst, group: \ - paddle.distributed.send(x, dst, group, use_calc_stream) + send_op = lambda x, dst, group: paddle.distributed.send( + x, dst, group, use_calc_stream + ) elif in_dygraph_mode(): send_op = paddle.distributed.isend return send_op(tensor.detach(), dst=dst_rank, group=group) -def _partial_recv_op(tensor, group, use_calc_stream, ring_id, src, nranks, - rank_id): +def _partial_recv_op( + tensor, group, use_calc_stream, ring_id, src, nranks, rank_id +): src_rank_in_group = src if group is None else group.get_group_rank(src) if _in_legacy_dygraph(): assert use_calc_stream - return _legacy_C_ops.partial_recv(tensor.detach(), 'use_calc_stream', - use_calc_stream, 'ring_id', ring_id, - 'peer', src_rank_in_group, 'num', - nranks, 'id', rank_id, 'dtype', - tensor.dtype, 'out_shape', - tensor.shape) + return _legacy_C_ops.partial_recv( + tensor.detach(), + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'peer', + src_rank_in_group, + 'num', + nranks, + 'id', + rank_id, + 'dtype', + tensor.dtype, + 'out_shape', + tensor.shape, + ) elif in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group - comm_op = group.process_group.recv_partial_on_calc_stream \ - if use_calc_stream else group.process_group.recv_partial + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) + comm_op = ( + group.process_group.recv_partial_on_calc_stream + if use_calc_stream + else group.process_group.recv_partial + ) return comm_op(tensor, src_rank_in_group, nranks, rank_id) -def recv_partial(tensor, - src=0, - nranks=1, - rank_id=0, - group=None, - use_calc_stream=True): +def recv_partial( + tensor, src=0, nranks=1, rank_id=0, group=None, use_calc_stream=True +): # src: local rank in group if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id - src_rank = _hcg._get_p2p_prev_rank( - ) if src == 0 else _hcg._get_p2p_next_rank() + src_rank = ( + _hcg._get_p2p_prev_rank() if src == 0 else _hcg._get_p2p_next_rank() + ) if _is_valid_send_recv_partial(tensor, nranks): - return _partial_recv_op(tensor, group, use_calc_stream, ring_id, - src_rank, nranks, rank_id) + return _partial_recv_op( + tensor, group, use_calc_stream, ring_id, src_rank, nranks, rank_id + ) else: if _in_legacy_dygraph() or use_calc_stream: recv_op = paddle.distributed.recv @@ -252,42 +298,52 @@ def recv_partial(tensor, return recv_op(tensor.detach(), src=src_rank, group=group) -def _partial_allgather_op(tensor, group, use_calc_stream, ring_id, nranks, - rank_id): +def _partial_allgather_op( + tensor, group, use_calc_stream, ring_id, nranks, rank_id +): if _in_legacy_dygraph(): - return _legacy_C_ops.partial_allgather_(tensor.detach(), - 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'nranks', nranks, - 'rank', rank_id) + return _legacy_C_ops.partial_allgather_( + tensor.detach(), + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + 'rank', + rank_id, + ) elif in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group - comm_op = group.process_group.all_gather_partial_on_calc_stream \ - if use_calc_stream else group.process_group.all_gather_partial + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) + comm_op = ( + group.process_group.all_gather_partial_on_calc_stream + if use_calc_stream + else group.process_group.all_gather_partial + ) return comm_op(tensor, tensor, nranks, rank_id) -def allgather_partial(tensor, - nranks=1, - rank_id=0, - group=None, - use_calc_stream=True): +def allgather_partial( + tensor, nranks=1, rank_id=0, group=None, use_calc_stream=True +): if not _is_valid_send_recv_partial(tensor, nranks): return tensor if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id - return _partial_allgather_op(tensor, group, use_calc_stream, ring_id, - nranks, rank_id) + return _partial_allgather_op( + tensor, group, use_calc_stream, ring_id, nranks, rank_id + ) -def _p2p_helper(tensor_send_next, - tensor_send_prev, - recv_prev, - recv_next, - sync_recv=True): +def _p2p_helper( + tensor_send_next, tensor_send_prev, recv_prev, recv_next, sync_recv=True +): global _hcg tensor_recv_prev = None @@ -310,15 +366,17 @@ def _p2p_helper(tensor_send_next, if isinstance(recv_shape_msg, tuple): tensor_recv_prev = [] for idx, shape in enumerate(recv_shape_msg): - tmp = paddle.empty(shape=shape, - dtype=number_2_dtype(recv_dtype_msg[idx])) + tmp = paddle.empty( + shape=shape, dtype=number_2_dtype(recv_dtype_msg[idx]) + ) tmp.stop_gradient = recv_stop_gradient[idx] tensor_recv_prev.append(tmp) tensor_recv_prev = tuple(tensor_recv_prev) else: tensor_recv_prev = paddle.empty( - shape=recv_shape_msg, dtype=number_2_dtype(recv_dtype_msg)) + shape=recv_shape_msg, dtype=number_2_dtype(recv_dtype_msg) + ) tensor_recv_prev.stop_gradient = recv_stop_gradient if recv_next: @@ -326,12 +384,15 @@ def _p2p_helper(tensor_send_next, tensor_recv_next = [] for idx, shape in enumerate(send_shape_msg): tensor_recv_next.append( - paddle.empty(shape=shape, - dtype=number_2_dtype(send_dtype_msg[idx]))) + paddle.empty( + shape=shape, dtype=number_2_dtype(send_dtype_msg[idx]) + ) + ) tensor_recv_next = tuple(tensor_recv_next) else: tensor_recv_next = paddle.empty( - shape=send_shape_msg, dtype=number_2_dtype(send_dtype_msg)) + shape=send_shape_msg, dtype=number_2_dtype(send_dtype_msg) + ) # TODO(Yuang Liu): use batch_isend_irecv replace all these comm ops tasks = [] @@ -340,51 +401,63 @@ def _p2p_helper(tensor_send_next, if isinstance(tensor_send_prev, tuple): for d in tensor_send_prev: paddle.distributed.wait(d, use_calc_stream=True) - send_partial(d, - dst=0, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.send_prev_group, - use_calc_stream=False) + send_partial( + d, + dst=0, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.send_prev_group, + use_calc_stream=False, + ) else: paddle.distributed.wait(tensor_send_prev, use_calc_stream=True) - send_partial(tensor_send_prev, - dst=0, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.send_prev_group, - use_calc_stream=False) + send_partial( + tensor_send_prev, + dst=0, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.send_prev_group, + use_calc_stream=False, + ) if tensor_recv_prev is not None: if isinstance(tensor_recv_prev, tuple): for d in tensor_recv_prev: - task = recv_partial(d, - src=0, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.recv_prev_group, - use_calc_stream=sync_recv) + task = recv_partial( + d, + src=0, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.recv_prev_group, + use_calc_stream=sync_recv, + ) if sync_recv: - allgather_partial(d, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + d, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) else: tasks.append(task) else: - task = recv_partial(tensor_recv_prev, - src=0, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.recv_prev_group, - use_calc_stream=sync_recv) + task = recv_partial( + tensor_recv_prev, + src=0, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.recv_prev_group, + use_calc_stream=sync_recv, + ) if sync_recv: - allgather_partial(tensor_recv_prev, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + tensor_recv_prev, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) else: tasks.append(task) @@ -392,52 +465,64 @@ def _p2p_helper(tensor_send_next, if isinstance(tensor_send_next, tuple): for d in tensor_send_next: paddle.distributed.wait(d, use_calc_stream=True) - send_partial(d, - dst=1, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.send_next_group, - use_calc_stream=False) + send_partial( + d, + dst=1, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.send_next_group, + use_calc_stream=False, + ) else: paddle.distributed.wait(tensor_send_next, use_calc_stream=True) - send_partial(tensor_send_next, - dst=1, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.send_next_group, - use_calc_stream=False) + send_partial( + tensor_send_next, + dst=1, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.send_next_group, + use_calc_stream=False, + ) if tensor_recv_next is not None: if isinstance(tensor_recv_next, tuple): for d in tensor_recv_next: - task = recv_partial(d, - src=1, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.recv_next_group, - use_calc_stream=sync_recv) + task = recv_partial( + d, + src=1, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.recv_next_group, + use_calc_stream=sync_recv, + ) if sync_recv: - allgather_partial(d, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + d, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) else: tasks.append(task) else: - task = recv_partial(tensor_recv_next, - src=1, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.recv_next_group, - use_calc_stream=sync_recv) + task = recv_partial( + tensor_recv_next, + src=1, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.recv_next_group, + use_calc_stream=sync_recv, + ) if sync_recv: - allgather_partial(tensor_recv_next, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + tensor_recv_next, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) else: tasks.append(task) @@ -463,11 +548,13 @@ def _p2p_helper(tensor_send_next, tensors_for_all_gather.append(tensor_recv_next) for tensor in tensors_for_all_gather: - allgather_partial(tensor, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + tensor, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) return tensor_recv_prev, tensor_recv_next @@ -480,11 +567,13 @@ def recv_forward(pp_first_stage, sync_recv=True): _send_recv_meta.recv_meta(_hcg.recv_prev_group) _send_recv_meta.has_recv_meta = _use_cache - input_tensor, _ = _p2p_helper(tensor_send_next=None, - tensor_send_prev=None, - recv_prev=True, - recv_next=False, - sync_recv=sync_recv) + input_tensor, _ = _p2p_helper( + tensor_send_next=None, + tensor_send_prev=None, + recv_prev=True, + recv_next=False, + sync_recv=sync_recv, + ) return input_tensor @@ -492,11 +581,13 @@ def recv_backward(pp_last_stage, sync_recv=True): if pp_last_stage: output_tensor_grad = None else: - _, output_tensor_grad = _p2p_helper(tensor_send_next=None, - tensor_send_prev=None, - recv_prev=False, - recv_next=True, - sync_recv=sync_recv) + _, output_tensor_grad = _p2p_helper( + tensor_send_next=None, + tensor_send_prev=None, + recv_prev=False, + recv_next=True, + sync_recv=sync_recv, + ) return output_tensor_grad @@ -507,28 +598,34 @@ def send_forward(output_tensor, pp_last_stage): _send_recv_meta.send_meta(output_tensor, _hcg.send_next_group) _send_recv_meta.has_send_meta = _use_cache - _p2p_helper(tensor_send_next=output_tensor, - tensor_send_prev=None, - recv_prev=False, - recv_next=False) + _p2p_helper( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=False, + recv_next=False, + ) def send_backward(input_tensor_grad, pp_first_stage): if not pp_first_stage: - _p2p_helper(tensor_send_next=None, - tensor_send_prev=input_tensor_grad, - recv_prev=False, - recv_next=False) + _p2p_helper( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=False, + recv_next=False, + ) def send_forward_recv_backward(output_tensor, pp_last_stage): if pp_last_stage: output_tensor_grad = None else: - _, output_tensor_grad = _p2p_helper(tensor_send_next=output_tensor, - tensor_send_prev=None, - recv_prev=False, - recv_next=True) + _, output_tensor_grad = _p2p_helper( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=False, + recv_next=True, + ) return output_tensor_grad @@ -536,16 +633,18 @@ def send_backward_recv_forward(input_tensor_grad, pp_first_stage): if pp_first_stage: input_tensor = None else: - input_tensor, _ = _p2p_helper(tensor_send_next=None, - tensor_send_prev=input_tensor_grad, - recv_prev=True, - recv_next=False) + input_tensor, _ = _p2p_helper( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=True, + recv_next=False, + ) return input_tensor -def send_forward_backward_recv_forward_backward(output_tensor, - input_tensor_grad, recv_prev, - recv_next): +def send_forward_backward_recv_forward_backward( + output_tensor, input_tensor_grad, recv_prev, recv_next +): # always have to send dytpe info to downstream if not _send_recv_meta.has_send_meta: _send_recv_meta.set_send_message(output_tensor) @@ -559,7 +658,8 @@ def send_forward_backward_recv_forward_backward(output_tensor, tensor_send_prev=input_tensor_grad, recv_prev=recv_prev, recv_next=recv_next, - sync_recv=False) + sync_recv=False, + ) return input_tensor, output_tensor_grad @@ -573,19 +673,23 @@ def send_forward_recv_forward(output_tensor, recv_prev): _send_recv_meta.recv_meta(_hcg.recv_prev_group) _send_recv_meta.has_recv_meta = _use_cache - input_tensor, _ = _p2p_helper(tensor_send_next=output_tensor, - tensor_send_prev=None, - recv_prev=recv_prev, - recv_next=False, - sync_recv=False) + input_tensor, _ = _p2p_helper( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=recv_prev, + recv_next=False, + sync_recv=False, + ) return input_tensor def send_backward_recv_backward(input_tensor_grad, recv_next): - _, output_tensor_grad = _p2p_helper(tensor_send_next=None, - tensor_send_prev=input_tensor_grad, - recv_prev=False, - recv_next=recv_next, - sync_recv=False) + _, output_tensor_grad = _p2p_helper( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=False, + recv_next=recv_next, + sync_recv=False, + ) return output_tensor_grad diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py index 9f5d868a157..e33f9d4689c 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py @@ -28,7 +28,7 @@ PADDLE_TO_NUMBER = { paddle.float32: 1, paddle.float64: 2, paddle.int32: 3, - paddle.int64: 4 + paddle.int64: 4, } NUMBER_TO_DTYPE = { @@ -36,7 +36,7 @@ NUMBER_TO_DTYPE = { 1: "float32", 2: "float64", 3: "int32", - 4: "int64" + 4: "int64", } @@ -88,7 +88,17 @@ def _all_gather(tensor, group=None, use_calc_stream=True): if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id - nranks = paddle.distributed.collective._get_global_group( - ).nranks if group is None else group.nranks - return _legacy_C_ops.c_allgather(tensor, 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id, 'nranks', nranks) + nranks = ( + paddle.distributed.collective._get_global_group().nranks + if group is None + else group.nranks + ) + return _legacy_C_ops.c_allgather( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + ) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py index 79813756490..2668b580899 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py @@ -31,7 +31,11 @@ import paddle from paddle.fluid import core from paddle.optimizer import Optimizer from paddle.fluid.clip import ClipGradByGlobalNorm -from paddle.distributed.collective import _get_global_group, broadcast, new_group +from paddle.distributed.collective import ( + _get_global_group, + broadcast, + new_group, +) from .group_sharded_storage import ParamStorage, GradStorage from .group_sharded_utils import Type, device_guard, GroupShardedClipGrad @@ -63,21 +67,24 @@ class GroupShardedOptimizerStage2(Optimizer): # 4. Support offload function. # 5. Support the establishment of independent communication groups. # 6. Broadcast_fp16 is not supported now. - def __init__(self, - params, - optim, - group=None, - offload=False, - device="gpu", - pertrain_sync_models=True, - dp_group=None, - **kw): + def __init__( + self, + params, + optim, + group=None, + offload=False, + device="gpu", + pertrain_sync_models=True, + dp_group=None, + **kw + ): super().__init__(learning_rate=optim._learning_rate, parameters=params) assert core.is_compiled_with_cuda(), "Only GPU is supported now" # Segmentation information - self._dtype_rank_params = OrderedDict( + self._dtype_rank_params = ( + OrderedDict() ) # {dtype:[param1,param2]} device, rank, params self._param2rank = {} self.__segment_params = [] @@ -92,8 +99,9 @@ class GroupShardedOptimizerStage2(Optimizer): # record the last task used for comm overlap for sharding stage 2 self._comm_task = None - assert hasattr(self._optim, "_master_weights" - ), "Must use optimizer with _master_weights attribute" + assert hasattr( + self._optim, "_master_weights" + ), "Must use optimizer with _master_weights attribute" # Support parameter group and parameter list self._local_params = [] @@ -104,10 +112,17 @@ class GroupShardedOptimizerStage2(Optimizer): self._local_params.extend(list(params)) self._default_device = device - self._pfp16 = len( - list( - filter(lambda x: x.trainable and x.dtype == Type.fp16.value, - self._local_params))) > 0 + self._pfp16 = ( + len( + list( + filter( + lambda x: x.trainable and x.dtype == Type.fp16.value, + self._local_params, + ) + ) + ) + > 0 + ) self._broadcast_overlap = False self._forward_pre_hook_remove_helper = [] @@ -116,12 +131,14 @@ class GroupShardedOptimizerStage2(Optimizer): # Have to sort the params to make sure all params are in the forward using order. self._broadcast_order_params = sorted( self.local_params, - key=lambda x: int(x.name.split('.')[0].split('_')[-1])) + key=lambda x: int(x.name.split('.')[0].split('_')[-1]), + ) except ValueError: self._broadcast_order_params = None - self._group = new_group( - _get_global_group().ranks) if group is None else group + self._group = ( + new_group(_get_global_group().ranks) if group is None else group + ) # only support to combine stage2 and dp hybrid parallel now. self._dp_group = dp_group @@ -141,15 +158,19 @@ class GroupShardedOptimizerStage2(Optimizer): ) self._optim._grad_clip = GroupShardedClipGrad( - self._optim._grad_clip, paddle.get_device(), self._group) + self._optim._grad_clip, paddle.get_device(), self._group + ) if self._optim._parameter_list and isinstance( - self._optim._parameter_list[0], dict): + self._optim._parameter_list[0], dict + ): for item in self._optim._param_groups: if "grad_clip" in item.keys(): item["grad_clip"] = self._optim._grad_clip if offload: - assert self._pfp16, "Only support offload strategy while using \'Adam\', \'AdamW\' and \'Momentum\' optimizer with AMP/Pure FP16" + assert ( + self._pfp16 + ), "Only support offload strategy while using \'Adam\', \'AdamW\' and \'Momentum\' optimizer with AMP/Pure FP16" self.offload = offload # Using for offload self.offload_device = "cpu" @@ -171,16 +192,17 @@ class GroupShardedOptimizerStage2(Optimizer): """ for p in self._local_params: - broadcast(p, - src=self._global_root_rank, - group=self._group, - sync_op=True) + broadcast( + p, src=self._global_root_rank, group=self._group, sync_op=True + ) if self._dp_group: - broadcast(p, - src=self._dp_group.ranks[0], - group=self._dp_group, - sync_op=True) + broadcast( + p, + src=self._dp_group.ranks[0], + group=self._dp_group, + sync_op=True, + ) def _update_task(self, task): if self._reduce_overlap: @@ -194,15 +216,15 @@ class GroupShardedOptimizerStage2(Optimizer): # Enable gradients' reduces overlap with backward calculation. self._reduce_overlap = reduce_overlap - def _set_broadcast_overlap(self, - broadcast_overlap, - layers=None, - num_groups=None): + def _set_broadcast_overlap( + self, broadcast_overlap, layers=None, num_groups=None + ): # Enable post optimizer broadcasts overlap with the forward calculation of next batch. self._broadcast_overlap = broadcast_overlap if self._broadcast_overlap: - assert layers is not None, \ - "To enable broadcast overlap forward, please pass the module to the function." + assert ( + layers is not None + ), "To enable broadcast overlap forward, please pass the module to the function." self._layers = layers warnings.warn( "Setting overlap broadcast means the `paddle.device.cuda.synchronize()` " @@ -212,7 +234,8 @@ class GroupShardedOptimizerStage2(Optimizer): # Params' names should be like column_linear_32.w_0 patter to get the best performance. warnings.warn( r"The param name passed to the optimizer doesn't follow .+_[0-9]+\..+ patter, " - "overlap broadcast may harm the performance.") + "overlap broadcast may harm the performance." + ) self._broadcast_order_params = self._local_params if num_groups is None or num_groups > len(self._broadcast_order_params): @@ -222,9 +245,9 @@ class GroupShardedOptimizerStage2(Optimizer): ) num_groups = 1 - assert isinstance( - num_groups, - int) and num_groups > 0, "num_groups should be a positive integer" + assert ( + isinstance(num_groups, int) and num_groups > 0 + ), "num_groups should be a positive integer" self._number_of_broadcast_groups = num_groups self._broadcast_groups = [ @@ -244,7 +267,8 @@ class GroupShardedOptimizerStage2(Optimizer): name=param.name, value=param.cast(dtype=Type.fp32.value).numpy(), place=core.CPUPlace(), - stop_gradient=param.stop_gradient) + stop_gradient=param.stop_gradient, + ) else: for param in trainable_params: if param.dtype == Type.fp16.value: @@ -253,8 +277,7 @@ class GroupShardedOptimizerStage2(Optimizer): self._optim._master_weights[param.name] = master_tensor def _update_opt_status(self): - """Update optimizer status and parameter storage information, and special functions to be developed. - """ + """Update optimizer status and parameter storage information, and special functions to be developed.""" # func 1 self._integration_params() @@ -302,14 +325,16 @@ class GroupShardedOptimizerStage2(Optimizer): if len(self._dtype_rank_params) == 0: # Assign the parameters of each rank according to the type trainable_params = list( - filter(lambda x: x.trainable, self._local_params)) + filter(lambda x: x.trainable, self._local_params) + ) for param in trainable_params: if param.dtype not in self._dtype_rank_params.keys(): self._dtype_rank_params[param.dtype] = [ [] for _ in range(self.world_size) ] - self._dtype_rank_params[param.dtype][self.param2rank[ - param.name]].append(param) + self._dtype_rank_params[param.dtype][ + self.param2rank[param.name] + ].append(param) # Sort per rank params by size for dtype in self._dtype_rank_params.keys(): @@ -329,7 +354,8 @@ class GroupShardedOptimizerStage2(Optimizer): if dtype not in self._rank_buffer_size.keys(): self._rank_buffer_size[dtype] = {} for dst_rank, per_rank_params in enumerate( - self.dtype_rank_params[dtype]): + self.dtype_rank_params[dtype] + ): if dst_rank not in self._rank_buffer_size[dtype].keys(): self._rank_buffer_size[dtype][dst_rank] = 0 for param in per_rank_params: @@ -337,11 +363,15 @@ class GroupShardedOptimizerStage2(Optimizer): continue size = param._numel() * align[dtype] remaining = size % alignment[self._default_device] - ali = 0 if remaining == 0 else alignment[ - self._default_device] - remaining + ali = ( + 0 + if remaining == 0 + else alignment[self._default_device] - remaining + ) align_ = ali // align[dtype] - self._rank_buffer_size[dtype][dst_rank] += param._numel( - ) + align_ + self._rank_buffer_size[dtype][dst_rank] += ( + param._numel() + align_ + ) self._param2align[param.name] = align_ return self._rank_buffer_size @@ -360,23 +390,27 @@ class GroupShardedOptimizerStage2(Optimizer): # Merge all the trainable params in a single InternalStorage trainable_params = list( - filter(lambda x: x.trainable, params)) + filter(lambda x: x.trainable, params) + ) if self._pfp16 and dst_rank == self._rank: self._generate_master_params(trainable_params) if trainable_params: param_storage = ParamStorage( size=self.rank_buffer_size[dtype][dst_rank], dtype=dtype, - device=self._default_device) + device=self._default_device, + ) - param_storage.add_rank_params(trainable_params, - self._param2align) + param_storage.add_rank_params( + trainable_params, self._param2align + ) self.param_storages[dtype][dst_rank] = param_storage # Clear the InternalStorage keys which are not in use anymore dtype_in_use = list(self.dtype_rank_params.keys()) dtype_to_pop = list( - filter(lambda x: x not in dtype_in_use, self.param_storages.keys())) + filter(lambda x: x not in dtype_in_use, self.param_storages.keys()) + ) for d in dtype_to_pop: self.param_storages.pop(d) @@ -386,8 +420,11 @@ class GroupShardedOptimizerStage2(Optimizer): for param in cpu_master_params: size = param._numel() * align[Type.fp32.value] remaining = size % alignment[self.offload_device] - ali = 0 if remaining == 0 else alignment[ - self.offload_device] - remaining + ali = ( + 0 + if remaining == 0 + else alignment[self.offload_device] - remaining + ) align_ = ali // align[Type.fp32.value] self.offload_buffer_size += param._numel() + align_ self.offload_param2align[param.name] = align_ @@ -397,10 +434,12 @@ class GroupShardedOptimizerStage2(Optimizer): self.offload_params = ParamStorage( size=self.offload_buffer_size, dtype=Type.fp32.value, - device=self.offload_device) + device=self.offload_device, + ) self.offload_params.buffer.name = "offload_buffer" self.offload_params.add_rank_params( - cpu_master_params, self.offload_param2align, False) + cpu_master_params, self.offload_param2align, False + ) self.offload_params.buffer.stop_gradient = False self.offload_grads = GradStorage( @@ -409,14 +448,16 @@ class GroupShardedOptimizerStage2(Optimizer): device=self.offload_device, destination=self._rank, parm2align=self.offload_param2align, - convert_cpu=True) + convert_cpu=True, + ) for p in cpu_master_params: self.offload_grads.add_grad( - p, self.offload_param2align[p.name]) + p, self.offload_param2align[p.name] + ) self._optim._master_weights[ - self.offload_params.buffer. - name] = self.offload_params.buffer + self.offload_params.buffer.name + ] = self.offload_params.buffer def _offload_acc_grad(self, param_name, grad_fp32_cpu): """accumulate grads with offload strategy""" @@ -424,12 +465,14 @@ class GroupShardedOptimizerStage2(Optimizer): if param_name in self._master_params.keys(): if self._master_params[param_name].grad is None: self._master_params[param_name]._copy_gradient_from( - grad_fp32_cpu) + grad_fp32_cpu + ) else: self._master_params[param_name].grad.add_(grad_fp32_cpu) self.offload_params.buffer._copy_gradient_from( - self.offload_grads.buffer) + self.offload_grads.buffer + ) def _offload_scale_grad(self, scale_size): """scale grads with offload strategy""" @@ -456,7 +499,7 @@ class GroupShardedOptimizerStage2(Optimizer): if self.offload: params_list = [self.offload_params.buffer] - #TODO(Baibaifan): Offload will support param_groups later + # TODO(Baibaifan): Offload will support param_groups later if not isinstance(self._optim._param_groups[0], dict): self._optim._parameter_list = params_list self._optim._param_groups = params_list @@ -468,8 +511,11 @@ class GroupShardedOptimizerStage2(Optimizer): for param in self._local_params: if param.name in self._master_params.keys(): - param.set_value(self._master_params[param.name].cuda( - self.dev_id).cast(dtype=param.dtype)) + param.set_value( + self._master_params[param.name] + .cuda(self.dev_id) + .cast(dtype=param.dtype) + ) else: self._optim.step() @@ -478,7 +524,8 @@ class GroupShardedOptimizerStage2(Optimizer): def minimize(self): raise RuntimeError( - "optimizer.minimize() not support now, please use optimizer.step()") + "optimizer.minimize() not support now, please use optimizer.step()" + ) def set_state_dict(self, state_dict): self._optim.set_state_dict(state_dict) @@ -501,10 +548,12 @@ class GroupShardedOptimizerStage2(Optimizer): else: for dtype_per_rank in self.param_storages.values(): for dst_rank, internal_storage in dtype_per_rank.items(): - broadcast(tensor=internal_storage.buffer, - src=self._group.ranks[dst_rank], - group=self._group, - sync_op=True) + broadcast( + tensor=internal_storage.buffer, + src=self._group.ranks[dst_rank], + group=self._group, + sync_op=True, + ) def _forward_pre_hook_function(self, tasks): # Since the layers will call pre hook by `forward_pre_hook(self, inputs)`, @@ -527,10 +576,12 @@ class GroupShardedOptimizerStage2(Optimizer): if x.trainable: group = self._broadcast_groups[group_idx] group_idx = (group_idx + 1) % self._number_of_broadcast_groups - task = broadcast(tensor=x, - src=group.ranks[self._param2rank[x.name]], - group=group, - sync_op=False) + task = broadcast( + tensor=x, + src=group.ranks[self._param2rank[x.name]], + group=group, + sync_op=False, + ) assert x.name not in param2task param2task[x.name] = task @@ -544,4 +595,6 @@ class GroupShardedOptimizerStage2(Optimizer): tasks.append(param2task[param.name]) self._forward_pre_hook_remove_helper.append( layer.register_forward_pre_hook( - self._forward_pre_hook_function(tasks))) + self._forward_pre_hook_function(tasks) + ) + ) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py index 6c0716f7bbb..044111cc349 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py @@ -58,38 +58,50 @@ class GroupShardedStage2(nn.Layer): # 5. Support the establishment of independent communication groups. def __init__( - self, - layer, - sharding_optimizer, - group=None, - sync_buffers=False, - buffer_max_size=2**23, #8MB - auto_refresh_trainable=True, - device="gpu", - dp_group=None): + self, + layer, + sharding_optimizer, + group=None, + sync_buffers=False, + buffer_max_size=2**23, # 8MB + auto_refresh_trainable=True, + device="gpu", + dp_group=None, + ): super().__init__() # training options self._layer = layer - self._sharding_optimizers = [ - sharding_optimizer - ] if not isinstance(sharding_optimizer, list) else sharding_optimizer + self._sharding_optimizers = ( + [sharding_optimizer] + if not isinstance(sharding_optimizer, list) + else sharding_optimizer + ) assert all( list( - map(lambda opt: isinstance(opt, GroupShardedOptimizerStage2), - self._sharding_optimizers)) + map( + lambda opt: isinstance(opt, GroupShardedOptimizerStage2), + self._sharding_optimizers, + ) + ) ), "Please use GroupShardedOptimizerStage2 optimizer" self._sync_buffers = sync_buffers self._auto_refresh_trainable = auto_refresh_trainable # Communication related attributes - self._group = collective.new_group( - collective._get_global_group().ranks) if group is None else group + self._group = ( + collective.new_group(collective._get_global_group().ranks) + if group is None + else group + ) self._world_size_scaling = 1.0 / self._group.nranks - assert self._group.nranks > 1, "Training must be distributed, ranks must be greater than 1" + assert ( + self._group.nranks > 1 + ), "Training must be distributed, ranks must be greater than 1" self._rank = self._group.rank self._global_root_rank = self._group.ranks[ - 0] # picking ranks index 0 as the reference + 0 + ] # picking ranks index 0 as the reference self._default_device = device self._dp_group = dp_group @@ -106,15 +118,17 @@ class GroupShardedStage2(nn.Layer): self._trainable_param2rank = {} self._trainable_param2align = {} self._trainable_params = list( - filter(lambda x: x.trainable, self._all_params)) + filter(lambda x: x.trainable, self._all_params) + ) self._trainable_mask = list(map(_trainable, self._trainable_params)) self._param_grads = [] # Set grad storage size & Display param sizes and model sizes model_size = sum([p._numel() for p in self._layer.parameters()]) assert buffer_max_size >= 0, "buffer_max_size must be GE than 0." - self._buffer_max_size = self._rank_buffer_size(buffer_max_size, - model_size) + self._buffer_max_size = self._rank_buffer_size( + buffer_max_size, model_size + ) self._use_grad_storage = buffer_max_size > 0 self._grad_storages = {} # {dtype: {rank: GradStorage}} self._has_grad_storage = [] @@ -123,11 +137,12 @@ class GroupShardedStage2(nn.Layer): # Offload # TODO(haohongxiang): Now it's not be supported for multi-optimizers using Offload strategy self._offload_optims = list( - filter(lambda optim: optim.offload, self._sharding_optimizers)) + filter(lambda optim: optim.offload, self._sharding_optimizers) + ) if len(self._offload_optims) > 0: - assert len( - self._sharding_optimizers - ) == 1, "Only support offload strategy for single optimizer" + assert ( + len(self._sharding_optimizers) == 1 + ), "Only support offload strategy for single optimizer" self._offload = len(self._offload_optims) > 0 self._offload_device = "cpu" @@ -170,17 +185,21 @@ class GroupShardedStage2(nn.Layer): return fw def set_state_dict(self, state_dict, use_structured_name=True): - self._layer.set_state_dict(state_dict, - use_structured_name=use_structured_name) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): + self._layer.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): return self._layer.state_dict( destination=destination, include_sublayers=include_sublayers, - structured_name_prefix=structured_name_prefix) + structured_name_prefix=structured_name_prefix, + ) def _clear_gradients(self): """ @@ -188,8 +207,10 @@ class GroupShardedStage2(nn.Layer): """ # Release grad storages for dtype in self._grad_storages.keys(): - if not self._offload and self._rank in self._grad_storages[ - dtype].keys(): + if ( + not self._offload + and self._rank in self._grad_storages[dtype].keys() + ): self._grad_storages[dtype][self._rank].buffer.zero_() # Release grads of params @@ -213,10 +234,13 @@ class GroupShardedStage2(nn.Layer): # Scale grad storages for dtype in self._grad_storages.keys(): - if not self._offload and self._rank in self._grad_storages[ - dtype].keys(): + if ( + not self._offload + and self._rank in self._grad_storages[dtype].keys() + ): self._grad_storages[dtype][self._rank].buffer.scale_( - scale=scale_factor) + scale=scale_factor + ) # Scale grads of params with paddle.no_grad(): @@ -246,7 +270,9 @@ class GroupShardedStage2(nn.Layer): Synchronously or asynchronously convert the data type of the layer, the device is not supported now. """ assert isinstance(device, str), "Device must be type str" - assert device == self._default_device, "New devices are not supported, because of the optimizer state is not sync" + assert ( + device == self._default_device + ), "New devices are not supported, because of the optimizer state is not sync" self._layer.to(device=device, dtype=dtype, blocking=blocking) @@ -254,14 +280,15 @@ class GroupShardedStage2(nn.Layer): self._fresh_trainable() def _fresh_trainable(self): - """ Whether to update training parameters. """ + """Whether to update training parameters.""" # Make sure that this is not done while gradients are waiting to be reduced (if no_sync context for instance) if reduce(lambda x, y: x or y, self._grad_reduced, False): logging.warning("Grads waiting to be reduced.") self._trainable_params = list( - filter(lambda x: x.trainable, self._all_params)) + filter(lambda x: x.trainable, self._all_params) + ) self._trainable_params.sort(key=lambda x: x._numel()) self._trainable_param2rank = {} @@ -271,14 +298,19 @@ class GroupShardedStage2(nn.Layer): optim._update_opt_status() # Get the parameters split by the optimizer according to rank - for per_rank_params in optim.dtype_rank_params.values( + for ( + per_rank_params + ) in ( + optim.dtype_rank_params.values() ): # all the params from all ranks for params in per_rank_params: for param in filter(lambda x: x.trainable, params): self._trainable_param2rank[ - param.name] = optim.param2rank[param.name] + param.name + ] = optim.param2rank[param.name] self._trainable_param2align[ - param.name] = optim._param2align[param.name] + param.name + ] = optim._param2align[param.name] # Create grad_storage self._setup_use_grad_storage() @@ -292,16 +324,17 @@ class GroupShardedStage2(nn.Layer): """ for buffer in self._layer.buffers(include_sublayers=True): - collective.broadcast(buffer, - self._global_root_rank, - self._group, - sync_op=True) + collective.broadcast( + buffer, self._global_root_rank, self._group, sync_op=True + ) if self._dp_group and self._dp_group.nranks > 1: - collective.broadcast(buffer, - self._dp_group.ranks[0], - self._dp_group, - sync_op=True) + collective.broadcast( + buffer, + self._dp_group.ranks[0], + self._dp_group, + sync_op=True, + ) def __getattr__(self, name): """Forward missing attributes to wrapped layer.""" @@ -327,9 +360,9 @@ class GroupShardedStage2(nn.Layer): # model._set_reduce_overlap(True) self._reduce_overlap = reduce_overlap if self._reduce_overlap: - assert len( - self._sharding_optimizers - ) == 1, "Only support comm overlap strategy for single optimizer" + assert ( + len(self._sharding_optimizers) == 1 + ), "Only support comm overlap strategy for single optimizer" self._sharding_optimizers[0]._set_reduce_overlap(reduce_overlap) def _get_reduce_fn(self, index, param, dst_rank): @@ -345,7 +378,9 @@ class GroupShardedStage2(nn.Layer): def reduce(*_): # Skip gradient reduction, do not change status information if self._grad_reduced[index]: - assert param.grad is not None, "Parameter gradient cannot be None" + assert ( + param.grad is not None + ), "Parameter gradient cannot be None" # Change reduce information self._grad_reduced[index] = False @@ -356,26 +391,35 @@ class GroupShardedStage2(nn.Layer): param.clear_gradient(False) elif self._offload: tmp_grad = param.grad.cast( - dtype=Type.fp32.value).cpu() + dtype=Type.fp32.value + ).cpu() self._sharding_optimizers[0]._offload_acc_grad( - param.name, tmp_grad) + param.name, tmp_grad + ) del tmp_grad param.clear_gradient(False) # Synchronize the reduce parameter gradient asynchronize self._sharding_optimizers[0]._update_task( - collective.reduce(tensor=param.grad, - dst=self._group.ranks[dst_rank], - group=self._group, - sync_op=not self._reduce_overlap)) + collective.reduce( + tensor=param.grad, + dst=self._group.ranks[dst_rank], + group=self._group, + sync_op=not self._reduce_overlap, + ) + ) if self._dp_group and self._dp_group.nranks > 1: - assert not self._reduce_overlap, 'dp + stage2 hybrid parallel only Synchronize due to the new communication lib.' - #TODO(wuhuachao):after the new communication lib upgrading, overlapping the comm of dp + stage2. - collective.all_reduce(tensor=param.grad, - group=self._dp_group, - sync_op=True) + assert ( + not self._reduce_overlap + ), 'dp + stage2 hybrid parallel only Synchronize due to the new communication lib.' + # TODO(wuhuachao):after the new communication lib upgrading, overlapping the comm of dp + stage2. + collective.all_reduce( + tensor=param.grad, + group=self._dp_group, + sync_op=True, + ) # Clear the task flow and trigger callback to clear the redundant gradient # self._clear_task_flow() @@ -388,7 +432,9 @@ class GroupShardedStage2(nn.Layer): def reduce(*_): # Skip gradient reduction, do not change status information if self._grad_reduced[index]: - assert param.grad is not None, "Parameter gradient cannot be None" + assert ( + param.grad is not None + ), "Parameter gradient cannot be None" # Change reduce information self._grad_reduced[index] = False @@ -410,9 +456,11 @@ class GroupShardedStage2(nn.Layer): for p in grad_storage._params: with device_guard(): tmp_grad = p.grad.cast( - dtype=Type.fp32.value) + dtype=Type.fp32.value + ) self._sharding_optimizers[ - 0]._offload_acc_grad(p.name, tmp_grad) + 0 + ]._offload_acc_grad(p.name, tmp_grad) p.clear_gradient(False) grad_storage._device = self._default_device grad_storage.buffer._clear_data() @@ -425,14 +473,20 @@ class GroupShardedStage2(nn.Layer): tensor=grad_storage.buffer, dst=self._group.ranks[grad_storage.destination], group=self._group, - sync_op=not self._reduce_overlap)) + sync_op=not self._reduce_overlap, + ) + ) if self._dp_group and self._dp_group.nranks > 1: - assert not self._reduce_overlap, 'dp + stage2 hybrid parallel only Synchronize due to the new communication lib.' - #TODO(wuhuachao):after the new communication lib upgrading, overlapping the comm of dp + stage2. - collective.all_reduce(tensor=grad_storage.buffer, - group=self._dp_group, - sync_op=True) + assert ( + not self._reduce_overlap + ), 'dp + stage2 hybrid parallel only Synchronize due to the new communication lib.' + # TODO(wuhuachao):after the new communication lib upgrading, overlapping the comm of dp + stage2. + collective.all_reduce( + tensor=grad_storage.buffer, + group=self._dp_group, + sync_op=True, + ) cleanup() @@ -460,7 +514,8 @@ class GroupShardedStage2(nn.Layer): reduce_function = self._get_reduce_fn(index, param, dst_rank) self._bw_hooks.append( - param._register_backward_hook(reduce_function)) + param._register_backward_hook(reduce_function) + ) def _setup_use_grad_storage(self): """ @@ -483,25 +538,32 @@ class GroupShardedStage2(nn.Layer): dtype=param.dtype, device=self._default_device, destination=dst_rank, - parm2align=self._trainable_param2align) + parm2align=self._trainable_param2align, + ) # Criteria to decide whether this parameter is to be put in GradStorage if self._grad_storages[param.dtype][dst_rank].can_add_grad_view( - param, self._trainable_param2align[param.name]): + param, self._trainable_param2align[param.name] + ): self._grad_storages[param.dtype][dst_rank].add_grad( - param, self._trainable_param2align[param.name]) + param, self._trainable_param2align[param.name] + ) self._has_grad_storage[index] = True else: self._param_grads.append(param.name) print( - "Can not add param: {}, param's shape: {}, param align: {}, grad_storages fill: {}, " - .format(param.name, param.shape, - self._trainable_param2align[param.name], - self._grad_storages[param.dtype][dst_rank]._fill)) + "Can not add param: {}, param's shape: {}, param align: {}, grad_storages fill: {}, ".format( + param.name, + param.shape, + self._trainable_param2align[param.name], + self._grad_storages[param.dtype][dst_rank]._fill, + ) + ) for dtype in self._grad_storages.keys(): self._grad_storage_list.extend( - list(self._grad_storages[dtype].values())) + list(self._grad_storages[dtype].values()) + ) # def _clear_task_flow(self): # """Try to consume the previous tasks.""" @@ -553,21 +615,27 @@ class GroupShardedStage2(nn.Layer): if Type.fp16.value in rank_buffer_size.keys(): # FP16 GradStorage and model size logger_.info( - "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.fp16.value] / 2**19, - model_size / 2**19)) + "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.fp16.value] / 2**19, + model_size / 2**19, + ) + ) if Type.bf16.value in rank_buffer_size.keys(): # FP16 GradStorage and model size logger_.info( - "====== BF16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.bf16.value] / 2**19, - model_size / 2**19)) + "====== BF16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.bf16.value] / 2**19, + model_size / 2**19, + ) + ) if Type.fp32.value in rank_buffer_size.keys(): # FP32 GradStorage and model size logger_.info( - "====== FP32 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.fp32.value] / 2**18, - model_size / 2**18)) + "====== FP32 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.fp32.value] / 2**18, + model_size / 2**18, + ) + ) return rank_buffer_size def _redefine_opt_step(self): diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py index 00ce653e1df..e80ea0531f3 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py @@ -72,16 +72,18 @@ class GroupShardedStage3(nn.Layer): # 3. Support offload function. # 4. Support the establishment of independent communication groups. - def __init__(self, - layer, - optimizer, - group=None, - sync_buffers=False, - device="gpu", - segment_size=2**20, - pertrain_sync_models=True, - offload=False, - sync_comm=False): + def __init__( + self, + layer, + optimizer, + group=None, + sync_buffers=False, + device="gpu", + segment_size=2**20, + pertrain_sync_models=True, + offload=False, + sync_comm=False, + ): super().__init__() # Default configs @@ -96,27 +98,40 @@ class GroupShardedStage3(nn.Layer): self._segment_size = segment_size global DEV - DEV = "cpu" if paddle.get_device() == "cpu" else paddle.get_device( - ).split(":")[0] + DEV = ( + "cpu" + if paddle.get_device() == "cpu" + else paddle.get_device().split(":")[0] + ) global DEV_ID - DEV_ID = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + DEV_ID = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) global param2dtype param2dtype = dict() # Communication group establishment - self._group = collective.new_group( - collective._get_global_group().ranks) if group is None else group + self._group = ( + collective.new_group(collective._get_global_group().ranks) + if group is None + else group + ) self._world_size_scaling = 1.0 / self._group.nranks - assert self._group.nranks > 1, "Training must be distributed, ranks must be greater than 1." + assert ( + self._group.nranks > 1 + ), "Training must be distributed, ranks must be greater than 1." self._rank = self._group.rank self._global_root_rank = self._group.ranks[ - 0] # picking ranks index 0 as the reference + 0 + ] # picking ranks index 0 as the reference # Parameter segmentation for global ranks # After flatten -> self._param2buffer_size, self._param2buffer, self._trainable_params self._param2buffer_size = dict() # {param.name: size} - self._param2buffer = dict( + self._param2buffer = ( + dict() ) # {param.name: [(start0, end0),(start1, end1), ...]} self._trainable_params = dict() # {id(layer): [trainable_params]} self._unslice_params = set() # param's numel <= segment_size @@ -124,9 +139,11 @@ class GroupShardedStage3(nn.Layer): self._grad_storages = dict() # {param.dtype: GradStorage} assert not isinstance( - optimizer, list), "Multiple optimizers are not supported now." - self._optim = _OptimizerWrapper(optimizer, self._offload, self._group, - self._update_params_slice) + optimizer, list + ), "Multiple optimizers are not supported now." + self._optim = _OptimizerWrapper( + optimizer, self._offload, self._group, self._update_params_slice + ) self._ori_parameter_list = self._optim._parameter_list self._ori_param_groups = self._optim._param_groups @@ -136,9 +153,11 @@ class GroupShardedStage3(nn.Layer): "While using ClipGradByGlobalNorm in GroupShardedStage3, the grad clip of original optimizer will be changed." ) self._optim._grad_clip = GroupShardedClipGrad( - self._optim._grad_clip, paddle.get_device(), self._group) + self._optim._grad_clip, paddle.get_device(), self._group + ) if self._optim._parameter_list and isinstance( - self._optim._parameter_list[0], dict): + self._optim._parameter_list[0], dict + ): for item in self._optim._param_groups: if "grad_clip" in item.keys(): item["grad_clip"] = self._optim._grad_clip @@ -177,22 +196,24 @@ class GroupShardedStage3(nn.Layer): """ for p in self._layer.parameters(): - collective.broadcast(p, - src=self._global_root_rank, - group=self._group, - sync_op=True) + collective.broadcast( + p, src=self._global_root_rank, group=self._group, sync_op=True + ) def _clear_gradients(self): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) # 1.Handle param's slice trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) for param in trainable_params: - assert hasattr(param, "fw_storage" - ), "Find {} don't have fw_storage attribute.".format( - param.name) + assert hasattr( + param, "fw_storage" + ), "Find {} don't have fw_storage attribute.".format(param.name) param.fw_storage.clear_gradient(False) param.bw_storage._clear() @@ -206,8 +227,10 @@ class GroupShardedStage3(nn.Layer): param.clear_gradient(False) tmp_var = param.cuda(DEV_ID) - if tmp_var.dtype == Type.fp32.value and param2dtype[ - param.name] == Type.fp16.value: + if ( + tmp_var.dtype == Type.fp32.value + and param2dtype[param.name] == Type.fp16.value + ): tmp_var = paddle.cast(tmp_var, Type.fp16.value) tmp_var._share_buffer_to(param) del tmp_var @@ -222,9 +245,11 @@ class GroupShardedStage3(nn.Layer): if not isinstance(self._optim._param_groups[0], dict): slice_params = [param.fw_storage for param in update_list] self._optim._parameter_list = slice_params + list( - self._unslice_params) + self._unslice_params + ) self._optim._param_groups = slice_params + list( - self._unslice_params) + self._unslice_params + ) else: for param_group in self._optim._param_groups: p_group = [] @@ -250,17 +275,21 @@ class GroupShardedStage3(nn.Layer): return fw def set_state_dict(self, state_dict, use_structured_name=True): - self._layer.set_state_dict(state_dict, - use_structured_name=use_structured_name) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): + self._layer.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): return self._layer.state_dict( destination=destination, include_sublayers=include_sublayers, - structured_name_prefix=structured_name_prefix) + structured_name_prefix=structured_name_prefix, + ) def _handle_unslice_params(self): buffer_size = dict() @@ -287,9 +316,11 @@ class GroupShardedStage3(nn.Layer): dtype=param.dtype, device=self._default_device, destination=self._rank, - parm2align=self._unslice_params2align) + parm2align=self._unslice_params2align, + ) self._grad_storages[param.dtype].add_grad( - param, self._unslice_params2align[param.name]) + param, self._unslice_params2align[param.name] + ) def _segment_rank_params(self, layer, name="last_layer"): """ @@ -329,8 +360,11 @@ class GroupShardedStage3(nn.Layer): align_ = self._param2align(param) offset = align_ + param._numel() - buffer_size = offset if offset % self._group.nranks == 0 else offset + self._group.nranks - ( - offset % self._group.nranks) + buffer_size = ( + offset + if offset % self._group.nranks == 0 + else offset + self._group.nranks - (offset % self._group.nranks) + ) self._param2buffer_size[param.name] = buffer_size # 2.Combination param buffer @@ -339,7 +373,8 @@ class GroupShardedStage3(nn.Layer): for rank_ in range(self._group.nranks): self._param2buffer[param.name].append( - (rank_ * pre_buffer, (rank_ + 1) * pre_buffer)) + (rank_ * pre_buffer, (rank_ + 1) * pre_buffer) + ) # Record param's dtype param2dtype[param.name] = param.dtype @@ -351,10 +386,11 @@ class GroupShardedStage3(nn.Layer): This is a function to simplify the handling of parameter InternalStorages. """ assert isinstance(buffer_size, int) - value = np.zeros( - buffer_size, - dtype=np.float16) if Type.fp16.value == param.dtype else np.zeros( - buffer_size, dtype=np.float32) + value = ( + np.zeros(buffer_size, dtype=np.float16) + if Type.fp16.value == param.dtype + else np.zeros(buffer_size, dtype=np.float32) + ) buffer = core.eager.Tensor(value=value, place=core.CPUPlace()) param_shape = param.shape @@ -376,16 +412,19 @@ class GroupShardedStage3(nn.Layer): if self._offload: with device_guard(): tmp_tensor = buffer._slice(start, end) - param.fw_storage = core.eager.Tensor(value=tmp_tensor, - place=core.CPUPlace(), - name="slice@" + param.name) + param.fw_storage = core.eager.Tensor( + value=tmp_tensor, + place=core.CPUPlace(), + name="slice@" + param.name, + ) with device_guard(): - param.master_weight = paddle.cast(param.fw_storage, - Type.fp32.value) + param.master_weight = paddle.cast( + param.fw_storage, Type.fp32.value + ) else: - param.fw_storage = core.eager.Tensor(value=buffer._slice( - start, end), - name="slice@" + param.name) + param.fw_storage = core.eager.Tensor( + value=buffer._slice(start, end), name="slice@" + param.name + ) param.status = "part" # Updata optimizer master weights @@ -414,20 +453,32 @@ class GroupShardedStage3(nn.Layer): self._register_forward_hooks(sub_layer) def _register_forward_all_hooks(self, sub_layer, task_flow): - def _forward_pre_hook(layer, inputs): - return ForwardPreHooks(layer, self._order_tracer, - self._trainable_params, - self._param2buffer_size, self._group, - self._sync_comm, self._offload, task_flow) + return ForwardPreHooks( + layer, + self._order_tracer, + self._trainable_params, + self._param2buffer_size, + self._group, + self._sync_comm, + self._offload, + task_flow, + ) def _forward_post_hook(layer, inputs, outputs): - return ForwardPostHooks.apply(outputs, layer, self._order_tracer, - self._trainable_params, - self._param2buffer, - self._param2buffer_size, self._rank, - self._group, self._sync_comm, - self._offload, task_flow) + return ForwardPostHooks.apply( + outputs, + layer, + self._order_tracer, + self._trainable_params, + self._param2buffer, + self._param2buffer_size, + self._rank, + self._group, + self._sync_comm, + self._offload, + task_flow, + ) # register previous forward hooks sub_layer.register_forward_pre_hook(_forward_pre_hook) @@ -442,10 +493,9 @@ class GroupShardedStage3(nn.Layer): """ for buffer in self._layer.buffers(include_sublayers=True): - collective.broadcast(buffer, - self._global_root_rank, - self._group, - sync_op=True) + collective.broadcast( + buffer, self._global_root_rank, self._group, sync_op=True + ) def __getattr__(self, name): """Forward missing attributes to wrapped layer.""" @@ -462,14 +512,16 @@ class GroupShardedStage3(nn.Layer): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) # 1.Handle param's slice for param in trainable_params: assert hasattr( - param, - "fw_storage"), "Find {} don't have fw_storage attribute".format( - param.name) + param, "fw_storage" + ), "Find {} don't have fw_storage attribute".format(param.name) # Gradient average if self._offload: with device_guard(): @@ -507,16 +559,21 @@ class GroupShardedStage3(nn.Layer): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) - t_flow = _allgather_buffer(trainable_params, - self._group, - param2buffer_size=self._param2buffer_size, - use_calc_stream=True, - task_flow=TaskFlow(), - sync_wait=True, - offload=self._offload, - convert2cpu=convert2cpu) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) + t_flow = _allgather_buffer( + trainable_params, + self._group, + param2buffer_size=self._param2buffer_size, + use_calc_stream=True, + task_flow=TaskFlow(), + sync_wait=True, + offload=self._offload, + convert2cpu=convert2cpu, + ) if convert2cpu: for param in trainable_params: t_flow.full_param[param.name][0]._share_buffer_to(param) @@ -527,15 +584,17 @@ class GroupShardedStage3(nn.Layer): def _register_backward_hooks(self): current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) for param in trainable_params: allreduce_function = self._get_allreduce_fn(param) param._register_backward_hook(allreduce_function) def _get_allreduce_fn(self, param): - @paddle.autograd.no_grad() def allreduce_(*_): if param.name in self._task_flow.full_grad.keys(): @@ -545,21 +604,25 @@ class GroupShardedStage3(nn.Layer): start, end = self._param2buffer[param.name][self._rank] if param.bw_storage is None: - param.bw_storage = full_grad._slice(start, - end).detach().clone() + param.bw_storage = ( + full_grad._slice(start, end).detach().clone() + ) if self._offload: param.bw_storage = _device2cpu(param.bw_storage, True) else: if self._offload: cpu_grad = _device2cpu( - full_grad._slice(start, end).detach().clone(), True) + full_grad._slice(start, end).detach().clone(), True + ) with device_guard(): param.bw_storage = paddle.add( - param.bw_storage, cpu_grad) + param.bw_storage, cpu_grad + ) else: param.bw_storage = paddle.add( param.bw_storage, - full_grad._slice(start, end).detach().clone()) + full_grad._slice(start, end).detach().clone(), + ) param.clear_gradient(False) del self._task_flow.full_grad[param.name] @@ -568,8 +631,12 @@ class GroupShardedStage3(nn.Layer): param.use_count = 0 param._clear_data() start, end = self._param2buffer[param.name][self._rank] - param.fw_storage = self._task_flow.full_param[ - param.name][0]._slice(start, end).detach().clone() + param.fw_storage = ( + self._task_flow.full_param[param.name][0] + ._slice(start, end) + .detach() + .clone() + ) param.status = "part" del self._task_flow.full_param[param.name] @@ -583,8 +650,9 @@ class GroupShardedStage3(nn.Layer): # CUDA alignment 256 bytes size = param._numel() * align[param.dtype] remaining = size % alignment[self._default_device] - ali = 0 if remaining == 0 else alignment[ - self._default_device] - remaining + ali = ( + 0 if remaining == 0 else alignment[self._default_device] - remaining + ) align_ = ali // align[param.dtype] return align_ @@ -618,8 +686,16 @@ class GroupShardedStage3(nn.Layer): self._optim.clear_grad = MethodType(_opt_clear, self._optim) -def ForwardPreHooks(layer, order_tracer, trainable_params, param2buffer_size, - group, sync_comm, offload, task_flow): +def ForwardPreHooks( + layer, + order_tracer, + trainable_params, + param2buffer_size, + group, + sync_comm, + offload, + task_flow, +): # Record layer's id layer_id = id(layer) @@ -634,35 +710,55 @@ def ForwardPreHooks(layer, order_tracer, trainable_params, param2buffer_size, # Whether to use calc stream task_flow.use_calc[layer_id] = use_calc # wait current layer params - _wait_layer(trainable_params[layer_id], task_flow, group, - param2buffer_size, use_calc, offload) - - if layer_id == order_tracer["layer"][-1]: return + _wait_layer( + trainable_params[layer_id], + task_flow, + group, + param2buffer_size, + use_calc, + offload, + ) + + if layer_id == order_tracer["layer"][-1]: + return order_ = order_tracer[layer_id] layer_id = order_tracer["layer"][order_ + 1] - _allgather_buffer(trainable_params[layer_id], - group, - param2buffer_size=param2buffer_size, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_id], + group, + param2buffer_size=param2buffer_size, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) return class ForwardPostHooks(PyLayer): - @staticmethod - def forward(ctx, inputs, layer, order_tracer, trainable_params, - param2buffer, param2buffer_size, rank, group, sync_comm, - offload, task_flow): + def forward( + ctx, + inputs, + layer, + order_tracer, + trainable_params, + param2buffer, + param2buffer_size, + rank, + group, + sync_comm, + offload, + task_flow, + ): layer_id = id(layer) # release current layer full params - _release_param(trainable_params[layer_id], param2buffer, rank, - task_flow, offload) + _release_param( + trainable_params[layer_id], param2buffer, rank, task_flow, offload + ) if layer_id not in order_tracer.keys(): order_ = order_tracer["order"] @@ -670,7 +766,7 @@ class ForwardPostHooks(PyLayer): order_tracer["order"] += 1 order_tracer["layer"].append(layer_id) - #Record fw info + # Record fw info ctx.order_tracer = order_tracer ctx.task_flow = task_flow ctx.group = group @@ -698,32 +794,43 @@ class ForwardPostHooks(PyLayer): # Allgather params synchronization if sync_comm: use_calc, sync_wait = True, True - _allgather_buffer(trainable_params[layer_id], - group, - param2buffer_size=param2buffer_size, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_id], + group, + param2buffer_size=param2buffer_size, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) else: - _wait_layer(trainable_params[layer_id], task_flow, group, - param2buffer_size, use_calc, offload) + _wait_layer( + trainable_params[layer_id], + task_flow, + group, + param2buffer_size, + use_calc, + offload, + ) # Create params's grad - _create_params_grad(trainable_params[layer_id], param2buffer_size, - task_flow) + _create_params_grad( + trainable_params[layer_id], param2buffer_size, task_flow + ) # Whether to use calc stream task_flow.use_calc[layer_id] = use_calc if layer_id != order_tracer["layer"][0] and not sync_comm: layer_next_id = order_tracer["layer"][order_tracer[layer_id] - 1] - _allgather_buffer(trainable_params[layer_next_id], - group, - param2buffer_size=param2buffer_size, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_next_id], + group, + param2buffer_size=param2buffer_size, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) return args @@ -733,22 +840,22 @@ class TaskFlow: Task flows, one way linked list for task acquisition. """ - def __init__(self, - full_param=dict(), - full_grad=dict(), - use_calc=dict(), - callback=None): + def __init__( + self, + full_param=dict(), + full_grad=dict(), + use_calc=dict(), + callback=None, + ): self.full_param = full_param self.full_grad = full_grad self.use_calc = use_calc self.callback = callback -def _release_param(trainable_params, - param2buffer, - rank, - task_flow, - offload=False): +def _release_param( + trainable_params, param2buffer, rank, task_flow, offload=False +): for param in trainable_params: # async communicate share weight not clear param.use_count -= 1 @@ -757,8 +864,12 @@ def _release_param(trainable_params, if param.name in task_flow.full_param.keys(): start, end = param2buffer[param.name][rank] with paddle.amp.auto_cast(enable=False): - param.fw_storage = task_flow.full_param[ - param.name][0]._slice(start, end).detach().clone() + param.fw_storage = ( + task_flow.full_param[param.name][0] + ._slice(start, end) + .detach() + .clone() + ) param.status = "part" del task_flow.full_param[param.name] @@ -767,12 +878,14 @@ def _release_param(trainable_params, return -def _wait_layer(trainable_params, - task_flow, - group, - param2buffer_size, - use_calc_stream, - offload=False): +def _wait_layer( + trainable_params, + task_flow, + group, + param2buffer_size, + use_calc_stream, + offload=False, +): for param in trainable_params: if param.status == "all": @@ -787,25 +900,29 @@ def _wait_layer(trainable_params, param.status = "all" param.use_count += 1 else: - _allgather_buffer(trainable_params, - group, - param2buffer_size=param2buffer_size, - use_calc_stream=True, - task_flow=task_flow, - sync_wait=True, - offload=offload) + _allgather_buffer( + trainable_params, + group, + param2buffer_size=param2buffer_size, + use_calc_stream=True, + task_flow=task_flow, + sync_wait=True, + offload=offload, + ) break return task_flow -def _allgather_buffer(trainable_params, - group, - param2buffer_size, - use_calc_stream, - task_flow, - sync_wait=False, - offload=False, - convert2cpu=False): +def _allgather_buffer( + trainable_params, + group, + param2buffer_size, + use_calc_stream, + task_flow, + sync_wait=False, + offload=False, + convert2cpu=False, +): for param in trainable_params: if param.status == "all": @@ -846,8 +963,9 @@ def _create_params_grad(trainable_params, param2buffer_size, task_flow): if param.name in task_flow.full_grad.keys(): continue assert isinstance(param2buffer_size[param.name], int) - temp_grad = paddle.zeros([param2buffer_size[param.name]], - dtype=param.dtype) + temp_grad = paddle.zeros( + [param2buffer_size[param.name]], dtype=param.dtype + ) temp_tensor = temp_grad._slice(0, param._numel()) temp_tensor.get_tensor()._set_dims(param.shape) param._copy_gradient_from(temp_tensor) @@ -875,13 +993,14 @@ def _UnsliceParam(param): def _VarBaseWrapper(param): varbase = param.fw_storage - tmp_param = EagerParamBase(shape=varbase.shape, - dtype=varbase.dtype, - name="slice@" + param.name) + tmp_param = EagerParamBase( + shape=varbase.shape, dtype=varbase.dtype, name="slice@" + param.name + ) varbase._share_buffer_to(tmp_param) tmp_param.regularizer = param.regularizer tmp_param.optimize_attr['learning_rate'] = param.optimize_attr[ - 'learning_rate'] + 'learning_rate' + ] varbase._clear() return tmp_param @@ -906,14 +1025,17 @@ def _device2cpu(trans_param, convert_dtype=False): def _cpu2device(param): tmp_p = param.fw_storage.cuda(DEV_ID) - if tmp_p.dtype == Type.fp32.value and param2dtype[ - param.name] == Type.fp16.value: + if ( + tmp_p.dtype == Type.fp32.value + and param2dtype[param.name] == Type.fp16.value + ): tmp_p = paddle.cast(tmp_p, Type.fp16.value) return tmp_p def _current_layer_params(layer): - return layer.parameters( - include_sublayers=False) + list(layer.extra_parameters) if hasattr( - layer, "extra_parameters") else layer.parameters( - include_sublayers=False) + return ( + layer.parameters(include_sublayers=False) + list(layer.extra_parameters) + if hasattr(layer, "extra_parameters") + else layer.parameters(include_sublayers=False) + ) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py index 66e9617f00e..72e9ebfcb7d 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py @@ -46,30 +46,40 @@ class InternalStorage: # The flatten tensor size = [size] if isinstance(size, int) else size if convert_cpu: - value = np.zeros( - size, - dtype=np.float16) if Type.fp16.value == dtype else np.zeros( - size, dtype=np.float32) + value = ( + np.zeros(size, dtype=np.float16) + if Type.fp16.value == dtype + else np.zeros(size, dtype=np.float32) + ) self.buffer = core.eager.Tensor(value=value, place=core.CPUPlace()) if dtype == Type.bf16.value: self.buffer = paddle.cast(self.buffer, dtype=paddle.bfloat16) else: self.buffer = paddle.zeros(size, dtype=dtype) - self.dev_id = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + self.dev_id = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) def to(self, device, dtype=None, keep_alignment=True): """ Move the underlying buffer """ - assert self.buffer is not None, "Cannot move a collapsed bucket, please rebuild it" - assert (dtype == Type.fp32.value - or Type.fp16.value), "Conversion type is not supported now" + assert ( + self.buffer is not None + ), "Cannot move a collapsed bucket, please rebuild it" + assert ( + dtype == Type.fp32.value or Type.fp16.value + ), "Conversion type is not supported now" if self._device != device: - tmp_buffer = self.buffer.cuda( - self.dev_id) if device == "gpu" else self.buffer.cpu() + tmp_buffer = ( + self.buffer.cuda(self.dev_id) + if device == "gpu" + else self.buffer.cpu() + ) for param in self._params: param.clear_gradient(False) @@ -107,17 +117,18 @@ class ParamStorage(InternalStorage): Add new parameters to the InternalStorage. Params becomes a view of this InternalStorage buffer. """ - assert all([ - id(param) not in self._param_ids for param in trainable_params - ]), "The same param cannot be checked in twice" + assert all( + [id(param) not in self._param_ids for param in trainable_params] + ), "The same param cannot be checked in twice" assert self.buffer is not None self.param2align = param2align cpu_param_shape = list() for param in trainable_params: - p_shape = self._add_param_as_view(param, param2align[param.name], - convert_gpu) + p_shape = self._add_param_as_view( + param, param2align[param.name], convert_gpu + ) cpu_param_shape.append(p_shape) if convert_gpu: @@ -127,8 +138,9 @@ class ParamStorage(InternalStorage): self._fill = 0 for idx, param in enumerate(trainable_params): - self._convert_buffer(param, cpu_param_shape[idx], - param2align[param.name]) + self._convert_buffer( + param, cpu_param_shape[idx], param2align[param.name] + ) self._params.append(param) self._param_ids.append(id(param)) @@ -138,7 +150,8 @@ class ParamStorage(InternalStorage): assert ( param.dtype == self.buffer.dtype ), "Different types for the InternalStorage and the param, cannot proceed: {} - {}".format( - param.dtype, self.buffer.dtype) + param.dtype, self.buffer.dtype + ) var_end = self._fill + param._numel() offset = var_end + align @@ -199,13 +212,9 @@ class GradStorage(InternalStorage): This is a basic class to simplify the handling of gradient InternalStorages """ - def __init__(self, - size, - dtype, - device, - destination, - parm2align, - convert_cpu=False): + def __init__( + self, size, dtype, device, destination, parm2align, convert_cpu=False + ): if isinstance(size, np.int64): size = size.tolist() super().__init__(size, dtype, device, convert_cpu) @@ -219,21 +228,21 @@ class GradStorage(InternalStorage): self.sent = False def reset_checked_in(self): - """ Reset the counter of the parameter grads which have been checked in - """ + """Reset the counter of the parameter grads which have been checked in""" self.params_checked_in = 0 self.sent = False @property def all_checked_in(self): - """ Judge all the expected gradient check-in happened """ + """Judge all the expected gradient check-in happened""" return len(self._params) == self.params_checked_in def can_add_grad_view(self, param, align): - """ Is there enough InternalStorage to add this parameter gradient, and whether this param have already checked in. - """ - return self._fill + param._numel() + align <= self._max_size and id( - param) not in self._param_ids + """Is there enough InternalStorage to add this parameter gradient, and whether this param have already checked in.""" + return ( + self._fill + param._numel() + align <= self._max_size + and id(param) not in self._param_ids + ) def to(self, device, dtype=None, keep_alignment=True): """ @@ -253,9 +262,9 @@ class GradStorage(InternalStorage): Add a new parameter gradient to the InternalStorage. Param.grad becomes a view of this InternalStorage buffer. """ - assert id( - param - ) not in self._param_ids, "The same gradients cannot be checked in twice" + assert ( + id(param) not in self._param_ids + ), "The same gradients cannot be checked in twice" self._add_grad_as_view(param, align) self._params.append(param) @@ -302,8 +311,9 @@ class GradStorage(InternalStorage): @paddle.autograd.no_grad() def _add_grad_as_view(self, param, align): - assert param._numel( - ) > 0, "Cannot add a gradient to a released InternalStorage, please rebuild" + assert ( + param._numel() > 0 + ), "Cannot add a gradient to a released InternalStorage, please rebuild" assert param.dtype == self.buffer.dtype grad_end = self._fill + param._numel() diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py index 0981b2cab76..4107bd83db9 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py @@ -39,13 +39,13 @@ class Type(Enum): """ Type of trainable parameters """ + fp16 = paddle.float16 bf16 = paddle.bfloat16 fp32 = paddle.float32 class GroupShardedClipGrad: - def __init__(self, clip, device, group): self._clip = clip self._device = device @@ -66,49 +66,64 @@ class GroupShardedClipGrad: merge_grad = g if g.type == core.VarDesc.VarType.SELECTED_ROWS: merge_grad = layers.get_tensor_from_selected_rows( - layers.merge_selected_rows(g)) + layers.merge_selected_rows(g) + ) square = layers.square(merge_grad) sum_square = layers.reduce_sum(square) if p.dtype == paddle.float16: - if p_slice: sum_square_fp16.append(sum_square) - else: unslice_params_fp16.append(sum_square) + if p_slice: + sum_square_fp16.append(sum_square) + else: + unslice_params_fp16.append(sum_square) elif p.dtype == paddle.float32: - if p_slice: sum_square_fp32.append(sum_square) - else: unslice_params_fp32.append(sum_square) + if p_slice: + sum_square_fp32.append(sum_square) + else: + unslice_params_fp32.append(sum_square) # global norm of non-distributed FP16 params_and_grads if len(sum_square_fp16) == 0: - global_norm_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_norm_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32) else: global_norm_fp16 = layers.concat(sum_square_fp16) global_norm_fp16 = layers.reduce_sum(global_norm_fp16) - global_norm_fp16 = paddle.cast(global_norm_fp16, - dtype=paddle.float32) + global_norm_fp16 = paddle.cast( + global_norm_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP16 params_and_grads for unslice parameters if len(unslice_params_fp16) == 0: - global_unslice_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_unslice_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32) else: global_unslice_fp16 = layers.concat(unslice_params_fp16) global_unslice_fp16 = layers.reduce_sum(global_unslice_fp16) - global_unslice_fp16 = paddle.cast(global_unslice_fp16, - dtype=paddle.float32) + global_unslice_fp16 = paddle.cast( + global_unslice_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP32 params_and_grads - global_norm_fp32 = layers.concat( - sum_square_fp32) if len(sum_square_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_norm_fp32 = ( + layers.concat(sum_square_fp32) + if len(sum_square_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_norm_fp32 = layers.reduce_sum(global_norm_fp32) # global norm of non-distributed FP32 params_and_grads for unslice parameters - global_unslice_fp32 = layers.concat(unslice_params_fp32) if len( - unslice_params_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_unslice_fp32 = ( + layers.concat(unslice_params_fp32) + if len(unslice_params_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_unslice_fp32 = layers.reduce_sum(global_unslice_fp32) global_unslice_var = global_unslice_fp16 + global_unslice_fp32 - global_norm_var = global_norm_fp16 + global_norm_fp32 + 1.0 / self._group.nranks * global_unslice_var + global_norm_var = ( + global_norm_fp16 + + global_norm_fp32 + + 1.0 / self._group.nranks * global_unslice_var + ) # add all reduce to get global norm of distributed params_and_grads dev_id = int(self._device.split(":")[1]) @@ -119,14 +134,14 @@ class GroupShardedClipGrad: paddle.distributed.all_reduce(global_norm_var, group=self._group) global_norm_var = layers.sqrt(global_norm_var) - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) - - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var, - y=max_global_norm)) + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) + + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var, y=max_global_norm), + ) clip_var_fp16 = paddle.cast(clip_var, paddle.float16) for p, g in params_grads: @@ -165,7 +180,6 @@ def device_guard(dev_id=0, device="cpu"): @dygraph_only def GroupShardedScaler(scaler): - def unscale_method(self, optimizer): if not self._enable: return @@ -177,14 +191,16 @@ def GroupShardedScaler(scaler): optimizer.update_scaler = True if getattr(optimizer._optim, '_param_groups', None) and isinstance( - optimizer._optim._param_groups[0], dict): + optimizer._optim._param_groups[0], dict + ): for group in optimizer._optim._param_groups: for param in group['params']: if param.grad is not None: param_grads.append(param.grad) if param.grad.dtype in [ - core.VarDesc.VarType.FP16, paddle.float16 + core.VarDesc.VarType.FP16, + paddle.float16, ]: param_grads_fp16.append(param.grad) else: @@ -194,7 +210,8 @@ def GroupShardedScaler(scaler): if param.grad is not None: param_grads.append(param.grad) if param.grad.dtype in [ - core.VarDesc.VarType.FP16, paddle.float16 + core.VarDesc.VarType.FP16, + paddle.float16, ]: param_grads_fp16.append(param.grad) else: @@ -204,27 +221,34 @@ def GroupShardedScaler(scaler): temp_found_inf_fp32 = to_variable(np.array([0]).astype(np.bool_)) device = "cpu" if optimizer.offload else "gpu" - dev_id = 0 if device == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 if device == "cpu" else int(paddle.get_device().split(":")[1]) + ) with device_guard(dev_id, device): if len(param_grads_fp16): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp16, - self._scale, - param_grads_fp16, - temp_found_inf_fp16) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp16, + self._scale, + param_grads_fp16, + temp_found_inf_fp16, + ) if len(param_grads_fp32): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp32, - self._scale, - param_grads_fp32, - temp_found_inf_fp32) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp32, + self._scale, + param_grads_fp32, + temp_found_inf_fp32, + ) self._found_inf = 1 if temp_found_inf_fp16 or temp_found_inf_fp32 else 0 is_found_inf = paddle.to_tensor([self._found_inf], dtype="int32") - paddle.distributed.all_reduce(is_found_inf, - op=paddle.distributed.ReduceOp.MAX, - group=optimizer._group) + paddle.distributed.all_reduce( + is_found_inf, + op=paddle.distributed.ReduceOp.MAX, + group=optimizer._group, + ) self._found_inf = is_found_inf.numpy()[0] scaler._unscale = MethodType(unscale_method, scaler) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage2.py index 5933d11037e..63ce53f01c4 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage2.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage2.py @@ -35,7 +35,9 @@ from paddle.distributed import collective as dist from paddle.distributed.collective import _get_global_group from ...utils.internal_storage import GradStorage -from ...meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 +from ...meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) from .sharding_utils import Taskflow, Type @@ -59,42 +61,55 @@ class ShardingStage2(nn.Layer): # 5. Support the establishment of independent communication groups. def __init__( - self, - layer, - sharding_optimizer, - group=None, - sync_buffers=False, - buffer_max_size=2**23, #8MB - auto_refresh_trainable=True, - device="gpu"): + self, + layer, + sharding_optimizer, + group=None, + sync_buffers=False, + buffer_max_size=2**23, # 8MB + auto_refresh_trainable=True, + device="gpu", + ): super().__init__() # training options self._layer = layer - self._sharding_optimizers = [ - sharding_optimizer - ] if not isinstance(sharding_optimizer, list) else sharding_optimizer + self._sharding_optimizers = ( + [sharding_optimizer] + if not isinstance(sharding_optimizer, list) + else sharding_optimizer + ) assert all( list( - map(lambda opt: isinstance(opt, ShardingOptimizerStage2), - self._sharding_optimizers)) + map( + lambda opt: isinstance(opt, ShardingOptimizerStage2), + self._sharding_optimizers, + ) + ) ), "Please use ShardingOptimizerStage2 optimizer" self._sync_buffers = sync_buffers self._auto_refresh_trainable = auto_refresh_trainable # Communication related attributes - self._group = dist.new_group( - _get_global_group().ranks) if group is None else group + self._group = ( + dist.new_group(_get_global_group().ranks) + if group is None + else group + ) self._world_size_scaling = 1.0 / self._group.nranks - assert self._group.nranks > 1, "Training must be distributed, ranks must be greater than 1" + assert ( + self._group.nranks > 1 + ), "Training must be distributed, ranks must be greater than 1" self._rank = self._group.rank self._global_root_rank = self._group.ranks[ - 0] # picking rank 0 as the reference + 0 + ] # picking rank 0 as the reference self._default_device = device # Global statistical parameters self._all_params = list( - chain(*[optim.local_params for optim in self._sharding_optimizers])) + chain(*[optim.local_params for optim in self._sharding_optimizers]) + ) self._trainable_params = [] self._grad_reduced = [] self._trainable_param2rank = {} @@ -103,11 +118,13 @@ class ShardingStage2(nn.Layer): self._param_grads = [] # Set grad storage size & Display param sizes and model sizes - model_size = sum([np.prod(p.shape) - for p in self._layer.parameters()]).item() + model_size = sum( + [np.prod(p.shape) for p in self._layer.parameters()] + ).item() assert buffer_max_size >= 0, "buffer_max_size must be GE than 0." - self._buffer_max_size = self._rank_buffer_size(buffer_max_size, - model_size) + self._buffer_max_size = self._rank_buffer_size( + buffer_max_size, model_size + ) self._use_grad_storage = buffer_max_size > 0 self._grad_storages = {} # {dtype: {rank: GradStorage}} self._has_grad_storage = [] @@ -116,11 +133,12 @@ class ShardingStage2(nn.Layer): # Offload # TODO(haohongxiang): Now it's not be supported for multi-optimizers using Offload strategy self._offload_optims = list( - filter(lambda optim: optim.offload, self._sharding_optimizers)) + filter(lambda optim: optim.offload, self._sharding_optimizers) + ) if len(self._offload_optims) > 0: - assert len( - self._sharding_optimizers - ) == 1, "Only support offload strategy for single optimizer" + assert ( + len(self._sharding_optimizers) == 1 + ), "Only support offload strategy for single optimizer" self._offload = self._sharding_optimizers[0].offload self._offload_device = "cpu" @@ -163,16 +181,19 @@ class ShardingStage2(nn.Layer): return fw def set_state_dict(self, state_dict, use_structured_name=True): - self._layer.set_state_dict(state_dict, - use_structured_name=use_structured_name) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): - return self._layer.state_dict(destination=None, - include_sublayers=True, - structured_name_prefix="") + self._layer.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): + return self._layer.state_dict( + destination=None, include_sublayers=True, structured_name_prefix="" + ) def _clear_gradients(self): """ @@ -180,8 +201,10 @@ class ShardingStage2(nn.Layer): """ # Release grad storages for dtype in self._grad_storages.keys(): - if not self._offload and self._rank in self._grad_storages[ - dtype].keys(): + if ( + not self._offload + and self._rank in self._grad_storages[dtype].keys() + ): self._grad_storages[dtype][self._rank].buffer.zero_() # Release grads of params @@ -199,10 +222,13 @@ class ShardingStage2(nn.Layer): """ # Scale grad storages for dtype in self._grad_storages.keys(): - if not self._offload and self._rank in self._grad_storages[ - dtype].keys(): + if ( + not self._offload + and self._rank in self._grad_storages[dtype].keys() + ): self._grad_storages[dtype][self._rank].buffer.scale_( - scale=self._world_size_scaling) + scale=self._world_size_scaling + ) # Scale grads of params for param in self._trainable_params: @@ -213,7 +239,8 @@ class ShardingStage2(nn.Layer): # Scale grads of master params with offload strategy if self._offload: self._sharding_optimizers[0]._offload_scale_grad( - self._world_size_scaling) + self._world_size_scaling + ) def _init_internal_storage(self, needs_fresh): """ @@ -232,7 +259,9 @@ class ShardingStage2(nn.Layer): Synchronously or asynchronously convert the data type of the layer, the device is not supported now. """ assert isinstance(device, str), "Device must be type str" - assert device == self._default_device, "New devices are not supported, because of the optimizer state is not sync" + assert ( + device == self._default_device + ), "New devices are not supported, because of the optimizer state is not sync" self._layer.to(device=device, dtype=dtype, blocking=blocking) @@ -240,14 +269,15 @@ class ShardingStage2(nn.Layer): self._fresh_trainable() def _fresh_trainable(self): - """ Whether to update training parameters. """ + """Whether to update training parameters.""" # Make sure that this is not done while gradients are waiting to be reduced (if no_sync context for instance) if reduce(lambda x, y: x or y, self._grad_reduced, False): logging.warning("Grads waiting to be reduced.") self._trainable_params = list( - filter(lambda x: x.trainable, self._all_params)) + filter(lambda x: x.trainable, self._all_params) + ) self._trainable_params.sort(key=lambda x: np.prod(x.shape)) self._trainable_param2rank = {} @@ -257,14 +287,19 @@ class ShardingStage2(nn.Layer): optim.update_opt_status() # Get the parameters split by the optimizer according to rank - for per_rank_params in optim.dtype_rank_params.values( + for ( + per_rank_params + ) in ( + optim.dtype_rank_params.values() ): # all the params from all ranks for params in per_rank_params: for param in filter(lambda x: x.trainable, params): self._trainable_param2rank[ - param.name] = optim.param2rank[param.name] + param.name + ] = optim.param2rank[param.name] self._trainable_param2align[ - param.name] = optim._param2align[param.name] + param.name + ] = optim._param2align[param.name] self._setup_use_grad_storage() @@ -278,10 +313,9 @@ class ShardingStage2(nn.Layer): """ for buffer in self._layer.buffers(include_sublayers=True): - dist.broadcast(buffer, - self._global_root_rank, - self._group, - sync_op=True) + dist.broadcast( + buffer, self._global_root_rank, self._group, sync_op=True + ) # Multi stream operation will be supported later dist.wait(tensor=buffer, group=self._group, use_calc_stream=True) @@ -315,7 +349,9 @@ class ShardingStage2(nn.Layer): def reduce(*_): # Skip gradient reduction, do not change status information if self._grad_reduced[index]: - assert param.grad is not None, "Parameter gradient cannot be None" + assert ( + param.grad is not None + ), "Parameter gradient cannot be None" # Change reduce information self._grad_reduced[index] = False @@ -327,22 +363,29 @@ class ShardingStage2(nn.Layer): elif self._offload: self._sharding_optimizers[0]._offload_acc_grad( param.name, - param.grad.cast(dtype=Type.fp32.value).cpu()) + param.grad.cast(dtype=Type.fp32.value).cpu(), + ) param.clear_gradient(False) # Synchronize the reduce parameter gradient self._tasks_flow.append( - Taskflow(task=dist.reduce( - tensor=param.grad, - dst=self._group.ranks[dst_rank], - group=self._group, - sync_op=True), - callback=cleanup)) + Taskflow( + task=dist.reduce( + tensor=param.grad, + dst=self._group.ranks[dst_rank], + group=self._group, + sync_op=True, + ), + callback=cleanup, + ) + ) # Multi stream operation will be supported later - dist.wait(tensor=param.grad, - group=self._group, - use_calc_stream=True) + dist.wait( + tensor=param.grad, + group=self._group, + use_calc_stream=True, + ) # Clear the task flow and trigger callback to clear the redundant gradient self._clear_task_flow() @@ -353,7 +396,9 @@ class ShardingStage2(nn.Layer): def reduce(*_): # Skip gradient reduction, do not change status information if self._grad_reduced[index]: - assert param.grad is not None, "Parameter gradient cannot be None" + assert ( + param.grad is not None + ), "Parameter gradient cannot be None" # Change reduce information self._grad_reduced[index] = False @@ -370,35 +415,43 @@ class ShardingStage2(nn.Layer): p.clear_gradient(False) p._gradient_set_empty(False) - grad_storage.buffer.value().get_tensor()._clear( - ) + grad_storage.buffer.value().get_tensor()._clear() elif self._offload: grad_storage.to(device=self._offload_device) for p in grad_storage._params: self._sharding_optimizers[ - 0]._offload_acc_grad( - p.name, - p.grad.cast(dtype=Type.fp32.value)) + 0 + ]._offload_acc_grad( + p.name, + p.grad.cast(dtype=Type.fp32.value), + ) p.clear_gradient(False) p._gradient_set_empty(False) grad_storage._device = self._default_device - grad_storage.buffer.value().get_tensor()._clear( - ) + grad_storage.buffer.value().get_tensor()._clear() # Reduce the bucket grad_storage.sent = True self._tasks_flow.append( - Taskflow(task=dist.reduce( - tensor=grad_storage.buffer, - dst=self._group.ranks[grad_storage.destination], - group=self._group, - sync_op=True), - callback=cleanup)) + Taskflow( + task=dist.reduce( + tensor=grad_storage.buffer, + dst=self._group.ranks[ + grad_storage.destination + ], + group=self._group, + sync_op=True, + ), + callback=cleanup, + ) + ) # Multi stream operation will be supported later - dist.wait(tensor=grad_storage.buffer, - group=self._group, - use_calc_stream=True) + dist.wait( + tensor=grad_storage.buffer, + group=self._group, + use_calc_stream=True, + ) # Clear the task flow and trigger callback to clear the redundant gradient self._clear_task_flow() @@ -424,7 +477,8 @@ class ShardingStage2(nn.Layer): reduce_function = self._get_reduce_fn(index, param, dst_rank) self._bw_hooks.append( - param._register_backward_hook(reduce_function)) + param._register_backward_hook(reduce_function) + ) def _setup_use_grad_storage(self): """ @@ -447,27 +501,36 @@ class ShardingStage2(nn.Layer): dtype=param.dtype, device=self._default_device, destination=dst_rank, - parm2align=self._trainable_param2align) + parm2align=self._trainable_param2align, + ) # Criteria to decide whether this parameter is to be put in GradStorage if self._grad_storages[param.dtype][dst_rank].can_add_grad_view( - param, self._trainable_param2align[param.name]): + param, self._trainable_param2align[param.name] + ): self._grad_storages[param.dtype][dst_rank].add_grad( - param, self._trainable_param2align[param.name]) + param, self._trainable_param2align[param.name] + ) self._has_grad_storage[index] = True else: self._param_grads.append(param.name) print( - "Can not add param: {}, param's shape: {}, param align: {}, grad_storages fill: {}, " - .format(param.name, param.shape, - self._trainable_param2align[param.name], - self._grad_storages[param.dtype][dst_rank]._fill)) + "Can not add param: {}, param's shape: {}, param align: {}, grad_storages fill: {}, ".format( + param.name, + param.shape, + self._trainable_param2align[param.name], + self._grad_storages[param.dtype][dst_rank]._fill, + ) + ) self._grad_storage_list = list( - chain(*[ - self._grad_storages[dtype].values() - for dtype in self._grad_storages.keys() - ])) + chain( + *[ + self._grad_storages[dtype].values() + for dtype in self._grad_storages.keys() + ] + ) + ) def _clear_task_flow(self): """Try to consume the previous tasks.""" @@ -518,15 +581,19 @@ class ShardingStage2(nn.Layer): if Type.fp16.value in rank_buffer_size.keys(): # FP16 GradStorage and model size print( - "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.fp16.value] / 2**19, - model_size / 2**19)) + "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.fp16.value] / 2**19, + model_size / 2**19, + ) + ) if Type.fp32.value in rank_buffer_size.keys(): # FP32 GradStorage and model size print( - "====== FP32 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.fp32.value] / 2**18, - model_size / 2**18)) + "====== FP32 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.fp32.value] / 2**18, + model_size / 2**18, + ) + ) return rank_buffer_size def _redefine_opt_step(self): diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage3.py b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage3.py index 02e701e8990..deae0cddd29 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage3.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage3.py @@ -59,16 +59,18 @@ class ShardingStage3(nn.Layer): # 3. Support offload function. # 4. Support the establishment of independent communication groups. - def __init__(self, - layer, - optimizer, - group=None, - sync_buffers=False, - device="gpu", - segment_size=2**15, - pertrain_sync_models=True, - offload=False, - sync_comm=False): + def __init__( + self, + layer, + optimizer, + group=None, + sync_buffers=False, + device="gpu", + segment_size=2**15, + pertrain_sync_models=True, + offload=False, + sync_comm=False, + ): super().__init__() # Default configs @@ -83,28 +85,41 @@ class ShardingStage3(nn.Layer): self._segment_size = segment_size global DEV - DEV = "cpu" if paddle.get_device() == "cpu" else paddle.get_device( - ).split(":")[0] + DEV = ( + "cpu" + if paddle.get_device() == "cpu" + else paddle.get_device().split(":")[0] + ) global DEV_ID - DEV_ID = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + DEV_ID = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) global param2dtype param2dtype = dict() # Communication group establishment - self._group = dist.new_group( - _get_global_group().ranks) if group is None else group + self._group = ( + dist.new_group(_get_global_group().ranks) + if group is None + else group + ) self._world_size_scaling = 1.0 / self._group.nranks - assert self._group.nranks > 1, "Training must be distributed, ranks must be greater than 1." + assert ( + self._group.nranks > 1 + ), "Training must be distributed, ranks must be greater than 1." self._rank = self._group.rank self._global_root_rank = self._group.ranks[ - 0] # picking rank 0 as the reference + 0 + ] # picking rank 0 as the reference self._global_ranks = self._group.ranks # Parameter segmentation for global ranks # After flatten -> self._param2buffer_size, self._param2buffer, self._trainable_params self._param2buffer_size = dict() # {param.name: size} - self._param2buffer = dict( + self._param2buffer = ( + dict() ) # {param.name: [(start0, end0),(start1, end1), ...]} self._trainable_params = dict() # {id(layer): [trainable_params]} self._unslice_params = set() # param's numel <= segment_size @@ -112,9 +127,11 @@ class ShardingStage3(nn.Layer): self._grad_storages = dict() # {param.dtype: GradStorage} assert not isinstance( - optimizer, list), "Multiple optimizers are not supported now." - self._optim = _OptimizerWrapper(optimizer, self._offload, self._group, - self._update_params_slice) + optimizer, list + ), "Multiple optimizers are not supported now." + self._optim = _OptimizerWrapper( + optimizer, self._offload, self._group, self._update_params_slice + ) self._ori_parameter_list = self._optim._parameter_list self._ori_param_groups = self._optim._param_groups @@ -123,9 +140,9 @@ class ShardingStage3(nn.Layer): logging.warning( "While using ClipGradByGlobalNorm in ShardingStage3, the grad clip of original optimizer will be changed." ) - self._optim._grad_clip = ShardingClipGrad(self._optim._grad_clip, - paddle.get_device(), - self._group) + self._optim._grad_clip = ShardingClipGrad( + self._optim._grad_clip, paddle.get_device(), self._group + ) # Synchronous all ranks models if pertrain_sync_models: @@ -161,10 +178,9 @@ class ShardingStage3(nn.Layer): """ for p in self._layer.parameters(): - dist.broadcast(p, - src=self._global_root_rank, - group=self._group, - sync_op=True) + dist.broadcast( + p, src=self._global_root_rank, group=self._group, sync_op=True + ) # Multi stream operation will be supported later dist.wait(tensor=p, group=self._group, use_calc_stream=True) @@ -174,12 +190,15 @@ class ShardingStage3(nn.Layer): current_layer_params = self._layer.parameters(include_sublayers=True) # 1.Handle param's slice trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) for param in trainable_params: - assert hasattr(param, "fw_storage" - ), "Find {} don't have fw_storage attribute.".format( - param.name) + assert hasattr( + param, "fw_storage" + ), "Find {} don't have fw_storage attribute.".format(param.name) param.fw_storage.clear_gradient(False) param.fw_storage._gradient_set_empty(False) @@ -195,8 +214,10 @@ class ShardingStage3(nn.Layer): param._gradient_set_empty(False) tmp_var = param.cuda(DEV_ID) - if tmp_var.dtype == Type.fp32.value and param2dtype[ - param.name] == Type.fp16.value: + if ( + tmp_var.dtype == Type.fp32.value + and param2dtype[param.name] == Type.fp16.value + ): tmp_var = paddle.cast(tmp_var, Type.fp16.value) tmp_var._share_buffer_to(param) tmp_var._clear() @@ -211,9 +232,11 @@ class ShardingStage3(nn.Layer): if not isinstance(self._optim._param_groups[0], dict): slice_params = [param.fw_storage for param in update_list] self._optim._parameter_list = slice_params + list( - self._unslice_params) + self._unslice_params + ) self._optim._param_groups = slice_params + list( - self._unslice_params) + self._unslice_params + ) else: for param_group in self._optim._param_groups: p_group = [] @@ -239,16 +262,19 @@ class ShardingStage3(nn.Layer): return fw def set_state_dict(self, state_dict, use_structured_name=True): - self._layer.set_state_dict(state_dict, - use_structured_name=use_structured_name) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): - return self._layer.state_dict(destination=None, - include_sublayers=True, - structured_name_prefix="") + self._layer.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): + return self._layer.state_dict( + destination=None, include_sublayers=True, structured_name_prefix="" + ) def _handle_unslice_params(self): buffer_size = dict() @@ -258,7 +284,8 @@ class ShardingStage3(nn.Layer): # Updata optimizer master weights if param.dtype == Type.fp16.value and not self._offload: self._optim._master_weights[param.name] = paddle.cast( - param, Type.fp32.value) + param, Type.fp32.value + ) if self._offload: param.master_weight = paddle.cast(param, Type.fp32.value).cpu() param2dtype[param.name] = param.dtype @@ -274,9 +301,11 @@ class ShardingStage3(nn.Layer): dtype=param.dtype, device=self._default_device, destination=self._rank, - parm2align=self._unslice_params2align) + parm2align=self._unslice_params2align, + ) self._grad_storages[param.dtype].add_grad( - param, self._unslice_params2align[param.name]) + param, self._unslice_params2align[param.name] + ) def _segment_rank_params(self, layer, name="last_layer"): """ @@ -316,8 +345,11 @@ class ShardingStage3(nn.Layer): align_ = self._param2align(param) offset = align_ + param._numel() - buffer_size = offset if offset % self._group.nranks == 0 else offset + self._group.nranks - ( - offset % self._group.nranks) + buffer_size = ( + offset + if offset % self._group.nranks == 0 + else offset + self._group.nranks - (offset % self._group.nranks) + ) self._param2buffer_size[param.name] = buffer_size # 2.Combination param buffer @@ -326,7 +358,8 @@ class ShardingStage3(nn.Layer): for rank_ in range(self._group.nranks): self._param2buffer[param.name].append( - (rank_ * pre_buffer, (rank_ + 1) * pre_buffer)) + (rank_ * pre_buffer, (rank_ + 1) * pre_buffer) + ) # Record param's dtype param2dtype[param.name] = param.dtype @@ -339,10 +372,11 @@ class ShardingStage3(nn.Layer): This is a function to simplify the handling of parameter InternalStorages. """ assert isinstance(buffer_size, int) - value = np.zeros( - buffer_size, - dtype=np.float16) if Type.fp16.value == param.dtype else np.zeros( - buffer_size, dtype=np.float32) + value = ( + np.zeros(buffer_size, dtype=np.float16) + if Type.fp16.value == param.dtype + else np.zeros(buffer_size, dtype=np.float32) + ) buffer = core.VarBase(value=value, place=core.CPUPlace()) param_shape = param.shape @@ -353,30 +387,37 @@ class ShardingStage3(nn.Layer): start, end = self._param2buffer[param.name][self._rank] # Copy the current param value - tmp_var = core.VarBase(tensor=buffer._slice(0, param._numel()), - place=core.CPUPlace()) + tmp_var = core.VarBase( + tensor=buffer._slice(0, param._numel()), place=core.CPUPlace() + ) param_cpu = param.cpu() - tmp_var.value().get_tensor().set(param_cpu.value().get_tensor(), - core.CPUPlace()) + tmp_var.value().get_tensor().set( + param_cpu.value().get_tensor(), core.CPUPlace() + ) param.value().get_tensor()._set_dims(param_shape) # Current rank param_storage if self._offload: - param.fw_storage = core.VarBase(buffer._slice(start, end), - core.CPUPlace(), - "slice@" + param.name) + param.fw_storage = core.VarBase( + buffer._slice(start, end), + core.CPUPlace(), + "slice@" + param.name, + ) with device_guard(device="cpu"): - param.master_weight = paddle.cast(param.fw_storage, - Type.fp32.value) + param.master_weight = paddle.cast( + param.fw_storage, Type.fp32.value + ) else: - param.fw_storage = core.VarBase(buffer._slice(start, end), - "slice@" + param.name) + param.fw_storage = core.VarBase( + buffer._slice(start, end), "slice@" + param.name + ) param.status = "part" # Updata optimizer master weights if param.dtype == Type.fp16.value and not self._offload: self._optim._master_weights[param.fw_storage.name] = paddle.cast( - param.fw_storage, Type.fp32.value) + param.fw_storage, Type.fp32.value + ) param._clear() def _register_forward_hooks(self, layer): @@ -398,20 +439,33 @@ class ShardingStage3(nn.Layer): self._register_forward_hooks(sub_layer) def _register_forward_all_hooks(self, sub_layer, task_flow): - def _forward_pre_hook(layer, inputs): - return ForwardPreHooks(layer, self._order_tracer, - self._trainable_params, self._param2buffer, - self._rank, self._group, self._sync_comm, - self._offload, task_flow) + return ForwardPreHooks( + layer, + self._order_tracer, + self._trainable_params, + self._param2buffer, + self._rank, + self._group, + self._sync_comm, + self._offload, + task_flow, + ) def _forward_post_hook(layer, inputs, outputs): - return ForwardPostHooks.apply(outputs, layer, self._order_tracer, - self._trainable_params, - self._param2buffer, - self._param2buffer_size, self._rank, - self._group, self._sync_comm, - self._offload, task_flow) + return ForwardPostHooks.apply( + outputs, + layer, + self._order_tracer, + self._trainable_params, + self._param2buffer, + self._param2buffer_size, + self._rank, + self._group, + self._sync_comm, + self._offload, + task_flow, + ) # register previous forward hooks sub_layer.register_forward_pre_hook(_forward_pre_hook) @@ -426,10 +480,9 @@ class ShardingStage3(nn.Layer): """ for buffer in self._layer.buffers(include_sublayers=True): - dist.broadcast(buffer, - self._global_root_rank, - self._group, - sync_op=True) + dist.broadcast( + buffer, self._global_root_rank, self._group, sync_op=True + ) # Multi stream operation will be supported later dist.wait(tensor=buffer, group=self._group, use_calc_stream=True) @@ -448,14 +501,16 @@ class ShardingStage3(nn.Layer): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) # 1.Handle param's slice for param in trainable_params: assert hasattr( - param, - "fw_storage"), "Find {} don't have fw_storage attribute".format( - param.name) + param, "fw_storage" + ), "Find {} don't have fw_storage attribute".format(param.name) # Gradient average if self._offload: with device_guard(device="cpu"): @@ -470,12 +525,14 @@ class ShardingStage3(nn.Layer): # 2.Handle unslice param for grad_storage in self._grad_storages.values(): grad_storage.buffer.scale_(scale=self._world_size_scaling) - dist.all_reduce(tensor=grad_storage.buffer, - group=self._group, - sync_op=True) - dist.wait(tensor=grad_storage.buffer, - group=self._group, - use_calc_stream=True) + dist.all_reduce( + tensor=grad_storage.buffer, group=self._group, sync_op=True + ) + dist.wait( + tensor=grad_storage.buffer, + group=self._group, + use_calc_stream=True, + ) if self._offload: for param in list(self._unslice_params): @@ -500,15 +557,20 @@ class ShardingStage3(nn.Layer): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) - t_flow = _allgather_buffer(trainable_params, - self._group, - use_calc_stream=True, - task_flow=TaskFlow(), - sync_wait=True, - offload=self._offload, - convert2cpu=convert2cpu) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) + t_flow = _allgather_buffer( + trainable_params, + self._group, + use_calc_stream=True, + task_flow=TaskFlow(), + sync_wait=True, + offload=self._offload, + convert2cpu=convert2cpu, + ) if convert2cpu: for param in trainable_params: t_flow.full_param[param.name]._share_buffer_to(param) @@ -519,49 +581,60 @@ class ShardingStage3(nn.Layer): def _register_backward_hooks(self): current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) for param in trainable_params: allreduce_function = self._get_allreduce_fn(param) param._register_backward_hook(allreduce_function) def _get_allreduce_fn(self, param): - @paddle.autograd.no_grad() def allreduce_(*_): if param.name in self._task_flow.full_grad.keys(): full_grad = self._task_flow.full_grad[param.name] # Only support sync allreduce current rank's layer now - dist.all_reduce(tensor=full_grad, - group=self._group, - sync_op=True) - dist.wait(tensor=full_grad, - group=self._group, - use_calc_stream=True) + dist.all_reduce( + tensor=full_grad, group=self._group, sync_op=True + ) + dist.wait( + tensor=full_grad, group=self._group, use_calc_stream=True + ) start, end = self._param2buffer[param.name][self._rank] if param.bw_storage is None: - param.bw_storage = core.VarBase(full_grad._slice( - start, end)).detach().clone() + param.bw_storage = ( + core.VarBase(full_grad._slice(start, end)) + .detach() + .clone() + ) if self._offload: param.bw_storage = _device2cpu(param.bw_storage, True) else: if self._offload: cpu_grad = _device2cpu( - core.VarBase(full_grad._slice( - start, end)).detach().clone(), True) + core.VarBase(full_grad._slice(start, end)) + .detach() + .clone(), + True, + ) with device_guard(device="cpu"): param.bw_storage = paddle.add( - param.bw_storage, cpu_grad) + param.bw_storage, cpu_grad + ) else: # param.bw_storage.add_( # core.VarBase(full_grad._slice(start, end)) # .detach().clone()) param.bw_storage = paddle.add( param.bw_storage, - core.VarBase(full_grad._slice( - start, end)).detach().clone()) + core.VarBase(full_grad._slice(start, end)) + .detach() + .clone(), + ) param.clear_gradient(False) param._gradient_set_empty(False) tmp_var = self._task_flow.full_grad.pop(param.name) @@ -572,10 +645,16 @@ class ShardingStage3(nn.Layer): param.use_count = 0 param._clear() start, end = self._param2buffer[param.name][self._rank] - param.fw_storage = core.VarBase( - self._task_flow.full_param[param.name]._slice( - start, end), - param.name + "@slice").detach().clone() + param.fw_storage = ( + core.VarBase( + self._task_flow.full_param[param.name]._slice( + start, end + ), + param.name + "@slice", + ) + .detach() + .clone() + ) param.status = "part" tmp_var = self._task_flow.full_param.pop(param.name) tmp_var._clear() @@ -590,8 +669,9 @@ class ShardingStage3(nn.Layer): # CUDA alignment 256 bytes size = param._numel() * align[param.dtype] remaining = size % alignment[self._default_device] - ali = 0 if remaining == 0 else alignment[ - self._default_device] - remaining + ali = ( + 0 if remaining == 0 else alignment[self._default_device] - remaining + ) align_ = ali // align[param.dtype] return align_ @@ -625,8 +705,17 @@ class ShardingStage3(nn.Layer): self._optim.clear_grad = MethodType(_opt_clear, self._optim) -def ForwardPreHooks(layer, order_tracer, trainable_params, param2buffer, rank, - group, sync_comm, offload, task_flow): +def ForwardPreHooks( + layer, + order_tracer, + trainable_params, + param2buffer, + rank, + group, + sync_comm, + offload, + task_flow, +): # Record layer's id layer_id = id(layer) @@ -641,34 +730,49 @@ def ForwardPreHooks(layer, order_tracer, trainable_params, param2buffer, rank, # Whether to use calc stream task_flow.use_calc[layer_id] = use_calc # wait current layer params - _wait_layer(trainable_params[layer_id], task_flow, group, use_calc, - offload) + _wait_layer( + trainable_params[layer_id], task_flow, group, use_calc, offload + ) - if layer_id == order_tracer["layer"][-1]: return + if layer_id == order_tracer["layer"][-1]: + return order_ = order_tracer[layer_id] layer_id = order_tracer["layer"][order_ + 1] - _allgather_buffer(trainable_params[layer_id], - group, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_id], + group, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) return class ForwardPostHooks(PyLayer): - @staticmethod - def forward(ctx, inputs, layer, order_tracer, trainable_params, - param2buffer, param2buffer_size, rank, group, sync_comm, - offload, task_flow): + def forward( + ctx, + inputs, + layer, + order_tracer, + trainable_params, + param2buffer, + param2buffer_size, + rank, + group, + sync_comm, + offload, + task_flow, + ): layer_id = id(layer) # release current layer full params - _release_param(trainable_params[layer_id], param2buffer, rank, - task_flow, offload) + _release_param( + trainable_params[layer_id], param2buffer, rank, task_flow, offload + ) if layer_id not in order_tracer.keys(): order_ = order_tracer["order"] @@ -676,7 +780,7 @@ class ForwardPostHooks(PyLayer): order_tracer["order"] += 1 order_tracer["layer"].append(layer_id) - #Record bw info + # Record bw info ctx.order_tracer = order_tracer ctx.task_flow = task_flow ctx.group = group @@ -705,30 +809,36 @@ class ForwardPostHooks(PyLayer): # Allgather params synchronization if sync_comm: use_calc, sync_wait = True, True - _allgather_buffer(trainable_params[layer_id], - group, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_id], + group, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) else: - _wait_layer(trainable_params[layer_id], task_flow, group, use_calc, - offload) + _wait_layer( + trainable_params[layer_id], task_flow, group, use_calc, offload + ) # Create params's grad - _create_params_grad(trainable_params[layer_id], param2buffer_size, - task_flow) + _create_params_grad( + trainable_params[layer_id], param2buffer_size, task_flow + ) # Whether to use calc stream task_flow.use_calc[layer_id] = use_calc if layer_id != order_tracer["layer"][0] and not sync_comm: layer_next_id = order_tracer["layer"][order_tracer[layer_id] - 1] - _allgather_buffer(trainable_params[layer_next_id], - group, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_next_id], + group, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) return args @@ -738,22 +848,22 @@ class TaskFlow: Task flows, one way linked list for task acquisition. """ - def __init__(self, - full_param=dict(), - full_grad=dict(), - use_calc=dict(), - callback=None): + def __init__( + self, + full_param=dict(), + full_grad=dict(), + use_calc=dict(), + callback=None, + ): self.full_param = full_param self.full_grad = full_grad self.use_calc = use_calc self.callback = callback -def _release_param(trainable_params, - param2buffer, - rank, - task_flow, - offload=False): +def _release_param( + trainable_params, param2buffer, rank, task_flow, offload=False +): for param in trainable_params: # async communicate share weight not clear param.use_count -= 1 @@ -762,9 +872,14 @@ def _release_param(trainable_params, if param.name in task_flow.full_param.keys(): start, end = param2buffer[param.name][rank] with paddle.amp.auto_cast(enable=False): - param.fw_storage = core.VarBase( - task_flow.full_param[param.name]._slice(start, end), - param.name + "@slice").detach().clone() + param.fw_storage = ( + core.VarBase( + task_flow.full_param[param.name]._slice(start, end), + param.name + "@slice", + ) + .detach() + .clone() + ) param.status = "part" tmp_var = task_flow.full_param.pop(param.name) tmp_var._clear() @@ -774,11 +889,9 @@ def _release_param(trainable_params, return -def _wait_layer(trainable_params, - task_flow, - group, - use_calc_stream, - offload=False): +def _wait_layer( + trainable_params, task_flow, group, use_calc_stream, offload=False +): paddle.device.cuda.synchronize() for param in trainable_params: if param.status == "all": @@ -786,30 +899,35 @@ def _wait_layer(trainable_params, continue if param.name in task_flow.full_param.keys(): full_param = task_flow.full_param[param.name] - core.VarBase(full_param._slice( - 0, param._numel()))._share_buffer_to(param) + core.VarBase(full_param._slice(0, param._numel()))._share_buffer_to( + param + ) param.fw_storage._clear() param.fw_storage = None param.status = "all" param.use_count += 1 else: - _allgather_buffer(trainable_params, - group, - use_calc_stream=True, - task_flow=task_flow, - sync_wait=True, - offload=offload) + _allgather_buffer( + trainable_params, + group, + use_calc_stream=True, + task_flow=task_flow, + sync_wait=True, + offload=offload, + ) break return task_flow -def _allgather_buffer(trainable_params, - group, - use_calc_stream, - task_flow, - sync_wait=False, - offload=False, - convert2cpu=False): +def _allgather_buffer( + trainable_params, + group, + use_calc_stream, + task_flow, + sync_wait=False, + offload=False, + convert2cpu=False, +): for param in trainable_params: if param.status == "all": @@ -820,18 +938,21 @@ def _allgather_buffer(trainable_params, param.fw_storage = _cpu2device(param) with paddle.amp.auto_cast(enable=False): - full_param = _all_gather(param.fw_storage, - group, - use_calc_stream=use_calc_stream) + full_param = _all_gather( + param.fw_storage, group, use_calc_stream=use_calc_stream + ) # Allgather current layer in the 1st step synchronously if sync_wait: with paddle.amp.auto_cast(enable=False): - dist.wait(tensor=full_param, - group=group, - use_calc_stream=use_calc_stream) - core.VarBase(full_param._slice( - 0, param._numel()))._share_buffer_to(param) + dist.wait( + tensor=full_param, + group=group, + use_calc_stream=use_calc_stream, + ) + core.VarBase(full_param._slice(0, param._numel()))._share_buffer_to( + param + ) param.fw_storage._clear() param.fw_storage = None param.status = "all" @@ -855,10 +976,12 @@ def _create_params_grad(trainable_params, param2buffer_size, task_flow): if param.name in task_flow.full_grad.keys(): continue assert isinstance(param2buffer_size[param.name], int) - temp_grad = paddle.zeros([param2buffer_size[param.name]], - dtype=param.dtype) + temp_grad = paddle.zeros( + [param2buffer_size[param.name]], dtype=param.dtype + ) param._copy_gradient_from( - core.VarBase(temp_grad._slice(0, param._numel()))) + core.VarBase(temp_grad._slice(0, param._numel())) + ) task_flow.full_grad[param.name] = temp_grad return task_flow @@ -882,13 +1005,14 @@ def _UnsliceParam(param): def _VarBaseWrapper(param): varbase = param.fw_storage - tmp_param = ParamBase(shape=varbase.shape, - dtype=varbase.dtype, - name="slice@" + param.name) + tmp_param = ParamBase( + shape=varbase.shape, dtype=varbase.dtype, name="slice@" + param.name + ) varbase._share_buffer_to(tmp_param) tmp_param.regularizer = param.regularizer tmp_param.optimize_attr['learning_rate'] = param.optimize_attr[ - 'learning_rate'] + 'learning_rate' + ] varbase._clear() return tmp_param @@ -913,14 +1037,17 @@ def _device2cpu(trans_param, convert_dtype=False): def _cpu2device(param): tmp_p = param.fw_storage.cuda(DEV_ID) - if tmp_p.dtype == Type.fp32.value and param2dtype[ - param.name] == Type.fp16.value: + if ( + tmp_p.dtype == Type.fp32.value + and param2dtype[param.name] == Type.fp16.value + ): tmp_p = paddle.cast(tmp_p, Type.fp16.value) return tmp_p def _current_layer_params(layer): - return layer.parameters( - include_sublayers=False) + list(layer.extra_parameters) if hasattr( - layer, "extra_parameters") else layer.parameters( - include_sublayers=False) + return ( + layer.parameters(include_sublayers=False) + list(layer.extra_parameters) + if hasattr(layer, "extra_parameters") + else layer.parameters(include_sublayers=False) + ) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py index 0e7725e3e21..07cf159c3e6 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py @@ -40,13 +40,13 @@ class Type(Enum): """ Type of trainable parameters """ + fp16 = paddle.float16 bf16 = paddle.bfloat16 fp32 = paddle.float32 class ShardingClipGrad: - def __init__(self, clip, device, group): self._clip = clip self._device = device @@ -67,49 +67,64 @@ class ShardingClipGrad: merge_grad = g if g.type == core.VarDesc.VarType.SELECTED_ROWS: merge_grad = layers.get_tensor_from_selected_rows( - layers.merge_selected_rows(g)) + layers.merge_selected_rows(g) + ) square = layers.square(merge_grad) sum_square = layers.reduce_sum(square) if p.dtype == paddle.float16: - if p_slice: sum_square_fp16.append(sum_square) - else: unslice_params_fp16.append(sum_square) + if p_slice: + sum_square_fp16.append(sum_square) + else: + unslice_params_fp16.append(sum_square) elif p.dtype == paddle.float32: - if p_slice: sum_square_fp32.append(sum_square) - else: unslice_params_fp32.append(sum_square) + if p_slice: + sum_square_fp32.append(sum_square) + else: + unslice_params_fp32.append(sum_square) # global norm of non-distributed FP16 params_and_grads if len(sum_square_fp16) == 0: - global_norm_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_norm_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32) else: global_norm_fp16 = layers.concat(sum_square_fp16) global_norm_fp16 = layers.reduce_sum(global_norm_fp16) - global_norm_fp16 = paddle.cast(global_norm_fp16, - dtype=paddle.float32) + global_norm_fp16 = paddle.cast( + global_norm_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP16 params_and_grads for unslice parameter if len(unslice_params_fp16) == 0: - global_unslice_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_unslice_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32) else: global_unslice_fp16 = layers.concat(unslice_params_fp16) global_unslice_fp16 = layers.reduce_sum(global_unslice_fp16) - global_unslice_fp16 = paddle.cast(global_unslice_fp16, - dtype=paddle.float32) + global_unslice_fp16 = paddle.cast( + global_unslice_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP32 params_and_grads - global_norm_fp32 = layers.concat( - sum_square_fp32) if len(sum_square_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_norm_fp32 = ( + layers.concat(sum_square_fp32) + if len(sum_square_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_norm_fp32 = layers.reduce_sum(global_norm_fp32) # global norm of non-distributed FP32 params_and_grads for unslice parameter - global_unslice_fp32 = layers.concat(unslice_params_fp32) if len( - unslice_params_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_unslice_fp32 = ( + layers.concat(unslice_params_fp32) + if len(unslice_params_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_unslice_fp32 = layers.reduce_sum(global_unslice_fp32) global_unslice_var = global_unslice_fp16 + global_unslice_fp32 - global_norm_var = global_norm_fp16 + global_norm_fp32 + 1.0 / self._group.nranks * global_unslice_var + global_norm_var = ( + global_norm_fp16 + + global_norm_fp32 + + 1.0 / self._group.nranks * global_unslice_var + ) # add all reduce to get global norm of distributed params_and_grads dev_id = int(self._device.split(":")[1]) @@ -117,14 +132,14 @@ class ShardingClipGrad: paddle.distributed.all_reduce(global_norm_var, group=self._group) global_norm_var = layers.sqrt(global_norm_var) - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) - - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var, - y=max_global_norm)) + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) + + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var, y=max_global_norm), + ) clip_var_fp16 = paddle.cast(clip_var, paddle.float16) for p, g in params_grads: @@ -163,7 +178,6 @@ def device_guard(dev_id=0, device="cpu"): @dygraph_only def ShardingScaler(scaler): - def unscale_method(self, optimizer): if not self._enable: return @@ -175,14 +189,16 @@ def ShardingScaler(scaler): optimizer.update_scaler = True if getattr(optimizer._optim, '_param_groups', None) and isinstance( - optimizer._optim._param_groups[0], dict): + optimizer._optim._param_groups[0], dict + ): for group in optimizer._optim._param_groups: for param in group['params']: if param._grad_ivar() is not None: param_grads.append(param._grad_ivar()) if param._grad_ivar().dtype in [ - core.VarDesc.VarType.FP16, paddle.float16 + core.VarDesc.VarType.FP16, + paddle.float16, ]: param_grads_fp16.append(param._grad_ivar()) else: @@ -192,7 +208,8 @@ def ShardingScaler(scaler): if param.grad is not None: param_grads.append(param.grad) if param.grad.dtype in [ - core.VarDesc.VarType.FP16, paddle.float16 + core.VarDesc.VarType.FP16, + paddle.float16, ]: param_grads_fp16.append(param.grad) else: @@ -202,27 +219,34 @@ def ShardingScaler(scaler): temp_found_inf_fp32 = to_variable(np.array([0]).astype(np.bool_)) device = "cpu" if optimizer.offload else "gpu" - dev_id = 0 if device == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 if device == "cpu" else int(paddle.get_device().split(":")[1]) + ) with device_guard(dev_id, device): if len(param_grads_fp16): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp16, - self._scale, - param_grads_fp16, - temp_found_inf_fp16) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp16, + self._scale, + param_grads_fp16, + temp_found_inf_fp16, + ) if len(param_grads_fp32): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp32, - self._scale, - param_grads_fp32, - temp_found_inf_fp32) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp32, + self._scale, + param_grads_fp32, + temp_found_inf_fp32, + ) self._found_inf = 1 if temp_found_inf_fp16 or temp_found_inf_fp32 else 0 is_found_inf = paddle.to_tensor([self._found_inf], dtype="int32") - paddle.distributed.all_reduce(is_found_inf, - op=paddle.distributed.ReduceOp.MAX, - group=optimizer.group) + paddle.distributed.all_reduce( + is_found_inf, + op=paddle.distributed.ReduceOp.MAX, + group=optimizer.group, + ) self._found_inf = is_found_inf.numpy()[0] scaler._unscale = MethodType(unscale_method, scaler) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding_parallel.py b/python/paddle/distributed/fleet/meta_parallel/sharding_parallel.py index fc3f195f7de..65fd317bd76 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding_parallel.py @@ -20,7 +20,6 @@ __all__ = [] class ShardingParallel(MetaParallelBase): - def __init__(self, layers, hcg, **kwargs): super(ShardingParallel, self).__init__(layers, hcg, **kwargs) diff --git a/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py b/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py index 2e2072e9a3e..5a7d0ae6c77 100755 --- a/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py @@ -15,14 +15,16 @@ from .meta_parallel_base import MetaParallelBase from ..utils.hybrid_parallel_util import broadcast_dp_parameters from ..utils.hybrid_parallel_util import broadcast_input_data -from ..utils.hybrid_parallel_util import broadcast_mp_parameters, broadcast_sharding_parameters +from ..utils.hybrid_parallel_util import ( + broadcast_mp_parameters, + broadcast_sharding_parameters, +) from ..utils.log_util import logger __all__ = [] class TensorParallel(MetaParallelBase): - def __init__(self, layers, hcg, **kwargs): super(TensorParallel, self).__init__(layers, hcg, **kwargs) diff --git a/python/paddle/distributed/fleet/metrics/metric.py b/python/paddle/distributed/fleet/metrics/metric.py index 6aa3793cd65..34c1c2968c8 100644 --- a/python/paddle/distributed/fleet/metrics/metric.py +++ b/python/paddle/distributed/fleet/metrics/metric.py @@ -257,7 +257,8 @@ def mae(abserr, total_ins_num, scope=None, util=None): abserr = np.array(scope.find_var(abserr).get_tensor()) if isinstance(total_ins_num, Variable): total_ins_num = np.array( - scope.find_var(total_ins_num.name).get_tensor()) + scope.find_var(total_ins_num.name).get_tensor() + ) elif isinstance(total_ins_num, str): total_ins_num = np.array(scope.find_var(total_ins_num).get_tensor()) @@ -306,7 +307,8 @@ def rmse(sqrerr, total_ins_num, scope=None, util=None): sqrerr = np.array(scope.find_var(sqrerr).get_tensor()) if isinstance(total_ins_num, Variable): total_ins_num = np.array( - scope.find_var(total_ins_num.name).get_tensor()) + scope.find_var(total_ins_num.name).get_tensor() + ) elif isinstance(total_ins_num, str): total_ins_num = np.array(scope.find_var(total_ins_num).get_tensor()) old_metric_shape = np.array(sqrerr.shape) @@ -355,7 +357,8 @@ def mse(sqrerr, total_ins_num, scope=None, util=None): sqrerr = np.array(scope.find_var(sqrerr).get_tensor()) if isinstance(total_ins_num, Variable): total_ins_num = np.array( - scope.find_var(total_ins_num.name).get_tensor()) + scope.find_var(total_ins_num.name).get_tensor() + ) elif isinstance(total_ins_num, str): total_ins_num = np.array(scope.find_var(total_ins_num).get_tensor()) old_metric_shape = np.array(sqrerr.shape) diff --git a/python/paddle/distributed/fleet/model.py b/python/paddle/distributed/fleet/model.py index 632e016f3d3..21e6d07ad55 100644 --- a/python/paddle/distributed/fleet/model.py +++ b/python/paddle/distributed/fleet/model.py @@ -15,7 +15,12 @@ import paddle from .base.topology import ParallelMode from .meta_parallel import TensorParallel -from .meta_parallel import PipelineParallel, ShardingParallel, PipelineParallelWithInterleave, PipelineLayer +from .meta_parallel import ( + PipelineParallel, + ShardingParallel, + PipelineParallelWithInterleave, + PipelineLayer, +) from paddle.fluid.dygraph.varbase_patch_methods import _grad_scalar from paddle.distributed import fleet @@ -89,19 +94,23 @@ def distributed_model(model): amp_enable = True amp_level = "O2" if strategy.amp_configs['use_pure_fp16'] else "O1" if amp_level.upper() == "O2": - model = paddle.amp.decorate(models=model, - optimizers=None, - level="O2", - master_weight=None, - save_dtype=None) + model = paddle.amp.decorate( + models=model, + optimizers=None, + level="O2", + master_weight=None, + save_dtype=None, + ) init_loss_scaling = strategy.amp_configs['init_loss_scaling'] incr_ratio = strategy.amp_configs['incr_ratio'] decr_ratio = strategy.amp_configs['decr_ratio'] incr_every_n_steps = strategy.amp_configs['incr_every_n_steps'] decr_every_n_nan_or_inf = strategy.amp_configs[ - 'decr_every_n_nan_or_inf'] + 'decr_every_n_nan_or_inf' + ] use_dynamic_loss_scaling = strategy.amp_configs[ - 'use_dynamic_loss_scaling'] + 'use_dynamic_loss_scaling' + ] global _grad_scalar _grad_scalar = paddle.amp.GradScaler( @@ -110,14 +119,16 @@ def distributed_model(model): decr_ratio=decr_ratio, incr_every_n_steps=incr_every_n_steps, decr_every_n_nan_or_inf=decr_every_n_nan_or_inf, - use_dynamic_loss_scaling=use_dynamic_loss_scaling) + use_dynamic_loss_scaling=use_dynamic_loss_scaling, + ) if strategy.heter_ccl_mode == True: distributed_model = paddle.DataParallel( model, comm_buffer_size=strategy.fuse_grad_size_in_MB, last_comm_buffer_size=strategy.last_comm_group_size_MB, - find_unused_parameters=strategy.find_unused_parameters) + find_unused_parameters=strategy.find_unused_parameters, + ) return distributed_model if fleet_env._hcg.get_parallel_mode() == ParallelMode.SHARDING_PARALLEL: @@ -127,8 +138,13 @@ def distributed_model(model): # NOTE (JZ-LIANG) init parameters broadcast within sharding group # normally it should be done inside DataParallel if fleet_env.sharding_degree > 1: - from paddle.distributed.fleet.utils.hybrid_parallel_util import broadcast_sharding_parameters - assert fleet_env.sharding_degree == fleet_env._hcg.get_sharding_parallel_world_size( + from paddle.distributed.fleet.utils.hybrid_parallel_util import ( + broadcast_sharding_parameters, + ) + + assert ( + fleet_env.sharding_degree + == fleet_env._hcg.get_sharding_parallel_world_size() ) broadcast_sharding_parameters(model, fleet_env._hcg) model = paddle.DataParallel( @@ -136,7 +152,8 @@ def distributed_model(model): comm_buffer_size=strategy.fuse_grad_size_in_MB, last_comm_buffer_size=strategy.last_comm_group_size_MB, find_unused_parameters=strategy.find_unused_parameters, - group=fleet_env._hcg.get_data_parallel_group()) + group=fleet_env._hcg.get_data_parallel_group(), + ) elif fleet_env._hcg.get_parallel_mode() == ParallelMode.TENSOR_PARALLEL: model = TensorParallel(model, fleet_env._hcg, strategy=strategy) elif fleet_env._hcg.get_parallel_mode() == ParallelMode.PIPELINE_PARALLEL: @@ -148,8 +165,8 @@ def distributed_model(model): model = PipelineParallel(model, fleet_env._hcg, strategy=strategy) else: # interleave pipeline - model = PipelineParallelWithInterleave(model, - fleet_env._hcg, - strategy=strategy) + model = PipelineParallelWithInterleave( + model, fleet_env._hcg, strategy=strategy + ) return model diff --git a/python/paddle/distributed/fleet/optimizer.py b/python/paddle/distributed/fleet/optimizer.py index 42567465c55..37a3a896f6b 100644 --- a/python/paddle/distributed/fleet/optimizer.py +++ b/python/paddle/distributed/fleet/optimizer.py @@ -21,27 +21,27 @@ from .utils.log_util import logger def _dygraph_distributed_optimizer(optimizer, strategy=None): """ - Optimizer for distributed training. - For the distributed training, this method would rebuild a new instance of DistributedOptimizer. - Which has basic Optimizer function and special features for distributed training. - Args: - optimizer(Optimizer): The executor to run for init server. - strategy(DistributedStrategy): Extra properties for distributed optimizer. - It is recommended to use DistributedStrategy in fleet.init(). The strategy - here is for compatibility. If the strategy in fleet.distributed_optimizer() - is not None, then it will overwrite the DistributedStrategy in fleet.init(), - which will take effect in distributed training. - Returns: - Fleet: instance of fleet. - Examples: - .. code-block:: python - import paddle - import paddle.distributed.fleet as fleet - fleet.init(is_collective=True) - strategy = fleet.DistributedStrategy() - optimizer = paddle.optimizer.SGD(learning_rate=0.001) - optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) - """ + Optimizer for distributed training. + For the distributed training, this method would rebuild a new instance of DistributedOptimizer. + Which has basic Optimizer function and special features for distributed training. + Args: + optimizer(Optimizer): The executor to run for init server. + strategy(DistributedStrategy): Extra properties for distributed optimizer. + It is recommended to use DistributedStrategy in fleet.init(). The strategy + here is for compatibility. If the strategy in fleet.distributed_optimizer() + is not None, then it will overwrite the DistributedStrategy in fleet.init(), + which will take effect in distributed training. + Returns: + Fleet: instance of fleet. + Examples: + .. code-block:: python + import paddle + import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) + strategy = fleet.DistributedStrategy() + optimizer = paddle.optimizer.SGD(learning_rate=0.001) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + """ fleet_env = fleet.fleet fleet_env.user_defined_optimizer = optimizer @@ -52,18 +52,21 @@ def _dygraph_distributed_optimizer(optimizer, strategy=None): "in fleet_env.init(). The strategy here is only for compatibility. " "If the strategy in fleet_env.distributed_optimizer() is " "not None, then it will overwrite the DistributedStrategy in fleet_env.init(), " - "which will take effect in distributed training.") + "which will take effect in distributed training." + ) fleet_env._user_defined_strategy = copy.deepcopy(strategy) fleet_env._context = {} if fleet_env.worker_num() > 1: if fleet_env._user_defined_strategy.heter_ccl_mode == False: - return HybridParallelOptimizer(optimizer, fleet_env._hcg, - fleet_env._user_defined_strategy) + return HybridParallelOptimizer( + optimizer, fleet_env._hcg, fleet_env._user_defined_strategy + ) else: - return HeterParallelOptimizer(optimizer, - fleet_env._user_defined_strategy) + return HeterParallelOptimizer( + optimizer, fleet_env._user_defined_strategy + ) else: return optimizer diff --git a/python/paddle/distributed/fleet/recompute/recompute.py b/python/paddle/distributed/fleet/recompute/recompute.py index 03b0c9d73f0..2657c60f02c 100755 --- a/python/paddle/distributed/fleet/recompute/recompute.py +++ b/python/paddle/distributed/fleet/recompute/recompute.py @@ -40,16 +40,23 @@ def detach_variable(inputs): def check_recompute_necessary(inputs): - if not any(input_.stop_gradient == False for input_ in inputs - if isinstance(input_, (core.eager.Tensor, paddle.Tensor))): + if not any( + input_.stop_gradient == False + for input_ in inputs + if isinstance(input_, (core.eager.Tensor, paddle.Tensor)) + ): logger.warning( "[Recompute]: None of the inputs to current recompute block need grad, " - "therefore there is NO need to recompute this block in backward !") + "therefore there is NO need to recompute this block in backward !" + ) @contextlib.contextmanager def swith_rng_state_tracker(rng_state, tracker): - from paddle.distributed.fleet.meta_parallel.parallel_layers.random import get_rng_state_tracker + from paddle.distributed.fleet.meta_parallel.parallel_layers.random import ( + get_rng_state_tracker, + ) + orig_cuda_rng_state = paddle.get_cuda_rng_state() orig_cuda_rng_tracker = get_rng_state_tracker().get_states_tracker() @@ -63,10 +70,11 @@ def swith_rng_state_tracker(rng_state, tracker): class LegacyRecomputeFunction(LegacyPyLayer): - @staticmethod def forward(ctx, run_function, preserve_rng_state, *args): - from paddle.distributed.fleet.meta_parallel.parallel_layers.random import get_rng_state_tracker + from paddle.distributed.fleet.meta_parallel.parallel_layers.random import ( + get_rng_state_tracker, + ) # store for recomputing ctx.run_function = run_function @@ -95,30 +103,37 @@ class LegacyRecomputeFunction(LegacyPyLayer): cur_device = paddle.get_device() if 'gpu:' not in cur_device: raise RuntimeError( - "Recompute with RNG perserve is not support current device: {}." - .format(cur_device)) + "Recompute with RNG perserve is not support current device: {}.".format( + cur_device + ) + ) ctx.fw_cuda_rng_state = paddle.get_cuda_rng_state() - ctx.fwd_cuda_rng_state_tracker = get_rng_state_tracker( - ).get_states_tracker() + ctx.fwd_cuda_rng_state_tracker = ( + get_rng_state_tracker().get_states_tracker() + ) # TODO support AMP tracer = framework._dygraph_tracer() - ctx.is_fw_autocast = False if tracer._amp_level == core.AmpLevel.O0 else True + ctx.is_fw_autocast = ( + False if tracer._amp_level == core.AmpLevel.O0 else True + ) if tracer._amp_level == core.AmpLevel.O2: ctx.amp_level = 'O2' elif tracer._amp_level in (core.AmpLevel.O1, core.AmpLevel.O0): ctx.amp_level = 'O1' else: - raise ValueError("unsupported amp level: {}".format( - tracer._amp_level)) + raise ValueError( + "unsupported amp level: {}".format(tracer._amp_level) + ) if tracer._amp_dtype == 'float16': ctx.amp_dtype = 'float16' elif tracer._amp_dtype in ('bfloat16', 'float32'): ctx.amp_dtype = 'bfloat16' else: - raise ValueError("unsupported amp dtype: {}".format( - tracer._amp_dtype)) + raise ValueError( + "unsupported amp dtype: {}".format(tracer._amp_dtype) + ) ctx.amp_white_list, ctx.amp_black_list = tracer._get_amp_op_list() @@ -145,27 +160,31 @@ class LegacyRecomputeFunction(LegacyPyLayer): # NOTE support AMP # need restore auto_cast state as well as w/b list if ctx.preserve_rng_state: - with swith_rng_state_tracker(ctx.fw_cuda_rng_state, - ctx.fwd_cuda_rng_state_tracker): + with swith_rng_state_tracker( + ctx.fw_cuda_rng_state, ctx.fwd_cuda_rng_state_tracker + ): with paddle.amp.auto_cast( - enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level, - dtype=ctx.amp_dtype): + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + dtype=ctx.amp_dtype, + ): detached_inputs = detach_variable(tuple(inputs)) outputs = ctx.run_function(*detached_inputs) else: - with paddle.amp.auto_cast(enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level, - dtype=ctx.amp_dtype): + with paddle.amp.auto_cast( + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + dtype=ctx.amp_dtype, + ): detached_inputs = detach_variable(tuple(inputs)) outputs = ctx.run_function(*detached_inputs) if isinstance(outputs, core.VarBase): - outputs = (outputs, ) + outputs = (outputs,) assert len(outputs) == len(args) # run backward() with only tensor that requires grad @@ -176,8 +195,10 @@ class LegacyRecomputeFunction(LegacyPyLayer): # the following backward_inputs_with_grad is used to avoid this case. backward_inputs_with_grad = [] for i in range(len(outputs)): - if isinstance(outputs[i], - core.VarBase) and not outputs[i].stop_gradient: + if ( + isinstance(outputs[i], core.VarBase) + and not outputs[i].stop_gradient + ): forward_outputs_with_grad.append(outputs[i]) backward_inputs_with_grad.append(args[i]) @@ -188,19 +209,24 @@ class LegacyRecomputeFunction(LegacyPyLayer): # actually backward with paddle.amp.auto_cast(enable=False): - paddle.autograd.backward(forward_outputs_with_grad, - backward_inputs_with_grad) + paddle.autograd.backward( + forward_outputs_with_grad, backward_inputs_with_grad + ) - grads = list(inp._grad_ivar() for inp in detached_inputs - if isinstance(inp, core.VarBase)) + grads = list( + inp._grad_ivar() + for inp in detached_inputs + if isinstance(inp, core.VarBase) + ) return grads class RecomputeFunction(PyLayer): - @staticmethod def forward(ctx, run_function, preserve_rng_state, *args, **kwargs): - from paddle.distributed.fleet.meta_parallel.parallel_layers.random import get_rng_state_tracker + from paddle.distributed.fleet.meta_parallel.parallel_layers.random import ( + get_rng_state_tracker, + ) # store for recomputing ctx.run_function = run_function @@ -230,30 +256,37 @@ class RecomputeFunction(PyLayer): cur_device = paddle.get_device() if 'gpu:' not in cur_device: raise RuntimeError( - "Recompute with RNG perserve is not support current device: {}." - .format(cur_device)) + "Recompute with RNG perserve is not support current device: {}.".format( + cur_device + ) + ) ctx.fw_cuda_rng_state = paddle.get_cuda_rng_state() - ctx.fwd_cuda_rng_state_tracker = get_rng_state_tracker( - ).get_states_tracker() + ctx.fwd_cuda_rng_state_tracker = ( + get_rng_state_tracker().get_states_tracker() + ) # TODO support AMP tracer = framework._dygraph_tracer() - ctx.is_fw_autocast = False if tracer._amp_level == core.AmpLevel.O0 else True + ctx.is_fw_autocast = ( + False if tracer._amp_level == core.AmpLevel.O0 else True + ) if tracer._amp_level == core.AmpLevel.O2: ctx.amp_level = 'O2' elif tracer._amp_level in (core.AmpLevel.O1, core.AmpLevel.O0): ctx.amp_level = 'O1' else: - raise ValueError("unsupported amp level: {}".format( - tracer._amp_level)) + raise ValueError( + "unsupported amp level: {}".format(tracer._amp_level) + ) if tracer._amp_dtype == 'float16': ctx.amp_dtype = 'float16' elif tracer._amp_dtype in ('bfloat16', 'float32'): ctx.amp_dtype = 'bfloat16' else: - raise ValueError("unsupported amp dtype: {}".format( - tracer._amp_dtype)) + raise ValueError( + "unsupported amp dtype: {}".format(tracer._amp_dtype) + ) ctx.amp_white_list, ctx.amp_black_list = tracer._get_amp_op_list() @@ -280,28 +313,33 @@ class RecomputeFunction(PyLayer): # NOTE support AMP # need restore auto_cast state as well as w/b list if ctx.preserve_rng_state: - with swith_rng_state_tracker(ctx.fw_cuda_rng_state, - ctx.fwd_cuda_rng_state_tracker): + with swith_rng_state_tracker( + ctx.fw_cuda_rng_state, ctx.fwd_cuda_rng_state_tracker + ): with paddle.amp.auto_cast( - enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level, - dtype=ctx.amp_dtype): + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + dtype=ctx.amp_dtype, + ): detached_inputs = detach_variable(tuple(inputs)) - outputs = ctx.run_function(*detached_inputs, - **ctx.kwargs) + outputs = ctx.run_function( + *detached_inputs, **ctx.kwargs + ) else: - with paddle.amp.auto_cast(enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level, - dtype=ctx.amp_dtype): + with paddle.amp.auto_cast( + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + dtype=ctx.amp_dtype, + ): detached_inputs = detach_variable(tuple(inputs)) outputs = ctx.run_function(*detached_inputs, **ctx.kwargs) if isinstance(outputs, (core.VarBase, core.eager.Tensor)): - outputs = (outputs, ) + outputs = (outputs,) assert len(outputs) == len(args) # run backward() with only tensor that requires grad @@ -312,10 +350,10 @@ class RecomputeFunction(PyLayer): # the following backward_inputs_with_grad is used to avoid this case. backward_inputs_with_grad = [] for i in range(len(outputs)): - if isinstance( - outputs[i], - (core.VarBase, - core.eager.Tensor)) and not outputs[i].stop_gradient: + if ( + isinstance(outputs[i], (core.VarBase, core.eager.Tensor)) + and not outputs[i].stop_gradient + ): forward_outputs_with_grad.append(outputs[i]) backward_inputs_with_grad.append(args[i]) @@ -326,17 +364,22 @@ class RecomputeFunction(PyLayer): # actually backward with paddle.amp.auto_cast(enable=False): - paddle.autograd.backward(forward_outputs_with_grad, - backward_inputs_with_grad) + paddle.autograd.backward( + forward_outputs_with_grad, backward_inputs_with_grad + ) if in_dygraph_mode(): grads = tuple( - inp._grad_ivar() for inp in detached_inputs - if isinstance(inp, (core.VarBase, core.eager.Tensor))) + inp._grad_ivar() + for inp in detached_inputs + if isinstance(inp, (core.VarBase, core.eager.Tensor)) + ) else: grads = list( - inp._grad_ivar() for inp in detached_inputs - if isinstance(inp, (core.VarBase, core.eager.Tensor))) + inp._grad_ivar() + for inp in detached_inputs + if isinstance(inp, (core.VarBase, core.eager.Tensor)) + ) return grads @@ -494,7 +537,6 @@ def recompute_sequential(ctx, functions, *args, **kwargs): preserve_rng_state = ctx.get('preserve_rng_state', True) def _run_func(begin, end, funcs): - def do_run(input): for i in range(begin, end + 1): input = funcs[i](input) @@ -510,8 +552,10 @@ def recompute_sequential(ctx, functions, *args, **kwargs): end = -1 for begin in range(0, segment_size * (segments - 1), segment_size): end = begin + segment_size - 1 - args = recompute(_run_func(begin, end, functions), - *args, - preserve_rng_state=preserve_rng_state, - **kwargs) + args = recompute( + _run_func(begin, end, functions), + *args, + preserve_rng_state=preserve_rng_state, + **kwargs + ) return _run_func(end + 1, len(functions) - 1, functions)(args) diff --git a/python/paddle/distributed/fleet/recompute/recompute_hybrid.py b/python/paddle/distributed/fleet/recompute/recompute_hybrid.py index 9c006c1e044..7f0d22726e5 100644 --- a/python/paddle/distributed/fleet/recompute/recompute_hybrid.py +++ b/python/paddle/distributed/fleet/recompute/recompute_hybrid.py @@ -17,7 +17,11 @@ from paddle.fluid import core from paddle.autograd import PyLayer from paddle.fluid import framework from ..meta_parallel.parallel_layers.random import get_rng_state_tracker -from .recompute import check_recompute_necessary, detach_variable, swith_rng_state_tracker +from .recompute import ( + check_recompute_necessary, + detach_variable, + swith_rng_state_tracker, +) from ..meta_parallel.pp_utils import utils __all__ = [] @@ -32,8 +36,11 @@ def _split_activation(tensor, mp_group): tensor_numel = paddle.numel(tensor) assert tensor_numel != 0, "can't recompute zero element" - assert tensor_numel % mp_degree == 0, "The capacity of the activation ({}) cannot be divisible by mp_degree({})".format( - tensor_numel, mp_degree) + assert ( + tensor_numel % mp_degree == 0 + ), "The capacity of the activation ({}) cannot be divisible by mp_degree({})".format( + tensor_numel, mp_degree + ) # use inplace operation to save memory data = tensor.flatten_() @@ -69,8 +76,16 @@ class _HPRecomputeFunction(PyLayer): """ @staticmethod - def forward(ctx, run_function, all_outputs, mp_group, offload, partition, - *args, **kwargs): + def forward( + ctx, + run_function, + all_outputs, + mp_group, + offload, + partition, + *args, + **kwargs + ): check_recompute_necessary(args) # store for recomputing @@ -80,8 +95,9 @@ class _HPRecomputeFunction(PyLayer): # store the rng states ctx.fwd_cuda_rng_state = paddle.get_cuda_rng_state() - ctx.fwd_cuda_rng_state_tracker = get_rng_state_tracker( - ).get_states_tracker() + ctx.fwd_cuda_rng_state_tracker = ( + get_rng_state_tracker().get_states_tracker() + ) # save config info ctx.mp_group = mp_group @@ -95,20 +111,25 @@ class _HPRecomputeFunction(PyLayer): tensor_inputs = [] cur_device = paddle.get_device() - assert 'gpu:' in paddle.get_device( + assert ( + 'gpu:' in paddle.get_device() ), "Recompute with RNG is not support current device: {}.".format( - cur_device) + cur_device + ) # TODO support AMP tracer = framework._dygraph_tracer() - ctx.is_fw_autocast = False if tracer._amp_level == core.AmpLevel.O0 else True + ctx.is_fw_autocast = ( + False if tracer._amp_level == core.AmpLevel.O0 else True + ) if tracer._amp_level == core.AmpLevel.O2: ctx.amp_level = 'O2' elif tracer._amp_level in (core.AmpLevel.O1, core.AmpLevel.O0): ctx.amp_level = 'O1' else: - raise ValueError("unsupported amp level: {}".format( - tracer._amp_level)) + raise ValueError( + "unsupported amp level: {}".format(tracer._amp_level) + ) ctx.amp_white_list, ctx.amp_black_list = tracer._get_amp_op_list() with paddle.no_grad(): @@ -119,8 +140,9 @@ class _HPRecomputeFunction(PyLayer): state = arg.stop_gradient if partition: ctx.tensor_shapes.append(arg.shape) - partition = _split_activation(arg.detach(), - mp_group).clone() + partition = _split_activation( + arg.detach(), mp_group + ).clone() # TODO(shenliang03) not use calculate stream to D2H to speed arg = partition.cpu() if offload else partition else: @@ -154,38 +176,44 @@ class _HPRecomputeFunction(PyLayer): for i, idx in enumerate(tensor_indices): if ctx.partition: state = tensors[i].stop_gradient - tensors[i] = _merge_activation( - tensors[i], - ctx.mp_group).detach().reshape_(tensor_shapes[i]) + tensors[i] = ( + _merge_activation(tensors[i], ctx.mp_group) + .detach() + .reshape_(tensor_shapes[i]) + ) tensors[i].stop_gradient = state - inputs[idx] = tensors[i].cuda( - device_id) if ctx.offload else tensors[i] + inputs[idx] = ( + tensors[i].cuda(device_id) if ctx.offload else tensors[i] + ) tracer = framework._dygraph_tracer() tracer._has_grad = True # need restore auto_cast state as well as w/b list - with swith_rng_state_tracker(ctx.fwd_cuda_rng_state, - ctx.fwd_cuda_rng_state_tracker): - with paddle.amp.auto_cast(enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level): + with swith_rng_state_tracker( + ctx.fwd_cuda_rng_state, ctx.fwd_cuda_rng_state_tracker + ): + with paddle.amp.auto_cast( + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + ): detached_inputs = detach_variable(tuple(inputs)) outputs = ctx.run_function(*detached_inputs, **ctx.kwargs) if isinstance(outputs, (core.VarBase, core.eager.Tensor)): - outputs = (outputs, ) + outputs = (outputs,) assert len(outputs) == len(args) forward_outputs_with_grad = [] backward_inputs = [] for i in range(len(outputs)): - if isinstance( - outputs[i], - (core.VarBase, - core.eager.Tensor)) and not outputs[i].stop_gradient: + if ( + isinstance(outputs[i], (core.VarBase, core.eager.Tensor)) + and not outputs[i].stop_gradient + ): forward_outputs_with_grad.append(outputs[i]) backward_inputs.append(args[i]) @@ -196,8 +224,11 @@ class _HPRecomputeFunction(PyLayer): # actually backward paddle.autograd.backward(forward_outputs_with_grad, backward_inputs) - grads = tuple(inp._grad_ivar() for inp in detached_inputs - if isinstance(inp, (core.VarBase, core.eager.Tensor))) + grads = tuple( + inp._grad_ivar() + for inp in detached_inputs + if isinstance(inp, (core.VarBase, core.eager.Tensor)) + ) return grads @@ -226,14 +257,17 @@ def recompute_hybrid(ctx, function, *args, **kwargs): """ mp_group = ctx.get('mp_group', None) - assert mp_group is not None, "ctx must contains mp_group and mp_group can not be None." + assert ( + mp_group is not None + ), "ctx must contains mp_group and mp_group can not be None." offload = ctx.get('offload', False) partition = ctx.get('partition', False) all_outputs = [] - _HPRecomputeFunction.apply(function, all_outputs, mp_group, offload, - partition, *args, **kwargs) + _HPRecomputeFunction.apply( + function, all_outputs, mp_group, offload, partition, *args, **kwargs + ) if len(all_outputs) == 1: return all_outputs[0] diff --git a/python/paddle/distributed/fleet/runtime/collective_runtime.py b/python/paddle/distributed/fleet/runtime/collective_runtime.py index 5b66bf79398..7ccd0ecf7d0 100644 --- a/python/paddle/distributed/fleet/runtime/collective_runtime.py +++ b/python/paddle/distributed/fleet/runtime/collective_runtime.py @@ -19,28 +19,32 @@ __all__ = [] class CollectiveRuntime(RuntimeBase): - def __init__(self): super(CollectiveRuntime, self).__init__() def _init_worker(self): logging.warn( - "You should not call 'init_worker' method for collective mode.") + "You should not call 'init_worker' method for collective mode." + ) def _run_worker(self): logging.warn( - "You should not call 'run_worker' method for collective mode.") + "You should not call 'run_worker' method for collective mode." + ) def _init_server(self, *args, **kwargs): logging.warn( - "You should not call 'init_server' method for collective mode.") + "You should not call 'init_server' method for collective mode." + ) def _run_server(self): logging.warn( - "You should not call 'run_server' method for collective mode.") + "You should not call 'run_server' method for collective mode." + ) def _stop_worker(self): logging.warn( - "You should not call 'stop_worker' method for collective mode.") + "You should not call 'stop_worker' method for collective mode." + ) # save inference model should be added here diff --git a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py index 062a6d5abf5..e2c5e5da29e 100644 --- a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py +++ b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py @@ -30,7 +30,6 @@ __all__ = [] class ParameterServerRuntime(RuntimeBase): - def __init__(self): super(ParameterServerRuntime, self).__init__() self._communicator = None @@ -46,7 +45,9 @@ class ParameterServerRuntime(RuntimeBase): def _get_distributed_strategy(self): strategy = None - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, + ) dist_strategy = self.context["valid_strategy"] k_steps = dist_strategy.a_sync_configs["k_steps"] @@ -66,19 +67,21 @@ class ParameterServerRuntime(RuntimeBase): return strategy def build_compiled_startegy(self): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import CompileTimeStrategy - - compiled_config = CompileTimeStrategy(self.origin_main_program, - self.origin_main_program, - self.async_strategy, - self.role_maker) + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + CompileTimeStrategy, + ) + + compiled_config = CompileTimeStrategy( + self.origin_main_program, + self.origin_main_program, + self.async_strategy, + self.role_maker, + ) return compiled_config - def _load_sparse_params(self, - executor, - dirname, - varnames, - main_program=None): + def _load_sparse_params( + self, executor, dirname, varnames, main_program=None + ): assert vars != None check_vars = [] load_prog = Program() @@ -88,12 +91,15 @@ class ParameterServerRuntime(RuntimeBase): return var.name in varnames load_vars = list( - filter(_in_varnames, - fluid.default_main_program().list_vars())) + filter(_in_varnames, fluid.default_main_program().list_vars()) + ) if main_program is None: main_program = self.origin_main_program - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, + ) + for each_var in load_vars: assert isinstance(each_var, Variable) @@ -104,29 +110,31 @@ class ParameterServerRuntime(RuntimeBase): if not os.path.exists(var_path): raise ValueError( "SelectedRows var {} can not find at {}".format( - new_var.name, var_path)) + new_var.name, var_path + ) + ) if os.path.isfile(var_path): - load_block.append_op(type='sparse_tensor_load', - inputs={}, - outputs={'Out': [new_var]}, - attrs={ - 'file_path': - os.path.join(dirname, origin_varname), - 'node_index': - self.role_maker._server_index(), - 'node_num': - self.role_maker._server_num(), - 'shape': - each_var.shape - }) + load_block.append_op( + type='sparse_tensor_load', + inputs={}, + outputs={'Out': [new_var]}, + attrs={ + 'file_path': os.path.join(dirname, origin_varname), + 'node_index': self.role_maker._server_index(), + 'node_num': self.role_maker._server_num(), + 'shape': each_var.shape, + }, + ) check_vars.append(each_var) executor.run(load_prog) def _load_distributed_params(self, dirname, varnames): from paddle.fluid.communicator import LargeScaleKV - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, + ) scale_kv = LargeScaleKV() for varname in varnames: @@ -136,12 +144,13 @@ class ParameterServerRuntime(RuntimeBase): @staticmethod def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, + ) origin_varname, _, _ = _get_varname_parts(var.name) if origin_varname.endswith("@GRAD"): @@ -150,25 +159,29 @@ class ParameterServerRuntime(RuntimeBase): if origin_varname == "learning_rate_0": return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable return is_valid def _init_worker(self): - def sync_strategy_envs(): kwargs = {} kwargs[ - "pserver_endpoints"] = self.role_maker._get_pserver_endpoints() + "pserver_endpoints" + ] = self.role_maker._get_pserver_endpoints() kwargs["trainer_id"] = self.role_maker._worker_index() return kwargs def geo_strategy_envs(): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, + ) def get_sparse_attrs(): opt_init_map = {} @@ -176,13 +189,17 @@ class ParameterServerRuntime(RuntimeBase): opt_init_map["fill_constant"] = ["value"] opt_init_map["uniform_random"] = ["seed", "min", "max"] opt_init_map["truncated_gaussian_random"] = [ - "seed", "mean", "std" + "seed", + "mean", + "std", ] - dist_varnames = get_sparse_tablenames(self.origin_main_program, - True) + dist_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) sparse_varnames = get_sparse_tablenames( - self.origin_main_program, False) + self.origin_main_program, False + ) if len(dist_varnames) != 0: raise ValueError( @@ -191,15 +208,18 @@ class ParameterServerRuntime(RuntimeBase): init_attrs = [] for value_name in sparse_varnames: - value_var = self.origin_main_program.global_block( - ).vars[value_name] + value_var = self.origin_main_program.global_block().vars[ + value_name + ] value_attr = [ value_name, - ",".join([str(dim) for dim in value_var.shape]) + ",".join([str(dim) for dim in value_var.shape]), ] for op in self.origin_startup_program.global_block().ops: - if op.type in opt_init_map.keys( - ) and value_name == op.output("Out")[0]: + if ( + op.type in opt_init_map.keys() + and value_name == op.output("Out")[0] + ): init_attr = [op.type] for attr in opt_init_map[op.type]: init_attr.append(str(op.attr(attr))) @@ -213,10 +233,15 @@ class ParameterServerRuntime(RuntimeBase): kwargs["sparse_attrs"] = get_sparse_attrs() return kwargs - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_lr_ops, _has_global_step + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_lr_ops, + _has_global_step, + ) - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import \ - SyncStrategy, GeoStrategy + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + SyncStrategy, + GeoStrategy, + ) trainer_config = self.async_strategy.get_trainer_runtime_config() print(trainer_config) @@ -228,7 +253,9 @@ class ParameterServerRuntime(RuntimeBase): wait_server_ready(self.role_maker._get_pserver_endpoints()) # for ps-heter mode, wait heter worker ready - if self.role_maker._is_heter_parameter_server_mode and self.role_maker._is_worker( + if ( + self.role_maker._is_heter_parameter_server_mode + and self.role_maker._is_worker() ): wait_server_ready(self.role_maker._get_heter_worker_endpoints()) @@ -252,15 +279,18 @@ class ParameterServerRuntime(RuntimeBase): if self.compiled_strategy.is_geo_mode(): recv_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=4) + recv_type=4 + ) else: recv_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=1) + recv_type=1 + ) from paddle.fluid.communicator import Communicator + self._communicator = Communicator( - trainer_config.mode, kwargs, - trainer_config.get_communicator_flags()) + trainer_config.mode, kwargs, trainer_config.get_communicator_flags() + ) self._communicator.init_with_ctx(send_ctx, recv_ctx) if not self._communicator.is_running(): @@ -271,21 +301,30 @@ class ParameterServerRuntime(RuntimeBase): def _get_executor(self): executor = fluid.Executor(fluid.CPUPlace()) if self.role_maker._is_heter_parameter_server_mode: - heter_worker_device_guard = self.context[ - "valid_strategy"].a_sync_configs[ - "heter_worker_device_guard"].upper() + heter_worker_device_guard = ( + self.context["valid_strategy"] + .a_sync_configs["heter_worker_device_guard"] + .upper() + ) if heter_worker_device_guard not in ["GPU", "XPU", "CPU"]: - raise ValueError("Heter Worker Not Support Device {}".format( - heter_worker_device_guard)) + raise ValueError( + "Heter Worker Not Support Device {}".format( + heter_worker_device_guard + ) + ) if self.role_maker._is_heter_worker(): if heter_worker_device_guard == "GPU": executor = Executor( fluid.CUDAPlace( - int(os.getenv("FLAGS_selected_gpus", "0")))) + int(os.getenv("FLAGS_selected_gpus", "0")) + ) + ) elif heter_worker_device_guard == "XPU": executor = Executor( fluid.XPUPlace( - int(os.getenv("FLAGS_selected_xpus", "0")))) + int(os.getenv("FLAGS_selected_xpus", "0")) + ) + ) return executor def _init_server(self, *args, **kwargs): @@ -297,8 +336,10 @@ class ParameterServerRuntime(RuntimeBase): model_dirname = None executor = self._get_executor() - if self.role_maker._is_heter_worker( - ) and self.context["valid_strategy"].a_sync_configs["launch_barrier"]: + if ( + self.role_maker._is_heter_worker() + and self.context["valid_strategy"].a_sync_configs["launch_barrier"] + ): # for heter trainer wait server ready wait_server_ready(self.role_maker._get_pserver_endpoints()) executor.run(fluid.default_startup_program()) @@ -310,26 +351,35 @@ class ParameterServerRuntime(RuntimeBase): sparse_varnames = self.compiled_strategy.get_sparse_varname_on_ps(False) sparse_related_optimize_varnames = [] for var_name in sparse_varnames: - sparse_related_optimize_varnames += self.compiled_strategy.get_optimize_varname_on_ps( - var_name) + sparse_related_optimize_varnames += ( + self.compiled_strategy.get_optimize_varname_on_ps(var_name) + ) sparse_related_optimize_varnames = list( - set(sparse_related_optimize_varnames)) + set(sparse_related_optimize_varnames) + ) distribtued_varnames = self.compiled_strategy.get_sparse_varname_on_ps( - True) + True + ) distributed_related_optimize_varnames = [] for var_name in distribtued_varnames: - distributed_related_optimize_varnames += self.compiled_strategy.get_optimize_varname_on_ps( - var_name) + distributed_related_optimize_varnames += ( + self.compiled_strategy.get_optimize_varname_on_ps(var_name) + ) distributed_related_optimize_varnames = list( - set(distributed_related_optimize_varnames)) + set(distributed_related_optimize_varnames) + ) remaining_vars = list( filter( ParameterServerRuntime.__exclude_vars( - sparse_varnames + distribtued_varnames + - sparse_related_optimize_varnames + - distributed_related_optimize_varnames), - fluid.default_main_program().list_vars())) + sparse_varnames + + distribtued_varnames + + sparse_related_optimize_varnames + + distributed_related_optimize_varnames + ), + fluid.default_main_program().list_vars(), + ) + ) if not model_dirname: return @@ -338,21 +388,26 @@ class ParameterServerRuntime(RuntimeBase): raise ValueError("There is no directory named '%s'", model_dirname) # load dense - fluid.io.load_vars(executor, - main_program=fluid.default_main_program(), - dirname=model_dirname, - vars=remaining_vars) + fluid.io.load_vars( + executor, + main_program=fluid.default_main_program(), + dirname=model_dirname, + vars=remaining_vars, + ) # load sparse - self._load_sparse_params(executor=executor, - dirname=model_dirname, - varnames=sparse_varnames + - sparse_related_optimize_varnames) + self._load_sparse_params( + executor=executor, + dirname=model_dirname, + varnames=sparse_varnames + sparse_related_optimize_varnames, + ) # load large scale - self._load_distributed_params(dirname=model_dirname, - varnames=distribtued_varnames + - distributed_related_optimize_varnames) + self._load_distributed_params( + dirname=model_dirname, + varnames=distribtued_varnames + + distributed_related_optimize_varnames, + ) def _run_server(self): executor = self._get_executor() @@ -365,8 +420,15 @@ class ParameterServerRuntime(RuntimeBase): def _get_optimizer_status(self, op, param_name): supported_opts = [ - "sgd", "adam", "adagrad", "adamax", "momentum", "lars_momentum", - "rmsprop", "decayed_adagrad", "ftrl" + "sgd", + "adam", + "adagrad", + "adamax", + "momentum", + "lars_momentum", + "rmsprop", + "decayed_adagrad", + "ftrl", ] reshaped_val_map = {} @@ -377,7 +439,9 @@ class ParameterServerRuntime(RuntimeBase): reshaped_val_map["momentum"] = ["velocity_0"] reshaped_val_map["lars_momentum"] = ["velocity_0"] reshaped_val_map["rmsprop"] = [ - "momentum_0", "mean_square_0", "mean_grad_0" + "momentum_0", + "mean_square_0", + "mean_grad_0", ] reshaped_val_map["decayed_adagrad"] = ["moment_0"] reshaped_val_map["ftrl"] = ["squared_0", "linear_0"] @@ -388,8 +452,10 @@ class ParameterServerRuntime(RuntimeBase): if op not in supported_opts: raise ValueError( - "fleet can not support optimizer: {}, only this can be supported: {}" - .format(op, supported_opts)) + "fleet can not support optimizer: {}, only this can be supported: {}".format( + op, supported_opts + ) + ) reshaped_names = [ param_name + "_" + val for val in reshaped_val_map[op] @@ -404,12 +470,17 @@ class ParameterServerRuntime(RuntimeBase): return reshaped_names, origin_names def _get_optimizer_op(self, param_name): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, + ) opts = _get_optimize_ops(self.origin_main_program) for op in opts: - if "Param" in op.input_names and \ - "LearningRate" in op.input_names and op.input("Param")[0] == param_name: + if ( + "Param" in op.input_names + and "LearningRate" in op.input_names + and op.input("Param")[0] == param_name + ): return op def _save_dense_params(self, executor, dirname, context, main_program): @@ -428,27 +499,24 @@ class ParameterServerRuntime(RuntimeBase): optimizer = self._get_optimizer_op(varname) reshaped_varnames, origin_varnames = self._get_optimizer_status( - optimizer.type, varname) + optimizer.type, varname + ) for var_name in [varname] + reshaped_varnames + origin_varnames: var = self.origin_main_program.global_block().vars[var_name] - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self.role_maker._worker_index(), - "shape": - var.shape, - "slice_shapes": - [",".join([str(i) for i in var.shape])], - "slice_varnames": [var.name], - "remote_varnames": [var.name], - "is_sparse": - False, - "endpoints": - var_ctx.split_endpoints(), - "file_path": - os.path.join(dirname, var.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self.role_maker._worker_index(), + "shape": var.shape, + "slice_shapes": [",".join([str(i) for i in var.shape])], + "slice_varnames": [var.name], + "remote_varnames": [var.name], + "is_sparse": False, + "endpoints": var_ctx.split_endpoints(), + "file_path": os.path.join(dirname, var.name), + }, + ) executor.run(prog) return local_vars @@ -467,7 +535,8 @@ class ParameterServerRuntime(RuntimeBase): optimizer = self._get_optimizer_op(varname) reshaped_varnames, origin_varnames = self._get_optimizer_status( - optimizer.type, varname) + optimizer.type, varname + ) var = self.origin_main_program.global_block().vars[varname] slice_shapes = [] @@ -476,37 +545,34 @@ class ParameterServerRuntime(RuntimeBase): for section in var_ctx.sections(): slice_shapes.append(str(section) + dims1) - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self.role_maker._worker_index(), - "shape": - var.shape, - "slice_shapes": - slice_shapes, - "slice_varnames": - var_ctx.split_varnames(), - "remote_varnames": - var_ctx.split_varnames(), - "is_sparse": - True, - "endpoints": - var_ctx.split_endpoints(), - "pserver_num": - len(self.role_maker._get_pserver_endpoints()), - "file_path": - os.path.join(dirname, var.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self.role_maker._worker_index(), + "shape": var.shape, + "slice_shapes": slice_shapes, + "slice_varnames": var_ctx.split_varnames(), + "remote_varnames": var_ctx.split_varnames(), + "is_sparse": True, + "endpoints": var_ctx.split_endpoints(), + "pserver_num": len( + self.role_maker._get_pserver_endpoints() + ), + "file_path": os.path.join(dirname, var.name), + }, + ) for reshaped_varname in reshaped_varnames: - var = self.origin_main_program.global_block( - ).vars[reshaped_varname] + var = self.origin_main_program.global_block().vars[ + reshaped_varname + ] slice_varnames = [] remote_varnames = [] for i in range(len(var_ctx.split_varnames())): - slice_varnames.append("{}.block{}".format( - reshaped_varname, i)) + slice_varnames.append( + "{}.block{}".format(reshaped_varname, i) + ) remote_varnames.append(reshaped_varname) block.append_op( @@ -519,32 +585,31 @@ class ParameterServerRuntime(RuntimeBase): "remote_varnames": remote_varnames, "is_sparse": True, "endpoints": var_ctx.split_endpoints(), - "pserver_num": - len(self.role_maker._get_pserver_endpoints()), - "file_path": os.path.join(dirname, var.name) - }) + "pserver_num": len( + self.role_maker._get_pserver_endpoints() + ), + "file_path": os.path.join(dirname, var.name), + }, + ) for origin_varname in origin_varnames: - var = self.origin_main_program.global_block( - ).vars[origin_varname] - - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self.role_maker._worker_index(), - "shape": - var.shape, - "slice_shapes": - [",".join([str(i) for i in var.shape])], - "slice_varnames": [origin_varname], - "remote_varnames": [origin_varname], - "is_sparse": - False, - "endpoints": - var_ctx.split_endpoints()[:1], - "file_path": - os.path.join(dirname, var.name) - }) + var = self.origin_main_program.global_block().vars[ + origin_varname + ] + + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self.role_maker._worker_index(), + "shape": var.shape, + "slice_shapes": [",".join([str(i) for i in var.shape])], + "slice_varnames": [origin_varname], + "remote_varnames": [origin_varname], + "is_sparse": False, + "endpoints": var_ctx.split_endpoints()[:1], + "file_path": os.path.join(dirname, var.name), + }, + ) executor.run(prog) return context.keys() @@ -553,58 +618,71 @@ class ParameterServerRuntime(RuntimeBase): block = prog.global_block() for name, var_ctx in context.items(): - block.append_op(type='checkpoint_notify', - attrs={ - "varname": name, - "mode": mode, - "slice_varnames": var_ctx.split_varnames(), - "remote_varnames": var_ctx.split_varnames(), - "endpoints": var_ctx.split_endpoints(), - "dirname": dirname - }) + block.append_op( + type='checkpoint_notify', + attrs={ + "varname": name, + "mode": mode, + "slice_varnames": var_ctx.split_varnames(), + "remote_varnames": var_ctx.split_varnames(), + "endpoints": var_ctx.split_endpoints(), + "dirname": dirname, + }, + ) executor.run(prog) return context.keys() - def _save_distributed_persistables(self, executor, dirname, main_program, - mode): + def _save_distributed_persistables( + self, executor, dirname, main_program, mode + ): dense_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=1, use_origin_program=True) + recv_type=1, use_origin_program=True + ) sparse_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=2, use_origin_program=True) + recv_type=2, use_origin_program=True + ) distributed_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=3, use_origin_program=True) + recv_type=3, use_origin_program=True + ) - recv_dense_varnames = self._save_dense_params(executor, dirname, - dense_ctx, main_program) + recv_dense_varnames = self._save_dense_params( + executor, dirname, dense_ctx, main_program + ) - recv_sparse_varnames = self._save_sparse_params(executor, dirname, - sparse_ctx, - main_program) + recv_sparse_varnames = self._save_sparse_params( + executor, dirname, sparse_ctx, main_program + ) recv_distributed_varnames = self._save_distributed_params( - executor, dirname, distributed_ctx, mode) + executor, dirname, distributed_ctx, mode + ) - saved_varnames = recv_dense_varnames + list( - recv_sparse_varnames) + list(recv_distributed_varnames) + saved_varnames = ( + recv_dense_varnames + + list(recv_sparse_varnames) + + list(recv_distributed_varnames) + ) remaining_vars = list( - filter(ParameterServerRuntime.__exclude_vars(saved_varnames), - main_program.list_vars())) - - fluid.io.save_vars(executor, - main_program=main_program, - dirname=dirname, - vars=remaining_vars) - - def _ps_inference_save_persistables(self, - executor, - dirname, - main_program=None, - mode=0, - **kwargs): + filter( + ParameterServerRuntime.__exclude_vars(saved_varnames), + main_program.list_vars(), + ) + ) + + fluid.io.save_vars( + executor, + main_program=main_program, + dirname=dirname, + vars=remaining_vars, + ) + + def _ps_inference_save_persistables( + self, executor, dirname, main_program=None, mode=0, **kwargs + ): """ This function filters out all variables with `persistable==True` from the give `main_program` and then saves these variables to the folder `dirname` @@ -634,16 +712,19 @@ class ParameterServerRuntime(RuntimeBase): "in fleet.save_persistables() function, main_program must be as Program type, CompiledProgram is not allowed" ) - self._save_distributed_persistables(executor, dirname, main_program, - mode) - - def _ps_inference_save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True): + self._save_distributed_persistables( + executor, dirname, main_program, mode + ) + + def _ps_inference_save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. @@ -664,14 +745,28 @@ class ParameterServerRuntime(RuntimeBase): raise TypeError( "in fleet.save_inference_model() function, main_program must be as Program type, CompiledProgram is not allowed" ) - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, main_program, - None, None, export_for_deployment) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + main_program, + None, + None, + export_for_deployment, + ) else: - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, - self.origin_main_program, None, None, - export_for_deployment, True) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + self.origin_main_program, + None, + None, + export_for_deployment, + True, + ) model_basename = "__model__" model_filename = os.path.join(dirname, model_basename) @@ -681,10 +776,9 @@ class ParameterServerRuntime(RuntimeBase): program = Program.parse_from_string(program_desc_str) program._copy_dist_param_info_from(fluid.default_main_program()) - self._ps_inference_save_persistables(executor, - dirname, - program, - mode=0) + self._ps_inference_save_persistables( + executor, dirname, program, mode=0 + ) def _save_inference_model(self, *args, **kwargs): self._ps_inference_save_inference_model(*args, **kwargs) diff --git a/python/paddle/distributed/fleet/runtime/runtime_base.py b/python/paddle/distributed/fleet/runtime/runtime_base.py index 38bb31ce3fc..2e8bacfbc3b 100644 --- a/python/paddle/distributed/fleet/runtime/runtime_base.py +++ b/python/paddle/distributed/fleet/runtime/runtime_base.py @@ -16,7 +16,6 @@ __all__ = [] class RuntimeBase(object): - def __init__(self): pass diff --git a/python/paddle/distributed/fleet/runtime/the_one_ps.py b/python/paddle/distributed/fleet/runtime/the_one_ps.py index e9765c9e2e6..5a0be9a1e01 100644 --- a/python/paddle/distributed/fleet/runtime/the_one_ps.py +++ b/python/paddle/distributed/fleet/runtime/the_one_ps.py @@ -35,8 +35,12 @@ PSERVER_SAVE_SUFFIX = ".shard" def parse_table_class(varname, o_main_program): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_distributed_sparse_op - from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_sparse_op + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_distributed_sparse_op, + ) + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_sparse_op, + ) for op in o_main_program.global_block().ops: if not is_distributed_sparse_op(op) and not is_sparse_op(op): @@ -44,7 +48,11 @@ def parse_table_class(varname, o_main_program): param_name = op.input("W")[0] - if param_name == varname and op.type == "lookup_table" or op.type == "lookup_table_v2": + if ( + param_name == varname + and op.type == "lookup_table" + or op.type == "lookup_table_v2" + ): if op.has_attr('table_class') and op.attr("table_class") != "none": return op.attr('table_class') else: @@ -90,7 +98,10 @@ def get_default_accessor_proto(accessor, varname, o_main_program): for sgd_param in [accessor.embed_sgd_param, accessor.embedx_sgd_param]: if not sgd_param.HasField("name"): sgd_param.name = "SparseAdaGradSGDRule" - if sgd_param.name == "SparseAdaGradSGDRule" or sgd_param.name == "StdAdaGradSGDRule": + if ( + sgd_param.name == "SparseAdaGradSGDRule" + or sgd_param.name == "StdAdaGradSGDRule" + ): if not sgd_param.adagrad.HasField("learning_rate"): sgd_param.adagrad.learning_rate = 0.05 if not sgd_param.adagrad.HasField("initial_g2sum"): @@ -130,17 +141,20 @@ def check_embedding_dim(accessor, varname, o_main_program): fea_dim = accessor.fea_dim if fea_dim != embedding_dim: raise ValueError( - "The fea_dim is wrong, it will be sparse_embedding_dim: {}, but got {}" - .format(embedding_dim, fea_dim)) + "The fea_dim is wrong, it will be sparse_embedding_dim: {}, but got {}".format( + embedding_dim, fea_dim + ) + ) embedx_dim = accessor.embedx_dim if embedx_dim != embedding_dim - 3: raise ValueError( - "The embedx_dim is wrong, it will be sparse_embedding_dim - 3: {}, but got {}" - .format(embedding_dim - 3, embedx_dim)) + "The embedx_dim is wrong, it will be sparse_embedding_dim - 3: {}, but got {}".format( + embedding_dim - 3, embedx_dim + ) + ) class Accessor: - def __init__(self): self.accessor_class = "" self.optimizer = None @@ -157,12 +171,12 @@ class Accessor: attrs += "\n" if self.optimizer is not None: attrs += self.optimizer.to_string(indent) - return accessor_str.format(conv_indent(indent), attrs, - conv_indent(indent)) + return accessor_str.format( + conv_indent(indent), attrs, conv_indent(indent) + ) class CommonAccessor: - def __init__(self): self.accessor_class = "" self.table_name = None @@ -183,26 +197,45 @@ class CommonAccessor: def define_optimize_map(self): opt_input_map = {} opt_input_map["sgd"] = [("Param", None), ("LearningRate", 1)] - opt_input_map["adam"] = [("Param", None), ("Moment1", None), - ("Moment2", None), ("Beta1Pow", 1), - ("Beta2Pow", 1), ("LearningRate", 1)] - opt_input_map["adam_d2sum"] = [("Param", None), ("D2Sum", None), - ("G2Sum", None), ("Moment", None), - ("MomentDecayRate", 1), - ("AdaDecayRate", 1), ("AdaEpsilon", 1), - ("LearningRate", 1)] + opt_input_map["adam"] = [ + ("Param", None), + ("Moment1", None), + ("Moment2", None), + ("Beta1Pow", 1), + ("Beta2Pow", 1), + ("LearningRate", 1), + ] + opt_input_map["adam_d2sum"] = [ + ("Param", None), + ("D2Sum", None), + ("G2Sum", None), + ("Moment", None), + ("MomentDecayRate", 1), + ("AdaDecayRate", 1), + ("AdaEpsilon", 1), + ("LearningRate", 1), + ] opt_input_map["sum"] = [("Param", None)] - opt_input_map["naive_adagrad"] = [("Param", None), ("G2Sum", 1), - ("LearningRate", 1)] + opt_input_map["naive_adagrad"] = [ + ("Param", None), + ("G2Sum", 1), + ("LearningRate", 1), + ] opt_attr_map = {} opt_attr_map["sgd"] = [] opt_attr_map["sum"] = [] opt_attr_map["naive_adagrad"] = [] - opt_attr_map["adam"] = [("beta1", "f"), ("beta2", "f"), - ("epsilon", "f")] - opt_attr_map["adam_d2sum"] = [("beta1", "f"), ("beta2", "f"), - ("epsilon", "f")] + opt_attr_map["adam"] = [ + ("beta1", "f"), + ("beta2", "f"), + ("epsilon", "f"), + ] + opt_attr_map["adam_d2sum"] = [ + ("beta1", "f"), + ("beta2", "f"), + ("epsilon", "f"), + ] opt_init_map = {} opt_init_map["gaussian_random"] = ["seed", "mean", "std"] @@ -215,8 +248,12 @@ class CommonAccessor: self.opt_init_map = opt_init_map def parse_entry(self, varname, o_main_program): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_distributed_sparse_op - from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_sparse_op + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_distributed_sparse_op, + ) + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_sparse_op, + ) for op in o_main_program.global_block().ops: if not is_distributed_sparse_op(op) and not is_sparse_op(op): @@ -250,8 +287,10 @@ class CommonAccessor: origin_var_name = value_name for op in o_startup_program.global_block().ops: - if op.type in self.opt_init_map.keys( - ) and origin_var_name == op.output("Out")[0]: + if ( + op.type in self.opt_init_map.keys() + and origin_var_name == op.output("Out")[0] + ): init_attr = [op.type] for attr in self.opt_init_map[op.type]: init_attr.append(str(op.attr(attr))) @@ -259,9 +298,19 @@ class CommonAccessor: break return attr_str - def parse_by_optimizer(self, grad_name, is_sparse, size, single_dim, - compiled_strategy, adam_d2sum): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops + def parse_by_optimizer( + self, + grad_name, + is_sparse, + size, + single_dim, + compiled_strategy, + adam_d2sum, + ): + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, + ) + param_name = compiled_strategy.grad_name_to_param_name[grad_name] main_program, startup_program = compiled_strategy.get_origin_programs() pserver_id = compiled_strategy.get_role_id() @@ -270,8 +319,9 @@ class CommonAccessor: oop = None for op in optimizer_ops: - if ("Param" in op.input_names) and (op.input("Param")[0] - == param_name): + if ("Param" in op.input_names) and ( + op.input("Param")[0] == param_name + ): oop = op break @@ -311,7 +361,7 @@ class CommonAccessor: for (formal_name, shape) in param_varnames: params.append(formal_name) if self.accessor_class == "adam_d2sum": - #for dims + # for dims if shape is None: if is_sparse: shape = single_dim @@ -319,18 +369,24 @@ class CommonAccessor: shape = self.get_shard(size, pserver_num, pserver_id) dims.append(shape) - #for initializers + # for initializers if formal_name == "Param" or formal_name == "LearningRate": - param = main_program.global_block().vars[oop.input( - formal_name)[0]] - #TODO: for dense learning_rate, can be different from sparse lr - if formal_name == "LearningRate" and param.name != "learning_rate_0": + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] + # TODO: for dense learning_rate, can be different from sparse lr + if ( + formal_name == "LearningRate" + and param.name != "learning_rate_0" + ): warnings.warn("will support decay soon") - param = main_program.global_block( - ).vars["learning_rate_0"] + param = main_program.global_block().vars[ + "learning_rate_0" + ] initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) elif formal_name == "MomentDecayRate": initializer = "fill_constant&0.99" elif formal_name == "AdaDecayRate": @@ -346,23 +402,30 @@ class CommonAccessor: initializer = "fill_constant&0" initializers.append(initializer) else: - param = main_program.global_block().vars[oop.input( - formal_name)[0]] - if formal_name == "LearningRate" and param.name != "learning_rate_0": + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] + if ( + formal_name == "LearningRate" + and param.name != "learning_rate_0" + ): warnings.warn("will support decay soon") - param = main_program.global_block( - ).vars["learning_rate_0"] + param = main_program.global_block().vars[ + "learning_rate_0" + ] if shape is None: if is_sparse: shape = single_dim else: - shape = self.get_shard(size, pserver_num, - pserver_id) + shape = self.get_shard( + size, pserver_num, pserver_id + ) dims.append(shape) initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) initializers.append(initializer) for (attr_varname, type_) in attr_varnames: @@ -401,12 +464,12 @@ class CommonAccessor: attrs += "initializers: \"{}\" ".format(initializer) attrs += "\n" - return accessor_str.format(conv_indent(indent), attrs, - conv_indent(indent)) + return accessor_str.format( + conv_indent(indent), attrs, conv_indent(indent) + ) class Tensor: - def __init__(self): self.main_program_id = None self.startup_program_id = None @@ -422,14 +485,15 @@ class Tensor: attrs += "startup_program_id: {} ".format(str(self.startup_program_id)) attrs += "main_program_id: {} ".format(str(self.main_program_id)) attrs += "tensor_table_class: \"{}\" ".format( - str(self.tensor_table_class)) + str(self.tensor_table_class) + ) attrs += "\n" - return program_str.format(conv_indent(indent), attrs, - conv_indent(indent)) + return program_str.format( + conv_indent(indent), attrs, conv_indent(indent) + ) class Table: - def __init__(self): self.id = -1 self.table_class = None @@ -458,9 +522,9 @@ class Table: if self.accessor_proto is not None: accessor_str = "{}accessor {{{}\n{}}}" - accessor_str = accessor_str.format(conv_indent(indent), - self.accessor_proto, - conv_indent(indent)) + accessor_str = accessor_str.format( + conv_indent(indent), self.accessor_proto, conv_indent(indent) + ) attrs += accessor_str + "\n" elif self.accessor is not None: attrs += self.accessor.to_string(indent) @@ -478,7 +542,6 @@ class Table: class Service: - def __init__(self): self.server_class = "BrpcPsServer" self.client_class = "BrpcPsClient" @@ -496,12 +559,12 @@ class Service: attrs += "start_server_port: {} ".format(self.start_server_port) attrs += "server_thread_num: {} ".format(self.server_thread_num) - return service_str.format(conv_indent(indent), attrs, - conv_indent(indent)) + return service_str.format( + conv_indent(indent), attrs, conv_indent(indent) + ) class DownpourServer: - def __init__(self): self.service = None self.tables = [] @@ -526,12 +589,12 @@ class DownpourServer: for table in self.tables: table_strs += "\n" table_strs += table.to_string(indent) - return server_str.format(conv_indent(indent), table_strs, - conv_indent(indent)) + return server_str.format( + conv_indent(indent), table_strs, conv_indent(indent) + ) class Server: - def __init__(self): self.servers = [] @@ -552,7 +615,6 @@ class Server: class DownpourWorker: - def __init__(self): self.tables = [] @@ -569,12 +631,12 @@ class DownpourWorker: table_strs += "\n" table_strs += table.to_string(indent) - return worker_str.format(conv_indent(indent), table_strs, - conv_indent(indent)) + return worker_str.format( + conv_indent(indent), table_strs, conv_indent(indent) + ) class Worker: - def __init__(self): self.workers = [] @@ -595,7 +657,6 @@ class Worker: class fsClient: - def __init__(self, proto): self.proto = proto self.uri = proto.uri @@ -605,6 +666,7 @@ class fsClient: def to_string(self): from google.protobuf import text_format + proto_txt = text_format.MessageToString(self.proto) if proto_txt: fs_str = "fs_client_param {{\n{}}}" @@ -614,7 +676,6 @@ class fsClient: class TheOnePSRuntime(RuntimeBase): - def __init__(self): super(TheOnePSRuntime, self).__init__() self._communicator = None @@ -634,8 +695,9 @@ class TheOnePSRuntime(RuntimeBase): def _get_distributed_strategy(self): strategy = None - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import \ - StrategyFactory + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, + ) dist_strategy = self.context["valid_strategy"] k_steps = dist_strategy.a_sync_configs["k_steps"] @@ -657,19 +719,24 @@ class TheOnePSRuntime(RuntimeBase): return strategy def build_compiled_startegy(self): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import CompileTimeStrategy - - compiled_config = CompileTimeStrategy(self.origin_main_program, - self.origin_main_program, - self.async_strategy, - self.role_maker) + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + CompileTimeStrategy, + ) + + compiled_config = CompileTimeStrategy( + self.origin_main_program, + self.origin_main_program, + self.async_strategy, + self.role_maker, + ) if self.async_strategy.use_ps_gpu: compiled_config.use_ps_gpu = True return compiled_config def _init_worker(self): - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import \ - SyncStrategy + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + SyncStrategy, + ) is_sync = self.compiled_strategy.is_sync_mode() worker = self._get_fleet_proto(is_server=False, is_sync=is_sync) @@ -690,7 +757,8 @@ class TheOnePSRuntime(RuntimeBase): def sync_strategy_envs(): kwargs = {} kwargs[ - "pserver_endpoints"] = self.role_maker._get_pserver_endpoints() + "pserver_endpoints" + ] = self.role_maker._get_pserver_endpoints() kwargs["trainer_id"] = self.role_maker._worker_index() return kwargs @@ -712,11 +780,13 @@ class TheOnePSRuntime(RuntimeBase): string_hosts.append(pshost.serialize_to_string()) dense_map = self.compiled_strategy.get_the_one_recv_context( - split_dense_table=self.role_maker._is_heter_parameter_server_mode) + split_dense_table=self.role_maker._is_heter_parameter_server_mode + ) send_ctx = self.compiled_strategy.get_the_one_send_context( split_dense_table=self.role_maker._is_heter_parameter_server_mode, use_origin_program=self.role_maker._is_heter_parameter_server_mode, - ep_list=endpoints) + ep_list=endpoints, + ) trainer_config = self.async_strategy.get_trainer_runtime_config() debug = bool(int(os.getenv("PSERVER_DEBUG", "0"))) @@ -732,7 +802,7 @@ class TheOnePSRuntime(RuntimeBase): kwargs['need_global_step'] = "0" kwargs["trainer_id"] = self.role_maker._role_id() kwargs["trainers"] = self.role_maker._worker_num() - #if self.role_maker._is_heter_worker(): + # if self.role_maker._is_heter_worker(): # kwargs["trainer_id"] += kwargs["trainers"] for table in server.servers[0].tables: @@ -745,13 +815,16 @@ class TheOnePSRuntime(RuntimeBase): kwargs.update(sync_kwargs) from paddle.fluid.communicator import Communicator, HeterClient + self._communicator = Communicator( - trainer_config.mode, kwargs, - trainer_config.get_communicator_flags()) - self._communicator.init_with_ctx(send_ctx, dense_map, proto_txt, - string_hosts, fluid.global_scope()) + trainer_config.mode, kwargs, trainer_config.get_communicator_flags() + ) + self._communicator.init_with_ctx( + send_ctx, dense_map, proto_txt, string_hosts, fluid.global_scope() + ) import paddle.distributed.fleet as fleet + fleet.util.barrier() info = self._communicator.get_client_info() if isinstance(info, list) and len(info) > 0: @@ -774,11 +847,14 @@ class TheOnePSRuntime(RuntimeBase): is_test = bool(int(os.getenv("TEST_MODE", "0"))) - if self.role_maker._is_first_worker( - ) and self.role_maker._is_heter_parameter_server_mode: + if ( + self.role_maker._is_first_worker() + and self.role_maker._is_heter_parameter_server_mode + ): # for ps-heter mode load all parameters on first_worker init_params = self.compiled_strategy.get_the_one_recv_context( - split_dense_table=True, use_origin_program=True) + split_dense_table=True, use_origin_program=True + ) else: init_params = dense_map @@ -798,8 +874,10 @@ class TheOnePSRuntime(RuntimeBase): if launch_barrier and launch_barrier_flag: # for trainer wait server ready wait_server_ready(self.role_maker._get_pserver_endpoints()) - if self.role_maker._is_heter_parameter_server_mode and self.role_maker._get_next_trainers( - ) != []: + if ( + self.role_maker._is_heter_parameter_server_mode + and self.role_maker._get_next_trainers() != [] + ): wait_server_ready(self.role_maker._get_next_trainers()) if self.role_maker._is_heter_parameter_server_mode: previous_trainers = [] @@ -808,14 +886,13 @@ class TheOnePSRuntime(RuntimeBase): next_trainers = [] if self.role_maker._get_next_trainers() != []: next_trainers = self.role_maker._get_next_trainers() - self._heter_client = HeterClient(next_trainers, - previous_trainers, - self.role_maker._role_id()) - - def _push_sparse_param(self, - var_name, - table_id=-1, - scope=fluid.global_scope()): + self._heter_client = HeterClient( + next_trainers, previous_trainers, self.role_maker._role_id() + ) + + def _push_sparse_param( + self, var_name, table_id=-1, scope=fluid.global_scope() + ): self._communicator.push_sparse_param(var_name, table_id, scope) def _get_executor(self): @@ -825,20 +902,23 @@ class TheOnePSRuntime(RuntimeBase): heter_device_type = self.role_maker._heter_device_type().upper() if heter_device_type not in ["GPU", "XPU", "CPU"]: raise ValueError( - "Heter Worker Not Support Device {}".format( - device_type)) + "Heter Worker Not Support Device {}".format(device_type) + ) if heter_device_type == "GPU": executor = Executor( fluid.CUDAPlace( - int(os.getenv("FLAGS_selected_gpus", "0")))) + int(os.getenv("FLAGS_selected_gpus", "0")) + ) + ) elif heter_device_type == "XPU": executor = Executor( fluid.XPUPlace( - int(os.getenv("FLAGS_selected_xpus", "0")))) + int(os.getenv("FLAGS_selected_xpus", "0")) + ) + ) return executor def _get_fleet_proto(self, is_server, is_sync, **kwargs): - def _build_merge_accessor(ctx): accessor = Accessor() accessor.accessor_class = "CommMergeAccessor" @@ -872,7 +952,8 @@ class TheOnePSRuntime(RuntimeBase): trainer_num = self.compiled_strategy.get_trainers() if self.role_maker._is_heter_parameter_server_mode: trainer_num += len( - self.role_maker._get_heter_worker_endpoints()) + self.role_maker._get_heter_worker_endpoints() + ) common.trainer_num = trainer_num common.attrs = "" common.dims = [] @@ -918,27 +999,32 @@ class TheOnePSRuntime(RuntimeBase): for table_name in tensor_table_dict: if tensor_table_dict[table_name]["startup_program"] != None: tensor_table_dict[table_name][ - "startup_program_id"] = program_idx + "startup_program_id" + ] = program_idx self._server_sub_program.append( - tensor_table_dict[table_name]["startup_program"].desc) + tensor_table_dict[table_name]["startup_program"].desc + ) program_idx += 1 if tensor_table_dict[table_name]["main_program"] != None: tensor_table_dict[table_name][ - "main_program_id"] = program_idx + "main_program_id" + ] = program_idx self._server_sub_program.append( - tensor_table_dict[table_name]["main_program"].desc) + tensor_table_dict[table_name]["main_program"].desc + ) program_idx += 1 # Todo: Hard code for lr_decay table apply table id - new_table = _build_tensor_table(len(tables), - tensor_table_dict[table_name]) + new_table = _build_tensor_table( + len(tables), tensor_table_dict[table_name] + ) tables.append(new_table) return tables def _get_tables(): send_ctx = self.compiled_strategy.get_the_one_send_context( use_origin_program=True, - split_dense_table=self.role_maker. - _is_heter_parameter_server_mode) + split_dense_table=self.role_maker._is_heter_parameter_server_mode, + ) tables = [] for idx, (name, ctx) in enumerate(send_ctx.items()): @@ -953,14 +1039,18 @@ class TheOnePSRuntime(RuntimeBase): table.type = "PS_SPARSE_TABLE" table.shard_num = 256 - common.table_name = self.compiled_strategy.grad_name_to_param_name[ - ctx.origin_varnames()[0]] + common.table_name = ( + self.compiled_strategy.grad_name_to_param_name[ + ctx.origin_varnames()[0] + ] + ) if self.compiled_strategy.is_geo_mode(): table.table_class = "MemorySparseGeoTable" else: all_table_proto = self.context[ - "user_defined_strategy"].sparse_table_configs + "user_defined_strategy" + ].sparse_table_configs table_proto = all_table_proto.add() for proto in all_table_proto: if proto.table_name == common.table_name: @@ -970,11 +1060,13 @@ class TheOnePSRuntime(RuntimeBase): table.table_class = table_proto.table_class else: table.table_class = parse_table_class( - common.table_name, self.origin_main_program) + common.table_name, self.origin_main_program + ) if table.table_class != 'MemorySparseTable': table.table_class = 'MemorySparseTable' warnings.warn( - "The PS mode must use MemorySparseTable.") + "The PS mode must use MemorySparseTable." + ) if table_proto.HasField("shard_num"): table.shard_num = table_proto.shard_num @@ -988,15 +1080,21 @@ class TheOnePSRuntime(RuntimeBase): warnings.warn( "The accessor of sparse table is not set, use default value." ) - get_default_accessor_proto(table_proto.accessor, - common.table_name, - self.origin_main_program) - check_embedding_dim(table_proto.accessor, - common.table_name, - self.origin_main_program) + get_default_accessor_proto( + table_proto.accessor, + common.table_name, + self.origin_main_program, + ) + check_embedding_dim( + table_proto.accessor, + common.table_name, + self.origin_main_program, + ) from google.protobuf import text_format + table.accessor_proto = text_format.MessageToString( - table_proto.accessor) + table_proto.accessor + ) else: table.type = "PS_DENSE_TABLE" table.table_class = "MemoryDenseTable" @@ -1005,14 +1103,18 @@ class TheOnePSRuntime(RuntimeBase): adam_d2sum = self.context["user_defined_strategy"].adam_d2sum common.parse_by_optimizer( - ctx.origin_varnames()[0], ctx.is_sparse(), + ctx.origin_varnames()[0], + ctx.is_sparse(), ctx.sections()[0], ctx.sections()[1] if ctx.is_sparse() else 1, - self.compiled_strategy, adam_d2sum) + self.compiled_strategy, + adam_d2sum, + ) if ctx.is_sparse(): - common.parse_entry(common.table_name, - self.origin_main_program) + common.parse_entry( + common.table_name, self.origin_main_program + ) if is_sync: common.sync = "true" @@ -1071,7 +1173,8 @@ class TheOnePSRuntime(RuntimeBase): server = self._get_fleet_proto(is_server=True, is_sync=is_sync) proto_txt = str(server) fs_client = fsClient( - self.context["user_defined_strategy"].fs_client_param) + self.context["user_defined_strategy"].fs_client_param + ) proto_txt = proto_txt + "\n" + fs_client.to_string() debug = bool(int(os.getenv("PSERVER_DEBUG", "0"))) @@ -1085,10 +1188,13 @@ class TheOnePSRuntime(RuntimeBase): string_hosts.append(pshost.serialize_to_string()) self._server = fluid.core.DistFleetWrapper() - self._server.init_server(proto_txt, string_hosts, role_id, trainers, - self._server_sub_program) + self._server.init_server( + proto_txt, string_hosts, role_id, trainers, self._server_sub_program + ) - from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, + ) dist_varnames = get_sparse_tablenames(self.origin_main_program, True) sparse_varnames = get_sparse_tablenames(self.origin_main_program, False) @@ -1101,8 +1207,10 @@ class TheOnePSRuntime(RuntimeBase): for var_name in var_names: if var_name not in distributed_varnames: raise ValueError( - "fleet.init server can only load sparse variables in {}" - .format(distributed_varnames)) + "fleet.init server can only load sparse variables in {}".format( + distributed_varnames + ) + ) load_varnames = var_names if dirname is None or not load_varnames: @@ -1132,19 +1240,22 @@ class TheOnePSRuntime(RuntimeBase): def _stop_worker(self): self._communicator.stop() if self.role_maker._is_heter_parameter_server_mode: - assert self._heter_client != None, "heter client should not be None in heterps mode" + assert ( + self._heter_client != None + ), "heter client should not be None in heterps mode" self._heter_client.stop() - #executor = self._get_executor() - #executor.close() + # executor = self._get_executor() + # executor.close() @staticmethod def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, + ) origin_varname, _, _ = _get_varname_parts(var.name) if origin_varname.endswith("@GRAD"): @@ -1153,9 +1264,11 @@ class TheOnePSRuntime(RuntimeBase): if origin_varname == "learning_rate_0": return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -1168,11 +1281,16 @@ class TheOnePSRuntime(RuntimeBase): model_path = os.path.join(dirname, "dnn_plugin") return model_path - def _save_sparse_params(self, executor, dirname, context, main_program, - mode): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames + def _save_sparse_params( + self, executor, dirname, context, main_program, mode + ): + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, + ) + distributed_varnames = get_sparse_tablenames( - self.compiled_strategy.origin_main_program, True) + self.compiled_strategy.origin_main_program, True + ) values = [] model_path = self._get_inference_model_path(dirname) for id, names in context.items(): @@ -1188,23 +1306,24 @@ class TheOnePSRuntime(RuntimeBase): # self._worker.save_all_model(dirname, mode) return values - def _save_distributed_persistables(self, - executor, - dirname, - main_program, - mode=0): + def _save_distributed_persistables( + self, executor, dirname, main_program, mode=0 + ): denses = self.compiled_strategy.get_the_one_recv_context( is_dense=True, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) sparses = self.compiled_strategy.get_the_one_recv_context( is_dense=False, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) - sparse_varnames = self._save_sparse_params(executor, dirname, sparses, - main_program, mode) + sparse_varnames = self._save_sparse_params( + executor, dirname, sparses, main_program, mode + ) recv_dense_varnames = [] for id, names in denses.items(): @@ -1214,24 +1333,25 @@ class TheOnePSRuntime(RuntimeBase): saved_varnames = sparse_varnames remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(saved_varnames), - main_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(saved_varnames), + main_program.list_vars(), + ) + ) import paddle + for var in remaining_vars: # if var.name not in recv_dense_varnames: # continue tensor = var.get_value() - paddle.save(tensor, - os.path.join(dirname, var.name), - use_binary_format=True) - - def _ps_inference_save_persistables(self, - executor, - dirname, - main_program=None, - mode=0, - **kwargs): + paddle.save( + tensor, os.path.join(dirname, var.name), use_binary_format=True + ) + + def _ps_inference_save_persistables( + self, executor, dirname, main_program=None, mode=0, **kwargs + ): """ This function filters out all variables with `persistable==True` from the give `main_program` and then saves these variables to the folder `dirname` @@ -1250,7 +1370,8 @@ class TheOnePSRuntime(RuntimeBase): if not isinstance(executor, Executor): raise TypeError( - "in fleet.save() function, executor must be as Executor type") + "in fleet.save() function, executor must be as Executor type" + ) if main_program is None: main_program = self.compiled_strategy.get_origin_ps_main_program() @@ -1265,14 +1386,16 @@ class TheOnePSRuntime(RuntimeBase): # mode) self._worker.save_all_model(dirname, mode) - def _ps_inference_save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True, - mode=0): + def _ps_inference_save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + mode=0, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. @@ -1285,10 +1408,14 @@ class TheOnePSRuntime(RuntimeBase): if not isinstance(executor, Executor): raise TypeError( - "in fleet.save() function, executor must be as Executor type") + "in fleet.save() function, executor must be as Executor type" + ) import paddle - program = self.origin_main_program if main_program is None else main_program + + program = ( + self.origin_main_program if main_program is None else main_program + ) if isinstance(program, CompiledProgram): raise TypeError( @@ -1299,8 +1426,9 @@ class TheOnePSRuntime(RuntimeBase): program.global_block().var(name) for name in feeded_var_names ] - infer_program = paddle.static.normalize_program(program, feed_vars, - target_vars) + infer_program = paddle.static.normalize_program( + program, feed_vars, target_vars + ) infer_program._copy_dist_param_info_from(program) @@ -1312,29 +1440,38 @@ class TheOnePSRuntime(RuntimeBase): sparses = self.compiled_strategy.get_the_one_recv_context( is_dense=False, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) - sparse_names = self._save_sparse_params(executor, dirname, sparses, - main_program, mode) + use_origin_program=True, + ) + sparse_names = self._save_sparse_params( + executor, dirname, sparses, main_program, mode + ) denses = self.compiled_strategy.get_the_one_recv_context( is_dense=True, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) # TODO(zhaocaibei123): for GEO: should call GeoCommunicator::RecvDense self._communicator.pull_dense(denses) generate_vars = self.context[ - "user_defined_strategy"].trainer_desc_configs["stat_var_names"] + "user_defined_strategy" + ].trainer_desc_configs["stat_var_names"] generate_vars = [var for var in generate_vars] remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(sparse_names), - infer_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(sparse_names), + infer_program.list_vars(), + ) + ) for var in remaining_vars: tensor = var.get_value() - paddle.save(tensor, - os.path.join(model_path, var.name), - use_binary_format=True) + paddle.save( + tensor, + os.path.join(model_path, var.name), + use_binary_format=True, + ) def _save_inference_model(self, *args, **kwargs): self._ps_inference_save_inference_model(*args, **kwargs) @@ -1343,9 +1480,13 @@ class TheOnePSRuntime(RuntimeBase): self._ps_inference_save_persistables(*args, **kwargs) def _load_sparse_params(self, dirname, context, main_program, mode): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, + ) + distributed_varnames = get_sparse_tablenames( - self.compiled_strategy.origin_main_program, True) + self.compiled_strategy.origin_main_program, True + ) values = [] for id, names in context.items(): if names[0] not in distributed_varnames: @@ -1356,10 +1497,9 @@ class TheOnePSRuntime(RuntimeBase): values.extend(names) return values - def _ps_inference_load_inference_model(self, - dirname, - mode=0, - main_program=None): + def _ps_inference_load_inference_model( + self, dirname, mode=0, main_program=None + ): if main_program is None: main_program = self.compiled_strategy.get_origin_ps_main_program() @@ -1371,14 +1511,17 @@ class TheOnePSRuntime(RuntimeBase): denses = self.compiled_strategy.get_the_one_recv_context( is_dense=True, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) sparses = self.compiled_strategy.get_the_one_recv_context( is_dense=False, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) - sparse_varnames = self._load_sparse_params(dirname, sparses, - main_program, mode) + sparse_varnames = self._load_sparse_params( + dirname, sparses, main_program, mode + ) recv_dense_varnames = [] for id, names in denses.items(): @@ -1387,14 +1530,18 @@ class TheOnePSRuntime(RuntimeBase): loaded_varnames = sparse_varnames remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(loaded_varnames), - main_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(loaded_varnames), + main_program.list_vars(), + ) + ) if dirname.startswith("afs:") or dirname.startswith("hdfs:"): model_path = "./dnn_plugin" else: model_path = os.path.join(dirname, "dnn_plugin") import paddle + for var in remaining_vars: if var.name not in recv_dense_varnames: continue @@ -1421,13 +1568,14 @@ class TheOnePSRuntime(RuntimeBase): else: threshold = 0 import paddle.distributed.fleet as fleet + fleet.util.barrier() if self.role_maker._is_first_worker(): sparses = self.compiled_strategy.get_the_one_recv_context( is_dense=False, - split_dense_table=self.role_maker. - _is_heter_parameter_server_mode, - use_origin_program=True) + split_dense_table=self.role_maker._is_heter_parameter_server_mode, + use_origin_program=True, + ) for id, names in sparses.items(): self._worker.shrink_sparse_table(id, threshold) diff --git a/python/paddle/distributed/fleet/scaler.py b/python/paddle/distributed/fleet/scaler.py index 583c2819d8d..60bc8201680 100644 --- a/python/paddle/distributed/fleet/scaler.py +++ b/python/paddle/distributed/fleet/scaler.py @@ -23,12 +23,12 @@ from paddle import _legacy_C_ops def distributed_scaler(scaler): - def unscale_method(self, optimizer): if not self._enable: return if getattr(optimizer, '_param_groups', None) and isinstance( - optimizer._param_groups[0], dict): + optimizer._param_groups[0], dict + ): param_grads = [] param_grads_fp16 = [] param_grads_fp32 = [] @@ -36,38 +36,47 @@ def distributed_scaler(scaler): for param in group['params']: if param._grad_ivar() is not None: param_grads.append(param._grad_ivar()) - if param._grad_ivar( - ).dtype == core.VarDesc.VarType.FP16: + if ( + param._grad_ivar().dtype + == core.VarDesc.VarType.FP16 + ): param_grads_fp16.append(param._grad_ivar()) else: param_grads_fp32.append(param._grad_ivar()) else: param_grads = [ - param._grad_ivar() for param in optimizer._parameter_list + param._grad_ivar() + for param in optimizer._parameter_list if param._grad_ivar() is not None ] param_grads_fp16 = [ - param._grad_ivar() for param in optimizer._parameter_list - if (param._grad_ivar() is not None) and ( - param._grad_ivar().dtype == core.VarDesc.VarType.FP16) + param._grad_ivar() + for param in optimizer._parameter_list + if (param._grad_ivar() is not None) + and (param._grad_ivar().dtype == core.VarDesc.VarType.FP16) ] param_grads_fp32 = [ - param._grad_ivar() for param in optimizer._parameter_list - if (param._grad_ivar() is not None) and ( - param._grad_ivar().dtype == core.VarDesc.VarType.FP32) + param._grad_ivar() + for param in optimizer._parameter_list + if (param._grad_ivar() is not None) + and (param._grad_ivar().dtype == core.VarDesc.VarType.FP32) ] temp_found_inf_fp16 = to_variable(np.array([0]).astype(np.bool_)) temp_found_inf_fp32 = to_variable(np.array([0]).astype(np.bool_)) if len(param_grads_fp16): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp16, - self._scale, - param_grads_fp16, - temp_found_inf_fp16) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp16, + self._scale, + param_grads_fp16, + temp_found_inf_fp16, + ) if len(param_grads_fp32): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp32, - self._scale, - param_grads_fp32, - temp_found_inf_fp32) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp32, + self._scale, + param_grads_fp32, + temp_found_inf_fp32, + ) self._found_inf = 1 if temp_found_inf_fp16 or temp_found_inf_fp32 else 0 is_found_inf = paddle.to_tensor([self._found_inf], dtype="int32") @@ -75,9 +84,9 @@ def distributed_scaler(scaler): # TODO(shenliang03) Since dp allreduce in the optimizer is # after the gradscaler, check_finite needs to synchronize global # information. In the future, we should use check_group to speed. - paddle.distributed.all_reduce(is_found_inf, - op=paddle.distributed.ReduceOp.MAX, - group=None) + paddle.distributed.all_reduce( + is_found_inf, op=paddle.distributed.ReduceOp.MAX, group=None + ) self._found_inf = is_found_inf.numpy()[0] # Only data_parallel doesn't need to modify scaler diff --git a/python/paddle/distributed/fleet/utils/__init__.py b/python/paddle/distributed/fleet/utils/__init__.py index 93fc890d05a..340370e5e55 100644 --- a/python/paddle/distributed/fleet/utils/__init__.py +++ b/python/paddle/distributed/fleet/utils/__init__.py @@ -22,14 +22,14 @@ import paddle from . import log_util # noqa: F401 from . import hybrid_parallel_util # noqa: F401 -__all__ = [ #noqa - "LocalFS", "recompute", "DistributedInfer", "HDFSClient" -] +__all__ = ["LocalFS", "recompute", "DistributedInfer", "HDFSClient"] # noqa -@deprecated(since="2.4.0", - update_to="paddle.distributed.fleet.recompute", - level=1, - reason="Please use new recompute API(fleet.recompute) ") +@deprecated( + since="2.4.0", + update_to="paddle.distributed.fleet.recompute", + level=1, + reason="Please use new recompute API(fleet.recompute) ", +) def recompute(function, *args, **kwargs): return fleet.recompute.recompute(function, *args, **kwargs) diff --git a/python/paddle/distributed/fleet/utils/fs.py b/python/paddle/distributed/fleet/utils/fs.py index cf0a6475d0e..9daddcdb856 100644 --- a/python/paddle/distributed/fleet/utils/fs.py +++ b/python/paddle/distributed/fleet/utils/fs.py @@ -47,7 +47,6 @@ class FSShellCmdAborted(ExecuteError): class FS(object): - @abc.abstractmethod def ls_dir(self, fs_path): raise NotImplementedError @@ -171,7 +170,8 @@ class LocalFS(FS): client.delete("test_mkdirs") """ assert not os.path.isfile(fs_path), "{} is already a file".format( - fs_path) + fs_path + ) os.system("mkdir -p {}".format(fs_path)) def rename(self, fs_src_path, fs_dst_path): @@ -379,9 +379,7 @@ class LocalFS(FS): def _handle_errors(max_time_out=None): - def decorator(f): - @functools.wraps(f) def handler(*args, **kwargs): o = args[0] @@ -400,16 +398,20 @@ def _handle_errors(max_time_out=None): # important: only ExecuteError need to retry except ExecuteError as e: if time.time() - start >= time_out: - raise FSTimeOut("args:{} timeout:{}".format( - args, - time.time() - start)) + raise FSTimeOut( + "args:{} timeout:{}".format( + args, time.time() - start + ) + ) time.sleep(inter) if time.time() - last_print_time > 30: - print("hadoop operator timeout:args:{} timeout:{}".format( - args, - time.time() - start)) + print( + "hadoop operator timeout:args:{} timeout:{}".format( + args, time.time() - start + ) + ) last_print_time = time.time() return handler @@ -443,11 +445,12 @@ class HDFSClient(FS): """ def __init__( - self, - hadoop_home, - configs, - time_out=5 * 60 * 1000, # ms - sleep_inter=1000): # ms + self, + hadoop_home, + configs, + time_out=5 * 60 * 1000, # ms + sleep_inter=1000, + ): # ms self.pre_commands = [] hadoop_bin = '%s/bin/hadoop' % hadoop_home self.pre_commands.append(hadoop_bin) @@ -463,7 +466,8 @@ class HDFSClient(FS): self._sleep_inter = sleep_inter self._base_cmd = " ".join(self.pre_commands) self._bd_err_re = re.compile( - r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:') + r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:' + ) def _run_cmd(self, cmd, redirect_stderr=False, retry_times=5): exe_cmd = "{} -{}".format(self._base_cmd, cmd) @@ -771,8 +775,9 @@ class HDFSClient(FS): procs = [] for i in range(multi_processes): process_datas = self._split_files(all_files, i, multi_processes) - p = multiprocessing.Process(target=__subprocess_upload, - args=(fs_path, process_datas)) + p = multiprocessing.Process( + target=__subprocess_upload, args=(fs_path, process_datas) + ) procs.append(p) p.start() @@ -841,8 +846,9 @@ class HDFSClient(FS): procs = [] for i in range(multi_processes): process_datas = self._split_files(all_files, i, multi_processes) - p = multiprocessing.Process(target=__subprocess_download, - args=(local_path, process_datas)) + p = multiprocessing.Process( + target=__subprocess_download, args=(local_path, process_datas) + ) procs.append(p) p.start() @@ -938,7 +944,8 @@ class HDFSClient(FS): if test_exists: if not self.is_exist(fs_src_path): raise FSFileNotExistsError( - "{} is not exists".format(fs_src_path)) + "{} is not exists".format(fs_src_path) + ) if self.is_exist(fs_dst_path): raise FSFileExistsError("{} exists already".format(fs_dst_path)) @@ -954,8 +961,7 @@ class HDFSClient(FS): if ret != 0: raise ExecuteError(cmd) except Exception as e: - if not self.is_exist(fs_src_path) and \ - self.is_exist(fs_dst_path): + if not self.is_exist(fs_src_path) and self.is_exist(fs_dst_path): return raise e @@ -1103,7 +1109,7 @@ class HDFSClient(FS): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] @@ -1121,13 +1127,15 @@ class HDFSClient(FS): file_list = [] - #concat filelist can speed up 'hadoop ls' + # concat filelist can speed up 'hadoop ls' str_concat = "" for path in path_list: str_concat += path + " " - cmd = "ls " + str_concat + " | awk '{if ($8 != \"\") {print $5\" \"$8 }}'" + cmd = ( + "ls " + str_concat + " | awk '{if ($8 != \"\") {print $5\" \"$8 }}'" + ) ret, lines = self._run_cmd(cmd) - if (len(lines) == 0): + if len(lines) == 0: logger.warning("list_files empty, path[%s]" % path_list) return [] for line in lines: @@ -1155,10 +1163,7 @@ class AFSClient(FS): client.ls_dir("hdfs:/test_hdfs_client") """ - def __init__( - self, - time_out=5 * 60 * 1000, # ms - sleep_inter=1000): # ms + def __init__(self, time_out=5 * 60 * 1000, sleep_inter=1000): # ms # ms self._fs = core.AfsWrapper() self._time_out = time_out @@ -1392,8 +1397,9 @@ class AFSClient(FS): procs = [] for i in range(multi_processes): process_datas = self._split_files(all_files, i, multi_processes) - p = multiprocessing.Process(target=__subprocess_download, - args=(local_path, process_datas)) + p = multiprocessing.Process( + target=__subprocess_download, args=(local_path, process_datas) + ) procs.append(p) p.start() @@ -1448,7 +1454,8 @@ class AFSClient(FS): if test_exists: if not self.is_exist(fs_src_path): raise FSFileNotExistsError( - "{} is not exists".format(fs_src_path)) + "{} is not exists".format(fs_src_path) + ) if self.is_exist(fs_dst_path): raise FSFileExistsError("{} exists already".format(fs_dst_path)) @@ -1552,7 +1559,7 @@ class AFSClient(FS): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] diff --git a/python/paddle/distributed/fleet/utils/http_server.py b/python/paddle/distributed/fleet/utils/http_server.py index a1251c46f3c..1dd78136d98 100644 --- a/python/paddle/distributed/fleet/utils/http_server.py +++ b/python/paddle/distributed/fleet/utils/http_server.py @@ -35,9 +35,9 @@ def get_logger(name, level, fmt): return logger -_http_server_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_http_server_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): @@ -167,7 +167,8 @@ class KVServer: start server until user calls stop to let it quit. """ self.listen_thread = threading.Thread( - target=lambda: self.http_server.serve_forever()) + target=lambda: self.http_server.serve_forever() + ) self.listen_thread.start() def stop(self): diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py index fd81c5b11f4..0c5bff02ed8 100644 --- a/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py +++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py @@ -181,15 +181,17 @@ class HybridParallelInferenceHelper(object): print(res) """ - def __init__(self, - startup_program, - main_program, - num_mp=1, - num_pp=1, - micro_batch_size=1, - beam_size=1, - init_comm=True, - role_maker=None): + def __init__( + self, + startup_program, + main_program, + num_mp=1, + num_pp=1, + micro_batch_size=1, + beam_size=1, + init_comm=True, + role_maker=None, + ): assert isinstance(startup_program, Program) assert isinstance(main_program, Program) @@ -225,7 +227,8 @@ class HybridParallelInferenceHelper(object): if role_maker is None: self.role_maker = fleet.base.role_maker.PaddleCloudRoleMaker( - is_collective=True) + is_collective=True + ) else: if isinstance(role_maker, fleet.base.role_maker.RoleMakerBase): assert role_maker._is_collective == True @@ -249,7 +252,8 @@ class HybridParallelInferenceHelper(object): self.global_nranks = self.nranks arr = np.arange(0, self.num_pp * self.num_mp).reshape( - [self.num_pp, self.num_mp]) + [self.num_pp, self.num_mp] + ) ipp, imp = np.where(arr == self.rank) ipp = ipp[0] imp = imp[0] @@ -269,31 +273,43 @@ class HybridParallelInferenceHelper(object): num_pp = len(dev_ids) num_pp = max(1, num_pp) assert num_pp == self.num_pp, 'num_pp: {}, self.num_pp: {}'.format( - num_pp, self.num_pp) + num_pp, self.num_pp + ) collective_helper = fleet.meta_optimizers.common.CollectiveHelper( - self.role_maker, wait_port=False) + self.role_maker, wait_port=False + ) # Create global rings - collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - self.global_endpoints, - self.global_rank, - self.global_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + self.global_endpoints, + self.global_rank, + self.global_ring_id, + True, + self.global_ring_id, + True, + ) # Create mp rings if self.num_mp > 1: mp_endpoints = [self.endpoints[mp_idx] for mp_idx in self.mp_group] mp_rank = [ - idx for idx, mp_idx in enumerate(self.mp_group) + idx + for idx, mp_idx in enumerate(self.mp_group) if mp_idx == self.rank ][0] - collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - mp_endpoints, mp_rank, - self.mp_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + mp_endpoints, + mp_rank, + self.mp_ring_id, + True, + self.global_ring_id, + True, + ) # Create pipeline rings if self.num_pp > 1: @@ -305,20 +321,32 @@ class HybridParallelInferenceHelper(object): second_node = self.pp_group[pair[1]] if self.rank != first_node and self.rank != second_node: collective_helper._init_communicator( - self._startup_program, None, None, None, None, False, - self.global_ring_id, True) + self._startup_program, + None, + None, + None, + None, + False, + self.global_ring_id, + True, + ) continue pipeline_endpoints = [ - self.endpoints[first_node], self.endpoints[second_node] + self.endpoints[first_node], + self.endpoints[second_node], ] pipeline_rank = 0 if self.rank == first_node else 1 - collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - pipeline_endpoints, - pipeline_rank, ring_id, - False, self.global_ring_id, - True) + collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + pipeline_endpoints, + pipeline_rank, + ring_id, + False, + self.global_ring_id, + True, + ) def _get_input_output_info(self, block): ''' @@ -345,7 +373,10 @@ class HybridParallelInferenceHelper(object): for each_block in self._main_program.blocks: for op in each_block.ops: for var_name in op.input_arg_names: - if var_name not in params or var_name in self._param_device_map: + if ( + var_name not in params + or var_name in self._param_device_map + ): continue device = op.attr(self._op_device_key) @@ -374,7 +405,8 @@ class HybridParallelInferenceHelper(object): if op.type == "while": sub_block_id = int(op.attr('sub_block').id) sub_used_var_names = self._split_program( - program, stage, sub_block_id) + program, stage, sub_block_id + ) used_var_names.update(sub_used_var_names) @@ -409,25 +441,24 @@ class HybridParallelInferenceHelper(object): return used_var_names - -# def _find_post_op(self, index, var_name): -# """ -# Find the post op that has variable named var_name as input. -# """ -# # bugfix for uniform hybrid parallelism -# if '.cast_fp32' in var_name: -# var_name = var_name.replace('.cast_fp32', '') -# if '.cast_fp16' in var_name: -# var_name = var_name.replace('.cast_fp16', '') - -# post_ops = self._input_var_to_op[var_name] -# if post_ops == None: return None -# result_op = None -# for post_op, post_idx in reversed(post_ops): -# if post_idx > index: -# result_op = post_op -# break -# return result_op + # def _find_post_op(self, index, var_name): + # """ + # Find the post op that has variable named var_name as input. + # """ + # # bugfix for uniform hybrid parallelism + # if '.cast_fp32' in var_name: + # var_name = var_name.replace('.cast_fp32', '') + # if '.cast_fp16' in var_name: + # var_name = var_name.replace('.cast_fp16', '') + + # post_ops = self._input_var_to_op[var_name] + # if post_ops == None: return None + # result_op = None + # for post_op, post_idx in reversed(post_ops): + # if post_idx > index: + # result_op = post_op + # break + # return result_op def _find_prev_op(self, index, var_name): """ @@ -435,7 +466,8 @@ class HybridParallelInferenceHelper(object): variable named var_name. """ prev_ops = self._output_var_to_op[var_name] - if prev_ops == None: return None + if prev_ops == None: + return None result_op = None for prev_op, prev_idx in reversed(prev_ops): if prev_idx < index: @@ -481,26 +513,31 @@ class HybridParallelInferenceHelper(object): pre_stage_id = None for op in block.ops: - assert op.has_attr(self._op_role_key), ("{} has no {} set .".format( - op.type, self._op_role_key)) + assert op.has_attr(self._op_role_key), "{} has no {} set .".format( + op.type, self._op_role_key + ) op_role = op.attr(self._op_role_key) - assert op_role == int(self._op_role.Forward), ( - "Only forward is supported for inference.") + assert op_role == int( + self._op_role.Forward + ), "Only forward is supported for inference." if not op._has_kernel(op.type): assert op.type in [ - "while", "conditional_block" - ], ("The only supported op without kernel is while.") + "while", + "conditional_block", + ], "The only supported op without kernel is while." sub_block_id = op.attr('sub_block').id sub_block = block.program.block(sub_block_id) self._check_validation(sub_block) - assert op.has_attr( - self._op_device_key), ("{} has no {} set.".format( - op.type, self._op_device_key)) + assert op.has_attr(self._op_device_key), "{} has no {} set.".format( + op.type, self._op_device_key + ) device = op.attr(self._op_device_key) - assert device, ("{} has no {} set.".format(op.type, - self._op_device_key)) - if device.split(':')[1] == "all": continue + assert device, "{} has no {} set.".format( + op.type, self._op_device_key + ) + if device.split(':')[1] == "all": + continue dev_type = device.split(':')[0] assert dev_type == self._device @@ -522,14 +559,17 @@ class HybridParallelInferenceHelper(object): for index, op in enumerate(list(block.ops)): cur_device = op.attr(self._op_device_key) - if cur_device.split(':')[-1] == "all": continue + if cur_device.split(':')[-1] == "all": + continue for var_name in op.input_arg_names: if not block.has_var(var_name) and block._find_var_recursive( - var_name): + var_name + ): continue var = block.var(var_name) # skip data var - if var.is_data: continue + if var.is_data: + continue prev_device = None generate_ops = self._output_var_to_op.get(var_name) if generate_ops is None: @@ -540,21 +580,24 @@ class HybridParallelInferenceHelper(object): prev_op = self._find_prev_op(index, var_name) if not prev_device: - prev_device = prev_op.attr(self._op_device_key) \ - if prev_op else None + prev_device = ( + prev_op.attr(self._op_device_key) if prev_op else None + ) if prev_device is None or prev_device.split(":")[-1] == "all": continue - if prev_device == cur_device: continue + if prev_device == cur_device: + continue if var_name not in input_var_to_device: input_var_to_device[var_name] = [] if (cur_device, prev_device) in input_var_to_device[var_name]: continue - assert self._device == cur_device.split( - ':')[0], "More than one device type found." + assert ( + self._device == cur_device.split(':')[0] + ), "More than one device type found." device_type = cur_device.split(':')[0] + ':' def _insert_send_recv(cur_id, prev_id): @@ -568,7 +611,8 @@ class HybridParallelInferenceHelper(object): _insert_send_recv(cur_id - 1, prev_id) _insert_send_recv(cur_id, cur_id - 1) input_var_to_device[var_name].append( - (cur_dev, prev_dev)) + (cur_dev, prev_dev) + ) return assert cur_id - prev_id == 1 @@ -577,7 +621,10 @@ class HybridParallelInferenceHelper(object): op_role = op.attr(self._op_role_key) var = block.vars[var_name] pair = (prev_id, cur_id) - if is_while_block and pair not in self._pipeline_pair_in_while: + if ( + is_while_block + and pair not in self._pipeline_pair_in_while + ): self._pipeline_pair_in_while.append(pair) # 1000 is just a magic number @@ -599,14 +646,16 @@ class HybridParallelInferenceHelper(object): self._op_role_key: op_role, 'use_calc_stream': True, 'peer': 1, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) extra_index_info['index'] += 1 var_shape = list(var.shape) if var_shape[0] < 0: if is_while_block: - var_shape[ - 0] = self.micro_batch_size * self.beam_size + var_shape[0] = ( + self.micro_batch_size * self.beam_size + ) else: var_shape[0] = self.micro_batch_size @@ -621,17 +670,24 @@ class HybridParallelInferenceHelper(object): self._op_role_key: op_role, 'use_calc_stream': True, 'peer': 0, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) extra_index_info['index'] += 1 - _insert_send_recv(int(cur_device.split(':')[1]), - int(prev_device.split(':')[1])) + _insert_send_recv( + int(cur_device.split(':')[1]), + int(prev_device.split(':')[1]), + ) block._sync_with_cpp() def _insert_sendrecv_ops_in_while_block( - self, block, sync_in_while_lastpp2firstpp_var_names, - sync_in_while_var_names, stage): + self, + block, + sync_in_while_lastpp2firstpp_var_names, + sync_in_while_var_names, + stage, + ): dev_ids = [] for pair in self._pipeline_pair_in_while: prev_id, cur_id = pair @@ -646,13 +702,16 @@ class HybridParallelInferenceHelper(object): first_id = min(dev_ids) last_id = max(dev_ids) - assert len(block.ops) > 2, "It must have more than 2 ops in while sub block, " \ - "layers.assign(layers.cast(cond_int, dtype='bool'), cond) must at end of while block, " \ + assert len(block.ops) > 2, ( + "It must have more than 2 ops in while sub block, " + "layers.assign(layers.cast(cond_int, dtype='bool'), cond) must at end of while block, " "because nccl cannot send bool dtype var" + ) index = len(block.ops) - 2 for prev_id in dev_ids: - if prev_id == cur_id: continue + if prev_id == cur_id: + continue assert cur_id > prev_id pair = (prev_id, cur_id) @@ -667,7 +726,10 @@ class HybridParallelInferenceHelper(object): ring_id = self._pp_ring_map[pair_key] if cur_id == last_id and prev_id == first_id: - var_names = sync_in_while_lastpp2firstpp_var_names + sync_in_while_var_names + var_names = ( + sync_in_while_lastpp2firstpp_var_names + + sync_in_while_var_names + ) else: var_names = sync_in_while_var_names @@ -679,19 +741,24 @@ class HybridParallelInferenceHelper(object): type='send_v2', inputs={'X': var}, attrs={ - self._op_device_key: - self._device + ':' + str(cur_id), + self._op_device_key: self._device + + ':' + + str(cur_id), self._op_role_key: int(self._op_role.Forward), 'use_calc_stream': True, 'peer': 0, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) else: var_shape = list(var.shape) print(var_name) if len(var.shape) > 0: - var_shape[0] = self.micro_batch_size if var_shape[ - 0] < 0 else var_shape[0] + var_shape[0] = ( + self.micro_batch_size + if var_shape[0] < 0 + else var_shape[0] + ) block._insert_op_without_sync( index=index, type='recv_v2', @@ -699,13 +766,15 @@ class HybridParallelInferenceHelper(object): attrs={ 'out_shape': var_shape, 'dtype': var.dtype, - self._op_device_key: - self._device + ':' + str(prev_id), + self._op_device_key: self._device + + ':' + + str(prev_id), self._op_role_key: int(self._op_role.Forward), 'use_calc_stream': True, 'peer': 1, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) index += 1 block._sync_with_cpp() @@ -721,13 +790,16 @@ class HybridParallelInferenceHelper(object): if op.type == 'while': sub_block_id = op.attr('sub_block').id num_while += 1 - if sub_block_id: return op, self._main_program.block(sub_block_id) + if sub_block_id: + return op, self._main_program.block(sub_block_id) return None, None - def gen_infer_program(self, - sync_in_while_lastpp2firstpp_var_names=None, - sync_in_while_var_names=None, - debug=False): + def gen_infer_program( + self, + sync_in_while_lastpp2firstpp_var_names=None, + sync_in_while_var_names=None, + debug=False, + ): """ Generate inference program. Params: @@ -764,15 +836,19 @@ class HybridParallelInferenceHelper(object): while_op, while_block = self._get_while_block() if while_block: out_var_to_op, in_var_to_op = self._get_input_output_info( - while_block) + while_block + ) self._output_var_to_op = out_var_to_op self._input_var_to_op = in_var_to_op self._insert_sendrecv_ops_for_boundaries(while_block, True) self._insert_sendrecv_ops_in_while_block( - while_block, sync_in_while_lastpp2firstpp_var_names, - sync_in_while_var_names, self._stage) + while_block, + sync_in_while_lastpp2firstpp_var_names, + sync_in_while_var_names, + self._stage, + ) # step3: split programs self._split_program(self._startup_program, self._stage, 0) diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py index 93b9ce4ef79..fec3e455f8a 100644 --- a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py +++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py @@ -15,7 +15,11 @@ from paddle import framework import paddle from paddle.fluid import core -from paddle.fluid.dygraph.parallel import _split_tensors, sync_params_buffers, build_groups +from paddle.fluid.dygraph.parallel import ( + _split_tensors, + sync_params_buffers, + build_groups, +) from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph from .log_util import logger @@ -30,7 +34,8 @@ def _apply_collective_grads(parameters, comm_group): for param in parameters: if param.trainable and (param._grad_ivar() is not None): g_var = param._grad_ivar() - assert not g_var._is_sparse( + assert ( + not g_var._is_sparse() ), "Now, it doesn't support sparse parameters" grad_vars.append(g_var) assert g_var not in grad_var_set @@ -38,19 +43,20 @@ def _apply_collective_grads(parameters, comm_group): coalesced_grads_and_vars = build_groups(grad_vars, 128 * 1024 * 1024) - nranks = paddle.distributed.get_world_size( - ) if comm_group is None else comm_group.nranks + nranks = ( + paddle.distributed.get_world_size() + if comm_group is None + else comm_group.nranks + ) for coalesced_grad, _, _ in coalesced_grads_and_vars: # need to div nranks div_factor = paddle.to_tensor(nranks, dtype=coalesced_grad.dtype) paddle.fluid.framework._dygraph_tracer().trace_op( type="elementwise_div", - inputs={ - 'X': coalesced_grad, - 'Y': div_factor - }, + inputs={'X': coalesced_grad, 'Y': div_factor}, outputs={'Out': coalesced_grad}, - attrs={'axis': -1}) + attrs={'axis': -1}, + ) paddle.distributed.all_reduce(coalesced_grad, group=comm_group) _split_tensors(coalesced_grads_and_vars) @@ -63,7 +69,8 @@ def _apply_collective_grads_eager(parameters, comm_group): for param in parameters: if param.trainable and (param._grad_ivar() is not None): g_var = param._grad_ivar() - assert not g_var.is_sparse( + assert ( + not g_var.is_sparse() ), "Now, it doesn't support sparse parameters" grad_vars.append(g_var) assert g_var not in grad_var_set @@ -71,8 +78,11 @@ def _apply_collective_grads_eager(parameters, comm_group): coalesced_grads_and_vars = build_groups(grad_vars, 128 * 1024 * 1024) - nranks = paddle.distributed.get_world_size( - ) if comm_group is None else comm_group.nranks + nranks = ( + paddle.distributed.get_world_size() + if comm_group is None + else comm_group.nranks + ) for coalesced_grad, _, _ in coalesced_grads_and_vars: # need to div nranks coalesced_grad.scale_(1.0 / nranks) @@ -87,20 +97,18 @@ def _broadcast_data_help(data, shape, dtype, hcg): mp_rank = hcg.get_model_parallel_rank() shape_gpu = paddle.to_tensor(shape, dtype="int32") - paddle.distributed.broadcast(shape_gpu, - src=src_rank, - group=model_parallel_group, - sync_op=True) + paddle.distributed.broadcast( + shape_gpu, src=src_rank, group=model_parallel_group, sync_op=True + ) if mp_rank != 0: input_data = paddle.zeros(shape_gpu, dtype=dtype) else: input_data = data - paddle.distributed.broadcast(input_data, - src=src_rank, - group=model_parallel_group, - sync_op=True) + paddle.distributed.broadcast( + input_data, src=src_rank, group=model_parallel_group, sync_op=True + ) if mp_rank != 0: if in_dygraph_mode(): @@ -109,7 +117,8 @@ def _broadcast_data_help(data, shape, dtype, hcg): else: data.value().get_tensor()._clear() data.value().get_tensor()._share_data_with( - input_data.value().get_tensor()) + input_data.value().get_tensor() + ) def broadcast_input_data(hcg, *inputs, **kwargs): @@ -117,8 +126,11 @@ def broadcast_input_data(hcg, *inputs, **kwargs): for v in inputs: if isinstance(v, (core.VarBase, core.eager.Tensor)): with framework.no_grad(): - if "gpu" in cur_device and in_dygraph_mode() \ - and not v.place.is_gpu_place(): + if ( + "gpu" in cur_device + and in_dygraph_mode() + and not v.place.is_gpu_place() + ): v_gpu = v.cuda(int(cur_device.split(":")[1])) v._clear_data() v_gpu._share_buffer_to(v) @@ -129,8 +141,11 @@ def broadcast_input_data(hcg, *inputs, **kwargs): for k, v in kwargs.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): with framework.no_grad(): - if "gpu" in cur_device and in_dygraph_mode() \ - and not v.place.is_gpu_place(): + if ( + "gpu" in cur_device + and in_dygraph_mode() + and not v.place.is_gpu_place() + ): v_gpu = v.cuda(int(cur_device.split(":")[1])) v._clear_data() v_gpu._share_buffer_to(v) @@ -144,26 +159,27 @@ def broadcast_input_data(hcg, *inputs, **kwargs): def broadcast_mp_parameters(model, hcg): model_parallel_group = hcg.get_model_parallel_group() src_rank = hcg.get_model_parallel_group_src_rank() - sync_params_buffers(model, - model_parallel_group, - src_rank, - is_model_parallel=True) + sync_params_buffers( + model, model_parallel_group, src_rank, is_model_parallel=True + ) def broadcast_dp_parameters(model, hcg): data_parallel_group = hcg.get_data_parallel_group() src_rank = hcg.get_data_parallel_group_src_rank() - sync_params_buffers(model, - data_parallel_group, - src_rank, - is_model_parallel=False) + sync_params_buffers( + model, data_parallel_group, src_rank, is_model_parallel=False + ) def fused_allreduce_gradients(parameter_list, hcg): data_parallel_group = None if hcg is None else hcg.get_data_parallel_group() logger.debug("dp start fuse allreduce gradients") - apply_func = _apply_collective_grads_eager if in_dygraph_mode( - ) else _apply_collective_grads + apply_func = ( + _apply_collective_grads_eager + if in_dygraph_mode() + else _apply_collective_grads + ) with framework.no_grad(): apply_func(parameter_list, data_parallel_group) @@ -182,7 +198,8 @@ def sharding_reduce_gradients(parameter_list, hcg): paddle.distributed.all_reduce( param.grad, group=hcg.get_sharding_parallel_group(), - sync_op=True) + sync_op=True, + ) elif _in_legacy_dygraph(): g_var = param._grad_ivar() @@ -195,20 +212,20 @@ def sharding_reduce_gradients(parameter_list, hcg): outputs={'Out': g_var}, attrs={ 'ring_id': hcg.get_sharding_parallel_group().id, - 'use_calc_stream': True - }) + 'use_calc_stream': True, + }, + ) # grad / sharding_rank - div_factor = paddle.to_tensor(sharding_nrank, - dtype=g_var.dtype) + div_factor = paddle.to_tensor( + sharding_nrank, dtype=g_var.dtype + ) paddle.fluid.framework._dygraph_tracer().trace_op( type="elementwise_div", - inputs={ - 'X': g_var, - 'Y': div_factor - }, + inputs={'X': g_var, 'Y': div_factor}, outputs={'Out': g_var}, - attrs={'axis': -1}) + attrs={'axis': -1}, + ) def broadcast_sharding_parameters(model, hcg): @@ -216,7 +233,6 @@ def broadcast_sharding_parameters(model, hcg): logger.debug("sharding start init parameters sync") sharding_parallel_group = hcg.get_sharding_parallel_group() src_rank = hcg.get_sharding_parallel_group_src_rank() - sync_params_buffers(model, - sharding_parallel_group, - src_rank, - is_model_parallel=False) + sync_params_buffers( + model, sharding_parallel_group, src_rank, is_model_parallel=False + ) diff --git a/python/paddle/distributed/fleet/utils/internal_storage.py b/python/paddle/distributed/fleet/utils/internal_storage.py index 89011a6fe1c..e0f2ef0a046 100644 --- a/python/paddle/distributed/fleet/utils/internal_storage.py +++ b/python/paddle/distributed/fleet/utils/internal_storage.py @@ -47,10 +47,11 @@ class InternalStorage: # The actual flat tensor size = [size] if isinstance(size, int) else size if convert_cpu: - value = np.zeros( - size, - dtype=np.float16) if Type.fp16.value == dtype else np.zeros( - size, dtype=np.float32) + value = ( + np.zeros(size, dtype=np.float16) + if Type.fp16.value == dtype + else np.zeros(size, dtype=np.float32) + ) self.buffer = core.VarBase(value=value, place=core.CPUPlace()) else: self.buffer = paddle.zeros(size, dtype=dtype) @@ -59,16 +60,25 @@ class InternalStorage: """ Move the underlying buffer """ - assert self.buffer is not None, "Cannot move a collapsed bucket, please rebuild it" - assert (dtype == Type.fp32.value - or Type.fp16.value), "Conversion type is not supported now" + assert ( + self.buffer is not None + ), "Cannot move a collapsed bucket, please rebuild it" + assert ( + dtype == Type.fp32.value or Type.fp16.value + ), "Conversion type is not supported now" - dev_id = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) if self._device != device: - tmp_buffer = self.buffer.cuda( - dev_id) if device == "gpu" else self.buffer.cpu() + tmp_buffer = ( + self.buffer.cuda(dev_id) + if device == "gpu" + else self.buffer.cpu() + ) for param in self._params: param.clear_gradient(False) param._gradient_set_empty(False) @@ -106,17 +116,18 @@ class ParamStorage(InternalStorage): Add new parameters to the InternalStorage. Params becomes a view of this InternalStorage buffer. """ - assert all([ - id(param) not in self._param_ids for param in trainable_params - ]), "The same param cannot be checked in twice" + assert all( + [id(param) not in self._param_ids for param in trainable_params] + ), "The same param cannot be checked in twice" assert self.buffer is not None self.param2align = param2align cpu_param_shape = list() for param in trainable_params: - p_shape = self._add_param_as_view(param, param2align[param.name], - convert_gpu) + p_shape = self._add_param_as_view( + param, param2align[param.name], convert_gpu + ) cpu_param_shape.append(p_shape) if convert_gpu: @@ -127,8 +138,9 @@ class ParamStorage(InternalStorage): self._fill = 0 for idx, param in enumerate(trainable_params): - self._convert_buffer(param, cpu_param_shape[idx], - param2align[param.name]) + self._convert_buffer( + param, cpu_param_shape[idx], param2align[param.name] + ) self._params.append(param) self._param_ids.append(id(param)) @@ -138,7 +150,8 @@ class ParamStorage(InternalStorage): assert ( param.dtype == self.buffer.dtype ), "Different types for the InternalStorage and the param, cannot proceed: {} - {}".format( - param.dtype, self.buffer.dtype) + param.dtype, self.buffer.dtype + ) var_end = self._fill + np.prod(param.shape) offset = var_end + align @@ -152,11 +165,15 @@ class ParamStorage(InternalStorage): param.stop_gradient = origin_state # Copy the current param value - dev_id = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) with device_guard(dev_id, "cpu"): tmp_var = core.VarBase( - tensor=self.buffer._slice(self._fill, var_end)) + tensor=self.buffer._slice(self._fill, var_end) + ) if convert_gpu: param_cpu = param.cpu() param.value().get_tensor()._clear() @@ -199,13 +216,9 @@ class GradStorage(InternalStorage): This is a basic class to simplify the handling of gradient InternalStorages """ - def __init__(self, - size, - dtype, - device, - destination, - parm2align, - convert_cpu=False): + def __init__( + self, size, dtype, device, destination, parm2align, convert_cpu=False + ): if isinstance(size, np.int64): size = size.tolist() super().__init__(size, dtype, device, convert_cpu) @@ -219,22 +232,21 @@ class GradStorage(InternalStorage): self.sent = False def reset_checked_in(self): - """ Reset the counter of the parameter grads which have been checked in - """ + """Reset the counter of the parameter grads which have been checked in""" self.params_checked_in = 0 self.sent = False @property def all_checked_in(self): - """ Judge all the expected gradient check-in happened """ + """Judge all the expected gradient check-in happened""" return len(self._params) == self.params_checked_in def can_add_grad_view(self, param, align): - """ Is there enough InternalStorage to add this parameter gradient, and whether this param have already checked in. - """ - return self._fill + np.prod( - param.shape) + align <= self._max_size and id( - param) not in self._param_ids + """Is there enough InternalStorage to add this parameter gradient, and whether this param have already checked in.""" + return ( + self._fill + np.prod(param.shape) + align <= self._max_size + and id(param) not in self._param_ids + ) def to(self, device, dtype=None, keep_alignment=True): """ @@ -254,9 +266,9 @@ class GradStorage(InternalStorage): Add a new parameter gradient to the InternalStorage. Param.grad becomes a view of this InternalStorage buffer. """ - assert id( - param - ) not in self._param_ids, "The same gradients cannot be checked in twice" + assert ( + id(param) not in self._param_ids + ), "The same gradients cannot be checked in twice" self._add_grad_as_view(param, align) self._params.append(param) @@ -304,9 +316,9 @@ class GradStorage(InternalStorage): @fluid.dygraph.no_grad def _add_grad_as_view(self, param, align): - assert np.prod( - self.buffer.shape - ) > 0, "Cannot add a gradient to a released InternalStorage, please rebuild" + assert ( + np.prod(self.buffer.shape) > 0 + ), "Cannot add a gradient to a released InternalStorage, please rebuild" assert param.dtype == self.buffer.dtype grad_end = self._fill + np.prod(param.shape) @@ -314,8 +326,11 @@ class GradStorage(InternalStorage): assert offset <= np.prod(self.buffer.shape) # Copy the current grad value to InternalStorage - dev_id = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) if self._device == "cpu": with device_guard(dev_id, self._device): tmp_var = core.VarBase(self.buffer._slice(self._fill, grad_end)) diff --git a/python/paddle/distributed/fleet/utils/log_util.py b/python/paddle/distributed/fleet/utils/log_util.py index 34b1caa863c..07bcbea4b41 100644 --- a/python/paddle/distributed/fleet/utils/log_util.py +++ b/python/paddle/distributed/fleet/utils/log_util.py @@ -67,7 +67,8 @@ def layer_to_str(base, *args, **kwargs): if kwargs: name += ", " if kwargs: - name += ", ".join("{}={}".format(key, str(value)) - for key, value in kwargs.items()) + name += ", ".join( + "{}={}".format(key, str(value)) for key, value in kwargs.items() + ) name += ")" return name diff --git a/python/paddle/distributed/fleet/utils/ps_util.py b/python/paddle/distributed/fleet/utils/ps_util.py index fbf713bed93..9b079d64bb5 100644 --- a/python/paddle/distributed/fleet/utils/ps_util.py +++ b/python/paddle/distributed/fleet/utils/ps_util.py @@ -29,21 +29,21 @@ class DistributedInfer: if main_program: self.origin_main_program = main_program.clone() else: - self.origin_main_program = paddle.static.default_main_program( - ).clone() + self.origin_main_program = ( + paddle.static.default_main_program().clone() + ) if startup_program: self.origin_startup_program = startup_program else: - self.origin_startup_program = paddle.static.default_startup_program( + self.origin_startup_program = ( + paddle.static.default_startup_program() ) self.sparse_table_maps = None - def init_distributed_infer_env(self, - exe, - loss, - role_maker=None, - dirname=None): + def init_distributed_infer_env( + self, exe, loss, role_maker=None, dirname=None + ): import paddle.distributed.fleet as fleet if fleet.fleet._runtime_handle is None: @@ -52,10 +52,12 @@ class DistributedInfer: fake_optimizer = paddle.optimizer.SGD() strategy = fleet.DistributedStrategy() strategy.a_sync = True - optimizer = fleet.distributed_optimizer(fake_optimizer, - strategy=strategy) - optimizer.minimize(loss, - startup_program=self.origin_startup_program) + optimizer = fleet.distributed_optimizer( + fake_optimizer, strategy=strategy + ) + optimizer.minimize( + loss, startup_program=self.origin_startup_program + ) if fleet.is_server(): fleet.init_server(dirname=dirname) @@ -88,36 +90,45 @@ class DistributedInfer: if dirname is not None and exe is not None: all_persist_vars = [ - v for v in self.origin_main_program.list_vars() + v + for v in self.origin_main_program.list_vars() if paddle.static.io.is_persistable(v) ] - dense_persist_vars = [(v.name, v) for v in all_persist_vars - if v.name not in sparse_table_maps] + dense_persist_vars = [ + (v.name, v) + for v in all_persist_vars + if v.name not in sparse_table_maps + ] need_load_vars = [ - v[1] for v in dense_persist_vars + v[1] + for v in dense_persist_vars if os.path.isfile(os.path.join(dirname, v[0])) ] - paddle.static.load_vars(exe, - dirname, - main_program=self.origin_main_program, - vars=need_load_vars) + paddle.static.load_vars( + exe, + dirname, + main_program=self.origin_main_program, + vars=need_load_vars, + ) def get_dist_infer_program(self): varname2tables = self._get_sparse_table_map() - convert_program = self._convert_program(self.origin_main_program, - varname2tables) + convert_program = self._convert_program( + self.origin_main_program, varname2tables + ) return convert_program def _convert_program(self, main_program, varname2tables): - def distributed_ops_pass(program): SPARSE_OP_TYPE_DICT = {"lookup_table": "W", "lookup_table_v2": "W"} def _get_pull_sparse_ops(_program): pull_sparse_ops = {} for op in _program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] ops = pull_sparse_ops.get(param_name, []) ops.append(op) @@ -125,7 +136,6 @@ class DistributedInfer: return pull_sparse_ops def _pull_sparse_fuse(_program, pull_sparse_ops): - def dag_check_up_and_reorder(program, inputs, outputs): global_block = program.global_block() min_output_index = len(global_block.ops) @@ -151,7 +161,8 @@ class DistributedInfer: if out_var.name in ins: output_indexes[idx] = 1 min_output_index = min( - min_output_index, idx) + min_output_index, idx + ) for i in range(len(global_block.ops)): if input_indexes[i] == 1 and output_indexes[i] == 1: @@ -162,8 +173,9 @@ class DistributedInfer: if min_output_index < max_input_index: move_ops = [] - for i in range(min_output_index + 1, - len(input_indexes)): + for i in range( + min_output_index + 1, len(input_indexes) + ): if input_indexes[i] == 1: move_ops.append((global_block.ops[i], i)) for i, op in enumerate(move_ops): @@ -179,8 +191,9 @@ class DistributedInfer: for k in range(0, len(op.input_names)): ins = op.input(op.input_names[k]) op_inputs.append(ins) - for j in range(pos - 1, min_output_index - 1, - -1): + for j in range( + pos - 1, min_output_index - 1, -1 + ): op1 = global_block.ops[j] if op1 in visited: continue @@ -209,27 +222,35 @@ class DistributedInfer: queue.sort() for index in queue: desc = global_block.desc._insert_op( - min_output_index) + min_output_index + ) desc.copy_from(global_block.ops[index].desc) global_block.desc._remove_op( - index + 1, index + 2) + index + 1, index + 2 + ) global_block.ops[index].desc = desc insert_op = global_block.ops.pop(index) input_state = input_indexes.pop(index) output_state = output_indexes.pop(index) - global_block.ops.insert(min_output_index, - insert_op) - input_indexes.insert(min_output_index, - input_state) - output_indexes.insert(min_output_index, - output_state) + global_block.ops.insert( + min_output_index, insert_op + ) + input_indexes.insert( + min_output_index, input_state + ) + output_indexes.insert( + min_output_index, output_state + ) min_output_index = min_output_index + 1 assert global_block.desc.op_size() == len( - global_block.ops) + global_block.ops + ) for i in range(len(global_block.ops)): - assert global_block.desc.op( - i) == global_block.ops[i].desc + assert ( + global_block.desc.op(i) + == global_block.ops[i].desc + ) for param, ops in pull_sparse_ops.items(): all_ops = program.global_block().ops @@ -243,8 +264,10 @@ class DistributedInfer: if w.name not in varname2tables.keys(): raise ValueError( - "can not find variable {}, please check your configuration" - .format(w.name)) + "can not find variable {}, please check your configuration".format( + w.name + ) + ) table_id = varname2tables[w.name] @@ -264,8 +287,9 @@ class DistributedInfer: program.global_block()._remove_op(idx) inputs_idxs = [-1] * len(inputs) - outputs_idxs = [len(program.global_block().ops) + 1 - ] * len(outputs) + outputs_idxs = [len(program.global_block().ops) + 1] * len( + outputs + ) for idx, op in enumerate(program.global_block().ops): for i in range(0, len(op.output_names)): @@ -273,13 +297,15 @@ class DistributedInfer: for in_id, in_var in enumerate(inputs): if in_var.name in outs: inputs_idxs[in_id] = max( - idx, inputs_idxs[in_id]) + idx, inputs_idxs[in_id] + ) for i in range(0, len(op.input_names)): ins = op.input(op.input_names[i]) for out_id, out_var in enumerate(outputs): if out_var.name in ins: outputs_idxs[out_id] = min( - idx, outputs_idxs[out_id]) + idx, outputs_idxs[out_id] + ) if min(outputs_idxs) - max(inputs_idxs) >= 1: distributed_idx = max(inputs_idxs) + 1 @@ -287,18 +313,16 @@ class DistributedInfer: program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Outputs": outputs}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "is_test": True, - "lookup_table_version": op_type - }) + "lookup_table_version": op_type, + }, + ) else: raise ValueError( "something wrong with Fleet, submit a issue is recommended" diff --git a/python/paddle/distributed/launch/context/__init__.py b/python/paddle/distributed/launch/context/__init__.py index 9936b534edf..037fc0efbc5 100644 --- a/python/paddle/distributed/launch/context/__init__.py +++ b/python/paddle/distributed/launch/context/__init__.py @@ -22,7 +22,6 @@ import logging class Context(object): - def __init__(self, enable_plugin=True): self.args, self.unknown_args = parse_args() self.envs = fetch_envs() @@ -54,8 +53,9 @@ class Context(object): return False if len(self.unknown_args) > 0: - self.logger.warning("Compatible mode enable with args {}".format( - self.unknown_args)) + self.logger.warning( + "Compatible mode enable with args {}".format(self.unknown_args) + ) return True return False @@ -75,7 +75,8 @@ class Context(object): logger = logging.getLogger("LAUNCH") logger.setLevel(self.args.log_level.upper() or level) formatter = logging.Formatter( - fmt='%(name)s %(levelname)s %(asctime)s %(message)s') + fmt='%(name)s %(levelname)s %(asctime)s %(message)s' + ) ch = logging.StreamHandler() ch.setFormatter(formatter) logger.addHandler(ch) diff --git a/python/paddle/distributed/launch/context/args_envs.py b/python/paddle/distributed/launch/context/args_envs.py index 104ad1b789f..2013ba6a3d7 100644 --- a/python/paddle/distributed/launch/context/args_envs.py +++ b/python/paddle/distributed/launch/context/args_envs.py @@ -52,126 +52,135 @@ def parse_args(): base_group = parser.add_argument_group("Base Parameters") - base_group.add_argument("--master", - type=str, - default=None, - help="the master/rendezvous server, ip:port") - - base_group.add_argument("--legacy", - type=bool, - default=False, - help="use legacy launch") - - base_group.add_argument("--rank", - type=int, - default=-1, - help="the node rank") - - base_group.add_argument("--log_level", - type=str, - default="INFO", - help="log level. Default INFO") - - base_group.add_argument("--nnodes", - type=str, - default="1", - help="the number of nodes, i.e. pod/node number") - - base_group.add_argument("--nproc_per_node", - type=int, - default=None, - help="the number of processes in a pod") + base_group.add_argument( + "--master", + type=str, + default=None, + help="the master/rendezvous server, ip:port", + ) + + base_group.add_argument( + "--legacy", type=bool, default=False, help="use legacy launch" + ) + + base_group.add_argument( + "--rank", type=int, default=-1, help="the node rank" + ) + + base_group.add_argument( + "--log_level", type=str, default="INFO", help="log level. Default INFO" + ) + + base_group.add_argument( + "--nnodes", + type=str, + default="1", + help="the number of nodes, i.e. pod/node number", + ) + + base_group.add_argument( + "--nproc_per_node", + type=int, + default=None, + help="the number of processes in a pod", + ) base_group.add_argument( "--log_dir", type=str, default="log", - help="the path for each process's log. Default ./log") - base_group.add_argument("--run_mode", - type=str, - default=None, - help="run mode of the job, collective/ps/ps-heter") - - base_group.add_argument("--job_id", - type=str, - default="default", - help="unique id of the job. Default default") - - base_group.add_argument("--devices", - "--gpus", - "--npus", - "--xpus", - type=str, - default=None, - help="accelerate devices. as --gpus,npus,xpus") + help="the path for each process's log. Default ./log", + ) + base_group.add_argument( + "--run_mode", + type=str, + default=None, + help="run mode of the job, collective/ps/ps-heter", + ) + + base_group.add_argument( + "--job_id", + type=str, + default="default", + help="unique id of the job. Default default", + ) + + base_group.add_argument( + "--devices", + "--gpus", + "--npus", + "--xpus", + type=str, + default=None, + help="accelerate devices. as --gpus,npus,xpus", + ) base_group.add_argument("--host", type=str, default=None, help="host ip") - base_group.add_argument("--ips", - type=str, - default=None, - help="nodes ips, e.g. 10.10.1.1,10.10.1.2") + base_group.add_argument( + "--ips", + type=str, + default=None, + help="nodes ips, e.g. 10.10.1.1,10.10.1.2", + ) - base_group.add_argument("--start_port", - type=int, - default=6070, - help="fix port start with") + base_group.add_argument( + "--start_port", type=int, default=6070, help="fix port start with" + ) - base_group.add_argument("training_script", - type=str, - help="the full path of py script," - "followed by arguments for the " - "training script") + base_group.add_argument( + "training_script", + type=str, + help="the full path of py script," + "followed by arguments for the " + "training script", + ) base_group.add_argument('training_script_args', nargs=REMAINDER) ps_group = parser.add_argument_group("Parameter-Server Parameters") # for parameter server - ps_group.add_argument("--servers", - type=str, - default='', - help="servers endpoints full list") - ps_group.add_argument("--trainers", - type=str, - default='', - help="trainers endpoints full list") - - ps_group.add_argument("--trainer_num", - type=int, - default=None, - help="number of trainers") - ps_group.add_argument("--server_num", - type=int, - default=None, - help="number of servers") - ps_group.add_argument("--gloo_port", - type=int, - default=6767, - help="gloo http port") - ps_group.add_argument("--with_gloo", - type=str, - default="1", - help="use gloo or not") + ps_group.add_argument( + "--servers", type=str, default='', help="servers endpoints full list" + ) + ps_group.add_argument( + "--trainers", type=str, default='', help="trainers endpoints full list" + ) + + ps_group.add_argument( + "--trainer_num", type=int, default=None, help="number of trainers" + ) + ps_group.add_argument( + "--server_num", type=int, default=None, help="number of servers" + ) + ps_group.add_argument( + "--gloo_port", type=int, default=6767, help="gloo http port" + ) + ps_group.add_argument( + "--with_gloo", type=str, default="1", help="use gloo or not" + ) # parameter elastic mode elastic_group = parser.add_argument_group("Elastic Parameters") - elastic_group.add_argument("--max_restart", - type=int, - default=3, - help="the times can restart. Default 3") + elastic_group.add_argument( + "--max_restart", + type=int, + default=3, + help="the times can restart. Default 3", + ) elastic_group.add_argument( "--elastic_level", type=int, default=-1, - help= - "elastic level: -1 disable, 0 failed exit, peers hold, 1 internal restart" + help="elastic level: -1 disable, 0 failed exit, peers hold, 1 internal restart", ) elastic_group.add_argument( "--elastic_timeout", type=int, default=30, - help="seconds to wait before elastic job begin to train") + help="seconds to wait before elastic job begin to train", + ) return parser.parse_known_args() diff --git a/python/paddle/distributed/launch/context/device.py b/python/paddle/distributed/launch/context/device.py index 14997df2459..f05bc1b7768 100644 --- a/python/paddle/distributed/launch/context/device.py +++ b/python/paddle/distributed/launch/context/device.py @@ -28,7 +28,6 @@ class DeviceType: class Device(object): - def __init__(self, dtype=None, memory="", labels=""): self._dtype = dtype self._memory = memory @@ -102,7 +101,8 @@ class Device(object): if 'PADDLE_XCCL_BACKEND' in os.environ: dev._dtype = DeviceType.CUSTOM_DEVICE visible_devices_str = '{}_VISIBLE_DEVICES'.format( - os.getenv('PADDLE_XCCL_BACKEND').upper()) + os.getenv('PADDLE_XCCL_BACKEND').upper() + ) if visible_devices_str in os.environ: visible_devices = os.getenv(visible_devices_str) elif 'CUDA_VISIBLE_DEVICES' in os.environ: @@ -127,7 +127,6 @@ class Device(object): @classmethod def detect_device(self): - def get_custom_devices_count(device_type): all_custom_devices = get_available_custom_device() all_custom_devices = [ @@ -144,7 +143,8 @@ class Device(object): dev._dtype = DeviceType.CUSTOM_DEVICE num = get_custom_devices_count(custom_device_type) visible_devices_str = '{}_VISIBLE_DEVICES'.format( - custom_device_type.upper()) + custom_device_type.upper() + ) if visible_devices_str in os.environ: visible_devices = os.getenv(visible_devices_str) elif fluid.core.is_compiled_with_cuda(): diff --git a/python/paddle/distributed/launch/context/event.py b/python/paddle/distributed/launch/context/event.py index cb39e1529fc..3859edb83f2 100644 --- a/python/paddle/distributed/launch/context/event.py +++ b/python/paddle/distributed/launch/context/event.py @@ -14,7 +14,6 @@ class Event(object): - def __init__(self, kind="status", message="", fatal=False): self.kind = kind self.message = message diff --git a/python/paddle/distributed/launch/context/node.py b/python/paddle/distributed/launch/context/node.py index e81c91e1bca..b8c1a2a14f5 100644 --- a/python/paddle/distributed/launch/context/node.py +++ b/python/paddle/distributed/launch/context/node.py @@ -22,7 +22,6 @@ from contextlib import closing class Node(object): - def __init__(self): # self.device = Device.detect_device() self.device = Device.parse_device() @@ -54,8 +53,9 @@ class Node(object): def _get_free_port(self, port=0): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) + s.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0) + ) try: s.bind(('', port)) return s.getsockname()[1] @@ -81,8 +81,8 @@ class Node(object): @classmethod def is_server_ready(self, ip, port): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - #sock.settimeout(0.01) - #sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # sock.settimeout(0.01) + # sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) result = sock.connect_ex((ip, int(port))) diff --git a/python/paddle/distributed/launch/context/resource.py b/python/paddle/distributed/launch/context/resource.py index d523c3c5cdf..b7baf6fd7b6 100644 --- a/python/paddle/distributed/launch/context/resource.py +++ b/python/paddle/distributed/launch/context/resource.py @@ -14,6 +14,5 @@ class Resource(object): - def __init__(self): self.devices = [] diff --git a/python/paddle/distributed/launch/controllers/collective.py b/python/paddle/distributed/launch/controllers/collective.py index dd0a3cc34aa..be38b4057f0 100644 --- a/python/paddle/distributed/launch/controllers/collective.py +++ b/python/paddle/distributed/launch/controllers/collective.py @@ -19,7 +19,6 @@ import json class CollectiveController(Controller): - @classmethod def enable(cls, ctx): # collective is the default mode @@ -31,7 +30,11 @@ class CollectiveController(Controller): return False def build_pod(self): - if self.ctx.args.master is None and self.ctx.args.start_port and self.ctx.args.ips: + if ( + self.ctx.args.master is None + and self.ctx.args.start_port + and self.ctx.args.ips + ): self._build_pod_with_args() else: self._build_pod_with_master() @@ -48,15 +51,18 @@ class CollectiveController(Controller): self.ctx.logger.debug("job endpoints: {}".format(job_endpoints)) - rank_offset = ips.index( - self.ctx.node.ip - ) * self.pod.replicas if self.ctx.node.ip in ips else 0 + rank_offset = ( + ips.index(self.ctx.node.ip) * self.pod.replicas + if self.ctx.node.ip in ips + else 0 + ) self.save_pod_log(job_endpoints) selected_dev_key = self.ctx.node.device.get_selected_device_key() selected_dev_list = self.ctx.node.device.get_selected_devices( - self.ctx.args.devices) + self.ctx.args.devices + ) for i in range(self.pod.replicas): e = { @@ -101,19 +107,24 @@ class CollectiveController(Controller): for p in self.ctx.node.get_free_ports(self.pod.replicas) ] - data = json.dumps({ - 'name': self.pod.name, - 'rank': self.pod.rank, - 'replicas': self.pod.replicas, - 'dtype': self.ctx.node.device.dtype, - 'candidate': '{}:{}'.format(self.ctx.node.ip, port), - 'endpoints': ",".join(endpoints), - }) - - peer_list, rank = self.master.sync_peers('/{}/info'.format(self.job.id), - self.pod.name, data, - self.job.replicas, - self.pod.rank) + data = json.dumps( + { + 'name': self.pod.name, + 'rank': self.pod.rank, + 'replicas': self.pod.replicas, + 'dtype': self.ctx.node.device.dtype, + 'candidate': '{}:{}'.format(self.ctx.node.ip, port), + 'endpoints': ",".join(endpoints), + } + ) + + peer_list, rank = self.master.sync_peers( + '/{}/info'.format(self.job.id), + self.pod.name, + data, + self.job.replicas, + self.pod.rank, + ) self.pod.rank = rank if len(peer_list) < 1: @@ -136,7 +147,8 @@ class CollectiveController(Controller): self.pod.reset() selected_dev_key = self.ctx.node.device.get_selected_device_key() selected_dev_list = self.ctx.node.device.get_selected_devices( - self.ctx.args.devices) + self.ctx.args.devices + ) for i in range(self.pod.replicas): e = { "PADDLE_MASTER": collective_master, @@ -170,7 +182,6 @@ class CollectiveController(Controller): class CollectiveElasticController(CollectiveController): - @classmethod def enable(cls, ctx): if ctx.args.master and ctx.args.master.startswith("etcd://"): @@ -200,9 +211,9 @@ class CollectiveElasticController(CollectiveController): self.ctx.logger.info("Waiting peer ready...") - ok, replicas = self.master.wait_peer_ready(self.job.replicas_min, - self.job.replicas_max, - timeout) + ok, replicas = self.master.wait_peer_ready( + self.job.replicas_min, self.job.replicas_max, timeout + ) if ok: self.job.replicas = replicas else: diff --git a/python/paddle/distributed/launch/controllers/controller.py b/python/paddle/distributed/launch/controllers/controller.py index 21b9ce2d5c8..9ff18d5b5d5 100644 --- a/python/paddle/distributed/launch/controllers/controller.py +++ b/python/paddle/distributed/launch/controllers/controller.py @@ -32,7 +32,6 @@ class ControleMode: class ControllerBase(object): - def __init__(self, ctx): signal.signal(signal.SIGTERM, self.signal_handler) signal.signal(signal.SIGABRT, self.signal_handler) @@ -43,9 +42,11 @@ class ControllerBase(object): self.watcher = Watcher(self.ctx) - self.job = Job(nnodes=self.ctx.args.nnodes, - mode=self.ctx.args.run_mode, - jid=self.ctx.args.job_id) + self.job = Job( + nnodes=self.ctx.args.nnodes, + mode=self.ctx.args.run_mode, + jid=self.ctx.args.job_id, + ) self.pod = Pod() self.ctx.set_envs({"POD_NAME": self.pod.name}) @@ -74,14 +75,14 @@ class ControllerBase(object): ''' watch self and peer status, return true to exit ''' - #TODO(kuizhiqing) unify ctx.status and master status + # TODO(kuizhiqing) unify ctx.status and master status self.ctx.logger.info("Watching {}".format(self.pod)) while not self.ctx.status.is_done(): status = self.pod.watch(timeout=2) - #if self.ctx.continous_log(): + # if self.ctx.continous_log(): # default to print log self.pod.logs() @@ -120,8 +121,10 @@ class ControllerBase(object): return False # peer failure - if self.ctx.status.is_restarting( - ) and self.master.get_status() != self.ctx.status.COMPLETED: + if ( + self.ctx.status.is_restarting() + and self.master.get_status() != self.ctx.status.COMPLETED + ): self.pod.stop(timeout=30) return False @@ -190,12 +193,9 @@ class Controller(ControllerBase): err = os.path.join(self.ctx.args.log_dir, err) return out, (err or out) - def new_container(self, - entrypoint=None, - envs={}, - use_ctx_env=True, - out=None, - err=None): + def new_container( + self, entrypoint=None, envs={}, use_ctx_env=True, out=None, err=None + ): c = Container( entrypoint=(entrypoint or self._get_entrypoint()), env=(self.ctx.get_envs() if use_ctx_env else {}), @@ -204,18 +204,19 @@ class Controller(ControllerBase): c.update_env(envs) return c - def add_container(self, - container=None, - entrypoint=None, - envs={}, - log_file=None, - is_init=False): + def add_container( + self, + container=None, + entrypoint=None, + envs={}, + log_file=None, + is_init=False, + ): if not container: - container = self.new_container(entrypoint=entrypoint, - envs=envs, - out=log_file, - err=log_file) + container = self.new_container( + entrypoint=entrypoint, envs=envs, out=log_file, err=log_file + ) if is_init: self.pod.add_init_container(container) @@ -241,8 +242,10 @@ class Controller(ControllerBase): if not self.ctx.args.log_dir: return - f = os.path.join(self.ctx.args.log_dir, - '{}.{}.log'.format(self.job.id, self.pod.name)) + f = os.path.join( + self.ctx.args.log_dir, + '{}.{}.log'.format(self.job.id, self.pod.name), + ) try: os.makedirs(os.path.dirname(f), exist_ok=True) with open(f, 'a+') as fd: diff --git a/python/paddle/distributed/launch/controllers/ipu_controller.py b/python/paddle/distributed/launch/controllers/ipu_controller.py index 92dc2960ab6..ea342e5ebe0 100644 --- a/python/paddle/distributed/launch/controllers/ipu_controller.py +++ b/python/paddle/distributed/launch/controllers/ipu_controller.py @@ -21,7 +21,6 @@ from paddle.distributed.launch.job.container import Container class IPUController(CollectiveController): - @classmethod def enable(cls, ctx): if ctx.args.training_script == "ipu": @@ -33,26 +32,31 @@ class IPUController(CollectiveController): def parse_ipu_args(self, args_list): parser = argparse.ArgumentParser() - parser.add_argument("--hosts", - type=str, - help="The hosts for IPU distributd training.") - parser.add_argument("--nproc_per_host", - type=int, - help="The number of processes launched per host.") - parser.add_argument("--ipus_per_replica", - type=int, - help="The number of IPUs requested per replica.") - parser.add_argument("--ipu_partition", - type=str, - help="The partition name of IPU devices.") - parser.add_argument("--vipu_server", - type=str, - help="The ip of the IPU device manager.") + parser.add_argument( + "--hosts", type=str, help="The hosts for IPU distributd training." + ) + parser.add_argument( + "--nproc_per_host", + type=int, + help="The number of processes launched per host.", + ) + parser.add_argument( + "--ipus_per_replica", + type=int, + help="The number of IPUs requested per replica.", + ) + parser.add_argument( + "--ipu_partition", + type=str, + help="The partition name of IPU devices.", + ) + parser.add_argument( + "--vipu_server", type=str, help="The ip of the IPU device manager." + ) parser.add_argument( "training_script", type=str, - help= - "The full path to the IPU distributed training program/script to be launched in parallel. e.g., ``training.py``." + help="The full path to the IPU distributed training program/script to be launched in parallel. e.g., ``training.py``.", ) parser.add_argument('training_script_args', nargs=argparse.REMAINDER) return parser.parse_args(args_list) @@ -64,19 +68,27 @@ class IPUController(CollectiveController): num_ipus = int(self.ctx.args.devices) # The number of replicas for data parallel - assert (num_ipus % poprun_args.ipus_per_replica) == 0, \ - "The number of IPUs:{} mod the number of IPUs per replica:{} must == 0".format(num_ipus, poprun_args.ipus_per_replica) + assert ( + num_ipus % poprun_args.ipus_per_replica + ) == 0, "The number of IPUs:{} mod the number of IPUs per replica:{} must == 0".format( + num_ipus, poprun_args.ipus_per_replica + ) num_replicas = num_ipus // poprun_args.ipus_per_replica self.ctx.logger.info( - "The number of total replicas is {}.".format(num_replicas)) + "The number of total replicas is {}.".format(num_replicas) + ) # The number of processes num_nodes = len(poprun_args.hosts.split(',')) num_procs = num_nodes * poprun_args.nproc_per_host self.ctx.logger.info( - "The number of total processes is {}.".format(num_procs)) - assert (num_replicas % num_procs) == 0, \ - "The number of replicas:{} mod the number of processes:{} must == 0".format(num_replicas, num_procs) + "The number of total processes is {}.".format(num_procs) + ) + assert ( + num_replicas % num_procs + ) == 0, "The number of replicas:{} mod the number of processes:{} must == 0".format( + num_replicas, num_procs + ) # hosts and endpoints hosts = poprun_args.hosts.replace(' ', '').split(',') @@ -87,26 +99,36 @@ class IPUController(CollectiveController): poprun_command.append('--num-instances={}'.format(num_procs)) poprun_command.append('--num-replicas={}'.format(num_replicas)) - poprun_command.append('--ipus-per-replica={}'.format( - poprun_args.ipus_per_replica)) + poprun_command.append( + '--ipus-per-replica={}'.format(poprun_args.ipus_per_replica) + ) poprun_command.append('--host={}'.format(','.join(hosts))) - poprun_command.append('--vipu-partition={}'.format( - poprun_args.ipu_partition)) - poprun_command.append('--vipu-server-host={}'.format( - poprun_args.vipu_server)) + poprun_command.append( + '--vipu-partition={}'.format(poprun_args.ipu_partition) + ) + poprun_command.append( + '--vipu-server-host={}'.format(poprun_args.vipu_server) + ) - poprun_command.extend([ - '--update-partition=no', '--vipu-server-timeout=120', - '--print-topology=yes', '--numa-aware=yes' - ]) + poprun_command.extend( + [ + '--update-partition=no', + '--vipu-server-timeout=120', + '--print-topology=yes', + '--numa-aware=yes', + ] + ) # global envs global_envs = '--mpi-local-args=\'' log_level = os.getenv('POPART_LOG_LEVEL', None) if log_level: global_envs += '-x POPART_LOG_LEVEL={} '.format(log_level) - global_envs += '-x PADDLE_TRAINERS_NUM={} -x PADDLE_TRAINER_ENDPOINTS={}'.format( - num_procs, ','.join(endpoints)) + global_envs += ( + '-x PADDLE_TRAINERS_NUM={} -x PADDLE_TRAINER_ENDPOINTS={}'.format( + num_procs, ','.join(endpoints) + ) + ) global_envs += '\'' poprun_command.append(global_envs) @@ -115,8 +137,10 @@ class IPUController(CollectiveController): cur_endpoint = endpoints[idx // poprun_args.nproc_per_host] rank_in_node = idx % poprun_args.nproc_per_host poprun_command.append( - '--instance-mpi-local-args={}:\"-x PADDLE_TRAINER_ID={} -x PADDLE_CURRENT_ENDPOINT={} -x PADDLE_RANK_IN_NODE={}\"' - .format(idx, idx, cur_endpoint, rank_in_node)) + '--instance-mpi-local-args={}:\"-x PADDLE_TRAINER_ID={} -x PADDLE_CURRENT_ENDPOINT={} -x PADDLE_RANK_IN_NODE={}\"'.format( + idx, idx, cur_endpoint, rank_in_node + ) + ) # executor poprun_command.append(sys.executable) @@ -142,12 +166,9 @@ class IPUController(CollectiveController): entrypoint = [" ".join(entrypoint)] return entrypoint - def new_container(self, - entrypoint=None, - envs={}, - use_ctx_env=True, - out=None, - err=None): + def new_container( + self, entrypoint=None, envs={}, use_ctx_env=True, out=None, err=None + ): c = Container( entrypoint=(entrypoint or self._get_entrypoint()), env=(self.ctx.get_envs() if use_ctx_env else {}), diff --git a/python/paddle/distributed/launch/controllers/master.py b/python/paddle/distributed/launch/controllers/master.py index da240ceffa9..128852e092c 100644 --- a/python/paddle/distributed/launch/controllers/master.py +++ b/python/paddle/distributed/launch/controllers/master.py @@ -64,7 +64,6 @@ class Master(object): class HTTPMaster(Master): - def lazy_init(self): if self.initialized: return @@ -83,7 +82,8 @@ class HTTPMaster(Master): break except Exception as e: self.ctx.logger.warning( - "start master failed {}".format(e)) + "start master failed {}".format(e) + ) time.sleep(0.1) continue else: @@ -94,7 +94,9 @@ class HTTPMaster(Master): print("Copy the following command to other nodes to run.") cmd = [ - sys.executable.split('/')[-1], "-m", "paddle.distributed.launch" + sys.executable.split('/')[-1], + "-m", + "paddle.distributed.launch", ] cmd.extend(["--master", self.endpoint]) cmd.extend(sys.argv[1:]) @@ -104,7 +106,8 @@ class HTTPMaster(Master): if int(self.ctx.args.rank) >= 0: self.ctx.logger.warning( - "--rank set in the command may not compatible in auto mode") + "--rank set in the command may not compatible in auto mode" + ) if '127.0.0.1' in self.endpoint: self.endpoint = self.endpoint.replace('127.0.0.1', self.ctx.node.ip) @@ -173,7 +176,6 @@ class HTTPMaster(Master): class ETCDMaster(Master): - def __init__(self, ctx): super().__init__(ctx) @@ -225,7 +227,8 @@ class ETCDMaster(Master): ii = int(six.ensure_str(k.key).split('/')[-1]) if ii < 0: self.ctx.logger.error( - "rank {} error in sync".format(ii)) + "rank {} error in sync".format(ii) + ) ret[ii] = six.ensure_str(v) return ret, rank else: @@ -241,7 +244,7 @@ class ETCDMaster(Master): lease = self.client.lease(ttl) - #self.client.delete_prefix(self.job_prefix) + # self.client.delete_prefix(self.job_prefix) beat_path = "{}/{}".format(self.heartbeat_prefix, pod_id) self.client.put(beat_path, pod_id.encode('latin-1'), lease=lease) @@ -250,16 +253,17 @@ class ETCDMaster(Master): self.ctx.status.restart() beat_watch = self.client.add_watch_prefix_callback( - self.heartbeat_prefix, _beat_watch) + self.heartbeat_prefix, _beat_watch + ) def _heartbeat(): while not self.ctx.status.is_done(): try: lease.refresh() if pod_id not in self.fetch_peer_alive(): - self.client.put(beat_path, - pod_id.encode('latin-1'), - lease=lease) + self.client.put( + beat_path, pod_id.encode('latin-1'), lease=lease + ) self.ctx.logger.debug("Heartbeat register again") except Exception as e: self.ctx.logger.error("Heartbeat error {}".format(e)) @@ -267,9 +271,9 @@ class ETCDMaster(Master): self.ctx.logger.debug("Heartbeat done") self.client.cancel_watch(beat_watch) - self.beat_thread = threading.Thread(name='heartbeat', - target=_heartbeat, - daemon=True) + self.beat_thread = threading.Thread( + name='heartbeat', target=_heartbeat, daemon=True + ) self.beat_thread.start() def fetch_peer_alive(self): @@ -311,7 +315,8 @@ class ETCDMaster(Master): assert self.client.put( self.job_prefix, status.encode('latin-1'), - lease=self.client.lease(600)), "set status failed {}".format(status) + lease=self.client.lease(600), + ), "set status failed {}".format(status) def get_status(self): return six.ensure_str(self.client.get(self.job_prefix)[0] or '') @@ -320,4 +325,4 @@ class ETCDMaster(Master): if hasattr(self, 'beat_thread'): self.ctx.status.done() # daemon thread - #self.beat_thread.join() + # self.beat_thread.join() diff --git a/python/paddle/distributed/launch/controllers/ps.py b/python/paddle/distributed/launch/controllers/ps.py index f785311a525..b78ab34d490 100644 --- a/python/paddle/distributed/launch/controllers/ps.py +++ b/python/paddle/distributed/launch/controllers/ps.py @@ -19,12 +19,15 @@ import os, shutil class PSController(Controller): - @classmethod def enable(cls, ctx): - if ctx.args.run_mode == ControleMode.PS or ctx.args.server_num or len( - ctx.args.servers) > 0 or ctx.args.trainer_num or len( - ctx.args.trainers) > 0: + if ( + ctx.args.run_mode == ControleMode.PS + or ctx.args.server_num + or len(ctx.args.servers) > 0 + or ctx.args.trainer_num + or len(ctx.args.trainers) > 0 + ): ctx.logger.debug("{} enabled".format(cls.__name__)) ctx.args.run_mode = ControleMode.PS return True @@ -59,6 +62,7 @@ class PSController(Controller): self.save_pod_log([server_endpoints, trainer_endpoints]) import tempfile + gloo_rendezvous_dir = tempfile.mkdtemp() if os.path.exists(gloo_rendezvous_dir): shutil.rmtree(gloo_rendezvous_dir) @@ -70,7 +74,7 @@ class PSController(Controller): "PADDLE_GLOO_RENDEZVOUS": "3", "PADDLE_GLOO_FS_PATH": gloo_rendezvous_dir, "PADDLE_GLOO_HTTP_ENDPOINT": gloo_http, - "PADDLE_WITH_GLOO": self.ctx.args.with_gloo + "PADDLE_WITH_GLOO": self.ctx.args.with_gloo, } for i in range(server_num): @@ -124,19 +128,24 @@ class PSController(Controller): for p in self.ctx.node.get_free_ports(trainer_num) ] - data = json.dumps({ - 'name': self.pod.name, - 'rank': self.pod.rank, - 'servers': servers, - 'trainers': trainers, - 'dtype': self.ctx.node.device.dtype, - 'gloo_port': self.ctx.node.get_free_port(), - }) + data = json.dumps( + { + 'name': self.pod.name, + 'rank': self.pod.rank, + 'servers': servers, + 'trainers': trainers, + 'dtype': self.ctx.node.device.dtype, + 'gloo_port': self.ctx.node.get_free_port(), + } + ) - peer_list, rank = self.master.sync_peers('/{}/info'.format(self.job.id), - self.pod.name, data, - self.job.replicas, - self.pod.rank) + peer_list, rank = self.master.sync_peers( + '/{}/info'.format(self.job.id), + self.pod.name, + data, + self.job.replicas, + self.pod.rank, + ) self.ctx.logger.debug("sync peers done {}".format(peer_list)) @@ -146,17 +155,19 @@ class PSController(Controller): server_endpoints = [j for i in peer_list for j in i['servers']] trainer_endpoints = [j for i in peer_list for j in i['trainers']] - #rank_offset = sum([i['replicas'] for i in peer_list[:rank]]) + # rank_offset = sum([i['replicas'] for i in peer_list[:rank]]) server_rank_offset = sum([len(i['servers']) for i in peer_list[:rank]]) trainer_rank_offset = sum( - [len(i['trainers']) for i in peer_list[:rank]]) + [len(i['trainers']) for i in peer_list[:rank]] + ) self.pod.rank = rank self.pod.replicas = server_num + trainer_num import tempfile + gloo_rendezvous_dir = tempfile.mkdtemp() if os.path.exists(gloo_rendezvous_dir): shutil.rmtree(gloo_rendezvous_dir) @@ -168,27 +179,21 @@ class PSController(Controller): "PADDLE_GLOO_RENDEZVOUS": "3", "PADDLE_GLOO_FS_PATH": gloo_rendezvous_dir, "PADDLE_GLOO_HTTP_ENDPOINT": gloo_http, - "PADDLE_WITH_GLOO": self.ctx.args.with_gloo + "PADDLE_WITH_GLOO": self.ctx.args.with_gloo, } for i in range(server_num): e = { - "PADDLE_NNODES": - "{}".format(self.job.replicas), - "PADDLE_PSERVERS_IP_PORT_LIST": - ",".join(server_endpoints), - "PADDLE_TRAINER_ENDPOINTS": - ",".join(trainer_endpoints), - "PADDLE_PORT": - server_endpoints[i + server_rank_offset].split(":")[1], - "PADDLE_ROLE": - "PSERVER", - "TRAINING_ROLE": - "PSERVER", - "PADDLE_TRAINERS_NUM": - "{}".format(len(trainer_endpoints)), - "POD_IP": - self.ctx.node.ip, + "PADDLE_NNODES": "{}".format(self.job.replicas), + "PADDLE_PSERVERS_IP_PORT_LIST": ",".join(server_endpoints), + "PADDLE_TRAINER_ENDPOINTS": ",".join(trainer_endpoints), + "PADDLE_PORT": server_endpoints[i + server_rank_offset].split( + ":" + )[1], + "PADDLE_ROLE": "PSERVER", + "TRAINING_ROLE": "PSERVER", + "PADDLE_TRAINERS_NUM": "{}".format(len(trainer_endpoints)), + "POD_IP": self.ctx.node.ip, } e.update(_gloo_envs) log_file = "serverlog.{}".format(i) @@ -196,24 +201,17 @@ class PSController(Controller): for i in range(trainer_num): e = { - "PADDLE_NNODES": - "{}".format(self.job.replicas), - "PADDLE_PSERVERS_IP_PORT_LIST": - ",".join(server_endpoints), - "PADDLE_TRAINER_ENDPOINTS": - ",".join(trainer_endpoints), - "PADDLE_PORT": - trainer_endpoints[i + trainer_rank_offset].split(":")[1], - "PADDLE_ROLE": - "TRAINER", - "TRAINING_ROLE": - "TRAINER", - "PADDLE_TRAINER_ID": - "{}".format(i + trainer_rank_offset), - "PADDLE_TRAINERS_NUM": - "{}".format(len(trainer_endpoints)), - "POD_IP": - self.ctx.node.ip, + "PADDLE_NNODES": "{}".format(self.job.replicas), + "PADDLE_PSERVERS_IP_PORT_LIST": ",".join(server_endpoints), + "PADDLE_TRAINER_ENDPOINTS": ",".join(trainer_endpoints), + "PADDLE_PORT": trainer_endpoints[i + trainer_rank_offset].split( + ":" + )[1], + "PADDLE_ROLE": "TRAINER", + "TRAINING_ROLE": "TRAINER", + "PADDLE_TRAINER_ID": "{}".format(i + trainer_rank_offset), + "PADDLE_TRAINERS_NUM": "{}".format(len(trainer_endpoints)), + "POD_IP": self.ctx.node.ip, } e.update(_gloo_envs) log_file = "workerlog.{}".format(i) diff --git a/python/paddle/distributed/launch/controllers/rpc.py b/python/paddle/distributed/launch/controllers/rpc.py index d68c389da9e..0d3c314ec77 100644 --- a/python/paddle/distributed/launch/controllers/rpc.py +++ b/python/paddle/distributed/launch/controllers/rpc.py @@ -18,7 +18,6 @@ import json class RpcController(Controller): - @classmethod def enable(cls, ctx): if ctx.args.run_mode == ControleMode.RPC: @@ -28,8 +27,9 @@ class RpcController(Controller): return False def build_pod(self): - assert (self.ctx.args.master - is not None), "Master is None, Please set master address!" + assert ( + self.ctx.args.master is not None + ), "Master is None, Please set master address!" self._build_pod_with_master() def _build_pod_with_master(self): @@ -47,14 +47,16 @@ class RpcController(Controller): for p in self.ctx.node.get_free_ports(self.pod.replicas) ] - data = json.dumps({ - "name": self.pod.name, - "rank": self.pod.rank, - "replicas": self.pod.replicas, - "dtype": self.ctx.node.device.dtype, - "candidate": "{}:{}".format(self.ctx.node.ip, port), - "endpoints": ",".join(endpoints), - }) + data = json.dumps( + { + "name": self.pod.name, + "rank": self.pod.rank, + "replicas": self.pod.replicas, + "dtype": self.ctx.node.device.dtype, + "candidate": "{}:{}".format(self.ctx.node.ip, port), + "endpoints": ",".join(endpoints), + } + ) peer_list, rank = self.master.sync_peers( "/{}/info".format(self.job.id), self.pod.name, diff --git a/python/paddle/distributed/launch/controllers/watcher.py b/python/paddle/distributed/launch/controllers/watcher.py index 4b8e346e790..a9c1f509666 100644 --- a/python/paddle/distributed/launch/controllers/watcher.py +++ b/python/paddle/distributed/launch/controllers/watcher.py @@ -20,7 +20,6 @@ from threading import Thread class Watcher(object): - def __init__(self, ctx): self.ctx = ctx @@ -31,8 +30,9 @@ class Watcher(object): # gpu log file self.gpus = self.ctx.args.devices or self.ctx.node.device.labels if len(self.gpus) > 0: - fn = os.path.join(self.ctx.args.log_dir, - "{}.gpu.log".format(self.ctx.args.job_id)) + fn = os.path.join( + self.ctx.args.log_dir, "{}.gpu.log".format(self.ctx.args.job_id) + ) os.makedirs(os.path.dirname(fn), exist_ok=True) self.gpu_fd = open(fn, 'w') else: diff --git a/python/paddle/distributed/launch/job/container.py b/python/paddle/distributed/launch/job/container.py index 55223bacdd8..6eb313ea579 100644 --- a/python/paddle/distributed/launch/job/container.py +++ b/python/paddle/distributed/launch/job/container.py @@ -89,7 +89,8 @@ class Container(object): def _valide_env(self): for k, v in self._env.items(): assert isinstance(k, str) and isinstance( - v, str), 'env {}:{} must be str'.format(k, v) + v, str + ), 'env {}:{} must be str'.format(k, v) def _get_fd(self, pth): if not pth: @@ -120,11 +121,13 @@ class Container(object): self._log_handler.seek(0, 2) self._log_start_offset = self._log_handler.tell() - self._proc = ProcessContext(self._entrypoint, - env=self._env, - out=self._stdout, - err=self._stderr, - shell=self._shell) + self._proc = ProcessContext( + self._entrypoint, + env=self._env, + out=self._stdout, + err=self._stderr, + shell=self._shell, + ) self._proc.start() @@ -159,13 +162,15 @@ class Container(object): return Status.FAILED def __str__(self): - return 'Container rank {} status {} cmd {} code {} log {} \nenv {}'.format( - self._rank, - self.status, - self._entrypoint, - self.exit_code, - self.errfile, - self._env, + return ( + 'Container rank {} status {} cmd {} code {} log {} \nenv {}'.format( + self._rank, + self.status, + self._entrypoint, + self.exit_code, + self.errfile, + self._env, + ) ) def logs(self, fn=None, offset=0, whence=1, limit=1000): diff --git a/python/paddle/distributed/launch/job/job.py b/python/paddle/distributed/launch/job/job.py index 4bad1209c18..f5c805e31bf 100644 --- a/python/paddle/distributed/launch/job/job.py +++ b/python/paddle/distributed/launch/job/job.py @@ -20,7 +20,6 @@ class JobMode: class Job(object): - def __init__(self, jid='default', mode=JobMode.COLLECTIVE, nnodes="1"): self._mode = mode self._id = jid @@ -34,8 +33,13 @@ class Job(object): def __str__(self): return "Job: {}, mode {}, replicas {}[{}:{}], elastic {}".format( - self.id, self.mode, self._replicas, self._replicas_min, - self._replicas_max, self.elastic) + self.id, + self.mode, + self._replicas, + self._replicas_min, + self._replicas_max, + self.elastic, + ) @property def mode(self): @@ -76,6 +80,9 @@ class Job(object): self._elastic = True else: self._replicas = int(np) - self._replicas_min, self._replicas_max = self._replicas, self._replicas + self._replicas_min, self._replicas_max = ( + self._replicas, + self._replicas, + ) self._elastic = False diff --git a/python/paddle/distributed/launch/job/pod.py b/python/paddle/distributed/launch/job/pod.py index 960a52aa3f3..b65aad6e0fc 100644 --- a/python/paddle/distributed/launch/job/pod.py +++ b/python/paddle/distributed/launch/job/pod.py @@ -21,17 +21,17 @@ import time class PodSepc(object): - def __init__(self): self._name = ''.join( - random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6)) + random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6) + ) # by controller self._init_containers: List[Container] = [] self._containers: List[Container] = [] - #self.resource: Resource = None - #self.status: Status = None + # self.resource: Resource = None + # self.status: Status = None self._rank = -1 self._init_timeout = None @@ -41,14 +41,13 @@ class PodSepc(object): class Pod(PodSepc): - def __init__(self): super().__init__() def __str__(self): - return "Pod: {}, replicas {}, status {}".format(self.name, - self.replicas, - self.status) + return "Pod: {}, replicas {}, status {}".format( + self.name, self.replicas, self.status + ) def failed_container(self): cs = [] @@ -183,11 +182,13 @@ class Pod(PodSepc): else: self._containers[idx].tail() - def watch(self, - all_list=[Status.COMPLETED], - any_list=[Status.FAILED], - interval=1, - timeout=-1): + def watch( + self, + all_list=[Status.COMPLETED], + any_list=[Status.FAILED], + interval=1, + timeout=-1, + ): ''' watch return if any container status in any_list or all container status in all_list diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py index a6a0c973208..c960239f733 100644 --- a/python/paddle/distributed/launch/main.py +++ b/python/paddle/distributed/launch/main.py @@ -290,6 +290,7 @@ def launch(): # legacy mode from paddle.distributed.fleet import launch + launch.launch() else: diff --git a/python/paddle/distributed/launch/plugins/__init__.py b/python/paddle/distributed/launch/plugins/__init__.py index 91a2dee09a3..2d7632d642c 100644 --- a/python/paddle/distributed/launch/plugins/__init__.py +++ b/python/paddle/distributed/launch/plugins/__init__.py @@ -27,7 +27,7 @@ def log(ctx): def process_args(ctx): # reset device by args - #argdev = ctx.args.gpus or ctx.args.xpus or ctx.args.npus + # argdev = ctx.args.gpus or ctx.args.xpus or ctx.args.npus argdev = ctx.args.devices if argdev: for d in argdev.split(','): @@ -49,7 +49,8 @@ def collective_compatible(ctx): ctx.args.master = eps[0] if ':' in eps[0] else '{}:6768'.format(eps[0]) ctx.args.nnodes = len(hosts) ctx.logger.info( - 'args reset by env PADDLE_TRAINER_ENDPOINTS\n{}'.format(eps)) + 'args reset by env PADDLE_TRAINER_ENDPOINTS\n{}'.format(eps) + ) if 'DISTRIBUTED_TRAINER_ENDPOINTS' in ctx.envs: eps = ctx.envs['DISTRIBUTED_TRAINER_ENDPOINTS'].split(',') @@ -57,7 +58,8 @@ def collective_compatible(ctx): ctx.args.master = eps[0] ctx.args.nnodes = len(hosts) ctx.logger.info( - 'args reset by env DISTRIBUTED_TRAINER_ENDPOINTS\n{}'.format(eps)) + 'args reset by env DISTRIBUTED_TRAINER_ENDPOINTS\n{}'.format(eps) + ) def rewrite_host_ip(ctx): @@ -72,9 +74,13 @@ def test_mode(ctx): if int(ctx.args.nnodes) < 2: ctx.args.nnodes = 2 ctx.args.training_script = '{}/test.py'.format( - os.path.dirname(__file__)) + os.path.dirname(__file__) + ) enabled_plugins = [ - test_mode, collective_compatible, rewrite_host_ip, process_args + test_mode, + collective_compatible, + rewrite_host_ip, + process_args, ] diff --git a/python/paddle/distributed/launch/plugins/test.py b/python/paddle/distributed/launch/plugins/test.py index ae4c111351e..3dd7ab886e6 100644 --- a/python/paddle/distributed/launch/plugins/test.py +++ b/python/paddle/distributed/launch/plugins/test.py @@ -31,13 +31,12 @@ class_dim = 102 # define a random dataset class RandomDataset(Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([3, 224, 224]).astype('float32') - label = np.random.randint(0, class_dim - 1, (1, )).astype('int64') + label = np.random.randint(0, class_dim - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -49,7 +48,8 @@ def optimizer_setting(parameter_list=None): learning_rate=base_lr, momentum=momentum_rate, weight_decay=paddle.regularizer.L2Decay(l2_decay), - parameters=parameter_list) + parameters=parameter_list, + ) return optimizer @@ -62,11 +62,13 @@ def train_resnet(): resnet = fleet.distributed_model(resnet) dataset = RandomDataset(batch_num * batch_size) - train_loader = DataLoader(dataset, - batch_size=batch_size, - shuffle=True, - drop_last=True, - num_workers=2) + train_loader = DataLoader( + dataset, + batch_size=batch_size, + shuffle=True, + drop_last=True, + num_workers=2, + ) print("Distributed training start...") for eop in range(epoch): @@ -86,14 +88,17 @@ def train_resnet(): optimizer.step() resnet.clear_gradients() - print("[Epoch %d, batch %d] loss: %.5f, acc1: %.5f, acc5: %.5f" % - (eop, batch_id, avg_loss, acc_top1, acc_top5)) + print( + "[Epoch %d, batch %d] loss: %.5f, acc1: %.5f, acc5: %.5f" + % (eop, batch_id, avg_loss, acc_top1, acc_top5) + ) print("Distributed training completed") if __name__ == '__main__': import os + nnodes = os.getenv('PADDLE_NNODES') cn = os.getenv('PADDLE_LOCAL_SIZE') print(f"Prepare distributed training with {nnodes} nodes {cn} cards") diff --git a/python/paddle/distributed/launch/utils/kv_client.py b/python/paddle/distributed/launch/utils/kv_client.py index a66ca800c58..8ed46053de0 100644 --- a/python/paddle/distributed/launch/utils/kv_client.py +++ b/python/paddle/distributed/launch/utils/kv_client.py @@ -17,10 +17,12 @@ import time class KVClient(object): - def __init__(self, endpoint='localhost:2379'): - self.endpoint = endpoint if endpoint.startswith( - "http://") else "http://{}".format(endpoint) + self.endpoint = ( + endpoint + if endpoint.startswith("http://") + else "http://{}".format(endpoint) + ) def put(self, key, value): key = key if key.startswith('/') else "/{}".format(key) diff --git a/python/paddle/distributed/launch/utils/kv_server.py b/python/paddle/distributed/launch/utils/kv_server.py index ddf5685c988..7c8807d3a16 100644 --- a/python/paddle/distributed/launch/utils/kv_server.py +++ b/python/paddle/distributed/launch/utils/kv_server.py @@ -22,7 +22,6 @@ import json class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): - def do_GET(self): with self.server.kv_lock: ret = {} @@ -69,7 +68,6 @@ class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): class KVServer(HTTPServer, object): - def __init__(self, port): super(KVServer, self).__init__(('', port), KVHandler) self.kv_lock = threading.Lock() @@ -90,8 +88,7 @@ class KVServer(HTTPServer, object): self.stopped = True -class PKVServer(): - +class PKVServer: def __init__(self, port): self._server = KVServer(port) @@ -114,11 +111,11 @@ class PKVServer(): if __name__ == '__main__': - #kv = PKVServer(8090) + # kv = PKVServer(8090) kv = KVServer(8090) kv.start() import time - #print("serve at 8090 for 600 s") + # print("serve at 8090 for 600 s") time.sleep(600) diff --git a/python/paddle/distributed/launch/utils/nvsmi.py b/python/paddle/distributed/launch/utils/nvsmi.py index 785704be3ff..d1a14f11666 100644 --- a/python/paddle/distributed/launch/utils/nvsmi.py +++ b/python/paddle/distributed/launch/utils/nvsmi.py @@ -19,7 +19,6 @@ import shutil class Info(object): - def __repr__(self): return str(self.__dict__) @@ -76,20 +75,28 @@ def query_smi(query=None, query_type="gpu", index=None, dtype=None): def get_gpu_info(index=None): q = "index,uuid,driver_version,name,gpu_serial,display_active,display_mode".split( - ",") + "," + ) d = [int, str, str, str, str, str, str] - index = index if index is None or isinstance( - index, list) else str(index).split(",") + index = ( + index + if index is None or isinstance(index, list) + else str(index).split(",") + ) return query_smi(q, index=index, dtype=d) def get_gpu_util(index=None): q = "index,utilization.gpu,memory.total,memory.used,memory.free,timestamp".split( - ",") + "," + ) d = [int, int, int, int, int, str] - index = index if index is None or isinstance( - index, list) else str(index).split(",") + index = ( + index + if index is None or isinstance(index, list) + else str(index).split(",") + ) return query_smi(q, index=index, dtype=d) @@ -97,8 +104,11 @@ def get_gpu_util(index=None): def get_gpu_process(index=None): q = "pid,process_name,gpu_uuid,gpu_name,used_memory".split(",") d = [int, str, str, str, int] - index = index if index is None or isinstance( - index, list) else str(index).split(",") + index = ( + index + if index is None or isinstance(index, list) + else str(index).split(",") + ) return query_smi(q, index=index, query_type="compute", dtype=d) diff --git a/python/paddle/distributed/launch/utils/process_context.py b/python/paddle/distributed/launch/utils/process_context.py index 5d8505aa66e..682a857f2ee 100644 --- a/python/paddle/distributed/launch/utils/process_context.py +++ b/python/paddle/distributed/launch/utils/process_context.py @@ -17,15 +17,16 @@ import os, sys, signal, time class ProcessContext(object): - - def __init__(self, - cmd, - env=os.environ, - out=sys.stdout, - err=sys.stderr, - group=True, - preexec_fn=None, - shell=False): + def __init__( + self, + cmd, + env=os.environ, + out=sys.stdout, + err=sys.stderr, + group=True, + preexec_fn=None, + shell=False, + ): self._cmd = cmd self._env = env self._preexec_fn = preexec_fn @@ -38,12 +39,14 @@ class ProcessContext(object): def _start(self): pre_fn = os.setsid if self._group else None - self._proc = subprocess.Popen(self._cmd, - env=self._env, - stdout=self._stdout, - stderr=self._stderr, - preexec_fn=self._preexec_fn or pre_fn, - shell=self._shell) + self._proc = subprocess.Popen( + self._cmd, + env=self._env, + stdout=self._stdout, + stderr=self._stderr, + preexec_fn=self._preexec_fn or pre_fn, + shell=self._shell, + ) def _close_std(self): try: diff --git a/python/paddle/distributed/metric/metrics.py b/python/paddle/distributed/metric/metrics.py index 9f8573183b3..64f62d85251 100644 --- a/python/paddle/distributed/metric/metrics.py +++ b/python/paddle/distributed/metric/metrics.py @@ -22,15 +22,17 @@ logger = get_logger(logging.INFO, name="metrics") # read metric config from yaml and init MetricMsg in fleet_wrapper -def init_metric(metric_ptr, - metric_yaml_path, - cmatch_rank_var="", - mask_var="", - uid_var="", - phase=-1, - cmatch_rank_group="", - ignore_rank=False, - bucket_size=1000000): +def init_metric( + metric_ptr, + metric_yaml_path, + cmatch_rank_var="", + mask_var="", + uid_var="", + phase=-1, + cmatch_rank_group="", + ignore_rank=False, + bucket_size=1000000, +): yaml_fobj = open(metric_yaml_path) if sys.version.startswith('2.7.13'): content = yaml.load(yaml_fobj) @@ -49,53 +51,102 @@ def init_metric(metric_ptr, phase = 1 if is_join else 0 if metric_runner['method'] == 'AucCalculator': - metric_ptr.init_metric(metric_runner['method'], - metric_runner['name'], - metric_runner['label'], - metric_runner['target'], cmatch_rank_var, - mask_var, uid_var, phase, cmatch_rank_group, - ignore_rank, bucket_size) + metric_ptr.init_metric( + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + cmatch_rank_var, + mask_var, + uid_var, + phase, + cmatch_rank_group, + ignore_rank, + bucket_size, + ) elif metric_runner['method'] == 'MultiTaskAucCalculator': metric_ptr.init_metric( - metric_runner['method'], metric_runner['name'], - metric_runner['label'], metric_runner['target'], - metric_runner['cmatch_var'], mask_var, uid_var, phase, - metric_runner['cmatch_group'], ignore_rank, bucket_size) + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + metric_runner['cmatch_var'], + mask_var, + uid_var, + phase, + metric_runner['cmatch_group'], + ignore_rank, + bucket_size, + ) elif metric_runner['method'] == 'CmatchRankAucCalculator': metric_ptr.init_metric( - metric_runner['method'], metric_runner['name'], - metric_runner['label'], metric_runner['target'], - metric_runner['cmatch_var'], mask_var, uid_var, phase, - metric_runner['cmatch_group'], metric_runner['ignore_rank'], - bucket_size) + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + metric_runner['cmatch_var'], + mask_var, + uid_var, + phase, + metric_runner['cmatch_group'], + metric_runner['ignore_rank'], + bucket_size, + ) elif metric_runner['method'] == 'MaskAucCalculator': - metric_ptr.init_metric(metric_runner['method'], - metric_runner['name'], - metric_runner['label'], - metric_runner['target'], cmatch_rank_var, - metric_runner['mask'], uid_var, phase, - cmatch_rank_group, ignore_rank, bucket_size) + metric_ptr.init_metric( + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + cmatch_rank_var, + metric_runner['mask'], + uid_var, + phase, + cmatch_rank_group, + ignore_rank, + bucket_size, + ) elif metric_runner['method'] == 'CmatchRankMaskAucCalculator': metric_ptr.init_metric( - metric_runner['method'], metric_runner['name'], - metric_runner['label'], metric_runner['target'], - metric_runner['cmatch_var'], metric_runner['mask'], uid_var, - phase, metric_runner['cmatch_group'], - metric_runner['ignore_rank'], bucket_size) + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + metric_runner['cmatch_var'], + metric_runner['mask'], + uid_var, + phase, + metric_runner['cmatch_group'], + metric_runner['ignore_rank'], + bucket_size, + ) elif metric_runner['method'] == 'WuAucCalculator': - metric_ptr.init_metric(metric_runner['method'], - metric_runner['name'], - metric_runner['label'], - metric_runner['target'], cmatch_rank_var, - mask_var, metric_runner['uid'], phase, - cmatch_rank_group, ignore_rank, bucket_size) + metric_ptr.init_metric( + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + cmatch_rank_var, + mask_var, + metric_runner['uid'], + phase, + cmatch_rank_group, + ignore_rank, + bucket_size, + ) else: - metric_ptr.init_metric(metric_runner['method'], - metric_runner['name'], - metric_runner['label'], - metric_runner['target'], cmatch_rank_var, - mask_var, phase, cmatch_rank_group, - ignore_rank, bucket_size) + metric_ptr.init_metric( + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + cmatch_rank_var, + mask_var, + phase, + cmatch_rank_group, + ignore_rank, + bucket_size, + ) def print_metric(metric_ptr, name): @@ -104,14 +155,27 @@ def print_metric(metric_ptr, name): """ if name.find("wuauc") != -1: metric = metric_ptr.get_wuauc_metric_msg(name) - monitor_msg = "%s: User Count=%.0f INS Count=%.0f UAUC=%.6f WUAUC=%.6f "\ - % (name, metric[0], metric[1], metric[4], metric[5]) + monitor_msg = ( + "%s: User Count=%.0f INS Count=%.0f UAUC=%.6f WUAUC=%.6f " + % (name, metric[0], metric[1], metric[4], metric[5]) + ) else: metric = metric_ptr.get_metric_msg(name) - monitor_msg = "%s: AUC=%.6f BUCKET_ERROR=%.6f MAE=%.6f RMSE=%.6f "\ - "Actual CTR=%.6f Predicted CTR=%.6f COPC=%.6f INS Count=%.0f"\ - % (name, metric[0], metric[1], metric[2], metric[3], metric[4], - metric[5], metric[6], metric[7]) + monitor_msg = ( + "%s: AUC=%.6f BUCKET_ERROR=%.6f MAE=%.6f RMSE=%.6f " + "Actual CTR=%.6f Predicted CTR=%.6f COPC=%.6f INS Count=%.0f" + % ( + name, + metric[0], + metric[1], + metric[2], + metric[3], + metric[4], + metric[5], + metric[6], + metric[7], + ) + ) # logger.info(monitor_msg) return monitor_msg diff --git a/python/paddle/distributed/models/moe/utils.py b/python/paddle/distributed/models/moe/utils.py index 4c6ac503449..2011e558701 100644 --- a/python/paddle/distributed/models/moe/utils.py +++ b/python/paddle/distributed/models/moe/utils.py @@ -51,10 +51,12 @@ def _number_count(numbers, upper_range): helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=numbers.dtype) - helper.append_op(type=op_type, - inputs={'numbers': numbers}, - outputs={'Out': out}, - attrs={'upper_range': upper_range}) + helper.append_op( + type=op_type, + inputs={'numbers': numbers}, + outputs={'Out': out}, + attrs={'upper_range': upper_range}, + ) return out @@ -98,29 +100,31 @@ def _assign_pos(x, cum_count): helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=cum_count.dtype) - helper.append_op(type=op_type, - inputs={ - 'X': [x], - 'cum_count': [cum_count], - "eff_num_len": [cum_count[-1]] - }, - outputs={'Out': [out]}) + helper.append_op( + type=op_type, + inputs={ + 'X': [x], + 'cum_count': [cum_count], + "eff_num_len": [cum_count[-1]], + }, + outputs={'Out': [out]}, + ) return out def _random_routing(topk_idx, topk_value, prob, topk=2): r""" - random routing topk gate idx - ``` - out = topk_idx - for i in len(topk_idx): - if topk * value[i][topk-1] < prob[i]: - out[i][topk-1] = -1 - ``` - Args: - topk_idx: gate idx, shape=(N, topk) - topk_value: values, shape = topk_idx.shape - prob: random prob, shape=(topk_idx.shape[0],) + random routing topk gate idx + ``` + out = topk_idx + for i in len(topk_idx): + if topk * value[i][topk-1] < prob[i]: + out[i][topk-1] = -1 + ``` + Args: + topk_idx: gate idx, shape=(N, topk) + topk_value: values, shape = topk_idx.shape + prob: random prob, shape=(topk_idx.shape[0],) """ if topk == 2: if in_dygraph_mode(): @@ -155,25 +159,27 @@ def _limit_by_capacity(expert_count, capacity, n_worker): print(out) # the result: [1, 2, 2, 4, 3, 3] """ if in_dygraph_mode(): - return _legacy_C_ops.limit_by_capacity(expert_count, capacity, - 'n_worker', n_worker) + return _legacy_C_ops.limit_by_capacity( + expert_count, capacity, 'n_worker', n_worker + ) elif _in_legacy_dygraph(): - return core.ops.limit_by_capacity(expert_count, capacity, 'n_worker', - n_worker) + return core.ops.limit_by_capacity( + expert_count, capacity, 'n_worker', n_worker + ) else: op_type = 'limit_by_capacity' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference( - dtype=expert_count.dtype) - - helper.append_op(type=op_type, - inputs={ - 'expert_count': expert_count, - 'capacity': capacity - }, - outputs={'Out': out}, - attrs={'n_worker': n_worker}) + dtype=expert_count.dtype + ) + + helper.append_op( + type=op_type, + inputs={'expert_count': expert_count, 'capacity': capacity}, + outputs={'Out': out}, + attrs={'n_worker': n_worker}, + ) return out @@ -202,30 +208,35 @@ def _prune_gate_by_capacity(gate_idx, expert_count, n_expert, n_worker): [1, 3, 3, 3, -1, 2, 1, 1]) """ if in_dygraph_mode(): - return _legacy_C_ops.prune_gate_by_capacity(gate_idx, expert_count, - "n_expert", n_expert, - "n_worker", n_worker) + return _legacy_C_ops.prune_gate_by_capacity( + gate_idx, expert_count, "n_expert", n_expert, "n_worker", n_worker + ) elif _in_legacy_dygraph(): - return core.ops.prune_gate_by_capacity(gate_idx, expert_count, - "n_expert", n_expert, "n_worker", - n_worker) - check_variable_and_dtype(gate_idx, 'GateIdx', ['int32', 'int64'], - 'paddle.distributed.utils.prune_gate_by_capacity') - check_variable_and_dtype(expert_count, 'ExpertCount', ['int32', 'int64'], - 'paddle.distributed.utils.prune_gate_by_capacity') + return core.ops.prune_gate_by_capacity( + gate_idx, expert_count, "n_expert", n_expert, "n_worker", n_worker + ) + check_variable_and_dtype( + gate_idx, + 'GateIdx', + ['int32', 'int64'], + 'paddle.distributed.utils.prune_gate_by_capacity', + ) + check_variable_and_dtype( + expert_count, + 'ExpertCount', + ['int32', 'int64'], + 'paddle.distributed.utils.prune_gate_by_capacity', + ) helper = LayerHelper('prune_gate_by_capacity', **locals()) new_gate_idx = helper.create_variable_for_type_inference( - dtype=gate_idx.dtype) - helper.append_op(type='prune_gate_by_capacity', - inputs={ - 'GateIdx': gate_idx, - "ExpertCount": expert_count - }, - outputs={'NewGateIdx': new_gate_idx}, - attrs={ - "n_expert": n_expert, - "n_worker": n_worker - }) + dtype=gate_idx.dtype + ) + helper.append_op( + type='prune_gate_by_capacity', + inputs={'GateIdx': gate_idx, "ExpertCount": expert_count}, + outputs={'NewGateIdx': new_gate_idx}, + attrs={"n_expert": n_expert, "n_worker": n_worker}, + ) return new_gate_idx diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py index 49cb60349f7..2a74e6bb2cf 100644 --- a/python/paddle/distributed/parallel.py +++ b/python/paddle/distributed/parallel.py @@ -26,7 +26,9 @@ from paddle.fluid.framework import _set_expected_place from paddle.fluid.dygraph import parallel_helper from paddle.distributed.fleet.launch_utils import check_backend from paddle.fluid.dygraph.parallel import ParallelEnv -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready # noqa: F401 +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) # noqa: F401 from paddle.distributed.collective import _set_group_map from paddle.distributed.collective import _set_group_map_by_name from paddle.distributed.collective import _get_group_map_by_name @@ -57,6 +59,7 @@ def _get_global_parallel_env(): def _start_kv_server(port, http_server_d, size): from paddle.distributed.fleet.utils.http_server import KVServer + http_server = KVServer(int(port), size=size) http_server.start() wait_seconds = 3 @@ -67,10 +70,15 @@ def _start_kv_server(port, http_server_d, size): def _is_cpuonly(backend): check_backend(backend) - if (backend in ['auto', 'nccl', 'bkcl', 'hccl', 'heter', 'cncl'] and - (core.is_compiled_with_cuda() or core.is_compiled_with_xpu() - or core.is_compiled_with_npu() - or core.is_compiled_with_mlu())) or backend == 'xccl': + if ( + backend in ['auto', 'nccl', 'bkcl', 'hccl', 'heter', 'cncl'] + and ( + core.is_compiled_with_cuda() + or core.is_compiled_with_xpu() + or core.is_compiled_with_npu() + or core.is_compiled_with_mlu() + ) + ) or backend == 'xccl': # passes 'auto' and can use cuda or xpu, use the default logics. so return False return False @@ -81,9 +89,10 @@ def _is_cpuonly(backend): def _check_var_exists(var_name): var = os.environ.get(var_name, None) if var is None: - raise ValueError("paddle.distributed initialize error, " - "environment variable %s is needed, but not set." % - var_name) + raise ValueError( + "paddle.distributed initialize error, " + "environment variable %s is needed, but not set." % var_name + ) def init_parallel_env(): @@ -161,15 +170,21 @@ def init_parallel_env(): backend = os.environ.get('PADDLE_DISTRI_BACKEND', 'auto') is_cpu_only = _is_cpuonly(backend) # 1. gpu xpu check, must be gpu or xpu, - if not (is_cpu_only or core.is_compiled_with_cuda() - or core.is_compiled_with_xpu() or core.is_compiled_with_npu() - or core.is_compiled_with_mlu()): + if not ( + is_cpu_only + or core.is_compiled_with_cuda() + or core.is_compiled_with_xpu() + or core.is_compiled_with_npu() + or core.is_compiled_with_mlu() + ): raise NotImplementedError( - "If you want to use CPU-only version, please use 'gloo' as backend") + "If you want to use CPU-only version, please use 'gloo' as backend" + ) if backend == "xccl": FLAGS_selected_custom_devices = 'FLAGS_selected_{}s'.format( - parallel_env.device_type) + parallel_env.device_type + ) _check_var_exists(FLAGS_selected_custom_devices) else: if not is_cpu_only and core.is_compiled_with_cuda(): @@ -197,8 +212,9 @@ def init_parallel_env(): # they need to call a function to change default place, # here just set correctly place to users if backend == "xccl": - place = core.CustomPlace(parallel_env.device_type, - parallel_env.device_id) + place = core.CustomPlace( + parallel_env.device_type, parallel_env.device_id + ) elif is_cpu_only: place = core.CPUPlace() elif core.is_compiled_with_cuda(): @@ -222,11 +238,15 @@ def init_parallel_env(): assert rank >= 0 and world_size > rank and world_size > 1, ( "rank must be non-negative and world_size must be the " "maximum rank plus one. Moreover, at least two processes are " - "required to create a process group.") + "required to create a process group." + ) master_addr = os.getenv("MASTER_ADDR", None) master_port = os.getenv("MASTER_PORT", None) - endpoints = ":".join([master_addr, master_port - ]) if master_addr and master_port else None + endpoints = ( + ":".join([master_addr, master_port]) + if master_addr and master_port + else None + ) if endpoints is None: endpoints = os.getenv("PADDLE_MASTER", None) if endpoints is None: @@ -235,23 +255,28 @@ def init_parallel_env(): "The environment variable 'MASTER_ADDR' and 'MASTER_PORT' " "must be specified, for example 'export MASTER_ADDR=127.0.0.1' " "and 'export MASTER_ADDR=54612'. Or you can start your training" - "with paddle.distributed.run module.") + "with paddle.distributed.run module." + ) master_addr, master_port = endpoints.split(":") master_port = int(master_port) is_master = rank == 0 stop_check_timeout = int(os.getenv("FLAGS_stop_check_timeout", "900")) - default_store = core.TCPStore(master_addr, - master_port, - is_master, - world_size, - timeout=stop_check_timeout) + default_store = core.TCPStore( + master_addr, + master_port, + is_master, + world_size, + timeout=stop_check_timeout, + ) _set_default_store(default_store) - pg = _new_process_group_impl(backend, - default_store, - rank, - world_size, - _default_group_name, - pg_options=None) + pg = _new_process_group_impl( + backend, + default_store, + rank, + world_size, + _default_group_name, + pg_options=None, + ) ranks = list(range(world_size)) group = Group(rank, 0, ranks, pg=pg, name=_default_group_name) _set_group_map_by_name(_default_group_name, group) @@ -277,8 +302,10 @@ def init_parallel_env(): size = {'_worker': parallel_env.world_size} if backend == "heter": size = {'_worker': len(node_num)} - http_server = Process(target=_start_kv_server, - args=(int(ep_rank_0[1]), http_server_d, size)) + http_server = Process( + target=_start_kv_server, + args=(int(ep_rank_0[1]), http_server_d, size), + ) http_server.daemon = True http_server_d["running"] = True http_server.start() @@ -296,22 +323,28 @@ def init_parallel_env(): # init nccl or hccl or bkcl or heter context if is_cpu_only: parallel_helper._set_parallel_ctx( - core.GLOOParallelContext(strategy, place)) - elif (backend == "heter"): + core.GLOOParallelContext(strategy, place) + ) + elif backend == "heter": parallel_helper._set_parallel_ctx( - core.HeterParallelContext(strategy, parallel_env.device_id)) + core.HeterParallelContext(strategy, parallel_env.device_id) + ) elif core.is_compiled_with_cuda(): parallel_helper._set_parallel_ctx( - core.NCCLParallelContext(strategy, place)) + core.NCCLParallelContext(strategy, place) + ) elif core.is_compiled_with_xpu(): parallel_helper._set_parallel_ctx( - core.BKCLParallelContext(strategy, place)) + core.BKCLParallelContext(strategy, place) + ) elif core.is_compiled_with_npu(): parallel_helper._set_parallel_ctx( - core.HCCLParallelContext(strategy, place)) + core.HCCLParallelContext(strategy, place) + ) elif core.is_compiled_with_mlu(): parallel_helper._set_parallel_ctx( - core.CNCLParallelContext(strategy, place)) + core.CNCLParallelContext(strategy, place) + ) if backend != "heter": other_endpoints = strategy.trainer_endpoints[:] diff --git a/python/paddle/distributed/parallel_with_gloo.py b/python/paddle/distributed/parallel_with_gloo.py index a5630239948..eeb91332070 100755 --- a/python/paddle/distributed/parallel_with_gloo.py +++ b/python/paddle/distributed/parallel_with_gloo.py @@ -17,7 +17,9 @@ from multiprocessing import Process, Manager # deprecated module import from paddle.fluid import core -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) __all__ = [] @@ -26,6 +28,7 @@ _global_gloo_ctx = None def _start_kv_server(port, http_server_d, size): from paddle.distributed.fleet.utils.http_server import KVServer + http_server = KVServer(int(port), size=size) http_server.start() wait_seconds = 3 @@ -89,8 +92,9 @@ def gloo_init_parallel_env(rank_id, rank_num, server_endpoint): test_gloo_init_with_multiprocess(2) """ - assert (rank_num < 2) is False, \ - "rank_num should greater than or equal to 2 for parallel environment initialzation." + assert ( + rank_num < 2 + ) is False, "rank_num should greater than or equal to 2 for parallel environment initialzation." # init gloo context manager = Manager() @@ -100,9 +104,10 @@ def gloo_init_parallel_env(rank_id, rank_num, server_endpoint): if rank_id == 0: # The scope for worker used by http server is '_worker' size = {'_worker': rank_num} - http_server_proc = Process(target=_start_kv_server, - args=(int(server_endpoint.split(":")[1]), - http_server_status, size)) + http_server_proc = Process( + target=_start_kv_server, + args=(int(server_endpoint.split(":")[1]), http_server_status, size), + ) http_server_proc.daemon = True http_server_status["running"] = True http_server_proc.start() diff --git a/python/paddle/distributed/passes/auto_parallel_amp.py b/python/paddle/distributed/passes/auto_parallel_amp.py index dc51cced37d..3305982f505 100644 --- a/python/paddle/distributed/passes/auto_parallel_amp.py +++ b/python/paddle/distributed/passes/auto_parallel_amp.py @@ -18,25 +18,48 @@ from paddle.fluid import unique_name from .pass_base import PassBase, register_pass from paddle.distributed.fleet.meta_optimizers.common import OpRole from paddle.fluid.data_feeder import check_variable_and_dtype, check_type -from paddle.distributed.auto_parallel.utils import get_loss_op, set_var_dist_attr -from paddle.distributed.auto_parallel.utils import naive_set_dist_op_attr_for_program_by_mesh_and_mapping -from paddle.distributed.auto_parallel.process_group import get_world_process_group -from paddle.fluid.contrib.mixed_precision.fp16_utils import AutoMixedPrecisionLists -from paddle.fluid.contrib.mixed_precision.fp16_utils import _keep_fp32_input, _keep_fp32_output, find_op_index -from paddle.fluid.contrib.mixed_precision.fp16_utils import _valid_types, find_true_post_op, find_true_prev_op -from paddle.fluid.contrib.mixed_precision.fp16_utils import _is_in_black_varnames, _dtype_to_str, _rename_arg -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute +from paddle.distributed.auto_parallel.utils import ( + get_loss_op, + set_var_dist_attr, +) +from paddle.distributed.auto_parallel.utils import ( + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) +from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + AutoMixedPrecisionLists, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _keep_fp32_input, + _keep_fp32_output, + find_op_index, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _valid_types, + find_true_post_op, + find_true_prev_op, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _is_in_black_varnames, + _dtype_to_str, + _rename_arg, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) from ..auto_parallel.utils import is_forward_op, is_backward_op, is_loss_op world_process_group = get_world_process_group() class AMPState(object): - def __init__(self, block): self._block = block - self._op_fp16_dict = { - } # op_id --> True/False. 'True' means that the current op is in fp16 mode. + self._op_fp16_dict = ( + {} + ) # op_id --> True/False. 'True' means that the current op is in fp16 mode. self._var_name_dict = {} # fwd_op_id --> {old_name: cast_name} self.is_train = False @@ -55,7 +78,8 @@ class AMPState(object): elif int(op.attr('op_role')) == int(OpRole.Backward): if op.desc.original_id() in dist_op_context.grad_op_id_to_op_id: fwd_op_id = dist_op_context.grad_op_id_to_op_id[ - op.desc.original_id()] + op.desc.original_id() + ] if self._is_fp16_op(fwd_op_id) == True: self._op_fp16_dict[op.desc.original_id()] = True elif self._is_fp16_op(fwd_op_id) == False: @@ -78,7 +102,8 @@ class AMPState(object): if op.type == 'create_py_reader' or op.type == 'read': continue if amp_lists.black_varnames is not None and _is_in_black_varnames( - op, amp_lists): + op, amp_lists + ): self._op_fp16_dict[op.desc.original_id()] = False continue if op.type in amp_lists.black_list: @@ -98,17 +123,24 @@ class AMPState(object): continue elif in_var.op is op: prev_op = find_true_prev_op( - ops, op, in_var_name) + ops, op, in_var_name + ) if prev_op is None: continue else: prev_op = in_var.op # if it's one of inputs - if self._is_fp16_op(prev_op.desc.original_id()) == False or \ - prev_op.type in amp_lists.black_list: + if ( + self._is_fp16_op(prev_op.desc.original_id()) + == False + or prev_op.type in amp_lists.black_list + ): is_black_op = True - elif self._is_fp16_op(prev_op.desc.original_id()) == True or \ - prev_op.type in amp_lists.white_list: + elif ( + self._is_fp16_op(prev_op.desc.original_id()) + == True + or prev_op.type in amp_lists.white_list + ): is_white_op = True if is_black_op: self._op_fp16_dict[op.desc.original_id()] = False @@ -131,19 +163,28 @@ class AMPState(object): break if self._is_fp16_op(op.desc.original_id()) == False: num_cast_ops = self._insert_cast_op_forward( - op, idx, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32, dist_context) + op, + idx, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + dist_context, + ) elif self._is_fp16_op(op.desc.original_id()) == True: num_cast_ops = self._insert_cast_op_forward( - op, idx, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, dist_context) + op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + dist_context, + ) else: pass idx += num_cast_ops + 1 self._block._sync_with_cpp() - def _insert_cast_op_forward(self, op, idx, src_dtype, dst_dtype, - dist_context): + def _insert_cast_op_forward( + self, op, idx, src_dtype, dst_dtype, dist_context + ): """ only for forward cast modified from paddle.fluid.contrib.mixed_precision @@ -152,38 +193,45 @@ class AMPState(object): var_name_dict = {} for in_name in op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_input( - op, in_name): + op, in_name + ): continue for in_var_name in op.input(in_name): in_var = self._block._find_var_recursive(in_var_name) if in_var.type not in _valid_types or in_var.dtype == dst_dtype: continue if in_var.dtype == src_dtype: - cast_name = in_var.name + '.cast_' + _dtype_to_str( - dst_dtype) + cast_name = ( + in_var.name + '.cast_' + _dtype_to_str(dst_dtype) + ) out_var = self._block.vars.get(cast_name) var_name_dict[in_var.name] = cast_name consume_op_attr = dist_context.get_op_dist_attr_for_program( - op) + op + ) assert consume_op_attr is not None if out_var is None or out_var.dtype != dst_dtype: # NOTE we make the cast op and var's dist attr as the op that consume the # cast var instead of the op which generates the var in_var_dist_attr = consume_op_attr.get_input_dist_attr( - in_var.name) + in_var.name + ) assert in_var_dist_attr is not None ref_mesh = in_var_dist_attr.process_mesh ref_mapping = in_var_dist_attr.dims_mapping consume_op_attr.set_input_dist_attr( - cast_name, in_var_dist_attr) + cast_name, in_var_dist_attr + ) out_var = self._block.create_var( name=cast_name, dtype=dst_dtype, persistable=False, - stop_gradient=in_var.stop_gradient) - set_var_dist_attr(dist_context, out_var, ref_mapping, - ref_mesh) + stop_gradient=in_var.stop_gradient, + ) + set_var_dist_attr( + dist_context, out_var, ref_mapping, ref_mesh + ) cast_op = self._block._insert_op_without_sync( idx, @@ -193,22 +241,29 @@ class AMPState(object): attrs={ "in_dtype": in_var.dtype, "out_dtype": out_var.dtype, - }) + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, ref_mapping, dist_context) + cast_op, ref_mesh, ref_mapping, dist_context + ) num_cast_ops += 1 else: in_var_dist_attr = consume_op_attr.get_input_dist_attr( - in_var.name) + in_var.name + ) consume_op_attr.set_input_dist_attr( - cast_name, in_var_dist_attr) + cast_name, in_var_dist_attr + ) _rename_arg(op, in_var.name, cast_name) else: if op.has_attr('in_dtype'): op._set_attr('in_dtype', dst_dtype) self._var_name_dict[op.desc.original_id()] = var_name_dict - if src_dtype == core.VarDesc.VarType.FP32 and dst_dtype == core.VarDesc.VarType.FP16: + if ( + src_dtype == core.VarDesc.VarType.FP32 + and dst_dtype == core.VarDesc.VarType.FP16 + ): for out_name in op.output_names: if _keep_fp32_output(op, out_name): continue @@ -238,8 +293,9 @@ class AMPState(object): # NOTE: the map in `grad_var_to_var` may be changed when the var is casted, # which will affect the dist_op to insert allreduce_sum op. op_dist_attr = dist_context.get_op_dist_attr_for_program(grad_op) - if is_backward_op(grad_op) and (is_forward_op(ops[idx - 1]) - or is_loss_op(ops[idx - 1])): + if is_backward_op(grad_op) and ( + is_forward_op(ops[idx - 1]) or is_loss_op(ops[idx - 1]) + ): if not op_dist_attr.is_recompute: appended_grad_times += 1 @@ -248,14 +304,22 @@ class AMPState(object): if grad_op_orig_id in dist_op_context.grad_op_id_to_op_id: if self._is_fp16_op(grad_op_orig_id) == False: # fp32 num_cast_ops = self._insert_cast_op_backward( - grad_op, idx, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32, dist_context, - appended_grad_times) + grad_op, + idx, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + dist_context, + appended_grad_times, + ) elif self._is_fp16_op(grad_op_orig_id) == True: # fp16 num_cast_ops = self._insert_cast_op_backward( - grad_op, idx, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, dist_context, - appended_grad_times) + grad_op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + dist_context, + appended_grad_times, + ) elif grad_op.type == "sum": in_var_name = grad_op.desc.input_arg_names()[0] src_dtype = self._block.var(in_var_name).dtype @@ -270,15 +334,24 @@ class AMPState(object): else: raise ValueError( "'{}' op is not supported in the complete amp pass.".format( - grad_op.type)) + grad_op.type + ) + ) idx += num_cast_ops + 1 self._block._sync_with_cpp() _update_backward_cast_ops(params_grads, dist_context) - def _insert_cast_op_backward(self, grad_op, idx, src_dtype, dst_dtype, - dist_context, appended_grad_times): - """ only for backward cast """ + def _insert_cast_op_backward( + self, + grad_op, + idx, + src_dtype, + dst_dtype, + dist_context, + appended_grad_times, + ): + """only for backward cast""" def _keep_fp32_input(op, in_name): op_type = op.type @@ -299,7 +372,8 @@ class AMPState(object): for in_name in grad_op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_input( - grad_op, in_name): + grad_op, in_name + ): for in_var_name in grad_op.input(in_name): in_var = self._block._find_var_recursive(in_var_name) assert in_var.dtype == core.VarDesc.VarType.FP32 @@ -309,24 +383,34 @@ class AMPState(object): in_var = self._block._find_var_recursive(in_var_name) if in_var.dtype == src_dtype: consume_op_attr = dist_context.get_op_dist_attr_for_program( - grad_op) + grad_op + ) if in_var_name in self._var_name_dict[fwd_op_id]: # NOTE: if in_var of consume grad_op has been casted before, # it should be renamed and reset dist_attr. cast_name = self._var_name_dict[fwd_op_id][in_var_name] grad_op.desc._rename_input(in_var_name, cast_name) in_var_dist_attr = consume_op_attr.get_input_dist_attr( - in_var_name) + in_var_name + ) consume_op_attr.set_input_dist_attr( - cast_name, in_var_dist_attr) + cast_name, in_var_dist_attr + ) else: - assert in_var.dtype == dst_dtype, "op [{}] expect input [{}] to be dtype [{}] BUT got [{}]. {}".format( - grad_op.type, in_name, dst_dtype, in_var.dtype, - str(grad_op)) + assert ( + in_var.dtype == dst_dtype + ), "op [{}] expect input [{}] to be dtype [{}] BUT got [{}]. {}".format( + grad_op.type, + in_name, + dst_dtype, + in_var.dtype, + str(grad_op), + ) for out_name in grad_op.output_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_output( - grad_op, out_name): + grad_op, out_name + ): for out_var_name in grad_op.output(out_name): out_var = self._block._find_var_recursive(out_var_name) assert out_var.dtype == core.VarDesc.VarType.FP32 @@ -334,7 +418,7 @@ class AMPState(object): for out_var_name in grad_op.output(out_name): out_var = self._block._find_var_recursive(out_var_name) - out_var_name_prefix = out_var_name[:out_var_name.find("@")] + out_var_name_prefix = out_var_name[: out_var_name.find("@")] fwd_var = self._block._find_var_recursive(out_var_name_prefix) # NOTE: the out_var's dtype of consume grad_op should equal to the fwd_var's dtype if out_var.dtype != fwd_var.dtype: @@ -345,34 +429,45 @@ class AMPState(object): # NOTE: if out_var of consume grad_op has been casted before, # it should be renamed and reset dist_attr, then we insert cast op to # convert the cast_var to original dtype - consume_op_attr = dist_context.get_op_dist_attr_for_program( - grad_op) + consume_op_attr = ( + dist_context.get_op_dist_attr_for_program(grad_op) + ) fwd_cast_name = self._var_name_dict[fwd_op_id][ - out_var_name_prefix] + out_var_name_prefix + ] suffix = "" if "@RENAME" in out_var_name: - suffix = out_var_name[out_var_name.find("@RENAME"):] + suffix = out_var_name[ + out_var_name.find("@RENAME") : + ] cast_name = fwd_cast_name + "@GRAD" + suffix cast_var = self._block.vars.get(cast_name) if cast_var is None or cast_var.dtype != dst_dtype: grad_op.desc._rename_output(out_var_name, cast_name) - out_var_dist_attr = consume_op_attr.get_output_dist_attr( - out_var_name) + out_var_dist_attr = ( + consume_op_attr.get_output_dist_attr( + out_var_name + ) + ) ref_mesh = out_var_dist_attr.process_mesh ref_mapping = out_var_dist_attr.dims_mapping consume_op_attr.set_output_dist_attr( - cast_name, out_var_dist_attr) + cast_name, out_var_dist_attr + ) assert ref_mapping is not None cast_var = self._block.create_var( name=cast_name, shape=out_var.shape, dtype=dst_dtype, persistable=False, - stop_gradient=out_var.stop_gradient) - set_var_dist_attr(dist_context, cast_var, - ref_mapping, ref_mesh) + stop_gradient=out_var.stop_gradient, + ) + set_var_dist_attr( + dist_context, cast_var, ref_mapping, ref_mesh + ) dist_op_context.grad_var_to_var[ - appended_grad_times][cast_name] = fwd_cast_name + appended_grad_times + ][cast_name] = fwd_cast_name cast_op = self._block._insert_op( idx + 1, @@ -382,13 +477,15 @@ class AMPState(object): attrs={ "in_dtype": cast_var.dtype, "out_dtype": out_var.dtype, - "op_role": OpRole.Backward - }) + "op_role": OpRole.Backward, + }, + ) cast_op._remove_attr("op_role_var") cast_op._remove_attr("op_namescope") cast_op._remove_attr("with_quant_attr") naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, ref_mapping, dist_context) + cast_op, ref_mesh, ref_mapping, dist_context + ) num_cast_ops += 1 else: assert out_var.dtype == dst_dtype @@ -409,15 +506,18 @@ def _update_backward_cast_ops(params_grads, dist_context): for p, g in params_grads: op = g.op if g.dtype == core.VarDesc.VarType.FP32 and op.type == 'cast': - if int(op.attr('op_role')) == int( - OpRole.Backward) and op.has_attr('op_role_var'): + if int(op.attr('op_role')) == int(OpRole.Backward) and op.has_attr( + 'op_role_var' + ): op._remove_attr("op_role_var") post_ops = find_true_post_op(main_block.ops, op, g.name) if post_ops: - raise ValueError("The cast op {0}'s output should not be" - "used by a non-optimize op, however, it" - "is used by {1}".format(op, post_ops[0])) + raise ValueError( + "The cast op {0}'s output should not be" + "used by a non-optimize op, however, it" + "is used by {1}".format(op, post_ops[0]) + ) if op == main_block.ops[-1]: continue @@ -425,23 +525,29 @@ def _update_backward_cast_ops(params_grads, dist_context): # add new op in the python and cpp at the same time new_op_desc = main_block.desc.append_op() new_op_desc.copy_from(op.desc) - new_op = paddle.fluid.framework.Operator(block=main_block, - desc=new_op_desc, - type=None, - inputs=None, - outputs=None, - attrs=None) + new_op = paddle.fluid.framework.Operator( + block=main_block, + desc=new_op_desc, + type=None, + inputs=None, + outputs=None, + attrs=None, + ) main_block.ops.append(new_op) # dist attr param_dist_attr = dist_context.get_tensor_dist_attr_for_program(p) output_dist_attr = dist_context.get_tensor_dist_attr_for_program( - main_block.var(op.output_arg_names[0])) + main_block.var(op.output_arg_names[0]) + ) assert param_dist_attr is not None assert output_dist_attr is not None naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, param_dist_attr.process_mesh, - param_dist_attr.dims_mapping, dist_context) + new_op, + param_dist_attr.process_mesh, + param_dist_attr.dims_mapping, + dist_context, + ) output_dist_attr.process_mesh = param_dist_attr.process_mesh output_dist_attr.dims_mapping = param_dist_attr.dims_mapping @@ -462,26 +568,34 @@ def _check_and_update_gradient(params_grads, loss_scaling, dist_context): grads = [g for _, g in params_grads] check_type(grads, 'x', (tuple, list), 'check_finite_and_unscale') for e in grads: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'check_finite_and_unscale') + check_variable_and_dtype( + e, + "x", + ['float16', 'float32', 'float64'], + 'check_finite_and_unscale', + ) found_inf = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ['find_infinite_scale', 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(['find_infinite_scale', 'tmp']) + ), shape=[1], dtype='bool', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) set_var_dist_attr(dist_context, found_inf, [-1], world_process_group.ranks) inputs = {'X': grads, 'Scale': loss_scaling} outputs = {'Out': grads, 'FoundInfinite': found_inf} attrs = {'op_role': OpRole.Optimize} - new_op = main_block.append_op(type='check_finite_and_unscale', - inputs=inputs, - outputs=outputs, - attrs=attrs) + new_op = main_block.append_op( + type='check_finite_and_unscale', + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) new_op_dist_attr = OperatorDistributedAttribute() new_op_dist_attr.process_mesh = world_process_group.ranks @@ -491,17 +605,18 @@ def _check_and_update_gradient(params_grads, loss_scaling, dist_context): for g in grads: g_dist_attr = dist_context.get_tensor_dist_attr_for_program(g) assert g_dist_attr is not None - new_op_dist_attr.set_input_dims_mapping(g.name, - g_dist_attr.dims_mapping) - new_op_dist_attr.set_output_dims_mapping(g.name, - g_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) + new_op_dist_attr.set_output_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) dist_context.set_op_dist_attr_for_program(new_op, new_op_dist_attr) return grads, found_inf @register_pass("auto_parallel_amp") class AMPPass(PassBase): - def __init__(self): super(AMPPass, self).__init__() self.set_attr("loss", None) @@ -552,7 +667,8 @@ class AMPPass(PassBase): amp_lists = AutoMixedPrecisionLists( set(self.get_attr("custom_white_list")), set(self.get_attr("custom_black_list")), - set(self.get_attr("custom_black_varnames"))) + set(self.get_attr("custom_black_varnames")), + ) with paddle.static.program_guard(main_program, startup_program): amp_state = AMPState(main_program.global_block()) @@ -566,10 +682,13 @@ class AMPPass(PassBase): self._init_amp_var() self._scale_loss() - if self.get_attr("use_dynamic_loss_scaling" - ) or self.get_attr("init_loss_scaling") != 1.0: + if ( + self.get_attr("use_dynamic_loss_scaling") + or self.get_attr("init_loss_scaling") != 1.0 + ): grads, found_inf = _check_and_update_gradient( - params_grads, self._loss_scaling, self.dist_context) + params_grads, self._loss_scaling, self.dist_context + ) if self.get_attr("use_dynamic_loss_scaling"): self._update_loss_scaling(grads, found_inf) @@ -580,9 +699,14 @@ class AMPPass(PassBase): shape=[1], value=self.get_attr("init_loss_scaling"), dtype='float32', - persistable=True) - set_var_dist_attr(self.dist_context, self._loss_scaling, [-1], - world_process_group.ranks) + persistable=True, + ) + set_var_dist_attr( + self.dist_context, + self._loss_scaling, + [-1], + world_process_group.ranks, + ) if self.get_attr("use_dynamic_loss_scaling"): self._num_good_steps = paddle.static.create_global_var( @@ -590,18 +714,28 @@ class AMPPass(PassBase): shape=[1], value=0, dtype='int32', - persistable=True) - set_var_dist_attr(self.dist_context, self._num_good_steps, [-1], - world_process_group.ranks) + persistable=True, + ) + set_var_dist_attr( + self.dist_context, + self._num_good_steps, + [-1], + world_process_group.ranks, + ) self._num_bad_steps = paddle.static.create_global_var( name=unique_name.generate("num_bad_steps"), shape=[1], value=0, dtype='int32', - persistable=True) - set_var_dist_attr(self.dist_context, self._num_bad_steps, [-1], - world_process_group.ranks) + persistable=True, + ) + set_var_dist_attr( + self.dist_context, + self._num_bad_steps, + [-1], + world_process_group.ranks, + ) def _scale_loss(self): @@ -613,18 +747,22 @@ class AMPPass(PassBase): assert loss is not None loss_op = loss.op loss_op_dist_attr = self.dist_context.get_op_dist_attr_for_program( - loss_op) + loss_op + ) if loss.dtype != core.VarDesc.VarType.FP32: tmp_name = unique_name.generate(loss.name + ".cast_fp32") - cast_loss = main_block.create_var(name=tmp_name, - dtype=core.VarDesc.VarType.FP32) + cast_loss = main_block.create_var( + name=tmp_name, dtype=core.VarDesc.VarType.FP32 + ) loss_dist_attr = self.dist_context.get_tensor_dist_attr_for_program( - loss) + loss + ) ref_mesh = loss_op_dist_attr.process_mesh self.dist_context.set_tensor_dist_attr_for_program( - cast_loss, loss_dist_attr) + cast_loss, loss_dist_attr + ) # forward loss_op_idx = find_op_index(main_block.desc, loss_op.desc) @@ -637,22 +775,28 @@ class AMPPass(PassBase): "in_dtype": loss.dtype, "out_dtype": core.VarDesc.VarType.FP32, 'op_role': loss_op.all_attrs()[OP_ROLE_KEY], - }) + }, + ) - loss_op._set_attr(OP_ROLE_KEY, - core.op_proto_and_checker_maker.OpRole.Forward) + loss_op._set_attr( + OP_ROLE_KEY, core.op_proto_and_checker_maker.OpRole.Forward + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, [-1], self.dist_context) + cast_op, ref_mesh, [-1], self.dist_context + ) # backward first_backward_op = main_block.ops[loss_op_idx + 2] - assert first_backward_op.type == "fill_constant" and int( - first_backward_op.all_attrs()[OP_ROLE_KEY]) == 257 + assert ( + first_backward_op.type == "fill_constant" + and int(first_backward_op.all_attrs()[OP_ROLE_KEY]) == 257 + ) cast_loss_grad = main_block.create_var( name=unique_name.generate(tmp_name + "@GRAD"), shape=loss.shape, dtype=core.VarDesc.VarType.FP32, - persistable=loss.persistable) + persistable=loss.persistable, + ) set_var_dist_attr(self.dist_context, cast_loss_grad, [-1], ref_mesh) pre_grad_name = first_backward_op.output_arg_names[0] @@ -666,14 +810,18 @@ class AMPPass(PassBase): "in_dtype": core.VarDesc.VarType.FP32, "out_dtype": core.VarDesc.VarType.FP16, 'op_role': core.op_proto_and_checker_maker.OpRole.Backward, - }) + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_grad_op, ref_mesh, [-1], self.dist_context) + cast_grad_op, ref_mesh, [-1], self.dist_context + ) loss_op = cast_op loss = cast_loss - if self.get_attr("use_dynamic_loss_scaling" - ) or self.get_attr("init_loss_scaling") != 1.0: + if ( + self.get_attr("use_dynamic_loss_scaling") + or self.get_attr("init_loss_scaling") != 1.0 + ): loss_op_idx = find_op_index(main_block.desc, loss_op.desc) @@ -683,63 +831,76 @@ class AMPPass(PassBase): name=unique_name.generate("scaled_loss"), shape=loss.shape, dtype=loss.dtype, - persistable=loss.persistable) - set_var_dist_attr(self.dist_context, self._scaled_loss, [-1], - ref_mesh) + persistable=loss.persistable, + ) + set_var_dist_attr( + self.dist_context, self._scaled_loss, [-1], ref_mesh + ) elementwise_mul_op = main_block._insert_op( loss_op_idx + 1, type='elementwise_mul', - inputs={ - 'X': [loss], - 'Y': [self._loss_scaling] - }, + inputs={'X': [loss], 'Y': [self._loss_scaling]}, outputs={'Out': [self._scaled_loss]}, attrs={ 'op_role': loss_op.all_attrs()[OP_ROLE_KEY], - }) - loss_op._set_attr(OP_ROLE_KEY, - core.op_proto_and_checker_maker.OpRole.Forward) + }, + ) + loss_op._set_attr( + OP_ROLE_KEY, core.op_proto_and_checker_maker.OpRole.Forward + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - elementwise_mul_op, ref_mesh, [-1], self.dist_context) + elementwise_mul_op, ref_mesh, [-1], self.dist_context + ) # backward first_backward_op = main_block.ops[loss_op_idx + 2] - assert first_backward_op.type == "fill_constant" and int( - first_backward_op.all_attrs()[OP_ROLE_KEY]) == 257 + assert ( + first_backward_op.type == "fill_constant" + and int(first_backward_op.all_attrs()[OP_ROLE_KEY]) == 257 + ) self._scaled_loss_grad = main_block.create_var( name=unique_name.generate("scaled_loss") + "@GRAD", shape=loss.shape, dtype=loss.dtype, - persistable=loss.persistable) - set_var_dist_attr(self.dist_context, self._scaled_loss_grad, [-1], - ref_mesh) + persistable=loss.persistable, + ) + set_var_dist_attr( + self.dist_context, self._scaled_loss_grad, [-1], ref_mesh + ) pre_grad_name = first_backward_op.output_arg_names[0] - first_backward_op._rename_output(pre_grad_name, - self._scaled_loss_grad.name) + first_backward_op._rename_output( + pre_grad_name, self._scaled_loss_grad.name + ) # FIXME(JZ-LIANG) a trick to insert backward op main_block._sync_with_cpp() elementwise_mul_grad_op_desc = main_block.desc._insert_op( - loss_op_idx + 3) + loss_op_idx + 3 + ) elementwise_mul_grad_op_desc.set_type("elementwise_mul_grad") elementwise_mul_grad_op_desc.set_input( - 'Out@GRAD', [self._scaled_loss_grad.name]) + 'Out@GRAD', [self._scaled_loss_grad.name] + ) elementwise_mul_grad_op_desc.set_input('X', [loss.name]) - elementwise_mul_grad_op_desc.set_input('Y', - [self._loss_scaling.name]) + elementwise_mul_grad_op_desc.set_input( + 'Y', [self._loss_scaling.name] + ) elementwise_mul_grad_op_desc.set_output('X@GRAD', [pre_grad_name]) elementwise_mul_grad_op_desc.set_output('Y@GRAD', []) elementwise_mul_grad_op_desc._set_attr( - OP_ROLE_KEY, core.op_proto_and_checker_maker.OpRole.Backward) + OP_ROLE_KEY, core.op_proto_and_checker_maker.OpRole.Backward + ) elementwise_mul_grad_op_desc._set_attr('axis', -1) elementwise_mul_grad_op = paddle.fluid.framework.Operator( - main_block, elementwise_mul_grad_op_desc) + main_block, elementwise_mul_grad_op_desc + ) main_block.ops.insert(loss_op_idx + 3, elementwise_mul_grad_op) main_block._sync_with_cpp() elementwise_mul_grad_op = main_block.ops[loss_op_idx + 3] assert elementwise_mul_grad_op.type == "elementwise_mul_grad" naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - elementwise_mul_grad_op, ref_mesh, [-1], self.dist_context) + elementwise_mul_grad_op, ref_mesh, [-1], self.dist_context + ) else: self._scaled_loss = loss @@ -751,31 +912,39 @@ class AMPPass(PassBase): main_block = paddle.static.default_main_program().global_block() main_block._sync_with_cpp() - check_variable_and_dtype(self._loss_scaling, "prev_loss_scaling", - ['float32', 'float64'], "update_loss_scaling") + check_variable_and_dtype( + self._loss_scaling, + "prev_loss_scaling", + ['float32', 'float64'], + "update_loss_scaling", + ) check_type(grads, 'x', (tuple, list), 'update_loss_scaling') for e in grads: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'update_loss_scaling') + check_variable_and_dtype( + e, "x", ['float16', 'float32', 'float64'], 'update_loss_scaling' + ) if e.dtype == core.VarDesc.VarType.FP16: - assert self._loss_scaling.dtype == core.VarDesc.VarType.FP32, \ - "The dtype of prev_loss_scaling should be float32 when the dtype of x is float16." + assert ( + self._loss_scaling.dtype == core.VarDesc.VarType.FP32 + ), "The dtype of prev_loss_scaling should be float32 when the dtype of x is float16." else: - assert self._loss_scaling.dtype == e.dtype, "The dtype of prev_loss_scaling should be equal to the dtype of x." + assert ( + self._loss_scaling.dtype == e.dtype + ), "The dtype of prev_loss_scaling should be equal to the dtype of x." inputs = { 'X': grads, 'FoundInfinite': found_inf, 'PrevLossScaling': self._loss_scaling, 'InGoodSteps': self._num_good_steps, - 'InBadSteps': self._num_bad_steps + 'InBadSteps': self._num_bad_steps, } outputs = { 'Out': grads, 'LossScaling': self._loss_scaling, 'OutGoodSteps': self._num_good_steps, - 'OutBadSteps': self._num_bad_steps + 'OutBadSteps': self._num_bad_steps, } attrs = { @@ -784,13 +953,15 @@ class AMPPass(PassBase): 'incr_ratio': self.get_attr("incr_ratio"), 'decr_ratio': self.get_attr("decr_ratio"), 'stop_update': self.get_attr("stop_update"), - 'op_role': OpRole.Optimize + 'op_role': OpRole.Optimize, } - new_op = main_block.append_op(type='update_loss_scaling', - inputs=inputs, - outputs=outputs, - attrs=attrs) + new_op = main_block.append_op( + type='update_loss_scaling', + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) new_op_dist_attr = OperatorDistributedAttribute() new_op_dist_attr.process_mesh = world_process_group.ranks @@ -800,10 +971,12 @@ class AMPPass(PassBase): for g in grads: g_dist_attr = self.dist_context.get_tensor_dist_attr_for_program(g) assert g_dist_attr is not None - new_op_dist_attr.set_input_dims_mapping(g.name, - g_dist_attr.dims_mapping) - new_op_dist_attr.set_output_dims_mapping(g.name, - g_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) + new_op_dist_attr.set_output_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) self.dist_context.set_op_dist_attr_for_program(new_op, new_op_dist_attr) main_block._sync_with_cpp() diff --git a/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py b/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py index 8470aa51099..fc938d65230 100644 --- a/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py +++ b/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py @@ -19,14 +19,25 @@ import paddle from paddle.fluid import unique_name from paddle.fluid.framework import default_main_program from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole -from paddle.distributed.auto_parallel.operators.common import is_data_parallel_scale_op, is_data_parallel_reduce_op -from paddle.distributed.auto_parallel.utils import find_higher_order_backward_op, is_loss_grad_op, is_optimize_op, ring_id_to_process_group +from paddle.distributed.auto_parallel.operators.common import ( + is_data_parallel_scale_op, + is_data_parallel_reduce_op, +) +from paddle.distributed.auto_parallel.utils import ( + find_higher_order_backward_op, + is_loss_grad_op, + is_optimize_op, + ring_id_to_process_group, +) from .pass_base import PassBase, PassType, register_pass # add new optimizers supporting rescale_grad here __rescale_grad_supported_opts__ = [ - 'lars_momentum', 'sparse_momentum', 'dgc_momentum', 'momentum', - 'merge_momentum' + 'lars_momentum', + 'sparse_momentum', + 'dgc_momentum', + 'momentum', + 'merge_momentum', ] # a heuristic number @@ -62,8 +73,9 @@ class DataParallelOptimizationPass(PassBase): def _check_self(self): if self.get_attr("dist_context") is None: return False - if (not isinstance(self.get_attr("global_rank"), - int)) or self.get_attr("global_rank") < 0: + if (not isinstance(self.get_attr("global_rank"), int)) or self.get_attr( + "global_rank" + ) < 0: return False return True @@ -140,8 +152,11 @@ class DataParallelOptimizationPass(PassBase): ), "Unexception: comm op [{}] has NOT ring id.".format(str(op)) group = ring_id_to_process_group(op.attr("ring_id")) - assert group is not None, "Unexception: data parallel group of [{}] from op [{}] is None".format( - grad_name, str(op)) + assert ( + group is not None + ), "Unexception: data parallel group of [{}] from op [{}] is None".format( + grad_name, str(op) + ) self._grad_name_to_group_map[grad_name] = group @@ -156,18 +171,21 @@ class DataParallelOptimizationPass(PassBase): # TODO support multiple optimizers in on network in future. # here we assume that the optimizer is unique in network. - elif is_optimize_op( - op) and op.type in __rescale_grad_supported_opts__: + elif ( + is_optimize_op(op) + and op.type in __rescale_grad_supported_opts__ + ): self._support_rescale_grad = True not_synchronized_grads = [] for grad_name in scaled_grads: if grad_name not in self._grad_name_to_group_map: not_synchronized_grads.append(grad_name) - assert len( + assert ( + len(not_synchronized_grads) == 0 + ), "Unexception: gradients [{}] is scaled BUT NOT synchronized.".format( not_synchronized_grads - ) == 0, "Unexception: gradients [{}] is scaled BUT NOT synchronized.".format( - not_synchronized_grads) + ) def is_data_parallel_applied(self): return len(self._group_to_grad_name_map) > 0 @@ -175,14 +193,21 @@ class DataParallelOptimizationPass(PassBase): def _could_be_prune(self): return self.dist_context.gradient_scale and ( - self._support_rescale_grad or self._all_dp_groups_same_degree()) + self._support_rescale_grad or self._all_dp_groups_same_degree() + ) def _all_dp_groups_same_degree(self): - return len( - set([ - len(group.ranks) - for group in self._group_to_grad_name_map.keys() - ])) == 1 + return ( + len( + set( + [ + len(group.ranks) + for group in self._group_to_grad_name_map.keys() + ] + ) + ) + == 1 + ) def _scale_backward_initial_grad(self): @@ -191,9 +216,10 @@ class DataParallelOptimizationPass(PassBase): for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): - assert op.type == 'fill_constant', \ - "loss_grad_op must be fill_constant op, " \ + assert op.type == 'fill_constant', ( + "loss_grad_op must be fill_constant op, " "but this op is {}".format(op.type) + ) assert op.has_attr('value') loss_scale = float(op.attr('value')) loss_scale = loss_scale / dp_degree @@ -215,28 +241,35 @@ class DataParallelOptimizationPass(PassBase): scaled_grads = set() for idx, op in reversed(list(enumerate(block.ops))): - if is_optimize_op( - op) and op.type in __rescale_grad_supported_opts__: + if ( + is_optimize_op(op) + and op.type in __rescale_grad_supported_opts__ + ): assert op.has_attr( 'rescale_grad' ), "Unexception: op [{}] is supported to have [rescale_grad] attribute.".format( - str(op)) - assert len( - op.input("Grad") - ) == 1, "Unexception: op [{}] is supported to have only one input grad var.".format( - str(op)) + str(op) + ) + assert ( + len(op.input("Grad")) == 1 + ), "Unexception: op [{}] is supported to have only one input grad var.".format( + str(op) + ) grad_name = op.input("Grad")[0] dp_degree = len( - list(self._grad_name_to_group_map[grad_name].ranks)) + list(self._grad_name_to_group_map[grad_name].ranks) + ) scaled_grads.add(grad_name) rescale_grad = float(op.attr('rescale_grad')) / dp_degree op._set_attr('rescale_grad', rescale_grad) - assert scaled_grads == set(self._grad_name_to_group_map.keys( - )), "Unexception: gradients [{}] are unscaled.".format( - set(self._grad_name_to_group_map.keys()) - scaled_grads) + assert scaled_grads == set( + self._grad_name_to_group_map.keys() + ), "Unexception: gradients [{}] are unscaled.".format( + set(self._grad_name_to_group_map.keys()) - scaled_grads + ) def _could_be_overlap(self): # NOTE current different nccl comm will use different cuda stream @@ -266,14 +299,13 @@ class DataParallelOptimizationPass(PassBase): op._set_attr('use_calc_stream', False) ring_id = op.attr("ring_id") - block._insert_op_without_sync(idx, - type='c_wait_compute', - inputs={'X': []}, - outputs={'Out': []}, - attrs={ - 'op_role': OpRole.Backward, - 'ring_id': ring_id - }) + block._insert_op_without_sync( + idx, + type='c_wait_compute', + inputs={'X': []}, + outputs={'Out': []}, + attrs={'op_role': OpRole.Backward, 'ring_id': ring_id}, + ) block._sync_with_cpp() @@ -307,8 +339,10 @@ class DataParallelOptimizationPass(PassBase): # other ops that might use communicating grad else: for input_var_name in op.input_arg_names: - for ring_id, unsync_grad_names in ring_id_to_un_sync_grad_map.items( - ): + for ( + ring_id, + unsync_grad_names, + ) in ring_id_to_un_sync_grad_map.items(): if input_var_name in unsync_grad_names: # need to sync before op_i if i in op_idx_to_sync_ring_id_map: @@ -328,14 +362,13 @@ class DataParallelOptimizationPass(PassBase): for i in sorted(indices, reverse=True): for ring_id in op_idx_to_sync_ring_id_map[i]: - block._insert_op_without_sync(i, - type='c_wait_comm', - inputs={'X': []}, - outputs={'Out': []}, - attrs={ - 'op_role': OpRole.Backward, - 'ring_id': ring_id - }) + block._insert_op_without_sync( + i, + type='c_wait_comm', + inputs={'X': []}, + outputs={'Out': []}, + attrs={'op_role': OpRole.Backward, 'ring_id': ring_id}, + ) def _could_be_fuse(self): # TODO support gradient fuse higher order gradient. @@ -423,36 +456,49 @@ class DataParallelOptimizationPass(PassBase): for i, group in enumerate(grad_groups[::-1]): # create coalecse tensor - group.coalesce_var = block.create_var(name=unique_name.generate( - 'coalecse_grad_{}'.format(i)), - dtype=group.dtype, - persistable=False, - stop_gradient=True) + group.coalesce_var = block.create_var( + name=unique_name.generate('coalecse_grad_{}'.format(i)), + dtype=group.dtype, + persistable=False, + stop_gradient=True, + ) # update allreduce & scale op if group.scale_op_idx != -1: scale_op = block.ops[group.scale_op_idx] - assert scale_op.type == 'scale', "should found scale op but found {}".format( - str(scale_op)) - scale_op._rename_input(scale_op.input_arg_names[0], - group.coalesce_var.name) - scale_op._rename_output(scale_op.output_arg_names[0], - group.coalesce_var.name) + assert ( + scale_op.type == 'scale' + ), "should found scale op but found {}".format(str(scale_op)) + scale_op._rename_input( + scale_op.input_arg_names[0], group.coalesce_var.name + ) + scale_op._rename_output( + scale_op.output_arg_names[0], group.coalesce_var.name + ) allreduce_op = block.ops[group.allreduce_op_idx] - assert allreduce_op.type == 'c_allreduce_sum', "should found c_allreduce_sum op but found {}".format( - str(allreduce_op)) - allreduce_op._rename_input(allreduce_op.input_arg_names[0], - group.coalesce_var.name) - allreduce_op._rename_output(allreduce_op.output_arg_names[0], - group.coalesce_var.name) + assert ( + allreduce_op.type == 'c_allreduce_sum' + ), "should found c_allreduce_sum op but found {}".format( + str(allreduce_op) + ) + allreduce_op._rename_input( + allreduce_op.input_arg_names[0], group.coalesce_var.name + ) + allreduce_op._rename_output( + allreduce_op.output_arg_names[0], group.coalesce_var.name + ) # remvoe un-used op - remove_op_indices = group.remove_wait_op_indices + group.remove_allreduce_op_indices + group.remove_scale_op_indices + remove_op_indices = ( + group.remove_wait_op_indices + + group.remove_allreduce_op_indices + + group.remove_scale_op_indices + ) for idx in sorted(remove_op_indices, reverse=True): - assert block.ops[ - idx].type in remove_op_types, "Unexception: try to remove op {}".format( - str(op)) + assert ( + block.ops[idx].type in remove_op_types + ), "Unexception: try to remove op {}".format(str(op)) block._remove_op(idx) # insert coalecse op @@ -464,22 +510,23 @@ class DataParallelOptimizationPass(PassBase): concated_ranks.append(len(shape)) grad_names = [grad.name for grad in group.gradients] - block._insert_op_without_sync(group.coalesce_op_idx, - type="coalesce_tensor", - inputs={"Input": grad_names}, - outputs={ - "Output": grad_names, - "FusedOutput": group.coalesce_var - }, - attrs={ - "copy_data": False, - "use_align": True, - "dtype": group.dtype, - "concated_shapes": - concated_shapes, - "concated_ranks": concated_ranks, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op_without_sync( + group.coalesce_op_idx, + type="coalesce_tensor", + inputs={"Input": grad_names}, + outputs={ + "Output": grad_names, + "FusedOutput": group.coalesce_var, + }, + attrs={ + "copy_data": False, + "use_align": True, + "dtype": group.dtype, + "concated_shapes": concated_shapes, + "concated_ranks": concated_ranks, + OP_ROLE_KEY: OpRole.Backward, + }, + ) block._sync_with_cpp() # TODO update dist attr @@ -487,6 +534,7 @@ class DataParallelOptimizationPass(PassBase): def summary(self, grad_groups=[]): # TODO: add logger module import logging + self._logger = logging.getLogger() self._logger.propagate = False if not self._logger.handlers: @@ -500,26 +548,31 @@ class DataParallelOptimizationPass(PassBase): if len(grad_groups) > 0: self._logger.info( - "origin {} allreduce ops are fused into {} coalecse allreduce ops." - .format(len(self._grad_name_to_group_map.keys()), - len(grad_groups))) + "origin {} allreduce ops are fused into {} coalecse allreduce ops.".format( + len(self._grad_name_to_group_map.keys()), len(grad_groups) + ) + ) self._logger.info("gradient fusing group are following: ") fused_grads = set() for i, group in enumerate(grad_groups): self._logger.info( "coalecse gradient [{}] is composed by: {}".format( - i, [grad.name for grad in group.gradients])) + i, [grad.name for grad in group.gradients] + ) + ) fused_grads.update([grad.name for grad in group.gradients]) - individual_grads = set( - self._grad_name_to_group_map.keys()) - set(fused_grads) + individual_grads = set(self._grad_name_to_group_map.keys()) - set( + fused_grads + ) self._logger.info( "the following [{}] gradients are not fused: ".format( - len(individual_grads))) + len(individual_grads) + ) + ) self._logger.info("individual gradient {}".format(individual_grads)) class GradientsGroup(object): - def __init__(self, ops, max_group_size): self.max_group_size = max_group_size self.ops = ops @@ -575,8 +628,11 @@ class GradientsGroup(object): grad_op_idx -= 1 grad_op = self.ops[grad_op_idx] - assert grad_var.name in grad_op.output_arg_names, "grad [{}] should be output of {}".format( - grad_var.name, str(grad_op)) + assert ( + grad_var.name in grad_op.output_arg_names + ), "grad [{}] should be output of {}".format( + grad_var.name, str(grad_op) + ) self.coalesce_op_idx = grad_op_idx def finalize(self): diff --git a/python/paddle/distributed/passes/auto_parallel_fp16.py b/python/paddle/distributed/passes/auto_parallel_fp16.py index 35a4831af3a..cf1b2d45290 100644 --- a/python/paddle/distributed/passes/auto_parallel_fp16.py +++ b/python/paddle/distributed/passes/auto_parallel_fp16.py @@ -20,12 +20,31 @@ from paddle.fluid.framework import default_main_program, default_startup_program from paddle.fluid import unique_name from .pass_base import register_pass from paddle.fluid.data_feeder import check_variable_and_dtype, check_type -from paddle.distributed.auto_parallel.utils import set_var_dist_attr, naive_set_dist_op_attr_for_program_by_mesh_and_mapping -from paddle.distributed.auto_parallel.process_group import get_world_process_group -from paddle.fluid.contrib.mixed_precision.fp16_utils import AutoMixedPrecisionLists -from paddle.fluid.contrib.mixed_precision.fp16_utils import _keep_layer_norm_scale_bias_to_fp32, _need_keep_fp32, _valid_types, _dtype_to_str -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute -from paddle.distributed.auto_parallel.utils import is_forward_op, is_backward_op, OP_ROLE_KEY, OpRole +from paddle.distributed.auto_parallel.utils import ( + set_var_dist_attr, + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) +from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + AutoMixedPrecisionLists, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _keep_layer_norm_scale_bias_to_fp32, + _need_keep_fp32, + _valid_types, + _dtype_to_str, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) +from paddle.distributed.auto_parallel.utils import ( + is_forward_op, + is_backward_op, + OP_ROLE_KEY, + OpRole, +) from .auto_parallel_amp import AMPPass world_process_group = get_world_process_group() @@ -39,11 +58,15 @@ __amp_skip_ops__ = [ def set_op_dtype_to_fp16(op): - if op.has_attr('in_dtype') and op.attr( - 'in_dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('in_dtype') + and op.attr('in_dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('in_dtype', core.VarDesc.VarType.FP16) - if op.has_attr('out_dtype') and op.attr( - 'out_dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('out_dtype') + and op.attr('out_dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('out_dtype', core.VarDesc.VarType.FP16) if op.has_attr('dtype') and op.attr('dtype') == core.VarDesc.VarType.FP32: op._set_attr('dtype', core.VarDesc.VarType.FP16) @@ -63,7 +86,12 @@ def _keep_fp32_input(op, in_name): return in_name not in {'X', 'FilterX', 'Z', 'FilterZ'} if op_type in ['fused_attention', 'fused_feedforward']: return in_name in { - 'LnScale', 'LnBias', 'Ln2Scale', 'Ln2Bias', "Ln1Scale", "Ln1Bias" + 'LnScale', + 'LnBias', + 'Ln2Scale', + 'Ln2Bias', + "Ln1Scale", + "Ln1Bias", } # backward if op_type in ['batch_norm_grad']: @@ -83,8 +111,12 @@ def _keep_fp32_output(op, out_name): return out_name not in {'Y', 'ConvX', 'ConvZ'} if op_type in ['fused_attention', 'fused_feedforward']: return out_name in { - 'LnMean', 'LnVariance', 'Ln2Mean', 'Ln2Variance', 'Ln1Mean', - 'Ln1Variance' + 'LnMean', + 'LnVariance', + 'Ln2Mean', + 'Ln2Variance', + 'Ln1Mean', + 'Ln1Variance', } # backward if op_type in ['layer_norm_grad']: @@ -95,24 +127,28 @@ def _keep_fp32_output(op, out_name): class FP16State(object): - - def __init__(self, - program, - amp_list, - dist_context, - use_fp16_guard, - input_data_var_names=None): + def __init__( + self, + program, + amp_list, + dist_context, + use_fp16_guard, + input_data_var_names=None, + ): self.program = program self.amp_list = amp_list self.use_fp16_guard = use_fp16_guard self.dist_context = dist_context - self.grad_op_to_op_map = self.dist_context.dist_op_context.grad_op_id_to_op_id + self.grad_op_to_op_map = ( + self.dist_context.dist_op_context.grad_op_id_to_op_id + ) if input_data_var_names: self.input_data_var_names = input_data_var_names else: self.input_data_var_names = [] - self._op_fp16_dict = { - } # op_id --> True/False. 'True' means that the op is should run in fp16 mode. + self._op_fp16_dict = ( + {} + ) # op_id --> True/False. 'True' means that the op is should run in fp16 mode. # a trick to determine leaf tensor node in program {varname: generator_op_id} self.forward_non_leaf_tensors = {} # record the cast ops that are inserted for a forward @@ -156,8 +192,9 @@ class FP16State(object): if op.type == "assign" and "array_" in op.input_arg_names[0]: self._op_fp16_dict[op.desc.original_id()] = False return - if _need_keep_fp32(op, self.amp_list.unsupported_list, - self.use_fp16_guard): + if _need_keep_fp32( + op, self.amp_list.unsupported_list, self.use_fp16_guard + ): self._op_fp16_dict[op.desc.original_id()] = False else: self._op_fp16_dict[op.desc.original_id()] = True @@ -170,8 +207,9 @@ class FP16State(object): if op.desc.original_id() in self.grad_op_to_op_map: fwd_op_id = self.grad_op_to_op_map[op.desc.original_id()] assert fwd_op_id in self._op_fp16_dict, "{}".format(str(op)) - self._op_fp16_dict[ - op.desc.original_id()] = self._op_fp16_dict[fwd_op_id] + self._op_fp16_dict[op.desc.original_id()] = self._op_fp16_dict[ + fwd_op_id + ] if int(op.attr('op_role')) == 257: self.is_train = True @@ -197,13 +235,18 @@ class FP16State(object): for op in block.ops: if is_forward_op(op): # NOTE (JZ-LIANG) un-expected cast op when user call "+, -, *, /" in python - if self._is_fp16_op(op.desc.original_id()) == True \ - or op.type == "cast": + if ( + self._is_fp16_op(op.desc.original_id()) == True + or op.type == "cast" + ): for in_name in op.input_names: if _keep_fp32_input(op, in_name): continue for in_var_name in op.input(in_name): - if in_var_name not in self.forward_non_leaf_tensors and in_var_name not in self.input_data_var_names: + if ( + in_var_name not in self.forward_non_leaf_tensors + and in_var_name not in self.input_data_var_names + ): self.set_var_to_fp16(in_var_name, block) for out_name in op.output_names: if _keep_fp32_output(op, out_name): @@ -249,22 +292,42 @@ class FP16State(object): elif is_forward_op(op): if self._is_fp16_op(op.desc.original_id()) == False: num_cast_ops = self._insert_forward_cast_ops( - op, idx, block, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32, self.dist_context) + op, + idx, + block, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + self.dist_context, + ) elif self._is_fp16_op(op.desc.original_id()) == True: num_cast_ops = self._insert_forward_cast_ops( - op, idx, block, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, self.dist_context) + op, + idx, + block, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + self.dist_context, + ) elif is_backward_op(op): if op.desc.original_id() in dist_op_context.grad_op_id_to_op_id: if self._is_fp16_op(op.desc.original_id()) == False: num_cast_ops = self._insert_backward_cast_ops( - op, idx, block, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32, self.dist_context) + op, + idx, + block, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + self.dist_context, + ) elif self._is_fp16_op(op.desc.original_id()) == True: num_cast_ops = self._insert_backward_cast_ops( - op, idx, block, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, self.dist_context) + op, + idx, + block, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + self.dist_context, + ) elif op.type == "sum": # all inputs dtype of sum should be equal and output dtype should follow input out_var_name = op.output_arg_names[0] @@ -272,41 +335,51 @@ class FP16State(object): out_var = block.var(out_var_name) in_var = block._find_var_recursive(in_var_name) for in_var_name in op.input_arg_names: - assert in_var.dtype == block.var( - in_var_name).dtype, "{}, {}, {}".format( - in_var, block.var(in_var_name), str(op)) + assert ( + in_var.dtype == block.var(in_var_name).dtype + ), "{}, {}, {}".format( + in_var, block.var(in_var_name), str(op) + ) out_var.desc.set_dtype(in_var.dtype) idx += num_cast_ops + 1 block._sync_with_cpp() - def _insert_forward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, - dist_context): + def _insert_forward_cast_ops( + self, op, idx, block, src_dtype, dst_dtype, dist_context + ): num_cast_ops = 0 for in_name in op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_input( - op, in_name): + op, in_name + ): continue consume_op_attr = dist_context.get_op_dist_attr_for_program(op) assert consume_op_attr is not None for in_var_name in op.input(in_name): in_var = block._find_var_recursive(in_var_name) - if in_var is None or in_var.type not in _valid_types or in_var.dtype == dst_dtype: + if ( + in_var is None + or in_var.type not in _valid_types + or in_var.dtype == dst_dtype + ): continue if in_var.dtype == src_dtype: - cast_name = in_var.name + '.cast_' + _dtype_to_str( - dst_dtype) + cast_name = ( + in_var.name + '.cast_' + _dtype_to_str(dst_dtype) + ) cast_var = block.vars.get(cast_name) self.forward_input_cast_ops[op.desc.original_id()] += [ (cast_name, in_var.name, dst_dtype, src_dtype, in_name) ] in_var_dist_attr = consume_op_attr.get_input_dist_attr( - in_var.name) + in_var.name + ) assert in_var_dist_attr is not None # truly insert cast op if cast_var is None or cast_var.dtype != dst_dtype: @@ -320,9 +393,11 @@ class FP16State(object): name=cast_name, dtype=dst_dtype, persistable=False, - stop_gradient=in_var.stop_gradient) - set_var_dist_attr(dist_context, cast_var, ref_mapping, - ref_mesh) + stop_gradient=in_var.stop_gradient, + ) + set_var_dist_attr( + dist_context, cast_var, ref_mapping, ref_mesh + ) cast_op = block._insert_op_without_sync( idx, @@ -332,23 +407,27 @@ class FP16State(object): attrs={ "in_dtype": in_var.dtype, "out_dtype": cast_var.dtype, - OP_ROLE_KEY: OpRole.Forward - }) + OP_ROLE_KEY: OpRole.Forward, + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, ref_mapping, dist_context) + cast_op, ref_mesh, ref_mapping, dist_context + ) num_cast_ops += 1 op._rename_input(in_var.name, cast_name) - consume_op_attr.set_input_dist_attr(cast_name, - in_var_dist_attr) + consume_op_attr.set_input_dist_attr( + cast_name, in_var_dist_attr + ) if op.has_attr('out_dtype') and op.attr('out_dtype') != -1: assert op.attr('out_dtype') == dst_dtype return num_cast_ops - def _insert_backward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, - dist_context): + def _insert_backward_cast_ops( + self, op, idx, block, src_dtype, dst_dtype, dist_context + ): num_cast_ops = 0 op_id = op.desc.id() @@ -364,10 +443,16 @@ class FP16State(object): if _keep_fp32_output(op, out_var.name): continue assert out_var.dtype == dst_dtype, "{}, {}".format( - str(out_var), dst_dtype) + str(out_var), dst_dtype + ) - for cast_name, src_name, dst_dtype, src_dtype, slot_name in self.forward_input_cast_ops[ - forward_op_id]: + for ( + cast_name, + src_name, + dst_dtype, + src_dtype, + slot_name, + ) in self.forward_input_cast_ops[forward_op_id]: # some forward output is not need by backward computation, e.g. logit in softmax_with_cross_entropy if slot_name not in op.input_names: @@ -375,8 +460,8 @@ class FP16State(object): # rename input assert src_name in op.input( - slot_name), "var: {} not in op's {}. {}".format( - src_name, slot_name, str(op)) + slot_name + ), "var: {} not in op's {}. {}".format(src_name, slot_name, str(op)) src_var_dist_attr = grad_op_attr.get_input_dist_attr(src_name) assert src_var_dist_attr is not None op._rename_input(src_name, cast_name) @@ -384,15 +469,16 @@ class FP16State(object): # create cast grad grad_slot_name = slot_name + "@GRAD" - assert grad_slot_name in op.output_names, "[{}], Current Op: {}".format( - grad_slot_name, str(op)) + assert ( + grad_slot_name in op.output_names + ), "[{}], Current Op: {}".format(grad_slot_name, str(op)) # some forward input maybe stop_gradient=True, e.g. input_mask if len(op.output(grad_slot_name)) == 0: continue - assert len( - op.output(grad_slot_name)) == 1, "[{}], Current Op: {}".format( - grad_slot_name, str(op)) + assert ( + len(op.output(grad_slot_name)) == 1 + ), "[{}], Current Op: {}".format(grad_slot_name, str(op)) grad_name = op.output(grad_slot_name)[0] grad = block.var(grad_name) grad_dist_attr = grad_op_attr.get_output_dist_attr(grad_name) @@ -401,15 +487,18 @@ class FP16State(object): ref_mapping = grad_dist_attr.dims_mapping cast_grad = block.create_var( - name=unique_name.generate_with_ignorable_key("".join( - [cast_name, '@GRAD'])), + name=unique_name.generate_with_ignorable_key( + "".join([cast_name, '@GRAD']) + ), dtype=dst_dtype, shape=grad.shape, type=grad.type, persistable=grad.persistable, - stop_gradient=grad.stop_gradient) + stop_gradient=grad.stop_gradient, + ) dist_context.set_tensor_dist_attr_for_program( - cast_grad, grad_dist_attr) + cast_grad, grad_dist_attr + ) op._rename_output(grad_name, cast_grad.name) grad_op_attr.set_output_dist_attr(cast_grad.name, grad_dist_attr) @@ -422,12 +511,14 @@ class FP16State(object): attrs={ "in_dtype": dst_dtype, "out_dtype": src_dtype, - OP_ROLE_KEY: OpRole.Backward - }) + OP_ROLE_KEY: OpRole.Backward, + }, + ) grad.desc.set_dtype(src_dtype) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, ref_mapping, dist_context) + cast_op, ref_mesh, ref_mapping, dist_context + ) num_cast_ops += 1 return num_cast_ops @@ -440,26 +531,34 @@ def _check_and_update_gradient(grads, loss_scaling, name, dist_context): check_type(grads, 'x', (tuple, list), 'check_finite_and_unscale') for e in grads: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'check_finite_and_unscale') + check_variable_and_dtype( + e, + "x", + ['float16', 'float32', 'float64'], + 'check_finite_and_unscale', + ) found_inf = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ['find_infinite_scale', name])), + name=unique_name.generate_with_ignorable_key( + ".".join(['find_infinite_scale', name]) + ), shape=[1], dtype='bool', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) set_var_dist_attr(dist_context, found_inf, [-1], world_process_group.ranks) inputs = {'X': grads, 'Scale': loss_scaling} outputs = {'Out': grads, 'FoundInfinite': found_inf} attrs = {'op_role': OpRole.Optimize} - new_op = main_block.append_op(type='check_finite_and_unscale', - inputs=inputs, - outputs=outputs, - attrs=attrs) + new_op = main_block.append_op( + type='check_finite_and_unscale', + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) new_op_dist_attr = OperatorDistributedAttribute() new_op_dist_attr.process_mesh = world_process_group.ranks @@ -469,10 +568,12 @@ def _check_and_update_gradient(grads, loss_scaling, name, dist_context): for g in grads: g_dist_attr = dist_context.get_tensor_dist_attr_for_program(g) assert g_dist_attr is not None - new_op_dist_attr.set_input_dims_mapping(g.name, - g_dist_attr.dims_mapping) - new_op_dist_attr.set_output_dims_mapping(g.name, - g_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) + new_op_dist_attr.set_output_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) dist_context.set_op_dist_attr_for_program(new_op, new_op_dist_attr) return grads, found_inf @@ -481,8 +582,9 @@ def _split_grads(params_grads): grads = [g for _, g in params_grads] fp32_grads = [g for g in grads if g.dtype == core.VarDesc.VarType.FP32] fp16_grads = [g for g in grads if g.dtype == core.VarDesc.VarType.FP16] - assert len(fp32_grads) + len(fp16_grads) == len(grads), \ - "Data types of all grads must be either fp16 or fp32." + assert len(fp32_grads) + len(fp16_grads) == len( + grads + ), "Data types of all grads must be either fp16 or fp32." return grads, fp32_grads, fp16_grads @@ -494,37 +596,45 @@ def _set_op_dist_attr_with_ranks(new_op, ranks, block, dist_context): var = block.var(var_name) var_dist_attr = dist_context.get_tensor_dist_attr_for_program(var) assert var_dist_attr is not None - new_op_dist_attr.set_input_dims_mapping(var_name, - var_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + var_name, var_dist_attr.dims_mapping + ) for var_name in new_op.output_arg_names: var = block.var(var_name) var_dist_attr = dist_context.get_tensor_dist_attr_for_program(var) assert var_dist_attr is not None - new_op_dist_attr.set_output_dims_mapping(var_name, - var_dist_attr.dims_mapping) + new_op_dist_attr.set_output_dims_mapping( + var_name, var_dist_attr.dims_mapping + ) dist_context.set_op_dist_attr_for_program(new_op, new_op_dist_attr) def _get_memcopy_idx(block, found_inf_var): # use reduce_any op for check_nan_inf as the anchor for now for idx, op in enumerate(block.ops): - if op.type == 'reduce_any' and op.output_arg_names[ - 0] == found_inf_var.name: + if ( + op.type == 'reduce_any' + and op.output_arg_names[0] == found_inf_var.name + ): return idx + 1 raise RuntimeError( - "not found the correct location for memcopy for found_inf_var.") + "not found the correct location for memcopy for found_inf_var." + ) def _insert_memcopy(block, idx, src_var, dist_context, direction="D2H"): src_name = src_var.name - output_var = block.create_var(name=unique_name.generate_with_ignorable_key( - src_name.join(['memcopy_'])), - dtype=src_var.dtype, - shape=src_var.shape, - type=core.VarDesc.VarType.LOD_TENSOR, - persistable=False, - stop_gradient=src_var.stop_gradient) + output_var = block.create_var( + name=unique_name.generate_with_ignorable_key( + src_name.join(['memcopy_']) + ), + dtype=src_var.dtype, + shape=src_var.shape, + type=core.VarDesc.VarType.LOD_TENSOR, + persistable=False, + stop_gradient=src_var.stop_gradient, + ) set_var_dist_attr(dist_context, output_var, [-1], world_process_group.ranks) @@ -535,16 +645,20 @@ def _insert_memcopy(block, idx, src_var, dist_context, direction="D2H"): dst_place_type = 1 else: raise NotImplementedError( - "direction [{}] is not supported yet.".format(direction)) + "direction [{}] is not supported yet.".format(direction) + ) attrs = {'dst_place_type': dst_place_type} - new_op = block._insert_op_without_sync(index=idx, - type='memcpy', - inputs={'X': [src_var]}, - outputs={'Out': [output_var]}, - attrs=attrs) - _set_op_dist_attr_with_ranks(new_op, world_process_group.ranks, block, - dist_context) + new_op = block._insert_op_without_sync( + index=idx, + type='memcpy', + inputs={'X': [src_var]}, + outputs={'Out': [output_var]}, + attrs=attrs, + ) + _set_op_dist_attr_with_ranks( + new_op, world_process_group.ranks, block, dist_context + ) block._sync_with_cpp() return output_var @@ -572,19 +686,21 @@ def cast_startup_program(): for op in startup_program.global_block().ops: if is_initialization_op(op): output_name = op.output_arg_names[0] - if param_to_dtype.get(output_name, - None) == core.VarDesc.VarType.FP16: + if ( + param_to_dtype.get(output_name, None) + == core.VarDesc.VarType.FP16 + ): assert op.has_attr( 'dtype' ), "initialization op is supported to has dtype attribute but got {}.".format( - str(op)) + str(op) + ) if op.attr('dtype') == core.VarDesc.VarType.FP32: op._set_attr('dtype', core.VarDesc.VarType.FP16) @register_pass("auto_parallel_fp16") class FP16Pass(AMPPass): - def __init__(self): super(FP16Pass, self).__init__() @@ -597,16 +713,22 @@ class FP16Pass(AMPPass): amp_list = AutoMixedPrecisionLists( set(self.get_attr("custom_white_list")), - set(self.get_attr("custom_black_list")), None) + set(self.get_attr("custom_black_list")), + None, + ) # NOTE don't not change input data dtype, since it is controled by dataloader # and which is out of control of FP16 Pass input_data_var_names = [var.name for var in self.get_attr("input_data")] with paddle.static.program_guard(main_program, startup_program): - fp16_state = FP16State(main_program, amp_list, self.dist_context, - self.get_attr("use_fp16_guard"), - input_data_var_names) + fp16_state = FP16State( + main_program, + amp_list, + self.dist_context, + self.get_attr("use_fp16_guard"), + input_data_var_names, + ) is_train = fp16_state._build_state() cast_startup_program() @@ -619,41 +741,63 @@ class FP16Pass(AMPPass): grads, fp32_grads, fp16_grads = _split_grads(params_grads) - if self.get_attr("use_dynamic_loss_scaling" - ) or self.get_attr("init_loss_scaling") != 1.0: + if ( + self.get_attr("use_dynamic_loss_scaling") + or self.get_attr("init_loss_scaling") != 1.0 + ): found_infs = [] if fp32_grads: with main_program._optimized_guard([]): _, found_inf_fp32 = _check_and_update_gradient( - fp32_grads, self._loss_scaling, "@fp32", - self.dist_context) + fp32_grads, + self._loss_scaling, + "@fp32", + self.dist_context, + ) found_infs.append(found_inf_fp32) if fp16_grads: with main_program._optimized_guard([]): _, found_inf_fp16 = _check_and_update_gradient( - fp16_grads, self._loss_scaling, "@fp16", - self.dist_context) + fp16_grads, + self._loss_scaling, + "@fp16", + self.dist_context, + ) found_infs.append(found_inf_fp16) with main_program._optimized_guard([]): block = main_program.global_block() all_infs = paddle.fluid.layers.concat(found_infs) - set_var_dist_attr(self.dist_context, all_infs, [-1], - world_process_group.ranks) + set_var_dist_attr( + self.dist_context, + all_infs, + [-1], + world_process_group.ranks, + ) new_op = block.ops[-1] assert new_op.type == "concat" - _set_op_dist_attr_with_ranks(new_op, - world_process_group.ranks, - block, self.dist_context) + _set_op_dist_attr_with_ranks( + new_op, + world_process_group.ranks, + block, + self.dist_context, + ) found_inf = paddle.fluid.layers.reduce_any(all_infs) - set_var_dist_attr(self.dist_context, found_inf, [-1], - world_process_group.ranks) + set_var_dist_attr( + self.dist_context, + found_inf, + [-1], + world_process_group.ranks, + ) new_op = block.ops[-1] assert new_op.type == "reduce_any" - _set_op_dist_attr_with_ranks(new_op, - world_process_group.ranks, - block, self.dist_context) + _set_op_dist_attr_with_ranks( + new_op, + world_process_group.ranks, + block, + self.dist_context, + ) if self.get_attr("use_dynamic_loss_scaling"): with main_program._optimized_guard([]): @@ -668,14 +812,15 @@ class FP16Pass(AMPPass): if self.get_attr("use_optimizer_fp16"): base_opt._multi_precision = False if isinstance( - base_opt, - (paddle.fluid.optimizer.Adam, paddle.optimizer.AdamW)): + base_opt, (paddle.fluid.optimizer.Adam, paddle.optimizer.AdamW) + ): with main_program._optimized_guard([]): # found_inf = paddle.tensor.creation._memcpy( # found_inf, paddle.CPUPlace()) insert_idx = _get_memcopy_idx(block, found_inf) - found_inf = _insert_memcopy(block, insert_idx, found_inf, - self.dist_context) + found_inf = _insert_memcopy( + block, insert_idx, found_inf, self.dist_context + ) base_opt._set_auxiliary_var('found_inf', found_inf.name) elif hasattr(base_opt, "_set_auxiliary_var"): base_opt._set_auxiliary_var('found_inf', found_inf.name) diff --git a/python/paddle/distributed/passes/auto_parallel_grad_clip.py b/python/paddle/distributed/passes/auto_parallel_grad_clip.py index 5108992ae55..ebb44bb0f12 100644 --- a/python/paddle/distributed/passes/auto_parallel_grad_clip.py +++ b/python/paddle/distributed/passes/auto_parallel_grad_clip.py @@ -20,8 +20,17 @@ import paddle from .pass_base import PassBase, register_pass from ..auto_parallel.reshard import Resharder from ..auto_parallel.process_group import get_world_process_group -from ..auto_parallel.utils import is_gradient_clip_op, is_optimize_op, OP_ROLE_KEY, OpRole, _get_comm_group -from ..auto_parallel.dist_attribute import TensorDistributedAttribute, OperatorDistributedAttribute +from ..auto_parallel.utils import ( + is_gradient_clip_op, + is_optimize_op, + OP_ROLE_KEY, + OpRole, + _get_comm_group, +) +from ..auto_parallel.dist_attribute import ( + TensorDistributedAttribute, + OperatorDistributedAttribute, +) def _get_params_grads(block): @@ -52,7 +61,8 @@ def _get_dpmp_topology(origin_topology, sharding_group): """ sharding_axis = 1 dp_sharding_topology = [ - origin_topology[0] // sharding_group.nranks, sharding_group.nranks + origin_topology[0] // sharding_group.nranks, + sharding_group.nranks, ] if dp_sharding_topology[0] == 1: sharding_axis = 0 @@ -108,22 +118,24 @@ def _get_dpmp_process_mesh(rank_id, topology, processes, sharding_group): return dpmp_topology, list(dpmp_processes_in_sharding) -def _is_about_global_norm(rank_id, tensor_shape, topology, processes, - dims_mapping, sharding_group): +def _is_about_global_norm( + rank_id, tensor_shape, topology, processes, dims_mapping, sharding_group +): # get current process_mesh where the parameter exist. dpmp_topology, dpmp_processes = _get_dpmp_process_mesh( - rank_id, topology, processes, sharding_group) + rank_id, topology, processes, sharding_group + ) - complete_shape = Resharder.compute_complete_shape(tensor_shape, - dpmp_topology, - dims_mapping) + complete_shape = Resharder.compute_complete_shape( + tensor_shape, dpmp_topology, dims_mapping + ) complete_partitions = [] complete_param_ranks = [] for process in dpmp_processes: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, dpmp_topology, - dpmp_processes) + process, complete_shape, dims_mapping, dpmp_topology, dpmp_processes + ) if partition_index not in complete_partitions: complete_partitions.append(partition_index) complete_param_ranks.append(process) @@ -132,7 +144,6 @@ def _is_about_global_norm(rank_id, tensor_shape, topology, processes, class ClipHelper(object): - def __init__(self, params_grads, rank_id, block, dist_context): params, _ = zip(*params_grads) self.params = list(params) @@ -154,9 +165,14 @@ class ClipHelper(object): topology = dist_attr.process_mesh.topology processes = dist_attr.process_mesh.processes dims_mapping = dist_attr.dims_mapping - return _is_about_global_norm(self.rank_id, param.shape, topology, - processes, dims_mapping, - self.sharding_group) + return _is_about_global_norm( + self.rank_id, + param.shape, + topology, + processes, + dims_mapping, + self.sharding_group, + ) def _get_dist_attr(self, name): var = self.block.vars[name] @@ -181,7 +197,8 @@ class ClipHelper(object): in_dist_attr.process_mesh = self.world_ranks in_dist_attr.dims_mapping = [-1] self.dist_context.set_tensor_dist_attr_for_program( - in_var, in_dist_attr) + in_var, in_dist_attr + ) op_dist_attr.set_input_dist_attr(in_name, in_dist_attr) for out_name in op.output_arg_names: out_var = self.block.vars[out_name] @@ -189,7 +206,8 @@ class ClipHelper(object): out_dist_attr.process_mesh = self.world_ranks out_dist_attr.dims_mapping = [-1] self.dist_context.set_tensor_dist_attr_for_program( - out_var, out_dist_attr) + out_var, out_dist_attr + ) op_dist_attr.set_output_dist_attr(out_name, out_dist_attr) self.dist_context.set_op_dist_attr_for_program(op, op_dist_attr) @@ -228,14 +246,18 @@ class ClipGradByGloblNormPass(PassBase): dist_params_grads = self.get_attr("params_grads", None) # dist_params_grads = _get_params_grads(block) - self.clip_helper = ClipHelper(dist_params_grads, rank_id, block, - dist_context) + self.clip_helper = ClipHelper( + dist_params_grads, rank_id, block, dist_context + ) self._remove_no_need_ops_vars(block) def _remove_no_need_ops_vars(self, block): removed_op_out_type = [ - 'clip_by_norm', 'squared_l2_norm', 'square', 'reduce_sum' + 'clip_by_norm', + 'squared_l2_norm', + 'square', + 'reduce_sum', ] removed_op_idx = set() @@ -248,12 +270,14 @@ class ClipGradByGloblNormPass(PassBase): input_name = op.input("X")[0] if input_name.find("@GRAD") != -1: #'clip_by_norm', 'squared_l2_norm', 'square' - param_name = input_name[:input_name.find("@GRAD")] + param_name = input_name[: input_name.find("@GRAD")] is_local = self.clip_helper._is_local_param(param_name) is_calculate = self.clip_helper._is_calcuate_norm( - param_name) - if not is_local or (not is_calculate - and op.type != 'clip_by_norm'): + param_name + ) + if not is_local or ( + not is_calculate and op.type != 'clip_by_norm' + ): removed_op_idx.add(idx) removed_tmp_var.update(set(op.output_arg_names)) else: @@ -265,20 +289,23 @@ class ClipGradByGloblNormPass(PassBase): elif op.type == 'elementwise_mul': input_name = op.input("X")[0] if input_name.find("@GRAD") != -1: - param_name = input_name[:input_name.find("@GRAD")] + param_name = input_name[: input_name.find("@GRAD")] is_local = self.clip_helper._is_local_param(param_name) if not is_local: removed_op_idx.add(idx) if block.ops[idx - 1].type == 'cast': removed_op_idx.add(idx - 1) removed_tmp_var.update( - set(block.ops[idx - 1].output_arg_names)) + set(block.ops[idx - 1].output_arg_names) + ) elif op.type == 'sum': reserved_vars = [] for input_name in op.input_arg_names: - if input_name not in removed_tmp_var and \ - self.clip_helper._is_local_var(input_name): + if ( + input_name not in removed_tmp_var + and self.clip_helper._is_local_var(input_name) + ): reserved_vars.append(input_name) if not reserved_vars: removed_op_idx.add(idx) @@ -286,7 +313,8 @@ class ClipGradByGloblNormPass(PassBase): if block.ops[idx + 1].type == 'cast': removed_op_idx.add(idx + 1) removed_tmp_var.update( - set(block.ops[idx + 1].output_arg_names)) + set(block.ops[idx + 1].output_arg_names) + ) else: op.desc.set_input("X", reserved_vars) @@ -320,10 +348,12 @@ class ClipGradByGloblNormPass(PassBase): 'dtype': input_var.dtype, 'value': 0, 'force_cpu': False, - OP_ROLE_KEY: OpRole.Optimize - }) - fill_constant_op._set_attr('op_namescope', - "/gradient_clip_pass") + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + fill_constant_op._set_attr( + 'op_namescope', "/gradient_clip_pass" + ) offset += 1 self.clip_helper._init_dist_attr(fill_constant_op) @@ -336,9 +366,11 @@ class ClipGradByGloblNormPass(PassBase): 'ring_id': 0, 'use_calc_stream': True, OP_ROLE_KEY: OpRole.Optimize, - }) - allreduce_op._set_attr('op_namescope', - "/gradient_clip_pass") + }, + ) + allreduce_op._set_attr( + 'op_namescope', "/gradient_clip_pass" + ) self.clip_helper._init_dist_attr(allreduce_op) for varname in removed_tmp_var: diff --git a/python/paddle/distributed/passes/auto_parallel_gradient_merge.py b/python/paddle/distributed/passes/auto_parallel_gradient_merge.py index 2c0af10e35d..282f909987d 100644 --- a/python/paddle/distributed/passes/auto_parallel_gradient_merge.py +++ b/python/paddle/distributed/passes/auto_parallel_gradient_merge.py @@ -19,9 +19,18 @@ from paddle.framework import core from paddle.fluid import layers from paddle.fluid.framework import device_guard from .pass_base import PassBase, PassType, register_pass -from paddle.distributed.auto_parallel.utils import set_var_dist_attr, is_optimize_op, OpRole, OP_ROLE_KEY -from paddle.distributed.auto_parallel.utils import naive_set_dist_op_attr_for_program_by_mesh_and_mapping -from paddle.distributed.auto_parallel.process_group import get_world_process_group +from paddle.distributed.auto_parallel.utils import ( + set_var_dist_attr, + is_optimize_op, + OpRole, + OP_ROLE_KEY, +) +from paddle.distributed.auto_parallel.utils import ( + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) +from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, +) world_process_group = get_world_process_group() @@ -56,79 +65,87 @@ def _remove_and_get_optimizer_op(main_program, dist_context): def _get_gm_cond_var(main_program, k_steps, dist_context): main_block = main_program.global_block() # Add const var - k_step_var = layers.create_global_var(name="gradient_merge_k", - shape=[1], - value=int(k_steps), - dtype='int32', - persistable=True, - force_cpu=True) + k_step_var = layers.create_global_var( + name="gradient_merge_k", + shape=[1], + value=int(k_steps), + dtype='int32', + persistable=True, + force_cpu=True, + ) set_var_dist_attr(dist_context, k_step_var, [-1], world_process_group.ranks) - zero_var = layers.create_global_var(name="gradient_merge_zero", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + zero_var = layers.create_global_var( + name="gradient_merge_zero", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) set_var_dist_attr(dist_context, zero_var, [-1], world_process_group.ranks) # Add step var & cond var - step_var = layers.create_global_var(name="gradient_merge_step", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + step_var = layers.create_global_var( + name="gradient_merge_step", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) set_var_dist_attr(dist_context, step_var, [-1], world_process_group.ranks) - cond_var = main_block.create_var(name="gradient_merge_cond", - shape=[1], - dtype='bool') + cond_var = main_block.create_var( + name="gradient_merge_cond", shape=[1], dtype='bool' + ) set_var_dist_attr(dist_context, cond_var, [-1], world_process_group.ranks) with device_guard("cpu"): # step_var += 1 - increment_op = main_block.append_op(type='increment', - inputs={'X': [step_var]}, - outputs={'Out': [step_var]}, - attrs={ - 'step': float(1.0), - OP_ROLE_KEY: OpRole.Backward - }) + increment_op = main_block.append_op( + type='increment', + inputs={'X': [step_var]}, + outputs={'Out': [step_var]}, + attrs={'step': float(1.0), OP_ROLE_KEY: OpRole.Backward}, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - increment_op, world_process_group.ranks, [-1], dist_context) + increment_op, world_process_group.ranks, [-1], dist_context + ) # step_var %= k_step - elementwise_mod_op = main_block.append_op(type='elementwise_mod', - inputs={ - 'X': step_var, - 'Y': k_step_var - }, - outputs={'Out': step_var}, - attrs={ - 'axis': -1, - 'use_mkldnn': False, - OP_ROLE_KEY: - OpRole.Backward - }) + elementwise_mod_op = main_block.append_op( + type='elementwise_mod', + inputs={'X': step_var, 'Y': k_step_var}, + outputs={'Out': step_var}, + attrs={ + 'axis': -1, + 'use_mkldnn': False, + OP_ROLE_KEY: OpRole.Backward, + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - elementwise_mod_op, world_process_group.ranks, [-1], dist_context) + elementwise_mod_op, world_process_group.ranks, [-1], dist_context + ) # cond_var = (step_var == 0) - equal_op = main_block.append_op(type='equal', - inputs={ - 'X': step_var, - 'Y': zero_var - }, - outputs={'Out': cond_var}, - attrs={OP_ROLE_KEY: OpRole.Backward}) + equal_op = main_block.append_op( + type='equal', + inputs={'X': step_var, 'Y': zero_var}, + outputs={'Out': cond_var}, + attrs={OP_ROLE_KEY: OpRole.Backward}, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - equal_op, world_process_group.ranks, [-1], dist_context) + equal_op, world_process_group.ranks, [-1], dist_context + ) return cond_var def _append_gradient_merge_backward_op( - main_program, startup_program, params_grads: List[Tuple[Any, Any]], - dist_context) -> Tuple[List[Tuple[Any, Any]], Dict[str, Any]]: + main_program, + startup_program, + params_grads: List[Tuple[Any, Any]], + dist_context, +) -> Tuple[List[Tuple[Any, Any]], Dict[str, Any]]: main_block = main_program.global_block() startup_block = startup_program.global_block() @@ -146,57 +163,66 @@ def _append_gradient_merge_backward_op( for param, grad in params_grads: param_name = param.name param_var = main_block.var(param_name) - assert (param_var is not None) + assert param_var is not None ref_dist_attr = dist_context.get_tensor_dist_attr_for_program(param_var) assert ref_dist_attr is not None - gradient_merge_var = main_block.create_var(name=param_name + - "@GRAD@GradientMerge", - shape=param_var.shape, - dtype=param_var.dtype, - persistable=True) + gradient_merge_var = main_block.create_var( + name=param_name + "@GRAD@GradientMerge", + shape=param_var.shape, + dtype=param_var.dtype, + persistable=True, + ) ref_process_mesh = ref_dist_attr.process_mesh ref_dims_mapping = ref_dist_attr.dims_mapping - set_var_dist_attr(dist_context, gradient_merge_var, ref_dims_mapping, - ref_process_mesh) + set_var_dist_attr( + dist_context, gradient_merge_var, ref_dims_mapping, ref_process_mesh + ) startup_gradient_merge_var = startup_block.create_var( name=param_name + "@GRAD@GradientMerge", shape=param_var.shape, dtype=param_var.dtype, - persistable=True) - startup_block.append_op(type="fill_constant", - outputs={"Out": startup_gradient_merge_var}, - attrs={ - "shape": param_var.shape, - "dtype": param_var.dtype, - "value": float(0), - }) + persistable=True, + ) + startup_block.append_op( + type="fill_constant", + outputs={"Out": startup_gradient_merge_var}, + attrs={ + "shape": param_var.shape, + "dtype": param_var.dtype, + "value": float(0), + }, + ) # grad_merge += grad - new_grad_op = main_block.append_op(type="elementwise_add", - inputs={ - 'X': grad, - 'Y': gradient_merge_var - }, - outputs={'Out': gradient_merge_var}, - attrs={ - 'axis': -1, - 'use_mkldnn': False, - OP_ROLE_KEY: OpRole.Backward - }) + new_grad_op = main_block.append_op( + type="elementwise_add", + inputs={'X': grad, 'Y': gradient_merge_var}, + outputs={'Out': gradient_merge_var}, + attrs={ + 'axis': -1, + 'use_mkldnn': False, + OP_ROLE_KEY: OpRole.Backward, + }, + ) new_params_to_grads.append([param, gradient_merge_var]) grad_to_gradient_merge[grad.name] = gradient_merge_var.name naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_grad_op, ref_process_mesh, ref_dims_mapping, dist_context) + new_grad_op, ref_process_mesh, ref_dims_mapping, dist_context + ) return new_params_to_grads, grad_to_gradient_merge def _create_cond_block_and_update_optimizer( - main_program, cond_var, new_params_to_grads: List[Tuple[Any, Any]], - grad_to_gradient_merge: Dict[str, str], optimize_ops_desc: List[Any], - k_steps, avg): - + main_program, + cond_var, + new_params_to_grads: List[Tuple[Any, Any]], + grad_to_gradient_merge: Dict[str, str], + optimize_ops_desc: List[Any], + k_steps, + avg, +): def true_apply_gradient(): cur_block_idx = main_program.current_block_idx cur_block = main_program.current_block() @@ -207,14 +233,16 @@ def _create_cond_block_and_update_optimizer( if avg: for param, new_grad in new_params_to_grads: # grad /= k_steps - cur_block.append_op(type='scale', - inputs={'X': new_grad}, - outputs={'Out': new_grad}, - attrs={ - 'scale': 1.0 / k_steps, - 'bias': 0.0, - 'bias_after_scale': False - }) + cur_block.append_op( + type='scale', + inputs={'X': new_grad}, + outputs={'Out': new_grad}, + attrs={ + 'scale': 1.0 / k_steps, + 'bias': 0.0, + 'bias_after_scale': False, + }, + ) new_grad.op._set_attr(OP_ROLE_KEY, OpRole.Optimize) # append optimizer ops @@ -222,16 +250,18 @@ def _create_cond_block_and_update_optimizer( new_op_desc = cur_block.desc.append_op() new_op_desc.copy_from(op_desc) - #update input/output + # update input/output for input_name in new_op_desc.input_arg_names(): if input_name in grad_to_gradient_merge: new_op_desc._rename_input( - input_name, grad_to_gradient_merge[input_name]) + input_name, grad_to_gradient_merge[input_name] + ) for output_name in new_op_desc.output_arg_names(): if output_name in grad_to_gradient_merge: new_op_desc._rename_output( - output_name, grad_to_gradient_merge[output_name]) + output_name, grad_to_gradient_merge[output_name] + ) # remove op_role_var if new_op_desc.has_attr(op_maker.kOpRoleVarAttrName()): @@ -249,10 +279,12 @@ def _create_cond_block_and_update_optimizer( # clear gradient_merge_vars for param, new_grad in new_params_to_grads: - layers.fill_constant(shape=new_grad.shape, - dtype=new_grad.dtype, - value=0.0, - out=new_grad) + layers.fill_constant( + shape=new_grad.shape, + dtype=new_grad.dtype, + value=0.0, + out=new_grad, + ) new_grad.op._set_attr(OP_ROLE_KEY, op_maker.OpRole.Optimize) layers.cond(cond_var, true_fn=true_apply_gradient, false_fn=None) @@ -260,8 +292,9 @@ def _create_cond_block_and_update_optimizer( cond_op._set_attr(OP_ROLE_KEY, OpRole.Optimize) -def parse_program(main_program, startup_program, params_grads, k_steps, avg, - dist_context): +def parse_program( + main_program, startup_program, params_grads, k_steps, avg, dist_context +): # 1 remove optimizer_op from main_program optimize_ops_desc = _remove_and_get_optimizer_op(main_program, dist_context) @@ -269,22 +302,30 @@ def parse_program(main_program, startup_program, params_grads, k_steps, avg, main_program._rollback() # 2 append gradient merge backward op to main_program - new_params_to_grads, grad_to_gradient_merge = _append_gradient_merge_backward_op( - main_program, startup_program, params_grads, dist_context) + ( + new_params_to_grads, + grad_to_gradient_merge, + ) = _append_gradient_merge_backward_op( + main_program, startup_program, params_grads, dist_context + ) # 3 create gradient_merge_cond cond_var = _get_gm_cond_var(main_program, k_steps, dist_context) # 4 create ConditionalBlock and append gradient merge optimizer ops - _create_cond_block_and_update_optimizer(main_program, cond_var, - new_params_to_grads, - grad_to_gradient_merge, - optimize_ops_desc, k_steps, avg) + _create_cond_block_and_update_optimizer( + main_program, + cond_var, + new_params_to_grads, + grad_to_gradient_merge, + optimize_ops_desc, + k_steps, + avg, + ) @register_pass("auto_parallel_gradient_merge_pass") class GradientMergePass(PassBase): - def __init__(self): super(GradientMergePass, self).__init__() self.set_attr("k_steps", -1) @@ -307,7 +348,13 @@ class GradientMergePass(PassBase): dist_context = self.get_attr("dist_context") params_grads = self.get_attr("params_grads") with paddle.static.program_guard(main_program, startup_program): - parse_program(main_program, startup_program, params_grads, k_steps, - avg, dist_context) + parse_program( + main_program, + startup_program, + params_grads, + k_steps, + avg, + dist_context, + ) main_program._sync_with_cpp() diff --git a/python/paddle/distributed/passes/auto_parallel_quantization.py b/python/paddle/distributed/passes/auto_parallel_quantization.py index c0ac93d8393..893b9d0d9e5 100644 --- a/python/paddle/distributed/passes/auto_parallel_quantization.py +++ b/python/paddle/distributed/passes/auto_parallel_quantization.py @@ -20,7 +20,10 @@ from paddle.fluid.contrib.slim.quantization import utils from paddle.fluid.contrib.slim.quantization import QuantizationTransformPassV2 from paddle.fluid.contrib.slim.quantization import AddQuantDequantPassV2 from paddle.fluid.contrib.slim.quantization import OutScaleForTrainingPass -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute, TensorDistributedAttribute +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, + TensorDistributedAttribute, +) from .pass_base import PassBase, register_pass @@ -34,7 +37,6 @@ def _node_id(node): @register_pass("auto_parallel_quantization") class QuantizationPass(PassBase): - def __init__(self): super(QuantizationPass, self).__init__() self.set_attr("dist_context", None) @@ -61,14 +63,19 @@ class QuantizationPass(PassBase): place = paddle.fluid.CUDAPlace(ParallelEnv().dev_id) # 1. Program convert to Graph, and this pass is only for train mode - main_graph = framework.IrGraph(core.Graph(main_program.desc), - for_test=False) + main_graph = framework.IrGraph( + core.Graph(main_program.desc), for_test=False + ) # 2. Prepare inputs transform_pass_ops = [] quant_dequant_ops = [] quantize_op_types = [ - 'conv2d', 'depthwise_conv2d', 'mul', 'matmul', 'matmul_v2' + 'conv2d', + 'depthwise_conv2d', + 'mul', + 'matmul', + 'matmul_v2', ] for op_type in quantize_op_types: if op_type in TRANSFORM_PASS_OP_TYPES: @@ -76,8 +83,11 @@ class QuantizationPass(PassBase): elif op_type in QUANT_DEQUANT_PASS_OP_TYPES: quant_dequant_ops.append(op_type) - weight_quantize_type = "channel_wise_abs_max" if self.get_attr( - 'channel_wise_abs_max') else "abs_max" + weight_quantize_type = ( + "channel_wise_abs_max" + if self.get_attr('channel_wise_abs_max') + else "abs_max" + ) # 3. Add quant op for ops which have parameters transform_pass = QuantizationTransformPassV2( @@ -94,7 +104,8 @@ class QuantizationPass(PassBase): weight_preprocess_func=None, act_preprocess_func=None, optimizer_func=None, - executor=None) + executor=None, + ) transform_pass.apply(main_graph) # 4. Add quant op for ops which don't have parameter @@ -103,12 +114,14 @@ class QuantizationPass(PassBase): place=place, quant_bits=self.get_attr('activation_bits'), skip_pattern=self.get_attr('not_quant_pattern'), - quantizable_op_type=quant_dequant_ops) + quantizable_op_type=quant_dequant_ops, + ) quant_dequant_pass.apply(main_graph) # 5. Gather quantitative information for the output - out_scale_training_pass = OutScaleForTrainingPass(scope=scope, - place=place) + out_scale_training_pass = OutScaleForTrainingPass( + scope=scope, place=place + ) out_scale_training_pass.apply(main_graph) # 6. Convert Graph back to Program @@ -132,34 +145,43 @@ class QuantizationPass(PassBase): for ip, quant_op in enumerate(block.ops): quant_op_dist_attr = OperatorDistributedAttribute() - if "quantize" in quant_op.type or \ - quant_op.type == "moving_average_abs_max_scale": + if ( + "quantize" in quant_op.type + or quant_op.type == "moving_average_abs_max_scale" + ): input_name = quant_op.desc.input('X')[0] if "quantize" in input_name: - input_name = input_name[:input_name.index(".quantized")] + input_name = input_name[ + : input_name.index(".quantized") + ] if quant_op.type == "moving_average_abs_max_scale": consume_op = main_program.blocks[ib].vars[input_name].op else: - consume_op = main_program.blocks[ib].ops[ip - - qat_offset] + consume_op = main_program.blocks[ib].ops[ + ip - qat_offset + ] consume_op_dist_attr = dist_context.get_dist_op_for_program( - consume_op).dist_attr + consume_op + ).dist_attr ref_process_mesh = consume_op_dist_attr.process_mesh if input_name in consume_op_dist_attr.outputs_dist_attrs: - consume_input_dist_attr = consume_op_dist_attr.outputs_dist_attrs[ - input_name] + consume_input_dist_attr = ( + consume_op_dist_attr.outputs_dist_attrs[input_name] + ) else: - consume_input_dist_attr = consume_op_dist_attr.inputs_dist_attrs[ - input_name] + consume_input_dist_attr = ( + consume_op_dist_attr.inputs_dist_attrs[input_name] + ) quant_op_dist_attr.impl_idx = 0 quant_op_dist_attr.impl_type = "default" quant_op_dist_attr.process_mesh = ref_process_mesh quant_op_dist_attr.set_input_dist_attr( - quant_op.desc.input('X')[0], consume_input_dist_attr) + quant_op.desc.input('X')[0], consume_input_dist_attr + ) for slot_name in quant_op.desc.input_names(): if slot_name == "X": @@ -170,26 +192,32 @@ class QuantizationPass(PassBase): tensor_dist_attr.process_mesh = ref_process_mesh tensor_dist_attr.dims_mapping = [-1] dist_context.set_tensor_dist_attr_for_program( - input_var, tensor_dist_attr) + input_var, tensor_dist_attr + ) quant_op_dist_attr.set_input_dist_attr( - in_name, tensor_dist_attr) + in_name, tensor_dist_attr + ) for slot_name in quant_op.desc.output_names(): output_name = quant_op.desc.output(slot_name)[0] output_var = block.vars[output_name] if slot_name == "Y": dist_context.set_tensor_dist_attr_for_program( - output_var, consume_input_dist_attr) + output_var, consume_input_dist_attr + ) quant_op_dist_attr.set_output_dist_attr( - output_name, consume_input_dist_attr) + output_name, consume_input_dist_attr + ) else: tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = ref_process_mesh tensor_dist_attr.dims_mapping = [-1] dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) quant_op_dist_attr.set_output_dist_attr( - output_name, tensor_dist_attr) + output_name, tensor_dist_attr + ) quant_op._set_attr("op_device", "") qat_offset += 1 @@ -199,59 +227,86 @@ class QuantizationPass(PassBase): origin_op = main_program.blocks[ib].ops[ip - qat_offset] quant_op.desc.set_original_id(origin_op.desc.original_id()) dist_origin_op = dist_context.get_dist_op_for_program( - origin_op) - assert dist_origin_op is not None, "origin op must have dist attr." + origin_op + ) + assert ( + dist_origin_op is not None + ), "origin op must have dist attr." origin_op_dist_attr = dist_origin_op.dist_attr quant_op_dist_attr.impl_idx = origin_op_dist_attr.impl_idx quant_op_dist_attr.impl_type = origin_op_dist_attr.impl_type - quant_op_dist_attr.process_mesh = origin_op_dist_attr.process_mesh + quant_op_dist_attr.process_mesh = ( + origin_op_dist_attr.process_mesh + ) for idx, input_name in enumerate(quant_op.input_arg_names): origin_input_name = origin_op.input_arg_names[idx] - origin_input_dist_attr = origin_op_dist_attr.inputs_dist_attrs[ - origin_input_name] + origin_input_dist_attr = ( + origin_op_dist_attr.inputs_dist_attrs[ + origin_input_name + ] + ) quant_op_dist_attr.set_input_dist_attr( - input_name, origin_input_dist_attr) + input_name, origin_input_dist_attr + ) if input_name not in main_program.blocks[ib].vars: origin_input_var = main_program.blocks[ib].vars[ - origin_input_name] - origin_in_tensor_dist_attr = dist_context.get_dist_tensor_for_program( - origin_input_var).dist_attr + origin_input_name + ] + origin_in_tensor_dist_attr = ( + dist_context.get_dist_tensor_for_program( + origin_input_var + ).dist_attr + ) quant_input_var = block.vars[input_name] dist_context.set_tensor_dist_attr_for_program( - quant_input_var, origin_in_tensor_dist_attr) + quant_input_var, origin_in_tensor_dist_attr + ) for idx, output_name in enumerate( - quant_op.output_arg_names): + quant_op.output_arg_names + ): origin_output_name = origin_op.output_arg_names[idx] - origin_output_dist_attr = origin_op_dist_attr.outputs_dist_attrs[ - origin_output_name] + origin_output_dist_attr = ( + origin_op_dist_attr.outputs_dist_attrs[ + origin_output_name + ] + ) quant_op_dist_attr.set_output_dist_attr( - output_name, origin_output_dist_attr) + output_name, origin_output_dist_attr + ) if output_name not in main_program.blocks[ib].vars: origin_output_var = main_program.blocks[ib].vars[ - origin_output_name] - origin_out_tensor_dist_attr = dist_context.get_dist_tensor_for_program( - origin_output_var).dist_attr + origin_output_name + ] + origin_out_tensor_dist_attr = ( + dist_context.get_dist_tensor_for_program( + origin_output_var + ).dist_attr + ) quant_output_var = block.vars[output_name] dist_context.set_tensor_dist_attr_for_program( - quant_output_var, origin_out_tensor_dist_attr) + quant_output_var, origin_out_tensor_dist_attr + ) dist_context.set_op_dist_attr_for_program( - quant_op, quant_op_dist_attr) + quant_op, quant_op_dist_attr + ) # recover vars' dist_attr for name, dst_var in block.vars.items(): if name in main_program.blocks[ib].vars: src_var = main_program.blocks[ib].vars[name] dist_tensor = dist_context.get_dist_tensor_for_program( - src_var) + src_var + ) if not dist_tensor: continue dist_context.set_tensor_dist_attr_for_program( - dst_var, dist_tensor.dist_attr) + dst_var, dist_tensor.dist_attr + ) context.set_attr("main_program", quant_program) context.set_attr("startup_program", startup_program) diff --git a/python/paddle/distributed/passes/auto_parallel_recompute.py b/python/paddle/distributed/passes/auto_parallel_recompute.py index 81bda0d3d39..26568fbe727 100644 --- a/python/paddle/distributed/passes/auto_parallel_recompute.py +++ b/python/paddle/distributed/passes/auto_parallel_recompute.py @@ -20,13 +20,20 @@ from paddle.fluid import framework as framework from paddle.fluid.framework import Variable from paddle.fluid.backward import _append_grad_suffix_, _get_no_grad_set_name from paddle.fluid.backward import ProgramStats, _rename_arg_, _find_op_path_ -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute -from paddle.distributed.auto_parallel.utils import get_loss_op, set_var_dist_attr, set_dist_op_desc_original_id -from paddle.distributed.auto_parallel.utils import naive_set_dist_op_attr_for_program_by_mesh_and_mapping +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) +from paddle.distributed.auto_parallel.utils import ( + get_loss_op, + set_var_dist_attr, + set_dist_op_desc_original_id, +) +from paddle.distributed.auto_parallel.utils import ( + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) class RecomputeState(ProgramStats): - def __init__(self, block, ops): super(RecomputeState, self).__init__(block=block, ops=ops) self._block = block @@ -52,7 +59,7 @@ class RecomputeState(ProgramStats): self.var_op_deps[name]["var_as_output_ops"] = [i] def get_recompute_segments(self, checkpoints): - """ get recompute segments from checkpoints """ + """get recompute segments from checkpoints""" segments = [] start_idx = -1 pre_segment_end_idx = -1 @@ -67,27 +74,37 @@ class RecomputeState(ProgramStats): segments.append([0, max(op_idx_list) + 1]) else: flag, min_idx, max_idx = self.is_subgraph( - [checkpoints[start_idx]], [checkpoints[start_idx + 1]]) + [checkpoints[start_idx]], [checkpoints[start_idx + 1]] + ) if flag: min_idx = self._update_segment_start( - min_idx, pre_segment_end_idx) + min_idx, pre_segment_end_idx + ) segments.append([min_idx, max_idx + 1]) else: logging.info( "Could not recompute op range [{}] - [{}] ".format( - min_idx, max_idx + 1)) + min_idx, max_idx + 1 + ) + ) start_idx += 1 for i, (idx1, idx2) in enumerate(segments): logging.info("recompute segment[{}]".format(i)) - logging.info("segment start op: [{}]: [{}] [{}]".format( - self._ops[idx1].desc.type(), - self._ops[idx1].desc.input_arg_names(), - self._ops[idx1].desc.output_arg_names())) - logging.info("segment end op: [{}]: [{}] [{}]".format( - self._ops[idx2 - 1].desc.type(), - self._ops[idx2 - 1].desc.input_arg_names(), - self._ops[idx2 - 1].desc.output_arg_names())) + logging.info( + "segment start op: [{}]: [{}] [{}]".format( + self._ops[idx1].desc.type(), + self._ops[idx1].desc.input_arg_names(), + self._ops[idx1].desc.output_arg_names(), + ) + ) + logging.info( + "segment end op: [{}]: [{}] [{}]".format( + self._ops[idx2 - 1].desc.type(), + self._ops[idx2 - 1].desc.input_arg_names(), + self._ops[idx2 - 1].desc.output_arg_names(), + ) + ) return segments @@ -115,44 +132,49 @@ class RecomputeState(ProgramStats): cur_op_dist_attr = dist_context.get_op_dist_attr_for_program(cur_op) # insert seed op to guarantee that two dropout op have the same outputs op_unique_name = unique_name.generate("seed") - var_unique_name = unique_name.generate_with_ignorable_key(".".join( - [op_unique_name, 'tmp'])) + var_unique_name = unique_name.generate_with_ignorable_key( + ".".join([op_unique_name, 'tmp']) + ) seed_var = self._block.create_var( name=var_unique_name, dtype='int32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) # set new seed_var's dist_attr ref_dims_mapping = [-1] ref_process_mesh = cur_op_dist_attr.process_mesh - seed_var_dist_attr = set_var_dist_attr(dist_context, seed_var, - ref_dims_mapping, - ref_process_mesh) - - seed = 0 if cur_op.attr("fix_seed") is False else int( - cur_op.attr("seed")) + seed_var_dist_attr = set_var_dist_attr( + dist_context, seed_var, ref_dims_mapping, ref_process_mesh + ) + + seed = ( + 0 + if cur_op.attr("fix_seed") is False + else int(cur_op.attr("seed")) + ) seed_op = self._block._insert_op_without_sync( index=cur_op.idx, type="seed", inputs={}, outputs={"Out": seed_var}, - attrs={ - "seed": seed, - "force_cpu": True - }) + attrs={"seed": seed, "force_cpu": True}, + ) # set new seed op's dist_attr naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - seed_op, ref_process_mesh, ref_dims_mapping, dist_context) + seed_op, ref_process_mesh, ref_dims_mapping, dist_context + ) # modify dropout op's desc self._ops.insert(op_idx, seed_op) cur_op.desc.set_input("Seed", [var_unique_name]) cur_op._remove_attr("fix_seed") cur_op._remove_attr("seed") - cur_op_dist_attr.set_input_dist_attr(seed_var.name, - seed_var_dist_attr) + cur_op_dist_attr.set_input_dist_attr( + seed_var.name, seed_var_dist_attr + ) op_idx += 2 self._block._sync_with_cpp() @@ -166,7 +188,7 @@ def _find_op_index(block, cur_op): def _get_stop_gradients(program, no_grad_set): - """ get no grad var """ + """get no grad var""" if no_grad_set is None: no_grad_set = set() else: @@ -183,8 +205,9 @@ def _get_stop_gradients(program, no_grad_set): return no_grad_set_name -def _add_needed_descs_to_block(descs, block, main_block, in_memory_vars, - dist_context): +def _add_needed_descs_to_block( + descs, block, main_block, in_memory_vars, dist_context +): """ Get the recomputed ops which will insert the backward part """ @@ -215,7 +238,6 @@ def _add_needed_descs_to_block(descs, block, main_block, in_memory_vars, @register_pass("auto_parallel_recompute") class RecomputePass(PassBase): - def __init__(self): super(RecomputePass, self).__init__() self.set_attr("checkpoints", None) @@ -259,12 +281,15 @@ class RecomputePass(PassBase): vars_should_be_hold = [] for segment in segments: vars_should_be_hold.extend( - rc_state.get_out_of_subgraph_vars(segment[0], segment[1])) + rc_state.get_out_of_subgraph_vars(segment[0], segment[1]) + ) cross_vars = set(vars_should_be_hold) - set(checkpoints) logging.info( "found [{}] vars which cross recompute segment: [{}]," "better checkpoints might be set to reduce those vars".format( - len(cross_vars), cross_vars)) + len(cross_vars), cross_vars + ) + ) vars_should_be_hold.extend(rc_state.get_reserved_vars()) vars_should_be_hold.extend(rc_state.get_input_nodes()) vars_should_be_hold = list(set(vars_should_be_hold)) @@ -275,14 +300,15 @@ class RecomputePass(PassBase): ckpt_ops_dict = {} buffer_block = main_block.program._create_block() for i, segment in enumerate(segments[::-1]): - fwd_ops = op_path[segment[0]:segment[1]] + fwd_ops = op_path[segment[0] : segment[1]] var_suffix = ".subprog_%d" % i for op in fwd_ops: input_and_output_names = [] input_and_output_names.extend(op.desc.input_arg_names()) input_and_output_names.extend(op.desc.output_arg_names()) - cur_op_dist_attr = self._dist_context.get_op_dist_attr_for_program( - op) + cur_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program(op) + ) assert cur_op_dist_attr is not None for name in input_and_output_names: if main_block.var(name).persistable or name in checkpoints: @@ -292,11 +318,13 @@ class RecomputePass(PassBase): if name not in var_name_dict: ref_process_mesh = cur_op_dist_attr.process_mesh if name in op.desc.input_arg_names(): - ref_dims_mapping = cur_op_dist_attr.get_input_dims_mapping( - name) + ref_dims_mapping = ( + cur_op_dist_attr.get_input_dims_mapping(name) + ) else: - ref_dims_mapping = cur_op_dist_attr.get_output_dims_mapping( - name) + ref_dims_mapping = ( + cur_op_dist_attr.get_output_dims_mapping(name) + ) # record recomputed var's old_name and new_name (old_name.subprog_XXX) # create new var with new name var_name_dict[name] = name + var_suffix @@ -307,15 +335,23 @@ class RecomputePass(PassBase): dtype=ref_var.dtype, type=ref_var.type, persistable=ref_var.persistable, - stop_gradient=ref_var.stop_gradient) + stop_gradient=ref_var.stop_gradient, + ) # set new recomputed var's dist attr - set_var_dist_attr(self._dist_context, rc_var, - ref_dims_mapping, ref_process_mesh) + set_var_dist_attr( + self._dist_context, + rc_var, + ref_dims_mapping, + ref_process_mesh, + ) # get recomputed segment's descs - segment_descs = _add_needed_descs_to_block(fwd_ops, buffer_block, - main_block, - vars_in_memory, - self._dist_context) + segment_descs = _add_needed_descs_to_block( + fwd_ops, + buffer_block, + main_block, + vars_in_memory, + self._dist_context, + ) # rename recomputed ops' input and output var name for key in var_name_dict: _rename_arg_(segment_descs, key, var_name_dict[key]) @@ -343,7 +379,10 @@ class RecomputePass(PassBase): # rename grad op's var_name which is not in 'vars_in_memory' for key in var_name_dict: - if key not in grad_op.input_arg_names + grad_op.output_arg_names: + if ( + key + not in grad_op.input_arg_names + grad_op.output_arg_names + ): continue self.reset_op_dist_attr(grad_op, var_name_dict) _rename_arg_([grad_op.desc], key, var_name_dict[key]) @@ -358,17 +397,20 @@ class RecomputePass(PassBase): idx -= 1 segment_descs = ckpt_ops_dict[fwd_op_id][1] for _, op_desc in reversed(list(enumerate(segment_descs))): - rc_op = main_block._insert_op_without_sync(idx, - type='nop') + rc_op = main_block._insert_op_without_sync( + idx, type='nop' + ) rc_desc = rc_op.desc rc_desc.copy_from(op_desc) rc_desc.set_original_id(rc_desc.id()) # set recomputed ops' dist attr fwd_op_dist_attr = self._dist_context.get_op_dist_attr_for_program_with_id( - op_desc.original_id()) + op_desc.original_id() + ) assert fwd_op_dist_attr is not None - self.set_op_dist_attr(rc_op, fwd_op_dist_attr, - var_name_dict) + self.set_op_dist_attr( + rc_op, fwd_op_dist_attr, var_name_dict + ) ckpt_ops_dict[fwd_op_id][0] = False @@ -380,13 +422,15 @@ class RecomputePass(PassBase): for input in op.desc.input_arg_names(): if input in var_name_dict.keys(): in_dist_attr = op_dist_attr.get_input_dist_attr(input) - op_dist_attr.set_input_dist_attr(var_name_dict[input], - in_dist_attr) + op_dist_attr.set_input_dist_attr( + var_name_dict[input], in_dist_attr + ) for output in op.desc.output_arg_names(): if output in var_name_dict.keys(): out_dist_attr = op_dist_attr.get_output_dist_attr(output) - op_dist_attr.set_output_dist_attr(var_name_dict[output], - out_dist_attr) + op_dist_attr.set_output_dist_attr( + var_name_dict[output], out_dist_attr + ) def set_op_dist_attr(self, op, old_dist_attr, var_name_dict): new_dist_attr = OperatorDistributedAttribute() @@ -397,16 +441,18 @@ class RecomputePass(PassBase): for input in old_dist_attr.inputs_dist_attrs.keys(): if input in var_name_dict.keys(): in_dist_attr = old_dist_attr.inputs_dist_attrs[input] - new_dist_attr.set_input_dist_attr(var_name_dict[input], - in_dist_attr) + new_dist_attr.set_input_dist_attr( + var_name_dict[input], in_dist_attr + ) else: in_dist_attr = old_dist_attr.inputs_dist_attrs[input] new_dist_attr.set_input_dist_attr(input, in_dist_attr) for output in old_dist_attr.outputs_dist_attrs.keys(): if output in var_name_dict.keys(): out_dist_attr = old_dist_attr.outputs_dist_attrs[output] - new_dist_attr.set_output_dist_attr(var_name_dict[output], - out_dist_attr) + new_dist_attr.set_output_dist_attr( + var_name_dict[output], out_dist_attr + ) else: out_dist_attr = old_dist_attr.outputs_dist_attrs[output] new_dist_attr.set_output_dist_attr(output, out_dist_attr) diff --git a/python/paddle/distributed/passes/auto_parallel_sharding.py b/python/paddle/distributed/passes/auto_parallel_sharding.py index 171188618ef..51262506598 100644 --- a/python/paddle/distributed/passes/auto_parallel_sharding.py +++ b/python/paddle/distributed/passes/auto_parallel_sharding.py @@ -17,27 +17,51 @@ from functools import reduce from paddle.framework import core from paddle.fluid import unique_name from .pass_base import PassBase, register_pass -from paddle.distributed.fleet.meta_optimizers.common import is_backward_op, is_optimizer_op +from paddle.distributed.fleet.meta_optimizers.common import ( + is_backward_op, + is_optimizer_op, +) from paddle.distributed.auto_parallel.process_group import new_process_group -from paddle.distributed.auto_parallel.operators.common import is_parameter_related, is_data_parallel_reduce_op -from paddle.distributed.auto_parallel.utils import _get_comm_group, naive_set_dist_op_attr_for_program_by_mesh_and_mapping, set_var_dist_attr +from paddle.distributed.auto_parallel.operators.common import ( + is_parameter_related, + is_data_parallel_reduce_op, +) +from paddle.distributed.auto_parallel.utils import ( + _get_comm_group, + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, + set_var_dist_attr, +) OpRole = core.op_proto_and_checker_maker.OpRole OP_ROLE_KEY = core.op_proto_and_checker_maker.kOpRoleAttrName() _skip_ops = [ - 'create_py_reader', 'create_double_buffer_reader', 'read', 'slice', 'split', - 'assign', "send_v2" + 'create_py_reader', + 'create_double_buffer_reader', + 'read', + 'slice', + 'split', + 'assign', + "send_v2", ] # update here to support new optimizers _supported_optimizer_type = [ - "adam", "adamax", "adamw", "decayed_adagrad", "momentum", "dgc_momentum", - "lars_momentum", "merged_momentum", "lamb", "sgd" + "adam", + "adamax", + "adamw", + "decayed_adagrad", + "momentum", + "dgc_momentum", + "lars_momentum", + "merged_momentum", + "lamb", + "sgd", ] def _is_reshard_op(op): - return op.desc.has_attr("op_namescope") and \ - "/auto_parallel/reshard" in op.desc.attr('op_namescope') + return op.desc.has_attr( + "op_namescope" + ) and "/auto_parallel/reshard" in op.desc.attr('op_namescope') # NOTE we add the "auto_parallel" prefix to the pass in order to @@ -46,7 +70,6 @@ def _is_reshard_op(op): # should use dist op instead of custom comm op @register_pass("auto_parallel_sharding") class ShardingPass(PassBase): - def __init__(self): super(ShardingPass, self).__init__() self.set_attr("dist_context", None) @@ -69,19 +92,22 @@ class ShardingPass(PassBase): if self.get_attr("stage") not in [1, 2, 3]: return False if self.get_attr("sharding_degree") is not None: - if (not isinstance(self.get_attr("sharding_degree"), int)) \ - or self.get_attr("sharding_degree") <= 1: + if ( + not isinstance(self.get_attr("sharding_degree"), int) + ) or self.get_attr("sharding_degree") <= 1: return False elif self.get_attr("degree") is not None: - if (not isinstance(self.get_attr("degree"), int)) \ - or self.get_attr("degree") <= 1: + if (not isinstance(self.get_attr("degree"), int)) or self.get_attr( + "degree" + ) <= 1: return False else: return False if len(self.get_attr("params_grads")) <= 0: return False - if (not isinstance(self.get_attr("global_rank"), - int)) or self.get_attr("global_rank") < 0: + if (not isinstance(self.get_attr("global_rank"), int)) or self.get_attr( + "global_rank" + ) < 0: return False return True @@ -92,12 +118,15 @@ class ShardingPass(PassBase): def _apply_single_impl(self, main_program, startup_program, context): self._dist_context = self.get_attr("dist_context") self.sharding_world_size = int( - self.get_attr("sharding_degree") or self.get_attr("degree")) + self.get_attr("sharding_degree") or self.get_attr("degree") + ) self.stage = int(self.get_attr("stage")) self.global_rank = int(self.get_attr("global_rank")) params_grads = self.get_attr("params_grads") - main_block, startup_block = main_program.global_block( - ), startup_program.global_block() + main_block, startup_block = ( + main_program.global_block(), + startup_program.global_block(), + ) self._build_sharding_groups(main_block, params_grads) self._shard_optimizer(main_block, startup_block, params_grads, context) @@ -119,7 +148,8 @@ class ShardingPass(PassBase): if _is_reshard_op(op): continue group = _inference_data_parallel_group_for_operator( - self.global_rank, op, self._dist_context) + self.global_rank, op, self._dist_context + ) if group is not None: self.dp_groups.add(group) @@ -127,32 +157,45 @@ class ShardingPass(PassBase): # genetated by auto search if len(self.dp_groups) != 1: raise NotImplementedError( - "So far Only and Exactly one data parallel group in network are supported, but got [{}] different data parallel groups" - .format(len(self.dp_groups))) + "So far Only and Exactly one data parallel group in network are supported, but got [{}] different data parallel groups".format( + len(self.dp_groups) + ) + ) def _build_sharding_infos(self, params_grads): for dp_group in self.dp_groups: - assert dp_group.nranks >= self.sharding_world_size, "sharding world size [{}] should not larger than dp world size [{}]".format( - self.sharding_world_size, dp_group.nranks) - assert dp_group.nranks % self.sharding_world_size == 0, "sharding world size [{}] should be divisible by dp world size [{}]".format( - self.sharding_world_size, dp_group.nranks) - assert self.global_rank in dp_group.ranks, "current ranks [{}] does NOT belong to the data parallel group [{}]".format( - self.global_rank, dp_group.ranks) - assert len( - params_grads - ) >= self.sharding_world_size, "number of parameters [{}] is not enough to be shard among [{}] ranks".format( - len(params_grads), self.sharding_world_size) + assert ( + dp_group.nranks >= self.sharding_world_size + ), "sharding world size [{}] should not larger than dp world size [{}]".format( + self.sharding_world_size, dp_group.nranks + ) + assert ( + dp_group.nranks % self.sharding_world_size == 0 + ), "sharding world size [{}] should be divisible by dp world size [{}]".format( + self.sharding_world_size, dp_group.nranks + ) + assert ( + self.global_rank in dp_group.ranks + ), "current ranks [{}] does NOT belong to the data parallel group [{}]".format( + self.global_rank, dp_group.ranks + ) + assert ( + len(params_grads) >= self.sharding_world_size + ), "number of parameters [{}] is not enough to be shard among [{}] ranks".format( + len(params_grads), self.sharding_world_size + ) # sharding hybrid data parallel: partial sharding param within if dp_group.nranks > self.sharding_world_size: self.partial_sharding = True - assert len( - self.dp_groups - ) == 1, "hybrid sharding and data parallelism are supported only when there is excatly one data parallel group in the network" + assert ( + len(self.dp_groups) == 1 + ), "hybrid sharding and data parallelism are supported only when there is excatly one data parallel group in the network" outer_dp_group, sharding_group = _get_dp_and_sharding_groups( - dp_group.ranks, self.sharding_world_size, self.global_rank) + dp_group.ranks, self.sharding_world_size, self.global_rank + ) sharding_group = new_process_group(sharding_group) self.outer_dp_group = new_process_group(outer_dp_group) else: @@ -160,14 +203,16 @@ class ShardingPass(PassBase): self._dist_context._sharding_group = sharding_group # TODO(JZ-LIANG) when support multiple dp groups in future, should group param and bind them to corresponding dp group - sharding_info = ShardingInfo(sharding_group, self.global_rank, - params_grads) + sharding_info = ShardingInfo( + sharding_group, self.global_rank, params_grads + ) self.sharding_infos.append(sharding_info) for param in sharding_info.params: self.varname_to_sharding_info[param.name] = sharding_info - def _shard_optimizer(self, main_block, startup_block, params_grads, - pass_context): + def _shard_optimizer( + self, main_block, startup_block, params_grads, pass_context + ): """ sharding all optimizer related ops and vars, include: gradient clip ops & vars @@ -189,7 +234,7 @@ class ShardingPass(PassBase): # shard amp related param_grad cast if _is_param_grad_fp32_cast_op(main_block, op): output_name = op.output_arg_names[0] - param_name = output_name[:output_name.find("@")] + param_name = output_name[: output_name.find("@")] if not self._is_parameter_in_local_shard(param_name): main_block._remove_op(idx, sync=False) main_block._remove_var(output_name, sync=False) @@ -198,7 +243,7 @@ class ShardingPass(PassBase): elif op.type in ["check_finite_and_unscale", "update_loss_scaling"]: reversed_x = [] for input_name in op.desc.input('X'): - param_name = input_name[:input_name.find("@")] + param_name = input_name[: input_name.find("@")] if self._is_parameter_in_local_shard(param_name): reversed_x.append(input_name) @@ -223,7 +268,8 @@ class ShardingPass(PassBase): "dtype": out_var.dtype, "value": 0, OP_ROLE_KEY: op_role, - }) + }, + ) else: main_block._remove_op(idx, sync=False) @@ -245,7 +291,7 @@ class ShardingPass(PassBase): if op.type in removed_op_type: input_name = op.input("X")[0] - param_name = input_name[:input_name.find("@GRAD")] + param_name = input_name[: input_name.find("@GRAD")] if not self._is_parameter_in_local_shard(param_name): removed_op_idx.add(idx) if op.type in ['squared_l2_norm', 'clip_by_norm']: @@ -283,9 +329,13 @@ class ShardingPass(PassBase): 'op_namescope': "/gradient_clip_model_parallelism", 'use_calc_stream': True, OP_ROLE_KEY: OpRole.Optimize, - }) - dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - main_block.var(sum_op_output)) + }, + ) + dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + main_block.var(sum_op_output) + ) + ) # assert dist_attr is not None # naive_set_dist_op_attr_for_program_by_mesh_and_mapping( # new_op, dist_attr.process_mesh, dist_attr.dims_mapping, @@ -304,7 +354,8 @@ class ShardingPass(PassBase): continue else: raise NotImplementedError( - "weight decay is NOT supported by now") + "weight decay is NOT supported by now" + ) main_block._sync_with_cpp() def _shard_optimizer_ops_and_states(self, main_block, startup_block): @@ -319,18 +370,24 @@ class ShardingPass(PassBase): assert len(op.input("Param")) == 1 param_name = op.input("Param")[0] if not self._is_parameter_in_local_shard(param_name): - should_removed_optimizer_states.extend([ - varname for varname in op.output_arg_names - if varname != param_name - ]) + should_removed_optimizer_states.extend( + [ + varname + for varname in op.output_arg_names + if varname != param_name + ] + ) main_block._remove_op(idx, sync=False) else: self.shared_params_grads.append( - self._get_param_grad(param_name)) + self._get_param_grad(param_name) + ) for idx, op in reversed(list(enumerate(startup_block.ops))): - if len(op.output_arg_names) == 1 and op.output_arg_names[ - 0] in should_removed_optimizer_states: + if ( + len(op.output_arg_names) == 1 + and op.output_arg_names[0] in should_removed_optimizer_states + ): startup_block._remove_op(idx, sync=False) for varname in should_removed_optimizer_states: @@ -352,26 +409,27 @@ class ShardingPass(PassBase): assert main_block.has_var(param.name) assert startup_block.has_var(param.name) - new_op = main_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': - sharding_info.group.id, - 'root': - sharding_info.get_var_rank( - param.name), - 'use_calc_stream': - True, - OP_ROLE_KEY: - OpRole.Optimize - }) - param_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - param) + new_op = main_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': sharding_info.group.id, + 'root': sharding_info.get_var_rank(param.name), + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + param_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program(param) + ) assert param_dist_attr is not None naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, param_dist_attr.process_mesh, - param_dist_attr.dims_mapping, self._dist_context) + new_op, + param_dist_attr.process_mesh, + param_dist_attr.dims_mapping, + self._dist_context, + ) main_block._sync_with_cpp() def _is_parameter_in_local_shard(self, param_name): @@ -397,12 +455,18 @@ class ShardingPass(PassBase): input_name = op.input_arg_names[0] base_name = _get_base_name_from_grad_name(input_name) sharding_info = self.varname_to_sharding_info[base_name] - _insert_reduce_op(main_block, idx, input_name, - sharding_info.group.id, - sharding_info.get_var_rank(base_name), - self._dist_context) - if not self.partial_sharding or not sharding_info.is_in_local_shard( - base_name): + _insert_reduce_op( + main_block, + idx, + input_name, + sharding_info.group.id, + sharding_info.get_var_rank(base_name), + self._dist_context, + ) + if ( + not self.partial_sharding + or not sharding_info.is_in_local_shard(base_name) + ): main_block._remove_op(idx + 1, sync=False) else: op._set_attr("ring_id", self.outer_dp_group.id) @@ -427,12 +491,17 @@ class ShardingPass(PassBase): dp_ring_ids = [group.id for group in self.dp_groups] for sharding_info in self.sharding_infos: - need_broadcast_vars, param_usage = sharding_info.get_broadcast_vars_and_param_usage( - main_block) + ( + need_broadcast_vars, + param_usage, + ) = sharding_info.get_broadcast_vars_and_param_usage(main_block) not_used_param_nane = [] for param_name in param_usage: - if param_usage[param_name] == 0 and sharding_info.get_var_rank( - param_name) != sharding_info.local_rank: + if ( + param_usage[param_name] == 0 + and sharding_info.get_var_rank(param_name) + != sharding_info.local_rank + ): not_used_param_nane.append(param_name) for idx, op in reversed(list(enumerate(main_block.ops))): @@ -442,8 +511,9 @@ class ShardingPass(PassBase): for input_name in op.desc.input_arg_names(): # NOTE hack for embedding op when AMP 02-3 # paddle amp force embedding (lookup table) to be run on fp32 - if _is_param_fp16_cast_op(main_block, op, - sharding_info.param_names): + if _is_param_fp16_cast_op( + main_block, op, sharding_info.param_names + ): continue if input_name not in need_broadcast_vars: continue @@ -451,28 +521,39 @@ class ShardingPass(PassBase): if root_rank == sharding_info.local_rank: broadcast_varname = input_name else: - broadcast_varname = unique_name.generate(input_name + - "@BroadCast") + broadcast_varname = unique_name.generate( + input_name + "@BroadCast" + ) input_var = main_block.var(input_name) - new_var = main_block.create_var(name=broadcast_varname, - shape=input_var.shape, - dtype=input_var.dtype, - persistable=False) - ref_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - input_var) + new_var = main_block.create_var( + name=broadcast_varname, + shape=input_var.shape, + dtype=input_var.dtype, + persistable=False, + ) + ref_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + input_var + ) + ) out_var_dist_attr = set_var_dist_attr( - self._dist_context, new_var, + self._dist_context, + new_var, ref_dist_attr.dims_mapping, - ref_dist_attr.process_mesh) + ref_dist_attr.process_mesh, + ) op._rename_input(input_name, broadcast_varname) - _insert_init_and_broadcast_op(main_block, idx, - broadcast_varname, - sharding_info.local_rank, - root_rank, - sharding_info.group.id, - op.attr('op_role'), - self._dist_context) + _insert_init_and_broadcast_op( + main_block, + idx, + broadcast_varname, + sharding_info.local_rank, + root_rank, + sharding_info.group.id, + op.attr('op_role'), + self._dist_context, + ) for idx, op in reversed(list(enumerate(main_block.ops))): if op.type != "cast": @@ -487,22 +568,33 @@ class ShardingPass(PassBase): assert len(op.output_arg_names) == 1 output_name = op.output_arg_names[0] - if op.type == "c_broadcast" and op.attr( - "ring_id") in dp_ring_ids: - if self.outer_dp_group and sharding_info.get_var_rank( - output_name) == sharding_info.local_rank: + if ( + op.type == "c_broadcast" + and op.attr("ring_id") in dp_ring_ids + ): + if ( + self.outer_dp_group + and sharding_info.get_var_rank(output_name) + == sharding_info.local_rank + ): op._set_attr("ring_id", self.outer_dp_group.id) else: startup_block._remove_op(idx, sync=False) continue - if op.type != "c_broadcast" and output_name in param_usage and sharding_info.get_var_rank( - output_name) != sharding_info.local_rank: + if ( + op.type != "c_broadcast" + and output_name in param_usage + and sharding_info.get_var_rank(output_name) + != sharding_info.local_rank + ): startup_block._remove_op(idx, sync=False) for param_name in param_usage: - if sharding_info.get_var_rank( - param_name) != sharding_info.local_rank: + if ( + sharding_info.get_var_rank(param_name) + != sharding_info.local_rank + ): main_block._remove_var(param_name, sync=False) startup_block._remove_var(param_name, sync=False) @@ -510,28 +602,42 @@ class ShardingPass(PassBase): startup_block._sync_with_cpp() -def _insert_init_and_broadcast_op(block, insert_idx, varname, local_rank, - root_rank, ring_id, op_role, dist_context): +def _insert_init_and_broadcast_op( + block, + insert_idx, + varname, + local_rank, + root_rank, + ring_id, + op_role, + dist_context, +): """ empty op for initialization """ broadcast_var = block.var(varname) broadcast_var_dist_attr = dist_context.get_tensor_dist_attr_for_program( - broadcast_var) - - new_op = block._insert_op_without_sync(insert_idx, - type='c_broadcast', - inputs={'X': varname}, - outputs={'Out': varname}, - attrs={ - 'ring_id': ring_id, - 'root': root_rank, - 'use_calc_stream': True, - OP_ROLE_KEY: op_role - }) + broadcast_var + ) + + new_op = block._insert_op_without_sync( + insert_idx, + type='c_broadcast', + inputs={'X': varname}, + outputs={'Out': varname}, + attrs={ + 'ring_id': ring_id, + 'root': root_rank, + 'use_calc_stream': True, + OP_ROLE_KEY: op_role, + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, broadcast_var_dist_attr.process_mesh, - broadcast_var_dist_attr.dims_mapping, dist_context) + new_op, + broadcast_var_dist_attr.process_mesh, + broadcast_var_dist_attr.dims_mapping, + dist_context, + ) if local_rank != root_rank: new_op = block._insert_op_without_sync( @@ -541,40 +647,50 @@ def _insert_init_and_broadcast_op(block, insert_idx, varname, local_rank, attrs={ "shape": broadcast_var.shape, "dtype": broadcast_var.dtype, - OP_ROLE_KEY: op_role - }) + OP_ROLE_KEY: op_role, + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, broadcast_var_dist_attr.process_mesh, - broadcast_var_dist_attr.dims_mapping, dist_context) + new_op, + broadcast_var_dist_attr.process_mesh, + broadcast_var_dist_attr.dims_mapping, + dist_context, + ) return -def _insert_reduce_op(block, - insert_idx, - reduce_var, - ring_id, - root_id, - dist_context, - op_role=OpRole.Backward, - use_calc_stream=True): - assert root_id >= 0, "root id should be a positive int, but now root id is {}".format( - root_id) - new_op = block._insert_op_without_sync(insert_idx, - type='c_reduce_sum', - inputs={'X': [reduce_var]}, - outputs={'Out': [reduce_var]}, - attrs={ - 'ring_id': ring_id, - 'root_id': root_id, - 'use_calc_stream': - use_calc_stream, - OP_ROLE_KEY: op_role - }) +def _insert_reduce_op( + block, + insert_idx, + reduce_var, + ring_id, + root_id, + dist_context, + op_role=OpRole.Backward, + use_calc_stream=True, +): + assert ( + root_id >= 0 + ), "root id should be a positive int, but now root id is {}".format(root_id) + new_op = block._insert_op_without_sync( + insert_idx, + type='c_reduce_sum', + inputs={'X': [reduce_var]}, + outputs={'Out': [reduce_var]}, + attrs={ + 'ring_id': ring_id, + 'root_id': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) dist_attr = dist_context.get_tensor_dist_attr_for_program( - block.var(reduce_var)) + block.var(reduce_var) + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, dist_attr.process_mesh, dist_attr.dims_mapping, dist_context) + new_op, dist_attr.process_mesh, dist_attr.dims_mapping, dist_context + ) def _get_dp_and_sharding_groups(origin_group, sharding_group_size, rank): @@ -589,23 +705,26 @@ def _get_dp_and_sharding_groups(origin_group, sharding_group_size, rank): def _is_gradient_clip_op(op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip") def _is_weight_decay_op(op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/regularization") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/regularization") def _is_param_grad_fp32_cast_op(block, op): if not is_backward_op(op): return False - if not _is_desired_cast_op(block, op, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32): + if not _is_desired_cast_op( + block, op, core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32 + ): return False output_name = op.desc.output_arg_names()[0] - base_name = output_name[:output_name.find("@")] + base_name = output_name[: output_name.find("@")] if not block.has_var(base_name): return False return block.var(base_name).is_parameter @@ -623,19 +742,20 @@ def _is_param_fp16_cast_op(block, op, params): return True -def _is_desired_cast_op(block, - op, - src_var_type=core.VarDesc.VarType.FP32, - dst_var_type=core.VarDesc.VarType.FP16): +def _is_desired_cast_op( + block, + op, + src_var_type=core.VarDesc.VarType.FP32, + dst_var_type=core.VarDesc.VarType.FP16, +): if op.type != "cast": return False - assert (len(op.desc.input_arg_names()) == 1) - assert (len(op.desc.output_arg_names()) == 1) + assert len(op.desc.input_arg_names()) == 1 + assert len(op.desc.output_arg_names()) == 1 input_var = block.var(op.desc.input_arg_names()[0]) output_var = block.var(op.desc.output_arg_names()[0]) - if input_var.dtype != src_var_type or \ - output_var.dtype != dst_var_type: + if input_var.dtype != src_var_type or output_var.dtype != dst_var_type: return False return True @@ -644,9 +764,9 @@ def _is_desired_cast_op(block, def _get_base_name_from_grad_name(grad_name): base_name = None if ".cast_fp16@GRAD" in grad_name: - base_name = grad_name[:grad_name.find(".cast_fp16@GRAD")] + base_name = grad_name[: grad_name.find(".cast_fp16@GRAD")] elif "@GRAD" in grad_name: - base_name = grad_name[:grad_name.find("@GRAD")] + base_name = grad_name[: grad_name.find("@GRAD")] return base_name @@ -696,9 +816,12 @@ def _inference_data_parallel_group_for_operator(rank_id, op, dist_context): # TODO(JZ-LIANG) replace with specific batch size dimension batch_size_axis = input_dim_mapping[0] if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, - batch_size_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + batch_size_axis, + rank_id, + ) dp_group = new_process_group(group_ranks) break @@ -717,20 +840,23 @@ def shard_parameters(params, group_size): rank = sizes.index(min(sizes)) mapping[rank].append(param) numel = reduce(lambda x, y: x * y, param.shape) - assert numel > 0, "param [{}] should larger than 0, but it is [{}]".format( - param.name, numel) + assert ( + numel > 0 + ), "param [{}] should larger than 0, but it is [{}]".format( + param.name, numel + ) sizes[rank] += numel return mapping class ShardingInfo(object): - def __init__(self, group, rank, params_grads): self.group = group self.params_grads = dict([(p.name, (p, g)) for p, g in params_grads]) - assert len(self.params_grads) == len(set( - self.params_grads)), "found duplicated param in params_grads" + assert len(self.params_grads) == len( + set(self.params_grads) + ), "found duplicated param in params_grads" self.params = [p for p, _ in params_grads] self.param_names = [p.name for p in self.params] @@ -796,7 +922,8 @@ class ShardingInfo(object): def get_param_grad(self, param_name): if not self.is_in_local_shard(param_name): raise ValueError( - "param[{}] not in current rank.".format(param_name)) + "param[{}] not in current rank.".format(param_name) + ) if param_name not in self.params_grads: raise ValueError('param[{}] not in params_grads'.format(param_name)) return self.params_grads.get(param_name, None) diff --git a/python/paddle/distributed/passes/cpp_pass.py b/python/paddle/distributed/passes/cpp_pass.py index c729a919c1a..17d54186dad 100644 --- a/python/paddle/distributed/passes/cpp_pass.py +++ b/python/paddle/distributed/passes/cpp_pass.py @@ -19,7 +19,6 @@ from paddle.fluid.framework import core, _apply_pass as _apply_cpp_pass @register_pass("fuse_elewise_add_act") class FuseElementwiseAddActPass(CPPPassWrapper): - def __init__(self): super(FuseElementwiseAddActPass, self).__init__() @@ -33,7 +32,6 @@ class FuseElementwiseAddActPass(CPPPassWrapper): @register_pass("fuse_bn_act") class FuseBatchNormActPass(CPPPassWrapper): - def __init__(self): super(FuseBatchNormActPass, self).__init__() @@ -47,7 +45,6 @@ class FuseBatchNormActPass(CPPPassWrapper): @register_pass("fuse_bn_add_act") class FuseBatchNormAddActPass(CPPPassWrapper): - def __init__(self): super(FuseBatchNormAddActPass, self).__init__() @@ -61,7 +58,6 @@ class FuseBatchNormAddActPass(CPPPassWrapper): @register_pass("fuse_relu_depthwise_conv") class FuseReluDepthwiseConvPass(CPPPassWrapper): - def __init__(self): super(FuseReluDepthwiseConvPass, self).__init__() @@ -75,14 +71,15 @@ class FuseReluDepthwiseConvPass(CPPPassWrapper): @register_pass("fuse_optimizer") class FuseOptimizerPass(CPPPassWrapper): - def __init__(self): super(FuseOptimizerPass, self).__init__() @property def cpp_name(self): return [ - "fuse_adam_op_pass", "fuse_sgd_op_pass", "fuse_momentum_op_pass" + "fuse_adam_op_pass", + "fuse_sgd_op_pass", + "fuse_momentum_op_pass", ] def _type(self): @@ -91,7 +88,6 @@ class FuseOptimizerPass(CPPPassWrapper): @register_pass("inplace_addto_op") class InplaceAddtoOpPass(CPPPassWrapper): - def __init__(self): super(InplaceAddtoOpPass, self).__init__() @@ -112,7 +108,6 @@ def _set_cinn_op_flag(flag_name, extra_ops): @register_pass("build_cinn") class BuildCINNPass(CPPPassWrapper): - def __init__(self): super(BuildCINNPass, self).__init__() self.set_attr("allow_ops", []) @@ -127,13 +122,15 @@ class BuildCINNPass(CPPPassWrapper): def _apply_single_impl(self, main_program, startup_program, context): - assert 'FLAGS_allow_cinn_ops' in core.globals( + assert ( + 'FLAGS_allow_cinn_ops' in core.globals() ), "PaddlePaddle is not compiled with CINN support" old_allow_ops = core.globals()['FLAGS_allow_cinn_ops'] old_deny_ops = core.globals()['FLAGS_deny_cinn_ops'] try: - _set_cinn_op_flag('FLAGS_allow_cinn_ops', - self.get_attr("allow_ops")) + _set_cinn_op_flag( + 'FLAGS_allow_cinn_ops', self.get_attr("allow_ops") + ) _set_cinn_op_flag('FLAGS_deny_cinn_ops', self.get_attr("deny_ops")) feed = self.get_attr('feed', []) @@ -142,18 +139,26 @@ class BuildCINNPass(CPPPassWrapper): if prune_program: tmp_main_program = Executor._prune_program( - main_program, feed, fetch_list, []) + main_program, feed, fetch_list, [] + ) tmp_main_program = Executor._add_fetch_ops( - tmp_main_program, fetch_list, 'fetch') + tmp_main_program, fetch_list, 'fetch' + ) else: tmp_main_program = Executor._add_fetch_ops( - main_program, fetch_list, 'fetch') - - _apply_cpp_pass(tmp_main_program, startup_program, self.cpp_name, - {}, self.cpp_attr_types) + main_program, fetch_list, 'fetch' + ) + + _apply_cpp_pass( + tmp_main_program, + startup_program, + self.cpp_name, + {}, + self.cpp_attr_types, + ) tmp_main_program = Executor._remove_fetch_ops(tmp_main_program) diff --git a/python/paddle/distributed/passes/fuse_all_reduce.py b/python/paddle/distributed/passes/fuse_all_reduce.py index 628caa0696a..dcd1976c89c 100644 --- a/python/paddle/distributed/passes/fuse_all_reduce.py +++ b/python/paddle/distributed/passes/fuse_all_reduce.py @@ -18,9 +18,9 @@ from .pass_base import PassBase, PassType, register_pass import numpy as np -def find_adjacent_match_sequences(iterable, - filter_func, - adjacent_filter_func=None): +def find_adjacent_match_sequences( + iterable, filter_func, adjacent_filter_func=None +): n = len(iterable) match_sequences = [] if adjacent_filter_func is None: @@ -30,8 +30,11 @@ def find_adjacent_match_sequences(iterable, while i < n and not filter_func(iterable[i]): i += 1 j = i + 1 - while j < n and filter_func(iterable[j]) and adjacent_filter_func( - iterable[i], iterable[j]): + while ( + j < n + and filter_func(iterable[j]) + and adjacent_filter_func(iterable[i], iterable[j]) + ): j += 1 if i < n and j <= n: match_sequences.append((i, j)) @@ -41,11 +44,13 @@ def find_adjacent_match_sequences(iterable, return match_sequences -def insert_fuse_all_reduce_ops(block, reversed_op_indices, input_var_names, - output_var_names, dtype, attrs): - fused_var = block.create_var(name=unique_name.generate( - "FusedOutput_{}".format(input_var_names[0])), - dtype=dtype) +def insert_fuse_all_reduce_ops( + block, reversed_op_indices, input_var_names, output_var_names, dtype, attrs +): + fused_var = block.create_var( + name=unique_name.generate("FusedOutput_{}".format(input_var_names[0])), + dtype=dtype, + ) # FIXME(zengjinle): here we assume that we use # c_sync_calc_stream/c_sync_comm_stream to do sync. @@ -56,16 +61,20 @@ def insert_fuse_all_reduce_ops(block, reversed_op_indices, input_var_names, for i, op_idx in enumerate(reversed_op_indices): prev_op_idx = op_idx - 1 - while prev_op_idx >= 0 and block.ops[ - prev_op_idx].type == "c_sync_calc_stream": + while ( + prev_op_idx >= 0 + and block.ops[prev_op_idx].type == "c_sync_calc_stream" + ): new_op_indices.append(prev_op_idx) prev_op_idx -= 1 if i > 0: next_op_idx = op_idx + 1 n = len(block.ops) - while next_op_idx < n and block.ops[ - next_op_idx].type == "c_sync_comm_stream": + while ( + next_op_idx < n + and block.ops[next_op_idx].type == "c_sync_comm_stream" + ): assert block.ops[next_op_idx].attr("ring_id") == ring_id new_op_indices.append(next_op_idx) @@ -102,21 +111,22 @@ def insert_fuse_all_reduce_ops(block, reversed_op_indices, input_var_names, } if not attrs["use_calc_stream"]: - block._insert_op_without_sync(insert_idx, - type="c_sync_calc_stream", - inputs={"X": fused_var}, - outputs={ - "Out": fused_var, - op_role_key: attrs[op_role_key] - }) + block._insert_op_without_sync( + insert_idx, + type="c_sync_calc_stream", + inputs={"X": fused_var}, + outputs={"Out": fused_var, op_role_key: attrs[op_role_key]}, + ) insert_idx += 1 # c_allreduce_sum should insert - block._insert_op_without_sync(insert_idx, - type="c_allreduce_sum", - inputs={"X": fused_var}, - outputs={"Out": fused_var}, - attrs=attrs) + block._insert_op_without_sync( + insert_idx, + type="c_allreduce_sum", + inputs={"X": fused_var}, + outputs={"Out": fused_var}, + attrs=attrs, + ) for op_idx in reversed_op_indices: block._remove_op(op_idx) @@ -185,9 +195,9 @@ def find_all_fuse_all_reduce_groups(block): return False return True - match_seqs = find_adjacent_match_sequences(collective_ops, - is_valid_allreduce_op, - is_same_adjacent_op) + match_seqs = find_adjacent_match_sequences( + collective_ops, is_valid_allreduce_op, is_same_adjacent_op + ) new_match_seqs = [] for i, j in match_seqs: new_match_seqs.append([collective_op_indices[k] for k in range(i, j)]) @@ -301,8 +311,13 @@ def insert_fuse_all_reduce_by_memory_size(block, groups, max_memory_size): if len(recorded_op_indices) > 1: attrs[op_role_var_key] = op_role_vars coalesce_op_kwargs = insert_fuse_all_reduce_ops( - block, recorded_op_indices, in_var_names, out_var_names, - dtype, attrs) + block, + recorded_op_indices, + in_var_names, + out_var_names, + dtype, + attrs, + ) coalesce_ops_kwargs.append(coalesce_op_kwargs) cur_mem_size = 0 @@ -321,8 +336,13 @@ def insert_fuse_all_reduce_by_memory_size(block, groups, max_memory_size): if len(recorded_op_indices) > 1: attrs[op_role_var_key] = op_role_vars coalesce_op_kwargs = insert_fuse_all_reduce_ops( - block, recorded_op_indices, in_var_names, out_var_names, dtype, - attrs) + block, + recorded_op_indices, + in_var_names, + out_var_names, + dtype, + attrs, + ) coalesce_ops_kwargs.append(coalesce_op_kwargs) block._sync_with_cpp() insert_coalesce_tensor_ops(block, coalesce_ops_kwargs) @@ -330,7 +350,6 @@ def insert_fuse_all_reduce_by_memory_size(block, groups, max_memory_size): @register_pass("fuse_all_reduce") class FuseAllReducePass(PassBase): - def __init__(self): super(FuseAllReducePass, self).__init__() self.set_attr("max_memory_size", -1) @@ -358,7 +377,9 @@ class FuseAllReducePass(PassBase): block = main_program.block(i) groups = find_all_fuse_all_reduce_groups(block) groups = split_fuse_all_reduce_groups_by_deps( - block, groups, op_deps[i]) - insert_fuse_all_reduce_by_memory_size(block, groups, - max_memory_size) + block, groups, op_deps[i] + ) + insert_fuse_all_reduce_by_memory_size( + block, groups, max_memory_size + ) main_program._sync_with_cpp() diff --git a/python/paddle/distributed/passes/pass_base.py b/python/paddle/distributed/passes/pass_base.py index e042ce4a15e..56cc262ca03 100644 --- a/python/paddle/distributed/passes/pass_base.py +++ b/python/paddle/distributed/passes/pass_base.py @@ -17,7 +17,6 @@ from paddle.fluid.framework import _apply_pass as _apply_cpp_pass class PassContext: - def __init__(self): self._applied_passes = [] self._attrs = {} @@ -84,7 +83,8 @@ class PassBase(ABC): def _check_conflict_including_common_rules(self, other_pass): return self._check_conflict(other_pass) and all( - [r(other_pass, self) for r in PassBase._COMMON_RULES]) + [r(other_pass, self) for r in PassBase._COMMON_RULES] + ) def apply(self, main_programs, startup_programs, context=None): if context is None: @@ -93,10 +93,12 @@ class PassBase(ABC): if not self._check_self(): return context - if not all([ + if not all( + [ self._check_conflict_including_common_rules(p) for p in context.passes - ]): + ] + ): return context assert isinstance(main_programs, list) @@ -107,8 +109,9 @@ class PassBase(ABC): return context def _apply_impl(self, main_programs, startup_programs, context): - for main_program, startup_program in zip(main_programs, - startup_programs): + for main_program, startup_program in zip( + main_programs, startup_programs + ): self._apply_single_impl(main_program, startup_program, context) @abstractmethod @@ -117,7 +120,6 @@ class PassBase(ABC): def register_pass(name): - def impl(cls): PassBase._register(name, cls) cls.name = name @@ -136,7 +138,6 @@ def new_pass(name, pass_attrs={}): class CPPPassWrapper(PassBase): - def __init__(self): super(CPPPassWrapper, self).__init__() @@ -155,21 +156,28 @@ class CPPPassWrapper(PassBase): return True def _apply_single_impl(self, main_program, startup_program, context): - _apply_cpp_pass(main_program, startup_program, self.cpp_name, - self._attrs, self.cpp_attr_types) + _apply_cpp_pass( + main_program, + startup_program, + self.cpp_name, + self._attrs, + self.cpp_attr_types, + ) def _fusion_opt_last_rule(pass_before, pass_after): - if pass_before._type( - ) == PassType.FUSION_OPT and pass_after._type() != PassType.FUSION_OPT: + if ( + pass_before._type() == PassType.FUSION_OPT + and pass_after._type() != PassType.FUSION_OPT + ): return False else: return True -def _make_rule_from_white_lists_dict(before_white_lists_dict, - after_white_lists_dict): - +def _make_rule_from_white_lists_dict( + before_white_lists_dict, after_white_lists_dict +): def collect_pass_names(white_lists_dict, result): for k, v in white_lists_dict.items(): result.add(k) @@ -196,7 +204,10 @@ def _make_rule_from_white_lists_dict(before_white_lists_dict, def rule(pass_before, pass_after): all_passes_after = compatible_pass_dict.get(pass_before.name) - if all_passes_after is None or pass_after.name not in compatible_pass_dict: + if ( + all_passes_after is None + or pass_after.name not in compatible_pass_dict + ): return True else: return pass_after.name in all_passes_after @@ -220,8 +231,9 @@ PassBase._AFTER_WHITE_LISTS_DICT = { PassBase._COMMON_RULES = [ _fusion_opt_last_rule, lambda pass_before, pass_after: type(pass_before) != type(pass_after), - _make_rule_from_white_lists_dict(PassBase._BEFORE_WHITE_LISTS_DICT, - PassBase._AFTER_WHITE_LISTS_DICT), + _make_rule_from_white_lists_dict( + PassBase._BEFORE_WHITE_LISTS_DICT, PassBase._AFTER_WHITE_LISTS_DICT + ), # Add more common rules here ] @@ -270,10 +282,12 @@ def _solve_pass_conflict(passes, context): old_passes = passes passes = [] for p in old_passes: - if all([ + if all( + [ p._check_conflict_including_common_rules(applied_p) for applied_p in context.passes - ]): + ] + ): passes.append(p) if not passes: @@ -287,14 +301,14 @@ def _solve_pass_conflict(passes, context): for i in range(n): for j in range(n): adjacent_matrix[i][j] = passes[ - j]._check_conflict_including_common_rules(passes[i]) + j + ]._check_conflict_including_common_rules(passes[i]) longest_path = _find_longest_path(adjacent_matrix) return [passes[idx] for idx in longest_path] class PassManager: - def __init__(self, passes, context=None, auto_solve_conflict=True): if context is None: context = PassContext() diff --git a/python/paddle/distributed/passes/pass_utils.py b/python/paddle/distributed/passes/pass_utils.py index e779048c56b..8e608bd9ed6 100644 --- a/python/paddle/distributed/passes/pass_utils.py +++ b/python/paddle/distributed/passes/pass_utils.py @@ -108,8 +108,9 @@ def split_program(program, op_indices): op_indices.append(op_num) for idx in range(len(op_indices) - 1): - assert op_indices[idx] < op_indices[ - idx + 1], "op_indices must be strictly sorted" + assert ( + op_indices[idx] < op_indices[idx + 1] + ), "op_indices must be strictly sorted" splitted_programs = [] for idx in range(len(op_indices) - 1): diff --git a/python/paddle/distributed/passes/ps_server_pass.py b/python/paddle/distributed/passes/ps_server_pass.py index 27c10fd1f5b..37e5622ea8e 100755 --- a/python/paddle/distributed/passes/ps_server_pass.py +++ b/python/paddle/distributed/passes/ps_server_pass.py @@ -15,16 +15,30 @@ import logging import paddle.fluid as fluid -from ..ps.utils.public import get_optimize_ops, get_ps_endpoint, get_role_id, get_trainers +from ..ps.utils.public import ( + get_optimize_ops, + get_ps_endpoint, + get_role_id, + get_trainers, +) from .pass_base import PassBase, register_pass from paddle.optimizer.lr import LRScheduler -from paddle.optimizer.lr import ExponentialDecay, InverseTimeDecay, NaturalExpDecay, NoamDecay -from paddle.fluid.layers.learning_rate_scheduler import exponential_decay, inverse_time_decay, natural_exp_decay, noam_decay +from paddle.optimizer.lr import ( + ExponentialDecay, + InverseTimeDecay, + NaturalExpDecay, + NoamDecay, +) +from paddle.fluid.layers.learning_rate_scheduler import ( + exponential_decay, + inverse_time_decay, + natural_exp_decay, + noam_decay, +) @register_pass("add_lr_decay_table_pass") class AddLrDecayTablePass(PassBase): - def __init__(self): super(AddLrDecayTablePass, self).__init__() @@ -34,13 +48,15 @@ class AddLrDecayTablePass(PassBase): def _check_conflict(self, other_pass): return True - def _add_tensor_table(self, - attrs, - feed_var_name, - fetch_var_name="", - startup_program=None, - main_program=None, - tensor_table_class=""): + def _add_tensor_table( + self, + attrs, + feed_var_name, + fetch_var_name="", + startup_program=None, + main_program=None, + tensor_table_class="", + ): tensor_table_dict = {} tensor_table_dict[feed_var_name] = {} tensor_table_dict[feed_var_name]["feed_var_name"] = feed_var_name @@ -48,13 +64,16 @@ class AddLrDecayTablePass(PassBase): tensor_table_dict[feed_var_name]["startup_program"] = startup_program tensor_table_dict[feed_var_name]["main_program"] = main_program tensor_table_dict[feed_var_name][ - "tensor_table_class"] = tensor_table_class + "tensor_table_class" + ] = tensor_table_class attrs['tensor_table'] = tensor_table_dict def _get_lr_sheduler_program(self, lr_sheduler, lr_decay_steps): schedler_decay = [ - 'NoamDecay', 'NaturalExpDecay', 'InverseTimeDecay', - 'ExponentialDecay' + 'NoamDecay', + 'NaturalExpDecay', + 'InverseTimeDecay', + 'ExponentialDecay', ] decay_main_program = fluid.framework.Program() @@ -63,48 +82,59 @@ class AddLrDecayTablePass(PassBase): if isinstance(lr_sheduler, ExponentialDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = exponential_decay(1.0, lr_decay_steps, lr_sheduler.gamma, - True) + lr = exponential_decay( + 1.0, lr_decay_steps, lr_sheduler.gamma, True + ) lr_name = lr.name logging.warn( "ExponentialDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n" "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) elif isinstance(lr_sheduler, NoamDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = noam_decay(lr_sheduler.d_model, lr_sheduler.warmup_steps, - 1.0) + lr = noam_decay( + lr_sheduler.d_model, lr_sheduler.warmup_steps, 1.0 + ) lr_name = lr.name - logging.warn("NoamDecay is set, warmup steps is [ %d ]" % - lr_sheduler.warmup_steps) + logging.warn( + "NoamDecay is set, warmup steps is [ %d ]" + % lr_sheduler.warmup_steps + ) elif isinstance(lr_sheduler, NaturalExpDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = natural_exp_decay(1.0, lr_decay_steps, lr_sheduler.gamma, - True) + lr = natural_exp_decay( + 1.0, lr_decay_steps, lr_sheduler.gamma, True + ) lr_name = lr.name logging.warn( "NaturalExpDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n" "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) elif isinstance(lr_sheduler, InverseTimeDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = inverse_time_decay(1.0, lr_decay_steps, lr_sheduler.gamma, - True) + lr = inverse_time_decay( + 1.0, lr_decay_steps, lr_sheduler.gamma, True + ) lr_name = lr.name logging.warn( "InverseTimeDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n" "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) else: raise ValueError( - "Not supported current LearningRate strategy, please use follow decay strategy: {}" - .format(schedler_decay)) + "Not supported current LearningRate strategy, please use follow decay strategy: {}".format( + schedler_decay + ) + ) return decay_main_program, decay_startup_program, lr_name @@ -113,21 +143,31 @@ class AddLrDecayTablePass(PassBase): if hasattr(attrs['origin_main_program'], 'lr_sheduler') == False: return - assert isinstance(attrs['origin_main_program'].lr_sheduler, - LRScheduler), "must be LRScheduler" + assert isinstance( + attrs['origin_main_program'].lr_sheduler, LRScheduler + ), "must be LRScheduler" ops = get_optimize_ops(attrs['origin_main_program']) - lr_decay_main_program, lr_decay_startup_program, lr_name = self._get_lr_sheduler_program( - attrs['origin_main_program'].lr_sheduler, attrs['lr_decay_steps']) - self._add_tensor_table(attrs, "@LR_DECAY_COUNTER@", lr_name, - lr_decay_startup_program, lr_decay_main_program, - "GlobalStepTable") + ( + lr_decay_main_program, + lr_decay_startup_program, + lr_name, + ) = self._get_lr_sheduler_program( + attrs['origin_main_program'].lr_sheduler, attrs['lr_decay_steps'] + ) + self._add_tensor_table( + attrs, + "@LR_DECAY_COUNTER@", + lr_name, + lr_decay_startup_program, + lr_decay_main_program, + "GlobalStepTable", + ) return @register_pass("add_listen_and_serv_pass") class AddListenAndServPass(PassBase): - def __init__(self): super(AddListenAndServPass, self).__init__() @@ -145,7 +185,6 @@ class AddListenAndServPass(PassBase): "lr_decay_block_id": None, "dense_optimize_blocks": None, "sparse_optimize_blocks": None, - # runtime attribute "endpoint": get_ps_endpoint(attrs['role_maker']), "pserver_id": get_role_id(attrs['role_maker']), @@ -153,17 +192,15 @@ class AddListenAndServPass(PassBase): "distributed_mode": attrs['ps_mode'], "rpc_get_thread_num": -1, "rpc_send_thread_num": -1, - "rpc_prefetch_thread_num": -1 + "rpc_prefetch_thread_num": -1, } - main_program.global_block().append_op(type="listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=opt) + main_program.global_block().append_op( + type="listen_and_serv", inputs={'X': []}, outputs={}, attrs=opt + ) @register_pass("add_rpc_global_flags_pass") class AddRpcGlobalFlagsPass(PassBase): - def __init__(self): super(AddRpcGlobalFlagsPass, self).__init__() @@ -179,7 +216,6 @@ class AddRpcGlobalFlagsPass(PassBase): @register_pass("add_optimizer_pass") class AddOptimizerPass(PassBase): - def __init__(self): super(AddOptimizerPass, self).__init__() @@ -195,7 +231,6 @@ class AddOptimizerPass(PassBase): @register_pass("add_geo_optimizer_pass") class AddGeoOptimizerPass(PassBase): - def __init__(self): super(AddGeoOptimizerPass, self).__init__() @@ -211,7 +246,6 @@ class AddGeoOptimizerPass(PassBase): @register_pass("build_pserver_startup_program_pass") class BuildPserverStartupProgramPass(PassBase): - def __init__(self): super(BuildPserverStartupProgramPass, self).__init__() @@ -227,7 +261,6 @@ class BuildPserverStartupProgramPass(PassBase): @register_pass("delete_unused_in_startup_pass") class DeleteUnusedInStartupPass(PassBase): - def __init__(self): super(DeleteUnusedInStartupPass, self).__init__() diff --git a/python/paddle/distributed/passes/ps_trainer_pass.py b/python/paddle/distributed/passes/ps_trainer_pass.py index 1b5eb1e442f..f99d9f316d4 100755 --- a/python/paddle/distributed/passes/ps_trainer_pass.py +++ b/python/paddle/distributed/passes/ps_trainer_pass.py @@ -25,7 +25,6 @@ from paddle.fluid.framework import Program, Parameter @register_pass("append_send_ops_pass") class AppendSendOpsPass(PassBase): # 该 pass 被多种模式复用 - def __init__(self): super(AppendSendOpsPass, self).__init__() @@ -35,8 +34,9 @@ class AppendSendOpsPass(PassBase): # 该 pass 被多种模式复用 def _check_conflict(self, other_pass): return True - def _append_send_op(self, program, union_vars, queue, is_sparse, table_id, - ps_mode): + def _append_send_op( + self, program, union_vars, queue, is_sparse, table_id, ps_mode + ): if queue == STEP_COUNTER: send_input_vars = [] else: @@ -48,43 +48,42 @@ class AppendSendOpsPass(PassBase): # 该 pass 被多种模式复用 dummy_output = [] if ps_mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]: dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) - program.global_block().append_op(type="send", - inputs={"X": send_input_vars}, - outputs={"Out": dummy_output}, - attrs={ - "send_varnames": [queue], - "is_sparse": - is_sparse, - "table_id": - table_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + name=framework.generate_control_dev_var_name() + ) + program.global_block().append_op( + type="send", + inputs={"X": send_input_vars}, + outputs={"Out": dummy_output}, + attrs={ + "send_varnames": [queue], + "is_sparse": is_sparse, + "table_id": table_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return dummy_output def _append_barrier_op(self, program, dummys, trainer_id): - program.global_block().append_op(type="send_barrier", - inputs={"X": dummys}, - outputs={"Out": []}, - attrs={ - "trainer_id": - trainer_id, - "half_async": - True, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + program.global_block().append_op( + type="send_barrier", + inputs={"X": dummys}, + outputs={"Out": []}, + attrs={ + "trainer_id": trainer_id, + "half_async": True, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs ps_mode = attrs['ps_mode'] - #if ps_mode == DistributedMode.GEO: + # if ps_mode == DistributedMode.GEO: # send_ctx = get_geo_trainer_send_context(attrs) # geo 模式, 没必要 send_ctx = get_the_one_send_context( - attrs, - split_dense_table=attrs['is_heter_ps_mode']) # async、sync 等各种模式 + attrs, split_dense_table=attrs['is_heter_ps_mode'] + ) # async、sync 等各种模式 dummys = [] for merged_name, send in send_ctx.items(): # embedding_0.w_0@GRAD @@ -99,9 +98,15 @@ class AppendSendOpsPass(PassBase): # 该 pass 被多种模式复用 is_sparse = 1 if send.is_sparse() else 0 is_sparse = 2 if send.is_distributed() else is_sparse dummys.append( - self._append_send_op(main_program, send.origin_varnames(), - merged_name, is_sparse, send.table_id(), - ps_mode)) + self._append_send_op( + main_program, + send.origin_varnames(), + merged_name, + is_sparse, + send.table_id(), + ps_mode, + ) + ) if ps_mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]: trainer_id = get_role_id(attrs['role_maker']) self._append_barrier_op(main_program, dummys, trainer_id) @@ -109,7 +114,6 @@ class AppendSendOpsPass(PassBase): # 该 pass 被多种模式复用 @register_pass("distributed_ops_pass") class DistributedOpsPass(PassBase): - def __init__(self): super(DistributedOpsPass, self).__init__() self.w_2_table_id = {} @@ -138,8 +142,10 @@ class DistributedOpsPass(PassBase): if len(entry) == 3 and entry[0] == 'show_click_entry': show_var_name = entry[1] click_var_name = entry[2] - if show_var_name in _program.global_block( - ).vars and click_var_name in _program.global_block().vars: + if ( + show_var_name in _program.global_block().vars + and click_var_name in _program.global_block().vars + ): show = _program.global_block().vars[show_var_name] clk = _program.global_block().vars[click_var_name] use_entry = True @@ -154,31 +160,37 @@ class DistributedOpsPass(PassBase): name="show", dtype=core.VarDesc.VarType.FP32, persistable=False, - stop_gradient=True) - _program.global_block()._insert_op(index=0, - type='fill_constant', - inputs={}, - outputs={'Out': show}, - attrs={ - 'shape': [1], - 'dtype': show.dtype, - 'value': 1, - }) + stop_gradient=True, + ) + _program.global_block()._insert_op( + index=0, + type='fill_constant', + inputs={}, + outputs={'Out': show}, + attrs={ + 'shape': [1], + 'dtype': show.dtype, + 'value': 1, + }, + ) clk = _program.global_block().create_var( name="clk", dtype=core.VarDesc.VarType.FP32, persistable=False, - stop_gradient=True) - _program.global_block()._insert_op(index=0, - type='fill_constant', - inputs={}, - outputs={'Out': clk}, - attrs={ - 'shape': [1], - 'dtype': clk.dtype, - 'value': 0, - }) + stop_gradient=True, + ) + _program.global_block()._insert_op( + index=0, + type='fill_constant', + inputs={}, + outputs={'Out': clk}, + attrs={ + 'shape': [1], + 'dtype': clk.dtype, + 'value': 0, + }, + ) for param, ops in push_sparse_ops.items(): all_ops = _program.global_block().ops @@ -203,27 +215,27 @@ class DistributedOpsPass(PassBase): for idx in op_idxs[::-1]: _program.global_block()._remove_op(idx) - _program.global_block().append_op(type="distributed_push_sparse", - inputs={ - "Ids": inputs, - 'W': w, - "Outputs": outputs, - "Shows": show, - "Clicks": clk, - }, - outputs={"Outputs": outputs}, - attrs={ - "is_distributed": - is_distributed, - "padding_idx": padding_idx, - "table_id": table_id, - "size": self.emb_size[param], - "use_cvm_op": use_cvm_op, - "slots": slots - }) + _program.global_block().append_op( + type="distributed_push_sparse", + inputs={ + "Ids": inputs, + 'W': w, + "Outputs": outputs, + "Shows": show, + "Clicks": clk, + }, + outputs={"Outputs": outputs}, + attrs={ + "is_distributed": is_distributed, + "padding_idx": padding_idx, + "table_id": table_id, + "size": self.emb_size[param], + "use_cvm_op": use_cvm_op, + "slots": slots, + }, + ) def _pull_sparse_fuse(self, _program, pull_sparse_ops, attrs, send_ctx): - def dag_check_up_and_reorder(program, inputs, outputs): global_block = program.global_block() min_output_index = len(global_block.ops) @@ -348,7 +360,8 @@ class DistributedOpsPass(PassBase): if table_id == -1: raise ValueError( - "can not find suitable sparse table, please check") + "can not find suitable sparse table, please check" + ) self.w_2_table_id[param] = table_id padding_idx = ops[0].attr("padding_idx") @@ -379,8 +392,9 @@ class DistributedOpsPass(PassBase): ins = op.input(op.input_names[i]) for out_id, out_var in enumerate(outputs): if out_var.name in ins: - outputs_idxs[out_id] = min(idx, - outputs_idxs[out_id]) + outputs_idxs[out_id] = min( + idx, outputs_idxs[out_id] + ) if attrs['use_ps_gpu']: gpups_inputs_idxs.extend(inputs_idxs) @@ -388,8 +402,9 @@ class DistributedOpsPass(PassBase): gpups_inputs.extend(inputs) gpups_outputs.extend(outputs) gpups_w_size.extend([w.shape[1]] * len(inputs)) - gpups_min_distributed_idx = min(min(op_idxs), - gpups_min_distributed_idx) + gpups_min_distributed_idx = min( + min(op_idxs), gpups_min_distributed_idx + ) continue if min(outputs_idxs) - max(inputs_idxs) >= 1: @@ -401,18 +416,16 @@ class DistributedOpsPass(PassBase): _program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Outputs": outputs}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "lookup_table_version": op_type, - "op_device": op_device - }) + "op_device": op_device, + }, + ) else: for i in range(len(inputs_idxs)): distributed_idx = op_idxs[i] @@ -420,34 +433,34 @@ class DistributedOpsPass(PassBase): _program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": [inputs[i]], - 'W': w - }, + inputs={"Ids": [inputs[i]], 'W': w}, outputs={"Outputs": [outputs[i]]}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "lookup_table_version": op_type, - "op_device": op_device - }) + "op_device": op_device, + }, + ) if attrs['use_ps_gpu'] and len(gpups_inputs) > 0: if max(gpups_inputs_idxs) > 0: raise ValueError("There can't be ops before embedding in gpups") - _program.global_block()._insert_op(index=gpups_min_distributed_idx, - type="pull_gpups_sparse", - inputs={ - "Ids": gpups_inputs, - }, - outputs={"Out": gpups_outputs}, - attrs={ - "size": gpups_w_size, - "is_distributed": True, - "is_sparse": True - }) + _program.global_block()._insert_op( + index=gpups_min_distributed_idx, + type="pull_gpups_sparse", + inputs={ + "Ids": gpups_inputs, + }, + outputs={"Out": gpups_outputs}, + attrs={ + "size": gpups_w_size, + "is_distributed": True, + "is_sparse": True, + }, + ) PSGPU = paddle.fluid.core.PSGPU() try: gpu_slot = [int(var.name) for var in gpups_inputs] @@ -466,8 +479,10 @@ class DistributedOpsPass(PassBase): ops = {} use_cvm_op = False for op in _program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] if attrs['is_heter_ps_mode'] and not attrs['is_fl_ps_mode']: # TODO: trick for matchnet, need to modify for heter_ps @@ -486,8 +501,10 @@ class DistributedOpsPass(PassBase): for op in _program.global_block().ops: if op.type in SPARSE_GRAD_OP_TYPE_DICT.keys(): param_name = op.input(SPARSE_GRAD_OP_TYPE_DICT[op.type])[0] - if param_name in pull_sparse_ids and op.input( - "Ids")[0] in pull_sparse_ids[param_name]: + if ( + param_name in pull_sparse_ids + and op.input("Ids")[0] in pull_sparse_ids[param_name] + ): ops = push_sparse_ops.get(param_name, []) ops.append(op) push_sparse_ops[param_name] = ops @@ -496,19 +513,25 @@ class DistributedOpsPass(PassBase): def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs - pull_sparse_ops, push_sparse_ops, use_cvm_op = self._get_pull_sparse_ops( - main_program, attrs) - print("is_heter_ps_mode in distributed_ops_pass {}?".format( - attrs['is_heter_ps_mode'])) + ( + pull_sparse_ops, + push_sparse_ops, + use_cvm_op, + ) = self._get_pull_sparse_ops(main_program, attrs) + print( + "is_heter_ps_mode in distributed_ops_pass {}?".format( + attrs['is_heter_ps_mode'] + ) + ) send_ctx = get_the_one_send_context( - attrs, split_dense_table=attrs['is_heter_ps_mode']) + attrs, split_dense_table=attrs['is_heter_ps_mode'] + ) self._pull_sparse_fuse(main_program, pull_sparse_ops, attrs, send_ctx) self._push_sparse_fuse(main_program, push_sparse_ops, attrs, use_cvm_op) @register_pass("delete_optimizer_pass") class DeleteOptimizesPass(PassBase): - def __init__(self): super(DeleteOptimizesPass, self).__init__() @@ -518,8 +541,9 @@ class DeleteOptimizesPass(PassBase): def _check_conflict(self, other_pass): return True - def _delete_optimizer_op_and_vars(self, _program, remote_optimize_ops, - local_optimize_ops): + def _delete_optimizer_op_and_vars( + self, _program, remote_optimize_ops, local_optimize_ops + ): local_optimize_vars = [] remote_optimize_vars = [] remote_optimize_op_role_vars = [] @@ -533,14 +557,18 @@ class DeleteOptimizesPass(PassBase): remote_optimize_op_role_vars.extend(op.attr("op_role_var")) remote_optimize_vars = list( - set(remote_optimize_vars - )) # param + grad + optimizer_state + learning_rate + set(remote_optimize_vars) + ) # param + grad + optimizer_state + learning_rate remote_optimize_op_role_vars = list( - set(remote_optimize_op_role_vars)) # param + grad + set(remote_optimize_op_role_vars) + ) # param + grad print( - "remote_optimize_vars: {}, remote_optimize_op_role_vars: {}, local_optimize_vars: {}" - .format(remote_optimize_vars, remote_optimize_op_role_vars, - local_optimize_vars)) + "remote_optimize_vars: {}, remote_optimize_op_role_vars: {}, local_optimize_vars: {}".format( + remote_optimize_vars, + remote_optimize_op_role_vars, + local_optimize_vars, + ) + ) for var in remote_optimize_vars: if var in local_optimize_vars: continue @@ -555,26 +583,32 @@ class DeleteOptimizesPass(PassBase): def _add_lr_var(self, main_program, attrs): # Todo: hard code for pe - lr_var = attrs['origin_main_program'].global_block( - ).vars["learning_rate_0"] - main_program.global_block().create_var(name=lr_var.name, - shape=lr_var.shape, - dtype=lr_var.dtype, - type=lr_var.type, - lod_level=lr_var.lod_level, - persistable=True) + lr_var = ( + attrs['origin_main_program'].global_block().vars["learning_rate_0"] + ) + main_program.global_block().create_var( + name=lr_var.name, + shape=lr_var.shape, + dtype=lr_var.dtype, + type=lr_var.type, + lod_level=lr_var.lod_level, + persistable=True, + ) def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs all_optimize_ops = get_optimize_ops(main_program) - remote_optimize_ops = get_optimize_ops(main_program, - attrs['remote_sparse']) + remote_optimize_ops = get_optimize_ops( + main_program, attrs['remote_sparse'] + ) lr_ops = get_lr_ops(main_program) remote_optimize_ops.extend(lr_ops) local_optimize_ops = list( - set(all_optimize_ops) - set(remote_optimize_ops)) - self._delete_optimizer_op_and_vars(main_program, remote_optimize_ops, - local_optimize_ops) + set(all_optimize_ops) - set(remote_optimize_ops) + ) + self._delete_optimizer_op_and_vars( + main_program, remote_optimize_ops, local_optimize_ops + ) if hasattr(attrs['origin_main_program'], 'lr_sheduler'): self._add_lr_var(main_program, attrs) @@ -582,7 +616,6 @@ class DeleteOptimizesPass(PassBase): @register_pass("delete_extra_optimizer_pass") class DeleteExtraOptimizerPass(PassBase): - def __init__(self): super(DeleteExtraOptimizerPass, self).__init__() @@ -598,10 +631,12 @@ class DeleteExtraOptimizerPass(PassBase): remote_optimize_op_role_vars = [] optimize_need_delete_vars = [] all_optimize_ops = get_optimize_ops(main_program) - remote_optimize_ops = get_optimize_ops(main_program, - attrs['remote_sparse']) + remote_optimize_ops = get_optimize_ops( + main_program, attrs['remote_sparse'] + ) local_optimize_ops = list( - set(all_optimize_ops) - set(remote_optimize_ops)) + set(all_optimize_ops) - set(remote_optimize_ops) + ) local_optimize_vars = [] for op in local_optimize_ops: @@ -638,7 +673,6 @@ class DeleteExtraOptimizerPass(PassBase): @register_pass("fake_init_ops_pass") class FakeInitOpsPass(PassBase): - def __init__(self): super(FakeInitOpsPass, self).__init__() @@ -649,19 +683,24 @@ class FakeInitOpsPass(PassBase): return True def _get_sparse_table_names(self, attrs): - dist_varnames = get_sparse_tablenames(attrs['origin_main_programs'], - True) - sparse_varnames = get_sparse_tablenames(attrs['origin_main_programs'], - False) + dist_varnames = get_sparse_tablenames( + attrs['origin_main_programs'], True + ) + sparse_varnames = get_sparse_tablenames( + attrs['origin_main_programs'], False + ) return list(set(dist_varnames + sparse_varnames)) - def _fake_init_sparsetable(self, startup_program, sparse_table_names, - attrs): + def _fake_init_sparsetable( + self, startup_program, sparse_table_names, attrs + ): # delete table init op for table_name in sparse_table_names: table_var = startup_program.global_block().vars[table_name] - if str(table_var).split( - ":")[0].strip().split()[-1] in attrs['local_sparse']: + if ( + str(table_var).split(":")[0].strip().split()[-1] + in attrs['local_sparse'] + ): continue table_param_init_op = [] for op in startup_program.global_block().ops: @@ -669,14 +708,16 @@ class FakeInitOpsPass(PassBase): table_param_init_op.append(op) init_op_num = len(table_param_init_op) if init_op_num != 1: - raise ValueError("table init op num should be 1, now is " + - str(init_op_num)) + raise ValueError( + "table init op num should be 1, now is " + str(init_op_num) + ) table_init_op = table_param_init_op[0] startup_program.global_block().append_op( type="fake_init", inputs={}, outputs={"Out": table_var}, - attrs={"shape": table_init_op.attr('shape')}) + attrs={"shape": table_init_op.attr('shape')}, + ) delete_ops(startup_program.global_block(), table_param_init_op) def _apply_single_impl(self, main_program, startup_program, pass_ctx): @@ -687,7 +728,6 @@ class FakeInitOpsPass(PassBase): @register_pass("ps_gpu_pass") class PsGpuPass(PassBase): - def __init__(self): super(PsGpuPass, self).__init__() @@ -706,14 +746,17 @@ class PsGpuPass(PassBase): if op.type != "pull_box_sparse" and op.type != "pull_gpups_sparse": continue grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, set(), []) + op.desc, set(), [] + ) for op_desc in grad_op_desc: new_op_desc = program.global_block().desc._insert_op( - insert_index + 1) + insert_index + 1 + ) new_op_desc.copy_from(op_desc) new_op_desc._set_attr(op_role_attr_name, backward) new_op = paddle.fluid.framework.Operator( - program.global_block(), new_op_desc) + program.global_block(), new_op_desc + ) program.global_block().ops.insert(insert_index + 1, new_op) program.global_block()._sync_with_cpp() @@ -787,7 +830,6 @@ class PsGpuPass(PassBase): @register_pass("ps_transpile_pass") class PsTranspilePass(PassBase): - def __init__(self): super(PsTranspilePass, self).__init__() @@ -801,17 +843,18 @@ class PsTranspilePass(PassBase): attrs = pass_ctx._attrs t = SingleProcessMultiThread() env = get_dist_env() - t.transpile(startup_program=startup_program, - main_program=main_program, - rank=env["trainer_id"], - endpoints=env["trainer_endpoints"], - current_endpoint=env['current_endpoint'], - wait_port=False) + t.transpile( + startup_program=startup_program, + main_program=main_program, + rank=env["trainer_id"], + endpoints=env["trainer_endpoints"], + current_endpoint=env['current_endpoint'], + wait_port=False, + ) @register_pass("split_heter_worker_ops_pass") class SplitHeterWorkerOpsPass(PassBase): - def __init__(self): super(SplitHeterWorkerOpsPass, self).__init__() @@ -821,9 +864,15 @@ class SplitHeterWorkerOpsPass(PassBase): def _check_conflict(self, other_pass): return True - def _create_heter_program(self, program, attrs, heter_program, - program_block_ops_list, heter_ops, - block_var_detail): + def _create_heter_program( + self, + program, + attrs, + heter_program, + program_block_ops_list, + heter_ops, + block_var_detail, + ): # This function mainly includes the following contents: # 1. For every heter block: # a) copy heter device op from origin program @@ -849,10 +898,12 @@ class SplitHeterWorkerOpsPass(PassBase): current_device = role_maker._heter_device_type().lower() stage_id = int(role_maker._get_stage_id()) - heter_block_ops_forward = program_block_ops_list[stage_id - - 1]["forward"] - heter_block_ops_backward = program_block_ops_list[stage_id - - 1]["backward"] + heter_block_ops_forward = program_block_ops_list[stage_id - 1][ + "forward" + ] + heter_block_ops_backward = program_block_ops_list[stage_id - 1][ + "backward" + ] heter_block = heter_program._create_block(pre_block_idx) optimizer_block.append(heter_block) @@ -874,59 +925,82 @@ class SplitHeterWorkerOpsPass(PassBase): for _, op in enumerate(heter_block_ops_backward): block_append_op(heter_program, program, heter_block_bp, op) - bp_entrance_vars = block_var_detail[stage_id - - 1]["backward"]["entrance"] - add_vars_by_var_list(bp_entrance_vars, program, heter_program, - heter_block_bp) + bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][ + "entrance" + ] + add_vars_by_var_list( + bp_entrance_vars, program, heter_program, heter_block_bp + ) bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"] - add_vars_by_var_list(bp_exit_vars, program, heter_program, - heter_block_bp) - backward_comm_info = get_communicate_var_info(program, - stage_id, - bp_entrance_vars, - type="backward") + add_vars_by_var_list( + bp_exit_vars, program, heter_program, heter_block_bp + ) + backward_comm_info = get_communicate_var_info( + program, stage_id, bp_entrance_vars, type="backward" + ) - grad_to_block_id.append(backward_comm_info["block_input_var_name"] + - ":" + str(heter_block_bp.idx)) + grad_to_block_id.append( + backward_comm_info["block_input_var_name"] + + ":" + + str(heter_block_bp.idx) + ) else: for _, op in enumerate(heter_block_ops_backward): block_append_op(heter_program, program, heter_block, op) - bp_entrance_vars = block_var_detail[stage_id - - 1]["backward"]["entrance"] - add_vars_by_var_list(bp_entrance_vars, program, heter_program, - heter_block) + bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][ + "entrance" + ] + add_vars_by_var_list( + bp_entrance_vars, program, heter_program, heter_block + ) bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"] - add_vars_by_var_list(bp_exit_vars, program, heter_program, - heter_block) + add_vars_by_var_list( + bp_exit_vars, program, heter_program, heter_block + ) heter_block_bp = heter_block - forward_comm_info = get_communicate_var_info(program, - stage_id, - entrance_vars, - type="forward") + forward_comm_info = get_communicate_var_info( + program, stage_id, entrance_vars, type="forward" + ) - grad_to_block_id.append(forward_comm_info["block_input_var_name"] + - ":" + str(heter_block.idx)) + grad_to_block_id.append( + forward_comm_info["block_input_var_name"] + + ":" + + str(heter_block.idx) + ) first_op_index_bp = len(heter_block_bp.ops) if stage_id <= len(block_var_detail) - 1: - static_var = insert_communicate_op(program, role_maker, heter_block, - stage_id, first_op_index_fp, - block_var_detail, current_device) - static_var_bp = insert_communicate_op(program, role_maker, - heter_block_bp, stage_id, - first_op_index_bp, - block_var_detail, current_device, - False) + static_var = insert_communicate_op( + program, + role_maker, + heter_block, + stage_id, + first_op_index_fp, + block_var_detail, + current_device, + ) + static_var_bp = insert_communicate_op( + program, + role_maker, + heter_block_bp, + stage_id, + first_op_index_bp, + block_var_detail, + current_device, + False, + ) # add send op send_grad_var_list = add_send_op( - program, heter_block_bp, - block_var_detail[stage_id - 1]["backward"]["persistables"]) + program, + heter_block_bp, + block_var_detail[stage_id - 1]["backward"]["persistables"], + ) # add step conter send_input_vars = [] @@ -941,13 +1015,15 @@ class SplitHeterWorkerOpsPass(PassBase): "pserver_id": get_role_id(role_maker), "distributed_mode": attrs['ps_mode'], "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } # append the listen_and_serv op - heter_program.global_block().append_op(type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + heter_program.global_block().append_op( + type="heter_listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) # TODO check heter program def _apply_single_impl(self, main_program, startup_program, pass_ctx): @@ -960,7 +1036,8 @@ class SplitHeterWorkerOpsPass(PassBase): attrs = pass_ctx._attrs default_deveice = "cpu" program, heter_ops, _, program_block_ops = find_heter_ops( - main_program, default_deveice) + main_program, default_deveice + ) if len(heter_ops) == 0: warnings.warn( "Currently running in Heter Parameter Server mode, but no OP running on heterogeneous devices, Please check your code." @@ -969,18 +1046,23 @@ class SplitHeterWorkerOpsPass(PassBase): return program_block_ops = union_forward_gradient_op(program_block_ops) - block_vars_detail = find_block_joints(program, program_block_ops, - heter_ops) + block_vars_detail = find_block_joints( + program, program_block_ops, heter_ops + ) heter_program = framework.Program() - self._create_heter_program(program, attrs, heter_program, - program_block_ops, heter_ops, - block_vars_detail) + self._create_heter_program( + program, + attrs, + heter_program, + program_block_ops, + heter_ops, + block_vars_detail, + ) main_program = heter_program @register_pass("split_trainer_ops_pass") class SplitTrainerOpsPass(PassBase): - def __init__(self): super(SplitTrainerOpsPass, self).__init__() @@ -990,8 +1072,9 @@ class SplitTrainerOpsPass(PassBase): def _check_conflict(self, other_pass): return True - def _replace_ops_by_communicate_op(self, program, attrs, heter_block_index, - ops_list, block_var_detail): + def _replace_ops_by_communicate_op( + self, program, attrs, heter_block_index, ops_list, block_var_detail + ): all_op = program.global_block().ops start_op = ops_list[0] first_op_idx = -1 @@ -1008,10 +1091,12 @@ class SplitTrainerOpsPass(PassBase): next_heter_worker_endpoints = get_next_stage_trainers(role_maker) entrance_var = block_var_detail[heter_block_index]["forward"][ - "entrance"] + "entrance" + ] - comm_info = get_communicate_var_info(program, heter_block_index + 1, - entrance_var) + comm_info = get_communicate_var_info( + program, heter_block_index + 1, entrance_var + ) program.global_block()._insert_op( index=first_op_idx, type="send_and_recv", @@ -1025,8 +1110,9 @@ class SplitTrainerOpsPass(PassBase): "next_endpoints": next_heter_worker_endpoints, "previous_endpoints": [], "trainer_id": get_role_id(role_maker), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return entrance_var @@ -1051,18 +1137,22 @@ class SplitTrainerOpsPass(PassBase): del attrs['merged_sparse_pairs'][index] return - def _remove_trainer_send_op(self, program, attrs, heter_block_index, - block_var_detail): + def _remove_trainer_send_op( + self, program, attrs, heter_block_index, block_var_detail + ): # if trainer do FF->BP->SEND, it has follow vars: var, var@GRAD # if trainer only do SEND, it has one var: var@GRAD # Delete Send op ,if trainer doesn't has pair var (var<->var@GRAD) - persistables = block_var_detail[heter_block_index]["forward"]["persistables"] + \ - block_var_detail[heter_block_index]["backward"]["persistables"] + persistables = ( + block_var_detail[heter_block_index]["forward"]["persistables"] + + block_var_detail[heter_block_index]["backward"]["persistables"] + ) need_remove_send_op = [] need_remove_grad_var = [] for op in find_send_op(program): - input_list, _ = find_op_input_output(program, - program.global_block(), op) + input_list, _ = find_op_input_output( + program, program.global_block(), op + ) for var_name in input_list: origin_var_name = var_name.split("@GRAD")[0] if origin_var_name in persistables: @@ -1073,8 +1163,14 @@ class SplitTrainerOpsPass(PassBase): for grad_var_name in need_remove_grad_var: self._remove_var_pair_by_grad(grad_var_name, attrs) - def _create_trainer_program(self, program, origin_program, attrs, - program_block_ops_list, block_var_detail): + def _create_trainer_program( + self, + program, + origin_program, + attrs, + program_block_ops_list, + block_var_detail, + ): # This function mainly includes the following contents: # 1. For every heter block in origin program # a) delete heter op and related variables @@ -1087,13 +1183,16 @@ class SplitTrainerOpsPass(PassBase): # 2. check every op's device static_var = [] for heter_block_index in range(1, len(program_block_ops_list)): - ops_list = program_block_ops_list[heter_block_index][ - "forward"] + program_block_ops_list[heter_block_index][ - "backward"] + ops_list = ( + program_block_ops_list[heter_block_index]["forward"] + + program_block_ops_list[heter_block_index]["backward"] + ) static_var += self._replace_ops_by_communicate_op( - program, attrs, heter_block_index, ops_list, block_var_detail) - self._remove_trainer_send_op(program, attrs, heter_block_index, - block_var_detail) + program, attrs, heter_block_index, ops_list, block_var_detail + ) + self._remove_trainer_send_op( + program, attrs, heter_block_index, block_var_detail + ) optimizer_block = [] grad_to_block_id = [] @@ -1101,40 +1200,46 @@ class SplitTrainerOpsPass(PassBase): bp_ops_list = program_block_ops_list[0]["backward"] delete_same_ops(program.global_block(), bp_ops_list) delete_trainer_useless_var(program, static_var) - backward_block = create_backward_block(program, origin_program, - bp_ops_list, block_var_detail) + backward_block = create_backward_block( + program, origin_program, bp_ops_list, block_var_detail + ) bp_entrance_vars = block_var_detail[0]["backward"]["entrance"] - backward_comm_info = get_communicate_var_info(origin_program, - 1, - bp_entrance_vars, - type="backward") - - grad_to_block_id.append(backward_comm_info["block_input_var_name"] + - ":" + str(backward_block.idx)) + backward_comm_info = get_communicate_var_info( + origin_program, 1, bp_entrance_vars, type="backward" + ) + + grad_to_block_id.append( + backward_comm_info["block_input_var_name"] + + ":" + + str(backward_block.idx) + ) optimizer_block.append(backward_block) role_maker = attrs['role_maker'] attrs = { "message_to_block_id": grad_to_block_id, "optimize_blocks": optimizer_block, # runtime attribute - "endpoint": - get_trainer_endpoint(role_maker), ## get trainer endpoint + "endpoint": get_trainer_endpoint( + role_maker + ), ## get trainer endpoint "fanin": 0, ## get heter worker "pserver_id": get_role_id(role_maker), "distributed_mode": attrs['ps_mode'], "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } # append the listen_and_serv op - program.global_block()._insert_op(index=0, - type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + program.global_block()._insert_op( + index=0, + type="heter_listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) ## TODO add check for bp block - #check_op_device(program.global_block(), DEFAULT_DEVICE) + # check_op_device(program.global_block(), DEFAULT_DEVICE) def _apply_single_impl(self, main_program, startup_program, pass_ctx): """ @@ -1146,20 +1251,26 @@ class SplitTrainerOpsPass(PassBase): attrs = pass_ctx._attrs default_device_ = 'cpu' program, heter_ops, default_ops, program_block_ops = find_heter_ops( - main_program, default_device_) + main_program, default_device_ + ) program_block_ops = union_forward_gradient_op(program_block_ops) - block_vars_detail = find_block_joints(program, program_block_ops, - heter_ops) + block_vars_detail = find_block_joints( + program, program_block_ops, heter_ops + ) trainer_program = program.clone() - self._create_trainer_program(trainer_program, program, attrs, - program_block_ops, block_vars_detail) + self._create_trainer_program( + trainer_program, + program, + attrs, + program_block_ops, + block_vars_detail, + ) main_program = trainer_program @register_pass("set_heter_pipeline_opt_pass") class SetHeterPipelineOptPass(PassBase): - def __init__(self): super(SetHeterPipelineOptPass, self).__init__() @@ -1173,32 +1284,31 @@ class SetHeterPipelineOptPass(PassBase): attrs = pass_ctx._attrs role_maker = attrs['role_maker'] num_microbatches = attrs['user_defined_strategy'].pipeline_configs[ - 'accumulate_steps'] + 'accumulate_steps' + ] startup_program._heter_pipeline_opt = { "startup_program": startup_program, "pipeline_stage": int(role_maker._get_stage_id()) - 1, "heter_place": role_maker._heter_device(), - "is_fl_mode": 1 + "is_fl_mode": 1, } main_program._heter_pipeline_opt = { "trainer": "HeterPipelineTrainer", "device_worker": "HeterSection", - "trainers": - role_maker._get_stage_trainers(), ## trainer num in each stage + "trainers": role_maker._get_stage_trainers(), ## trainer num in each stage "trainer_id": int(role_maker._role_id()), "pipeline_stage": int(role_maker._get_stage_id()) - 1, "num_pipeline_stages": int(role_maker._get_num_stage()), "section_program": main_program, "num_microbatches": num_microbatches, "heter_place": role_maker._heter_device(), - "is_fl_mode": 1 + "is_fl_mode": 1, } @register_pass("split_fl_ops_pass") class SplitFlOpsPass(PassBase): - def __init__(self): super(SplitFlOpsPass, self).__init__() self.PART_A_DEVICE_FlAG = 'gpu:0' @@ -1231,10 +1341,17 @@ class SplitFlOpsPass(PassBase): block = self.ori_main_program.block(0) for op in block.ops: device = op.attr(OP_DEVICE_KEY) - if device == self.PART_A_DEVICE_FlAG or device == '' or device == self.PART_A_JOINT_OP_DEVICE_FlAG: + if ( + device == self.PART_A_DEVICE_FlAG + or device == '' + or device == self.PART_A_JOINT_OP_DEVICE_FlAG + ): program = party_program_map['a'] self.partA_ops.append(op) - elif device == self.PART_B_DEVICE_FlAG or device == self.PART_B_JOINT_OP_DEVICE_FlAG: + elif ( + device == self.PART_B_DEVICE_FlAG + or device == self.PART_B_JOINT_OP_DEVICE_FlAG + ): program = party_program_map['b'] self.partB_ops.append(op) op_desc = op.desc @@ -1257,21 +1374,24 @@ class SplitFlOpsPass(PassBase): outputs={'Out': []}, attrs={ 'mode': 'forward', # mode 直接关联前向和反向 channel 选择 - 'send_var_name': - self.partA_to_partB_tensor_name + ["microbatch_id"], + 'send_var_name': self.partA_to_partB_tensor_name + + ["microbatch_id"], 'recv_var_name': [], 'message_name': comm_info, - 'next_endpoints': - get_next_stage_trainers(self.role_maker), # partB_endpoints - 'previous_endpoints': - get_previous_stage_trainers(self.role_maker), + 'next_endpoints': get_next_stage_trainers( + self.role_maker + ), # partB_endpoints + 'previous_endpoints': get_previous_stage_trainers( + self.role_maker + ), 'trainer_id': get_role_id(self.role_maker), # global id - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return def _insert_partB_communicate_op(self, block, idx): - comm_info = ("backward_joint_{}_{}@fl_ps".format(2, 1)) + comm_info = "backward_joint_{}_{}@fl_ps".format(2, 1) block._insert_op( idx, type='send_and_recv', @@ -1279,17 +1399,20 @@ class SplitFlOpsPass(PassBase): outputs={'Out': []}, attrs={ 'mode': 'backward', - 'send_var_name': - self.partB_to_partA_grad_name + ["microbatch_id"], + 'send_var_name': self.partB_to_partA_grad_name + + ["microbatch_id"], 'recv_var_name': [], 'message_name': comm_info, - 'next_endpoints': - get_next_stage_trainers(self.role_maker), # partA_endpoints - 'previous_endpoints': - get_previous_stage_trainers(self.role_maker), + 'next_endpoints': get_next_stage_trainers( + self.role_maker + ), # partA_endpoints + 'previous_endpoints': get_previous_stage_trainers( + self.role_maker + ), 'trainer_id': get_role_id(self.role_maker), # global id - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return def _create_var_for_block(self, vars, block): @@ -1308,7 +1431,8 @@ class SplitFlOpsPass(PassBase): trainable=source_var.trainable, optimize_attr=source_var.optimize_attr, regularizer=source_var.regularizer, - error_clip=source_var.error_clip) + error_clip=source_var.error_clip, + ) else: dest_var = block._clone_variable(source_var, False) dest_var.stop_gradient = source_var.stop_gradient @@ -1362,15 +1486,18 @@ class SplitFlOpsPass(PassBase): def _find_dense_grad_vars(self, bp_op_list): program = self.ori_main_program bp_op_input, bp_op_output = find_ops_list_input_output( - program, bp_op_list) - return (screen_persistables(program, bp_op_input) + - screen_persistables(program, bp_op_output)) + program, bp_op_list + ) + return screen_persistables(program, bp_op_input) + screen_persistables( + program, bp_op_output + ) def _get_partA_program(self, block): # 1. create block 0 # 1.1 insert send op - op_idx = self._find_joint_forward_op(block, - self.PART_A_JOINT_OP_DEVICE_FlAG) + op_idx = self._find_joint_forward_op( + block, self.PART_A_JOINT_OP_DEVICE_FlAG + ) op_list = [] for i in range(len(block.ops)): op = block.ops[i] @@ -1388,8 +1515,9 @@ class SplitFlOpsPass(PassBase): bp_op_list = get_bp_op_list(block) push_sparse_op_list = get_distributed_push_sparse_op_list(block) # logger.info('bp_op_list: {}'.format(bp_op_list)) - second_block = self._get_block_by_idx(bp_op_list + push_sparse_op_list, - self.partA_program, 1) + second_block = self._get_block_by_idx( + bp_op_list + push_sparse_op_list, self.partA_program, 1 + ) # 2.1. insert partA recv op block_input_flag = "backward_joint_{}_{}@fl_ps".format(2, 1) grad_to_block_id = block_input_flag + ":" + str(second_block.idx) @@ -1401,13 +1529,15 @@ class SplitFlOpsPass(PassBase): "pserver_id": get_role_id(self.role_maker), "distributed_mode": self.ps_mode, "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } - second_block._insert_op(index=0, - type='heter_listen_and_serv', - inputs={'X': []}, - outputs={}, - attrs=attrs) + second_block._insert_op( + index=0, + type='heter_listen_and_serv', + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) # 2.2 insert push dense grad op send_ops = find_send_op(self.ori_main_program) # push dense delete_same_ops(block, send_ops) @@ -1417,9 +1547,11 @@ class SplitFlOpsPass(PassBase): def _get_partB_program(self, block): op_idx1 = self._find_joint_forward_op( - block, self.PART_B_JOINT_OP_DEVICE_FlAG) # elementwise_add op - op_idx2 = self._find_joint_backward_op(block, - self.PART_B_JOINT_OP_DEVICE_FlAG) + block, self.PART_B_JOINT_OP_DEVICE_FlAG + ) # elementwise_add op + op_idx2 = self._find_joint_backward_op( + block, self.PART_B_JOINT_OP_DEVICE_FlAG + ) op_cnt = 0 op_list1 = [] op_list2 = [] @@ -1458,16 +1590,18 @@ class SplitFlOpsPass(PassBase): "pserver_id": 1, # TODO "distributed_mode": self.ps_mode, "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } - first_block._insert_op(index=len(op_list1), - type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + first_block._insert_op( + index=len(op_list1), + type="heter_listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) - #logger.info('partB-first_block:{}'.format(first_block)) - #logger.info('partB-second_block:{}'.format(second_block)) + # logger.info('partB-first_block:{}'.format(first_block)) + # logger.info('partB-second_block:{}'.format(second_block)) def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs @@ -1482,8 +1616,9 @@ class SplitFlOpsPass(PassBase): prog_a = party_program_map['a'] _main_file = ps_log_root_dir + '6_fl_A_main_program.prototxt' debug_program(_main_file, prog_a) - self._get_partB_to_partA_grad(prog_a.global_block(), - self.PART_A_JOINT_OP_DEVICE_FlAG) + self._get_partB_to_partA_grad( + prog_a.global_block(), self.PART_A_JOINT_OP_DEVICE_FlAG + ) prog_b = party_program_map['b'] _main_file = ps_log_root_dir + '6_fl_B_main_program.prototxt' diff --git a/python/paddle/distributed/ps/coordinator.py b/python/paddle/distributed/ps/coordinator.py index a012f338a51..3a6e0756df4 100755 --- a/python/paddle/distributed/ps/coordinator.py +++ b/python/paddle/distributed/ps/coordinator.py @@ -26,7 +26,8 @@ import logging logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) formatter = logging.Formatter( - fmt='%(asctime)s %(levelname)-2s [%(filename)s:%(lineno)d] %(message)s') + fmt='%(asctime)s %(levelname)-2s [%(filename)s:%(lineno)d] %(message)s' +) ch = logging.StreamHandler() ch.setFormatter(formatter) logger.addHandler(ch) @@ -46,7 +47,6 @@ class FLStrategy: class ClientSelectorBase(abc.ABC): - def __init__(self, fl_clients_info_mp): self.fl_clients_info_mp = fl_clients_info_mp self.clients_info = {} @@ -58,17 +58,19 @@ class ClientSelectorBase(abc.ABC): for client_id, info in self.fl_clients_info_mp.items(): self.fl_client_info_desc = the_one_ps_pb2.FLClientInfo() - text_format.Parse(bytes(info, encoding="utf8"), - self.fl_client_info_desc) + text_format.Parse( + bytes(info, encoding="utf8"), self.fl_client_info_desc + ) self.clients_info[client_id] = {} self.clients_info[client_id][ - ClientInfoAttr. - DEVICE_TYPE] = self.fl_client_info_desc.device_type + ClientInfoAttr.DEVICE_TYPE + ] = self.fl_client_info_desc.device_type self.clients_info[client_id][ - ClientInfoAttr. - COMPUTE_CAPACITY] = self.fl_client_info_desc.compute_capacity + ClientInfoAttr.COMPUTE_CAPACITY + ] = self.fl_client_info_desc.compute_capacity self.clients_info[client_id][ - ClientInfoAttr.BANDWIDTH] = self.fl_client_info_desc.bandwidth + ClientInfoAttr.BANDWIDTH + ] = self.fl_client_info_desc.bandwidth @abc.abstractmethod def select(self): @@ -76,7 +78,6 @@ class ClientSelectorBase(abc.ABC): class ClientSelector(ClientSelectorBase): - def __init__(self, fl_clients_info_mp): super().__init__(fl_clients_info_mp) self.__fl_strategy = {} @@ -84,8 +85,11 @@ class ClientSelector(ClientSelectorBase): def select(self): self.parse_from_string() for client_id in self.clients_info: - logger.info("fl-ps > client {} info : {}".format( - client_id, self.clients_info[client_id])) + logger.info( + "fl-ps > client {} info : {}".format( + client_id, self.clients_info[client_id] + ) + ) # ......... to implement ...... # fl_strategy_desc = the_one_ps_pb2.FLStrategy() fl_strategy_desc.iteration_num = 99 @@ -97,7 +101,6 @@ class ClientSelector(ClientSelectorBase): class FLClientBase(abc.ABC): - def __init__(self): pass @@ -113,8 +116,9 @@ class FLClientBase(abc.ABC): self.startup_program = paddle.static.default_startup_program() self._client_ptr = fleet.get_fl_client() self._coordinators = self.role_maker._get_coordinator_endpoints() - logger.info("fl-ps > coordinator enpoints: {}".format( - self._coordinators)) + logger.info( + "fl-ps > coordinator enpoints: {}".format(self._coordinators) + ) self.strategy_handlers = dict() self.exe = None self.use_cuda = int(self.config.get("runner.use_gpu")) @@ -129,8 +133,11 @@ class FLClientBase(abc.ABC): def set_train_dataset_info(self, train_dataset, train_file_list): self.train_dataset = train_dataset self.train_file_list = train_file_list - logger.info("fl-ps > {}, data_feed_desc:\n {}".format( - type(self.train_dataset), self.train_dataset._desc())) + logger.info( + "fl-ps > {}, data_feed_desc:\n {}".format( + type(self.train_dataset), self.train_dataset._desc() + ) + ) def set_test_dataset_info(self, test_dataset, test_file_list): self.test_dataset = test_dataset @@ -161,7 +168,8 @@ class FLClientBase(abc.ABC): if self.config.get("runner.need_dump"): self.debug = True dump_fields_path = "{}/epoch_{}".format( - self.config.get("runner.dump_fields_path"), self.epoch_idx) + self.config.get("runner.dump_fields_path"), self.epoch_idx + ) dump_fields = self.config.get("runner.dump_fields", []) dump_param = self.config.get("runner.dump_param", []) persist_vars_list = self.main_program.all_parameters() @@ -170,11 +178,13 @@ class FLClientBase(abc.ABC): for param in persist_vars_list ] logger.info( - "fl-ps > persist_vars_list: {}".format(persist_vars_name)) + "fl-ps > persist_vars_list: {}".format(persist_vars_name) + ) if dump_fields_path is not None: self.main_program._fleet_opt[ - 'dump_fields_path'] = dump_fields_path + 'dump_fields_path' + ] = dump_fields_path if dump_fields is not None: self.main_program._fleet_opt["dump_fields"] = dump_fields if dump_param is not None: @@ -186,7 +196,6 @@ class FLClientBase(abc.ABC): class FLClient(FLClientBase): - def __init__(self): super(FLClient, self).__init__() @@ -195,12 +204,13 @@ class FLClient(FLClientBase): state_info = { ClientInfoAttr.DEVICE_TYPE: "Andorid", ClientInfoAttr.COMPUTE_CAPACITY: 10, - ClientInfoAttr.BANDWIDTH: 100 + ClientInfoAttr.BANDWIDTH: 100, } client_info = the_one_ps_pb2.FLClientInfo() client_info.device_type = state_info[ClientInfoAttr.DEVICE_TYPE] client_info.compute_capacity = state_info[ - ClientInfoAttr.COMPUTE_CAPACITY] + ClientInfoAttr.COMPUTE_CAPACITY + ] client_info.bandwidth = state_info[ClientInfoAttr.BANDWIDTH] str_msg = text_format.MessageToString(client_info) return str_msg @@ -224,7 +234,7 @@ class FLClient(FLClientBase): state_info = { "client id": self.worker_index, "auc": 0.9, - "epoch": self.epoch_idx + "epoch": self.epoch_idx, } self.push_fl_client_info_sync(state_info) strategy_dict = self.pull_fl_strategy() @@ -242,13 +252,18 @@ class FLClient(FLClientBase): def pull_fl_strategy(self): strategy_dict = {} - fl_strategy_str = self._client_ptr.pull_fl_strategy( + fl_strategy_str = ( + self._client_ptr.pull_fl_strategy() ) # block: wait for coordinator's strategy arrived - logger.info("fl-ps > fl client recved fl_strategy(str):\n{}".format( - fl_strategy_str)) + logger.info( + "fl-ps > fl client recved fl_strategy(str):\n{}".format( + fl_strategy_str + ) + ) fl_strategy_desc = the_one_ps_pb2.FLStrategy() - text_format.Parse(bytes(fl_strategy_str, encoding="utf8"), - fl_strategy_desc) + text_format.Parse( + bytes(fl_strategy_str, encoding="utf8"), fl_strategy_desc + ) strategy_dict["next_state"] = fl_strategy_desc.next_state return strategy_dict @@ -262,8 +277,9 @@ class FLClient(FLClientBase): self.register_handlers('train', self.callback_train) self.register_handlers('infer', self.callback_infer) self.register_handlers('finish', self.callback_finish) - self.register_handlers('initialize_model_params', - self.callback_initialize_model_params) + self.register_handlers( + 'initialize_model_params', self.callback_initialize_model_params + ) self.register_handlers('init_worker', self.callback_init_worker) self.register_handlers('save_model', self.callback_save_model) @@ -282,12 +298,14 @@ class FLClient(FLClientBase): "Epoch {} Var {}".format(self.epoch_idx, var_name) for var_name in self.metrics ] - self.exe.train_from_dataset(program=self.main_program, - dataset=self.train_dataset, - fetch_list=self.fetch_vars, - fetch_info=fetch_info, - print_period=self.print_step, - debug=self.debug) + self.exe.train_from_dataset( + program=self.main_program, + dataset=self.train_dataset, + fetch_list=self.fetch_vars, + fetch_info=fetch_info, + print_period=self.print_step, + debug=self.debug, + ) self.epoch_idx += 1 epoch_time = time.time() - epoch_start_time epoch_speed = self.train_example_nums / epoch_time @@ -299,12 +317,14 @@ class FLClient(FLClientBase): "Epoch {} Var {}".format(self.epoch_idx, var_name) for var_name in self.metrics ] - self.exe.infer_from_dataset(program=self.main_program, - dataset=self.test_dataset, - fetch_list=self.fetch_vars, - fetch_info=fetch_info, - print_period=self.print_step, - debug=self.debug) + self.exe.infer_from_dataset( + program=self.main_program, + dataset=self.test_dataset, + fetch_list=self.fetch_vars, + fetch_info=fetch_info, + print_period=self.print_step, + debug=self.debug, + ) def callback_save_model(self): model_dir = "{}/{}".format(self.save_model_path, self.epoch_idx) @@ -318,12 +338,14 @@ class FLClient(FLClientBase): fleet.stop_worker() def print_program(self): - with open("./{}_worker_main_program.prototxt".format(self.worker_index), - 'w+') as f: + with open( + "./{}_worker_main_program.prototxt".format(self.worker_index), 'w+' + ) as f: f.write(str(self.main_program)) with open( - "./{}_worker_startup_program.prototxt".format( - self.worker_index), 'w+') as f: + "./{}_worker_startup_program.prototxt".format(self.worker_index), + 'w+', + ) as f: f.write(str(self.startup_program)) def print_train_statical_info(self): @@ -332,7 +354,6 @@ class FLClient(FLClientBase): class Coordinator(object): - def __init__(self, ps_hosts): self._communicator = FLCommunicator(ps_hosts) self._client_selector = None @@ -344,7 +365,8 @@ class Coordinator(object): logger.info("fl-ps > running make_fl_strategy(loop) in coordinator\n") while True: # 1. get all fl clients reported info - str_map = self._communicator.query_fl_clients_info( + str_map = ( + self._communicator.query_fl_clients_info() ) # block: wait for all fl clients info reported # 2. generate fl strategy self._client_selector = ClientSelector(str_map) diff --git a/python/paddle/distributed/ps/the_one_ps.py b/python/paddle/distributed/ps/the_one_ps.py index 78ecef22b63..86766d60ae8 100755 --- a/python/paddle/distributed/ps/the_one_ps.py +++ b/python/paddle/distributed/ps/the_one_ps.py @@ -24,15 +24,21 @@ from paddle.fluid.compiler import CompiledProgram from paddle.fluid.executor import Executor from paddle.fluid.parallel_executor import ParallelExecutor from paddle.distributed.fleet.runtime.runtime_base import RuntimeBase -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) from paddle.distributed.fleet.proto import the_one_ps_pb2 from paddle.fluid.communicator import Communicator, HeterClient from google.protobuf import text_format from paddle.distributed.ps.coordinator import Coordinator __all__ = [ - 'Table', 'SparseTable', 'GeoSparseTable', 'BarrierTable', 'TensorTable', - 'DenseTable' + 'Table', + 'SparseTable', + 'GeoSparseTable', + 'BarrierTable', + 'TensorTable', + 'DenseTable', ] @@ -52,7 +58,11 @@ def parse_table_class(varname, program_id, context): param_name = op.input("W")[0] - if param_name == varname and op.type == "lookup_table" or op.type == "lookup_table_v2": + if ( + param_name == varname + and op.type == "lookup_table" + or op.type == "lookup_table_v2" + ): if op.has_attr('table_class') and op.attr("table_class") != "none": return op.attr('table_class') else: @@ -65,37 +75,47 @@ def check_embedding_dim(accessor_proto, varname, program_id, context): for var in main_program.list_vars(): if var.name == varname: embedding_dim = var.shape[1] - print('new var: {}, {}, {}'.format(var, embedding_dim, - accessor_proto.fea_dim)) + print( + 'new var: {}, {}, {}'.format( + var, embedding_dim, accessor_proto.fea_dim + ) + ) break fea_dim = accessor_proto.fea_dim if accessor_proto.accessor_class == "SparseAccessor": if fea_dim != embedding_dim + 2: raise ValueError( - "The fea_dim is wrong, it will be sparse_embedding_dim + 2: {}, but got {}" - .format(embedding_dim + 2, fea_dim)) + "The fea_dim is wrong, it will be sparse_embedding_dim + 2: {}, but got {}".format( + embedding_dim + 2, fea_dim + ) + ) else: if fea_dim != embedding_dim: raise ValueError( - "The fea_dim is wrong, it will be sparse_embedding_dim: {}, but got {}" - .format(embedding_dim, fea_dim)) + "The fea_dim is wrong, it will be sparse_embedding_dim: {}, but got {}".format( + embedding_dim, fea_dim + ) + ) embedx_dim = accessor_proto.embedx_dim if accessor_proto.accessor_class == "SparseAccessor": if embedx_dim != embedding_dim - 1: raise ValueError( - "The embedx_dim is wrong, it will be sparse_embedding_dim - 1: {}, but got {}" - .format(embedding_dim - 1, embedx_dim)) + "The embedx_dim is wrong, it will be sparse_embedding_dim - 1: {}, but got {}".format( + embedding_dim - 1, embedx_dim + ) + ) else: if embedx_dim != embedding_dim - 3: raise ValueError( - "The embedx_dim is wrong, it will be sparse_embedding_dim - 3: {}, but got {}" - .format(embedding_dim - 3, embedx_dim)) + "The embedx_dim is wrong, it will be sparse_embedding_dim - 3: {}, but got {}".format( + embedding_dim - 3, embedx_dim + ) + ) class Service: - def __init__(self): pass @@ -108,7 +128,6 @@ class Service: class GpuService(Service): - def __init__(self): super(GpuService, self).__init__() @@ -118,7 +137,6 @@ class GpuService(Service): class Accessor: - def __init__(self): self.accessor_class = "" self.optimizer = None @@ -126,10 +144,12 @@ class Accessor: self.embedding_dim = 0 # TableAccessorParameter accessor - def _set(self, accessor_proto, varname, program_id, context, - common_accessor): + def _set( + self, accessor_proto, varname, program_id, context, common_accessor + ): main_program, startup_program, idx = get_program_by_id( - context, program_id) + context, program_id + ) embedding_dim = 0 for var in main_program.list_vars(): if var.name == varname: @@ -184,7 +204,8 @@ class Accessor: ctr_accessor_param.ssd_unseenday_threshold = 1 for sgd_param in [ - accessor_proto.embed_sgd_param, accessor_proto.embedx_sgd_param + accessor_proto.embed_sgd_param, + accessor_proto.embedx_sgd_param, ]: if not sgd_param.HasField("name"): if common_accessor.accessor_class == "sgd": @@ -194,7 +215,10 @@ class Accessor: else: # for fl-ps, because geo accessor is 'sum' sgd_param.name = "SparseAdamSGDRule" - if sgd_param.name == "SparseAdaGradSGDRule" or sgd_param.name == "StdAdaGradSGDRule": + if ( + sgd_param.name == "SparseAdaGradSGDRule" + or sgd_param.name == "StdAdaGradSGDRule" + ): if not sgd_param.adagrad.HasField("learning_rate"): sgd_param.adagrad.learning_rate = 0.05 if not sgd_param.adagrad.HasField("initial_g2sum"): @@ -206,42 +230,52 @@ class Accessor: if sgd_param.name == "SparseNaiveSGDRule": if not sgd_param.naive.HasField("learning_rate"): - learning_rate = common_accessor.initializers[-1].split( - "&")[1] + learning_rate = common_accessor.initializers[-1].split("&")[ + 1 + ] sgd_param.naive.learning_rate = float(learning_rate) if not sgd_param.naive.HasField("initial_range"): - initial_range = common_accessor.initializers[0].split( - "&")[-1] + initial_range = common_accessor.initializers[0].split("&")[ + -1 + ] sgd_param.naive.initial_range = float(initial_range) if len(sgd_param.naive.weight_bounds) == 0: sgd_param.naive.weight_bounds.extend([-10.0, 10.0]) - if sgd_param.name == "SparseAdamSGDRule" or sgd_param.name == "SparseSharedAdamSGDRule": + if ( + sgd_param.name == "SparseAdamSGDRule" + or sgd_param.name == "SparseSharedAdamSGDRule" + ): if not sgd_param.adam.HasField("learning_rate"): - learning_rate = common_accessor.initializers[-1].split( - "&")[1] + learning_rate = common_accessor.initializers[-1].split("&")[ + 1 + ] sgd_param.adam.learning_rate = float(learning_rate) if not sgd_param.adam.HasField("initial_range"): - initial_range = common_accessor.initializers[0].split( - "&")[-1] + initial_range = common_accessor.initializers[0].split("&")[ + -1 + ] sgd_param.adam.initial_range = float(initial_range) attr_list = [x.split("&") for x in common_accessor.attrs] - if not sgd_param.adam.HasField( - "beta1_decay_rate" - ) and common_accessor.accessor_class == "adam": + if ( + not sgd_param.adam.HasField("beta1_decay_rate") + and common_accessor.accessor_class == "adam" + ): sgd_param.adam.beta1_decay_rate = float(attr_list[0][1]) else: sgd_param.adam.beta1_decay_rate = 0.9 - if not sgd_param.adam.HasField( - "beta2_decay_rate" - ) and common_accessor.accessor_class == "adam": + if ( + not sgd_param.adam.HasField("beta2_decay_rate") + and common_accessor.accessor_class == "adam" + ): sgd_param.adam.beta2_decay_rate = float(attr_list[1][1]) else: sgd_param.adam.beta2_decay_rate = 0.999 - if not sgd_param.adam.HasField( - "ada_epsilon" - ) and common_accessor.accessor_class == "adam": + if ( + not sgd_param.adam.HasField("ada_epsilon") + and common_accessor.accessor_class == "adam" + ): sgd_param.adam.ada_epsilon = float(attr_list[2][1]) else: sgd_param.adam.ada_epsilon = 1e-08 @@ -250,7 +284,6 @@ class Accessor: class CommonAccessor(Accessor): - def __init__(self): super(CommonAccessor, self).__init__() self.table_name = '' @@ -269,27 +302,46 @@ class CommonAccessor(Accessor): def define_optimize_map(self): opt_input_map = {} opt_input_map["sgd"] = [("Param", None), ("LearningRate", 1)] - opt_input_map["adam"] = [("Param", None), ("Moment1", None), - ("Moment2", None), ("Beta1Pow", 1), - ("Beta2Pow", 1), ("LearningRate", 1)] - opt_input_map["adam_d2sum"] = [("Param", None), ("D2Sum", None), - ("G2Sum", None), ("Moment", None), - ("MomentDecayRate", 1), - ("AdaDecayRate", 1), ("AdaEpsilon", 1), - ("LearningRate", 1)] + opt_input_map["adam"] = [ + ("Param", None), + ("Moment1", None), + ("Moment2", None), + ("Beta1Pow", 1), + ("Beta2Pow", 1), + ("LearningRate", 1), + ] + opt_input_map["adam_d2sum"] = [ + ("Param", None), + ("D2Sum", None), + ("G2Sum", None), + ("Moment", None), + ("MomentDecayRate", 1), + ("AdaDecayRate", 1), + ("AdaEpsilon", 1), + ("LearningRate", 1), + ] opt_input_map["sum"] = [("Param", None)] - opt_input_map["naive_adagrad"] = [("Param", None), ("G2Sum", 1), - ("LearningRate", 1)] + opt_input_map["naive_adagrad"] = [ + ("Param", None), + ("G2Sum", 1), + ("LearningRate", 1), + ] opt_input_map["summary"] = [("Param", None), ("SummaryDecayRate", 1)] opt_attr_map = {} opt_attr_map["sgd"] = [] opt_attr_map["sum"] = [] opt_attr_map["naive_adagrad"] = [] - opt_attr_map["adam"] = [("beta1", "f"), ("beta2", "f"), - ("epsilon", "f")] - opt_attr_map["adam_d2sum"] = [("beta1", "f"), ("beta2", "f"), - ("epsilon", "f")] + opt_attr_map["adam"] = [ + ("beta1", "f"), + ("beta2", "f"), + ("epsilon", "f"), + ] + opt_attr_map["adam_d2sum"] = [ + ("beta1", "f"), + ("beta2", "f"), + ("epsilon", "f"), + ] opt_attr_map["summary"] = [("summary_decay_rate", "f")] opt_init_map = {} @@ -304,7 +356,8 @@ class CommonAccessor(Accessor): def parse_entry(self, varname, program_id, context): main_program, startup_program, idx = get_program_by_id( - context, program_id) + context, program_id + ) for op in main_program.global_block().ops: if not is_distributed_sparse_op(op) and not is_sparse_op(op): continue @@ -337,8 +390,10 @@ class CommonAccessor(Accessor): origin_var_name = value_name # print("get_initializer_attr param name:", value_name) for op in o_startup_program.global_block().ops: - if op.type in self.opt_init_map.keys( - ) and origin_var_name == op.output("Out")[0]: + if ( + op.type in self.opt_init_map.keys() + and origin_var_name == op.output("Out")[0] + ): init_attr = [op.type] # print("get_initializer_attr op type:", op.type) for attr in self.opt_init_map[op.type]: @@ -359,7 +414,8 @@ class CommonAccessor(Accessor): # ctx.table_id(), ctx.is_datanorm_table())) main_program, startup_program, idx = get_program_by_id( - context, ctx.program_id()) + context, ctx.program_id() + ) pserver_id = get_role_id(context['role_maker']) pserver_num = len(get_ps_endpoints(context['role_maker'])) optimizer_ops = get_optimize_ops(main_program) @@ -369,8 +425,9 @@ class CommonAccessor(Accessor): for op in optimizer_ops: if ("Param" in op.input_names) and ( - op.input("Param")[0] - == context['grad_name_to_param_name'][grad_name]): + op.input("Param")[0] + == context['grad_name_to_param_name'][grad_name] + ): oop = op break @@ -409,7 +466,8 @@ class CommonAccessor(Accessor): else: if oop.type != 'sgd' and oop.type != 'adam': raise ValueError( - "The dense optimizer in PS is only supported SGD or Adam!") + "The dense optimizer in PS is only supported SGD or Adam!" + ) param_varnames = self.opt_input_map[oop.type] attr_varnames = self.opt_attr_map[oop.type] self.accessor_class = oop.type @@ -417,7 +475,7 @@ class CommonAccessor(Accessor): for (formal_name, shape) in param_varnames: params.append(formal_name) if self.accessor_class == "adam_d2sum": - #for dims + # for dims if shape is None: if is_sparse: shape = single_dim @@ -425,19 +483,24 @@ class CommonAccessor(Accessor): shape = self.get_shard(size, pserver_num, pserver_id) dims.append(shape) - #for initializers + # for initializers if formal_name == "Param" or formal_name == "LearningRate": - param = main_program.global_block().vars[oop.input( - formal_name)[0]] - #TODO: for dense learning_rate, can be different from sparse lr - if formal_name == "LearningRate" and param.name != "learning_rate_" + str( - idx): + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] + # TODO: for dense learning_rate, can be different from sparse lr + if ( + formal_name == "LearningRate" + and param.name != "learning_rate_" + str(idx) + ): warnings.warn("will support decay soon") param = main_program.global_block().vars[ - "learning_rate_" + str(idx)] + "learning_rate_" + str(idx) + ] initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) elif formal_name == "MomentDecayRate": initializer = "fill_constant&0.99" elif formal_name == "AdaDecayRate": @@ -448,7 +511,7 @@ class CommonAccessor(Accessor): initializer = "fill_constant&0" initializers.append(initializer) elif self.accessor_class == "summary": - #for dims + # for dims if shape is None: if is_sparse: shape = single_dim @@ -456,13 +519,15 @@ class CommonAccessor(Accessor): shape = self.get_shard(size, pserver_num, pserver_id) dims.append(shape) - #for initializers + # for initializers if formal_name == "Param": - param = main_program.global_block().vars[oop.input( - formal_name)[0]] + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) elif formal_name == "SummaryDecayRate": initializer = "fill_constant&0.999999" else: @@ -474,32 +539,39 @@ class CommonAccessor(Accessor): initializer = "fill_constant&0" initializers.append(initializer) else: - param = main_program.global_block().vars[oop.input( - formal_name)[0]] - if formal_name == "LearningRate" and param.name != "learning_rate_" + str( - idx): + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] + if ( + formal_name == "LearningRate" + and param.name != "learning_rate_" + str(idx) + ): warnings.warn("will support decay soon") param = main_program.global_block().vars[ - "learning_rate_" + str(idx)] + "learning_rate_" + str(idx) + ] if shape is None: if is_sparse: shape = single_dim else: - shape = self.get_shard(size, pserver_num, - pserver_id) + shape = self.get_shard( + size, pserver_num, pserver_id + ) dims.append(shape) initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) initializers.append(initializer) if self.accessor_class == 'summary': datanorm_ops = get_datanorm_ops(main_program) for op in datanorm_ops: if ("BatchSize" in op.input_names) and ( - op.input("BatchSize")[0] - == context['grad_name_to_param_name'][grad_name]): + op.input("BatchSize")[0] + == context['grad_name_to_param_name'][grad_name] + ): oop = op break @@ -528,23 +600,24 @@ class CommonAccessor(Accessor): class Tensor: - def __init__(self, tesnor_dcit): self.tensor_dict = tesnor_dcit def _set(self, tensor_proto): tensor_proto.main_program_id = self.tensor_dict.get( - "main_program_id", 0) + "main_program_id", 0 + ) tensor_proto.startup_program_id = self.tensor_dict.get( - "startup_program_id", 0) + "startup_program_id", 0 + ) tensor_proto.feed_var_name = self.tensor_dict.get("feed_var_name", '') tensor_proto.fetch_var_name = self.tensor_dict.get("fetch_var_name", '') tensor_proto.tensor_table_class = self.tensor_dict.get( - "tensor_table_class", '') + "tensor_table_class", '' + ) class Table: - def __init__(self): self.table_class = None self.shard_num = -1 @@ -559,7 +632,6 @@ class Table: class BarrierTable(Table): - def __init__(self, context, idx): super(BarrierTable, self).__init__() self.type = None @@ -595,7 +667,6 @@ class BarrierTable(Table): class TensorTable(Table): - def __init__(self, idx, tensor_dict, role_maker): super(TensorTable, self).__init__() self.idx = idx @@ -610,7 +681,8 @@ class TensorTable(Table): table_proto.accessor.accessor_class = "CommMergeAccessor" table_proto.common.table_name = self.tensor_dict.get( - "feed_var_name", '') + "feed_var_name", '' + ) table_proto.common.trainer_num = get_trainers(self.role_maker) tensor = Tensor(self.tensor_dict) @@ -618,7 +690,6 @@ class TensorTable(Table): class SparseTable(Table): - def __init__(self, context, send_ctx): super(SparseTable, self).__init__() self.context = context @@ -629,31 +700,39 @@ class SparseTable(Table): def _set(self, table_proto): ctx = self.ctx - if ctx.is_tensor_table() or len( - ctx.origin_varnames()) < 1 or (ctx.is_sparse() == False): + if ( + ctx.is_tensor_table() + or len(ctx.origin_varnames()) < 1 + or (ctx.is_sparse() == False) + ): return table_proto.table_id = ctx.table_id() table_proto.table_class = self.table_class table_proto.type = the_one_ps_pb2.PS_SPARSE_TABLE table_proto.shard_num = self.shard_num if table_proto.sparse_table_cache_file_num > len( - get_ps_endpoints(self.context['role_maker'])): + get_ps_endpoints(self.context['role_maker']) + ): table_proto.sparse_table_cache_file_num = len( - get_ps_endpoints(self.context['role_maker'])) + get_ps_endpoints(self.context['role_maker']) + ) self.common.table_name = self.context['grad_name_to_param_name'][ - ctx.origin_varnames()[0]] + ctx.origin_varnames()[0] + ] self.common.parse_by_optimizer(ctx, self.context) - self.common.parse_entry(self.common.table_name, ctx.program_id(), - self.context) + self.common.parse_entry( + self.common.table_name, ctx.program_id(), self.context + ) self.common.sync = True if self.context['is_sync'] else False self.common._set(table_proto.common) print('new table_name: {}'.format(self.common.table_name)) all_table_proto = self.context[ - "user_defined_strategy"].sparse_table_configs + "user_defined_strategy" + ].sparse_table_configs usr_table_proto = all_table_proto.add() for proto in all_table_proto: if proto.table_name == self.common.table_name: @@ -679,11 +758,17 @@ class SparseTable(Table): ) if usr_table_proto.HasField("enable_sparse_table_cache"): - table_proto.enable_sparse_table_cache = usr_table_proto.enable_sparse_table_cache + table_proto.enable_sparse_table_cache = ( + usr_table_proto.enable_sparse_table_cache + ) if usr_table_proto.HasField("sparse_table_cache_rate"): - table_proto.sparse_table_cache_rate = usr_table_proto.sparse_table_cache_rate + table_proto.sparse_table_cache_rate = ( + usr_table_proto.sparse_table_cache_rate + ) if usr_table_proto.HasField("sparse_table_cache_file_num"): - table_proto.sparse_table_cache_file_num = usr_table_proto.sparse_table_cache_file_num + table_proto.sparse_table_cache_file_num = ( + usr_table_proto.sparse_table_cache_file_num + ) if usr_table_proto.HasField("enable_revert"): table_proto.enable_revert = usr_table_proto.enable_revert if usr_table_proto.HasField("shard_merge_rate"): @@ -691,19 +776,29 @@ class SparseTable(Table): if usr_table_proto.accessor.ByteSize() == 0: warnings.warn( - "The accessor of sparse table is not set, use default value.") + "The accessor of sparse table is not set, use default value." + ) table_proto.accessor.ParseFromString( - usr_table_proto.accessor.SerializeToString()) - self.accessor._set(table_proto.accessor, self.common.table_name, - ctx.program_id(), self.context, self.common) + usr_table_proto.accessor.SerializeToString() + ) + self.accessor._set( + table_proto.accessor, + self.common.table_name, + ctx.program_id(), + self.context, + self.common, + ) - check_embedding_dim(table_proto.accessor, self.common.table_name, - ctx.program_id(), self.context) + check_embedding_dim( + table_proto.accessor, + self.common.table_name, + ctx.program_id(), + self.context, + ) class GeoSparseTable(SparseTable): - def __init__(self, context, send_ctx): super(GeoSparseTable, self).__init__(context, send_ctx) self.table_class = "MemorySparseGeoTable" @@ -712,8 +807,11 @@ class GeoSparseTable(SparseTable): def _set(self, table_proto): ctx = self.ctx - if ctx.is_tensor_table() or len( - ctx.origin_varnames()) < 1 or (ctx.is_sparse() == False): + if ( + ctx.is_tensor_table() + or len(ctx.origin_varnames()) < 1 + or (ctx.is_sparse() == False) + ): return table_proto.table_id = ctx.table_id() table_proto.table_class = self.table_class @@ -725,16 +823,17 @@ class GeoSparseTable(SparseTable): table_proto.accessor.embedx_dim = ctx.sections()[1] self.common.table_name = self.context['grad_name_to_param_name'][ - ctx.origin_varnames()[0]] + ctx.origin_varnames()[0] + ] self.common.parse_by_optimizer(ctx, self.context) - self.common.parse_entry(self.common.table_name, ctx.program_id(), - self.context) + self.common.parse_entry( + self.common.table_name, ctx.program_id(), self.context + ) self.common.sync = False self.common._set(table_proto.common) class DenseTable(Table): - def __init__(self, context, send_ctx): super(DenseTable, self).__init__() self.context = context @@ -743,8 +842,11 @@ class DenseTable(Table): def _set(self, table_proto): ctx = self.ctx - if ctx.is_tensor_table() or len( - ctx.origin_varnames()) < 1 or (ctx.is_sparse() == True): + if ( + ctx.is_tensor_table() + or len(ctx.origin_varnames()) < 1 + or (ctx.is_sparse() == True) + ): return table_proto.table_id = ctx.table_id() @@ -759,15 +861,15 @@ class DenseTable(Table): self.common.table_name = "MergedDense" self.common.parse_by_optimizer(ctx, self.context) - self.common.parse_entry(self.common.table_name, ctx.program_id(), - self.context) + self.common.parse_entry( + self.common.table_name, ctx.program_id(), self.context + ) self.common.sync = True if self.context['is_sync'] else False self.common._set(table_proto.common) class Server: - def __init__(self): pass @@ -776,7 +878,6 @@ class Server: class DownpourServer(Server): - def __init__(self): super(DownpourServer, self).__init__() @@ -785,7 +886,6 @@ class DownpourServer(Server): class Worker: - def __init__(self): pass @@ -794,7 +894,6 @@ class Worker: class DownpourWorker(Worker): - def __init__(self): super(DownpourWorker, self).__init__() @@ -803,7 +902,6 @@ class DownpourWorker(Worker): class fsClient: - def __init__(self, fs_client_param): self.fs_client_param = fs_client_param @@ -817,7 +915,6 @@ class fsClient: class PsDescBuilder(object): - def __init__(self, context): self.context = context self.is_sync = context['is_sync'] @@ -827,7 +924,8 @@ class PsDescBuilder(object): self.barrier_table_id = None self.send_ctx = get_the_one_send_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) self.tensor_table_dict = {} # TODO self._server_sub_program = [] @@ -846,8 +944,11 @@ class PsDescBuilder(object): self._server_sub_program.append(Program().desc) tables = [] for table_name in self.tensor_table_dict: - tables.append(globals()['TensorTable'](len(tables), tensor_dict, - self.context['role_maker'])) + tables.append( + globals()['TensorTable']( + len(tables), tensor_dict, self.context['role_maker'] + ) + ) program_idx += 1 return tables @@ -857,14 +958,17 @@ class PsDescBuilder(object): print("idx, name, ctx:", idx, name, ctx) if ctx.is_sparse(): if self.ps_mode == DistributedMode.GEO: - if (self.context['local_sparse'] - and name[:-5] in self.context['local_sparse']) or ( - not self.context['local_sparse']): - tables.append(globals()['GeoSparseTable'](self.context, - ctx)) + if ( + self.context['local_sparse'] + and name[:-5] in self.context['local_sparse'] + ) or (not self.context['local_sparse']): + tables.append( + globals()['GeoSparseTable'](self.context, ctx) + ) else: - tables.append(globals()['SparseTable'](self.context, - ctx)) + tables.append( + globals()['SparseTable'](self.context, ctx) + ) else: tables.append(globals()['SparseTable'](self.context, ctx)) else: @@ -888,37 +992,45 @@ class PsDescBuilder(object): def build_worker_desc(self): for table in self.tables: - table_proto = self.ps_desc.worker_param.downpour_worker_param.downpour_table_param.add( + table_proto = ( + self.ps_desc.worker_param.downpour_worker_param.downpour_table_param.add() ) table._set(table_proto) - table_proto = self.ps_desc.server_param.downpour_server_param.downpour_table_param.add( + table_proto = ( + self.ps_desc.server_param.downpour_server_param.downpour_table_param.add() ) table._set(table_proto) if type(table) == BarrierTable and self.barrier_table_id is None: self.barrier_table_id = table.idx self.service._set( - self.ps_desc.server_param.downpour_server_param.service_param) + self.ps_desc.server_param.downpour_server_param.service_param + ) self.fs_client._set(self.ps_desc.fs_client_param) return text_format.MessageToString(self.ps_desc) def build_server_desc(self): self.sparse_table_maps = {} for table in self.tables: - table_proto = self.ps_desc.server_param.downpour_server_param.downpour_table_param.add( + table_proto = ( + self.ps_desc.server_param.downpour_server_param.downpour_table_param.add() ) table._set(table_proto) - if table_proto.type == the_one_ps_pb2.PS_SPARSE_TABLE and table_proto.common is not None: + if ( + table_proto.type == the_one_ps_pb2.PS_SPARSE_TABLE + and table_proto.common is not None + ): self.sparse_table_maps[ - table_proto.common.table_name] = table_proto.table_id + table_proto.common.table_name + ] = table_proto.table_id self.service._set( - self.ps_desc.server_param.downpour_server_param.service_param) + self.ps_desc.server_param.downpour_server_param.service_param + ) self.fs_client._set(self.ps_desc.fs_client_param) return text_format.MessageToString(self.ps_desc) class TheOnePSRuntime(RuntimeBase): - def __init__(self): super(TheOnePSRuntime, self).__init__() self._communicator = None @@ -936,30 +1048,41 @@ class TheOnePSRuntime(RuntimeBase): self.debug = bool(int(os.getenv("PSERVER_DEBUG", "0"))) self.origin_main_program = context["origin_main_program"] - self.origin_main_programs = context.get("origin_main_programs", - [self.origin_main_program]) + self.origin_main_programs = context.get( + "origin_main_programs", [self.origin_main_program] + ) self.context["origin_main_programs"] = self.origin_main_programs self.context["origin_startup_programs"] = context.get( - 'origin_startup_programs', [context['origin_startup_program']]) + 'origin_startup_programs', [context['origin_startup_program']] + ) self.context[ - 'is_heter_ps_mode'] = self.role_maker._is_heter_parameter_server_mode + 'is_heter_ps_mode' + ] = self.role_maker._is_heter_parameter_server_mode self.is_heter_ps_mode = self.context['is_heter_ps_mode'] self.context['trainer'] = TrainerRuntimeConfig( - context['valid_strategy']) + context['valid_strategy'] + ) self.context['ps_mode'] = self.context['trainer'].mode self.context['use_ps_gpu'] = context['valid_strategy'].a_sync_configs[ - 'use_ps_gpu'] - self.context['is_sync'] = True if self.context[ - 'ps_mode'] == DistributedMode.SYNC else False + 'use_ps_gpu' + ] + self.context['is_sync'] = ( + True if self.context['ps_mode'] == DistributedMode.SYNC else False + ) self.context['grad_name_to_param_name'] = {} self.context['tensor_table'] = {} # FL self.context['local_sparse'] = context[ - "user_defined_strategy"].trainer_desc_configs["local_sparse"] + "user_defined_strategy" + ].trainer_desc_configs["local_sparse"] self.context['remote_sparse'] = context[ - "user_defined_strategy"].trainer_desc_configs["remote_sparse"] - print("fl-ps > local_sparse: {}, remote_sparse: {}".format( - self.context['local_sparse'], self.context['remote_sparse'])) + "user_defined_strategy" + ].trainer_desc_configs["remote_sparse"] + print( + "fl-ps > local_sparse: {}, remote_sparse: {}".format( + self.context['local_sparse'], self.context['remote_sparse'] + ) + ) build_var_distributed(self.context) @@ -993,7 +1116,7 @@ class TheOnePSRuntime(RuntimeBase): scope = scopes[idx] table_id = ctx.table_id() var_names = recv_map[table_id] - #print("init params:", idx, table_id, var_names) + # print("init params:", idx, table_id, var_names) self._worker.push_dense_params(scope, table_id, var_names) all_var_names.extend(var_names) return all_var_names @@ -1007,7 +1130,7 @@ class TheOnePSRuntime(RuntimeBase): scope = scopes[idx] table_id = ctx.table_id() var_names = recv_map[table_id] - #print("pull all dense:", idx, table_id, var_names) + # print("pull all dense:", idx, table_id, var_names) self._worker.pull_dense_params(scope, table_id, var_names) all_var_names.extend(var_names) return all_var_names @@ -1056,16 +1179,19 @@ class TheOnePSRuntime(RuntimeBase): def sync_strategy_envs(): kwargs = {} kwargs[ - "pserver_endpoints"] = self.role_maker._get_pserver_endpoints() + "pserver_endpoints" + ] = self.role_maker._get_pserver_endpoints() kwargs["trainer_id"] = self.role_maker._worker_index() return kwargs dense_map = get_the_one_recv_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) send_ctx = get_the_one_send_context( self.context, split_dense_table=self.is_heter_ps_mode, - ep_list=self.endpoints) + ep_list=self.endpoints, + ) self._send_ctx = send_ctx trainer_config = self.context['trainer'] @@ -1097,24 +1223,34 @@ class TheOnePSRuntime(RuntimeBase): print("fl-ps > with_coordinator? {}".format(self.with_coordinator)) print("fl-ps > coordinator addr: {}".format(self.coordinator_hosts)) if self.with_coordinator: - self._worker.init_fl_worker(self.coordinator_hosts, self.role_id, - self.trainer_endpoint) + self._worker.init_fl_worker( + self.coordinator_hosts, self.role_id, self.trainer_endpoint + ) - if self.context[ - 'ps_mode'] == DistributedMode.GEO or self.is_heter_ps_mode: + if ( + self.context['ps_mode'] == DistributedMode.GEO + or self.is_heter_ps_mode + ): self._communicator = Communicator( - trainer_config.mode, kwargs, - trainer_config.get_communicator_flags()) - self._communicator.init_with_ctx(send_ctx, dense_map, worker_desc, - self.string_hosts, - fluid.global_scope()) + trainer_config.mode, + kwargs, + trainer_config.get_communicator_flags(), + ) + self._communicator.init_with_ctx( + send_ctx, + dense_map, + worker_desc, + self.string_hosts, + fluid.global_scope(), + ) fleet.util.barrier() # info = self._communicator.get_client_info() info = self._worker.get_client_info() if isinstance(info, list) and len(info) > 0: all_info = self.role_maker._all_gather( - info[0]) # 收集其他 client 的 service 地址 + info[0] + ) # 收集其他 client 的 service 地址 # for unittest if not isinstance(all_info, list): warnings.warn("gloo may not initialize correctly") @@ -1143,8 +1279,10 @@ class TheOnePSRuntime(RuntimeBase): self.scopes = scopes if not is_test: - if self.context[ - 'ps_mode'] == DistributedMode.GEO or self.is_heter_ps_mode == True: + if ( + self.context['ps_mode'] == DistributedMode.GEO + or self.is_heter_ps_mode == True + ): self._communicator.init_params(dense_map) else: if not self.context['use_ps_gpu']: @@ -1158,8 +1296,10 @@ class TheOnePSRuntime(RuntimeBase): self._pull_all_dense(scopes, send_ctx, dense_map) fleet.util.barrier() - if self.context[ - 'ps_mode'] == DistributedMode.GEO or self.is_heter_ps_mode == True: + if ( + self.context['ps_mode'] == DistributedMode.GEO + or self.is_heter_ps_mode == True + ): if not self._communicator.is_running(): self._communicator.start() else: @@ -1169,8 +1309,10 @@ class TheOnePSRuntime(RuntimeBase): launch_barrier_flag = int(os.getenv("FLAGS_LAUNCH_BARRIER", "1")) if launch_barrier and launch_barrier_flag: wait_server_ready(self.role_maker._get_pserver_endpoints()) - if self.is_heter_ps_mode and self.role_maker._get_next_trainers( - ) != []: + if ( + self.is_heter_ps_mode + and self.role_maker._get_next_trainers() != [] + ): wait_server_ready(self.role_maker._get_next_trainers()) if self.is_heter_ps_mode: previous_trainers = [] @@ -1180,8 +1322,8 @@ class TheOnePSRuntime(RuntimeBase): if self.role_maker._get_next_trainers() != []: next_trainers = self.role_maker._get_next_trainers() self._heter_client = HeterClient( - next_trainers, previous_trainers, - self.role_maker._role_id()) # --> HeterClient::GetInstance + next_trainers, previous_trainers, self.role_maker._role_id() + ) # --> HeterClient::GetInstance def _init_coordinator(self, scopes=None): if self._coordinator == None: @@ -1189,12 +1331,13 @@ class TheOnePSRuntime(RuntimeBase): print(">>> curr node ip: {}".format(self.coordinator_hosts[0])) print(">>> all trainer endpoints: {}".format(self.trainer_endpoints)) - self._coordinator.start_coordinator(self.coordinator_hosts[0], - self.trainer_endpoints) + self._coordinator.start_coordinator( + self.coordinator_hosts[0], self.trainer_endpoints + ) def _make_fl_strategy(self): if self._coordinator == None: - assert ("Coordinator py object is null!") + assert "Coordinator py object is null!" else: self._coordinator.make_fl_strategy() @@ -1208,12 +1351,18 @@ class TheOnePSRuntime(RuntimeBase): print("server_desc: \n{}".format(server_desc)) self._server = fluid.core.DistFleetWrapper() - self._server.init_server(server_desc, self.string_hosts, self.role_id, - trainers, self._server_sub_program) + self._server.init_server( + server_desc, + self.string_hosts, + self.role_id, + trainers, + self._server_sub_program, + ) dist_varnames = get_sparse_tablenames(self.origin_main_programs, True) - sparse_varnames = get_sparse_tablenames(self.origin_main_programs, - False) + sparse_varnames = get_sparse_tablenames( + self.origin_main_programs, False + ) distributed_varnames = dist_varnames + sparse_varnames @@ -1223,8 +1372,10 @@ class TheOnePSRuntime(RuntimeBase): for var_name in var_names: if var_name not in distributed_varnames: raise ValueError( - "fleet.init server can only load sparse variables in {}" - .format(distributed_varnames)) + "fleet.init server can only load sparse variables in {}".format( + distributed_varnames + ) + ) load_varnames = var_names if dirname is None or not load_varnames: @@ -1249,17 +1400,19 @@ class TheOnePSRuntime(RuntimeBase): self._communicator.stop() self._worker.stop_worker() if self.is_heter_ps_mode: - assert self._heter_client != None, "heter client should not be None in heterps mode" + assert ( + self._heter_client != None + ), "heter client should not be None in heterps mode" self._heter_client.stop() @staticmethod def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False from .utils.public import _get_varname_parts + origin_varname, _, _ = _get_varname_parts(var.name) if origin_varname.endswith("@GRAD"): return False @@ -1267,9 +1420,11 @@ class TheOnePSRuntime(RuntimeBase): if origin_varname.startswith("learning_rate_"): return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -1282,35 +1437,35 @@ class TheOnePSRuntime(RuntimeBase): model_path = os.path.join(dirname, "dnn_plugin") return model_path - def _ps_save_dense_params(self, - executor, - dirname, - scope, - program, - var_names=None): + def _ps_save_dense_params( + self, executor, dirname, scope, program, var_names=None + ): dense_map = get_the_one_recv_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) send_ctx = get_the_one_send_context( self.context, split_dense_table=self.is_heter_ps_mode, - ep_list=self.endpoints) + ep_list=self.endpoints, + ) if program is None or len(self.origin_main_programs) == 1: program = self.origin_main_programs[0] dense_var_names = self._pull_dense(program, scope, send_ctx, dense_map) save_var_names = dense_var_names if var_names is None else var_names vars = [program.global_block().var(i) for i in save_var_names] import paddle + with paddle.static.scope_guard(scope): - paddle.static.save_vars(executor, - "./", - program, - vars=vars, - filename=dirname) - - def _save_sparse_params(self, executor, dirname, context, main_program, - mode): - distributed_varnames = get_sparse_tablenames(self.origin_main_programs, - True) + paddle.static.save_vars( + executor, "./", program, vars=vars, filename=dirname + ) + + def _save_sparse_params( + self, executor, dirname, context, main_program, mode + ): + distributed_varnames = get_sparse_tablenames( + self.origin_main_programs, True + ) values = [] model_path = self._get_inference_model_path(dirname) for id, names in context.items(): @@ -1326,12 +1481,9 @@ class TheOnePSRuntime(RuntimeBase): # self._worker.save_all_model(dirname, mode) return values - def _save_distributed_persistables(self, - executor, - dirname, - main_program=None, - mode=0, - **kwargs): + def _save_distributed_persistables( + self, executor, dirname, main_program=None, mode=0, **kwargs + ): """ This function filters out all variables with `persistable==True` from the give `main_program` and then saves these variables to the folder `dirname` @@ -1350,7 +1502,8 @@ class TheOnePSRuntime(RuntimeBase): if not isinstance(executor, Executor): raise TypeError( - "in fleet.save() function, executor must be as Executor type") + "in fleet.save() function, executor must be as Executor type" + ) if main_program is None: main_program = self.context['origin_main_program'] @@ -1362,14 +1515,16 @@ class TheOnePSRuntime(RuntimeBase): self._worker.save_all_model(dirname, mode) - def _ps_inference_save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True, - mode=0): + def _ps_inference_save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + mode=0, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. @@ -1382,11 +1537,16 @@ class TheOnePSRuntime(RuntimeBase): if not isinstance(executor, Executor): raise TypeError( - "in fleet.save() function, executor must be as Executor type") + "in fleet.save() function, executor must be as Executor type" + ) import paddle - program = self.origin_main_programs[ - 0] if main_program is None else main_program + + program = ( + self.origin_main_programs[0] + if main_program is None + else main_program + ) _, _, idx = get_program_by_id(self.context, id(program)) scope = self.scopes[idx] print("save inference model scope idx:", idx) @@ -1400,8 +1560,9 @@ class TheOnePSRuntime(RuntimeBase): program.global_block().var(name) for name in feeded_var_names ] - infer_program = paddle.static.normalize_program(program, feed_vars, - target_vars) + infer_program = paddle.static.normalize_program( + program, feed_vars, target_vars + ) infer_program._copy_dist_param_info_from(program) @@ -1413,30 +1574,40 @@ class TheOnePSRuntime(RuntimeBase): sparses = get_the_one_recv_context( self.context, is_dense=False, - split_dense_table=self.is_heter_ps_mode) - sparse_names = self._save_sparse_params(executor, dirname, sparses, - main_program, mode) + split_dense_table=self.is_heter_ps_mode, + ) + sparse_names = self._save_sparse_params( + executor, dirname, sparses, main_program, mode + ) dense_map = get_the_one_recv_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) send_ctx = get_the_one_send_context( self.context, split_dense_table=self.is_heter_ps_mode, - ep_list=self.endpoints) + ep_list=self.endpoints, + ) self._pull_dense(program, scope, send_ctx, dense_map) generate_vars = self.context[ - "user_defined_strategy"].trainer_desc_configs["stat_var_names"] + "user_defined_strategy" + ].trainer_desc_configs["stat_var_names"] generate_vars = [var for var in generate_vars] remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(sparse_names), - infer_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(sparse_names), + infer_program.list_vars(), + ) + ) for var in remaining_vars: tensor = var.get_value(scope) - paddle.save(tensor, - os.path.join(model_path, var.name), - use_binary_format=True) + paddle.save( + tensor, + os.path.join(model_path, var.name), + use_binary_format=True, + ) def _save_cache_model(self, dirname, **kwargs): mode = kwargs.get("mode", 1) @@ -1447,7 +1618,7 @@ class TheOnePSRuntime(RuntimeBase): if self.role_maker._is_first_worker(): cache_threshold = self._worker.get_cache_threshold(table_id) - #check cache threshold right or not + # check cache threshold right or not fleet.util.barrier() if self.role_maker._is_first_worker(): @@ -1469,8 +1640,9 @@ class TheOnePSRuntime(RuntimeBase): fleet.util.barrier() def _load_sparse_params(self, dirname, context, main_program, mode): - distributed_varnames = get_sparse_tablenames(self.origin_main_programs, - True) + distributed_varnames = get_sparse_tablenames( + self.origin_main_programs, True + ) values = [] for id, names in context.items(): if names[0] not in distributed_varnames: @@ -1481,12 +1653,14 @@ class TheOnePSRuntime(RuntimeBase): values.extend(names) return values - def _ps_inference_load_inference_model(self, - dirname, - mode=0, - main_program=None): - main_program = self.origin_main_programs[ - 0] if main_program is None else main_program + def _ps_inference_load_inference_model( + self, dirname, mode=0, main_program=None + ): + main_program = ( + self.origin_main_programs[0] + if main_program is None + else main_program + ) _, _, idx = get_program_by_id(self.context, id(main_program)) scope = self.scopes[idx] print("load inference model scope idx:", idx) @@ -1499,17 +1673,21 @@ class TheOnePSRuntime(RuntimeBase): sparses = get_the_one_recv_context( self.context, is_dense=False, - split_dense_table=self.is_heter_ps_mode) + split_dense_table=self.is_heter_ps_mode, + ) - sparse_varnames = self._load_sparse_params(dirname, sparses, - main_program, mode) + sparse_varnames = self._load_sparse_params( + dirname, sparses, main_program, mode + ) dense_map = get_the_one_recv_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) send_ctx = get_the_one_send_context( self.context, split_dense_table=self.is_heter_ps_mode, - ep_list=self.endpoints) + ep_list=self.endpoints, + ) recv_dense_varnames = [] for _, names in dense_map.items(): @@ -1518,11 +1696,15 @@ class TheOnePSRuntime(RuntimeBase): loaded_varnames = sparse_varnames remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(loaded_varnames), - main_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(loaded_varnames), + main_program.list_vars(), + ) + ) model_path = self._get_inference_model_path(dirname) import paddle + for var in remaining_vars: if var.name not in recv_dense_varnames: continue @@ -1586,8 +1768,8 @@ class TheOnePSRuntime(RuntimeBase): sparses = get_the_one_recv_context( self.context, is_dense=False, - split_dense_table=self.role_maker. - _is_heter_parameter_server_mode) + split_dense_table=self.role_maker._is_heter_parameter_server_mode, + ) for id, names in sparses.items(): self._worker.shrink_sparse_table(id, threshold) diff --git a/python/paddle/distributed/ps/utils/ps_factory.py b/python/paddle/distributed/ps/utils/ps_factory.py index 1f522ec27f8..0726fe15dc4 100755 --- a/python/paddle/distributed/ps/utils/ps_factory.py +++ b/python/paddle/distributed/ps/utils/ps_factory.py @@ -16,14 +16,18 @@ from .ps_program_builder import * # noqa: F403 from .public import * # noqa: F403 __all__ = [ - 'PsProgramBuilder', 'GeoPsProgramBuilder', 'CpuSyncPsProgramBuilder', - 'CpuAsyncPsProgramBuilder', 'GpuPsProgramBuilder', - 'HeterAsyncPsProgramBuilder', 'FlPsProgramBuilder', 'NuPsProgramBuilder' + 'PsProgramBuilder', + 'GeoPsProgramBuilder', + 'CpuSyncPsProgramBuilder', + 'CpuAsyncPsProgramBuilder', + 'GpuPsProgramBuilder', + 'HeterAsyncPsProgramBuilder', + 'FlPsProgramBuilder', + 'NuPsProgramBuilder', ] class PsProgramBuilderFactory(object): - def __init__(self): pass diff --git a/python/paddle/distributed/ps/utils/public.py b/python/paddle/distributed/ps/utils/public.py index c5835c730ee..75182a49721 100755 --- a/python/paddle/distributed/ps/utils/public.py +++ b/python/paddle/distributed/ps/utils/public.py @@ -22,9 +22,9 @@ import paddle.fluid as fluid from paddle.fluid import core import paddle.fluid.framework as framework -#logging.basicConfig( +# logging.basicConfig( # format='%(levelname)s - %(asctime)s - %(pathname)s: %(lineno)s - %(message)s', level=logging.INFO) -#logger = logging.getLogger(__name__) +# logger = logging.getLogger(__name__) OP_NAME_SCOPE = "op_namescope" CLIP_OP_NAME_SCOPE = "gradient_clip" @@ -47,7 +47,7 @@ SPARSE_OP_LIST = ["lookup_table", "lookup_table_v2"] SPARSE_OP_TYPE_DICT = {"lookup_table": "W", "lookup_table_v2": "W"} SPARSE_GRAD_OP_TYPE_DICT = { "lookup_table_grad": "W", - "lookup_table_v2_grad": "W" + "lookup_table_v2_grad": "W", } DEFAULT_DEVICE = 'cpu' @@ -58,13 +58,13 @@ DATA_NORM_GRAD_NAME = [x + "@GRAD" for x in DATA_NORM_NAME] def logger_config(log_path, logging_name): logger = logging.getLogger(logging_name) logger.setLevel(level=logging.WARNING) - handler = logging.FileHandler(log_path, - mode='a', - encoding='UTF-8', - delay=True) + handler = logging.FileHandler( + log_path, mode='a', encoding='UTF-8', delay=True + ) handler.setLevel(logging.INFO) formatter = logging.Formatter( - '%(levelname)s - %(asctime)s - %(pathname)s: %(lineno)s - %(message)s') + '%(levelname)s - %(asctime)s - %(pathname)s: %(lineno)s - %(message)s' + ) handler.setFormatter(formatter) console = logging.StreamHandler() console.setLevel(logging.DEBUG) @@ -74,8 +74,9 @@ def logger_config(log_path, logging_name): ps_log_root_dir = './ps_log/' -logger = logger_config(log_path='./ps_usr_print_log', - logging_name='ps_usr_print_log') +logger = logger_config( + log_path='./ps_usr_print_log', logging_name='ps_usr_print_log' +) class DistributedMode: @@ -88,7 +89,6 @@ class DistributedMode: class TrainerRuntimeConfig(object): - def __init__(self, valid_strategy): self.mode = None num_threads = os.getenv("CPU_NUM", "1") @@ -107,21 +107,28 @@ class TrainerRuntimeConfig(object): self.runtime_configs = {} self.runtime_configs['communicator_max_merge_var_num'] = os.getenv( - "FLAGS_communicator_max_merge_var_num", send_queue_size) + "FLAGS_communicator_max_merge_var_num", send_queue_size + ) self.runtime_configs['communicator_send_queue_size'] = os.getenv( - "FLAGS_communicator_send_queue_size", send_queue_size) + "FLAGS_communicator_send_queue_size", send_queue_size + ) self.runtime_configs[ - 'communicator_independent_recv_thread'] = os.getenv( - "FLAGS_communicator_independent_recv_thread", "1") + 'communicator_independent_recv_thread' + ] = os.getenv("FLAGS_communicator_independent_recv_thread", "1") self.runtime_configs[ - 'communicator_min_send_grad_num_before_recv'] = os.getenv( - "FLAGS_communicator_min_send_grad_num_before_recv", num_threads) + 'communicator_min_send_grad_num_before_recv' + ] = os.getenv( + "FLAGS_communicator_min_send_grad_num_before_recv", num_threads + ) self.runtime_configs['communicator_thread_pool_size'] = os.getenv( - "FLAGS_communicator_thread_pool_size", "5") + "FLAGS_communicator_thread_pool_size", "5" + ) self.runtime_configs['communicator_send_wait_times'] = os.getenv( - "FLAGS_communicator_send_wait_times", "5") + "FLAGS_communicator_send_wait_times", "5" + ) self.runtime_configs['communicator_is_sgd_optimizer'] = os.getenv( - "FLAGS_communicator_is_sgd_optimizer", "1") + "FLAGS_communicator_is_sgd_optimizer", "1" + ) def get_communicator_flags(self): need_keys = [] @@ -130,46 +137,62 @@ class TrainerRuntimeConfig(object): if self.mode is None or self.mode == DistributedMode.ASYNC: need_keys = self.runtime_configs.keys() mode_str = "async" - elif self.mode == DistributedMode.SYNC or self.mode == DistributedMode.HALF_ASYNC: + elif ( + self.mode == DistributedMode.SYNC + or self.mode == DistributedMode.HALF_ASYNC + ): mode_str = "sync or half_async" need_keys = [ 'communicator_max_merge_var_num', - 'communicator_send_wait_times', 'communicator_thread_pool_size', - 'communicator_send_queue_size' + 'communicator_send_wait_times', + 'communicator_thread_pool_size', + 'communicator_send_queue_size', ] elif self.mode == DistributedMode.GEO: mode_str = "GEO" need_keys = [ - 'communicator_thread_pool_size', 'communicator_send_wait_times', - 'communicator_max_merge_var_num', 'communicator_send_queue_size' + 'communicator_thread_pool_size', + 'communicator_send_wait_times', + 'communicator_max_merge_var_num', + 'communicator_send_queue_size', ] else: raise ValueError("Unsupported Mode") - if self.mode == DistributedMode.SYNC or self.mode == DistributedMode.HALF_ASYNC: + if ( + self.mode == DistributedMode.SYNC + or self.mode == DistributedMode.HALF_ASYNC + ): max_merge_var_num = self.runtime_configs[ - 'communicator_max_merge_var_num'] + 'communicator_max_merge_var_num' + ] send_queue_size = self.runtime_configs[ - 'communicator_send_queue_size'] + 'communicator_send_queue_size' + ] if max_merge_var_num != num_threads: print( 'WARNING: In {} mode, communicator_max_merge_var_num ' 'must be equal to CPU_NUM. But received, ' 'communicator_max_merge_var_num = {}, CPU_NUM = ' - '{}. communicator_max_merge_var_num will be forced to {}.'. - format(mode_str, max_merge_var_num, num_threads, - num_threads)) + '{}. communicator_max_merge_var_num will be forced to {}.'.format( + mode_str, max_merge_var_num, num_threads, num_threads + ) + ) self.runtime_configs[ - 'communicator_max_merge_var_num'] = num_threads + 'communicator_max_merge_var_num' + ] = num_threads if send_queue_size != num_threads: - print('WARNING: In {} mode, communicator_send_queue_size ' - 'must be equal to CPU_NUM. But received, ' - 'communicator_send_queue_size = {}, CPU_NUM = ' - '{}. communicator_send_queue_size will be forced to {}.'. - format(mode_str, send_queue_size, num_threads, - num_threads)) + print( + 'WARNING: In {} mode, communicator_send_queue_size ' + 'must be equal to CPU_NUM. But received, ' + 'communicator_send_queue_size = {}, CPU_NUM = ' + '{}. communicator_send_queue_size will be forced to {}.'.format( + mode_str, send_queue_size, num_threads, num_threads + ) + ) self.runtime_configs[ - 'communicator_send_queue_size'] = num_threads + 'communicator_send_queue_size' + ] = num_threads return dict((key, str(self.runtime_configs[key])) for key in need_keys) @@ -178,9 +201,9 @@ def get_lr_ops(program): lr_ops = [] for index, op in enumerate(program.global_block().ops): role_id = int(op.attr(RPC_OP_ROLE_ATTR_NAME)) - if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or \ - role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) | \ - int(OPT_OP_ROLE_ATTR_VALUE): + if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or role_id == int( + LR_SCHED_OP_ROLE_ATTR_VALUE + ) | int(OPT_OP_ROLE_ATTR_VALUE): lr_ops.append(op) return lr_ops @@ -190,16 +213,20 @@ def get_optimize_ops(_program, remote_sparse=[]): opt_ops = [] for op in block.ops: if _is_opt_role_op(op): - if len(remote_sparse) > 0 and op.input( - "Param" - )[0] not in remote_sparse: # for fl: only delete remote sparse optimize + if ( + len(remote_sparse) > 0 + and op.input("Param")[0] not in remote_sparse + ): # for fl: only delete remote sparse optimize continue # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs() \ - and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE): + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + ): op._set_attr( "op_role", - int(core.op_proto_and_checker_maker.OpRole.Backward)) + int(core.op_proto_and_checker_maker.OpRole.Backward), + ) continue opt_ops.append(op) return opt_ops @@ -228,7 +255,7 @@ def get_dist_env(): 'trainer_id': trainer_id, 'num_trainers': num_trainers, 'current_endpoint': current_endpoint, - 'trainer_endpoints': trainer_endpoints + 'trainer_endpoints': trainer_endpoints, } @@ -276,8 +303,10 @@ def is_distributed_sparse_op(op): if op.type in SPARSE_OP_LIST and op.attr('is_distributed') is True: return True - if op.type == "distributed_lookup_table" and op.attr( - 'is_distributed') is True: + if ( + op.type == "distributed_lookup_table" + and op.attr('is_distributed') is True + ): return True return False @@ -288,12 +317,17 @@ def get_sparse_tablename(op): def is_sparse_op(op): - if op.type in SPARSE_OP_LIST and op.attr('is_sparse') is True and op.attr( - 'is_distributed') is False: + if ( + op.type in SPARSE_OP_LIST + and op.attr('is_sparse') is True + and op.attr('is_distributed') is False + ): return True - if op.type == "distributed_lookup_table" and op.attr( - 'is_distributed') is False: + if ( + op.type == "distributed_lookup_table" + and op.attr('is_distributed') is False + ): return True return False @@ -320,12 +354,14 @@ def get_trainers(role_maker): return role_maker.worker_num() -def get_dense_send_context(program, - send_ctx, - idx, - merged_dense_pairs, - trainer_id, - split_dense_table=False): +def get_dense_send_context( + program, + send_ctx, + idx, + merged_dense_pairs, + trainer_id, + split_dense_table=False, +): if len(merged_dense_pairs) < 1: return idx if not split_dense_table: @@ -356,10 +392,23 @@ def get_dense_send_context(program, # print("public get_dense_send_context dense_table:", grad_name, # var_numel, origin_varnames) from paddle.fluid.core import CommContext - dense_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], origin_varnames, trainer_id, - aggregate, False, False, idx, False, False, - id(program), []) + + dense_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + origin_varnames, + trainer_id, + aggregate, + False, + False, + idx, + False, + False, + id(program), + [], + ) send_ctx[grad_name] = dense_ctx idx += 1 @@ -379,10 +428,23 @@ def get_dense_send_context(program, # print("public get_dense_send_context data_norm table:", grad_name, # var_numel, origin_varnames) from paddle.fluid.core import CommContext - data_norm_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], origin_varnames, trainer_id, - aggregate, False, False, idx, False, True, - id(program), []) + + data_norm_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + origin_varnames, + trainer_id, + aggregate, + False, + False, + idx, + False, + True, + id(program), + [], + ) send_ctx[grad_name] = data_norm_ctx idx += 1 else: @@ -394,10 +456,23 @@ def get_dense_send_context(program, grad_name = origin_varname aggregate = True from paddle.fluid.core import CommContext - dense_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], [origin_varname], trainer_id, - aggregate, False, False, idx, False, False, - id(program), []) + + dense_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + [origin_varname], + trainer_id, + aggregate, + False, + False, + idx, + False, + False, + id(program), + [], + ) send_ctx[grad_name] = dense_ctx idx += 1 return idx @@ -405,8 +480,10 @@ def get_dense_send_context(program, def get_geo_trainer_send_context(attrs): if attrs['ps_mode'] != DistributedMode.GEO: - raise ValueError("ps mode: {} not matched {}", - format(ps_mode, "get_geo_trainer_send_context")) + raise ValueError( + "ps mode: {} not matched {}", + format(ps_mode, "get_geo_trainer_send_context"), + ) send_ctx = {} trainer_id = get_role_id(attrs['role_maker']) origin_programs = attrs['origin_main_programs'] @@ -422,16 +499,32 @@ def get_geo_trainer_send_context(attrs): if param_name in attrs['remote_sparse']: # for recall/ncf model continue - is_distributed = True if param_name in distibuted_varnames else False + is_distributed = ( + True if param_name in distibuted_varnames else False + ) var = program.global_block().vars[grad.merged_var.name] var_numel = reduce(lambda x, y: x * y, var.shape[1:]) from paddle.fluid.core import CommContext - print("public get_the_geo_send_context sparse: ", grad_name, - var_numel) - sparse_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], [grad_name], trainer_id, True, - True, is_distributed, idx, False, False, - id(program), []) + + print( + "public get_the_geo_send_context sparse: ", grad_name, var_numel + ) + sparse_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + [grad_name], + trainer_id, + True, + True, + is_distributed, + idx, + False, + False, + id(program), + [], + ) idx += 1 send_ctx[sparse_ctx.var_name()] = sparse_ctx @@ -452,8 +545,23 @@ def _step_ctx(idx, role_maker): sections = [1] * len(endpoints) names = [name] * len(endpoints) from paddle.fluid.core import CommContext - ctx = CommContext(name, names, endpoints, sections, [name], trainer_id, - True, False, False, idx, True, False, -1, []) + + ctx = CommContext( + name, + names, + endpoints, + sections, + [name], + trainer_id, + True, + False, + False, + idx, + True, + False, + -1, + [], + ) return name, ctx @@ -483,7 +591,9 @@ def get_the_one_send_context(attrs, split_dense_table=False, ep_list=None): for i in range(len(ep_list)): splited_varname.append("{}.block{}".format(param_name, i)) - is_distributed = True if param_name in distibuted_varnames else False + is_distributed = ( + True if param_name in distibuted_varnames else False + ) var = program.global_block().vars[grad.merged_var.name] @@ -493,20 +603,43 @@ def get_the_one_send_context(attrs, split_dense_table=False, ep_list=None): if grad_name in send_ctx: continue from paddle.fluid.core import CommContext - print("public get_the_one_send_context sparse: ", grad_name, - splited_varname, shape) - sparse_ctx = CommContext(grad_name, splited_varname, ep_list, shape, - [grad_name], trainer_id, True, True, - is_distributed, idx, False, False, - id(program), remote_sparse_ids) + + print( + "public get_the_one_send_context sparse: ", + grad_name, + splited_varname, + shape, + ) + sparse_ctx = CommContext( + grad_name, + splited_varname, + ep_list, + shape, + [grad_name], + trainer_id, + True, + True, + is_distributed, + idx, + False, + False, + id(program), + remote_sparse_ids, + ) idx += 1 send_ctx[sparse_ctx.var_name()] = sparse_ctx for i, program in enumerate(origin_programs): merged_dense_pairs = attrs['merged_dense_pairs'][i] - idx = get_dense_send_context(program, send_ctx, idx, merged_dense_pairs, - trainer_id, split_dense_table) + idx = get_dense_send_context( + program, + send_ctx, + idx, + merged_dense_pairs, + trainer_id, + split_dense_table, + ) if len(attrs['tensor_table']) > 0 and attrs['is_worker']: name, ctx = _step_ctx(idx, attrs['role_maker']) @@ -517,8 +650,11 @@ def get_the_one_send_context(attrs, split_dense_table=False, ep_list=None): def find_heter_ops(program, default_device="cpu"): if default_device not in DEVICE_LIST: - raise ValueError("Given device {} is not in device list {}".format( - default_device, DEVICE_LIST)) + raise ValueError( + "Given device {} is not in device list {}".format( + default_device, DEVICE_LIST + ) + ) def _is_heter_op(op, current_heter_device, default_device="cpu"): heter_devices = list(DEVICE_LIST) @@ -527,10 +663,13 @@ def find_heter_ops(program, default_device="cpu"): op_type = op.type if op_device in heter_devices: return True - elif op_type in COMMUNICATE_OPS_TYPE and current_heter_device != default_device: + elif ( + op_type in COMMUNICATE_OPS_TYPE + and current_heter_device != default_device + ): # for distributed communciate ops: send & recv & barrier etc. # Todo: need update this method - #op._set_attr('op_device', current_heter_device) + # op._set_attr('op_device', current_heter_device) return True elif op_device == None or op_device == default_device: op._set_attr('op_device', default_device) @@ -565,16 +704,20 @@ def find_heter_ops(program, default_device="cpu"): op = op_list[i] if "_grad" in op.type: forward_op_type = op.type.split("_grad")[0] - if forward_op_type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + forward_op_type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[forward_op_type])[0] if param_name in var2idx: ## insert sum op & remove sum op from var2idx and origin place op_list = list(block.ops) sum_op = op_list[var2idx[param_name]] sum_op_inputs = { - sum_op.input_names[0]: - [block.vars[input] for input in sum_op.input_arg_names] + sum_op.input_names[0]: [ + block.vars[input] + for input in sum_op.input_arg_names + ] } sum_op_outputs = { sum_op.output_names[0]: [ @@ -582,11 +725,13 @@ def find_heter_ops(program, default_device="cpu"): for output in sum_op.output_arg_names ] } - block._insert_op(index=i + 1, - type=sum_op.type, - inputs=sum_op_inputs, - outputs=sum_op_outputs, - attrs=sum_op.all_attrs()) + block._insert_op( + index=i + 1, + type=sum_op.type, + inputs=sum_op_inputs, + outputs=sum_op_outputs, + attrs=sum_op.all_attrs(), + ) block._remove_op(var2idx[param_name] + 1) var2idx.pop(param_name) for var_ in var2idx: @@ -607,9 +752,9 @@ def find_heter_ops(program, default_device="cpu"): for no_grad_var in output_vars_no_grad: if no_grad_var in var2idx: """ - insert sum op & remove sum op from var2idx and origin place + insert sum op & remove sum op from var2idx and origin place - """ + """ op_list = list(block.ops) sum_op = op_list[var2idx[no_grad_var]] sum_op_inputs = { @@ -624,11 +769,13 @@ def find_heter_ops(program, default_device="cpu"): for output in sum_op.output_arg_names ] } - block._insert_op(index=i + 1, - type=sum_op.type, - inputs=sum_op_inputs, - outputs=sum_op_outputs, - attrs=sum_op.all_attrs()) + block._insert_op( + index=i + 1, + type=sum_op.type, + inputs=sum_op_inputs, + outputs=sum_op_outputs, + attrs=sum_op.all_attrs(), + ) block._remove_op(var2idx[no_grad_var] + 1) var2idx.pop(no_grad_var) for var_ in var2idx: @@ -641,12 +788,16 @@ def find_heter_ops(program, default_device="cpu"): pre_op = op_list[i - 1] if "_grad" in pre_op.type: forward_op_type = pre_op.type.split("_grad")[0] - if forward_op_type in SPARSE_OP_TYPE_DICT.keys() \ - and pre_op.attr('remote_prefetch') is True: + if ( + forward_op_type in SPARSE_OP_TYPE_DICT.keys() + and pre_op.attr('remote_prefetch') is True + ): param_name = pre_op.input( - SPARSE_OP_TYPE_DICT[forward_op_type])[0] + SPARSE_OP_TYPE_DICT[forward_op_type] + )[0] if param_name == origin_var and op.attr( - "op_device") == pre_op.attr("op_device"): + "op_device" + ) == pre_op.attr("op_device"): continue else: var2idx[origin_var] = i @@ -699,7 +850,8 @@ def find_heter_ops(program, default_device="cpu"): # for cpu-op block append if len(current_default_block_ops) > 1: default_ops[default_device][ - block_index] = current_default_block_ops + block_index + ] = current_default_block_ops program_block_ops.append(current_default_block_ops) current_default_block_ops = [] block_index += 1 @@ -755,8 +907,10 @@ def find_heter_ops(program, default_device="cpu"): for _, heter_block in heter_block_dict.items(): total_heter_ops += len(heter_block) print( - "There are {} OPs in your main_program, and contains {} heter-OPs which is made up of {} heter-blocks." - .format(len(block.ops), total_heter_ops, heter_blocks)) + "There are {} OPs in your main_program, and contains {} heter-OPs which is made up of {} heter-blocks.".format( + len(block.ops), total_heter_ops, heter_blocks + ) + ) return origin_porgram, heter_ops, default_ops, program_block_ops @@ -773,11 +927,14 @@ def union_forward_gradient_op(program_block_ops_list): """ block_length = len(program_block_ops_list) union_program_block_ops_list = [] - assert block_length % 2 != 0, "the length of program_block_ops_list should be odd" + assert ( + block_length % 2 != 0 + ), "the length of program_block_ops_list should be odd" for i in range(0, block_length // 2): block_op_list = {"forward": program_block_ops_list[i]} block_op_list.update( - {"backward": program_block_ops_list[block_length - 1 - i]}) + {"backward": program_block_ops_list[block_length - 1 - i]} + ) union_program_block_ops_list.append(block_op_list) block_op_list = {"forward": [], "backward": []} @@ -791,13 +948,15 @@ def union_forward_gradient_op(program_block_ops_list): def find_block_joints(program, program_block_ops_list, heter_ops): - block_var_detail = find_entrance_exit_private(program, - program_block_ops_list) - block_var_detail = entrance_exit_check(program, program_block_ops_list, - block_var_detail, heter_ops) - block_var_detail = delete_block_useless_exit(program, - program_block_ops_list, - block_var_detail) + block_var_detail = find_entrance_exit_private( + program, program_block_ops_list + ) + block_var_detail = entrance_exit_check( + program, program_block_ops_list, block_var_detail, heter_ops + ) + block_var_detail = delete_block_useless_exit( + program, program_block_ops_list, block_var_detail + ) return block_var_detail @@ -822,9 +981,11 @@ def find_entrance_exit_private(program, program_block_ops_list): for index, block_op_list in enumerate(program_block_ops_list): ## forward block_input, block_output = find_ops_list_input_output( - program, block_op_list["forward"]) + program, block_op_list["forward"] + ) persistables = screen_persistables( - program, block_input) + screen_persistables(program, block_output) + program, block_input + ) + screen_persistables(program, block_output) # find entrance & exit block_private_vars = list(set(block_input) & set(block_output)) block_entrance = list(set(block_input) - set(block_private_vars)) @@ -834,35 +995,40 @@ def find_entrance_exit_private(program, program_block_ops_list): "entrance": block_entrance, "exit": block_exit, "private": block_private_vars, - "persistables": persistables + "persistables": persistables, } } ## backward bp_block_input, bp_block_output = find_ops_list_input_output( - program, block_op_list["backward"]) + program, block_op_list["backward"] + ) bp_persistables = screen_persistables( - program, bp_block_input) + screen_persistables( - program, bp_block_output) + program, bp_block_input + ) + screen_persistables(program, bp_block_output) # find entrance & exit bp_block_private_vars = list(set(bp_block_input) & set(bp_block_output)) bp_block_entrance = list( - set(bp_block_input) - set(bp_block_private_vars)) + set(bp_block_input) - set(bp_block_private_vars) + ) bp_block_exit = list(set(bp_block_output) - set(bp_block_private_vars)) - detail.update({ - "backward": { - "entrance": bp_block_entrance, - "exit": bp_block_exit, - "private": bp_block_private_vars, - "persistables": bp_persistables + detail.update( + { + "backward": { + "entrance": bp_block_entrance, + "exit": bp_block_exit, + "private": bp_block_private_vars, + "persistables": bp_persistables, + } } - }) + ) block_var_detail.append(detail) return block_var_detail -def entrance_exit_check(program, program_block_ops_list, block_var_detail, - heter_ops): +def entrance_exit_check( + program, program_block_ops_list, block_var_detail, heter_ops +): for index in range(len(block_var_detail) - 1, -1, -1): if index - 1 < 0: break @@ -872,9 +1038,11 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, backward_entrance = block_var_detail[index]["backward"]["entrance"] - forward_all = block_var_detail[index]["forward"][ - "entrance"] + block_var_detail[index]["forward"][ - "private"] + block_var_detail[index]["forward"]["exit"] + forward_all = ( + block_var_detail[index]["forward"]["entrance"] + + block_var_detail[index]["forward"]["private"] + + block_var_detail[index]["forward"]["exit"] + ) for var in backward_entrance: if not ("@GRAD" in var) and not (var in forward_all): @@ -885,18 +1053,24 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, if previous_block_exit == current_block_entrance: continue exist_vars = list( - set(previous_block_exit) & set(current_block_entrance)) + set(previous_block_exit) & set(current_block_entrance) + ) need_add_vars = list(set(current_block_entrance) - set(exist_vars)) # var in different stage should not be ignored, since they are not placed in the same program & device - #need_add_vars = find_need_var_from_previous_block( + # need_add_vars = find_need_var_from_previous_block( # need_add_vars, block_var_detail, index, heter_ops) - previous_block_private = block_var_detail[index - - 1]["forward"]["private"] - previous_block_entrance = block_var_detail[index - - 1]["forward"]["entrance"] + previous_block_private = block_var_detail[index - 1]["forward"][ + "private" + ] + previous_block_entrance = block_var_detail[index - 1]["forward"][ + "entrance" + ] for var in need_add_vars: - if var not in previous_block_private and var not in previous_block_entrance: + if ( + var not in previous_block_private + and var not in previous_block_entrance + ): previous_block_entrance.append(var) previous_block_exit.append(var) if var not in current_block_entrance: @@ -912,27 +1086,35 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, if previous_block_exit == current_block_entrance: continue exist_vars = list( - set(previous_block_exit) & set(current_block_entrance)) + set(previous_block_exit) & set(current_block_entrance) + ) need_add_vars = list(set(current_block_entrance) - set(exist_vars)) need_ignore_vars = [] for var in need_add_vars: if "@GRAD" not in var: need_ignore_vars.append(var) need_add_vars = list( - set(need_add_vars).difference(set(need_ignore_vars))) - previous_block_private = block_var_detail[index + - 1]["backward"]["private"] - previous_block_entrance = block_var_detail[index + - 1]["backward"]["entrance"] + set(need_add_vars).difference(set(need_ignore_vars)) + ) + previous_block_private = block_var_detail[index + 1]["backward"][ + "private" + ] + previous_block_entrance = block_var_detail[index + 1]["backward"][ + "entrance" + ] for var in need_add_vars: - if var not in previous_block_private and var not in previous_block_entrance: + if ( + var not in previous_block_private + and var not in previous_block_entrance + ): previous_block_entrance.append(var) previous_block_exit.append(var) return block_var_detail -def delete_block_useless_exit(program, program_block_ops_list, - block_var_detail): +def delete_block_useless_exit( + program, program_block_ops_list, block_var_detail +): ## forward for index in range(len(block_var_detail)): if index == len(block_var_detail) - 1: @@ -951,8 +1133,9 @@ def delete_block_useless_exit(program, program_block_ops_list, if index - 1 < 0: break current_block_exit = block_var_detail[index]["backward"]["exit"] - next_block_entrance = block_var_detail[index - - 1]["backward"]["entrance"] + next_block_entrance = block_var_detail[index - 1]["backward"][ + "entrance" + ] need_delete_var = [] for var in current_block_exit: if var not in next_block_entrance: @@ -963,19 +1146,20 @@ def delete_block_useless_exit(program, program_block_ops_list, return block_var_detail -def get_communicate_var_info(program, - block_index, - entrance_var_list, - type="forward"): +def get_communicate_var_info( + program, block_index, entrance_var_list, type="forward" +): input_var_reshape_dim = [] input_var_reshape_name = [] if type == "forward": block_input_var_name = "forward_joint_{}_{}@Heter".format( - block_index - 1, block_index) + block_index - 1, block_index + ) else: block_input_var_name = "backward_joint_{}_{}@Heter".format( - block_index + 1, block_index) + block_index + 1, block_index + ) entrance_var_list.sort() # input @@ -998,12 +1182,15 @@ def get_communicate_var_info(program, def add_vars_by_var_list(var_name_list, origin_program, program, block): for var_name in var_name_list: - if var_name not in program.global_block( - ).vars and var_name not in block.vars: + if ( + var_name not in program.global_block().vars + and var_name not in block.vars + ): var = origin_program.global_block().vars[var_name] if var.persistable: - program.global_block()._clone_variable(var, - force_persistable=False) + program.global_block()._clone_variable( + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) @@ -1082,11 +1269,14 @@ def block_append_op(program, origin_program, block, op): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: - if var.name not in program.global_block( - ).vars and var.name not in block.vars: + if ( + var.name not in program.global_block().vars + and var.name not in block.vars + ): if var.persistable: program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) @@ -1095,20 +1285,22 @@ def block_append_op(program, origin_program, block, op): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: - if var.name not in program.global_block( - ).vars and var.name not in block.vars: + if ( + var.name not in program.global_block().vars + and var.name not in block.vars + ): if var.persistable: program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) if "_grad" not in op.type: # for forward op - return block.append_op(type=op.type, - inputs=inputs, - outputs=outputs, - attrs=op.all_attrs()) + return block.append_op( + type=op.type, inputs=inputs, outputs=outputs, attrs=op.all_attrs() + ) else: # for grad op op_desc = op.desc @@ -1134,49 +1326,54 @@ def get_next_stage_trainers(role_maker): return role_maker.get_next_trainers() -def insert_communicate_op(orign_program, - role_maker, - heter_block, - stage_id, - first_op_index, - block_var_detail, - device, - is_forward=True): +def insert_communicate_op( + orign_program, + role_maker, + heter_block, + stage_id, + first_op_index, + block_var_detail, + device, + is_forward=True, +): if is_forward: next_heter_worker_endpoints = get_next_stage_trainers(role_maker) previous_heter_worker_endpoints = get_previous_stage_trainers( - role_maker) + role_maker + ) entrance_var = block_var_detail[stage_id]["forward"]["entrance"] - comm_info = get_communicate_var_info(orign_program, stage_id + 1, - entrance_var) + comm_info = get_communicate_var_info( + orign_program, stage_id + 1, entrance_var + ) else: next_heter_worker_endpoints = get_next_stage_trainers(role_maker) previous_heter_worker_endpoints = get_previous_stage_trainers( - role_maker) + role_maker + ) entrance_var = block_var_detail[stage_id - 1]["backward"]["exit"] - comm_info = get_communicate_var_info(orign_program, stage_id - 1, - entrance_var, "backward") - - heter_block._insert_op(index=first_op_index, - type="send_and_recv", - inputs={"X": heter_block.vars[entrance_var[0]]}, - outputs={"Out": []}, - attrs={ - "mode": "forward" if is_forward else "backward", - "send_var_name": - entrance_var + ["microbatch_id"], - "recv_var_name": [], - "message_name": - comm_info["block_input_var_name"], - "next_endpoints": next_heter_worker_endpoints, - "previous_endpoints": - previous_heter_worker_endpoints, - "trainer_id": get_role_id(role_maker), - "op_device": device, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + comm_info = get_communicate_var_info( + orign_program, stage_id - 1, entrance_var, "backward" + ) + + heter_block._insert_op( + index=first_op_index, + type="send_and_recv", + inputs={"X": heter_block.vars[entrance_var[0]]}, + outputs={"Out": []}, + attrs={ + "mode": "forward" if is_forward else "backward", + "send_var_name": entrance_var + ["microbatch_id"], + "recv_var_name": [], + "message_name": comm_info["block_input_var_name"], + "next_endpoints": next_heter_worker_endpoints, + "previous_endpoints": previous_heter_worker_endpoints, + "trainer_id": get_role_id(role_maker), + "op_device": device, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return entrance_var @@ -1185,8 +1382,9 @@ def get_the_one_recv_context(context, is_dense=True, split_dense_table=False): recv_id_maps = {} grad_name_to_param_name = {} if is_dense: - send_ctx = get_the_one_send_context(context, - split_dense_table=split_dense_table) + send_ctx = get_the_one_send_context( + context, split_dense_table=split_dense_table + ) for idx, (name, ctx) in enumerate(send_ctx.items()): if ctx.is_sparse(): continue @@ -1201,9 +1399,9 @@ def get_the_one_recv_context(context, is_dense=True, split_dense_table=False): param_names.append(param_name) recv_id_maps[ctx.table_id()] = param_names else: - send_ctx = get_the_one_send_context(context, - split_dense_table=False, - ep_list=None) + send_ctx = get_the_one_send_context( + context, split_dense_table=False, ep_list=None + ) for idx, (name, ctx) in enumerate(send_ctx.items()): if not ctx.is_sparse(): continue @@ -1225,15 +1423,15 @@ def _get_varname_parts(varname): block_part = "" trainer_idx = varname.find(".trainer_") if trainer_idx >= 0: - trainer_part = varname[trainer_idx + 1:] + trainer_part = varname[trainer_idx + 1 :] else: trainer_idx = len(varname) block_index = varname.find(".block") if block_index >= 0: - block_part = varname[block_index + 1:trainer_idx] + block_part = varname[block_index + 1 : trainer_idx] else: block_index = len(varname) - orig_var_name = varname[0:min(block_index, trainer_idx)] + orig_var_name = varname[0 : min(block_index, trainer_idx)] return orig_var_name, block_part, trainer_part @@ -1256,7 +1454,6 @@ def get_var_mem_size(var): class MergedVariable: - def __init__(self, merged, ordered, offsets): self.merged_var = merged self.ordered_vars = ordered @@ -1276,8 +1473,8 @@ def build_var_distributed(context): context["merged_variable_map"] = {} for origin_program in origin_programs: sparse_pairs, dense_pairs = get_param_grads(origin_program) - #print("public build_var_distributed sparse_pairs:", sparse_pairs) - #print("public build_var_distributed dense_pairs:", dense_pairs) + # print("public build_var_distributed sparse_pairs:", sparse_pairs) + # print("public build_var_distributed dense_pairs:", dense_pairs) origin_for_sparse = [] origin_for_dense = [] merged_sparse_pairs = [] @@ -1297,7 +1494,7 @@ def build_var_distributed(context): m_grad = MergedVariable(grad, [grad], [0]) merged_variables_pairs.append((m_param, m_grad)) merged_dense_pairs.append((m_param, m_grad)) - #print("public build_var_distributed merged_dense_pairs:", + # print("public build_var_distributed merged_dense_pairs:", # merged_dense_pairs) for sparse_pair in origin_for_sparse: @@ -1307,15 +1504,17 @@ def build_var_distributed(context): m_grad = MergedVariable(grad, [grad], [0]) merged_variables_pairs.append((m_param, m_grad)) merged_sparse_pairs.append((m_param, m_grad)) - #print("public build_var_distributed merged_sparse_pairs:", + # print("public build_var_distributed merged_sparse_pairs:", # merged_sparse_pairs) for merged in merged_variables_pairs: m_param, m_grad = merged context["merged_variable_map"][ - m_param.merged_var.name] = m_param.merged_var + m_param.merged_var.name + ] = m_param.merged_var context["merged_variable_map"][ - m_grad.merged_var.name] = m_grad.merged_var + m_grad.merged_var.name + ] = m_grad.merged_var param_merges = [] param_merges.extend(origin_for_sparse) @@ -1353,14 +1552,14 @@ def _is_opt_role_op(op): # optimize op_maker = core.op_proto_and_checker_maker optimize_role = core.op_proto_and_checker_maker.OpRole.Optimize - if op_maker.kOpRoleAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(optimize_role): + if op_maker.kOpRoleAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(optimize_role): return True return False def get_param_grads(origin_program): - def _get_params_grads(sparse_varnames): block = origin_program.global_block() @@ -1373,8 +1572,10 @@ def get_param_grads(origin_program): for op in block.ops: if _is_opt_role_op(op): # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs() \ - and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE): + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + ): op._set_attr("op_role", role_id) continue if op.attr(OP_ROLE_VAR_ATTR_NAME): @@ -1382,8 +1583,10 @@ def get_param_grads(origin_program): grad_name = op.attr(OP_ROLE_VAR_ATTR_NAME)[1] if param_name not in optimize_params: optimize_params.add(param_name) - param_grad = (origin_var_dict[param_name], - origin_var_dict[grad_name]) + param_grad = ( + origin_var_dict[param_name], + origin_var_dict[grad_name], + ) if param_name in sparse_varnames: sparse_param_grads.append(param_grad) @@ -1394,8 +1597,10 @@ def get_param_grads(origin_program): def _get_sparse_varnames(): varnames = [] for op in origin_program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] varnames.append(param_name) @@ -1437,13 +1642,13 @@ def find_op_input_output(program, block, op): def add_send_op(program, block, _vars): - def _get_send_op_dict(): send_op_dict = {} send_op_list = find_send_op(program) for op in send_op_list: - input_list, _ = find_op_input_output(program, - program.global_block(), op) + input_list, _ = find_op_input_output( + program, program.global_block(), op + ) for var in input_list: send_op_dict[var] = op return send_op_dict @@ -1472,21 +1677,23 @@ def add_send_op(program, block, _vars): for table_id in table_dict: dummy_output = block.create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) send_input_vars = [ block.vars[union_var] for union_var in table_dict[table_id]['var_list'] ] - block.append_op(type="send", - inputs={"X": send_input_vars}, - outputs={"Out": dummy_output}, - attrs={ - "send_varnames": - table_dict[table_id]['send_varnames'], - "is_sparse": is_sparse, - "table_id": table_id, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + block.append_op( + type="send", + inputs={"X": send_input_vars}, + outputs={"Out": dummy_output}, + attrs={ + "send_varnames": table_dict[table_id]['send_varnames'], + "is_sparse": is_sparse, + "table_id": table_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return send_grad_var_list @@ -1503,21 +1710,26 @@ def delete_trainer_useless_var(program, static_var): program_useful_var_list = [] for op in program.global_block().ops: input_var_list, output_var_list = find_op_input_output( - program, program.global_block(), op) + program, program.global_block(), op + ) op_var_list = list(set(input_var_list).union(set(output_var_list))) program_useful_var_list = list( - set(program_useful_var_list).union(set(op_var_list))) + set(program_useful_var_list).union(set(op_var_list)) + ) program_useful_var_list += static_var program_useless_var_list = list( set(get_vars_name_in_block(program.global_block())).difference( - set(program_useful_var_list))) + set(program_useful_var_list) + ) + ) for var in program_useless_var_list: program.global_block()._remove_var(var) return program_useless_var_list -def create_backward_block(program, origin_program, bp_ops_list, - block_var_detail): +def create_backward_block( + program, origin_program, bp_ops_list, block_var_detail +): pre_block_idx = program.num_blocks - 1 heter_block = program._create_block(pre_block_idx) @@ -1526,8 +1738,10 @@ def create_backward_block(program, origin_program, bp_ops_list, send_varnames = op.attr('send_varnames') is_skip = False for varname in send_varnames: - if varname not in program.global_block( - ).vars and varname not in heter_block.vars: + if ( + varname not in program.global_block().vars + and varname not in heter_block.vars + ): is_skip = True break if is_skip == True: @@ -1543,12 +1757,14 @@ def create_backward_block(program, origin_program, bp_ops_list, def is_backward_op(op): return op_role_attr_name in op.attr_names and ( - int(op.attr(op_role_attr_name)) & int(op_role.Backward)) + int(op.attr(op_role_attr_name)) & int(op_role.Backward) + ) def is_forward_op(op): - return op_role_attr_name in op.attr_names and (int( - op.attr(op_role_attr_name)) == int(op_role.Forward)) + return op_role_attr_name in op.attr_names and ( + int(op.attr(op_role_attr_name)) == int(op_role.Forward) + ) def is_push_sparse_op(op): @@ -1591,11 +1807,13 @@ def check_program(program): for op in block.ops: input_var_names = op.desc.input_arg_names() output_var_names = op.desc.output_arg_names() - for var_name in (input_var_names + output_var_names): + for var_name in input_var_names + output_var_names: if not block._find_var_recursive(str(var_name)): raise ValueError( 'var: {} needed by op is not found in block: {}'.format( - str(var_name), block_idx)) + str(var_name), block_idx + ) + ) block_idx += 1 print('program checked valid') diff --git a/python/paddle/distributed/rpc/rpc.py b/python/paddle/distributed/rpc/rpc.py index c61b1a805b0..6a93f27517c 100644 --- a/python/paddle/distributed/rpc/rpc.py +++ b/python/paddle/distributed/rpc/rpc.py @@ -26,7 +26,7 @@ from paddle.distributed.launch.context import Node WorkerInfo = namedtuple("WorkerInfo", ["name", "rank", "ip", "port"]) _DEFAULT_RPC_TIMEOUT = -1 -_MAX_RPC_TIMEOUT_MS = 0x7fffffff +_MAX_RPC_TIMEOUT_MS = 0x7FFFFFFF _BARRIER_TIMEOUT_MAX_DAYS = 99999999 # tcp store for `_barrier_never_timeout` _barrier_store = None @@ -55,8 +55,9 @@ def _exchange_all_service_infos(world_size): s = set() for rank in range(world_size): info = pickle.loads(_barrier_store.get(str(rank))) - assert (info.name not in s - ), "The Worker name must be unique, but name `{}` is repeated." + assert ( + info.name not in s + ), "The Worker name must be unique, but name `{}` is repeated." s.add(info.name) all_infos.append(info) return all_infos @@ -94,22 +95,30 @@ def init_rpc(name, rank=None, world_size=None, master_endpoint=None): """ rank = int(os.environ["PADDLE_TRAINER_ID"]) if rank is None else rank - world_size = int( - os.environ["PADDLE_TRAINERS_NUM"]) if world_size is None else world_size + world_size = ( + int(os.environ["PADDLE_TRAINERS_NUM"]) + if world_size is None + else world_size + ) worker_endpoint = os.getenv("PADDLE_WORKER_ENDPOINT", None) if worker_endpoint is None: worker_endpoint = _gen_endpoint() logger.info("Trainer {}: worker endpoint: {}".format(rank, worker_endpoint)) - master_endpoint = (master_endpoint if master_endpoint != None else - os.environ["PADDLE_MASTER_ENDPOINT"]) + master_endpoint = ( + master_endpoint + if master_endpoint != None + else os.environ["PADDLE_MASTER_ENDPOINT"] + ) master_addr, master_port = master_endpoint.split(":") master_port = int(master_port) stop_check_timeout = int(os.getenv("FLAGS_stop_check_timeout", "900")) - store = core.TCPStore(master_addr, - master_port, - rank == 0, - world_size, - timeout=stop_check_timeout) + store = core.TCPStore( + master_addr, + master_port, + rank == 0, + world_size, + timeout=stop_check_timeout, + ) _set_barrier_store(store) ip, port = worker_endpoint.split(":") port = int(port) @@ -117,8 +126,9 @@ def init_rpc(name, rank=None, world_size=None, master_endpoint=None): all_infos = _exchange_all_service_infos(world_size) c_infos = [] for node_info in all_infos: - info = core.WorkerInfo(node_info.name, node_info.rank, node_info.ip, - node_info.port) + info = core.WorkerInfo( + node_info.name, node_info.rank, node_info.ip, node_info.port + ) c_infos.append(info) core.init_and_set_agent_instance(name, c_infos) core.rpc_start_worker() @@ -226,7 +236,7 @@ def _barrier_never_timeout(global_rank, global_world_size): global _barrier_count barrier_prefix = "Barrier/" + str(_barrier_count) + "/" _barrier_count += 1 - is_master = (global_rank == 0) + is_master = global_rank == 0 def _check_keys_ready(wait_keys): start_time = time.time() @@ -235,11 +245,13 @@ def _barrier_never_timeout(global_rank, global_world_size): elapse_time = time.time() - start_time if datetime.timedelta(seconds=elapse_time) > timeout: raise RuntimeError( - "Keys {} are not ready sinck rank {} is waiting them.". - format(wait_keys, global_rank)) + "Keys {} are not ready sinck rank {} is waiting them.".format( + wait_keys, global_rank + ) + ) wait_keys = list( - filter(lambda key: int(_barrier_store.get(key)) != 1, - wait_keys)) + filter(lambda key: int(_barrier_store.get(key)) != 1, wait_keys) + ) if is_master: # the master will add key, wait for all workers'exiting key and exit in the end. diff --git a/python/paddle/distributed/sharding/__init__.py b/python/paddle/distributed/sharding/__init__.py index e938c12d5af..3a710ca8059 100644 --- a/python/paddle/distributed/sharding/__init__.py +++ b/python/paddle/distributed/sharding/__init__.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .group_sharded import group_sharded_parallel, save_group_sharded_model # noqa: F401 +from .group_sharded import ( + group_sharded_parallel, + save_group_sharded_model, +) # noqa: F401 __all__ = ['group_sharded_parallel', 'save_group_sharded_model'] diff --git a/python/paddle/distributed/sharding/group_sharded.py b/python/paddle/distributed/sharding/group_sharded.py index 1474f639547..4137075c3f9 100644 --- a/python/paddle/distributed/sharding/group_sharded.py +++ b/python/paddle/distributed/sharding/group_sharded.py @@ -22,31 +22,49 @@ from paddle.distributed.utils.log_utils import get_logger from paddle.fluid.framework import in_dygraph_mode # Old version -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ShardingStage3 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ( + ShardingStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ( + ShardingScaler, +) # New version -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import GroupShardedStage3 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import ( + GroupShardedStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) logger_ = get_logger(logging.WARNING) -def group_sharded_parallel(model, - optimizer, - level, - scaler=None, - group=None, - offload=False, - sync_buffers=False, - buffer_max_size=2**23, - segment_size=2**20, - sync_comm=False, - dp_group=None): +def group_sharded_parallel( + model, + optimizer, + level, + scaler=None, + group=None, + offload=False, + sync_buffers=False, + buffer_max_size=2**23, + segment_size=2**20, + sync_comm=False, + dp_group=None, +): """ Use group_sharded_parallel can perform group shared configuration on the model, optimizer and GradScaler. Level has three string options, 'os', 'os_g' and 'p_g_os' corresponds to three different usage scenarios: optimizer state segmentation, optimizer state + gradient segmentation, and parameter + gradient + optimizer state segmentation. Usually, optimizer state + gradient segmentation is actually a re optimization of optimizer state segmentation, so optimizer state + gradient segmentation can be used to realize optimizer state segmentation. @@ -101,13 +119,16 @@ def group_sharded_parallel(model, """ # check optition type assert isinstance( - model, - paddle.nn.Layer), "The model must be the instance of paddle.nn.Layer." + model, paddle.nn.Layer + ), "The model must be the instance of paddle.nn.Layer." assert isinstance( optimizer, Optimizer ), "The optimizer must be the instance of paddle.optimizer.Optimizer." - assert level in ['os', 'os_g', - 'p_g_os'], "The level must be os, os_g or p_g_os." + assert level in [ + 'os', + 'os_g', + 'p_g_os', + ], "The level must be os, os_g or p_g_os." def check_dtype(param): return param.dtype == paddle.float16 @@ -126,40 +147,51 @@ def group_sharded_parallel(model, optim=optimizer, group=group, offload=offload, - dp_group=dp_group) - model = GroupShardedStage2(model, - optimizer, - group=group, - sync_buffers=sync_buffers, - buffer_max_size=buffer_max_size, - dp_group=dp_group) + dp_group=dp_group, + ) + model = GroupShardedStage2( + model, + optimizer, + group=group, + sync_buffers=sync_buffers, + buffer_max_size=buffer_max_size, + dp_group=dp_group, + ) else: - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - group=group, - offload=offload) - model = ShardingStage2(model, - optimizer, - group=group, - sync_buffers=sync_buffers, - buffer_max_size=buffer_max_size) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), + optim=optimizer, + group=group, + offload=offload, + ) + model = ShardingStage2( + model, + optimizer, + group=group, + sync_buffers=sync_buffers, + buffer_max_size=buffer_max_size, + ) elif level == 'p_g_os': if in_dygraph_mode(): - model = GroupShardedStage3(model, - optimizer=optimizer, - group=group, - sync_buffers=sync_buffers, - segment_size=segment_size, - offload=offload, - sync_comm=sync_comm) + model = GroupShardedStage3( + model, + optimizer=optimizer, + group=group, + sync_buffers=sync_buffers, + segment_size=segment_size, + offload=offload, + sync_comm=sync_comm, + ) else: - model = ShardingStage3(model, - optimizer=optimizer, - group=group, - sync_buffers=sync_buffers, - segment_size=segment_size, - offload=offload, - sync_comm=sync_comm) + model = ShardingStage3( + model, + optimizer=optimizer, + group=group, + sync_buffers=sync_buffers, + segment_size=segment_size, + offload=offload, + sync_comm=sync_comm, + ) else: raise ValueError("Please enter the correct level.") if isinstance(scaler, paddle.amp.GradScaler): @@ -222,7 +254,8 @@ def save_group_sharded_model(model, output, optimizer=None): save_group_sharded_model(model, optimizer, output=output_dir) """ logger_.info( - "==========Begin to save group sharded model and optimizer==========") + "==========Begin to save group sharded model and optimizer==========" + ) assert not os.path.isfile( output ), "Saving directory ({}) should be a directory, not a file".format(output) @@ -246,4 +279,5 @@ def save_group_sharded_model(model, output, optimizer=None): output_opt = os.path.join(output, "model.pdopt") paddle.save(optimizer._optim.state_dict(), output_opt) logger_.info( - "==========End to save group sharded model and optimizer==========") + "==========End to save group sharded model and optimizer==========" + ) diff --git a/python/paddle/distributed/spawn.py b/python/paddle/distributed/spawn.py index 7deebdf549e..2df9118ac4e 100644 --- a/python/paddle/distributed/spawn.py +++ b/python/paddle/distributed/spawn.py @@ -18,11 +18,22 @@ import signal import sys import warnings -from paddle.distributed.utils.launch_utils import _print_arguments, _prepare_trainer_env, get_host_name_ip -from paddle.distributed.cloud_utils import get_cluster_and_pod, _get_trainers_num +from paddle.distributed.utils.launch_utils import ( + _print_arguments, + _prepare_trainer_env, + get_host_name_ip, +) +from paddle.distributed.cloud_utils import ( + get_cluster_and_pod, + _get_trainers_num, +) from paddle.distributed.fleet.launch import get_cluster_from_args from paddle.distributed.fleet.cloud_utils import use_paddlecloud -from paddle.distributed.fleet.launch_utils import DeviceMode, check_backend, block_windows_and_macos +from paddle.distributed.fleet.launch_utils import ( + DeviceMode, + check_backend, + block_windows_and_macos, +) from paddle.device import get_device # deprecated module import @@ -33,7 +44,6 @@ __all__ = [] class ParallelEnvArgs(object): - def __init__(self): # Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17.. self.cluster_node_ips = None @@ -64,17 +74,27 @@ def _py_supported_check(): "Use `paddle.distributed.spawn` to start parallel training " "requires python version greater than 3.4, if your python " "is lower than this version, please use " - "`paddle.distributed.launch` instead.") + "`paddle.distributed.launch` instead." + ) def _options_valid_check(options): # `print_config` keeped as a debug options, not show to users supported_options = [ - 'start_method', 'ips', 'gpus', 'xpus', 'mlus', 'print_config', 'backend' + 'start_method', + 'ips', + 'gpus', + 'xpus', + 'mlus', + 'print_config', + 'backend', ] deprecated_options = [ - 'selected_devices', 'started_port', 'cluster_node_ips', 'node_ip', - 'use_paddlecloud' + 'selected_devices', + 'started_port', + 'cluster_node_ips', + 'node_ip', + 'use_paddlecloud', ] for key in options: if key not in supported_options: @@ -82,11 +102,14 @@ def _options_valid_check(options): warnings.warn( "The config option (%s) of `paddle.distributed.spawn` is deprecated. " "Please use the latest config options stated in the `spawn` API documentation." - % key, DeprecationWarning) + % key, + DeprecationWarning, + ) else: raise ValueError( "The config option (%s) of `paddle.distributed.spawn` is not supported." - % key) + % key + ) def _get_default_nprocs(): @@ -101,8 +124,10 @@ def _get_default_nprocs(): return multiprocessing.cpu_count() else: raise RuntimeError( - "`paddle.distributed.spawn` does not support parallel training on device `{}` now." - .format(device)) + "`paddle.distributed.spawn` does not support parallel training on device `{}` now.".format( + device + ) + ) def _get_default_backend(): @@ -117,8 +142,10 @@ def _get_default_backend(): return 'gloo' else: raise RuntimeError( - "`paddle.distributed.spawn` does not support parallel training on device `{}` now." - .format(device)) + "`paddle.distributed.spawn` does not support parallel training on device `{}` now.".format( + device + ) + ) def _get_node_ip(ips): @@ -180,23 +207,28 @@ def _get_subprocess_env_list(nprocs, options): "the number of visible devices(%d) is less than the number " "of spawn processes(%d), please ensure that the correct " "`nprocs` argument is passed or the environment variable " - "`CUDA_VISIBLE_DEVICES` is correctly configured." % - (len(env_devices_list), nprocs)) + "`CUDA_VISIBLE_DEVICES` is correctly configured." + % (len(env_devices_list), nprocs) + ) args.selected_devices = ",".join( - [str(env_devices_list[x]) for x in range(0, nprocs)]) + [str(env_devices_list[x]) for x in range(0, nprocs)] + ) else: selected_device_list = args.selected_devices.split(',') if len(selected_device_list) != nprocs: raise ValueError( "The number of selected devices(%s) is not equal to " "the number of spawn processes(%d), please ensure that the " - "correct `nprocs` and `gpus` arguments are passed." % - (len(selected_device_list), nprocs)) + "correct `nprocs` and `gpus` arguments are passed." + % (len(selected_device_list), nprocs) + ) for card_id in selected_device_list: if card_id not in env_devices_list: - raise ValueError("The selected gpu card %s cannot found in " - "CUDA_VISIBLE_DEVICES (%s)." % - (card_id, ",".join(env_devices_list))) + raise ValueError( + "The selected gpu card %s cannot found in " + "CUDA_VISIBLE_DEVICES (%s)." + % (card_id, ",".join(env_devices_list)) + ) elif options['backend'] == 'bkcl': args.selected_devices = options.get('xpus', None) @@ -215,23 +247,28 @@ def _get_subprocess_env_list(nprocs, options): "the number of visible devices(%d) is less than the number " "of spawn processes(%d), please ensure that the correct " "`nprocs` argument is passed or the environment variable " - "`XPU_VISIBLE_DEVICES` is correctly configured." % - (len(env_devices_list), nprocs)) + "`XPU_VISIBLE_DEVICES` is correctly configured." + % (len(env_devices_list), nprocs) + ) args.selected_devices = ",".join( - [str(env_devices_list[x]) for x in range(0, nprocs)]) + [str(env_devices_list[x]) for x in range(0, nprocs)] + ) else: selected_device_list = args.selected_devices.split(',') if len(selected_device_list) != nprocs: raise ValueError( "The number of selected devices(%s) is not equal to " "the number of spawn processes(%d), please ensure that the " - "correct `nprocs` and `xpus` arguments are passed." % - (len(selected_device_list), nprocs)) + "correct `nprocs` and `xpus` arguments are passed." + % (len(selected_device_list), nprocs) + ) for card_id in selected_device_list: if card_id not in env_devices_list: - raise ValueError("The selected xpu card %s cannot found in " - "XPU_VISIBLE_DEVICES (%s)." % - (card_id, ",".join(env_devices_list))) + raise ValueError( + "The selected xpu card %s cannot found in " + "XPU_VISIBLE_DEVICES (%s)." + % (card_id, ",".join(env_devices_list)) + ) elif options['backend'] == 'cncl': args.selected_devices = options.get('mlus', None) if args.selected_devices is None: @@ -249,23 +286,28 @@ def _get_subprocess_env_list(nprocs, options): "the number of visible devices(%d) is less than the number " "of spawn processes(%d), please ensure that the correct " "`nprocs` argument is passed or the environment variable " - "`MLU_VISIBLE_DEVICES` is correctly configured." % - (len(env_devices_list), nprocs)) + "`MLU_VISIBLE_DEVICES` is correctly configured." + % (len(env_devices_list), nprocs) + ) args.selected_devices = ",".join( - [str(env_devices_list[x]) for x in range(0, nprocs)]) + [str(env_devices_list[x]) for x in range(0, nprocs)] + ) else: selected_device_list = args.selected_devices.split(',') if len(selected_device_list) != nprocs: raise ValueError( "The number of selected devices(%s) is not equal to " "the number of spawn processes(%d), please ensure that the " - "correct `nprocs` and `mlus` arguments are passed." % - (len(selected_device_list), nprocs)) + "correct `nprocs` and `mlus` arguments are passed." + % (len(selected_device_list), nprocs) + ) for card_id in selected_device_list: if card_id not in env_devices_list: - raise ValueError("The selected mlu card %s cannot found in " - "MLU_VISIBLE_DEVICES (%s)." % - (card_id, ",".join(env_devices_list))) + raise ValueError( + "The selected mlu card %s cannot found in " + "MLU_VISIBLE_DEVICES (%s)." + % (card_id, ",".join(env_devices_list)) + ) elif options['backend'] == 'gloo': # TODO check gpu / xpu flag must not exist warnings.warn( @@ -275,14 +317,15 @@ def _get_subprocess_env_list(nprocs, options): args.paddle_cpuonly = True args.selected_devices = None args.ips = args.cluster_node_ips - assert options.get( - 'use_paddlecloud', - None) is None, "CPUONLY spawn doesn't support use paddle cloud" - assert len( - args.cluster_node_ips.split(',') - ) <= 1, "CPUONLY spawn only support single trainer, that is len(ips)=1, but got %s." - assert _get_trainers_num( - ) == 1, "CPUONLY spawn doesn't support multi-trainer" + assert ( + options.get('use_paddlecloud', None) is None + ), "CPUONLY spawn doesn't support use paddle cloud" + assert ( + len(args.cluster_node_ips.split(',')) <= 1 + ), "CPUONLY spawn only support single trainer, that is len(ips)=1, but got %s." + assert ( + _get_trainers_num() == 1 + ), "CPUONLY spawn doesn't support multi-trainer" # set other inner args args.node_ip = options.get('node_ip', None) @@ -298,15 +341,17 @@ def _get_subprocess_env_list(nprocs, options): # get cluster and pod config if options['backend'] == 'gloo': devices_per_proc = [x for x in range(0, nprocs)] - cluster, pod = get_cluster_from_args(args, DeviceMode.CPU, - devices_per_proc) + cluster, pod = get_cluster_from_args( + args, DeviceMode.CPU, devices_per_proc + ) else: cluster, pod = get_cluster_and_pod(args) # prepare subprocess env list for trainer in pod.trainers: processes_env_list.append( - _prepare_trainer_env(cluster, trainer, options['backend'])) + _prepare_trainer_env(cluster, trainer, options['backend']) + ) # [Debug] print config args.print_config = options.get('print_config', False) @@ -340,7 +385,7 @@ def _set_trainer_env(env_dict, backend): elif backend == 'cncl': set_flags({'FLAGS_selected_mlus': env_dict['FLAGS_selected_mlus']}) else: - #NOTE(xiongkun) why not raise Error ? + # NOTE(xiongkun) why not raise Error ? # So far, we added support for CPU parallel, and will be applied when paddle is not # compiled with cuda or xp. just do nothing. pass @@ -362,12 +407,12 @@ def _func_wrapper(func, args, error_queue, return_queue, env_dict, backend): pass except Exception: import traceback + error_queue.put(traceback.format_exc()) sys.exit(1) class MultiprocessContext(object): - def __init__(self, processes, error_queues, return_queues): _py_supported_check() self.error_queues = error_queues @@ -379,16 +424,16 @@ class MultiprocessContext(object): self.return_queues = return_queues self.processes = processes self.sentinels = { - process.sentinel: index - for index, process in enumerate(processes) + process.sentinel: index for index, process in enumerate(processes) } def join(self, timeout=None): if len(self.sentinels) == 0: return True - ready = multiprocessing.connection.wait(self.sentinels.keys(), - timeout=timeout) + ready = multiprocessing.connection.wait( + self.sentinels.keys(), timeout=timeout + ) error_index = None for sentinel in ready: @@ -414,16 +459,22 @@ class MultiprocessContext(object): exitcode = self.processes[error_index].exitcode if exitcode < 0: name = signal.Signals(-exitcode).name - raise Exception("Process %d terminated with signal %s." % - (error_index, name)) + raise Exception( + "Process %d terminated with signal %s." + % (error_index, name) + ) else: - raise Exception("Process %d terminated with exit code %d." % - (error_index, exitcode)) + raise Exception( + "Process %d terminated with exit code %d." + % (error_index, exitcode) + ) original_trace = self.error_queues[error_index].get() - msg = "\n\n----------------------------------------------\n" \ - "Process %d terminated with the following error:\n" \ - "----------------------------------------------\n\n" % error_index + msg = ( + "\n\n----------------------------------------------\n" + "Process %d terminated with the following error:\n" + "----------------------------------------------\n\n" % error_index + ) msg += original_trace raise Exception(msg) @@ -584,9 +635,17 @@ def spawn(func, args=(), nprocs=-1, join=True, daemon=False, **options): for i in range(nprocs): error_queue = mp.SimpleQueue() return_queue = mp.SimpleQueue() - process = mp.Process(target=_func_wrapper, - args=(func, args, error_queue, return_queue, - procs_env_list[i], options['backend'])) + process = mp.Process( + target=_func_wrapper, + args=( + func, + args, + error_queue, + return_queue, + procs_env_list[i], + options['backend'], + ), + ) process.daemon = daemon process.start() error_queues.append(error_queue) diff --git a/python/paddle/distributed/utils/launch_utils.py b/python/paddle/distributed/utils/launch_utils.py index a542523e605..88acc643ead 100644 --- a/python/paddle/distributed/utils/launch_utils.py +++ b/python/paddle/distributed/utils/launch_utils.py @@ -33,12 +33,18 @@ def get_cluster_from_args(args, selected_gpus): node_ip = args.node_ip node_rank = node_ips.index(node_ip) - logger.debug("parsed from args:node_ips:{} node_ip:{} node_rank:{}".format( - node_ips, node_ip, node_rank)) + logger.debug( + "parsed from args:node_ips:{} node_ip:{} node_rank:{}".format( + node_ips, node_ip, node_rank + ) + ) free_ports = None - if not args.use_paddlecloud and len( - node_ips) <= 1 and args.started_port is None: + if ( + not args.use_paddlecloud + and len(node_ips) <= 1 + and args.started_port is None + ): free_ports = find_free_ports(len(selected_gpus)) if free_ports is not None: free_ports = list(free_ports) @@ -60,6 +66,7 @@ def get_cluster_from_args(args, selected_gpus): def get_gpus(selected_gpus): if selected_gpus is None: from paddle.fluid import core + gpus_num = core.get_cuda_device_count() gpus = [str(x) for x in range(0, gpus_num)] else: @@ -72,48 +79,56 @@ def get_gpus(selected_gpus): # therefore selected_gpus=0,1,2,3 cuda_visible_devices_list = cuda_visible_devices.split(',') for x in selected_gpus.split(','): - assert x in cuda_visible_devices_list, "Can't find "\ - "your selected_gpus %s in CUDA_VISIBLE_DEVICES[%s]."\ - % (x, cuda_visible_devices) + assert x in cuda_visible_devices_list, ( + "Can't find " + "your selected_gpus %s in CUDA_VISIBLE_DEVICES[%s]." + % (x, cuda_visible_devices) + ) gpus = [ cuda_visible_devices_list.index(x.strip()) for x in selected_gpus.split(',') ] - logger.info("Change selected_gpus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "CUDA_VISIBLE_DEVICES:{}".format( - selected_gpus, gpus, cuda_visible_devices_list)) + logger.info( + "Change selected_gpus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "CUDA_VISIBLE_DEVICES:{}".format( + selected_gpus, gpus, cuda_visible_devices_list + ) + ) return gpus class Hdfs(object): - def __init__(self): self.hdfs_ugi = None self.hdfs_name = None self.hdfs_path = None def is_valid(self): - return self.hdfs_ugi is not None and \ - self.hdfs_name is not None and \ - self.hdfs_path is not None + return ( + self.hdfs_ugi is not None + and self.hdfs_name is not None + and self.hdfs_path is not None + ) def __str__(self): return "hdfs_ugi:{} hdfs_name:{} hdfs_path{}".format( - self.hdfs_ugi, self.hdfs_name, self.hdfs_path) + self.hdfs_ugi, self.hdfs_name, self.hdfs_path + ) def __eq__(self, n): - return self.hdfs_ugi == n.hdfs_ugi and \ - self.hdfs_name == n.hdfs_name and \ - self.hdfs_path == n.hdfs_path + return ( + self.hdfs_ugi == n.hdfs_ugi + and self.hdfs_name == n.hdfs_name + and self.hdfs_path == n.hdfs_path + ) def __ne__(self, n): return not self == n class Cluster(object): - def __init__(self, hdfs): self.job_server = None self.pods = [] @@ -122,8 +137,11 @@ class Cluster(object): def __str__(self): return "job_server:{} pods:{} job_stage_flag:{} hdfs:{}".format( - self.job_server, [str(pod) for pod in self.pods], - self.job_stage_flag, self.hdfs) + self.job_server, + [str(pod) for pod in self.pods], + self.job_stage_flag, + self.hdfs, + ) def __eq__(self, cluster): if len(self.pods) != len(cluster.pods): @@ -161,8 +179,9 @@ class Cluster(object): r = [] for pod in self.pods: ep = "{}:{}".format(pod.addr, pod.port) - assert pod.port != None and pod.addr != None, "{} not a valid endpoint".format( - ep) + assert ( + pod.port != None and pod.addr != None + ), "{} not a valid endpoint".format(ep) r.append(ep) return r @@ -176,7 +195,6 @@ class Cluster(object): class JobServer(object): - def __init__(self): self.endpoint = None @@ -191,22 +209,21 @@ class JobServer(object): class Trainer(object): - def __init__(self): self.gpus = [] self.endpoint = None self.rank = None def __str__(self): - return "gpu:{} endpoint:{} rank:{}".format(self.gpus, self.endpoint, - self.rank) + return "gpu:{} endpoint:{} rank:{}".format( + self.gpus, self.endpoint, self.rank + ) def __eq__(self, t): if len(self.gpus) != len(t.gpus): return False - if self.endpoint != t.endpoint or \ - self.rank != t.rank: + if self.endpoint != t.endpoint or self.rank != t.rank: return False for a, b in zip(self.gpus, t.gpus): @@ -223,7 +240,6 @@ class Trainer(object): class Pod(object): - def __init__(self): self.rank = None self.id = None @@ -233,27 +249,38 @@ class Pod(object): self.gpus = [] def __str__(self): - return "rank:{} id:{} addr:{} port:{} visible_gpu:{} trainers:{}".format( - self.rank, self.id, self.addr, self.port, self.gpus, - [str(t) for t in self.trainers]) + return ( + "rank:{} id:{} addr:{} port:{} visible_gpu:{} trainers:{}".format( + self.rank, + self.id, + self.addr, + self.port, + self.gpus, + [str(t) for t in self.trainers], + ) + ) def __eq__(self, pod): - if self.rank != pod.rank or \ - self.id != pod.id or \ - self.addr != pod.addr or \ - self.port != pod.port: + if ( + self.rank != pod.rank + or self.id != pod.id + or self.addr != pod.addr + or self.port != pod.port + ): logger.debug("pod {} != {}".format(self, pod)) return False if len(self.trainers) != len(pod.trainers): - logger.debug("trainers {} != {}".format(self.trainers, - pod.trainers)) + logger.debug( + "trainers {} != {}".format(self.trainers, pod.trainers) + ) return False for i in range(len(self.trainers)): if self.trainers[i] != pod.trainers[i]: - logger.debug("trainer {} != {}".format(self.trainers[i], - pod.trainers[i])) + logger.debug( + "trainer {} != {}".format(self.trainers[i], pod.trainers[i]) + ) return False return True @@ -310,7 +337,7 @@ def terminate_local_procs(procs): p.log_fn.close() logger.debug("terminate process id:{}".format(p.proc.pid)) - #wait all process terminiated + # wait all process terminiated time.sleep(3) for step in range(0, 50): alive = False @@ -347,15 +374,16 @@ def add_arguments(argname, type, default, help, argparser, **kwargs): args = parser.parse_args() """ type = strtobool if type == bool else type - argparser.add_argument("--" + argname, - default=default, - type=type, - help=help + ' Default: %(default)s.', - **kwargs) + argparser.add_argument( + "--" + argname, + default=default, + type=type, + help=help + ' Default: %(default)s.', + **kwargs + ) def find_free_ports(num): - def __free_port(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(('', 0)) @@ -386,30 +414,30 @@ def _prepare_trainer_env(cluster, trainer, backend=None): backend = get_backend_by_compile_flag() # for compatibility if backend == 'bkcl': proc_env = { - "FLAGS_selected_xpus": - "%s" % ",".join([str(g) for g in trainer.gpus]), + "FLAGS_selected_xpus": "%s" + % ",".join([str(g) for g in trainer.gpus]), "PADDLE_TRAINER_ID": "%d" % trainer.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % trainer.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } elif backend == 'nccl': proc_env = { - "FLAGS_selected_gpus": - "%s" % ",".join([str(g) for g in trainer.gpus]), + "FLAGS_selected_gpus": "%s" + % ",".join([str(g) for g in trainer.gpus]), "PADDLE_TRAINER_ID": "%d" % trainer.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % trainer.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } elif backend == 'cncl': proc_env = { - "FLAGS_selected_mlus": - "%s" % ",".join([str(g) for g in trainer.gpus]), + "FLAGS_selected_mlus": "%s" + % ",".join([str(g) for g in trainer.gpus]), "PADDLE_TRAINER_ID": "%d" % trainer.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % trainer.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } elif backend == 'gloo': # NOTE (xiongkun) default fall back into cpu only @@ -418,8 +446,7 @@ def _prepare_trainer_env(cluster, trainer, backend=None): "PADDLE_CURRENT_ENDPOINT": "%s" % trainer.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), - "PADDLE_DISTRI_BACKEND": - backend, # only add here, other will be auto + "PADDLE_DISTRI_BACKEND": backend, # only add here, other will be auto } else: raise ValueError("backend must be one of 'gloo, nccl, bkcl'") @@ -428,7 +455,6 @@ def _prepare_trainer_env(cluster, trainer, backend=None): class TrainerProc(object): - def __init__(self): self.proc = None self.log_fn = None @@ -438,16 +464,14 @@ class TrainerProc(object): self.cmd = None -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - log_dir=None): +def start_local_trainers( + cluster, pod, training_script, training_script_args, log_dir=None +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -493,8 +517,9 @@ def pull_worker_log(tp): except UnicodeEncodeError: sys.stdout.write( 'UnicodeEncodeError occurs at this line. ' - 'Please refer to the original log file "%s"\n' % - tp.log_fn.name) + 'Please refer to the original log file "%s"\n' + % tp.log_fn.name + ) tp.log_offset = fin.tell() @@ -525,14 +550,18 @@ def watch_local_trainers(procs, nranks): raise except SystemExit: logger.error( - "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log." - .format(nranks, error_rank)) + "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log.".format( + nranks, error_rank + ) + ) terminate_local_procs(procs) raise except: logger.error( - "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log." - .format(nranks, error_rank)) + "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log.".format( + nranks, error_rank + ) + ) terminate_local_procs(procs) raise diff --git a/python/paddle/distributed/utils/moe_utils.py b/python/paddle/distributed/utils/moe_utils.py index d6dbfdfab58..cd7c0e758d4 100644 --- a/python/paddle/distributed/utils/moe_utils.py +++ b/python/paddle/distributed/utils/moe_utils.py @@ -18,11 +18,9 @@ from paddle.fluid.data_feeder import check_variable_and_dtype from paddle import _legacy_C_ops -def global_scatter(x, - local_count, - global_count, - group=None, - use_calc_stream=True): +def global_scatter( + x, local_count, global_count, group=None, use_calc_stream=True +): """ The global_scatter operator distributes the data of x to n_expert * world_size experts according to local_count, and then receives data according to global_count. The expert refers to a user-defined expert network, @@ -110,42 +108,49 @@ def global_scatter(x, ring_id = 0 if group is None else group.id if _non_static_mode(): - return _legacy_C_ops.global_scatter(x, local_count, \ - global_count, \ - 'use_calc_stream', use_calc_stream, \ - 'ring_id', ring_id) + return _legacy_C_ops.global_scatter( + x, + local_count, + global_count, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + ) else: op_type = 'global_scatter' check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'global_scatter') - check_variable_and_dtype(local_count, 'local_count', ['int64'], - 'global_scatter') - check_variable_and_dtype(global_count, 'global_count', ['int64'], - 'global_scatter') + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'global_scatter', + ) + check_variable_and_dtype( + local_count, 'local_count', ['int64'], 'global_scatter' + ) + check_variable_and_dtype( + global_count, 'global_count', ['int64'], 'global_scatter' + ) helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=op_type, - inputs={ - 'X': [x], - 'local_count': [local_count], - 'global_count': [global_count], - }, - outputs={'Out': [out]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + helper.append_op( + type=op_type, + inputs={ + 'X': [x], + 'local_count': [local_count], + 'global_count': [global_count], + }, + outputs={'Out': [out]}, + attrs={'ring_id': ring_id, 'use_calc_stream': use_calc_stream}, + ) return out -def global_gather(x, - local_count, - global_count, - group=None, - use_calc_stream=True): +def global_gather( + x, local_count, global_count, group=None, use_calc_stream=True +): """ The global_gather operator gathers the data of x into n_expert * world_size experts according to global_count, and then receives data according to local_count. The expert refers to a user-defined expert network, n_expert refers to the number of expert networks owned by each card, and world_size refers to the number of graphics cards running the network. @@ -223,33 +228,45 @@ def global_gather(x, ring_id = 0 if group is None else group.id if _non_static_mode(): - return _legacy_C_ops.global_gather(x, local_count, \ - global_count, \ - 'use_calc_stream', use_calc_stream, \ - 'ring_id', ring_id) + return _legacy_C_ops.global_gather( + x, + local_count, + global_count, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + ) else: op_type = 'global_gather' check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'global_gather') + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'global_gather', + ) - check_variable_and_dtype(local_count, 'local_count', ['int64'], - 'global_gather') + check_variable_and_dtype( + local_count, 'local_count', ['int64'], 'global_gather' + ) - check_variable_and_dtype(global_count, 'global_count', ['int64'], - 'global_gather') + check_variable_and_dtype( + global_count, 'global_count', ['int64'], 'global_gather' + ) helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=op_type, - inputs={ - 'X': [x], - 'local_count': [local_count], - 'global_count': [global_count] - }, - outputs={'Out': [out]}, - attrs={ - 'ring_id': group, - 'use_calc_stream': use_calc_stream, - }) + helper.append_op( + type=op_type, + inputs={ + 'X': [x], + 'local_count': [local_count], + 'global_count': [global_count], + }, + outputs={'Out': [out]}, + attrs={ + 'ring_id': group, + 'use_calc_stream': use_calc_stream, + }, + ) return out diff --git a/python/paddle/distribution/__init__.py b/python/paddle/distribution/__init__.py index 805675a0f23..77b83fa6a94 100644 --- a/python/paddle/distribution/__init__.py +++ b/python/paddle/distribution/__init__.py @@ -25,15 +25,26 @@ from paddle.distribution.lognormal import LogNormal from paddle.distribution.multinomial import Multinomial from paddle.distribution.normal import Normal from paddle.distribution.transform import * # noqa: F403 -from paddle.distribution.transformed_distribution import \ - TransformedDistribution +from paddle.distribution.transformed_distribution import TransformedDistribution from paddle.distribution.uniform import Uniform from paddle.distribution.laplace import Laplace __all__ = [ # noqa - 'Beta', 'Categorical', 'Dirichlet', 'Distribution', 'ExponentialFamily', - 'Multinomial', 'Normal', 'Uniform', 'kl_divergence', 'register_kl', - 'Independent', 'TransformedDistribution', 'Laplace', 'LogNormal', 'Gumbel' + 'Beta', + 'Categorical', + 'Dirichlet', + 'Distribution', + 'ExponentialFamily', + 'Multinomial', + 'Normal', + 'Uniform', + 'kl_divergence', + 'register_kl', + 'Independent', + 'TransformedDistribution', + 'Laplace', + 'LogNormal', + 'Gumbel', ] __all__.extend(transform.__all__) diff --git a/python/paddle/distribution/beta.py b/python/paddle/distribution/beta.py index 3474ee87b4c..f0eaa1dfe17 100644 --- a/python/paddle/distribution/beta.py +++ b/python/paddle/distribution/beta.py @@ -92,20 +92,19 @@ class Beta(exponential_family.ExponentialFamily): self.alpha, self.beta = paddle.broadcast_tensors([alpha, beta]) self._dirichlet = dirichlet.Dirichlet( - paddle.stack([self.alpha, self.beta], -1)) + paddle.stack([self.alpha, self.beta], -1) + ) super(Beta, self).__init__(self._dirichlet._batch_shape) @property def mean(self): - """Mean of beta distribution. - """ + """Mean of beta distribution.""" return self.alpha / (self.alpha + self.beta) @property def variance(self): - """Variance of beat distribution - """ + """Variance of beat distribution""" sum = self.alpha + self.beta return self.alpha * self.beta / (sum.pow(2) * (sum + 1)) diff --git a/python/paddle/distribution/categorical.py b/python/paddle/distribution/categorical.py index 269e2b55671..bd6c570ca1f 100644 --- a/python/paddle/distribution/categorical.py +++ b/python/paddle/distribution/categorical.py @@ -15,9 +15,9 @@ import numpy as np import paddle from paddle.distribution import distribution -from paddle.fluid.data_feeder import (check_type, convert_dtype) +from paddle.fluid.data_feeder import check_type, convert_dtype from paddle.fluid.framework import _non_static_mode -from paddle.fluid.layers import (ops, tensor) +from paddle.fluid.layers import ops, tensor from paddle.tensor import multinomial @@ -90,9 +90,12 @@ class Categorical(distribution.Distribution): name(str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. """ if not _non_static_mode(): - check_type(logits, 'logits', - (np.ndarray, tensor.Variable, list, tuple), - 'Categorical') + check_type( + logits, + 'logits', + (np.ndarray, tensor.Variable, list, tuple), + 'Categorical', + ) self.name = name if name is not None else 'Categorical' self.dtype = 'float32' @@ -101,8 +104,10 @@ class Categorical(distribution.Distribution): self.logits = logits self.dtype = convert_dtype(logits.dtype) else: - if isinstance(logits, np.ndarray) and str( - logits.dtype) in ['float32', 'float64']: + if isinstance(logits, np.ndarray) and str(logits.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = logits.dtype self.logits = self._to_tensor(logits)[0] if self.dtype != convert_dtype(self.logits.dtype): @@ -149,13 +154,15 @@ class Categorical(distribution.Distribution): if len(logits_shape) > 1: sample_shape = shape + logits_shape[:-1] logits = paddle.reshape( - self.logits, [np.prod(logits_shape[:-1]), logits_shape[-1]]) + self.logits, [np.prod(logits_shape[:-1]), logits_shape[-1]] + ) else: sample_shape = shape logits = self.logits - sample_index = multinomial(self._logits_to_probs(logits), num_samples, - True) + sample_index = multinomial( + self._logits_to_probs(logits), num_samples, True + ) # multinomial sample shape is (logits.shape[:-1], num_samples), need to # tanspose to (num_samples, logits.shape[:-1]) @@ -203,21 +210,22 @@ class Categorical(distribution.Distribution): if not _non_static_mode(): check_type(other, 'other', Categorical, 'kl_divergence') - logits = self.logits - \ - paddle.max(self.logits, axis=-1, keepdim=True) + logits = self.logits - paddle.max(self.logits, axis=-1, keepdim=True) other_logits = other.logits - paddle.max( - other.logits, axis=-1, keepdim=True) + other.logits, axis=-1, keepdim=True + ) e_logits = ops.exp(logits) other_e_logits = ops.exp(other_logits) z = paddle.sum(e_logits, axis=-1, keepdim=True) other_z = paddle.sum(other_e_logits, axis=-1, keepdim=True) prob = e_logits / z kl = paddle.sum( - prob * - (logits - paddle.log(z) - other_logits + paddle.log(other_z)), + prob + * (logits - paddle.log(z) - other_logits + paddle.log(other_z)), axis=-1, keepdim=True, - name=name) + name=name, + ) return kl @@ -246,8 +254,7 @@ class Categorical(distribution.Distribution): """ name = self.name + '_entropy' - logits = self.logits - \ - paddle.max(self.logits, axis=-1, keepdim=True) + logits = self.logits - paddle.max(self.logits, axis=-1, keepdim=True) e_logits = ops.exp(logits) z = paddle.sum(e_logits, axis=-1, keepdim=True) prob = e_logits / z @@ -293,17 +300,20 @@ class Categorical(distribution.Distribution): """ name = self.name + '_probs' if len(self._prob.shape) == 1: # batch_shape is empty - return paddle.gather(self._prob, - value.reshape([-1], name=name), - name=name).reshape(value.shape, name=name) + return paddle.gather( + self._prob, value.reshape([-1], name=name), name=name + ).reshape(value.shape, name=name) else: if len(value.shape) == 1: return paddle.take_along_axis( self._prob, - paddle.reshape(value, - (len(self._prob.shape) - 1) * [1] + [-1], - name=name), - axis=-1) + paddle.reshape( + value, + (len(self._prob.shape) - 1) * [1] + [-1], + name=name, + ), + axis=-1, + ) else: return paddle.take_along_axis(self._prob, value, axis=-1) diff --git a/python/paddle/distribution/constraint.py b/python/paddle/distribution/constraint.py index 4cde3d30a56..1891d201779 100644 --- a/python/paddle/distribution/constraint.py +++ b/python/paddle/distribution/constraint.py @@ -15,21 +15,18 @@ import paddle class Constraint(object): - """Constraint condition for random variable. - """ + """Constraint condition for random variable.""" def __call__(self, value): raise NotImplementedError class Real(Constraint): - def __call__(self, value): return value == value class Range(Constraint): - def __init__(self, lower, upper): self._lower = lower self._upper = upper @@ -40,16 +37,15 @@ class Range(Constraint): class Positive(Constraint): - def __call__(self, value): - return value >= 0. + return value >= 0.0 class Simplex(Constraint): - def __call__(self, value): - return paddle.all(value >= 0, - axis=-1) and ((value.sum(-1) - 1).abs() < 1e-6) + return paddle.all(value >= 0, axis=-1) and ( + (value.sum(-1) - 1).abs() < 1e-6 + ) real = Real() diff --git a/python/paddle/distribution/dirichlet.py b/python/paddle/distribution/dirichlet.py index 7a8e9180968..c57bc41c018 100644 --- a/python/paddle/distribution/dirichlet.py +++ b/python/paddle/distribution/dirichlet.py @@ -73,11 +73,13 @@ class Dirichlet(exponential_family.ExponentialFamily): def __init__(self, concentration): if concentration.dim() < 1: raise ValueError( - "`concentration` parameter must be at least one dimensional") + "`concentration` parameter must be at least one dimensional" + ) self.concentration = concentration - super(Dirichlet, self).__init__(concentration.shape[:-1], - concentration.shape[-1:]) + super(Dirichlet, self).__init__( + concentration.shape[:-1], concentration.shape[-1:] + ) @property def mean(self): @@ -97,7 +99,8 @@ class Dirichlet(exponential_family.ExponentialFamily): """ concentration0 = self.concentration.sum(-1, keepdim=True) return (self.concentration * (concentration0 - self.concentration)) / ( - concentration0.pow(2) * (concentration0 + 1)) + concentration0.pow(2) * (concentration0 + 1) + ) def sample(self, shape=()): """Sample from dirichlet distribution. @@ -125,9 +128,11 @@ class Dirichlet(exponential_family.ExponentialFamily): Args: value (Tensor): Value to be evaluated. """ - return ((paddle.log(value) * (self.concentration - 1.0)).sum(-1) + - paddle.lgamma(self.concentration.sum(-1)) - - paddle.lgamma(self.concentration).sum(-1)) + return ( + (paddle.log(value) * (self.concentration - 1.0)).sum(-1) + + paddle.lgamma(self.concentration.sum(-1)) + - paddle.lgamma(self.concentration).sum(-1) + ) def entropy(self): """Entropy of Dirichlet distribution. @@ -137,15 +142,18 @@ class Dirichlet(exponential_family.ExponentialFamily): """ concentration0 = self.concentration.sum(-1) k = self.concentration.shape[-1] - return (paddle.lgamma(self.concentration).sum(-1) - - paddle.lgamma(concentration0) - - (k - concentration0) * paddle.digamma(concentration0) - - ((self.concentration - 1.0) * - paddle.digamma(self.concentration)).sum(-1)) + return ( + paddle.lgamma(self.concentration).sum(-1) + - paddle.lgamma(concentration0) + - (k - concentration0) * paddle.digamma(concentration0) + - ( + (self.concentration - 1.0) * paddle.digamma(self.concentration) + ).sum(-1) + ) @property def _natural_parameters(self): - return (self.concentration, ) + return (self.concentration,) def _log_normalizer(self, x): return x.lgamma().sum(-1) - paddle.lgamma(x.sum(-1)) @@ -154,8 +162,9 @@ class Dirichlet(exponential_family.ExponentialFamily): def _dirichlet(concentration, name=None): op_type = 'dirichlet' - check_variable_and_dtype(concentration, 'concentration', - ['float32', 'float64'], op_type) + check_variable_and_dtype( + concentration, 'concentration', ['float32', 'float64'], op_type + ) if in_dygraph_mode(): return paddle._C_ops.dirichlet(concentration) @@ -164,9 +173,12 @@ def _dirichlet(concentration, name=None): else: helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference( - dtype=concentration.dtype) - helper.append_op(type=op_type, - inputs={"Alpha": concentration}, - outputs={'Out': out}, - attrs={}) + dtype=concentration.dtype + ) + helper.append_op( + type=op_type, + inputs={"Alpha": concentration}, + outputs={'Out': out}, + attrs={}, + ) return out diff --git a/python/paddle/distribution/distribution.py b/python/paddle/distribution/distribution.py index 14861d3794e..65a899c0fac 100644 --- a/python/paddle/distribution/distribution.py +++ b/python/paddle/distribution/distribution.py @@ -24,8 +24,12 @@ import warnings import numpy as np import paddle from paddle import _C_ops, _legacy_C_ops -from paddle.fluid.data_feeder import (check_variable_and_dtype, convert_dtype) -from paddle.fluid.framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph +from paddle.fluid.data_feeder import check_variable_and_dtype, convert_dtype +from paddle.fluid.framework import ( + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, +) from paddle.fluid.layers import tensor @@ -45,10 +49,16 @@ class Distribution(object): def __init__(self, batch_shape=(), event_shape=()): - self._batch_shape = batch_shape if isinstance( - batch_shape, tuple) else tuple(batch_shape) - self._event_shape = event_shape if isinstance( - event_shape, tuple) else tuple(event_shape) + self._batch_shape = ( + batch_shape + if isinstance(batch_shape, tuple) + else tuple(batch_shape) + ) + self._event_shape = ( + event_shape + if isinstance(event_shape, tuple) + else tuple(event_shape) + ) super(Distribution, self).__init__() @@ -147,7 +157,8 @@ class Distribution(object): if is_variable and is_number: raise ValueError( - 'if one argument is Tensor, all arguments should be Tensor') + 'if one argument is Tensor, all arguments should be Tensor' + ) return is_variable @@ -162,15 +173,17 @@ class Distribution(object): """ numpy_args = [] variable_args = [] - tmp = 0. + tmp = 0.0 for arg in args: if isinstance(arg, float): arg = [arg] if not isinstance(arg, (list, tuple, np.ndarray, tensor.Variable)): raise TypeError( - "Type of input args must be float, list, numpy.ndarray or Tensor, but received type {}" - .format(type(arg))) + "Type of input args must be float, list, numpy.ndarray or Tensor, but received type {}".format( + type(arg) + ) + ) arg_np = np.array(arg) arg_dtype = arg_np.dtype @@ -208,20 +221,24 @@ class Distribution(object): value (Tensor): Change value's dtype if value's dtype is different from param. """ if _non_static_mode(): - if value.dtype != param.dtype and convert_dtype( - value.dtype) in ['float32', 'float64']: + if value.dtype != param.dtype and convert_dtype(value.dtype) in [ + 'float32', + 'float64', + ]: warnings.warn( "dtype of input 'value' needs to be the same as parameters of distribution class. dtype of 'value' will be converted." ) if in_dygraph_mode(): return _C_ops.cast(value, param.dtype) if _in_legacy_dygraph(): - return _legacy_C_ops.cast(value, 'in_dtype', value.dtype, - 'out_dtype', param.dtype) + return _legacy_C_ops.cast( + value, 'in_dtype', value.dtype, 'out_dtype', param.dtype + ) return value - check_variable_and_dtype(value, 'value', ['float32', 'float64'], - 'log_prob') + check_variable_and_dtype( + value, 'value', ['float32', 'float64'], 'log_prob' + ) if value.dtype != param.dtype: warnings.warn( "dtype of input 'value' needs to be the same as parameters of distribution class. dtype of 'value' will be converted." @@ -236,8 +253,11 @@ class Distribution(object): multi-dimensional, values of last axis denote the probabilities of occurrence of each of the events. """ - return (paddle.log(probs) - paddle.log1p(-probs)) \ - if is_binary else paddle.log(probs) + return ( + (paddle.log(probs) - paddle.log1p(-probs)) + if is_binary + else paddle.log(probs) + ) def _logits_to_probs(self, logits, is_binary=False): r""" @@ -245,5 +265,8 @@ class Distribution(object): log odds, whereas for the multi-dimensional case, the values along the last dimension denote the log probabilities of the events. """ - return paddle.nn.functional.sigmoid(logits) \ - if is_binary else paddle.nn.functional.softmax(logits, axis=-1) + return ( + paddle.nn.functional.sigmoid(logits) + if is_binary + else paddle.nn.functional.softmax(logits, axis=-1) + ) diff --git a/python/paddle/distribution/exponential_family.py b/python/paddle/distribution/exponential_family.py index 301c0cea13a..0939e73f726 100644 --- a/python/paddle/distribution/exponential_family.py +++ b/python/paddle/distribution/exponential_family.py @@ -62,9 +62,9 @@ class ExponentialFamily(distribution.Distribution): log_norm = self._log_normalizer(*natural_parameters) if _non_static_mode(): - grads = paddle.grad(log_norm.sum(), - natural_parameters, - create_graph=True) + grads = paddle.grad( + log_norm.sum(), natural_parameters, create_graph=True + ) else: grads = paddle.static.gradients(log_norm.sum(), natural_parameters) diff --git a/python/paddle/distribution/gumbel.py b/python/paddle/distribution/gumbel.py index 225e327cc91..1f16fd5529b 100644 --- a/python/paddle/distribution/gumbel.py +++ b/python/paddle/distribution/gumbel.py @@ -70,7 +70,8 @@ class Gumbel(TransformedDistribution): if not isinstance(loc, (numbers.Real, framework.Variable)): raise TypeError( - f"Expected type of loc is Real|Variable, but got {type(loc)}") + f"Expected type of loc is Real|Variable, but got {type(loc)}" + ) if not isinstance(scale, (numbers.Real, framework.Variable)): raise TypeError( f"Expected type of scale is Real|Variable, but got {type(scale)}" @@ -90,7 +91,8 @@ class Gumbel(TransformedDistribution): finfo = np.finfo(dtype='float32') self.base_dist = paddle.distribution.Uniform( paddle.full_like(self.loc, float(finfo.tiny)), - paddle.full_like(self.loc, float(1 - finfo.eps))) + paddle.full_like(self.loc, float(1 - finfo.eps)), + ) self.transforms = () @@ -136,9 +138,11 @@ class Gumbel(TransformedDistribution): Tensor: The variance value. """ - temp = paddle.full(shape=self.loc.shape, - fill_value=math.pi * math.pi, - dtype=self.scale.dtype) + temp = paddle.full( + shape=self.loc.shape, + fill_value=math.pi * math.pi, + dtype=self.scale.dtype, + ) return paddle.pow(self.scale, 2) * temp / 6 @@ -230,13 +234,19 @@ class Gumbel(TransformedDistribution): """ exp_trans = paddle.distribution.ExpTransform() affine_trans_1 = paddle.distribution.AffineTransform( - paddle.full(shape=self.scale.shape, - fill_value=0, - dtype=self.loc.dtype), -paddle.ones_like(self.scale)) + paddle.full( + shape=self.scale.shape, fill_value=0, dtype=self.loc.dtype + ), + -paddle.ones_like(self.scale), + ) affine_trans_2 = paddle.distribution.AffineTransform( - self.loc, -self.scale) + self.loc, -self.scale + ) return affine_trans_2.forward( exp_trans.inverse( affine_trans_1.forward( - exp_trans.inverse(self._base.sample(shape))))) + exp_trans.inverse(self._base.sample(shape)) + ) + ) + ) diff --git a/python/paddle/distribution/independent.py b/python/paddle/distribution/independent.py index 9f02d802fc8..a12ab32b85a 100644 --- a/python/paddle/distribution/independent.py +++ b/python/paddle/distribution/independent.py @@ -61,11 +61,14 @@ class Independent(distribution.Distribution): self._reinterpreted_batch_rank = reinterpreted_batch_rank shape = base.batch_shape + base.event_shape - super(Independent, - self).__init__(batch_shape=shape[:len(base.batch_shape) - - reinterpreted_batch_rank], - event_shape=shape[len(base.batch_shape) - - reinterpreted_batch_rank:]) + super(Independent, self).__init__( + batch_shape=shape[ + : len(base.batch_shape) - reinterpreted_batch_rank + ], + event_shape=shape[ + len(base.batch_shape) - reinterpreted_batch_rank : + ], + ) @property def mean(self): @@ -79,15 +82,17 @@ class Independent(distribution.Distribution): return self._base.sample(shape) def log_prob(self, value): - return self._sum_rightmost(self._base.log_prob(value), - self._reinterpreted_batch_rank) + return self._sum_rightmost( + self._base.log_prob(value), self._reinterpreted_batch_rank + ) def prob(self, value): return self.log_prob(value).exp() def entropy(self): - return self._sum_rightmost(self._base.entropy(), - self._reinterpreted_batch_rank) + return self._sum_rightmost( + self._base.entropy(), self._reinterpreted_batch_rank + ) def _sum_rightmost(self, value, n): return value.sum(list(range(-n, 0))) if n > 0 else value diff --git a/python/paddle/distribution/kl.py b/python/paddle/distribution/kl.py index aa288a57ba0..4f95366391a 100644 --- a/python/paddle/distribution/kl.py +++ b/python/paddle/distribution/kl.py @@ -85,8 +85,9 @@ def register_kl(cls_p, cls_q): def kl_beta_beta(): pass # insert implementation here """ - if (not issubclass(cls_p, Distribution) - or not issubclass(cls_q, Distribution)): + if not issubclass(cls_p, Distribution) or not issubclass( + cls_q, Distribution + ): raise TypeError('cls_p and cls_q must be subclass of Distribution') def decorator(f): @@ -100,8 +101,11 @@ def _dispatch(cls_p, cls_q): """Multiple dispatch into concrete implement function.""" # find all matched super class pair of p and q - matchs = [(super_p, super_q) for super_p, super_q in _REGISTER_TABLE - if issubclass(cls_p, super_p) and issubclass(cls_q, super_q)] + matchs = [ + (super_p, super_q) + for super_p, super_q in _REGISTER_TABLE + if issubclass(cls_p, super_p) and issubclass(cls_q, super_q) + ] if not matchs: raise NotImplementedError @@ -110,16 +114,20 @@ def _dispatch(cls_p, cls_q): if _REGISTER_TABLE[left_p, left_q] is not _REGISTER_TABLE[right_p, right_q]: warnings.warn( - 'Ambiguous kl_divergence({}, {}). Please register_kl({}, {})'. - format(cls_p.__name__, cls_q.__name__, left_p.__name__, - right_q.__name__), RuntimeWarning) + 'Ambiguous kl_divergence({}, {}). Please register_kl({}, {})'.format( + cls_p.__name__, + cls_q.__name__, + left_p.__name__, + right_q.__name__, + ), + RuntimeWarning, + ) return _REGISTER_TABLE[left_p, left_q] @functools.total_ordering class _Compare(object): - def __init__(self, *classes): self.classes = classes @@ -137,22 +145,33 @@ class _Compare(object): @register_kl(Beta, Beta) def _kl_beta_beta(p, q): - return ((q.alpha.lgamma() + q.beta.lgamma() + (p.alpha + p.beta).lgamma()) - - (p.alpha.lgamma() + p.beta.lgamma() + (q.alpha + q.beta).lgamma()) + - ((p.alpha - q.alpha) * p.alpha.digamma()) + - ((p.beta - q.beta) * p.beta.digamma()) + - (((q.alpha + q.beta) - (p.alpha + p.beta)) * - (p.alpha + p.beta).digamma())) + return ( + (q.alpha.lgamma() + q.beta.lgamma() + (p.alpha + p.beta).lgamma()) + - (p.alpha.lgamma() + p.beta.lgamma() + (q.alpha + q.beta).lgamma()) + + ((p.alpha - q.alpha) * p.alpha.digamma()) + + ((p.beta - q.beta) * p.beta.digamma()) + + ( + ((q.alpha + q.beta) - (p.alpha + p.beta)) + * (p.alpha + p.beta).digamma() + ) + ) @register_kl(Dirichlet, Dirichlet) def _kl_dirichlet_dirichlet(p, q): return ( - (p.concentration.sum(-1).lgamma() - q.concentration.sum(-1).lgamma()) - - ((p.concentration.lgamma() - q.concentration.lgamma()).sum(-1)) + - (((p.concentration - q.concentration) * - (p.concentration.digamma() - - p.concentration.sum(-1).digamma().unsqueeze(-1))).sum(-1))) + (p.concentration.sum(-1).lgamma() - q.concentration.sum(-1).lgamma()) + - ((p.concentration.lgamma() - q.concentration.lgamma()).sum(-1)) + + ( + ( + (p.concentration - q.concentration) + * ( + p.concentration.digamma() + - p.concentration.sum(-1).digamma().unsqueeze(-1) + ) + ).sum(-1) + ) + ) @register_kl(Categorical, Categorical) @@ -177,8 +196,7 @@ def _kl_laplace_laplace(p, q): @register_kl(ExponentialFamily, ExponentialFamily) def _kl_expfamily_expfamily(p, q): - """Compute kl-divergence using `Bregman divergences `_ - """ + """Compute kl-divergence using `Bregman divergences `_""" if not type(p) == type(q): raise NotImplementedError @@ -194,19 +212,22 @@ def _kl_expfamily_expfamily(p, q): try: if _non_static_mode(): - p_grads = paddle.grad(p_log_norm, - p_natural_params, - create_graph=True) + p_grads = paddle.grad( + p_log_norm, p_natural_params, create_graph=True + ) else: p_grads = paddle.static.gradients(p_log_norm, p_natural_params) except RuntimeError as e: raise TypeError( - "Cann't compute kl_divergence({cls_p}, {cls_q}) use bregman divergence. Please register_kl({cls_p}, {cls_q})." - .format(cls_p=type(p).__name__, cls_q=type(q).__name__)) from e + "Cann't compute kl_divergence({cls_p}, {cls_q}) use bregman divergence. Please register_kl({cls_p}, {cls_q}).".format( + cls_p=type(p).__name__, cls_q=type(q).__name__ + ) + ) from e kl = q._log_normalizer(*q_natural_params) - p_log_norm - for p_param, q_param, p_grad in zip(p_natural_params, q_natural_params, - p_grads): + for p_param, q_param, p_grad in zip( + p_natural_params, q_natural_params, p_grads + ): term = (q_param - p_param) * p_grad kl -= _sum_rightmost(term, len(q.event_shape)) diff --git a/python/paddle/distribution/laplace.py b/python/paddle/distribution/laplace.py index 2b117f93bac..a7a9301724c 100644 --- a/python/paddle/distribution/laplace.py +++ b/python/paddle/distribution/laplace.py @@ -55,7 +55,8 @@ class Laplace(distribution.Distribution): def __init__(self, loc, scale): if not isinstance(loc, (numbers.Real, framework.Variable)): raise TypeError( - f"Expected type of loc is Real|Variable, but got {type(loc)}") + f"Expected type of loc is Real|Variable, but got {type(loc)}" + ) if not isinstance(scale, (numbers.Real, framework.Variable)): raise TypeError( @@ -68,8 +69,9 @@ class Laplace(distribution.Distribution): if isinstance(scale, numbers.Real): scale = paddle.full(shape=(), fill_value=scale) - if (len(scale.shape) > 0 or len(loc.shape) > 0) and (loc.dtype - == scale.dtype): + if (len(scale.shape) > 0 or len(loc.shape) > 0) and ( + loc.dtype == scale.dtype + ): self.loc, self.scale = paddle.broadcast_tensors([loc, scale]) else: self.loc, self.scale = loc, scale @@ -135,10 +137,14 @@ class Laplace(distribution.Distribution): value = paddle.full(shape=(), fill_value=value) if value.dtype != self.scale.dtype: value = paddle.cast(value, self.scale.dtype) - if len(self.scale.shape) > 0 or len(self.loc.shape) > 0 or len( - value.shape) > 0: + if ( + len(self.scale.shape) > 0 + or len(self.loc.shape) > 0 + or len(value.shape) > 0 + ): loc, scale, value = paddle.broadcast_tensors( - [self.loc, self.scale, value]) + [self.loc, self.scale, value] + ) else: loc, scale = self.loc, self.scale @@ -178,7 +184,7 @@ class Laplace(distribution.Distribution): loc, scale, value = self._validate_value(value) log_scale = -paddle.log(2 * scale) - return (log_scale - paddle.abs(value - loc) / scale) + return log_scale - paddle.abs(value - loc) / scale def entropy(self): r"""Entropy of Laplace distribution. @@ -238,8 +244,11 @@ class Laplace(distribution.Distribution): # [0.54758132]) """ loc, scale, value = self._validate_value(value) - iterm = (0.5 * (value - loc).sign() * - paddle.expm1(-(value - loc).abs() / scale)) + iterm = ( + 0.5 + * (value - loc).sign() + * paddle.expm1(-(value - loc).abs() / scale) + ) return 0.5 - iterm @@ -276,7 +285,7 @@ class Laplace(distribution.Distribution): loc, scale, value = self._validate_value(value) term = value - 0.5 - return (loc - scale * (term).sign() * paddle.log1p(-2 * term.abs())) + return loc - scale * (term).sign() * paddle.log1p(-2 * term.abs()) def sample(self, shape=()): r"""Generate samples of the specified shape. @@ -299,7 +308,8 @@ class Laplace(distribution.Distribution): """ if not isinstance(shape, tuple): raise TypeError( - f'Expected shape should be tuple[int], but got {type(shape)}') + f'Expected shape should be tuple[int], but got {type(shape)}' + ) with paddle.no_grad(): return self.rsample(shape) @@ -325,19 +335,22 @@ class Laplace(distribution.Distribution): """ eps = self._get_eps() - shape = self._extend_shape(shape) or (1, ) - uniform = paddle.uniform(shape=shape, - min=float(np.nextafter(-1, 1)) + eps / 2, - max=1. - eps / 2, - dtype=self.loc.dtype) + shape = self._extend_shape(shape) or (1,) + uniform = paddle.uniform( + shape=shape, + min=float(np.nextafter(-1, 1)) + eps / 2, + max=1.0 - eps / 2, + dtype=self.loc.dtype, + ) if len(self.scale.shape) == 0 and len(self.loc.shape) == 0: loc, scale, uniform = paddle.broadcast_tensors( - [self.loc, self.scale, uniform]) + [self.loc, self.scale, uniform] + ) else: loc, scale = self.loc, self.scale - return (loc - scale * uniform.sign() * paddle.log1p(-uniform.abs())) + return loc - scale * uniform.sign() * paddle.log1p(-uniform.abs()) def _get_eps(self): """ @@ -351,8 +364,10 @@ class Laplace(distribution.Distribution): Float: An eps value by different data types. """ eps = 1.19209e-07 - if (self.loc.dtype == paddle.float64 - or self.loc.dtype == paddle.complex128): + if ( + self.loc.dtype == paddle.float64 + or self.loc.dtype == paddle.complex128 + ): eps = 2.22045e-16 return eps @@ -400,7 +415,7 @@ class Laplace(distribution.Distribution): var_ratio = other.scale / self.scale t = paddle.abs(self.loc - other.loc) - term1 = ((self.scale * paddle.exp(-t / self.scale) + t) / other.scale) + term1 = (self.scale * paddle.exp(-t / self.scale) + t) / other.scale term2 = paddle.log(var_ratio) return term1 + term2 - 1 diff --git a/python/paddle/distribution/lognormal.py b/python/paddle/distribution/lognormal.py index b171e1ecbc6..f49eee4d2e0 100644 --- a/python/paddle/distribution/lognormal.py +++ b/python/paddle/distribution/lognormal.py @@ -15,8 +15,7 @@ import paddle from paddle.distribution.normal import Normal from paddle.distribution.transform import ExpTransform -from paddle.distribution.transformed_distribution import \ - TransformedDistribution +from paddle.distribution.transformed_distribution import TransformedDistribution class LogNormal(TransformedDistribution): @@ -104,8 +103,9 @@ class LogNormal(TransformedDistribution): Returns: Tensor: variance value. """ - return (paddle.expm1(self._base.variance) * - paddle.exp(2 * self._base.mean + self._base.variance)) + return paddle.expm1(self._base.variance) * paddle.exp( + 2 * self._base.mean + self._base.variance + ) def entropy(self): r"""Shannon entropy in nats. diff --git a/python/paddle/distribution/multinomial.py b/python/paddle/distribution/multinomial.py index 22f8aa820f9..56bb3da4481 100644 --- a/python/paddle/distribution/multinomial.py +++ b/python/paddle/distribution/multinomial.py @@ -14,6 +14,7 @@ import paddle from paddle.distribution import categorical, distribution + try: from collections.abc import Iterable except: @@ -77,12 +78,14 @@ class Multinomial(distribution.Distribution): if probs.dim() < 1: raise ValueError( - 'probs parameter shoule not be none and over one dimension') + 'probs parameter shoule not be none and over one dimension' + ) self.probs = probs / probs.sum(-1, keepdim=True) self.total_count = total_count self._categorical = categorical.Categorical( - logits=self._probs_to_logits(probs)) + logits=self._probs_to_logits(probs) + ) super(Multinomial, self).__init__(probs.shape[:-1], probs.shape[-1:]) @@ -128,11 +131,15 @@ class Multinomial(distribution.Distribution): value = paddle.cast(value, self.probs.dtype) logits, value = paddle.broadcast_tensors( - [paddle.log(self.probs), value]) + [paddle.log(self.probs), value] + ) logits[(value == 0) & (paddle.isinf(logits))] = 0 - return (paddle.lgamma(value.sum(-1) + 1) - - paddle.lgamma(value + 1).sum(-1) + (value * logits).sum(-1)) + return ( + paddle.lgamma(value.sum(-1) + 1) + - paddle.lgamma(value + 1).sum(-1) + + (value * logits).sum(-1) + ) def sample(self, shape=()): """draw sample data from multinomial distribution @@ -143,11 +150,17 @@ class Multinomial(distribution.Distribution): if not isinstance(shape, Iterable): raise TypeError('sample shape must be Iterable object.') - samples = self._categorical.sample([ - self.total_count, - ] + list(shape)) - return paddle.nn.functional.one_hot(samples, self.probs.shape[-1]).cast( - self.probs.dtype).sum(0) + samples = self._categorical.sample( + [ + self.total_count, + ] + + list(shape) + ) + return ( + paddle.nn.functional.one_hot(samples, self.probs.shape[-1]) + .cast(self.probs.dtype) + .sum(0) + ) def entropy(self): """entropy of multinomial distribution @@ -155,18 +168,18 @@ class Multinomial(distribution.Distribution): Returns: Tensor: entropy value """ - n = paddle.full(shape=[1], - fill_value=self.total_count, - dtype=self.probs.dtype) + n = paddle.full( + shape=[1], fill_value=self.total_count, dtype=self.probs.dtype + ) support = paddle.arange( - self.total_count + 1, - dtype=self.probs.dtype).reshape((-1, ) + - (1, ) * len(self.probs.shape))[1:] + self.total_count + 1, dtype=self.probs.dtype + ).reshape((-1,) + (1,) * len(self.probs.shape))[1:] binomial_pmf = paddle.exp(self._binomial_logpmf(n, support)) - return ((n * self._categorical.entropy() - paddle.lgamma(n + 1)) + - ((binomial_pmf * paddle.lgamma(support + 1)).sum([0, -1]))) + return (n * self._categorical.entropy() - paddle.lgamma(n + 1)) + ( + (binomial_pmf * paddle.lgamma(support + 1)).sum([0, -1]) + ) def _binomial_logpmf(self, count, value): logits = self._probs_to_logits(self.probs, is_binary=True) @@ -175,9 +188,11 @@ class Multinomial(distribution.Distribution): factor_k = paddle.lgamma(value + 1) factor_nmk = paddle.lgamma(count - value + 1) - norm = (count * _clip_by_zero(logits) + - count * paddle.log1p(paddle.exp(-paddle.abs(logits))) - - factor_n) + norm = ( + count * _clip_by_zero(logits) + + count * paddle.log1p(paddle.exp(-paddle.abs(logits))) + - factor_n + ) return value * logits - factor_k - factor_nmk - norm diff --git a/python/paddle/distribution/normal.py b/python/paddle/distribution/normal.py index f76f13b250e..a05c8e4703b 100644 --- a/python/paddle/distribution/normal.py +++ b/python/paddle/distribution/normal.py @@ -16,10 +16,17 @@ import math import numpy as np import paddle from paddle.distribution import distribution -from paddle.fluid.data_feeder import (check_type, convert_dtype) +from paddle.fluid.data_feeder import check_type, convert_dtype from paddle.fluid.framework import _non_static_mode -from paddle.fluid.layers import (elementwise_add, elementwise_div, - elementwise_sub, nn, ops, tensor) +from paddle.fluid.layers import ( + elementwise_add, + elementwise_div, + elementwise_sub, + nn, + ops, + tensor, +) + try: from collections.abc import Iterable except: @@ -89,12 +96,18 @@ class Normal(distribution.Distribution): def __init__(self, loc, scale, name=None): if not _non_static_mode(): - check_type(loc, 'loc', - (int, float, np.ndarray, tensor.Variable, list, tuple), - 'Normal') - check_type(scale, 'scale', - (int, float, np.ndarray, tensor.Variable, list, tuple), - 'Normal') + check_type( + loc, + 'loc', + (int, float, np.ndarray, tensor.Variable, list, tuple), + 'Normal', + ) + check_type( + scale, + 'scale', + (int, float, np.ndarray, tensor.Variable, list, tuple), + 'Normal', + ) self.batch_size_unknown = False self.all_arg_is_float = False @@ -114,11 +127,15 @@ class Normal(distribution.Distribution): else: if isinstance(loc, float) and isinstance(scale, float): self.all_arg_is_float = True - if isinstance(loc, np.ndarray) and str( - loc.dtype) in ['float32', 'float64']: + if isinstance(loc, np.ndarray) and str(loc.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = loc.dtype - elif isinstance(scale, np.ndarray) and str( - scale.dtype) in ['float32', 'float64']: + elif isinstance(scale, np.ndarray) and str(scale.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = scale.dtype # pylint: disable=unbalanced-tuple-unpacking self.loc, self.scale = self._to_tensor(loc, scale) @@ -169,22 +186,21 @@ class Normal(distribution.Distribution): if self.batch_size_unknown: output_shape = shape + batch_shape zero_tmp = tensor.fill_constant_batch_size_like( - self.loc + self.scale, batch_shape + shape, self.dtype, 0.) + self.loc + self.scale, batch_shape + shape, self.dtype, 0.0 + ) zero_tmp_reshape = nn.reshape(zero_tmp, output_shape) zero_tmp_shape = nn.shape(zero_tmp_reshape) - normal_random_tmp = nn.gaussian_random(zero_tmp_shape, - mean=0., - std=1., - seed=seed, - dtype=self.dtype) + normal_random_tmp = nn.gaussian_random( + zero_tmp_shape, mean=0.0, std=1.0, seed=seed, dtype=self.dtype + ) output = normal_random_tmp * (zero_tmp_reshape + self.scale) output = elementwise_add(output, self.loc, name=name) return output else: output_shape = shape + batch_shape output = nn.gaussian_random( - output_shape, mean=0., std=1., seed=seed, dtype=self.dtype) * ( - tensor.zeros(output_shape, dtype=self.dtype) + self.scale) + output_shape, mean=0.0, std=1.0, seed=seed, dtype=self.dtype + ) * (tensor.zeros(output_shape, dtype=self.dtype) + self.scale) output = elementwise_add(output, self.loc, name=name) if self.all_arg_is_float: return nn.reshape(output, shape, name=name) @@ -206,7 +222,7 @@ class Normal(distribution.Distribution): shape = self._extend_shape(tuple(shape)) eps = paddle.normal(shape=shape) - return (self.loc + eps * self.scale) + return self.loc + eps * self.scale def entropy(self): r"""Shannon entropy in nats. @@ -227,13 +243,14 @@ class Normal(distribution.Distribution): """ name = self.name + '_entropy' batch_shape = list((self.loc + self.scale).shape) - zero_tmp = tensor.fill_constant_batch_size_like(self.loc + self.scale, - batch_shape, self.dtype, - 0.) - return elementwise_add(0.5 + zero_tmp, - 0.5 * math.log(2 * math.pi) + nn.log( - (self.scale + zero_tmp)), - name=name) + zero_tmp = tensor.fill_constant_batch_size_like( + self.loc + self.scale, batch_shape, self.dtype, 0.0 + ) + return elementwise_add( + 0.5 + zero_tmp, + 0.5 * math.log(2 * math.pi) + nn.log((self.scale + zero_tmp)), + name=name, + ) def log_prob(self, value): """Log probability density/mass function. @@ -250,10 +267,11 @@ class Normal(distribution.Distribution): var = self.scale * self.scale log_scale = nn.log(self.scale) - return elementwise_sub(-1. * ((value - self.loc) * (value - self.loc)) / - (2. * var), - log_scale + math.log(math.sqrt(2. * math.pi)), - name=name) + return elementwise_sub( + -1.0 * ((value - self.loc) * (value - self.loc)) / (2.0 * var), + log_scale + math.log(math.sqrt(2.0 * math.pi)), + name=name, + ) def probs(self, value): """Probability density/mass function. @@ -269,10 +287,13 @@ class Normal(distribution.Distribution): value = self._check_values_dtype_in_probs(self.loc, value) var = self.scale * self.scale - return elementwise_div(ops.exp(-1. * ((value - self.loc) * - (value - self.loc)) / (2. * var)), - (math.sqrt(2 * math.pi) * self.scale), - name=name) + return elementwise_div( + ops.exp( + -1.0 * ((value - self.loc) * (value - self.loc)) / (2.0 * var) + ), + (math.sqrt(2 * math.pi) * self.scale), + name=name, + ) def kl_divergence(self, other): r"""The KL-divergence between two normal distributions. @@ -312,9 +333,9 @@ class Normal(distribution.Distribution): name = self.name + '_kl_divergence' var_ratio = self.scale / other.scale - var_ratio = (var_ratio * var_ratio) + var_ratio = var_ratio * var_ratio t1 = (self.loc - other.loc) / other.scale - t1 = (t1 * t1) - return elementwise_add(0.5 * var_ratio, - 0.5 * (t1 - 1. - nn.log(var_ratio)), - name=name) + t1 = t1 * t1 + return elementwise_add( + 0.5 * var_ratio, 0.5 * (t1 - 1.0 - nn.log(var_ratio)), name=name + ) diff --git a/python/paddle/distribution/transform.py b/python/paddle/distribution/transform.py index 6cb0e048bab..f31dfe1b154 100644 --- a/python/paddle/distribution/transform.py +++ b/python/paddle/distribution/transform.py @@ -20,20 +20,33 @@ import typing import paddle import paddle.nn.functional as F -from paddle.distribution import (constraint, distribution, - transformed_distribution, variable) +from paddle.distribution import ( + constraint, + distribution, + transformed_distribution, + variable, +) __all__ = [ # noqa - 'Transform', 'AbsTransform', 'AffineTransform', 'ChainTransform', - 'ExpTransform', 'IndependentTransform', 'PowerTransform', - 'ReshapeTransform', 'SigmoidTransform', 'SoftmaxTransform', - 'StackTransform', 'StickBreakingTransform', 'TanhTransform' + 'Transform', + 'AbsTransform', + 'AffineTransform', + 'ChainTransform', + 'ExpTransform', + 'IndependentTransform', + 'PowerTransform', + 'ReshapeTransform', + 'SigmoidTransform', + 'SoftmaxTransform', + 'StackTransform', + 'StickBreakingTransform', + 'TanhTransform', ] class Type(enum.Enum): - """Mapping type of a transformation. - """ + """Mapping type of a transformation.""" + BIJECTION = 'bijection' # bijective(injective and surjective) INJECTION = 'injection' # injective-only SURJECTION = 'surjection' # surjective-only @@ -41,8 +54,7 @@ class Type(enum.Enum): @classmethod def is_injective(cls, _type): - """Both bijection and injection are injective mapping. - """ + """Both bijection and injection are injective mapping.""" return _type in (cls.BIJECTION, cls.INJECTION) @@ -138,7 +150,8 @@ class Transform(object): """ if isinstance(input, distribution.Distribution): return transformed_distribution.TransformedDistribution( - input, [self]) + input, [self] + ) if isinstance(input, Transform): return ChainTransform([self, input]) return self.forward(input) @@ -157,11 +170,13 @@ class Transform(object): """ if not isinstance(x, paddle.fluid.framework.Variable): raise TypeError( - f"Expected 'x' is a Tensor or Real, but got {type(x)}.") + f"Expected 'x' is a Tensor or Real, but got {type(x)}." + ) if x.dim() < self._domain.event_rank: raise ValueError( f'The dimensions of x({x.dim()}) should be ' - f'grater than or equal to {self._domain.event_rank}') + f'grater than or equal to {self._domain.event_rank}' + ) return self._forward(x) def inverse(self, y): @@ -176,11 +191,13 @@ class Transform(object): """ if not isinstance(y, paddle.fluid.framework.Variable): raise TypeError( - f"Expected 'y' is a Tensor or Real, but got {type(y)}.") + f"Expected 'y' is a Tensor or Real, but got {type(y)}." + ) if y.dim() < self._codomain.event_rank: raise ValueError( f'The dimensions of y({y.dim()}) should be ' - f'grater than or equal to {self._codomain.event_rank}') + f'grater than or equal to {self._codomain.event_rank}' + ) return self._inverse(y) def forward_log_det_jacobian(self, x): @@ -196,16 +213,21 @@ class Transform(object): """ if not isinstance(x, paddle.fluid.framework.Variable): raise TypeError( - f"Expected 'y' is a Tensor or Real, but got {type(x)}.") - if isinstance(x, paddle.fluid.framework.Variable - ) and x.dim() < self._domain.event_rank: + f"Expected 'y' is a Tensor or Real, but got {type(x)}." + ) + if ( + isinstance(x, paddle.fluid.framework.Variable) + and x.dim() < self._domain.event_rank + ): raise ValueError( f'The dimensions of x({x.dim()}) should be ' - f'grater than or equal to {self._domain.event_rank}') + f'grater than or equal to {self._domain.event_rank}' + ) if not self._is_injective(): raise NotImplementedError( "forward_log_det_jacobian can't be implemented for non-injective" - "transforms.") + "transforms." + ) return self._call_forward_log_det_jacobian(x) @@ -226,7 +248,8 @@ class Transform(object): if y.dim() < self._codomain.event_rank: raise ValueError( f'The dimensions of y({y.dim()}) should be ' - f'grater than or equal to {self._codomain.event_rank}') + f'grater than or equal to {self._codomain.event_rank}' + ) return self._call_inverse_log_det_jacobian(y) def forward_shape(self, shape): @@ -240,7 +263,8 @@ class Transform(object): """ if not isinstance(shape, typing.Sequence): raise TypeError( - f"Expected shape is Sequence[int] type, but got {type(shape)}.") + f"Expected shape is Sequence[int] type, but got {type(shape)}." + ) return self._forward_shape(shape) def inverse_shape(self, shape): @@ -254,7 +278,8 @@ class Transform(object): """ if not isinstance(shape, typing.Sequence): raise TypeError( - f"Expected shape is Sequence[int] type, but got {type(shape)}.") + f"Expected shape is Sequence[int] type, but got {type(shape)}." + ) return self._inverse_shape(shape) @property @@ -287,7 +312,8 @@ class Transform(object): return -self._inverse_log_det_jacobian(self.forward(x)) raise NotImplementedError( 'Neither _forward_log_det_jacobian nor _inverse_log_det_jacobian' - 'is implemented. One of them is required.') + 'is implemented. One of them is required.' + ) def _call_inverse_log_det_jacobian(self, y): """Inner method called by ``inverse_log_det_jacobian``""" @@ -297,7 +323,8 @@ class Transform(object): return -self._forward_log_det_jacobian(self._inverse(y)) raise NotImplementedError( 'Neither _forward_log_det_jacobian nor _inverse_log_det_jacobian ' - 'is implemented. One of them is required') + 'is implemented. One of them is required' + ) def _forward_shape(self, shape): """Inner method called by ``forward_shape``, which is used to infer the @@ -420,7 +447,8 @@ class AffineTransform(Transform): raise TypeError(f"Expected 'loc' is a Tensor, but got {type(loc)}") if not isinstance(scale, paddle.fluid.framework.Variable): raise TypeError( - f"Expected scale is a Tensor, but got {type(scale)}") + f"Expected scale is a Tensor, but got {type(scale)}" + ) self._loc = loc self._scale = scale super(AffineTransform, self).__init__() @@ -446,13 +474,17 @@ class AffineTransform(Transform): return tuple( paddle.broadcast_shape( paddle.broadcast_shape(shape, self._loc.shape), - self._scale.shape)) + self._scale.shape, + ) + ) def _inverse_shape(self, shape): return tuple( paddle.broadcast_shape( paddle.broadcast_shape(shape, self._loc.shape), - self._scale.shape)) + self._scale.shape, + ) + ) @property def _domain(self): @@ -504,7 +536,8 @@ class ChainTransform(Transform): ) if not all(isinstance(t, Transform) for t in transforms): raise TypeError( - "All elements of transforms should be Transform type.") + "All elements of transforms should be Transform type." + ) self.transforms = transforms super(ChainTransform, self).__init__() @@ -523,11 +556,12 @@ class ChainTransform(Transform): return y def _forward_log_det_jacobian(self, x): - value = 0. + value = 0.0 event_rank = self._domain.event_rank for t in self.transforms: - value += self._sum_rightmost(t.forward_log_det_jacobian(x), - event_rank - t._domain.event_rank) + value += self._sum_rightmost( + t.forward_log_det_jacobian(x), event_rank - t._domain.event_rank + ) x = t.forward(x) event_rank += t._codomain.event_rank - t._domain.event_rank return value @@ -682,7 +716,8 @@ class IndependentTransform(Transform): def __init__(self, base, reinterpreted_batch_rank): if not isinstance(base, Transform): raise TypeError( - f"Expected 'base' is Transform type, but get {type(base)}") + f"Expected 'base' is Transform type, but get {type(base)}" + ) if reinterpreted_batch_rank <= 0: raise ValueError( f"Expected 'reinterpreted_batch_rank' is grater than zero, but got {reinterpreted_batch_rank}" @@ -707,7 +742,8 @@ class IndependentTransform(Transform): def _forward_log_det_jacobian(self, x): return self._base.forward_log_det_jacobian(x).sum( - list(range(-self._reinterpreted_batch_rank, 0))) + list(range(-self._reinterpreted_batch_rank, 0)) + ) def _forward_shape(self, shape): return self._base.forward_shape(shape) @@ -717,13 +753,15 @@ class IndependentTransform(Transform): @property def _domain(self): - return variable.Independent(self._base._domain, - self._reinterpreted_batch_rank) + return variable.Independent( + self._base._domain, self._reinterpreted_batch_rank + ) @property def _codomain(self): - return variable.Independent(self._base._codomain, - self._reinterpreted_batch_rank) + return variable.Independent( + self._base._codomain, self._reinterpreted_batch_rank + ) class PowerTransform(Transform): @@ -757,7 +795,8 @@ class PowerTransform(Transform): def __init__(self, power): if not isinstance(power, paddle.fluid.framework.Variable): raise TypeError( - f"Expected 'power' is a tensor, but got {type(power)}") + f"Expected 'power' is a tensor, but got {type(power)}" + ) self._power = power super(PowerTransform, self).__init__() @@ -826,13 +865,16 @@ class ReshapeTransform(Transform): def __init__(self, in_event_shape, out_event_shape): if not isinstance(in_event_shape, typing.Sequence) or not isinstance( - out_event_shape, typing.Sequence): + out_event_shape, typing.Sequence + ): raise TypeError( f"Expected type of 'in_event_shape' and 'out_event_shape' is " f"Squence[int], but got 'in_event_shape': {in_event_shape}, " - f"'out_event_shape': {out_event_shape}") + f"'out_event_shape': {out_event_shape}" + ) if functools.reduce(operator.mul, in_event_shape) != functools.reduce( - operator.mul, out_event_shape): + operator.mul, out_event_shape + ): raise ValueError( f"The numel of 'in_event_shape' should be 'out_event_shape', " f"but got {functools.reduce(operator.mul, in_event_shape)}!={functools.reduce(operator.mul, out_event_shape)}" @@ -860,39 +902,45 @@ class ReshapeTransform(Transform): def _forward(self, x): return x.reshape( - tuple(x.shape)[:x.dim() - len(self._in_event_shape)] + - self._out_event_shape) + tuple(x.shape)[: x.dim() - len(self._in_event_shape)] + + self._out_event_shape + ) def _inverse(self, y): return y.reshape( - tuple(y.shape)[:y.dim() - len(self._out_event_shape)] + - self._in_event_shape) + tuple(y.shape)[: y.dim() - len(self._out_event_shape)] + + self._in_event_shape + ) def _forward_shape(self, shape): if len(shape) < len(self._in_event_shape): raise ValueError( f"Expected length of 'shape' is not less than {len(self._in_event_shape)}, but got {len(shape)}" ) - if shape[-len(self._in_event_shape):] != self._in_event_shape: + if shape[-len(self._in_event_shape) :] != self._in_event_shape: raise ValueError( f"Event shape mismatch, expected: {self._in_event_shape}, but got {shape[-len(self._in_event_shape):]}" ) - return tuple(shape[:-len(self._in_event_shape)]) + self._out_event_shape + return ( + tuple(shape[: -len(self._in_event_shape)]) + self._out_event_shape + ) def _inverse_shape(self, shape): if len(shape) < len(self._out_event_shape): raise ValueError( f"Expected 'shape' length is not less than {len(self._out_event_shape)}, but got {len(shape)}" ) - if shape[-len(self._out_event_shape):] != self._out_event_shape: + if shape[-len(self._out_event_shape) :] != self._out_event_shape: raise ValueError( f"Event shape mismatch, expected: {self._out_event_shape}, but got {shape[-len(self._out_event_shape):]}" ) - return tuple(shape[:-len(self._out_event_shape)]) + self._in_event_shape + return ( + tuple(shape[: -len(self._out_event_shape)]) + self._in_event_shape + ) def _forward_log_det_jacobian(self, x): # paddle.zeros not support zero dimension Tensor. - shape = x.shape[:x.dim() - len(self._in_event_shape)] or [1] + shape = x.shape[: x.dim() - len(self._in_event_shape)] or [1] return paddle.zeros(shape, dtype=x.dtype) @@ -927,7 +975,7 @@ class SigmoidTransform(Transform): @property def _codomain(self): - return variable.Variable(False, 0, constraint.Range(0., 1.)) + return variable.Variable(False, 0, constraint.Range(0.0, 1.0)) def _forward(self, x): return F.sigmoid(x) @@ -996,7 +1044,7 @@ class SoftmaxTransform(Transform): class StackTransform(Transform): - r""" ``StackTransform`` applies a sequence of transformations along the + r"""``StackTransform`` applies a sequence of transformations along the specific axis. Args: @@ -1043,7 +1091,8 @@ class StackTransform(Transform): ) if not all(isinstance(t, Transform) for t in transforms): raise TypeError( - 'Expected all element in transforms is Transform Type.') + 'Expected all element in transforms is Transform Type.' + ) if not isinstance(axis, int): raise TypeError(f"Expected 'axis' is int, but got{type(axis)}.") @@ -1063,34 +1112,45 @@ class StackTransform(Transform): def _forward(self, x): self._check_size(x) - return paddle.stack([ - t.forward(v) - for v, t in zip(paddle.unstack(x, self._axis), self._transforms) - ], self._axis) + return paddle.stack( + [ + t.forward(v) + for v, t in zip(paddle.unstack(x, self._axis), self._transforms) + ], + self._axis, + ) def _inverse(self, y): self._check_size(y) - return paddle.stack([ - t.inverse(v) - for v, t in zip(paddle.unstack(y, self._axis), self._transforms) - ], self._axis) + return paddle.stack( + [ + t.inverse(v) + for v, t in zip(paddle.unstack(y, self._axis), self._transforms) + ], + self._axis, + ) def _forward_log_det_jacobian(self, x): self._check_size(x) - return paddle.stack([ - t.forward_log_det_jacobian(v) - for v, t in zip(paddle.unstack(x, self._axis), self._transforms) - ], self._axis) + return paddle.stack( + [ + t.forward_log_det_jacobian(v) + for v, t in zip(paddle.unstack(x, self._axis), self._transforms) + ], + self._axis, + ) def _check_size(self, v): if not (-v.dim() <= self._axis < v.dim()): raise ValueError( f'Input dimensions {v.dim()} should be grater than stack ' - f'transform axis {self._axis}.') + f'transform axis {self._axis}.' + ) if v.shape[self._axis] != len(self._transforms): raise ValueError( f'Input size along {self._axis} should be equal to the ' - f'length of transforms.') + f'length of transforms.' + ) @property def _domain(self): @@ -1098,8 +1158,9 @@ class StackTransform(Transform): @property def _codomain(self): - return variable.Stack([t._codomain for t in self._transforms], - self._axis) + return variable.Stack( + [t._codomain for t in self._transforms], self._axis + ) class StickBreakingTransform(Transform): @@ -1132,8 +1193,9 @@ class StickBreakingTransform(Transform): offset = x.shape[-1] + 1 - paddle.ones([x.shape[-1]]).cumsum(-1) z = F.sigmoid(x - offset.log()) z_cumprod = (1 - z).cumprod(-1) - return F.pad(z, [0] * 2 * (len(x.shape) - 1) + [0, 1], value=1) * \ - F.pad(z_cumprod, [0] * 2 * (len(x.shape) - 1) + [1, 0], value=1) + return F.pad(z, [0] * 2 * (len(x.shape) - 1) + [0, 1], value=1) * F.pad( + z_cumprod, [0] * 2 * (len(x.shape) - 1) + [1, 0], value=1 + ) def _inverse(self, y): y_crop = y[..., :-1] @@ -1151,12 +1213,12 @@ class StickBreakingTransform(Transform): def _forward_shape(self, shape): if not shape: raise ValueError(f"Expected 'shape' is not empty, but got {shape}") - return shape[:-1] + (shape[-1] + 1, ) + return shape[:-1] + (shape[-1] + 1,) def _inverse_shape(self, shape): if not shape: raise ValueError(f"Expected 'shape' is not empty, but got {shape}") - return shape[:-1] + (shape[-1] - 1, ) + return shape[:-1] + (shape[-1] - 1,) @property def _domain(self): @@ -1220,4 +1282,4 @@ class TanhTransform(Transform): See details: https://github.com/tensorflow/probability/blob/master/tensorflow_probability/python/bijectors/tanh.py#L69-L80 """ - return 2. * (math.log(2.) - x - F.softplus(-2. * x)) + return 2.0 * (math.log(2.0) - x - F.softplus(-2.0 * x)) diff --git a/python/paddle/distribution/transformed_distribution.py b/python/paddle/distribution/transformed_distribution.py index 8433ed0c910..1f97150cc9a 100644 --- a/python/paddle/distribution/transformed_distribution.py +++ b/python/paddle/distribution/transformed_distribution.py @@ -65,8 +65,9 @@ class TransformedDistribution(distribution.Distribution): self._base = base self._transforms = transforms if not transforms: - super(TransformedDistribution, - self).__init__(base.batch_shape, base.event_shape) + super(TransformedDistribution, self).__init__( + base.batch_shape, base.event_shape + ) return if len(base.batch_shape + base.event_shape) < chain._domain.event_rank: raise ValueError( @@ -74,15 +75,23 @@ class TransformedDistribution(distribution.Distribution): ) if chain._domain.event_rank > len(base.event_shape): base = independent.Independent( - (base, chain._domain.event_rank - len(base.event_shape))) + (base, chain._domain.event_rank - len(base.event_shape)) + ) - transformed_shape = chain.forward_shape(base.batch_shape + - base.event_shape) - transformed_event_rank = chain._codomain.event_rank + \ - max(len(base.event_shape) - chain._domain.event_rank, 0) + transformed_shape = chain.forward_shape( + base.batch_shape + base.event_shape + ) + transformed_event_rank = chain._codomain.event_rank + max( + len(base.event_shape) - chain._domain.event_rank, 0 + ) super(TransformedDistribution, self).__init__( - transformed_shape[:len(transformed_shape) - transformed_event_rank], - transformed_shape[len(transformed_shape) - transformed_event_rank:]) + transformed_shape[ + : len(transformed_shape) - transformed_event_rank + ], + transformed_shape[ + len(transformed_shape) - transformed_event_rank : + ], + ) def sample(self, shape=()): """Sample from ``TransformedDistribution``. @@ -127,12 +136,13 @@ class TransformedDistribution(distribution.Distribution): for t in reversed(self._transforms): x = t.inverse(y) event_rank += t._domain.event_rank - t._codomain.event_rank - log_prob = log_prob - \ - _sum_rightmost(t.forward_log_det_jacobian( - x), event_rank - t._domain.event_rank) + log_prob = log_prob - _sum_rightmost( + t.forward_log_det_jacobian(x), event_rank - t._domain.event_rank + ) y = x - log_prob += _sum_rightmost(self._base.log_prob(y), - event_rank - len(self._base.event_shape)) + log_prob += _sum_rightmost( + self._base.log_prob(y), event_rank - len(self._base.event_shape) + ) return log_prob diff --git a/python/paddle/distribution/uniform.py b/python/paddle/distribution/uniform.py index 5f9dc6c3d37..4e48dc60622 100644 --- a/python/paddle/distribution/uniform.py +++ b/python/paddle/distribution/uniform.py @@ -15,10 +15,19 @@ import numpy as np from paddle import _C_ops, _legacy_C_ops from paddle.distribution import distribution -from paddle.fluid.data_feeder import (check_type, convert_dtype) -from paddle.fluid.framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph -from paddle.fluid.layers import (elementwise_add, elementwise_div, - elementwise_sub, nn, tensor) +from paddle.fluid.data_feeder import check_type, convert_dtype +from paddle.fluid.framework import ( + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, +) +from paddle.fluid.layers import ( + elementwise_add, + elementwise_div, + elementwise_sub, + nn, + tensor, +) class Uniform(distribution.Distribution): @@ -86,12 +95,18 @@ class Uniform(distribution.Distribution): def __init__(self, low, high, name=None): if not _non_static_mode(): - check_type(low, 'low', - (int, float, np.ndarray, tensor.Variable, list, tuple), - 'Uniform') - check_type(high, 'high', - (int, float, np.ndarray, tensor.Variable, list, tuple), - 'Uniform') + check_type( + low, + 'low', + (int, float, np.ndarray, tensor.Variable, list, tuple), + 'Uniform', + ) + check_type( + high, + 'high', + (int, float, np.ndarray, tensor.Variable, list, tuple), + 'Uniform', + ) self.all_arg_is_float = False self.batch_size_unknown = False @@ -111,11 +126,15 @@ class Uniform(distribution.Distribution): else: if isinstance(low, float) and isinstance(high, float): self.all_arg_is_float = True - if isinstance(low, np.ndarray) and str( - low.dtype) in ['float32', 'float64']: + if isinstance(low, np.ndarray) and str(low.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = low.dtype - elif isinstance(high, np.ndarray) and str( - high.dtype) in ['float32', 'float64']: + elif isinstance(high, np.ndarray) and str(high.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = high.dtype # pylint: disable=unbalanced-tuple-unpacking self.low, self.high = self._to_tensor(low, high) @@ -145,27 +164,33 @@ class Uniform(distribution.Distribution): if self.batch_size_unknown: output_shape = shape + batch_shape zero_tmp = tensor.fill_constant_batch_size_like( - self.low + self.high, batch_shape + shape, self.dtype, 0.) + self.low + self.high, batch_shape + shape, self.dtype, 0.0 + ) uniform_random_tmp = nn.uniform_random_batch_size_like( zero_tmp, zero_tmp.shape, dtype=self.dtype, - min=0., - max=1., - seed=seed) + min=0.0, + max=1.0, + seed=seed, + ) zero_tmp_reshape = nn.reshape(zero_tmp, output_shape) - uniform_random_tmp_reshape = nn.reshape(uniform_random_tmp, - output_shape) - output = uniform_random_tmp_reshape * (zero_tmp_reshape + - self.high - self.low) + uniform_random_tmp_reshape = nn.reshape( + uniform_random_tmp, output_shape + ) + output = uniform_random_tmp_reshape * ( + zero_tmp_reshape + self.high - self.low + ) output = elementwise_add(output, self.low, name=name) return output else: output_shape = shape + batch_shape output = nn.uniform_random( - output_shape, dtype=self.dtype, min=0., max=1., - seed=seed) * (tensor.zeros(output_shape, dtype=self.dtype) + - (self.high - self.low)) + output_shape, dtype=self.dtype, min=0.0, max=1.0, seed=seed + ) * ( + tensor.zeros(output_shape, dtype=self.dtype) + + (self.high - self.low) + ) output = elementwise_add(output, self.low, name=name) if self.all_arg_is_float: return nn.reshape(output, shape, name=name) @@ -194,10 +219,12 @@ class Uniform(distribution.Distribution): return nn.log(lb * ub) - nn.log(self.high - self.low) if _in_legacy_dygraph(): - lb = _legacy_C_ops.cast(lb_bool, 'in_dtype', lb_bool.dtype, - 'out_dtype', value.dtype) - ub = _legacy_C_ops.cast(ub_bool, 'in_dtype', ub_bool.dtype, - 'out_dtype', value.dtype) + lb = _legacy_C_ops.cast( + lb_bool, 'in_dtype', lb_bool.dtype, 'out_dtype', value.dtype + ) + ub = _legacy_C_ops.cast( + ub_bool, 'in_dtype', ub_bool.dtype, 'out_dtype', value.dtype + ) return nn.log(lb * ub) - nn.log(self.high - self.low) name = self.name + '_log_prob' @@ -205,9 +232,9 @@ class Uniform(distribution.Distribution): ub_bool = value < self.high lb = tensor.cast(lb_bool, dtype=value.dtype) ub = tensor.cast(ub_bool, dtype=value.dtype) - return elementwise_sub(nn.log(lb * ub), - nn.log(self.high - self.low), - name=name) + return elementwise_sub( + nn.log(lb * ub), nn.log(self.high - self.low), name=name + ) def probs(self, value): """Probability density/mass function. @@ -230,10 +257,12 @@ class Uniform(distribution.Distribution): return (lb * ub) / (self.high - self.low) if _in_legacy_dygraph(): - lb = _legacy_C_ops.cast(lb_bool, 'in_dtype', lb_bool.dtype, - 'out_dtype', value.dtype) - ub = _legacy_C_ops.cast(ub_bool, 'in_dtype', ub_bool.dtype, - 'out_dtype', value.dtype) + lb = _legacy_C_ops.cast( + lb_bool, 'in_dtype', lb_bool.dtype, 'out_dtype', value.dtype + ) + ub = _legacy_C_ops.cast( + ub_bool, 'in_dtype', ub_bool.dtype, 'out_dtype', value.dtype + ) return (lb * ub) / (self.high - self.low) name = self.name + '_probs' diff --git a/python/paddle/distribution/variable.py b/python/paddle/distribution/variable.py index f6cced04164..2bde6c79aeb 100644 --- a/python/paddle/distribution/variable.py +++ b/python/paddle/distribution/variable.py @@ -43,13 +43,11 @@ class Variable(object): class Real(Variable): - def __init__(self, event_rank=0): super(Real, self).__init__(False, event_rank, constraint.real) class Positive(Variable): - def __init__(self, event_rank=0): super(Positive, self).__init__(False, event_rank, constraint.positive) @@ -66,23 +64,24 @@ class Independent(Variable): def __init__(self, base, reinterpreted_batch_rank): self._base = base self._reinterpreted_batch_rank = reinterpreted_batch_rank - super(Independent, - self).__init__(base.is_discrete, - base.event_rank + reinterpreted_batch_rank) + super(Independent, self).__init__( + base.is_discrete, base.event_rank + reinterpreted_batch_rank + ) def constraint(self, value): ret = self._base.constraint(value) if ret.dim() < self._reinterpreted_batch_rank: raise ValueError( "Input dimensions must be equal or grater than {}".format( - self._reinterpreted_batch_rank)) - return ret.reshape(ret.shape[:ret.dim() - - self.reinterpreted_batch_rank] + - (-1, )).all(-1) + self._reinterpreted_batch_rank + ) + ) + return ret.reshape( + ret.shape[: ret.dim() - self.reinterpreted_batch_rank] + (-1,) + ).all(-1) class Stack(Variable): - def __init__(self, vars, axis=0): self._vars = vars self._axis = axis @@ -102,12 +101,18 @@ class Stack(Variable): if not (-value.dim() <= self._axis < value.dim()): raise ValueError( f'Input dimensions {value.dim()} should be grater than stack ' - f'constraint axis {self._axis}.') - - return paddle.stack([ - var.check(value) - for var, value in zip(self._vars, paddle.unstack(value, self._axis)) - ], self._axis) + f'constraint axis {self._axis}.' + ) + + return paddle.stack( + [ + var.check(value) + for var, value in zip( + self._vars, paddle.unstack(value, self._axis) + ) + ], + self._axis, + ) real = Real() diff --git a/python/paddle/fft.py b/python/paddle/fft.py index b873d7b1df6..8bc95cd37e9 100644 --- a/python/paddle/fft.py +++ b/python/paddle/fft.py @@ -51,61 +51,76 @@ __all__ = [ def _check_normalization(norm): if norm not in ['forward', 'backward', 'ortho']: raise ValueError( - "Unexpected norm: {}. Norm should be forward, backward or ortho". - format(norm)) + "Unexpected norm: {}. Norm should be forward, backward or ortho".format( + norm + ) + ) def _check_fft_n(n): if not isinstance(n, int): raise ValueError( - "Invalid FFT argument n({}), it shoule be an integer.".format(n)) + "Invalid FFT argument n({}), it shoule be an integer.".format(n) + ) if n <= 0: raise ValueError( - "Invalid FFT argument n({}), it should be positive.".format(n)) + "Invalid FFT argument n({}), it should be positive.".format(n) + ) def _check_fft_shape(x, s): ndim = x.ndim if not isinstance(s, Sequence): raise ValueError( - "Invaid FFT argument s({}), it should be a sequence of integers.") + "Invaid FFT argument s({}), it should be a sequence of integers." + ) if len(s) > ndim: raise ValueError( "Length of FFT argument s should not be larger than the rank of input. " - "Received s: {}, rank of x: {}".format(s, ndim)) + "Received s: {}, rank of x: {}".format(s, ndim) + ) for size in s: if not isinstance(size, int) or size <= 0: - raise ValueError("FFT sizes {} contains invalid value ({})".format( - s, size)) + raise ValueError( + "FFT sizes {} contains invalid value ({})".format(s, size) + ) def _check_fft_axis(x, axis): ndim = x.ndim if not isinstance(axis, int): raise ValueError( - "Invalid FFT axis ({}), it shoule be an integer.".format(axis)) + "Invalid FFT axis ({}), it shoule be an integer.".format(axis) + ) if axis < -ndim or axis >= ndim: raise ValueError( "Invalid FFT axis ({}), it should be in range [-{}, {})".format( - axis, ndim, ndim)) + axis, ndim, ndim + ) + ) def _check_fft_axes(x, axes): ndim = x.ndim if not isinstance(axes, Sequence): raise ValueError( - "Invalid FFT axes ({}), it should be a sequence of integers.". - format(axes)) + "Invalid FFT axes ({}), it should be a sequence of integers.".format( + axes + ) + ) if len(axes) > ndim: raise ValueError( "Length of fft axes should not be larger than the rank of input. " - "Received, len of axes: {}, rank of x: {}".format(len(axes), ndim)) + "Received, len of axes: {}, rank of x: {}".format(len(axes), ndim) + ) for axis in axes: if not isinstance(axis, int) or axis < -ndim or axis >= ndim: raise ValueError( - "FFT axes {} contains invalid value ({}), it should be in range [-{}, {})" - .format(axes, axis, ndim, ndim)) + "FFT axes {} contains invalid value ({}), it should be in range [-{}, {})".format( + axes, axis, ndim, ndim + ) + ) def _resize_fft_input(x, s, axes): @@ -127,10 +142,12 @@ def _resize_fft_input(x, s, axes): slices.append((0, s[i])) if axes_to_slice: - x = paddle.slice(x, - axes_to_slice, - starts=[item[0] for item in slices], - ends=[item[1] for item in slices]) + x = paddle.slice( + x, + axes_to_slice, + starts=[item[0] for item in slices], + ends=[item[1] for item in slices], + ) if axes_to_pad: padding_widths = [0] * (2 * ndim) for axis, pad in zip(axes_to_pad, paddings): @@ -146,8 +163,9 @@ def _normalize_axes(x, axes): def _check_at_least_ndim(x, rank): if x.ndim < rank: - raise ValueError("The rank of the input ({}) should >= {}".format( - x.ndim, rank)) + raise ValueError( + "The rank of the input ({}) should >= {}".format(x.ndim, rank) + ) # public APIs 1d @@ -197,13 +215,9 @@ def fft(x, n=None, axis=-1, norm="backward", name=None): """ if is_integer(x) or is_floating_point(x): - return fft_r2c(x, - n, - axis, - norm, - forward=True, - onesided=False, - name=name) + return fft_r2c( + x, n, axis, norm, forward=True, onesided=False, name=name + ) else: return fft_c2c(x, n, axis, norm, forward=True, name=name) @@ -266,13 +280,9 @@ def ifft(x, n=None, axis=-1, norm="backward", name=None): """ if is_integer(x) or is_floating_point(x): - return fft_r2c(x, - n, - axis, - norm, - forward=False, - onesided=False, - name=name) + return fft_r2c( + x, n, axis, norm, forward=False, onesided=False, name=name + ) else: return fft_c2c(x, n, axis, norm, forward=False, name=name) @@ -536,13 +546,9 @@ def fftn(x, s=None, axes=None, norm="backward", name=None): # [-8.-8.j 0.+0.j 0.+0.j 0.-0.j]]] """ if is_integer(x) or is_floating_point(x): - return fftn_r2c(x, - s, - axes, - norm, - forward=True, - onesided=False, - name=name) + return fftn_r2c( + x, s, axes, norm, forward=True, onesided=False, name=name + ) else: return fftn_c2c(x, s, axes, norm, forward=True, name=name) @@ -608,13 +614,9 @@ def ifftn(x, s=None, axes=None, norm="backward", name=None): # (-0.1666666716337204+0.28867512941360474j)]]) """ if is_integer(x) or is_floating_point(x): - return fftn_r2c(x, - s, - axes, - norm, - forward=False, - onesided=False, - name=name) + return fftn_r2c( + x, s, axes, norm, forward=False, onesided=False, name=name + ) else: return fftn_c2c(x, s, axes, norm, forward=False, name=name) @@ -914,13 +916,17 @@ def fft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return fftn(x, s, axes, norm, name) @@ -979,13 +985,17 @@ def ifft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return ifftn(x, s, axes, norm, name) @@ -1038,13 +1048,17 @@ def rfft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return rfftn(x, s, axes, norm, name) @@ -1090,13 +1104,17 @@ def irfft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return irfftn(x, s, axes, norm, name) @@ -1135,13 +1153,17 @@ def hfft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return hfftn(x, s, axes, norm, name) @@ -1187,13 +1209,17 @@ def ihfft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return ihfftn(x, s, axes, norm, name) @@ -1417,10 +1443,9 @@ def fft_c2c(x, n, axis, norm, forward, name): dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1442,8 +1467,16 @@ def fft_r2c(x, n, axis, norm, forward, onesided, name): if in_dygraph_mode(): out = _C_ops.fft_r2c(x, axes, norm, forward, onesided) elif _in_legacy_dygraph(): - attrs = ('axes', axes, 'normalization', norm, 'forward', forward, - 'onesided', onesided) + attrs = ( + 'axes', + axes, + 'normalization', + norm, + 'forward', + forward, + 'onesided', + onesided, + ) out = getattr(_legacy_C_ops, op_type)(x, *attrs) else: inputs = { @@ -1458,12 +1491,12 @@ def fft_r2c(x, n, axis, norm, forward, onesided, name): helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference( - _real_to_complex_dtype(dtype)) + _real_to_complex_dtype(dtype) + ) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1491,8 +1524,16 @@ def fft_c2r(x, n, axis, norm, forward, name): out = _C_ops.fft_c2r(x, axes, norm, forward, 0) elif _in_legacy_dygraph(): if n is not None: - attrs = ('axes', axes, 'normalization', norm, 'forward', forward, - 'last_dim_size', n) + attrs = ( + 'axes', + axes, + 'normalization', + norm, + 'forward', + forward, + 'last_dim_size', + n, + ) else: attrs = ('axes', axes, 'normalization', norm, 'forward', forward) out = getattr(_legacy_C_ops, op_type)(x, *attrs) @@ -1506,12 +1547,12 @@ def fft_c2r(x, n, axis, norm, forward, name): helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference( - _complex_to_real_dtype(dtype)) + _complex_to_real_dtype(dtype) + ) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1539,8 +1580,10 @@ def fftn_c2c(x, s, axes, norm, forward, name): if s is not None: if len(s) != len(axes): raise ValueError( - "Length of s ({}) and length of axes ({}) does not match.". - format(len(s), len(axes))) + "Length of s ({}) and length of axes ({}) does not match.".format( + len(s), len(axes) + ) + ) s = [s[i] for i in axes_argsoft] if s is not None: @@ -1562,10 +1605,9 @@ def fftn_c2c(x, s, axes, norm, forward, name): dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1591,8 +1633,10 @@ def fftn_r2c(x, s, axes, norm, forward, onesided, name): if s is not None: if len(s) != len(axes): raise ValueError( - "Length of s ({}) and length of axes ({}) does not match.". - format(len(s), len(axes))) + "Length of s ({}) and length of axes ({}) does not match.".format( + len(s), len(axes) + ) + ) s = [s[i] for i in axes_argsoft] + [s[-1]] if s is not None: @@ -1604,8 +1648,16 @@ def fftn_r2c(x, s, axes, norm, forward, onesided, name): if in_dygraph_mode(): out = _C_ops.fft_r2c(x, axes, norm, forward, onesided) elif _in_legacy_dygraph(): - attrs = ('axes', axes, 'normalization', norm, 'forward', forward, - 'onesided', onesided) + attrs = ( + 'axes', + axes, + 'normalization', + norm, + 'forward', + forward, + 'onesided', + onesided, + ) out = getattr(_legacy_C_ops, op_type)(x, *attrs) else: inputs = { @@ -1620,12 +1672,12 @@ def fftn_r2c(x, s, axes, norm, forward, onesided, name): helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference( - _real_to_complex_dtype(dtype)) + _real_to_complex_dtype(dtype) + ) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1654,8 +1706,10 @@ def fftn_c2r(x, s, axes, norm, forward, name): if s is not None: if len(s) != len(axes): raise ValueError( - "Length of s ({}) and length of axes ({}) does not match.". - format(len(s), len(axes))) + "Length of s ({}) and length of axes ({}) does not match.".format( + len(s), len(axes) + ) + ) s = [s[i] for i in axes_argsoft] + [s[-1]] if s is not None: @@ -1673,8 +1727,16 @@ def fftn_c2r(x, s, axes, norm, forward, name): out = _C_ops.fft_c2r(x, axes, norm, forward, 0) elif _in_legacy_dygraph(): if s: - attrs = ('axes', axes, 'normalization', norm, 'forward', forward, - 'last_dim_size', s[-1]) + attrs = ( + 'axes', + axes, + 'normalization', + norm, + 'forward', + forward, + 'last_dim_size', + s[-1], + ) else: attrs = ('axes', axes, 'normalization', norm, 'forward', forward) out = getattr(_legacy_C_ops, op_type)(x, *attrs) @@ -1688,10 +1750,10 @@ def fftn_c2r(x, s, axes, norm, forward, name): helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference( - _complex_to_real_dtype(dtype)) + _complex_to_real_dtype(dtype) + ) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 00f222b3e8b..53d50a8b4a3 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -22,8 +22,9 @@ core_suffix = 'so' if os.name == 'nt': core_suffix = 'pyd' -legacy_core = os.path.abspath( - os.path.dirname(__file__)) + os.sep + 'core.' + core_suffix +legacy_core = ( + os.path.abspath(os.path.dirname(__file__)) + os.sep + 'core.' + core_suffix +) if os.path.exists(legacy_core): sys.stderr.write('Deleting legacy file ' + legacy_core + '\n') try: @@ -34,6 +35,7 @@ if os.path.exists(legacy_core): # import all class inside framework into fluid module from . import framework from .framework import * + # import all class inside executor into fluid module from . import executor from .executor import * @@ -70,10 +72,23 @@ from .param_attr import ParamAttr, WeightNormParamAttr from .data_feeder import DataFeeder from .core import LoDTensor, LoDTensorArray, Scope, _Scope -from .core import CPUPlace, XPUPlace, CUDAPlace, CUDAPinnedPlace, NPUPlace, IPUPlace, MLUPlace, CustomPlace +from .core import ( + CPUPlace, + XPUPlace, + CUDAPlace, + CUDAPinnedPlace, + NPUPlace, + IPUPlace, + MLUPlace, + CustomPlace, +) from .incubate import fleet -from .transpiler import DistributeTranspiler, \ - memory_optimize, release_memory, DistributeTranspilerConfig +from .transpiler import ( + DistributeTranspiler, + memory_optimize, + release_memory, + DistributeTranspilerConfig, +) from .lod_tensor import create_lod_tensor, create_random_int_lodtensor from . import clip from . import profiler @@ -93,7 +108,14 @@ from .dygraph.varbase_patch_methods import monkey_patch_varbase from . import generator from .core import _cuda_synchronize from .generator import Generator -from .trainer_desc import TrainerDesc, DistMultiTrainer, PipelineTrainer, HeterPipelineTrainer, MultiTrainer, HeterXpuTrainer +from .trainer_desc import ( + TrainerDesc, + DistMultiTrainer, + PipelineTrainer, + HeterPipelineTrainer, + MultiTrainer, + HeterXpuTrainer, +) from .transpiler import HashName, RoundRobin from .backward import append_backward @@ -101,10 +123,18 @@ Tensor = LoDTensor enable_imperative = enable_dygraph disable_imperative = disable_dygraph -__all__ = framework.__all__ + executor.__all__ + \ - trainer_desc.__all__ + transpiler.__all__ + \ - parallel_executor.__all__ + lod_tensor.__all__ + \ - data_feed_desc.__all__ + compiler.__all__ + backward.__all__ + generator.__all__ + [ +__all__ = ( + framework.__all__ + + executor.__all__ + + trainer_desc.__all__ + + transpiler.__all__ + + parallel_executor.__all__ + + lod_tensor.__all__ + + data_feed_desc.__all__ + + compiler.__all__ + + backward.__all__ + + generator.__all__ + + [ 'io', 'initializer', 'embedding', @@ -142,8 +172,9 @@ __all__ = framework.__all__ + executor.__all__ + \ 'install_check', 'save', 'load', - '_cuda_synchronize' + '_cuda_synchronize', ] +) def __bootstrap__(): @@ -169,18 +200,21 @@ def __bootstrap__(): num_threads = 1 if num_threads > 1: - print('WARNING: OMP_NUM_THREADS set to {0}, not 1. The computation ' - 'speed will not be optimized if you use data parallel. It will ' - 'fail if this PaddlePaddle binary is compiled with OpenBlas since' - ' OpenBlas does not support multi-threads.'.format(num_threads), - file=sys.stderr) + print( + 'WARNING: OMP_NUM_THREADS set to {0}, not 1. The computation ' + 'speed will not be optimized if you use data parallel. It will ' + 'fail if this PaddlePaddle binary is compiled with OpenBlas since' + ' OpenBlas does not support multi-threads.'.format(num_threads), + file=sys.stderr, + ) print('PLEASE USE OMP_NUM_THREADS WISELY.', file=sys.stderr) os.environ['OMP_NUM_THREADS'] = str(num_threads) flag_prefix = "FLAGS_" read_env_flags = [ - key[len(flag_prefix):] for key in core.globals().keys() + key[len(flag_prefix) :] + for key in core.globals().keys() if key.startswith(flag_prefix) ] diff --git a/python/paddle/fluid/average.py b/python/paddle/fluid/average.py index d31b4d49648..2d83246df7f 100644 --- a/python/paddle/fluid/average.py +++ b/python/paddle/fluid/average.py @@ -14,6 +14,7 @@ import numpy as np import warnings + """ Class of all kinds of Average. @@ -27,8 +28,11 @@ __all__ = ["WeightedAverage"] def _is_number_(var): - return isinstance(var, int) or isinstance( - var, float) or (isinstance(var, np.ndarray) and var.shape == (1, )) + return ( + isinstance(var, int) + or isinstance(var, float) + or (isinstance(var, np.ndarray) and var.shape == (1,)) + ) def _is_number_or_matrix_(var): @@ -59,8 +63,10 @@ class WeightedAverage(object): def __init__(self): warnings.warn( - "The %s is deprecated, please use fluid.metrics.Accuracy instead." % - (self.__class__.__name__), Warning) + "The %s is deprecated, please use fluid.metrics.Accuracy instead." + % (self.__class__.__name__), + Warning, + ) self.reset() def reset(self): @@ -70,7 +76,8 @@ class WeightedAverage(object): def add(self, value, weight): if not _is_number_or_matrix_(value): raise ValueError( - "The 'value' must be a number(int, float) or a numpy ndarray.") + "The 'value' must be a number(int, float) or a numpy ndarray." + ) if not _is_number_(weight): raise ValueError("The 'weight' must be a number(int, float).") @@ -84,5 +91,6 @@ class WeightedAverage(object): def eval(self): if self.numerator is None or self.denominator is None: raise ValueError( - "There is no data to be averaged in WeightedAverage.") + "There is no data to be averaged in WeightedAverage." + ) return self.numerator / self.denominator diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py index 8da8faeefb1..16708268bd5 100755 --- a/python/paddle/fluid/backward.py +++ b/python/paddle/fluid/backward.py @@ -26,6 +26,7 @@ from . import log_helper import paddle.fluid from .data_feeder import check_type import warnings + try: from collections.abc import Sequence except: @@ -36,13 +37,12 @@ __all__ = [ 'gradients', ] -_logger = log_helper.get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = log_helper.get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class ProgramStats(object): - def __init__(self, block, ops): self.block = block self.ops = ops @@ -52,8 +52,10 @@ class ProgramStats(object): def get_input_nodes(self): input_names = [] for name in self.var_op_deps: - if len(self.var_op_deps[name]["var_as_output_ops"]) == 0 and \ - len(self.var_op_deps[name]["var_as_input_ops"]) > 0: + if ( + len(self.var_op_deps[name]["var_as_output_ops"]) == 0 + and len(self.var_op_deps[name]["var_as_input_ops"]) > 0 + ): if self.block.var(name).persistable: continue input_names.append(name) @@ -115,16 +117,21 @@ class ProgramStats(object): """ def is_amp_cast(op): - return op.desc.type() == 'cast' and self.block.var( - op.desc.input_arg_names()[0]).persistable + return ( + op.desc.type() == 'cast' + and self.block.var(op.desc.input_arg_names()[0]).persistable + ) idx_ = min_idx - 1 updated_min_idx = min_idx while idx_ > pre_segment_end_idx: if is_amp_cast(self.ops[idx_]): - _logger.info("found amp-cast op: {}, : {}".format( - self.ops[idx_].desc.type(), - self.ops[idx_].desc.input_arg_names()[0])) + _logger.info( + "found amp-cast op: {}, : {}".format( + self.ops[idx_].desc.type(), + self.ops[idx_].desc.input_arg_names()[0], + ) + ) updated_min_idx = idx_ idx_ -= 1 else: @@ -138,7 +145,8 @@ class ProgramStats(object): for j, name in enumerate(op.desc.input_arg_names()): if name in self.var_op_deps: self.op_deps[i]["in_ops"].extend( - self.var_op_deps[name]["var_as_output_ops"]) + self.var_op_deps[name]["var_as_output_ops"] + ) for j, name in enumerate(op.desc.input_arg_names()): if name in self.var_op_deps: self.var_op_deps[name]["var_as_input_ops"].extend([i]) @@ -164,13 +172,15 @@ class ProgramStats(object): if name not in self.var_op_deps: _logger.info( "Recompute Optimizer: deleted %s from checkpoints, because it is not used in paddle program." - % name) + % name + ) elif self.var_op_deps[name]["var_as_output_ops"] == []: # input nodes sorted_checkpoints.append((name, -1)) else: sorted_checkpoints.append( - (name, max(self.var_op_deps[name]["var_as_output_ops"]))) + (name, max(self.var_op_deps[name]["var_as_output_ops"])) + ) sorted_checkpoints = sorted(sorted_checkpoints, key=lambda x: x[1]) return [x[0] for x in sorted_checkpoints] @@ -191,17 +201,20 @@ class ProgramStats(object): continue # add a seed op so that the two dropout op can generate same output op_unique_name = unique_name.generate("seed") - var_unique_name = unique_name.generate_with_ignorable_key(".".join( - [op_unique_name, 'tmp'])) + var_unique_name = unique_name.generate_with_ignorable_key( + ".".join([op_unique_name, 'tmp']) + ) added_var = self.block.create_var( name=var_unique_name, dtype='int32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) seed = 0 if op.attr("fix_seed") is False else int(op.attr("seed")) - op_device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + op_device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) op_device = "" if op.desc.has_attr(op_device_attr_name): @@ -209,15 +222,13 @@ class ProgramStats(object): # Setting the force_cpu of seed to true will make the output of seed in cpu memory, # reduce the synchronous copy from GPU to CPU in dropout, and reduce the communication hang - added_op = self.block._insert_op(index=op.idx, - type='seed', - inputs={}, - outputs={'Out': [added_var]}, - attrs={ - 'seed': seed, - 'op_device': op_device, - 'force_cpu': True - }) + added_op = self.block._insert_op( + index=op.idx, + type='seed', + inputs={}, + outputs={'Out': [added_var]}, + attrs={'seed': seed, 'op_device': op_device, 'force_cpu': True}, + ) self.ops.insert(op_idx, added_op) # modify dropout op desc so that it accept a seed var as input op.desc.set_input("Seed", [var_unique_name]) @@ -228,22 +239,24 @@ class ProgramStats(object): def _pretty_op_desc_(op_desc, prefix): - out_s = "%s\tname:[%s]\n%s \tinputs:[%s]\n%s \toutputs:[%s]" % \ - (prefix + "_op", str(op_desc.type()), prefix + "_input", " ".join(op_desc.input_arg_names()), - prefix + "_output", " ".join(op_desc.output_arg_names())) + out_s = "%s\tname:[%s]\n%s \tinputs:[%s]\n%s \toutputs:[%s]" % ( + prefix + "_op", + str(op_desc.type()), + prefix + "_input", + " ".join(op_desc.input_arg_names()), + prefix + "_output", + " ".join(op_desc.output_arg_names()), + ) return out_s -def _add_needed_descs_to_block(descs, - block, - main_block, - in_memory_vars, - grad_op_id_to_fwd_op=None): +def _add_needed_descs_to_block( + descs, block, main_block, in_memory_vars, grad_op_id_to_fwd_op=None +): if len(descs) == 0: return [] result_descs = [] - op_role_attr_name = \ - core.op_proto_and_checker_maker.kOpRoleAttrName() + op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() backward = core.op_proto_and_checker_maker.OpRole.Backward for desc in descs: origin_desc = desc @@ -275,8 +288,7 @@ def _add_descs_to_block(descs, block, grad_op_id_to_fwd_op=None): if len(descs) == 0: return [] result_descs = [] - op_role_attr_name = \ - core.op_proto_and_checker_maker.kOpRoleAttrName() + op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() backward = core.op_proto_and_checker_maker.OpRole.Backward for desc in descs: if isinstance(desc, framework.Operator): @@ -298,8 +310,10 @@ def _add_descs_to_block(descs, block, grad_op_id_to_fwd_op=None): def _find_loss_op_(loss): for op in reversed(loss.block.ops): assert isinstance(op, framework.Operator) - if len(op.output_arg_names - ) == 1 and op.output_arg_names[0] == loss.name: + if ( + len(op.output_arg_names) == 1 + and op.output_arg_names[0] == loss.name + ): loss.op = op break if loss.op is None: @@ -340,21 +354,30 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): op_desc.set_input( para, list( - map(lambda arg: arg.decode() - if isinstance(arg, bytes) else arg, args))) + map( + lambda arg: arg.decode() if isinstance(arg, bytes) else arg, + args, + ) + ), + ) for para, args in outputs.items(): op_desc.set_output( para, list( - map(lambda arg: arg.decode() - if isinstance(arg, bytes) else arg, args))) + map( + lambda arg: arg.decode() if isinstance(arg, bytes) else arg, + args, + ) + ), + ) op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() op_device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName() if op_role_attr_name not in attrs: attrs[ - op_role_attr_name] = core.op_proto_and_checker_maker.OpRole.Backward + op_role_attr_name + ] = core.op_proto_and_checker_maker.OpRole.Backward if op_device_attr_name not in attrs: attrs[op_device_attr_name] = "" for name, val in attrs.items(): @@ -367,20 +390,23 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): def _create_loss_op_desc_(loss): op_desc = _create_op_desc_( - "fill_constant", {}, {"Out": [_append_grad_suffix_(loss.name)]}, { + "fill_constant", + {}, + {"Out": [_append_grad_suffix_(loss.name)]}, + { "shape": [1], - "value": - 1.0, - "dtype": - loss.dtype, - "force_cpu": - False, - core.op_proto_and_checker_maker.kOpRoleAttrName(): - int(core.op_proto_and_checker_maker.OpRole.Backward) + "value": 1.0, + "dtype": loss.dtype, + "force_cpu": False, + core.op_proto_and_checker_maker.kOpRoleAttrName(): int( + core.op_proto_and_checker_maker.OpRole.Backward + ) | int(core.op_proto_and_checker_maker.OpRole.Loss), - core.op_proto_and_checker_maker.kOpDeviceAttrName(): - loss.op.attr(core.op_proto_and_checker_maker.kOpDeviceAttrName()) - }) + core.op_proto_and_checker_maker.kOpDeviceAttrName(): loss.op.attr( + core.op_proto_and_checker_maker.kOpDeviceAttrName() + ), + }, + ) return op_desc @@ -397,8 +423,10 @@ def _infer_var_data_type_shape_(grad_var_name, block): else: # TODO(jiabin): Maybe we should not to this to cause some unexpected error on dtype warnings.warn( - "Set grad var: {} dtype to default FP32, since we can't find its related forward var" - .format(grad_var_name)) + "Set grad var: {} dtype to default FP32, since we can't find its related forward var".format( + grad_var_name + ) + ) grad_var.set_dtype(core.VarDesc.VarType.FP32) @@ -435,7 +463,7 @@ def _strip_grad_suffix_(name): pos = name.find(core.grad_var_suffix()) new_name = name[:pos] if pos != -1 else name new_pos = name.rfind('grad/') - return new_name[new_pos + 5:] if new_pos != -1 else new_name + return new_name[new_pos + 5 :] if new_pos != -1 else new_name def _append_grad_suffix_(name): @@ -446,30 +474,28 @@ def _append_grad_suffix_(name): return name + core.grad_var_suffix() -def _accumulate_gradients_by_sum_op_(var_name, - renamed_vars, - pending_sum_ops, - op_idx, - op_device=""): +def _accumulate_gradients_by_sum_op_( + var_name, renamed_vars, pending_sum_ops, op_idx, op_device="" +): """ Use sum op to accumulate_gradients, the gradients are stored in renamed_vars. """ if op_idx not in pending_sum_ops.keys(): pending_sum_ops[op_idx] = [] pending_sum_ops[op_idx].append( - _create_op_desc_("sum", {"X": renamed_vars[var_name]}, - {"Out": [var_name]}, { - "use_mkldnn": False, - "op_device": op_device - })) + _create_op_desc_( + "sum", + {"X": renamed_vars[var_name]}, + {"Out": [var_name]}, + {"use_mkldnn": False, "op_device": op_device}, + ) + ) renamed_vars[var_name] = [var_name] -def _accumulate_gradients_by_add_ops_(var_name, - renamed_vars, - pending_sum_ops, - op_idx, - op_device=""): +def _accumulate_gradients_by_add_ops_( + var_name, renamed_vars, pending_sum_ops, op_idx, op_device="" +): """ Use several inplace add op to accumulate_gradients, the gradients are stored in renamed_vars. """ @@ -484,20 +510,19 @@ def _accumulate_gradients_by_add_ops_(var_name, else: out_name = var_name pending_sum_ops[op_idx].append( - _create_op_desc_("grad_add", { - "X": [x_name], - "Y": [y_name] - }, {"Out": [out_name]}, { - "use_mkldnn": False, - "op_device": op_device - })) + _create_op_desc_( + "grad_add", + {"X": [x_name], "Y": [y_name]}, + {"Out": [out_name]}, + {"use_mkldnn": False, "op_device": op_device}, + ) + ) renamed_vars[var_name] = [var_name] -def _addup_repetitive_outputs_(op_descs, - block_idx, - grad_var_to_var=None, - grad_op_id_to_fwd_op=None): +def _addup_repetitive_outputs_( + op_descs, block_idx, grad_var_to_var=None, grad_op_id_to_fwd_op=None +): """ In backward part, an variable may be the output of more than one ops. And one op may yield its multiple outputs to the same variable. @@ -510,14 +535,15 @@ def _addup_repetitive_outputs_(op_descs, """ _MAX_ADD_NUM_ = framework._global_flags()['FLAGS_max_inplace_grad_add'] - #pending_sum_ops = [] + # pending_sum_ops = [] pending_sum_ops = collections.OrderedDict() var_rename_count = collections.defaultdict(int) renamed_vars = collections.defaultdict(list) renamed_var_start_idx = collections.defaultdict(list) var_device = collections.defaultdict(str) for idx, op_desc in enumerate(op_descs): - op_device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + op_device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) op_device = "" if op_desc.has_attr(op_device_attr_name): @@ -527,13 +553,21 @@ def _addup_repetitive_outputs_(op_descs, continue if len(renamed_vars[var_name]) > 1: if len(renamed_vars[var_name]) > _MAX_ADD_NUM_: - _accumulate_gradients_by_sum_op_(var_name, renamed_vars, - pending_sum_ops, idx, - var_device[var_name]) + _accumulate_gradients_by_sum_op_( + var_name, + renamed_vars, + pending_sum_ops, + idx, + var_device[var_name], + ) else: - _accumulate_gradients_by_add_ops_(var_name, renamed_vars, - pending_sum_ops, idx, - var_device[var_name]) + _accumulate_gradients_by_add_ops_( + var_name, + renamed_vars, + pending_sum_ops, + idx, + var_device[var_name], + ) for param_idx, param_name in enumerate(op_desc.output_names()): arg_names = op_desc.output(param_name) @@ -542,8 +576,10 @@ def _addup_repetitive_outputs_(op_descs, continue # if "@RENAME@" in var_name: # continue - if var_name == core.empty_var_name( - ) or var_name in op_desc.input_arg_names(): + if ( + var_name == core.empty_var_name() + or var_name in op_desc.input_arg_names() + ): # empty variable or inplace op continue if len(renamed_vars[var_name]) == 0: @@ -552,14 +588,20 @@ def _addup_repetitive_outputs_(op_descs, renamed_var_start_idx[var_name] = idx else: if len(renamed_vars[var_name]) == 1: - new_name = var_name + "@RENAME@block" + str(block_idx) + "@" + \ - str(var_rename_count[var_name]) + new_name = ( + var_name + + "@RENAME@block" + + str(block_idx) + + "@" + + str(var_rename_count[var_name]) + ) var_rename_count[var_name] += 1 # Build the mapping between the new_name and var_name (Only for auto parallel) if grad_var_to_var is not None: if var_name in grad_var_to_var: grad_var_to_var[new_name] = grad_var_to_var[ - var_name] + var_name + ] else: grad_var_to_var[new_name] = var_name # rename original var_name @@ -568,31 +610,45 @@ def _addup_repetitive_outputs_(op_descs, # new_name, 0, idx) # rename arg from idx of the first appearance # in backward, not always from 0 - _rename_arg_(op_descs, var_name, new_name, - renamed_var_start_idx[var_name], idx) + _rename_arg_( + op_descs, + var_name, + new_name, + renamed_var_start_idx[var_name], + idx, + ) _rename_arg_(pending_sum_ops, var_name, new_name) for p in op_desc.output_names()[:param_idx]: p_arg_names = op_desc.output(p) if var_name in p_arg_names: - op_desc.set_output(p, [ - new_name if x == var_name else x - for x in p_arg_names - ]) + op_desc.set_output( + p, + [ + new_name if x == var_name else x + for x in p_arg_names + ], + ) arg_names = [ new_name if x == var_name else x for x in arg_names[:arg_idx] ] + arg_names[arg_idx:] - new_name = var_name + "@RENAME@block" + str(block_idx) + "@" + \ - str(var_rename_count[var_name]) + new_name = ( + var_name + + "@RENAME@block" + + str(block_idx) + + "@" + + str(var_rename_count[var_name]) + ) var_rename_count[var_name] += 1 # Build the mapping between the new_name and var_name (Only for auto parallel) if grad_var_to_var is not None: if var_name in grad_var_to_var: grad_var_to_var[new_name] = grad_var_to_var[ - var_name] + var_name + ] else: grad_var_to_var[new_name] = var_name arg_names[arg_idx] = new_name @@ -604,19 +660,27 @@ def _addup_repetitive_outputs_(op_descs, for var_name, inputs in renamed_vars.items(): if len(renamed_vars[var_name]) > 1: if len(renamed_vars[var_name]) > _MAX_ADD_NUM_: - _accumulate_gradients_by_sum_op_(var_name, renamed_vars, - pending_sum_ops, len(op_descs), - var_device[var_name]) + _accumulate_gradients_by_sum_op_( + var_name, + renamed_vars, + pending_sum_ops, + len(op_descs), + var_device[var_name], + ) else: - _accumulate_gradients_by_add_ops_(var_name, - renamed_vars, pending_sum_ops, - len(op_descs), - var_device[var_name]) + _accumulate_gradients_by_add_ops_( + var_name, + renamed_vars, + pending_sum_ops, + len(op_descs), + var_device[var_name], + ) op_descs_len = len(op_descs) # sum_op descs are sorted according to their insert position for key, value in collections.OrderedDict( - reversed(list(pending_sum_ops.items()))).items(): + reversed(list(pending_sum_ops.items())) + ).items(): # NOTE(zhiqiu): Since reversed, the idx of op_descs to be inserted will remains correct. # For example, [0, 1, 2], and we want to insert 'a' at idx 1, 'b' at idx 2, and the expected result is [0, 1, 'a', 2, 'b']. @@ -626,19 +690,24 @@ def _addup_repetitive_outputs_(op_descs, for i, op in enumerate(value): # update the mapping between fwd and bwd target_idx = idx - 1 if idx == op_descs_len else idx + i - if grad_op_id_to_fwd_op is not None and grad_op_id_to_fwd_op.get( - op_descs[target_idx].original_id(), None) is not None: + if ( + grad_op_id_to_fwd_op is not None + and grad_op_id_to_fwd_op.get( + op_descs[target_idx].original_id(), None + ) + is not None + ): grad_op_id_to_fwd_op[op.original_id()] = grad_op_id_to_fwd_op[ - op_descs[target_idx].original_id()] + op_descs[target_idx].original_id() + ] op_descs.insert(idx + i, op) return op_descs -def _remove_no_grad_branch_(op_descs, - no_grad_set, - grad_op_id_to_fwd_op=None, - target_vars=[]): +def _remove_no_grad_branch_( + op_descs, no_grad_set, grad_op_id_to_fwd_op=None, target_vars=[] +): """ Remove unnecessary grad ops A grad op can be removed in two cases: @@ -651,19 +720,25 @@ def _remove_no_grad_branch_(op_descs, out_arg_names = op_desc.output_arg_names() if len(out_arg_names) == 0 or _all_in_set_(out_arg_names, no_grad_set): return True - if _all_in_set_([ - name for name in op_desc.input_arg_names() + if _all_in_set_( + [ + name + for name in op_desc.input_arg_names() if name.find(core.grad_var_suffix()) != -1 - ], no_grad_set): + ], + no_grad_set, + ): no_grad_set.update(set(out_arg_names) - target_grad_var_names) return True return False # Remove ops whose outputs are all in no_grad_dict target_grad_var_names = set( - [var.name + core.grad_var_suffix() for var in target_vars]) + [var.name + core.grad_var_suffix() for var in target_vars] + ) op_descs = [ - op_desc for op_desc in op_descs + op_desc + for op_desc in op_descs if not _op_can_be_removed_(op_desc, no_grad_set) ] # Insert fill_any_like_op with value 0 @@ -675,16 +750,21 @@ def _remove_no_grad_branch_(op_descs, x_in = _strip_grad_suffix_(arg) # the reason should be: arg can be input of another grad op # and the op is a not-to-remove op - new_op_desc = _create_op_desc_("fill_any_like", {"X": [x_in]}, - {"Out": [arg]}, { - 'value': 0, - 'dtype': -1 - }) + new_op_desc = _create_op_desc_( + "fill_any_like", + {"X": [x_in]}, + {"Out": [arg]}, + {'value': 0, 'dtype': -1}, + ) # update the mapping between fwd and bwd - if grad_op_id_to_fwd_op is not None and grad_op_id_to_fwd_op.get( - op_desc.original_id(), None) is not None: - grad_op_id_to_fwd_op[new_op_desc.original_id( - )] = grad_op_id_to_fwd_op[op_desc.original_id()] + if ( + grad_op_id_to_fwd_op is not None + and grad_op_id_to_fwd_op.get(op_desc.original_id(), None) + is not None + ): + grad_op_id_to_fwd_op[ + new_op_desc.original_id() + ] = grad_op_id_to_fwd_op[op_desc.original_id()] to_insert.append((new_op_desc, idx)) list([op_descs.insert(p[1], p[0]) for p in reversed(to_insert)]) @@ -711,7 +791,6 @@ def _find_not_need_ops(grad_op_descs, forward_ops, input_grad_names_set): """ class Var(object): - def __init__(self, var_name): self.var_name = var_name self.gen_op = None @@ -727,7 +806,6 @@ def _find_not_need_ops(grad_op_descs, forward_ops, input_grad_names_set): self.pendding_ops.append(op) class Op(object): - def __init__(self, op_desc): self.op_desc = op_desc self.inputs = [] @@ -768,8 +846,9 @@ def _find_not_need_ops(grad_op_descs, forward_ops, input_grad_names_set): return op_node # Record the forward vars - forward_vars_set = set() if input_grad_names_set is None else set( - input_grad_names_set) + forward_vars_set = ( + set() if input_grad_names_set is None else set(input_grad_names_set) + ) for op in forward_ops: forward_vars_set.update(op.desc.input_arg_names()) forward_vars_set.update(op.desc.output_arg_names()) @@ -824,14 +903,16 @@ def serialize_op_decs(op_desc): return proto.__str__() -def _append_backward_ops_with_checkpoints_(block, - ops, - target_vars, - target_block, - no_grad_dict, - grad_to_var, - checkpoints, - grad_op_id_to_fwd_op=None): +def _append_backward_ops_with_checkpoints_( + block, + ops, + target_vars, + target_block, + no_grad_dict, + grad_to_var, + checkpoints, + grad_op_id_to_fwd_op=None, +): """ Create grad ops with forward ops, and insert them into given block @@ -898,16 +979,20 @@ def _append_backward_ops_with_checkpoints_(block, # min_idx: checkpoint_1' s input op # max_idx: checkpoint_2' s output op flag, min_idx, max_idx = program_stat.is_subgraph( - [checkpoints_name[start_idx]], - [checkpoints_name[start_idx + 1]]) + [checkpoints_name[start_idx]], [checkpoints_name[start_idx + 1]] + ) if flag: # max_idx + 1 since the exact and used segment end idx is max_idx min_idx = program_stat._update_segment_start( - min_idx, pre_segment_end_idx) + min_idx, pre_segment_end_idx + ) segments.append([min_idx, max_idx + 1]) else: - _logger.info("Could not recompute op range [{}] - [{}] ".format( - min_idx, max_idx + 1)) + _logger.info( + "Could not recompute op range [{}] - [{}] ".format( + min_idx, max_idx + 1 + ) + ) start_idx += 1 @@ -918,26 +1003,42 @@ def _append_backward_ops_with_checkpoints_(block, for i, (idx1, idx2) in enumerate(recompute_segments): _logger.info("recompute segment[{}]".format(i)) - _logger.info("segment start op: [{}]: [{}]".format( - ops[idx1].desc.type(), ops[idx1].desc.input_arg_names())) - _logger.info("segment end op: [{}]: [{}]".format( - ops[idx2 - 1].desc.type(), ops[idx2 - 1].desc.input_arg_names())) + _logger.info( + "segment start op: [{}]: [{}]".format( + ops[idx1].desc.type(), ops[idx1].desc.input_arg_names() + ) + ) + _logger.info( + "segment end op: [{}]: [{}]".format( + ops[idx2 - 1].desc.type(), ops[idx2 - 1].desc.input_arg_names() + ) + ) _logger.info("recompute segment[{}]".format(i)) - _logger.info("segment start op: [{}]: [{}]".format( - ops[idx1].desc.type(), ops[idx1].desc.input_arg_names())) - _logger.info("segment end op: [{}]: [{}]".format( - ops[idx2 - 1].desc.type(), ops[idx2 - 1].desc.input_arg_names())) + _logger.info( + "segment start op: [{}]: [{}]".format( + ops[idx1].desc.type(), ops[idx1].desc.input_arg_names() + ) + ) + _logger.info( + "segment end op: [{}]: [{}]".format( + ops[idx2 - 1].desc.type(), ops[idx2 - 1].desc.input_arg_names() + ) + ) # 2) go through all forward ops and induct all variables that will be hold in memory vars_should_be_hold = [] # a. variables that are used across segments will be held in memory for segment in recompute_segments: vars_should_be_hold.extend( - program_stat.get_out_of_subgraph_vars(segment[0], segment[1])) + program_stat.get_out_of_subgraph_vars(segment[0], segment[1]) + ) cross_vars = set(vars_should_be_hold) - set(checkpoints_name) - _logger.info("found [{}] vars which cross recompute segment: [{}], better checkpoints might be set to reduce those vars".format( \ - len(cross_vars), cross_vars)) + _logger.info( + "found [{}] vars which cross recompute segment: [{}], better checkpoints might be set to reduce those vars".format( + len(cross_vars), cross_vars + ) + ) # b. output of seed op should be kept in memory vars_should_be_hold.extend(program_stat.get_reserved_vars()) @@ -957,11 +1058,14 @@ def _append_backward_ops_with_checkpoints_(block, gap_ops = ops[0:max_calculated_op_position] for op in reversed(gap_ops): if op.has_attr("sub_block"): - raise Exception("Recompute don't support ops with sub_block" - "invoke op: %s" % - _pretty_op_desc_(op.desc, "with_sub_block")) + raise Exception( + "Recompute don't support ops with sub_block" + "invoke op: %s" + % _pretty_op_desc_(op.desc, "with_sub_block") + ) grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, no_grad_dict[block.idx], []) + op.desc, no_grad_dict[block.idx], [] + ) # record the mapping between fwd and bwd if grad_op_id_to_fwd_op is not None: @@ -973,21 +1077,25 @@ def _append_backward_ops_with_checkpoints_(block, op_device = op.desc.attr(device_attr_name) for op_desc in grad_op_desc: op_desc._set_attr(device_attr_name, op_device) - added_descs = _add_descs_to_block(grad_op_desc, local_block, - grad_op_id_to_fwd_op) + added_descs = _add_descs_to_block( + grad_op_desc, local_block, grad_op_id_to_fwd_op + ) grad_op_descs.extend(added_descs) grad_to_var.update(op_grad_to_var) for i, segment in enumerate(recompute_segments[::-1]): - gap_ops = ops[segment[1]:max_calculated_op_position] + gap_ops = ops[segment[1] : max_calculated_op_position] max_calculated_op_position = segment[0] for op in reversed(gap_ops): if op.has_attr("sub_block"): - raise Exception("Recompute don't support ops with sub_block" - "invoke op: %s" % - _pretty_op_desc_(op.desc, "with_sub_block")) + raise Exception( + "Recompute don't support ops with sub_block" + "invoke op: %s" + % _pretty_op_desc_(op.desc, "with_sub_block") + ) grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, no_grad_dict[block.idx], []) + op.desc, no_grad_dict[block.idx], [] + ) # record the mapping between fwd and bwd if grad_op_id_to_fwd_op is not None: @@ -999,19 +1107,22 @@ def _append_backward_ops_with_checkpoints_(block, op_device = op.desc.attr(device_attr_name) for op_desc in grad_op_desc: op_desc._set_attr(device_attr_name, op_device) - added_descs = _add_descs_to_block(grad_op_desc, local_block, - grad_op_id_to_fwd_op) + added_descs = _add_descs_to_block( + grad_op_desc, local_block, grad_op_id_to_fwd_op + ) grad_op_descs.extend(added_descs) grad_to_var.update(op_grad_to_var) - ff_ops = ops[segment[0]:segment[1]] + ff_ops = ops[segment[0] : segment[1]] var_suffix = ".subprog_%d" % i for op in ff_ops: if op.has_attr("sub_block"): - raise Exception("Recompute don't support ops with sub_block" - "invoke op: %s" % - _pretty_op_desc_(op.desc, "with_sub_block")) + raise Exception( + "Recompute don't support ops with sub_block" + "invoke op: %s" + % _pretty_op_desc_(op.desc, "with_sub_block") + ) input_and_output_names = [] input_and_output_names.extend(op.desc.input_arg_names()) input_and_output_names.extend(op.desc.output_arg_names()) @@ -1025,19 +1136,22 @@ def _append_backward_ops_with_checkpoints_(block, # we should create the rename var in subprog, otherwise its VarType will be BOOL ref_var = block.program.global_block().var(name) - block.create_var(name=var_name_dict[name], - shape=ref_var.shape, - dtype=ref_var.dtype, - type=ref_var.type, - persistable=ref_var.persistable, - stop_gradient=ref_var.stop_gradient) + block.create_var( + name=var_name_dict[name], + shape=ref_var.shape, + dtype=ref_var.dtype, + type=ref_var.type, + persistable=ref_var.persistable, + stop_gradient=ref_var.stop_gradient, + ) # 3.a. add ops in current recompute_segment as forward recomputation ops - buffer_descs = _add_needed_descs_to_block(ff_ops, buffer_block, block, - vars_in_memory, - grad_op_id_to_fwd_op) - added_descs = _add_descs_to_block(ff_ops, local_block, - grad_op_id_to_fwd_op) + buffer_descs = _add_needed_descs_to_block( + ff_ops, buffer_block, block, vars_in_memory, grad_op_id_to_fwd_op + ) + added_descs = _add_descs_to_block( + ff_ops, local_block, grad_op_id_to_fwd_op + ) # 3.b. rename all non-checkpoint variables in recomputation ops for key in var_name_dict: @@ -1049,13 +1163,15 @@ def _append_backward_ops_with_checkpoints_(block, # 3.c. add backward ops for all ops in current segment for op_desc in reversed(added_descs): grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op_desc, no_grad_dict[block.idx], []) + op_desc, no_grad_dict[block.idx], [] + ) # record the mapping between fwd and bwd if grad_op_id_to_fwd_op is not None: for g_op_desc in grad_op_desc: - grad_op_id_to_fwd_op[g_op_desc.original_id( - )] = grad_op_id_to_fwd_op[op_desc.original_id()] + grad_op_id_to_fwd_op[ + g_op_desc.original_id() + ] = grad_op_id_to_fwd_op[op_desc.original_id()] # Set device for grad_op according to forward Op if op_desc.has_attr(device_attr_name): @@ -1070,21 +1186,33 @@ def _append_backward_ops_with_checkpoints_(block, # 3.d. add sum op for repetitive_outputs grad_op_descs = _addup_repetitive_outputs_( - grad_op_descs, block.idx, grad_op_id_to_fwd_op=grad_op_id_to_fwd_op) + grad_op_descs, block.idx, grad_op_id_to_fwd_op=grad_op_id_to_fwd_op + ) # 4) remove no grad branch as it is in _remove_no_grad_branch_ - grad_op_descs = _remove_no_grad_branch_(grad_op_descs, - no_grad_dict[block.idx], - grad_op_id_to_fwd_op, target_vars) - added_descs = _add_descs_to_block(grad_op_descs, target_block, - grad_op_id_to_fwd_op) - return program_stat, checkpoints_name, vars_should_be_hold, recompute_segments - - -def _get_sub_block_path(sub_block, - sub_block_op_desc, - no_grad_set, - op_path_dict, - sub_block_target_names=None): + grad_op_descs = _remove_no_grad_branch_( + grad_op_descs, + no_grad_dict[block.idx], + grad_op_id_to_fwd_op, + target_vars, + ) + added_descs = _add_descs_to_block( + grad_op_descs, target_block, grad_op_id_to_fwd_op + ) + return ( + program_stat, + checkpoints_name, + vars_should_be_hold, + recompute_segments, + ) + + +def _get_sub_block_path( + sub_block, + sub_block_op_desc, + no_grad_set, + op_path_dict, + sub_block_target_names=None, +): """ Get output vars in subblock which will be assigned to parent block. It is used to find the grad path in subblock. @@ -1102,8 +1230,8 @@ def _get_sub_block_path(sub_block, """ assert sub_block_op_desc.has_attr( - "sub_block") and sub_block.idx == sub_block_op_desc._block_attr_id( - "sub_block") + "sub_block" + ) and sub_block.idx == sub_block_op_desc._block_attr_id("sub_block") assert isinstance(sub_block_target_names, (set, type(None))) if sub_block_target_names is None: @@ -1123,8 +1251,9 @@ def _get_sub_block_path(sub_block, # Step2: find op path of sub-block is_while = sub_block_op_desc.type in ["while"] - sub_block_op_path = _find_op_path_(sub_block, sub_outputs, [], - no_grad_set, op_path_dict, is_while) + sub_block_op_path = _find_op_path_( + sub_block, sub_outputs, [], no_grad_set, op_path_dict, is_while + ) return sub_block_op_path return sub_block.ops @@ -1132,8 +1261,9 @@ def _get_sub_block_path(sub_block, def _is_grad_op_(op): op_maker = core.op_proto_and_checker_maker backward = core.op_proto_and_checker_maker.OpRole.Backward - if op_maker.kOpRoleVarAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(backward): + if op_maker.kOpRoleVarAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(backward): return True return False @@ -1142,18 +1272,20 @@ def _rename_grad_name_(name, grad_order): return 'grad/' * grad_order + name -def _append_backward_ops_(block, - ops, - target_vars, - target_block, - no_grad_dict, - grad_to_var, - callbacks=None, - input_grad_names_set=None, - op_path_dict=None, - distop_context=None, - rename_var_map=None, - grad_op_id_to_fwd_op=None): +def _append_backward_ops_( + block, + ops, + target_vars, + target_block, + no_grad_dict, + grad_to_var, + callbacks=None, + input_grad_names_set=None, + op_path_dict=None, + distop_context=None, + rename_var_map=None, + grad_op_id_to_fwd_op=None, +): """ Create all grad ops, and insert them into given block @@ -1180,18 +1312,22 @@ def _append_backward_ops_(block, """ # Build the mapping between the forward op and backward op (Only for auto parallel) - def update_distop_context(distop_context, op_grad_to_var, - appending_grad_times): + def update_distop_context( + distop_context, op_grad_to_var, appending_grad_times + ): distop_context.grad_var_to_var[appending_grad_times].update( - op_grad_to_var) + op_grad_to_var + ) for op_desc in grad_op_desc: - assert op_desc.original_id( - ) not in distop_context.grad_op_id_to_op_id + assert ( + op_desc.original_id() not in distop_context.grad_op_id_to_op_id + ) distop_context.grad_op_id_to_op_id[ - op_desc.original_id()] = op.desc.original_id() + op_desc.original_id() + ] = op.desc.original_id() if callbacks is not None: - assert (isinstance(callbacks, (list, tuple))) + assert isinstance(callbacks, (list, tuple)) for cb in callbacks: if not hasattr(cb, '__call__'): raise ValueError("'callback' must be a callable object.") @@ -1216,16 +1352,18 @@ def _append_backward_ops_(block, pre_input_grad_names_set = copy.copy(input_grad_names_set) input_grad_names_set = None sub_block_path = op_path_dict[op._block_attr_id("sub_block")] - _append_backward_ops_(sub_block, - sub_block_path, - target_vars, - grad_sub_block, - no_grad_dict, - grad_to_var, - callbacks, - input_grad_names_set, - op_path_dict, - grad_op_id_to_fwd_op=grad_op_id_to_fwd_op) + _append_backward_ops_( + sub_block, + sub_block_path, + target_vars, + grad_sub_block, + no_grad_dict, + grad_to_var, + callbacks, + input_grad_names_set, + op_path_dict, + grad_op_id_to_fwd_op=grad_op_id_to_fwd_op, + ) input_grad_names_set = pre_input_grad_names_set program._rollback() @@ -1233,7 +1371,8 @@ def _append_backward_ops_(block, # Getting op's corresponding grad_op grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, no_grad_dict[block.idx], grad_sub_block_list) + op.desc, no_grad_dict[block.idx], grad_sub_block_list + ) # record the mapping between fwd and bwd if grad_op_id_to_fwd_op is not None: @@ -1242,15 +1381,22 @@ def _append_backward_ops_(block, # Build the mapping between the forward op and backward op (Only for auto parallel) if distop_context is not None: - update_distop_context(distop_context, op_grad_to_var, - program._appending_grad_times) + update_distop_context( + distop_context, op_grad_to_var, program._appending_grad_times + ) else: - default_ctx = getattr(paddle.distributed.auto_parallel.dist_context, - '_g_default_distributed_context', None) + default_ctx = getattr( + paddle.distributed.auto_parallel.dist_context, + '_g_default_distributed_context', + None, + ) if default_ctx is not None: distop_context = default_ctx.dist_op_context - update_distop_context(distop_context, op_grad_to_var, - program._appending_grad_times) + update_distop_context( + distop_context, + op_grad_to_var, + program._appending_grad_times, + ) # Set device for grad_op according to forward Op device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName() @@ -1278,7 +1424,8 @@ def _append_backward_ops_(block, continue if block.desc.find_var(name.encode("ascii")): new_name = _rename_grad_name_( - name, program._appending_grad_times) + name, program._appending_grad_times + ) op_desc._rename_output(name, new_name) rename_var_map[name] = new_name @@ -1286,8 +1433,8 @@ def _append_backward_ops_(block, # Build the mapping between the grad var name and var name (Only for auto parallel) if distop_context is not None: distop_context.grad_var_to_var[ - program._appending_grad_times][ - new_name] = op_grad_to_var[name] + program._appending_grad_times + ][new_name] = op_grad_to_var[name] op_grad_to_var[new_name] = op_grad_to_var[name] op_grad_to_var.pop(name) @@ -1296,12 +1443,15 @@ def _append_backward_ops_(block, # But this strategy is not suited for while op for some control flow, # for example, for while op, the grads maybe generated in next loop. if input_grad_names_set is not None: - is_grad_name = lambda name: name.find(core.grad_var_suffix( - )) != -1 or name in input_grad_names_set + is_grad_name = ( + lambda name: name.find(core.grad_var_suffix()) != -1 + or name in input_grad_names_set + ) is_append_grad = False for op_desc in grad_op_desc: input_grad_names = [ - name for name in op_desc.input_arg_names() + name + for name in op_desc.input_arg_names() if is_grad_name(name) ] # some code of gradient ops, like increment, are not very @@ -1325,19 +1475,24 @@ def _append_backward_ops_(block, grad_var_to_var = None if distop_context is not None: grad_var_to_var = distop_context.grad_var_to_var[ - program._appending_grad_times] + program._appending_grad_times + ] # sum parameter's gradients' var given multiple var gradient grad_op_descs = _addup_repetitive_outputs_( grad_op_descs, block.idx, grad_var_to_var, - grad_op_id_to_fwd_op=grad_op_id_to_fwd_op) + grad_op_id_to_fwd_op=grad_op_id_to_fwd_op, + ) # if all outputs of the grad op are in no_grad_set, then just remove and fill zero # if all inputs of the grad op are in no_grad_set, just remove this op - grad_op_descs = _remove_no_grad_branch_(grad_op_descs, - no_grad_dict[block.idx], - grad_op_id_to_fwd_op, target_vars) + grad_op_descs = _remove_no_grad_branch_( + grad_op_descs, + no_grad_dict[block.idx], + grad_op_id_to_fwd_op, + target_vars, + ) # remove some backward ops not_need_ops = _find_not_need_ops(grad_op_descs, ops, input_grad_names_set) @@ -1355,7 +1510,7 @@ def _append_backward_ops_(block, new_op_desc._set_attr(op_role_attr_name, backward) grad_to_var["__current_op_desc__"] = new_op_desc if callbacks is not None: - assert (isinstance(callbacks, (list, tuple))) + assert isinstance(callbacks, (list, tuple)) for cb in callbacks: cb(block=target_block, context=grad_to_var) @@ -1376,8 +1531,10 @@ def _find_parent_op_(sub_block): block_desc = program.block(block_id).desc for op_idx in range(block_desc.op_size()): op = block_desc.op(op_idx) - if op.has_attr("sub_block") and op._block_attr_id( - "sub_block") == sub_block_id: + if ( + op.has_attr("sub_block") + and op._block_attr_id("sub_block") == sub_block_id + ): return op # NOTE(paddle-dev): When optimizer is added in conditional block, @@ -1431,11 +1588,13 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): ] inputs = [ - var for var in op_desc.input_arg_names() + var + for var in op_desc.input_arg_names() if var != core.empty_var_name() ] outputs = [ - var for var in op_desc.output_arg_names() + var + for var in op_desc.output_arg_names() if var != core.empty_var_name() ] @@ -1450,7 +1609,8 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): ''' if grad_var_ins: existing_grad_var_ins = [ - var for var in grad_var_ins + var + for var in grad_var_ins if block.desc.has_var_recursive(var.encode()) or var in parent_op_vars ] @@ -1470,8 +1630,10 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): new_vars = set() # create new gradient variables for grad_var_name in op_desc.output_arg_names(): - if block.desc.has_var_recursive(grad_var_name.encode( - )) or grad_var_name == core.empty_var_name(): + if ( + block.desc.has_var_recursive(grad_var_name.encode()) + or grad_var_name == core.empty_var_name() + ): continue block.desc.var(grad_var_name.encode()) new_vars.add(grad_var_name) @@ -1550,21 +1712,26 @@ def _get_no_grad_set_name(no_grad_set): else: raise TypeError( "The type of no_grad_set's member must be paddle.fluid.Variable or str, but received %s." - % (type(no_grad_var))) + % (type(no_grad_var)) + ) else: raise TypeError( - "The type of no_grad_set should be set or list or tuple, but received {}" - .format(type(no_grad_set))) + "The type of no_grad_set should be set or list or tuple, but received {}".format( + type(no_grad_set) + ) + ) return no_grad_set_name @framework.static_only -def append_backward(loss, - parameter_list=None, - no_grad_set=None, - callbacks=None, - checkpoints=None, - distop_context=None): +def append_backward( + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + checkpoints=None, + distop_context=None, +): """ :api_attr: Static Graph @@ -1665,11 +1832,13 @@ def append_backward(loss, p_g_list6 = paddle.static.append_backward(loss=avg_loss, parameter_list=all_weights, no_grad_set=set(all_weights)) """ - grad_op_id_to_fwd_op = { - } # for cuda graph usage, recording the mapping between grad op original id to fwd op + grad_op_id_to_fwd_op = ( + {} + ) # for cuda graph usage, recording the mapping between grad op original id to fwd op - check_type(loss, 'loss', framework.Variable, - 'paddle.static.append_backward') + check_type( + loss, 'loss', framework.Variable, 'paddle.static.append_backward' + ) if loss.op is None: # the loss is from a cloned program. Find loss op manually. @@ -1678,11 +1847,16 @@ def append_backward(loss, loss.op._set_attr( core.op_proto_and_checker_maker.kOpRoleAttrName(), int(core.op_proto_and_checker_maker.OpRole.Forward) - | int(core.op_proto_and_checker_maker.OpRole.Loss)) + | int(core.op_proto_and_checker_maker.OpRole.Loss), + ) if callbacks is not None: - check_type(callbacks, 'callbacks', (list, tuple), - 'paddle.static.append_backward') + check_type( + callbacks, + 'callbacks', + (list, tuple), + 'paddle.static.append_backward', + ) program = loss.block.program root_block = program.block(0) @@ -1712,14 +1886,16 @@ def append_backward(loss, if is_in_control_flow: # create grad block if in switch control flow. target_grad_block = program._create_block( - parent_idx=current_block.parent_idx) + parent_idx=current_block.parent_idx + ) target_grad_block._set_forward_block_idx(current_block_idx) # after _create_block, program.current_block changes else: target_grad_block = root_block son_parent_block_idx_dict = _get_son_parent_block_idx_dict( - program, current_block_idx) + program, current_block_idx + ) block_fwd_op_num_dict = {} # block_id: fwd_op_num for idx in son_parent_block_idx_dict: @@ -1736,18 +1912,22 @@ def append_backward(loss, block = program.block(block_idx) block_no_grad_set = set( - map(_strip_grad_suffix_, no_grad_dict[block_idx])) + map(_strip_grad_suffix_, no_grad_dict[block_idx]) + ) op_path_dict = dict() - op_path = _find_op_path_(block, [loss], [], block_no_grad_set, - op_path_dict) + op_path = _find_op_path_( + block, [loss], [], block_no_grad_set, op_path_dict + ) - no_grad_vars = _find_no_grad_vars(block, op_path, [loss], - block_no_grad_set) + no_grad_vars = _find_no_grad_vars( + block, op_path, [loss], block_no_grad_set + ) block_no_grad_set.update(no_grad_vars) no_grad_dict[block_idx].update( - list(map(_append_grad_suffix_, block_no_grad_set))) + list(map(_append_grad_suffix_, block_no_grad_set)) + ) input_grad_names_set = None # For double backward, input_grad_names is used for filtering @@ -1762,22 +1942,27 @@ def append_backward(loss, # TODO: support _append_backward_ops_with_checkpoints_ in # sub-block (control flow) is_recompute = False - if checkpoints != None and \ - isinstance(checkpoints, list) and \ - len(checkpoints) > 0: + if ( + checkpoints != None + and isinstance(checkpoints, list) + and len(checkpoints) > 0 + ): is_recompute = True - program_stat, checkpoint_names, \ - vars_should_be_hold, \ - recompute_segments = \ - _append_backward_ops_with_checkpoints_( - root_block, - op_path, - [loss], - root_block, - no_grad_dict, - grad_to_var, - checkpoints, - grad_op_id_to_fwd_op) + ( + program_stat, + checkpoint_names, + vars_should_be_hold, + recompute_segments, + ) = _append_backward_ops_with_checkpoints_( + root_block, + op_path, + [loss], + root_block, + no_grad_dict, + grad_to_var, + checkpoints, + grad_op_id_to_fwd_op, + ) else: _append_backward_ops_( block, # the block where forward ops are in @@ -1790,22 +1975,27 @@ def append_backward(loss, input_grad_names_set=input_grad_names_set, op_path_dict=op_path_dict, distop_context=distop_context, - grad_op_id_to_fwd_op=grad_op_id_to_fwd_op) + grad_op_id_to_fwd_op=grad_op_id_to_fwd_op, + ) grad_info_map = dict() # if in control flow, target_grad_block is a created new block which only contains grad ops, # so fwd_op_num is set to 0. - fwd_op_num = block_fwd_op_num_dict[ - current_block_idx] if not is_in_control_flow else 0 + fwd_op_num = ( + block_fwd_op_num_dict[current_block_idx] + if not is_in_control_flow + else 0 + ) # Because append_backward may be called multiple times, # we need rename the internal gradient variables so that they have # different names. _rename_grad_(target_grad_block, fwd_op_num, grad_to_var, {}) - _append_backward_vars_(target_grad_block, fwd_op_num, grad_to_var, - grad_info_map) + _append_backward_vars_( + target_grad_block, fwd_op_num, grad_to_var, grad_info_map + ) program.current_block_idx = current_block_idx program._sync_with_cpp() @@ -1817,13 +2007,20 @@ def append_backward(loss, op._cuda_graph_attr = fwd_op._cuda_graph_attr if parameter_list is not None: - check_type(parameter_list, 'parameter_list', (list, tuple, set), - 'fluid.backward.append_backward') + check_type( + parameter_list, + 'parameter_list', + (list, tuple, set), + 'fluid.backward.append_backward', + ) parameters = [] for i, param in enumerate(parameter_list): - check_type(param, 'parameter_list[%s]' % i, - (framework.Variable, str), - 'fluid.backward.append_backward') + check_type( + param, + 'parameter_list[%s]' % i, + (framework.Variable, str), + 'fluid.backward.append_backward', + ) if isinstance(param, framework.Variable): parameters.append(param.name) elif isinstance(param, str): @@ -1840,8 +2037,11 @@ def append_backward(loss, grad_info = grad_info_map[param] grad_block = grad_info[1] if not grad_block.has_var(grad_info[0]): - raise ValueError("grad block[{0}] did not have grad var {1}".format( - grad_info[1], grad_info[0])) + raise ValueError( + "grad block[{0}] did not have grad var {1}".format( + grad_info[1], grad_info[0] + ) + ) # Get the param var from the global block param_var = program.global_block().var(param) grad_var = grad_block.var(grad_info[0]) @@ -1856,8 +2056,9 @@ def append_backward(loss, for p, g in params_and_grads: if g is None: continue - ops = grad_block.ops if is_in_control_flow else program.global_block( - ).ops + ops = ( + grad_block.ops if is_in_control_flow else program.global_block().ops + ) for op in reversed(ops): assert isinstance(op, framework.Operator) if g.name in op.output_arg_names: @@ -1924,8 +2125,9 @@ def _get_output_names(cur_block, targets): if _some_in_set_(op.desc.output_arg_names(), current_output_names): for name in op.desc.input_arg_names(): current_output_names.add(name) - if not block.desc.find_var(name.encode()) \ - and parent_block.desc.find_var(name.encode()): + if not block.desc.find_var( + name.encode() + ) and parent_block.desc.find_var(name.encode()): parent_block_output_names.add(name) block = parent_block @@ -1945,8 +2147,11 @@ def _find_no_grad_vars(block, op_path, targets, no_grad_set): # If the op has sub_block, it is too complicated to find the correct no_grad_var. if not op.has_attr("sub_block"): for out_var in op.desc.output_arg_names(): - if out_var not in output_names and out_var not in op.desc.input_arg_names( - ) and not block.vars[out_var].stop_gradient: + if ( + out_var not in output_names + and out_var not in op.desc.input_arg_names() + and not block.vars[out_var].stop_gradient + ): no_grad_var.append(out_var) for name in op.desc.input_arg_names(): if name not in no_grad_set: @@ -1954,12 +2159,9 @@ def _find_no_grad_vars(block, op_path, targets, no_grad_set): return set(no_grad_var) -def _find_op_path_(block, - targets, - inputs, - no_grad_set, - op_path_dict=None, - is_while=False): +def _find_op_path_( + block, targets, inputs, no_grad_set, op_path_dict=None, is_while=False +): """ It is used to find the grad path in `block`. @@ -1986,9 +2188,9 @@ def _find_op_path_(block, # All the inputs of the block are used if inputs is empty, if inputs: for i, op in enumerate(block.ops): - if _some_in_set_(op.desc.input_arg_names(), - input_names) and core.has_non_empty_grad_op_maker( - op.type): + if _some_in_set_( + op.desc.input_arg_names(), input_names + ) and core.has_non_empty_grad_op_maker(op.type): for name in op.desc.output_arg_names(): if name not in no_grad_set: input_names.add(name) @@ -2000,14 +2202,14 @@ def _find_op_path_(block, sub_block_id = op._block_attr_id("sub_block") sub_block = block.program.block(sub_block_id) sub_block_target_names = output_names & set(op.output_arg_names) - sub_block_path = _get_sub_block_path(sub_block, op, set(), - op_path_dict, - sub_block_target_names) + sub_block_path = _get_sub_block_path( + sub_block, op, set(), op_path_dict, sub_block_target_names + ) op_path_dict[sub_block_id] = sub_block_path - if _some_in_set_(op.desc.output_arg_names(), - output_names) and core.has_non_empty_grad_op_maker( - op.type): + if _some_in_set_( + op.desc.output_arg_names(), output_names + ) and core.has_non_empty_grad_op_maker(op.type): for name in op.desc.input_arg_names(): if name not in no_grad_set: output_names.add(name) @@ -2018,8 +2220,9 @@ def _find_op_path_(block, # If block is while block, dealing with op specifically again. # TODO(liym27): Consider special types of ops. for i, op in reversed(list(enumerate(block.ops))): - if relevant_op_flags[i] == False \ - and _some_in_set_(op.desc.output_arg_names(), output_names): + if relevant_op_flags[i] == False and _some_in_set_( + op.desc.output_arg_names(), output_names + ): relevant_op_flags[i] = True op_path = [ @@ -2072,7 +2275,8 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): if len(targets) != len(target_gradients): raise ValueError( - "Should have the same number of target_gradients as targets") + "Should have the same number of target_gradients as targets" + ) if no_grad_set is None: no_grad_set = set() @@ -2093,16 +2297,24 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): if grad is None: target_shape = target.name + '_shape' block.desc.append_op().copy_from( - _create_op_desc_("shape", {'Input': [target.name]}, - {"Out": [target_shape]}, {})) + _create_op_desc_( + "shape", + {'Input': [target.name]}, + {"Out": [target_shape]}, + {}, + ) + ) input_grad_names_set.add(target_shape) - op_desc = _create_op_desc_("fill_constant", - {"ShapeTensor": [target_shape]}, - {"Out": [grad_name]}, { - "shape": target.shape, - "value": 1.0, - "dtype": target.dtype, - }) + op_desc = _create_op_desc_( + "fill_constant", + {"ShapeTensor": [target_shape]}, + {"Out": [grad_name]}, + { + "shape": target.shape, + "value": 1.0, + "dtype": target.dtype, + }, + ) block.desc.append_op().copy_from(op_desc) input_grad_names_set.add(grad_name) @@ -2111,8 +2323,9 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): raise ValueError("all targets must be in the same block") if target.shape != grad.shape: raise ValueError( - "The shapes of target and grad are different: %s %s" % - (target.name, grad.name)) + "The shapes of target and grad are different: %s %s" + % (target.name, grad.name) + ) target_grad_map[_append_grad_suffix_(target.name)] = grad.name input_grad_names_set.add(grad.name) rename_var_map[grad_name] = grad.name @@ -2131,26 +2344,30 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): block_no_grad_set = set(map(_strip_grad_suffix_, no_grad_dict[0])) op_path_dict = dict() - op_path = _find_op_path_(block, targets, inputs, block_no_grad_set, - op_path_dict) + op_path = _find_op_path_( + block, targets, inputs, block_no_grad_set, op_path_dict + ) # find no grad var by op_path - no_grad_vars = _find_no_grad_vars(block, op_path, targets, - block_no_grad_set) + no_grad_vars = _find_no_grad_vars( + block, op_path, targets, block_no_grad_set + ) block_no_grad_set.update(no_grad_vars) no_grad_dict[0].update(list(map(_append_grad_suffix_, block_no_grad_set))) grad_to_var = dict() grad_info_map = dict() - _append_backward_ops_(block, - op_path, - targets, - block, - no_grad_dict, - grad_to_var, - input_grad_names_set=input_grad_names_set, - op_path_dict=op_path_dict, - rename_var_map=rename_var_map) + _append_backward_ops_( + block, + op_path, + targets, + block, + no_grad_dict, + grad_to_var, + input_grad_names_set=input_grad_names_set, + op_path_dict=op_path_dict, + rename_var_map=rename_var_map, + ) # Because calc_gradient may be called multiple times, # we need rename the internal gradient variables so that they have @@ -2214,13 +2431,24 @@ def gradients(targets, inputs, target_gradients=None, no_grad_set=None): z = paddle.static.gradients([y], x) print(z) # [var x@GRAD : LOD_TENSOR.shape(-1, 2, 8, 8).dtype(float32).stop_gradient(False)] """ - check_type(targets, 'targets', (framework.Variable, list, tuple), - 'paddle.static.gradients') - check_type(inputs, 'inputs', (framework.Variable, list, tuple), - 'paddle.static.gradients') - check_type(target_gradients, 'target_gradients', - (framework.Variable, list, tuple, type(None)), - 'paddle.static.gradients') + check_type( + targets, + 'targets', + (framework.Variable, list, tuple), + 'paddle.static.gradients', + ) + check_type( + inputs, + 'inputs', + (framework.Variable, list, tuple), + 'paddle.static.gradients', + ) + check_type( + target_gradients, + 'target_gradients', + (framework.Variable, list, tuple, type(None)), + 'paddle.static.gradients', + ) outs = calc_gradient(targets, inputs, target_gradients, no_grad_set) return _as_list(outs) @@ -2263,10 +2491,18 @@ def gradients_with_optimizer(program, optimizer, inputs=None, outputs=None): print(opt_ops) """ - check_type(program, 'program', paddle.fluid.Program, - 'paddle.static.gradients_with_optimizer') - check_type(optimizer, 'optimizer', paddle.optimizer.Optimizer, - 'paddle.static.gradients_with_optimizer') + check_type( + program, + 'program', + paddle.fluid.Program, + 'paddle.static.gradients_with_optimizer', + ) + check_type( + optimizer, + 'optimizer', + paddle.optimizer.Optimizer, + 'paddle.static.gradients_with_optimizer', + ) if inputs is None or outputs is None: in_set = set() @@ -2285,9 +2521,12 @@ def gradients_with_optimizer(program, optimizer, inputs=None, outputs=None): grads = gradients(outputs, inputs) with program_guard(program, None): - pram_grads = [(pram, grad) for pram, grad in zip(inputs, grads) - if isinstance(pram, paddle.fluid.framework.Parameter) - and grad is not None] + pram_grads = [ + (pram, grad) + for pram, grad in zip(inputs, grads) + if isinstance(pram, paddle.fluid.framework.Parameter) + and grad is not None + ] optimize_ops = optimizer.apply_gradients(pram_grads) diff --git a/python/paddle/fluid/clip.py b/python/paddle/fluid/clip.py index fd5c4579468..93ad5564cb2 100644 --- a/python/paddle/fluid/clip.py +++ b/python/paddle/fluid/clip.py @@ -29,8 +29,11 @@ from .framework import default_main_program from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'set_gradient_clip', 'ErrorClipByValue', 'ClipGradByValue', - 'ClipGradByNorm', 'ClipGradByGlobalNorm' + 'set_gradient_clip', + 'ErrorClipByValue', + 'ClipGradByValue', + 'ClipGradByNorm', + 'ClipGradByGlobalNorm', ] _clip_by_global_norm_using_mp_type_flag = False @@ -49,9 +52,10 @@ def _clip_by_global_norm_using_mp_type(*args): def _cast_to_mp_type_if_enabled(x): - if (x.dtype == core.VarDesc.VarType.FP16 - or x.dtype == core.VarDesc.VarType.BF16 - ) and _clip_by_global_norm_using_mp_type(): + if ( + x.dtype == core.VarDesc.VarType.FP16 + or x.dtype == core.VarDesc.VarType.BF16 + ) and _clip_by_global_norm_using_mp_type(): return x.astype(core.VarDesc.VarType.FP32) else: return x @@ -63,8 +67,11 @@ def _squared_l2_norm(x): """ x = _cast_to_mp_type_if_enabled(x) - if core.is_compiled_with_xpu( - ) or x.dtype == core.VarDesc.VarType.FP16 or x.dtype == core.VarDesc.VarType.BF16: + if ( + core.is_compiled_with_xpu() + or x.dtype == core.VarDesc.VarType.FP16 + or x.dtype == core.VarDesc.VarType.BF16 + ): square = layers.square(x) sum_square = layers.reduce_sum(square) return sum_square @@ -86,7 +93,6 @@ def _squared_l2_norm(x): class BaseErrorClipAttr(object): - def __str__(self): raise NotImplementedError() @@ -161,8 +167,9 @@ def error_clip_callback(block, context): for grad_n in [n for n in op_desc.output_arg_names() if n in grad_to_var]: fwd_var = block._var_recursive(grad_to_var[grad_n]) error_clip = getattr(fwd_var, "error_clip", None) - if not (error_clip is None - or isinstance(error_clip, BaseErrorClipAttr)): + if not ( + error_clip is None or isinstance(error_clip, BaseErrorClipAttr) + ): raise TypeError( "Variable's error_clip should be an instance of BaseErrorClipAttr or None." ) @@ -171,7 +178,6 @@ def error_clip_callback(block, context): class ClipGradBase(object): - def __init__(self): super(ClipGradBase, self).__init__() @@ -194,7 +200,8 @@ class ClipGradBase(object): warnings.warn( "'set_gradient_clip' will be ineffective, because you have " "set 'need_clip' in 'ParamAttr'. So, 'set_gradient_clip' " - "is redundant and you can remove it.") + "is redundant and you can remove it." + ) break return self._static_clip(params_grads) @@ -249,7 +256,7 @@ class ClipGradByValue(ClipGradBase): def __init__(self, max, min=None): super(ClipGradByValue, self).__init__() if min is None: - assert (max > 0.0) + assert max > 0.0 min = -max self.max = float(max) self.min = float(min) @@ -465,10 +472,9 @@ class ClipGradByGlobalNorm(ClipGradBase): sdg.step() """ - def __init__(self, - clip_norm, - group_name="default_group", - auto_skip_clip=False): + def __init__( + self, clip_norm, group_name="default_group", auto_skip_clip=False + ): super(ClipGradByGlobalNorm, self).__init__() self.clip_norm = float(clip_norm) self.group_name = group_name @@ -500,7 +506,10 @@ class ClipGradByGlobalNorm(ClipGradBase): merge_grad = layers.get_tensor_from_selected_rows(merge_grad) sum_square = _squared_l2_norm(merge_grad) - if sum_square.dtype == core.VarDesc.VarType.FP16 or sum_square.dtype == core.VarDesc.VarType.BF16: + if ( + sum_square.dtype == core.VarDesc.VarType.FP16 + or sum_square.dtype == core.VarDesc.VarType.BF16 + ): sum_square_list_fp16.append(sum_square) elif sum_square.dtype == core.VarDesc.VarType.FP32: sum_square_list_fp32.append(sum_square) @@ -508,8 +517,12 @@ class ClipGradByGlobalNorm(ClipGradBase): sum_square_list.append(sum_square) # all parameters have been filterd out - if len(sum_square_list) + len(sum_square_list_fp16) + len( - sum_square_list_fp32) == 0: + if ( + len(sum_square_list) + + len(sum_square_list_fp16) + + len(sum_square_list_fp32) + == 0 + ): return params_grads sum_dtype = 'float64' if len(sum_square_list) > 0 else "float32" @@ -528,22 +541,23 @@ class ClipGradByGlobalNorm(ClipGradBase): global_norm_var.append(global_norm_var_fp64) global_norm_var = paddle.add_n(global_norm_var) global_norm_var = layers.sqrt(global_norm_var) - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) need_clip = False if not self.auto_skip_clip: # always apply clip need_clip = True - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var, - y=max_global_norm)) + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var, y=max_global_norm), + ) elif global_norm_var > max_global_norm: # only when global_norm_var > max_global_norm, grad need clip need_clip = True - clip_var = layers.elementwise_div(x=max_global_norm, - y=global_norm_var) + clip_var = layers.elementwise_div( + x=max_global_norm, y=global_norm_var + ) for p, g in params_grads: if g is None: @@ -553,8 +567,11 @@ class ClipGradByGlobalNorm(ClipGradBase): continue # TODO(wangxi): use inplace elementwise_mul if need_clip: - clip_input = (clip_var.astype(g.dtype) - if clip_var.dtype != g.dtype else clip_var) + clip_input = ( + clip_var.astype(g.dtype) + if clip_var.dtype != g.dtype + else clip_var + ) new_grad = layers.elementwise_mul(g, clip_input) params_and_grads.append((p, new_grad)) else: @@ -578,7 +595,8 @@ class ClipGradByGlobalNorm(ClipGradBase): if g.type == core.VarDesc.VarType.SELECTED_ROWS: merge_grad = layers.merge_selected_rows(g) merge_grad = layers.get_tensor_from_selected_rows( - merge_grad) + merge_grad + ) sum_square = _squared_l2_norm(merge_grad) if sum_square.dtype == core.VarDesc.VarType.FP16: sum_square_list_fp16.append(sum_square) @@ -588,8 +606,12 @@ class ClipGradByGlobalNorm(ClipGradBase): sum_square_list.append(sum_square) # all parameters have been filterd out - if len(sum_square_list) + len(sum_square_list_fp16) + len( - sum_square_list_fp32) == 0: + if ( + len(sum_square_list) + + len(sum_square_list_fp16) + + len(sum_square_list_fp32) + == 0 + ): return params_grads with p.block.program._optimized_guard([p, g]): @@ -598,10 +620,14 @@ class ClipGradByGlobalNorm(ClipGradBase): global_norm_var = [] if len(sum_square_list_fp16) > 0: global_norm_var_fp16 = layers.sums(sum_square_list_fp16) - if sum_square_list_fp32 or sum_square_list or not _allow_pure_fp16_global_norm_clip( + if ( + sum_square_list_fp32 + or sum_square_list + or not _allow_pure_fp16_global_norm_clip() ): global_norm_var.append( - global_norm_var_fp16.astype(sum_dtype)) + global_norm_var_fp16.astype(sum_dtype) + ) else: global_norm_var.append(global_norm_var_fp16) if len(sum_square_list_fp32) > 0: @@ -610,23 +636,28 @@ class ClipGradByGlobalNorm(ClipGradBase): global_norm_var.append(global_norm_var_fp32) else: global_norm_var.append( - global_norm_var_fp32.astype(sum_dtype)) + global_norm_var_fp32.astype(sum_dtype) + ) if len(sum_square_list) > 0: # fp64 global_norm_var_other_dtype = layers.sums(sum_square_list) global_norm_var.append(global_norm_var_other_dtype) - global_norm_var = layers.sums(global_norm_var) if len( - global_norm_var) > 1 else global_norm_var[0] + global_norm_var = ( + layers.sums(global_norm_var) + if len(global_norm_var) > 1 + else global_norm_var[0] + ) global_norm_var = layers.sqrt(x=global_norm_var) max_global_norm = layers.fill_constant( - shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) - scale_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=max_global_norm, - y=global_norm_var)) + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) + scale_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max( + x=max_global_norm, y=global_norm_var + ), + ) param_new_grad_name_dict = dict() for p, g in params_grads: if g is None: @@ -638,29 +669,32 @@ class ClipGradByGlobalNorm(ClipGradBase): with p.block.program._optimized_guard([p, g]): new_g = _cast_to_mp_type_if_enabled(g) # inplace - scale_input = (scale_var.astype('float16') if - new_g.dtype == core.VarDesc.VarType.FP16 and - scale_var.dtype != core.VarDesc.VarType.FP16 - else scale_var) + scale_input = ( + scale_var.astype('float16') + if new_g.dtype == core.VarDesc.VarType.FP16 + and scale_var.dtype != core.VarDesc.VarType.FP16 + else scale_var + ) # NOTE(Yuang Liu): For pure dp with gradient merge, the p and g # will be in different blocks with the gradient clip related ops. # We need to handle the correct block, otherwise will encounter # a 'NotFoundError' during compile time. block = default_main_program().current_block() - block.append_op(type='elementwise_mul', - inputs={ - 'X': new_g, - 'Y': scale_input - }, - outputs={'Out': new_g}) + block.append_op( + type='elementwise_mul', + inputs={'X': new_g, 'Y': scale_input}, + outputs={'Out': new_g}, + ) if new_g is not g: - block.append_op(type='cast', - inputs={'X': new_g}, - outputs={'Out': g}, - attrs={ - 'in_dtype': new_g.dtype, - 'out_dtype': g.dtype - }) + block.append_op( + type='cast', + inputs={'X': new_g}, + outputs={'Out': g}, + attrs={ + 'in_dtype': new_g.dtype, + 'out_dtype': g.dtype, + }, + ) param_new_grad_name_dict[p.name] = g.name params_and_grads.append((p, g)) @@ -673,7 +707,8 @@ class ClipGradByGlobalNorm(ClipGradBase): context[self.group_name] = [] context[self.group_name + "_clip_value"] = self.clip_norm context[self.group_name + "_clip"] = layers.fill_constant( - shape=[1], dtype=grad.dtype, value=self.clip_norm) + shape=[1], dtype=grad.dtype, value=self.clip_norm + ) else: if not self.clip_norm == context[self.group_name + "_clip_value"]: raise ValueError( @@ -696,20 +731,19 @@ class ClipGradByGlobalNorm(ClipGradBase): group_norm_var = layers.sums(input=self.context[self.group_name]) group_norm_var = layers.sqrt(x=group_norm_var) clip_var = self.context[self.group_name + "_clip"] - group_scale_var = layers.elementwise_div(x=clip_var, - y=layers.elementwise_max( - x=clip_var, - y=group_norm_var)) - assert group_scale_var.shape == (1, ) + group_scale_var = layers.elementwise_div( + x=clip_var, + y=layers.elementwise_max(x=clip_var, y=group_norm_var), + ) + assert group_scale_var.shape == (1,) self.context[group_scale_name] = group_scale_var # inplace - param.block.append_op(type='elementwise_mul', - inputs={ - 'X': grad, - 'Y': self.context[group_scale_name] - }, - outputs={'Out': grad}) + param.block.append_op( + type='elementwise_mul', + inputs={'X': grad, 'Y': self.context[group_scale_name]}, + outputs={'Out': grad}, + ) return param, grad @@ -804,22 +838,26 @@ def set_gradient_clip(clip, param_list=None, program=None): """ - warnings.warn("Caution! 'set_gradient_clip' is not recommended " - "and may be deprecated in future! " - "We recommend a new strategy: set 'grad_clip' " - "when initializing the 'optimizer'. " - "This method can reduce the mistakes, please " - "refer to documention of 'optimizer'.") + warnings.warn( + "Caution! 'set_gradient_clip' is not recommended " + "and may be deprecated in future! " + "We recommend a new strategy: set 'grad_clip' " + "when initializing the 'optimizer'. " + "This method can reduce the mistakes, please " + "refer to documention of 'optimizer'." + ) if not isinstance(clip, ClipGradBase): raise TypeError( - "'clip' should be an instance of ClipGradBase's derived class") + "'clip' should be an instance of ClipGradBase's derived class" + ) if program is None: program = framework.default_main_program() for op in program.block(0).ops: if 'op_namescope' in op.all_attrs() and "optimizer" in op.attr( - "op_namescope"): + "op_namescope" + ): warnings.warn( "'minimize' has been invoked before, this will make 'set_gradient_clip' " "be ineffective! Please invoke 'set_gradient_clip' before 'minimize'." @@ -844,14 +882,16 @@ def append_gradient_clip_ops(param_grads): for p, g in param_grads: if g is None: continue - with p.block.program._optimized_guard( - [p, g]), framework.name_scope('gradient_clip'): + with p.block.program._optimized_guard([p, g]), framework.name_scope( + 'gradient_clip' + ): clip_attr = getattr(p, 'gradient_clip_attr', None) if clip_attr is None: return param_grads if not isinstance(clip_attr, ClipGradBase): raise TypeError( - "clip attribute should be an instance of GradientClipBase") + "clip attribute should be an instance of GradientClipBase" + ) clip_attr._process_context(context=context, param=p, grad=g) @@ -860,8 +900,9 @@ def append_gradient_clip_ops(param_grads): for p, g in param_grads: if g is None: continue - with p.block.program._optimized_guard( - [p, g]), framework.name_scope('gradient_clip'): + with p.block.program._optimized_guard([p, g]), framework.name_scope( + 'gradient_clip' + ): param, new_grad = clip_attr._create_operators(param=p, grad=g) param_new_grad_name_dict[param.name] = new_grad.name res.append([param, new_grad]) @@ -885,12 +926,16 @@ def _correct_clip_op_role_var(params_grads, param_new_grad_name_dict): continue block_id_list.append(block_id) for op in param.block.program.global_block().ops: - if op.has_attr("op_namescope") and "gradient_clip" in op.attr( - "op_namescope") and op.attr('op_role_var'): + if ( + op.has_attr("op_namescope") + and "gradient_clip" in op.attr("op_namescope") + and op.attr('op_role_var') + ): param_name = op.attr('op_role_var')[0] if param_name in param_new_grad_name_dict: correct_p_g = [ - param_name, param_new_grad_name_dict[param_name] + param_name, + param_new_grad_name_dict[param_name], ] op._set_attr('op_role_var', correct_p_g) diff --git a/python/paddle/fluid/communicator.py b/python/paddle/fluid/communicator.py index 4704628f2ae..be3fb5ea22d 100755 --- a/python/paddle/fluid/communicator.py +++ b/python/paddle/fluid/communicator.py @@ -27,6 +27,7 @@ # limitations under the License. from .executor import global_scope + """ Communicator is used for async distribute training in distribute_transpiler mode. It's a wrapper of a cpp class Communicator and should be used inside fleet API. @@ -38,7 +39,6 @@ __all__ = ['Communicator', 'FLCommunicator', 'LargeScaleKV'] class Communicator(object): - def __init__(self, mode, kwargs=None, envs=None): """ Communicator is used for async distribute training in distribute_transpiler mode. @@ -69,7 +69,8 @@ class Communicator(object): else: if mode == DistributedMode.SYNC: envs["pserver_endpoints"] = ','.join( - kwargs["pserver_endpoints"]) + kwargs["pserver_endpoints"] + ) envs["trainers"] = str(kwargs["trainers"]) envs["trainer_id"] = str(kwargs["trainer_id"]) @@ -93,26 +94,32 @@ class Communicator(object): self.send_ctx_ = None self.recv_ctx_ = None - def init_with_ctx(self, - send_ctx, - recv_ctx, - proto_txt, - unit64_hosts, - scope=None): + def init_with_ctx( + self, send_ctx, recv_ctx, proto_txt, unit64_hosts, scope=None + ): if scope == None: scope = global_scope() - self.communicator_ = core.DistCommunicator(self.mode, proto_txt, - unit64_hosts, send_ctx, - recv_ctx, scope, self.envs) + self.communicator_ = core.DistCommunicator( + self.mode, + proto_txt, + unit64_hosts, + send_ctx, + recv_ctx, + scope, + self.envs, + ) self.send_ctx_ = send_ctx self.recv_ctx_ = recv_ctx - def create_client_to_client_connection(self, - pserver_timeout_ms=500000, - pserver_connect_timeout_ms=10000, - max_retry=3): + def create_client_to_client_connection( + self, + pserver_timeout_ms=500000, + pserver_connect_timeout_ms=10000, + max_retry=3, + ): self.communicator_.create_client_to_client_connection( - pserver_timeout_ms, pserver_connect_timeout_ms, max_retry) + pserver_timeout_ms, pserver_connect_timeout_ms, max_retry + ) def get_client_info(self): return self.communicator_.get_client_info() @@ -209,7 +216,6 @@ class Communicator(object): class FLCommunicator(Communicator): ## only for coordinator - def __init__(self, ps_hosts, kwargs=None): mode = None super(FLCommunicator, self).__init__(mode, kwargs) @@ -221,8 +227,9 @@ class FLCommunicator(Communicator): ## only for coordinator def start_coordinator(self, self_endpoint, trainer_endpoints): if self.communicator_ != None: - self.communicator_.start_coordinator(self_endpoint, - trainer_endpoints) + self.communicator_.start_coordinator( + self_endpoint, trainer_endpoints + ) return def save_fl_strategy(self, mp): @@ -240,7 +247,6 @@ class FLCommunicator(Communicator): ## only for coordinator class LargeScaleKV(object): - def __init__(self): self.scale_kv = core.LargeScaleKV() @@ -255,10 +261,10 @@ class LargeScaleKV(object): class HeterClient(object): - def __init__(self, endpoint, previous_endpoint, trainer_id): - self.heter_client_ = core.HeterClient(endpoint, previous_endpoint, - trainer_id) + self.heter_client_ = core.HeterClient( + endpoint, previous_endpoint, trainer_id + ) def stop(self): self.heter_client_.stop() diff --git a/python/paddle/fluid/compiler.py b/python/paddle/fluid/compiler.py index 7ac8e4c30ab..1adef41f86a 100644 --- a/python/paddle/fluid/compiler.py +++ b/python/paddle/fluid/compiler.py @@ -22,8 +22,11 @@ from .framework import cuda_places, cpu_places, xpu_places from . import core __all__ = [ - 'CompiledProgram', 'ExecutionStrategy', 'BuildStrategy', - 'IpuCompiledProgram', 'IpuStrategy' + 'CompiledProgram', + 'ExecutionStrategy', + 'BuildStrategy', + 'IpuCompiledProgram', + 'IpuStrategy', ] ExecutionStrategy = core.ParallelExecutor.ExecutionStrategy @@ -40,8 +43,7 @@ def _place_obj(place): def _is_pserver_mode(main_program): - main = main_program if main_program \ - else framework.default_main_program() + main = main_program if main_program else framework.default_main_program() for op in main.global_block().ops: if op.type in ["send", "recv"]: return True @@ -50,8 +52,11 @@ def _is_pserver_mode(main_program): def _has_backward_op(graph): for node in graph.nodes(): - if node.is_op() and node.op() is not None and \ - node.op().type().endswith("_grad"): + if ( + node.is_op() + and node.op() is not None + and node.op().type().endswith("_grad") + ): return True return False @@ -60,7 +65,8 @@ def _prune_feed_ops(program): # prune the feed ops in the program. pop_idx = [] for i, op in enumerate(program.global_block().ops): - if op.type == "feed": pop_idx.append(i) + if op.type == "feed": + pop_idx.append(i) for index in pop_idx[::-1]: program.global_block()._remove_op(index) @@ -69,8 +75,9 @@ def _has_optimize_op(block): for op in block.ops: op_maker = core.op_proto_and_checker_maker optimize = core.op_proto_and_checker_maker.OpRole.Optimize - if op_maker.kOpRoleVarAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(optimize): + if op_maker.kOpRoleVarAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(optimize): return True return False @@ -93,7 +100,8 @@ def _should_broadcast_or_not_exists(program, var_name): if var is None: return True is_distributed = getattr(var, '_is_distributed', False) or getattr( - var, 'is_distributed', False) + var, 'is_distributed', False + ) return not is_distributed @@ -159,7 +167,8 @@ class CompiledProgram(object): else: raise TypeError( "The type of program_to_graph parameter is wrong, expected Graph or Program, but received %s" - % type(program_or_graph)) + % type(program_or_graph) + ) self._scope = None self._place = None @@ -173,12 +182,14 @@ class CompiledProgram(object): self._build_strategy = build_strategy self._exec_strategy = None - def with_data_parallel(self, - loss_name=None, - build_strategy=None, - exec_strategy=None, - share_vars_from=None, - places=None): + def with_data_parallel( + self, + loss_name=None, + build_strategy=None, + exec_strategy=None, + share_vars_from=None, + places=None, + ): """ This interface is used to transform the input Program or Graph to a multi-graph to run the model in data parallel mode. Users can use the build_strategy and @@ -284,14 +295,19 @@ class CompiledProgram(object): feed={"X": test_data}, fetch_list=[loss.name]) """ - assert not self._is_data_parallel, "Already compiled with parallel, cannot be recompiled." - assert not self._is_inference, "Cannot compile with both data parallel and inference." + assert ( + not self._is_data_parallel + ), "Already compiled with parallel, cannot be recompiled." + assert ( + not self._is_inference + ), "Cannot compile with both data parallel and inference." self._is_data_parallel = True # FIXME(zcd): Currently, the build_strategy can be set during creating # CompiledProgram or calling with_data_parallel, and it may be confusing, # but in the long run, we should set up build_strategy only when creating # CompiledProgram, and exec_strategy should be deprecated. - if build_strategy is not None: self._build_strategy = build_strategy + if build_strategy is not None: + self._build_strategy = build_strategy self._exec_strategy = exec_strategy self._loss_name = loss_name self._share_vars_from = share_vars_from @@ -301,7 +317,9 @@ class CompiledProgram(object): self._places = _get_paddle_place(places) if _has_backward_op(self._graph): - assert self._loss_name is not None, "The loss name of CompiledProgram is None. The loss name should be set if CompiledProgram contains backward part." + assert ( + self._loss_name is not None + ), "The loss name of CompiledProgram is None. The loss name should be set if CompiledProgram contains backward part." if self._places is not None: if not isinstance(self._places, (list, tuple)): @@ -310,20 +328,26 @@ class CompiledProgram(object): return self def _with_inference_optimize(self, config): - """ Add inference optimize + """Add inference optimize Args: config: instance of `NativeConfig` or `AnalysisConfig` to create predictor Returns: self """ - assert not self._is_data_parallel, "Cannot compile with both data parallel and inference" - assert not self._is_inference, "Already compiled with inference, cannot be recompiled." - - assert any([ - isinstance(config, InferNativeConfig), - isinstance(config, InferAnalysisConfig) - ]) + assert ( + not self._is_data_parallel + ), "Cannot compile with both data parallel and inference" + assert ( + not self._is_inference + ), "Already compiled with inference, cannot be recompiled." + + assert any( + [ + isinstance(config, InferNativeConfig), + isinstance(config, InferAnalysisConfig), + ] + ) self._is_inference = True self._infer_config = config return self @@ -340,18 +364,23 @@ class CompiledProgram(object): if not self._share_vars_from._is_data_parallel: raise ValueError( "The shared Program is not data parallel, cannot " - "share variables from it.") + "share variables from it." + ) if self._share_vars_from._executor is None: raise ValueError( "The shared Program is not compiled and executed, so there is no " - "variables to share.") + "variables to share." + ) self._local_scopes = self._share_vars_from._executor.local_scopes() else: assert scope is not None, "" self._local_scopes = [] - assert isinstance(places, tuple) or isinstance(places, list), \ - "Currently , The places type can only be list or tuple, but the input type is {}.".format(type(places)) + assert isinstance(places, tuple) or isinstance( + places, list + ), "Currently , The places type can only be list or tuple, but the input type is {}.".format( + type(places) + ) if self._build_strategy is None: self._build_strategy = BuildStrategy() @@ -372,41 +401,61 @@ class CompiledProgram(object): else: self._exec_strategy.num_threads = len(places) * 2 - if "FLAGS_use_cinn" in core.globals() and core.globals( - )["FLAGS_use_cinn"] and self._exec_strategy.num_threads != 1: - warnings.warn("At present, when CINN is turned on, each process can " \ - "only contain one thread, so reset the number of threads to 1 here.") + if ( + "FLAGS_use_cinn" in core.globals() + and core.globals()["FLAGS_use_cinn"] + and self._exec_strategy.num_threads != 1 + ): + warnings.warn( + "At present, when CINN is turned on, each process can " + "only contain one thread, so reset the number of threads to 1 here." + ) self._exec_strategy.num_threads = 1 if self._build_strategy.num_trainers > 1: - assert self._is_data_parallel, \ - "If you use multi-trainer to train the model, you should use "\ + assert self._is_data_parallel, ( + "If you use multi-trainer to train the model, you should use " "the data parallel model, i.e. calling with_data_parallel function." + ) # TODO(wuyi): trainer endpoings should be passed in through # build_strategy, not program.xxx. # TODO(gongwb): let user to set them once. - if self._program and self._build_strategy.num_trainers > 1 and \ - self._program._trainers_endpoints: + if ( + self._program + and self._build_strategy.num_trainers > 1 + and self._program._trainers_endpoints + ): tps = self._program._trainers_endpoints assert self._build_strategy.num_trainers == len( - tps), "The trainer numbers is not equal to endpoint numbers." + tps + ), "The trainer numbers is not equal to endpoint numbers." self._build_strategy.trainers_endpoints = tps if self._program: self._build_strategy.nccl_comm_num = self._program._nccl_comm_num - self._build_strategy.use_hierarchical_allreduce = self._program._use_hierarchical_allreduce - self._build_strategy.hierarchical_allreduce_inter_nranks = self._program._hierarchical_allreduce_inter_nranks + self._build_strategy.use_hierarchical_allreduce = ( + self._program._use_hierarchical_allreduce + ) + self._build_strategy.hierarchical_allreduce_inter_nranks = ( + self._program._hierarchical_allreduce_inter_nranks + ) if self._build_strategy.sync_batch_norm: self._build_strategy.enable_sequential_execution = True if self._program is not None and self._program._enable_dgc: - assert self._exec_strategy._use_device == DeviceType.CUDA, "DGC only used under CUDA environment." - assert self._build_strategy.num_trainers * len( - places) > 1, "DGC is not avaliable for single card training." - assert self._build_strategy.reduce_strategy == BuildStrategy.ReduceStrategy.AllReduce, "DGC \ + assert ( + self._exec_strategy._use_device == DeviceType.CUDA + ), "DGC only used under CUDA environment." + assert ( + self._build_strategy.num_trainers * len(places) > 1 + ), "DGC is not avaliable for single card training." + assert ( + self._build_strategy.reduce_strategy + == BuildStrategy.ReduceStrategy.AllReduce + ), "DGC \ only can be used for AllReduce BuildStrategy." # DGC doesn't support fuse for now, close fuse. @@ -414,11 +463,17 @@ class CompiledProgram(object): self._persistable_vars = [] for node in self._graph.nodes(): - if node.is_var() and node.var() is not None and node.var().persistable() and \ - node.var().type() != core.VarDesc.VarType.RAW: + if ( + node.is_var() + and node.var() is not None + and node.var().persistable() + and node.var().type() != core.VarDesc.VarType.RAW + ): name = node.name() - if self._program is not None and _should_broadcast_or_not_exists( - self._program, name): + if ( + self._program is not None + and _should_broadcast_or_not_exists(self._program, name) + ): self._persistable_vars.append(node.name()) places = list(map(_place_obj, places)) @@ -429,11 +484,16 @@ class CompiledProgram(object): self._persistable_vars = list(set(self._persistable_vars)) self._persistable_vars.sort() - return core.ParallelExecutor(places, self._persistable_vars, - self._loss_name if self._loss_name else '', - self._scope, self._local_scopes, - self._exec_strategy, self._build_strategy, - self._graph) + return core.ParallelExecutor( + places, + self._persistable_vars, + self._loss_name if self._loss_name else '', + self._scope, + self._local_scopes, + self._exec_strategy, + self._build_strategy, + self._graph, + ) def _compile_inference(self): return core.create_paddle_predictor(self._infer_config) @@ -471,28 +531,31 @@ class CompiledProgram(object): # Todo(liym27):If optimizer is used in control flow, # training on multi-places is not supported now, will # be supported later. - if len(self._places) > 1 and \ - _has_optimizer_in_control_flow(self._program): + if len(self._places) > 1 and _has_optimizer_in_control_flow( + self._program + ): raise NotImplementedError( "If optimizer is used in control flow, " - "training on multi-places is not supported now.") + "training on multi-places is not supported now." + ) if isinstance(self._place, core.CUDAPlace): use_device = DeviceType.CUDA elif isinstance(self._place, core.XPUPlace): use_device = DeviceType.XPU else: use_device = DeviceType.CPU - self._executor = self._compile_data_parallel(use_device=use_device, - scope=self._scope, - places=self._places) + self._executor = self._compile_data_parallel( + use_device=use_device, scope=self._scope, places=self._places + ) return self def _get_places(self, place, place_list): - has_set_place = (place_list is not None) + has_set_place = place_list is not None if has_set_place: for p in place_list: - assert p._type() == place._type(), \ - "Place type not match. You may set wrong type of places." + assert ( + p._type() == place._type() + ), "Place type not match. You may set wrong type of places." else: if isinstance(place, core.CUDAPlace): place_list = cuda_places() @@ -515,9 +578,9 @@ class IpuDynamicPatcher(object): pass @staticmethod - def convert_concrete_program(ipu_strategy, - concrete_program, - class_instance=None): + def convert_concrete_program( + ipu_strategy, concrete_program, class_instance=None + ): """ Convert the ConcreteProgram to IPUConcreteProgram. """ @@ -553,7 +616,8 @@ class IpuDynamicPatcher(object): # copy the bias and filters for param_or_buffer in concrete_program.parameters: param_or_buffer_tensor = scope.var( - param_or_buffer.name).get_tensor() + param_or_buffer.name + ).get_tensor() src_tensor = param_or_buffer.value().get_tensor() param_or_buffer_tensor._share_data_with(src_tensor) @@ -576,13 +640,16 @@ class IpuDynamicPatcher(object): dtype=var_tmp.dtype, type=var_tmp.type, shape=var_tmp.shape, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) device = optimizer._get_device_for_param(param_name) with device_guard(device): optimizer.helper.set_variable_initializer( - var, initializer=Constant(value=0.0)) + var, initializer=Constant(value=0.0) + ) param_or_lr_tensor = scope.find_var( - var_tmp.name).get_tensor() + var_tmp.name + ).get_tensor() optim_tensor = var.value().get_tensor() param_or_lr_tensor._share_data_with(optim_tensor) optimizer._accumulators[k][param_name] = var @@ -595,16 +662,19 @@ class IpuDynamicPatcher(object): to_fp16_var_names = paddle.static.amp.cast_model_to_fp16( concrete_program.main_program, amp_list, - use_fp16_guard=False) + use_fp16_guard=False, + ) paddle.static.amp.cast_parameters_to_fp16( paddle.CPUPlace(), concrete_program.main_program, - to_fp16_var_names=to_fp16_var_names) - - program = IpuCompiledProgram(concrete_program.main_program, - ipu_strategy=ipu_strategy, - scope=scope).compile( - feed_list, fetch_list) + to_fp16_var_names=to_fp16_var_names, + ) + + program = IpuCompiledProgram( + concrete_program.main_program, + ipu_strategy=ipu_strategy, + scope=scope, + ).compile(feed_list, fetch_list) return program main_program = func_compile() @@ -613,7 +683,7 @@ class IpuDynamicPatcher(object): @staticmethod def patch_program_cache(ipu_strategy): - """ Monkey patch ProgramCache discriptor to support dynamic2static in IPU. + """Monkey patch ProgramCache discriptor to support dynamic2static in IPU. Args: ipu_strategy: The ipu_strategy used in dynamic graph. @@ -621,53 +691,70 @@ class IpuDynamicPatcher(object): Returns: None """ - from ..fluid.dygraph.dygraph_to_static.program_translator import ProgramCache - from ..fluid.dygraph.dygraph_to_static.program_translator import CacheKey + from ..fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramCache, + ) + from ..fluid.dygraph.dygraph_to_static.program_translator import ( + CacheKey, + ) from ..fluid.dygraph.dygraph_to_static import logging_utils - from ..fluid.dygraph.dygraph_to_static.program_translator import MAX_TRACED_PROGRAM_COUNT - from ..fluid.dygraph.dygraph_to_static.partial_program import partial_program_from + from ..fluid.dygraph.dygraph_to_static.program_translator import ( + MAX_TRACED_PROGRAM_COUNT, + ) + from ..fluid.dygraph.dygraph_to_static.partial_program import ( + partial_program_from, + ) old_getter = ProgramCache.__getitem__ def patch_getter(self, item): if not isinstance(item, CacheKey): raise ValueError( - 'type(item) should be CacheKey, but received %s' % - type(item).__name__) + 'type(item) should be CacheKey, but received %s' + % type(item).__name__ + ) item_id = hash(item) self._recent_key = item_id if item_id not in self._caches or ipu_strategy.need_compile: if item_id in self._caches: logging_utils.warn( - "ipu_strategy chances detected. Please sync weights.") + "ipu_strategy chances detected. Please sync weights." + ) if self._caches and not ipu_strategy.need_compile: logging_utils.warn( "dynamic2static on IPU doesn't support mutiple caches. Please make sure" - "dynamic inputs is not used.") + "dynamic inputs is not used." + ) concrete_program, _ = self._build_once(item) concrete_program = IpuDynamicPatcher.convert_concrete_program( - ipu_strategy, concrete_program, item.class_instance) + ipu_strategy, concrete_program, item.class_instance + ) - self._caches[item_id] = (concrete_program, - partial_program_from(concrete_program)) + self._caches[item_id] = ( + concrete_program, + partial_program_from(concrete_program), + ) # Note: raise warnings if number of traced program is more than `max_tracing_count` current_tracing_count = len(self._caches) if current_tracing_count > MAX_TRACED_PROGRAM_COUNT: logging_utils.warn( "Current traced program number: {} > `max_tracing_count`:{}. Too much cached programs will bring expensive overhead. " - "The reason may be: (1) passing tensors with different shapes, (2) passing python objects instead of tensors." - .format(current_tracing_count, - MAX_TRACED_PROGRAM_COUNT)) + "The reason may be: (1) passing tensors with different shapes, (2) passing python objects instead of tensors.".format( + current_tracing_count, MAX_TRACED_PROGRAM_COUNT + ) + ) return self._caches[item_id] setattr(ProgramCache, '__getitem__', patch_getter) IpuDynamicPatcher.patcher_cache.append( - [ProgramCache, '__getitem__', old_getter]) + [ProgramCache, '__getitem__', old_getter] + ) @staticmethod def patch_lr_scheduler(ipu_strategy): from paddle.optimizer.lr import LRScheduler + # For IPU dynamic graph usage, lr_var is not synced in executor as static mode do. # Manually set lr to ipu_strategy to update the lr. old_step = LRScheduler.step @@ -718,10 +805,8 @@ class IpuStrategy(object): 'on_chip': 0, 'use_replicated_tensor_sharding': 1, }, # set optimizer location - 'accumulation_and_replication_reduction_type': - 1, # popart::ReductionType::Mean - 'mean_accumulation_and_replication_reduction_strategy': - 1, # popart::MeanReductionStrategy::Post + 'accumulation_and_replication_reduction_type': 1, # popart::ReductionType::Mean + 'mean_accumulation_and_replication_reduction_strategy': 1, # popart::MeanReductionStrategy::Post } self._ipu_strategy.set_options(default_options) self.has_custom_ops = False @@ -732,6 +817,7 @@ class IpuStrategy(object): "Can not use IpuStrategy in non IPU compiled environment, please re-compile with WITH_IPU=ON." ) from paddle import in_dynamic_mode + if in_dynamic_mode(): self.register_patch() @@ -797,6 +883,7 @@ class IpuStrategy(object): ipu_strategy.set_optimizer(optimizer) """ from paddle import in_dynamic_mode + if in_dynamic_mode(): self._optimizer = optimizer optimizer_attrs = self.parse_optimizer(optimizer) @@ -831,6 +918,7 @@ class IpuStrategy(object): def get_lr(): from paddle.optimizer.lr import LRScheduler + if isinstance(optimizer._learning_rate, float): return {"lr": optimizer._learning_rate} elif isinstance(optimizer._learning_rate, LRScheduler): @@ -842,11 +930,13 @@ class IpuStrategy(object): optimizer_attrs.update(fn()) return optimizer_attrs - def set_graph_config(self, - num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False): + def set_graph_config( + self, + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ): """ Set graph configuration to the IpuStrategy instance. @@ -889,11 +979,13 @@ class IpuStrategy(object): } self.set_options(options) - def set_pipelining_config(self, - enable_pipelining=False, - batches_per_step=1, - enable_gradient_accumulation=False, - accumulation_factor=1): + def set_pipelining_config( + self, + enable_pipelining=False, + batches_per_step=1, + enable_gradient_accumulation=False, + accumulation_factor=1, + ): """ Set pipelining configuration to the IpuStrategy instance. Used to optimize the throughput performance. @@ -967,11 +1059,9 @@ class IpuStrategy(object): } self.set_options(options) - def add_custom_op(self, - paddle_op, - popart_op=None, - domain='custom.ops', - version=1): + def add_custom_op( + self, paddle_op, popart_op=None, domain='custom.ops', version=1 + ): """ Add a mapping to use popart custom ops running on the IPU. @@ -1201,8 +1291,9 @@ class IpuCompiledProgram(object): if not isinstance(program, framework.Program): raise TypeError( - "The type of program is wrong, expected Program, but got %s" % - type(program)) + "The type of program is wrong, expected Program, but got %s" + % type(program) + ) self._program = program self._compiled = False @@ -1212,6 +1303,7 @@ class IpuCompiledProgram(object): else: # import here to avoiding confused import paddle + self._scope = paddle.static.global_scope() if ipu_strategy is not None: diff --git a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py index 10bc30d13d1..429feda47bb 100644 --- a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py +++ b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py @@ -64,13 +64,15 @@ class InitState(object): See `StateCell`. """ - def __init__(self, - init=None, - shape=None, - value=0.0, - init_boot=None, - need_reorder=False, - dtype='float32'): + def __init__( + self, + init=None, + shape=None, + value=0.0, + init_boot=None, + need_reorder=False, + dtype='float32', + ): if init is not None: self._init = init elif init_boot is None: @@ -78,10 +80,9 @@ class InitState(object): 'init_boot must be provided to infer the shape of InitState .\n' ) else: - self._init = layers.fill_constant_batch_size_like(input=init_boot, - value=value, - shape=shape, - dtype=dtype) + self._init = layers.fill_constant_batch_size_like( + input=init_boot, value=value, shape=shape, dtype=dtype + ) self._shape = shape self._value = value @@ -98,12 +99,12 @@ class InitState(object): class _MemoryState(object): - def __init__(self, state_name, rnn_obj, init_state): self._state_name = state_name # each is a rnn.memory self._rnn_obj = rnn_obj self._state_mem = self._rnn_obj.memory( - init=init_state.value, need_reorder=init_state.need_reorder) + init=init_state.value, need_reorder=init_state.need_reorder + ) def get_state(self): return self._state_mem @@ -113,7 +114,6 @@ class _MemoryState(object): class _ArrayState(object): - def __init__(self, state_name, block, init_state): self._state_name = state_name self._block = block @@ -121,33 +121,36 @@ class _ArrayState(object): self._state_array = self._block.create_var( name=unique_name.generate('array_state_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=init_state.value.dtype) + dtype=init_state.value.dtype, + ) self._counter = self._block.create_var( name=unique_name.generate('array_state_counter'), type=core.VarDesc.VarType.LOD_TENSOR, - dtype='int64') + dtype='int64', + ) # initialize counter - self._block.append_op(type='fill_constant', - inputs={}, - outputs={'Out': [self._counter]}, - attrs={ - 'shape': [1], - 'dtype': self._counter.dtype, - 'value': float(0.0), - 'force_cpu': True - }) + self._block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [self._counter]}, + attrs={ + 'shape': [1], + 'dtype': self._counter.dtype, + 'value': float(0.0), + 'force_cpu': True, + }, + ) self._counter.stop_gradient = True # write initial state - block.append_op(type='write_to_array', - inputs={ - 'X': init_state.value, - 'I': self._counter - }, - outputs={'Out': self._state_array}) + block.append_op( + type='write_to_array', + inputs={'X': init_state.value, 'I': self._counter}, + outputs={'Out': self._state_array}, + ) def get_state(self): state = layers.array_read(array=self._state_array, i=self._counter) @@ -220,8 +223,9 @@ class StateCell(object): def _leave_decoder(self, decoder_obj): if not self._in_decoder: - raise ValueError('StateCell not in decoder, ' - 'invalid leaving operation.') + raise ValueError( + 'StateCell not in decoder, ' 'invalid leaving operation.' + ) if self._cur_decoder_obj != decoder_obj: raise ValueError('Inconsistent decoder object in StateCell.') @@ -242,29 +246,35 @@ class StateCell(object): state = self._cur_states[state_name] if not isinstance(state, InitState): - raise ValueError('Current type of state is %s, should be ' - 'an InitState object.' % type(state)) + raise ValueError( + 'Current type of state is %s, should be ' + 'an InitState object.' % type(state) + ) self._states_holder[state_name] = {} if self._cur_decoder_obj.type == _DecoderType.TRAINING: - self._states_holder[state_name][id(self._cur_decoder_obj)] \ - = _MemoryState(state_name, - self._cur_decoder_obj.dynamic_rnn, - state) + self._states_holder[state_name][ + id(self._cur_decoder_obj) + ] = _MemoryState( + state_name, self._cur_decoder_obj.dynamic_rnn, state + ) elif self._cur_decoder_obj.type == _DecoderType.BEAM_SEARCH: - self._states_holder[state_name][id(self._cur_decoder_obj)] \ - = _ArrayState(state_name, - self._cur_decoder_obj._parent_block(), - state) + self._states_holder[state_name][ + id(self._cur_decoder_obj) + ] = _ArrayState( + state_name, self._cur_decoder_obj._parent_block(), state + ) else: - raise ValueError('Unknown decoder type, only support ' - '[TRAINING, BEAM_SEARCH]') + raise ValueError( + 'Unknown decoder type, only support ' + '[TRAINING, BEAM_SEARCH]' + ) # Read back, since current state should be LoDTensor - self._cur_states[state_name] = \ - self._states_holder[state_name][ - id(self._cur_decoder_obj)].get_state() + self._cur_states[state_name] = self._states_holder[state_name][ + id(self._cur_decoder_obj) + ].get_state() self._switched_decoder = True @@ -284,7 +294,8 @@ class StateCell(object): if state_name not in self._cur_states: raise ValueError( 'Unknown state %s. Please make sure _switch_decoder() ' - 'invoked.' % state_name) + 'invoked.' % state_name + ) return self._cur_states[state_name] @@ -328,8 +339,10 @@ class StateCell(object): def _decorator(state_cell): if state_cell == self: - raise TypeError('Updater should only accept a StateCell object ' - 'as argument.') + raise TypeError( + 'Updater should only accept a StateCell object ' + 'as argument.' + ) updater(state_cell) return _decorator @@ -353,9 +366,11 @@ class StateCell(object): for input_name, input_value in inputs.items(): if input_name not in self._inputs: - raise ValueError('Unknown input %s. ' - 'Please make sure %s in input ' - 'place holder.' % (input_name, input_name)) + raise ValueError( + 'Unknown input %s. ' + 'Please make sure %s in input ' + 'place holder.' % (input_name, input_name) + ) self._inputs[input_name] = input_value self._state_updater(self) @@ -368,10 +383,13 @@ class StateCell(object): for state_name, decoder_state in self._states_holder.items(): if id(self._cur_decoder_obj) not in decoder_state: - raise ValueError('Unknown decoder object, please make sure ' - 'switch_decoder been invoked.') + raise ValueError( + 'Unknown decoder object, please make sure ' + 'switch_decoder been invoked.' + ) decoder_state[id(self._cur_decoder_obj)].update_state( - self._cur_states[state_name]) + self._cur_states[state_name] + ) def out_state(self): """ @@ -409,6 +427,7 @@ class TrainingDecoder(object): decoder.state_cell.update_states() decoder.output(current_score) """ + BEFORE_DECODER = 0 IN_DECODER = 1 AFTER_DECODER = 2 @@ -495,8 +514,10 @@ class TrainingDecoder(object): Variable: The specified output of the RNN cell. """ if self._status != TrainingDecoder.AFTER_DECODER: - raise ValueError('Output of training decoder can only be visited ' - 'outside the block.') + raise ValueError( + 'Output of training decoder can only be visited ' + 'outside the block.' + ) return self._dynamic_rnn(*args, **kwargs) def output(self, *outputs): @@ -520,8 +541,10 @@ class TrainingDecoder(object): def _assert_in_decoder_block(self, method): if self._status != TrainingDecoder.IN_DECODER: - raise ValueError('%s should be invoked inside block of ' - 'TrainingDecoder object.' % method) + raise ValueError( + '%s should be invoked inside block of ' + 'TrainingDecoder object.' % method + ) class BeamSearchDecoder(object): @@ -569,42 +592,44 @@ class BeamSearchDecoder(object): decoder.decode() translation_ids, translation_scores = decoder() """ + BEFORE_BEAM_SEARCH_DECODER = 0 IN_BEAM_SEARCH_DECODER = 1 AFTER_BEAM_SEARCH_DECODER = 2 - def __init__(self, - state_cell, - init_ids, - init_scores, - target_dict_dim, - word_dim, - input_var_dict={}, - topk_size=50, - sparse_emb=True, - max_len=100, - beam_size=1, - end_id=1, - name=None): + def __init__( + self, + state_cell, + init_ids, + init_scores, + target_dict_dim, + word_dim, + input_var_dict={}, + topk_size=50, + sparse_emb=True, + max_len=100, + beam_size=1, + end_id=1, + name=None, + ): self._helper = LayerHelper('beam_search_decoder', name=name) self._counter = layers.zeros(shape=[1], dtype='int64') self._counter.stop_gradient = True self._type = _DecoderType.BEAM_SEARCH - self._max_len = layers.fill_constant(shape=[1], - dtype='int64', - value=max_len) - self._cond = layers.less_than(x=self._counter, - y=layers.fill_constant(shape=[1], - dtype='int64', - value=max_len)) + self._max_len = layers.fill_constant( + shape=[1], dtype='int64', value=max_len + ) + self._cond = layers.less_than( + x=self._counter, + y=layers.fill_constant(shape=[1], dtype='int64', value=max_len), + ) self._while_op = layers.While(self._cond) self._state_cell = state_cell self._state_cell._enter_decoder(self) self._status = BeamSearchDecoder.BEFORE_BEAM_SEARCH_DECODER - self._zero_idx = layers.fill_constant(shape=[1], - value=0, - dtype='int64', - force_cpu=True) + self._zero_idx = layers.fill_constant( + shape=[1], value=0, dtype='int64', force_cpu=True + ) self._array_dict = {} self._array_link = [] self._ids_array = None @@ -637,13 +662,13 @@ class BeamSearchDecoder(object): layers.increment(x=self._counter, value=1.0, in_place=True) for value, array in self._array_link: - layers.array_write(x=value, - i=self._counter, - array=array) + layers.array_write( + x=value, i=self._counter, array=array + ) - layers.less_than(x=self._counter, - y=self._max_len, - cond=self._cond) + layers.less_than( + x=self._counter, y=self._max_len, cond=self._cond + ) self._status = BeamSearchDecoder.AFTER_BEAM_SEARCH_DECODER self._state_cell._leave_decoder(self) @@ -656,11 +681,9 @@ class BeamSearchDecoder(object): """ Stop the generation process in advance. Could be used as "break". """ - layers.fill_constant(shape=[1], - value=0, - dtype='bool', - force_cpu=True, - out=self._cond) + layers.fill_constant( + shape=[1], value=0, dtype='bool', force_cpu=True, out=self._cond + ) def decode(self): """ @@ -675,32 +698,39 @@ class BeamSearchDecoder(object): """ with self.block(): prev_ids = self.read_array(init=self._init_ids, is_ids=True) - prev_scores = self.read_array(init=self._init_scores, - is_scores=True) + prev_scores = self.read_array( + init=self._init_scores, is_scores=True + ) prev_ids_embedding = layers.embedding( input=prev_ids, size=[self._target_dict_dim, self._word_dim], dtype='float32', - is_sparse=self._sparse_emb) + is_sparse=self._sparse_emb, + ) feed_dict = {} update_dict = {} for init_var_name, init_var in self._input_var_dict.items(): if init_var_name not in self.state_cell._inputs: - raise ValueError('Variable ' + init_var_name + - ' not found in StateCell!\n') + raise ValueError( + 'Variable ' + + init_var_name + + ' not found in StateCell!\n' + ) read_var = self.read_array(init=init_var) update_dict[init_var_name] = read_var feed_var_expanded = layers.sequence_expand( - read_var, prev_scores) + read_var, prev_scores + ) feed_dict[init_var_name] = feed_var_expanded for state_str in self._state_cell._state_names: prev_state = self.state_cell.get_state(state_str) prev_state_expanded = layers.sequence_expand( - prev_state, prev_scores) + prev_state, prev_scores + ) self.state_cell.set_state(state_str, prev_state_expanded) for i, input_name in enumerate(self._state_cell._inputs): @@ -709,23 +739,29 @@ class BeamSearchDecoder(object): self.state_cell.compute_state(inputs=feed_dict) current_state = self.state_cell.out_state() - current_state_with_lod = layers.lod_reset(x=current_state, - y=prev_scores) - scores = layers.fc(input=current_state_with_lod, - size=self._target_dict_dim, - act='softmax') + current_state_with_lod = layers.lod_reset( + x=current_state, y=prev_scores + ) + scores = layers.fc( + input=current_state_with_lod, + size=self._target_dict_dim, + act='softmax', + ) topk_scores, topk_indices = layers.topk(scores, k=self._topk_size) - accu_scores = layers.elementwise_add(x=layers.log(x=topk_scores), - y=layers.reshape(prev_scores, - shape=[-1]), - axis=0) - selected_ids, selected_scores = layers.beam_search(prev_ids, - prev_scores, - topk_indices, - accu_scores, - self._beam_size, - end_id=1, - level=0) + accu_scores = layers.elementwise_add( + x=layers.log(x=topk_scores), + y=layers.reshape(prev_scores, shape=[-1]), + axis=0, + ) + selected_ids, selected_scores = layers.beam_search( + prev_ids, + prev_scores, + topk_indices, + accu_scores, + self._beam_size, + end_id=1, + level=0, + ) with layers.Switch() as switch: with switch.case(layers.is_empty(selected_ids)): @@ -760,8 +796,10 @@ class BeamSearchDecoder(object): self._assert_in_decoder_block('read_array') if is_ids and is_scores: - raise ValueError('Shouldn\'t mark current array be ids array and' - 'scores array at the same time.') + raise ValueError( + 'Shouldn\'t mark current array be ids array and' + 'scores array at the same time.' + ) if not isinstance(init, Variable): raise TypeError('The input argument `init` must be a Variable.') @@ -770,13 +808,13 @@ class BeamSearchDecoder(object): array = parent_block.create_var( name=unique_name.generate('beam_search_decoder_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=init.dtype) - parent_block.append_op(type='write_to_array', - inputs={ - 'X': init, - 'I': self._zero_idx - }, - outputs={'Out': array}) + dtype=init.dtype, + ) + parent_block.append_op( + type='write_to_array', + inputs={'X': init, 'I': self._zero_idx}, + outputs={'Out': array}, + ) if is_ids: self._ids_array = array @@ -800,7 +838,8 @@ class BeamSearchDecoder(object): if not isinstance(array, Variable): raise TypeError( - 'The input argument `array` of must be a Variable.') + 'The input argument `array` of must be a Variable.' + ) if not isinstance(value, Variable): raise TypeError('The input argument `value` of must be a Variable.') @@ -819,12 +858,16 @@ class BeamSearchDecoder(object): as id, holds the score for each generated token. """ if self._status != BeamSearchDecoder.AFTER_BEAM_SEARCH_DECODER: - raise ValueError('Output of BeamSearchDecoder object can ' - 'only be visited outside the block.') - return layers.beam_search_decode(ids=self._ids_array, - scores=self._scores_array, - beam_size=self._beam_size, - end_id=self._end_id) + raise ValueError( + 'Output of BeamSearchDecoder object can ' + 'only be visited outside the block.' + ) + return layers.beam_search_decode( + ids=self._ids_array, + scores=self._scores_array, + beam_size=self._beam_size, + end_id=self._end_id, + ) @property def state_cell(self): @@ -847,5 +890,7 @@ class BeamSearchDecoder(object): def _assert_in_decoder_block(self, method): if self._status != BeamSearchDecoder.IN_BEAM_SEARCH_DECODER: - raise ValueError('%s should be invoked inside block of ' - 'BeamSearchDecoder object.' % method) + raise ValueError( + '%s should be invoked inside block of ' + 'BeamSearchDecoder object.' % method + ) diff --git a/python/paddle/fluid/contrib/extend_optimizer/extend_optimizer_with_weight_decay.py b/python/paddle/fluid/contrib/extend_optimizer/extend_optimizer_with_weight_decay.py index 6a87bb54d3f..59b6ac38884 100644 --- a/python/paddle/fluid/contrib/extend_optimizer/extend_optimizer_with_weight_decay.py +++ b/python/paddle/fluid/contrib/extend_optimizer/extend_optimizer_with_weight_decay.py @@ -18,10 +18,10 @@ __all__ = ["extend_with_decoupled_weight_decay"] class DecoupledWeightDecay(object): - def __init__(self, coeff=0.0, apply_decay_param_fun=None, **kwargs): - if not isinstance(coeff, float) and \ - not isinstance(coeff, framework.Variable): + if not isinstance(coeff, float) and not isinstance( + coeff, framework.Variable + ): raise TypeError("coeff should be float or Variable.") self._params_name = set() self._apply_decay_param_fun = apply_decay_param_fun @@ -47,19 +47,28 @@ class DecoupledWeightDecay(object): # If no gradient then we don't need to do anything if grad is None: continue - if self._apply_decay_param_fun is not None \ - and not self._apply_decay_param_fun(param.name): + if ( + self._apply_decay_param_fun is not None + and not self._apply_decay_param_fun(param.name) + ): continue if isinstance(self._coeff, float): - assert param.dtype is not paddle.fluid.core.VarDesc.VarType.FP32, \ - "the type of coeff(float) and parameter(%s) is not consistent."%(self._coeff.dtype) + assert ( + param.dtype is not paddle.fluid.core.VarDesc.VarType.FP32 + ), ( + "the type of coeff(float) and parameter(%s) is not consistent." + % (self._coeff.dtype) + ) else: - assert self._coeff.dtype == param.dtype, \ - "the type of coeff(%s) and parameter(%s) is not consistent."%(self._coeff.dtype, param.dtype) + assert self._coeff.dtype == param.dtype, ( + "the type of coeff(%s) and parameter(%s) is not consistent." + % (self._coeff.dtype, param.dtype) + ) with param.block.program._optimized_guard( - [param, grad]), framework.name_scope('weight decay'): + [param, grad] + ), framework.name_scope('weight decay'): assert param.name not in self._params_name scaled_params.append((param, grad, param * self._coeff)) self._params_name.add(param.name) @@ -71,27 +80,31 @@ class DecoupledWeightDecay(object): def apply_optimize(self, **kargs): return super(DecoupledWeightDecay, self).apply_optimize(**kargs) - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - params_grads = self.backward(loss=loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + params_grads = self.backward( + loss=loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) scaled_params = self._scale_parameters(params_grads) for p_grad_sgrad in scaled_params: param, grad, scaled_param = p_grad_sgrad with param.block.program._optimized_guard( - [param, grad]), framework.name_scope('weight decay'): + [param, grad] + ), framework.name_scope('weight decay'): updated_param = paddle.fluid.layers.elementwise_sub( - x=param, y=scaled_param) + x=param, y=scaled_param + ) paddle.fluid.layers.assign(input=updated_param, output=param) - optimize_ops = self.apply_optimize(loss=loss, - params_grads=params_grads, - startup_program=startup_program) + optimize_ops = self.apply_optimize( + loss=loss, + params_grads=params_grads, + startup_program=startup_program, + ) return optimize_ops, params_grads def __str__(self): @@ -126,10 +139,12 @@ def extend_with_decoupled_weight_decay(base_optimizer): """ if not issubclass(base_optimizer, paddle.fluid.optimizer.Optimizer): raise TypeError( - "The input(base_optimizer) should be a derived class of Optimizer.") + "The input(base_optimizer) should be a derived class of Optimizer." + ) - class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecay, - base_optimizer): + class OptimizerWithDecoupledWeightDecay( + DecoupledWeightDecay, base_optimizer + ): """ OptimizerWithDecoupledWeightDecay is used to update the optimized parameters with the parameters before optimization. For more information, please refer: @@ -145,7 +160,8 @@ def extend_with_decoupled_weight_decay(base_optimizer): """ def __init__(self, weight_decay, apply_decay_param_fun=None, **kwargs): - super(OptimizerWithDecoupledWeightDecay, - self).__init__(weight_decay, apply_decay_param_fun, **kwargs) + super(OptimizerWithDecoupledWeightDecay, self).__init__( + weight_decay, apply_decay_param_fun, **kwargs + ) return OptimizerWithDecoupledWeightDecay diff --git a/python/paddle/fluid/contrib/layers/metric_op.py b/python/paddle/fluid/contrib/layers/metric_op.py index 82d1a145f5b..414fcf5b6cd 100755 --- a/python/paddle/fluid/contrib/layers/metric_op.py +++ b/python/paddle/fluid/contrib/layers/metric_op.py @@ -78,171 +78,195 @@ def ctr_metric_bundle(input, label, ins_tag_weight=None): """ if ins_tag_weight is None: - ins_tag_weight = tensor.fill_constant(shape=[1, 1], - dtype="float32", - value=1.0) + ins_tag_weight = tensor.fill_constant( + shape=[1, 1], dtype="float32", value=1.0 + ) assert input.shape == label.shape helper = LayerHelper("ctr_metric_bundle", **locals()) - local_abserr = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_sqrerr = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_prob = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_q = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_pos_num = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_ins_num = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - - tmp_res_elesub = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[-1]) - tmp_res_sigmoid = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[-1]) - tmp_ones = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[-1]) - - batch_prob = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_abserr = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_sqrerr = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_q = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_pos_num = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_ins_num = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) + local_abserr = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_sqrerr = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_prob = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_q = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_pos_num = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_ins_num = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + + tmp_res_elesub = helper.create_global_variable( + persistable=False, dtype='float32', shape=[-1] + ) + tmp_res_sigmoid = helper.create_global_variable( + persistable=False, dtype='float32', shape=[-1] + ) + tmp_ones = helper.create_global_variable( + persistable=False, dtype='float32', shape=[-1] + ) + + batch_prob = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_abserr = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_sqrerr = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_q = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_pos_num = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_ins_num = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) for var in [ - local_abserr, batch_abserr, local_sqrerr, batch_sqrerr, local_prob, - batch_prob, local_q, batch_q, batch_pos_num, batch_ins_num, - local_pos_num, local_ins_num + local_abserr, + batch_abserr, + local_sqrerr, + batch_sqrerr, + local_prob, + batch_prob, + local_q, + batch_q, + batch_pos_num, + batch_ins_num, + local_pos_num, + local_ins_num, ]: - helper.set_variable_initializer(var, Constant(value=0.0, - force_cpu=True)) - - helper.append_op(type="elementwise_sub", - inputs={ - "X": [input], - "Y": [label] - }, - outputs={"Out": [tmp_res_elesub]}) - - helper.append_op(type="squared_l2_norm", - inputs={"X": [tmp_res_elesub]}, - outputs={"Out": [batch_sqrerr]}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_sqrerr], - "Y": [local_sqrerr] - }, - outputs={"Out": [local_sqrerr]}) - - helper.append_op(type="l1_norm", - inputs={"X": [tmp_res_elesub]}, - outputs={"Out": [batch_abserr]}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_abserr], - "Y": [local_abserr] - }, - outputs={"Out": [local_abserr]}) - - helper.append_op(type="reduce_sum", - inputs={"X": [input]}, - outputs={"Out": [batch_prob]}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_prob], - "Y": [local_prob] - }, - outputs={"Out": [local_prob]}) - helper.append_op(type="sigmoid", - inputs={"X": [input]}, - outputs={"Out": [tmp_res_sigmoid]}) - helper.append_op(type="reduce_sum", - inputs={"X": [tmp_res_sigmoid]}, - outputs={"Out": [batch_q]}) - - helper.append_op(type="reduce_sum", - inputs={"X": [label]}, - outputs={"Out": [batch_pos_num]}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_pos_num], - "Y": [local_pos_num] - }, - outputs={"Out": [local_pos_num]}) - - helper.append_op(type='fill_constant_batch_size_like', - inputs={"Input": label}, - outputs={'Out': [tmp_ones]}, - attrs={ - 'shape': [-1, 1], - 'dtype': tmp_ones.dtype, - 'value': float(1.0), - }) - helper.append_op(type="reduce_sum", - inputs={"X": [tmp_ones]}, - outputs={"Out": [batch_ins_num]}) - - #if data is fake, return 0 + helper.set_variable_initializer( + var, Constant(value=0.0, force_cpu=True) + ) + + helper.append_op( + type="elementwise_sub", + inputs={"X": [input], "Y": [label]}, + outputs={"Out": [tmp_res_elesub]}, + ) + + helper.append_op( + type="squared_l2_norm", + inputs={"X": [tmp_res_elesub]}, + outputs={"Out": [batch_sqrerr]}, + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_sqrerr], "Y": [local_sqrerr]}, + outputs={"Out": [local_sqrerr]}, + ) + + helper.append_op( + type="l1_norm", + inputs={"X": [tmp_res_elesub]}, + outputs={"Out": [batch_abserr]}, + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_abserr], "Y": [local_abserr]}, + outputs={"Out": [local_abserr]}, + ) + + helper.append_op( + type="reduce_sum", inputs={"X": [input]}, outputs={"Out": [batch_prob]} + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_prob], "Y": [local_prob]}, + outputs={"Out": [local_prob]}, + ) + helper.append_op( + type="sigmoid", + inputs={"X": [input]}, + outputs={"Out": [tmp_res_sigmoid]}, + ) + helper.append_op( + type="reduce_sum", + inputs={"X": [tmp_res_sigmoid]}, + outputs={"Out": [batch_q]}, + ) + + helper.append_op( + type="reduce_sum", + inputs={"X": [label]}, + outputs={"Out": [batch_pos_num]}, + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_pos_num], "Y": [local_pos_num]}, + outputs={"Out": [local_pos_num]}, + ) + + helper.append_op( + type='fill_constant_batch_size_like', + inputs={"Input": label}, + outputs={'Out': [tmp_ones]}, + attrs={ + 'shape': [-1, 1], + 'dtype': tmp_ones.dtype, + 'value': float(1.0), + }, + ) + helper.append_op( + type="reduce_sum", + inputs={"X": [tmp_ones]}, + outputs={"Out": [batch_ins_num]}, + ) + + # if data is fake, return 0 inputs_slice = {'Input': ins_tag_weight} attrs = {'axes': [0]} attrs['starts'] = [0] attrs['ends'] = [1] - helper.append_op(type="slice", - inputs=inputs_slice, - attrs=attrs, - outputs={"Out": ins_tag_weight}) + helper.append_op( + type="slice", + inputs=inputs_slice, + attrs=attrs, + outputs={"Out": ins_tag_weight}, + ) axis = helper.kwargs.get('axis', 0) - helper.append_op(type="elementwise_mul", - inputs={ - "X": [batch_ins_num], - "Y": [ins_tag_weight] - }, - outputs={"Out": [batch_ins_num]}, - attrs={'axis': axis}) - - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_ins_num], - "Y": [local_ins_num] - }, - outputs={"Out": [local_ins_num]}) - - helper.append_op(type="elementwise_mul", - inputs={ - "X": [batch_q], - "Y": [ins_tag_weight] - }, - outputs={"Out": [batch_q]}, - attrs={'axis': axis}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_q], - "Y": [local_q] - }, - outputs={"Out": [local_q]}) - - return local_sqrerr, local_abserr, local_prob, local_q, local_pos_num, local_ins_num + helper.append_op( + type="elementwise_mul", + inputs={"X": [batch_ins_num], "Y": [ins_tag_weight]}, + outputs={"Out": [batch_ins_num]}, + attrs={'axis': axis}, + ) + + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_ins_num], "Y": [local_ins_num]}, + outputs={"Out": [local_ins_num]}, + ) + + helper.append_op( + type="elementwise_mul", + inputs={"X": [batch_q], "Y": [ins_tag_weight]}, + outputs={"Out": [batch_q]}, + attrs={'axis': axis}, + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_q], "Y": [local_q]}, + outputs={"Out": [local_q]}, + ) + + return ( + local_sqrerr, + local_abserr, + local_prob, + local_q, + local_pos_num, + local_ins_num, + ) diff --git a/python/paddle/fluid/contrib/layers/nn.py b/python/paddle/fluid/contrib/layers/nn.py index e09bdcb0d8f..08ab46dfac9 100644 --- a/python/paddle/fluid/contrib/layers/nn.py +++ b/python/paddle/fluid/contrib/layers/nn.py @@ -25,7 +25,12 @@ from paddle.fluid.layer_helper import LayerHelper from paddle.fluid.layers import utils from ... import unique_name from paddle.fluid.initializer import Normal, Constant, NumpyArrayInitializer -from paddle.fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from paddle.fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from paddle.fluid import core from paddle.fluid.param_attr import ParamAttr @@ -36,21 +41,33 @@ import warnings from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'fused_elemwise_activation', 'sequence_topk_avg_pooling', 'var_conv_2d', - 'match_matrix_tensor', 'tree_conv', 'fused_embedding_seq_pool', - 'multiclass_nms2', 'search_pyramid_hash', 'shuffle_batch', 'partial_concat', - 'sparse_embedding', 'partial_sum', 'tdm_child', 'rank_attention', - 'tdm_sampler', 'batch_fc', '_pull_box_extended_sparse', 'bilateral_slice', - 'correlation', 'fused_bn_add_act', 'fused_seqpool_cvm' + 'fused_elemwise_activation', + 'sequence_topk_avg_pooling', + 'var_conv_2d', + 'match_matrix_tensor', + 'tree_conv', + 'fused_embedding_seq_pool', + 'multiclass_nms2', + 'search_pyramid_hash', + 'shuffle_batch', + 'partial_concat', + 'sparse_embedding', + 'partial_sum', + 'tdm_child', + 'rank_attention', + 'tdm_sampler', + 'batch_fc', + '_pull_box_extended_sparse', + 'bilateral_slice', + 'correlation', + 'fused_bn_add_act', + 'fused_seqpool_cvm', ] -def fused_elemwise_activation(x, - y, - functor_list, - axis=-1, - scale=0.0, - save_intermediate_out=True): +def fused_elemwise_activation( + x, y, functor_list, axis=-1, scale=0.0, save_intermediate_out=True +): """ **Fused elementwise_add/mul and activation layers** @@ -89,104 +106,103 @@ def fused_elemwise_activation(x, if not isinstance(functor_list, list) or len(functor_list) != 2: raise ValueError( - 'functor_list should be a list of str, and the length should be 2.') + 'functor_list should be a list of str, and the length should be 2.' + ) helper = LayerHelper('fused_elemwise_activation', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) intermediate_out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='fused_elemwise_activation', - inputs={ - 'X': x, - 'Y': y - }, - outputs={ - 'Out': out, - 'IntermediateOut': intermediate_out - }, - attrs={ - 'axis': axis, - 'scale': scale, - 'save_intermediate_out': save_intermediate_out, - 'functor_list': functor_list - }) + helper.append_op( + type='fused_elemwise_activation', + inputs={'X': x, 'Y': y}, + outputs={'Out': out, 'IntermediateOut': intermediate_out}, + attrs={ + 'axis': axis, + 'scale': scale, + 'save_intermediate_out': save_intermediate_out, + 'functor_list': functor_list, + }, + ) return out -def var_conv_2d(input, - row, - col, - input_channel, - output_channel, - filter_size, - stride=1, - param_attr=None, - act=None, - dtype='float32', - name=None): +def var_conv_2d( + input, + row, + col, + input_channel, + output_channel, + filter_size, + stride=1, + param_attr=None, + act=None, + dtype='float32', + name=None, +): r""" - The var_conv_2d layer calculates the output base on the :attr:`input` with variable length, - row, col, input channel, filter size and strides. Both :attr:`input`, :attr:`row`, - and :attr:`col` are 1-level LodTensor. The convolution operation is same as conv2d layer with - padding. Besides, input.dims[1] should be 1. - - .. code-block:: text - - If input_channel is 2 and given row lodTensor and col lodTensor as follows: - row.lod = [[5, 4]] - col.lod = [[6, 7]] - input is a lodTensor: - input.lod = [[60, 56]] # where 60 = input_channel * 5 * 6 - input.dims = [116, 1] # where 116 = 60 + 56 - - If set output_channel is 3, filter_size is [3, 3], stride is [1, 1]: - # where 90 = output_channel * [(5-1)/stride + 1] * [(6-1)/stride + 1] - output.lod = [[90, 84]] - output.dims = [174, 1] # where 174 = 90 + 84 - - Args: - input (Variable): The input should be 1-level LodTensor with dims[1] equals 1. - row (Variable): The row should be 1-level LodTensor to provide height information. - col (Variable): The col should be 1-level LodTensor to provide width information. - input_channel (int): The number of input channel. - output_channel (int): The number of output channel. - filter_size (int|tuple|None): The filter size. If filter_size is a tuple, - it must contain two integers, (filter_size_H, filter_size_W). - Otherwise, the filter will be a square. - stride (int|tuple): The stride size. If stride is a tuple, it must - contain two integers, (stride_H, stride_W). Otherwise, the - stride_H = stride_W = stride. Default: stride = 1. - param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights - of var_conv2d. If it is set to None or one attribute of ParamAttr, var_conv2d - will create ParamAttr as param_attr. If the Initializer of the param_attr - is not set, the parameter is initialized with :math:`Normal(0.0, std)`, - and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{ - 0.5}`. Default: None. - act (str): Activation type, if it is set to None, activation is not appended. - Default: None - dtype ('float32'): The data type of parameter and output. - name (str|None): A name for this layer(optional). If set None, the layer - will be named automatically. Default: None - - Returns: - Variable: Output variable with LoD specified by this layer. - - Examples: - .. code-block:: python - - import numpy as np - from paddle.fluid import layers - from paddle.fluid import contrib - - x_lod_tensor = layers.data(name='x', shape=[1], lod_level=1) - row_lod_tensor = layers.data(name='row', shape=[6], lod_level=1) - col_lod_tensor = layers.data(name='col', shape=[6], lod_level=1) - out = contrib.var_conv_2d(input=x_lod_tensor, - row=row_lod_tensor, - col=col_lod_tensor, - input_channel=3, - output_channel=5, - filter_size=[3, 3], - stride=1) + The var_conv_2d layer calculates the output base on the :attr:`input` with variable length, + row, col, input channel, filter size and strides. Both :attr:`input`, :attr:`row`, + and :attr:`col` are 1-level LodTensor. The convolution operation is same as conv2d layer with + padding. Besides, input.dims[1] should be 1. + + .. code-block:: text + + If input_channel is 2 and given row lodTensor and col lodTensor as follows: + row.lod = [[5, 4]] + col.lod = [[6, 7]] + input is a lodTensor: + input.lod = [[60, 56]] # where 60 = input_channel * 5 * 6 + input.dims = [116, 1] # where 116 = 60 + 56 + + If set output_channel is 3, filter_size is [3, 3], stride is [1, 1]: + # where 90 = output_channel * [(5-1)/stride + 1] * [(6-1)/stride + 1] + output.lod = [[90, 84]] + output.dims = [174, 1] # where 174 = 90 + 84 + + Args: + input (Variable): The input should be 1-level LodTensor with dims[1] equals 1. + row (Variable): The row should be 1-level LodTensor to provide height information. + col (Variable): The col should be 1-level LodTensor to provide width information. + input_channel (int): The number of input channel. + output_channel (int): The number of output channel. + filter_size (int|tuple|None): The filter size. If filter_size is a tuple, + it must contain two integers, (filter_size_H, filter_size_W). + Otherwise, the filter will be a square. + stride (int|tuple): The stride size. If stride is a tuple, it must + contain two integers, (stride_H, stride_W). Otherwise, the + stride_H = stride_W = stride. Default: stride = 1. + param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights + of var_conv2d. If it is set to None or one attribute of ParamAttr, var_conv2d + will create ParamAttr as param_attr. If the Initializer of the param_attr + is not set, the parameter is initialized with :math:`Normal(0.0, std)`, + and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{ + 0.5}`. Default: None. + act (str): Activation type, if it is set to None, activation is not appended. + Default: None + dtype ('float32'): The data type of parameter and output. + name (str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Default: None + + Returns: + Variable: Output variable with LoD specified by this layer. + + Examples: + .. code-block:: python + + import numpy as np + from paddle.fluid import layers + from paddle.fluid import contrib + + x_lod_tensor = layers.data(name='x', shape=[1], lod_level=1) + row_lod_tensor = layers.data(name='row', shape=[6], lod_level=1) + col_lod_tensor = layers.data(name='col', shape=[6], lod_level=1) + out = contrib.var_conv_2d(input=x_lod_tensor, + row=row_lod_tensor, + col=col_lod_tensor, + input_channel=3, + output_channel=5, + filter_size=[3, 3], + stride=1) """ helper = LayerHelper('var_conv_2d', **locals()) x_shape = list(input.shape) @@ -197,7 +213,7 @@ def var_conv_2d(input, filter_shape = [ int(output_channel), - int(input_channel) * filter_size[0] * filter_size[1] + int(input_channel) * filter_size[0] * filter_size[1], ] filter_param = helper.create_parameter( attr=helper.param_attr, @@ -206,39 +222,35 @@ def var_conv_2d(input, ) conv_res = helper.create_variable_for_type_inference(dtype) - tmp_res = helper.create_variable_for_type_inference(dtype, - stop_gradient=True) - - helper.append_op(type='var_conv_2d', - inputs={ - 'X': input, - 'ROW': row, - 'COLUMN': col, - 'W': filter_param, - }, - outputs={ - "Out": conv_res, - "Col": tmp_res - }, - attrs={ - 'InputChannel': input_channel, - 'OutputChannel': output_channel, - 'StrideH': stride[0], - 'StrideW': stride[1], - 'KernelH': filter_size[0], - 'KernelW': filter_size[1], - }) + tmp_res = helper.create_variable_for_type_inference( + dtype, stop_gradient=True + ) + + helper.append_op( + type='var_conv_2d', + inputs={ + 'X': input, + 'ROW': row, + 'COLUMN': col, + 'W': filter_param, + }, + outputs={"Out": conv_res, "Col": tmp_res}, + attrs={ + 'InputChannel': input_channel, + 'OutputChannel': output_channel, + 'StrideH': stride[0], + 'StrideW': stride[1], + 'KernelH': filter_size[0], + 'KernelW': filter_size[1], + }, + ) return helper.append_activation(conv_res) -def match_matrix_tensor(x, - y, - channel_num, - act=None, - param_attr=None, - dtype='float32', - name=None): +def match_matrix_tensor( + x, y, channel_num, act=None, param_attr=None, dtype='float32', name=None +): """ Calculate the semantic matching matrix of two word sequences with variable length. Given a query A of length `n` and a title B of length `m`, the input shape are respectively @@ -296,28 +308,28 @@ def match_matrix_tensor(x, x_shape = list(x.shape) y_shape = list(y.shape) - assert len(x_shape) == 2 and len( - y_shape) == 2 and x_shape[-1] == y_shape[-1] + assert ( + len(x_shape) == 2 and len(y_shape) == 2 and x_shape[-1] == y_shape[-1] + ) weight_shape = [x_shape[-1], channel_num, y_shape[-1]] - w = helper.create_parameter(attr=helper.param_attr, - shape=weight_shape, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=weight_shape, dtype=dtype, is_bias=False + ) mm_res = helper.create_variable_for_type_inference(dtype) - tmp_res = helper.create_variable_for_type_inference(dtype, - stop_gradient=True) - helper.append_op(type='match_matrix_tensor', - inputs={ - 'X': x, - 'Y': y, - 'W': w, - }, - outputs={ - "Out": mm_res, - "Tmp": tmp_res - }, - attrs={'dim_t': channel_num}) + tmp_res = helper.create_variable_for_type_inference( + dtype, stop_gradient=True + ) + helper.append_op( + type='match_matrix_tensor', + inputs={ + 'X': x, + 'Y': y, + 'W': w, + }, + outputs={"Out": mm_res, "Tmp": tmp_res}, + attrs={'dim_t': channel_num}, + ) return helper.append_activation(mm_res), tmp_res @@ -375,72 +387,67 @@ def sequence_topk_avg_pooling(input, row, col, topks, channel_num): """ helper = LayerHelper('sequence_topk_avg_pooling', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - pos = helper.create_variable_for_type_inference(dtype=helper.input_dtype(), - stop_gradient=True) - helper.append_op(type='sequence_topk_avg_pooling', - inputs={ - 'X': input, - 'ROW': row, - 'COLUMN': col - }, - outputs={ - 'Out': out, - 'pos': pos - }, - attrs={ - 'topks': topks, - 'channel_num': channel_num - }) + pos = helper.create_variable_for_type_inference( + dtype=helper.input_dtype(), stop_gradient=True + ) + helper.append_op( + type='sequence_topk_avg_pooling', + inputs={'X': input, 'ROW': row, 'COLUMN': col}, + outputs={'Out': out, 'pos': pos}, + attrs={'topks': topks, 'channel_num': channel_num}, + ) return out -def tree_conv(nodes_vector, - edge_set, - output_size, - num_filters=1, - max_depth=2, - act='tanh', - param_attr=None, - bias_attr=None, - name=None): +def tree_conv( + nodes_vector, + edge_set, + output_size, + num_filters=1, + max_depth=2, + act='tanh', + param_attr=None, + bias_attr=None, + name=None, +): """ - ${comment} -Args : nodes_vector(${nodes_vector_type}) : $ { nodes_vector_comment } -edge_set(${edge_set_type}) : $ { edge_set_comment } - output_size(int): output feature width - num_filters(int): number of filters, Default 1 - max_depth(int): max depth of filters, Default 2 - act(str): activation function, Default tanh - param_attr(ParamAttr): the parameter attribute for the filters, Default None - bias_attr(ParamAttr): the parameter attribute for the bias of this layer, Default None - name(str): a name of this layer(optional). If set None, the layer will be named automatically, Default None - - Returns: - out(${out_type}): ${ - out_comment - } - - Examples: - .. code-block:: python - - import paddle.fluid as fluid + ${comment} + Args : nodes_vector(${nodes_vector_type}) : $ { nodes_vector_comment } + edge_set(${edge_set_type}) : $ { edge_set_comment } + output_size(int): output feature width + num_filters(int): number of filters, Default 1 + max_depth(int): max depth of filters, Default 2 + act(str): activation function, Default tanh + param_attr(ParamAttr): the parameter attribute for the filters, Default None + bias_attr(ParamAttr): the parameter attribute for the bias of this layer, Default None + name(str): a name of this layer(optional). If set None, the layer will be named automatically, Default None + + Returns: + out(${out_type}): ${ + out_comment + } + + Examples: + .. code-block:: python - # 10 for max_node_size of dataset, 5 for vector width - nodes_vector = fluid.layers.data( - name='vectors', shape=[10, 5], dtype='float32') - # 10 for max_node_size of dataset, 2 for every edge has two nodes - # edges must be directional - edge_set = fluid.layers.data(name='edge_set', shape=[ - 10, 2], dtype='float32') - # the shape of output will be [10, 6, 1], - # 10 for max_node_size of dataset, 6 for output size, 1 for 1 filter - out_vector = fluid.layers.tree_conv(nodes_vector, edge_set, 6, 1, 2) -#After reshape, output tensor could be nodes_vector for next tree convolution - out_vector = fluid.layers.reshape(out_vector, shape=[-1, 10, 6]) - out_vector_2 = fluid.layers.tree_conv(out_vector, edge_set, 3, 4, 2) -#also output tensor could be pooling(the pooling in paper called global pooling) - pooled = fluid.layers.reduce_max(out_vector, dim=2) # global pooling + import paddle.fluid as fluid + + # 10 for max_node_size of dataset, 5 for vector width + nodes_vector = fluid.layers.data( + name='vectors', shape=[10, 5], dtype='float32') + # 10 for max_node_size of dataset, 2 for every edge has two nodes + # edges must be directional + edge_set = fluid.layers.data(name='edge_set', shape=[ + 10, 2], dtype='float32') + # the shape of output will be [10, 6, 1], + # 10 for max_node_size of dataset, 6 for output size, 1 for 1 filter + out_vector = fluid.layers.tree_conv(nodes_vector, edge_set, 6, 1, 2) + #After reshape, output tensor could be nodes_vector for next tree convolution + out_vector = fluid.layers.reshape(out_vector, shape=[-1, 10, 6]) + out_vector_2 = fluid.layers.tree_conv(out_vector, edge_set, 3, 4, 2) + #also output tensor could be pooling(the pooling in paper called global pooling) + pooled = fluid.layers.reduce_max(out_vector, dim=2) # global pooling """ check_type(nodes_vector, 'nodes_vector', (Variable), 'tree_conv') check_type(edge_set, 'edge_set', (Variable), 'tree_conv') @@ -449,21 +456,18 @@ edge_set(${edge_set_type}) : $ { edge_set_comment } dtype = helper.input_dtype('nodes_vector') feature_size = nodes_vector.shape[2] W_shape = [feature_size, 3, output_size, num_filters] - W = helper.create_parameter(attr=param_attr, - shape=W_shape, - dtype=dtype, - is_bias=False) + W = helper.create_parameter( + attr=param_attr, shape=W_shape, dtype=dtype, is_bias=False + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='tree_conv', - inputs={ - 'NodesVector': nodes_vector, - 'EdgeSet': edge_set, - 'Filter': W - }, - outputs={ - 'Out': out, - }, - attrs={'max_depth': max_depth}) + helper.append_op( + type='tree_conv', + inputs={'NodesVector': nodes_vector, 'EdgeSet': edge_set, 'Filter': W}, + outputs={ + 'Out': out, + }, + attrs={'max_depth': max_depth}, + ) if helper.bias_attr: pre_activation = helper.append_bias_op(out) else: @@ -471,13 +475,15 @@ edge_set(${edge_set_type}) : $ { edge_set_comment } return helper.append_activation(pre_activation) -def fused_embedding_seq_pool(input, - size, - is_sparse=False, - padding_idx=None, - combiner='sum', - param_attr=None, - dtype='float32'): +def fused_embedding_seq_pool( + input, + size, + is_sparse=False, + padding_idx=None, + combiner='sum', + param_attr=None, + dtype='float32', +): r""" **Embedding Sequence pool** @@ -520,33 +526,33 @@ def fused_embedding_seq_pool(input, is_sparse=False) """ helper = LayerHelper('fused_embedding_seq_pool', **locals()) - w = helper.create_parameter(attr=helper.param_attr, - shape=size, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False + ) out = helper.create_variable_for_type_inference(dtype) - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) - helper.append_op(type='fused_embedding_seq_pool', - inputs={ - 'Ids': input, - 'W': w - }, - outputs={'Out': out}, - attrs={ - 'is_sparse': is_sparse, - 'combiner': combiner, - 'padding_idx': padding_idx - }) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) + helper.append_op( + type='fused_embedding_seq_pool', + inputs={'Ids': input, 'W': w}, + outputs={'Out': out}, + attrs={ + 'is_sparse': is_sparse, + 'combiner': combiner, + 'padding_idx': padding_idx, + }, + ) return out -def fused_seqpool_cvm(input, - pool_type, - cvm, - pad_value=0.0, - use_cvm=True, - cvm_offset=2): +def fused_seqpool_cvm( + input, pool_type, cvm, pad_value=0.0, use_cvm=True, cvm_offset=2 +): """ :api_attr: Static Graph @@ -592,13 +598,15 @@ def fused_seqpool_cvm(input, if pool_type.upper() != 'SUM': raise ValueError( "fused_seqpool_cvm only support SUM pooling now, and your type is: " - + pool_type) + + pool_type + ) check_type(input, 'input', list, 'fused_seqpool_cvm') if isinstance(input, list): for _input in input: - check_variable_and_dtype(_input, 'input', ['float32'], - 'fused_seqpool_cvm') + check_variable_and_dtype( + _input, 'input', ['float32'], 'fused_seqpool_cvm' + ) dtype = helper.input_dtype() inputs = helper.multiple_input() @@ -607,33 +615,34 @@ def fused_seqpool_cvm(input, for i in range(len(inputs)) ] - helper.append_op(type="fused_seqpool_cvm", - inputs={ - "X": inputs, - "CVM": cvm - }, - outputs={"Out": outs}, - attrs={ - "pooltype": pool_type.upper(), - "pad_value": pad_value, - "use_cvm": use_cvm, - "cvm_offset": cvm_offset, - }) + helper.append_op( + type="fused_seqpool_cvm", + inputs={"X": inputs, "CVM": cvm}, + outputs={"Out": outs}, + attrs={ + "pooltype": pool_type.upper(), + "pad_value": pad_value, + "use_cvm": use_cvm, + "cvm_offset": cvm_offset, + }, + ) return outs -def multiclass_nms2(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=0, - return_index=False, - name=None): +def multiclass_nms2( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=0, + return_index=False, + name=None, +): """ **Multiclass NMS2** @@ -729,24 +738,20 @@ def multiclass_nms2(bboxes, output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) index = helper.create_variable_for_type_inference(dtype='int') - helper.append_op(type="multiclass_nms2", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs={ - 'Out': output, - 'Index': index - }) + helper.append_op( + type="multiclass_nms2", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs={'Out': output, 'Index': index}, + ) output.stop_gradient = True index.stop_gradient = True @@ -755,24 +760,26 @@ def multiclass_nms2(bboxes, return output -def search_pyramid_hash(input, - num_emb, - space_len, - pyramid_layer, - rand_len, - drop_out_percent, - is_training, - use_filter, - white_list_len, - black_list_len, - seed, - lr, - param_attr=None, - param_attr_wl=None, - param_attr_bl=None, - name=None, - distribute_update_vars=None, - dtype='float32'): +def search_pyramid_hash( + input, + num_emb, + space_len, + pyramid_layer, + rand_len, + drop_out_percent, + is_training, + use_filter, + white_list_len, + black_list_len, + seed, + lr, + param_attr=None, + param_attr_wl=None, + param_attr_bl=None, + name=None, + distribute_update_vars=None, + dtype='float32', +): """ **Pyramid hash embedding** @@ -809,28 +816,25 @@ def search_pyramid_hash(input, helper = LayerHelper('search_pyramid_hash', **locals()) w_shape = [space_len + rand_len, 1] - w = helper.create_parameter(attr=param_attr, - shape=w_shape, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=param_attr, shape=w_shape, dtype=dtype, is_bias=False + ) w.stop_gradient = True input_vars = {'X': input, 'W': w} if white_list_len > 0: wl_shape = [white_list_len, 1] - white_list = helper.create_parameter(attr=param_attr_wl, - shape=wl_shape, - dtype=dtype, - is_bias=False) + white_list = helper.create_parameter( + attr=param_attr_wl, shape=wl_shape, dtype=dtype, is_bias=False + ) white_list.stop_gradient = True input_vars['WhiteList'] = white_list if black_list_len >= 0: bl_shape = [black_list_len, 1] - black_list = helper.create_parameter(attr=param_attr_bl, - shape=bl_shape, - dtype=dtype, - is_bias=False) + black_list = helper.create_parameter( + attr=param_attr_bl, shape=bl_shape, dtype=dtype, is_bias=False + ) black_list.stop_gradient = True input_vars['BlackList'] = black_list @@ -847,33 +851,32 @@ def search_pyramid_hash(input, for param in distribute_update_vars: if param not in special_name_list: raise ValueError( - "Pyramid Hash layer didn't have parameter {}".format(param)) + "Pyramid Hash layer didn't have parameter {}".format(param) + ) distribute_update_vars_str = ",".join(distribute_update_vars) res = helper.create_variable_for_type_inference(dtype) drop_pos = helper.create_variable_for_type_inference(dtype) x_temp_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pyramid_hash', - inputs=input_vars, - outputs={ - "Out": res, - "X_Temp_Out": x_temp_out, - 'DropPos': drop_pos - }, - attrs={ - 'num_emb': num_emb, - 'space_len': space_len, - 'pyramid_layer': pyramid_layer, - 'rand_len': rand_len, - 'drop_out_percent': drop_out_percent, - 'is_training': is_training, - 'use_filter': use_filter, - 'white_list_len': white_list_len, - 'black_list_len': black_list_len, - 'seed': seed, - 'lr': lr, - 'distribute_update_vars': distribute_update_vars_str - }) + helper.append_op( + type='pyramid_hash', + inputs=input_vars, + outputs={"Out": res, "X_Temp_Out": x_temp_out, 'DropPos': drop_pos}, + attrs={ + 'num_emb': num_emb, + 'space_len': space_len, + 'pyramid_layer': pyramid_layer, + 'rand_len': rand_len, + 'drop_out_percent': drop_out_percent, + 'is_training': is_training, + 'use_filter': use_filter, + 'white_list_len': white_list_len, + 'black_list_len': black_list_len, + 'seed': seed, + 'lr': lr, + 'distribute_update_vars': distribute_update_vars_str, + }, + ) return res @@ -930,18 +933,14 @@ def shuffle_batch(x, seed=None): seed = helper.create_variable( name=unique_name.generate("shuffle_batch_seed"), dtype="int64", - persistable=False) - helper.append_op(type='shuffle_batch', - inputs={ - 'X': x, - 'Seed': seed - }, - outputs={ - 'Out': out, - 'ShuffleIdx': shuffle_idx, - 'SeedOut': seed - }, - attrs=op_attrs) + persistable=False, + ) + helper.append_op( + type='shuffle_batch', + inputs={'X': x, 'Seed': seed}, + outputs={'Out': out, 'ShuffleIdx': shuffle_idx, 'SeedOut': seed}, + attrs=op_attrs, + ) return out @@ -987,23 +986,28 @@ def partial_concat(input, start_index=0, length=-1): if not isinstance(input, list): warnings.warn( "The type of input in partial_concat should be list, but received %s." - % (type(input))) + % (type(input)) + ) input = [input] for id, x in enumerate(input): check_variable_and_dtype( - x, 'input[' + str(id) + ']', + x, + 'input[' + str(id) + ']', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'partial_concat') + 'partial_concat', + ) check_type(start_index, 'start_index', (int), 'partial_concat') check_type(length, 'length', (int), 'partial_concat') inputs = {'X': input} attrs = {'start_index': start_index, 'length': length} helper = LayerHelper('partial_concat', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='partial_concat', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='partial_concat', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + ) return out @@ -1046,9 +1050,12 @@ def partial_sum(input, start_index=0, length=-1): out = exe.run(feed={"x":xx, "y":yy}, fetch_list=[sum]) """ for id, x in enumerate(input): - check_variable_and_dtype(x, 'input[' + str(id) + ']', - ['float32', 'float64', 'int32', 'int64'], - 'partial_sum') + check_variable_and_dtype( + x, + 'input[' + str(id) + ']', + ['float32', 'float64', 'int32', 'int64'], + 'partial_sum', + ) inputs = {'X': input} attrs = {} @@ -1056,22 +1063,23 @@ def partial_sum(input, start_index=0, length=-1): attrs['length'] = length helper = LayerHelper('partial_sum', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='partial_sum', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='partial_sum', inputs=inputs, outputs={'Out': [out]}, attrs=attrs + ) return out -def sparse_embedding(input, - size, - padding_idx=None, - is_test=False, - entry=None, - table_class="MemorySparseTable", - param_attr=None, - dtype='float32', - slot=None): +def sparse_embedding( + input, + size, + padding_idx=None, + is_test=False, + entry=None, + table_class="MemorySparseTable", + param_attr=None, + dtype='float32', + slot=None, +): r""" :api_attr: Static Graph @@ -1183,25 +1191,39 @@ def sparse_embedding(input, helper = LayerHelper('sparse_embedding', **locals()) - check_variable_and_dtype(input, 'input', ['int64'], - 'fluid.contrib.layers.sparse_embedding') + check_variable_and_dtype( + input, 'input', ['int64'], 'fluid.contrib.layers.sparse_embedding' + ) - check_dtype(dtype, 'dtype', ['float32', 'float64'], - 'paddle.static.nn.sparse_embedding') + check_dtype( + dtype, + 'dtype', + ['float32', 'float64'], + 'paddle.static.nn.sparse_embedding', + ) - w = helper.create_parameter(attr=helper.param_attr, - shape=size, - type=core.VarDesc.VarType.SELECTED_ROWS, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, + shape=size, + type=core.VarDesc.VarType.SELECTED_ROWS, + dtype=dtype, + is_bias=False, + ) tmp = helper.create_variable_for_type_inference(dtype) - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) if table_class not in [ - "CommonSparseTable", "SSDSparseTable", "MemorySparseTable" + "CommonSparseTable", + "SSDSparseTable", + "MemorySparseTable", ]: raise ValueError( "table_class must be in [CommonSparseTable, SSDSparseTable, MemorySparseTable]" @@ -1211,7 +1233,9 @@ def sparse_embedding(input, if entry is not None: if entry.__class__.__name__ not in [ - "ProbabilityEntry", "CountFilterEntry", "ShowClickEntry" + "ProbabilityEntry", + "CountFilterEntry", + "ShowClickEntry", ]: raise ValueError( "entry must be instance in [paddle.distributed.ProbabilityEntry, paddle.distributed.CountFilterEntry, paddle.distributed.ShowClickEntry]" @@ -1221,22 +1245,21 @@ def sparse_embedding(input, if slot == None: slot = 0 - helper.append_op(type='lookup_table', - inputs={ - 'Ids': input, - 'W': w - }, - outputs={'Out': tmp}, - attrs={ - 'padding_idx': padding_idx, - 'is_sparse': True, - 'is_distributed': True, - 'remote_prefetch': True, - 'is_test': is_test, - 'entry': entry_str, - 'table_class': table_class, - 'slot': slot - }) + helper.append_op( + type='lookup_table', + inputs={'Ids': input, 'W': w}, + outputs={'Out': tmp}, + attrs={ + 'padding_idx': padding_idx, + 'is_sparse': True, + 'is_distributed': True, + 'remote_prefetch': True, + 'is_test': is_test, + 'entry': entry_str, + 'table_class': table_class, + 'slot': slot, + }, + ) return tmp @@ -1298,48 +1321,46 @@ def tdm_child(x, node_nums, child_nums, param_attr=None, dtype='int32'): exe.run(fluid.default_startup_program()) xx = np.array([[2],[3]]).reshape((2,1)).astype("int32") child_res, leaf_mask_res = exe.run(feed={"x":xx}, fetch_list=[child, leaf_mask]) - """ + """ helper = LayerHelper("tdm_child", **locals()) - check_dtype(dtype, 'dtype', ['int32', 'int64'], - 'fluid.contrib.layers.tdm_child') + check_dtype( + dtype, 'dtype', ['int32', 'int64'], 'fluid.contrib.layers.tdm_child' + ) c_dtype = convert_np_dtype_to_dtype_(dtype) - tree_info = helper.create_parameter(attr=helper.param_attr, - shape=[node_nums, 3 + child_nums], - dtype=dtype, - default_initializer=Constant(0)) + tree_info = helper.create_parameter( + attr=helper.param_attr, + shape=[node_nums, 3 + child_nums], + dtype=dtype, + default_initializer=Constant(0), + ) tree_info.stop_gradient = True child = helper.create_variable_for_type_inference(dtype=dtype) leaf_mask = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='tdm_child', - inputs={ - 'X': x, - 'TreeInfo': tree_info - }, - outputs={ - 'Child': child, - 'LeafMask': leaf_mask - }, - attrs={ - 'child_nums': child_nums, - 'dtype': c_dtype - }, - stop_gradient=True) + helper.append_op( + type='tdm_child', + inputs={'X': x, 'TreeInfo': tree_info}, + outputs={'Child': child, 'LeafMask': leaf_mask}, + attrs={'child_nums': child_nums, 'dtype': c_dtype}, + stop_gradient=True, + ) return (child, leaf_mask) -def tdm_sampler(x, - neg_samples_num_list, - layer_node_num_list, - leaf_node_num, - tree_travel_attr=None, - tree_layer_attr=None, - output_positive=True, - output_list=True, - seed=0, - tree_dtype='int32', - dtype='int32'): +def tdm_sampler( + x, + neg_samples_num_list, + layer_node_num_list, + leaf_node_num, + tree_travel_attr=None, + tree_layer_attr=None, + output_positive=True, + output_list=True, + seed=0, + tree_dtype='int32', + dtype='int32', +): """ **Tdm Sampler** According to the input positive samples at leaf node(x), do negative sampling layer by layer on the given tree. @@ -1426,18 +1447,25 @@ def tdm_sampler(x, """ helper = LayerHelper("tdm_sampler", **locals()) - check_dtype(tree_dtype, 'tree_dtype', ['int32', 'int64'], - 'fluid.contrib.layers.tdm_sampler') - check_dtype(dtype, 'dtype', ['int32', 'int64'], - 'fluid.contrib.layers.tdm_sampler') + check_dtype( + tree_dtype, + 'tree_dtype', + ['int32', 'int64'], + 'fluid.contrib.layers.tdm_sampler', + ) + check_dtype( + dtype, 'dtype', ['int32', 'int64'], 'fluid.contrib.layers.tdm_sampler' + ) c_dtype = convert_np_dtype_to_dtype_(dtype) if len(neg_samples_num_list) != len(layer_node_num_list): raise ValueError( "The shape of negative samples list must match the shape of layers. " "But received len of neg_samples_num_list: {}," - "and len of layer_node_num_list: {}, please check your input.". - format(len(neg_samples_num_list), len(layer_node_num_list))) + "and len of layer_node_num_list: {}, please check your input.".format( + len(neg_samples_num_list), len(layer_node_num_list) + ) + ) assert leaf_node_num is not None, "leaf_node_num should not be None here." layer_nums = 0 @@ -1452,21 +1480,31 @@ def tdm_sampler(x, "The number of negative samples must be less than the number of nodes " "in the layer {}, But received negative nums {}, and num of node at layer {} " "is {}, please check your input.".format( - layer_idx, neg_samples_num_list[layer_idx], layer_idx, - layer_node_num_list[layer_idx])) - assert leaf_node_num < node_nums, "leaf_node_num must be less than total node nums." + layer_idx, + neg_samples_num_list[layer_idx], + layer_idx, + layer_node_num_list[layer_idx], + ) + ) + assert ( + leaf_node_num < node_nums + ), "leaf_node_num must be less than total node nums." travel_shape = [leaf_node_num, layer_nums] - travel = helper.create_parameter(attr=tree_travel_attr, - shape=travel_shape, - dtype=tree_dtype, - default_initializer=Constant(0)) + travel = helper.create_parameter( + attr=tree_travel_attr, + shape=travel_shape, + dtype=tree_dtype, + default_initializer=Constant(0), + ) layer_shape = [node_nums, 1] - layer = helper.create_parameter(attr=tree_layer_attr, - shape=layer_shape, - dtype=tree_dtype, - default_initializer=Constant(0)) + layer = helper.create_parameter( + attr=tree_layer_attr, + shape=layer_shape, + dtype=tree_dtype, + default_initializer=Constant(0), + ) out = helper.create_variable_for_type_inference(dtype=dtype) out.stop_gradient = True @@ -1477,24 +1515,18 @@ def tdm_sampler(x, mask = helper.create_variable_for_type_inference(dtype=dtype) mask.stop_gradient = True - helper.append_op(type='tdm_sampler', - inputs={ - "X": x, - "Travel": travel, - "Layer": layer - }, - outputs={ - 'Out': out, - 'Labels': labels, - 'Mask': mask - }, - attrs={ - 'neg_samples_num_list': neg_samples_num_list, - 'output_positive': output_positive, - 'layer_offset_lod': tree_layer_offset_lod, - 'seed': seed, - 'dtype': c_dtype - }) + helper.append_op( + type='tdm_sampler', + inputs={"X": x, "Travel": travel, "Layer": layer}, + outputs={'Out': out, 'Labels': labels, 'Mask': mask}, + attrs={ + 'neg_samples_num_list': neg_samples_num_list, + 'output_positive': output_positive, + 'layer_offset_lod': tree_layer_offset_lod, + 'seed': seed, + 'dtype': c_dtype, + }, + ) if output_list: output_list = [] @@ -1506,31 +1538,30 @@ def tdm_sampler(x, positive_flag = 0 for layer_sample_num in neg_samples_num_list: - end_offset = start_offset + \ - layer_sample_num + positive_flag - layer_samples = slice(out, - axes=[1], - starts=[start_offset], - ends=[end_offset]) - layer_labels = slice(labels, - axes=[1], - starts=[start_offset], - ends=[end_offset]) - layer_mask = slice(mask, - axes=[1], - starts=[start_offset], - ends=[end_offset]) - - layer_samples = reshape(layer_samples, - [-1, layer_sample_num + positive_flag, 1]) + end_offset = start_offset + layer_sample_num + positive_flag + layer_samples = slice( + out, axes=[1], starts=[start_offset], ends=[end_offset] + ) + layer_labels = slice( + labels, axes=[1], starts=[start_offset], ends=[end_offset] + ) + layer_mask = slice( + mask, axes=[1], starts=[start_offset], ends=[end_offset] + ) + + layer_samples = reshape( + layer_samples, [-1, layer_sample_num + positive_flag, 1] + ) layer_samples.stop_gradient = True - layer_labels = reshape(layer_labels, - [-1, layer_sample_num + positive_flag, 1]) + layer_labels = reshape( + layer_labels, [-1, layer_sample_num + positive_flag, 1] + ) layer_labels.stop_gradient = True - layer_mask = reshape(layer_mask, - [-1, layer_sample_num + positive_flag, 1]) + layer_mask = reshape( + layer_mask, [-1, layer_sample_num + positive_flag, 1] + ) layer_mask.stop_gradient = True output_list.append(layer_samples) @@ -1545,12 +1576,14 @@ def tdm_sampler(x, return (out, labels, mask) -def rank_attention(input, - rank_offset, - rank_param_shape, - rank_param_attr, - max_rank=3, - max_size=0): +def rank_attention( + input, + rank_offset, + rank_param_shape, + rank_param_attr, + max_rank=3, + max_size=0, +): """ **Rank Attention layer** This Op can calculate rank attention between input and rank_param, and @@ -1588,32 +1621,25 @@ def rank_attention(input, input_shape = input.shape assert input_shape[1] * max_rank * max_rank == rank_param_shape[0] - rank_param = helper.create_parameter(attr=rank_param_attr, - shape=rank_param_shape, - dtype=dtype) + rank_param = helper.create_parameter( + attr=rank_param_attr, shape=rank_param_shape, dtype=dtype + ) rank_param.stop_gradient = False output = helper.create_variable_for_type_inference(dtype) - input_help = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) - ins_rank = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) - - helper.append_op(type="rank_attention", - inputs={ - "X": input, - "RankOffset": rank_offset, - "RankParam": rank_param - }, - outputs={ - "Out": output, - "InputHelp": input_help, - "InsRank": ins_rank - }, - attrs={ - "MaxRank": max_rank, - "MaxSize": max_size - }) + input_help = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) + ins_rank = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) + + helper.append_op( + type="rank_attention", + inputs={"X": input, "RankOffset": rank_offset, "RankParam": rank_param}, + outputs={"Out": output, "InputHelp": input_help, "InsRank": ins_rank}, + attrs={"MaxRank": max_rank, "MaxSize": max_size}, + ) return output @@ -1666,22 +1692,18 @@ def batch_fc(input, param_size, param_attr, bias_size, bias_attr, act=None): dtype = helper.input_dtype() check_dtype(dtype, 'input', ['float32', 'float64'], 'batch_fc') - w = helper.create_parameter(attr=param_attr, - shape=param_size, - dtype=dtype, - is_bias=False) - b = helper.create_parameter(attr=bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=param_attr, shape=param_size, dtype=dtype, is_bias=False + ) + b = helper.create_parameter( + attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=False + ) pre_act = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="batch_fc", - inputs={ - "Input": input, - "W": w, - "Bias": b - }, - outputs={"Out": pre_act}) + helper.append_op( + type="batch_fc", + inputs={"Input": input, "W": w, "Bias": b}, + outputs={"Out": pre_act}, + ) return helper.append_activation(pre_act) @@ -1720,16 +1742,12 @@ def _pull_box_extended_sparse(input, size, extend_size=64, dtype='float32'): helper.create_variable_for_type_inference(dtype) for i in range(len(inputs)) ] - helper.append_op(type='pull_box_extended_sparse', - inputs={'Ids': inputs}, - outputs={ - 'Out': outs, - 'OutExtend': outs_extend - }, - attrs={ - 'emb_size': size, - 'emb_extended_size': extend_size - }) + helper.append_op( + type='pull_box_extended_sparse', + inputs={'Ids': inputs}, + outputs={'Out': outs, 'OutExtend': outs_extend}, + attrs={'emb_size': size, 'emb_extended_size': extend_size}, + ) if len(outs) == 1: return outs[0], outs_extend[0] return outs, outs_extend @@ -1738,8 +1756,8 @@ def _pull_box_extended_sparse(input, size, extend_size=64, dtype='float32'): def bilateral_slice(x, guide, grid, has_offset, name=None): """ :alias_main: paddle.nn.functional.bilateral_slice - :alias: paddle.nn.functional.bilateral_slice,paddle.nn.functional.vision.bilateral_slice - :old_api: paddle.fluid.layers.bilateral_slice + :alias: paddle.nn.functional.bilateral_slice,paddle.nn.functional.vision.bilateral_slice + :old_api: paddle.fluid.layers.bilateral_slice This operation implements bilateral slicing on the input according to the guide map. For more information of bilateral slicing, please refer to Deep Bilateral Learning for Real-Time Image Enhancement _ @@ -1783,28 +1801,34 @@ def bilateral_slice(x, guide, grid, has_offset, name=None): return getattr(_legacy_C_ops, "bilateral_slice")(x, grid, guide, *attrs) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'bilateral_slice') - check_variable_and_dtype(guide, 'guide', ['float32', 'float64'], - 'bilateral_slice') - check_variable_and_dtype(grid, 'grid', ['float32', 'float64'], - 'bilateral_slice') + check_variable_and_dtype( + guide, 'guide', ['float32', 'float64'], 'bilateral_slice' + ) + check_variable_and_dtype( + grid, 'grid', ['float32', 'float64'], 'bilateral_slice' + ) helper = LayerHelper("bilateral_slice", **locals()) out = helper.create_variable_for_type_inference(x.dtype) inputs = {'X': x, 'Guide': guide, 'Grid': grid} - helper.append_op(type='bilateral_slice', - inputs=inputs, - attrs={'has_offset': has_offset}, - outputs={'Out': out}) + helper.append_op( + type='bilateral_slice', + inputs=inputs, + attrs={'has_offset': has_offset}, + outputs={'Out': out}, + ) return out -def correlation(x, - y, - pad_size, - kernel_size, - max_displacement, - stride1, - stride2, - corr_type_multiply=1): +def correlation( + x, + y, + pad_size, + kernel_size, + max_displacement, + stride1, + stride2, + corr_type_multiply=1, +): """ This operation compute correlation of two tensor. @@ -1852,40 +1876,52 @@ def correlation(x, """ if paddle.fluid._non_static_mode(): - attrs = ("pad_size", pad_size, "kernel_size", kernel_size, - "max_displacement", max_displacement, "stride1", stride1, - "stride2", stride2, "corr_type_multiply", corr_type_multiply) + attrs = ( + "pad_size", + pad_size, + "kernel_size", + kernel_size, + "max_displacement", + max_displacement, + "stride1", + stride1, + "stride2", + stride2, + "corr_type_multiply", + corr_type_multiply, + ) output = getattr(_legacy_C_ops, "correlation")(x, y, *attrs) else: helper = LayerHelper("correlation", **locals()) output = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="correlation", - inputs={ - "Input1": x, - "Input2": y - }, - attrs={ - "pad_size": pad_size, - "kernel_size": kernel_size, - "max_displacement": max_displacement, - "stride1": stride1, - "stride2": stride2, - "corr_type_multiply": corr_type_multiply - }, - outputs={"Output": output}) + helper.append_op( + type="correlation", + inputs={"Input1": x, "Input2": y}, + attrs={ + "pad_size": pad_size, + "kernel_size": kernel_size, + "max_displacement": max_displacement, + "stride1": stride1, + "stride2": stride2, + "corr_type_multiply": corr_type_multiply, + }, + outputs={"Output": output}, + ) return output -def fused_bn_add_act(x, - y, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - moving_mean_name=None, - moving_variance_name=None, - act=None, - name=None): +def fused_bn_add_act( + x, + y, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + moving_mean_name=None, + moving_variance_name=None, + act=None, + name=None, +): r""" This Op performs batch norm on input x, and adds the result to input y. Then it performs activation on the sum. The data format of inputs must be NHWC @@ -1906,14 +1942,14 @@ def fused_bn_add_act(x, numerical stability. Default is 1e-5. param_attr(ParamAttr, optional): The parameter attribute for Parameter `scale` of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm - will create ParamAttr as param_attr, the name of scale can be set in ParamAttr. - If the Initializer of the param_attr is not set, the parameter is initialized - with Xavier. Default: None. + will create ParamAttr as param_attr, the name of scale can be set in ParamAttr. + If the Initializer of the param_attr is not set, the parameter is initialized + with Xavier. Default: None. bias_attr(ParamAttr, optional): The parameter attribute for the bias of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm - will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. - If the Initializer of the bias_attr is not set, the bias is initialized zero. - Default: None. + will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. + If the Initializer of the bias_attr is not set, the bias is initialized zero. + Default: None. moving_mean_name(str, optional): The name of moving_mean which store the global Mean. If it is set to None, batch_norm will save global mean with a random name, otherwise, batch_norm will save global mean with the string. @@ -1988,10 +2024,12 @@ def fused_bn_add_act(x, """ helper = LayerHelper('fused_bn_add_act', **locals()) - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'fused_bn_add_act') - check_variable_and_dtype(y, 'input', ['float16', 'float32', 'float64'], - 'fused_bn_add_act') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'fused_bn_add_act' + ) + check_variable_and_dtype( + y, 'input', ['float16', 'float32', 'float64'], 'fused_bn_add_act' + ) bn_param_dtype = core.VarDesc.VarType.FP32 x_shape = x.shape @@ -1999,25 +2037,35 @@ def fused_bn_add_act(x, param_shape = [channel_num] # create parameter - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=bn_param_dtype, - default_initializer=Constant(1.0)) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=bn_param_dtype, - is_bias=True) - mean = helper.create_parameter(attr=ParamAttr(name=moving_mean_name, - initializer=Constant(0.0), - trainable=False), - shape=param_shape, - dtype=bn_param_dtype) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=bn_param_dtype, + default_initializer=Constant(1.0), + ) + bias = helper.create_parameter( + attr=helper.bias_attr, + shape=param_shape, + dtype=bn_param_dtype, + is_bias=True, + ) + mean = helper.create_parameter( + attr=ParamAttr( + name=moving_mean_name, initializer=Constant(0.0), trainable=False + ), + shape=param_shape, + dtype=bn_param_dtype, + ) mean.stop_gradient = True - variance = helper.create_parameter(attr=ParamAttr(name=moving_variance_name, - initializer=Constant(1.0), - trainable=False), - shape=param_shape, - dtype=bn_param_dtype) + variance = helper.create_parameter( + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + ), + shape=param_shape, + dtype=bn_param_dtype, + ) variance.stop_gradient = True # create output @@ -2025,14 +2073,18 @@ def fused_bn_add_act(x, mean_out = mean # variance and variance out share the same memory variance_out = variance - saved_mean = helper.create_variable_for_type_inference(dtype=bn_param_dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) reserve_space = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.FP16, stop_gradient=True) + dtype=core.VarDesc.VarType.FP16, stop_gradient=True + ) batch_norm_out = helper.create_variable_for_type_inference( - core.VarDesc.VarType.FP16) + core.VarDesc.VarType.FP16 + ) inputs = { "X": x, @@ -2048,51 +2100,50 @@ def fused_bn_add_act(x, "VarianceOut": variance_out, "SavedMean": saved_mean, "SavedVariance": saved_variance, - "ReserveSpace": reserve_space + "ReserveSpace": reserve_space, } - helper.append_op(type="fused_bn_add_activation", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="fused_bn_add_activation", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) return batch_norm_out -def pow2_decay_with_linear_warmup(warmup_steps, - total_steps, - base_lr, - end_lr, - dtype='float32', - name=None): +def pow2_decay_with_linear_warmup( + warmup_steps, total_steps, base_lr, end_lr, dtype='float32', name=None +): if paddle.fluid._non_static_mode(): raise NotImplementedError( - "pow2_decay_with_linear_warmup does not support dygraph mode yet.") + "pow2_decay_with_linear_warmup does not support dygraph mode yet." + ) helper = LayerHelper("pow2_decay_with_linear_warmup", **locals()) lr = helper.create_global_variable(persistable=True, dtype=dtype, shape=[1]) helper.set_variable_initializer( - lr, Constant(value=float(base_lr) / warmup_steps)) + lr, Constant(value=float(base_lr) / warmup_steps) + ) - step = helper.create_global_variable(persistable=True, - dtype='int64', - shape=[1]) + step = helper.create_global_variable( + persistable=True, dtype='int64', shape=[1] + ) helper.set_variable_initializer(step, Constant(value=0)) - assert warmup_steps <= total_steps, "warmup_steps cannot be larger than total_steps" - - helper.append_op(type="pow2_decay_with_linear_warmup", - inputs={ - "LearningRate": lr, - "Step": step - }, - outputs={ - "LearningRateOut": lr, - "StepOut": step - }, - attrs={ - "warmup_steps": warmup_steps, - "total_steps": total_steps, - "base_lr": base_lr, - "end_lr": end_lr, - }) + assert ( + warmup_steps <= total_steps + ), "warmup_steps cannot be larger than total_steps" + + helper.append_op( + type="pow2_decay_with_linear_warmup", + inputs={"LearningRate": lr, "Step": step}, + outputs={"LearningRateOut": lr, "StepOut": step}, + attrs={ + "warmup_steps": warmup_steps, + "total_steps": total_steps, + "base_lr": base_lr, + "end_lr": end_lr, + }, + ) return lr diff --git a/python/paddle/fluid/contrib/layers/rnn_impl.py b/python/paddle/fluid/contrib/layers/rnn_impl.py index 95ce0a6ba53..668ce445f78 100644 --- a/python/paddle/fluid/contrib/layers/rnn_impl.py +++ b/python/paddle/fluid/contrib/layers/rnn_impl.py @@ -74,18 +74,21 @@ class BasicGRUUnit(Layer): """ - def __init__(self, - name_scope, - hidden_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - dtype='float32'): + def __init__( + self, + name_scope, + hidden_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + dtype='float32', + ): super(BasicGRUUnit, self).__init__(name_scope, dtype) # reserve old school _full_name and _helper for static graph save load - self._full_name = unique_name.generate(name_scope + "/" + - self.__class__.__name__) + self._full_name = unique_name.generate( + name_scope + "/" + self.__class__.__name__ + ) self._helper = LayerObjectHelper(self._full_name) self._name = name_scope @@ -98,7 +101,7 @@ class BasicGRUUnit(Layer): def _build_once(self, input, pre_hidden): self._input_size = input.shape[-1] - assert (self._input_size > 0) + assert self._input_size > 0 if self._param_attr is not None and self._param_attr.name is not None: gate_param_attr = copy.deepcopy(self._param_attr) @@ -112,12 +115,14 @@ class BasicGRUUnit(Layer): self._gate_weight = self.create_parameter( attr=gate_param_attr, shape=[self._input_size + self._hiden_size, 2 * self._hiden_size], - dtype=self._dtype) + dtype=self._dtype, + ) self._candidate_weight = self.create_parameter( attr=candidate_param_attr, shape=[self._input_size + self._hiden_size, self._hiden_size], - dtype=self._dtype) + dtype=self._dtype, + ) if self._bias_attr is not None and self._bias_attr.name is not None: gate_bias_attr = copy.deepcopy(self._bias_attr) @@ -128,14 +133,18 @@ class BasicGRUUnit(Layer): gate_bias_attr = self._bias_attr candidate_bias_attr = self._bias_attr - self._gate_bias = self.create_parameter(attr=gate_bias_attr, - shape=[2 * self._hiden_size], - dtype=self._dtype, - is_bias=True) - self._candidate_bias = self.create_parameter(attr=candidate_bias_attr, - shape=[self._hiden_size], - dtype=self._dtype, - is_bias=True) + self._gate_bias = self.create_parameter( + attr=gate_bias_attr, + shape=[2 * self._hiden_size], + dtype=self._dtype, + is_bias=True, + ) + self._candidate_bias = self.create_parameter( + attr=candidate_bias_attr, + shape=[self._hiden_size], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input, pre_hidden): concat_input_hidden = layers.concat([input, pre_hidden], 1) @@ -149,8 +158,9 @@ class BasicGRUUnit(Layer): r_hidden = r * pre_hidden - candidate = layers.matmul(layers.concat([input, r_hidden], 1), - self._candidate_weight) + candidate = layers.matmul( + layers.concat([input, r_hidden], 1), self._candidate_weight + ) candidate = layers.elementwise_add(candidate, self._candidate_bias) c = self._activation(candidate) @@ -159,20 +169,22 @@ class BasicGRUUnit(Layer): return new_hidden -def basic_gru(input, - init_hidden, - hidden_size, - num_layers=1, - sequence_length=None, - dropout_prob=0.0, - bidirectional=False, - batch_first=True, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - dtype='float32', - name='basic_gru'): +def basic_gru( + input, + init_hidden, + hidden_size, + num_layers=1, + sequence_length=None, + dropout_prob=0.0, + bidirectional=False, + batch_first=True, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + dtype='float32', + name='basic_gru', +): r""" GRU implementation using basic operator, supports multiple layers and bidirectional gru. @@ -273,8 +285,16 @@ def basic_gru(input, else: layer_bias_attr = bias_attr fw_unit_list.append( - BasicGRUUnit(new_name, hidden_size, layer_param_attr, - layer_bias_attr, gate_activation, activation, dtype)) + BasicGRUUnit( + new_name, + hidden_size, + layer_param_attr, + layer_bias_attr, + gate_activation, + activation, + dtype, + ) + ) if bidirectional: bw_unit_list = [] @@ -292,9 +312,16 @@ def basic_gru(input, layer_bias_attr = bias_attr bw_unit_list.append( - BasicGRUUnit(new_name, hidden_size, layer_param_attr, - layer_bias_attr, gate_activation, activation, - dtype)) + BasicGRUUnit( + new_name, + hidden_size, + layer_param_attr, + layer_bias_attr, + gate_activation, + activation, + dtype, + ) + ) if batch_first: input = layers.transpose(input, [1, 0, 2]) @@ -302,9 +329,9 @@ def basic_gru(input, mask = None if sequence_length: max_seq_len = layers.shape(input)[0] - mask = layers.sequence_mask(sequence_length, - maxlen=max_seq_len, - dtype='float32') + mask = layers.sequence_mask( + sequence_length, maxlen=max_seq_len, dtype='float32' + ) mask = layers.transpose(mask, [1, 0]) direc_num = 1 @@ -312,12 +339,12 @@ def basic_gru(input, direc_num = 2 if init_hidden: init_hidden = layers.reshape( - init_hidden, shape=[num_layers, direc_num, -1, hidden_size]) + init_hidden, shape=[num_layers, direc_num, -1, hidden_size] + ) - def get_single_direction_output(rnn_input, - unit_list, - mask=None, - direc_index=0): + def get_single_direction_output( + rnn_input, unit_list, mask=None, direc_index=0 + ): rnn = StaticRNN() with rnn.step(): step_input = rnn.step_input(rnn_input) @@ -329,16 +356,20 @@ def basic_gru(input, if init_hidden: pre_hidden = rnn.memory(init=init_hidden[i, direc_index]) else: - pre_hidden = rnn.memory(batch_ref=rnn_input, - shape=[-1, hidden_size], - ref_batch_dim_idx=1) + pre_hidden = rnn.memory( + batch_ref=rnn_input, + shape=[-1, hidden_size], + ref_batch_dim_idx=1, + ) new_hidden = unit_list[i](step_input, pre_hidden) if mask: new_hidden = layers.elementwise_mul( - new_hidden, step_mask, axis=0) - layers.elementwise_mul( - pre_hidden, (step_mask - 1), axis=0) + new_hidden, step_mask, axis=0 + ) - layers.elementwise_mul( + pre_hidden, (step_mask - 1), axis=0 + ) rnn.update_memory(pre_hidden, new_hidden) rnn.step_output(new_hidden) @@ -362,26 +393,25 @@ def basic_gru(input, last_hidden_array.append(last_hidden) last_hidden_output = layers.concat(last_hidden_array, axis=0) - last_hidden_output = layers.reshape(last_hidden_output, - shape=[num_layers, -1, hidden_size]) + last_hidden_output = layers.reshape( + last_hidden_output, shape=[num_layers, -1, hidden_size] + ) return rnn_output, last_hidden_output # seq_len, batch_size, hidden_size - fw_rnn_out, fw_last_hidden = get_single_direction_output(input, - fw_unit_list, - mask, - direc_index=0) + fw_rnn_out, fw_last_hidden = get_single_direction_output( + input, fw_unit_list, mask, direc_index=0 + ) if bidirectional: bw_input = layers.reverse(input, axis=[0]) bw_mask = None if mask: bw_mask = layers.reverse(mask, axis=[0]) - bw_rnn_out, bw_last_hidden = get_single_direction_output(bw_input, - bw_unit_list, - bw_mask, - direc_index=1) + bw_rnn_out, bw_last_hidden = get_single_direction_output( + bw_input, bw_unit_list, bw_mask, direc_index=1 + ) bw_rnn_out = layers.reverse(bw_rnn_out, axis=[0]) @@ -389,7 +419,8 @@ def basic_gru(input, last_hidden = layers.concat([fw_last_hidden, bw_last_hidden], axis=1) last_hidden = layers.reshape( - last_hidden, shape=[num_layers * direc_num, -1, hidden_size]) + last_hidden, shape=[num_layers * direc_num, -1, hidden_size] + ) if batch_first: rnn_out = layers.transpose(rnn_out, [1, 0, 2]) @@ -405,22 +436,24 @@ def basic_gru(input, return rnn_out, last_hidden -def basic_lstm(input, - init_hidden, - init_cell, - hidden_size, - num_layers=1, - sequence_length=None, - dropout_prob=0.0, - bidirectional=False, - batch_first=True, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - dtype='float32', - name='basic_lstm'): +def basic_lstm( + input, + init_hidden, + init_cell, + hidden_size, + num_layers=1, + sequence_length=None, + dropout_prob=0.0, + bidirectional=False, + batch_first=True, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + dtype='float32', + name='basic_lstm', +): r""" LSTM implementation using basic operators, supports multiple layers and bidirectional LSTM. @@ -535,14 +568,17 @@ def basic_lstm(input, else: layer_bias_attr = bias_attr fw_unit_list.append( - BasicLSTMUnit(new_name, - hidden_size, - param_attr=layer_param_attr, - bias_attr=layer_bias_attr, - gate_activation=gate_activation, - activation=activation, - forget_bias=forget_bias, - dtype=dtype)) + BasicLSTMUnit( + new_name, + hidden_size, + param_attr=layer_param_attr, + bias_attr=layer_bias_attr, + gate_activation=gate_activation, + activation=activation, + forget_bias=forget_bias, + dtype=dtype, + ) + ) if bidirectional: bw_unit_list = [] @@ -559,14 +595,17 @@ def basic_lstm(input, else: layer_bias_attr = param_attr bw_unit_list.append( - BasicLSTMUnit(new_name, - hidden_size, - param_attr=layer_param_attr, - bias_attr=layer_bias_attr, - gate_activation=gate_activation, - activation=activation, - forget_bias=forget_bias, - dtype=dtype)) + BasicLSTMUnit( + new_name, + hidden_size, + param_attr=layer_param_attr, + bias_attr=layer_bias_attr, + gate_activation=gate_activation, + activation=activation, + forget_bias=forget_bias, + dtype=dtype, + ) + ) if batch_first: input = layers.transpose(input, [1, 0, 2]) @@ -574,9 +613,9 @@ def basic_lstm(input, mask = None if sequence_length: max_seq_len = layers.shape(input)[0] - mask = layers.sequence_mask(sequence_length, - maxlen=max_seq_len, - dtype='float32') + mask = layers.sequence_mask( + sequence_length, maxlen=max_seq_len, dtype='float32' + ) mask = layers.transpose(mask, [1, 0]) @@ -586,15 +625,16 @@ def basic_lstm(input, # convert to [num_layers, 2, batch_size, hidden_size] if init_hidden: init_hidden = layers.reshape( - init_hidden, shape=[num_layers, direc_num, -1, hidden_size]) + init_hidden, shape=[num_layers, direc_num, -1, hidden_size] + ) init_cell = layers.reshape( - init_cell, shape=[num_layers, direc_num, -1, hidden_size]) + init_cell, shape=[num_layers, direc_num, -1, hidden_size] + ) # forward direction - def get_single_direction_output(rnn_input, - unit_list, - mask=None, - direc_index=0): + def get_single_direction_output( + rnn_input, unit_list, mask=None, direc_index=0 + ): rnn = StaticRNN() with rnn.step(): step_input = rnn.step_input(rnn_input) @@ -607,21 +647,28 @@ def basic_lstm(input, pre_hidden = rnn.memory(init=init_hidden[i, direc_index]) pre_cell = rnn.memory(init=init_cell[i, direc_index]) else: - pre_hidden = rnn.memory(batch_ref=rnn_input, - shape=[-1, hidden_size]) - pre_cell = rnn.memory(batch_ref=rnn_input, - shape=[-1, hidden_size]) + pre_hidden = rnn.memory( + batch_ref=rnn_input, shape=[-1, hidden_size] + ) + pre_cell = rnn.memory( + batch_ref=rnn_input, shape=[-1, hidden_size] + ) - new_hidden, new_cell = unit_list[i](step_input, pre_hidden, - pre_cell) + new_hidden, new_cell = unit_list[i]( + step_input, pre_hidden, pre_cell + ) if mask: new_hidden = layers.elementwise_mul( - new_hidden, step_mask, axis=0) - layers.elementwise_mul( - pre_hidden, (step_mask - 1), axis=0) + new_hidden, step_mask, axis=0 + ) - layers.elementwise_mul( + pre_hidden, (step_mask - 1), axis=0 + ) new_cell = layers.elementwise_mul( - new_cell, step_mask, axis=0) - layers.elementwise_mul( - pre_cell, (step_mask - 1), axis=0) + new_cell, step_mask, axis=0 + ) - layers.elementwise_mul( + pre_cell, (step_mask - 1), axis=0 + ) rnn.update_memory(pre_hidden, new_hidden) rnn.update_memory(pre_cell, new_cell) @@ -634,7 +681,8 @@ def basic_lstm(input, step_input = layers.dropout( step_input, dropout_prob=dropout_prob, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn.step_output(step_input) @@ -652,17 +700,20 @@ def basic_lstm(input, last_cell_array.append(last_cell) last_hidden_output = layers.concat(last_hidden_array, axis=0) - last_hidden_output = layers.reshape(last_hidden_output, - shape=[num_layers, -1, hidden_size]) + last_hidden_output = layers.reshape( + last_hidden_output, shape=[num_layers, -1, hidden_size] + ) last_cell_output = layers.concat(last_cell_array, axis=0) - last_cell_output = layers.reshape(last_cell_output, - shape=[num_layers, -1, hidden_size]) + last_cell_output = layers.reshape( + last_cell_output, shape=[num_layers, -1, hidden_size] + ) return rnn_output, last_hidden_output, last_cell_output # seq_len, batch_size, hidden_size fw_rnn_out, fw_last_hidden, fw_last_cell = get_single_direction_output( - input, fw_unit_list, mask, direc_index=0) + input, fw_unit_list, mask, direc_index=0 + ) if bidirectional: bw_input = layers.reverse(input, axis=[0]) @@ -670,18 +721,21 @@ def basic_lstm(input, if mask: bw_mask = layers.reverse(mask, axis=[0]) bw_rnn_out, bw_last_hidden, bw_last_cell = get_single_direction_output( - bw_input, bw_unit_list, bw_mask, direc_index=1) + bw_input, bw_unit_list, bw_mask, direc_index=1 + ) bw_rnn_out = layers.reverse(bw_rnn_out, axis=[0]) rnn_out = layers.concat([fw_rnn_out, bw_rnn_out], axis=2) last_hidden = layers.concat([fw_last_hidden, bw_last_hidden], axis=1) last_hidden = layers.reshape( - last_hidden, shape=[num_layers * direc_num, -1, hidden_size]) + last_hidden, shape=[num_layers * direc_num, -1, hidden_size] + ) last_cell = layers.concat([fw_last_cell, bw_last_cell], axis=1) last_cell = layers.reshape( - last_cell, shape=[num_layers * direc_num, -1, hidden_size]) + last_cell, shape=[num_layers * direc_num, -1, hidden_size] + ) if batch_first: rnn_out = layers.transpose(rnn_out, [1, 0, 2]) @@ -769,19 +823,22 @@ class BasicLSTMUnit(Layer): """ - def __init__(self, - name_scope, - hidden_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - dtype='float32'): + def __init__( + self, + name_scope, + hidden_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + dtype='float32', + ): super(BasicLSTMUnit, self).__init__(name_scope, dtype) # reserve old school _full_name and _helper for static graph save load - self._full_name = unique_name.generate(name_scope + "/" + - self.__class__.__name__) + self._full_name = unique_name.generate( + name_scope + "/" + self.__class__.__name__ + ) self._helper = LayerObjectHelper(self._full_name) self._name = name_scope @@ -790,25 +847,28 @@ class BasicLSTMUnit(Layer): self._bias_attr = bias_attr self._gate_activation = gate_activation or layers.sigmoid self._activation = activation or layers.tanh - self._forget_bias = layers.fill_constant([1], - dtype=dtype, - value=forget_bias) + self._forget_bias = layers.fill_constant( + [1], dtype=dtype, value=forget_bias + ) self._forget_bias.stop_gradient = False self._dtype = dtype def _build_once(self, input, pre_hidden, pre_cell): self._input_size = input.shape[-1] - assert (self._input_size > 0) + assert self._input_size > 0 self._weight = self.create_parameter( attr=self._param_attr, shape=[self._input_size + self._hiden_size, 4 * self._hiden_size], - dtype=self._dtype) + dtype=self._dtype, + ) - self._bias = self.create_parameter(attr=self._bias_attr, - shape=[4 * self._hiden_size], - dtype=self._dtype, - is_bias=True) + self._bias = self.create_parameter( + attr=self._bias_attr, + shape=[4 * self._hiden_size], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input, pre_hidden, pre_cell): concat_input_hidden = layers.concat([input, pre_hidden], 1) @@ -819,8 +879,10 @@ class BasicLSTMUnit(Layer): new_cell = layers.elementwise_add( layers.elementwise_mul( pre_cell, - layers.sigmoid(layers.elementwise_add(f, self._forget_bias))), - layers.elementwise_mul(layers.sigmoid(i), layers.tanh(j))) + layers.sigmoid(layers.elementwise_add(f, self._forget_bias)), + ), + layers.elementwise_mul(layers.sigmoid(i), layers.tanh(j)), + ) new_hidden = layers.tanh(new_cell) * layers.sigmoid(o) return new_hidden, new_cell diff --git a/python/paddle/fluid/contrib/memory_usage_calc.py b/python/paddle/fluid/contrib/memory_usage_calc.py index 607f8b45da1..8a8dcb55a00 100644 --- a/python/paddle/fluid/contrib/memory_usage_calc.py +++ b/python/paddle/fluid/contrib/memory_usage_calc.py @@ -66,7 +66,8 @@ def memory_usage(program, batch_size): if not isinstance(program, Program): raise TypeError( "Calculating Memory Usage requires Program as its Parameter." - "But you passed in %s" % (type(program))) + "But you passed in %s" % (type(program)) + ) if batch_size <= 0: raise ValueError("The batch size need to be positive.") @@ -88,8 +89,9 @@ def memory_usage(program, batch_size): if x < 0: if neg_dim_count >= 1: raise ValueError( - "Var %s has more than one negative dim." % - (var_name)) + "Var %s has more than one negative dim." + % (var_name) + ) neg_dim_count += 1 data_count *= batch_size * (-x) else: diff --git a/python/paddle/fluid/contrib/mixed_precision/amp_nn.py b/python/paddle/fluid/contrib/mixed_precision/amp_nn.py index f9552fa705c..f336c87a11e 100644 --- a/python/paddle/fluid/contrib/mixed_precision/amp_nn.py +++ b/python/paddle/fluid/contrib/mixed_precision/amp_nn.py @@ -39,8 +39,12 @@ def check_finite_and_unscale(x, scale, name=None, float_status=None): """ check_type(x, 'x', (tuple, list), 'check_finite_and_unscale') for e in x: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'check_finite_and_unscale') + check_variable_and_dtype( + e, + "x", + ['float16', 'float32', 'float64'], + 'check_finite_and_unscale', + ) helper = LayerHelper("check_finite_and_unscale", **locals()) @@ -52,29 +56,34 @@ def check_finite_and_unscale(x, scale, name=None, float_status=None): inputs = {'X': x, 'Scale': scale} if core.is_compiled_with_npu(): - check_variable_and_dtype(float_status, "float_status", - ['float16', 'float32'], - 'check_finite_and_unscale') + check_variable_and_dtype( + float_status, + "float_status", + ['float16', 'float32'], + 'check_finite_and_unscale', + ) inputs['FloatStatus'] = float_status outputs = {'Out': x, 'FoundInfinite': found_inf} - helper.append_op(type='check_finite_and_unscale', - inputs=inputs, - outputs=outputs) + helper.append_op( + type='check_finite_and_unscale', inputs=inputs, outputs=outputs + ) return x, found_inf -def update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - stop_update=False, - name=None): +def update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + stop_update=False, + name=None, +): """ Update loss scaling according to overall gradients. If all gradients is finite after incr_every_n_steps, loss scaling will increase by incr_ratio. @@ -102,23 +111,39 @@ def update_loss_scaling(x, loss scaling. """ - check_variable_and_dtype(prev_loss_scaling, "prev_loss_scaling", - ['float32', 'float64'], "update_loss_scaling") + check_variable_and_dtype( + prev_loss_scaling, + "prev_loss_scaling", + ['float32', 'float64'], + "update_loss_scaling", + ) check_type(x, 'x', (tuple, list), 'update_loss_scaling') for e in x: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'update_loss_scaling') + check_variable_and_dtype( + e, "x", ['float16', 'float32', 'float64'], 'update_loss_scaling' + ) if e.dtype == core.VarDesc.VarType.FP16: - assert prev_loss_scaling.dtype == core.VarDesc.VarType.FP32, \ - "The dtype of prev_loss_scaling should be float32 when the dtype of x is float16." + assert ( + prev_loss_scaling.dtype == core.VarDesc.VarType.FP32 + ), "The dtype of prev_loss_scaling should be float32 when the dtype of x is float16." else: - assert prev_loss_scaling.dtype == e.dtype, "The dtype of prev_loss_scaling should be equal to the dtype of x." + assert ( + prev_loss_scaling.dtype == e.dtype + ), "The dtype of prev_loss_scaling should be equal to the dtype of x." if in_dygraph_mode(): - _C_ops.update_loss_scaling_(x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps, - incr_every_n_steps, decr_every_n_nan_or_inf, - incr_ratio, decr_ratio, stop_update) + _C_ops.update_loss_scaling_( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + stop_update, + ) return x helper = LayerHelper("update_loss_scaling", **locals()) @@ -128,14 +153,14 @@ def update_loss_scaling(x, 'FoundInfinite': found_inf, 'PrevLossScaling': prev_loss_scaling, 'InGoodSteps': num_good_steps, - 'InBadSteps': num_bad_steps + 'InBadSteps': num_bad_steps, } outputs = { 'Out': x, 'LossScaling': prev_loss_scaling, 'OutGoodSteps': num_good_steps, - 'OutBadSteps': num_bad_steps + 'OutBadSteps': num_bad_steps, } attrs = { @@ -150,9 +175,8 @@ def update_loss_scaling(x, else: attrs['stop_update'] = stop_update - helper.append_op(type='update_loss_scaling', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='update_loss_scaling', inputs=inputs, outputs=outputs, attrs=attrs + ) return x diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py index 3f799809af9..33694f4d127 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py @@ -15,8 +15,11 @@ import copy from paddle.fluid import core -from ..fp16_lists import white_list as white_list_fp16, black_list as black_list_fp16,\ - gray_list as gray_list_fp16 +from ..fp16_lists import ( + white_list as white_list_fp16, + black_list as black_list_fp16, + gray_list as gray_list_fp16, +) __all__ = ["AutoMixedPrecisionListsBF16"] @@ -40,10 +43,12 @@ class AutoMixedPrecisionListsBF16(object): paddle.static.amp.AutoMixedPrecisionListsBF16(custom_fp32_list={'lstm'}) """ - def __init__(self, - custom_bf16_list=None, - custom_fp32_list=None, - custom_fp32_varnames=None): + def __init__( + self, + custom_bf16_list=None, + custom_fp32_list=None, + custom_fp32_varnames=None, + ): self._custom_bf16_list = custom_bf16_list self._custom_fp32_list = custom_fp32_list self.bf16_list = copy.copy(bf16_list) @@ -61,8 +66,9 @@ class AutoMixedPrecisionListsBF16(object): if self._custom_bf16_list and self._custom_fp32_list: for op_name in self._custom_bf16_list: if op_name in self._custom_fp32_list: - raise ValueError("Custom bf16 list overlap " - "custom fp32 list") + raise ValueError( + "Custom bf16 list overlap " "custom fp32 list" + ) if self._custom_bf16_list: for op_name in self._custom_bf16_list: if op_name in self.fp32_list: @@ -92,13 +98,28 @@ bf16_list = { # depends on the prev_op type gray_list = { - 'elementwise_add', 'elementwise_sub', 'elementwise_mul', 'elementwise_div', - 'relu', 'layer_norm', 'slice', 'concat', 'uniform_random', 'reshape2', - 'transpose2', 'pool2d', 'sigmoid', 'cast', 'scale', 'fill_constant', 'split' + 'elementwise_add', + 'elementwise_sub', + 'elementwise_mul', + 'elementwise_div', + 'relu', + 'layer_norm', + 'slice', + 'concat', + 'uniform_random', + 'reshape2', + 'transpose2', + 'pool2d', + 'sigmoid', + 'cast', + 'scale', + 'fill_constant', + 'split', } _, _, _sys_unsupported_bf16_list = core.op_supported_infos( - 'CPU', core.VarDesc.VarType.BF16) + 'CPU', core.VarDesc.VarType.BF16 +) unsupported_list = _sys_unsupported_bf16_list fp32_list = black_list_fp16.copy().copy() diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py index 94734a8088d..25468b563aa 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py @@ -19,8 +19,13 @@ from .... import global_scope from ....log_helper import get_logger from ....wrapped_decorator import signature_safe_contextmanager from .amp_lists import AutoMixedPrecisionListsBF16 -from ..fp16_utils import find_true_prev_op, find_true_post_op, _rename_arg, \ - find_op_index, _rename_op_input +from ..fp16_utils import ( + find_true_prev_op, + find_true_post_op, + _rename_arg, + find_op_index, + _rename_op_input, +) import collections import struct @@ -28,17 +33,21 @@ import logging import numpy as np __all__ = [ - "bf16_guard", "rewrite_program_bf16", "cast_model_to_bf16", - "cast_parameters_to_bf16", "convert_float_to_uint16" + "bf16_guard", + "rewrite_program_bf16", + "cast_model_to_bf16", + "cast_parameters_to_bf16", + "convert_float_to_uint16", ] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) _valid_types = [ - core.VarDesc.VarType.LOD_TENSOR, core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR_ARRAY + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, ] _bf16_guard_pattern = "__use_bf16__" @@ -48,7 +57,8 @@ def convert_float_to_uint16(in_list): in_list = np.asarray(in_list) out = np.vectorize( lambda x: struct.unpack('> 16, - otypes=[np.uint16])(in_list.flat) + otypes=[np.uint16], + )(in_list.flat) return np.reshape(out, in_list.shape) @@ -83,7 +93,9 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): for in_name in op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and op.type in [ - 'batch_norm', 'fused_bn_add_activation', 'layer_norm' + 'batch_norm', + 'fused_bn_add_activation', + 'layer_norm', ]: if in_name not in {'X', 'Z'}: continue @@ -99,26 +111,34 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): name=cast_name, dtype=dest_dtype, persistable=False, - stop_gradient=in_var.stop_gradient) - - block._insert_op(idx, - type="cast", - inputs={"X": in_var}, - outputs={"Out": out_var}, - attrs={ - "in_dtype": in_var.dtype, - "out_dtype": out_var.dtype - }) + stop_gradient=in_var.stop_gradient, + ) + + block._insert_op( + idx, + type="cast", + inputs={"X": in_var}, + outputs={"Out": out_var}, + attrs={ + "in_dtype": in_var.dtype, + "out_dtype": out_var.dtype, + }, + ) num_cast_ops += 1 _rename_arg(op, in_var.name, out_var.name) else: if op.has_attr('in_dtype'): op._set_attr('in_dtype', dest_dtype) - if src_dtype == core.VarDesc.VarType.FP32 and dest_dtype == core.VarDesc.VarType.BF16: + if ( + src_dtype == core.VarDesc.VarType.FP32 + and dest_dtype == core.VarDesc.VarType.BF16 + ): for out_name in op.output_names: - if op.type in [ - 'batch_norm', 'fused_bn_add_activation', 'layer_norm' - ] and out_name != 'Y': + if ( + op.type + in ['batch_norm', 'fused_bn_add_activation', 'layer_norm'] + and out_name != 'Y' + ): continue for out_var_name in op.output(out_name): out_var = block.var(out_var_name) @@ -131,31 +151,36 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): return num_cast_ops -def _insert_cast_post_op(block, op, idx, src_dtype, dest_dtype, target_name, - op_var_rename_map): +def _insert_cast_post_op( + block, op, idx, src_dtype, dest_dtype, target_name, op_var_rename_map +): num_cast_ops = 0 target_var = block.var(target_name) if target_var.type not in _valid_types or target_var.dtype == dest_dtype: return num_cast_ops - assert target_var.dtype == src_dtype, \ - "The real dtype({}) is not equal to the src dtype({})".format(_dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype)) + assert ( + target_var.dtype == src_dtype + ), "The real dtype({}) is not equal to the src dtype({})".format( + _dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype) + ) cast_name = target_var.name + '.cast_' + _dtype_to_str(dest_dtype) cast_var = block.vars.get(cast_name) if cast_var is None or cast_var.dtype != dest_dtype: - cast_var = block.create_var(name=cast_name, - dtype=dest_dtype, - persistable=False, - stop_gradient=target_var.stop_gradient) - block._insert_op(idx, - type="cast", - inputs={"X": target_var}, - outputs={"Out": cast_var}, - attrs={ - "in_dtype": target_var.dtype, - "out_dtype": cast_var.dtype - }) + cast_var = block.create_var( + name=cast_name, + dtype=dest_dtype, + persistable=False, + stop_gradient=target_var.stop_gradient, + ) + block._insert_op( + idx, + type="cast", + inputs={"X": target_var}, + outputs={"Out": cast_var}, + attrs={"in_dtype": target_var.dtype, "out_dtype": cast_var.dtype}, + ) num_cast_ops += 1 op_var_rename_map[block.idx][target_var.name] = cast_var.name @@ -192,8 +217,9 @@ def _need_keep_fp32(op, unsupported_op_list, use_bf16_guard): return True if use_bf16_guard: - if op.has_attr("op_namescope") and \ - (_bf16_guard_pattern in op.attr("op_namescope")): + if op.has_attr("op_namescope") and ( + _bf16_guard_pattern in op.attr("op_namescope") + ): # op in bf16 guard return False else: @@ -238,12 +264,14 @@ def are_post_ops_bf16(post_ops, keep_fp32_ops): return True -def cast_initializers_to_bf16(startup_prog, - amp_lists, - block, - all_ops, - keep_fp32_ops, - to_bf16_var_names=None): +def cast_initializers_to_bf16( + startup_prog, + amp_lists, + block, + all_ops, + keep_fp32_ops, + to_bf16_var_names=None, +): prepend_ops = startup_prog.global_block().ops for op in prepend_ops: if str(op.type) in amp_lists.bf16_initializer_list: @@ -265,17 +293,21 @@ def cast_initializers_to_bf16(startup_prog, for out_var in op_out_vars: if out_var.dtype == core.VarDesc.VarType.FP32: out_var.desc.set_dtype(core.VarDesc.VarType.BF16) - if to_bf16_var_names is not None and out_var.name in to_bf16_var_names: + if ( + to_bf16_var_names is not None + and out_var.name in to_bf16_var_names + ): to_bf16_var_names.remove(out_var.name) - if op.has_attr('dtype') and op.attr( - 'dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('dtype') + and op.attr('dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('dtype', core.VarDesc.VarType.BF16) -def cast_model_to_bf16(program, - startup_prog=None, - amp_lists=None, - use_bf16_guard=True): +def cast_model_to_bf16( + program, startup_prog=None, amp_lists=None, use_bf16_guard=True +): """ Traverse all ops in the whole model and set their inputs and outputs to the bf16 data type. This function will do some special processing for @@ -307,7 +339,9 @@ def cast_model_to_bf16(program, continue # processed below for in_name in op.input_names: if op.type in { - 'batch_norm', 'fused_bn_add_activation', 'layer_norm' + 'batch_norm', + 'fused_bn_add_activation', + 'layer_norm', } and in_name not in {'X', 'Z'}: continue for in_var_name in op.input(in_name): @@ -316,13 +350,17 @@ def cast_model_to_bf16(program, in_var = block.var(in_var_name) except ValueError as e: _logger.debug( - "-- {}, try to get it in the global block --". - format(e)) + "-- {}, try to get it in the global block --".format( + e + ) + ) in_var = global_block.var(in_var_name) if in_var is not None: _logger.debug( - "-- var {} is got in the global block --". - format(in_var_name)) + "-- var {} is got in the global block --".format( + in_var_name + ) + ) if in_var is None or in_var.type not in _valid_types: continue @@ -332,13 +370,17 @@ def cast_model_to_bf16(program, to_bf16_var_names.add(in_var_name) _logger.debug( - "-- op type: {}, in var name: {}, in var dtype: {} --". - format(op.type, in_var_name, in_var.dtype)) + "-- op type: {}, in var name: {}, in var dtype: {} --".format( + op.type, in_var_name, in_var.dtype + ) + ) for out_name in op.output_names: - if op.type in { - 'batch_norm', 'fused_bn_add_activation', 'layer_norm' - } and out_name != 'Y': + if ( + op.type + in {'batch_norm', 'fused_bn_add_activation', 'layer_norm'} + and out_name != 'Y' + ): continue for out_var_name in op.output(out_name): out_var = None @@ -346,13 +388,17 @@ def cast_model_to_bf16(program, out_var = block.var(out_var_name) except ValueError as e: _logger.debug( - "-- {}, try to get it in the global block --". - format(e)) + "-- {}, try to get it in the global block --".format( + e + ) + ) out_var = global_block.var(out_var_name) if out_var is not None: _logger.debug( - "-- var {} is got in the global block --". - format(out_var_name)) + "-- var {} is got in the global block --".format( + out_var_name + ) + ) if out_var is None or out_var.type not in _valid_types: continue @@ -361,11 +407,15 @@ def cast_model_to_bf16(program, out_var.desc.set_dtype(core.VarDesc.VarType.BF16) _logger.debug( - "-- op type: {}, out var name: {}, out var dtype: {} --" - .format(op.type, out_var_name, out_var.dtype)) + "-- op type: {}, out var name: {}, out var dtype: {} --".format( + op.type, out_var_name, out_var.dtype + ) + ) for attr_name in ['in_dtype', 'out_dtype', 'dtype']: - if op.has_attr(attr_name) and op.attr( - attr_name) == core.VarDesc.VarType.FP32: + if ( + op.has_attr(attr_name) + and op.attr(attr_name) == core.VarDesc.VarType.FP32 + ): op._set_attr(attr_name, core.VarDesc.VarType.BF16) if op.has_attr('use_mkldnn'): op._set_attr('use_mkldnn', True) @@ -373,8 +423,14 @@ def cast_model_to_bf16(program, op._set_attr('mkldnn_data_type', 'bfloat16') if startup_prog is not None: - cast_initializers_to_bf16(startup_prog, amp_lists, global_block, - ops, keep_fp32_ops, to_bf16_var_names) + cast_initializers_to_bf16( + startup_prog, + amp_lists, + global_block, + ops, + keep_fp32_ops, + to_bf16_var_names, + ) # process ops in keep_fp32_ops op_var_rename_map = [ @@ -388,14 +444,22 @@ def cast_model_to_bf16(program, num_cast_ops = 0 if op not in keep_fp32_ops: if op in to_bf16_pre_cast_ops: - in_var_cast_num = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP32, - core.VarDesc.VarType.BF16) + in_var_cast_num = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.BF16, + ) num_cast_ops += in_var_cast_num else: - pre_cast_num = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.BF16, - core.VarDesc.VarType.FP32) + pre_cast_num = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.BF16, + core.VarDesc.VarType.FP32, + ) num_cast_ops += pre_cast_num for out_var_name in op.output_arg_names: out_var = block.vars.get(out_var_name) @@ -408,10 +472,14 @@ def cast_model_to_bf16(program, if post_op in keep_fp32_ops: continue post_cast_num = _insert_cast_post_op( - block, op, idx + pre_cast_num + 1, + block, + op, + idx + pre_cast_num + 1, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.BF16, out_var_name, - op_var_rename_map) + core.VarDesc.VarType.BF16, + out_var_name, + op_var_rename_map, + ) num_cast_ops += post_cast_num idx += num_cast_ops + 1 @@ -482,7 +550,8 @@ def rewrite_program_bf16(main_prog, amp_lists=None): continue if amp_lists.fp32_varnames is not None and _is_in_fp32_varnames( - op, amp_lists): + op, amp_lists + ): fp32_op_set.add(op) continue @@ -508,11 +577,15 @@ def rewrite_program_bf16(main_prog, amp_lists=None): else: prev_op = in_var.op # if it's one of inputs - if prev_op in fp32_op_set or \ - prev_op.type in amp_lists.fp32_list: + if ( + prev_op in fp32_op_set + or prev_op.type in amp_lists.fp32_list + ): is_fp32_op = True - elif prev_op in bf16_op_set or \ - prev_op.type in amp_lists.bf16_list: + elif ( + prev_op in bf16_op_set + or prev_op.type in amp_lists.bf16_list + ): is_bf16_op = True if is_fp32_op: fp32_op_set.add(op) @@ -530,20 +603,30 @@ def rewrite_program_bf16(main_prog, amp_lists=None): op = ops[idx] num_cast_ops = 0 if op in fp32_op_set: - num_cast_ops = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.BF16, - core.VarDesc.VarType.FP32) + num_cast_ops = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.BF16, + core.VarDesc.VarType.FP32, + ) elif op in bf16_op_set: if op.has_attr('use_mkldnn'): op._set_attr('use_mkldnn', True) op._set_attr('mkldnn_data_type', 'bfloat16') - elif op.has_attr('dtype') and op.attr( - 'dtype') == core.VarDesc.VarType.FP32: + elif ( + op.has_attr('dtype') + and op.attr('dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('dtype', core.VarDesc.VarType.BF16) - num_cast_ops = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP32, - core.VarDesc.VarType.BF16) + num_cast_ops = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.BF16, + ) else: pass diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py b/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py index 0551cf1aace..91106865822 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py @@ -12,10 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.fluid import (core, default_main_program, layers, program_guard, - unique_name) -from .amp_utils import (rewrite_program_bf16, cast_model_to_bf16, - cast_parameters_to_bf16) +from paddle.fluid import ( + core, + default_main_program, + layers, + program_guard, + unique_name, +) +from .amp_utils import ( + rewrite_program_bf16, + cast_model_to_bf16, + cast_parameters_to_bf16, +) from .amp_lists import AutoMixedPrecisionListsBF16 import types import warnings @@ -58,20 +66,24 @@ class OptimizerWithMixedPrecision(object): # Ensure the data type of learning rate vars is float32 (same as the # master parameter dtype) if isinstance(self._optimizer._learning_rate, float): - self._optimizer._learning_rate_map[default_main_program()] = \ - layers.create_global_var( - name=unique_name.generate("learning_rate"), - shape=[1], - value=float(self._optimizer._learning_rate), - dtype='float32', - persistable=True) - - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + self._optimizer._learning_rate_map[ + default_main_program() + ] = layers.create_global_var( + name=unique_name.generate("learning_rate"), + shape=[1], + value=float(self._optimizer._learning_rate), + dtype='float32', + persistable=True, + ) + + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ Backward propagation or auto differentiation for gradients' computation. @@ -96,24 +108,25 @@ class OptimizerWithMixedPrecision(object): if self._use_pure_bf16: self._to_bf16_var_names = cast_model_to_bf16( - self._train_program, startup_program, self._amp_lists, - self._use_bf16_guard) + self._train_program, + startup_program, + self._amp_lists, + self._use_bf16_guard, + ) else: rewrite_program_bf16(self._train_program, self._amp_lists) if loss.dtype != core.VarDesc.VarType.FP32: loss = loss.astype('float32') - params_grads = self._optimizer.backward(loss, startup_program, - parameter_list, no_grad_set, - callbacks) + params_grads = self._optimizer.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) return params_grads - def amp_init(self, - place, - scope=None, - test_program=None, - use_bf16_test=False): + def amp_init( + self, place, scope=None, test_program=None, use_bf16_test=False + ): """ Init the amp training, such as cast fp32 parameters to bf16 type. @@ -165,16 +178,20 @@ class OptimizerWithMixedPrecision(object): optimizer.amp_init(place, scope=paddle.static.global_scope()) """ - assert self._train_program is not None, \ - "Please call the minimize method first." + assert ( + self._train_program is not None + ), "Please call the minimize method first." if self._use_pure_bf16: - cast_parameters_to_bf16(place, self._train_program, scope, - self._to_bf16_var_names) + cast_parameters_to_bf16( + place, self._train_program, scope, self._to_bf16_var_names + ) if test_program is not None: if self._use_pure_bf16: - cast_model_to_bf16(test_program, - amp_lists=self._amp_lists, - use_bf16_guard=self._use_bf16_guard) + cast_model_to_bf16( + test_program, + amp_lists=self._amp_lists, + use_bf16_guard=self._use_bf16_guard, + ) elif use_bf16_test: rewrite_program_bf16(test_program, amp_lists=self._amp_lists) @@ -197,11 +214,9 @@ class OptimizerWithMixedPrecision(object): optimize_ops = self.apply_gradients(params_grads) return optimize_ops - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ Perform optimization by minimizing the given loss. @@ -217,26 +232,28 @@ class OptimizerWithMixedPrecision(object): list of scaled parameters and gradients. """ opt_dict = self._optimizer.__class__.__dict__ - if 'minimize' in opt_dict and isinstance(opt_dict['minimize'], - types.FunctionType): + if 'minimize' in opt_dict and isinstance( + opt_dict['minimize'], types.FunctionType + ): warnings.warn( "The decorated optimizer has its own `minimize` method, but it will not be executed." ) - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) optimize_ops = self.apply_optimize(loss, startup_program, params_grads) return optimize_ops, params_grads -def decorate_bf16(optimizer, - amp_lists=None, - use_pure_bf16=False, - use_bf16_guard=None): +def decorate_bf16( + optimizer, amp_lists=None, use_pure_bf16=False, use_bf16_guard=None +): """ Decorate the given optimizer to adapt to the mixed-precision training. @@ -252,7 +269,7 @@ def decorate_bf16(optimizer, enabled. Examples 1: - .. code-block:: python + .. code-block:: python # fp32&bf16 list based strategy example import paddle @@ -316,7 +333,8 @@ def decorate_bf16(optimizer, if use_bf16_guard is None: use_bf16_guard = use_pure_bf16 - mp_optimizer = OptimizerWithMixedPrecision(optimizer, amp_lists, - use_pure_bf16, use_bf16_guard) + mp_optimizer = OptimizerWithMixedPrecision( + optimizer, amp_lists, use_pure_bf16, use_bf16_guard + ) return mp_optimizer diff --git a/python/paddle/fluid/contrib/mixed_precision/decorator.py b/python/paddle/fluid/contrib/mixed_precision/decorator.py index 70aefac5a59..75554ff9c81 100644 --- a/python/paddle/fluid/contrib/mixed_precision/decorator.py +++ b/python/paddle/fluid/contrib/mixed_precision/decorator.py @@ -63,10 +63,19 @@ class OptimizerWithMixedPrecision(object): """ - def __init__(self, optimizer, amp_lists, init_loss_scaling, - use_dynamic_loss_scaling, incr_every_n_steps, - decr_every_n_nan_or_inf, incr_ratio, decr_ratio, use_pure_fp16, - use_fp16_guard): + def __init__( + self, + optimizer, + amp_lists, + init_loss_scaling, + use_dynamic_loss_scaling, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + use_pure_fp16, + use_fp16_guard, + ): self._optimizer = optimizer self._amp_lists = amp_lists self._param_grads = None @@ -97,9 +106,10 @@ class OptimizerWithMixedPrecision(object): self._is_distributed = flag def get_loss_scaling(self): - """Return the real-time loss scaling factor. - """ - assert self._loss_scaling is not None, 'Please call minimize() before calling get_loss_scaling().' + """Return the real-time loss scaling factor.""" + assert ( + self._loss_scaling is not None + ), 'Please call minimize() before calling get_loss_scaling().' return self._loss_scaling def get_scaled_loss(self): @@ -117,7 +127,8 @@ class OptimizerWithMixedPrecision(object): shape=[1], value=self._init_loss_scaling, dtype='float32', - persistable=True) + persistable=True, + ) if self._use_dynamic_loss_scaling: self._num_good_steps = layers.create_global_var( @@ -125,31 +136,37 @@ class OptimizerWithMixedPrecision(object): shape=[1], value=0, dtype='int32', - persistable=True) + persistable=True, + ) self._num_bad_steps = layers.create_global_var( name=unique_name.generate("num_bad_steps"), shape=[1], value=0, dtype='int32', - persistable=True) + persistable=True, + ) # Ensure the data type of learning rate vars is float32 (same as the # master parameter dtype) if isinstance(self._optimizer._learning_rate, float): - self._optimizer._learning_rate_map[default_main_program()] = \ - layers.create_global_var( - name=unique_name.generate("learning_rate"), - shape=[1], - value=float(self._optimizer._learning_rate), - dtype='float32', - persistable=True) - - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + self._optimizer._learning_rate_map[ + default_main_program() + ] = layers.create_global_var( + name=unique_name.generate("learning_rate"), + shape=[1], + value=float(self._optimizer._learning_rate), + dtype='float32', + persistable=True, + ) + + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ Backward propagation or auto differentiation for gradients' computation. @@ -171,9 +188,9 @@ class OptimizerWithMixedPrecision(object): # NOTE(zhiqiu): _float_status is only used for NPU. if core.is_compiled_with_npu(): - float_status = paddle.static.data(name="float_status", - shape=[8], - dtype='float32') + float_status = paddle.static.data( + name="float_status", shape=[8], dtype='float32' + ) self._train_program.global_block().append_op( type="alloc_float_status", outputs={"FloatStatus": float_status}, @@ -192,7 +209,8 @@ class OptimizerWithMixedPrecision(object): if self._use_pure_fp16: self._to_fp16_var_names = cast_model_to_fp16( - self._train_program, self._amp_lists, self._use_fp16_guard) + self._train_program, self._amp_lists, self._use_fp16_guard + ) else: rewrite_program(self._train_program, self._amp_lists) @@ -205,10 +223,13 @@ class OptimizerWithMixedPrecision(object): else: self._scaled_loss = loss - params_grads = self._optimizer.backward(self._scaled_loss, - startup_program, - parameter_list, no_grad_set, - callbacks) + params_grads = self._optimizer.backward( + self._scaled_loss, + startup_program, + parameter_list, + no_grad_set, + callbacks, + ) if self._supports_check_nan_inf(): self._add_cast_ops_to_startup_program(startup_program) return params_grads @@ -216,8 +237,11 @@ class OptimizerWithMixedPrecision(object): def _add_cast_ops_to_startup_program(self, startup_program): names = list(self._to_fp16_var_names) if self._to_fp16_var_names else [] names.sort() - startup_program = default_startup_program( - ) if startup_program is None else startup_program + startup_program = ( + default_startup_program() + if startup_program is None + else startup_program + ) block = startup_program.global_block() param_names = [p.name for p in block.all_parameters()] for name in names: @@ -225,23 +249,23 @@ class OptimizerWithMixedPrecision(object): continue tmp = block.create_var(dtype=core.VarDesc.VarType.FP32) - block.append_op(type='assign', - inputs={'X': [name]}, - outputs={'Out': [tmp]}) - block.append_op(type='cast', - inputs={'X': [tmp]}, - outputs={'Out': [name]}, - attrs={ - 'in_dtype': core.VarDesc.VarType.FP32, - 'out_dtype': core.VarDesc.VarType.FP16, - }) + block.append_op( + type='assign', inputs={'X': [name]}, outputs={'Out': [tmp]} + ) + block.append_op( + type='cast', + inputs={'X': [tmp]}, + outputs={'Out': [name]}, + attrs={ + 'in_dtype': core.VarDesc.VarType.FP32, + 'out_dtype': core.VarDesc.VarType.FP16, + }, + ) self._to_fp16_var_names = None - def amp_init(self, - place, - scope=None, - test_program=None, - use_fp16_test=False): + def amp_init( + self, place, scope=None, test_program=None, use_fp16_test=False + ): """ Init the amp training, such as cast fp32 parameters to fp16 type. @@ -297,15 +321,18 @@ class OptimizerWithMixedPrecision(object): if paddle.is_compiled_with_cuda() and len(paddle.static.cuda_places()) > 0: run_example_code() """ - assert self._train_program is not None, \ - "Please call the minimize method first." + assert ( + self._train_program is not None + ), "Please call the minimize method first." if self._use_pure_fp16: - cast_parameters_to_fp16(place, self._train_program, scope, - self._to_fp16_var_names) + cast_parameters_to_fp16( + place, self._train_program, scope, self._to_fp16_var_names + ) if test_program is not None: if self._use_pure_fp16: - cast_model_to_fp16(test_program, self._amp_lists, - self._use_fp16_guard) + cast_model_to_fp16( + test_program, self._amp_lists, self._use_fp16_guard + ) elif use_fp16_test: rewrite_program(test_program, self._amp_lists) @@ -327,7 +354,10 @@ class OptimizerWithMixedPrecision(object): # When not using dynamic loss scaling and the init loss scaling value is equal to 1.0, # the model can be optimized. - if not self._use_dynamic_loss_scaling and self._init_loss_scaling == 1.0: + if ( + not self._use_dynamic_loss_scaling + and self._init_loss_scaling == 1.0 + ): return self._optimizer.apply_gradients(params_grads) if self._supports_check_nan_inf(): @@ -346,13 +376,16 @@ class OptimizerWithMixedPrecision(object): real_optimizer = self._optimizer while hasattr(real_optimizer, "inner_opt"): real_optimizer = real_optimizer.inner_opt - if isinstance(real_optimizer, - (paddle.fluid.optimizer.Adam, paddle.optimizer.AdamW)): + if isinstance( + real_optimizer, + (paddle.fluid.optimizer.Adam, paddle.optimizer.AdamW), + ): # NOTE(zhiqiu): Since found_inf needs to be on cpu in adam op, we # copy it in advance to avoid multiple time copies. with self._train_program._optimized_guard([]): found_inf = paddle.tensor.creation._memcpy( - found_inf, paddle.CPUPlace()) + found_inf, paddle.CPUPlace() + ) real_optimizer._set_auxiliary_var('found_inf', found_inf) elif hasattr(real_optimizer, "_set_auxiliary_var"): real_optimizer._set_auxiliary_var('found_inf', found_inf) @@ -363,8 +396,9 @@ class OptimizerWithMixedPrecision(object): grads = [g for _, g in params_grads] fp32_grads = [g for g in grads if g.dtype == core.VarDesc.VarType.FP32] fp16_grads = [g for g in grads if g.dtype == core.VarDesc.VarType.FP16] - assert len(fp32_grads) + len(fp16_grads) == len(grads), \ - "Data types of all grads must be either fp16 or fp32." + assert len(fp32_grads) + len(fp16_grads) == len( + grads + ), "Data types of all grads must be either fp16 or fp32." return grads, fp32_grads, fp16_grads def _check_finite_and_unscale(self, params_grads): @@ -380,7 +414,8 @@ class OptimizerWithMixedPrecision(object): grads, self._loss_scaling, name="find_infinite_scale", - float_status=self._float_status) + float_status=self._float_status, + ) found_infs.append(found_inf) else: for p, g in params_grads: @@ -391,7 +426,8 @@ class OptimizerWithMixedPrecision(object): ], self._loss_scaling, name="find_infinite_scale", - float_status=self._float_status) + float_status=self._float_status, + ) found_infs.append(found_inf) elif self._use_pure_fp16: if fp32_grads: @@ -400,7 +436,8 @@ class OptimizerWithMixedPrecision(object): fp32_grads, self._loss_scaling, name="find_infinite_scale_fp32", - float_status=self._float_status) + float_status=self._float_status, + ) found_infs.append(fp32_found_inf) if fp16_grads: with self._train_program._optimized_guard(fp16_grads): @@ -408,7 +445,8 @@ class OptimizerWithMixedPrecision(object): fp16_grads, self._loss_scaling, name="find_infinite_scale_fp16", - float_status=self._float_status) + float_status=self._float_status, + ) found_infs.append(fp16_found_inf) else: with self._train_program._optimized_guard(grads): @@ -416,7 +454,8 @@ class OptimizerWithMixedPrecision(object): grads, self._loss_scaling, name="find_infinite_scale", - float_status=self._float_status) + float_status=self._float_status, + ) if self._is_distributed or self._use_pure_fp16: with self._train_program._optimized_guard([]): @@ -439,7 +478,8 @@ class OptimizerWithMixedPrecision(object): self._incr_ratio, self._decr_ratio, stop_update=self._optimizer._get_stop_update_var(), - name="update_loss_scaling") + name="update_loss_scaling", + ) return grads, fp32_grads, fp16_grads = self._split_grads(params_grads) @@ -447,42 +487,48 @@ class OptimizerWithMixedPrecision(object): stop_update = False with self._train_program._optimized_guard([]): if fp32_grads: - update_loss_scaling(fp32_grads, - found_inf, - self._loss_scaling, - self._num_good_steps, - self._num_bad_steps, - self._incr_every_n_steps, - self._decr_every_n_nan_or_inf, - self._incr_ratio, - self._decr_ratio, - stop_update=stop_update, - name="update_loss_scaling_fp32") + update_loss_scaling( + fp32_grads, + found_inf, + self._loss_scaling, + self._num_good_steps, + self._num_bad_steps, + self._incr_every_n_steps, + self._decr_every_n_nan_or_inf, + self._incr_ratio, + self._decr_ratio, + stop_update=stop_update, + name="update_loss_scaling_fp32", + ) stop_update = True if fp16_grads: - update_loss_scaling(fp16_grads, - found_inf, - self._loss_scaling, - self._num_good_steps, - self._num_bad_steps, - self._incr_every_n_steps, - self._decr_every_n_nan_or_inf, - self._incr_ratio, - self._decr_ratio, - stop_update=stop_update, - name="update_loss_scaling_fp16") + update_loss_scaling( + fp16_grads, + found_inf, + self._loss_scaling, + self._num_good_steps, + self._num_bad_steps, + self._incr_every_n_steps, + self._decr_every_n_nan_or_inf, + self._incr_ratio, + self._decr_ratio, + stop_update=stop_update, + name="update_loss_scaling_fp16", + ) else: with self._train_program._optimized_guard([]): - update_loss_scaling(grads, - found_inf, - self._loss_scaling, - self._num_good_steps, - self._num_bad_steps, - self._incr_every_n_steps, - self._decr_every_n_nan_or_inf, - self._incr_ratio, - self._decr_ratio, - name="update_loss_scaling") + update_loss_scaling( + grads, + found_inf, + self._loss_scaling, + self._num_good_steps, + self._num_bad_steps, + self._incr_every_n_steps, + self._decr_every_n_nan_or_inf, + self._incr_ratio, + self._decr_ratio, + name="update_loss_scaling", + ) def apply_optimize(self, loss, startup_program, params_grads): program = loss.block.program @@ -490,11 +536,9 @@ class OptimizerWithMixedPrecision(object): optimize_ops = self.apply_gradients(params_grads) return optimize_ops - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ Perform optimization by minimizing the given loss. @@ -511,33 +555,39 @@ class OptimizerWithMixedPrecision(object): """ opt_dict = self._optimizer.__class__.__dict__ - if 'minimize' in opt_dict and isinstance(opt_dict['minimize'], - types.FunctionType): + if 'minimize' in opt_dict and isinstance( + opt_dict['minimize'], types.FunctionType + ): warnings.warn( "The decorated optimizer has its own `minimize` method, but it will not be executed." ) - scaled_params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + scaled_params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self.apply_optimize(loss, startup_program, - scaled_params_grads) + optimize_ops = self.apply_optimize( + loss, startup_program, scaled_params_grads + ) return optimize_ops, scaled_params_grads -def decorate(optimizer, - amp_lists=None, - init_loss_scaling=2**15, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - incr_ratio=2.0, - decr_ratio=0.8, - use_dynamic_loss_scaling=True, - use_pure_fp16=False, - use_fp16_guard=None): +def decorate( + optimizer, + amp_lists=None, + init_loss_scaling=2**15, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=2, + incr_ratio=2.0, + decr_ratio=0.8, + use_dynamic_loss_scaling=True, + use_pure_fp16=False, + use_fp16_guard=None, +): """ Decorate the given optimizer to adapt to the mixed-precision training. @@ -564,7 +614,7 @@ def decorate(optimizer, enabled. Examples 1: - .. code-block:: python + .. code-block:: python # black&white list based strategy example import paddle @@ -635,8 +685,16 @@ def decorate(optimizer, use_fp16_guard = use_pure_fp16 mp_optimizer = OptimizerWithMixedPrecision( - optimizer, amp_lists, init_loss_scaling, use_dynamic_loss_scaling, - incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, - use_pure_fp16, use_fp16_guard) + optimizer, + amp_lists, + init_loss_scaling, + use_dynamic_loss_scaling, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + use_pure_fp16, + use_fp16_guard, + ) return mp_optimizer diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py index b2767b1dd1c..ef8f222bac2 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py @@ -19,7 +19,10 @@ __all__ = ["CustomOpLists", "AutoMixedPrecisionLists"] # lookup_table fp16 is slower than fp32, though fp16 is supported. _extra_unsupported_fp16_list = { - 'lookup_table', 'lookup_table_v2', 'scatter', 'scatter_grad' + 'lookup_table', + 'lookup_table_v2', + 'scatter', + 'scatter_grad', } @@ -36,10 +39,12 @@ class AutoMixedPrecisionLists(object): custom_black_varnames (set): Users' custom black varibles' names. """ - def __init__(self, - custom_white_list=None, - custom_black_list=None, - custom_black_varnames=None): + def __init__( + self, + custom_white_list=None, + custom_black_list=None, + custom_black_varnames=None, + ): self._custom_white_list = custom_white_list self._custom_black_list = custom_black_list self.white_list = copy.copy(white_list) @@ -56,8 +61,9 @@ class AutoMixedPrecisionLists(object): if self._custom_white_list and self._custom_black_list: for op_name in self._custom_white_list: if op_name in self._custom_black_list: - raise ValueError("Custom white list overlap " - "custom black list") + raise ValueError( + "Custom white list overlap " "custom black list" + ) if self._custom_white_list: for op_name in self._custom_white_list: if op_name in self.black_list: @@ -175,17 +181,23 @@ gray_list = { _sys_unsupported_fp16_list = [] if core.is_compiled_with_xpu(): _, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'XPU', core.VarDesc.VarType.FP16) + 'XPU', core.VarDesc.VarType.FP16 + ) elif core.is_compiled_with_npu(): _, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'NPU', core.VarDesc.VarType.FP16) + 'NPU', core.VarDesc.VarType.FP16 + ) elif core.is_compiled_with_mlu(): _, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'MLU', core.VarDesc.VarType.FP16) + 'MLU', core.VarDesc.VarType.FP16 + ) else: _, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'GPU', core.VarDesc.VarType.FP16) + 'GPU', core.VarDesc.VarType.FP16 + ) -unsupported_fp16_list = _extra_unsupported_fp16_list | _sys_unsupported_fp16_list +unsupported_fp16_list = ( + _extra_unsupported_fp16_list | _sys_unsupported_fp16_list +) CustomOpLists = AutoMixedPrecisionLists diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py index ad91e8ba799..e50a2457147 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py @@ -25,13 +25,14 @@ import numpy as np __all__ = ["fp16_guard", "cast_model_to_fp16", "cast_parameters_to_fp16"] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) _valid_types = [ - core.VarDesc.VarType.LOD_TENSOR, core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR_ARRAY + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, ] _fp16_guard_pattern = "__use_fp16__" @@ -106,7 +107,12 @@ def _keep_fp32_input(op, in_name): return in_name not in {'X', 'FilterX', 'Z', 'FilterZ'} if op_type in ['fused_attention', 'fused_feedforward']: return in_name in { - 'LnScale', 'LnBias', 'Ln2Scale', 'Ln2Bias', "Ln1Scale", "Ln1Bias" + 'LnScale', + 'LnBias', + 'Ln2Scale', + 'Ln2Bias', + "Ln1Scale", + "Ln1Bias", } if op_type == 'fused_multi_transformer': return in_name in {'LnScale', 'LnBias', 'FFNLnScale', 'FFNLnBias'} @@ -123,8 +129,12 @@ def _keep_fp32_output(op, out_name): return out_name not in {'Y', 'ConvX', 'ConvZ'} if op_type in ['fused_attention', 'fused_feedforward']: return out_name in { - 'LnMean', 'LnVariance', 'Ln2Mean', 'Ln2Variance', 'Ln1Mean', - 'Ln1Variance' + 'LnMean', + 'LnVariance', + 'Ln2Mean', + 'Ln2Variance', + 'Ln1Mean', + 'Ln1Variance', } return False @@ -147,7 +157,8 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): for in_name in op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_input( - op, in_name): + op, in_name + ): continue for in_var_name in op.input(in_name): in_var = block._find_var_recursive(in_var_name) @@ -163,11 +174,15 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): # set cast_op device to `all`, can reduce send cast_var. # TODO: need remove this after we unified the dynamic # and static pipeline interface. - if src_dtype == core.VarDesc.VarType.FP32 and in_var.stop_gradient: + if ( + src_dtype == core.VarDesc.VarType.FP32 + and in_var.stop_gradient + ): prev_op = None if in_var.op is op: - prev_op = find_true_prev_op(block.ops, op, - in_var_name) + prev_op = find_true_prev_op( + block.ops, op, in_var_name + ) elif in_var.op is not None: prev_op = in_var.op @@ -175,33 +190,40 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): if prev_op is not None: prev_op_device = prev_op.attr('op_device') - if prev_op_device is not None and 'all' in prev_op_device: + if ( + prev_op_device is not None + and 'all' in prev_op_device + ): op_device = prev_op_device out_var = block.create_var( name=cast_name, dtype=dest_dtype, persistable=False, - stop_gradient=in_var.stop_gradient) - - block._insert_op_without_sync(idx, - type="cast", - inputs={"X": in_var}, - outputs={"Out": out_var}, - attrs={ - "in_dtype": in_var.dtype, - "out_dtype": - out_var.dtype, - "op_device": op_device, - "op_role": - op.attr("op_role"), - }) + stop_gradient=in_var.stop_gradient, + ) + + block._insert_op_without_sync( + idx, + type="cast", + inputs={"X": in_var}, + outputs={"Out": out_var}, + attrs={ + "in_dtype": in_var.dtype, + "out_dtype": out_var.dtype, + "op_device": op_device, + "op_role": op.attr("op_role"), + }, + ) num_cast_ops += 1 _rename_arg(op, in_var.name, out_var.name) else: if op.has_attr('in_dtype'): op._set_attr('in_dtype', dest_dtype) - if src_dtype == core.VarDesc.VarType.FP32 and dest_dtype == core.VarDesc.VarType.FP16: + if ( + src_dtype == core.VarDesc.VarType.FP32 + and dest_dtype == core.VarDesc.VarType.FP16 + ): for out_name in op.output_names: if _keep_fp32_output(op, out_name): continue @@ -216,35 +238,42 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): return num_cast_ops -def _insert_cast_post_op(block, op, idx, src_dtype, dest_dtype, target_name, - op_var_rename_map): +def _insert_cast_post_op( + block, op, idx, src_dtype, dest_dtype, target_name, op_var_rename_map +): num_cast_ops = 0 target_var = block.var(target_name) if target_var.type not in _valid_types or target_var.dtype == dest_dtype: return num_cast_ops - assert target_var.dtype == src_dtype, \ - "The real dtype({}) is not equal to the src dtype({})".format( - _dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype)) + assert ( + target_var.dtype == src_dtype + ), "The real dtype({}) is not equal to the src dtype({})".format( + _dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype) + ) cast_name = target_var.name + '.cast_' + _dtype_to_str(dest_dtype) cast_var = block.vars.get(cast_name) if cast_var is None or cast_var.dtype != dest_dtype: - cast_var = block.create_var(name=cast_name, - dtype=dest_dtype, - persistable=False, - stop_gradient=target_var.stop_gradient) - block._insert_op(idx, - type="cast", - inputs={"X": target_var}, - outputs={"Out": cast_var}, - attrs={ - "in_dtype": target_var.dtype, - "out_dtype": cast_var.dtype, - "op_device": op.attr("op_device"), - "op_role": op.attr("op_role"), - }) + cast_var = block.create_var( + name=cast_name, + dtype=dest_dtype, + persistable=False, + stop_gradient=target_var.stop_gradient, + ) + block._insert_op( + idx, + type="cast", + inputs={"X": target_var}, + outputs={"Out": cast_var}, + attrs={ + "in_dtype": target_var.dtype, + "out_dtype": cast_var.dtype, + "op_device": op.attr("op_device"), + "op_role": op.attr("op_role"), + }, + ) num_cast_ops += 1 op_var_rename_map[block.idx][target_var.name] = cast_var.name @@ -270,8 +299,10 @@ def find_true_prev_op(ops, cur_op, var_name): prev_op.append(op) if prev_op: if not len(prev_op) == 1: - raise ValueError("There must be only one previous op " - "that outputs {0} variable".format(var_name)) + raise ValueError( + "There must be only one previous op " + "that outputs {0} variable".format(var_name) + ) else: return prev_op[0] return None @@ -313,8 +344,7 @@ def find_true_post_op(ops, cur_op, var_name, search_all=False): def find_op_index(block_desc, cur_op_desc): - """ - """ + """ """ for idx in range(block_desc.op_size()): if cur_op_desc == block_desc.op(idx): return idx @@ -348,8 +378,9 @@ def _need_keep_fp32(op, unsupported_op_list, use_fp16_guard): return True if use_fp16_guard: - if op.has_attr("op_namescope") and \ - (_fp16_guard_pattern in op.attr("op_namescope")): + if op.has_attr("op_namescope") and ( + _fp16_guard_pattern in op.attr("op_namescope") + ): # op in fp16 guard return False else: @@ -419,7 +450,8 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): for in_name in op.input_names: # for ipu, all inputs must be converted to fp16 if not core.is_compiled_with_ipu() and _keep_fp32_input( - op, in_name): + op, in_name + ): continue for in_var_name in op.input(in_name): in_var = None @@ -427,13 +459,17 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): in_var = block.var(in_var_name) except ValueError as e: _logger.debug( - "-- {}, try to get it in the global block --". - format(e)) + "-- {}, try to get it in the global block --".format( + e + ) + ) in_var = global_block.var(in_var_name) if in_var is not None: _logger.debug( - "-- var {} is got in the global block --". - format(in_var_name)) + "-- var {} is got in the global block --".format( + in_var_name + ) + ) if in_var is None or in_var.type not in _valid_types: continue @@ -443,13 +479,16 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): to_fp16_var_names.add(in_var_name) _logger.debug( - "-- op type: {}, in var name: {}, in var dtype: {} --". - format(op.type, in_var_name, in_var.dtype)) + "-- op type: {}, in var name: {}, in var dtype: {} --".format( + op.type, in_var_name, in_var.dtype + ) + ) for out_name in op.output_names: # for ipu, all outputs must be converted to fp16 if not core.is_compiled_with_ipu() and _keep_fp32_output( - op, out_name): + op, out_name + ): continue for out_var_name in op.output(out_name): out_var = None @@ -457,13 +496,17 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): out_var = block.var(out_var_name) except ValueError as e: _logger.debug( - "-- {}, try to get it in the global block --". - format(e)) + "-- {}, try to get it in the global block --".format( + e + ) + ) out_var = global_block.var(out_var_name) if out_var is not None: _logger.debug( - "-- var {} is got in the global block --". - format(out_var_name)) + "-- var {} is got in the global block --".format( + out_var_name + ) + ) if out_var is None or out_var.type not in _valid_types: continue @@ -472,16 +515,24 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): out_var.desc.set_dtype(core.VarDesc.VarType.FP16) _logger.debug( - "-- op type: {}, out var name: {}, out var dtype: {} --" - .format(op.type, out_var_name, out_var.dtype)) - if op.has_attr('in_dtype') and op.attr( - 'in_dtype') == core.VarDesc.VarType.FP32: + "-- op type: {}, out var name: {}, out var dtype: {} --".format( + op.type, out_var_name, out_var.dtype + ) + ) + if ( + op.has_attr('in_dtype') + and op.attr('in_dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('in_dtype', core.VarDesc.VarType.FP16) - if op.has_attr('out_dtype') and op.attr( - 'out_dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('out_dtype') + and op.attr('out_dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('out_dtype', core.VarDesc.VarType.FP16) - if op.has_attr('dtype') and op.attr( - 'dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('dtype') + and op.attr('dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('dtype', core.VarDesc.VarType.FP16) # process ops in keep_fp32_ops @@ -495,9 +546,13 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): op = ops[idx] num_cast_ops = 0 if op in keep_fp32_ops: - pre_cast_num = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32) + pre_cast_num = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + ) num_cast_ops += pre_cast_num for out_var_name in op.output_arg_names: out_var = block.vars.get(out_var_name) @@ -510,10 +565,14 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): if post_op in keep_fp32_ops: continue post_cast_num = _insert_cast_post_op( - block, op, idx + pre_cast_num + 1, + block, + op, + idx + pre_cast_num + 1, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, out_var_name, - op_var_rename_map) + core.VarDesc.VarType.FP16, + out_var_name, + op_var_rename_map, + ) num_cast_ops += post_cast_num idx += num_cast_ops + 1 @@ -583,7 +642,8 @@ def rewrite_program(main_prog, amp_lists): continue if amp_lists.black_varnames is not None and _is_in_black_varnames( - op, amp_lists): + op, amp_lists + ): black_op_set.add(op) continue @@ -609,11 +669,15 @@ def rewrite_program(main_prog, amp_lists): else: prev_op = in_var.op # if it's one of inputs - if prev_op in black_op_set or \ - prev_op.type in amp_lists.black_list: + if ( + prev_op in black_op_set + or prev_op.type in amp_lists.black_list + ): is_black_op = True - elif prev_op in white_op_set or \ - prev_op.type in amp_lists.white_list: + elif ( + prev_op in white_op_set + or prev_op.type in amp_lists.white_list + ): is_white_op = True if is_black_op: black_op_set.add(op) @@ -631,13 +695,21 @@ def rewrite_program(main_prog, amp_lists): op = ops[idx] num_cast_ops = 0 if op in black_op_set: - num_cast_ops = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32) + num_cast_ops = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + ) elif op in white_op_set: - num_cast_ops = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16) + num_cast_ops = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + ) else: pass @@ -668,13 +740,16 @@ def update_role_var_grad(main_prog, params_grads): if role & int(BACKWARD) and op.has_attr('op_role_var'): op._remove_attr("op_role_var") else: - raise ValueError("The cast op {0} must be in BACKWARD role " - "and have op_role_var attr.".format(op)) + raise ValueError( + "The cast op {0} must be in BACKWARD role " + "and have op_role_var attr.".format(op) + ) fp16_grad_name = op.input(op.input_names[0])[0] op_for_fp16_grad = find_true_prev_op(block.ops, op, fp16_grad_name) - op_role_var_attr_name = \ + op_role_var_attr_name = ( core.op_proto_and_checker_maker.kOpRoleVarAttrName() + ) attr_val = [p.name, fp16_grad_name] if op_for_fp16_grad.has_attr(op_role_var_attr_name): attr_val.extend(op_for_fp16_grad.attr(op_role_var_attr_name)) @@ -688,18 +763,22 @@ def update_role_var_grad(main_prog, params_grads): continue post_ops = find_true_post_op(block.ops, op, g.name) if post_ops: - raise ValueError("The cast op {0}'s output should not be" - "used by a non-optimize op, however, it" - "is used by {1}".format(op, post_ops[0])) + raise ValueError( + "The cast op {0}'s output should not be" + "used by a non-optimize op, however, it" + "is used by {1}".format(op, post_ops[0]) + ) # add new op in the python and cpp at the same time new_op_desc = block.desc.append_op() new_op_desc.copy_from(op.desc) - new_op = framework.Operator(block=block, - desc=new_op_desc, - type=None, - inputs=None, - outputs=None, - attrs=None) + new_op = framework.Operator( + block=block, + desc=new_op_desc, + type=None, + inputs=None, + outputs=None, + attrs=None, + ) block.ops.append(new_op) op_idx = find_op_index(block.desc, op.desc) if op_idx == -1: diff --git a/python/paddle/fluid/contrib/model_stat.py b/python/paddle/fluid/contrib/model_stat.py index 3bcc3306895..0c18deec5da 100644 --- a/python/paddle/fluid/contrib/model_stat.py +++ b/python/paddle/fluid/contrib/model_stat.py @@ -151,8 +151,10 @@ def _format_summary(collected_ops_list): _verify_dependent_package() from prettytable import PrettyTable + summary_table = PrettyTable( - ["No.", "TYPE", "INPUT", "OUTPUT", "PARAMs", "FLOPs"]) + ["No.", "TYPE", "INPUT", "OUTPUT", "PARAMs", "FLOPs"] + ) summary_table.align = 'r' total = {} @@ -200,8 +202,9 @@ def _print_summary(summary_table, total): parmas = total['params'] flops = total['flops'] print(summary_table) - print('Total PARAMs: {}({:.4f}M)'.format(sum(parmas), - sum(parmas) / (10**6))) + print( + 'Total PARAMs: {}({:.4f}M)'.format(sum(parmas), sum(parmas) / (10**6)) + ) print('Total FLOPs: {}({:.2f}G)'.format(sum(flops), sum(flops) / 10**9)) print( "Notice: \n now supported ops include [Conv, DepthwiseConv, FC(mul), BatchNorm, Pool, Activation(sigmoid, tanh, relu, leaky_relu, prelu)]" diff --git a/python/paddle/fluid/contrib/op_frequence.py b/python/paddle/fluid/contrib/op_frequence.py index 5203aa308e3..ee435ac657a 100644 --- a/python/paddle/fluid/contrib/op_frequence.py +++ b/python/paddle/fluid/contrib/op_frequence.py @@ -43,8 +43,10 @@ def op_freq_statistic(program): """ if not isinstance(program, Program): - raise TypeError("The input type should be Porgram." - "But you passed in %s" % (type(program))) + raise TypeError( + "The input type should be Porgram." + "But you passed in %s" % (type(program)) + ) uni_op_freq = OrderedDict() adj_2_op_freq = OrderedDict() @@ -78,8 +80,10 @@ def op_freq_statistic(program): else: op_in_ops[op.type] = [var_gen_op[var_name][-1]] else: - print("Var's generate op is not found,%s, %s" % - (var_name, op.type)) + print( + "Var's generate op is not found,%s, %s" + % (var_name, op.type) + ) for var_name in op.output_arg_names: if var_gen_op.has_key(var_name): @@ -95,11 +99,11 @@ def op_freq_statistic(program): else: adj_2_op_freq[op_op] = 1 - uni_op_freq = sorted(uni_op_freq.items(), - key=lambda item: item[1], - reverse=True) - adj_2_op_freq = sorted(adj_2_op_freq.items(), - key=lambda item: item[1], - reverse=True) + uni_op_freq = sorted( + uni_op_freq.items(), key=lambda item: item[1], reverse=True + ) + adj_2_op_freq = sorted( + adj_2_op_freq.items(), key=lambda item: item[1], reverse=True + ) return uni_op_freq, adj_2_op_freq diff --git a/python/paddle/fluid/contrib/optimizer.py b/python/paddle/fluid/contrib/optimizer.py index dc6dc213a8f..53386bc6666 100644 --- a/python/paddle/fluid/contrib/optimizer.py +++ b/python/paddle/fluid/contrib/optimizer.py @@ -104,31 +104,35 @@ class Momentum(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate, - momentum, - parameter_list=None, - use_nesterov=False, - regularization=None, - grad_clip=None, - multi_precision=False, - rescale_grad=1.0, - name=None): + def __init__( + self, + learning_rate, + momentum, + parameter_list=None, + use_nesterov=False, + regularization=None, + grad_clip=None, + multi_precision=False, + rescale_grad=1.0, + name=None, + ): assert learning_rate is not None assert momentum is not None predicate = lambda regular: isinstance(regular, L2DecayRegularizer) py_regular = None if predicate(regularization) else regularization - super(Momentum, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=py_regular, - grad_clip=grad_clip, - name=name) + super(Momentum, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=py_regular, + grad_clip=grad_clip, + name=name, + ) self.type = "momentum" self._momentum = momentum self._use_nesterov = bool(use_nesterov) self._regularization_method = "" self._regularization_coeff = 0 - if (isinstance(regularization, L2DecayRegularizer)): + if isinstance(regularization, L2DecayRegularizer): self._regularization_method = "l2_decay" self._regularization_coeff = regularization._regularization_coeff self._multi_precision = multi_precision @@ -140,19 +144,23 @@ class Momentum(Optimizer): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -168,15 +176,22 @@ class Momentum(Optimizer): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _create_accumulators(self, block, parameters): @@ -187,7 +202,10 @@ class Momentum(Optimizer): master_p = self._create_master_weight(p) self._add_accumulator(self._velocity_acc_str, master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Momentum optimizer." @@ -197,23 +215,42 @@ class Momentum(Optimizer): def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - velocity_acc = self._get_accumulator(self._velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._velocity_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) if framework._non_static_mode(): _, _, _ = _legacy_C_ops.momentum( - param_and_grad[0], param_and_grad[1], velocity_acc, lr, - master_weight, param_and_grad[0], velocity_acc, master_weight, - 'mu', self._momentum, 'use_nesterov', self._use_nesterov, - 'regularization_method', self._regularization_method, - 'regularization_coeff', self._regularization_coeff, - 'multi_precision', find_master) + param_and_grad[0], + param_and_grad[1], + velocity_acc, + lr, + master_weight, + param_and_grad[0], + velocity_acc, + master_weight, + 'mu', + self._momentum, + 'use_nesterov', + self._use_nesterov, + 'regularization_method', + self._regularization_method, + 'regularization_coeff', + self._regularization_coeff, + 'multi_precision', + find_master, + ) return None attrs = { @@ -222,17 +259,17 @@ class Momentum(Optimizer): "regularization_method": self._regularization_method, "regularization_coeff": self._regularization_coeff, "multi_precision": find_master, - "rescale_grad": self._rescale_grad + "rescale_grad": self._rescale_grad, } inputs = { "Param": [param_and_grad[0]], "Grad": [param_and_grad[1]], "Velocity": [velocity_acc], - "LearningRate": [lr] + "LearningRate": [lr], } outputs = { "ParamOut": [param_and_grad[0]], - "VelocityOut": [velocity_acc] + "VelocityOut": [velocity_acc], } if find_master: @@ -240,10 +277,12 @@ class Momentum(Optimizer): outputs["MasterParamOut"] = master_weight # create the momentum optimize op - momentum_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + momentum_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return momentum_op diff --git a/python/paddle/fluid/contrib/quantize/quantize_transpiler.py b/python/paddle/fluid/contrib/quantize/quantize_transpiler.py index de4c1004086..6e225fdbcc8 100644 --- a/python/paddle/fluid/contrib/quantize/quantize_transpiler.py +++ b/python/paddle/fluid/contrib/quantize/quantize_transpiler.py @@ -15,7 +15,11 @@ import collections import numpy as np -from paddle.fluid.framework import default_main_program, default_startup_program, program_guard +from paddle.fluid.framework import ( + default_main_program, + default_startup_program, + program_guard, +) from paddle.fluid.layer_helper import LayerHelper from paddle.fluid import unique_name from paddle.fluid import core @@ -57,13 +61,13 @@ def _original_var_name(var_name): Return the original variable name. """ if var_name.endswith('.quantized.dequantized'): - return var_name[:-len('.quantized.dequantized')] + return var_name[: -len('.quantized.dequantized')] if var_name.endswith('.quantized'): - return var_name[:-len('.quantized')] + return var_name[: -len('.quantized')] if var_name.endswith('.dequantized'): - return var_name[:-len('.dequantized')] + return var_name[: -len('.dequantized')] if var_name.endswith('.scale'): - return var_name[:-len('.scale')] + return var_name[: -len('.scale')] else: return var_name @@ -78,14 +82,15 @@ def quant(x, scale, num_bits): class QuantizeTranspiler(object): - - def __init__(self, - weight_bits=8, - activation_bits=8, - activation_quantize_type='abs_max', - weight_quantize_type='abs_max', - window_size=10000, - moving_rate=0.9): + def __init__( + self, + weight_bits=8, + activation_bits=8, + activation_quantize_type='abs_max', + weight_quantize_type='abs_max', + window_size=10000, + moving_rate=0.9, + ): """ Convert and rewrite the fluid Program according to weight and activation quantization type. @@ -123,12 +128,14 @@ class QuantizeTranspiler(object): raise ValueError( "Unknown weight_quantize_type: '%s'. It can only be ", "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'.", - str(weight_quantize_type)) + str(weight_quantize_type), + ) if activation_quantize_type not in quant_type: raise ValueError( "Unknown activation_quantize_type : '%s'. It can only be ", "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'.", - str(activation_quantize_type)) + str(activation_quantize_type), + ) self.weight_quantize_type = weight_quantize_type self.activation_quantize_type = activation_quantize_type @@ -137,8 +144,9 @@ class QuantizeTranspiler(object): self.moving_rate = moving_rate self.helper = LayerHelper(self.__class__.__name__) self.fake_quant_op_types = [ - 'fake_quantize_abs_max', 'fake_quantize_range_abs_max', - 'fake_quantize_moving_average_abs_max' + 'fake_quantize_abs_max', + 'fake_quantize_range_abs_max', + 'fake_quantize_moving_average_abs_max', ] self.fake_dequant_op_types = ['fake_dequantize_max_abs'] self.is_test = None @@ -157,8 +165,11 @@ class QuantizeTranspiler(object): """ self.is_test = False program = default_main_program() if program is None else program - startup_program = default_startup_program() if startup_program is \ - None else startup_program + startup_program = ( + default_startup_program() + if startup_program is None + else startup_program + ) # marked the variable which has been quantized and dequantized. dequanted_vars = [ @@ -173,20 +184,28 @@ class QuantizeTranspiler(object): block_id = block.idx # insert quant op and dequant op for name in op.input_arg_names: - #if share input between ops + # if share input between ops if name in dequanted_vars[block_id]: dequant_var = dequanted_vars[block_id][name] else: var = block.var(name) - quant_bits = self.weight_bits if var.name in params \ - else self.activation_bits - quant_type = self.weight_quantize_type if var.name \ - in params else self.activation_quantize_type + quant_bits = ( + self.weight_bits + if var.name in params + else self.activation_bits + ) + quant_type = ( + self.weight_quantize_type + if var.name in params + else self.activation_quantize_type + ) quant_var, scale_var = self._insert_quant_op( - block, idx, var, quant_bits, quant_type) + block, idx, var, quant_bits, quant_type + ) dequant_var = self._insert_dequant_op( - block, idx + 1, quant_var, scale_var, quant_bits) + block, idx + 1, quant_var, scale_var, quant_bits + ) dequanted_vars[block_id][name] = dequant_var # rename the forward op inputs op._rename_input(name, dequant_var.name) @@ -200,8 +219,9 @@ class QuantizeTranspiler(object): op._rename_input(name, dequant_var.name) no_dequanted_input_vars = False if no_dequanted_input_vars: - raise ValueError("There is no dequanted inputs for op %s." % - (op.type)) + raise ValueError( + "There is no dequanted inputs for op %s." % (op.type) + ) with program_guard(program, startup_program): self._create_global_step() @@ -217,8 +237,10 @@ class QuantizeTranspiler(object): _transpile_backward(block, op) def _create_global_step(self): - if self.weight_quantize_type == 'range_abs_max' or \ - self.activation_quantize_type == 'range_abs_max': + if ( + self.weight_quantize_type == 'range_abs_max' + or self.activation_quantize_type == 'range_abs_max' + ): self.global_step = autoincreased_step_counter() def freeze_program(self, program, place, scope=None): @@ -263,7 +285,7 @@ class QuantizeTranspiler(object): max_range = None scale_var = None for name in op.input_arg_names: - #rename input name of the op to the input name of last op which has be removed + # rename input name of the op to the input name of last op which has be removed if name in op_in_rename_map[block_id]: op._rename_input(name, op_in_rename_map[block_id][name]) @@ -278,23 +300,25 @@ class QuantizeTranspiler(object): scale_var = scale_v if len(op.output_arg_names) != 1: - raise ValueError("Only support one output, but op %s has" - " more than one output." % (op.type)) + raise ValueError( + "Only support one output, but op %s has" + " more than one output." % (op.type) + ) out_var = block.var(op.output_arg_names[0]) - dequant_var = block.create_var(name=_dequantized_var_name( - out_var.name), - type=out_var.type, - shape=out_var.shape, - dtype=out_var.dtype) + dequant_var = block.create_var( + name=_dequantized_var_name(out_var.name), + type=out_var.type, + shape=out_var.shape, + dtype=out_var.dtype, + ) # insert fake_dequantize_op - dequant_op = block._insert_op(idx + 1, - type="fake_dequantize_max_abs", - attrs={'max_range': float(max_range)}, - inputs={ - "X": out_var, - 'Scale': scale_var - }, - outputs={"Out": dequant_var}) + dequant_op = block._insert_op( + idx + 1, + type="fake_dequantize_max_abs", + attrs={'max_range': float(max_range)}, + inputs={"X": out_var, 'Scale': scale_var}, + outputs={"Out": dequant_var}, + ) op_out_rename_map[block_id][out_var.name] = dequant_var.name return dequant_var @@ -315,8 +339,9 @@ class QuantizeTranspiler(object): # input of the followed ops(of fc/conv) to the dquant_op for name in op.input_arg_names: if name in op_out_rename_map[block_id]: - op._rename_input(name, - op_out_rename_map[block_id][name]) + op._rename_input( + name, op_out_rename_map[block_id][name] + ) if op_type in self.fake_quant_op_types: in_arg_name = op.input('X')[0] @@ -346,7 +371,7 @@ class QuantizeTranspiler(object): # remove the unused var in ProgramDesc self._remove_unused_var(program) - #program = program.clone() + # program = program.clone() def convert_to_int8(self, program, place, scope=None): scope = global_scope() if scope is None else scope @@ -363,7 +388,8 @@ class QuantizeTranspiler(object): name=int8_var_name.encode('ascii'), type=var.type, dtype=core.VarDesc.VarType.INT8, - shape=var.shape) + shape=var.shape, + ) tensor = _load_var(var.name) @@ -392,7 +418,7 @@ class QuantizeTranspiler(object): for op in block.ops: args += op.input_arg_names args += op.output_arg_names - args = list(set(args)) #vals of all left ops + args = list(set(args)) # vals of all left ops var_names = block.vars.keys() # all vals sub_block_remove_vars = [] for var in var_names: @@ -406,39 +432,45 @@ class QuantizeTranspiler(object): block._remove_var(v) def _insert_quant_abs_max_op(self, block, idx, var, quant_bits): - """Insert fake_quantize_abs_max op. - """ - quant_var = block.create_var(name=_quantized_var_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) - scale = block.create_var(name=_quantized_scale_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) - quant_op = block._insert_op(idx, - type='fake_quantize_abs_max', - attrs={'bit_length': quant_bits}, - inputs={'X': var}, - outputs={ - 'Out': quant_var, - 'OutScale': scale - }) + """Insert fake_quantize_abs_max op.""" + quant_var = block.create_var( + name=_quantized_var_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) + scale = block.create_var( + name=_quantized_scale_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) + quant_op = block._insert_op( + idx, + type='fake_quantize_abs_max', + attrs={'bit_length': quant_bits}, + inputs={'X': var}, + outputs={'Out': quant_var, 'OutScale': scale}, + ) return quant_var, scale def _insert_quant_range_abs_max_op(self, block, idx, var, quant_bits): - """Insert fake_quantize_range_abs_max - """ - quant_var = block.create_var(name=_quantized_var_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) - scale = self.helper.create_parameter(attr=ParamAttr( - name=_quantized_scale_name(var.name), - initializer=Constant(0.001), - trainable=False), - shape=[1], - dtype=var.dtype) + """Insert fake_quantize_range_abs_max""" + quant_var = block.create_var( + name=_quantized_var_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) + scale = self.helper.create_parameter( + attr=ParamAttr( + name=_quantized_scale_name(var.name), + initializer=Constant(0.001), + trainable=False, + ), + shape=[1], + dtype=var.dtype, + ) scale.stop_gradient = True ins = {'X': var, 'InScale': scale} @@ -449,9 +481,11 @@ class QuantizeTranspiler(object): name=unique_name.generate('scales'), persistable=True, dtype=var.dtype, - shape=[self.window_size]) - self.helper.set_variable_initializer(scales, - initializer=Constant(value=0)) + shape=[self.window_size], + ) + self.helper.set_variable_initializer( + scales, initializer=Constant(value=0) + ) ins['Iter'] = self.global_step outs['OutScales'] = scales @@ -459,45 +493,56 @@ class QuantizeTranspiler(object): attrs = { 'window_size': self.window_size, 'bit_length': quant_bits, - 'is_test': self.is_test + 'is_test': self.is_test, } - quant_op = block._insert_op(idx, - type='fake_quantize_range_abs_max', - attrs=attrs, - inputs=ins, - outputs=outs) + quant_op = block._insert_op( + idx, + type='fake_quantize_range_abs_max', + attrs=attrs, + inputs=ins, + outputs=outs, + ) return quant_var, scale - def _insert_quant_moving_average_abs_max_op(self, block, idx, var, - quant_bits): - """Insert fake_quantize_moving_average_abs_max - """ - quant_var = block.create_var(name=_quantized_var_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) + def _insert_quant_moving_average_abs_max_op( + self, block, idx, var, quant_bits + ): + """Insert fake_quantize_moving_average_abs_max""" + quant_var = block.create_var( + name=_quantized_var_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) state = self.helper.create_global_variable( name=unique_name.generate('state'), persistable=True, dtype=var.dtype, - shape=[1]) - self.helper.set_variable_initializer(state, - initializer=Constant(value=1)) + shape=[1], + ) + self.helper.set_variable_initializer( + state, initializer=Constant(value=1) + ) accum = self.helper.create_global_variable( name=unique_name.generate('accum'), persistable=True, dtype=var.dtype, - shape=[1]) - self.helper.set_variable_initializer(accum, - initializer=Constant(value=1)) - scale = self.helper.create_parameter(attr=ParamAttr( - name=_quantized_scale_name(var.name), - initializer=Constant(0.001), - trainable=False), - shape=[1], - dtype=var.dtype) + shape=[1], + ) + self.helper.set_variable_initializer( + accum, initializer=Constant(value=1) + ) + scale = self.helper.create_parameter( + attr=ParamAttr( + name=_quantized_scale_name(var.name), + initializer=Constant(0.001), + trainable=False, + ), + shape=[1], + dtype=var.dtype, + ) scale.stop_gradient = True ins = {'X': var, 'InScale': scale} @@ -511,14 +556,16 @@ class QuantizeTranspiler(object): attrs = { 'bit_length': quant_bits, 'moving_rate': self.moving_rate, - 'is_test': self.is_test + 'is_test': self.is_test, } - quant_op = block._insert_op(idx, - type='fake_quantize_moving_average_abs_max', - attrs=attrs, - inputs=ins, - outputs=outs) + quant_op = block._insert_op( + idx, + type='fake_quantize_moving_average_abs_max', + attrs=attrs, + inputs=ins, + outputs=outs, + ) return quant_var, scale @@ -529,28 +576,31 @@ class QuantizeTranspiler(object): if quant_type == 'abs_max': return self._insert_quant_abs_max_op(block, idx, var, quant_bits) elif quant_type == 'range_abs_max': - return self._insert_quant_range_abs_max_op(block, idx, var, - quant_bits) + return self._insert_quant_range_abs_max_op( + block, idx, var, quant_bits + ) elif quant_type == 'moving_average_abs_max': return self._insert_quant_moving_average_abs_max_op( - block, idx, var, quant_bits) + block, idx, var, quant_bits + ) def _insert_dequant_op(self, block, idx, var, scale, quant_bits): """ Insert fake_quantize_op """ - dequant_var = block.create_var(name=_dequantized_var_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) + dequant_var = block.create_var( + name=_dequantized_var_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) # insert fake_dequantize_op max_range = (1 << (quant_bits - 1)) - 1 - dequant_op = block._insert_op(idx, - type="fake_dequantize_max_abs", - attrs={'max_range': float(max_range)}, - inputs={ - "X": var, - 'Scale': scale - }, - outputs={"Out": dequant_var}) + dequant_op = block._insert_op( + idx, + type="fake_dequantize_max_abs", + attrs={'max_range': float(max_range)}, + inputs={"X": var, 'Scale': scale}, + outputs={"Out": dequant_var}, + ) return dequant_var diff --git a/python/paddle/fluid/contrib/slim/quantization/adaround.py b/python/paddle/fluid/contrib/slim/quantization/adaround.py index 04d894b055d..9dd00ddadc6 100644 --- a/python/paddle/fluid/contrib/slim/quantization/adaround.py +++ b/python/paddle/fluid/contrib/slim/quantization/adaround.py @@ -20,44 +20,52 @@ import logging import paddle.fluid as fluid from ....log_helper import get_logger -from .utils import load_variable_data, set_variable_data, stable_sigmoid, quant_tensor, dequant_tensor, _channelwise_quant_axis1_ops, calculate_quant_cos_error, bias_correction_w - -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +from .utils import ( + load_variable_data, + set_variable_data, + stable_sigmoid, + quant_tensor, + dequant_tensor, + _channelwise_quant_axis1_ops, + calculate_quant_cos_error, + bias_correction_w, +) + +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) GAMMA = -0.1 ZETA = 1.1 def compute_soft_rounding(alpha_v): - return fluid.layers.clip(fluid.layers.sigmoid(alpha_v) * (ZETA - GAMMA) + - GAMMA, - min=0, - max=1) + return fluid.layers.clip( + fluid.layers.sigmoid(alpha_v) * (ZETA - GAMMA) + GAMMA, min=0, max=1 + ) def compute_soft_rounding_np(alpha_v): - return np.clip(stable_sigmoid(alpha_v) * (ZETA - GAMMA) + GAMMA, - a_min=0, - a_max=1) + return np.clip( + stable_sigmoid(alpha_v) * (ZETA - GAMMA) + GAMMA, a_min=0, a_max=1 + ) class AdaRoundLoss(object): - def __init__(self, reg_param=0.01, default_beta_range=(20, 2)): self.default_reg_param = reg_param self.default_beta_range = default_beta_range def compute_recon_loss(self, ada_quantized_output, orig_output): - square_cost = fluid.layers.square_error_cost(ada_quantized_output, - orig_output) + square_cost = fluid.layers.square_error_cost( + ada_quantized_output, orig_output + ) recon_loss = fluid.layers.reduce_mean( - fluid.layers.reduce_sum(square_cost, dim=-1)) + fluid.layers.reduce_sum(square_cost, dim=-1) + ) return recon_loss def compute_round_loss(self, alpha_v, warm_start, beta): - def round_loss_fn(): # compute rectified sigmoid of parameter 'alpha' which maps it between zero and one h_v = compute_soft_rounding(alpha_v) @@ -65,8 +73,9 @@ class AdaRoundLoss(object): # calculate regularization term - which ensures parameter to converge to exactly zeros and ones # at the end of optimization reg_term = fluid.layers.reduce_sum( - -fluid.layers.pow(fluid.layers.abs(2 * h_v - 1), factor=beta) + - 1) + -fluid.layers.pow(fluid.layers.abs(2 * h_v - 1), factor=beta) + + 1 + ) # calculate the rounding loss round_loss = self.default_reg_param * reg_term @@ -74,8 +83,12 @@ class AdaRoundLoss(object): return round_loss round_loss = fluid.layers.cond( - warm_start, lambda: fluid.layers.fill_constant( - shape=[1], dtype='float32', value=0.0), round_loss_fn) + warm_start, + lambda: fluid.layers.fill_constant( + shape=[1], dtype='float32', value=0.0 + ), + round_loss_fn, + ) return round_loss @@ -88,29 +101,32 @@ class AdaRoundLoss(object): warm_start_end_iter = warm_start * max_iter # compute relative iteration of current iteration - rel_iter = (cur_iter - warm_start_end_iter) / (max_iter - - warm_start_end_iter) - beta = end_beta + 0.5 * (start_beta - - end_beta) * (1 + np.cos(rel_iter * np.pi)) + rel_iter = (cur_iter - warm_start_end_iter) / ( + max_iter - warm_start_end_iter + ) + beta = end_beta + 0.5 * (start_beta - end_beta) * ( + 1 + np.cos(rel_iter * np.pi) + ) return beta class AdaRound(object): - - def __init__(self, - scale, - weight_tensor, - scope=None, - weight_var_name=None, - weight_op_type=None, - is_train=True, - num_iterations=1000): + def __init__( + self, + scale, + weight_tensor, + scope=None, + weight_var_name=None, + weight_op_type=None, + is_train=True, + num_iterations=1000, + ): self.is_train = is_train self.num_iterations = num_iterations self.warm_start = 0.1 self.weight_bits = 8 - self.offset = 0. # zero-point offset + self.offset = 0.0 # zero-point offset self.adaround_loss = AdaRoundLoss() self.ori_weight_tensor = weight_tensor self.scale = scale @@ -134,19 +150,23 @@ class AdaRound(object): shape=alpha.shape, dtype="float32", name=var_name + ".alpha", - default_initializer=fluid.initializer.NumpyArrayInitializer(alpha)) - - def _calculate_output_with_adarounded_weights(self, program, place, exe, - data, fp32_fetch_list, - weight_tensor_dequant): - set_variable_data(self.scope, place, self.weight_var_name, - weight_tensor_dequant) - - adaround_out_tensor = exe.run(program=program, - feed=data, - fetch_list=[fp32_fetch_list], - return_numpy=True, - scope=self.scope) + default_initializer=fluid.initializer.NumpyArrayInitializer(alpha), + ) + + def _calculate_output_with_adarounded_weights( + self, program, place, exe, data, fp32_fetch_list, weight_tensor_dequant + ): + set_variable_data( + self.scope, place, self.weight_var_name, weight_tensor_dequant + ) + + adaround_out_tensor = exe.run( + program=program, + feed=data, + fetch_list=[fp32_fetch_list], + return_numpy=True, + scope=self.scope, + ) return adaround_out_tensor def _calculate_quant_weight(self): @@ -154,9 +174,11 @@ class AdaRound(object): h_alpha = compute_soft_rounding_np(np_alpha) # Scale the tensor - tensor_scale = quant_tensor(self.ori_weight_tensor.copy(), - self.scale, - quant_axis=self.quant_axis) + tensor_scale = quant_tensor( + self.ori_weight_tensor.copy(), + self.scale, + quant_axis=self.quant_axis, + ) weight_tensor = np.floor(tensor_scale) @@ -168,10 +190,11 @@ class AdaRound(object): weight_tensor_quant = self._calculate_quant_weight() # Dequantize the tensor - weight_tensor_dequant = dequant_tensor(weight_tensor_quant + - self.offset, - self.scale, - quant_axis=self.quant_axis) + weight_tensor_dequant = dequant_tensor( + weight_tensor_quant + self.offset, + self.scale, + quant_axis=self.quant_axis, + ) return weight_tensor_dequant def update_final_weights(self): @@ -180,37 +203,42 @@ class AdaRound(object): def get_loss(self, beta, warm_start, adaround_out_tensor, orig_out_tensor): round_loss = self.adaround_loss.compute_round_loss( - self.alpha_v, warm_start, beta) + self.alpha_v, warm_start, beta + ) recon_loss = self.adaround_loss.compute_recon_loss( - adaround_out_tensor, orig_out_tensor) + adaround_out_tensor, orig_out_tensor + ) loss = round_loss + recon_loss losses = { 'loss': loss, 'round_loss': round_loss, - 'recon_loss': recon_loss + 'recon_loss': recon_loss, } return losses def update_beta_warm(self, cur_iteration): warm_start = cur_iteration < self.num_iterations * self.warm_start - beta = self.adaround_loss.compute_beta(self.num_iterations, - cur_iteration, self.warm_start) + beta = self.adaround_loss.compute_beta( + self.num_iterations, cur_iteration, self.warm_start + ) return beta, warm_start -def run_adaround(data_loader, - fp32_program, - fetch_list, - exe, - scope, - place, - quantized_op_pairs, - weight_op_pairs, - scale_dict, - num_iterations=1000, - lr=0.001, - bias_correction=False, - fast_mode=True): +def run_adaround( + data_loader, + fp32_program, + fetch_list, + exe, + scope, + place, + quantized_op_pairs, + weight_op_pairs, + scale_dict, + num_iterations=1000, + lr=0.001, + bias_correction=False, + fast_mode=True, +): fetch_op_name = fetch_list[0].name final_weight_tensor_quant_dict = {} for weight_var_name, quant_op_out_name in quantized_op_pairs.items(): @@ -224,7 +252,8 @@ def run_adaround(data_loader, if _op.type == "fetch": _op._rename_input(fetch_op_name, quant_op_out_name) fp32_fetch_list = fp32_program.global_block().var( - quant_op_out_name) + quant_op_out_name + ) fetch_op_name = quant_op_out_name # build adaround program @@ -235,29 +264,37 @@ def run_adaround(data_loader, with fluid.program_guard(train_program, startup_program): with fluid.unique_name.guard(): # initialize adaround - adaround = AdaRound(scale, - weight_var_tensor, - scope=scope, - weight_var_name=weight_var_name, - weight_op_type=weight_op_type, - num_iterations=num_iterations) - orig_out_tensor = fluid.data(name='orig_out_tensor', - shape=fp32_fetch_list.shape, - dtype='float32') - adaround_out_tensor = fluid.data(name='adaround_out_tensor', - shape=fp32_fetch_list.shape, - dtype='float32') - beta_tensor = fluid.data(name='beta', - shape=[1], - dtype='float32') - warm_start_tensor = fluid.data(name='warm_start', - shape=[1], - dtype='bool') - - train_fetches_loss = adaround.get_loss(beta_tensor, - warm_start_tensor, - adaround_out_tensor, - orig_out_tensor) + adaround = AdaRound( + scale, + weight_var_tensor, + scope=scope, + weight_var_name=weight_var_name, + weight_op_type=weight_op_type, + num_iterations=num_iterations, + ) + orig_out_tensor = fluid.data( + name='orig_out_tensor', + shape=fp32_fetch_list.shape, + dtype='float32', + ) + adaround_out_tensor = fluid.data( + name='adaround_out_tensor', + shape=fp32_fetch_list.shape, + dtype='float32', + ) + beta_tensor = fluid.data( + name='beta', shape=[1], dtype='float32' + ) + warm_start_tensor = fluid.data( + name='warm_start', shape=[1], dtype='bool' + ) + + train_fetches_loss = adaround.get_loss( + beta_tensor, + warm_start_tensor, + adaround_out_tensor, + orig_out_tensor, + ) optimizer = fluid.optimizer.Adam(learning_rate=lr) loss = train_fetches_loss['loss'] optimizer.minimize(loss) @@ -269,21 +306,32 @@ def run_adaround(data_loader, prev_start_time = start_time start_time = time.time() # run fp32 model - np_orig_out_tensor = exe.run(program=fp32_program, - feed=data, - fetch_list=[fp32_fetch_list], - return_numpy=True, - scope=scope) + np_orig_out_tensor = exe.run( + program=fp32_program, + feed=data, + fetch_list=[fp32_fetch_list], + return_numpy=True, + scope=scope, + ) - adaround_weight_tensor_dequant = adaround._calculate_adarounded_weights( + adaround_weight_tensor_dequant = ( + adaround._calculate_adarounded_weights() + ) + np_adaround_out_tensor = ( + adaround._calculate_output_with_adarounded_weights( + fp32_program, + place, + exe, + data, + fp32_fetch_list, + adaround_weight_tensor_dequant, + ) ) - np_adaround_out_tensor = adaround._calculate_output_with_adarounded_weights( - fp32_program, place, exe, data, fp32_fetch_list, - adaround_weight_tensor_dequant) # If the cosine distance of the two tensor is small, skip training - cos_error = calculate_quant_cos_error(np_orig_out_tensor[0], - np_adaround_out_tensor[0]) + cos_error = calculate_quant_cos_error( + np_orig_out_tensor[0], np_adaround_out_tensor[0] + ) if fast_mode and cos_error > 0.99: _logger.info("The cosine error is small, skip training.") break @@ -292,22 +340,30 @@ def run_adaround(data_loader, 'orig_out_tensor': np_orig_out_tensor[0], 'adaround_out_tensor': np_adaround_out_tensor[0], 'beta': beta, - 'warm_start': warm_start + 'warm_start': warm_start, } out = exe.run( train_program, feed=feed_dict, fetch_list=[v.name for v in train_fetches_loss.values()], - return_numpy=True) + return_numpy=True, + ) _logger.info( - "Iter {:d}, lr {:.5f}, loss {:.5f}, loss_round {:.5f}, loss_recon {:.5f}, time {:.5f}s" - .format(i, lr, np.mean(out[0]), np.mean(out[1]), - np.mean(out[2]), start_time - prev_start_time)) + "Iter {:d}, lr {:.5f}, loss {:.5f}, loss_round {:.5f}, loss_recon {:.5f}, time {:.5f}s".format( + i, + lr, + np.mean(out[0]), + np.mean(out[1]), + np.mean(out[2]), + start_time - prev_start_time, + ) + ) sys.stdout.flush() if i == num_iterations: break final_weight_tensor_quant_dict[ - weight_var_name] = adaround.update_final_weights() + weight_var_name + ] = adaround.update_final_weights() if bias_correction: final_weight_tensor_quant_dict[weight_var_name] = bias_correction_w( @@ -315,11 +371,16 @@ def run_adaround(data_loader, final_weight_tensor_quant_dict[weight_var_name], scale, adaround.quant_axis, - weight_bits=adaround.weight_bits) + weight_bits=adaround.weight_bits, + ) del adaround # update adarounded calibrated weights for weight_var_name in quantized_op_pairs.keys(): - set_variable_data(scope, place, weight_var_name, - final_weight_tensor_quant_dict[weight_var_name]) + set_variable_data( + scope, + place, + weight_var_name, + final_weight_tensor_quant_dict[weight_var_name], + ) diff --git a/python/paddle/fluid/contrib/slim/quantization/cal_kl_threshold.py b/python/paddle/fluid/contrib/slim/quantization/cal_kl_threshold.py index 69cd3f64061..ea3b1876f23 100644 --- a/python/paddle/fluid/contrib/slim/quantization/cal_kl_threshold.py +++ b/python/paddle/fluid/contrib/slim/quantization/cal_kl_threshold.py @@ -17,9 +17,9 @@ import math import numpy as np from ....log_helper import get_logger -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) __all__ = ['cal_kl_threshold'] @@ -38,11 +38,13 @@ def expand_quantized_bins(quantized_bins, reference_bins): if zero_count == num_merged_bins: avg_bin_ele = 0 else: - avg_bin_ele = quantized_bins[idx] / (num_merged_bins - zero_count + - 0.0) + avg_bin_ele = quantized_bins[idx] / ( + num_merged_bins - zero_count + 0.0 + ) for idx1 in range(j_start, j_end): - expanded_quantized_bins[idx1] = (0 if reference_bins[idx1] == 0 else - avg_bin_ele) + expanded_quantized_bins[idx1] = ( + 0 if reference_bins[idx1] == 0 else avg_bin_ele + ) j_start += num_merged_bins j_end += num_merged_bins if (idx + 1) == len(quantized_bins) - 1: @@ -65,8 +67,12 @@ def safe_entropy(reference_distr_P, P_sum, candidate_distr_Q, Q_sum): tmp_sum2 += 0 else: if q_idx == 0: - _logger.error("Fatal error!, idx = " + str(idx) + - " qindex = 0! p_idx = " + str(p_idx)) + _logger.error( + "Fatal error!, idx = " + + str(idx) + + " qindex = 0! p_idx = " + + str(p_idx) + ) tmp_sum1 += p_idx * (math.log(Q_sum * p_idx)) tmp_sum2 += p_idx * (math.log(P_sum * q_idx)) return (tmp_sum1 - tmp_sum2) / P_sum @@ -84,7 +90,7 @@ def cal_kl_threshold(hist, bin_width, bits): assert hist.ndim == 1 hist_bins = hist.shape[0] starting_iter = int((hist_bins - 1) * 0.5) - quant_range = 2**(bits - 1) - 1 + quant_range = 2 ** (bits - 1) - 1 P_sum = np.sum(np.array(hist).ravel()) min_kl_divergence = 0 @@ -105,16 +111,19 @@ def cal_kl_threshold(hist, bin_width, bits): j_end = num_merged_bins for idx in range(quant_range): candidate_distr_Q_quantized[idx] = sum( - candidate_distr_Q[j_start:j_end]) + candidate_distr_Q[j_start:j_end] + ) j_start += num_merged_bins j_end += num_merged_bins if (idx + 1) == quant_range - 1: j_end = i - candidate_distr_Q = expand_quantized_bins(candidate_distr_Q_quantized, - reference_distr_bins) + candidate_distr_Q = expand_quantized_bins( + candidate_distr_Q_quantized, reference_distr_bins + ) Q_sum = sum(candidate_distr_Q) - kl_divergence = safe_entropy(reference_distr_P, P_sum, - candidate_distr_Q, Q_sum) + kl_divergence = safe_entropy( + reference_distr_P, P_sum, candidate_distr_Q, Q_sum + ) if not kl_inited: min_kl_divergence = kl_divergence min_kl_index = i diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/fuse_utils.py b/python/paddle/fluid/contrib/slim/quantization/imperative/fuse_utils.py index 4ae949bf0fe..b01ceddd1ca 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/fuse_utils.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/fuse_utils.py @@ -51,20 +51,20 @@ def fuse_conv_bn(model): def fuse_layers(model, layers_to_fuse, inplace=False): ''' - fuse layers in layers_to_fuse - - Args: - model(paddle.nn.Layer): The model to be fused. - layers_to_fuse(list): The layers' names to be fused. For - example,"fuse_list = [["conv1", "bn1"], ["conv2", "bn2"]]". - A TypeError would be raised if "fuse" was set as - True but "fuse_list" was None. - Default: None. - inplace(bool): Whether apply fusing to the input model. - Default: False. - - Return - fused_model(paddle.nn.Layer): The fused model. + fuse layers in layers_to_fuse + + Args: + model(paddle.nn.Layer): The model to be fused. + layers_to_fuse(list): The layers' names to be fused. For + example,"fuse_list = [["conv1", "bn1"], ["conv2", "bn2"]]". + A TypeError would be raised if "fuse" was set as + True but "fuse_list" was None. + Default: None. + inplace(bool): Whether apply fusing to the input model. + Default: False. + + Return + fused_model(paddle.nn.Layer): The fused model. ''' if inplace == False: model = copy.deepcopy(model) @@ -78,12 +78,14 @@ def _fuse_layers(model, layers_list): layer_list = [] for layer_name in layers_list: parent_layer, sub_name = utils.find_parent_layer_and_sub_name( - model, layer_name) + model, layer_name + ) layer_list.append(getattr(parent_layer, sub_name)) new_layers = _fuse_func(layer_list) for i, item in enumerate(layers_list): parent_layer, sub_name = utils.find_parent_layer_and_sub_name( - model, item) + model, item + ) setattr(parent_layer, sub_name, new_layers[i]) @@ -109,10 +111,13 @@ def _fuse_func(layer_list): def _fuse_conv_bn(conv, bn): '''fuse conv and bn for train or eval''' - assert(conv.training == bn.training),\ - "Conv and BN both must be in the same mode (train or eval)." + assert ( + conv.training == bn.training + ), "Conv and BN both must be in the same mode (train or eval)." if conv.training: - assert bn._num_features == conv._out_channels, 'Output channel of Conv2d must match num_features of BatchNorm2d' + assert ( + bn._num_features == conv._out_channels + ), 'Output channel of Conv2d must match num_features of BatchNorm2d' raise NotImplementedError else: return _fuse_conv_bn_eval(conv, bn) @@ -120,17 +125,23 @@ def _fuse_conv_bn(conv, bn): def _fuse_conv_bn_eval(conv, bn): '''fuse conv and bn for eval''' - assert (not (conv.training or bn.training)), "Fusion only for eval!" + assert not (conv.training or bn.training), "Fusion only for eval!" fused_conv = copy.deepcopy(conv) - fused_weight, fused_bias = _fuse_conv_bn_weights(fused_conv.weight, - fused_conv.bias, bn._mean, - bn._variance, bn._epsilon, - bn.weight, bn.bias) + fused_weight, fused_bias = _fuse_conv_bn_weights( + fused_conv.weight, + fused_conv.bias, + bn._mean, + bn._variance, + bn._epsilon, + bn.weight, + bn.bias, + ) fused_conv.weight.set_value(fused_weight) if fused_conv.bias is None: fused_conv.bias = paddle.create_parameter( - shape=[fused_conv._out_channels], is_bias=True, dtype=bn.bias.dtype) + shape=[fused_conv._out_channels], is_bias=True, dtype=bn.bias.dtype + ) fused_conv.bias.set_value(fused_bias) return fused_conv @@ -144,19 +155,22 @@ def _fuse_conv_bn_weights(conv_w, conv_b, bn_rm, bn_rv, bn_eps, bn_w, bn_b): if bn_b is None: bn_b = paddle.zeros_like(bn_rm) bn_var_rsqrt = paddle.rsqrt(bn_rv + bn_eps) - conv_w = conv_w * \ - (bn_w * bn_var_rsqrt).reshape([-1] + [1] * (len(conv_w.shape) - 1)) + conv_w = conv_w * (bn_w * bn_var_rsqrt).reshape( + [-1] + [1] * (len(conv_w.shape) - 1) + ) conv_b = (conv_b - bn_rm) * bn_var_rsqrt * bn_w + bn_b return conv_w, conv_b def _fuse_linear_bn(linear, bn): '''fuse linear and bn''' - assert (linear.training == bn.training),\ - "Linear and BN both must be in the same mode (train or eval)." + assert ( + linear.training == bn.training + ), "Linear and BN both must be in the same mode (train or eval)." if linear.training: - assert bn._num_features == linear.weight.shape[ - 1], 'Output channel of Linear must match num_features of BatchNorm' + assert ( + bn._num_features == linear.weight.shape[1] + ), 'Output channel of Linear must match num_features of BatchNorm' raise NotImplementedError else: return _fuse_linear_bn_eval(linear, bn) @@ -164,26 +178,32 @@ def _fuse_linear_bn(linear, bn): def _fuse_linear_bn_eval(linear, bn): '''fuse linear and bn for eval''' - assert (not (linear.training or bn.training)), "Fusion only for eval!" + assert not (linear.training or bn.training), "Fusion only for eval!" fused_linear = copy.deepcopy(linear) - fused_weight, fused_bias = _fuse_linear_bn_weights(fused_linear.weight, - fused_linear.bias, - bn._mean, bn._variance, - bn._epsilon, bn.weight, - bn.bias) + fused_weight, fused_bias = _fuse_linear_bn_weights( + fused_linear.weight, + fused_linear.bias, + bn._mean, + bn._variance, + bn._epsilon, + bn.weight, + bn.bias, + ) fused_linear.weight.set_value(fused_weight) if fused_linear.bias is None: fused_linear.bias = paddle.create_parameter( shape=[fused_linear.weight.shape[1]], is_bias=True, - dtype=bn.bias.dtype) + dtype=bn.bias.dtype, + ) fused_linear.bias.set_value(fused_bias) return fused_linear -def _fuse_linear_bn_weights(linear_w, linear_b, bn_rm, bn_rv, bn_eps, bn_w, - bn_b): +def _fuse_linear_bn_weights( + linear_w, linear_b, bn_rm, bn_rv, bn_eps, bn_w, bn_b +): '''fuse weights and bias of linear and bn''' if linear_b is None: linear_b = paddle.zeros_like(bn_rm) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq.py b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq.py index 23affb658cc..9c028736d68 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq.py @@ -31,9 +31,9 @@ from .ptq_registry import PTQRegistry __all__ = ['ImperativePTQ'] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class ImperativePTQ(object): @@ -75,17 +75,20 @@ class ImperativePTQ(object): Return quantized_model(paddle.nn.Layer): The quantized model. """ - assert isinstance(model, paddle.nn.Layer), \ - "The model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The model must be the instance of paddle.nn.Layer." if not inplace: model = copy.deepcopy(model) if fuse: model.eval() model = fuse_utils.fuse_layers(model, fuse_list) for name, layer in model.named_sublayers(): - if PTQRegistry.is_supported_layer(layer) \ - and utils.is_leaf_layer(layer) \ - and not self._is_skip_layer(layer): + if ( + PTQRegistry.is_supported_layer(layer) + and utils.is_leaf_layer(layer) + and not self._is_skip_layer(layer) + ): # Add quant config quant_config = copy.deepcopy(self._quant_config) @@ -98,7 +101,8 @@ class ImperativePTQ(object): quant_hook_handle = layer.register_forward_post_hook(hook) quant_config.quant_hook_handle = quant_hook_handle layer._forward_post_hooks.move_to_end( - quant_hook_handle._hook_id, last=False) + quant_hook_handle._hook_id, last=False + ) return model @@ -133,8 +137,9 @@ class ImperativePTQ(object): None """ - assert isinstance(model, paddle.nn.Layer), \ - "The model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The model must be the instance of paddle.nn.Layer." # Convert and save dygraph quantized model self._convert(model) @@ -156,12 +161,16 @@ class ImperativePTQ(object): model_filename = basename + INFER_MODEL_SUFFIX params_filename = basename + INFER_PARAMS_SUFFIX - [infer_program, feed_target_names, - fetch_targets] = (paddle.fluid.io.load_inference_model( - dirname=dirname, - executor=exe, - model_filename=model_filename, - params_filename=params_filename)) + [ + infer_program, + feed_target_names, + fetch_targets, + ] = paddle.fluid.io.load_inference_model( + dirname=dirname, + executor=exe, + model_filename=model_filename, + params_filename=params_filename, + ) # Process inference program self._clean_up(infer_program) @@ -169,13 +178,15 @@ class ImperativePTQ(object): self._remove_scale_op(infer_program) # Save final program - paddle.fluid.io.save_inference_model(dirname=dirname, - feeded_var_names=feed_target_names, - target_vars=fetch_targets, - executor=exe, - main_program=infer_program.clone(), - model_filename=model_filename, - params_filename=params_filename) + paddle.fluid.io.save_inference_model( + dirname=dirname, + feeded_var_names=feed_target_names, + target_vars=fetch_targets, + executor=exe, + main_program=infer_program.clone(), + model_filename=model_filename, + params_filename=params_filename, + ) if is_dynamic_mode: paddle.disable_static() @@ -213,8 +224,9 @@ class ImperativePTQ(object): Returns: None """ - assert isinstance(model, paddle.nn.Layer), \ - "The input model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The input model must be the instance of paddle.nn.Layer." total_num = 0 cur_num = 0 @@ -226,8 +238,9 @@ class ImperativePTQ(object): if self._is_quant_layer(sub_layer): cur_num += 1 if cur_num % 5 == 0: - _logger.info("Process the %s / %s layer" % - (cur_num, total_num)) + _logger.info( + "Process the %s / %s layer" % (cur_num, total_num) + ) quant_config = sub_layer._quant_config @@ -236,7 +249,7 @@ class ImperativePTQ(object): quant_config.out_act_quantizer.cal_thresholds() if PTQRegistry.is_simulated_quant_layer(sub_layer): - weights = (sub_layer.weight, ) + weights = (sub_layer.weight,) quant_config.wt_quantizer.sample_data(sub_layer, weights) quant_config.wt_quantizer.cal_thresholds() @@ -250,8 +263,9 @@ class ImperativePTQ(object): Returns: None """ - assert isinstance(sub_layer, paddle.nn.Layer), \ - "The input model must be the instance of paddle.nn.Layer." + assert isinstance( + sub_layer, paddle.nn.Layer + ), "The input model must be the instance of paddle.nn.Layer." layer_info = PTQRegistry.layer_info(sub_layer) @@ -272,12 +286,14 @@ class ImperativePTQ(object): Returns: None """ - assert isinstance(model, paddle.nn.Layer), \ - "The input model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The input model must be the instance of paddle.nn.Layer." for name, sub_layer in model.named_sublayers(): - if self._is_quant_layer(sub_layer) \ - and PTQRegistry.is_simulated_quant_layer(sub_layer): + if self._is_quant_layer( + sub_layer + ) and PTQRegistry.is_simulated_quant_layer(sub_layer): quant_config = sub_layer._quant_config assert quant_config.enable_in_act_quantizer == True @@ -303,15 +319,17 @@ class ImperativePTQ(object): "activation_bits": in_act_quantizer.quant_bits, } - quant_layer = quant_layers.__dict__[quant_layer_name](sub_layer, - **kwargs) + quant_layer = quant_layers.__dict__[quant_layer_name]( + sub_layer, **kwargs + ) # save the input thresholds assert hasattr(quant_layer, "_fake_quant_input") assert hasattr(quant_layer._fake_quant_input, "_scale") assert len(in_act_quantizer.thresholds) == 1 - input_threshold = np.array([in_act_quantizer.thresholds[0]], - dtype=np.float32) + input_threshold = np.array( + [in_act_quantizer.thresholds[0]], dtype=np.float32 + ) quant_layer._fake_quant_input._scale.set_value(input_threshold) assert hasattr(quant_layer, "_fake_quant_weight") @@ -319,20 +337,24 @@ class ImperativePTQ(object): assert len(wt_quantizer.thresholds) == 1 weight_threshold = wt_quantizer.thresholds[0] if isinstance(weight_threshold, list): - weight_threshold = np.array(weight_threshold, - dtype=np.float32) + weight_threshold = np.array( + weight_threshold, dtype=np.float32 + ) else: - weight_threshold = np.array([weight_threshold], - dtype=np.float32) + weight_threshold = np.array( + [weight_threshold], dtype=np.float32 + ) quant_layer._fake_quant_weight._scale.set_value( - weight_threshold) + weight_threshold + ) # save the output thresholds self._save_output_thresholds(quant_layer, quant_config) # replace the layer - parent_layer, sub_name = \ - utils.find_parent_layer_and_sub_name(model, name) + parent_layer, sub_name = utils.find_parent_layer_and_sub_name( + model, name + ) setattr(parent_layer, sub_name, quant_layer) def _gather_input_thresholds(self, program, scope): @@ -351,30 +373,37 @@ class ImperativePTQ(object): if previous_op is None: continue - if "quantize_dequantize" in previous_op.type or \ - previous_op.type == "moving_average_abs_max_scale": + if ( + "quantize_dequantize" in previous_op.type + or previous_op.type == "moving_average_abs_max_scale" + ): attr_name = previous_op.output('OutScale')[0] in_threshold = utils.load_variable_data(scope, attr_name) in_threshold = utils.fp_numpy_to_naive(in_threshold) argname, index = utils._get_input_name_index( - op, in_var_name) - op._set_attr(argname + str(index) + "_threshold", - in_threshold) + op, in_var_name + ) + op._set_attr( + argname + str(index) + "_threshold", in_threshold + ) op._set_attr("with_quant_attr", True) else: for out_var_name in utils._get_op_output_var_names( - previous_op): + previous_op + ): if out_var_name != in_var_name: continue argname, index = utils._get_output_name_index( - previous_op, out_var_name) + previous_op, out_var_name + ) attr_name = argname + str(index) + "_threshold" if not previous_op.has_attr(attr_name): continue threshold = previous_op.attr(attr_name) argname, index = utils._get_input_name_index( - op, in_var_name) + op, in_var_name + ) attr_name = argname + str(index) + "_threshold" op._set_attr(attr_name, threshold) op._set_attr("with_quant_attr", True) @@ -390,8 +419,11 @@ class ImperativePTQ(object): """ def _helper(op, next_op, old_attr_name, new_attr_name): - if op.has_attr(old_attr_name) and next_op.has_attr(old_attr_name) \ - and op.attr(old_attr_name) == next_op.attr(old_attr_name): + if ( + op.has_attr(old_attr_name) + and next_op.has_attr(old_attr_name) + and op.attr(old_attr_name) == next_op.attr(old_attr_name) + ): threshold = op.attr(old_attr_name) op._remove_attr(old_attr_name) next_op._remove_attr(old_attr_name) @@ -417,8 +449,8 @@ class ImperativePTQ(object): old_attr_name = argname + str(index) + "_threshold" argname, index = utils._get_output_name_index( - next_op, - next_op.output("Out")[0]) + next_op, next_op.output("Out")[0] + ) new_attr_name = argname + str(index) + "_threshold" _helper(op, next_op, old_attr_name, new_attr_name) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_hooks.py b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_hooks.py index 41c9b07195a..319beee0ed7 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_hooks.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_hooks.py @@ -23,10 +23,11 @@ def quant_forward_post_hook(layer, inputs, outputs): """ The forward_post_hook for PTQ. """ - assert hasattr(layer, '_quant_config'), \ - "The layer should have _quant_config attr" + assert hasattr( + layer, '_quant_config' + ), "The layer should have _quant_config attr" qc = layer._quant_config if qc.enable_in_act_quantizer: qc.in_act_quantizer.sample_data(layer, inputs) - qc.out_act_quantizer.sample_data(layer, (outputs, )) + qc.out_act_quantizer.sample_data(layer, (outputs,)) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_quantizer.py b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_quantizer.py index 9b751a93a0f..ba881f88efc 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_quantizer.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_quantizer.py @@ -24,9 +24,13 @@ from . import utils from ..cal_kl_threshold import cal_kl_threshold __all__ = [ - 'BaseQuantizer', 'AbsmaxQuantizer', 'PerChannelAbsmaxQuantizer', - 'KLQuantizer', 'HistQuantizer', 'SUPPORT_ACT_QUANTIZERS', - 'SUPPORT_WT_QUANTIZERS' + 'BaseQuantizer', + 'AbsmaxQuantizer', + 'PerChannelAbsmaxQuantizer', + 'KLQuantizer', + 'HistQuantizer', + 'SUPPORT_ACT_QUANTIZERS', + 'SUPPORT_WT_QUANTIZERS', ] @@ -50,25 +54,25 @@ def merge_max_value(old, new): return new -def combine_abs_max_and_hist(tensor, origin_max, origin_hist, bins, - upsample_bins): - """ - """ +def combine_abs_max_and_hist( + tensor, origin_max, origin_hist, bins, upsample_bins +): + """ """ new_max = abs_max_value(tensor) if new_max == 0.0: return origin_max, origin_hist elif origin_max == 0.0: - new_hist, _ = np.histogram(paddle.abs(tensor).numpy(), - range=(0, new_max), - bins=bins) + new_hist, _ = np.histogram( + paddle.abs(tensor).numpy(), range=(0, new_max), bins=bins + ) new_hist = new_hist.astype(np.float32) return new_max, new_hist elif new_max <= origin_max: - new_hist, _ = np.histogram(paddle.abs(tensor).numpy(), - range=(0, origin_max), - bins=bins) + new_hist, _ = np.histogram( + paddle.abs(tensor).numpy(), range=(0, origin_max), bins=bins + ) new_hist = new_hist.astype(np.float32) new_hist += origin_hist return origin_max, new_hist @@ -81,17 +85,18 @@ def combine_abs_max_and_hist(tensor, origin_max, origin_hist, bins, upsampled_hist = np.repeat(origin_hist, upsample_bins) expanded_hist = np.zeros((bins * downsampe_bins), dtype=np.float32) - expanded_hist[0:bins * upsample_bins] = upsampled_hist - cumsumed_hist = np.cumsum( - expanded_hist, dtype=np.float64)[downsampe_bins - 1::downsampe_bins] + expanded_hist[0 : bins * upsample_bins] = upsampled_hist + cumsumed_hist = np.cumsum(expanded_hist, dtype=np.float64)[ + downsampe_bins - 1 :: downsampe_bins + ] shift_cumsumed_hist = np.zeros((bins), dtype=np.float64) shift_cumsumed_hist[1:] = cumsumed_hist[0:-1] sampled_hist = (cumsumed_hist - shift_cumsumed_hist) / upsample_bins sampled_hist = sampled_hist.astype(np.float32) - new_hist, _ = np.histogram(paddle.abs(tensor).numpy(), - range=(0, new_max), - bins=bins) + new_hist, _ = np.histogram( + paddle.abs(tensor).numpy(), range=(0, new_max), bins=bins + ) new_hist = new_hist.astype(np.float32) new_hist += sampled_hist @@ -165,16 +170,16 @@ class PerChannelAbsmaxQuantizer(BaseQuantizer): ] abs_max_vals_list.append(abs_max_vals) - self.abs_max_vals = merge_max_value(self.abs_max_vals, - abs_max_vals_list) + self.abs_max_vals = merge_max_value( + self.abs_max_vals, abs_max_vals_list + ) def cal_thresholds(self): self.thresholds = self.abs_max_vals class BaseHistQuantizer(BaseQuantizer, metaclass=abc.ABCMeta): - """ - """ + """ """ def __init__(self, quant_bits=8, bins=1024, upsample_bins=64): super(BaseHistQuantizer, self).__init__(quant_bits) @@ -194,9 +199,11 @@ class BaseHistQuantizer(BaseQuantizer, metaclass=abc.ABCMeta): if abs_max_vals[idx] == 0.0: self.hists.append(None) else: - hist, _ = np.histogram(paddle.abs(tensor).numpy(), - range=(0., abs_max_vals[idx]), - bins=self.bins) + hist, _ = np.histogram( + paddle.abs(tensor).numpy(), + range=(0.0, abs_max_vals[idx]), + bins=self.bins, + ) hist = hist.astype(np.float32) self.hists.append(hist) else: @@ -205,8 +212,12 @@ class BaseHistQuantizer(BaseQuantizer, metaclass=abc.ABCMeta): for idx, tensor in enumerate(tensors): new_abs_max, new_hist = combine_abs_max_and_hist( - tensor, self.abs_max_vals[idx], self.hists[idx], self.bins, - self.upsample_bins) + tensor, + self.abs_max_vals[idx], + self.hists[idx], + self.bins, + self.upsample_bins, + ) self.abs_max_vals[idx] = new_abs_max self.hists[idx] = new_hist @@ -216,19 +227,15 @@ class BaseHistQuantizer(BaseQuantizer, metaclass=abc.ABCMeta): class HistQuantizer(BaseHistQuantizer): - """ - """ + """ """ - def __init__(self, - quant_bits=8, - bins=1024, - upsample_bins=64, - hist_percent=0.99999): + def __init__( + self, quant_bits=8, bins=1024, upsample_bins=64, hist_percent=0.99999 + ): super(HistQuantizer, self).__init__(quant_bits, bins, upsample_bins) self.hist_percent = hist_percent def cal_thresholds(self): - def _helper(abs_max, hist, percent): assert hist.ndim == 1 and percent < 1.0 hist = hist / np.sum(hist, dtype=np.float64) @@ -240,14 +247,14 @@ class HistQuantizer(BaseHistQuantizer): if self.hists[idx] is None: self.thresholds.append(self.abs_max_vals[idx]) else: - threshold = _helper(self.abs_max_vals[idx], self.hists[idx], - self.hist_percent) + threshold = _helper( + self.abs_max_vals[idx], self.hists[idx], self.hist_percent + ) self.thresholds.append(threshold) class KLQuantizer(BaseHistQuantizer): - """ - """ + """ """ def __init__(self, quant_bits=8, bins=1024, upsample_bins=64): super(KLQuantizer, self).__init__(quant_bits, bins, upsample_bins) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_registry.py b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_registry.py index a6b8033bc78..5bc7fc0c6b3 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_registry.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_registry.py @@ -48,10 +48,15 @@ PTQ_LAYERS_INFO = [ ] QUANT_LAYERS_INFO = [ - LayerInfo(paddle.nn.quant.quant_layers.QuantizedConv2D, ['Input'], - ['Filter'], ['Output']), - LayerInfo(paddle.nn.quant.quant_layers.QuantizedLinear, ['X'], ['Y'], - ['Out']), + LayerInfo( + paddle.nn.quant.quant_layers.QuantizedConv2D, + ['Input'], + ['Filter'], + ['Output'], + ), + LayerInfo( + paddle.nn.quant.quant_layers.QuantizedLinear, ['X'], ['Y'], ['Out'] + ), ] SIMULATED_LAYERS = [paddle.nn.Conv2D, paddle.nn.Linear] @@ -61,6 +66,7 @@ class PTQRegistry(object): """ Register the supported layers for PTQ and provide layers info. """ + supported_layers_map = {} registered_layers_map = {} is_inited = False @@ -89,8 +95,9 @@ class PTQRegistry(object): flag(bool): Whther the layer is supported. """ cls._init() - return layer in cls.supported_layers_map or \ - isinstance(layer, tuple(cls.supported_layers_map.keys())) + return layer in cls.supported_layers_map or isinstance( + layer, tuple(cls.supported_layers_map.keys()) + ) @classmethod def is_registered_layer(cls, layer): @@ -102,8 +109,9 @@ class PTQRegistry(object): flag(bool): Wether the layer is register layer_info. """ cls._init() - return layer in cls.registered_layers_map or \ - isinstance(layer, tuple(cls.registered_layers_map.keys())) + return layer in cls.registered_layers_map or isinstance( + layer, tuple(cls.registered_layers_map.keys()) + ) @classmethod def is_simulated_quant_layer(cls, layer): @@ -114,8 +122,9 @@ class PTQRegistry(object): Returns: flag(bool): Whther the layer is supported. """ - return layer in SIMULATED_LAYERS or \ - isinstance(layer, tuple(SIMULATED_LAYERS)) + return layer in SIMULATED_LAYERS or isinstance( + layer, tuple(SIMULATED_LAYERS) + ) @classmethod def layer_info(cls, layer): @@ -126,8 +135,9 @@ class PTQRegistry(object): Returns: layer_info(LayerInfo): The layer info of the input layer. """ - assert cls.is_registered_layer(layer), \ - "The input layer is not register." + assert cls.is_registered_layer( + layer + ), "The input layer is not register." for layer_key, layer_info in cls.registered_layers_map.items(): if layer == layer_key or isinstance(layer, layer_key): diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py b/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py index 423fb0fcd52..255e8e03a20 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py @@ -38,17 +38,20 @@ from . import fuse_utils __all__ = ['ImperativeQuantAware'] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def lazy_import_fleet(layer_name_map, fake_quant_input_layers): from paddle.distributed import fleet + layer_name_map[ - 'ColumnParallelLinear'] = fleet.meta_parallel.parallel_layers.mp_layers.ColumnParallelLinear + 'ColumnParallelLinear' + ] = fleet.meta_parallel.parallel_layers.mp_layers.ColumnParallelLinear layer_name_map[ - 'RowParallelLinear'] = fleet.meta_parallel.parallel_layers.mp_layers.RowParallelLinear + 'RowParallelLinear' + ] = fleet.meta_parallel.parallel_layers.mp_layers.RowParallelLinear fake_quant_input_layers.append(fleet.meta_parallel.RowParallelLinear) fake_quant_input_layers.append(fleet.meta_parallel.ColumnParallelLinear) return layer_name_map, fake_quant_input_layers @@ -59,22 +62,27 @@ class ImperativeQuantAware(object): Applying quantization aware training (QAT) to the dgraph model. """ - def __init__(self, - quantizable_layer_type=[ - 'Conv2D', 'Linear', 'Conv2DTranspose', - 'ColumnParallelLinear', 'RowParallelLinear' - ], - weight_quantize_type='abs_max', - activation_quantize_type='moving_average_abs_max', - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - fuse_conv_bn=False, - weight_preprocess_layer=None, - act_preprocess_layer=None, - weight_quantize_layer=None, - act_quantize_layer=None, - onnx_format=False): + def __init__( + self, + quantizable_layer_type=[ + 'Conv2D', + 'Linear', + 'Conv2DTranspose', + 'ColumnParallelLinear', + 'RowParallelLinear', + ], + weight_quantize_type='abs_max', + activation_quantize_type='moving_average_abs_max', + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + fuse_conv_bn=False, + weight_preprocess_layer=None, + act_preprocess_layer=None, + weight_quantize_layer=None, + act_quantize_layer=None, + onnx_format=False, + ): """ The constructor for ImperativeQuantAware. @@ -222,13 +230,14 @@ class ImperativeQuantAware(object): "weight_preprocess_layer": weight_preprocess_layer, "act_preprocess_layer": act_preprocess_layer, "weight_quantize_layer": weight_quantize_layer, - "act_quantize_layer": act_quantize_layer + "act_quantize_layer": act_quantize_layer, } self._quantize_inputs = ImperativeQuantizeInputs(**kwargs) self._quantize_outputs = ImperativeQuantizeOutputs( - moving_rate, activation_bits, onnx_format) + moving_rate, activation_bits, onnx_format + ) def quantize(self, model): """ @@ -278,8 +287,9 @@ class ImperativeQuantAware(object): # fake quant logical. imperative_qat.quantize(model) """ - assert isinstance(model, dygraph.Layer), \ - "The model must be the instance of dygraph.Layer." + assert isinstance( + model, dygraph.Layer + ), "The model must be the instance of dygraph.Layer." if self.fuse_conv_bn: fuse_utils.fuse_conv_bn(model) @@ -289,8 +299,9 @@ class ImperativeQuantAware(object): return model def save_quantized_model(self, layer, path, input_spec=None, **config): - self._quantize_outputs.save_quantized_model(layer, path, input_spec, - **config) + self._quantize_outputs.save_quantized_model( + layer, path, input_spec, **config + ) class ImperativeQuantizeInputs(object): @@ -299,17 +310,19 @@ class ImperativeQuantizeInputs(object): logic both for activation inputs and weight inputs. """ - def __init__(self, - quantizable_layer_type=['Conv2D', 'Linear', 'Conv2DTranspose'], - weight_quantize_type='abs_max', - activation_quantize_type='moving_average_abs_max', - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_preprocess_layer=None, - act_preprocess_layer=None, - weight_quantize_layer=None, - act_quantize_layer=None): + def __init__( + self, + quantizable_layer_type=['Conv2D', 'Linear', 'Conv2DTranspose'], + weight_quantize_type='abs_max', + activation_quantize_type='moving_average_abs_max', + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_preprocess_layer=None, + act_preprocess_layer=None, + weight_quantize_layer=None, + act_quantize_layer=None, + ): """ The constructor for ImperativeQuantizeInputs. @@ -317,48 +330,66 @@ class ImperativeQuantizeInputs(object): """ super(ImperativeQuantizeInputs, self).__init__() self.layer_name_map, self.fake_quant_input_layers = lazy_import_fleet( - utils.layer_name_map, utils.fake_quant_input_layers) + utils.layer_name_map, utils.fake_quant_input_layers + ) self._quantizable_layer_type = tuple( - self.layer_name_map[layer] if layer in - self.layer_name_map else layer for layer in quantizable_layer_type) + self.layer_name_map[layer] + if layer in self.layer_name_map + else layer + for layer in quantizable_layer_type + ) for layer in self._quantizable_layer_type: - assert not isinstance(layer, str) \ - and layer in self.fake_quant_input_layers, \ - "%s is unspported to be quantized." % layer + assert ( + not isinstance(layer, str) + and layer in self.fake_quant_input_layers + ), ("%s is unspported to be quantized." % layer) quantize_type = { - 'abs_max', 'moving_average_abs_max', 'channel_wise_abs_max', - 'lsq_weight', 'channel_wise_lsq_weight' + 'abs_max', + 'moving_average_abs_max', + 'channel_wise_abs_max', + 'lsq_weight', + 'channel_wise_lsq_weight', } act_quantize_type = {'moving_average_abs_max', 'lsq_act'} - assert weight_quantize_type != 'moving_average_abs_max' \ - and weight_quantize_type in quantize_type, \ - "Unsupported weight_quantize_type: %s. It can only " \ + assert ( + weight_quantize_type != 'moving_average_abs_max' + and weight_quantize_type in quantize_type + ), ( + "Unsupported weight_quantize_type: %s. It can only " "be abs_max or channel_wise_abs_max." % weight_quantize_type + ) # TODO (jc): activation_quantize_type supports range_abs_max - assert activation_quantize_type in act_quantize_type, \ - "Unsupported activation_quantize_type: %s. It can " \ - "only be moving_average_abs_max or lsq_act now." \ + assert activation_quantize_type in act_quantize_type, ( + "Unsupported activation_quantize_type: %s. It can " + "only be moving_average_abs_max or lsq_act now." % activation_quantize_type - - bits_check = lambda bits: isinstance(bits, int) \ - and bits >= 0 and bits <= 16 - assert bits_check(weight_bits), \ - "weight_bits should be 1, 2,... or 16." - assert bits_check(activation_bits), \ - "activation_bits should be 1, 2,... or 16." - - layer_check = lambda method: method is None or \ - issubclass(method, dygraph.layers.Layer) - assert layer_check(weight_preprocess_layer), \ - "weight_preprocess should be nn.Layer." - assert layer_check(act_preprocess_layer), \ - "act_preprocess should be nn.Layer." - assert layer_check(weight_quantize_layer), \ - "weight_quantize should be nn.Layer." - assert layer_check(act_quantize_layer), \ - "act_quantize should be nn.Layer." + ) + + bits_check = ( + lambda bits: isinstance(bits, int) and bits >= 0 and bits <= 16 + ) + assert bits_check(weight_bits), "weight_bits should be 1, 2,... or 16." + assert bits_check( + activation_bits + ), "activation_bits should be 1, 2,... or 16." + + layer_check = lambda method: method is None or issubclass( + method, dygraph.layers.Layer + ) + assert layer_check( + weight_preprocess_layer + ), "weight_preprocess should be nn.Layer." + assert layer_check( + act_preprocess_layer + ), "act_preprocess should be nn.Layer." + assert layer_check( + weight_quantize_layer + ), "weight_quantize should be nn.Layer." + assert layer_check( + act_quantize_layer + ), "act_quantize should be nn.Layer." self._kwargs = { "weight_quantize_type": weight_quantize_type, @@ -369,7 +400,7 @@ class ImperativeQuantizeInputs(object): "weight_pre_layer": weight_preprocess_layer, "act_pre_layer": act_preprocess_layer, "weight_quant_layer": weight_quantize_layer, - "act_quant_layer": act_quantize_layer + "act_quant_layer": act_quantize_layer, } def apply(self, model): @@ -385,17 +416,20 @@ class ImperativeQuantizeInputs(object): None """ - assert isinstance(model, dygraph.Layer), \ - "The model must be the instance of dygraph.Layer." + assert isinstance( + model, dygraph.Layer + ), "The model must be the instance of dygraph.Layer." for name, cur_layer in model.named_sublayers(): - if not isinstance(cur_layer, self._quantizable_layer_type) \ - or (hasattr(cur_layer, "skip_quant") \ - and cur_layer.skip_quant == True): + if not isinstance(cur_layer, self._quantizable_layer_type) or ( + hasattr(cur_layer, "skip_quant") + and cur_layer.skip_quant == True + ): continue - parent_layer, sub_name = \ - utils.find_parent_layer_and_sub_name(model, name) + parent_layer, sub_name = utils.find_parent_layer_and_sub_name( + model, name + ) cur_quant_layer = self._get_input_quantized_layer(cur_layer) setattr(parent_layer, sub_name, cur_quant_layer) @@ -407,9 +441,9 @@ class ImperativeQuantizeInputs(object): if isinstance(layer, value): quant_layer_name = 'Quantized' + key break - assert quant_layer_name is not None, \ - "The layer %s is unsupported to be quantized." \ - % layer.full_name() + assert quant_layer_name is not None, ( + "The layer %s is unsupported to be quantized." % layer.full_name() + ) return quant_layers.__dict__[quant_layer_name](layer, **self._kwargs) @@ -445,8 +479,9 @@ class ImperativeQuantizeOutputs(object): Returns: None """ - assert isinstance(model, dygraph.Layer), \ - "The model must be the instance of dygraph.Layer." + assert isinstance( + model, dygraph.Layer + ), "The model must be the instance of dygraph.Layer." for cur_name, cur_layer in model.named_sublayers(): if '_act_preprocess' in cur_name: @@ -454,17 +489,20 @@ class ImperativeQuantizeOutputs(object): if not self._is_target_layer(cur_layer): continue - parent_layer, sub_name = \ - utils.find_parent_layer_and_sub_name(model, cur_name) + parent_layer, sub_name = utils.find_parent_layer_and_sub_name( + model, cur_name + ) reduce_type = None if isinstance(cur_layer, tuple(utils.fake_quant_output_layers)): cur_quant_layer = quant_layers.FakeQuantMAOutputScaleLayer( - cur_layer, self._moving_rate, reduce_type=reduce_type) + cur_layer, self._moving_rate, reduce_type=reduce_type + ) else: cur_quant_layer = quant_layers.MAOutputScaleLayer( - cur_layer, self._moving_rate, reduce_type=reduce_type) + cur_layer, self._moving_rate, reduce_type=reduce_type + ) setattr(parent_layer, sub_name, cur_quant_layer) @@ -496,8 +534,9 @@ class ImperativeQuantizeOutputs(object): Returns: None """ - assert isinstance(model, dygraph.Layer), \ - "The model must be the instance of dygraph.Layer." + assert isinstance( + model, dygraph.Layer + ), "The model must be the instance of dygraph.Layer." paddle.jit.save(layer=model, path=path, input_spec=input_spec, **config) @@ -515,11 +554,16 @@ class ImperativeQuantizeOutputs(object): model_filename = basename + INFER_MODEL_SUFFIX params_filename = basename + INFER_PARAMS_SUFFIX - [infer_program, feed_target_names, fetch_targets - ] = (load_inference_model(dirname=dirname, - executor=exe, - model_filename=model_filename, - params_filename=params_filename)) + [ + infer_program, + feed_target_names, + fetch_targets, + ] = load_inference_model( + dirname=dirname, + executor=exe, + model_filename=model_filename, + params_filename=params_filename, + ) if not self._onnx_format: self._gather_scales(infer_program, scope, fetch_targets) @@ -539,7 +583,8 @@ class ImperativeQuantizeOutputs(object): else: graph = IrGraph(core.Graph(infer_program.desc), for_test=False) transform_pass = ReplaceFakeQuantDequantPass( - scope, place, quant_bits=self._activation_bits) + scope, place, quant_bits=self._activation_bits + ) for sub_graph in graph.all_sub_graphs(): sub_graph._for_test = True transform_pass.apply(sub_graph) @@ -555,14 +600,16 @@ class ImperativeQuantizeOutputs(object): move_persistable_var_to_global_block(infer_program) - save_inference_model(dirname=dirname, - feeded_var_names=feed_target_names, - target_vars=fetch_targets, - executor=exe, - main_program=infer_program.clone(), - model_filename=model_filename, - params_filename=params_filename, - clip_extra=clip_extra) + save_inference_model( + dirname=dirname, + feeded_var_names=feed_target_names, + target_vars=fetch_targets, + executor=exe, + main_program=infer_program.clone(), + model_filename=model_filename, + params_filename=params_filename, + clip_extra=clip_extra, + ) if is_dynamic_mode: paddle.disable_static() @@ -576,12 +623,16 @@ class ImperativeQuantizeOutputs(object): return False if self._onnx_format: - return True if isinstance(layer, tuple( - utils.fake_quant_wrap_layers)) else False + return ( + True + if isinstance(layer, tuple(utils.fake_quant_wrap_layers)) + else False + ) flag = False - if utils.is_leaf_layer(layer) and \ - not isinstance(layer, tuple(utils.fake_quant_leaf_layers)): + if utils.is_leaf_layer(layer) and not isinstance( + layer, tuple(utils.fake_quant_leaf_layers) + ): flag = True if isinstance(layer, tuple(utils.fake_quant_wrap_layers)): @@ -600,8 +651,9 @@ class ImperativeQuantizeOutputs(object): def _gather_input_scale(): target_ops = [] - skip_ops = utils.fake_quantize_dequantize_op_types + \ - ["moving_average_abs_max_scale"] + skip_ops = utils.fake_quantize_dequantize_op_types + [ + "moving_average_abs_max_scale" + ] for block in program.blocks: for op in block.ops: if op.type not in skip_ops: @@ -611,16 +663,19 @@ class ImperativeQuantizeOutputs(object): for in_var_name in utils._get_op_input_var_names(op): previous_op = utils.find_previous_op(op.block, in_var_name) - if previous_op is not None and \ - ("quantize_dequantize" in previous_op.type or \ - previous_op.type == "moving_average_abs_max_scale"): + if previous_op is not None and ( + "quantize_dequantize" in previous_op.type + or previous_op.type == "moving_average_abs_max_scale" + ): scale_name = previous_op.output('OutScale')[0] in_scale = utils.load_variable_data(scope, scale_name) in_scale = utils.fp_numpy_to_naive(in_scale) argname, index = utils._get_input_name_index( - op, in_var_name) - op._set_attr(argname + str(index) + "_threshold", - in_scale) + op, in_var_name + ) + op._set_attr( + argname + str(index) + "_threshold", in_scale + ) op._set_attr("with_quant_attr", True) def _gather_output_scale(): @@ -646,7 +701,8 @@ class ImperativeQuantizeOutputs(object): if res is not None: argname, index = res previous_op._set_attr( - argname + str(index) + "_threshold", out_scale) + argname + str(index) + "_threshold", out_scale + ) previous_op._set_attr("out_threshold", out_scale) previous_op._set_attr("with_quant_attr", True) @@ -678,12 +734,20 @@ class ImperativeQuantizeOutputs(object): 2. the previous ops of the input op are not fake_quantize_dequantize ops """ target_op_types = [ - "conv2d", "depthwise_conv2d", "matmul", "conv2d_transpose" + "conv2d", + "depthwise_conv2d", + "matmul", + "conv2d_transpose", ] if in_op.type not in target_op_types: return False - previous_ops = [utils.find_previous_op(block, arg_name) \ - for arg_name in in_op.input_arg_names] - return any(op is not None and op.type not in \ - utils.fake_quantize_dequantize_op_types for op in previous_ops) + previous_ops = [ + utils.find_previous_op(block, arg_name) + for arg_name in in_op.input_arg_names + ] + return any( + op is not None + and op.type not in utils.fake_quantize_dequantize_op_types + for op in previous_ops + ) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py b/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py index a30d775165e..d771b51e09d 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py @@ -18,7 +18,12 @@ import numpy as np import paddle import paddle.nn.quant.quant_layers as quant_layers -from ..utils import _get_op_input_var_names, _get_op_output_var_names, _get_output_name_index, _get_input_name_index +from ..utils import ( + _get_op_input_var_names, + _get_op_output_var_names, + _get_output_name_index, + _get_input_name_index, +) layer_name_map = { 'Conv2DTranspose': paddle.nn.Conv2DTranspose, @@ -54,8 +59,10 @@ fake_quant_input_layers = [ # TODO(jc): fix the problem of adding duplicate fake_quant ops # paddle.nn.AdaptiveAvgPool2D, paddle.nn.AvgPool2D, paddle.nn.ReLU,paddle.nn.LeakyReLU fake_quant_output_layers = [ - paddle.nn.quant.add, paddle.nn.quant.subtract, paddle.nn.quant.multiply, - paddle.nn.quant.divide + paddle.nn.quant.add, + paddle.nn.quant.subtract, + paddle.nn.quant.multiply, + paddle.nn.quant.divide, ] fake_quant_leaf_layers = [ @@ -66,24 +73,28 @@ fake_quant_leaf_layers = [ ] fake_quant_wrap_layers = [ - quant_layers.QuantizedConv2D, quant_layers.QuantizedLinear, + quant_layers.QuantizedConv2D, + quant_layers.QuantizedLinear, quant_layers.QuantizedConv2DTranspose, quant_layers.QuantizedColumnParallelLinear, - quant_layers.QuantizedRowParallelLinear + quant_layers.QuantizedRowParallelLinear, ] # The weight format of these layers is Cin * Cout * H * W spec_channel_axis_layers = [paddle.nn.Conv2DTranspose, paddle.nn.Linear] weight_op_types = [ - "conv2d", "depthwise_conv2d", "matmul", "conv2d_transpose", - "depthwise_conv2d_transpose" + "conv2d", + "depthwise_conv2d", + "matmul", + "conv2d_transpose", + "depthwise_conv2d_transpose", ] fake_quantize_dequantize_op_types = [ "fake_quantize_dequantize_abs_max", "fake_channel_wise_quantize_dequantize_abs_max", - "fake_quantize_dequantize_moving_average_abs_max" + "fake_quantize_dequantize_moving_average_abs_max", ] @@ -92,8 +103,7 @@ def load_variable_data(scope, var_name): Load variable value from scope """ var_node = scope.find_var(var_name) - assert var_node is not None, \ - "Can not find " + var_name + " in the scope." + assert var_node is not None, "Can not find " + var_name + " in the scope." return np.array(var_node.get_tensor()) @@ -131,8 +141,9 @@ def find_parent_layer_and_sub_name(model, name): Returns: parent_layer, subname """ - assert isinstance(model, paddle.nn.Layer), \ - "The model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The model must be the instance of paddle.nn.Layer." assert len(name) > 0, "The input (name) should not be empty." last_idx = 0 @@ -164,8 +175,7 @@ def is_leaf_layer(layer): """ Whether the layer is leaf layer. """ - return isinstance(layer, paddle.nn.Layer) \ - and len(layer.sublayers()) == 0 + return isinstance(layer, paddle.nn.Layer) and len(layer.sublayers()) == 0 def fp_numpy_to_naive(x_np): diff --git a/python/paddle/fluid/contrib/slim/quantization/post_training_quantization.py b/python/paddle/fluid/contrib/slim/quantization/post_training_quantization.py index 5c0fff9abe4..73d1e31159a 100644 --- a/python/paddle/fluid/contrib/slim/quantization/post_training_quantization.py +++ b/python/paddle/fluid/contrib/slim/quantization/post_training_quantization.py @@ -32,7 +32,14 @@ from .... import unique_name from ....executor import global_scope, Executor from ....framework import IrGraph from ....log_helper import get_logger -from .quantization_pass import QuantizationTransformPass, QuantizationTransformPassV2, QuantizationFreezePass, QuantWeightPass, AddQuantDequantPass, AddQuantDequantPassV2 +from .quantization_pass import ( + QuantizationTransformPass, + QuantizationTransformPassV2, + QuantizationFreezePass, + QuantWeightPass, + AddQuantDequantPass, + AddQuantDequantPassV2, +) from .cal_kl_threshold import cal_kl_threshold from .adaround import run_adaround from . import utils @@ -43,9 +50,9 @@ __all__ = [ 'PostTrainingQuantizationProgram', ] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def _all_persistable_var_names(program): @@ -68,8 +75,9 @@ def _remove_unused_var_nodes(graph): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, graph.all_var_nodes() + ) } graph.safe_remove_nodes(all_unused_vars) return graph @@ -84,12 +92,9 @@ def _remove_ctrl_vars(graph): return graph -def _apply_pass(scope, - graph, - pass_name, - attrs=None, - attr_values=None, - debug=False): +def _apply_pass( + scope, graph, pass_name, attrs=None, attr_values=None, debug=False +): ir_pass = core.get_pass(pass_name) cpp_graph = graph.graph if not cpp_graph.has('__param_scope__'): @@ -114,37 +119,39 @@ class PostTrainingQuantization(object): quantized variables. """ - def __init__(self, - executor, - model_dir, - scope=None, - model_filename=None, - params_filename=None, - batch_generator=None, - sample_generator=None, - data_loader=None, - batch_size=10, - batch_nums=None, - algo="KL", - hist_percent=0.99999, - quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], - round_type='round', - learning_rate=0.001, - is_full_quantize=False, - bias_correction=False, - activation_bits=8, - weight_bits=8, - activation_quantize_type='range_abs_max', - weight_quantize_type='channel_wise_abs_max', - onnx_format=False, - freeze_model=True, - optimize_model=False, - is_use_cache_file=False, - skip_tensor_list=None, - same_scale_tensor_list=None, - cache_dir=None, - scale_dict=None, - return_graph=False): + def __init__( + self, + executor, + model_dir, + scope=None, + model_filename=None, + params_filename=None, + batch_generator=None, + sample_generator=None, + data_loader=None, + batch_size=10, + batch_nums=None, + algo="KL", + hist_percent=0.99999, + quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], + round_type='round', + learning_rate=0.001, + is_full_quantize=False, + bias_correction=False, + activation_bits=8, + weight_bits=8, + activation_quantize_type='range_abs_max', + weight_quantize_type='channel_wise_abs_max', + onnx_format=False, + freeze_model=True, + optimize_model=False, + is_use_cache_file=False, + skip_tensor_list=None, + same_scale_tensor_list=None, + cache_dir=None, + scale_dict=None, + return_graph=False, + ): ''' Constructor. @@ -274,38 +281,65 @@ class PostTrainingQuantization(object): ''' self._support_activation_quantize_type = [ - 'range_abs_max', 'moving_average_abs_max', 'abs_max' + 'range_abs_max', + 'moving_average_abs_max', + 'abs_max', ] self._support_weight_quantize_type = ['abs_max', 'channel_wise_abs_max'] self._support_algo_type = [ - 'KL', 'hist', 'avg', 'mse', 'emd', 'abs_max', 'min_max', 'ptf' + 'KL', + 'hist', + 'avg', + 'mse', + 'emd', + 'abs_max', + 'min_max', + 'ptf', ] assert round_type in ['adaround', 'round'] self._round_type = round_type self._learning_rate = learning_rate self._dynamic_quantize_op_type = ['lstm'] - self._support_quantize_op_type = \ - list(set(utils._weight_supported_quantizable_op_type + - utils._act_supported_quantizable_op_type + - self._dynamic_quantize_op_type)) + self._support_quantize_op_type = list( + set( + utils._weight_supported_quantizable_op_type + + utils._act_supported_quantizable_op_type + + self._dynamic_quantize_op_type + ) + ) # Check inputs assert executor is not None, "The executor cannot be None." - assert any([gen is not None] for gen in [sample_generator, - batch_generator, data_loader]), "The sample_generator, batch_generator " \ + assert any( + [gen is not None] + for gen in [sample_generator, batch_generator, data_loader] + ), ( + "The sample_generator, batch_generator " "and data_loader cannot be None in the same time." + ) if data_loader is not None: - assert isinstance(data_loader, (io.DataLoader, type(isgeneratorfunction), reader.GeneratorLoader)), \ - "data_loader only accepts `paddle.io.DataLoader` or Generator instance." + assert isinstance( + data_loader, + ( + io.DataLoader, + type(isgeneratorfunction), + reader.GeneratorLoader, + ), + ), "data_loader only accepts `paddle.io.DataLoader` or Generator instance." assert batch_size > 0, "The batch_size should be greater than 0." - assert algo in self._support_algo_type, \ - "The algo should be KL, hist, mse, avg, abs_max, min_max or ptf." - assert activation_quantize_type in self._support_activation_quantize_type, \ - "The activation_quantize_type ({}) should in ({}).".format( - activation_quantize_type, self._support_activation_quantize_type) - assert weight_quantize_type in self._support_weight_quantize_type, \ - "The weight_quantize_type ({}) shoud in ({}).".format( - weight_quantize_type, self._support_weight_quantize_type) + assert ( + algo in self._support_algo_type + ), "The algo should be KL, hist, mse, avg, abs_max, min_max or ptf." + assert ( + activation_quantize_type in self._support_activation_quantize_type + ), "The activation_quantize_type ({}) should in ({}).".format( + activation_quantize_type, self._support_activation_quantize_type + ) + assert ( + weight_quantize_type in self._support_weight_quantize_type + ), "The weight_quantize_type ({}) shoud in ({}).".format( + weight_quantize_type, self._support_weight_quantize_type + ) # Save input params self._bias_correction = bias_correction @@ -333,8 +367,9 @@ class PostTrainingQuantization(object): else: self._quantizable_op_type = quantizable_op_type for op_type in self._quantizable_op_type: - assert op_type in self._support_quantize_op_type, \ + assert op_type in self._support_quantize_op_type, ( op_type + " is not supported for quantization." + ) self._optimize_model = optimize_model # Define variables @@ -389,16 +424,18 @@ class PostTrainingQuantization(object): if self._algo in ["KL", "hist"]: batch_id = 0 with tqdm( - total=self._batch_nums, - bar_format= - 'Preparation stage, Run batch:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + total=self._batch_nums, + bar_format='Preparation stage, Run batch:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for data in self._data_loader(): - self._executor.run(program=self._program, - feed=data, - fetch_list=self._fetch_list, - return_numpy=False, - scope=self._scope) + self._executor.run( + program=self._program, + feed=data, + fetch_list=self._fetch_list, + return_numpy=False, + scope=self._scope, + ) self._collect_activation_abs_min_max() batch_id += 1 t.update() @@ -407,16 +444,19 @@ class PostTrainingQuantization(object): self._init_sampling_act_histogram() batch_id = 0 - with tqdm(total=self._batch_nums, - bar_format= - 'Sampling stage, Run batch:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=self._batch_nums, + bar_format='Sampling stage, Run batch:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for data in self._data_loader(): - self._executor.run(program=self._program, - feed=data, - fetch_list=self._fetch_list, - return_numpy=False, - scope=self._scope) + self._executor.run( + program=self._program, + feed=data, + fetch_list=self._fetch_list, + return_numpy=False, + scope=self._scope, + ) self._sampling() batch_id += 1 t.update() @@ -425,8 +465,9 @@ class PostTrainingQuantization(object): if self._algo == 'avg': for var_name in self._quantized_act_var_name: - self._quantized_threshold[var_name] = \ - np.array(self._quantized_var_avg[var_name]).mean() + self._quantized_threshold[var_name] = np.array( + self._quantized_var_avg[var_name] + ).mean() if self._algo in ["KL", "hist"]: self._calculate_kl_hist_threshold() @@ -444,10 +485,13 @@ class PostTrainingQuantization(object): if not self.FLAG: self._save_output_threshold() - if any(op_type in self._quantizable_op_type - for op_type in self._dynamic_quantize_op_type): + if any( + op_type in self._quantizable_op_type + for op_type in self._dynamic_quantize_op_type + ): self._collect_dynamic_quantize_op_threshold( - self._dynamic_quantize_op_type) + self._dynamic_quantize_op_type + ) utils.move_persistable_var_to_global_block(self._program) @@ -463,23 +507,24 @@ class PostTrainingQuantization(object): scale_dict = self._quantized_var_threshold else: scale_dict = self._quantized_threshold - run_adaround(self._data_loader, - self._program, - self._fetch_list, - self._executor, - self._scope, - self._place, - self._quantized_op_pairs, - self._weight_op_pairs, - scale_dict, - num_iterations=self._batch_nums, - bias_correction=self._bias_correction, - lr=self._learning_rate) - - def save_quantized_model(self, - save_model_path, - model_filename=None, - params_filename=None): + run_adaround( + self._data_loader, + self._program, + self._fetch_list, + self._executor, + self._scope, + self._place, + self._quantized_op_pairs, + self._weight_op_pairs, + scale_dict, + num_iterations=self._batch_nums, + bias_correction=self._bias_correction, + lr=self._learning_rate, + ) + + def save_quantized_model( + self, save_model_path, model_filename=None, params_filename=None + ): ''' Save the quantized model to the disk. @@ -494,14 +539,16 @@ class PostTrainingQuantization(object): Returns: None ''' - io.save_inference_model(dirname=save_model_path, - model_filename=model_filename, - params_filename=params_filename, - feeded_var_names=self._feed_list, - target_vars=self._fetch_list, - executor=self._executor, - main_program=self._program, - clip_extra=self._clip_extra) + io.save_inference_model( + dirname=save_model_path, + model_filename=model_filename, + params_filename=params_filename, + feeded_var_names=self._feed_list, + target_vars=self._fetch_list, + executor=self._executor, + main_program=self._program, + clip_extra=self._clip_extra, + ) _logger.info("The quantized model is saved in " + save_model_path) def _load_model_data(self): @@ -510,36 +557,49 @@ class PostTrainingQuantization(object): ''' if self._program is None: _logger.info("Load model and set data loader ...") - [self._program, self._feed_list, self._fetch_list] = \ - io.load_inference_model(dirname=self._model_dir, - executor=self._executor, - model_filename=self._model_filename, - params_filename=self._params_filename) + [ + self._program, + self._feed_list, + self._fetch_list, + ] = io.load_inference_model( + dirname=self._model_dir, + executor=self._executor, + model_filename=self._model_filename, + params_filename=self._params_filename, + ) if self._optimize_model: self._optimize_fp32_model() - feed_vars = [framework._get_var(str(var_name), self._program) \ - for var_name in self._feed_list] + feed_vars = [ + framework._get_var(str(var_name), self._program) + for var_name in self._feed_list + ] if self._data_loader is not None: - self._batch_nums = self._batch_nums if self._batch_nums else len( - self._data_loader) + self._batch_nums = ( + self._batch_nums if self._batch_nums else len(self._data_loader) + ) return - self._data_loader = io.DataLoader.from_generator(feed_list=feed_vars, - capacity=3 * - self._batch_size, - iterable=True) + self._data_loader = io.DataLoader.from_generator( + feed_list=feed_vars, capacity=3 * self._batch_size, iterable=True + ) if self._sample_generator is not None: - self._data_loader.set_sample_generator(self._sample_generator, - batch_size=self._batch_size, - drop_last=True, - places=self._place) + self._data_loader.set_sample_generator( + self._sample_generator, + batch_size=self._batch_size, + drop_last=True, + places=self._place, + ) elif self._batch_generator is not None: - self._data_loader.set_batch_generator(self._batch_generator, - places=self._place) - self._batch_nums = self._batch_nums if self._batch_nums else len( - list(self._data_loader)) + self._data_loader.set_batch_generator( + self._batch_generator, places=self._place + ) + self._batch_nums = ( + self._batch_nums + if self._batch_nums + else len(list(self._data_loader)) + ) def _optimize_fp32_model(self): ''' @@ -552,8 +612,9 @@ class PostTrainingQuantization(object): graph = _apply_pass(self._scope, graph, 'depthwise_conv_bn_fuse_pass') graph = _apply_pass(self._scope, graph, 'conv_transpose_bn_fuse_pass') graph = _apply_pass(self._scope, graph, 'conv_eltwiseadd_bn_fuse_pass') - graph = _apply_pass(self._scope, graph, - 'depthwise_conv_eltwiseadd_bn_fuse_pass') + graph = _apply_pass( + self._scope, graph, 'depthwise_conv_eltwiseadd_bn_fuse_pass' + ) self._program = graph.to_program() @@ -584,26 +645,39 @@ class PostTrainingQuantization(object): op._set_attr("op_namescope", "skip_quant") op_type = op.type - if self._is_full_quantize and \ - op_type not in self._quantizable_op_type: - _logger.warning(op_type + - " is not supported for quantization.") + if ( + self._is_full_quantize + and op_type not in self._quantizable_op_type + ): + _logger.warning( + op_type + " is not supported for quantization." + ) # For quantized ops, sample inputs and outputs if op_type in self._quantizable_op_type: - collect_var_name(utils._get_op_input_var_names(op), - persistable_var_names, op_type) - collect_var_name(utils._get_op_output_var_names(op), - persistable_var_names, op_type) + collect_var_name( + utils._get_op_input_var_names(op), + persistable_var_names, + op_type, + ) + collect_var_name( + utils._get_op_output_var_names(op), + persistable_var_names, + op_type, + ) # collect quanted op output var name for out_var_name in utils._get_op_output_var_names(op): for in_var_name in utils._get_op_input_var_names(op): if in_var_name in persistable_var_names: self._quantized_op_pairs[ - in_var_name] = out_var_name + in_var_name + ] = out_var_name # For other op, only sample output scale elif op_type in self._out_scale_op_list: - collect_var_name(utils._get_op_output_var_names(op), - persistable_var_names, op_type) + collect_var_name( + utils._get_op_output_var_names(op), + persistable_var_names, + op_type, + ) def _set_activation_persistable(self): ''' @@ -651,15 +725,19 @@ class PostTrainingQuantization(object): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value _logger.info("MSE searching stage ...") for var_name in self._quantized_act_var_name: @@ -673,16 +751,19 @@ class PostTrainingQuantization(object): while s <= 1.0: scale = s * abs_max_value s += 0.02 - bins = 2**(self._activation_bits - 1) - 1 + bins = 2 ** (self._activation_bits - 1) - 1 if self._onnx_format: - quant_var = np.clip(np.round(var_tensor / scale * bins), - -bins - 1, bins) + quant_var = np.clip( + np.round(var_tensor / scale * bins), -bins - 1, bins + ) quant_dequant_var = quant_var / bins * scale else: - quant_dequant_var = np.round( - np.clip(var_tensor, 0.0, scale) / scale * - bins) / bins * scale - mse_loss = ((var_tensor - quant_dequant_var)**2).mean() + quant_dequant_var = ( + np.round(np.clip(var_tensor, 0.0, scale) / scale * bins) + / bins + * scale + ) + mse_loss = ((var_tensor - quant_dequant_var) ** 2).mean() if mse_loss <= self._best_calibration_loss[var_name]: self._best_calibration_loss[var_name] = mse_loss self._quantized_threshold[var_name] = scale @@ -695,15 +776,19 @@ class PostTrainingQuantization(object): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value _logger.info("EMD searching stage ...") for var_name in self._quantized_act_var_name: @@ -717,18 +802,21 @@ class PostTrainingQuantization(object): while s <= 1.0: scale = s * abs_max_value s += 0.02 - bins = 2**(self._activation_bits - 1) - 1 + bins = 2 ** (self._activation_bits - 1) - 1 if self._onnx_format: - quant_var = np.clip(np.round(var_tensor / scale * bins), - -bins - 1, bins) + quant_var = np.clip( + np.round(var_tensor / scale * bins), -bins - 1, bins + ) quant_dequant_var = quant_var / bins * scale else: - quant_dequant_var = np.round( - np.clip(var_tensor, 0.0, scale) / scale * - bins) / bins * scale + quant_dequant_var = ( + np.round(np.clip(var_tensor, 0.0, scale) / scale * bins) + / bins + * scale + ) emd_loss = np.abs( - np.mean(var_tensor) - np.mean(quant_dequant_var)) + np.abs( - np.std(var_tensor) - np.std(quant_dequant_var)) + np.mean(var_tensor) - np.mean(quant_dequant_var) + ) + np.abs(np.std(var_tensor) - np.std(quant_dequant_var)) if emd_loss <= self._best_calibration_loss[var_name]: self._best_calibration_loss[var_name] = emd_loss self._quantized_threshold[var_name] = scale @@ -741,24 +829,34 @@ class PostTrainingQuantization(object): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value for var_name in self._quantized_act_var_name: var_tensor = utils.load_variable_data(self._scope, var_name) abs_max_value = float(np.max(np.abs(var_tensor))) - if (var_name not in self._quantized_var_avg): + if var_name not in self._quantized_var_avg: self._quantized_var_avg[var_name] = [] - abs_avg_value = float(np.mean(np.max( \ - np.abs(var_tensor.reshape(var_tensor.shape[0], -1)), axis=(1)))) + abs_avg_value = float( + np.mean( + np.max( + np.abs(var_tensor.reshape(var_tensor.shape[0], -1)), + axis=(1), + ) + ) + ) self._quantized_var_avg[var_name].append(abs_avg_value) continue @@ -770,22 +868,27 @@ class PostTrainingQuantization(object): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value for var_name in self._quantized_act_var_name: var_tensor = utils.load_variable_data(self._scope, var_name) abs_max_value = float(np.max(np.abs(var_tensor))) - if (var_name not in self._quantized_threshold) or \ - (abs_max_value > self._quantized_threshold[var_name]): + if (var_name not in self._quantized_threshold) or ( + abs_max_value > self._quantized_threshold[var_name] + ): self._quantized_threshold[var_name] = abs_max_value def _sample_min_max(self): @@ -798,8 +901,10 @@ class PostTrainingQuantization(object): elif self._weight_quantize_type == "channel_wise_abs_max": min_value = [] max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): min_value.append(float(np.min(var_tensor[:, i]))) max_value.append(float(np.max(var_tensor[:, i]))) @@ -814,11 +919,13 @@ class PostTrainingQuantization(object): var_tensor = utils.load_variable_data(self._scope, var_name) min_value = float(np.min(var_tensor)) max_value = float(np.max(var_tensor)) - if (var_name not in self._quantized_var_min) or \ - (min_value < self._quantized_var_min[var_name]): + if (var_name not in self._quantized_var_min) or ( + min_value < self._quantized_var_min[var_name] + ): self._quantized_var_min[var_name] = min_value - if (var_name not in self._quantized_var_max) or \ - (max_value > self._quantized_var_max[var_name]): + if (var_name not in self._quantized_var_max) or ( + max_value > self._quantized_var_max[var_name] + ): self._quantized_var_max[var_name] = max_value def _sample_histogram(self): @@ -841,39 +948,47 @@ class PostTrainingQuantization(object): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value for var_name in self._quantized_act_var_name: var_tensor = utils.load_variable_data(self._scope, var_name) abs_max_value = float(np.max(np.abs(var_tensor))) - q_max = 2**(self._activation_bits - 1) - 1 + q_max = 2 ** (self._activation_bits - 1) - 1 scale8 = abs_max_value / q_max scale4 = scale8 / 2 scale2 = scale4 / 2 scale1 = scale2 / 2 - quant_dequant_var_scale1 = np.clip(np.round(var_tensor / scale1), 0, - q_max) * scale1 - quant_dequant_var_scale2 = np.clip(np.round(var_tensor / scale2), 0, - q_max) * scale2 - quant_dequant_var_scale4 = np.clip(np.round(var_tensor / scale4), 0, - q_max) * scale4 - quant_dequant_var_scale8 = np.clip(np.round(var_tensor / scale8), 0, - q_max) * scale8 + quant_dequant_var_scale1 = ( + np.clip(np.round(var_tensor / scale1), 0, q_max) * scale1 + ) + quant_dequant_var_scale2 = ( + np.clip(np.round(var_tensor / scale2), 0, q_max) * scale2 + ) + quant_dequant_var_scale4 = ( + np.clip(np.round(var_tensor / scale4), 0, q_max) * scale4 + ) + quant_dequant_var_scale8 = ( + np.clip(np.round(var_tensor / scale8), 0, q_max) * scale8 + ) score1 = utils.l2_loss(var_tensor, quant_dequant_var_scale1) score2 = utils.l2_loss(var_tensor, quant_dequant_var_scale2) score4 = utils.l2_loss(var_tensor, quant_dequant_var_scale4) score8 = utils.l2_loss(var_tensor, quant_dequant_var_scale8) score = [score1, score2, score4, score8] - mask = 2**score.index(min(score)) + mask = 2 ** score.index(min(score)) scale = scale1 * mask threshold = q_max * scale self._quantized_threshold[var_name] = threshold @@ -882,18 +997,21 @@ class PostTrainingQuantization(object): ''' Save input threshold to the quantized op. ''' - assert self._algo == "min_max", \ - "The algo should be min_max to save input threshold." + assert ( + self._algo == "min_max" + ), "The algo should be min_max to save input threshold." for block_id in range(len(self._program.blocks)): for op in self._program.blocks[block_id].ops: if op.type in self._quantizable_op_type: for var_name in utils._get_op_input_var_names(op): assert var_name in self._quantized_var_min assert var_name in self._quantized_var_max - op._set_attr(var_name + ".min", - self._quantized_var_min[var_name]) - op._set_attr(var_name + ".max", - self._quantized_var_max[var_name]) + op._set_attr( + var_name + ".min", self._quantized_var_min[var_name] + ) + op._set_attr( + var_name + ".max", self._quantized_var_max[var_name] + ) op._set_attr("with_quant_attr", True) def _collect_activation_abs_min_max(self): @@ -908,7 +1026,8 @@ class PostTrainingQuantization(object): max_value = float(np.max(var_tensor)) if var_name not in self._sampling_act_abs_min_max: self._sampling_act_abs_min_max[var_name] = [ - min_value, max_value + min_value, + max_value, ] else: if min_value < self._sampling_act_abs_min_max[var_name][0]: @@ -924,9 +1043,9 @@ class PostTrainingQuantization(object): if var_name not in self._sampling_act_histogram: min_val = self._sampling_act_abs_min_max[var_name][0] max_val = self._sampling_act_abs_min_max[var_name][1] - hist, hist_edeges = np.histogram([], - bins=self._histogram_bins, - range=(min_val, max_val)) + hist, hist_edeges = np.histogram( + [], bins=self._histogram_bins, range=(min_val, max_val) + ) self._sampling_act_histogram[var_name] = [hist, hist_edeges] def _calculate_kl_hist_threshold(self): @@ -943,26 +1062,32 @@ class PostTrainingQuantization(object): weight_threshold = float(np.max(np.abs(weight_data))) elif self._weight_quantize_type == "channel_wise_abs_max": weight_threshold = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(weight_data.shape[1]): weight_threshold.append( - float(np.max(np.abs(weight_data[:, i])))) + float(np.max(np.abs(weight_data[:, i]))) + ) else: for i in range(weight_data.shape[0]): weight_threshold.append( - float(np.max(np.abs(weight_data[i])))) + float(np.max(np.abs(weight_data[i]))) + ) self._quantized_var_threshold[var_name] = weight_threshold for var_name in self._quantized_act_var_name: hist, hist_edeges = self._sampling_act_histogram[var_name] if self._algo == "KL": bin_width = hist_edeges[1] - hist_edeges[0] - self._quantized_var_threshold[var_name] = \ - cal_kl_threshold(hist, bin_width, self._activation_bits) + self._quantized_var_threshold[var_name] = cal_kl_threshold( + hist, bin_width, self._activation_bits + ) elif self._algo == "hist": - self._quantized_var_threshold[var_name] = \ - self._get_hist_scaling_factor(hist, hist_edeges) + self._quantized_var_threshold[ + var_name + ] = self._get_hist_scaling_factor(hist, hist_edeges) def _update_program(self): ''' @@ -986,7 +1111,8 @@ class PostTrainingQuantization(object): activation_bits=self._activation_bits, activation_quantize_type=self._activation_quantize_type, weight_quantize_type=self._weight_quantize_type, - quantizable_op_type=major_quantizable_op_types) + quantizable_op_type=major_quantizable_op_types, + ) else: transform_pass = QuantizationTransformPassV2( scope=self._scope, @@ -995,7 +1121,8 @@ class PostTrainingQuantization(object): activation_bits=self._activation_bits, activation_quantize_type=self._activation_quantize_type, weight_quantize_type=self._weight_quantize_type, - quantizable_op_type=major_quantizable_op_types) + quantizable_op_type=major_quantizable_op_types, + ) for sub_graph in graph.all_sub_graphs(): # Insert fake_quant/fake_dequantize op must in test graph, so @@ -1012,13 +1139,15 @@ class PostTrainingQuantization(object): add_quant_dequant_pass = AddQuantDequantPass( scope=self._scope, place=self._place, - quantizable_op_type=minor_quantizable_op_types) + quantizable_op_type=minor_quantizable_op_types, + ) else: add_quant_dequant_pass = AddQuantDequantPassV2( scope=self._scope, place=self._place, quantizable_op_type=minor_quantizable_op_types, - is_full_quantized=True) + is_full_quantized=True, + ) for sub_graph in graph.all_sub_graphs(): sub_graph._for_test = True @@ -1038,41 +1167,49 @@ class PostTrainingQuantization(object): for tensor_name in tensor_list: if '#' in tensor_name: real_tensor_name, opera, scalar = tensor_name.split( - '#') + '#' + ) if real_tensor_name not in scale_dict.keys(): continue if opera == '*': scale_dict[real_tensor_name] = float( - scale_dict[real_tensor_name]) * float( - scalar) + scale_dict[real_tensor_name] + ) * float(scalar) elif opera == '/': scale_dict[real_tensor_name] = float( - scale_dict[real_tensor_name]) / float( - scalar) - max_scale = scale_dict[ - real_tensor_name] if max_scale is None else max( - max_scale, scale_dict[real_tensor_name]) + scale_dict[real_tensor_name] + ) / float(scalar) + max_scale = ( + scale_dict[real_tensor_name] + if max_scale is None + else max( + max_scale, scale_dict[real_tensor_name] + ) + ) else: if tensor_name not in scale_dict.keys(): continue - max_scale = scale_dict[ - tensor_name] if max_scale is None else max( - max_scale, scale_dict[tensor_name]) + max_scale = ( + scale_dict[tensor_name] + if max_scale is None + else max(max_scale, scale_dict[tensor_name]) + ) for tensor_name in tensor_list: if '#' in tensor_name: real_tensor_name, opera, scalar = tensor_name.split( - '#') + '#' + ) if real_tensor_name not in scale_dict.keys(): continue if opera == '*': scale_dict[ - real_tensor_name] = max_scale / float( - scalar) + real_tensor_name + ] = max_scale / float(scalar) elif opera == '/': scale_dict[ - real_tensor_name] = max_scale * float( - scalar) + real_tensor_name + ] = max_scale * float(scalar) else: if tensor_name not in scale_dict.keys(): continue @@ -1080,11 +1217,18 @@ class PostTrainingQuantization(object): self._scale_dict = scale_dict for key, val in self._scale_dict.items(): - utils.set_variable_data(self._scope, self._place, key + "@scale", - np.array([val], dtype=np.float32)) - utils.set_variable_data(self._scope, self._place, - key + ".quant_dequant@scale", - np.array([val], dtype=np.float32)) + utils.set_variable_data( + self._scope, + self._place, + key + "@scale", + np.array([val], dtype=np.float32), + ) + utils.set_variable_data( + self._scope, + self._place, + key + ".quant_dequant@scale", + np.array([val], dtype=np.float32), + ) if not self._onnx_format: # apply QuantizationFreezePass, and obtain the final quant model @@ -1097,7 +1241,8 @@ class PostTrainingQuantization(object): round_type=self._round_type, activation_bits=self._activation_bits, weight_quantize_type=self._weight_quantize_type, - quantizable_op_type=major_quantizable_op_types) + quantizable_op_type=major_quantizable_op_types, + ) for sub_graph in graph.all_sub_graphs(): sub_graph._for_test = True @@ -1116,16 +1261,20 @@ class PostTrainingQuantization(object): ''' self._calibration_scales = {} - def save_info(op_node, out_var_name, threshold_map, out_info_name, - quantized_type): - assert out_var_name in threshold_map, \ - "The output ({}) of {} node does not have threshold.".format( - out_var_name, op_node.type) + def save_info( + op_node, out_var_name, threshold_map, out_info_name, quantized_type + ): + assert ( + out_var_name in threshold_map + ), "The output ({}) of {} node does not have threshold.".format( + out_var_name, op_node.type + ) if self._onnx_format: # For easy extension, every var_node set a dict to save parameters of quant. self._calibration_scales[var_name] = {} self._calibration_scales[var_name]['scale'] = threshold_map[ - var_name] + var_name + ] else: op_node._set_attr(out_info_name, threshold_map[var_name]) op_node._set_attr("with_quant_attr", True) @@ -1134,42 +1283,78 @@ class PostTrainingQuantization(object): def analysis_and_save_info(op_node, out_var_name): argname_index = utils._get_output_name_index(op_node, out_var_name) - assert argname_index is not None, \ + assert argname_index is not None, ( out_var_name + " is not the output of the op" + ) if self._algo == "KL": # For compatibility, we save output threshold by two methods. - save_info(op_node, out_var_name, self._quantized_var_threshold, - "out_threshold", "post_kl") save_info( - op_node, out_var_name, self._quantized_var_threshold, + op_node, + out_var_name, + self._quantized_var_threshold, + "out_threshold", + "post_kl", + ) + save_info( + op_node, + out_var_name, + self._quantized_var_threshold, argname_index[0] + str(argname_index[1]) + "_threshold", - "post_kl") + "post_kl", + ) elif self._algo == "hist": # For compatibility, we save output threshold by two methods. - save_info(op_node, out_var_name, self._quantized_var_threshold, - "out_threshold", "post_hist") save_info( - op_node, out_var_name, self._quantized_var_threshold, + op_node, + out_var_name, + self._quantized_var_threshold, + "out_threshold", + "post_hist", + ) + save_info( + op_node, + out_var_name, + self._quantized_var_threshold, argname_index[0] + str(argname_index[1]) + "_threshold", - "post_hist") + "post_hist", + ) elif self._algo in ["avg", "abs_max", "mse", "emd", "ptf"]: - save_info(op_node, out_var_name, self._quantized_threshold, - "out_threshold", "post_" + str(self._algo)) save_info( - op_node, out_var_name, self._quantized_threshold, + op_node, + out_var_name, + self._quantized_threshold, + "out_threshold", + "post_" + str(self._algo), + ) + save_info( + op_node, + out_var_name, + self._quantized_threshold, argname_index[0] + str(argname_index[1]) + "_threshold", - "post_" + str(self._algo)) + "post_" + str(self._algo), + ) elif self._algo == "min_max": - save_info(op_node, out_var_name, self._quantized_var_min, - "out_min", "post_min_max") - save_info(op_node, out_var_name, self._quantized_var_max, - "out_max", "post_min_max") + save_info( + op_node, + out_var_name, + self._quantized_var_min, + "out_min", + "post_min_max", + ) + save_info( + op_node, + out_var_name, + self._quantized_var_max, + "out_max", + "post_min_max", + ) for block_id in range(len(self._program.blocks)): for op in self._program.blocks[block_id].ops: - if op.type in (self._quantizable_op_type + - self._out_scale_op_list): + if op.type in ( + self._quantizable_op_type + self._out_scale_op_list + ): out_var_names = utils._get_op_output_var_names(op) for var_name in out_var_names: analysis_and_save_info(op, var_name) @@ -1221,55 +1406,77 @@ class PostTrainingQuantization(object): class PostTrainingQuantizationProgram(PostTrainingQuantization): - - def __init__(self, - executor, - program, - feed_list=None, - fetch_list=None, - scope=None, - batch_generator=None, - sample_generator=None, - data_loader=None, - batch_size=10, - batch_nums=None, - algo="KL", - hist_percent=0.99999, - quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], - round_type='round', - learning_rate=0.001, - is_full_quantize=False, - bias_correction=False, - activation_bits=8, - weight_bits=8, - activation_quantize_type='range_abs_max', - weight_quantize_type='channel_wise_abs_max', - onnx_format=False, - freeze_model=True, - optimize_model=False, - is_use_cache_file=False, - skip_tensor_list=None, - same_scale_tensor_list=None, - cache_dir=None, - scale_dict=None, - return_graph=True): - super().__init__(executor, scope, None, None, None, batch_generator, - sample_generator, data_loader, batch_size, batch_nums, - algo, hist_percent, quantizable_op_type, round_type, - learning_rate, is_full_quantize, bias_correction, - activation_bits, weight_bits, activation_quantize_type, - weight_quantize_type, onnx_format, freeze_model, - optimize_model, is_use_cache_file, skip_tensor_list, - same_scale_tensor_list, cache_dir, scale_dict, - return_graph) + def __init__( + self, + executor, + program, + feed_list=None, + fetch_list=None, + scope=None, + batch_generator=None, + sample_generator=None, + data_loader=None, + batch_size=10, + batch_nums=None, + algo="KL", + hist_percent=0.99999, + quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], + round_type='round', + learning_rate=0.001, + is_full_quantize=False, + bias_correction=False, + activation_bits=8, + weight_bits=8, + activation_quantize_type='range_abs_max', + weight_quantize_type='channel_wise_abs_max', + onnx_format=False, + freeze_model=True, + optimize_model=False, + is_use_cache_file=False, + skip_tensor_list=None, + same_scale_tensor_list=None, + cache_dir=None, + scale_dict=None, + return_graph=True, + ): + super().__init__( + executor, + scope, + None, + None, + None, + batch_generator, + sample_generator, + data_loader, + batch_size, + batch_nums, + algo, + hist_percent, + quantizable_op_type, + round_type, + learning_rate, + is_full_quantize, + bias_correction, + activation_bits, + weight_bits, + activation_quantize_type, + weight_quantize_type, + onnx_format, + freeze_model, + optimize_model, + is_use_cache_file, + skip_tensor_list, + same_scale_tensor_list, + cache_dir, + scale_dict, + return_graph, + ) self.FLAG = False self._program = program if self._program is not None: self.FLAG = True - assert feed_list is not None, \ - "Feed list should not be None." - assert fetch_list is not None, \ - "Fetch list should not be None." + assert feed_list is not None, "Feed list should not be None." + assert fetch_list is not None, "Fetch list should not be None." self._feed_list = feed_list self._fetch_list = fetch_list @@ -1298,15 +1505,17 @@ class WeightQuantization(object): self._model_filename = model_filename self._params_filename = params_filename - def quantize_weight_to_int(self, - save_model_dir, - save_model_filename=None, - save_params_filename=None, - quantizable_op_type=["conv2d", "mul"], - weight_bits=8, - weight_quantize_type="channel_wise_abs_max", - generate_test_model=False, - threshold_rate=0.0): + def quantize_weight_to_int( + self, + save_model_dir, + save_model_filename=None, + save_params_filename=None, + quantizable_op_type=["conv2d", "mul"], + weight_bits=8, + weight_quantize_type="channel_wise_abs_max", + generate_test_model=False, + threshold_rate=0.0, + ): ''' In order to reduce the size of model, this api quantizes the weight of some ops from float32 to int8/16. In the inference stage, the @@ -1342,28 +1551,45 @@ class WeightQuantization(object): value will be optimized. Default is 0.0. ''' for op_type in quantizable_op_type: - assert op_type in self._supported_quantizable_op_type, \ - "Input error:" + op_type + \ - " is not supported for weight quantization." - assert weight_bits in [8, 16], \ - "Input error: weight_bits should be 8 or 16." - assert weight_quantize_type in self._supported_weight_quantize_type, \ - "Input error: weight_quantize_type should in {}".format( - self._supported_weight_quantize_type) + assert op_type in self._supported_quantizable_op_type, ( + "Input error:" + + op_type + + " is not supported for weight quantization." + ) + assert weight_bits in [ + 8, + 16, + ], "Input error: weight_bits should be 8 or 16." + assert ( + weight_quantize_type in self._supported_weight_quantize_type + ), "Input error: weight_quantize_type should in {}".format( + self._supported_weight_quantize_type + ) quantized_model_dir = os.path.join(save_model_dir, "quantized_model") - self._quantize_weight_to_int(quantized_model_dir, save_model_filename, - save_params_filename, quantizable_op_type, - weight_bits, weight_quantize_type, False, - threshold_rate) + self._quantize_weight_to_int( + quantized_model_dir, + save_model_filename, + save_params_filename, + quantizable_op_type, + weight_bits, + weight_quantize_type, + False, + threshold_rate, + ) if generate_test_model: test_model_dir = os.path.join(save_model_dir, "test_model") - self._quantize_weight_to_int(test_model_dir, save_model_filename, - save_params_filename, - quantizable_op_type, weight_bits, - weight_quantize_type, True, - threshold_rate) + self._quantize_weight_to_int( + test_model_dir, + save_model_filename, + save_params_filename, + quantizable_op_type, + weight_bits, + weight_quantize_type, + True, + threshold_rate, + ) def convert_weight_to_fp16(self, save_model_dir): """ @@ -1379,11 +1605,12 @@ class WeightQuantization(object): place = core.CPUPlace() exe = Executor(place) scope = global_scope() - [infer_program, feed_list, fetch_list] = \ - io.load_inference_model(dirname=self._model_dir, - executor=exe, - model_filename=self._model_filename, - params_filename=self._params_filename) + [infer_program, feed_list, fetch_list] = io.load_inference_model( + dirname=self._model_dir, + executor=exe, + model_filename=self._model_filename, + params_filename=self._params_filename, + ) # Clone and save fp16 weights save_program = framework.Program() @@ -1391,27 +1618,31 @@ class WeightQuantization(object): save_var_map = {} for var in infer_program.list_vars(): - if (var.type == core.VarDesc.VarType.RAW) or \ - (not var.persistable) or (var.name in ['feed', 'fetch']) \ - or (var.dtype != core.VarDesc.VarType.FP32): + if ( + (var.type == core.VarDesc.VarType.RAW) + or (not var.persistable) + or (var.name in ['feed', 'fetch']) + or (var.dtype != core.VarDesc.VarType.FP32) + ): continue - #new_var = _clone_var_to_block_(var, save_block) + # new_var = _clone_var_to_block_(var, save_block) new_var = save_block._clone_variable(var) if self._params_filename is not None: save_var_map[new_var.name] = new_var else: - save_file_path = os.path.join(os.path.normpath(save_model_dir), - new_var.name) - save_block.append_op(type='save', - inputs={'X': [new_var]}, - outputs={}, - attrs={ - 'file_path': - os.path.normpath(save_file_path), - 'save_as_fp16': - True - }) + save_file_path = os.path.join( + os.path.normpath(save_model_dir), new_var.name + ) + save_block.append_op( + type='save', + inputs={'X': [new_var]}, + outputs={}, + attrs={ + 'file_path': os.path.normpath(save_file_path), + 'save_as_fp16': True, + }, + ) if self._params_filename is not None: save_var_list = [] @@ -1420,33 +1651,44 @@ class WeightQuantization(object): saved_params_var = save_block.create_var( type=core.VarDesc.VarType.RAW, - name=unique_name.generate("saved_params")) + name=unique_name.generate("saved_params"), + ) saved_params_var.desc.set_persistable(True) - save_path = os.path.join(os.path.normpath(save_model_dir), - self._params_filename) - save_block.append_op(type='save_combine', - inputs={'X': save_var_list}, - outputs={'Y': saved_params_var}, - attrs={ - 'file_path': save_path, - 'save_as_fp16': True - }) + save_path = os.path.join( + os.path.normpath(save_model_dir), self._params_filename + ) + save_block.append_op( + type='save_combine', + inputs={'X': save_var_list}, + outputs={'Y': saved_params_var}, + attrs={'file_path': save_path, 'save_as_fp16': True}, + ) save_program._sync_with_cpp() exe.run(save_program) # Copy model - model_filename = "__model__" if self._model_filename is None \ - else self._model_filename + model_filename = ( + "__model__" + if self._model_filename is None + else self._model_filename + ) src_model = os.path.join(self._model_dir, model_filename) dest_model = os.path.join(save_model_dir, model_filename) shutil.copyfile(src_model, dest_model) - def _quantize_weight_to_int(self, save_model_dir, save_model_filename, - save_params_filename, quantizable_op_type, - weight_bits, weight_quantize_type, for_test, - threshold_rate): + def _quantize_weight_to_int( + self, + save_model_dir, + save_model_filename, + save_params_filename, + quantizable_op_type, + weight_bits, + weight_quantize_type, + for_test, + threshold_rate, + ): """ Generate quantized model or fake quantized model. """ @@ -1454,11 +1696,12 @@ class WeightQuantization(object): place = core.CPUPlace() exe = Executor(place) scope = global_scope() - [program, feed_list, fetch_list] = \ - io.load_inference_model(dirname=self._model_dir, - executor=exe, - model_filename=self._model_filename, - params_filename=self._params_filename) + [program, feed_list, fetch_list] = io.load_inference_model( + dirname=self._model_dir, + executor=exe, + model_filename=self._model_filename, + params_filename=self._params_filename, + ) quantized_ops = [] for index in range(program.num_blocks): @@ -1474,22 +1717,32 @@ class WeightQuantization(object): if var_name in persistable_var_names: if weight_quantize_type == "abs_max": self._weight_abs_max_quantization( - scope, place, weight_bits, threshold_rate, op, - var_name, for_test) + scope, + place, + weight_bits, + threshold_rate, + op, + var_name, + for_test, + ) elif weight_quantize_type == "channel_wise_abs_max": self._weight_channel_wise_abs_max_quantization( - scope, place, weight_bits, op, var_name, for_test) - - io.save_inference_model(dirname=save_model_dir, - feeded_var_names=feed_list, - target_vars=fetch_list, - executor=exe, - main_program=program, - model_filename=save_model_filename, - params_filename=save_params_filename) - - def _weight_abs_max_quantization(self, scope, place, weight_bits, - threshold_rate, op, var_name, for_test): + scope, place, weight_bits, op, var_name, for_test + ) + + io.save_inference_model( + dirname=save_model_dir, + feeded_var_names=feed_list, + target_vars=fetch_list, + executor=exe, + main_program=program, + model_filename=save_model_filename, + params_filename=save_params_filename, + ) + + def _weight_abs_max_quantization( + self, scope, place, weight_bits, threshold_rate, op, var_name, for_test + ): ''' Use abs_max method to quantize weight. ''' @@ -1501,23 +1754,28 @@ class WeightQuantization(object): if abs(threshold_rate) < 1e-10: threshold_value = np.max(np.abs(weight_data)) else: - threshold_value = self._calculate_threshold(\ - weight_data, threshold_rate) + threshold_value = self._calculate_threshold( + weight_data, threshold_rate + ) weight_data[weight_data > threshold_value] = threshold_value weight_data[weight_data < -threshold_value] = -threshold_value scale = threshold_value / quantize_range - quantized_weight_data = \ - np.around(weight_data / scale).astype(save_weight_dtype) + quantized_weight_data = np.around(weight_data / scale).astype( + save_weight_dtype + ) # Set weight data if not for_test: - utils.set_variable_data(scope, place, var_name, - quantized_weight_data) + utils.set_variable_data( + scope, place, var_name, quantized_weight_data + ) else: - dequantized_weight_data = \ - (quantized_weight_data * scale).astype(np.float32) - utils.set_variable_data(scope, place, var_name, - dequantized_weight_data) + dequantized_weight_data = (quantized_weight_data * scale).astype( + np.float32 + ) + utils.set_variable_data( + scope, place, var_name, dequantized_weight_data + ) # Save info op._set_attr('quantization_type', 'post_weight_abs_max') @@ -1525,9 +1783,9 @@ class WeightQuantization(object): op._set_attr(var_name + "_quant_scale", [scale]) # Save as list op._set_attr("with_quant_attr", True) - def _weight_channel_wise_abs_max_quantization(self, scope, place, - weight_bits, op, var_name, - for_test): + def _weight_channel_wise_abs_max_quantization( + self, scope, place, weight_bits, op, var_name, for_test + ): ''' Use channel_wise_abs_max method to quantize weight. ''' @@ -1537,32 +1795,42 @@ class WeightQuantization(object): # Get quantized scale and weight data weight_data = utils.load_variable_data(scope, var_name) if op.type == "mul": - scales, quantized_weight_data = \ - self._mul_channel_wise_quantization(weight_data, - quantize_range, save_weight_dtype) + scales, quantized_weight_data = self._mul_channel_wise_quantization( + weight_data, quantize_range, save_weight_dtype + ) elif op.type in ["conv2d", "depthwise_conv2d"]: - scales, quantized_weight_data = \ - self._conv_channel_wise_quantization(weight_data, - quantize_range, save_weight_dtype) + ( + scales, + quantized_weight_data, + ) = self._conv_channel_wise_quantization( + weight_data, quantize_range, save_weight_dtype + ) else: _logger.error(op.type + " is not supported by weight quantization") # Set weight data if not for_test: - utils.set_variable_data(scope, place, var_name, - quantized_weight_data) + utils.set_variable_data( + scope, place, var_name, quantized_weight_data + ) else: if op.type == "mul": - dequantized_weight_data = \ - self._mul_channel_wise_dequantization(quantized_weight_data, scales) + dequantized_weight_data = self._mul_channel_wise_dequantization( + quantized_weight_data, scales + ) elif op.type in ["conv2d", "depthwise_conv2d"]: - dequantized_weight_data = \ - self._conv_channel_wise_dequantization(quantized_weight_data, scales) + dequantized_weight_data = ( + self._conv_channel_wise_dequantization( + quantized_weight_data, scales + ) + ) else: - _logger.error(op.type + - " is not supported by weight quantization") - utils.set_variable_data(scope, place, var_name, - dequantized_weight_data) + _logger.error( + op.type + " is not supported by weight quantization" + ) + utils.set_variable_data( + scope, place, var_name, dequantized_weight_data + ) # Save info op._set_attr('quantization_type', 'post_weight_channel_wise_abs_max') @@ -1570,67 +1838,77 @@ class WeightQuantization(object): op._set_attr(var_name + "_quant_scale", scales) op._set_attr("with_quant_attr", True) - def _conv_channel_wise_quantization(self, weight_data, quantize_range, - save_weight_dtype): + def _conv_channel_wise_quantization( + self, weight_data, quantize_range, save_weight_dtype + ): ''' Get channel wise scale for the weights of conv2d and depthwise_conv2d, and quantize the weights. ''' scales = [] - quantized_weight_data = np.zeros_like(weight_data, - dtype=save_weight_dtype) + quantized_weight_data = np.zeros_like( + weight_data, dtype=save_weight_dtype + ) channel_num = weight_data.shape[0] for i in range(channel_num): scale = np.max(np.abs(weight_data[i])) / quantize_range scales.append(scale) - quantized_weight_data[i] = \ - np.around(weight_data[i] / scale).astype(save_weight_dtype) + quantized_weight_data[i] = np.around(weight_data[i] / scale).astype( + save_weight_dtype + ) return scales, quantized_weight_data def _conv_channel_wise_dequantization(self, quantized_weight_data, scales): ''' For conv2d and depthwise_conv2d, dequantize the weights to fp32. ''' - dequantized_weight_data = np.zeros_like(quantized_weight_data, - dtype=np.float32) + dequantized_weight_data = np.zeros_like( + quantized_weight_data, dtype=np.float32 + ) for i in range(len(scales)): - dequantized_weight_data[i] = \ - (quantized_weight_data[i] * scales[i]).astype(np.float32) + dequantized_weight_data[i] = ( + quantized_weight_data[i] * scales[i] + ).astype(np.float32) return dequantized_weight_data - def _mul_channel_wise_quantization(self, weight_data, quantize_range, - save_weight_dtype): + def _mul_channel_wise_quantization( + self, weight_data, quantize_range, save_weight_dtype + ): ''' Get channel wise scale for the weights of conv2d and depthwise_conv2d, and quantize the weights. ''' scales = [] - quantized_weight_data = np.zeros_like(weight_data, - dtype=save_weight_dtype) + quantized_weight_data = np.zeros_like( + weight_data, dtype=save_weight_dtype + ) channel_num = weight_data.shape[-1] for i in range(channel_num): scale = np.max(np.abs(weight_data[:, i])) / quantize_range scales.append(scale) - quantized_weight_data[:, i] = \ - np.around(weight_data[:, i] / scale).astype(save_weight_dtype) + quantized_weight_data[:, i] = np.around( + weight_data[:, i] / scale + ).astype(save_weight_dtype) return scales, quantized_weight_data def _mul_channel_wise_dequantization(self, quantized_weight_data, scales): ''' For mul, dequantize the weights to fp32. ''' - dequantized_weight_data = np.zeros_like(quantized_weight_data, - dtype=np.float32) + dequantized_weight_data = np.zeros_like( + quantized_weight_data, dtype=np.float32 + ) for i in range(len(scales)): - dequantized_weight_data[:, i] = \ - (quantized_weight_data[:, i] * scales[i]).astype(np.float32) + dequantized_weight_data[:, i] = ( + quantized_weight_data[:, i] * scales[i] + ).astype(np.float32) return dequantized_weight_data def _calculate_threshold(self, input, threshold_rate, histogram_bins=5000): input_abs = np.abs(input) - hist, hist_edeges = np.histogram(input_abs, - bins=histogram_bins, - range=(0, np.max(input_abs))) + hist, hist_edeges = np.histogram( + input_abs, bins=histogram_bins, range=(0, np.max(input_abs)) + ) hist = hist / float(sum(hist)) hist_sum = 0 hist_index = 0 diff --git a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py index f8de55ee3ca..c7723097f4d 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py @@ -36,13 +36,15 @@ class Quant2Int8MkldnnPass(object): passes (`cpu_quantize_pass`, `cpu_quantize_squash_pass`). """ - def __init__(self, - _ops_to_quantize, - _op_ids_to_skip=None, - _scope=None, - _place=None, - _core=None, - _debug=False): + def __init__( + self, + _ops_to_quantize, + _op_ids_to_skip=None, + _scope=None, + _place=None, + _core=None, + _debug=False, + ): self._scope = _scope self._place = _get_paddle_place(_place) self._core = _core @@ -52,19 +54,26 @@ class Quant2Int8MkldnnPass(object): 'fake_quantize_range_abs_max', ] self._fake_dequantize_types = [ - 'fake_dequantize_max_abs', 'fake_channel_wise_dequantize_max_abs' + 'fake_dequantize_max_abs', + 'fake_channel_wise_dequantize_max_abs', ] self._fake_quantize_dequantize_types = [ 'fake_quantize_dequantize_abs_max', 'fake_quantize_dequantize_moving_average_abs_max', - 'fake_channel_wise_quantize_dequantize_abs_max' + 'fake_channel_wise_quantize_dequantize_abs_max', ] self._ops_to_quantize = _ops_to_quantize - self._op_ids_to_skip = _op_ids_to_skip if _op_ids_to_skip is not None else set( - [-1]) + self._op_ids_to_skip = ( + _op_ids_to_skip if _op_ids_to_skip is not None else set([-1]) + ) self._scale_immutable_ops = [ - 'transpose2', 'reshape2', 'pool2d', 'slice', 'shape', - 'nearest_interp', 'nearest_interp_v2' + 'transpose2', + 'reshape2', + 'pool2d', + 'slice', + 'shape', + 'nearest_interp', + 'nearest_interp_v2', ] self._scale_ops = ['scale'] self._conv_ops = ['conv2d', 'depthwise_conv2d'] @@ -84,8 +93,9 @@ class Quant2Int8MkldnnPass(object): self._pass_group = 'int8' def apply(self, graph): - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' self._reset_pass_idx_and_group('int8') graph = self._label_skip_quantized_op(graph) @@ -105,8 +115,9 @@ class Quant2Int8MkldnnPass(object): return graph def prepare_and_optimize_fp32(self, graph): - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' self._reset_pass_idx_and_group('fp32') graph = self._optimize_fp32_graph(graph) @@ -130,10 +141,10 @@ class Quant2Int8MkldnnPass(object): return any(op.name() in op_types for op in graph.all_op_nodes()) def _is_any_of_op_types_quantized(self, op_types, graph): - return self._is_any_of_op_types_in_graph( - op_types, graph) and (self._is_quantizing_all_ops() - or any(op_type in self._ops_to_quantize - for op_type in op_types)) + return self._is_any_of_op_types_in_graph(op_types, graph) and ( + self._is_quantizing_all_ops() + or any(op_type in self._ops_to_quantize for op_type in op_types) + ) def _is_conv_quantized(self, graph): return self._is_any_of_op_types_quantized(self._conv_ops, graph) @@ -153,8 +164,9 @@ class Quant2Int8MkldnnPass(object): """ target_ops = self._conv_ops + self._mul_ops + self._matmul_ops for op_node in graph.all_op_nodes(): - if op_node.name() in target_ops and \ - not op_node.op().has_attr("quantization_type"): + if op_node.name() in target_ops and not op_node.op().has_attr( + "quantization_type" + ): is_quantized_op = True for var_node in op_node.inputs: for front_op_node in var_node.inputs: @@ -181,21 +193,25 @@ class Quant2Int8MkldnnPass(object): for op in graph.all_op_nodes(): if op.name() in fake_ops: bit_length = op.op().attr("bit_length") - assert bit_length == 8, 'Unsupported number quantization bits ({}). Only 8 is supported now.'.format( - bit_length) + assert ( + bit_length == 8 + ), 'Unsupported number quantization bits ({}). Only 8 is supported now.'.format( + bit_length + ) input_name = op.input("X")[0] scale_name = op.input("InScale")[0] output_name = op.output("Out")[0] # Gather new weight scales after folding batchnorm in convolution scale = np.array( - 1.0 / self._load_param(self._scope, scale_name)[0]).astype( - np.float64) + 1.0 / self._load_param(self._scope, scale_name)[0] + ).astype(np.float64) scale[scale == np.Inf] = 0.0 lod_tensor = self._convert_scale2tensor(scale) use_unsigned_int = False - self._add_scale_for_vars([input_name, output_name], - use_unsigned_int, lod_tensor) + self._add_scale_for_vars( + [input_name, output_name], use_unsigned_int, lod_tensor + ) return graph @@ -205,15 +221,16 @@ class Quant2Int8MkldnnPass(object): input_name = op.input("X")[0] if op.op().has_attr("max_range"): _max_range = np.array(op.op().attr("max_range")).astype( - np.float64) + np.float64 + ) self._weight_thresholds[input_name] = np.array( - self._s8_max * self._s8_max / _max_range).astype( - np.float64) + self._s8_max * self._s8_max / _max_range + ).astype(np.float64) else: scale_name = op.input("Scales")[0] self._weight_thresholds[input_name] = np.array( - self._load_param(self._scope, - scale_name)).astype(np.float64) + self._load_param(self._scope, scale_name) + ).astype(np.float64) return graph @@ -229,14 +246,13 @@ class Quant2Int8MkldnnPass(object): use_unsigned_int = False for output_name in op.op().outputs(): for out_var_name in op.op().output(output_name): - self._add_scale_for_vars([out_var_name], - use_unsigned_int, - scale_lod_tensor) + self._add_scale_for_vars( + [out_var_name], use_unsigned_int, scale_lod_tensor + ) return graph def _propagate_scales(self, graph): - def _update_scale_op_in_scale(op, input, output): unsigned, tensor = self._var_quant_scales[output] scale = np.array(tensor) * op.op().attr("scale") @@ -254,24 +270,28 @@ class Quant2Int8MkldnnPass(object): output_name = op.output("Out")[0] tensor_names = [input_name, output_name] - if all(name not in self._var_quant_scales - for name in tensor_names): + if all( + name not in self._var_quant_scales + for name in tensor_names + ): waiting_for_scale.update(tensor_names) continue elif input_name in self._var_quant_scales: self._var_quant_scales[ - output_name] = self._var_quant_scales[input_name] + output_name + ] = self._var_quant_scales[input_name] elif output_name in self._var_quant_scales: self._var_quant_scales[ - input_name] = self._var_quant_scales[output_name] + input_name + ] = self._var_quant_scales[output_name] elif op.name() == 'concat': output_name = op.output("Out")[0] if output_name in self._var_quant_scales: input_names = op.input("X") for input_name in input_names: self._var_quant_scales[ - input_name] = self._var_quant_scales[ - output_name] + input_name + ] = self._var_quant_scales[output_name] elif op.name() in self._scale_ops: input_name = op.input("X")[0] output_name = op.output("Out")[0] @@ -282,8 +302,10 @@ class Quant2Int8MkldnnPass(object): waiting_for_scale = _update_scales(graph) waiting_for_scale_prev = set() - while len(waiting_for_scale - ) != 0 and waiting_for_scale != waiting_for_scale_prev: + while ( + len(waiting_for_scale) != 0 + and waiting_for_scale != waiting_for_scale_prev + ): waiting_for_scale_prev = waiting_for_scale waiting_for_scale = _update_scales(graph) @@ -305,27 +327,31 @@ class Quant2Int8MkldnnPass(object): def _remove_fake_quantize(self, graph, op): fake_quant_in = graph._find_node_by_name(op.inputs, op.input("X")[0]) - fake_quant_in_scale = graph._find_node_by_name(op.inputs, - op.input("InScale")[0]) - fake_quant_out = graph._find_node_by_name(op.outputs, - op.output("Out")[0]) + fake_quant_in_scale = graph._find_node_by_name( + op.inputs, op.input("InScale")[0] + ) + fake_quant_out = graph._find_node_by_name( + op.outputs, op.output("Out")[0] + ) fake_quant_out_scale = graph._find_node_by_name( - op.outputs, - op.output("OutScale")[0]) + op.outputs, op.output("OutScale")[0] + ) next_ops = fake_quant_out.outputs for next_op in next_ops: self._swap_inputs(next_op, fake_quant_out, fake_quant_in) graph.link_to(fake_quant_in, next_op) graph.safe_remove_nodes( - {op, fake_quant_in_scale, fake_quant_out, fake_quant_out_scale}) + {op, fake_quant_in_scale, fake_quant_out, fake_quant_out_scale} + ) return graph def _remove_fake_dequantize(self, graph, op): fake_dequant_in = graph._find_node_by_name(op.inputs, op.input("X")[0]) - fake_dequant_out = graph._find_node_by_name(op.outputs, - op.output("Out")[0]) + fake_dequant_out = graph._find_node_by_name( + op.outputs, op.output("Out")[0] + ) next_ops = fake_dequant_out.outputs for next_op in next_ops: @@ -338,13 +364,15 @@ class Quant2Int8MkldnnPass(object): def _swap_inputs(self, op, old_input, new_input): for input_name in op.op().input_names(): if old_input.name() in op.input(input_name): - op.op().set_input(input_name, [ - new_input.name() if x == old_input.name() else x - for x in op.input(input_name) - ]) + op.op().set_input( + input_name, + [ + new_input.name() if x == old_input.name() else x + for x in op.input(input_name) + ], + ) def _dequantize_weights(self, graph): - def _is_int8_weights(op_node, weight_name): weight_var_name = op_node.input(weight_name)[0] if self._scope.find_var(weight_var_name) is None: @@ -373,8 +401,10 @@ class Quant2Int8MkldnnPass(object): w_fp32 = np.multiply(np.divide(weight, self._s8_max), scales) else: raise ValueError( - "The size of weight scales vector ({}) does not match the dimensions ({}) of the weights tensor {}." - .format(scales.size, weight.shape, weight_var_name)) + "The size of weight scales vector ({}) does not match the dimensions ({}) of the weights tensor {}.".format( + scales.size, weight.shape, weight_var_name + ) + ) w_fp32 = w_fp32.reshape(weight.shape).astype(np.float32) self._restore_var(weight_var_name, w_fp32) @@ -384,8 +414,9 @@ class Quant2Int8MkldnnPass(object): def _update_activations(self, graph): for op in graph.all_op_nodes(): - if op.name( - ) in self._conv_ops and not op.op().has_attr("fuse_activation"): + if op.name() in self._conv_ops and not op.op().has_attr( + "fuse_activation" + ): activation = "" if op.op().has_attr("fuse_relu") and op.op().attr("fuse_relu"): activation = "relu" @@ -403,8 +434,9 @@ class Quant2Int8MkldnnPass(object): def _optimize_fp32_graph(self, graph): graph = self._update_activations(graph) graph = self._remove_ctrl_vars(graph) - graph = self._apply_pass(graph, 'mkldnn_placement_pass', - ['mkldnn_enabled_op_types'], [set()]) + graph = self._apply_pass( + graph, 'mkldnn_placement_pass', ['mkldnn_enabled_op_types'], [set()] + ) # remove dropout ops graph = self._apply_pass(graph, 'simplify_with_basic_ops_pass') graph = self._apply_pass(graph, 'layer_norm_fuse_pass') @@ -433,29 +465,34 @@ class Quant2Int8MkldnnPass(object): graph = self._apply_pass(graph, 'conv_eltwiseadd_bn_fuse_pass') graph = self._apply_pass(graph, 'conv_affine_channel_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_transpose_bn_fuse_pass') - graph = self._apply_pass(graph, - 'conv_transpose_eltwiseadd_bn_fuse_pass') + graph = self._apply_pass( + graph, 'conv_transpose_eltwiseadd_bn_fuse_pass' + ) graph = self._apply_pass(graph, 'conv_bias_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_transpose_bias_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_elementwise_add_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_activation_mkldnn_fuse_pass') - graph = self._apply_pass(graph, 'fc_fuse_pass', - ['use_gpu', 'use_fc_padding'], [False, False]) + graph = self._apply_pass( + graph, 'fc_fuse_pass', ['use_gpu', 'use_fc_padding'], [False, False] + ) graph = self._apply_pass(graph, 'repeated_fc_relu_fuse_pass') if self._is_fc_quantized(graph): # Disabled due to topology-dependent speed-up graph = self._apply_pass(graph, 'fc_mkldnn_pass') graph = self._apply_pass(graph, 'fc_act_mkldnn_fuse_pass') - graph = self._apply_pass(graph, - 'matmul_transpose_reshape_mkldnn_fuse_pass') - graph = self._apply_pass(graph, - 'matmul_elementwise_add_mkldnn_fuse_pass') + graph = self._apply_pass( + graph, 'matmul_transpose_reshape_mkldnn_fuse_pass' + ) + graph = self._apply_pass( + graph, 'matmul_elementwise_add_mkldnn_fuse_pass' + ) graph = self._apply_pass(graph, 'matmul_activation_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'batch_norm_act_fuse_pass') graph = self._apply_pass(graph, 'softplus_activation_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'scale_matmul_fuse_pass') - graph = self._apply_pass(graph, - 'reshape_transpose_matmul_mkldnn_fuse_pass') + graph = self._apply_pass( + graph, 'reshape_transpose_matmul_mkldnn_fuse_pass' + ) # the following pass should be the last one since it will work on all fused ops. graph = self._apply_pass(graph, 'runtime_context_cache_pass') return graph @@ -474,8 +511,10 @@ class Quant2Int8MkldnnPass(object): ir_pass.apply(cpp_graph) if self._debug: graph.draw( - '.', '{}_{}_{}'.format(self._pass_group, self._pass_idx, - pass_name), graph.all_op_nodes()) + '.', + '{}_{}_{}'.format(self._pass_group, self._pass_idx, pass_name), + graph.all_op_nodes(), + ) self._remove_unused_var_nodes(graph) self._pass_idx += 1 return graph @@ -502,8 +541,10 @@ class Quant2Int8MkldnnPass(object): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) return graph @@ -515,45 +556,60 @@ class Quant2Int8MkldnnPass(object): return graph def _compute_weight_scales(self, graph): - def _compute_var_scales(ops, w_name, axis): for op in graph.all_op_nodes(): if op.op().type() in ops: weight_var_name = op.input(w_name)[0] weights = np.array( - self._load_param(self._scope, weight_var_name)) - scales = 1.0 / np.amax(np.abs( - weights.reshape(weights.shape[0], -1)).astype( - np.float64), - axis=axis) + self._load_param(self._scope, weight_var_name) + ) + scales = 1.0 / np.amax( + np.abs(weights.reshape(weights.shape[0], -1)).astype( + np.float64 + ), + axis=axis, + ) scales[scales == np.Inf] = 0.0 lod_tensor = self._convert_scale2tensor(scales) use_unsigned_int = False - self._var_quant_scales[weight_var_name] = (use_unsigned_int, - lod_tensor) + self._var_quant_scales[weight_var_name] = ( + use_unsigned_int, + lod_tensor, + ) def _compute_single_gru_weight_scales(wx_var_name, wh_var_name): wx = np.array(self._load_param(self._scope, wx_var_name)) wh = np.array(self._load_param(self._scope, wh_var_name)) OC = wh.shape[0] - scale_ur = 1.0 / np.max(np.abs( - np.concatenate([ - wx[:, :2 * OC], - wh.flatten()[:2 * OC * OC].reshape(OC, 2 * OC) - ], - axis=0)), - axis=0) - scale_o = 1.0 / np.max(np.abs( - np.concatenate([ - wx[:, 2 * OC:], - wh.flatten()[2 * OC * OC:].reshape(OC, OC) - ], - axis=0)), - axis=0) - - gru_weights_scale = np.concatenate([scale_ur, - scale_o]).astype('float') + scale_ur = 1.0 / np.max( + np.abs( + np.concatenate( + [ + wx[:, : 2 * OC], + wh.flatten()[: 2 * OC * OC].reshape(OC, 2 * OC), + ], + axis=0, + ) + ), + axis=0, + ) + scale_o = 1.0 / np.max( + np.abs( + np.concatenate( + [ + wx[:, 2 * OC :], + wh.flatten()[2 * OC * OC :].reshape(OC, OC), + ], + axis=0, + ) + ), + axis=0, + ) + + gru_weights_scale = np.concatenate([scale_ur, scale_o]).astype( + 'float' + ) return self._convert_scale2tensor(gru_weights_scale) @@ -563,21 +619,26 @@ class Quant2Int8MkldnnPass(object): assert len(op.input(wx_name)) == len( op.input(wh_name) ), 'Mismatch in number of weights inputs ({} for WeightX vs. {} for WeightH).'.format( - len(op.input(wx_name)), len(op.input(wh_name))) + len(op.input(wx_name)), len(op.input(wh_name)) + ) for i, wx_var_name in enumerate(op.input(wx_name)): wh_var_name = op.input(wh_name)[i] use_unsigned_int = False lod_tensor = _compute_single_gru_weight_scales( - wx_var_name, wh_var_name) - self._var_quant_scales[wx_var_name] = (use_unsigned_int, - lod_tensor) + wx_var_name, wh_var_name + ) + self._var_quant_scales[wx_var_name] = ( + use_unsigned_int, + lod_tensor, + ) def _compute_single_lstm_weight_scales(wx_var_name, wh_var_name): wx = np.array(self._load_param(self._scope, wx_var_name)) wh = np.array(self._load_param(self._scope, wh_var_name)) lstm_weights_scale = 1.0 / np.max( - np.abs(np.concatenate([wx[:, :], wh[:, :]], axis=0)), axis=0) + np.abs(np.concatenate([wx[:, :], wh[:, :]], axis=0)), axis=0 + ) lstm_weights_scale = lstm_weights_scale.astype('float') return self._convert_scale2tensor(lstm_weights_scale) @@ -588,14 +649,18 @@ class Quant2Int8MkldnnPass(object): assert len(op.input(wx_name)) == len( op.input(wh_name) ), 'Mismatch in number of weights inputs ({} for WeightX vs. {} for WeightH).'.format( - len(op.input(wx_name)), len(op.input(wh_name))) + len(op.input(wx_name)), len(op.input(wh_name)) + ) for i, wx_var_name in enumerate(op.input(wx_name)): wh_var_name = op.input(wh_name)[i] use_unsigned_int = False lod_tensor = _compute_single_lstm_weight_scales( - wx_var_name, wh_var_name) - self._var_quant_scales[wx_var_name] = (use_unsigned_int, - lod_tensor) + wx_var_name, wh_var_name + ) + self._var_quant_scales[wx_var_name] = ( + use_unsigned_int, + lod_tensor, + ) _compute_var_scales(self._conv_ops, "Filter", axis=1) _compute_var_scales(self._fc_ops, "W", axis=0) @@ -606,7 +671,6 @@ class Quant2Int8MkldnnPass(object): return graph def _update_relu_output_scales(self, graph): - def _set_unsigned_scale(graph, ops, op_out_name, predicate): ''' Sets the type of an output scale of a passed op type(s) to 'unsigned int8' if the @@ -617,7 +681,8 @@ class Quant2Int8MkldnnPass(object): if op.name() in ops: out_name = op.output(op_out_name)[0] if out_name in self._var_quant_scales and predicate( - op.op()): + op.op() + ): is_unsigned, tensor = self._var_quant_scales[out_name] if is_unsigned is False: # If the variable is signed, it means that the scales for this var @@ -625,23 +690,26 @@ class Quant2Int8MkldnnPass(object): # to fill the entire range of uint8 scale = np.array(tensor) * 2 tensor = self._convert_scale2tensor( - scale.astype(np.float64)) + scale.astype(np.float64) + ) self._var_quant_scales[out_name] = (True, tensor) return graph def conv_predicate(op): return op.attr("fuse_activation") in self._relu_ops - graph = _set_unsigned_scale(graph, self._conv_ops, "Output", - conv_predicate) + graph = _set_unsigned_scale( + graph, self._conv_ops, "Output", conv_predicate + ) def fc_predicate(op): return op.attr("activation_type") in self._relu_ops graph = _set_unsigned_scale(graph, self._fc_ops, "Out", fc_predicate) - graph = _set_unsigned_scale(graph, self._relu_ops, 'Out', - lambda op: True) + graph = _set_unsigned_scale( + graph, self._relu_ops, 'Out', lambda op: True + ) return graph @@ -650,15 +718,21 @@ class Quant2Int8MkldnnPass(object): def _quantize_fp32_graph(self, graph): graph = self._apply_pass(graph, 'scale_matmul_fuse_pass') - graph = self._apply_pass(graph, - 'reshape_transpose_matmul_mkldnn_fuse_pass') - graph = self._apply_pass(graph, 'cpu_quantize_placement_pass', - ['quantize_enabled_op_types'], - [self._ops_to_quantize]) graph = self._apply_pass( - graph, 'cpu_quantize_pass', ['quant_var_scales', 'data_layout'], - [self._var_quant_scales, - self._get_data_layout(graph)]) + graph, 'reshape_transpose_matmul_mkldnn_fuse_pass' + ) + graph = self._apply_pass( + graph, + 'cpu_quantize_placement_pass', + ['quantize_enabled_op_types'], + [self._ops_to_quantize], + ) + graph = self._apply_pass( + graph, + 'cpu_quantize_pass', + ['quant_var_scales', 'data_layout'], + [self._var_quant_scales, self._get_data_layout(graph)], + ) graph = self._apply_pass(graph, 'cpu_quantize_squash_pass') graph = self._apply_pass(graph, 'int8_scale_calculation_mkldnn_pass') graph = self._apply_pass(graph, 'params_quantization_mkldnn_pass') diff --git a/python/paddle/fluid/contrib/slim/quantization/quant_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/quantization/quant_int8_mkldnn_pass.py index d56aeb79f3f..73c611db012 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quant_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quant_int8_mkldnn_pass.py @@ -66,7 +66,7 @@ class QuantInt8MkldnnPass(object): self._quantize_type = [ 'fake_quantize_moving_average_abs_max', - 'fake_quantize_range_abs_max' + 'fake_quantize_range_abs_max', ] self._dequantize_type = ['fake_dequantize_max_abs'] self._quantize_dequantize_type = [ @@ -92,8 +92,9 @@ class QuantInt8MkldnnPass(object): graph(IrGraph): the applied graph. """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' ops = graph.all_op_nodes() persistable_vars = [p.name() for p in graph.all_persistable_nodes()] @@ -104,7 +105,8 @@ class QuantInt8MkldnnPass(object): input_name = op_node.input("X")[0] scale_name = op_node.input("Scale")[0] self._in_scale[input_name] = self._load_param( - self._scope, scale_name)[0] + self._scope, scale_name + )[0] self._max_range[input_name] = op_node.op().attr("max_range") self._new_output[input_name] = op_node.output("Out")[0] @@ -114,7 +116,8 @@ class QuantInt8MkldnnPass(object): input_name = op_node.input("X")[0] scale_name = op_node.input("InScale")[0] self._in_scale[input_name] = self._load_param( - self._scope, scale_name)[0] + self._scope, scale_name + )[0] # self._max_range[input_name] = op_node.op().attr("max_range") self._new_output[input_name] = op_node.output("Out")[0] @@ -142,29 +145,30 @@ class QuantInt8MkldnnPass(object): output_name = op_node.output("Output")[0] # Convert int8 range weights to fp32 range weights weight = self._load_param(self._scope, weight_name) - w_fp32 = np.divide(np.multiply(weight, self._s8_max), - self._max_range[output_name]) + w_fp32 = np.divide( + np.multiply(weight, self._s8_max), self._max_range[output_name] + ) w_fp32 = w_fp32.reshape(weight.shape) self._restore_var(weight_name, w_fp32) - input_var_node = graph._find_node_by_name(op_node.inputs, - op_node.input("Input")[0]) + input_var_node = graph._find_node_by_name( + op_node.inputs, op_node.input("Input")[0] + ) weight_var_node = graph._find_node_by_name(op_node.inputs, weight_name) # Set fake_dequantize_abs_max's output as new output of conv2d output_var_node = graph._find_node_by_name( - graph.all_var_nodes(), self._new_output[output_name]) + graph.all_var_nodes(), self._new_output[output_name] + ) attrs = { - name: op_node.op().attr(name) - for name in op_node.op().attr_names() + name: op_node.op().attr(name) for name in op_node.op().attr_names() } - conv_op_node = graph.create_op_node(op_type='conv2d', - attrs=attrs, - inputs={ - 'Input': input_var_node, - 'Filter': weight_var_node - }, - outputs={'Output': output_var_node}) + conv_op_node = graph.create_op_node( + op_type='conv2d', + attrs=attrs, + inputs={'Input': input_var_node, 'Filter': weight_var_node}, + outputs={'Output': output_var_node}, + ) # Based on the Quant's scales to calculate the scales of MKL-DNN INT8 conv2d scale_in = self._s8_max / self._in_scale[output_name] @@ -187,29 +191,30 @@ class QuantInt8MkldnnPass(object): output_name = op_node.output("Out")[0] # Convert int8 range weights to fp32 range weights weight = self._load_param(self._scope, weight_name) - w_fp32 = np.divide(np.multiply(weight, self._s8_max), - self._max_range[output_name]) + w_fp32 = np.divide( + np.multiply(weight, self._s8_max), self._max_range[output_name] + ) w_fp32 = w_fp32.reshape(weight.shape) self._restore_var(weight_name, w_fp32) - input_var_node = graph._find_node_by_name(op_node.inputs, - op_node.input("X")[0]) + input_var_node = graph._find_node_by_name( + op_node.inputs, op_node.input("X")[0] + ) weight_var_node = graph._find_node_by_name(op_node.inputs, weight_name) # Set fake_dequantize_abs_max's output as new output of mul output_var_node = graph._find_node_by_name( - graph.all_var_nodes(), self._new_output[output_name]) + graph.all_var_nodes(), self._new_output[output_name] + ) attrs = { - name: op_node.op().attr(name) - for name in op_node.op().attr_names() + name: op_node.op().attr(name) for name in op_node.op().attr_names() } - mul_op_node = graph.create_op_node(op_type='mul', - attrs=attrs, - inputs={ - 'X': input_var_node, - 'Y': weight_var_node - }, - outputs={'Out': output_var_node}) + mul_op_node = graph.create_op_node( + op_type='mul', + attrs=attrs, + inputs={'X': input_var_node, 'Y': weight_var_node}, + outputs={'Out': output_var_node}, + ) # Based on the Quant's scales to calculate MKL-DNN INT8 mul's scales scale_in = self._s8_max / self._in_scale[output_name] @@ -230,30 +235,35 @@ class QuantInt8MkldnnPass(object): """ Transform fake_quantize_xx op to quantize mkldnn op in the graph. """ - input_var_node = graph._find_node_by_name(op_node.inputs, - op_node.input("X")[0]) - output_var_node = graph._find_node_by_name(op_node.outputs, - op_node.output("Out")[0]) - scale_in = self._s8_max / self._load_param( - self._scope, - op_node.input("InScale")[0])[0] + input_var_node = graph._find_node_by_name( + op_node.inputs, op_node.input("X")[0] + ) + output_var_node = graph._find_node_by_name( + op_node.outputs, op_node.output("Out")[0] + ) + scale_in = ( + self._s8_max + / self._load_param(self._scope, op_node.input("InScale")[0])[0] + ) quant_op_node = graph.create_op_node( op_type='quantize', attrs={ 'data_format': 'MKLDNNLAYOUT', 'use_mkldnn': 1, 'Scale': scale_in, - 'is_negative_input': 1 + 'is_negative_input': 1, }, inputs={'Input': input_var_node}, - outputs={'Output': output_var_node}) + outputs={'Output': output_var_node}, + ) graph.link_to(input_var_node, quant_op_node) graph.link_to(quant_op_node, output_var_node) graph.safe_remove_nodes(op_node) def _remove_fake_dequantize_op(self, graph, op_node): - input_var_node = graph._find_node_by_name(op_node.inputs, - op_node.input("X")[0]) + input_var_node = graph._find_node_by_name( + op_node.inputs, op_node.input("X")[0] + ) graph.safe_remove_nodes(op_node) def _load_param(self, scope, param_name): @@ -275,7 +285,9 @@ class QuantInt8MkldnnPass(object): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) diff --git a/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py b/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py index abd6a7822d8..81aac8823d8 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py @@ -14,6 +14,7 @@ import collections import numpy as np + try: from tqdm import tqdm except: @@ -47,12 +48,15 @@ __all__ = [ ] _fake_quant_op_list = [ - 'fake_quantize_abs_max', 'fake_quantize_range_abs_max', - 'fake_quantize_moving_average_abs_max', 'fake_channel_wise_quantize_abs_max' + 'fake_quantize_abs_max', + 'fake_quantize_range_abs_max', + 'fake_quantize_moving_average_abs_max', + 'fake_channel_wise_quantize_abs_max', ] _fake_dequant_op_list = [ - 'fake_dequantize_max_abs', 'fake_channel_wise_dequantize_max_abs' + 'fake_dequantize_max_abs', + 'fake_channel_wise_dequantize_max_abs', ] _fake_quant_dequant_op_list = [ @@ -67,12 +71,11 @@ _SCALE_DEFAULT_VALUE = 0.001 def _init_var_node(var_node, value, scope, place): - assert isinstance(value, - np.ndarray), 'The type of value should be numpy array.' - assert scope is not None, \ - 'The scope cannot be set None.' - assert place is not None, \ - 'The place cannot be set None.' + assert isinstance( + value, np.ndarray + ), 'The type of value should be numpy array.' + assert scope is not None, 'The scope cannot be set None.' + assert place is not None, 'The place cannot be set None.' tensor = scope.var(var_node.name()).get_tensor() tensor.set(value, place) @@ -84,8 +87,9 @@ def _is_input_all_not_persistable(graph, op_node): is_input_all_not_persistable = True for var_name in utils._get_op_input_var_names(op_node): in_node = graph._find_node_by_name(op_node.inputs, var_name) - is_input_all_not_persistable = (is_input_all_not_persistable and \ - (not in_node.persistable())) + is_input_all_not_persistable = is_input_all_not_persistable and ( + not in_node.persistable() + ) return is_input_all_not_persistable @@ -109,24 +113,26 @@ class QuantizationTransformPass(object): the quantized ops's inputs. """ - def __init__(self, - scope=None, - place=None, - weight_bits=8, - activation_bits=8, - activation_quantize_type='abs_max', - weight_quantize_type='abs_max', - window_size=10000, - moving_rate=0.9, - skip_pattern=['skip_quant'], - quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul'], - weight_quantize_func=None, - act_quantize_func=None, - weight_preprocess_func=None, - act_preprocess_func=None, - optimizer_func=None, - executor=None, - is_test=None): + def __init__( + self, + scope=None, + place=None, + weight_bits=8, + activation_bits=8, + activation_quantize_type='abs_max', + weight_quantize_type='abs_max', + window_size=10000, + moving_rate=0.9, + skip_pattern=['skip_quant'], + quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul'], + weight_quantize_func=None, + act_quantize_func=None, + weight_preprocess_func=None, + act_preprocess_func=None, + optimizer_func=None, + executor=None, + is_test=None, + ): r""" Constructor. @@ -215,21 +221,26 @@ class QuantizationTransformPass(object): self._optimizer = optimizer_func self._exe = executor quant_type = [ - 'abs_max', 'channel_wise_abs_max', 'range_abs_max', - 'moving_average_abs_max' + 'abs_max', + 'channel_wise_abs_max', + 'range_abs_max', + 'moving_average_abs_max', ] - assert activation_quantize_type != 'channel_wise_abs_max', \ - "The activation quantization type does not support 'channel_wise_abs_max'." + assert ( + activation_quantize_type != 'channel_wise_abs_max' + ), "The activation quantization type does not support 'channel_wise_abs_max'." if activation_quantize_type not in quant_type: raise ValueError( "Unknown activation_quantize_type : '%s'. It can only be " - "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'." % - (str(activation_quantize_type))) + "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'." + % (str(activation_quantize_type)) + ) if weight_quantize_type not in quant_type: raise ValueError( "Unknown weight_quantize_type: '%s'. It can only be " "'abs_max' or 'channel_wise_abs_max' or 'range_abs_max' " - "or 'moving_average_abs_max'." % (str(weight_quantize_type))) + "or 'moving_average_abs_max'." % (str(weight_quantize_type)) + ) self._activation_quantize_type = activation_quantize_type self._weight_quantize_type = weight_quantize_type @@ -238,8 +249,9 @@ class QuantizationTransformPass(object): self._quantizable_ops = quantizable_op_type for op in self._quantizable_ops: - assert op in utils._weight_supported_quantizable_op_type, \ + assert op in utils._weight_supported_quantizable_op_type, ( op + " is not supported for quantization." + ) self._quantizable_grad_ops = [ '%s_grad' % (op) for op in self._quantizable_ops ] @@ -260,8 +272,9 @@ class QuantizationTransformPass(object): Returns: None """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() # marked the variable which has been dequantized. @@ -272,13 +285,18 @@ class QuantizationTransformPass(object): def _quant_preprocess(op_node): user_skipped = False if isinstance(self._skip_pattern, list): - user_skipped = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") \ - for pattern in self._skip_pattern) + user_skipped = op_node.op().has_attr("op_namescope") and any( + pattern in op_node.op().attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - user_skipped = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find( - self._skip_pattern) != -1 + user_skipped = ( + op_node.op().has_attr("op_namescope") + and op_node.op() + .attr("op_namescope") + .find(self._skip_pattern) + != -1 + ) if user_skipped: op_node.op()._set_attr("skip_quant", True) @@ -297,8 +315,9 @@ class QuantizationTransformPass(object): name = var_node.name() if name in processed_vars: continue - is_weight = True if var_node.name() in persistable_vars \ - else False + is_weight = ( + True if var_node.name() in persistable_vars else False + ) # if var node is weight and weight_preprocess_func is not None, # will insert weight preprocess func @@ -308,11 +327,14 @@ class QuantizationTransformPass(object): # to preorocess activation before quantization if is_weight and self._weight_preprocess_func is not None: var_node = self._insert_func( - graph, self._weight_preprocess_func, var_node, op) - elif not is_weight and self._act_preprocess_func is not None: - var_node = self._insert_func(graph, - self._act_preprocess_func, - var_node, op) + graph, self._weight_preprocess_func, var_node, op + ) + elif ( + not is_weight and self._act_preprocess_func is not None + ): + var_node = self._insert_func( + graph, self._act_preprocess_func, var_node, op + ) # if var node is weight and weight_quantize_func is not None, # will insert weight quantize func to quantize and dequantize weight @@ -320,32 +342,55 @@ class QuantizationTransformPass(object): # will insert act quantize func to quantize and dequantize activation if is_weight and self._weight_quantize_func is not None: target_out_node = self._insert_func( - graph, self._weight_quantize_func, var_node, op) + graph, self._weight_quantize_func, var_node, op + ) processed_vars.append(name) continue elif not is_weight and self._act_quantize_func is not None: target_out_node = self._insert_func( - graph, self._act_quantize_func, var_node, op) + graph, self._act_quantize_func, var_node, op + ) processed_vars.append(name) continue - quant_bits = self._weight_bits if var_node.name() in persistable_vars \ + quant_bits = ( + self._weight_bits + if var_node.name() in persistable_vars else self._activation_bits - quant_type = self._weight_quantize_type if is_weight \ + ) + quant_type = ( + self._weight_quantize_type + if is_weight else self._activation_quantize_type - if quant_type == 'channel_wise_abs_max': # Weight quantization - quant_axis = 1 if op.name() in \ - utils._channelwise_quant_axis1_ops else 0 - quant_var_node, scale_var_node = self._insert_channel_quant_op( - graph, var_node, name, quant_bits, quant_axis) + ) + if ( + quant_type == 'channel_wise_abs_max' + ): # Weight quantization + quant_axis = ( + 1 + if op.name() in utils._channelwise_quant_axis1_ops + else 0 + ) + ( + quant_var_node, + scale_var_node, + ) = self._insert_channel_quant_op( + graph, var_node, name, quant_bits, quant_axis + ) dequant_var_node = self._insert_channel_dequant_op( - graph, quant_var_node, [scale_var_node], - [quant_bits], quant_axis) + graph, + quant_var_node, + [scale_var_node], + [quant_bits], + quant_axis, + ) else: quant_var_node, scale_var_node = self._insert_quant_op( - graph, var_node, name, quant_bits, quant_type) + graph, var_node, name, quant_bits, quant_type + ) dequant_var_node = self._insert_dequant_op( - graph, quant_var_node, scale_var_node, quant_bits) + graph, quant_var_node, scale_var_node, quant_bits + ) dequantized_vars[name] = dequant_var_node graph.update_input_link(var_node, dequant_var_node, op) @@ -373,17 +418,20 @@ class QuantizationTransformPass(object): # Do the preproccess of quantization, such as skipping some ops # for not being quantized. for op in ops: - if op.name() in self._quantizable_ops or \ - op.name() in self._quantizable_grad_ops: + if ( + op.name() in self._quantizable_ops + or op.name() in self._quantizable_grad_ops + ): _quant_preprocess(op) # Insert mapping table to solve the problem in saving inference model. graph.out_node_mapping_table = dict() # The process of _transform_forward and _transform_backward is needed in two for loops. # The loop for transforming the forward graph: - with tqdm(total=len(ops), - bar_format= - 'Adding quant op with weight:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(ops), + bar_format='Adding quant op with weight:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op in ops: if op.name() in self._quantizable_ops: if not self._is_skip_quant(graph, op) and _has_weight(op): @@ -397,8 +445,10 @@ class QuantizationTransformPass(object): return graph def _create_global_step(self, graph): - if self._weight_quantize_type == 'range_abs_max' or \ - self._activation_quantize_type == 'range_abs_max': + if ( + self._weight_quantize_type == 'range_abs_max' + or self._activation_quantize_type == 'range_abs_max' + ): counter_name = '@STEP_COUNTER@' for node in graph.all_var_nodes(): if node.name() == counter_name: @@ -408,21 +458,27 @@ class QuantizationTransformPass(object): name=counter_name, var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=core.VarDesc.VarType.INT64) - _init_var_node(global_step_in, np.zeros([1], dtype='int64'), - self._scope, self._place) + var_dtype=core.VarDesc.VarType.INT64, + ) + _init_var_node( + global_step_in, + np.zeros([1], dtype='int64'), + self._scope, + self._place, + ) global_step_out = graph.create_var_node_from_desc( - global_step_in.var()) + global_step_in.var() + ) # The attribute of `op_role` is needed by ParallelExecutor. increment_op = graph.create_op_node( op_type='increment', attrs={ 'step': 1.0, - 'op_role': - core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, inputs={'X': global_step_in}, - outputs={'Out': global_step_out}) + outputs={'Out': global_step_out}, + ) graph.link_to(global_step_in, increment_op) graph.link_to(increment_op, global_step_out) self._global_step = global_step_out @@ -432,14 +488,17 @@ class QuantizationTransformPass(object): Insert fake_quantize_op in the graph. """ if quant_type == 'abs_max': - return self._insert_quant_abs_max_op(graph, var_node, name, - quant_bits) + return self._insert_quant_abs_max_op( + graph, var_node, name, quant_bits + ) elif quant_type == 'range_abs_max': - return self._insert_quant_range_abs_max_op(graph, var_node, name, - quant_bits) + return self._insert_quant_range_abs_max_op( + graph, var_node, name, quant_bits + ) elif quant_type == 'moving_average_abs_max': return self._insert_quant_moving_average_abs_max_op( - graph, var_node, name, quant_bits) + graph, var_node, name, quant_bits + ) def _insert_quant_abs_max_op(self, graph, var_node, name, quant_bits): """ @@ -451,33 +510,37 @@ class QuantizationTransformPass(object): name=self._quantized_var_name(name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) scale_name = self._quantized_scale_name(name) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: scale_value = np.array( - self._scope.find_var(scale_name).get_tensor()) + self._scope.find_var(scale_name).get_tensor() + ) except: scale_value = np.zeros([1], dtype=data_type) scale_var_node = graph.create_persistable_node( name=scale_name, var_type=var_node.type(), shape=[1], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_var_node, scale_value, self._scope, self._place) quant_op_node = graph.create_op_node( op_type='fake_quantize_abs_max', attrs={ 'bit_length': quant_bits, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, inputs={'X': var_node}, - outputs={ - 'Out': quant_var_node, - 'OutScale': scale_var_node - }) + outputs={'Out': quant_var_node, 'OutScale': scale_var_node}, + ) graph.link_to(var_node, quant_op_node) graph.link_to(quant_op_node, quant_var_node) graph.link_to(quant_op_node, scale_var_node) @@ -493,21 +556,27 @@ class QuantizationTransformPass(object): name=self._quantized_var_name(name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) scale_name = self._quantized_scale_name(name) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: scale_value = np.array( - self._scope.find_var(scale_name).get_tensor()) + self._scope.find_var(scale_name).get_tensor() + ) except: scale_value = np.array([_SCALE_DEFAULT_VALUE], dtype=data_type) scale_in_node = graph.create_persistable_node( name=scale_name, var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_in_node, scale_value, self._scope, self._place) scale_out_node = graph.create_var_node_from_desc(scale_in_node.var()) @@ -520,12 +589,19 @@ class QuantizationTransformPass(object): name=unique_name.generate('scales'), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[self._window_size], - var_dtype=var_node.dtype()) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(scales_node, - np.zeros([self._window_size], dtype=data_type), - self._scope, self._place) + var_dtype=var_node.dtype(), + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + scales_node, + np.zeros([self._window_size], dtype=data_type), + self._scope, + self._place, + ) inputs['Iter'] = self._global_step outputs['OutScales'] = scales_node @@ -533,13 +609,14 @@ class QuantizationTransformPass(object): 'window_size': self._window_size, 'bit_length': quant_bits, 'is_test': self._is_test, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, } quant_op_node = graph.create_op_node( op_type='fake_quantize_range_abs_max', attrs=attrs, inputs=inputs, - outputs=outputs) + outputs=outputs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_in_node, quant_op_node) @@ -552,28 +629,34 @@ class QuantizationTransformPass(object): return quant_var_node, scale_out_node - def _insert_quant_moving_average_abs_max_op(self, graph, var_node, name, - quant_bits): - """Insert fake_quantize_moving_average_abs_max - """ + def _insert_quant_moving_average_abs_max_op( + self, graph, var_node, name, quant_bits + ): + """Insert fake_quantize_moving_average_abs_max""" quant_var_node = graph.create_var_node( name=self._quantized_var_name(name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) scale_name = self._quantized_scale_name(name) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: scale_value = np.array( - self._scope.find_var(scale_name).get_tensor()) + self._scope.find_var(scale_name).get_tensor() + ) except: scale_value = np.array([_SCALE_DEFAULT_VALUE], dtype=data_type) scale_in_node = graph.create_persistable_node( name=scale_name, var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_in_node, scale_value, self._scope, self._place) scale_out_node = graph.create_var_node_from_desc(scale_in_node.var()) @@ -584,22 +667,37 @@ class QuantizationTransformPass(object): name=unique_name.generate('state'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(state_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + state_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) accum_in_node = graph.create_persistable_node( name=unique_name.generate('accum'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - _init_var_node(accum_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + accum_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) state_out_node = graph.create_var_node_from_desc( - state_in_node.var()) + state_in_node.var() + ) accum_out_node = graph.create_var_node_from_desc( - accum_in_node.var()) + accum_in_node.var() + ) ins['InState'] = state_in_node ins['InAccum'] = accum_in_node @@ -610,14 +708,15 @@ class QuantizationTransformPass(object): 'bit_length': quant_bits, 'moving_rate': self._moving_rate, 'is_test': self._is_test, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, } quant_op_node = graph.create_op_node( op_type='fake_quantize_moving_average_abs_max', attrs=attrs, inputs=ins, - outputs=outs) + outputs=outs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_in_node, quant_op_node) @@ -632,8 +731,9 @@ class QuantizationTransformPass(object): return quant_var_node, scale_out_node - def _insert_channel_quant_op(self, graph, var_node, name, quant_bits, - quant_axis): + def _insert_channel_quant_op( + self, graph, var_node, name, quant_bits, quant_axis + ): """ Insert fake_channel_wise_quantize_abs_max op in the graph. """ @@ -643,21 +743,28 @@ class QuantizationTransformPass(object): name=self._quantized_var_name(name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) scale_name = self._quantized_scale_name(name) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: scale_value = np.array( - self._scope.find_var(scale_name).get_tensor()) + self._scope.find_var(scale_name).get_tensor() + ) except: - scale_value = np.zeros([var_node.shape()[quant_axis]], - dtype=data_type) + scale_value = np.zeros( + [var_node.shape()[quant_axis]], dtype=data_type + ) scale_var_node = graph.create_persistable_node( name=self._quantized_scale_name(name), var_type=var_node.type(), shape=[var_node.shape()[quant_axis]], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_var_node, scale_value, self._scope, self._place) quant_op_node = graph.create_op_node( op_type='fake_channel_wise_quantize_abs_max', @@ -665,13 +772,11 @@ class QuantizationTransformPass(object): 'bit_length': quant_bits, 'quant_axis': quant_axis, 'is_test': self._is_test, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, inputs={'X': var_node}, - outputs={ - 'Out': quant_var_node, - 'OutScale': scale_var_node - }) + outputs={'Out': quant_var_node, 'OutScale': scale_var_node}, + ) graph.link_to(var_node, quant_op_node) graph.link_to(quant_op_node, quant_var_node) graph.link_to(quant_op_node, scale_var_node) @@ -687,26 +792,26 @@ class QuantizationTransformPass(object): name=self._dequantized_var_name(var_node.name()), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) max_range = (1 << (quant_bits - 1)) - 1 dequant_op_node = graph.create_op_node( op_type='fake_dequantize_max_abs', attrs={ 'max_range': float(max_range), - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward - }, - inputs={ - 'X': var_node, - 'Scale': scale_var_node + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, - outputs={'Out': dequant_var_node}) + inputs={'X': var_node, 'Scale': scale_var_node}, + outputs={'Out': dequant_var_node}, + ) graph.link_to(var_node, dequant_op_node) graph.link_to(scale_var_node, dequant_op_node) graph.link_to(dequant_op_node, dequant_var_node) return dequant_var_node - def _insert_channel_dequant_op(self, graph, var_node, scale_var_nodes, - quant_bits, quant_axis): + def _insert_channel_dequant_op( + self, graph, var_node, scale_var_nodes, quant_bits, quant_axis + ): """ Insert fake_channel_wise_dequantize_max_abs in the graph. """ @@ -716,19 +821,18 @@ class QuantizationTransformPass(object): name=self._dequantized_var_name(var_node.name()), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) dequant_op_node = graph.create_op_node( op_type='fake_channel_wise_dequantize_max_abs', attrs={ 'quant_bits': quant_bits, 'quant_axis': quant_axis, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward - }, - inputs={ - 'X': var_node, - 'Scales': scale_var_nodes + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, - outputs={'Out': dequant_var_node}) + inputs={'X': var_node, 'Scales': scale_var_nodes}, + outputs={'Out': dequant_var_node}, + ) graph.link_to(var_node, dequant_op_node) for scale_n in scale_var_nodes: graph.link_to(scale_n, dequant_op_node) @@ -815,27 +919,34 @@ class QuantizationTransformPass(object): startup_program = Program() with program_guard(tmp_program, startup_program): with unique_name.guard(var_node.name() + "_"): - in_node = data(var_node.name() + '_tmp_input', - shape=var_node.shape(), - dtype='float32') + in_node = data( + var_node.name() + '_tmp_input', + shape=var_node.shape(), + dtype='float32', + ) out_node = func(in_node) graph.out_node_mapping_table[out_node.name] = var_node.name() # loss shape must be 1 when minimize loss = mean(out_node) if not graph._for_test: - assert self._optimizer, "optimizer_func must be set when graph is test graph" + assert ( + self._optimizer + ), "optimizer_func must be set when graph is test graph" in_node.stop_gradient = False optimizer = self._optimizer() optimizer.minimize(loss) with scope_guard(self._scope): self._exe.run(startup_program) - tmp_graph = IrGraph(core.Graph(tmp_program.desc), - for_test=graph._for_test) - in_node = tmp_graph._find_node_by_name(tmp_graph.all_var_nodes(), - in_node.name) - out_node = tmp_graph._find_node_by_name(tmp_graph.all_var_nodes(), - out_node.name) + tmp_graph = IrGraph( + core.Graph(tmp_program.desc), for_test=graph._for_test + ) + in_node = tmp_graph._find_node_by_name( + tmp_graph.all_var_nodes(), in_node.name + ) + out_node = tmp_graph._find_node_by_name( + tmp_graph.all_var_nodes(), out_node.name + ) in_node_params = [] in_op_node = [] @@ -854,10 +965,12 @@ class QuantizationTransformPass(object): for node in in_op_node: self._copy_graph(graph, tmp_graph, node) - target_in_node = graph._find_node_by_name(graph.all_var_nodes(), - in_node.name()) - target_out_node = graph._find_node_by_name(graph.all_var_nodes(), - out_node.name()) + target_in_node = graph._find_node_by_name( + graph.all_var_nodes(), in_node.name() + ) + target_out_node = graph._find_node_by_name( + graph.all_var_nodes(), out_node.name() + ) loss_node = graph._find_node_by_name(graph.all_var_nodes(), loss.name) outputs = target_in_node.outputs for node in outputs: @@ -867,16 +980,17 @@ class QuantizationTransformPass(object): # update grad if not graph._for_test: op_out = op.outputs[0] - op_out_grad = graph._find_node_by_name(graph.all_var_nodes(), - op_out.name() + "@GRAD") + op_out_grad = graph._find_node_by_name( + graph.all_var_nodes(), op_out.name() + "@GRAD" + ) # find op's gradient op, such as conv2d_grad op_grad = op_out_grad.outputs[0] target_out_grad_node = graph._find_node_by_name( - graph.all_var_nodes(), - target_out_node.name() + "@GRAD") + graph.all_var_nodes(), target_out_node.name() + "@GRAD" + ) in_node_grad = graph._find_node_by_name( - graph.all_var_nodes(), - target_in_node.name() + "@GRAD") + graph.all_var_nodes(), target_in_node.name() + "@GRAD" + ) in_node_grad_op = in_node_grad.inputs # update op_grad's input graph.update_input_link(var_node, target_out_node, op_grad) @@ -888,8 +1002,9 @@ class QuantizationTransformPass(object): op_grad_out = node # update op_grad's output if op_grad_out is not None: - graph.update_output_link(op_grad_out, target_out_grad_node, - op_grad) + graph.update_output_link( + op_grad_out, target_out_grad_node, op_grad + ) else: graph.link_to(op_grad, target_out_grad_node) @@ -934,31 +1049,37 @@ class QuantizationTransformPass(object): Analyse whether the op node skips quantization. """ is_skip = False - if op_node.op().has_attr("skip_quant") and \ - op_node.op().attr("skip_quant"): + if op_node.op().has_attr("skip_quant") and op_node.op().attr( + "skip_quant" + ): is_skip = True # if the inputs of mul and matmul are not all persistable, use # AddQuantDequantPass to quantize them. - if op_node.name() in ["mul", "matmul"] and \ - _is_input_all_not_persistable(graph, op_node): + if op_node.name() in [ + "mul", + "matmul", + ] and _is_input_all_not_persistable(graph, op_node): is_skip = True - if op_node.op().has_attr("quantization_type") and \ - op_node.op().attr("quantization_type") == "qat_without_weight": + if ( + op_node.op().has_attr("quantization_type") + and op_node.op().attr("quantization_type") == "qat_without_weight" + ): is_skip = True return is_skip class QuantizationFreezePass(object): - - def __init__(self, - scope, - place, - bias_correction=False, - weight_bits=8, - activation_bits=8, - round_type='round', - weight_quantize_type='abs_max', - quantizable_op_type=None): + def __init__( + self, + scope, + place, + bias_correction=False, + weight_bits=8, + activation_bits=8, + round_type='round', + weight_quantize_type='abs_max', + quantizable_op_type=None, + ): """ The freeze pass is used to adjust the quantize operator order, for example: 1) `activation -> quant -> dequant -> conv2d` will be frozen into @@ -984,10 +1105,8 @@ class QuantizationFreezePass(object): quantizable_op_type(list[str]): This input param will be removed latter. The pass will process all quantized op, so it is not necessary to set the input param. """ - assert scope is not None, \ - 'The scope cannot be set None.' - assert place is not None, \ - 'The place cannot be set None.' + assert scope is not None, 'The scope cannot be set None.' + assert place is not None, 'The place cannot be set None.' self._scope = scope self._bias_correction = bias_correction self._place = _get_paddle_place(place) @@ -1020,21 +1139,26 @@ class QuantizationFreezePass(object): if hasattr(graph, 'out_node_mapping_table'): if input_arg_name in graph.out_node_mapping_table.keys(): input_arg_name = graph.out_node_mapping_table[ - input_arg_name] + input_arg_name + ] if input_arg_name not in persistable_vars: scale_v = graph._find_node_by_name( - op_node.outputs, - op_node.output('OutScale')[0]) + op_node.outputs, op_node.output('OutScale')[0] + ) self._quant_var_scale_map[input_arg_name] = scale_v else: # Obtain scale from OutScale var node scale_v = self._load_var(op_node.output('OutScale')[0]) assert scale_v.ndim in [ - 1, 2 + 1, + 2, ], "the dim of scale_v should be 1 or 2" if scale_v.ndim == 2: scale_v = scale_v[0] - if scale_v.size == 1 and self._weight_quantize_type == 'abs_max': + if ( + scale_v.size == 1 + and self._weight_quantize_type == 'abs_max' + ): scale_v = scale_v[0] else: scale_v = scale_v.tolist() @@ -1043,14 +1167,18 @@ class QuantizationFreezePass(object): if self._round_type == 'round': param_v = self._load_var(input_arg_name) if any( - _check_grandchild_op_node(op_node, op) - for op in utils._channelwise_quant_axis1_ops): + _check_grandchild_op_node(op_node, op) + for op in utils._channelwise_quant_axis1_ops + ): quant_axis = 1 else: quant_axis = 0 quantized_param_v = utils.quant_tensor( - param_v.copy(), scale_v, quant_axis, - self._weight_bits) + param_v.copy(), + scale_v, + quant_axis, + self._weight_bits, + ) quantized_param_v = np.round(quantized_param_v) # Weight bias correction if self._bias_correction == True: @@ -1059,7 +1187,8 @@ class QuantizationFreezePass(object): quantized_param_v, scale_v, quant_axis, - weight_bits=self._weight_bits) + weight_bits=self._weight_bits, + ) quantized_param_v = np.round(quantized_param_v) self._restore_var(input_arg_name, quantized_param_v) self._remove_fake_quant_and_dequant_op(graph, op_node) @@ -1075,13 +1204,19 @@ class QuantizationFreezePass(object): ops = graph.all_op_nodes() for op_node in ops: op_node_desc = op_node.op() - if op_node_desc.has_attr("quantization_type") and \ - op_node_desc.attr("quantization_type") == "qat_with_weight": + if ( + op_node_desc.has_attr("quantization_type") + and op_node_desc.attr("quantization_type") == "qat_with_weight" + ): if self._weight_quantize_type == 'channel_wise_abs_max': - quant_axis = 1 if op_node.name() in \ - utils._channelwise_quant_axis1_ops else 0 + quant_axis = ( + 1 + if op_node.name() in utils._channelwise_quant_axis1_ops + else 0 + ) self._insert_post_channel_dequant_op( - graph, op_node, quant_axis) + graph, op_node, quant_axis + ) else: self._insert_post_dequant_op(graph, op_node) @@ -1105,7 +1240,8 @@ class QuantizationFreezePass(object): self._op_input_rename_map[k.node] = v else: self._op_input_rename_map[k.node] = self._op_input_rename_map[ - v.node] + v.node + ] graph.safe_remove_nodes(op_node) def _insert_post_channel_dequant_op(self, graph, op_node, quant_axis): @@ -1123,35 +1259,47 @@ class QuantizationFreezePass(object): scale_v = self._quant_var_scale_map[original_var_name] if original_var_name in persistable_vars: assert isinstance( - scale_v, - list), 'The scale of parameter %s is not a list.' % ( - original_var_name) + scale_v, list + ), 'The scale of parameter %s is not a list.' % ( + original_var_name + ) channel_scale = np.array(scale_v) else: assert isinstance(scale_v, IrNode) scale_var_node = self._quant_var_scale_map[original_var_name] if len(op_node.output_arg_names()) != 1: - raise ValueError("Only support one output, but op %s has" - " more than one output." % (op_node.name())) + raise ValueError( + "Only support one output, but op %s has" + " more than one output." % (op_node.name()) + ) output_var_node = graph._find_node_by_name( - op_node.outputs, - op_node.output_arg_names()[0]) + op_node.outputs, op_node.output_arg_names()[0] + ) weight_scale_node = graph.create_persistable_node( name=unique_name.generate('channel_scale'), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[channel_scale.shape[0]], - var_dtype=output_var_node.dtype()) - data_type = 'float64' if output_var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(weight_scale_node, channel_scale.astype(data_type), - self._scope, self._place) + var_dtype=output_var_node.dtype(), + ) + data_type = ( + 'float64' + if output_var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + weight_scale_node, + channel_scale.astype(data_type), + self._scope, + self._place, + ) dequant_var_node = graph.create_var_node( name=self._dequantized_var_name(output_var_node.name()), var_type=output_var_node.type(), shape=output_var_node.shape(), - var_dtype=output_var_node.dtype()) + var_dtype=output_var_node.dtype(), + ) x_num_col_dims = 1 if op_node.name() in ['matmul', 'matmul_v2', 'mul']: x_num_col_dims = len(op_node.outputs[0].shape()) - 1 @@ -1163,13 +1311,14 @@ class QuantizationFreezePass(object): 'quant_bits': [self._weight_bits, self._activation_bits], 'quant_axis': quant_axis, 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, - 'x_num_col_dims': x_num_col_dims + 'x_num_col_dims': x_num_col_dims, }, inputs={ 'X': output_var_node, - 'Scales': [weight_scale_node, scale_var_node] + 'Scales': [weight_scale_node, scale_var_node], }, - outputs={'Out': dequant_var_node}) + outputs={'Out': dequant_var_node}, + ) graph.link_to(output_var_node, dequant_op_node) graph.link_to(scale_var_node, dequant_op_node) graph.link_to(weight_scale_node, dequant_op_node) @@ -1195,8 +1344,10 @@ class QuantizationFreezePass(object): scale_v = self._quant_var_scale_map[original_var_name] if original_var_name in persistable_vars: assert self._is_float( - scale_v), 'The scale of parameter %s is not a float.' % ( - original_var_name) + scale_v + ), 'The scale of parameter %s is not a float.' % ( + original_var_name + ) scale_v = 1e-8 if scale_v == 0.0 else scale_v max_range *= param_range / scale_v else: @@ -1205,28 +1356,29 @@ class QuantizationFreezePass(object): scale_var_node = self._quant_var_scale_map[original_var_name] if len(op_node.output_arg_names()) != 1: - raise ValueError("Only support one output, but op %s has" - " more than one output." % (op_node.name())) + raise ValueError( + "Only support one output, but op %s has" + " more than one output." % (op_node.name()) + ) output_var_node = graph._find_node_by_name( - op_node.outputs, - op_node.output_arg_names()[0]) + op_node.outputs, op_node.output_arg_names()[0] + ) dequant_var_node = graph.create_var_node( name=self._dequantized_var_name(output_var_node.name()), var_type=output_var_node.type(), shape=output_var_node.shape(), - var_dtype=output_var_node.dtype()) + var_dtype=output_var_node.dtype(), + ) dequant_op_node = graph.create_op_node( op_type='fake_dequantize_max_abs', attrs={ 'max_range': float(max_range), - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward - }, - inputs={ - 'X': output_var_node, - 'Scale': scale_var_node + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, - outputs={'Out': dequant_var_node}) + inputs={'X': output_var_node, 'Scale': scale_var_node}, + outputs={'Out': dequant_var_node}, + ) graph.link_to(output_var_node, dequant_op_node) graph.link_to(scale_var_node, dequant_op_node) graph.link_to(dequant_op_node, dequant_var_node) @@ -1252,8 +1404,10 @@ class QuantizationFreezePass(object): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) @@ -1262,13 +1416,13 @@ class QuantizationFreezePass(object): Return the original variable name. """ if var_name.endswith('.quantized.dequantized'): - return var_name[:-len('.quantized.dequantized')] + return var_name[: -len('.quantized.dequantized')] if var_name.endswith('.quantized'): - return var_name[:-len('.quantized')] + return var_name[: -len('.quantized')] if var_name.endswith('.dequantized'): - return var_name[:-len('.dequantized')] + return var_name[: -len('.dequantized')] if var_name.endswith('@scale'): - return var_name[:-len('@scale')] + return var_name[: -len('@scale')] else: return var_name @@ -1279,12 +1433,14 @@ class QuantizationFreezePass(object): return "%s.dequantized" % (var_name) def _is_float(self, v): - return isinstance(v, float) or isinstance(v, np.float32) \ + return ( + isinstance(v, float) + or isinstance(v, np.float32) or isinstance(v, np.float64) + ) class ConvertToInt8Pass(object): - def __init__(self, scope, place, quantizable_op_type=None): """ Convert the weights into int8_t type. @@ -1297,10 +1453,8 @@ class ConvertToInt8Pass(object): quantizable_op_type(list[str]): This input param will be removed latter. The pass will process all quantized op, so it is not necessary to set the input param. """ - assert scope is not None, \ - 'The scope cannot be set None.' - assert place is not None, \ - 'The place cannot be set None.' + assert scope is not None, 'The scope cannot be set None.' + assert place is not None, 'The place cannot be set None.' self._scope = scope self._place = _get_paddle_place(place) @@ -1318,17 +1472,21 @@ class ConvertToInt8Pass(object): ops = graph.all_op_nodes() input_map = {} for op_node in ops: - if op_node.op().has_attr("quantization_type") and \ - op_node.op().attr("quantization_type") == "qat_with_weight": + if ( + op_node.op().has_attr("quantization_type") + and op_node.op().attr("quantization_type") == "qat_with_weight" + ): for var_node in op_node.inputs: name = var_node.name() if name in persistable_vars: if name not in input_map: int8_var_node = self._convert_to_int8( - graph, var_node) + graph, var_node + ) input_map[name] = int8_var_node - graph.update_input_link(var_node, input_map[name], - op_node) + graph.update_input_link( + var_node, input_map[name], op_node + ) # remove the unused var node in the graph self._remove_unused_var_nodes(graph) @@ -1341,7 +1499,8 @@ class ConvertToInt8Pass(object): name=int8_var_node_name, var_type=var_node.type(), shape=var_node.shape(), - var_dtype=core.VarDesc.VarType.INT8) + var_dtype=core.VarDesc.VarType.INT8, + ) array = self._load_var(var_node.name()) self._scope.var(int8_var_node_name) self._store_var(int8_var_node_name, array, np.int8) @@ -1366,14 +1525,15 @@ class ConvertToInt8Pass(object): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) class TransformForMobilePass(object): - def __init__(self): """ This pass is used to convert the frozen graph for paddle-mobile execution. @@ -1416,13 +1576,14 @@ class TransformForMobilePass(object): class OutScaleForTrainingPass(object): - - def __init__(self, - scope=None, - place=None, - moving_rate=0.9, - is_test=None, - scale_dict=None): + def __init__( + self, + scope=None, + place=None, + moving_rate=0.9, + is_test=None, + scale_dict=None, + ): """ This pass is used for calculating output scales of some operators. These output scales may be used by tensorRT or some other inference engines. @@ -1449,48 +1610,61 @@ class OutScaleForTrainingPass(object): Args: graph(IrGraph): the target graph. """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() target_ops = [] for op in graph.all_op_nodes(): if op.name() in self._teller_set: target_ops.append(op) - with tqdm(total=len(target_ops), - bar_format='Adding OutScale op:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(target_ops), + bar_format='Adding OutScale op:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op in target_ops: for output_var_name in utils._get_op_output_var_names(op): - in_node = graph._find_node_by_name(op.outputs, - output_var_name) - if in_node.dtype() not in \ - [core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]: + in_node = graph._find_node_by_name( + op.outputs, output_var_name + ) + if in_node.dtype() not in [ + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, + ]: continue - data_type = 'float64' if in_node.dtype() \ - == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if in_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: graph._find_node_by_name( graph.all_var_nodes(), - self._scale_name(in_node.name())) + self._scale_name(in_node.name()), + ) continue except: scale_node = graph.create_persistable_node( name=self._scale_name(in_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=in_node.dtype()) + var_dtype=in_node.dtype(), + ) if self._scale_dict is not None: try: scale_value = np.array( - [self._scale_dict[in_node.name()]]) + [self._scale_dict[in_node.name()]] + ) except: scale_value = np.ones([1], dtype=data_type) else: scale_value = np.ones([1], dtype=data_type) - _init_var_node(scale_node, scale_value, self._scope, - self._place) + _init_var_node( + scale_node, scale_value, self._scope, self._place + ) ins = {'X': in_node} outs = {'OutScale': scale_node} @@ -1499,22 +1673,32 @@ class OutScaleForTrainingPass(object): name=unique_name.generate('scale_state@'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=in_node.dtype(), - shape=[1]) - _init_var_node(state_in_node, - np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + state_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) accum_in_node = graph.create_persistable_node( name=unique_name.generate('scale_accum@'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=in_node.dtype(), - shape=[1]) - _init_var_node(accum_in_node, - np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + accum_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) state_out_node = graph.create_var_node_from_desc( - state_in_node.var()) + state_in_node.var() + ) accum_out_node = graph.create_var_node_from_desc( - accum_in_node.var()) + accum_in_node.var() + ) ins['InState'] = state_in_node ins['InAccum'] = accum_in_node @@ -1524,14 +1708,14 @@ class OutScaleForTrainingPass(object): attrs = { 'moving_rate': self._moving_rate, 'is_test': self._is_test, - 'op_role': - core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, } scale_op_node = graph.create_op_node( op_type='moving_average_abs_max_scale', attrs=attrs, inputs=ins, - outputs=outs) + outputs=outs, + ) next_op_node = None if len(in_node.outputs) > 0: @@ -1558,7 +1742,6 @@ class OutScaleForTrainingPass(object): class OutScaleForInferencePass(object): - def __init__(self, scope=None): """ This pass is used for setting output scales of some operators. @@ -1578,34 +1761,45 @@ class OutScaleForInferencePass(object): Args: graph(IrGraph): the target graph. """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' op_nodes = graph.all_op_nodes() for op_node in op_nodes: if op_node.name() in self._teller_set: var_names = utils._get_op_output_var_names(op_node) for var_name in var_names: - in_node = graph._find_node_by_name(op_node.outputs, - var_name) - if in_node.dtype() not in \ - [core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]: + in_node = graph._find_node_by_name( + op_node.outputs, var_name + ) + if in_node.dtype() not in [ + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, + ]: continue scale_name = self._scale_name(var_name) scale_var = self._scope.find_var(scale_name) - assert scale_var is not None, \ - "Can not find {} variable in the scope".format(scale_name) + assert ( + scale_var is not None + ), "Can not find {} variable in the scope".format( + scale_name + ) scale_value = np.array(scale_var.get_tensor())[0] # For compatibility, we save output threshold by two methods. op_node.op()._set_attr("out_threshold", float(scale_value)) argname_index = utils._get_output_name_index( - op_node, var_name) - assert argname_index is not None, \ + op_node, var_name + ) + assert argname_index is not None, ( var_name + " is not the output of the op" - op_node.op()._set_attr(argname_index[0] + str(argname_index[1]) \ - + "_threshold", float(scale_value)) + ) + op_node.op()._set_attr( + argname_index[0] + str(argname_index[1]) + "_threshold", + float(scale_value), + ) op_node.op()._set_attr("with_quant_attr", True) graph.resolve_hazard() return graph @@ -1626,16 +1820,18 @@ class AddQuantDequantPass(object): # To be compatible with PaddleSlim, not remove _activation_type for now _activation_type = ["relu", "relu6", "leaky_relu", "tanh", "swish"] - def __init__(self, - scope=None, - place=None, - moving_rate=0.9, - quant_bits=8, - skip_pattern=["skip_quant"], - quantizable_op_type=["elementwise_add", "pool2d"], - is_full_quantized=False, - is_test=None, - scale_dict=None): + def __init__( + self, + scope=None, + place=None, + moving_rate=0.9, + quant_bits=8, + skip_pattern=["skip_quant"], + quantizable_op_type=["elementwise_add", "pool2d"], + is_full_quantized=False, + is_test=None, + scale_dict=None, + ): """ Constructor. @@ -1671,8 +1867,9 @@ class AddQuantDequantPass(object): else: self._quantizable_op_type = quantizable_op_type for op_type in quantizable_op_type: - assert op_type in utils._act_supported_quantizable_op_type, \ + assert op_type in utils._act_supported_quantizable_op_type, ( op_type + " is not supported for quantization." + ) self._quantizable_grad_op_type = [ '%s_grad' % (op) for op in self._quantizable_op_type ] @@ -1690,50 +1887,71 @@ class AddQuantDequantPass(object): Returns: None """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() dequantized_vars_map = collections.OrderedDict() # Forward stage, insert quant_dequant op all_op_nodes = graph.all_op_nodes() - with tqdm(total=len(all_op_nodes), - bar_format= - 'Adding quant activation op:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(all_op_nodes), + bar_format='Adding quant activation op:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op_node in all_op_nodes: if op_node.name() in self._quantizable_op_type: is_skip = False if isinstance(self._skip_pattern, list): - is_skip = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") for pattern in self._skip_pattern) + is_skip = op_node.op().has_attr("op_namescope") and any( + pattern in op_node.op().attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - is_skip = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find(self._skip_pattern) != -1 - is_quantized = op_node.op().has_attr("quantization_type") and \ - op_node.op().attr("quantization_type") == "qat_with_weight" - if is_skip or is_quantized or \ - (not _is_input_all_not_persistable(graph, op_node)): + is_skip = ( + op_node.op().has_attr("op_namescope") + and op_node.op() + .attr("op_namescope") + .find(self._skip_pattern) + != -1 + ) + is_quantized = ( + op_node.op().has_attr("quantization_type") + and op_node.op().attr("quantization_type") + == "qat_with_weight" + ) + if ( + is_skip + or is_quantized + or (not _is_input_all_not_persistable(graph, op_node)) + ): continue - op_node.op()._set_attr("quantization_type", - "qat_without_weight") + op_node.op()._set_attr( + "quantization_type", "qat_without_weight" + ) op_node.op()._set_attr("activation_bits", self._quant_bits) op_node.op()._set_attr("with_quant_attr", True) arg_names = utils._get_op_input_var_names(op_node) for arg_name in arg_names: in_node = graph._find_node_by_name( - op_node.inputs, arg_name) + op_node.inputs, arg_name + ) if arg_name in dequantized_vars_map: quant_var_node = dequantized_vars_map[arg_name] else: - quant_var_node, _ = \ - self._inser_quant_dequant_moving_average_abs_max_op( - graph, in_node, self._quant_bits) + ( + quant_var_node, + _, + ) = self._inser_quant_dequant_moving_average_abs_max_op( + graph, in_node, self._quant_bits + ) dequantized_vars_map[arg_name] = quant_var_node - graph.update_input_link(in_node, quant_var_node, - op_node) + graph.update_input_link( + in_node, quant_var_node, op_node + ) t.update() # Backward stage, update input link @@ -1742,35 +1960,45 @@ class AddQuantDequantPass(object): for input_name in op_node.input_arg_names(): if input_name in dequantized_vars_map: in_node = graph._find_node_by_name( - op_node.inputs, input_name) + op_node.inputs, input_name + ) dequant_var_node = dequantized_vars_map[input_name] - graph.update_input_link(in_node, dequant_var_node, - op_node) + graph.update_input_link( + in_node, dequant_var_node, op_node + ) graph.resolve_hazard() return graph - def _inser_quant_dequant_moving_average_abs_max_op(self, graph, var_node, - quant_bits): - """Insert fake_quantize_dequantize_moving_average_abs_max op. - """ - quant_var_node = graph.create_var_node(name="{}.quant_dequant".format( - var_node.name()), - var_type=var_node.type(), - shape=var_node.shape(), - var_dtype=var_node.dtype()) + def _inser_quant_dequant_moving_average_abs_max_op( + self, graph, var_node, quant_bits + ): + """Insert fake_quantize_dequantize_moving_average_abs_max op.""" + quant_var_node = graph.create_var_node( + name="{}.quant_dequant".format(var_node.name()), + var_type=var_node.type(), + shape=var_node.shape(), + var_dtype=var_node.dtype(), + ) scale_name = "{}.quant_dequant@scale".format(var_node.name()) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: - if self._scale_dict is not None and var_node.name( - ) in self._scale_dict.keys(): - scale_value = np.array([self._scale_dict[var_node.name()]], - dtype=data_type) + if ( + self._scale_dict is not None + and var_node.name() in self._scale_dict.keys() + ): + scale_value = np.array( + [self._scale_dict[var_node.name()]], dtype=data_type + ) else: scale_value = np.array( self._scope.find_var(scale_name).get_tensor(), - dtype=data_type) + dtype=data_type, + ) except: scale_value = np.array([_SCALE_DEFAULT_VALUE], dtype=data_type) @@ -1778,7 +2006,8 @@ class AddQuantDequantPass(object): name="{}.quant_dequant@scale".format(var_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_in_node, scale_value, self._scope, self._place) scale_out_node = graph.create_var_node_from_desc(scale_in_node.var()) @@ -1789,22 +2018,37 @@ class AddQuantDequantPass(object): name=unique_name.generate('quant_dequant.state'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(state_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + state_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) accum_in_node = graph.create_persistable_node( name=unique_name.generate('quant_dequant.accum'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - _init_var_node(accum_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + accum_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) state_out_node = graph.create_var_node_from_desc( - state_in_node.var()) + state_in_node.var() + ) accum_out_node = graph.create_var_node_from_desc( - accum_in_node.var()) + accum_in_node.var() + ) ins['InState'] = state_in_node ins['InAccum'] = accum_in_node @@ -1815,14 +2059,15 @@ class AddQuantDequantPass(object): 'bit_length': quant_bits, 'moving_rate': self._moving_rate, 'is_test': self._is_test, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, } quant_op_node = graph.create_op_node( op_type='fake_quantize_dequantize_moving_average_abs_max', attrs=attrs, inputs=ins, - outputs=outs) + outputs=outs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_in_node, quant_op_node) @@ -1856,15 +2101,17 @@ class InsertQuantizeLinear(object): scale_dict(dict, optional): calibration ranges of tensors output. """ - def __init__(self, - place, - scope, - quant_bits=8, - quant_axis=-1, - channel_wise=False, - moving_rate=0.9, - is_test=True, - scale_dict=None): + def __init__( + self, + place, + scope, + quant_bits=8, + quant_axis=-1, + channel_wise=False, + moving_rate=0.9, + is_test=True, + scale_dict=None, + ): self._place = place self._scope = scope self.quant_bits = quant_bits @@ -1881,32 +2128,42 @@ class InsertQuantizeLinear(object): name=self._quantized_var_name(var_name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + var_dtype=var_node.dtype(), + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) scale_name = self._quantized_scale_name(var_name) if self.channel_wise: scale_var_shape = var_node.shape()[self.quant_axis] scale_var_type = core.VarDesc.VarType.LOD_TENSOR - init_scale_value = np.ones(scale_var_shape, - dtype=data_type) * _SCALE_DEFAULT_VALUE + init_scale_value = ( + np.ones(scale_var_shape, dtype=data_type) * _SCALE_DEFAULT_VALUE + ) else: scale_var_shape = 1 scale_var_type = var_node.type() init_scale_value = np.array([_SCALE_DEFAULT_VALUE], dtype=data_type) - if self._scale_dict is not None and var_node.name( - ) in self._scale_dict.keys(): - init_scale_value = np.array([self._scale_dict[var_node.name()]], - dtype=data_type) + if ( + self._scale_dict is not None + and var_node.name() in self._scale_dict.keys() + ): + init_scale_value = np.array( + [self._scale_dict[var_node.name()]], dtype=data_type + ) scale_var_node = graph.create_persistable_node( name=scale_name, var_type=scale_var_type, shape=[scale_var_shape], - var_dtype=var_node.dtype()) - _init_var_node(scale_var_node, init_scale_value, self._scope, - self._place) + var_dtype=var_node.dtype(), + ) + _init_var_node( + scale_var_node, init_scale_value, self._scope, self._place + ) zero_point_node = None if zero_point_node is None: @@ -1914,10 +2171,14 @@ class InsertQuantizeLinear(object): name=self._zero_point_name(quant_var_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=scale_var_node.shape(), - var_dtype=core.VarDesc.VarType.INT32) - _init_var_node(zero_point_node, - np.zeros(scale_var_node.shape(), dtype="int32"), - self._scope, self._place) + var_dtype=core.VarDesc.VarType.INT32, + ) + _init_var_node( + zero_point_node, + np.zeros(scale_var_node.shape(), dtype="int32"), + self._scope, + self._place, + ) inputs = {"X": var_node, "Scale": scale_var_node} if zero_point_node is not None: @@ -1928,27 +2189,43 @@ class InsertQuantizeLinear(object): outputs = {"Y": quant_var_node} if not self._is_test: scale_out_node = graph.create_var_node_from_desc( - scale_var_node.var()) + scale_var_node.var() + ) state_in_node = graph.create_persistable_node( name=unique_name.generate('state'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(state_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + state_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) accum_in_node = graph.create_persistable_node( name=unique_name.generate('accum'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - _init_var_node(accum_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + accum_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) state_out_node = graph.create_var_node_from_desc( - state_in_node.var()) + state_in_node.var() + ) accum_out_node = graph.create_var_node_from_desc( - accum_in_node.var()) + accum_in_node.var() + ) outputs["OutScale"] = scale_out_node inputs['InState'] = state_in_node @@ -1958,10 +2235,12 @@ class InsertQuantizeLinear(object): attrs["is_test"] = self._is_test attrs['moving_rate'] = self._moving_rate - quant_op_node = graph.create_op_node(op_type="quantize_linear", - attrs=attrs, - inputs=inputs, - outputs=outputs) + quant_op_node = graph.create_op_node( + op_type="quantize_linear", + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_var_node, quant_op_node) @@ -1983,7 +2262,8 @@ class InsertQuantizeLinear(object): name=self._dequantized_var_name(var_node.name()), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) zero_point_node = None if zero_point_node is None: @@ -1991,10 +2271,14 @@ class InsertQuantizeLinear(object): name=self._zero_point_name(dequant_var_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=scale_var_node.shape(), - var_dtype=core.VarDesc.VarType.INT32) - _init_var_node(zero_point_node, - np.zeros(scale_var_node.shape(), dtype="int32"), - self._scope, self._place) + var_dtype=core.VarDesc.VarType.INT32, + ) + _init_var_node( + zero_point_node, + np.zeros(scale_var_node.shape(), dtype="int32"), + self._scope, + self._place, + ) inputs = {"X": var_node, "Scale": scale_var_node} if zero_point_node is not None: @@ -2003,10 +2287,12 @@ class InsertQuantizeLinear(object): attrs = {"quant_axis": self.quant_axis, "bit_length": self.quant_bits} attrs["op_role"] = core.op_proto_and_checker_maker.OpRole.Forward - quant_op_node = graph.create_op_node(op_type="dequantize_linear", - attrs=attrs, - inputs=inputs, - outputs={"Y": dequant_var_node}) + quant_op_node = graph.create_op_node( + op_type="dequantize_linear", + attrs=attrs, + inputs=inputs, + outputs={"Y": dequant_var_node}, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_var_node, quant_op_node) @@ -2046,24 +2332,26 @@ class QuantizationTransformPassV2(QuantizationTransformPass): the quantized ops's inputs. It is used in the new format of quantization. """ - def __init__(self, - scope=None, - place=None, - weight_bits=8, - activation_bits=8, - activation_quantize_type='abs_max', - weight_quantize_type='abs_max', - window_size=10000, - moving_rate=0.9, - skip_pattern=['skip_quant'], - quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul'], - weight_quantize_func=None, - act_quantize_func=None, - weight_preprocess_func=None, - act_preprocess_func=None, - optimizer_func=None, - executor=None, - is_test=None): + def __init__( + self, + scope=None, + place=None, + weight_bits=8, + activation_bits=8, + activation_quantize_type='abs_max', + weight_quantize_type='abs_max', + window_size=10000, + moving_rate=0.9, + skip_pattern=['skip_quant'], + quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul'], + weight_quantize_func=None, + act_quantize_func=None, + weight_preprocess_func=None, + act_preprocess_func=None, + optimizer_func=None, + executor=None, + is_test=None, + ): r""" Args: scope(paddle.Scope): When activation use 'range_abs_max' as the quantize @@ -2149,21 +2437,26 @@ class QuantizationTransformPassV2(QuantizationTransformPass): self._optimizer = optimizer_func self._exe = executor quant_type = [ - 'abs_max', 'channel_wise_abs_max', 'range_abs_max', - 'moving_average_abs_max' + 'abs_max', + 'channel_wise_abs_max', + 'range_abs_max', + 'moving_average_abs_max', ] - assert activation_quantize_type != 'channel_wise_abs_max', \ - "The activation quantization type does not support 'channel_wise_abs_max'." + assert ( + activation_quantize_type != 'channel_wise_abs_max' + ), "The activation quantization type does not support 'channel_wise_abs_max'." if activation_quantize_type not in quant_type: raise ValueError( "Unknown activation_quantize_type : '%s'. It can only be " - "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'." % - (str(activation_quantize_type))) + "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'." + % (str(activation_quantize_type)) + ) if weight_quantize_type not in quant_type: raise ValueError( "Unknown weight_quantize_type: '%s'. It can only be " "'abs_max' or 'channel_wise_abs_max' or 'range_abs_max' " - "or 'moving_average_abs_max'." % (str(weight_quantize_type))) + "or 'moving_average_abs_max'." % (str(weight_quantize_type)) + ) self._activation_quantize_type = activation_quantize_type self._weight_quantize_type = weight_quantize_type @@ -2172,8 +2465,9 @@ class QuantizationTransformPassV2(QuantizationTransformPass): self._quantizable_ops = quantizable_op_type for op in self._quantizable_ops: - assert op in utils._weight_supported_quantizable_op_type, \ + assert op in utils._weight_supported_quantizable_op_type, ( op + " is not supported for quantization." + ) self._quantizable_grad_ops = [ '%s_grad' % (op) for op in self._quantizable_ops ] @@ -2191,13 +2485,16 @@ class QuantizationTransformPassV2(QuantizationTransformPass): def _quant_preprocess(self, op_node): user_skipped = False if isinstance(self._skip_pattern, list): - user_skipped = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") \ - for pattern in self._skip_pattern) + user_skipped = op_node.op().has_attr("op_namescope") and any( + pattern in op_node.op().attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - user_skipped = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find( - self._skip_pattern) != -1 + user_skipped = ( + op_node.op().has_attr("op_namescope") + and op_node.op().attr("op_namescope").find(self._skip_pattern) + != -1 + ) if user_skipped: op_node.op()._set_attr("skip_quant", True) @@ -2215,8 +2512,9 @@ class QuantizationTransformPassV2(QuantizationTransformPass): name = var_node.name() if name in self.processed_vars: continue - is_weight = True if var_node.name() in self.persistable_vars \ - else False + is_weight = ( + True if var_node.name() in self.persistable_vars else False + ) # if var node is weight and weight_preprocess_func is not None, # will insert weight preprocess func @@ -2225,13 +2523,13 @@ class QuantizationTransformPassV2(QuantizationTransformPass): # will insert activation preprocess func # to preorocess activation before quantization if is_weight and self._weight_preprocess_func is not None: - var_node = self._insert_func(graph, - self._weight_preprocess_func, - var_node, op) + var_node = self._insert_func( + graph, self._weight_preprocess_func, var_node, op + ) elif not is_weight and self._act_preprocess_func is not None: - var_node = self._insert_func(graph, - self._act_preprocess_func, - var_node, op) + var_node = self._insert_func( + graph, self._act_preprocess_func, var_node, op + ) # if var node is weight and weight_quantize_func is not None, # will insert weight quantize func to quantize and dequantize weight @@ -2239,26 +2537,36 @@ class QuantizationTransformPassV2(QuantizationTransformPass): # will insert act quantize func to quantize and dequantize activation if is_weight and self._weight_quantize_func is not None: target_out_node = self._insert_func( - graph, self._weight_quantize_func, var_node, op) + graph, self._weight_quantize_func, var_node, op + ) self.processed_vars.append(name) continue elif not is_weight and self._act_quantize_func is not None: - target_out_node = self._insert_func(graph, - self._act_quantize_func, - var_node, op) + target_out_node = self._insert_func( + graph, self._act_quantize_func, var_node, op + ) self.processed_vars.append(name) continue - quant_bits = self._weight_bits if var_node.name() in self.persistable_vars \ + quant_bits = ( + self._weight_bits + if var_node.name() in self.persistable_vars else self._activation_bits - quant_type = self._weight_quantize_type if is_weight \ + ) + quant_type = ( + self._weight_quantize_type + if is_weight else self._activation_quantize_type + ) quant_axis = -1 channel_wise = False if quant_type == 'channel_wise_abs_max': # Weight quantization channel_wise = True - quant_axis = 1 if op.name() in \ - utils._channelwise_quant_axis1_ops else 0 + quant_axis = ( + 1 + if op.name() in utils._channelwise_quant_axis1_ops + else 0 + ) insert_quant_pass = InsertQuantizeLinear( self._place, self._scope, @@ -2266,11 +2574,17 @@ class QuantizationTransformPassV2(QuantizationTransformPass): quant_axis=quant_axis, channel_wise=channel_wise, moving_rate=self._moving_rate, - is_test=self._is_test) - quant_var_node, scale_var_node = insert_quant_pass.insert_quant_op( - graph, var_node, var_name=name) + is_test=self._is_test, + ) + ( + quant_var_node, + scale_var_node, + ) = insert_quant_pass.insert_quant_op( + graph, var_node, var_name=name + ) dequant_var_node = insert_quant_pass.insert_dequant_op( - graph, quant_var_node, scale_var_node) + graph, quant_var_node, scale_var_node + ) self.dequantized_vars[name] = dequant_var_node graph.update_input_link(var_node, dequant_var_node, op) @@ -2304,8 +2618,9 @@ class QuantizationTransformPassV2(QuantizationTransformPass): Returns: None """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() @@ -2317,21 +2632,25 @@ class QuantizationTransformPassV2(QuantizationTransformPass): # Do the preproccess of quantization, such as skipping some ops # for not being quantized. for op in ops: - if op.name() in self._quantizable_ops or \ - op.name() in self._quantizable_grad_ops: + if ( + op.name() in self._quantizable_ops + or op.name() in self._quantizable_grad_ops + ): self._quant_preprocess(op) # Insert mapping table to solve the problem in saving inference model. graph.out_node_mapping_table = dict() # The process of _transform_forward and _transform_backward is needed in two for loops. # The loop for transforming the forward graph: - with tqdm(total=len(ops), - bar_format= - 'Adding quant op with weight:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(ops), + bar_format='Adding quant op with weight:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op in ops: if op.name() in self._quantizable_ops: - if not self._is_skip_quant(graph, - op) and self._has_weight(op): + if not self._is_skip_quant(graph, op) and self._has_weight( + op + ): self._transform_forward(graph, op) t.update() # The loop for renaming the inputs of backward op. @@ -2350,16 +2669,18 @@ class AddQuantDequantPassV2(object): # To be compatible with PaddleSlim, not remove _activation_type for now _activation_type = ["relu", "relu6", "leaky_relu", "tanh", "swish"] - def __init__(self, - scope=None, - place=None, - moving_rate=0.9, - quant_bits=8, - skip_pattern=["skip_quant"], - quantizable_op_type=["elementwise_add", "pool2d"], - is_full_quantized=False, - is_test=None, - scale_dict=None): + def __init__( + self, + scope=None, + place=None, + moving_rate=0.9, + quant_bits=8, + skip_pattern=["skip_quant"], + quantizable_op_type=["elementwise_add", "pool2d"], + is_full_quantized=False, + is_test=None, + scale_dict=None, + ): """ Args: scope(paddle.Scope): The scope is used to initialize these new parameters. @@ -2409,8 +2730,9 @@ class AddQuantDequantPassV2(object): else: self._quantizable_op_type = quantizable_op_type for op_type in quantizable_op_type: - assert op_type in utils._act_supported_quantizable_op_type, \ + assert op_type in utils._act_supported_quantizable_op_type, ( op_type + " is not supported for quantization." + ) self._quantizable_grad_op_type = [ '%s_grad' % (op) for op in self._quantizable_op_type ] @@ -2429,8 +2751,9 @@ class AddQuantDequantPassV2(object): Returns: None """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() dequantized_vars_map = collections.OrderedDict() @@ -2441,28 +2764,40 @@ class AddQuantDequantPassV2(object): # Forward stage, insert quant_dequant op all_op_nodes = graph.all_op_nodes() - with tqdm(total=len(all_op_nodes), - bar_format= - 'Adding quant activation op:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(all_op_nodes), + bar_format='Adding quant activation op:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op_node in all_op_nodes: if op_node.name() in self._quantizable_op_type: is_skip = False if isinstance(self._skip_pattern, list): - is_skip = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") for pattern in self._skip_pattern) + is_skip = op_node.op().has_attr("op_namescope") and any( + pattern in op_node.op().attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - is_skip = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find(self._skip_pattern) != -1 - is_quantized = op_node.op().has_attr("quantization_type") and \ - op_node.op().attr("quantization_type") == "qat_with_weight" + is_skip = ( + op_node.op().has_attr("op_namescope") + and op_node.op() + .attr("op_namescope") + .find(self._skip_pattern) + != -1 + ) + is_quantized = ( + op_node.op().has_attr("quantization_type") + and op_node.op().attr("quantization_type") + == "qat_with_weight" + ) if is_skip or is_quantized: continue arg_names = utils._get_op_input_var_names(op_node) for arg_name in arg_names: in_node = graph._find_node_by_name( - op_node.inputs, arg_name) + op_node.inputs, arg_name + ) if in_node.persistable(): continue if arg_name in dequantized_vars_map: @@ -2476,14 +2811,23 @@ class AddQuantDequantPassV2(object): channel_wise=False, moving_rate=self._moving_rate, is_test=self._is_test, - scale_dict=self._scale_dict) - quant_var_node, scale_var_node = insert_quant_pass.insert_quant_op( - graph, in_node) - dequant_var_node = insert_quant_pass.insert_dequant_op( - graph, quant_var_node, scale_var_node) + scale_dict=self._scale_dict, + ) + ( + quant_var_node, + scale_var_node, + ) = insert_quant_pass.insert_quant_op( + graph, in_node + ) + dequant_var_node = ( + insert_quant_pass.insert_dequant_op( + graph, quant_var_node, scale_var_node + ) + ) dequantized_vars_map[arg_name] = dequant_var_node - graph.update_input_link(in_node, dequant_var_node, - op_node) + graph.update_input_link( + in_node, dequant_var_node, op_node + ) t.update() # Backward stage, update input link @@ -2492,10 +2836,12 @@ class AddQuantDequantPassV2(object): for input_name in op_node.input_arg_names(): if input_name in dequantized_vars_map: in_node = graph._find_node_by_name( - op_node.inputs, input_name) + op_node.inputs, input_name + ) dequant_var_node = dequantized_vars_map[input_name] - graph.update_input_link(in_node, dequant_var_node, - op_node) + graph.update_input_link( + in_node, dequant_var_node, op_node + ) return graph @@ -2536,13 +2882,16 @@ class ReplaceFakeQuantDequantPass(object): assert self._place != None, "place must not be None." def apply(self, graph): - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' fake_quant_dequant_ops = [] for op in graph.all_op_nodes(): - if op.name() in _fake_quant_dequant_op_list or op.name( - ) == "moving_average_abs_max_scale": + if ( + op.name() in _fake_quant_dequant_op_list + or op.name() == "moving_average_abs_max_scale" + ): fake_quant_dequant_ops.append(op) for _op in fake_quant_dequant_ops: @@ -2555,13 +2904,18 @@ class ReplaceFakeQuantDequantPass(object): def _replace_op(self, graph, op): x_node = graph._find_node_by_name(op.inputs, op.input("X")[0]) out_node = graph._find_node_by_name(op.outputs, op.output("Out")[0]) - scale_node = graph._find_node_by_name(op.outputs, - op.output("OutScale")[0]) - - quant_axis = op.op().attr("quant_axis") if op.op().has_attr( - "quant_axis") else -1 - bit_length = op.op().attr("bit_length") if op.op().has_attr( - "bit_length") else self._quant_bits + scale_node = graph._find_node_by_name( + op.outputs, op.output("OutScale")[0] + ) + + quant_axis = ( + op.op().attr("quant_axis") if op.op().has_attr("quant_axis") else -1 + ) + bit_length = ( + op.op().attr("bit_length") + if op.op().has_attr("bit_length") + else self._quant_bits + ) zero_point_node = None quanted_node = x_node @@ -2570,43 +2924,46 @@ class ReplaceFakeQuantDequantPass(object): name=self._zero_point_name(quanted_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=scale_node.shape(), - var_dtype=core.VarDesc.VarType.INT32) - _init_var_node(zero_point_node, - np.zeros(scale_node.shape(), dtype="int32"), - self._scope, self._place) - - quant_var_node = graph.create_var_node(name=self._quantized_var_name( - x_node.name()), - var_type=x_node.type(), - shape=x_node.shape(), - var_dtype=x_node.dtype()) - quant_op_node = graph.create_op_node(op_type="quantize_linear", - attrs={ - "quant_axis": quant_axis, - "bit_length": bit_length - }, - inputs={ - "X": x_node, - "Scale": scale_node, - "ZeroPoint": zero_point_node - }, - outputs={"Y": quant_var_node}) + var_dtype=core.VarDesc.VarType.INT32, + ) + _init_var_node( + zero_point_node, + np.zeros(scale_node.shape(), dtype="int32"), + self._scope, + self._place, + ) + + quant_var_node = graph.create_var_node( + name=self._quantized_var_name(x_node.name()), + var_type=x_node.type(), + shape=x_node.shape(), + var_dtype=x_node.dtype(), + ) + quant_op_node = graph.create_op_node( + op_type="quantize_linear", + attrs={"quant_axis": quant_axis, "bit_length": bit_length}, + inputs={ + "X": x_node, + "Scale": scale_node, + "ZeroPoint": zero_point_node, + }, + outputs={"Y": quant_var_node}, + ) graph.link_to(x_node, quant_op_node) graph.link_to(scale_node, quant_op_node) if zero_point_node is not None: graph.link_to(zero_point_node, quant_op_node) graph.link_to(quant_op_node, quant_var_node) - dequant_op_node = graph.create_op_node(op_type="dequantize_linear", - attrs={ - "quant_axis": quant_axis, - "bit_length": bit_length - }, - inputs={ - "X": quant_var_node, - "Scale": scale_node, - "ZeroPoint": zero_point_node - }, - outputs={"Y": out_node}) + dequant_op_node = graph.create_op_node( + op_type="dequantize_linear", + attrs={"quant_axis": quant_axis, "bit_length": bit_length}, + inputs={ + "X": quant_var_node, + "Scale": scale_node, + "ZeroPoint": zero_point_node, + }, + outputs={"Y": out_node}, + ) graph.link_to(quant_var_node, dequant_op_node) graph.link_to(scale_node, dequant_op_node) if zero_point_node is not None: @@ -2657,12 +3014,14 @@ class QuantWeightPass(object): quant_weight_pass.apply(graph) """ - def __init__(self, - scope, - place, - bias_correction=False, - quant_bits=8, - save_int_weight=True): + def __init__( + self, + scope, + place, + bias_correction=False, + quant_bits=8, + save_int_weight=True, + ): self._place = _get_paddle_place(place) self._scope = scope self._bias_correction = bias_correction @@ -2672,8 +3031,9 @@ class QuantWeightPass(object): assert self._place != None, "place must not be None." def apply(self, graph): - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' fake_quant_ops_for_weight = [] fake_quant_ops = [ @@ -2682,17 +3042,21 @@ class QuantWeightPass(object): for _op in fake_quant_ops: x_node = graph._find_node_by_name(_op.inputs, _op.input("X")[0]) if x_node.persistable(): - scale_node = graph._find_node_by_name(_op.inputs, - _op.input("Scale")[0]) + scale_node = graph._find_node_by_name( + _op.inputs, _op.input("Scale")[0] + ) zero_point_node = graph._find_node_by_name( - _op.inputs, - _op.input("ZeroPoint")[0]) - out_node = graph._find_node_by_name(_op.outputs, - _op.output("Y")[0]) + _op.inputs, _op.input("ZeroPoint")[0] + ) + out_node = graph._find_node_by_name( + _op.outputs, _op.output("Y")[0] + ) scale_v = self._load_var(scale_node.name()) - assert scale_v.ndim in [1, 2 - ], "the dim of scale_v should be 1 or 2" + assert scale_v.ndim in [ + 1, + 2, + ], "the dim of scale_v should be 1 or 2" if scale_v.ndim == 2: scale_v = scale_v[0] if scale_v.size == 1 and _op.name() == 'abs_max': @@ -2702,24 +3066,28 @@ class QuantWeightPass(object): param_v = self._load_var(x_node.name()) quant_axis = _op.op().attr("quant_axis") bits_length = _op.op().attr("bit_length") - quantized_param_v = utils.quant_tensor(param_v.copy(), - scale_v, - quant_axis, - bits_length, - onnx_format=True) + quantized_param_v = utils.quant_tensor( + param_v.copy(), + scale_v, + quant_axis, + bits_length, + onnx_format=True, + ) if self._bias_correction == True: quantized_param_v = utils.bias_correction_w( param_v, quantized_param_v, scale_v, quant_axis, - weight_bits=bits_length) + weight_bits=bits_length, + ) if self._save_int_weight: # cast weight type to int if self._quant_bits == 8: save_weight_dtype = np.int8 quantized_param_v = quantized_param_v.astype( - save_weight_dtype) + save_weight_dtype + ) self._restore_var(x_node.name(), quantized_param_v) for next_op_node in out_node.outputs: @@ -2739,8 +3107,10 @@ class QuantWeightPass(object): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) @@ -2775,24 +3145,29 @@ class AddQuantDequantForInferencePass(object): Args: graph(IrGraph): the target graph. """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' dequant_node_map = {} dequantized_vars_map = collections.OrderedDict() for op_node in graph.all_op_nodes(): if op_node.name() in self._teller_set: var_names = utils._get_op_output_var_names(op_node) for var_name in var_names: - out_node = graph._find_node_by_name(op_node.outputs, - var_name) - if out_node.dtype() not in \ - [core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]: + out_node = graph._find_node_by_name( + op_node.outputs, var_name + ) + if out_node.dtype() not in [ + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, + ]: continue if var_name in dequantized_vars_map: dequant_var_node = dequantized_vars_map[var_name] else: dequant_var_node = self._insert_quant_dequant_op( - graph, out_node) + graph, out_node + ) dequantized_vars_map[var_name] = dequant_var_node dequant_node_map[var_name] = dequant_var_node @@ -2805,10 +3180,11 @@ class AddQuantDequantForInferencePass(object): for var_name in var_names: if var_name in dequant_node_map: in_node = graph._find_node_by_name( - op_node.inputs, var_name) - graph.update_input_link(in_node, - dequant_node_map[var_name], - op_node) + op_node.inputs, var_name + ) + graph.update_input_link( + in_node, dequant_node_map[var_name], op_node + ) return graph @@ -2826,22 +3202,29 @@ class AddQuantDequantForInferencePass(object): name="{}.quantized".format(var_name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) - scale_var_node = graph._find_node_by_name(graph.all_persistable_nodes(), - self._scale_name(var_name)) + var_dtype=var_node.dtype(), + ) + scale_var_node = graph._find_node_by_name( + graph.all_persistable_nodes(), self._scale_name(var_name) + ) try: zero_point_node = graph._find_node_by_name( graph.all_persistable_nodes(), - "{}@zero_point".format(quant_var_node.name())) + "{}@zero_point".format(quant_var_node.name()), + ) except: zero_point_node = graph.create_persistable_node( name="{}@zero_point".format(quant_var_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=scale_var_node.shape(), - var_dtype=core.VarDesc.VarType.INT32) - _init_var_node(zero_point_node, - np.zeros(scale_var_node.shape(), dtype="int32"), - self._scope, self._place) + var_dtype=core.VarDesc.VarType.INT32, + ) + _init_var_node( + zero_point_node, + np.zeros(scale_var_node.shape(), dtype="int32"), + self._scope, + self._place, + ) inputs = {"X": var_node, "Scale": scale_var_node} if zero_point_node is not None: @@ -2851,10 +3234,12 @@ class AddQuantDequantForInferencePass(object): attrs["op_role"] = core.op_proto_and_checker_maker.OpRole.Forward outputs = {"Y": quant_var_node} - quant_op_node = graph.create_op_node(op_type="quantize_linear", - attrs=attrs, - inputs=inputs, - outputs=outputs) + quant_op_node = graph.create_op_node( + op_type="quantize_linear", + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_var_node, quant_op_node) @@ -2867,7 +3252,8 @@ class AddQuantDequantForInferencePass(object): name="{}.dequantized".format(quant_var_node.name()), var_type=quant_var_node.type(), shape=quant_var_node.shape(), - var_dtype=quant_var_node.dtype()) + var_dtype=quant_var_node.dtype(), + ) inputs = {"X": quant_var_node, "Scale": scale_var_node} if zero_point_node is not None: @@ -2876,10 +3262,12 @@ class AddQuantDequantForInferencePass(object): attrs = {"quant_axis": -1, "bit_length": self._quant_bits} attrs["op_role"] = core.op_proto_and_checker_maker.OpRole.Forward - dequant_op_node = graph.create_op_node(op_type="dequantize_linear", - attrs=attrs, - inputs=inputs, - outputs={"Y": dequant_var_node}) + dequant_op_node = graph.create_op_node( + op_type="dequantize_linear", + attrs=attrs, + inputs=inputs, + outputs={"Y": dequant_var_node}, + ) graph.link_to(quant_var_node, dequant_op_node) graph.link_to(scale_var_node, dequant_op_node) diff --git a/python/paddle/fluid/contrib/slim/quantization/quantize_transpiler_v2.py b/python/paddle/fluid/contrib/slim/quantization/quantize_transpiler_v2.py index ce20d2e695a..12f6f3c53d2 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quantize_transpiler_v2.py +++ b/python/paddle/fluid/contrib/slim/quantization/quantize_transpiler_v2.py @@ -24,9 +24,9 @@ from ....param_attr import ParamAttr from ....initializer import Constant from ....log_helper import get_logger -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def find_next_ops(block, var_name): @@ -45,24 +45,24 @@ def load_variable_data(scope, var_name): Load variable value from scope ''' var_node = scope.find_var(var_name) - assert var_node is not None, \ - "Cannot find " + var_name + " in scope." + assert var_node is not None, "Cannot find " + var_name + " in scope." return np.array(var_node.get_tensor()) class QuantizeTranspilerV2(object): - - def __init__(self, - weight_bits=8, - activation_bits=8, - weight_quantize_type='abs_max', - activation_quantize_type='moving_average_abs_max', - quantizable_op_type=[ - 'conv2d', - 'depthwise_conv2d', - 'mul', - ], - skip_pattern=['skip_quant']): + def __init__( + self, + weight_bits=8, + activation_bits=8, + weight_quantize_type='abs_max', + activation_quantize_type='moving_average_abs_max', + quantizable_op_type=[ + 'conv2d', + 'depthwise_conv2d', + 'mul', + ], + skip_pattern=['skip_quant'], + ): """ Apply fake quant for the quantized ops. @@ -81,18 +81,26 @@ class QuantizeTranspilerV2(object): self._weight_bits = weight_bits self._activation_bits = activation_bits - assert activation_quantize_type in \ - ["abs_max", "moving_average_abs_max"], \ - "activation_quantize_type should be abs_max " \ + assert activation_quantize_type in [ + "abs_max", + "moving_average_abs_max", + ], ( + "activation_quantize_type should be abs_max " "or moving_average_abs_max for now." - assert weight_quantize_type in ["abs_max", "channel_wise_abs_max"], \ - "weight_quantize_type should be abs_max or channel_wise_abs_max." + ) + assert weight_quantize_type in [ + "abs_max", + "channel_wise_abs_max", + ], "weight_quantize_type should be abs_max or channel_wise_abs_max." self._activation_quantize_type = activation_quantize_type self._weight_quantize_type = weight_quantize_type for op_type in quantizable_op_type: - assert op_type in ['conv2d', 'depthwise_conv2d', 'mul'], \ - "Quantize op should be ['conv2d', 'depthwise_conv2d', 'mul']" + assert op_type in [ + 'conv2d', + 'depthwise_conv2d', + 'mul', + ], "Quantize op should be ['conv2d', 'depthwise_conv2d', 'mul']" self._quantizable_ops = quantizable_op_type self._quantizable_grad_ops = [ '%s_grad' % (op) for op in self._quantizable_ops @@ -115,10 +123,12 @@ class QuantizeTranspilerV2(object): Returns: None """ - assert isinstance(program, Program), \ - "program must be the instance of Program" - assert isinstance(startup_program, Program), \ - "startup_program must be the instance of Program" + assert isinstance( + program, Program + ), "program must be the instance of Program" + assert isinstance( + startup_program, Program + ), "startup_program must be the instance of Program" var_rename_map = [ collections.OrderedDict() for _ in range(len(program.blocks)) @@ -127,16 +137,19 @@ class QuantizeTranspilerV2(object): for block in program.blocks: ops = list(block.ops) for op in ops: - if op.type in self._quantizable_ops and \ - (not self._is_skip_quant(op)): - self._transform_forward(block, op, var_rename_map, - is_test) + if op.type in self._quantizable_ops and ( + not self._is_skip_quant(op) + ): + self._transform_forward( + block, op, var_rename_map, is_test + ) for block in program.blocks: ops = list(block.ops) for op in ops: - if op.type in self._quantizable_grad_ops and \ - (not self._is_skip_quant(op)): + if op.type in self._quantizable_grad_ops and ( + not self._is_skip_quant(op) + ): self._transform_backward(block, op, var_rename_map) def convert(self, test_program, scope=None): @@ -153,8 +166,10 @@ class QuantizeTranspilerV2(object): for block in test_program.blocks: for op in block.ops: - if op.has_attr("quantization_type") \ - and op.attr("quantization_type") == "qat_with_weight": + if ( + op.has_attr("quantization_type") + and op.attr("quantization_type") == "qat_with_weight" + ): # quant op -> var1 -> fake op -> var2 assert len(op.output_arg_names) == 1 var1_name = op.output_arg_names[0] @@ -188,29 +203,40 @@ class QuantizeTranspilerV2(object): else: in_var = block.var(in_name) target_dtype = [ - core.VarDesc.VarType.FP32, core.VarDesc.VarType.FP16 + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, ] if in_var.dtype not in target_dtype: continue - quant_bits = self._weight_bits if in_var.persistable \ - else self._activation_bits - quant_type = self._weight_quantize_type if in_var.persistable \ - else self._activation_quantize_type + quant_bits = ( + self._weight_bits + if in_var.persistable + else self._activation_bits + ) + quant_type = ( + self._weight_quantize_type + if in_var.persistable + else self._activation_quantize_type + ) if quant_type == "abs_max": new_var = self._insert_abs_max_fq_op( - block, idx, in_var, quant_bits) + block, idx, in_var, quant_bits + ) elif quant_type == "moving_average_abs_max": new_var = self._insert_ma_abs_max_fq_op( - block, idx, in_var, quant_bits, is_test) + block, idx, in_var, quant_bits, is_test + ) elif quant_type == "channel_wise_abs_max": ch_axis = 1 if op.type in self._out_ch_axis1_ops else 0 new_var = self._insert_pc_abs_max_fq_op( - block, idx, in_var, quant_bits, ch_axis) + block, idx, in_var, quant_bits, ch_axis + ) else: - _logger.error("Don't support the quant_type: %s" % - quant_type) + _logger.error( + "Don't support the quant_type: %s" % quant_type + ) continue new_in_name = new_var.name @@ -225,7 +251,8 @@ class QuantizeTranspilerV2(object): idx = block.ops.index(op) out_var = block.var(out_name) new_out_var = self._insert_ma_abs_max_scale_op( - block, idx + 1, out_var, is_test, True) + block, idx + 1, out_var, is_test, True + ) for next_op in next_ops: if "_grad" not in next_op.type: @@ -237,13 +264,15 @@ class QuantizeTranspilerV2(object): """ user_skipped = False if isinstance(self._skip_pattern, list): - user_skipped = op.has_attr("op_namescope") and \ - any(pattern in op.attr("op_namescope") \ - for pattern in self._skip_pattern) + user_skipped = op.has_attr("op_namescope") and any( + pattern in op.attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - user_skipped = op.has_attr("op_namescope") and \ - op.attr("op_namescope").find( - self._skip_pattern) != -1 + user_skipped = ( + op.has_attr("op_namescope") + and op.attr("op_namescope").find(self._skip_pattern) != -1 + ) return user_skipped def _transform_backward(self, block, op, var_rename_map): @@ -259,75 +288,92 @@ class QuantizeTranspilerV2(object): op._rename_input(name, new_var_name) no_dequanted_input_vars = False if no_dequanted_input_vars: - raise ValueError("There is no dequanted inputs for op %s." % - (op.type)) + raise ValueError( + "There is no dequanted inputs for op %s." % (op.type) + ) def _insert_abs_max_fq_op(self, block, idx, in_var, quant_bits): """ Inset abs max fake quant op. """ - quant_dequant_var = block.create_var(type=in_var.type, - name="{}.quant_dequant".format( - in_var.name), - shape=in_var.shape, - dtype=in_var.dtype) - scale_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.scale".format(in_var.name), - initializer=Constant(0.), - trainable=False), - shape=[1], - dtype=in_var.dtype) + quant_dequant_var = block.create_var( + type=in_var.type, + name="{}.quant_dequant".format(in_var.name), + shape=in_var.shape, + dtype=in_var.dtype, + ) + scale_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.scale".format(in_var.name), + initializer=Constant(0.0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) scale_var.stop_gradient = True inputs = {'X': in_var} outputs = {'Out': quant_dequant_var, 'OutScale': scale_var} attrs = {'bit_length': quant_bits} - block._insert_op(idx, - type='fake_quantize_dequantize_abs_max', - attrs=attrs, - inputs=inputs, - outputs=outputs) + block._insert_op( + idx, + type='fake_quantize_dequantize_abs_max', + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) return quant_dequant_var def _insert_ma_abs_max_fq_op(self, block, idx, in_var, quant_bits, is_test): """ Insert moving average abs max fake quant op. """ - quant_dequant_var = block.create_var(type=in_var.type, - name="{}.quant_dequant".format( - in_var.name), - shape=in_var.shape, - dtype=in_var.dtype) - - scale_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.scale".format(in_var.name), - initializer=Constant(0.), - trainable=False), - shape=[1], - dtype=in_var.dtype) + quant_dequant_var = block.create_var( + type=in_var.type, + name="{}.quant_dequant".format(in_var.name), + shape=in_var.shape, + dtype=in_var.dtype, + ) + + scale_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.scale".format(in_var.name), + initializer=Constant(0.0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) scale_var.stop_gradient = True if not is_test: - state_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.state".format(in_var.name), - initializer=Constant(0), - trainable=False), - shape=[1], - dtype=in_var.dtype) + state_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.state".format(in_var.name), + initializer=Constant(0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) state_var.stop_gradient = True - accum_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.accum".format(in_var.name), - initializer=Constant(0), - trainable=False), - shape=[1], - dtype=in_var.dtype) + accum_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.accum".format(in_var.name), + initializer=Constant(0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) accum_var.stop_gradient = True attrs = { 'moving_rate': self._moving_rate, 'bit_length': quant_bits, - 'is_test': is_test + 'is_test': is_test, } inputs = {'X': in_var, 'InScale': scale_var} outputs = {'Out': quant_dequant_var, 'OutScale': scale_var} @@ -337,56 +383,64 @@ class QuantizeTranspilerV2(object): outputs['OutState'] = state_var outputs['OutAccum'] = accum_var - block._insert_op(idx, - type='fake_quantize_dequantize_moving_average_abs_max', - attrs=attrs, - inputs=inputs, - outputs=outputs) + block._insert_op( + idx, + type='fake_quantize_dequantize_moving_average_abs_max', + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) return quant_dequant_var def _insert_pc_abs_max_fq_op(self, block, idx, in_var, quant_bits, ch_axis): """ Insert per channel abs max fake quant op. """ - quant_dequant_var = block.create_var(type=in_var.type, - name="{}.quant_dequant".format( - in_var.name), - shape=in_var.shape, - dtype=in_var.dtype) - - scale_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.scale".format(in_var.name), - initializer=Constant(0.), - trainable=False), - shape=[in_var.shape[ch_axis]], - dtype=in_var.dtype) + quant_dequant_var = block.create_var( + type=in_var.type, + name="{}.quant_dequant".format(in_var.name), + shape=in_var.shape, + dtype=in_var.dtype, + ) + + scale_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.scale".format(in_var.name), + initializer=Constant(0.0), + trainable=False, + ), + shape=[in_var.shape[ch_axis]], + dtype=in_var.dtype, + ) scale_var.stop_gradient = True inputs = {'X': in_var} outputs = {'Out': quant_dequant_var, 'OutScale': scale_var} attrs = {'bit_length': quant_bits, 'quant_axis': ch_axis} - block._insert_op(idx, - type='fake_channel_wise_quantize_dequantize_abs_max', - attrs=attrs, - inputs=inputs, - outputs=outputs) + block._insert_op( + idx, + type='fake_channel_wise_quantize_dequantize_abs_max', + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) return quant_dequant_var - def _insert_ma_abs_max_scale_op(self, - block, - idx, - in_var, - is_test, - has_out_var=False): + def _insert_ma_abs_max_scale_op( + self, block, idx, in_var, is_test, has_out_var=False + ): """ Insert moving average abs max scale op. """ - scale_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.outscale.scale".format(in_var.name), - initializer=Constant(0.), - trainable=False), - shape=[1], - dtype=in_var.dtype) + scale_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.outscale.scale".format(in_var.name), + initializer=Constant(0.0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) scale_var.stop_gradient = True attrs = {'moving_rate': self._moving_rate, 'is_test': is_test} @@ -394,20 +448,26 @@ class QuantizeTranspilerV2(object): outputs = {'OutScale': scale_var} if not is_test: - state_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.outscale.state".format(in_var.name), - initializer=Constant(0), - trainable=False), - shape=[1], - dtype=in_var.dtype) + state_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.outscale.state".format(in_var.name), + initializer=Constant(0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) state_var.stop_gradient = True - accum_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.outscale.accum".format(in_var.name), - initializer=Constant(0), - trainable=False), - shape=[1], - dtype=in_var.dtype) + accum_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.outscale.accum".format(in_var.name), + initializer=Constant(0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) accum_var.stop_gradient = True inputs['InState'] = state_var @@ -416,18 +476,22 @@ class QuantizeTranspilerV2(object): outputs['OutAccum'] = accum_var if has_out_var: - out_var = block.create_var(type=in_var.type, - name="{}.tmp".format(in_var.name), - shape=in_var.shape, - dtype=in_var.dtype) + out_var = block.create_var( + type=in_var.type, + name="{}.tmp".format(in_var.name), + shape=in_var.shape, + dtype=in_var.dtype, + ) outputs['Out'] = out_var - block._insert_op(idx, - type='moving_average_abs_max_scale', - attrs=attrs, - inputs=inputs, - outputs=outputs) + block._insert_op( + idx, + type='moving_average_abs_max_scale', + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) if has_out_var: return out_var diff --git a/python/paddle/fluid/contrib/slim/quantization/utils.py b/python/paddle/fluid/contrib/slim/quantization/utils.py index 158f7e07a0d..11e39116389 100644 --- a/python/paddle/fluid/contrib/slim/quantization/utils.py +++ b/python/paddle/fluid/contrib/slim/quantization/utils.py @@ -18,8 +18,12 @@ from ....framework import IrNode from ....framework import Operator _weight_supported_quantizable_op_type = [ - 'conv2d', 'depthwise_conv2d', 'conv2d_transpose', 'mul', 'matmul', - 'matmul_v2' + 'conv2d', + 'depthwise_conv2d', + 'conv2d_transpose', + 'mul', + 'matmul', + 'matmul_v2', ] _act_supported_quantizable_op_type = [ @@ -114,13 +118,19 @@ _act_supported_quantizable_op_type = [ ] QUANT_SUPPORTED_OP_TYPE_LIST = list( - set(_weight_supported_quantizable_op_type + - _act_supported_quantizable_op_type)) + set( + _weight_supported_quantizable_op_type + + _act_supported_quantizable_op_type + ) +) _out_scale_op_list = QUANT_SUPPORTED_OP_TYPE_LIST _channelwise_quant_axis1_ops = [ - 'conv2d_transpose', 'mul', 'matmul', 'matmul_v2' + 'conv2d_transpose', + 'mul', + 'matmul', + 'matmul_v2', ] # list op real input and output names, to avoid processing input such as AxisTensor. @@ -230,11 +240,11 @@ def _get_op_input_var_names(op): Returns: input_var_names or None. """ - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." var_names = [] - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in _op_real_in_out_name: return [] @@ -250,11 +260,11 @@ def _get_op_input_var_names(op): def _get_op_output_var_names(op): """ """ - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." var_names = [] - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in _op_real_in_out_name: return [] @@ -270,10 +280,10 @@ def _get_op_output_var_names(op): def _get_input_name_index(op, input_var_name): """Get the input name and index of the var_name in the op""" - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in _op_real_in_out_name: return None @@ -288,10 +298,10 @@ def _get_input_name_index(op, input_var_name): def _get_output_name_index(op, output_var_name): """Get the output name and index of the var_name in the op""" - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in _op_real_in_out_name: return None @@ -310,8 +320,7 @@ def load_variable_data(scope, var_name): Load variable value from scope ''' var_node = scope.find_var(var_name) - assert var_node is not None, \ - "Cannot find " + var_name + " in scope." + assert var_node is not None, "Cannot find " + var_name + " in scope." return np.array(var_node.get_tensor()) @@ -319,8 +328,9 @@ def set_variable_data(scope, place, var_name, np_value): ''' Set the value of var node by name, if the node exits, ''' - assert isinstance(np_value, np.ndarray), \ - 'The type of value should be numpy array.' + assert isinstance( + np_value, np.ndarray + ), 'The type of value should be numpy array.' var_node = scope.find_var(var_name) if var_node != None: tensor = var_node.get_tensor() @@ -405,10 +415,12 @@ def bias_correction_w(x, x_quant, scale_v, quant_axis, weight_bits=8): x_dequant[:, i] = x_quant[:, i] * s / bnt quant_bias = x - x_dequant mean_bias = np.array( - [quant_bias[:, i].mean() for i in range(quant_bias.shape[1])]) + [quant_bias[:, i].mean() for i in range(quant_bias.shape[1])] + ) std_orig = np.array([x[:, i].std() for i in range(x.shape[1])]) std_quant = np.array( - [x_dequant[:, i].std() for i in range(x_dequant.shape[1])]) + [x_dequant[:, i].std() for i in range(x_dequant.shape[1])] + ) std_bias = std_orig / (std_quant + eps) else: x_dequant = x_quant * scale_v / bnt @@ -419,8 +431,9 @@ def bias_correction_w(x, x_quant, scale_v, quant_axis, weight_bits=8): mean_bias = np.resize(mean_bias, x.shape) x_dequant = (mean_bias + x_dequant) * std_bias - quantized_param_v = quant_tensor(x_dequant, scale_v, quant_axis, - weight_bits) + quantized_param_v = quant_tensor( + x_dequant, scale_v, quant_axis, weight_bits + ) return quantized_param_v @@ -430,8 +443,10 @@ def stable_sigmoid(x): def calculate_quant_cos_error(orig_tensor, qdq_tensor): - cos_sim = np.inner(orig_tensor.flatten(), qdq_tensor.flatten()) \ - / (np.linalg.norm(orig_tensor.flatten()) * np.linalg.norm(qdq_tensor.flatten())) + cos_sim = np.inner(orig_tensor.flatten(), qdq_tensor.flatten()) / ( + np.linalg.norm(orig_tensor.flatten()) + * np.linalg.norm(qdq_tensor.flatten()) + ) return cos_sim @@ -454,11 +469,10 @@ def move_persistable_var_to_global_block(program): def l2_loss(gt, pred): - return ((gt - pred)**2).mean() + return ((gt - pred) ** 2).mean() class tqdm(object): - def __init__(self, total, bar_format='Loading|{bar}', ncols=80): self.total = total self.bar_format = bar_format @@ -470,8 +484,9 @@ class tqdm(object): a = "=" * round((self.n / self.total) * self.ncols) b = " " * (self.ncols - len(a)) prefix = self.bar_format.split('|')[0] - sys.stderr.write("\r{}|{}=>{}| {}/{}".format(prefix, a, b, self.n, - self.total)) + sys.stderr.write( + "\r{}|{}=>{}| {}/{}".format(prefix, a, b, self.n, self.total) + ) sys.stderr.flush() def __enter__(self): diff --git a/python/paddle/fluid/contrib/slim/tests/convert_model2dot.py b/python/paddle/fluid/contrib/slim/tests/convert_model2dot.py index 3573f53e22d..7bb7de706bc 100644 --- a/python/paddle/fluid/contrib/slim/tests/convert_model2dot.py +++ b/python/paddle/fluid/contrib/slim/tests/convert_model2dot.py @@ -26,20 +26,21 @@ paddle.enable_static() def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--model_path', - type=str, - default='', - help='A path to a model.') - parser.add_argument('--save_graph_dir', - type=str, - default='', - help='A path to save the graph.') + parser.add_argument( + '--model_path', type=str, default='', help='A path to a model.' + ) + parser.add_argument( + '--save_graph_dir', + type=str, + default='', + help='A path to save the graph.', + ) parser.add_argument( '--save_graph_name', type=str, default='', - help= - 'A name to save the graph. Default - name from model path will be used') + help='A name to save the graph. Default - name from model path will be used', + ) test_args, args = parser.parse_known_args(namespace=unittest) return test_args, sys.argv[:1] + args @@ -51,12 +52,19 @@ def generate_dot_for_model(model_path, save_graph_dir, save_graph_name): inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, exe, 'model', 'params' + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) if not os.path.exists(save_graph_dir): os.makedirs(save_graph_dir) @@ -65,12 +73,17 @@ def generate_dot_for_model(model_path, save_graph_dir, save_graph_name): save_graph_name = model_name graph.draw(save_graph_dir, save_graph_name, graph.all_op_nodes()) print( - "Success! Generated dot and pdf files for {0} model, that can be found at {1} named {2}.\n" - .format(model_name, save_graph_dir, save_graph_name)) + "Success! Generated dot and pdf files for {0} model, that can be found at {1} named {2}.\n".format( + model_name, save_graph_dir, save_graph_name + ) + ) if __name__ == '__main__': global test_args test_args, remaining_args = parse_args() - generate_dot_for_model(test_args.model_path, test_args.save_graph_dir, - test_args.save_graph_name) + generate_dot_for_model( + test_args.model_path, + test_args.save_graph_dir, + test_args.save_graph_name, + ) diff --git a/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py b/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py index d69241d6cb9..43ace1b7700 100644 --- a/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py +++ b/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py @@ -24,9 +24,9 @@ from paddle.nn import BatchNorm1D from paddle.fluid.log_helper import get_logger -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def fix_model_dict(model): @@ -37,16 +37,18 @@ def fix_model_dict(model): if name.endswith("bias"): value = np.zeros_like(p_value).astype('float32') else: - value = np.random.normal( - loc=0.0, scale=0.01, - size=np.product(p_shape)).reshape(p_shape).astype('float32') + value = ( + np.random.normal(loc=0.0, scale=0.01, size=np.product(p_shape)) + .reshape(p_shape) + .astype('float32') + ) fixed_state[name] = value model.set_dict(fixed_state) return model def pre_hook(layer, input): - input_return = (input[0] * 2) + input_return = input[0] * 2 return input_return @@ -59,8 +61,9 @@ def train_lenet(lenet, reader, optimizer): lenet.train() for batch_id, data in enumerate(reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') + x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) y_data = np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) img = paddle.to_tensor(x_data) @@ -82,7 +85,6 @@ def train_lenet(lenet, reader, optimizer): class ImperativeLenet(fluid.dygraph.Layer): - def __init__(self, num_classes=10): super(ImperativeLenet, self).__init__() conv2d_w1_attr = fluid.ParamAttr(name="conv2d_w_1") @@ -95,36 +97,55 @@ class ImperativeLenet(fluid.dygraph.Layer): fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") self.features = Sequential( - Conv2D(in_channels=1, - out_channels=6, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2d_w1_attr, - bias_attr=False), BatchNorm2D(6), ReLU(), + Conv2D( + in_channels=1, + out_channels=6, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2d_w1_attr, + bias_attr=False, + ), + BatchNorm2D(6), + ReLU(), + MaxPool2D(kernel_size=2, stride=2), + Conv2D( + in_channels=6, + out_channels=16, + kernel_size=5, + stride=1, + padding=0, + weight_attr=conv2d_w2_attr, + bias_attr=conv2d_b2_attr, + ), + BatchNorm2D(16), + PReLU(), MaxPool2D(kernel_size=2, stride=2), - Conv2D(in_channels=6, - out_channels=16, - kernel_size=5, - stride=1, - padding=0, - weight_attr=conv2d_w2_attr, - bias_attr=conv2d_b2_attr), BatchNorm2D(16), PReLU(), - MaxPool2D(kernel_size=2, stride=2)) + ) self.fc = Sequential( - Linear(in_features=400, - out_features=120, - weight_attr=fc_w1_attr, - bias_attr=fc_b1_attr), LeakyReLU(), - Linear(in_features=120, - out_features=84, - weight_attr=fc_w2_attr, - bias_attr=fc_b2_attr), Sigmoid(), - Linear(in_features=84, - out_features=num_classes, - weight_attr=fc_w3_attr, - bias_attr=fc_b3_attr), Softmax()) + Linear( + in_features=400, + out_features=120, + weight_attr=fc_w1_attr, + bias_attr=fc_b1_attr, + ), + LeakyReLU(), + Linear( + in_features=120, + out_features=84, + weight_attr=fc_w2_attr, + bias_attr=fc_b2_attr, + ), + Sigmoid(), + Linear( + in_features=84, + out_features=num_classes, + weight_attr=fc_w3_attr, + bias_attr=fc_b3_attr, + ), + Softmax(), + ) self.add = paddle.nn.quant.add() self.quant_stub = paddle.nn.quant.QuantStub() @@ -139,7 +160,6 @@ class ImperativeLenet(fluid.dygraph.Layer): class ImperativeLenetWithSkipQuant(fluid.dygraph.Layer): - def __init__(self, num_classes=10): super(ImperativeLenetWithSkipQuant, self).__init__() @@ -153,48 +173,58 @@ class ImperativeLenetWithSkipQuant(fluid.dygraph.Layer): fc_b1_attr = fluid.ParamAttr(name="fc_b_1") fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") - self.conv2d_0 = Conv2D(in_channels=1, - out_channels=6, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2d_w1_attr, - bias_attr=conv2d_b1_attr) + self.conv2d_0 = Conv2D( + in_channels=1, + out_channels=6, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2d_w1_attr, + bias_attr=conv2d_b1_attr, + ) self.conv2d_0.skip_quant = True self.batch_norm_0 = BatchNorm2D(6) self.relu_0 = ReLU() self.pool2d_0 = MaxPool2D(kernel_size=2, stride=2) - self.conv2d_1 = Conv2D(in_channels=6, - out_channels=16, - kernel_size=5, - stride=1, - padding=0, - weight_attr=conv2d_w2_attr, - bias_attr=conv2d_b2_attr) + self.conv2d_1 = Conv2D( + in_channels=6, + out_channels=16, + kernel_size=5, + stride=1, + padding=0, + weight_attr=conv2d_w2_attr, + bias_attr=conv2d_b2_attr, + ) self.conv2d_1.skip_quant = False self.batch_norm_1 = BatchNorm2D(16) self.relu6_0 = ReLU6() self.pool2d_1 = MaxPool2D(kernel_size=2, stride=2) - self.linear_0 = Linear(in_features=400, - out_features=120, - weight_attr=fc_w1_attr, - bias_attr=fc_b1_attr) + self.linear_0 = Linear( + in_features=400, + out_features=120, + weight_attr=fc_w1_attr, + bias_attr=fc_b1_attr, + ) self.linear_0.skip_quant = True self.leaky_relu_0 = LeakyReLU() - self.linear_1 = Linear(in_features=120, - out_features=84, - weight_attr=fc_w2_attr, - bias_attr=fc_b2_attr) + self.linear_1 = Linear( + in_features=120, + out_features=84, + weight_attr=fc_w2_attr, + bias_attr=fc_b2_attr, + ) self.linear_1.skip_quant = False self.sigmoid_0 = Sigmoid() - self.linear_2 = Linear(in_features=84, - out_features=num_classes, - weight_attr=fc_w3_attr, - bias_attr=fc_b3_attr) + self.linear_2 = Linear( + in_features=84, + out_features=num_classes, + weight_attr=fc_w3_attr, + bias_attr=fc_b3_attr, + ) self.linear_2.skip_quant = False self.softmax_0 = Softmax() @@ -221,24 +251,28 @@ class ImperativeLenetWithSkipQuant(fluid.dygraph.Layer): class ImperativeLinearBn(fluid.dygraph.Layer): - def __init__(self): super(ImperativeLinearBn, self).__init__() fc_w_attr = paddle.ParamAttr( name="fc_weight", - initializer=paddle.nn.initializer.Constant(value=0.5)) + initializer=paddle.nn.initializer.Constant(value=0.5), + ) fc_b_attr = paddle.ParamAttr( name="fc_bias", - initializer=paddle.nn.initializer.Constant(value=1.0)) + initializer=paddle.nn.initializer.Constant(value=1.0), + ) bn_w_attr = paddle.ParamAttr( name="bn_weight", - initializer=paddle.nn.initializer.Constant(value=0.5)) - - self.linear = Linear(in_features=10, - out_features=10, - weight_attr=fc_w_attr, - bias_attr=fc_b_attr) + initializer=paddle.nn.initializer.Constant(value=0.5), + ) + + self.linear = Linear( + in_features=10, + out_features=10, + weight_attr=fc_w_attr, + bias_attr=fc_b_attr, + ) self.bn = BatchNorm1D(10, weight_attr=bn_w_attr) def forward(self, inputs): @@ -249,17 +283,17 @@ class ImperativeLinearBn(fluid.dygraph.Layer): class ImperativeLinearBn_hook(fluid.dygraph.Layer): - def __init__(self): super(ImperativeLinearBn_hook, self).__init__() fc_w_attr = paddle.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.Constant(value=0.5)) + initializer=paddle.nn.initializer.Constant(value=0.5), + ) - self.linear = Linear(in_features=10, - out_features=10, - weight_attr=fc_w_attr) + self.linear = Linear( + in_features=10, out_features=10, weight_attr=fc_w_attr + ) self.bn = BatchNorm1D(10) forward_pre = self.linear.register_forward_pre_hook(pre_hook) diff --git a/python/paddle/fluid/contrib/slim/tests/quant2_int8_image_classification_comparison.py b/python/paddle/fluid/contrib/slim/tests/quant2_int8_image_classification_comparison.py index c3d7f1a254e..dcd5d6de313 100644 --- a/python/paddle/fluid/contrib/slim/tests/quant2_int8_image_classification_comparison.py +++ b/python/paddle/fluid/contrib/slim/tests/quant2_int8_image_classification_comparison.py @@ -40,50 +40,50 @@ def parse_args(): '--skip_batch_num', type=int, default=0, - help='Number of the first minibatches to skip in performance statistics.' + help='Number of the first minibatches to skip in performance statistics.', + ) + parser.add_argument( + '--quant_model', type=str, default='', help='A path to a Quant model.' + ) + parser.add_argument( + '--fp32_model', type=str, default='', help='A path to an FP32 model.' ) - parser.add_argument('--quant_model', - type=str, - default='', - help='A path to a Quant model.') - parser.add_argument('--fp32_model', - type=str, - default='', - help='A path to an FP32 model.') parser.add_argument('--infer_data', type=str, default='', help='Data file.') parser.add_argument( '--batch_num', type=int, default=0, - help= - 'Number of batches to process. 0 or less means whole dataset. Default: 0.' + help='Number of batches to process. 0 or less means whole dataset. Default: 0.', + ) + parser.add_argument( + '--acc_diff_threshold', + type=float, + default=0.01, + help='Accepted accuracy difference threshold.', ) - parser.add_argument('--acc_diff_threshold', - type=float, - default=0.01, - help='Accepted accuracy difference threshold.') parser.add_argument( '--ops_to_quantize', type=str, default='', - help= - 'A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.' + help='A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.', ) parser.add_argument( '--op_ids_to_skip', type=str, default='', - help='A comma separated list of operator ids to skip in quantization.') + help='A comma separated list of operator ids to skip in quantization.', + ) parser.add_argument( '--targets', type=str, default='quant,int8,fp32', - help= - 'A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"' + help='A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"', + ) + parser.add_argument( + '--debug', + action='store_true', + help='If used, the graph of Quant model is drawn.', ) - parser.add_argument('--debug', - action='store_true', - help='If used, the graph of Quant model is drawn.') test_args, args = parser.parse_known_args(namespace=unittest) return test_args, sys.argv[:1] + args @@ -95,7 +95,6 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): """ def _reader_creator(self, data_file='data.bin'): - def reader(): with open(data_file, 'rb') as fp: num = fp.read(8) @@ -114,7 +113,8 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): fp.seek(imgs_offset + img_size * step) img = fp.read(img_size) img = struct.unpack_from( - '{}f'.format(img_ch * img_w * img_h), img) + '{}f'.format(img_ch * img_w * img_h), img + ) img = np.array(img) img.shape = (img_ch, img_w, img_h) fp.seek(labels_offset + label_size * step) @@ -148,14 +148,14 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): name = op_node.name() if name in ['depthwise_conv2d']: input_var_node = graph._find_node_by_name( - op_node.inputs, - op_node.input("Input")[0]) + op_node.inputs, op_node.input("Input")[0] + ) weight_var_node = graph._find_node_by_name( - op_node.inputs, - op_node.input("Filter")[0]) + op_node.inputs, op_node.input("Filter")[0] + ) output_var_node = graph._find_node_by_name( - graph.all_var_nodes(), - op_node.output("Output")[0]) + graph.all_var_nodes(), op_node.output("Output")[0] + ) attrs = { name: op_node.op().attr(name) for name in op_node.op().attr_names() @@ -164,11 +164,9 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): conv_op_node = graph.create_op_node( op_type='conv2d', attrs=attrs, - inputs={ - 'Input': input_var_node, - 'Filter': weight_var_node - }, - outputs={'Output': output_var_node}) + inputs={'Input': input_var_node, 'Filter': weight_var_node}, + outputs={'Output': output_var_node}, + ) graph.link_to(input_var_node, conv_op_node) graph.link_to(weight_var_node, conv_op_node) @@ -177,28 +175,37 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): return graph - def _predict(self, - test_reader=None, - model_path=None, - batch_size=1, - batch_num=1, - skip_batch_num=0, - target='quant'): + def _predict( + self, + test_reader=None, + model_path=None, + batch_size=1, + batch_num=1, + skip_batch_num=0, + target='quant', + ): assert target in ['quant', 'int8', 'fp32'] place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, exe, 'model', 'params' + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) - if (self._debug): + if self._debug: graph.draw('.', 'quant_orig', graph.all_op_nodes()) quant_transform_pass = Quant2Int8MkldnnPass( self._quantized_ops, @@ -206,10 +213,11 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): _scope=inference_scope, _place=place, _core=core, - _debug=self._debug) - if (target == 'quant'): + _debug=self._debug, + ) + if target == 'quant': graph = self._prepare_for_fp32_mkldnn(graph) - elif (target == 'int8'): + elif target == 'int8': graph = quant_transform_pass.apply(graph) else: # target == fp32 graph = quant_transform_pass.prepare_and_optimize_fp32(graph) @@ -238,30 +246,35 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): images = np.array(images).astype('float32') labels = np.array([x[1] for x in data]).astype('int64') - if (target == 'fp32'): + if target == 'fp32': # FP32 models have accuracy measuring layers labels = labels.reshape([-1, 1]) start = time.time() - out = exe.run(inference_program, - feed={ - feed_target_names[0]: images, - feed_target_names[1]: labels - }, - fetch_list=fetch_targets) + out = exe.run( + inference_program, + feed={ + feed_target_names[0]: images, + feed_target_names[1]: labels, + }, + fetch_list=fetch_targets, + ) batch_time = (time.time() - start) * 1000 # in miliseconds batch_acc1, batch_acc5 = out[1][0], out[2][0] outputs.append(batch_acc1) else: # Quant INT8 models do not have accuracy measuring layers start = time.time() - out = exe.run(inference_program, - feed={feed_target_names[0]: images}, - fetch_list=fetch_targets) + out = exe.run( + inference_program, + feed={feed_target_names[0]: images}, + fetch_list=fetch_targets, + ) batch_time = (time.time() - start) * 1000 # in miliseconds outputs.append(out[0]) # Calculate accuracy result batch_acc1, batch_acc5 = self._get_batch_accuracy( - out[0], labels) + out[0], labels + ) infer_accs1.append(batch_acc1) infer_accs5.append(batch_acc5) @@ -272,10 +285,17 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): fpses.append(fps) iters += 1 appx = ' (warm-up)' if iters <= skip_batch_num else '' - _logger.info('batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' - 'latency: {3:.4f} ms, fps: {4:.2f}'.format( - iters, batch_acc1, batch_acc5, - batch_time / batch_size, fps, appx)) + _logger.info( + 'batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' + 'latency: {3:.4f} ms, fps: {4:.2f}'.format( + iters, + batch_acc1, + batch_acc5, + batch_time / batch_size, + fps, + appx, + ) + ) # Postprocess benchmark data batch_latencies = batch_times[skip_batch_num:] @@ -287,18 +307,24 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): acc1_avg = np.mean(infer_accs1) acc5_avg = np.mean(infer_accs5) _logger.info( - 'Total inference run time: {:.2f} s'.format(infer_total_time)) + 'Total inference run time: {:.2f} s'.format(infer_total_time) + ) return outputs, acc1_avg, acc5_avg, fps_avg, latency_avg def _print_performance(self, title, fps, lat): - _logger.info('{0}: avg fps: {1:.2f}, avg latency: {2:.4f} ms'.format( - title, fps, lat)) + _logger.info( + '{0}: avg fps: {1:.2f}, avg latency: {2:.4f} ms'.format( + title, fps, lat + ) + ) def _print_accuracy(self, title, acc1, acc5): _logger.info( - '{0}: avg top1 accuracy: {1:.4f}, avg top5 accuracy: {2:.4f}'. - format(title, acc1, acc5)) + '{0}: avg top1 accuracy: {1:.4f}, avg top5 accuracy: {2:.4f}'.format( + title, acc1, acc5 + ) + ) def _summarize_performance(self, int8_fps, int8_lat, fp32_fps, fp32_lat): _logger.info('--- Performance summary ---') @@ -306,8 +332,9 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): if fp32_lat >= 0: self._print_performance('FP32', fp32_fps, fp32_lat) - def _summarize_accuracy(self, quant_acc1, quant_acc5, int8_acc1, int8_acc5, - fp32_acc1, fp32_acc5): + def _summarize_accuracy( + self, quant_acc1, quant_acc5, int8_acc1, int8_acc5, fp32_acc1, fp32_acc5 + ): _logger.info('--- Accuracy summary ---') self._print_accuracy('Quant', quant_acc1, quant_acc5) self._print_accuracy('INT8', int8_acc1, int8_acc5) @@ -316,8 +343,10 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): def _compare_accuracy(self, threshold, quant_acc1, int8_acc1): _logger.info( - 'Accepted top1 accuracy drop threshold: {0}. (condition: (Quant_top1_acc - IN8_top1_acc) <= threshold && Quant_top1_acc > 0.5 && INT8_top1_acc > 0.5)' - .format(threshold)) + 'Accepted top1 accuracy drop threshold: {0}. (condition: (Quant_top1_acc - IN8_top1_acc) <= threshold && Quant_top1_acc > 0.5 && INT8_top1_acc > 0.5)'.format( + threshold + ) + ) # We assume valid accuracy to be at least 0.5 assert quant_acc1 > 0.5 assert int8_acc1 > 0.5 @@ -334,9 +363,13 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): return quant_model_path = test_case_args.quant_model - assert quant_model_path, 'The Quant model path cannot be empty. Please, use the --quant_model option.' + assert ( + quant_model_path + ), 'The Quant model path cannot be empty. Please, use the --quant_model option.' data_path = test_case_args.infer_data - assert data_path, 'The dataset path cannot be empty. Please, use the --infer_data option.' + assert ( + data_path + ), 'The dataset path cannot be empty. Please, use the --infer_data option.' fp32_model_path = test_case_args.fp32_model batch_size = test_case_args.batch_size batch_num = test_case_args.batch_num @@ -347,12 +380,14 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): self._quantized_ops = set() if test_case_args.ops_to_quantize: self._quantized_ops = self._strings_from_csv( - test_case_args.ops_to_quantize) + test_case_args.ops_to_quantize + ) self._op_ids_to_skip = set([-1]) if test_case_args.op_ids_to_skip: self._op_ids_to_skip = self._ints_from_csv( - test_case_args.op_ids_to_skip) + test_case_args.op_ids_to_skip + ) self._targets = self._strings_from_csv(test_case_args.targets) assert self._targets.intersection( @@ -368,61 +403,99 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): _logger.info('Batch number: {}'.format(batch_num)) _logger.info('Accuracy drop threshold: {}.'.format(acc_diff_threshold)) _logger.info( - 'Quantized ops: {}.'.format(','.join(self._quantized_ops) if self. - _quantized_ops else 'all quantizable')) - _logger.info('Op ids to skip quantization: {}.'.format( - ','.join(map(str, self._op_ids_to_skip) - ) if test_case_args.op_ids_to_skip else 'none')) + 'Quantized ops: {}.'.format( + ','.join(self._quantized_ops) + if self._quantized_ops + else 'all quantizable' + ) + ) + _logger.info( + 'Op ids to skip quantization: {}.'.format( + ','.join(map(str, self._op_ids_to_skip)) + if test_case_args.op_ids_to_skip + else 'none' + ) + ) _logger.info('Targets: {}.'.format(','.join(self._targets))) if 'quant' in self._targets: _logger.info('--- Quant prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) - quant_output, quant_acc1, quant_acc5, quant_fps, quant_lat = self._predict( + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) + ( + quant_output, + quant_acc1, + quant_acc5, + quant_fps, + quant_lat, + ) = self._predict( val_reader, quant_model_path, batch_size, batch_num, skip_batch_num, - target='quant') + target='quant', + ) self._print_performance('Quant', quant_fps, quant_lat) self._print_accuracy('Quant', quant_acc1, quant_acc5) if 'int8' in self._targets: _logger.info('--- INT8 prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) - int8_output, int8_acc1, int8_acc5, int8_fps, int8_lat = self._predict( + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) + ( + int8_output, + int8_acc1, + int8_acc5, + int8_fps, + int8_lat, + ) = self._predict( val_reader, quant_model_path, batch_size, batch_num, skip_batch_num, - target='int8') + target='int8', + ) self._print_performance('INT8', int8_fps, int8_lat) self._print_accuracy('INT8', int8_acc1, int8_acc5) fp32_acc1 = fp32_acc5 = fp32_fps = fp32_lat = -1 if 'fp32' in self._targets and fp32_model_path: _logger.info('--- FP32 prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) - fp32_output, fp32_acc1, fp32_acc5, fp32_fps, fp32_lat = self._predict( + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) + ( + fp32_output, + fp32_acc1, + fp32_acc5, + fp32_fps, + fp32_lat, + ) = self._predict( val_reader, fp32_model_path, batch_size, batch_num, skip_batch_num, - target='fp32') + target='fp32', + ) self._print_performance('FP32', fp32_fps, fp32_lat) self._print_accuracy('FP32', fp32_acc1, fp32_acc5) if {'int8', 'fp32'}.issubset(self._targets): self._summarize_performance(int8_fps, int8_lat, fp32_fps, fp32_lat) if {'int8', 'quant'}.issubset(self._targets): - self._summarize_accuracy(quant_acc1, quant_acc5, int8_acc1, - int8_acc5, fp32_acc1, fp32_acc5) + self._summarize_accuracy( + quant_acc1, + quant_acc5, + int8_acc1, + int8_acc5, + fp32_acc1, + fp32_acc5, + ) self._compare_accuracy(acc_diff_threshold, quant_acc1, int8_acc1) diff --git a/python/paddle/fluid/contrib/slim/tests/quant2_int8_lstm_model.py b/python/paddle/fluid/contrib/slim/tests/quant2_int8_lstm_model.py index 0a9abe61e0e..71bac0208e4 100644 --- a/python/paddle/fluid/contrib/slim/tests/quant2_int8_lstm_model.py +++ b/python/paddle/fluid/contrib/slim/tests/quant2_int8_lstm_model.py @@ -25,35 +25,33 @@ from save_quant_model import transform_and_save_int8_model def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--fp32_model', - type=str, - default='', - help='A path to a FP32 model.') - parser.add_argument('--quant_model', - type=str, - default='', - help='A path to a quant model.') + parser.add_argument( + '--fp32_model', type=str, default='', help='A path to a FP32 model.' + ) + parser.add_argument( + '--quant_model', type=str, default='', help='A path to a quant model.' + ) parser.add_argument('--infer_data', type=str, default='', help='Data file.') parser.add_argument( '--warmup_iter', type=int, default=1, - help='Number of the first iterations to skip in performance statistics.' + help='Number of the first iterations to skip in performance statistics.', + ) + parser.add_argument( + '--acc_diff_threshold', + type=float, + default=0.01, + help='Accepted accuracy difference threshold.', + ) + parser.add_argument( + '--num_threads', type=int, default=1, help='Number of threads.' ) - parser.add_argument('--acc_diff_threshold', - type=float, - default=0.01, - help='Accepted accuracy difference threshold.') - parser.add_argument('--num_threads', - type=int, - default=1, - help='Number of threads.') parser.add_argument( '--mkldnn_cache_capacity', type=int, default=0, - help= - 'Mkldnn cache capacity. The default value in Python API is 15, which can slow down int8 models. Default 0 means unlimited cache.' + help='Mkldnn cache capacity. The default value in Python API is 15, which can slow down int8 models. Default 0 means unlimited cache.', ) test_args, args = parser.parse_known_args(namespace=unittest) @@ -61,7 +59,6 @@ def parse_args(): class TestLstmModelPTQ(unittest.TestCase): - def get_warmup_tensor(self, data_path, place): data = [] with open(data_path, 'rb') as in_f: @@ -75,11 +72,13 @@ class TestLstmModelPTQ(unittest.TestCase): seq_len = (alllen >> 16) & 0xFFFF label = in_f.read(4 * label_len) - label = np.frombuffer(label, - dtype=np.int32).reshape([len(label) // 4]) + label = np.frombuffer(label, dtype=np.int32).reshape( + [len(label) // 4] + ) feat = in_f.read(4 * seq_len * 8) feat = np.frombuffer(feat, dtype=np.float32).reshape( - [len(feat) // 4 // 8, 8]) + [len(feat) // 4 // 8, 8] + ) lod_feat = [feat.shape[0]] minputs = fluid.create_lod_tensor(feat, [lod_feat], place) @@ -97,13 +96,15 @@ class TestLstmModelPTQ(unittest.TestCase): inputs = data[1:] return warmup_data, inputs - def set_config(self, - model_path, - num_threads, - mkldnn_cache_capacity, - warmup_data=None, - use_analysis=False, - enable_ptq=False): + def set_config( + self, + model_path, + num_threads, + mkldnn_cache_capacity, + warmup_data=None, + use_analysis=False, + enable_ptq=False, + ): config = AnalysisConfig(model_path) config.set_cpu_math_library_num_threads(num_threads) if use_analysis: @@ -120,19 +121,27 @@ class TestLstmModelPTQ(unittest.TestCase): config.quantizer_config().set_quant_batch_size(1) return config - def run_program(self, - model_path, - data_path, - num_threads, - mkldnn_cache_capacity, - warmup_iter, - use_analysis=False, - enable_ptq=False): + def run_program( + self, + model_path, + data_path, + num_threads, + mkldnn_cache_capacity, + warmup_iter, + use_analysis=False, + enable_ptq=False, + ): place = fluid.CPUPlace() warmup_data, inputs = self.get_warmup_tensor(data_path, place) warmup_data = [item[0] for item in warmup_data] - config = self.set_config(model_path, num_threads, mkldnn_cache_capacity, - warmup_data, use_analysis, enable_ptq) + config = self.set_config( + model_path, + num_threads, + mkldnn_cache_capacity, + warmup_data, + use_analysis, + enable_ptq, + ) predictor = create_paddle_predictor(config) data = [item[0] for item in inputs] @@ -164,13 +173,17 @@ class TestLstmModelPTQ(unittest.TestCase): if this_label_data[0] <= 6350: all_ctc_num += 1 - if np_ctc_out.shape[0] == 1 and np_ctc_out.all( - ) == this_label_data.all(): + if ( + np_ctc_out.shape[0] == 1 + and np_ctc_out.all() == this_label_data.all() + ): ok_ctc_num += 1 else: all_ctc_num += 1 - if np_ctc_out.shape[0] == this_label.shape[ - 0] and np_ctc_out.all() == this_label_data.all(): + if ( + np_ctc_out.shape[0] == this_label.shape[0] + and np_ctc_out.all() == this_label_data.all() + ): ok_ctc_num += 1 if all_ctc_num > 1000 or all_hz_num > 1000: @@ -187,44 +200,75 @@ class TestLstmModelPTQ(unittest.TestCase): return fp32_model = test_case_args.fp32_model - assert fp32_model, 'The FP32 model path cannot be empty. Please, use the --fp32_model option.' + assert ( + fp32_model + ), 'The FP32 model path cannot be empty. Please, use the --fp32_model option.' quant_model = test_case_args.quant_model - assert quant_model, 'The quant model path cannot be empty. Please, use the --quant_model option.' + assert ( + quant_model + ), 'The quant model path cannot be empty. Please, use the --quant_model option.' infer_data = test_case_args.infer_data - assert infer_data, 'The dataset path cannot be empty. Please, use the --infer_data option.' + assert ( + infer_data + ), 'The dataset path cannot be empty. Please, use the --infer_data option.' num_threads = test_case_args.num_threads mkldnn_cache_capacity = test_case_args.mkldnn_cache_capacity warmup_iter = test_case_args.warmup_iter acc_diff_threshold = test_case_args.acc_diff_threshold - (fp32_hx_acc, fp32_ctc_acc, - fp32_fps) = self.run_program(fp32_model, infer_data, num_threads, - mkldnn_cache_capacity, warmup_iter, False, - False) - - (int8_hx_acc, int8_ctc_acc, - int8_fps) = self.run_program(fp32_model, infer_data, num_threads, - mkldnn_cache_capacity, warmup_iter, True, - True) + (fp32_hx_acc, fp32_ctc_acc, fp32_fps) = self.run_program( + fp32_model, + infer_data, + num_threads, + mkldnn_cache_capacity, + warmup_iter, + False, + False, + ) + + (int8_hx_acc, int8_ctc_acc, int8_fps) = self.run_program( + fp32_model, + infer_data, + num_threads, + mkldnn_cache_capacity, + warmup_iter, + True, + True, + ) quant_model_save_path = quant_model + "_int8" # transform model to quant2 - transform_and_save_int8_model(quant_model, quant_model_save_path, - "fusion_lstm,concat") - - (quant_hx_acc, quant_ctc_acc, - quant_fps) = self.run_program(quant_model_save_path, infer_data, - num_threads, mkldnn_cache_capacity, - warmup_iter, True, False) - - print("FP32: fps {0}, hx_acc {1}, ctc_acc {2}".format( - fp32_fps, fp32_hx_acc, fp32_ctc_acc)) - - print("PTQ_INT8: fps {0}, hx_acc {1}, ctc_acc {2}".format( - int8_fps, int8_hx_acc, int8_ctc_acc)) - - print("QUANT2_INT8: fps {0}, hx_acc {1}, ctc_acc {2}".format( - quant_fps, quant_hx_acc, quant_ctc_acc)) + transform_and_save_int8_model( + quant_model, quant_model_save_path, "fusion_lstm,concat" + ) + + (quant_hx_acc, quant_ctc_acc, quant_fps) = self.run_program( + quant_model_save_path, + infer_data, + num_threads, + mkldnn_cache_capacity, + warmup_iter, + True, + False, + ) + + print( + "FP32: fps {0}, hx_acc {1}, ctc_acc {2}".format( + fp32_fps, fp32_hx_acc, fp32_ctc_acc + ) + ) + + print( + "PTQ_INT8: fps {0}, hx_acc {1}, ctc_acc {2}".format( + int8_fps, int8_hx_acc, int8_ctc_acc + ) + ) + + print( + "QUANT2_INT8: fps {0}, hx_acc {1}, ctc_acc {2}".format( + quant_fps, quant_hx_acc, quant_ctc_acc + ) + ) sys.stdout.flush() diff --git a/python/paddle/fluid/contrib/slim/tests/quant2_int8_nlp_comparison.py b/python/paddle/fluid/contrib/slim/tests/quant2_int8_nlp_comparison.py index fecead6d6de..3b997fa0d50 100644 --- a/python/paddle/fluid/contrib/slim/tests/quant2_int8_nlp_comparison.py +++ b/python/paddle/fluid/contrib/slim/tests/quant2_int8_nlp_comparison.py @@ -39,57 +39,56 @@ def parse_args(): '--skip_batch_num', type=int, default=0, - help='Number of the first minibatches to skip in performance statistics.' + help='Number of the first minibatches to skip in performance statistics.', + ) + parser.add_argument( + '--quant_model', type=str, default='', help='A path to a Quant model.' ) - parser.add_argument('--quant_model', - type=str, - default='', - help='A path to a Quant model.') parser.add_argument( '--fp32_model', type=str, default='', - help= - 'A path to an FP32 model. If empty, the Quant model will be used for FP32 inference.' + help='A path to an FP32 model. If empty, the Quant model will be used for FP32 inference.', ) parser.add_argument('--infer_data', type=str, default='', help='Data file.') - parser.add_argument('--labels', - type=str, - default='', - help='File with labels.') + parser.add_argument( + '--labels', type=str, default='', help='File with labels.' + ) parser.add_argument( '--batch_num', type=int, default=0, - help= - 'Number of batches to process. 0 or less means whole dataset. Default: 0.' + help='Number of batches to process. 0 or less means whole dataset. Default: 0.', + ) + parser.add_argument( + '--acc_diff_threshold', + type=float, + default=0.01, + help='Accepted accuracy difference threshold.', ) - parser.add_argument('--acc_diff_threshold', - type=float, - default=0.01, - help='Accepted accuracy difference threshold.') parser.add_argument( '--ops_to_quantize', type=str, default='', - help= - 'A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.' + help='A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.', ) parser.add_argument( '--op_ids_to_skip', type=str, default='', - help='A comma separated list of operator ids to skip in quantization.') + help='A comma separated list of operator ids to skip in quantization.', + ) parser.add_argument( '--targets', type=str, default='quant,int8,fp32', - help= - 'A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"' + help='A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"', + ) + parser.add_argument( + '--debug', + action='store_true', + help='If used, the graph of Quant model is drawn.', ) - parser.add_argument('--debug', - action='store_true', - help='If used, the graph of Quant model is drawn.') test_args, args = parser.parse_known_args(namespace=unittest) @@ -116,16 +115,16 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): for i in range(len(data_lines)): data_fields = data_lines[i].split(';') - assert len( - data_fields - ) >= 2, "The number of data fields in the dataset is less than 2" + assert ( + len(data_fields) >= 2 + ), "The number of data fields in the dataset is less than 2" buffers = [] shape = [] for j in range(2): data = data_fields[j].split(':') - assert len( - data - ) >= 2, "Size of data in the dataset is less than 2" + assert ( + len(data) >= 2 + ), "Size of data in the dataset is less than 2" # Shape is stored under index 0, while data under 1 shape = data[0].split() shape.pop(0) @@ -149,42 +148,53 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): correct += 1 return correct - def _predict(self, - test_reader=None, - model_path=None, - batch_size=1, - batch_num=1, - skip_batch_num=0, - target='quant'): + def _predict( + self, + test_reader=None, + model_path=None, + batch_size=1, + batch_num=1, + skip_batch_num=0, + target='quant', + ): assert target in ['quant', 'int8', 'fp32'] place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, exe, 'model', 'params' + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) - if (self._debug): + if self._debug: graph.draw('.', 'quant_orig', graph.all_op_nodes()) - if (target != 'quant'): + if target != 'quant': quant_transform_pass = Quant2Int8MkldnnPass( self._quantized_ops, _op_ids_to_skip=self._op_ids_to_skip, _scope=inference_scope, _place=place, _core=core, - _debug=self._debug) - if (target == 'int8'): + _debug=self._debug, + ) + if target == 'int8': graph = quant_transform_pass.apply(graph) else: # target == fp32 graph = quant_transform_pass.prepare_and_optimize_fp32( - graph) + graph + ) inference_program = graph.to_program() @@ -205,12 +215,14 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): labels = np.array([x[2] for x in data]).astype('int64') start = time.time() - out = exe.run(inference_program, - feed={ - feed_target_names[0]: input0, - feed_target_names[1]: input1 - }, - fetch_list=fetch_targets) + out = exe.run( + inference_program, + feed={ + feed_target_names[0]: input0, + feed_target_names[1]: input1, + }, + fetch_list=fetch_targets, + ) batch_time = (time.time() - start) * 1000 # in miliseconds batch_times.append(batch_time) batch_correct = self._get_batch_correct(out, labels) @@ -224,8 +236,10 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): iters += 1 appx = ' (warm-up)' if iters <= skip_batch_num else '' _logger.info( - 'batch {0}{4}, acc: {1:.4f}, latency: {2:.4f} ms, predictions per sec: {3:.2f}' - .format(iters, batch_acc, latency, pps, appx)) + 'batch {0}{4}, acc: {1:.4f}, latency: {2:.4f} ms, predictions per sec: {3:.2f}'.format( + iters, batch_acc, latency, pps, appx + ) + ) # Postprocess benchmark data infer_total_time = time.time() - infer_start_time @@ -236,14 +250,17 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): pps_avg = np.average(ppses) acc_avg = float(np.sum(total_correct)) / float(total_samples) _logger.info( - 'Total inference run time: {:.2f} s'.format(infer_total_time)) + 'Total inference run time: {:.2f} s'.format(infer_total_time) + ) return acc_avg, pps_avg, latency_avg def _print_performance(self, title, pps, lat): _logger.info( - '{0}: avg predictions per sec: {1:.2f}, avg latency: {2:.4f} ms'. - format(title, pps, lat)) + '{0}: avg predictions per sec: {1:.2f}, avg latency: {2:.4f} ms'.format( + title, pps, lat + ) + ) def _print_accuracy(self, title, acc): _logger.info('{0}: avg accuracy: {1:.6f}'.format(title, acc)) @@ -263,8 +280,10 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): def _compare_accuracy(self, threshold, quant_acc, int8_acc): _logger.info( - 'Accepted accuracy drop threshold: {0}. (condition: (Quant_acc - INT8_acc) <= threshold)' - .format(threshold)) + 'Accepted accuracy drop threshold: {0}. (condition: (Quant_acc - INT8_acc) <= threshold)'.format( + threshold + ) + ) # Random outputs give accuracy about 0.33, we assume valid accuracy to be at least 0.5 assert quant_acc > 0.5 assert int8_acc > 0.5 @@ -281,9 +300,13 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): return quant_model_path = test_case_args.quant_model - assert quant_model_path, 'The Quant model path cannot be empty. Please, use the --quant_model option.' + assert ( + quant_model_path + ), 'The Quant model path cannot be empty. Please, use the --quant_model option.' data_path = test_case_args.infer_data - assert data_path, 'The dataset path cannot be empty. Please, use the --infer_data option.' + assert ( + data_path + ), 'The dataset path cannot be empty. Please, use the --infer_data option.' fp32_model_path = test_case_args.fp32_model labels_path = test_case_args.labels batch_size = test_case_args.batch_size @@ -295,12 +318,14 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): self._quantized_ops = set() if test_case_args.ops_to_quantize: self._quantized_ops = self._strings_from_csv( - test_case_args.ops_to_quantize) + test_case_args.ops_to_quantize + ) self._op_ids_to_skip = set([-1]) if test_case_args.op_ids_to_skip: self._op_ids_to_skip = self._ints_from_csv( - test_case_args.op_ids_to_skip) + test_case_args.op_ids_to_skip + ) self._targets = self._strings_from_csv(test_case_args.targets) assert self._targets.intersection( @@ -317,53 +342,70 @@ class QuantInt8NLPComparisonTest(unittest.TestCase): _logger.info('Batch number: {}'.format(batch_num)) _logger.info('Accuracy drop threshold: {}.'.format(acc_diff_threshold)) _logger.info( - 'Quantized ops: {}.'.format(','.join(self._quantized_ops) if self. - _quantized_ops else 'all quantizable')) - _logger.info('Op ids to skip quantization: {}.'.format( - ','.join(map(str, self._op_ids_to_skip) - ) if test_case_args.op_ids_to_skip else 'none')) + 'Quantized ops: {}.'.format( + ','.join(self._quantized_ops) + if self._quantized_ops + else 'all quantizable' + ) + ) + _logger.info( + 'Op ids to skip quantization: {}.'.format( + ','.join(map(str, self._op_ids_to_skip)) + if test_case_args.op_ids_to_skip + else 'none' + ) + ) _logger.info('Targets: {}.'.format(','.join(self._targets))) if 'quant' in self._targets: _logger.info('--- Quant prediction start ---') - val_reader = paddle.batch(self._reader_creator( - data_path, labels_path), - batch_size=batch_size) - quant_acc, quant_pps, quant_lat = self._predict(val_reader, - quant_model_path, - batch_size, - batch_num, - skip_batch_num, - target='quant') + val_reader = paddle.batch( + self._reader_creator(data_path, labels_path), + batch_size=batch_size, + ) + quant_acc, quant_pps, quant_lat = self._predict( + val_reader, + quant_model_path, + batch_size, + batch_num, + skip_batch_num, + target='quant', + ) self._print_performance('Quant', quant_pps, quant_lat) self._print_accuracy('Quant', quant_acc) if 'int8' in self._targets: _logger.info('--- INT8 prediction start ---') - val_reader = paddle.batch(self._reader_creator( - data_path, labels_path), - batch_size=batch_size) - int8_acc, int8_pps, int8_lat = self._predict(val_reader, - quant_model_path, - batch_size, - batch_num, - skip_batch_num, - target='int8') + val_reader = paddle.batch( + self._reader_creator(data_path, labels_path), + batch_size=batch_size, + ) + int8_acc, int8_pps, int8_lat = self._predict( + val_reader, + quant_model_path, + batch_size, + batch_num, + skip_batch_num, + target='int8', + ) self._print_performance('INT8', int8_pps, int8_lat) self._print_accuracy('INT8', int8_acc) fp32_acc = fp32_pps = fp32_lat = -1 if 'fp32' in self._targets and fp32_model_path: _logger.info('--- FP32 prediction start ---') - val_reader = paddle.batch(self._reader_creator( - data_path, labels_path), - batch_size=batch_size) - fp32_acc, fp32_pps, fp32_lat = self._predict(val_reader, - fp32_model_path, - batch_size, - batch_num, - skip_batch_num, - target='fp32') + val_reader = paddle.batch( + self._reader_creator(data_path, labels_path), + batch_size=batch_size, + ) + fp32_acc, fp32_pps, fp32_lat = self._predict( + val_reader, + fp32_model_path, + batch_size, + batch_num, + skip_batch_num, + target='fp32', + ) self._print_performance('FP32', fp32_pps, fp32_lat) self._print_accuracy('FP32', fp32_acc) diff --git a/python/paddle/fluid/contrib/slim/tests/quant_int8_image_classification_comparison.py b/python/paddle/fluid/contrib/slim/tests/quant_int8_image_classification_comparison.py index ec88664c2ed..e3aecd48c34 100644 --- a/python/paddle/fluid/contrib/slim/tests/quant_int8_image_classification_comparison.py +++ b/python/paddle/fluid/contrib/slim/tests/quant_int8_image_classification_comparison.py @@ -40,27 +40,29 @@ def parse_args(): '--skip_batch_num', type=int, default=0, - help='Number of the first minibatches to skip in performance statistics.' + help='Number of the first minibatches to skip in performance statistics.', + ) + parser.add_argument( + '--debug', + action='store_true', + help='If used, the graph of Quant model is drawn.', + ) + parser.add_argument( + '--quant_model', type=str, default='', help='A path to a Quant model.' ) - parser.add_argument('--debug', - action='store_true', - help='If used, the graph of Quant model is drawn.') - parser.add_argument('--quant_model', - type=str, - default='', - help='A path to a Quant model.') parser.add_argument('--infer_data', type=str, default='', help='Data file.') parser.add_argument( '--batch_num', type=int, default=0, - help= - 'Number of batches to process. 0 or less means whole dataset. Default: 0.' + help='Number of batches to process. 0 or less means whole dataset. Default: 0.', + ) + parser.add_argument( + '--acc_diff_threshold', + type=float, + default=0.01, + help='Accepted accuracy difference threshold.', ) - parser.add_argument('--acc_diff_threshold', - type=float, - default=0.01, - help='Accepted accuracy difference threshold.') test_args, args = parser.parse_known_args(namespace=unittest) return test_args, sys.argv[:1] + args @@ -72,7 +74,6 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): """ def _reader_creator(self, data_file='data.bin'): - def reader(): with open(data_file, 'rb') as fp: num = fp.read(8) @@ -91,7 +92,8 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): fp.seek(imgs_offset + img_size * step) img = fp.read(img_size) img = struct.unpack_from( - '{}f'.format(img_ch * img_w * img_h), img) + '{}f'.format(img_ch * img_w * img_h), img + ) img = np.array(img) img.shape = (img_ch, img_w, img_h) fp.seek(labels_offset + label_size * step) @@ -125,14 +127,14 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): name = op_node.name() if name in ['depthwise_conv2d']: input_var_node = graph._find_node_by_name( - op_node.inputs, - op_node.input("Input")[0]) + op_node.inputs, op_node.input("Input")[0] + ) weight_var_node = graph._find_node_by_name( - op_node.inputs, - op_node.input("Filter")[0]) + op_node.inputs, op_node.input("Filter")[0] + ) output_var_node = graph._find_node_by_name( - graph.all_var_nodes(), - op_node.output("Output")[0]) + graph.all_var_nodes(), op_node.output("Output")[0] + ) attrs = { name: op_node.op().attr(name) for name in op_node.op().attr_names() @@ -141,11 +143,9 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): conv_op_node = graph.create_op_node( op_type='conv2d', attrs=attrs, - inputs={ - 'Input': input_var_node, - 'Filter': weight_var_node - }, - outputs={'Output': output_var_node}) + inputs={'Input': input_var_node, 'Filter': weight_var_node}, + outputs={'Output': output_var_node}, + ) graph.link_to(input_var_node, conv_op_node) graph.link_to(weight_var_node, conv_op_node) @@ -154,31 +154,41 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): return graph - def _predict(self, - test_reader=None, - model_path=None, - batch_size=1, - batch_num=1, - skip_batch_num=0, - transform_to_int8=False): + def _predict( + self, + test_reader=None, + model_path=None, + batch_size=1, + batch_num=1, + skip_batch_num=0, + transform_to_int8=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, exe, 'model', 'params' + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) - if (self._debug): + if self._debug: graph.draw('.', 'quant_orig', graph.all_op_nodes()) - if (transform_to_int8): - mkldnn_int8_pass = QuantInt8MkldnnPass(_scope=inference_scope, - _place=place) + if transform_to_int8: + mkldnn_int8_pass = QuantInt8MkldnnPass( + _scope=inference_scope, _place=place + ) graph = mkldnn_int8_pass.apply(graph) else: graph = self._prepare_for_fp32_mkldnn(graph) @@ -205,13 +215,16 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): labels = np.array([x[1] for x in data]).astype('int64') start = time.time() - out = exe.run(inference_program, - feed={feed_target_names[0]: images}, - fetch_list=fetch_targets) + out = exe.run( + inference_program, + feed={feed_target_names[0]: images}, + fetch_list=fetch_targets, + ) batch_time = (time.time() - start) * 1000 # in miliseconds outputs.append(out[0]) batch_acc1, batch_acc5 = self._get_batch_accuracy( - out[0], labels) + out[0], labels + ) infer_accs1.append(batch_acc1) infer_accs5.append(batch_acc5) samples = len(data) @@ -221,10 +234,17 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): fpses.append(fps) iters += 1 appx = ' (warm-up)' if iters <= skip_batch_num else '' - _logger.info('batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' - 'latency: {3:.4f} ms, fps: {4:.2f}'.format( - iters, batch_acc1, batch_acc5, - batch_time / batch_size, fps, appx)) + _logger.info( + 'batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' + 'latency: {3:.4f} ms, fps: {4:.2f}'.format( + iters, + batch_acc1, + batch_acc5, + batch_time / batch_size, + fps, + appx, + ) + ) # Postprocess benchmark data batch_latencies = batch_times[skip_batch_num:] @@ -236,29 +256,43 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): acc1_avg = np.mean(infer_accs1) acc5_avg = np.mean(infer_accs5) _logger.info( - 'Total inference run time: {:.2f} s'.format(infer_total_time)) + 'Total inference run time: {:.2f} s'.format(infer_total_time) + ) return outputs, acc1_avg, acc5_avg, fps_avg, latency_avg def _summarize_performance(self, fp32_fps, fp32_lat, int8_fps, int8_lat): _logger.info('--- Performance summary ---') - _logger.info('FP32: avg fps: {0:.2f}, avg latency: {1:.4f} ms'.format( - fp32_fps, fp32_lat)) - _logger.info('INT8: avg fps: {0:.2f}, avg latency: {1:.4f} ms'.format( - int8_fps, int8_lat)) - - def _compare_accuracy(self, fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, - threshold): + _logger.info( + 'FP32: avg fps: {0:.2f}, avg latency: {1:.4f} ms'.format( + fp32_fps, fp32_lat + ) + ) + _logger.info( + 'INT8: avg fps: {0:.2f}, avg latency: {1:.4f} ms'.format( + int8_fps, int8_lat + ) + ) + + def _compare_accuracy( + self, fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, threshold + ): _logger.info('--- Accuracy summary ---') _logger.info( - 'Accepted top1 accuracy drop threshold: {0}. (condition: (FP32_top1_acc - IN8_top1_acc) <= threshold)' - .format(threshold)) + 'Accepted top1 accuracy drop threshold: {0}. (condition: (FP32_top1_acc - IN8_top1_acc) <= threshold)'.format( + threshold + ) + ) _logger.info( - 'FP32: avg top1 accuracy: {0:.4f}, avg top5 accuracy: {1:.4f}'. - format(fp32_acc1, fp32_acc5)) + 'FP32: avg top1 accuracy: {0:.4f}, avg top5 accuracy: {1:.4f}'.format( + fp32_acc1, fp32_acc5 + ) + ) _logger.info( - 'INT8: avg top1 accuracy: {0:.4f}, avg top5 accuracy: {1:.4f}'. - format(int8_acc1, int8_acc5)) + 'INT8: avg top1 accuracy: {0:.4f}, avg top5 accuracy: {1:.4f}'.format( + int8_acc1, int8_acc5 + ) + ) assert fp32_acc1 > 0.0 assert int8_acc1 > 0.0 assert fp32_acc1 - int8_acc1 <= threshold @@ -268,9 +302,13 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): return quant_model_path = test_case_args.quant_model - assert quant_model_path, 'The Quant model path cannot be empty. Please, use the --quant_model option.' + assert ( + quant_model_path + ), 'The Quant model path cannot be empty. Please, use the --quant_model option.' data_path = test_case_args.infer_data - assert data_path, 'The dataset path cannot be empty. Please, use the --infer_data option.' + assert ( + data_path + ), 'The dataset path cannot be empty. Please, use the --infer_data option.' batch_size = test_case_args.batch_size batch_num = test_case_args.batch_num skip_batch_num = test_case_args.skip_batch_num @@ -285,29 +323,34 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): _logger.info('Accuracy drop threshold: {0}.'.format(acc_diff_threshold)) _logger.info('--- Quant FP32 prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) fp32_output, fp32_acc1, fp32_acc5, fp32_fps, fp32_lat = self._predict( val_reader, quant_model_path, batch_size, batch_num, skip_batch_num, - transform_to_int8=False) + transform_to_int8=False, + ) _logger.info('--- Quant INT8 prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) int8_output, int8_acc1, int8_acc5, int8_fps, int8_lat = self._predict( val_reader, quant_model_path, batch_size, batch_num, skip_batch_num, - transform_to_int8=True) + transform_to_int8=True, + ) self._summarize_performance(fp32_fps, fp32_lat, int8_fps, int8_lat) - self._compare_accuracy(fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, - acc_diff_threshold) + self._compare_accuracy( + fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, acc_diff_threshold + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/save_quant_model.py b/python/paddle/fluid/contrib/slim/tests/save_quant_model.py index cb15b3da473..b743615f575 100644 --- a/python/paddle/fluid/contrib/slim/tests/save_quant_model.py +++ b/python/paddle/fluid/contrib/slim/tests/save_quant_model.py @@ -27,88 +27,102 @@ paddle.enable_static() def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--quant_model_path', - type=str, - default='', - help='A path to a Quant model.') - parser.add_argument('--int8_model_save_path', - type=str, - default='', - help='Saved optimized and quantized INT8 model') + parser.add_argument( + '--quant_model_path', + type=str, + default='', + help='A path to a Quant model.', + ) + parser.add_argument( + '--int8_model_save_path', + type=str, + default='', + help='Saved optimized and quantized INT8 model', + ) parser.add_argument( '--ops_to_quantize', type=str, default='', - help= - 'A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.' + help='A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.', ) parser.add_argument( '--op_ids_to_skip', type=str, default='', - help='A comma separated list of operator ids to skip in quantization.') - parser.add_argument('--debug', - action='store_true', - help='If used, the graph of Quant model is drawn.') + help='A comma separated list of operator ids to skip in quantization.', + ) + parser.add_argument( + '--debug', + action='store_true', + help='If used, the graph of Quant model is drawn.', + ) parser.add_argument( '--quant_model_filename', type=str, default="", - help= - 'The input model`s file name. If empty, search default `__model__` and separate parameter files and use them or in case if not found, attempt loading `model` and `params` files.' + help='The input model`s file name. If empty, search default `__model__` and separate parameter files and use them or in case if not found, attempt loading `model` and `params` files.', ) parser.add_argument( '--quant_params_filename', type=str, default="", - help= - 'If quant_model_filename is empty, this field is ignored. The input model`s all parameters file name. If empty load parameters from separate files.' + help='If quant_model_filename is empty, this field is ignored. The input model`s all parameters file name. If empty load parameters from separate files.', ) parser.add_argument( '--save_model_filename', type=str, default="__model__", - help= - 'The name of file to save the inference program itself. If is set None, a default filename __model__ will be used.' + help='The name of file to save the inference program itself. If is set None, a default filename __model__ will be used.', ) parser.add_argument( '--save_params_filename', type=str, default=None, - help= - 'The name of file to save all related parameters. If it is set None, parameters will be saved in separate files' + help='The name of file to save all related parameters. If it is set None, parameters will be saved in separate files', ) test_args, args = parser.parse_known_args(namespace=unittest) return test_args, sys.argv[:1] + args -def transform_and_save_int8_model(original_path, - save_path, - ops_to_quantize='', - op_ids_to_skip='', - debug=False, - quant_model_filename='', - quant_params_filename='', - save_model_filename="__model__", - save_params_filename=None): +def transform_and_save_int8_model( + original_path, + save_path, + ops_to_quantize='', + op_ids_to_skip='', + debug=False, + quant_model_filename='', + quant_params_filename='', + save_model_filename="__model__", + save_params_filename=None, +): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if not quant_model_filename: if os.path.exists(os.path.join(original_path, '__model__')): - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(original_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(original_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(original_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + original_path, exe, 'model', 'params' + ) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(original_path, exe, - quant_model_filename, - quant_params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + original_path, exe, quant_model_filename, quant_params_filename + ) ops_to_quantize_set = set() print(ops_to_quantize) @@ -121,7 +135,7 @@ def transform_and_save_int8_model(original_path, op_ids_to_skip_set = set(map(int, op_ids_to_skip.split(','))) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) - if (debug): + if debug: graph.draw('.', 'quant_orig', graph.all_op_nodes()) transform_to_mkldnn_int8_pass = Quant2Int8MkldnnPass( ops_to_quantize_set, @@ -129,27 +143,38 @@ def transform_and_save_int8_model(original_path, _scope=inference_scope, _place=place, _core=core, - _debug=debug) + _debug=debug, + ) graph = transform_to_mkldnn_int8_pass.apply(graph) inference_program = graph.to_program() with fluid.scope_guard(inference_scope): - fluid.io.save_inference_model(save_path, - feed_target_names, - fetch_targets, - exe, - inference_program, - model_filename=save_model_filename, - params_filename=save_params_filename) + fluid.io.save_inference_model( + save_path, + feed_target_names, + fetch_targets, + exe, + inference_program, + model_filename=save_model_filename, + params_filename=save_params_filename, + ) print( - "Success! INT8 model obtained from the Quant model can be found at {}\n" - .format(save_path)) + "Success! INT8 model obtained from the Quant model can be found at {}\n".format( + save_path + ) + ) if __name__ == '__main__': global test_args test_args, remaining_args = parse_args() transform_and_save_int8_model( - test_args.quant_model_path, test_args.int8_model_save_path, - test_args.ops_to_quantize, test_args.op_ids_to_skip, test_args.debug, - test_args.quant_model_filename, test_args.quant_params_filename, - test_args.save_model_filename, test_args.save_params_filename) + test_args.quant_model_path, + test_args.int8_model_save_path, + test_args.ops_to_quantize, + test_args.op_ids_to_skip, + test_args.debug, + test_args.quant_model_filename, + test_args.quant_params_filename, + test_args.save_model_filename, + test_args.save_params_filename, + ) diff --git a/python/paddle/fluid/contrib/slim/tests/test_graph.py b/python/paddle/fluid/contrib/slim/tests/test_graph.py index dd08eb1cbde..482c7237bfc 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_graph.py +++ b/python/paddle/fluid/contrib/slim/tests/test_graph.py @@ -29,19 +29,23 @@ os.environ["CPU_NUM"] = "1" def conv_block(): img = fluid.layers.data(name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) avg_loss = paddle.mean(loss) @@ -49,7 +53,6 @@ def conv_block(): class TestGraph(unittest.TestCase): - def graph_apis(self, use_cuda=False, for_ci=True): main = fluid.Program() startup = fluid.Program() @@ -65,25 +68,27 @@ class TestGraph(unittest.TestCase): build_strategy.memory_optimize = False build_strategy.enable_inplace = False origin_binary = fluid.CompiledProgram(graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) backup_binary = fluid.CompiledProgram( - backup_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + backup_graph.graph + ).with_data_parallel(loss_name=loss.name, build_strategy=build_strategy) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) def _train(binary): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss.name]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss.name] + ) if not for_ci: print('{}: {}'.format('loss', loss_v)) @@ -98,18 +103,18 @@ class TestGraph(unittest.TestCase): var.set(var_array, place) sum_before = np.sum( - np.array( - fluid.global_scope().find_var('conv2d_1.w_0').get_tensor())) + np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor()) + ) fluid.io._save_persistable_nodes(exe, checkponit_dir, graph) _set_zero('conv2d_1.w_0', fluid.global_scope(), place) set_after = np.sum( - np.array( - fluid.global_scope().find_var('conv2d_1.w_0').get_tensor())) + np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor()) + ) self.assertEqual(set_after, 0) fluid.io._load_persistable_nodes(exe, checkponit_dir, graph) sum_after = np.sum( - np.array( - fluid.global_scope().find_var('conv2d_1.w_0').get_tensor())) + np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor()) + ) self.assertEqual(sum_before, sum_after) marked_nodes = set() diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py index 743e45e927a..5d990d4a986 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py @@ -42,9 +42,9 @@ os.environ["CPU_NUM"] = "1" if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def get_vaild_warning_num(warning, w): @@ -56,7 +56,6 @@ def get_vaild_warning_num(warning, w): class ImperativeLenet(fluid.dygraph.Layer): - def __init__(self, num_classes=10): super(ImperativeLenet, self).__init__() conv2d_w1_attr = fluid.ParamAttr(name="conv2d_w_1") @@ -69,36 +68,55 @@ class ImperativeLenet(fluid.dygraph.Layer): fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") self.features = Sequential( - Conv2D(in_channels=1, - out_channels=6, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2d_w1_attr, - bias_attr=False), BatchNorm2D(6), ReLU(), + Conv2D( + in_channels=1, + out_channels=6, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2d_w1_attr, + bias_attr=False, + ), + BatchNorm2D(6), + ReLU(), + MaxPool2D(kernel_size=2, stride=2), + Conv2D( + in_channels=6, + out_channels=16, + kernel_size=5, + stride=1, + padding=0, + weight_attr=conv2d_w2_attr, + bias_attr=conv2d_b2_attr, + ), + BatchNorm2D(16), + PReLU(), MaxPool2D(kernel_size=2, stride=2), - Conv2D(in_channels=6, - out_channels=16, - kernel_size=5, - stride=1, - padding=0, - weight_attr=conv2d_w2_attr, - bias_attr=conv2d_b2_attr), BatchNorm2D(16), PReLU(), - MaxPool2D(kernel_size=2, stride=2)) + ) self.fc = Sequential( - Linear(in_features=400, - out_features=120, - weight_attr=fc_w1_attr, - bias_attr=fc_b1_attr), LeakyReLU(), - Linear(in_features=120, - out_features=84, - weight_attr=fc_w2_attr, - bias_attr=fc_b2_attr), Sigmoid(), - Linear(in_features=84, - out_features=num_classes, - weight_attr=fc_w3_attr, - bias_attr=fc_b3_attr), Softmax()) + Linear( + in_features=400, + out_features=120, + weight_attr=fc_w1_attr, + bias_attr=fc_b1_attr, + ), + LeakyReLU(), + Linear( + in_features=120, + out_features=84, + weight_attr=fc_w2_attr, + bias_attr=fc_b2_attr, + ), + Sigmoid(), + Linear( + in_features=84, + out_features=num_classes, + weight_attr=fc_w3_attr, + bias_attr=fc_b3_attr, + ), + Softmax(), + ) def forward(self, inputs): x = self.features(inputs) @@ -109,13 +127,14 @@ class ImperativeLenet(fluid.dygraph.Layer): class TestImperativeOutSclae(unittest.TestCase): - def setUp(self): self.root_path = tempfile.TemporaryDirectory() - self.param_save_path = os.path.join(self.root_path.name, - "lenet.pdparams") - self.save_path = os.path.join(self.root_path.name, - "lenet_dynamic_outscale_infer_model") + self.param_save_path = os.path.join( + self.root_path.name, "lenet.pdparams" + ) + self.save_path = os.path.join( + self.root_path.name, "lenet_dynamic_outscale_infer_model" + ) def tearDown(self): self.root_path.cleanup() @@ -128,7 +147,8 @@ class TestImperativeOutSclae(unittest.TestCase): activation_quantize_type = 'moving_average_abs_max' imperative_out_scale = ImperativeQuantAware( weight_quantize_type=weight_quantize_type, - activation_quantize_type=activation_quantize_type) + activation_quantize_type=activation_quantize_type, + ) with fluid.dygraph.guard(): np.random.seed(seed) @@ -139,11 +159,12 @@ class TestImperativeOutSclae(unittest.TestCase): lenet = fix_model_dict(lenet) imperative_out_scale.quantize(lenet) - reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=32, - drop_last=True) - adam = AdamOptimizer(learning_rate=lr, - parameter_list=lenet.parameters()) + reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=32, drop_last=True + ) + adam = AdamOptimizer( + learning_rate=lr, parameter_list=lenet.parameters() + ) loss_list = train_lenet(lenet, reader, adam) lenet.eval() @@ -151,8 +172,10 @@ class TestImperativeOutSclae(unittest.TestCase): paddle.save(save_dict, self.param_save_path) for i in range(len(loss_list) - 1): - self.assertTrue(loss_list[i] > loss_list[i + 1], - msg='Failed to do the imperative qat.') + self.assertTrue( + loss_list[i] > loss_list[i + 1], + msg='Failed to do the imperative qat.', + ) with fluid.dygraph.guard(): lenet = ImperativeLenet() @@ -160,11 +183,12 @@ class TestImperativeOutSclae(unittest.TestCase): imperative_out_scale.quantize(lenet) lenet.set_dict(load_dict) - reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=32, - drop_last=True) - adam = AdamOptimizer(learning_rate=lr, - parameter_list=lenet.parameters()) + reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=32, drop_last=True + ) + adam = AdamOptimizer( + learning_rate=lr, parameter_list=lenet.parameters() + ) loss_list = train_lenet(lenet, reader, adam) lenet.eval() @@ -172,13 +196,17 @@ class TestImperativeOutSclae(unittest.TestCase): layer=lenet, path=self.save_path, input_spec=[ - paddle.static.InputSpec(shape=[None, 1, 28, 28], - dtype='float32') - ]) + paddle.static.InputSpec( + shape=[None, 1, 28, 28], dtype='float32' + ) + ], + ) for i in range(len(loss_list) - 1): - self.assertTrue(loss_list[i] > loss_list[i + 1], - msg='Failed to do the imperative qat.') + self.assertTrue( + loss_list[i] > loss_list[i + 1], + msg='Failed to do the imperative qat.', + ) def test_out_scale_acc(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_ptq.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_ptq.py index 6727356b327..759e74907e1 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_ptq.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_ptq.py @@ -30,12 +30,16 @@ from paddle.fluid.log_helper import get_logger from paddle.dataset.common import download from paddle.fluid.framework import _test_eager_guard -from imperative_test_utils import fix_model_dict, ImperativeLenet, ImperativeLinearBn +from imperative_test_utils import ( + fix_model_dict, + ImperativeLenet, + ImperativeLinearBn, +) from imperative_test_utils import ImperativeLinearBn_hook -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestFuseLinearBn(unittest.TestCase): @@ -54,27 +58,30 @@ class TestFuseLinearBn(unittest.TestCase): quant_h = ptq.quantize(model_h, fuse=True, fuse_list=f_l) for name, layer in quant_model.named_sublayers(): if name in f_l: - assert not (isinstance(layer, nn.BatchNorm1D) - or isinstance(layer, nn.BatchNorm2D)) + assert not ( + isinstance(layer, nn.BatchNorm1D) + or isinstance(layer, nn.BatchNorm2D) + ) out = model(inputs) out_h = model_h(inputs) out_quant = quant_model(inputs) out_quant_h = quant_h(inputs) cos_sim_func = nn.CosineSimilarity(axis=0) - print('fuse linear+bn', cos_sim_func(out.flatten(), - out_quant.flatten())) + print( + 'fuse linear+bn', cos_sim_func(out.flatten(), out_quant.flatten()) + ) print(cos_sim_func(out_h.flatten(), out_quant_h.flatten())) class TestImperativePTQ(unittest.TestCase): - """ - """ + """ """ @classmethod def setUpClass(cls): cls.download_path = 'dygraph_int8/download' - cls.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - cls.download_path) + cls.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + cls.download_path + ) cls.lenet_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/lenet_pretrained.tar.gz" cls.lenet_md5 = "953b802fb73b52fae42896e3c24f0afb" @@ -87,7 +94,8 @@ class TestImperativePTQ(unittest.TestCase): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_model(self, data_url, data_md5, folder_name): @@ -114,23 +122,29 @@ class TestImperativePTQ(unittest.TestCase): 'batch_norm2d_0': [[0.37673383951187134], [0.44249194860458374]], 're_lu_0': [[0.44249194860458374], [0.25804123282432556]], 'max_pool2d_0': [[0.25804123282432556], [0.25804123282432556]], - 'linear_0': [[1.7058950662612915], [14.405526161193848], - [0.4373355209827423]], + 'linear_0': [ + [1.7058950662612915], + [14.405526161193848], + [0.4373355209827423], + ], 'add_0': [[1.7058950662612915, 0.0], [1.7058950662612915]], } def model_test(self, model, batch_num=-1, batch_size=8): model.eval() - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) eval_acc_top1_list = [] for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape(-1, 1) + x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) + y_data = ( + np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -141,8 +155,11 @@ class TestImperativePTQ(unittest.TestCase): eval_acc_top1_list.append(float(acc_top1.numpy())) if batch_id % 50 == 0: - _logger.info("Test | At step {}: acc1 = {:}, acc5 = {:}".format( - batch_id, acc_top1.numpy(), acc_top5.numpy())) + _logger.info( + "Test | At step {}: acc1 = {:}, acc5 = {:}".format( + batch_id, acc_top1.numpy(), acc_top5.numpy() + ) + ) if batch_num > 0 and batch_id + 1 >= batch_num: break @@ -153,31 +170,39 @@ class TestImperativePTQ(unittest.TestCase): def program_test(self, program_path, batch_num=-1, batch_size=8): exe = paddle.static.Executor(paddle.CPUPlace()) - [inference_program, feed_target_names, fetch_targets - ] = (paddle.static.load_inference_model(program_path, exe)) - - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - - top1_correct_num = 0. - total_num = 0. + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(program_path, exe) + + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + + top1_correct_num = 0.0 + total_num = 0.0 for batch_id, data in enumerate(test_reader()): - img = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') + img = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) label = np.array([x[1] for x in data]).astype('int64') feed = {feed_target_names[0]: img} - results = exe.run(inference_program, - feed=feed, - fetch_list=fetch_targets) + results = exe.run( + inference_program, feed=feed, fetch_list=fetch_targets + ) pred = np.argmax(results[0], axis=1) top1_correct_num += np.sum(np.equal(pred, label)) total_num += len(img) if total_num % 50 == 49: - _logger.info("Test | Test num {}: acc1 = {:}".format( - total_num, top1_correct_num / total_num)) + _logger.info( + "Test | Test num {}: acc1 = {:}".format( + total_num, top1_correct_num / total_num + ) + ) if batch_num > 0 and batch_id + 1 >= batch_num: break @@ -189,8 +214,9 @@ class TestImperativePTQ(unittest.TestCase): self.set_vars() # Load model - params_path = self.download_model(self.lenet_url, self.lenet_md5, - "lenet") + params_path = self.download_model( + self.lenet_url, self.lenet_md5, "lenet" + ) params_path += "/lenet_pretrained/lenet.pdparams" model = ImperativeLenet() @@ -198,25 +224,28 @@ class TestImperativePTQ(unittest.TestCase): model.set_state_dict(model_state_dict) # Quantize, calibrate and save quant_model = self.ptq.quantize(model) - before_acc_top1 = self.model_test(quant_model, self.batch_num, - self.batch_size) + before_acc_top1 = self.model_test( + quant_model, self.batch_num, self.batch_size + ) input_spec = [ paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32') ] with tempfile.TemporaryDirectory(prefix="imperative_ptq_") as tmpdir: save_path = os.path.join(tmpdir, "model") - self.ptq.save_quantized_model(model=quant_model, - path=save_path, - input_spec=input_spec) + self.ptq.save_quantized_model( + model=quant_model, path=save_path, input_spec=input_spec + ) print('Quantized model saved in {%s}' % save_path) - after_acc_top1 = self.model_test(quant_model, self.batch_num, - self.batch_size) + after_acc_top1 = self.model_test( + quant_model, self.batch_num, self.batch_size + ) paddle.enable_static() - infer_acc_top1 = self.program_test(save_path, self.batch_num, - self.batch_size) + infer_acc_top1 = self.program_test( + save_path, self.batch_num, self.batch_size + ) paddle.disable_static() # Check @@ -224,11 +253,15 @@ class TestImperativePTQ(unittest.TestCase): print('After converted acc_top1: %s' % after_acc_top1) print('Infer acc_top1: %s' % infer_acc_top1) - self.assertTrue(after_acc_top1 >= self.eval_acc_top1, - msg="The test acc {%f} is less than {%f}." % - (after_acc_top1, self.eval_acc_top1)) - self.assertTrue(infer_acc_top1 >= after_acc_top1, - msg='The acc is lower after converting model.') + self.assertTrue( + after_acc_top1 >= self.eval_acc_top1, + msg="The test acc {%f} is less than {%f}." + % (after_acc_top1, self.eval_acc_top1), + ) + self.assertTrue( + infer_acc_top1 >= after_acc_top1, + msg='The acc is lower after converting model.', + ) end_time = time.time() print("total time: %ss \n" % (end_time - start_time)) @@ -240,15 +273,15 @@ class TestImperativePTQ(unittest.TestCase): class TestImperativePTQfuse(TestImperativePTQ): - def func_ptq(self): start_time = time.time() self.set_vars() # Load model - params_path = self.download_model(self.lenet_url, self.lenet_md5, - "lenet") + params_path = self.download_model( + self.lenet_url, self.lenet_md5, "lenet" + ) params_path += "/lenet_pretrained/lenet.pdparams" model = ImperativeLenet() @@ -259,27 +292,32 @@ class TestImperativePTQfuse(TestImperativePTQ): quant_model = self.ptq.quantize(model, fuse=True, fuse_list=f_l) for name, layer in quant_model.named_sublayers(): if name in f_l: - assert not (isinstance(layer, nn.BatchNorm1D) - or isinstance(layer, nn.BatchNorm2D)) - before_acc_top1 = self.model_test(quant_model, self.batch_num, - self.batch_size) + assert not ( + isinstance(layer, nn.BatchNorm1D) + or isinstance(layer, nn.BatchNorm2D) + ) + before_acc_top1 = self.model_test( + quant_model, self.batch_num, self.batch_size + ) input_spec = [ paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32') ] with tempfile.TemporaryDirectory(prefix="imperative_ptq_") as tmpdir: save_path = os.path.join(tmpdir, "model") - self.ptq.save_quantized_model(model=quant_model, - path=save_path, - input_spec=input_spec) + self.ptq.save_quantized_model( + model=quant_model, path=save_path, input_spec=input_spec + ) print('Quantized model saved in {%s}' % save_path) - after_acc_top1 = self.model_test(quant_model, self.batch_num, - self.batch_size) + after_acc_top1 = self.model_test( + quant_model, self.batch_num, self.batch_size + ) paddle.enable_static() - infer_acc_top1 = self.program_test(save_path, self.batch_num, - self.batch_size) + infer_acc_top1 = self.program_test( + save_path, self.batch_num, self.batch_size + ) paddle.disable_static() # Check @@ -287,15 +325,19 @@ class TestImperativePTQfuse(TestImperativePTQ): print('After converted acc_top1: %s' % after_acc_top1) print('Infer acc_top1: %s' % infer_acc_top1) - #Check whether the quant_model is correct after converting. - #The acc of quantized model should be higher than 0.95. - self.assertTrue(after_acc_top1 >= self.eval_acc_top1, - msg="The test acc {%f} is less than {%f}." % - (after_acc_top1, self.eval_acc_top1)) - #Check the saved infer_model.The acc of infer model - #should not be lower than the one of dygraph model. - self.assertTrue(infer_acc_top1 >= after_acc_top1, - msg='The acc is lower after converting model.') + # Check whether the quant_model is correct after converting. + # The acc of quantized model should be higher than 0.95. + self.assertTrue( + after_acc_top1 >= self.eval_acc_top1, + msg="The test acc {%f} is less than {%f}." + % (after_acc_top1, self.eval_acc_top1), + ) + # Check the saved infer_model.The acc of infer model + # should not be lower than the one of dygraph model. + self.assertTrue( + infer_acc_top1 >= after_acc_top1, + msg='The acc is lower after converting model.', + ) end_time = time.time() print("total time: %ss \n" % (end_time - start_time)) @@ -307,7 +349,6 @@ class TestImperativePTQfuse(TestImperativePTQ): class TestImperativePTQHist(TestImperativePTQ): - def set_vars(self): config = PTQConfig(HistQuantizer(), AbsmaxQuantizer()) self.ptq = ImperativePTQ(config) @@ -317,19 +358,24 @@ class TestImperativePTQHist(TestImperativePTQ): self.eval_acc_top1 = 0.98 self.gt_thresholds = { - 'conv2d_0': [[0.99853515625], [0.35732391771364225], - [0.10933732241392136]], + 'conv2d_0': [ + [0.99853515625], + [0.35732391771364225], + [0.10933732241392136], + ], 'batch_norm2d_0': [[0.35732391771364225], [0.4291427868761275]], 're_lu_0': [[0.4291427868761275], [0.2359918110742001]], 'max_pool2d_0': [[0.2359918110742001], [0.25665526917146053]], - 'linear_0': [[1.7037603475152991], [14.395224522473026], - [0.4373355209827423]], + 'linear_0': [ + [1.7037603475152991], + [14.395224522473026], + [0.4373355209827423], + ], 'add_0': [[1.7037603475152991, 0.0], [1.7037603475152991]], } class TestImperativePTQKL(TestImperativePTQ): - def set_vars(self): config = PTQConfig(KLQuantizer(), PerChannelAbsmaxQuantizer()) self.ptq = ImperativePTQ(config) @@ -339,13 +385,20 @@ class TestImperativePTQKL(TestImperativePTQ): self.eval_acc_top1 = 0.98 conv2d_1_wt_thresholds = [ - 0.18116560578346252, 0.17079241573810577, 0.1702047884464264, - 0.179476797580719, 0.1454375684261322, 0.22981858253479004 + 0.18116560578346252, + 0.17079241573810577, + 0.1702047884464264, + 0.179476797580719, + 0.1454375684261322, + 0.22981858253479004, ] self.gt_thresholds = { 'conv2d_0': [[0.99267578125], [0.37695913558696836]], - 'conv2d_1': [[0.19189296757394914], [0.24514256547263358], - [conv2d_1_wt_thresholds]], + 'conv2d_1': [ + [0.19189296757394914], + [0.24514256547263358], + [conv2d_1_wt_thresholds], + ], 'batch_norm2d_0': [[0.37695913558696836], [0.27462541429440535]], 're_lu_0': [[0.27462541429440535], [0.19189296757394914]], 'max_pool2d_0': [[0.19189296757394914], [0.19189296757394914]], diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py index a507fbe4161..e037cea8f55 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py @@ -29,7 +29,10 @@ from paddle.fluid.dygraph.container import Sequential from paddle.nn import Linear, Conv2D, Softmax, Conv2DTranspose from paddle.fluid.log_helper import get_logger from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX -from paddle.nn.quant.quant_layers import QuantizedConv2D, QuantizedConv2DTranspose +from paddle.nn.quant.quant_layers import ( + QuantizedConv2D, + QuantizedConv2DTranspose, +) from paddle.fluid.framework import _test_eager_guard from imperative_test_utils import fix_model_dict, ImperativeLenet @@ -39,9 +42,9 @@ os.environ["CPU_NUM"] = "1" if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeQat(unittest.TestCase): @@ -67,26 +70,28 @@ class TestImperativeQat(unittest.TestCase): weight_quantize_type=self.weight_quantize_type, activation_quantize_type=self.activation_quantize_type, fuse_conv_bn=self.fuse_conv_bn, - onnx_format=self.onnx_format) + onnx_format=self.onnx_format, + ) with fluid.dygraph.guard(): # For CI coverage - conv1 = Conv2D(in_channels=3, - out_channels=2, - kernel_size=3, - stride=1, - padding=1, - padding_mode='replicate') + conv1 = Conv2D( + in_channels=3, + out_channels=2, + kernel_size=3, + stride=1, + padding=1, + padding_mode='replicate', + ) quant_conv1 = QuantizedConv2D(conv1) data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') quant_conv1(fluid.dygraph.to_variable(data)) conv_transpose = Conv2DTranspose(4, 6, (3, 3)) quant_conv_transpose = QuantizedConv2DTranspose(conv_transpose) - x_var = paddle.uniform((2, 4, 8, 8), - dtype='float32', - min=-1.0, - max=1.0) + x_var = paddle.uniform( + (2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0 + ) quant_conv_transpose(x_var) seed = 1 @@ -97,23 +102,29 @@ class TestImperativeQat(unittest.TestCase): lenet = ImperativeLenet() lenet = fix_model_dict(lenet) imperative_qat.quantize(lenet) - adam = AdamOptimizer(learning_rate=0.001, - parameter_list=lenet.parameters()) + adam = AdamOptimizer( + learning_rate=0.001, parameter_list=lenet.parameters() + ) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=32, - drop_last=True) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=32) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=32, drop_last=True + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=32 + ) epoch_num = 1 for epoch in range(epoch_num): lenet.train() for batch_id, data in enumerate(train_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) @@ -126,52 +137,65 @@ class TestImperativeQat(unittest.TestCase): lenet.clear_gradients() if batch_id % 100 == 0: _logger.info( - "Train | At epoch {} step {}: loss = {:}, acc= {:}". - format(epoch, batch_id, avg_loss.numpy(), - acc.numpy())) + "Train | At epoch {} step {}: loss = {:}, acc= {:}".format( + epoch, batch_id, avg_loss.numpy(), acc.numpy() + ) + ) if batch_id == 500: # For shortening CI time break lenet.eval() eval_acc_top1_list = [] for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) out = lenet(img) - acc_top1 = fluid.layers.accuracy(input=out, - label=label, - k=1) - acc_top5 = fluid.layers.accuracy(input=out, - label=label, - k=5) + acc_top1 = fluid.layers.accuracy( + input=out, label=label, k=1 + ) + acc_top5 = fluid.layers.accuracy( + input=out, label=label, k=5 + ) if batch_id % 100 == 0: eval_acc_top1_list.append(float(acc_top1.numpy())) _logger.info( - "Test | At epoch {} step {}: acc1 = {:}, acc5 = {:}" - .format(epoch, batch_id, acc_top1.numpy(), - acc_top5.numpy())) + "Test | At epoch {} step {}: acc1 = {:}, acc5 = {:}".format( + epoch, + batch_id, + acc_top1.numpy(), + acc_top5.numpy(), + ) + ) # check eval acc eval_acc_top1 = sum(eval_acc_top1_list) / len( - eval_acc_top1_list) + eval_acc_top1_list + ) print('eval_acc_top1', eval_acc_top1) - self.assertTrue(eval_acc_top1 > 0.9, - msg="The test acc {%f} is less than 0.9." % - eval_acc_top1) + self.assertTrue( + eval_acc_top1 > 0.9, + msg="The test acc {%f} is less than 0.9." % eval_acc_top1, + ) # test the correctness of `paddle.jit.save` data = next(test_reader()) - test_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape(-1, 1) + test_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) + ) test_img = fluid.dygraph.to_variable(test_data) label = fluid.dygraph.to_variable(y_data) lenet.eval() @@ -184,9 +208,11 @@ class TestImperativeQat(unittest.TestCase): layer=lenet, path=os.path.join(tmpdir, "lenet"), input_spec=[ - paddle.static.InputSpec(shape=[None, 1, 28, 28], - dtype='float32') - ]) + paddle.static.InputSpec( + shape=[None, 1, 28, 28], dtype='float32' + ) + ], + ) print('Quantized model saved in %s' % tmpdir) if core.is_compiled_with_cuda(): @@ -194,15 +220,21 @@ class TestImperativeQat(unittest.TestCase): else: place = core.CPUPlace() exe = fluid.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model( - dirname=tmpdir, - executor=exe, - model_filename="lenet" + INFER_MODEL_SUFFIX, - params_filename="lenet" + INFER_PARAMS_SUFFIX) - quant_out, = exe.run(inference_program, - feed={feed_target_names[0]: test_data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + dirname=tmpdir, + executor=exe, + model_filename="lenet" + INFER_MODEL_SUFFIX, + params_filename="lenet" + INFER_PARAMS_SUFFIX, + ) + (quant_out,) = exe.run( + inference_program, + feed={feed_target_names[0]: test_data}, + fetch_list=fetch_targets, + ) paddle.disable_static() quant_out = fluid.dygraph.to_variable(quant_out) quant_acc = fluid.layers.accuracy(quant_out, label).numpy() @@ -217,7 +249,6 @@ class TestImperativeQat(unittest.TestCase): class TestImperativeQatONNXFormat(unittest.TestCase): - def set_vars(self): self.weight_quantize_type = 'abs_max' self.activation_quantize_type = 'moving_average_abs_max' diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_amp.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_amp.py index b92d19cab25..ee0edb445e7 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_amp.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_amp.py @@ -33,9 +33,9 @@ os.environ["CPU_NUM"] = "1" if paddle.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeQatAmp(unittest.TestCase): @@ -46,12 +46,14 @@ class TestImperativeQatAmp(unittest.TestCase): @classmethod def setUpClass(cls): cls.root_path = tempfile.TemporaryDirectory( - prefix="imperative_qat_amp_") + prefix="imperative_qat_amp_" + ) cls.save_path = os.path.join(cls.root_path.name, "model") cls.download_path = 'dygraph_int8/download' - cls.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - cls.download_path) + cls.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + cls.download_path + ) cls.lenet_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/lenet_pretrained.tar.gz" cls.lenet_md5 = "953b802fb73b52fae42896e3c24f0afb" @@ -68,7 +70,8 @@ class TestImperativeQatAmp(unittest.TestCase): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_model(self, data_url, data_md5, folder_name): @@ -93,17 +96,21 @@ class TestImperativeQatAmp(unittest.TestCase): def model_train(self, model, batch_num=-1, batch_size=32, use_amp=False): model.train() - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) - adam = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) + adam = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=500) for batch_id, data in enumerate(train_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape(-1, 1) + x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) + y_data = ( + np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -130,8 +137,11 @@ class TestImperativeQatAmp(unittest.TestCase): model.clear_gradients() if batch_id % 100 == 0: - _logger.info("Train | step {}: loss = {:}, acc= {:}".format( - batch_id, avg_loss.numpy(), acc.numpy())) + _logger.info( + "Train | step {}: loss = {:}, acc= {:}".format( + batch_id, avg_loss.numpy(), acc.numpy() + ) + ) if batch_num > 0 and batch_id + 1 >= batch_num: break @@ -139,15 +149,18 @@ class TestImperativeQatAmp(unittest.TestCase): def model_test(self, model, batch_num=-1, batch_size=32, use_amp=False): model.eval() - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) acc_top1_list = [] for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape(-1, 1) + x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) + y_data = ( + np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -159,8 +172,11 @@ class TestImperativeQatAmp(unittest.TestCase): acc_top1_list.append(float(acc_top1.numpy())) if batch_id % 100 == 0: - _logger.info("Test | At step {}: acc1 = {:}, acc5 = {:}".format( - batch_id, acc_top1.numpy(), acc_top5.numpy())) + _logger.info( + "Test | At step {}: acc1 = {:}, acc5 = {:}".format( + batch_id, acc_top1.numpy(), acc_top5.numpy() + ) + ) if batch_num > 0 and batch_id + 1 >= batch_num: break @@ -173,8 +189,9 @@ class TestImperativeQatAmp(unittest.TestCase): self.set_vars() - params_path = self.download_model(self.lenet_url, self.lenet_md5, - "lenet") + params_path = self.download_model( + self.lenet_url, self.lenet_md5, "lenet" + ) params_path += "/lenet_pretrained/lenet.pdparams" with fluid.dygraph.guard(): @@ -183,24 +200,31 @@ class TestImperativeQatAmp(unittest.TestCase): model.set_state_dict(model_state_dict) _logger.info("Test fp32 model") - fp32_acc_top1 = self.model_test(model, self.test_batch_num, - self.test_batch_size) + fp32_acc_top1 = self.model_test( + model, self.test_batch_num, self.test_batch_size + ) self.qat.quantize(model) use_amp = True - self.model_train(model, self.train_batch_num, self.train_batch_size, - use_amp) + self.model_train( + model, self.train_batch_num, self.train_batch_size, use_amp + ) _logger.info("Test int8 model") - int8_acc_top1 = self.model_test(model, self.test_batch_num, - self.test_batch_size, use_amp) - - _logger.info('fp32_acc_top1: %f, int8_acc_top1: %f' % - (fp32_acc_top1, int8_acc_top1)) - self.assertTrue(int8_acc_top1 > fp32_acc_top1 - 0.01, - msg='fp32_acc_top1: %f, int8_acc_top1: %f' % - (fp32_acc_top1, int8_acc_top1)) + int8_acc_top1 = self.model_test( + model, self.test_batch_num, self.test_batch_size, use_amp + ) + + _logger.info( + 'fp32_acc_top1: %f, int8_acc_top1: %f' + % (fp32_acc_top1, int8_acc_top1) + ) + self.assertTrue( + int8_acc_top1 > fp32_acc_top1 - 0.01, + msg='fp32_acc_top1: %f, int8_acc_top1: %f' + % (fp32_acc_top1, int8_acc_top1), + ) input_spec = [ paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32') diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py index 30ea0346d8b..45ea756ce27 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py @@ -31,13 +31,12 @@ os.environ["CPU_NUM"] = "1" if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeQatChannelWise(TestImperativeQat): - def set_vars(self): self.weight_quantize_type = 'channel_wise_abs_max' self.activation_quantize_type = 'moving_average_abs_max' @@ -48,7 +47,6 @@ class TestImperativeQatChannelWise(TestImperativeQat): class TestImperativeQatChannelWiseONNXFormat(TestImperativeQat): - def set_vars(self): self.weight_quantize_type = 'channel_wise_abs_max' self.activation_quantize_type = 'moving_average_abs_max' diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py index 898d89f0142..dc1dee13cf8 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py @@ -31,13 +31,12 @@ os.environ["CPU_NUM"] = "1" if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeQatfuseBN(TestImperativeQat): - def set_vars(self): self.weight_quantize_type = 'abs_max' self.activation_quantize_type = 'moving_average_abs_max' diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py index 8269abbab4d..7e94c8dedd5 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py @@ -23,14 +23,21 @@ import logging import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.optimizer import SGDOptimizer, AdamOptimizer, MomentumOptimizer +from paddle.fluid.optimizer import ( + SGDOptimizer, + AdamOptimizer, + MomentumOptimizer, +) from paddle.fluid.contrib.slim.quantization import ImperativeQuantAware from paddle.fluid.dygraph.container import Sequential from paddle.nn import ReLU, ReLU6, LeakyReLU, Sigmoid, Softmax, PReLU from paddle.nn import Linear, Conv2D, Softmax, BatchNorm2D, MaxPool2D from paddle.fluid.log_helper import get_logger from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX -from paddle.nn.quant.quant_layers import QuantizedConv2D, QuantizedConv2DTranspose +from paddle.nn.quant.quant_layers import ( + QuantizedConv2D, + QuantizedConv2DTranspose, +) from paddle.fluid.framework import _test_eager_guard from imperative_test_utils import fix_model_dict @@ -40,13 +47,12 @@ os.environ["CPU_NUM"] = "1" if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class ImperativeLenet(fluid.dygraph.Layer): - def __init__(self, num_classes=10): super(ImperativeLenet, self).__init__() conv2d_w1_attr = fluid.ParamAttr(name="conv2d_w_1") @@ -59,36 +65,55 @@ class ImperativeLenet(fluid.dygraph.Layer): fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") self.features = Sequential( - Conv2D(in_channels=1, - out_channels=6, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2d_w1_attr, - bias_attr=False), BatchNorm2D(6), ReLU(), + Conv2D( + in_channels=1, + out_channels=6, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2d_w1_attr, + bias_attr=False, + ), + BatchNorm2D(6), + ReLU(), + MaxPool2D(kernel_size=2, stride=2), + Conv2D( + in_channels=6, + out_channels=16, + kernel_size=5, + stride=1, + padding=0, + weight_attr=conv2d_w2_attr, + bias_attr=conv2d_b2_attr, + ), + BatchNorm2D(16), + PReLU(), MaxPool2D(kernel_size=2, stride=2), - Conv2D(in_channels=6, - out_channels=16, - kernel_size=5, - stride=1, - padding=0, - weight_attr=conv2d_w2_attr, - bias_attr=conv2d_b2_attr), BatchNorm2D(16), PReLU(), - MaxPool2D(kernel_size=2, stride=2)) + ) self.fc = Sequential( - Linear(in_features=400, - out_features=120, - weight_attr=fc_w1_attr, - bias_attr=fc_b1_attr), LeakyReLU(), - Linear(in_features=120, - out_features=84, - weight_attr=fc_w2_attr, - bias_attr=fc_b2_attr), Sigmoid(), - Linear(in_features=84, - out_features=num_classes, - weight_attr=fc_w3_attr, - bias_attr=fc_b3_attr), Softmax()) + Linear( + in_features=400, + out_features=120, + weight_attr=fc_w1_attr, + bias_attr=fc_b1_attr, + ), + LeakyReLU(), + Linear( + in_features=120, + out_features=84, + weight_attr=fc_w2_attr, + bias_attr=fc_b2_attr, + ), + Sigmoid(), + Linear( + in_features=84, + out_features=num_classes, + weight_attr=fc_w3_attr, + bias_attr=fc_b3_attr, + ), + Softmax(), + ) def forward(self, inputs): x = self.features(inputs) @@ -98,7 +123,6 @@ class ImperativeLenet(fluid.dygraph.Layer): class TestImperativeQatLSQ(unittest.TestCase): - def set_vars(self): self.weight_quantize_type = 'channel_wise_lsq_weight' self.activation_quantize_type = 'lsq_act' @@ -111,7 +135,8 @@ class TestImperativeQatLSQ(unittest.TestCase): imperative_qat = ImperativeQuantAware( weight_quantize_type=self.weight_quantize_type, activation_quantize_type=self.activation_quantize_type, - fuse_conv_bn=self.fuse_conv_bn) + fuse_conv_bn=self.fuse_conv_bn, + ) seed = 100 np.random.seed(seed) @@ -121,22 +146,26 @@ class TestImperativeQatLSQ(unittest.TestCase): lenet = ImperativeLenet() lenet = fix_model_dict(lenet) imperative_qat.quantize(lenet) - optimizer = MomentumOptimizer(learning_rate=0.1, - parameter_list=lenet.parameters(), - momentum=0.9) + optimizer = MomentumOptimizer( + learning_rate=0.1, parameter_list=lenet.parameters(), momentum=0.9 + ) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=64, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=64, drop_last=True + ) test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=32) epoch_num = 2 for epoch in range(epoch_num): lenet.train() for batch_id, data in enumerate(train_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) @@ -151,42 +180,53 @@ class TestImperativeQatLSQ(unittest.TestCase): if batch_id % 100 == 0: _logger.info( - "Train | At epoch {} step {}: loss = {:}, acc= {:}". - format(epoch, batch_id, avg_loss.numpy(), acc.numpy())) + "Train | At epoch {} step {}: loss = {:}, acc= {:}".format( + epoch, batch_id, avg_loss.numpy(), acc.numpy() + ) + ) lenet.eval() eval_acc_top1_list = [] with paddle.no_grad(): for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) out = lenet(img) - acc_top1 = fluid.layers.accuracy(input=out, - label=label, - k=1) - acc_top5 = fluid.layers.accuracy(input=out, - label=label, - k=5) + acc_top1 = fluid.layers.accuracy( + input=out, label=label, k=1 + ) + acc_top5 = fluid.layers.accuracy( + input=out, label=label, k=5 + ) if batch_id % 100 == 0: eval_acc_top1_list.append(float(acc_top1.numpy())) _logger.info( - "Test | At epoch {} step {}: acc1 = {:}, acc5 = {:}" - .format(epoch, batch_id, acc_top1.numpy(), - acc_top5.numpy())) + "Test | At epoch {} step {}: acc1 = {:}, acc5 = {:}".format( + epoch, + batch_id, + acc_top1.numpy(), + acc_top5.numpy(), + ) + ) # check eval acc eval_acc_top1 = sum(eval_acc_top1_list) / len(eval_acc_top1_list) print('eval_acc_top1', eval_acc_top1) - self.assertTrue(eval_acc_top1 > 0.9, - msg="The test acc {%f} is less than 0.9." % - eval_acc_top1) + self.assertTrue( + eval_acc_top1 > 0.9, + msg="The test acc {%f} is less than 0.9." % eval_acc_top1, + ) def test_qat(self): self.func_qat() diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_user_defined.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_user_defined.py index 2ae75744cf8..89e2362b51d 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_user_defined.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_user_defined.py @@ -32,21 +32,21 @@ from paddle.fluid.framework import _test_eager_guard os.environ["CPU_NUM"] = "1" -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class PACT(nn.Layer): - def __init__(self, init_value=20): super(PACT, self).__init__() alpha_attr = paddle.ParamAttr( name=self.full_name() + ".pact", - initializer=paddle.nn.initializer.Constant(value=init_value)) - self.alpha = self.create_parameter(shape=[1], - attr=alpha_attr, - dtype='float32') + initializer=paddle.nn.initializer.Constant(value=init_value), + ) + self.alpha = self.create_parameter( + shape=[1], attr=alpha_attr, dtype='float32' + ) def forward(self, x): out_left = paddle.nn.functional.relu(x - self.alpha) @@ -56,31 +56,30 @@ class PACT(nn.Layer): class CustomQAT(nn.Layer): - def __init__(self): super(CustomQAT, self).__init__() - attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Constant( - value=1.0)) - self.u_param = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') - self.l_param = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') - self.alpha_param = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') - self.upper = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') + attr = paddle.ParamAttr( + initializer=paddle.nn.initializer.Constant(value=1.0) + ) + self.u_param = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) + self.l_param = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) + self.alpha_param = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) + self.upper = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) self.upper.stop_gradient = True - self.lower = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') + self.lower = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) self.lower.stop_gradient = True def forward(self, x): - def clip(x, upper, lower): x = x + paddle.nn.functional.relu(lower - x) x = x - paddle.nn.functional.relu(x - upper) @@ -111,7 +110,6 @@ class CustomQAT(nn.Layer): class ModelForConv2dT(nn.Layer): - def __init__(self, num_classes=10): super(ModelForConv2dT, self).__init__() self.features = nn.Conv2DTranspose(4, 6, (3, 3)) @@ -125,29 +123,34 @@ class ModelForConv2dT(nn.Layer): class ImperativeLenet(paddle.nn.Layer): - def __init__(self, num_classes=10, classifier_activation='softmax'): super(ImperativeLenet, self).__init__() self.features = Sequential( - Conv2D(num_channels=1, - num_filters=6, - filter_size=3, - stride=1, - padding=1), + Conv2D( + num_channels=1, + num_filters=6, + filter_size=3, + stride=1, + padding=1, + ), Pool2D(pool_size=2, pool_type='max', pool_stride=2), - Conv2D(num_channels=6, - num_filters=16, - filter_size=5, - stride=1, - padding=0), - Pool2D(pool_size=2, pool_type='max', pool_stride=2)) + Conv2D( + num_channels=6, + num_filters=16, + filter_size=5, + stride=1, + padding=0, + ), + Pool2D(pool_size=2, pool_type='max', pool_stride=2), + ) self.fc = Sequential( Linear(input_dim=400, output_dim=120), Linear(input_dim=120, output_dim=84), - Linear(input_dim=84, - output_dim=num_classes, - act=classifier_activation)) + Linear( + input_dim=84, output_dim=num_classes, act=classifier_activation + ), + ) def forward(self, inputs): x = self.features(inputs) @@ -158,7 +161,6 @@ class ImperativeLenet(paddle.nn.Layer): class TestUserDefinedActPreprocess(unittest.TestCase): - def setUp(self): _logger.info("test act_preprocess") self.imperative_qat = ImperativeQuantAware(act_preprocess_layer=PACT) @@ -178,9 +180,13 @@ class TestUserDefinedActPreprocess(unittest.TestCase): if name.endswith("bias"): value = np.zeros_like(p_value).astype('float32') else: - value = np.random.normal( - loc=0.0, scale=0.01, - size=np.product(p_shape)).reshape(p_shape).astype('float32') + value = ( + np.random.normal( + loc=0.0, scale=0.01, size=np.product(p_shape) + ) + .reshape(p_shape) + .astype('float32') + ) fixed_state[name] = value param_init_map[param.name] = value lenet.set_dict(fixed_state) @@ -188,10 +194,10 @@ class TestUserDefinedActPreprocess(unittest.TestCase): imperative_qat.quantize(lenet) adam = Adam(learning_rate=0.001, parameters=lenet.parameters()) dynamic_loss_rec = [] - #for CI coverage + # for CI coverage conv_transpose = ModelForConv2dT() imperative_qat.quantize(conv_transpose) - x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv_transpose(x_var) def train(model): @@ -200,10 +206,14 @@ class TestUserDefinedActPreprocess(unittest.TestCase): for epoch in range(epoch_num): model.train() for batch_id, data in enumerate(train_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -216,19 +226,24 @@ class TestUserDefinedActPreprocess(unittest.TestCase): model.clear_gradients() if batch_id % 50 == 0: _logger.info( - "Train | At epoch {} step {}: loss = {:}, acc= {:}". - format(epoch, batch_id, avg_loss.numpy(), - acc.numpy())) + "Train | At epoch {} step {}: loss = {:}, acc= {:}".format( + epoch, batch_id, avg_loss.numpy(), acc.numpy() + ) + ) break def test(model): model.eval() avg_acc = [[], []] for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -241,11 +256,13 @@ class TestUserDefinedActPreprocess(unittest.TestCase): if batch_id % 100 == 0: _logger.info( "Test | step {}: acc1 = {:}, acc5 = {:}".format( - batch_id, acc_top1.numpy(), acc_top5.numpy())) + batch_id, acc_top1.numpy(), acc_top5.numpy() + ) + ) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=512, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=512, drop_last=True + ) test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=512) train(lenet) test(lenet) @@ -257,25 +274,23 @@ class TestUserDefinedActPreprocess(unittest.TestCase): class TestUserDefinedWeightPreprocess(TestUserDefinedActPreprocess): - def setUp(self): _logger.info("test weight_preprocess") self.imperative_qat = ImperativeQuantAware(weight_preprocess_layer=PACT) class TestUserDefinedActQuantize(TestUserDefinedActPreprocess): - def setUp(self): _logger.info("test act_quantize") self.imperative_qat = ImperativeQuantAware(act_quantize_layer=CustomQAT) class TestUserDefinedWeightQuantize(TestUserDefinedActPreprocess): - def setUp(self): _logger.info("test weight_quantize") self.imperative_qat = ImperativeQuantAware( - weight_quantize_layer=CustomQAT) + weight_quantize_layer=CustomQAT + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_skip_op.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_skip_op.py index f7608ee66b4..131866095ad 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_skip_op.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_skip_op.py @@ -29,20 +29,23 @@ from paddle.nn import Linear, Conv2D, Softmax, BatchNorm from paddle.fluid.dygraph.nn import Pool2D from paddle.fluid.log_helper import get_logger -from imperative_test_utils import fix_model_dict, train_lenet, ImperativeLenetWithSkipQuant +from imperative_test_utils import ( + fix_model_dict, + train_lenet, + ImperativeLenetWithSkipQuant, +) from paddle.fluid.framework import _test_eager_guard os.environ["CPU_NUM"] = "1" if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeOutSclae(unittest.TestCase): - def func_out_scale_acc(self): paddle.disable_static() seed = 1000 @@ -51,16 +54,17 @@ class TestImperativeOutSclae(unittest.TestCase): qat = ImperativeQuantAware() np.random.seed(seed) - reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=512, - drop_last=True) + reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=512, drop_last=True + ) lenet = ImperativeLenetWithSkipQuant() lenet = fix_model_dict(lenet) qat.quantize(lenet) - adam = AdamOptimizer(learning_rate=lr, - parameter_list=lenet.parameters()) + adam = AdamOptimizer( + learning_rate=lr, parameter_list=lenet.parameters() + ) dynamic_loss_rec = [] lenet.train() loss_list = train_lenet(lenet, reader, adam) @@ -70,13 +74,15 @@ class TestImperativeOutSclae(unittest.TestCase): path = "./save_dynamic_quant_infer_model/lenet" save_dir = "./save_dynamic_quant_infer_model" - qat.save_quantized_model(layer=lenet, - path=path, - input_spec=[ - paddle.static.InputSpec( - shape=[None, 1, 28, 28], - dtype='float32') - ]) + qat.save_quantized_model( + layer=lenet, + path=path, + input_spec=[ + paddle.static.InputSpec( + shape=[None, 1, 28, 28], dtype='float32' + ) + ], + ) paddle.enable_static() @@ -86,12 +92,16 @@ class TestImperativeOutSclae(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = (fluid.io.load_inference_model( - dirname=save_dir, - executor=exe, - model_filename="lenet" + INFER_MODEL_SUFFIX, - params_filename="lenet" + INFER_PARAMS_SUFFIX)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + dirname=save_dir, + executor=exe, + model_filename="lenet" + INFER_MODEL_SUFFIX, + params_filename="lenet" + INFER_PARAMS_SUFFIX, + ) model_ops = inference_program.global_block().ops conv2d_count, matmul_count = 0, 0 @@ -105,10 +115,12 @@ class TestImperativeOutSclae(unittest.TestCase): conv2d_skip_count += 1 if conv2d_count > 0: self.assertTrue( - 'fake_quantize_dequantize' in model_ops[i - 1].type) + 'fake_quantize_dequantize' in model_ops[i - 1].type + ) else: self.assertTrue( - 'fake_quantize_dequantize' not in model_ops[i - 1].type) + 'fake_quantize_dequantize' not in model_ops[i - 1].type + ) conv2d_count += 1 if op.type == 'matmul': @@ -117,10 +129,12 @@ class TestImperativeOutSclae(unittest.TestCase): matmul_skip_count += 1 if matmul_count > 0: self.assertTrue( - 'fake_quantize_dequantize' in model_ops[i - 1].type) + 'fake_quantize_dequantize' in model_ops[i - 1].type + ) else: self.assertTrue( - 'fake_quantize_dequantize' not in model_ops[i - 1].type) + 'fake_quantize_dequantize' not in model_ops[i - 1].type + ) matmul_count += 1 if find_conv2d: diff --git a/python/paddle/fluid/contrib/slim/tests/test_moving_average_abs_max_scale_op.py b/python/paddle/fluid/contrib/slim/tests/test_moving_average_abs_max_scale_op.py index 04814ed9fe1..1ec463192d9 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_moving_average_abs_max_scale_op.py +++ b/python/paddle/fluid/contrib/slim/tests/test_moving_average_abs_max_scale_op.py @@ -28,46 +28,51 @@ def init_data(batch_size=32, img_shape=[784], label_range=9): assert isinstance(img_shape, list) input_shape = [batch_size] + img_shape img = np.random.random(size=input_shape).astype(np.float32) - label = np.array( - [np.random.randint(0, label_range) for _ in range(batch_size)]).reshape( - (-1, 1)).astype("int64") + label = ( + np.array([np.random.randint(0, label_range) for _ in range(batch_size)]) + .reshape((-1, 1)) + .astype("int64") + ) return img, label class TestMovingAverageAbsMaxScaleOp(unittest.TestCase): - def check_backward(self, use_cuda): main_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - image = fluid.layers.data(name='image', - shape=[784], - dtype='float32') + image = fluid.layers.data( + name='image', shape=[784], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') fc_tmp = fluid.layers.fc(image, size=10, act='softmax') out_scale = quant_layers.MovingAverageAbsMaxScale( - name=fc_tmp.name, dtype=fc_tmp.dtype) + name=fc_tmp.name, dtype=fc_tmp.dtype + ) fc_tmp_1 = out_scale(fc_tmp) cross_entropy = fluid.layers.softmax_with_cross_entropy( - fc_tmp, label) + fc_tmp, label + ) loss = fluid.layers.reduce_mean(cross_entropy) sgd = fluid.optimizer.SGD(learning_rate=1e-3) sgd.minimize(loss) moving_average_abs_max_scale_ops = [ - op for op in main_program.blocks[0].ops + op + for op in main_program.blocks[0].ops if op.type == u'moving_average_abs_max_scale' ] - assert len( - moving_average_abs_max_scale_ops - ) == 1, "The number of moving_average_abs_max_scale_ops should be 1." + assert ( + len(moving_average_abs_max_scale_ops) == 1 + ), "The number of moving_average_abs_max_scale_ops should be 1." place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_program) binary = fluid.compiler.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name) + main_program + ).with_data_parallel(loss_name=loss.name) img, label = init_data() feed_dict = {"image": img, "label": label} diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_lstm_model.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_lstm_model.py index 4c55f58e05e..7eb7f4d479e 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_lstm_model.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_lstm_model.py @@ -34,19 +34,23 @@ np.random.seed(0) class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.download_path = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.download_path) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.download_path + ) self.root_path = tempfile.TemporaryDirectory() - self.int8_model_path = os.path.join(self.root_path.name, - "post_training_quantization") + self.int8_model_path = os.path.join( + self.root_path.name, "post_training_quantization" + ) try: os.system("mkdir -p " + self.int8_model_path) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model_path, str(e))) + print( + "Failed to create {} due to {}".format( + self.int8_model_path, str(e) + ) + ) sys.exit(-1) def tearDown(self): @@ -55,7 +59,8 @@ class TestPostTrainingQuantization(unittest.TestCase): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_model(self, data_url, data_md5, folder_name): @@ -69,7 +74,6 @@ class TestPostTrainingQuantization(unittest.TestCase): return data_cache_folder def get_batch_reader(self, data_path, place): - def reader(): with open(data_path, 'rb') as in_file: while True: @@ -83,13 +87,15 @@ class TestPostTrainingQuantization(unittest.TestCase): label = in_file.read(4 * label_len) label = np.frombuffer(label, dtype=np.int32).reshape( - [len(label) // 4]) + [len(label) // 4] + ) if label.shape[0] != 1 or label[0] > 6350: continue feat = in_file.read(4 * seq_len * 8) feat = np.frombuffer(feat, dtype=np.float32).reshape( - [len(feat) // 4 // 8, 8]) + [len(feat) // 4 // 8, 8] + ) lod_feat = [feat.shape[0]] minputs = fluid.create_lod_tensor(feat, [lod_feat], place) @@ -98,7 +104,6 @@ class TestPostTrainingQuantization(unittest.TestCase): return reader def get_simple_reader(self, data_path, place): - def reader(): with open(data_path, 'rb') as in_file: while True: @@ -112,13 +117,15 @@ class TestPostTrainingQuantization(unittest.TestCase): label = in_file.read(4 * label_len) label = np.frombuffer(label, dtype=np.int32).reshape( - [len(label) // 4]) + [len(label) // 4] + ) if label.shape[0] != 1 or label[0] > 6350: continue feat = in_file.read(4 * seq_len * 8) feat = np.frombuffer(feat, dtype=np.float32).reshape( - [len(feat) // 4 // 8, 8]) + [len(feat) // 4 // 8, 8] + ) lod_feat = [feat.shape[0]] minputs = fluid.create_lod_tensor(feat, [lod_feat], place) @@ -130,8 +137,11 @@ class TestPostTrainingQuantization(unittest.TestCase): print("test model path:" + model_path) place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) val_reader = self.get_simple_reader(data_path, place) @@ -140,10 +150,12 @@ class TestPostTrainingQuantization(unittest.TestCase): periods = [] for batch_id, (data, label) in enumerate(val_reader()): t1 = time.time() - cls_out, ctc_out = exe.run(infer_program, - feed={feed_dict[0]: data}, - fetch_list=fetch_targets, - return_numpy=False) + cls_out, ctc_out = exe.run( + infer_program, + feed={feed_dict[0]: data}, + fetch_list=fetch_targets, + return_numpy=False, + ) t2 = time.time() periods.append(t2 - t1) @@ -161,86 +173,117 @@ class TestPostTrainingQuantization(unittest.TestCase): acc = right_num / all_num return (latency, acc) - def generate_quantized_model(self, - model_path, - data_path, - algo="KL", - round_type="round", - quantizable_op_type=["conv2d"], - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - batch_size=10, - batch_nums=10, - onnx_format=False): + def generate_quantized_model( + self, + model_path, + data_path, + algo="KL", + round_type="round", + quantizable_op_type=["conv2d"], + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + batch_size=10, + batch_nums=10, + onnx_format=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.global_scope() batch_generator = self.get_batch_reader(data_path, place) - ptq = PostTrainingQuantization(executor=exe, - model_dir=model_path, - batch_generator=batch_generator, - batch_nums=batch_nums, - algo=algo, - quantizable_op_type=quantizable_op_type, - round_type=round_type, - is_full_quantize=is_full_quantize, - optimize_model=is_optimize_model, - onnx_format=onnx_format, - is_use_cache_file=is_use_cache_file) + ptq = PostTrainingQuantization( + executor=exe, + model_dir=model_path, + batch_generator=batch_generator, + batch_nums=batch_nums, + algo=algo, + quantizable_op_type=quantizable_op_type, + round_type=round_type, + is_full_quantize=is_full_quantize, + optimize_model=is_optimize_model, + onnx_format=onnx_format, + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() if onnx_format: ptq._clip_extra = False ptq.save_quantized_model(self.int8_model_path) - def run_test(self, - model_name, - model_url, - model_md5, - data_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - infer_iterations, - quant_iterations, - onnx_format=False): + def run_test( + self, + model_name, + model_url, + model_md5, + data_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + infer_iterations, + quant_iterations, + onnx_format=False, + ): fp32_model_path = self.download_model(model_url, model_md5, model_name) fp32_model_path = os.path.join(fp32_model_path, model_name) data_path = self.download_model(data_url, data_md5, data_name) data_path = os.path.join(data_path, data_name) - print("Start FP32 inference for {0} on {1} samples ...".format( - model_name, infer_iterations)) - (fp32_latency, fp32_acc) = self.run_program(fp32_model_path, data_path, - infer_iterations) - - print("Start post training quantization for {0} on {1} samples ...". - format(model_name, quant_iterations)) - self.generate_quantized_model(fp32_model_path, data_path, algo, - round_type, quantizable_op_type, - is_full_quantize, is_use_cache_file, - is_optimize_model, 10, quant_iterations, - onnx_format) - - print("Start INT8 inference for {0} on {1} samples ...".format( - model_name, infer_iterations)) - (int8_latency, int8_acc) = self.run_program(self.int8_model_path, - data_path, infer_iterations) + print( + "Start FP32 inference for {0} on {1} samples ...".format( + model_name, infer_iterations + ) + ) + (fp32_latency, fp32_acc) = self.run_program( + fp32_model_path, data_path, infer_iterations + ) + + print( + "Start post training quantization for {0} on {1} samples ...".format( + model_name, quant_iterations + ) + ) + self.generate_quantized_model( + fp32_model_path, + data_path, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + 10, + quant_iterations, + onnx_format, + ) + + print( + "Start INT8 inference for {0} on {1} samples ...".format( + model_name, infer_iterations + ) + ) + (int8_latency, int8_acc) = self.run_program( + self.int8_model_path, data_path, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) - print("FP32 {0}: batch_size {1}, latency {2} s, acc {3}.".format( - model_name, 1, fp32_latency, fp32_acc)) - print("INT8 {0}: batch_size {1}, latency {2} s, acc1 {3}.\n".format( - model_name, 1, int8_latency, int8_acc)) + print( + "FP32 {0}: batch_size {1}, latency {2} s, acc {3}.".format( + model_name, 1, fp32_latency, fp32_acc + ) + ) + print( + "INT8 {0}: batch_size {1}, latency {2} s, acc1 {3}.\n".format( + model_name, 1, int8_latency, int8_acc + ) + ) sys.stdout.flush() delta_value = fp32_acc - int8_acc @@ -248,7 +291,6 @@ class TestPostTrainingQuantization(unittest.TestCase): class TestPostTrainingAvgForLSTM(TestPostTrainingQuantization): - def test_post_training_avg(self): model_name = "nlp_lstm_fp32_model" model_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/nlp_lstm_fp32_model.tar.gz" @@ -265,14 +307,26 @@ class TestPostTrainingAvgForLSTM(TestPostTrainingQuantization): diff_threshold = 0.02 infer_iterations = 100 quant_iterations = 10 - self.run_test(model_name, model_url, model_md5, data_name, data_url, - data_md5, algo, round_type, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, infer_iterations, quant_iterations) + self.run_test( + model_name, + model_url, + model_md5, + data_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + infer_iterations, + quant_iterations, + ) class TestPostTrainingAvgForLSTMONNXFormat(TestPostTrainingQuantization): - def not_test_post_training_avg_onnx_format(self): model_name = "nlp_lstm_fp32_model" model_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/nlp_lstm_fp32_model.tar.gz" @@ -290,22 +344,24 @@ class TestPostTrainingAvgForLSTMONNXFormat(TestPostTrainingQuantization): infer_iterations = 100 quant_iterations = 10 onnx_format = True - self.run_test(model_name, - model_url, - model_md5, - data_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - infer_iterations, - quant_iterations, - onnx_format=onnx_format) + self.run_test( + model_name, + model_url, + model_md5, + data_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + infer_iterations, + quant_iterations, + onnx_format=onnx_format, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mnist.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mnist.py index 807bdbf8a9a..6ff54f7c970 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mnist.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mnist.py @@ -33,19 +33,23 @@ np.random.seed(0) class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.root_path = tempfile.TemporaryDirectory() - self.int8_model_path = os.path.join(self.root_path.name, - "post_training_quantization") + self.int8_model_path = os.path.join( + self.root_path.name, "post_training_quantization" + ) self.download_path = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.download_path) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.download_path + ) try: os.system("mkdir -p " + self.int8_model_path) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model_path, str(e))) + print( + "Failed to create {} due to {}".format( + self.int8_model_path, str(e) + ) + ) sys.exit(-1) def tearDown(self): @@ -54,7 +58,8 @@ class TestPostTrainingQuantization(unittest.TestCase): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_model(self, data_url, data_md5, folder_name): @@ -71,8 +76,11 @@ class TestPostTrainingQuantization(unittest.TestCase): print("test model path:" + model_path) place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) val_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size) img_shape = [1, 28, 28] @@ -80,14 +88,17 @@ class TestPostTrainingQuantization(unittest.TestCase): cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(img_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(img_shape) for x in data]).astype( + "float32" + ) input_label = np.array([x[1] for x in data]).astype("int64") t1 = time.time() - out = exe.run(infer_program, - feed={feed_dict[0]: image}, - fetch_list=fetch_targets) + out = exe.run( + infer_program, + feed={feed_dict[0]: image}, + fetch_list=fetch_targets, + ) t2 = time.time() period = t2 - t1 periods.append(period) @@ -105,91 +116,117 @@ class TestPostTrainingQuantization(unittest.TestCase): acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) - def generate_quantized_model(self, - model_path, - algo="KL", - round_type="round", - quantizable_op_type=["conv2d"], - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - batch_size=10, - batch_nums=10, - onnx_format=False, - skip_tensor_list=None, - bias_correction=False): + def generate_quantized_model( + self, + model_path, + algo="KL", + round_type="round", + quantizable_op_type=["conv2d"], + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + batch_size=10, + batch_nums=10, + onnx_format=False, + skip_tensor_list=None, + bias_correction=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) val_reader = paddle.dataset.mnist.train() - ptq = PostTrainingQuantization(executor=exe, - model_dir=model_path, - sample_generator=val_reader, - batch_size=batch_size, - batch_nums=batch_nums, - algo=algo, - quantizable_op_type=quantizable_op_type, - round_type=round_type, - is_full_quantize=is_full_quantize, - optimize_model=is_optimize_model, - bias_correction=bias_correction, - onnx_format=onnx_format, - skip_tensor_list=skip_tensor_list, - is_use_cache_file=is_use_cache_file) + ptq = PostTrainingQuantization( + executor=exe, + model_dir=model_path, + sample_generator=val_reader, + batch_size=batch_size, + batch_nums=batch_nums, + algo=algo, + quantizable_op_type=quantizable_op_type, + round_type=round_type, + is_full_quantize=is_full_quantize, + optimize_model=is_optimize_model, + bias_correction=bias_correction, + onnx_format=onnx_format, + skip_tensor_list=skip_tensor_list, + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() ptq.save_quantized_model(self.int8_model_path) - def run_test(self, - model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size=10, - infer_iterations=10, - quant_iterations=5, - bias_correction=False, - onnx_format=False, - skip_tensor_list=None): + def run_test( + self, + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size=10, + infer_iterations=10, + quant_iterations=5, + bias_correction=False, + onnx_format=False, + skip_tensor_list=None, + ): origin_model_path = self.download_model(data_url, data_md5, model_name) origin_model_path = os.path.join(origin_model_path, model_name) - print("Start FP32 inference for {0} on {1} images ...".format( - model_name, infer_iterations * batch_size)) - (fp32_throughput, fp32_latency, - fp32_acc1) = self.run_program(origin_model_path, batch_size, - infer_iterations) - - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model_name, quant_iterations * batch_size)) - self.generate_quantized_model(origin_model_path, algo, round_type, - quantizable_op_type, is_full_quantize, - is_use_cache_file, is_optimize_model, - batch_size, quant_iterations, onnx_format, - skip_tensor_list, bias_correction) - - print("Start INT8 inference for {0} on {1} images ...".format( - model_name, infer_iterations * batch_size)) - (int8_throughput, int8_latency, - int8_acc1) = self.run_program(self.int8_model_path, batch_size, - infer_iterations) + print( + "Start FP32 inference for {0} on {1} images ...".format( + model_name, infer_iterations * batch_size + ) + ) + (fp32_throughput, fp32_latency, fp32_acc1) = self.run_program( + origin_model_path, batch_size, infer_iterations + ) + + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model_name, quant_iterations * batch_size + ) + ) + self.generate_quantized_model( + origin_model_path, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + batch_size, + quant_iterations, + onnx_format, + skip_tensor_list, + bias_correction, + ) + + print( + "Start INT8 inference for {0} on {1} images ...".format( + model_name, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1) = self.run_program( + self.int8_model_path, batch_size, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}." - .format(model_name, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.".format( + model_name, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.\n" - .format(model_name, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.\n".format( + model_name, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = fp32_acc1 - int8_acc1 @@ -197,10 +234,11 @@ class TestPostTrainingQuantization(unittest.TestCase): class TestPostTrainingKLForMnist(TestPostTrainingQuantization): - def test_post_training_kl(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "KL" round_type = "round" @@ -212,17 +250,29 @@ class TestPostTrainingKLForMnist(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTraininghistForMnist(TestPostTrainingQuantization): - def test_post_training_hist(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "hist" round_type = "round" @@ -234,17 +284,29 @@ class TestPostTraininghistForMnist(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingmseForMnist(TestPostTrainingQuantization): - def test_post_training_mse(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "mse" round_type = "round" @@ -256,17 +318,29 @@ class TestPostTrainingmseForMnist(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingemdForMnist(TestPostTrainingQuantization): - def test_post_training_mse(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "emd" round_type = "round" @@ -278,17 +352,29 @@ class TestPostTrainingemdForMnist(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingavgForMnist(TestPostTrainingQuantization): - def test_post_training_avg(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "avg" round_type = "round" @@ -300,17 +386,29 @@ class TestPostTrainingavgForMnist(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingAbsMaxForMnist(TestPostTrainingQuantization): - def test_post_training_abs_max(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "abs_max" round_type = "round" @@ -322,17 +420,29 @@ class TestPostTrainingAbsMaxForMnist(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 10 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingmseAdaroundForMnist(TestPostTrainingQuantization): - def test_post_training_mse(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "mse" round_type = "adaround" @@ -345,27 +455,30 @@ class TestPostTrainingmseAdaroundForMnist(TestPostTrainingQuantization): infer_iterations = 50 quant_iterations = 5 bias_correction = True - self.run_test(model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - bias_correction=bias_correction) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + bias_correction=bias_correction, + ) class TestPostTrainingKLAdaroundForMnist(TestPostTrainingQuantization): - def test_post_training_kl(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "KL" round_type = "adaround" @@ -377,17 +490,29 @@ class TestPostTrainingKLAdaroundForMnist(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingmseForMnistONNXFormat(TestPostTrainingQuantization): - def test_post_training_mse_onnx_format(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "mse" round_type = "round" @@ -400,28 +525,32 @@ class TestPostTrainingmseForMnistONNXFormat(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - onnx_format=onnx_format) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + onnx_format=onnx_format, + ) class TestPostTrainingmseForMnistONNXFormatFullQuant( - TestPostTrainingQuantization): - + TestPostTrainingQuantization +): def test_post_training_mse_onnx_format_full_quant(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "mse" round_type = "round" @@ -434,27 +563,30 @@ class TestPostTrainingmseForMnistONNXFormatFullQuant( batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - onnx_format=onnx_format) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + onnx_format=onnx_format, + ) class TestPostTrainingavgForMnistSkipOP(TestPostTrainingQuantization): - def test_post_training_avg_skip_op(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "avg" round_type = "round" @@ -467,20 +599,22 @@ class TestPostTrainingavgForMnistSkipOP(TestPostTrainingQuantization): infer_iterations = 50 quant_iterations = 5 skip_tensor_list = ["fc_0.w_0"] - self.run_test(model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - skip_tensor_list=skip_tensor_list) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + skip_tensor_list=skip_tensor_list, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mobilenetv1.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mobilenetv1.py index 41add5e8b8f..37daeab1186 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mobilenetv1.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mobilenetv1.py @@ -77,13 +77,14 @@ def process_image(sample, mode, color_jitter, rotate): return img, sample[1] -def _reader_creator(file_list, - mode, - shuffle=False, - color_jitter=False, - rotate=False, - data_dir=DATA_DIR): - +def _reader_creator( + file_list, + mode, + shuffle=False, + color_jitter=False, + rotate=False, + data_dir=DATA_DIR, +): def reader(): with open(file_list) as flist: full_lines = [line.strip() for line in flist] @@ -98,10 +99,9 @@ def _reader_creator(file_list, continue yield img_path, int(label) - mapper = functools.partial(process_image, - mode=mode, - color_jitter=color_jitter, - rotate=rotate) + mapper = functools.partial( + process_image, mode=mode, color_jitter=color_jitter, rotate=rotate + ) return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE) @@ -112,11 +112,11 @@ def val(data_dir=DATA_DIR): class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.int8_download = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.int8_download) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.int8_download + ) self.data_cache_folder = '' data_urls = [] data_md5s = [] @@ -129,29 +129,34 @@ class TestPostTrainingQuantization(unittest.TestCase): 'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partab' ) data_md5s.append('1e9f15f64e015e58d6f9ec3210ed18b5') - self.data_cache_folder = self.download_data(data_urls, data_md5s, - "full_data", False) + self.data_cache_folder = self.download_data( + data_urls, data_md5s, "full_data", False + ) else: data_urls.append( 'http://paddle-inference-dist.bj.bcebos.com/int8/calibration_test_data.tar.gz' ) data_md5s.append('1b6c1c434172cca1bf9ba1e4d7a3157d') - self.data_cache_folder = self.download_data(data_urls, data_md5s, - "small_data", False) + self.data_cache_folder = self.download_data( + data_urls, data_md5s, "small_data", False + ) # reader/decorator.py requires the relative path to the data folder if not os.path.exists("./data/ILSVRC2012"): - cmd = 'rm -rf {0} && ln -s {1} {0}'.format("data", - self.data_cache_folder) + cmd = 'rm -rf {0} && ln -s {1} {0}'.format( + "data", self.data_cache_folder + ) os.system(cmd) self.batch_size = 1 if os.environ.get('DATASET') == 'full' else 50 - self.infer_iterations = 50000 if os.environ.get( - 'DATASET') == 'full' else 2 + self.infer_iterations = ( + 50000 if os.environ.get('DATASET') == 'full' else 2 + ) self.root_path = tempfile.TemporaryDirectory() - self.int8_model = os.path.join(self.root_path.name, - "post_training_quantization") + self.int8_model = os.path.join( + self.root_path.name, "post_training_quantization" + ) def tearDown(self): self.root_path.cleanup() @@ -159,7 +164,8 @@ class TestPostTrainingQuantization(unittest.TestCase): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_data(self, data_urls, data_md5s, folder_name, is_model=True): @@ -171,13 +177,15 @@ class TestPostTrainingQuantization(unittest.TestCase): download(data_urls[i], self.int8_download, data_md5s[i]) file_names.append(data_urls[i].split('/')[-1]) - zip_path = os.path.join(self.cache_folder, - 'full_imagenet_val.tar.gz') + zip_path = os.path.join( + self.cache_folder, 'full_imagenet_val.tar.gz' + ) if not os.path.exists(zip_path): cat_command = 'cat' for file_name in file_names: - cat_command += ' ' + os.path.join(self.cache_folder, - file_name) + cat_command += ' ' + os.path.join( + self.cache_folder, file_name + ) cat_command += ' > ' + zip_path os.system(cat_command) @@ -197,8 +205,11 @@ class TestPostTrainingQuantization(unittest.TestCase): image_shape = [3, 224, 224] place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) val_reader = paddle.batch(val(), batch_size) iterations = infer_iterations @@ -206,18 +217,18 @@ class TestPostTrainingQuantization(unittest.TestCase): cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(image_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(image_shape) for x in data]).astype( + "float32" + ) label = np.array([x[1] for x in data]).astype("int64") label = label.reshape([-1, 1]) t1 = time.time() - _, acc1, _ = exe.run(infer_program, - feed={ - feed_dict[0]: image, - feed_dict[1]: label - }, - fetch_list=fetch_targets) + _, acc1, _ = exe.run( + infer_program, + feed={feed_dict[0]: image, feed_dict[1]: label}, + fetch_list=fetch_targets, + ) t2 = time.time() period = t2 - t1 periods.append(period) @@ -236,22 +247,25 @@ class TestPostTrainingQuantization(unittest.TestCase): acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) - def generate_quantized_model(self, - model_path, - quantizable_op_type, - batch_size, - algo="KL", - round_type="round", - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - batch_nums=10, - onnx_format=False): + def generate_quantized_model( + self, + model_path, + quantizable_op_type, + batch_size, + algo="KL", + round_type="round", + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + batch_nums=10, + onnx_format=False, + ): try: os.system("mkdir " + self.int8_model) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model, str(e))) + print( + "Failed to create {} due to {}".format(self.int8_model, str(e)) + ) sys.exit(-1) place = fluid.CPUPlace() @@ -259,68 +273,92 @@ class TestPostTrainingQuantization(unittest.TestCase): scope = fluid.global_scope() val_reader = val() - ptq = PostTrainingQuantization(executor=exe, - sample_generator=val_reader, - model_dir=model_path, - batch_size=batch_size, - batch_nums=batch_nums, - algo=algo, - quantizable_op_type=quantizable_op_type, - round_type=round_type, - is_full_quantize=is_full_quantize, - optimize_model=is_optimize_model, - onnx_format=onnx_format, - is_use_cache_file=is_use_cache_file) + ptq = PostTrainingQuantization( + executor=exe, + sample_generator=val_reader, + model_dir=model_path, + batch_size=batch_size, + batch_nums=batch_nums, + algo=algo, + quantizable_op_type=quantizable_op_type, + round_type=round_type, + is_full_quantize=is_full_quantize, + optimize_model=is_optimize_model, + onnx_format=onnx_format, + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() ptq.save_quantized_model(self.int8_model) - def run_test(self, - model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=False, - batch_nums=10): + def run_test( + self, + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=False, + batch_nums=10, + ): infer_iterations = self.infer_iterations batch_size = self.batch_size model_cache_folder = self.download_data(data_urls, data_md5s, model) - print("Start FP32 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) + print( + "Start FP32 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) (fp32_throughput, fp32_latency, fp32_acc1) = self.run_program( - os.path.join(model_cache_folder, "model"), batch_size, - infer_iterations) - - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model, batch_nums * batch_size)) - self.generate_quantized_model(os.path.join(model_cache_folder, "model"), - quantizable_op_type, batch_size, algo, - round_type, is_full_quantize, - is_use_cache_file, is_optimize_model, - batch_nums, onnx_format) - - print("Start INT8 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) - (int8_throughput, int8_latency, - int8_acc1) = self.run_program(self.int8_model, batch_size, - infer_iterations) + os.path.join(model_cache_folder, "model"), + batch_size, + infer_iterations, + ) + + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model, batch_nums * batch_size + ) + ) + self.generate_quantized_model( + os.path.join(model_cache_folder, "model"), + quantizable_op_type, + batch_size, + algo, + round_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + batch_nums, + onnx_format, + ) + + print( + "Start INT8 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1) = self.run_program( + self.int8_model, batch_size, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}." - .format(model, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.".format( + model, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n" - .format(model, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n".format( + model, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = fp32_acc1 - int8_acc1 @@ -328,7 +366,6 @@ class TestPostTrainingQuantization(unittest.TestCase): class TestPostTrainingKLForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_kl_mobilenetv1(self): model = "MobileNet-V1" algo = "KL" @@ -348,13 +385,21 @@ class TestPostTrainingKLForMobilenetv1(TestPostTrainingQuantization): is_optimize_model = True diff_threshold = 0.025 batch_nums = 3 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) class TestPostTrainingavgForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_avg_mobilenetv1(self): model = "MobileNet-V1" algo = "avg" @@ -372,13 +417,21 @@ class TestPostTrainingavgForMobilenetv1(TestPostTrainingQuantization): is_use_cache_file = False is_optimize_model = True diff_threshold = 0.025 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) class TestPostTraininghistForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_hist_mobilenetv1(self): model = "MobileNet-V1" algo = "hist" @@ -397,21 +450,22 @@ class TestPostTraininghistForMobilenetv1(TestPostTrainingQuantization): is_optimize_model = True diff_threshold = 0.03 batch_nums = 3 - self.run_test(model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_nums=batch_nums) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_nums=batch_nums, + ) class TestPostTrainingAbsMaxForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_abs_max_mobilenetv1(self): model = "MobileNet-V1" algo = "abs_max" @@ -429,13 +483,21 @@ class TestPostTrainingAbsMaxForMobilenetv1(TestPostTrainingQuantization): is_optimize_model = False # The accuracy diff of post-training quantization (abs_max) maybe bigger diff_threshold = 0.05 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) class TestPostTrainingAvgONNXFormatForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_onnx_format_mobilenetv1(self): model = "MobileNet-V1" algo = "emd" @@ -455,18 +517,20 @@ class TestPostTrainingAvgONNXFormatForMobilenetv1(TestPostTrainingQuantization): onnx_format = True diff_threshold = 0.05 batch_nums = 3 - self.run_test(model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=onnx_format, - batch_nums=batch_nums) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=onnx_format, + batch_nums=batch_nums, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_program_resnet50.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_program_resnet50.py index 5854d40529d..345853636a4 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_program_resnet50.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_program_resnet50.py @@ -23,8 +23,12 @@ import contextlib import numpy as np import paddle.fluid as fluid from PIL import Image, ImageEnhance -from paddle.fluid.contrib.slim.quantization import PostTrainingQuantizationProgram -from test_post_training_quantization_mobilenetv1 import TestPostTrainingQuantization +from paddle.fluid.contrib.slim.quantization import ( + PostTrainingQuantizationProgram, +) +from test_post_training_quantization_mobilenetv1 import ( + TestPostTrainingQuantization, +) paddle.enable_static() @@ -76,13 +80,14 @@ def process_image(sample, mode, color_jitter, rotate): return img, sample[1] -def _reader_creator(file_list, - mode, - shuffle=False, - color_jitter=False, - rotate=False, - data_dir=DATA_DIR): - +def _reader_creator( + file_list, + mode, + shuffle=False, + color_jitter=False, + rotate=False, + data_dir=DATA_DIR, +): def reader(): with open(file_list) as flist: full_lines = [line.strip() for line in flist] @@ -97,10 +102,9 @@ def _reader_creator(file_list, continue yield img_path, int(label) - mapper = functools.partial(process_image, - mode=mode, - color_jitter=color_jitter, - rotate=rotate) + mapper = functools.partial( + process_image, mode=mode, color_jitter=color_jitter, rotate=rotate + ) return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE) @@ -111,31 +115,33 @@ def val(data_dir=DATA_DIR): class TestPostTrainingQuantizationProgram(TestPostTrainingQuantization): - def run_program(self, model_path, batch_size, infer_iterations): image_shape = [3, 224, 224] place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) val_reader = paddle.batch(val(), batch_size) iterations = infer_iterations test_info = [] cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(image_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(image_shape) for x in data]).astype( + "float32" + ) label = np.array([x[1] for x in data]).astype("int64") label = label.reshape([-1, 1]) t1 = time.time() - _, acc1, _ = exe.run(infer_program, - feed={ - feed_dict[0]: image, - feed_dict[1]: label - }, - fetch_list=fetch_targets) + _, acc1, _ = exe.run( + infer_program, + feed={feed_dict[0]: image, feed_dict[1]: label}, + fetch_list=fetch_targets, + ) t2 = time.time() period = t2 - t1 periods.append(period) @@ -152,10 +158,19 @@ class TestPostTrainingQuantizationProgram(TestPostTrainingQuantization): throughput = cnt / np.sum(periods) latency = np.average(periods) acc1 = np.sum(test_info) / cnt - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) - return (throughput, latency, acc1, infer_program, feed_dict, - fetch_targets) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) + return ( + throughput, + latency, + acc1, + infer_program, + feed_dict, + fetch_targets, + ) def generate_quantized_model( self, @@ -173,25 +188,27 @@ class TestPostTrainingQuantizationProgram(TestPostTrainingQuantization): try: os.system("mkdir " + self.int8_model) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model, str(e))) + print( + "Failed to create {} due to {}".format(self.int8_model, str(e)) + ) sys.exit(-1) place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.global_scope() val_reader = val() - same_scale_tensor_list = [[ - 'batch_norm_3.tmp_2#/#1', 'batch_norm_4.tmp_2#*#1' - ], ['batch_norm_27.tmp_2', 'batch_norm_26.tmp_2'], - [ - 'test_scale_name_not_in_scale_dict1', - 'test_scale_name_not_in_scale_dict2' - ], - [ - 'test_scale_name_not_in_scale_dict1#/#1', - 'test_scale_name_not_in_scale_dict2#/#1' - ]] + same_scale_tensor_list = [ + ['batch_norm_3.tmp_2#/#1', 'batch_norm_4.tmp_2#*#1'], + ['batch_norm_27.tmp_2', 'batch_norm_26.tmp_2'], + [ + 'test_scale_name_not_in_scale_dict1', + 'test_scale_name_not_in_scale_dict2', + ], + [ + 'test_scale_name_not_in_scale_dict1#/#1', + 'test_scale_name_not_in_scale_dict2#/#1', + ], + ] ptq = PostTrainingQuantizationProgram( executor=exe, program=program, @@ -206,56 +223,86 @@ class TestPostTrainingQuantizationProgram(TestPostTrainingQuantization): is_use_cache_file=is_use_cache_file, feed_list=feed_list, fetch_list=fetch_list, - same_scale_tensor_list=same_scale_tensor_list) + same_scale_tensor_list=same_scale_tensor_list, + ) ptq.quantize() ptq.save_quantized_model(self.int8_model) - def run_test(self, - model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=False): + def run_test( + self, + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=False, + ): infer_iterations = self.infer_iterations batch_size = self.batch_size sample_iterations = self.sample_iterations model_cache_folder = self.download_data(data_urls, data_md5s, model) - print("Start FP32 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) - (fp32_throughput, fp32_latency, fp32_acc1, infer_program, feed_dict, - fetch_targets) = self.run_program( - os.path.join(model_cache_folder, "model"), batch_size, - infer_iterations) - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model, sample_iterations * batch_size)) - self.generate_quantized_model(infer_program, quantizable_op_type, - feed_dict, fetch_targets, algo, - round_type, is_full_quantize, - is_use_cache_file, is_optimize_model, - onnx_format) - - print("Start INT8 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) - (int8_throughput, int8_latency, int8_acc1, _, _, - _) = self.run_program(self.int8_model, batch_size, infer_iterations) + print( + "Start FP32 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) + ( + fp32_throughput, + fp32_latency, + fp32_acc1, + infer_program, + feed_dict, + fetch_targets, + ) = self.run_program( + os.path.join(model_cache_folder, "model"), + batch_size, + infer_iterations, + ) + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model, sample_iterations * batch_size + ) + ) + self.generate_quantized_model( + infer_program, + quantizable_op_type, + feed_dict, + fetch_targets, + algo, + round_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + onnx_format, + ) + + print( + "Start INT8 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1, _, _, _) = self.run_program( + self.int8_model, batch_size, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}." - .format(model, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.".format( + model, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n" - .format(model, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n".format( + model, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = fp32_acc1 - int8_acc1 @@ -263,8 +310,8 @@ class TestPostTrainingQuantizationProgram(TestPostTrainingQuantization): class TestPostTrainingProgramAbsMaxForResnet50( - TestPostTrainingQuantizationProgram): - + TestPostTrainingQuantizationProgram +): def test_post_training_abs_max_resnet50(self): model = "ResNet-50" algo = "abs_max" @@ -278,9 +325,18 @@ class TestPostTrainingProgramAbsMaxForResnet50( is_use_cache_file = False is_optimize_model = False diff_threshold = 0.025 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_resnet50.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_resnet50.py index c79499100ce..65e1d391399 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_resnet50.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_resnet50.py @@ -14,14 +14,15 @@ import sys import unittest -from test_post_training_quantization_mobilenetv1 import TestPostTrainingQuantization +from test_post_training_quantization_mobilenetv1 import ( + TestPostTrainingQuantization, +) import paddle paddle.enable_static() class TestPostTrainingForResnet50(TestPostTrainingQuantization): - def test_post_training_resnet50(self): model = "ResNet-50" algo = "min_max" @@ -35,13 +36,21 @@ class TestPostTrainingForResnet50(TestPostTrainingQuantization): is_use_cache_file = False is_optimize_model = False diff_threshold = 0.025 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) class TestPostTrainingForResnet50ONNXFormat(TestPostTrainingQuantization): - def test_post_training_resnet50(self): model = "ResNet-50" algo = "min_max" @@ -56,17 +65,19 @@ class TestPostTrainingForResnet50ONNXFormat(TestPostTrainingQuantization): is_optimize_model = False diff_threshold = 0.025 onnx_format = True - self.run_test(model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=onnx_format) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=onnx_format, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_while.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_while.py index f4eaf5d9bc7..628d120f45e 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_while.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_while.py @@ -32,27 +32,34 @@ np.random.seed(0) class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.download_path = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.download_path) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.download_path + ) self.timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()) - self.int8_model_path = os.path.join(os.getcwd(), - "post_training_" + self.timestamp) + self.int8_model_path = os.path.join( + os.getcwd(), "post_training_" + self.timestamp + ) try: os.system("mkdir -p " + self.int8_model_path) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model_path, str(e))) + print( + "Failed to create {} due to {}".format( + self.int8_model_path, str(e) + ) + ) sys.exit(-1) def tearDown(self): try: os.system("rm -rf {}".format(self.int8_model_path)) except Exception as e: - print("Failed to delete {} due to {}".format( - self.int8_model_path, str(e))) + print( + "Failed to delete {} due to {}".format( + self.int8_model_path, str(e) + ) + ) def cache_unzipping(self, target_folder, zip_path): cmd = 'tar xf {0} -C {1}'.format(zip_path, target_folder) @@ -72,10 +79,16 @@ class TestPostTrainingQuantization(unittest.TestCase): print("test model path:" + model_path) place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, - model_filename='model.pdmodel', - params_filename='model.pdiparams', executor=exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, + model_filename='model.pdmodel', + params_filename='model.pdiparams', + executor=exe, + ) val_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size) img_shape = [1, 28, 28] @@ -83,14 +96,17 @@ class TestPostTrainingQuantization(unittest.TestCase): cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(img_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(img_shape) for x in data]).astype( + "float32" + ) input_label = np.array([x[1] for x in data]).astype("int64") t1 = time.time() - out = exe.run(infer_program, - feed={feed_dict[0]: image}, - fetch_list=fetch_targets) + out = exe.run( + infer_program, + feed={feed_dict[0]: image}, + fetch_list=fetch_targets, + ) t2 = time.time() period = t2 - t1 periods.append(period) @@ -108,16 +124,18 @@ class TestPostTrainingQuantization(unittest.TestCase): acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) - def generate_quantized_model(self, - model_path, - algo="KL", - quantizable_op_type=["conv2d"], - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - batch_size=10, - batch_nums=10, - is_data_loader=False): + def generate_quantized_model( + self, + model_path, + algo="KL", + quantizable_op_type=["conv2d"], + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + batch_size=10, + batch_nums=10, + is_data_loader=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -146,63 +164,81 @@ class TestPostTrainingQuantization(unittest.TestCase): quantizable_op_type=quantizable_op_type, is_full_quantize=is_full_quantize, optimize_model=is_optimize_model, - is_use_cache_file=is_use_cache_file) + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() - ptq.save_quantized_model(self.int8_model_path, - model_filename='model.pdmodel', - params_filename='model.pdiparams') - - def run_test(self, - model_name, - data_url, - data_md5, - algo, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size=10, - infer_iterations=10, - quant_iterations=5, - is_data_loader=False): + ptq.save_quantized_model( + self.int8_model_path, + model_filename='model.pdmodel', + params_filename='model.pdiparams', + ) + + def run_test( + self, + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size=10, + infer_iterations=10, + quant_iterations=5, + is_data_loader=False, + ): origin_model_path = self.download_model(data_url, data_md5, model_name) - #origin_model_path = os.path.join(origin_model_path, model_name) - - print("Start FP32 inference for {0} on {1} images ...".format( - model_name, infer_iterations * batch_size)) - (fp32_throughput, fp32_latency, - fp32_acc1) = self.run_program(origin_model_path, batch_size, - infer_iterations) - - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model_name, quant_iterations * batch_size)) - self.generate_quantized_model(origin_model_path, - algo, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - batch_size, - quant_iterations, - is_data_loader=is_data_loader) - - print("Start INT8 inference for {0} on {1} images ...".format( - model_name, infer_iterations * batch_size)) - (int8_throughput, int8_latency, - int8_acc1) = self.run_program(self.int8_model_path, batch_size, - infer_iterations) + # origin_model_path = os.path.join(origin_model_path, model_name) + + print( + "Start FP32 inference for {0} on {1} images ...".format( + model_name, infer_iterations * batch_size + ) + ) + (fp32_throughput, fp32_latency, fp32_acc1) = self.run_program( + origin_model_path, batch_size, infer_iterations + ) + + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model_name, quant_iterations * batch_size + ) + ) + self.generate_quantized_model( + origin_model_path, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + batch_size, + quant_iterations, + is_data_loader=is_data_loader, + ) + + print( + "Start INT8 inference for {0} on {1} images ...".format( + model_name, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1) = self.run_program( + self.int8_model_path, batch_size, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}." - .format(model_name, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.".format( + model_name, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.\n" - .format(model_name, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.\n".format( + model_name, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = fp32_acc1 - int8_acc1 @@ -210,10 +246,11 @@ class TestPostTrainingQuantization(unittest.TestCase): class TestPostTrainingKLForWhile(TestPostTrainingQuantization): - def test_post_training_kl(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "KL" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -224,17 +261,28 @@ class TestPostTrainingKLForWhile(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTraininghistForWhile(TestPostTrainingQuantization): - def test_post_training_hist(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "hist" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -245,17 +293,28 @@ class TestPostTraininghistForWhile(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingmseForWhile(TestPostTrainingQuantization): - def test_post_training_mse(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "mse" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -266,17 +325,28 @@ class TestPostTrainingmseForWhile(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingavgForWhile(TestPostTrainingQuantization): - def test_post_training_avg(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "avg" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -287,17 +357,28 @@ class TestPostTrainingavgForWhile(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingMinMaxForWhile(TestPostTrainingQuantization): - def test_post_training_min_max(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "min_max" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -308,17 +389,28 @@ class TestPostTrainingMinMaxForWhile(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingAbsMaxForWhile(TestPostTrainingQuantization): - def test_post_training_abs_max(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "abs_max" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -329,23 +421,35 @@ class TestPostTrainingAbsMaxForWhile(TestPostTrainingQuantization): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) - self.run_test(model_name, - data_url, - data_md5, - algo, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - is_data_loader=True) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + is_data_loader=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py index dea0fcd4897..0f7a43ebebd 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestQuant2Int8MkldnnPassMul(unittest.TestCase): - def op_name(self): return "mul" @@ -39,7 +38,8 @@ class TestQuant2Int8MkldnnPassMul(unittest.TestCase): self.mul_weights_size = [3, 5] self.mul_output_size = [1, 5] self.mul_input = np.random.random(self.mul_input_size).astype( - self.dtype) + self.dtype + ) self.mul_weights = np.ones(self.mul_weights_size, self.dtype) self.mul_weights_bad = np.ones([1, 1], self.dtype) self.mul_output = np.ndarray(self.mul_output_size).astype(self.dtype) @@ -49,23 +49,22 @@ class TestQuant2Int8MkldnnPassMul(unittest.TestCase): "mul_input": self.mul_input, "mul_weights": self.mul_weights, "mul_output": self.mul_output, - "mul_weights_bad": self.mul_weights_bad + "mul_weights_bad": self.mul_weights_bad, } def prepare_program_mul(self, program): block = program.global_block() for name in self.variables_mul: - block.create_var(name=name, - dtype="float32", - shape=self.variables_mul[name].shape) - - mul_op1 = block.append_op(type=self.op_name(), - inputs={ - "X": block.var('mul_input'), - "Y": block.var('mul_weights') - }, - outputs={"Out": block.var('mul_output')}, - attrs={'use_mkldnn': self.use_mkldnn}) + block.create_var( + name=name, dtype="float32", shape=self.variables_mul[name].shape + ) + + mul_op1 = block.append_op( + type=self.op_name(), + inputs={"X": block.var('mul_input'), "Y": block.var('mul_weights')}, + outputs={"Out": block.var('mul_output')}, + attrs={'use_mkldnn': self.use_mkldnn}, + ) def test_dequantize_op_weights(self): program = fluid.Program() @@ -80,11 +79,13 @@ class TestQuant2Int8MkldnnPassMul(unittest.TestCase): break assert op_node != "", "op of type %s not found" % self.op_name() - qpass = Quant2Int8MkldnnPass(self.quantized_ops, - _scope=self.scope, - _place=self.place, - _core=core, - _debug=False) + qpass = Quant2Int8MkldnnPass( + self.quantized_ops, + _scope=self.scope, + _place=self.place, + _core=core, + _debug=False, + ) qpass._weight_thresholds["mul_output"] = self.mul_output_scale param = self.scope.var("mul_weights").get_tensor() param.set(self.variables_mul["mul_weights"], self.place) @@ -92,9 +93,30 @@ class TestQuant2Int8MkldnnPassMul(unittest.TestCase): assert np.allclose( self.scope.find_var("mul_weights").get_tensor(), - [[1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.], - [1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.], - [1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.]]) + [ + [ + 1.0 / 127.0, + 2.0 / 127.0, + 3.0 / 127.0, + 4.0 / 127.0, + 5.0 / 127.0, + ], + [ + 1.0 / 127.0, + 2.0 / 127.0, + 3.0 / 127.0, + 4.0 / 127.0, + 5.0 / 127.0, + ], + [ + 1.0 / 127.0, + 2.0 / 127.0, + 3.0 / 127.0, + 4.0 / 127.0, + 5.0 / 127.0, + ], + ], + ) param = self.scope.var("mul_weights").get_tensor() param.set(self.variables_mul["mul_weights_bad"], self.place) @@ -103,13 +125,11 @@ class TestQuant2Int8MkldnnPassMul(unittest.TestCase): class TestQuant2Int8MkldnnPassMatmulV2(TestQuant2Int8MkldnnPassMul): - def op_name(self): return "matmul_v2" class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): - def setUp(self): self.scope = fluid.Scope() self.place = fluid.CPUPlace() @@ -131,7 +151,8 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): self.filter2 = np.random.random(self.filter_size2).astype(self.dtype) self.conv_output = np.ndarray(self.conv_output_size).astype(self.dtype) self.conv_output2 = np.ndarray(self.conv_output2_size).astype( - self.dtype) + self.dtype + ) self.quantized_ops = 'conv2d' self.variables = { "input": self.input, @@ -144,15 +165,12 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): def prepare_program_conv2d(self, program): block = program.global_block() for name in self.variables: - block.create_var(name=name, - dtype="float32", - shape=self.variables[name].shape) + block.create_var( + name=name, dtype="float32", shape=self.variables[name].shape + ) conv2d_op1 = block.append_op( type="conv2d", - inputs={ - "Input": block.var('input'), - 'Filter': block.var('filter') - }, + inputs={"Input": block.var('input'), 'Filter': block.var('filter')}, outputs={"Output": block.var('conv_output')}, attrs={ 'strides': self.stride, @@ -162,13 +180,14 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu': True - }) + 'fuse_relu': True, + }, + ) conv2d_op2 = block.append_op( type="conv2d", inputs={ "Input": block.var('conv_output'), - 'Filter': block.var('filter2') + 'Filter': block.var('filter2'), }, outputs={"Output": block.var('conv_output2')}, attrs={ @@ -178,8 +197,9 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): 'dilations': self.dilations, 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format - }) + 'data_format': self.data_format, + }, + ) def remove_fuse_activation_attribute(self, graph): for op in graph.all_op_nodes(): @@ -204,16 +224,17 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): graph = IrGraph(core.Graph(program.desc), for_test=True) graph = self.remove_fuse_activation_attribute(graph) self.check_graph_before_pass(graph) - quant2_int8_mkldnn_pass = Quant2Int8MkldnnPass(self.quantized_ops, - _scope=self.scope, - _place=self.place, - _core=core, - _debug=False) + quant2_int8_mkldnn_pass = Quant2Int8MkldnnPass( + self.quantized_ops, + _scope=self.scope, + _place=self.place, + _core=core, + _debug=False, + ) graph = quant2_int8_mkldnn_pass._update_activations(graph) self.check_graph_after_pass(graph) class TestQuant2Int8MkldnnPassNearestInterp(unittest.TestCase): - def op_name(self): return "nearest_interp" @@ -236,7 +257,8 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): self.input = np.random.random(self.input_size).astype(self.dtype) self.filter = np.random.random(self.filter_size).astype(self.dtype) self.conv_output = np.ndarray(self.conv_output_size).astype( - self.dtype) + self.dtype + ) # nearest_interp self.out_h = 1 @@ -246,16 +268,20 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): self.data_layout = 'NCHW' self.nearest_interp_output_size = [1, 1, 2, 2] self.nearest_interp_output = np.ndarray( - self.nearest_interp_output_size).astype(self.dtype) + self.nearest_interp_output_size + ).astype(self.dtype) # dropout self.dropout_prob = 0.5 self.dropout_out = np.ndarray( - self.nearest_interp_output_size).astype(self.dtype) + self.nearest_interp_output_size + ).astype(self.dtype) self.dropout_mask = np.ndarray(self.nearest_interp_output_size) self.quantized_ops = { - "conv2d", "nearest_interp", "nearest_interp_v2" + "conv2d", + "nearest_interp", + "nearest_interp_v2", } self.variables = { "input": self.input, @@ -263,55 +289,61 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): "conv_output": self.conv_output, "nearest_interp_output": self.nearest_interp_output, "dropout_out": self.dropout_out, - 'dropout_mask': self.dropout_mask + 'dropout_mask': self.dropout_mask, } def prepare_program(self, program): block = program.global_block() for name in self.variables: - block.create_var(name=name, - dtype="float32", - shape=self.variables[name].shape) - block.append_op(type="conv2d", - inputs={ - "Input": block.var('input'), - 'Filter': block.var('filter') - }, - outputs={"Output": block.var('conv_output')}, - attrs={ - 'strides': self.stride, - 'paddings': self.pad, - 'groups': self.groups, - 'dilations': self.dilations, - 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format, - 'fuse_relu': True - }) - block.append_op(type=self.op_name(), - inputs={ - "X": block.var('conv_output'), - }, - outputs={"Out": block.var('nearest_interp_output')}, - attrs={ - 'interp_method': self.interp_method, - 'out_h': self.out_h, - 'out_w': self.out_w, - 'scale': self.scale, - 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn - }) - block.append_op(type='dropout', - inputs={ - "X": block.var('nearest_interp_output'), - }, - outputs={ - 'Out': block.var('dropout_out'), - 'Mask': block.var('dropout_mask') - }, - attrs={ - 'dropout_prob': self.dropout_prob, - }) + block.create_var( + name=name, dtype="float32", shape=self.variables[name].shape + ) + block.append_op( + type="conv2d", + inputs={ + "Input": block.var('input'), + 'Filter': block.var('filter'), + }, + outputs={"Output": block.var('conv_output')}, + attrs={ + 'strides': self.stride, + 'paddings': self.pad, + 'groups': self.groups, + 'dilations': self.dilations, + 'use_cudnn': self.use_cudnn, + 'use_mkldnn': self.use_mkldnn, + 'data_format': self.data_format, + 'fuse_relu': True, + }, + ) + block.append_op( + type=self.op_name(), + inputs={ + "X": block.var('conv_output'), + }, + outputs={"Out": block.var('nearest_interp_output')}, + attrs={ + 'interp_method': self.interp_method, + 'out_h': self.out_h, + 'out_w': self.out_w, + 'scale': self.scale, + 'data_layout': self.data_layout, + 'use_mkldnn': self.use_mkldnn, + }, + ) + block.append_op( + type='dropout', + inputs={ + "X": block.var('nearest_interp_output'), + }, + outputs={ + 'Out': block.var('dropout_out'), + 'Mask': block.var('dropout_mask'), + }, + attrs={ + 'dropout_prob': self.dropout_prob, + }, + ) def check_graph_after_pass(self, graph): for op in graph.all_op_nodes(): @@ -329,12 +361,21 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): _scope=self.scope, _place=self.place, _core=core, - _debug=False) - - input_scale_tensor = quant2_int8_mkldnn_pass._convert_scale2tensor( - np.array(self.scale).astype(np.float64)) - output_scale_tensor = quant2_int8_mkldnn_pass._convert_scale2tensor( - np.array(1. / self.scale * self.scale).astype(np.float64)) + _debug=False, + ) + + input_scale_tensor = ( + quant2_int8_mkldnn_pass._convert_scale2tensor( + np.array(self.scale).astype(np.float64) + ) + ) + output_scale_tensor = ( + quant2_int8_mkldnn_pass._convert_scale2tensor( + np.array(1.0 / self.scale * self.scale).astype( + np.float64 + ) + ) + ) var_scale = { "input": (False, input_scale_tensor), "filter": (False, input_scale_tensor), @@ -347,7 +388,6 @@ class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): self.check_graph_after_pass(graph) class TestQuant2Int8MkldnnPassNearestInterpV2(unittest.TestCase): - def op_name(self): return "nearest_interp_v2" diff --git a/python/paddle/fluid/contrib/slim/tests/test_quantization_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quantization_mkldnn_pass.py index da3b0139e02..23b89512454 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quantization_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quantization_mkldnn_pass.py @@ -29,19 +29,23 @@ os.environ["CPU_NUM"] = "1" def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) avg_loss = paddle.mean(loss) @@ -49,12 +53,11 @@ def conv_net(img, label): class TestMKLDNNTransformBasedFreezePass(unittest.TestCase): - def setUp(self): self.quantizable_op_and_inputs = { 'conv2d': ['Input', 'Filter'], 'depthwise_conv2d': ['Input', 'Filter'], - 'mul': ['X', 'Y'] + 'mul': ['X', 'Y'], } def check_program(self, program): @@ -74,25 +77,27 @@ class TestMKLDNNTransformBasedFreezePass(unittest.TestCase): startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) return [img, label], loss - def mkldnn_based_freeze_graph(self, - use_cuda, - seed, - activation_quant_type, - weight_quant_type='abs_max', - quant_perf=False, - for_ci=False): + def mkldnn_based_freeze_graph( + self, + use_cuda, + seed, + activation_quant_type, + weight_quant_type='abs_max', + quant_perf=False, + for_ci=False, + ): random.seed(0) np.random.seed(0) @@ -115,42 +120,48 @@ class TestMKLDNNTransformBasedFreezePass(unittest.TestCase): scope=scope, place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) transform_pass.apply(main_graph) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) transform_pass.apply(test_graph) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) quantized_test_program = test_graph.to_program() iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) # Training the model to get the weights value with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( - scope=scope, place=place, weight_quantize_type=weight_quant_type) + scope=scope, place=place, weight_quantize_type=weight_quant_type + ) freeze_pass.apply(test_graph) # Transform quantized graph for MKL-DNN INT8 inference @@ -163,8 +174,14 @@ class TestMKLDNNTransformBasedFreezePass(unittest.TestCase): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test_mkldnn' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test_mkldnn' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) mkldnn_program = test_graph.to_program() # Check the transformation weights of conv2d and mul @@ -178,9 +195,16 @@ class TestMKLDNNTransformBasedFreezePass(unittest.TestCase): # output self.check_program(mkldnn_program) if not for_ci: - print('{}: {}'.format( - 'w_mkldnn' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_mkldnn))) + print( + '{}: {}'.format( + 'w_mkldnn' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_mkldnn), + ) + ) def test_mkldnn_graph_cpu_static(self): with fluid.unique_name.guard(): @@ -189,13 +213,15 @@ class TestMKLDNNTransformBasedFreezePass(unittest.TestCase): seed=2, activation_quant_type='range_abs_max', weight_quant_type='abs_max', - for_ci=True) + for_ci=True, + ) self.mkldnn_based_freeze_graph( False, seed=2, activation_quant_type='moving_average_abs_max', weight_quant_type='abs_max', - for_ci=True) + for_ci=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_quantization_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quantization_pass.py index 4310c259ca5..be42ab5cf2e 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quantization_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quantization_pass.py @@ -45,50 +45,47 @@ def linear_fc(num): def residual_block(num, quant_skip_pattern=None): - - def conv_bn_layer(input, - ch_out, - filter_size, - stride, - padding, - act='relu', - bias_attr=False): - tmp = fluid.layers.conv2d(input=input, - filter_size=filter_size, - num_filters=ch_out, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) + def conv_bn_layer( + input, ch_out, filter_size, stride, padding, act='relu', bias_attr=False + ): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr, + ) return fluid.layers.batch_norm(input=tmp, act=act) - data = fluid.layers.data(name='image', - shape=[1, 1, 32, 32], - dtype='float32', - append_batch_size=False) - label = fluid.layers.data(name='label', - shape=[1, 1], - dtype='int64', - append_batch_size=False) + data = fluid.layers.data( + name='image', + shape=[1, 1, 32, 32], + dtype='float32', + append_batch_size=False, + ) + label = fluid.layers.data( + name='label', shape=[1, 1], dtype='int64', append_batch_size=False + ) hidden = data for _ in range(num): conv = conv_bn_layer(hidden, 16, 3, 1, 1, act=None, bias_attr=True) short = conv_bn_layer(hidden, 16, 1, 1, 0, act=None) hidden = fluid.layers.elementwise_add(x=conv, y=short, act='relu') - matmul_weight = fluid.layers.create_parameter(shape=[1, 16, 32, 32], - dtype='float32') + matmul_weight = fluid.layers.create_parameter( + shape=[1, 16, 32, 32], dtype='float32' + ) hidden = fluid.layers.matmul(hidden, matmul_weight, True, True) if quant_skip_pattern: with fluid.name_scope(quant_skip_pattern): - pool = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) + pool = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) else: - pool = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) + pool = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) fc = fluid.layers.fc(input=pool, size=10) loss = fluid.layers.cross_entropy(input=fc, label=label) loss = paddle.mean(loss) @@ -96,21 +93,25 @@ def residual_block(num, quant_skip_pattern=None): def conv_net(img, label, quant_skip_pattern): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) hidden = fluid.layers.fc(input=conv_pool_2, size=100, act='relu') with fluid.name_scope(quant_skip_pattern): prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') @@ -120,17 +121,16 @@ def conv_net(img, label, quant_skip_pattern): class TestQuantizationTransformPass(unittest.TestCase): - def setUp(self): self.quantizable_op_and_inputs = { 'conv2d': ['Input', 'Filter'], 'depthwise_conv2d': ['Input', 'Filter'], - 'mul': ['X', 'Y'] + 'mul': ['X', 'Y'], } self.quantizable_grad_op_inputs = { 'conv2d_grad': ['Input', 'Filter'], 'depthwise_conv2d_grad': ['Input', 'Filter'], - 'mul_grad': ['X', 'Y'] + 'mul_grad': ['X', 'Y'], } def check_program(self, program): @@ -141,7 +141,8 @@ class TestQuantizationTransformPass(unittest.TestCase): if op.type in self.quantizable_op_and_inputs: for arg_name in op.input_arg_names: self.assertTrue( - arg_name.endswith('.quantized.dequantized')) + arg_name.endswith('.quantized.dequantized') + ) quantized_ops.add(arg_name) for op in block.ops: @@ -150,13 +151,13 @@ class TestQuantizationTransformPass(unittest.TestCase): for pname in self.quantizable_grad_op_inputs[op.type]: arg_name = op.input(pname)[0] self.assertTrue( - arg_name.endswith('.quantized.dequantized')) + arg_name.endswith('.quantized.dequantized') + ) self.assertTrue(arg_name in quantized_ops) - def linear_fc_quant(self, - activation_quant_type, - weight_quantize_type, - for_ci=True): + def linear_fc_quant( + self, activation_quant_type, weight_quantize_type, for_ci=True + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -169,15 +170,17 @@ class TestQuantizationTransformPass(unittest.TestCase): scope=fluid.global_scope(), place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quantize_type) + weight_quantize_type=weight_quantize_type, + ) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) - graph.draw('.', 'quantize_fc_' + activation_quant_type, - marked_nodes) + graph.draw( + '.', 'quantize_fc_' + activation_quant_type, marked_nodes + ) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) @@ -186,8 +189,9 @@ class TestQuantizationTransformPass(unittest.TestCase): for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) - val_graph.draw('.', 'val_fc_' + activation_quant_type, - val_marked_nodes) + val_graph.draw( + '.', 'val_fc_' + activation_quant_type, val_marked_nodes + ) def test_linear_fc_quant_abs_max(self): self.linear_fc_quant('abs_max', 'abs_max', for_ci=True) @@ -196,15 +200,17 @@ class TestQuantizationTransformPass(unittest.TestCase): self.linear_fc_quant('range_abs_max', 'abs_max', for_ci=True) def test_linear_fc_quant_moving_average_abs_max(self): - self.linear_fc_quant('moving_average_abs_max', - 'channel_wise_abs_max', - for_ci=True) - - def residual_block_quant(self, - activation_quant_type, - weight_quantize_type, - quantizable_op_type, - for_ci=True): + self.linear_fc_quant( + 'moving_average_abs_max', 'channel_wise_abs_max', for_ci=True + ) + + def residual_block_quant( + self, + activation_quant_type, + weight_quantize_type, + quantizable_op_type, + for_ci=True, + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -218,15 +224,17 @@ class TestQuantizationTransformPass(unittest.TestCase): place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quantize_type, - quantizable_op_type=quantizable_op_type) + quantizable_op_type=quantizable_op_type, + ) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) - graph.draw('.', 'quantize_residual_' + activation_quant_type, - marked_nodes) + graph.draw( + '.', 'quantize_residual_' + activation_quant_type, marked_nodes + ) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) @@ -235,53 +243,54 @@ class TestQuantizationTransformPass(unittest.TestCase): for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) - val_graph.draw('.', 'val_residual_' + activation_quant_type, - val_marked_nodes) + val_graph.draw( + '.', 'val_residual_' + activation_quant_type, val_marked_nodes + ) def test_residual_block_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('abs_max', - 'abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'abs_max', 'abs_max', quantizable_op_type, for_ci=True + ) def test_residual_block_range_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('range_abs_max', - 'abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'range_abs_max', 'abs_max', quantizable_op_type, for_ci=True + ) def test_residual_block_moving_average_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('moving_average_abs_max', - 'channel_wise_abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'moving_average_abs_max', + 'channel_wise_abs_max', + quantizable_op_type, + for_ci=True, + ) class TestQuantizationFreezePass(unittest.TestCase): - - def freeze_graph(self, - use_cuda, - seed, - activation_quant_type, - bias_correction=False, - weight_quant_type='abs_max', - for_ci=True, - quant_skip_pattern='skip_quant'): - + def freeze_graph( + self, + use_cuda, + seed, + activation_quant_type, + bias_correction=False, + weight_quant_type='abs_max', + for_ci=True, + quant_skip_pattern='skip_quant', + ): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label, quant_skip_pattern) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.001) @@ -310,14 +319,16 @@ class TestQuantizationFreezePass(unittest.TestCase): place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type, - skip_pattern=quant_skip_pattern) + skip_pattern=quant_skip_pattern, + ) transform_pass.apply(main_graph) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type, - skip_pattern=quant_skip_pattern) + skip_pattern=quant_skip_pattern, + ) transform_pass.apply(test_graph) dev_name = '_gpu_' if use_cuda else '_cpu_' if not for_ci: @@ -326,57 +337,85 @@ class TestQuantizationFreezePass(unittest.TestCase): if op.name().find('quantize') > -1: marked_nodes.add(op) main_graph.draw( - '.', 'main' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'main' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) marked_nodes = set() for op in test_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) quantized_test_program = test_graph.to_program() iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) if not for_ci: - print('{}: {}'.format( - 'loss' + dev_name + activation_quant_type + '_' + - weight_quant_type, loss_v)) + print( + '{}: {}'.format( + 'loss' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + loss_v, + ) + ) test_data = next(test_reader()) with fluid.program_guard(quantized_test_program): - w_var = fluid.framework._get_var('conv2d_1.w_0.quantized', - quantized_test_program) + w_var = fluid.framework._get_var( + 'conv2d_1.w_0.quantized', quantized_test_program + ) # Testing with fluid.scope_guard(scope): - test_loss1, w_quant = exe.run(program=quantized_test_program, - feed=feeder.feed(test_data), - fetch_list=[loss, w_var]) + test_loss1, w_quant = exe.run( + program=quantized_test_program, + feed=feeder.feed(test_data), + fetch_list=[loss, w_var], + ) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( - scope=scope, place=place, bias_correction=bias_correction, \ - weight_quantize_type=weight_quant_type) + scope=scope, + place=place, + bias_correction=bias_correction, + weight_quantize_type=weight_quant_type, + ) freeze_pass.apply(test_graph) if not for_ci: marked_nodes = set() @@ -384,32 +423,68 @@ class TestQuantizationFreezePass(unittest.TestCase): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test_freeze' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test_freeze' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) server_program = test_graph.to_program() with fluid.scope_guard(scope): - test_loss2, = exe.run(program=server_program, - feed=feeder.feed(test_data), - fetch_list=[loss]) + (test_loss2,) = exe.run( + program=server_program, + feed=feeder.feed(test_data), + fetch_list=[loss], + ) self.assertAlmostEqual(test_loss1, test_loss2, delta=5e-3) if not for_ci: - print('{}: {}'.format( - 'test_loss1' + dev_name + activation_quant_type + '_' + - weight_quant_type, test_loss1)) - print('{}: {}'.format( - 'test_loss2' + dev_name + activation_quant_type + '_' + - weight_quant_type, test_loss2)) + print( + '{}: {}'.format( + 'test_loss1' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + test_loss1, + ) + ) + print( + '{}: {}'.format( + 'test_loss2' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + test_loss2, + ) + ) w_freeze = np.array(scope.find_var('conv2d_1.w_0').get_tensor()) # Maybe failed, this is due to the calculation precision # self.assertAlmostEqual(np.sum(w_freeze), np.sum(w_quant)) if not for_ci: - print('{}: {}'.format( - 'w_freeze' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_freeze))) - print('{}: {}'.format( - 'w_quant' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_quant))) + print( + '{}: {}'.format( + 'w_freeze' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_freeze), + ) + ) + print( + '{}: {}'.format( + 'w_quant' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_quant), + ) + ) # Convert parameter to 8-bit. convert_int8_pass = ConvertToInt8Pass(scope=scope, place=place) @@ -420,30 +495,62 @@ class TestQuantizationFreezePass(unittest.TestCase): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test_int8' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test_int8' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) server_program_int8 = test_graph.to_program() # Save the 8-bit parameter and model file. with fluid.scope_guard(scope): fluid.io.save_inference_model( - 'server_int8' + dev_name + activation_quant_type + '_' + - weight_quant_type, ['image', 'label'], [loss], exe, - server_program_int8) + 'server_int8' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + ['image', 'label'], + [loss], + exe, + server_program_int8, + ) # Test whether the 8-bit parameter and model file can be loaded successfully. [infer, feed, fetch] = fluid.io.load_inference_model( - 'server_int8' + dev_name + activation_quant_type + '_' + - weight_quant_type, exe) + 'server_int8' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + exe, + ) # Check the loaded 8-bit weight. w_8bit = np.array(scope.find_var('conv2d_1.w_0.int8').get_tensor()) self.assertEqual(w_8bit.dtype, np.int8) self.assertEqual(np.sum(w_8bit), np.sum(w_freeze)) if not for_ci: - print('{}: {}'.format( - 'w_8bit' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_8bit))) - print('{}: {}'.format( - 'w_freeze' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_freeze))) + print( + '{}: {}'.format( + 'w_8bit' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_8bit), + ) + ) + print( + '{}: {}'.format( + 'w_freeze' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_freeze), + ) + ) mobile_pass = TransformForMobilePass() mobile_pass.apply(test_graph) @@ -453,129 +560,164 @@ class TestQuantizationFreezePass(unittest.TestCase): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test_mobile' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test_mobile' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) mobile_program = test_graph.to_program() with fluid.scope_guard(scope): fluid.io.save_inference_model( - 'mobile_int8' + dev_name + activation_quant_type + '_' + - weight_quant_type, ['image', 'label'], [loss], exe, - mobile_program) + 'mobile_int8' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + ['image', 'label'], + [loss], + exe, + mobile_program, + ) def test_freeze_graph_cuda_dynamic(self): if fluid.core.is_compiled_with_cuda(): with fluid.unique_name.guard(): - self.freeze_graph(True, - seed=1, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=True) + self.freeze_graph( + True, + seed=1, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) with fluid.unique_name.guard(): - self.freeze_graph(True, - seed=1, - activation_quant_type='abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + self.freeze_graph( + True, + seed=1, + activation_quant_type='abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) def test_freeze_graph_cpu_dynamic(self): with fluid.unique_name.guard(): - self.freeze_graph(False, - seed=2, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(False, - seed=2, - activation_quant_type='abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + self.freeze_graph( + False, + seed=2, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) + self.freeze_graph( + False, + seed=2, + activation_quant_type='abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) def test_freeze_graph_cuda_static(self): if fluid.core.is_compiled_with_cuda(): with fluid.unique_name.guard(): - self.freeze_graph(True, - seed=1, - activation_quant_type='range_abs_max', - bias_correction=True, - weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(True, - seed=1, - activation_quant_type='range_abs_max', - weight_quant_type='abs_max', - for_ci=True) + self.freeze_graph( + True, + seed=1, + activation_quant_type='range_abs_max', + bias_correction=True, + weight_quant_type='abs_max', + for_ci=True, + ) + self.freeze_graph( + True, + seed=1, + activation_quant_type='range_abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) self.freeze_graph( True, seed=1, activation_quant_type='moving_average_abs_max', weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(True, - seed=1, - activation_quant_type='range_abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) + self.freeze_graph( + True, + seed=1, + activation_quant_type='range_abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) self.freeze_graph( True, seed=1, activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) self.freeze_graph( True, seed=1, activation_quant_type='moving_average_abs_max', bias_correction=True, weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) def test_freeze_graph_cpu_static(self): with fluid.unique_name.guard(): - self.freeze_graph(False, - seed=2, - activation_quant_type='range_abs_max', - weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(False, - seed=2, - activation_quant_type='moving_average_abs_max', - weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(False, - seed=2, - activation_quant_type='range_abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) - self.freeze_graph(False, - seed=2, - activation_quant_type='moving_average_abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + self.freeze_graph( + False, + seed=2, + activation_quant_type='range_abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) + self.freeze_graph( + False, + seed=2, + activation_quant_type='moving_average_abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) + self.freeze_graph( + False, + seed=2, + activation_quant_type='range_abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) + self.freeze_graph( + False, + seed=2, + activation_quant_type='moving_average_abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) def quant_dequant_residual_block(num, quant_skip_pattern=None): - - def conv_bn_layer(input, - ch_out, - filter_size, - stride, - padding, - act='relu', - bias_attr=False): - tmp = fluid.layers.conv2d(input=input, - filter_size=filter_size, - num_filters=ch_out, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) + def conv_bn_layer( + input, ch_out, filter_size, stride, padding, act='relu', bias_attr=False + ): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr, + ) return fluid.layers.batch_norm(input=tmp, act=act) data1 = fluid.layers.data(name='image', shape=[1, 32, 32], dtype='float32') - data2 = fluid.layers.data(name='matmul_input', - shape=[16, 32, 32], - dtype='float32') + data2 = fluid.layers.data( + name='matmul_input', shape=[16, 32, 32], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') hidden = data1 for _ in range(num): @@ -585,43 +727,37 @@ def quant_dequant_residual_block(num, quant_skip_pattern=None): hidden = fluid.layers.matmul(hidden, data2, True, True) if isinstance(quant_skip_pattern, str): with fluid.name_scope(quant_skip_pattern): - pool1 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) - pool2 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='max', - pool_stride=2) - pool_add = fluid.layers.elementwise_add(x=pool1, - y=pool2, - act='relu') + pool1 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) + pool2 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='max', pool_stride=2 + ) + pool_add = fluid.layers.elementwise_add( + x=pool1, y=pool2, act='relu' + ) elif isinstance(quant_skip_pattern, list): - assert len( - quant_skip_pattern - ) > 1, 'test config error: the len of quant_skip_pattern list should be greater than 1.' + assert ( + len(quant_skip_pattern) > 1 + ), 'test config error: the len of quant_skip_pattern list should be greater than 1.' with fluid.name_scope(quant_skip_pattern[0]): - pool1 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) - pool2 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='max', - pool_stride=2) + pool1 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) + pool2 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='max', pool_stride=2 + ) with fluid.name_scope(quant_skip_pattern[1]): - pool_add = fluid.layers.elementwise_add(x=pool1, - y=pool2, - act='relu') + pool_add = fluid.layers.elementwise_add( + x=pool1, y=pool2, act='relu' + ) else: - pool1 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) - pool2 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='max', - pool_stride=2) + pool1 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) + pool2 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='max', pool_stride=2 + ) pool_add = fluid.layers.elementwise_add(x=pool1, y=pool2, act='relu') fc = fluid.layers.fc(input=pool_add, size=10) loss = fluid.layers.cross_entropy(input=fc, label=label) @@ -630,7 +766,6 @@ def quant_dequant_residual_block(num, quant_skip_pattern=None): class TestAddQuantDequantPass(unittest.TestCase): - def setUp(self): self._target_ops = {'elementwise_add', 'pool2d'} self._target_grad_ops = {'elementwise_add_grad', 'pool2d_grad'} @@ -641,32 +776,40 @@ class TestAddQuantDequantPass(unittest.TestCase): if op_node.name() in self._target_ops: user_skipped = False if isinstance(skip_pattern, list): - user_skipped = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") for pattern in skip_pattern) + user_skipped = op_node.op().has_attr( + "op_namescope" + ) and any( + pattern in op_node.op().attr("op_namescope") + for pattern in skip_pattern + ) elif isinstance(skip_pattern, str): - user_skipped = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find(skip_pattern) != -1 + user_skipped = ( + op_node.op().has_attr("op_namescope") + and op_node.op().attr("op_namescope").find(skip_pattern) + != -1 + ) if user_skipped: continue in_nodes_all_not_persistable = True for input_name in op_node.input_arg_names(): - in_node = graph._find_node_by_name(op_node.inputs, - input_name) - in_nodes_all_not_persistable = (in_nodes_all_not_persistable - and - not in_node.persistable()) + in_node = graph._find_node_by_name( + op_node.inputs, input_name + ) + in_nodes_all_not_persistable = ( + in_nodes_all_not_persistable + and not in_node.persistable() + ) if not in_nodes_all_not_persistable: continue input_names = op_node.input_arg_names() for input_name in input_names: self.assertTrue(input_name.endswith('.quant_dequant')) - def residual_block_quant(self, - quantizable_op_type, - skip_pattern=None, - for_ci=True): + def residual_block_quant( + self, quantizable_op_type, skip_pattern=None, for_ci=True + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -679,7 +822,8 @@ class TestAddQuantDequantPass(unittest.TestCase): scope=fluid.global_scope(), place=place, skip_pattern=skip_pattern, - quantizable_op_type=quantizable_op_type) + quantizable_op_type=quantizable_op_type, + ) add_quant_dequant_pass.apply(graph) if not for_ci: marked_nodes = set() @@ -699,35 +843,36 @@ class TestAddQuantDequantPass(unittest.TestCase): def test_residual_block(self): quantizable_op_type = ['elementwise_add', 'pool2d', 'mul', 'matmul'] - self.residual_block_quant(quantizable_op_type, - skip_pattern=None, - for_ci=True) + self.residual_block_quant( + quantizable_op_type, skip_pattern=None, for_ci=True + ) def test_residual_block_skip_pattern(self): quantizable_op_type = ['elementwise_add', 'pool2d', 'mul', 'matmul'] - self.residual_block_quant(quantizable_op_type, - skip_pattern='skip_quant', - for_ci=True) + self.residual_block_quant( + quantizable_op_type, skip_pattern='skip_quant', for_ci=True + ) def test_residual_block_skip_pattern_1(self): quantizable_op_type = ['elementwise_add', 'pool2d', 'mul', 'matmul'] - self.residual_block_quant(quantizable_op_type, - skip_pattern=['skip_quant1', 'skip_quant2'], - for_ci=True) + self.residual_block_quant( + quantizable_op_type, + skip_pattern=['skip_quant1', 'skip_quant2'], + for_ci=True, + ) class TestQuantizationTransformPassV2(unittest.TestCase): - def setUp(self): self.quantizable_op_and_inputs = { 'conv2d': ['Input', 'Filter'], 'depthwise_conv2d': ['Input', 'Filter'], - 'mul': ['X', 'Y'] + 'mul': ['X', 'Y'], } self.quantizable_grad_op_inputs = { 'conv2d_grad': ['Input', 'Filter'], 'depthwise_conv2d_grad': ['Input', 'Filter'], - 'mul_grad': ['X', 'Y'] + 'mul_grad': ['X', 'Y'], } def check_program(self, program): @@ -738,7 +883,8 @@ class TestQuantizationTransformPassV2(unittest.TestCase): if op.type in self.quantizable_op_and_inputs: for arg_name in op.input_arg_names: self.assertTrue( - arg_name.endswith('.quantized.dequantized')) + arg_name.endswith('.quantized.dequantized') + ) quantized_ops.add(arg_name) for op in block.ops: @@ -747,13 +893,13 @@ class TestQuantizationTransformPassV2(unittest.TestCase): for pname in self.quantizable_grad_op_inputs[op.type]: arg_name = op.input(pname)[0] self.assertTrue( - arg_name.endswith('.quantized.dequantized')) + arg_name.endswith('.quantized.dequantized') + ) self.assertTrue(arg_name in quantized_ops) - def linear_fc_quant(self, - activation_quant_type, - weight_quantize_type, - for_ci=True): + def linear_fc_quant( + self, activation_quant_type, weight_quantize_type, for_ci=True + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -766,15 +912,17 @@ class TestQuantizationTransformPassV2(unittest.TestCase): scope=fluid.global_scope(), place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quantize_type) + weight_quantize_type=weight_quantize_type, + ) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) - graph.draw('.', 'quantize_fc_' + activation_quant_type, - marked_nodes) + graph.draw( + '.', 'quantize_fc_' + activation_quant_type, marked_nodes + ) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) @@ -783,8 +931,9 @@ class TestQuantizationTransformPassV2(unittest.TestCase): for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) - val_graph.draw('.', 'val_fc_' + activation_quant_type, - val_marked_nodes) + val_graph.draw( + '.', 'val_fc_' + activation_quant_type, val_marked_nodes + ) def test_linear_fc_quant_abs_max(self): self.linear_fc_quant('abs_max', 'abs_max', for_ci=True) @@ -792,11 +941,13 @@ class TestQuantizationTransformPassV2(unittest.TestCase): def test_linear_fc_quant_channel_wise_abs_max(self): self.linear_fc_quant('abs_max', 'channel_wise_abs_max', for_ci=True) - def residual_block_quant(self, - activation_quant_type, - weight_quantize_type, - quantizable_op_type, - for_ci=True): + def residual_block_quant( + self, + activation_quant_type, + weight_quantize_type, + quantizable_op_type, + for_ci=True, + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -810,15 +961,17 @@ class TestQuantizationTransformPassV2(unittest.TestCase): place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quantize_type, - quantizable_op_type=quantizable_op_type) + quantizable_op_type=quantizable_op_type, + ) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) - graph.draw('.', 'quantize_residual_' + activation_quant_type, - marked_nodes) + graph.draw( + '.', 'quantize_residual_' + activation_quant_type, marked_nodes + ) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) @@ -827,22 +980,21 @@ class TestQuantizationTransformPassV2(unittest.TestCase): for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) - val_graph.draw('.', 'val_residual_' + activation_quant_type, - val_marked_nodes) + val_graph.draw( + '.', 'val_residual_' + activation_quant_type, val_marked_nodes + ) def test_residual_block_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('abs_max', - 'abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'abs_max', 'abs_max', quantizable_op_type, for_ci=True + ) def test_residual_block_channel_wise_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('abs_max', - 'channel_wise_abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'abs_max', 'channel_wise_abs_max', quantizable_op_type, for_ci=True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_quantization_scale_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quantization_scale_pass.py index a8466ad7371..46e37002460 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quantization_scale_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quantization_scale_pass.py @@ -34,21 +34,25 @@ os.environ["CPU_NUM"] = "1" def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) hidden = fluid.layers.fc(input=conv_pool_2, size=100, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -57,25 +61,25 @@ def conv_net(img, label): class TestQuantizationScalePass(unittest.TestCase): - - def quantization_scale(self, - use_cuda, - seed, - activation_quant_type, - weight_quant_type='abs_max', - for_ci=False): - + def quantization_scale( + self, + use_cuda, + seed, + activation_quant_type, + weight_quant_type='abs_max', + for_ci=False, + ): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.0001) @@ -104,7 +108,8 @@ class TestQuantizationScalePass(unittest.TestCase): scope=scope, place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) transform_pass.apply(main_graph) transform_pass.apply(test_graph) @@ -133,20 +138,22 @@ class TestQuantizationScalePass(unittest.TestCase): build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) if not for_ci: print('{}: {}'.format('loss' + dev_name, loss_v)) @@ -155,7 +162,8 @@ class TestQuantizationScalePass(unittest.TestCase): # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( - scope=scope, place=place, weight_quantize_type=weight_quant_type) + scope=scope, place=place, weight_quantize_type=weight_quant_type + ) freeze_pass.apply(test_graph) server_program = test_graph.to_program() @@ -168,16 +176,21 @@ class TestQuantizationScalePass(unittest.TestCase): tempdir = tempfile.TemporaryDirectory() mapping_table_path = os.path.join( - tempdir.name, 'quant_scale_model' + dev_name + '.txt') + tempdir.name, 'quant_scale_model' + dev_name + '.txt' + ) save_path = os.path.join(tempdir.name, 'quant_scale_model' + dev_name) with open(mapping_table_path, 'w') as f: f.write(str(server_program)) with fluid.scope_guard(scope): - fluid.io.save_inference_model(save_path, ['image', 'label'], [loss], - exe, - server_program, - clip_extra=True) + fluid.io.save_inference_model( + save_path, + ['image', 'label'], + [loss], + exe, + server_program, + clip_extra=True, + ) tempdir.cleanup() def test_quant_scale_cuda(self): @@ -188,7 +201,8 @@ class TestQuantizationScalePass(unittest.TestCase): seed=1, activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) def test_quant_scale_cpu(self): with fluid.unique_name.guard(): @@ -197,7 +211,8 @@ class TestQuantizationScalePass(unittest.TestCase): seed=2, activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_quantize_transpiler_v2.py b/python/paddle/fluid/contrib/slim/tests/test_quantize_transpiler_v2.py index 3893177cbf0..3b487ae1fae 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quantize_transpiler_v2.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quantize_transpiler_v2.py @@ -19,7 +19,9 @@ import numpy as np import paddle.fluid as fluid import paddle from paddle.fluid.framework import IrGraph -from paddle.fluid.contrib.slim.quantization.quantize_transpiler_v2 import QuantizeTranspilerV2 +from paddle.fluid.contrib.slim.quantization.quantize_transpiler_v2 import ( + QuantizeTranspilerV2, +) from paddle.fluid import core paddle.enable_static() @@ -29,20 +31,24 @@ os.environ["CPU_NUM"] = "1" def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) with fluid.name_scope("skip_quant"): hidden = fluid.layers.fc(input=conv_pool_1, size=100, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') @@ -52,25 +58,25 @@ def conv_net(img, label): class TestQuantizeProgramPass(unittest.TestCase): - - def quantize_program(self, - use_cuda, - seed, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=False): - + def quantize_program( + self, + use_cuda, + seed, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=False, + ): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.0001) @@ -89,8 +95,9 @@ class TestQuantizeProgramPass(unittest.TestCase): test_program = test_program.clone(for_test=True) if not for_ci: - train_graph = IrGraph(core.Graph(train_program.desc), - for_test=False) + train_graph = IrGraph( + core.Graph(train_program.desc), for_test=False + ) train_graph.draw('.', 'train_program_1') test_graph = IrGraph(core.Graph(test_program.desc), for_test=True) test_graph.draw('.', 'test_program_1') @@ -98,7 +105,8 @@ class TestQuantizeProgramPass(unittest.TestCase): # 2 Apply quantization qt = QuantizeTranspilerV2( activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) qt.apply(train_program, startup_program, is_test=False) qt.apply(test_program, startup_program, is_test=True) @@ -109,8 +117,9 @@ class TestQuantizeProgramPass(unittest.TestCase): with fluid.scope_guard(scope): exe.run(startup_program) if not for_ci: - train_graph = IrGraph(core.Graph(train_program.desc), - for_test=False) + train_graph = IrGraph( + core.Graph(train_program.desc), for_test=False + ) train_graph.draw('.', 'train_program_2') test_graph = IrGraph(core.Graph(test_program.desc), for_test=True) test_graph.draw('.', 'test_program_2') @@ -120,19 +129,21 @@ class TestQuantizeProgramPass(unittest.TestCase): build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(train_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for idx in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) if not for_ci and idx % 20 == 0: print('{}: {}'.format('loss', np.mean(loss_v))) @@ -142,19 +153,24 @@ class TestQuantizeProgramPass(unittest.TestCase): qt.convert(test_program, scope) if not for_ci: with fluid.scope_guard(scope): - fluid.io.save_inference_model('./infer_model', - ['image', 'label'], [loss], - exe, - test_program, - clip_extra=True) + fluid.io.save_inference_model( + './infer_model', + ['image', 'label'], + [loss], + exe, + test_program, + clip_extra=True, + ) def test_gpu_1(self): if fluid.core.is_compiled_with_cuda(): - self.quantize_program(use_cuda=True, - seed=1, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=True) + self.quantize_program( + use_cuda=True, + seed=1, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) def test_gpu_2(self): if fluid.core.is_compiled_with_cuda(): @@ -163,21 +179,26 @@ class TestQuantizeProgramPass(unittest.TestCase): seed=1, activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) def test_cpu_1(self): - self.quantize_program(use_cuda=False, - seed=2, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=True) + self.quantize_program( + use_cuda=False, + seed=2, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) def test_cpu_2(self): - self.quantize_program(use_cuda=False, - seed=2, - activation_quant_type='moving_average_abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + self.quantize_program( + use_cuda=False, + seed=2, + activation_quant_type='moving_average_abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_user_defined_quantization.py b/python/paddle/fluid/contrib/slim/tests/test_user_defined_quantization.py index 361440f0238..96635700666 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_user_defined_quantization.py +++ b/python/paddle/fluid/contrib/slim/tests/test_user_defined_quantization.py @@ -36,21 +36,25 @@ os.environ["CPU_NUM"] = "1" def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) hidden = fluid.layers.fc(input=conv_pool_2, size=100, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -66,41 +70,44 @@ def pact(x, name=None): name=x.name + '_pact', initializer=fluid.initializer.ConstantInitializer(value=init_thres), regularizer=fluid.regularizer.L2Decay(0.0001), - learning_rate=1) + learning_rate=1, + ) u_param = helper.create_parameter(attr=u_param_attr, shape=[1], dtype=dtype) x = fluid.layers.elementwise_sub( - x, fluid.layers.relu(fluid.layers.elementwise_sub(x, u_param))) + x, fluid.layers.relu(fluid.layers.elementwise_sub(x, u_param)) + ) x = fluid.layers.elementwise_add( - x, fluid.layers.relu(fluid.layers.elementwise_sub(-u_param, x))) + x, fluid.layers.relu(fluid.layers.elementwise_sub(-u_param, x)) + ) return x class TestUserDefinedQuantization(unittest.TestCase): - - def quantization_scale(self, - use_cuda, - seed, - activation_quant_type, - weight_quant_type='abs_max', - for_ci=False, - act_preprocess_func=None, - weight_preprocess_func=None, - act_quantize_func=None, - weight_quantize_func=None): - + def quantization_scale( + self, + use_cuda, + seed, + activation_quant_type, + weight_quant_type='abs_max', + for_ci=False, + act_preprocess_func=None, + weight_preprocess_func=None, + act_quantize_func=None, + weight_quantize_func=None, + ): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) img.stop_gradient = False - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.SGD(learning_rate=0.0001) @@ -149,7 +156,8 @@ class TestUserDefinedQuantization(unittest.TestCase): act_quantize_func=act_quantize_func, weight_quantize_func=weight_quantize_func, optimizer_func=get_optimizer, - executor=exe) + executor=exe, + ) train_transform_pass.apply(main_graph) test_transform_pass = QuantizationTransformPass( scope=scope, @@ -161,7 +169,8 @@ class TestUserDefinedQuantization(unittest.TestCase): act_quantize_func=act_quantize_func, weight_quantize_func=weight_quantize_func, optimizer_func=get_optimizer, - executor=exe) + executor=exe, + ) test_transform_pass.apply(test_graph) save_dict(test_graph.out_node_mapping_table, mapping_table_path) @@ -180,20 +189,22 @@ class TestUserDefinedQuantization(unittest.TestCase): build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) out_scale_infer_pass = OutScaleForInferencePass(scope=scope) out_scale_infer_pass.apply(test_graph) @@ -203,7 +214,8 @@ class TestUserDefinedQuantization(unittest.TestCase): place=place, weight_bits=8, activation_bits=8, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) mapping_table = load_dict(mapping_table_path) test_graph.out_node_mapping_table = mapping_table @@ -220,7 +232,8 @@ class TestUserDefinedQuantization(unittest.TestCase): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - act_preprocess_func=pact) + act_preprocess_func=pact, + ) def test_act_preprocess_cpu(self): with fluid.unique_name.guard(): @@ -230,7 +243,8 @@ class TestUserDefinedQuantization(unittest.TestCase): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - act_preprocess_func=pact) + act_preprocess_func=pact, + ) def test_weight_preprocess_cuda(self): if fluid.core.is_compiled_with_cuda(): @@ -241,7 +255,8 @@ class TestUserDefinedQuantization(unittest.TestCase): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - weight_preprocess_func=pact) + weight_preprocess_func=pact, + ) def test_weight_preprocess_cpu(self): with fluid.unique_name.guard(): @@ -251,7 +266,8 @@ class TestUserDefinedQuantization(unittest.TestCase): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - weight_preprocess_func=pact) + weight_preprocess_func=pact, + ) def test_act_quantize_cuda(self): if fluid.core.is_compiled_with_cuda(): @@ -262,7 +278,8 @@ class TestUserDefinedQuantization(unittest.TestCase): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - act_quantize_func=pact) + act_quantize_func=pact, + ) def test_act_quantize_cpu(self): with fluid.unique_name.guard(): @@ -272,7 +289,8 @@ class TestUserDefinedQuantization(unittest.TestCase): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - act_quantize_func=pact) + act_quantize_func=pact, + ) def test_weight_quantize_cuda(self): if fluid.core.is_compiled_with_cuda(): @@ -283,7 +301,8 @@ class TestUserDefinedQuantization(unittest.TestCase): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - weight_quantize_func=pact) + weight_quantize_func=pact, + ) def test_weight_quantize_cpu(self): with fluid.unique_name.guard(): @@ -293,7 +312,8 @@ class TestUserDefinedQuantization(unittest.TestCase): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - weight_quantize_func=pact) + weight_quantize_func=pact, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_weight_quantization_mobilenetv1.py b/python/paddle/fluid/contrib/slim/tests/test_weight_quantization_mobilenetv1.py index e6af06c6cea..929eb34994b 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_weight_quantization_mobilenetv1.py +++ b/python/paddle/fluid/contrib/slim/tests/test_weight_quantization_mobilenetv1.py @@ -28,8 +28,7 @@ def _load_variable_data(scope, var_name): Load variable value from scope ''' var_node = scope.find_var(var_name) - assert var_node is not None, \ - "Cannot find " + var_name + " in scope." + assert var_node is not None, "Cannot find " + var_name + " in scope." return np.array(var_node.get_tensor()) @@ -37,8 +36,9 @@ def _set_variable_data(scope, place, var_name, np_value): ''' Set the value of var node by name, if the node exits, ''' - assert isinstance(np_value, np.ndarray), \ - 'The type of value should be numpy array.' + assert isinstance( + np_value, np.ndarray + ), 'The type of value should be numpy array.' var_node = scope.find_var(var_name) if var_node != None: tensor = var_node.get_tensor() @@ -46,11 +46,11 @@ def _set_variable_data(scope, place, var_name, np_value): class TestWeightQuantization(unittest.TestCase): - def setUp(self): self.weight_quantization_dir = 'weight_quantization' - self.cache_folder = os.path.join(DATA_HOME, - self.weight_quantization_dir) + self.cache_folder = os.path.join( + DATA_HOME, self.weight_quantization_dir + ) def download_model(self, model_name, data_url, data_md5): download(data_url, self.weight_quantization_dir, data_md5) @@ -66,21 +66,32 @@ class TestWeightQuantization(unittest.TestCase): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) - def quantize_to_int(self, model_name, model_data_url, model_data_md5, - weight_bits, quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate): - - model_dir = self.download_model(model_name, model_data_url, - model_data_md5) + def quantize_to_int( + self, + model_name, + model_data_url, + model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ): + + model_dir = self.download_model( + model_name, model_data_url, model_data_md5 + ) load_model_dir = os.path.join(model_dir, model_name) timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()) save_model_dir = os.path.join( os.getcwd(), - model_name + "_wq_" + str(weight_bits) + "_" + timestamp) + model_name + "_wq_" + str(weight_bits) + "_" + timestamp, + ) weight_quant = WeightQuantization(model_dir=load_model_dir) weight_quant.quantize_weight_to_int( @@ -89,39 +100,54 @@ class TestWeightQuantization(unittest.TestCase): quantizable_op_type=quantizable_op_type, weight_quantize_type=weight_quantize_type, generate_test_model=generate_test_model, - threshold_rate=threshold_rate) + threshold_rate=threshold_rate, + ) print("finish weight quantization for " + model_name + "\n") try: os.system("rm -rf {}".format(save_model_dir)) except Exception as e: - print("Failed to delete {} due to {}".format( - save_model_dir, str(e))) - - def convert_to_fp16(self, model_name, model_data_url, model_data_md5, - model_filename, params_filename): - model_dir = self.download_model(model_name, model_data_url, - model_data_md5) + print( + "Failed to delete {} due to {}".format(save_model_dir, str(e)) + ) + + def convert_to_fp16( + self, + model_name, + model_data_url, + model_data_md5, + model_filename, + params_filename, + ): + model_dir = self.download_model( + model_name, model_data_url, model_data_md5 + ) load_model_dir = os.path.join(model_dir, model_name) timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()) - save_model_dir = os.path.join(os.getcwd(), - model_name + "_wq_fp16_" + timestamp) + save_model_dir = os.path.join( + os.getcwd(), model_name + "_wq_fp16_" + timestamp + ) - weight_quant = WeightQuantization(load_model_dir, model_filename, - params_filename) + weight_quant = WeightQuantization( + load_model_dir, model_filename, params_filename + ) weight_quant.convert_weight_to_fp16(save_model_dir) - print("finish converting the data type of weights to fp16 for " + - model_name) + print( + "finish converting the data type of weights to fp16 for " + + model_name + ) print("fp16 model saved in " + save_model_dir + "\n") input_data = np.ones([1, 3, 224, 224], dtype=np.float32) - res_fp32 = self.run_models(load_model_dir, model_filename, - params_filename, input_data, False) - res_fp16 = self.run_models(save_model_dir, model_filename, - params_filename, input_data, True) + res_fp32 = self.run_models( + load_model_dir, model_filename, params_filename, input_data, False + ) + res_fp16 = self.run_models( + save_model_dir, model_filename, params_filename, input_data, True + ) np.testing.assert_allclose( res_fp32, @@ -129,40 +155,60 @@ class TestWeightQuantization(unittest.TestCase): rtol=1e-05, atol=1e-08, equal_nan=True, - err_msg='Failed to test the accuracy of the fp32 and fp16 model.') + err_msg='Failed to test the accuracy of the fp32 and fp16 model.', + ) try: os.system("rm -rf {}".format(save_model_dir)) except Exception as e: - print("Failed to delete {} due to {}".format( - save_model_dir, str(e))) - - def run_models(self, model_dir, model_filename, params_filename, input_data, - is_fp16_model): + print( + "Failed to delete {} due to {}".format(save_model_dir, str(e)) + ) + + def run_models( + self, + model_dir, + model_filename, + params_filename, + input_data, + is_fp16_model, + ): print(model_dir) place = paddle.CPUPlace() exe = paddle.static.Executor(place) scope = paddle.static.Scope() with paddle.static.scope_guard(scope): - [inference_program, feed_target_names, fetch_targets] = \ - paddle.fluid.io.load_inference_model(model_dir, exe, - model_filename=model_filename, - params_filename=params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.fluid.io.load_inference_model( + model_dir, + exe, + model_filename=model_filename, + params_filename=params_filename, + ) if is_fp16_model: for var in inference_program.list_vars(): - if (var.type == paddle.fluid.core.VarDesc.VarType.RAW) or \ - (not var.persistable) or (var.name in ['feed', 'fetch']) \ - or (var.dtype != paddle.fluid.core.VarDesc.VarType.FP16): + if ( + (var.type == paddle.fluid.core.VarDesc.VarType.RAW) + or (not var.persistable) + or (var.name in ['feed', 'fetch']) + or (var.dtype != paddle.fluid.core.VarDesc.VarType.FP16) + ): continue tensor = _load_variable_data(scope, var.name) - _set_variable_data(scope, place, var.name, - tensor.astype(np.float32)) - - results = exe.run(inference_program, - feed={feed_target_names[0]: input_data}, - fetch_list=fetch_targets) + _set_variable_data( + scope, place, var.name, tensor.astype(np.float32) + ) + + results = exe.run( + inference_program, + feed={feed_target_names[0]: input_data}, + fetch_list=fetch_targets, + ) return np.array(results[0]) @@ -181,10 +227,16 @@ class TestWeightQuantizationMobilenetv1(TestWeightQuantization): weight_quantize_type = "abs_max" generate_test_model = True threshold_rate = 0.0 - self.quantize_to_int(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, weight_bits, - quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate) + self.quantize_to_int( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ) def test_weight_quantization_mobilenetv1_8bit_channel_wise_abs_max(self): weight_bits = 8 @@ -192,10 +244,16 @@ class TestWeightQuantizationMobilenetv1(TestWeightQuantization): weight_quantize_type = "channel_wise_abs_max" generate_test_model = True threshold_rate = 0.0 - self.quantize_to_int(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, weight_bits, - quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate) + self.quantize_to_int( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ) def test_weight_quantization_mobilenetv1_16bit_abs_max(self): weight_bits = 16 @@ -203,10 +261,16 @@ class TestWeightQuantizationMobilenetv1(TestWeightQuantization): weight_quantize_type = "abs_max" generate_test_model = False threshold_rate = 0 - self.quantize_to_int(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, weight_bits, - quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate) + self.quantize_to_int( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ) def test_weight_quantization_mobilenetv1_16bit_channel_wise_abs_max(self): weight_bits = 16 @@ -214,24 +278,38 @@ class TestWeightQuantizationMobilenetv1(TestWeightQuantization): weight_quantize_type = "channel_wise_abs_max" generate_test_model = False threshold_rate = 1e-9 - self.quantize_to_int(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, weight_bits, - quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate) + self.quantize_to_int( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ) def test_mobilenetv1_fp16_combined(self): model_filename = '__model__' params_filename = '__params__' - self.convert_to_fp16(self.comb_model_name, self.comb_model_data_url, - self.comb_model_data_md5, model_filename, - params_filename) + self.convert_to_fp16( + self.comb_model_name, + self.comb_model_data_url, + self.comb_model_data_md5, + model_filename, + params_filename, + ) def test_mobilenetv1_fp16_nocombined(self): model_filename = None params_filename = None - self.convert_to_fp16(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, model_filename, - params_filename) + self.convert_to_fp16( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + model_filename, + params_filename, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/sparsity/__init__.py b/python/paddle/fluid/contrib/sparsity/__init__.py index 43c645057e8..fcb7acda377 100644 --- a/python/paddle/fluid/contrib/sparsity/__init__.py +++ b/python/paddle/fluid/contrib/sparsity/__init__.py @@ -30,8 +30,19 @@ from .asp import reset_excluded_layers from .supported_layer_list import add_supported_layer __all__ = [ - 'calculate_density', 'check_mask_1d', 'get_mask_1d', 'check_mask_2d', - 'get_mask_2d_greedy', 'get_mask_2d_best', 'create_mask', 'check_sparsity', - 'MaskAlgo', 'CheckMethod', 'decorate', 'prune_model', 'set_excluded_layers', - 'reset_excluded_layers', 'add_supported_layer' + 'calculate_density', + 'check_mask_1d', + 'get_mask_1d', + 'check_mask_2d', + 'get_mask_2d_greedy', + 'get_mask_2d_best', + 'create_mask', + 'check_sparsity', + 'MaskAlgo', + 'CheckMethod', + 'decorate', + 'prune_model', + 'set_excluded_layers', + 'reset_excluded_layers', + 'add_supported_layer', ] diff --git a/python/paddle/fluid/contrib/sparsity/asp.py b/python/paddle/fluid/contrib/sparsity/asp.py index ac97bcecf32..d770bd36e39 100644 --- a/python/paddle/fluid/contrib/sparsity/asp.py +++ b/python/paddle/fluid/contrib/sparsity/asp.py @@ -25,14 +25,19 @@ from paddle.fluid import global_scope, program_guard, layers from paddle.fluid.initializer import ConstantInitializer from paddle.fluid.contrib import sparsity from paddle.fluid import core -from paddle.fluid.contrib.sparsity.supported_layer_list import supported_layers_and_prune_func_map +from paddle.fluid.contrib.sparsity.supported_layer_list import ( + supported_layers_and_prune_func_map, +) from paddle.fluid.contrib.sparsity.supported_layer_list import _default_pruning OpRole = core.op_proto_and_checker_maker.OpRole OP_ROLE_KEY = core.op_proto_and_checker_maker.kOpRoleAttrName() __all__ = [ - 'decorate', 'prune_model', 'set_excluded_layers', 'reset_excluded_layers' + 'decorate', + 'prune_model', + 'set_excluded_layers', + 'reset_excluded_layers', ] @@ -118,8 +123,9 @@ def set_excluded_layers(param_names, main_program=None): """ if main_program is None: main_program = paddle.static.default_main_program() - ASPHelper.set_excluded_layers(param_names=param_names, - main_program=main_program) + ASPHelper.set_excluded_layers( + param_names=param_names, main_program=main_program + ) def reset_excluded_layers(main_program=None): @@ -437,32 +443,39 @@ def prune_model(model, n=2, m=4, mask_algo='mask_1d', with_mask=True): MaskAlgo_mapping = { 'mask_1d': sparsity.MaskAlgo.MASK_1D, 'mask_2d_greedy': sparsity.MaskAlgo.MASK_2D_GREEDY, - 'mask_2d_best': sparsity.MaskAlgo.MASK_2D_BEST + 'mask_2d_best': sparsity.MaskAlgo.MASK_2D_BEST, } - assert (mask_algo in MaskAlgo_mapping), \ - 'The "mask_algo" should be one of ["mask_1d", "mask_2d_greedy", "mask_2d_best"]' + assert ( + mask_algo in MaskAlgo_mapping + ), 'The "mask_algo" should be one of ["mask_1d", "mask_2d_greedy", "mask_2d_best"]' prune_func = None if isinstance(model, paddle.nn.Layer): prune_func = ASPHelper.prune_model_by_layer elif isinstance(model, paddle.static.Program): prune_func = ASPHelper.prune_model_by_program - if hasattr(model, "distributed_info_") and \ - model.distributed_info_["sharding_degree"] > 1 and \ - paddle.fluid.is_compiled_with_cuda(): + if ( + hasattr(model, "distributed_info_") + and model.distributed_info_["sharding_degree"] > 1 + and paddle.fluid.is_compiled_with_cuda() + ): gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0)) place = paddle.CUDAPlace(gpu_id) else: raise TypeError( - "model should be paddle.nn.Layer or paddle.static.Program, but got {}" - .format(type(model))) + "model should be paddle.nn.Layer or paddle.static.Program, but got {}".format( + type(model) + ) + ) - return prune_func(place, - model, - n=n, - m=m, - mask_algo=MaskAlgo_mapping[mask_algo], - with_mask=with_mask) + return prune_func( + place, + model, + n=n, + m=m, + mask_algo=MaskAlgo_mapping[mask_algo], + with_mask=with_mask, + ) class ProgramASPInfo(object): @@ -547,18 +560,21 @@ class ASPHelper(object): # default_main_program as the key. main_prog = paddle.static.default_main_program() startup_prog = paddle.static.default_startup_program() - ASPHelper._create_mask_variables(main_prog, startup_prog, - optimizer._parameter_list) + ASPHelper._create_mask_variables( + main_prog, startup_prog, optimizer._parameter_list + ) return OptimizerWithSparsityGuarantee(optimizer) @classmethod - def prune_model_by_program(cls, - place, - main_program=None, - n=2, - m=4, - mask_algo=sparsity.MaskAlgo.MASK_1D, - with_mask=True): + def prune_model_by_program( + cls, + place, + main_program=None, + n=2, + m=4, + mask_algo=sparsity.MaskAlgo.MASK_1D, + with_mask=True, + ): r""" This is the implementation of `sparsity.prune_model`, for details please see explanation in `sparsity.prune_model`. """ @@ -574,34 +590,43 @@ class ASPHelper(object): prune_func = ASPHelper._get_prune_func_by_name(param.name) - weight_pruned_nparray, weight_sparse_mask = \ - prune_func(weight_nparray, m, n, mask_algo, param.name) + weight_pruned_nparray, weight_sparse_mask = prune_func( + weight_nparray, m, n, mask_algo, param.name + ) weight_pruned_nparray = weight_pruned_nparray.astype( - weight_nparray.dtype) + weight_nparray.dtype + ) weight_tensor.set(weight_pruned_nparray, place) if with_mask: weight_mask_param = global_scope().find_var( - ASPHelper._get_mask_name(param.name)) - assert weight_mask_param is not None, \ - 'Cannot find {} variable, please call optimizer.minimize (' \ - 'paddle.sparsity.decorate(optimizer).minimize(loss)' \ - ' and initialization (exe.run(startup_program)) first!'.format(ASPHelper._get_mask_name(param.name)) + ASPHelper._get_mask_name(param.name) + ) + assert weight_mask_param is not None, ( + 'Cannot find {} variable, please call optimizer.minimize (' + 'paddle.sparsity.decorate(optimizer).minimize(loss)' + ' and initialization (exe.run(startup_program)) first!'.format( + ASPHelper._get_mask_name(param.name) + ) + ) weight_mask_tensor = weight_mask_param.get_tensor() weight_sparse_mask = weight_sparse_mask.astype( - np.array(weight_mask_tensor).dtype) + np.array(weight_mask_tensor).dtype + ) weight_mask_tensor.set(weight_sparse_mask, place) asp_info.update_masks(param.name, weight_sparse_mask) return asp_info.masks.copy() @classmethod - def prune_model_by_layer(cls, - place, - layer, - n=2, - m=4, - mask_algo=sparsity.MaskAlgo.MASK_1D, - with_mask=True): + def prune_model_by_layer( + cls, + place, + layer, + n=2, + m=4, + mask_algo=sparsity.MaskAlgo.MASK_1D, + with_mask=True, + ): r""" This is the implementation of `sparsity.prune_model`, for details please see explanation in `sparsity.prune_model`. """ @@ -615,19 +640,25 @@ class ASPHelper(object): prune_func = ASPHelper._get_prune_func_by_name(param.name) - weight_pruned_nparray, weight_sparse_mask = \ - prune_func(weight_nparray, m, n, mask_algo, param.name) + weight_pruned_nparray, weight_sparse_mask = prune_func( + weight_nparray, m, n, mask_algo, param.name + ) weight_pruned_nparray = weight_pruned_nparray.astype( - weight_nparray.dtype) + weight_nparray.dtype + ) param.set_value(weight_pruned_nparray) if with_mask: weight_mask_param = asp_info.mask_vars.get( - param.name, None) - assert weight_mask_param is not None, \ - 'Cannot find {} variable, please call sparsity.decorate() to' \ - ' decorate your optimizer first!'.format(ASPHelper._get_mask_name(param.name)) + param.name, None + ) + assert weight_mask_param is not None, ( + 'Cannot find {} variable, please call sparsity.decorate() to' + ' decorate your optimizer first!'.format( + ASPHelper._get_mask_name(param.name) + ) + ) weight_mask_param.set_value(weight_sparse_mask) asp_info.update_masks(param.name, weight_sparse_mask) @@ -639,14 +670,17 @@ class ASPHelper(object): target_program = None for param in layer.parameters(): target_program = param.block.program - assert target_program is not None, \ - 'Cannot get paddle.static.Program from Paddle.nn.Layer.' - return ASPHelper.prune_model_by_program(place, - target_program, - n=n, - m=m, - mask_algo=mask_algo, - with_mask=with_mask) + assert ( + target_program is not None + ), 'Cannot get paddle.static.Program from Paddle.nn.Layer.' + return ASPHelper.prune_model_by_program( + place, + target_program, + n=n, + m=m, + mask_algo=mask_algo, + with_mask=with_mask, + ) @staticmethod def _get_mask_name(param_name): @@ -728,13 +762,16 @@ class ASPHelper(object): param_name_no_weight_suffix = param_name_list[0] param_type_suffix = param_name_list[1] - layer_name = param_name_no_weight_suffix[:param_name_no_weight_suffix. - rfind('_')] + layer_name = param_name_no_weight_suffix[ + : param_name_no_weight_suffix.rfind('_') + ] if ASPHelper.PADDLE_WEIGHT_SUFFIX not in param_type_suffix: return False - if param_name_no_weight_suffix in supported_layers_and_prune_func_map or \ - layer_name in supported_layers_and_prune_func_map: + if ( + param_name_no_weight_suffix in supported_layers_and_prune_func_map + or layer_name in supported_layers_and_prune_func_map + ): return True return False @@ -745,23 +782,27 @@ class ASPHelper(object): param_name_no_weight_suffix = param_name.split('.')[0] if func is None: func = supported_layers_and_prune_func_map.get( - param_name_no_weight_suffix, None) + param_name_no_weight_suffix, None + ) if func is None: - layer_name = param_name_no_weight_suffix[: - param_name_no_weight_suffix - .rfind('_')] + layer_name = param_name_no_weight_suffix[ + : param_name_no_weight_suffix.rfind('_') + ] func = supported_layers_and_prune_func_map.get( - layer_name, _default_pruning) + layer_name, _default_pruning + ) return func @classmethod - def _minimize(cls, - optimizer, - loss, - main_program=None, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def _minimize( + cls, + optimizer, + loss, + main_program=None, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ): r""" This function is a decorator of `minimize` function in `Optimizer`. There are three steps: @@ -792,7 +833,8 @@ class ASPHelper(object): startup_program = paddle.static.default_startup_program() optimizer_ops, params_and_grads = optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) params_only = [pg[0] for pg in params_and_grads] cls._create_mask_variables(main_program, startup_program, params_only) @@ -819,8 +861,9 @@ class ASPHelper(object): optimizer.step() main_prog = paddle.static.default_main_program() with paddle.fluid.dygraph.no_grad(): - ASPHelper._insert_sparse_mask_ops(main_prog, - optimizer._parameter_list) + ASPHelper._insert_sparse_mask_ops( + main_prog, optimizer._parameter_list + ) @classmethod def _create_mask_variables(cls, main_program, startup_program, params): @@ -842,7 +885,8 @@ class ASPHelper(object): name=ASPHelper._get_mask_name(param.name), shape=param.shape, dtype=param.dtype, - default_initializer=ConstantInitializer(value=1.0)) + default_initializer=ConstantInitializer(value=1.0), + ) mask_param.stop_gradient = True mask_param.trainable = False asp_info.update_mask_vars(param.name, mask_param) @@ -861,17 +905,16 @@ class ASPHelper(object): asp_info = cls._get_program_asp_info(main_program) for param in params: if param.name in asp_info.mask_vars: - block.append_op(type='elementwise_mul', - inputs={ - "X": param, - 'Y': asp_info.mask_vars[param.name] - }, - outputs={'Out': param}, - attrs={ - 'axis': -1, - 'use_mkldnn': False, - OP_ROLE_KEY: int(OpRole.Optimize) - }) + block.append_op( + type='elementwise_mul', + inputs={"X": param, 'Y': asp_info.mask_vars[param.name]}, + outputs={'Out': param}, + attrs={ + 'axis': -1, + 'use_mkldnn': False, + OP_ROLE_KEY: int(OpRole.Optimize), + }, + ) class OptimizerWithSparsityGuarantee(object): @@ -889,11 +932,9 @@ class OptimizerWithSparsityGuarantee(object): def __getattr__(self, item): return getattr(self._optimizer, item) - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): r""" This function is to call `ASPHelper.minimize()` and return its return @@ -906,11 +947,13 @@ class OptimizerWithSparsityGuarantee(object): list: operators from :attr:`optimizer`.minimize(:attr:`loss`). list: pairs of parameters and their gradients. """ - return ASPHelper._minimize(self._optimizer, - loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + return ASPHelper._minimize( + self._optimizer, + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) @dygraph_only def step(self): @@ -940,7 +983,8 @@ class OptimizerWithSparsityGuarantee(object): """ state_dict = self._optimizer.state_dict() asp_info = ASPHelper._get_program_asp_info( - paddle.static.default_main_program()) + paddle.static.default_main_program() + ) for param_name, var in asp_info.mask_vars.items(): state_dict.update({ASPHelper._get_mask_name(param_name): var}) return state_dict @@ -955,11 +999,13 @@ class OptimizerWithSparsityGuarantee(object): None """ asp_info = ASPHelper._get_program_asp_info( - paddle.static.default_main_program()) + paddle.static.default_main_program() + ) for param_name, var in asp_info.mask_vars.items(): param_mask_name = ASPHelper._get_mask_name(param_name) - assert param_mask_name in state_dict, \ - "The {} is not found.".format(param_mask_name) + assert param_mask_name in state_dict, "The {} is not found.".format( + param_mask_name + ) var.set_value(state_dict[param_mask_name]) asp_info.update_masks(param_name, var.numpy()) return self._optimizer.set_state_dict(state_dict) diff --git a/python/paddle/fluid/contrib/sparsity/supported_layer_list.py b/python/paddle/fluid/contrib/sparsity/supported_layer_list.py index 38dd428e0f0..f55a877b4b7 100644 --- a/python/paddle/fluid/contrib/sparsity/supported_layer_list.py +++ b/python/paddle/fluid/contrib/sparsity/supported_layer_list.py @@ -23,9 +23,9 @@ from ...log_helper import get_logger __all__ = ['add_supported_layer'] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def _default_pruning(weight_nparray, m, n, func_name, param_name): @@ -38,13 +38,17 @@ def _default_pruning(weight_nparray, m, n, func_name, param_name): exlude_cond_shape4 = len(shape) == 4 and shape[1] < m if exlude_cond_shape2: _logger.warning( - '{} is not pruned because the first dimension of {} is smaller than {}' - .format(param_name, shape, m)) + '{} is not pruned because the first dimension of {} is smaller than {}'.format( + param_name, shape, m + ) + ) return weight_pruned_nparray, weight_sparse_mask if exlude_cond_shape4: _logger.warning( - '{} is not pruned because the second dimension of {} is smaller than {}' - .format(param_name, shape, m)) + '{} is not pruned because the second dimension of {} is smaller than {}'.format( + param_name, shape, m + ) + ) return weight_pruned_nparray, weight_sparse_mask checked_func_name = sparsity.CheckMethod.get_checking_method(func_name) @@ -60,13 +64,13 @@ def _default_pruning(weight_nparray, m, n, func_name, param_name): # sparsity/utils is row-major pruning. That is the reason we have to transpose weight # matrices beforce invoking create_mask. Then we transpose the result mask to make # sure its shape to be the same as the input weight. - weight_sparse_mask = sparsity.create_mask(weight_nparray.T, - func_name=func_name, - n=n, - m=m).T + weight_sparse_mask = sparsity.create_mask( + weight_nparray.T, func_name=func_name, n=n, m=m + ).T weight_pruned_nparray = np.multiply(weight_nparray, weight_sparse_mask) - assert sparsity.check_sparsity(weight_pruned_nparray.T, n=n, m=m, func_name=checked_func_name), \ - 'Pruning {} weight matrix failure!!!'.format(param_name) + assert sparsity.check_sparsity( + weight_pruned_nparray.T, n=n, m=m, func_name=checked_func_name + ), 'Pruning {} weight matrix failure!!!'.format(param_name) return weight_pruned_nparray, weight_sparse_mask @@ -93,13 +97,18 @@ def add_supported_layer(layer, pruning_func=None): name = layer elif isinstance(layer, paddle.fluid.dygraph.layers.Layer): name = paddle.fluid.dygraph.layers._convert_camel_to_snake( - type(layer).__name__) + type(layer).__name__ + ) elif issubclass(layer, paddle.fluid.dygraph.layers.Layer): name = paddle.fluid.dygraph.layers._convert_camel_to_snake( - layer.__name__) + layer.__name__ + ) else: - assert "The type of layer should be string of Layer, but got {}!".format( - type(layer)) + assert ( + "The type of layer should be string of Layer, but got {}!".format( + type(layer) + ) + ) if pruning_func is None: pruning_func = _default_pruning _supported_layers_and_prune_func_map_lock.acquire() diff --git a/python/paddle/fluid/contrib/sparsity/utils.py b/python/paddle/fluid/contrib/sparsity/utils.py index 82846920031..b5be3887380 100644 --- a/python/paddle/fluid/contrib/sparsity/utils.py +++ b/python/paddle/fluid/contrib/sparsity/utils.py @@ -25,9 +25,16 @@ from itertools import permutations import threading __all__ = [ - 'calculate_density', 'check_mask_1d', 'get_mask_1d', 'check_mask_2d', - 'get_mask_2d_greedy', 'get_mask_2d_best', 'create_mask', 'check_sparsity', - 'MaskAlgo', 'CheckMethod' + 'calculate_density', + 'check_mask_1d', + 'get_mask_1d', + 'check_mask_2d', + 'get_mask_2d_greedy', + 'get_mask_2d_best', + 'create_mask', + 'check_sparsity', + 'MaskAlgo', + 'CheckMethod', ] @@ -74,8 +81,9 @@ class CheckMethod(Enum): CheckMethod.get_checking_method(MaskAlgo.MASK_2D_BEST) # CheckMethod.CHECK_2D """ - assert isinstance(mask_algo, MaskAlgo), \ - "mask_algo should be MaskAlgo type" + assert isinstance( + mask_algo, MaskAlgo + ), "mask_algo should be MaskAlgo type" if mask_algo == MaskAlgo.MASK_1D: return CheckMethod.CHECK_1D else: @@ -124,7 +132,7 @@ def _reshape_1d(mat, m): remainder = mat.shape[1] % m if mat.shape[1] % m > 0: mat_padded = np.zeros((mat.shape[0], mat.shape[1] + (m - remainder))) - mat_padded[:, :mat.shape[1]] = mat + mat_padded[:, : mat.shape[1]] = mat shape = mat_padded.shape return mat_padded.reshape(-1, m), shape else: @@ -211,7 +219,7 @@ def get_mask_1d(mat, n, m): min_order_indices = np.argsort(np.absolute(sub_mat)) mask_flattern[i, min_order_indices[:n].tolist()] = 0 mask_flattern = mask_flattern.reshape(shape) - mask[:, :] = mask_flattern[:, :mat.shape[1]] + mask[:, :] = mask_flattern[:, : mat.shape[1]] return mask @@ -237,12 +245,12 @@ def _reshape_2d(mat, m): remainder_0 = mat.shape[0] % m remainder_1 = mat.shape[1] % m - new_shape = (mat.shape[0] if remainder_0 == 0 \ - else mat.shape[0] + (m - remainder_0), - mat.shape[1] if remainder_1 == 0 \ - else mat.shape[1] + (m - remainder_1)) + new_shape = ( + mat.shape[0] if remainder_0 == 0 else mat.shape[0] + (m - remainder_0), + mat.shape[1] if remainder_1 == 0 else mat.shape[1] + (m - remainder_1), + ) mat_padded = np.zeros(new_shape) - mat_padded[:mat.shape[0], :mat.shape[1]] = mat + mat_padded[: mat.shape[0], : mat.shape[1]] = mat mat_flattern = np.empty(new_shape).reshape(-1, m * m) curr_idx = 0 @@ -250,9 +258,9 @@ def _reshape_2d(mat, m): row_end = row_start + m for col_start in range(0, mat_padded.shape[1], m): col_end = col_start + m - sub_mat = np.squeeze(mat_padded[row_start:row_end, \ - col_start:col_end] \ - .reshape(-1)) + sub_mat = np.squeeze( + mat_padded[row_start:row_end, col_start:col_end].reshape(-1) + ) mat_flattern[curr_idx] = sub_mat curr_idx += 1 return mat_flattern, mat_padded.shape @@ -302,8 +310,9 @@ def check_mask_2d(mat, n, m): mat_padded, shape = _reshape_2d(mat, m) for sub_mat in mat_padded: sub_mask = np.absolute(np.squeeze(sub_mat.reshape(m, m))) > 0 - if (np.sum(np.sum(sub_mask, axis=1) > (m-n)) != 0) and \ - (np.sum(np.sum(sub_mask, axis=0) > (m-n)) != 0): + if (np.sum(np.sum(sub_mask, axis=1) > (m - n)) != 0) and ( + np.sum(np.sum(sub_mask, axis=0) > (m - n)) != 0 + ): return False return True @@ -348,15 +357,17 @@ def get_mask_2d_greedy(mat, n, m): sub_mask = np.squeeze(mask_padded[idx]) min_order_1d_indices = np.argsort(sub_mat) - min_order_2d_indices = [(int(x / m), x % m) - for x in min_order_1d_indices] + min_order_2d_indices = [ + (int(x / m), x % m) for x in min_order_1d_indices + ] row_counter = collections.Counter() col_counter = collections.Counter() for i in range(len(min_order_1d_indices) - 1, -1, -1): matrix_entry = min_order_2d_indices[i] - if (row_counter[matrix_entry[0]] == n) or \ - (col_counter[matrix_entry[1]] == n): + if (row_counter[matrix_entry[0]] == n) or ( + col_counter[matrix_entry[1]] == n + ): continue sub_mask[matrix_entry[0], matrix_entry[1]] = 1.0 @@ -371,7 +382,7 @@ def get_mask_2d_greedy(mat, n, m): col_end = col_start + m mask[row_start:row_end, col_start:col_end] = mask_padded[curr_idx] curr_idx += 1 - return mask[:mat.shape[0], :mat.shape[1]] + return mask[: mat.shape[0], : mat.shape[1]] _valid_2d_patterns_lock = threading.Lock() @@ -404,8 +415,11 @@ def _compute_valid_2d_patterns(n, m): patterns = patterns + patterns patterns = np.asarray(list(set(permutations(patterns, m)))) - valid = ((patterns.sum(axis=1) <= n).sum( - axis=1) == m).nonzero()[0].reshape(-1) + valid = ( + ((patterns.sum(axis=1) <= n).sum(axis=1) == m) + .nonzero()[0] + .reshape(-1) + ) valid_patterns = np.empty((valid.shape[0], m, m)) valid_patterns[:] = patterns[valid[:]] @@ -452,9 +466,10 @@ def get_mask_2d_best(mat, n, m): mat_flattern, shape = _reshape_2d(mat, m) mask_flattern = np.ones_like(mat_flattern).reshape(-1, m, m) - pmax = np.argmax(np.matmul(mat_flattern, - patterns.reshape(patterns.shape[0], m * m).T), - axis=1) + pmax = np.argmax( + np.matmul(mat_flattern, patterns.reshape(patterns.shape[0], m * m).T), + axis=1, + ) mask_flattern[:] = patterns[pmax[:]] mask = np.empty(shape) @@ -466,7 +481,7 @@ def get_mask_2d_best(mat, n, m): col_end = col_start + m mask[row_start:row_end, col_start:col_end] = mask_flattern[curr_idx] curr_idx += 1 - return mask[:mat.shape[0], :mat.shape[1]] + return mask[: mat.shape[0], : mat.shape[1]] def create_mask(tensor, func_name=MaskAlgo.MASK_1D, n=2, m=4): @@ -506,9 +521,10 @@ def create_mask(tensor, func_name=MaskAlgo.MASK_1D, n=2, m=4): dtype = tensor.dtype t = tensor.astype(float) - assert isinstance(func_name, MaskAlgo), \ - "func_name argumet of create_mask is only accepted as type MaskAlgo. " \ - "But got {}".format(type(func_name)) + assert isinstance(func_name, MaskAlgo), ( + "func_name argumet of create_mask is only accepted as type MaskAlgo. " + "But got {}".format(type(func_name)) + ) func = getattr(sys.modules[__name__], func_name.value, None) if len(shape) == 1: t = t.reshape(1, shape[0]) @@ -518,14 +534,20 @@ def create_mask(tensor, func_name=MaskAlgo.MASK_1D, n=2, m=4): t = t.reshape(shape[0] * shape[1], shape[2]) # 4d-tensor conv (h, w, in, out) -> (h*w*out, in) in GemmConvKernel Op elif len(shape) == 4: - t = t.transpose([0, 1, 3, 2]).reshape(shape[0] * shape[1] * shape[3], - shape[2]) + t = t.transpose([0, 1, 3, 2]).reshape( + shape[0] * shape[1] * shape[3], shape[2] + ) mask = func(t, n=n, m=m) - return mask.reshape([shape[0], shape[1], shape[3], - shape[2]]).transpose([0, 1, 3, 2]).astype(dtype) + return ( + mask.reshape([shape[0], shape[1], shape[3], shape[2]]) + .transpose([0, 1, 3, 2]) + .astype(dtype) + ) else: - raise ValueError("The dimension of input tensor is not supported in create_mask, " \ - "Only dimension < 4 is supported but got {}".format(len(shape))) + raise ValueError( + "The dimension of input tensor is not supported in create_mask, " + "Only dimension < 4 is supported but got {}".format(len(shape)) + ) mask = func(t, n=n, m=m) return mask.reshape(shape).astype(dtype) @@ -564,9 +586,10 @@ def check_sparsity(tensor, func_name=CheckMethod.CHECK_1D, n=2, m=4): shape = tensor.shape t = tensor.astype(float) - assert type(func_name) == CheckMethod, \ - "func_name argumet of check_sparsity is only accepted as type CheckMethod. " \ - "But got {}".format(type(func_name)) + assert type(func_name) == CheckMethod, ( + "func_name argumet of check_sparsity is only accepted as type CheckMethod. " + "But got {}".format(type(func_name)) + ) func = getattr(sys.modules[__name__], func_name.value, None) if len(shape) == 1: t = t.reshape(1, shape[0]) @@ -576,10 +599,13 @@ def check_sparsity(tensor, func_name=CheckMethod.CHECK_1D, n=2, m=4): t = t.reshape(shape[0] * shape[1], shape[2]) # 4d-tensor conv (h, w, in, out) -> (h*w*out, in) in GemmConvKernel Op elif len(shape) == 4: - t = t.transpose([0, 1, 3, - 2]).reshape([shape[0] * shape[1] * shape[3], shape[2]]) + t = t.transpose([0, 1, 3, 2]).reshape( + [shape[0] * shape[1] * shape[3], shape[2]] + ) else: - raise ValueError("The dimension of input tensor is not supported in create_mask, " \ - "Only dimension < 4 is supported but got {}".format(len(shape))) + raise ValueError( + "The dimension of input tensor is not supported in create_mask, " + "Only dimension < 4 is supported but got {}".format(len(shape)) + ) return func(t, n=n, m=m) diff --git a/python/paddle/fluid/contrib/tests/test_amp_list.py b/python/paddle/fluid/contrib/tests/test_amp_list.py index 93c99b5ea1c..4a30185229a 100644 --- a/python/paddle/fluid/contrib/tests/test_amp_list.py +++ b/python/paddle/fluid/contrib/tests/test_amp_list.py @@ -14,11 +14,12 @@ import paddle import unittest -from paddle.fluid.contrib.mixed_precision.fp16_lists import AutoMixedPrecisionLists +from paddle.fluid.contrib.mixed_precision.fp16_lists import ( + AutoMixedPrecisionLists, +) class TestAMPList(unittest.TestCase): - def test_main(self): custom_white_list = [ 'lookup_table', @@ -31,8 +32,11 @@ class TestAMPList(unittest.TestCase): self.assertTrue(op not in amp_list.unsupported_list) default_black_list = [ - 'linear_interp_v2', 'nearest_interp_v2', 'bilinear_interp_v2', - 'bicubic_interp_v2', 'trilinear_interp_v2' + 'linear_interp_v2', + 'nearest_interp_v2', + 'bilinear_interp_v2', + 'bicubic_interp_v2', + 'trilinear_interp_v2', ] for op in default_black_list: self.assertTrue(op in amp_list.black_list) diff --git a/python/paddle/fluid/contrib/tests/test_bf16_utils.py b/python/paddle/fluid/contrib/tests/test_bf16_utils.py index c456b1263ce..5d528f22659 100644 --- a/python/paddle/fluid/contrib/tests/test_bf16_utils.py +++ b/python/paddle/fluid/contrib/tests/test_bf16_utils.py @@ -22,7 +22,6 @@ paddle.enable_static() class AMPTest(unittest.TestCase): - def setUp(self): self.bf16_list = copy.copy(amp.bf16.amp_lists.bf16_list) self.fp32_list = copy.copy(amp.bf16.amp_lists.fp32_list) @@ -63,7 +62,8 @@ class AMPTest(unittest.TestCase): self.fp32_list.add('matmul_v2') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'matmul_v2'}) + custom_fp32_list={'matmul_v2'} + ) def test_amp_lists_5(self): # 5. w=None, b={'matmul_v2'} @@ -71,43 +71,47 @@ class AMPTest(unittest.TestCase): self.bf16_list.remove('matmul_v2') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'matmul_v2'}) + custom_fp32_list={'matmul_v2'} + ) def test_amp_lists_6(self): # 6. w=None, b={'lstm'} self.fp32_list.add('lstm') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'lstm'}) + custom_fp32_list={'lstm'} + ) def test_amp_lists_7(self): self.fp32_list.add('reshape2') self.gray_list.remove('reshape2') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'reshape2'}) + custom_fp32_list={'reshape2'} + ) def test_amp_list_8(self): self.bf16_list.add('reshape2') self.gray_list.remove('reshape2') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_bf16_list={'reshape2'}) + custom_bf16_list={'reshape2'} + ) class AMPTest2(unittest.TestCase): - def test_amp_lists_(self): # 7. w={'lstm'} b={'lstm'} # raise ValueError - self.assertRaises(ValueError, amp.bf16.AutoMixedPrecisionListsBF16, - {'lstm'}, {'lstm'}) + self.assertRaises( + ValueError, amp.bf16.AutoMixedPrecisionListsBF16, {'lstm'}, {'lstm'} + ) def test_find_op_index(self): block = fluid.default_main_program().global_block() op_desc = core.OpDesc() idx = amp.bf16.amp_utils.find_op_index(block.desc, op_desc) - assert (idx == -1) + assert idx == -1 def test_is_in_fp32_varnames(self): block = fluid.default_main_program().global_block() @@ -115,17 +119,19 @@ class AMPTest2(unittest.TestCase): var1 = block.create_var(name="X", shape=[3], dtype='float32') var2 = block.create_var(name="Y", shape=[3], dtype='float32') var3 = block.create_var(name="Z", shape=[3], dtype='float32') - op1 = block.append_op(type="abs", - inputs={"X": [var1]}, - outputs={"Out": [var2]}) - op2 = block.append_op(type="abs", - inputs={"X": [var2]}, - outputs={"Out": [var3]}) + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]} + ) + op2 = block.append_op( + type="abs", inputs={"X": [var2]}, outputs={"Out": [var3]} + ) amp_lists_1 = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_varnames={'X'}) + custom_fp32_varnames={'X'} + ) assert amp.bf16.amp_utils._is_in_fp32_varnames(op1, amp_lists_1) amp_lists_2 = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_varnames={'Y'}) + custom_fp32_varnames={'Y'} + ) assert amp.bf16.amp_utils._is_in_fp32_varnames(op2, amp_lists_2) assert amp.bf16.amp_utils._is_in_fp32_varnames(op1, amp_lists_2) @@ -136,14 +142,14 @@ class AMPTest2(unittest.TestCase): var1 = block.create_var(name="X", shape=[3], dtype='float32') var2 = block.create_var(name="Y", shape=[3], dtype='float32') var3 = block.create_var(name="Z", shape=[3], dtype='float32') - op1 = block.append_op(type="abs", - inputs={"X": [var1]}, - outputs={"Out": [var2]}) - op2 = block.append_op(type="abs", - inputs={"X": [var2]}, - outputs={"Out": [var3]}) + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]} + ) + op2 = block.append_op( + type="abs", inputs={"X": [var2]}, outputs={"Out": [var3]} + ) res = amp.bf16.amp_utils.find_true_post_op(block.ops, op1, "Y") - assert (res == [op2]) + assert res == [op2] def test_find_true_post_op_with_search_all(self): program = fluid.Program() @@ -152,27 +158,23 @@ class AMPTest2(unittest.TestCase): var1 = block.create_var(name="X", shape=[3], dtype='float32') var2 = block.create_var(name="Y", shape=[3], dtype='float32') - inititializer_op = startup_block._prepend_op(type="fill_constant", - outputs={"Out": var1}, - attrs={ - "shape": var1.shape, - "dtype": var1.dtype, - "value": 1.0 - }) - - op1 = block.append_op(type="abs", - inputs={"X": [var1]}, - outputs={"Out": [var2]}) - result = amp.bf16.amp_utils.find_true_post_op(block.ops, - inititializer_op, - "X", - search_all=False) - assert (len(result) == 0) - result = amp.bf16.amp_utils.find_true_post_op(block.ops, - inititializer_op, - "X", - search_all=True) - assert (result == [op1]) + inititializer_op = startup_block._prepend_op( + type="fill_constant", + outputs={"Out": var1}, + attrs={"shape": var1.shape, "dtype": var1.dtype, "value": 1.0}, + ) + + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]} + ) + result = amp.bf16.amp_utils.find_true_post_op( + block.ops, inititializer_op, "X", search_all=False + ) + assert len(result) == 0 + result = amp.bf16.amp_utils.find_true_post_op( + block.ops, inititializer_op, "X", search_all=True + ) + assert result == [op1] if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/tests/test_correlation.py b/python/paddle/fluid/contrib/tests/test_correlation.py index d3ab5dff601..553bb912bc0 100644 --- a/python/paddle/fluid/contrib/tests/test_correlation.py +++ b/python/paddle/fluid/contrib/tests/test_correlation.py @@ -21,22 +21,28 @@ import paddle paddle.enable_static() -def corr(x_1, - x_2, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1, - corr_multiply=1): +def corr( + x_1, + x_2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + corr_multiply=1, +): K = kernel_size - rinput1 = np.pad(x_1, ((0, 0), (0, 0), (pad_size, pad_size), - (pad_size, pad_size)), - mode='constant') - rinput2 = np.pad(x_2, ((0, 0), (0, 0), (pad_size, pad_size), - (pad_size, pad_size)), - mode='constant') + rinput1 = np.pad( + x_1, + ((0, 0), (0, 0), (pad_size, pad_size), (pad_size, pad_size)), + mode='constant', + ) + rinput2 = np.pad( + x_2, + ((0, 0), (0, 0), (pad_size, pad_size), (pad_size, pad_size)), + mode='constant', + ) rinput1 = np.transpose(rinput1, (0, 2, 3, 1)) rinput2 = np.transpose(rinput2, (0, 2, 3, 1)) B = int(rinput1.shape[0]) @@ -55,17 +61,23 @@ def corr(x_1, y1_index = j + pad_size x2_index = x1_index + k y2_index = y1_index + l - output[b, l + d + D * (k + d), i, - j] = np.mean(rinput1[b, x1_index:x1_index + K, - y1_index:y1_index + K] * - rinput2[b, x2_index:x2_index + K, - y2_index:y2_index + K]) + output[b, l + d + D * (k + d), i, j] = np.mean( + rinput1[ + b, + x1_index : x1_index + K, + y1_index : y1_index + K, + ] + * rinput2[ + b, + x2_index : x2_index + K, + y2_index : y2_index + K, + ] + ) return output class TestCorrelationOp(unittest.TestCase): - def test_check_output(self): if not fluid.core.is_compiled_with_cuda(): return @@ -73,34 +85,42 @@ class TestCorrelationOp(unittest.TestCase): np.set_printoptions(threshold=np.inf) x_shape = (2, 10, 3, 3) x_type = 'float32' - x1 = fluid.layers.data(name='x1', - shape=x_shape, - dtype=x_type, - append_batch_size=False, - stop_gradient=False) - x2 = fluid.layers.data(name='x2', - shape=x_shape, - dtype=x_type, - append_batch_size=False, - stop_gradient=False) + x1 = fluid.layers.data( + name='x1', + shape=x_shape, + dtype=x_type, + append_batch_size=False, + stop_gradient=False, + ) + x2 = fluid.layers.data( + name='x2', + shape=x_shape, + dtype=x_type, + append_batch_size=False, + stop_gradient=False, + ) x1_np = np.random.randn(2, 3, 4, 5).astype(x_type) x2_np = np.random.randn(2, 3, 4, 5).astype(x_type) - out_np = corr(x1_np, - x2_np, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1) - - out = fluid.contrib.correlation(x1, - x2, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1) + out_np = corr( + x1_np, + x2_np, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + ) + + out = fluid.contrib.correlation( + x1, + x2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + ) loss = fluid.layers.reduce_mean(out) optimizer = fluid.optimizer.Momentum(0.0001, 0.9) @@ -108,33 +128,31 @@ class TestCorrelationOp(unittest.TestCase): place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - res = exe.run(feed={ - 'x1': x1_np, - 'x2': x2_np - }, - fetch_list=[out.name, loss.name]) + res = exe.run( + feed={'x1': x1_np, 'x2': x2_np}, fetch_list=[out.name, loss.name] + ) np.testing.assert_allclose(res[0], out_np, rtol=1e-05, atol=1e-8) class Net(fluid.dygraph.Layer): - def __init__(self, name_scope): super(Net, self).__init__(name_scope) def forward(self, x1, x2): - y = fluid.contrib.correlation(x1, - x2, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1) + y = fluid.contrib.correlation( + x1, + x2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + ) return y class TestCorrelationOpDyGraph(unittest.TestCase): - def test_check_output(self): if not fluid.core.is_compiled_with_cuda(): return @@ -146,13 +164,15 @@ class TestCorrelationOpDyGraph(unittest.TestCase): with fluid.dygraph.guard(place): x1_np = np.random.randn(2, 3, 4, 5).astype(x_type) x2_np = np.random.randn(2, 3, 4, 5).astype(x_type) - out_np = corr(x1_np, - x2_np, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1) + out_np = corr( + x1_np, + x2_np, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + ) x1 = to_variable(x1_np) x2 = to_variable(x2_np) diff --git a/python/paddle/fluid/contrib/tests/test_fp16_utils.py b/python/paddle/fluid/contrib/tests/test_fp16_utils.py index 54753ce4479..af245a30fed 100644 --- a/python/paddle/fluid/contrib/tests/test_fp16_utils.py +++ b/python/paddle/fluid/contrib/tests/test_fp16_utils.py @@ -22,12 +22,11 @@ paddle.enable_static() class AMPTest(unittest.TestCase): - def test_find_op_index(self): block = fluid.default_main_program().global_block() op_desc = core.OpDesc() idx = fp16_utils.find_op_index(block.desc, op_desc) - assert (idx == -1) + assert idx == -1 def test_find_true_post_op(self): block = fluid.default_main_program().global_block() @@ -35,14 +34,14 @@ class AMPTest(unittest.TestCase): var1 = block.create_var(name="X", shape=[3], dtype='float32') var2 = block.create_var(name="Y", shape=[3], dtype='float32') var3 = block.create_var(name="Z", shape=[3], dtype='float32') - op1 = block.append_op(type="abs", - inputs={"X": [var1]}, - outputs={"Out": [var2]}) - op2 = block.append_op(type="abs", - inputs={"X": [var2]}, - outputs={"Out": [var3]}) + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]} + ) + op2 = block.append_op( + type="abs", inputs={"X": [var2]}, outputs={"Out": [var3]} + ) res = fp16_utils.find_true_post_op(block.ops, op1, "Y") - assert (res == [op2]) + assert res == [op2] if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/tests/test_image_classification_fp16.py b/python/paddle/fluid/contrib/tests/test_image_classification_fp16.py index bf09a145450..7edaeb2760b 100644 --- a/python/paddle/fluid/contrib/tests/test_image_classification_fp16.py +++ b/python/paddle/fluid/contrib/tests/test_image_classification_fp16.py @@ -29,21 +29,18 @@ paddle.enable_static() def resnet_cifar10(input, depth=32): - - def conv_bn_layer(input, - ch_out, - filter_size, - stride, - padding, - act='relu', - bias_attr=False): - tmp = fluid.layers.conv2d(input=input, - filter_size=filter_size, - num_filters=ch_out, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) + def conv_bn_layer( + input, ch_out, filter_size, stride, padding, act='relu', bias_attr=False + ): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr, + ) return fluid.layers.batch_norm(input=tmp, act=act) def shortcut(input, ch_in, ch_out, stride): @@ -66,33 +63,31 @@ def resnet_cifar10(input, depth=32): assert (depth - 2) % 6 == 0 n = (depth - 2) // 6 - conv1 = conv_bn_layer(input=input, - ch_out=16, - filter_size=3, - stride=1, - padding=1) + conv1 = conv_bn_layer( + input=input, ch_out=16, filter_size=3, stride=1, padding=1 + ) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) res2 = layer_warp(basicblock, res1, 16, 32, n, 2) res3 = layer_warp(basicblock, res2, 32, 64, n, 2) - pool = fluid.layers.pool2d(input=res3, - pool_size=8, - pool_type='avg', - pool_stride=1) + pool = fluid.layers.pool2d( + input=res3, pool_size=8, pool_type='avg', pool_stride=1 + ) return pool def vgg16_bn_drop(input): - def conv_block(input, num_filter, groups, dropouts): - return fluid.nets.img_conv_group(input=input, - pool_size=2, - pool_stride=2, - conv_num_filter=[num_filter] * groups, - conv_filter_size=3, - conv_act='relu', - conv_with_batchnorm=True, - conv_batchnorm_drop_rate=dropouts, - pool_type='max') + return fluid.nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max', + ) conv1 = conv_block(input, 64, 2, [0.3, 0]) conv2 = conv_block(conv1, 128, 2, [0.4, 0]) @@ -117,9 +112,9 @@ def train(net_type, use_cuda, save_dirname, is_local): train_program.random_seed = 123 startup_prog.random_seed = 456 with fluid.program_guard(train_program, startup_prog): - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') if net_type == "vgg": @@ -133,7 +128,8 @@ def train(net_type, use_cuda, save_dirname, is_local): logits = fluid.layers.fc(input=net, size=classdim, act="softmax") cost, predict = fluid.layers.softmax_with_cross_entropy( - logits, label, return_softmax=True) + logits, label, return_softmax=True + ) avg_cost = paddle.mean(cost) acc = fluid.layers.accuracy(input=predict, label=label) @@ -143,11 +139,14 @@ def train(net_type, use_cuda, save_dirname, is_local): optimizer = fluid.optimizer.Lamb(learning_rate=0.001) amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_varnames={"loss", "conv2d_0.w_0"}) - mp_optimizer = decorate(optimizer=optimizer, - amp_lists=amp_lists, - init_loss_scaling=8.0, - use_dynamic_loss_scaling=True) + custom_black_varnames={"loss", "conv2d_0.w_0"} + ) + mp_optimizer = decorate( + optimizer=optimizer, + amp_lists=amp_lists, + init_loss_scaling=8.0, + use_dynamic_loss_scaling=True, + ) mp_optimizer.minimize(avg_cost) loss_scaling = mp_optimizer.get_loss_scaling() @@ -157,11 +156,13 @@ def train(net_type, use_cuda, save_dirname, is_local): PASS_NUM = 1 # no shuffle for unit test - train_reader = paddle.batch(paddle.dataset.cifar.train10(), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.dataset.cifar.train10(), batch_size=BATCH_SIZE + ) - test_reader = paddle.batch(paddle.dataset.cifar.test10(), - batch_size=BATCH_SIZE) + test_reader = paddle.batch( + paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -175,18 +176,25 @@ def train(net_type, use_cuda, save_dirname, is_local): np_scaled_loss, loss = exe.run( main_program, feed=feeder.feed(data), - fetch_list=[scaled_loss, avg_cost]) + fetch_list=[scaled_loss, avg_cost], + ) print( - 'PassID {0:1}, BatchID {1:04}, train loss {2:2.4}, scaled train closs {3:2.4}' - .format(pass_id, batch_id + 1, float(loss), - float(np_scaled_loss))) + 'PassID {0:1}, BatchID {1:04}, train loss {2:2.4}, scaled train closs {3:2.4}'.format( + pass_id, + batch_id + 1, + float(loss), + float(np_scaled_loss), + ) + ) if (batch_id % 10) == 0: acc_list = [] avg_loss_list = [] for tid, test_data in enumerate(test_reader()): - loss_t, acc_t = exe.run(program=test_program, - feed=feeder.feed(test_data), - fetch_list=[avg_cost, acc]) + loss_t, acc_t = exe.run( + program=test_program, + feed=feeder.feed(test_data), + fetch_list=[avg_cost, acc], + ) if math.isnan(float(loss_t)): sys.exit("got NaN loss, training failed.") acc_list.append(float(acc_t)) @@ -197,16 +205,23 @@ def train(net_type, use_cuda, save_dirname, is_local): avg_loss_value = numpy.array(avg_loss_list).mean() print( - 'PassID {0:1}, BatchID {1:04}, test loss {2:2.2}, acc {3:2.2}' - .format(pass_id, batch_id + 1, float(avg_loss_value), - float(acc_value))) + 'PassID {0:1}, BatchID {1:04}, test loss {2:2.2}, acc {3:2.2}'.format( + pass_id, + batch_id + 1, + float(avg_loss_value), + float(acc_value), + ) + ) if acc_value > 0.08: # Low threshold for speeding up CI fluid.io.save_inference_model( - save_dirname, ["pixel"], [predict], + save_dirname, + ["pixel"], + [predict], exe, main_program=train_program, - clip_extra=True) + clip_extra=True, + ) return if is_local: @@ -226,8 +241,9 @@ def train(net_type, use_cuda, save_dirname, is_local): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -247,8 +263,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # The input's dimension of conv should be 4-D or 5-D. # Use normilized image pixels as input data, which should be in the range [0, 1.0]. @@ -257,22 +276,25 @@ def infer(use_cuda, save_dirname=None): # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("infer results: ", results[0]) - fluid.io.save_inference_model(save_dirname, - feed_target_names, - fetch_targets, - exe, - inference_program, - clip_extra=True) + fluid.io.save_inference_model( + save_dirname, + feed_target_names, + fetch_targets, + exe, + inference_program, + clip_extra=True, + ) class TestImageClassification(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -286,18 +308,22 @@ class TestImageClassification(unittest.TestCase): # Directory for saving the trained model save_dirname = os.path.join( self.temp_dir.name, - "image_classification_" + net_type + ".inference.model") + "image_classification_" + net_type + ".inference.model", + ) train(net_type, use_cuda, save_dirname, is_local) - #infer(use_cuda, save_dirname) + # infer(use_cuda, save_dirname) def test_amp_lists(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists() self.assertEqual(amp_lists.white_list, white_list) @@ -306,106 +332,130 @@ class TestImageClassification(unittest.TestCase): def test_amp_lists_1(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 1. w={'exp}, b=None white_list.add('exp') black_list.remove('exp') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - {'exp'}) + {'exp'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_2(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 2. w={'tanh'}, b=None white_list.add('tanh') gray_list.remove('tanh') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - {'tanh'}) + {'tanh'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_3(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 3. w={'lstm'}, b=None white_list.add('lstm') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - {'lstm'}) + {'lstm'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_4(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 4. w=None, b={'conv2d'} white_list.remove('conv2d') black_list.add('conv2d') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_list={'conv2d'}) + custom_black_list={'conv2d'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_5(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 5. w=None, b={'tanh'} black_list.add('tanh') gray_list.remove('tanh') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_list={'tanh'}) + custom_black_list={'tanh'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_6(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 6. w=None, b={'lstm'} black_list.add('lstm') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_list={'lstm'}) + custom_black_list={'lstm'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) @@ -413,9 +463,12 @@ class TestImageClassification(unittest.TestCase): def test_amp_lists_7(self): # 7. w={'lstm'} b={'lstm'} # raise ValueError - self.assertRaises(ValueError, - fluid.contrib.mixed_precision.AutoMixedPrecisionLists, - {'lstm'}, {'lstm'}) + self.assertRaises( + ValueError, + fluid.contrib.mixed_precision.AutoMixedPrecisionLists, + {'lstm'}, + {'lstm'}, + ) def test_vgg_cuda(self): with self.scope_prog_guard(): @@ -436,37 +489,43 @@ class TestImageClassification(unittest.TestCase): class TestAmpWithNonIterableDataLoader(unittest.TestCase): - def decorate_with_data_loader(self): main_prog = paddle.static.Program() start_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): with paddle.fluid.unique_name.guard(): - image = fluid.layers.data(name='image', - shape=[3, 224, 224], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + image = fluid.layers.data( + name='image', shape=[3, 224, 224], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) py_reader = fluid.io.DataLoader.from_generator( feed_list=[image, label], capacity=4, iterable=False, - use_double_buffer=False) + use_double_buffer=False, + ) net = vgg16_bn_drop(image) logits = fluid.layers.fc(input=net, size=10, act="softmax") cost, predict = fluid.layers.softmax_with_cross_entropy( - logits, label, return_softmax=True) + logits, label, return_softmax=True + ) avg_cost = paddle.mean(cost) optimizer = fluid.optimizer.Lamb(learning_rate=0.001) - amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_varnames={"loss", "conv2d_0.w_0"}) - mp_optimizer = decorate(optimizer=optimizer, - amp_lists=amp_lists, - init_loss_scaling=8.0, - use_dynamic_loss_scaling=True) + amp_lists = ( + fluid.contrib.mixed_precision.AutoMixedPrecisionLists( + custom_black_varnames={"loss", "conv2d_0.w_0"} + ) + ) + mp_optimizer = decorate( + optimizer=optimizer, + amp_lists=amp_lists, + init_loss_scaling=8.0, + use_dynamic_loss_scaling=True, + ) mp_optimizer.minimize(avg_cost) diff --git a/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py b/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py index ab79242ae00..c000d55fccd 100644 --- a/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py +++ b/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py @@ -30,7 +30,8 @@ def convert_uint16_to_float(in_list): in_list = np.asarray(in_list) out = np.vectorize( lambda x: struct.unpack(' 0 else: sys.stderr.write('Do not get AVX flag on %s\n' % sysstr) @@ -162,10 +187,10 @@ def avx_supported(): def run_shell_command(cmd): import subprocess - out, err = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True).communicate() + + out, err = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ).communicate() if err: return None else: @@ -174,8 +199,9 @@ def run_shell_command(cmd): def get_dso_path(core_so, dso_name): if core_so and dso_name: - return run_shell_command("ldd %s|grep %s|awk '{print $3}'" % - (core_so, dso_name)) + return run_shell_command( + "ldd %s|grep %s|awk '{print $3}'" % (core_so, dso_name) + ) else: return None @@ -184,6 +210,7 @@ def load_dso(dso_absolute_path): if dso_absolute_path: try: from ctypes import cdll + cdll.LoadLibrary(dso_absolute_path) except: warnings.warn("Load {} failed".format(dso_absolute_path)) @@ -242,12 +269,14 @@ if platform.system().lower() == 'linux': try: from . import libpaddle + if avx_supported() and not libpaddle.is_compiled_with_avx(): sys.stderr.write( "Hint: Your machine support AVX, but the installed paddlepaddle doesn't have avx core. " "Hence, no-avx core with worse preformance will be imported.\nIf you like, you could " "reinstall paddlepaddle by 'python -m pip install --force-reinstall paddlepaddle-gpu[==version]' " - "to get better performance.\n") + "to get better performance.\n" + ) # assign tensor alias libpaddle.LoDTensor = libpaddle.Tensor @@ -278,6 +307,7 @@ try: from .libpaddle import _Profiler, _ProfilerResult, _RecordEvent from .libpaddle import _set_current_stream from .libpaddle import _get_phi_kernel_name + if sys.platform != 'win32': from .libpaddle import _set_process_pids from .libpaddle import _erase_process_pids @@ -290,12 +320,18 @@ try: except Exception as e: if has_paddle_dy_lib: sys.stderr.write( - 'Error: Can not import paddle core while this file exists: ' + - current_path + os.sep + 'libpaddle.' + dy_lib_suffix + '\n') + 'Error: Can not import paddle core while this file exists: ' + + current_path + + os.sep + + 'libpaddle.' + + dy_lib_suffix + + '\n' + ) if not avx_supported() and libpaddle.is_compiled_with_avx(): sys.stderr.write( "Error: Your machine doesn't support AVX, but the installed PaddlePaddle is avx core, " - "you should reinstall paddlepaddle with no-avx core.\n") + "you should reinstall paddlepaddle with no-avx core.\n" + ) raise e @@ -312,22 +348,26 @@ def set_paddle_custom_device_lib_path(lib_path): # set paddle lib path def set_paddle_lib_path(): - site_dirs = site.getsitepackages() if hasattr( - site, - 'getsitepackages') else [x for x in sys.path if 'site-packages' in x] + site_dirs = ( + site.getsitepackages() + if hasattr(site, 'getsitepackages') + else [x for x in sys.path if 'site-packages' in x] + ) for site_dir in site_dirs: lib_dir = os.path.sep.join([site_dir, 'paddle', 'libs']) if os.path.exists(lib_dir): _set_paddle_lib_path(lib_dir) set_paddle_custom_device_lib_path( - os.path.sep.join([lib_dir, '..', '..', 'paddle-plugins'])) + os.path.sep.join([lib_dir, '..', '..', 'paddle-plugins']) + ) return if hasattr(site, 'USER_SITE'): lib_dir = os.path.sep.join([site.USER_SITE, 'paddle', 'libs']) if os.path.exists(lib_dir): _set_paddle_lib_path(lib_dir) set_paddle_custom_device_lib_path( - os.path.sep.join([lib_dir, '..', '..', 'paddle-plugins'])) + os.path.sep.join([lib_dir, '..', '..', 'paddle-plugins']) + ) set_paddle_lib_path() diff --git a/python/paddle/fluid/data.py b/python/paddle/fluid/data.py index 99e38b9470b..00173a29c28 100644 --- a/python/paddle/fluid/data.py +++ b/python/paddle/fluid/data.py @@ -114,11 +114,13 @@ def data(name, shape, dtype='float32', lod_level=0): if shape[i] is None: shape[i] = -1 - return helper.create_global_variable(name=name, - shape=shape, - dtype=dtype, - type=core.VarDesc.VarType.LOD_TENSOR, - stop_gradient=True, - lod_level=lod_level, - is_data=True, - need_check_feed=True) + return helper.create_global_variable( + name=name, + shape=shape, + dtype=dtype, + type=core.VarDesc.VarType.LOD_TENSOR, + stop_gradient=True, + lod_level=lod_level, + is_data=True, + need_check_feed=True, + ) diff --git a/python/paddle/fluid/data_feed_desc.py b/python/paddle/fluid/data_feed_desc.py index 0aa72cadaf5..7a58c5ef5e1 100644 --- a/python/paddle/fluid/data_feed_desc.py +++ b/python/paddle/fluid/data_feed_desc.py @@ -175,7 +175,8 @@ class DataFeedDesc(object): ) for name in dense_slots_name: self.proto_desc.multi_slot_desc.slots[ - self.__name_to_index[name]].is_dense = True + self.__name_to_index[name] + ].is_dense = True def set_use_slots(self, use_slots_name): """ @@ -220,7 +221,8 @@ class DataFeedDesc(object): ) for name in use_slots_name: self.proto_desc.multi_slot_desc.slots[ - self.__name_to_index[name]].is_used = True + self.__name_to_index[name] + ].is_used = True def desc(self): """ diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py index a6bf7c6d440..e84d41e5620 100644 --- a/python/paddle/fluid/data_feeder.py +++ b/python/paddle/fluid/data_feeder.py @@ -18,7 +18,13 @@ import os import multiprocessing import warnings -from .framework import Variable, default_main_program, _current_expected_place, _non_static_mode, _in_eager_without_dygraph_check +from .framework import ( + Variable, + default_main_program, + _current_expected_place, + _non_static_mode, + _in_eager_without_dygraph_check, +) from .framework import _cpu_num, _cuda_ids __all__ = ['DataFeeder'] @@ -45,18 +51,46 @@ def convert_dtype(dtype): return _PADDLE_DTYPE_2_NUMPY_DTYPE[dtype] elif isinstance(dtype, type): if dtype in [ - bool, np.float16, np.uint16, np.float32, np.float64, np.int8, - np.int16, np.int32, np.int64, np.uint8, np.complex64, - np.complex128 + bool, + np.float16, + np.uint16, + np.float32, + np.float64, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.complex64, + np.complex128, ]: return dtype.__name__ else: if dtype in [ - 'bool', 'float16', 'uint16', 'float32', 'float64', 'int8', - 'int16', 'int32', 'int64', 'uint8', 'complex64', 'complex128', - u'bool', u'float16', u'uint16', u'float32', u'float64', u'int8', - u'int16', u'int32', u'int64', u'uint8', u'complex64', - u'complex128' + 'bool', + 'float16', + 'uint16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + 'complex64', + 'complex128', + u'bool', + u'float16', + u'uint16', + u'float32', + u'float64', + u'int8', + u'int16', + u'int32', + u'int64', + u'uint8', + u'complex64', + u'complex128', ]: # this code is a little bit dangerous, since error could happen # when casting no-ascii code to str in python2. @@ -70,14 +104,13 @@ def convert_dtype(dtype): raise TypeError( "dtype must be any of [bool, float16, uint16, float32, float64, int8, int16, " - "int32, int64, uint8, complex64, complex128], but received %s" % dtype) + "int32, int64, uint8, complex64, complex128], but received %s" % dtype + ) -def check_variable_and_dtype(input, - input_name, - expected_dtype, - op_name, - extra_message=''): +def check_variable_and_dtype( + input, input_name, expected_dtype, op_name, extra_message='' +): check_type(input, input_name, Variable, op_name, extra_message) check_dtype(input.dtype, input_name, expected_dtype, op_name, extra_message) @@ -97,60 +130,76 @@ def check_type(input, input_name, expected_type, op_name, extra_message=''): # @declarative in transformation from dygrah to static layer. We add VarBase in # expected_type to skip checking because varBase may be created and used in unusual way. from .dygraph.base import in_declarative_mode + # Need a better design to be fix this. if in_declarative_mode(): if not isinstance(expected_type, tuple): - expected_type = (expected_type, ) - expected_type += (core.VarBase, ) + expected_type = (expected_type,) + expected_type += (core.VarBase,) if _in_eager_without_dygraph_check(): - expected_type += (core.eager.Tensor, ) + expected_type += (core.eager.Tensor,) elif isinstance(input, core.VarBase): raise TypeError( "Please use `with fluid.dygraph.guard()` as context or `fluid.enable_dygraph()` to switch to imperative mode firstly. " "Because received '{}' in {} is a imperative Variable.".format( - input_name, op_name)) + input_name, op_name + ) + ) elif hasattr(core, "eager"): if isinstance(input, core.eager.Tensor): raise TypeError( "Please use `with fluid.dygraph.guard()` as context or `fluid.enable_dygraph()` to switch to imperative mode firstly. " "Because received '{}' in {} is a imperative Variable.".format( - input_name, op_name)) + input_name, op_name + ) + ) if not isinstance(input, expected_type): raise TypeError( - "The type of '%s' in %s must be %s, but received %s. %s" % - (input_name, op_name, expected_type, type(input), extra_message)) + "The type of '%s' in %s must be %s, but received %s. %s" + % (input_name, op_name, expected_type, type(input), extra_message) + ) -def check_dtype(input_dtype, - input_name, - expected_dtype, - op_name, - extra_message=''): +def check_dtype( + input_dtype, input_name, expected_dtype, op_name, extra_message='' +): # See NOTE [ Why skip dynamic graph check ] if _non_static_mode(): return if convert_dtype(input_dtype) in ['float16']: warnings.warn( - "The data type of '%s' in %s only support float16 in GPU now. %s" % - (input_name, op_name, extra_message)) + "The data type of '%s' in %s only support float16 in GPU now. %s" + % (input_name, op_name, extra_message) + ) if convert_dtype(input_dtype) in ['uint16'] and op_name not in [ - 'reshape', 'lookup_table', 'scale' + 'reshape', + 'lookup_table', + 'scale', ]: warnings.warn( "The data type of '%s' in %s only support bfloat16 in OneDNN now. %s" - % (input_name, op_name, extra_message)) + % (input_name, op_name, extra_message) + ) if convert_dtype(input_dtype) not in expected_dtype: raise TypeError( - "The data type of '%s' in %s must be %s, but received %s. %s" % - (input_name, op_name, expected_dtype, convert_dtype(input_dtype), - extra_message)) - - -def check_shape(shape, + "The data type of '%s' in %s must be %s, but received %s. %s" + % ( + input_name, op_name, - expected_shape_type=(list, tuple, Variable), - expected_element_type=(int, Variable), - expected_tensor_dtype=('int32', 'int64')): + expected_dtype, + convert_dtype(input_dtype), + extra_message, + ) + ) + + +def check_shape( + shape, + op_name, + expected_shape_type=(list, tuple, Variable), + expected_element_type=(int, Variable), + expected_tensor_dtype=('int32', 'int64'), +): # See NOTE [ Why skip dynamic graph check ] if _non_static_mode(): return @@ -160,16 +209,19 @@ def check_shape(shape, check_type(item, 'element of shape', expected_element_type, op_name) if expected_tensor_dtype is not None and isinstance(item, Variable): check_dtype( - item.dtype, 'element of shape', expected_tensor_dtype, + item.dtype, + 'element of shape', + expected_tensor_dtype, op_name, - 'If element of shape is Tensor, its data type should be {}'. - format(', '.join(expected_tensor_dtype))) + 'If element of shape is Tensor, its data type should be {}'.format( + ', '.join(expected_tensor_dtype) + ), + ) if expected_tensor_dtype is not None and isinstance(shape, Variable): check_dtype(shape.dtype, 'shape', expected_tensor_dtype, op_name) class DataToLoDTensorConverter(object): - def __init__(self, place, lod_level, shape, dtype): self.place = place self.lod_level = lod_level @@ -203,8 +255,10 @@ class DataToLoDTensorConverter(object): for s1, s2 in zip(self.shape, shape): if s1 != s2 and s1 >= 0 and s2 >= 0: raise ValueError( - "Shape not match. What is defined in data layer is {}, but receive {}" - .format(self.shape, shape)) + "Shape not match. What is defined in data layer is {}, but receive {}".format( + self.shape, shape + ) + ) def done(self): arr = np.array(self.data, dtype=self.dtype) @@ -214,8 +268,10 @@ class DataToLoDTensorConverter(object): arr = arr.reshape(self.shape) except ValueError: raise ValueError( - "Reshape error. What is defined in data layer is {}, but receive {}" - .format(self.shape, arr.shape)) + "Reshape error. What is defined in data layer is {}, but receive {}".format( + self.shape, arr.shape + ) + ) t = core.LoDTensor() t.set(arr, self.place) if self.lod_level > 0: @@ -225,7 +281,6 @@ class DataToLoDTensorConverter(object): class BatchedTensorProvider(object): - def __init__(self, feed_list, place, batch_size, generator, drop_last): self.place = place self.batch_size = batch_size @@ -236,10 +291,13 @@ class BatchedTensorProvider(object): for var in feed_list: assert var.lod_level == 0, "lod_level must be 0" self.converters.append( - DataToLoDTensorConverter(place=self.place, - lod_level=0, - shape=var.shape, - dtype=var.dtype)) + DataToLoDTensorConverter( + place=self.place, + lod_level=0, + shape=var.shape, + dtype=var.dtype, + ) + ) def _done(self): return [c.done() for c in self.converters] @@ -380,18 +438,23 @@ class DataFeeder(object): """ converter = [] - for lod_level, shape, dtype in zip(self.feed_lod_level, - self.feed_shapes, self.feed_dtypes): + for lod_level, shape, dtype in zip( + self.feed_lod_level, self.feed_shapes, self.feed_dtypes + ): converter.append( - DataToLoDTensorConverter(place=self.place, - lod_level=lod_level, - shape=shape, - dtype=dtype)) + DataToLoDTensorConverter( + place=self.place, + lod_level=lod_level, + shape=shape, + dtype=dtype, + ) + ) for each_sample in iterable: assert len(each_sample) == len(converter), ( - "The number of fields in data (%d) does not match " + - "len(feed_list) (%d)") % (len(each_sample), len(converter)) + "The number of fields in data (%d) does not match " + + "len(feed_list) (%d)" + ) % (len(each_sample), len(converter)) for each_converter, each_slot in zip(converter, each_sample): each_converter.feed(each_slot) ret_dict = {} @@ -465,10 +528,12 @@ class DataFeeder(object): ] if len(iterable) != len(places): - raise ValueError("feed_parallel takes multiple mini-batches. Each " - "mini-batch will be feed on each device. The " - "number of devices and number of mini-batches " - "must be same.") + raise ValueError( + "feed_parallel takes multiple mini-batches. Each " + "mini-batch will be feed on each device. The " + "number of devices and number of mini-batches " + "must be same." + ) place = self.place for p, batch in zip(places, iterable): @@ -484,11 +549,9 @@ class DataFeeder(object): else: return _cpu_num() - def decorate_reader(self, - reader, - multi_devices, - num_places=None, - drop_last=True): + def decorate_reader( + self, reader, multi_devices, num_places=None, drop_last=True + ): """ Decorate the reader (generator) to fit multiple devices. The reader generate multiple mini-batches. Each mini-batch will be fed into a single device. @@ -566,6 +629,7 @@ class DataFeeder(object): raise ValueError( "The data batch which cannot fit for devices will be " "dropped is not implementation. Other strategies are " - "not implemented") + "not implemented" + ) return __reader_creator__ diff --git a/python/paddle/fluid/dataloader/__init__.py b/python/paddle/fluid/dataloader/__init__.py index 9136db3de80..c0b2052283b 100644 --- a/python/paddle/fluid/dataloader/__init__.py +++ b/python/paddle/fluid/dataloader/__init__.py @@ -24,7 +24,9 @@ from .dataloader_iter import * from . import sampler from .sampler import * -__all__ = dataset.__all__ \ - + batch_sampler.__all__ \ - + dataloader_iter.__all__ \ - + sampler.__all__ +__all__ = ( + dataset.__all__ + + batch_sampler.__all__ + + dataloader_iter.__all__ + + sampler.__all__ +) diff --git a/python/paddle/fluid/dataloader/batch_sampler.py b/python/paddle/fluid/dataloader/batch_sampler.py index 7c87bd3d4cd..624754ae286 100644 --- a/python/paddle/fluid/dataloader/batch_sampler.py +++ b/python/paddle/fluid/dataloader/batch_sampler.py @@ -96,36 +96,51 @@ class BatchSampler(Sampler): """ - def __init__(self, - dataset=None, - sampler=None, - shuffle=False, - batch_size=1, - drop_last=False): + def __init__( + self, + dataset=None, + sampler=None, + shuffle=False, + batch_size=1, + drop_last=False, + ): if dataset is None: - assert sampler is not None, \ - "either dataset or sampler should be set" - assert isinstance(sampler, Sampler), \ - "sampler should be a paddle.io.Sampler, but got {}".format(type(sampler)) + assert ( + sampler is not None + ), "either dataset or sampler should be set" + assert isinstance( + sampler, Sampler + ), "sampler should be a paddle.io.Sampler, but got {}".format( + type(sampler) + ) assert not shuffle, "shuffle should be False when sampler is set" self.sampler = sampler else: - assert not isinstance(dataset, IterableDataset), \ - "dataset should not be a paddle.io.IterableDataset" - assert sampler is None, \ - "should not set both dataset and sampler" - assert isinstance(shuffle, bool), \ - "shuffle should be a boolean value, but got {}".format(type(shuffle)) + assert not isinstance( + dataset, IterableDataset + ), "dataset should not be a paddle.io.IterableDataset" + assert sampler is None, "should not set both dataset and sampler" + assert isinstance( + shuffle, bool + ), "shuffle should be a boolean value, but got {}".format( + type(shuffle) + ) if shuffle: self.sampler = RandomSampler(dataset) else: self.sampler = SequenceSampler(dataset) - assert isinstance(batch_size, int) and batch_size > 0, \ - "batch_size should be a positive integer, but got {}".format(batch_size) + assert ( + isinstance(batch_size, int) and batch_size > 0 + ), "batch_size should be a positive integer, but got {}".format( + batch_size + ) self.batch_size = batch_size - assert isinstance(drop_last, bool), \ - "drop_last should be a boolean value, but got {}".format(type(drop_last)) + assert isinstance( + drop_last, bool + ), "drop_last should be a boolean value, but got {}".format( + type(drop_last) + ) self.drop_last = drop_last def __iter__(self): @@ -145,7 +160,6 @@ class BatchSampler(Sampler): class _InfiniteIterableSampler(object): - def __init__(self, dataset, batch_size=1): assert isinstance( dataset, IterableDataset @@ -214,36 +228,41 @@ class DistributedBatchSampler(BatchSampler): break """ - def __init__(self, - dataset, - batch_size, - num_replicas=None, - rank=None, - shuffle=False, - drop_last=False): + def __init__( + self, + dataset, + batch_size, + num_replicas=None, + rank=None, + shuffle=False, + drop_last=False, + ): self.dataset = dataset - assert isinstance(batch_size, int) and batch_size > 0, \ - "batch_size should be a positive integer" + assert ( + isinstance(batch_size, int) and batch_size > 0 + ), "batch_size should be a positive integer" self.batch_size = batch_size - assert isinstance(shuffle, bool), \ - "shuffle should be a boolean value" + assert isinstance(shuffle, bool), "shuffle should be a boolean value" self.shuffle = shuffle - assert isinstance(drop_last, bool), \ - "drop_last should be a boolean number" + assert isinstance( + drop_last, bool + ), "drop_last should be a boolean number" from paddle.fluid.dygraph.parallel import ParallelEnv if num_replicas is not None: - assert isinstance(num_replicas, int) and num_replicas > 0, \ - "num_replicas should be a positive integer" + assert ( + isinstance(num_replicas, int) and num_replicas > 0 + ), "num_replicas should be a positive integer" self.nranks = num_replicas else: self.nranks = ParallelEnv().nranks if rank is not None: - assert isinstance(rank, int) and rank >= 0, \ - "rank should be a non-negative integer" + assert ( + isinstance(rank, int) and rank >= 0 + ), "rank should be a non-negative integer" self.local_rank = rank else: self.local_rank = ParallelEnv().local_rank @@ -256,7 +275,7 @@ class DistributedBatchSampler(BatchSampler): def __iter__(self): num_samples = len(self.dataset) indices = np.arange(num_samples).tolist() - indices += indices[:(self.total_size - len(indices))] + indices += indices[: (self.total_size - len(indices))] assert len(indices) == self.total_size if self.shuffle: np.random.RandomState(self.epoch).shuffle(indices) @@ -269,16 +288,21 @@ class DistributedBatchSampler(BatchSampler): assert last_batch_size % self.nranks == 0 last_local_batch_size = last_batch_size // self.nranks - for i in range(self.local_rank * self.batch_size, - len(indices) - last_batch_size, - self.batch_size * self.nranks): - subsampled_indices.extend(indices[i:i + self.batch_size]) + for i in range( + self.local_rank * self.batch_size, + len(indices) - last_batch_size, + self.batch_size * self.nranks, + ): + subsampled_indices.extend(indices[i : i + self.batch_size]) - indices = indices[len(indices) - last_batch_size:] + indices = indices[len(indices) - last_batch_size :] subsampled_indices.extend( - indices[self.local_rank * - last_local_batch_size:(self.local_rank + 1) * - last_local_batch_size]) + indices[ + self.local_rank + * last_local_batch_size : (self.local_rank + 1) + * last_local_batch_size + ] + ) return subsampled_indices if self.nranks > 1: diff --git a/python/paddle/fluid/dataloader/collate.py b/python/paddle/fluid/dataloader/collate.py index 3c46b54156d..50b86ca41e5 100644 --- a/python/paddle/fluid/dataloader/collate.py +++ b/python/paddle/fluid/dataloader/collate.py @@ -66,18 +66,20 @@ def default_collate_fn(batch): return batch elif isinstance(sample, Mapping): return { - key: default_collate_fn([d[key] for d in batch]) - for key in sample + key: default_collate_fn([d[key] for d in batch]) for key in sample } elif isinstance(sample, Sequence): sample_fields_num = len(sample) if not all(len(sample) == sample_fields_num for sample in iter(batch)): raise RuntimeError( - "fileds number not same among samples in a batch") + "fileds number not same among samples in a batch" + ) return [default_collate_fn(fields) for fields in zip(*batch)] - raise TypeError("batch data con only contains: tensor, numpy.ndarray, " - "dict, list, number, but got {}".format(type(sample))) + raise TypeError( + "batch data con only contains: tensor, numpy.ndarray, " + "dict, list, number, but got {}".format(type(sample)) + ) def default_convert_fn(batch): diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index 71e4a9f83ad..83d95c47925 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -24,7 +24,11 @@ import threading import numpy as np import multiprocessing from collections import namedtuple -from paddle.fluid.framework import _set_expected_place, _current_expected_place, set_flags +from paddle.fluid.framework import ( + _set_expected_place, + _current_expected_place, + set_flags, +) # NOTE: queue has a different name in python2 and python3 import queue @@ -34,13 +38,23 @@ import paddle.profiler as profiler from paddle.profiler.utils import in_profiler_mode from .. import core, layers from ..framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph -from ..multiprocess_utils import _set_SIGCHLD_handler, MP_STATUS_CHECK_INTERVAL, CleanupFuncRegistrar +from ..multiprocess_utils import ( + _set_SIGCHLD_handler, + MP_STATUS_CHECK_INTERVAL, + CleanupFuncRegistrar, +) from .fetcher import _IterableDatasetFetcher, _MapDatasetFetcher from .batch_sampler import _InfiniteIterableSampler from .collate import default_collate_fn, default_convert_fn -from .worker import ParentWatchDog, get_worker_info, _worker_loop, \ - _DatasetKind, _IterableDatasetStopIteration, _WorkerException, \ - _ResumeIteration +from .worker import ( + ParentWatchDog, + get_worker_info, + _worker_loop, + _DatasetKind, + _IterableDatasetStopIteration, + _WorkerException, + _ResumeIteration, +) from .flat import _flatten_batch, _restore_batch from paddle.profiler.timer import benchmark @@ -98,7 +112,9 @@ class _DataLoaderIterBase(object): self._use_buffer_reader = loader.use_buffer_reader self._prefetch_factor = loader.prefetch_factor self._use_shared_memory = loader.use_shared_memory - self._timeout = loader.timeout if loader.timeout > 0 else MP_STATUS_CHECK_INTERVAL + self._timeout = ( + loader.timeout if loader.timeout > 0 else MP_STATUS_CHECK_INTERVAL + ) self._worker_init_fn = loader.worker_init_fn self._dataset_kind = loader.dataset_kind self._pin_memory = loader.pin_memory @@ -155,8 +171,12 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): super(_DataLoaderIterSingleProcess, self).__init__(loader) self._dataset_fetcher = _DatasetKind.create_fetcher( - self._dataset_kind, self._dataset, self._auto_collate_batch, - self._collate_fn, self._drop_last) + self._dataset_kind, + self._dataset, + self._auto_collate_batch, + self._collate_fn, + self._drop_last, + ) # NOTE: _structrue_infos used to record the data structure of # batch to restore batch structure after reading Tensor @@ -170,7 +190,8 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): # iteration, set blocking_queue can cache "self._prefetch_factor" iteration datas # at most here self._blocking_queue_capacity = self._prefetch_factor * len( - self._places) + self._places + ) self._init_thread() self._shutdown = False @@ -187,20 +208,30 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): ] # if only 1 place, do not need to keep order self._blocking_queue = core.init_lod_tensor_blocking_queue( - core.Variable(), self._blocking_queue_capacity, - len(self._places) > 1) + core.Variable(), + self._blocking_queue_capacity, + len(self._places) > 1, + ) self._reader = core.create_py_reader( - self._blocking_queue, self._var_names, self._shapes, self._dtypes, - self._need_check_feed, self._places, self._use_buffer_reader, True, - self._pin_memory) - - self._thread = threading.Thread(target=self._thread_loop, - args=(_current_expected_place(), )) + self._blocking_queue, + self._var_names, + self._shapes, + self._dtypes, + self._need_check_feed, + self._places, + self._use_buffer_reader, + True, + self._pin_memory, + ) + + self._thread = threading.Thread( + target=self._thread_loop, args=(_current_expected_place(),) + ) self._thread.daemon = True self._thread.start() def _thread_loop(self, legacy_expected_place): - #NOTE(zhiqiu): Set the expected place for new thread as the same as father thread, + # NOTE(zhiqiu): Set the expected place for new thread as the same as father thread, # and it will call platform::SetDeviceId() in c++ internally. # If we do not set cudaDeviceId in new thread, the default cudaDeviceId will be 0, # Which may cost hundreds of MB of GPU memory on CUDAPlace(0) if calling some cuda @@ -215,19 +246,22 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): # read data from dataset in mini-batch # with paddle.fluid.dygraph.guard(place=paddle.CPUPlace()): # read data from dataset in mini-batch - batch = self._dataset_fetcher.fetch(indices, - self._thread_done_event) + batch = self._dataset_fetcher.fetch( + indices, self._thread_done_event + ) except StopIteration: self._exit_thread_expectedly() return - if batch is None or self._thread_done_event.is_set(): break + if batch is None or self._thread_done_event.is_set(): + break # flat batch and record structure infos batch, structure = _flatten_batch(batch) self._structure_infos.append(structure) - if self._thread_done_event.is_set(): break + if self._thread_done_event.is_set(): + break try: # pack as LoDTensorArray @@ -242,7 +276,8 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): array.append(slot) - if self._thread_done_event.is_set(): break + if self._thread_done_event.is_set(): + break try: self._blocking_queue.push(array) @@ -259,14 +294,16 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): if in_profiler_mode(): trace_event = profiler.RecordEvent( name="_DataLoaderIterSingleProcess", - event_type=profiler.TracerEventType.Dataloader) + event_type=profiler.TracerEventType.Dataloader, + ) trace_event.begin() try: benchmark().check_if_need_record(self) benchmark().before_reader() if in_dygraph_mode(): data = core.eager.read_next_tensor_list( - self._reader.read_next_list()[0]) + self._reader.read_next_list()[0] + ) data = _restore_batch(data, self._structure_infos.pop(0)) else: if _in_legacy_dygraph(): @@ -281,8 +318,9 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): self._structure_infos.pop(0) for _ in range(len(self._places)) ] - data = [_restore_batch(d, s) \ - for d, s in zip(data, structs)] + data = [ + _restore_batch(d, s) for d, s in zip(data, structs) + ] # static graph organized data on multi-device with list, if # place number is 1, there is only 1 device, extra the data # from list for devices to be compatible with dygraph mode @@ -341,15 +379,17 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): class _DataLoaderIterMultiProcess(_DataLoaderIterBase): - def __init__(self, loader): super(_DataLoaderIterMultiProcess, self).__init__(loader) self._persistent_workers = loader._persistent_workers self._resume_worker_cnt = 0 - assert self._num_workers > 0, "Multi-process DataLoader " \ - "invalid num_workers({})".format(self._num_workers) + assert ( + self._num_workers > 0 + ), "Multi-process DataLoader " "invalid num_workers({})".format( + self._num_workers + ) # subprocess wrokers' result queue self._data_queue = None @@ -370,7 +410,8 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): # output data for at least "_prefetch_factor" iterations(Note that len(_places) # batches will be composed as an iteration output) self._outstanding_capacity = self._prefetch_factor * max( - self._num_workers, len(self._places)) + self._num_workers, len(self._places) + ) # see _try_put_indices self._thread_lock = threading.Lock() @@ -405,12 +446,22 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): self._indices_queues.append(indices_queue) worker = multiprocessing.Process( target=_worker_loop, - args=(self._dataset, self._dataset_kind, indices_queue, - self._data_queue, self._workers_done_event, - self._auto_collate_batch, self._collate_fn, - self._drop_last, self._worker_init_fn, i, - self._num_workers, self._use_shared_memory, - self._base_seed)) + args=( + self._dataset, + self._dataset_kind, + indices_queue, + self._data_queue, + self._workers_done_event, + self._auto_collate_batch, + self._collate_fn, + self._drop_last, + self._worker_init_fn, + i, + self._num_workers, + self._use_shared_memory, + self._base_seed, + ), + ) worker.daemon = True worker.start() self._workers.append(worker) @@ -438,17 +489,25 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): ] # if only 1 place, do not need to keep order self._blocking_queue = core.init_lod_tensor_blocking_queue( - core.Variable(), self._outstanding_capacity, - len(self._places) > 1) + core.Variable(), self._outstanding_capacity, len(self._places) > 1 + ) self._reader = core.create_py_reader( - self._blocking_queue, self._var_names, self._shapes, self._dtypes, - self._need_check_feed, self._places, self._use_buffer_reader, True, - self._pin_memory) + self._blocking_queue, + self._var_names, + self._shapes, + self._dtypes, + self._need_check_feed, + self._places, + self._use_buffer_reader, + True, + self._pin_memory, + ) self._thread_done_event = threading.Event() # thread event is only need in multi-processing mode - self._thread = threading.Thread(target=self._thread_loop, - args=(_current_expected_place(), )) + self._thread = threading.Thread( + target=self._thread_loop, args=(_current_expected_place(),) + ) self._thread.daemon = True self._thread.start() @@ -471,7 +530,8 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): while self._blocking_queue.size() >= len(self._places): if in_dygraph_mode(): data = core.eager.read_next_tensor_list( - self._reader.read_next_list()[0]) + self._reader.read_next_list()[0] + ) else: if _in_legacy_dygraph(): self._reader.read_next_var_list() @@ -497,8 +557,9 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): self._try_put_indices() def _shutdown_worker(self, worker_id, shutdown=False): - if self._worker_status[worker_id] or (self._persistent_workers - and shutdown): + if self._worker_status[worker_id] or ( + self._persistent_workers and shutdown + ): self._indices_queues[worker_id].put(None) self._worker_status[worker_id] = False @@ -526,7 +587,7 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): self._shutdown = True def _thread_loop(self, legacy_expected_place): - #NOTE(zhiqiu): Set the expected place for new thread as the same as father thread, + # NOTE(zhiqiu): Set the expected place for new thread as the same as father thread, # and it will call platform::SetDeviceId() in c++ internally. # If we do not set cudaDeviceId in new thread, the default cudaDeviceId will be 0, # Which may cost hundreds of MB of GPU memory on CUDAPlace(0) if calling some cuda @@ -555,8 +616,8 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): # serializable, cannot be create in workers for slot in batch: if isinstance( - slot, - (paddle.Tensor, core.eager.Tensor)): + slot, (paddle.Tensor, core.eager.Tensor) + ): slot = slot.value().get_tensor() elif not isinstance(slot, core.LoDTensor): tmp = core.LoDTensor() @@ -607,8 +668,10 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): if self._batches_outstanding < len(self._places): return None - if self._rcvd_idx in self._task_infos and \ - len(self._task_infos[self._rcvd_idx]) == 3: + if ( + self._rcvd_idx in self._task_infos + and len(self._task_infos[self._rcvd_idx]) == 3 + ): info = self._task_infos.pop(self._rcvd_idx) self._structure_infos.append(info[2]) return info[1] @@ -637,8 +700,10 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): if len(failed_workers) > 0: self._exit_thread_unexpectedly() pids = ', '.join(str(w.pid) for w in failed_workers) - raise RuntimeError("DataLoader {} workers exit unexpectedly, " \ - "pids: {}".format(len(failed_workers), pids)) + raise RuntimeError( + "DataLoader {} workers exit unexpectedly, " + "pids: {}".format(len(failed_workers), pids) + ) # get(timeout) will call _poll(timeout) and may raise IOError if isinstance(e, queue.Empty) or isinstance(e, IOError): @@ -646,12 +711,15 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): continue self._exit_thread_unexpectedly() - logging.error("DataLoader reader thread failed({}) to read data from " \ - "workers' result queue.".format(e)) + logging.error( + "DataLoader reader thread failed({}) to read data from " + "workers' result queue.".format(e) + ) six.reraise(*sys.exc_info()) else: if self._dataset_kind == _DatasetKind.ITER and isinstance( - data, _IterableDatasetStopIteration): + data, _IterableDatasetStopIteration + ): # if a worker get StopIteraion, we shutdown this worker, # note that this batch indices to trigger StopIteration # is discard, outstanding batch number should be decrease @@ -667,8 +735,11 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): idx, batch, structure = data - if isinstance(idx, _ResumeIteration) and batch is None \ - and structure is None: + if ( + isinstance(idx, _ResumeIteration) + and batch is None + and structure is None + ): return idx if isinstance(batch, _WorkerException): @@ -684,8 +755,9 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): continue def _try_put_indices(self): - assert self._batches_outstanding <= self._outstanding_capacity, \ - "too many indices have been put to queue" + assert ( + self._batches_outstanding <= self._outstanding_capacity + ), "too many indices have been put to queue" # In multi-process mode for IterableDataset, _try_put_indices will # be called both in main process(for our implement has blocking queue, # and blocking queue read is in main process) and thread, which may @@ -709,7 +781,7 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): return self._indices_queues[worker_idx].put((self._send_idx, indices)) - self._task_infos[self._send_idx] = (worker_idx, ) + self._task_infos[self._send_idx] = (worker_idx,) self._batches_outstanding += 1 self._send_idx += 1 @@ -723,7 +795,8 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): if in_profiler_mode(): trace_event = profiler.RecordEvent( name="_DataLoaderIterMultiProcess", - event_type=profiler.TracerEventType.Dataloader) + event_type=profiler.TracerEventType.Dataloader, + ) trace_event.begin() try: benchmark().check_if_need_record(self) @@ -744,7 +817,8 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): if in_dygraph_mode(): data = core.eager.read_next_tensor_list( - self._reader.read_next_list()[0]) + self._reader.read_next_list()[0] + ) data = _restore_batch(data, self._structure_infos.pop(0)) else: if _in_legacy_dygraph(): @@ -759,8 +833,9 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): self._structure_infos.pop(0) for _ in range(len(self._places)) ] - data = [_restore_batch(d, s) \ - for d, s in zip(data, structs)] + data = [ + _restore_batch(d, s) for d, s in zip(data, structs) + ] # static graph organized data on multi-device with list, if # place number is 1, there is only 1 device, extra the data # from list for devices to be compatible with dygraph mode diff --git a/python/paddle/fluid/dataloader/dataset.py b/python/paddle/fluid/dataloader/dataset.py index eaab078d7e8..5fe52196cd8 100755 --- a/python/paddle/fluid/dataloader/dataset.py +++ b/python/paddle/fluid/dataloader/dataset.py @@ -16,8 +16,13 @@ import paddle from .. import framework __all__ = [ - "Dataset", "IterableDataset", "TensorDataset", "ComposeDataset", - "ChainDataset", "random_split", "Subset" + "Dataset", + "IterableDataset", + "TensorDataset", + "ComposeDataset", + "ChainDataset", + "random_split", + "Subset", ] @@ -67,12 +72,16 @@ class Dataset(object): pass def __getitem__(self, idx): - raise NotImplementedError("'{}' not implement in class "\ - "{}".format('__getitem__', self.__class__.__name__)) + raise NotImplementedError( + "'{}' not implement in class " + "{}".format('__getitem__', self.__class__.__name__) + ) def __len__(self): - raise NotImplementedError("'{}' not implement in class "\ - "{}".format('__len__', self.__class__.__name__)) + raise NotImplementedError( + "'{}' not implement in class " + "{}".format('__len__', self.__class__.__name__) + ) class IterableDataset(Dataset): @@ -210,16 +219,22 @@ class IterableDataset(Dataset): pass def __iter__(self): - raise NotImplementedError("'{}' not implement in class "\ - "{}".format('__iter__', self.__class__.__name__)) + raise NotImplementedError( + "'{}' not implement in class " + "{}".format('__iter__', self.__class__.__name__) + ) def __getitem__(self, idx): - raise RuntimeError("'{}' should not be called for IterableDataset" \ - "{}".format('__getitem__', self.__class__.__name__)) + raise RuntimeError( + "'{}' should not be called for IterableDataset" + "{}".format('__getitem__', self.__class__.__name__) + ) def __len__(self): - raise RuntimeError("'{}' should not be called for IterableDataset" \ - "{}".format('__len__', self.__class__.__name__)) + raise RuntimeError( + "'{}' should not be called for IterableDataset" + "{}".format('__len__', self.__class__.__name__) + ) class TensorDataset(Dataset): @@ -261,9 +276,11 @@ class TensorDataset(Dataset): def __init__(self, tensors): if not framework._non_static_mode(): raise RuntimeError( - "TensorDataset con only be used in imperative mode") - assert all([tensor.shape[0] == tensors[0].shape[0] for tensor in tensors]), \ - "tensors not have same shape of the 1st dimension" + "TensorDataset con only be used in imperative mode" + ) + assert all( + [tensor.shape[0] == tensors[0].shape[0] for tensor in tensors] + ), "tensors not have same shape of the 1st dimension" self.tensors = tensors def __getitem__(self, index): @@ -330,13 +347,16 @@ class ComposeDataset(Dataset): self.datasets = list(datasets) assert len(self.datasets) > 0, "input datasets shoule not be empty" for i, dataset in enumerate(self.datasets): - assert isinstance(dataset, Dataset), \ - "each input dataset should be paddle.io.Dataset" - assert not isinstance(dataset, IterableDataset), \ - "paddle.io.IterableDataset not supported" + assert isinstance( + dataset, Dataset + ), "each input dataset should be paddle.io.Dataset" + assert not isinstance( + dataset, IterableDataset + ), "paddle.io.IterableDataset not supported" if i > 0: - assert len(dataset) == len(self.datasets[i-1]), \ - "lengths of datasets should be same" + assert len(dataset) == len( + self.datasets[i - 1] + ), "lengths of datasets should be same" def __len__(self): return len(self.datasets[0]) @@ -391,8 +411,9 @@ class ChainDataset(IterableDataset): self.datasets = list(datasets) assert len(self.datasets) > 0, "input datasets shoule not be empty" for i, dataset in enumerate(self.datasets): - assert isinstance(dataset, IterableDataset), \ - "ChainDataset only support paddle.io.IterableDataset" + assert isinstance( + dataset, IterableDataset + ), "ChainDataset only support paddle.io.IterableDataset" def __iter__(self): for dataset in self.datasets: @@ -492,7 +513,7 @@ def random_split(dataset, lengths, generator=None): # For example var.item() and var.tolist() indices = paddle.randperm(sum(lengths)).numpy().tolist() return [ - Subset(dataset, indices[offset - length:offset]) + Subset(dataset, indices[offset - length : offset]) for offset, length in zip(_accumulate(lengths), lengths) ] diff --git a/python/paddle/fluid/dataloader/fetcher.py b/python/paddle/fluid/dataloader/fetcher.py index 387032cdfbb..b4a2ff6b923 100644 --- a/python/paddle/fluid/dataloader/fetcher.py +++ b/python/paddle/fluid/dataloader/fetcher.py @@ -20,7 +20,6 @@ _WARNING_TO_LOG = True class _DatasetFetcher(object): - def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): self.dataset = dataset self.auto_collate_batch = auto_collate_batch @@ -37,48 +36,55 @@ class _DatasetFetcher(object): # done_event argument to check DataLoader exit status between # ecah sample processing in the batch def fetch(self, batch_indices, done_event=None): - raise NotImplementedError("'fetch' not implement for class {}".format( - self.__class__.__name__)) + raise NotImplementedError( + "'fetch' not implement for class {}".format(self.__class__.__name__) + ) def _log_warning(self): # only log warning on GPU 0 when distributed launch from ...distributed import get_world_size, get_rank + if get_world_size() >= 2 and get_rank() != 0: return - warn_str = "Detect dataset only contains single fileds, return format " \ - "changed since Paddle 2.1. In Paddle <= 2.0, DataLoader add " \ - "a list surround output data(e.g. return [data]), and in " \ - "Paddle >= 2.1, DataLoader return the single filed directly " \ - "(e.g. return data). For example, in following code: \n\n" - warn_str += \ - "import numpy as np\n" \ - "from paddle.io import DataLoader, Dataset\n\n" \ - "class RandomDataset(Dataset):\n" \ - " def __getitem__(self, idx):\n" \ - " data = np.random.random((2, 3)).astype('float32')\n\n" \ - " return data\n\n" \ - " def __len__(self):\n" \ - " return 10\n\n" \ - "dataset = RandomDataset()\n" \ - "loader = DataLoader(dataset, batch_size=1)\n" \ - "data = next(loader())\n\n" - - warn_str += "In Paddle <= 2.0, data is in format '[Tensor(shape=(1, 2, 3), " \ - "dtype=float32)]', and in Paddle >= 2.1, data is in format" \ - " 'Tensor(shape=(1, 2, 3), dtype=float32)'\n" - - logger = get_logger("DataLoader", - logging.INFO, - fmt='%(levelname)s: %(message)s') + warn_str = ( + "Detect dataset only contains single fileds, return format " + "changed since Paddle 2.1. In Paddle <= 2.0, DataLoader add " + "a list surround output data(e.g. return [data]), and in " + "Paddle >= 2.1, DataLoader return the single filed directly " + "(e.g. return data). For example, in following code: \n\n" + ) + warn_str += ( + "import numpy as np\n" + "from paddle.io import DataLoader, Dataset\n\n" + "class RandomDataset(Dataset):\n" + " def __getitem__(self, idx):\n" + " data = np.random.random((2, 3)).astype('float32')\n\n" + " return data\n\n" + " def __len__(self):\n" + " return 10\n\n" + "dataset = RandomDataset()\n" + "loader = DataLoader(dataset, batch_size=1)\n" + "data = next(loader())\n\n" + ) + + warn_str += ( + "In Paddle <= 2.0, data is in format '[Tensor(shape=(1, 2, 3), " + "dtype=float32)]', and in Paddle >= 2.1, data is in format" + " 'Tensor(shape=(1, 2, 3), dtype=float32)'\n" + ) + + logger = get_logger( + "DataLoader", logging.INFO, fmt='%(levelname)s: %(message)s' + ) logger.warning(warn_str) class _IterableDatasetFetcher(_DatasetFetcher): - def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): - super(_IterableDatasetFetcher, - self).__init__(dataset, auto_collate_batch, collate_fn, drop_last) + super(_IterableDatasetFetcher, self).__init__( + dataset, auto_collate_batch, collate_fn, drop_last + ) self.dataset_iter = iter(dataset) def fetch(self, batch_indices, done_event=None): @@ -94,13 +100,13 @@ class _IterableDatasetFetcher(_DatasetFetcher): else: return None - if len(data) == 0 or (self.drop_last - and len(data) < len(batch_indices)): + if len(data) == 0 or ( + self.drop_last and len(data) < len(batch_indices) + ): raise StopIteration global _WARNING_TO_LOG - if not isinstance(data[0], (Sequence, Mapping)) \ - and _WARNING_TO_LOG: + if not isinstance(data[0], (Sequence, Mapping)) and _WARNING_TO_LOG: self._log_warning() _WARNING_TO_LOG = False else: @@ -112,10 +118,10 @@ class _IterableDatasetFetcher(_DatasetFetcher): class _MapDatasetFetcher(_DatasetFetcher): - def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): - super(_MapDatasetFetcher, self).__init__(dataset, auto_collate_batch, - collate_fn, drop_last) + super(_MapDatasetFetcher, self).__init__( + dataset, auto_collate_batch, collate_fn, drop_last + ) def fetch(self, batch_indices, done_event=None): if self.auto_collate_batch: @@ -127,8 +133,7 @@ class _MapDatasetFetcher(_DatasetFetcher): return None global _WARNING_TO_LOG - if not isinstance(data[0], (Sequence, Mapping)) \ - and _WARNING_TO_LOG: + if not isinstance(data[0], (Sequence, Mapping)) and _WARNING_TO_LOG: self._log_warning() _WARNING_TO_LOG = False else: diff --git a/python/paddle/fluid/dataloader/flat.py b/python/paddle/fluid/dataloader/flat.py index 5baf4cc853e..6f3c6edf0b0 100644 --- a/python/paddle/fluid/dataloader/flat.py +++ b/python/paddle/fluid/dataloader/flat.py @@ -36,39 +36,47 @@ def _flatten_batch(batch): def _flatten(batch, flat_batch, structure, field_idx): if isinstance(batch, Sequence): for field in batch: - if isinstance(field, (np.ndarray, paddle.Tensor, - paddle.fluid.core.eager.Tensor)): + if isinstance( + field, + (np.ndarray, paddle.Tensor, paddle.fluid.core.eager.Tensor), + ): structure.append('{}{}'.format(FIELD_PREFIX, field_idx)) flat_batch.append(field) field_idx += 1 elif isinstance(field, (str, bytes, numbers.Number)): structure.append(field) elif isinstance(field, Sequence): - field_struct, field_idx = _flatten(field, flat_batch, [], - field_idx) + field_struct, field_idx = _flatten( + field, flat_batch, [], field_idx + ) structure.append(field_struct) elif isinstance(field, Mapping): - field_struct, field_idx = _flatten(field, flat_batch, {}, - field_idx) + field_struct, field_idx = _flatten( + field, flat_batch, {}, field_idx + ) structure.append(field_struct) else: structure.append(field) elif isinstance(batch, Mapping): for k, field in batch.items(): - if isinstance(field, (np.ndarray, paddle.Tensor, - paddle.fluid.core.eager.Tensor)): + if isinstance( + field, + (np.ndarray, paddle.Tensor, paddle.fluid.core.eager.Tensor), + ): structure[k] = '{}{}'.format(FIELD_PREFIX, field_idx) flat_batch.append(field) field_idx += 1 elif isinstance(field, (str, bytes, numbers.Number)): structure[k] = field elif isinstance(field, Sequence): - field_struct, field_idx = _flatten(field, flat_batch, [], - field_idx) + field_struct, field_idx = _flatten( + field, flat_batch, [], field_idx + ) structure[k] = field_struct elif isinstance(field, Mapping): - field_struct, field_idx = _flatten(field, flat_batch, {}, - field_idx) + field_struct, field_idx = _flatten( + field, flat_batch, {}, field_idx + ) structure[k] = field_struct else: structure[k] = field @@ -100,8 +108,9 @@ def _restore_batch(flat_batch, structure): if isinstance(field, str) and field.startswith(FIELD_PREFIX): cur_field_idx = int(field.replace(FIELD_PREFIX, '')) field_idx = max(field_idx, cur_field_idx) - assert flat_batch[cur_field_idx] is not None, \ - "flat_batch[{}] parsed repeatly" + assert ( + flat_batch[cur_field_idx] is not None + ), "flat_batch[{}] parsed repeatly" structure[i] = flat_batch[cur_field_idx] flat_batch[cur_field_idx] = None elif isinstance(field, (str, bytes, numbers.Number)): @@ -113,8 +122,9 @@ def _restore_batch(flat_batch, structure): if isinstance(field, str) and field.startswith(FIELD_PREFIX): cur_field_idx = int(field.replace(FIELD_PREFIX, '')) field_idx = max(field_idx, cur_field_idx) - assert flat_batch[cur_field_idx] is not None, \ - "flat_batch[{}] parsed repeatly" + assert ( + flat_batch[cur_field_idx] is not None + ), "flat_batch[{}] parsed repeatly" structure[k] = flat_batch[cur_field_idx] flat_batch[cur_field_idx] = None elif isinstance(field, (str, bytes, numbers.Number)): @@ -126,8 +136,7 @@ def _restore_batch(flat_batch, structure): return field_idx - assert isinstance(flat_batch, Sequence), \ - "flat_batch is not a list or tuple" + assert isinstance(flat_batch, Sequence), "flat_batch is not a list or tuple" # no np.array in dataset, no output tensor from blocking queue # simply return structure @@ -136,8 +145,9 @@ def _restore_batch(flat_batch, structure): # sample only contains single fields if isinstance(structure, (str, bytes)): - assert structure == '{}{}'.format(FIELD_PREFIX, 0), \ - "invalid structure: {}".format(structure) + assert structure == '{}{}'.format( + FIELD_PREFIX, 0 + ), "invalid structure: {}".format(structure) return flat_batch[0] field_idx = _restore(structure, 0) assert field_idx + 1 == len(flat_batch), "Tensor parse incomplete" diff --git a/python/paddle/fluid/dataloader/sampler.py b/python/paddle/fluid/dataloader/sampler.py index 77b8ff09ba0..3626ed63e52 100644 --- a/python/paddle/fluid/dataloader/sampler.py +++ b/python/paddle/fluid/dataloader/sampler.py @@ -16,7 +16,10 @@ import numpy as np from .. import core __all__ = [ - "Sampler", "SequenceSampler", "RandomSampler", "WeightedRandomSampler" + "Sampler", + "SequenceSampler", + "RandomSampler", + "WeightedRandomSampler", ] @@ -185,19 +188,19 @@ class RandomSampler(Sampler): see `paddle.io.Sampler` """ - def __init__(self, - data_source, - replacement=False, - num_samples=None, - generator=None): + def __init__( + self, data_source, replacement=False, num_samples=None, generator=None + ): self.data_source = data_source self.replacement = replacement self._num_samples = num_samples self.generator = generator if not isinstance(self.replacement, bool): - raise TypeError("expect boolean value for replacement, but got " - "replacement={}".format(self.replacement)) + raise TypeError( + "expect boolean value for replacement, but got " + "replacement={}".format(self.replacement) + ) if self._num_samples is not None and not replacement: raise ValueError( @@ -205,8 +208,10 @@ class RandomSampler(Sampler): ) if not isinstance(self.num_samples, int) or self.num_samples <= 0: - raise ValueError("num_samples should be a positive integer, " - "but got num_samples={}".format(self.num_samples)) + raise ValueError( + "num_samples should be a positive integer, " + "but got num_samples={}".format(self.num_samples) + ) @property def num_samples(self): @@ -225,13 +230,14 @@ class RandomSampler(Sampler): yield index else: if self.replacement: - for index in np.random.choice(np.arange(n), - self.num_samples, - replace=True).tolist(): + for index in np.random.choice( + np.arange(n), self.num_samples, replace=True + ).tolist(): yield index else: - for index in np.random.choice(np.arange(n), n, - replace=False).tolist(): + for index in np.random.choice( + np.arange(n), n, replace=False + ).tolist(): yield index def __len__(self): @@ -243,31 +249,29 @@ def _weighted_sample(weights, num_samples, replacement=True): weights = weights.numpy() if isinstance(weights, (list, tuple)): weights = np.array(weights) - assert isinstance(weights, np.ndarray), \ - "weights should be paddle.Tensor, numpy.ndarray, list or tuple" - assert len(weights.shape) <= 2, \ - "weights should be a 1-D or 2-D array" + assert isinstance( + weights, np.ndarray + ), "weights should be paddle.Tensor, numpy.ndarray, list or tuple" + assert len(weights.shape) <= 2, "weights should be a 1-D or 2-D array" weights = weights.reshape((-1, weights.shape[-1])) - assert np.all(weights >= 0.), \ - "weights should be positive value" - assert not np.any(weights == np.inf), \ - "weights shoule not be INF" - assert not np.any(weights == np.nan), \ - "weights shoule not be NaN" - - non_zeros = np.sum(weights > 0., axis=1) - assert np.all(non_zeros > 0), \ - "weights should have positive values" + assert np.all(weights >= 0.0), "weights should be positive value" + assert not np.any(weights == np.inf), "weights shoule not be INF" + assert not np.any(weights == np.nan), "weights shoule not be NaN" + + non_zeros = np.sum(weights > 0.0, axis=1) + assert np.all(non_zeros > 0), "weights should have positive values" if not replacement: - assert np.all(non_zeros >= num_samples), \ - "weights positive value number should not " \ + assert np.all(non_zeros >= num_samples), ( + "weights positive value number should not " "less than num_samples when replacement=False" + ) weights = weights / weights.sum(axis=1) rets = [] for i in range(weights.shape[0]): - ret = np.random.choice(weights.shape[1], num_samples, replacement, - weights[i]) + ret = np.random.choice( + weights.shape[1], num_samples, replacement, weights[i] + ) rets.append(ret) return np.array(rets) @@ -311,8 +315,9 @@ class WeightedRandomSampler(Sampler): self.replacement = replacement def __iter__(self): - idxs = _weighted_sample(self.weights, self.num_samples, - self.replacement) + idxs = _weighted_sample( + self.weights, self.num_samples, self.replacement + ) return iter(idxs.reshape((-1)).tolist()) def __len__(self): diff --git a/python/paddle/fluid/dataloader/worker.py b/python/paddle/fluid/dataloader/worker.py index 06ea7ef9d72..f0aa32e7745 100644 --- a/python/paddle/fluid/dataloader/worker.py +++ b/python/paddle/fluid/dataloader/worker.py @@ -21,7 +21,11 @@ import traceback from collections import namedtuple from .. import core from .fetcher import _IterableDatasetFetcher, _MapDatasetFetcher -from ..multiprocess_utils import _cleanup_mmap, CleanupFuncRegistrar, MP_STATUS_CHECK_INTERVAL +from ..multiprocess_utils import ( + _cleanup_mmap, + CleanupFuncRegistrar, + MP_STATUS_CHECK_INTERVAL, +) from ..framework import _non_static_mode, _in_eager_without_dygraph_check from .flat import _flatten_batch @@ -32,7 +36,6 @@ __all__ = ['get_worker_info'] class _IterableDatasetStopIteration(object): - def __init__(self, worker_id): self.worker_id = worker_id @@ -46,20 +49,22 @@ class _DatasetKind(object): ITER = 1 @staticmethod - def create_fetcher(kind, dataset, auto_collate_batch, collate_fn, - drop_last): + def create_fetcher( + kind, dataset, auto_collate_batch, collate_fn, drop_last + ): if kind == _DatasetKind.MAP: - return _MapDatasetFetcher(dataset, auto_collate_batch, collate_fn, - drop_last) + return _MapDatasetFetcher( + dataset, auto_collate_batch, collate_fn, drop_last + ) elif kind == _DatasetKind.ITER: - return _IterableDatasetFetcher(dataset, auto_collate_batch, - collate_fn, drop_last) + return _IterableDatasetFetcher( + dataset, auto_collate_batch, collate_fn, drop_last + ) else: raise NotImplementedError("unknown Dataset kind {}".format(kind)) class ParentWatchDog(object): - def __init__(self): self._parent_pid = os.getppid() self._parent_alive = True @@ -151,13 +156,15 @@ class WorkerInfo(object): def __setattr__(self, key, val): if self.__initialized: - raise RuntimeError("Cannot assign attributes to {} objects".format( - self.__class__.__name__)) + raise RuntimeError( + "Cannot assign attributes to {} objects".format( + self.__class__.__name__ + ) + ) return super(WorkerInfo, self).__setattr__(key, val) class _WorkerException(object): - def __init__(self, worker_id, exc_info=None): self.worker_id = worker_id exc_info = exc_info or sys.exc_info() @@ -166,7 +173,8 @@ class _WorkerException(object): def reraise(self): msg = "DataLoader worker({}) caught {} with message:\n{}".format( - self.worker_id, self.exc_type.__name__, self.exc_msg) + self.worker_id, self.exc_type.__name__, self.exc_msg + ) if getattr(self.exc_type, "message", None): raise self.exc_type(message=msg) raise self.exc_type(msg) @@ -204,12 +212,12 @@ class _WorkerException(object): # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -INIT_A = 0x43b0d7e5 -MULT_A = 0x931e8875 -INIT_B = 0x8b51f9dd -MULT_B = 0x58f38ded -MIX_MULT_L = 0xca01f9dd -MIX_MULT_R = 0x4973f715 +INIT_A = 0x43B0D7E5 +MULT_A = 0x931E8875 +INIT_B = 0x8B51F9DD +MULT_B = 0x58F38DED +MIX_MULT_L = 0xCA01F9DD +MIX_MULT_R = 0x4973F715 XSHIFT = np.dtype(np.uint32).itemsize * 8 // 2 MASK32 = 0xFFFFFFFF @@ -255,9 +263,21 @@ def _generate_states(base_seed=0, worker_id=0): return states -def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, - auto_collate_batch, collate_fn, drop_last, init_fn, worker_id, - num_workers, use_shared_memory, base_seed): +def _worker_loop( + dataset, + dataset_kind, + indices_queue, + out_queue, + done_event, + auto_collate_batch, + collate_fn, + drop_last, + init_fn, + worker_id, + num_workers, + use_shared_memory, + base_seed, +): try: # NOTE: [ mmap files clear ] When the child process exits unexpectedly, # some shared memory objects may have been applied for but have not yet @@ -282,18 +302,20 @@ def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, np.random.seed(_generate_states(base_seed, worker_id)) global _worker_info - _worker_info = WorkerInfo(id=worker_id, - num_workers=num_workers, - dataset=dataset, - seed=base_seed) + _worker_info = WorkerInfo( + id=worker_id, + num_workers=num_workers, + dataset=dataset, + seed=base_seed, + ) init_exception = None try: if init_fn is not None: init_fn(worker_id) - fetcher = _DatasetKind.create_fetcher(dataset_kind, dataset, - auto_collate_batch, - collate_fn, drop_last) + fetcher = _DatasetKind.create_fetcher( + dataset_kind, dataset, auto_collate_batch, collate_fn, drop_last + ) except: init_exception = _WorkerException(worker_id) @@ -309,15 +331,16 @@ def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, if isinstance(data, _ResumeIteration): out_queue.put((data, None, None)) iterator_drained = False - fetcher = _DatasetKind.create_fetcher(dataset_kind, dataset, - auto_collate_batch, - collate_fn, True) + fetcher = _DatasetKind.create_fetcher( + dataset_kind, dataset, auto_collate_batch, collate_fn, True + ) continue # None as poison piil, so worker event should be set if data is None: - assert done_event.is_set() or iterator_drained, \ - "get None when worker done_event set" + assert ( + done_event.is_set() or iterator_drained + ), "get None when worker done_event set" break # If worker done event is set but get still get data in # indices_queue, remaining data should be get and skipped. @@ -338,8 +361,10 @@ def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, with paddle.fluid.dygraph.guard(place=paddle.CPUPlace()): batch = fetcher.fetch(indices) except Exception as e: - if isinstance( - e, StopIteration) and dataset_kind == _DatasetKind.ITER: + if ( + isinstance(e, StopIteration) + and dataset_kind == _DatasetKind.ITER + ): out_queue.put(_IterableDatasetStopIteration(worker_id)) iterator_drained = True else: @@ -355,10 +380,12 @@ def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, if _in_eager_without_dygraph_check(): return core._array_to_share_memory_tensor(tensor) return tensor._share_memory() + tensor_list = [ core._array_to_share_memory_tensor(b) - if isinstance(b, np.ndarray) \ - else tensor_share_memory(b) for b in batch + if isinstance(b, np.ndarray) + else tensor_share_memory(b) + for b in batch ] out_queue.put((idx, tensor_list, structure)) core._remove_tensor_list_mmap_fds(tensor_list) diff --git a/python/paddle/fluid/dataset.py b/python/paddle/fluid/dataset.py index 4c7e8bb5378..ff68300d4a0 100644 --- a/python/paddle/fluid/dataset.py +++ b/python/paddle/fluid/dataset.py @@ -36,7 +36,7 @@ class DatasetFactory(object): """ def __init__(self): - """ Init. """ + """Init.""" pass def create_dataset(self, datafeed_class="QueueDataset"): @@ -59,15 +59,16 @@ class DatasetFactory(object): dataset = globals()[datafeed_class]() return dataset except: - raise ValueError("datafeed class %s does not exist" % - datafeed_class) + raise ValueError( + "datafeed class %s does not exist" % datafeed_class + ) class DatasetBase(object): - """ Base dataset class. """ + """Base dataset class.""" def __init__(self): - """ Init. """ + """Init.""" # define class name here # to decide whether we need create in memory instance self.proto_desc = data_feed_pb2.DataFeedDesc() @@ -374,7 +375,7 @@ class InMemoryDataset(DatasetBase): @deprecated(since="2.0.0", update_to="paddle.distributed.InMemoryDataset") def __init__(self): - """ Init. """ + """Init.""" super(InMemoryDataset, self).__init__() self.proto_desc.name = "MultiSlotInMemoryDataFeed" self.fleet_send_batch_size = None @@ -389,18 +390,22 @@ class InMemoryDataset(DatasetBase): self.fleet_send_sleep_seconds = None self.trainer_num = -1 - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_feed_type") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._set_feed_type", + ) def set_feed_type(self, data_feed_type): """ Set data_feed_desc """ self.proto_desc.name = data_feed_type - if (self.proto_desc.name == "SlotRecordInMemoryDataFeed"): + if self.proto_desc.name == "SlotRecordInMemoryDataFeed": self.dataset = core.Dataset("SlotRecordDataset") - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._prepare_to_run") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._prepare_to_run", + ) def _prepare_to_run(self): """ Set data_feed_desc before load or shuffle, @@ -423,8 +428,8 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to= - "paddle.distributed.InMemoryDataset._dynamic_adjust_before_train") + update_to="paddle.distributed.InMemoryDataset._dynamic_adjust_before_train", + ) def _dynamic_adjust_before_train(self, thread_num): if not self.is_user_set_queue_num: if self.use_ps_gpu: @@ -435,7 +440,7 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._dynamic_adjust_after_train" + update_to="paddle.distributed.InMemoryDataset._dynamic_adjust_after_train", ) def _dynamic_adjust_after_train(self): if not self.is_user_set_queue_num: @@ -445,8 +450,10 @@ class InMemoryDataset(DatasetBase): self.dataset.dynamic_adjust_channel_num(self.thread_num, False) self.dataset.dynamic_adjust_readers_num(self.thread_num) - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_queue_num") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._set_queue_num", + ) def set_queue_num(self, queue_num): """ Set Dataset output queue num, training threads get data from queues @@ -465,9 +472,10 @@ class InMemoryDataset(DatasetBase): self.is_user_set_queue_num = True self.queue_num = queue_num - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_parse_ins_id" - ) + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._set_parse_ins_id", + ) def set_parse_ins_id(self, parse_ins_id): """ Set id Dataset need to parse insid @@ -487,7 +495,8 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_parse_content") + update_to="paddle.distributed.InMemoryDataset._set_parse_content", + ) def set_parse_content(self, parse_content): """ Set if Dataset need to parse content @@ -539,9 +548,10 @@ class InMemoryDataset(DatasetBase): """ self.trainer_num = trainer_num - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_merge_by_sid" - ) + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._set_merge_by_sid", + ) def set_merge_by_sid(self, merge_by_sid): """ Set if Dataset need to merge sid. If not, one ins means one Pv. @@ -633,7 +643,7 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_fleet_send_batch_size" + update_to="paddle.distributed.InMemoryDataset._set_fleet_send_batch_size", ) def set_fleet_send_batch_size(self, fleet_send_batch_size=1024): """ @@ -654,8 +664,8 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to= - "paddle.distributed.InMemoryDataset._set_fleet_send_sleep_seconds") + update_to="paddle.distributed.InMemoryDataset._set_fleet_send_sleep_seconds", + ) def set_fleet_send_sleep_seconds(self, fleet_send_sleep_seconds=0): """ Set fleet send sleep time, default is 0 @@ -675,7 +685,8 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_merge_by_lineid") + update_to="paddle.distributed.InMemoryDataset._set_merge_by_lineid", + ) def set_merge_by_lineid(self, merge_size=2): """ Set merge by line id, instances of same line id will be merged after @@ -698,8 +709,8 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to= - "paddle.distributed.InMemoryDataset._set_generate_unique_feasigns") + update_to="paddle.distributed.InMemoryDataset._set_generate_unique_feasigns", + ) def set_generate_unique_feasigns(self, generate_uni_feasigns, shard_num): self.dataset.set_generate_unique_feasigns(generate_uni_feasigns) self.gen_uni_feasigns = generate_uni_feasigns @@ -707,13 +718,14 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to= - "paddle.distributed.InMemoryDataset._generate_local_tables_unlock") - def generate_local_tables_unlock(self, table_id, fea_dim, read_thread_num, - consume_thread_num, shard_num): - self.dataset.generate_local_tables_unlock(table_id, fea_dim, - read_thread_num, - consume_thread_num, shard_num) + update_to="paddle.distributed.InMemoryDataset._generate_local_tables_unlock", + ) + def generate_local_tables_unlock( + self, table_id, fea_dim, read_thread_num, consume_thread_num, shard_num + ): + self.dataset.generate_local_tables_unlock( + table_id, fea_dim, read_thread_num, consume_thread_num, shard_num + ) def set_date(self, date): """ @@ -738,8 +750,10 @@ class InMemoryDataset(DatasetBase): if self.use_ps_gpu and core._is_compiled_with_heterps(): self.psgpu.set_date(year, month, day) - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.load_into_memory") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.load_into_memory", + ) def load_into_memory(self, is_shuffle=False): """ Load data into memory @@ -766,7 +780,8 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.preload_into_memory") + update_to="paddle.distributed.InMemoryDataset.preload_into_memory", + ) def preload_into_memory(self, thread_num=None): """ Load data into memory in async mode @@ -792,9 +807,10 @@ class InMemoryDataset(DatasetBase): self.dataset.create_preload_readers() self.dataset.preload_into_memory() - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.wait_preload_done" - ) + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.wait_preload_done", + ) def wait_preload_done(self): """ Wait preload_into_memory done @@ -813,8 +829,10 @@ class InMemoryDataset(DatasetBase): self.dataset.wait_preload_done() self.dataset.destroy_preload_readers() - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.local_shuffle") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.local_shuffle", + ) def local_shuffle(self): """ Local shuffle @@ -832,8 +850,10 @@ class InMemoryDataset(DatasetBase): """ self.dataset.local_shuffle() - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.global_shuffle") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.global_shuffle", + ) def global_shuffle(self, fleet=None, thread_num=12): """ Global shuffle. @@ -893,8 +913,10 @@ class InMemoryDataset(DatasetBase): else: fleet._role_maker.barrier_worker() - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.release_memory") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.release_memory", + ) def release_memory(self): """ :api_attr: Static Graph @@ -946,7 +968,8 @@ class InMemoryDataset(DatasetBase): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.get_memory_data_size") + update_to="paddle.distributed.InMemoryDataset.get_memory_data_size", + ) def get_memory_data_size(self, fleet=None): """ Get memory data size, user can call this function to know the num @@ -975,18 +998,21 @@ class InMemoryDataset(DatasetBase): """ import numpy as np + local_data_size = self.dataset.get_memory_data_size() local_data_size = np.array([local_data_size]) if fleet is not None: global_data_size = local_data_size * 0 - fleet._role_maker.all_reduce_worker(local_data_size, - global_data_size) + fleet._role_maker.all_reduce_worker( + local_data_size, global_data_size + ) return global_data_size[0] return local_data_size[0] @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.get_shuffle_data_size") + update_to="paddle.distributed.InMemoryDataset.get_shuffle_data_size", + ) def get_shuffle_data_size(self, fleet=None): """ Get shuffle data size, user can call this function to know the num @@ -1017,6 +1043,7 @@ class InMemoryDataset(DatasetBase): """ import numpy as np + local_data_size = self.dataset.get_shuffle_data_size() local_data_size = np.array([local_data_size]) print('global shuffle local_data_size: ', local_data_size) @@ -1025,8 +1052,9 @@ class InMemoryDataset(DatasetBase): if hasattr(fleet, "util"): global_data_size = fleet.util.all_reduce(local_data_size) else: - fleet._role_maker.all_reduce_worker(local_data_size, - global_data_size) + fleet._role_maker.all_reduce_worker( + local_data_size, global_data_size + ) return global_data_size[0] return local_data_size[0] @@ -1070,16 +1098,20 @@ class InMemoryDataset(DatasetBase): self.proto_desc.graph_config.walk_len = config.get("walk_len", 20) self.proto_desc.graph_config.window = config.get("window", 5) self.proto_desc.graph_config.once_sample_startid_len = config.get( - "once_sample_startid_len", 8000) + "once_sample_startid_len", 8000 + ) self.proto_desc.graph_config.sample_times_one_chunk = config.get( - "sample_times_one_chunk", 10) + "sample_times_one_chunk", 10 + ) self.proto_desc.graph_config.batch_size = config.get("batch_size", 1) self.proto_desc.graph_config.debug_mode = config.get("debug_mode", 0) self.proto_desc.graph_config.first_node_type = config.get( - "first_node_type", "") + "first_node_type", "" + ) self.proto_desc.graph_config.meta_path = config.get("meta_path", "") self.proto_desc.graph_config.gpu_graph_training = config.get( - "gpu_graph_training", True) + "gpu_graph_training", True + ) self.dataset.set_gpu_graph_mode(True) @@ -1103,8 +1135,10 @@ class QueueDataset(DatasetBase): super(QueueDataset, self).__init__() self.proto_desc.name = "MultiSlotDataFeed" - @deprecated(since="2.0.0", - update_to="paddle.distributed.QueueDataset._prepare_to_run") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.QueueDataset._prepare_to_run", + ) def _prepare_to_run(self): """ Set data_feed_desc/thread num/filelist before run, @@ -1139,7 +1173,8 @@ class QueueDataset(DatasetBase): """ raise NotImplementedError( "QueueDataset does not support local shuffle, " - "please use InMemoryDataset for local_shuffle") + "please use InMemoryDataset for local_shuffle" + ) def global_shuffle(self, fleet=None): """ @@ -1165,7 +1200,8 @@ class QueueDataset(DatasetBase): """ raise NotImplementedError( "QueueDataset does not support global shuffle, " - "please use InMemoryDataset for global_shuffle") + "please use InMemoryDataset for global_shuffle" + ) class FileInstantDataset(DatasetBase): @@ -1194,7 +1230,8 @@ class FileInstantDataset(DatasetBase): """ raise NotImplementedError( "FileInstantDataset does not support local shuffle, " - "please use InMemoryDataset for local_shuffle") + "please use InMemoryDataset for local_shuffle" + ) def global_shuffle(self, fleet=None): """ @@ -1203,7 +1240,8 @@ class FileInstantDataset(DatasetBase): """ raise NotImplementedError( "FileInstantDataset does not support global shuffle, " - "please use InMemoryDataset for global_shuffle") + "please use InMemoryDataset for global_shuffle" + ) class BoxPSDataset(InMemoryDataset): @@ -1289,7 +1327,7 @@ class BoxPSDataset(InMemoryDataset): filelist = ["a.txt", "b.txt"] dataset.set_filelist(filelist) dataset.load_into_memory() - """ + """ self._prepare_to_run() self.boxps.load_into_memory() diff --git a/python/paddle/fluid/debugger.py b/python/paddle/fluid/debugger.py index 0c2d61db2d2..36f38df38e3 100644 --- a/python/paddle/fluid/debugger.py +++ b/python/paddle/fluid/debugger.py @@ -52,8 +52,9 @@ def repr_data_type(type): def repr_tensor(proto): - return "tensor(type={}, shape={})".format(_dtype2str_[int(proto.data_type)], - str(proto.dims)) + return "tensor(type={}, shape={})".format( + _dtype2str_[int(proto.data_type)], str(proto.dims) + ) reprtpl = "{ttype} {name} ({reprs})" @@ -65,30 +66,37 @@ def repr_lodtensor(proto): level = proto.type.lod_tensor.lod_level reprs = repr_tensor(proto.type.lod_tensor.tensor) - return reprtpl.format(ttype="LoDTensor" if level > 0 else "Tensor", - name=proto.name, - reprs="level=%d, %s" % - (level, reprs) if level > 0 else reprs) + return reprtpl.format( + ttype="LoDTensor" if level > 0 else "Tensor", + name=proto.name, + reprs="level=%d, %s" % (level, reprs) if level > 0 else reprs, + ) def repr_selected_rows(proto): if proto.type.type != framework_pb2.VarType.SELECTED_ROWS: return - return reprtpl.format(ttype="SelectedRows", - name=proto.name, - reprs=repr_tensor(proto.type.selected_rows)) + return reprtpl.format( + ttype="SelectedRows", + name=proto.name, + reprs=repr_tensor(proto.type.selected_rows), + ) def repr_tensor_array(proto): if proto.type.type != framework_pb2.VarType.LOD_TENSOR_ARRAY: return - return reprtpl.format(ttype="TensorArray", - name=proto.name, - reprs="level=%d, %s" % - (proto.type.tensor_array.lod_level, - repr_tensor(proto.type.lod_tensor.tensor))) + return reprtpl.format( + ttype="TensorArray", + name=proto.name, + reprs="level=%d, %s" + % ( + proto.type.tensor_array.lod_level, + repr_tensor(proto.type.lod_tensor.tensor), + ), + ) type_handlers = [ @@ -115,19 +123,23 @@ def pprint_program_codes(program_desc): def pprint_block_codes(block_desc, show_backward=False): - def is_op_backward(op_desc): - if op_desc.type.endswith('_grad'): return True + if op_desc.type.endswith('_grad'): + return True def is_var_backward(var): - if "@GRAD" in var.parameter: return True + if "@GRAD" in var.parameter: + return True for arg in var.arguments: - if "@GRAD" in arg: return True + if "@GRAD" in arg: + return True for var in op_desc.inputs: - if is_var_backward(var): return True + if is_var_backward(var): + return True for var in op_desc.outputs: - if is_var_backward(var): return True + if is_var_backward(var): + return True return False def is_var_backward(var_desc): @@ -135,7 +147,8 @@ def pprint_block_codes(block_desc, show_backward=False): if type(block_desc) is not framework_pb2.BlockDesc: block_desc = framework_pb2.BlockDesc.FromString( - block_desc.desc.serialize_to_string()) + block_desc.desc.serialize_to_string() + ) var_reprs = [] op_reprs = [] for var in block_desc.vars: @@ -144,7 +157,8 @@ def pprint_block_codes(block_desc, show_backward=False): var_reprs.append(repr_var(var)) for op in block_desc.ops: - if not show_backward and is_op_backward(op): continue + if not show_backward and is_op_backward(op): + continue op_reprs.append(repr_op(op)) tpl = "// block-{idx} parent-{pidx}\n// variables\n{vars}\n\n// operators\n{ops}\n" @@ -182,7 +196,8 @@ def _repr_op_fill_constant(optype, inputs, outputs, attrs): return "{output} = {data} [shape={shape}]".format( output=','.join(outputs), data=attrs['value'], - shape=str(attrs['shape'])) + shape=str(attrs['shape']), + ) op_repr_handlers = [ @@ -216,13 +231,16 @@ def repr_op(opdesc): for handler in op_repr_handlers: res = handler(opdesc.type, inputs, outputs, attr_dict) - if res: return res + if res: + return res - return tpl.format(outputs=', '.join(outputs), - optype=opdesc.type, - inputs=', '.join(inputs), - attrs="{%s}" % ','.join(attrs), - is_target=", is_target" if is_target else "") + return tpl.format( + outputs=', '.join(outputs), + optype=opdesc.type, + inputs=', '.join(inputs), + attrs="{%s}" % ','.join(attrs), + is_target=", is_target" if is_target else "", + ) def draw_block_graphviz(block, highlights=None, path="./temp.dot"): @@ -237,7 +255,8 @@ def draw_block_graphviz(block, highlights=None, path="./temp.dot"): desc = framework_pb2.BlockDesc.FromString(bytes(protostr)) def need_highlight(name): - if highlights is None: return False + if highlights is None: + return False for pattern in highlights: assert type(pattern) is str if re.match(pattern, name): @@ -250,9 +269,11 @@ def draw_block_graphviz(block, highlights=None, path="./temp.dot"): # TODO(gongwb): format the var.type # create var if var.persistable: - varn = graph.add_param(var.name, - str(var.type).replace("\n", "
", 1), - highlight=need_highlight(var.name)) + varn = graph.add_param( + var.name, + str(var.type).replace("\n", "
", 1), + highlight=need_highlight(var.name), + ) else: varn = graph.add_arg(var.name, highlight=need_highlight(var.name)) vars[var.name] = varn @@ -264,7 +285,8 @@ def draw_block_graphviz(block, highlights=None, path="./temp.dot"): vars[arg] = graph.add_arg(arg, highlight=need_highlight(arg)) varn = vars[arg] highlight = need_highlight(op.description) or need_highlight( - varn.description) + varn.description + ) if op2var: graph.add_edge(op, varn, highlight=highlight) else: diff --git a/python/paddle/fluid/device_worker.py b/python/paddle/fluid/device_worker.py index bf264f5a647..ee82c7ebbdc 100644 --- a/python/paddle/fluid/device_worker.py +++ b/python/paddle/fluid/device_worker.py @@ -14,8 +14,12 @@ """Defination of device workers.""" __all__ = [ - 'DeviceWorker', 'Hogwild', 'DownpourSGD', 'Section', 'DownpourSGDOPT', - 'HeterSection' + 'DeviceWorker', + 'Hogwild', + 'DownpourSGD', + 'Section', + 'DownpourSGDOPT', + 'HeterSection', ] @@ -67,7 +71,8 @@ class DeviceWorker(object): """ raise NotImplementedError( "DeviceWorker does not implement gen_worker_desc, " - "please use Hogwild or DownpourSGD, etc.") + "please use Hogwild or DownpourSGD, etc." + ) class Hogwild(DeviceWorker): @@ -90,10 +95,16 @@ class Hogwild(DeviceWorker): trainer_desc.device_worker_name = "HogwildWorker" if self._infer: # just ignore feed op for inference model - trainer_desc.hogwild_param.skip_ops.extend([ - "feed", "push_sparse", "push_sparse_v2", "push_dense", - "distributed_push_sparse", "send" - ]) + trainer_desc.hogwild_param.skip_ops.extend( + [ + "feed", + "push_sparse", + "push_sparse_v2", + "push_dense", + "distributed_push_sparse", + "send", + ] + ) dense_table_set = set() program_id = str(id(self._program)) @@ -114,8 +125,11 @@ class Hogwild(DeviceWorker): from paddle.fluid.incubate.fleet.parameter_server import version - if version.is_transpiler( - ) and "fleet_desc" not in opt_info and "program_configs" not in opt_info: + if ( + version.is_transpiler() + and "fleet_desc" not in opt_info + and "program_configs" not in opt_info + ): return program_configs = opt_info["program_configs"] @@ -126,8 +140,10 @@ class Hogwild(DeviceWorker): if pid == program_id: pc = downpour.program_config.add() pc.program_id = program_id - print("device worker pull dense:", - program_configs[program_id]["pull_dense"]) + print( + "device worker pull dense:", + program_configs[program_id]["pull_dense"], + ) for i in program_configs[program_id]["push_sparse"]: pc.push_sparse_table_id.extend([i]) for i in program_configs[program_id]["push_dense"]: @@ -143,36 +159,43 @@ class Hogwild(DeviceWorker): trainer_desc.device_worker_name = "HogwildWorker" pull_thread = trainer_desc.pull_dense_param pull_thread.device_num = trainer_desc.thread_num - if opt_info.get("program_id_to_worker") is None and opt_info.get( - "dense_table_config") is None: + if ( + opt_info.get("program_id_to_worker") is None + and opt_info.get("dense_table_config") is None + ): raise ValueError( - "opt_info must have program_id_to_worker or dense_table_config") + "opt_info must have program_id_to_worker or dense_table_config" + ) if opt_info.get("program_id_to_worker") is not None: prog_id_to_worker = opt_info["program_id_to_worker"] if prog_id_to_worker.get(program_id) is None: - raise ValueError("%s not found in program_id_to_worker" % - program_id) + raise ValueError( + "%s not found in program_id_to_worker" % program_id + ) worker = opt_info["program_id_to_worker"][program_id] for i in worker.get_desc().dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) - dense_table.table_id = \ - i.table_id + dense_table.table_id = i.table_id sparse_len = len(worker.get_desc().sparse_table) for i in range(sparse_len): sparse_table = downpour.sparse_table.add() - sparse_table.table_id = worker.get_desc( - ).sparse_table[i].table_id + sparse_table.table_id = ( + worker.get_desc().sparse_table[i].table_id + ) sparse_table.sparse_key_name.extend( - worker.get_desc().sparse_table[i].slot_key) + worker.get_desc().sparse_table[i].slot_key + ) sparse_table.sparse_value_name.extend( - worker.get_desc().sparse_table[i].slot_value) + worker.get_desc().sparse_table[i].slot_value + ) sparse_table.sparse_grad_name.extend( - worker.get_desc().sparse_table[i].slot_gradient) - sparse_table.fea_dim = \ - self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim + worker.get_desc().sparse_table[i].slot_gradient + ) + sparse_table.fea_dim = self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ + i + ].accessor.fea_dim # not use emb_dim sparse_table.emb_dim = -1 # not use hard code click @@ -184,7 +207,8 @@ class Hogwild(DeviceWorker): dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( - i.dense_gradient_variable_name) + i.dense_gradient_variable_name + ) hogwild.skip_ops.extend(worker.get_desc().skip_op) else: dense_table_config = opt_info.get("dense_table_config") @@ -196,7 +220,8 @@ class Hogwild(DeviceWorker): if self._infer: hogwild.skip_ops.extend( - ["push_sparse", "push_sparse_v2", "push_dense"]) + ["push_sparse", "push_sparse_v2", "push_dense"] + ) class DownpourLite(DeviceWorker): @@ -220,10 +245,16 @@ class DownpourLite(DeviceWorker): trainer_desc.device_worker_name = "DownpourLiteWorker" if self._infer: # just ignore feed op for inference model - trainer_desc.downpour_param.skip_ops.extend([ - "feed", "push_sparse", "push_sparse_v2", "push_dense", - "distributed_push_sparse", "send" - ]) + trainer_desc.downpour_param.skip_ops.extend( + [ + "feed", + "push_sparse", + "push_sparse_v2", + "push_dense", + "distributed_push_sparse", + "send", + ] + ) dense_table_set = set() program_id = str(id(self._program)) @@ -242,8 +273,11 @@ class DownpourLite(DeviceWorker): from paddle.fluid.incubate.fleet.parameter_server import version - if version.is_transpiler( - ) and "fleet_desc" not in opt_info and "program_configs" not in opt_info: + if ( + version.is_transpiler() + and "fleet_desc" not in opt_info + and "program_configs" not in opt_info + ): return program_configs = opt_info["program_configs"] @@ -254,8 +288,10 @@ class DownpourLite(DeviceWorker): if pid == program_id: pc = downpour.program_config.add() pc.program_id = program_id - print("device worker pull dense:", - program_configs[program_id]["pull_dense"]) + print( + "device worker pull dense:", + program_configs[program_id]["pull_dense"], + ) for i in program_configs[program_id]["push_sparse"]: pc.push_sparse_table_id.extend([i]) for i in program_configs[program_id]["push_dense"]: @@ -270,36 +306,43 @@ class DownpourLite(DeviceWorker): pull_thread = trainer_desc.pull_dense_param pull_thread.device_num = trainer_desc.thread_num - if opt_info.get("program_id_to_worker") is None and opt_info.get( - "dense_table_config") is None: + if ( + opt_info.get("program_id_to_worker") is None + and opt_info.get("dense_table_config") is None + ): raise ValueError( - "opt_info must have program_id_to_worker or dense_table_config") + "opt_info must have program_id_to_worker or dense_table_config" + ) if opt_info.get("program_id_to_worker") is not None: prog_id_to_worker = opt_info["program_id_to_worker"] if prog_id_to_worker.get(program_id) is None: - raise ValueError("%s not found in program_id_to_worker" % - program_id) + raise ValueError( + "%s not found in program_id_to_worker" % program_id + ) worker = opt_info["program_id_to_worker"][program_id] for i in worker.get_desc().dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) - dense_table.table_id = \ - i.table_id + dense_table.table_id = i.table_id sparse_len = len(worker.get_desc().sparse_table) for i in range(sparse_len): sparse_table = downpour.sparse_table.add() - sparse_table.table_id = worker.get_desc( - ).sparse_table[i].table_id + sparse_table.table_id = ( + worker.get_desc().sparse_table[i].table_id + ) sparse_table.sparse_key_name.extend( - worker.get_desc().sparse_table[i].slot_key) + worker.get_desc().sparse_table[i].slot_key + ) sparse_table.sparse_value_name.extend( - worker.get_desc().sparse_table[i].slot_value) + worker.get_desc().sparse_table[i].slot_value + ) sparse_table.sparse_grad_name.extend( - worker.get_desc().sparse_table[i].slot_gradient) - sparse_table.fea_dim = \ - self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim + worker.get_desc().sparse_table[i].slot_gradient + ) + sparse_table.fea_dim = self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ + i + ].accessor.fea_dim # not use emb_dim sparse_table.emb_dim = -1 # not use hard code click @@ -311,7 +354,8 @@ class DownpourLite(DeviceWorker): dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( - i.dense_gradient_variable_name) + i.dense_gradient_variable_name + ) downpour.skip_ops.extend(worker.get_desc().skip_op) else: dense_table_config = opt_info.get("dense_table_config") @@ -323,7 +367,8 @@ class DownpourLite(DeviceWorker): if self._infer: downpour.skip_ops.extend( - ["push_sparse", "push_sparse_v2", "push_dense"]) + ["push_sparse", "push_sparse_v2", "push_dense"] + ) class DownpourSGD(DeviceWorker): @@ -377,43 +422,53 @@ class DownpourSGD(DeviceWorker): mc_map.value = value break - trainer_desc.device_worker_name = opt_info.get("worker_class", - "DownpourWorker") + trainer_desc.device_worker_name = opt_info.get( + "worker_class", "DownpourWorker" + ) pull_thread = trainer_desc.pull_dense_param pull_thread.device_num = trainer_desc.thread_num if opt_info.get("program_id_to_worker") is None: raise ValueError("opt_info must have program_id_to_worker") prog_id_to_worker = opt_info["program_id_to_worker"] if prog_id_to_worker.get(program_id) is None: - raise ValueError("%s not found in program_id_to_worker" % - program_id) + raise ValueError( + "%s not found in program_id_to_worker" % program_id + ) worker = opt_info["program_id_to_worker"][program_id] for i in worker.get_desc().dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) - dense_table.table_id = \ - i.table_id + dense_table.table_id = i.table_id sparse_len = len(worker.get_desc().sparse_table) for i in range(sparse_len): sparse_table = downpour.sparse_table.add() sparse_table.table_id = worker.get_desc().sparse_table[i].table_id sparse_table.sparse_key_name.extend( - worker.get_desc().sparse_table[i].slot_key) + worker.get_desc().sparse_table[i].slot_key + ) sparse_table.sparse_value_name.extend( - worker.get_desc().sparse_table[i].slot_value) + worker.get_desc().sparse_table[i].slot_value + ) sparse_table.sparse_grad_name.extend( - worker.get_desc().sparse_table[i].slot_gradient) - if opt_info["use_cvm"] or "no_cvm" in opt_info and opt_info[ - "no_cvm"] == True: - sparse_table.emb_dim = \ - self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim + worker.get_desc().sparse_table[i].slot_gradient + ) + if ( + opt_info["use_cvm"] + or "no_cvm" in opt_info + and opt_info["no_cvm"] == True + ): + sparse_table.emb_dim = self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ + i + ].accessor.fea_dim sparse_table.fea_dim = sparse_table.emb_dim else: - sparse_table.emb_dim = \ + sparse_table.emb_dim = ( self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim - 2 + i + ].accessor.fea_dim + - 2 + ) sparse_table.fea_dim = sparse_table.emb_dim + 2 # TODO(guru4elephant): hard code here, need to improve sparse_table.label_var_name = "click" @@ -427,7 +482,8 @@ class DownpourSGD(DeviceWorker): dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( - i.dense_gradient_variable_name) + i.dense_gradient_variable_name + ) downpour.skip_ops.extend(worker.get_desc().skip_op) if self._infer: downpour.push_dense = False @@ -485,43 +541,56 @@ class DownpourSGDOPT(DeviceWorker): raise ValueError("opt_info must have program_id_to_worker") prog_id_to_worker = opt_info["program_id_to_worker"] if prog_id_to_worker.get(program_id) is None: - raise ValueError("%s not found in program_id_to_worker" % - program_id) + raise ValueError( + "%s not found in program_id_to_worker" % program_id + ) worker = opt_info["program_id_to_worker"][program_id] for i in worker.get_desc().dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) - dense_table.table_id = \ - i.table_id + dense_table.table_id = i.table_id sparse_len = len(worker.get_desc().sparse_table) for i in range(sparse_len): sparse_table = downpour.sparse_table.add() sparse_table.table_id = worker.get_desc().sparse_table[i].table_id sparse_table.sparse_key_name.extend( - worker.get_desc().sparse_table[i].slot_key) + worker.get_desc().sparse_table[i].slot_key + ) sparse_table.sparse_value_name.extend( - worker.get_desc().sparse_table[i].slot_value) + worker.get_desc().sparse_table[i].slot_value + ) sparse_table.sparse_grad_name.extend( - worker.get_desc().sparse_table[i].slot_gradient) - if opt_info["use_cvm"] or "no_cvm" in opt_info and opt_info[ - "no_cvm"] == True: - sparse_table.emb_dim = \ - self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim + worker.get_desc().sparse_table[i].slot_gradient + ) + if ( + opt_info["use_cvm"] + or "no_cvm" in opt_info + and opt_info["no_cvm"] == True + ): + sparse_table.emb_dim = self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ + i + ].accessor.fea_dim sparse_table.fea_dim = sparse_table.emb_dim else: - sparse_table.emb_dim = \ + sparse_table.emb_dim = ( self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim - 2 + i + ].accessor.fea_dim + - 2 + ) sparse_table.fea_dim = sparse_table.emb_dim + 2 # TODO(guru4elephant): hard code here, need to improve sparse_table.label_var_name = "click" - if "local_tables" in opt_info and sparse_table.table_id in opt_info[ - "local_tables"]: + if ( + "local_tables" in opt_info + and sparse_table.table_id in opt_info["local_tables"] + ): sparse_table.is_local = True - if "async_tables" in opt_info and sparse_table.table_id in opt_info[ - "async_tables"]: + if ( + "async_tables" in opt_info + and sparse_table.table_id in opt_info["async_tables"] + ): sparse_table.is_async = True if opt_info["stat_var_names"]: for i in opt_info["stat_var_names"]: @@ -533,7 +602,8 @@ class DownpourSGDOPT(DeviceWorker): dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( - i.dense_gradient_variable_name) + i.dense_gradient_variable_name + ) downpour.skip_ops.extend(worker.get_desc().skip_op) if self._infer: downpour.push_dense = False @@ -555,6 +625,7 @@ class Section(DeviceWorker): """ from google.protobuf import text_format from . import core + trainer_desc.device_worker_name = "SectionWorker" pipeline_opt = self._program._pipeline_opt section_param = trainer_desc.section_param @@ -567,16 +638,16 @@ class Section(DeviceWorker): # then runs Backward phase for all microbatches. # 1F1B scheduler, which runs forward phase and backward phase altertively # after startup phase. - assert schedule_mode_str in [ - "F-then-B", "1F1B" - ], ("The schedule mode " - "for pipeline must be one of F-then-B or 1F1B") + assert schedule_mode_str in ["F-then-B", "1F1B"], ( + "The schedule mode " "for pipeline must be one of F-then-B or 1F1B" + ) schedule_mode = 0 if schedule_mode_str == "F-then-B" else 1 section_param.schedule_mode = schedule_mode cfg = section_param.section_config program = pipeline_opt["section_program"] cfg.program_desc.ParseFromString( - program._get_desc().serialize_to_string()) + program._get_desc().serialize_to_string() + ) # TODO: why does not work # cfg.program_desc.CopyFrom(program.program._get_desc()) place = pipeline_opt["place"] @@ -604,23 +675,27 @@ class HeterSection(DeviceWorker): """ from google.protobuf import text_format from . import core + trainer_desc.device_worker_name = "HeterSectionWorker" heter_pipeline_opt = self._program._heter_pipeline_opt heter_section_param = trainer_desc.heter_section_param heter_section_param.num_microbatches = heter_pipeline_opt[ - "num_microbatches"] + "num_microbatches" + ] heter_section_param.pipeline_stage = heter_pipeline_opt[ - "pipeline_stage"] + "pipeline_stage" + ] heter_section_param.num_pipeline_stages = heter_pipeline_opt[ - "num_pipeline_stages"] + "num_pipeline_stages" + ] cfg = heter_section_param.section_config program = heter_pipeline_opt["section_program"] cfg.program_desc.ParseFromString( - program._get_desc().serialize_to_string()) + program._get_desc().serialize_to_string() + ) class DeviceWorkerFactory(object): - def _create_device_worker(self, worker_type): classname = worker_type.capitalize() return globals()[classname]() diff --git a/python/paddle/fluid/distribute_lookup_table.py b/python/paddle/fluid/distribute_lookup_table.py index 74824f68324..cff2388bfdb 100644 --- a/python/paddle/fluid/distribute_lookup_table.py +++ b/python/paddle/fluid/distribute_lookup_table.py @@ -70,8 +70,10 @@ def find_distributed_lookup_table(program): if table_name is None: table_name = op.input("W")[0] if table_name != op.input("W")[0]: - raise RuntimeError("all distributed lookup_table_ops" - " should have only one table") + raise RuntimeError( + "all distributed lookup_table_ops" + " should have only one table" + ) else: if table_name is not None: assert op.input("W")[0] != table_name diff --git a/python/paddle/fluid/distributed/downpour.py b/python/paddle/fluid/distributed/downpour.py index 69d592d0d5f..c710b7337cc 100644 --- a/python/paddle/fluid/distributed/downpour.py +++ b/python/paddle/fluid/distributed/downpour.py @@ -16,8 +16,12 @@ from .node import DownpourWorker from ..backward import append_backward import ps_pb2 as pslib from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table -from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_inputs -from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_outputs +from paddle.fluid.distribute_lookup_table import ( + find_distributed_lookup_table_inputs, +) +from paddle.fluid.distribute_lookup_table import ( + find_distributed_lookup_table_outputs, +) from google.protobuf import text_format @@ -47,15 +51,21 @@ class DownpourSGD(object): self.window_ = window self.type = "downpour" self.data_norm_name = [ - ".batch_size", ".batch_square_sum", ".batch_sum", - ".batch_size@GRAD", ".batch_square_sum@GRAD", ".batch_sum@GRAD" + ".batch_size", + ".batch_square_sum", + ".batch_sum", + ".batch_size@GRAD", + ".batch_square_sum@GRAD", + ".batch_sum@GRAD", ] - def minimize(self, - losses, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, + losses, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ): """ DownpounSGD is a distributed optimizer so that user can call minimize to generate backward @@ -76,30 +86,44 @@ class DownpourSGD(object): raise ValueError('losses is a list, just lick [model.cost]') table_name = find_distributed_lookup_table(losses[0].block.program) prefetch_slots = find_distributed_lookup_table_inputs( - losses[0].block.program, table_name) + losses[0].block.program, table_name + ) prefetch_slots_emb = find_distributed_lookup_table_outputs( - losses[0].block.program, table_name) + losses[0].block.program, table_name + ) ps_param = pslib.PSParameter() server = DownpourServer() worker = DownpourWorker(self.window_) sparse_table_index = 0 - server.add_sparse_table(sparse_table_index, self.learning_rate_, - prefetch_slots, prefetch_slots_emb) - worker.add_sparse_table(sparse_table_index, self.learning_rate_, - prefetch_slots, prefetch_slots_emb) + server.add_sparse_table( + sparse_table_index, + self.learning_rate_, + prefetch_slots, + prefetch_slots_emb, + ) + worker.add_sparse_table( + sparse_table_index, + self.learning_rate_, + prefetch_slots, + prefetch_slots_emb, + ) dense_table_index = 1 program_configs = [] param_grads_list = [] for loss_index in range(len(losses)): program_config = ps_param.trainer_param.program_config.add() - program_config.program_id = str(id( - losses[loss_index].block.program)) + program_config.program_id = str( + id(losses[loss_index].block.program) + ) program_config.pull_sparse_table_id.extend([sparse_table_index]) program_config.push_sparse_table_id.extend([sparse_table_index]) - params_grads = sorted(append_backward(losses[loss_index], - parameter_list, no_grad_set), - key=lambda x: x[0].name) + params_grads = sorted( + append_backward( + losses[loss_index], parameter_list, no_grad_set + ), + key=lambda x: x[0].name, + ) param_grads_list.append(params_grads) params = [] grads = [] @@ -121,19 +145,28 @@ class DownpourSGD(object): data_norm_grads.append(i[1]) if not is_data_norm_data: grads.append(i[1]) - server.add_dense_table(dense_table_index, self.learning_rate_, - params, grads) - worker.add_dense_table(dense_table_index, self.learning_rate_, - params, grads) + server.add_dense_table( + dense_table_index, self.learning_rate_, params, grads + ) + worker.add_dense_table( + dense_table_index, self.learning_rate_, params, grads + ) program_config.pull_dense_table_id.extend([dense_table_index]) program_config.push_dense_table_id.extend([dense_table_index]) if len(data_norm_params) != 0 and len(data_norm_grads) != 0: dense_table_index += 1 - server.add_data_norm_table(dense_table_index, - self.learning_rate_, - data_norm_params, data_norm_grads) - worker.add_dense_table(dense_table_index, self.learning_rate_, - data_norm_params, data_norm_grads) + server.add_data_norm_table( + dense_table_index, + self.learning_rate_, + data_norm_params, + data_norm_grads, + ) + worker.add_dense_table( + dense_table_index, + self.learning_rate_, + data_norm_params, + data_norm_grads, + ) program_config.pull_dense_table_id.extend([dense_table_index]) program_config.push_dense_table_id.extend([dense_table_index]) dense_table_index += 1 diff --git a/python/paddle/fluid/distributed/fleet.py b/python/paddle/fluid/distributed/fleet.py index 9874492a413..08b46ce0473 100644 --- a/python/paddle/fluid/distributed/fleet.py +++ b/python/paddle/fluid/distributed/fleet.py @@ -19,9 +19,7 @@ __all__ = ['Fleet'] class Fleet(object): - """ - - """ + """ """ def __init__(self): self.instance_ = ps_instance.PaddlePSInstance() @@ -38,11 +36,13 @@ class Fleet(object): def init_pserver(self, opt_info): if "fleet_desc" in opt_info: self.dist_desc_str_ = text_format.MessageToString( - opt_info["fleet_desc"]) + opt_info["fleet_desc"] + ) self.dist_desc_ = opt_info["fleet_desc"] else: print( - "You should run distributed optimization to get opt_info first") + "You should run distributed optimization to get opt_info first" + ) sys.exit(-1) self.fleet_.init_server(self.dist_desc_str_) ip = self.fleet_.start_server() @@ -55,17 +55,22 @@ class Fleet(object): def init_worker(self, opt_info): if "fleet_desc" in opt_info: self.dist_desc_str_ = text_format.MessageToString( - opt_info["fleet_desc"]) + opt_info["fleet_desc"] + ) self.dist_desc_ = opt_info["fleet_desc"] else: print( - "You should run distributed optimization to get opt_info first") + "You should run distributed optimization to get opt_info first" + ) sys.exit(-1) self.instance_.barrier_all() ips = self.instance.gather_ips() - self.fleet_.init_worker(self.dist_desc_str_, ips, - self.instance_.get_node_cnt(), - self.instance._rankid) + self.fleet_.init_worker( + self.dist_desc_str_, + ips, + self.instance_.get_node_cnt(), + self.instance._rankid, + ) self.instance.barrier_worker() def init_pserver_model(self): diff --git a/python/paddle/fluid/distributed/helper.py b/python/paddle/fluid/distributed/helper.py index 08e6fca6165..9511ce2db62 100644 --- a/python/paddle/fluid/distributed/helper.py +++ b/python/paddle/fluid/distributed/helper.py @@ -26,23 +26,26 @@ class FileSystem(object): fs = FileSystm() """ - def __init__(self, - fs_type="afs", - uri="afs://xx", - user=None, - passwd=None, - hadoop_bin=""): + def __init__( + self, + fs_type="afs", + uri="afs://xx", + user=None, + passwd=None, + hadoop_bin="", + ): assert user != None assert passwd != None assert hadoop_bin != None import ps_pb2 as pslib + self.fs_client = pslib.FsClientParameter() self.fs_client.uri = uri self.fs_client.user = user self.fs_client.passwd = passwd - #self.fs_client.buffer_size = 0 + # self.fs_client.buffer_size = 0 self.fs_client.hadoop_bin = hadoop_bin - #self.fs_client.afs_conf = afs_conf if not afs_conf else "" + # self.fs_client.afs_conf = afs_conf if not afs_conf else "" def get_desc(self): """ @@ -63,6 +66,7 @@ class MPIHelper(object): def __init__(self): from mpi4py import MPI + self.comm = MPI.COMM_WORLD self.MPI = MPI @@ -74,11 +78,13 @@ class MPIHelper(object): def get_ip(self): import socket + local_ip = socket.gethostbyname(socket.gethostname()) return local_ip def get_hostname(self): import socket + return socket.gethostname() def finalize(self): diff --git a/python/paddle/fluid/distributed/node.py b/python/paddle/fluid/distributed/node.py index 90ac44ada14..793787d0fd7 100644 --- a/python/paddle/fluid/distributed/node.py +++ b/python/paddle/fluid/distributed/node.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and import ps_pb2 as pslib + # NOTE: reduce removed in fuctools in python3 from functools import reduce class Server(object): """ - A Server basic class. + A Server basic class. """ def __init__(self): @@ -27,7 +28,7 @@ class Server(object): class Worker(object): """ - A Worker basic class. + A Worker basic class. """ def __init__(self): @@ -36,23 +37,30 @@ class Worker(object): class DownpourServer(Server): """ - DownpourServer class is used to generate server program_desc - Args: - server: it is pslib.ServerParameter() - Examples: - server = DownpourServer() + DownpourServer class is used to generate server program_desc + Args: + server: it is pslib.ServerParameter() + Examples: + server = DownpourServer() """ def __init__(self): self.server_ = pslib.ServerParameter() self.server_.downpour_server_param.service_param.start_server_port = 0 - self.server_.downpour_server_param.service_param.server_class = "DownpourBrpcPsServer" - self.server_.downpour_server_param.service_param.client_class = "DownpourBrpcPsClient" - self.server_.downpour_server_param.service_param.service_class = "DownpourPsService" + self.server_.downpour_server_param.service_param.server_class = ( + "DownpourBrpcPsServer" + ) + self.server_.downpour_server_param.service_param.client_class = ( + "DownpourBrpcPsClient" + ) + self.server_.downpour_server_param.service_param.service_class = ( + "DownpourPsService" + ) self.server_.downpour_server_param.service_param.server_thread_num = 12 - def add_sparse_table(self, table_id, learning_rate, slot_key_vars, - slot_value_var): + def add_sparse_table( + self, table_id, learning_rate, slot_key_vars, slot_value_var + ): r""" Args: table_id(int): id of sparse params table @@ -108,8 +116,9 @@ class DownpourServer(Server): table.accessor.dense_sgd_param.adam.mom_decay_rate = 0.99 table.accessor.dense_sgd_param.naive.learning_rate = 0.0002 fea_dim = 0 - for param in filter(lambda x: x.name.find("embedding") == -1, - param_var): + for param in filter( + lambda x: x.name.find("embedding") == -1, param_var + ): fea_dim += reduce(lambda x, y: x * y, param.shape, 1) table.accessor.fea_dim = fea_dim @@ -122,20 +131,21 @@ class DownpourServer(Server): class DownpourWorker(Worker): """ - DownpourWorker class is used to generate worker program_desc - Args: - window (int): push params frequency - worker: it is pslib.DownpourTrainerParameter - Examples: - worker = DownpourWorker(1) + DownpourWorker class is used to generate worker program_desc + Args: + window (int): push params frequency + worker: it is pslib.DownpourTrainerParameter + Examples: + worker = DownpourWorker(1) """ def __init__(self, window): self.window = window self.worker_ = pslib.DownpourTrainerParameter() - def add_sparse_table(self, table_id, learning_rate, slot_key_vars, - slot_value_vars): + def add_sparse_table( + self, table_id, learning_rate, slot_key_vars, slot_value_vars + ): r""" Args: table_id(int): id of sparse params table @@ -151,7 +161,8 @@ class DownpourWorker(Worker): table.slot_key.extend([var.name for var in slot_key_vars]) table.slot_value.extend([var.name for var in slot_value_vars]) table.slot_gradient.extend( - [var.name + "@GRAD" for var in slot_value_vars]) + [var.name + "@GRAD" for var in slot_value_vars] + ) def add_dense_table(self, table_id, learning_rate, param_vars, grad_vars): r""" @@ -167,11 +178,16 @@ class DownpourWorker(Worker): table = self.worker_.dense_table.add() table.table_id = table_id table.dense_variable_name.extend( - filter(lambda x: x.find("embedding") == -1, - [p.name for p in param_vars])) + filter( + lambda x: x.find("embedding") == -1, + [p.name for p in param_vars], + ) + ) table.dense_gradient_variable_name.extend( - filter(lambda x: x.find("embedding") == -1, - [g.name for g in grad_vars])) + filter( + lambda x: x.find("embedding") == -1, [g.name for g in grad_vars] + ) + ) def get_desc(self): """ diff --git a/python/paddle/fluid/distributed/ps_instance.py b/python/paddle/fluid/distributed/ps_instance.py index 6b19d7ca62e..370e1b19647 100644 --- a/python/paddle/fluid/distributed/ps_instance.py +++ b/python/paddle/fluid/distributed/ps_instance.py @@ -16,12 +16,12 @@ from .helper import MPIHelper class PaddlePSInstance(object): """ - PaddlePSInstance class is used to generate A instance of server or worker - Args: - server_worker_mode: is a value 0 or 1, default is 1 - proc_per_node: process per node, default is 2 - Examples: - instance = PaddlePSInstance(1, 2) + PaddlePSInstance class is used to generate A instance of server or worker + Args: + server_worker_mode: is a value 0 or 1, default is 1 + proc_per_node: process per node, default is 2 + Examples: + instance = PaddlePSInstance(1, 2) """ def __init__(self, server_worker_mode=1, proc_per_node=2): @@ -35,7 +35,7 @@ class PaddlePSInstance(object): self._worker_num = self._nodes * self._proc_per_node / 2 self._server_num = self._nodes * self._proc_per_node / 2 self._total_server_worker = self._worker_num + self._server_num - self._node_type = None #IDLE=-1, WORKER=1, SERVER=0 + self._node_type = None # IDLE=-1, WORKER=1, SERVER=0 self._set_nodetype() self._comm = None self._split_comm() @@ -104,7 +104,7 @@ class PaddlePSInstance(object): def set_ip(self, ip): """ - set server ip + set server ip """ self._ip = ip diff --git a/python/paddle/fluid/distributed/ps_pb2.py b/python/paddle/fluid/distributed/ps_pb2.py index f1262ebae12..8a131a2548c 100644 --- a/python/paddle/fluid/distributed/ps_pb2.py +++ b/python/paddle/fluid/distributed/ps_pb2.py @@ -24,6 +24,7 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -34,7 +35,8 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto2', serialized_pb=_b( '\n\x08ps.proto\x12\x06paddle\"\x9e\x02\n\x0bPSParameter\x12\x14\n\x0cworker_class\x18\x01 \x01(\t\x12\x14\n\x0cserver_class\x18\x02 \x01(\t\x12\x16\n\x0einstance_class\x18\x03 \x01(\t\x12-\n\x0cworker_param\x18\x65 \x01(\x0b\x32\x17.paddle.WorkerParameter\x12-\n\x0cserver_param\x18\x66 \x01(\x0b\x32\x17.paddle.ServerParameter\x12\x38\n\rtrainer_param\x18\xad\x02 \x01(\x0b\x32 .paddle.DownpourTrainerParameter\x12\x33\n\x0f\x66s_client_param\x18\xf5\x03 \x01(\x0b\x32\x19.paddle.FsClientParameter\"Q\n\x0fWorkerParameter\x12>\n\x15\x64ownpour_worker_param\x18\x01 \x01(\x0b\x32\x1f.paddle.DownpourWorkerParameter\"Q\n\x0fServerParameter\x12>\n\x15\x64ownpour_server_param\x18\x01 \x01(\x0b\x32\x1f.paddle.DownpourServerParameter\"O\n\x17\x44ownpourWorkerParameter\x12\x34\n\x14\x64ownpour_table_param\x18\x01 \x03(\x0b\x32\x16.paddle.TableParameter\"\xfd\x01\n\x18\x44ownpourTrainerParameter\x12\x30\n\x0b\x64\x65nse_table\x18\x01 \x03(\x0b\x32\x1b.paddle.DenseTableParameter\x12\x32\n\x0csparse_table\x18\x02 \x03(\x0b\x32\x1c.paddle.SparseTableParameter\x12\x1d\n\x15push_sparse_per_batch\x18\x03 \x01(\x05\x12\x1c\n\x14push_dense_per_batch\x18\x04 \x01(\x05\x12\x0f\n\x07skip_op\x18\x05 \x03(\t\x12-\n\x0eprogram_config\x18\x06 \x03(\x0b\x32\x15.paddle.ProgramConfig\"\x99\x01\n\rProgramConfig\x12\x12\n\nprogram_id\x18\x01 \x02(\t\x12\x1c\n\x14push_sparse_table_id\x18\x02 \x03(\x05\x12\x1b\n\x13push_dense_table_id\x18\x03 \x03(\x05\x12\x1c\n\x14pull_sparse_table_id\x18\x04 \x03(\x05\x12\x1b\n\x13pull_dense_table_id\x18\x05 \x03(\x05\"{\n\x13\x44\x65nseTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x05\x12\x1b\n\x13\x64\x65nse_variable_name\x18\x02 \x03(\t\x12$\n\x1c\x64\x65nse_gradient_variable_name\x18\x03 \x03(\t\x12\x0f\n\x07\x66\x65\x61_dim\x18\x04 \x01(\x05\"z\n\x14SparseTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x05\x12\x13\n\x0b\x66\x65\x61ture_dim\x18\x02 \x01(\x05\x12\x10\n\x08slot_key\x18\x03 \x03(\t\x12\x12\n\nslot_value\x18\x04 \x03(\t\x12\x15\n\rslot_gradient\x18\x05 \x03(\t\"\x86\x01\n\x17\x44ownpourServerParameter\x12\x34\n\x14\x64ownpour_table_param\x18\x01 \x03(\x0b\x32\x16.paddle.TableParameter\x12\x35\n\rservice_param\x18\x02 \x01(\x0b\x32\x1e.paddle.ServerServiceParameter\"\xd7\x01\n\x16ServerServiceParameter\x12*\n\x0cserver_class\x18\x01 \x01(\t:\x14\x44ownpourBrpcPsServer\x12*\n\x0c\x63lient_class\x18\x02 \x01(\t:\x14\x44ownpourBrpcPsClient\x12(\n\rservice_class\x18\x03 \x01(\t:\x11\x44ownpourPsService\x12\x1c\n\x11start_server_port\x18\x04 \x01(\r:\x01\x30\x12\x1d\n\x11server_thread_num\x18\x05 \x01(\r:\x02\x31\x32\"\xbf\x01\n\x0eTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x04\x12\x13\n\x0btable_class\x18\x02 \x01(\t\x12\x12\n\nshared_num\x18\x03 \x01(\x04\x12\x30\n\x08\x61\x63\x63\x65ssor\x18\x04 \x01(\x0b\x32\x1e.paddle.TableAccessorParameter\x12\x1f\n\x04type\x18\x05 \x01(\x0e\x32\x11.paddle.TableType\x12\x1f\n\x10\x63ompress_in_save\x18\x06 \x01(\x08:\x05\x66\x61lse\"\xf1\x02\n\x16TableAccessorParameter\x12\x16\n\x0e\x61\x63\x63\x65ssor_class\x18\x01 \x01(\t\x12\x38\n\x10sparse_sgd_param\x18\x02 \x01(\x0b\x32\x1e.paddle.SparseSGDRuleParameter\x12\x36\n\x0f\x64\x65nse_sgd_param\x18\x03 \x01(\x0b\x32\x1d.paddle.DenseSGDRuleParameter\x12\x0f\n\x07\x66\x65\x61_dim\x18\x04 \x01(\r\x12\x12\n\nembedx_dim\x18\x05 \x01(\r\x12\x18\n\x10\x65mbedx_threshold\x18\x06 \x01(\r\x12G\n\x17\x64ownpour_accessor_param\x18\x07 \x01(\x0b\x32&.paddle.DownpourTableAccessorParameter\x12\x45\n\x19table_accessor_save_param\x18\x08 \x03(\x0b\x32\".paddle.TableAccessorSaveParameter\"\xce\x01\n\x1e\x44ownpourTableAccessorParameter\x12\x14\n\x0cnonclk_coeff\x18\x01 \x01(\x02\x12\x13\n\x0b\x63lick_coeff\x18\x02 \x01(\x02\x12\x16\n\x0e\x62\x61se_threshold\x18\x03 \x01(\x02\x12\x17\n\x0f\x64\x65lta_threshold\x18\x04 \x01(\x02\x12\x17\n\x0f\x64\x65lta_keep_days\x18\x05 \x01(\x02\x12\x1d\n\x15show_click_decay_rate\x18\x06 \x01(\x02\x12\x18\n\x10\x64\x65lete_threshold\x18\x07 \x01(\x02\"S\n\x1aTableAccessorSaveParameter\x12\r\n\x05param\x18\x01 \x01(\r\x12\x11\n\tconverter\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65\x63onverter\x18\x03 \x01(\t\"e\n\x10PsRequestMessage\x12\x0e\n\x06\x63md_id\x18\x01 \x02(\r\x12\x10\n\x08table_id\x18\x02 \x01(\r\x12\x0e\n\x06params\x18\x03 \x03(\x0c\x12\x11\n\tclient_id\x18\x04 \x01(\x05\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\"w\n\x16SparseSGDRuleParameter\x12\x15\n\rlearning_rate\x18\x01 \x01(\x01\x12\x15\n\rinitial_g2sum\x18\x02 \x01(\x01\x12\x18\n\rinitial_range\x18\x03 \x01(\x01:\x01\x30\x12\x15\n\rweight_bounds\x18\x04 \x03(\x02\"\xe1\x01\n\x15\x44\x65nseSGDRuleParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x04\x61\x64\x61m\x18\x02 \x01(\x0b\x32\x18.paddle.AdamSGDParameter\x12(\n\x05naive\x18\x03 \x01(\x0b\x32\x19.paddle.NaiveSGDParameter\x12,\n\x07summary\x18\x04 \x01(\x0b\x32\x1b.paddle.SummarySGDParameter\x12:\n\x0emoving_average\x18\x05 \x01(\x0b\x32\".paddle.MovingAverageRuleParameter\"\x86\x01\n\x10\x41\x64\x61mSGDParameter\x12\x15\n\rlearning_rate\x18\x01 \x01(\x01\x12\x16\n\x0e\x61vg_decay_rate\x18\x02 \x01(\x01\x12\x16\n\x0e\x61\x64\x61_decay_rate\x18\x03 \x01(\x01\x12\x13\n\x0b\x61\x64\x61_epsilon\x18\x04 \x01(\x01\x12\x16\n\x0emom_decay_rate\x18\x05 \x01(\x01\"B\n\x11NaiveSGDParameter\x12\x15\n\rlearning_rate\x18\x01 \x01(\x01\x12\x16\n\x0e\x61vg_decay_rate\x18\x02 \x01(\x01\";\n\x13SummarySGDParameter\x12$\n\x12summary_decay_rate\x18\x01 \x01(\x01:\x08\x30.999999\".\n\x1aMovingAverageRuleParameter\x12\x10\n\x08momentum\x18\x01 \x01(\x01\"I\n\x11PsResponseMessage\x12\x13\n\x08\x65rr_code\x18\x01 \x02(\x05:\x01\x30\x12\x11\n\x07\x65rr_msg\x18\x02 \x02(\t:\x00\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xd5\x01\n\x11\x46sClientParameter\x12:\n\x07\x66s_type\x18\x01 \x01(\x0e\x32#.paddle.FsClientParameter.FsApiType:\x04HDFS\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x0e\n\x06passwd\x18\x04 \x01(\t\x12\x13\n\x0b\x62uffer_size\x18\x05 \x01(\x05\x12\x12\n\nhadoop_bin\x18\x33 \x01(\t\x12\x10\n\x08\x61\x66s_conf\x18\x65 \x01(\t\"\x1e\n\tFsApiType\x12\x08\n\x04HDFS\x10\x00\x12\x07\n\x03\x41\x46S\x10\x01*4\n\tTableType\x12\x13\n\x0fPS_SPARSE_TABLE\x10\x00\x12\x12\n\x0ePS_DENSE_TABLE\x10\x01*\xbd\x02\n\x07PsCmdID\x12\x17\n\x13PS_PULL_DENSE_TABLE\x10\x00\x12\x17\n\x13PS_PUSH_DENSE_TABLE\x10\x01\x12\x18\n\x14PS_PULL_SPARSE_TABLE\x10\x02\x12\x18\n\x14PS_PUSH_SPARSE_TABLE\x10\x03\x12\x13\n\x0fPS_SHRINK_TABLE\x10\x04\x12\x15\n\x11PS_SAVE_ONE_TABLE\x10\x05\x12\x15\n\x11PS_SAVE_ALL_TABLE\x10\x06\x12\x15\n\x11PS_LOAD_ONE_TABLE\x10\x07\x12\x15\n\x11PS_LOAD_ALL_TABLE\x10\x08\x12\x16\n\x12PS_CLEAR_ONE_TABLE\x10\t\x12\x16\n\x12PS_CLEAR_ALL_TABLE\x10\n\x12\x17\n\x13PS_PUSH_DENSE_PARAM\x10\x0b\x12\x12\n\x0ePS_STOP_SERVER\x10\x0c\x32K\n\tPsService\x12>\n\x07service\x12\x18.paddle.PsRequestMessage\x1a\x19.paddle.PsResponseMessageB\x03\x80\x01\x01' - )) + ), +) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _TABLETYPE = _descriptor.EnumDescriptor( @@ -43,16 +45,12 @@ _TABLETYPE = _descriptor.EnumDescriptor( filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='PS_SPARSE_TABLE', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_DENSE_TABLE', - index=1, - number=1, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='PS_SPARSE_TABLE', index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_DENSE_TABLE', index=1, number=1, options=None, type=None + ), ], containing_type=None, options=None, @@ -68,71 +66,73 @@ _PSCMDID = _descriptor.EnumDescriptor( filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='PS_PULL_DENSE_TABLE', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_DENSE_TABLE', - index=1, - number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PULL_SPARSE_TABLE', - index=2, - number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_SPARSE_TABLE', - index=3, - number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SHRINK_TABLE', - index=4, - number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ONE_TABLE', - index=5, - number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ALL_TABLE', - index=6, - number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_LOAD_ONE_TABLE', - index=7, - number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_LOAD_ALL_TABLE', - index=8, - number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CLEAR_ONE_TABLE', - index=9, - number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CLEAR_ALL_TABLE', - index=10, - number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_DENSE_PARAM', - index=11, - number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_STOP_SERVER', - index=12, - number=12, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='PS_PULL_DENSE_TABLE', + index=0, + number=0, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_DENSE_TABLE', + index=1, + number=1, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PULL_SPARSE_TABLE', + index=2, + number=2, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_SPARSE_TABLE', + index=3, + number=3, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_SHRINK_TABLE', index=4, number=4, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ONE_TABLE', index=5, number=5, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ALL_TABLE', index=6, number=6, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_LOAD_ONE_TABLE', index=7, number=7, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_LOAD_ALL_TABLE', index=8, number=8, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_CLEAR_ONE_TABLE', + index=9, + number=9, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_CLEAR_ALL_TABLE', + index=10, + number=10, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_DENSE_PARAM', + index=11, + number=11, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_STOP_SERVER', index=12, number=12, options=None, type=None + ), ], containing_type=None, options=None, @@ -164,16 +164,12 @@ _FSCLIENTPARAMETER_FSAPITYPE = _descriptor.EnumDescriptor( filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='HDFS', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='AFS', - index=1, - number=1, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='HDFS', index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='AFS', index=1, number=1, options=None, type=None + ), ], containing_type=None, options=None, @@ -189,36 +185,40 @@ _PSPARAMETER = _descriptor.Descriptor( file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='worker_class', - full_name='paddle.PSParameter.worker_class', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='server_class', - full_name='paddle.PSParameter.server_class', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='worker_class', + full_name='paddle.PSParameter.worker_class', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='server_class', + full_name='paddle.PSParameter.server_class', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='instance_class', full_name='paddle.PSParameter.instance_class', @@ -234,37 +234,42 @@ _PSPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='worker_param', - full_name='paddle.PSParameter.worker_param', - index=3, - number=101, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='server_param', - full_name='paddle.PSParameter.server_param', - index=4, - number=102, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='worker_param', + full_name='paddle.PSParameter.worker_param', + index=3, + number=101, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='server_param', + full_name='paddle.PSParameter.server_param', + index=4, + number=102, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='trainer_param', full_name='paddle.PSParameter.trainer_param', @@ -280,7 +285,8 @@ _PSPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fs_client_param', full_name='paddle.PSParameter.fs_client_param', @@ -296,7 +302,8 @@ _PSPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -332,7 +339,8 @@ _WORKERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -368,7 +376,8 @@ _SERVERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -404,7 +413,8 @@ _DOWNPOURWORKERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -440,7 +450,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_table', full_name='paddle.DownpourTrainerParameter.sparse_table', @@ -456,7 +467,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_sparse_per_batch', full_name='paddle.DownpourTrainerParameter.push_sparse_per_batch', @@ -472,7 +484,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_dense_per_batch', full_name='paddle.DownpourTrainerParameter.push_dense_per_batch', @@ -488,7 +501,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='skip_op', full_name='paddle.DownpourTrainerParameter.skip_op', @@ -504,7 +518,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='program_config', full_name='paddle.DownpourTrainerParameter.program_config', @@ -520,7 +535,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -541,21 +557,23 @@ _PROGRAMCONFIG = _descriptor.Descriptor( file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='program_id', - full_name='paddle.ProgramConfig.program_id', - index=0, - number=1, - type=9, - cpp_type=9, - label=2, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='program_id', + full_name='paddle.ProgramConfig.program_id', + index=0, + number=1, + type=9, + cpp_type=9, + label=2, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='push_sparse_table_id', full_name='paddle.ProgramConfig.push_sparse_table_id', @@ -571,7 +589,8 @@ _PROGRAMCONFIG = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_dense_table_id', full_name='paddle.ProgramConfig.push_dense_table_id', @@ -587,7 +606,8 @@ _PROGRAMCONFIG = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='pull_sparse_table_id', full_name='paddle.ProgramConfig.pull_sparse_table_id', @@ -603,7 +623,8 @@ _PROGRAMCONFIG = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='pull_dense_table_id', full_name='paddle.ProgramConfig.pull_dense_table_id', @@ -619,7 +640,8 @@ _PROGRAMCONFIG = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -655,7 +677,8 @@ _DENSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_variable_name', full_name='paddle.DenseTableParameter.dense_variable_name', @@ -671,7 +694,8 @@ _DENSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_gradient_variable_name', full_name='paddle.DenseTableParameter.dense_gradient_variable_name', @@ -687,7 +711,8 @@ _DENSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fea_dim', full_name='paddle.DenseTableParameter.fea_dim', @@ -703,7 +728,8 @@ _DENSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -739,7 +765,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='feature_dim', full_name='paddle.SparseTableParameter.feature_dim', @@ -755,7 +782,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_key', full_name='paddle.SparseTableParameter.slot_key', @@ -771,7 +799,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_value', full_name='paddle.SparseTableParameter.slot_value', @@ -787,7 +816,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_gradient', full_name='paddle.SparseTableParameter.slot_gradient', @@ -803,7 +833,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -839,7 +870,8 @@ _DOWNPOURSERVERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='service_param', full_name='paddle.DownpourServerParameter.service_param', @@ -855,7 +887,8 @@ _DOWNPOURSERVERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -891,7 +924,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='client_class', full_name='paddle.ServerServiceParameter.client_class', @@ -907,7 +941,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='service_class', full_name='paddle.ServerServiceParameter.service_class', @@ -923,7 +958,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='start_server_port', full_name='paddle.ServerServiceParameter.start_server_port', @@ -939,7 +975,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='server_thread_num', full_name='paddle.ServerServiceParameter.server_thread_num', @@ -955,7 +992,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -976,21 +1014,23 @@ _TABLEPARAMETER = _descriptor.Descriptor( file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='table_id', - full_name='paddle.TableParameter.table_id', - index=0, - number=1, - type=4, - cpp_type=4, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='table_id', + full_name='paddle.TableParameter.table_id', + index=0, + number=1, + type=4, + cpp_type=4, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='table_class', full_name='paddle.TableParameter.table_class', @@ -1006,7 +1046,8 @@ _TABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='shared_num', full_name='paddle.TableParameter.shared_num', @@ -1022,37 +1063,42 @@ _TABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='accessor', - full_name='paddle.TableParameter.accessor', - index=3, - number=4, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='type', - full_name='paddle.TableParameter.type', - index=4, - number=5, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='accessor', + full_name='paddle.TableParameter.accessor', + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='type', + full_name='paddle.TableParameter.type', + index=4, + number=5, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='compress_in_save', full_name='paddle.TableParameter.compress_in_save', @@ -1068,7 +1114,8 @@ _TABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1104,7 +1151,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_sgd_param', full_name='paddle.TableAccessorParameter.sparse_sgd_param', @@ -1120,7 +1168,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_sgd_param', full_name='paddle.TableAccessorParameter.dense_sgd_param', @@ -1136,7 +1185,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fea_dim', full_name='paddle.TableAccessorParameter.fea_dim', @@ -1152,7 +1202,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_dim', full_name='paddle.TableAccessorParameter.embedx_dim', @@ -1168,7 +1219,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_threshold', full_name='paddle.TableAccessorParameter.embedx_threshold', @@ -1184,7 +1236,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='downpour_accessor_param', full_name='paddle.TableAccessorParameter.downpour_accessor_param', @@ -1200,7 +1253,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='table_accessor_save_param', full_name='paddle.TableAccessorParameter.table_accessor_save_param', @@ -1216,7 +1270,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1252,7 +1307,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='click_coeff', full_name='paddle.DownpourTableAccessorParameter.click_coeff', @@ -1268,7 +1324,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='base_threshold', full_name='paddle.DownpourTableAccessorParameter.base_threshold', @@ -1284,7 +1341,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delta_threshold', full_name='paddle.DownpourTableAccessorParameter.delta_threshold', @@ -1300,7 +1358,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delta_keep_days', full_name='paddle.DownpourTableAccessorParameter.delta_keep_days', @@ -1316,11 +1375,11 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='show_click_decay_rate', - full_name= - 'paddle.DownpourTableAccessorParameter.show_click_decay_rate', + full_name='paddle.DownpourTableAccessorParameter.show_click_decay_rate', index=5, number=6, type=2, @@ -1333,7 +1392,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delete_threshold', full_name='paddle.DownpourTableAccessorParameter.delete_threshold', @@ -1349,7 +1409,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1385,7 +1446,8 @@ _TABLEACCESSORSAVEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='converter', full_name='paddle.TableAccessorSaveParameter.converter', @@ -1401,7 +1463,8 @@ _TABLEACCESSORSAVEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='deconverter', full_name='paddle.TableAccessorSaveParameter.deconverter', @@ -1417,7 +1480,8 @@ _TABLEACCESSORSAVEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1438,21 +1502,23 @@ _PSREQUESTMESSAGE = _descriptor.Descriptor( file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='cmd_id', - full_name='paddle.PsRequestMessage.cmd_id', - index=0, - number=1, - type=13, - cpp_type=3, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='cmd_id', + full_name='paddle.PsRequestMessage.cmd_id', + index=0, + number=1, + type=13, + cpp_type=3, + label=2, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='table_id', full_name='paddle.PsRequestMessage.table_id', @@ -1468,22 +1534,25 @@ _PSREQUESTMESSAGE = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='params', - full_name='paddle.PsRequestMessage.params', - index=2, - number=3, - type=12, - cpp_type=9, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='params', + full_name='paddle.PsRequestMessage.params', + index=2, + number=3, + type=12, + cpp_type=9, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='client_id', full_name='paddle.PsRequestMessage.client_id', @@ -1499,22 +1568,25 @@ _PSREQUESTMESSAGE = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='data', - full_name='paddle.PsRequestMessage.data', - index=4, - number=5, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='data', + full_name='paddle.PsRequestMessage.data', + index=4, + number=5, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -1550,7 +1622,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_g2sum', full_name='paddle.SparseSGDRuleParameter.initial_g2sum', @@ -1566,7 +1639,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseSGDRuleParameter.initial_range', @@ -1582,7 +1656,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseSGDRuleParameter.weight_bounds', @@ -1598,7 +1673,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1634,7 +1710,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='adam', full_name='paddle.DenseSGDRuleParameter.adam', @@ -1650,7 +1727,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='naive', full_name='paddle.DenseSGDRuleParameter.naive', @@ -1666,7 +1744,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='summary', full_name='paddle.DenseSGDRuleParameter.summary', @@ -1682,7 +1761,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='moving_average', full_name='paddle.DenseSGDRuleParameter.moving_average', @@ -1698,7 +1778,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1734,7 +1815,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='avg_decay_rate', full_name='paddle.AdamSGDParameter.avg_decay_rate', @@ -1750,7 +1832,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_decay_rate', full_name='paddle.AdamSGDParameter.ada_decay_rate', @@ -1766,7 +1849,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_epsilon', full_name='paddle.AdamSGDParameter.ada_epsilon', @@ -1782,7 +1866,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='mom_decay_rate', full_name='paddle.AdamSGDParameter.mom_decay_rate', @@ -1798,7 +1883,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1834,7 +1920,8 @@ _NAIVESGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='avg_decay_rate', full_name='paddle.NaiveSGDParameter.avg_decay_rate', @@ -1850,7 +1937,8 @@ _NAIVESGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1886,7 +1974,8 @@ _SUMMARYSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1922,7 +2011,8 @@ _MOVINGAVERAGERULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1958,7 +2048,8 @@ _PSRESPONSEMESSAGE = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='err_msg', full_name='paddle.PsResponseMessage.err_msg', @@ -1974,22 +2065,25 @@ _PSRESPONSEMESSAGE = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='data', - full_name='paddle.PsResponseMessage.data', - index=2, - number=3, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='data', + full_name='paddle.PsResponseMessage.data', + index=2, + number=3, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -2025,52 +2119,59 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='uri', - full_name='paddle.FsClientParameter.uri', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='user', - full_name='paddle.FsClientParameter.user', - index=2, - number=3, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='passwd', - full_name='paddle.FsClientParameter.passwd', - index=3, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='uri', + full_name='paddle.FsClientParameter.uri', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='user', + full_name='paddle.FsClientParameter.user', + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='passwd', + full_name='paddle.FsClientParameter.passwd', + index=3, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='buffer_size', full_name='paddle.FsClientParameter.buffer_size', @@ -2086,7 +2187,8 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='hadoop_bin', full_name='paddle.FsClientParameter.hadoop_bin', @@ -2102,7 +2204,8 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='afs_conf', full_name='paddle.FsClientParameter.afs_conf', @@ -2118,7 +2221,8 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2137,75 +2241,102 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( _PSPARAMETER.fields_by_name['worker_param'].message_type = _WORKERPARAMETER _PSPARAMETER.fields_by_name['server_param'].message_type = _SERVERPARAMETER _PSPARAMETER.fields_by_name[ - 'trainer_param'].message_type = _DOWNPOURTRAINERPARAMETER + 'trainer_param' +].message_type = _DOWNPOURTRAINERPARAMETER _PSPARAMETER.fields_by_name['fs_client_param'].message_type = _FSCLIENTPARAMETER _WORKERPARAMETER.fields_by_name[ - 'downpour_worker_param'].message_type = _DOWNPOURWORKERPARAMETER + 'downpour_worker_param' +].message_type = _DOWNPOURWORKERPARAMETER _SERVERPARAMETER.fields_by_name[ - 'downpour_server_param'].message_type = _DOWNPOURSERVERPARAMETER + 'downpour_server_param' +].message_type = _DOWNPOURSERVERPARAMETER _DOWNPOURWORKERPARAMETER.fields_by_name[ - 'downpour_table_param'].message_type = _TABLEPARAMETER + 'downpour_table_param' +].message_type = _TABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'dense_table'].message_type = _DENSETABLEPARAMETER + 'dense_table' +].message_type = _DENSETABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'sparse_table'].message_type = _SPARSETABLEPARAMETER + 'sparse_table' +].message_type = _SPARSETABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'program_config'].message_type = _PROGRAMCONFIG + 'program_config' +].message_type = _PROGRAMCONFIG _DOWNPOURSERVERPARAMETER.fields_by_name[ - 'downpour_table_param'].message_type = _TABLEPARAMETER + 'downpour_table_param' +].message_type = _TABLEPARAMETER _DOWNPOURSERVERPARAMETER.fields_by_name[ - 'service_param'].message_type = _SERVERSERVICEPARAMETER + 'service_param' +].message_type = _SERVERSERVICEPARAMETER _TABLEPARAMETER.fields_by_name[ - 'accessor'].message_type = _TABLEACCESSORPARAMETER + 'accessor' +].message_type = _TABLEACCESSORPARAMETER _TABLEPARAMETER.fields_by_name['type'].enum_type = _TABLETYPE _TABLEACCESSORPARAMETER.fields_by_name[ - 'sparse_sgd_param'].message_type = _SPARSESGDRULEPARAMETER + 'sparse_sgd_param' +].message_type = _SPARSESGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'dense_sgd_param'].message_type = _DENSESGDRULEPARAMETER + 'dense_sgd_param' +].message_type = _DENSESGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'downpour_accessor_param'].message_type = _DOWNPOURTABLEACCESSORPARAMETER + 'downpour_accessor_param' +].message_type = _DOWNPOURTABLEACCESSORPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'table_accessor_save_param'].message_type = _TABLEACCESSORSAVEPARAMETER + 'table_accessor_save_param' +].message_type = _TABLEACCESSORSAVEPARAMETER _DENSESGDRULEPARAMETER.fields_by_name['adam'].message_type = _ADAMSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name['naive'].message_type = _NAIVESGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name[ - 'summary'].message_type = _SUMMARYSGDPARAMETER + 'summary' +].message_type = _SUMMARYSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name[ - 'moving_average'].message_type = _MOVINGAVERAGERULEPARAMETER + 'moving_average' +].message_type = _MOVINGAVERAGERULEPARAMETER _FSCLIENTPARAMETER.fields_by_name[ - 'fs_type'].enum_type = _FSCLIENTPARAMETER_FSAPITYPE + 'fs_type' +].enum_type = _FSCLIENTPARAMETER_FSAPITYPE _FSCLIENTPARAMETER_FSAPITYPE.containing_type = _FSCLIENTPARAMETER DESCRIPTOR.message_types_by_name['PSParameter'] = _PSPARAMETER DESCRIPTOR.message_types_by_name['WorkerParameter'] = _WORKERPARAMETER DESCRIPTOR.message_types_by_name['ServerParameter'] = _SERVERPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourWorkerParameter'] = _DOWNPOURWORKERPARAMETER + 'DownpourWorkerParameter' +] = _DOWNPOURWORKERPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourTrainerParameter'] = _DOWNPOURTRAINERPARAMETER + 'DownpourTrainerParameter' +] = _DOWNPOURTRAINERPARAMETER DESCRIPTOR.message_types_by_name['ProgramConfig'] = _PROGRAMCONFIG DESCRIPTOR.message_types_by_name['DenseTableParameter'] = _DENSETABLEPARAMETER DESCRIPTOR.message_types_by_name['SparseTableParameter'] = _SPARSETABLEPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourServerParameter'] = _DOWNPOURSERVERPARAMETER + 'DownpourServerParameter' +] = _DOWNPOURSERVERPARAMETER DESCRIPTOR.message_types_by_name[ - 'ServerServiceParameter'] = _SERVERSERVICEPARAMETER + 'ServerServiceParameter' +] = _SERVERSERVICEPARAMETER DESCRIPTOR.message_types_by_name['TableParameter'] = _TABLEPARAMETER DESCRIPTOR.message_types_by_name[ - 'TableAccessorParameter'] = _TABLEACCESSORPARAMETER + 'TableAccessorParameter' +] = _TABLEACCESSORPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourTableAccessorParameter'] = _DOWNPOURTABLEACCESSORPARAMETER + 'DownpourTableAccessorParameter' +] = _DOWNPOURTABLEACCESSORPARAMETER DESCRIPTOR.message_types_by_name[ - 'TableAccessorSaveParameter'] = _TABLEACCESSORSAVEPARAMETER + 'TableAccessorSaveParameter' +] = _TABLEACCESSORSAVEPARAMETER DESCRIPTOR.message_types_by_name['PsRequestMessage'] = _PSREQUESTMESSAGE DESCRIPTOR.message_types_by_name[ - 'SparseSGDRuleParameter'] = _SPARSESGDRULEPARAMETER + 'SparseSGDRuleParameter' +] = _SPARSESGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'DenseSGDRuleParameter'] = _DENSESGDRULEPARAMETER + 'DenseSGDRuleParameter' +] = _DENSESGDRULEPARAMETER DESCRIPTOR.message_types_by_name['AdamSGDParameter'] = _ADAMSGDPARAMETER DESCRIPTOR.message_types_by_name['NaiveSGDParameter'] = _NAIVESGDPARAMETER DESCRIPTOR.message_types_by_name['SummarySGDParameter'] = _SUMMARYSGDPARAMETER DESCRIPTOR.message_types_by_name[ - 'MovingAverageRuleParameter'] = _MOVINGAVERAGERULEPARAMETER + 'MovingAverageRuleParameter' +] = _MOVINGAVERAGERULEPARAMETER DESCRIPTOR.message_types_by_name['PsResponseMessage'] = _PSRESPONSEMESSAGE DESCRIPTOR.message_types_by_name['FsClientParameter'] = _FSCLIENTPARAMETER DESCRIPTOR.enum_types_by_name['TableType'] = _TABLETYPE @@ -2213,215 +2344,259 @@ DESCRIPTOR.enum_types_by_name['PsCmdID'] = _PSCMDID PSParameter = _reflection.GeneratedProtocolMessageType( 'PSParameter', - (_message.Message, ), - dict(DESCRIPTOR=_PSPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PSParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PSParameter) + ), +) _sym_db.RegisterMessage(PSParameter) WorkerParameter = _reflection.GeneratedProtocolMessageType( 'WorkerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_WORKERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.WorkerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_WORKERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.WorkerParameter) + ), +) _sym_db.RegisterMessage(WorkerParameter) ServerParameter = _reflection.GeneratedProtocolMessageType( 'ServerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SERVERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ServerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SERVERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ServerParameter) + ), +) _sym_db.RegisterMessage(ServerParameter) DownpourWorkerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourWorkerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURWORKERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourWorkerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURWORKERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourWorkerParameter) + ), +) _sym_db.RegisterMessage(DownpourWorkerParameter) DownpourTrainerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourTrainerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURTRAINERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourTrainerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURTRAINERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourTrainerParameter) + ), +) _sym_db.RegisterMessage(DownpourTrainerParameter) ProgramConfig = _reflection.GeneratedProtocolMessageType( 'ProgramConfig', - (_message.Message, ), - dict(DESCRIPTOR=_PROGRAMCONFIG, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ProgramConfig) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PROGRAMCONFIG, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ProgramConfig) + ), +) _sym_db.RegisterMessage(ProgramConfig) DenseTableParameter = _reflection.GeneratedProtocolMessageType( 'DenseTableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DENSETABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DenseTableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DENSETABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DenseTableParameter) + ), +) _sym_db.RegisterMessage(DenseTableParameter) SparseTableParameter = _reflection.GeneratedProtocolMessageType( 'SparseTableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSETABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseTableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSETABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseTableParameter) + ), +) _sym_db.RegisterMessage(SparseTableParameter) DownpourServerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourServerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURSERVERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourServerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURSERVERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourServerParameter) + ), +) _sym_db.RegisterMessage(DownpourServerParameter) ServerServiceParameter = _reflection.GeneratedProtocolMessageType( 'ServerServiceParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SERVERSERVICEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ServerServiceParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SERVERSERVICEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ServerServiceParameter) + ), +) _sym_db.RegisterMessage(ServerServiceParameter) TableParameter = _reflection.GeneratedProtocolMessageType( 'TableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_TABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.TableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_TABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.TableParameter) + ), +) _sym_db.RegisterMessage(TableParameter) TableAccessorParameter = _reflection.GeneratedProtocolMessageType( 'TableAccessorParameter', - (_message.Message, ), - dict(DESCRIPTOR=_TABLEACCESSORPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.TableAccessorParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_TABLEACCESSORPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.TableAccessorParameter) + ), +) _sym_db.RegisterMessage(TableAccessorParameter) DownpourTableAccessorParameter = _reflection.GeneratedProtocolMessageType( 'DownpourTableAccessorParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_DOWNPOURTABLEACCESSORPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.DownpourTableAccessorParameter) - )) + ), +) _sym_db.RegisterMessage(DownpourTableAccessorParameter) TableAccessorSaveParameter = _reflection.GeneratedProtocolMessageType( 'TableAccessorSaveParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_TABLEACCESSORSAVEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.TableAccessorSaveParameter) - )) + ), +) _sym_db.RegisterMessage(TableAccessorSaveParameter) PsRequestMessage = _reflection.GeneratedProtocolMessageType( 'PsRequestMessage', - (_message.Message, ), - dict(DESCRIPTOR=_PSREQUESTMESSAGE, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PsRequestMessage) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSREQUESTMESSAGE, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PsRequestMessage) + ), +) _sym_db.RegisterMessage(PsRequestMessage) SparseSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseSGDRuleParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSESGDRULEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseSGDRuleParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSESGDRULEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseSGDRuleParameter) + ), +) _sym_db.RegisterMessage(SparseSGDRuleParameter) DenseSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'DenseSGDRuleParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DENSESGDRULEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DenseSGDRuleParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DENSESGDRULEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DenseSGDRuleParameter) + ), +) _sym_db.RegisterMessage(DenseSGDRuleParameter) AdamSGDParameter = _reflection.GeneratedProtocolMessageType( 'AdamSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_ADAMSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.AdamSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_ADAMSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.AdamSGDParameter) + ), +) _sym_db.RegisterMessage(AdamSGDParameter) NaiveSGDParameter = _reflection.GeneratedProtocolMessageType( 'NaiveSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_NAIVESGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.NaiveSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_NAIVESGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.NaiveSGDParameter) + ), +) _sym_db.RegisterMessage(NaiveSGDParameter) SummarySGDParameter = _reflection.GeneratedProtocolMessageType( 'SummarySGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SUMMARYSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SummarySGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SUMMARYSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SummarySGDParameter) + ), +) _sym_db.RegisterMessage(SummarySGDParameter) MovingAverageRuleParameter = _reflection.GeneratedProtocolMessageType( 'MovingAverageRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_MOVINGAVERAGERULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.MovingAverageRuleParameter) - )) + ), +) _sym_db.RegisterMessage(MovingAverageRuleParameter) PsResponseMessage = _reflection.GeneratedProtocolMessageType( 'PsResponseMessage', - (_message.Message, ), - dict(DESCRIPTOR=_PSRESPONSEMESSAGE, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PsResponseMessage) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSRESPONSEMESSAGE, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PsResponseMessage) + ), +) _sym_db.RegisterMessage(PsResponseMessage) FsClientParameter = _reflection.GeneratedProtocolMessageType( 'FsClientParameter', - (_message.Message, ), - dict(DESCRIPTOR=_FSCLIENTPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.FsClientParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_FSCLIENTPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.FsClientParameter) + ), +) _sym_db.RegisterMessage(FsClientParameter) DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), - _b('\200\001\001')) +DESCRIPTOR._options = _descriptor._ParseOptions( + descriptor_pb2.FileOptions(), _b('\200\001\001') +) # @@protoc_insertion_point(module_scope) diff --git a/python/paddle/fluid/dygraph/amp/auto_cast.py b/python/paddle/fluid/dygraph/amp/auto_cast.py index 682f14fab5b..bb1acc7c09b 100644 --- a/python/paddle/fluid/dygraph/amp/auto_cast.py +++ b/python/paddle/fluid/dygraph/amp/auto_cast.py @@ -12,10 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.fluid.wrapped_decorator import signature_safe_contextmanager, wrap_decorator +from paddle.fluid.wrapped_decorator import ( + signature_safe_contextmanager, + wrap_decorator, +) from paddle.fluid import core import contextlib -from paddle.fluid.framework import Variable, _non_static_mode, OpProtoHolder, Parameter, _dygraph_tracer, dygraph_only, set_flags, get_flags +from paddle.fluid.framework import ( + Variable, + _non_static_mode, + OpProtoHolder, + Parameter, + _dygraph_tracer, + dygraph_only, + set_flags, + get_flags, +) import warnings import copy import functools @@ -103,12 +115,11 @@ def amp_state(): return _g_amp_state_ -#NOTE(zhiqiu): similar as paddle.fluid.contrib.mixed_precision.fp16_lists.AutoMixedPrecisionLists._update_list +# NOTE(zhiqiu): similar as paddle.fluid.contrib.mixed_precision.fp16_lists.AutoMixedPrecisionLists._update_list # The reason why not use AutoMixedPrecisionLists is that custom_black_varnames is not suitable for imperative mode. -def _update_list(custom_white_list, - custom_black_list, - level='O1', - dtype='float16'): +def _update_list( + custom_white_list, custom_black_list, level='O1', dtype='float16' +): """ Update black and white list according to users' custom list. """ @@ -129,8 +140,9 @@ def _update_list(custom_white_list, if custom_white_list and custom_black_list: for op_name in custom_white_list: if op_name in custom_black_list: - raise ValueError("Custom white list overlap " - "custom black list") + raise ValueError( + "Custom white list overlap " "custom black list" + ) if custom_white_list: for op_name in custom_white_list: if op_name in _black_list: @@ -190,17 +202,29 @@ def pure_fp16_initialize(models): for layer in models[idx].sublayers(include_self=True): layer._casted_by_pure_fp16 = True if (layer._dtype == 'float16') or isinstance( - layer, (paddle.nn.BatchNorm, paddle.nn.BatchNorm1D, - paddle.nn.BatchNorm2D, paddle.nn.BatchNorm3D, - paddle.nn.LayerNorm, paddle.nn.SyncBatchNorm)): + layer, + ( + paddle.nn.BatchNorm, + paddle.nn.BatchNorm1D, + paddle.nn.BatchNorm2D, + paddle.nn.BatchNorm3D, + paddle.nn.LayerNorm, + paddle.nn.SyncBatchNorm, + ), + ): continue - if isinstance(layer, (paddle.incubate.nn.FusedFeedForward, - paddle.incubate.nn.FusedMultiHeadAttention)): + if isinstance( + layer, + ( + paddle.incubate.nn.FusedFeedForward, + paddle.incubate.nn.FusedMultiHeadAttention, + ), + ): layer._amp_decorate(dtype='float16') continue - layer._to_impl(dtype='float16', - include_sublayers=False, - floating_only=True) + layer._to_impl( + dtype='float16', include_sublayers=False, floating_only=True + ) return models @@ -208,9 +232,9 @@ def pure_fp16_initialize(models): def pure_bf16_initialize(models): for idx in range(len(models)): for layer in models[idx].sublayers(include_self=True): - layer._to_impl(dtype='bfloat16', - include_sublayers=False, - floating_only=True) + layer._to_impl( + dtype='bfloat16', include_sublayers=False, floating_only=True + ) return models @@ -218,8 +242,10 @@ def check_models(models): for model in models: if not isinstance(model, paddle.nn.Layer): raise RuntimeError( - "Current train mode is pure fp16, models should be paddle.nn.Layer, but receive {}." - .format(type(model))) + "Current train mode is pure fp16, models should be paddle.nn.Layer, but receive {}.".format( + type(model) + ) + ) if isinstance(model, paddle.DataParallel): raise RuntimeError( "For distributed AMP training, you should first use paddle.amp.decorate() to decotate origin model, and then call paddle.DataParallel get distributed model." @@ -229,20 +255,25 @@ def check_models(models): def check_optimizers(optimizers): for optimizer in optimizers: if not isinstance( - optimizer, - (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer)): + optimizer, + (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer), + ): raise RuntimeError( - "Current train mode is pure fp16, optimizers should be paddle.optimizer.Optimizer or paddle.fluid.optimizer.Optimizer, but receive {}." - .format(type(optimizer))) + "Current train mode is pure fp16, optimizers should be paddle.optimizer.Optimizer or paddle.fluid.optimizer.Optimizer, but receive {}.".format( + type(optimizer) + ) + ) @signature_safe_contextmanager @dygraph_only -def amp_guard(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O1', - dtype='float16'): +def amp_guard( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O1', + dtype='float16', +): """ :api_attr: imperative @@ -306,19 +337,23 @@ def amp_guard(enable=True, tracer = _dygraph_tracer() if not tracer: raise ValueError( - "current_tracer is None, maybe it is not in imperative mode.") + "current_tracer is None, maybe it is not in imperative mode." + ) # check device_type: # NOTE: Now, amp only support gpu for float16 and bfloat16, xpu for float16, mlu for float16, npu for float16. # Maybe we will support cpu for bfloat16. - if enable and not (tracer._expected_place.is_gpu_place() - or tracer._expected_place.is_xpu_place() - or tracer._expected_place.is_mlu_place() - or tracer._expected_place.is_npu_place() - or tracer._expected_place.is_custom_place()): + if enable and not ( + tracer._expected_place.is_gpu_place() + or tracer._expected_place.is_xpu_place() + or tracer._expected_place.is_mlu_place() + or tracer._expected_place.is_npu_place() + or tracer._expected_place.is_custom_place() + ): warnings.warn( 'amp_guard can only be enabled on CUDAPlace, XPUPlace, MLUPlace, NPUPlace, and CustomPlace, current place is %s, so it makes no effect.' - % tracer._expected_place) + % tracer._expected_place + ) enable = False # For npu: if tracer._expected_place.is_npu_place() and (dtype == 'bfloat16'): @@ -343,14 +378,20 @@ def amp_guard(enable=True, prop = paddle.device.cuda.get_device_capability() warnings.warn( "For float16, amp only support NVIDIA GPU with Compute Capability 7.0 or higher, current GPU is: %s, with Compute Capability: %d.%d." - % (paddle.device.cuda.get_device_name(), prop[0], prop[1])) + % (paddle.device.cuda.get_device_name(), prop[0], prop[1]) + ) elif (dtype == 'bfloat16') and not _is_gpu_bfloat16_supported(): prop = paddle.device.cuda.get_device_capability() cuda_version = paddle.version.cuda() warnings.warn( "For bfloat16, amp only support NVIDIA GPU with Compute Capability 8.0 or higher and CUDA Version 11.0 or higher, current GPU is: %s, with Compute Capability: %d.%d, current CUDA Version is: %s." - % (paddle.device.cuda.get_device_name(), prop[0], prop[1], - cuda_version)) + % ( + paddle.device.cuda.get_device_name(), + prop[0], + prop[1], + cuda_version, + ) + ) amp_dtype = dtype @@ -381,8 +422,9 @@ def amp_guard(enable=True, _black_list = BF16_BLACK_LIST if custom_white_list or custom_black_list: - _white_list, _black_list = _update_list(custom_white_list, - custom_black_list, level, dtype) + _white_list, _black_list = _update_list( + custom_white_list, custom_black_list, level, dtype + ) if not enable: amp_level = AMP_LEVEL.O0 @@ -422,7 +464,6 @@ def amp_guard(enable=True, class StateDictHook(object): - def __init__(self, save_dtype): self._save_dtype = save_dtype @@ -437,12 +478,14 @@ class StateDictHook(object): @dygraph_only -def amp_decorate(models, - optimizers=None, - level='O1', - dtype='float16', - master_weight=None, - save_dtype=None): +def amp_decorate( + models, + optimizers=None, + level='O1', + dtype='float16', + master_weight=None, + save_dtype=None, +): """ Decorate models and optimizers for auto-mixed-precision. When level is O1(amp), the decorate will do nothing. When level is O2(pure fp16), the decorate will cast all parameters of models to FP16, except BatchNorm and LayerNorm. @@ -527,7 +570,8 @@ def amp_decorate(models, models_is_list = True else: raise TypeError( - "models must be either a single model or a list of models.") + "models must be either a single model or a list of models." + ) if dtype == 'float16': models = pure_fp16_initialize(models=models) elif dtype == 'bfloat16': @@ -539,8 +583,9 @@ def amp_decorate(models, # check optimizers optimizers_is_list = False if isinstance( - optimizers, - (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer)): + optimizers, + (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer), + ): optimizers_is_list = False optimizers = [optimizers] check_optimizers(optimizers) @@ -563,7 +608,8 @@ def amp_decorate(models, if not (save_dtype in ['float16', 'bfloat16', 'float32', 'float64']): raise ValueError( "save_dtype can only be float16 float32 or float64, but your input save_dtype is %s." - % save_dtype) + % save_dtype + ) for idx in range(len(models)): for layer in models[idx].sublayers(include_self=True): layer.register_state_dict_hook(StateDictHook(save_dtype)) diff --git a/python/paddle/fluid/dygraph/amp/loss_scaler.py b/python/paddle/fluid/dygraph/amp/loss_scaler.py index ce2e766d436..6ab153e4a2e 100644 --- a/python/paddle/fluid/dygraph/amp/loss_scaler.py +++ b/python/paddle/fluid/dygraph/amp/loss_scaler.py @@ -14,7 +14,11 @@ from paddle.fluid import core from paddle.fluid.dygraph import to_variable -from paddle.fluid.framework import _varbase_creator, _dygraph_tracer, dygraph_only +from paddle.fluid.framework import ( + _varbase_creator, + _dygraph_tracer, + dygraph_only, +) from paddle.fluid.data_feeder import check_type from ...wrapped_decorator import signature_safe_contextmanager, wrap_decorator import warnings @@ -89,28 +93,34 @@ class AmpScaler(object): """ @dygraph_only - def __init__(self, - enable=True, - init_loss_scaling=2.**15, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=1, - use_dynamic_loss_scaling=True): + def __init__( + self, + enable=True, + init_loss_scaling=2.0**15, + incr_ratio=2.0, + decr_ratio=0.5, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=1, + use_dynamic_loss_scaling=True, + ): tracer = _dygraph_tracer() if not tracer: raise ValueError( - "current_tracer is None, maybe it is not in imperative mode.") + "current_tracer is None, maybe it is not in imperative mode." + ) - if enable and not (tracer._expected_place.is_gpu_place() - or tracer._expected_place.is_xpu_place() - or tracer._expected_place.is_mlu_place() - or tracer._expected_place.is_npu_place() - or tracer._expected_place.is_custom_place()): + if enable and not ( + tracer._expected_place.is_gpu_place() + or tracer._expected_place.is_xpu_place() + or tracer._expected_place.is_mlu_place() + or tracer._expected_place.is_npu_place() + or tracer._expected_place.is_custom_place() + ): warnings.warn( 'AmpScaler can only be enabled on CUDAPlace, XPUPlace, MLUPlace, NPUPlace and CustomPlace, current place is %s, so it makes no effect.' - % tracer._expected_place) + % tracer._expected_place + ) enable = False self._enable = enable @@ -130,13 +140,17 @@ class AmpScaler(object): self._found_inf = to_variable(np.array([0]).astype(np.bool_)) self._temp_found_inf_fp16 = to_variable( - np.array([0]).astype(np.bool_)) + np.array([0]).astype(np.bool_) + ) self._temp_found_inf_bf16 = to_variable( - np.array([0]).astype(np.bool_)) + np.array([0]).astype(np.bool_) + ) self._temp_found_inf_fp32 = to_variable( - np.array([0]).astype(np.bool_)) + np.array([0]).astype(np.bool_) + ) self._scale = to_variable( - np.array([self._init_loss_scaling]).astype(np.float32)) + np.array([self._init_loss_scaling]).astype(np.float32) + ) self._cache_founf_inf = None self._optimizer_states = defaultdict(_refresh_optimizer_state) @@ -260,7 +274,8 @@ class AmpScaler(object): raise RuntimeError("unscale_() is being called after step().") if getattr(optimizer, '_param_groups', None) and isinstance( - optimizer._param_groups[0], dict): + optimizer._param_groups[0], dict + ): param_grads = [] param_grads_fp16 = [] param_grads_bf16 = [] @@ -269,29 +284,37 @@ class AmpScaler(object): for param in group['params']: if param._grad_ivar() is not None: param_grads.append(param._grad_ivar()) - if param._grad_ivar( - ).dtype == core.VarDesc.VarType.FP16: + if ( + param._grad_ivar().dtype + == core.VarDesc.VarType.FP16 + ): param_grads_fp16.append(param._grad_ivar()) - elif param._grad_ivar( - ).dtype == core.VarDesc.VarType.BF16: + elif ( + param._grad_ivar().dtype + == core.VarDesc.VarType.BF16 + ): param_grads_bf16.append(param._grad_ivar()) else: param_grads_fp32.append(param._grad_ivar()) else: param_grads = [ - param._grad_ivar() for param in optimizer._parameter_list + param._grad_ivar() + for param in optimizer._parameter_list if param._grad_ivar() is not None ] param_grads_fp16 = [ - param for param in param_grads + param + for param in param_grads if param.dtype == core.VarDesc.VarType.FP16 ] param_grads_bf16 = [ - param for param in param_grads + param + for param in param_grads if param.dtype == core.VarDesc.VarType.BF16 ] param_grads_fp32 = [ - param for param in param_grads + param + for param in param_grads if param.dtype == core.VarDesc.VarType.FP32 ] if core.is_compiled_with_npu(): @@ -300,31 +323,56 @@ class AmpScaler(object): if len(param_grads_fp16): _legacy_C_ops.check_finite_and_unscale( - param_grads_fp16, self._scale, float_status, - param_grads_fp16, self._temp_found_inf_fp16) + param_grads_fp16, + self._scale, + float_status, + param_grads_fp16, + self._temp_found_inf_fp16, + ) if len(param_grads_bf16): _legacy_C_ops.check_finite_and_unscale( - param_grads_bf16, self._scale, float_status, - param_grads_bf16, self._temp_found_inf_bf16) + param_grads_bf16, + self._scale, + float_status, + param_grads_bf16, + self._temp_found_inf_bf16, + ) if len(param_grads_fp32): _legacy_C_ops.check_finite_and_unscale( - param_grads_fp32, self._scale, float_status, - param_grads_fp32, self._temp_found_inf_fp32) + param_grads_fp32, + self._scale, + float_status, + param_grads_fp32, + self._temp_found_inf_fp32, + ) else: if len(param_grads_fp16): _legacy_C_ops.check_finite_and_unscale( - param_grads_fp16, self._scale, param_grads_fp16, - self._temp_found_inf_fp16) + param_grads_fp16, + self._scale, + param_grads_fp16, + self._temp_found_inf_fp16, + ) if len(param_grads_bf16): _legacy_C_ops.check_finite_and_unscale( - param_grads_bf16, self._scale, param_grads_bf16, - self._temp_found_inf_bf16) + param_grads_bf16, + self._scale, + param_grads_bf16, + self._temp_found_inf_bf16, + ) if len(param_grads_fp32): _legacy_C_ops.check_finite_and_unscale( - param_grads_fp32, self._scale, param_grads_fp32, - self._temp_found_inf_fp32) - - self._found_inf = self._temp_found_inf_fp16 or self._temp_found_inf_bf16 or self._temp_found_inf_fp32 + param_grads_fp32, + self._scale, + param_grads_fp32, + self._temp_found_inf_fp32, + ) + + self._found_inf = ( + self._temp_found_inf_fp16 + or self._temp_found_inf_bf16 + or self._temp_found_inf_fp32 + ) optimizer_state["state"] = OptimizerState.UNSCALED @@ -340,9 +388,12 @@ class AmpScaler(object): self._decr_count = self._decr_count + 1 if self._decr_count == self._decr_every_n_nan_or_inf: print( - 'Found inf or nan, current scale is: {}, decrease to: {}*{}' - .format(float(self._scale), float(self._scale), - float(self._decr_ratio))) + 'Found inf or nan, current scale is: {}, decrease to: {}*{}'.format( + float(self._scale), + float(self._scale), + float(self._decr_ratio), + ) + ) self._scale = self._scale * self._decr_ratio self._decr_count = 0 else: @@ -390,7 +441,8 @@ class AmpScaler(object): """ self._init_loss_scaling = new_init_loss_scaling self._scale = to_variable( - np.array([self._init_loss_scaling]).astype(np.float32)) + np.array([self._init_loss_scaling]).astype(np.float32) + ) def get_incr_ratio(self): """ @@ -481,16 +533,20 @@ class AmpScaler(object): decr_count(int): The number of recent consecutive skipped steps. use_dynamic_loss_scaling(bool): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamicly. Default is True. """ - return { - "scale": self._scale.numpy(), - "incr_ratio": self._incr_ratio, - "decr_ratio": self._decr_ratio, - "incr_every_n_steps": self._incr_every_n_steps, - "decr_every_n_nan_or_inf": self._decr_every_n_nan_or_inf, - "incr_count": self._incr_count, - "decr_count": self._decr_count, - "use_dynamic_loss_scaling": self._use_dynamic_loss_scaling - } if self._enable else {} + return ( + { + "scale": self._scale.numpy(), + "incr_ratio": self._incr_ratio, + "decr_ratio": self._decr_ratio, + "incr_every_n_steps": self._incr_every_n_steps, + "decr_every_n_nan_or_inf": self._decr_every_n_nan_or_inf, + "incr_count": self._incr_count, + "decr_count": self._decr_count, + "use_dynamic_loss_scaling": self._use_dynamic_loss_scaling, + } + if self._enable + else {} + ) def load_state_dict(self, state_dict): """ @@ -505,11 +561,13 @@ class AmpScaler(object): if len(state_dict) == 0: raise RuntimeError( "The input state dict is empty, possibly because it was saved " - "from a disabled instance of GradScaler.") + "from a disabled instance of GradScaler." + ) self._init_loss_scaling = state_dict["scale"][0] self._scale = to_variable( - np.array([self._init_loss_scaling]).astype(np.float32)) + np.array([self._init_loss_scaling]).astype(np.float32) + ) self._incr_ratio = state_dict["incr_ratio"] self._decr_ratio = state_dict["decr_ratio"] self._incr_every_n_steps = state_dict["incr_every_n_steps"] diff --git a/python/paddle/fluid/dygraph/base.py b/python/paddle/fluid/dygraph/base.py index 82ef8065837..593ce558f21 100644 --- a/python/paddle/fluid/dygraph/base.py +++ b/python/paddle/fluid/dygraph/base.py @@ -25,13 +25,23 @@ from .tracer import Tracer import logging from ..data_feeder import convert_dtype import warnings -from ..framework import _get_paddle_place, _in_legacy_dygraph, _in_eager_without_dygraph_check +from ..framework import ( + _get_paddle_place, + _in_legacy_dygraph, + _in_eager_without_dygraph_check, +) import paddle import warnings __all__ = [ - 'no_grad', 'no_grad_', 'grad', 'guard', 'enable_dygraph', 'disable_dygraph', - 'enabled', 'to_variable' + 'no_grad', + 'no_grad_', + 'grad', + 'guard', + 'enable_dygraph', + 'disable_dygraph', + 'enabled', + 'to_variable', ] # Flag that indicates whether running code under `@declarative` @@ -46,8 +56,9 @@ def in_declarative_mode(): return _in_declarative_mode_ -def declarative_unsupport_argument_warning(func_name, input_names, inputs, - support_values): +def declarative_unsupport_argument_warning( + func_name, input_names, inputs, support_values +): """ Warning if inputs do not elementwisely equals to support_values. It's a utility function for dy2static when dygraph interface have @@ -56,12 +67,13 @@ def declarative_unsupport_argument_warning(func_name, input_names, inputs, """ for name, inp, sup in zip(input_names, inputs, support_values): if inp != sup: - warnings.warn(f"{func_name} has unsupported parameter in jit: " + - f"{name}, jit will discard it") + warnings.warn( + f"{func_name} has unsupported parameter in jit: " + + f"{name}, jit will discard it" + ) def _switch_to_static_graph_(func): - def __impl__(*args, **kwargs): with framework._dygraph_guard(None): return func(*args, **kwargs) @@ -101,8 +113,11 @@ _functional_dygraph_context_manager = None @signature_safe_contextmanager def param_guard(parameters): # Note: parameters is a reference of self._parameters or self._buffers - if in_declarative_mode( - ) and not framework._non_static_mode() and parameters: + if ( + in_declarative_mode() + and not framework._non_static_mode() + and parameters + ): origin_parameters = parameters.copy() for name, var_base in parameters.items(): if isinstance(var_base, list): @@ -126,8 +141,9 @@ def _convert_into_variable(tensor): if new_var is not None: assert isinstance(new_var, framework.Variable) # Convert ParamBase into Parameter with same attributes in dy2stat. - elif isinstance(tensor, - (framework.EagerParamBase, framework.ParamBase)): + elif isinstance( + tensor, (framework.EagerParamBase, framework.ParamBase) + ): new_var = tensor._to_static_var(to_parameter=True) else: # Note(Aurelius84): Convert VarBase in self._buffers into Variable with @@ -140,8 +156,9 @@ def _convert_into_variable(tensor): # non-persistable. See case of `drop_state` in lstm api. is_persistable = len(tensor.shape) > 0 - new_var = tensor._to_static_var(to_parameter=False, - persistable=is_persistable) + new_var = tensor._to_static_var( + to_parameter=False, persistable=is_persistable + ) return new_var else: return tensor @@ -207,7 +224,8 @@ def enable_dygraph(place=None): global _functional_dygraph_context_manager if _functional_dygraph_context_manager is None: _functional_dygraph_context_manager = guard( - place=_get_paddle_place(place)) + place=_get_paddle_place(place) + ) _functional_dygraph_context_manager.__enter__() # call disable_dygraph when Python exit @@ -368,7 +386,6 @@ class no_grad_: """ def __call__(self, func): - @decorator.decorator def _decorate_function(func, *args, **kwargs): with self: @@ -448,14 +465,16 @@ def guard(place=None): @framework.non_static_only -def grad(outputs, - inputs, - grad_outputs=None, - retain_graph=None, - create_graph=False, - only_inputs=True, - allow_unused=False, - no_grad_vars=None): +def grad( + outputs, + inputs, + grad_outputs=None, + retain_graph=None, + create_graph=False, + only_inputs=True, + allow_unused=False, + no_grad_vars=None, +): ''' .. note:: **This API is ONLY available in imperative mode.** @@ -581,16 +600,18 @@ def grad(outputs, # dy1 = [3], dy2 = [4] grad_y1 = paddle.to_tensor(3.0) print(test_dygraph_grad([grad_y1, grad_value])) # [24.] - ''' + ''' if in_declarative_mode(): # In dy2static context, we call static interface `gradients` # to calculate grads. from paddle.static import gradients + declarative_unsupport_argument_warning( "paddle.grad", ["retain_graph", "create_grad", "only_inputs", "allow_unused"], [retain_graph, create_graph, only_inputs, allow_unused], - [None, False, True, False]) + [None, False, True, False], + ) return gradients(outputs, inputs, grad_outputs, no_grad_vars) def check_in_out(in_out_list, name): @@ -605,9 +626,8 @@ def grad(outputs, ), "Elements of {} must be Tensor".format(name) else: assert isinstance( - each_var, - core.VarBase), "Elements of {} must be Variable".format( - name) + each_var, core.VarBase + ), "Elements of {} must be Variable".format(name) return in_out_list else: if _in_eager_without_dygraph_check(): @@ -642,7 +662,8 @@ def grad(outputs, if len(grad_outputs) > 0: assert len(grad_outputs) == len( - outputs), "The length of grad_outputs must be equal to outputs" + outputs + ), "The length of grad_outputs must be equal to outputs" if no_grad_vars is None: no_grad_vars = [] @@ -655,12 +676,12 @@ def grad(outputs, for var in no_grad_vars: if _in_eager_without_dygraph_check(): assert isinstance( - var, - core.eager.Tensor), "no_grad_vars can only contains Tensor" + var, core.eager.Tensor + ), "no_grad_vars can only contains Tensor" else: assert isinstance( - var, - core.VarBase), "no_grad_vars can only contains Variable" + var, core.VarBase + ), "no_grad_vars can only contains Variable" else: if _in_eager_without_dygraph_check(): raise AssertionError( @@ -676,8 +697,9 @@ def grad(outputs, if retain_graph is None: retain_graph = create_graph - assert isinstance(retain_graph, - bool), "retain_graph must be None, True or False" + assert isinstance( + retain_graph, bool + ), "retain_graph must be None, True or False" assert isinstance(allow_unused, bool), "allow_unused must be True or False" @@ -685,17 +707,30 @@ def grad(outputs, assert only_inputs, "only_inputs=False is not supported yet" if _in_eager_without_dygraph_check(): - return core.eager.run_partial_grad(outputs, inputs, grad_outputs, - retain_graph, create_graph, - only_inputs, allow_unused, - no_grad_vars) + return core.eager.run_partial_grad( + outputs, + inputs, + grad_outputs, + retain_graph, + create_graph, + only_inputs, + allow_unused, + no_grad_vars, + ) else: place = core.Place() place.set_place(framework._current_expected_place()) - return core.dygraph_partial_grad(inputs, outputs, grad_outputs, - no_grad_vars, place, create_graph, - retain_graph, allow_unused, - only_inputs) + return core.dygraph_partial_grad( + inputs, + outputs, + grad_outputs, + no_grad_vars, + place, + create_graph, + retain_graph, + allow_unused, + only_inputs, + ) @framework.dygraph_only @@ -755,20 +790,30 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): y.shape # [3L, 2L] """ - support_type = (list, tuple, np.ndarray, core.eager.Tensor, core.VarBase, - framework.Variable, core.Tensor, core.LoDTensor) + support_type = ( + list, + tuple, + np.ndarray, + core.eager.Tensor, + core.VarBase, + framework.Variable, + core.Tensor, + core.LoDTensor, + ) if not isinstance(value, support_type): raise TypeError( "The type of 'value' in fluid.dygraph.to_variable must be %s, but received %s." - % (support_type, type(value))) + % (support_type, type(value)) + ) if isinstance(value, (core.eager.Tensor, core.VarBase, framework.Variable)): return value elif isinstance(value, (core.Tensor, core.LoDTensor)): return core.VarBase(value) else: - if isinstance(framework._current_expected_place(), - framework.core.CPUPlace): - #TODO(zhiqiu): we found two problems when enable zero_copy on CPUPlace. + if isinstance( + framework._current_expected_place(), framework.core.CPUPlace + ): + # TODO(zhiqiu): we found two problems when enable zero_copy on CPUPlace. # (1): eigen requires 16-bytes alignments, but the data of numpy array may not statisfy. # Details: https://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html # (2): when used in flask framework, it may result in hang. @@ -780,7 +825,9 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): ) zero_copy = False else: - assert not zero_copy, "zero_copy mode can only be used with CPUPlace" + assert ( + not zero_copy + ), "zero_copy mode can only be used with CPUPlace" if not isinstance(value, np.ndarray): value = np.array(value) @@ -791,13 +838,20 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): value = value.astype(dtype) if _in_eager_without_dygraph_check(): - return core.eager.Tensor(value, framework._current_expected_place(), - False, zero_copy, name if name else None, - True) + return core.eager.Tensor( + value, + framework._current_expected_place(), + False, + zero_copy, + name if name else None, + True, + ) else: - py_var = core.VarBase(value=value, - place=framework._current_expected_place(), - persistable=False, - zero_copy=zero_copy, - name=name if name else '') + py_var = core.VarBase( + value=value, + place=framework._current_expected_place(), + persistable=False, + zero_copy=zero_copy, + name=name if name else '', + ) return py_var diff --git a/python/paddle/fluid/dygraph/checkpoint.py b/python/paddle/fluid/dygraph/checkpoint.py index 8306e702598..f7a1db032f4 100644 --- a/python/paddle/fluid/dygraph/checkpoint.py +++ b/python/paddle/fluid/dygraph/checkpoint.py @@ -15,14 +15,27 @@ import os import collections import functools -from ..framework import Variable, default_main_program, _non_static_mode, dygraph_only, Parameter, ParamBase, _varbase_creator, _dygraph_tracer, EagerParamBase +from ..framework import ( + Variable, + default_main_program, + _non_static_mode, + dygraph_only, + Parameter, + ParamBase, + _varbase_creator, + _dygraph_tracer, + EagerParamBase, +) import pickle from . import learning_rate_scheduler import warnings from .. import core from .base import guard from paddle.fluid.dygraph.jit import _SaveLoadConfig -from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers +from paddle.fluid.dygraph.io import ( + _construct_program_holders, + _construct_params_and_buffers, +) __all__ = [ 'save_dygraph', @@ -38,7 +51,8 @@ def _parse_load_config(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.fluid.load_dygraph` is not supported." - % (key)) + % (key) + ) # construct inner config inner_config = _SaveLoadConfig() @@ -85,7 +99,9 @@ def save_dygraph(state_dict, model_path): ''' base_name = os.path.basename(model_path) - assert base_name != "", "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received filename is empty string." + assert ( + base_name != "" + ), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received filename is empty string." suffix = ".pdparams" assert len(state_dict) > 0, "state_dict is empty, no need to save" @@ -193,7 +209,10 @@ def load_dygraph(model_path, **configs): with open(params_file_path, 'rb') as f: para_dict = pickle.load(f, encoding='latin1') - if not config.keep_name_table and "StructuredToParameterName@@" in para_dict: + if ( + not config.keep_name_table + and "StructuredToParameterName@@" in para_dict + ): del para_dict["StructuredToParameterName@@"] if os.path.exists(opti_file_path): @@ -202,8 +221,9 @@ def load_dygraph(model_path, **configs): else: # check model path if not os.path.isdir(model_prefix): - raise ValueError("Model saved directory '%s' is not exists." % - model_prefix) + raise ValueError( + "Model saved directory '%s' is not exists." % model_prefix + ) # check whether model file exists if config.model_filename is None: @@ -222,8 +242,9 @@ def load_dygraph(model_path, **configs): # NOTE(chenweihang): `jit.save` doesn't save optimizer state # 1. load program desc & construct _ProgramHolder - programs = _construct_program_holders(model_path, - config.model_filename) + programs = _construct_program_holders( + model_path, config.model_filename + ) # 2. load layer parameters & buffers # NOTE: using fluid.dygraph.guard() here will cause import error in py2 @@ -232,7 +253,8 @@ def load_dygraph(model_path, **configs): model_prefix, programs, config.params_filename, - append_suffix=False) + append_suffix=False, + ) # 3. construct state_dict para_dict = dict() @@ -248,10 +270,15 @@ def load_dygraph(model_path, **configs): structured_para_dict = dict() for var_name in para_dict: structured_name = extra_var_info[var_name].get( - 'structured_name', None) - assert structured_name is not None, "Cannot find saved variable (%s)'s structured name in saved model." % var_name + 'structured_name', None + ) + assert structured_name is not None, ( + "Cannot find saved variable (%s)'s structured name in saved model." + % var_name + ) structured_para_dict[structured_name] = para_dict[ - var_name] + var_name + ] para_dict = structured_para_dict else: # load state dict by `io.save_params/persistables` save format @@ -282,7 +309,8 @@ def load_dygraph(model_path, **configs): type='load', inputs={}, outputs={'Out': new_var}, - attrs={'file_path': os.path.join(model_path, name)}) + attrs={'file_path': os.path.join(model_path, name)}, + ) load_var_list.append(new_var) # 3. construct state_dict diff --git a/python/paddle/fluid/dygraph/container.py b/python/paddle/fluid/dygraph/container.py index 854df393557..db4ad356102 100644 --- a/python/paddle/fluid/dygraph/container.py +++ b/python/paddle/fluid/dygraph/container.py @@ -222,8 +222,10 @@ class LayerList(Layer): if isinstance(idx, int): if not (-len(self) <= idx < len(self)): raise IndexError( - 'index {} is out of range, should be an integer in range [{}, {})' - .format(idx, -len(self), len(self))) + 'index {} is out of range, should be an integer in range [{}, {})'.format( + idx, -len(self), len(self) + ) + ) if idx < 0: idx += len(self) return idx @@ -248,7 +250,8 @@ class LayerList(Layer): delattr(self, str(idx)) str_indices = [str(i) for i in range(len(self._sub_layers))] self._sub_layers = OrderedDict( - list(zip(str_indices, self._sub_layers.values()))) + list(zip(str_indices, self._sub_layers.values())) + ) def __len__(self): return len(self._sub_layers) @@ -297,9 +300,11 @@ class LayerList(Layer): linears.insert(-1, another) print(linears[-2] is another) # True """ - assert isinstance(index, int) and \ - -len(self._sub_layers) <= index < len(self._sub_layers), \ - "index should be an integer in range [{}, {})".format(-len(self), len(self)) + assert isinstance(index, int) and -len(self._sub_layers) <= index < len( + self._sub_layers + ), "index should be an integer in range [{}, {})".format( + -len(self), len(self) + ) index = self._get_abs_idx(index) for i in range(len(self._sub_layers), index, -1): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py index a48258e30d5..ca6f1e652e9 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py @@ -14,9 +14,13 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class AssertTransformer(BaseTransformer): @@ -35,8 +39,15 @@ class AssertTransformer(BaseTransformer): self.visit(self.root) def visit_Assert(self, node): - convert_assert_node = gast.parse('_jst.Assert({test}, {msg})'.format( - test=ast_to_source_code(node.test), - msg=ast_to_source_code(node.msg) if node.msg else "")).body[0].value + convert_assert_node = ( + gast.parse( + '_jst.Assert({test}, {msg})'.format( + test=ast_to_source_code(node.test), + msg=ast_to_source_code(node.msg) if node.msg else "", + ) + ) + .body[0] + .value + ) return gast.Expr(value=convert_assert_node) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py index ff7a9a2a957..3ae91027f5a 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py @@ -19,24 +19,60 @@ import os from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.early_return_transformer import EarlyReturnTransformer -from paddle.fluid.dygraph.dygraph_to_static.assert_transformer import AssertTransformer -from paddle.fluid.dygraph.dygraph_to_static.basic_api_transformer import BasicApiTransformer -from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import BreakContinueTransformer -from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import BreakTransformOptimizer -from paddle.fluid.dygraph.dygraph_to_static.call_transformer import CallTransformer -from paddle.fluid.dygraph.dygraph_to_static.cast_transformer import CastTransformer -from paddle.fluid.dygraph.dygraph_to_static.typehint_transformer import TypeHintTransformer -from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import IfElseTransformer -from paddle.fluid.dygraph.dygraph_to_static.logical_transformer import LogicalTransformer -from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import LoopTransformer -from paddle.fluid.dygraph.dygraph_to_static.print_transformer import PrintTransformer -from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ReturnTransformer -from paddle.fluid.dygraph.dygraph_to_static.create_variable_transformer import CreateVariableTransformer -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.tensor_shape_transformer import TensorShapeTransformer -from paddle.fluid.dygraph.dygraph_to_static.decorator_transformer import DecoratorTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.early_return_transformer import ( + EarlyReturnTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.assert_transformer import ( + AssertTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.basic_api_transformer import ( + BasicApiTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import ( + BreakContinueTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import ( + BreakTransformOptimizer, +) +from paddle.fluid.dygraph.dygraph_to_static.call_transformer import ( + CallTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.cast_transformer import ( + CastTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.typehint_transformer import ( + TypeHintTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import ( + IfElseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.logical_transformer import ( + LogicalTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import ( + LoopTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.print_transformer import ( + PrintTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + ReturnTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.create_variable_transformer import ( + CreateVariableTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + StaticAnalysisVisitor, +) +from paddle.fluid.dygraph.dygraph_to_static.tensor_shape_transformer import ( + TensorShapeTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.decorator_transformer import ( + DecoratorTransformer, +) from paddle.fluid.dygraph.dygraph_to_static import logging_utils from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code @@ -51,8 +87,11 @@ def apply_optimization(transformers): And not all optimized transformations are applied by default. It's controlled by 'export FLAGS_optim_transformation=1' """ - flag = str( - os.environ.get('FLAGS_optim_transformation')) in ['1', 'True', 'true'] + flag = str(os.environ.get('FLAGS_optim_transformation')) in [ + '1', + 'True', + 'true', + ] if flag: transformers.insert(3, BreakTransformOptimizer) @@ -69,7 +108,8 @@ class DygraphToStaticAst(BaseTransformer): # save root for some analysis may need global AST self.root = root self.static_analysis_visitor = StaticAnalysisVisitor(root) - self.static_analysis_root = self.static_analysis_visitor.get_node_wrapper_root( + self.static_analysis_root = ( + self.static_analysis_visitor.get_node_wrapper_root() ) self.decorate_func_name = None self.transfer_from_node_type(self.static_analysis_root) @@ -77,12 +117,14 @@ class DygraphToStaticAst(BaseTransformer): def _apply(self, transformer, node_wrapper, log_level): transformer(node_wrapper).transform() - self.translator_logger.log_transformed_code(log_level, self.root, - transformer.__name__) + self.translator_logger.log_transformed_code( + log_level, self.root, transformer.__name__ + ) def transfer_from_node_type(self, node_wrapper): self.translator_logger.log( - 1, "Source code: \n{}".format(ast_to_source_code(self.root))) + 1, "Source code: \n{}".format(ast_to_source_code(self.root)) + ) # Generic transformation self.visit(node_wrapper.node) @@ -110,7 +152,8 @@ class DygraphToStaticAst(BaseTransformer): self._apply(transformer, node_wrapper, log_level=index + 1) self.translator_logger.log_transformed_code( - logging_utils.LOG_AllTransformer, self.root, "All Transformers") + logging_utils.LOG_AllTransformer, self.root, "All Transformers" + ) def visit_FunctionDef(self, node): if self.decorate_func_name is None: diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/base_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/base_transformer.py index b77c5b3c701..cb8f1562b3e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/base_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/base_transformer.py @@ -20,16 +20,23 @@ from paddle.fluid.dygraph.dygraph_to_static.utils import create_assign_node from paddle.fluid.dygraph.dygraph_to_static.utils import ORIGI_INFO from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_INDEX_PREFIX from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_TUPLE_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_TUPLE_INDEX_PREFIX +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_TUPLE_INDEX_PREFIX, +) from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_VAR_LEN_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_VAR_NAME_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_ZIP_TO_LIST_PREFIX +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_VAR_NAME_PREFIX, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_ZIP_TO_LIST_PREFIX, +) from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_TARGET_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_ITERATOR_PREFIX +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_ITERATOR_PREFIX, +) class BaseTransformer(gast.NodeTransformer): - def visit(self, node): if not isinstance(node, gast.AST): msg = ('Expected "gast.AST", but got "{}".').format(type(node)) @@ -41,7 +48,7 @@ class BaseTransformer(gast.NodeTransformer): iter_result = result if iter_result is not node and iter_result is not None: if not isinstance(iter_result, (list, tuple)): - iter_result = (iter_result, ) + iter_result = (iter_result,) if origin_info is not None: for n in iter_result: setattr(n, ORIGI_INFO, origin_info) @@ -50,10 +57,10 @@ class BaseTransformer(gast.NodeTransformer): class RenameTransformer(BaseTransformer): - def __init__(self, node): assert isinstance( - node, gast.AST), "RenameTransformer only accepts gast.AST as input" + node, gast.AST + ), "RenameTransformer only accepts gast.AST as input" self.root = node self.old_name = "" self.new_name = "" @@ -113,7 +120,8 @@ class NameNodeReplaceTransformer(BaseTransformer): names = node.names def replace(s): - if s == self.target_name: return self.replace_node.id + if s == self.target_name: + return self.replace_node.id return s node.names = list(map(replace, names)) @@ -121,7 +129,7 @@ class NameNodeReplaceTransformer(BaseTransformer): class ForLoopTuplePreTransformer(BaseTransformer): - """ pre-process of for loop. + """pre-process of for loop. >>> for A in B: >>> C @@ -152,24 +160,28 @@ class ForLoopTuplePreTransformer(BaseTransformer): assign_iterator_node = gast.parse( f"{tuple_iterator} = _jst.Indexable({ast_to_source_code(node.iter).strip()})" ).body[0] - node.target = gast.Name(id=tuple_target, - ctx=gast.Store(), - annotation=None, - type_comment=None) - node.iter = gast.Name(id=tuple_iterator, - ctx=gast.Load(), - annotation=None, - type_comment=None) + node.target = gast.Name( + id=tuple_target, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ) + node.iter = gast.Name( + id=tuple_iterator, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ) node.body[0:0] = self.tuple_to_stmts(origin_tuple_node, tuple_target) # return a list will insert a list of node replace the original for node. return [assign_iterator_node, node] def tuple_node_to_unpack_structure(self, node): - """ Create a sequence to represents the structure of nest. - For example: `a, (b,c), [d,e,f]` is represented by - `[1, [1,1], [1,1,1]]`. the `1` is just a notation. + """Create a sequence to represents the structure of nest. + For example: `a, (b,c), [d,e,f]` is represented by + `[1, [1,1], [1,1,1]]`. the `1` is just a notation. - Specially, `a` is represented by `1`. + Specially, `a` is represented by `1`. """ ret = [] if not isinstance(node, (gast.Tuple, gast.List)): @@ -181,7 +193,9 @@ class ForLoopTuplePreTransformer(BaseTransformer): def tuple_to_stmts(self, node, tuple_name): structure_str = str(self.tuple_node_to_unpack_structure(node)) node_str = ast_to_source_code(node).strip() - assign_node_str = f"{node_str} = _jst.Unpack({tuple_name}, {structure_str})" + assign_node_str = ( + f"{node_str} = _jst.Unpack({tuple_name}, {structure_str})" + ) assign_node = gast.parse(assign_node_str).body[0] return [assign_node] @@ -213,8 +227,9 @@ class ForNodeVisitor(object): # 2. gast.For node main parts self.target = for_node.target # NOTE: type may be Node or list[Node] - self.iter_args = for_node.iter if self.is_for_iter( - ) else for_node.iter.args + self.iter_args = ( + for_node.iter if self.is_for_iter() else for_node.iter.args + ) self.body = for_node.body # 3. key shared node or names @@ -236,7 +251,8 @@ class ForNodeVisitor(object): self.iter_var_len_name = unique_name.generate(FOR_ITER_VAR_LEN_PREFIX) # - created zip to list var : __for_loop_iter_zip_0 self.iter_zip_to_list_name = unique_name.generate( - FOR_ITER_ZIP_TO_LIST_PREFIX) + FOR_ITER_ZIP_TO_LIST_PREFIX + ) # - var.numpy()/var # - for x in var|var.numpy() @@ -262,17 +278,22 @@ class ForNodeVisitor(object): return None def is_for_range_iter(self): - return isinstance(self.node.iter, gast.Call) and isinstance( - self.node.iter.func, - gast.Name) and self.node.iter.func.id == "range" + return ( + isinstance(self.node.iter, gast.Call) + and isinstance(self.node.iter.func, gast.Name) + and self.node.iter.func.id == "range" + ) def is_for_iter(self): - if isinstance(self.node.iter, - (gast.Name, gast.Attribute, gast.List, gast.Tuple)): + if isinstance( + self.node.iter, (gast.Name, gast.Attribute, gast.List, gast.Tuple) + ): return True - elif isinstance(self.node.iter, gast.Call) and isinstance( - self.node.iter.func, - gast.Attribute) and self.node.iter.func.attr == 'numpy': + elif ( + isinstance(self.node.iter, gast.Call) + and isinstance(self.node.iter.func, gast.Attribute) + and self.node.iter.func.attr == 'numpy' + ): return True elif isinstance(self.node.iter, gast.Subscript): return True @@ -280,17 +301,23 @@ class ForNodeVisitor(object): return False def is_for_enumerate_iter(self): - return isinstance(self.node.iter, gast.Call) and isinstance( - self.node.iter.func, - gast.Name) and self.node.iter.func.id == "enumerate" + return ( + isinstance(self.node.iter, gast.Call) + and isinstance(self.node.iter.func, gast.Name) + and self.node.iter.func.id == "enumerate" + ) def _args_check(self): if self.is_for_range_iter(): self.args_length = len(self.iter_args) - assert self.args_length >= 1 and self.args_length <= 3, "range() function takes 1 to 3 arguments" + assert ( + self.args_length >= 1 and self.args_length <= 3 + ), "range() function takes 1 to 3 arguments" elif self.is_for_enumerate_iter(): self.args_length = len(self.iter_args) - assert self.args_length >= 1 and self.args_length <= 2, "enumerate() function takes 1 to 2 arguments" + assert ( + self.args_length >= 1 and self.args_length <= 2 + ), "enumerate() function takes 1 to 2 arguments" else: self.args_length = None @@ -324,8 +351,9 @@ class ForNodeVisitor(object): target_node, assign_node = self._build_assign_var_slice_node() body_stmts[0:0] = [assign_node] for body_node in body_stmts: - NameNodeReplaceTransformer(body_node, self.iter_var_name, - target_node) + NameNodeReplaceTransformer( + body_node, self.iter_var_name, target_node + ) body_stmts.append(self._build_index_increase_node(step_node)) return init_stmts, cond_stmt, body_stmts @@ -346,8 +374,9 @@ class ForNodeVisitor(object): target_node, assign_node = self._build_assign_var_slice_node() body_stmts[0:0] = [assign_node] for body_node in body_stmts: - NameNodeReplaceTransformer(body_node, self.iter_var_name, - target_node) + NameNodeReplaceTransformer( + body_node, self.iter_var_name, target_node + ) body_stmts.append(self._build_index_increase_node(step_node)) body_stmts.append(self._build_enum_increase_node()) @@ -360,7 +389,8 @@ class ForNodeVisitor(object): index_init_value_str = '0' else: index_init_value_str = ast_to_source_code( - self.iter_args[0]).strip() + self.iter_args[0] + ).strip() index_init_var_name = self.iter_var_name else: @@ -368,7 +398,8 @@ class ForNodeVisitor(object): index_init_var_name = self.iter_idx_name index_init_node_source_str = "{target} = {value}".format( - target=index_init_var_name, value=index_init_value_str) + target=index_init_var_name, value=index_init_value_str + ) index_init_node = gast.parse(index_init_node_source_str).body[0] @@ -376,16 +407,20 @@ class ForNodeVisitor(object): def _build_var_len_assign_node(self): # get the length of iterable variable - if isinstance(self.iter_node, gast.Call) and isinstance( - self.iter_node.func, - gast.Attribute) and self.iter_node.func.attr == 'numpy': + if ( + isinstance(self.iter_node, gast.Call) + and isinstance(self.iter_node.func, gast.Attribute) + and self.iter_node.func.attr == 'numpy' + ): iter_var_name = ast_to_source_code( - self.iter_node.func.value).strip() + self.iter_node.func.value + ).strip() else: iter_var_name = ast_to_source_code(self.iter_node).strip() convert_len_node_source_str = '{} = _jst.Len({})'.format( - self.iter_var_len_name, iter_var_name) + self.iter_var_len_name, iter_var_name + ) convert_len_node = gast.parse(convert_len_node_source_str).body[0] @@ -403,18 +438,22 @@ class ForNodeVisitor(object): """ new_nodes = [] if isinstance(self.iter_node, gast.Call) and isinstance( - self.iter_node.func, gast.Name): + self.iter_node.func, gast.Name + ): if self.iter_node.func.id == 'zip': iter_var_name = ast_to_source_code(self.iter_node).strip() zip_to_list_str = "{target} = list({value})".format( - target=self.iter_zip_to_list_name, value=iter_var_name) + target=self.iter_zip_to_list_name, value=iter_var_name + ) zip_to_list_node = gast.parse(zip_to_list_str).body[0] new_nodes.append(zip_to_list_node) - self.iter_node = gast.Name(id=self.iter_zip_to_list_name, - ctx=gast.Load(), - annotation=None, - type_comment=None) + self.iter_node = gast.Name( + id=self.iter_zip_to_list_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ) return new_nodes @@ -424,27 +463,35 @@ class ForNodeVisitor(object): else: init_value_str = '0' - enum_init_node_source_str = "{} = {}".format(self.enum_idx_name, - init_value_str) + enum_init_node_source_str = "{} = {}".format( + self.enum_idx_name, init_value_str + ) enum_init_node = gast.parse(enum_init_node_source_str).body[0] return enum_init_node def _build_compare_node(self): if self.is_for_range_iter(): - compare_node = self.iter_args[ - 0] if self.args_length == 1 else self.iter_args[1] + compare_node = ( + self.iter_args[0] + if self.args_length == 1 + else self.iter_args[1] + ) else: - compare_node = gast.Name(id=self.iter_var_len_name, - ctx=gast.Load(), - annotation=None, - type_comment=None) + compare_node = gast.Name( + id=self.iter_var_len_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ) return compare_node def _build_step_node(self): if self.is_for_range_iter(): - step_node = self.iter_args[ - 2] if self.args_length == 3 else gast.Constant(value=1, - kind=None) + step_node = ( + self.iter_args[2] + if self.args_length == 3 + else gast.Constant(value=1, kind=None) + ) else: step_node = gast.Constant(value=1, kind=None) return step_node @@ -453,62 +500,82 @@ class ForNodeVisitor(object): if not isinstance(step_node, (gast.Constant, gast.UnaryOp)): raise NotImplementedError( "Dynamic-to-Static only supports the step value is a constant or negative constant in 'for-range' statements, " - "such as '2', '-3'. But received: '{}'. Please fix code to be compatible with Dynamic-to-Static." - .format(ast_to_source_code(step_node).strip())) + "such as '2', '-3'. But received: '{}'. Please fix code to be compatible with Dynamic-to-Static.".format( + ast_to_source_code(step_node).strip() + ) + ) if isinstance(step_node, gast.UnaryOp) or step_node.value < 0: # eg: # range(max, min, -2) # -> # i > min - return gast.Compare(left=gast.Name( - id=self.iter_var_name - if self.is_for_range_iter() else self.iter_idx_name, - ctx=gast.Load(), - annotation=None, - type_comment=None), - ops=[gast.Gt()], - comparators=[compare_node]) + return gast.Compare( + left=gast.Name( + id=self.iter_var_name + if self.is_for_range_iter() + else self.iter_idx_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ops=[gast.Gt()], + comparators=[compare_node], + ) else: # eg: # range(min, max, 2) # -> # i < max - return gast.Compare(left=gast.Name( - id=self.iter_var_name - if self.is_for_range_iter() else self.iter_idx_name, - ctx=gast.Load(), - annotation=None, - type_comment=None), - ops=[gast.Lt()], - comparators=[compare_node]) + return gast.Compare( + left=gast.Name( + id=self.iter_var_name + if self.is_for_range_iter() + else self.iter_idx_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ops=[gast.Lt()], + comparators=[compare_node], + ) def _build_index_increase_node(self, step_node): - return gast.AugAssign(target=gast.Name( - id=self.iter_var_name - if self.is_for_range_iter() else self.iter_idx_name, - ctx=gast.Store(), - annotation=None, - type_comment=None), - op=gast.Add(), - value=step_node) + return gast.AugAssign( + target=gast.Name( + id=self.iter_var_name + if self.is_for_range_iter() + else self.iter_idx_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ), + op=gast.Add(), + value=step_node, + ) def _build_assign_var_slice_node(self): var_slice_str = "{}[{}]".format( - ast_to_source_code(self.iter_node).strip(), self.iter_idx_name) + ast_to_source_code(self.iter_node).strip(), self.iter_idx_name + ) var_slice_node = gast.parse(var_slice_str).body[0].value new_iter_var_name = unique_name.generate(FOR_ITER_VAR_NAME_PREFIX) - target_node, assign_node = create_assign_node(new_iter_var_name, - var_slice_node) + target_node, assign_node = create_assign_node( + new_iter_var_name, var_slice_node + ) return target_node, assign_node def _build_enum_increase_node(self): - return gast.AugAssign(target=gast.Name(id=self.enum_idx_name, - ctx=gast.Store(), - annotation=None, - type_comment=None), - op=gast.Add(), - value=gast.Constant(value=1, kind=None)) + return gast.AugAssign( + target=gast.Name( + id=self.enum_idx_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ), + op=gast.Add(), + value=gast.Constant(value=1, kind=None), + ) def _get_iter_var_name(self): if self.is_for_range_iter(): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py index 55afb7ae6d6..d6c32a1fc2b 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py @@ -15,9 +15,13 @@ import astor from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static import utils -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code @@ -139,9 +143,11 @@ class AttributeJstTransformer(BaseTransformer): assert isinstance( node, gast.AST ), "Input non-gast.AST node for the initialization of ToTensorTransformer." - self.interested_name = set([ - 'size', - ]) + self.interested_name = set( + [ + 'size', + ] + ) self.root = node def transform(self): @@ -151,12 +157,21 @@ class AttributeJstTransformer(BaseTransformer): def visit_Attribute(self, node): assert isinstance(node, gast.Attribute) assert isinstance(node.attr, str) - if isinstance(node.ctx, - gast.Load) and node.attr in self.interested_name: + if ( + isinstance(node.ctx, gast.Load) + and node.attr in self.interested_name + ): attr = node.attr value = node.value - node = gast.parse("_jst.Attr({}, \"{}\")".format( - ast_to_source_code(value).strip(), attr)).body[0].value + node = ( + gast.parse( + "_jst.Attr({}, \"{}\")".format( + ast_to_source_code(value).strip(), attr + ) + ) + .body[0] + .value + ) self.generic_visit(node) return node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py index 8bfc7187eca..6726e5f14f4 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py @@ -17,9 +17,15 @@ from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list from paddle.fluid.dygraph.dygraph_to_static.utils import BaseNodeVisitor -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_bool_node -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ForNodeVisitor +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_bool_node, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + ForNodeVisitor, +) __all__ = ['BreakContinueTransformer'] @@ -35,8 +41,8 @@ class ForToWhileTransformer(BaseTransformer): def __init__(self, parent_node, loop_node, condition_node): assert isinstance( - loop_node, - gast.For), "loop_node is not gast.For in ForToWhileTransformer" + loop_node, gast.For + ), "loop_node is not gast.For in ForToWhileTransformer" self.parent_node = parent_node self.loop_node = loop_node self.condition_node = condition_node @@ -47,7 +53,7 @@ class ForToWhileTransformer(BaseTransformer): i = index_in_list(body_list, self.loop_node) if i != -1: new_stmts = self.get_for_stmt_nodes(body_list[i]) - body_list[i:i + 1] = new_stmts + body_list[i : i + 1] = new_stmts i += len(new_stmts) return new_stmts if hasattr(self.parent_node, 'orelse'): @@ -55,7 +61,7 @@ class ForToWhileTransformer(BaseTransformer): i = index_in_list(body_list, self.loop_node) if i != -1: new_stmts = self.get_for_stmt_nodes(body_list[i]) - body_list[i:i + 1] = new_stmts + body_list[i : i + 1] = new_stmts i += len(new_stmts) return new_stmts raise ValueError( @@ -64,7 +70,8 @@ class ForToWhileTransformer(BaseTransformer): def get_for_stmt_nodes(self, node): assert isinstance( - node, gast.For), "Input node is NOT gast.For in get_for_stmt_nodes" + node, gast.For + ), "Input node is NOT gast.For in get_for_stmt_nodes" # 1. parse current gast.For node current_for_node_parser = ForNodeVisitor(node) @@ -74,13 +81,14 @@ class ForToWhileTransformer(BaseTransformer): init_stmts, cond_stmt, body_stmts = stmts_tuple # 2. append break statement - new_cond_stmt = gast.BoolOp(op=gast.And(), - values=[cond_stmt, self.condition_node]) + new_cond_stmt = gast.BoolOp( + op=gast.And(), values=[cond_stmt, self.condition_node] + ) # 3. construct gast.While node - while_node = gast.While(test=new_cond_stmt, - body=body_stmts, - orelse=node.orelse) + while_node = gast.While( + test=new_cond_stmt, body=body_stmts, orelse=node.orelse + ) init_stmts.append(while_node) return init_stmts @@ -132,7 +140,8 @@ class BreakContinueTransformer(BaseNodeVisitor): # block can be a node containing stmt list. We should remove all stmts # after the 'break/continue' and set the V to True here. first_block_index = self._remove_stmts_after_break_continue( - node, variable_name, loop_node_index) + node, variable_name, loop_node_index + ) # 3. Add 'if not V' for stmts in ancestor blocks between the first one # (exclusive) and the ancestor loop (inclusive) @@ -142,19 +151,25 @@ class BreakContinueTransformer(BaseNodeVisitor): assign_false_node = create_bool_node(variable_name, False) self._add_stmt_before_cur_node(loop_node_index, assign_false_node) - cond_var_node = gast.UnaryOp(op=gast.Not(), - operand=gast.Name(id=variable_name, - ctx=gast.Load(), - annotation=None, - type_comment=None)) + cond_var_node = gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=variable_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ) if isinstance(loop_node, gast.While): - loop_node.test = gast.BoolOp(op=gast.And(), - values=[loop_node.test, cond_var_node]) + loop_node.test = gast.BoolOp( + op=gast.And(), values=[loop_node.test, cond_var_node] + ) elif isinstance(loop_node, gast.For): parent_node = self.ancestor_nodes[loop_node_index - 1] - for_to_while = ForToWhileTransformer(parent_node, loop_node, - cond_var_node) + for_to_while = ForToWhileTransformer( + parent_node, loop_node, cond_var_node + ) for_to_while.transform() def visit_Continue(self, node): @@ -169,7 +184,8 @@ class BreakContinueTransformer(BaseNodeVisitor): # block can be a node containing stmt list. We should remove all stmts # after the 'break/continue' and set the V to True here. first_block_index = self._remove_stmts_after_break_continue( - node, variable_name, loop_node_index) + node, variable_name, loop_node_index + ) # 3. Add 'if not V' for stmts in ancestor blocks between the first one # (exclusive) and the ancestor loop (inclusive) @@ -179,44 +195,51 @@ class BreakContinueTransformer(BaseNodeVisitor): assign_false_node = create_bool_node(variable_name, False) loop_node.body.insert(0, assign_false_node) - def _remove_stmts_after_break_continue(self, break_continue_node, - break_continue_name, - loop_node_index): + def _remove_stmts_after_break_continue( + self, break_continue_node, break_continue_name, loop_node_index + ): for first_block_index in range( - len(self.ancestor_nodes) - 1, loop_node_index - 1, -1): + len(self.ancestor_nodes) - 1, loop_node_index - 1, -1 + ): first_block = self.ancestor_nodes[first_block_index] - if hasattr(first_block, - "body") and self._replace_break_continue_in_stmt_list( - first_block.body, break_continue_node, - break_continue_name): + if hasattr( + first_block, "body" + ) and self._replace_break_continue_in_stmt_list( + first_block.body, break_continue_node, break_continue_name + ): return first_block_index - if hasattr(first_block, - "orelse") and self._replace_break_continue_in_stmt_list( - first_block.orelse, break_continue_node, - break_continue_name): + if hasattr( + first_block, "orelse" + ) and self._replace_break_continue_in_stmt_list( + first_block.orelse, break_continue_node, break_continue_name + ): return first_block_index return first_block_index - def _replace_if_stmt(self, loop_node_index, first_block_index, - break_continue_name): + def _replace_if_stmt( + self, loop_node_index, first_block_index, break_continue_name + ): for i in range(first_block_index - 1, loop_node_index - 1, -1): cur_node = self.ancestor_nodes[i] son_node = self.ancestor_nodes[i + 1] - if hasattr(cur_node, - 'body') and self._replace_after_node_to_if_in_stmt_list( - cur_node.body, son_node, break_continue_name): + if hasattr( + cur_node, 'body' + ) and self._replace_after_node_to_if_in_stmt_list( + cur_node.body, son_node, break_continue_name + ): continue if hasattr( - cur_node, - 'orelse') and self._replace_after_node_to_if_in_stmt_list( - cur_node.orelse, son_node, break_continue_name): + cur_node, 'orelse' + ) and self._replace_after_node_to_if_in_stmt_list( + cur_node.orelse, son_node, break_continue_name + ): continue - def _replace_break_continue_in_stmt_list(self, stmt_list, - break_continue_node, - break_continue_name): + def _replace_break_continue_in_stmt_list( + self, stmt_list, break_continue_node, break_continue_name + ): i = index_in_list(stmt_list, break_continue_node) if i == -1: return False @@ -224,8 +247,9 @@ class BreakContinueTransformer(BaseNodeVisitor): stmt_list[i:] = [assign_true_node] return True - def _replace_after_node_to_if_in_stmt_list(self, stmt_list, node, - break_continue_name): + def _replace_after_node_to_if_in_stmt_list( + self, stmt_list, node, break_continue_name + ): i = index_in_list(stmt_list, node) if i == -1: return False @@ -234,28 +258,37 @@ class BreakContinueTransformer(BaseNodeVisitor): # No need to add, we consider this as added successfully return True - if_stmt = gast.If(test=gast.UnaryOp(op=gast.Not(), - operand=gast.Name( - id=break_continue_name, - ctx=gast.Store(), - annotation=None, - type_comment=None)), - body=stmt_list[i + 1:], - orelse=[]) - stmt_list[i + 1:] = [] + if_stmt = gast.If( + test=gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=break_continue_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ), + ), + body=stmt_list[i + 1 :], + orelse=[], + ) + stmt_list[i + 1 :] = [] stmt_list.append(if_stmt) return True def _add_stmt_before_cur_node(self, cur_node_index, stmt_node): cur_node = self.ancestor_nodes[cur_node_index] parent_node = self.ancestor_nodes[cur_node_index - 1] - if hasattr(parent_node, - "body") and self._add_stmt_into_list_before_node( - parent_node.body, cur_node, stmt_node): + if hasattr( + parent_node, "body" + ) and self._add_stmt_into_list_before_node( + parent_node.body, cur_node, stmt_node + ): return True - if hasattr(parent_node, - "orelse") and self._add_stmt_into_list_before_node( - parent_node.orelse, cur_node, stmt_node): + if hasattr( + parent_node, "orelse" + ) and self._add_stmt_into_list_before_node( + parent_node.orelse, cur_node, stmt_node + ): return True return False @@ -330,11 +363,13 @@ class BreakTransformOptimizer(BaseNodeVisitor): if isinstance(loop_node, gast.While): loop_node.test = gast.BoolOp( - op=gast.And(), values=[loop_node.test, cond_var_node]) + op=gast.And(), values=[loop_node.test, cond_var_node] + ) elif isinstance(loop_node, gast.For): parent_node = self.ancestor_nodes[loop_node_index - 1] - for_to_while = ForToWhileTransformer(parent_node, loop_node, - cond_var_node) + for_to_while = ForToWhileTransformer( + parent_node, loop_node, cond_var_node + ) for_to_while.transform() def _is_break_cond_pattern(self, break_node, loop_node): @@ -351,8 +386,10 @@ class BreakTransformOptimizer(BaseNodeVisitor): is_matched = False if isinstance(parent_if_node, gast.If): # gast.If only contains `break` - break_first_in_if = parent_if_node.body[0] == break_node and len( - parent_if_node.orelse) == 0 + break_first_in_if = ( + parent_if_node.body[0] == break_node + and len(parent_if_node.orelse) == 0 + ) # gast.If is first node of loop_node if_first_in_loop = loop_node.body[0] == parent_if_node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py index d6af6ffced7..a29fe8f9d6b 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py @@ -14,10 +14,14 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) PDB_SET = "pdb.set_trace" @@ -47,7 +51,10 @@ class CallTransformer(BaseTransformer): func_str = ast_to_source_code(node.func).strip() try: - from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import is_builtin + from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import ( + is_builtin, + ) + need_convert_builtin_func_list = { 'len', 'zip', diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py index 06c79703381..7a5821e12f5 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py @@ -14,9 +14,13 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class CastTransformer(BaseTransformer): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py index 3f136182839..5adf810eef4 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py @@ -26,12 +26,26 @@ import six import builtins from paddle.fluid.dygraph.container import Sequential -from paddle.fluid.dygraph.dygraph_to_static.convert_operators import convert_len, convert_zip -from paddle.fluid.dygraph.dygraph_to_static.convert_operators import convert_range, convert_enumerate -from paddle.fluid.dygraph.dygraph_to_static.logging_utils import TranslatorLogger -from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction -from paddle.fluid.dygraph.dygraph_to_static.program_translator import convert_to_static -from paddle.fluid.dygraph.dygraph_to_static.program_translator import unwrap_decorators +from paddle.fluid.dygraph.dygraph_to_static.convert_operators import ( + convert_len, + convert_zip, +) +from paddle.fluid.dygraph.dygraph_to_static.convert_operators import ( + convert_range, + convert_enumerate, +) +from paddle.fluid.dygraph.dygraph_to_static.logging_utils import ( + TranslatorLogger, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + StaticFunction, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + convert_to_static, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + unwrap_decorators, +) from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_func, unwrap from paddle.fluid.dygraph.layers import Layer @@ -39,7 +53,14 @@ __all__ = ["convert_call"] # TODO(liym27): A better way to do this. BUILTIN_LIKELY_MODULES = [ - collections, pdb, copy, inspect, re, six, numpy, logging + collections, + pdb, + copy, + inspect, + re, + six, + numpy, + logging, ] # The api(s) should be considered as plain function and convert # them into static layer code. @@ -65,8 +86,8 @@ class ConversionOptions(object): def is_builtin(func, name=None): - """ predict whether a function is a builtin function with name={name}. - if name == None, then any builtin function will return True + """predict whether a function is a builtin function with name={name}. + if name == None, then any builtin function will return True """ def name_judge(): @@ -93,8 +114,10 @@ def is_unsupported(func): if func_in_dict: translator_logger.log( 2, - "Whitelist: {} is part of built-in module and does not have to be transformed." - .format(func)) + "Whitelist: {} is part of built-in module and does not have to be transformed.".format( + func + ), + ) return True # NOTE: should be placed before `is_paddle_func` @@ -104,8 +127,10 @@ def is_unsupported(func): if is_paddle_func(func): translator_logger.log( 2, - "Whitelist: {} is part of Paddle module and does not have to be transformed." - .format(func)) + "Whitelist: {} is part of Paddle module and does not have to be transformed.".format( + func + ), + ) return True @@ -145,8 +170,9 @@ def convert_call(func): # [1. 1. 1.]] """ - translator_logger.log(1, - "Convert callable object: convert {}.".format(func)) + translator_logger.log( + 1, "Convert callable object: convert {}.".format(func) + ) func_self = None converted_call = None @@ -158,8 +184,10 @@ def convert_call(func): if options is not None and options.not_convert: translator_logger.log( 2, - "{} is not converted when it is decorated by 'paddle.jit.not_to_static'." - .format(func)) + "{} is not converted when it is decorated by 'paddle.jit.not_to_static'.".format( + func + ), + ) return func if is_builtin(func, "len"): @@ -183,9 +211,15 @@ def convert_call(func): # occasion. number_of_stars = 30 translator_logger.warn( - "\n\n" + "*" * number_of_stars + - "\nYour function:`{}` doesn't support to transform to static function because it is a generator function, it will be run as-is." - .format(func.__name__) + "\n" + "*" * number_of_stars + "\n\n") + "\n\n" + + "*" * number_of_stars + + "\nYour function:`{}` doesn't support to transform to static function because it is a generator function, it will be run as-is.".format( + func.__name__ + ) + + "\n" + + "*" * number_of_stars + + "\n\n" + ) return func if inspect.isfunction(func): @@ -214,10 +248,12 @@ def convert_call(func): _, fn = unwrap_decorators(fn) global_functions.add(fn) elif inspect.isclass(fn): - if isinstance(fn.__dict__.get(func.__name__, None), - staticmethod): + if isinstance( + fn.__dict__.get(func.__name__, None), staticmethod + ): global_functions.add( - func) # Add func to ensure that we will convert + func + ) # Add func to ensure that we will convert if func in global_functions: converted_call = convert_to_static(func) @@ -227,8 +263,10 @@ def convert_call(func): # If func is not in __globals__, it does not need to be transformed # because it has been transformed before. translator_logger.warn( - "{} doesn't have to be transformed to static function because it has been transformed before, it will be run as-is." - .format(func)) + "{} doesn't have to be transformed to static function because it has been transformed before, it will be run as-is.".format( + func + ) + ) converted_call = func except AttributeError: # NOTE: @@ -274,12 +312,15 @@ def convert_call(func): func_self = None if func_self else func_self else: raise NotImplementedError( - "Callable {} can not be transformed at present.".format(func)) + "Callable {} can not be transformed at present.".format(func) + ) if converted_call is None: translator_logger.warn( - "{} doesn't have to be transformed to static function, and it will be run as-is." - .format(func)) + "{} doesn't have to be transformed to static function, and it will be run as-is.".format( + func + ) + ) return func if func_self: diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py b/python/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py index 2e8aa40541c..0fa48c4260c 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py @@ -15,16 +15,46 @@ import re import paddle from paddle.fluid.data_feeder import convert_dtype -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + to_static_variable, +) from paddle.fluid.framework import core, Variable from paddle.fluid.layers import Assert, Print from paddle.fluid.layers import range as paddle_range -from paddle.fluid.layers import array_length, array_read, array_write, create_array -from paddle.fluid.layers import assign, fill_constant, slice, reduce_all, reduce_any -from paddle.fluid.layers import cast, control_flow, logical_and, logical_not, logical_or, nn -from paddle.fluid.layers.control_flow import cond, while_loop, less_than, increment -from paddle.fluid.dygraph.dygraph_to_static.return_transformer import RETURN_NO_VALUE_VAR_NAME -from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar, Dygraph2StaticException +from paddle.fluid.layers import ( + array_length, + array_read, + array_write, + create_array, +) +from paddle.fluid.layers import ( + assign, + fill_constant, + slice, + reduce_all, + reduce_any, +) +from paddle.fluid.layers import ( + cast, + control_flow, + logical_and, + logical_not, + logical_or, + nn, +) +from paddle.fluid.layers.control_flow import ( + cond, + while_loop, + less_than, + increment, +) +from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + RETURN_NO_VALUE_VAR_NAME, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + UndefinedVar, + Dygraph2StaticException, +) from paddle.fluid.dygraph.dygraph_to_static.utils import GetterSetterHelper from paddle.fluid.layers.utils import copy_mutable_vars @@ -37,8 +67,10 @@ def convert_attr(x, attr): def indexable(x, code=None): - if isinstance(x, Variable): return x - if hasattr(x, '__len__') and hasattr(x, '__getitem__'): return x + if isinstance(x, Variable): + return x + if hasattr(x, '__len__') and hasattr(x, '__getitem__'): + return x if hasattr(x, '__iter__'): return [i for i in x] else: @@ -46,8 +78,7 @@ def indexable(x, code=None): def unpack_by_structure(target, structure): - """ unified unpack interface for paddle and python. - """ + """unified unpack interface for paddle and python.""" if isinstance(target, Variable): return _unpack_by_structure_paddle(target, structure) else: @@ -55,8 +86,7 @@ def unpack_by_structure(target, structure): def _unpack_by_structure_python(target, structure): - """ TODO(xiongkun): analysis the differences between python and paddle unpack. - """ + """TODO(xiongkun): analysis the differences between python and paddle unpack.""" return _unpack_by_structure_paddle(target, structure) @@ -75,12 +105,9 @@ def _unpack_by_structure_paddle(target, structure): return ret -def convert_while_loop(cond, - body, - getter, - setter, - return_name_ids=None, - push_pop_names=None): +def convert_while_loop( + cond, body, getter, setter, return_name_ids=None, push_pop_names=None +): """ A function representation of a Python ``while`` statement. @@ -100,8 +127,9 @@ def convert_while_loop(cond, # If loop_vars is changed during cond callable, then it causes bug, but current logical_and/logical_not/... doesn't change the loop_vars. pred = cond() if isinstance(pred, Variable): - _run_paddle_while(cond, body, getter, setter, return_name_ids, - push_pop_names) + _run_paddle_while( + cond, body, getter, setter, return_name_ids, push_pop_names + ) else: _run_py_while(cond, body, getter, setter) @@ -117,19 +145,21 @@ def _convert_tensor_arrray_if_necessary(setterhelper, push_pop_names): else: return v - setterhelper.set(push_pop_names, - [maybe_to_tensor_array(v) for v in push_pop_vars]) + setterhelper.set( + push_pop_names, [maybe_to_tensor_array(v) for v in push_pop_vars] + ) -def _run_paddle_while(cond, body, getter, setter, return_name_ids, - push_pop_names): +def _run_paddle_while( + cond, body, getter, setter, return_name_ids, push_pop_names +): # NOTE: loop_vars of Paddle op `control_flow.while_loop` must be Paddle Tensors. helper = GetterSetterHelper(getter, setter, return_name_ids, push_pop_names) _convert_tensor_arrray_if_necessary(helper, push_pop_names) def new_body_fn(*args): - """ wrap the body() and add return value for `while_loop` - the args may be differ from getter(). + """wrap the body() and add return value for `while_loop` + the args may be differ from getter(). """ mutable_loop_vars = args helper.set(return_name_ids, mutable_loop_vars) @@ -137,8 +167,8 @@ def _run_paddle_while(cond, body, getter, setter, return_name_ids, return helper.get(return_name_ids) def new_cond_fn(*args): - """ cond is a zero-args function, which is not - compatible with `while_loop`. + """cond is a zero-args function, which is not + compatible with `while_loop`. """ return cond() @@ -147,8 +177,9 @@ def _run_paddle_while(cond, body, getter, setter, return_name_ids, to_static_variable(var) if not isinstance(var, UndefinedVar) else var for var in helper.get(return_name_ids) ] - helper.set(return_name_ids, - loop_vars) # change the non-local var to variable + helper.set( + return_name_ids, loop_vars + ) # change the non-local var to variable # variable maybe modified to inner var. change it into loop_vars = control_flow.while_loop(new_cond_fn, new_body_fn, loop_vars) helper.set(return_name_ids, loop_vars) @@ -160,8 +191,10 @@ def _run_py_while(cond, body, getter, setter): pred = cond() if isinstance(pred, Variable): raise Dygraph2StaticException( - "python while pred change from bool to variable.") - if not pred: break + "python while pred change from bool to variable." + ) + if not pred: + break body() @@ -293,13 +326,15 @@ def _run_py_logical_not(x): return not x -def convert_ifelse(pred, - true_fn, - false_fn, - get_args, - set_args, - return_name_ids, - push_pop_names=None): +def convert_ifelse( + pred, + true_fn, + false_fn, + get_args, + set_args, + return_name_ids, + push_pop_names=None, +): """ A function representation of a Python ``if/else`` statement. @@ -317,63 +352,84 @@ def convert_ifelse(pred, """ if isinstance(pred, Variable): - out = _run_paddle_cond(pred, true_fn, false_fn, get_args, set_args, - return_name_ids, push_pop_names) + out = _run_paddle_cond( + pred, + true_fn, + false_fn, + get_args, + set_args, + return_name_ids, + push_pop_names, + ) else: - out = _run_py_ifelse(pred, true_fn, false_fn, get_args, set_args, - return_name_ids) + out = _run_py_ifelse( + pred, true_fn, false_fn, get_args, set_args, return_name_ids + ) return out -def _run_paddle_cond(pred, true_fn, false_fn, get_args, set_args, - return_name_ids, push_pop_names): +def _run_paddle_cond( + pred, true_fn, false_fn, get_args, set_args, return_name_ids, push_pop_names +): """ Paddle cond API will evaluate both ture_fn and false_fn codes. """ - helper = GetterSetterHelper(get_args, set_args, return_name_ids, - push_pop_names) + helper = GetterSetterHelper( + get_args, set_args, return_name_ids, push_pop_names + ) _convert_tensor_arrray_if_necessary(helper, push_pop_names) pred = cast_bool_if_necessary(pred) init_args = helper.get(return_name_ids) def new_true_fn(): - #init args may contain mutable python container like [var, 2], we copy then like in while_loop + # init args may contain mutable python container like [var, 2], we copy then like in while_loop helper.set(return_name_ids, copy_mutable_vars(init_args)) ret = true_fn() # IfExpr will return a non-None return value, so we just return ret. # We assume normal return has no return value. - if ret is None: return helper.get(return_name_ids) - else: return ret + if ret is None: + return helper.get(return_name_ids) + else: + return ret def new_false_fn(): - #init args may contain mutable python container like [var, 2], we copy then like in while_loop + # init args may contain mutable python container like [var, 2], we copy then like in while_loop helper.set(return_name_ids, copy_mutable_vars(init_args)) ret = false_fn() - if ret is None: return helper.get(return_name_ids) - else: return ret + if ret is None: + return helper.get(return_name_ids) + else: + return ret try: - cond_outs = control_flow.cond(pred, new_true_fn, new_false_fn, None, - return_name_ids) + cond_outs = control_flow.cond( + pred, new_true_fn, new_false_fn, None, return_name_ids + ) except Exception as e: - if re.search("Unsupported return type of true_fn and false_fn in cond", - str(e)): + if re.search( + "Unsupported return type of true_fn and false_fn in cond", str(e) + ): raise Dygraph2StaticException( - "Your if/else have different return type. TODO: add link to modifty. {}" - .format(str(e))) + "Your if/else have different return type. TODO: add link to modifty. {}".format( + str(e) + ) + ) if re.search("Incompatible return values of", str(e)): raise Dygraph2StaticException( - "Your if/else have different number of return value. TODO: add link to modifty. {}" - .format(str(e))) + "Your if/else have different number of return value. TODO: add link to modifty. {}".format( + str(e) + ) + ) raise e get_args = lambda: helper.get(return_name_ids) set_args = lambda vs: helper.set(return_name_ids, vs) return _recover_args_state(cond_outs, get_args, set_args, return_name_ids) -def _run_py_ifelse(pred, true_fn, false_fn, get_args, set_args, - return_name_ids): +def _run_py_ifelse( + pred, true_fn, false_fn, get_args, set_args, return_name_ids +): """ Evaluate python original branch function if-else. """ @@ -387,20 +443,22 @@ def _remove_no_value_return_var(out): align_ret = out[0] if isinstance(align_ret, tuple): for index, item in enumerate(align_ret): - if isinstance(item, Variable) and (RETURN_NO_VALUE_VAR_NAME - in item.name): + if isinstance(item, Variable) and ( + RETURN_NO_VALUE_VAR_NAME in item.name + ): # return None if index == 0: - processed_out = (None, ) + out[1:] + processed_out = (None,) + out[1:] elif index == 1: processed_out = align_ret[:1] + out[1:] else: - processed_out = (align_ret[:index], ) + out[1:] + processed_out = (align_ret[:index],) + out[1:] break for index, item in enumerate(processed_out): - if isinstance(item, Variable) and (RETURN_NO_VALUE_VAR_NAME - in item.name): + if isinstance(item, Variable) and ( + RETURN_NO_VALUE_VAR_NAME in item.name + ): processed_out = processed_out[:index] if not processed_out: @@ -415,14 +473,17 @@ def _remove_no_value_return_var(out): def _check_no_undefined_var(outs, names, branch_name): - if names is None: return + if names is None: + return if not isinstance(outs, (list, tuple)): outs = [outs] for var, name in zip(list(outs), names): if isinstance(var, UndefinedVar): raise ValueError( - "Required '{}' must be initialized both in if-else branch, but found it not initialized in '{}'." - .format(name, branch_name)) + "Required '{}' must be initialized both in if-else branch, but found it not initialized in '{}'.".format( + name, branch_name + ) + ) def _recover_args_state(outs, get_args, set_args, return_name_ids): @@ -443,10 +504,11 @@ def _recover_args_state(outs, get_args, set_args, return_name_ids): assert num_outs <= num_args if num_args == 1: - final_outs = (outs, ) if not isinstance(outs, - (list, tuple)) else tuple(outs) + final_outs = ( + (outs,) if not isinstance(outs, (list, tuple)) else tuple(outs) + ) else: - outs = (outs, ) if num_outs == 1 else tuple(outs) + outs = (outs,) if num_outs == 1 else tuple(outs) final_outs = outs + init_args[num_outs:] set_args(final_outs) @@ -464,8 +526,8 @@ def convert_len(var): if isinstance(var, Variable): assert var.ndim > 0, "len() of a 0D tensor is wrong" if var.type in [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.SELECTED_ROWS + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, ]: # Note: Length of var may be known ahead of time in dygraph, # but it probably represents batch size which can be variant. @@ -478,7 +540,8 @@ def convert_len(var): else: raise TypeError( 'len(var) only supports LoDTensor/LoDTensorArray/SelectedRows, but received %s.' - % type(var)) + % type(var) + ) else: if isinstance(var, VariableTuple): return var.__len__() @@ -490,16 +553,17 @@ def convert_zip(*args): if isinstance(arg, Variable) and arg.shape[0] == -1: raise RuntimeError( "Not support zip(tensor, ...) when tensor.shape[0] == -1, " - "but found args[{}].shape[0] == -1 in 'zip'".format(str(i))) + "but found args[{}].shape[0] == -1 in 'zip'".format(str(i)) + ) return zip(*args) # TODO(xiongkun): delete when list is ready. class VariableTuple: """ - this class will cause enumerate can't be wrapped by other iterator change function. - this will be fixed when list is producted. - VariableTuple can only deal with variables which is fixed. + this class will cause enumerate can't be wrapped by other iterator change function. + this will be fixed when list is producted. + VariableTuple can only deal with variables which is fixed. """ def __init__(self, var, start=0): @@ -527,7 +591,8 @@ def convert_enumerate(*args): def convert_range(*args): has_variable = any(map(lambda x: isinstance(x, Variable), args)) if has_variable: - if len(args) == 1: return paddle_range(0, args[0], 1, paddle.int64) + if len(args) == 1: + return paddle_range(0, args[0], 1, paddle.int64) if len(args) == 2: return paddle_range(args[0], args[1], 1, paddle.int64) if len(args) == 3: @@ -581,8 +646,12 @@ def convert_shape_compare(left, *args): """ args_len = len(args) - assert args_len >= 2, "convert_shape_compare needs at least one right compare variable" - assert args_len % 2 == 0, "Illegal input for convert_shape_compare, *args should be op(str), var, op(str), var ..." + assert ( + args_len >= 2 + ), "convert_shape_compare needs at least one right compare variable" + assert ( + args_len % 2 == 0 + ), "Illegal input for convert_shape_compare, *args should be op(str), var, op(str), var ..." num_cmp = args_len // 2 if isinstance(left, Variable): @@ -590,7 +659,12 @@ def convert_shape_compare(left, *args): element_wise_result = eval("x " + op_str + " y") if op_str == "!=": return reduce_any(element_wise_result) - elif op_str == "is" or op_str == "is not" or op_str == "in" or op_str == "not in": + elif ( + op_str == "is" + or op_str == "is not" + or op_str == "in" + or op_str == "not in" + ): return element_wise_result else: return reduce_all(element_wise_result) @@ -601,8 +675,9 @@ def convert_shape_compare(left, *args): cmp_op = args[i * 2] cmp_right = args[i * 2 + 1] cur_result = reduce_compare(cmp_left, cmp_op, cmp_right) - final_result = convert_logical_and(lambda: final_result, - lambda: cur_result) + final_result = convert_logical_and( + lambda: final_result, lambda: cur_result + ) return final_result else: cmp_left = left @@ -633,13 +708,23 @@ def convert_var_dtype(var, dtype): if isinstance(var, Variable): src_dtype = convert_dtype(var.dtype) assert src_dtype in [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'uint8' + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', ], "The dtype of var {} is {}, which is not supported in the cast op.".format( - var.name, src_dtype) + var.name, src_dtype + ) assert dtype in [ - 'bool', 'int', 'float' + 'bool', + 'int', + 'float', ], "The casted target dtype is {}, which is not supported in type casting.".format( - dtype) + dtype + ) cast_map = { 'bool': 'bool', 'int': 'int32', @@ -733,7 +818,6 @@ def _run_paddle_pop(array, *args): # TODO(liym27): A better way to slice tensor array. # Maybe support start == end for slice op. def _slice_tensor_array(array, start, end): - def true_fn(): null_array = create_array("float32") return null_array diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/create_variable_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/create_variable_transformer.py index 78ca606dcec..3432765191f 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/create_variable_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/create_variable_transformer.py @@ -13,21 +13,28 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.utils import FunctionNameLivenessAnalysis -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_undefined_var -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FunctionNameLivenessAnalysis, +) +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_undefined_var, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class CreateVariableTransformer(BaseTransformer): - """ - """ + """ """ def __init__(self, wrapper_root): - assert isinstance( - wrapper_root, AstNodeWrapper - ), "Type of input node should be AstNodeWrapper, but received %s ." % type( - wrapper_root) + assert isinstance(wrapper_root, AstNodeWrapper), ( + "Type of input node should be AstNodeWrapper, but received %s ." + % type(wrapper_root) + ) self.root = wrapper_root.node FunctionNameLivenessAnalysis(self.root) @@ -38,7 +45,7 @@ class CreateVariableTransformer(BaseTransformer): self.visit(self.root) def visit_FunctionDef(self, node): - #attributes = set(filter(lambda x: '.' in x, node.pd_scope.modified_vars())) + # attributes = set(filter(lambda x: '.' in x, node.pd_scope.modified_vars())) self.generic_visit(node) bodys = node.body names = sorted(node.pd_scope.created_vars()) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/decorator_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/decorator_transformer.py index db88d1670e1..55ca1f9d9ac 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/decorator_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/decorator_transformer.py @@ -14,17 +14,31 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.utils import create_funcDef_node, ast_to_source_code, is_paddle_api, Dygraph2StaticException +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_funcDef_node, + ast_to_source_code, + is_paddle_api, + Dygraph2StaticException, +) import warnings import re from paddle.fluid.dygraph.dygraph_to_static.utils import RE_PYNAME, RE_PYMODULE IGNORE_NAMES = [ - 'declarative', 'to_static', 'dygraph_to_static_func', 'wraps', - 'staticmethod', 'classmethod', 'decorator' + 'declarative', + 'to_static', + 'dygraph_to_static_func', + 'wraps', + 'staticmethod', + 'classmethod', + 'decorator', ] @@ -34,10 +48,10 @@ class DecoratorTransformer(BaseTransformer): """ def __init__(self, wrapper_root): - assert isinstance( - wrapper_root, AstNodeWrapper - ), "Type of input node should be AstNodeWrapper, but received %s ." % type( - wrapper_root) + assert isinstance(wrapper_root, AstNodeWrapper), ( + "Type of input node should be AstNodeWrapper, but received %s ." + % type(wrapper_root) + ) self.root = wrapper_root.node self.ancestor_nodes = [] @@ -70,16 +84,21 @@ class DecoratorTransformer(BaseTransformer): # 1: @_jst.Call(a.b.c.d.deco)() # 2: @q.w.e.r.deco() re_tmp = re.match( - r'({module})*({name}\(){{0,1}}({module})*({name})(\)){{0,1}}\(.*$' - .format(name=RE_PYNAME, module=RE_PYMODULE), deco_full_name) + r'({module})*({name}\(){{0,1}}({module})*({name})(\)){{0,1}}\(.*$'.format( + name=RE_PYNAME, module=RE_PYMODULE + ), + deco_full_name, + ) deco_name = re_tmp.group(4) else: # match case like: # @a.d.g.deco re_tmp = re.match( - r'({module})*({name})$'.format(name=RE_PYNAME, - module=RE_PYMODULE), - deco_full_name) + r'({module})*({name})$'.format( + name=RE_PYNAME, module=RE_PYMODULE + ), + deco_full_name, + ) deco_name = re_tmp.group(2) if deco_name in IGNORE_NAMES: continue @@ -95,25 +114,37 @@ class DecoratorTransformer(BaseTransformer): if '_jst.Call' in deco_full_name: # in this case , the deco_full_name will be like: # '_jst.Call(deco)(5)' - rematch = re.match(r'\_jst\.Call\((.+?)\)\((.*)\)', - deco_full_name) + rematch = re.match( + r'\_jst\.Call\((.+?)\)\((.*)\)', deco_full_name + ) re_name = rematch.group(1) re_args = rematch.group(2) re_args_with_func = deco_target + ', ' + re_args - decofun_str = 'try:\n\t{0} = _jst.Call({1})({2})\nexcept:\n\t{0} = _jst.Call({1})({3})({4})'\ - .format(decoed_func, re_name, re_args_with_func, re_args, deco_target) + decofun_str = 'try:\n\t{0} = _jst.Call({1})({2})\nexcept:\n\t{0} = _jst.Call({1})({3})({4})'.format( + decoed_func, + re_name, + re_args_with_func, + re_args, + deco_target, + ) else: # paddle api will not be transformed to '_jst.Call' rematch = re.match(r'(.+?)\((.*)\)', deco_full_name) re_name = rematch.group(1) re_args = rematch.group(2) re_args_with_func = deco_target + ', ' + re_args - decofun_str = 'try:\n\t{0} = {1}({2})\nexcept:\n\t{0} = {1}({3})({4})'\ - .format(decoed_func, re_name, re_args_with_func, re_args, deco_target) + decofun_str = 'try:\n\t{0} = {1}({2})\nexcept:\n\t{0} = {1}({3})({4})'.format( + decoed_func, + re_name, + re_args_with_func, + re_args, + deco_target, + ) else: decofun_str = '{} = _jst.Call({})({})'.format( - decoed_func, deco_full_name, deco_target) + decoed_func, deco_full_name, deco_target + ) decofun_nodes.extend(gast.parse(decofun_str).body) deco_target = decoed_func @@ -121,12 +152,14 @@ class DecoratorTransformer(BaseTransformer): if not decofun_nodes: return node - orig_func_node = gast.FunctionDef(name='_orig_' + node.name, - args=node.args, - body=node.body, - decorator_list=[], - returns=None, - type_comment=None) + orig_func_node = gast.FunctionDef( + name='_orig_' + node.name, + args=node.args, + body=node.body, + decorator_list=[], + returns=None, + type_comment=None, + ) args = [arg.id for arg in node.args.args] arg_str = ','.join(args) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/early_return_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/early_return_transformer.py index e20f50b609e..1ce75b27786 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/early_return_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/early_return_transformer.py @@ -13,8 +13,12 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class EarlyReturnTransformer(BaseTransformer): @@ -23,10 +27,10 @@ class EarlyReturnTransformer(BaseTransformer): """ def __init__(self, wrapper_root): - assert isinstance( - wrapper_root, AstNodeWrapper - ), "Type of input node should be AstNodeWrapper, but received %s ." % type( - wrapper_root) + assert isinstance(wrapper_root, AstNodeWrapper), ( + "Type of input node should be AstNodeWrapper, but received %s ." + % type(wrapper_root) + ) self.root = wrapper_root.node def transform(self): @@ -39,7 +43,8 @@ class EarlyReturnTransformer(BaseTransformer): assert isinstance( node, gast.If ), "Type of input node should be gast.If, but received %s ." % type( - node) + node + ) for child in node.body: if isinstance(child, gast.Return): return True @@ -60,9 +65,11 @@ class EarlyReturnTransformer(BaseTransformer): if isinstance(node, gast.If) and self.is_define_return_in_if(node): destination_nodes = node.orelse # handle stmt like `if/elif/elif` - while len(destination_nodes) > 0 and \ - isinstance(destination_nodes[0], gast.If) and \ - self.is_define_return_in_if(destination_nodes[0]): + while ( + len(destination_nodes) > 0 + and isinstance(destination_nodes[0], gast.If) + and self.is_define_return_in_if(destination_nodes[0]) + ): destination_nodes = destination_nodes[0].orelse return result_nodes diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/error.py b/python/paddle/fluid/dygraph/dygraph_to_static/error.py index d05a8ce4d84..7e05b5db891 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/error.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/error.py @@ -19,8 +19,15 @@ import linecache import re import numpy as np -from paddle.fluid.dygraph.dygraph_to_static.origin_info import Location, OriginInfo, global_origin_info_map -from paddle.fluid.dygraph.dygraph_to_static.utils import _is_api_in_module_helper, RE_PYMODULE +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + Location, + OriginInfo, + global_origin_info_map, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + _is_api_in_module_helper, + RE_PYMODULE, +) ERROR_DATA = "Error data about original source code information and traceback." @@ -72,10 +79,17 @@ class TraceBackFrame(OriginInfo): def formated_message(self): # self.source_code may be empty in some functions. # For example, decorator generated function - return ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'File "{}", line {}, in {}\n\t{}'.format( - self.location.filepath, self.location.lineno, self.function_name, - self.source_code.lstrip() - if isinstance(self.source_code, str) else self.source_code) + return ( + ' ' * BLANK_COUNT_BEFORE_FILE_STR + + 'File "{}", line {}, in {}\n\t{}'.format( + self.location.filepath, + self.location.lineno, + self.function_name, + self.source_code.lstrip() + if isinstance(self.source_code, str) + else self.source_code, + ) + ) class TraceBackFrameRange(OriginInfo): @@ -118,26 +132,35 @@ class TraceBackFrameRange(OriginInfo): for i in range(len(self.source_code)): # if source_code[i] is empty line between two code line, dont add blank if self.source_code[i]: - self.source_code[i] = ' ' * ( - blank_count[i] - min_black_count + - BLANK_COUNT_BEFORE_FILE_STR * 2) + self.source_code[i] + self.source_code[i] = ( + ' ' + * ( + blank_count[i] + - min_black_count + + BLANK_COUNT_BEFORE_FILE_STR * 2 + ) + + self.source_code[i] + ) def formated_message(self): - msg = ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'File "{}", line {}, in {}\n'.format( - self.location.filepath, self.location.lineno, self.function_name) + msg = ( + ' ' * BLANK_COUNT_BEFORE_FILE_STR + + 'File "{}", line {}, in {}\n'.format( + self.location.filepath, self.location.lineno, self.function_name + ) + ) # add empty line after range code return msg + '\n'.join(self.source_code) class SuggestionDict(object): - def __init__(self): # {(keywords): (suggestions)} self.suggestion_dict = { - ('is not initialized.', 'Hint:', 'IsInitialized'): - ("Please ensure all your sublayers are inheritted from nn.Layer.", - "Please ensure there is no tensor created explicitly depended on external data, we suggest to register it as buffer tensor. See https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/04_dygraph_to_static/export_model/principle_cn.html#parameters-buffers for details" - ) + ('is not initialized.', 'Hint:', 'IsInitialized'): ( + "Please ensure all your sublayers are inheritted from nn.Layer.", + "Please ensure there is no tensor created explicitly depended on external data, we suggest to register it as buffer tensor. See https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/04_dygraph_to_static/export_model/principle_cn.html#parameters-buffers for details", + ) } def keys(self): @@ -156,8 +179,9 @@ class ErrorData(object): Error data attached to an exception which is raised in un-transformed code. """ - def __init__(self, error_type, error_value, origin_traceback, - origin_info_map): + def __init__( + self, error_type, error_value, origin_traceback, origin_info_map + ): self.error_type = error_type self.error_value = error_value self.origin_traceback = origin_traceback @@ -182,24 +206,30 @@ class ErrorData(object): func_str = None for frame in tb: searched_name = re.search( - r'({module})*{name}'.format(module=RE_PYMODULE, - name=frame.name), error_line) + r'({module})*{name}'.format( + module=RE_PYMODULE, name=frame.name + ), + error_line, + ) if searched_name: func_str = searched_name.group(0) break try: - module_result = eval("_is_api_in_module_helper({}, '{}')".format( - func_str, "numpy")) - is_numpy_api_err = module_result or (func_str.startswith("numpy.") - or func_str.startswith("np.")) + module_result = eval( + "_is_api_in_module_helper({}, '{}')".format(func_str, "numpy") + ) + is_numpy_api_err = module_result or ( + func_str.startswith("numpy.") or func_str.startswith("np.") + ) except Exception: is_numpy_api_err = False if is_numpy_api_err and func_str: return [ - "TypeError: Code '{}' called numpy API {}, please use Paddle API to replace it." - .format(error_line, func_str), - " values will be changed to variables by dy2static, numpy api can not handle variables" + "TypeError: Code '{}' called numpy API {}, please use Paddle API to replace it.".format( + error_line, func_str + ), + " values will be changed to variables by dy2static, numpy api can not handle variables", ] else: return format_exception @@ -219,45 +249,56 @@ class ErrorData(object): # Simplify error value to improve readability if error is raised in runtime if self.in_runtime: if int( - os.getenv(SIMPLIFY_ERROR_ENV_NAME, - DEFAULT_SIMPLIFY_NEW_ERROR)): + os.getenv(SIMPLIFY_ERROR_ENV_NAME, DEFAULT_SIMPLIFY_NEW_ERROR) + ): self._simplify_error_value() message_lines.append(str(self.error_value)) return '\n'.join(message_lines) # Step2: Optimizes stack information with source code information of dygraph from user. user_code_traceback_index = [] - for i, (filepath, lineno, funcname, - code) in enumerate(self.origin_traceback): - dygraph_func_info = self.origin_info_map.get((filepath, lineno), - None) + for i, (filepath, lineno, funcname, code) in enumerate( + self.origin_traceback + ): + dygraph_func_info = self.origin_info_map.get( + (filepath, lineno), None + ) if dygraph_func_info: user_code_traceback_index.append(i) # Add user code traceback for i in user_code_traceback_index: filepath, lineno, funcname, code = self.origin_traceback[i] - dygraph_func_info = self.origin_info_map.get((filepath, lineno), - None) + dygraph_func_info = self.origin_info_map.get( + (filepath, lineno), None + ) if i == user_code_traceback_index[-1]: traceback_frame = TraceBackFrameRange( - dygraph_func_info.location, dygraph_func_info.function_name) + dygraph_func_info.location, dygraph_func_info.function_name + ) else: traceback_frame = TraceBackFrame( - dygraph_func_info.location, dygraph_func_info.function_name, - dygraph_func_info.source_code) + dygraph_func_info.location, + dygraph_func_info.function_name, + dygraph_func_info.source_code, + ) message_lines.append(traceback_frame.formated_message()) error_line = traceback_frame.error_line message_lines.append("") # Add paddle traceback after user code traceback - paddle_traceback_start_index = user_code_traceback_index[ - -1] + 1 if user_code_traceback_index else 0 + paddle_traceback_start_index = ( + user_code_traceback_index[-1] + 1 + if user_code_traceback_index + else 0 + ) for filepath, lineno, funcname, code in self.origin_traceback[ - paddle_traceback_start_index:]: - traceback_frame = TraceBackFrame(Location(filepath, lineno), - funcname, code) + paddle_traceback_start_index: + ]: + traceback_frame = TraceBackFrame( + Location(filepath, lineno), funcname, code + ) message_lines.append(traceback_frame.formated_message()) message_lines.append("") @@ -265,10 +306,12 @@ class ErrorData(object): # NOTE: `format_exception` is a list, its length is 1 in most cases, but sometimes its length # is gather than 1, for example, the error_type is IndentationError. format_exception = traceback.format_exception_only( - self.error_type, self.error_value) + self.error_type, self.error_value + ) if error_line is not None: - format_exception = self.numpy_api_check(format_exception, - error_line) + format_exception = self.numpy_api_check( + format_exception, error_line + ) error_message = [ " " * BLANK_COUNT_BEFORE_FILE_STR + line @@ -280,17 +323,23 @@ class ErrorData(object): def _create_revise_suggestion(self, bottom_error_message): revise_suggestions = [ - '', ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'Revise suggestion: ' + '', + ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'Revise suggestion: ', ] for keywords in self.suggestion_dict.keys(): contain_keywords = [ True for i in keywords if i in ''.join(bottom_error_message) ] if len(contain_keywords) == len( - keywords): # all keywords should be in bottom_error_message + keywords + ): # all keywords should be in bottom_error_message for suggestion in self.suggestion_dict[keywords]: - suggestion_msg = ' ' * BLANK_COUNT_BEFORE_FILE_STR * 2 + '{}. {}'.format( - str(len(revise_suggestions) - 1), suggestion) + suggestion_msg = ( + ' ' * BLANK_COUNT_BEFORE_FILE_STR * 2 + + '{}. {}'.format( + str(len(revise_suggestions) - 1), suggestion + ) + ) revise_suggestions.append(suggestion_msg) return revise_suggestions if len(revise_suggestions) > 2 else [] @@ -313,12 +362,12 @@ class ErrorData(object): start_trace = "outputs = static_func(*inputs)" start_idx = error_value_lines_strip.index(start_trace) - error_value_lines = error_value_lines[start_idx + 1:] - error_value_lines_strip = error_value_lines_strip[start_idx + 1:] + error_value_lines = error_value_lines[start_idx + 1 :] + error_value_lines_strip = error_value_lines_strip[start_idx + 1 :] # use empty line to locate the bottom_error_message empty_line_idx = error_value_lines_strip.index('') - bottom_error_message = error_value_lines[empty_line_idx + 1:] + bottom_error_message = error_value_lines[empty_line_idx + 1 :] revise_suggestion = self._create_revise_suggestion(bottom_error_message) error_traceback = [] @@ -336,14 +385,18 @@ class ErrorData(object): if error_value_lines_strip[i].startswith("File "): re_result = re.search(pattern, error_value_lines_strip[i]) tmp_filepath, lineno_str, function_name = re_result.groups() - code = error_value_lines_strip[ - i + 1] if i + 1 < len(error_value_lines_strip) else '' + code = ( + error_value_lines_strip[i + 1] + if i + 1 < len(error_value_lines_strip) + else '' + ) if static_info_map.get((tmp_filepath, int(lineno_str))): user_code_traceback_index.append(len(error_traceback)) error_traceback.append( - (tmp_filepath, int(lineno_str), function_name, code)) + (tmp_filepath, int(lineno_str), function_name, code) + ) error_frame = [] # Add user code traceback @@ -351,20 +404,27 @@ class ErrorData(object): filepath, lineno, funcname, code = error_traceback[i] if i == user_code_traceback_index[-1]: traceback_frame = TraceBackFrameRange( - Location(filepath, lineno), funcname) + Location(filepath, lineno), funcname + ) else: - traceback_frame = TraceBackFrame(Location(filepath, lineno), - funcname, code) + traceback_frame = TraceBackFrame( + Location(filepath, lineno), funcname, code + ) error_frame.append(traceback_frame.formated_message()) error_frame.append("") # Add paddle traceback after user code traceback - paddle_traceback_start_index = user_code_traceback_index[ - -1] + 1 if user_code_traceback_index else 0 + paddle_traceback_start_index = ( + user_code_traceback_index[-1] + 1 + if user_code_traceback_index + else 0 + ) for filepath, lineno, funcname, code in error_traceback[ - paddle_traceback_start_index:]: - traceback_frame = TraceBackFrame(Location(filepath, lineno), - funcname, code) + paddle_traceback_start_index: + ]: + traceback_frame = TraceBackFrame( + Location(filepath, lineno), funcname, code + ) error_frame.append(traceback_frame.formated_message()) error_frame.append("") diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py b/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py index 730c02fbc5e..039df7e84e4 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py @@ -48,8 +48,9 @@ class FunctionSpec(object): self._arg_names, self._default_kwargs = parse_arg_and_kwargs(function) # parse *args self.varargs_name = parse_varargs_name(function) - if self.varargs_name is not None and isinstance(function.__self__, - TranslatedLayer): + if self.varargs_name is not None and isinstance( + function.__self__, TranslatedLayer + ): self._arg_names += function.__self__._input_args_names def unified_args_and_kwargs(self, args, kwargs): @@ -72,8 +73,12 @@ class FunctionSpec(object): """ if len(self._arg_names) < len(args): error_msg = "The decorated function `{}` requires {} arguments: {}, but received {} with {}.".format( - self._dygraph_function.__name__, len(self._arg_names), - self._arg_names, len(args), args) + self._dygraph_function.__name__, + len(self._arg_names), + self._arg_names, + len(args), + args, + ) if args and inspect.isclass(args[0]): error_msg += "\n\tMaybe the function has more than one decorator, we don't support this for now." raise NotImplementedError(error_msg) @@ -90,9 +95,13 @@ class FunctionSpec(object): else: if arg_name not in self._default_kwargs: raise ValueError( - "`{}()` requires `{}` arguments, but not found in input `args`: {} and `kwargs`: {}." - .format(self._dygraph_function.__name__, arg_name, args, - kwargs)) + "`{}()` requires `{}` arguments, but not found in input `args`: {} and `kwargs`: {}.".format( + self._dygraph_function.__name__, + arg_name, + args, + kwargs, + ) + ) args.append(self._default_kwargs[arg_name]) return tuple(args), kwargs @@ -135,16 +144,20 @@ class FunctionSpec(object): # So we don't support to deal this case while specificing `input_spec` currently. if kwargs: raise ValueError( - "{} got unexpected keyword arguments: {}. Cannot trace the function when `input_spec` is specificed." - .format(self._dygraph_function.__name__, kwargs)) + "{} got unexpected keyword arguments: {}. Cannot trace the function when `input_spec` is specificed.".format( + self._dygraph_function.__name__, kwargs + ) + ) # Note: The length of `input_spec` can be greater than `args`, # because `args` may contains non-tensor value merged form `kwargs` # after `unified_args_and_kwargs`. if len(args) < len(self._input_spec): raise ValueError( - "Requires len(arguments) >= len(input_spec), but received len(args):{} < len(InputSpec): {}" - .format(len(args), len(self._input_spec))) + "Requires len(arguments) >= len(input_spec), but received len(args):{} < len(InputSpec): {}".format( + len(args), len(self._input_spec) + ) + ) # replace argument with corresponding InputSpec. args_with_spec = convert_to_input_spec(args, self._input_spec) @@ -154,8 +167,9 @@ class FunctionSpec(object): # If without specificing name in input_spec, add default name # according to argument name from decorated function. - args_with_spec = replace_spec_empty_name(self._arg_names, - args_with_spec) + args_with_spec = replace_spec_empty_name( + self._arg_names, args_with_spec + ) return args_with_spec, kwargs_with_spec @@ -182,7 +196,8 @@ class FunctionSpec(object): dtype=var_spec.dtype, is_data=True, need_check_feed=False, - stop_gradient=stop_gradient) + stop_gradient=stop_gradient, + ) else: feed_layer = var_spec inputs.append(feed_layer) @@ -195,15 +210,19 @@ class FunctionSpec(object): """ if not isinstance(input_spec, (tuple, list)): raise TypeError( - "The type(input_spec) should be one of (tuple, list), but received {}." - .format(type_name(input_spec))) + "The type(input_spec) should be one of (tuple, list), but received {}.".format( + type_name(input_spec) + ) + ) return tuple(input_spec) def __repr__(self): return "function: {}({}), input_spec: {}".format( - self._dygraph_function.__name__, ','.join(self._arg_names), - self._input_spec) + self._dygraph_function.__name__, + ','.join(self._arg_names), + self._input_spec, + ) @property def dygraph_function(self): @@ -242,8 +261,10 @@ def get_parameters(layer_instance, include_sublayer=True): params = layer_instance._parameters else: raise TypeError( - "Type of `layer_instance` should be nn.Layer, but received {}". - format(type_name(layer_instance))) + "Type of `layer_instance` should be nn.Layer, but received {}".format( + type_name(layer_instance) + ) + ) return params @@ -264,8 +285,10 @@ def get_buffers(layer_instance, include_sublayer=True): buffers = layer_instance._buffers else: raise TypeError( - "Type of `layer_instance` should be nn.Layer, but received {}". - format(type_name(layer_instance))) + "Type of `layer_instance` should be nn.Layer, but received {}".format( + type_name(layer_instance) + ) + ) return buffers @@ -284,12 +307,17 @@ def convert_to_input_spec(inputs, input_spec): def check_type_and_len(input, spec, check_length=False): if type(input) is not type(spec): - raise TypeError('type(input) should be {}, but received {}.'.format( - type(spec), type(input))) + raise TypeError( + 'type(input) should be {}, but received {}.'.format( + type(spec), type(input) + ) + ) if check_length and len(input) < len(spec): raise ValueError( - 'Requires len(inputs) >= len(input_spec), but received len(inputs):{} < len(input_spec):{}' - .format(len(inputs), len(input_spec))) + 'Requires len(inputs) >= len(input_spec), but received len(inputs):{} < len(input_spec):{}'.format( + len(inputs), len(input_spec) + ) + ) if isinstance(input_spec, (tuple, list)): input_with_spec = [] @@ -302,13 +330,15 @@ def convert_to_input_spec(inputs, input_spec): # Note: If the rest inputs contain tensor or numpy.ndarray # without specific InputSpec, raise warning. if len(inputs) > len(input_spec): - for rest_input in inputs[len(input_spec):]: + for rest_input in inputs[len(input_spec) :]: if isinstance(rest_input, (core.VarBase, np.ndarray)): logging_utils.warn( "The inputs constain `{}` without specificing InputSpec, its shape and dtype will be treated immutable. " - "Please specific InputSpec information in `@to_static` if you expect them as mutable inputs." - .format(type_name(rest_input))) - input_with_spec.extend(inputs[len(input_spec):]) + "Please specific InputSpec information in `@to_static` if you expect them as mutable inputs.".format( + type_name(rest_input) + ) + ) + input_with_spec.extend(inputs[len(input_spec) :]) return input_with_spec elif isinstance(input_spec, dict): @@ -317,7 +347,8 @@ def convert_to_input_spec(inputs, input_spec): for name, input in inputs.items(): if name in input_spec: input_with_spec[name] = convert_to_input_spec( - input, input_spec[name]) + input, input_spec[name] + ) else: input_with_spec[name] = input return input_with_spec @@ -354,7 +385,7 @@ def replace_spec_empty_name(args_name, input_with_spec): print([in_var.name for in_var in foo.inputs]) # [x, y] """ input_with_spec = list(input_with_spec) - candidate_arg_names = args_name[:len(input_with_spec)] + candidate_arg_names = args_name[: len(input_with_spec)] for i, arg_name in enumerate(candidate_arg_names): input_spec = input_with_spec[i] @@ -405,11 +436,13 @@ def _hash_spec_names(args_specs, kwargs_specs): the former has one input ('x'), but the latter has two input ('x', 'y'). """ spec_names = [ - spec.name for spec in flatten(args_specs) + spec.name + for spec in flatten(args_specs) if isinstance(spec, paddle.static.InputSpec) ] spec_names += [ - spec.name for spec in flatten(kwargs_specs) + spec.name + for spec in flatten(kwargs_specs) if isinstance(spec, paddle.static.InputSpec) ] i, name_ids = 0, {} diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py index 5d51b0f48c2..8120e79c1da 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py @@ -23,16 +23,47 @@ from collections import defaultdict from paddle.utils import gast from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.utils import create_funcDef_node, ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.utils import create_assign_node, FunctionNameLivenessAnalysis -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_undefined_var -from paddle.fluid.dygraph.dygraph_to_static.utils import create_nonlocal_stmt_nodes -from paddle.fluid.dygraph.dygraph_to_static.utils import create_get_args_node, create_set_args_node -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_INDEX_PREFIX, FOR_ITER_TUPLE_PREFIX, FOR_ITER_TUPLE_INDEX_PREFIX, FOR_ITER_VAR_LEN_PREFIX, FOR_ITER_VAR_NAME_PREFIX, FOR_ITER_ZIP_TO_LIST_PREFIX, FOR_ITER_TARGET_PREFIX, FOR_ITER_ITERATOR_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import GetterSetterHelper, create_name_str +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_funcDef_node, + ast_to_source_code, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_assign_node, + FunctionNameLivenessAnalysis, +) +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + StaticAnalysisVisitor, +) +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_undefined_var, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_nonlocal_stmt_nodes, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_get_args_node, + create_set_args_node, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_INDEX_PREFIX, + FOR_ITER_TUPLE_PREFIX, + FOR_ITER_TUPLE_INDEX_PREFIX, + FOR_ITER_VAR_LEN_PREFIX, + FOR_ITER_VAR_NAME_PREFIX, + FOR_ITER_ZIP_TO_LIST_PREFIX, + FOR_ITER_TARGET_PREFIX, + FOR_ITER_ITERATOR_PREFIX, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + GetterSetterHelper, + create_name_str, +) TRUE_FUNC_PREFIX = 'true_fn' FALSE_FUNC_PREFIX = 'false_fn' @@ -47,13 +78,14 @@ class IfElseTransformer(BaseTransformer): """ def __init__(self, wrapper_root): - assert isinstance( - wrapper_root, AstNodeWrapper - ), "Type of input node should be AstNodeWrapper, but received %s ." % type( - wrapper_root) + assert isinstance(wrapper_root, AstNodeWrapper), ( + "Type of input node should be AstNodeWrapper, but received %s ." + % type(wrapper_root) + ) self.root = wrapper_root.node FunctionNameLivenessAnalysis( - self.root) # name analysis of current ast tree. + self.root + ) # name analysis of current ast tree. def transform(self): """ @@ -63,16 +95,31 @@ class IfElseTransformer(BaseTransformer): def visit_If(self, node): self.generic_visit(node) - true_func_node, false_func_node, get_args_node, set_args_node, return_name_ids, push_pop_ids = transform_if_else( - node, self.root) - - new_node = create_convert_ifelse_node(return_name_ids, push_pop_ids, - node.test, true_func_node, - false_func_node, get_args_node, - set_args_node) - - return [get_args_node, set_args_node, true_func_node, false_func_node - ] + [new_node] + ( + true_func_node, + false_func_node, + get_args_node, + set_args_node, + return_name_ids, + push_pop_ids, + ) = transform_if_else(node, self.root) + + new_node = create_convert_ifelse_node( + return_name_ids, + push_pop_ids, + node.test, + true_func_node, + false_func_node, + get_args_node, + set_args_node, + ) + + return [ + get_args_node, + set_args_node, + true_func_node, + false_func_node, + ] + [new_node] def visit_Call(self, node): # Remove `numpy()` statement, like `Tensor.numpy()[i]` -> `Tensor[i]` @@ -89,8 +136,9 @@ class IfElseTransformer(BaseTransformer): """ self.generic_visit(node) - new_node = create_convert_ifelse_node(None, None, node.test, node.body, - node.orelse, None, None, True) + new_node = create_convert_ifelse_node( + None, None, node.test, node.body, node.orelse, None, None, True + ) # Note: A blank line will be added separately if transform gast.Expr # into source code. Using gast.Expr.value instead to avoid syntax error # in python. @@ -101,7 +149,6 @@ class IfElseTransformer(BaseTransformer): class NameVisitor(gast.NodeVisitor): - def __init__(self, after_node=None, end_node=None): # The start node (exclusive) of the visitor self.after_node = after_node @@ -171,7 +218,8 @@ class NameVisitor(gast.NodeVisitor): # Blocks the vars in `if.body` and only inserts the vars both created in 'if/else' branch # into name_ids. new_name_ids = self._find_new_name_ids( - body_name_ids, else_name_ids) + body_name_ids, else_name_ids + ) for new_name_id in new_name_ids: before_if_name_ids[new_name_id].append(gast.Store()) @@ -186,7 +234,8 @@ class NameVisitor(gast.NodeVisitor): self.generic_visit(node) return blacklist = {'True', 'False', 'None'} - if node.id in blacklist: return + if node.id in blacklist: + return if node.id in self._def_func_names: return if not self._is_call_func_name_node(node): @@ -205,7 +254,10 @@ class NameVisitor(gast.NodeVisitor): # NOTE: We skip to visit names of get_args and set_args, because they contains # nonlocal statement such as 'nonlocal x, self' where 'self' should not be # parsed as returned value in contron flow. - if GET_ARGS_FUNC_PREFIX in node.name or SET_ARGS_FUNC_PREFIX in node.name: + if ( + GET_ARGS_FUNC_PREFIX in node.name + or SET_ARGS_FUNC_PREFIX in node.name + ): return if not self._in_range: @@ -236,7 +288,6 @@ class NameVisitor(gast.NodeVisitor): return copy.deepcopy(self.name_ids) def _find_new_name_ids(self, body_name_ids, else_name_ids): - def is_required_ctx(ctxs, required_ctx): for ctx in ctxs: if isinstance(ctx, required_ctx): @@ -244,13 +295,14 @@ class NameVisitor(gast.NodeVisitor): return False candidate_name_ids = set(body_name_ids.keys()) & set( - else_name_ids.keys()) + else_name_ids.keys() + ) store_ctx = gast.Store new_name_ids = set() for name_id in candidate_name_ids: - if is_required_ctx(body_name_ids[name_id], - store_ctx) and is_required_ctx( - else_name_ids[name_id], store_ctx): + if is_required_ctx( + body_name_ids[name_id], store_ctx + ) and is_required_ctx(else_name_ids[name_id], store_ctx): new_name_ids.add(name_id) return new_name_ids @@ -262,8 +314,10 @@ class NameVisitor(gast.NodeVisitor): parent_node = self.ancestor_nodes[-2] if isinstance(parent_node, gast.Call) and parent_node.func == node: # e.g: var_list.append(elem), var_list is also a name_id. - should_skip = isinstance( - node, gast.Attribute) and node.attr in white_func_names + should_skip = ( + isinstance(node, gast.Attribute) + and node.attr in white_func_names + ) if not should_skip: return True return False @@ -290,8 +344,10 @@ def _valid_nonlocal_names(return_name_ids, nonlocal_names): for name in return_name_ids: if name not in nonlocal_names: raise ValueError( - "Required returned var '{}' must be in 'nonlocal' statement '', but not found." - .format(name)) + "Required returned var '{}' must be in 'nonlocal' statement '', but not found.".format( + name + ) + ) nonlocal_names.remove(name) return return_name_ids + nonlocal_names @@ -313,15 +369,21 @@ def transform_if_else(node, root): # TODO(dev): Need a better way to deal this. # LoopTransformer will create some special vars, which is not visiable by users. so we can sure it's safe to remove them. filter_names = [ - ARGS_NAME, FOR_ITER_INDEX_PREFIX, FOR_ITER_TUPLE_PREFIX, - FOR_ITER_TARGET_PREFIX, FOR_ITER_ITERATOR_PREFIX, - FOR_ITER_TUPLE_INDEX_PREFIX, FOR_ITER_VAR_LEN_PREFIX, - FOR_ITER_VAR_NAME_PREFIX, FOR_ITER_ZIP_TO_LIST_PREFIX + ARGS_NAME, + FOR_ITER_INDEX_PREFIX, + FOR_ITER_TUPLE_PREFIX, + FOR_ITER_TARGET_PREFIX, + FOR_ITER_ITERATOR_PREFIX, + FOR_ITER_TUPLE_INDEX_PREFIX, + FOR_ITER_VAR_LEN_PREFIX, + FOR_ITER_VAR_NAME_PREFIX, + FOR_ITER_ZIP_TO_LIST_PREFIX, ] def remove_if(x): for name in filter_names: - if x.startswith(name): return False + if x.startswith(name): + return False return True nonlocal_names = list(filter(remove_if, nonlocal_names)) @@ -329,40 +391,53 @@ def transform_if_else(node, root): nonlocal_stmt_node = create_nonlocal_stmt_nodes(nonlocal_names) - empty_arg_node = gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]) + empty_arg_node = gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ) true_func_node = create_funcDef_node( nonlocal_stmt_node + node.body, name=unique_name.generate(TRUE_FUNC_PREFIX), input_args=empty_arg_node, - return_name_ids=[]) + return_name_ids=[], + ) false_func_node = create_funcDef_node( nonlocal_stmt_node + node.orelse, name=unique_name.generate(FALSE_FUNC_PREFIX), input_args=empty_arg_node, - return_name_ids=[]) + return_name_ids=[], + ) helper = GetterSetterHelper(None, None, nonlocal_names, push_pop_ids) get_args_node = create_get_args_node(helper.union()) set_args_node = create_set_args_node(helper.union()) - return true_func_node, false_func_node, get_args_node, set_args_node, return_name_ids, push_pop_ids - - -def create_convert_ifelse_node(return_name_ids, - push_pop_ids, - pred, - true_func, - false_func, - get_args_func, - set_args_func, - is_if_expr=False): + return ( + true_func_node, + false_func_node, + get_args_node, + set_args_node, + return_name_ids, + push_pop_ids, + ) + + +def create_convert_ifelse_node( + return_name_ids, + push_pop_ids, + pred, + true_func, + false_func, + get_args_func, + set_args_func, + is_if_expr=False, +): """ Create `paddle.jit.dy2static.convert_ifelse( pred, true_fn, false_fn, get_args, set_args, return_name_ids)` @@ -377,16 +452,19 @@ def create_convert_ifelse_node(return_name_ids, convert_ifelse_layer = gast.parse( '_jst.IfElse(' - '{pred}, {true_fn}, {false_fn}, {get_args}, {set_args}, {return_name_ids}, push_pop_names={push_pop_ids})' - .format( + '{pred}, {true_fn}, {false_fn}, {get_args}, {set_args}, {return_name_ids}, push_pop_names={push_pop_ids})'.format( pred=ast_to_source_code(pred), true_fn=true_func_source, false_fn=false_func_source, - get_args=get_args_func.name if not is_if_expr else - 'lambda: None', #TODO: better way to deal with this + get_args=get_args_func.name + if not is_if_expr + else 'lambda: None', # TODO: better way to deal with this set_args=set_args_func.name - if not is_if_expr else 'lambda args: None', + if not is_if_expr + else 'lambda args: None', return_name_ids=create_name_str(return_name_ids), - push_pop_ids=create_name_str(push_pop_ids))).body[0] + push_pop_ids=create_name_str(push_pop_ids), + ) + ).body[0] return convert_ifelse_layer diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py b/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py index f0a4672794c..6f73a23316d 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py @@ -29,7 +29,6 @@ LOG_AllTransformer = 100 def synchronized(func): - def wrapper(*args, **kwargs): with threading.Lock(): return func(*args, **kwargs) @@ -59,7 +58,8 @@ class TranslatorLogger(object): self._logger = log_helper.get_logger( self.logger_name, 1, - fmt='%(asctime)s %(name)s %(levelname)s: %(message)s') + fmt='%(asctime)s %(name)s %(levelname)s: %(message)s', + ) self._verbosity_level = None self._transformed_code_level = None self._need_to_echo_log_to_stdout = None @@ -155,16 +155,19 @@ class TranslatorLogger(object): if self.need_to_echo_log_to_stdout: self._output_to_stdout('INFO: ' + msg_with_level, *args) - def log_transformed_code(self, level, ast_node, transformer_name, *args, - **kwargs): + def log_transformed_code( + self, level, ast_node, transformer_name, *args, **kwargs + ): if self.has_code_level(level): source_code = ast_to_source_code(ast_node) if level == LOG_AllTransformer: - header_msg = "After the last level ast transformer: '{}', the transformed code:\n" \ - .format(transformer_name) + header_msg = "After the last level ast transformer: '{}', the transformed code:\n".format( + transformer_name + ) else: - header_msg = "After the level {} ast transformer: '{}', the transformed code:\n"\ - .format(level, transformer_name) + header_msg = "After the level {} ast transformer: '{}', the transformed code:\n".format( + level, transformer_name + ) msg = header_msg + source_code self.logger.info(msg, *args, **kwargs) @@ -271,5 +274,6 @@ def log(level, msg, *args, **kwargs): def log_transformed_code(level, ast_node, transformer_name, *args, **kwargs): - _TRANSLATOR_LOGGER.log_transformed_code(level, ast_node, transformer_name, - *args, **kwargs) + _TRANSLATOR_LOGGER.log_transformed_code( + level, ast_node, transformer_name, *args, **kwargs + ) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py index b7006d935ec..3ad623a8ff0 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py @@ -14,7 +14,9 @@ from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) cmpop_type_to_str = { gast.Eq: "==", @@ -26,7 +28,7 @@ cmpop_type_to_str = { gast.Is: "is", gast.IsNot: "is not", gast.In: "in", - gast.NotIn: "not in" + gast.NotIn: "not in", } @@ -70,7 +72,8 @@ class LogicalTransformer(BaseTransformer): new_node = self._create_bool_op_node(node.values, 'Or') else: raise TypeError( - "Only supports and/or syntax in control flow if statement.") + "Only supports and/or syntax in control flow if statement." + ) return new_node def _create_bool_op_node(self, nodes, api_type): @@ -80,10 +83,11 @@ class LogicalTransformer(BaseTransformer): according to the actual order. In `convert_logical_and(lambda:x>1, lambda:y<1)`, `lambda:y<1` must be run after `lambda:x>1`, If `x>1` is False, `y<1` should NOT be run. ''' - assert len( - nodes - ) > 1, "The length of BoolOp should be at least 2, but received {}.".format( - len(nodes)) + assert ( + len(nodes) > 1 + ), "The length of BoolOp should be at least 2, but received {}.".format( + len(nodes) + ) if len(nodes) > 2: # Creates logic_and/logic_or node recursively. pre_logic_node = self._create_bool_op_node(nodes[:2], api_type) @@ -95,7 +99,8 @@ class LogicalTransformer(BaseTransformer): args = [ast_to_source_code(child) for child in nodes] new_node_str = "_jst.{}(lambda:{}, lambda:{})".format( - api_type, args[0], args[1]) + api_type, args[0], args[1] + ) # NOTE: gast.parse return Module(body=[expr(...)]) new_node = gast.parse(new_node_str).body[0].value return new_node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py index 38dc7468e08..b61f9f6e43c 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py @@ -17,21 +17,44 @@ from paddle.utils import gast from collections import defaultdict from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + StaticAnalysisVisitor, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import generate_name_node from paddle.fluid.dygraph.dygraph_to_static.utils import get_attribute_full_name -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_undefined_var -from paddle.fluid.dygraph.dygraph_to_static.utils import create_nonlocal_stmt_nodes, create_get_args_node, create_set_args_node -from paddle.fluid.dygraph.dygraph_to_static.utils import FunctionNameLivenessAnalysis +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_undefined_var, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_nonlocal_stmt_nodes, + create_get_args_node, + create_set_args_node, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FunctionNameLivenessAnalysis, +) from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import ARGS_NAME -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import RenameTransformer -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ForLoopTuplePreTransformer -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ForNodeVisitor -from paddle.fluid.dygraph.dygraph_to_static.utils import GetterSetterHelper, create_name_str +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + RenameTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + ForLoopTuplePreTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + ForNodeVisitor, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + GetterSetterHelper, + create_name_str, +) __all__ = ['LoopTransformer', 'NameVisitor'] @@ -42,8 +65,14 @@ FOR_CONDITION_PREFIX = 'for_loop_condition' FOR_BODY_PREFIX = 'for_loop_body' -def create_while_nodes(condition_name, body_name, loop_var_names, - push_pop_names, getter_name, setter_name): +def create_while_nodes( + condition_name, + body_name, + loop_var_names, + push_pop_names, + getter_name, + setter_name, +): """ Returns a list of gast.Node which represents the calling of Paddle controlflow while_loop. @@ -79,13 +108,21 @@ def create_while_nodes(condition_name, body_name, loop_var_names, # set doesn't have order so we convert it to list loop_var_names = list(loop_var_names) assign_loop_var_names = [] - for name in (loop_var_names): + for name in loop_var_names: assign_loop_var_names.append(name) while_func_name = "_jst.While" - while_node_str = "{}({}, {}, {}, {}, return_name_ids={}, push_pop_names={})".format( - while_func_name, condition_name, body_name, getter_name, setter_name, - create_name_str(loop_var_names), create_name_str(push_pop_names)) + while_node_str = ( + "{}({}, {}, {}, {}, return_name_ids={}, push_pop_names={})".format( + while_func_name, + condition_name, + body_name, + getter_name, + setter_name, + create_name_str(loop_var_names), + create_name_str(push_pop_names), + ) + ) while_node = gast.parse(while_node_str).body[0] ret = [while_node] @@ -123,14 +160,16 @@ class NameVisitor(gast.NodeVisitor): self.type_vars = set() self.static_analysis_visitor = StaticAnalysisVisitor(root_node) - self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( + self.node_to_wrapper_map = ( + self.static_analysis_visitor.get_node_to_wrapper_map() ) self.visit(root_node) def get_loop_var_names(self, node): assert isinstance( - node, (gast.While, gast.For)), "Input node is not gast loop node" + node, (gast.While, gast.For) + ), "Input node is not gast loop node" loop_var_names = set() create_var_names = set() read_context = {type(gast.Load()), type(gast.AugLoad())} @@ -141,7 +180,8 @@ class NameVisitor(gast.NodeVisitor): var_name_to_ctxs = defaultdict(list) for var_node in in_loop_vars_list: var_name_to_ctxs[self._var_node_to_name(var_node)].append( - var_node.ctx) + var_node.ctx + ) in_loop_vars = set(in_loop_vars_list) in_loop_vars = self._remove_unnecessary_vars(in_loop_vars, node) @@ -149,13 +189,17 @@ class NameVisitor(gast.NodeVisitor): before_loop_body_vars = self.before_loop_body_vars[node] before_loop_body_vars = self._remove_unnecessary_vars( - before_loop_body_vars, node) + before_loop_body_vars, node + ) before_loop_name_strs = self._var_nodes_to_names(before_loop_body_vars) - after_loop_vars = self.current_seen_vars - before_loop_body_vars - in_loop_vars + after_loop_vars = ( + self.current_seen_vars - before_loop_body_vars - in_loop_vars + ) after_loop_vars = self._remove_unnecessary_vars(after_loop_vars, node) - after_loop_name_strs = self._var_nodes_to_names(after_loop_vars, - read_context) + after_loop_name_strs = self._var_nodes_to_names( + after_loop_vars, read_context + ) condition_vars = self.condition_vars[node] condition_names = self._var_nodes_to_names(condition_vars) @@ -204,8 +248,10 @@ class NameVisitor(gast.NodeVisitor): if isinstance(ctx, gast.Store): is_created = True - if isinstance(var_name_to_ctxs[name][0], - gast.Load) and is_created: + if ( + isinstance(var_name_to_ctxs[name][0], gast.Load) + and is_created + ): loop_var_names.add(name) create_var_names.add(name) @@ -223,7 +269,7 @@ class NameVisitor(gast.NodeVisitor): write_context = { type(gast.Store()), type(gast.AugStore()), - type(gast.Del()) + type(gast.Del()), } for loop_node in self.current_loop: @@ -322,8 +368,10 @@ class NameVisitor(gast.NodeVisitor): def _node_var_type_is_basic(self, node_var_type): basic_types = { - NodeVarType.BOOLEAN, NodeVarType.INT, NodeVarType.FLOAT, - NodeVarType.STRING + NodeVarType.BOOLEAN, + NodeVarType.INT, + NodeVarType.FLOAT, + NodeVarType.STRING, } for t in node_var_type: if t in basic_types: @@ -411,7 +459,8 @@ class NameVisitor(gast.NodeVisitor): target_vars = [target_node] vars_of_list_generator = vars_of_list_generator | set( - target_vars) + target_vars + ) # 1.2 vars from target vars used in elt_node target_var_names = {var.id for var in target_vars} @@ -424,7 +473,8 @@ class NameVisitor(gast.NodeVisitor): for node in elt_nodes: vars_of_list_generator |= filter_name_nodes_from( - node, target_var_names) + node, target_var_names + ) # 2. Get target vars or vars from target vars used in for-loop but the for-loop is # 1) not the "loop_node" itself @@ -450,15 +500,18 @@ class NameVisitor(gast.NodeVisitor): target_vars = [target_node] target_vars_of_for_node = target_vars_of_for_node | set( - target_vars) + target_vars + ) # 2.2 vars from target vars used in for-loop target_vars_name_strs = {var.id for var in target_vars_of_for_node} for var in loop_vars: if not isinstance(var, gast.Name): continue - if var.id in target_vars_name_strs and var not in self.condition_vars[ - loop_node]: + if ( + var.id in target_vars_name_strs + and var not in self.condition_vars[loop_node] + ): target_vars_of_for_node.add(var) removed_vars = target_vars_of_for_node | vars_of_list_generator @@ -506,11 +559,11 @@ class LoopTransformer(BaseTransformer): while i < len(body_list): if isinstance(body_list[i], gast.While): new_stmts = self.get_while_stmt_nodes(body_list[i]) - body_list[i:i + 1] = new_stmts + body_list[i : i + 1] = new_stmts i += len(new_stmts) elif isinstance(body_list[i], gast.For): new_stmts = self.get_for_stmt_nodes(body_list[i]) - body_list[i:i + 1] = new_stmts + body_list[i : i + 1] = new_stmts i += len(new_stmts) else: i += 1 @@ -536,8 +589,10 @@ class LoopTransformer(BaseTransformer): return [node] init_stmts, cond_stmt, body_stmts = stmts_tuple # 2. get original loop vars - loop_var_names, create_var_names = node.pd_scope.modified_vars( - ), node.pd_scope.created_vars() + loop_var_names, create_var_names = ( + node.pd_scope.modified_vars(), + node.pd_scope.created_vars(), + ) push_pop_names = list(node.pd_scope.variadic_length_vars()) # TODO: Remove the bunch of code? We have the unique format `for A in B:` # NOTE: in 'for x in var' or 'for i, x in enumerate(var)' cases, @@ -577,53 +632,64 @@ class LoopTransformer(BaseTransformer): # 5. create & append condition function node condition_func_node = gast.FunctionDef( name=unique_name.generate(FOR_CONDITION_PREFIX), - args=gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]), + args=gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ), body=nonlocal_stmt_node + [gast.Return(value=cond_stmt)], decorator_list=[], returns=None, - type_comment=None) + type_comment=None, + ) new_stmts.append(condition_func_node) # 6. create & append loop body function node # append return values for loop body body_func_node = gast.FunctionDef( name=unique_name.generate(FOR_BODY_PREFIX), - args=gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]), + args=gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ), body=nonlocal_stmt_node + body_stmts, decorator_list=[], returns=None, - type_comment=None) + type_comment=None, + ) new_stmts.append(body_func_node) helper = GetterSetterHelper(None, None, nonlocal_names, push_pop_names) get_args_node = create_get_args_node(helper.union()) set_args_node = create_set_args_node(helper.union()) # 7. create & append while loop node - while_loop_nodes = create_while_nodes(condition_func_node.name, - body_func_node.name, - nonlocal_names, push_pop_names, - get_args_node.name, - set_args_node.name) + while_loop_nodes = create_while_nodes( + condition_func_node.name, + body_func_node.name, + nonlocal_names, + push_pop_names, + get_args_node.name, + set_args_node.name, + ) new_stmts.extend([get_args_node, set_args_node]) new_stmts.extend(while_loop_nodes) return new_stmts def get_while_stmt_nodes(self, node): - loop_var_names, create_var_names = node.pd_scope.modified_vars( - ), node.pd_scope.created_vars() + loop_var_names, create_var_names = ( + node.pd_scope.modified_vars(), + node.pd_scope.created_vars(), + ) push_pop_names = list(node.pd_scope.variadic_length_vars()) new_stmts = [] @@ -648,45 +714,54 @@ class LoopTransformer(BaseTransformer): condition_func_node = gast.FunctionDef( name=unique_name.generate(WHILE_CONDITION_PREFIX), - args=gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]), + args=gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ), body=nonlocal_stmt_node + [gast.Return(value=node.test)], decorator_list=[], returns=None, - type_comment=None) + type_comment=None, + ) new_stmts.append(condition_func_node) new_body = node.body body_func_node = gast.FunctionDef( name=unique_name.generate(WHILE_BODY_PREFIX), - args=gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]), + args=gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ), body=nonlocal_stmt_node + new_body, decorator_list=[], returns=None, - type_comment=None) + type_comment=None, + ) new_stmts.append(body_func_node) helper = GetterSetterHelper(None, None, nonlocal_names, push_pop_names) get_args_node = create_get_args_node(helper.union()) set_args_node = create_set_args_node(helper.union()) - while_loop_nodes = create_while_nodes(condition_func_node.name, - body_func_node.name, - nonlocal_names, push_pop_names, - get_args_node.name, - set_args_node.name) + while_loop_nodes = create_while_nodes( + condition_func_node.name, + body_func_node.name, + nonlocal_names, + push_pop_names, + get_args_node.name, + set_args_node.name, + ) new_stmts.extend([get_args_node, set_args_node]) new_stmts.extend(while_loop_nodes) return new_stmts diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py index 1399eeb2b6b..4bfc73a3256 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py @@ -20,6 +20,7 @@ from paddle.fluid import core from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap from paddle.fluid.dygraph.dygraph_to_static.utils import ORIGI_INFO from paddle.fluid.framework import Program + try: from collections.abc import Sequence except: @@ -30,6 +31,7 @@ class Location(object): """ Location information of source code. """ + __slots__ = ( "filepath", "lineno", @@ -42,8 +44,9 @@ class Location(object): self.col_offset = col_offset def __str__(self): - return "location: {}:{}:{}".format(self.filepath, self.lineno, - self.col_offset) + return "location: {}:{}:{}".format( + self.filepath, self.lineno, self.col_offset + ) @property def line_location(self): @@ -54,6 +57,7 @@ class OriginInfo(object): """ Original information of source code. """ + __slots__ = ( "location", "function_name", @@ -67,17 +71,26 @@ class OriginInfo(object): def __str__(self): return "{} \nsource_code: {} in function {}\n ".format( - self.location, self.source_code, self.function_name) + self.location, self.source_code, self.function_name + ) def formated_message(self): flag_for_origin_info = "(* user code *)" return ' File "{}", line {}, in {} {}\n\t{}'.format( - self.location.filepath, self.location.lineno, self.function_name, - flag_for_origin_info, self.source_code.lstrip()) + self.location.filepath, + self.location.lineno, + self.function_name, + flag_for_origin_info, + self.source_code.lstrip(), + ) def as_frame(self): - return (self.location.filepath, self.location.lineno, - self.function_name, self.source_code.lstrip()) + return ( + self.location.filepath, + self.location.lineno, + self.function_name, + self.source_code.lstrip(), + ) class OriginInfoAttacher(gast.NodeTransformer): @@ -142,9 +155,9 @@ class OriginInfoAttacher(gast.NodeTransformer): global_origin_info_map = {} -def create_and_update_origin_info_map(transformed_node, - static_func, - is_global=True): +def create_and_update_origin_info_map( + transformed_node, static_func, is_global=True +): """ Creates a original information map between transformed static function and original dygraph function. @@ -162,9 +175,11 @@ def create_and_update_origin_info_map(transformed_node, static_node = attach_origin_info(static_node, static_func) for t_node, s_node in ast_walk(transformed_node, static_node): - assert type(t_node) == type(s_node), \ - "The node types should be the same, but received type(t_node) is {}, and type(s_node) is {}." \ - .format(type(t_node), type(s_node)) + assert type(t_node) == type( + s_node + ), "The node types should be the same, but received type(t_node) is {}, and type(s_node) is {}.".format( + type(t_node), type(s_node) + ) dygraph_info = getattr(t_node, ORIGI_INFO, None) static_info = getattr(s_node, ORIGI_INFO, None) @@ -174,9 +189,15 @@ def create_and_update_origin_info_map(transformed_node, exist_origin_info = origin_info_map.get(static_loc) if exist_origin_info is not None: - if exist_origin_info.location.lineno >= dygraph_info.location.lineno: + if ( + exist_origin_info.location.lineno + >= dygraph_info.location.lineno + ): continue - if exist_origin_info.location.col_offset <= dygraph_info.location.col_offset: + if ( + exist_origin_info.location.col_offset + <= dygraph_info.location.col_offset + ): continue origin_info_map[static_loc] = dygraph_info @@ -229,12 +250,15 @@ def ast_walk(transformed_node, static_node): # Node types should be strictly required, but there is no strict distinction between gast.Load and gast.Param # in the ast transformation process. if isinstance(t_node, (gast.Load, gast.Param)) or isinstance( - s_node, (gast.Load, gast.Param)): + s_node, (gast.Load, gast.Param) + ): continue - assert type(t_node) == type(s_node), \ - "The node types should be the same, but received type(t_node) is {}, and type(s_node) is {}."\ - .format(type(t_node), type(s_node)) + assert type(t_node) == type( + s_node + ), "The node types should be the same, but received type(t_node) is {}, and type(s_node) is {}.".format( + type(t_node), type(s_node) + ) yield t_node, s_node @@ -289,11 +313,11 @@ def update_op_callstack_with_origin_info(program): loc = Location(filepath, lineno) dygraph_func_info = global_origin_info_map.get(loc.line_location) if dygraph_func_info: - filepath, lineno, funcname, code = \ - dygraph_func_info.as_frame() + filepath, lineno, funcname, code = dygraph_func_info.as_frame() callstack[i] = ' File "{}", line {}, in {}'.format( - filepath, lineno, funcname) + filepath, lineno, funcname + ) callstack[i + 1] = ' {}'.format(code) return callstack diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py index ebc8b351f78..e0129e84669 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py @@ -16,19 +16,32 @@ import numpy as np import paddle from paddle.fluid import framework, backward, core, program_guard -from paddle.fluid.executor import _is_enable_standalone_executor, _is_dy2st_enable_standalone_executor +from paddle.fluid.executor import ( + _is_enable_standalone_executor, + _is_dy2st_enable_standalone_executor, +) from paddle.fluid.dygraph import layers from paddle.fluid.dygraph.base import switch_to_static_graph from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.return_transformer import RETURN_NO_VALUE_MAGIC_NUM +from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + RETURN_NO_VALUE_MAGIC_NUM, +) from paddle.fluid.layers.utils import flatten from paddle.fluid.layers.utils import pack_sequence_as from paddle.fluid.layers.utils import _hash_with_id from paddle.fluid.compiler import BuildStrategy from paddle.fluid.framework import _apply_pass -from paddle.fluid.contrib.mixed_precision.decorator import AutoMixedPrecisionLists -from paddle.fluid.contrib.mixed_precision.fp16_utils import rewrite_program, cast_model_to_fp16 -from paddle.fluid.dygraph.amp.auto_cast import _in_amp_guard, _in_pure_fp16_guard +from paddle.fluid.contrib.mixed_precision.decorator import ( + AutoMixedPrecisionLists, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + rewrite_program, + cast_model_to_fp16, +) +from paddle.fluid.dygraph.amp.auto_cast import ( + _in_amp_guard, + _in_pure_fp16_guard, +) from paddle import _legacy_C_ops @@ -61,7 +74,8 @@ class NestSequence(object): var_ids = [] for idx, var in enumerate(self.__input_list): if isinstance( - var, (framework.Variable, core.VarBase, core.eager.Tensor)): + var, (framework.Variable, core.VarBase, core.eager.Tensor) + ): var_ids.append(idx) return var_ids @@ -74,15 +88,17 @@ class NestSequence(object): warning_types = set() for var in self.__input_list: if not isinstance( - var, - (framework.Variable, core.VarBase, core.eager.Tensor)): + var, (framework.Variable, core.VarBase, core.eager.Tensor) + ): warning_types.add(type(var)) if warning_types: logging_utils.warn( "Output of traced function contains non-tensor type values: {}. " "Currently, We don't support to update them while training and will return " - "what we first saw. Please try to return them as tensor.". - format(list(warning_types))) + "what we first saw. Please try to return them as tensor.".format( + list(warning_types) + ) + ) @property def var_ids(self): @@ -136,12 +152,9 @@ class PartialProgramLayer: Layer: A Layer object that run all ops internally in static mode. """ - def __init__(self, - main_program, - inputs, - outputs, - parameters=None, - **kwargs): + def __init__( + self, main_program, inputs, outputs, parameters=None, **kwargs + ): super(PartialProgramLayer, self).__init__() self._inputs = NestSequence(inputs) self._outputs = NestSequence(outputs, need_check=True) @@ -164,7 +177,8 @@ class PartialProgramLayer: # For AMP training self._amp_list = AutoMixedPrecisionLists( custom_white_list=custom_white_list, - custom_black_list=custom_black_list) + custom_black_list=custom_black_list, + ) # program_id -> list(scope) self._scope_cache = {} @@ -200,7 +214,8 @@ class PartialProgramLayer: return self._origin_main_program.clone(for_test=is_infer_mode) else: train_program = self._append_backward_desc( - self._origin_main_program) + self._origin_main_program + ) # Note: Only set grad type once after initializing train program. So we put it here. self._set_grad_type(self._params, train_program) return train_program @@ -220,16 +235,18 @@ class PartialProgramLayer: @switch_to_static_graph def _create_pure_fp16_program(self, is_infer_mode=False): pure_fp16_program = self._origin_main_program.clone( - for_test=is_infer_mode) + for_test=is_infer_mode + ) with program_guard(pure_fp16_program): - cast_model_to_fp16(pure_fp16_program, - self._amp_list, - use_fp16_guard=False) + cast_model_to_fp16( + pure_fp16_program, self._amp_list, use_fp16_guard=False + ) if is_infer_mode: return pure_fp16_program else: train_pure_fp16_program = self._append_backward_desc( - pure_fp16_program) + pure_fp16_program + ) self._set_grad_type(self._params, train_pure_fp16_program) return train_pure_fp16_program @@ -237,23 +254,27 @@ class PartialProgramLayer: def _create_forward_backward_train_program(self): whole_program = self._create_program() forward_end_op_index = self._infer_program.desc.block(0).op_size() - return self._get_forward_backward_program_form(whole_program, - forward_end_op_index) + return self._get_forward_backward_program_form( + whole_program, forward_end_op_index + ) @switch_to_static_graph def _create_forward_backward_train_amp_program(self): whole_program = self._create_amp_program() forward_end_op_index = self._infer_amp_program.desc.block(0).op_size() - return self._get_forward_backward_program_form(whole_program, - forward_end_op_index) + return self._get_forward_backward_program_form( + whole_program, forward_end_op_index + ) @switch_to_static_graph def _create_forward_backward_train_pure_fp16_program(self): whole_program = self._create_pure_fp16_program() forward_end_op_index = self._infer_pure_fp16_program.desc.block( - 0).op_size() - return self._get_forward_backward_program_form(whole_program, - forward_end_op_index) + 0 + ).op_size() + return self._get_forward_backward_program_form( + whole_program, forward_end_op_index + ) @LazyInitialized def _train_program(self): @@ -349,8 +370,9 @@ class PartialProgramLayer: @LazyInitialized def _train_program_id(self): program_id = _hash_with_id(self._train_program, self) - core._set_cached_executor_build_strategy(program_id, - self._build_strategy) + core._set_cached_executor_build_strategy( + program_id, self._build_strategy + ) return program_id @LazyInitialized @@ -360,8 +382,9 @@ class PartialProgramLayer: @LazyInitialized def _train_amp_program_id(self): program_id = _hash_with_id(self._train_amp_program, self) - core._set_cached_executor_build_strategy(program_id, - self._build_strategy) + core._set_cached_executor_build_strategy( + program_id, self._build_strategy + ) return program_id @LazyInitialized @@ -371,8 +394,9 @@ class PartialProgramLayer: @LazyInitialized def _train_pure_fp16_program_id(self): program_id = _hash_with_id(self._train_pure_fp16_program, self) - core._set_cached_executor_build_strategy(program_id, - self._build_strategy) + core._set_cached_executor_build_strategy( + program_id, self._build_strategy + ) return program_id @LazyInitialized @@ -408,8 +432,9 @@ class PartialProgramLayer: return main_program - def prepare_gradient_aggregation(self, start_idx, main_program, - target_program): + def prepare_gradient_aggregation( + self, start_idx, main_program, target_program + ): """ Why we need add gradient aggregation operation ? In some cases, if non leaf nodes are used as output, gradient overwriting will occur, such as @@ -427,8 +452,8 @@ class PartialProgramLayer: if exist a op whose inputs is var, then return True """ if not isinstance(var, framework.Variable) or var.type not in [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.SELECTED_ROWS + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, ]: return False if var.dtype not in [paddle.float32, paddle.float64]: @@ -445,20 +470,28 @@ class PartialProgramLayer: new_grad_name = var.name + suffix + "@GRAD" finded_ops = list( filter( - lambda x: x[0] >= start_idx and any([ - out_arg == var_grad_name - for out_arg in x[1].output_arg_names - ]), enumerate(target_program.block(0).ops))) + lambda x: x[0] >= start_idx + and any( + [ + out_arg == var_grad_name + for out_arg in x[1].output_arg_names + ] + ), + enumerate(target_program.block(0).ops), + ) + ) # len(finded_ops) may equals zero when stop_gradient works. # len(finded_ops) may > 1, because we may have fill_constant op. if len(finded_ops) == 0: return None # step1: create a new var named var.name@GRAD - target_program.block(0).create_var(name=new_grad_name, - type=var.type, - dtype=var.dtype, - shape=var.shape) + target_program.block(0).create_var( + name=new_grad_name, + type=var.type, + dtype=var.dtype, + shape=var.shape, + ) # step2: rename the var.name@GRAD to var.name@GRAD@dy2static for idx, op in finded_ops: op._rename_input(var_grad_name, new_grad_name) @@ -469,11 +502,13 @@ class PartialProgramLayer: finded_ops[-1][0] + 1, type='sum', inputs={'X': [var_grad_name, new_grad_name]}, - outputs={"Out": var_grad_name}) + outputs={"Out": var_grad_name}, + ) return None to_processed_vars = list( - filter(_need_aggregation, self._outputs.tolist())) + filter(_need_aggregation, self._outputs.tolist()) + ) for _var in to_processed_vars: _insert_aggregation_ops_for_var(target_program, _var) @@ -489,8 +524,9 @@ class PartialProgramLayer: if targets and self._params: backward.gradients(targets=targets, inputs=[]) - start_idx = len( - main_program.block(0).ops) + 2 * len(self._outputs.tolist()) + start_idx = len(main_program.block(0).ops) + 2 * len( + self._outputs.tolist() + ) self.prepare_gradient_aggregation(start_idx, main_program, program) @@ -509,7 +545,10 @@ class PartialProgramLayer: found_param = False for block in program.blocks: for op in block.ops: - if param.name in op.input_arg_names or param.name in op.output_arg_names: + if ( + param.name in op.input_arg_names + or param.name in op.output_arg_names + ): required_params.append(param) found_param = True break @@ -526,15 +565,21 @@ class PartialProgramLayer: var_desc = block.vars[name].desc var_base = None if not framework._in_eager_mode_: - var_base = core.VarBase(var_desc.dtype(), - var_desc.shape(), - var_desc.name(), - var_desc.type(), False) + var_base = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) else: - var_base = core.eager.Tensor(var_desc.dtype(), - var_desc.shape(), - var_desc.name(), - var_desc.type(), False) + var_base = core.eager.Tensor( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) double_grads.append(var_base) return self._valid_vars(double_grads) @@ -554,36 +599,62 @@ class PartialProgramLayer: attrs = [ 'global_block', - self.program.desc.block(0), 'start_op_index', 0, 'end_op_index', - self._get_end_op_index(), 'is_test', not self.training, - 'program_id', self.program_id + self.program.desc.block(0), + 'start_op_index', + 0, + 'end_op_index', + self._get_end_op_index(), + 'is_test', + not self.training, + 'program_id', + self.program_id, ] if self._cuda_graph_capture_mode: attrs.extend( - ('cuda_graph_capture_mode', self._cuda_graph_capture_mode, - 'cuda_graph_pool_id', self._cuda_graph_pool_id)) - - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + ( + 'cuda_graph_capture_mode', + self._cuda_graph_capture_mode, + 'cuda_graph_pool_id', + self._cuda_graph_pool_id, + ) + ) + + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: attrs.extend( - ('forward_global_block', self.forward_program.desc.block(0), - 'backward_global_block', self.backward_program.desc.block(0))) + ( + 'forward_global_block', + self.forward_program.desc.block(0), + 'backward_global_block', + self.backward_program.desc.block(0), + ) + ) _legacy_C_ops.run_program( - self._valid_vars(in_vars), self._valid_vars(self._params), + self._valid_vars(in_vars), + self._valid_vars(self._params), self._valid_vars(out_vars), - self._create_scope_vec(program_id=self.program_id, - use_scope_cache=True), - self._double_grads, self._cuda_graph_vec, *attrs) + self._create_scope_vec( + program_id=self.program_id, use_scope_cache=True + ), + self._double_grads, + self._cuda_graph_vec, + *attrs + ) else: - _legacy_C_ops.run_program(self._valid_vars(in_vars), - self._valid_vars(self._params), - self._valid_vars(out_vars), - self._create_scope_vec(), - self._double_grads, self._cuda_graph_vec, - *attrs) + _legacy_C_ops.run_program( + self._valid_vars(in_vars), + self._valid_vars(self._params), + self._valid_vars(out_vars), + self._create_scope_vec(), + self._double_grads, + self._cuda_graph_vec, + *attrs + ) restored_nest_out = self._restore_out(out_vars) return self._remove_no_value(restored_nest_out) @@ -591,9 +662,11 @@ class PartialProgramLayer: if _in_pure_fp16_guard(): for i, var in enumerate(in_vars): name = var.name - if (self.program.global_block().has_var(name) - and self.program.global_block().var(name).dtype - == paddle.float16): + if ( + self.program.global_block().has_var(name) + and self.program.global_block().var(name).dtype + == paddle.float16 + ): in_vars[i] = var.astype('float16') in_vars[i].name = name @@ -624,25 +697,32 @@ class PartialProgramLayer: return self._infer_program @switch_to_static_graph - def _get_forward_backward_program_form(self, whole_program, - forward_end_op_index): + def _get_forward_backward_program_form( + self, whole_program, forward_end_op_index + ): forward_builded_program = add_build_strategy_for( - whole_program, 0, forward_end_op_index, self._build_strategy) + whole_program, 0, forward_end_op_index, self._build_strategy + ) backward_start_op_index = forward_end_op_index + 2 * len( - self._outputs.var_ids) + self._outputs.var_ids + ) backward_end_op_index = whole_program.desc.block(0).op_size() backward_builded_program = add_build_strategy_for( - whole_program, backward_start_op_index, backward_end_op_index, - self._build_strategy) - self._apply_inplace_pass(forward_builded_program, - backward_builded_program) + whole_program, + backward_start_op_index, + backward_end_op_index, + self._build_strategy, + ) + self._apply_inplace_pass( + forward_builded_program, backward_builded_program + ) return [forward_builded_program, backward_builded_program] def _apply_inplace_pass(self, forward_program, backward_program): attr_types = { "use_cuda": "bool", "mem_opt_skip_vars": "list[str]", - "for_partial_block": "bool" + "for_partial_block": "bool", } empty_startup_program = paddle.static.Program() use_cuda = True if core.is_compiled_with_cuda() else False @@ -664,22 +744,33 @@ class PartialProgramLayer: forward_mem_opt_skip_vars.append(var.desc.name()) backward_mem_opt_skip_vars.append(var.desc.name()) for var_name in core.parse_safe_eager_deletion_skip_vars( - backward_program.desc): + backward_program.desc + ): forward_mem_opt_skip_vars.append(var_name) attrs = { "use_cuda": use_cuda, "mem_opt_skip_vars": forward_mem_opt_skip_vars, - "for_partial_block": True + "for_partial_block": True, } - _apply_pass(forward_program, empty_startup_program, - "buffer_shared_inplace_pass", attrs, attr_types) + _apply_pass( + forward_program, + empty_startup_program, + "buffer_shared_inplace_pass", + attrs, + attr_types, + ) attrs = { "use_cuda": use_cuda, "mem_opt_skip_vars": backward_mem_opt_skip_vars, - "for_partial_block": True + "for_partial_block": True, } - _apply_pass(backward_program, empty_startup_program, - "buffer_shared_inplace_pass", attrs, attr_types) + _apply_pass( + backward_program, + empty_startup_program, + "buffer_shared_inplace_pass", + attrs, + attr_types, + ) def _prepare(self, inputs): """ @@ -695,23 +786,28 @@ class PartialProgramLayer: if isinstance(value, np.ndarray): var = None if not framework._in_eager_mode_: - var = core.VarBase(value=value, - name=self._inputs[i].desc.name(), - persistable=False, - place=expected_place, - zero_copy=True) + var = core.VarBase( + value=value, + name=self._inputs[i].desc.name(), + persistable=False, + place=expected_place, + zero_copy=True, + ) else: - var = core.eager.Tensor(value=value, - name=self._inputs[i].desc.name(), - persistable=False, - place=expected_place, - zero_copy=True) + var = core.eager.Tensor( + value=value, + name=self._inputs[i].desc.name(), + persistable=False, + place=expected_place, + zero_copy=True, + ) elif isinstance(value, (core.VarBase, core.eager.Tensor)): # NOTE(Aurelius84): If var is on CPUPlace, it will be transformed multi times # into CUDAPlace when it's as input of multi Ops. so we move it in advance # to avoid this problem. if value.stop_gradient and not value.place._equals( - expected_place): + expected_place + ): var = value._copy_to(expected_place, False) var.stop_gradient = True else: @@ -734,12 +830,21 @@ class PartialProgramLayer: return out_varbase_map[var_desc.name()] if not framework._in_eager_mode_: - var_base = core.VarBase(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var_base = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) else: - var_base = core.eager.Tensor(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), - False) + var_base = core.eager.Tensor( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) var_base.stop_gradient = var.stop_gradient out_varbase_map[var_desc.name()] = var_base return var_base @@ -752,20 +857,30 @@ class PartialProgramLayer: def _create_scope_vec(self, program_id=None, use_scope_cache=False): # Hold forward variables tmp_scope_vec = None - inner_scope = self._get_scope(program_id=program_id, - use_scope_cache=use_scope_cache) + inner_scope = self._get_scope( + program_id=program_id, use_scope_cache=use_scope_cache + ) if not framework._in_eager_mode_: - tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [], - "program_out_scope", - core.VarDesc.VarType.STEP_SCOPES, True) + tmp_scope_vec = core.VarBase( + core.VarDesc.VarType.FP32, + [], + "program_out_scope", + core.VarDesc.VarType.STEP_SCOPES, + True, + ) tmp_scope_vec.value().set_scope(inner_scope) else: tmp_scope_vec = [inner_scope] return tmp_scope_vec def _create_cuda_graph_vec(self): - var = core.VarBase(core.VarDesc.VarType.FP32, [], "cuda_graph", - core.VarDesc.VarType.RAW, True) + var = core.VarBase( + core.VarDesc.VarType.FP32, + [], + "cuda_graph", + core.VarDesc.VarType.RAW, + True, + ) var.stop_gradient = True return var @@ -788,8 +903,9 @@ class PartialProgramLayer: return main_program.clone(for_test=True) def _is_no_value(self, var): - if isinstance(var, - (core.VarBase, core.eager.Tensor)) and var.shape == [1]: + if isinstance(var, (core.VarBase, core.eager.Tensor)) and var.shape == [ + 1 + ]: # NOTE: .numpy() will insert MemcpySync operation, it hits performance. if var.numpy()[0] == RETURN_NO_VALUE_MAGIC_NUM: return True @@ -805,13 +921,14 @@ class PartialProgramLayer: return out_vars elif isinstance(out_vars, (tuple, list)): if isinstance(out_vars, tuple): - res = tuple(var for var in out_vars - if not self._is_no_value(var)) + res = tuple( + var for var in out_vars if not self._is_no_value(var) + ) else: # isinstance(out_vars, list) res = [var for var in out_vars if not self._is_no_value(var)] - has_removed = (len(out_vars) > len(res)) + has_removed = len(out_vars) > len(res) # len(out_vars) > len(res) means we have removed var. This is # preventing out_vars is empty or just one element at the beginning if len(res) == 0 and has_removed: @@ -860,15 +977,18 @@ class PartialProgramLayer: if not isinstance(self._params, (list, tuple)): raise TypeError( "Type of self._params in PartialProgramLayer should be list or tuple, but received %s." - % type(self._params)) + % type(self._params) + ) param_and_buffer_names_set = set() for i, var in enumerate(self._params): # self._params constains parameters and buffers with persistable=True. if not isinstance(var, (core.VarBase, core.eager.Tensor)): raise TypeError( - 'Type of self._params[{}] in PartialProgramLayer should be Parameter or Variable, but received {}.' - .format(i, type(var))) + 'Type of self._params[{}] in PartialProgramLayer should be Parameter or Variable, but received {}.'.format( + i, type(var) + ) + ) param_and_buffer_names_set.add(var.name) for block in main_program.blocks: @@ -882,7 +1002,8 @@ class PartialProgramLayer: "\n\tRevise suggestion: " "\n\t\t1. Please ensure all your sublayers are inheritted from nn.Layer." "\n\t\t2. Please use nn.ParameterList and nn.LayerList as container instead of using a native Python container such as List" - % name) + % name + ) def _valid_vars(self, vars): """ @@ -899,13 +1020,23 @@ def _create_fake_var(): """ if not framework._in_eager_mode_: return [ - core.VarBase(core.VarDesc.VarType.FP32, [], "Fake_var", - core.VarDesc.VarType.RAW, False) + core.VarBase( + core.VarDesc.VarType.FP32, + [], + "Fake_var", + core.VarDesc.VarType.RAW, + False, + ) ] else: return [ - core.eager.Tensor(core.VarDesc.VarType.FP32, [], "Fake_var", - core.VarDesc.VarType.RAW, False) + core.eager.Tensor( + core.VarDesc.VarType.FP32, + [], + "Fake_var", + core.VarDesc.VarType.RAW, + False, + ) ] @@ -914,23 +1045,27 @@ def partial_program_from(concrete_program): if inputs and isinstance(inputs[0], layers.Layer): inputs = inputs[1:] - return PartialProgramLayer(concrete_program.main_program, inputs, - concrete_program.outputs, - concrete_program.parameters, - **concrete_program.kwargs) + return PartialProgramLayer( + concrete_program.main_program, + inputs, + concrete_program.outputs, + concrete_program.parameters, + **concrete_program.kwargs + ) @switch_to_static_graph -def add_build_strategy_for(program, - start_op_index, - end_op_index, - build_strategy=None): - if (start_op_index < end_op_index): +def add_build_strategy_for( + program, start_op_index, end_op_index, build_strategy=None +): + if start_op_index < end_op_index: compiled_program = paddle.static.CompiledProgram( core.Graph(program.desc, start_op_index, end_op_index), - build_strategy=build_strategy) - compiled_program._compile(core.Scope(), - framework._current_expected_place()) + build_strategy=build_strategy, + ) + compiled_program._compile( + core.Scope(), framework._current_expected_place() + ) ir_graph = framework.IrGraph(compiled_program._graph) builded_program = ir_graph.to_program() if hasattr(compiled_program._program, 'lr_sheduler'): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py index 93a16322b0b..fdbd585a71b 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py @@ -14,8 +14,13 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, + StaticAnalysisVisitor, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class PrintTransformer(BaseTransformer): @@ -31,7 +36,8 @@ class PrintTransformer(BaseTransformer): self.root = wrapper_root.node self.static_analysis_visitor = StaticAnalysisVisitor(self.root) - self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( + self.node_to_wrapper_map = ( + self.static_analysis_visitor.get_node_to_wrapper_map() ) def transform(self): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py index ad327a7f2e7..a8a0cd71212 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py @@ -29,19 +29,36 @@ from paddle.fluid.dygraph.base import switch_to_static_graph from paddle.fluid.dygraph.dygraph_to_static import DygraphToStaticAst from paddle.fluid.dygraph.dygraph_to_static import error from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.origin_info import attach_origin_info -from paddle.fluid.dygraph.dygraph_to_static.origin_info import create_and_update_origin_info_map -from paddle.fluid.dygraph.dygraph_to_static.origin_info import update_op_callstack_with_origin_info -from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + attach_origin_info, +) +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + create_and_update_origin_info_map, +) +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + update_op_callstack_with_origin_info, +) +from paddle.fluid.dygraph.dygraph_to_static.partial_program import ( + partial_program_from, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import input_specs_compatible from paddle.fluid.dygraph.dygraph_to_static.utils import type_name from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap -from paddle.fluid.dygraph.dygraph_to_static.utils import make_hashable, ALREADY_D2S -from paddle.fluid.dygraph.dygraph_to_static.function_spec import FunctionSpec, _hash_spec_names -from paddle.fluid.dygraph.dygraph_to_static.function_spec import get_buffers, get_parameters +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + make_hashable, + ALREADY_D2S, +) +from paddle.fluid.dygraph.dygraph_to_static.function_spec import ( + FunctionSpec, + _hash_spec_names, +) +from paddle.fluid.dygraph.dygraph_to_static.function_spec import ( + get_buffers, + get_parameters, +) from paddle.fluid.wrapped_decorator import signature_safe_contextmanager __all__ = ['ProgramTranslator', 'convert_to_static'] @@ -145,13 +162,24 @@ class CacheKey(object): """ Cached key for ProgramCache. """ + __slots__ = [ - 'function_spec', 'input_args_with_spec', 'input_kwargs_with_spec', - 'class_instance', 'kwargs', '_spec_names_id' + 'function_spec', + 'input_args_with_spec', + 'input_kwargs_with_spec', + 'class_instance', + 'kwargs', + '_spec_names_id', ] - def __init__(self, function_spec, input_args_with_spec, - input_kwargs_with_spec, class_instance, **kwargs): + def __init__( + self, + function_spec, + input_args_with_spec, + input_kwargs_with_spec, + class_instance, + **kwargs + ): """ Initializes a cache key. @@ -168,8 +196,9 @@ class CacheKey(object): self.class_instance = class_instance # NOTE: `kwargs` is usually not considered as basic member for `__hash__` self.kwargs = kwargs - self._spec_names_id = _hash_spec_names(input_args_with_spec, - input_kwargs_with_spec) + self._spec_names_id = _hash_spec_names( + input_args_with_spec, input_kwargs_with_spec + ) @classmethod def from_func_and_args(cls, function_spec, args, kwargs, class_instance): @@ -187,22 +216,34 @@ class CacheKey(object): args = args[1:] # 2. convert tensor and numpy array into InputSpec _args, _kwargs = function_spec.unified_args_and_kwargs(args, kwargs) - input_args_with_spec, input_kwargs_with_spec = function_spec.args_to_input_spec( - _args, _kwargs) + ( + input_args_with_spec, + input_kwargs_with_spec, + ) = function_spec.args_to_input_spec(_args, _kwargs) # 3. check whether hit the cache or build a new program for the input arguments - return CacheKey(function_spec, input_args_with_spec, - input_kwargs_with_spec, class_instance) + return CacheKey( + function_spec, + input_args_with_spec, + input_kwargs_with_spec, + class_instance, + ) def __hash__(self): error_msg = "Arguments to a `@paddle.jit.to_static` must be a hashable Python objects (or nested structures of these types)." with_hook = self.kwargs.get("with_hook", False) is_train = self.kwargs.get("is_train", False) - return hash((id(self.function_spec), - make_hashable(self.input_args_with_spec, error_msg), - make_hashable(self.input_kwargs_with_spec, - error_msg), self._spec_names_id, - self.class_instance, with_hook, is_train)) + return hash( + ( + id(self.function_spec), + make_hashable(self.input_args_with_spec, error_msg), + make_hashable(self.input_kwargs_with_spec, error_msg), + self._spec_names_id, + self.class_instance, + with_hook, + is_train, + ) + ) def __eq__(self, other): return (type(self) is type(other)) and hash(self) == hash(other) @@ -212,8 +253,11 @@ class CacheKey(object): def __repr__(self): return "id(function_spec): {}, input_args_with_spec: {}, input_kwargs_with_spec: {}, class_instance: {}".format( - id(self.function_spec), self.input_args_with_spec, - self.input_kwargs_with_spec, self.class_instance) + id(self.function_spec), + self.input_args_with_spec, + self.input_kwargs_with_spec, + self.class_instance, + ) def unwrap_decorators(func): @@ -260,9 +304,11 @@ class StaticFunction(object): if not hasattr(self._class_instance, '_original_funcs'): raise TypeError( "When using 'to_static' to convert method of a class, " - "please ensure the class inherits from nn.Layer") + "please ensure the class inherits from nn.Layer" + ) self._class_instance._original_funcs[ - function.__name__] = self._dygraph_function + function.__name__ + ] = self._dygraph_function else: self._dygraph_function = function self._class_instance = None @@ -286,21 +332,29 @@ class StaticFunction(object): return self._property def train(self): - if isinstance(self._class_instance, - layers.Layer) and self._class_instance.training == False: + if ( + isinstance(self._class_instance, layers.Layer) + and self._class_instance.training == False + ): raise RuntimeError( "Failed to switch train mode. {} is a Layer's method, " "please use Layer.train() to switch train mode.".format( - self.dygraph_function)) + self.dygraph_function + ) + ) self._training = True def eval(self): - if isinstance(self._class_instance, - layers.Layer) and self._class_instance.training == True: + if ( + isinstance(self._class_instance, layers.Layer) + and self._class_instance.training == True + ): raise RuntimeError( "Failed to switch eval mode. {} is a Layer's method, " "please use Layer.eval() to switch eval mode.".format( - self.dygraph_function)) + self.dygraph_function + ) + ) self._training = False def __get__(self, instance, owner): @@ -338,8 +392,9 @@ class StaticFunction(object): return self._descriptor_cache[instance] def _clone(self): - return self.__class__(self._dygraph_function, self._input_spec, - **self._kwargs) + return self.__class__( + self._dygraph_function, self._input_spec, **self._kwargs + ) def __call__(self, *args, **kwargs): """ @@ -364,7 +419,8 @@ class StaticFunction(object): logging_utils.warn( "The decorator '@paddle.jit.to_static' does NOT work when setting ProgramTranslator.enable to False. " "We will just return dygraph output. If you would like to get static graph output, please call API " - "ProgramTranslator.enable(True)") + "ProgramTranslator.enable(True)" + ) return self._call_dygraph_function(*args, **kwargs) if not _non_static_mode(): @@ -372,21 +428,26 @@ class StaticFunction(object): "Failed to run the callable object {} decorated by '@paddle.jit.to_static', " "because it is NOT in dynamic mode. Please disable the static mode to enter dynamic mode with the " "following API: paddle.disable_static().".format( - self.dygraph_function)) + self.dygraph_function + ) + ) # 2. trace ops from dygraph layers and cache the generated program. args, kwargs = self._function_spec.unified_args_and_kwargs(args, kwargs) try: concrete_program, partial_program_layer = self.get_concrete_program( - *args, **kwargs, is_train=self._is_train_mode()) + *args, **kwargs, is_train=self._is_train_mode() + ) # 3. synchronize self.training attribute. if isinstance(self._class_instance, layers.Layer): partial_program_layer.training = self._class_instance.training else: partial_program_layer.training = self._training - partial_program_layer._cuda_graph_capture_mode = self._cuda_graph_capture_mode + partial_program_layer._cuda_graph_capture_mode = ( + self._cuda_graph_capture_mode + ) partial_program_layer._cuda_graph_pool_id = self._cuda_graph_pool_id # 4. return outputs. @@ -404,7 +465,8 @@ class StaticFunction(object): else: logging_utils.warn( "Please file an issue at 'https://github.com/PaddlePaddle/Paddle/issues'" - " if you can't handle this {} yourself.".format(type(e))) + " if you can't handle this {} yourself.".format(type(e)) + ) raise e def _is_train_mode(self): @@ -412,7 +474,8 @@ class StaticFunction(object): if not hasattr(self._class_instance, 'training'): raise TypeError( "When using 'to_static' to convert method of a class, " - "please ensure the class inherits from nn.Layer") + "please ensure the class inherits from nn.Layer" + ) return self._class_instance.training else: return self._training @@ -430,7 +493,8 @@ class StaticFunction(object): """ if self._class_instance is not None: dygraph_function = self._dygraph_function.__get__( - self._class_instance) + self._class_instance + ) else: dygraph_function = self._dygraph_function @@ -460,23 +524,30 @@ class StaticFunction(object): with_hook = kwargs.get("with_hook", False) is_train = kwargs.get("is_train", True) - if "is_train" in kwargs: kwargs.pop("is_train") - if "with_hook" in kwargs: kwargs.pop("with_hook") + if "is_train" in kwargs: + kwargs.pop("is_train") + if "with_hook" in kwargs: + kwargs.pop("with_hook") # 1. unify args/kwargs and replace Tensor with InputSpec if len(args) != len(self._function_spec.args_name): args, kwargs = self._function_spec.unified_args_and_kwargs( - args, kwargs) - input_args_with_spec, input_kwargs_with_spec = self._function_spec.args_to_input_spec( - args, kwargs) + args, kwargs + ) + ( + input_args_with_spec, + input_kwargs_with_spec, + ) = self._function_spec.args_to_input_spec(args, kwargs) # 2. generate cache key - cache_key = CacheKey(self._function_spec, - input_args_with_spec, - input_kwargs_with_spec, - self._class_instance, - **self._kwargs, - with_hook=with_hook, - is_train=is_train) + cache_key = CacheKey( + self._function_spec, + input_args_with_spec, + input_kwargs_with_spec, + self._class_instance, + **self._kwargs, + with_hook=with_hook, + is_train=is_train + ) # 3. check whether hit the cache or build a new program for the input arguments concrete_program, partial_program_layer = self._program_cache[cache_key] @@ -533,9 +604,9 @@ class StaticFunction(object): """ return self.concrete_program_specify_input_spec(input_spec=None) - def concrete_program_specify_input_spec(self, - input_spec=None, - with_hook=False): + def concrete_program_specify_input_spec( + self, input_spec=None, with_hook=False + ): """ Returns recent ConcreteProgram instance of decorated function while specifying input_spec. If the self._function_spec already has @@ -556,30 +627,37 @@ class StaticFunction(object): desired_input_spec = input_spec if self._function_spec.input_spec is not None: if input_spec is not None and not input_specs_compatible( - flatten(input_spec), - flatten(self._function_spec.input_spec)): + flatten(input_spec), flatten(self._function_spec.input_spec) + ): raise ValueError( - "The `input_spec`: {} used to construct concrete_program is conflict with the `input_spec`: {} in `@paddle.jit.to_static`" - .format(input_spec, self._function_spec.input_spec)) + "The `input_spec`: {} used to construct concrete_program is conflict with the `input_spec`: {} in `@paddle.jit.to_static`".format( + input_spec, self._function_spec.input_spec + ) + ) # NOTE(chenweihang): we should always translated program based on the `input_spec` # decorated on forward if it is valid desired_input_spec = self._function_spec.input_spec if input_spec is not None: logging_utils.warn( - "\n\nYou have specified `input_spec` both in function definition (higher priority) and `paddle.jit.save` (will be ignored.)\n\n\t Using: {}\n\n\t Ignore: {}\n" - .format(desired_input_spec, input_spec)) + "\n\nYou have specified `input_spec` both in function definition (higher priority) and `paddle.jit.save` (will be ignored.)\n\n\t Using: {}\n\n\t Ignore: {}\n".format( + desired_input_spec, input_spec + ) + ) - has_input_spec = (desired_input_spec is not None) + has_input_spec = desired_input_spec is not None if has_input_spec: concrete_program, _ = self.get_concrete_program( *desired_input_spec, with_hook=with_hook, - is_train=self._is_train_mode()) + is_train=self._is_train_mode() + ) return concrete_program else: raise ValueError( - "No valid transformed program for {}.\n\t Please specific `input_spec` in `@paddle.jit.to_static` or feed input tensor to call the decorated function at once.\n" - .format(self._function_spec)) + "No valid transformed program for {}.\n\t Please specific `input_spec` in `@paddle.jit.to_static` or feed input tensor to call the decorated function at once.\n".format( + self._function_spec + ) + ) elif with_hook: cache_key = self._program_cache._recent_cache_key cache_key.kwargs["with_hook"] = True @@ -589,11 +667,15 @@ class StaticFunction(object): # If more than one programs have been cached, return the recent converted program by default. elif cached_program_len > 1: logging_utils.warn( - "Current {} has more than one cached programs: {}, the last traced progam will be return by default." - .format(self._function_spec, cached_program_len)) + "Current {} has more than one cached programs: {}, the last traced progam will be return by default.".format( + self._function_spec, cached_program_len + ) + ) - cache_key, (concrete_program, - partial_layer) = self._program_cache.last() + cache_key, ( + concrete_program, + partial_layer, + ) = self._program_cache.last() return concrete_program def rollback(self): @@ -639,11 +721,15 @@ class StaticFunction(object): # only rollback sub-functions on path of top _dygraph_function func_name = self._dygraph_function.__name__ - assert func_name in self._class_instance._original_funcs, "Not Found function '{}' in class '{}'.".format( - func_name, self._class_instance.__name__) + assert ( + func_name in self._class_instance._original_funcs + ), "Not Found function '{}' in class '{}'.".format( + func_name, self._class_instance.__name__ + ) func = self._class_instance._original_funcs[func_name] - setattr(self._class_instance, func_name, - func.__get__(self._class_instance)) + setattr( + self._class_instance, func_name, func.__get__(self._class_instance) + ) for sublayer in self._class_instance.sublayers(include_self=False): rollback_impl(sublayer) @@ -686,12 +772,15 @@ class StaticFunction(object): net_name = type(self._class_instance).__name__ logging_utils.log( level=-1, - msg="Not recommend to deepcopy '{}' decorated with @to_static, it has side effect that will" \ - " rollback into original state before @to_static. Please deepcopy '{}' before applying @to_static." - .format(net_name, net_name)) + msg="Not recommend to deepcopy '{}' decorated with @to_static, it has side effect that will" + " rollback into original state before @to_static. Please deepcopy '{}' before applying @to_static.".format( + net_name, net_name + ), + ) self.rollback() - return self._dygraph_function.__get__(memo[id( - self._class_instance)]) + return self._dygraph_function.__get__( + memo[id(self._class_instance)] + ) else: return self._dygraph_function @@ -703,7 +792,8 @@ class StaticFunction(object): self._raise_when_property() concrete_program = self.concrete_program inputs = [ - var for var in flatten(concrete_program.inputs) + var + for var in flatten(concrete_program.inputs) if isinstance(var, framework.Variable) ] return inputs @@ -716,7 +806,8 @@ class StaticFunction(object): self._raise_when_property() concrete_program = self.concrete_program outputs = [ - var for var in flatten(concrete_program.outputs) + var + for var in flatten(concrete_program.outputs) if isinstance(var, framework.Variable) ] @@ -751,7 +842,9 @@ def _verify_init_in_dynamic_mode(class_instance): " `paddle.jit.to_static` is only available in dynamic mode. Please call `paddle.disable_static()` before " "initializing your Layer class `{}` . Because parameters of Layer class should be initialized firstly " "in dynamic mode while applying transformation.".format( - class_instance)) + class_instance + ) + ) class HookHelper(object): @@ -764,22 +857,25 @@ class HookHelper(object): self.func = func self.class_instance = class_instance self.with_hook = with_hook - self.need_apply_hook = with_hook and isinstance( - self.class_instance, layers.Layer) and getattr( - func, "__name__") == "forward" + self.need_apply_hook = ( + with_hook + and isinstance(self.class_instance, layers.Layer) + and getattr(func, "__name__") == "forward" + ) def apply_pre_hooks(self, inputs): """ Apply _forward_pre_hooks from outermost layer """ - if not self.need_apply_hook: return inputs + if not self.need_apply_hook: + return inputs inputs = inputs[1:] for forward_pre_hook in self.class_instance._forward_pre_hooks.values(): hook_result = forward_pre_hook(self.class_instance, inputs) if hook_result is not None: if not isinstance(hook_result, tuple): - hook_result = (hook_result, ) + hook_result = (hook_result,) inputs = hook_result return [self.class_instance] + list(inputs) @@ -788,13 +884,16 @@ class HookHelper(object): """ Apply _forward_post_hooks from outermost layer """ - if not self.need_apply_hook: return outputs + if not self.need_apply_hook: + return outputs inputs = inputs[1:] - for forward_post_hook in self.class_instance._forward_post_hooks.values( - ): - hook_result = forward_post_hook(self.class_instance, inputs, - outputs) + for ( + forward_post_hook + ) in self.class_instance._forward_post_hooks.values(): + hook_result = forward_post_hook( + self.class_instance, inputs, outputs + ) if hook_result is not None: outputs = hook_result @@ -805,18 +904,25 @@ class HookHelper(object): class ConcreteProgram(object): __slots__ = [ - 'inputs', 'outputs', 'main_program', "startup_program", "parameters", - "function", 'kwargs' + 'inputs', + 'outputs', + 'main_program', + "startup_program", + "parameters", + "function", + 'kwargs', ] - def __init__(self, - inputs, - outputs, - parameters, - function, - main_program, - startup_program=None, - **kwargs): + def __init__( + self, + inputs, + outputs, + parameters, + function, + main_program, + startup_program=None, + **kwargs + ): self.inputs = inputs self.outputs = outputs self.main_program = main_program @@ -827,8 +933,9 @@ class ConcreteProgram(object): @staticmethod @switch_to_static_graph - def from_func_spec(func_spec, input_spec, input_kwargs_spec, class_instance, - **kwargs): + def from_func_spec( + func_spec, input_spec, input_kwargs_spec, class_instance, **kwargs + ): """ Builds the main_program with specialized inputs and returns outputs of program as fetch_list. @@ -844,38 +951,44 @@ class ConcreteProgram(object): dygraph_function = func_spec.dygraph_function static_func = convert_to_static(dygraph_function) # apply pre\post hook for outermost layer - hook_helper = HookHelper(dygraph_function, class_instance, - kwargs.get("with_hook", False)) + hook_helper = HookHelper( + dygraph_function, class_instance, kwargs.get("with_hook", False) + ) main_program, startup_program = framework.Program(), framework.Program() # Note: The random seed should be synchronized into cached program # if set in `fluid.dygraph_guard` because some ops rely on it, such as # `fluid.layers.dropout`. main_program.random_seed = framework.default_main_program().random_seed - startup_program.random_seed = framework.default_startup_program( - ).random_seed + startup_program.random_seed = ( + framework.default_startup_program().random_seed + ) from paddle.fluid.dygraph.base import _switch_declarative_mode_guard_ + with framework.program_guard(main_program, startup_program): with _switch_declarative_mode_guard_(is_declarative=True): # 1. Adds `fluid.data` layers for input if needed static_inputs = func_spec.to_static_inputs_with_spec( - input_spec, main_program) + input_spec, main_program + ) _kwargs = func_spec.to_static_inputs_with_spec( - input_kwargs_spec, main_program) + input_kwargs_spec, main_program + ) if class_instance: - static_inputs = tuple([class_instance] + - list(static_inputs)) + static_inputs = tuple( + [class_instance] + list(static_inputs) + ) # 2. Gets all ParamBases and buffered VarBases in the function all_parameters_and_buffers = _extract_indeed_params_buffers( - class_instance) + class_instance + ) # 3. Builds program only once and returns the output Variables. - with param_guard(get_parameters( - class_instance, - False)), param_guard(get_buffers(class_instance, - False)): + with param_guard( + get_parameters(class_instance, False) + ), param_guard(get_buffers(class_instance, False)): try: # only for jit.save, do nothing while train and eval process inputs = hook_helper.apply_pre_hooks(static_inputs) @@ -893,20 +1006,24 @@ class ConcreteProgram(object): raise if outputs is not None: - need_wrap_into_list = not isinstance( - outputs, (tuple, list)) or len(outputs) == 1 + need_wrap_into_list = ( + not isinstance(outputs, (tuple, list)) + or len(outputs) == 1 + ) if need_wrap_into_list: outputs = [outputs] main_program = update_op_callstack_with_origin_info(main_program) - return ConcreteProgram(inputs=static_inputs, - outputs=outputs, - parameters=all_parameters_and_buffers, - function=dygraph_function, - main_program=main_program, - startup_program=startup_program, - **kwargs) + return ConcreteProgram( + inputs=static_inputs, + outputs=outputs, + parameters=all_parameters_and_buffers, + function=dygraph_function, + main_program=main_program, + startup_program=startup_program, + **kwargs + ) def _extract_indeed_params_buffers(class_instance): @@ -938,13 +1055,16 @@ class ProgramCache(object): input_spec=cache_key.input_args_with_spec, input_kwargs_spec=cache_key.input_kwargs_with_spec, class_instance=cache_key.class_instance, - **cache_key.kwargs) + **cache_key.kwargs + ) return concrete_program, partial_program_from(concrete_program) def __getitem__(self, item): if not isinstance(item, CacheKey): - raise ValueError('type(item) should be CacheKey, but received %s' % - type_name(item)) + raise ValueError( + 'type(item) should be CacheKey, but received %s' + % type_name(item) + ) item_id = hash(item) self._recent_cache_key = item self._recent_key = item_id @@ -955,16 +1075,19 @@ class ProgramCache(object): if current_tracing_count > MAX_TRACED_PROGRAM_COUNT: logging_utils.warn( "Current traced program number: {} > `max_tracing_count`:{}. Too much cached programs will bring expensive overhead. " - "The reason may be: (1) passing tensors with different shapes, (2) passing python objects instead of tensors." - .format(current_tracing_count, MAX_TRACED_PROGRAM_COUNT)) + "The reason may be: (1) passing tensors with different shapes, (2) passing python objects instead of tensors.".format( + current_tracing_count, MAX_TRACED_PROGRAM_COUNT + ) + ) return self._caches[item_id] def get_program(self, item): if not isinstance(item, CacheKey): raise ValueError( - "Input item's type should be FunctionSpec, but received %s" % - type_name(item)) + "Input item's type should be FunctionSpec, but received %s" + % type_name(item) + ) item_id = hash(item) if item_id not in self._caches: raise RuntimeError( @@ -973,8 +1096,9 @@ class ProgramCache(object): return self._caches[item_id] def last(self): - assert len( - self._caches) >= 1, "No valid cached program in ProgramCache." + assert ( + len(self._caches) >= 1 + ), "No valid cached program in ProgramCache." assert self._recent_key is not None return self._recent_key, self._caches[self._recent_key] @@ -1082,8 +1206,12 @@ class ProgramTranslator(object): print(func(x)) # [[0. 0.]] """ - check_type(enable_to_static, "enable_to_static", bool, - "ProgramTranslator.enable") + check_type( + enable_to_static, + "enable_to_static", + bool, + "ProgramTranslator.enable", + ) self.enable_to_static = enable_to_static def get_output(self, dygraph_func, *args, **kwargs): @@ -1137,8 +1265,11 @@ class ProgramTranslator(object): try: function_spec = FunctionSpec(dygraph_func) cache_key = CacheKey.from_func_and_args( - function_spec, args, kwargs, - getattr(dygraph_func, '__self__', None)) + function_spec, + args, + kwargs, + getattr(dygraph_func, '__self__', None), + ) _, partial_program_layer = self._program_cache[cache_key] if args and isinstance(args[0], layers.Layer): @@ -1162,7 +1293,8 @@ class ProgramTranslator(object): else: logging_utils.warn( "Please file an issue at 'https://github.com/PaddlePaddle/Paddle/issues'" - " if you can't handle this {} yourself.".format(type(e))) + " if you can't handle this {} yourself.".format(type(e)) + ) raise e def get_func(self, dygraph_func): @@ -1267,24 +1399,28 @@ class ProgramTranslator(object): function_spec = FunctionSpec(dygraph_func) cache_key = CacheKey.from_func_and_args( - function_spec, args, kwargs, getattr(dygraph_func, '__self__', - None)) + function_spec, args, kwargs, getattr(dygraph_func, '__self__', None) + ) concrete_program, partial_program_layer = self._program_cache[cache_key] # Note: concrete_program hold all input/output infos include non-Variable input_vars = [ - var for var in concrete_program.inputs + var + for var in concrete_program.inputs if isinstance(var, framework.Variable) ] output_vars = [ - var for var in concrete_program.outputs + var + for var in concrete_program.outputs if isinstance(var, framework.Variable) ] - return concrete_program.main_program, \ - concrete_program.startup_program, \ - input_vars, \ - output_vars + return ( + concrete_program.main_program, + concrete_program.startup_program, + input_vars, + output_vars, + ) def get_code(self, dygraph_func): """ diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py index 88277bc832f..80bebbf501e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py @@ -16,15 +16,23 @@ from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list -from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import ForToWhileTransformer -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_fill_constant_node +from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import ( + ForToWhileTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_fill_constant_node, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException from paddle.fluid.dygraph.dygraph_to_static.utils import ORIGI_INFO __all__ = [ - 'RETURN_NO_VALUE_MAGIC_NUM', 'RETURN_NO_VALUE_VAR_NAME', 'ReturnTransformer' + 'RETURN_NO_VALUE_MAGIC_NUM', + 'RETURN_NO_VALUE_VAR_NAME', + 'ReturnTransformer', ] # Constant for the name of the variable which stores the boolean state that we @@ -45,7 +53,7 @@ RETURN_VALUE_INIT_NAME = '__return_value_init' # should return. # Assign not support float64, use float32 value as magic number. -RETURN_NO_VALUE_MAGIC_NUM = 1.77113e+27 +RETURN_NO_VALUE_MAGIC_NUM = 1.77113e27 RETURN_NO_VALUE_VAR_NAME = "__no_value_return_var" @@ -91,7 +99,8 @@ class ReturnAnalysisVisitor(gast.NodeVisitor): def __init__(self, root_node): self.root = root_node assert isinstance( - self.root, gast.FunctionDef), "Input is not gast.FunctionDef node" + self.root, gast.FunctionDef + ), "Input is not gast.FunctionDef node" # the number of return statements self.count_return = 0 @@ -157,7 +166,8 @@ class SingleReturnTransformer(BaseTransformer): def __init__(self, root): self.root = root assert isinstance( - self.root, gast.FunctionDef), "Input is not gast.FunctionDef node" + self.root, gast.FunctionDef + ), "Input is not gast.FunctionDef node" self.ancestor_nodes = [] @@ -206,15 +216,18 @@ class SingleReturnTransformer(BaseTransformer): self.generic_visit(node) return node - def append_assign_to_return_node(self, value, parent_node_of_return, - return_name, assign_nodes): + def append_assign_to_return_node( + self, value, parent_node_of_return, return_name, assign_nodes + ): self.assert_parent_is_not_while(parent_node_of_return) assert value in [True, False], "value must be True or False." if isinstance(parent_node_of_return, gast.If): # Prepend control flow boolean nodes such as '__return@1 = True' node_str = "{} = _jst.create_bool_as_type({}, {})".format( return_name, - ast_to_source_code(parent_node_of_return.test).strip(), value) + ast_to_source_code(parent_node_of_return.test).strip(), + value, + ) assign_node = gast.parse(node_str).body[0] assign_nodes.append(assign_node) @@ -236,18 +249,26 @@ class SingleReturnTransformer(BaseTransformer): value_name = self.return_value_name if value_name is not None: node.body.append( - gast.Return(value=gast.Name(id=value_name, - ctx=gast.Load(), - annotation=None, - type_comment=None))) - assign_return_value_node = gast.Assign(targets=[ - gast.Name(id=value_name, - ctx=gast.Store(), - annotation=None, - type_comment=None) - ], - value=gast.Constant( - kind=None, value=None)) + gast.Return( + value=gast.Name( + id=value_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ) + ) + ) + assign_return_value_node = gast.Assign( + targets=[ + gast.Name( + id=value_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ) + ], + value=gast.Constant(kind=None, value=None), + ) node.body.insert(0, assign_return_value_node) # Prepend no value placeholders @@ -269,37 +290,52 @@ class SingleReturnTransformer(BaseTransformer): if index_in_list(branch_node, cur_node) != -1: if cur_node == node: self._replace_return_in_stmt_list( - branch_node, cur_node, return_name, - max_return_length, parent_node_of_return) + branch_node, + cur_node, + return_name, + max_return_length, + parent_node_of_return, + ) self._replace_after_node_to_if_in_stmt_list( - branch_node, cur_node, return_name, - parent_node_of_return) + branch_node, + cur_node, + return_name, + parent_node_of_return, + ) _deal_branches("body") _deal_branches("orelse") # If return node in while loop, add `not return_name` in gast.While.test if isinstance(ancestor, gast.While): - cond_var_node = gast.UnaryOp(op=gast.Not(), - operand=gast.Name( - id=return_name, - ctx=gast.Load(), - annotation=None, - type_comment=None)) + cond_var_node = gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=return_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ) ancestor.test = gast.BoolOp( - op=gast.And(), values=[ancestor.test, cond_var_node]) + op=gast.And(), values=[ancestor.test, cond_var_node] + ) continue # If return node in for loop, add `not return_name` in gast.While.test if isinstance(ancestor, gast.For): - cond_var_node = gast.UnaryOp(op=gast.Not(), - operand=gast.Name( - id=return_name, - ctx=gast.Load(), - annotation=None, - type_comment=None)) + cond_var_node = gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=return_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ) parent_node = self.ancestor_nodes[ancestor_index - 1] - for_to_while = ForToWhileTransformer(parent_node, ancestor, - cond_var_node) + for_to_while = ForToWhileTransformer( + parent_node, ancestor, cond_var_node + ) new_stmts = for_to_while.transform() while_node = new_stmts[-1] self.ancestor_nodes[ancestor_index] = while_node @@ -308,8 +344,14 @@ class SingleReturnTransformer(BaseTransformer): break # return_node is replaced so we shouldn't return here - def _replace_return_in_stmt_list(self, stmt_list, return_node, return_name, - max_return_length, parent_node_of_return): + def _replace_return_in_stmt_list( + self, + stmt_list, + return_node, + return_name, + max_return_length, + parent_node_of_return, + ): assert max_return_length >= 0, "Input illegal max_return_length" i = index_in_list(stmt_list, return_node) @@ -317,24 +359,31 @@ class SingleReturnTransformer(BaseTransformer): return False assign_nodes = [] - self.append_assign_to_return_node(True, parent_node_of_return, - return_name, assign_nodes) + self.append_assign_to_return_node( + True, parent_node_of_return, return_name, assign_nodes + ) return_length = get_return_size(return_node) # In this case we should NOT append RETURN_NO_VALUE placeholder if return_node.value is not None: if self.return_value_name is None: self.return_value_name = unique_name.generate( - RETURN_VALUE_PREFIX) + RETURN_VALUE_PREFIX + ) assign_nodes.append( - gast.Assign(targets=[ - gast.Name(id=self.return_value_name, - ctx=gast.Store(), - annotation=None, - type_comment=None) - ], - value=return_node.value)) + gast.Assign( + targets=[ + gast.Name( + id=self.return_value_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ) + ], + value=return_node.value, + ) + ) return_origin_info = getattr(return_node, ORIGI_INFO, None) setattr(assign_nodes[-1], ORIGI_INFO, return_origin_info) @@ -343,9 +392,9 @@ class SingleReturnTransformer(BaseTransformer): stmt_list[i:] = assign_nodes return True - def _replace_after_node_to_if_in_stmt_list(self, stmt_list, node, - return_name, - parent_node_of_return): + def _replace_after_node_to_if_in_stmt_list( + self, stmt_list, node, return_name, parent_node_of_return + ): i = index_in_list(stmt_list, node) if i < 0 or i >= len(stmt_list): return False @@ -353,20 +402,26 @@ class SingleReturnTransformer(BaseTransformer): # No need to add, we consider this as added successfully return True - if_stmt = gast.If(test=gast.UnaryOp(op=gast.Not(), - operand=gast.Name( - id=return_name, - ctx=gast.Store(), - annotation=None, - type_comment=None)), - body=stmt_list[i + 1:], - orelse=[]) - - stmt_list[i + 1:] = [if_stmt] + if_stmt = gast.If( + test=gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=return_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ), + ), + body=stmt_list[i + 1 :], + orelse=[], + ) + + stmt_list[i + 1 :] = [if_stmt] # Here assume that the parent node of return is gast.If assign_nodes = [] - self.append_assign_to_return_node(False, parent_node_of_return, - return_name, assign_nodes) + self.append_assign_to_return_node( + False, parent_node_of_return, return_name, assign_nodes + ) stmt_list[i:i] = assign_nodes return True diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py index 16723bea10e..bf07523e12d 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py @@ -14,7 +14,13 @@ from paddle.utils import gast from .logging_utils import warn -from .utils import is_paddle_api, is_dygraph_api, is_numpy_api, index_in_list, ast_to_source_code +from .utils import ( + is_paddle_api, + is_dygraph_api, + is_numpy_api, + index_in_list, + ast_to_source_code, +) __all__ = ['AstNodeWrapper', 'NodeVarType', 'StaticAnalysisVisitor'] @@ -26,6 +32,7 @@ class NodeVarType(object): tensor variable in if clause may lead to different conversion from dygraph to static graph. """ + ERROR = -1 # Returns when static analysis gets error UNKNOWN = 0 # Reserve for AST nodes have not known the type STATEMENT = 1 # For nodes representing statement (non-variable type) @@ -62,7 +69,7 @@ class NodeVarType(object): "int": INT, "float": FLOAT, "bool": BOOLEAN, - "str": STRING + "str": STRING, } @staticmethod @@ -76,9 +83,12 @@ class NodeVarType(object): return in_type1 supported_types = [ - NodeVarType.BOOLEAN, NodeVarType.INT, NodeVarType.FLOAT, - NodeVarType.NUMPY_NDARRAY, NodeVarType.TENSOR, - NodeVarType.PADDLE_RETURN_TYPES + NodeVarType.BOOLEAN, + NodeVarType.INT, + NodeVarType.FLOAT, + NodeVarType.NUMPY_NDARRAY, + NodeVarType.TENSOR, + NodeVarType.PADDLE_RETURN_TYPES, ] if in_type1 not in supported_types: @@ -122,14 +132,14 @@ class AstVarScope(object): AstVarScope is a class holding the map from current scope variable to its type. """ + SCOPE_TYPE_SCRIPT = 0 SCOPE_TYPE_FUNCTION = 1 SCOPE_TYPE_CLASS = 2 - def __init__(self, - scope_name='', - scope_type=SCOPE_TYPE_SCRIPT, - parent_scope=None): + def __init__( + self, scope_name='', scope_type=SCOPE_TYPE_SCRIPT, parent_scope=None + ): self.sub_scopes = [] self.name_to_id = {} self.id_to_type = {} @@ -158,8 +168,9 @@ class AstVarScope(object): num_id = self.cur_id self.cur_id += 1 self.name_to_id[var_name] = num_id - self.id_to_type[num_id] = node_var_type if isinstance( - node_var_type, set) else {node_var_type} + self.id_to_type[num_id] = ( + node_var_type if isinstance(node_var_type, set) else {node_var_type} + ) def get_var_type(self, var_name): if var_name in self.name_to_id: @@ -179,20 +190,24 @@ class AstVarEnv(object): self.cur_scope = AstVarScope() def enter_scope(self, scope_name, scope_type): - self.cur_scope = AstVarScope(scope_name, - scope_type, - parent_scope=self.cur_scope) + self.cur_scope = AstVarScope( + scope_name, scope_type, parent_scope=self.cur_scope + ) return self.cur_scope def exit_scope(self): - assert self.cur_scope.parent_scope is not None, "Call exit_scope in "\ + assert self.cur_scope.parent_scope is not None, ( + "Call exit_scope in " "AstVarEnv when current scope doesn't have parent scope." + ) self.cur_scope = self.cur_scope.parent_scope return self.cur_scope def get_parent_scope(self): - assert self.cur_scope.parent_scope is not None, "Call parent_scope in "\ + assert self.cur_scope.parent_scope is not None, ( + "Call parent_scope in " "AstVarEnv when current scope doesn't have parent scope." + ) return self.cur_scope.parent_scope def add_var_type(self, var_name, node_var_type): @@ -252,11 +267,13 @@ class StaticAnalysisVisitor(object): self.ancestor_wrappers.append(cur_wrapper) for child in gast.iter_child_nodes(node): if isinstance(child, gast.FunctionDef) or isinstance( - child, gast.AsyncFunctionDef): + child, gast.AsyncFunctionDef + ): # TODO: current version is function name mapping to its type # consider complex case involving parameters - self.var_env.enter_scope(child.name, - AstVarScope.SCOPE_TYPE_FUNCTION) + self.var_env.enter_scope( + child.name, AstVarScope.SCOPE_TYPE_FUNCTION + ) func_type = self.dfs_visit(child) self.var_env.exit_scope() else: @@ -284,8 +301,10 @@ class StaticAnalysisVisitor(object): return True def _get_constant_node_type(self, node): - assert isinstance(node, gast.Constant), \ - "Type of input node should be gast.Constant, but received %s" % type(node) + assert isinstance(node, gast.Constant), ( + "Type of input node should be gast.Constant, but received %s" + % type(node) + ) # singleton: None, True or False if node.value is None: return {NodeVarType.NONE} @@ -338,7 +357,8 @@ class StaticAnalysisVisitor(object): for sub_target in target.elts: if isinstance(sub_target, gast.Name): self.node_to_wrapper_map[ - sub_target].node_var_type = ret_type + sub_target + ].node_var_type = ret_type self.var_env.set_var_type(sub_target.id, ret_type) return ret_type @@ -349,9 +369,12 @@ class StaticAnalysisVisitor(object): # if annotation and value(Constant) are diffent type, we use value type if node.value: node_value_type = self.node_to_wrapper_map[ - node.value].node_var_type - if not (node_value_type - & {NodeVarType.UNKNOWN, NodeVarType.STATEMENT}): + node.value + ].node_var_type + if not ( + node_value_type + & {NodeVarType.UNKNOWN, NodeVarType.STATEMENT} + ): ret_type = node_value_type if isinstance(node.target, gast.Name): self.node_to_wrapper_map[node.target].node_var_type = ret_type @@ -365,8 +388,9 @@ class StaticAnalysisVisitor(object): return {NodeVarType.BOOLEAN} # If node is child of functionDef.arguments parent_node_wrapper = cur_wrapper.parent - if parent_node_wrapper and isinstance(parent_node_wrapper.node, - gast.arguments): + if parent_node_wrapper and isinstance( + parent_node_wrapper.node, gast.arguments + ): return self._get_func_argument_type(parent_node_wrapper, node) @@ -378,7 +402,10 @@ class StaticAnalysisVisitor(object): return {NodeVarType.NONE} return_type = self.node_to_wrapper_map[node.value].node_var_type - assert self.var_env.cur_scope.scope_type == AstVarScope.SCOPE_TYPE_FUNCTION, "Return at non-function scope" + assert ( + self.var_env.cur_scope.scope_type + == AstVarScope.SCOPE_TYPE_FUNCTION + ), "Return at non-function scope" func_name = self.var_env.cur_scope.scope_name parent_scope = self.var_env.get_parent_scope() parent_scope.add_var_type(func_name, return_type) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py index b11f18d2658..e7a882b28a2 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py @@ -15,8 +15,12 @@ from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class TensorShapeTransformer(BaseTransformer): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/typehint_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/typehint_transformer.py index f258b98b507..3fddc0bcf58 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/typehint_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/typehint_transformer.py @@ -15,9 +15,13 @@ from paddle.utils import gast import warnings -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static import utils -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class TypeHintTransformer(BaseTransformer): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py index 8e1950b21fc..8b4a7ac645e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py @@ -98,10 +98,18 @@ RE_PYMODULE = r'[a-zA-Z0-9_]+\.' # FullArgSpec is valid from Python3. Defined a Namedtuple to # to make it available in Python2. -FullArgSpec = collections.namedtuple('FullArgSpec', [ - 'args', 'varargs', 'varkw', 'defaults', 'kwonlyargs', 'kwonlydefaults', - 'annotations' -]) +FullArgSpec = collections.namedtuple( + 'FullArgSpec', + [ + 'args', + 'varargs', + 'varkw', + 'defaults', + 'kwonlyargs', + 'kwonlydefaults', + 'annotations', + ], +) def data_layer_not_check(name, shape, dtype='float32', lod_level=0): @@ -139,20 +147,26 @@ def data_layer_not_check(name, shape, dtype='float32', lod_level=0): if shape[i] is None: shape[i] = -1 - return helper.create_global_variable(name=name, - shape=shape, - dtype=dtype, - type=core.VarDesc.VarType.LOD_TENSOR, - stop_gradient=True, - lod_level=lod_level, - is_data=True, - need_check_feed=False) + return helper.create_global_variable( + name=name, + shape=shape, + dtype=dtype, + type=core.VarDesc.VarType.LOD_TENSOR, + stop_gradient=True, + lod_level=lod_level, + is_data=True, + need_check_feed=False, + ) def create_undefined_variable(): - from paddle.fluid.dygraph.dygraph_to_static.return_transformer import RETURN_NO_VALUE_MAGIC_NUM - var = data_layer_not_check(unique_name.generate("undefined_var"), [1], - "float64") + from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + RETURN_NO_VALUE_MAGIC_NUM, + ) + + var = data_layer_not_check( + unique_name.generate("undefined_var"), [1], "float64" + ) var.stop_gradient = False # the variable is created in block(0), we append assign in block(0) either. helper = LayerHelper('create_undefined_variable', **locals()) @@ -164,17 +178,16 @@ def create_undefined_variable(): class UndefinedVar: - def __init__(self, name): self.name = name def check(self): raise UnboundLocalError( - "local variable '{}' should be created before using it.") + "local variable '{}' should be created before using it." + ) class Dygraph2StaticException(Exception): - def __init__(self, message): super().__init__(message) @@ -191,13 +204,15 @@ def getfullargspec(target): return inspect.getfullargspec(target) else: argspec = inspect.getargspec(target) - return FullArgSpec(args=argspec.args, - varargs=argspec.varargs, - varkw=argspec.keywords, - defaults=argspec.defaults, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}) + return FullArgSpec( + args=argspec.args, + varargs=argspec.varargs, + varkw=argspec.keywords, + defaults=argspec.defaults, + kwonlyargs=[], + kwonlydefaults=None, + annotations={}, + ) def parse_arg_and_kwargs(function): @@ -214,7 +229,7 @@ def parse_arg_and_kwargs(function): default_values = fullargspec.defaults if default_values: assert len(default_values) <= len(arg_names) - default_kwarg_names = arg_names[-len(default_values):] + default_kwarg_names = arg_names[-len(default_values) :] default_kwargs = dict(zip(default_kwarg_names, default_values)) return arg_names, default_kwargs @@ -288,8 +303,9 @@ def is_api_in_module(node, module_prefix): from paddle.fluid.dygraph import to_variable from paddle import to_tensor - return eval("_is_api_in_module_helper({}, '{}')".format( - func_str, module_prefix)) + return eval( + "_is_api_in_module_helper({}, '{}')".format(func_str, module_prefix) + ) except Exception: return False @@ -320,12 +336,15 @@ def is_numpy_api(node): func_str = astor.to_source(gast.gast_to_ast(node.func)) try: import numpy as np - module_result = eval("_is_api_in_module_helper({}, '{}')".format( - func_str, "numpy")) + + module_result = eval( + "_is_api_in_module_helper({}, '{}')".format(func_str, "numpy") + ) # BUG: np.random.uniform doesn't have module and cannot be analyzed # TODO: find a better way - return module_result or (func_str.startswith("numpy.") - or func_str.startswith("np.")) + return module_result or ( + func_str.startswith("numpy.") or func_str.startswith("np.") + ) except Exception: return False @@ -334,6 +353,7 @@ def _delete_keywords_from(node): assert isinstance(node, gast.Call) func_src = astor.to_source(gast.gast_to_ast(node.func)) import paddle.fluid as fluid + full_args = eval("inspect.getargspec({})".format(func_src)) full_args_name = full_args[0] @@ -347,7 +367,8 @@ def to_static_api(dygraph_class): else: raise NotImplementedError( "Paddle dygraph API {} cannot be converted " - "to static graph at present.".format(dygraph_class)) + "to static graph at present.".format(dygraph_class) + ) def _add_keywords_to(node, dygraph_api_name): @@ -358,8 +379,10 @@ def _add_keywords_to(node, dygraph_api_name): ast_keyword.arg = "size" node.keywords.append( - gast.keyword(arg="num_flatten_dims", - value=gast.Constant(value=-1, kind=None))) + gast.keyword( + arg="num_flatten_dims", value=gast.Constant(value=-1, kind=None) + ) + ) if dygraph_api_name == "BilinearTensorProduct": for ast_keyword in node.keywords: @@ -378,15 +401,17 @@ def to_static_ast(node, class_node): assert isinstance(class_node, gast.Call) static_api = to_static_api(class_node.func.attr) - node.func = gast.Attribute(attr=static_api, - ctx=gast.Load(), - value=gast.Attribute(attr='layers', - ctx=gast.Load(), - value=gast.Name( - ctx=gast.Load(), - id='fluid', - annotation=None, - type_comment=None))) + node.func = gast.Attribute( + attr=static_api, + ctx=gast.Load(), + value=gast.Attribute( + attr='layers', + ctx=gast.Load(), + value=gast.Name( + ctx=gast.Load(), id='fluid', annotation=None, type_comment=None + ), + ), + ) update_args_of_func(node, class_node, 'forward') @@ -409,10 +434,13 @@ def update_args_of_func(node, dygraph_node, method_name): class_src = astor.to_source(gast.gast_to_ast(dygraph_node.func)) import paddle.fluid as fluid + if method_name == "__init__" or eval( - "issubclass({}, fluid.dygraph.Layer)".format(class_src)): - full_args = eval("inspect.getargspec({}.{})".format( - class_src, method_name)) + "issubclass({}, fluid.dygraph.Layer)".format(class_src) + ): + full_args = eval( + "inspect.getargspec({}.{})".format(class_src, method_name) + ) full_args_name = [ arg_name for arg_name in full_args[0] if arg_name != "self" ] @@ -427,21 +455,24 @@ def update_args_of_func(node, dygraph_node, method_name): def create_api_shape_node(tensor_shape_node): - assert isinstance(tensor_shape_node, - (gast.Name, gast.Attribute, gast.Subscript)) + assert isinstance( + tensor_shape_node, (gast.Name, gast.Attribute, gast.Subscript) + ) if isinstance(tensor_shape_node, gast.Name): api_shape_node = gast.Call( func=gast.parse('paddle.shape').body[0].value, args=[tensor_shape_node], - keywords=[]) + keywords=[], + ) return api_shape_node if isinstance(tensor_shape_node, gast.Attribute): api_shape_node = gast.Call( func=gast.parse('paddle.shape').body[0].value, args=[tensor_shape_node.value], - keywords=[]) + keywords=[], + ) return api_shape_node if isinstance(tensor_shape_node, gast.Subscript): @@ -451,14 +482,15 @@ def create_api_shape_node(tensor_shape_node): def get_constant_variable_node(name, value, shape=[1], dtype='int64'): - return gast.parse('%s = paddle.full(%s, "%s", %s)' % - (name, str(shape), str(value), dtype)) + return gast.parse( + '%s = paddle.full(%s, "%s", %s)' % (name, str(shape), str(value), dtype) + ) def get_attribute_full_name(node): assert isinstance( - node, - gast.Attribute), "Input non-Attribute node to get attribute full name" + node, gast.Attribute + ), "Input non-Attribute node to get attribute full name" return astor.to_source(gast.gast_to_ast(node)).strip() @@ -476,15 +508,15 @@ def generate_name_node(name_ids, ctx=gast.Load(), gen_tuple_if_single=False): name_ids = [name_ids] if not isinstance(name_ids, (list, tuple, set)): raise TypeError( - 'name_ids must be list or tuple or set, but received %s' % - type(type(name_ids))) + 'name_ids must be list or tuple or set, but received %s' + % type(type(name_ids)) + ) def create_node_for_name(name): if '.' not in name: - return gast.Name(id=name, - ctx=ctx, - annotation=None, - type_comment=None) + return gast.Name( + id=name, ctx=ctx, annotation=None, type_comment=None + ) return gast.parse(name).body[0].value gast_names = [create_node_for_name(name_id) for name_id in name_ids] @@ -506,12 +538,14 @@ def create_funcDef_node(nodes, name, input_args, return_name_ids): nodes.append(gast.Return(value=generate_name_node(return_name_ids))) else: nodes.append(gast.Return(value=None)) - func_def_node = gast.FunctionDef(name=name, - args=input_args, - body=nodes, - decorator_list=[], - returns=None, - type_comment=None) + func_def_node = gast.FunctionDef( + name=name, + args=input_args, + body=nodes, + decorator_list=[], + returns=None, + type_comment=None, + ) return func_def_node @@ -571,12 +605,14 @@ def ast_to_func(ast_root, dyfunc, delete_on_exit=True): source = ast_to_source_code(ast_root) source = _inject_import_statements() + source temp_dir = get_temp_dir() - f = tempfile.NamedTemporaryFile(mode='w', - prefix=func_prefix(dyfunc), - suffix='.py', - delete=False, - dir=temp_dir, - encoding='utf-8') + f = tempfile.NamedTemporaryFile( + mode='w', + prefix=func_prefix(dyfunc), + suffix='.py', + delete=False, + dir=temp_dir, + encoding='utf-8', + ) with f: module_name = os.path.basename(f.name[:-3]) f.write(source) @@ -598,8 +634,9 @@ def ast_to_func(ast_root, dyfunc, delete_on_exit=True): callable_func = getattr(module, func_name) else: raise ValueError( - 'Function: %s doesn\'t exist in the Module transformed from AST.' % - func_name) + 'Function: %s doesn\'t exist in the Module transformed from AST.' + % func_name + ) # After transform dygraph function into callable_func saved in tmp file, # it lost the global variables from imported statements or defined in source file. # Recovers the necessary variables by `__globals__`. @@ -610,10 +647,14 @@ def ast_to_func(ast_root, dyfunc, delete_on_exit=True): def _inject_import_statements(): import_statements = [ - "import paddle", "from paddle import Tensor", - "import paddle.fluid as fluid", "import paddle.jit.dy2static as _jst", - "from typing import *", "import numpy as np", "import warnings", - "warnings.filterwarnings('ignore', category=DeprecationWarning)" + "import paddle", + "from paddle import Tensor", + "import paddle.fluid as fluid", + "import paddle.jit.dy2static as _jst", + "from typing import *", + "import numpy as np", + "import warnings", + "warnings.filterwarnings('ignore', category=DeprecationWarning)", ] return '\n'.join(import_statements) + '\n' @@ -636,8 +677,10 @@ def func_to_source_code(function, dedent=True): """ if not (inspect.isfunction(function) or inspect.ismethod(function)): raise TypeError( - "The type of 'function' should be a function or method, but received {}." - .format(type(function).__name__)) + "The type of 'function' should be a function or method, but received {}.".format( + type(function).__name__ + ) + ) source_code_list, _ = inspect.getsourcelines(function) # Replace comments with blank lines so that error messages are not misplaced source_code_list = [ @@ -657,8 +700,9 @@ def ast_to_source_code(ast_node): """ if not isinstance(ast_node, (gast.AST, ast.AST)): raise TypeError( - "Type of ast_root should be gast.AST or ast.AST, but received %s." % - type(ast_node)) + "Type of ast_root should be gast.AST or ast.AST, but received %s." + % type(ast_node) + ) if isinstance(ast_node, gast.AST): ast_node = gast.gast_to_ast(ast_node) @@ -674,8 +718,17 @@ def is_candidate_node(node): """ Nodes with specified type will be dependent on tensor. """ - is_compare_node = isinstance(node, (gast.Compare, gast.BoolOp, gast.UnaryOp, - gast.For, gast.If, gast.While)) + is_compare_node = isinstance( + node, + ( + gast.Compare, + gast.BoolOp, + gast.UnaryOp, + gast.For, + gast.If, + gast.While, + ), + ) # TODO(Aurelius84): `.numpy()` may be an customized function, # and should consider a more elegant way to solve this problem. has_numpy_attr = ".numpy()" in ast_to_source_code(node) @@ -691,9 +744,9 @@ def compare_with_none(node): # node.comparators is a list. if isinstance(child, list): child = child[0] - if (isinstance(child, gast.Constant) - and child.value is None) or (isinstance(child, gast.Name) - and child.id == 'None'): + if (isinstance(child, gast.Constant) and child.value is None) or ( + isinstance(child, gast.Name) and child.id == 'None' + ): return True return False @@ -728,20 +781,22 @@ class IsControlFlowVisitor(gast.NodeVisitor): because reshape_op may be called before this statement. """ - def __init__(self, - ast_node, - static_analysis_visitor=None, - node_var_type_map=None): + def __init__( + self, ast_node, static_analysis_visitor=None, node_var_type_map=None + ): assert isinstance( ast_node, gast.AST ), "Type of input node should be gast.AST, but received %s." % type( - ast_node) + ast_node + ) self.ast_root = ast_node if static_analysis_visitor is None: from .static_analysis import StaticAnalysisVisitor + static_analysis_visitor = StaticAnalysisVisitor(ast_node) self.static_analysis_visitor = static_analysis_visitor - self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( + self.node_to_wrapper_map = ( + self.static_analysis_visitor.get_node_to_wrapper_map() ) self.node_var_type_map = node_var_type_map @@ -770,7 +825,10 @@ class IsControlFlowVisitor(gast.NodeVisitor): if isinstance(node.iter, gast.Call): # for in range(var[0]|var.numpy()[0]) or for in enumerate(var|var.numpy()) if isinstance(node.iter.func, gast.Name): - if node.iter.func.id == "range" or node.iter.func.id == "enumerate": + if ( + node.iter.func.id == "range" + or node.iter.func.id == "enumerate" + ): for arg in node.iter.args: self.visit(arg) else: @@ -869,7 +927,9 @@ class IsControlFlowVisitor(gast.NodeVisitor): return node def _is_node_with_tensor(self, node, name_id): - from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType + from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + NodeVarType, + ) # Look up the node_var_type_map by name_id. if self.node_var_type_map: @@ -899,7 +959,7 @@ def unwrap(func): return hasattr(f, '__wrapped__') unwrapped_f = func - while (_is_wrapped(unwrapped_f)): + while _is_wrapped(unwrapped_f): unwrapped_f = unwrapped_f.__wrapped__ return unwrapped_f @@ -923,10 +983,12 @@ def input_specs_compatible(src_input_specs, desired_input_specs): if spec not in desired_input_specs: return False else: - for (src_spec, desired_spec) in zip(src_input_specs, - desired_input_specs): + for (src_spec, desired_spec) in zip( + src_input_specs, desired_input_specs + ): if isinstance(src_spec, paddle.static.InputSpec) or isinstance( - desired_spec, paddle.static.InputSpec): + desired_spec, paddle.static.InputSpec + ): if not _compatible_tensor_spec(src_spec, desired_spec): return False else: @@ -986,15 +1048,14 @@ def _compatible_non_tensor_spec(src_spec, desired_spec): class NameScope: - def __init__(self): """ - A NameScope is a object which manager all the variable names. - only FunctionDef and Controlflow node will have a namescope property. + A NameScope is a object which manager all the variable names. + only FunctionDef and Controlflow node will have a namescope property. - type can be "function" and "controlflow" + type can be "function" and "controlflow" - we don't analyze the read only variable because they don't affect the analysis. + we don't analyze the read only variable because they don't affect the analysis. """ self.globals = set() self.nonlocals = set() @@ -1010,8 +1071,8 @@ class NameScope: self.father = father def existed_vars(self): - """ vars existing in current scope. - they must not contain qualified names. + """vars existing in current scope. + they must not contain qualified names. """ local_vars = self.w_vars - self.globals - self.nonlocals - self.args return set(filter(lambda x: '.' not in x, local_vars)) @@ -1040,18 +1101,20 @@ class NameScope: f"Find variable `{var}` defined in global scope" f" and call `{var}.append() or {var}.pop()`" f", which will be ignored and never be transfered into" - f" tensor array.") + f" tensor array." + ) else: non_global_push_pop_names.append(var) return set(non_global_push_pop_names) def control_flow_vars(self): valid_names = self.w_vars - tmp = self.father.global_vars & valid_names, + tmp = (self.father.global_vars & valid_names,) return {"global": tmp, "nonlocal": self.w_vars - tmp} def _is_simple_name(self, name): - if '.' in name or '[' in name: return False + if '.' in name or '[' in name: + return False return True def is_global_var(self, name): @@ -1062,11 +1125,14 @@ class NameScope: Only valid after FunctionNameLivenessAnalysis visitor. """ assert self._is_simple_name( - name), "is_global_var accept a simple name, but get `{name}`." + name + ), "is_global_var accept a simple name, but get `{name}`." ancestor = self while ancestor is not None: - if name in ancestor.globals: return True - if name in (ancestor.nonlocals | ancestor.w_vars): return False + if name in ancestor.globals: + return True + if name in (ancestor.nonlocals | ancestor.w_vars): + return False ancestor = ancestor.father return True @@ -1082,46 +1148,46 @@ class NameScope: class FunctionNameLivenessAnalysis(gast.NodeVisitor): - """ analyze the liveness of a function. - - every variables stored in this scope will be collected, - in addition with global/nonlocal information and - push_pop information. - - 1. global variable is stored in node.var_globals. - 2. nonlocal variable is stored in node.var_nonlocals. - 3. arguments is stored in node.var_args. - 4. if a variable's push and pop attribute is called, - it will be collected in push_pop_vars. They are - used for transformation to tensor_array. - NOTE: push_pop_vars **may not** in w_vars. - a.push(0) don't modify the variable a, but the content - of a. - - For example: - - def func(*args, **kargs): - a = 12 - global i,j - nonlocal x,y - print(a) - i = k - b = [] - c = [1,2,3] - for m in range(10): - q = 12 - b.push(1) - c.pop() - - After this visitor we have: - # node is the FunctionDef node with name: "func" - node.pd_scope = NameScope( - globals = ['i', 'j'], - nonlocals = ['x', 'y'], - args = ['args', 'kargs'], - wr_vars = ['a', 'i', 'q', 'm', 'c', 'b'] - push_pop_vars = ['b', 'c'] - ) + """analyze the liveness of a function. + + every variables stored in this scope will be collected, + in addition with global/nonlocal information and + push_pop information. + + 1. global variable is stored in node.var_globals. + 2. nonlocal variable is stored in node.var_nonlocals. + 3. arguments is stored in node.var_args. + 4. if a variable's push and pop attribute is called, + it will be collected in push_pop_vars. They are + used for transformation to tensor_array. + NOTE: push_pop_vars **may not** in w_vars. + a.push(0) don't modify the variable a, but the content + of a. + + For example: + + def func(*args, **kargs): + a = 12 + global i,j + nonlocal x,y + print(a) + i = k + b = [] + c = [1,2,3] + for m in range(10): + q = 12 + b.push(1) + c.pop() + + After this visitor we have: + # node is the FunctionDef node with name: "func" + node.pd_scope = NameScope( + globals = ['i', 'j'], + nonlocals = ['x', 'y'], + args = ['args', 'kargs'], + wr_vars = ['a', 'i', 'q', 'm', 'c', 'b'] + push_pop_vars = ['b', 'c'] + ) """ def __init__(self, root_node): @@ -1141,25 +1207,26 @@ class FunctionNameLivenessAnalysis(gast.NodeVisitor): return self._get_name_scope(self.scope_node_stack[-1]) def _father_name_scope(self): - if len(self.scope_node_stack) == 1: return None + if len(self.scope_node_stack) == 1: + return None return self._get_name_scope(self.scope_node_stack[-2]) def _nearest_function_scope(self): - if len(self.scope_node_stack) == 1: return None + if len(self.scope_node_stack) == 1: + return None for node in self.scope_node_stack[-2::-1]: if isinstance(node, gast.FunctionDef): return self._get_name_scope(node) def visit_ListComp(self, node): - """ [ i for i in range(10) ] - In this case, `i` will not created in FunctionScope. - We don't collect `i` by not calling generic_visit. + """[ i for i in range(10) ] + In this case, `i` will not created in FunctionScope. + We don't collect `i` by not calling generic_visit. """ pass def visit_DictComp(self, node): - """ the same as ListComp. - """ + """the same as ListComp.""" pass def visit_Name(self, node): @@ -1169,62 +1236,86 @@ class FunctionNameLivenessAnalysis(gast.NodeVisitor): self._current_name_scope().w_vars.add(node.id) def visit_FunctionDef(self, node): - def pre_func(): self._current_name_scope().args |= set( - self._get_argument_names(node)) + self._get_argument_names(node) + ) def post_func(): - """ NOTE: why we need merge w_vars and push_pop_vars here ? - because we do ifelse_transformer after loop_transformer. Loops will changed into functioons. but we know this function will be called in if. so we add w_vars to father function scope. + """NOTE: why we need merge w_vars and push_pop_vars here ? + because we do ifelse_transformer after loop_transformer. Loops will changed into functioons. but we know this function will be called in if. so we add w_vars to father function scope. """ - from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import WHILE_CONDITION_PREFIX, WHILE_BODY_PREFIX, FOR_CONDITION_PREFIX, FOR_BODY_PREFIX - from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import TRUE_FUNC_PREFIX, FALSE_FUNC_PREFIX + from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import ( + WHILE_CONDITION_PREFIX, + WHILE_BODY_PREFIX, + FOR_CONDITION_PREFIX, + FOR_BODY_PREFIX, + ) + from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import ( + TRUE_FUNC_PREFIX, + FALSE_FUNC_PREFIX, + ) + control_flow_function_def = [ - WHILE_BODY_PREFIX, WHILE_BODY_PREFIX, FOR_CONDITION_PREFIX, - FOR_BODY_PREFIX, TRUE_FUNC_PREFIX, FALSE_FUNC_PREFIX + WHILE_BODY_PREFIX, + WHILE_BODY_PREFIX, + FOR_CONDITION_PREFIX, + FOR_BODY_PREFIX, + TRUE_FUNC_PREFIX, + FALSE_FUNC_PREFIX, ] def is_control_flow_def_node(): for prefix in control_flow_function_def: - if node.name.startswith(prefix): return True + if node.name.startswith(prefix): + return True return False if self._father_name_scope() and is_control_flow_def_node(): - self._father_name_scope().w_vars |= self._current_name_scope( - ).w_vars - self._father_name_scope( - ).push_pop_vars |= self._current_name_scope().push_pop_vars + self._father_name_scope().w_vars |= ( + self._current_name_scope().w_vars + ) + self._father_name_scope().push_pop_vars |= ( + self._current_name_scope().push_pop_vars + ) self._visit_scope_node(node, pre_func, post_func) def _visit_scope_node(self, node, pre_func, post_func): - """ scope node main visit logic. - pre_func and post_func is callbacks + """scope node main visit logic. + pre_func and post_func is callbacks """ self._reset_name_scope(node) self.scope_node_stack.append(node) self._current_name_scope().set_father(self._nearest_function_scope()) - if pre_func: pre_func() + if pre_func: + pre_func() self.generic_visit(node) - if post_func: post_func() + if post_func: + post_func() self.scope_node_stack.pop() def _visit_controlflow_node(self, node): - def post_func(): self._father_name_scope().merge_from(self._current_name_scope()) self._nearest_function_scope().merge_from( - self._current_name_scope()) - self._current_name_scope().created = self._nearest_function_scope( - ).existed_vars() - node.before_created + self._current_name_scope() + ) + self._current_name_scope().created = ( + self._nearest_function_scope().existed_vars() + - node.before_created + ) # gather created vars into father and used in CreateUndefinedVarTransform - self._nearest_function_scope().created |= self._current_name_scope( - ).created + self._nearest_function_scope().created |= ( + self._current_name_scope().created + ) def pre_func(): - setattr(node, "before_created", - self._nearest_function_scope().existed_vars()) + setattr( + node, + "before_created", + self._nearest_function_scope().existed_vars(), + ) self._visit_scope_node(node, pre_func, post_func) @@ -1262,12 +1353,13 @@ class FunctionNameLivenessAnalysis(gast.NodeVisitor): self._current_name_scope().push_pop_vars.add(name) def _get_argument_names(self, node): - """ get all arguments name in the functiondef node. - this node is local to the function and shouldn't - be created. + """get all arguments name in the functiondef node. + this node is local to the function and shouldn't + be created. """ assert isinstance( - node, gast.FunctionDef), "Input node is not function define node" + node, gast.FunctionDef + ), "Input node is not function define node" names = [a for a in node.args.args] names.append(node.args.vararg) names.append(node.args.kwarg) @@ -1288,7 +1380,9 @@ def create_get_args_node(names): func_def = """ def {func_name}(): return - """.format(func_name=unique_name.generate(GET_ARGS_FUNC_PREFIX)) + """.format( + func_name=unique_name.generate(GET_ARGS_FUNC_PREFIX) + ) return gast.parse(textwrap.dedent(func_def)).body[0] assert isinstance(names, (list, tuple)) @@ -1307,7 +1401,8 @@ def create_get_args_node(names): func_def = template.format( func_name=unique_name.generate(GET_ARGS_FUNC_PREFIX), nonlocal_vars=nonlocal_vars, - vars=",".join(names)) + vars=",".join(names), + ) return gast.parse(textwrap.dedent(func_def)).body[0] @@ -1324,8 +1419,9 @@ def create_set_args_node(names): func_def = """ def {func_name}({args}): pass - """.format(func_name=unique_name.generate(SET_ARGS_FUNC_PREFIX), - args=ARGS_NAME) + """.format( + func_name=unique_name.generate(SET_ARGS_FUNC_PREFIX), args=ARGS_NAME + ) return gast.parse(textwrap.dedent(func_def)).body[0] assert isinstance(names, (list, tuple)) @@ -1345,7 +1441,8 @@ def create_set_args_node(names): func_name=unique_name.generate(SET_ARGS_FUNC_PREFIX), args=ARGS_NAME, nonlocal_vars=nonlocal_vars, - vars=",".join(names)) + vars=",".join(names), + ) return gast.parse(textwrap.dedent(func_def)).body[0] @@ -1355,8 +1452,8 @@ def create_nonlocal_stmt_nodes(names): mapped = list(filter(lambda n: '.' not in n, names)) mapped = list(filter(lambda n: '[' not in n, mapped)) names = sorted( - mapped, - key=mapped.index) # to keep the order, we can't use set() to unique + mapped, key=mapped.index + ) # to keep the order, we can't use set() to unique if not names: return [] func_code = "nonlocal {}".format(','.join(names)) @@ -1364,10 +1461,10 @@ def create_nonlocal_stmt_nodes(names): class GetterSetterHelper: - """ we have two classes of names in setter and getter function: - w_vars(loop_vars) + push_pop_vars - To simplify the setter logic in convert_while and convert_cond, - we extract the helper class here. + """we have two classes of names in setter and getter function: + w_vars(loop_vars) + push_pop_vars + To simplify the setter logic in convert_while and convert_cond, + we extract the helper class here. """ def __init__(self, getter_func, setter_func, *name_lists): @@ -1383,22 +1480,33 @@ class GetterSetterHelper: return self._union def get(self, names): - if names is None: names = [] + if names is None: + names = [] vars = self.getter() - if vars is None: return tuple() + if vars is None: + return tuple() for n in names: - assert n in self.name2id, "the name `{}` not in name union set`{}`.".format( - n, self.name2id.keys()) + assert ( + n in self.name2id + ), "the name `{}` not in name union set`{}`.".format( + n, self.name2id.keys() + ) return tuple(map(lambda n: vars[self.name2id[n]], names)) def set(self, names, values): - if names is None: names = [] - if values is None: values = [] + if names is None: + names = [] + if values is None: + values = [] vars = self.getter() - if vars is None: return + if vars is None: + return for n in names: - assert n in self.name2id, "the name `{}` not in name union set`{}`.".format( - n, self.name2id.keys()) + assert ( + n in self.name2id + ), "the name `{}` not in name union set`{}`.".format( + n, self.name2id.keys() + ) vars = list(vars) indices = list(map(lambda n: self.name2id[n], names)) for i, v in zip(indices, values): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py index 8afe558ecfc..f28f1993621 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py @@ -17,7 +17,10 @@ import textwrap from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.framework import Variable -from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar, create_undefined_variable +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + UndefinedVar, + create_undefined_variable, +) from paddle.fluid.layers.utils import map_structure, is_sequence __all__ = [ @@ -37,16 +40,19 @@ def create_fill_constant_node(name, value=0): func_code = "{} = paddle.full(shape=[1], ".format(name) if isinstance(value, bool): func_code += "dtype='bool', fill_value={}, name='{}')".format( - value, name) + value, name + ) return gast.parse(func_code).body[0] if isinstance(value, float): func_code += "dtype='float64', fill_value={}, name='{}')".format( - value, name) + value, name + ) return gast.parse(func_code).body[0] if isinstance(value, int): func_code += "dtype='int64', fill_value={}, name='{}')".format( - value, name) + value, name + ) return gast.parse(func_code).body[0] diff --git a/python/paddle/fluid/dygraph/inplace_utils.py b/python/paddle/fluid/dygraph/inplace_utils.py index 968a957b660..fb27a5674b7 100644 --- a/python/paddle/fluid/dygraph/inplace_utils.py +++ b/python/paddle/fluid/dygraph/inplace_utils.py @@ -23,13 +23,14 @@ from paddle import _C_ops, _legacy_C_ops # in dygraph mode. If static mode is used, the inplace mechanism will not be used, and the static method # of the original API will be called. def _inplace_apis_in_dygraph_only_(func): - def __impl__(*args, **kwargs): if not _non_static_mode(): origin_api_name = func.__name__[:-1] warnings.warn( - "In static mode, {}() is the same as {}() and does not perform inplace operation." - .format(func.__name__, origin_api_name)) + "In static mode, {}() is the same as {}() and does not perform inplace operation.".format( + func.__name__, origin_api_name + ) + ) origin_func = "{}.{}".format(func.__module__, origin_api_name) return eval(origin_func)(*args, **kwargs) return func(*args, **kwargs) diff --git a/python/paddle/fluid/dygraph/io.py b/python/paddle/fluid/dygraph/io.py index 9ddb46798cf..d2e7ada85a7 100644 --- a/python/paddle/fluid/dygraph/io.py +++ b/python/paddle/fluid/dygraph/io.py @@ -27,8 +27,14 @@ from paddle.fluid.layers import nn from paddle.fluid.layers.utils import _hash_with_id from paddle.fluid.dygraph.base import switch_to_static_graph from paddle.fluid.framework import _non_static_mode -from paddle.fluid.executor import _is_enable_standalone_executor, _is_dy2st_enable_standalone_executor -from paddle.fluid.dygraph.dygraph_to_static.partial_program import add_build_strategy_for, LazyInitialized +from paddle.fluid.executor import ( + _is_enable_standalone_executor, + _is_dy2st_enable_standalone_executor, +) +from paddle.fluid.dygraph.dygraph_to_static.partial_program import ( + add_build_strategy_for, + LazyInitialized, +) from paddle import _C_ops, _legacy_C_ops __all__ = ['TranslatedLayer'] @@ -50,17 +56,20 @@ def _load_program_desc(model_file_path): program_desc = core.ProgramDesc(program_desc_str) if not core._is_program_version_supported(program_desc._version()): - raise ValueError("Unsupported program version: %d\n" % - program_desc._version()) + raise ValueError( + "Unsupported program version: %d\n" % program_desc._version() + ) return program_desc def _is_persistable(var_desc): - if var_desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var_desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var_desc.type() == core.VarDesc.VarType.READER or \ - var_desc.type() == core.VarDesc.VarType.RAW: + if ( + var_desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var_desc.type() == core.VarDesc.VarType.FETCH_LIST + or var_desc.type() == core.VarDesc.VarType.READER + or var_desc.type() == core.VarDesc.VarType.RAW + ): return False return var_desc.persistable() @@ -204,9 +213,11 @@ def _rename_var_program_desc(program_desc, include=None, exclude=None): name_old = var.name() is_double_grad_var = "@GRAD" in name_old has_double_grad = has_double_grad or is_double_grad_var - should_rename = (include is None or name_old in include) and ( - exclude is None - or name_old not in exclude) and not is_double_grad_var + should_rename = ( + (include is None or name_old in include) + and (exclude is None or name_old not in exclude) + and not is_double_grad_var + ) if should_rename: temp_name = name_old.split('_') if len(temp_name) > 1 and temp_name[-1].isnumeric(): @@ -215,9 +226,12 @@ def _rename_var_program_desc(program_desc, include=None, exclude=None): temp_name = name_old while True: name_new = _generate_unique_var_name_sync_with_main_program( - temp_name) - if name_new not in old_names[:var_idx] + old_names[var_idx + - 1:]: + temp_name + ) + if ( + name_new + not in old_names[:var_idx] + old_names[var_idx + 1 :] + ): break else: name_new = name_old @@ -237,13 +251,16 @@ def _rename_var_program_desc(program_desc, include=None, exclude=None): var_name = var.name() if "@GRAD" in var_name and name_old in var_name: new_var_name = var_name.replace( - name_old, dict_rename_var_old_new[name_old]) + name_old, dict_rename_var_old_new[name_old] + ) double_grad_rename_dict[var_name] = new_var_name for var_name in double_grad_rename_dict: dict_rename_var_old_new[var_name] = double_grad_rename_dict[ - var_name] + var_name + ] dict_rename_var_new_old[ - double_grad_rename_dict[var_name]] = var_name + double_grad_rename_dict[var_name] + ] = var_name # Rename on program desc for b_idx in range(program_desc.num_blocks()): @@ -252,27 +269,38 @@ def _rename_var_program_desc(program_desc, include=None, exclude=None): op = cur_block.op(op_idx) for input_arg_name in op.input_arg_names(): if input_arg_name in dict_rename_var_old_new: - if input_arg_name != dict_rename_var_old_new[input_arg_name]: + if ( + input_arg_name + != dict_rename_var_old_new[input_arg_name] + ): op._rename_input( input_arg_name, - dict_rename_var_old_new[input_arg_name]) + dict_rename_var_old_new[input_arg_name], + ) if cur_block.has_var(input_arg_name.encode()): cur_block._rename_var( input_arg_name.encode(), - dict_rename_var_old_new[input_arg_name].encode( - )) + dict_rename_var_old_new[ + input_arg_name + ].encode(), + ) for output_arg_name in op.output_arg_names(): if output_arg_name in dict_rename_var_old_new: - if output_arg_name != dict_rename_var_old_new[ - output_arg_name]: + if ( + output_arg_name + != dict_rename_var_old_new[output_arg_name] + ): op._rename_output( output_arg_name, - dict_rename_var_old_new[output_arg_name]) + dict_rename_var_old_new[output_arg_name], + ) if cur_block.has_var(output_arg_name.encode()): cur_block._rename_var( output_arg_name.encode(), - dict_rename_var_old_new[output_arg_name].encode( - )) + dict_rename_var_old_new[ + output_arg_name + ].encode(), + ) program_desc.flush() return dict_rename_var_new_old, dict_rename_var_old_new @@ -327,7 +355,8 @@ class _ProgramHolder(object): self._infer_program_desc = self._preprocess(program_desc) # forward + backward program self._train_program_desc = self._append_backward_desc( - self._infer_program_desc) + self._infer_program_desc + ) # forward: @switch_to_static_graph @@ -348,11 +377,13 @@ class _ProgramHolder(object): def _create_backward_train_program(self): whole_program = _build_program_by_desc(self._train_program_desc) start_op_index = self._infer_program_desc.block(0).op_size() + 2 * len( - self._output_descs) + self._output_descs + ) end_op_index = whole_program.desc.block(0).op_size() - if (start_op_index < end_op_index): - return add_build_strategy_for(whole_program, start_op_index, - end_op_index) + if start_op_index < end_op_index: + return add_build_strategy_for( + whole_program, start_op_index, end_op_index + ) else: return paddle.static.Program() @@ -400,7 +431,8 @@ class _ProgramHolder(object): # rename persistable variables of 'program_desc' list_persistable_var = _get_persistable_var_names(program_desc) rename_new_old_dict, _ = _rename_var_program_desc( - program_desc, list_persistable_var) + program_desc, list_persistable_var + ) # 1. Prune original program # remove feed, fetch and scale-1 op, remove op_callstack attr ops_to_remove = [] @@ -412,14 +444,17 @@ class _ProgramHolder(object): feed_var_name = op.input('X')[0].encode() root_block._remove_var(feed_var_name) self._input_descs.append( - root_block.find_var(op.output('Out')[0].encode())) + root_block.find_var(op.output('Out')[0].encode()) + ) elif op.type() == 'scale' and op.output('Out')[0].startswith( - 'save_infer_model/scale_'): + 'save_infer_model/scale_' + ): ops_to_remove.append(i) out_var_name = op.output('Out')[0].encode() root_block._remove_var(out_var_name) self._output_descs.append( - root_block.find_var(op.input('X')[0].encode())) + root_block.find_var(op.input('X')[0].encode()) + ) elif op.type() == 'fetch': ops_to_remove.append(i) fetch_var_name = op.output('Out')[0].encode() @@ -427,7 +462,8 @@ class _ProgramHolder(object): # NOTE: some old pre-train models have no extra scale_op if not op.input('X')[0].startswith('save_infer_model/scale_'): self._output_descs.append( - root_block.find_var(op.input('X')[0].encode())) + root_block.find_var(op.input('X')[0].encode()) + ) else: if op.has_attr("op_callstack"): op.remove_attr("op_callstack") @@ -472,7 +508,8 @@ class _ProgramHolder(object): # there will be a problem of duplicate names, so here is unified # to add the LOADED suffix to the parameters of the model loaded self._suffix_varname_dict = _get_loaded_var_new_old( - program_desc, rename_new_old_dict) + program_desc, rename_new_old_dict + ) # - get persistable var self._persistable_names = _get_persistable_var_names(program_desc) @@ -486,9 +523,9 @@ class _ProgramHolder(object): with framework.program_guard(program): for i, out in enumerate(self._output_descs): var = program.global_block().var(out.name()) - var = nn.scale(var, - 1., - name="translated_layer/scale_{}".format(i)) + var = nn.scale( + var, 1.0, name="translated_layer/scale_{}".format(i) + ) scale_output_vars.append(var) # 2. update output names & descs for i, var in enumerate(scale_output_vars): @@ -513,15 +550,19 @@ class _ProgramHolder(object): block = program.block(block_idx) for op in block.ops: if op.type == "batch_norm": - if "ReserveSpace" not in op.output_names or len( - op.output("ReserveSpace")) == 0: + if ( + "ReserveSpace" not in op.output_names + or len(op.output("ReserveSpace")) == 0 + ): reserve_space = block.create_var( name=unique_name.generate_with_ignorable_key( - ".".join(["reserve_space", 'tmp'])), + ".".join(["reserve_space", 'tmp']) + ), dtype=block.var(op.input("X")[0]).dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) op.desc.set_output("ReserveSpace", [reserve_space.name]) return program @@ -567,9 +608,9 @@ class _ProgramHolder(object): # NOTE: [compatible] deal with model saved by save_inference_model, # which need get var info from program desc -def _load_persistable_vars_by_program(model_path, - program_holder, - params_filename=None): +def _load_persistable_vars_by_program( + model_path, program_holder, params_filename=None +): # make sure the path has been checked persistable_vars = _get_persistable_vars(program_holder.infer_program) load_var_dict = {} @@ -578,37 +619,43 @@ def _load_persistable_vars_by_program(model_path, if _is_parameter(each_var, program_holder.infer_program): # create output varbase if framework._in_eager_without_dygraph_check(): - new_var = framework.EagerParamBase(shape=each_var.shape(), - dtype=each_var.dtype(), - name=each_var.name(), - type=each_var.type(), - persistable=True) + new_var = framework.EagerParamBase( + shape=each_var.shape(), + dtype=each_var.dtype(), + name=each_var.name(), + type=each_var.type(), + persistable=True, + ) else: - new_var = framework.ParamBase(shape=each_var.shape(), - dtype=each_var.dtype(), - name=each_var.name(), - type=each_var.type(), - persistable=True) + new_var = framework.ParamBase( + shape=each_var.shape(), + dtype=each_var.dtype(), + name=each_var.name(), + type=each_var.type(), + persistable=True, + ) else: - new_var = framework._varbase_creator(type=each_var.type(), - name=each_var.name(), - shape=each_var.shape(), - dtype=each_var.dtype(), - persistable=True) + new_var = framework._varbase_creator( + type=each_var.type(), + name=each_var.name(), + shape=each_var.shape(), + dtype=each_var.dtype(), + persistable=True, + ) if params_filename is None: framework._dygraph_tracer().trace_op( type='load', inputs={}, outputs={'Out': new_var}, - attrs={'file_path': os.path.join(model_path, orig_each_name)}) + attrs={'file_path': os.path.join(model_path, orig_each_name)}, + ) new_var.stop_gradient = False load_var_dict[each_var.name()] = new_var if params_filename is not None: load_var_list = [] dict_name_old_new = { - v: k - for k, v in program_holder._suffix_varname_dict.items() + v: k for k, v in program_holder._suffix_varname_dict.items() } for name in sorted(dict_name_old_new.keys()): load_var_list.append(load_var_dict[dict_name_old_new[name]]) @@ -617,7 +664,8 @@ def _load_persistable_vars_by_program(model_path, type='load_combine', inputs={}, outputs={'Out': load_var_list}, - attrs={'file_path': os.path.join(model_path, params_filename)}) + attrs={'file_path': os.path.join(model_path, params_filename)}, + ) for each_var in persistable_vars: if not _is_parameter(each_var, program_holder.infer_program): @@ -639,8 +687,9 @@ def _load_persistable_vars_by_program(model_path, return load_var_dict -def _load_persistable_vars(model_path, var_info_path, program_holder, - params_filename): +def _load_persistable_vars( + model_path, var_info_path, program_holder, params_filename +): # 1. load extra var info with open(var_info_path, 'rb') as f: extra_var_info = pickle.load(f) @@ -649,8 +698,7 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, load_var_dict = dict() load_var_list = [] inv_suffix_varname_dict = { - value: key - for key, value in program_holder._suffix_varname_dict.items() + value: key for key, value in program_holder._suffix_varname_dict.items() } # NOTE(chenweihang): we need load persistable vars based the program, @@ -661,7 +709,8 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, raise RuntimeError( "The model to be loaded is not complete." "The variable `%s` of program cannot be found in loaded model.", - name) + name, + ) # get suffix var name, see [why need to append suffix to persistable vars] new_name = inv_suffix_varname_dict[name] # create output varbase @@ -674,7 +723,8 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, ], # only to pass check, this shape is not meaningful dtype=core.VarDesc.VarType.FP32, name=new_name, - persistable=True) + persistable=True, + ) else: new_var = framework.ParamBase( shape=[ @@ -682,10 +732,12 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, ], # only to pass check, this shape is not meaningful dtype=core.VarDesc.VarType.FP32, name=new_name, - persistable=True) + persistable=True, + ) else: - new_var = framework._varbase_creator(name=new_name, - persistable=True) + new_var = framework._varbase_creator( + name=new_name, persistable=True + ) new_var.stop_gradient = extra_var_info[name]['stop_gradient'] load_var_dict[new_name] = new_var @@ -698,10 +750,12 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, if len(extra_var_info) != 0: raise ValueError("The model to be loaded is incomplete.") else: - framework._dygraph_tracer().trace_op(type='load_combine', - inputs={}, - outputs={'Out': load_var_list}, - attrs={'file_path': var_file_path}) + framework._dygraph_tracer().trace_op( + type='load_combine', + inputs={}, + outputs={'Out': load_var_list}, + attrs={'file_path': var_file_path}, + ) return load_var_dict @@ -723,17 +777,18 @@ def _construct_program_holders(model_path, model_filename=None): # [compatible] if assign model_filename, only can load one program as Layer.forward model_filename = os.path.basename(model_filename) model_file_path = os.path.join(model_path, model_filename) - model_name = model_filename[:-len(INFER_MODEL_SUFFIX)] - #Load every file that meets the requirements in the directory model_path. + model_name = model_filename[: -len(INFER_MODEL_SUFFIX)] + # Load every file that meets the requirements in the directory model_path. for filename in os.listdir(model_path): if model_filename == filename: func_name = 'forward' model_file_path = os.path.join(model_path, model_filename) elif filename.endswith(INFER_MODEL_SUFFIX) and filename.startswith( - model_name): - parsing_names = filename[len(model_name - ):-len(INFER_MODEL_SUFFIX) + - 1].split('.') + model_name + ): + parsing_names = filename[ + len(model_name) : -len(INFER_MODEL_SUFFIX) + 1 + ].split('.') if len(parsing_names) == 3 and len(parsing_names[1]) > 0: func_name = parsing_names[1] model_file_path = os.path.join(model_path, filename) @@ -742,7 +797,8 @@ def _construct_program_holders(model_path, model_filename=None): else: continue program_holder_dict[func_name] = _ProgramHolder( - _load_program_desc(model_file_path)) + _load_program_desc(model_file_path) + ) else: for _, _, file_names in os.walk(model_path): for name in file_names: @@ -754,30 +810,32 @@ def _construct_program_holders(model_path, model_filename=None): else: method_name.replace('model', '') program_holder_dict[method_name] = _ProgramHolder( - _load_program_desc(model_file_path)) + _load_program_desc(model_file_path) + ) return program_holder_dict -def _construct_params_and_buffers(model_path, - programs, - params_filename=None, - append_suffix=True): +def _construct_params_and_buffers( + model_path, programs, params_filename=None, append_suffix=True +): var_info_filename = str(params_filename) + ".info" var_info_path = os.path.join(model_path, var_info_filename) params_path = os.path.join(model_path, str(params_filename)) if os.path.exists(var_info_path): - var_dict = _load_persistable_vars(model_path, var_info_path, - programs['forward'], params_filename) - model_name = params_filename[:-len(INFER_PARAMS_SUFFIX)] - #Load every file that meets the requirements in the directory model_path. + var_dict = _load_persistable_vars( + model_path, var_info_path, programs['forward'], params_filename + ) + model_name = params_filename[: -len(INFER_PARAMS_SUFFIX)] + # Load every file that meets the requirements in the directory model_path. for file_name in os.listdir(model_path): if file_name.startswith(model_name) and file_name.endswith( - INFER_PARAMS_SUFFIX): - parsing_names = file_name[len(model_name - ):-len(INFER_PARAMS_SUFFIX) + - 1].split('.') + INFER_PARAMS_SUFFIX + ): + parsing_names = file_name[ + len(model_name) : -len(INFER_PARAMS_SUFFIX) + 1 + ].split('.') if len(parsing_names) == 3 and len(parsing_names[1]) > 0: func_name = parsing_names[1] else: @@ -786,15 +844,17 @@ def _construct_params_and_buffers(model_path, continue var_info_path = os.path.join(model_path, var_info_filename) var_dict.update( - _load_persistable_vars(model_path, var_info_path, - programs[func_name], file_name)) + _load_persistable_vars( + model_path, var_info_path, programs[func_name], file_name + ) + ) elif params_filename is not None and not os.path.exists(params_path): # When saving XX, there is only '*.pdmodel' return dict() else: - var_dict = _load_persistable_vars_by_program(model_path, - programs['forward'], - params_filename) + var_dict = _load_persistable_vars_by_program( + model_path, programs['forward'], params_filename + ) if not append_suffix: var_dict = _remove_varname_suffix(var_dict, programs['forward']) @@ -807,13 +867,23 @@ def _valid_vars(vars): return vars if framework._in_eager_without_dygraph_check(): return [ - core.eager.Tensor(core.VarDesc.VarType.FP32, [], "Fake_var", - core.VarDesc.VarType.RAW, False) + core.eager.Tensor( + core.VarDesc.VarType.FP32, + [], + "Fake_var", + core.VarDesc.VarType.RAW, + False, + ) ] else: return [ - core.VarBase(core.VarDesc.VarType.FP32, [], "Fake_var", - core.VarDesc.VarType.RAW, False) + core.VarBase( + core.VarDesc.VarType.FP32, + [], + "Fake_var", + core.VarDesc.VarType.RAW, + False, + ) ] @@ -825,7 +895,8 @@ def _run_dygraph(instance, input, program_holder): if not isinstance(value, (np.ndarray, core.VarBase, core.eager.Tensor)): raise TypeError( "The type of input in TranslatedLayer must be numpy array or Variable(VarBase), but received %s." - % type(value)) + % type(value) + ) # NOTE: In order to unify the API, firstly convert the input to VarBase if isinstance(value, np.ndarray): if framework._in_eager_without_dygraph_check(): @@ -834,13 +905,16 @@ def _run_dygraph(instance, input, program_holder): name=program_holder.input_descs[i].name(), persistable=False, place=framework._current_expected_place(), - zero_copy=True) + zero_copy=True, + ) else: - var = core.VarBase(value=value, - name=program_holder.input_descs[i].name(), - persistable=False, - place=framework._current_expected_place(), - zero_copy=True) + var = core.VarBase( + value=value, + name=program_holder.input_descs[i].name(), + persistable=False, + place=framework._current_expected_place(), + zero_copy=True, + ) else: var = value # NOTE: we changed var name here, @@ -862,67 +936,112 @@ def _run_dygraph(instance, input, program_holder): else: raise ValueError( "The persistable variable %s does not exist in current TranslatedLayer." - % var_name) + % var_name + ) output_vars = [] for var_desc in program_holder.output_descs: if framework._in_eager_without_dygraph_check(): - var = core.eager.Tensor(dtype=var_desc.dtype(), - dims=var_desc.shape(), - name=var_desc.name(), - type=var_desc.type(), - persistable=False) + var = core.eager.Tensor( + dtype=var_desc.dtype(), + dims=var_desc.shape(), + name=var_desc.name(), + type=var_desc.type(), + persistable=False, + ) else: - var = core.VarBase(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) output_vars.append(var) # hold forward variables if framework._in_eager_without_dygraph_check(): tmp_scope_vec = [program_holder.scope] else: - tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [], - "program_out_scope", - core.VarDesc.VarType.STEP_SCOPES, True) + tmp_scope_vec = core.VarBase( + core.VarDesc.VarType.FP32, + [], + "program_out_scope", + core.VarDesc.VarType.STEP_SCOPES, + True, + ) tmp_scope_vec.value().set_scope(program_holder.scope) double_grad_vars = [] for var_desc in program_holder.double_grad_descs: if framework._in_eager_without_dygraph_check(): - var = core.eager.Tensor(dtype=var_desc.dtype(), - dims=var_desc.shape(), - name=var_desc.name(), - type=var_desc.type(), - persistable=False) + var = core.eager.Tensor( + dtype=var_desc.dtype(), + dims=var_desc.shape(), + name=var_desc.name(), + type=var_desc.type(), + persistable=False, + ) else: - var = core.VarBase(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) double_grad_vars.append(var) # 2. run program by op - trace_program = program_holder.infer_program if instance._is_test else program_holder.train_program - forward_program = program_holder._infer_program_desc if instance._is_test else program_holder.forward_program + trace_program = ( + program_holder.infer_program + if instance._is_test + else program_holder.train_program + ) + forward_program = ( + program_holder._infer_program_desc + if instance._is_test + else program_holder.forward_program + ) end_op_index = program_holder.infer_program.block(0).op_size() attrs = [ 'global_block', - trace_program.block(0), 'start_op_index', 0, 'end_op_index', - end_op_index, 'is_test', instance._is_test, 'program_id', - _hash_with_id(trace_program, instance) + trace_program.block(0), + 'start_op_index', + 0, + 'end_op_index', + end_op_index, + 'is_test', + instance._is_test, + 'program_id', + _hash_with_id(trace_program, instance), ] - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: attrs.extend( - ('forward_global_block', forward_program.block(0), - 'backward_global_block', program_holder.backward_program.block(0))) - - _legacy_C_ops.run_program(_valid_vars(input_vars), - _valid_vars(persistable_vars), - _valid_vars(output_vars), tmp_scope_vec, - _valid_vars(double_grad_vars), None, *attrs) + ( + 'forward_global_block', + forward_program.block(0), + 'backward_global_block', + program_holder.backward_program.block(0), + ) + ) + + _legacy_C_ops.run_program( + _valid_vars(input_vars), + _valid_vars(persistable_vars), + _valid_vars(output_vars), + tmp_scope_vec, + _valid_vars(double_grad_vars), + None, + *attrs + ) # NOTE: [ why need set param's gradient type here ] # if user set sparse gradient mode, the param's gradient @@ -951,8 +1070,11 @@ def _run_dygraph(instance, input, program_holder): def drop_scope_if_no_grad(instance, scope_vec): tracer = framework._dygraph_tracer() - scope = scope_vec.value().get_scope() if isinstance( - scope_vec, (core.VarBase)) else scope_vec[0] + scope = ( + scope_vec.value().get_scope() + if isinstance(scope_vec, (core.VarBase)) + else scope_vec[0] + ) if (not instance._is_test) and (not tracer._has_grad): scope.drop_kids() @@ -961,15 +1083,22 @@ def _run_static_graph(input, program_holder, trace_program): main_program = framework.default_main_program() param_var_names = _get_persistable_var_names(trace_program) _, dict_rename_var_old_new = _rename_var_program_desc( - trace_program, exclude=param_var_names) + trace_program, exclude=param_var_names + ) trace_program.flush() output_names = [var.name() for var in program_holder.output_descs] # append blocks from 'trace_program' - _append_block(main_program, trace_program, program_holder, input, - dict_rename_var_old_new) + _append_block( + main_program, + trace_program, + program_holder, + input, + dict_rename_var_old_new, + ) main_program._sync_with_cpp() - outs = _get_output_from_program(main_program, program_holder, - dict_rename_var_old_new) + outs = _get_output_from_program( + main_program, program_holder, dict_rename_var_old_new + ) if len(outs) == 1: outs = outs[0] return outs @@ -997,11 +1126,13 @@ def _collect_current_and_parent_var(program, block_idx): return vars -def _append_block(dest_program, - src_program_desc, - program_holder, - input_variables, - dict_rename_var_old_new=None): +def _append_block( + dest_program, + src_program_desc, + program_holder, + input_variables, + dict_rename_var_old_new=None, +): ''' Append Variables and Operators in 'src_program_desc' to dest_program. @@ -1015,28 +1146,35 @@ def _append_block(dest_program, ''' origin_block_idx = dest_program.current_block_idx - param_var_names = _collect_current_and_parent_var(dest_program, - origin_block_idx) - append_var_from_block_desc_static(dest_program.block(origin_block_idx), - src_program_desc.block(0), - exclude=param_var_names) + param_var_names = _collect_current_and_parent_var( + dest_program, origin_block_idx + ) + append_var_from_block_desc_static( + dest_program.block(origin_block_idx), + src_program_desc.block(0), + exclude=param_var_names, + ) name_inp_desc = [inp.name() for inp in program_holder.input_descs] input_names = [inp.name for inp in input_variables] if len(name_inp_desc) != len(input_names): raise ValueError( - "The number of input is invalid, expected {}, but received {}.". - format(len(name_inp_desc), len(input_names))) + "The number of input is invalid, expected {}, but received {}.".format( + len(name_inp_desc), len(input_names) + ) + ) for i, out_name in enumerate(name_inp_desc): if dict_rename_var_old_new: out_name = dict_rename_var_old_new[out_name] dest_program.block(origin_block_idx).append_op( type='assign', inputs={'X': [input_names[i]]}, - outputs={'Out': [out_name]}) + outputs={'Out': [out_name]}, + ) append_ops = append_op_from_block_desc_static( - dest_program.block(origin_block_idx), src_program_desc.block(0)) + dest_program.block(origin_block_idx), src_program_desc.block(0) + ) dest_program._sync_with_cpp() offset_block_idx = dest_program.num_blocks - 1 @@ -1050,11 +1188,12 @@ def _append_block(dest_program, else: parent_idx = origin_block_idx dest_block = dest_program._create_block(parent_idx=parent_idx) - append_var_from_block_desc_static(dest_block, - src_block, - exclude=param_var_names) + append_var_from_block_desc_static( + dest_block, src_block, exclude=param_var_names + ) append_ops += append_op_from_block_desc_static( - dest_block, src_block) + dest_block, src_block + ) dest_program._sync_with_cpp() for op in append_ops: @@ -1064,15 +1203,16 @@ def _append_block(dest_program, origin_id = sub.id if isinstance(sub, framework.Block): origin_id = sub.idx - op._set_attr('sub_block', - dest_program.block(offset_block_idx + origin_id)) + op._set_attr( + 'sub_block', dest_program.block(offset_block_idx + origin_id) + ) dest_program._sync_with_cpp() dest_program.current_block_idx = origin_block_idx -def _get_output_from_program(program, - program_holder, - dict_rename_var_old_new=None): +def _get_output_from_program( + program, program_holder, dict_rename_var_old_new=None +): """ Get output name of 'program' according to program_holder """ @@ -1121,20 +1261,21 @@ def append_op_from_desc_static(block, op_desc): op_type = op_desc.type() op_append = block.desc.append_op() op_append.copy_from(op_desc) - op = framework.Operator(block=block, - desc=op_append, - type=op_type, - inputs=None, - outputs=None, - attrs=None) + op = framework.Operator( + block=block, + desc=op_append, + type=op_type, + inputs=None, + outputs=None, + attrs=None, + ) block.ops.append(op) return op -def append_var_from_block_desc_static(block, - src_block_desc, - include=None, - exclude=None): +def append_var_from_block_desc_static( + block, src_block_desc, include=None, exclude=None +): """ Append Variables of 'src_block_desc' to current block. If 'include' is not `None`,variables that are not in include are not append. @@ -1153,13 +1294,14 @@ def append_var_from_block_desc_static(block, for var_desc in src_block_desc.all_vars(): var_desc_name = var_desc.name() should_append = (include is None or var_desc_name in include) and ( - exclude is None or var_desc_name not in exclude) + exclude is None or var_desc_name not in exclude + ) if not block.has_var(var_desc_name) and should_append: var_type = var_desc.type() if var_type in [ - core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.LOD_TENSOR_ARRAY + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, ]: data_type = var_desc.dtype() var_shape = var_desc.shape() @@ -1167,8 +1309,8 @@ def append_var_from_block_desc_static(block, data_type = None var_shape = None if var_type in [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.LOD_TENSOR_ARRAY + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, ]: lod_level = var_desc.lod_level() else: @@ -1187,7 +1329,9 @@ def append_var_from_block_desc_static(block, shape=var_shape, lod_level=lod_level, persistable=var_desc.persistable(), - set_need_check_feed=var_desc.need_check_feed())) + set_need_check_feed=var_desc.need_check_feed(), + ) + ) return vars_append @@ -1312,8 +1456,9 @@ class TranslatedLayer(layers.Layer): # the TranslatedLayer object holded var names count started from 0 with unique_name.guard(): for name, var in persistable_vars.items(): - if isinstance(var, - (framework.ParamBase, framework.EagerParamBase)): + if isinstance( + var, (framework.ParamBase, framework.EagerParamBase) + ): dy_name = _generate_unique_var_name(PARAMETER_NAME_PREFIX) self._persistable_var_name_dict[name] = dy_name self.add_parameter(dy_name, var) @@ -1347,7 +1492,8 @@ class TranslatedLayer(layers.Layer): # 2. load layer parameters & buffers persistable_vars = _construct_params_and_buffers( - model_path, programs, params_filename) + model_path, programs, params_filename + ) # 3. construct TranslatedLayer object translated_layer = TranslatedLayer(programs, persistable_vars) @@ -1359,9 +1505,12 @@ class TranslatedLayer(layers.Layer): ins.name() for ins in program_holder.input_descs ] setattr( - TranslatedLayer, method_name, + TranslatedLayer, + method_name, TranslatedLayer._execution_method_creator( - method_name, program_holder)) + method_name, program_holder + ), + ) # 5. set TranslatedLayer's default mode to eval translated_layer.eval() @@ -1370,7 +1519,6 @@ class TranslatedLayer(layers.Layer): @staticmethod def _execution_method_creator(method_name, program_holder): - def __i_m_p_l__(self, *input): program_holder = self._program_holder_dict[__i_m_p_l__.__name__] # When using jit.save, it runs in static graph mode. @@ -1383,7 +1531,8 @@ class TranslatedLayer(layers.Layer): # because '_run_static_graph' modifies 'ProgramDesc', 'OpDesc.op_size()' will return a very large wrong number. # A Segmentation fault error may occur if used 'p=ProgramDesc(program_holder.infer_program)'. p = framework.Program._construct_from_desc( - core.ProgramDesc(program_holder.infer_program)) + core.ProgramDesc(program_holder.infer_program) + ) return _run_static_graph(input, program_holder, p.desc) __i_m_p_l__.__name__ = method_name @@ -1496,8 +1645,9 @@ class TranslatedLayer(layers.Layer): program_holder = self._program_holder_dict.get(method_name, None) if program_holder is None: raise ValueError( - "The method `%s` does not exist in loaded TranslatedLayer." % - method_name) + "The method `%s` does not exist in loaded TranslatedLayer." + % method_name + ) return program_holder def _input_spec(self, method_name='forward'): @@ -1507,9 +1657,11 @@ class TranslatedLayer(layers.Layer): # 2. build input spec by input desc input_spec = [] for var_desc in program_holder.input_descs: - spec = paddle.static.InputSpec(shape=var_desc.shape(), - dtype=var_desc.dtype(), - name=var_desc.name()) + spec = paddle.static.InputSpec( + shape=var_desc.shape(), + dtype=var_desc.dtype(), + name=var_desc.name(), + ) input_spec.append(spec) return input_spec @@ -1524,9 +1676,11 @@ class TranslatedLayer(layers.Layer): # NOTE(chenweihang): InputSpec describes a tensor, not just input. # Maybe the name is not good enough. Here we use InputSpec to # construct the description of Output tensor - spec = paddle.static.InputSpec(shape=var_desc.shape(), - dtype=var_desc.dtype(), - name=var_desc.name()) + spec = paddle.static.InputSpec( + shape=var_desc.shape(), + dtype=var_desc.dtype(), + name=var_desc.name(), + ) output_spec.append(spec) return output_spec diff --git a/python/paddle/fluid/dygraph/jit.py b/python/paddle/fluid/dygraph/jit.py index c659af4a783..acca96a9376 100644 --- a/python/paddle/fluid/dygraph/jit.py +++ b/python/paddle/fluid/dygraph/jit.py @@ -24,25 +24,65 @@ from typing import Text, Tuple, Any, List import paddle from paddle.fluid import core, dygraph -from paddle.fluid.compiler import BuildStrategy, CompiledProgram, ExecutionStrategy +from paddle.fluid.compiler import ( + BuildStrategy, + CompiledProgram, + ExecutionStrategy, +) from paddle.fluid.data_feeder import check_type from paddle.fluid.layers.utils import flatten, pack_sequence_as -from paddle.fluid.dygraph.base import program_desc_tracing_guard, switch_to_static_graph +from paddle.fluid.dygraph.base import ( + program_desc_tracing_guard, + switch_to_static_graph, +) from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import ConversionOptions, CONVERSION_OPTIONS -from paddle.fluid.dygraph.dygraph_to_static.logging_utils import set_code_level, set_verbosity -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator, StaticFunction, unwrap_decorators -from paddle.fluid.dygraph.io import TranslatedLayer, INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX, INFER_PARAMS_INFO_SUFFIX, INFER_PROPERTY_SUFFIX +from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import ( + ConversionOptions, + CONVERSION_OPTIONS, +) +from paddle.fluid.dygraph.dygraph_to_static.logging_utils import ( + set_code_level, + set_verbosity, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, + StaticFunction, + unwrap_decorators, +) +from paddle.fluid.dygraph.io import ( + TranslatedLayer, + INFER_MODEL_SUFFIX, + INFER_PARAMS_SUFFIX, + INFER_PARAMS_INFO_SUFFIX, + INFER_PROPERTY_SUFFIX, +) from paddle.fluid.dygraph.layers import Layer from paddle.fluid.executor import Executor, scope_guard -from paddle.fluid.framework import Block, ParamBase, Program, Variable, Parameter, EagerParamBase -from paddle.fluid.framework import _current_expected_place, _dygraph_guard, _dygraph_tracer +from paddle.fluid.framework import ( + Block, + ParamBase, + Program, + Variable, + Parameter, + EagerParamBase, +) +from paddle.fluid.framework import ( + _current_expected_place, + _dygraph_guard, + _dygraph_tracer, +) from paddle.fluid.framework import dygraph_only, _non_static_mode from paddle.fluid.wrapped_decorator import wrap_decorator __all__ = [ - 'TracedLayer', 'declarative', 'dygraph_to_static_func', 'set_code_level', - 'set_verbosity', 'save', 'load', 'not_to_static' + 'TracedLayer', + 'declarative', + 'dygraph_to_static_func', + 'set_code_level', + 'set_verbosity', + 'save', + 'load', + 'not_to_static', ] @@ -62,8 +102,10 @@ def _extract_vars(inputs, result_list, err_tag='inputs'): _extract_vars(var, result_list, err_tag) else: raise TypeError( - "The type of 'each element of {}' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received {}." - .format(err_tag, type(inputs))) + "The type of 'each element of {}' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received {}.".format( + err_tag, type(inputs) + ) + ) def extract_vars(inputs, err_tag='inputs'): @@ -127,7 +169,8 @@ def _dygraph_to_static_func_(dygraph_func): logging_utils.warn( "The decorator 'dygraph_to_static_func' doesn't work in " "dygraph mode or set ProgramTranslator.enable to False. " - "We will just return dygraph output.") + "We will just return dygraph output." + ) return dygraph_func(*args, **kwargs) static_func = program_translator.get_func(dygraph_func) return static_func(*args, **kwargs) @@ -158,10 +201,9 @@ def copy_decorator_attrs(original_func, decorated_obj): return decorated_obj -def declarative(function=None, - input_spec=None, - build_strategy=None, - property=False): +def declarative( + function=None, input_spec=None, build_strategy=None, property=False +): """ Converts imperative dygraph APIs into declarative function APIs. Decorator @declarative handles the Program and Executor of static mode and returns @@ -213,20 +255,25 @@ def declarative(function=None, _, python_func = unwrap_decorators(python_func) # Step 2. copy some attributes from original python function. - static_layer = copy_decorator_attrs(original_func=python_func, - decorated_obj=StaticFunction( - function=python_func, - input_spec=input_spec, - build_strategy=build_strategy, - property=property)) + static_layer = copy_decorator_attrs( + original_func=python_func, + decorated_obj=StaticFunction( + function=python_func, + input_spec=input_spec, + build_strategy=build_strategy, + property=property, + ), + ) return static_layer build_strategy = build_strategy or BuildStrategy() if not isinstance(build_strategy, BuildStrategy): raise TypeError( - "Required type(build_strategy) shall be `paddle.static.BuildStrategy`, but received {}" - .format(type(build_strategy).__name__)) + "Required type(build_strategy) shall be `paddle.static.BuildStrategy`, but received {}".format( + type(build_strategy).__name__ + ) + ) # for usage: `declarative(foo, ...)` if function is not None: @@ -234,8 +281,10 @@ def declarative(function=None, if isinstance(function.forward, StaticFunction): class_name = function.__class__.__name__ logging_utils.warn( - "`{}.forward` has already been decorated somewhere. It will be redecorated to replace previous one." - .format(class_name)) + "`{}.forward` has already been decorated somewhere. It will be redecorated to replace previous one.".format( + class_name + ) + ) function.forward = decorated(function.forward) return function else: @@ -286,7 +335,6 @@ def not_to_static(func=None): class _SaveLoadConfig(object): - def __init__(self): self._output_spec = None self._model_filename = None @@ -321,12 +369,14 @@ class _SaveLoadConfig(object): if not isinstance(spec, list): raise TypeError( "The config `output_spec` should be 'list', but received input type is %s." - % type(input)) + % type(input) + ) for var in spec: if not isinstance(var, core.VarBase): raise TypeError( "The element in config `output_spec` list should be 'Variable', but received element's type is %s." - % type(var)) + % type(var) + ) self._output_spec = spec @property @@ -340,7 +390,8 @@ class _SaveLoadConfig(object): if not isinstance(filename, str): raise TypeError( "The config `model_filename` should be str, but received input's type is %s." - % type(filename)) + % type(filename) + ) if len(filename) == 0: raise ValueError("The config `model_filename` is empty string.") self._model_filename = filename @@ -356,7 +407,8 @@ class _SaveLoadConfig(object): if not isinstance(filename, str): raise TypeError( "The config `params_filename` should be str, but received input's type is %s." - % type(filename)) + % type(filename) + ) if len(filename) == 0: raise ValueError("The config `params_filename` is empty string.") self._params_filename = filename @@ -372,14 +424,18 @@ class _SaveLoadConfig(object): if not isinstance(value, bool): raise TypeError( "The config `keep_name_table` should be bool value, but received input's type is %s." - % type(value)) + % type(value) + ) self._keep_name_table = value def _parse_save_configs(configs): supported_configs = [ - 'output_spec', "with_hook", "combine_params", "clip_extra", - "skip_forward" + 'output_spec', + "with_hook", + "combine_params", + "clip_extra", + "skip_forward", ] # input check @@ -387,7 +443,8 @@ def _parse_save_configs(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.jit.save` is not supported." - % (key)) + % (key) + ) # construct inner config inner_config = _SaveLoadConfig() @@ -408,7 +465,8 @@ def _parse_load_config(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.jit.load` is not supported." - % (key)) + % (key) + ) # construct inner config inner_config = _SaveLoadConfig() @@ -419,14 +477,18 @@ def _parse_load_config(configs): def _get_input_var_names(inputs, input_spec): - name_none_error = "The %s's name is None. " \ - "When using jit.save, please set InputSepc's name in " \ - "to_static(input_spec=[]) and jit.save(input_spec=[]) " \ + name_none_error = ( + "The %s's name is None. " + "When using jit.save, please set InputSepc's name in " + "to_static(input_spec=[]) and jit.save(input_spec=[]) " "and make sure they are consistent." - name_no_exists_error = "The tensor `%s` does not exists. " \ - "Please make sure the name of InputSpec or example Tensor " \ - "in input_spec is the same as the name of InputSpec in " \ + ) + name_no_exists_error = ( + "The tensor `%s` does not exists. " + "Please make sure the name of InputSpec or example Tensor " + "in input_spec is the same as the name of InputSpec in " "`to_static` decorated on the Layer.forward method." + ) result_list = [] input_var_names = [ var.name for var in flatten(inputs) if isinstance(var, Variable) @@ -437,7 +499,8 @@ def _get_input_var_names(inputs, input_spec): else: # fileter out non-tensor type spec infos. input_spec = [ - spec for spec in input_spec + spec + for spec in input_spec if isinstance(spec, paddle.static.InputSpec) ] @@ -469,10 +532,12 @@ def _get_input_var_names(inputs, input_spec): def _get_output_vars(outputs, output_spec, with_hook=False): - name_no_exists_error = "The tensor `%s` does not exists. " \ - "Please make sure the name of example Tensor " \ - "in configs.output_spec is the output tensor of " \ + name_no_exists_error = ( + "The tensor `%s` does not exists. " + "Please make sure the name of example Tensor " + "in configs.output_spec is the output tensor of " "Layer.forward method." + ) if output_spec and with_hook: raise RuntimeError( "Currently not support specify output_spec while founding pre/post hooks in your outermost layer." @@ -519,12 +584,14 @@ def _build_load_path_and_config(path, config): raise ValueError( "The %s.pdmodel and %s directory exist at the same time, " "don't know which one to load, please make sure that the specified target " - "of ``path`` is unique." % (path, path)) + "of ``path`` is unique." % (path, path) + ) elif not prefix_format_exist and not directory_format_exist: - raise ValueError("The ``path`` (%s) to load model not exists. " - "Please make sure that *.pdmodel exists or " - "don't using ``skip_forward=True`` to jit.save." % - path) + raise ValueError( + "The ``path`` (%s) to load model not exists. " + "Please make sure that *.pdmodel exists or " + "don't using ``skip_forward=True`` to jit.save." % path + ) else: if prefix_format_exist: file_prefix = os.path.basename(path) @@ -533,13 +600,15 @@ def _build_load_path_and_config(path, config): warnings.warn( "When loading the result saved with the " "specified file prefix, the ``model_filename`` config does " - "not take effect.") + "not take effect." + ) config.model_filename = file_prefix + INFER_MODEL_SUFFIX if config.params_filename is not None: warnings.warn( "When loading the result saved with the " "specified file prefix, the ``params_filename`` config does " - "not take effect.") + "not take effect." + ) config.params_filename = file_prefix + INFER_PARAMS_SUFFIX else: # Compatible with the old save_inference_model format @@ -553,7 +622,7 @@ _save_pre_hooks = [] class HookRemoveHelper(object): - """ A HookRemoveHelper that can be used to remove hook. """ + """A HookRemoveHelper that can be used to remove hook.""" def __init__(self, hook): self._hook = hook @@ -638,7 +707,6 @@ def _remove_save_pre_hook(hook): @wrap_decorator def _run_save_pre_hooks(func): - def wrapper(layer, path, input_spec=None, **configs): global _save_pre_hooks for hook in _save_pre_hooks: @@ -826,11 +894,15 @@ def save(layer, path, input_spec=None, **configs): "The paddle.jit.save doesn't work when setting ProgramTranslator.enable to False." ) - if not (isinstance(layer, Layer) or inspect.isfunction(layer) - or isinstance(layer, StaticFunction)): + if not ( + isinstance(layer, Layer) + or inspect.isfunction(layer) + or isinstance(layer, StaticFunction) + ): raise TypeError( "The input of paddle.jit.save should be 'Layer' or 'Function', but received input type is %s." - % type(layer)) + % type(layer) + ) elif inspect.isfunction(layer) or isinstance(layer, StaticFunction): warnings.warn( 'What you save is a function, and `jit.save` will generate the name of the model file according to `path` you specify. When loading these files with `jit.load`, you get a `TranslatedLayer` whose inference result is the same as the inference result of the function you saved.' @@ -852,7 +924,8 @@ def save(layer, path, input_spec=None, **configs): raise ValueError( "The input path MUST be format of dirname/file_prefix " "[dirname\\file_prefix in Windows system], but received " - "file_prefix is empty string.") + "file_prefix is empty string." + ) dirname = os.path.dirname(path) if dirname and not os.path.exists(dirname): @@ -864,23 +937,28 @@ def save(layer, path, input_spec=None, **configs): if isinstance(layer, Layer): for attr_func in dir(inner_layer): static_func = getattr(inner_layer, attr_func, None) - if isinstance(static_func, - StaticFunction) and 'forward' != attr_func: + if ( + isinstance(static_func, StaticFunction) + and 'forward' != attr_func + ): raise ValueError( "If there are static functions other than 'forward' that need to be saved, the input 'input_spec' should be None, but received the type of 'input_spec' is %s." - % type(input_spec)) + % type(input_spec) + ) if not isinstance(input_spec, (list, tuple)): raise TypeError( "The input input_spec should be 'list', but received input_spec's type is %s." - % type(input_spec)) + % type(input_spec) + ) inner_input_spec = [] for var in flatten(input_spec): if isinstance(var, paddle.static.InputSpec): inner_input_spec.append(var) elif isinstance(var, (core.VarBase, core.eager.Tensor, Variable)): inner_input_spec.append( - paddle.static.InputSpec.from_tensor(var)) + paddle.static.InputSpec.from_tensor(var) + ) else: # NOTE(Aurelius84): Support non-Tensor type in `input_spec`. inner_input_spec.append(var) @@ -917,12 +995,18 @@ def save(layer, path, input_spec=None, **configs): # property method to be exported immediate_val = static_func() property_vals.append( - (immediate_val, - layer.__class__.__name__ + '.' + attr_func)) + ( + immediate_val, + layer.__class__.__name__ + '.' + attr_func, + ) + ) continue - concrete_program = static_func.concrete_program_specify_input_spec( - inner_input_spec, with_hook=with_hook) + concrete_program = ( + static_func.concrete_program_specify_input_spec( + inner_input_spec, with_hook=with_hook + ) + ) elif 'forward' == attr_func: if configs.skip_forward: # do not jit.save forward function @@ -932,12 +1016,17 @@ def save(layer, path, input_spec=None, **configs): # inner_input_spec is list[InputSpec], it should be packed with same structure # as original input_spec here. if inner_input_spec: - inner_input_spec = pack_sequence_as(input_spec, - inner_input_spec) - static_forward = declarative(inner_layer.forward, - input_spec=inner_input_spec) - concrete_program = static_forward.concrete_program_specify_input_spec( - with_hook=with_hook) + inner_input_spec = pack_sequence_as( + input_spec, inner_input_spec + ) + static_forward = declarative( + inner_layer.forward, input_spec=inner_input_spec + ) + concrete_program = ( + static_forward.concrete_program_specify_input_spec( + with_hook=with_hook + ) + ) # the input_spec has been used in declarative, which is equal to # @declarative with input_spec and jit.save without input_spec, # avoid needless warning @@ -953,20 +1042,27 @@ def save(layer, path, input_spec=None, **configs): property_vals.append((immediate_val, attr_func)) continue - concrete_program = attr_func.concrete_program_specify_input_spec( - inner_input_spec) + concrete_program = ( + attr_func.concrete_program_specify_input_spec( + inner_input_spec + ) + ) else: if inner_input_spec: - inner_input_spec = pack_sequence_as(input_spec, - inner_input_spec) - static_function = declarative(attr_func, - input_spec=inner_input_spec) + inner_input_spec = pack_sequence_as( + input_spec, inner_input_spec + ) + static_function = declarative( + attr_func, input_spec=inner_input_spec + ) concrete_program = static_function.concrete_program if static_function._class_instance is None: warnings.warn( - '`jit.save` will only save the `Program`, not the parameters. If you have to save the parameters, please make sure that {} is a member function of `paddle.nn.Layer` and the saved parameters are in `state_dict`' - .format(layer)) + '`jit.save` will only save the `Program`, not the parameters. If you have to save the parameters, please make sure that {} is a member function of `paddle.nn.Layer` and the saved parameters are in `state_dict`'.format( + layer + ) + ) # when save multi `StaticFunction`, all `StaticFunction` share params. dygraph_state_dict = None @@ -974,7 +1070,8 @@ def save(layer, path, input_spec=None, **configs): dygraph_state_dict = inner_layer.to_static_state_dict() elif isinstance(attr_func, StaticFunction): if attr_func._class_instance: - dygraph_state_dict = attr_func._class_instance.to_static_state_dict( + dygraph_state_dict = ( + attr_func._class_instance.to_static_state_dict() ) if dygraph_state_dict: @@ -998,19 +1095,25 @@ def save(layer, path, input_spec=None, **configs): tgt_var.set_vocab(scr_tensor) else: param_or_buffer_tensor = scope.var( - param_or_buffer.name).get_tensor() - #src_tensor = param_or_buffer.value().get_tensor() - src_tensor = state_var_dict[ - param_or_buffer.name].value().get_tensor() + param_or_buffer.name + ).get_tensor() + # src_tensor = param_or_buffer.value().get_tensor() + src_tensor = ( + state_var_dict[param_or_buffer.name] + .value() + .get_tensor() + ) param_or_buffer_tensor._share_data_with(src_tensor) # record var info if param_or_buffer.name not in extra_var_info: extra_info_dict = dict() if param_or_buffer.name in state_names_dict: extra_info_dict['structured_name'] = state_names_dict[ - param_or_buffer.name] + param_or_buffer.name + ] extra_info_dict[ - 'stop_gradient'] = param_or_buffer.stop_gradient + 'stop_gradient' + ] = param_or_buffer.stop_gradient if isinstance(param_or_buffer, (ParamBase, EagerParamBase)): extra_info_dict['trainable'] = param_or_buffer.trainable extra_var_info[param_or_buffer.name] = extra_info_dict @@ -1024,8 +1127,9 @@ def save(layer, path, input_spec=None, **configs): # - prune inputs: # - the input_spec length < len((concrete_program.inputs) - 1 # - the input_spec's name should be in concrete_program.inputs - input_var_names = _get_input_var_names(concrete_program.inputs, - inner_input_spec) + input_var_names = _get_input_var_names( + concrete_program.inputs, inner_input_spec + ) # NOTE(chenweihang): [ Get output variables ] # the rule is like [ Get input variables name ]. For output var, @@ -1033,8 +1137,9 @@ def save(layer, path, input_spec=None, **configs): # var name of output, and we don't recommended to use output_spec # print(concrete_program.main_program) # print(concrete_program.outputs, configs.output_spec) - output_vars = _get_output_vars(concrete_program.outputs, - configs.output_spec, with_hook) + output_vars = _get_output_vars( + concrete_program.outputs, configs.output_spec, with_hook + ) # 5. save inference model from paddle.fluid.io import save_inference_model @@ -1048,7 +1153,9 @@ def save(layer, path, input_spec=None, **configs): params_filename = file_prefix + INFER_PARAMS_SUFFIX else: model_filename = file_prefix + '.' + attr_func + INFER_MODEL_SUFFIX - params_filename = file_prefix + '.' + attr_func + INFER_PARAMS_SUFFIX + params_filename = ( + file_prefix + '.' + attr_func + INFER_PARAMS_SUFFIX + ) with scope_guard(scope): save_inference_model( @@ -1061,12 +1168,14 @@ def save(layer, path, input_spec=None, **configs): params_filename=params_filename, export_for_deployment=configs._export_for_deployment, program_only=configs._program_only, - clip_extra=configs.clip_extra) + clip_extra=configs.clip_extra, + ) if combine_params: clone_main_program = concrete_program.main_program.clone() clone_main_program = clone_main_program._prune_with_input( - input_var_names, output_vars) + input_var_names, output_vars + ) for block in clone_main_program.blocks: combine_vars.update(block.vars) @@ -1080,15 +1189,16 @@ def save(layer, path, input_spec=None, **configs): params_filename = file_prefix + INFER_PARAMS_SUFFIX with scope_guard(scope): - paddle.static.save_vars(Executor(_current_expected_place()), - dirname=model_path, - vars=list( - filter(paddle.fluid.io.is_persistable, - ordered_vars)), - filename=params_filename) + paddle.static.save_vars( + Executor(_current_expected_place()), + dirname=model_path, + vars=list(filter(paddle.fluid.io.is_persistable, ordered_vars)), + filename=params_filename, + ) # save property - property_save_path = os.path.join(os.path.normpath(model_path), - file_prefix + INFER_PROPERTY_SUFFIX) + property_save_path = os.path.join( + os.path.normpath(model_path), file_prefix + INFER_PROPERTY_SUFFIX + ) _save_property(property_save_path, property_vals) # NOTE(chenweihang): [ Save extra variable info ] @@ -1350,11 +1460,9 @@ def load(path, **configs): @dygraph_only -def _trace(layer, - inputs, - feed_prefix='feed_', - fetch_prefix='fetch_', - tmp_prefix='t_'): +def _trace( + layer, inputs, feed_prefix='feed_', fetch_prefix='fetch_', tmp_prefix='t_' +): assert isinstance(layer, Layer) if not isinstance(inputs, (list, tuple)): @@ -1372,8 +1480,14 @@ def _trace(layer, outputs = original_outputs out_vars = extract_vars(outputs, err_tag='outputs') - program_desc, feed_names, fetch_names, parameters = tracer.create_program_desc( - var_list, feed_prefix, out_vars, fetch_prefix, tmp_prefix) + ( + program_desc, + feed_names, + fetch_names, + parameters, + ) = tracer.create_program_desc( + var_list, feed_prefix, out_vars, fetch_prefix, tmp_prefix + ) tracer.reset() with _dygraph_guard(None): @@ -1483,7 +1597,8 @@ class TracedLayer(object): assert isinstance( layer, Layer ), "The type of 'layer' in fluid.dygraph.jit.TracedLayer.trace must be fluid.dygraph.Layer, but received {}.".format( - type(layer)) + type(layer) + ) outs, prog, feed, fetch, parameters = _trace(layer, inputs) traced = TracedLayer(prog, parameters, feed, fetch) return outs, traced @@ -1533,25 +1648,30 @@ class TracedLayer(object): assert isinstance( build_strategy, (type(None), BuildStrategy) ), "The type of 'build_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.BuildStrategy, but received {}.".format( - type(build_strategy)) + type(build_strategy) + ) assert isinstance( exec_strategy, (type(None), ExecutionStrategy) ), "The type of 'exec_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.ExecutionStrategy, but received {}.".format( - type(exec_strategy)) + type(exec_strategy) + ) self._build_strategy = build_strategy self._exec_strategy = exec_strategy @switch_to_static_graph def _compile(self): self._compiled_program = CompiledProgram( - self._program).with_data_parallel( - build_strategy=self._build_strategy, - exec_strategy=self._exec_strategy, - places=self._place) + self._program + ).with_data_parallel( + build_strategy=self._build_strategy, + exec_strategy=self._exec_strategy, + places=self._place, + ) def _build_feed(self, inputs): - assert isinstance(inputs, (list, tuple)), \ - "Inputs should be a list or tuple of variables" + assert isinstance( + inputs, (list, tuple) + ), "Inputs should be a list or tuple of variables" assert len(inputs) == len(self._feed_names) feed_dict = {} if _non_static_mode(): @@ -1565,9 +1685,9 @@ class TracedLayer(object): @switch_to_static_graph def _run(self, feed): - return self._exe.run(self._compiled_program, - feed=feed, - fetch_list=self._fetch_names) + return self._exe.run( + self._compiled_program, feed=feed, fetch_list=self._fetch_names + ) def __call__(self, inputs): with scope_guard(self._scope): @@ -1631,22 +1751,40 @@ class TracedLayer(object): fetch, = exe.run(program, feed={feed_vars[0]: in_np}, fetch_list=fetch_vars) print(fetch.shape) # (2, 10) """ - check_type(path, "path", str, - "fluid.dygraph.jit.TracedLayer.save_inference_model") - check_type(feed, "feed", (type(None), list), - "fluid.dygraph.jit.TracedLayer.save_inference_model") + check_type( + path, + "path", + str, + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) + check_type( + feed, + "feed", + (type(None), list), + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) if isinstance(feed, list): for f in feed: check_type( - f, "each element of feed", int, - "fluid.dygraph.jit.TracedLayer.save_inference_model") - check_type(fetch, "fetch", (type(None), list), - "fluid.dygraph.jit.TracedLayer.save_inference_model") + f, + "each element of feed", + int, + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) + check_type( + fetch, + "fetch", + (type(None), list), + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) if isinstance(fetch, list): for f in fetch: check_type( - f, "each element of fetch", int, - "fluid.dygraph.jit.TracedLayer.save_inference_model") + f, + "each element of fetch", + int, + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) clip_extra = kwargs.get('clip_extra', True) # path check file_prefix = os.path.basename(path) @@ -1654,7 +1792,8 @@ class TracedLayer(object): raise ValueError( "The input path MUST be format of dirname/file_prefix " "[dirname\\file_prefix in Windows system], but received " - "file_prefix is empty string.") + "file_prefix is empty string." + ) dirname = os.path.dirname(path) if dirname and not os.path.exists(dirname): @@ -1680,11 +1819,13 @@ class TracedLayer(object): model_filename = file_prefix + INFER_MODEL_SUFFIX params_filename = file_prefix + INFER_PARAMS_SUFFIX - save_inference_model(dirname=dirname, - feeded_var_names=feeded_var_names, - target_vars=target_vars, - executor=self._exe, - main_program=self._program.clone(), - model_filename=model_filename, - params_filename=params_filename, - clip_extra=clip_extra) + save_inference_model( + dirname=dirname, + feeded_var_names=feeded_var_names, + target_vars=target_vars, + executor=self._exe, + main_program=self._program.clone(), + model_filename=model_filename, + params_filename=params_filename, + clip_extra=clip_extra, + ) diff --git a/python/paddle/fluid/dygraph/layer_hooks.py b/python/paddle/fluid/dygraph/layer_hooks.py index a231e7cf874..8a373cd17c8 100644 --- a/python/paddle/fluid/dygraph/layer_hooks.py +++ b/python/paddle/fluid/dygraph/layer_hooks.py @@ -37,13 +37,16 @@ def record_program_ops_pre_hook(layer, inputs): if not _non_static_mode(): if layer._op_recorder.start < 0: layer._op_recorder.start = len( - default_main_program().current_block().ops) + default_main_program().current_block().ops + ) layer._op_recorder.is_valid = True else: layer._op_recorder.is_valid = False warnings.warn( - "{} has recorded the op information before. Please check whether you call this layer twice." - .format(layer._full_name)) + "{} has recorded the op information before. Please check whether you call this layer twice.".format( + layer._full_name + ) + ) return None @@ -56,7 +59,7 @@ def set_op_customized_attrs_post_hook(layer, inputs, outputs): start = layer._op_recorder.start end = len(default_main_program().current_block().ops) - assert (start >= 0 and end >= start) + assert start >= 0 and end >= start ops = default_main_program().current_block().ops[start:end] layer._op_recorder.end = end diff --git a/python/paddle/fluid/dygraph/layer_object_helper.py b/python/paddle/fluid/dygraph/layer_object_helper.py index bdd34abd5a8..b07798b72e6 100644 --- a/python/paddle/fluid/dygraph/layer_object_helper.py +++ b/python/paddle/fluid/dygraph/layer_object_helper.py @@ -22,16 +22,17 @@ from ..dygraph_utils import _append_activation_in_dygraph class LayerObjectHelper(LayerHelperBase): - def __init__(self, name): super(LayerObjectHelper, self).__init__(name, layer_type=name) - def append_op(self, - type=None, - inputs=None, - outputs=None, - attrs=None, - stop_gradient=None): + def append_op( + self, + type=None, + inputs=None, + outputs=None, + attrs=None, + stop_gradient=None, + ): """append an operator for this layer object. Args: @@ -48,7 +49,8 @@ class LayerObjectHelper(LayerHelperBase): inputs=inputs, outputs=outputs, attrs=attrs, - stop_gradient=stop_gradient) + stop_gradient=stop_gradient, + ) def _multiple_input(self, inputs_in): inputs = inputs_in @@ -73,8 +75,9 @@ class LayerObjectHelper(LayerHelperBase): param_attr = [param_attr] if len(param_attr) != 1 and len(param_attr) != length: - raise ValueError("parameter number mismatch in {}".format( - self.name)) + raise ValueError( + "parameter number mismatch in {}".format(self.name) + ) elif len(param_attr) == 1 and length != 1: tmp = [None] * length for i in range(length): @@ -93,8 +96,9 @@ class LayerObjectHelper(LayerHelperBase): """ param_attr_in = ParamAttr._to_attr(param_attr_in) if isinstance(param_attr_in, bool): - raise ValueError('Param_attr should not be False in {}'.format( - self.name)) + raise ValueError( + 'Param_attr should not be False in {}'.format(self.name) + ) inputs = inputs_in if (inputs_in is not None) else [] inputs = self._multiple_input(inputs) param_attrs = self._multiple_param_attr(len(inputs), param_attr_in) @@ -116,8 +120,10 @@ class LayerObjectHelper(LayerHelperBase): if dtype is None: dtype = each.dtype elif dtype != each.dtype: - raise ValueError("Data Type mismatch: %d to %d in %s" % - (dtype, each.dtype, self.name)) + raise ValueError( + "Data Type mismatch: %d to %d in %s" + % (dtype, each.dtype, self.name) + ) return dtype def get_parameter(self, name): @@ -130,8 +136,9 @@ class LayerObjectHelper(LayerHelperBase): """ param = self.main_program.global_block().var(name) if not isinstance(param, Parameter): - raise ValueError("no Parameter name %s found in %s" % - (name, self.name)) + raise ValueError( + "no Parameter name %s found in %s" % (name, self.name) + ) return param # TODO: this should not be called anymore after all activation func move to Layers @@ -153,7 +160,8 @@ class LayerObjectHelper(LayerHelperBase): act = {'type': act} else: raise TypeError( - str(act) + " should be unicode or str in %s ", self.name) + str(act) + " should be unicode or str in %s ", self.name + ) if (use_cudnn is not None) and use_cudnn: act['use_cudnn'] = use_cudnn @@ -162,15 +170,18 @@ class LayerObjectHelper(LayerHelperBase): act['use_mkldnn'] = use_mkldnn act_type = act.pop('type') if _non_static_mode(): - res = _append_activation_in_dygraph(input_var, act_type, use_cudnn, - use_mkldnn) + res = _append_activation_in_dygraph( + input_var, act_type, use_cudnn, use_mkldnn + ) return res else: tmp = self.create_variable_for_type_inference(dtype=input_var.dtype) - self.append_op(type=act_type, - inputs={"X": [input_var]}, - outputs={"Out": [tmp]}, - attrs=act) + self.append_op( + type=act_type, + inputs={"X": [input_var]}, + outputs={"Out": [tmp]}, + attrs=act, + ) return tmp def is_instance(self, param, cls): @@ -186,4 +197,8 @@ class LayerObjectHelper(LayerHelperBase): if not isinstance(param, cls): raise TypeError( "The input {0} parameter of method {1} must be {2}, in layer {3}", - param, self.layer_type, cls.__name__, self.name) + param, + self.layer_type, + cls.__name__, + self.name, + ) diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py index 29f2e186bd2..889a910d8b9 100644 --- a/python/paddle/fluid/dygraph/layers.py +++ b/python/paddle/fluid/dygraph/layers.py @@ -31,12 +31,25 @@ from . import parallel_helper from .. import unique_name from paddle.fluid import core from .layer_object_helper import LayerObjectHelper -from .layer_hooks import record_program_ops_pre_hook, set_op_customized_attrs_post_hook, LayerOpsRecoder -from .base import program_desc_tracing_guard, param_guard, in_declarative_mode, _convert_into_variable +from .layer_hooks import ( + record_program_ops_pre_hook, + set_op_customized_attrs_post_hook, + LayerOpsRecoder, +) +from .base import ( + program_desc_tracing_guard, + param_guard, + in_declarative_mode, + _convert_into_variable, +) from paddle.fluid import framework from ..param_attr import ParamAttr from paddle.fluid.executor import Executor, global_scope -from paddle.fluid.framework import _non_static_mode, convert_np_dtype_to_dtype_, in_dygraph_mode +from paddle.fluid.framework import ( + _non_static_mode, + convert_np_dtype_to_dtype_, + in_dygraph_mode, +) from paddle.fluid.framework import Program, program_guard from paddle.fluid.framework import _current_expected_place as _get_device from paddle.fluid.core import VarDesc @@ -66,7 +79,7 @@ def _addindent(string, indent): class HookRemoveHelper(object): - """ A HookRemoveHelper that can be used to remove hook. """ + """A HookRemoveHelper that can be used to remove hook.""" next_hook_id = 0 @@ -397,12 +410,14 @@ class Layer(object): self._forward_pre_hooks[hook_remove_helper._hook_id] = hook return hook_remove_helper - def create_parameter(self, - shape, - attr=None, - dtype=None, - is_bias=False, - default_initializer=None): + def create_parameter( + self, + shape, + attr=None, + dtype=None, + is_bias=False, + default_initializer=None, + ): """Create parameters for this layer. Parameters: @@ -442,12 +457,15 @@ class Layer(object): temp_attr = copy.deepcopy(attr) if isinstance(temp_attr, str) and temp_attr == "": temp_attr = None - return self._helper.create_parameter(temp_attr, shape, dtype, is_bias, - default_initializer) - - @deprecated(since="2.0.0", - update_to="paddle.nn.Layer.create_tensor", - reason="New api in create_tensor, easier to use.") + return self._helper.create_parameter( + temp_attr, shape, dtype, is_bias, default_initializer + ) + + @deprecated( + since="2.0.0", + update_to="paddle.nn.Layer.create_tensor", + reason="New api in create_tensor, easier to use.", + ) def create_variable(self, name=None, persistable=None, dtype=None): """ @@ -487,14 +505,16 @@ class Layer(object): if name is not None: var_name = ".".join([self._full_name, name]) else: - var_name = unique_name.generate(".".join( - [self._full_name, "_generated_var"])) + var_name = unique_name.generate( + ".".join([self._full_name, "_generated_var"]) + ) return self._helper.main_program.current_block().create_var( name=var_name, persistable=persistable, dtype=dtype, - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) # TODO: Add more parameter list when we need them def create_tensor(self, name=None, persistable=None, dtype=None): @@ -537,14 +557,16 @@ class Layer(object): if name is not None: var_name = ".".join([self._full_name, name]) else: - var_name = unique_name.generate(".".join( - [self._full_name, "_generated_var"])) + var_name = unique_name.generate( + ".".join([self._full_name, "_generated_var"]) + ) return self._helper.main_program.current_block().create_var( name=var_name, persistable=persistable, dtype=dtype, - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) def parameters(self, include_sublayers=True): """Returns a list of all Parameters from current layer and its sub-layers. @@ -562,8 +584,10 @@ class Layer(object): """ ret = [ - param for _, param in self.named_parameters( - include_sublayers=include_sublayers) + param + for _, param in self.named_parameters( + include_sublayers=include_sublayers + ) ] return ret @@ -677,9 +701,11 @@ class Layer(object): """ params_set = set() - named_sublayers = self.named_sublayers( - prefix=prefix, include_self=True) if include_sublayers else zip( - [prefix], [self]) + named_sublayers = ( + self.named_sublayers(prefix=prefix, include_self=True) + if include_sublayers + else zip([prefix], [self]) + ) for layer_prefix, sublayer in named_sublayers: params = sublayer._parameters.items() for key, param in params: @@ -723,9 +749,9 @@ class Layer(object): if layer is None: continue layer_prefix = prefix + ('.' if prefix else '') + key - for p, l in layer.named_sublayers(prefix=layer_prefix, - include_self=True, - layers_set=layers_set): + for p, l in layer.named_sublayers( + prefix=layer_prefix, include_self=True, layers_set=layers_set + ): yield p, l def register_buffer(self, name, tensor, persistable=True): @@ -768,25 +794,32 @@ class Layer(object): if '_buffers' not in self.__dict__: raise ValueError( - "super(YourLayer, self).__init__() should be called first") + "super(YourLayer, self).__init__() should be called first" + ) elif not isinstance(name, str): raise TypeError( - "The name of buffer should be a string, but received {}.". - format(type(name).__name__)) + "The name of buffer should be a string, but received {}.".format( + type(name).__name__ + ) + ) elif '.' in name: raise KeyError( "The name of buffer can not contain `.`, " "because when you access the newly added buffer in the " - "form of `self.**.**`, it will cause AttributeError.") + "form of `self.**.**`, it will cause AttributeError." + ) elif name == '': raise KeyError("The name of buffer can not be empty.") elif hasattr(self, name) and name not in self._buffers: raise KeyError("attribute '{}' already exists.".format(name)) - elif tensor is not None and not (type(tensor) == core.VarBase - or type(tensor) == core.eager.Tensor): + elif tensor is not None and not ( + type(tensor) == core.VarBase or type(tensor) == core.eager.Tensor + ): raise TypeError( - "The registered buffer should be a Paddle.Tensor, but received {}." - .format(type(tensor).__name__)) + "The registered buffer should be a Paddle.Tensor, but received {}.".format( + type(tensor).__name__ + ) + ) else: self._buffers[name] = tensor if persistable: @@ -819,8 +852,10 @@ class Layer(object): """ ret = [ - buffer for _, buffer in self.named_buffers( - include_sublayers=include_sublayers) + buffer + for _, buffer in self.named_buffers( + include_sublayers=include_sublayers + ) ] return ret @@ -861,9 +896,11 @@ class Layer(object): """ buffers_set = set() - named_sublayers = self.named_sublayers( - prefix=prefix, include_self=True) if include_sublayers else zip( - [prefix], [self]) + named_sublayers = ( + self.named_sublayers(prefix=prefix, include_self=True) + if include_sublayers + else zip([prefix], [self]) + ) for layer_prefix, sublayer in named_sublayers: buffers = sublayer._buffers.items() for key, buffer in buffers: @@ -909,7 +946,7 @@ class Layer(object): hook_result = forward_pre_hook(self, inputs) if hook_result is not None: if not isinstance(hook_result, tuple): - hook_result = (hook_result, ) + hook_result = (hook_result,) inputs = hook_result if not self._built: @@ -919,16 +956,20 @@ class Layer(object): # TODO(liuyuhui) Only xpu broadcast parameters here. # The other device is to call _sync_params_buffers in DataParallel # to realize the parameter synchronization among multiply cards. - if parallel_helper._is_data_parallel_mode( - ) and paddle.is_compiled_with_xpu(): + if ( + parallel_helper._is_data_parallel_mode() + and paddle.is_compiled_with_xpu() + ): parallel_helper._broadcast_parameters( - self._parameters.values()) + self._parameters.values() + ) self._built = True if in_profiler_mode(): - with profiler.RecordEvent(self.__class__.__name__, - profiler.TracerEventType.Forward): + with profiler.RecordEvent( + self.__class__.__name__, profiler.TracerEventType.Forward + ): outputs = self.forward(*inputs, **kwargs) else: outputs = self.forward(*inputs, **kwargs) @@ -941,8 +982,14 @@ class Layer(object): return outputs def __call__(self, *inputs, **kwargs): - if (not in_declarative_mode()) and (not self._forward_pre_hooks) \ - and (not self._forward_post_hooks) and (not self._built) and in_dygraph_mode() and (not in_profiler_mode()): + if ( + (not in_declarative_mode()) + and (not self._forward_pre_hooks) + and (not self._forward_post_hooks) + and (not self._built) + and in_dygraph_mode() + and (not in_profiler_mode()) + ): self._build_once(*inputs, **kwargs) return self.forward(*inputs, **kwargs) else: @@ -999,7 +1046,7 @@ class Layer(object): for prefix, layer in model.named_sublayers(): print(prefix, layer) """ - assert (isinstance(sublayer, Layer) or sublayer == None) + assert isinstance(sublayer, Layer) or sublayer == None self._sub_layers[name] = sublayer return sublayer @@ -1036,32 +1083,42 @@ class Layer(object): """ if '_parameters' not in self.__dict__: raise RuntimeError( - "super(YourLayer, self).__init__() should be called firstly.") + "super(YourLayer, self).__init__() should be called firstly." + ) elif not isinstance(name, str): raise TypeError( - "The name of parameter should be a string, but received {}.". - format(type(name).__name__)) + "The name of parameter should be a string, but received {}.".format( + type(name).__name__ + ) + ) elif '.' in name: raise KeyError( "The name of parameter can not contain `.`, " "because when you access the newly added parameter in the " - "form of `self.**.**`, it will cause AttributeError.") + "form of `self.**.**`, it will cause AttributeError." + ) elif name == '': raise KeyError("The name of parameter can not be empty.") elif hasattr(self, name) and name not in self._parameters: raise KeyError("The parameter '{}' already exists.".format(name)) - elif parameter is not None and not isinstance(parameter, - framework.Parameter): + elif parameter is not None and not isinstance( + parameter, framework.Parameter + ): raise TypeError( - "The parameter to be added should be a Parameter, but received {}." - .format(type(parameter).__name__)) + "The parameter to be added should be a Parameter, but received {}.".format( + type(parameter).__name__ + ) + ) else: if parameter is None: self._parameters[name] = None if len(self._loaddict_holder) > 0: - assert parameter.name in self._loaddict_holder, "Parameter not found, Can't not find [ {} ] in state_dict".format( - parameter.name) + assert ( + parameter.name in self._loaddict_holder + ), "Parameter not found, Can't not find [ {} ] in state_dict".format( + parameter.name + ) parameter.set_value(self._loaddict_holder[parameter.name]) @@ -1080,37 +1137,50 @@ class Layer(object): """ def is_already_registered(is_pre_hook): - layers_hooks = self._forward_pre_hooks if is_pre_hook else self._forward_post_hooks - candidate_hook = record_program_ops_pre_hook if is_pre_hook else set_op_customized_attrs_post_hook + layers_hooks = ( + self._forward_pre_hooks + if is_pre_hook + else self._forward_post_hooks + ) + candidate_hook = ( + record_program_ops_pre_hook + if is_pre_hook + else set_op_customized_attrs_post_hook + ) already_registed = False if layers_hooks: last_key = next(reversed(layers_hooks)) - already_registed = (layers_hooks[last_key] == candidate_hook) + already_registed = layers_hooks[last_key] == candidate_hook return already_registed if not isinstance(attrs, dict): raise TypeError( "attrs should be type(dict), but received {}".format( - type(attrs).__name__)) + type(attrs).__name__ + ) + ) # NOTE: Overwrite behavior for same key. self._customized_attrs.update(attrs) if not is_already_registered(is_pre_hook=True): pre_hook_helper = self.register_forward_pre_hook( - record_program_ops_pre_hook) + record_program_ops_pre_hook + ) assert len(self._op_recorder.hooks) == 0 self._op_recorder.hooks = [pre_hook_helper] # manually register post_hook to ensure it is inserted into the head. if not is_already_registered(is_pre_hook=False): post_hook_helper = self.register_forward_post_hook( - set_op_customized_attrs_post_hook) + set_op_customized_attrs_post_hook + ) if len(self._forward_post_hooks) > 1: - self._forward_post_hooks.move_to_end(post_hook_helper._hook_id, - last=False) + self._forward_post_hooks.move_to_end( + post_hook_helper._hook_id, last=False + ) assert len(self._op_recorder.hooks) == 1 @@ -1143,7 +1213,6 @@ class Layer(object): return object.__getattribute__(self, name) def __setattr__(self, name, value): - def _remove_if_exist(*dicts): for d in dicts: if name in d: @@ -1155,10 +1224,14 @@ class Layer(object): if isinstance(value, framework.Parameter): if params is None: raise ValueError( - "super(YourLayer, self).__init__() should be called first") + "super(YourLayer, self).__init__() should be called first" + ) if len(self._loaddict_holder) > 0: - assert value.name in self._loaddict_holder, "Parameter not found, Can't not find [ {} ] in state_dict".format( - value.name) + assert ( + value.name in self._loaddict_holder + ), "Parameter not found, Can't not find [ {} ] in state_dict".format( + value.name + ) value.set_value(self._loaddict_holder[value.name]) @@ -1167,9 +1240,10 @@ class Layer(object): elif params is not None and name in params: if value is not None: raise TypeError( - "assignment to parameter '{}' should be of type Parameter or None, but got '{}'" - .format(name, - type(value).__name__)) + "assignment to parameter '{}' should be of type Parameter or None, but got '{}'".format( + name, type(value).__name__ + ) + ) params[name] = None else: layers = self.__dict__.get('_sub_layers', None) @@ -1184,9 +1258,10 @@ class Layer(object): elif layers is not None and name in layers: if value is not None: raise TypeError( - "assignment to sublayer '{}' should be of type Layer or None, but got '{}'" - .format(name, - type(value).__name__)) + "assignment to sublayer '{}' should be of type Layer or None, but got '{}'".format( + name, type(value).__name__ + ) + ) layers[name] = None else: _buffers = self.__dict__.get('_buffers', None) @@ -1195,8 +1270,9 @@ class Layer(object): raise ValueError( "super(YourLayer, self).__init__() should be called first" ) - _remove_if_exist(self.__dict__, self._parameters, - self._sub_layers) + _remove_if_exist( + self.__dict__, self._parameters, self._sub_layers + ) # Set persistable=False by default. Only `register_buffer` can # add a persistable buffer. if name not in self._buffers: @@ -1210,6 +1286,7 @@ class Layer(object): # value via `assign`. if type(value) == framework.Variable: from paddle import assign + # Note(zhhsplendid): the condition below happens in PaddleGan model, # but should all non-Variable _buffers[name] be re-assign? We # should consider it in the future. I current wrote this as @@ -1217,18 +1294,23 @@ class Layer(object): if in_declarative_mode() and _buffers[name] is None: raise RuntimeError( 'In Dy2stat, self.{0} is a buffer and self.{0} is ' - 'not allowed to be set to Variable when self.{0} is None.' - .format(name)) - elif _buffers[name] is None or type(getattr( - self, name)) == core.VarBase: + 'not allowed to be set to Variable when self.{0} is None.'.format( + name + ) + ) + elif ( + _buffers[name] is None + or type(getattr(self, name)) == core.VarBase + ): _buffers[name] = assign(value) else: assign(value, getattr(self, name)) elif value is not None: raise TypeError( - "assignment to buffers '{}' should be of type core.VarBase or None, but got '{}'" - .format(name, - type(value).__name__)) + "assignment to buffers '{}' should be of type core.VarBase or None, but got '{}'".format( + name, type(value).__name__ + ) + ) else: # Assigning None will remove the buffer, but if re-assign a new varBase to it, # it will be remarked as a buffer with same `persistable` attribute. @@ -1315,10 +1397,12 @@ class Layer(object): self._state_dict_hooks[hook_remove_helper._hook_id] = hook return hook_remove_helper - def _obtain_parameters_buffers(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): + def _obtain_parameters_buffers( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): """ The difference from state_dict() is that state_dict_hook will not be called, but the original types of parameters and buffers will be maintained. @@ -1329,7 +1413,10 @@ class Layer(object): if data is not None: destination[structured_name_prefix + name] = data for name, buffer in self._buffers.items(): - if buffer is not None and name not in self._non_persistable_buffer_names_set: + if ( + buffer is not None + and name not in self._non_persistable_buffer_names_set + ): destination[structured_name_prefix + name] = buffer if include_sublayers: @@ -1338,17 +1425,22 @@ class Layer(object): destination_temp = destination.copy() destination_temp.update( layer_item._obtain_parameters_buffers( - destination_temp, include_sublayers, - structured_name_prefix + layer_name + ".")) + destination_temp, + include_sublayers, + structured_name_prefix + layer_name + ".", + ) + ) destination = destination_temp return destination - def _state_dict_impl(self, - destination=None, - include_sublayers=True, - structured_name_prefix="", - include_non_persistable_buffer=False, - use_hook=True): + def _state_dict_impl( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + include_non_persistable_buffer=False, + use_hook=True, + ): """ Get all parameters and persistable buffers of current layer and its sub-layers. And set them into a dict @@ -1366,7 +1458,10 @@ class Layer(object): destination[structured_name_prefix + name] = data for name, buffer in self._buffers.items(): if not include_non_persistable_buffer: - if buffer is not None and name not in self._non_persistable_buffer_names_set: + if ( + buffer is not None + and name not in self._non_persistable_buffer_names_set + ): destination[structured_name_prefix + name] = buffer else: if buffer is not None: @@ -1378,9 +1473,13 @@ class Layer(object): destination_temp = destination.copy() destination_temp.update( layer_item._state_dict_impl( - destination_temp, include_sublayers, + destination_temp, + include_sublayers, structured_name_prefix + layer_name + ".", - include_non_persistable_buffer, use_hook)) + include_non_persistable_buffer, + use_hook, + ) + ) destination = destination_temp if use_hook: for state_dict_hook in self._state_dict_hooks.values(): @@ -1390,11 +1489,13 @@ class Layer(object): return destination - def to_static_state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix="", - use_hook=True): + def to_static_state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + use_hook=True, + ): ''' Get all parameters and buffers of current layer and its sub-layers. And set them into a dict @@ -1422,13 +1523,16 @@ class Layer(object): include_sublayers=include_sublayers, structured_name_prefix=structured_name_prefix, include_non_persistable_buffer=True, - use_hook=use_hook) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix="", - use_hook=True): + use_hook=use_hook, + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + use_hook=True, + ): ''' Get all parameters and persistable buffers of current layer and its sub-layers. And set them into a dict @@ -1456,7 +1560,8 @@ class Layer(object): include_sublayers=include_sublayers, structured_name_prefix=structured_name_prefix, include_non_persistable_buffer=False, - use_hook=use_hook) + use_hook=use_hook, + ) @framework.deprecate_stat_dict def set_state_dict(self, state_dict, use_structured_name=True): @@ -1488,22 +1593,31 @@ class Layer(object): state = state_dict.get(key, None) if state is None: raise ValueError( - "{} is not found in the provided dict.".format(key)) - if (isinstance(state, dict) or isinstance(state, list)): - if (len(state) != len(param)): - raise ValueError("{} receieves the length of {}, " - "but the expected shape is {}".format( - key, len(state), len(param))) + "{} is not found in the provided dict.".format(key) + ) + if isinstance(state, dict) or isinstance(state, list): + if len(state) != len(param): + raise ValueError( + "{} receieves the length of {}, " + "but the expected shape is {}".format( + key, len(state), len(param) + ) + ) else: return param, state else: - state_shape = state.shape() if inspect.ismethod( - state.shape) else state.shape + state_shape = ( + state.shape() + if inspect.ismethod(state.shape) + else state.shape + ) if list(state_shape) != list(param.shape): raise ValueError( - "{} receives a shape {}, but the expected shape is {}.". - format(key, list(state_shape), list(param.shape))) + "{} receives a shape {}, but the expected shape is {}.".format( + key, list(state_shape), list(param.shape) + ) + ) return param, state matched_param_state = [] @@ -1542,7 +1656,9 @@ class Layer(object): # restore parameter states core._create_loaded_parameter( [param for param, state in matched_param_state], - global_scope(), executor) + global_scope(), + executor, + ) for param, state in matched_param_state: _set_var(param, state) except ValueError as e: @@ -1598,11 +1714,13 @@ class Layer(object): # [ 0.33960250, 0.96878713]]) ''' - return self._to_impl(device=device, - dtype=dtype, - blocking=blocking, - include_sublayers=True, - floating_only=False) + return self._to_impl( + device=device, + dtype=dtype, + blocking=blocking, + include_sublayers=True, + floating_only=False, + ) def _apply(self, func, device, dtype, blocking, include_sublayers=True): if include_sublayers: @@ -1616,8 +1734,9 @@ class Layer(object): if param.grad is not None: with no_grad(): - grad_applied = func(param._grad_ivar(), device, dtype, - blocking) + grad_applied = func( + param._grad_ivar(), device, dtype, blocking + ) for key, buf in self._buffers.items(): if buf is not None: @@ -1641,12 +1760,14 @@ class Layer(object): # Note(zhangbo): Paddle GPU minimum memory allocation unit is 256 bytes, waiting_alloc_memory will comput ‘t’ occupied memory space. # Coefficient 1.2 is used to avoid OOM that may occur in this critical state when the memory is just enough. waiting_alloc_memory = ( - (np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 + ((np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 + ) gpu_memory_available = core.gpu_memory_available() if gpu_memory_available < waiting_alloc_memory: # Copy param / Tensor to cpu - t_used = t._copy_to(paddle.CPUPlace(), - blocking) # k-v type will error + t_used = t._copy_to( + paddle.CPUPlace(), blocking + ) # k-v type will error # Release mem of t t.value().get_tensor()._clear() else: @@ -1657,7 +1778,8 @@ class Layer(object): # 2. cast param / Tensor to dtype if dtype is not None and dtype != t_used.dtype: with paddle.fluid.framework._dygraph_place_guard( - place=t_used.place): + place=t_used.place + ): t_casted = t_used.cast(dtype=dtype) else: t_casted = t_used @@ -1675,12 +1797,14 @@ class Layer(object): return t - def _to_impl(self, - device=None, - dtype=None, - blocking=None, - include_sublayers=True, - floating_only=False): + def _to_impl( + self, + device=None, + dtype=None, + blocking=None, + include_sublayers=True, + floating_only=False, + ): ''' Cast the parameters and buffers of Layer by the give device, dtype and blocking. @@ -1709,20 +1833,28 @@ class Layer(object): if device is not None: if isinstance(device, str): device = paddle.device._convert_to_place(device) - elif isinstance(device, (core.CPUPlace, core.CUDAPlace, - core.CUDAPinnedPlace, core.XPUPlace)): + elif isinstance( + device, + ( + core.CPUPlace, + core.CUDAPlace, + core.CUDAPinnedPlace, + core.XPUPlace, + ), + ): pass else: raise ValueError( "device value error, must be str, paddle.CPUPlace(), paddle.CUDAPlace(), paddle.CUDAPinnedPlace() or paddle.XPUPlace(), but the type of device is " - + type(device).__name__) + + type(device).__name__ + ) if blocking is None: blocking = True else: assert isinstance( - blocking, - bool), "blocking value error, must be the True, False or None" + blocking, bool + ), "blocking value error, must be the True, False or None" def transform(t, device, dtype, blocking): if floating_only and (not paddle.is_floating_point(t)): diff --git a/python/paddle/fluid/dygraph/learning_rate_scheduler.py b/python/paddle/fluid/dygraph/learning_rate_scheduler.py index 0620425910e..65219a9b176 100644 --- a/python/paddle/fluid/dygraph/learning_rate_scheduler.py +++ b/python/paddle/fluid/dygraph/learning_rate_scheduler.py @@ -20,9 +20,18 @@ from ..framework import Variable from ..data_feeder import check_type __all__ = [ - 'NoamDecay', 'PiecewiseDecay', 'NaturalExpDecay', 'ExponentialDecay', - 'InverseTimeDecay', 'PolynomialDecay', 'CosineDecay', 'LinearLrWarmup', - 'ReduceLROnPlateau', 'StepDecay', 'MultiStepDecay', 'LambdaDecay' + 'NoamDecay', + 'PiecewiseDecay', + 'NaturalExpDecay', + 'ExponentialDecay', + 'InverseTimeDecay', + 'PolynomialDecay', + 'CosineDecay', + 'LinearLrWarmup', + 'ReduceLROnPlateau', + 'StepDecay', + 'MultiStepDecay', + 'LambdaDecay', ] @@ -57,12 +66,14 @@ class LearningRateDecay(object): learning rate variable """ from .. import layers + lr = layers.create_global_var( name=unique_name.generate("learning_rate"), shape=[1], value=float(lr), dtype=self.dtype, - persistable=False) + persistable=False, + ) return lr # Note: If you want to change what optimizer.state_dict stores, just overwrite this functions, @@ -83,7 +94,8 @@ class LearningRateDecay(object): assert value.shape == [ 1 ], "shape of Variable in state_dict must be [1] {}".format( - value.shape) + value.shape + ) value = value.numpy()[0] state_dict[key] = value @@ -105,8 +117,10 @@ class LearningRateDecay(object): self.__dict__[key] = state_dict[key] else: raise RuntimeError( - "Please check whether state_dict is correct for optimizer. Can't find [ {} ] in state_dict" - .format(key)) + "Please check whether state_dict is correct for optimizer. Can't find [ {} ] in state_dict".format( + key + ) + ) if len(state_dict) > len(self.keys): warnings.warn( "There are some unused values in state_dict. Maybe the optimizer have different 'LearningRateDecay' when invoking state_dict and set_dict" @@ -238,14 +252,16 @@ class NaturalExpDecay(LearningRateDecay): """ - def __init__(self, - learning_rate, - decay_steps, - decay_rate, - staircase=False, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + decay_steps, + decay_rate, + staircase=False, + begin=0, + step=1, + dtype='float32', + ): super(NaturalExpDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.decay_steps = decay_steps @@ -254,11 +270,13 @@ class NaturalExpDecay(LearningRateDecay): def step(self): from .. import layers + div_res = self.create_lr_var(self.step_num / self.decay_steps) if self.staircase: div_res = layers.floor(div_res) decayed_lr = self.learning_rate * layers.exp( - -1 * self.decay_rate * div_res) + -1 * self.decay_rate * div_res + ) return decayed_lr @@ -312,22 +330,24 @@ class ExponentialDecay(LearningRateDecay): base_lr = 0.1 with fluid.dygraph.guard(): sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.dygraph.ExponentialDecay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.dygraph.ExponentialDecay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) """ - def __init__(self, - learning_rate, - decay_steps, - decay_rate, - staircase=False, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + decay_steps, + decay_rate, + staircase=False, + begin=0, + step=1, + dtype='float32', + ): super(ExponentialDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.decay_steps = decay_steps @@ -336,6 +356,7 @@ class ExponentialDecay(LearningRateDecay): def step(self): from .. import layers + div_res = self.create_lr_var(self.step_num / self.decay_steps) if self.staircase: div_res = layers.floor(div_res) @@ -389,23 +410,25 @@ class InverseTimeDecay(LearningRateDecay): with fluid.dygraph.guard(): emb = fluid.dygraph.Embedding([10, 10]) sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.dygraph.InverseTimeDecay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True), + learning_rate=fluid.dygraph.InverseTimeDecay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True), parameter_list = emb.parameters()) """ - def __init__(self, - learning_rate, - decay_steps, - decay_rate, - staircase=False, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + decay_steps, + decay_rate, + staircase=False, + begin=0, + step=1, + dtype='float32', + ): super(InverseTimeDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.decay_steps = decay_steps @@ -414,6 +437,7 @@ class InverseTimeDecay(LearningRateDecay): def step(self): from .. import layers + div_res = self.create_lr_var(self.step_num / self.decay_steps) if self.staircase: div_res = layers.floor(div_res) @@ -480,15 +504,17 @@ class PolynomialDecay(LearningRateDecay): """ - def __init__(self, - learning_rate, - decay_steps, - end_learning_rate=0.0001, - power=1.0, - cycle=False, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + decay_steps, + end_learning_rate=0.0001, + power=1.0, + cycle=False, + begin=0, + step=1, + dtype='float32', + ): super(PolynomialDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.decay_steps = decay_steps @@ -498,22 +524,27 @@ class PolynomialDecay(LearningRateDecay): def step(self): from .. import layers + tmp_step_num = self.step_num tmp_decay_steps = self.decay_steps if self.cycle: div_res = layers.ceil( - self.create_lr_var(tmp_step_num / float(self.decay_steps))) + self.create_lr_var(tmp_step_num / float(self.decay_steps)) + ) if tmp_step_num == 0: div_res = self.create_lr_var(1.0) tmp_decay_steps = self.decay_steps * div_res else: tmp_step_num = self.create_lr_var( - tmp_step_num if tmp_step_num < self.decay_steps else self. - decay_steps) + tmp_step_num + if tmp_step_num < self.decay_steps + else self.decay_steps + ) - decayed_lr = (self.learning_rate - self.end_learning_rate) * \ - ((1 - tmp_step_num / tmp_decay_steps) ** self.power) + self.end_learning_rate + decayed_lr = (self.learning_rate - self.end_learning_rate) * ( + (1 - tmp_step_num / tmp_decay_steps) ** self.power + ) + self.end_learning_rate return decayed_lr @@ -545,22 +576,24 @@ class CosineDecay(LearningRateDecay): None. Examples: - .. code-block:: python + .. code-block:: python - base_lr = 0.1 + base_lr = 0.1 with fluid.dygraph.guard(): optimizer = fluid.optimizer.SGD( - learning_rate = fluid.dygraph.CosineDecay( - base_lr, 10000, 120) ) + learning_rate = fluid.dygraph.CosineDecay( + base_lr, 10000, 120) ) """ - def __init__(self, - learning_rate, - step_each_epoch, - epochs, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + step_each_epoch, + epochs, + begin=0, + step=1, + dtype='float32', + ): super(CosineDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.step_each_epoch = step_each_epoch @@ -568,10 +601,15 @@ class CosineDecay(LearningRateDecay): def step(self): from .. import layers + cur_epoch = layers.floor( - self.create_lr_var(self.step_num / self.step_each_epoch)) - decayed_lr = self.learning_rate * 0.5 * ( - layers.cos(cur_epoch * math.pi / self.epochs) + 1) + self.create_lr_var(self.step_num / self.step_each_epoch) + ) + decayed_lr = ( + self.learning_rate + * 0.5 + * (layers.cos(cur_epoch * math.pi / self.epochs) + 1) + ) return decayed_lr @@ -621,13 +659,15 @@ class NoamDecay(LearningRateDecay): parameter_list = emb.parameters()) """ - def __init__(self, - d_model, - warmup_steps, - begin=1, - step=1, - dtype='float32', - learning_rate=1.0): + def __init__( + self, + d_model, + warmup_steps, + begin=1, + step=1, + dtype='float32', + learning_rate=1.0, + ): super(NoamDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.d_model = d_model @@ -635,10 +675,14 @@ class NoamDecay(LearningRateDecay): def step(self): from .. import layers + a = self.create_lr_var(self.step_num**-0.5) b = self.create_lr_var((self.warmup_steps**-1.5) * self.step_num) - lr_value = self.learning_rate * (self.d_model** - -0.5) * layers.elementwise_min(a, b) + lr_value = ( + self.learning_rate + * (self.d_model**-0.5) + * layers.elementwise_min(a, b) + ) return lr_value @@ -698,28 +742,37 @@ class LinearLrWarmup(LearningRateDecay): """ - def __init__(self, - learning_rate, - warmup_steps, - start_lr, - end_lr, - begin=1, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + warmup_steps, + start_lr, + end_lr, + begin=1, + step=1, + dtype='float32', + ): super(LinearLrWarmup, self).__init__(begin, step, dtype) - type_check = isinstance(learning_rate, float) or isinstance( - learning_rate, int) or isinstance(learning_rate, LearningRateDecay) + type_check = ( + isinstance(learning_rate, float) + or isinstance(learning_rate, int) + or isinstance(learning_rate, LearningRateDecay) + ) if not type_check: raise TypeError( - "the type of learning_rate should be [int, float or LearningRateDecay], the current type is {}" - .format(learning_rate)) + "the type of learning_rate should be [int, float or LearningRateDecay], the current type is {}".format( + learning_rate + ) + ) self.learning_rate = learning_rate self.warmup_steps = warmup_steps self.start_lr = start_lr - assert end_lr > start_lr, "end_lr {} must be greater than start_lr {}".format( - end_lr, start_lr) - self.lr_ratio_before_warmup = (float(end_lr) - - float(start_lr)) / float(warmup_steps) + assert ( + end_lr > start_lr + ), "end_lr {} must be greater than start_lr {}".format(end_lr, start_lr) + self.lr_ratio_before_warmup = (float(end_lr) - float(start_lr)) / float( + warmup_steps + ) def step(self): base_lr = self.learning_rate @@ -727,6 +780,7 @@ class LinearLrWarmup(LearningRateDecay): base_lr = base_lr() from .. import layers + if self.step_num < self.warmup_steps: return self.lr_ratio_before_warmup * self.step_num + self.start_lr else: @@ -812,18 +866,20 @@ class ReduceLROnPlateau(LearningRateDecay): """ - def __init__(self, - learning_rate, - mode='min', - decay_rate=0.1, - patience=10, - verbose=False, - threshold=1e-4, - threshold_mode='rel', - cooldown=0, - min_lr=0, - eps=1e-8, - dtype='float32'): + def __init__( + self, + learning_rate, + mode='min', + decay_rate=0.1, + patience=10, + verbose=False, + threshold=1e-4, + threshold_mode='rel', + cooldown=0, + min_lr=0, + eps=1e-8, + dtype='float32', + ): super(ReduceLROnPlateau, self).__init__(dtype=dtype) mode = mode.lower() if mode not in ['min', 'max']: @@ -838,15 +894,21 @@ class ReduceLROnPlateau(LearningRateDecay): threshold_mode = threshold_mode.lower() if threshold_mode not in ['rel', 'abs']: - raise ValueError('threshold mode ' + threshold_mode + - ' is unknown!') + raise ValueError( + 'threshold mode ' + threshold_mode + ' is unknown!' + ) self.threshold_mode = threshold_mode - check_type(learning_rate, 'learning_rate', (float, int, Variable), - 'ReduceLROnPlateau') + check_type( + learning_rate, + 'learning_rate', + (float, int, Variable), + 'ReduceLROnPlateau', + ) if not isinstance(learning_rate, (float, int, Variable)): raise TypeError( "The type of 'learning_rate' in 'ReduceLROnPlateau' must be 'float, int, Variable', but received %s." - % type(learning_rate)) + % type(learning_rate) + ) self.learning_rate = learning_rate self.verbose = verbose @@ -865,8 +927,11 @@ class ReduceLROnPlateau(LearningRateDecay): # "cooldown_counter / best_loss / num_bad_epochs / epoch_num / learning_rate" will be stored. def _state_keys(self): self.keys = [ - 'cooldown_counter', 'best_loss', 'num_bad_epochs', 'epoch_num', - 'learning_rate' + 'cooldown_counter', + 'best_loss', + 'num_bad_epochs', + 'epoch_num', + 'learning_rate', ] def __call__(self): @@ -893,9 +958,13 @@ class ReduceLROnPlateau(LearningRateDecay): # loss must be 1-D Tensor with shape [1] check_type(loss, 'loss', Variable, 'ReduceLROnPlateau.step') - assert len(loss.shape) == 1 and loss.shape[0] == 1, "the loss.shape " \ - "should be (1L,), but the current loss.shape is {}. Maybe that " \ - "you should call paddle.mean to process it first.".format(loss.shape) + assert len(loss.shape) == 1 and loss.shape[0] == 1, ( + "the loss.shape " + "should be (1L,), but the current loss.shape is {}. Maybe that " + "you should call paddle.mean to process it first.".format( + loss.shape + ) + ) self.epoch_num += 1 if self.cooldown_counter > 0: @@ -909,18 +978,24 @@ class ReduceLROnPlateau(LearningRateDecay): if self.num_bad_epochs > self.patience: from .. import layers + self.cooldown_counter = self.cooldown self.num_bad_epochs = 0 new_lr = layers.elementwise_max( - self.learning_rate * self.decay_rate, self.min_lr) + self.learning_rate * self.decay_rate, self.min_lr + ) if self.learning_rate - new_lr > self.eps: if self.verbose: - old_lr = self.learning_rate.numpy()[0] if isinstance( - self.learning_rate, - Variable) else self.learning_rate - print('Epoch {}: reducing learning rate from {} to {}.'. - format(self.epoch_num, old_lr, - new_lr.numpy()[0])) + old_lr = ( + self.learning_rate.numpy()[0] + if isinstance(self.learning_rate, Variable) + else self.learning_rate + ) + print( + 'Epoch {}: reducing learning rate from {} to {}.'.format( + self.epoch_num, old_lr, new_lr.numpy()[0] + ) + ) self.learning_rate = new_lr def _is_better(self, current, best): @@ -952,7 +1027,8 @@ class _LearningRateEpochDecay(LearningRateDecay): if not isinstance(learning_rate, (float, int)): raise TypeError( "The type of 'learning_rate' must be 'float, int', but received %s." - % type(learning_rate)) + % type(learning_rate) + ) if learning_rate < 0: raise ValueError("Invalid learning rate: {}".format(learning_rate)) @@ -1057,8 +1133,9 @@ class StepDecay(_LearningRateEpochDecay): def __init__(self, learning_rate, step_size, decay_rate=0.1): if not isinstance(step_size, int): raise TypeError( - "The type of 'step_size' must be 'int', but received %s." % - type(step_size)) + "The type of 'step_size' must be 'int', but received %s." + % type(step_size) + ) if decay_rate >= 1.0: raise ValueError('decay_rate should be < 1.0.') @@ -1134,12 +1211,15 @@ class MultiStepDecay(_LearningRateEpochDecay): if not isinstance(milestones, (tuple, list)): raise TypeError( "The type of 'milestones' in 'MultiStepDecay' must be 'tuple, list', but received %s." - % type(milestones)) + % type(milestones) + ) - if not all([ + if not all( + [ milestones[i] < milestones[i + 1] for i in range(len(milestones) - 1) - ]): + ] + ): raise ValueError('The elements of milestones must be incremented') if decay_rate >= 1.0: raise ValueError('decay_rate should be < 1.0.') @@ -1154,7 +1234,7 @@ class MultiStepDecay(_LearningRateEpochDecay): if self.epoch_num < self.milestones[i]: return self.base_lr * (decay_rate**i) - return self.base_lr * (decay_rate**len(self.milestones)) + return self.base_lr * (decay_rate ** len(self.milestones)) class LambdaDecay(_LearningRateEpochDecay): @@ -1213,7 +1293,8 @@ class LambdaDecay(_LearningRateEpochDecay): if not callable(lr_lambda): raise TypeError( "The type of 'lr_lambda' in 'LambdaDecay' must be 'function', but received %s." - % type(lr_lambda)) + % type(lr_lambda) + ) self.lr_lambda = lr_lambda super(LambdaDecay, self).__init__(learning_rate) diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py index ac894a3acc7..de71f68c1fe 100644 --- a/python/paddle/fluid/dygraph/math_op_patch.py +++ b/python/paddle/fluid/dygraph/math_op_patch.py @@ -13,7 +13,13 @@ # limitations under the License. from .. import core -from ..framework import Variable, convert_np_dtype_to_dtype_, _varbase_creator, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Variable, + convert_np_dtype_to_dtype_, + _varbase_creator, + _in_legacy_dygraph, + in_dygraph_mode, +) from ..layers.layer_function_generator import OpProtoHolder from . import no_grad from .. import framework @@ -70,13 +76,22 @@ def monkey_patch_math_varbase(): @no_grad def create_tensor(value, dtype, shape): if framework._in_eager_mode_: - out = _C_ops.full(shape, value, dtype, - framework._current_expected_place()) + out = _C_ops.full( + shape, value, dtype, framework._current_expected_place() + ) else: out = _varbase_creator(dtype=dtype) - out = _legacy_C_ops.fill_constant(out, 'dtype', dtype, 'shape', - shape, 'value', value, - 'force_cpu', False) + out = _legacy_C_ops.fill_constant( + out, + 'dtype', + dtype, + 'shape', + shape, + 'value', + value, + 'force_cpu', + False, + ) out.stop_gradient = True return out @@ -110,8 +125,9 @@ def monkey_patch_math_varbase(): dtype = convert_np_dtype_to_dtype_(dtype) if _in_legacy_dygraph(): - return _legacy_C_ops.cast(self, 'in_dtype', self.dtype, 'out_dtype', - dtype) + return _legacy_C_ops.cast( + self, 'in_dtype', self.dtype, 'out_dtype', dtype + ) return _C_ops.cast(self, dtype) def _scalar_elementwise_op_(var, scale, bias): @@ -124,7 +140,9 @@ def monkey_patch_math_varbase(): def _float_(var): numel = np.prod(var.shape) - assert numel == 1, "only one element variable can be converted to float." + assert ( + numel == 1 + ), "only one element variable can be converted to float." tensor = var.value().get_tensor() assert tensor._is_initialized(), "variable's tensor is not initialized" return float(var.numpy().flatten()[0]) @@ -154,7 +172,9 @@ def monkey_patch_math_varbase(): def _index_(var): numel = np.prod(var.shape) - assert numel == 1, "only one element variable can be converted to python index." + assert ( + numel == 1 + ), "only one element variable can be converted to python index." tensor = var.value().get_tensor() assert tensor._is_initialized(), "variable's tensor is not initialized" return int(var.numpy().flatten()[0]) @@ -196,12 +216,13 @@ def monkey_patch_math_varbase(): return _scalar_elementwise_op_(var, 1.0 / value, 0.0) # for binary operator such as elementwise, compare - def _binary_creator_(method_name, - op_type, - reverse=False, - scalar_method=None, - call_final_api=False): - + def _binary_creator_( + method_name, + op_type, + reverse=False, + scalar_method=None, + call_final_api=False, + ): def __impl__(self, other_var): # 1. scalar exists cases # we need combine the tensor.dtype and scalar.dtype, cast correct object @@ -224,8 +245,9 @@ def monkey_patch_math_varbase(): # so the calculation result here and the calculation result of numpy are # different after 6 decimal point. If necessary, we can also use float64 here. # torch's behavior here is consistent with ours - if (op_type == "divide" or op_type == "elementwise_div" - ) and self.dtype in _supported_int_dtype_: + if ( + op_type == "divide" or op_type == "elementwise_div" + ) and self.dtype in _supported_int_dtype_: self = astype(self, 'float32') # here use `scale` replace `elementwise` to get better performance # but only +, -, *, / can use this method @@ -244,36 +266,47 @@ def monkey_patch_math_varbase(): if not isinstance(other_var, other_var_should_be): if isinstance(other_var, complex): import paddle + other_var = paddle.to_tensor(other_var, dtype='complex64') else: if reverse: - other_var = create_tensor(other_var, - dtype=lhs_dtype, - shape=self.shape) + other_var = create_tensor( + other_var, dtype=lhs_dtype, shape=self.shape + ) else: # add fill_op - other_var = create_scalar(value=other_var, - dtype=lhs_dtype) + other_var = create_scalar( + value=other_var, dtype=lhs_dtype + ) # 3. promote types or unify right var type to left var rhs_dtype = other_var.dtype if lhs_dtype != rhs_dtype: if method_name in _supported_promote_complex_types_ and ( - lhs_dtype in _complex_dtypes - or rhs_dtype in _complex_dtypes): + lhs_dtype in _complex_dtypes or rhs_dtype in _complex_dtypes + ): # only when lhs_dtype or rhs_dtype is complex type, # the dtype will promote, in other cases, directly # use lhs_dtype, this is consistent will original rule promote_dtype = core._promote_types_if_complex_exists( - lhs_dtype, rhs_dtype) - self = self if lhs_dtype == promote_dtype else astype( - self, promote_dtype) - other_var = other_var if rhs_dtype == promote_dtype else astype( - other_var, promote_dtype) + lhs_dtype, rhs_dtype + ) + self = ( + self + if lhs_dtype == promote_dtype + else astype(self, promote_dtype) + ) + other_var = ( + other_var + if rhs_dtype == promote_dtype + else astype(other_var, promote_dtype) + ) else: warnings.warn( - 'The dtype of left and right variables are not the same, left dtype is {}, but right dtype is {}, the right dtype will convert to {}' - .format(lhs_dtype, rhs_dtype, lhs_dtype)) + 'The dtype of left and right variables are not the same, left dtype is {}, but right dtype is {}, the right dtype will convert to {}'.format( + lhs_dtype, rhs_dtype, lhs_dtype + ) + ) other_var = astype(other_var, lhs_dtype) if reverse: @@ -281,8 +314,9 @@ def monkey_patch_math_varbase(): self = other_var other_var = tmp - if (op_type == "divide" or op_type == "elementwise_div" - ) and self.dtype in _supported_int_dtype_: + if ( + op_type == "divide" or op_type == "elementwise_div" + ) and self.dtype in _supported_int_dtype_: self = astype(self, 'float32') other_var = astype(other_var, 'float32') @@ -315,7 +349,9 @@ def monkey_patch_math_varbase(): Returns: Tensor - """.format(comment) + """.format( + comment + ) __impl__.__name__ = method_name return __impl__ @@ -332,39 +368,78 @@ def monkey_patch_math_varbase(): ('ndim', _ndim_), ('size', _size_), ('T', _T_), - ('__add__', - _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_)), + ( + '__add__', + _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_), + ), # a+b == b+a. Do not need to reverse explicitly - ('__radd__', - _binary_creator_('__radd__', 'elementwise_add', False, _scalar_add_)), - ('__sub__', - _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_)), - ('__rsub__', - _binary_creator_('__rsub__', 'elementwise_sub', True, _scalar_rsub_)), - ('__mul__', - _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_)), + ( + '__radd__', + _binary_creator_( + '__radd__', 'elementwise_add', False, _scalar_add_ + ), + ), + ( + '__sub__', + _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_), + ), + ( + '__rsub__', + _binary_creator_( + '__rsub__', 'elementwise_sub', True, _scalar_rsub_ + ), + ), + ( + '__mul__', + _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_), + ), ## a*b == b*a. Do not need to reverse explicitly - ('__rmul__', - _binary_creator_('__rmul__', 'elementwise_mul', False, _scalar_mul_)), - ('__div__', - _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_)), - ('__truediv__', - _binary_creator_('__truediv__', 'elementwise_div', False, - _scalar_div_)), - ('__rdiv__', _binary_creator_('__rdiv__', 'elementwise_div', True, - None)), - ('__rtruediv__', - _binary_creator_('rtruediv__', 'elementwise_div', True, None)), - ('__pow__', _binary_creator_('__pow__', 'elementwise_pow', False, - None)), - ('__rpow__', _binary_creator_('__rpow__', 'elementwise_pow', True, - None)), - ('__floordiv__', - _binary_creator_('__floordiv__', 'elementwise_floordiv', False, None)), - ('__mod__', _binary_creator_('__mod__', 'elementwise_mod', False, - None)), - ('__matmul__', _binary_creator_('__matmul__', "matmul_v2", False, - None)), + ( + '__rmul__', + _binary_creator_( + '__rmul__', 'elementwise_mul', False, _scalar_mul_ + ), + ), + ( + '__div__', + _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_), + ), + ( + '__truediv__', + _binary_creator_( + '__truediv__', 'elementwise_div', False, _scalar_div_ + ), + ), + ( + '__rdiv__', + _binary_creator_('__rdiv__', 'elementwise_div', True, None), + ), + ( + '__rtruediv__', + _binary_creator_('rtruediv__', 'elementwise_div', True, None), + ), + ( + '__pow__', + _binary_creator_('__pow__', 'elementwise_pow', False, None), + ), + ( + '__rpow__', + _binary_creator_('__rpow__', 'elementwise_pow', True, None), + ), + ( + '__floordiv__', + _binary_creator_( + '__floordiv__', 'elementwise_floordiv', False, None + ), + ), + ( + '__mod__', + _binary_creator_('__mod__', 'elementwise_mod', False, None), + ), + ( + '__matmul__', + _binary_creator_('__matmul__', "matmul_v2", False, None), + ), ## for logical compare ('__eq__', _binary_creator_('__eq__', 'equal', False, None)), ('__ne__', _binary_creator_('__ne__', 'not_equal', False, None)), @@ -372,7 +447,7 @@ def monkey_patch_math_varbase(): ('__le__', _binary_creator_('__le__', 'less_equal', False, None)), ('__gt__', _binary_creator_('__gt__', 'greater_than', False, None)), ('__ge__', _binary_creator_('__ge__', 'greater_equal', False, None)), - ('__array_ufunc__', None) + ('__array_ufunc__', None), ] eager_methods = [ @@ -390,7 +465,7 @@ def monkey_patch_math_varbase(): ('T', _T_), # for logical compare ('__eq__', _binary_creator_('__eq__', 'equal', False, None, True)), - ('__array_ufunc__', None) + ('__array_ufunc__', None), ] eager_cpp_level_patch = [ @@ -447,12 +522,16 @@ def monkey_patch_math_varbase(): setattr(local_tensor, method_name, method_impl) else: import paddle.tensor + # Tensor method from module paddle.tensor for method_name in paddle.tensor.tensor_method_func: - if hasattr(local_tensor, method_name): continue + if hasattr(local_tensor, method_name): + continue method_impl = getattr(paddle.tensor, method_name, None) - if method_impl: setattr(local_tensor, method_name, method_impl) + if method_impl: + setattr(local_tensor, method_name, method_impl) for magic_method, origin_method in paddle.tensor.magic_method_func: impl = getattr(paddle.tensor, origin_method, None) - if impl: setattr(local_tensor, magic_method, impl) + if impl: + setattr(local_tensor, magic_method, impl) diff --git a/python/paddle/fluid/dygraph/nn.py b/python/paddle/fluid/dygraph/nn.py index 27ac082da79..60202c2a6b1 100644 --- a/python/paddle/fluid/dygraph/nn.py +++ b/python/paddle/fluid/dygraph/nn.py @@ -18,8 +18,24 @@ from ..layers import utils from ..layers import nn as F from .. import dygraph_utils from . import layers -from ..framework import Variable, _non_static_mode, OpProtoHolder, Parameter, _dygraph_tracer, _varbase_creator, default_main_program, _global_flags, in_dygraph_mode, _in_legacy_dygraph -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..framework import ( + Variable, + _non_static_mode, + OpProtoHolder, + Parameter, + _dygraph_tracer, + _varbase_creator, + default_main_program, + _global_flags, + in_dygraph_mode, + _in_legacy_dygraph, +) +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from ..param_attr import ParamAttr from ..initializer import Normal, Constant, NumpyArrayInitializer from .. import unique_name @@ -33,10 +49,25 @@ import paddle.utils.deprecated as deprecated from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'Conv2D', 'Conv3D', 'Pool2D', 'Linear', 'BatchNorm', 'Dropout', 'Embedding', - 'GRUUnit', 'InstanceNorm', 'LayerNorm', 'NCE', 'PRelu', - 'BilinearTensorProduct', 'Conv2DTranspose', 'Conv3DTranspose', 'GroupNorm', - 'SpectralNorm', 'TreeConv', 'Flatten' + 'Conv2D', + 'Conv3D', + 'Pool2D', + 'Linear', + 'BatchNorm', + 'Dropout', + 'Embedding', + 'GRUUnit', + 'InstanceNorm', + 'LayerNorm', + 'NCE', + 'PRelu', + 'BilinearTensorProduct', + 'Conv2DTranspose', + 'Conv3DTranspose', + 'GroupNorm', + 'SpectralNorm', + 'TreeConv', + 'Flatten', ] @@ -157,24 +188,30 @@ class Conv2D(layers.Layer): """ - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - dtype='float32'): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + dtype='float32', + ): assert param_attr is not False, "param_attr should not be False here." super(Conv2D, self).__init__() - if (core.is_compiled_with_cuda() and paddle.fluid.get_flags( - "FLAGS_conv2d_disable_cudnn")["FLAGS_conv2d_disable_cudnn"]): + if ( + core.is_compiled_with_cuda() + and paddle.fluid.get_flags("FLAGS_conv2d_disable_cudnn")[ + "FLAGS_conv2d_disable_cudnn" + ] + ): use_cudnn = False self._num_channels = num_channels @@ -193,17 +230,22 @@ class Conv2D(layers.Layer): self._bias_attr = bias_attr self._dtype = dtype - if (self._num_channels == self._groups - and num_filters % self._num_channels == 0 - and not self._use_cudnn and not self._use_mkldnn): + if ( + self._num_channels == self._groups + and num_filters % self._num_channels == 0 + and not self._use_cudnn + and not self._use_mkldnn + ): self._l_type = 'depthwise_conv2d' else: self._l_type = 'conv2d' # NPU only supports depthwise_conv2d when "input_channel = output_channel = groups" if core.is_compiled_with_npu(): - if (self._num_channels == self._groups - and self._num_channels == self._num_filters): + if ( + self._num_channels == self._groups + and self._num_channels == self._num_filters + ): self._l_type = 'depthwise_conv2d' else: self._l_type = 'conv2d' @@ -219,48 +261,75 @@ class Conv2D(layers.Layer): filter_shape = [self._num_filters, num_filter_channels] + filter_size def _get_default_param_initializer(): - filter_elem_num = filter_size[0] * filter_size[ - 1] * self._num_channels - std = (2.0 / filter_elem_num)**0.5 + filter_elem_num = ( + filter_size[0] * filter_size[1] * self._num_channels + ) + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) self.weight = self.create_parameter( attr=self._param_attr, shape=filter_shape, dtype=self._dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): if in_dygraph_mode() and self._l_type == "conv2d": - pre_bias = _C_ops.conv2d(input, self.weight, self._stride, - self._padding, "EXPLICIT", - self._groups if self._groups else 1, - self._dilation, "NCHW", False, -1, False) + pre_bias = _C_ops.conv2d( + input, + self.weight, + self._stride, + self._padding, + "EXPLICIT", + self._groups if self._groups else 1, + self._dilation, + "NCHW", + False, + -1, + False, + ) if self.bias is not None: pre_act = F.elementwise_add(pre_bias, self.bias, axis=1) else: pre_act = pre_bias return dygraph_utils._append_activation_in_dygraph( - pre_act, self._act, use_mkldnn=self._use_mkldnn) - - if _non_static_mode() and (self._l_type == 'conv2d' - or self._l_type == 'depthwise_conv2d'): - attrs = ('strides', self._stride, 'paddings', self._padding, - 'dilations', self._dilation, 'groups', - self._groups if self._groups else 1, 'use_cudnn', - self._use_cudnn, 'use_mkldnn', self._use_mkldnn) + pre_act, self._act, use_mkldnn=self._use_mkldnn + ) + + if _non_static_mode() and ( + self._l_type == 'conv2d' or self._l_type == 'depthwise_conv2d' + ): + attrs = ( + 'strides', + self._stride, + 'paddings', + self._padding, + 'dilations', + self._dilation, + 'groups', + self._groups if self._groups else 1, + 'use_cudnn', + self._use_cudnn, + 'use_mkldnn', + self._use_mkldnn, + ) out = _legacy_C_ops.conv2d(input, self.weight, *attrs) pre_bias = out pre_act = dygraph_utils._append_bias_in_dygraph( - pre_bias, self.bias, 1, use_mkldnn=self._use_mkldnn) + pre_bias, self.bias, 1, use_mkldnn=self._use_mkldnn + ) return dygraph_utils._append_activation_in_dygraph( - pre_act, self._act, use_mkldnn=self._use_mkldnn) + pre_act, self._act, use_mkldnn=self._use_mkldnn + ) inputs = { 'Input': [input], 'Filter': [self.weight], @@ -274,32 +343,33 @@ class Conv2D(layers.Layer): 'use_mkldnn': self._use_mkldnn, } - check_variable_and_dtype(input, 'input', - ['float16', 'float32', 'float64'], 'Conv2D') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'Conv2D' + ) pre_bias = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) - self._helper.append_op(type=self._l_type, - inputs={ - 'Input': input, - 'Filter': self.weight, - }, - outputs={"Output": pre_bias}, - attrs=attrs) + self._helper.append_op( + type=self._l_type, + inputs={ + 'Input': input, + 'Filter': self.weight, + }, + outputs={"Output": pre_bias}, + attrs=attrs, + ) if self.bias is not None: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={ - 'axis': 1, - 'use_mkldnn': self._use_mkldnn - }) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1, 'use_mkldnn': self._use_mkldnn}, + ) else: pre_act = pre_bias @@ -418,19 +488,21 @@ class Conv3D(layers.Layer): """ - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - dtype='float32'): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + dtype='float32', + ): assert param_attr is not False, "param_attr should not be False here." super(Conv3D, self).__init__() self._num_channels = num_channels @@ -457,52 +529,61 @@ class Conv3D(layers.Layer): filter_shape = [self._num_filters, num_filter_channels] + filter_size def _get_default_param_initializer(): - filter_elem_num = filter_size[0] * filter_size[1] * filter_size[ - 2] * self._num_channels - std = (2.0 / filter_elem_num)**0.5 + filter_elem_num = ( + filter_size[0] + * filter_size[1] + * filter_size[2] + * self._num_channels + ) + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) self.weight = self.create_parameter( attr=self._param_attr, shape=filter_shape, dtype=self._dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): pre_bias = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - - self._helper.append_op(type='conv3d', - inputs={ - 'Input': input, - 'Filter': self.weight, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': self._stride, - 'paddings': self._padding, - 'dilations': self._dilation, - 'groups': - self._groups if self._groups else 1, - 'use_cudnn': self._use_cudnn, - 'use_mkldnn': False - }) + dtype=self._dtype + ) + + self._helper.append_op( + type='conv3d', + inputs={ + 'Input': input, + 'Filter': self.weight, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': self._stride, + 'paddings': self._padding, + 'dilations': self._dilation, + 'groups': self._groups if self._groups else 1, + 'use_cudnn': self._use_cudnn, + 'use_mkldnn': False, + }, + ) if self.bias is not None: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}, + ) else: pre_act = pre_bias @@ -651,23 +732,27 @@ class Conv3DTranspose(layers.Layer): """ - def __init__(self, - num_channels, - num_filters, - filter_size, - padding=0, - stride=1, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - dtype='float32'): + def __init__( + self, + num_channels, + num_filters, + filter_size, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + dtype='float32', + ): super(Conv3DTranspose, self).__init__() if not isinstance(use_cudnn, bool): raise ValueError("use_cudnn should be True or False") - assert param_attr is not False, "param_attr should not be False in conv3d_transpose." + assert ( + param_attr is not False + ), "param_attr should not be False in conv3d_transpose." self._padding = utils.convert_to_list(padding, 3, 'padding') self._stride = utils.convert_to_list(stride, 3, 'stride') self._dilation = utils.convert_to_list(dilation, 3, 'dilation') @@ -682,46 +767,50 @@ class Conv3DTranspose(layers.Layer): self._dtype = dtype self._filter_size = utils.convert_to_list( - self._filter_size, 3, 'conv3d_transpose.filter_size') - - filter_shape = [self._num_channels, self._num_filters // self._groups - ] + self._filter_size - self.weight = self.create_parameter(dtype=self._dtype, - shape=filter_shape, - attr=self._param_attr) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self._filter_size, 3, 'conv3d_transpose.filter_size' + ) + + filter_shape = [ + self._num_channels, + self._num_filters // self._groups, + ] + self._filter_size + self.weight = self.create_parameter( + dtype=self._dtype, shape=filter_shape, attr=self._param_attr + ) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): pre_bias = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type="conv3d_transpose", - inputs={ - 'Input': [input], - 'Filter': [self.weight] - }, - outputs={'Output': pre_bias}, - attrs={ - 'strides': self._stride, - 'paddings': self._padding, - 'dilations': self._dilation, - 'groups': - self._groups if self._groups else 1, - 'use_cudnn': self._use_cudnn - }) + dtype=self._dtype + ) + self._helper.append_op( + type="conv3d_transpose", + inputs={'Input': [input], 'Filter': [self.weight]}, + outputs={'Output': pre_bias}, + attrs={ + 'strides': self._stride, + 'paddings': self._padding, + 'dilations': self._dilation, + 'groups': self._groups if self._groups else 1, + 'use_cudnn': self._use_cudnn, + }, + ) if self._bias_attr: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}, + ) else: pre_act = pre_bias @@ -835,27 +924,31 @@ class Pool2D(layers.Layer): """ - def __init__(self, - pool_size=-1, - pool_type="max", - pool_stride=1, - pool_padding=0, - global_pooling=False, - use_cudnn=True, - ceil_mode=False, - exclusive=True, - data_format="NCHW"): + def __init__( + self, + pool_size=-1, + pool_type="max", + pool_stride=1, + pool_padding=0, + global_pooling=False, + use_cudnn=True, + ceil_mode=False, + exclusive=True, + data_format="NCHW", + ): data_format = data_format.upper() # supprt NHWC, nhwc, etc. pool_type = pool_type.lower() # supprt max, Max, etc. if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if global_pooling is False and pool_size == -1: raise ValueError( "When the global_pooling is False, pool_size must be passed " - "and be a valid value. Received pool_size: " + str(pool_size)) + "and be a valid value. Received pool_size: " + str(pool_size) + ) if not isinstance(use_cudnn, bool): raise ValueError("use_cudnn should be True or False") @@ -865,14 +958,16 @@ class Pool2D(layers.Layer): if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) super(Pool2D, self).__init__() self._pool_type = pool_type self._pool_size = utils.convert_to_list(pool_size, 2, 'pool_size') - self._pool_padding = utils.convert_to_list(pool_padding, 2, - 'pool_padding') + self._pool_padding = utils.convert_to_list( + pool_padding, 2, 'pool_padding' + ) self._pool_stride = utils.convert_to_list(pool_stride, 2, 'pool_stride') self._global_pooling = global_pooling self._use_cudnn = use_cudnn @@ -884,23 +979,51 @@ class Pool2D(layers.Layer): def forward(self, input): if _non_static_mode(): if not self._use_mkldnn and in_dygraph_mode(): - return _C_ops.pool2d(input, self._pool_size, self._pool_stride, - self._pool_padding, self._ceil_mode, - self._exclusive, self._data_format, - self._pool_type, self._global_pooling, - False, "EXPLICIT", self._use_cudnn) - - attrs = ('pooling_type', self._pool_type, 'ksize', self._pool_size, - 'global_pooling', self._global_pooling, 'strides', - self._pool_stride, 'paddings', self._pool_padding, - 'use_cudnn', self._use_cudnn, 'ceil_mode', self._ceil_mode, - 'use_mkldnn', self._use_mkldnn, 'exclusive', - self._exclusive, 'data_format', self._data_format) + return _C_ops.pool2d( + input, + self._pool_size, + self._pool_stride, + self._pool_padding, + self._ceil_mode, + self._exclusive, + self._data_format, + self._pool_type, + self._global_pooling, + False, + "EXPLICIT", + self._use_cudnn, + ) + + attrs = ( + 'pooling_type', + self._pool_type, + 'ksize', + self._pool_size, + 'global_pooling', + self._global_pooling, + 'strides', + self._pool_stride, + 'paddings', + self._pool_padding, + 'use_cudnn', + self._use_cudnn, + 'ceil_mode', + self._ceil_mode, + 'use_mkldnn', + self._use_mkldnn, + 'exclusive', + self._exclusive, + 'data_format', + self._data_format, + ) return _legacy_C_ops.pool2d(input, *attrs) check_variable_and_dtype( - input, 'input', ['int8', 'uint8', 'float16', 'float32', 'float64'], - 'Pool2D') + input, + 'input', + ['int8', 'uint8', 'float16', 'float32', 'float64'], + 'Pool2D', + ) attrs = { "pooling_type": self._pool_type, @@ -918,10 +1041,12 @@ class Pool2D(layers.Layer): pool_out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type=self._l_type, - inputs={"X": input}, - outputs={"Out": pool_out}, - attrs=attrs) + self._helper.append_op( + type=self._l_type, + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs=attrs, + ) return pool_out @@ -977,44 +1102,60 @@ class Linear(layers.Layer): res = linear(data) # [30, 10, 64] """ - def __init__(self, - input_dim, - output_dim, - param_attr=None, - bias_attr=None, - act=None, - dtype="float32"): + def __init__( + self, + input_dim, + output_dim, + param_attr=None, + bias_attr=None, + act=None, + dtype="float32", + ): super(Linear, self).__init__() self._act = act self._dtype = dtype - self.weight = self.create_parameter(shape=[input_dim, output_dim], - attr=param_attr, - dtype=dtype, - is_bias=False) - self.bias = self.create_parameter(shape=[output_dim], - attr=bias_attr, - dtype=dtype, - is_bias=True) + self.weight = self.create_parameter( + shape=[input_dim, output_dim], + attr=param_attr, + dtype=dtype, + is_bias=False, + ) + self.bias = self.create_parameter( + shape=[output_dim], attr=bias_attr, dtype=dtype, is_bias=True + ) self._use_mkldnn = _global_flags()["FLAGS_use_mkldnn"] def forward(self, input): if _non_static_mode(): pre_bias = _varbase_creator(dtype=input.dtype) - _legacy_C_ops.matmul(input, self.weight, pre_bias, 'transpose_X', - False, 'transpose_Y', False, "alpha", 1, - "use_mkldnn", self._use_mkldnn) + _legacy_C_ops.matmul( + input, + self.weight, + pre_bias, + 'transpose_X', + False, + 'transpose_Y', + False, + "alpha", + 1, + "use_mkldnn", + self._use_mkldnn, + ) pre_act = dygraph_utils._append_bias_in_dygraph( pre_bias, self.bias, axis=len(input.shape) - 1, - use_mkldnn=self._use_mkldnn) + use_mkldnn=self._use_mkldnn, + ) return dygraph_utils._append_activation_in_dygraph( - pre_act, self._act, use_mkldnn=self._use_mkldnn) + pre_act, self._act, use_mkldnn=self._use_mkldnn + ) - check_variable_and_dtype(input, 'input', - ['float16', 'float32', 'float64'], "Linear") + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], "Linear" + ) attrs = { "transpose_X": False, @@ -1025,23 +1166,22 @@ class Linear(layers.Layer): inputs = {"X": [input], "Y": [self.weight]} tmp = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="matmul", - inputs=inputs, - outputs={"Out": tmp}, - attrs=attrs) + self._helper.append_op( + type="matmul", inputs=inputs, outputs={"Out": tmp}, attrs=attrs + ) if self.bias is not None: pre_activation = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [tmp], - 'Y': [self.bias] - }, - outputs={'Out': [pre_activation]}, - attrs={ - 'axis': len(input.shape) - 1, - 'use_mkldnn': self._use_mkldnn - }) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [tmp], 'Y': [self.bias]}, + outputs={'Out': [pre_activation]}, + attrs={ + 'axis': len(input.shape) - 1, + 'use_mkldnn': self._use_mkldnn, + }, + ) else: pre_activation = tmp return self._helper.append_activation(pre_activation, act=self._act) @@ -1115,16 +1255,20 @@ class InstanceNorm(layers.Layer): """ - def __init__(self, - num_channels, - epsilon=1e-5, - param_attr=None, - bias_attr=None, - dtype='float32'): + def __init__( + self, + num_channels, + epsilon=1e-5, + param_attr=None, + bias_attr=None, + dtype='float32', + ): super(InstanceNorm, self).__init__() if param_attr == False or bias_attr == False: - assert bias_attr == param_attr, "param_attr and bias_attr must be set to Fasle at the same time in InstanceNorm" + assert ( + bias_attr == param_attr + ), "param_attr and bias_attr must be set to Fasle at the same time in InstanceNorm" self._epsilon = epsilon self._param_attr = param_attr self._bias_attr = bias_attr @@ -1136,29 +1280,34 @@ class InstanceNorm(layers.Layer): shape=[num_channels], dtype=self._dtype, default_initializer=Constant(1.0), - is_bias=False) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[num_channels], - dtype=self._dtype, - default_initializer=Constant(0.0), - is_bias=True) + is_bias=False, + ) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[num_channels], + dtype=self._dtype, + default_initializer=Constant(0.0), + is_bias=True, + ) else: self.scale = None self.bias = None def forward(self, input): if in_dygraph_mode(): - out = _C_ops.instance_norm(input, self.scale, self.bias, - self._epsilon) + out = _C_ops.instance_norm( + input, self.scale, self.bias, self._epsilon + ) return out if _in_legacy_dygraph(): - out, _, _ = _legacy_C_ops.instance_norm(input, self.scale, - self.bias, 'epsilon', - self._epsilon) + out, _, _ = _legacy_C_ops.instance_norm( + input, self.scale, self.bias, 'epsilon', self._epsilon + ) return out - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - "InstanceNorm") + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], "InstanceNorm" + ) attrs = {"epsilon": self._epsilon} @@ -1168,22 +1317,24 @@ class InstanceNorm(layers.Layer): inputs = {"X": [input]} saved_mean = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) saved_variance = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) instance_norm_out = self._helper.create_variable_for_type_inference( - self._dtype) + self._dtype + ) outputs = { "Y": [instance_norm_out], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } - self._helper.append_op(type="instance_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="instance_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return instance_norm_out @@ -1285,29 +1436,33 @@ class BatchNorm(layers.Layer): hidden1 = batch_norm(x) """ - def __init__(self, - num_channels, - act=None, - is_test=False, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - dtype='float32', - data_layout='NCHW', - in_place=False, - moving_mean_name=None, - moving_variance_name=None, - do_model_average_for_mean_and_var=True, - use_global_stats=False, - trainable_statistics=False): + def __init__( + self, + num_channels, + act=None, + is_test=False, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + dtype='float32', + data_layout='NCHW', + in_place=False, + moving_mean_name=None, + moving_variance_name=None, + do_model_average_for_mean_and_var=True, + use_global_stats=False, + trainable_statistics=False, + ): super(BatchNorm, self).__init__() self._param_attr = param_attr self._bias_attr = bias_attr self._act = act self._use_mkldnn = _global_flags()["FLAGS_use_mkldnn"] - assert bias_attr is not False, "bias_attr should not be False in batch_norm." + assert ( + bias_attr is not False + ), "bias_attr should not be False in batch_norm." if dtype == "float16": self._dtype = "float32" @@ -1317,34 +1472,48 @@ class BatchNorm(layers.Layer): param_shape = [num_channels] # create parameter - self.weight = self.create_parameter(attr=self._param_attr, - shape=param_shape, - dtype=self._dtype, - default_initializer=Constant(1.0)) - self.weight.stop_gradient = use_global_stats and self._param_attr.learning_rate == 0. - - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - dtype=self._dtype, - is_bias=True) - self.bias.stop_gradient = use_global_stats and self._param_attr.learning_rate == 0. - - self._mean = self.create_parameter(attr=ParamAttr( - name=moving_mean_name, - initializer=Constant(0.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=self._dtype) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=param_shape, + dtype=self._dtype, + default_initializer=Constant(1.0), + ) + self.weight.stop_gradient = ( + use_global_stats and self._param_attr.learning_rate == 0.0 + ) + + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=True, + ) + self.bias.stop_gradient = ( + use_global_stats and self._param_attr.learning_rate == 0.0 + ) + + self._mean = self.create_parameter( + attr=ParamAttr( + name=moving_mean_name, + initializer=Constant(0.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=self._dtype, + ) self._mean.stop_gradient = True - self._variance = self.create_parameter(attr=ParamAttr( - name=moving_variance_name, - initializer=Constant(1.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=self._dtype) + self._variance = self.create_parameter( + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=self._dtype, + ) self._variance.stop_gradient = True self._in_place = in_place @@ -1366,29 +1535,61 @@ class BatchNorm(layers.Layer): if _non_static_mode(): if in_dygraph_mode(): batch_norm_out, t1, t2, t3, t4, _ = _C_ops.batch_norm( - input, self.weight, self.bias, self._mean, self._variance, - self._momentum, self._epsilon, self._data_layout, - not self.training, self._use_global_stats, - self._trainable_statistics, False) + input, + self.weight, + self.bias, + self._mean, + self._variance, + self._momentum, + self._epsilon, + self._data_layout, + not self.training, + self._use_global_stats, + self._trainable_statistics, + False, + ) return dygraph_utils._append_activation_in_dygraph( - batch_norm_out, act=self._act, use_mkldnn=self._use_mkldnn) + batch_norm_out, act=self._act, use_mkldnn=self._use_mkldnn + ) elif _in_legacy_dygraph(): - attrs = ("momentum", self._momentum, "epsilon", self._epsilon, - "is_test", not self.training, "data_layout", - self._data_layout, "use_mkldnn", self._use_mkldnn, - "fuse_with_relu", self._fuse_with_relu, - "use_global_stats", self._use_global_stats, - 'trainable_statistics', self._trainable_statistics) + attrs = ( + "momentum", + self._momentum, + "epsilon", + self._epsilon, + "is_test", + not self.training, + "data_layout", + self._data_layout, + "use_mkldnn", + self._use_mkldnn, + "fuse_with_relu", + self._fuse_with_relu, + "use_global_stats", + self._use_global_stats, + 'trainable_statistics', + self._trainable_statistics, + ) batch_norm_out, _, _, _, _, _ = _legacy_C_ops.batch_norm( - input, self.weight, self.bias, self._mean, self._variance, - None, mean_out, variance_out, *attrs) + input, + self.weight, + self.bias, + self._mean, + self._variance, + None, + mean_out, + variance_out, + *attrs + ) return dygraph_utils._append_activation_in_dygraph( - batch_norm_out, act=self._act, use_mkldnn=self._use_mkldnn) + batch_norm_out, act=self._act, use_mkldnn=self._use_mkldnn + ) - check_variable_and_dtype(input, 'input', - ['float16', 'float32', 'float64'], 'BatchNorm') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'BatchNorm' + ) attrs = { "momentum": self._momentum, @@ -1406,33 +1607,38 @@ class BatchNorm(layers.Layer): "Scale": [self.weight], "Bias": [self.bias], "Mean": [self._mean], - "Variance": [self._variance] + "Variance": [self._variance], } saved_mean = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) saved_variance = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) reserve_space = self._helper.create_variable_for_type_inference( - dtype=self._helper.input_dtype(input), stop_gradient=True) + dtype=self._helper.input_dtype(input), stop_gradient=True + ) - batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference( - self._dtype) + batch_norm_out = ( + input + if self._in_place + else self._helper.create_variable_for_type_inference(self._dtype) + ) outputs = { "Y": [batch_norm_out], "MeanOut": [mean_out], "VarianceOut": [variance_out], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } if reserve_space is not None: outputs["ReserveSpace"] = [reserve_space] - self._helper.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) # Currently, we don't support inplace in dygraph mode return self._helper.append_activation(batch_norm_out, self._act) @@ -1440,78 +1646,82 @@ class BatchNorm(layers.Layer): class Dropout(layers.Layer): """ - This interface is used to construct a callable object of the ``Dropout`` class. - For more details, refer to code examples. + This interface is used to construct a callable object of the ``Dropout`` class. + For more details, refer to code examples. - Drop or keep each element of input independently. Dropout is a regularization - technique for reducing overfitting by preventing neuron co-adaption during - training. The dropout operator randomly sets (according to the given dropout - probability) the outputs of some units to zero, while others are remain - unchanged. + Drop or keep each element of input independently. Dropout is a regularization + technique for reducing overfitting by preventing neuron co-adaption during + training. The dropout operator randomly sets (according to the given dropout + probability) the outputs of some units to zero, while others are remain + unchanged. - Dropout layer can be removed for efficiency concern. + Dropout layer can be removed for efficiency concern. - Parameters: - p (float, optional): Probability of setting units to zero. Default: 0.5 - seed (int, optional): A Python integer used to create random seeds. If this - parameter is set to None, a random seed is used. - NOTE: If an integer seed is given, always the same output - units will be dropped. DO NOT use a fixed seed in training. Default: None. - dropout_implementation(string, optional): ['downgrade_in_infer'(default)|'upscale_in_train'] + Parameters: + p (float, optional): Probability of setting units to zero. Default: 0.5 + seed (int, optional): A Python integer used to create random seeds. If this + parameter is set to None, a random seed is used. + NOTE: If an integer seed is given, always the same output + units will be dropped. DO NOT use a fixed seed in training. Default: None. + dropout_implementation(string, optional): ['downgrade_in_infer'(default)|'upscale_in_train'] - 1. downgrade_in_infer(default), downgrade the outcome at inference + 1. downgrade_in_infer(default), downgrade the outcome at inference - - train: out = input * mask - - inference: out = input * (1.0 - p) + - train: out = input * mask + - inference: out = input * (1.0 - p) - (mask is a tensor same shape with input, value is 0 or 1 - ratio of 0 is dropout_prob) - 2. upscale_in_train, upscale the outcome at training time + (mask is a tensor same shape with input, value is 0 or 1 + ratio of 0 is dropout_prob) + 2. upscale_in_train, upscale the outcome at training time - - train: out = input * mask / ( 1.0 - p ) - - inference: out = input + - train: out = input * mask / ( 1.0 - p ) + - inference: out = input - (mask is a tensor same shape with input, value is 0 or 1 - ratio of 0 is p) - is_test (bool, optional): A flag indicating whether it is in test phrase or not. - This flag only has effect on static graph mode. For dygraph mode, please use ``eval()``. - Default: False. + (mask is a tensor same shape with input, value is 0 or 1 + ratio of 0 is p) + is_test (bool, optional): A flag indicating whether it is in test phrase or not. + This flag only has effect on static graph mode. For dygraph mode, please use ``eval()``. + Default: False. - Returns: - None + Returns: + None - Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - from paddle.fluid.dygraph.base import to_variable - import numpy as np - - x = np.random.random(size=(3, 10, 3, 7)).astype('float32') - with fluid.dygraph.guard(): - x = to_variable(x) - m = fluid.dygraph.Dropout(p=0.5) - droped_train = m(x) - # switch to eval mode - m.eval() - droped_eval = m(x) - """ - - def __init__(self, - p=0.5, - seed=None, - dropout_implementation="downgrade_in_infer", - is_test=False): + import paddle.fluid as fluid + from paddle.fluid.dygraph.base import to_variable + import numpy as np + + x = np.random.random(size=(3, 10, 3, 7)).astype('float32') + with fluid.dygraph.guard(): + x = to_variable(x) + m = fluid.dygraph.Dropout(p=0.5) + droped_train = m(x) + # switch to eval mode + m.eval() + droped_eval = m(x) + """ + + def __init__( + self, + p=0.5, + seed=None, + dropout_implementation="downgrade_in_infer", + is_test=False, + ): super(Dropout, self).__init__() assert isinstance(p, (float, int)), "p argument should be a number" assert 0 <= p <= 1, "p argument should between 0 and 1" self._dropout_prob = p assert seed is None or isinstance( - seed, int), "seed argument should be None or a integer" + seed, int + ), "seed argument should be None or a integer" self._seed = seed assert dropout_implementation in ( - 'downgrade_in_infer', 'upscale_in_train' + 'downgrade_in_infer', + 'upscale_in_train', ), "dropout_implementation argument should be 'downgrade_in_infer' or 'upscale_in_train'" self._dropout_implementation = dropout_implementation self._is_test = is_test @@ -1525,8 +1735,9 @@ class Dropout(layers.Layer): self._seed = prog.random_seed attrs = { 'dropout_prob': self._dropout_prob, - 'is_test': - not self.training if _non_static_mode() else self._is_test, + 'is_test': not self.training + if _non_static_mode() + else self._is_test, 'fix_seed': self._seed is not None, 'seed': self._seed if self._seed is not None else 0, 'dropout_implementation': self._dropout_implementation, @@ -1539,23 +1750,23 @@ class Dropout(layers.Layer): out = self._helper.create_variable_for_type_inference(dtype=input.dtype) mask = self._helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) - - self._helper.append_op(type='dropout', - inputs={'X': [input]}, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs=attrs) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) + + self._helper.append_op( + type='dropout', + inputs={'X': [input]}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs=attrs, + ) return out class Embedding(layers.Layer): r""" :alias_main: paddle.nn.Embedding - :alias: paddle.nn.Embedding,paddle.nn.layer.Embedding,paddle.nn.layer.common.Embedding - :old_api: paddle.fluid.dygraph.Embedding + :alias: paddle.nn.Embedding,paddle.nn.layer.Embedding,paddle.nn.layer.common.Embedding + :old_api: paddle.fluid.dygraph.Embedding **Embedding Layer** @@ -1659,19 +1870,26 @@ class Embedding(layers.Layer): static_rlt3 = emb(base.to_variable(inp_word)) """ - def __init__(self, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): + def __init__( + self, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', + ): super(Embedding, self).__init__() self._size = size self._is_sparse = is_sparse self._is_distributed = is_distributed - self._padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) + self._padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) self._param_attr = param_attr self._dtype = dtype @@ -1679,36 +1897,48 @@ class Embedding(layers.Layer): if self._remote_prefetch: assert self._is_sparse is True and self._is_distributed is False - self.weight = self.create_parameter(attr=self._param_attr, - shape=self._size, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False, + ) def forward(self, input): if _non_static_mode(): return _legacy_C_ops.lookup_table_v2( - self.weight, input, 'is_sparse', self._is_sparse, - 'is_distributed', self._is_distributed, 'remote_prefetch', - self._remote_prefetch, 'padding_idx', self._padding_idx) + self.weight, + input, + 'is_sparse', + self._is_sparse, + 'is_distributed', + self._is_distributed, + 'remote_prefetch', + self._remote_prefetch, + 'padding_idx', + self._padding_idx, + ) - check_variable_and_dtype(input, 'input', - ['uint8', 'int8', 'int16', 'int32', 'int64'], - 'Embedding') + check_variable_and_dtype( + input, + 'input', + ['uint8', 'int8', 'int16', 'int32', 'int64'], + 'Embedding', + ) attrs = { 'is_sparse': self._is_sparse, 'is_distributed': self._is_distributed, 'remote_prefetch': self._remote_prefetch, - 'padding_idx': self._padding_idx + 'padding_idx': self._padding_idx, } out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type='lookup_table_v2', - inputs={ - 'Ids': input, - 'W': self.weight - }, - outputs={'Out': out}, - attrs=attrs) + self._helper.append_op( + type='lookup_table_v2', + inputs={'Ids': input, 'W': self.weight}, + outputs={'Out': out}, + attrs=attrs, + ) return out @@ -1716,8 +1946,8 @@ class Embedding(layers.Layer): class LayerNorm(layers.Layer): r""" :alias_main: paddle.nn.LayerNorm - :alias: paddle.nn.LayerNorm,paddle.nn.layer.LayerNorm,paddle.nn.layer.norm.LayerNorm - :old_api: paddle.fluid.dygraph.LayerNorm + :alias: paddle.nn.LayerNorm,paddle.nn.layer.LayerNorm,paddle.nn.layer.norm.LayerNorm + :old_api: paddle.fluid.dygraph.LayerNorm This interface is used to construct a callable object of the ``LayerNorm`` class. For more details, refer to code examples. @@ -1784,15 +2014,17 @@ class LayerNorm(layers.Layer): """ - def __init__(self, - normalized_shape, - scale=True, - shift=True, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None, - dtype='float32'): + def __init__( + self, + normalized_shape, + scale=True, + shift=True, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + dtype='float32', + ): super(LayerNorm, self).__init__() if isinstance(normalized_shape, numbers.Integral): normalized_shape = [normalized_shape] @@ -1811,7 +2043,8 @@ class LayerNorm(layers.Layer): attr=self._param_attr, shape=param_shape, dtype=self._dtype, - default_initializer=Constant(1.0)) + default_initializer=Constant(1.0), + ) else: if self._param_attr: logging.warn("param_attr are only available with scale is True") @@ -1819,10 +2052,12 @@ class LayerNorm(layers.Layer): if self._shift: assert self._bias_attr is not False - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=True, + ) else: if self._bias_attr: logging.warn("bias_attr are only available with shift is True") @@ -1833,31 +2068,50 @@ class LayerNorm(layers.Layer): input_ndim = len(input_shape) normalized_ndim = len(self._normalized_shape) self._begin_norm_axis = input_ndim - normalized_ndim - if input_ndim < normalized_ndim or input_shape[ - self._begin_norm_axis:] != self._normalized_shape: + if ( + input_ndim < normalized_ndim + or input_shape[self._begin_norm_axis :] != self._normalized_shape + ): str_normalized_shape = str(self._normalized_shape) - raise ValueError('Given normalized_shape is ' + - str_normalized_shape + - ', expected input with shape [*, ' + - str_normalized_shape[1:] + - ', but got input shape ' + str(input_shape)) + raise ValueError( + 'Given normalized_shape is ' + + str_normalized_shape + + ', expected input with shape [*, ' + + str_normalized_shape[1:] + + ', but got input shape ' + + str(input_shape) + ) if _non_static_mode(): if in_dygraph_mode(): - pre_act, _, _, = _C_ops.layer_norm(input, self.weight, - self.bias, self._epsilon, - self._begin_norm_axis, False) + pre_act, _, _, = _C_ops.layer_norm( + input, + self.weight, + self.bias, + self._epsilon, + self._begin_norm_axis, + False, + ) return dygraph_utils._append_activation_in_dygraph( - pre_act, act=self._act) + pre_act, act=self._act + ) else: pre_act, _, _ = _legacy_C_ops.layer_norm( - input, self.weight, self.bias, 'epsilon', self._epsilon, - 'begin_norm_axis', self._begin_norm_axis) + input, + self.weight, + self.bias, + 'epsilon', + self._epsilon, + 'begin_norm_axis', + self._begin_norm_axis, + ) return dygraph_utils._append_activation_in_dygraph( - pre_act, act=self._act) + pre_act, act=self._act + ) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'LayerNorm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'LayerNorm' + ) inputs = dict() inputs['X'] = [input] @@ -1867,28 +2121,33 @@ class LayerNorm(layers.Layer): inputs['Bias'] = [self.bias] attrs = { "epsilon": self._epsilon, - "begin_norm_axis": self._begin_norm_axis + "begin_norm_axis": self._begin_norm_axis, } # create output mean_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) variance_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) layer_norm_out = self._helper.create_variable_for_type_inference( - self._dtype) - - self._helper.append_op(type="layer_norm", - inputs=inputs, - outputs={ - "Y": layer_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": self._epsilon, - "begin_norm_axis": self._begin_norm_axis - }) + self._dtype + ) + + self._helper.append_op( + type="layer_norm", + inputs=inputs, + outputs={ + "Y": layer_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={ + "epsilon": self._epsilon, + "begin_norm_axis": self._begin_norm_axis, + }, + ) return self._helper.append_activation(layer_norm_out, act=self._act) @@ -2002,14 +2261,16 @@ class GRUUnit(layers.Layer): """ - def __init__(self, - size, - param_attr=None, - bias_attr=None, - activation='tanh', - gate_activation='sigmoid', - origin_mode=False, - dtype='float32'): + def __init__( + self, + size, + param_attr=None, + bias_attr=None, + activation='tanh', + gate_activation='sigmoid', + origin_mode=False, + dtype='float32', + ): super(GRUUnit, self).__init__() self._bias_attr = bias_attr activation_dict = dict( @@ -2024,52 +2285,64 @@ class GRUUnit(layers.Layer): self._dtype = dtype size = size // 3 # create weight - self.weight = self.create_parameter(attr=param_attr, - shape=[size, 3 * size], - dtype=dtype) + self.weight = self.create_parameter( + attr=param_attr, shape=[size, 3 * size], dtype=dtype + ) # create bias bias_size = [1, 3 * size] self._bias_size = bias_size - self.bias = self.create_parameter(attr=bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) def forward(self, input, hidden): if _non_static_mode(): gate, reset_hidden_pre, updated_hidden = _legacy_C_ops.gru_unit( - input, hidden, self.weight, self.bias, 'activation', - self.activation, 'gate_activation', self.gate_activation) + input, + hidden, + self.weight, + self.bias, + 'activation', + self.activation, + 'gate_activation', + self.gate_activation, + ) return updated_hidden, reset_hidden_pre, gate - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'GRUUnit') - check_variable_and_dtype(hidden, 'hidden', ['float32', 'float64'], - 'GRUUnit') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'GRUUnit' + ) + check_variable_and_dtype( + hidden, 'hidden', ['float32', 'float64'], 'GRUUnit' + ) inputs = { 'Input': [input], 'HiddenPrev': [hidden], - 'Weight': [self.weight] + 'Weight': [self.weight], } if self.bias is not None: inputs['Bias'] = [self.bias] gate = self._helper.create_variable_for_type_inference(self._dtype) reset_hidden_pre = self._helper.create_variable_for_type_inference( - self._dtype) + self._dtype + ) updated_hidden = self._helper.create_variable_for_type_inference( - self._dtype) - self._helper.append_op(type='gru_unit', - inputs=inputs, - outputs={ - 'Gate': gate, - 'ResetHiddenPrev': reset_hidden_pre, - 'Hidden': updated_hidden, - }, - attrs={ - 'activation': self.activation, - 'gate_activation': self.gate_activation, - }) + self._dtype + ) + self._helper.append_op( + type='gru_unit', + inputs=inputs, + outputs={ + 'Gate': gate, + 'ResetHiddenPrev': reset_hidden_pre, + 'Hidden': updated_hidden, + }, + attrs={ + 'activation': self.activation, + 'gate_activation': self.gate_activation, + }, + ) return updated_hidden, reset_hidden_pre, gate @@ -2161,25 +2434,29 @@ class NCE(layers.Layer): """ - def __init__(self, - num_total_classes, - dim, - sample_weight=None, - param_attr=None, - bias_attr=None, - num_neg_samples=None, - sampler="uniform", - custom_dist=None, - seed=0, - is_sparse=False, - dtype='float32'): + def __init__( + self, + num_total_classes, + dim, + sample_weight=None, + param_attr=None, + bias_attr=None, + num_neg_samples=None, + sampler="uniform", + custom_dist=None, + seed=0, + is_sparse=False, + dtype='float32', + ): super(NCE, self).__init__() self._param_attr = param_attr self._bias_attr = bias_attr self._num_total_classes = num_total_classes self._dtype = dtype self._inputs = dict() - self._inputs['SampleWeight'] = sample_weight if sample_weight is not None else [] + self._inputs['SampleWeight'] = ( + sample_weight if sample_weight is not None else [] + ) if sampler == "uniform": sampler = 0 elif sampler == "log_uniform": @@ -2235,16 +2512,20 @@ class NCE(layers.Layer): attr=ParamAttr(), shape=numpy_array.shape, dtype=numpy_array.dtype, - default_initializer=NumpyArrayInitializer(numpy_array)) + default_initializer=NumpyArrayInitializer(numpy_array), + ) ret.stop_gradient = True return ret self._inputs['CustomDistProbs'] = _init_by_numpy_array( - np.array(custom_dist).astype('float32')) + np.array(custom_dist).astype('float32') + ) self._inputs['CustomDistAlias'] = _init_by_numpy_array( - np.array(alias_).astype('int32')) + np.array(alias_).astype('int32') + ) self._inputs['CustomDistAliasProbs'] = _init_by_numpy_array( - np.array(alias_probs_).astype('float32')) + np.array(alias_probs_).astype('float32') + ) sampler = 2 else: raise Exception("Unsupported sampler type.") @@ -2264,64 +2545,88 @@ class NCE(layers.Layer): 'seed': seed, 'sampler': sampler, 'is_sparse': is_sparse, - 'remote_prefetch': remote_prefetch + 'remote_prefetch': remote_prefetch, } self.weight = self.create_parameter( attr=self._param_attr, shape=[self._num_total_classes, dim], is_bias=False, - dtype=self._dtype) + dtype=self._dtype, + ) if self._bias_attr: self.bias = self.create_parameter( attr=self._bias_attr, shape=[self._num_total_classes, 1], is_bias=True, - dtype=self._dtype) + dtype=self._dtype, + ) self._inputs['Bias'] = self.bias self._inputs['Weight'] = self.weight def forward(self, input, label, sample_weight=None): if _non_static_mode(): - attrs = ('num_total_classes', self._attrs['num_total_classes'], - 'num_neg_samples', self._attrs['num_neg_samples'], 'seed', - self._attrs['seed'], 'sampler', self._attrs['sampler'], - 'is_sparse', self._attrs['is_sparse'], 'remote_prefetch', - self._attrs['remote_prefetch']) - cost, _, _ = _legacy_C_ops.nce(input, label, self.weight, self.bias, - self._inputs['SampleWeight'], - self._inputs['CustomDistProbs'], - self._inputs['CustomDistAlias'], - self._inputs['CustomDistAliasProbs'], - *attrs) + attrs = ( + 'num_total_classes', + self._attrs['num_total_classes'], + 'num_neg_samples', + self._attrs['num_neg_samples'], + 'seed', + self._attrs['seed'], + 'sampler', + self._attrs['sampler'], + 'is_sparse', + self._attrs['is_sparse'], + 'remote_prefetch', + self._attrs['remote_prefetch'], + ) + cost, _, _ = _legacy_C_ops.nce( + input, + label, + self.weight, + self.bias, + self._inputs['SampleWeight'], + self._inputs['CustomDistProbs'], + self._inputs['CustomDistAlias'], + self._inputs['CustomDistAliasProbs'], + *attrs + ) return cost / (self._num_neg_samples + 1) check_variable_and_dtype(input, "input", ['float32', 'float64'], "NCE") check_variable_and_dtype(label, "label", ['int64'], "NCE") - check_type(sample_weight, 'sample_weight', (Variable, type(None)), - 'NCE') + check_type( + sample_weight, 'sample_weight', (Variable, type(None)), 'NCE' + ) assert isinstance(input, Variable) assert isinstance(label, Variable) self._inputs['Input'] = input self._inputs['Label'] = label - self._inputs['SampleWeight'] = sample_weight if sample_weight is not None else [] + self._inputs['SampleWeight'] = ( + sample_weight if sample_weight is not None else [] + ) cost = self._helper.create_variable_for_type_inference( - dtype=input.dtype) + dtype=input.dtype + ) sample_logits = self._helper.create_variable_for_type_inference( - dtype=input.dtype) + dtype=input.dtype + ) sample_labels = self._helper.create_variable_for_type_inference( - dtype=label.dtype) - - self._helper.append_op(type='nce', - inputs=self._inputs, - outputs={ - 'Cost': cost, - 'SampleLogits': sample_logits, - 'SampleLabels': sample_labels - }, - attrs=self._attrs) + dtype=label.dtype + ) + + self._helper.append_op( + type='nce', + inputs=self._inputs, + outputs={ + 'Cost': cost, + 'SampleLogits': sample_logits, + 'SampleLabels': sample_labels, + }, + attrs=self._attrs, + ) return cost / (self._num_neg_samples + 1) @@ -2385,12 +2690,14 @@ class PRelu(layers.Layer): """ - def __init__(self, - mode, - channel=None, - input_shape=None, - param_attr=None, - dtype='float32'): + def __init__( + self, + mode, + channel=None, + input_shape=None, + param_attr=None, + dtype='float32', + ): # need specify name_scope since snake-cased 'PRelu' is 'p_relu' super(PRelu, self).__init__(name_scope='prelu') self._mode = mode @@ -2400,26 +2707,27 @@ class PRelu(layers.Layer): self._alpha_shape = [1] elif mode == 'channel': assert isinstance( - channel, - int), "channel argument is required when mode is 'channel'." - #NOTE(zhiqiu): The _alpha_shape should be [1, channel] + [1] * len(input_shape[2:]), not [1, channel, 1, 1]. + channel, int + ), "channel argument is required when mode is 'channel'." + # NOTE(zhiqiu): The _alpha_shape should be [1, channel] + [1] * len(input_shape[2:]), not [1, channel, 1, 1]. # However, the suffix 1 in the list is useless, since the tensor is viewed as one demension array during kernel calculation. # And, input_shape is not required when mode is 'channel', so it is simplified. - #NOTE(zhiqiu): Revert shape to [1, channel, 1, 1] for compatibility with saved model of old version. + # NOTE(zhiqiu): Revert shape to [1, channel, 1, 1] for compatibility with saved model of old version. self._alpha_shape = [1, channel, 1, 1] elif mode == 'element': assert isinstance( - input_shape, - (list, tuple - )), "input_shape argument is required when mode is 'element'." + input_shape, (list, tuple) + ), "input_shape argument is required when mode is 'element'." self._alpha_shape = [1] + list(input_shape)[1:] else: raise ValueError('mode should be one of all, channel, element.') - self.weight = self.create_parameter(attr=self._param_attr, - shape=self._alpha_shape, - dtype='float32', - is_bias=False, - default_initializer=Constant(1.0)) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=self._alpha_shape, + dtype='float32', + is_bias=False, + default_initializer=Constant(1.0), + ) def forward(self, input): if in_dygraph_mode(): @@ -2427,13 +2735,12 @@ class PRelu(layers.Layer): check_variable_and_dtype(input, 'input', ['float32'], 'PRelu') out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="prelu", - inputs={ - "X": input, - 'Alpha': self.weight - }, - attrs={"mode": self._mode}, - outputs={"Out": out}) + self._helper.append_op( + type="prelu", + inputs={"X": input, 'Alpha': self.weight}, + attrs={"mode": self._mode}, + outputs={"Out": out}, + ) return out @@ -2492,15 +2799,17 @@ class BilinearTensorProduct(layers.Layer): """ - def __init__(self, - input1_dim, - input2_dim, - output_dim, - name=None, - act=None, - param_attr=None, - bias_attr=None, - dtype='float32'): + def __init__( + self, + input1_dim, + input2_dim, + output_dim, + name=None, + act=None, + param_attr=None, + bias_attr=None, + dtype='float32', + ): super(BilinearTensorProduct, self).__init__() self._param_attr = param_attr self._bias_attr = bias_attr @@ -2513,38 +2822,50 @@ class BilinearTensorProduct(layers.Layer): self._dtype = dtype param_shape = [self._output_dim, self._input1_dim, self._input2_dim] - self.weight = self.create_parameter(attr=self._param_attr, - shape=param_shape, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=False, + ) bias_size = [1, self._output_dim] - self.bias = self.create_parameter(attr=self._bias_attr, - shape=bias_size, - dtype=self._dtype, - is_bias=True) - - @deprecated(since="2.0.0", - update_to="paddle.nn.Bilinear", - reason="New name and new args in Bilinear, easier to use.") + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=bias_size, + dtype=self._dtype, + is_bias=True, + ) + + @deprecated( + since="2.0.0", + update_to="paddle.nn.Bilinear", + reason="New name and new args in Bilinear, easier to use.", + ) def forward(self, x, y): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'BilinearTensorProduct') - check_variable_and_dtype(y, 'y', ['float32', 'float64'], - 'BilinearTensorProduct') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'BilinearTensorProduct' + ) + check_variable_and_dtype( + y, 'y', ['float32', 'float64'], 'BilinearTensorProduct' + ) self._inputs = {"X": x, "Y": y, "Weight": self.weight} if self.bias is not None: self._inputs["Bias"] = self.bias if self._name is not None: - out = self._helper.create_variable(name=".".join( - [self.full_name(), self._name]), - dtype=self._dtype, - persistable=False) + out = self._helper.create_variable( + name=".".join([self.full_name(), self._name]), + dtype=self._dtype, + persistable=False, + ) else: - out = self._helper.create_variable(dtype=self._dtype, - persistable=False) - self._helper.append_op(type="bilinear_tensor_product", - inputs=self._inputs, - outputs={"Out": out}) + out = self._helper.create_variable( + dtype=self._dtype, persistable=False + ) + self._helper.append_op( + type="bilinear_tensor_product", + inputs=self._inputs, + outputs={"Out": out}, + ) # add activation return self._helper.append_activation(out, act=self._act) @@ -2668,22 +2989,26 @@ class Conv2DTranspose(layers.Layer): """ - def __init__(self, - num_channels, - num_filters, - filter_size, - output_size=None, - padding=0, - stride=1, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - dtype='float32'): + def __init__( + self, + num_channels, + num_filters, + filter_size, + output_size=None, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + dtype='float32', + ): super(Conv2DTranspose, self).__init__() - assert param_attr is not False, "param_attr should not be False in conv2d_transpose." + assert ( + param_attr is not False + ), "param_attr should not be False in conv2d_transpose." self._param_attr = param_attr self._bias_attr = bias_attr self._act = act @@ -2698,9 +3023,11 @@ class Conv2DTranspose(layers.Layer): self._output_size = output_size self._dtype = dtype - if (self._num_channels == self._groups - and self._num_filters == self._num_channels - and not self._use_cudnn): + if ( + self._num_channels == self._groups + and self._num_filters == self._num_channels + and not self._use_cudnn + ): self._op_type = 'depthwise_conv2d_transpose' else: self._op_type = 'conv2d_transpose' @@ -2710,63 +3037,91 @@ class Conv2DTranspose(layers.Layer): self._dilation = utils.convert_to_list(self._dilation, 2, 'dilation') self._filter_size = utils.convert_to_list( - self._filter_size, 2, 'conv2d_transpose.filter_size') + self._filter_size, 2, 'conv2d_transpose.filter_size' + ) if self._output_size is None: self._output_size = [] elif isinstance(self._output_size, list): if utils._contain_var(self._output_size): self._output_size = utils._convert_to_tensor_list( - self._output_size) + self._output_size + ) else: self._output_size = utils.convert_to_list( - self._output_size, 2, 'output_size') + self._output_size, 2, 'output_size' + ) elif isinstance(self._output_size, int): - self._output_size = utils.convert_to_list(self._output_size, 2, - 'output_size') + self._output_size = utils.convert_to_list( + self._output_size, 2, 'output_size' + ) elif isinstance(self._output_size, Variable): - check_dtype(self._output_size.dtype, 'output_size', - ['int32', 'int64'], 'Conv2DTranspose') + check_dtype( + self._output_size.dtype, + 'output_size', + ['int32', 'int64'], + 'Conv2DTranspose', + ) if len(self._output_size.shape) == 1 and ( - self._output_size.shape[0] == 1 - or self._output_size.shape[0] == 2): + self._output_size.shape[0] == 1 + or self._output_size.shape[0] == 2 + ): if self._output_size.shape[0] == 1: self._output_size = [self._output_size, self._output_size] else: raise ValueError( - "output_size must contain one or two integers.") + "output_size must contain one or two integers." + ) else: raise ValueError("output_size should be list or int or Tensor") self._padding = utils.convert_to_list(self._padding, 2, 'padding') self._groups = 1 if self._groups is None else self._groups - filter_shape = [self._num_channels, self._num_filters // self._groups - ] + self._filter_size + filter_shape = [ + self._num_channels, + self._num_filters // self._groups, + ] + self._filter_size - self.weight = self.create_parameter(dtype=self._dtype, - shape=filter_shape, - attr=self._param_attr) + self.weight = self.create_parameter( + dtype=self._dtype, shape=filter_shape, attr=self._param_attr + ) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): if _non_static_mode(): op = getattr(_legacy_C_ops, self._op_type) - out = op(input, self.weight, 'output_size', self._output_size, - 'strides', self._stride, 'paddings', self._padding, - 'dilations', self._dilation, 'groups', self._groups, - 'use_cudnn', self._use_cudnn) + out = op( + input, + self.weight, + 'output_size', + self._output_size, + 'strides', + self._stride, + 'paddings', + self._padding, + 'dilations', + self._dilation, + 'groups', + self._groups, + 'use_cudnn', + self._use_cudnn, + ) pre_bias = out pre_act = dygraph_utils._append_bias_in_dygraph( - pre_bias, self.bias, 1) - return dygraph_utils._append_activation_in_dygraph(pre_act, - act=self._act) + pre_bias, self.bias, 1 + ) + return dygraph_utils._append_activation_in_dygraph( + pre_act, act=self._act + ) - check_variable_and_dtype(input, 'input', - ['float16', 'float32', 'float64'], - "Conv2DTranspose") + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], "Conv2DTranspose" + ) inputs = {'Input': [input], 'Filter': [self.weight]} attrs = { @@ -2775,26 +3130,29 @@ class Conv2DTranspose(layers.Layer): 'paddings': self._padding, 'dilations': self._dilation, 'groups': self._groups, - 'use_cudnn': self._use_cudnn + 'use_cudnn': self._use_cudnn, } pre_bias = self._helper.create_variable_for_type_inference( - dtype=input.dtype) - self._helper.append_op(type=self._op_type, - inputs=inputs, - outputs={'Output': pre_bias}, - attrs=attrs) + dtype=input.dtype + ) + self._helper.append_op( + type=self._op_type, + inputs=inputs, + outputs={'Output': pre_bias}, + attrs=attrs, + ) if self.bias is not None: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}, + ) else: pre_act = pre_bias @@ -2834,16 +3192,19 @@ class SequenceConv(layers.Layer): Variable: output of sequence_conv """ - def __init__(self, - name_scope, - num_filters, - filter_size=3, - filter_stride=1, - padding=None, - bias_attr=None, - param_attr=None, - act=None): - assert not _non_static_mode( + def __init__( + self, + name_scope, + num_filters, + filter_size=3, + filter_stride=1, + padding=None, + bias_attr=None, + param_attr=None, + act=None, + ): + assert ( + not _non_static_mode() ), "SequenceConv is not supported by dynamic graph mode yet!" super(SequenceConv, self).__init__(name_scope) self._num_filters = num_filters @@ -2857,39 +3218,43 @@ class SequenceConv(layers.Layer): def _build_once(self, input): self._dtype = self._helper.input_dtype(input) filter_shape = [self._filter_size * input.shape[1], self._num_filters] - self.weight = self.create_parameter(attr=self._param_attr, - shape=filter_shape, - dtype=self._dtype) + self.weight = self.create_parameter( + attr=self._param_attr, shape=filter_shape, dtype=self._dtype + ) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): pre_bias = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type='sequence_conv', - inputs={ - 'X': [input], - 'Filter': [self.weight], - }, - outputs={"Out": pre_bias}, - attrs={ - 'contextStride': self._filter_stride, - 'contextStart': -int(self._filter_size // 2), - 'contextLength': self._filter_size - }) + self._helper.append_op( + type='sequence_conv', + inputs={ + 'X': [input], + 'Filter': [self.weight], + }, + outputs={"Out": pre_bias}, + attrs={ + 'contextStride': self._filter_stride, + 'contextStart': -int(self._filter_size // 2), + 'contextLength': self._filter_size, + }, + ) if self.bias is not None: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}, + ) else: pre_act = pre_bias @@ -2944,12 +3309,11 @@ class RowConv(layers.Layer): """ - def __init__(self, - name_scope, - future_context_size, - param_attr=None, - act=None): - assert not _non_static_mode( + def __init__( + self, name_scope, future_context_size, param_attr=None, act=None + ): + assert ( + not _non_static_mode() ), "RowConv is not supported by dynamic graph mode yet!" super(RowConv, self).__init__(name_scope) self._act = act @@ -2959,27 +3323,28 @@ class RowConv(layers.Layer): def _build_once(self, input): self._dtype = self._helper.input_dtype(input) filter_shape = [self._future_context_size + 1, input.shape[1]] - self.weight = self.create_parameter(attr=self._param_attr, - shape=filter_shape, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=filter_shape, + dtype=self._dtype, + is_bias=False, + ) def forward(self, input): out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type='row_conv', - inputs={ - 'X': [input], - 'Filter': [self.weight] - }, - outputs={'Out': [out]}) + self._helper.append_op( + type='row_conv', + inputs={'X': [input], 'Filter': [self.weight]}, + outputs={'Out': [out]}, + ) return self._helper.append_activation(out, act=self._act) class GroupNorm(layers.Layer): """ :alias_main: paddle.nn.GroupNorm - :alias: paddle.nn.GroupNorm,paddle.nn.layer.GroupNorm,paddle.nn.layer.norm.GroupNorm - :old_api: paddle.fluid.dygraph.GroupNorm + :alias: paddle.nn.GroupNorm,paddle.nn.layer.GroupNorm,paddle.nn.layer.norm.GroupNorm + :old_api: paddle.fluid.dygraph.GroupNorm This interface is used to construct a callable object of the ``GroupNorm`` class. For more details, refer to code examples. @@ -3016,15 +3381,17 @@ class GroupNorm(layers.Layer): """ - def __init__(self, - channels, - groups, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None, - data_layout='NCHW', - dtype='float32'): + def __init__( + self, + channels, + groups, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + data_layout='NCHW', + dtype='float32', + ): super(GroupNorm, self).__init__() self._param_attr = param_attr self._bias_attr = bias_attr @@ -3038,31 +3405,44 @@ class GroupNorm(layers.Layer): param_shape = [self._channels] - self.weight = self.create_parameter(attr=self._param_attr or False, - shape=param_shape, - dtype=self._dtype, - default_initializer=Constant(1.0)) + self.weight = self.create_parameter( + attr=self._param_attr or False, + shape=param_shape, + dtype=self._dtype, + default_initializer=Constant(1.0), + ) - self.bias = self.create_parameter(attr=self._bias_attr or False, - shape=param_shape, - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr or False, + shape=param_shape, + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): mean_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) variance_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) if in_dygraph_mode(): - out = _C_ops.group_norm(input, self.weight, self.bias, - self._epsilon, self._groups, "NCHW") + out = _C_ops.group_norm( + input, + self.weight, + self.bias, + self._epsilon, + self._groups, + "NCHW", + ) return dygraph_utils._append_activation_in_dygraph(out, self._act) elif _in_legacy_dygraph(): attrs = ('epsilon', self._epsilon, 'groups', self._groups) - out, _, _ = _legacy_C_ops.group_norm(input, self.weight, self.bias, - mean_out, variance_out, *attrs) + out, _, _ = _legacy_C_ops.group_norm( + input, self.weight, self.bias, mean_out, variance_out, *attrs + ) return dygraph_utils._append_activation_in_dygraph(out, self._act) else: @@ -3074,19 +3454,19 @@ class GroupNorm(layers.Layer): # create output group_norm_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - - self._helper.append_op(type="group_norm", - inputs=inputs, - outputs={ - "Y": group_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": self._epsilon, - "groups": self._groups - }) + dtype=self._dtype + ) + + self._helper.append_op( + type="group_norm", + inputs=inputs, + outputs={ + "Y": group_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": self._epsilon, "groups": self._groups}, + ) return self._helper.append_activation(group_norm_out, self._act) @@ -3150,12 +3530,9 @@ class SpectralNorm(layers.Layer): """ - def __init__(self, - weight_shape, - dim=0, - power_iters=1, - eps=1e-12, - dtype='float32'): + def __init__( + self, weight_shape, dim=0, power_iters=1, eps=1e-12, dtype='float32' + ): super(SpectralNorm, self).__init__() self._power_iters = power_iters self._eps = eps @@ -3163,48 +3540,61 @@ class SpectralNorm(layers.Layer): self._dtype = dtype self._weight_shape = list(weight_shape) - assert np.prod(self._weight_shape) > 0,\ - "Any dimension of `weight_shape` cannot be equal to 0." - assert dim < len(self._weight_shape), \ - ("The input `dim` should be less than the " + assert ( + np.prod(self._weight_shape) > 0 + ), "Any dimension of `weight_shape` cannot be equal to 0." + assert dim < len(self._weight_shape), ( + "The input `dim` should be less than the " "length of `weight_shape`, but received dim=" - "{}".format(dim)) + "{}".format(dim) + ) h = self._weight_shape[self._dim] w = np.prod(self._weight_shape) // h - self.weight_u = self.create_parameter(attr=ParamAttr(), - shape=[h], - dtype=self._dtype, - default_initializer=Normal( - 0., 1.)) + self.weight_u = self.create_parameter( + attr=ParamAttr(), + shape=[h], + dtype=self._dtype, + default_initializer=Normal(0.0, 1.0), + ) self.weight_u.stop_gradient = True - self.weight_v = self.create_parameter(attr=ParamAttr(), - shape=[w], - dtype=self._dtype, - default_initializer=Normal( - 0., 1.)) + self.weight_v = self.create_parameter( + attr=ParamAttr(), + shape=[w], + dtype=self._dtype, + default_initializer=Normal(0.0, 1.0), + ) self.weight_v.stop_gradient = True def forward(self, weight): if in_dygraph_mode(): - return _C_ops.spectral_norm(weight, self.weight_u, self.weight_v, - self._dim, self._power_iters, self._eps) + return _C_ops.spectral_norm( + weight, + self.weight_u, + self.weight_v, + self._dim, + self._power_iters, + self._eps, + ) - check_variable_and_dtype(weight, "weight", ['float32', 'float64'], - 'SpectralNorm') + check_variable_and_dtype( + weight, "weight", ['float32', 'float64'], 'SpectralNorm' + ) inputs = {'Weight': weight, 'U': self.weight_u, 'V': self.weight_v} out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="spectral_norm", - inputs=inputs, - outputs={ - "Out": out, - }, - attrs={ - "dim": self._dim, - "power_iters": self._power_iters, - "eps": self._eps, - }) + self._helper.append_op( + type="spectral_norm", + inputs=inputs, + outputs={ + "Out": out, + }, + attrs={ + "dim": self._dim, + "power_iters": self._power_iters, + "eps": self._eps, + }, + ) return out @@ -3254,16 +3644,18 @@ class TreeConv(layers.Layer): ret = treeConv(fluid.dygraph.base.to_variable(nodes_vector), fluid.dygraph.base.to_variable(edge_set)) """ - def __init__(self, - feature_size, - output_size, - num_filters=1, - max_depth=2, - act='tanh', - param_attr=None, - bias_attr=None, - name=None, - dtype='float32'): + def __init__( + self, + feature_size, + output_size, + num_filters=1, + max_depth=2, + act='tanh', + param_attr=None, + bias_attr=None, + name=None, + dtype='float32', + ): super(TreeConv, self).__init__() self._name = name self._feature_size = feature_size @@ -3276,45 +3668,52 @@ class TreeConv(layers.Layer): self._dtype = dtype w_shape = [self._feature_size, 3, self._output_size, self._num_filters] if self._bias_attr: - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) - self.weight = self.create_parameter(attr=self._param_attr, - shape=w_shape, - dtype=self._dtype, - is_bias=False) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=w_shape, + dtype=self._dtype, + is_bias=False, + ) def forward(self, nodes_vector, edge_set): check_type(nodes_vector, 'nodes_vector', (Variable), 'TreeConv') check_type(edge_set, 'edge_set', (Variable), 'TreeConv') if self._name: - out = self.create_variable(name=self._name, - dtype=self._dtype, - persistable=False) + out = self.create_variable( + name=self._name, dtype=self._dtype, persistable=False + ) else: out = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='tree_conv', - inputs={ - 'NodesVector': nodes_vector, - 'EdgeSet': edge_set, - 'Filter': self.weight - }, - outputs={ - 'Out': out, - }, - attrs={'max_depth': self._max_depth}) + dtype=self._dtype + ) + self._helper.append_op( + type='tree_conv', + inputs={ + 'NodesVector': nodes_vector, + 'EdgeSet': edge_set, + 'Filter': self.weight, + }, + outputs={ + 'Out': out, + }, + attrs={'max_depth': self._max_depth}, + ) if self._bias_attr: pre_activation = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [out], - 'Y': [self.bias] - }, - outputs={'Out': [pre_activation]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [out], 'Y': [self.bias]}, + outputs={'Out': [pre_activation]}, + attrs={'axis': 1}, + ) else: pre_activation = out return self._helper.append_activation(pre_activation, act=self._act) @@ -3353,7 +3752,7 @@ class Flatten(layers.Layer): self.stop_axis = stop_axis def forward(self, input): - out = paddle.tensor.manipulation.flatten(input, - start_axis=self.start_axis, - stop_axis=self.stop_axis) + out = paddle.tensor.manipulation.flatten( + input, start_axis=self.start_axis, stop_axis=self.stop_axis + ) return out diff --git a/python/paddle/fluid/dygraph/parallel.py b/python/paddle/fluid/dygraph/parallel.py index bd0a4c9e5d2..51e0527e4fa 100644 --- a/python/paddle/fluid/dygraph/parallel.py +++ b/python/paddle/fluid/dygraph/parallel.py @@ -30,7 +30,12 @@ from paddle.fluid.dygraph import to_variable, no_grad from paddle.utils import deprecated from ..layers import collective from paddle.fluid.dygraph import base as imperative_base -from paddle.fluid.framework import ParamBase, _in_legacy_dygraph, _non_static_mode, in_dygraph_mode +from paddle.fluid.framework import ( + ParamBase, + _in_legacy_dygraph, + _non_static_mode, + in_dygraph_mode, +) __all__ = ["prepare_context", "ParallelEnv", "DataParallel"] @@ -50,24 +55,29 @@ def prepare_context(strategy=None): strategy.current_endpoint = Env().current_endpoint if strategy.nranks < 2: return - assert framework._non_static_mode() is True, \ - "dygraph.prepare_context should be used with dygraph mode." + assert ( + framework._non_static_mode() is True + ), "dygraph.prepare_context should be used with dygraph mode." place = framework._current_expected_place() - assert place is not None, \ - "dygraph.prepare_context should be used in fluid.dygraph.guard(place) guard." + assert ( + place is not None + ), "dygraph.prepare_context should be used in fluid.dygraph.guard(place) guard." if not parallel_helper._is_parallel_ctx_initialized(): if isinstance(place, core.CUDAPlace): parallel_helper._set_parallel_ctx( - core.NCCLParallelContext(strategy, place)) + core.NCCLParallelContext(strategy, place) + ) elif isinstance(place, core.XPUPlace): parallel_helper._set_parallel_ctx( - core.BKCLParallelContext(strategy, place)) + core.BKCLParallelContext(strategy, place) + ) elif isinstance(place, core.NPUPlace): parallel_helper._set_parallel_ctx( - core.HCCLParallelContext(strategy, place)) + core.HCCLParallelContext(strategy, place) + ) else: # TODO(Yancey1989): add Gloo Parallel Context to support CPU parallel computation - assert ("Only support CUDAPlace or XPUPlace or NPUPlace for now.") + assert "Only support CUDAPlace or XPUPlace or NPUPlace for now." parallel_helper._init_parallel_ctx() return strategy @@ -122,9 +132,11 @@ class ParallelEnv(object): # imperative only support one gpu or xpu if self._device_type != "": FLAGS_selected_custom_devices = 'FLAGS_selected_{}s'.format( - self._device_type) - selected_custom_devices = os.getenv(FLAGS_selected_custom_devices, - "0").split(",") + self._device_type + ) + selected_custom_devices = os.getenv( + FLAGS_selected_custom_devices, "0" + ).split(",") self._device_id = int(selected_custom_devices[0]) else: if core.is_compiled_with_cuda(): @@ -140,14 +152,17 @@ class ParallelEnv(object): selected_mlus = os.getenv("FLAGS_selected_mlus", "0").split(",") self._device_id = int(selected_mlus[0]) - self._trainer_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS", - "").split(",") + self._trainer_endpoints = os.getenv( + "PADDLE_TRAINER_ENDPOINTS", "" + ).split(",") self._current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT", "") self._nrings = int(os.getenv("FLAGS_nccl_nrings", "1")) - assert self._nrings > 0, \ - "nccl_nrings must be an integer greater than 0." - assert self._nrings < 9, \ - "nccl_nrings should be less than 9, which is enough in most scenarios." + assert ( + self._nrings > 0 + ), "nccl_nrings must be an integer greater than 0." + assert ( + self._nrings < 9 + ), "nccl_nrings should be less than 9, which is enough in most scenarios." @property def rank(self): @@ -297,6 +312,7 @@ def _build_default_parallel_strategy(): def _coalesce_tensors(var_groups): from ..layers import nn + coalesced_grads_and_grad_vars = [] for group_id, grad_vars in var_groups.items(): flattened_vars = [] @@ -304,43 +320,50 @@ def _coalesce_tensors(var_groups): for g_var in grad_vars: g_var_shapes.append(g_var.shape) flattened_vars.append( - nn.reshape(x=g_var, shape=[np.prod(g_var.shape)])) + nn.reshape(x=g_var, shape=[np.prod(g_var.shape)]) + ) coalesced_grad = nn.concat(flattened_vars) coalesced_grads_and_grad_vars.append( - [coalesced_grad, grad_vars, g_var_shapes]) + [coalesced_grad, grad_vars, g_var_shapes] + ) return coalesced_grads_and_grad_vars @framework.dygraph_only def _reshape_inplace(x, shape): x_shape = framework._varbase_creator(dtype=x.dtype) - framework._dygraph_tracer().trace_op(type="reshape2", - inputs={'X': x}, - outputs={ - 'Out': x, - 'XShape': x_shape - }, - attrs={'shape': shape}) + framework._dygraph_tracer().trace_op( + type="reshape2", + inputs={'X': x}, + outputs={'Out': x, 'XShape': x_shape}, + attrs={'shape': shape}, + ) @framework.dygraph_only def _split_tensors(coalesced_grads_and_grad_vars): if _in_legacy_dygraph(): - for coalesced_grad, origin_grad_vars, grad_shapes in coalesced_grads_and_grad_vars: + for ( + coalesced_grad, + origin_grad_vars, + grad_shapes, + ) in coalesced_grads_and_grad_vars: grad_var_len = [np.prod(g_shape) for g_shape in grad_shapes] framework._dygraph_tracer().trace_op( type='split', inputs={'X': coalesced_grad}, outputs={'Out': origin_grad_vars}, - attrs={ - 'sections': grad_var_len, - 'axis': 0 - }) + attrs={'sections': grad_var_len, 'axis': 0}, + ) for g_var, g_shape in zip(origin_grad_vars, grad_shapes): _reshape_inplace(x=g_var, shape=g_shape) assert g_var.shape == g_shape elif in_dygraph_mode(): - for coalesced_grad, origin_grad_vars, grad_shapes in coalesced_grads_and_grad_vars: + for ( + coalesced_grad, + origin_grad_vars, + grad_shapes, + ) in coalesced_grads_and_grad_vars: grad_var_len = [np.prod(g_shape) for g_shape in grad_shapes] attrs = () attrs += ('sections', grad_var_len) @@ -357,7 +380,8 @@ def scale_loss(loss): return loss loss_scale = to_variable( - np.array([ParallelEnv().world_size]).astype("float32")) + np.array([ParallelEnv().world_size]).astype("float32") + ) loss_scale.stop_gradient = True scaled_loss = loss / loss_scale return scaled_loss @@ -385,16 +409,16 @@ def build_groups(vars, group_size): @imperative_base.no_grad @framework.dygraph_only -def sync_params_buffers(model, - comm_group=None, - src_rank=0, - is_model_parallel=False): +def sync_params_buffers( + model, comm_group=None, src_rank=0, is_model_parallel=False +): model_vars = [] for _, param in model._obtain_parameters_buffers().items(): if not isinstance(param, (core.VarBase, core.eager.Tensor)): raise TypeError( - "The data type of '%s' must be Varbase or eager.Tensor" % - param.name) + "The data type of '%s' must be Varbase or eager.Tensor" + % param.name + ) # is_distributed param not need to sync when in mp mode if isinstance(param, (ParamBase, core.eager.Tensor)): @@ -416,10 +440,9 @@ def sync_params_buffers(model, coalesced_vars = build_groups(model_vars, 128 * 1024 * 1024) for coalesced_var, _, _ in coalesced_vars: - paddle.distributed.broadcast(coalesced_var, - src=src_rank, - group=comm_group, - sync_op=True) + paddle.distributed.broadcast( + coalesced_var, src=src_rank, group=comm_group, sync_op=True + ) for coalesced_var, origin_vars, var_shapes in coalesced_vars: var_len = [np.prod(v_shape) for v_shape in var_shapes] @@ -427,10 +450,8 @@ def sync_params_buffers(model, type='split', inputs={'X': coalesced_var}, outputs={'Out': origin_vars}, - attrs={ - 'sections': var_len, - 'axis': 0 - }) + attrs={'sections': var_len, 'axis': 0}, + ) class DataParallel(layers.Layer): @@ -594,25 +615,30 @@ class DataParallel(layers.Layer): """ - def __init__(self, - layers, - strategy=None, - comm_buffer_size=25, - last_comm_buffer_size=1, - find_unused_parameters=False, - group=None): - super(DataParallel, - self).__init__(layers.full_name() + "_data_parallel") - - assert _non_static_mode(), \ - "It's not supported to construct DataParallel in static mode." + def __init__( + self, + layers, + strategy=None, + comm_buffer_size=25, + last_comm_buffer_size=1, + find_unused_parameters=False, + group=None, + ): + super(DataParallel, self).__init__( + layers.full_name() + "_data_parallel" + ) + + assert ( + _non_static_mode() + ), "It's not supported to construct DataParallel in static mode." self._layers = layers self.find_unused_parameters = find_unused_parameters self.grad_need_sync = True self.group = group - self.var_dtype = core.eager.Tensor if in_dygraph_mode( - ) else core.VarBase + self.var_dtype = ( + core.eager.Tensor if in_dygraph_mode() else core.VarBase + ) # NOTE(chenweihang): The ParallelStrategy here is not strictly a strategy. # It just stores some environment variables, which can be constructed by @@ -625,16 +651,21 @@ class DataParallel(layers.Layer): if self._strategy.nranks > 1: # check the environment - assert parallel_helper.__parallel_ctx__clz__ is not None, \ - "ParallelContext must be initialized before. You should use init_parallel_env() before" \ - "constructing the DataParallel." + assert parallel_helper.__parallel_ctx__clz__ is not None, ( + "ParallelContext must be initialized before. You should use init_parallel_env() before" + "constructing the DataParallel." + ) if in_dygraph_mode(): - self.group = paddle.distributed.collective._get_default_group( - ) if self.group is None else self.group + self.group = ( + paddle.distributed.collective._get_default_group() + if self.group is None + else self.group + ) - assert isinstance(self.group, paddle.distributed.collective.Group), \ - "ProcessGroup must be an instance of Group in DataParallel." + assert isinstance( + self.group, paddle.distributed.collective.Group + ), "ProcessGroup must be an instance of Group in DataParallel." # sync buffer and params # TODO(liuyuhui) Currently not support xpu. xpu is @@ -647,14 +678,17 @@ class DataParallel(layers.Layer): # the size of the group, Default: 1MB. The role of this small group is: # when the last group allreduce, the overlap cannot work. Making the # the last group small is useful to improve performance. - self.last_comm_buffer_size = int(last_comm_buffer_size * 1024 * - 1024) + self.last_comm_buffer_size = int( + last_comm_buffer_size * 1024 * 1024 + ) self.init_reducer() else: - warnings.warn("The program will return to single-card operation. " - "Please check 1, whether you use spawn or fleetrun " - "to start the program. 2, Whether it is a multi-card " - "program. 3, Is the current environment multi-card.") + warnings.warn( + "The program will return to single-card operation. " + "Please check 1, whether you use spawn or fleetrun " + "to start the program. 2, Whether it is a multi-card " + "program. 3, Is the current environment multi-card." + ) def init_reducer(self): layers_param = [] @@ -665,16 +699,19 @@ class DataParallel(layers.Layer): continue params_set.add(param) if not isinstance(param, self.var_dtype): - raise TypeError("The data type of '%s' must be '%s'" % - (param.name, self.var_dtype)) + raise TypeError( + "The data type of '%s' must be '%s'" + % (param.name, self.var_dtype) + ) if param.trainable: layers_param.append((sublayer, param)) trainable_parameters = [param for _, param in layers_param] - assert len(trainable_parameters) > 0, \ - "This model does not have any parameters to train, and " \ + assert len(trainable_parameters) > 0, ( + "This model does not have any parameters to train, and " "does not need to use DataParallel" + ) # NOTE(shenliang03): Here we can only use the attributes to judge whether # parameter is sparse(or SelectedRows). The reason is that the sparse message @@ -695,24 +732,34 @@ class DataParallel(layers.Layer): if in_dygraph_mode(): self.group_indices = core.eager_assign_group_by_size( - trainable_parameters, is_sparse_gradient, - [self.last_comm_buffer_size, self.comm_buffer_size]) + trainable_parameters, + is_sparse_gradient, + [self.last_comm_buffer_size, self.comm_buffer_size], + ) self._reducer = core.EagerReducer( - trainable_parameters, list(reversed(self.group_indices)), - is_sparse_gradient, self.group.process_group, + trainable_parameters, + list(reversed(self.group_indices)), + is_sparse_gradient, + self.group.process_group, [self.last_comm_buffer_size, self.comm_buffer_size], - self.find_unused_parameters) + self.find_unused_parameters, + ) elif _in_legacy_dygraph(): self.group_indices = core.assign_group_by_size( - trainable_parameters, is_sparse_gradient, - [self.last_comm_buffer_size, self.comm_buffer_size]) + trainable_parameters, + is_sparse_gradient, + [self.last_comm_buffer_size, self.comm_buffer_size], + ) self._reducer = core.Reducer( - trainable_parameters, list(reversed(self.group_indices)), - is_sparse_gradient, parallel_helper.__parallel_ctx__clz__, + trainable_parameters, + list(reversed(self.group_indices)), + is_sparse_gradient, + parallel_helper.__parallel_ctx__clz__, [self.last_comm_buffer_size, self.comm_buffer_size], - self.find_unused_parameters) + self.find_unused_parameters, + ) def _find_varbase(self, obj): var_type = core.eager.Tensor if in_dygraph_mode() else core.VarBase @@ -771,14 +818,19 @@ class DataParallel(layers.Layer): def forward(self, *inputs, **kwargs): outputs = self._layers(*inputs, **kwargs) - if self._strategy.nranks > 1 and framework._dygraph_tracer( - )._has_grad and self.grad_need_sync: - self._reducer.prepare_for_backward(list( - self._find_varbase(outputs))) + if ( + self._strategy.nranks > 1 + and framework._dygraph_tracer()._has_grad + and self.grad_need_sync + ): + self._reducer.prepare_for_backward( + list(self._find_varbase(outputs)) + ) return outputs - @deprecated(since="2.0.0", - reason="This method does not need to be called anymore.") + @deprecated( + since="2.0.0", reason="This method does not need to be called anymore." + ) def scale_loss(self, loss): """ Deprecated method, now ``scale_loss`` is an empty method, @@ -786,8 +838,9 @@ class DataParallel(layers.Layer): """ return loss - @deprecated(since="2.0.0", - reason="This method does not need to be called anymore.") + @deprecated( + since="2.0.0", reason="This method does not need to be called anymore." + ) def apply_collective_grads(self): """ Deprecated method, now ``apply_collective_grads`` is an empty method, @@ -795,10 +848,12 @@ class DataParallel(layers.Layer): """ return - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): ''' Get all parameters and persistable buffers of current layer and its sub-layers. And set them into a dict @@ -828,7 +883,8 @@ class DataParallel(layers.Layer): return self._layers.state_dict( destination=destination, include_sublayers=include_sublayers, - structured_name_prefix=structured_name_prefix) + structured_name_prefix=structured_name_prefix, + ) @framework.deprecate_stat_dict def set_state_dict(self, state_dict, use_structured_name=True): @@ -861,8 +917,9 @@ class DataParallel(layers.Layer): ''' - self._layers.set_state_dict(state_dict, - use_structured_name=use_structured_name) + self._layers.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) # [aliases] Compatible with old method names set_dict = set_state_dict diff --git a/python/paddle/fluid/dygraph/parallel_helper.py b/python/paddle/fluid/dygraph/parallel_helper.py index bc0bb460352..32d7974af31 100644 --- a/python/paddle/fluid/dygraph/parallel_helper.py +++ b/python/paddle/fluid/dygraph/parallel_helper.py @@ -20,8 +20,10 @@ __parallel_ctx__clz__ = None def _is_data_parallel_mode(): global __parallel_ctx__clz__ - return __parallel_ctx__clz__ is not None and int( - os.getenv("PADDLE_TRAINERS_NUM", "1")) > 1 + return ( + __parallel_ctx__clz__ is not None + and int(os.getenv("PADDLE_TRAINERS_NUM", "1")) > 1 + ) def _is_parallel_ctx_initialized(): @@ -31,15 +33,17 @@ def _is_parallel_ctx_initialized(): def _set_parallel_ctx(ccl_parallel_context): global __parallel_ctx__clz__ - assert __parallel_ctx__clz__ is None, \ - "ParallelContext can only be initialized once." + assert ( + __parallel_ctx__clz__ is None + ), "ParallelContext can only be initialized once." __parallel_ctx__clz__ = ccl_parallel_context def _init_parallel_ctx(): global __parallel_ctx__clz__ - assert __parallel_ctx__clz__ is not None, \ - "ParallelContext should be initialized." + assert ( + __parallel_ctx__clz__ is not None + ), "ParallelContext should be initialized." __parallel_ctx__clz__.init() @@ -47,7 +51,8 @@ def _broadcast_parameters(parameters): for param in parameters: # In model parallel, some parameters are split into multiple devices, # so we could not broadcast these parameters. - if param.is_distributed: continue + if param.is_distributed: + continue if isinstance(param, Parameter) and param.trainable: collective._broadcast(param, 0, sync_mode=True) diff --git a/python/paddle/fluid/dygraph/rnn.py b/python/paddle/fluid/dygraph/rnn.py index e7053366297..4ae9c5ba5fe 100644 --- a/python/paddle/fluid/dygraph/rnn.py +++ b/python/paddle/fluid/dygraph/rnn.py @@ -13,7 +13,16 @@ # limitations under the License. from . import Layer -from ..layers import sigmoid, tanh, concat, fill_constant, matmul, elementwise_add, elementwise_mul, split +from ..layers import ( + sigmoid, + tanh, + concat, + fill_constant, + matmul, + elementwise_add, + elementwise_mul, + split, +) import copy __all__ = ['LSTMCell', 'GRUCell'] @@ -111,16 +120,18 @@ class LSTMCell(Layer): """ - def __init__(self, - hidden_size, - input_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - use_cudnn_impl=True, - dtype='float64'): + def __init__( + self, + hidden_size, + input_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + use_cudnn_impl=True, + dtype='float64', + ): super(LSTMCell, self).__init__(dtype) self._hidden_size = hidden_size @@ -134,7 +145,10 @@ class LSTMCell(Layer): if self._use_cudnn_impl: - if self._param_attr is not None and self._param_attr.name is not None: + if ( + self._param_attr is not None + and self._param_attr.name is not None + ): weight_ih_param_attr = copy.deepcopy(self._param_attr) weight_hh_param_attr = copy.deepcopy(self._param_attr) weight_ih_param_attr.name += "_weight_ih" @@ -155,40 +169,50 @@ class LSTMCell(Layer): self._weight_ih = self.create_parameter( attr=weight_ih_param_attr, shape=[4 * self._hidden_size, self._input_size], - dtype=self._dtype) + dtype=self._dtype, + ) self._weight_hh = self.create_parameter( attr=weight_hh_param_attr, shape=[4 * self._hidden_size, self._hidden_size], - dtype=self._dtype) - - self._bias_ih = self.create_parameter(attr=bias_ih_param_attr, - shape=[4 * self._hidden_size], - dtype=self._dtype, - is_bias=True) - self._bias_hh = self.create_parameter(attr=bias_hh_param_attr, - shape=[4 * self._hidden_size], - dtype=self._dtype, - is_bias=True) + dtype=self._dtype, + ) + + self._bias_ih = self.create_parameter( + attr=bias_ih_param_attr, + shape=[4 * self._hidden_size], + dtype=self._dtype, + is_bias=True, + ) + self._bias_hh = self.create_parameter( + attr=bias_hh_param_attr, + shape=[4 * self._hidden_size], + dtype=self._dtype, + is_bias=True, + ) else: - self._forget_bias = fill_constant([1], - dtype=dtype, - value=forget_bias) + self._forget_bias = fill_constant( + [1], dtype=dtype, value=forget_bias + ) self._forget_bias.stop_gradient = False self._weight = self.create_parameter( attr=self._param_attr, shape=[ - self._input_size + self._hidden_size, 4 * self._hidden_size + self._input_size + self._hidden_size, + 4 * self._hidden_size, ], - dtype=dtype) + dtype=dtype, + ) - self._bias = self.create_parameter(attr=self._bias_attr, - shape=[4 * self._hidden_size], - dtype=dtype, - is_bias=True) + self._bias = self.create_parameter( + attr=self._bias_attr, + shape=[4 * self._hidden_size], + dtype=dtype, + is_bias=True, + ) def forward(self, input, pre_hidden, pre_cell): @@ -226,9 +250,12 @@ class LSTMCell(Layer): new_cell = elementwise_add( elementwise_mul( pre_cell, - self._gate_activation(elementwise_add(f, - self._forget_bias))), - elementwise_mul(sigmoid(i), tanh(j))) + self._gate_activation( + elementwise_add(f, self._forget_bias) + ), + ), + elementwise_mul(sigmoid(i), tanh(j)), + ) new_hidden = self._activation(new_cell) * self._gate_activation(o) return new_hidden, new_cell @@ -312,15 +339,17 @@ class GRUCell(Layer): """ - def __init__(self, - hidden_size, - input_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - use_cudnn_impl=True, - dtype='float64'): + def __init__( + self, + hidden_size, + input_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + use_cudnn_impl=True, + dtype='float64', + ): super(GRUCell, self).__init__() self._hidden_size = hidden_size @@ -334,7 +363,10 @@ class GRUCell(Layer): if self._use_cudnn_impl: - if self._param_attr is not None and self._param_attr.name is not None: + if ( + self._param_attr is not None + and self._param_attr.name is not None + ): weight_ih_param_attr = copy.deepcopy(self._param_attr) weight_hh_param_attr = copy.deepcopy(self._param_attr) weight_ih_param_attr.name += "_weight_ih" @@ -355,25 +387,34 @@ class GRUCell(Layer): self._weight_ih = self.create_parameter( attr=weight_ih_param_attr, shape=[3 * self._hidden_size, self._input_size], - dtype=self._dtype) + dtype=self._dtype, + ) self._weight_hh = self.create_parameter( attr=weight_hh_param_attr, shape=[3 * self._hidden_size, self._hidden_size], - dtype=self._dtype) - - self._bias_ih = self.create_parameter(attr=bias_ih_param_attr, - shape=[3 * self._hidden_size], - dtype=self._dtype, - is_bias=True) - self._bias_hh = self.create_parameter(attr=bias_hh_param_attr, - shape=[3 * self._hidden_size], - dtype=self._dtype, - is_bias=True) + dtype=self._dtype, + ) + + self._bias_ih = self.create_parameter( + attr=bias_ih_param_attr, + shape=[3 * self._hidden_size], + dtype=self._dtype, + is_bias=True, + ) + self._bias_hh = self.create_parameter( + attr=bias_hh_param_attr, + shape=[3 * self._hidden_size], + dtype=self._dtype, + is_bias=True, + ) else: - if self._param_attr is not None and self._param_attr.name is not None: + if ( + self._param_attr is not None + and self._param_attr.name is not None + ): gate_weight_param_attr = copy.deepcopy(self._param_attr) candidate_weight_param_attr = copy.deepcopy(self._param_attr) gate_weight_param_attr.name += "_gate_weight" @@ -394,25 +435,30 @@ class GRUCell(Layer): self._gate_weight = self.create_parameter( attr=gate_weight_param_attr, shape=[ - self._input_size + self._hidden_size, 2 * self._hidden_size + self._input_size + self._hidden_size, + 2 * self._hidden_size, ], - dtype=dtype) + dtype=dtype, + ) self._candidate_weight = self.create_parameter( attr=candidate_weight_param_attr, shape=[self._input_size + self._hidden_size, self._hidden_size], - dtype=dtype) + dtype=dtype, + ) self._gate_bias = self.create_parameter( attr=gate_bias_param_attr, shape=[2 * self._hidden_size], dtype=dtype, - is_bias=True) + is_bias=True, + ) self._candidate_bias = self.create_parameter( attr=candidate_bias_param_attr, shape=[self._hidden_size], dtype=dtype, - is_bias=True) + is_bias=True, + ) def forward(self, input, pre_hidden): @@ -450,8 +496,9 @@ class GRUCell(Layer): r_hidden = r * pre_hidden - candidate = matmul(concat([input, r_hidden], 1), - self._candidate_weight) + candidate = matmul( + concat([input, r_hidden], 1), self._candidate_weight + ) candidate = elementwise_add(candidate, self._candidate_bias) c = self._activation(candidate) diff --git a/python/paddle/fluid/dygraph/tracer.py b/python/paddle/fluid/dygraph/tracer.py index 1553efad41a..a274296cce3 100644 --- a/python/paddle/fluid/dygraph/tracer.py +++ b/python/paddle/fluid/dygraph/tracer.py @@ -24,7 +24,7 @@ name_mapping = { "src_index": "Src_index", "dst_index": "Dst_index", "out": "Out", - "dst_count": "Dst_count" + "dst_count": "Dst_count", }, "matmul_v2": { "final_op_name": "matmul", @@ -106,13 +106,15 @@ class Tracer(core.Tracer): self._train_mode = True - def eager_legacy_trace_op(self, - op_type, - inputs, - outputs, - attrs, - stop_gradient=False, - inplace_map=None): + def eager_legacy_trace_op( + self, + op_type, + inputs, + outputs, + attrs, + stop_gradient=False, + inplace_map=None, + ): function_ptr = _legacy_C_ops.__dict__[op_type] core_ops_args_info = _legacy_C_ops.get_core_ops_args_info() @@ -186,14 +188,17 @@ class Tracer(core.Tracer): if isinstance(returns[i], list): for j in range(len(returns[i])): outputs[retname][j].reconstruct_from_( - returns[i][j], False) + returns[i][j], False + ) else: if isinstance(outputs[retname], list): outputs[retname][0].reconstruct_from_( - returns[i], False) + returns[i], False + ) else: outputs[retname].reconstruct_from_( - returns[i], False) + returns[i], False + ) elif isinstance(returns, list): assert len(outputs.keys()) == 1 key = list(outputs.keys())[0] @@ -207,13 +212,15 @@ class Tracer(core.Tracer): else: outputs[key].reconstruct_from_(returns, False) - def eager_trace_op(self, - op_type, - inputs, - outputs, - attrs, - stop_gradient=False, - inplace_map=None): + def eager_trace_op( + self, + op_type, + inputs, + outputs, + attrs, + stop_gradient=False, + inplace_map=None, + ): assert op_type in name_mapping.keys() op_type = name_mapping[op_type]["final_op_name"] @@ -273,7 +280,8 @@ class Tracer(core.Tracer): if isinstance(returns[i], list): for j in range(len(returns[i])): outputs[retname][j].reconstruct_from_( - returns[i][j], False) + returns[i][j], False + ) else: outputs[retname][0].reconstruct_from_(returns[i], False) elif isinstance(returns, list): @@ -289,13 +297,15 @@ class Tracer(core.Tracer): else: outputs[key].reconstruct_from_(returns, False) - def trace_op(self, - type, - inputs, - outputs, - attrs, - stop_gradient=False, - inplace_map=None): + def trace_op( + self, + type, + inputs, + outputs, + attrs, + stop_gradient=False, + inplace_map=None, + ): if not framework._in_legacy_dygraph(): # inputs : {"sum": [tensor], ...} # outputs : {"sum": [tensor], ...} @@ -303,16 +313,23 @@ class Tracer(core.Tracer): type = name_mapping[type]["final_op_name"] assert type in _legacy_C_ops.__dict__ - self.eager_trace_op(type, inputs, outputs, attrs, stop_gradient, - inplace_map) + self.eager_trace_op( + type, inputs, outputs, attrs, stop_gradient, inplace_map + ) else: - self.eager_legacy_trace_op(type, inputs, outputs, attrs, - stop_gradient, inplace_map) + self.eager_legacy_trace_op( + type, inputs, outputs, attrs, stop_gradient, inplace_map + ) else: - self.trace(type, inputs, outputs, attrs, - framework._current_expected_place(), self._has_grad - and not stop_gradient, - inplace_map if inplace_map else {}) + self.trace( + type, + inputs, + outputs, + attrs, + framework._current_expected_place(), + self._has_grad and not stop_gradient, + inplace_map if inplace_map else {}, + ) def train_mode(self): self._train_mode = True diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index 618bcd540f0..7dbcbb6f98e 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -23,7 +23,15 @@ from .. import framework from ..framework import convert_np_dtype_to_dtype_, _in_legacy_dygraph from .. import core from .. import unique_name -from ..framework import Variable, Parameter, ParamBase, _getitem_impl_, _setitem_impl_, EagerParamBase, in_dygraph_mode +from ..framework import ( + Variable, + Parameter, + ParamBase, + _getitem_impl_, + _setitem_impl_, + EagerParamBase, + in_dygraph_mode, +) from .base import switch_to_static_graph from .math_op_patch import monkey_patch_math_varbase from .parallel import scale_loss @@ -43,8 +51,9 @@ class TensorHookRemoveHelper(object): """ def __init__(self, tensor, hook_id): - self._tensor = tensor if framework._in_eager_mode_ else weakref.ref( - tensor) + self._tensor = ( + tensor if framework._in_eager_mode_ else weakref.ref(tensor) + ) self._hook_id = hook_id def remove(self): @@ -62,7 +71,9 @@ class TensorHookRemoveHelper(object): else: warnings.warn( "The backward hook (ID: %d) of Tensor `%s` you want to remove does not exist or has been removed." - % (self._hook_id, tensor.name), RuntimeWarning) + % (self._hook_id, tensor.name), + RuntimeWarning, + ) return False @@ -70,7 +81,6 @@ _already_patch_repr = False def monkey_patch_varbase(): - @switch_to_static_graph def _to_static_var(self, to_parameter=False, **kwargs): """ @@ -111,8 +121,9 @@ def monkey_patch_varbase(): attr_names = [] for name in dir(self): if name not in attr_not_need_keys: - if not inspect.ismethod(getattr( - self, name)) and not name.startswith('_'): + if not inspect.ismethod( + getattr(self, name) + ) and not name.startswith('_'): attr_names.append(name) attr_kwargs = {name: getattr(self, name) for name in attr_names} @@ -169,38 +180,45 @@ def monkey_patch_varbase(): base_tensor = core.eager.Tensor else: base_tensor = core.VarBase - assert isinstance(value, (np.ndarray, base_tensor, dict, str)), \ - "Variable set_value function, arguments type only support Variable, numpy, VarBase, dict, string." + assert isinstance( + value, (np.ndarray, base_tensor, dict, str) + ), "Variable set_value function, arguments type only support Variable, numpy, VarBase, dict, string." if isinstance(value, (dict, str)): assert len(self) == len( value ), "Variable length not match, Variable [ {} ] need tensor with length {} but load set tensor with length {}".format( - self.name, len(self), len(value)) + self.name, len(self), len(value) + ) if isinstance(value, dict): self.value().set_vocab(value) else: self.value().set_string_list(value) else: - assert self.shape == list(value.shape), \ - "Variable Shape not match, Variable [ {} ] need tensor with shape {} but load set tensor with shape {}".format( - self.name, self.shape, value.shape) + assert self.shape == list( + value.shape + ), "Variable Shape not match, Variable [ {} ] need tensor with shape {} but load set tensor with shape {}".format( + self.name, self.shape, value.shape + ) if isinstance(value, base_tensor): dtype = value.dtype else: dtype = convert_np_dtype_to_dtype_(value.dtype) - assert self.dtype == dtype, \ - "Variable dtype not match, Variable [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( - self.name, self.dtype, dtype) + assert ( + self.dtype == dtype + ), "Variable dtype not match, Variable [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( + self.name, self.dtype, dtype + ) # NOTE(wuweilong): self could be VarBase or Tensor, the subsequent behavior are defined in different files # if self is VarBase, method value() return Variable that bindded in imperative.cc, get_tensor() bindded in pybind.cc # if self is Tensor, method value() return self that defined in this file, get_tensor() defined in eager_method.cc # this Interface behavior will be unifed in the future. - self.value().get_tensor().set(value, - framework._current_expected_place()) + self.value().get_tensor().set( + value, framework._current_expected_place() + ) @framework.dygraph_only def backward(self, grad_tensor=None, retain_graph=False): @@ -258,7 +276,8 @@ def monkey_patch_varbase(): if framework._non_static_mode(): if in_profiler_mode(): record_event = profiler.RecordEvent( - "Gradient Backward", profiler.TracerEventType.Backward) + "Gradient Backward", profiler.TracerEventType.Backward + ) record_event.begin() if grad_tensor is not None: if framework._in_eager_mode_: @@ -269,9 +288,11 @@ def monkey_patch_varbase(): assert isinstance( grad_tensor, paddle.Tensor ), "The type of grad_tensor must be paddle.Tensor" - assert grad_tensor.shape == self.shape, \ - "Tensor shape not match, Tensor of grad_tensor [ {} ] with shape {} mismatch Tensor [ {} ] with shape {}".format( - grad_tensor.name, grad_tensor.shape, self.name, self.shape) + assert ( + grad_tensor.shape == self.shape + ), "Tensor shape not match, Tensor of grad_tensor [ {} ] with shape {} mismatch Tensor [ {} ] with shape {}".format( + grad_tensor.name, grad_tensor.shape, self.name, self.shape + ) if framework._in_eager_mode_: if grad_tensor is None: @@ -281,36 +302,46 @@ def monkey_patch_varbase(): if _grad_scalar: # When using amp with Fleet DistributedStrategy, we do loss scaling implicitly. self = _grad_scalar.scale(self) - if paddle.is_compiled_with_xpu() or paddle.is_compiled_with_npu( - ) or paddle.is_compiled_with_mlu(): + if ( + paddle.is_compiled_with_xpu() + or paddle.is_compiled_with_npu() + or paddle.is_compiled_with_mlu() + ): # TODO(liuyuhui): Currently only for xpu. Will be removed in the future. scaled_loss = scale_loss(self) if framework._in_eager_mode_: - core.eager.run_backward([scaled_loss], grad_tensor, - retain_graph) + core.eager.run_backward( + [scaled_loss], grad_tensor, retain_graph + ) else: - core.dygraph_run_backward([scaled_loss], [grad_tensor], - retain_graph, - framework._dygraph_tracer()) + core.dygraph_run_backward( + [scaled_loss], + [grad_tensor], + retain_graph, + framework._dygraph_tracer(), + ) else: if framework._in_eager_mode_: core.eager.run_backward([self], grad_tensor, retain_graph) else: - core.dygraph_run_backward([self], [grad_tensor], - retain_graph, - framework._dygraph_tracer()) + core.dygraph_run_backward( + [self], + [grad_tensor], + retain_graph, + framework._dygraph_tracer(), + ) if in_profiler_mode(): record_event.end() else: raise ValueError( - "Variable.backward() is only available in DyGraph mode") + "Variable.backward() is only available in DyGraph mode" + ) @framework.dygraph_only @deprecated( since="2.1.0", level=1, - reason= - "Please use tensor.grad, which returns the tensor value of the gradient." + reason="Please use tensor.grad, which returns the tensor value of the gradient.", ) def gradient(self): """ @@ -347,9 +378,10 @@ def monkey_patch_varbase(): new_ivar = self._grad_ivar()._copy_to(core.CPUPlace(), True) if self._grad_ivar().type == core.VarDesc.VarType.SELECTED_ROWS: - return (np.array( - new_ivar.value().get_selected_rows().get_tensor()), - np.array(new_ivar.value().get_selected_rows().rows())) + return ( + np.array(new_ivar.value().get_selected_rows().get_tensor()), + np.array(new_ivar.value().get_selected_rows().rows()), + ) else: return np.array(new_ivar.value().get_tensor()) @@ -414,7 +446,8 @@ def monkey_patch_varbase(): """ if self.stop_gradient is True: raise RuntimeError( - "Cannot register hook on a tensor that stop gradient.") + "Cannot register hook on a tensor that stop gradient." + ) hook_id = self._register_grad_hook(hook) helper = TensorHookRemoveHelper(self, hook_id) @@ -430,21 +463,28 @@ def monkey_patch_varbase(): if isinstance(device, str): device = paddle.device._convert_to_place(device) elif isinstance( - device, - (core.CPUPlace, core.CUDAPlace, core.CUDAPinnedPlace, - core.XPUPlace, core.CustomPlace)): + device, + ( + core.CPUPlace, + core.CUDAPlace, + core.CUDAPinnedPlace, + core.XPUPlace, + core.CustomPlace, + ), + ): pass else: raise ValueError( "device value error, must be str, paddle.CPUPlace(), paddle.CUDAPlace(), paddle.CUDAPinnedPlace(), paddle.XPUPlace() or paddle.CustomPlace(), but the type of device is " - + type(device).__name__) + + type(device).__name__ + ) if blocking is None: blocking = True else: assert isinstance( - blocking, - bool), "blocking value error, must be the True, False or None" + blocking, bool + ), "blocking value error, must be the True, False or None" def transform(t, device, dtype, blocking): if device is None: @@ -461,7 +501,8 @@ def monkey_patch_varbase(): # waiting_alloc_memory will compute the memory space occupied by 't'. # Coefficient 1.2 is used to avoid OOM that may occur in this critical state when the memory is just enough. waiting_alloc_memory = ( - (t._numel() * size_dtype) / 256 + 1) * 256 * 1.2 + ((t._numel() * size_dtype) / 256 + 1) * 256 * 1.2 + ) gpu_memory_available = core.gpu_memory_available() if gpu_memory_available < waiting_alloc_memory: # Copy Tensor to cpu @@ -477,7 +518,8 @@ def monkey_patch_varbase(): # 2. cast Tensor to dtype if dtype is not None and dtype != t_used.dtype: with paddle.fluid.framework._dygraph_place_guard( - place=t_used.place): + place=t_used.place + ): t_casted = t_used.cast(dtype=dtype) else: t_casted = t_used @@ -523,10 +565,12 @@ def monkey_patch_varbase(): # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=False, [500.]) """ - msg = 'tensor.grad will return the tensor value of the gradient.' \ - ' This is an incompatible upgrade for tensor.grad API. ' \ - ' It\'s return type changes from numpy.ndarray in version 2.0 to paddle.Tensor in version 2.1.0. ' \ + msg = ( + 'tensor.grad will return the tensor value of the gradient.' + ' This is an incompatible upgrade for tensor.grad API. ' + ' It\'s return type changes from numpy.ndarray in version 2.0 to paddle.Tensor in version 2.1.0. ' ' If you want to get the numpy value of the gradient, you can use :code:`x.grad.numpy()`' + ) warning_msg = "\033[93m\nWarning:\n%s \033[0m" % (msg) # ensure ANSI escape sequences print correctly in cmd and powershell if sys.platform.lower() == 'win32': @@ -622,9 +666,11 @@ def monkey_patch_varbase(): """ if framework._in_eager_mode_: from paddle.tensor.to_string import tensor_to_string + return tensor_to_string(self) else: from paddle.tensor.to_string import to_string + return to_string(self) def __deepcopy__(self, memo): @@ -667,7 +713,9 @@ def monkey_patch_varbase(): def __nonzero__(self): numel = np.prod(self.shape) - assert numel == 1, "When Variable is used as the condition of if/while , Variable can only contain one element." + assert ( + numel == 1 + ), "When Variable is used as the condition of if/while , Variable can only contain one element." if framework._in_eager_mode_: assert self._is_initialized(), "tensor not initialized" return bool(np.all(self.numpy() > 0)) @@ -711,21 +759,22 @@ def monkey_patch_varbase(): for slice_item in item: if isinstance(slice_item, slice): - if isinstance(slice_item.start, Variable) \ - or isinstance(slice_item.stop, Variable) \ - or isinstance(slice_item.step, Variable): + if ( + isinstance(slice_item.start, Variable) + or isinstance(slice_item.stop, Variable) + or isinstance(slice_item.step, Variable) + ): return True else: - if isinstance( - slice_item, - (Variable, np.ndarray)) and Variable.dtype != paddle.bool: + if ( + isinstance(slice_item, (Variable, np.ndarray)) + and Variable.dtype != paddle.bool + ): return True return False def __getitem__(self, item): - def is_list_tuple(index, contain_type): - def _is_list_tuple(item): if isinstance(item, (tuple, list)): for s in item: @@ -753,7 +802,6 @@ def monkey_patch_varbase(): return self._getitem_index_not_tensor(item) def __setitem__(self, item, value): - def contain_tensor_or_list(item): if not isinstance(item, tuple): item = [item] @@ -813,7 +861,8 @@ def monkey_patch_varbase(): self._unset_fake_empty() else: raise TypeError( - "_set_grad_ivar is only supported for Parameter Tensor") + "_set_grad_ivar is only supported for Parameter Tensor" + ) @framework.dygraph_only def value(self): @@ -971,25 +1020,30 @@ def monkey_patch_varbase(): if framework._in_eager_mode_ and not hasattr(core, "eager"): return - for method_name, method in (("__bool__", __bool__), ("__nonzero__", - __nonzero__), - ("_to_static_var", - _to_static_var), ("set_value", set_value), - ("block", block), ("backward", backward), - ("clear_grad", clear_grad), ("inplace_version", - inplace_version), - ("gradient", gradient), ("register_hook", - register_hook), - ("__str__", __str__), ("__repr__", __str__), - ("__deepcopy__", __deepcopy__), ("__module__", - "paddle"), - ("__array__", - __array__), ("__getitem__", - __getitem__), ("item", item), - ("__setitem__", - __setitem__), ("_to", _to), ("values", values), - ("to_dense", to_dense), ("to_sparse_coo", - to_sparse_coo)): + for method_name, method in ( + ("__bool__", __bool__), + ("__nonzero__", __nonzero__), + ("_to_static_var", _to_static_var), + ("set_value", set_value), + ("block", block), + ("backward", backward), + ("clear_grad", clear_grad), + ("inplace_version", inplace_version), + ("gradient", gradient), + ("register_hook", register_hook), + ("__str__", __str__), + ("__repr__", __str__), + ("__deepcopy__", __deepcopy__), + ("__module__", "paddle"), + ("__array__", __array__), + ("__getitem__", __getitem__), + ("item", item), + ("__setitem__", __setitem__), + ("_to", _to), + ("values", values), + ("to_dense", to_dense), + ("to_sparse_coo", to_sparse_coo), + ): if framework._in_eager_mode_: setattr(core.eager.Tensor, method_name, method) else: diff --git a/python/paddle/fluid/dygraph_utils.py b/python/paddle/fluid/dygraph_utils.py index d93915e8bb5..e89f8f59108 100644 --- a/python/paddle/fluid/dygraph_utils.py +++ b/python/paddle/fluid/dygraph_utils.py @@ -18,10 +18,9 @@ from paddle import _C_ops, _legacy_C_ops @dygraph_only -def _append_activation_in_dygraph(input, - act=None, - use_cudnn=None, - use_mkldnn=None): +def _append_activation_in_dygraph( + input, act=None, use_cudnn=None, use_mkldnn=None +): """Append activation in dygraph mode. Args: @@ -60,5 +59,6 @@ def _append_bias_in_dygraph(input, bias=None, axis=1, use_mkldnn=False): if bias is None: return input - return _legacy_C_ops.elementwise_add(input, bias, 'axis', axis, - 'use_mkldnn', use_mkldnn) + return _legacy_C_ops.elementwise_add( + input, bias, 'axis', axis, 'use_mkldnn', use_mkldnn + ) diff --git a/python/paddle/fluid/entry_attr.py b/python/paddle/fluid/entry_attr.py index 4d481943229..51e9f5a6d51 100644 --- a/python/paddle/fluid/entry_attr.py +++ b/python/paddle/fluid/entry_attr.py @@ -37,7 +37,6 @@ class EntryAttr(object): class ProbabilityEntry(EntryAttr): - def __init__(self, probability): super(ProbabilityEntry, self).__init__() @@ -55,17 +54,18 @@ class ProbabilityEntry(EntryAttr): class CountFilterEntry(EntryAttr): - def __init__(self, count_filter): super(CountFilterEntry, self).__init__() if not isinstance(count_filter, int): raise ValueError( - "count_filter must be a valid integer greater than 0") + "count_filter must be a valid integer greater than 0" + ) if count_filter < 0: raise ValueError( - "count_filter must be a valid integer greater or equal than 0") + "count_filter must be a valid integer greater or equal than 0" + ) self._name = "count_filter_entry" self._count_filter = count_filter diff --git a/python/paddle/fluid/evaluator.py b/python/paddle/fluid/evaluator.py index 6c3ab7c497e..06f01ecdf5e 100644 --- a/python/paddle/fluid/evaluator.py +++ b/python/paddle/fluid/evaluator.py @@ -31,12 +31,14 @@ __all__ = [ def _clone_var_(block, var): assert isinstance(var, Variable) - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True, + ) class Evaluator(object): @@ -66,7 +68,9 @@ class Evaluator(object): def __init__(self, name, **kwargs): warnings.warn( "The %s is deprecated, because maintain a modified program inside evaluator cause bug easily, please use fluid.metrics.%s instead." - % (self.__class__.__name__, self.__class__.__name__), Warning) + % (self.__class__.__name__, self.__class__.__name__), + Warning, + ) self.states = [] self.metrics = [] self.helper = LayerHelper(name, **kwargs) @@ -86,10 +90,9 @@ class Evaluator(object): for var in self.states: assert isinstance(var, Variable) g_var = _clone_var_(reset_program.current_block(), var) - layers.fill_constant(shape=g_var.shape, - value=0.0, - dtype=g_var.dtype, - out=g_var) + layers.fill_constant( + shape=g_var.shape, value=0.0, dtype=g_var.dtype, out=g_var + ) executor.run(reset_program) @@ -114,11 +117,12 @@ class Evaluator(object): Returns: State variable """ - state = self.helper.create_variable(name="_".join( - [unique_name.generate(self.helper.name), suffix]), - persistable=True, - dtype=dtype, - shape=shape) + state = self.helper.create_variable( + name="_".join([unique_name.generate(self.helper.name), suffix]), + persistable=True, + dtype=dtype, + shape=shape, + ) self.states.append(state) return state @@ -169,27 +173,41 @@ class ChunkEvaluator(Evaluator): if main_program.current_block().idx != 0: raise ValueError("You can only invoke Evaluator in root block") - self.num_infer_chunks = self._create_state(dtype='int64', - shape=[1], - suffix='num_infer_chunks') - self.num_label_chunks = self._create_state(dtype='int64', - shape=[1], - suffix='num_label_chunks') + self.num_infer_chunks = self._create_state( + dtype='int64', shape=[1], suffix='num_infer_chunks' + ) + self.num_label_chunks = self._create_state( + dtype='int64', shape=[1], suffix='num_label_chunks' + ) self.num_correct_chunks = self._create_state( - dtype='int64', shape=[1], suffix='num_correct_chunks') - precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks = layers.chunk_eval( + dtype='int64', shape=[1], suffix='num_correct_chunks' + ) + ( + precision, + recall, + f1_score, + num_infer_chunks, + num_label_chunks, + num_correct_chunks, + ) = layers.chunk_eval( input=input, label=label, chunk_scheme=chunk_scheme, num_chunk_types=num_chunk_types, excluded_chunk_types=excluded_chunk_types, ) - layers.sums(input=[self.num_infer_chunks, num_infer_chunks], - out=self.num_infer_chunks) - layers.sums(input=[self.num_label_chunks, num_label_chunks], - out=self.num_label_chunks) - layers.sums(input=[self.num_correct_chunks, num_correct_chunks], - out=self.num_correct_chunks) + layers.sums( + input=[self.num_infer_chunks, num_infer_chunks], + out=self.num_infer_chunks, + ) + layers.sums( + input=[self.num_label_chunks, num_label_chunks], + out=self.num_label_chunks, + ) + layers.sums( + input=[self.num_correct_chunks, num_correct_chunks], + out=self.num_correct_chunks, + ) self.metrics.extend([precision, recall, f1_score]) @@ -199,18 +217,31 @@ class ChunkEvaluator(Evaluator): block = eval_program.current_block() num_infer_chunks, num_label_chunks, num_correct_chunks = executor.run( eval_program, - fetch_list=[_clone_var_(block, state) for state in self.states]) + fetch_list=[_clone_var_(block, state) for state in self.states], + ) num_infer_chunks = num_infer_chunks[0] num_label_chunks = num_label_chunks[0] num_correct_chunks = num_correct_chunks[0] - precision = float( - num_correct_chunks) / num_infer_chunks if num_infer_chunks else 0 - recall = float( - num_correct_chunks) / num_label_chunks if num_label_chunks else 0 - f1_score = float(2 * precision * recall) / ( - precision + recall) if num_correct_chunks else 0 - return np.array([precision], dtype='float32'), np.array( - [recall], dtype='float32'), np.array([f1_score], dtype='float32') + precision = ( + float(num_correct_chunks) / num_infer_chunks + if num_infer_chunks + else 0 + ) + recall = ( + float(num_correct_chunks) / num_label_chunks + if num_label_chunks + else 0 + ) + f1_score = ( + float(2 * precision * recall) / (precision + recall) + if num_correct_chunks + else 0 + ) + return ( + np.array([precision], dtype='float32'), + np.array([recall], dtype='float32'), + np.array([f1_score], dtype='float32'), + ) class EditDistance(Evaluator): @@ -250,31 +281,35 @@ class EditDistance(Evaluator): if main_program.current_block().idx != 0: raise ValueError("You can only invoke Evaluator in root block") - self.total_distance = self._create_state(dtype='float32', - shape=[1], - suffix='total_distance') - self.seq_num = self._create_state(dtype='int64', - shape=[1], - suffix='seq_num') - self.instance_error = self._create_state(dtype='int64', - shape=[1], - suffix='instance_error') - distances, seq_num = layers.edit_distance(input=input, - label=label, - ignored_tokens=ignored_tokens) + self.total_distance = self._create_state( + dtype='float32', shape=[1], suffix='total_distance' + ) + self.seq_num = self._create_state( + dtype='int64', shape=[1], suffix='seq_num' + ) + self.instance_error = self._create_state( + dtype='int64', shape=[1], suffix='instance_error' + ) + distances, seq_num = layers.edit_distance( + input=input, label=label, ignored_tokens=ignored_tokens + ) zero = layers.fill_constant(shape=[1], value=0.0, dtype='float32') compare_result = layers.equal(distances, zero) compare_result_int = layers.cast(x=compare_result, dtype='int64') seq_right_count = layers.reduce_sum(compare_result_int) - instance_error_count = layers.elementwise_sub(x=seq_num, - y=seq_right_count) + instance_error_count = layers.elementwise_sub( + x=seq_num, y=seq_right_count + ) total_distance = layers.reduce_sum(distances) - layers.sums(input=[self.total_distance, total_distance], - out=self.total_distance) + layers.sums( + input=[self.total_distance, total_distance], out=self.total_distance + ) layers.sums(input=[self.seq_num, seq_num], out=self.seq_num) - layers.sums(input=[self.instance_error, instance_error_count], - out=self.instance_error) + layers.sums( + input=[self.instance_error, instance_error_count], + out=self.instance_error, + ) self.metrics.append(total_distance) self.metrics.append(instance_error_count) @@ -289,10 +324,12 @@ class EditDistance(Evaluator): seq_num = layers.cast(x=seq_num, dtype='float32') instance_error = layers.cast(x=instance_error, dtype='float32') avg_distance = layers.elementwise_div(x=total_distance, y=seq_num) - avg_instance_error = layers.elementwise_div(x=instance_error, - y=seq_num) - result = executor.run(eval_program, - fetch_list=[avg_distance, avg_instance_error]) + avg_instance_error = layers.elementwise_div( + x=instance_error, y=seq_num + ) + result = executor.run( + eval_program, fetch_list=[avg_distance, avg_instance_error] + ) return np.array(result[0]), np.array(result[1]) @@ -355,16 +392,18 @@ class DetectionMAP(Evaluator): 'accum_map_v' is the accumulative mAP of one pass. """ - def __init__(self, - input, - gt_label, - gt_box, - gt_difficult=None, - class_num=None, - background_label=0, - overlap_threshold=0.5, - evaluate_difficult=True, - ap_version='integral'): + def __init__( + self, + input, + gt_label, + gt_box, + gt_difficult=None, + class_num=None, + background_label=0, + overlap_threshold=0.5, + evaluate_difficult=True, + ap_version='integral', + ): super(DetectionMAP, self).__init__("map_eval") gt_label = layers.cast(x=gt_label, dtype=gt_box.dtype) @@ -375,26 +414,29 @@ class DetectionMAP(Evaluator): label = layers.concat([gt_label, gt_box], axis=1) # calculate mean average precision (mAP) of current mini-batch - map = detection.detection_map(input, - label, - class_num, - background_label, - overlap_threshold=overlap_threshold, - evaluate_difficult=evaluate_difficult, - ap_version=ap_version) + map = detection.detection_map( + input, + label, + class_num, + background_label, + overlap_threshold=overlap_threshold, + evaluate_difficult=evaluate_difficult, + ap_version=ap_version, + ) self._create_state(dtype='int32', shape=None, suffix='accum_pos_count') self._create_state(dtype='float32', shape=None, suffix='accum_true_pos') - self._create_state(dtype='float32', - shape=None, - suffix='accum_false_pos') + self._create_state( + dtype='float32', shape=None, suffix='accum_false_pos' + ) self.has_state = None - var = self.helper.create_variable(persistable=True, - dtype='int32', - shape=[1]) - self.helper.set_variable_initializer(var, - initializer=Constant(value=int(0))) + var = self.helper.create_variable( + persistable=True, dtype='int32', shape=[1] + ) + self.helper.set_variable_initializer( + var, initializer=Constant(value=int(0)) + ) self.has_state = var # calculate accumulative mAP @@ -408,12 +450,15 @@ class DetectionMAP(Evaluator): has_state=self.has_state, input_states=self.states, out_states=self.states, - ap_version=ap_version) + ap_version=ap_version, + ) - layers.fill_constant(shape=self.has_state.shape, - value=1, - dtype=self.has_state.dtype, - out=self.has_state) + layers.fill_constant( + shape=self.has_state.shape, + value=1, + dtype=self.has_state.dtype, + out=self.has_state, + ) self.cur_map = map self.accum_map = accum_map @@ -426,8 +471,7 @@ class DetectionMAP(Evaluator): reset_program = Program() with program_guard(main_program=reset_program): var = _clone_var_(reset_program.current_block(), self.has_state) - layers.fill_constant(shape=var.shape, - value=0, - dtype=var.dtype, - out=var) + layers.fill_constant( + shape=var.shape, value=0, dtype=var.dtype, out=var + ) executor.run(reset_program) diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index 489ed0b476a..ba8221bb2f0 100755 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -144,10 +144,12 @@ def as_numpy(tensor, copy=False): assert isinstance(tensor, core.LoDTensor) lod = tensor.lod() if len(lod) > 0: - raise RuntimeError("Some of your fetched tensors hold LoD information. \ + raise RuntimeError( + "Some of your fetched tensors hold LoD information. \ They can not be completely cast to Python ndarray. \ Please set the parameter 'return_numpy' as 'False' to \ - return LoDTensor itself directly.") + return LoDTensor itself directly." + ) if tensor._is_initialized(): if copy: return np.array(tensor) @@ -237,21 +239,29 @@ def check_feed_shape_type(var, feed, num_places=1): if diff_shape is not None: raise ValueError( 'The fed Variable %r should have dimensions = %d, shape = ' - '%r, but received fed shape %r on each device' % - (var.name, len(var.shape), var.shape, diff_shape)) + '%r, but received fed shape %r on each device' + % (var.name, len(var.shape), var.shape, diff_shape) + ) if not dtype_is_compatible_with(feed._dtype(), var.dtype): - var_dtype_format = convert_dtype(var.dtype) if isinstance( - var.dtype, core.VarDesc.VarType) else var.dtype - feed_dtype_format = convert_dtype(feed._dtype()) if isinstance( - feed._dtype(), core.VarDesc.VarType) else feed._dtype() + var_dtype_format = ( + convert_dtype(var.dtype) + if isinstance(var.dtype, core.VarDesc.VarType) + else var.dtype + ) + feed_dtype_format = ( + convert_dtype(feed._dtype()) + if isinstance(feed._dtype(), core.VarDesc.VarType) + else feed._dtype() + ) raise ValueError( - 'The data type of fed Variable %r must be %r, but received %r' % - (var.name, var_dtype_format, feed_dtype_format)) + 'The data type of fed Variable %r must be %r, but received %r' + % (var.name, var_dtype_format, feed_dtype_format) + ) return True def has_feed_operators(block, feed_targets, feed_holder_name): - """ Check whether the block already has feed operators. + """Check whether the block already has feed operators. Return false if the block does not have any feed operators. If some feed operators have been prepended to the block, check that @@ -280,20 +290,22 @@ def has_feed_operators(block, feed_targets, feed_holder_name): if feed_target_name not in feed_targets: raise Exception( "'feed_targets' does not have {} variable".format( - feed_target_name)) + feed_target_name + ) + ) else: break if feed_count > 0 and feed_count != len(feed_targets): raise Exception( - "Feed operators in program desc do not match 'feed_targets'") + "Feed operators in program desc do not match 'feed_targets'" + ) return feed_count > 0 -def has_fetch_operators(block, - fetch_targets, - fetch_holder_name, - fetch_op='fetch'): - """ Check whether the block already has fetch operators. +def has_fetch_operators( + block, fetch_targets, fetch_holder_name, fetch_op='fetch' +): + """Check whether the block already has fetch operators. Return false if the block does not have any fetch operators. If some fetch operators have been appended to the block, check that @@ -321,25 +333,25 @@ def has_fetch_operators(block, assert op.desc.output('Out')[0] == fetch_holder_name fetch_target_name = op.desc.input('X')[0] if fetch_target_name not in [ - var.desc.name() for var in fetch_targets + var.desc.name() for var in fetch_targets ]: raise Exception( "'fetch_targets' does not have {} variable".format( - fetch_target_name)) + fetch_target_name + ) + ) idx = op.desc.attr('col') assert fetch_target_name == fetch_targets[idx].desc.name() if fetch_count > 0 and fetch_count != len(fetch_targets): raise Exception( - "Fetch operators in program desc do not match 'fetch_targets'") + "Fetch operators in program desc do not match 'fetch_targets'" + ) return fetch_count > 0 -def _add_feed_fetch_ops(program, - feed, - fetch_list, - feed_var_name, - fetch_var_name, - use_fetch_v2=False): +def _add_feed_fetch_ops( + program, feed, fetch_list, feed_var_name, fetch_var_name, use_fetch_v2=False +): tmp_program = program.clone() global_block = tmp_program.global_block() @@ -350,7 +362,8 @@ def _add_feed_fetch_ops(program, feed_var = global_block.create_var( name=feed_var_name, type=core.VarDesc.VarType.FEED_MINIBATCH, - persistable=True) + persistable=True, + ) if fetch_var_name in global_block.vars: fetch_var = global_block.var(fetch_var_name) @@ -358,21 +371,25 @@ def _add_feed_fetch_ops(program, fetch_var = global_block.create_var( name=fetch_var_name, type=core.VarDesc.VarType.FETCH_LIST, - persistable=True) + persistable=True, + ) # prepend feed operators if not has_feed_operators(global_block, feed, feed_var_name): for i, name in enumerate(feed): if global_block.has_var(name): out = global_block.var(name) - global_block._prepend_op(type='feed', - inputs={'X': [feed_var]}, - outputs={'Out': [out]}, - attrs={'col': i}) + global_block._prepend_op( + type='feed', + inputs={'X': [feed_var]}, + outputs={'Out': [out]}, + attrs={'col': i}, + ) else: warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % name) + % name + ) if use_fetch_v2: fetch_op = 'fetch_v2' @@ -380,22 +397,26 @@ def _add_feed_fetch_ops(program, fetch_op = 'fetch' # append fetch_operators - if not has_fetch_operators(global_block, fetch_list, fetch_var_name, - fetch_op): + if not has_fetch_operators( + global_block, fetch_list, fetch_var_name, fetch_op + ): for i, var in enumerate(fetch_list): assert isinstance(var, Variable) or isinstance( - var, str), ("Wrong type for fetch_list[%s]: %s" % - (i, type(var))) - global_block.append_op(type=fetch_op, - inputs={'X': [var]}, - outputs={'Out': [fetch_var]}, - attrs={'col': i}) + var, str + ), "Wrong type for fetch_list[%s]: %s" % (i, type(var)) + global_block.append_op( + type=fetch_op, + inputs={'X': [var]}, + outputs={'Out': [fetch_var]}, + attrs={'col': i}, + ) return tmp_program -def _apply_inplace_addto_pass(program, enable_inplace, enable_addto, - skip_var_names): +def _apply_inplace_addto_pass( + program, enable_inplace, enable_addto, skip_var_names +): use_cuda = True if core.is_compiled_with_cuda() else False attrs = {"use_cuda": use_cuda, "mem_opt_skip_vars": skip_var_names} @@ -404,12 +425,14 @@ def _apply_inplace_addto_pass(program, enable_inplace, enable_addto, empty_startup_program = Program() if enable_inplace: pass_name = "buffer_shared_inplace_pass" - _apply_pass(program, empty_startup_program, pass_name, attrs, - attr_types) + _apply_pass( + program, empty_startup_program, pass_name, attrs, attr_types + ) if enable_addto and use_cuda: pass_name = "inplace_addto_op_pass" - _apply_pass(program, empty_startup_program, pass_name, attrs, - attr_types) + _apply_pass( + program, empty_startup_program, pass_name, attrs, attr_types + ) def _fetch_var(name, scope=None, return_numpy=True): @@ -438,7 +461,8 @@ def _fetch_var(name, scope=None, return_numpy=True): assert var is not None, ( "Cannot find " + name + " in scope. Perhaps you need to make the" " variable persistable by using var.persistable = True in your" - " program.") + " program." + ) tensor = var.get_tensor() if return_numpy: tensor = as_numpy(tensor, copy=True) @@ -446,7 +470,6 @@ def _fetch_var(name, scope=None, return_numpy=True): def _to_name_str(var): - def _to_str(var): if isinstance(var, Variable): return var.desc.name() @@ -471,19 +494,26 @@ def _to_name_str(var): def _is_enable_standalone_executor(): - return framework._enable_standalone_executor_ is None or framework._enable_standalone_executor_ in [ - 1, '1', True, 'True', 'true' - ] + return ( + framework._enable_standalone_executor_ is None + or framework._enable_standalone_executor_ + in [1, '1', True, 'True', 'true'] + ) def _is_dy2st_enable_standalone_executor(): return framework._dy2st_enable_standalone_executor_ in [ - 1, '1', True, 'True', 'true' + 1, + '1', + True, + 'True', + 'true', ] def _prepare_fleet_executor(): from ..distributed.fleet.proto import fleet_executor_desc_pb2 + trainer_endpoints_str = os.getenv("PADDLE_TRAINER_ENDPOINTS", "") trainer_endpoints = trainer_endpoints_str.split(',') fleet_exe_desc = fleet_executor_desc_pb2.FleetExecutorDesc() @@ -501,7 +531,8 @@ def _prepare_fleet_executor(): def _get_strong_program_cache_key_for_new_exe(program, feed, fetch_list): return program.desc.cached_hash_str() + _get_program_cache_key( - feed, fetch_list) + feed, fetch_list + ) def _get_strong_program_cache_key(program, feed, fetch_list): @@ -512,10 +543,16 @@ def _get_strong_program_cache_key(program, feed, fetch_list): block_str.append(var_name) return "\n".join(block_str) - inner_program = program._program if isinstance( - program, compiler.CompiledProgram) else program - return _get_varname_from_block(inner_program.blocks[0]) + str( - id(program)) + _get_program_cache_key(feed, fetch_list) + inner_program = ( + program._program + if isinstance(program, compiler.CompiledProgram) + else program + ) + return ( + _get_varname_from_block(inner_program.blocks[0]) + + str(id(program)) + + _get_program_cache_key(feed, fetch_list) + ) def _get_program_cache_key(feed, fetch_list): @@ -531,30 +568,35 @@ def _get_program_cache_key(feed, fetch_list): def _as_lodtensor(data, place, dtype=None): """ - Convert numpy.ndarray to Tensor, its only support Tensor without LoD information. - For higher dimensional sequence data, please use LoDTensor directly. + Convert numpy.ndarray to Tensor, its only support Tensor without LoD information. + For higher dimensional sequence data, please use LoDTensor directly. - Examples: - >>> import paddle.fluid as fluid - >>> place = fluid.CPUPlace() - >>> exe = fluid.executor(place) - >>> data = np.array(size=(100, 200, 300)) - >>> np_outs = map(lambda x: fluid.executor._as_lodtensor(x, place), data) - >>> ... + Examples: + >>> import paddle.fluid as fluid + >>> place = fluid.CPUPlace() + >>> exe = fluid.executor(place) + >>> data = np.array(size=(100, 200, 300)) + >>> np_outs = map(lambda x: fluid.executor._as_lodtensor(x, place), data) + >>> ... - Args: - data(numpy.ndarray|list|tuple|scalar): a instance of array, scalar, list or tuple - data(core.Place): the place of created tensor - dtype(core.VarDesc.VarType|str): the expected data type of created tensor + Args: + data(numpy.ndarray|list|tuple|scalar): a instance of array, scalar, list or tuple + data(core.Place): the place of created tensor + dtype(core.VarDesc.VarType|str): the expected data type of created tensor - Returns: - LoDTensor - """ - #NOTE(zhiqiu): convert python builtin, like float, int, and list, to numpy ndarray + Returns: + LoDTensor + """ + # NOTE(zhiqiu): convert python builtin, like float, int, and list, to numpy ndarray if not isinstance(data, np.ndarray): - assert dtype is not None, 'The dtype should be given when feed data is not np.ndarray' - dtype = convert_dtype(dtype) if isinstance( - dtype, core.VarDesc.VarType) else dtype + assert ( + dtype is not None + ), 'The dtype should be given when feed data is not np.ndarray' + dtype = ( + convert_dtype(dtype) + if isinstance(dtype, core.VarDesc.VarType) + else dtype + ) if np.isscalar(data): data = np.array([data]).astype(dtype) elif isinstance(data, (list, tuple)): @@ -569,7 +611,9 @@ def _as_lodtensor(data, place, dtype=None): else: raise TypeError( "Convert data of type {} to Tensor is not supported".format( - type(data))) + type(data) + ) + ) # convert numpy.ndarray to tensor tensor = core.LoDTensor() @@ -578,7 +622,6 @@ def _as_lodtensor(data, place, dtype=None): class FetchHandler(object): - def __init__(self, var_dict=None, period_secs=60): assert var_dict != None self.var_dict = var_dict @@ -592,7 +635,8 @@ class FetchHandler(object): @staticmethod def help(): - print(""" + print( + """ class FetchHandlerExample(FetchHandler): def handler(self, res_dict): print(res_dict["auc"]) @@ -601,11 +645,11 @@ class FetchHandlerExample(FetchHandler): auc = Variable() var_dict = {"auc": auc} handler = FetchHandlerExample(var_dict=var_dict) -""") +""" + ) class _StandaloneExecutor(object): - def __init__(self, place, main_program, scope): self._place = core.Place() self._place.set_place(place) @@ -625,8 +669,9 @@ class _StandaloneExecutor(object): """ fetch_list = self._check_fetch(fetch_list) - tensors = self._new_exe.run(scope, feed_names, - fetch_list)._move_to_list() + tensors = self._new_exe.run( + scope, feed_names, fetch_list + )._move_to_list() if return_numpy: return as_numpy(tensors, copy=True) else: @@ -658,8 +703,9 @@ class _StandaloneExecutor(object): if not isinstance(feed, dict): raise TypeError( - "feed requires dict as its Parameter. But you passed in %s" % - (type(feed))) + "feed requires dict as its Parameter. But you passed in %s" + % (type(feed)) + ) global_block = self._main_program.global_block() for feed_name in list(feed.keys()): @@ -667,7 +713,8 @@ class _StandaloneExecutor(object): feed.pop(feed_name) warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % feed_name) + % feed_name + ) return feed @@ -681,19 +728,27 @@ class _StandaloneExecutor(object): fetch_var = fetch_var.name elif not isinstance(fetch_var, str): raise TypeError( - "Required fetch_var shall be str|Variable, but received {}". - format(type(fetch_var).__name__)) + "Required fetch_var shall be str|Variable, but received {}".format( + type(fetch_var).__name__ + ) + ) res.append(fetch_var) return res class _ExecutorCache(object): - class _CachedData(object): - - def __init__(self, program, feed, fetch_list, feed_var_name, - fetch_var_name, place, scope): + def __init__( + self, + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + place, + scope, + ): self.program = program self.feed = feed self.fetch_list = fetch_list @@ -709,18 +764,25 @@ class _ExecutorCache(object): # The program holds no _program, maybe it is constructed by graph. # Convert graph to program in order to generate key. self.program._program = framework.IrGraph( - self.program._graph).to_program() + self.program._graph + ).to_program() self.key = hash( _get_strong_program_cache_key_for_new_exe( - self.program._program, feed, fetch_list)) + self.program._program, feed, fetch_list + ) + ) else: self.key = hash( _get_strong_program_cache_key_for_new_exe( - self.program, feed, fetch_list)) + self.program, feed, fetch_list + ) + ) def __eq__(self, other): - return isinstance( - other, _ExecutorCache._CachedData) and self.key == other.key + return ( + isinstance(other, _ExecutorCache._CachedData) + and self.key == other.key + ) def __hash__(self): return self.key @@ -730,21 +792,41 @@ class _ExecutorCache(object): # the _ExecutorCache instance, otherwise a global cache may not be released after # the Executor instance deleted self._get_cached_program_and_executor = lru_cache(maxsize=8)( - self._get_program_and_executor) + self._get_program_and_executor + ) def clear(self): self._get_cached_program_and_executor.cache_clear() - def get_program_and_executor(self, program, feed, fetch_list, feed_var_name, - fetch_var_name, place, scope): + def get_program_and_executor( + self, + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + place, + scope, + ): return self._get_cached_program_and_executor( - self._CachedData(program, feed, fetch_list, feed_var_name, - fetch_var_name, place, scope)) + self._CachedData( + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + place, + scope, + ) + ) def _get_program_and_executor(self, cached_data): program = cached_data.program - inner_program = program._program if isinstance( - program, compiler.CompiledProgram) else program + inner_program = ( + program._program + if isinstance(program, compiler.CompiledProgram) + else program + ) feed = cached_data.feed fetch_list = cached_data.fetch_list feed_var_name = cached_data.feed_var_name @@ -754,9 +836,13 @@ class _ExecutorCache(object): # To apply IR pass, compile the Program to IrGraph and convert it back to Program if isinstance(program, compiler.CompiledProgram) or isinstance( - program._graph, compiler.CompiledProgram): - compiled_program = program if isinstance( - program, compiler.CompiledProgram) else program._graph + program._graph, compiler.CompiledProgram + ): + compiled_program = ( + program + if isinstance(program, compiler.CompiledProgram) + else program._graph + ) build_strategy = compiled_program._build_strategy # print(f"Program before convert:\n {inner_program}", flush=True) compiled_program._compile(scope, place) @@ -771,21 +857,26 @@ class _ExecutorCache(object): else: build_strategy = None from paddle.incubate.autograd import prim_enabled, prim2orig + if prim_enabled() and program == default_main_program(): prim2orig() inner_program = program - program = _add_feed_fetch_ops(program=inner_program, - feed=feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name, - use_fetch_v2=True) + program = _add_feed_fetch_ops( + program=inner_program, + feed=feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + use_fetch_v2=True, + ) - if os.environ.get('FLAGS_CONVERT_GRAPH_TO_PROGRAM', None) in [ - 1, '1', True, 'True', 'true' - ] and not program._is_start_up_program_: + if ( + os.environ.get('FLAGS_CONVERT_GRAPH_TO_PROGRAM', None) + in [1, '1', True, 'True', 'true'] + and not program._is_start_up_program_ + ): if program.num_blocks > 1: # If there are multiple blocks in the program, subblock will not be executed with the new executor in temporary logging.warning("There are more than 1 block in program.") @@ -796,13 +887,22 @@ class _ExecutorCache(object): # standalone executor will apply buffer_shared_inplace_pass and # inplace_addto_op_pass to program according to build_strategy - enable_inplace = True if build_strategy is None or build_strategy.enable_inplace else False - enable_addto = True if build_strategy is not None and build_strategy.enable_addto else False + enable_inplace = ( + True + if build_strategy is None or build_strategy.enable_inplace + else False + ) + enable_addto = ( + True + if build_strategy is not None and build_strategy.enable_addto + else False + ) if enable_inplace or enable_addto: # inplace should skip feed and fetch var skip_var_names = eval(_get_program_cache_key(feed, fetch_list)) - _apply_inplace_addto_pass(program, enable_inplace, enable_addto, - skip_var_names) + _apply_inplace_addto_pass( + program, enable_inplace, enable_addto, skip_var_names + ) new_program = program.clone() new_exe = _StandaloneExecutor(place, new_program, scope) @@ -906,7 +1006,8 @@ class Executor(object): self._prepare_to_run_called = False self._auto_checkpoint_name = unique_name.generate( - "__auto_checkpoint_executor__") + "__auto_checkpoint_executor__" + ) # NOTE: Whether to use experimental executor `StandaloneExecutor`. self._enable_interpreter_core = _is_enable_standalone_executor() @@ -976,8 +1077,9 @@ class Executor(object): var = global_block.var(feed_target_name) if var.dtype != core.VarDesc.VarType.STRINGS: if not isinstance(cur_feed, core.LoDTensor): - cur_feed = _as_lodtensor(cur_feed, self.place, - var.dtype) + cur_feed = _as_lodtensor( + cur_feed, self.place, var.dtype + ) check_feed_shape_type(var, cur_feed) idx = op.desc.attr('col') core.set_feed_variable(scope, cur_feed, feed_var_name, idx) @@ -1015,14 +1117,19 @@ class Executor(object): _optimize_ops.append(item) else: raise TypeError( - "The operator in fetch_list is not an optimize_op") - elif isinstance(item, Variable) or isinstance( - item, str) or isinstance(item, str): + "The operator in fetch_list is not an optimize_op" + ) + elif ( + isinstance(item, Variable) + or isinstance(item, str) + or isinstance(item, str) + ): _fetch_list.append(item) else: raise TypeError( "The item in fetch_list should be str, variable or optimize_op, but received %s.", - type(item)) + type(item), + ) for index, item in enumerate(fetch_list): # NOTE(zhiqiu): to support (optimizer_ops, param_and_grads) and optimizer_ops in fetch_list @@ -1034,9 +1141,10 @@ class Executor(object): elif isinstance(item, tuple): if not isinstance(item[0], (list, tuple)): raise TypeError( - "Requires fetch_list[{}][0] shall be one of (list, tuple) when type(fetch_list[{}]) is `tuple`, but received fetch_list[{}][0]'s type is `{}`." - .format(index, index, index, - type(item[0]).__name__)) + "Requires fetch_list[{}][0] shall be one of (list, tuple) when type(fetch_list[{}]) is `tuple`, but received fetch_list[{}][0]'s type is `{}`.".format( + index, index, index, type(item[0]).__name__ + ) + ) for i in item[0]: _get_targets(_optimize_ops, _fetch_list, i) else: @@ -1045,11 +1153,9 @@ class Executor(object): return _fetch_list, _optimize_ops @classmethod - def _prune_program(cls, - program, - feed=None, - fetch_list=None, - optimize_ops=None): + def _prune_program( + cls, program, feed=None, fetch_list=None, optimize_ops=None + ): """ Prune operators and variables which are not needed to generate :code:`fetch_list` and optimize operators. @@ -1141,7 +1247,8 @@ class Executor(object): feed.pop(feed_name) warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % feed_name) + % feed_name + ) elif isinstance(feed, list) or isinstance(feed, tuple): for i, each in enumerate(feed): @@ -1150,7 +1257,8 @@ class Executor(object): each.pop(feed_name) warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % feed_name) + % feed_name + ) return feed ''' @@ -1185,9 +1293,18 @@ class Executor(object): del trainer_instance self._default_executor.close() - def _run_parallel(self, program, scope, feed, fetch_list, fetch_var_name, - return_numpy, return_merged): + def _run_parallel( + self, + program, + scope, + feed, + fetch_list, + fetch_var_name, + return_numpy, + return_merged, + ): from paddle.optimizer.lr import LRScheduler + exe = program._executor # TODO(zhenghuihuang): quantization uses Graph in CompiledProgram # instead of program. We will add support for checking Vars in Graph @@ -1202,9 +1319,11 @@ class Executor(object): if not isinstance(feed_tensor, core.LoDTensor): # always set to CPU place, since the tensor need to be split # it is fast in CPU - feed_tensor = _as_lodtensor(feed[feed_name], - core.CPUPlace(), - var.dtype if var else None) + feed_tensor = _as_lodtensor( + feed[feed_name], + core.CPUPlace(), + var.dtype if var else None, + ) if need_check_feed: check_feed_shape_type(var, feed_tensor, exe.device_count()) feed_tensor_dict[feed_name] = feed_tensor @@ -1215,16 +1334,20 @@ class Executor(object): for i, each in enumerate(feed): if not isinstance(each, dict): raise TypeError( - "Each element of feed list should be a dict") + "Each element of feed list should be a dict" + ) res_dict = dict() for feed_name in each: tensor = each[feed_name] - var = global_block.var( - feed_name) if need_check_feed else None + var = ( + global_block.var(feed_name) if need_check_feed else None + ) if not isinstance(tensor, core.LoDTensor): - tensor = _as_lodtensor(each[feed_name], - program._places[i], - var.dtype if var else None) + tensor = _as_lodtensor( + each[feed_name], + program._places[i], + var.dtype if var else None, + ) if need_check_feed: check_feed_shape_type(var, tensor) res_dict[feed_name] = tensor @@ -1245,23 +1368,26 @@ class Executor(object): ) else: exe.feed_and_split_tensor_into_local_scopes( - {lr_sheduler._var_name: lr_tensor}) + {lr_sheduler._var_name: lr_tensor} + ) fetch_var_names = list(map(_to_name_str, fetch_list)) tensors = exe.run(fetch_var_names, return_merged)._move_to_list() return as_numpy(tensors) if return_numpy else tensors - def run(self, - program=None, - feed=None, - fetch_list=None, - feed_var_name='feed', - fetch_var_name='fetch', - scope=None, - return_numpy=True, - use_program_cache=False, - return_merged=True, - use_prune=False): + def run( + self, + program=None, + feed=None, + fetch_list=None, + feed_var_name='feed', + fetch_var_name='fetch', + scope=None, + return_numpy=True, + use_program_cache=False, + return_merged=True, + use_prune=False, + ): """ Run the specified :code:`Program` or :code:`CompiledProgram`. It should be noted that the executor will execute all the operators in :code:`Program` or :code:`CompiledProgram` without pruning some @@ -1436,32 +1562,49 @@ class Executor(object): """ # Temporary FLAGS, just for testing the performance of program cache force_use_program_cache = os.environ.get( - 'FLAGS_FORCE_USE_PROGRAM_CACHE', None) + 'FLAGS_FORCE_USE_PROGRAM_CACHE', None + ) if force_use_program_cache is not None: use_program_cache = force_use_program_cache in [ - 1, '1', True, 'True', 'true' + 1, + '1', + True, + 'True', + 'true', ] self._log_force_set_program_cache(use_program_cache) try: - res = self._run_impl(program=program, - feed=feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name, - scope=scope, - return_numpy=return_numpy, - use_program_cache=use_program_cache, - use_prune=use_prune, - return_merged=return_merged) + res = self._run_impl( + program=program, + feed=feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + scope=scope, + return_numpy=return_numpy, + use_program_cache=use_program_cache, + use_prune=use_prune, + return_merged=return_merged, + ) core.update_autotune_status() return res except Exception as e: six.reraise(*sys.exc_info()) - def _run_impl(self, program, feed, fetch_list, feed_var_name, - fetch_var_name, scope, return_numpy, use_program_cache, - return_merged, use_prune): + def _run_impl( + self, + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + scope, + return_numpy, + use_program_cache, + return_merged, + use_prune, + ): if self._closed: raise RuntimeError("Attempted to use a closed Executor") @@ -1480,17 +1623,19 @@ class Executor(object): program=program, feed=feed, fetch_list=fetch_list, - with_standalone_executor=self. - _fleet_executor_with_standalone) + with_standalone_executor=self._fleet_executor_with_standalone, + ) if "startup_program" in program._pipeline_opt: program = program._pipeline_opt["startup_program"] else: - return self._run_pipeline(program, - fetch_list=fetch_list, - use_program_cache=use_program_cache) + return self._run_pipeline( + program, + fetch_list=fetch_list, + use_program_cache=use_program_cache, + ) if isinstance(program, Program) and program._heter_pipeline_opt: - #print("program._heter_pipeline_opt: {}".format( + # print("program._heter_pipeline_opt: {}".format( # program._heter_pipeline_opt)) ## change default executor heter_place = program._heter_pipeline_opt["heter_place"] @@ -1500,20 +1645,26 @@ class Executor(object): self._default_executor = core.Executor(p) # TODO(zhangminxu): support heterps pipeline training using exe.run if "startup_program" in program._heter_pipeline_opt: - #print("get startup_program from _pipeline_opt") + # print("get startup_program from _pipeline_opt") program = program._heter_pipeline_opt["startup_program"] - if isinstance(program, Program) and \ - len(program.global_block().ops) == 0: + if ( + isinstance(program, Program) + and len(program.global_block().ops) == 0 + ): if use_default_main_program: - error_info = "Now you are using default_main_program, "\ - "but there are no operators in the program to be executed. "\ - "Please ensure you create model correctly or you can pass "\ + error_info = ( + "Now you are using default_main_program, " + "but there are no operators in the program to be executed. " + "Please ensure you create model correctly or you can pass " "the Program or the CompiledProgram manually." + ) else: - error_info = "There are no operators in the program to be executed. "\ - "If you pass Program manually, please use fluid.program_guard "\ + error_info = ( + "There are no operators in the program to be executed. " + "If you pass Program manually, please use fluid.program_guard " "to ensure the current Program is being used." + ) warnings.warn(error_info) if scope is None: @@ -1523,27 +1674,36 @@ class Executor(object): _origin_fetch_list = fetch_list _origin_program = program fetch_list, optimize_ops = self._split_optimize_ops_in_fetch_list( - fetch_list) + fetch_list + ) if optimize_ops: use_prune = True if use_prune: - cache_key = _get_strong_program_cache_key(program, feed, - _origin_fetch_list) + cache_key = _get_strong_program_cache_key( + program, feed, _origin_fetch_list + ) cached_pruned_program = self._get_pruned_program_cache(cache_key) if cached_pruned_program is None: if isinstance(program, compiler.CompiledProgram): program_scope_cache = self._get_pruned_program_scope_cache( - str(id(_origin_program))) + str(id(_origin_program)) + ) # copy the original program, so it can be cached. program = copy.copy(program) # share the local scopes for same original CompiledProgram. program._share_vars_from = program_scope_cache - if self._get_pruned_program_scope_cache( - str(id(_origin_program))) is None: + if ( + self._get_pruned_program_scope_cache( + str(id(_origin_program)) + ) + is None + ): self._add_pruned_program_scope_cache( - str(id(_origin_program)), program) - pruned_program = self._prune_program(program, feed, fetch_list, - optimize_ops) + str(id(_origin_program)), program + ) + pruned_program = self._prune_program( + program, feed, fetch_list, optimize_ops + ) self._add_pruned_program_cache(cache_key, pruned_program) else: pruned_program = cached_pruned_program @@ -1556,68 +1716,97 @@ class Executor(object): return False use_standalone_executor_for_distribution = os.environ.get( - 'FLAGS_CONVERT_GRAPH_TO_PROGRAM', - None) in [1, '1', True, 'True', 'true'] + 'FLAGS_CONVERT_GRAPH_TO_PROGRAM', None + ) in [1, '1', True, 'True', 'true'] - compiled = isinstance(program, - compiler.CompiledProgram) or isinstance( - program._graph, compiler.CompiledProgram) + compiled = isinstance( + program, compiler.CompiledProgram + ) or isinstance(program._graph, compiler.CompiledProgram) if compiled: - compiled_program = program if isinstance( - program, compiler.CompiledProgram) else program._graph + compiled_program = ( + program + if isinstance(program, compiler.CompiledProgram) + else program._graph + ) # Unsupported case 1: data parallel - if compiled_program._is_data_parallel and len( + if ( + compiled_program._is_data_parallel + and len( compiled_program._get_places( - place, compiled_program._places)) != 1: + place, compiled_program._places + ) + ) + != 1 + ): warnings.warn( "Standalone executor is not used for data parallel", - UserWarning) + UserWarning, + ) return False # Unsupported case 2: parallel graph if core.globals()['FLAGS_enable_parallel_graph'] in [ - 1, '1', True, 'True', 'true' + 1, + '1', + True, + 'True', + 'true', ]: warnings.warn( "Standalone executor is not used for parallel graph", - UserWarning) + UserWarning, + ) return False # Unsupported case 3: inference if compiled_program._is_inference: warnings.warn( "Standalone executor is not used for inference", - UserWarning) + UserWarning, + ) return False # Unsupported case 4: CUDA Graph - if compiled_program._build_strategy is not None and compiled_program._build_strategy.allow_cuda_graph_capture: + if ( + compiled_program._build_strategy is not None + and compiled_program._build_strategy.allow_cuda_graph_capture + ): warnings.warn( "Standalone executor is not used for CUDA Graph", - UserWarning) + UserWarning, + ) return False # Unsupported case 5: async mode - if compiled_program._build_strategy is not None and compiled_program._build_strategy.async_mode: + if ( + compiled_program._build_strategy is not None + and compiled_program._build_strategy.async_mode + ): warnings.warn( "Standalone executor is not used for async mode", - UserWarning) + UserWarning, + ) return False # delete this code after supporting fleet from paddle.distributed.fleet import fleet + if fleet._role_maker is not None: - warnings.warn("Standalone executor is not used for fleet", - UserWarning) + warnings.warn( + "Standalone executor is not used for fleet", UserWarning + ) return use_standalone_executor_for_distribution return True # NOTE: This is an experimental feature. If `export FLAGS_USE_STANDALONE_EXECUTOR=1 `, # use StandaloneExecutor to run the program. - if return_merged and self._enable_interpreter_core and _can_use_interpreter_core( - program, self.place): + if ( + return_merged + and self._enable_interpreter_core + and _can_use_interpreter_core(program, self.place) + ): if feed is None: feed = {} @@ -1627,18 +1816,27 @@ class Executor(object): if not isinstance(feed, dict): raise TypeError( "feed requires dict as its Parameter. But you passed in %s" - % (type(feed))) + % (type(feed)) + ) feed = self._update_feed(program, feed) program, new_exe = self._executor_cache.get_program_and_executor( - program, feed, fetch_list, feed_var_name, fetch_var_name, - self.place, scope) + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + self.place, + scope, + ) self._feed_data(program, feed, feed_var_name, scope) if hasattr(program, 'lr_sheduler'): from paddle.optimizer.lr import LRScheduler - assert isinstance(program.lr_sheduler, - LRScheduler), "must be LRScheduler" + + assert isinstance( + program.lr_sheduler, LRScheduler + ), "must be LRScheduler" lr_sheduler = program.lr_sheduler lr_value = lr_sheduler() lr_var = program.global_block().vars[lr_sheduler._var_name] @@ -1652,8 +1850,9 @@ class Executor(object): else: tensor._copy_from(cpu_tensor, self.place) - return new_exe.run(scope, list(feed.keys()), fetch_list, - return_numpy) + return new_exe.run( + scope, list(feed.keys()), fetch_list, return_numpy + ) compiled = isinstance(program, compiler.CompiledProgram) @@ -1668,13 +1867,15 @@ class Executor(object): varobj = global_block.vars[varname] # Can not check var build by fluid.layers.data(), bucause fluid.layers.data() had not set need_check_feed - if vardesc.persistable() == False and \ - vardesc.type() == core.VarDesc.VarType.LOD_TENSOR and \ - vardesc.need_check_feed() == True and \ - varobj.stop_gradient == True and \ - varobj.is_data == True and \ - varobj.belong_to_optimizer == False and \ - varname not in feed: + if ( + vardesc.persistable() == False + and vardesc.type() == core.VarDesc.VarType.LOD_TENSOR + and vardesc.need_check_feed() == True + and varobj.stop_gradient == True + and varobj.is_data == True + and varobj.belong_to_optimizer == False + and varname not in feed + ): raise ValueError('Need feed data for variable %s' % varname) acp._auto_checkpoint(self, program) @@ -1682,46 +1883,63 @@ class Executor(object): # For backward compatibility, run directly. if not compiled: # In distributed training, the compiled program is saved in Program._graph - has_compiled_graph = isinstance(program._graph, - compiler.CompiledProgram) + has_compiled_graph = isinstance( + program._graph, compiler.CompiledProgram + ) if has_compiled_graph: program._graph._compile(scope, self.place) # _graph in program does not support inference since the _graph is optimized # through optimizer.minimize function and should not be used as inference graph # assert not program._graph._is_inference - return self._run_parallel(program._graph, - scope=scope, - feed=feed, - fetch_list=fetch_list, - fetch_var_name=fetch_var_name, - return_numpy=return_numpy, - return_merged=return_merged) - - return self._run_program(program, - feed=feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name, - scope=scope, - return_numpy=return_numpy, - use_program_cache=use_program_cache) + return self._run_parallel( + program._graph, + scope=scope, + feed=feed, + fetch_list=fetch_list, + fetch_var_name=fetch_var_name, + return_numpy=return_numpy, + return_merged=return_merged, + ) + + return self._run_program( + program, + feed=feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + scope=scope, + return_numpy=return_numpy, + use_program_cache=use_program_cache, + ) program._compile(scope, self.place) if program._is_inference: return self._run_inference(program._executor, feed) else: - return self._run_parallel(program, - scope=scope, - feed=feed, - fetch_list=fetch_list, - fetch_var_name=fetch_var_name, - return_numpy=return_numpy, - return_merged=return_merged) - - def _run_program(self, program, feed, fetch_list, feed_var_name, - fetch_var_name, scope, return_numpy, use_program_cache): + return self._run_parallel( + program, + scope=scope, + feed=feed, + fetch_list=fetch_list, + fetch_var_name=fetch_var_name, + return_numpy=return_numpy, + return_merged=return_merged, + ) + + def _run_program( + self, + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + scope, + return_numpy, + use_program_cache, + ): from paddle.optimizer.lr import LRScheduler + if feed is None: feed = {} elif isinstance(feed, (list, tuple)): @@ -1730,19 +1948,22 @@ class Executor(object): if not isinstance(feed, dict): raise TypeError( - "feed requires dict as its Parameter. But you passed in %s" % - (type(feed))) + "feed requires dict as its Parameter. But you passed in %s" + % (type(feed)) + ) assert program is not None, "The program should not be Empty" if not isinstance(program, Program): raise TypeError( "Executor requires Program as its Parameter. But you passed in %s" - % (type(program))) + % (type(program)) + ) if not isinstance(fetch_var_name, str): raise TypeError( "The name of fetch variable requires string as its Parameter. But you passed in %s" - % (type(fetch_var_name))) + % (type(fetch_var_name)) + ) if use_program_cache: cache_key = _get_strong_program_cache_key(program, feed, fetch_list) @@ -1755,35 +1976,41 @@ class Executor(object): feed=feed, fetch_list=fetch_list, feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name) + fetch_var_name=fetch_var_name, + ) self._add_program_cache(cache_key, cached_program) fetch_list_str = list(map(_to_name_str, fetch_list)) cached_ctx = self._default_executor.prepare( - cached_program.desc, 0, fetch_list_str, False) + cached_program.desc, 0, fetch_list_str, False + ) # currently, we cache program, vars, sub_scope here # we suppose that in a life cycle of training, a user # will not create many programs. So, here the basic # rule of caching is to cache all unseen (program, var, scope) # when a user use use_program_cache. cached_scope = scope.new_scope() - self._default_executor.create_variables(cached_program.desc, - cached_scope, 0) + self._default_executor.create_variables( + cached_program.desc, cached_scope, 0 + ) self._add_ctx_cache(cache_key, cached_ctx) self._add_scope_cache(cache_key, cached_scope) program = cached_program ctx = cached_ctx scope = cached_scope else: - program = _add_feed_fetch_ops(program=program, - feed=feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name) + program = _add_feed_fetch_ops( + program=program, + feed=feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + ) self._feed_data(program, feed, feed_var_name, scope) if hasattr(program, 'lr_sheduler'): - assert isinstance(program.lr_sheduler, - LRScheduler), "must be LRScheduler" + assert isinstance( + program.lr_sheduler, LRScheduler + ), "must be LRScheduler" lr_sheduler = program.lr_sheduler lr_value = lr_sheduler() lr_var = program.global_block().vars[lr_sheduler._var_name] @@ -1792,11 +2019,13 @@ class Executor(object): tensor.set(data, self.place) if not use_program_cache: - self._default_executor.run(program.desc, scope, 0, True, True, - [fetch_var_name]) + self._default_executor.run( + program.desc, scope, 0, True, True, [fetch_var_name] + ) else: - self._default_executor.run_prepared_ctx(ctx, scope, False, False, - False) + self._default_executor.run_prepared_ctx( + ctx, scope, False, False, False + ) arr = scope.find_var(fetch_var_name).get_fetch_list() tensors = arr._move_to_list() if return_numpy: @@ -1811,13 +2040,16 @@ class Executor(object): is_fetch_var = lambda var: isinstance(var, (Variable, str)) is_tuple_list = lambda var: isinstance(var, (tuple, list)) - if fetch_list is None: return [] - if is_fetch_var(fetch_list): return [fetch_list] + if fetch_list is None: + return [] + if is_fetch_var(fetch_list): + return [fetch_list] - assert is_tuple_list(fetch_list), \ - "Currently , The fetch_list type only should be list or tuple, \n"\ - "but the input type is {}. For more information please refer to \n"\ + assert is_tuple_list(fetch_list), ( + "Currently , The fetch_list type only should be list or tuple, \n" + "but the input type is {}. For more information please refer to \n" "the executor.run(...).".format(type(fetch_list)) + ) res = [] for i, var in enumerate(fetch_list): @@ -1831,9 +2063,10 @@ class Executor(object): res.append(var) else: raise TypeError( - "Require fetch_list[{}] 's type shall be one of (Variable, str), but received {}." - .format(i, - type(var).__name__)) + "Require fetch_list[{}] 's type shall be one of (Variable, str), but received {}.".format( + i, type(var).__name__ + ) + ) return res @@ -1850,25 +2083,30 @@ class Executor(object): pipeline_num = filelist_length print( "Pipeline training: setting the pipeline num to %d is enough because there are only %d files" - % (filelist_length, filelist_length)) + % (filelist_length, filelist_length) + ) if filelist_length < pipeline_num * pipeline_opt["concurrency_list"][0]: print( "Pipeline training: setting the 1st element in concurrency_list to %d is enough because there are only %d files" - % (filelist_length // pipeline_num, filelist_length)) - pipeline_opt["concurrency_list"][ - 0] = filelist_length // pipeline_num + % (filelist_length // pipeline_num, filelist_length) + ) + pipeline_opt["concurrency_list"][0] = ( + filelist_length // pipeline_num + ) dataset.set_thread(pipeline_opt["concurrency_list"][0] * pipeline_num) return pipeline_num - def _prepare_trainer(self, - program=None, - dataset=None, - scope=None, - thread=0, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100): + def _prepare_trainer( + self, + program=None, + dataset=None, + scope=None, + thread=0, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + ): is_heter = 0 use_ps_gpu = 0 if not program._fleet_opt is None: @@ -1889,16 +2127,19 @@ class Executor(object): if is_heter: from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.utils.fleet_util import FleetUtil + fu = FleetUtil() ret = fu.split_program_by_device(program) if not compiled: # TODO: Need a better way to distinguish and specify different execution mode if program._pipeline_opt: trainer = TrainerFactory()._create_trainer( - program._pipeline_opt) + program._pipeline_opt + ) elif program._heter_pipeline_opt: trainer = TrainerFactory()._create_trainer( - program._heter_pipeline_opt) + program._heter_pipeline_opt + ) else: trainer = TrainerFactory()._create_trainer(program._fleet_opt) trainer._set_thread_barrier(program._is_distributed) @@ -1908,13 +2149,16 @@ class Executor(object): else: if program._pipeline_opt: trainer = TrainerFactory()._create_trainer( - program.program._pipeline_opt) + program.program._pipeline_opt + ) elif program._heter_pipeline_opt: trainer = TrainerFactory()._create_trainer( - program.program._heter_pipeline_opt) + program.program._heter_pipeline_opt + ) else: trainer = TrainerFactory()._create_trainer( - program.program._fleet_opt) + program.program._fleet_opt + ) trainer._set_program(program.program) if thread <= 0: @@ -1923,7 +2167,8 @@ class Executor(object): elif dataset.thread_num <= 0: raise RuntimeError( "You should set thread num first, either in Dataset" - "or in Executor.train_from_dataset") + "or in Executor.train_from_dataset" + ) else: trainer._set_thread(dataset.thread_num) else: @@ -1933,19 +2178,22 @@ class Executor(object): trainer._set_fetch_var_and_info(fetch_list, fetch_info, print_period) return scope, trainer - def _run_from_dataset(self, - program=None, - dataset=None, - scope=None, - thread=0, - is_infer=False, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): + def _run_from_dataset( + self, + program=None, + dataset=None, + scope=None, + thread=0, + is_infer=False, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): if program._pipeline_opt is not None: import paddle + if dataset is not None: raise RuntimeError("dataset should be None for pipeline mode") # The following fake dataset is created to call @@ -1956,24 +2204,28 @@ class Executor(object): data_vars.append(var) if core.is_compiled_with_npu(): dataset = paddle.fluid.DatasetFactory().create_dataset( - 'InMemoryDataset') + 'InMemoryDataset' + ) else: dataset = paddle.fluid.DatasetFactory().create_dataset( - 'FileInstantDataset') + 'FileInstantDataset' + ) dataset.set_batch_size(1) dataset.set_thread(1) dataset.set_filelist(['None']) dataset.set_use_var(data_vars) elif program._heter_pipeline_opt is not None: stage_id = program._heter_pipeline_opt["pipeline_stage"] - #print("test_fl_stage_id: {}".format(stage_id)) + # print("test_fl_stage_id: {}".format(stage_id)) heter_place = program._heter_pipeline_opt["heter_place"] if stage_id != 0: if "is_fl_mode" not in program._heter_pipeline_opt: import paddle + if dataset is not None: raise RuntimeError( - "dataset should be None for heter pipeline mode") + "dataset should be None for heter pipeline mode" + ) # The following fake dataset is created to call # the _prepare_trainer api, and it is meaningless. data_vars = [] @@ -1981,7 +2233,8 @@ class Executor(object): if var.is_data: data_vars.append(var) dataset = paddle.fluid.DatasetFactory().create_dataset( - 'InMemoryDataset') + 'InMemoryDataset' + ) dataset.set_batch_size(1) dataset.set_thread(1) dataset.set_filelist(['None']) @@ -1989,7 +2242,8 @@ class Executor(object): else: if dataset is None: raise RuntimeError( - "dataset is need and should be initialized") + "dataset is need and should be initialized" + ) ## change default executor heter_place = framework._get_paddle_place(heter_place) p = core.Place() @@ -2016,7 +2270,8 @@ class Executor(object): feed=[], fetch_list=real_fetch_list, feed_var_name='feed', - fetch_var_name='fetch') + fetch_var_name='fetch', + ) main_block = program._pipeline_opt["section_program"].block(0) for op in main_block.ops: # set the op_role of fetch op to Optimize to avoid @@ -2024,16 +2279,19 @@ class Executor(object): if op.type == 'fetch': op._set_attr( 'op_role', - core.op_proto_and_checker_maker.OpRole.Optimize) + core.op_proto_and_checker_maker.OpRole.Optimize, + ) fetch_list = None - scope, trainer = self._prepare_trainer(program=program, - dataset=dataset, - scope=scope, - thread=thread, - debug=debug, - fetch_list=fetch_list, - fetch_info=fetch_info, - print_period=print_period) + scope, trainer = self._prepare_trainer( + program=program, + dataset=dataset, + scope=scope, + thread=thread, + debug=debug, + fetch_list=fetch_list, + fetch_info=fetch_info, + print_period=print_period, + ) trainer._set_infer(is_infer) trainer._gen_trainer_desc() @@ -2048,8 +2306,11 @@ class Executor(object): dataset._dynamic_adjust_before_train(trainer.proto_desc.thread_num) if program._heter_pipeline_opt is None: - trainer_instance = self._default_executor.init_for_dataset( # -->InitForDataset - program.desc, trainer._desc(), scope, dataset.dataset) + trainer_instance = ( + self._default_executor.init_for_dataset( # -->InitForDataset + program.desc, trainer._desc(), scope, dataset.dataset + ) + ) else: # cache trainer instance for heterps pipeline training if fetch_list == None: @@ -2058,8 +2319,9 @@ class Executor(object): trainer_instance = self._get_trainer_cache(cache_key) if trainer_instance is None: trainer_instance = self._default_executor.init_for_dataset( - program.desc, trainer._desc(), scope, dataset.dataset) - #print("test_fl_ps - trainer_desc: {}\n".format(trainer)) + program.desc, trainer._desc(), scope, dataset.dataset + ) + # print("test_fl_ps - trainer_desc: {}\n".format(trainer)) self._add_trainer_cache(cache_key, trainer_instance) else: trainer_instance.ResetDataset(dataset.dataset) @@ -2086,18 +2348,20 @@ class Executor(object): return None - def _prepare_pipeline_ctx(self, - program=None, - dataset=None, - scope=None, - thread=0, - is_infer=False, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None, - use_program_cache=False): + def _prepare_pipeline_ctx( + self, + program=None, + dataset=None, + scope=None, + thread=0, + is_infer=False, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + use_program_cache=False, + ): assert program._pipeline_opt is not None assert dataset is None, "dataset should be None for pipeline mode" @@ -2117,10 +2381,12 @@ class Executor(object): data_vars.append(var) if core.is_compiled_with_npu(): dataset = paddle.fluid.DatasetFactory().create_dataset( - 'InMemoryDataset') + 'InMemoryDataset' + ) else: dataset = paddle.fluid.DatasetFactory().create_dataset( - 'FileInstantDataset') + 'FileInstantDataset' + ) dataset.set_batch_size(1) dataset.set_thread(1) dataset.set_filelist(['None']) @@ -2141,11 +2407,13 @@ class Executor(object): if fetch_var_name in real_program.global_block().vars: real_fetch_list.append(fetch_var) - real_program = _add_feed_fetch_ops(program=real_program, - feed=[], - fetch_list=real_fetch_list, - feed_var_name='feed', - fetch_var_name='fetch') + real_program = _add_feed_fetch_ops( + program=real_program, + feed=[], + fetch_list=real_fetch_list, + feed_var_name='feed', + fetch_var_name='fetch', + ) main_block = real_program.block(0) for op in main_block.ops: # set the op_role of fetch op to Optimize to avoid @@ -2153,7 +2421,8 @@ class Executor(object): if op.type == 'fetch': op._set_attr( 'op_role', - core.op_proto_and_checker_maker.OpRole.Optimize) + core.op_proto_and_checker_maker.OpRole.Optimize, + ) return real_program, real_fetch_list real_program, real_fetch_list = _get_real_program_fetch_list() @@ -2161,14 +2430,16 @@ class Executor(object): program._pipeline_opt["section_program"] = real_program fetch_list = None - scope, trainer = self._prepare_trainer(program=program, - dataset=dataset, - scope=scope, - thread=thread, - debug=debug, - fetch_list=fetch_list, - fetch_info=fetch_info, - print_period=print_period) + scope, trainer = self._prepare_trainer( + program=program, + dataset=dataset, + scope=scope, + thread=thread, + debug=debug, + fetch_list=fetch_list, + fetch_info=fetch_info, + print_period=print_period, + ) trainer._set_infer(is_infer) trainer._gen_trainer_desc() @@ -2183,59 +2454,84 @@ class Executor(object): trainer_desc = trainer._desc() # slow, cache trainer_instance = self._default_executor.init_for_dataset( - program.desc, trainer_desc, scope, dataset.dataset) + program.desc, trainer_desc, scope, dataset.dataset + ) ctx = [scope, real_fetch_list, trainer_instance] - if use_program_cache: self._add_ctx_cache(cache_key, ctx) + if use_program_cache: + self._add_ctx_cache(cache_key, ctx) return ctx - def _prepare_fleet_executor_carrier(self, - carrier_id="", - program=None, - scope=None, - fleet_opt=None, - with_standalone_executor=False): - num_micro_batches = fleet_opt[ - "num_micro_batches"] if "num_micro_batches" in fleet_opt else 1 + def _prepare_fleet_executor_carrier( + self, + carrier_id="", + program=None, + scope=None, + fleet_opt=None, + with_standalone_executor=False, + ): + num_micro_batches = ( + fleet_opt["num_micro_batches"] + if "num_micro_batches" in fleet_opt + else 1 + ) cur_rank = int(os.getenv("PADDLE_TRAINER_ID", 0)) trainer_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS", "").split(',') nrank = len(trainer_endpoints) - assert 'scheduler' in fleet_opt or 'tasks' in fleet_opt, \ - "Fleet executor need configuration for scheduler, you can choose from 1F1B or Origin. " \ + assert 'scheduler' in fleet_opt or 'tasks' in fleet_opt, ( + "Fleet executor need configuration for scheduler, you can choose from 1F1B or Origin. " "Or you can provide a list of task nodes to init fleet executor directly." + ) if 'tasks' in fleet_opt: - assert 'task_id_to_rank' in fleet_opt, "If you provide tasks to init fleet executor," \ - " task_id_to_rank should also be provided." + assert 'task_id_to_rank' in fleet_opt, ( + "If you provide tasks to init fleet executor," + " task_id_to_rank should also be provided." + ) print('fleet executor will use user defined task nodes') tasks = [task.task_node() for task in fleet_opt['tasks']] task_id_to_rank = fleet_opt['task_id_to_rank'] else: scheduler = fleet_opt['scheduler'] if scheduler == '1F1B': - from paddle.distributed.fleet.fleet_executor_utils import run1f1b - if "dist_strategy" not in fleet_opt or \ - "pp_degree" not in fleet_opt["dist_strategy"] or \ - fleet_opt["dist_strategy"]["pp_degree"] == 1: + from paddle.distributed.fleet.fleet_executor_utils import ( + run1f1b, + ) + + if ( + "dist_strategy" not in fleet_opt + or "pp_degree" not in fleet_opt["dist_strategy"] + or fleet_opt["dist_strategy"]["pp_degree"] == 1 + ): warnings.warn("Using 1F1B scheduler with pp_degree == 1.") tasks, task_id_to_rank = run1f1b( - program, cur_rank, fleet_opt.get('num_micro_batches', 1), - fleet_opt.get('dist_strategy', {}), nrank, - with_standalone_executor) + program, + cur_rank, + fleet_opt.get('num_micro_batches', 1), + fleet_opt.get('dist_strategy', {}), + nrank, + with_standalone_executor, + ) elif scheduler == 'Origin': from paddle.distributed.fleet.fleet_executor_utils import origin - if "dist_strategy" in fleet_opt and \ - "pp_degree" in fleet_opt["dist_strategy"]: - assert fleet_opt["dist_strategy"]["pp_degree"] == 1, \ - "For pipeline mode, the scheduler should be 1F1B instead of Origin." + + if ( + "dist_strategy" in fleet_opt + and "pp_degree" in fleet_opt["dist_strategy"] + ): + assert ( + fleet_opt["dist_strategy"]["pp_degree"] == 1 + ), "For pipeline mode, the scheduler should be 1F1B instead of Origin." if "num_micro_batches" in fleet_opt: - assert fleet_opt["num_micro_batches"] == 1, \ - "For origin scheduler mode, the num micro batches should be 1." + assert ( + fleet_opt["num_micro_batches"] == 1 + ), "For origin scheduler mode, the num micro batches should be 1." tasks, task_id_to_rank = origin(program, cur_rank) else: - raise "Fleet_executor only supports 1F1B and Origin scheduler, " \ - "but received " + str(scheduler) + "." + raise "Fleet_executor only supports 1F1B and Origin scheduler, " "but received " + str( + scheduler + ) + "." # NOTE: have to hold these vars, otherwise will be destructed fleet_opt['tasks'] = tasks fleet_opt['task_id_to_rank'] = task_id_to_rank @@ -2243,16 +2539,26 @@ class Executor(object): place.set_place(self.place) # NOTE: the last argument is used to force create some vars in root scope, # won't be used during train. - self._fleet_executor.init(carrier_id, program.desc, scope, place, - num_micro_batches, tasks, task_id_to_rank, []) - - def _run_using_fleet_executor(self, - program=None, - feed=None, - feed_var_name="feed", - fetch_var_name="fetch", - fetch_list=None, - with_standalone_executor=False): + self._fleet_executor.init( + carrier_id, + program.desc, + scope, + place, + num_micro_batches, + tasks, + task_id_to_rank, + [], + ) + + def _run_using_fleet_executor( + self, + program=None, + feed=None, + feed_var_name="feed", + fetch_var_name="fetch", + fetch_list=None, + with_standalone_executor=False, + ): cache_key = _get_strong_program_cache_key(program, feed, fetch_list) cached_program = self._get_program_cache(cache_key) cached_scope = self._get_scope_cache(cache_key) @@ -2260,16 +2566,20 @@ class Executor(object): cached_scope = global_scope() self._add_scope_cache(cache_key, cached_scope) if cached_program is None: - assert program._pipeline_opt, "program should have _pipeline_opt to start carrier" + assert ( + program._pipeline_opt + ), "program should have _pipeline_opt to start carrier" real_feed = [] if feed is None else feed real_program = program if "section_program" in program._pipeline_opt: real_program = program._pipeline_opt["section_program"] - cached_program = _add_feed_fetch_ops(program=real_program, - feed=real_feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name) + cached_program = _add_feed_fetch_ops( + program=real_program, + feed=real_feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + ) main_block = cached_program.block(0) for op in main_block.ops: # set the op_role of fetch op to Optimize to avoid @@ -2277,7 +2587,8 @@ class Executor(object): if op.type == 'fetch': op._set_attr( 'op_role', - core.op_proto_and_checker_maker.OpRole.Optimize) + core.op_proto_and_checker_maker.OpRole.Optimize, + ) self._add_program_cache(cache_key, cached_program) fleet_opt = program._pipeline_opt["fleet_opt"] if 'tasks' in fleet_opt: @@ -2291,9 +2602,11 @@ class Executor(object): feed_task = fleet_opt['tasks'][0] print("Inserting feed ops for task", feed_task.task_id()) feed_program = feed_task.get_program() - feed_program = self._add_feed_ops(program=feed_program, - feed=real_feed, - feed_var_name=feed_var_name) + feed_program = self._add_feed_ops( + program=feed_program, + feed=real_feed, + feed_var_name=feed_var_name, + ) feed_task.set_program(feed_program) # Insert fetch ops @@ -2303,7 +2616,8 @@ class Executor(object): fetch_program = self._add_fetch_ops( program=fetch_program, fetch_list=fetch_list, - fetch_var_name=fetch_var_name) + fetch_var_name=fetch_var_name, + ) main_block = fetch_program.block(0) for op in main_block.ops: # set the op_role of fetch op to Optimize to avoid @@ -2311,7 +2625,8 @@ class Executor(object): if op.type == 'fetch': op._set_attr( 'op_role', - core.op_proto_and_checker_maker.OpRole.Optimize) + core.op_proto_and_checker_maker.OpRole.Optimize, + ) fetch_task.set_program(fetch_program) self._prepare_fleet_executor_carrier( @@ -2319,7 +2634,8 @@ class Executor(object): program=cached_program, scope=cached_scope, fleet_opt=fleet_opt, - with_standalone_executor=with_standalone_executor) + with_standalone_executor=with_standalone_executor, + ) if feed: # NOTE: don't have to traverse programs in task nodes, @@ -2328,14 +2644,16 @@ class Executor(object): self._feed_data(cached_program, feed, feed_var_name, cached_scope) from paddle.optimizer.lr import LRScheduler + if hasattr(program, 'lr_sheduler'): lr_sheduler = program.lr_sheduler assert isinstance(lr_sheduler, LRScheduler), "must be LRScheduler" lr_value = lr_sheduler() lr_var = program.global_block().vars[lr_sheduler._var_name] data = np.array([lr_value]).astype(convert_dtype(lr_var.dtype)) - tensor = core.get_variable_tensor(cached_scope, - lr_sheduler._var_name) + tensor = core.get_variable_tensor( + cached_scope, lr_sheduler._var_name + ) tensor.set(data, self.place) self._fleet_executor.run(cache_key) @@ -2357,30 +2675,32 @@ class Executor(object): feed_var = global_block.create_var( name=feed_var_name, type=core.VarDesc.VarType.FEED_MINIBATCH, - persistable=True) + persistable=True, + ) # prepend feed operators if not has_feed_operators(global_block, feed, feed_var_name): for i, name in enumerate(feed): if global_block.has_var(name): out = global_block.var(name) - global_block._prepend_op(type='feed', - inputs={'X': [feed_var]}, - outputs={'Out': [out]}, - attrs={'col': i}) + global_block._prepend_op( + type='feed', + inputs={'X': [feed_var]}, + outputs={'Out': [out]}, + attrs={'col': i}, + ) else: warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % name) + % name + ) return tmp_program @classmethod - def _add_fetch_ops(cls, - program, - fetch_list, - fetch_var_name, - use_fetch_v2=False): + def _add_fetch_ops( + cls, program, fetch_list, fetch_var_name, use_fetch_v2=False + ): tmp_program = program.clone() global_block = tmp_program.global_block() @@ -2391,7 +2711,8 @@ class Executor(object): fetch_var = global_block.create_var( name=fetch_var_name, type=core.VarDesc.VarType.FETCH_LIST, - persistable=True) + persistable=True, + ) if use_fetch_v2: fetch_op = 'fetch_v2' @@ -2399,16 +2720,19 @@ class Executor(object): fetch_op = 'fetch' # append fetch_operators - if not has_fetch_operators(global_block, fetch_list, fetch_var_name, - fetch_op): + if not has_fetch_operators( + global_block, fetch_list, fetch_var_name, fetch_op + ): for i, var in enumerate(fetch_list): assert isinstance(var, Variable) or isinstance( - var, str), ("Wrong type for fetch_list[%s]: %s" % - (i, type(var))) - global_block.append_op(type=fetch_op, - inputs={'X': [var]}, - outputs={'Out': [fetch_var]}, - attrs={'col': i}) + var, str + ), "Wrong type for fetch_list[%s]: %s" % (i, type(var)) + global_block.append_op( + type=fetch_op, + inputs={'X': [var]}, + outputs={'Out': [fetch_var]}, + attrs={'col': i}, + ) return tmp_program @@ -2423,25 +2747,36 @@ class Executor(object): return tmp_program - def _run_pipeline(self, - program=None, - dataset=None, - scope=None, - thread=0, - is_infer=False, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None, - use_program_cache=False): - scope, real_fetch_list, trainer_instance = \ - self._prepare_pipeline_ctx(program, dataset, scope, thread, - is_infer, debug, fetch_list, fetch_info, - print_period, fetch_handler, - use_program_cache) + def _run_pipeline( + self, + program=None, + dataset=None, + scope=None, + thread=0, + is_infer=False, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + use_program_cache=False, + ): + scope, real_fetch_list, trainer_instance = self._prepare_pipeline_ctx( + program, + dataset, + scope, + thread, + is_infer, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + use_program_cache, + ) from paddle.optimizer.lr import LRScheduler + if hasattr(program, 'lr_sheduler'): lr_sheduler = program.lr_sheduler assert isinstance(lr_sheduler, LRScheduler), "must be LRScheduler" @@ -2463,16 +2798,18 @@ class Executor(object): return None - def infer_from_dataset(self, - program=None, - dataset=None, - scope=None, - thread=0, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): + def infer_from_dataset( + self, + program=None, + dataset=None, + scope=None, + thread=0, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): """ Infer from a pre-defined Dataset. Dataset is defined in paddle.fluid.dataset. Given a program, either a program or compiled program, infer_from_dataset will @@ -2528,26 +2865,39 @@ class Executor(object): dataset=dataset) """ - return self._run_from_dataset(program, dataset, scope, thread, True, - debug, fetch_list, fetch_info, - print_period, fetch_handler) - - def start_heter_trainer(self, - program=None, - scope=None, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): - scope, trainer = self._prepare_trainer(program=program, - dataset=None, - scope=scope, - thread=1, - debug=debug, - fetch_list=fetch_list, - fetch_info=fetch_info, - print_period=print_period) + return self._run_from_dataset( + program, + dataset, + scope, + thread, + True, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + ) + + def start_heter_trainer( + self, + program=None, + scope=None, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): + scope, trainer = self._prepare_trainer( + program=program, + dataset=None, + scope=scope, + thread=1, + debug=debug, + fetch_list=fetch_list, + fetch_info=fetch_info, + print_period=print_period, + ) trainer._set_infer(False) trainer._gen_trainer_desc() @@ -2555,32 +2905,35 @@ class Executor(object): self._dump_debug_info(program=program, trainer=trainer) trainer_instance = self._default_executor.init_for_dataset( - program.desc, trainer._desc(), scope, None) + program.desc, trainer._desc(), scope, None + ) - #if fetch_handler is not None: + # if fetch_handler is not None: # scope0 = trainer_instance.get_worker_scope(0) # fetch_monitor = FetchHandlerMonitor(scope0, fetch_handler) # fetch_monitor.start() # self._default_executor.run_from_dataset(trainer_instance) # fetch_monitor.stop() # self._default_executor.release_trainer(trainer_instance) - #else: + # else: self._default_executor.run_from_dataset(trainer_instance) - #self._default_executor.release_trainer(trainer_instance) + # self._default_executor.release_trainer(trainer_instance) return trainer_instance - def train_from_dataset(self, - program=None, - dataset=None, - scope=None, - thread=0, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): + def train_from_dataset( + self, + program=None, + dataset=None, + scope=None, + thread=0, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): """ Train from a pre-defined Dataset. Dataset is defined in paddle.fluid.dataset. Given a program, either a program or compiled program, train_from_dataset will @@ -2635,6 +2988,15 @@ class Executor(object): dataset=dataset) """ - return self._run_from_dataset(program, dataset, scope, thread, False, - debug, fetch_list, fetch_info, - print_period, fetch_handler) + return self._run_from_dataset( + program, + dataset, + scope, + thread, + False, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + ) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 99abb76d213..46bd46d706a 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -72,7 +72,7 @@ ZERO_VAR_SUFFIX = core.kZeroVarSuffix() CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName() _dygraph_tracer_ = None -_in_eager_mode_ = (os.environ.get('FLAGS_enable_eager_mode', '1') == '1') +_in_eager_mode_ = os.environ.get('FLAGS_enable_eager_mode', '1') == '1' _global_expected_place_ = None _current_device = None global_prog_seed = 0 @@ -81,10 +81,12 @@ _already_patch_eager_tensor = False _already_patch_varbase = False _current_cuda_graph_mode = None _global_flags_ = core.globals() -_enable_standalone_executor_ = (os.environ.get('FLAGS_USE_STANDALONE_EXECUTOR', - None)) -_dy2st_enable_standalone_executor_ = (os.environ.get( - 'FLAGS_DY2ST_USE_STANDALONE_EXECUTOR', 1)) +_enable_standalone_executor_ = os.environ.get( + 'FLAGS_USE_STANDALONE_EXECUTOR', None +) +_dy2st_enable_standalone_executor_ = os.environ.get( + 'FLAGS_DY2ST_USE_STANDALONE_EXECUTOR', 1 +) # Some explanation of our execution system 2022.03 # For now we have 3 kinds of execution system, since we refactored dygraph mode to @@ -145,6 +147,7 @@ def _update_monkey_methods(is_eager): def _switch_tensor_bind_type(is_eager): import paddle + if is_eager: paddle.Tensor = core.eager.Tensor else: @@ -179,8 +182,11 @@ def _fallback_legacy_dygraph(): global _is_first_import_ need_fallback = False # Only enable eager on CPU/GPU/XPU - is_not_support = core.is_compiled_with_npu() or core.is_compiled_with_ipu( - ) or core.is_compiled_with_mlu() + is_not_support = ( + core.is_compiled_with_npu() + or core.is_compiled_with_ipu() + or core.is_compiled_with_mlu() + ) if _in_eager_mode_ and is_not_support: # switch into legacy dygraph mode @@ -358,7 +364,6 @@ def set_ipu_shard(call_func, index=-1, stage=-1): """ def decorate(func): - def wrapper(*args, **kwargs): with ipu_shard_guard(index=index, stage=stage): return func(*args, **kwargs) @@ -366,16 +371,17 @@ def set_ipu_shard(call_func, index=-1, stage=-1): return wrapper from .dygraph.layers import Layer + if not isinstance(call_func, Layer): if callable(call_func): return decorate(call_func) else: raise TypeError( - "Unsupported type. Only accept paddle.nn.Layer or function.") + "Unsupported type. Only accept paddle.nn.Layer or function." + ) # patch paddle.nn.Layer class BlockFn(type(call_func)): - def __call__(self, *args, **kwargs): with ipu_shard_guard(index=index, stage=stage): return super().__call__(*args, **kwargs) @@ -387,62 +393,68 @@ def set_ipu_shard(call_func, index=-1, stage=-1): def require_version(min_version, max_version=None): """ - Check if the installed version of PaddlePaddle is in [min_version, max_version], - if the installed version is lower than ``min_version`` or higher than ``max_version``, - an exception will be thrown, NO returns if the installed version is satisfied. + Check if the installed version of PaddlePaddle is in [min_version, max_version], + if the installed version is lower than ``min_version`` or higher than ``max_version``, + an exception will be thrown, NO returns if the installed version is satisfied. - Args: - min_version (str): the minimum version required (like '1.4.0'). - max_version (str, optional): the max version required (like '1.6.0'), default is None, - meaning any version equal or higher than ``min_version`` is acceptable. + Args: + min_version (str): the minimum version required (like '1.4.0'). + max_version (str, optional): the max version required (like '1.6.0'), default is None, + meaning any version equal or higher than ``min_version`` is acceptable. - Returns: - None. + Returns: + None. - Raises: - TypeError: if the type of ``min_version`` is not str. - TypeError: if the type of ``max_version`` is not str or type(None). - ValueError: if the value of ``min_version`` is not in version format. - ValueError: if the value of ``max_version`` is not in version format or None. - Exception: if the installed version is lower than ``min_version`` or higher than ``max_version``. + Raises: + TypeError: if the type of ``min_version`` is not str. + TypeError: if the type of ``max_version`` is not str or type(None). + ValueError: if the value of ``min_version`` is not in version format. + ValueError: if the value of ``max_version`` is not in version format or None. + Exception: if the installed version is lower than ``min_version`` or higher than ``max_version``. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - import paddle.fluid as fluid + import paddle.fluid as fluid - # any version >= 0.1.0 is acceptable. - fluid.require_version('0.1.0') + # any version >= 0.1.0 is acceptable. + fluid.require_version('0.1.0') - # if 0.1.0 <= version <= 10.0.0, it is acceptable. - fluid.require_version(min_version='0.1.0', max_version='10.0.0') - """ + # if 0.1.0 <= version <= 10.0.0, it is acceptable. + fluid.require_version(min_version='0.1.0', max_version='10.0.0') + """ if not isinstance(min_version, str): raise TypeError( "The type of 'min_version' in require_version must be str, but received %s." - % (type(min_version))) + % (type(min_version)) + ) if not isinstance(max_version, (str, type(None))): raise TypeError( "The type of 'max_version' in require_version must be str or type(None), but received %s." - % (type(max_version))) + % (type(max_version)) + ) check_format = re.match(r'\d+(\.\d+){0,3}', min_version) if check_format is None or check_format.group() != min_version: raise ValueError( "The value of 'min_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', " - "like '1.5.2.0', but received %s" % min_version) + "like '1.5.2.0', but received %s" % min_version + ) if max_version is not None: check_format = re.match(r'\d+(\.\d+){0,3}', max_version) if check_format is None or check_format.group() != max_version: raise ValueError( "The value of 'max_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', " - "like '1.5.2.0', but received %s" % max_version) + "like '1.5.2.0', but received %s" % max_version + ) version_installed = [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] zero_version = ['0', '0', '0', '0'] @@ -459,75 +471,86 @@ def require_version(min_version, max_version=None): warnings.warn( "PaddlePaddle version in [%s, %s] required, but %s installed. " "Maybe you are using a develop version, " - "please make sure the version is good with your code." % - (min_version, max_version, fluid_version.full_version)) + "please make sure the version is good with your code." + % (min_version, max_version, fluid_version.full_version) + ) else: warnings.warn( "PaddlePaddle version %s or higher is required, but %s installed, " "Maybe you are using a develop version, " - "please make sure the version is good with your code." % - (min_version, fluid_version.full_version)) + "please make sure the version is good with your code." + % (min_version, fluid_version.full_version) + ) return min_version_split = min_version.split('.') - min_version_to_check = min_version_split + zero_version[ - len(min_version_split):] + min_version_to_check = ( + min_version_split + zero_version[len(min_version_split) :] + ) if max_version is not None: max_version_split = max_version.split('.') - max_version_to_check = max_version_split + zero_version[ - len(max_version_split):] + max_version_to_check = ( + max_version_split + zero_version[len(max_version_split) :] + ) - if version_cmp(version_installed, - max_version_to_check) > 0 or version_cmp( - version_installed, min_version_to_check) < 0: + if ( + version_cmp(version_installed, max_version_to_check) > 0 + or version_cmp(version_installed, min_version_to_check) < 0 + ): raise Exception( "VersionError: PaddlePaddle version in [%s, %s] required, but %s installed." - % (min_version, max_version, fluid_version.full_version)) + % (min_version, max_version, fluid_version.full_version) + ) else: if version_cmp(version_installed, min_version_to_check) < 0: raise Exception( "VersionError: PaddlePaddle version %s or higher is required, but %s installed, " "please upgrade your PaddlePaddle to %s or other higher version." - % (min_version, fluid_version.full_version, min_version)) + % (min_version, fluid_version.full_version, min_version) + ) def _dygraph_not_support_(func): - def __impl__(*args, **kwargs): - assert not _non_static_mode( - ), "We don't support %s in dynamic graph mode" % func.__name__ + assert not _non_static_mode(), ( + "We don't support %s in dynamic graph mode" % func.__name__ + ) return func(*args, **kwargs) return __impl__ def _dygraph_only_(func): - def __impl__(*args, **kwargs): - assert _non_static_mode( - ), "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." % func.__name__ + assert _non_static_mode(), ( + "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." + % func.__name__ + ) return func(*args, **kwargs) return __impl__ def _non_static_only_(func): - def __impl__(*args, **kwargs): from .dygraph.base import in_declarative_mode - assert _non_static_mode() or in_declarative_mode( - ), "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." % func.__name__ + + assert _non_static_mode() or in_declarative_mode(), ( + "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." + % func.__name__ + ) return func(*args, **kwargs) return __impl__ def _static_only_(func): - def __impl__(*args, **kwargs): - assert not _non_static_mode( - ), "In PaddlePaddle 2.x, we turn on dynamic graph mode by default, and '%s()' is only supported in static graph mode. So if you want to use this api, please call 'paddle.enable_static()' before this api to enter static graph mode." % func.__name__ + assert not _non_static_mode(), ( + "In PaddlePaddle 2.x, we turn on dynamic graph mode by default, and '%s()' is only supported in static graph mode. So if you want to use this api, please call 'paddle.enable_static()' before this api to enter static graph mode." + % func.__name__ + ) return func(*args, **kwargs) return __impl__ @@ -546,14 +569,14 @@ def _set_pipeline_stage(stage): # TODO(zhiqiu): We should make VarBase consistent with Variable in future, for example, by inheritting # same base class. def _fake_interface_only_(func): - def __impl__(*args, **kwargs): raise AssertionError( "'%s' only can be called by `paddle.Tensor` in dynamic graph mode. Suggestions:\n" " 1. If you are in static graph mode, you can switch to dynamic graph mode by turning off `paddle.enable_static()` or calling `paddle.disable_static()`.\n" " 2. If you are using `@paddle.jit.to_static`, you can turn off ProgramTranslator by calling `paddle.jit.ProgramTranslator().enable(False)`. " "If you have to translate dynamic graph to static graph, please use other API to replace '%s'." - % (func.__name__, func.__name__)) + % (func.__name__, func.__name__) + ) return __impl__ @@ -564,13 +587,13 @@ def _fake_interface_only_(func): # NOTE(chenweihang): not using `wrap_decorator` here is because `wrap_decorator` will # move kwargs to args, which doesn't work in this decorate case def deprecate_stat_dict(func): - @functools.wraps(func) def wrapper(*args, **kwargs): if 'stat_dict' in kwargs: warnings.warn( "The argument `stat_dict` has deprecated, please change it to `state_dict`.", - DeprecationWarning) + DeprecationWarning, + ) kwargs['state_dict'] = kwargs['stat_dict'] kwargs.pop('stat_dict') return func(*args, **kwargs) @@ -676,7 +699,9 @@ def _cpu_num(): 'And if this parameter are set as N (equal to the number of physical CPU core) the program may be faster.\n\n' 'export CPU_NUM={} # for example, set CPU_NUM as number of physical CPU core which is {}.\n\n' '!!! The default number of CPU_NUM=1.\n'.format( - multiprocessing.cpu_count(), multiprocessing.cpu_count())) + multiprocessing.cpu_count(), multiprocessing.cpu_count() + ) + ) os.environ['CPU_NUM'] = str(1) cpu_num = os.environ.get('CPU_NUM') return int(cpu_num) @@ -858,8 +883,7 @@ def cuda_places(device_ids=None): cuda_places = static.cuda_places() """ - assert core.is_compiled_with_cuda(), \ - "Not compiled with CUDA" + assert core.is_compiled_with_cuda(), "Not compiled with CUDA" if device_ids is None: device_ids = _cuda_ids() elif not isinstance(device_ids, (list, tuple)): @@ -898,8 +922,7 @@ def xpu_places(device_ids=None): paddle.enable_static() xpu_places = static.xpu_places() """ - assert core.is_compiled_with_xpu(), \ - "Not compiled with XPU" + assert core.is_compiled_with_xpu(), "Not compiled with XPU" if device_ids is None: device_ids = _xpu_ids() elif not isinstance(device_ids, (list, tuple)): @@ -939,8 +962,7 @@ def npu_places(device_ids=None): paddle.enable_static() npu_places = static.npu_places() """ - assert core.is_compiled_with_npu(), \ - "Not compiled with NPU" + assert core.is_compiled_with_npu(), "Not compiled with NPU" if device_ids is None: device_ids = _npu_ids() elif not isinstance(device_ids, (list, tuple)): @@ -1008,8 +1030,7 @@ def cuda_pinned_places(device_count=None): cuda_pinned_places = fluid.cuda_pinned_places(1) """ - assert core.is_compiled_with_cuda(), \ - "Not compiled with CUDA" + assert core.is_compiled_with_cuda(), "Not compiled with CUDA" if device_count is None: device_count = len(_cuda_ids()) return [core.CUDAPinnedPlace()] * device_count @@ -1049,8 +1070,7 @@ def mlu_places(device_ids=None): paddle.enable_static() mlu_places = static.mlu_places() """ - assert core.is_compiled_with_mlu(), \ - "Not compiled with MLU" + assert core.is_compiled_with_mlu(), "Not compiled with MLU" if device_ids is None: device_ids = _mlu_ids() elif not isinstance(device_ids, (list, tuple)): @@ -1059,7 +1079,6 @@ def mlu_places(device_ids=None): class NameScope(object): - def __init__(self, name="", parent=None): self._children = dict() self._name = name @@ -1070,8 +1089,9 @@ class NameScope(object): new_child = NameScope(prefix, self) self._children[prefix] = [new_child] else: - new_child = NameScope(prefix + "_%d" % len(self._children[prefix]), - self) + new_child = NameScope( + prefix + "_%d" % len(self._children[prefix]), self + ) self._children[prefix].append(new_child) return new_child @@ -1161,6 +1181,7 @@ def _full_name_scope(): def generate_control_dev_var_name(): import random + return CONTROL_DEP_VAR_PREFIX + "@" + str(random.random()) @@ -1234,8 +1255,9 @@ def dtype_is_floating(dtype): dtype = convert_np_dtype_to_dtype_(dtype) return dtype in [ - core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP64 + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP64, ] @@ -1255,16 +1277,20 @@ def _debug_string_(proto, throw_on_error=True): if not proto.IsInitialized(error_fields) and throw_on_error: raise ValueError( "{0} are not initialized.\nThe message is {1}:\n".format( - error_fields, proto)) + error_fields, proto + ) + ) return proto.__str__() -def _varbase_creator(type=core.VarDesc.VarType.LOD_TENSOR, - name=None, - shape=None, - dtype=None, - persistable=None, - **kwargs): +def _varbase_creator( + type=core.VarDesc.VarType.LOD_TENSOR, + name=None, + shape=None, + dtype=None, + persistable=None, + **kwargs +): if dtype is not None: if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) @@ -1272,16 +1298,21 @@ def _varbase_creator(type=core.VarDesc.VarType.LOD_TENSOR, if _in_eager_mode_: eager_tensor = core.eager.Tensor( dtype if dtype else core.VarDesc.VarType.FP32, - list(shape) if shape else [], name, + list(shape) if shape else [], + name, type if type else core.VarDesc.VarType.LOD_TENSOR, - True if persistable else False) + True if persistable else False, + ) eager_tensor.retain_grads() return eager_tensor else: - return core.VarBase(dtype if dtype else core.VarDesc.VarType.FP32, - list(shape) if shape else [], name, - type if type else core.VarDesc.VarType.LOD_TENSOR, - True if persistable else False) + return core.VarBase( + dtype if dtype else core.VarDesc.VarType.FP32, + list(shape) if shape else [], + name, + type if type else core.VarDesc.VarType.LOD_TENSOR, + True if persistable else False, + ) def _all_is_type(vals, expected_type): @@ -1291,12 +1322,12 @@ def _all_is_type(vals, expected_type): NOTE: BuiltIn all() will always return True if vals is empty. """ assert isinstance(vals, (list, tuple)) - if not vals: return False + if not vals: + return False return all(isinstance(v, expected_type) for v in vals) class VariableMetaClass(type): - @classmethod def __instancecheck__(cls, instance): t = type(instance) @@ -1309,7 +1340,6 @@ class VariableMetaClass(type): class ParameterMetaClass(VariableMetaClass): - @classmethod def __instancecheck__(cls, instance): t = type(instance) @@ -1365,21 +1395,23 @@ class Variable(metaclass=VariableMetaClass): """ - def __init__(self, - block, - type=core.VarDesc.VarType.LOD_TENSOR, - name=None, - shape=None, - dtype=None, - lod_level=None, - capacity=None, - persistable=None, - error_clip=None, - stop_gradient=False, - is_data=False, - need_check_feed=False, - belong_to_optimizer=False, - **kwargs): + def __init__( + self, + block, + type=core.VarDesc.VarType.LOD_TENSOR, + name=None, + shape=None, + dtype=None, + lod_level=None, + capacity=None, + persistable=None, + error_clip=None, + stop_gradient=False, + is_data=False, + need_check_feed=False, + belong_to_optimizer=False, + **kwargs + ): self.block = block if name is None: name = unique_name.generate('_generated_var') @@ -1409,10 +1441,11 @@ class Variable(metaclass=VariableMetaClass): if is_new_var: self.desc.set_type(type) elif self.desc.type() != type: - raise ValueError("Variable '{0}' has been created before. The " - "previous type is {1}, the new type is {2}. They" - " are not matched".format(self.name, - self.desc.type(), type)) + raise ValueError( + "Variable '{0}' has been created before. The " + "previous type is {1}, the new type is {2}. They" + " are not matched".format(self.name, self.desc.type(), type) + ) if shape is not None: if is_new_var: @@ -1424,29 +1457,32 @@ class Variable(metaclass=VariableMetaClass): raise ValueError( "Variable '{0}' has been created before. The previous " "shape is {1}, the new shape is {2}. They are not " - "matched.".format(self.name, old_shape, shape)) + "matched.".format(self.name, old_shape, shape) + ) if dtype is not None: if is_new_var: self.desc.set_dtype(dtype) else: old_dtype = self.dtype if dtype != old_dtype: - raise ValueError("Variable '{0}' has been created before. " - "The previous data type is {1}, the new " - "data type is {2}. They are not " - "matched.".format(self.name, old_dtype, - dtype)) + raise ValueError( + "Variable '{0}' has been created before. " + "The previous data type is {1}, the new " + "data type is {2}. They are not " + "matched.".format(self.name, old_dtype, dtype) + ) if lod_level is not None: if is_new_var: self.desc.set_lod_level(lod_level) else: if lod_level != self.lod_level: - raise ValueError("Variable '{0}' has been created before. " - "The previous lod_level is {1}, the new " - "lod_level is {2}. They are not " - "matched".format(self.name, self.lod_level, - lod_level)) + raise ValueError( + "Variable '{0}' has been created before. " + "The previous lod_level is {1}, the new " + "lod_level is {2}. They are not " + "matched".format(self.name, self.lod_level, lod_level) + ) if persistable is not None: if is_new_var: self.desc.set_persistable(persistable) @@ -1456,7 +1492,9 @@ class Variable(metaclass=VariableMetaClass): "Variable '{0}' has been created before." "The previous persistable is {1}, the new " "persistable is {2}. They are not matched".format( - self.name, self.persistable, persistable)) + self.name, self.persistable, persistable + ) + ) if need_check_feed and is_new_var: self.desc.set_need_check_feed(need_check_feed) @@ -1497,20 +1535,22 @@ class Variable(metaclass=VariableMetaClass): y = x.detach() """ - assert self.type == core.VarDesc.VarType.SELECTED_ROWS or \ - self.type == core.VarDesc.VarType.LOD_TENSOR, \ - "only support a variable with SELECTED_ROWS or LOD_TENSOR to be detached" + assert ( + self.type == core.VarDesc.VarType.SELECTED_ROWS + or self.type == core.VarDesc.VarType.LOD_TENSOR + ), "only support a variable with SELECTED_ROWS or LOD_TENSOR to be detached" output = self.block.create_var( name=unique_name.generate_with_ignorable_key("detach_" + self.name), dtype=self.dtype, type=self.type, persistable=self.persistable, - stop_gradient=True) + stop_gradient=True, + ) - self.block.append_op(type='share_data', - inputs={'X': [self]}, - outputs={'Out': [output]}) + self.block.append_op( + type='share_data', inputs={'X': [self]}, outputs={'Out': [output]} + ) return output @fake_interface_only @@ -1700,14 +1740,20 @@ class Variable(metaclass=VariableMetaClass): """ # VarType.LOD_TENSOR -> LOD_TENSOR type_str = str(self.type).split('.')[1] - if self.type == core.VarDesc.VarType.SELECTED_ROWS or self.type == core.VarDesc.VarType.LOD_TENSOR: + if ( + self.type == core.VarDesc.VarType.SELECTED_ROWS + or self.type == core.VarDesc.VarType.LOD_TENSOR + ): dtype_str = str(self.dtype).split('.')[1] - var_str = "{name} : {type}.shape{shape}.dtype({dtype}).stop_gradient({stop_gradient})".\ - format(name=self.name, type=type_str, shape=self.shape, - dtype=dtype_str, stop_gradient=self.stop_gradient) + var_str = "{name} : {type}.shape{shape}.dtype({dtype}).stop_gradient({stop_gradient})".format( + name=self.name, + type=type_str, + shape=self.shape, + dtype=dtype_str, + stop_gradient=self.stop_gradient, + ) else: - var_str = "{name} : {type})".\ - format(name=self.name, type=type_str) + var_str = "{name} : {type})".format(name=self.name, type=type_str) if self.is_parameter: if self.trainable: @@ -1720,12 +1766,16 @@ class Variable(metaclass=VariableMetaClass): if self.persistable: var_str = "persist " + var_str - from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context + from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, + ) + dist_context = get_default_distributed_context() dist_tensor = dist_context.get_dist_tensor_for_program(self) if dist_tensor is not None: - var_str += ", {name} = {value}".format(name="dist_attr", - value=dist_tensor) + var_str += ", {name} = {value}".format( + name="dist_attr", value=dist_tensor + ) return var_str @@ -1759,12 +1809,13 @@ class Variable(metaclass=VariableMetaClass): print(new_variable.to_string(True, True)) """ assert isinstance(throw_on_error, bool) and isinstance( - with_details, bool) + with_details, bool + ) protostr = self.desc.serialize_to_string() proto = framework_pb2.VarDesc.FromString(bytes(protostr)) res_str = _debug_string_(proto, throw_on_error) if with_details: - additional_attr = ("error_clip", ) + additional_attr = ("error_clip",) for attr_name in additional_attr: res_str += "%s: %s\n" % (attr_name, getattr(self, attr_name)) @@ -2061,21 +2112,22 @@ class Variable(metaclass=VariableMetaClass): dtype=self.dtype, type=self.type, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) input_shape = self.block.create_var( name=unique_name.generate_with_ignorable_key(self.name + '.tmp'), dtype=self.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - - self.block.append_op(type='transpose2', - inputs={'X': [self]}, - outputs={ - 'Out': [out], - 'XShape': [input_shape] - }, - attrs={'axis': perm}) + stop_gradient=False, + ) + + self.block.append_op( + type='transpose2', + inputs={'X': [self]}, + outputs={'Out': [out], 'XShape': [input_shape]}, + attrs={'axis': perm}, + ) return out def clone(self): @@ -2106,11 +2158,12 @@ class Variable(metaclass=VariableMetaClass): dtype=self.dtype, type=self.type, persistable=self.persistable, - stop_gradient=self.stop_gradient) + stop_gradient=self.stop_gradient, + ) - self.block.append_op(type='assign', - inputs={'X': [self]}, - outputs={'Out': [output]}) + self.block.append_op( + type='assign', inputs={'X': [self]}, outputs={'Out': [output]} + ) return output def _set_error_clip(self, error_clip): @@ -2176,8 +2229,9 @@ class Variable(metaclass=VariableMetaClass): start = upper if step < 0 else lower else: start = slice.start - start = max(start + - length, lower) if start < 0 else min(start, upper) + start = ( + max(start + length, lower) if start < 0 else min(start, upper) + ) # Compute stop. if slice.stop is None: @@ -2223,11 +2277,15 @@ class Variable(metaclass=VariableMetaClass): for index, o in enumerate(item): if isinstance(o, int): start = int(o) - if (index > 0 and index >= self.shape[index]) \ - or (index < 0 and (index + self.shape[index]) < 0): + if (index > 0 and index >= self.shape[index]) or ( + index < 0 and (index + self.shape[index]) < 0 + ): raise IndexError("invalid index") - start = max(start + self.shape[index], 0) if start < 0 else min( - start, self.shape[index]) + start = ( + max(start + self.shape[index], 0) + if start < 0 + else min(start, self.shape[index]) + ) starts.append(start) ends.append(start + 1) elif isinstance(o, slice): @@ -2245,30 +2303,31 @@ class Variable(metaclass=VariableMetaClass): if not copy: return self.block.create_var( name=unique_name.generate_with_ignorable_key(self.name), - dtype=self.dtype) + dtype=self.dtype, + ) else: return self def _sliceVar(self, axes, starts, ends): new_var = self._cloneVar() - self.block.append_op(type="slice", - inputs={'Input': [self]}, - outputs={'Out': [new_var]}, - attrs={ - 'axes': axes, - 'starts': starts, - 'ends': ends - }) + self.block.append_op( + type="slice", + inputs={'Input': [self]}, + outputs={'Out': [new_var]}, + attrs={'axes': axes, 'starts': starts, 'ends': ends}, + ) return new_var def _concatVar(self, inputs, axis): new_var = self._cloneVar() - self.block.append_op(type="concat", - inputs={'X': inputs}, - outputs={'Out': [new_var]}, - attrs={ - 'axis': axis, - }) + self.block.append_op( + type="concat", + inputs={'X': inputs}, + outputs={'Out': [new_var]}, + attrs={ + 'axis': axis, + }, + ) return new_var def _sliceAndConcatVar(self, item, axis): @@ -2282,21 +2341,24 @@ class Variable(metaclass=VariableMetaClass): vars = [] if step > 0: while start < stop: - vars.append(self._sliceVar([axis], [start], - [start + 1])) + vars.append( + self._sliceVar([axis], [start], [start + 1]) + ) start += step else: while start > stop: - vars.append(self._sliceVar([axis], [start], - [start + 1])) + vars.append( + self._sliceVar([axis], [start], [start + 1]) + ) start += step return self._concatVar(vars, axis) elif isinstance(item, int): if self.shape[axis] < 0: return self._cloneVar(True) index = int(item) - if (index > 0 and index >= self.shape[axis]) \ - or (index < 0 and (index + self.shape[axis]) < 0): + if (index > 0 and index >= self.shape[axis]) or ( + index < 0 and (index + self.shape[axis]) < 0 + ): raise IndexError("invalid index") return self._sliceVar([axis], [index], [index + 1]) else: @@ -2353,17 +2415,21 @@ class Variable(metaclass=VariableMetaClass): # can not be imported at the begainning of this file. # Therefore, the above two modules are dynamically imported. from .executor import global_scope + if scope is not None and not isinstance(scope, core._Scope): raise TypeError( - "`scope` should be None or `paddle.static.Scope` type, but received {}." - .format(type(scope))) + "`scope` should be None or `paddle.static.Scope` type, but received {}.".format( + type(scope) + ) + ) if scope is None: scope = global_scope() var_temp = scope.find_var(self.name) if var_temp is None: - raise ValueError("Can not find Variable '{}' in the Scope.".format( - self.name)) + raise ValueError( + "Can not find Variable '{}' in the Scope.".format(self.name) + ) t = var_temp.get_tensor() return t @@ -2417,21 +2483,26 @@ class Variable(metaclass=VariableMetaClass): if not (isinstance(value, np.ndarray) or hasattr(value, '__array__')): raise TypeError( - "`value` should be `numpy.ndarray` or `LoDTensor`, but received {}." - .format(type(value))) + "`value` should be `numpy.ndarray` or `LoDTensor`, but received {}.".format( + type(value) + ) + ) if scope is not None and not isinstance(scope, core._Scope): raise TypeError( - "`scope` should be None or `paddle.static.Scope` type, but received {}." - .format(type(scope))) + "`scope` should be None or `paddle.static.Scope` type, but received {}.".format( + type(scope) + ) + ) if scope is None: scope = global_scope() var_temp = scope.find_var(self.name) if var_temp is None: - raise ValueError("Can not find Variable '{}' in the Scope.".format( - self.name)) + raise ValueError( + "Can not find Variable '{}' in the Scope.".format(self.name) + ) t = var_temp.get_tensor() @@ -2442,8 +2513,10 @@ class Variable(metaclass=VariableMetaClass): value_shape = value.shape if list(t.shape()) != list(value_shape): raise ValueError( - "{} expected a shape {}, but the received shape is {}.". - format(self.name, list(t.shape()), list(value_shape))) + "{} expected a shape {}, but the received shape is {}.".format( + self.name, list(t.shape()), list(value_shape) + ) + ) p = t._place() if p.is_cpu_place(): @@ -2492,11 +2565,12 @@ class Variable(metaclass=VariableMetaClass): output = self.block.create_var( name=unique_name.generate_with_ignorable_key(self.name + "_size"), - dtype=core.VarDesc.VarType.INT64) + dtype=core.VarDesc.VarType.INT64, + ) - self.block.append_op(type='size', - inputs={'Input': [self]}, - outputs={'Out': [output]}) + self.block.append_op( + type='size', inputs={'Input': [self]}, outputs={'Out': [output]} + ) return output def _set_attr(self, name, val): @@ -2595,8 +2669,8 @@ class OpProtoHolder(object): def __init__(self): assert not hasattr( - self.__class__, - '_instance'), 'Please use `instance()` to get OpProtoHolder object!' + self.__class__, '_instance' + ), 'Please use `instance()` to get OpProtoHolder object!' op_protos = get_all_op_protos() self.op_proto_map = {} for proto in op_protos: @@ -2632,7 +2706,7 @@ class OpProtoHolder(object): core.op_proto_and_checker_maker.kOpRoleVarAttrName(), core.op_proto_and_checker_maker.kOpNameScopeAttrName(), core.op_proto_and_checker_maker.kOpCreationCallstackAttrName(), - core.op_proto_and_checker_maker.kOpDeviceAttrName() + core.op_proto_and_checker_maker.kOpDeviceAttrName(), } @@ -2679,24 +2753,44 @@ class Operator(object): inputs={"X": [var1, var2, var3]}, outputs={"Out": [var1]}) """ + OP_WITHOUT_KERNEL_SET = { - 'feed', 'fetch', 'recurrent', 'go', 'rnn_memory_helper_grad', - 'conditional_block', 'while', 'send', 'recv', 'listen_and_serv', - 'fl_listen_and_serv', 'ncclInit', 'select', 'checkpoint_notify', - 'gen_bkcl_id', 'c_gen_bkcl_id', 'gen_nccl_id', 'c_gen_nccl_id', - 'c_comm_init', 'c_sync_calc_stream', 'c_sync_comm_stream', - 'queue_generator', 'dequeue', 'enqueue', 'heter_listen_and_serv', - 'c_wait_comm', 'c_wait_compute', 'c_gen_hccl_id', 'c_comm_init_hccl', - 'copy_cross_scope', 'c_gen_cncl_id' + 'feed', + 'fetch', + 'recurrent', + 'go', + 'rnn_memory_helper_grad', + 'conditional_block', + 'while', + 'send', + 'recv', + 'listen_and_serv', + 'fl_listen_and_serv', + 'ncclInit', + 'select', + 'checkpoint_notify', + 'gen_bkcl_id', + 'c_gen_bkcl_id', + 'gen_nccl_id', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_sync_calc_stream', + 'c_sync_comm_stream', + 'queue_generator', + 'dequeue', + 'enqueue', + 'heter_listen_and_serv', + 'c_wait_comm', + 'c_wait_compute', + 'c_gen_hccl_id', + 'c_comm_init_hccl', + 'copy_cross_scope', + 'c_gen_cncl_id', } - def __init__(self, - block, - desc, - type=None, - inputs=None, - outputs=None, - attrs=None): + def __init__( + self, block, desc, type=None, inputs=None, outputs=None, attrs=None + ): # read attr type index from op proto to avoid unexpected type # conversions, e.g. narrowing conversion like double to float try: @@ -2710,7 +2804,8 @@ class Operator(object): if _non_static_mode(): if type is None: raise ValueError( - "`type` to initialized an Operator can not be None.") + "`type` to initialized an Operator can not be None." + ) self._type = type self.attrs = attrs if attrs else {} else: @@ -2730,11 +2825,14 @@ class Operator(object): if op_maker.kOpRoleAttrName() not in op_attrs: op_attrs[ - op_maker.kOpRoleAttrName()] = self.block.program._op_role + op_maker.kOpRoleAttrName() + ] = self.block.program._op_role role_var_name = op_maker.kOpRoleVarAttrName() - if len(self.block.program._op_role_var - ) != 0 and role_var_name not in op_attrs: + if ( + len(self.block.program._op_role_var) != 0 + and role_var_name not in op_attrs + ): op_attrs[role_var_name] = self.block.program._op_role_var if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0: @@ -2749,16 +2847,20 @@ class Operator(object): return if type is None: raise ValueError( - "`type` to initialized an Operator can not be None.") + "`type` to initialized an Operator can not be None." + ) else: callstack_var_name = op_maker.kOpCreationCallstackAttrName() op_attrs[callstack_var_name] = [] for frame in traceback.extract_stack(): op_attrs[callstack_var_name].append( ' File "{}", line {}, in {}'.format( - frame[0], frame[1], frame[2])) - op_attrs[callstack_var_name].append(' {}'.format( - frame[3])) + frame[0], frame[1], frame[2] + ) + ) + op_attrs[callstack_var_name].append( + ' {}'.format(frame[3]) + ) self.desc.set_type(type) proto = OpProtoHolder.instance().get_op_proto(type) @@ -2774,20 +2876,25 @@ class Operator(object): op_device = op_maker.kOpDeviceAttrName() op_attrs[op_device] = _current_device else: - warnings.warn("The Op(%s) is not support to set device." % - type) + warnings.warn( + "The Op(%s) is not support to set device." % type + ) if 'force_cpu' in op_attrs: - if (type == 'less_than' and op_attrs['force_cpu'] != None - ) or op_attrs['force_cpu'] != False: + if ( + type == 'less_than' and op_attrs['force_cpu'] != None + ) or op_attrs['force_cpu'] != False: warnings.warn( "The Attr(force_cpu) of Op(%s) will be deprecated in the future, " "please use 'device_guard' instead. 'device_guard' has higher priority when they are " - "used at the same time." % type) + "used at the same time." % type + ) if _current_pipeline_stage is not None: - pipeline_attr_name = 'pipeline_stage' + core.kAutoParallelSuffix( + pipeline_attr_name = ( + 'pipeline_stage' + core.kAutoParallelSuffix() + ) + self._update_desc_attr( + pipeline_attr_name, _current_pipeline_stage ) - self._update_desc_attr(pipeline_attr_name, - _current_pipeline_stage) def find_name(var_list, name): for var_name in var_list: @@ -2798,8 +2905,9 @@ class Operator(object): if inputs is not None: for in_proto in proto.inputs: found = find_name(inputs, in_proto.name) - assert found or in_proto.dispensable, "Input {} not found".format( - in_proto.name) + assert ( + found or in_proto.dispensable + ), "Input {} not found".format(in_proto.name) if found: in_args = inputs[in_proto.name] if not isinstance(in_args, (list, tuple)): @@ -2807,7 +2915,8 @@ class Operator(object): if not in_proto.duplicable and len(in_args) > 1: raise ValueError( "Input %s expects only one input, but %d are given." - % (in_proto.name, len(in_args))) + % (in_proto.name, len(in_args)) + ) in_arg_names = [] for index, arg in enumerate(in_args): if isinstance(arg, str): @@ -2821,8 +2930,9 @@ class Operator(object): "The type of '%s' in operator %s should be " "one of [basestring(), str, Varibale] in python2, " "or one of [str, bytes, Variable] in python3." - "but received : %s" % - (in_proto.name, type, arg)) + "but received : %s" + % (in_proto.name, type, arg) + ) self.desc.set_input(in_proto.name, in_arg_names) else: self.desc.set_input(in_proto.name, []) @@ -2833,9 +2943,12 @@ class Operator(object): continue if not ((m.name in outputs) or m.dispensable): raise ValueError( - ("Incorrect setting for output(s) of " - "operator \"%s\", should set: [%s].") % - (type, m.name)) + ( + "Incorrect setting for output(s) of " + "operator \"%s\", should set: [%s]." + ) + % (type, m.name) + ) for out_proto in proto.outputs: if out_proto.name not in outputs: continue @@ -2845,7 +2958,8 @@ class Operator(object): if not out_proto.duplicable and len(out_args) > 1: raise ValueError( "Output %s expects only one output, but %d are given." - % (out_proto.name, len(out_args))) + % (out_proto.name, len(out_args)) + ) out_arg_names = [] for arg in out_args: if isinstance(arg, str): @@ -2866,27 +2980,32 @@ class Operator(object): raise TypeError("'attrs' should be a dict.") for attr in proto.attrs: attr_name = attr.name - if (attr_name - not in op_attrs) or (op_attrs[attr_name] is None): + if (attr_name not in op_attrs) or ( + op_attrs[attr_name] is None + ): continue attr_val = op_attrs[attr_name] self._update_desc_attr(attr_name, attr_val) for attr_name in extra_attrs_map.keys(): - if (attr_name - not in op_attrs) or (op_attrs[attr_name] is None): - self._update_desc_attr(attr_name, - extra_attrs_map[attr_name]) + if (attr_name not in op_attrs) or ( + op_attrs[attr_name] is None + ): + self._update_desc_attr( + attr_name, extra_attrs_map[attr_name] + ) else: self._update_desc_attr(attr_name, op_attrs[attr_name]) # proto.attrs doesn't include ipu_index if core.is_compiled_with_ipu(): if global_ipu_index >= 0: - self._update_desc_attr(ipu_index_attr_name, - global_ipu_index) + self._update_desc_attr( + ipu_index_attr_name, global_ipu_index + ) if global_ipu_stage >= 0: - self._update_desc_attr(ipu_stage_attr_name, - global_ipu_stage) + self._update_desc_attr( + ipu_stage_attr_name, global_ipu_stage + ) self.desc.check_attrs() if self._has_kernel(type): @@ -2945,7 +3064,8 @@ class Operator(object): assert isinstance( skip_op_callstack, bool ), "skip_op_callstack parameter's type is error, expect bool, received {}".format( - type(skip_op_callstack)) + type(skip_op_callstack) + ) outputs_str = "{" for i in range(0, len(self.output_names)): outputs_str += "{name}=".format(name=self.output_names[i]) @@ -2975,9 +3095,9 @@ class Operator(object): attr_type = self.desc.attr_type(name, True) if attr_type == core.AttrType.VAR: attr_var_name = self.desc.attr(name, True).name() - a = "{name} = Var['{value}']".format(name=name, - type=attr_type, - value=attr_var_name) + a = "{name} = Var['{value}']".format( + name=name, type=attr_type, value=attr_var_name + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " @@ -2988,7 +3108,8 @@ class Operator(object): "'%s'" % var.name() for var in self.desc.attr(name, True) ] a = "{name} = Vars[{value}]".format( - name=name, type=attr_type, value=','.join(attr_var_names)) + name=name, type=attr_type, value=','.join(attr_var_names) + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " @@ -2996,7 +3117,8 @@ class Operator(object): if attr_type == core.AttrType.BLOCK: a = "{name} = block[{value}]".format( - name=name, type=attr_type, value=self._block_attr_id(name)) + name=name, type=attr_type, value=self._block_attr_id(name) + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " @@ -3004,17 +3126,19 @@ class Operator(object): if attr_type == core.AttrType.BLOCKS: a = "{name} = blocks{value}".format( - name=name, - type=attr_type, - value=self._blocks_attr_ids(name)) + name=name, type=attr_type, value=self._blocks_attr_ids(name) + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " continue # it is bytes of serialized protobuf - if is_compiled_with_cinn( - ) and self.type == 'cinn_launch' and name == 'compilation_key': + if ( + is_compiled_with_cinn() + and self.type == 'cinn_launch' + and name == 'compilation_key' + ): key = self.desc.attr(name) v = core.get_serialize_comile_key(key) prog = Program() @@ -3026,28 +3150,36 @@ class Operator(object): else: value = self.desc.attr(name) - a = "{name} = {value}".format(name=name, - type=attr_type, - value=value) + a = "{name} = {value}".format( + name=name, type=attr_type, value=value + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " - from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context + from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, + ) + dist_context = get_default_distributed_context() dist_op = dist_context.get_dist_op_for_program(self) if dist_op is not None: - attrs_str += ", {name} = {value}".format(name="dist_attr", - value=dist_op) + attrs_str += ", {name} = {value}".format( + name="dist_attr", value=dist_op + ) if outputs_str != "{}": - op_str = "{outputs} = {op_type}(inputs={inputs}, {attrs})".\ - format(outputs=outputs_str, op_type=self.type, - inputs=inputs_str, attrs=attrs_str) + op_str = "{outputs} = {op_type}(inputs={inputs}, {attrs})".format( + outputs=outputs_str, + op_type=self.type, + inputs=inputs_str, + attrs=attrs_str, + ) else: - op_str = "{op_type}(inputs={inputs}, {attrs})".\ - format(op_type=self.type, inputs=inputs_str, attrs=attrs_str) + op_str = "{op_type}(inputs={inputs}, {attrs})".format( + op_type=self.type, inputs=inputs_str, attrs=attrs_str + ) return op_str def __str__(self): @@ -3133,7 +3265,8 @@ class Operator(object): if op == self: return i raise ValueError( - "Can't find op itself in it's block. It could be a bug of Paddle.") + "Can't find op itself in it's block. It could be a bug of Paddle." + ) def has_attr(self, name): """ @@ -3195,8 +3328,9 @@ class Operator(object): self.desc.set_block_attr(name, val.desc) elif isinstance(val, list) and val and _all_is_type(val, Block): self.desc.set_blocks_attr(name, [v.desc for v in val]) - elif isinstance(val, core.BlockDesc) or \ - isinstance(val, core.ProgramDesc): + elif isinstance(val, core.BlockDesc) or isinstance( + val, core.ProgramDesc + ): self.desc.set_serialized_attr(name, val.serialize_to_string()) else: self._update_desc_plain_attr(name, val) @@ -3277,7 +3411,7 @@ class Operator(object): """ id = self._block_attr_id(name) - assert (id >= 0 and id < len(self.block.program.blocks)) + assert id >= 0 and id < len(self.block.program.blocks) return self.block.program.blocks[id] def _blocks_attr(self, name): @@ -3292,7 +3426,7 @@ class Operator(object): """ attrs = [] for i in self._blocks_attr_ids(name): - assert (i >= 0 and i < len(self.block.program.blocks)) + assert i >= 0 and i < len(self.block.program.blocks) attrs.append(self.block.program.blocks[i]) return attrs @@ -3321,8 +3455,11 @@ class Operator(object): Variable: the Variable attribute. """ attr_type = self.desc.attr_type(name, True) - assert attr_type == core.AttrType.VAR, "Required type attr({}) is Variable, but received {}".format( - name, attr_type) + assert ( + attr_type == core.AttrType.VAR + ), "Required type attr({}) is Variable, but received {}".format( + name, attr_type + ) attr_var_name = self.desc.attr(name, True).name() return self.block._var_recursive(attr_var_name) @@ -3337,8 +3474,11 @@ class Operator(object): Variables: the Variables attribute. """ attr_type = self.desc.attr_type(name, True) - assert attr_type == core.AttrType.VARS, "Required type attr({}) is list[Variable], but received {}".format( - name, attr_type) + assert ( + attr_type == core.AttrType.VARS + ), "Required type attr({}) is list[Variable], but received {}".format( + name, attr_type + ) attr_vars = [ self.block._var_recursive(var.name()) for var in self.desc.attr(name, True) @@ -3485,7 +3625,8 @@ class Block(object): assert isinstance( skip_op_callstack, bool ), "skip_op_callstack parameter's type is error, expect bool, received {}".format( - type(skip_op_callstack)) + type(skip_op_callstack) + ) block_str = "{ // block " block_str += "{}\n".format(self.idx) for var in list(self.vars.values()): @@ -3493,7 +3634,8 @@ class Block(object): block_str += "\n" for op in self.ops: block_str += " {}\n".format( - op._to_readable_code(skip_op_callstack)) + op._to_readable_code(skip_op_callstack) + ) block_str += "}" return block_str @@ -3512,17 +3654,22 @@ class Block(object): str: The debug string. """ assert isinstance(throw_on_error, bool) and isinstance( - with_details, bool) + with_details, bool + ) if with_details: re_add_indent = re.compile(r"\n(.)") res_str = "blocks {\n idx: %d\n parent_idx: %d" % ( - self.idx, self.parent_idx) + self.idx, + self.parent_idx, + ) for var in list(self.vars.values()): res_str += "\n vars {\n %s }" % re_add_indent.sub( - r"\n \1", var.to_string(throw_on_error, with_details)) + r"\n \1", var.to_string(throw_on_error, with_details) + ) for op in self.ops: res_str += "\n ops {\n %s }" % re_add_indent.sub( - r"\n \1", op.to_string(throw_on_error)) + r"\n \1", op.to_string(throw_on_error) + ) res_str += "\n}" else: protostr = self.desc.serialize_to_string() @@ -3580,8 +3727,9 @@ class Block(object): """ if not isinstance(name, str): raise TypeError( - "var require string as parameter, but get %s instead." % - (type(name))) + "var require string as parameter, but get %s instead." + % (type(name)) + ) v = self.vars.get(name, None) if v is None: raise ValueError("var %s not in this block" % name) @@ -3647,8 +3795,11 @@ class Block(object): return list(self.iter_parameters()) def iter_parameters(self): - return (item[1] for item in self.vars.items() - if isinstance(item[1], Parameter)) + return ( + item[1] + for item in self.vars.items() + if isinstance(item[1], Parameter) + ) def create_var(self, *args, **kwargs): if _non_static_mode(): @@ -3680,8 +3831,9 @@ class Block(object): """ # Ensure the type of name and new_name is str name = name.decode() if isinstance(name, bytes) else name - new_name = new_name.decode() if isinstance(new_name, - bytes) else new_name + new_name = ( + new_name.decode() if isinstance(new_name, bytes) else new_name + ) if not self.has_var(name): raise ValueError("var %s is not in current block" % name) @@ -3705,43 +3857,51 @@ class Block(object): d = self.desc.find_var(new_name.encode()) if var_type == "Parameter": if in_dygraph_mode(): - var = EagerParamBase(d.shape(), - d.dtype(), - type=orig_var_type, - name=new_name, - stop_gradient=stop_gradient, - trainable=trainable, - optimize_attr=optimize_attr, - regularizer=regularizer, - error_clip=error_clip) + var = EagerParamBase( + d.shape(), + d.dtype(), + type=orig_var_type, + name=new_name, + stop_gradient=stop_gradient, + trainable=trainable, + optimize_attr=optimize_attr, + regularizer=regularizer, + error_clip=error_clip, + ) else: if _in_legacy_dygraph(): - var = ParamBase(d.shape(), - d.dtype(), - type=orig_var_type, - name=new_name, - stop_gradient=stop_gradient, - trainable=trainable, - optimize_attr=optimize_attr, - regularizer=regularizer, - error_clip=error_clip) + var = ParamBase( + d.shape(), + d.dtype(), + type=orig_var_type, + name=new_name, + stop_gradient=stop_gradient, + trainable=trainable, + optimize_attr=optimize_attr, + regularizer=regularizer, + error_clip=error_clip, + ) else: - var = Parameter(self, - d.shape(), - d.dtype(), - type=orig_var_type, - name=new_name, - stop_gradient=stop_gradient, - trainable=trainable, - optimize_attr=optimize_attr, - regularizer=regularizer, - error_clip=error_clip) + var = Parameter( + self, + d.shape(), + d.dtype(), + type=orig_var_type, + name=new_name, + stop_gradient=stop_gradient, + trainable=trainable, + optimize_attr=optimize_attr, + regularizer=regularizer, + error_clip=error_clip, + ) elif var_type == "Variable": - var = Variable(self, - type=orig_var_type, - name=new_name, - error_clip=error_clip, - stop_gradient=stop_gradient) + var = Variable( + self, + type=orig_var_type, + name=new_name, + error_clip=error_clip, + stop_gradient=stop_gradient, + ) # rename the python side, _sync_with_cpp will only add # new vars/ops to python side. @@ -3778,8 +3938,9 @@ class Block(object): # Think of "c_broadcast" and "c_sync_comm_stream" as a special case here. # NOTE: "coalesce_tensor" is a special case for rnn with cudnn support if op.type in [ - "c_broadcast", "c_sync_comm_stream", - "coalesce_tensor" + "c_broadcast", + "c_sync_comm_stream", + "coalesce_tensor", ]: continue init_ops.append(op) @@ -3789,9 +3950,12 @@ class Block(object): init_ops = _is_inited_by(global_block, param) init_ops_len = len(init_ops) if init_ops_len > 1: - raise RuntimeError("param " + param.name + - " is inited by multiple init ops " + - str(init_ops)) + raise RuntimeError( + "param " + + param.name + + " is inited by multiple init ops " + + str(init_ops) + ) elif init_ops_len == 1: # TODO already inited, do nothing, should log a warning pass @@ -3813,24 +3977,31 @@ class Block(object): warnings.warn( "Op `%s` is executed through `append_op` under the dynamic mode, " "the corresponding API implementation needs to be upgraded to " - "using `_C_ops` method." % type, DeprecationWarning) - op = Operator(block=self, - desc=None, - type=type, - inputs=None, - outputs=None, - attrs=attrs) + "using `_C_ops` method." % type, + DeprecationWarning, + ) + op = Operator( + block=self, + desc=None, + type=type, + inputs=None, + outputs=None, + attrs=attrs, + ) # record ops in tracer rather than blocks # # TODO(minqiyang): add op stop_gradient support in static mode too. # currently, we only support stop_gradient in dygraph mode. - _dygraph_tracer().trace_op(type, kwargs.get("inputs", {}), - kwargs.get("outputs", - {}), attrs if attrs else {}, - kwargs.get("stop_gradient", False), - inplace_map) + _dygraph_tracer().trace_op( + type, + kwargs.get("inputs", {}), + kwargs.get("outputs", {}), + attrs if attrs else {}, + kwargs.get("stop_gradient", False), + inplace_map, + ) else: from paddle.fluid.dygraph.base import param_guard @@ -3841,12 +4012,14 @@ class Block(object): inputs = kwargs.get("inputs", None) outputs = kwargs.get("outputs", None) with param_guard(inputs), param_guard(outputs): - op = Operator(block=self, - desc=op_desc, - type=kwargs.get("type", None), - inputs=inputs, - outputs=outputs, - attrs=kwargs.get("attrs", None)) + op = Operator( + block=self, + desc=op_desc, + type=kwargs.get("type", None), + inputs=inputs, + outputs=outputs, + attrs=kwargs.get("attrs", None), + ) self.ops.append(op) @@ -3913,25 +4086,27 @@ class Block(object): if _non_static_mode(): type = kwargs.get("type", None) attrs = kwargs.get("attrs", {}) - op = Operator(self, - None, - type=type, - inputs=None, - outputs=None, - attrs=attrs) - - _dygraph_tracer().trace_op(type, kwargs.get("inputs", {}), - kwargs.get("outputs", {}), - attrs if attrs else {}, - kwargs.get("stop_gradient", False)) + op = Operator( + self, None, type=type, inputs=None, outputs=None, attrs=attrs + ) + + _dygraph_tracer().trace_op( + type, + kwargs.get("inputs", {}), + kwargs.get("outputs", {}), + attrs if attrs else {}, + kwargs.get("stop_gradient", False), + ) else: op_desc = self.desc._prepend_op() - op = Operator(self, - op_desc, - type=kwargs.get("type", None), - inputs=kwargs.get("inputs", None), - outputs=kwargs.get("outputs", None), - attrs=kwargs.get("attrs", None)) + op = Operator( + self, + op_desc, + type=kwargs.get("type", None), + inputs=kwargs.get("inputs", None), + outputs=kwargs.get("outputs", None), + attrs=kwargs.get("attrs", None), + ) self.ops.insert(0, op) return op @@ -3948,17 +4123,21 @@ class Block(object): if var.has_stop_gradient(): is_stop_gradient = var.stop_gradient() if var.has_is_parameter() and var.is_parameter(): - self.create_parameter(name=var.name(), - desc=var, - type=var.type(), - shape=var.shape(), - dtype=var.dtype(), - stop_gradient=is_stop_gradient) + self.create_parameter( + name=var.name(), + desc=var, + type=var.type(), + shape=var.shape(), + dtype=var.dtype(), + stop_gradient=is_stop_gradient, + ) else: - self.create_var(name=var.name(), - desc=var, - type=var.type(), - stop_gradient=is_stop_gradient) + self.create_var( + name=var.name(), + desc=var, + type=var.type(), + stop_gradient=is_stop_gradient, + ) # sync variables removed from c++ end for var in list(self.vars.keys()): @@ -4004,9 +4183,12 @@ class Block(object): ops_in_cpp_index = 0 ops_in_python_index = 0 while ops_in_python_index < len( - self.ops) and ops_in_cpp_index < len(ops_in_cpp): - if self.ops[ops_in_python_index].desc != ops_in_cpp[ - ops_in_cpp_index]: + self.ops + ) and ops_in_cpp_index < len(ops_in_cpp): + if ( + self.ops[ops_in_python_index].desc + != ops_in_cpp[ops_in_cpp_index] + ): del self.ops[ops_in_python_index] else: ops_in_cpp_index += 1 @@ -4032,7 +4214,8 @@ class Block(object): """ if not isinstance(other, Block): raise TypeError( - "_copy_param_info_from should be invoked with Block") + "_copy_param_info_from should be invoked with Block" + ) for p in other.iter_parameters(): assert isinstance(p, Parameter) v = self.vars.get(p.name, None) @@ -4042,28 +4225,32 @@ class Block(object): assert isinstance(v, Variable) new_p = None if in_dygraph_mode(): - new_p = EagerParamBase(shape=v.shape, - dtype=v.dtype, - type=v.type, - lod_level=v.lod_level, - stop_gradient=p.stop_gradient, - trainable=p.trainable, - optimize_attr=p.optimize_attr, - regularizer=p.regularizer, - error_clip=p.error_clip, - name=v.name) + new_p = EagerParamBase( + shape=v.shape, + dtype=v.dtype, + type=v.type, + lod_level=v.lod_level, + stop_gradient=p.stop_gradient, + trainable=p.trainable, + optimize_attr=p.optimize_attr, + regularizer=p.regularizer, + error_clip=p.error_clip, + name=v.name, + ) else: if _in_legacy_dygraph(): - new_p = ParamBase(shape=v.shape, - dtype=v.dtype, - type=v.type, - lod_level=v.lod_level, - stop_gradient=p.stop_gradient, - trainable=p.trainable, - optimize_attr=p.optimize_attr, - regularizer=p.regularizer, - error_clip=p.error_clip, - name=v.name) + new_p = ParamBase( + shape=v.shape, + dtype=v.dtype, + type=v.type, + lod_level=v.lod_level, + stop_gradient=p.stop_gradient, + trainable=p.trainable, + optimize_attr=p.optimize_attr, + regularizer=p.regularizer, + error_clip=p.error_clip, + name=v.name, + ) else: new_p = Parameter( block=self, @@ -4071,13 +4258,15 @@ class Block(object): dtype=v.dtype, type=v.type, lod_level=v.lod_level - if v.type == core.VarDesc.VarType.LOD_TENSOR else None, + if v.type == core.VarDesc.VarType.LOD_TENSOR + else None, stop_gradient=p.stop_gradient, trainable=p.trainable, optimize_attr=p.optimize_attr, regularizer=p.regularizer, error_clip=p.error_clip, - name=v.name) + name=v.name, + ) self.vars[new_p.name] = new_p def _clone_variable(self, var, force_persistable=True): @@ -4097,13 +4286,13 @@ class Block(object): ret_var = None # make STEP_SCOPES var can be safely cloned. if var.type == core.VarDesc.VarType.STEP_SCOPES: - ret_var = self.create_var(name=var.name, - persistable=var.persistable, - type=var.type) + ret_var = self.create_var( + name=var.name, persistable=var.persistable, type=var.type + ) elif var.type == core.VarDesc.VarType.RAW: - ret_var = self.create_var(name=var.name, - persistable=var.persistable, - type=var.type) + ret_var = self.create_var( + name=var.name, persistable=var.persistable, type=var.type + ) elif var.type == core.VarDesc.VarType.SELECTED_ROWS: ret_var = self.create_var( name=var.name, @@ -4112,7 +4301,8 @@ class Block(object): type=var.type, persistable=True if force_persistable else var.persistable, is_data=var.is_data, - need_check_feed=var.desc.need_check_feed()) + need_check_feed=var.desc.need_check_feed(), + ) else: ret_var = self.create_var( name=var.name, @@ -4122,7 +4312,8 @@ class Block(object): lod_level=var.lod_level, persistable=True if force_persistable else var.persistable, is_data=var.is_data, - need_check_feed=var.desc.need_check_feed()) + need_check_feed=var.desc.need_check_feed(), + ) return ret_var @@ -4132,17 +4323,20 @@ class Block(object): # re-constructed inside this method. The underlying VarDesc(OpDesc) # of some old Python Variables(all old Python Operators) may have # been destructed. -def _apply_pass(main_program, - startup_program, - pass_name, - pass_attrs={}, - pass_attr_types={}): +def _apply_pass( + main_program, startup_program, pass_name, pass_attrs={}, pass_attr_types={} +): assert isinstance(pass_attrs, dict), "pass_attrs must be dict" assert isinstance(pass_attr_types, dict), "pass_attr_types must be dict" tmp_main_program = core.ProgramDesc(main_program.desc) tmp_startup_program = core.ProgramDesc(startup_program.desc) - attrs = core.apply_pass(tmp_main_program, tmp_startup_program, pass_name, - pass_attrs, pass_attr_types) + attrs = core.apply_pass( + tmp_main_program, + tmp_startup_program, + pass_name, + pass_attrs, + pass_attr_types, + ) main_program._rebuild_from_desc(tmp_main_program) startup_program._rebuild_from_desc(tmp_startup_program) return attrs @@ -4160,8 +4354,9 @@ class IrNode(object): Args: node(core.Node): C++ Node. """ - assert isinstance(node, - core.Node), 'node must be the instance of core.Node.' + assert isinstance( + node, core.Node + ), 'node must be the instance of core.Node.' self.node = node def name(self): @@ -4337,8 +4532,9 @@ class IrVarNode(IrNode): Args: node(core.Node): C++ Node. """ - assert isinstance(node, core.Node) and node.is_var(), \ - 'node must be the instance of core.Node and it must be a variable node.' + assert ( + isinstance(node, core.Node) and node.is_var() + ), 'node must be the instance of core.Node and it must be a variable node.' super(IrVarNode, self).__init__(node) self.node = node @@ -4349,8 +4545,9 @@ class IrVarNode(IrNode): Args: shape(list): shape to be set. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." self.node.var().set_shape(shape) def persistable(self): @@ -4360,8 +4557,9 @@ class IrVarNode(IrNode): Returns: bool: indicate whether the variable is persistable. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." return self.node.var().persistable() def type(self): @@ -4371,8 +4569,9 @@ class IrVarNode(IrNode): Returns: core.VarDesc.VarType: the variable type. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." return self.node.var().type() def dtype(self): @@ -4382,8 +4581,9 @@ class IrVarNode(IrNode): Returns: core.VarDesc.VarType: the variable data type. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." return self.node.var().dtype() def shape(self): @@ -4393,8 +4593,9 @@ class IrVarNode(IrNode): Returns: list: the variable shape. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." return self.node.var().shape() @property @@ -4430,8 +4631,9 @@ class IrOpNode(IrNode): Args: node(core.Node): C++ Node. """ - assert isinstance(node, core.Node) and node.is_op(), \ - 'node must be the instance of core.Node and it must be a operator node.' + assert ( + isinstance(node, core.Node) and node.is_op() + ), 'node must be the instance of core.Node and it must be a operator node.' super(IrOpNode, self).__init__(node) self.node = node @@ -4443,8 +4645,9 @@ class IrOpNode(IrNode): old_input_name(str): the old input name. new_input_name(str): the new input name. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." self.node.op()._rename_input(old_input_name, new_input_name) def rename_output(self, old_output_name, new_output_name): @@ -4455,8 +4658,9 @@ class IrOpNode(IrNode): old_output_name(str): the old output name. new_output_name(str): the new output name. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." self.node.op()._rename_output(old_output_name, new_output_name) def input(self, name): @@ -4469,8 +4673,9 @@ class IrOpNode(IrNode): Returns: list(str): the argument name list. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().input(name) def output(self, name): @@ -4483,8 +4688,9 @@ class IrOpNode(IrNode): Returns: list(str): the argument name list. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().output(name) def set_type(self, new_type): @@ -4494,8 +4700,9 @@ class IrOpNode(IrNode): Args: new_type(str): new operator type to be set. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().set_type(new_type) def set_attr(self, name, val): @@ -4512,8 +4719,9 @@ class IrOpNode(IrNode): """ Update the value of the op desc's attribute by attribute's name. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." desc = self.node.op() if isinstance(val, Variable): desc.set_var_attr(name, val.desc) @@ -4523,8 +4731,9 @@ class IrOpNode(IrNode): desc.set_block_attr(name, val.desc) elif isinstance(val, list) and val and _all_is_type(val, Block): desc.set_blocks_attr(name, [v.desc for v in val]) - elif isinstance(val, core.BlockDesc) or \ - isinstance(val, core.ProgramDesc): + elif isinstance(val, core.BlockDesc) or isinstance( + val, core.ProgramDesc + ): desc.set_serialized_attr(name, val.serialize_to_string()) else: desc._set_attr(name, val) @@ -4536,8 +4745,9 @@ class IrOpNode(IrNode): Returns: list(str): input arguments' names of this op node. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().input_arg_names() def output_arg_names(self): @@ -4547,8 +4757,9 @@ class IrOpNode(IrNode): Returns: list(str): output arguments' names of this op node. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().output_arg_names() @property @@ -4589,7 +4800,8 @@ class IrGraph(object): for_test(bool): True for the test graph and false for the train graph. """ assert isinstance( - graph, core.Graph), 'graph must be the instance of core.Graph.' + graph, core.Graph + ), 'graph must be the instance of core.Graph.' self.graph = graph self._for_test = for_test @@ -4630,8 +4842,11 @@ class IrGraph(object): """ persistable_nodes = set() for node in self.graph.nodes(): - if node.is_var() and node.var() is not None and node.var( - ).persistable(): + if ( + node.is_var() + and node.var() is not None + and node.var().persistable() + ): persistable_nodes.add(node) return {IrVarNode(p) for p in persistable_nodes} @@ -4738,13 +4953,15 @@ class IrGraph(object): for input_name, var_nodes in inputs.items(): if not isinstance(var_nodes, list): var_nodes = [var_nodes] - op_desc.set_input(input_name, - [var_node.name() for var_node in var_nodes]) + op_desc.set_input( + input_name, [var_node.name() for var_node in var_nodes] + ) for output_name, var_nodes in outputs.items(): if not isinstance(var_nodes, list): var_nodes = [var_nodes] - op_desc.set_output(output_name, - [var_node.name() for var_node in var_nodes]) + op_desc.set_output( + output_name, [var_node.name() for var_node in var_nodes] + ) return IrOpNode(self.graph.create_op_node(op_desc)) def create_op_node_from_desc(self, op_desc): @@ -4768,9 +4985,11 @@ class IrGraph(object): new_input_node(IrNode): the new input node of the giving op_node. op_node(IrOpNode): the operator node that is needed to update input's link. """ - assert old_input_node.node in self.graph.nodes() and new_input_node.node in \ - self.graph.nodes() and op_node.node in self.graph.nodes(), \ - 'The three arguments(old_input_node&new_input_node&op_node) must be in the graph nodes.' + assert ( + old_input_node.node in self.graph.nodes() + and new_input_node.node in self.graph.nodes() + and op_node.node in self.graph.nodes() + ), 'The three arguments(old_input_node&new_input_node&op_node) must be in the graph nodes.' old_input_node.remove_output(op_node) op_node.remove_input(old_input_node) new_input_node.append_output(op_node) @@ -4786,9 +5005,11 @@ class IrGraph(object): new_output_node(IrNode): the new output node of the giving op_node. op_node(IrOpNode): the operator node that is needed to update input's link. """ - assert old_output_node.node in self.graph.nodes() and new_output_node.node in \ - self.graph.nodes() and op_node.node in self.graph.nodes(), \ - 'The three arguments(old_output_node &new_output_node &op_node) must be in the graph nodes.' + assert ( + old_output_node.node in self.graph.nodes() + and new_output_node.node in self.graph.nodes() + and op_node.node in self.graph.nodes() + ), 'The three arguments(old_output_node &new_output_node &op_node) must be in the graph nodes.' old_output_node.remove_input(op_node) op_node.remove_output(old_output_node) new_output_node.append_input(op_node) @@ -4804,9 +5025,11 @@ class IrGraph(object): node_out(IrNode): the output node. """ assert node_in.node in self.graph.nodes(), ( - 'node_in(%s) must be in the graph nodes.' % node_in.node.name()) + 'node_in(%s) must be in the graph nodes.' % node_in.node.name() + ) assert node_out.node in self.graph.nodes(), ( - 'node_out(%s) must be in the graph nodes.' % node_out.node.name()) + 'node_out(%s) must be in the graph nodes.' % node_out.node.name() + ) node_in.append_output(node_out) node_out.append_input(node_in) @@ -4843,8 +5066,8 @@ class IrGraph(object): ] else: var_nodes[each_var_name].append( - self._find_node_by_name(node.outputs, - each_var_name)) + self._find_node_by_name(node.outputs, each_var_name) + ) self.graph.resolve_hazard(var_nodes) def has_circle(self): @@ -4906,13 +5129,15 @@ class IrGraph(object): def _convert_to_pdf(dot_file_path): pdf_save_path = os.path.splitext(dot_file_path)[0] + '.pdf' - exited_code = subprocess.call('dot -Tpdf ' + dot_file_path + - ' -o ' + pdf_save_path, - shell=True) + exited_code = subprocess.call( + 'dot -Tpdf ' + dot_file_path + ' -o ' + pdf_save_path, + shell=True, + ) if exited_code != 0: print('The dot command is needed for creating pdf files.') - print('The {} is saved as the dot filetype.'.format( - dot_file_path)) + print( + 'The {} is saved as the dot filetype.'.format(dot_file_path) + ) remove_ctr_vars = set() if remove_ctr_var: @@ -4969,7 +5194,8 @@ class IrGraph(object): if n.name() == node_name: target_node = n assert target_node is not None, ( - "Cannot find the target node (%s)in the giving set." % node_name) + "Cannot find the target node (%s)in the giving set." % node_name + ) return target_node def _update_desc_attr(self, desc, name, val): @@ -4984,8 +5210,9 @@ class IrGraph(object): desc.set_block_attr(name, val.desc) elif isinstance(val, list) and val and _all_is_type(val, Block): desc.set_blocks_attr(name, [v.desc for v in val]) - elif isinstance(val, core.BlockDesc) or \ - isinstance(val, core.ProgramDesc): + elif isinstance(val, core.BlockDesc) or isinstance( + val, core.ProgramDesc + ): desc.set_serialized_attr(name, val.serialize_to_string()) else: desc._set_attr(name, val) @@ -5087,7 +5314,8 @@ class Program(object): # identifier for auto checkpoint self._auto_checkpoint_name = unique_name.generate( - "__auto_checkpoint_program__") + "__auto_checkpoint_program__" + ) # compiled program, i.e. Graph self._graph = None @@ -5107,7 +5335,7 @@ class Program(object): all_new_vars = [] block_num = new_desc.num_blocks() for idx in range(block_num): - if (idx > (len(self.blocks) - 1)): + if idx > (len(self.blocks) - 1): self._create_block() new_block_desc = new_desc.block(idx) all_new_vars.append([]) @@ -5119,60 +5347,75 @@ class Program(object): old_var = None kwargs = { - 'type': - new_var_desc.type(), - 'name': - new_var_desc.name(), - 'shape': - get_var_desc_attr_or_none(new_var_desc, "shape", [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR_ARRAY, - ]), - 'dtype': - get_var_desc_attr_or_none(new_var_desc, "dtype", [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR_ARRAY, - ]), - 'lod_level': - get_var_desc_attr_or_none(new_var_desc, "lod_level", [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.LOD_TENSOR_ARRAY, - ]), - 'error_clip': - old_var.error_clip if old_var is not None else None, - 'stop_gradient': - old_var.stop_gradient if old_var is not None else False, - 'is_data': - old_var.is_data if old_var is not None else False, - 'need_check_feed': - new_var_desc.need_check_feed(), - 'belong_to_optimizer': - old_var.belong_to_optimizer - if old_var is not None else False, + 'type': new_var_desc.type(), + 'name': new_var_desc.name(), + 'shape': get_var_desc_attr_or_none( + new_var_desc, + "shape", + [ + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ], + ), + 'dtype': get_var_desc_attr_or_none( + new_var_desc, + "dtype", + [ + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ], + ), + 'lod_level': get_var_desc_attr_or_none( + new_var_desc, + "lod_level", + [ + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ], + ), + 'error_clip': old_var.error_clip + if old_var is not None + else None, + 'stop_gradient': old_var.stop_gradient + if old_var is not None + else False, + 'is_data': old_var.is_data + if old_var is not None + else False, + 'need_check_feed': new_var_desc.need_check_feed(), + 'belong_to_optimizer': old_var.belong_to_optimizer + if old_var is not None + else False, } if isinstance(old_var, Parameter): - kwargs.update({ - 'trainable': old_var.trainable, - 'optimize_attr': old_var.optimize_attr, - 'regularizer': old_var.regularizer, - 'do_model_average': old_var.do_model_average, - 'need_clip': old_var.need_clip, - 'is_distributed': old_var.is_distributed, - 'is_parameter': old_var.is_parameter, - }) - block_new_vars.append({ - 'class': Parameter, - 'kwargs': copy.deepcopy(kwargs), - }) + kwargs.update( + { + 'trainable': old_var.trainable, + 'optimize_attr': old_var.optimize_attr, + 'regularizer': old_var.regularizer, + 'do_model_average': old_var.do_model_average, + 'need_clip': old_var.need_clip, + 'is_distributed': old_var.is_distributed, + 'is_parameter': old_var.is_parameter, + } + ) + block_new_vars.append( + { + 'class': Parameter, + 'kwargs': copy.deepcopy(kwargs), + } + ) else: kwargs['persistable'] = new_var_desc.persistable() - block_new_vars.append({ - 'class': Variable, - 'kwargs': copy.deepcopy(kwargs), - }) + block_new_vars.append( + { + 'class': Variable, + 'kwargs': copy.deepcopy(kwargs), + } + ) return all_new_vars @@ -5404,7 +5647,8 @@ class Program(object): assert isinstance( skip_op_callstack, bool ), "skip_op_callstack parameter's type is error, expect bool, received {}".format( - type(skip_op_callstack)) + type(skip_op_callstack) + ) program_str = "" for block in self.blocks: program_str += block._to_readable_code(skip_op_callstack) @@ -5446,11 +5690,13 @@ class Program(object): assert isinstance( throw_on_error, bool ), "The type of throw_on_error parameter is wrong, expected bool, but received {}.".format( - type(throw_on_error)) + type(throw_on_error) + ) assert isinstance( with_details, bool ), "The type of with_details parameter is wrong, expected bool, but received {}.".format( - type(with_details)) + type(with_details) + ) if with_details: res_str = "" @@ -5654,7 +5900,8 @@ class Program(object): if for_test: forward_prog = Program() forward_prog.desc, pruned_origin_block_id_map = core.prune_backward( - self.desc) + self.desc + ) forward_prog.blocks = [ Block(forward_prog, i) for i in range(forward_prog.desc.num_blocks()) @@ -5732,7 +5979,8 @@ class Program(object): if not isinstance(var, str): raise ValueError( "All feeded_var_names of Program._prune_with_input() can only be " - "str, but received %s." % type(var)) + "str, but received %s." % type(var) + ) # find out all variables that can be generated or updated with given feed generatable_vars = set() @@ -5760,7 +6008,8 @@ class Program(object): else: raise ValueError( "All targets of Program._prune_with_input() can only be " - "Variable or Operator, but received %s." % type(t)) + "Variable or Operator, but received %s." % type(t) + ) # NOTEZ(zhiqiu): For variable to be fed in fetch_list, there two cases: # (1) the variable is leaf, it has no op that generates it; @@ -5794,7 +6043,8 @@ class Program(object): res = Program() res.desc, pruned_origin_block_id_map = core.prune( - self.desc, set(feeded_var_names), targets_idx) + self.desc, set(feeded_var_names), targets_idx + ) res.blocks = [Block(res, i) for i in range(res.desc.num_blocks())] res._sync_with_cpp() @@ -5833,8 +6083,10 @@ class Program(object): root_block = res.desc.block(0) if prune_read_op: while True: - if read_op_idx >= root_block.op_size() or root_block.op( - read_op_idx).type() == 'read': + if ( + read_op_idx >= root_block.op_size() + or root_block.op(read_op_idx).type() == 'read' + ): break read_op_idx += 1 if read_op_idx < root_block.op_size(): @@ -5926,14 +6178,22 @@ class Program(object): # for name in remove_output_list: # op.remove_output(name) - op_quant_name = core.op_proto_and_checker_maker.kOpWithQuantAttrName( + op_quant_name = ( + core.op_proto_and_checker_maker.kOpWithQuantAttrName() + ) + quant = ( + bool(op.attr(op_quant_name)) + if op_quant_name in op.attr_names() + else False ) - quant = bool(op.attr(op_quant_name) - ) if op_quant_name in op.attr_names() else False quant_attrs = [ - op_quant_name, "quantization_type", "skip_quant", - "activation_bits", "bit_length", "quantize_weight_bits", - "weight_quant_scale" + op_quant_name, + "quantization_type", + "skip_quant", + "activation_bits", + "bit_length", + "quantize_weight_bits", + "weight_quant_scale", ] for extra_attr_name in extra_attrs_map.keys(): op.remove_attr(extra_attr_name) @@ -6096,7 +6356,8 @@ class Program(object): if not isinstance(seed, int): raise ValueError( "Program.random_seed's input seed must be an integer, but received %s." - % type(seed)) + % type(seed) + ) self._seed = seed def __repr__(self): @@ -6193,8 +6454,11 @@ class Program(object): Block: The new block. """ new_block_idx = len(self.blocks) - parent = self.current_block() if parent_idx is None else self.block( - parent_idx) + parent = ( + self.current_block() + if parent_idx is None + else self.block(parent_idx) + ) self.desc.append_block(parent.desc) self.current_block_idx = new_block_idx self.blocks.append(Block(self, self.current_block_idx)) @@ -6240,7 +6504,8 @@ class Program(object): if not isinstance(other, Program): raise TypeError( "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s" - % type(other)) + % type(other) + ) self.global_block()._copy_param_info_from(other.global_block()) @@ -6257,7 +6522,8 @@ class Program(object): if not isinstance(other, Program): raise TypeError( "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s" - % type(other)) + % type(other) + ) self._is_distributed = other._is_distributed self._is_chief = other._is_chief self._parameters_on_pservers = other._parameters_on_pservers @@ -6285,12 +6551,12 @@ class Program(object): if not isinstance(other, Program): raise TypeError( "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s" - % type(other)) + % type(other) + ) if not pruned_origin_block_id_map: pruned_origin_block_id_map = { - i: i - for i in range(self.desc.num_blocks()) + i: i for i in range(self.desc.num_blocks()) } # NOTE(zhiqiu): All vars in cloned program exist in original program. @@ -6418,10 +6684,13 @@ class Program(object): # can not be imported at the begainning of this file. # Therefore, the above two modules are dynamically imported. from .executor import global_scope + if scope is not None and not isinstance(scope, core._Scope): raise TypeError( - "`scope` should be None or `paddle.static.Scope'` type, but received {}." - .format(type(scope))) + "`scope` should be None or `paddle.static.Scope'` type, but received {}.".format( + type(scope) + ) + ) if scope is None: scope = global_scope() @@ -6429,15 +6698,19 @@ class Program(object): if not isinstance(mode, str): raise TypeError( "Type of `mode` should be string, but received {}.".format( - type(mode))) + type(mode) + ) + ) def is_parameter(var): return isinstance(var, Parameter) def is_persistable(var): - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -6456,8 +6729,10 @@ class Program(object): return is_parameter(var) or is_belong_to_optimizer(var) else: raise ValueError( - "`mode` string should be 'param', 'opt' or 'all', but received {}." - .format(mode)) + "`mode` string should be 'param', 'opt' or 'all', but received {}.".format( + mode + ) + ) var_list = filter(condition, self.list_vars()) @@ -6466,8 +6741,10 @@ class Program(object): var_temp = scope.find_var(var.name) if var_temp is None: raise ValueError( - "Can not find Variable '{}' in the scope. Make sure it is initialized" - .format(var.name)) + "Can not find Variable '{}' in the scope. Make sure it is initialized".format( + var.name + ) + ) state_dict[var.name] = var_temp.get_tensor() return state_dict @@ -6517,10 +6794,14 @@ class Program(object): if not isinstance(state_dict, dict): raise TypeError( "Type of `state_dict` should be dict, but received {}.".format( - type(state_dict))) + type(state_dict) + ) + ) vars_dict = {var.name: var for var in self.list_vars()} - condition = True if 'StructuredToParameterName@@' in state_dict else False + condition = ( + True if 'StructuredToParameterName@@' in state_dict else False + ) for name, value in state_dict.items(): if condition: if name == "StructuredToParameterName@@": @@ -6532,14 +6813,20 @@ class Program(object): vars_dict[name].set_value(value, scope) except ValueError as err: warnings.warn( - ("Skip loading for '{}'. ".format(name) + str(err))) + ("Skip loading for '{}'. ".format(name) + str(err)) + ) except TypeError as err: warnings.warn( - ("Skip loading for '{}'. ".format(name) + str(err))) + ("Skip loading for '{}'. ".format(name) + str(err)) + ) else: warnings.warn( - ("Skip loading for '{0}'. Because '{0}' not in the program." - .format(name))) + ( + "Skip loading for '{0}'. Because '{0}' not in the program.".format( + name + ) + ) + ) class Parameter(Variable, metaclass=ParameterMetaClass): @@ -6566,12 +6853,14 @@ class Parameter(Variable, metaclass=ParameterMetaClass): in optimizer. Default is True. """ - def __init__(self, - block, - shape, - dtype, - type=core.VarDesc.VarType.LOD_TENSOR, - **kwargs): + def __init__( + self, + block, + shape, + dtype, + type=core.VarDesc.VarType.LOD_TENSOR, + **kwargs + ): if shape is None: raise ValueError("The shape of Parameter should not be None") if dtype is None: @@ -6581,15 +6870,18 @@ class Parameter(Variable, metaclass=ParameterMetaClass): if each < 0: raise ValueError( "Each dimension of shape for Parameter must be greater than 0, but received %s" - % list(shape)) - - Variable.__init__(self, - block, - persistable=True, - shape=shape, - dtype=dtype, - type=type, - **kwargs) + % list(shape) + ) + + Variable.__init__( + self, + block, + persistable=True, + shape=shape, + dtype=dtype, + type=type, + **kwargs + ) self.trainable = kwargs.get('trainable', True) self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0}) @@ -6630,11 +6922,17 @@ class Parameter(Variable, metaclass=ParameterMetaClass): print(debug_str) """ assert isinstance(throw_on_error, bool) and isinstance( - with_details, bool) + with_details, bool + ) if with_details: res_str = Variable.to_string(self, throw_on_error, True) - additional_attr = ("trainable", "optimize_attr", "regularizer", - "do_model_average", "need_clip") + additional_attr = ( + "trainable", + "optimize_attr", + "regularizer", + "do_model_average", + "need_clip", + ) for attr_name in additional_attr: res_str += "%s: %s\n" % (attr_name, getattr(self, attr_name)) else: @@ -6680,7 +6978,8 @@ class ParamBase(core.VarBase): if each < 0: raise ValueError( "Each dimension of shape for Parameter must be greater than 0, but received %s" - % list(shape)) + % list(shape) + ) if dtype is not None: if not isinstance(dtype, core.VarDesc.VarType): @@ -6688,10 +6987,13 @@ class ParamBase(core.VarBase): name = kwargs.get('name', unique_name.generate('_param_base')) - super(ParamBase, - self).__init__(dtype if dtype else core.VarDesc.VarType.FP32, - list(shape) if shape else [], name, - core.VarDesc.VarType.LOD_TENSOR, True) + super(ParamBase, self).__init__( + dtype if dtype else core.VarDesc.VarType.FP32, + list(shape) if shape else [], + name, + core.VarDesc.VarType.LOD_TENSOR, + True, + ) trainable = kwargs.get('trainable', True) self.stop_gradient = not trainable @@ -6718,7 +7020,8 @@ class ParamBase(core.VarBase): else: raise ValueError( "The type of trainable MUST be bool, but the type is ", - type(trainable)) + type(trainable), + ) def __str__(self): """ @@ -6739,7 +7042,8 @@ class ParamBase(core.VarBase): # [-0.54217887, 0.48439729, 0.34082305]]) """ return "Parameter containing:\n{tensor}".format( - tensor=super(ParamBase, self).__str__()) + tensor=super(ParamBase, self).__str__() + ) def __deepcopy__(self, memo): """ @@ -6822,7 +7126,8 @@ class EagerParamBase(_core_eager_eagertensor): if each < 0: raise ValueError( "Each dimension of shape for Parameter must be greater than 0, but received %s" - % list(shape)) + % list(shape) + ) if dtype is not None: if not isinstance(dtype, core.VarDesc.VarType): @@ -6833,10 +7138,13 @@ class EagerParamBase(_core_eager_eagertensor): if isinstance(shape, core.eager.Tensor): shape = shape.numpy() - super(EagerParamBase, - self).__init__(dtype if dtype else core.VarDesc.VarType.FP32, - list(shape) if shape else [], name, - core.VarDesc.VarType.LOD_TENSOR, True) + super(EagerParamBase, self).__init__( + dtype if dtype else core.VarDesc.VarType.FP32, + list(shape) if shape else [], + name, + core.VarDesc.VarType.LOD_TENSOR, + True, + ) self.retain_grads() trainable = kwargs.get('trainable', True) @@ -6860,7 +7168,9 @@ class EagerParamBase(_core_eager_eagertensor): @dygraph_only def initialize(self): - assert self._init_func is not None, "Required self._init_func is not None, but received None." + assert ( + self._init_func is not None + ), "Required self._init_func is not None, but received None." self._init_func() # clear function handle to release resource self._init_func = None @@ -6876,13 +7186,16 @@ class EagerParamBase(_core_eager_eagertensor): else: raise ValueError( "The type of trainable MUST be bool, but the type is ", - type(trainable)) + type(trainable), + ) def _create_init_op(self, block): """ Call init_op_creator function to create initializer operation in block. """ - assert self._init_op_creator is not None, "Required self._init_op_creator is not None, but received None." + assert ( + self._init_op_creator is not None + ), "Required self._init_op_creator is not None, but received None." self._init_op_creator(block) def __str__(self): @@ -6904,7 +7217,8 @@ class EagerParamBase(_core_eager_eagertensor): # [-0.54217887, 0.48439729, 0.34082305]]) """ return "Parameter containing:\n{tensor}".format( - tensor=super(EagerParamBase, self).__str__()) + tensor=super(EagerParamBase, self).__str__() + ) def __deepcopy__(self, memo): """ @@ -7091,12 +7405,18 @@ def program_guard(main_program, startup_program=None): """ from .data_feeder import check_type - check_type(main_program, 'main_program', Program, - 'paddle.static.program_guard') + + check_type( + main_program, 'main_program', Program, 'paddle.static.program_guard' + ) main_program = switch_main_program(main_program) if startup_program is not None: - check_type(startup_program, 'startup_program', Program, - 'paddle.static.program_guard') + check_type( + startup_program, + 'startup_program', + Program, + 'paddle.static.program_guard', + ) # Tag the program __is_start_up as True startup_program._is_start_up_program_ = True startup_program = switch_startup_program(startup_program) @@ -7219,7 +7539,8 @@ def device_guard(device=None): if device not in ['cpu', 'gpu', 'npu', 'xpu', 'mlu', '', None]: raise ValueError( "The Attr(device) should be 'cpu' 'npu' 'xpu' 'mlu' or 'gpu', and it can also be empty string or None " - "when there is no need to specify device. But received %s" % device) + "when there is no need to specify device. But received %s" % device + ) if index: device = ":".join([device, index]) pre_device = switch_device(device) @@ -7249,9 +7570,11 @@ def _cuda_graph_guard(cuda_graph_attr=None): cuda_graph_attr(str|None): The cuda graph attr with the format of: cuda_graph_capture_mode;memory_pool_id;cuda_graph_id """ - assert not _non_static_mode( + assert ( + not _non_static_mode() ), "cuda_graph_guard only works under static mode" - assert core.is_compiled_with_cuda( + assert ( + core.is_compiled_with_cuda() ), "cuda_graph_guard context can be only used when Paddle is compiled with cuda" pre_mode = _switch_cuda_graph_mode(cuda_graph_attr) try: @@ -7281,7 +7604,8 @@ def set_flags(flags): _global_flags()[key] = value else: raise ValueError( - "Flag %s cannot set its value through this function." % (key)) + "Flag %s cannot set its value through this function." % (key) + ) def get_flags(flags): @@ -7308,22 +7632,24 @@ def get_flags(flags): flags_value = {} if isinstance(flags, (list, tuple)): for key in flags: - if (_global_flags().is_public(key)): + if _global_flags().is_public(key): value = _global_flags()[key] temp = {key: value} flags_value.update(temp) else: raise ValueError( - 'Flag %s cannot get its value through this function.' % - (key)) + 'Flag %s cannot get its value through this function.' + % (key) + ) elif isinstance(flags, str): - if (_global_flags().is_public(flags)): + if _global_flags().is_public(flags): value = _global_flags()[flags] temp = {flags: value} flags_value.update(temp) else: raise ValueError( - 'Flag %s cannot get its value through this function.' % (flags)) + 'Flag %s cannot get its value through this function.' % (flags) + ) else: raise TypeError('Flags in get_flags should be a list, tuple or string.') return flags_value @@ -7333,20 +7659,32 @@ def _get_paddle_place(place): "convert the string to paddle Place" if place is None: return place - if isinstance(place, (core.Place, core.XPUPlace, core.CPUPlace, - core.CUDAPinnedPlace, core.CUDAPlace, core.NPUPlace, - core.IPUPlace, core.MLUPlace, core.CustomPlace)): + if isinstance( + place, + ( + core.Place, + core.XPUPlace, + core.CPUPlace, + core.CUDAPinnedPlace, + core.CUDAPlace, + core.NPUPlace, + core.IPUPlace, + core.MLUPlace, + core.CustomPlace, + ), + ): return place if not isinstance(place, str): raise ValueError( - "place only support string which is 'Place' and so on.") + "place only support string which is 'Place' and so on." + ) place = place.lower() - if (place == "cpu"): + if place == "cpu": return core.CPUPlace() - if (place == "device"): + if place == "device": return core.Place() # GPU @@ -7354,8 +7692,9 @@ def _get_paddle_place(place): if place == "gpu_pinned" or place == "gpu" or avaliable_gpu_place: if not core.is_compiled_with_cuda(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with CUDA".format(avaliable_gpu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with CUDA".format(avaliable_gpu_place) + ) if place == "gpu_pinned": return core.CUDAPinnedPlace() elif place == "gpu": @@ -7371,8 +7710,9 @@ def _get_paddle_place(place): if avaliable_xpu_place: if not core.is_compiled_with_xpu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with XPU".format(avaliable_xpu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with XPU".format(avaliable_xpu_place) + ) place_info_list = place.split(':', 1) device_id = place_info_list[1] device_id = int(device_id) @@ -7383,8 +7723,9 @@ def _get_paddle_place(place): if avaliable_npu_place: if not core.is_compiled_with_npu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with NPU".format(avaliable_npu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with NPU".format(avaliable_npu_place) + ) place_info_list = place.split(':', 1) device_id = place_info_list[1] device_id = int(device_id) @@ -7395,8 +7736,9 @@ def _get_paddle_place(place): if avaliable_ipu_place: if not core.is_compiled_with_ipu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with IPU".format(avaliable_ipu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with IPU".format(avaliable_ipu_place) + ) place_info_list = place.split(':', 1) device_id = place_info_list[1] device_id = int(device_id) @@ -7407,16 +7749,19 @@ def _get_paddle_place(place): if avaliable_mlu_place: if not core.is_compiled_with_mlu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with MLU".format(avaliable_mlu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with MLU".format(avaliable_mlu_place) + ) place_info_list = place.split(':', 1) device_id = place_info_list[1] device_id = int(device_id) return core.MLUPlace(device_id) raise ValueError( - "Paddle supports CPUPlace, CUDAPlace,CUDAPinnedPlace, XPUPlace, IPUPlace, MLUPlace and NPUPlace, but received {}." - .format(place)) + "Paddle supports CPUPlace, CUDAPlace,CUDAPinnedPlace, XPUPlace, IPUPlace, MLUPlace and NPUPlace, but received {}.".format( + place + ) + ) def _get_paddle_place_list(places): diff --git a/python/paddle/fluid/generator.py b/python/paddle/fluid/generator.py index 5bbe7a0e12c..4982de95f49 100644 --- a/python/paddle/fluid/generator.py +++ b/python/paddle/fluid/generator.py @@ -43,4 +43,5 @@ class Generator(core.Generator): else: raise ValueError( "Generator class with %s does is not supported yet, currently only support generator with CPUPlace " - % place) + % place + ) diff --git a/python/paddle/fluid/graphviz.py b/python/paddle/fluid/graphviz.py index 9579bff409e..b7ef1c7531f 100644 --- a/python/paddle/fluid/graphviz.py +++ b/python/paddle/fluid/graphviz.py @@ -26,7 +26,6 @@ def crepr(v): class Rank(object): - def __init__(self, kind, name, priority): ''' kind: str @@ -42,8 +41,12 @@ class Rank(object): if not self.nodes: return '' - return '{' + 'rank={};'.format(self.kind) + \ - ','.join([node.name for node in self.nodes]) + '}' + return ( + '{' + + 'rank={};'.format(self.kind) + + ','.join([node.name for node in self.nodes]) + + '}' + ) class Graph(object): @@ -84,28 +87,36 @@ class Graph(object): def compile(self, dot_path): file = open(dot_path, 'w') file.write(self.__str__()) - image_path = os.path.join(os.path.dirname(dot_path), - dot_path[:-3] + "pdf") + image_path = os.path.join( + os.path.dirname(dot_path), dot_path[:-3] + "pdf" + ) cmd = ["dot", "-Tpdf", dot_path, "-o", image_path] - subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) logging.warning("write block debug graph to {}".format(image_path)) return image_path def show(self, dot_path): image = self.compile(dot_path) cmd = ["open", image] - subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) def _rank_repr(self): - ranks = sorted(self.rank_groups.items(), - key=functools.cmp_to_key( - lambda a, b: a[1].priority > b[1].priority)) + ranks = sorted( + self.rank_groups.items(), + key=functools.cmp_to_key( + lambda a, b: a[1].priority > b[1].priority + ), + ) repr = [] for x in ranks: repr.append(str(x[1])) @@ -118,8 +129,9 @@ class Graph(object): ] for attr in self.attrs: - reprs.append("{key}={value};".format(key=attr, - value=crepr(self.attrs[attr]))) + reprs.append( + "{key}={value};".format(key=attr, value=crepr(self.attrs[attr])) + ) reprs.append(self._rank_repr()) @@ -147,14 +159,18 @@ class Node(object): reprs = '{name} [label={label} {extra} ];'.format( name=self.name, label=self.label, - extra=',' + ','.join("%s=%s" % (key, crepr(value)) - for key, value in self.attrs.items()) - if self.attrs else "") + extra=',' + + ','.join( + "%s=%s" % (key, crepr(value)) + for key, value in self.attrs.items() + ) + if self.attrs + else "", + ) return reprs class Edge(object): - def __init__(self, source, target, **attrs): ''' Link source to target. @@ -171,9 +187,15 @@ class Edge(object): repr = "{source} -> {target} {extra}".format( source=self.source.name, target=self.target.name, - extra="" if not self.attrs else "[" + - ','.join("{}={}".format(attr[0], crepr(attr[1])) - for attr in self.attrs.items()) + "]") + extra="" + if not self.attrs + else "[" + + ','.join( + "{}={}".format(attr[0], crepr(attr[1])) + for attr in self.attrs.items() + ) + + "]", + ) return repr @@ -202,31 +224,34 @@ class GraphPreviewGenerator(object): self.graph.show(path) def add_param(self, name, data_type, highlight=False): - label = '\n'.join([ - '<', - ' ', - ' ', - ' ', - ' ', - ' ' - ' ', - '
', - ' ', - name, - ' ', - '
', - str(data_type), - '
>', - ]) - return self.graph.node(label, - prefix="param", - description=name, - shape="none", - style="rounded,filled,bold", - width="1.3", - color="#148b97" if not highlight else "orange", - fontcolor="#ffffff", - fontname="Arial") + label = '\n'.join( + [ + '<', + ' ', + ' ', + ' ', + ' ', + ' ' ' ', + '
', + ' ', + name, + ' ', + '
', + str(data_type), + '
>', + ] + ) + return self.graph.node( + label, + prefix="param", + description=name, + shape="none", + style="rounded,filled,bold", + width="1.3", + color="#148b97" if not highlight else "orange", + fontcolor="#ffffff", + fontname="Arial", + ) def add_op(self, opType, **kwargs): highlight = False @@ -247,21 +272,25 @@ class GraphPreviewGenerator(object): ) def add_arg(self, name, highlight=False): - return self.graph.node(crepr(name), - prefix="arg", - description=name, - shape="box", - style="rounded,filled,bold", - fontname="Arial", - fontcolor="#999999", - color="#dddddd" if not highlight else "orange") + return self.graph.node( + crepr(name), + prefix="arg", + description=name, + shape="box", + style="rounded,filled,bold", + fontname="Arial", + fontcolor="#999999", + color="#dddddd" if not highlight else "orange", + ) def add_edge(self, source, target, **kwargs): highlight = False if 'highlight' in kwargs: highlight = kwargs['highlight'] del kwargs['highlight'] - return self.graph.edge(source, - target, - color="#00000" if not highlight else "orange", - **kwargs) + return self.graph.edge( + source, + target, + color="#00000" if not highlight else "orange", + **kwargs + ) diff --git a/python/paddle/fluid/incubate/checkpoint/auto_checkpoint.py b/python/paddle/fluid/incubate/checkpoint/auto_checkpoint.py index e0d8062877c..134723cdbc0 100644 --- a/python/paddle/fluid/incubate/checkpoint/auto_checkpoint.py +++ b/python/paddle/fluid/incubate/checkpoint/auto_checkpoint.py @@ -55,7 +55,8 @@ def _get_logger(log_level, name="auto_checkpoint"): log_handler = logging.StreamHandler() log_format = logging.Formatter( - '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s') + '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s' + ) log_handler.setFormatter(log_format) logger.addHandler(log_handler) @@ -63,12 +64,12 @@ def _get_logger(log_level, name="auto_checkpoint"): def _thread_checker(): - assert current_thread().name == "MainThread", \ - "auto checkpoint must run under main thread" + assert ( + current_thread().name == "MainThread" + ), "auto checkpoint must run under main thread" class AutoCheckpointChecker(object): - def __init__(self): self._run_env = None self._platform = None @@ -91,35 +92,43 @@ class AutoCheckpointChecker(object): self._hdfs_name = os.environ["PADDLE_EDL_HDFS_NAME"] self._hdfs_ugi = os.environ["PADDLE_EDL_HDFS_UGI"] self._hdfs_checkpoint_path = os.environ[ - "PADDLE_EDL_HDFS_CHECKPOINT_PATH"] + "PADDLE_EDL_HDFS_CHECKPOINT_PATH" + ] self._trainer_id = int(os.environ["PADDLE_TRAINER_ID"]) self._ce_test = int(os.getenv("PADDLE_EDL_ONLY_FOR_CE_TEST", "0")) self._fs_cache = os.getenv("PADDLE_EDL_FS_CACHE", ".cache") self._save_checkpoint_inter = int( - os.getenv("PADDLE_EDL_SAVE_CHECKPOINT_INTER", "900")) # s + os.getenv("PADDLE_EDL_SAVE_CHECKPOINT_INTER", "900") + ) # s if not self._ce_test: - assert len(self._hdfs_home) > 3 and \ - len(self._hdfs_name) > 6 and \ - len(self._hdfs_ugi) > 3 and \ - len(self._hdfs_checkpoint_path) > 0, "hdfs environ must set" + assert ( + len(self._hdfs_home) > 3 + and len(self._hdfs_name) > 6 + and len(self._hdfs_ugi) > 3 + and len(self._hdfs_checkpoint_path) > 0 + ), "hdfs environ must set" else: - assert len(self._hdfs_home) > 3 and \ - len(self._hdfs_checkpoint_path) > 0, "hdfs environ must set" + assert ( + len(self._hdfs_home) > 3 + and len(self._hdfs_checkpoint_path) > 0 + ), "hdfs environ must set" except Exception as e: logger.fatal("exception:{}".format(e)) sys.exit(1) def get_range_checkpoint_path(self, name): - return "{}/{}/range/{}".format(self.hdfs_checkpoint_path, self.job_id, - name) + return "{}/{}/range/{}".format( + self.hdfs_checkpoint_path, self.job_id, name + ) def get_exe_checkpoint_path(self, name): - return "{}/{}/exe/{}".format(self.hdfs_checkpoint_path, self.job_id, - name) + return "{}/{}/exe/{}".format( + self.hdfs_checkpoint_path, self.job_id, name + ) def get_job_path(self): return "{}/{}".format(self.hdfs_checkpoint_path, self.job_id) @@ -132,22 +141,30 @@ class AutoCheckpointChecker(object): if _non_static_mode(): return False - return self._run_env is not None and \ - self._platform is not None and \ - self._job_id is not None and \ - self._hdfs_home is not None and \ - self._hdfs_name is not None and \ - self._hdfs_ugi is not None and \ - self._hdfs_checkpoint_path is not None and \ - self._trainer_id is not None + return ( + self._run_env is not None + and self._platform is not None + and self._job_id is not None + and self._hdfs_home is not None + and self._hdfs_name is not None + and self._hdfs_ugi is not None + and self._hdfs_checkpoint_path is not None + and self._trainer_id is not None + ) def __str__(self): return "run_env:{} platform:{} job_id:{} \ hdfs_home:{} hdfs_name:{} hdfs_ugi:{} \ hdfs_checkpoint_path:{} trainer_id:{} ce_test".format( - self._run_env, self._platform, self._hdfs_home, self._hdfs_name, - self._hdfs_ugi, self._hdfs_checkpoint_path, self._trainer_id, - self._ce_test) + self._run_env, + self._platform, + self._hdfs_home, + self._hdfs_name, + self._hdfs_ugi, + self._hdfs_checkpoint_path, + self._trainer_id, + self._ce_test, + ) @property def trainer_id(self): @@ -191,7 +208,6 @@ class AutoCheckpointChecker(object): class ExeTrainStatus(SerializableBase): - def __init__(self): self._epoch_no = -1 # start epoch_no self._hash_key = None @@ -207,13 +223,15 @@ class ExeTrainStatus(SerializableBase): self._file_name = "exe_train_status" def __eq__(self, t): - return self._epoch_no == t._epoch_no and \ - self._hash_key == t._hash_key and \ - self._key == t._key and \ - self._checkpoint_path == t._checkpoint_path and \ - self._checkpoint_no == t._checkpoint_no and \ - self._exe_name == t._exe_name and \ - self._program_name == t._program_name + return ( + self._epoch_no == t._epoch_no + and self._hash_key == t._hash_key + and self._key == t._key + and self._checkpoint_path == t._checkpoint_path + and self._checkpoint_no == t._checkpoint_no + and self._exe_name == t._exe_name + and self._program_name == t._program_name + ) def __ne__(self, t): return not self == t @@ -256,7 +274,7 @@ class ExeTrainStatus(SerializableBase): "restored_from": self._restored_from, "exe_name": self._exe_name, "program_name": self._program_name, - "checkpoint_no": self._checkpoint_no + "checkpoint_no": self._checkpoint_no, } def __str__(self): @@ -264,12 +282,9 @@ class ExeTrainStatus(SerializableBase): class TrainEpochRange(SerializableBase): - - def __init__(self, - max_epoch_num, - name, - checkpoint_inter=None, - restored=True): + def __init__( + self, max_epoch_num, name, checkpoint_inter=None, restored=True + ): self._max_epoch_num = max_epoch_num self._epoch_no = -1 # current epoch_no self._name = name @@ -282,8 +297,9 @@ class TrainEpochRange(SerializableBase): self._save_checkpoint_inter = checkpoint_inter else: self._save_checkpoint_inter = self._checker.save_checkpoint_inter - assert self._save_checkpoint_inter >= 0, "checkpointer:{} must >=0".format( - self._save_checkpoint_inter) + assert ( + self._save_checkpoint_inter >= 0 + ), "checkpointer:{} must >=0".format(self._save_checkpoint_inter) self._last_checkpoint_time = time.time() self._load_cp_nos = None @@ -301,13 +317,14 @@ class TrainEpochRange(SerializableBase): config = { "fs.default.name": self._checker.hdfs_name, - "hadoop.job.ugi": self._checker.hdfs_ugi + "hadoop.job.ugi": self._checker.hdfs_ugi, } if self._checker.ce_test: config = None from paddle.distributed.fleet.utils.fs import HDFSClient + self._hdfs = HDFSClient(self._checker.hdfs_home, config) self._cper = CheckpointSaver(self._hdfs) @@ -321,10 +338,13 @@ class TrainEpochRange(SerializableBase): epoch_no = -1 for i in cp_nos[::-1]: t = TrainEpochRange(self._max_epoch_num, self.name, restored=False) - self._cper.load_checkpoint(self._checkpoint_path, [t], - self._checker.trainer_id, - checkpoint_no=i, - local_cache_path=self._checker._fs_cache) + self._cper.load_checkpoint( + self._checkpoint_path, + [t], + self._checker.trainer_id, + checkpoint_no=i, + local_cache_path=self._checker._fs_cache, + ) cps.append(t) logger.debug("look for valid:{} t:{}".format(i, t._serialize())) if epoch_no < 0: @@ -344,14 +364,18 @@ class TrainEpochRange(SerializableBase): if g_acp_type == CONST_ACP_TYPE: # get the last one - self._cper.load_checkpoint(self._checkpoint_path, [self], - self._checker.trainer_id, - local_cache_path=self._checker._fs_cache) + self._cper.load_checkpoint( + self._checkpoint_path, + [self], + self._checker.trainer_id, + local_cache_path=self._checker._fs_cache, + ) self._restored_from = CONST_CHECKPOINT self._checkpoint_epoch_no = self._epoch_no - logger.info("load tain_epoch_range checkpoint:{}".format( - self._serialize())) + logger.info( + "load tain_epoch_range checkpoint:{}".format(self._serialize()) + ) elif g_acp_type == CONST_DACP_TYPE: t, i = self._look_for_valid(self._load_cp_nos) @@ -359,15 +383,19 @@ class TrainEpochRange(SerializableBase): self._restored_from = CONST_MEMORYINIT return - self._cper.load_checkpoint(self._checkpoint_path, [self], - self._checker.trainer_id, - checkpoint_no=i, - local_cache_path=self._checker._fs_cache) + self._cper.load_checkpoint( + self._checkpoint_path, + [self], + self._checker.trainer_id, + checkpoint_no=i, + local_cache_path=self._checker._fs_cache, + ) self._restored_from = CONST_CHECKPOINT self._checkpoint_epoch_no = self._epoch_no - logger.info("load tain_epoch_range checkpoint:{}".format( - self._serialize())) + logger.info( + "load tain_epoch_range checkpoint:{}".format(self._serialize()) + ) else: assert False, "not supported acp_type:{}".format(g_acp_type) @@ -378,7 +406,7 @@ class TrainEpochRange(SerializableBase): "name": self._name, "checkpoint_path": self._checkpoint_path, "restored_from": self._restored_from, - "checkpoint_epoch_no": self._checkpoint_epoch_no + "checkpoint_epoch_no": self._checkpoint_epoch_no, } return d @@ -438,12 +466,16 @@ class TrainEpochRange(SerializableBase): self._max_epoch_num = sys.maxint assert self._epoch_no >= -1, "self._epoch_no:{} must >=-1".format( - self._epoch_no) + self._epoch_no + ) self._last_checkpoint_time = time.time() start = self._epoch_no + 1 - logger.info("started epoch_no:{} max_epoch_num:{}".format( - start, self._max_epoch_num)) + logger.info( + "started epoch_no:{} max_epoch_num:{}".format( + start, self._max_epoch_num + ) + ) for i in range(start, self._max_epoch_num): self._epoch_no = i @@ -458,11 +490,16 @@ class TrainEpochRange(SerializableBase): # not save last one because exe and program can't be restored. if self._checker.trainer_id == 0: - if time.time() - self._last_checkpoint_time >= \ - self._save_checkpoint_inter: + if ( + time.time() - self._last_checkpoint_time + >= self._save_checkpoint_inter + ): if g_acp_type == CONST_ACP_TYPE: # not save the last one - if self._max_epoch_num > 0 and self._epoch_no != self._max_epoch_num - 1: + if ( + self._max_epoch_num > 0 + and self._epoch_no != self._max_epoch_num - 1 + ): self._save_checkpoint() elif g_acp_type == CONST_DACP_TYPE: self._save_checkpoint() @@ -484,9 +521,11 @@ class TrainEpochRange(SerializableBase): p = self._checker.get_exe_checkpoint_path(t._hash_key) t._epoch_no = self.get() path, checkpoint_no = self._cper.save_checkpoint( - p, [m], + p, + [m], self._checker.trainer_id, - local_cache_path=self._checker._fs_cache) + local_cache_path=self._checker._fs_cache, + ) # index info t._checkpoint_path = path t._checkpoint_no = checkpoint_no @@ -496,10 +535,14 @@ class TrainEpochRange(SerializableBase): logger.debug("save executor checkpoint:{}".format(t._serialize())) if len(self._exe_status) > 0: - self._cper.save_checkpoint(self._checkpoint_path, [self], - local_cache_path=self._checker._fs_cache) - logger.info("save train_epoch_range checkpoint:{}".format( - self._serialize())) + self._cper.save_checkpoint( + self._checkpoint_path, + [self], + local_cache_path=self._checker._fs_cache, + ) + logger.info( + "save train_epoch_range checkpoint:{}".format(self._serialize()) + ) self._generate_flag() @@ -538,13 +581,13 @@ def _check_program_oprole(program): def _can_auto_checkpoint(prog): - if not isinstance(prog, compiler.CompiledProgram) and \ - not isinstance(prog, Program): + if not isinstance(prog, compiler.CompiledProgram) and not isinstance( + prog, Program + ): return False if isinstance(prog, compiler.CompiledProgram): - if prog._program is None or \ - prog._program._is_distributed: + if prog._program is None or prog._program._is_distributed: return False else: if prog._is_distributed: @@ -564,8 +607,11 @@ def _can_auto_checkpoint(prog): g_program_attr[program._auto_checkpoint_name] = ret if not ret: - logger.debug("program {} need't to auto checkpoint".format( - program._auto_checkpoint_name)) + logger.debug( + "program {} need't to auto checkpoint".format( + program._auto_checkpoint_name + ) + ) return False return g_checker.valid() and g_train_epoch_range is not None @@ -597,7 +643,8 @@ def train_epoch_range(max_epoch_num, save_checkpoint_inter=None): global g_acp_type if not _get_checker().valid(): logger.warning( - "auto checkpoint will take effect automaticly on PaddleCloud") + "auto checkpoint will take effect automaticly on PaddleCloud" + ) for i in _normal_yield(max_epoch_num): yield i @@ -617,7 +664,8 @@ def train_epoch_range(max_epoch_num, save_checkpoint_inter=None): g_train_epoch_range = TrainEpochRange( max_epoch_num, g_checker.generate_range_name(), - checkpoint_inter=save_checkpoint_inter) + checkpoint_inter=save_checkpoint_inter, + ) for i in g_train_epoch_range.next(): yield i @@ -643,12 +691,16 @@ def _auto_checkpoint(exe, prog): assert program._auto_checkpoint_name != None exe_status = g_train_epoch_range._exe_status - key = _get_running_key(exe._auto_checkpoint_name, - program._auto_checkpoint_name) + key = _get_running_key( + exe._auto_checkpoint_name, program._auto_checkpoint_name + ) if g_train_epoch_range.restored_from == CONST_CHECKPOINT: - assert key in exe_status, "when restored key:{} must be in train_epoch_range:{}".format( - key, g_train_epoch_range) + assert ( + key in exe_status + ), "when restored key:{} must be in train_epoch_range:{}".format( + key, g_train_epoch_range + ) t = None if key in exe_status: @@ -656,10 +708,13 @@ def _auto_checkpoint(exe, prog): if t._restored_from is None: a = CheckpointSaver(g_train_epoch_range._hdfs) m = PaddleModel(exe, program) - a.load_checkpoint(g_checker.get_exe_checkpoint_path(key), [m], - trainer_id=g_checker.trainer_id, - checkpoint_no=t._checkpoint_no, - local_cache_path=g_checker._fs_cache) + a.load_checkpoint( + g_checker.get_exe_checkpoint_path(key), + [m], + trainer_id=g_checker.trainer_id, + checkpoint_no=t._checkpoint_no, + local_cache_path=g_checker._fs_cache, + ) t._restored_from = CONST_CHECKPOINT logger.info("load executor checkpoint {}".format(t)) t._exe = exe diff --git a/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py b/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py index c8aeb50f157..21e305afc16 100644 --- a/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py +++ b/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py @@ -16,7 +16,6 @@ from ...compiler import CompiledProgram class SerializableBase(object): - def serialize(self, path): raise NotImplementedError @@ -25,7 +24,6 @@ class SerializableBase(object): class PaddleModel(SerializableBase): - def __init__(self, exe, program): self._exe = exe self._origin_program = program @@ -37,30 +35,33 @@ class PaddleModel(SerializableBase): def serialize(self, path): from ...io import save_persistables - save_persistables(executor=self._exe, - dirname=path, - main_program=self._program, - filename=self._file_name) + + save_persistables( + executor=self._exe, + dirname=path, + main_program=self._program, + filename=self._file_name, + ) def deserialize(self, path): from ...io import load_persistables - load_persistables(executor=self._exe, - dirname=path, - main_program=self._program, - filename=self._file_name) + load_persistables( + executor=self._exe, + dirname=path, + main_program=self._program, + filename=self._file_name, + ) -class CheckpointSaver(object): +class CheckpointSaver(object): def __init__(self, fs): self._fs = fs self._checkpoint_prefix = "__paddle_checkpoint__" - def save_checkpoint(self, - path, - slists, - trainer_id=None, - local_cache_path=".cache"): + def save_checkpoint( + self, path, slists, trainer_id=None, local_cache_path=".cache" + ): """ Serialize objects in slists to path Return really saved path and checkpoint_no @@ -69,7 +70,8 @@ class CheckpointSaver(object): self._fs.mkdirs(path) else: assert self._fs.is_dir(path), "path:{} must be a directory".format( - path) + path + ) max_no = self._get_last_checkpoint_no(path) if max_no < 0: @@ -81,13 +83,14 @@ class CheckpointSaver(object): saved_path = tmp_path from paddle.distributed.fleet.utils.fs import LocalFS + local_fs = LocalFS() cache_path = None if self._fs.need_upload_download(): - cache_path = "{}/{}.{}.saved_cache".format(local_cache_path, - self._checkpoint_prefix, - max_no) + cache_path = "{}/{}.{}.saved_cache".format( + local_cache_path, self._checkpoint_prefix, max_no + ) if trainer_id is not None: cache_path = "{}.{}".format(cache_path, trainer_id) @@ -95,8 +98,9 @@ class CheckpointSaver(object): if not local_fs.is_exist(cache_path): local_fs.mkdirs(cache_path) else: - assert local_fs.is_dir(cache_path), \ - "cache path:{} must be a directory".format(cache_path) + assert local_fs.is_dir( + cache_path + ), "cache path:{} must be a directory".format(cache_path) saved_path = cache_path @@ -111,13 +115,15 @@ class CheckpointSaver(object): return real_path, max_no - def load_checkpoint(self, - path, - slists, - trainer_id, - local_cache_path=".cache", - checkpoint_no=None, - ignore_empty=True): + def load_checkpoint( + self, + path, + slists, + trainer_id, + local_cache_path=".cache", + checkpoint_no=None, + ignore_empty=True, + ): """ Deserialize objects in slists from path Return really load path @@ -137,11 +143,12 @@ class CheckpointSaver(object): assert checkpoint_no >= 0 from paddle.distributed.fleet.utils.fs import LocalFS + local_fs = LocalFS() if self._fs.need_upload_download(): - cache_path = "{}/{}.{}.load_cache".format(local_cache_path, - self._checkpoint_prefix, - checkpoint_no) + cache_path = "{}/{}.{}.load_cache".format( + local_cache_path, self._checkpoint_prefix, checkpoint_no + ) if trainer_id is not None: cache_path = "{}.{}".format(cache_path, trainer_id) @@ -151,8 +158,9 @@ class CheckpointSaver(object): if local_fs.is_exist(cache_path): local_fs.delete(cache_path) - real_path = "{}/{}.{}".format(path, self._checkpoint_prefix, - checkpoint_no) + real_path = "{}/{}.{}".format( + path, self._checkpoint_prefix, checkpoint_no + ) load_path = real_path if self._fs.need_upload_download(): self._fs.download(real_path, cache_path) @@ -217,8 +225,9 @@ class CheckpointSaver(object): try: n = int(g[1]) if n not in s: - path = "{}/{}.{}".format(root_path, self._checkpoint_prefix, - n) + path = "{}/{}.{}".format( + root_path, self._checkpoint_prefix, n + ) self._fs.delete(path) except Exception as e: print(e) diff --git a/python/paddle/fluid/incubate/data_generator/__init__.py b/python/paddle/fluid/incubate/data_generator/__init__.py index 60bfa9eb110..10e4fba92dd 100644 --- a/python/paddle/fluid/incubate/data_generator/__init__.py +++ b/python/paddle/fluid/incubate/data_generator/__init__.py @@ -31,8 +31,9 @@ class DataGenerator(object): def _set_line_limit(self, line_limit): if not isinstance(line_limit, int): - raise ValueError("line_limit%s must be in int type" % - type(line_limit)) + raise ValueError( + "line_limit%s must be in int type" % type(line_limit) + ) if line_limit < 1: raise ValueError("line_limit can not less than 1") self._line_limit = line_limit @@ -140,7 +141,8 @@ class DataGenerator(object): Return a string data that can be read directly by the datafeed. ''' raise NotImplementedError( - "pls use MultiSlotDataGenerator or PairWiseDataGenerator") + "pls use MultiSlotDataGenerator or PairWiseDataGenerator" + ) def generate_sample(self, line): ''' @@ -172,8 +174,9 @@ class DataGenerator(object): return local_iter ''' raise NotImplementedError( - "Please rewrite this function to return a list or tuple: " + - "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)") + "Please rewrite this function to return a list or tuple: " + + "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)" + ) def generate_batch(self, samples): ''' @@ -214,7 +217,6 @@ class DataGenerator(object): # add more generalized DataGenerator that can adapt user-defined slot # for example, [(name, float_list), (name, str_list), (name, int_list)] class MultiSlotStringDataGenerator(DataGenerator): - def _gen_str(self, line): ''' Further processing the output of the process() function rewritten by @@ -238,7 +240,8 @@ class MultiSlotStringDataGenerator(DataGenerator): if not isinstance(line, list) and not isinstance(line, tuple): raise ValueError( "the output of process() must be in list or tuple type" - "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]") + "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]" + ) output = "" for index, item in enumerate(line): name, elements = item @@ -252,7 +255,6 @@ class MultiSlotStringDataGenerator(DataGenerator): class MultiSlotDataGenerator(DataGenerator): - def _gen_str(self, line): ''' Further processing the output of the process() function rewritten by @@ -281,7 +283,8 @@ class MultiSlotDataGenerator(DataGenerator): if not isinstance(line, list) and not isinstance(line, tuple): raise ValueError( "the output of process() must be in list or tuple type" - "Example: [('words', [1926, 08, 17]), ('label', [1])]") + "Example: [('words', [1926, 08, 17]), ('label', [1])]" + ) output = "" if self._proto_info is None: @@ -291,8 +294,9 @@ class MultiSlotDataGenerator(DataGenerator): if not isinstance(name, str): raise ValueError("name%s must be in str type" % type(name)) if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) + raise ValueError( + "elements%s must be in list type" % type(elements) + ) if not elements: raise ValueError( "the elements of each field can not be empty, you need padding it in process()." @@ -305,10 +309,12 @@ class MultiSlotDataGenerator(DataGenerator): if isinstance(elem, float): self._proto_info[-1] = (name, "float") elif not isinstance(elem, int) and not isinstance( - elem, long): + elem, long + ): raise ValueError( - "the type of element%s must be in int or float" % - type(elem)) + "the type of element%s must be in int or float" + % type(elem) + ) output += " " + str(elem) else: if len(line) != len(self._proto_info): @@ -320,8 +326,9 @@ class MultiSlotDataGenerator(DataGenerator): if not isinstance(name, str): raise ValueError("name%s must be in str type" % type(name)) if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) + raise ValueError( + "elements%s must be in list type" % type(elements) + ) if not elements: raise ValueError( "the elements of each field can not be empty, you need padding it in process()." @@ -329,7 +336,8 @@ class MultiSlotDataGenerator(DataGenerator): if name != self._proto_info[index][0]: raise ValueError( "the field name of two given line are not match: require<%s>, get<%s>." - % (self._proto_info[index][0], name)) + % (self._proto_info[index][0], name) + ) if output: output += " " output += str(len(elements)) @@ -338,9 +346,11 @@ class MultiSlotDataGenerator(DataGenerator): if isinstance(elem, float): self._proto_info[index] = (name, "float") elif not isinstance(elem, int) and not isinstance( - elem, long): + elem, long + ): raise ValueError( "the type of element%s must be in int or float" - % type(elem)) + % type(elem) + ) output += " " + str(elem) return output + "\n" diff --git a/python/paddle/fluid/incubate/fleet/base/fleet_base.py b/python/paddle/fluid/incubate/fleet/base/fleet_base.py index f49dbafb554..ea03e930560 100644 --- a/python/paddle/fluid/incubate/fleet/base/fleet_base.py +++ b/python/paddle/fluid/incubate/fleet/base/fleet_base.py @@ -21,7 +21,9 @@ from paddle.optimizer import SGD as SGD_v2 from paddle.fluid.incubate.fleet.base.mode import Mode from paddle.distributed.fleet.base.role_maker import RoleMakerBase -from paddle.fluid.contrib.mixed_precision.decorator import OptimizerWithMixedPrecision +from paddle.fluid.contrib.mixed_precision.decorator import ( + OptimizerWithMixedPrecision, +) from . import mode __all__ = ['Fleet', 'DistributedOptimizer'] @@ -38,6 +40,7 @@ class Fleet(object): Returns: None """ + __metaclass__ = abc.ABCMeta def __init__(self, mode): @@ -179,7 +182,7 @@ class Fleet(object): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] @@ -199,10 +202,14 @@ class Fleet(object): self._executor = Executor(fluid.CPUPlace()) if role_maker and not isinstance(role_maker, RoleMakerBase): - from paddle.fluid.incubate.fleet.base.role_maker import RoleMakerBase as RoleMakerBaseIncubate + from paddle.fluid.incubate.fleet.base.role_maker import ( + RoleMakerBase as RoleMakerBaseIncubate, + ) + if role_maker and not isinstance(role_maker, RoleMakerBaseIncubate): raise TypeError( - "role_maker must be an instance of RoleMakerBase") + "role_maker must be an instance of RoleMakerBase" + ) self._role_maker = role_maker self._role_maker.generate_role() @@ -245,13 +252,15 @@ class Fleet(object): pass @abc.abstractmethod - def save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + ): pass @abc.abstractmethod @@ -277,24 +286,29 @@ class DistributedOptimizer(object): None """ + __metaclass__ = abc.ABCMeta def __init__(self, optimizer, strategy=None): - if not isinstance(optimizer, SGD.__bases__) \ - and not isinstance(optimizer, OptimizerWithMixedPrecision) \ - and not isinstance(optimizer, SGD_v2.__base__): + if ( + not isinstance(optimizer, SGD.__bases__) + and not isinstance(optimizer, OptimizerWithMixedPrecision) + and not isinstance(optimizer, SGD_v2.__base__) + ): raise TypeError("optimizer must be an instance of Optimizer") self._optimizer = optimizer self._strategy = strategy @abc.abstractmethod - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ First part of `minimize`, do auto-diff to append backward ops for the current program. @@ -341,12 +355,14 @@ class DistributedOptimizer(object): pass @abc.abstractmethod - def minimize(self, - losses, - scopes=None, - startup_programs=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, + losses, + scopes=None, + startup_programs=None, + parameter_list=None, + no_grad_set=None, + ): """ Add operations to minimize `loss` by updating `parameter_list`. diff --git a/python/paddle/fluid/incubate/fleet/base/mode.py b/python/paddle/fluid/incubate/fleet/base/mode.py index a940630d4c6..80bd7e6b4b5 100644 --- a/python/paddle/fluid/incubate/fleet/base/mode.py +++ b/python/paddle/fluid/incubate/fleet/base/mode.py @@ -19,6 +19,7 @@ class Mode: """ There are various mode for fleet, each of them is designed for different model. """ + TRANSPILER = 1 PSLIB = 2 COLLECTIVE = 3 diff --git a/python/paddle/fluid/incubate/fleet/base/role_maker.py b/python/paddle/fluid/incubate/fleet/base/role_maker.py index 31a20a409b6..3349dcf275b 100644 --- a/python/paddle/fluid/incubate/fleet/base/role_maker.py +++ b/python/paddle/fluid/incubate/fleet/base/role_maker.py @@ -19,8 +19,13 @@ import os import time __all__ = [ - 'Role', 'RoleMakerBase', 'MPISymetricRoleMaker', 'UserDefinedRoleMaker', - 'UserDefinedCollectiveRoleMaker', 'PaddleCloudRoleMaker', 'GeneralRoleMaker' + 'Role', + 'RoleMakerBase', + 'MPISymetricRoleMaker', + 'UserDefinedRoleMaker', + 'UserDefinedCollectiveRoleMaker', + 'PaddleCloudRoleMaker', + 'GeneralRoleMaker', ] @@ -145,8 +150,11 @@ class RoleMakerBase(object): def to_string(self): return "role: {}, current_id: {}, worker_endpoints: {}, server_endpoints: {}".format( - self._role, self._current_id, self._worker_endpoints, - self._server_endpoints) + self._role, + self._current_id, + self._worker_endpoints, + self._server_endpoints, + ) def all_gather(self, input): """ @@ -196,6 +204,7 @@ class MPIRoleMaker(RoleMakerBase): """Init.""" super(MPIRoleMaker, self).__init__() from mpi4py import MPI + self.MPI = MPI self._comm = MPI.COMM_WORLD self._node_type_comm = None @@ -251,6 +260,7 @@ class MPIRoleMaker(RoleMakerBase): def get_local_ip(self): """Return get local ip.""" import socket + self._ip = socket.gethostbyname(socket.gethostname()) return self._ip @@ -347,6 +357,7 @@ class MPISymetricRoleMaker(MPIRoleMaker): """ if self._pserver_rand_port <= 0: import random + random.seed(self._server_num()) # port will be randomly generated from 60001 to 63999 # random seed is server num so that all nodes will get @@ -495,7 +506,8 @@ class PaddleCloudRoleMaker(RoleMakerBase): # Environment variable PADDLE_PSERVERS_IP_PORT_LIST must be set # format: string(ip:port), eg. 127.0.0.1:6001 eplist = os.environ["PADDLE_PSERVERS_IP_PORT_LIST"].split( - ",") + "," + ) # note that, we usually assign the same port to different ips # if we run parameter server training in local mode # port should be different in environment variables @@ -505,7 +517,8 @@ class PaddleCloudRoleMaker(RoleMakerBase): if training_role not in ["TRAINER", "PSERVER"]: raise ValueError( - "TRAINING_ROLE must be PSERVER or TRAINER") + "TRAINING_ROLE must be PSERVER or TRAINER" + ) if training_role == "TRAINER": role = Role.WORKER @@ -518,7 +531,8 @@ class PaddleCloudRoleMaker(RoleMakerBase): current_id = eplist.index(curr_endpoint) else: raise ValueError( - "TRAINING_ROLE must be PSERVER or TRAINER") + "TRAINING_ROLE must be PSERVER or TRAINER" + ) except ValueError as ve: raise ValueError( "something wrong with PaddleCloud, please check environment" @@ -530,12 +544,15 @@ class PaddleCloudRoleMaker(RoleMakerBase): self._current_id = current_id else: self._current_id = int(os.getenv("PADDLE_TRAINER_ID", "0")) - self._training_role = os.getenv("PADDLE_TRAINING_ROLE", - "TRAINER") - assert (self._training_role == "TRAINER") + self._training_role = os.getenv( + "PADDLE_TRAINING_ROLE", "TRAINER" + ) + assert self._training_role == "TRAINER" self._worker_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS") self._current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT") - assert self._worker_endpoints is not None, "can't find PADDLE_TRAINER_ENDPOINTS" + assert ( + self._worker_endpoints is not None + ), "can't find PADDLE_TRAINER_ENDPOINTS" self._worker_endpoints = self._worker_endpoints.split(",") self._trainers_num = len(self._worker_endpoints) @@ -630,7 +647,8 @@ class GeneralRoleMaker(RoleMakerBase): self._is_barrier_all = 1 if "PADDLE_IS_BARRIER_ALL_ROLE" in os.environ: self._is_barrier_all = int( - os.environ["PADDLE_IS_BARRIER_ALL_ROLE"]) + os.environ["PADDLE_IS_BARRIER_ALL_ROLE"] + ) if training_role == "TRAINER": role = Role.WORKER current_id = int(os.environ["PADDLE_TRAINER_ID"]) @@ -638,12 +656,13 @@ class GeneralRoleMaker(RoleMakerBase): size_d = { "trainer": len(worker_endpoints), "pserver": len(eplist), - "all": len(worker_endpoints) + len(eplist) + "all": len(worker_endpoints) + len(eplist), } # child process for http server - self._http_server = Process(target=self.__start_kv_server, - args=(self._http_server_d, - size_d)) + self._http_server = Process( + target=self.__start_kv_server, + args=(self._http_server_d, size_d), + ) self._http_server.daemon = True # set running status to True self._http_server_d["running"] = True @@ -657,15 +676,21 @@ class GeneralRoleMaker(RoleMakerBase): gloo.set_size(len(worker_endpoints)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) if len(self._http_ip_port) != 0: - gloo.set_http_store(self._http_ip_port[0], - int(self._http_ip_port[1]), - "trainer") + gloo.set_http_store( + self._http_ip_port[0], + int(self._http_ip_port[1]), + "trainer", + ) else: - gloo.set_hdfs_store(self._hdfs_path + "/trainer", - self._hdfs_name, self._hdfs_ugi) + gloo.set_hdfs_store( + self._hdfs_path + "/trainer", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo if self._use_ps_gpu or self._use_metric: @@ -676,7 +701,9 @@ class GeneralRoleMaker(RoleMakerBase): Gloo_strategy.ip_port = int(self._http_ip_port[1]) Default_init_timeout_seconds = 3600 Default_run_timeout_seconds = 9999999 - Gloo_strategy.init_seconds = Default_init_timeout_seconds + Gloo_strategy.init_seconds = ( + Default_init_timeout_seconds + ) Gloo_strategy.run_seconds = Default_run_timeout_seconds Gloo = fluid.core.GlooParallelContext(Gloo_strategy) Gloo.init() @@ -700,14 +727,21 @@ class GeneralRoleMaker(RoleMakerBase): gloo.set_size(len(eplist)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) if len(self._http_ip_port) != 0: - gloo.set_http_store(self._http_ip_port[0], - int(self._http_ip_port[1]), "pserver") + gloo.set_http_store( + self._http_ip_port[0], + int(self._http_ip_port[1]), + "pserver", + ) else: - gloo.set_hdfs_store(self._hdfs_path + "/pserver", - self._hdfs_name, self._hdfs_ugi) + gloo.set_hdfs_store( + self._hdfs_path + "/pserver", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo @@ -717,14 +751,17 @@ class GeneralRoleMaker(RoleMakerBase): gloo.set_size(len(all_list)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) if len(self._http_ip_port) != 0: - gloo.set_http_store(self._http_ip_port[0], - int(self._http_ip_port[1]), "all") + gloo.set_http_store( + self._http_ip_port[0], int(self._http_ip_port[1]), "all" + ) else: - gloo.set_hdfs_store(self._hdfs_path + "/all", self._hdfs_name, - self._hdfs_ugi) + gloo.set_hdfs_store( + self._hdfs_path + "/all", self._hdfs_name, self._hdfs_ugi + ) gloo.init() self._all_comm = gloo self._trainers_num = trainers_num @@ -977,8 +1014,12 @@ class GeneralRoleMaker(RoleMakerBase): gateway = None if len(item) > gateway_idx: gateway = item[gateway_idx] - if gateway and gateway != '*' and gateway != "0.0.0.0" and len( - item) > iface_idx: + if ( + gateway + and gateway != '*' + and gateway != "0.0.0.0" + and len(item) > iface_idx + ): return item[iface_idx] return "lo" @@ -986,8 +1027,9 @@ class GeneralRoleMaker(RoleMakerBase): """ get default physical interface """ - res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( - "\n") + res = ( + os.popen("ip -f inet addr | awk NR%3==1").read().strip().split("\n") + ) for item in res: if "BROADCAST" in item: return item.split(":")[1].strip() @@ -995,6 +1037,7 @@ class GeneralRoleMaker(RoleMakerBase): def __start_kv_server(self, http_server_d, size_d): from paddle.fluid.incubate.fleet.utils.http_server import KVServer + http_server = KVServer(int(self._http_ip_port[1]), size_d) http_server.start() wait_seconds = 5 @@ -1028,7 +1071,8 @@ class HeterRoleMaker(GeneralRoleMaker): xpu_num = len(xpu_endpoints) if training_role not in ["TRAINER", "PSERVER", "XPU"]: raise ValueError( - "TRAINING_ROLE must be PSERVER or TRAINER or XPU") + "TRAINING_ROLE must be PSERVER or TRAINER or XPU" + ) if training_role == "TRAINER": role = Role.WORKER current_id = int(os.environ["PADDLE_TRAINER_ID"]) @@ -1040,11 +1084,14 @@ class HeterRoleMaker(GeneralRoleMaker): gloo.set_size(len(worker_endpoints)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/trainer", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/trainer", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo elif training_role == "XPU": @@ -1058,11 +1105,14 @@ class HeterRoleMaker(GeneralRoleMaker): gloo.set_size(len(xpu_endpoints)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/xpu", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/xpu", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo elif training_role == "PSERVER": @@ -1083,11 +1133,14 @@ class HeterRoleMaker(GeneralRoleMaker): gloo.set_size(len(eplist)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/pserver", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/pserver", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo @@ -1099,11 +1152,14 @@ class HeterRoleMaker(GeneralRoleMaker): gloo.set_size(len(heter_list)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/heter", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/heter", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._heter_comm = gloo @@ -1114,11 +1170,14 @@ class HeterRoleMaker(GeneralRoleMaker): gloo.set_size(len(all_list)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/all", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/all", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._all_comm = gloo @@ -1167,8 +1226,7 @@ class HeterRoleMaker(GeneralRoleMaker): self._heter_comm.barrier() def xpu_num(self): - """ - """ + """ """ if not self._role_is_generated: self.generate_role() return len(self._xpu_endpoints) @@ -1181,18 +1239,21 @@ class UserDefinedRoleMaker(RoleMakerBase): on each physical node, It can be assign by user. """ - def __init__(self, - current_id=0, - role=Role.WORKER, - worker_num=0, - server_endpoints=None): + def __init__( + self, + current_id=0, + role=Role.WORKER, + worker_num=0, + server_endpoints=None, + ): super(UserDefinedRoleMaker, self).__init__() if not isinstance(server_endpoints, list): raise TypeError("server_endpoints must be as string list") elif len(server_endpoints) <= 0: raise ValueError( - "the length of server_endpoints list must be greater than 0") + "the length of server_endpoints list must be greater than 0" + ) elif len(server_endpoints) != len(set(server_endpoints)): raise ValueError("server_endpoints can't have duplicate elements") else: @@ -1213,9 +1274,11 @@ class UserDefinedRoleMaker(RoleMakerBase): else: if current_id < 0: raise ValueError( - "current_id must be greater than or equal to 0") + "current_id must be greater than or equal to 0" + ) elif self._role == Role.SERVER and current_id >= len( - server_endpoints): + server_endpoints + ): raise ValueError( "if role is Role.SERVER, current_id must be less than or equal to len(server_endpoints) - 1" ) @@ -1263,7 +1326,8 @@ class UserDefinedCollectiveRoleMaker(RoleMakerBase): raise TypeError("worker_endpoints must be as string list") elif len(worker_endpoints) <= 0: raise ValueError( - "the length of worker_endpoints list must be greater than 0") + "the length of worker_endpoints list must be greater than 0" + ) elif len(worker_endpoints) != len(set(worker_endpoints)): raise ValueError("worker_endpoints can't have duplicate elements") else: @@ -1279,7 +1343,8 @@ class UserDefinedCollectiveRoleMaker(RoleMakerBase): else: if current_id < 0: raise ValueError( - "current_id must be greater than or equal to 0") + "current_id must be greater than or equal to 0" + ) elif current_id >= len(worker_endpoints): raise ValueError( "current_id must be less than or equal to len(worker_endpoints) - 1" diff --git a/python/paddle/fluid/incubate/fleet/collective/__init__.py b/python/paddle/fluid/incubate/fleet/collective/__init__.py index 8887a75f1d3..229d6e44bd6 100644 --- a/python/paddle/fluid/incubate/fleet/collective/__init__.py +++ b/python/paddle/fluid/incubate/fleet/collective/__init__.py @@ -26,7 +26,10 @@ from paddle.fluid.incubate.fleet.base.fleet_base import Mode from paddle.fluid.incubate.fleet.base.fleet_base import DistributedOptimizer from paddle.fluid import compiler -from paddle.fluid.incubate.checkpoint.checkpoint_saver import PaddleModel, CheckpointSaver +from paddle.fluid.incubate.checkpoint.checkpoint_saver import ( + PaddleModel, + CheckpointSaver, +) import paddle @@ -38,19 +41,16 @@ import shutil class LambConfig(object): - def __init__(self): pass class DistFCConfig(object): - def __init__(self): pass class Collective(Fleet): - def __init__(self): super(Collective, self).__init__(Mode.COLLECTIVE) self._local_ip = 0 @@ -64,60 +64,73 @@ class Collective(Fleet): def init_worker(self): logging.warn( - "You should not call 'init_worker' method for collective mode.") + "You should not call 'init_worker' method for collective mode." + ) def run_worker(self, main_programs=None, scopes=None): logging.warn( - "You should not call 'run_worker' method for collective mode.") + "You should not call 'run_worker' method for collective mode." + ) def init_server(self, model_dir=None): logging.warn( - "You should not call 'init_server' method for collective mode.") + "You should not call 'init_server' method for collective mode." + ) def run_server(self): logging.warn( - "You should not call 'run_server' method for collective mode.") + "You should not call 'run_server' method for collective mode." + ) def stop_worker(self): logging.warn( - "You should not call 'stop_worker' method for collective mode.") + "You should not call 'stop_worker' method for collective mode." + ) def distributed_optimizer(self, optimizer, strategy=None): - self._optimizer = \ - CollectiveOptimizer(optimizer, strategy) + self._optimizer = CollectiveOptimizer(optimizer, strategy) return self._optimizer - def save_inference_model(self, - executor, - dirname, - feeded_var_names=None, - target_vars=None, - main_program=None, - export_for_deployment=True): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names=None, + target_vars=None, + main_program=None, + export_for_deployment=True, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. """ - assert isinstance(executor, Executor), \ - "In fleet.save_inference_model() function, executor must be as" \ + assert isinstance(executor, Executor), ( + "In fleet.save_inference_model() function, executor must be as" " Executor type." + ) if main_program is None: main_program = self._origin_program - assert isinstance(main_program, Program), \ - "In fleet.save_inference_model() function, main_program " \ + assert isinstance(main_program, Program), ( + "In fleet.save_inference_model() function, main_program " "must be as Program type." - - io.save_inference_model(dirname, feeded_var_names, target_vars, - executor, main_program, None, None, - export_for_deployment) - - def save_persistables(self, - executor, - dirname, - main_program=None, - filename=None): + ) + + io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + main_program, + None, + None, + export_for_deployment, + ) + + def save_persistables( + self, executor, dirname, main_program=None, filename=None + ): """ This function filters out all variables with `persistable==True` from the give `main_program` and then saves these variables to the folder @@ -128,28 +141,32 @@ class Collective(Fleet): files, set `filename` None; if you would like to save all variables in a single file, use `filename` to specify the file name. """ - assert isinstance(executor, Executor), \ - "In fleet.save_inference_model() function, executor must be as" \ + assert isinstance(executor, Executor), ( + "In fleet.save_inference_model() function, executor must be as" " Executor type." + ) if main_program is None: main_program = self._origin_program - assert isinstance(main_program, Program), \ - "In fleet.save_inference_model() function, main_program " \ + assert isinstance(main_program, Program), ( + "In fleet.save_inference_model() function, main_program " "must be as Program type." + ) io.save_persistables(executor, dirname, main_program, filename=filename) - def save_checkpoint(self, - executor, - path, - trainer_id, - train_status, - fs, - main_program=None, - local_cache_path=".cache", - remain_all_checkpoint=True): + def save_checkpoint( + self, + executor, + path, + trainer_id, + train_status, + fs, + main_program=None, + local_cache_path=".cache", + remain_all_checkpoint=True, + ): """ This function save persistables and current epoch num to path. """ @@ -163,22 +180,25 @@ class Collective(Fleet): path=path, slists=[m, t], trainer_id=trainer_id, - local_cache_path=local_cache_path) + local_cache_path=local_cache_path, + ) if not remain_all_checkpoint: c.clean_redundant_checkpoints(path) return real_path, checkpoint_no - def load_checkpoint(self, - executor, - path, - trainer_id, - train_status, - fs, - main_program=None, - local_cache_path=".cache", - ignore_empty=True): + def load_checkpoint( + self, + executor, + path, + trainer_id, + train_status, + fs, + main_program=None, + local_cache_path=".cache", + ignore_empty=True, + ): """ This function load persistables and current epoch num from path. """ @@ -188,10 +208,13 @@ class Collective(Fleet): m = PaddleModel(executor, main_program) c = CheckpointSaver(fs) - return c.load_checkpoint(path, [m, train_status], - trainer_id=trainer_id, - ignore_empty=ignore_empty, - local_cache_path=local_cache_path) + return c.load_checkpoint( + path, + [m, train_status], + trainer_id=trainer_id, + ignore_empty=ignore_empty, + local_cache_path=local_cache_path, + ) fleet = Collective() @@ -230,18 +253,21 @@ class CollectiveOpBasedOptimizer(DistributedOptimizer): def __init__(self, optimizer, strategy=None): assert isinstance( - strategy, - DistributedStrategy), "strategy must be DistributedStrategy" + strategy, DistributedStrategy + ), "strategy must be DistributedStrategy" super(CollectiveOpBasedOptimizer, self).__init__(optimizer, strategy) - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self._optimizer.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self._optimizer.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): return self._optimizer.apply_gradients(params_grads) @@ -263,22 +289,26 @@ class CollectiveOptimizer(DistributedOptimizer): strategy = DistributedStrategy() super(CollectiveOptimizer, self).__init__(optimizer, strategy) self._forward_recompute = strategy.forward_recompute - if (not isinstance(strategy.recompute_checkpoints, list)): - raise ValueError("DistStrategy.recompute_checkpoints should" - "be a List") + if not isinstance(strategy.recompute_checkpoints, list): + raise ValueError( + "DistStrategy.recompute_checkpoints should" "be a List" + ) self._recompute_checkpoints = strategy.recompute_checkpoints self._use_amp = strategy.use_amp self._amp_loss_scaling = strategy.amp_loss_scaling self.print_config = False - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self._optimizer.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self._optimizer.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): return self._optimizer.apply_gradients(params_grads) @@ -295,29 +325,40 @@ class CollectiveOptimizer(DistributedOptimizer): if strategy.use_local_sgd: strategy.mode = "collective" strategy.collective_mode = "local_sgd" - self._check_condition("use_local_sgd", - use_dgc=main_program._enable_dgc, - use_dist_fc=strategy.use_dist_fc, - use_lamb=main_program._use_lamb) + self._check_condition( + "use_local_sgd", + use_dgc=main_program._enable_dgc, + use_dist_fc=strategy.use_dist_fc, + use_lamb=main_program._use_lamb, + ) if strategy.use_dist_fc: - self._check_condition("use_dist_fc", - use_dgc=main_program._enable_dgc, - use_local_sgd=strategy.use_local_sgd, - use_lamb=main_program._use_lamb) - assert strategy.dist_fc_config is not None, "DistributedStrategy.dist_fc_config should be set" + self._check_condition( + "use_dist_fc", + use_dgc=main_program._enable_dgc, + use_local_sgd=strategy.use_local_sgd, + use_lamb=main_program._use_lamb, + ) + assert ( + strategy.dist_fc_config is not None + ), "DistributedStrategy.dist_fc_config should be set" if strategy._ut4grad_allreduce: strategy.mode = "collective" strategy.collective_mode = "grad_allreduce" - self._check_condition("_ut4grad_allreduce", - use_dgc=main_program._enable_dgc, - use_lamb=main_program._use_lamb) + self._check_condition( + "_ut4grad_allreduce", + use_dgc=main_program._enable_dgc, + use_lamb=main_program._use_lamb, + ) - if self._strategy.collective_mode=="local_sgd" \ - or self._strategy.collective_mode == "grad_allreduce": - assert self._strategy.mode == "collective", \ - "local_sgd and grad_allreduce can be used under collective mode" + if ( + self._strategy.collective_mode == "local_sgd" + or self._strategy.collective_mode == "grad_allreduce" + ): + assert ( + self._strategy.mode == "collective" + ), "local_sgd and grad_allreduce can be used under collective mode" def _transpile(self, startup_program, main_program): """ @@ -330,9 +371,12 @@ class CollectiveOptimizer(DistributedOptimizer): trainers_num = fleet.worker_num() if self.print_config: - print("worker_endpoints:{} trainers_num:{} current_endpoint:{} \ - trainer_id:{}".format(worker_endpoints, trainers_num, - current_endpoint, trainer_id)) + print( + "worker_endpoints:{} trainers_num:{} current_endpoint:{} \ + trainer_id:{}".format( + worker_endpoints, trainers_num, current_endpoint, trainer_id + ) + ) # call transpiler config = dist_transpiler.DistributeTranspilerConfig() @@ -340,15 +384,21 @@ class CollectiveOptimizer(DistributedOptimizer): config.collective_mode = self._strategy.collective_mode config.nccl_comm_num = self._strategy.nccl_comm_num - config.use_hierarchical_allreduce = self._strategy.use_hierarchical_allreduce - config.hierarchical_allreduce_inter_nranks = self._strategy.hierarchical_allreduce_inter_nranks + config.use_hierarchical_allreduce = ( + self._strategy.use_hierarchical_allreduce + ) + config.hierarchical_allreduce_inter_nranks = ( + self._strategy.hierarchical_allreduce_inter_nranks + ) t = dist_transpiler.DistributeTranspiler(config=config) - t.transpile(trainer_id=trainer_id, - trainers=worker_endpoints_env, - startup_program=startup_program, - program=main_program, - current_endpoint=current_endpoint) + t.transpile( + trainer_id=trainer_id, + trainers=worker_endpoints_env, + startup_program=startup_program, + program=main_program, + current_endpoint=current_endpoint, + ) def _get_node_ips_from_endpoints(self, endpoints): ss = set() @@ -415,11 +465,18 @@ class CollectiveOptimizer(DistributedOptimizer): ) if self.print_config: - print("node_num:", node_num, "num_threads:", - exec_strategy.num_threads, "use_hierarchical_allreduce:", - self._strategy.use_hierarchical_allreduce, "nccl_comm_num:", - self._strategy.nccl_comm_num, "FLAGS_sync_nccl_allreduce:", - sync_allreduce) + print( + "node_num:", + node_num, + "num_threads:", + exec_strategy.num_threads, + "use_hierarchical_allreduce:", + self._strategy.use_hierarchical_allreduce, + "nccl_comm_num:", + self._strategy.nccl_comm_num, + "FLAGS_sync_nccl_allreduce:", + sync_allreduce, + ) self._transpile(startup_program, main_program) @@ -437,19 +494,20 @@ class CollectiveOptimizer(DistributedOptimizer): loss_name=self._loss.name, build_strategy=self._strategy, exec_strategy=self._strategy.exec_strategy, - share_vars_from=None) + share_vars_from=None, + ) return self._compiled_program def raiseOptimizeError(self, strategy_name, optimize_name): - raise ValueError("can not use {0} when you set DistStrategy.{1} " - "as True".format(optimize_name, strategy_name)) - - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + raise ValueError( + "can not use {0} when you set DistStrategy.{1} " + "as True".format(optimize_name, strategy_name) + ) + + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ minimize a program through loss Args: @@ -470,28 +528,36 @@ class CollectiveOptimizer(DistributedOptimizer): # check optimizer conflicts if self._forward_recompute: if self._recompute_checkpoints == []: - raise ValueError("please set strategy.recompute_checkpoints" - "when set strategy.forward_recompute as True") + raise ValueError( + "please set strategy.recompute_checkpoints" + "when set strategy.forward_recompute as True" + ) if self._optimizer.__class__.__name__ in [ - "RecomputeOptimizer", "OptimizerWithMixedPrecision" + "RecomputeOptimizer", + "OptimizerWithMixedPrecision", ]: - self.raiseOptimizeError("forward_recompute", - self._optimizer.__class__.__name__) + self.raiseOptimizeError( + "forward_recompute", self._optimizer.__class__.__name__ + ) - self._optimizer = \ - fluid.optimizer.RecomputeOptimizer(self._optimizer) + self._optimizer = fluid.optimizer.RecomputeOptimizer( + self._optimizer + ) self._optimizer._set_checkpoints(self._recompute_checkpoints) if self._use_amp: if self._optimizer.__class__.__name__ in [ - "OptimizerWithMixedPrecision", "DGCMomentumOptimizer" + "OptimizerWithMixedPrecision", + "DGCMomentumOptimizer", ]: - self.raiseOptimizeError("mixed_precision", - self._optimizer.__class__.__name__) + self.raiseOptimizeError( + "mixed_precision", self._optimizer.__class__.__name__ + ) self._optimizer = fluid.contrib.mixed_precision.decorate( self._optimizer, init_loss_scaling=self._amp_loss_scaling, - use_dynamic_loss_scaling=True) + use_dynamic_loss_scaling=True, + ) main_program = loss.block.program if startup_program is None: @@ -500,11 +566,13 @@ class CollectiveOptimizer(DistributedOptimizer): self._loss = loss - self._check_collective_mode(main_program, self._optimizer, - self._strategy) + self._check_collective_mode( + main_program, self._optimizer, self._strategy + ) optimize_ops, param_grads = self._optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) fleet._origin_program = main_program.clone(for_test=False) fleet._transpiled_program = main_program diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py b/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py index 67c1a0f0f8b..5ba4f927322 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py @@ -29,26 +29,43 @@ from paddle.fluid.executor import Executor from paddle.fluid.parallel_executor import ParallelExecutor from paddle.fluid.optimizer import Optimizer -from paddle.fluid.transpiler.distribute_transpiler import DistributeTranspilerConfig +from paddle.fluid.transpiler.distribute_transpiler import ( + DistributeTranspilerConfig, +) from paddle.fluid.incubate.fleet.base.fleet_base import Fleet from paddle.fluid.incubate.fleet.base.mode import Mode from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker from paddle.fluid.incubate.fleet.parameter_server import version -from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, +) from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_lr_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.public import _has_global_step -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import TrainerRuntimeConfig, DistributedStrategy, \ - SyncStrategy, AsyncStrategy, HalfAsyncStrategy, GeoStrategy, StrategyFactory +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _has_global_step, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + TrainerRuntimeConfig, + DistributedStrategy, + SyncStrategy, + AsyncStrategy, + HalfAsyncStrategy, + GeoStrategy, + StrategyFactory, +) from paddle.fluid.transpiler.details.checkport import wait_server_ready from paddle.fluid.incubate.fleet.parameter_server.mode import PSMode from paddle.fluid.incubate.fleet.base.fleet_base import DistributedOptimizer -from paddle.fluid.incubate.fleet.parameter_server.ir import trainer_pass as worker -from paddle.fluid.incubate.fleet.parameter_server.ir import pserver_pass as server +from paddle.fluid.incubate.fleet.parameter_server.ir import ( + trainer_pass as worker, +) +from paddle.fluid.incubate.fleet.parameter_server.ir import ( + pserver_pass as server, +) from paddle.fluid.incubate.fleet.parameter_server.ir import public as public @@ -105,25 +122,29 @@ class FleetTranspiler(Fleet): def sync_strategy_envs(): kwargs = {} kwargs[ - "pserver_endpoints"] = self._role_maker.get_pserver_endpoints() + "pserver_endpoints" + ] = self._role_maker.get_pserver_endpoints() kwargs["trainer_id"] = self._role_maker.worker_index() return kwargs def geo_strategy_envs(): - def get_sparse_attrs(): opt_init_map = {} opt_init_map["gaussian_random"] = ["seed", "mean", "std"] opt_init_map["fill_constant"] = ["value"] opt_init_map["uniform_random"] = ["seed", "min", "max"] opt_init_map["truncated_gaussian_random"] = [ - "seed", "mean", "std" + "seed", + "mean", + "std", ] - dist_varnames = get_sparse_tablenames(self._origin_main_program, - True) + dist_varnames = get_sparse_tablenames( + self._origin_main_program, True + ) sparse_varnames = get_sparse_tablenames( - self._origin_main_program, False) + self._origin_main_program, False + ) if len(dist_varnames) != 0: raise ValueError( @@ -132,15 +153,18 @@ class FleetTranspiler(Fleet): init_attrs = [] for value_name in sparse_varnames: - value_var = self._origin_main_program.global_block( - ).vars[value_name] + value_var = self._origin_main_program.global_block().vars[ + value_name + ] value_attr = [ value_name, - ",".join([str(dim) for dim in value_var.shape]) + ",".join([str(dim) for dim in value_var.shape]), ] for op in self._origin_startup_program.global_block().ops: - if op.type in opt_init_map.keys( - ) and value_name == op.output("Out")[0]: + if ( + op.type in opt_init_map.keys() + and value_name == op.output("Out")[0] + ): init_attr = [op.type] for attr in opt_init_map[op.type]: init_attr.append(str(op.attr(attr))) @@ -185,15 +209,18 @@ class FleetTranspiler(Fleet): if self.compiled_config.is_geo_mode(): recv_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=4) + recv_type=4 + ) else: recv_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=1) + recv_type=1 + ) from paddle.fluid.communicator import Communicator + self._communicator = Communicator( - trainer_config.mode, kwargs, - trainer_config.get_communicator_flags()) + trainer_config.mode, kwargs, trainer_config.get_communicator_flags() + ) self._communicator.init_with_ctx(send_ctx, recv_ctx) @@ -201,7 +228,8 @@ class FleetTranspiler(Fleet): self._communicator.start() else: raise ValueError( - "Communicator can only be inited once, please check") + "Communicator can only be inited once, please check" + ) def init_worker(self): """ @@ -231,23 +259,31 @@ class FleetTranspiler(Fleet): raise ValueError("There is no directory named '%s'", model_dir) sparse_varnames = self.compiled_config.get_sparse_varname_on_ps( - True) - distribtued_varnames = self.compiled_config.get_sparse_varname_on_ps( - False) + True + ) + distribtued_varnames = ( + self.compiled_config.get_sparse_varname_on_ps(False) + ) remaining_vars = list( filter( - FleetTranspiler.__exclude_vars(sparse_varnames + - distribtued_varnames), - self.main_program.list_vars())) + FleetTranspiler.__exclude_vars( + sparse_varnames + distribtued_varnames + ), + self.main_program.list_vars(), + ) + ) - fluid.io.load_vars(self._executor, - main_program=self.main_program, - dirname=model_dir, - vars=remaining_vars) + fluid.io.load_vars( + self._executor, + main_program=self.main_program, + dirname=model_dir, + vars=remaining_vars, + ) - self._load_sparse_params(dirname=model_dir, - varnames=sparse_varnames) + self._load_sparse_params( + dirname=model_dir, varnames=sparse_varnames + ) # todo(tangwei12) load distributed vars # self._load_sparse_params(dirname=model_dir, varnames=distribtued_varnames) @@ -366,13 +402,15 @@ class FleetTranspiler(Fleet): self._optimizer = ParameterServerOptimizer(optimizer, _strategy) return self._optimizer - def save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. @@ -402,14 +440,28 @@ class FleetTranspiler(Fleet): raise TypeError( "in fleet.save_inference_model() function, main_program must be as Program type, CompiledProgram is not allowed" ) - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, main_program, - None, None, export_for_deployment) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + main_program, + None, + None, + export_for_deployment, + ) else: - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, - self._origin_main_program, None, None, - export_for_deployment, True) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + self._origin_main_program, + None, + None, + export_for_deployment, + True, + ) model_basename = "__model__" model_filename = os.path.join(dirname, model_basename) @@ -423,6 +475,7 @@ class FleetTranspiler(Fleet): def _load_sparse_params(self, dirname, varnames): from paddle.fluid.communicator import LargeScaleKV + scale_kv = LargeScaleKV() for varname in varnames: origin_varname, _, _ = public._get_varname_parts(varname) @@ -431,8 +484,15 @@ class FleetTranspiler(Fleet): def _get_optimizer_status(self, op, param_name): supported_opts = [ - "sgd", "adam", "adagrad", "adamax", "momentum", "lars_momentum", - "rmsprop", "decayed_adagrad", "ftrl" + "sgd", + "adam", + "adagrad", + "adamax", + "momentum", + "lars_momentum", + "rmsprop", + "decayed_adagrad", + "ftrl", ] reshaped_val_map = {} @@ -443,7 +503,9 @@ class FleetTranspiler(Fleet): reshaped_val_map["momentum"] = ["velocity_0"] reshaped_val_map["lars_momentum"] = ["velocity_0"] reshaped_val_map["rmsprop"] = [ - "momentum_0", "mean_square_0", "mean_grad_0" + "momentum_0", + "mean_square_0", + "mean_grad_0", ] reshaped_val_map["decayed_adagrad"] = ["moment_0"] reshaped_val_map["ftrl"] = ["squared_0", "linear_0"] @@ -454,8 +516,10 @@ class FleetTranspiler(Fleet): if op not in supported_opts: raise ValueError( - "fleet can not support optimizer: {}, only this can be supported: {}" - .format(op, supported_opts)) + "fleet can not support optimizer: {}, only this can be supported: {}".format( + op, supported_opts + ) + ) reshaped_names = [ param_name + "_" + val for val in reshaped_val_map[op] @@ -472,8 +536,11 @@ class FleetTranspiler(Fleet): def _get_optimizer_op(self, param_name): opts = public._get_optimize_ops(self._origin_main_program) for op in opts: - if "Param" in op.input_names and \ - "LearningRate" in op.input_names and op.input("Param")[0] == param_name: + if ( + "Param" in op.input_names + and "LearningRate" in op.input_names + and op.input("Param")[0] == param_name + ): return op def _save_dense_params(self, executor, dirname, context, main_program): @@ -492,27 +559,24 @@ class FleetTranspiler(Fleet): optimizer = self._get_optimizer_op(varname) reshaped_varnames, origin_varnames = self._get_optimizer_status( - optimizer.type, varname) + optimizer.type, varname + ) for var_name in [varname] + reshaped_varnames + origin_varnames: var = self._origin_main_program.global_block().vars[var_name] - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self._role_maker.worker_index(), - "shape": - var.shape, - "slice_shapes": - [",".join([str(i) for i in var.shape])], - "slice_varnames": [var.name], - "remote_varnames": [var.name], - "is_sparse": - False, - "endpoints": - var_ctx.split_endpoints(), - "file_path": - os.path.join(dirname, var.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self._role_maker.worker_index(), + "shape": var.shape, + "slice_shapes": [",".join([str(i) for i in var.shape])], + "slice_varnames": [var.name], + "remote_varnames": [var.name], + "is_sparse": False, + "endpoints": var_ctx.split_endpoints(), + "file_path": os.path.join(dirname, var.name), + }, + ) executor.run(prog) return local_vars @@ -531,7 +595,8 @@ class FleetTranspiler(Fleet): optimizer = self._get_optimizer_op(varname) reshaped_varnames, origin_varnames = self._get_optimizer_status( - optimizer.type, varname) + optimizer.type, varname + ) var = self._origin_main_program.global_block().vars[varname] slice_shapes = [] @@ -540,37 +605,34 @@ class FleetTranspiler(Fleet): for section in var_ctx.sections(): slice_shapes.append(str(section) + dims1) - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self._role_maker.worker_index(), - "shape": - var.shape, - "slice_shapes": - slice_shapes, - "slice_varnames": - var_ctx.split_varnames(), - "remote_varnames": - var_ctx.split_varnames(), - "is_sparse": - True, - "endpoints": - var_ctx.split_endpoints(), - "pserver_num": - len(self._role_maker.get_pserver_endpoints()), - "file_path": - os.path.join(dirname, var.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self._role_maker.worker_index(), + "shape": var.shape, + "slice_shapes": slice_shapes, + "slice_varnames": var_ctx.split_varnames(), + "remote_varnames": var_ctx.split_varnames(), + "is_sparse": True, + "endpoints": var_ctx.split_endpoints(), + "pserver_num": len( + self._role_maker.get_pserver_endpoints() + ), + "file_path": os.path.join(dirname, var.name), + }, + ) for reshaped_varname in reshaped_varnames: - var = self._origin_main_program.global_block( - ).vars[reshaped_varname] + var = self._origin_main_program.global_block().vars[ + reshaped_varname + ] slice_varnames = [] remote_varnames = [] for i in range(len(var_ctx.split_varnames())): - slice_varnames.append("{}.block{}".format( - reshaped_varname, i)) + slice_varnames.append( + "{}.block{}".format(reshaped_varname, i) + ) remote_varnames.append(reshaped_varname) block.append_op( @@ -583,97 +645,112 @@ class FleetTranspiler(Fleet): "remote_varnames": remote_varnames, "is_sparse": True, "endpoints": var_ctx.split_endpoints(), - "pserver_num": - len(self._role_maker.get_pserver_endpoints()), - "file_path": os.path.join(dirname, var.name) - }) + "pserver_num": len( + self._role_maker.get_pserver_endpoints() + ), + "file_path": os.path.join(dirname, var.name), + }, + ) for origin_varname in origin_varnames: - var = self._origin_main_program.global_block( - ).vars[origin_varname] - - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self._role_maker.worker_index(), - "shape": - var.shape, - "slice_shapes": - [",".join([str(i) for i in var.shape])], - "slice_varnames": [origin_varname], - "remote_varnames": [origin_varname], - "is_sparse": - False, - "endpoints": - var_ctx.split_endpoints()[:1], - "file_path": - os.path.join(dirname, var.name) - }) + var = self._origin_main_program.global_block().vars[ + origin_varname + ] + + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self._role_maker.worker_index(), + "shape": var.shape, + "slice_shapes": [",".join([str(i) for i in var.shape])], + "slice_varnames": [origin_varname], + "remote_varnames": [origin_varname], + "is_sparse": False, + "endpoints": var_ctx.split_endpoints()[:1], + "file_path": os.path.join(dirname, var.name), + }, + ) executor.run(prog) return context.keys() - def _save_distributed_params(self, executor, dirname, context, - main_program): + def _save_distributed_params( + self, executor, dirname, context, main_program + ): prog = Program() block = prog.global_block() for name, var_ctx in context.items(): - block.append_op(type='checkpoint_notify', - attrs={ - "varname": name, - "is_slice": True, - "slice_varnames": var_ctx.split_varnames(), - "remote_varnames": var_ctx.split_varnames(), - "endpoints": var_ctx.split_endpoints(), - "dirname": dirname - }) + block.append_op( + type='checkpoint_notify', + attrs={ + "varname": name, + "is_slice": True, + "slice_varnames": var_ctx.split_varnames(), + "remote_varnames": var_ctx.split_varnames(), + "endpoints": var_ctx.split_endpoints(), + "dirname": dirname, + }, + ) executor.run(prog) return context.keys() def _save_distributed_persistables(self, executor, dirname, main_program): dense_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=1) + recv_type=1 + ) sparse_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=2) + recv_type=2 + ) distributed_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=3) + recv_type=3 + ) - recv_dense_varnames = self._save_dense_params(executor, dirname, - dense_ctx, main_program) + recv_dense_varnames = self._save_dense_params( + executor, dirname, dense_ctx, main_program + ) - recv_sparse_varnames = self._save_sparse_params(executor, dirname, - sparse_ctx, - main_program) + recv_sparse_varnames = self._save_sparse_params( + executor, dirname, sparse_ctx, main_program + ) recv_distributed_varnames = self._save_distributed_params( - executor, dirname, distributed_ctx, main_program) + executor, dirname, distributed_ctx, main_program + ) - saved_varnames = recv_dense_varnames + list( - recv_sparse_varnames) + list(recv_distributed_varnames) + saved_varnames = ( + recv_dense_varnames + + list(recv_sparse_varnames) + + list(recv_distributed_varnames) + ) remaining_vars = list( - filter(FleetTranspiler.__exclude_vars(saved_varnames), - main_program.list_vars())) + filter( + FleetTranspiler.__exclude_vars(saved_varnames), + main_program.list_vars(), + ) + ) - fluid.io.save_vars(executor, - main_program=main_program, - dirname=dirname, - vars=remaining_vars) + fluid.io.save_vars( + executor, + main_program=main_program, + dirname=dirname, + vars=remaining_vars, + ) def save_persistables(self, executor, dirname, main_program=None, **kwargs): """ - This function filters out all variables with `persistable==True` from the - give `main_program` and then saves these variables to the folder `dirname` - or file `filename`. - - The `dirname` is used to specify the folder where persistable variables - are going to be saved. If you would like to save variables in separate - files, set `filename` None; -if you would like to save all variables in a - single file, use `filename` to specify the file name. + This function filters out all variables with `persistable==True` from the + give `main_program` and then saves these variables to the folder `dirname` + or file `filename`. + + The `dirname` is used to specify the folder where persistable variables + are going to be saved. If you would like to save variables in separate + files, set `filename` None; + if you would like to save all variables in a + single file, use `filename` to specify the file name. """ if self._inner_mode == PSMode.PSLIB: @@ -702,12 +779,12 @@ if you would like to save all variables in a "in fleet.save_persistables() function, main_program must be as Program type, CompiledProgram is not allowed" ) - self._save_distributed_persistables(save_executor, dirname, - main_program) + self._save_distributed_persistables( + save_executor, dirname, main_program + ) @staticmethod def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False @@ -719,9 +796,11 @@ if you would like to save all variables in a if origin_varname == "learning_rate_0": return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -760,7 +839,8 @@ class ParameterServerOptimizer(DistributedOptimizer): "Currently, distributed optimizer only support Adam" "Will config built-in adam for you." "We will support more functions in DistributedOptimizer", - sys.stderr) + sys.stderr, + ) self._optimizer_name = "DistributedAdam" self._optimizer = globals()[self._optimizer_name](optimizer) @@ -770,16 +850,22 @@ class ParameterServerOptimizer(DistributedOptimizer): self._window = 1 self.type = "downpour" self.data_norm_name = [ - ".batch_size", ".batch_square_sum", ".batch_sum", - ".batch_size@GRAD", ".batch_square_sum@GRAD", ".batch_sum@GRAD" + ".batch_size", + ".batch_square_sum", + ".batch_sum", + ".batch_size@GRAD", + ".batch_square_sum@GRAD", + ".batch_sum@GRAD", ] - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): raise NotImplementedError() def apply_gradients(self, params_grads): @@ -799,7 +885,8 @@ class ParameterServerOptimizer(DistributedOptimizer): _startup = worker.fake_init_ops_pass(_startup, compiled_config) _startup = worker.init_from_server_pass(_startup, compiled_config) _startup = worker.delet_extra_optimizes_pass( - _startup, compiled_config) + _startup, compiled_config + ) else: _main = worker.append_send_ops_pass(_main, compiled_config) _startup = _startup @@ -814,57 +901,76 @@ class ParameterServerOptimizer(DistributedOptimizer): _main = server.add_listen_and_serv_pass(_main, compiled_config) _main = server.add_rpc_global_flags_pass(_main, compiled_config) _main = server.add_optimizer_pass(_main, compiled_config) - _main = server.large_scale_sparse_pass(_main, _main, - compiled_config, False) + _main = server.large_scale_sparse_pass( + _main, _main, compiled_config, False + ) _startup = server.build_pserver_startup_program_pass( - _startup, _main, compiled_config) - _startup = server.large_scale_sparse_pass(_startup, _main, - compiled_config, True) + _startup, _main, compiled_config + ) + _startup = server.large_scale_sparse_pass( + _startup, _main, compiled_config, True + ) if not compiled_config.is_sync_mode(): _main = server.delete_unused_in_main_pass( - _main, compiled_config) + _main, compiled_config + ) _startup = server.delete_unused_in_startup_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) else: _main = server.add_listen_and_serv_pass(_main, compiled_config) _main = server.add_rpc_global_flags_pass(_main, compiled_config) _main = server.add_geo_optimizer_pass(_main, compiled_config) - _main = server.large_scale_sparse_pass(_main, _main, - compiled_config, False) + _main = server.large_scale_sparse_pass( + _main, _main, compiled_config, False + ) _startup = server.build_pserver_startup_program_pass( - _startup, _main, compiled_config) - _startup = server.large_scale_sparse_pass(_startup, _main, - compiled_config, True) + _startup, _main, compiled_config + ) + _startup = server.large_scale_sparse_pass( + _startup, _main, compiled_config, True + ) _startup = server.delete_unused_in_startup_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) return _main, _startup - def minimize(self, - losses, - scopes=None, - startup_programs=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, + losses, + scopes=None, + startup_programs=None, + parameter_list=None, + no_grad_set=None, + ): if isinstance(losses, list): raise ValueError("need implement later") - self._optimizer.minimize(losses, startup_programs, parameter_list, - no_grad_set) + self._optimizer.minimize( + losses, startup_programs, parameter_list, no_grad_set + ) fleet._origin_main_program = default_main_program().clone( - for_test=False) + for_test=False + ) fleet._origin_startup_program = default_startup_program().clone( - for_test=False) + for_test=False + ) compiled_config = public.CompileTimeStrategy( - fleet._origin_main_program, fleet._origin_startup_program, - self._strategy, fleet._role_maker) + fleet._origin_main_program, + fleet._origin_startup_program, + self._strategy, + fleet._role_maker, + ) fleet.compiled_config = compiled_config - fleet.main_program, fleet.startup_program = \ - self._build_trainer_programs(compiled_config) if fleet.is_worker() \ + fleet.main_program, fleet.startup_program = ( + self._build_trainer_programs(compiled_config) + if fleet.is_worker() else self._build_pserver_programs(compiled_config) + ) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/distributed_strategy.py b/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/distributed_strategy.py index 8e40fa81ebb..e035b9136c2 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/distributed_strategy.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/distributed_strategy.py @@ -13,45 +13,61 @@ # limitations under the License. __all__ = [ - "TrainerRuntimeConfig", "DistributedStrategy", "SyncStrategy", - "AsyncStrategy", "HalfAsyncStrategy", "GeoStrategy", "StrategyFactory" + "TrainerRuntimeConfig", + "DistributedStrategy", + "SyncStrategy", + "AsyncStrategy", + "HalfAsyncStrategy", + "GeoStrategy", + "StrategyFactory", ] import os import paddle.fluid as fluid -from paddle.fluid.transpiler.distribute_transpiler import DistributeTranspilerConfig, ServerRuntimeConfig +from paddle.fluid.transpiler.distribute_transpiler import ( + DistributeTranspilerConfig, + ServerRuntimeConfig, +) from paddle.fluid.incubate.fleet.parameter_server.mode import DistributedMode class TrainerRuntimeConfig(object): - def __init__(self): self.mode = None num_threads = os.getenv("CPU_NUM", "1") self.runtime_configs = {} self.runtime_configs['communicator_max_merge_var_num'] = os.getenv( - "FLAGS_communicator_max_merge_var_num", num_threads) + "FLAGS_communicator_max_merge_var_num", num_threads + ) self.runtime_configs['communicator_send_queue_size'] = os.getenv( - "FLAGS_communicator_send_queue_size", num_threads) + "FLAGS_communicator_send_queue_size", num_threads + ) self.runtime_configs[ - 'communicator_independent_recv_thread'] = os.getenv( - "FLAGS_communicator_independent_recv_thread", "1") + 'communicator_independent_recv_thread' + ] = os.getenv("FLAGS_communicator_independent_recv_thread", "1") self.runtime_configs[ - 'communicator_min_send_grad_num_before_recv'] = os.getenv( - "FLAGS_communicator_min_send_grad_num_before_recv", num_threads) + 'communicator_min_send_grad_num_before_recv' + ] = os.getenv( + "FLAGS_communicator_min_send_grad_num_before_recv", num_threads + ) self.runtime_configs['communicator_thread_pool_size'] = os.getenv( - "FLAGS_communicator_thread_pool_size", "5") + "FLAGS_communicator_thread_pool_size", "5" + ) self.runtime_configs['communicator_send_wait_times'] = os.getenv( - "FLAGS_communicator_send_wait_times", "5") + "FLAGS_communicator_send_wait_times", "5" + ) self.runtime_configs['communicator_is_sgd_optimizer'] = os.getenv( - "FLAGS_communicator_is_sgd_optimizer", "1") + "FLAGS_communicator_is_sgd_optimizer", "1" + ) # not used self.runtime_configs['rpc_deadline'] = os.getenv( - "FLAGS_rpc_deadline", "180000") + "FLAGS_rpc_deadline", "180000" + ) self.runtime_configs['rpc_retry_times'] = os.getenv( - "FLAGS_rpc_retry_times", "3") + "FLAGS_rpc_retry_times", "3" + ) def get_communicator_flags(self): need_keys = [] @@ -60,45 +76,62 @@ class TrainerRuntimeConfig(object): if self.mode is None or self.mode == DistributedMode.ASYNC: need_keys = self.runtime_configs.keys() mode_str = "async" - elif self.mode == DistributedMode.SYNC or self.mode == DistributedMode.HALF_ASYNC: + elif ( + self.mode == DistributedMode.SYNC + or self.mode == DistributedMode.HALF_ASYNC + ): mode_str = "sync or half_async" need_keys = [ 'communicator_max_merge_var_num', - 'communicator_send_wait_times', 'communicator_thread_pool_size', - 'communicator_send_queue_size' + 'communicator_send_wait_times', + 'communicator_thread_pool_size', + 'communicator_send_queue_size', ] elif self.mode == DistributedMode.GEO: mode_str = "GEO" need_keys = [ - 'communicator_thread_pool_size', 'communicator_send_wait_times', - 'communicator_max_merge_var_num', 'communicator_send_queue_size' + 'communicator_thread_pool_size', + 'communicator_send_wait_times', + 'communicator_max_merge_var_num', + 'communicator_send_queue_size', ] else: raise ValueError("Unsupported Mode") - if self.mode == DistributedMode.SYNC or self.mode == DistributedMode.HALF_ASYNC: + if ( + self.mode == DistributedMode.SYNC + or self.mode == DistributedMode.HALF_ASYNC + ): max_merge_var_num = self.runtime_configs[ - 'communicator_max_merge_var_num'] + 'communicator_max_merge_var_num' + ] send_queue_size = self.runtime_configs[ - 'communicator_send_queue_size'] + 'communicator_send_queue_size' + ] if max_merge_var_num != num_threads: - print('WARNING: In {} mode, communicator_max_merge_var_num ' - 'must be equal to CPU_NUM. But received, ' - 'communicator_max_merge_var_num = {}, CPU_NUM = ' - '{}. communicator_max_merge_var_num will be fored to {}.'. - format(mode_str, max_merge_var_num, num_threads, - num_threads)) + print( + 'WARNING: In {} mode, communicator_max_merge_var_num ' + 'must be equal to CPU_NUM. But received, ' + 'communicator_max_merge_var_num = {}, CPU_NUM = ' + '{}. communicator_max_merge_var_num will be fored to {}.'.format( + mode_str, max_merge_var_num, num_threads, num_threads + ) + ) self.runtime_configs[ - 'communicator_max_merge_var_num'] = num_threads + 'communicator_max_merge_var_num' + ] = num_threads if send_queue_size != num_threads: - print('WARNING: In {} mode, communicator_send_queue_size ' - 'must be equal to CPU_NUM. But received, ' - 'communicator_send_queue_size = {}, CPU_NUM = ' - '{}. communicator_send_queue_size will be fored to {}.'. - format(mode_str, send_queue_size, num_threads, - num_threads)) + print( + 'WARNING: In {} mode, communicator_send_queue_size ' + 'must be equal to CPU_NUM. But received, ' + 'communicator_send_queue_size = {}, CPU_NUM = ' + '{}. communicator_send_queue_size will be fored to {}.'.format( + mode_str, send_queue_size, num_threads, num_threads + ) + ) self.runtime_configs[ - 'communicator_send_queue_size'] = num_threads + 'communicator_send_queue_size' + ] = num_threads return dict((key, str(self.runtime_configs[key])) for key in need_keys) @@ -128,7 +161,6 @@ class TrainerRuntimeConfig(object): class PSLibRuntimeConfig(object): - def __init__(self): self.runtime_configs = {} @@ -137,7 +169,6 @@ class PSLibRuntimeConfig(object): class DistributedStrategy(object): - def __init__(self): self._program_config = DistributeTranspilerConfig() self._trainer_runtime_config = TrainerRuntimeConfig() @@ -150,7 +181,9 @@ class DistributedStrategy(object): self._execute_strategy.num_threads = num_threads if num_threads > 1: - self._build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + self._build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) self.debug_opt = None self.use_ps_gpu = False @@ -162,11 +195,13 @@ class DistributedStrategy(object): if self.debug_opt is not None and isinstance(self.debug_opt, dict): opt_info["dump_slot"] = bool(self.debug_opt.get("dump_slot", 0)) opt_info["dump_converter"] = str( - self.debug_opt.get("dump_converter", "")) + self.debug_opt.get("dump_converter", "") + ) opt_info["dump_fields"] = self.debug_opt.get("dump_fields", []) opt_info["dump_file_num"] = self.debug_opt.get("dump_file_num", 16) opt_info["dump_fields_path"] = self.debug_opt.get( - "dump_fields_path", "") + "dump_fields_path", "" + ) opt_info["dump_param"] = self.debug_opt.get("dump_param", []) return opt_info @@ -182,8 +217,10 @@ class DistributedStrategy(object): setattr(self._program_config, key, config[key]) else: raise ValueError( - "DistributeTranspilerConfig doesn't have key: {}". - format(key)) + "DistributeTranspilerConfig doesn't have key: {}".format( + key + ) + ) else: raise TypeError( "program_config only accept input type: dict or DistributeTranspilerConfig" @@ -207,7 +244,8 @@ class DistributedStrategy(object): self._trainer_runtime_config.runtime_configs[key] = Value else: raise ValueError( - "TrainerRuntimeConfig doesn't have key: {}".format(key)) + "TrainerRuntimeConfig doesn't have key: {}".format(key) + ) else: raise TypeError( "trainer_runtime_config only accept input type: dict or TrainerRuntimeConfig" @@ -237,7 +275,8 @@ class DistributedStrategy(object): setattr(self._server_runtime_config, key, config[key]) else: raise ValueError( - "ServerRuntimeConfig doesn't have key: {}".format(key)) + "ServerRuntimeConfig doesn't have key: {}".format(key) + ) else: raise TypeError( "server_runtime_config only accept input type: dict or ServerRuntimeConfig" @@ -261,7 +300,8 @@ class DistributedStrategy(object): setattr(self._execute_strategy, key, config[key]) else: raise ValueError( - "ExecutionStrategy doesn't have key: {}".format(key)) + "ExecutionStrategy doesn't have key: {}".format(key) + ) else: raise TypeError( "execute_strategy only accept input type: dict or ExecutionStrategy" @@ -285,10 +325,12 @@ class DistributedStrategy(object): setattr(self._build_strategy, key, config[key]) else: raise ValueError( - "BuildStrategy doesn't have key: {}".format(key)) + "BuildStrategy doesn't have key: {}".format(key) + ) else: raise TypeError( - "build_strategy only accept input type: dict or BuildStrategy") + "build_strategy only accept input type: dict or BuildStrategy" + ) self.check_build_strategy() def check_build_strategy(self): @@ -298,7 +340,6 @@ class DistributedStrategy(object): class SyncStrategy(DistributedStrategy): - def __init__(self): super(SyncStrategy, self).__init__() self.check_program_config() @@ -327,7 +368,6 @@ class SyncStrategy(DistributedStrategy): class AsyncStrategy(DistributedStrategy): - def __init__(self): super(AsyncStrategy, self).__init__() self.check_program_config() @@ -354,7 +394,6 @@ class AsyncStrategy(DistributedStrategy): class HalfAsyncStrategy(DistributedStrategy): - def __init__(self): super(HalfAsyncStrategy, self).__init__() self.check_program_config() @@ -382,7 +421,6 @@ class HalfAsyncStrategy(DistributedStrategy): class GeoStrategy(DistributedStrategy): - def __init__(self, update_frequency=100): super(GeoStrategy, self).__init__() self._program_config.geo_sgd_need_push_nums = update_frequency @@ -401,10 +439,12 @@ class GeoStrategy(DistributedStrategy): self._trainer_runtime_config.mode = DistributedMode.GEO self._trainer_runtime_config.runtime_configs[ - 'communicator_send_queue_size'] = self._program_config.geo_sgd_need_push_nums + 'communicator_send_queue_size' + ] = self._program_config.geo_sgd_need_push_nums self._trainer_runtime_config.runtime_configs[ - 'communicator_max_merge_var_num'] = self._program_config.geo_sgd_need_push_nums + 'communicator_max_merge_var_num' + ] = self._program_config.geo_sgd_need_push_nums def check_server_runtime_config(self): pass @@ -417,7 +457,6 @@ class GeoStrategy(DistributedStrategy): class StrategyFactory(object): - def __init_(self): pass diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/heter_trainer_pass.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/heter_trainer_pass.py index bcb83a56370..f799a69e2c8 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/heter_trainer_pass.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/heter_trainer_pass.py @@ -18,13 +18,27 @@ import paddle.fluid.core as core import paddle.fluid.framework as framework from paddle.fluid.transpiler.details.program_utils import delete_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import find_heter_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import union_forward_gradient_op -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import create_heter_program -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import create_trainer_program -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import find_block_joints -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import find_op_input_output -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import get_vars_name_in_block +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + find_heter_ops, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + union_forward_gradient_op, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + create_heter_program, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + create_trainer_program, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + find_block_joints, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + find_op_input_output, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + get_vars_name_in_block, +) def split_heter_worker_ops_pass(program, config, stage_id, device): @@ -36,7 +50,8 @@ def split_heter_worker_ops_pass(program, config, stage_id, device): """ default_deveice = "cpu" program, heter_ops, _, program_block_ops = find_heter_ops( - program, default_deveice) + program, default_deveice + ) if len(heter_ops) == 0: warnings.warn( "Currently running in Heter Parameter Server mode, but no OP running on heterogeneous devices, Please check your code." @@ -46,8 +61,16 @@ def split_heter_worker_ops_pass(program, config, stage_id, device): program_block_ops = union_forward_gradient_op(program_block_ops) block_vars_detail = find_block_joints(program, program_block_ops, heter_ops) heter_program = framework.Program() - create_heter_program(program, config, heter_program, program_block_ops, - heter_ops, block_vars_detail, device, stage_id) + create_heter_program( + program, + config, + heter_program, + program_block_ops, + heter_ops, + block_vars_detail, + device, + stage_id, + ) return heter_program @@ -61,11 +84,13 @@ def split_trainer_ops_pass(program, config, default_device="cpu"): # Todo: support user define default_device (MrChengmo) default_device_ = default_device program, heter_ops, default_ops, program_block_ops = find_heter_ops( - program, default_device_) + program, default_device_ + ) program_block_ops = union_forward_gradient_op(program_block_ops) block_vars_detail = find_block_joints(program, program_block_ops, heter_ops) trainer_program = program.clone() - create_trainer_program(trainer_program, program, config, program_block_ops, - block_vars_detail) + create_trainer_program( + trainer_program, program, config, program_block_ops, block_vars_detail + ) return trainer_program diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/pserver_pass.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/pserver_pass.py index 65d186c855c..1d10bc15168 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/pserver_pass.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/pserver_pass.py @@ -17,12 +17,22 @@ import collections from paddle.fluid import core from paddle.fluid.framework import Block -from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, +) from paddle.fluid.incubate.fleet.parameter_server.ir.public import _orig_varname -from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts -from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_distributed_sparse_op -from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablename -from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_distributed_sparse_op, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablename, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, +) from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_lr_ops LEARNING_RATE_DECAY_COUNTER = "@LR_DECAY_COUNTER@" @@ -33,8 +43,7 @@ LR_SCHED_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.LRSched def _is_optimizer_op(op): - if "Param" in op.input_names and \ - "LearningRate" in op.input_names: + if "Param" in op.input_names and "LearningRate" in op.input_names: return True return False @@ -75,12 +84,12 @@ def _get_optimizer_input_shape(op_type, varkey, orig_shape, param_shape): pass else: raise ValueError( - "Not supported optimizer for distributed training: %s" % op_type) + "Not supported optimizer for distributed training: %s" % op_type + ) return orig_shape def _append_pserver_non_opt_ops(optimize_block, opt_op, origin_program, config): - def _get_pserver_grad_param_var(var, var_dict): """ Return pserver side grad/param variable, return None @@ -120,8 +129,8 @@ def _append_pserver_non_opt_ops(optimize_block, opt_op, origin_program, config): # for ops like clipping and weight decay, get the split var(xxx.block0) # for inputs / outputs grad_block = _get_pserver_grad_param_var( - var, - program.global_block().vars) + var, program.global_block().vars + ) if grad_block: varlist[i] = grad_block elif var.name not in program.global_block().vars: @@ -131,16 +140,17 @@ def _append_pserver_non_opt_ops(optimize_block, opt_op, origin_program, config): varlist[i] = program.global_block().vars[var.name] inputs[key] = varlist - outputs = _get_output_map_from_op(origin_program.global_block().vars, - opt_op) + outputs = _get_output_map_from_op( + origin_program.global_block().vars, opt_op + ) for key, varlist in outputs.items(): if not isinstance(varlist, list): varlist = [varlist] for i in range(len(varlist)): var = varlist[i] grad_block = _get_pserver_grad_param_var( - var, - program.global_block().vars) + var, program.global_block().vars + ) if grad_block: varlist[i] = grad_block elif var.name not in program.global_block().vars: @@ -150,15 +160,24 @@ def _append_pserver_non_opt_ops(optimize_block, opt_op, origin_program, config): varlist[i] = program.global_block().vars[var.name] outputs[key] = varlist - return optimize_block.append_op(type=opt_op.type, - inputs=inputs, - outputs=outputs, - attrs=opt_op.all_attrs()) - - -def _append_pserver_ops(optimize_block, opt_op, endpoint, grad_to_block_id, - origin_program, merged_var, sparse_grad_to_param, - config): + return optimize_block.append_op( + type=opt_op.type, + inputs=inputs, + outputs=outputs, + attrs=opt_op.all_attrs(), + ) + + +def _append_pserver_ops( + optimize_block, + opt_op, + endpoint, + grad_to_block_id, + origin_program, + merged_var, + sparse_grad_to_param, + config, +): program = optimize_block.program pserver_block = program.global_block() new_inputs = collections.OrderedDict() @@ -180,8 +199,10 @@ def _append_pserver_ops(optimize_block, opt_op, endpoint, grad_to_block_id, for pairs in config.merged_variables_pairs: merged_p = pairs[0] if merged_p.merged_var.name == orig_varname: - if merged_p.merged_var.name == merged_p.ordered_vars[ - 0].name: + if ( + merged_p.merged_var.name + == merged_p.ordered_vars[0].name + ): unmerged_vars.append(merged_p.ordered_vars[0]) else: merged_vars.append(merged_p.merged_var) @@ -209,8 +230,10 @@ def _append_pserver_ops(optimize_block, opt_op, endpoint, grad_to_block_id, # because it will create a new tensor for # decayed gradient but not inplace modify the origin one origin_grad_name = opt_op.input(key)[0] - if core.kNewGradSuffix( - ) in origin_grad_name and pserver_block.has_var(origin_grad_name): + if ( + core.kNewGradSuffix() in origin_grad_name + and pserver_block.has_var(origin_grad_name) + ): new_grad = pserver_block.var(origin_grad_name) new_inputs[key] = new_grad else: @@ -220,10 +243,12 @@ def _append_pserver_ops(optimize_block, opt_op, endpoint, grad_to_block_id, if not param_block: return - tmpvar = pserver_block.create_var(name=param_block.name, - persistable=True, - dtype=param_block.dtype, - shape=param_block.shape) + tmpvar = pserver_block.create_var( + name=param_block.name, + persistable=True, + dtype=param_block.dtype, + shape=param_block.shape, + ) new_inputs[key] = tmpvar elif key == "LearningRate": @@ -238,40 +263,52 @@ def _append_pserver_ops(optimize_block, opt_op, endpoint, grad_to_block_id, name=origin_var.name, persistable=origin_var.persistable, dtype=origin_var.dtype, - shape=origin_var.shape) + shape=origin_var.shape, + ) new_inputs[key] = tmpvar for key in opt_op.input_names: new_shape = None if key in [ - "Param", "Grad", "LearningRate", "MasterParam", "Beta1Tensor", - "Beta2Tensor" + "Param", + "Grad", + "LearningRate", + "MasterParam", + "Beta1Tensor", + "Beta2Tensor", ]: continue var = origin_program.global_block().vars[opt_op.input(key)[0]] param_var = new_inputs["Param"] # update accumulator variable shape - new_shape = _get_optimizer_input_shape(opt_op.type, key, var.shape, - param_var.shape) - tmpvar = pserver_block.create_var(name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=new_shape) + new_shape = _get_optimizer_input_shape( + opt_op.type, key, var.shape, param_var.shape + ) + tmpvar = pserver_block.create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=new_shape, + ) new_inputs[key] = tmpvar # change output's ParamOut variable - outputs = _get_output_map_from_op(origin_program.global_block().vars, - opt_op) + outputs = _get_output_map_from_op( + origin_program.global_block().vars, opt_op + ) outputs["ParamOut"] = new_inputs["Param"] - optimize_block.append_op(type=opt_op.type, - inputs=new_inputs, - outputs=outputs, - attrs=opt_op.all_attrs()) + optimize_block.append_op( + type=opt_op.type, + inputs=new_inputs, + outputs=outputs, + attrs=opt_op.all_attrs(), + ) # record sparse grad to param name if new_inputs["Grad"].type == core.VarDesc.VarType.SELECTED_ROWS: sparse_grad_to_param.append( - str(new_inputs["Grad"].name) + ":" + str(new_inputs["Param"].name)) + str(new_inputs["Grad"].name) + ":" + str(new_inputs["Param"].name) + ) def _get_input_map_from_op(varmap, op): @@ -316,7 +353,6 @@ def add_listen_and_serv_pass(program, config): "lr_decay_block_id": None, "dense_optimize_blocks": None, "sparse_optimize_blocks": None, - # runtime attribute "endpoint": config.get_ps_endpoint(), "pserver_id": config.get_role_id(), @@ -324,14 +360,13 @@ def add_listen_and_serv_pass(program, config): "distributed_mode": config.get_distributed_mode(), "rpc_get_thread_num": -1, "rpc_send_thread_num": -1, - "rpc_prefetch_thread_num": -1 + "rpc_prefetch_thread_num": -1, } # step5 append the listen_and_serv op - program.global_block().append_op(type="listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + program.global_block().append_op( + type="listen_and_serv", inputs={'X': []}, outputs={}, attrs=attrs + ) return program @@ -346,7 +381,8 @@ def add_rpc_global_flags_pass(program, config): if get_threads < 1 or send_threads < 1 or pull_threads < 1: raise ValueError( - "error arguments in get_threads/send_threads/pull_threads") + "error arguments in get_threads/send_threads/pull_threads" + ) op._set_attr("rpc_get_thread_num", get_threads) op._set_attr("rpc_send_thread_num", send_threads) @@ -356,18 +392,20 @@ def add_rpc_global_flags_pass(program, config): def _clone_var(block, var, persistable=True): - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=persistable) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=persistable, + ) def add_optimizer_pass(program, config): - - def _append_pserver_grad_merge_ops(optimize_block, grad_varname_for_block, - endpoint, grad_to_block_id): + def _append_pserver_grad_merge_ops( + optimize_block, grad_varname_for_block, endpoint, grad_to_block_id + ): trainers = config.get_trainers() program = optimize_block.program @@ -375,8 +413,7 @@ def add_optimizer_pass(program, config): grad_block = None for g in config.param_grad_ep_mapping[endpoint]["grads"]: - if _orig_varname(g.name) == \ - _orig_varname(grad_varname_for_block): + if _orig_varname(g.name) == _orig_varname(grad_varname_for_block): grad_block = g break @@ -386,41 +423,48 @@ def add_optimizer_pass(program, config): return None orig_varname, block_name, trainer_name = _get_varname_parts( - grad_block.name) + grad_block.name + ) if block_name: merged_var_name = '.'.join([orig_varname, block_name]) else: merged_var_name = orig_varname - merged_var = pserver_block.create_var(name=grad_block.name, - persistable=True, - type=grad_block.type, - dtype=grad_block.dtype, - shape=grad_block.shape) + merged_var = pserver_block.create_var( + name=grad_block.name, + persistable=True, + type=grad_block.type, + dtype=grad_block.dtype, + shape=grad_block.shape, + ) grad_to_block_id.append(merged_var.name + ":" + str(optimize_block.idx)) if config.is_sync_mode() and trainers > 1: vars2merge = [] for i in range(trainers): - per_trainer_name = "%s.trainer_%d" % \ - (merged_var_name, i) + per_trainer_name = "%s.trainer_%d" % (merged_var_name, i) per_trainer_var = pserver_block.create_var( name=per_trainer_name, persistable=False, type=grad_block.type, dtype=grad_block.dtype, - shape=grad_block.shape) + shape=grad_block.shape, + ) vars2merge.append(per_trainer_var) - optimize_block.append_op(type="sum", - inputs={"X": vars2merge}, - outputs={"Out": merged_var}, - attrs={"use_mkldnn": False}) - optimize_block.append_op(type="scale", - inputs={"X": merged_var}, - outputs={"Out": merged_var}, - attrs={"scale": 1.0 / float(trainers)}) + optimize_block.append_op( + type="sum", + inputs={"X": vars2merge}, + outputs={"Out": merged_var}, + attrs={"use_mkldnn": False}, + ) + optimize_block.append_op( + type="scale", + inputs={"X": merged_var}, + outputs={"Out": merged_var}, + attrs={"scale": 1.0 / float(trainers)}, + ) return merged_var origin_program = config.get_origin_main_program() @@ -450,8 +494,10 @@ def add_optimizer_pass(program, config): for pairs in config.merged_variables_pairs: merged_p = pairs[0] if merged_p.merged_var.name == orig_varname: - if merged_p.merged_var.name == merged_p.ordered_vars[ - 0].name: + if ( + merged_p.merged_var.name + == merged_p.ordered_vars[0].name + ): unmerged_varnames.append(merged_p.ordered_vars[0].name) else: merged_varnames.append(merged_p.merged_var.name) @@ -473,9 +519,16 @@ def add_optimizer_pass(program, config): def __append_optimize_op__(op, block, grad_to_block_id, merged_var, lr_ops): if _is_optimizer_op(op): - _append_pserver_ops(block, op, ps_endpoint, grad_to_block_id, - origin_program, merged_var, - sparse_grad_to_param, config) + _append_pserver_ops( + block, + op, + ps_endpoint, + grad_to_block_id, + origin_program, + merged_var, + sparse_grad_to_param, + config, + ) elif op not in lr_ops: _append_pserver_non_opt_ops(block, op, origin_program, config) @@ -506,8 +559,9 @@ def add_optimizer_pass(program, config): lr_decay_block = program._create_block(program.num_blocks - 1) optimize_blocks.append(lr_decay_block) for op in lr_ops: - cloned_op = _append_pserver_non_opt_ops(lr_decay_block, op, - origin_program, config) + cloned_op = _append_pserver_non_opt_ops( + lr_decay_block, op, origin_program, config + ) # append sub blocks to pserver_program in lr_decay_op # todo(tangwei12): __clone_lr_op_sub_block__ lr_decay_block_id = lr_decay_block.idx @@ -529,18 +583,25 @@ def add_optimizer_pass(program, config): grad_varname_for_block = op.attr(OP_ROLE_VAR_ATTR_NAME)[1] if op.attr(OP_ROLE_VAR_ATTR_NAME)[0] == optimize_target_param_name: merged_var = _append_pserver_grad_merge_ops( - per_opt_block, grad_varname_for_block, ps_endpoint, - grad_to_block_id) + per_opt_block, + grad_varname_for_block, + ps_endpoint, + grad_to_block_id, + ) if merged_var: break # append optimize op once then append other ops. if merged_var: for _, op in enumerate(optimize_ops): # optimizer is connected to itself - if op.attr(OP_ROLE_VAR_ATTR_NAME)[0] == optimize_target_param_name and \ - op not in global_ops: - __append_optimize_op__(op, per_opt_block, grad_to_block_id, - merged_var, lr_ops) + if ( + op.attr(OP_ROLE_VAR_ATTR_NAME)[0] + == optimize_target_param_name + and op not in global_ops + ): + __append_optimize_op__( + op, per_opt_block, grad_to_block_id, merged_var, lr_ops + ) # dedup grad to ids list grad_to_block_id = list(set(grad_to_block_id)) @@ -549,8 +610,9 @@ def add_optimizer_pass(program, config): opt_state_block = program._create_block(program.num_blocks - 1) optimize_blocks.append(opt_state_block) for glb_op in global_ops: - __append_optimize_op__(glb_op, opt_state_block, grad_to_block_id, - None, lr_ops) + __append_optimize_op__( + glb_op, opt_state_block, grad_to_block_id, None, lr_ops + ) if len(optimize_blocks) == 0: pre_block_idx = program.num_blocks - 1 @@ -590,8 +652,10 @@ def large_scale_sparse_pass(program, main_program, config, is_startup=False): origin_name = _orig_varname(param_name) o_main_program = config.get_origin_main_program() for op in o_main_program.global_block().ops: - if is_distributed_sparse_op(op) and get_sparse_tablename( - op) == origin_name: + if ( + is_distributed_sparse_op(op) + and get_sparse_tablename(op) == origin_name + ): entry = op.attr("entry") return entry @@ -604,8 +668,10 @@ def large_scale_sparse_pass(program, main_program, config, is_startup=False): for value_name in acture_value_names: origin_var_name = _orig_varname(value_name) for op in o_startup_program.global_block().ops: - if op.type in opt_init_map.keys( - ) and origin_var_name == op.output("Out")[0]: + if ( + op.type in opt_init_map.keys() + and origin_var_name == op.output("Out")[0] + ): init_attr = [op.type] for attr in opt_init_map[op.type]: init_attr.append(str(op.attr(attr))) @@ -646,8 +712,16 @@ def large_scale_sparse_pass(program, main_program, config, is_startup=False): break return grad, opt_idx, value_names, value_dims, acture_names, fuse - def add_fuse_large_scale_op(block, global_block, table_name, value_names, - acture_names, grad, is_entry, opt_idx): + def add_fuse_large_scale_op( + block, + global_block, + table_name, + value_names, + acture_names, + grad, + is_entry, + opt_idx, + ): op = block.ops[opt_idx] @@ -655,99 +729,108 @@ def large_scale_sparse_pass(program, main_program, config, is_startup=False): grad = main_program.global_block().vars[op.input("Grad")[0]] lr = main_program.global_block().vars[op.input("LearningRate")[0]] - block._insert_op(opt_idx, - type="lookup_sparse_table_fuse_sgd", - inputs={ - "Grad": grad, - "LearningRate": lr - }, - attrs={ - "is_entry": is_entry, - "tablename": table_name, - "value_names": value_names - }) + block._insert_op( + opt_idx, + type="lookup_sparse_table_fuse_sgd", + inputs={"Grad": grad, "LearningRate": lr}, + attrs={ + "is_entry": is_entry, + "tablename": table_name, + "value_names": value_names, + }, + ) elif op.type == "adam": grad = main_program.global_block().vars[op.input("Grad")[0]] lr = main_program.global_block().vars[op.input("LearningRate")[0]] - beta1_pow = main_program.global_block().vars[op.input("Beta1Pow") - [0]] - beta2_pow = main_program.global_block().vars[op.input("Beta2Pow") - [0]] - beta1_pow_o = main_program.global_block().vars[op.output( - "Beta1PowOut")[0]] - beta2_pow_o = main_program.global_block().vars[op.output( - "Beta2PowOut")[0]] + beta1_pow = main_program.global_block().vars[ + op.input("Beta1Pow")[0] + ] + beta2_pow = main_program.global_block().vars[ + op.input("Beta2Pow")[0] + ] + beta1_pow_o = main_program.global_block().vars[ + op.output("Beta1PowOut")[0] + ] + beta2_pow_o = main_program.global_block().vars[ + op.output("Beta2PowOut")[0] + ] beta1 = op.attr('beta1') beta2 = op.attr('beta2') epsilon = op.attr('epsilon') - block._insert_op(opt_idx, - type="lookup_sparse_table_fuse_adam", - inputs={ - "Grad": grad, - "LearningRate": lr, - "Beta1Pow": beta1_pow, - "Beta2Pow": beta2_pow - }, - outputs={ - "Beta1PowOut": beta1_pow_o, - "Beta2PowOut": beta2_pow_o - }, - attrs={ - "beta1": beta1, - "beta2": beta2, - "epsilon": epsilon, - "is_entry": is_entry, - "tablename": table_name, - "value_names": value_names - }) + block._insert_op( + opt_idx, + type="lookup_sparse_table_fuse_adam", + inputs={ + "Grad": grad, + "LearningRate": lr, + "Beta1Pow": beta1_pow, + "Beta2Pow": beta2_pow, + }, + outputs={ + "Beta1PowOut": beta1_pow_o, + "Beta2PowOut": beta2_pow_o, + }, + attrs={ + "beta1": beta1, + "beta2": beta2, + "epsilon": epsilon, + "is_entry": is_entry, + "tablename": table_name, + "value_names": value_names, + }, + ) else: raise ValueError("only support sgd/adam optimizer now") - def add_large_scale_op(block, global_block, table_name, value_names, - acture_names, grad, is_entry, opt_idx): - ids = global_block.create_var(name="kSparseIDs@{}".format(table_name), - persistable=False, - dtype="int64", - shape=[1, 1], - lod_level=0) + def add_large_scale_op( + block, + global_block, + table_name, + value_names, + acture_names, + grad, + is_entry, + opt_idx, + ): + ids = global_block.create_var( + name="kSparseIDs@{}".format(table_name), + persistable=False, + dtype="int64", + shape=[1, 1], + lod_level=0, + ) # insert grad split to ids and tensor op - block._insert_op(opt_idx, - type="lookup_sparse_table_grad_split", - inputs={"Grad": grad}, - outputs={ - "Row": ids, - "Value": grad - }, - attrs={ - "tablename": table_name, - "is_entry": is_entry - }) + block._insert_op( + opt_idx, + type="lookup_sparse_table_grad_split", + inputs={"Grad": grad}, + outputs={"Row": ids, "Value": grad}, + attrs={"tablename": table_name, "is_entry": is_entry}, + ) # insert read at first vars = [global_block.vars[acture_name] for acture_name in acture_names] - block._insert_op(opt_idx + 1, - type="lookup_sparse_table_read", - inputs={"Ids": ids}, - outputs={"Out": vars}, - attrs={ - "tablename": table_name, - "value_names": value_names - }) + block._insert_op( + opt_idx + 1, + type="lookup_sparse_table_read", + inputs={"Ids": ids}, + outputs={"Out": vars}, + attrs={"tablename": table_name, "value_names": value_names}, + ) # append write at last inputs = {"Ids": ids, "In": vars} - block.append_op(type="lookup_sparse_table_write", - inputs=inputs, - outputs={}, - attrs={ - "tablename": table_name, - "value_names": value_names - }) + block.append_op( + type="lookup_sparse_table_write", + inputs=inputs, + outputs={}, + attrs={"tablename": table_name, "value_names": value_names}, + ) op = get_op_by_type(main_program.global_block(), "listen_and_serv") @@ -775,27 +858,53 @@ def large_scale_sparse_pass(program, main_program, config, is_startup=False): for param, blockid in param_blockid_map.items(): opt_block = program.block(blockid) - grad, opt_idx, value_names, value_dims, acture_names, fuse = \ - get_optimizer_values(opt_block) + ( + grad, + opt_idx, + value_names, + value_dims, + acture_names, + fuse, + ) = get_optimizer_values(opt_block) entry_attr = get_entry_attr(param) is_entry = False if entry_attr == "none" else True if fuse: - add_fuse_large_scale_op(opt_block, program.global_block(), - param, value_names, acture_names, grad, - is_entry, opt_idx) + add_fuse_large_scale_op( + opt_block, + program.global_block(), + param, + value_names, + acture_names, + grad, + is_entry, + opt_idx, + ) else: - add_large_scale_op(opt_block, program.global_block(), param, - value_names, acture_names, grad, is_entry, - opt_idx) + add_large_scale_op( + opt_block, + program.global_block(), + param, + value_names, + acture_names, + grad, + is_entry, + opt_idx, + ) else: large_scale_kv_metas = [] for param, blockid in param_blockid_map.items(): opt_block = main_program.block(blockid) - grad, opt_idx, value_names, value_dims, acture_names, fuse = \ - get_optimizer_values(opt_block) + ( + grad, + opt_idx, + value_names, + value_dims, + acture_names, + fuse, + ) = get_optimizer_values(opt_block) entry_attr = get_entry_attr(param) @@ -811,10 +920,18 @@ def large_scale_sparse_pass(program, main_program, config, is_startup=False): cached_str = ",".join(acture_names + [ids_name]) init_attr_str = get_initializer_attrs(acture_names) - meta_str = ":".join([ - param, names_str, dims_str, mode, grad.name, cached_str, - init_attr_str, entry_attr - ]) + meta_str = ":".join( + [ + param, + names_str, + dims_str, + mode, + grad.name, + cached_str, + init_attr_str, + entry_attr, + ] + ) print("large_scale_metas: {}".format(meta_str)) large_scale_kv_metas.append(meta_str) @@ -822,7 +939,8 @@ def large_scale_sparse_pass(program, main_program, config, is_startup=False): type="lookup_sparse_table_init", inputs=None, outputs=None, - attrs={"large_scale_metas": large_scale_kv_metas}) + attrs={"large_scale_metas": large_scale_kv_metas}, + ) # todo: need delete unused var. return program @@ -843,7 +961,8 @@ def get_distributed_from_listen_and_serv(program, origin_program): def delete_unused_in_main_pass(program, config): origin_program = config.get_origin_main_program() sparse_params = get_distributed_from_listen_and_serv( - program, origin_program) + program, origin_program + ) for var in sparse_params: if program.global_block().has_var(var): @@ -854,7 +973,8 @@ def delete_unused_in_main_pass(program, config): def delete_unused_in_startup_pass(program, main_program, config): origin_program = config.get_origin_main_program() sparse_params = get_distributed_from_listen_and_serv( - main_program, origin_program) + main_program, origin_program + ) remove_ops = [] for op in program.global_block().ops: @@ -937,15 +1057,19 @@ def build_pserver_startup_program_pass(program, p_main_program, config): new_inputs = _get_input_map_from_op(pserver_vars, op) if op.type in [ - "gaussian_random", "fill_constant", "uniform_random", - "truncated_gaussian_random" + "gaussian_random", + "fill_constant", + "uniform_random", + "truncated_gaussian_random", ]: op._set_attr("shape", list(new_outputs["Out"].shape)) - program.global_block().append_op(type=op.type, - inputs=new_inputs, - outputs=new_outputs, - attrs=op.all_attrs()) + program.global_block().append_op( + type=op.type, + inputs=new_inputs, + outputs=new_outputs, + attrs=op.all_attrs(), + ) return program @@ -954,8 +1078,9 @@ def add_geo_optimizer_pass(program, config): endpoint = config.get_ps_endpoint() params = [p for p in config.param_grad_ep_mapping[endpoint]["params"]] - sparse_tablenames = get_sparse_tablenames(config.get_origin_main_program(), - False) + sparse_tablenames = get_sparse_tablenames( + config.get_origin_main_program(), False + ) for param in params: _clone_var(program.global_block(), param) @@ -978,15 +1103,17 @@ def add_geo_optimizer_pass(program, config): if origin_varname in sparse_tablenames: sparse_grad_to_param.append(":".join([delta_var_name, param.name])) - delta_var = pserver_block.create_var(name=delta_var_name, - persistable=False, - type=param.type, - dtype=param.dtype, - shape=param.shape) - - per_opt_block.append_op(type="sum", - inputs={"X": [param, delta_var]}, - outputs={"Out": param}) + delta_var = pserver_block.create_var( + name=delta_var_name, + persistable=False, + type=param.type, + dtype=param.dtype, + shape=param.shape, + ) + + per_opt_block.append_op( + type="sum", inputs={"X": [param, delta_var]}, outputs={"Out": param} + ) param_to_block_id.append(delta_var_name + ":" + str(per_opt_block.idx)) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/public.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/public.py index 1778ace9c39..9b762bb9ee7 100755 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/public.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/public.py @@ -26,7 +26,10 @@ from paddle.fluid.core import CommContext import paddle.fluid.framework as framework from paddle.fluid.incubate.fleet.parameter_server.mode import DistributedMode from paddle.fluid.incubate.fleet.parameter_server.ir import vars_metatools -from paddle.fluid.incubate.fleet.parameter_server.ir.ps_dispatcher import RoundRobin, PSDispatcher +from paddle.fluid.incubate.fleet.parameter_server.ir.ps_dispatcher import ( + RoundRobin, + PSDispatcher, +) from paddle.fluid.transpiler.details.program_utils import delete_ops OP_NAME_SCOPE = "op_namescope" @@ -49,9 +52,9 @@ def _get_lr_ops(program): lr_ops = [] for index, op in enumerate(program.global_block().ops): role_id = int(op.attr(RPC_OP_ROLE_ATTR_NAME)) - if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or \ - role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) | \ - int(OPT_OP_ROLE_ATTR_VALUE): + if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or role_id == int( + LR_SCHED_OP_ROLE_ATTR_VALUE + ) | int(OPT_OP_ROLE_ATTR_VALUE): lr_ops.append(op) return lr_ops @@ -68,12 +71,17 @@ def _has_global_step(lr_ops): def is_sparse_op(op): - if op.type in SPARSE_OP_LIST and op.attr('is_sparse') is True and op.attr( - 'is_distributed') is False: + if ( + op.type in SPARSE_OP_LIST + and op.attr('is_sparse') is True + and op.attr('is_distributed') is False + ): return True - if op.type == "distributed_lookup_table" and op.attr( - 'is_distributed') is False: + if ( + op.type == "distributed_lookup_table" + and op.attr('is_distributed') is False + ): return True return False @@ -83,8 +91,10 @@ def is_distributed_sparse_op(op): if op.type in SPARSE_OP_LIST and op.attr('is_distributed') is True: return True - if op.type == "distributed_lookup_table" and op.attr( - 'is_distributed') is True: + if ( + op.type == "distributed_lookup_table" + and op.attr('is_distributed') is True + ): return True return False @@ -108,7 +118,6 @@ def get_sparse_tablenames(program, is_distributed): class MergedVariable: - def __init__(self, merged, ordered, offsets): self.merged_var = merged self.ordered_vars = ordered @@ -128,7 +137,6 @@ def Singleton(cls): @Singleton class CompileTimeStrategy(object): - def __init__(self, main_program, startup_program, strategy, role_maker): self.min_block_size = 81920 @@ -268,20 +276,24 @@ class CompileTimeStrategy(object): def get_origin_ps_startup_program(self): return self.origin_ps_startup_program - def add_tensor_table(self, - feed_var_name, - fetch_var_name="", - startup_program=None, - main_program=None, - tensor_table_class=""): + def add_tensor_table( + self, + feed_var_name, + fetch_var_name="", + startup_program=None, + main_program=None, + tensor_table_class="", + ): self.tensor_table_dict[feed_var_name] = {} self.tensor_table_dict[feed_var_name]["feed_var_name"] = feed_var_name self.tensor_table_dict[feed_var_name]["fetch_var_name"] = fetch_var_name self.tensor_table_dict[feed_var_name][ - "startup_program"] = startup_program + "startup_program" + ] = startup_program self.tensor_table_dict[feed_var_name]["main_program"] = main_program self.tensor_table_dict[feed_var_name][ - "tensor_table_class"] = tensor_table_class + "tensor_table_class" + ] = tensor_table_class def get_tensor_table_dict(self): return self.tensor_table_dict @@ -289,8 +301,9 @@ class CompileTimeStrategy(object): def get_sparse_varname_on_ps(self, is_distributed, endpoint=None): if not endpoint: endpoint = self.get_ps_endpoint() - varnames = get_sparse_tablenames(self.get_origin_main_program(), - is_distributed) + varnames = get_sparse_tablenames( + self.get_origin_main_program(), is_distributed + ) ps_sparse_varnames = [] for varname in varnames: @@ -313,13 +326,19 @@ class CompileTimeStrategy(object): # check all input for key in op.input_names: if key in [ - "Param", "Grad", "LearningRate", "Beta1Tensor", - "Beta2Tensor" + "Param", + "Grad", + "LearningRate", + "Beta1Tensor", + "Beta2Tensor", ]: continue # check varibale shape related param, e.g: Moment1 - optimize_var_names += self._get_optimizer_param_related_var_name( - op, op.type, key) + optimize_var_names += ( + self._get_optimizer_param_related_var_name( + op, op.type, key + ) + ) return optimize_var_names def _get_optimizer_param_related_var_name(self, op, op_type, varkey): @@ -346,18 +365,13 @@ class CompileTimeStrategy(object): pass else: raise ValueError( - "Not supported optimizer for distributed training: %s" % - op_type) + "Not supported optimizer for distributed training: %s" % op_type + ) return related_var_names - def build_ctx(self, - vars, - mapping, - is_grad, - is_sparse, - is_send, - is_distributed=False): - + def build_ctx( + self, vars, mapping, is_grad, is_sparse, is_send, is_distributed=False + ): def get_grad_var_ep(slices): names = [] eps = [] @@ -369,10 +383,12 @@ class CompileTimeStrategy(object): names.append("{}.delta".format(slice.name)) else: names.append(slice.name) - elif is_grad and self.is_sync_mode( - ) and self.get_trainers() > 1: - names.append("{}.trainer_{}".format(slice.name, - self.get_role_id())) + elif ( + is_grad and self.is_sync_mode() and self.get_trainers() > 1 + ): + names.append( + "{}.trainer_{}".format(slice.name, self.get_role_id()) + ) else: names.append(slice.name) @@ -400,21 +416,33 @@ class CompileTimeStrategy(object): trainer_id = self.get_role_id() aggregate = True - ctx = CommContext(name, names, eps, sections, origin_varnames, - trainer_id, aggregate, is_sparse, is_distributed, []) + ctx = CommContext( + name, + names, + eps, + sections, + origin_varnames, + trainer_id, + aggregate, + is_sparse, + is_distributed, + [], + ) return ctx def get_trainer_send_context(self): send_ctx = {} - distibuted_varnames = get_sparse_tablenames(self.origin_main_program, - True) + distibuted_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) idx = 0 if not self.is_geo_mode(): for merged in self.merged_dense_pairs: grad = merged[1] - ctx = self.build_ctx(grad, self.grad_var_mapping, True, False, - True) + ctx = self.build_ctx( + grad, self.grad_var_mapping, True, False, True + ) send_ctx[ctx.var_name()] = ctx for merged in self.merged_sparse_pairs: @@ -423,10 +451,18 @@ class CompileTimeStrategy(object): param_name = param.merged_var.name - is_distributed = True if param_name in distibuted_varnames else False - - ctx = self.build_ctx(grad, self.grad_var_mapping, True, True, - True, is_distributed) + is_distributed = ( + True if param_name in distibuted_varnames else False + ) + + ctx = self.build_ctx( + grad, + self.grad_var_mapping, + True, + True, + True, + is_distributed, + ) send_ctx[ctx.var_name()] = ctx idx += 1 @@ -437,21 +473,39 @@ class CompileTimeStrategy(object): for pairs in self.origin_sparse_pairs: param, grad = pairs param_name = param.name - is_distributed = True if param_name in distibuted_varnames else False - - param_ctx = self.build_ctx(param, self.param_var_mapping, False, - True, True, is_distributed) - grad_ctx = self.build_ctx(grad, self.grad_var_mapping, True, - True, True, is_distributed) - - ctx = CommContext(param_ctx.var_name(), - param_ctx.split_varnames(), - param_ctx.split_endpoints(), - param_ctx.sections(), - grad_ctx.origin_varnames(), - param_ctx.trainer_id(), param_ctx.aggregate(), - param_ctx.is_sparse(), - param_ctx.is_distributed(), []) + is_distributed = ( + True if param_name in distibuted_varnames else False + ) + + param_ctx = self.build_ctx( + param, + self.param_var_mapping, + False, + True, + True, + is_distributed, + ) + grad_ctx = self.build_ctx( + grad, + self.grad_var_mapping, + True, + True, + True, + is_distributed, + ) + + ctx = CommContext( + param_ctx.var_name(), + param_ctx.split_varnames(), + param_ctx.split_endpoints(), + param_ctx.sections(), + grad_ctx.origin_varnames(), + param_ctx.trainer_id(), + param_ctx.aggregate(), + param_ctx.is_sparse(), + param_ctx.is_distributed(), + [], + ) send_ctx[ctx.var_name()] = ctx idx += 1 @@ -461,24 +515,34 @@ class CompileTimeStrategy(object): def get_communicator_send_context(self): send_ctx = {} - distibuted_varnames = get_sparse_tablenames(self.origin_main_program, - True) + distibuted_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) idx = 0 if self.is_geo_mode(): for pairs in self.merged_dense_pairs: param = pairs[0] - ctx = self.build_ctx(param, self.param_var_mapping, False, - False, True) + ctx = self.build_ctx( + param, self.param_var_mapping, False, False, True + ) send_ctx[ctx.var_name()] = ctx for pairs in self.merged_sparse_pairs: param = pairs[0] param_name = param.merged_var.name - is_distributed = True if param_name in distibuted_varnames else False - - ctx = self.build_ctx(param, self.param_var_mapping, False, True, - True, is_distributed) + is_distributed = ( + True if param_name in distibuted_varnames else False + ) + + ctx = self.build_ctx( + param, + self.param_var_mapping, + False, + True, + True, + is_distributed, + ) send_ctx[ctx.var_name()] = ctx idx += 1 name, ctx = self._step_ctx(idx) @@ -486,18 +550,27 @@ class CompileTimeStrategy(object): else: for merged in self.merged_dense_pairs: grad = merged[1] - ctx = self.build_ctx(grad, self.grad_var_mapping, True, False, - True) + ctx = self.build_ctx( + grad, self.grad_var_mapping, True, False, True + ) send_ctx[ctx.var_name()] = ctx for merged in self.merged_sparse_pairs: param, grad = merged param_name = param.merged_var.name - is_distributed = True if param_name in distibuted_varnames else False - - ctx = self.build_ctx(grad, self.grad_var_mapping, True, True, - True, is_distributed) + is_distributed = ( + True if param_name in distibuted_varnames else False + ) + + ctx = self.build_ctx( + grad, + self.grad_var_mapping, + True, + True, + True, + is_distributed, + ) send_ctx[ctx.var_name()] = ctx idx += 1 @@ -505,13 +578,14 @@ class CompileTimeStrategy(object): send_ctx[name] = ctx return send_ctx - def get_communicator_recv_context(self, - recv_type=1, - use_origin_program=False): + def get_communicator_recv_context( + self, recv_type=1, use_origin_program=False + ): # recv_type # 1 : DENSE 2. SPARSE 3. DISTRIBUTED 4. ALL - distibuted_varnames = get_sparse_tablenames(self.origin_main_program, - True) + distibuted_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) sparse_varnames = [] for pairs in self.origin_sparse_pairs: param, grad = pairs @@ -521,26 +595,33 @@ class CompileTimeStrategy(object): sparse_recv_ctx = {} distributed_recv_ctx = {} - variables_pairs = self.merged_variables_pairs if not use_origin_program else self.origin_merged_variables_pairs + variables_pairs = ( + self.merged_variables_pairs + if not use_origin_program + else self.origin_merged_variables_pairs + ) for merged in variables_pairs: params = merged[0] if params.merged_var.name in sparse_varnames: continue - ctx = self.build_ctx(params, self.param_var_mapping, False, False, - False, False) + ctx = self.build_ctx( + params, self.param_var_mapping, False, False, False, False + ) dense_recv_ctx[ctx.var_name()] = ctx for pairs in self.origin_sparse_pairs: param, grad = pairs if param.name in distibuted_varnames: - ctx = self.build_ctx(param, self.param_var_mapping, False, True, - False, True) + ctx = self.build_ctx( + param, self.param_var_mapping, False, True, False, True + ) distributed_recv_ctx[ctx.var_name()] = ctx else: - ctx = self.build_ctx(param, self.param_var_mapping, False, True, - False, False) + ctx = self.build_ctx( + param, self.param_var_mapping, False, True, False, False + ) sparse_recv_ctx[ctx.var_name()] = ctx if recv_type == 1: @@ -564,28 +645,44 @@ class CompileTimeStrategy(object): idx = 0 distibuted_varnames = get_sparse_tablenames( - self.origin_main_program, True) + self.origin_main_program, True + ) for merged in self.merged_sparse_pairs: param, grad = merged grad_name = grad.merged_var.name param_name = param.merged_var.name - is_distributed = True if param_name in distibuted_varnames else False + is_distributed = ( + True if param_name in distibuted_varnames else False + ) var = self.origin_main_program.global_block().vars[ - grad.merged_var.name] + grad.merged_var.name + ] var_numel = reduce(lambda x, y: x * y, var.shape[1:]) - sparse_ctx = CommContext(grad_name, [grad_name], - ["127.0.0.1:6071"], [var_numel], - [grad_name], trainer_id, True, True, - is_distributed, idx, False, False, -1, - []) + sparse_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + [grad_name], + trainer_id, + True, + True, + is_distributed, + idx, + False, + False, + -1, + [], + ) idx += 1 send_ctx[sparse_ctx.var_name()] = sparse_ctx if len(send_ctx) == 0: raise ValueError( - "GeoSGD require sparse parameters in your net.") + "GeoSGD require sparse parameters in your net." + ) if len(self.tensor_table_dict) > 0 and self.role_maker._is_worker(): name, ctx = self._step_ctx(idx) @@ -595,12 +692,14 @@ class CompileTimeStrategy(object): else: return self.get_the_one_send_context(split_dense_table) - def get_dense_send_context(self, - send_ctx, - idx, - merged_dense_pairs, - trainer_id, - split_dense_table=False): + def get_dense_send_context( + self, + send_ctx, + idx, + merged_dense_pairs, + trainer_id, + split_dense_table=False, + ): if len(merged_dense_pairs) < 1: return idx if not split_dense_table: @@ -610,52 +709,87 @@ class CompileTimeStrategy(object): grad = merged[1] origin_varnames.append(grad.merged_var.name) var = self.origin_main_program.global_block().vars[ - grad.merged_var.name] + grad.merged_var.name + ] var_numel += reduce(lambda x, y: x * y, var.shape) grad_name = "Dense@Grad" trainer_id = self.get_role_id() aggregate = True - dense_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], origin_varnames, trainer_id, - aggregate, False, False, idx, False, False, - -1, []) + dense_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + origin_varnames, + trainer_id, + aggregate, + False, + False, + idx, + False, + False, + -1, + [], + ) send_ctx[grad_name] = dense_ctx idx += 1 else: for merged in merged_dense_pairs: grad = merged[1] origin_varname = grad.merged_var.name - var = self.origin_main_program.global_block( - ).vars[origin_varname] + var = self.origin_main_program.global_block().vars[ + origin_varname + ] var_numel = reduce(lambda x, y: x * y, var.shape) grad_name = origin_varname aggregate = True - dense_ctx = CommContext(grad_name, [grad_name], - ["127.0.0.1:6071"], [var_numel], - [origin_varname], trainer_id, aggregate, - False, False, idx, False, False, -1, []) + dense_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + [origin_varname], + trainer_id, + aggregate, + False, + False, + idx, + False, + False, + -1, + [], + ) send_ctx[grad_name] = dense_ctx idx += 1 return idx - def get_the_one_send_context(self, - split_dense_table=False, - use_origin_program=False, - ep_list=None): + def get_the_one_send_context( + self, split_dense_table=False, use_origin_program=False, ep_list=None + ): if ep_list is None: ep_list = ["127.0.0.1:6071"] send_ctx = {} trainer_id = self.get_role_id() idx = 0 - merged_dense_pairs = self.origin_merged_dense_pairs if use_origin_program else self.merged_dense_pairs - merged_sparse_pairs = self.origin_merged_sparse_pairs if use_origin_program else self.merged_sparse_pairs + merged_dense_pairs = ( + self.origin_merged_dense_pairs + if use_origin_program + else self.merged_dense_pairs + ) + merged_sparse_pairs = ( + self.origin_merged_sparse_pairs + if use_origin_program + else self.merged_sparse_pairs + ) - idx += self.get_dense_send_context(send_ctx, idx, merged_dense_pairs, - trainer_id, split_dense_table) + idx += self.get_dense_send_context( + send_ctx, idx, merged_dense_pairs, trainer_id, split_dense_table + ) - distibuted_varnames = get_sparse_tablenames(self.origin_main_program, - True) + distibuted_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) for merged in merged_sparse_pairs: param, grad = merged grad_name = grad.merged_var.name @@ -665,17 +799,33 @@ class CompileTimeStrategy(object): for i in range(len(ep_list)): splited_varname.append("{}.block{}".format(param_name, i)) - is_distributed = True if param_name in distibuted_varnames else False + is_distributed = ( + True if param_name in distibuted_varnames else False + ) var = self.origin_main_program.global_block().vars[ - grad.merged_var.name] + grad.merged_var.name + ] shape = list(var.shape) shape[0] = 0 if is_distributed else shape[0] - sparse_ctx = CommContext(grad_name, splited_varname, ep_list, shape, - [grad_name], trainer_id, True, True, - is_distributed, idx, False, False, -1, []) + sparse_ctx = CommContext( + grad_name, + splited_varname, + ep_list, + shape, + [grad_name], + trainer_id, + True, + True, + is_distributed, + idx, + False, + False, + -1, + [], + ) idx += 1 send_ctx[sparse_ctx.var_name()] = sparse_ctx @@ -686,15 +836,15 @@ class CompileTimeStrategy(object): return send_ctx - def get_the_one_recv_context(self, - is_dense=True, - split_dense_table=False, - use_origin_program=False): + def get_the_one_recv_context( + self, is_dense=True, split_dense_table=False, use_origin_program=False + ): recv_id_maps = {} if is_dense: send_ctx = self.get_the_one_send_context( split_dense_table=split_dense_table, - use_origin_program=use_origin_program) + use_origin_program=use_origin_program, + ) for idx, (name, ctx) in enumerate(send_ctx.items()): if ctx.is_sparse(): continue @@ -752,8 +902,22 @@ class CompileTimeStrategy(object): endpoints = self.get_ps_endpoints() sections = [1] * len(endpoints) names = [name] * len(endpoints) - ctx = CommContext(name, names, endpoints, sections, [name], trainer_id, - True, False, False, idx, True, False, -1, []) + ctx = CommContext( + name, + names, + endpoints, + sections, + [name], + trainer_id, + True, + False, + False, + idx, + True, + False, + -1, + [], + ) return name, ctx def _create_vars_from_blocklist(self, block_list): @@ -784,20 +948,23 @@ class CompileTimeStrategy(object): if len(split) == 1: var_mapping[varname] = [orig_var] - self.var_distributed.add_distributed_var(origin_var=orig_var, - slice_var=orig_var, - block_id=0, - offset=0, - is_slice=False, - vtype="Param") + self.var_distributed.add_distributed_var( + origin_var=orig_var, + slice_var=orig_var, + block_id=0, + offset=0, + is_slice=False, + vtype="Param", + ) else: var_mapping[varname] = [] orig_shape = orig_var.shape orig_dim1_flatten = 1 if len(orig_shape) >= 2: - orig_dim1_flatten = reduce(lambda x, y: x * y, - orig_shape[1:]) + orig_dim1_flatten = reduce( + lambda x, y: x * y, orig_shape[1:] + ) for i, block in enumerate(split): size = block[1] @@ -813,7 +980,8 @@ class CompileTimeStrategy(object): dtype=orig_var.dtype, type=orig_var.type, lod_level=orig_var.lod_level, - persistable=False) + persistable=False, + ) var_mapping[varname].append(slice_var) self.var_distributed.add_distributed_var( @@ -822,7 +990,8 @@ class CompileTimeStrategy(object): block_id=i, offset=-1, is_slice=False, - vtype="Param") + vtype="Param", + ) return var_mapping @@ -871,11 +1040,9 @@ class CompileTimeStrategy(object): self.param_grad_ep_mapping[ep]["params"].append(recv_vars[i]) self.param_grad_ep_mapping[ep]["grads"].append(send_vars[i]) - def _slice_variable(self, - var_list, - slice_count, - min_block_size, - uniform=False): + def _slice_variable( + self, var_list, slice_count, min_block_size, uniform=False + ): """ We may need to split dense tensor to one or more blocks and put them equally onto parameter server. One block is a sub-tensor @@ -906,7 +1073,8 @@ class CompileTimeStrategy(object): else: split_count = slice_count max_pserver_count = int( - math.floor(var_numel / float(min_block_size))) + math.floor(var_numel / float(min_block_size)) + ) if max_pserver_count == 0: max_pserver_count = 1 if max_pserver_count < slice_count: @@ -922,10 +1090,12 @@ class CompileTimeStrategy(object): # update split_count after aligning split_count = int(math.ceil(var_numel / float(block_size))) for block_id in range(split_count): - curr_block_size = min(block_size, - var_numel - ((block_id) * block_size)) - block = vars_metatools.VarBlock(var.name, block_id, - curr_block_size) + curr_block_size = min( + block_size, var_numel - ((block_id) * block_size) + ) + block = vars_metatools.VarBlock( + var.name, block_id, curr_block_size + ) blocks.append(str(block)) else: block_size = var.shape[0] / slice_count @@ -966,13 +1136,13 @@ class CompileTimeStrategy(object): # when we slice var up into blocks, we will slice the var according to # pserver services' count. A pserver may have two or more listening ports. - grad_blocks = self._slice_variable(grad_list, - len(self.get_ps_endpoints()), - min_block_size, uniform) + grad_blocks = self._slice_variable( + grad_list, len(self.get_ps_endpoints()), min_block_size, uniform + ) - param_blocks = self._slice_variable(param_list, - len(self.get_ps_endpoints()), - min_block_size, uniform) + param_blocks = self._slice_variable( + param_list, len(self.get_ps_endpoints()), min_block_size, uniform + ) return param_blocks, grad_blocks def _var_slice_and_distribute(self): @@ -982,15 +1152,17 @@ class CompileTimeStrategy(object): # 3. grad_param_mapping : grad.blockx->param.blockx # 4. param_grad_ep_mapping : ep->{"params" : [], "grads" : [] } - dps, dgs = self._get_param_grad_blocks(self.merged_dense_pairs, - self.min_block_size, False) - sps, sgs = self._get_param_grad_blocks(self.merged_sparse_pairs, - self.min_block_size, True) + dps, dgs = self._get_param_grad_blocks( + self.merged_dense_pairs, self.min_block_size, False + ) + sps, sgs = self._get_param_grad_blocks( + self.merged_sparse_pairs, self.min_block_size, True + ) param_blocks = dps + sps grad_blocks = dgs + sgs - assert (len(grad_blocks) == len(param_blocks)) + assert len(grad_blocks) == len(param_blocks) # origin_param_name->[splited_param_vars] self.param_var_mapping = self._create_vars_from_blocklist(param_blocks) @@ -1001,8 +1173,9 @@ class CompileTimeStrategy(object): for g, p in zip(grad_blocks, param_blocks): g_name, g_bid, _ = g.split(":") p_name, p_bid, _ = p.split(":") - self.grad_param_mapping[self.grad_var_mapping[g_name][int(g_bid)]] = \ - self.param_var_mapping[p_name][int(p_bid)] + self.grad_param_mapping[ + self.grad_var_mapping[g_name][int(g_bid)] + ] = self.param_var_mapping[p_name][int(p_bid)] print_maps = {} for k, v in self.grad_param_mapping.items(): @@ -1011,10 +1184,8 @@ class CompileTimeStrategy(object): # create mapping of endpoint->split var to create pserver side program self.param_grad_ep_mapping = collections.OrderedDict() [ - self.param_grad_ep_mapping.update({ep: { - "params": [], - "grads": [] - }}) for ep in self.get_ps_endpoints() + self.param_grad_ep_mapping.update({ep: {"params": [], "grads": []}}) + for ep in self.get_ps_endpoints() ] def _build_var_distributed(self): @@ -1055,7 +1226,8 @@ class CompileTimeStrategy(object): for merged in self.merged_variables_pairs: m_param, m_grad = merged self.merged_variable_map[ - m_param.merged_var.name] = m_param.merged_var + m_param.merged_var.name + ] = m_param.merged_var self.merged_variable_map[m_grad.merged_var.name] = m_grad.merged_var param_merges = [] @@ -1096,8 +1268,10 @@ class CompileTimeStrategy(object): for op in block.ops: if _is_opt_role_op(op): # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs() \ - and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE): + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + ): op._set_attr("op_role", role_id) continue if op.attr(OP_ROLE_VAR_ATTR_NAME): @@ -1105,8 +1279,10 @@ class CompileTimeStrategy(object): grad_name = op.attr(OP_ROLE_VAR_ATTR_NAME)[1] if param_name not in optimize_params: optimize_params.add(param_name) - param_grad = (origin_var_dict[param_name], - origin_var_dict[grad_name]) + param_grad = ( + origin_var_dict[param_name], + origin_var_dict[grad_name], + ) if param_name in sparse_varnames: sparse_param_grads.append(param_grad) @@ -1117,8 +1293,10 @@ class CompileTimeStrategy(object): def _get_sparse_varnames(): varnames = [] for op in origin_program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] varnames.append(param_name) @@ -1126,7 +1304,8 @@ class CompileTimeStrategy(object): sparse_varnames = _get_sparse_varnames() sparse_param_grads, dense_param_grads = _get_params_grads( - sparse_varnames) + sparse_varnames + ) return sparse_param_grads, dense_param_grads @@ -1160,8 +1339,9 @@ def _is_opt_role_op(op): # optimize op_maker = core.op_proto_and_checker_maker optimize_role = core.op_proto_and_checker_maker.OpRole.Optimize - if op_maker.kOpRoleAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(optimize_role): + if op_maker.kOpRoleAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(optimize_role): return True return False @@ -1172,11 +1352,14 @@ def _get_optimize_ops(_program): for op in block.ops: if _is_opt_role_op(op): # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs() \ - and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE): + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + ): op._set_attr( "op_role", - int(core.op_proto_and_checker_maker.OpRole.Backward)) + int(core.op_proto_and_checker_maker.OpRole.Backward), + ) continue opt_ops.append(op) return opt_ops @@ -1185,17 +1368,28 @@ def _get_optimize_ops(_program): def _add_lr_decay_table_pass(main_program, compiled_config, lr_decay_steps): if hasattr(compiled_config.origin_main_program, 'lr_sheduler'): from paddle.optimizer.lr import LRScheduler - assert isinstance(compiled_config.origin_main_program.lr_sheduler, - LRScheduler), "must be LRScheduler" + + assert isinstance( + compiled_config.origin_main_program.lr_sheduler, LRScheduler + ), "must be LRScheduler" ops = _get_optimize_ops(compiled_config.origin_main_program) lr_param_dict = _get_lr_param_dict(ops) - lr_decay_main_program, lr_decay_startup_program, lr_name = _get_lr_sheduler_program( - compiled_config.origin_main_program.lr_sheduler, lr_param_dict, - lr_decay_steps) - compiled_config.add_tensor_table("@LR_DECAY_COUNTER@", lr_name, - lr_decay_startup_program, - lr_decay_main_program, - "GlobalStepTable") + ( + lr_decay_main_program, + lr_decay_startup_program, + lr_name, + ) = _get_lr_sheduler_program( + compiled_config.origin_main_program.lr_sheduler, + lr_param_dict, + lr_decay_steps, + ) + compiled_config.add_tensor_table( + "@LR_DECAY_COUNTER@", + lr_name, + lr_decay_startup_program, + lr_decay_main_program, + "GlobalStepTable", + ) def _get_lr_param_dict(opt_ops): @@ -1211,11 +1405,26 @@ def _get_lr_param_dict(opt_ops): def _get_lr_sheduler_program(lr_sheduler, lr_param_dict, lr_decay_steps): schedler_decay = [ - 'NoamDecay', 'NaturalExpDecay', 'InverseTimeDecay', 'ExponentialDecay' + 'NoamDecay', + 'NaturalExpDecay', + 'InverseTimeDecay', + 'ExponentialDecay', ] - from paddle.optimizer.lr import ExponentialDecay, NoamDecay, PiecewiseDecay, NaturalExpDecay, InverseTimeDecay - from paddle.fluid.layers.learning_rate_scheduler import exponential_decay, noam_decay, piecewise_decay, natural_exp_decay, inverse_time_decay + from paddle.optimizer.lr import ( + ExponentialDecay, + NoamDecay, + PiecewiseDecay, + NaturalExpDecay, + InverseTimeDecay, + ) + from paddle.fluid.layers.learning_rate_scheduler import ( + exponential_decay, + noam_decay, + piecewise_decay, + natural_exp_decay, + inverse_time_decay, + ) decay_main_program = fluid.framework.Program() decay_startup_program = fluid.framework.Program() @@ -1230,13 +1439,16 @@ def _get_lr_sheduler_program(lr_sheduler, lr_param_dict, lr_decay_steps): "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) elif isinstance(lr_sheduler, NoamDecay): with fluid.program_guard(decay_main_program, decay_startup_program): lr = noam_decay(lr_sheduler.d_model, lr_sheduler.warmup_steps, 1.0) lr_name = lr.name - logging.warn("NoamDecay is set, warmup steps is [ %d ]" % - lr_sheduler.warmup_steps) + logging.warn( + "NoamDecay is set, warmup steps is [ %d ]" + % lr_sheduler.warmup_steps + ) elif isinstance(lr_sheduler, NaturalExpDecay): with fluid.program_guard(decay_main_program, decay_startup_program): lr = natural_exp_decay(1.0, lr_decay_steps, lr_sheduler.gamma, True) @@ -1246,22 +1458,27 @@ def _get_lr_sheduler_program(lr_sheduler, lr_param_dict, lr_decay_steps): "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) elif isinstance(lr_sheduler, InverseTimeDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = inverse_time_decay(1.0, lr_decay_steps, lr_sheduler.gamma, - True) + lr = inverse_time_decay( + 1.0, lr_decay_steps, lr_sheduler.gamma, True + ) lr_name = lr.name logging.warn( "InverseTimeDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n" "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) else: raise ValueError( - "Not supported current LearningRate strategy, please use follow decay strategy: {}" - .format(schedler_decay)) + "Not supported current LearningRate strategy, please use follow decay strategy: {}".format( + schedler_decay + ) + ) return decay_main_program, decay_startup_program, lr_name @@ -1273,15 +1490,15 @@ def _get_varname_parts(varname): block_part = "" trainer_idx = varname.find(".trainer_") if trainer_idx >= 0: - trainer_part = varname[trainer_idx + 1:] + trainer_part = varname[trainer_idx + 1 :] else: trainer_idx = len(varname) block_index = varname.find(".block") if block_index >= 0: - block_part = varname[block_index + 1:trainer_idx] + block_part = varname[block_index + 1 : trainer_idx] else: block_index = len(varname) - orig_var_name = varname[0:min(block_index, trainer_idx)] + orig_var_name = varname[0 : min(block_index, trainer_idx)] return orig_var_name, block_part, trainer_part diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py index acd337720ca..fa818e3c413 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py @@ -24,9 +24,13 @@ import paddle.fluid.core as core import paddle.fluid.framework as framework from paddle.fluid.transpiler.details.program_utils import delete_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, +) from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_lr_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, +) from paddle.fluid.incubate.fleet.parameter_server.mode import DistributedMode OP_NAME_SCOPE = "op_namescope" @@ -42,7 +46,7 @@ op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() SPARSE_OP_TYPE_DICT = {"lookup_table": "W", "lookup_table_v2": "W"} SPARSE_GRAD_OP_TYPE_DICT = { "lookup_table_grad": "W", - "lookup_table_v2_grad": "W" + "lookup_table_v2_grad": "W", } DEVICE_LIST = ["cpu", "gpu", "xpu"] COMMUNICATE_OPS_TYPE = ["send", "recv", "fetch_barrier", "send_barrier"] @@ -50,7 +54,6 @@ DEFAULT_DEVICE = 'cpu' def delete_optimizer_pass(program, config): - def _delete_optimizer_op_and_vars(_program, optimize_ops): optimize_vars = [] optimize_op_role_vars = [] @@ -75,14 +78,17 @@ def delete_optimizer_pass(program, config): def _add_lr_var(main_program, compiled_config): # Todo: hard code for pe - lr_var = compiled_config.origin_main_program.global_block( - ).vars["learning_rate_0"] - main_program.global_block().create_var(name=lr_var.name, - shape=lr_var.shape, - dtype=lr_var.dtype, - type=lr_var.type, - lod_level=lr_var.lod_level, - persistable=True) + lr_var = compiled_config.origin_main_program.global_block().vars[ + "learning_rate_0" + ] + main_program.global_block().create_var( + name=lr_var.name, + shape=lr_var.shape, + dtype=lr_var.dtype, + type=lr_var.type, + lod_level=lr_var.lod_level, + persistable=True, + ) optimizer_ops = _get_optimize_ops(program) lr_ops = _get_lr_ops(program) @@ -98,7 +104,8 @@ def delete_optimizer_pass(program, config): def distributed_ops_pass(program, config, use_ps_gpu=False): trainer_id = config.get_role_id() send_ctx = config.get_the_one_send_context( - split_dense_table=config.is_heter_ps_mode) + split_dense_table=config.is_heter_ps_mode + ) w_2_table_id = {} emb_size = {} @@ -108,8 +115,10 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): push_sparse_ops = {} ops = {} for op in _program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] if config.is_heter_ps_mode: # trick for matchnet, need to modify @@ -123,15 +132,16 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): for op in _program.global_block().ops: if op.type in SPARSE_GRAD_OP_TYPE_DICT.keys(): param_name = op.input(SPARSE_GRAD_OP_TYPE_DICT[op.type])[0] - if param_name in pull_sparse_ids and op.input( - "Ids")[0] in pull_sparse_ids[param_name]: + if ( + param_name in pull_sparse_ids + and op.input("Ids")[0] in pull_sparse_ids[param_name] + ): ops = push_sparse_ops.get(param_name, []) ops.append(op) push_sparse_ops[param_name] = ops return pull_sparse_ops, push_sparse_ops def _pull_sparse_fuse(_program, pull_sparse_ops, use_ps_gpu): - def dag_check_up_and_reorder(program, inputs, outputs): global_block = program.global_block() min_output_index = len(global_block.ops) @@ -248,7 +258,8 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): if table_id == -1: raise ValueError( - "can not find suitable sparse table, please check") + "can not find suitable sparse table, please check" + ) w_2_table_id[param] = table_id padding_idx = ops[0].attr("padding_idx") @@ -279,8 +290,9 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): ins = op.input(op.input_names[i]) for out_id, out_var in enumerate(outputs): if out_var.name in ins: - outputs_idxs[out_id] = min(idx, - outputs_idxs[out_id]) + outputs_idxs[out_id] = min( + idx, outputs_idxs[out_id] + ) if min(outputs_idxs) - max(inputs_idxs) >= 1: if max(inputs_idxs) == -1: @@ -292,32 +304,28 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): program.global_block()._insert_op( index=distributed_idx, type="pull_gpups_sparse", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Out": outputs}, attrs={ "size": [w.shape[1] for i in inputs], "is_distributed": True, - "is_sparse": True - }) + "is_sparse": True, + }, + ) else: program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Outputs": outputs}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "lookup_table_version": op_type, - "op_device": op_device - }) + "op_device": op_device, + }, + ) else: for i in range(len(inputs_idxs)): distributed_idx = op_idxs[i] @@ -325,18 +333,16 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": [inputs[i]], - 'W': w - }, + inputs={"Ids": [inputs[i]], 'W': w}, outputs={"Outputs": [outputs[i]]}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "lookup_table_version": op_type, - "op_device": op_device - }) + "op_device": op_device, + }, + ) def _push_sparse_fuse(_program, push_sparse_ops, use_ps_gpu): if use_ps_gpu: @@ -357,8 +363,10 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): if len(entry) == 3 and entry[0] == 'show_click_entry': show_var_name = entry[1] click_var_name = entry[2] - if show_var_name in program.global_block( - ).vars and click_var_name in program.global_block().vars: + if ( + show_var_name in program.global_block().vars + and click_var_name in program.global_block().vars + ): show = program.global_block().vars[show_var_name] clk = program.global_block().vars[click_var_name] use_entry = True @@ -373,7 +381,8 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): name="show", dtype=core.VarDesc.VarType.INT64, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) program.global_block()._insert_op( index=0, type='fill_constant', @@ -383,14 +392,16 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): 'shape': [1], 'dtype': show.dtype, 'value': 1, - #OP_ROLE_KEY: OpRole.Forward - }) + # OP_ROLE_KEY: OpRole.Forward + }, + ) clk = program.global_block().create_var( name="clk", dtype=core.VarDesc.VarType.INT64, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) program.global_block()._insert_op( index=0, type='fill_constant', @@ -400,8 +411,9 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): 'shape': [1], 'dtype': clk.dtype, 'value': 0, - #OP_ROLE_KEY: OpRole.Forward - }) + # OP_ROLE_KEY: OpRole.Forward + }, + ) for param, ops in push_sparse_ops.items(): all_ops = program.global_block().ops @@ -423,35 +435,35 @@ def distributed_ops_pass(program, config, use_ps_gpu=False): for idx in op_idxs[::-1]: program.global_block()._remove_op(idx) - -# if use_ps_gpu: -# program.global_block().append_op( -# type="push_box_sparse", -# inputs={"Ids": inputs, -# 'Out': outputs}, -# outputs={"Out": outputs}, -# attrs={ -# "size": w.shape[1], -# "is_distributed": True, -# "is_sparse": True -# }) -# else: - program.global_block().append_op(type="distributed_push_sparse", - inputs={ - "Ids": inputs, - 'W': w, - "Outputs": outputs, - "Shows": show, - "Clicks": clk - }, - outputs={"Outputs": outputs}, - attrs={ - "is_distributed": - is_distributed, - "padding_idx": padding_idx, - "table_id": table_id, - "size": emb_size[param] - }) + # if use_ps_gpu: + # program.global_block().append_op( + # type="push_box_sparse", + # inputs={"Ids": inputs, + # 'Out': outputs}, + # outputs={"Out": outputs}, + # attrs={ + # "size": w.shape[1], + # "is_distributed": True, + # "is_sparse": True + # }) + # else: + program.global_block().append_op( + type="distributed_push_sparse", + inputs={ + "Ids": inputs, + 'W': w, + "Outputs": outputs, + "Shows": show, + "Clicks": clk, + }, + outputs={"Outputs": outputs}, + attrs={ + "is_distributed": is_distributed, + "padding_idx": padding_idx, + "table_id": table_id, + "size": emb_size[param], + }, + ) pull_sparse_ops, push_sparse_ops = _get_pull_sparse_ops(program) _pull_sparse_fuse(program, pull_sparse_ops, use_ps_gpu) @@ -476,40 +488,40 @@ def append_send_ops_pass(program, config): dummy_output = [] if mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]: dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) - - program.global_block().append_op(type="send", - inputs={"X": send_input_vars}, - outputs={"Out": dummy_output}, - attrs={ - "send_varnames": [queue], - "is_sparse": - is_sparse, - "table_id": - table_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + name=framework.generate_control_dev_var_name() + ) + + program.global_block().append_op( + type="send", + inputs={"X": send_input_vars}, + outputs={"Out": dummy_output}, + attrs={ + "send_varnames": [queue], + "is_sparse": is_sparse, + "table_id": table_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return dummy_output def _append_barrier_op(dummys): - program.global_block().append_op(type="send_barrier", - inputs={"X": dummys}, - outputs={"Out": []}, - attrs={ - "trainer_id": - trainer_id, - "half_async": - True, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + program.global_block().append_op( + type="send_barrier", + inputs={"X": dummys}, + outputs={"Out": []}, + attrs={ + "trainer_id": trainer_id, + "half_async": True, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) dummys = [] sends = config.get_the_one_trainer_send_context( - split_dense_table=config.is_heter_ps_mode) + split_dense_table=config.is_heter_ps_mode + ) for merged_name, send in sends.items(): if send.is_sparse() and not config.is_geo_mode(): @@ -517,8 +529,10 @@ def append_send_ops_pass(program, config): is_sparse = 1 if send.is_sparse() else 0 is_sparse = 2 if send.is_distributed() else is_sparse dummys.append( - _append_send_op(send.origin_varnames(), merged_name, is_sparse, - send.table_id())) + _append_send_op( + send.origin_varnames(), merged_name, is_sparse, send.table_id() + ) + ) if mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]: _append_barrier_op(dummys) @@ -532,19 +546,19 @@ def init_from_server_pass(program, config): return program fetch_barrier_out = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) - - program.global_block().append_op(type="fetch_barrier", - inputs={}, - outputs={"Out": fetch_barrier_out}, - attrs={ - "endpoints": - config.get_ps_endpoints(), - "trainer_id": - config.get_role_id(), - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + name=framework.generate_control_dev_var_name() + ) + + program.global_block().append_op( + type="fetch_barrier", + inputs={}, + outputs={"Out": fetch_barrier_out}, + attrs={ + "endpoints": config.get_ps_endpoints(), + "trainer_id": config.get_role_id(), + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return program @@ -566,14 +580,16 @@ def fake_init_ops_pass(program, config): table_param_init_op.append(op) init_op_num = len(table_param_init_op) if init_op_num != 1: - raise ValueError("table init op num should be 1, now is " + - str(init_op_num)) + raise ValueError( + "table init op num should be 1, now is " + str(init_op_num) + ) table_init_op = table_param_init_op[0] program.global_block().append_op( type="fake_init", inputs={}, outputs={"Out": table_var}, - attrs={"shape": table_init_op.attr('shape')}) + attrs={"shape": table_init_op.attr('shape')}, + ) delete_ops(program.global_block(), table_param_init_op) sparse_tables = _get_sparse_table_names() @@ -583,7 +599,6 @@ def fake_init_ops_pass(program, config): def ps_gpu_pass(program): - def _add_push_box_sparse_op(program): op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() backward = core.op_proto_and_checker_maker.OpRole.Backward @@ -591,7 +606,8 @@ def ps_gpu_pass(program): if op.type != "pull_box_sparse" and op.type != "pull_gpups_sparse": continue grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, set(), []) + op.desc, set(), [] + ) for op_desc in grad_op_desc: new_op_desc = program.global_block().desc.append_op() new_op_desc.copy_from(op_desc) @@ -700,8 +716,11 @@ def delete_extra_optimizes_pass(program, config): def find_heter_ops(program, default_device="cpu"): if default_device not in DEVICE_LIST: - raise ValueError("Given device {} is not in device list {}".format( - default_device, DEVICE_LIST)) + raise ValueError( + "Given device {} is not in device list {}".format( + default_device, DEVICE_LIST + ) + ) def _is_heter_op(op, current_heter_device, default_device="cpu"): heter_devices = list(DEVICE_LIST) @@ -710,10 +729,13 @@ def find_heter_ops(program, default_device="cpu"): op_type = op.type if op_device in heter_devices: return True - elif op_type in COMMUNICATE_OPS_TYPE and current_heter_device != default_device: + elif ( + op_type in COMMUNICATE_OPS_TYPE + and current_heter_device != default_device + ): # for distributed communciate ops: send & recv & barrier etc. # Todo: need update this method - #op._set_attr('op_device', current_heter_device) + # op._set_attr('op_device', current_heter_device) return True elif op_device == None or op_device == default_device: op._set_attr('op_device', default_device) @@ -748,16 +770,20 @@ def find_heter_ops(program, default_device="cpu"): op = op_list[i] if "_grad" in op.type: forward_op_type = op.type.split("_grad")[0] - if forward_op_type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + forward_op_type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[forward_op_type])[0] if param_name in var2idx: ## insert sum op & remove sum op from var2idx and origin place op_list = list(block.ops) sum_op = op_list[var2idx[param_name]] sum_op_inputs = { - sum_op.input_names[0]: - [block.vars[input] for input in sum_op.input_arg_names] + sum_op.input_names[0]: [ + block.vars[input] + for input in sum_op.input_arg_names + ] } sum_op_outputs = { sum_op.output_names[0]: [ @@ -765,11 +791,13 @@ def find_heter_ops(program, default_device="cpu"): for output in sum_op.output_arg_names ] } - block._insert_op(index=i + 1, - type=sum_op.type, - inputs=sum_op_inputs, - outputs=sum_op_outputs, - attrs=sum_op.all_attrs()) + block._insert_op( + index=i + 1, + type=sum_op.type, + inputs=sum_op_inputs, + outputs=sum_op_outputs, + attrs=sum_op.all_attrs(), + ) block._remove_op(var2idx[param_name] + 1) var2idx.pop(param_name) for var_ in var2idx: @@ -790,9 +818,9 @@ def find_heter_ops(program, default_device="cpu"): for no_grad_var in output_vars_no_grad: if no_grad_var in var2idx: """ - insert sum op & remove sum op from var2idx and origin place + insert sum op & remove sum op from var2idx and origin place - """ + """ op_list = list(block.ops) sum_op = op_list[var2idx[no_grad_var]] sum_op_inputs = { @@ -807,11 +835,13 @@ def find_heter_ops(program, default_device="cpu"): for output in sum_op.output_arg_names ] } - block._insert_op(index=i + 1, - type=sum_op.type, - inputs=sum_op_inputs, - outputs=sum_op_outputs, - attrs=sum_op.all_attrs()) + block._insert_op( + index=i + 1, + type=sum_op.type, + inputs=sum_op_inputs, + outputs=sum_op_outputs, + attrs=sum_op.all_attrs(), + ) block._remove_op(var2idx[no_grad_var] + 1) var2idx.pop(no_grad_var) for var_ in var2idx: @@ -824,12 +854,16 @@ def find_heter_ops(program, default_device="cpu"): pre_op = op_list[i - 1] if "_grad" in pre_op.type: forward_op_type = pre_op.type.split("_grad")[0] - if forward_op_type in SPARSE_OP_TYPE_DICT.keys() \ - and pre_op.attr('remote_prefetch') is True: + if ( + forward_op_type in SPARSE_OP_TYPE_DICT.keys() + and pre_op.attr('remote_prefetch') is True + ): param_name = pre_op.input( - SPARSE_OP_TYPE_DICT[forward_op_type])[0] + SPARSE_OP_TYPE_DICT[forward_op_type] + )[0] if param_name == origin_var and op.attr( - "op_device") == pre_op.attr("op_device"): + "op_device" + ) == pre_op.attr("op_device"): continue else: var2idx[origin_var] = i @@ -882,7 +916,8 @@ def find_heter_ops(program, default_device="cpu"): # for cpu-op block append if len(current_default_block_ops) > 1: default_ops[default_device][ - block_index] = current_default_block_ops + block_index + ] = current_default_block_ops program_block_ops.append(current_default_block_ops) current_default_block_ops = [] block_index += 1 @@ -938,14 +973,24 @@ def find_heter_ops(program, default_device="cpu"): for _, heter_block in heter_block_dict.items(): total_heter_ops += len(heter_block) print( - "There are {} OPs in your main_program, and contains {} heter-OPs which is made up of {} heter-blocks." - .format(len(block.ops), total_heter_ops, heter_blocks)) + "There are {} OPs in your main_program, and contains {} heter-OPs which is made up of {} heter-blocks.".format( + len(block.ops), total_heter_ops, heter_blocks + ) + ) return origin_porgram, heter_ops, default_ops, program_block_ops -def create_heter_program(program, config, heter_program, program_block_ops_list, - heter_ops, block_var_detail, current_device, stage_id): +def create_heter_program( + program, + config, + heter_program, + program_block_ops_list, + heter_ops, + block_var_detail, + current_device, + stage_id, +): # This function mainly includes the following contents: # 1. For every heter block: # a) copy heter device op from origin program @@ -993,57 +1038,76 @@ def create_heter_program(program, config, heter_program, program_block_ops_list, for _, op in enumerate(heter_block_ops_backward): block_append_op(heter_program, program, heter_block_bp, op) - bp_entrance_vars = block_var_detail[stage_id - - 1]["backward"]["entrance"] - add_vars_by_var_list(bp_entrance_vars, program, heter_program, - heter_block_bp) + bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][ + "entrance" + ] + add_vars_by_var_list( + bp_entrance_vars, program, heter_program, heter_block_bp + ) bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"] - add_vars_by_var_list(bp_exit_vars, program, heter_program, - heter_block_bp) - backward_comm_info = get_communicate_var_info(program, - stage_id, - bp_entrance_vars, - type="backward") + add_vars_by_var_list( + bp_exit_vars, program, heter_program, heter_block_bp + ) + backward_comm_info = get_communicate_var_info( + program, stage_id, bp_entrance_vars, type="backward" + ) - grad_to_block_id.append(backward_comm_info["block_input_var_name"] + - ":" + str(heter_block_bp.idx)) + grad_to_block_id.append( + backward_comm_info["block_input_var_name"] + + ":" + + str(heter_block_bp.idx) + ) else: for _, op in enumerate(heter_block_ops_backward): block_append_op(heter_program, program, heter_block, op) - bp_entrance_vars = block_var_detail[stage_id - - 1]["backward"]["entrance"] - add_vars_by_var_list(bp_entrance_vars, program, heter_program, - heter_block) + bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][ + "entrance" + ] + add_vars_by_var_list( + bp_entrance_vars, program, heter_program, heter_block + ) bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"] add_vars_by_var_list(bp_exit_vars, program, heter_program, heter_block) heter_block_bp = heter_block - forward_comm_info = get_communicate_var_info(program, - stage_id, - entrance_vars, - type="forward") + forward_comm_info = get_communicate_var_info( + program, stage_id, entrance_vars, type="forward" + ) - grad_to_block_id.append(forward_comm_info["block_input_var_name"] + ":" + - str(heter_block.idx)) + grad_to_block_id.append( + forward_comm_info["block_input_var_name"] + ":" + str(heter_block.idx) + ) first_op_index_bp = len(heter_block_bp.ops) if stage_id <= len(block_var_detail) - 1: - static_var = insert_communicate_op(program, config, heter_block, - stage_id, first_op_index_fp, - block_var_detail, current_device) - static_var_bp = insert_communicate_op(program, config, heter_block_bp, - stage_id, first_op_index_bp, - block_var_detail, current_device, - False) + static_var = insert_communicate_op( + program, + config, + heter_block, + stage_id, + first_op_index_fp, + block_var_detail, + current_device, + ) + static_var_bp = insert_communicate_op( + program, + config, + heter_block_bp, + stage_id, + first_op_index_bp, + block_var_detail, + current_device, + False, + ) # add send op - send_grad_var_list = add_heter_send_op(program, heter_program, - heter_block_bp, - block_var_detail[stage_id - 1]) + send_grad_var_list = add_heter_send_op( + program, heter_program, heter_block_bp, block_var_detail[stage_id - 1] + ) # --------------- # add step conter @@ -1064,9 +1128,9 @@ def create_heter_program(program, config, heter_program, program_block_ops_list, # add info in listen&serv attrs = { - #"mode": "sync", - #"trainers": config.get_trainers(), - #"trainer_id": config.get_role_id() + config.get_trainers(), + # "mode": "sync", + # "trainers": config.get_trainers(), + # "trainer_id": config.get_role_id() + config.get_trainers(), "message_to_block_id": grad_to_block_id, "optimize_blocks": optimizer_block, # runtime attribute @@ -1075,13 +1139,12 @@ def create_heter_program(program, config, heter_program, program_block_ops_list, "pserver_id": config.get_role_id(), "distributed_mode": config.get_distributed_mode(), "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } # append the listen_and_serv op - heter_program.global_block().append_op(type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + heter_program.global_block().append_op( + type="heter_listen_and_serv", inputs={'X': []}, outputs={}, attrs=attrs + ) check_heter_compile_time_strategy(program, config, send_grad_var_list) @@ -1093,14 +1156,16 @@ def check_heter_compile_time_strategy(program, config, send_grad_var_list): origin_grad_var_list = list(set(origin_grad_var_list)) send_grad_var_list = list(set(send_grad_var_list)) useless_grad_var_list = list( - set(origin_grad_var_list) - set(send_grad_var_list)) + set(origin_grad_var_list) - set(send_grad_var_list) + ) for useless_grad_var in useless_grad_var_list: config.remove_var_pair_by_grad(useless_grad_var) -def create_trainer_program(program, origin_program, config, - program_block_ops_list, block_var_detail): +def create_trainer_program( + program, origin_program, config, program_block_ops_list, block_var_detail +): # This function mainly includes the following contents: # 1. For every heter block in origin program # a) delete heter op and related variables @@ -1113,13 +1178,16 @@ def create_trainer_program(program, origin_program, config, # 2. check every op's device static_var = [] for heter_block_index in range(1, len(program_block_ops_list)): - ops_list = program_block_ops_list[heter_block_index][ - "forward"] + program_block_ops_list[heter_block_index]["backward"] - static_var += replace_ops_by_communicate_op(program, config, - heter_block_index, ops_list, - block_var_detail) - remove_trainer_send_op(program, config, heter_block_index, - block_var_detail) + ops_list = ( + program_block_ops_list[heter_block_index]["forward"] + + program_block_ops_list[heter_block_index]["backward"] + ) + static_var += replace_ops_by_communicate_op( + program, config, heter_block_index, ops_list, block_var_detail + ) + remove_trainer_send_op( + program, config, heter_block_index, block_var_detail + ) optimizer_block = [] grad_to_block_id = [] @@ -1127,23 +1195,26 @@ def create_trainer_program(program, origin_program, config, bp_ops_list = program_block_ops_list[0]["backward"] delete_same_ops(program.global_block(), bp_ops_list) delete_trainer_useless_var(config, program, static_var) - backward_block = create_backward_block(program, origin_program, config, - bp_ops_list, block_var_detail) + backward_block = create_backward_block( + program, origin_program, config, bp_ops_list, block_var_detail + ) bp_entrance_vars = block_var_detail[0]["backward"]["entrance"] - backward_comm_info = get_communicate_var_info(origin_program, - 1, - bp_entrance_vars, - type="backward") - - grad_to_block_id.append(backward_comm_info["block_input_var_name"] + ":" + - str(backward_block.idx)) + backward_comm_info = get_communicate_var_info( + origin_program, 1, bp_entrance_vars, type="backward" + ) + + grad_to_block_id.append( + backward_comm_info["block_input_var_name"] + + ":" + + str(backward_block.idx) + ) optimizer_block.append(backward_block) attrs = { - #"mode": "sync", - #"trainers": config.get_trainers(), - #"trainer_id": config.get_role_id(), + # "mode": "sync", + # "trainers": config.get_trainers(), + # "trainer_id": config.get_role_id(), "message_to_block_id": grad_to_block_id, "optimize_blocks": optimizer_block, # runtime attribute @@ -1152,68 +1223,74 @@ def create_trainer_program(program, origin_program, config, "pserver_id": config.get_role_id(), "distributed_mode": config.get_distributed_mode(), "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } # append the listen_and_serv op - program.global_block()._insert_op(index=0, - type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + program.global_block()._insert_op( + index=0, + type="heter_listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) ## TODO add check for bp block check_op_device(program.global_block(), DEFAULT_DEVICE) -def insert_communicate_op(orign_program, - config, - heter_block, - stage_id, - first_op_index, - block_var_detail, - device, - is_forward=True): +def insert_communicate_op( + orign_program, + config, + heter_block, + stage_id, + first_op_index, + block_var_detail, + device, + is_forward=True, +): if is_forward: next_heter_worker_endpoints = config.get_next_stage_trainers() previous_heter_worker_endpoints = config.get_previous_stage_trainers() entrance_var = block_var_detail[stage_id]["forward"]["entrance"] - comm_info = get_communicate_var_info(orign_program, stage_id + 1, - entrance_var) + comm_info = get_communicate_var_info( + orign_program, stage_id + 1, entrance_var + ) else: next_heter_worker_endpoints = config.get_next_stage_trainers() - #if next_heter_worker_endpoints == "": + # if next_heter_worker_endpoints == "": # next_heter_worker_endpoints = [] previous_heter_worker_endpoints = config.get_previous_stage_trainers() entrance_var = block_var_detail[stage_id - 1]["backward"]["exit"] - comm_info = get_communicate_var_info(orign_program, stage_id - 1, - entrance_var, "backward") - - heter_block._insert_op(index=first_op_index, - type="send_and_recv", - inputs={"X": heter_block.vars[entrance_var[0]]}, - outputs={"Out": []}, - attrs={ - "mode": "forward" if is_forward else "backward", - "send_var_name": - entrance_var + ["microbatch_id"], - "recv_var_name": [], - "message_name": - comm_info["block_input_var_name"], - "next_endpoints": next_heter_worker_endpoints, - "previous_endpoints": - previous_heter_worker_endpoints, - "trainer_id": config.get_role_id(), - "op_device": device, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + comm_info = get_communicate_var_info( + orign_program, stage_id - 1, entrance_var, "backward" + ) + + heter_block._insert_op( + index=first_op_index, + type="send_and_recv", + inputs={"X": heter_block.vars[entrance_var[0]]}, + outputs={"Out": []}, + attrs={ + "mode": "forward" if is_forward else "backward", + "send_var_name": entrance_var + ["microbatch_id"], + "recv_var_name": [], + "message_name": comm_info["block_input_var_name"], + "next_endpoints": next_heter_worker_endpoints, + "previous_endpoints": previous_heter_worker_endpoints, + "trainer_id": config.get_role_id(), + "op_device": device, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return entrance_var -def create_backward_block(program, origin_program, config, bp_ops_list, - block_var_detail): +def create_backward_block( + program, origin_program, config, bp_ops_list, block_var_detail +): pre_block_idx = program.num_blocks - 1 heter_block = program._create_block(pre_block_idx) @@ -1222,8 +1299,10 @@ def create_backward_block(program, origin_program, config, bp_ops_list, send_varnames = op.attr('send_varnames') is_skip = False for varname in send_varnames: - if varname not in program.global_block( - ).vars and varname not in heter_block.vars: + if ( + varname not in program.global_block().vars + and varname not in heter_block.vars + ): is_skip = True break if is_skip == True: @@ -1237,8 +1316,9 @@ def create_backward_block(program, origin_program, config, bp_ops_list, return heter_block -def replace_ops_by_communicate_op(program, config, heter_block_index, ops_list, - block_var_detail): +def replace_ops_by_communicate_op( + program, config, heter_block_index, ops_list, block_var_detail +): all_op = program.global_block().ops start_op = ops_list[0] first_op_idx = -1 @@ -1256,10 +1336,12 @@ def replace_ops_by_communicate_op(program, config, heter_block_index, ops_list, next_heter_worker_endpoints = config.get_next_stage_trainers() entrance_var = block_var_detail[heter_block_index]["forward"][ - "entrance"] + "entrance" + ] - comm_info = get_communicate_var_info(program, heter_block_index + 1, - entrance_var) + comm_info = get_communicate_var_info( + program, heter_block_index + 1, entrance_var + ) program.global_block()._insert_op( index=first_op_idx, type="send_and_recv", @@ -1273,25 +1355,30 @@ def replace_ops_by_communicate_op(program, config, heter_block_index, ops_list, "next_endpoints": next_heter_worker_endpoints, "previous_endpoints": [], "trainer_id": config.get_role_id(), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return entrance_var -def remove_trainer_send_op(program, config, heter_block_index, - block_var_detail): +def remove_trainer_send_op( + program, config, heter_block_index, block_var_detail +): # if trainer do FF->BP->SEND, it has follow vars: var, var@GRAD # if trainer only do SEND, it has one var: var@GRAD # Delete Send op ,if trainer doesn't has pair var (var<->var@GRAD) - persistables = block_var_detail[heter_block_index]["forward"]["persistables"] + \ - block_var_detail[heter_block_index]["backward"]["persistables"] + persistables = ( + block_var_detail[heter_block_index]["forward"]["persistables"] + + block_var_detail[heter_block_index]["backward"]["persistables"] + ) need_remove_send_op = [] need_remove_grad_var = [] for op in find_send_op(program): - input_list, _ = find_op_input_output(program, program.global_block(), - op) + input_list, _ = find_op_input_output( + program, program.global_block(), op + ) for var_name in input_list: origin_var_name = var_name.split("@GRAD")[0] if origin_var_name in persistables: @@ -1304,13 +1391,13 @@ def remove_trainer_send_op(program, config, heter_block_index, def add_heter_send_op(program, heter_program, block, block_var_detail): - def _get_send_op_dict(): send_op_dict = {} send_op_list = find_send_op(program) for op in send_op_list: - input_list, _ = find_op_input_output(program, - program.global_block(), op) + input_list, _ = find_op_input_output( + program, program.global_block(), op + ) for var in input_list: send_op_dict[var] = op return send_op_dict @@ -1345,21 +1432,23 @@ def add_heter_send_op(program, heter_program, block, block_var_detail): for table_id in table_dict: dummy_output = block.create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) send_input_vars = [ block.vars[union_var] for union_var in table_dict[table_id]['var_list'] ] - block.append_op(type="send", - inputs={"X": send_input_vars}, - outputs={"Out": dummy_output}, - attrs={ - "send_varnames": - table_dict[table_id]['send_varnames'], - "is_sparse": is_sparse, - "table_id": table_id, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + block.append_op( + type="send", + inputs={"X": send_input_vars}, + outputs={"Out": dummy_output}, + attrs={ + "send_varnames": table_dict[table_id]['send_varnames'], + "is_sparse": is_sparse, + "table_id": table_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return send_grad_var_list @@ -1372,19 +1461,20 @@ def find_send_op(program): return send_op_list -def get_communicate_var_info(program, - block_index, - entrance_var_list, - type="forward"): +def get_communicate_var_info( + program, block_index, entrance_var_list, type="forward" +): input_var_reshape_dim = [] input_var_reshape_name = [] if type == "forward": block_input_var_name = "forward_joint_{}_{}@Heter".format( - block_index - 1, block_index) + block_index - 1, block_index + ) else: block_input_var_name = "backward_joint_{}_{}@Heter".format( - block_index + 1, block_index) + block_index + 1, block_index + ) entrance_var_list.sort() # input @@ -1402,7 +1492,7 @@ def get_communicate_var_info(program, # output # var -> reshape -> var@Heter_SERVER_BLOCK@INPUT_RESHAPE_VAR -> concat -> Heter_SERVER_BLOCK_index@JOINT_VAR - #for var_name in exit_var_list: + # for var_name in exit_var_list: # var = program.global_block().vars[var_name] # shape = var.shape # # if len(shape) < 2 or shape[0] != -1: @@ -1473,11 +1563,14 @@ def union_forward_gradient_op(program_block_ops_list): ''' union_program_block_ops_list = [] - assert block_length % 2 != 0, "the length of program_block_ops_list should be odd" + assert ( + block_length % 2 != 0 + ), "the length of program_block_ops_list should be odd" for i in range(0, block_length // 2): block_op_list = {"forward": program_block_ops_list[i]} block_op_list.update( - {"backward": program_block_ops_list[block_length - 1 - i]}) + {"backward": program_block_ops_list[block_length - 1 - i]} + ) union_program_block_ops_list.append(block_op_list) block_op_list = {"forward": [], "backward": []} @@ -1491,13 +1584,15 @@ def union_forward_gradient_op(program_block_ops_list): def find_block_joints(program, program_block_ops_list, heter_ops): - block_var_detail = find_entrance_exit_private(program, - program_block_ops_list) - block_var_detail = entrance_exit_check(program, program_block_ops_list, - block_var_detail, heter_ops) - block_var_detail = delete_block_useless_exit(program, - program_block_ops_list, - block_var_detail) + block_var_detail = find_entrance_exit_private( + program, program_block_ops_list + ) + block_var_detail = entrance_exit_check( + program, program_block_ops_list, block_var_detail, heter_ops + ) + block_var_detail = delete_block_useless_exit( + program, program_block_ops_list, block_var_detail + ) return block_var_detail @@ -1508,9 +1603,11 @@ def find_entrance_exit_private(program, program_block_ops_list): for index, block_op_list in enumerate(program_block_ops_list): ## forward block_input, block_output = find_ops_list_input_output( - program, block_op_list["forward"]) + program, block_op_list["forward"] + ) persistables = screen_persistables( - program, block_input) + screen_persistables(program, block_output) + program, block_input + ) + screen_persistables(program, block_output) # find entrance & exit block_private_vars = list(set(block_input) & set(block_output)) block_entrance = list(set(block_input) - set(block_private_vars)) @@ -1520,35 +1617,40 @@ def find_entrance_exit_private(program, program_block_ops_list): "entrance": block_entrance, "exit": block_exit, "private": block_private_vars, - "persistables": persistables + "persistables": persistables, } } ## backward bp_block_input, bp_block_output = find_ops_list_input_output( - program, block_op_list["backward"]) + program, block_op_list["backward"] + ) bp_persistables = screen_persistables( - program, bp_block_input) + screen_persistables( - program, bp_block_output) + program, bp_block_input + ) + screen_persistables(program, bp_block_output) # find entrance & exit bp_block_private_vars = list(set(bp_block_input) & set(bp_block_output)) bp_block_entrance = list( - set(bp_block_input) - set(bp_block_private_vars)) + set(bp_block_input) - set(bp_block_private_vars) + ) bp_block_exit = list(set(bp_block_output) - set(bp_block_private_vars)) - detail.update({ - "backward": { - "entrance": bp_block_entrance, - "exit": bp_block_exit, - "private": bp_block_private_vars, - "persistables": bp_persistables + detail.update( + { + "backward": { + "entrance": bp_block_entrance, + "exit": bp_block_exit, + "private": bp_block_private_vars, + "persistables": bp_persistables, + } } - }) + ) block_var_detail.append(detail) return block_var_detail -def entrance_exit_check(program, program_block_ops_list, block_var_detail, - heter_ops): +def entrance_exit_check( + program, program_block_ops_list, block_var_detail, heter_ops +): for index in range(len(block_var_detail) - 1, -1, -1): if index - 1 < 0: break @@ -1558,9 +1660,11 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, backward_entrance = block_var_detail[index]["backward"]["entrance"] - forward_all = block_var_detail[index]["forward"][ - "entrance"] + block_var_detail[index]["forward"][ - "private"] + block_var_detail[index]["forward"]["exit"] + forward_all = ( + block_var_detail[index]["forward"]["entrance"] + + block_var_detail[index]["forward"]["private"] + + block_var_detail[index]["forward"]["exit"] + ) for var in backward_entrance: if not ("@GRAD" in var) and not (var in forward_all): @@ -1571,18 +1675,24 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, if previous_block_exit == current_block_entrance: continue exist_vars = list( - set(previous_block_exit) & set(current_block_entrance)) + set(previous_block_exit) & set(current_block_entrance) + ) need_add_vars = list(set(current_block_entrance) - set(exist_vars)) # var in different stage should not be ignored, since they are not placed in the same program & device - #need_add_vars = find_need_var_from_previous_block( + # need_add_vars = find_need_var_from_previous_block( # need_add_vars, block_var_detail, index, heter_ops) - previous_block_private = block_var_detail[index - - 1]["forward"]["private"] - previous_block_entrance = block_var_detail[index - - 1]["forward"]["entrance"] + previous_block_private = block_var_detail[index - 1]["forward"][ + "private" + ] + previous_block_entrance = block_var_detail[index - 1]["forward"][ + "entrance" + ] for var in need_add_vars: - if var not in previous_block_private and var not in previous_block_entrance: + if ( + var not in previous_block_private + and var not in previous_block_entrance + ): previous_block_entrance.append(var) previous_block_exit.append(var) if not var in current_block_entrance: @@ -1598,27 +1708,35 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, if previous_block_exit == current_block_entrance: continue exist_vars = list( - set(previous_block_exit) & set(current_block_entrance)) + set(previous_block_exit) & set(current_block_entrance) + ) need_add_vars = list(set(current_block_entrance) - set(exist_vars)) need_ignore_vars = [] for var in need_add_vars: if not "@GRAD" in var: need_ignore_vars.append(var) need_add_vars = list( - set(need_add_vars).difference(set(need_ignore_vars))) - previous_block_private = block_var_detail[index + - 1]["backward"]["private"] - previous_block_entrance = block_var_detail[index + - 1]["backward"]["entrance"] + set(need_add_vars).difference(set(need_ignore_vars)) + ) + previous_block_private = block_var_detail[index + 1]["backward"][ + "private" + ] + previous_block_entrance = block_var_detail[index + 1]["backward"][ + "entrance" + ] for var in need_add_vars: - if var not in previous_block_private and var not in previous_block_entrance: + if ( + var not in previous_block_private + and var not in previous_block_entrance + ): previous_block_entrance.append(var) previous_block_exit.append(var) return block_var_detail -def find_need_var_from_previous_block(need_add_vars, block_var_detail, - current_index, heter_ops): +def find_need_var_from_previous_block( + need_add_vars, block_var_detail, current_index, heter_ops +): # create index_device_map index_device_map = {} for index in range(len(block_var_detail)): @@ -1633,15 +1751,21 @@ def find_need_var_from_previous_block(need_add_vars, block_var_detail, # if need_add_var in current device, no need communicate for var in need_add_vars: - while (pre_index >= 0): + while pre_index >= 0: previous_block_private = block_var_detail[pre_index]["private"] previous_block_exit = block_var_detail[pre_index]["exit"] previous_block_entrance = block_var_detail[pre_index]["entrance"] - total_var = previous_block_private + previous_block_exit + previous_block_entrance + total_var = ( + previous_block_private + + previous_block_exit + + previous_block_entrance + ) if var in total_var: - if index_device_map[current_index] == index_device_map[ - pre_index] and index_device_map[ - current_index] == DEFAULT_DEVICE: + if ( + index_device_map[current_index] + == index_device_map[pre_index] + and index_device_map[current_index] == DEFAULT_DEVICE + ): need_ignore_var.append(var) break pre_index -= 1 @@ -1650,8 +1774,9 @@ def find_need_var_from_previous_block(need_add_vars, block_var_detail, return need_add_vars -def delete_block_useless_exit(program, program_block_ops_list, - block_var_detail): +def delete_block_useless_exit( + program, program_block_ops_list, block_var_detail +): ## forward for index in range(len(block_var_detail)): if index == len(block_var_detail) - 1: @@ -1670,8 +1795,9 @@ def delete_block_useless_exit(program, program_block_ops_list, if index - 1 < 0: break current_block_exit = block_var_detail[index]["backward"]["exit"] - next_block_entrance = block_var_detail[index - - 1]["backward"]["entrance"] + next_block_entrance = block_var_detail[index - 1]["backward"][ + "entrance" + ] need_delete_var = [] for var in current_block_exit: if var not in next_block_entrance: @@ -1706,71 +1832,82 @@ def screen_persistables(program, var_list): return need_remove -def insert_reshape_op(program, - block, - index, - var_name, - new_var_name, - new_var_shape=None): +def insert_reshape_op( + program, block, index, var_name, new_var_name, new_var_shape=None +): input_var = block.vars[var_name] if new_var_name not in block.vars: - out = block.create_var(name=new_var_name, - shape=new_var_shape, - dtype=input_var.dtype, - type=input_var.type) + out = block.create_var( + name=new_var_name, + shape=new_var_shape, + dtype=input_var.dtype, + type=input_var.type, + ) else: out = block.vars[new_var_name] new_var_shape = out.shape - x_shape = block.create_var(name="{}.xshape@Heter".format(var_name), - dtype=input_var.dtype) - block._insert_op(index=index, - type="reshape2", - inputs={"X": input_var}, - attrs={'shape': new_var_shape}, - outputs={ - "Out": out, - "XShape": x_shape - }) - - -def insert_send_concat_op(program, block, index, var_name_list, new_var_name, - new_var_shape): + x_shape = block.create_var( + name="{}.xshape@Heter".format(var_name), dtype=input_var.dtype + ) + block._insert_op( + index=index, + type="reshape2", + inputs={"X": input_var}, + attrs={'shape': new_var_shape}, + outputs={"Out": out, "XShape": x_shape}, + ) + + +def insert_send_concat_op( + program, block, index, var_name_list, new_var_name, new_var_shape +): input_var_list = [block.vars[var_name] for var_name in var_name_list] - out = program.global_block().create_var(name=new_var_name, - shape=new_var_shape, - dtype=input_var_list[0].dtype, - type=input_var_list[0].type) - - block._insert_op(index=index, - type='concat', - inputs={"X": input_var_list}, - outputs={'Out': [out]}, - attrs={ - 'axis': -1, - 'use_stack': False - }) - - -def insert_recv_slice_op(program, block, index, var_name, var_shape, dtype, - type, new_var_name_list, new_var_shape_list): + out = program.global_block().create_var( + name=new_var_name, + shape=new_var_shape, + dtype=input_var_list[0].dtype, + type=input_var_list[0].type, + ) + + block._insert_op( + index=index, + type='concat', + inputs={"X": input_var_list}, + outputs={'Out': [out]}, + attrs={'axis': -1, 'use_stack': False}, + ) + + +def insert_recv_slice_op( + program, + block, + index, + var_name, + var_shape, + dtype, + type, + new_var_name_list, + new_var_shape_list, +): if var_name not in program.global_block().vars: - input_var = program.global_block().create_var(name=var_name, - shape=var_shape, - dtype=dtype, - type=type) + input_var = program.global_block().create_var( + name=var_name, shape=var_shape, dtype=dtype, type=type + ) else: input_var = program.global_block().vars[var_name] out_list = [] for i in range(len(new_var_name_list)): if new_var_name_list[i] not in block.vars: - out = block.create_var(name=new_var_name_list[i], - shape=new_var_shape_list[i], - dtype=input_var.dtype, - type=input_var.type) + out = block.create_var( + name=new_var_name_list[i], + shape=new_var_shape_list[i], + dtype=input_var.dtype, + type=input_var.type, + ) else: out = block.vars[new_var_name_list[i]] out_list.append(out) @@ -1787,25 +1924,31 @@ def insert_recv_slice_op(program, block, index, var_name, var_shape, dtype, attrs['starts'] = starts attrs['ends'] = ends - block._insert_op(index=index, - type='slice', - inputs={'Input': input_var}, - attrs=attrs, - outputs={'Out': out_list[i]}) + block._insert_op( + index=index, + type='slice', + inputs={'Input': input_var}, + attrs=attrs, + outputs={'Out': out_list[i]}, + ) start_index = end_index index += 1 -def add_heter_trainer_useful_vars(config, program, heter_program, heter_block, - static_var): +def add_heter_trainer_useful_vars( + config, program, heter_program, heter_block, static_var +): static_var = list(set(static_var)) for var_name in static_var: - if var_name not in heter_program.global_block( - ).vars and var_name not in heter_block.vars: + if ( + var_name not in heter_program.global_block().vars + and var_name not in heter_block.vars + ): var = program.global_block().vars[var_name] if var.persistable: heter_program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: heter_block._clone_variable(var, force_persistable=False) @@ -1815,14 +1958,18 @@ def delete_trainer_useless_var(config, program, static_var): program_useful_var_list = [] for op in program.global_block().ops: input_var_list, output_var_list = find_op_input_output( - program, program.global_block(), op) + program, program.global_block(), op + ) op_var_list = list(set(input_var_list).union(set(output_var_list))) program_useful_var_list = list( - set(program_useful_var_list).union(set(op_var_list))) + set(program_useful_var_list).union(set(op_var_list)) + ) program_useful_var_list += static_var program_useless_var_list = list( set(get_vars_name_in_block(program.global_block())).difference( - set(program_useful_var_list))) + set(program_useful_var_list) + ) + ) for var in program_useless_var_list: program.global_block()._remove_var(var) return program_useless_var_list @@ -1837,11 +1984,14 @@ def block_append_op(program, origin_program, block, op): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: - if var.name not in program.global_block( - ).vars and var.name not in block.vars: + if ( + var.name not in program.global_block().vars + and var.name not in block.vars + ): if var.persistable: program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) @@ -1850,20 +2000,22 @@ def block_append_op(program, origin_program, block, op): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: - if var.name not in program.global_block( - ).vars and var.name not in block.vars: + if ( + var.name not in program.global_block().vars + and var.name not in block.vars + ): if var.persistable: program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) if "_grad" not in op.type: # for forward op - return block.append_op(type=op.type, - inputs=inputs, - outputs=outputs, - attrs=op.all_attrs()) + return block.append_op( + type=op.type, inputs=inputs, outputs=outputs, attrs=op.all_attrs() + ) else: # for grad op op_desc = op.desc @@ -1885,12 +2037,15 @@ def block_append_op(program, origin_program, block, op): def add_vars_by_var_list(var_name_list, origin_program, program, block): for var_name in var_name_list: - if var_name not in program.global_block( - ).vars and var_name not in block.vars: + if ( + var_name not in program.global_block().vars + and var_name not in block.vars + ): var = origin_program.global_block().vars[var_name] if var.persistable: - program.global_block()._clone_variable(var, - force_persistable=False) + program.global_block()._clone_variable( + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/ufind.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/ufind.py index 0e30d0e3f9c..fef6f24570c 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/ufind.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/ufind.py @@ -14,7 +14,7 @@ class UnionFind(object): - """ Union-find data structure. + """Union-find data structure. Union-find is a data structure that keeps track of a set of elements partitioned into a number of disjoint (non-overlapping) subsets. diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py index b1beecd66d5..745c05d986a 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py @@ -29,7 +29,6 @@ dtype_to_size = { class VarBlock: - def __init__(self, varname, offset, size): self.varname = varname # NOTE: real offset is offset * size @@ -48,8 +47,9 @@ def create_var_struct(var): else: raise ValueError("can only support SELECTED_ROWS/LOD_TENSOR now") - return VarStruct(var.name, var.shape, var.dtype, var.type, lod_level, - var.persistable) + return VarStruct( + var.name, var.shape, var.dtype, var.type, lod_level, var.persistable + ) class VarStruct(object): @@ -70,8 +70,14 @@ class VarStruct(object): def __str__(self): return "N: {}, S: {}, D: {}, T: {}, LL: {}, P: {}, M: {}".format( - self.name, self.shape, self.dtype, self.type, self.lod_level, - self.persistable, self.m_size) + self.name, + self.shape, + self.dtype, + self.type, + self.lod_level, + self.persistable, + self.m_size, + ) class VarDistributed(object): @@ -81,14 +87,16 @@ class VarDistributed(object): the slice var's properties, such as type/shape/offset/endpoint. """ - def __init__(self, - origin_var, - slice_var, - is_slice=None, - block_id=None, - offset=None, - vtype=None, - endpoint=None): + def __init__( + self, + origin_var, + slice_var, + is_slice=None, + block_id=None, + offset=None, + vtype=None, + endpoint=None, + ): """ Args: origin_var(Variable|VarStruct): origin var properties @@ -138,26 +146,45 @@ class VarDistributed(object): """ assert isinstance(var1, VarStruct) and isinstance(var2, VarStruct) - return var1.name == var2.name and \ - var1.type == var2.type and \ - var1.shape == var2.shape and \ - var1.dtype == var2.dtype and \ - var1.lod_level == var2.lod_level and \ - var1.persistable == var2.persistable + return ( + var1.name == var2.name + and var1.type == var2.type + and var1.shape == var2.shape + and var1.dtype == var2.dtype + and var1.lod_level == var2.lod_level + and var1.persistable == var2.persistable + ) def __str__(self): - origin_var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})". \ - format(i="{", e="}", name=self.origin.name, type=self.origin.type, - shape=self.origin.shape, dtype=self.origin.dtype) - - slice_var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})" \ - ".slice({is_slice}).block({block_id}).offset({offset})". \ - format(i="{", e="}", name=self.slice.name, type=self.slice.type, - shape=self.slice.shape, dtype=self.slice.dtype, - is_slice=self.is_slice, block_id=self.block_id, offset=self.offset) + origin_var_str = ( + "{name} : fluid.{type}.shape{shape}.astype({dtype})".format( + i="{", + e="}", + name=self.origin.name, + type=self.origin.type, + shape=self.origin.shape, + dtype=self.origin.dtype, + ) + ) + + slice_var_str = ( + "{name} : fluid.{type}.shape{shape}.astype({dtype})" + ".slice({is_slice}).block({block_id}).offset({offset})".format( + i="{", + e="}", + name=self.slice.name, + type=self.slice.type, + shape=self.slice.shape, + dtype=self.slice.dtype, + is_slice=self.is_slice, + block_id=self.block_id, + offset=self.offset, + ) + ) return "var owned: {}, origin var: ( {} ), slice var: ( {} ), endpoint: {} ".format( - self.vtype, origin_var_str, slice_var_str, self.endpoint) + self.vtype, origin_var_str, slice_var_str, self.endpoint + ) class VarsDistributed(object): @@ -171,14 +198,16 @@ class VarsDistributed(object): def __init__(self): self.distributed_vars = [] - def add_distributed_var(self, - origin_var, - slice_var, - is_slice=None, - block_id=None, - offset=None, - vtype=None, - endpoint=None): + def add_distributed_var( + self, + origin_var, + slice_var, + is_slice=None, + block_id=None, + offset=None, + vtype=None, + endpoint=None, + ): """ add distributed var in this. @@ -194,5 +223,13 @@ class VarsDistributed(object): None """ self.distributed_vars.append( - VarDistributed(origin_var, slice_var, is_slice, block_id, offset, - vtype, endpoint)) + VarDistributed( + origin_var, + slice_var, + is_slice, + block_id, + offset, + vtype, + endpoint, + ) + ) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/mode.py b/python/paddle/fluid/incubate/fleet/parameter_server/mode.py index 0733f9b8a23..623e919ba35 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/mode.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/mode.py @@ -17,6 +17,7 @@ class PSMode: """ There are various mode for fleet, each of them is designed for different model. """ + TRANSPILER = 1 PSLIB = 2 diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py index ef6c34b23fd..39d6d3ff1c7 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py @@ -49,8 +49,9 @@ class PSLib(Fleet): if isinstance(role_maker, HeterRoleMaker): self._heter_ptr = fluid.core.Heter() - def _set_client_communication_config(self, request_timeout_ms, - connect_timeout_ms, max_retry): + def _set_client_communication_config( + self, request_timeout_ms, connect_timeout_ms, max_retry + ): self._client2client_request_timeout_ms = request_timeout_ms self._client2client_connect_timeout_ms = connect_timeout_ms self._client2client_max_retry = max_retry @@ -70,33 +71,41 @@ class PSLib(Fleet): if len(self._main_programs) == 0: raise ValueError( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) if self._opt_info: if "fleet_desc" in self._opt_info: self._dist_desc_str = text_format.MessageToString( - self._opt_info["fleet_desc"]) + self._opt_info["fleet_desc"] + ) self._dist_desc = self._opt_info["fleet_desc"] else: raise Exception( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) # barrier_all for init_server, wait for server starts if isinstance(self._role_maker, HeterRoleMaker): if self._role_maker.is_xpu(): local_endpoint = self._role_maker.get_local_endpoint() local_endpoint = local_endpoint.split(":") - self._heter_ptr.start_xpu_service(str(local_endpoint[0]), - int(local_endpoint[1])) + self._heter_ptr.start_xpu_service( + str(local_endpoint[0]), int(local_endpoint[1]) + ) self._role_maker._barrier_all() self.all_ips_ = self._role_maker._all_gather(self._local_ip) # worker_index * 2 is for compatible with older versions of pslib - self._fleet_ptr.init_worker(self._dist_desc_str, self.all_ips_, - self._role_maker._get_size(), - self._role_maker.worker_index() * 2) + self._fleet_ptr.init_worker( + self._dist_desc_str, + self.all_ips_, + self._role_maker._get_size(), + self._role_maker.worker_index() * 2, + ) if isinstance(self._role_maker, HeterRoleMaker): if self._role_maker.is_worker(): self._heter_ptr.set_xpu_list( - self._role_maker._xpu_endpoints) + self._role_maker._xpu_endpoints + ) self._heter_ptr.create_client2xpu_connection() # barrier_all for init_worker self._role_maker._barrier_all() @@ -110,7 +119,8 @@ class PSLib(Fleet): self._fleet_ptr.set_client2client_config( self._client2client_request_timeout_ms, self._client2client_connect_timeout_ms, - self._client2client_max_retry) + self._client2client_max_retry, + ) self._fleet_ptr.create_client2client_connection() # barrier for init model self._role_maker._barrier_worker() @@ -136,19 +146,22 @@ class PSLib(Fleet): var_name = table.dense_variable_name[i] if scope.find_var(var_name) is None: raise ValueError( - "var " + var_name + - " not found in scope, " + - "you should run startup program first") + "var " + + var_name + + " not found in scope, " + + "you should run startup program first" + ) var_name_list.append(var_name) if not self._opt_info["use_ps_gpu"]: - self._fleet_ptr.init_model(scope, - int(table.table_id), - var_name_list) + self._fleet_ptr.init_model( + scope, int(table.table_id), var_name_list + ) # barrier for init model done self._role_maker._barrier_worker() else: raise NameError( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) def init_server(self, model_dir=None, **kwargs): """ @@ -177,83 +190,102 @@ class PSLib(Fleet): def run_server(self): """ - init_pserver(): will be called by user. When a user knows current process is_worker(), he/she - should call init_pserver() to initialize global information about parameter server + init_pserver(): will be called by user. When a user knows current process is_worker(), he/she + should call init_pserver() to initialize global information about parameter server """ if self._opt_info: if "fleet_desc" in self._opt_info: self._dist_desc_str = text_format.MessageToString( - self._opt_info["fleet_desc"]) + self._opt_info["fleet_desc"] + ) self._dist_desc = self._opt_info["fleet_desc"] else: raise Exception( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) # server_index * 2 is for compatible with older versions of pslib - self._fleet_ptr.init_server(self._dist_desc_str, - self._role_maker.server_index() * 2) + self._fleet_ptr.init_server( + self._dist_desc_str, self._role_maker.server_index() * 2 + ) if isinstance(self._role_maker, MPISymetricRoleMaker): self._local_ip = self._fleet_ptr.run_server() else: local_endpoint = self._role_maker.get_local_endpoint() local_endpoint = local_endpoint.split(":") self._local_ip = self._fleet_ptr.run_server( - str(local_endpoint[0]), int(local_endpoint[1])) + str(local_endpoint[0]), int(local_endpoint[1]) + ) # barrier_all for init_server self._role_maker._barrier_all() self.all_ips_ = self._role_maker._all_gather(self._local_ip) - self._fleet_ptr.gather_servers(self.all_ips_, - self._role_maker._get_size()) + self._fleet_ptr.gather_servers( + self.all_ips_, self._role_maker._get_size() + ) # barrier_all for init_worker, wait all workers start self._role_maker._barrier_all() else: raise Exception( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) def end_pass(self, scope): if self._role_maker.worker_index() < self._role_maker.xpu_num(): self._heter_ptr.end_pass(scope, self._role_maker.worker_index()) self._heter_ptr.stop_xpu_service(self._role_maker.worker_index()) - def train_from_dataset(self, - executor, - program=None, - dataset=None, - scope=None, - thread=0, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): - """ - - """ + def train_from_dataset( + self, + executor, + program=None, + dataset=None, + scope=None, + thread=0, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): + """ """ if self._role_maker.is_worker(): self._role_maker._barrier_heter() - executor.train_from_dataset(program, dataset, scope, thread, debug, - fetch_list, fetch_info, print_period, - fetch_handler) - - def start_heter_trainer(self, - executor, - program=None, - scope=None, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): - """ - - """ - - trainer_instance = executor.start_heter_trainer(program, scope, debug, - fetch_list, fetch_info, - print_period, - fetch_handler) + executor.train_from_dataset( + program, + dataset, + scope, + thread, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + ) + + def start_heter_trainer( + self, + executor, + program=None, + scope=None, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): + """ """ + + trainer_instance = executor.start_heter_trainer( + program, + scope, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + ) if self._role_maker.is_xpu(): print("barrier heter") self._role_maker._barrier_heter() @@ -293,13 +325,15 @@ class PSLib(Fleet): self._optimizer = DownpourOptimizer(optimizer, strategy) return self._optimizer - def save_inference_model(self, - executor, - dirname, - feeded_var_names=None, - target_vars=None, - main_program=None, - export_for_deployment=True): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names=None, + target_vars=None, + main_program=None, + export_for_deployment=True, + ): """ save pserver model called from a worker Args: @@ -369,12 +403,9 @@ class PSLib(Fleet): self._fleet_ptr.save_model(dirname, mode) self._role_maker._barrier_worker() - def save_model_with_whitelist(self, - executor, - dirname, - whitelist_path, - main_program=None, - **kwargs): + def save_model_with_whitelist( + self, executor, dirname, whitelist_path, main_program=None, **kwargs + ): """ save whitelist, mode is consistent with fleet.save_persistables, when using fleet, it will save sparse and dense feature @@ -400,8 +431,9 @@ class PSLib(Fleet): self._fleet_ptr.client_flush() self._role_maker._barrier_worker() if self._role_maker.is_first_worker(): - self._fleet_ptr.save_model_with_whitelist(table_id, dirname, mode, - whitelist_path) + self._fleet_ptr.save_model_with_whitelist( + table_id, dirname, mode, whitelist_path + ) self._role_maker._barrier_worker() def save_multi_table_one_path(self, table_ids, model_dir, **kwargs): @@ -422,8 +454,9 @@ class PSLib(Fleet): mode = kwargs.get("mode", 0) self._role_maker._barrier_worker() if self._role_maker.is_first_worker(): - self._fleet_ptr.save_multi_table_one_path(table_ids, model_dir, - mode) + self._fleet_ptr.save_multi_table_one_path( + table_ids, model_dir, mode + ) self._role_maker._barrier_worker() def save_cache_model(self, executor, dirname, main_program=None, **kwargs): @@ -452,12 +485,13 @@ class PSLib(Fleet): if self._role_maker.is_first_worker(): cache_threshold = self._fleet_ptr.get_cache_threshold(table_id) - #check cache threshold right or not + # check cache threshold right or not self._role_maker._barrier_worker() if self._role_maker.is_first_worker(): - self._fleet_ptr.cache_shuffle(table_id, dirname, mode, - cache_threshold) + self._fleet_ptr.cache_shuffle( + table_id, dirname, mode, cache_threshold + ) self._role_maker._barrier_worker() @@ -516,8 +550,9 @@ class PSLib(Fleet): break if skip: continue - self._fleet_ptr.shrink_dense_table(i.table_id, scope, - var_list, decay, emb_dim) + self._fleet_ptr.shrink_dense_table( + i.table_id, scope, var_list, decay, emb_dim + ) self._role_maker._barrier_worker() def clear_one_table(self, table_id): @@ -587,8 +622,9 @@ class PSLib(Fleet): self._role_maker._barrier_worker() mode = kwargs.get("mode", 0) if self._role_maker.is_first_worker(): - self._fleet_ptr.load_table_with_whitelist(table_id, model_path, - mode) + self._fleet_ptr.load_table_with_whitelist( + table_id, model_path, mode + ) self._role_maker._barrier_worker() def load_one_table(self, table_id, model_path, **kwargs): @@ -631,20 +667,27 @@ class PSLib(Fleet): load_combine = kwargs.get("load_combine", False) self._role_maker._barrier_worker() if scope is not None and model_proto_file is not None: - self._load_one_table_from_paddle_model(scope, table_id, model_path, - model_proto_file, var_names, - load_combine) + self._load_one_table_from_paddle_model( + scope, + table_id, + model_path, + model_proto_file, + var_names, + load_combine, + ) elif self._role_maker.is_first_worker(): self._fleet_ptr.load_model_one_table(table_id, model_path, mode) self._role_maker._barrier_worker() - def _load_one_table_from_paddle_model(self, - scope, - table_id, - model_path, - model_proto_file, - var_names=None, - load_combine=False): + def _load_one_table_from_paddle_model( + self, + scope, + table_id, + model_path, + model_proto_file, + var_names=None, + load_combine=False, + ): """ load params from paddle model, and push params to pserver Args: @@ -660,26 +703,46 @@ class PSLib(Fleet): if self._role_maker.is_first_worker(): # get fs config from fleet_desc fs_name = self._opt_info["fleet_desc"].fs_client_param.uri - fs_ugi = self._opt_info["fleet_desc"].fs_client_param.user + "," + \ - self._opt_info["fleet_desc"].fs_client_param.passwd + fs_ugi = ( + self._opt_info["fleet_desc"].fs_client_param.user + + "," + + self._opt_info["fleet_desc"].fs_client_param.passwd + ) hadoop_bin = self._opt_info["fleet_desc"].fs_client_param.hadoop_bin # download model_path if it's hdfs/afs if model_path.startswith("hdfs:") or model_path.startswith("afs:"): dest = "./model_for_load_table_%s" % table_id - cmd = hadoop_bin + " fs -D fs.default.name=" + fs_name + \ - " -D hadoop.job.ugi=" + fs_ugi + " -get " + model_path + \ - " " + dest + cmd = ( + hadoop_bin + + " fs -D fs.default.name=" + + fs_name + + " -D hadoop.job.ugi=" + + fs_ugi + + " -get " + + model_path + + " " + + dest + ) ret = os.system(cmd) if ret != 0: raise RuntimeError("download model failed") model_path = dest # download model_proto_file if it's hdfs/afs - if model_proto_file.startswith("hdfs:") or \ - model_proto_file.startswith("afs:"): + if model_proto_file.startswith( + "hdfs:" + ) or model_proto_file.startswith("afs:"): dest = "./model_proto_file_for_load_table_%s" % table_id - cmd = hadoop_bin + " fs -D fs.default.name=" + fs_name + \ - " -D hadoop.job.ugi=" + fs_ugi + " -get " + \ - model_proto_file + " " + dest + cmd = ( + hadoop_bin + + " fs -D fs.default.name=" + + fs_name + + " -D hadoop.job.ugi=" + + fs_ugi + + " -get " + + model_proto_file + + " " + + dest + ) ret = os.system(cmd) if ret != 0: raise RuntimeError("download model proto file failed") @@ -697,8 +760,14 @@ class PSLib(Fleet): if skip: continue self._fleet_ptr.load_from_paddle_model( - scope, table_id, var_names, model_path, - model_proto_file, table_var_names, load_combine) + scope, + table_id, + var_names, + model_path, + model_proto_file, + table_var_names, + load_combine, + ) self._role_maker._barrier_worker() def confirm(self): @@ -784,7 +853,8 @@ class PSLib(Fleet): if self._role_maker.is_first_worker(): if prefix is not None: self._fleet_ptr.save_model_one_table_with_prefix( - table_id, model_dir, mode, prefix) + table_id, model_dir, mode, prefix + ) else: self._fleet_ptr.save_model_one_table(table_id, model_dir, mode) self._role_maker._barrier_worker() @@ -808,13 +878,15 @@ class PSLib(Fleet): fleet = PSLib() -def _prepare_params(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): +def _prepare_params( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): """ preprocess params, this interface is not for users. Args: @@ -844,16 +916,18 @@ def _prepare_params(input, if d_size.get(name) is None: d_size[name] = size elif d_size[name] != size: - raise ValueError("embedding size error: %s vs %s" % - (size, d_size[name])) + raise ValueError( + "embedding size error: %s vs %s" % (size, d_size[name]) + ) # check embedding accessor accessor = FLEET_GLOBAL_DICT["cur_accessor"] if d_accessor.get(name) is None: d_accessor[name] = accessor elif d_accessor[name] != accessor: - raise ValueError("embedding size error: %s vs %s" % - (d_accessor[name], accessor)) + raise ValueError( + "embedding size error: %s vs %s" % (d_accessor[name], accessor) + ) # check embedding table id if d_table.get(name) is None: @@ -869,13 +943,15 @@ def _prepare_params(input, raise ValueError("dtype must be float32") -def _fleet_embedding(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): +def _fleet_embedding( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): """ add fleet embedding, this interface is not for users. Args: @@ -888,8 +964,9 @@ def _fleet_embedding(input, dtype(str): data type of output """ # check and set params - _prepare_params(input, size, is_sparse, is_distributed, padding_idx, - param_attr, dtype) + _prepare_params( + input, size, is_sparse, is_distributed, padding_idx, param_attr, dtype + ) name = param_attr.name size = size[-1] if padding_idx is None: @@ -904,16 +981,19 @@ def _fleet_embedding(input, ctr_label_name=FLEET_GLOBAL_DICT["click_name"], padding_id=padding_idx, dtype=dtype, - scale_sparse_grad=FLEET_GLOBAL_DICT["scale_sparse_grad"]) - - -def _fleet_embedding_v2(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): + scale_sparse_grad=FLEET_GLOBAL_DICT["scale_sparse_grad"], + ) + + +def _fleet_embedding_v2( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): """ add fleet embedding v2, this interface is not for users. Args: @@ -926,8 +1006,9 @@ def _fleet_embedding_v2(input, dtype(str): data type of output """ # check and set params - _prepare_params(input, size, is_sparse, is_distributed, padding_idx, - param_attr, dtype) + _prepare_params( + input, size, is_sparse, is_distributed, padding_idx, param_attr, dtype + ) name = param_attr.name size = size[-1] if padding_idx is None: @@ -942,7 +1023,8 @@ def _fleet_embedding_v2(input, ctr_label_name=FLEET_GLOBAL_DICT["click_name"], padding_id=padding_idx, dtype=dtype, - scale_sparse_grad=FLEET_GLOBAL_DICT["scale_sparse_grad"]) + scale_sparse_grad=FLEET_GLOBAL_DICT["scale_sparse_grad"], + ) class fleet_embedding(object): @@ -1012,17 +1094,20 @@ class DownpourOptimizer(DistributedOptimizer): "Currently, distributed optimizer only support Adam" "Will config built-in adam for you." "We will support more functions in DistributedOptimizer", - sys.stderr) + sys.stderr, + ) self._optimizer_name = "DistributedAdam" self._distributed_optimizer = globals()[self._optimizer_name](optimizer) - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ Currently, backward function can not be called through DistributedOptimizer """ @@ -1050,7 +1135,8 @@ class DownpourOptimizer(DistributedOptimizer): current_endpoint = '' num_trainers = 0 if os.getenv('PADDLE_TRAINER_ENDPOINTS') and os.getenv( - 'PADDLE_CURRENT_ENDPOINT'): + 'PADDLE_CURRENT_ENDPOINT' + ): trainer_endpoints = os.getenv('PADDLE_TRAINER_ENDPOINTS') current_endpoint = os.getenv('PADDLE_CURRENT_ENDPOINT') num_trainers = len(trainer_endpoints.split(',')) @@ -1059,7 +1145,7 @@ class DownpourOptimizer(DistributedOptimizer): 'trainer_id': trainer_id, 'num_trainers': num_trainers, 'current_endpoint': current_endpoint, - 'trainer_endpoints': trainer_endpoints + 'trainer_endpoints': trainer_endpoints, } def _remove_collective_op_for_embedding(self, loss, table_name): @@ -1086,13 +1172,15 @@ class DownpourOptimizer(DistributedOptimizer): for index in need_remove_op_index: block._remove_op(index) - def minimize(self, - losses, - scopes=None, - startup_programs=None, - parameter_list=None, - no_grad_set=None, - program_mode="all_reduce"): + def minimize( + self, + losses, + scopes=None, + startup_programs=None, + parameter_list=None, + no_grad_set=None, + program_mode="all_reduce", + ): """ minimize a program through loss, loss can be a list in DistributedOptimizer. Note that in parameter server mode, a worker will not get anything about optimize_os @@ -1115,13 +1203,17 @@ class DownpourOptimizer(DistributedOptimizer): if not isinstance(losses, list): losses = [losses] - optimize_ops, param_grads, opt_info = \ - self._distributed_optimizer._minimize( - losses, - startup_programs, - parameter_list, - no_grad_set, - self._strategy) + ( + optimize_ops, + param_grads, + opt_info, + ) = self._distributed_optimizer._minimize( + losses, + startup_programs, + parameter_list, + no_grad_set, + self._strategy, + ) opt_info["mpi_rank"] = fleet.worker_index() opt_info["mpi_size"] = fleet.worker_num() fleet._set_opt_info(opt_info) @@ -1140,13 +1232,18 @@ class DownpourOptimizer(DistributedOptimizer): fleet._scopes = scopes if opt_info["use_ps_gpu"]: from paddle.fluid.transpiler.collective import MultiThread + # check start program if program_mode not in [ - "all_reduce", "fuse_all_reduce", "all_gather", - "all_reduce_xpu" + "all_reduce", + "fuse_all_reduce", + "all_gather", + "all_reduce_xpu", ]: - raise ValueError("You should set program_mode in [ all_reduce, \ - fuse_all_reduce, all_gather, all_reduce_xpu ]") + raise ValueError( + "You should set program_mode in [ all_reduce, \ + fuse_all_reduce, all_gather, all_reduce_xpu ]" + ) env = self.get_dist_env() if not isinstance(losses, list): startup_programs = [startup_programs] @@ -1155,19 +1252,23 @@ class DownpourOptimizer(DistributedOptimizer): t = MultiThread(trans_mode=program_mode) start_program = startup_programs[i] main_program = programs[i] - t.transpile(startup_program=start_program, - main_program=main_program, - rank=env["trainer_id"], - endpoints=env["trainer_endpoints"], - current_endpoint=env['current_endpoint'], - wait_port=False) + t.transpile( + startup_program=start_program, + main_program=main_program, + rank=env["trainer_id"], + endpoints=env["trainer_endpoints"], + current_endpoint=env['current_endpoint'], + wait_port=False, + ) if i > 0: - self._remove_collective_ops(start_program, - "c_comm_init_all") + self._remove_collective_ops( + start_program, "c_comm_init_all" + ) for i in range(0, len(losses)): loss = losses[i] embedding_table = self._distributed_optimizer._find_multi_distributed_lookup_table( - [loss]) + [loss] + ) self._remove_collective_op_for_embedding(loss, embedding_table) return [optimize_ops, param_grads] diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py index 308261cea06..d1c9fae5ccb 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py @@ -13,14 +13,15 @@ """Defination of Server and Worker.""" from . import ps_pb2 as pslib + # NOTE: reduce removed in fuctools in python3 from functools import reduce class Server(object): """ - A Server basic class - it's a base class, does not have implementation + A Server basic class + it's a base class, does not have implementation """ def __init__(self): @@ -29,8 +30,8 @@ class Server(object): class Worker(object): """ - A Worker basic class. - it's a base class, does not have implementation + A Worker basic class. + it's a base class, does not have implementation """ def __init__(self): @@ -39,18 +40,24 @@ class Worker(object): class DownpourServer(Server): """ - DownpourServer class is used to generate server program_desc - Args: - server: it is pslib.ServerParameter() - Examples: - server = DownpourServer() + DownpourServer class is used to generate server program_desc + Args: + server: it is pslib.ServerParameter() + Examples: + server = DownpourServer() """ def __init__(self): self._server = pslib.ServerParameter() - self._server.downpour_server_param.service_param.server_class = "DownpourBrpcPsServer" - self._server.downpour_server_param.service_param.client_class = "DownpourBrpcPsClient" - self._server.downpour_server_param.service_param.service_class = "DownpourPsService" + self._server.downpour_server_param.service_param.server_class = ( + "DownpourBrpcPsServer" + ) + self._server.downpour_server_param.service_param.client_class = ( + "DownpourBrpcPsClient" + ) + self._server.downpour_server_param.service_param.service_class = ( + "DownpourPsService" + ) self._server.downpour_server_param.service_param.start_server_port = 0 self._server.downpour_server_param.service_param.server_thread_num = 12 @@ -68,27 +75,60 @@ class DownpourServer(Server): if table.type == pslib.PS_SPARSE_TABLE: return else: - raise ValueError("expect table %s type=%s, but actual type=%s" \ - %(table_id, pslib.PS_SPARSE_TABLE, table.type)) + raise ValueError( + "expect table %s type=%s, but actual type=%s" + % (table_id, pslib.PS_SPARSE_TABLE, table.type) + ) if strategy is None: strategy = dict() table = self._server.downpour_server_param.downpour_table_param.add() table.table_id = table_id table.type = pslib.PS_SPARSE_TABLE - support_sparse_key_list = ['sparse_table_class', 'sparse_compress_in_save', 'sparse_shard_num', \ - 'sparse_accessor_class', 'sparse_learning_rate', 'sparse_initial_g2sum', 'sparse_initial_range', \ - 'sparse_weight_bounds', 'sparse_embedx_dim', 'sparse_embedx_threshold', 'sparse_nonclk_coeff', \ - 'sparse_click_coeff', 'sparse_base_threshold', 'sparse_delta_threshold', 'sparse_delta_keep_days', \ - 'sparse_delete_after_unseen_days', 'sparse_show_click_decay_rate', 'sparse_delete_threshold', \ - 'sparse_converter', 'sparse_deconverter', 'sparse_enable_cache', 'sparse_cache_rate', \ - 'sparse_cache_file_num', 'sparse_beta1_decay_rate', 'sparse_beta2_decay_rate', \ - 'sparse_ada_epsilon', 'sparse_optimizer', 'sparse_ssd_unseenday_threshold', \ - 'embed_sparse_optimizer', 'embed_sparse_learning_rate', 'embed_sparse_weight_bounds', \ - 'embed_sparse_initial_range', 'embed_sparse_initial_g2sum', 'embed_sparse_beta1_decay_rate', \ - 'embed_sparse_beta2_decay_rate', 'embedx_sparse_optimizer', 'embedx_sparse_learning_rate', \ - 'embedx_sparse_weight_bounds', 'embedx_sparse_initial_range', 'embedx_sparse_initial_g2sum', \ - 'embedx_sparse_beta1_decay_rate', 'embedx_sparse_beta2_decay_rate'] + support_sparse_key_list = [ + 'sparse_table_class', + 'sparse_compress_in_save', + 'sparse_shard_num', + 'sparse_accessor_class', + 'sparse_learning_rate', + 'sparse_initial_g2sum', + 'sparse_initial_range', + 'sparse_weight_bounds', + 'sparse_embedx_dim', + 'sparse_embedx_threshold', + 'sparse_nonclk_coeff', + 'sparse_click_coeff', + 'sparse_base_threshold', + 'sparse_delta_threshold', + 'sparse_delta_keep_days', + 'sparse_delete_after_unseen_days', + 'sparse_show_click_decay_rate', + 'sparse_delete_threshold', + 'sparse_converter', + 'sparse_deconverter', + 'sparse_enable_cache', + 'sparse_cache_rate', + 'sparse_cache_file_num', + 'sparse_beta1_decay_rate', + 'sparse_beta2_decay_rate', + 'sparse_ada_epsilon', + 'sparse_optimizer', + 'sparse_ssd_unseenday_threshold', + 'embed_sparse_optimizer', + 'embed_sparse_learning_rate', + 'embed_sparse_weight_bounds', + 'embed_sparse_initial_range', + 'embed_sparse_initial_g2sum', + 'embed_sparse_beta1_decay_rate', + 'embed_sparse_beta2_decay_rate', + 'embedx_sparse_optimizer', + 'embedx_sparse_learning_rate', + 'embedx_sparse_weight_bounds', + 'embedx_sparse_initial_range', + 'embedx_sparse_initial_g2sum', + 'embedx_sparse_beta1_decay_rate', + 'embedx_sparse_beta2_decay_rate', + ] for key in strategy: if key not in support_sparse_key_list: @@ -100,21 +140,30 @@ class DownpourServer(Server): if table_class not in support_table_calss: raise ValueError( "support sparse_table_class: [ 'DownpourSparseTable', 'DownpourSparseSSDTable'], \ - but actual %s" % (table_class)) + but actual %s" + % (table_class) + ) else: table_class = 'DownpourSparseTable' table.table_class = table_class - if table_class == 'DownpourSparseTable' or table_class == 'DownpourSparseSSDTable': + if ( + table_class == 'DownpourSparseTable' + or table_class == 'DownpourSparseSSDTable' + ): table.enable_sparse_table_cache = strategy.get( - 'sparse_enable_cache', True) + 'sparse_enable_cache', True + ) table.sparse_table_cache_rate = strategy.get( - 'sparse_cache_rate', 0.00055) + 'sparse_cache_rate', 0.00055 + ) table.sparse_table_cache_file_num = strategy.get( - 'sparse_cache_file_num', 16) - table.compress_in_save = strategy.get('sparse_compress_in_save', - True) + 'sparse_cache_file_num', 16 + ) + table.compress_in_save = strategy.get( + 'sparse_compress_in_save', True + ) table.shard_num = strategy.get('sparse_shard_num', 1000) # DownpourFeatureValueAccessor: for ctr task, has cvm, embedding and sgd info # DownpourCtrAccessor : for ctr task, has cvm, slot, embedding and sgd info @@ -123,10 +172,13 @@ class DownpourServer(Server): # DownpourUnitAccessor : for ctr task, has cvm, slot, embedding and sgd info support_accessor_class = [ - 'DownpourFeatureValueAccessor', 'DownpourCtrAccessor', - 'DownpourCtrDymfAccessor', 'DownpourSparseValueAccessor', - 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', - 'DownpourDoubleUnitAccessor' + 'DownpourFeatureValueAccessor', + 'DownpourCtrAccessor', + 'DownpourCtrDymfAccessor', + 'DownpourSparseValueAccessor', + 'DownpourCtrDoubleAccessor', + 'DownpourUnitAccessor', + 'DownpourDoubleUnitAccessor', ] if strategy.get('sparse_accessor_class') is not None: accessor_class = strategy.get('sparse_accessor_class') @@ -134,56 +186,76 @@ class DownpourServer(Server): raise ValueError( "support sparse_accessor_class: ['DownpourFeatureValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDymfAccessor', \ 'DownpourSparseValueAccessor', 'DownpourCtrDoubleAccessor'], \ - but actual %s" % (accessor_class)) + but actual %s" + % (accessor_class) + ) else: accessor_class = 'DownpourCtrAccessor' table.accessor.accessor_class = accessor_class - if accessor_class == 'DownpourFeatureValueAccessor' \ - or accessor_class == 'DownpourCtrAccessor' \ - or accessor_class == 'DownpourCtrDymfAccessor' \ - or accessor_class == 'DownpourCtrDoubleAccessor': + if ( + accessor_class == 'DownpourFeatureValueAccessor' + or accessor_class == 'DownpourCtrAccessor' + or accessor_class == 'DownpourCtrDymfAccessor' + or accessor_class == 'DownpourCtrDoubleAccessor' + ): table.accessor.sparse_sgd_param.learning_rate = strategy.get( - 'sparse_learning_rate', 0.05) + 'sparse_learning_rate', 0.05 + ) table.accessor.sparse_sgd_param.initial_g2sum = strategy.get( - 'sparse_initial_g2sum', 3) + 'sparse_initial_g2sum', 3 + ) table.accessor.sparse_sgd_param.initial_range = strategy.get( - 'sparse_initial_range', 1e-4) + 'sparse_initial_range', 1e-4 + ) if strategy.get('sparse_weight_bounds') is None: table.accessor.sparse_sgd_param.weight_bounds.extend( - [-10, 10]) + [-10, 10] + ) else: table.accessor.sparse_sgd_param.weight_bounds.extend( - strategy.get('sparse_weight_bounds')) + strategy.get('sparse_weight_bounds') + ) table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8) table.accessor.embedx_threshold = strategy.get( - 'sparse_embedx_threshold', 10) + 'sparse_embedx_threshold', 10 + ) table.accessor.fea_dim = int(table.accessor.embedx_dim) + 3 - table.accessor.downpour_accessor_param.nonclk_coeff = strategy.get( - 'sparse_nonclk_coeff', 0.1) - table.accessor.downpour_accessor_param.click_coeff = strategy.get( - 'sparse_click_coeff', 1) - table.accessor.downpour_accessor_param.base_threshold = strategy.get( - 'sparse_base_threshold', 1.5) - table.accessor.downpour_accessor_param.delta_threshold = strategy.get( - 'sparse_delta_threshold', 0.25) - table.accessor.downpour_accessor_param.delta_keep_days = strategy.get( - 'sparse_delta_keep_days', 16) + table.accessor.downpour_accessor_param.nonclk_coeff = ( + strategy.get('sparse_nonclk_coeff', 0.1) + ) + table.accessor.downpour_accessor_param.click_coeff = ( + strategy.get('sparse_click_coeff', 1) + ) + table.accessor.downpour_accessor_param.base_threshold = ( + strategy.get('sparse_base_threshold', 1.5) + ) + table.accessor.downpour_accessor_param.delta_threshold = ( + strategy.get('sparse_delta_threshold', 0.25) + ) + table.accessor.downpour_accessor_param.delta_keep_days = ( + strategy.get('sparse_delta_keep_days', 16) + ) table.accessor.downpour_accessor_param.delete_after_unseen_days = strategy.get( - 'sparse_delete_after_unseen_days', 30) + 'sparse_delete_after_unseen_days', 30 + ) table.accessor.downpour_accessor_param.ssd_unseenday_threshold = strategy.get( - 'sparse_ssd_unseenday_threshold', 1) - table.accessor.downpour_accessor_param.show_click_decay_rate = strategy.get( - 'sparse_show_click_decay_rate', 0.98) - table.accessor.downpour_accessor_param.delete_threshold = strategy.get( - 'sparse_delete_threshold', 0.8) + 'sparse_ssd_unseenday_threshold', 1 + ) + table.accessor.downpour_accessor_param.show_click_decay_rate = ( + strategy.get('sparse_show_click_decay_rate', 0.98) + ) + table.accessor.downpour_accessor_param.delete_threshold = ( + strategy.get('sparse_delete_threshold', 0.8) + ) converter = strategy.get( 'sparse_converter', - "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)") + "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)", + ) deconverter = strategy.get( 'sparse_deconverter', - "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)" + "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)", ) table1 = table.accessor.table_accessor_save_param.add() @@ -201,52 +273,69 @@ class DownpourServer(Server): table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8) table.accessor.fea_dim = int(table.accessor.embedx_dim) if optimizer_name == "naive": - table.accessor.sparse_commonsgd_param.naive.learning_rate = \ - strategy.get('sparse_learning_rate', 0.05) - table.accessor.sparse_commonsgd_param.naive.initial_range = \ - strategy.get('sparse_initial_range', 1e-4) + table.accessor.sparse_commonsgd_param.naive.learning_rate = strategy.get( + 'sparse_learning_rate', 0.05 + ) + table.accessor.sparse_commonsgd_param.naive.initial_range = strategy.get( + 'sparse_initial_range', 1e-4 + ) if strategy.get('sparse_weight_bounds') is None: table.accessor.sparse_commonsgd_param.naive.weight_bounds.extend( - [-10, 10]) + [-10, 10] + ) else: table.accessor.sparse_commonsgd_param.naive.weight_bounds.extend( - strategy.get('sparse_weight_bounds')) + strategy.get('sparse_weight_bounds') + ) elif optimizer_name == "adagrad": - table.accessor.sparse_commonsgd_param.adagrad.learning_rate = \ - strategy.get('sparse_learning_rate', 0.05) - table.accessor.sparse_commonsgd_param.adagrad.initial_range = \ - strategy.get('sparse_initial_range', 1e-4) + table.accessor.sparse_commonsgd_param.adagrad.learning_rate = strategy.get( + 'sparse_learning_rate', 0.05 + ) + table.accessor.sparse_commonsgd_param.adagrad.initial_range = strategy.get( + 'sparse_initial_range', 1e-4 + ) table.accessor.sparse_commonsgd_param.adagrad.initial_g2sum = strategy.get( - 'sparse_initial_g2sum', 3) + 'sparse_initial_g2sum', 3 + ) if strategy.get('sparse_weight_bounds') is None: table.accessor.sparse_commonsgd_param.adagrad.weight_bounds.extend( - [-10, 10]) + [-10, 10] + ) else: table.accessor.sparse_commonsgd_param.adagrad.weight_bounds.extend( - strategy.get('sparse_weight_bounds')) + strategy.get('sparse_weight_bounds') + ) elif optimizer_name == "adam": - table.accessor.sparse_commonsgd_param.adam.learning_rate = \ + table.accessor.sparse_commonsgd_param.adam.learning_rate = ( strategy.get('sparse_learning_rate', 0.001) - table.accessor.sparse_commonsgd_param.adam.initial_range = \ + ) + table.accessor.sparse_commonsgd_param.adam.initial_range = ( strategy.get('sparse_initial_range', 1e-4) + ) table.accessor.sparse_commonsgd_param.adam.beta1_decay_rate = strategy.get( - 'sparse_beta1_decay_rate', 0.9) + 'sparse_beta1_decay_rate', 0.9 + ) table.accessor.sparse_commonsgd_param.adam.beta2_decay_rate = strategy.get( - 'sparse_beta2_decay_rate', 0.999) - table.accessor.sparse_commonsgd_param.adam.ada_epsilon = strategy.get( - 'sparse_ada_epsilon', 1e-8) + 'sparse_beta2_decay_rate', 0.999 + ) + table.accessor.sparse_commonsgd_param.adam.ada_epsilon = ( + strategy.get('sparse_ada_epsilon', 1e-8) + ) if strategy.get('sparse_weight_bounds') is None: table.accessor.sparse_commonsgd_param.adam.weight_bounds.extend( - [-10, 10]) + [-10, 10] + ) else: table.accessor.sparse_commonsgd_param.adam.weight_bounds.extend( - strategy.get('sparse_weight_bounds')) + strategy.get('sparse_weight_bounds') + ) converter = strategy.get( 'sparse_converter', - "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)") + "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)", + ) deconverter = strategy.get( 'sparse_deconverter', - "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)" + "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)", ) table1 = table.accessor.table_accessor_save_param.add() @@ -258,15 +347,21 @@ class DownpourServer(Server): table2.param = 2 table2.converter = converter table2.deconverter = deconverter - elif accessor_class == 'DownpourUnitAccessor' or accessor_class == 'DownpourDoubleUnitAccessor': + elif ( + accessor_class == 'DownpourUnitAccessor' + or accessor_class == 'DownpourDoubleUnitAccessor' + ): self.add_sparse_table_common_config(table, strategy) - self.add_sparse_optimizer(table.accessor.embed_sgd_param, - strategy, "embed_") - self.add_sparse_optimizer(table.accessor.embedx_sgd_param, - strategy, "embedx_") + self.add_sparse_optimizer( + table.accessor.embed_sgd_param, strategy, "embed_" + ) + self.add_sparse_optimizer( + table.accessor.embedx_sgd_param, strategy, "embedx_" + ) - def add_dense_table(self, table_id, param_var, grad_var, strategy, - sparse_table_names): + def add_dense_table( + self, table_id, param_var, grad_var, strategy, sparse_table_names + ): """ Args: table_id(int): id of sparse params table @@ -292,45 +387,72 @@ class DownpourServer(Server): table.accessor.fea_dim = fea_dim return else: - raise ValueError("expect table %s type=%s, but actual type=%s" \ - %(table_id, pslib.PS_DENSE_TABLE, table.type)) + raise ValueError( + "expect table %s type=%s, but actual type=%s" + % (table_id, pslib.PS_DENSE_TABLE, table.type) + ) if strategy is None: strategy = dict() table = self._server.downpour_server_param.downpour_table_param.add() table.table_id = table_id - support_dense_key_list = ['dense_table_class', 'dense_compress_in_save', 'dense_accessor_class', \ - 'dense_optimizer', 'dense_learning_rate', 'dense_avg_decay', 'dense_ada_decay', \ - 'dense_ada_epsilon', 'dense_mom_decay', 'dense_naive_lr'] + support_dense_key_list = [ + 'dense_table_class', + 'dense_compress_in_save', + 'dense_accessor_class', + 'dense_optimizer', + 'dense_learning_rate', + 'dense_avg_decay', + 'dense_ada_decay', + 'dense_ada_epsilon', + 'dense_mom_decay', + 'dense_naive_lr', + ] for key in strategy: if key not in support_dense_key_list: raise ValueError("strategy key '%s' not support" % (key)) - table.table_class = strategy.get('dense_table_class', - "DownpourDenseTable") + table.table_class = strategy.get( + 'dense_table_class', "DownpourDenseTable" + ) table.type = pslib.PS_DENSE_TABLE table.compress_in_save = strategy.get('dense_compress_in_save', True) table.accessor.accessor_class = strategy.get( - 'dense_accessor_class', "DownpourDenseValueAccessor") + 'dense_accessor_class', "DownpourDenseValueAccessor" + ) table.accessor.dense_sgd_param.name = strategy.get( - 'dense_optimizer', "adam") + 'dense_optimizer', "adam" + ) table.accessor.dense_sgd_param.adam.learning_rate = strategy.get( - 'dense_learning_rate', 5e-06) + 'dense_learning_rate', 5e-06 + ) table.accessor.dense_sgd_param.adam.avg_decay_rate = strategy.get( - 'dense_avg_decay', 0.999993) + 'dense_avg_decay', 0.999993 + ) table.accessor.dense_sgd_param.adam.ada_decay_rate = strategy.get( - 'dense_ada_decay', 0.9999) + 'dense_ada_decay', 0.9999 + ) table.accessor.dense_sgd_param.adam.ada_epsilon = strategy.get( - 'dense_ada_epsilon', 1e-8) + 'dense_ada_epsilon', 1e-8 + ) table.accessor.dense_sgd_param.adam.mom_decay_rate = strategy.get( - 'dense_mom_decay', 0.99) + 'dense_mom_decay', 0.99 + ) table.accessor.dense_sgd_param.naive.learning_rate = strategy.get( - 'dense_naive_lr', 0.0002) + 'dense_naive_lr', 0.0002 + ) table.accessor.fea_dim = fea_dim - def add_data_norm_table(self, table_id, learning_rate, param_var, grad_var, - strategy, sparse_table_names): + def add_data_norm_table( + self, + table_id, + learning_rate, + param_var, + grad_var, + strategy, + sparse_table_names, + ): """ Args: table_id(int): id of datanorm table @@ -357,13 +479,20 @@ class DownpourServer(Server): table.accessor.fea_dim = fea_dim return else: - raise ValueError("expect table %s type=%s, but actual type=%s" \ - %(table_id, pslib.PS_DENSE_TABLE, table.type)) + raise ValueError( + "expect table %s type=%s, but actual type=%s" + % (table_id, pslib.PS_DENSE_TABLE, table.type) + ) if strategy is None: strategy = dict() - support_datanorm_key_list = ['datanorm_table_class', 'datanorm_compress_in_save', \ - 'datanorm_accessor_class', 'datanorm_operation', 'datanorm_decay_rate'] + support_datanorm_key_list = [ + 'datanorm_table_class', + 'datanorm_compress_in_save', + 'datanorm_accessor_class', + 'datanorm_operation', + 'datanorm_decay_rate', + ] for key in strategy: if key not in support_datanorm_key_list: @@ -371,91 +500,119 @@ class DownpourServer(Server): table = self._server.downpour_server_param.downpour_table_param.add() table.table_id = table_id - table.table_class = strategy.get('datanorm_table_class', - 'DownpourDenseTable') + table.table_class = strategy.get( + 'datanorm_table_class', 'DownpourDenseTable' + ) table.type = pslib.PS_DENSE_TABLE table.compress_in_save = strategy.get('datanorm_compress_in_save', True) table.accessor.accessor_class = strategy.get( - 'datanorm_accessor_class', 'DownpourDenseValueAccessor') + 'datanorm_accessor_class', 'DownpourDenseValueAccessor' + ) table.accessor.dense_sgd_param.name = strategy.get( - 'datanorm_operation', 'summary') - table.accessor.dense_sgd_param.summary.summary_decay_rate = strategy.get( - 'datanorm_decay_rate', 0.999999) + 'datanorm_operation', 'summary' + ) + table.accessor.dense_sgd_param.summary.summary_decay_rate = ( + strategy.get('datanorm_decay_rate', 0.999999) + ) table.accessor.fea_dim = fea_dim def add_sparse_optimizer(self, sgd, strategy, prefix): optimizer_name = strategy.get(prefix + "sparse_optimizer", "adagrad") sgd.name = optimizer_name if optimizer_name == "naive": - sgd.naive.learning_rate = \ - strategy.get(prefix + 'sparse_learning_rate', 0.05) - sgd.naive.initial_range = \ - strategy.get(prefix + 'sparse_initial_range', 1e-4) + sgd.naive.learning_rate = strategy.get( + prefix + 'sparse_learning_rate', 0.05 + ) + sgd.naive.initial_range = strategy.get( + prefix + 'sparse_initial_range', 1e-4 + ) bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10]) sgd.naive.weight_bounds.extend(bounds) elif optimizer_name == "adagrad": - sgd.adagrad.learning_rate = \ - strategy.get(prefix + 'sparse_learning_rate', 0.05) - sgd.adagrad.initial_range = \ - strategy.get(prefix + 'sparse_initial_range', 1e-4) + sgd.adagrad.learning_rate = strategy.get( + prefix + 'sparse_learning_rate', 0.05 + ) + sgd.adagrad.initial_range = strategy.get( + prefix + 'sparse_initial_range', 1e-4 + ) if prefix == "embed_": sgd.adagrad.initial_range = 0 sgd.adagrad.initial_g2sum = strategy.get( - prefix + 'sparse_initial_g2sum', 3) + prefix + 'sparse_initial_g2sum', 3 + ) bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10]) sgd.adagrad.weight_bounds.extend(bounds) elif optimizer_name == "std_adagrad": - sgd.adagrad.learning_rate = \ - strategy.get(prefix + 'sparse_learning_rate', 0.05) - sgd.adagrad.initial_range = \ - strategy.get(prefix + 'sparse_initial_range', 1e-4) + sgd.adagrad.learning_rate = strategy.get( + prefix + 'sparse_learning_rate', 0.05 + ) + sgd.adagrad.initial_range = strategy.get( + prefix + 'sparse_initial_range', 1e-4 + ) if prefix == "embed_": sgd.adagrad.initial_range = 0 sgd.adagrad.initial_g2sum = strategy.get( - prefix + 'sparse_initial_g2sum', 3) + prefix + 'sparse_initial_g2sum', 3 + ) bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10]) sgd.adagrad.weight_bounds.extend(bounds) elif optimizer_name == "adam": - sgd.adam.learning_rate = \ - strategy.get(prefix + 'sparse_learning_rate', 0.001) - sgd.adam.initial_range = \ - strategy.get(prefix + 'sparse_initial_range', 1e-4) + sgd.adam.learning_rate = strategy.get( + prefix + 'sparse_learning_rate', 0.001 + ) + sgd.adam.initial_range = strategy.get( + prefix + 'sparse_initial_range', 1e-4 + ) sgd.adam.beta1_decay_rate = strategy.get( - prefix + 'sparse_beta1_decay_rate', 0.9) + prefix + 'sparse_beta1_decay_rate', 0.9 + ) sgd.adam.beta2_decay_rate = strategy.get( - prefix + 'sparse_beta2_decay_rate', 0.999) - sgd.adam.ada_epsilon = strategy.get(prefix + 'sparse_ada_epsilon', - 1e-8) + prefix + 'sparse_beta2_decay_rate', 0.999 + ) + sgd.adam.ada_epsilon = strategy.get( + prefix + 'sparse_ada_epsilon', 1e-8 + ) bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10]) sgd.adam.weight_bounds.extend(bounds) def add_sparse_table_common_config(self, table, strategy): table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8) table.accessor.embedx_threshold = strategy.get( - 'sparse_embedx_threshold', 10) + 'sparse_embedx_threshold', 10 + ) table.accessor.fea_dim = int(table.accessor.embedx_dim) + 3 table.accessor.downpour_accessor_param.nonclk_coeff = strategy.get( - 'sparse_nonclk_coeff', 0.1) + 'sparse_nonclk_coeff', 0.1 + ) table.accessor.downpour_accessor_param.click_coeff = strategy.get( - 'sparse_click_coeff', 1) + 'sparse_click_coeff', 1 + ) table.accessor.downpour_accessor_param.base_threshold = strategy.get( - 'sparse_base_threshold', 1.5) + 'sparse_base_threshold', 1.5 + ) table.accessor.downpour_accessor_param.delta_threshold = strategy.get( - 'sparse_delta_threshold', 0.25) + 'sparse_delta_threshold', 0.25 + ) table.accessor.downpour_accessor_param.delta_keep_days = strategy.get( - 'sparse_delta_keep_days', 16) - table.accessor.downpour_accessor_param.delete_after_unseen_days = strategy.get( - 'sparse_delete_after_unseen_days', 30) - table.accessor.downpour_accessor_param.show_click_decay_rate = strategy.get( - 'sparse_show_click_decay_rate', 0.98) + 'sparse_delta_keep_days', 16 + ) + table.accessor.downpour_accessor_param.delete_after_unseen_days = ( + strategy.get('sparse_delete_after_unseen_days', 30) + ) + table.accessor.downpour_accessor_param.show_click_decay_rate = ( + strategy.get('sparse_show_click_decay_rate', 0.98) + ) table.accessor.downpour_accessor_param.delete_threshold = strategy.get( - 'sparse_delete_threshold', 0.8) + 'sparse_delete_threshold', 0.8 + ) converter = strategy.get( 'sparse_converter', - "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)") + "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)", + ) deconverter = strategy.get( 'sparse_deconverter', - "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)") + "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)", + ) table1 = table.accessor.table_accessor_save_param.add() table1.param = 1 @@ -476,23 +633,21 @@ class DownpourServer(Server): class DownpourWorker(Worker): """ - DownpourWorker class is used to generate worker program_desc - Args: - window (int): push params frequency - worker: it is pslib.DownpourTrainerParameter - Examples: - worker = DownpourWorker(1) + DownpourWorker class is used to generate worker program_desc + Args: + window (int): push params frequency + worker: it is pslib.DownpourTrainerParameter + Examples: + worker = DownpourWorker(1) """ def __init__(self, window): self.window = window self._worker = pslib.DownpourTrainerParameter() - def add_sparse_table(self, - table_id, - slot_key_vars, - slot_value_vars, - slot_value_grads=None): + def add_sparse_table( + self, table_id, slot_key_vars, slot_value_vars, slot_value_grads=None + ): """ Args: table_id(int): id of sparse params table @@ -503,8 +658,9 @@ class DownpourWorker(Worker): return None """ if slot_value_grads is None: - slot_value_grad_names = \ - [var.name + "@GRAD" for var in slot_value_vars] + slot_value_grad_names = [ + var.name + "@GRAD" for var in slot_value_vars + ] else: value_to_key = {} for i in range(len(slot_key_vars)): @@ -514,12 +670,19 @@ class DownpourWorker(Worker): for var in slot_value_vars: if var.name + "@GRAD" in all_grad_names: slot_value_grad_names.append(var.name + "@GRAD") - sorted_slot_value_vars = [i for i in slot_value_vars if \ - i.name + "@GRAD" in slot_value_grad_names] - sorted_slot_value_vars += [i for i in slot_value_vars if \ - i.name + "@GRAD" not in slot_value_grad_names] - sorted_slot_key_vars = \ - [value_to_key[v.name] for v in sorted_slot_value_vars] + sorted_slot_value_vars = [ + i + for i in slot_value_vars + if i.name + "@GRAD" in slot_value_grad_names + ] + sorted_slot_value_vars += [ + i + for i in slot_value_vars + if i.name + "@GRAD" not in slot_value_grad_names + ] + sorted_slot_key_vars = [ + value_to_key[v.name] for v in sorted_slot_value_vars + ] target_table = None for table in self._worker.sparse_table: @@ -528,8 +691,9 @@ class DownpourWorker(Worker): key_names = [var.name for var in sorted_slot_key_vars] for key_name in key_names: if key_name not in keys: - raise ValueError("sparse table %s slot_key error" % - table_id) + raise ValueError( + "sparse table %s slot_key error" % table_id + ) target_table = table break @@ -542,8 +706,15 @@ class DownpourWorker(Worker): table.slot_value.extend([var.name for var in sorted_slot_value_vars]) table.slot_gradient.extend(slot_value_grad_names) - def add_dense_table(self, table_id, learning_rate, param_vars, grad_vars, - dense_start_table_id, sparse_table_names): + def add_dense_table( + self, + table_id, + learning_rate, + param_vars, + grad_vars, + dense_start_table_id, + sparse_table_names, + ): r""" Args: table_id(int): id of sparse params table @@ -580,22 +751,25 @@ class DownpourWorker(Worker): if dense_param_name == desc_dense_param_name: desc_dense_grad_name = list( - table.dense_gradient_variable_name) + table.dense_gradient_variable_name + ) desc_dense_grad_name.sort() if dense_grad_name == desc_dense_grad_name: return else: raise ValueError( "dense table %s dense_gradient_variable_name " - "error" % table_id) + "error" % table_id + ) else: raise ValueError( - "dense table %s dense_variable_name error" % table_id) + "dense table %s dense_variable_name error" % table_id + ) table = self._worker.dense_table.add() table.table_id = table_id - #def cmp_fc(x, y): + # def cmp_fc(x, y): # if x.startswith("fc_") and y.startswith("fc_"): # index_x = x.find('.') # index_y = y.find('.') @@ -616,8 +790,8 @@ class DownpourWorker(Worker): # else: # return 1 - #table.dense_variable_name.extend(sorted(dense_param_name, cmp_fc)) - #table.dense_gradient_variable_name.extend( + # table.dense_variable_name.extend(sorted(dense_param_name, cmp_fc)) + # table.dense_gradient_variable_name.extend( # sorted(dense_grad_name, cmp_fc)) table.dense_variable_name.extend(dense_param_name) table.dense_gradient_variable_name.extend(dense_grad_name) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/optimizer_factory.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/optimizer_factory.py index 35cda4c34b0..ed9db255146 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/optimizer_factory.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/optimizer_factory.py @@ -17,8 +17,12 @@ __all__ = ["DistributedAdam", "FLEET_GLOBAL_DICT"] import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table -from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_inputs -from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_outputs +from paddle.fluid.distribute_lookup_table import ( + find_distributed_lookup_table_inputs, +) +from paddle.fluid.distribute_lookup_table import ( + find_distributed_lookup_table_outputs, +) from google.protobuf import text_format from collections import OrderedDict import copy @@ -61,11 +65,13 @@ class DistributedOptimizerImplBase(object): self._learning_rate = optimizer._learning_rate self._regularization = optimizer.regularization - def minimize(self, - losses, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, + losses, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ): """ Args: losses(Variable): loss variable defined by user @@ -90,15 +96,24 @@ class DistributedAdam(DistributedOptimizerImplBase): self._window = 1 self.type = "downpour" self.data_norm_name = [ - ".batch_size", ".batch_square_sum", ".batch_sum", - ".batch_size@GRAD", ".batch_square_sum@GRAD", ".batch_sum@GRAD" + ".batch_size", + ".batch_square_sum", + ".batch_sum", + ".batch_size@GRAD", + ".batch_square_sum@GRAD", + ".batch_sum@GRAD", ] self.supported_embedding_types = [ - "lookup_table", "pull_sparse", "pull_sparse_v2", "pull_box_sparse", - "pull_gpups_sparse" + "lookup_table", + "pull_sparse", + "pull_sparse_v2", + "pull_box_sparse", + "pull_gpups_sparse", ] self.supported_embedding_grad_types = [ - "lookup_table_grad", "push_sparse", "push_sparse_v2" + "lookup_table_grad", + "push_sparse", + "push_sparse_v2", ] op_maker = core.op_proto_and_checker_maker self.op_role_key = op_maker.kOpRoleAttrName() @@ -122,7 +137,8 @@ class DistributedAdam(DistributedOptimizerImplBase): if op.type in self.supported_embedding_types: if op.input("W")[0] in table_names: inputs_dict[op.input("W")[0]].extend( - [local_vars[name] for name in op.input("Ids")]) + [local_vars[name] for name in op.input("Ids")] + ) return inputs_dict def _find_distributed_lookup_table_outputs(self, program, table_names): @@ -144,7 +160,8 @@ class DistributedAdam(DistributedOptimizerImplBase): if op.type in self.supported_embedding_types: if op.input("W")[0] in table_names: outputs_dict[op.input("W")[0]].extend( - [local_vars[name] for name in op.output("Out")]) + [local_vars[name] for name in op.output("Out")] + ) return outputs_dict def _find_distributed_lookup_table_grads(self, program, table_names): @@ -157,12 +174,14 @@ class DistributedAdam(DistributedOptimizerImplBase): if op.type in self.supported_embedding_grad_types: if op.input("W")[0] in table_names: grads_dict[op.input("W")[0]].extend( - [local_vars[name] for name in op.input("Out@GRAD")]) + [local_vars[name] for name in op.input("Out@GRAD")] + ) return grads_dict def _is_optimizer_op(self, op): - return self.op_role_key in op.attr_names and \ - int(op.all_attrs()[self.op_role_key]) & int(OpRole.Optimize) + return self.op_role_key in op.attr_names and int( + op.all_attrs()[self.op_role_key] + ) & int(OpRole.Optimize) def _remove_optimize_op_for_embedding(self, loss, table_name): """ @@ -217,8 +236,9 @@ class DistributedAdam(DistributedOptimizerImplBase): return False return True - def _generte_cond_para_map(self, op, _fill_value_dict, _equal_fill_dict, - _now_program, _all_params): + def _generte_cond_para_map( + self, op, _fill_value_dict, _equal_fill_dict, _now_program, _all_params + ): # generate cond value to parameter map recursively cond_str = op.input('Cond')[0] vars_ = op.input('Input') @@ -241,9 +261,13 @@ class DistributedAdam(DistributedOptimizerImplBase): ops_cond = _now_program.block(int(op.attr('sub_block').id)).ops for op in ops_cond: if op.type == 'conditional_block': - self._generte_cond_para_map(op, _fill_value_dict, - _equal_fill_dict, _now_program, - _all_params) + self._generte_cond_para_map( + op, + _fill_value_dict, + _equal_fill_dict, + _now_program, + _all_params, + ) def _has_conditional_block(self, loss): now_program = loss.block.program @@ -256,8 +280,10 @@ class DistributedAdam(DistributedOptimizerImplBase): def _check_params_grads(self, params, grads): if len(params) != len(grads): - raise ValueError("params size != grads size, %s vs %s" % - (len(params), len(grads))) + raise ValueError( + "params size != grads size, %s vs %s" + % (len(params), len(grads)) + ) pname2grad = dict() for i in range(len(params)): @@ -269,13 +295,15 @@ class DistributedAdam(DistributedOptimizerImplBase): return pname2grad - def _generate_multi_dense_table(self, - params, - grads, - cond_params, - other_params, - sparse_table_names, - dense_table_id=0): + def _generate_multi_dense_table( + self, + params, + grads, + cond_params, + other_params, + sparse_table_names, + dense_table_id=0, + ): # generate multi dense table by cond value pname2grad = self._check_params_grads(params, grads) root_params_list = [] @@ -307,7 +335,14 @@ class DistributedAdam(DistributedOptimizerImplBase): lists_params[name2key[p.name]].append(p) lists_grads[name2key[p.name]].append(pname2grad[p.name]) - return dense_tables, cond2denseid, lists_params, lists_grads, root_params_list, root_grads_list + return ( + dense_tables, + cond2denseid, + lists_params, + lists_grads, + root_params_list, + root_grads_list, + ) def _gen_distributed_emb_to_size_dict(self, program): d_size = dict() @@ -321,13 +356,16 @@ class DistributedAdam(DistributedOptimizerImplBase): if d_size.get(table_name) is None: d_size[table_name] = emb_size elif d_size[table_name] != emb_size: - raise ValueError("embedding size error: %s vs %s" % - (emb_size, d_size[table_name])) + raise ValueError( + "embedding size error: %s vs %s" + % (emb_size, d_size[table_name]) + ) return d_size - def _check_config_fleet_with_program_op(self, strategy, table_name, - emb_to_size): + def _check_config_fleet_with_program_op( + self, strategy, table_name, emb_to_size + ): if strategy.get(table_name) is None: strategy[table_name] = dict() st = strategy[table_name] @@ -337,67 +375,93 @@ class DistributedAdam(DistributedOptimizerImplBase): accessor = st["sparse_accessor_class"] # set sparse_embedx_dim in the strategy according to accessor and use_cvm config - if accessor == "DownpourFeatureValueAccessor" \ - or accessor == "DownpourCtrAccessor" \ - or accessor == "DownpourCtrDymfAccessor" \ - or accessor == "DownpourDoubleUnitAccessor" \ - or accessor == "DownpourUnitAccessor": - if st.get("sparse_embedx_dim") is not None \ - and strategy.get("use_cvm") == True \ - and st["sparse_embedx_dim"] != emb_to_size[table_name] - 3: + if ( + accessor == "DownpourFeatureValueAccessor" + or accessor == "DownpourCtrAccessor" + or accessor == "DownpourCtrDymfAccessor" + or accessor == "DownpourDoubleUnitAccessor" + or accessor == "DownpourUnitAccessor" + ): + if ( + st.get("sparse_embedx_dim") is not None + and strategy.get("use_cvm") == True + and st["sparse_embedx_dim"] != emb_to_size[table_name] - 3 + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding dim - 3 = %s" % - (st["sparse_embedx_dim"], emb_to_size[table_name] - 3)) - if st.get("sparse_embedx_dim") is not None \ - and strategy.get("use_cvm") == False \ - and st["sparse_embedx_dim"] != emb_to_size[table_name] - 1: + " equal to embedding dim - 3 = %s" + % (st["sparse_embedx_dim"], emb_to_size[table_name] - 3) + ) + if ( + st.get("sparse_embedx_dim") is not None + and strategy.get("use_cvm") == False + and st["sparse_embedx_dim"] != emb_to_size[table_name] - 1 + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding dim - 1 = %s" % - (st["sparse_embedx_dim"], emb_to_size[table_name] - 1)) - if st.get("sparse_embedx_dim") is None \ - and strategy.get("use_cvm") == True: + " equal to embedding dim - 1 = %s" + % (st["sparse_embedx_dim"], emb_to_size[table_name] - 1) + ) + if ( + st.get("sparse_embedx_dim") is None + and strategy.get("use_cvm") == True + ): logger.warning( "sparse embedding dim for table name '{}' is: {}, while sparse_embedx_dim " "with same sparse table name is not set in config_fleet.py. " - "Hence automatically set sparse_embedx_dim = {} - 3.". - format(table_name, emb_to_size[table_name], - emb_to_size[table_name])) + "Hence automatically set sparse_embedx_dim = {} - 3.".format( + table_name, + emb_to_size[table_name], + emb_to_size[table_name], + ) + ) st["sparse_embedx_dim"] = emb_to_size[table_name] - 3 - if st.get("sparse_embedx_dim") is None \ - and strategy.get("use_cvm") == False: + if ( + st.get("sparse_embedx_dim") is None + and strategy.get("use_cvm") == False + ): logger.warning( "sparse embedding dim for table name '{}' is: {}, while sparse_embedx_dim " "with same sparse table name is not set in config_fleet.py. " - "Hence automatically set sparse_embedx_dim = {} - 1.". - format(table_name, emb_to_size[table_name], - emb_to_size[table_name])) + "Hence automatically set sparse_embedx_dim = {} - 1.".format( + table_name, + emb_to_size[table_name], + emb_to_size[table_name], + ) + ) st["sparse_embedx_dim"] = emb_to_size[table_name] - 1 elif accessor == "DownpourSparseValueAccessor": - if st.get("sparse_embedx_dim") is not None \ - and st["sparse_embedx_dim"] != emb_to_size[table_name]: + if ( + st.get("sparse_embedx_dim") is not None + and st["sparse_embedx_dim"] != emb_to_size[table_name] + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding dim = %s" % - (st["sparse_embedx_dim"], emb_to_size[table_name])) + " equal to embedding dim = %s" + % (st["sparse_embedx_dim"], emb_to_size[table_name]) + ) if st.get("sparse_embedx_dim") is None: logger.warning( "sparse embedding dim for table name '{}' is: {}, while sparse_embedx_dim " "with same sparse table name is not set in config_fleet.py. " "Hence automatically set sparse_embedx_dim = {}.".format( - table_name, emb_to_size[table_name], - emb_to_size[table_name])) + table_name, + emb_to_size[table_name], + emb_to_size[table_name], + ) + ) st["sparse_embedx_dim"] = emb_to_size[table_name] return strategy - def _minimize(self, - losses, - startup_program=None, - parameter_list=None, - no_grad_set=None, - strategy={}): + def _minimize( + self, + losses, + startup_program=None, + parameter_list=None, + no_grad_set=None, + strategy={}, + ): """ DownpounSGD is a distributed optimizer so that user can call minimize to generate backward @@ -439,9 +503,10 @@ class DistributedAdam(DistributedOptimizerImplBase): parameters = parameter_list[num] prog_id = str(id(loss.block.program)) # param_grads of program - params_grads = sorted(fluid.backward.append_backward( - loss, parameters, no_grad_set), - key=lambda x: x[0].name) + params_grads = sorted( + fluid.backward.append_backward(loss, parameters, no_grad_set), + key=lambda x: x[0].name, + ) flag_use_ps_gpu = strategy.get("use_ps_gpu", False) if flag_use_ps_gpu: @@ -451,9 +516,11 @@ class DistributedAdam(DistributedOptimizerImplBase): optimize_ops = optimizer.apply_optimize( loss, startup_program=startup_program[num], - params_grads=params_grads) + params_grads=params_grads, + ) embedding_table = self._find_multi_distributed_lookup_table( - [loss]) + [loss] + ) self._remove_optimize_op_for_embedding(loss, embedding_table) # has condition_block op means multi-task flag_multi_task = self._has_conditional_block(loss) @@ -476,9 +543,13 @@ class DistributedAdam(DistributedOptimizerImplBase): if op.type == 'equal': equal_fill_dict[op.output('Out')[0]] = op.input('Y')[0] if op.type == 'conditional_block': - self._generte_cond_para_map(op, fill_value_dict, - equal_fill_dict, - now_program, all_params) + self._generte_cond_para_map( + op, + fill_value_dict, + equal_fill_dict, + now_program, + all_params, + ) if prog_id not in program_id_set: program_id_set.add(prog_id) @@ -493,28 +564,32 @@ class DistributedAdam(DistributedOptimizerImplBase): # get {table_name: emb_size} dict from program ops emb_to_size = self._gen_distributed_emb_to_size_dict( - loss.block.program) + loss.block.program + ) # get inputs_dict inputs_dict = self._find_distributed_lookup_table_inputs( - loss.block.program, sparse_table) + loss.block.program, sparse_table + ) prog_id_to_inputs_dict[prog_id] = inputs_dict # get outputs_dict outputs_dict = self._find_distributed_lookup_table_outputs( - loss.block.program, sparse_table) + loss.block.program, sparse_table + ) prog_id_to_outputs_dict[prog_id] = outputs_dict prog_id_to_worker[prog_id] = DownpourWorker(self._window) grads_dict = self._find_distributed_lookup_table_grads( - loss.block.program, sparse_table) + loss.block.program, sparse_table + ) prog_id_to_sparse_grads[prog_id] = grads_dict if prog_id not in prog_id_to_param_grads: prog_id_to_param_grads[prog_id] = [] prog_id_to_param_grads[prog_id].append(params_grads) - #if strategy.get("parallel_compute") + # if strategy.get("parallel_compute") # if user specify a fleet_desc.prototxt file, then load the file # instead of creating default fleet_desc.prototxt. @@ -527,17 +602,20 @@ class DistributedAdam(DistributedOptimizerImplBase): if len(ps_param.trainer_param) == 1: for k in prog_id_to_worker: prog_id_to_worker[k].get_desc().CopyFrom( - ps_param.trainer_param[0]) + ps_param.trainer_param[0] + ) else: if len(ps_param.trainer_param) != len(prog_id_to_worker): raise ValueError( - "trainer param size != program size, %s vs %s" % - (len(ps_param.trainer_param), len(prog_id_to_worker))) + "trainer param size != program size, %s vs %s" + % (len(ps_param.trainer_param), len(prog_id_to_worker)) + ) idx = 0 # prog_id_to_worker is OrderedDict for k in prog_id_to_worker: prog_id_to_worker[k].get_desc().CopyFrom( - ps_param.trainer_param[idx]) + ps_param.trainer_param[idx] + ) idx += 1 # check config in op defination and fleet config @@ -550,10 +628,13 @@ class DistributedAdam(DistributedOptimizerImplBase): if len(sparse_table_to_index) != len(emb_to_table): raise ValueError( "sparse tables from program != sparse tables from op: %s " - "vs %s" % (len(sparse_table_to_index), len(emb_to_table))) + "vs %s" % (len(sparse_table_to_index), len(emb_to_table)) + ) for key in sparse_table_to_index: - if key not in emb_to_table or \ - sparse_table_to_index[key] != emb_to_table[key]: + if ( + key not in emb_to_table + or sparse_table_to_index[key] != emb_to_table[key] + ): print("sparse_table_to_index ", sparse_table_to_index) print("emb_to_table ", emb_to_table) raise ValueError("key error: %s" % key) @@ -564,8 +645,9 @@ class DistributedAdam(DistributedOptimizerImplBase): accessor = None if st.get("sparse_accessor_class") is not None: accessor = st["sparse_accessor_class"] - tables = \ + tables = ( server.get_desc().downpour_server_param.downpour_table_param + ) for table in tables: if table.table_id == sparse_table_to_index[key]: accessor = table.accessor.accessor_class @@ -574,44 +656,57 @@ class DistributedAdam(DistributedOptimizerImplBase): for loss in losses: for op in loss.block.program.global_block().ops: if op.type in self.supported_embedding_types: - if accessor is not None \ - and op.has_attr("AccessorClass"): + if accessor is not None and op.has_attr( + "AccessorClass" + ): op._set_attr("AccessorClass", accessor) if one_slot is None: - one_slot = loss.block.program. \ - global_block().var(op.input("Ids")[0]) + one_slot = ( + loss.block.program.global_block().var( + op.input("Ids")[0] + ) + ) # if accessor is None, use default accessor in op definition if accessor is None: accessor = emb_to_accessor[key] # set sparse_embedx_dim in strategy, # user do not have to set it in config_fleet - if accessor == "DownpourFeatureValueAccessor" \ - or accessor == "DownpourCtrDymfAccessor" \ - or accessor == "DownpourCtrAccessor" \ - or accessor == "DownpourDoubleUnitAccessor" \ - or accessor == "DownpourUnitAccessor": - if st.get("sparse_embedx_dim") is not None \ - and st["sparse_embedx_dim"] != emb_to_size[key] - 3: + if ( + accessor == "DownpourFeatureValueAccessor" + or accessor == "DownpourCtrDymfAccessor" + or accessor == "DownpourCtrAccessor" + or accessor == "DownpourDoubleUnitAccessor" + or accessor == "DownpourUnitAccessor" + ): + if ( + st.get("sparse_embedx_dim") is not None + and st["sparse_embedx_dim"] != emb_to_size[key] - 3 + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding size - 3 = %s" % - (st["sparse_embedx_dim"], emb_to_size[key] - 3)) + " equal to embedding size - 3 = %s" + % (st["sparse_embedx_dim"], emb_to_size[key] - 3) + ) st["sparse_embedx_dim"] = emb_to_size[key] - 3 elif accessor == "DownpourSparseValueAccessor": - if st.get("sparse_embedx_dim") is not None \ - and st["sparse_embedx_dim"] != emb_to_size[key]: + if ( + st.get("sparse_embedx_dim") is not None + and st["sparse_embedx_dim"] != emb_to_size[key] + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding size = %s" % - (st["sparse_embedx_dim"], emb_to_size[key])) + " equal to embedding size = %s" + % (st["sparse_embedx_dim"], emb_to_size[key]) + ) st["sparse_embedx_dim"] = emb_to_size[key] # ServerParameter add all sparse tables for tn in sparse_table_to_index: sparse_table_index = sparse_table_to_index[tn] st = self._check_config_fleet_with_program_op( - strategy, tn, emb_to_size) + strategy, tn, emb_to_size + ) if st.get(tn) is not None: server.add_sparse_table(sparse_table_index, st[tn]) else: @@ -629,8 +724,12 @@ class DistributedAdam(DistributedOptimizerImplBase): for tn in prog_id_to_sparse_table[prog_id]: sparse_table_index = sparse_table_to_index[tn] grads_dict = prog_id_to_sparse_grads[prog_id] - worker.add_sparse_table(sparse_table_index, inputs_dict[tn], - outputs_dict[tn], grads_dict[tn]) + worker.add_sparse_table( + sparse_table_index, + inputs_dict[tn], + outputs_dict[tn], + grads_dict[tn], + ) dense_start_table_id = len(sparse_table_to_index) dense_table_index = len(sparse_table_to_index) @@ -644,12 +743,13 @@ class DistributedAdam(DistributedOptimizerImplBase): program_id_set.add(program_id) worker = prog_id_to_worker[program_id] sparse_table_names = prog_id_to_sparse_table[program_id] - sparse_table_index = \ - [sparse_table_to_index[i] for i in sparse_table_names] + sparse_table_index = [ + sparse_table_to_index[i] for i in sparse_table_names + ] program_configs[program_id] = { "pull_sparse": [t_index for t_index in sparse_table_index], - "push_sparse": [t_index for t_index in sparse_table_index] + "push_sparse": [t_index for t_index in sparse_table_index], } params_grads = prog_id_to_param_grads[program_id] @@ -679,12 +779,24 @@ class DistributedAdam(DistributedOptimizerImplBase): multi_task_dense_tables_push = [] multi_task_dense_tables_pull = [] if flag_multi_task: - dense_tables, cond2denseid, lists_params, lists_grads, root_params_list, root_grads_list = self._generate_multi_dense_table( - params, grads, self._cond_params, - self._other_params, sparse_table_names, - dense_table_index) + ( + dense_tables, + cond2denseid, + lists_params, + lists_grads, + root_params_list, + root_grads_list, + ) = self._generate_multi_dense_table( + params, + grads, + self._cond_params, + self._other_params, + sparse_table_names, + dense_table_index, + ) program_configs[program_id][ - 'cond2denseid'] = cond2denseid + 'cond2denseid' + ] = cond2denseid multi_task_dense_tables_push = dense_tables multi_task_dense_tables_pull = dense_tables[:] @@ -692,56 +804,76 @@ class DistributedAdam(DistributedOptimizerImplBase): if flag_multi_task: server_dense_table_index = dense_table_index if len(root_params_list) > 0: - server.add_dense_table(server_dense_table_index, - root_params_list, - root_grads_list, - strategy['dense_table'], - sparse_table_names) + server.add_dense_table( + server_dense_table_index, + root_params_list, + root_grads_list, + strategy['dense_table'], + sparse_table_names, + ) server_dense_table_index += 1 for i in range(len(lists_params)): - server.add_dense_table(server_dense_table_index, - lists_params[i], - lists_grads[i], - strategy['dense_table'], - sparse_table_names) + server.add_dense_table( + server_dense_table_index, + lists_params[i], + lists_grads[i], + strategy['dense_table'], + sparse_table_names, + ) server_dense_table_index += 1 else: - server.add_dense_table(dense_table_index, params, - grads, - strategy['dense_table'], - sparse_table_names) + server.add_dense_table( + dense_table_index, + params, + grads, + strategy['dense_table'], + sparse_table_names, + ) else: - server.add_dense_table(dense_table_index, params, grads, - None, sparse_table_names) + server.add_dense_table( + dense_table_index, + params, + grads, + None, + sparse_table_names, + ) if flag_multi_task: if len(root_params_list) > 0: - worker.add_dense_table(dense_table_index, - self._learning_rate, - root_params_list, - root_grads_list, - dense_start_table_id, - sparse_table_names) + worker.add_dense_table( + dense_table_index, + self._learning_rate, + root_params_list, + root_grads_list, + dense_start_table_id, + sparse_table_names, + ) dense_table_index += 1 for i in range(len(lists_params)): - worker.add_dense_table(dense_table_index, - self._learning_rate, - lists_params[i], - lists_grads[i], - dense_start_table_id, - sparse_table_names) + worker.add_dense_table( + dense_table_index, + self._learning_rate, + lists_params[i], + lists_grads[i], + dense_start_table_id, + sparse_table_names, + ) dense_table_index += 1 dense_table_index -= 1 else: - worker.add_dense_table(dense_table_index, - self._learning_rate, params, - grads, dense_start_table_id, - sparse_table_names) + worker.add_dense_table( + dense_table_index, + self._learning_rate, + params, + grads, + dense_start_table_id, + sparse_table_names, + ) if FLEET_GLOBAL_DICT["enable"]: cur_prog = losses[loss_index].block.program @@ -751,30 +883,39 @@ class DistributedAdam(DistributedOptimizerImplBase): attrs={ "InputNames": [i.name for i in grads], "TableId": dense_table_index, - "ScaleDataNorm": - strategy.get("scale_datanorm", -1) - }) - - if "pull_dense" in program_configs[ - program_id] and "push_dense" in program_configs[ - program_id] and len(program_configs[program_id] - ["pull_dense"]) > 0: + "ScaleDataNorm": strategy.get( + "scale_datanorm", -1 + ), + }, + ) + + if ( + "pull_dense" in program_configs[program_id] + and "push_dense" in program_configs[program_id] + and len(program_configs[program_id]["pull_dense"]) > 0 + ): if flag_multi_task: program_configs[program_id]["pull_dense"].extend( - multi_task_dense_tables_pull) + multi_task_dense_tables_pull + ) program_configs[program_id]["push_dense"].extend( - multi_task_dense_tables_push) + multi_task_dense_tables_push + ) else: program_configs[program_id]["pull_dense"].extend( - [dense_table_index]) + [dense_table_index] + ) program_configs[program_id]["push_dense"].extend( - [dense_table_index]) + [dense_table_index] + ) else: if flag_multi_task: program_configs[program_id][ - "pull_dense"] = multi_task_dense_tables_pull + "pull_dense" + ] = multi_task_dense_tables_pull program_configs[program_id][ - "push_dense"] = multi_task_dense_tables_push + "push_dense" + ] = multi_task_dense_tables_push else: program_configs[program_id]["pull_dense"] = [ dense_table_index @@ -787,22 +928,31 @@ class DistributedAdam(DistributedOptimizerImplBase): dense_table_index += 1 if strategy.get('datanorm_table') is not None: server.add_data_norm_table( - dense_table_index, self._learning_rate, - data_norm_params, data_norm_grads, - strategy['datanorm_table'], sparse_table_names) + dense_table_index, + self._learning_rate, + data_norm_params, + data_norm_grads, + strategy['datanorm_table'], + sparse_table_names, + ) else: - server.add_data_norm_table(dense_table_index, - self._learning_rate, - data_norm_params, - data_norm_grads, None, - sparse_table_names) - - worker.add_dense_table(dense_table_index, - self._learning_rate, - data_norm_params, - data_norm_grads, - dense_start_table_id, - sparse_table_names) + server.add_data_norm_table( + dense_table_index, + self._learning_rate, + data_norm_params, + data_norm_grads, + None, + sparse_table_names, + ) + + worker.add_dense_table( + dense_table_index, + self._learning_rate, + data_norm_params, + data_norm_grads, + dense_start_table_id, + sparse_table_names, + ) if FLEET_GLOBAL_DICT["enable"]: cur_prog = losses[loss_index].block.program @@ -810,18 +960,22 @@ class DistributedAdam(DistributedOptimizerImplBase): type="push_dense", inputs={"Ids": one_slot}, attrs={ - "InputNames": - [i.name for i in data_norm_grads], - "TableId": - dense_table_index, - "ScaleDataNorm": - strategy.get("scale_datanorm", -1) - }) + "InputNames": [ + i.name for i in data_norm_grads + ], + "TableId": dense_table_index, + "ScaleDataNorm": strategy.get( + "scale_datanorm", -1 + ), + }, + ) program_configs[program_id]["pull_dense"].extend( - [dense_table_index]) + [dense_table_index] + ) program_configs[program_id]["push_dense"].extend( - [dense_table_index]) + [dense_table_index] + ) dense_table_index += 1 # Todo(guru4elephant): figure out how to support more sparse parameters @@ -865,36 +1019,47 @@ class DistributedAdam(DistributedOptimizerImplBase): opt_info["use_cvm"] = strategy.get("use_cvm", False) opt_info["no_cvm"] = strategy.get("no_cvm", False) opt_info["scale_sparse_gradient_with_batch_size"] = strategy.get( - "scale_sparse_gradient_with_batch_size", True) - opt_info["worker_class"] = strategy.get("worker_class", - "DownpourWorker") + "scale_sparse_gradient_with_batch_size", True + ) + opt_info["worker_class"] = strategy.get( + "worker_class", "DownpourWorker" + ) opt_info["stat_var_names"] = strategy.get("stat_var_names", []) opt_info["local_tables"] = strategy.get("local_tables", []) opt_info["async_tables"] = strategy.get("async_tables", []) opt_info["async_tables"] = strategy.get("async_tables", []) opt_info["scale_datanorm"] = strategy.get("scale_datanorm", -1) - opt_info["check_nan_var_names"] = strategy.get("check_nan_var_names", - []) + opt_info["check_nan_var_names"] = strategy.get( + "check_nan_var_names", [] + ) opt_info["dump_slot"] = False opt_info["dump_converter"] = "" opt_info["dump_fields"] = strategy.get("dump_fields", []) opt_info["dump_file_num"] = strategy.get("dump_file_num", 16) opt_info["user_define_dump_filename"] = strategy.get( - "user_define_dump_filename", "") + "user_define_dump_filename", "" + ) opt_info["dump_fields_path"] = strategy.get("dump_fields_path", "") opt_info["dump_param"] = strategy.get("dump_param", []) gpus_env = os.getenv("FLAGS_selected_gpus", "0") opt_info["worker_places"] = [int(s) for s in gpus_env.split(",")] opt_info["use_ps_gpu"] = strategy.get("use_ps_gpu", False) if server._server.downpour_server_param.downpour_table_param[ - 0].accessor.accessor_class in [ - "DownpourCtrAccessor", "DownpourCtrDoubleAccessor", - "DownpourUnitAccessor", "DownpourDoubleUnitAccessor", - "DownpourCtrDymfAccessor" - ]: + 0 + ].accessor.accessor_class in [ + "DownpourCtrAccessor", + "DownpourCtrDoubleAccessor", + "DownpourUnitAccessor", + "DownpourDoubleUnitAccessor", + "DownpourCtrDymfAccessor", + ]: opt_info["dump_slot"] = True - elif server._server.downpour_server_param.downpour_table_param[ - 0].accessor.accessor_class == "DownpourSparseValueAccessor": + elif ( + server._server.downpour_server_param.downpour_table_param[ + 0 + ].accessor.accessor_class + == "DownpourSparseValueAccessor" + ): opt_info["no_cvm"] = True opt_info["adjust_ins_weight"] = strategy.get("adjust_ins_weight", {}) opt_info["copy_table"] = strategy.get("copy_table", {}) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py index eec51ef827c..4291115b0bc 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py @@ -24,6 +24,7 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -34,7 +35,8 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto2', serialized_pb=_b( '\n\x08ps.proto\x12\x06paddle\"\xb5\x02\n\x0bPSParameter\x12\x14\n\x0cworker_class\x18\x01 \x01(\t\x12\x14\n\x0cserver_class\x18\x02 \x01(\t\x12\x16\n\x0einstance_class\x18\x03 \x01(\t\x12\x15\n\x0binit_gflags\x18\x04 \x01(\t:\x00\x12-\n\x0cworker_param\x18\x65 \x01(\x0b\x32\x17.paddle.WorkerParameter\x12-\n\x0cserver_param\x18\x66 \x01(\x0b\x32\x17.paddle.ServerParameter\x12\x38\n\rtrainer_param\x18\xad\x02 \x03(\x0b\x32 .paddle.DownpourTrainerParameter\x12\x33\n\x0f\x66s_client_param\x18\xf5\x03 \x01(\x0b\x32\x19.paddle.FsClientParameter\"Q\n\x0fWorkerParameter\x12>\n\x15\x64ownpour_worker_param\x18\x01 \x01(\x0b\x32\x1f.paddle.DownpourWorkerParameter\"Q\n\x0fServerParameter\x12>\n\x15\x64ownpour_server_param\x18\x01 \x01(\x0b\x32\x1f.paddle.DownpourServerParameter\"O\n\x17\x44ownpourWorkerParameter\x12\x34\n\x14\x64ownpour_table_param\x18\x01 \x03(\x0b\x32\x16.paddle.TableParameter\"\xfd\x01\n\x18\x44ownpourTrainerParameter\x12\x30\n\x0b\x64\x65nse_table\x18\x01 \x03(\x0b\x32\x1b.paddle.DenseTableParameter\x12\x32\n\x0csparse_table\x18\x02 \x03(\x0b\x32\x1c.paddle.SparseTableParameter\x12\x1d\n\x15push_sparse_per_batch\x18\x03 \x01(\x05\x12\x1c\n\x14push_dense_per_batch\x18\x04 \x01(\x05\x12\x0f\n\x07skip_op\x18\x05 \x03(\t\x12-\n\x0eprogram_config\x18\x06 \x03(\x0b\x32\x15.paddle.ProgramConfig\"\x99\x01\n\rProgramConfig\x12\x12\n\nprogram_id\x18\x01 \x02(\t\x12\x1c\n\x14push_sparse_table_id\x18\x02 \x03(\x05\x12\x1b\n\x13push_dense_table_id\x18\x03 \x03(\x05\x12\x1c\n\x14pull_sparse_table_id\x18\x04 \x03(\x05\x12\x1b\n\x13pull_dense_table_id\x18\x05 \x03(\x05\"{\n\x13\x44\x65nseTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x05\x12\x1b\n\x13\x64\x65nse_variable_name\x18\x02 \x03(\t\x12$\n\x1c\x64\x65nse_gradient_variable_name\x18\x03 \x03(\t\x12\x0f\n\x07\x66\x65\x61_dim\x18\x04 \x01(\x05\"z\n\x14SparseTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x05\x12\x13\n\x0b\x66\x65\x61ture_dim\x18\x02 \x01(\x05\x12\x10\n\x08slot_key\x18\x03 \x03(\t\x12\x12\n\nslot_value\x18\x04 \x03(\t\x12\x15\n\rslot_gradient\x18\x05 \x03(\t\"\x86\x01\n\x17\x44ownpourServerParameter\x12\x34\n\x14\x64ownpour_table_param\x18\x01 \x03(\x0b\x32\x16.paddle.TableParameter\x12\x35\n\rservice_param\x18\x02 \x01(\x0b\x32\x1e.paddle.ServerServiceParameter\"\xd7\x01\n\x16ServerServiceParameter\x12*\n\x0cserver_class\x18\x01 \x01(\t:\x14\x44ownpourBrpcPsServer\x12*\n\x0c\x63lient_class\x18\x02 \x01(\t:\x14\x44ownpourBrpcPsClient\x12(\n\rservice_class\x18\x03 \x01(\t:\x11\x44ownpourPsService\x12\x1c\n\x11start_server_port\x18\x04 \x01(\r:\x01\x30\x12\x1d\n\x11server_thread_num\x18\x05 \x01(\r:\x02\x31\x32\"\xc0\x02\n\x0eTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x04\x12\x13\n\x0btable_class\x18\x02 \x01(\t\x12\x17\n\tshard_num\x18\x03 \x01(\x04:\x04\x31\x30\x30\x30\x12\x30\n\x08\x61\x63\x63\x65ssor\x18\x04 \x01(\x0b\x32\x1e.paddle.TableAccessorParameter\x12\x1f\n\x04type\x18\x05 \x01(\x0e\x32\x11.paddle.TableType\x12\x1f\n\x10\x63ompress_in_save\x18\x06 \x01(\x08:\x05\x66\x61lse\x12\'\n\x19\x65nable_sparse_table_cache\x18\x07 \x01(\x08:\x04true\x12(\n\x17sparse_table_cache_rate\x18\x08 \x01(\x01:\x07\x30.00055\x12\'\n\x1bsparse_table_cache_file_num\x18\t \x01(\r:\x02\x31\x36\"\xc1\x04\n\x16TableAccessorParameter\x12\x16\n\x0e\x61\x63\x63\x65ssor_class\x18\x01 \x01(\t\x12\x38\n\x10sparse_sgd_param\x18\x02 \x01(\x0b\x32\x1e.paddle.SparseSGDRuleParameter\x12\x36\n\x0f\x64\x65nse_sgd_param\x18\x03 \x01(\x0b\x32\x1d.paddle.DenseSGDRuleParameter\x12\x13\n\x07\x66\x65\x61_dim\x18\x04 \x01(\r:\x02\x31\x31\x12\x15\n\nembedx_dim\x18\x05 \x01(\r:\x01\x38\x12\x1c\n\x10\x65mbedx_threshold\x18\x06 \x01(\r:\x02\x31\x30\x12G\n\x17\x64ownpour_accessor_param\x18\x07 \x01(\x0b\x32&.paddle.DownpourTableAccessorParameter\x12\x45\n\x19table_accessor_save_param\x18\x08 \x03(\x0b\x32\".paddle.TableAccessorSaveParameter\x12\x44\n\x16sparse_commonsgd_param\x18\t \x01(\x0b\x32$.paddle.SparseCommonSGDRuleParameter\x12=\n\x0f\x65mbed_sgd_param\x18\n \x01(\x0b\x32$.paddle.SparseCommonSGDRuleParameter\x12>\n\x10\x65mbedx_sgd_param\x18\x0b \x01(\x0b\x32$.paddle.SparseCommonSGDRuleParameter\"\xba\x02\n\x1e\x44ownpourTableAccessorParameter\x12\x19\n\x0cnonclk_coeff\x18\x01 \x01(\x02:\x03\x30.1\x12\x16\n\x0b\x63lick_coeff\x18\x02 \x01(\x02:\x01\x31\x12\x1b\n\x0e\x62\x61se_threshold\x18\x03 \x01(\x02:\x03\x31.5\x12\x1d\n\x0f\x64\x65lta_threshold\x18\x04 \x01(\x02:\x04\x30.25\x12\x1b\n\x0f\x64\x65lta_keep_days\x18\x05 \x01(\x02:\x02\x31\x36\x12#\n\x15show_click_decay_rate\x18\x06 \x01(\x02:\x04\x30.98\x12\x1d\n\x10\x64\x65lete_threshold\x18\x07 \x01(\x02:\x03\x30.8\x12$\n\x18\x64\x65lete_after_unseen_days\x18\x08 \x01(\x02:\x02\x33\x30\x12\"\n\x17ssd_unseenday_threshold\x18\t \x01(\x05:\x01\x31\"S\n\x1aTableAccessorSaveParameter\x12\r\n\x05param\x18\x01 \x01(\r\x12\x11\n\tconverter\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65\x63onverter\x18\x03 \x01(\t\"e\n\x10PsRequestMessage\x12\x0e\n\x06\x63md_id\x18\x01 \x02(\r\x12\x10\n\x08table_id\x18\x02 \x01(\r\x12\x0e\n\x06params\x18\x03 \x03(\x0c\x12\x11\n\tclient_id\x18\x04 \x01(\x05\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\"\x85\x01\n\x16SparseSGDRuleParameter\x12\x1b\n\rlearning_rate\x18\x01 \x01(\x01:\x04\x30.05\x12\x18\n\rinitial_g2sum\x18\x02 \x01(\x01:\x01\x33\x12\x1d\n\rinitial_range\x18\x03 \x01(\x01:\x06\x30.0001\x12\x15\n\rweight_bounds\x18\x04 \x03(\x02\"\xc6\x01\n\x1cSparseCommonSGDRuleParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x32\n\x05naive\x18\x02 \x01(\x0b\x32#.paddle.SparseNaiveSGDRuleParameter\x12\x36\n\x07\x61\x64\x61grad\x18\x03 \x01(\x0b\x32%.paddle.SparseAdagradSGDRuleParameter\x12,\n\x04\x61\x64\x61m\x18\x04 \x01(\x0b\x32\x1e.paddle.SparseAdamSGDParameter\"p\n\x1bSparseNaiveSGDRuleParameter\x12\x1b\n\rlearning_rate\x18\x01 \x01(\x01:\x04\x30.05\x12\x1d\n\rinitial_range\x18\x02 \x01(\x01:\x06\x30.0001\x12\x15\n\rweight_bounds\x18\x03 \x03(\x02\"\x8c\x01\n\x1dSparseAdagradSGDRuleParameter\x12\x1b\n\rlearning_rate\x18\x01 \x01(\x01:\x04\x30.05\x12\x18\n\rinitial_g2sum\x18\x02 \x01(\x01:\x01\x33\x12\x1d\n\rinitial_range\x18\x03 \x01(\x01:\x06\x30.0001\x12\x15\n\rweight_bounds\x18\x04 \x03(\x02\"\xc8\x01\n\x16SparseAdamSGDParameter\x12\x1c\n\rlearning_rate\x18\x01 \x01(\x01:\x05\x30.001\x12\x1d\n\rinitial_range\x18\x02 \x01(\x01:\x06\x30.0001\x12\x1d\n\x10\x62\x65ta1_decay_rate\x18\x03 \x01(\x01:\x03\x30.9\x12\x1f\n\x10\x62\x65ta2_decay_rate\x18\x04 \x01(\x01:\x05\x30.999\x12\x1a\n\x0b\x61\x64\x61_epsilon\x18\x05 \x01(\x01:\x05\x31\x65-08\x12\x15\n\rweight_bounds\x18\x06 \x03(\x02\"\xe1\x01\n\x15\x44\x65nseSGDRuleParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x04\x61\x64\x61m\x18\x02 \x01(\x0b\x32\x18.paddle.AdamSGDParameter\x12(\n\x05naive\x18\x03 \x01(\x0b\x32\x19.paddle.NaiveSGDParameter\x12,\n\x07summary\x18\x04 \x01(\x0b\x32\x1b.paddle.SummarySGDParameter\x12:\n\x0emoving_average\x18\x05 \x01(\x0b\x32\".paddle.MovingAverageRuleParameter\"\xac\x01\n\x10\x41\x64\x61mSGDParameter\x12\x1c\n\rlearning_rate\x18\x01 \x01(\x01:\x05\x35\x65-06\x12 \n\x0e\x61vg_decay_rate\x18\x02 \x01(\x01:\x08\x30.999993\x12\x1e\n\x0e\x61\x64\x61_decay_rate\x18\x03 \x01(\x01:\x06\x30.9999\x12\x1a\n\x0b\x61\x64\x61_epsilon\x18\x04 \x01(\x01:\x05\x31\x65-08\x12\x1c\n\x0emom_decay_rate\x18\x05 \x01(\x01:\x04\x30.99\"J\n\x11NaiveSGDParameter\x12\x1d\n\rlearning_rate\x18\x01 \x01(\x01:\x06\x30.0002\x12\x16\n\x0e\x61vg_decay_rate\x18\x02 \x01(\x01\";\n\x13SummarySGDParameter\x12$\n\x12summary_decay_rate\x18\x01 \x01(\x01:\x08\x30.999999\".\n\x1aMovingAverageRuleParameter\x12\x10\n\x08momentum\x18\x01 \x01(\x01\"I\n\x11PsResponseMessage\x12\x13\n\x08\x65rr_code\x18\x01 \x02(\x05:\x01\x30\x12\x11\n\x07\x65rr_msg\x18\x02 \x02(\t:\x00\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xd5\x01\n\x11\x46sClientParameter\x12:\n\x07\x66s_type\x18\x01 \x01(\x0e\x32#.paddle.FsClientParameter.FsApiType:\x04HDFS\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x0e\n\x06passwd\x18\x04 \x01(\t\x12\x13\n\x0b\x62uffer_size\x18\x05 \x01(\x05\x12\x12\n\nhadoop_bin\x18\x33 \x01(\t\x12\x10\n\x08\x61\x66s_conf\x18\x65 \x01(\t\"\x1e\n\tFsApiType\x12\x08\n\x04HDFS\x10\x00\x12\x07\n\x03\x41\x46S\x10\x01*4\n\tTableType\x12\x13\n\x0fPS_SPARSE_TABLE\x10\x00\x12\x12\n\x0ePS_DENSE_TABLE\x10\x01*\xba\x04\n\x07PsCmdID\x12\x17\n\x13PS_PULL_DENSE_TABLE\x10\x00\x12\x17\n\x13PS_PUSH_DENSE_TABLE\x10\x01\x12\x18\n\x14PS_PULL_SPARSE_TABLE\x10\x02\x12\x18\n\x14PS_PUSH_SPARSE_TABLE\x10\x03\x12\x13\n\x0fPS_SHRINK_TABLE\x10\x04\x12\x15\n\x11PS_SAVE_ONE_TABLE\x10\x05\x12\x15\n\x11PS_SAVE_ALL_TABLE\x10\x06\x12\x15\n\x11PS_LOAD_ONE_TABLE\x10\x07\x12\x15\n\x11PS_LOAD_ALL_TABLE\x10\x08\x12\x16\n\x12PS_CLEAR_ONE_TABLE\x10\t\x12\x16\n\x12PS_CLEAR_ALL_TABLE\x10\n\x12\x17\n\x13PS_PUSH_DENSE_PARAM\x10\x0b\x12\x12\n\x0ePS_STOP_SERVER\x10\x0c\x12\x1b\n\x17PS_SAVE_ONE_CACHE_TABLE\x10\r\x12\x1a\n\x16PS_GET_CACHE_THRESHOLD\x10\x0e\x12\x14\n\x10PS_CACHE_SHUFFLE\x10\x0f\x12\x11\n\rPS_COPY_TABLE\x10\x10\x12\x1c\n\x18PS_COPY_TABLE_BY_FEASIGN\x10\x11\x12(\n$PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY\x10\x12\x12(\n$PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY\x10\x13\x12\x17\n\x13PS_PRINT_TABLE_STAT\x10\x14\x12\x0e\n\nPS_S2S_MSG\x10\x65\x32K\n\tPsService\x12>\n\x07service\x12\x18.paddle.PsRequestMessage\x1a\x19.paddle.PsResponseMessageB\x06\x80\x01\x01\xf8\x01\x01' - )) + ), +) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _TABLETYPE = _descriptor.EnumDescriptor( @@ -43,16 +45,12 @@ _TABLETYPE = _descriptor.EnumDescriptor( filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='PS_SPARSE_TABLE', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_DENSE_TABLE', - index=1, - number=1, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='PS_SPARSE_TABLE', index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_DENSE_TABLE', index=1, number=1, options=None, type=None + ), ], containing_type=None, options=None, @@ -68,118 +66,128 @@ _PSCMDID = _descriptor.EnumDescriptor( filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='PS_PULL_DENSE_TABLE', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_DENSE_TABLE', - index=1, - number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PULL_SPARSE_TABLE', - index=2, - number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_SPARSE_TABLE', - index=3, - number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SHRINK_TABLE', - index=4, - number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ONE_TABLE', - index=5, - number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ALL_TABLE', - index=6, - number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_LOAD_ONE_TABLE', - index=7, - number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_LOAD_ALL_TABLE', - index=8, - number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CLEAR_ONE_TABLE', - index=9, - number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CLEAR_ALL_TABLE', - index=10, - number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_DENSE_PARAM', - index=11, - number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_STOP_SERVER', - index=12, - number=12, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ONE_CACHE_TABLE', - index=13, - number=13, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_GET_CACHE_THRESHOLD', - index=14, - number=14, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CACHE_SHUFFLE', - index=15, - number=15, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_COPY_TABLE', - index=16, - number=16, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_COPY_TABLE_BY_FEASIGN', - index=17, - number=17, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='PS_PULL_DENSE_TABLE', + index=0, + number=0, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_DENSE_TABLE', + index=1, + number=1, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PULL_SPARSE_TABLE', + index=2, + number=2, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_SPARSE_TABLE', + index=3, + number=3, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_SHRINK_TABLE', index=4, number=4, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ONE_TABLE', index=5, number=5, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ALL_TABLE', index=6, number=6, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_LOAD_ONE_TABLE', index=7, number=7, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_LOAD_ALL_TABLE', index=8, number=8, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_CLEAR_ONE_TABLE', + index=9, + number=9, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_CLEAR_ALL_TABLE', + index=10, + number=10, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_DENSE_PARAM', + index=11, + number=11, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_STOP_SERVER', index=12, number=12, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ONE_CACHE_TABLE', + index=13, + number=13, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_GET_CACHE_THRESHOLD', + index=14, + number=14, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_CACHE_SHUFFLE', + index=15, + number=15, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_COPY_TABLE', index=16, number=16, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_COPY_TABLE_BY_FEASIGN', + index=17, + number=17, + options=None, + type=None, + ), _descriptor.EnumValueDescriptor( name='PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY', index=18, number=18, options=None, - type=None), + type=None, + ), _descriptor.EnumValueDescriptor( name='PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY', index=19, number=19, options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PRINT_TABLE_STAT', - index=20, - number=20, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_S2S_MSG', - index=21, - number=101, - options=None, - type=None), + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PRINT_TABLE_STAT', + index=20, + number=20, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_S2S_MSG', index=21, number=101, options=None, type=None + ), ], containing_type=None, options=None, @@ -220,16 +228,12 @@ _FSCLIENTPARAMETER_FSAPITYPE = _descriptor.EnumDescriptor( filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='HDFS', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='AFS', - index=1, - number=1, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='HDFS', index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='AFS', index=1, number=1, options=None, type=None + ), ], containing_type=None, options=None, @@ -245,36 +249,40 @@ _PSPARAMETER = _descriptor.Descriptor( file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='worker_class', - full_name='paddle.PSParameter.worker_class', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='server_class', - full_name='paddle.PSParameter.server_class', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='worker_class', + full_name='paddle.PSParameter.worker_class', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='server_class', + full_name='paddle.PSParameter.server_class', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='instance_class', full_name='paddle.PSParameter.instance_class', @@ -290,52 +298,59 @@ _PSPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='init_gflags', - full_name='paddle.PSParameter.init_gflags', - index=3, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=True, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='worker_param', - full_name='paddle.PSParameter.worker_param', - index=4, - number=101, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='server_param', - full_name='paddle.PSParameter.server_param', - index=5, - number=102, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='init_gflags', + full_name='paddle.PSParameter.init_gflags', + index=3, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=True, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='worker_param', + full_name='paddle.PSParameter.worker_param', + index=4, + number=101, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='server_param', + full_name='paddle.PSParameter.server_param', + index=5, + number=102, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='trainer_param', full_name='paddle.PSParameter.trainer_param', @@ -351,7 +366,8 @@ _PSPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fs_client_param', full_name='paddle.PSParameter.fs_client_param', @@ -367,7 +383,8 @@ _PSPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -403,7 +420,8 @@ _WORKERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -439,7 +457,8 @@ _SERVERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -475,7 +494,8 @@ _DOWNPOURWORKERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -511,7 +531,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_table', full_name='paddle.DownpourTrainerParameter.sparse_table', @@ -527,7 +548,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_sparse_per_batch', full_name='paddle.DownpourTrainerParameter.push_sparse_per_batch', @@ -543,7 +565,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_dense_per_batch', full_name='paddle.DownpourTrainerParameter.push_dense_per_batch', @@ -559,7 +582,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='skip_op', full_name='paddle.DownpourTrainerParameter.skip_op', @@ -575,7 +599,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='program_config', full_name='paddle.DownpourTrainerParameter.program_config', @@ -591,7 +616,8 @@ _DOWNPOURTRAINERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -612,21 +638,23 @@ _PROGRAMCONFIG = _descriptor.Descriptor( file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='program_id', - full_name='paddle.ProgramConfig.program_id', - index=0, - number=1, - type=9, - cpp_type=9, - label=2, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='program_id', + full_name='paddle.ProgramConfig.program_id', + index=0, + number=1, + type=9, + cpp_type=9, + label=2, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='push_sparse_table_id', full_name='paddle.ProgramConfig.push_sparse_table_id', @@ -642,7 +670,8 @@ _PROGRAMCONFIG = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_dense_table_id', full_name='paddle.ProgramConfig.push_dense_table_id', @@ -658,7 +687,8 @@ _PROGRAMCONFIG = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='pull_sparse_table_id', full_name='paddle.ProgramConfig.pull_sparse_table_id', @@ -674,7 +704,8 @@ _PROGRAMCONFIG = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='pull_dense_table_id', full_name='paddle.ProgramConfig.pull_dense_table_id', @@ -690,7 +721,8 @@ _PROGRAMCONFIG = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -726,7 +758,8 @@ _DENSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_variable_name', full_name='paddle.DenseTableParameter.dense_variable_name', @@ -742,7 +775,8 @@ _DENSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_gradient_variable_name', full_name='paddle.DenseTableParameter.dense_gradient_variable_name', @@ -758,7 +792,8 @@ _DENSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fea_dim', full_name='paddle.DenseTableParameter.fea_dim', @@ -774,7 +809,8 @@ _DENSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -810,7 +846,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='feature_dim', full_name='paddle.SparseTableParameter.feature_dim', @@ -826,7 +863,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_key', full_name='paddle.SparseTableParameter.slot_key', @@ -842,7 +880,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_value', full_name='paddle.SparseTableParameter.slot_value', @@ -858,7 +897,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_gradient', full_name='paddle.SparseTableParameter.slot_gradient', @@ -874,7 +914,8 @@ _SPARSETABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -910,7 +951,8 @@ _DOWNPOURSERVERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='service_param', full_name='paddle.DownpourServerParameter.service_param', @@ -926,7 +968,8 @@ _DOWNPOURSERVERPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -962,7 +1005,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='client_class', full_name='paddle.ServerServiceParameter.client_class', @@ -978,7 +1022,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='service_class', full_name='paddle.ServerServiceParameter.service_class', @@ -994,7 +1039,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='start_server_port', full_name='paddle.ServerServiceParameter.start_server_port', @@ -1010,7 +1056,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='server_thread_num', full_name='paddle.ServerServiceParameter.server_thread_num', @@ -1026,7 +1073,8 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1047,21 +1095,23 @@ _TABLEPARAMETER = _descriptor.Descriptor( file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='table_id', - full_name='paddle.TableParameter.table_id', - index=0, - number=1, - type=4, - cpp_type=4, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='table_id', + full_name='paddle.TableParameter.table_id', + index=0, + number=1, + type=4, + cpp_type=4, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='table_class', full_name='paddle.TableParameter.table_class', @@ -1077,52 +1127,59 @@ _TABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='shard_num', - full_name='paddle.TableParameter.shard_num', - index=2, - number=3, - type=4, - cpp_type=4, - label=1, - has_default_value=True, - default_value=1000, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='accessor', - full_name='paddle.TableParameter.accessor', - index=3, - number=4, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='type', - full_name='paddle.TableParameter.type', - index=4, - number=5, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='shard_num', + full_name='paddle.TableParameter.shard_num', + index=2, + number=3, + type=4, + cpp_type=4, + label=1, + has_default_value=True, + default_value=1000, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='accessor', + full_name='paddle.TableParameter.accessor', + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='type', + full_name='paddle.TableParameter.type', + index=4, + number=5, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='compress_in_save', full_name='paddle.TableParameter.compress_in_save', @@ -1138,7 +1195,8 @@ _TABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='enable_sparse_table_cache', full_name='paddle.TableParameter.enable_sparse_table_cache', @@ -1154,7 +1212,8 @@ _TABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_table_cache_rate', full_name='paddle.TableParameter.sparse_table_cache_rate', @@ -1170,7 +1229,8 @@ _TABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_table_cache_file_num', full_name='paddle.TableParameter.sparse_table_cache_file_num', @@ -1186,7 +1246,8 @@ _TABLEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1222,7 +1283,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_sgd_param', full_name='paddle.TableAccessorParameter.sparse_sgd_param', @@ -1238,7 +1300,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_sgd_param', full_name='paddle.TableAccessorParameter.dense_sgd_param', @@ -1254,7 +1317,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fea_dim', full_name='paddle.TableAccessorParameter.fea_dim', @@ -1270,7 +1334,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_dim', full_name='paddle.TableAccessorParameter.embedx_dim', @@ -1286,7 +1351,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_threshold', full_name='paddle.TableAccessorParameter.embedx_threshold', @@ -1302,7 +1368,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='downpour_accessor_param', full_name='paddle.TableAccessorParameter.downpour_accessor_param', @@ -1318,7 +1385,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='table_accessor_save_param', full_name='paddle.TableAccessorParameter.table_accessor_save_param', @@ -1334,7 +1402,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_commonsgd_param', full_name='paddle.TableAccessorParameter.sparse_commonsgd_param', @@ -1350,7 +1419,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embed_sgd_param', full_name='paddle.TableAccessorParameter.embed_sgd_param', @@ -1366,7 +1436,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_sgd_param', full_name='paddle.TableAccessorParameter.embedx_sgd_param', @@ -1382,7 +1453,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1418,7 +1490,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='click_coeff', full_name='paddle.DownpourTableAccessorParameter.click_coeff', @@ -1434,7 +1507,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='base_threshold', full_name='paddle.DownpourTableAccessorParameter.base_threshold', @@ -1450,7 +1524,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delta_threshold', full_name='paddle.DownpourTableAccessorParameter.delta_threshold', @@ -1466,7 +1541,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delta_keep_days', full_name='paddle.DownpourTableAccessorParameter.delta_keep_days', @@ -1482,11 +1558,11 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='show_click_decay_rate', - full_name= - 'paddle.DownpourTableAccessorParameter.show_click_decay_rate', + full_name='paddle.DownpourTableAccessorParameter.show_click_decay_rate', index=5, number=6, type=2, @@ -1499,7 +1575,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delete_threshold', full_name='paddle.DownpourTableAccessorParameter.delete_threshold', @@ -1515,11 +1592,11 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delete_after_unseen_days', - full_name= - 'paddle.DownpourTableAccessorParameter.delete_after_unseen_days', + full_name='paddle.DownpourTableAccessorParameter.delete_after_unseen_days', index=7, number=8, type=2, @@ -1532,11 +1609,11 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ssd_unseenday_threshold', - full_name= - 'paddle.DownpourTableAccessorParameter.ssd_unseenday_threshold', + full_name='paddle.DownpourTableAccessorParameter.ssd_unseenday_threshold', index=8, number=9, type=5, @@ -1549,7 +1626,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1585,7 +1663,8 @@ _TABLEACCESSORSAVEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='converter', full_name='paddle.TableAccessorSaveParameter.converter', @@ -1601,7 +1680,8 @@ _TABLEACCESSORSAVEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='deconverter', full_name='paddle.TableAccessorSaveParameter.deconverter', @@ -1617,7 +1697,8 @@ _TABLEACCESSORSAVEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1638,21 +1719,23 @@ _PSREQUESTMESSAGE = _descriptor.Descriptor( file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='cmd_id', - full_name='paddle.PsRequestMessage.cmd_id', - index=0, - number=1, - type=13, - cpp_type=3, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='cmd_id', + full_name='paddle.PsRequestMessage.cmd_id', + index=0, + number=1, + type=13, + cpp_type=3, + label=2, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='table_id', full_name='paddle.PsRequestMessage.table_id', @@ -1668,22 +1751,25 @@ _PSREQUESTMESSAGE = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='params', - full_name='paddle.PsRequestMessage.params', - index=2, - number=3, - type=12, - cpp_type=9, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='params', + full_name='paddle.PsRequestMessage.params', + index=2, + number=3, + type=12, + cpp_type=9, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='client_id', full_name='paddle.PsRequestMessage.client_id', @@ -1699,22 +1785,25 @@ _PSREQUESTMESSAGE = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='data', - full_name='paddle.PsRequestMessage.data', - index=4, - number=5, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='data', + full_name='paddle.PsRequestMessage.data', + index=4, + number=5, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -1750,7 +1839,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_g2sum', full_name='paddle.SparseSGDRuleParameter.initial_g2sum', @@ -1766,7 +1856,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseSGDRuleParameter.initial_range', @@ -1782,7 +1873,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseSGDRuleParameter.weight_bounds', @@ -1798,7 +1890,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1834,7 +1927,8 @@ _SPARSECOMMONSGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='naive', full_name='paddle.SparseCommonSGDRuleParameter.naive', @@ -1850,7 +1944,8 @@ _SPARSECOMMONSGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='adagrad', full_name='paddle.SparseCommonSGDRuleParameter.adagrad', @@ -1866,7 +1961,8 @@ _SPARSECOMMONSGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='adam', full_name='paddle.SparseCommonSGDRuleParameter.adam', @@ -1882,7 +1978,8 @@ _SPARSECOMMONSGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1918,7 +2015,8 @@ _SPARSENAIVESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseNaiveSGDRuleParameter.initial_range', @@ -1934,7 +2032,8 @@ _SPARSENAIVESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseNaiveSGDRuleParameter.weight_bounds', @@ -1950,7 +2049,8 @@ _SPARSENAIVESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1986,7 +2086,8 @@ _SPARSEADAGRADSGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_g2sum', full_name='paddle.SparseAdagradSGDRuleParameter.initial_g2sum', @@ -2002,7 +2103,8 @@ _SPARSEADAGRADSGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseAdagradSGDRuleParameter.initial_range', @@ -2018,7 +2120,8 @@ _SPARSEADAGRADSGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseAdagradSGDRuleParameter.weight_bounds', @@ -2034,7 +2137,8 @@ _SPARSEADAGRADSGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2070,7 +2174,8 @@ _SPARSEADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseAdamSGDParameter.initial_range', @@ -2086,7 +2191,8 @@ _SPARSEADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='beta1_decay_rate', full_name='paddle.SparseAdamSGDParameter.beta1_decay_rate', @@ -2102,7 +2208,8 @@ _SPARSEADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='beta2_decay_rate', full_name='paddle.SparseAdamSGDParameter.beta2_decay_rate', @@ -2118,7 +2225,8 @@ _SPARSEADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_epsilon', full_name='paddle.SparseAdamSGDParameter.ada_epsilon', @@ -2134,7 +2242,8 @@ _SPARSEADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseAdamSGDParameter.weight_bounds', @@ -2150,7 +2259,8 @@ _SPARSEADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2186,7 +2296,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='adam', full_name='paddle.DenseSGDRuleParameter.adam', @@ -2202,7 +2313,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='naive', full_name='paddle.DenseSGDRuleParameter.naive', @@ -2218,7 +2330,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='summary', full_name='paddle.DenseSGDRuleParameter.summary', @@ -2234,7 +2347,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='moving_average', full_name='paddle.DenseSGDRuleParameter.moving_average', @@ -2250,7 +2364,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2286,7 +2401,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='avg_decay_rate', full_name='paddle.AdamSGDParameter.avg_decay_rate', @@ -2302,7 +2418,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_decay_rate', full_name='paddle.AdamSGDParameter.ada_decay_rate', @@ -2318,7 +2435,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_epsilon', full_name='paddle.AdamSGDParameter.ada_epsilon', @@ -2334,7 +2452,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='mom_decay_rate', full_name='paddle.AdamSGDParameter.mom_decay_rate', @@ -2350,7 +2469,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2386,7 +2506,8 @@ _NAIVESGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='avg_decay_rate', full_name='paddle.NaiveSGDParameter.avg_decay_rate', @@ -2402,7 +2523,8 @@ _NAIVESGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2438,7 +2560,8 @@ _SUMMARYSGDPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2474,7 +2597,8 @@ _MOVINGAVERAGERULEPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2510,7 +2634,8 @@ _PSRESPONSEMESSAGE = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='err_msg', full_name='paddle.PsResponseMessage.err_msg', @@ -2526,22 +2651,25 @@ _PSRESPONSEMESSAGE = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='data', - full_name='paddle.PsResponseMessage.data', - index=2, - number=3, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='data', + full_name='paddle.PsResponseMessage.data', + index=2, + number=3, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -2577,52 +2705,59 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='uri', - full_name='paddle.FsClientParameter.uri', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='user', - full_name='paddle.FsClientParameter.user', - index=2, - number=3, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='passwd', - full_name='paddle.FsClientParameter.passwd', - index=3, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='uri', + full_name='paddle.FsClientParameter.uri', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='user', + full_name='paddle.FsClientParameter.user', + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='passwd', + full_name='paddle.FsClientParameter.passwd', + index=3, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='buffer_size', full_name='paddle.FsClientParameter.buffer_size', @@ -2638,7 +2773,8 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='hadoop_bin', full_name='paddle.FsClientParameter.hadoop_bin', @@ -2654,7 +2790,8 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='afs_conf', full_name='paddle.FsClientParameter.afs_conf', @@ -2670,7 +2807,8 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2689,95 +2827,132 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor( _PSPARAMETER.fields_by_name['worker_param'].message_type = _WORKERPARAMETER _PSPARAMETER.fields_by_name['server_param'].message_type = _SERVERPARAMETER _PSPARAMETER.fields_by_name[ - 'trainer_param'].message_type = _DOWNPOURTRAINERPARAMETER + 'trainer_param' +].message_type = _DOWNPOURTRAINERPARAMETER _PSPARAMETER.fields_by_name['fs_client_param'].message_type = _FSCLIENTPARAMETER _WORKERPARAMETER.fields_by_name[ - 'downpour_worker_param'].message_type = _DOWNPOURWORKERPARAMETER + 'downpour_worker_param' +].message_type = _DOWNPOURWORKERPARAMETER _SERVERPARAMETER.fields_by_name[ - 'downpour_server_param'].message_type = _DOWNPOURSERVERPARAMETER + 'downpour_server_param' +].message_type = _DOWNPOURSERVERPARAMETER _DOWNPOURWORKERPARAMETER.fields_by_name[ - 'downpour_table_param'].message_type = _TABLEPARAMETER + 'downpour_table_param' +].message_type = _TABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'dense_table'].message_type = _DENSETABLEPARAMETER + 'dense_table' +].message_type = _DENSETABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'sparse_table'].message_type = _SPARSETABLEPARAMETER + 'sparse_table' +].message_type = _SPARSETABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'program_config'].message_type = _PROGRAMCONFIG + 'program_config' +].message_type = _PROGRAMCONFIG _DOWNPOURSERVERPARAMETER.fields_by_name[ - 'downpour_table_param'].message_type = _TABLEPARAMETER + 'downpour_table_param' +].message_type = _TABLEPARAMETER _DOWNPOURSERVERPARAMETER.fields_by_name[ - 'service_param'].message_type = _SERVERSERVICEPARAMETER + 'service_param' +].message_type = _SERVERSERVICEPARAMETER _TABLEPARAMETER.fields_by_name[ - 'accessor'].message_type = _TABLEACCESSORPARAMETER + 'accessor' +].message_type = _TABLEACCESSORPARAMETER _TABLEPARAMETER.fields_by_name['type'].enum_type = _TABLETYPE _TABLEACCESSORPARAMETER.fields_by_name[ - 'sparse_sgd_param'].message_type = _SPARSESGDRULEPARAMETER + 'sparse_sgd_param' +].message_type = _SPARSESGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'dense_sgd_param'].message_type = _DENSESGDRULEPARAMETER + 'dense_sgd_param' +].message_type = _DENSESGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'downpour_accessor_param'].message_type = _DOWNPOURTABLEACCESSORPARAMETER + 'downpour_accessor_param' +].message_type = _DOWNPOURTABLEACCESSORPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'table_accessor_save_param'].message_type = _TABLEACCESSORSAVEPARAMETER + 'table_accessor_save_param' +].message_type = _TABLEACCESSORSAVEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'sparse_commonsgd_param'].message_type = _SPARSECOMMONSGDRULEPARAMETER + 'sparse_commonsgd_param' +].message_type = _SPARSECOMMONSGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'embed_sgd_param'].message_type = _SPARSECOMMONSGDRULEPARAMETER + 'embed_sgd_param' +].message_type = _SPARSECOMMONSGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'embedx_sgd_param'].message_type = _SPARSECOMMONSGDRULEPARAMETER + 'embedx_sgd_param' +].message_type = _SPARSECOMMONSGDRULEPARAMETER _SPARSECOMMONSGDRULEPARAMETER.fields_by_name[ - 'naive'].message_type = _SPARSENAIVESGDRULEPARAMETER + 'naive' +].message_type = _SPARSENAIVESGDRULEPARAMETER _SPARSECOMMONSGDRULEPARAMETER.fields_by_name[ - 'adagrad'].message_type = _SPARSEADAGRADSGDRULEPARAMETER + 'adagrad' +].message_type = _SPARSEADAGRADSGDRULEPARAMETER _SPARSECOMMONSGDRULEPARAMETER.fields_by_name[ - 'adam'].message_type = _SPARSEADAMSGDPARAMETER + 'adam' +].message_type = _SPARSEADAMSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name['adam'].message_type = _ADAMSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name['naive'].message_type = _NAIVESGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name[ - 'summary'].message_type = _SUMMARYSGDPARAMETER + 'summary' +].message_type = _SUMMARYSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name[ - 'moving_average'].message_type = _MOVINGAVERAGERULEPARAMETER + 'moving_average' +].message_type = _MOVINGAVERAGERULEPARAMETER _FSCLIENTPARAMETER.fields_by_name[ - 'fs_type'].enum_type = _FSCLIENTPARAMETER_FSAPITYPE + 'fs_type' +].enum_type = _FSCLIENTPARAMETER_FSAPITYPE _FSCLIENTPARAMETER_FSAPITYPE.containing_type = _FSCLIENTPARAMETER DESCRIPTOR.message_types_by_name['PSParameter'] = _PSPARAMETER DESCRIPTOR.message_types_by_name['WorkerParameter'] = _WORKERPARAMETER DESCRIPTOR.message_types_by_name['ServerParameter'] = _SERVERPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourWorkerParameter'] = _DOWNPOURWORKERPARAMETER + 'DownpourWorkerParameter' +] = _DOWNPOURWORKERPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourTrainerParameter'] = _DOWNPOURTRAINERPARAMETER + 'DownpourTrainerParameter' +] = _DOWNPOURTRAINERPARAMETER DESCRIPTOR.message_types_by_name['ProgramConfig'] = _PROGRAMCONFIG DESCRIPTOR.message_types_by_name['DenseTableParameter'] = _DENSETABLEPARAMETER DESCRIPTOR.message_types_by_name['SparseTableParameter'] = _SPARSETABLEPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourServerParameter'] = _DOWNPOURSERVERPARAMETER + 'DownpourServerParameter' +] = _DOWNPOURSERVERPARAMETER DESCRIPTOR.message_types_by_name[ - 'ServerServiceParameter'] = _SERVERSERVICEPARAMETER + 'ServerServiceParameter' +] = _SERVERSERVICEPARAMETER DESCRIPTOR.message_types_by_name['TableParameter'] = _TABLEPARAMETER DESCRIPTOR.message_types_by_name[ - 'TableAccessorParameter'] = _TABLEACCESSORPARAMETER + 'TableAccessorParameter' +] = _TABLEACCESSORPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourTableAccessorParameter'] = _DOWNPOURTABLEACCESSORPARAMETER + 'DownpourTableAccessorParameter' +] = _DOWNPOURTABLEACCESSORPARAMETER DESCRIPTOR.message_types_by_name[ - 'TableAccessorSaveParameter'] = _TABLEACCESSORSAVEPARAMETER + 'TableAccessorSaveParameter' +] = _TABLEACCESSORSAVEPARAMETER DESCRIPTOR.message_types_by_name['PsRequestMessage'] = _PSREQUESTMESSAGE DESCRIPTOR.message_types_by_name[ - 'SparseSGDRuleParameter'] = _SPARSESGDRULEPARAMETER + 'SparseSGDRuleParameter' +] = _SPARSESGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'SparseCommonSGDRuleParameter'] = _SPARSECOMMONSGDRULEPARAMETER + 'SparseCommonSGDRuleParameter' +] = _SPARSECOMMONSGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'SparseNaiveSGDRuleParameter'] = _SPARSENAIVESGDRULEPARAMETER + 'SparseNaiveSGDRuleParameter' +] = _SPARSENAIVESGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'SparseAdagradSGDRuleParameter'] = _SPARSEADAGRADSGDRULEPARAMETER + 'SparseAdagradSGDRuleParameter' +] = _SPARSEADAGRADSGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'SparseAdamSGDParameter'] = _SPARSEADAMSGDPARAMETER + 'SparseAdamSGDParameter' +] = _SPARSEADAMSGDPARAMETER DESCRIPTOR.message_types_by_name[ - 'DenseSGDRuleParameter'] = _DENSESGDRULEPARAMETER + 'DenseSGDRuleParameter' +] = _DENSESGDRULEPARAMETER DESCRIPTOR.message_types_by_name['AdamSGDParameter'] = _ADAMSGDPARAMETER DESCRIPTOR.message_types_by_name['NaiveSGDParameter'] = _NAIVESGDPARAMETER DESCRIPTOR.message_types_by_name['SummarySGDParameter'] = _SUMMARYSGDPARAMETER DESCRIPTOR.message_types_by_name[ - 'MovingAverageRuleParameter'] = _MOVINGAVERAGERULEPARAMETER + 'MovingAverageRuleParameter' +] = _MOVINGAVERAGERULEPARAMETER DESCRIPTOR.message_types_by_name['PsResponseMessage'] = _PSRESPONSEMESSAGE DESCRIPTOR.message_types_by_name['FsClientParameter'] = _FSCLIENTPARAMETER DESCRIPTOR.enum_types_by_name['TableType'] = _TABLETYPE @@ -2785,254 +2960,303 @@ DESCRIPTOR.enum_types_by_name['PsCmdID'] = _PSCMDID PSParameter = _reflection.GeneratedProtocolMessageType( 'PSParameter', - (_message.Message, ), - dict(DESCRIPTOR=_PSPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PSParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PSParameter) + ), +) _sym_db.RegisterMessage(PSParameter) WorkerParameter = _reflection.GeneratedProtocolMessageType( 'WorkerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_WORKERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.WorkerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_WORKERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.WorkerParameter) + ), +) _sym_db.RegisterMessage(WorkerParameter) ServerParameter = _reflection.GeneratedProtocolMessageType( 'ServerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SERVERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ServerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SERVERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ServerParameter) + ), +) _sym_db.RegisterMessage(ServerParameter) DownpourWorkerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourWorkerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURWORKERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourWorkerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURWORKERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourWorkerParameter) + ), +) _sym_db.RegisterMessage(DownpourWorkerParameter) DownpourTrainerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourTrainerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURTRAINERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourTrainerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURTRAINERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourTrainerParameter) + ), +) _sym_db.RegisterMessage(DownpourTrainerParameter) ProgramConfig = _reflection.GeneratedProtocolMessageType( 'ProgramConfig', - (_message.Message, ), - dict(DESCRIPTOR=_PROGRAMCONFIG, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ProgramConfig) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PROGRAMCONFIG, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ProgramConfig) + ), +) _sym_db.RegisterMessage(ProgramConfig) DenseTableParameter = _reflection.GeneratedProtocolMessageType( 'DenseTableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DENSETABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DenseTableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DENSETABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DenseTableParameter) + ), +) _sym_db.RegisterMessage(DenseTableParameter) SparseTableParameter = _reflection.GeneratedProtocolMessageType( 'SparseTableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSETABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseTableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSETABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseTableParameter) + ), +) _sym_db.RegisterMessage(SparseTableParameter) DownpourServerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourServerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURSERVERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourServerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURSERVERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourServerParameter) + ), +) _sym_db.RegisterMessage(DownpourServerParameter) ServerServiceParameter = _reflection.GeneratedProtocolMessageType( 'ServerServiceParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SERVERSERVICEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ServerServiceParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SERVERSERVICEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ServerServiceParameter) + ), +) _sym_db.RegisterMessage(ServerServiceParameter) TableParameter = _reflection.GeneratedProtocolMessageType( 'TableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_TABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.TableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_TABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.TableParameter) + ), +) _sym_db.RegisterMessage(TableParameter) TableAccessorParameter = _reflection.GeneratedProtocolMessageType( 'TableAccessorParameter', - (_message.Message, ), - dict(DESCRIPTOR=_TABLEACCESSORPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.TableAccessorParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_TABLEACCESSORPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.TableAccessorParameter) + ), +) _sym_db.RegisterMessage(TableAccessorParameter) DownpourTableAccessorParameter = _reflection.GeneratedProtocolMessageType( 'DownpourTableAccessorParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_DOWNPOURTABLEACCESSORPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.DownpourTableAccessorParameter) - )) + ), +) _sym_db.RegisterMessage(DownpourTableAccessorParameter) TableAccessorSaveParameter = _reflection.GeneratedProtocolMessageType( 'TableAccessorSaveParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_TABLEACCESSORSAVEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.TableAccessorSaveParameter) - )) + ), +) _sym_db.RegisterMessage(TableAccessorSaveParameter) PsRequestMessage = _reflection.GeneratedProtocolMessageType( 'PsRequestMessage', - (_message.Message, ), - dict(DESCRIPTOR=_PSREQUESTMESSAGE, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PsRequestMessage) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSREQUESTMESSAGE, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PsRequestMessage) + ), +) _sym_db.RegisterMessage(PsRequestMessage) SparseSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseSGDRuleParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSESGDRULEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseSGDRuleParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSESGDRULEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseSGDRuleParameter) + ), +) _sym_db.RegisterMessage(SparseSGDRuleParameter) SparseCommonSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseCommonSGDRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_SPARSECOMMONSGDRULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.SparseCommonSGDRuleParameter) - )) + ), +) _sym_db.RegisterMessage(SparseCommonSGDRuleParameter) SparseNaiveSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseNaiveSGDRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_SPARSENAIVESGDRULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.SparseNaiveSGDRuleParameter) - )) + ), +) _sym_db.RegisterMessage(SparseNaiveSGDRuleParameter) SparseAdagradSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseAdagradSGDRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_SPARSEADAGRADSGDRULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.SparseAdagradSGDRuleParameter) - )) + ), +) _sym_db.RegisterMessage(SparseAdagradSGDRuleParameter) SparseAdamSGDParameter = _reflection.GeneratedProtocolMessageType( 'SparseAdamSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSEADAMSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseAdamSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSEADAMSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseAdamSGDParameter) + ), +) _sym_db.RegisterMessage(SparseAdamSGDParameter) DenseSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'DenseSGDRuleParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DENSESGDRULEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DenseSGDRuleParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DENSESGDRULEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DenseSGDRuleParameter) + ), +) _sym_db.RegisterMessage(DenseSGDRuleParameter) AdamSGDParameter = _reflection.GeneratedProtocolMessageType( 'AdamSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_ADAMSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.AdamSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_ADAMSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.AdamSGDParameter) + ), +) _sym_db.RegisterMessage(AdamSGDParameter) NaiveSGDParameter = _reflection.GeneratedProtocolMessageType( 'NaiveSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_NAIVESGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.NaiveSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_NAIVESGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.NaiveSGDParameter) + ), +) _sym_db.RegisterMessage(NaiveSGDParameter) SummarySGDParameter = _reflection.GeneratedProtocolMessageType( 'SummarySGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SUMMARYSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SummarySGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SUMMARYSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SummarySGDParameter) + ), +) _sym_db.RegisterMessage(SummarySGDParameter) MovingAverageRuleParameter = _reflection.GeneratedProtocolMessageType( 'MovingAverageRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_MOVINGAVERAGERULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.MovingAverageRuleParameter) - )) + ), +) _sym_db.RegisterMessage(MovingAverageRuleParameter) PsResponseMessage = _reflection.GeneratedProtocolMessageType( 'PsResponseMessage', - (_message.Message, ), - dict(DESCRIPTOR=_PSRESPONSEMESSAGE, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PsResponseMessage) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSRESPONSEMESSAGE, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PsResponseMessage) + ), +) _sym_db.RegisterMessage(PsResponseMessage) FsClientParameter = _reflection.GeneratedProtocolMessageType( 'FsClientParameter', - (_message.Message, ), - dict(DESCRIPTOR=_FSCLIENTPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.FsClientParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_FSCLIENTPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.FsClientParameter) + ), +) _sym_db.RegisterMessage(FsClientParameter) DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), - _b('\200\001\001\370\001\001')) +DESCRIPTOR._options = _descriptor._ParseOptions( + descriptor_pb2.FileOptions(), _b('\200\001\001\370\001\001') +) # @@protoc_insertion_point(module_scope) diff --git a/python/paddle/fluid/incubate/fleet/tests/ctr_dataset_reader.py b/python/paddle/fluid/incubate/fleet/tests/ctr_dataset_reader.py index 2530a611336..97914fe1612 100644 --- a/python/paddle/fluid/incubate/fleet/tests/ctr_dataset_reader.py +++ b/python/paddle/fluid/incubate/fleet/tests/ctr_dataset_reader.py @@ -20,9 +20,9 @@ import paddle import paddle.distributed.fleet as fleet from paddle.fluid.log_helper import get_logger -logger = get_logger("paddle", - logging.INFO, - fmt='%(asctime)s - %(levelname)s - %(message)s') +logger = get_logger( + "paddle", logging.INFO, fmt='%(asctime)s - %(levelname)s - %(message)s' +) DATA_URL = "http://paddle-ctr-data.bj.bcebos.com/avazu_ctr_data.tgz" DATA_MD5 = "c11df99fbd14e53cd4bfa6567344b26e" @@ -59,17 +59,16 @@ def load_lr_input_record(sent): class DatasetCtrReader(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def iter(): fs = line.strip().split('\t') dnn_input = load_dnn_input_record(fs[0]) lr_input = load_lr_input_record(fs[1]) click = [int(fs[2])] - yield ("dnn_data", dnn_input), \ - ("lr_data", lr_input), \ - ("click", click) + yield ("dnn_data", dnn_input), ("lr_data", lr_input), ( + "click", + click, + ) return iter @@ -85,7 +84,9 @@ def prepare_data(): lines = f.readlines() err_info = "wrong meta format" assert len(lines) == 2, err_info - assert 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1], err_info + assert ( + 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1] + ), err_info res = map(int, [_.split(':')[1] for _ in lines]) res = list(res) dnn_input_dim = res[0] diff --git a/python/paddle/fluid/incubate/fleet/tests/fleet_deep_ctr.py b/python/paddle/fluid/incubate/fleet/tests/fleet_deep_ctr.py index 1b763c6ed59..d2c843ca4d0 100644 --- a/python/paddle/fluid/incubate/fleet/tests/fleet_deep_ctr.py +++ b/python/paddle/fluid/incubate/fleet/tests/fleet_deep_ctr.py @@ -19,16 +19,20 @@ import time import paddle import paddle.fluid as fluid import paddle.fluid.incubate.fleet.base.role_maker as role_maker -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, +) from paddle.fluid.log_helper import get_logger import ctr_dataset_reader -logger = get_logger("fluid", - logging.INFO, - fmt='%(asctime)s - %(levelname)s - %(message)s') +logger = get_logger( + "fluid", logging.INFO, fmt='%(asctime)s - %(levelname)s - %(message)s' +) def parse_args(): @@ -39,48 +43,64 @@ def parse_args(): '--role', type=str, default='pserver', # trainer or pserver - help='The path for model to store (default: models)') + help='The path for model to store (default: models)', + ) parser.add_argument( '--endpoints', type=str, default='127.0.0.1:6000', - help='The pserver endpoints, like: 127.0.0.1:6000,127.0.0.1:6001') + help='The pserver endpoints, like: 127.0.0.1:6000,127.0.0.1:6001', + ) parser.add_argument( '--current_endpoint', type=str, default='127.0.0.1:6000', - help='The path for model to store (default: 127.0.0.1:6000)') - parser.add_argument('--trainer_id', - type=int, - default=0, - help='The path for model to store (default: models)') - parser.add_argument('--trainers', - type=int, - default=1, - help='The num of trainers, (default: 1)') + help='The path for model to store (default: 127.0.0.1:6000)', + ) + parser.add_argument( + '--trainer_id', + type=int, + default=0, + help='The path for model to store (default: models)', + ) + parser.add_argument( + '--trainers', + type=int, + default=1, + help='The num of trainers, (default: 1)', + ) return parser.parse_args() def model(): - dnn_input_dim, lr_input_dim, train_file_path = ctr_dataset_reader.prepare_data( - ) + ( + dnn_input_dim, + lr_input_dim, + train_file_path, + ) = ctr_dataset_reader.prepare_data() """ network definition """ - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] @@ -92,8 +112,10 @@ def model(): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, pool_type="sum") dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): @@ -101,9 +123,11 @@ def model(): input=dnn_out, size=dim, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01)), - name='dnn-fc-%d' % i) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -113,16 +137,19 @@ def model(): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) predict = fluid.layers.fc(input=merge_layer, size=2, act='softmax') acc = fluid.layers.accuracy(input=predict, label=label) - auc_var, batch_auc_var, auc_states = fluid.layers.auc(input=predict, - label=label) + auc_var, batch_auc_var, auc_states = fluid.layers.auc( + input=predict, label=label + ) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(x=cost) @@ -140,9 +167,11 @@ def train(args): role = role_maker.UserDefinedRoleMaker( current_id=current_id, role=role_maker.Role.WORKER - if args.role.upper() == "TRAINER" else role_maker.Role.SERVER, + if args.role.upper() == "TRAINER" + else role_maker.Role.SERVER, worker_num=args.trainers, - server_endpoints=endpoints) + server_endpoints=endpoints, + ) exe = fluid.Executor(fluid.CPUPlace()) fleet.init(role) @@ -183,15 +212,18 @@ def train(args): logger.info("epoch {} start".format(epoch_id)) pass_start = time.time() dataset.set_filelist(filelist) - exe.train_from_dataset(program=fleet.main_program, - dataset=dataset, - fetch_list=[avg_cost], - fetch_info=["cost"], - print_period=100, - debug=False) + exe.train_from_dataset( + program=fleet.main_program, + dataset=dataset, + fetch_list=[avg_cost], + fetch_info=["cost"], + print_period=100, + debug=False, + ) pass_time = time.time() - pass_start - logger.info("epoch {} finished, pass_time {}".format( - epoch_id, pass_time)) + logger.info( + "epoch {} finished, pass_time {}".format(epoch_id, pass_time) + ) fleet.stop_worker() diff --git a/python/paddle/fluid/incubate/fleet/utils/fleet_util.py b/python/paddle/fluid/incubate/fleet/utils/fleet_util.py index 365043f92d0..b15892e0f5e 100644 --- a/python/paddle/fluid/incubate/fleet/utils/fleet_util.py +++ b/python/paddle/fluid/incubate/fleet/utils/fleet_util.py @@ -32,9 +32,9 @@ OpRole = core.op_proto_and_checker_maker.OpRole __all__ = ["FleetUtil", "GPUPSUtil"] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s %(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s %(levelname)s: %(message)s' +) fleet = None @@ -57,14 +57,21 @@ class FleetUtil(object): op_maker = core.op_proto_and_checker_maker self.op_role_key = op_maker.kOpRoleAttrName() if mode == "pslib": - from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet as fleet_pslib + from paddle.fluid.incubate.fleet.parameter_server.pslib import ( + fleet as fleet_pslib, + ) + fleet = fleet_pslib elif mode == "transpiler": - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet as fleet_transpiler + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet as fleet_transpiler, + ) + fleet = fleet_transpiler else: raise ValueError( - "Please choose one mode from [\"pslib\", \"transpiler\"]") + "Please choose one mode from [\"pslib\", \"transpiler\"]" + ) def rank0_print(self, s): """ @@ -124,11 +131,13 @@ class FleetUtil(object): return _logger.error(s) - def set_zero(self, - var_name, - scope=fluid.global_scope(), - place=fluid.CPUPlace(), - param_type="int64"): + def set_zero( + self, + var_name, + scope=fluid.global_scope(), + place=fluid.CPUPlace(), + param_type="int64", + ): """ Set tensor of a Variable to zero. @@ -150,11 +159,13 @@ class FleetUtil(object): param_array = np.zeros(param._get_dims()).astype(param_type) param.set(param_array, place) - def print_global_auc(self, - scope=fluid.global_scope(), - stat_pos="_generated_var_2", - stat_neg="_generated_var_3", - print_prefix=""): + def print_global_auc( + self, + scope=fluid.global_scope(), + stat_pos="_generated_var_2", + stat_neg="_generated_var_3", + print_prefix="", + ): r""" Print global auc of all distributed workers. @@ -189,10 +200,12 @@ class FleetUtil(object): auc_value = self.get_global_auc(scope, stat_pos, stat_neg) self.rank0_print(print_prefix + " global auc = %s" % auc_value) - def get_global_auc(self, - scope=fluid.global_scope(), - stat_pos="_generated_var_2", - stat_neg="_generated_var_3"): + def get_global_auc( + self, + scope=fluid.global_scope(), + stat_pos="_generated_var_2", + stat_neg="_generated_var_3", + ): """ Get global auc of all distributed workers. @@ -324,15 +337,17 @@ class FleetUtil(object): """ fleet.save_persistables(None, path, mode=mode) - def _get_xbox_str(self, - output_path, - day, - model_path, - xbox_base_key, - data_path, - hadoop_fs_name, - monitor_data={}, - mode="patch"): + def _get_xbox_str( + self, + output_path, + day, + model_path, + xbox_base_key, + data_path, + hadoop_fs_name, + monitor_data={}, + mode="patch", + ): xbox_dict = collections.OrderedDict() if mode == "base": xbox_dict["id"] = str(xbox_base_key) @@ -344,7 +359,7 @@ class FleetUtil(object): xbox_dict["id"] = str(int(time.time())) xbox_dict["key"] = str(xbox_base_key) if model_path.startswith("hdfs:") or model_path.startswith("afs:"): - model_path = model_path[model_path.find(":") + 1:] + model_path = model_path[model_path.find(":") + 1 :] xbox_dict["input"] = hadoop_fs_name + model_path.rstrip("/") + "/000" xbox_dict["record_count"] = "111111" xbox_dict["partition_type"] = "2" @@ -360,20 +375,23 @@ class FleetUtil(object): xbox_dict["job_id"] = job_id_with_host # currently hard code here, set monitor_data empty string xbox_dict["monitor_data"] = "" - xbox_dict["monitor_path"] = output_path.rstrip("/") + "/monitor/" \ - + day + ".txt" + xbox_dict["monitor_path"] = ( + output_path.rstrip("/") + "/monitor/" + day + ".txt" + ) xbox_dict["mpi_size"] = str(fleet.worker_num()) return json.dumps(xbox_dict) - def write_model_donefile(self, - output_path, - day, - pass_id, - xbox_base_key, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME", - donefile_name="donefile.txt"): + def write_model_donefile( + self, + output_path, + day, + pass_id, + xbox_base_key, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + donefile_name="donefile.txt", + ): """ write donefile when save model @@ -414,11 +432,16 @@ class FleetUtil(object): if fleet.worker_index() == 0: donefile_path = output_path + "/" + donefile_name - content = "%s\t%lu\t%s\t%s\t%d" % (day, xbox_base_key,\ - model_path, pass_id, 0) + content = "%s\t%lu\t%s\t%s\t%d" % ( + day, + xbox_base_key, + model_path, + pass_id, + 0, + ) configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if client.is_file(donefile_path): @@ -428,8 +451,9 @@ class FleetUtil(object): pass_list = [i.split("\t")[3] for i in pre_content_list] exist = False for i in range(len(day_list)): - if int(day) == int(day_list[i]) and \ - int(pass_id) == int(pass_list[i]): + if int(day) == int(day_list[i]) and int(pass_id) == int( + pass_list[i] + ): exist = True break if not exist: @@ -438,30 +462,36 @@ class FleetUtil(object): f.write(content + "\n") client.delete(donefile_path) client.upload(donefile_name, output_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) else: - self.rank0_error("not write %s because %s/%s already " - "exists" % (donefile_name, day, pass_id)) + self.rank0_error( + "not write %s because %s/%s already " + "exists" % (donefile_name, day, pass_id) + ) else: with open(donefile_name, "w") as f: f.write(content + "\n") client.upload(donefile_name, output_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) fleet._role_maker._barrier_worker() - def write_xbox_donefile(self, - output_path, - day, - pass_id, - xbox_base_key, - data_path, - hadoop_fs_name, - hadoop_fs_ugi, - monitor_data={}, - hadoop_home="$HADOOP_HOME", - donefile_name=None): + def write_xbox_donefile( + self, + output_path, + day, + pass_id, + xbox_base_key, + data_path, + hadoop_fs_name, + hadoop_fs_ugi, + monitor_data={}, + hadoop_home="$HADOOP_HOME", + donefile_name=None, + ): """ write delta donefile or xbox base donefile @@ -518,12 +548,19 @@ class FleetUtil(object): if fleet.worker_index() == 0: donefile_path = output_path + "/" + donefile_name - xbox_str = self._get_xbox_str(output_path, day, model_path, \ - xbox_base_key, data_path, hadoop_fs_name, monitor_data={}, - mode=mode) + xbox_str = self._get_xbox_str( + output_path, + day, + model_path, + xbox_base_key, + data_path, + hadoop_fs_name, + monitor_data={}, + mode=mode, + ) configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if client.is_file(donefile_path): @@ -532,9 +569,11 @@ class FleetUtil(object): last_day = last_dict["input"].split("/")[-3] last_pass = last_dict["input"].split("/")[-2].split("-")[-1] exist = False - if int(day) < int(last_day) or \ - int(day) == int(last_day) and \ - int(pass_id) <= int(last_pass): + if ( + int(day) < int(last_day) + or int(day) == int(last_day) + and int(pass_id) <= int(last_pass) + ): exist = True if not exist: with open(donefile_name, "w") as f: @@ -542,29 +581,35 @@ class FleetUtil(object): f.write(xbox_str + "\n") client.delete(donefile_path) client.upload(donefile_name, output_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) else: - self.rank0_error("not write %s because %s/%s already " - "exists" % (donefile_name, day, pass_id)) + self.rank0_error( + "not write %s because %s/%s already " + "exists" % (donefile_name, day, pass_id) + ) else: with open(donefile_name, "w") as f: f.write(xbox_str + "\n") client.upload(donefile_name, output_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) fleet._role_maker._barrier_worker() - def write_cache_donefile(self, - output_path, - day, - pass_id, - key_num, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME", - donefile_name="sparse_cache.meta", - **kwargs): + def write_cache_donefile( + self, + output_path, + day, + pass_id, + key_num, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + donefile_name="sparse_cache.meta", + **kwargs + ): """ write cache donefile @@ -613,15 +658,18 @@ class FleetUtil(object): donefile_path = model_path + "/" + donefile_name configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if client.is_file(donefile_path): - self.rank0_error( \ - "not write because %s already exists" % donefile_path) + self.rank0_error( + "not write because %s already exists" % donefile_path + ) else: - meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" \ - % (file_num, key_num) + meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" % ( + file_num, + key_num, + ) with open(donefile_name, "w") as f: f.write(meta_str) client.upload(donefile_name, model_path) @@ -779,10 +827,9 @@ class FleetUtil(object): suffix_name = "/%s/delta-%s" % (day, pass_id) model_path = output_path.rstrip("/") + suffix_name self.rank0_print("going to save_cache_model %s" % model_path) - key_num = fleet.save_cache_model(None, - model_path, - mode=mode, - table_id=table_id) + key_num = fleet.save_cache_model( + None, model_path, mode=mode, table_id=table_id + ) self.rank0_print("save_cache_model done") return key_num @@ -813,10 +860,9 @@ class FleetUtil(object): suffix_name = "/%s/base" % day model_path = output_path.rstrip("/") + suffix_name self.rank0_print("going to save_cache_base_model %s" % model_path) - key_num = fleet.save_cache_model(None, - model_path, - mode=2, - table_id=table_id) + key_num = fleet.save_cache_model( + None, model_path, mode=2, table_id=table_id + ) self.rank0_print("save_cache_base_model done") return key_num @@ -839,8 +885,11 @@ class FleetUtil(object): fleet._role_maker._barrier_worker() if fleet._role_maker.is_first_worker(): prog_id = str(id(program)) - tables = fleet._opt_info["program_id_to_worker"][prog_id].\ - get_desc().dense_table + tables = ( + fleet._opt_info["program_id_to_worker"][prog_id] + .get_desc() + .dense_table + ) prog_conf = fleet._opt_info['program_configs'][prog_id] prog_tables = {} for key in prog_conf: @@ -855,27 +904,33 @@ class FleetUtil(object): for i in range(0, len(table.dense_variable_name)): var_name = table.dense_variable_name[i] if scope.find_var(var_name) is None: - raise ValueError("var " + var_name + - " not found in scope " + - "when pull dense") + raise ValueError( + "var " + + var_name + + " not found in scope " + + "when pull dense" + ) var_name_list.append(var_name) - fleet._fleet_ptr.pull_dense(scope, int(table.table_id), - var_name_list) + fleet._fleet_ptr.pull_dense( + scope, int(table.table_id), var_name_list + ) fleet._role_maker._barrier_worker() - def save_paddle_inference_model(self, - executor, - scope, - program, - feeded_vars, - target_vars, - output_path, - day, - pass_id, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME", - save_combine=True): + def save_paddle_inference_model( + self, + executor, + scope, + program, + feeded_vars, + target_vars, + output_path, + day, + pass_id, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + save_combine=True, + ): """ save paddle inference model, and upload to hdfs dnn_plugin path @@ -925,26 +980,31 @@ class FleetUtil(object): target_vars=target_vars, executor=executor, main_program=program.clone(), - params_filename="params") + params_filename="params", + ) else: fluid.io.save_inference_model( dirname=model_name, feeded_var_names=feeded_var_names, target_vars=target_vars, executor=executor, - main_program=program.clone()) + main_program=program.clone(), + ) configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if pass_id == "-1": dest = "%s/%s/base/dnn_plugin/" % (output_path, day) else: - dest = "%s/%s/delta-%s/dnn_plugin/" % (output_path, day, - pass_id) + dest = "%s/%s/delta-%s/dnn_plugin/" % ( + output_path, + day, + pass_id, + ) if not client.is_exist(dest): client.makedirs(dest) @@ -952,19 +1012,21 @@ class FleetUtil(object): fleet._role_maker._barrier_worker() - def save_paddle_params(self, - executor, - scope, - program, - model_name, - output_path, - day, - pass_id, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME", - var_names=None, - save_combine=True): + def save_paddle_params( + self, + executor, + scope, + program, + model_name, + output_path, + day, + pass_id, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + var_names=None, + save_combine=True, + ): """ save paddle model, and upload to hdfs dnn_plugin path @@ -1028,36 +1090,39 @@ class FleetUtil(object): vars = [program.global_block().var(i) for i in var_names] with fluid.scope_guard(scope): if save_combine: - fluid.io.save_vars(executor, - "./", - program, - vars=vars, - filename=model_name) + fluid.io.save_vars( + executor, "./", program, vars=vars, filename=model_name + ) else: fluid.io.save_vars(executor, model_name, program, vars=vars) configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if pass_id == "-1": dest = "%s/%s/base/dnn_plugin/" % (output_path, day) else: - dest = "%s/%s/delta-%s/dnn_plugin/" % (output_path, day, - pass_id) + dest = "%s/%s/delta-%s/dnn_plugin/" % ( + output_path, + day, + pass_id, + ) if not client.is_exist(dest): client.mkdirs(dest) client.upload(model_name, dest, multi_processes=5, overwrite=True) fleet._role_maker._barrier_worker() - def get_last_save_xbox_base(self, - output_path, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME"): + def get_last_save_xbox_base( + self, + output_path, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + ): r""" get last saved base xbox info from xbox_base_done.txt @@ -1086,7 +1151,7 @@ class FleetUtil(object): donefile_path = output_path + "/xbox_base_done.txt" configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if not client.is_file(donefile_path): @@ -1098,11 +1163,13 @@ class FleetUtil(object): xbox_base_key = int(last_dict["key"]) return [last_day, last_path, xbox_base_key] - def get_last_save_xbox(self, - output_path, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME"): + def get_last_save_xbox( + self, + output_path, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + ): r""" get last saved xbox info from xbox_patch_done.txt @@ -1131,7 +1198,7 @@ class FleetUtil(object): donefile_path = output_path + "/xbox_patch_done.txt" configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if not client.is_file(donefile_path): @@ -1144,11 +1211,13 @@ class FleetUtil(object): xbox_base_key = int(last_dict["key"]) return [last_day, last_pass, last_path, xbox_base_key] - def get_last_save_model(self, - output_path, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME"): + def get_last_save_model( + self, + output_path, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + ): r""" get last saved model info from donefile.txt @@ -1180,7 +1249,7 @@ class FleetUtil(object): donefile_path = output_path + "/donefile.txt" configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if not client.is_file(donefile_path): @@ -1193,8 +1262,9 @@ class FleetUtil(object): xbox_base_key = int(content[1]) return [last_save_day, last_save_pass, last_path, xbox_base_key] - def get_online_pass_interval(self, days, hours, split_interval, - split_per_pass, is_data_hourly_placed): + def get_online_pass_interval( + self, days, hours, split_interval, split_per_pass, is_data_hourly_placed + ): """ get online pass interval @@ -1254,16 +1324,18 @@ class FleetUtil(object): return online_pass_interval - def get_global_metrics(self, - scope=fluid.global_scope(), - stat_pos_name="_generated_var_2", - stat_neg_name="_generated_var_3", - sqrerr_name="sqrerr", - abserr_name="abserr", - prob_name="prob", - q_name="q", - pos_ins_num_name="pos", - total_ins_num_name="total"): + def get_global_metrics( + self, + scope=fluid.global_scope(), + stat_pos_name="_generated_var_2", + stat_neg_name="_generated_var_3", + sqrerr_name="sqrerr", + abserr_name="abserr", + prob_name="prob", + q_name="q", + pos_ins_num_name="pos", + total_ins_num_name="total", + ): r""" get global metrics, including auc, bucket_error, mae, rmse, actual_ctr, predicted_ctr, copc, mean_predict_qvalue, total_ins_num. @@ -1317,8 +1389,10 @@ class FleetUtil(object): similarity_norm, label) """ - if scope.find_var(stat_pos_name) is None or \ - scope.find_var(stat_neg_name) is None: + if ( + scope.find_var(stat_pos_name) is None + or scope.find_var(stat_neg_name) is None + ): self.rank0_print("not found auc bucket") return [None] * 9 elif scope.find_var(sqrerr_name) is None: @@ -1337,8 +1411,9 @@ class FleetUtil(object): self.rank0_print("not found pos_ins_num_name=%s" % pos_ins_num_name) return [None] * 9 elif scope.find_var(total_ins_num_name) is None: - self.rank0_print("not found total_ins_num_name=%s" % \ - total_ins_num_name) + self.rank0_print( + "not found total_ins_num_name=%s" % total_ins_num_name + ) return [None] * 9 # barrier worker to ensure all workers finished training @@ -1427,8 +1502,9 @@ class FleetUtil(object): adjust_ctr = ctr_sum / impression_sum if adjust_ctr == 0: continue - relative_error = \ - math.sqrt((1 - adjust_ctr) / (adjust_ctr * impression_sum)) + relative_error = math.sqrt( + (1 - adjust_ctr) / (adjust_ctr * impression_sum) + ) if relative_error < k_relative_error_bound: actual_ctr = click_sum / impression_sum relative_ctr_error = abs(actual_ctr / adjust_ctr - 1) @@ -1439,22 +1515,30 @@ class FleetUtil(object): bucket_error = error_sum / error_count if error_count > 0 else 0.0 return [ - auc, bucket_error, mae, rmse, return_actual_ctr, predicted_ctr, - copc, mean_predict_qvalue, - int(total_ins_num) + auc, + bucket_error, + mae, + rmse, + return_actual_ctr, + predicted_ctr, + copc, + mean_predict_qvalue, + int(total_ins_num), ] - def print_global_metrics(self, - scope=fluid.global_scope(), - stat_pos_name="_generated_var_2", - stat_neg_name="_generated_var_3", - sqrerr_name="sqrerr", - abserr_name="abserr", - prob_name="prob", - q_name="q", - pos_ins_num_name="pos", - total_ins_num_name="total", - print_prefix=""): + def print_global_metrics( + self, + scope=fluid.global_scope(), + stat_pos_name="_generated_var_2", + stat_neg_name="_generated_var_3", + sqrerr_name="sqrerr", + abserr_name="abserr", + prob_name="prob", + q_name="q", + pos_ins_num_name="pos", + total_ins_num_name="total", + print_prefix="", + ): r""" print global metrics, including auc, bucket_error, mae, rmse, actual_ctr, predicted_ctr, copc, mean_predict_qvalue, total_ins_num. @@ -1505,8 +1589,10 @@ class FleetUtil(object): similarity_norm, label) """ - if scope.find_var(stat_pos_name) is None or \ - scope.find_var(stat_neg_name) is None: + if ( + scope.find_var(stat_pos_name) is None + or scope.find_var(stat_neg_name) is None + ): self.rank0_print("not found auc bucket") return elif scope.find_var(sqrerr_name) is None: @@ -1525,26 +1611,56 @@ class FleetUtil(object): self.rank0_print("not found pos_ins_num_name=%s" % pos_ins_num_name) return elif scope.find_var(total_ins_num_name) is None: - self.rank0_print("not found total_ins_num_name=%s" % \ - total_ins_num_name) + self.rank0_print( + "not found total_ins_num_name=%s" % total_ins_num_name + ) return - auc, bucket_error, mae, rmse, actual_ctr, predicted_ctr, copc,\ - mean_predict_qvalue, total_ins_num = self.get_global_metrics(\ - scope, stat_pos_name, stat_neg_name, sqrerr_name, abserr_name,\ - prob_name, q_name, pos_ins_num_name, total_ins_num_name) + ( + auc, + bucket_error, + mae, + rmse, + actual_ctr, + predicted_ctr, + copc, + mean_predict_qvalue, + total_ins_num, + ) = self.get_global_metrics( + scope, + stat_pos_name, + stat_neg_name, + sqrerr_name, + abserr_name, + prob_name, + q_name, + pos_ins_num_name, + total_ins_num_name, + ) self.rank0_print( "%s global AUC=%.6f BUCKET_ERROR=%.6f MAE=%.6f " "RMSE=%.6f Actural_CTR=%.6f Predicted_CTR=%.6f " - "COPC=%.6f MEAN Q_VALUE=%.6f Ins number=%s" % - (print_prefix, auc, bucket_error, mae, rmse, actual_ctr, - predicted_ctr, copc, mean_predict_qvalue, total_ins_num)) + "COPC=%.6f MEAN Q_VALUE=%.6f Ins number=%s" + % ( + print_prefix, + auc, + bucket_error, + mae, + rmse, + actual_ctr, + predicted_ctr, + copc, + mean_predict_qvalue, + total_ins_num, + ) + ) def program_type_trans(self, prog_dir, prog_fn, is_text): return utils.program_type_trans(prog_dir, prog_fn, is_text) - def draw_from_program_file(self, model_filename, is_text, output_dir, - output_filename): + def draw_from_program_file( + self, model_filename, is_text, output_dir, output_filename + ): """draw program from file""" program = utils.load_program(model_filename, is_text) utils.graphviz(program.global_block(), output_dir, output_filename) @@ -1554,14 +1670,17 @@ class FleetUtil(object): utils.graphviz(program.global_block(), output_dir, output_name) def check_two_programs(self, config): - train_prog = utils.load_program(config.train_prog_path, - config.is_text_train_program) - pruned_prog = utils.load_program(config.pruned_prog_path, - config.is_text_pruned_program) + train_prog = utils.load_program( + config.train_prog_path, config.is_text_train_program + ) + pruned_prog = utils.load_program( + config.pruned_prog_path, config.is_text_pruned_program + ) if config.draw: pruned_dir = os.path.dirname(config.pruned_prog_path) - self.draw_from_program(pruned_prog, pruned_dir, - config.draw_out_name) + self.draw_from_program( + pruned_prog, pruned_dir, config.draw_out_name + ) res = utils.check_pruned_program_vars(train_prog, pruned_prog) if res: _logger.info("check_programs succeed.") @@ -1574,9 +1693,14 @@ class FleetUtil(object): def check_vars_and_dump(self, config): _logger.info("start check_vars_and_dump.") results = utils.check_saved_vars_try_dump( - config.dump_model_dir, config.dump_program_filename, - config.is_text_dump_program, config.feed_config, - config.fetch_config, config.batch_size, config.save_params_filename) + config.dump_model_dir, + config.dump_program_filename, + config.is_text_dump_program, + config.feed_config, + config.fetch_config, + config.batch_size, + config.save_params_filename, + ) _logger.info("check_vars_and_dump succeed.") return results @@ -1607,8 +1731,9 @@ class FleetUtil(object): utils.parse_program(program, output_dir) def _is_optimizer_op(self, op): - return self.op_role_key in op.attr_names and \ - int(op.all_attrs()[self.op_role_key]) & int(OpRole.Optimize) + return self.op_role_key in op.attr_names and int( + op.all_attrs()[self.op_role_key] + ) & int(OpRole.Optimize) def split_program_by_device(self, program): ops_list = [] @@ -1619,8 +1744,11 @@ class FleetUtil(object): if self._is_optimizer_op(op): break if op.has_attr("op_device"): - cur_attr = op.attr( - "op_device") if op.attr("op_device") != "" else type_cpu + cur_attr = ( + op.attr("op_device") + if op.attr("op_device") != "" + else type_cpu + ) if pre is None or pre != cur_attr: ops_list.append([]) type_list.append(cur_attr) @@ -1710,8 +1838,9 @@ class FleetUtil(object): send_list[i].extend(list(in_from_pre[i + 1])) prog = program.clone() if merged_type_list[i] != type_cpu: - prog = prog._prune_with_input(list(in_from_pre[i]), - list(send_list[i])) + prog = prog._prune_with_input( + list(in_from_pre[i]), list(send_list[i]) + ) program_list.append(prog) else: program_list.append(prog) @@ -1729,8 +1858,13 @@ class FleetUtil(object): print("warning: non heter program") return None else: - return [start_list[heter_index], end_list[heter_index], send_list[heter_index], \ - recv_list[heter_index], program_list[heter_index]] + return [ + start_list[heter_index], + end_list[heter_index], + send_list[heter_index], + recv_list[heter_index], + program_list[heter_index], + ] class GPUPSUtil(FleetUtil): @@ -1921,12 +2055,14 @@ class GPUPSUtil(FleetUtil): os.remove("donefile.txt") return [last_save_day, last_save_pass, last_path, xbox_base_key] - def write_model_donefile(self, - output_path, - day, - pass_id, - xbox_base_key, - donefile_name="donefile.txt"): + def write_model_donefile( + self, + output_path, + day, + pass_id, + xbox_base_key, + donefile_name="donefile.txt", + ): """ write donefile when save model @@ -1965,8 +2101,13 @@ class GPUPSUtil(FleetUtil): if fleet.worker_index() == 0: donefile_path = output_path + "/" + donefile_name - content = "%s\t%lu\t%s\t%s\t%d" % (day, xbox_base_key,\ - model_path, pass_id, 0) + content = "%s\t%lu\t%s\t%s\t%d" % ( + day, + xbox_base_key, + model_path, + pass_id, + 0, + ) if self._afs.is_file(donefile_path): self._afs.download(donefile_path, donefile_name) pre_content = "" @@ -1978,8 +2119,9 @@ class GPUPSUtil(FleetUtil): os.remove(donefile_name) exist = False for i in range(len(day_list)): - if int(day) == int(day_list[i]) and \ - int(pass_id) == int(pass_list[i]): + if int(day) == int(day_list[i]) and int(pass_id) == int( + pass_list[i] + ): exist = True break if not exist: @@ -1988,29 +2130,35 @@ class GPUPSUtil(FleetUtil): f.write(content + "\n") self._afs.delete(donefile_path) self._afs.upload(donefile_name, donefile_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) else: - self.rank0_error("not write %s because %s/%s already " - "exists" % (donefile_name, day, pass_id)) + self.rank0_error( + "not write %s because %s/%s already " + "exists" % (donefile_name, day, pass_id) + ) else: with open(donefile_name, "w") as f: f.write(content + "\n") self._afs.upload(donefile_name, donefile_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) - - def write_xbox_donefile(self, - output_path, - day, - pass_id, - xbox_base_key, - data_path, - hadoop_fs_name, - hadoop_fs_ugi, - monitor_data={}, - hadoop_home="$HADOOP_HOME", - donefile_name=None): + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) + + def write_xbox_donefile( + self, + output_path, + day, + pass_id, + xbox_base_key, + data_path, + hadoop_fs_name, + hadoop_fs_ugi, + monitor_data={}, + hadoop_home="$HADOOP_HOME", + donefile_name=None, + ): """ write delta donefile or xbox base donefile @@ -2063,9 +2211,16 @@ class GPUPSUtil(FleetUtil): data_path = ",".join(data_path) if fleet.worker_index() == 0: donefile_path = output_path + "/" + donefile_name - xbox_str = self._get_xbox_str(output_path, day, model_path, \ - xbox_base_key, data_path, hadoop_fs_name, monitor_data={}, - mode=mode) + xbox_str = self._get_xbox_str( + output_path, + day, + model_path, + xbox_base_key, + data_path, + hadoop_fs_name, + monitor_data={}, + mode=mode, + ) if self._afs.is_exist(donefile_path): self.rank0_info("exist %s succeed" % (donefile_path)) @@ -2080,9 +2235,11 @@ class GPUPSUtil(FleetUtil): os.remove(donefile_name) self.rank0_info("remove %s succeed" % (donefile_name)) exist = False - if int(day) < int(last_day) or \ - int(day) == int(last_day) and \ - int(pass_id) <= int(last_pass): + if ( + int(day) < int(last_day) + or int(day) == int(last_day) + and int(pass_id) <= int(last_pass) + ): exist = True if not exist: with open(donefile_name, "w") as f: @@ -2090,25 +2247,31 @@ class GPUPSUtil(FleetUtil): f.write(xbox_str + "\n") self._afs.delete(donefile_path) self._afs.upload(donefile_name, donefile_path) - self.rank0_info("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_info( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) else: - self.rank0_info("not write %s because %s/%s already " - "exists" % (donefile_name, day, pass_id)) + self.rank0_info( + "not write %s because %s/%s already " + "exists" % (donefile_name, day, pass_id) + ) else: with open(donefile_name, "w") as f: f.write(xbox_str + "\n") self._afs.upload(donefile_name, donefile_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) - - def write_cache_donefile(self, - output_path, - day, - pass_id, - key_num, - donefile_name="sparse_cache.meta", - **kwargs): + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) + + def write_cache_donefile( + self, + output_path, + day, + pass_id, + key_num, + donefile_name="sparse_cache.meta", + **kwargs + ): """ write cache donefile @@ -2154,25 +2317,30 @@ class GPUPSUtil(FleetUtil): donefile_path = model_path + "/" + donefile_name if self._afs.is_file(donefile_path): - self.rank0_error( \ - "not write because %s already exists" % donefile_path) + self.rank0_error( + "not write because %s already exists" % donefile_path + ) else: - meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" \ - % (file_num, key_num) + meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" % ( + file_num, + key_num, + ) with open(donefile_name, "w") as f: f.write(meta_str) self._afs.upload(donefile_name, donefile_path) self.rank0_error("write %s succeed" % donefile_path) - def _get_xbox_str(self, - output_path, - day, - model_path, - xbox_base_key, - data_path, - hadoop_fs_name, - monitor_data={}, - mode="patch"): + def _get_xbox_str( + self, + output_path, + day, + model_path, + xbox_base_key, + data_path, + hadoop_fs_name, + monitor_data={}, + mode="patch", + ): xbox_dict = collections.OrderedDict() if mode == "base": xbox_dict["id"] = str(xbox_base_key) @@ -2184,7 +2352,7 @@ class GPUPSUtil(FleetUtil): xbox_dict["id"] = str(int(time.time())) xbox_dict["key"] = str(xbox_base_key) if model_path.startswith("hdfs:") or model_path.startswith("afs:"): - model_path = model_path[model_path.find(":") + 1:] + model_path = model_path[model_path.find(":") + 1 :] xbox_dict["input"] = hadoop_fs_name + model_path.rstrip("/") + "/000" xbox_dict["record_count"] = "111111" xbox_dict["partition_type"] = "2" @@ -2194,7 +2362,8 @@ class GPUPSUtil(FleetUtil): xbox_dict["job_id"] = os.environ.get("PADDLE_JOB_ID", "") # currently hard code here, set monitor_data empty string xbox_dict["monitor_data"] = "" - xbox_dict["monitor_path"] = output_path.rstrip("/") + "/monitor/" \ - + day + ".txt" + xbox_dict["monitor_path"] = ( + output_path.rstrip("/") + "/monitor/" + day + ".txt" + ) xbox_dict["mpi_size"] = str(fleet.worker_num()) return json.dumps(xbox_dict) diff --git a/python/paddle/fluid/incubate/fleet/utils/hdfs.py b/python/paddle/fluid/incubate/fleet/utils/hdfs.py index 41aff381959..c35e266357e 100644 --- a/python/paddle/fluid/incubate/fleet/utils/hdfs.py +++ b/python/paddle/fluid/incubate/fleet/utils/hdfs.py @@ -24,8 +24,17 @@ import copy import errno import time import logging -#from . import fs -from paddle.distributed.fleet.utils.fs import FS, LocalFS, FSFileExistsError, FSFileNotExistsError, ExecuteError, FSTimeOut, FSShellCmdAborted + +# from . import fs +from paddle.distributed.fleet.utils.fs import ( + FS, + LocalFS, + FSFileExistsError, + FSFileNotExistsError, + ExecuteError, + FSTimeOut, + FSShellCmdAborted, +) from paddle.fluid import core import functools @@ -35,9 +44,7 @@ __all__ = ["HDFSClient"] def _handle_errors(max_time_out=None): - def decorator(f): - @functools.wraps(f) def handler(*args, **kwargs): o = args[0] @@ -53,19 +60,23 @@ def _handle_errors(max_time_out=None): while True: try: return f(*args, **kwargs) - #important: only ExecuteError need to retry + # important: only ExecuteError need to retry except ExecuteError as e: if time.time() - start >= time_out: - raise FSTimeOut("args:{} timeout:{}".format( - args, - time.time() - start)) + raise FSTimeOut( + "args:{} timeout:{}".format( + args, time.time() - start + ) + ) time.sleep(inter) if time.time() - last_print_time > 30: - print("hadoop operator timeout:args:{} timeout:{}".format( - args, - time.time() - start)) + print( + "hadoop operator timeout:args:{} timeout:{}".format( + args, time.time() - start + ) + ) last_print_time = time.time() return handler @@ -74,13 +85,13 @@ def _handle_errors(max_time_out=None): class HDFSClient(FS): - def __init__( - self, - hadoop_home, - configs, - time_out=5 * 60 * 1000, #ms - sleep_inter=1000): #ms + self, + hadoop_home, + configs, + time_out=5 * 60 * 1000, # ms + sleep_inter=1000, + ): # ms # Raise exception if JAVA_HOME not exists. self.pre_commands = [] @@ -98,7 +109,8 @@ class HDFSClient(FS): self._sleep_inter = sleep_inter self._base_cmd = " ".join(self.pre_commands) self._bd_err_re = re.compile( - r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:') + r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:' + ) def _run_cmd(self, cmd, redirect_stderr=False): exe_cmd = "{} -{}".format(self._base_cmd, cmd) @@ -269,7 +281,8 @@ class HDFSClient(FS): if test_exists: if not self.is_exist(fs_src_path): raise FSFileNotExistsError( - "{} is not exists".format(fs_src_path)) + "{} is not exists".format(fs_src_path) + ) if self.is_exist(fs_dst_path): raise FSFileExistsError("{} exists already".format(fs_dst_path)) @@ -285,8 +298,7 @@ class HDFSClient(FS): if ret != 0: raise ExecuteError(cmd) except Exception as e: - if not self.is_exist(fs_src_path) and \ - self.is_exist(fs_dst_path): + if not self.is_exist(fs_src_path) and self.is_exist(fs_dst_path): return raise e diff --git a/python/paddle/fluid/incubate/fleet/utils/http_server.py b/python/paddle/fluid/incubate/fleet/utils/http_server.py index 685228f0749..269f912073e 100644 --- a/python/paddle/fluid/incubate/fleet/utils/http_server.py +++ b/python/paddle/fluid/incubate/fleet/utils/http_server.py @@ -14,6 +14,7 @@ """Http Server.""" import logging + # NOTE: HTTPServer has a different name in python2 and python3 from http.server import HTTPServer import http.server as SimpleHTTPServer @@ -32,9 +33,9 @@ def get_logger(name, level, fmt): return logger -_http_server_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_http_server_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): @@ -164,7 +165,8 @@ class KVServer: start server until user calls stop to let it quit. """ self.listen_thread = threading.Thread( - target=lambda: self.http_server.serve_forever()) + target=lambda: self.http_server.serve_forever() + ) self.listen_thread.start() def stop(self): diff --git a/python/paddle/fluid/incubate/fleet/utils/utils.py b/python/paddle/fluid/incubate/fleet/utils/utils.py index c154fcb2f1d..c675fea39bc 100644 --- a/python/paddle/fluid/incubate/fleet/utils/utils.py +++ b/python/paddle/fluid/incubate/fleet/utils/utils.py @@ -28,9 +28,13 @@ from paddle.fluid.framework import Program from paddle.fluid.proto import framework_pb2 __all__ = [ - "load_program", "save_program", "program_type_trans", - "check_saved_vars_try_dump", "parse_program", "check_pruned_program_vars", - "graphviz" + "load_program", + "save_program", + "program_type_trans", + "check_saved_vars_try_dump", + "parse_program", + "check_pruned_program_vars", + "graphviz", ] logger = logging.getLogger(__name__) @@ -45,7 +49,8 @@ all_vars_out_fn = "vars_all.log" ops_out_fn = "ops.log" feed_fetch_type_list = [ - core.VarDesc.VarType.FEED_MINIBATCH, core.VarDesc.VarType.FETCH_LIST + core.VarDesc.VarType.FEED_MINIBATCH, + core.VarDesc.VarType.FETCH_LIST, ] not_expected_op_types = ["lookup_table"] @@ -85,12 +90,16 @@ def save_program(program, model_filename='__model__', is_text=False): def check_pruned_program_vars(train_prog, pruned_prog): is_match = True - pruned_vars = [(v.name, v) for v in pruned_prog.list_vars() - if fluid.io.is_persistable(v)] + pruned_vars = [ + (v.name, v) + for v in pruned_prog.list_vars() + if fluid.io.is_persistable(v) + ] pruned_vars = OrderedDict(pruned_vars) pruned_vars_name = [name for name in pruned_vars] logger.info( - "persistable vars in pruned program: {}".format(pruned_vars_name)) + "persistable vars in pruned program: {}".format(pruned_vars_name) + ) for var_name in pruned_vars: var = pruned_vars[var_name] @@ -102,14 +111,23 @@ def check_pruned_program_vars(train_prog, pruned_prog): except ValueError as e: logger.error( "not find variable '%s' in train program. please check pruning." - % var_name) + % var_name + ) logger.error(e) continue - if var.shape != train_prog_var.shape or var.dtype != train_prog_var.dtype: + if ( + var.shape != train_prog_var.shape + or var.dtype != train_prog_var.dtype + ): logger.error( - "variable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}" - .format(var_name, var.shape, var.dtype, train_prog_var.shape, - train_prog_var.dtype)) + "variable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}".format( + var_name, + var.shape, + var.dtype, + train_prog_var.shape, + train_prog_var.dtype, + ) + ) is_match = False return is_match @@ -119,10 +137,12 @@ def graphviz(block, output_dir="", filename='debug'): pdf_path = os.path.join(output_dir, filename + '.pdf') debugger.draw_block_graphviz(block, path=dot_path) cmd = ["dot", "-Tpdf", dot_path, "-o", pdf_path] - p = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) p.wait() @@ -134,17 +154,18 @@ def program_type_trans(prog_dir, prog_fn, is_text): def append_save_op(block, var, path): - block.append_op(type='save', - inputs={'X': [var]}, - outputs={}, - attrs={'file_path': path}) + block.append_op( + type='save', inputs={'X': [var]}, outputs={}, attrs={'file_path': path} + ) def append_load_op(block, var, path): - block.append_op(type='load', - inputs={}, - outputs={'Out': [var]}, - attrs={'file_path': path}) + block.append_op( + type='load', + inputs={}, + outputs={'Out': [var]}, + attrs={'file_path': path}, + ) def save_var(np_array, var_name, shape_list, dtype, save_path): @@ -199,19 +220,34 @@ def feed_gen(batch_size, feeded_vars_dims, feeded_vars_filelist): return batch_feed -def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, - batch_size, feed_config, fetch_config, save_filename, - saved_params): +def try_load_model_vars( + dump_dir, + dump_prog_fn, + is_text_dump_program, + batch_size, + feed_config, + fetch_config, + save_filename, + saved_params, +): place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.core.Scope() with fluid.scope_guard(scope): if is_text_dump_program: - dump_prog_fn = program_type_trans(dump_dir, dump_prog_fn, - is_text_dump_program) - inference_program, feed_target_names, fetch_targets = \ - fluid.io.load_inference_model(dump_dir, exe, model_filename=dump_prog_fn, - params_filename=save_filename) + dump_prog_fn = program_type_trans( + dump_dir, dump_prog_fn, is_text_dump_program + ) + ( + inference_program, + feed_target_names, + fetch_targets, + ) = fluid.io.load_inference_model( + dump_dir, + exe, + model_filename=dump_prog_fn, + params_filename=save_filename, + ) # check program vars and saved vars shape orig_para_shape = { @@ -222,13 +258,17 @@ def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, var_temp = fluid.global_scope().find_var(each_var.name) assert var_temp != None, "can't not find var: " + each_var.name new_shape = (np.array(var_temp.get_tensor())).shape - assert each_var.name in orig_para_shape, each_var.name + "MUST in var list" + assert each_var.name in orig_para_shape, ( + each_var.name + "MUST in var list" + ) orig_shape = orig_para_shape.get(each_var.name) if new_shape != orig_shape: raise RuntimeError( "Shape not matching: the Program requires a parameter with a shape of ({}), " - "while the loaded parameter (namely [ {} ]) has a shape of ({})." - .format(orig_shape, each_var.name, new_shape)) + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_shape, each_var.name, new_shape + ) + ) # check feed/fetch vars in program and config fetch_targets_names = [v.name for v in fetch_targets] @@ -238,10 +278,15 @@ def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, logger.warning("no fetch targets in program.") fetch_list = fetch_targets feed_name_list = feed_target_names - if feed_config.feeded_vars_names is not None and feed_target_names != feed_config.feeded_vars_names: + if ( + feed_config.feeded_vars_names is not None + and feed_target_names != feed_config.feeded_vars_names + ): logger.warning( - "feed vars in program and config are diff: feed in program: {}. feed in config {}." - .format(feed_target_names, feed_config.feeded_vars_names)) + "feed vars in program and config are diff: feed in program: {}. feed in config {}.".format( + feed_target_names, feed_config.feeded_vars_names + ) + ) feed_name_list = feed_config.feeded_vars_names # remove feed op in inference_program. new feed op will be added in exe.run global_block = inference_program.global_block() @@ -252,10 +297,15 @@ def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, need_to_remove_op_index.append(i) for index in need_to_remove_op_index[::-1]: global_block._remove_op(index) - if fetch_config.fetch_vars_names is not None and fetch_targets_names != fetch_config.fetch_vars_names: + if ( + fetch_config.fetch_vars_names is not None + and fetch_targets_names != fetch_config.fetch_vars_names + ): logger.warning( - "fetch vars in program and config are diff: fetch in program: {}. fetch in config {}." - .format(fetch_targets_names, fetch_config.fetch_vars_names)) + "fetch vars in program and config are diff: fetch in program: {}. fetch in config {}.".format( + fetch_targets_names, fetch_config.fetch_vars_names + ) + ) fetch_list = [ inference_program.global_block().var(i) for i in fetch_config.fetch_vars_names @@ -275,70 +325,101 @@ def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, # try dump fetch_targets feed_tensors = [] - assert len(feed_config.feeded_vars_names) == len( - feed_config.feeded_vars_dims) == len(feed_config.feeded_vars_types) + assert ( + len(feed_config.feeded_vars_names) + == len(feed_config.feeded_vars_dims) + == len(feed_config.feeded_vars_types) + ) # check program vars and feed tensor shape in config for i in range(len(feed_config.feeded_vars_names)): var = inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) if not isinstance(feed_config.feeded_vars_dims[i], (list, tuple)): - tensor_shape = (feed_config.feeded_vars_dims[i], ) + tensor_shape = (feed_config.feeded_vars_dims[i],) else: tensor_shape = tuple(feed_config.feeded_vars_dims[i]) feed_config.feeded_vars_dims[i] = tensor_shape var_shape = var.shape[1:] if tensor_shape != var_shape: raise RuntimeError( - "feed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}" - .format(feed_config.feeded_vars_names[i], var_shape, - tensor_shape)) + "feed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}".format( + feed_config.feeded_vars_names[i], + var_shape, + tensor_shape, + ) + ) if not feed_config.feeded_vars_filelist: logger.info("generate random feed vars.") for i in range(len(feed_config.feeded_vars_names)): var = inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) # create fake feed tensor. if lod_level > 1, should create_lod_tensor() if var.lod_level == 0: feed_tensors.append( - np.array(np.random.random( - tuple([batch_size] + - list(feed_config.feeded_vars_dims[i]))), - dtype=feed_config.feeded_vars_types[i])) + np.array( + np.random.random( + tuple( + [batch_size] + + list(feed_config.feeded_vars_dims[i]) + ) + ), + dtype=feed_config.feeded_vars_types[i], + ) + ) elif var.lod_level == 1: - t = np.array(np.random.random( - tuple([batch_size] + - list(feed_config.feeded_vars_dims[i]))), - dtype=feed_config.feeded_vars_types[i]) + t = np.array( + np.random.random( + tuple( + [batch_size] + + list(feed_config.feeded_vars_dims[i]) + ) + ), + dtype=feed_config.feeded_vars_types[i], + ) feed_tensors.append( - fluid.create_lod_tensor(t, [[1] * batch_size], place)) + fluid.create_lod_tensor(t, [[1] * batch_size], place) + ) else: raise RuntimeError( "vars with lod_level >= 2 is not supported now in this infer program check tool." ) - results = exe.run(inference_program, - feed={ - name: feed_tensors[i] - for i, name in enumerate(feed_name_list) - }, - fetch_list=fetch_list, - return_numpy=return_numpy) + results = exe.run( + inference_program, + feed={ + name: feed_tensors[i] + for i, name in enumerate(feed_name_list) + }, + fetch_list=fetch_list, + return_numpy=return_numpy, + ) else: - logger.info("load feed vars from files: {}.".format( - feed_config.feeded_vars_filelist)) + logger.info( + "load feed vars from files: {}.".format( + feed_config.feeded_vars_filelist + ) + ) feed_vars = [ inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) for i in range(len(feed_config.feeded_vars_names)) ] feeder = fluid.DataFeeder(feed_list=feed_vars, place=place) - batch_feed = feed_gen(batch_size, feed_config.feeded_vars_dims, - feed_config.feeded_vars_filelist) + batch_feed = feed_gen( + batch_size, + feed_config.feeded_vars_dims, + feed_config.feeded_vars_filelist, + ) slots = [batch_feed] - results = exe.run(inference_program, - feed=feeder.feed(slots), - fetch_list=fetch_list, - return_numpy=return_numpy) + results = exe.run( + inference_program, + feed=feeder.feed(slots), + fetch_list=fetch_list, + return_numpy=return_numpy, + ) for i, v in enumerate(fetch_list): logger.info("fetch_targets name: %s" % v.name) logger.info("fetch_targets: {}".format(results[i])) @@ -350,31 +431,46 @@ def check_not_expected_ops(prog): for op in prog.global_block().ops: if op.type in not_expected_op_types and op.type not in op_types_set: logger.warning( - "find op type '{}' in program, please check if your program is pruned correctly !" - .format(op.type)) + "find op type '{}' in program, please check if your program is pruned correctly !".format( + op.type + ) + ) op_types_set.add(op.type) -def check_saved_vars_try_dump(dump_dir, - dump_prog_fn, - is_text_dump_program, - feed_config, - fetch_config, - batch_size=1, - save_filename=None): - dump_prog = load_program(os.path.join(dump_dir, dump_prog_fn), - is_text_dump_program) +def check_saved_vars_try_dump( + dump_dir, + dump_prog_fn, + is_text_dump_program, + feed_config, + fetch_config, + batch_size=1, + save_filename=None, +): + dump_prog = load_program( + os.path.join(dump_dir, dump_prog_fn), is_text_dump_program + ) saved_params = [ v for v in dump_prog.list_vars() if fluid.io.is_persistable(v) ] - logger.info("persistable vars in dump program: {}".format( - [v.name for v in saved_params])) + logger.info( + "persistable vars in dump program: {}".format( + [v.name for v in saved_params] + ) + ) check_not_expected_ops(dump_prog) - return try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, - batch_size, feed_config, fetch_config, - save_filename, saved_params) + return try_load_model_vars( + dump_dir, + dump_prog_fn, + is_text_dump_program, + batch_size, + feed_config, + fetch_config, + save_filename, + saved_params, + ) def parse_program(program, output_dir): @@ -383,13 +479,16 @@ def parse_program(program, output_dir): persistable_vars = [ v for v in program.list_vars() if fluid.io.is_persistable(v) ] - output["persistable_vars"] = [{ - 'name': str(v.name), - 'shape': str(v.shape), - 'lod_level': int(v.lod_level), - 'dtype': str(v.dtype), - 'type': str(v.type) - } for v in persistable_vars] + output["persistable_vars"] = [ + { + 'name': str(v.name), + 'shape': str(v.shape), + 'lod_level': int(v.lod_level), + 'dtype': str(v.dtype), + 'type': str(v.type), + } + for v in persistable_vars + ] with open(os.path.join(output_dir, persistable_vars_out_fn), 'w') as f: f.write("persistable vars:\n") for var in output["persistable_vars"]: @@ -398,15 +497,17 @@ def parse_program(program, output_dir): # all vars all_vars = [v for v in program.list_vars()] - output["all_vars"] = [{ - 'name': str(v.name), - 'shape': str(v.shape), - 'lod_level': int(v.lod_level), - 'dtype': str(v.dtype) - } if v.type not in feed_fetch_type_list else { - 'name': str(v.name), - 'type': str(v.type) - } for v in all_vars] + output["all_vars"] = [ + { + 'name': str(v.name), + 'shape': str(v.shape), + 'lod_level': int(v.lod_level), + 'dtype': str(v.dtype), + } + if v.type not in feed_fetch_type_list + else {'name': str(v.name), 'type': str(v.type)} + for v in all_vars + ] with open(os.path.join(output_dir, all_vars_out_fn), 'w') as f: f.write("all vars:\n") for var in output["all_vars"]: @@ -415,11 +516,14 @@ def parse_program(program, output_dir): # ops ops = program.global_block().ops - output["ops"] = [{ - 'type': op.type, - 'input_arg_names': str(op.input_arg_names), - 'output_arg_names': str(op.output_arg_names) - } for op in ops] + output["ops"] = [ + { + 'type': op.type, + 'input_arg_names': str(op.input_arg_names), + 'output_arg_names': str(op.output_arg_names), + } + for op in ops + ] with open(os.path.join(output_dir, ops_out_fn), 'w') as f: f.write("ops:\n") for op in output["ops"]: diff --git a/python/paddle/fluid/inference/__init__.py b/python/paddle/fluid/inference/__init__.py index d6b8b102487..51127de403f 100644 --- a/python/paddle/fluid/inference/__init__.py +++ b/python/paddle/fluid/inference/__init__.py @@ -12,7 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .wrapper import Config, DataType, PlaceType, PrecisionType, Tensor, Predictor +from .wrapper import ( + Config, + DataType, + PlaceType, + PrecisionType, + Tensor, + Predictor, +) from .wrapper import convert_to_mixed_precision -from ..core import create_predictor, get_version, _get_phi_kernel_name, get_num_bytes_of_data_type, PredictorPool, get_trt_compile_version, get_trt_runtime_version +from ..core import ( + create_predictor, + get_version, + _get_phi_kernel_name, + get_num_bytes_of_data_type, + PredictorPool, + get_trt_compile_version, + get_trt_runtime_version, +) diff --git a/python/paddle/fluid/inference/wrapper.py b/python/paddle/fluid/inference/wrapper.py index 83811012e52..ffad69335a4 100644 --- a/python/paddle/fluid/inference/wrapper.py +++ b/python/paddle/fluid/inference/wrapper.py @@ -33,8 +33,9 @@ def tensor_copy_from_cpu(self, data): ''' Support input type check based on tensor.copy_from_cpu. ''' - if isinstance(data, np.ndarray) or (isinstance(data, list) and len(data) > 0 - and isinstance(data[0], str)): + if isinstance(data, np.ndarray) or ( + isinstance(data, list) and len(data) > 0 and isinstance(data[0], str) + ): self.copy_from_cpu_bind(data) else: raise TypeError( @@ -50,17 +51,20 @@ def tensor_share_external_data(self, data): self.share_external_data_bind(data) else: raise TypeError( - "In share_external_data, we only support LoDTensor data type.") + "In share_external_data, we only support LoDTensor data type." + ) -def convert_to_mixed_precision(model_file: str, - params_file: str, - mixed_model_file: str, - mixed_params_file: str, - mixed_precision: PrecisionType, - backend: PlaceType, - keep_io_types: bool = True, - black_list: Set = set()): +def convert_to_mixed_precision( + model_file: str, + params_file: str, + mixed_model_file: str, + mixed_params_file: str, + mixed_precision: PrecisionType, + backend: PlaceType, + keep_io_types: bool = True, + black_list: Set = set(), +): ''' Convert a fp32 model to mixed precision model. @@ -80,9 +84,16 @@ def convert_to_mixed_precision(model_file: str, os.makedirs(mixed_model_dirname) if not os.path.exists(mixed_params_dirname): os.makedirs(mixed_params_dirname) - convert_to_mixed_precision_bind(model_file, params_file, mixed_model_file, - mixed_params_file, mixed_precision, backend, - keep_io_types, black_list) + convert_to_mixed_precision_bind( + model_file, + params_file, + mixed_model_file, + mixed_params_file, + mixed_precision, + backend, + keep_io_types, + black_list, + ) Tensor.copy_from_cpu = tensor_copy_from_cpu diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 540ea40833d..3e6c9ef3674 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -16,7 +16,13 @@ import math import functools from . import framework from . import core -from .framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph, default_main_program, _current_expected_place +from .framework import ( + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, + default_main_program, + _current_expected_place, +) from .lazy_init import lazy_init_helper from .framework import program_guard import numpy as np @@ -27,10 +33,22 @@ from paddle import _C_ops, _legacy_C_ops import paddle __all__ = [ - 'Constant', 'Uniform', 'Normal', 'TruncatedNormal', 'Xavier', 'Bilinear', - 'MSRA', 'ConstantInitializer', 'UniformInitializer', 'NormalInitializer', - 'TruncatedNormalInitializer', 'XavierInitializer', 'BilinearInitializer', - 'MSRAInitializer', 'NumpyArrayInitializer', 'set_global_initializer' + 'Constant', + 'Uniform', + 'Normal', + 'TruncatedNormal', + 'Xavier', + 'Bilinear', + 'MSRA', + 'ConstantInitializer', + 'UniformInitializer', + 'NormalInitializer', + 'TruncatedNormalInitializer', + 'XavierInitializer', + 'BilinearInitializer', + 'MSRAInitializer', + 'NumpyArrayInitializer', + 'set_global_initializer', ] _global_weight_initializer_ = None @@ -56,8 +74,7 @@ class Initializer(object): return self._lazy_init(param, block) def forward(self, param, block=None): - """Add corresponding initialization operations to the network - """ + """Add corresponding initialization operations to the network""" raise NotImplementedError() def _lazy_init(self, param, block=None): @@ -74,8 +91,9 @@ class Initializer(object): # Add hook function for initializing param in dygraph mode param.set_init_func(functools.partial(self.forward, param, block)) - param._init_op_creator = functools.partial(init_op_creator, - self.forward, param) + param._init_op_creator = functools.partial( + init_op_creator, self.forward, param + ) return param @@ -160,35 +178,47 @@ class ConstantInitializer(Initializer): """ block = self._check_block(block) - assert (isinstance(var, framework.Variable) - or isinstance(var, framework.EagerParamBase)) + assert isinstance(var, framework.Variable) or isinstance( + var, framework.EagerParamBase + ) assert isinstance(block, framework.Block) if in_dygraph_mode(): place = _current_expected_place() if self._force_cpu: place = core.CPUPlace() - _C_ops.full_(var, var.shape, str(float(self._value)), var.dtype, - place) + _C_ops.full_( + var, var.shape, str(float(self._value)), var.dtype, place + ) return None elif _in_legacy_dygraph(): - _legacy_C_ops.fill_constant(var, 'value', float(self._value), - 'force_cpu', self._force_cpu, 'dtype', - int(var.dtype), 'str_value', - str(float(self._value)), 'shape', - var.shape) + _legacy_C_ops.fill_constant( + var, + 'value', + float(self._value), + 'force_cpu', + self._force_cpu, + 'dtype', + int(var.dtype), + 'str_value', + str(float(self._value)), + 'shape', + var.shape, + ) return None else: - op = block.append_op(type="fill_constant", - outputs={"Out": var}, - attrs={ - "shape": var.shape, - "dtype": int(var.dtype), - "value": float(self._value), - 'str_value': str(float(self._value)), - 'force_cpu': self._force_cpu - }, - stop_gradient=True) + op = block.append_op( + type="fill_constant", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "dtype": int(var.dtype), + "value": float(self._value), + 'str_value': str(float(self._value)), + 'force_cpu': self._force_cpu, + }, + stop_gradient=True, + ) var.op = op return op @@ -214,16 +244,12 @@ class UniformInitializer(Initializer): import paddle.fluid as fluid x = fluid.data(name='x', shape=[None, 1], dtype='float32') fc = fluid.layers.fc(input=x, size=10, - param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5)) + param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5)) """ - def __init__(self, - low=-1.0, - high=1.0, - seed=0, - diag_num=0, - diag_step=0, - diag_val=1.0): + def __init__( + self, low=-1.0, high=1.0, seed=0, diag_num=0, diag_step=0, diag_val=1.0 + ): assert low is not None assert high is not None assert high >= low @@ -232,7 +258,7 @@ class UniformInitializer(Initializer): assert diag_step is not None assert diag_val is not None if diag_num > 0 or diag_step > 0: - assert (diag_num > 0 and diag_step > 0) + assert diag_num > 0 and diag_step > 0 super(UniformInitializer, self).__init__() self._low = low self._high = high @@ -255,9 +281,12 @@ class UniformInitializer(Initializer): block = self._check_block(block) assert isinstance(block, framework.Block) - check_variable_and_dtype(var, "Out", - ["uint16", "float16", "float32", "float64"], - "uniform_random") + check_variable_and_dtype( + var, + "Out", + ["uint16", "float16", "float32", "float64"], + "uniform_random", + ) if self._seed == 0: self._seed = block.program.random_seed @@ -265,62 +294,88 @@ class UniformInitializer(Initializer): # to be compatible of fp16 initializers if var.dtype == VarDesc.VarType.FP16: out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['uniform_random', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['uniform_random', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var if framework._non_static_mode(): if in_dygraph_mode(): - out_var = _C_ops.uniform_random(var.shape, out_dtype, self._low, - self._high, self._seed, - _current_expected_place()) + out_var = _C_ops.uniform_random( + var.shape, + out_dtype, + self._low, + self._high, + self._seed, + _current_expected_place(), + ) elif _in_legacy_dygraph(): out_var = _legacy_C_ops.uniform_random( - 'shape', var.shape, 'min', self._low, 'max', self._high, - 'seed', self._seed, 'dtype', out_dtype, 'diag_num', - self._diag_num, 'diag_step', self._diag_step, 'diag_val', - self._diag_val) + 'shape', + var.shape, + 'min', + self._low, + 'max', + self._high, + 'seed', + self._seed, + 'dtype', + out_dtype, + 'diag_num', + self._diag_num, + 'diag_step', + self._diag_step, + 'diag_val', + self._diag_val, + ) if var.dtype == VarDesc.VarType.FP16: if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type="uniform_random", - inputs={}, - outputs={"Out": out_var}, - attrs={ - "shape": var.shape, - "dtype": out_dtype, - "min": self._low, - "max": self._high, - "seed": self._seed, - "diag_num": self._diag_num, - "diag_step": self._diag_step, - "diag_val": self._diag_val - }, - stop_gradient=True) + op = block.append_op( + type="uniform_random", + inputs={}, + outputs={"Out": out_var}, + attrs={ + "shape": var.shape, + "dtype": out_dtype, + "min": self._low, + "max": self._high, + "seed": self._seed, + "diag_num": self._diag_num, + "diag_step": self._diag_step, + "diag_val": self._diag_val, + }, + stop_gradient=True, + ) if var.dtype == VarDesc.VarType.FP16: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -368,40 +423,61 @@ class NormalInitializer(Initializer): assert isinstance(block, framework.Block) - check_variable_and_dtype(var, "Out", - ["uint16", "float16", "float32", "float64"], - "guassian_random") + check_variable_and_dtype( + var, + "Out", + ["uint16", "float16", "float32", "float64"], + "guassian_random", + ) if self._seed == 0: self._seed = block.program.random_seed if in_dygraph_mode(): place = _current_expected_place() - out_var = _C_ops.gaussian_random(var.shape, self._mean, - self._std_dev, self._seed, - var.dtype, place) + out_var = _C_ops.gaussian_random( + var.shape, + self._mean, + self._std_dev, + self._seed, + var.dtype, + place, + ) out_var._share_underline_tensor_to(var) return None if _in_legacy_dygraph(): out_var = _legacy_C_ops.gaussian_random( - 'shape', var.shape, 'dtype', var.dtype, 'mean', self._mean, - 'std', self._std_dev, 'seed', self._seed, 'use_mkldnn', False) + 'shape', + var.shape, + 'dtype', + var.dtype, + 'mean', + self._mean, + 'std', + self._std_dev, + 'seed', + self._seed, + 'use_mkldnn', + False, + ) out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type="gaussian_random", - outputs={"Out": var}, - attrs={ - "shape": var.shape, - "dtype": var.dtype, - "mean": self._mean, - "std": self._std_dev, - "seed": self._seed, - "use_mkldnn": False - }, - stop_gradient=True) + op = block.append_op( + type="gaussian_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "dtype": var.dtype, + "mean": self._mean, + "std": self._std_dev, + "seed": self._seed, + "use_mkldnn": False, + }, + stop_gradient=True, + ) var.op = op return op @@ -454,20 +530,28 @@ class TruncatedNormalInitializer(Initializer): # to be compatible of fp16 initalizers if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['truncated_gaussian_random', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['truncated_gaussian_random', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var if in_dygraph_mode(): out_var = _C_ops.truncated_gaussian_random( - var.shape, self._mean, self._std_dev, self._seed, out_dtype, - _current_expected_place()) + var.shape, + self._mean, + self._std_dev, + self._seed, + out_dtype, + _current_expected_place(), + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: var_tmp = _C_ops.cast(out_var, var.dtype) var_tmp._share_underline_tensor_to(var) @@ -477,35 +561,46 @@ class TruncatedNormalInitializer(Initializer): if _in_legacy_dygraph(): out_var = _legacy_C_ops.truncated_gaussian_random( - 'shape', var.shape, 'dtype', out_dtype, 'mean', self._mean, - 'std', self._std_dev, 'seed', self._seed) + 'shape', + var.shape, + 'dtype', + out_dtype, + 'mean', + self._mean, + 'std', + self._std_dev, + 'seed', + self._seed, + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', out_var.dtype, - 'out_dtype', var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, 'in_dtype', out_var.dtype, 'out_dtype', var.dtype + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type="truncated_gaussian_random", - outputs={"Out": out_var}, - attrs={ - "shape": var.shape, - "dtype": out_dtype, - "mean": self._mean, - "std": self._std_dev, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="truncated_gaussian_random", + outputs={"Out": out_var}, + attrs={ + "shape": var.shape, + "dtype": out_dtype, + "mean": self._mean, + "std": self._std_dev, + "seed": self._seed, + }, + stop_gradient=True, + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -578,9 +673,12 @@ class XavierInitializer(Initializer): block = self._check_block(block) assert isinstance(block, framework.Block) - check_variable_and_dtype(var, "Out", - ["uint16", "float16", "float32", "float64"], - "xavier_init") + check_variable_and_dtype( + var, + "Out", + ["uint16", "float16", "float32", "float64"], + "xavier_init", + ) f_in, f_out = self._compute_fans(var) @@ -593,14 +691,18 @@ class XavierInitializer(Initializer): # to be compatible of fp16 initalizers if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['xavier_init', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['xavier_init', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var @@ -609,34 +711,62 @@ class XavierInitializer(Initializer): if self._uniform: limit = math.sqrt(6.0 / float(fan_in + fan_out)) if in_dygraph_mode(): - out_var = _C_ops.uniform_random(out_var.shape, out_dtype, - -limit, limit, self._seed, - _current_expected_place()) + out_var = _C_ops.uniform_random( + out_var.shape, + out_dtype, + -limit, + limit, + self._seed, + _current_expected_place(), + ) elif _in_legacy_dygraph(): out_var = _legacy_C_ops.uniform_random( - 'shape', out_var.shape, 'min', -limit, 'max', limit, - 'seed', self._seed, 'dtype', out_dtype) + 'shape', + out_var.shape, + 'min', + -limit, + 'max', + limit, + 'seed', + self._seed, + 'dtype', + out_dtype, + ) else: std = math.sqrt(2.0 / float(fan_in + fan_out)) if in_dygraph_mode(): place = _current_expected_place() - out_var = _C_ops.gaussian_random(out_var.shape, 0.0, std, - self._seed, out_dtype, - place) + out_var = _C_ops.gaussian_random( + out_var.shape, 0.0, std, self._seed, out_dtype, place + ) else: out_var = _legacy_C_ops.gaussian_random( - 'shape', out_var.shape, 'dtype', out_dtype, 'mean', 0.0, - 'std', std, 'seed', self._seed) + 'shape', + out_var.shape, + 'dtype', + out_dtype, + 'mean', + 0.0, + 'std', + std, + 'seed', + self._seed, + ) if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) @@ -644,39 +774,43 @@ class XavierInitializer(Initializer): else: if self._uniform: limit = math.sqrt(6.0 / float(fan_in + fan_out)) - op = block.append_op(type="uniform_random", - inputs={}, - outputs={"Out": out_var}, - attrs={ - "shape": out_var.shape, - "dtype": out_dtype, - "min": -limit, - "max": limit, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="uniform_random", + inputs={}, + outputs={"Out": out_var}, + attrs={ + "shape": out_var.shape, + "dtype": out_dtype, + "min": -limit, + "max": limit, + "seed": self._seed, + }, + stop_gradient=True, + ) else: std = math.sqrt(2.0 / float(fan_in + fan_out)) - op = block.append_op(type="gaussian_random", - outputs={"Out": out_var}, - attrs={ - "shape": out_var.shape, - "dtype": out_var.dtype, - "mean": 0.0, - "std": std, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="gaussian_random", + outputs={"Out": out_var}, + attrs={ + "shape": out_var.shape, + "dtype": out_var.dtype, + "mean": 0.0, + "std": std, + "seed": self._seed, + }, + stop_gradient=True, + ) if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -725,14 +859,15 @@ class MSRAInitializer(Initializer): """ - def __init__(self, - uniform=True, - fan_in=None, - seed=0, - negative_slope=0, - nonlinearity='relu'): - """Constructor for MSRAInitializer - """ + def __init__( + self, + uniform=True, + fan_in=None, + seed=0, + negative_slope=0, + nonlinearity='relu', + ): + """Constructor for MSRAInitializer""" assert uniform is not None assert seed is not None super(MSRAInitializer, self).__init__() @@ -767,14 +902,18 @@ class MSRAInitializer(Initializer): # to be compatible of fp16 initalizers if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['masra_init', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['masra_init', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var @@ -784,34 +923,62 @@ class MSRAInitializer(Initializer): gain = calculate_gain(self._nonlinearity, self._negative_slope) limit = gain * math.sqrt(3.0 / float(fan_in)) if in_dygraph_mode(): - out_var = _C_ops.uniform_random(var.shape, out_dtype, - -limit, limit, self._seed, - _current_expected_place()) + out_var = _C_ops.uniform_random( + var.shape, + out_dtype, + -limit, + limit, + self._seed, + _current_expected_place(), + ) else: out_var = _legacy_C_ops.uniform_random( - 'shape', out_var.shape, 'min', -limit, 'max', limit, - 'seed', self._seed, 'dtype', int(out_dtype)) + 'shape', + out_var.shape, + 'min', + -limit, + 'max', + limit, + 'seed', + self._seed, + 'dtype', + int(out_dtype), + ) else: gain = calculate_gain(self._nonlinearity, self._negative_slope) std = gain / math.sqrt(float(fan_in)) if in_dygraph_mode(): place = _current_expected_place() - out_var = _C_ops.gaussian_random(out_var.shape, 0.0, std, - self._seed, out_dtype, - place) + out_var = _C_ops.gaussian_random( + out_var.shape, 0.0, std, self._seed, out_dtype, place + ) else: out_var = _legacy_C_ops.gaussian_random( - 'shape', out_var.shape, 'dtype', int(out_dtype), 'mean', - 0.0, 'std', std, 'seed', self._seed) + 'shape', + out_var.shape, + 'dtype', + int(out_dtype), + 'mean', + 0.0, + 'std', + std, + 'seed', + self._seed, + ) if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) @@ -820,41 +987,45 @@ class MSRAInitializer(Initializer): if self._uniform: gain = calculate_gain(self._nonlinearity, self._negative_slope) limit = gain * math.sqrt(3.0 / float(fan_in)) - op = block.append_op(type="uniform_random", - inputs={}, - outputs={"Out": out_var}, - attrs={ - "shape": out_var.shape, - "dtype": int(out_dtype), - "min": -limit, - "max": limit, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="uniform_random", + inputs={}, + outputs={"Out": out_var}, + attrs={ + "shape": out_var.shape, + "dtype": int(out_dtype), + "min": -limit, + "max": limit, + "seed": self._seed, + }, + stop_gradient=True, + ) else: gain = calculate_gain(self._nonlinearity, self._negative_slope) std = gain / math.sqrt(float(fan_in)) - op = block.append_op(type="gaussian_random", - outputs={"Out": out_var}, - attrs={ - "shape": out_var.shape, - "dtype": int(out_dtype), - "mean": 0.0, - "std": std, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="gaussian_random", + outputs={"Out": out_var}, + attrs={ + "shape": out_var.shape, + "dtype": int(out_dtype), + "mean": 0.0, + "std": std, + "seed": self._seed, + }, + stop_gradient=True, + ) if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -905,8 +1076,7 @@ class BilinearInitializer(Initializer): """ def __init__(self): - """Constructor for BilinearInitializer. - """ + """Constructor for BilinearInitializer.""" super(BilinearInitializer, self).__init__() def forward(self, var, block=None): @@ -937,9 +1107,9 @@ class BilinearInitializer(Initializer): weight = np.zeros(np.prod(var.shape), dtype='float32') size = shape[3] # factor - f = np.ceil(size / 2.) + f = np.ceil(size / 2.0) # center - c = (2 * f - 1 - f % 2) / (2. * f) + c = (2 * f - 1 - f % 2) / (2.0 * f) for i in range(np.prod(shape)): x = i % size y = (i / size) % size @@ -948,15 +1118,20 @@ class BilinearInitializer(Initializer): # to be compatible of fp16 initalizers if var.dtype in [ - VarDesc.VarType.FP16, VarDesc.VarType.BF16, VarDesc.VarType.FP64 + VarDesc.VarType.FP16, + VarDesc.VarType.BF16, + VarDesc.VarType.FP64, ]: out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['bilinear_init', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['bilinear_init', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var @@ -972,46 +1147,64 @@ class BilinearInitializer(Initializer): if framework._non_static_mode(): if in_dygraph_mode(): - _C_ops.assign_value_(out_var, list(shape), out_dtype, values, - _current_expected_place()) + _C_ops.assign_value_( + out_var, + list(shape), + out_dtype, + values, + _current_expected_place(), + ) elif _in_legacy_dygraph(): - _legacy_C_ops.assign_value(out_var, 'shape', list(shape), - 'dtype', out_dtype, value_name, - values) + _legacy_C_ops.assign_value( + out_var, + 'shape', + list(shape), + 'dtype', + out_dtype, + value_name, + values, + ) if var.dtype in [ - VarDesc.VarType.FP16, VarDesc.VarType.BF16, - VarDesc.VarType.FP64 + VarDesc.VarType.FP16, + VarDesc.VarType.BF16, + VarDesc.VarType.FP64, ]: if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type='assign_value', - outputs={'Out': [out_var]}, - attrs={ - 'dtype': out_dtype, - 'shape': list(shape), - value_name: values - }) + op = block.append_op( + type='assign_value', + outputs={'Out': [out_var]}, + attrs={ + 'dtype': out_dtype, + 'shape': list(shape), + value_name: values, + }, + ) if var.dtype in [ - VarDesc.VarType.FP16, VarDesc.VarType.BF16, - VarDesc.VarType.FP64 + VarDesc.VarType.FP16, + VarDesc.VarType.BF16, + VarDesc.VarType.FP64, ]: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -1039,6 +1232,7 @@ class NumpyArrayInitializer(Initializer): def __init__(self, value): import numpy + assert isinstance(value, numpy.ndarray) super(NumpyArrayInitializer, self).__init__() self._value = value @@ -1063,12 +1257,15 @@ class NumpyArrayInitializer(Initializer): if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: out_dtype = VarDesc.VarType.FP32 np_value = self._value.astype("float32") - out_var = block.create_var(name=unique_name.generate(".".join( - ['numpy_array_init', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['numpy_array_init', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_var = var out_dtype = var.dtype @@ -1083,47 +1280,64 @@ class NumpyArrayInitializer(Initializer): else: raise ValueError("Unsupported dtype %s", self._value.dtype) if self._value.size > 1024 * 1024 * 1024: - raise ValueError("The size of input is too big. Please consider " - "saving it to file and 'load_op' to load it") + raise ValueError( + "The size of input is too big. Please consider " + "saving it to file and 'load_op' to load it" + ) if framework._non_static_mode(): if in_dygraph_mode(): - _C_ops.assign_value_(out_var, - list(self._value.shape), out_dtype, values, - _current_expected_place()) + _C_ops.assign_value_( + out_var, + list(self._value.shape), + out_dtype, + values, + _current_expected_place(), + ) elif _in_legacy_dygraph(): - _legacy_C_ops.assign_value(out_var, 'shape', - list(self._value.shape), 'dtype', - out_dtype, value_name, values) + _legacy_C_ops.assign_value( + out_var, + 'shape', + list(self._value.shape), + 'dtype', + out_dtype, + value_name, + values, + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type='assign_value', - outputs={'Out': out_var}, - attrs={ - 'dtype': out_dtype, - 'shape': list(self._value.shape), - value_name: values - }, - stop_gradient=True) + op = block.append_op( + type='assign_value', + outputs={'Out': out_var}, + attrs={ + 'dtype': out_dtype, + 'shape': list(self._value.shape), + value_name: values, + }, + stop_gradient=True, + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -1179,13 +1393,21 @@ def set_global_initializer(weight_init, bias_init=None): nn.initializer.set_global_initializer(None) """ - check_type(weight_init, 'weight_init', (Initializer, type(None)), - 'set_global_initializer') + check_type( + weight_init, + 'weight_init', + (Initializer, type(None)), + 'set_global_initializer', + ) global _global_weight_initializer_ _global_weight_initializer_ = weight_init - check_type(bias_init, 'bias_init', (Initializer, type(None)), - 'set_global_initializer') + check_type( + bias_init, + 'bias_init', + (Initializer, type(None)), + 'set_global_initializer', + ) global _global_bias_initializer_ _global_bias_initializer_ = bias_init @@ -1244,14 +1466,16 @@ def calculate_gain(nonlinearity, param=None): 'tanh': 5.0 / 3, 'relu': math.sqrt(2.0), 'leaky_relu': math.sqrt(2.0 / (1 + param**2)), - 'selu': 3.0 / 4 + 'selu': 3.0 / 4, } if nonlinearity in recommended_gain.keys(): return recommended_gain[nonlinearity] else: raise ValueError( "nonlinearity function {} is not suppported now.".format( - nonlinearity)) + nonlinearity + ) + ) # We short the class name, since users will use the initializer with the package diff --git a/python/paddle/fluid/input.py b/python/paddle/fluid/input.py index bfab41f86a8..255a17d6483 100644 --- a/python/paddle/fluid/input.py +++ b/python/paddle/fluid/input.py @@ -25,8 +25,8 @@ __all__ = ['one_hot', 'embedding'] def one_hot(input, depth, allow_out_of_range=False): """ :alias_main: paddle.nn.functional.one_hot - :alias: paddle.nn.functional.one_hot,paddle.nn.functional.common.one_hot - :old_api: paddle.fluid.one_hot + :alias: paddle.nn.functional.one_hot,paddle.nn.functional.common.one_hot + :old_api: paddle.fluid.one_hot The operator converts each id in the input to an one-hot vector with a depth length. The value in the vector dimension corresponding to the id @@ -122,23 +122,27 @@ def one_hot(input, depth, allow_out_of_range=False): depth.stop_gradient = True inputs = {'X': input, 'depth_tensor': depth} attrs = {'allow_out_of_range': allow_out_of_range} - helper.append_op(type="one_hot_v2", - inputs=inputs, - attrs=attrs, - outputs={'Out': one_hot_out}, - stop_gradient=True) + helper.append_op( + type="one_hot_v2", + inputs=inputs, + attrs=attrs, + outputs={'Out': one_hot_out}, + stop_gradient=True, + ) return one_hot_out @static_only @deprecated(since='2.0.0', update_to='paddle.nn.functional.embedding') -def embedding(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): +def embedding( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): r""" :api_attr: Static Graph @@ -310,28 +314,35 @@ def embedding(input, helper = LayerHelper('embedding', **locals()) check_variable_and_dtype(input, 'input', ['int64'], 'fluid.embedding') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64', 'uint16'], - 'fluid.embedding') + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64', 'uint16'], + 'fluid.embedding', + ) remote_prefetch = is_sparse and (not is_distributed) if remote_prefetch: assert is_sparse is True and is_distributed is False - w = helper.create_parameter(attr=helper.param_attr, - shape=size, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False + ) tmp = helper.create_variable_for_type_inference(dtype) - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) - helper.append_op(type='lookup_table_v2', - inputs={ - 'Ids': input, - 'W': w - }, - outputs={'Out': tmp}, - attrs={ - 'is_sparse': is_sparse, - 'is_distributed': is_distributed, - 'remote_prefetch': remote_prefetch, - 'padding_idx': padding_idx - }) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) + helper.append_op( + type='lookup_table_v2', + inputs={'Ids': input, 'W': w}, + outputs={'Out': tmp}, + attrs={ + 'is_sparse': is_sparse, + 'is_distributed': is_distributed, + 'remote_prefetch': remote_prefetch, + 'padding_idx': padding_idx, + }, + ) return tmp diff --git a/python/paddle/fluid/install_check.py b/python/paddle/fluid/install_check.py index 299275be5e2..74bc8df61bb 100644 --- a/python/paddle/fluid/install_check.py +++ b/python/paddle/fluid/install_check.py @@ -14,7 +14,13 @@ import os import paddle -from .framework import Program, program_guard, unique_name, cuda_places, cpu_places +from .framework import ( + Program, + program_guard, + unique_name, + cuda_places, + cpu_places, +) from .param_attr import ParamAttr from .initializer import Constant from . import layers @@ -31,13 +37,11 @@ __all__ = ['run_check'] class SimpleLayer(Layer): - def __init__(self, input_size): super(SimpleLayer, self).__init__() self._linear1 = nn.Linear( - input_size, - 3, - param_attr=ParamAttr(initializer=Constant(value=0.1))) + input_size, 3, param_attr=ParamAttr(initializer=Constant(value=0.1)) + ) def forward(self, inputs): x = self._linear1(inputs) @@ -74,7 +78,8 @@ def run_check(): except Exception as e: logging.warning( "You are using GPU version Paddle Fluid, But Your CUDA Device is not set properly" - "\n Original Error is {}".format(e)) + "\n Original Error is {}".format(e) + ) return 0 device_list = cuda_places() else: @@ -100,22 +105,29 @@ def run_check(): simple_layer = SimpleLayer(input_size=2) out = simple_layer(inp) exe = executor.Executor( - core.CUDAPlace(0) if core.is_compiled_with_cuda() and - (core.get_cuda_device_count() > 0) else core.CPUPlace()) + core.CUDAPlace(0) + if core.is_compiled_with_cuda() + and (core.get_cuda_device_count() > 0) + else core.CPUPlace() + ) loss = paddle.mean(out) loss.persistable = True optimizer.SGD(learning_rate=0.01).minimize(loss) startup_prog.random_seed = 1 compiled_prog = compiler.CompiledProgram( - train_prog).with_data_parallel( - build_strategy=build_strategy, - loss_name=loss.name, - places=device_list) + train_prog + ).with_data_parallel( + build_strategy=build_strategy, + loss_name=loss.name, + places=device_list, + ) exe.run(startup_prog) - exe.run(compiled_prog, - feed={inp.name: np_inp_muti}, - fetch_list=[loss.name]) + exe.run( + compiled_prog, + feed={inp.name: np_inp_muti}, + fetch_list=[loss.name], + ) def test_simple_exe(): train_prog = Program() @@ -124,20 +136,26 @@ def run_check(): with executor.scope_guard(scope): with program_guard(train_prog, startup_prog): with unique_name.guard(): - inp0 = layers.data(name="inp", - shape=[2, 2], - append_batch_size=False) + inp0 = layers.data( + name="inp", shape=[2, 2], append_batch_size=False + ) simple_layer0 = SimpleLayer(input_size=2) out0 = simple_layer0(inp0) param_grads = backward.append_backward( out0, - parameter_list=[simple_layer0._linear1.weight.name])[0] + parameter_list=[simple_layer0._linear1.weight.name], + )[0] exe0 = executor.Executor( - core.CUDAPlace(0) if core.is_compiled_with_cuda() and - (core.get_cuda_device_count() > 0) else core.CPUPlace()) + core.CUDAPlace(0) + if core.is_compiled_with_cuda() + and (core.get_cuda_device_count() > 0) + else core.CPUPlace() + ) exe0.run(startup_prog) - exe0.run(feed={inp0.name: np_inp_single}, - fetch_list=[out0.name, param_grads[1].name]) + exe0.run( + feed={inp0.name: np_inp_single}, + fetch_list=[out0.name, param_grads[1].name], + ) test_simple_exe() @@ -160,6 +178,7 @@ def run_check(): print("\n Original Error is: {}".format(e)) print( "Your Paddle Fluid is installed successfully ONLY for SINGLE GPU or CPU! " - "\n Let's start deep Learning with Paddle Fluid now") + "\n Let's start deep Learning with Paddle Fluid now" + ) paddle.disable_static() diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 1af1d0a606d..c5e55a95186 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -28,10 +28,28 @@ import paddle from paddle.fluid import layers from paddle.fluid.executor import Executor, global_scope from paddle.fluid.evaluator import Evaluator -from paddle.fluid.framework import Program, Parameter, default_main_program, default_startup_program, Variable, \ - program_guard, dygraph_not_support, static_only -from paddle.reader import cache, map_readers, buffered, compose, chain, shuffle, \ - ComposeNotAligned, firstn, xmap_readers, multiprocess_reader +from paddle.fluid.framework import ( + Program, + Parameter, + default_main_program, + default_startup_program, + Variable, + program_guard, + dygraph_not_support, + static_only, +) +from paddle.reader import ( + cache, + map_readers, + buffered, + compose, + chain, + shuffle, + ComposeNotAligned, + firstn, + xmap_readers, + multiprocess_reader, +) from .wrapped_decorator import signature_safe_contextmanager from paddle.fluid.compiler import CompiledProgram from paddle.fluid.log_helper import get_logger @@ -64,13 +82,12 @@ __all__ = [ 'get_program_persistable_vars', ] + reader.__all__ -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class _open_buffer(object): - def __init__(self, buffer): self.buffer = buffer @@ -79,7 +96,6 @@ class _open_buffer(object): class _buffer_reader(_open_buffer): - def __init__(self, buffer): super(_buffer_reader, self).__init__(buffer) self.initial_tell = self.buffer.tell() @@ -91,7 +107,6 @@ class _buffer_reader(_open_buffer): class _buffer_writer(_open_buffer): - def __exit__(self, *args): self.buffer.flush() @@ -111,7 +126,8 @@ def _open_file_buffer(path_or_buffer, mode): return _buffer_reader(path_or_buffer) else: raise ValueError( - "Expected 'r' or 'w' in mode but got {}".format(mode)) + "Expected 'r' or 'w' in mode but got {}".format(mode) + ) def _is_memory_buffer(buffer): @@ -163,9 +179,11 @@ def is_persistable(var): param = fluid.default_main_program().global_block().var('fc.b') res = fluid.io.is_persistable(param) """ - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -232,18 +250,22 @@ def get_program_persistable_vars(program): def _clone_var_in_block_(block, var): assert isinstance(var, Variable) if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR: - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True, + ) else: - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + persistable=True, + ) @signature_safe_contextmanager @@ -268,21 +290,25 @@ def _get_valid_program(main_program=None): "The type of input main_program is invalid, expected tyep is Program, but received None" ) warnings.warn( - "The input is a CompiledProgram, this is not recommended.") + "The input is a CompiledProgram, this is not recommended." + ) if not isinstance(main_program, Program): raise TypeError( "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program)) + % type(main_program) + ) return main_program @dygraph_not_support -def save_vars(executor, - dirname, - main_program=None, - vars=None, - predicate=None, - filename=None): +def save_vars( + executor, + dirname, + main_program=None, + vars=None, + predicate=None, + filename=None, +): """ Save specific variables in the `Program` to files. @@ -360,11 +386,13 @@ def save_vars(executor, main_program = _get_valid_program(main_program) if vars is None: - return save_vars(executor, - main_program=main_program, - dirname=dirname, - vars=list(filter(predicate, main_program.list_vars())), - filename=filename) + return save_vars( + executor, + main_program=main_program, + dirname=dirname, + vars=list(filter(predicate, main_program.list_vars())), + filename=filename, + ) else: params_var_name = "saved_params" # give warning when there is no var in model @@ -384,13 +412,15 @@ def save_vars(executor, continue new_var = _clone_var_in_block_(save_block, each_var) if filename is None and save_to_memory is False: - save_file_path = os.path.join(os.path.normpath(dirname), - new_var.name) + save_file_path = os.path.join( + os.path.normpath(dirname), new_var.name + ) save_block.append_op( type='save', inputs={'X': [new_var]}, outputs={}, - attrs={'file_path': os.path.normpath(save_file_path)}) + attrs={'file_path': os.path.normpath(save_file_path)}, + ) else: save_var_map[new_var.name] = new_var @@ -403,16 +433,19 @@ def save_vars(executor, if save_to_memory is False: save_path = os.path.join(os.path.normpath(dirname), filename) - saved_params = save_block.create_var(type=core.VarDesc.VarType.RAW, - name=params_var_name) + saved_params = save_block.create_var( + type=core.VarDesc.VarType.RAW, name=params_var_name + ) saved_params.desc.set_persistable(True) - save_block.append_op(type='save_combine', - inputs={'X': save_var_list}, - outputs={'Y': saved_params}, - attrs={ - 'file_path': save_path, - 'save_to_memory': save_to_memory - }) + save_block.append_op( + type='save_combine', + inputs={'X': save_var_list}, + outputs={'Y': saved_params}, + attrs={ + 'file_path': save_path, + 'save_to_memory': save_to_memory, + }, + ) # NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc, # which leads to diff on save_program and its desc. Call _sync_with_cpp @@ -489,12 +522,14 @@ def save_params(executor, dirname, main_program=None, filename=None): # The parameters weights and bias of the fc layer in the network are going to # be saved in different files in the path "./my_paddle_model" """ - return save_vars(executor, - dirname=dirname, - main_program=main_program, - vars=None, - predicate=is_parameter, - filename=filename) + return save_vars( + executor, + dirname=dirname, + main_program=main_program, + vars=None, + predicate=is_parameter, + filename=filename, + ) def _save_distributed_persistables(executor, dirname, main_program): @@ -568,21 +603,24 @@ def _save_distributed_persistables(executor, dirname, main_program): tmp = [str(dim) for dim in slice.shape] slice_shapes.append(",".join(tmp)) - block.append_op(type='recv_save', - attrs={ - "trainer_id": 0, - "shape": origin.shape, - "slice_shapes": slice_shapes, - "slice_varnames": slice_varnames, - "remote_varnames": remote_varnames, - "endpoints": endpoints, - "file_path": os.path.join(dirname, origin.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": 0, + "shape": origin.shape, + "slice_shapes": slice_shapes, + "slice_varnames": slice_varnames, + "remote_varnames": remote_varnames, + "endpoints": endpoints, + "file_path": os.path.join(dirname, origin.name), + }, + ) executor.run(prog) - def __save_distributed_lookup_tables(executor, dirname, - distributed_lookup_table, endpoints): + def __save_distributed_lookup_tables( + executor, dirname, distributed_lookup_table, endpoints + ): """ because the distributed lookup table may too huge to merge and save at one place, it will be saved at parameter server independent respectively. @@ -599,20 +637,20 @@ def _save_distributed_persistables(executor, dirname, main_program): attrs['epmap'] = endpoints attrs['dir'] = lookup_table_filename attrs['lookup_table'] = distributed_lookup_table - block.append_op(type='checkpoint_notify', - inputs={}, - outputs={}, - attrs=attrs) + block.append_op( + type='checkpoint_notify', inputs={}, outputs={}, attrs=attrs + ) executor.run(prog) def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -626,8 +664,11 @@ def _save_distributed_persistables(executor, dirname, main_program): "'_save_distributed_persistables' just be designed for distributed training." ) - remote_params_map = main_program._parameters_on_pservers.get_distributed_vars_by_vtypes( - ["Optimizer", "RemotePrefetch"], groupby=True) + remote_params_map = ( + main_program._parameters_on_pservers.get_distributed_vars_by_vtypes( + ["Optimizer", "RemotePrefetch"], groupby=True + ) + ) exclude_var_names = [] if remote_params_map: @@ -640,19 +681,22 @@ def _save_distributed_persistables(executor, dirname, main_program): exclude_var_names.append(main_program._distributed_lookup_table) local_vars = list( - filter(__exclude_vars(exclude_var_names), main_program.list_vars())) - save_vars(executor, - main_program=main_program, - dirname=dirname, - vars=local_vars) + filter(__exclude_vars(exclude_var_names), main_program.list_vars()) + ) + save_vars( + executor, main_program=main_program, dirname=dirname, vars=local_vars + ) if main_program._is_chief: if remote_params_map: __save_remote_params(executor, dirname, remote_params_map) if main_program._distributed_lookup_table: __save_distributed_lookup_tables( - executor, dirname, main_program._distributed_lookup_table, - main_program._endpoints) + executor, + dirname, + main_program._distributed_lookup_table, + main_program._endpoints, + ) @dygraph_not_support @@ -714,24 +758,28 @@ def save_persistables(executor, dirname, main_program=None, filename=None): # "./my_paddle_model" """ if main_program and main_program._is_distributed: - return _save_distributed_persistables(executor, - dirname=dirname, - main_program=main_program) + return _save_distributed_persistables( + executor, dirname=dirname, main_program=main_program + ) else: - return save_vars(executor, - dirname=dirname, - main_program=main_program, - vars=None, - predicate=is_persistable, - filename=filename) - - -def load_vars(executor, - dirname, - main_program=None, - vars=None, - predicate=None, - filename=None): + return save_vars( + executor, + dirname=dirname, + main_program=main_program, + vars=None, + predicate=is_persistable, + filename=filename, + ) + + +def load_vars( + executor, + dirname, + main_program=None, + vars=None, + predicate=None, + filename=None, +): """ :api_attr: Static Graph @@ -820,13 +868,16 @@ def load_vars(executor, if not isinstance(main_program, Program): raise TypeError( "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program)) + % type(main_program) + ) - load_vars(executor, - dirname=dirname, - main_program=main_program, - vars=list(filter(predicate, main_program.list_vars())), - filename=filename) + load_vars( + executor, + dirname=dirname, + main_program=main_program, + vars=list(filter(predicate, main_program.list_vars())), + filename=filename, + ) else: load_prog = Program() load_block = load_prog.global_block() @@ -837,7 +888,8 @@ def load_vars(executor, if not isinstance(main_program, Program): raise TypeError( "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program)) + % type(main_program) + ) # save origin param shape orig_para_shape = {} @@ -854,7 +906,8 @@ def load_vars(executor, if isinstance(each_var, Parameter): orig_para_shape[each_var.name] = tuple( - each_var.desc.get_shape()) + each_var.desc.get_shape() + ) if each_var.type == core.VarDesc.VarType.SELECTED_ROWS: sparse_vars.append(each_var) @@ -872,7 +925,8 @@ def load_vars(executor, type='load', inputs={}, outputs={'Out': [new_var]}, - attrs={'file_path': os.path.join(dirname, new_var.name)}) + attrs={'file_path': os.path.join(dirname, new_var.name)}, + ) else: load_var_map[new_var.name] = new_var @@ -881,7 +935,8 @@ def load_vars(executor, if filename is not None: raise ValueError( - "SelectedRows can not be load with load_combine") + "SelectedRows can not be load with load_combine" + ) new_var = _clone_var_in_block_(load_block, each_var) @@ -889,14 +944,17 @@ def load_vars(executor, if not os.path.exists(var_path): raise ValueError( "SelectedRows var {} can not find at {}".format( - new_var.name, var_path)) + new_var.name, var_path + ) + ) if os.path.isfile(var_path): load_block.append_op( type='load', inputs={}, outputs={'Out': [new_var]}, - attrs={'file_path': os.path.join(dirname, new_var.name)}) + attrs={'file_path': os.path.join(dirname, new_var.name)}, + ) else: blocks = [] block_paths = os.listdir(var_path) @@ -907,23 +965,29 @@ def load_vars(executor, slices = [] for block in blocks: - slice = load_block.create_var(name=block, - type=new_var.type, - shape=new_var.shape, - dtype=new_var.dtype, - persistable=False) + slice = load_block.create_var( + name=block, + type=new_var.type, + shape=new_var.shape, + dtype=new_var.dtype, + persistable=False, + ) slices.append(slice) file_path = os.path.join(var_path, block, "Param") - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [slice]}, - attrs={'file_path': file_path}) + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice]}, + attrs={'file_path': file_path}, + ) - load_block.append_op(type='lookup_sparse_table_merge', - inputs={'X': slices}, - outputs={'Out': new_var}, - attrs={}) + load_block.append_op( + type='lookup_sparse_table_merge', + inputs={'X': slices}, + outputs={'Out': new_var}, + attrs={}, + ) if filename is not None: load_var_list = [] @@ -933,13 +997,15 @@ def load_vars(executor, if vars_from_memory is False: filename = os.path.join(dirname, filename) - load_block.append_op(type='load_combine', - inputs={}, - outputs={"Out": load_var_list}, - attrs={ - 'file_path': filename, - 'model_from_memory': vars_from_memory - }) + load_block.append_op( + type='load_combine', + inputs={}, + outputs={"Out": load_var_list}, + attrs={ + 'file_path': filename, + 'model_from_memory': vars_from_memory, + }, + ) executor.run(load_prog) # check var shape @@ -949,13 +1015,17 @@ def load_vars(executor, var_temp = paddle.fluid.global_scope().find_var(each_var.name) assert var_temp != None, "can't not find var: " + each_var.name new_shape = (np.array(var_temp.get_tensor())).shape - assert each_var.name in orig_para_shape, each_var.name + "MUST in var list" + assert each_var.name in orig_para_shape, ( + each_var.name + "MUST in var list" + ) orig_shape = orig_para_shape.get(each_var.name) if new_shape != orig_shape: raise RuntimeError( "Variable's shape does not match, the Program requires a parameter with the shape of ({}), " - "while the loaded parameter (namely [ {} ]) has a shape of ({})." - .format(orig_shape, each_var.name, new_shape)) + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_shape, each_var.name, new_shape + ) + ) @dygraph_not_support @@ -1012,11 +1082,13 @@ def load_params(executor, dirname, main_program=None, filename=None): fluid.io.load_params(executor=exe, dirname=param_path, main_program=None) """ - load_vars(executor, - dirname=dirname, - main_program=main_program, - predicate=is_parameter, - filename=filename) + load_vars( + executor, + dirname=dirname, + main_program=main_program, + predicate=is_parameter, + filename=filename, + ) @dygraph_not_support @@ -1064,15 +1136,17 @@ def load_persistables(executor, dirname, main_program=None, filename=None): """ if main_program and main_program._is_distributed: - _load_distributed_persistables(executor, - dirname=dirname, - main_program=main_program) + _load_distributed_persistables( + executor, dirname=dirname, main_program=main_program + ) else: - load_vars(executor, - dirname=dirname, - main_program=main_program, - predicate=is_persistable, - filename=filename) + load_vars( + executor, + dirname=dirname, + main_program=main_program, + predicate=is_persistable, + filename=filename, + ) def _load_distributed_persistables(executor, dirname, main_program=None): @@ -1122,35 +1196,38 @@ def _load_distributed_persistables(executor, dirname, main_program=None): offset = param.offset if is_slice: - slice = load_block.create_var(name=slice_var.name, - type=slice_var.type, - shape=slice_var.shape, - dtype=slice_var.dtype, - persistable=True) - - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [slice]}, - attrs={ - 'file_path': - os.path.join(dirname, origin_var.name), - 'seek': - offset, - 'shape': - slice.shape - }) + slice = load_block.create_var( + name=slice_var.name, + type=slice_var.type, + shape=slice_var.shape, + dtype=slice_var.dtype, + persistable=True, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice]}, + attrs={ + 'file_path': os.path.join(dirname, origin_var.name), + 'seek': offset, + 'shape': slice.shape, + }, + ) else: - origin = load_block.create_var(name="{}".format( - origin_var.name), - type=origin_var.type, - shape=origin_var.shape, - dtype=origin_var.dtype, - persistable=True) + origin = load_block.create_var( + name="{}".format(origin_var.name), + type=origin_var.type, + shape=origin_var.shape, + dtype=origin_var.dtype, + persistable=True, + ) load_block.append_op( type='load', inputs={}, outputs={'Out': [origin]}, - attrs={'file_path': os.path.join(dirname, origin_var.name)}) + attrs={'file_path': os.path.join(dirname, origin_var.name)}, + ) load_block.append_op( type='delete_var', @@ -1172,63 +1249,78 @@ def _load_distributed_persistables(executor, dirname, main_program=None): "'_load_distributed_persistables' need current_endpoint set in DistributeTranspiler.transpile" ) - need_load_vars = main_program._parameters_on_pservers.get_distributed_vars_by_ep( - main_program._ps_endpoint) + need_load_vars = ( + main_program._parameters_on_pservers.get_distributed_vars_by_ep( + main_program._ps_endpoint + ) + ) __load_persistable_vars(executor, dirname, need_load_vars) -def prepend_feed_ops(inference_program, - feed_target_names, - feed_holder_name='feed'): +def prepend_feed_ops( + inference_program, feed_target_names, feed_holder_name='feed' +): if len(feed_target_names) == 0: return global_block = inference_program.global_block() - feed_var = global_block.create_var(name=feed_holder_name, - type=core.VarDesc.VarType.FEED_MINIBATCH, - persistable=True) + feed_var = global_block.create_var( + name=feed_holder_name, + type=core.VarDesc.VarType.FEED_MINIBATCH, + persistable=True, + ) for i, name in enumerate(feed_target_names): if not global_block.has_var(name): raise ValueError( "The feeded_var_names[{i}]: '{name}' doesn't exist in pruned inference program. " "Please check whether '{name}' is a valid feed_var name, or remove it from feeded_var_names " - "if '{name}' is not involved in the target_vars calculation.". - format(i=i, name=name)) + "if '{name}' is not involved in the target_vars calculation.".format( + i=i, name=name + ) + ) out = global_block.var(name) - global_block._prepend_op(type='feed', - inputs={'X': [feed_var]}, - outputs={'Out': [out]}, - attrs={'col': i}) + global_block._prepend_op( + type='feed', + inputs={'X': [feed_var]}, + outputs={'Out': [out]}, + attrs={'col': i}, + ) -def append_fetch_ops(inference_program, - fetch_target_names, - fetch_holder_name='fetch'): +def append_fetch_ops( + inference_program, fetch_target_names, fetch_holder_name='fetch' +): global_block = inference_program.global_block() - fetch_var = global_block.create_var(name=fetch_holder_name, - type=core.VarDesc.VarType.FETCH_LIST, - persistable=True) + fetch_var = global_block.create_var( + name=fetch_holder_name, + type=core.VarDesc.VarType.FETCH_LIST, + persistable=True, + ) for i, name in enumerate(fetch_target_names): - global_block.append_op(type='fetch', - inputs={'X': [name]}, - outputs={'Out': [fetch_var]}, - attrs={'col': i}) + global_block.append_op( + type='fetch', + inputs={'X': [name]}, + outputs={'Out': [fetch_var]}, + attrs={'col': i}, + ) @static_only @deprecated(since="2.0.0", update_to="paddle.static.save_inference_model") -def save_inference_model(dirname, - feeded_var_names, - target_vars, - executor, - main_program=None, - model_filename=None, - params_filename=None, - export_for_deployment=True, - program_only=False, - clip_extra=True): +def save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + main_program=None, + model_filename=None, + params_filename=None, + export_for_deployment=True, + program_only=False, + clip_extra=True, +): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` . @@ -1314,16 +1406,19 @@ def save_inference_model(dirname, elif export_for_deployment: if len(feeded_var_names) > 0: # TODO(paddle-dev): polish these code blocks - if not (bool(feeded_var_names) - and all(isinstance(name, str) - for name in feeded_var_names)): + if not ( + bool(feeded_var_names) + and all(isinstance(name, str) for name in feeded_var_names) + ): raise ValueError("'feed_var_names' should be a list of str.") if isinstance(target_vars, Variable): target_vars = [target_vars] elif export_for_deployment: - if not (bool(target_vars) - and all(isinstance(var, Variable) for var in target_vars)): + if not ( + bool(target_vars) + and all(isinstance(var, Variable) for var in target_vars) + ): raise ValueError("'target_vars' should be a list of Variable.") main_program = _get_valid_program(main_program) @@ -1384,7 +1479,8 @@ def save_inference_model(dirname, main_program.desc.flush() main_program = main_program._prune_with_input( - feeded_var_names=feeded_var_names, targets=target_vars) + feeded_var_names=feeded_var_names, targets=target_vars + ) main_program = main_program._inference_optimize(prune_read_op=True) fetch_var_names = [v.name for v in target_vars] @@ -1394,7 +1490,8 @@ def save_inference_model(dirname, name=target_v.name, shape=target_v.shape, dtype=target_v.dtype, - persistable=target_v.persistable) + persistable=target_v.persistable, + ) prepend_feed_ops(main_program, feeded_var_names) append_fetch_ops(main_program, fetch_var_names) @@ -1404,14 +1501,18 @@ def save_inference_model(dirname, with open(model_basename, "wb") as f: f.write( main_program._remove_training_info( - clip_extra=clip_extra).desc.serialize_to_string()) + clip_extra=clip_extra + ).desc.serialize_to_string() + ) else: # TODO(panyx0718): Save more information so that it can also be used # for training and more flexible post-processing. with open(model_basename + ".main_program", "wb") as f: f.write( main_program._remove_training_info( - clip_extra=clip_extra).desc.serialize_to_string()) + clip_extra=clip_extra + ).desc.serialize_to_string() + ) if program_only: warnings.warn( @@ -1430,11 +1531,13 @@ def save_inference_model(dirname, @static_only @deprecated(since="2.0.0", update_to="paddle.static.load_inference_model") -def load_inference_model(dirname, - executor, - model_filename=None, - params_filename=None, - pserver_endpoints=None): +def load_inference_model( + dirname, + executor, + model_filename=None, + params_filename=None, + pserver_endpoints=None, +): """ Load the inference model from a given directory. By this API, you can get the model structure(Inference Program) and model parameters. If you just want to load @@ -1534,8 +1637,9 @@ def load_inference_model(dirname, if model_filename is None: model_filename = '__model__' - model_filename = os.path.join(load_dirname, - os.path.basename(model_filename)) + model_filename = os.path.join( + load_dirname, os.path.basename(model_filename) + ) if params_filename is not None: params_filename = os.path.basename(params_filename) @@ -1554,8 +1658,9 @@ def load_inference_model(dirname, program = Program.parse_from_string(program_desc_str) if not core._is_program_version_supported(program._version()): - raise ValueError("Unsupported program version: %d\n" % - program._version()) + raise ValueError( + "Unsupported program version: %d\n" % program._version() + ) # Binary data also need versioning. load_persistables(executor, load_dirname, program, params_filename) @@ -1666,8 +1771,10 @@ def _save_persistable_nodes(executor, dirname, graph): var_list = [] for node in persistable_nodes: var_desc = node.var() - if var_desc.type() == core.VarDesc.VarType.RAW or \ - var_desc.type() == core.VarDesc.VarType.READER: + if ( + var_desc.type() == core.VarDesc.VarType.RAW + or var_desc.type() == core.VarDesc.VarType.READER + ): continue var = program.global_block().create_var( name=var_desc.name(), @@ -1675,7 +1782,8 @@ def _save_persistable_nodes(executor, dirname, graph): dtype=var_desc.dtype(), type=var_desc.type(), lod_level=var_desc.lod_level(), - persistable=var_desc.persistable()) + persistable=var_desc.persistable(), + ) var_list.append(var) save_vars(executor=executor, dirname=dirname, vars=var_list) @@ -1705,8 +1813,10 @@ def _load_persistable_nodes(executor, dirname, graph): for node in persistable_nodes: var_desc = node.var() - if var_desc.type() == core.VarDesc.VarType.RAW or \ - var_desc.type() == core.VarDesc.VarType.READER: + if ( + var_desc.type() == core.VarDesc.VarType.RAW + or var_desc.type() == core.VarDesc.VarType.READER + ): continue var = program.global_block().create_var( name=var_desc.name(), @@ -1714,7 +1824,8 @@ def _load_persistable_nodes(executor, dirname, graph): dtype=var_desc.dtype(), type=var_desc.type(), lod_level=var_desc.lod_level(), - persistable=var_desc.persistable()) + persistable=var_desc.persistable(), + ) if _exist(var): var_list.append(var) else: @@ -1731,7 +1842,8 @@ def _unpack_saved_dict(saved_obj, protocol): for key, value in saved_obj.items(): if isinstance(value, np.ndarray): MAX_NUMBER_OF_ELEMENT = int( - (2**30 - 1) / value.dtype.itemsize) + (2**30 - 1) / value.dtype.itemsize + ) num_element = np.prod(value.shape) if num_element > MAX_NUMBER_OF_ELEMENT: unpack_infor[key] = {} @@ -1739,15 +1851,19 @@ def _unpack_saved_dict(saved_obj, protocol): unpack_infor[key]["slices"] = [] value = value.flatten() for i in range( - int( - math.ceil(num_element * 1.0 / - MAX_NUMBER_OF_ELEMENT))): + int( + math.ceil( + num_element * 1.0 / MAX_NUMBER_OF_ELEMENT + ) + ) + ): part_name = key + "@@." + str(i) unpack_infor[key]["slices"].append(part_name) temp_saved_obj[part_name] = value[ - i * - MAX_NUMBER_OF_ELEMENT:MAX_NUMBER_OF_ELEMENT * - (i + 1)] + i + * MAX_NUMBER_OF_ELEMENT : MAX_NUMBER_OF_ELEMENT + * (i + 1) + ] if unpack_infor: for key, value in unpack_infor.items(): @@ -1767,7 +1883,8 @@ def _pack_loaded_dict(load_obj): for key, value in load_obj[unpack_info].items(): slices = [load_obj[part] for part in value["slices"]] load_obj[key] = np.concatenate(slices).reshape( - value["OriginShape"]) + value["OriginShape"] + ) removes += value["slices"] for key in removes: load_obj.pop(key) @@ -1778,7 +1895,6 @@ def _pack_loaded_dict(load_obj): @static_only def _legacy_save(param_dict, model_path, protocol=2): - def get_tensor(var): if isinstance(var, (core.VarBase, core.eager.Tensor)): return var.numpy() @@ -1789,14 +1905,16 @@ def _legacy_save(param_dict, model_path, protocol=2): param_dict = {name: get_tensor(param_dict[name]) for name in param_dict} # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if _is_file_path( - model_path - ) and sys.platform == 'darwin' and sys.version_info.major == 3: + if ( + _is_file_path(model_path) + and sys.platform == 'darwin' + and sys.version_info.major == 3 + ): pickle_bytes = pickle.dumps(param_dict, protocol=protocol) with open(model_path, 'wb') as f: max_bytes = 2**30 for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i:i + max_bytes]) + f.write(pickle_bytes[i : i + max_bytes]) else: with _open_file_buffer(model_path, 'wb') as f: pickle.dump(param_dict, f, protocol=protocol) @@ -1843,8 +1961,9 @@ def save(program, model_path, protocol=4, **configs): """ base_name = os.path.basename(model_path) - assert base_name != "", \ - "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string." + assert ( + base_name != "" + ), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string." if 'pickle_protocol' in configs: protocol = configs['pickle_protocol'] warnings.warn( @@ -1852,13 +1971,16 @@ def save(program, model_path, protocol=4, **configs): ) if not isinstance(protocol, int): - raise ValueError("The 'protocol' MUST be `int`, but received {}".format( - type(protocol))) + raise ValueError( + "The 'protocol' MUST be `int`, but received {}".format( + type(protocol) + ) + ) if protocol < 2 or protocol > 4: raise ValueError( - "Expected 1<'protocol'<5, but received protocol={}".format( - protocol)) + "Expected 1<'protocol'<5, but received protocol={}".format(protocol) + ) dir_name = os.path.dirname(model_path) if dir_name and not os.path.exists(dir_name): @@ -1879,13 +2001,14 @@ def save(program, model_path, protocol=4, **configs): with open(model_path + ".pdparams", 'wb') as f: max_bytes = 2**30 for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i:i + max_bytes]) + f.write(pickle_bytes[i : i + max_bytes]) else: with open(model_path + ".pdparams", 'wb') as f: pickle.dump(param_dict, f, protocol=protocol) optimizer_var_list = list( - filter(is_belong_to_optimizer, program.list_vars())) + filter(is_belong_to_optimizer, program.list_vars()) + ) opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list} with open(model_path + ".pdopt", 'wb') as f: @@ -1971,8 +2094,10 @@ def load(program, model_path, executor=None, var_list=None): # model file save by fluid.save not found, try to load model file saved with # [save_vars, save_params, save_persistables] _logger.debug( - "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]" - .format(parameter_file_name)) + "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format( + parameter_file_name + ) + ) if executor is None: raise ValueError( "executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]" @@ -1988,30 +2113,36 @@ def load(program, model_path, executor=None, var_list=None): for root, dirs, files in os.walk(model_path, topdown=False): for f in files: binary_file_set.add( - os.path.join(root, f).replace("\\", "/")) + os.path.join(root, f).replace("\\", "/") + ) program_var_list = list(program.list_vars()) loaded_var_list = [] for var in program_var_list: var_path = os.path.join(model_path, var.name).replace("\\", "/") - load_condition = var_list_names is None or var.name in var_list_names + load_condition = ( + var_list_names is None or var.name in var_list_names + ) if var_path in binary_file_set and load_condition: loaded_var_list.append(var) binary_file_set.remove(var_path) if len(binary_file_set) > 0: unused_var_list = " ".join(list(binary_file_set)) - _logger.warning("variable file [ %s ] not used" % - (" ".join(list(binary_file_set)))) + _logger.warning( + "variable file [ %s ] not used" + % (" ".join(list(binary_file_set))) + ) try: - load_vars(executor=executor, - dirname=model_path, - vars=loaded_var_list) + load_vars( + executor=executor, dirname=model_path, vars=loaded_var_list + ) except RuntimeError as e: _logger.error(e) raise e except: raise RuntimeError( "Failed to load model file, please make sure model file is saved with the " - "following APIs: save_params, save_persistables, save_vars") + "following APIs: save_params, save_persistables, save_vars" + ) return elif os.path.isfile(model_path): @@ -2026,21 +2157,26 @@ def load(program, model_path, executor=None, var_list=None): for var in var_list: if var.name not in program_var_name_set: raise LookupError( - "loaded var [{}] is not in program variable list") + "loaded var [{}] is not in program variable list" + ) dir_name, file_name = os.path.split(model_path) try: - load_vars(executor=executor, - dirname=dir_name, - vars=var_list, - filename=file_name) + load_vars( + executor=executor, + dirname=dir_name, + vars=var_list, + filename=file_name, + ) except RuntimeError as e: _logger.error(e) raise e except: - raise RuntimeError("Failed to load model file , please make sure model file is saved with the " \ - "the following APIs: [ save_params, save_persistables, save_vars ]. " \ - "When these API called, filename CANNOT be None") + raise RuntimeError( + "Failed to load model file , please make sure model file is saved with the " + "the following APIs: [ save_params, save_persistables, save_vars ]. " + "When these API called, filename CANNOT be None" + ) return @@ -2073,9 +2209,9 @@ def load(program, model_path, executor=None, var_list=None): parameter_list = list(filter(is_parameter, program.list_vars())) if executor: - paddle.fluid.core._create_loaded_parameter(parameter_list, - global_scope(), - executor._default_executor) + paddle.fluid.core._create_loaded_parameter( + parameter_list, global_scope(), executor._default_executor + ) with open(parameter_file_name, 'rb') as f: # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' @@ -2085,29 +2221,36 @@ def load(program, model_path, executor=None, var_list=None): load_dict = pickle.load(f, encoding='latin1') load_dict = _pack_loaded_dict(load_dict) for v in parameter_list: - assert v.name in load_dict, \ - "Can not find [{}] in model file [{}]".format( - v.name, parameter_file_name) + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, parameter_file_name + ) set_var(v, load_dict[v.name]) optimizer_var_list = list( - filter(is_belong_to_optimizer, program.list_vars())) + filter(is_belong_to_optimizer, program.list_vars()) + ) if len(optimizer_var_list) > 0: opt_file_name = model_prefix + ".pdopt" - assert os.path.exists(opt_file_name), \ - "Optimizer file [{}] not exits".format(opt_file_name) + assert os.path.exists( + opt_file_name + ), "Optimizer file [{}] not exits".format(opt_file_name) if executor: paddle.fluid.core._create_loaded_parameter( - optimizer_var_list, global_scope(), executor._default_executor) + optimizer_var_list, global_scope(), executor._default_executor + ) with open(opt_file_name, 'rb') as f: load_dict = pickle.load(f, encoding='latin1') for v in optimizer_var_list: - assert v.name in load_dict, \ - "Can not find [{}] in model file [{}]".format( - v.name, opt_file_name) + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, opt_file_name + ) set_var(v, load_dict[v.name]) @@ -2160,13 +2303,16 @@ def load_program_state(model_path, var_list=None): # model file saved with fluid.save is not found, try to load model file saved with # [save_vars, save_params, save_persistables] _logger.debug( - "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]" - .format(parameter_file_name)) + "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format( + parameter_file_name + ) + ) var_name_list = [] if var_list is None and os.path.isfile(model_path): raise ValueError( - "var_list can not be None when model_path is a file type") + "var_list can not be None when model_path is a file type" + ) for root, dirs, files in os.walk(model_path, topdown=False): for f in files: @@ -2187,27 +2333,34 @@ def load_program_state(model_path, var_list=None): shape=var.shape, dtype=var.dtype, type=var.type, - lod_level=var.lod_level if var.desc.type() - == core.VarDesc.VarType.LOD_TENSOR else None, - persistable=True) - - def _load_vars_with_try_catch(exe, - dirname, - vars, - filename, - raise_error=True): + lod_level=var.lod_level + if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR + else None, + persistable=True, + ) + + def _load_vars_with_try_catch( + exe, dirname, vars, filename, raise_error=True + ): try: - load_vars(executor=exe, - dirname=dirname, - vars=vars, - filename=filename) + load_vars( + executor=exe, + dirname=dirname, + vars=vars, + filename=filename, + ) return True except: - error_str = "Failed to load model/variables `%s`, please make sure " \ - "model/variables file is saved with the following APIs: " \ - "save_params, save_persistables, save_vars." - filenames = [var.name for var in vars - ] if filename is None else filename + error_str = ( + "Failed to load model/variables `%s`, please make sure " + "model/variables file is saved with the following APIs: " + "save_params, save_persistables, save_vars." + ) + filenames = ( + [var.name for var in vars] + if filename is None + else filename + ) if raise_error: raise RuntimeError(error_str % filenames) else: @@ -2224,16 +2377,19 @@ def load_program_state(model_path, var_list=None): dir_name, file_name = os.path.split(model_path) for var in var_list: loaded_var_list.append(clone_var_to_block(load_block, var)) - _load_vars_with_try_catch(exe, dir_name, loaded_var_list, - file_name) + _load_vars_with_try_catch( + exe, dir_name, loaded_var_list, file_name + ) else: # var_list can be None or not None if var_list is not None: for var in var_list: loaded_var_list.append( - clone_var_to_block(load_block, var)) - _load_vars_with_try_catch(exe, model_path, loaded_var_list, - None) + clone_var_to_block(load_block, var) + ) + _load_vars_with_try_catch( + exe, model_path, loaded_var_list, None + ) else: for var_name in var_name_list: # NOTE(chenweihang): If identify which files the user wants @@ -2241,21 +2397,25 @@ def load_program_state(model_path, var_list=None): # If a file does not exist, we only warn the user that the # file may be an irrelevant file, but does not throw an error # to ensure that other legal variables can be loaded. - temp_var = load_block.create_var(name=var_name, - persistable=True) - if _load_vars_with_try_catch(exe, model_path, - [temp_var], None, False): + temp_var = load_block.create_var( + name=var_name, persistable=True + ) + if _load_vars_with_try_catch( + exe, model_path, [temp_var], None, False + ): loaded_var_list.append(temp_var) res_dict = {} for var in loaded_var_list: res_dict[var.name] = np.asarray( - paddle.fluid.global_scope().find_var(var.name).get_tensor()) + paddle.fluid.global_scope().find_var(var.name).get_tensor() + ) return res_dict - assert os.path.exists(parameter_file_name), \ - "Parameter file [{}] not exits".format(parameter_file_name) + assert os.path.exists( + parameter_file_name + ), "Parameter file [{}] not exits".format(parameter_file_name) with open(parameter_file_name, 'rb') as f: # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' @@ -2318,25 +2478,32 @@ def set_program_state(program, state_dict): used_para_list = {} for para in parameter_list: var_temp = paddle.fluid.global_scope().find_var(para.name) - assert var_temp != None, \ - "Variable [ {} ] Not found, Please make sure run startup program".format(para.name) + assert ( + var_temp != None + ), "Variable [ {} ] Not found, Please make sure run startup program".format( + para.name + ) if para.name in state_dict: # set value from state dict orig_para_np = np.array(var_temp.get_tensor()) new_para_np = state_dict[para.name] - assert orig_para_np.shape == new_para_np.shape, \ - "Parameter's shape does not match, the Program requires a parameter with the shape of ({}), " \ - "while the loaded parameter (namely [ {} ]) has a shape of ({})." \ - .format(orig_para_np.shape, para.name, new_para_np.shape) - assert orig_para_np.dtype == new_para_np.dtype, \ - "Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), " \ - "while the loaded parameter (namely [ {} ]) has a dtype of ({})." \ - .format(orig_para_np.dtype, para.name, new_para_np.dtype) + assert orig_para_np.shape == new_para_np.shape, ( + "Parameter's shape does not match, the Program requires a parameter with the shape of ({}), " + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_para_np.shape, para.name, new_para_np.shape + ) + ) + assert orig_para_np.dtype == new_para_np.dtype, ( + "Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), " + "while the loaded parameter (namely [ {} ]) has a dtype of ({}).".format( + orig_para_np.dtype, para.name, new_para_np.dtype + ) + ) ten = var_temp.get_tensor() ten_place = ten._place() - #assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \ + # assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \ # "Place not support, only support CPUPlace and GPUPlace, now is {}".format(str(ten_place)) py_place = paddle.fluid.CPUPlace() if ten_place.is_cuda_pinned_place(): @@ -2368,5 +2535,7 @@ def set_program_state(program, state_dict): unused_para_list.append(k) if len(unused_para_list) > 0: warnings.warn( - "This list is not set, Because of Paramerter not found in program. There are: {}" - .format(" ".join(unused_para_list))) + "This list is not set, Because of Paramerter not found in program. There are: {}".format( + " ".join(unused_para_list) + ) + ) diff --git a/python/paddle/fluid/ir.py b/python/paddle/fluid/ir.py index 3e58af416d5..7dbe815b5e0 100644 --- a/python/paddle/fluid/ir.py +++ b/python/paddle/fluid/ir.py @@ -20,10 +20,12 @@ from . import core, unique_name from .framework import _apply_pass, OpProtoHolder from .proto import framework_pb2 + try: from .proto import pass_desc_pb2 except ModuleNotFoundError: import sys + sys.path.append(path.join(path.dirname(__file__), 'proto')) from .proto import pass_desc_pb2 @@ -65,9 +67,9 @@ def _update_grad_persistable(main_program): g_var.persistable = True -def apply_build_strategy(main_program, startup_program, build_strategy, - pass_attrs): - +def apply_build_strategy( + main_program, startup_program, build_strategy, pass_attrs +): def update_attr(attrs, attr_types, name, value, typ=None): if name not in attrs: attrs[name] = value @@ -80,8 +82,13 @@ def apply_build_strategy(main_program, startup_program, build_strategy, update_attr(attrs, attr_types, "nranks", 1, "size_t") update_attr(attrs, attr_types, "use_cuda", False, "bool") # TODO(zjl): how to skip fetch variables ? - update_attr(attrs, attr_types, "mem_opt_skip_vars", - get_data_vars(main_program), "list[str]") + update_attr( + attrs, + attr_types, + "mem_opt_skip_vars", + get_data_vars(main_program), + "list[str]", + ) _apply_pass(main_program, startup_program, name, attrs, attr_types) _update_grad_persistable(main_program) @@ -109,12 +116,14 @@ def apply_build_strategy(main_program, startup_program, build_strategy, apply_pass("fuse_elewise_add_act_pass") build_strategy.fuse_elewise_add_act_ops = False if build_strategy.fuse_all_optimizer_ops: - apply_pass([ - "coalesce_grad_tensor_pass", - "fuse_adam_op_pass", - "fuse_sgd_op_pass", - "fuse_momentum_op_pass", - ]) + apply_pass( + [ + "coalesce_grad_tensor_pass", + "fuse_adam_op_pass", + "fuse_sgd_op_pass", + "fuse_momentum_op_pass", + ] + ) build_strategy.fuse_all_optimizer_ops = False # TODO(zjl): support fuse all reduce ops if build_strategy.cache_runtime_context: @@ -147,8 +156,10 @@ class RegisterPassHelper(object): input_spec = self._input_specs.get(arg_name) if isinstance(input_spec, paddle.static.InputSpec): args.append( - PassDesc.VarHelper(arg_name, input_spec.shape, - input_spec.dtype)) + PassDesc.VarHelper( + arg_name, input_spec.shape, input_spec.dtype + ) + ) elif isinstance(input_spec, paddle.ParamAttr): args.append(paddle.ParamAttr(arg_name)) else: @@ -158,12 +169,15 @@ class RegisterPassHelper(object): def _prune_program_desc(self, ops): for op_desc in ops: default_attrs = core.get_op_attrs_default_value( - op_desc.type.encode()) + op_desc.type.encode() + ) remove_attrs = list() for attr in op_desc.attrs: # attr must not in if attr.name not in [ - "op_namescope", "op_callstack", "op_device" + "op_namescope", + "op_callstack", + "op_device", ]: attr_list_fields = attr.ListFields() # attr format must be: name, type, value @@ -192,8 +206,10 @@ class RegisterPassHelper(object): op_outs = out.Outputs() if len(op_outs) != 1: raise ValueError( - "Operator '{}' has multiple outputs, please specify one output variable." - .format(out._type)) + "Operator '{}' has multiple outputs, please specify one output variable.".format( + out._type + ) + ) for op_out in op_outs.values(): vars.extend(op_out) else: @@ -205,7 +221,6 @@ class RegisterPassHelper(object): return vars, program.current_block().ops def _convert_vars_to_pass_desc(self, patterns, replaces, desc): - def _add_element_conditions(conditions, elements): for element in elements: if element._condition: @@ -257,11 +272,14 @@ class RegisterPassHelper(object): pass_desc = multi_pass_desc.pass_descs.add() # Convert ProgramDescs of pattern and replace subgraphs. pattern_vars, pattern_ops = self._func_to_program_desc( - pattern, pass_desc.pattern) + pattern, pass_desc.pattern + ) replace_vars, replace_ops = self._func_to_program_desc( - replace, pass_desc.replace) - self._convert_vars_to_pass_desc(pattern_vars, replace_vars, - pass_desc) + replace, pass_desc.replace + ) + self._convert_vars_to_pass_desc( + pattern_vars, replace_vars, pass_desc + ) self._convert_ops_to_pass_desc(pattern_ops, replace_ops, pass_desc) if switch_static_mode: paddle.disable_static() @@ -269,9 +287,7 @@ class RegisterPassHelper(object): class PassDesc(object): - class AttrHelper(object): - def __init__(self, obj, name, element_index=None): self._obj = obj self._name = name @@ -283,9 +299,9 @@ class PassDesc(object): self._mapped = None def __getitem__(self, index): - element = PassDesc.AttrHelper(self._obj, - self._name, - element_index=index) + element = PassDesc.AttrHelper( + self._obj, self._name, element_index=index + ) self._elements.append(element) return element @@ -311,8 +327,9 @@ class PassDesc(object): raise NotImplementedError("Unimplemented transform operation.") def _clone_with_operation(self, type, value=None): - attr = PassDesc.AttrHelper(self._obj, self._name, - self._element_index) + attr = PassDesc.AttrHelper( + self._obj, self._name, self._element_index + ) self._elements.append(attr) if value is None: attr._operation_type = type @@ -329,19 +346,23 @@ class PassDesc(object): def __sub__(self, value): return self._clone_with_operation( - pass_desc_pb2.PassDesc.OperationType.kSub, value) + pass_desc_pb2.PassDesc.OperationType.kSub, value + ) def __add__(self, value): return self._clone_with_operation( - pass_desc_pb2.PassDesc.OperationType.kAdd, value) + pass_desc_pb2.PassDesc.OperationType.kAdd, value + ) def Mod(self, value): return self._clone_with_operation( - pass_desc_pb2.PassDesc.OperationType.kMod, value) + pass_desc_pb2.PassDesc.OperationType.kMod, value + ) def Size(self): return self._clone_with_operation( - pass_desc_pb2.PassDesc.OperationType.kSize) + pass_desc_pb2.PassDesc.OperationType.kSize + ) def _set_with_condition(self, type, value): condition = pass_desc_pb2.PassDesc.AttrCondition() @@ -356,36 +377,36 @@ class PassDesc(object): self._condition = condition def EQ(self, value): - self._set_with_condition(pass_desc_pb2.PassDesc.ConditionType.kEQ, - value) - - def MappedPattern(self, - var=None, - op=None, - index=0, - name=None, - element_index=None): + self._set_with_condition( + pass_desc_pb2.PassDesc.ConditionType.kEQ, value + ) + + def MappedPattern( + self, var=None, op=None, index=0, name=None, element_index=None + ): if all([var, op]): raise ValueError("Only mapped one of which var or op.") def mapped_var(pattern_ops): raise NotImplementedError( - "Mapping to variable is not implemented.") + "Mapping to variable is not implemented." + ) def mapped_op(pattern_ops): ops = [o for o in pattern_ops if o._type == op] if len(ops) <= index: raise ValueError( "Index '{}' of operator '{}' is incorrect.".format( - index, op)) - return PassDesc.AttrHelper(ops[index], - name, - element_index=element_index) + index, op + ) + ) + return PassDesc.AttrHelper( + ops[index], name, element_index=element_index + ) self._mapped = mapped_op if var is None else mapped_var class VarHelper(paddle.static.Variable): - def __init__(self, *args, **kwargs): block = paddle.static.default_main_program().current_block() self._var = paddle.static.data(*args, **kwargs) @@ -402,7 +423,6 @@ class PassDesc(object): return attr class OpHelper(object): - def __init__(self, type=None): self._type = type @@ -414,18 +434,23 @@ class PassDesc(object): def __call__(self, *args, **kwargs): if len(args) > 0: raise ValueError( - "Each input argument needs to specify a parameter name.") + "Each input argument needs to specify a parameter name." + ) for (in_name, in_args) in kwargs.items(): op_input = self._inputs.get(in_name) if op_input is None: raise ValueError( "Operator '{}' does not have input named '{}'.".format( - self._type, in_name)) + self._type, in_name + ) + ) if isinstance(in_args, (list, tuple)): if len(in_args) == 0: raise ValueError( - "Input '{}' of operator '{}' cannot be empty.". - format(in_name, self._type)) + "Input '{}' of operator '{}' cannot be empty.".format( + in_name, self._type + ) + ) else: in_args = [in_args] for in_arg in in_args: @@ -433,8 +458,10 @@ class PassDesc(object): op_outs = in_arg.Outputs() if len(op_outs) != 1: raise ValueError( - "The size of outputs of operator '{}' is not equal 1, please specify one output variable." - .format(in_arg._type)) + "The size of outputs of operator '{}' is not equal 1, please specify one output variable.".format( + in_arg._type + ) + ) for op_out in op_outs.values(): op_input.extend(op_out) else: @@ -453,7 +480,9 @@ class PassDesc(object): if self._proto is None: raise AttributeError( "type object 'OpHelper' has no attribute '{}'".format( - self._type)) + self._type + ) + ) self._index = len(block.ops) self._desc = block.desc.append_op() self._desc.set_type(self._type) @@ -480,7 +509,9 @@ class PassDesc(object): if output is None: raise ValueError( "Operator '{}' does not have output named '{}'.".format( - self._type, name)) + self._type, name + ) + ) return output def Outputs(self): @@ -541,7 +572,8 @@ def RegisterPass(function=None, input_specs=dict()): signature = inspect.signature(python_func) if len(signature.parameters) > 0: raise NotImplementedError( - "Pass function with parameter is not supported now.") + "Pass function with parameter is not supported now." + ) elif len(signature.parameters) == 0: pass_pairs = python_func() if _is_pass_pair(pass_pairs): diff --git a/python/paddle/fluid/layer_helper.py b/python/paddle/fluid/layer_helper.py index fc63bbea4e5..595aa656f29 100644 --- a/python/paddle/fluid/layer_helper.py +++ b/python/paddle/fluid/layer_helper.py @@ -14,7 +14,13 @@ import copy -from .framework import Parameter, dtype_is_floating, _non_static_mode, OpProtoHolder, _global_flags +from .framework import ( + Parameter, + dtype_is_floating, + _non_static_mode, + OpProtoHolder, + _global_flags, +) from . import unique_name from paddle.fluid.initializer import Constant, Xavier from .param_attr import ParamAttr @@ -25,7 +31,6 @@ from .dygraph_utils import _append_activation_in_dygraph class LayerHelper(LayerHelperBase): - def __init__(self, layer_type, **kwargs): self.kwargs = kwargs name = self.kwargs.get('name', None) @@ -35,8 +40,9 @@ class LayerHelper(LayerHelperBase): if name is None: self.kwargs['name'] = unique_name.generate(layer_type) - super(LayerHelper, self).__init__(self.kwargs['name'], - layer_type=layer_type) + super(LayerHelper, self).__init__( + self.kwargs['name'], layer_type=layer_type + ) def append_op(self, *args, **kwargs): return self.main_program.current_block().append_op(*args, **kwargs) @@ -65,7 +71,7 @@ class LayerHelper(LayerHelperBase): def bias_attr(self): return ParamAttr._to_attr(self.kwargs.get('bias_attr', None)) - #TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of param_attr + # TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of param_attr def multiple_param_attr(self, length): param_attr = self.param_attr if isinstance(param_attr, ParamAttr): @@ -93,8 +99,9 @@ class LayerHelper(LayerHelperBase): if dtype is None: dtype = each.dtype elif dtype != each.dtype: - raise ValueError("Data Type mismatch: %d to %d" % - (dtype, each.dtype)) + raise ValueError( + "Data Type mismatch: %d to %d" % (dtype, each.dtype) + ) return dtype def get_parameter(self, name): @@ -103,7 +110,7 @@ class LayerHelper(LayerHelperBase): raise ValueError("no Parameter name %s found" % name) return param - #TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of bias_attr + # TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of bias_attr def append_bias_op(self, input_var, dim_start=1, dim_end=None): """ Append bias operator and return its output. If the user does not set @@ -123,21 +130,19 @@ class LayerHelper(LayerHelperBase): if not bias_attr: return input_var - b = self.create_parameter(attr=bias_attr, - shape=size, - dtype=input_var.dtype, - is_bias=True) + b = self.create_parameter( + attr=bias_attr, shape=size, dtype=input_var.dtype, is_bias=True + ) tmp = self.create_variable_for_type_inference(dtype=input_var.dtype) - self.append_op(type='elementwise_add', - inputs={ - 'X': [input_var], - 'Y': [b] - }, - outputs={'Out': [tmp]}, - attrs={'axis': dim_start}) + self.append_op( + type='elementwise_add', + inputs={'X': [input_var], 'Y': [b]}, + outputs={'Out': [tmp]}, + attrs={'axis': dim_start}, + ) return tmp - #TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of act + # TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of act def append_activation(self, input_var): act = self.kwargs.get('act', None) if act is None: @@ -152,24 +157,27 @@ class LayerHelper(LayerHelperBase): use_cudnn = self.kwargs.get('use_cudnn') act['use_cudnn'] = use_cudnn use_mkldnn = self.kwargs.get( - 'use_mkldnn', - _global_flags().get("FLAGS_use_mkldnn", False)) + 'use_mkldnn', _global_flags().get("FLAGS_use_mkldnn", False) + ) if use_mkldnn: act['use_mkldnn'] = use_mkldnn act_type = act.pop('type') if _non_static_mode(): - res = _append_activation_in_dygraph(input_var, act_type, use_cudnn, - use_mkldnn) + res = _append_activation_in_dygraph( + input_var, act_type, use_cudnn, use_mkldnn + ) return res else: tmp = self.create_variable_for_type_inference(dtype=input_var.dtype) - self.append_op(type=act_type, - inputs={"X": [input_var]}, - outputs={"Out": [tmp]}, - attrs=act) + self.append_op( + type=act_type, + inputs={"X": [input_var]}, + outputs={"Out": [tmp]}, + attrs=act, + ) return tmp - #TODO (jiabin): should we remove this since it has never be used + # TODO (jiabin): should we remove this since it has never be used def _get_default_initializer(self, dtype): if dtype is None or dtype_is_floating(dtype) is True: return Xavier() @@ -177,9 +185,13 @@ class LayerHelper(LayerHelperBase): # For integer and boolean types, initialize with all zeros return Constant() - #TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of kwargs + # TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of kwargs def is_instance(self, param_name, cls): param = self.kwargs.get(param_name, None) if not isinstance(param, cls): - raise TypeError("The input {0} parameter of method {1} must be {2}", - param_name, self.layer_type, cls.__name__) + raise TypeError( + "The input {0} parameter of method {1} must be {2}", + param_name, + self.layer_type, + cls.__name__, + ) diff --git a/python/paddle/fluid/layer_helper_base.py b/python/paddle/fluid/layer_helper_base.py index 3a688cf9f44..1ea8d504add 100644 --- a/python/paddle/fluid/layer_helper_base.py +++ b/python/paddle/fluid/layer_helper_base.py @@ -15,7 +15,14 @@ import copy import numpy as np -from .framework import Variable, default_main_program, default_startup_program, _non_static_mode, _current_expected_place, _in_eager_without_dygraph_check +from .framework import ( + Variable, + default_main_program, + default_startup_program, + _non_static_mode, + _current_expected_place, + _in_eager_without_dygraph_check, +) from . import unique_name from .param_attr import ParamAttr, WeightNormParamAttr from . import core @@ -81,123 +88,151 @@ class LayerHelperBase(object): """ if isinstance(value, np.ndarray): if _in_eager_without_dygraph_check(): - return core.eager.Tensor(value, _current_expected_place(), - False, False, name if name else None, - True) + return core.eager.Tensor( + value, + _current_expected_place(), + False, + False, + name if name else None, + True, + ) else: - py_var = core.VarBase(value=value, - name=name if name else '', - persistable=False, - place=_current_expected_place(), - zero_copy=False) + py_var = core.VarBase( + value=value, + name=name if name else '', + persistable=False, + place=_current_expected_place(), + zero_copy=False, + ) return py_var elif isinstance(value, (core.VarBase, Variable, core.eager.Tensor)): return value else: raise TypeError( "The type of input value is invalid, expected type is 'ndarray' or 'Variable', but received %s" - % type(value)) + % type(value) + ) def _create_weight_normalize(self, attr, shape, dtype): from .layers import elementwise_mul, elementwise_div, reshape # Remove these ops when LayerHelper and layers support indicating # program and block. - def __norm_op(x, - out=None, - p=2, - dim=None, - keep_dim=False, - block=self.startup_program.global_block()): + def __norm_op( + x, + out=None, + p=2, + dim=None, + keep_dim=False, + block=self.startup_program.global_block(), + ): if out is None: out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_norm'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_norm']) + ), dtype=dtype, - persistable=False) + persistable=False, + ) abs_out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_abs'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_abs']) + ), dtype=dtype, - persistable=False) - block.append_op(type='abs', - inputs={'X': x}, - outputs={'Out': abs_out}) + persistable=False, + ) + block.append_op( + type='abs', inputs={'X': x}, outputs={'Out': abs_out} + ) pow_out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_pow'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_pow']) + ), dtype=dtype, - persistable=False) - block.append_op(type='pow', - inputs={'X': abs_out}, - outputs={'Out': pow_out}, - attrs={'factor': float(p)}) + persistable=False, + ) + block.append_op( + type='pow', + inputs={'X': abs_out}, + outputs={'Out': pow_out}, + attrs={'factor': float(p)}, + ) sum_out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_sum'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_sum']) + ), dtype=dtype, - persistable=False) - block.append_op(type='reduce_sum', - inputs={'X': pow_out}, - outputs={'Out': sum_out}, - attrs={ - 'dim': dim, - 'keep_dim': keep_dim, - 'reduce_all': True if dim is None else False - }) - block.append_op(type='pow', - inputs={'X': sum_out}, - outputs={'Out': out}, - attrs={'factor': 1. / p}) + persistable=False, + ) + block.append_op( + type='reduce_sum', + inputs={'X': pow_out}, + outputs={'Out': sum_out}, + attrs={ + 'dim': dim, + 'keep_dim': keep_dim, + 'reduce_all': True if dim is None else False, + }, + ) + block.append_op( + type='pow', + inputs={'X': sum_out}, + outputs={'Out': out}, + attrs={'factor': 1.0 / p}, + ) return out - def __reshape_op(x, - shape, - out=None, - block=self.startup_program.global_block()): + def __reshape_op( + x, shape, out=None, block=self.startup_program.global_block() + ): if out is None: out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_reshape'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_reshape']) + ), dtype=dtype, - persistable=False) + persistable=False, + ) x_shape = block.create_var(name="Xshape", dtype=x.dtype) - block.append_op(type="reshape2", - inputs={'X': x}, - attrs={'shape': shape}, - outputs={ - "Out": out, - "XShape": x_shape - }) + block.append_op( + type="reshape2", + inputs={'X': x}, + attrs={'shape': shape}, + outputs={"Out": out, "XShape": x_shape}, + ) return out - def __transpose_op(x, - axis, - out=None, - block=self.startup_program.global_block()): + def __transpose_op( + x, axis, out=None, block=self.startup_program.global_block() + ): if out is None: out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_transpose'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_transpose']) + ), dtype=dtype, - persistable=False) - block.append_op(type='transpose', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'axis': axis}) + persistable=False, + ) + block.append_op( + type='transpose', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'axis': axis}, + ) return out - def __norm_except_dim(x, - out=None, - dim=None, - block=self.startup_program.global_block()): + def __norm_except_dim( + x, out=None, dim=None, block=self.startup_program.global_block() + ): """Computes the norm over all dimensions except dim""" if out is None: out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_norm'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_norm']) + ), dtype=dtype, - persistable=False) + persistable=False, + ) if dim is None: __norm_op(x, out, dim=dim, block=block) elif dim == 0: @@ -214,11 +249,12 @@ class LayerHelperBase(object): perm = list(range(len(x.shape))) perm[0], perm[dim] = dim, 0 transpose = __transpose_op(x, perm, block=block) - out_shape = [transpose.shape[0] - ] + [1] * (len(transpose.shape) - 1) - reshape = __reshape_op(transpose, - shape=[transpose.shape[0], -1], - block=block) + out_shape = [transpose.shape[0]] + [1] * ( + len(transpose.shape) - 1 + ) + reshape = __reshape_op( + transpose, shape=[transpose.shape[0], -1], block=block + ) norm = __norm_op(reshape, dim=[1], block=block) reshape2 = __reshape_op(norm, shape=out_shape, block=block) __transpose_op(reshape2, perm, out=out, block=block) @@ -226,18 +262,22 @@ class LayerHelperBase(object): def __weight_normalize(g, v, dim): """Calculations for weight normalization""" - norm = __norm_except_dim(v, - dim=dim, - block=self.main_program.current_block()) + norm = __norm_except_dim( + v, dim=dim, block=self.main_program.current_block() + ) scale = elementwise_div( - x=g, y=norm) # The shapes of g and norm are the same. + x=g, y=norm + ) # The shapes of g and norm are the same. # Currently, elementwise_mul only support broadcast when the shape # of y is a subset of the shape of x. Thus, we reshape y to squeeze # to achieve the subset. - w = elementwise_mul(x=v, - y=scale if dim is None else reshape( - x=scale, shape=[v.shape[dim]]), - axis=-1 if dim is None else dim) + w = elementwise_mul( + x=v, + y=scale + if dim is None + else reshape(x=scale, shape=[v.shape[dim]]), + axis=-1 if dim is None else dim, + ) # To serialize the original parameter for inference, maybe a # parameter rather than a variable should be returned. return w @@ -261,39 +301,49 @@ class LayerHelperBase(object): g_param = self.startup_program.global_block().create_parameter( dtype=dtype, shape=g_param_shape, - **g_param_attr._to_kwargs(with_initializer=False)) + **g_param_attr._to_kwargs(with_initializer=False) + ) v_param = self.startup_program.global_block().create_parameter( dtype=dtype, shape=v_param_shape, - **v_param_attr._to_kwargs(with_initializer=True)) - __norm_except_dim(x=v_param, - out=g_param, - dim=attr.dim, - block=self.startup_program.global_block()) + **v_param_attr._to_kwargs(with_initializer=True) + ) + __norm_except_dim( + x=v_param, + out=g_param, + dim=attr.dim, + block=self.startup_program.global_block(), + ) # keep g_param shape to be consistent with that in main_program - __reshape_op(g_param, - g_param_shape, - out=g_param, - block=self.startup_program.global_block()) + __reshape_op( + g_param, + g_param_shape, + out=g_param, + block=self.startup_program.global_block(), + ) # Add weight normalization to main_program g_param = self.main_program.global_block().create_parameter( - dtype=dtype, shape=g_param_shape, **g_param_attr._to_kwargs()) + dtype=dtype, shape=g_param_shape, **g_param_attr._to_kwargs() + ) v_param = self.main_program.global_block().create_parameter( - dtype=dtype, shape=v_param_shape, **v_param_attr._to_kwargs()) + dtype=dtype, shape=v_param_shape, **v_param_attr._to_kwargs() + ) w_param = __weight_normalize(g_param, v_param, dim=attr.dim) return w_param # TODO: hide the func after we move the layers to Layers - def create_parameter(self, - attr, - shape, - dtype=None, - is_bias=False, - default_initializer=None, - stop_gradient=False, - type=core.VarDesc.VarType.LOD_TENSOR): + def create_parameter( + self, + attr, + shape, + dtype=None, + is_bias=False, + default_initializer=None, + stop_gradient=False, + type=core.VarDesc.VarType.LOD_TENSOR, + ): """Create parameters for this layers. Args: @@ -312,36 +362,45 @@ class LayerHelperBase(object): return None assert isinstance(attr, ParamAttr) for i, size in enumerate(shape): - assert size > 0, ("Expected every dim's size to be larger than 0, " - "but the size of the {}-th dim is {}".format( - i, size)) + assert size > 0, ( + "Expected every dim's size to be larger than 0, " + "but the size of the {}-th dim is {}".format(i, size) + ) # set global dtype if not dtype: dtype = self.__dtype if is_bias: suffix = 'b' - default_initializer = _global_bias_initializer( - ) if _global_bias_initializer() is not None else default_initializer + default_initializer = ( + _global_bias_initializer() + if _global_bias_initializer() is not None + else default_initializer + ) else: suffix = 'w' - default_initializer = _global_weight_initializer( - ) if _global_weight_initializer( - ) is not None else default_initializer + default_initializer = ( + _global_weight_initializer() + if _global_weight_initializer() is not None + else default_initializer + ) if attr.name is None: attr.name = unique_name.generate(".".join([self.name, suffix])) if default_initializer is None and attr.initializer is None: if isinstance(dtype, core.VarDesc.VarType): - if dtype != core.VarDesc.VarType.FP32 and \ - dtype != core.VarDesc.VarType.FP64 and \ - dtype != core.VarDesc.VarType.FP16 and \ - dtype != core.VarDesc.VarType.BF16: + if ( + dtype != core.VarDesc.VarType.FP32 + and dtype != core.VarDesc.VarType.FP64 + and dtype != core.VarDesc.VarType.FP16 + and dtype != core.VarDesc.VarType.BF16 + ): raise TypeError( "Can not create parameter with default initializer when dtype is not float type. Set default_initializer to fit the parameter dtype!" ) else: - if not (dtype.startswith("float") - or dtype in ["double", "uint16"]): + if not ( + dtype.startswith("float") or dtype in ["double", "uint16"] + ): raise TypeError( "Can not create parameter with default initializer when dtype is not float type. Set default_initializer to fit the parameter dtype!" ) @@ -368,26 +427,29 @@ class LayerHelperBase(object): "parameter name [{}] have be been used. " "In dygraph mode, the name of parameter can't be same." "Please check the parameter attr value passed to self.create_parameter or " - "constructor of dygraph Layers".format(attr.name)) + "constructor of dygraph Layers".format(attr.name) + ) return self.main_program.global_block().create_parameter( dtype=dtype, shape=shape, type=type, stop_gradient=stop_gradient, - **attr._to_kwargs(with_initializer=True)) + **attr._to_kwargs(with_initializer=True) + ) else: self.startup_program.global_block().create_parameter( dtype=dtype, shape=shape, type=type, - **attr._to_kwargs(with_initializer=True)) + **attr._to_kwargs(with_initializer=True) + ) return self.main_program.global_block().create_parameter( - dtype=dtype, shape=shape, type=type, **attr._to_kwargs()) + dtype=dtype, shape=shape, type=type, **attr._to_kwargs() + ) - def create_variable_for_type_inference(self, - dtype, - stop_gradient=False, - shape=None): + def create_variable_for_type_inference( + self, dtype, stop_gradient=False, shape=None + ): """Create a temporary variable that should be type inferred layer. Note: @@ -400,18 +462,19 @@ class LayerHelperBase(object): if not dtype: dtype = self.__dtype return self.main_program.current_block().create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'tmp']) + ), dtype=dtype, shape=shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=stop_gradient) + stop_gradient=stop_gradient, + ) - def create_sparse_variable_for_type_inference(self, - dtype, - stop_gradient=False, - shape=None): + def create_sparse_variable_for_type_inference( + self, dtype, stop_gradient=False, shape=None + ): """Create a temporary sparse variable that should be type inferred layer. Note: @@ -424,13 +487,15 @@ class LayerHelperBase(object): if not dtype: dtype = self.__dtype return self.main_program.current_block().create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'tmp']) + ), dtype=dtype, shape=shape, type=core.VarDesc.VarType.SPARSE_COO, persistable=False, - stop_gradient=stop_gradient) + stop_gradient=stop_gradient, + ) def create_variable(self, *args, **kwargs): """Create Variable for this layers. @@ -449,7 +514,8 @@ class LayerHelperBase(object): Returns(Variable): the created variable. """ return self.main_program.global_block().create_var( - *args, persistable=persistable, **kwargs) + *args, persistable=persistable, **kwargs + ) def create_or_get_global_variable(self, name, *args, **kwargs): """ @@ -464,9 +530,9 @@ class LayerHelperBase(object): def set_variable_initializer(self, var, initializer): """Set target Variable's initializer - Args: - var: target Variable - initializer: initializer to use + Args: + var: target Variable + initializer: initializer to use """ assert isinstance(var, Variable) if _non_static_mode(): @@ -478,4 +544,5 @@ class LayerHelperBase(object): dtype=var.dtype, shape=var.shape, persistable=True, - initializer=initializer) + initializer=initializer, + ) diff --git a/python/paddle/fluid/layers/collective.py b/python/paddle/fluid/layers/collective.py index df3147a27cd..7764235dc2f 100644 --- a/python/paddle/fluid/layers/collective.py +++ b/python/paddle/fluid/layers/collective.py @@ -35,40 +35,38 @@ def _allreduce(x, out=None, reduce_type="sum", sync_mode=False): if out is None: out = helper.create_variable( - name=unique_name.generate_with_ignorable_key(".".join( - [x.name, 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join([x.name, 'tmp']) + ), shape=x.shape, dtype=x.dtype, type=x.type, persistable=x.persistable, - stop_gradient=True) - helper.append_op(type='allreduce', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - "reduce_type": red_typ_int, - "sync_mode": sync_mode - }) + stop_gradient=True, + ) + helper.append_op( + type='allreduce', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={"reduce_type": red_typ_int, "sync_mode": sync_mode}, + ) return out def _broadcast(x, root, sync_mode=False): helper = LayerHelper("broadcast", **locals()) - helper.append_op(type='broadcast', - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={ - "sync_mode": sync_mode, - "root": root - }) + helper.append_op( + type='broadcast', + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={"sync_mode": sync_mode, "root": root}, + ) return x -def _c_allreduce(x, - out=None, - reduce_type='sum', - ring_id=0, - use_calc_stream=False): +def _c_allreduce( + x, out=None, reduce_type='sum', ring_id=0, use_calc_stream=False +): helper = LayerHelper('c_allreduce', **locals()) if reduce_type not in ['sum', 'prob', 'max', 'min']: @@ -77,34 +75,37 @@ def _c_allreduce(x, op_type = 'c_allreduce_' + reduce_type if out is None: out = helper.create_variable( - name=unique_name.generate_with_ignorable_key('.'.join( - [x.name, op_type])), + name=unique_name.generate_with_ignorable_key( + '.'.join([x.name, op_type]) + ), shape=x.shape, dtype=x.dtype, type=x.type, - persistable=x.persistable) - - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + persistable=x.persistable, + ) + + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'ring_id': ring_id, 'use_calc_stream': use_calc_stream}, + ) return out def _c_broadcast(x, root=0, ring_id=0, use_calc_stream=False): op_type = 'c_broadcast' helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={ - 'root': root, - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={ + 'root': root, + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + }, + ) return x @@ -121,28 +122,39 @@ def _c_allgather(x, nranks, ring_id=0, use_calc_stream=False): return out if _in_legacy_dygraph(): - attrs = ('nranks', nranks, 'ring_id', ring_id, 'use_calc_stream', - use_calc_stream) + attrs = ( + 'nranks', + nranks, + 'ring_id', + ring_id, + 'use_calc_stream', + use_calc_stream, + ) return _legacy_C_ops.c_allgather(x, *attrs) helper = LayerHelper(op_type, **locals()) out_shape = list(x.shape[:]) if out_shape[0] > 0: out_shape[0] *= nranks - out = helper.create_variable(name=unique_name.generate_with_ignorable_key( - '.'.join([x.name, op_type])), - shape=out_shape, - dtype=x.dtype, - type=x.type, - persistable=x.persistable) - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'nranks': nranks, - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + out = helper.create_variable( + name=unique_name.generate_with_ignorable_key( + '.'.join([x.name, op_type]) + ), + shape=out_shape, + dtype=x.dtype, + type=x.type, + persistable=x.persistable, + ) + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={ + 'nranks': nranks, + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + }, + ) return out @@ -152,28 +164,34 @@ def _c_reducescatter(x, nranks, ring_id=0, use_calc_stream=False): if x.shape[0] > 0 and x.shape[0] % nranks != 0: raise ValueError( - 'x.shape[0](%d) cannot be evenly divided by nranks(%d)' % - (x.shape[0], nranks)) + 'x.shape[0](%d) cannot be evenly divided by nranks(%d)' + % (x.shape[0], nranks) + ) op_type = 'c_reducescatter' helper = LayerHelper(op_type, **locals()) out_shape = list(x.shape[:]) if out_shape[0] > 0: out_shape[0] //= nranks - out = helper.create_variable(name=unique_name.generate_with_ignorable_key( - '.'.join([x.name, op_type])), - shape=out_shape, - dtype=x.dtype, - type=x.type, - persistable=x.persistable) - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'nranks': nranks, - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + out = helper.create_variable( + name=unique_name.generate_with_ignorable_key( + '.'.join([x.name, op_type]) + ), + shape=out_shape, + dtype=x.dtype, + type=x.type, + persistable=x.persistable, + ) + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={ + 'nranks': nranks, + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + }, + ) return out @@ -187,8 +205,10 @@ def _c_sync_calc_stream(x): def _c_sync_comm_stream(x, ring_id): op_type = 'c_sync_comm_stream' helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={'ring_id': ring_id}) + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={'ring_id': ring_id}, + ) return x diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index b800e67fe70..eb654458fbf 100755 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -17,24 +17,66 @@ from ..wrapped_decorator import signature_safe_contextmanager from .layer_function_generator import autodoc, templatedoc from .tensor import assign, cast, fill_constant from .. import core -from ..framework import Program, Variable, Operator, _non_static_mode, static_only, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Program, + Variable, + Operator, + _non_static_mode, + static_only, + _in_legacy_dygraph, + in_dygraph_mode, +) from ..layer_helper import LayerHelper, unique_name from .nn import logical_and, logical_not, logical_or -from .utils import assert_same_structure, map_structure, hold_mutable_vars, copy_mutable_vars, padding_to_same_structure, is_sequence, pack_sequence_as, flatten, to_sequence +from .utils import ( + assert_same_structure, + map_structure, + hold_mutable_vars, + copy_mutable_vars, + padding_to_same_structure, + is_sequence, + pack_sequence_as, + flatten, + to_sequence, +) import numpy import warnings from functools import reduce, partial -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from ... import compat as cpt from ..backward import _infer_var_data_type_shape_ from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'While', 'Switch', 'increment', 'array_write', 'create_array', 'less_than', - 'less_equal', 'greater_than', 'greater_equal', 'equal', 'not_equal', - 'array_read', 'array_length', 'cond', 'IfElse', 'DynamicRNN', 'StaticRNN', - 'reorder_lod_tensor_by_rank', 'Print', 'Assert', 'is_empty', 'case', - 'switch_case', 'while_loop' + 'While', + 'Switch', + 'increment', + 'array_write', + 'create_array', + 'less_than', + 'less_equal', + 'greater_than', + 'greater_equal', + 'equal', + 'not_equal', + 'array_read', + 'array_length', + 'cond', + 'IfElse', + 'DynamicRNN', + 'StaticRNN', + 'reorder_lod_tensor_by_rank', + 'Print', + 'Assert', + 'is_empty', + 'case', + 'switch_case', + 'while_loop', ] @@ -59,12 +101,11 @@ def select_output(input, outputs, mask): check_variable_and_dtype(mask, 'mask', ['int32'], 'select_output') check_type(outputs, 'outputs', (list, tuple), 'select_output') - helper.append_op(type='select_output', - inputs={ - 'X': input, - 'Mask': mask - }, - outputs={'Out': outputs}) + helper.append_op( + type='select_output', + inputs={'X': input, 'Mask': mask}, + outputs={'Out': outputs}, + ) return outputs @@ -83,7 +124,8 @@ def _select_input_infer_shape(first_shape, second_shape): ) return second_shape out_shape = list( - map(lambda a, b: a if a == b else -1, first_shape, second_shape)) + map(lambda a, b: a if a == b else -1, first_shape, second_shape) + ) return out_shape @@ -107,32 +149,34 @@ def select_input(inputs, mask): check_variable_and_dtype(mask, 'mask', ['int32'], 'select_input') # Select input should expand the shape. If it is - 1 and valid number, use - 1 first. If the dim is different, an error will be reported directly - #assert inputs[0].dtype == inputs[1].dtype, f"Expect the inputs should have the same dtype, but get {inputs[0].dtype} and {inputs[1].dtype}" + # assert inputs[0].dtype == inputs[1].dtype, f"Expect the inputs should have the same dtype, but get {inputs[0].dtype} and {inputs[1].dtype}" output_shape = _select_input_infer_shape(inputs[0].shape, inputs[1].shape) output_dtype = inputs[1].dtype output_type = inputs[1].type - out = helper.create_variable(dtype=output_dtype, - shape=output_shape, - type=output_type) - helper.append_op(type='select_input', - inputs={ - 'X': inputs, - 'Mask': mask - }, - outputs={'Out': out}) + out = helper.create_variable( + dtype=output_dtype, shape=output_shape, type=output_type + ) + helper.append_op( + type='select_input', + inputs={'X': inputs, 'Mask': mask}, + outputs={'Out': out}, + ) return out def select_input_with_buildin_type(inputs, mask, name): - from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable + from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + to_static_variable, + ) from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar + false_var, true_var = inputs if isinstance(false_var, UndefinedVar) and isinstance( - true_var, UndefinedVar): - """ None -> UndefinedVar, so the real value is a [None, UndefinedVar] or [None, None], we just return None. - """ + true_var, UndefinedVar + ): + """None -> UndefinedVar, so the real value is a [None, UndefinedVar] or [None, None], we just return None.""" return None if isinstance(false_var, Variable) and isinstance(true_var, Variable): @@ -140,50 +184,63 @@ def select_input_with_buildin_type(inputs, mask, name): return select_input(inputs, mask) except Exception as e: raise RuntimeError( - f"Exceptions throwed while doing select_input on {name}:\n{e}") + f"Exceptions throwed while doing select_input on {name}:\n{e}" + ) - elif (isinstance(false_var, support_ret_buildin_type) - and isinstance(false_var, type(true_var))): + elif isinstance(false_var, support_ret_buildin_type) and isinstance( + false_var, type(true_var) + ): if false_var == true_var: return false_var else: inputs = [ to_static_variable(false_var), - to_static_variable(true_var) + to_static_variable(true_var), ] # Deal with the situations like this: false_var is int and true_var is Variable - elif ((isinstance(false_var, support_ret_buildin_type) - and isinstance(true_var, Variable)) - or (isinstance(true_var, support_ret_buildin_type) - and isinstance(false_var, Variable))): + elif ( + isinstance(false_var, support_ret_buildin_type) + and isinstance(true_var, Variable) + ) or ( + isinstance(true_var, support_ret_buildin_type) + and isinstance(false_var, Variable) + ): inputs = [to_static_variable(false_var), to_static_variable(true_var)] warnings.warn( "Return results from different branches in cond are not same type: " "false_var returned by fasle_fn is '{}' and true_var of true_fn is " - "'{}'".format(type(false_var), type(true_var))) - elif ((isinstance(false_var, UndefinedVar) - and isinstance(true_var, (Variable, ) + support_ret_buildin_type)) - or (isinstance(true_var, UndefinedVar) - and isinstance(false_var, - (Variable, ) + support_ret_buildin_type))): + "'{}'".format(type(false_var), type(true_var)) + ) + elif ( + isinstance(false_var, UndefinedVar) + and isinstance(true_var, (Variable,) + support_ret_buildin_type) + ) or ( + isinstance(true_var, UndefinedVar) + and isinstance(false_var, (Variable,) + support_ret_buildin_type) + ): def create_var_if_not_undefined_var(a): - if isinstance(a, UndefinedVar): return a + if isinstance(a, UndefinedVar): + return a return to_static_variable(a) true_var, false_var = to_static_variable(true_var), to_static_variable( - false_var) + false_var + ) inputs = [false_var, true_var] else: raise TypeError( "Unsupported return type of true_fn and false_fn in cond: false_var " - "returned by fasle_fn is '{}' and true_var of true_fn is '{}'". - format(type(false_var), type(true_var))) + "returned by fasle_fn is '{}' and true_var of true_fn is '{}'".format( + type(false_var), type(true_var) + ) + ) try: return select_input(inputs, mask) except Exception as e: raise RuntimeError( - f"Exceptions throwed while doing select_input on {name}:\n{e}") + f"Exceptions throwed while doing select_input on {name}:\n{e}" + ) def split_lod_tensor(input, mask, level=0): @@ -220,23 +277,26 @@ def split_lod_tensor(input, mask, level=0): input=x, mask=y, level=level) """ - check_type(input, 'input', (Variable, list, tuple, type(None)), - 'fluid.layers.split_lod_tensor') + check_type( + input, + 'input', + (Variable, list, tuple, type(None)), + 'fluid.layers.split_lod_tensor', + ) check_type(mask, 'mask', (Variable, list), 'fluid.layers.split_lod_tensor') check_type(level, 'level', int, 'fluid.layers.split_lod_tensor') helper = LayerHelper('split_lod_tensor', **locals()) out_true = helper.create_variable_for_type_inference(dtype=input.dtype) out_false = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='split_lod_tensor', - inputs={ - 'X': input, - 'Mask': mask, - }, - outputs={ - 'OutTrue': out_true, - 'OutFalse': out_false - }, - attrs={'level': level}) + helper.append_op( + type='split_lod_tensor', + inputs={ + 'X': input, + 'Mask': mask, + }, + outputs={'OutTrue': out_true, 'OutFalse': out_false}, + attrs={'level': level}, + ) return out_true, out_false @@ -278,37 +338,48 @@ def merge_lod_tensor(in_true, in_false, x, mask, level=0): in_true=out_true, in_false=out_false, mask=y, x=x, level=level) """ helper = LayerHelper('merge_lod_tensor', **locals()) - check_type(x, 'x', (Variable, list, tuple, type(None)), - 'fluid.layers.merge_lod_tensor') + check_type( + x, + 'x', + (Variable, list, tuple, type(None)), + 'fluid.layers.merge_lod_tensor', + ) check_type(mask, 'mask', (Variable, list), 'fluid.layers.merge_lod_tensor') - check_type(in_true, 'in_true', (Variable, list, tuple, type(None)), - 'fluid.layers.merge_lod_tensor') - check_type(in_false, 'in_false', (Variable, list, tuple, type(None)), - 'fluid.layers.merge_lod_tensor') + check_type( + in_true, + 'in_true', + (Variable, list, tuple, type(None)), + 'fluid.layers.merge_lod_tensor', + ) + check_type( + in_false, + 'in_false', + (Variable, list, tuple, type(None)), + 'fluid.layers.merge_lod_tensor', + ) out = helper.create_variable_for_type_inference(dtype=in_true.dtype) - helper.append_op(type='merge_lod_tensor', - inputs={ - 'X': x, - 'Mask': mask, - 'InTrue': in_true, - 'InFalse': in_false - }, - outputs={'Out': out}, - attrs={'level': level}) + helper.append_op( + type='merge_lod_tensor', + inputs={'X': x, 'Mask': mask, 'InTrue': in_true, 'InFalse': in_false}, + outputs={'Out': out}, + attrs={'level': level}, + ) return out @static_only -def Print(input, - first_n=-1, - message=None, - summarize=20, - print_tensor_name=True, - print_tensor_type=True, - print_tensor_shape=True, - print_tensor_layout=True, - print_tensor_lod=True, - print_phase='both'): +def Print( + input, + first_n=-1, + message=None, + summarize=20, + print_tensor_name=True, + print_tensor_type=True, + print_tensor_shape=True, + print_tensor_layout=True, + print_tensor_lod=True, + print_phase='both', +): ''' :api_attr: Static Graph @@ -366,26 +437,31 @@ def Print(input, # - dtype: long # - data: [3 3 3 3 3 3] ''' - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64', 'bool'], - 'fluid.layers.Print') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64', 'int32', 'int64', 'bool'], + 'fluid.layers.Print', + ) helper = LayerHelper('print' + "_" + input.name, **locals()) output = helper.create_variable_for_type_inference(input.dtype) - helper.append_op(type='print', - inputs={'In': input}, - outputs={'Out': output}, - attrs={ - 'first_n': first_n, - 'summarize': summarize, - 'message': message or "", - 'print_tensor_name': print_tensor_name, - 'print_tensor_type': print_tensor_type, - 'print_tensor_shape': print_tensor_shape, - 'print_tensor_layout': print_tensor_layout, - 'print_tensor_lod': print_tensor_lod, - 'print_phase': print_phase.upper() - }) + helper.append_op( + type='print', + inputs={'In': input}, + outputs={'Out': output}, + attrs={ + 'first_n': first_n, + 'summarize': summarize, + 'message': message or "", + 'print_tensor_name': print_tensor_name, + 'print_tensor_type': print_tensor_type, + 'print_tensor_shape': print_tensor_shape, + 'print_tensor_layout': print_tensor_layout, + 'print_tensor_lod': print_tensor_lod, + 'print_phase': print_phase.upper(), + }, + ) return output @@ -452,12 +528,11 @@ def Assert(cond, data=None, summarize=20, name=None): layer_name = name if name else ('assert_' + cond.name) helper = LayerHelper(layer_name, **locals()) - op = helper.append_op(type="assert", - inputs={ - "Cond": cond, - "Data": [] if data is None else list(data) - }, - attrs={"summarize": summarize}) + op = helper.append_op( + type="assert", + inputs={"Cond": cond, "Data": [] if data is None else list(data)}, + attrs={"summarize": summarize}, + ) return op @@ -507,8 +582,9 @@ class BlockGuardWithCompletion(BlockGuard): return False self.rnn.status = StaticRNN.AFTER_RNN_BLOCK self.rnn._complete_op() - return super(BlockGuardWithCompletion, - self).__exit__(exc_type, exc_val, exc_tb) + return super(BlockGuardWithCompletion, self).__exit__( + exc_type, exc_val, exc_tb + ) class StaticRNNMemoryLink(object): @@ -580,6 +656,7 @@ class StaticRNN(object): result = rnn() """ + BEFORE_RNN_BLOCK = 0 IN_RNN_BLOCK = 1 AFTER_RNN_BLOCK = 2 @@ -605,13 +682,15 @@ class StaticRNN(object): if self.status != StaticRNN.IN_RNN_BLOCK: raise ValueError("You must invoke {0} in rnn block".format(method)) - def memory(self, - init=None, - shape=None, - batch_ref=None, - init_value=0.0, - init_batch_dim_idx=0, - ref_batch_dim_idx=1): + def memory( + self, + init=None, + shape=None, + batch_ref=None, + init_value=0.0, + init_batch_dim_idx=0, + ref_batch_dim_idx=1, + ): """ Create a memory variable for static rnn. If the :code:`init` is not None, :code:`memory` will be initialized by @@ -637,97 +716,118 @@ class StaticRNN(object): Examples 1: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # create prev memory parameter, batch size comes from word - prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # use hidden to update prev - rnn.update_memory(prev, hidden) + import paddle.fluid as fluid + import paddle.fluid.layers as layers + + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # create prev memory parameter, batch size comes from word + prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # use hidden to update prev + rnn.update_memory(prev, hidden) Examples 2: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - boot_memory = fluid.layers.data(name='boot', shape=[hidden_size], dtype='float32', lod_level=1) - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # init memory - prev = rnn.memory(init=boot_memory) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # update hidden with prev - rnn.update_memory(prev, hidden) + import paddle.fluid as fluid + import paddle.fluid.layers as layers + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + boot_memory = fluid.layers.data(name='boot', shape=[hidden_size], dtype='float32', lod_level=1) + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # init memory + prev = rnn.memory(init=boot_memory) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # update hidden with prev + rnn.update_memory(prev, hidden) """ self._assert_in_rnn_block_('memory') - check_type(init, "init", (Variable, type(None)), - "fluid.layers.StaticRNN.memory") - check_type(shape, "shape", (list, tuple, type(None)), - "fluid.layers.StaticRNN.memory") - check_type(batch_ref, "batch_ref", (Variable, type(None)), - "fluid.layers.StaticRNN.memory") + check_type( + init, + "init", + (Variable, type(None)), + "fluid.layers.StaticRNN.memory", + ) + check_type( + shape, + "shape", + (list, tuple, type(None)), + "fluid.layers.StaticRNN.memory", + ) + check_type( + batch_ref, + "batch_ref", + (Variable, type(None)), + "fluid.layers.StaticRNN.memory", + ) if init is None: if shape is None or batch_ref is None: raise ValueError( - "if init is None, memory at least need shape and batch_ref") + "if init is None, memory at least need shape and batch_ref" + ) parent_block = self._parent_block() - var_name = unique_name.generate_with_ignorable_key("@".join( - [self.helper.name, "memory_boot"])) - boot_var = parent_block.create_var(name=var_name, - shape=shape, - dtype=batch_ref.dtype, - persistable=False) - - parent_block.append_op(type="fill_constant_batch_size_like", - inputs={'Input': [batch_ref]}, - outputs={'Out': [boot_var]}, - attrs={ - 'value': init_value, - 'shape': boot_var.shape, - 'dtype': boot_var.dtype, - 'input_dim_idx': ref_batch_dim_idx, - 'output_dim_idx': init_batch_dim_idx - }) + var_name = unique_name.generate_with_ignorable_key( + "@".join([self.helper.name, "memory_boot"]) + ) + boot_var = parent_block.create_var( + name=var_name, + shape=shape, + dtype=batch_ref.dtype, + persistable=False, + ) + + parent_block.append_op( + type="fill_constant_batch_size_like", + inputs={'Input': [batch_ref]}, + outputs={'Out': [boot_var]}, + attrs={ + 'value': init_value, + 'shape': boot_var.shape, + 'dtype': boot_var.dtype, + 'input_dim_idx': ref_batch_dim_idx, + 'output_dim_idx': init_batch_dim_idx, + }, + ) return self.memory(init=boot_var) else: pre_mem = self.helper.create_variable( - name=unique_name.generate_with_ignorable_key("@".join( - [self.helper.name, "mem"])), + name=unique_name.generate_with_ignorable_key( + "@".join([self.helper.name, "mem"]) + ), dtype=init.dtype, - shape=init.shape) - self.memories[pre_mem.name] = StaticRNNMemoryLink(init=init, - pre_mem=pre_mem) + shape=init.shape, + ) + self.memories[pre_mem.name] = StaticRNNMemoryLink( + init=init, pre_mem=pre_mem + ) return pre_mem def step_input(self, x): @@ -744,29 +844,29 @@ class StaticRNN(object): Examples: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # create prev memory parameter, batch size comes from word - prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # use hidden to update prev - rnn.update_memory(prev, hidden) + import paddle.fluid as fluid + import paddle.fluid.layers as layers + + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # create prev memory parameter, batch size comes from word + prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # use hidden to update prev + rnn.update_memory(prev, hidden) """ self._assert_in_rnn_block_('step_input') @@ -776,10 +876,9 @@ class StaticRNN(object): elif x.shape[0] != -1 and self.seq_len != x.shape[0]: raise ValueError("Static RNN only take fix seq_len input") - ipt = self.helper.create_variable(name=x.name, - dtype=x.dtype, - shape=list(x.shape[1:]), - type=x.type) + ipt = self.helper.create_variable( + name=x.name, dtype=x.dtype, shape=list(x.shape[1:]), type=x.type + ) self.inputs.append(ipt) return ipt @@ -796,47 +895,50 @@ class StaticRNN(object): Examples: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # create prev memory parameter, batch size comes from word - prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # use hidden to update prev - rnn.update_memory(prev, hidden) - rnn.step_output(hidden) - - result = rnn() + import paddle.fluid as fluid + import paddle.fluid.layers as layers + + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # create prev memory parameter, batch size comes from word + prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # use hidden to update prev + rnn.update_memory(prev, hidden) + rnn.step_output(hidden) + + result = rnn() """ self._assert_in_rnn_block_('step_output') check_type(o, "o", Variable, "fluid.layers.StaticRNN.step_output") tmp_o = self.helper.create_variable_for_type_inference(dtype=o.dtype) - self.helper.append_op(type='rnn_memory_helper', - inputs={'X': [o]}, - outputs={'Out': tmp_o}, - attrs={'dtype': o.dtype}) + self.helper.append_op( + type='rnn_memory_helper', + inputs={'X': [o]}, + outputs={'Out': tmp_o}, + attrs={'dtype': o.dtype}, + ) - out_var = self._parent_block().create_var(name=tmp_o.name, - shape=[self.seq_len] + - list(tmp_o.shape), - dtype=tmp_o.dtype) + out_var = self._parent_block().create_var( + name=tmp_o.name, + shape=[self.seq_len] + list(tmp_o.shape), + dtype=tmp_o.dtype, + ) self.outputs.append(out_var) @@ -853,33 +955,33 @@ class StaticRNN(object): Examples: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # create prev memory parameter, batch size comes from word - prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # use hidden to update prev - rnn.update_memory(prev, hidden) - # mark each step's hidden and word as output - rnn.output(hidden, word) - - result = rnn() + import paddle.fluid as fluid + import paddle.fluid.layers as layers + + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # create prev memory parameter, batch size comes from word + prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # use hidden to update prev + rnn.update_memory(prev, hidden) + # mark each step's hidden and word as output + rnn.output(hidden, word) + + result = rnn() """ for each in outputs: self.step_output(each) @@ -952,7 +1054,8 @@ class StaticRNN(object): ] step_scope = parent_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) + type=core.VarDesc.VarType.STEP_SCOPES + ) inlinks = [parent_block.var(i.name) for i in self.inputs] outlinks = self.outputs @@ -964,39 +1067,41 @@ class StaticRNN(object): for _, mem in self.memories.items(): boot_memories.append(mem.init) pre_memories.append(mem.pre_mem.name) - assert mem.mem is not None, "%s should be updated in every step." % ( - mem.init.name) + assert ( + mem.mem is not None + ), "%s should be updated in every step." % (mem.init.name) mem_var = rnn_block.var(mem.mem.name) assert isinstance(mem_var, Variable) new_mem = self.helper.create_variable_for_type_inference( - dtype=mem_var.dtype) - rnn_block.append_op(type='rnn_memory_helper', - inputs={'X': [mem_var]}, - outputs={'Out': [new_mem]}, - attrs={'dtype': mem_var.dtype}) + dtype=mem_var.dtype + ) + rnn_block.append_op( + type='rnn_memory_helper', + inputs={'X': [mem_var]}, + outputs={'Out': [new_mem]}, + attrs={'dtype': mem_var.dtype}, + ) memories.append(new_mem.name) - parent_block.append_op(type='recurrent', - inputs={ - 'inputs': inlinks, - 'initial_states': boot_memories, - 'parameters': parameters - }, - outputs={ - 'outputs': outlinks, - 'step_scopes': [step_scope] - }, - attrs={ - 'has_states': len(pre_memories) > 0, - 'ex_states': pre_memories, - 'states': memories, - 'sub_block': rnn_block - }) + parent_block.append_op( + type='recurrent', + inputs={ + 'inputs': inlinks, + 'initial_states': boot_memories, + 'parameters': parameters, + }, + outputs={'outputs': outlinks, 'step_scopes': [step_scope]}, + attrs={ + 'has_states': len(pre_memories) > 0, + 'ex_states': pre_memories, + 'states': memories, + 'sub_block': rnn_block, + }, + ) class WhileGuard(BlockGuard): - def __init__(self, while_op): if not isinstance(while_op, While): raise TypeError("WhileGuard takes a while op") @@ -1015,8 +1120,9 @@ class WhileGuard(BlockGuard): return super(WhileGuard, self).__exit__(exc_type, exc_val, exc_tb) -def get_inputs_outputs_in_block(current_block, inner_inputs, inner_outputs, - helper): +def get_inputs_outputs_in_block( + current_block, inner_inputs, inner_outputs, helper +): """ Find inputs and outputs in current control flow block. :param current_block: Current control flow block. @@ -1047,7 +1153,8 @@ def get_inputs_outputs_in_block(current_block, inner_inputs, inner_outputs, for iname in op.input_names: for in_var_name in op.input(iname): if in_var_name not in inner_outputs and not is_ignore_vars( - op, in_var_name): + op, in_var_name + ): inner_inputs.add(in_var_name) for oname in op.output_names: @@ -1063,8 +1170,11 @@ def get_inputs_outputs_in_block(current_block, inner_inputs, inner_outputs, current_block_var = None if current_block.has_var(in_var_name): current_block_var = current_block.var(in_var_name) - if not parent_block_var and current_block_var and \ - current_block_var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not parent_block_var + and current_block_var + and current_block_var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): remove_inner_inputs.add(in_var_name) inner_inputs = inner_inputs - remove_inner_inputs @@ -1154,8 +1264,10 @@ class While(object): check_variable_and_dtype(cond, 'cond', ['bool'], 'fluid.layers.While') if reduce(lambda a, b: a * b, cond.shape, 1) != 1: raise TypeError( - "condition expected shape as [1], but given shape as {0}.". - format(list(cond.shape))) + "condition expected shape as [1], but given shape as {0}.".format( + list(cond.shape) + ) + ) self.cond_var = cond self.is_test = is_test @@ -1166,12 +1278,14 @@ class While(object): main_program = self.helper.main_program while_block = main_program.current_block() parent_block = main_program.block( - main_program.current_block().parent_idx) + main_program.current_block().parent_idx + ) inner_outputs = {self.cond_var.name} x_name_list = set() x_name_list, inner_outputs = get_inputs_outputs_in_block( - while_block, x_name_list, inner_outputs, self.helper) + while_block, x_name_list, inner_outputs, self.helper + ) out_vars = [] for inner_out_name in inner_outputs: @@ -1185,23 +1299,21 @@ class While(object): x_name_list -= {self.cond_var.name} step_scope = parent_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) + type=core.VarDesc.VarType.STEP_SCOPES + ) parent_block.append_op( type='while', inputs={ - 'X': - [parent_block._var_recursive(x_name) for x_name in x_name_list], - 'Condition': [self.cond_var] - }, - outputs={ - 'Out': out_vars, - 'StepScopes': [step_scope] + 'X': [ + parent_block._var_recursive(x_name) + for x_name in x_name_list + ], + 'Condition': [self.cond_var], }, - attrs={ - 'sub_block': while_block, - "is_test": self.is_test - }) + outputs={'Out': out_vars, 'StepScopes': [step_scope]}, + attrs={'sub_block': while_block, "is_test": self.is_test}, + ) support_ret_buildin_type = (bool, float, int) @@ -1213,14 +1325,17 @@ def assign_skip_lod_tensor_array(input, output): """ def has_shape_diff(x_var, y_var): - if len(x_var.shape) != len(y_var.shape): return True + if len(x_var.shape) != len(y_var.shape): + return True for x_dim, y_dim in zip(x_var.shape, y_var.shape): - if x_dim != y_dim and -1 not in [x_dim, y_dim]: return True + if x_dim != y_dim and -1 not in [x_dim, y_dim]: + return True return False if not isinstance(input, (Variable, core.VarBase)): if isinstance(output, Variable) and isinstance( - input, support_ret_buildin_type): + input, support_ret_buildin_type + ): assign(input, output) else: output = input @@ -1229,15 +1344,21 @@ def assign_skip_lod_tensor_array(input, output): if input.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: main_program = input.block.program parent_block = main_program.block( - main_program.current_block().parent_idx) + main_program.current_block().parent_idx + ) if parent_block and not parent_block._find_var_recursive(input.name): assign(input, output) else: - if isinstance(output, Variable) and isinstance( - input, Variable) and has_shape_diff(input, output): + if ( + isinstance(output, Variable) + and isinstance(input, Variable) + and has_shape_diff(input, output) + ): warnings.warn( - "In dy2static mode, we attemp to assign a variable with shape {} into a variable with shape{}, which is not always right." - .format(input.shape, output.shape)) + "In dy2static mode, we attemp to assign a variable with shape {} into a variable with shape{}, which is not always right.".format( + input.shape, output.shape + ) + ) assign(input, output) @@ -1253,7 +1374,7 @@ def while_loop(cond, body, loop_vars, is_test=False, name=None): Args: cond(Callable): A callable returning a boolean tensor controlling whether to continue looping. And ``cond`` takes - as many arguments as ``loop_vars`` . + as many arguments as ``loop_vars`` . body(Callable): A callable returning a tuple or list of tensors or LoDTensorArrays of the same arity (length and structure) and types as ``loops_vars`` . And ``body`` takes as many arguments as ``loop_vars`` . loop_vars(list|tuple): A list or tuple of tensors or LoDTensorArrays that is passed to both ``cond`` and ``body`` . @@ -1299,23 +1420,26 @@ def while_loop(cond, body, loop_vars, is_test=False, name=None): raise ValueError("loop_vars in while_loop should not be empty") pre_cond = cond(*loop_vars) - check_variable_and_dtype(pre_cond, 'var of cond returned', ['bool'], - 'fluid.layers.while_loop') + check_variable_and_dtype( + pre_cond, 'var of cond returned', ['bool'], 'fluid.layers.while_loop' + ) if reduce(lambda a, b: a * b, pre_cond.shape, 1) != 1: raise TypeError( "the shape of the variable returned by cond should be [1]," - "but given shape as {0}.".format(list(pre_cond.shape))) + "but given shape as {0}.".format(list(pre_cond.shape)) + ) if _non_static_mode(): now_cond = pre_cond.numpy()[0] - while (now_cond): + while now_cond: output_vars = body(*loop_vars) if not isinstance(output_vars, (list, tuple)): output_vars = [output_vars] if len(output_vars) != len(loop_vars): raise ValueError( "body in while_loop should return the same arity " - "(length and structure) and types as loop_vars") + "(length and structure) and types as loop_vars" + ) now_cond = cond(*output_vars).numpy()[0] map_structure(assign_skip_lod_tensor_array, output_vars, loop_vars) return loop_vars @@ -1340,7 +1464,8 @@ def while_loop(cond, body, loop_vars, is_test=False, name=None): except ValueError as e: raise ValueError( "body in while_loop should return the same arity " - "(length and structure) as loop_vars: {0}".format(e)) + "(length and structure) as loop_vars: {0}".format(e) + ) now_cond = cond(*output_vars) map_structure(assign_skip_lod_tensor_array, output_vars, loop_vars) assign(now_cond, pre_cond) @@ -1348,19 +1473,24 @@ def while_loop(cond, body, loop_vars, is_test=False, name=None): def _deal_with_undefined_var(output_vars, loop_vars): - """ Deal with undefined var cases, We create undefined variable based on the results of body(). - In Dy2Static, we use undefined var to represent the var created in control flow. This function - expand the loop_vars and replace original loop_vars. - 1. UndefinedVar = Variable # create a variable - 2. UndefinedVar = None # create a undefined var with RETURN_NO_VALUE_MAGIC_NUM - 3. UndefinedVar = List(int) # create a list of variable - 4. UndefinedVar = value # create a variable + """Deal with undefined var cases, We create undefined variable based on the results of body(). + In Dy2Static, we use undefined var to represent the var created in control flow. This function + expand the loop_vars and replace original loop_vars. + 1. UndefinedVar = Variable # create a variable + 2. UndefinedVar = None # create a undefined var with RETURN_NO_VALUE_MAGIC_NUM + 3. UndefinedVar = List(int) # create a list of variable + 4. UndefinedVar = value # create a variable """ - from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar, create_undefined_variable + from paddle.fluid.dygraph.dygraph_to_static.utils import ( + UndefinedVar, + create_undefined_variable, + ) def create_var_like(o_var): - if isinstance(o_var, - (Variable, ) + support_ret_buildin_type) or o_var is None: + if ( + isinstance(o_var, (Variable,) + support_ret_buildin_type) + or o_var is None + ): return create_undefined_variable() if is_sequence(o_var): """ @@ -1431,16 +1561,21 @@ def lod_rank_table(x, level=0): check_type(x, 'x', (Variable, list), 'lod_rank_table') if isinstance(x, (list)): for i, input_x in enumerate(x): - check_type(input_x, 'input[' + str(i) + ']', Variable, - 'lod_rank_table') + check_type( + input_x, 'input[' + str(i) + ']', Variable, 'lod_rank_table' + ) helper = LayerHelper("lod_rank_table", **locals()) - table = helper.create_variable(type=core.VarDesc.VarType.LOD_RANK_TABLE, - name=unique_name.generate("lod_rank_table")) - helper.append_op(type='lod_rank_table', - inputs={'X': x}, - outputs={'Out': table}, - attrs={'level': level}) + table = helper.create_variable( + type=core.VarDesc.VarType.LOD_RANK_TABLE, + name=unique_name.generate("lod_rank_table"), + ) + helper.append_op( + type='lod_rank_table', + inputs={'X': x}, + outputs={'Out': table}, + attrs={'level': level}, + ) return table @@ -1463,9 +1598,11 @@ def max_sequence_len(rank_table): """ helper = LayerHelper("max_seqence_len", **locals()) res = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="max_sequence_len", - inputs={"RankTable": rank_table}, - outputs={"Out": res}) + helper.append_op( + type="max_sequence_len", + inputs={"RankTable": rank_table}, + outputs={"Out": res}, + ) return res @@ -1501,24 +1638,32 @@ def lod_tensor_to_array(x, table): check_type(x, 'x', (Variable, list), 'lod_tensor_to_array') if isinstance(x, (list)): for i, input_x in enumerate(x): - check_type(input_x, 'input[' + str(i) + ']', Variable, - 'lod_tensor_to_array') + check_type( + input_x, + 'input[' + str(i) + ']', + Variable, + 'lod_tensor_to_array', + ) check_type(table, 'table', (Variable, list), 'lod_tensor_to_array') if isinstance(table, (list)): for i, table_x in enumerate(table): - check_type(table_x, 'table[' + str(i) + ']', Variable, - 'lod_tensor_to_array') + check_type( + table_x, + 'table[' + str(i) + ']', + Variable, + 'lod_tensor_to_array', + ) helper = LayerHelper("lod_tensor_to_array", **locals()) array = helper.create_variable( name=unique_name.generate("lod_tensor_to_array"), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=x.dtype) - helper.append_op(type='lod_tensor_to_array', - inputs={ - 'X': x, - 'RankTable': table - }, - outputs={'Out': array}) + dtype=x.dtype, + ) + helper.append_op( + type='lod_tensor_to_array', + inputs={'X': x, 'RankTable': table}, + outputs={'Out': array}, + ) return array @@ -1547,22 +1692,29 @@ def array_to_lod_tensor(x, table): check_type(x, 'x', (Variable, list), 'array_to_lod_tensor') if isinstance(x, (list)): for i, input_x in enumerate(x): - check_type(input_x, 'input[' + str(i) + ']', Variable, - 'array_to_lod_tensor') + check_type( + input_x, + 'input[' + str(i) + ']', + Variable, + 'array_to_lod_tensor', + ) check_type(table, 'table', (Variable, list), 'array_to_lod_tensor') if isinstance(table, (list)): for i, table_x in enumerate(table): - check_type(table_x, 'table[' + str(i) + ']', Variable, - 'array_to_lod_tensor') + check_type( + table_x, + 'table[' + str(i) + ']', + Variable, + 'array_to_lod_tensor', + ) helper = LayerHelper("array_to_lod_tensor", **locals()) tmp = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="array_to_lod_tensor", - inputs={ - 'X': x, - 'RankTable': table - }, - outputs={'Out': tmp}) + helper.append_op( + type="array_to_lod_tensor", + inputs={'X': x, 'RankTable': table}, + outputs={'Out': tmp}, + ) return tmp @@ -1590,17 +1742,20 @@ def increment(x, value=1.0, in_place=True): if in_dygraph_mode(): return _C_ops.increment_(x, value) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'increment') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'increment' + ) helper = LayerHelper("increment", **locals()) if not in_place: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: out = x - helper.append_op(type='increment', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={'step': float(value)}) + helper.append_op( + type='increment', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'step': float(value)}, + ) return out @@ -1668,8 +1823,8 @@ def array_write(x, i, array=None): if array is None: array = create_array(x.dtype) assert isinstance( - array, - list), "The 'array' in array_write must be a list in dygraph mode" + array, list + ), "The 'array' in array_write must be a list in dygraph mode" assert i <= len( array ), "The index 'i' should not be greater than the length of 'array' in dygraph mode" @@ -1683,22 +1838,24 @@ def array_write(x, i, array=None): check_type(x, 'x', (Variable), 'array_write') helper = LayerHelper('array_write', **locals()) if array is not None: - if not isinstance( - array, Variable - ) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError( - "array should be tensor array vairable in array_write Op") + "array should be tensor array vairable in array_write Op" + ) if array is None: array = helper.create_variable( name="{0}.out".format(helper.name), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=x.dtype) - helper.append_op(type='write_to_array', - inputs={ - 'X': [x], - 'I': [i] - }, - outputs={'Out': [array]}) + dtype=x.dtype, + ) + helper.append_op( + type='write_to_array', + inputs={'X': [x], 'I': [i]}, + outputs={'Out': [array]}, + ) return array @@ -1729,16 +1886,20 @@ def create_array(dtype, initialized_list=None): if initialized_list is not None: if not isinstance(initialized_list, (list, tuple)): raise TypeError( - "Require type(initialized_list) should be list/tuple, but received {}" - .format(type(initialized_list))) + "Require type(initialized_list) should be list/tuple, but received {}".format( + type(initialized_list) + ) + ) array = list(initialized_list) # NOTE: Only support plain list like [x, y,...], not support nested list in static mode. for val in array: if not isinstance(val, Variable): raise TypeError( - "All values in `initialized_list` should be Variable, but recevied {}." - .format(type(val))) + "All values in `initialized_list` should be Variable, but recevied {}.".format( + type(val) + ) + ) if _non_static_mode(): return array @@ -1747,7 +1908,8 @@ def create_array(dtype, initialized_list=None): tensor_array = helper.create_variable( name="{0}.out".format(helper.name), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=dtype) + dtype=dtype, + ) for val in array: array_write(x=val, i=array_length(tensor_array), array=tensor_array) @@ -1784,10 +1946,12 @@ def less_than(x, y, force_cpu=None, cond=None, name=None): print(result) # [True, False, False, False] """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "less_than") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "less_than") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "less_than" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "less_than" + ) if cond is not None: check_type(cond, "cond", Variable, "less_than") if force_cpu != None: @@ -1802,13 +1966,12 @@ def less_than(x, y, force_cpu=None, cond=None, name=None): if force_cpu is not None: attrs['force_cpu'] = force_cpu - helper.append_op(type='less_than', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}, - attrs=attrs) + helper.append_op( + type='less_than', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [cond]}, + attrs=attrs, + ) return cond @@ -1816,8 +1979,8 @@ def less_than(x, y, force_cpu=None, cond=None, name=None): def less_equal(x, y, cond=None, name=None): """ :alias_main: paddle.less_equal - :alias: paddle.less_equal,paddle.tensor.less_equal,paddle.tensor.logic.less_equal - :old_api: paddle.fluid.layers.less_equal + :alias: paddle.less_equal,paddle.tensor.less_equal,paddle.tensor.logic.less_equal + :old_api: paddle.fluid.layers.less_equal This OP returns the truth value of :math:`x <= y` elementwise, which is equivalent function to the overloaded operator `<=`. @@ -1843,10 +2006,12 @@ def less_equal(x, y, cond=None, name=None): out1 = label<= limit #out1=[True, False] """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "less_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "less_equal") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "less_equal" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "less_equal" + ) if cond is not None: check_type(cond, "cond", Variable, "less_equal") @@ -1857,13 +2022,12 @@ def less_equal(x, y, cond=None, name=None): attrs = dict() - helper.append_op(type='less_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}, - attrs=attrs) + helper.append_op( + type='less_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [cond]}, + attrs=attrs, + ) return cond @@ -1871,8 +2035,8 @@ def less_equal(x, y, cond=None, name=None): def greater_than(x, y, cond=None, name=None): """ :alias_main: paddle.greater_than - :alias: paddle.greater_than,paddle.tensor.greater_than,paddle.tensor.logic.greater_than - :old_api: paddle.fluid.layers.greater_than + :alias: paddle.greater_than,paddle.tensor.greater_than,paddle.tensor.logic.greater_than + :old_api: paddle.fluid.layers.greater_than This OP returns the truth value of :math:`x > y` elementwise, which is equivalent function to the overloaded operator `>`. @@ -1897,10 +2061,12 @@ def greater_than(x, y, cond=None, name=None): out = fluid.layers.greater_than(x=label, y=limit) #out=[False, True] out1 = label > limit #out1=[False, True] """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "greater_than") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "greater_than") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "greater_than" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "greater_than" + ) if cond is not None: check_type(cond, "cond", Variable, "greater_than") @@ -1914,13 +2080,12 @@ def greater_than(x, y, cond=None, name=None): if in_dygraph_mode(): return _C_ops.greater_than(x, y, -1) else: - helper.append_op(type='greater_than', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}, - attrs=attrs) + helper.append_op( + type='greater_than', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [cond]}, + attrs=attrs, + ) return cond @@ -1928,8 +2093,8 @@ def greater_than(x, y, cond=None, name=None): def greater_equal(x, y, cond=None, name=None): """ :alias_main: paddle.greater_equal - :alias: paddle.greater_equal,paddle.tensor.greater_equal,paddle.tensor.logic.greater_equal - :old_api: paddle.fluid.layers.greater_equal + :alias: paddle.greater_equal,paddle.tensor.greater_equal,paddle.tensor.logic.greater_equal + :old_api: paddle.fluid.layers.greater_equal This OP returns the truth value of :math:`x >= y` elementwise, which is equivalent function to the overloaded operator `>=`. @@ -1956,10 +2121,12 @@ def greater_equal(x, y, cond=None, name=None): out_1 = label >= limit #out1=[True, False] """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "greater_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "greater_equal") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "greater_equal" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "greater_equal" + ) if cond is not None: check_type(cond, "cond", Variable, "greater_equal") @@ -1970,13 +2137,12 @@ def greater_equal(x, y, cond=None, name=None): attrs = dict() - helper.append_op(type='greater_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}, - attrs=attrs) + helper.append_op( + type='greater_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [cond]}, + attrs=attrs, + ) return cond @@ -2013,10 +2179,12 @@ def equal(x, y, cond=None, name=None): default_axis = -1 return _C_ops.equal(x, y, default_axis) - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "equal") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "equal" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "equal" + ) if cond is not None: check_type(cond, "cond", Variable, "equal") @@ -2025,20 +2193,17 @@ def equal(x, y, cond=None, name=None): cond = helper.create_variable_for_type_inference(dtype='bool') cond.stop_gradient = True - helper.append_op(type='equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}) + helper.append_op( + type='equal', inputs={'X': [x], 'Y': [y]}, outputs={'Out': [cond]} + ) return cond def not_equal(x, y, cond=None, name=None): """ :alias_main: paddle.not_equal - :alias: paddle.not_equal,paddle.tensor.not_equal,paddle.tensor.logic.not_equal - :old_api: paddle.fluid.layers.not_equal + :alias: paddle.not_equal,paddle.tensor.not_equal,paddle.tensor.logic.not_equal + :old_api: paddle.fluid.layers.not_equal This OP returns the truth value of :math:`x != y` elementwise, which is equivalent function to the overloaded operator `!=`. @@ -2062,10 +2227,12 @@ def not_equal(x, y, cond=None, name=None): limit = fluid.layers.fill_constant(shape=[1], value=1, dtype='int64') out = fluid.layers.not_equal(x=label, y=limit) """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "not_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "not_equal") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "not_equal" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "not_equal" + ) if cond is not None: check_type(cond, "cond", Variable, "not_equal") @@ -2074,12 +2241,9 @@ def not_equal(x, y, cond=None, name=None): cond = helper.create_variable_for_type_inference(dtype='bool') cond.stop_gradient = True - helper.append_op(type='not_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}) + helper.append_op( + type='not_equal', inputs={'X': [x], 'Y': [y]}, outputs={'Out': [cond]} + ) return cond @@ -2146,8 +2310,8 @@ def array_read(array, i): """ if _non_static_mode(): assert isinstance( - array, - list), "The 'array' in array_read must be list in dygraph mode" + array, list + ), "The 'array' in array_read must be list in dygraph mode" assert isinstance( i, Variable ), "The index 'i' in array_read must be Variable in dygraph mode" @@ -2159,17 +2323,17 @@ def array_read(array, i): check_variable_and_dtype(i, 'i', ['int64'], 'array_read') helper = LayerHelper('array_read', **locals()) - if not isinstance( - array, - Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError("array should be tensor array vairable") out = helper.create_variable_for_type_inference(dtype=array.dtype) - helper.append_op(type='read_from_array', - inputs={ - 'X': [array], - 'I': [i] - }, - outputs={'Out': [out]}) + helper.append_op( + type='read_from_array', + inputs={'X': [array], 'I': [i]}, + outputs={'Out': [out]}, + ) return out @@ -2203,14 +2367,12 @@ def shrink_memory(x, i, table): check_type(i, 'i', Variable, 'shrink_memory') check_type(table, 'table', Variable, 'shrink_memory') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='shrink_rnn_memory', - inputs={ - 'X': [x], - 'I': [i], - 'RankTable': [table] - }, - outputs={'Out': [out]}, - attrs={}) + helper.append_op( + type='shrink_rnn_memory', + inputs={'X': [x], 'I': [i], 'RankTable': [table]}, + outputs={'Out': [out]}, + attrs={}, + ) return out @@ -2262,22 +2424,24 @@ def array_length(array): if _non_static_mode(): assert isinstance( - array, - list), "The 'array' in array_write must be a list in dygraph mode" + array, list + ), "The 'array' in array_write must be a list in dygraph mode" return len(array) - if not isinstance( - array, - Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError( - "array should be tensor array vairable in array_length Op") + "array should be tensor array vairable in array_length Op" + ) helper = LayerHelper('array_length', **locals()) tmp = helper.create_variable_for_type_inference(dtype='int64') tmp.stop_gradient = True - helper.append_op(type='lod_array_length', - inputs={'X': [array]}, - outputs={'Out': [tmp]}) + helper.append_op( + type='lod_array_length', inputs={'X': [array]}, outputs={'Out': [tmp]} + ) return tmp @@ -2299,8 +2463,9 @@ class ConditionalBlockGuard(BlockGuard): def __exit__(self, exc_type, exc_val, exc_tb): self.block.complete() - return super(ConditionalBlockGuard, - self).__exit__(exc_type, exc_val, exc_tb) + return super(ConditionalBlockGuard, self).__exit__( + exc_type, exc_val, exc_tb + ) class ConditionalBlock(object): @@ -2346,10 +2511,9 @@ class ConditionalBlock(object): intermediate = set() params = set() - params, intermediate = get_inputs_outputs_in_block(inside_block, - params, - intermediate, - helper=self.helper) + params, intermediate = get_inputs_outputs_in_block( + inside_block, params, intermediate, helper=self.helper + ) # Todo(liym27) Here assume that all params are in recursive parent block # but when minimize() called in control flow, some params may be in @@ -2365,25 +2529,25 @@ class ConditionalBlock(object): out_list.append(inner_var) step_scope = parent_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) + type=core.VarDesc.VarType.STEP_SCOPES + ) conditional_block_op = parent_block.append_op( type='conditional_block', inputs={ 'Cond': self.inputs, 'Input': param_list, }, - outputs={ - 'Out': out_list, - 'Scope': [step_scope] - }, + outputs={'Out': out_list, 'Scope': [step_scope]}, attrs={ 'sub_block': inside_block, - 'is_scalar_condition': self.is_scalar_condition - }) + 'is_scalar_condition': self.is_scalar_condition, + }, + ) if self.need_append_conditional_block_grad(inside_block): - self.append_conditional_block_grad(parent_block, inside_block, - conditional_block_op) + self.append_conditional_block_grad( + parent_block, inside_block, conditional_block_op + ) def need_append_conditional_block_grad(self, inside_block): grad_sub_block_idx = inside_block.backward_block_idx @@ -2391,10 +2555,13 @@ class ConditionalBlock(object): # if inside_block have grad_block and grad_block is not itself, # we will append conditional block grad. - return grad_sub_block_idx != -1 and grad_sub_block_idx != inside_block_idx + return ( + grad_sub_block_idx != -1 and grad_sub_block_idx != inside_block_idx + ) - def append_conditional_block_grad(self, parent_block, inside_block, - conditional_block_op): + def append_conditional_block_grad( + self, parent_block, inside_block, conditional_block_op + ): ''' Append op `conditional_block_grad` manually. When `optimizer.minimize/append_backward` is called in Paddle control flow, @@ -2433,7 +2600,8 @@ class ConditionalBlock(object): param_list.append(inner_var.name) grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - conditional_block_op.desc, set(), [grad_sub_block.desc]) + conditional_block_op.desc, set(), [grad_sub_block.desc] + ) # append op_desc in grad_op_descs to target_block op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() @@ -2443,13 +2611,16 @@ class ConditionalBlock(object): new_op_desc._set_attr(op_role_attr_name, backward) # set input and output manually new_op_desc.set_input('Input', param_list) - new_op_desc.set_output('Input@GRAD', - [param + "@GRAD" for param in param_list]) + new_op_desc.set_output( + 'Input@GRAD', [param + "@GRAD" for param in param_list] + ) new_vars = set() for grad_var_name in new_op_desc.output_arg_names(): - if grad_sub_block.desc.has_var_recursive(grad_var_name.encode( - )) or grad_var_name == core.empty_var_name(): + if ( + grad_sub_block.desc.has_var_recursive(grad_var_name.encode()) + or grad_var_name == core.empty_var_name() + ): continue grad_sub_block.desc.var(grad_var_name.encode()) new_vars.add(grad_var_name) @@ -2472,16 +2643,20 @@ def copy_var_to_parent_block(var, layer_helper): return var prog = layer_helper.main_program parent_idx = prog.current_block().parent_idx - assert parent_idx >= 0, "Got wrong parent block index when assigning var to parent scope in control_flow" + assert ( + parent_idx >= 0 + ), "Got wrong parent block index when assigning var to parent scope in control_flow" parent_block = prog.block(parent_idx) - if var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - and parent_block._find_var_recursive(var.name): + if ( + var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + and parent_block._find_var_recursive(var.name) + ): parent_block_var = var else: - parent_block_var = parent_block.create_var(dtype=var.dtype, - shape=var.shape, - type=var.type) + parent_block_var = parent_block.create_var( + dtype=var.dtype, shape=var.shape, type=var.type + ) assign(var, parent_block_var) return parent_block_var @@ -2594,15 +2769,19 @@ def cond(pred, true_fn=None, false_fn=None, name=None, return_names=None): if true_fn is not None: if not callable(true_fn): raise TypeError( - "The true_fn in cond must be callable, but received {}". - format(type(true_fn).__name__)) + "The true_fn in cond must be callable, but received {}".format( + type(true_fn).__name__ + ) + ) return true_fn() else: if false_fn is not None: if not callable(false_fn): raise TypeError( - "The false_fn in cond must be callable, but received {}" - .format(type(false_fn).__name__)) + "The false_fn in cond must be callable, but received {}".format( + type(false_fn).__name__ + ) + ) return false_fn() return None @@ -2616,25 +2795,32 @@ def cond(pred, true_fn=None, false_fn=None, name=None, return_names=None): if not callable(true_fn): raise TypeError( "The true_fn in cond must be callable, but received {}".format( - type(true_fn).__name__)) + type(true_fn).__name__ + ) + ) true_cond_block = ConditionalBlock([pred], is_scalar_condition=True) with true_cond_block.block(): origin_true_output = true_fn() if origin_true_output is not None: - true_output = map_structure(copy_to_parent_func, - origin_true_output) + true_output = map_structure( + copy_to_parent_func, origin_true_output + ) if false_fn is not None: if not callable(false_fn): raise TypeError( "The false_fn in cond must be callable, but received {}".format( - type(false_fn).__name__)) - false_cond_block = ConditionalBlock([logical_not(pred)], - is_scalar_condition=True) + type(false_fn).__name__ + ) + ) + false_cond_block = ConditionalBlock( + [logical_not(pred)], is_scalar_condition=True + ) with false_cond_block.block(): origin_false_output = false_fn() if origin_false_output is not None: - false_output = map_structure(copy_to_parent_func, - origin_false_output) + false_output = map_structure( + copy_to_parent_func, origin_false_output + ) if true_output is None and false_output is None: return None @@ -2642,11 +2828,13 @@ def cond(pred, true_fn=None, false_fn=None, name=None, return_names=None): if true_output is None: raise ValueError( "Incompatible return values of true_fn and false_fn in cond: " - "true_fn returns None while false_fn returns non-None") + "true_fn returns None while false_fn returns non-None" + ) if false_output is None: raise ValueError( "Incompatible return values of true_fn and false_fn in cond: " - "true_fn returns non-None while false_fn returns None") + "true_fn returns non-None while false_fn returns None" + ) # Merge ture and false output if they are not None if return_names is None: @@ -2664,22 +2852,28 @@ def cond(pred, true_fn=None, false_fn=None, name=None, return_names=None): # a = 1 # Because we can not use variable to express 'None' true_output, false_output = expand_undefined_var( - true_output, false_output, return_names) + true_output, false_output, return_names + ) if len(to_sequence(true_output)) != len(to_sequence(false_output)): raise ValueError( - "true fn returns {} vars, but false fn returns {} vars, which is not equals" - .format(len(to_sequence(true_output)), - len(to_sequence(false_output)))) - for true_out, false_out, return_name in zip(to_sequence(true_output), - to_sequence(false_output), - to_sequence(return_names)): + "true fn returns {} vars, but false fn returns {} vars, which is not equals".format( + len(to_sequence(true_output)), len(to_sequence(false_output)) + ) + ) + for true_out, false_out, return_name in zip( + to_sequence(true_output), + to_sequence(false_output), + to_sequence(return_names), + ): try: assert_same_structure(true_out, false_out, check_types=False) except ValueError as e: raise ValueError( - "Incompatible return values of `{}` in true_fn and false_fn in cond: {}" - .format(return_name, e)) + "Incompatible return values of `{}` in true_fn and false_fn in cond: {}".format( + return_name, e + ) + ) def check_ret_none(seq_true, seq_false, seq_names): length = len(seq_true) @@ -2687,32 +2881,53 @@ def cond(pred, true_fn=None, false_fn=None, name=None, return_names=None): f_true = flatten(seq_true[i]) f_false = flatten(seq_false[i]) for idx in range(len(f_true)): - if f_true[idx] is None and f_false[idx] is not None or f_false[ - idx] is None and f_true[idx] is not None: + if ( + f_true[idx] is None + and f_false[idx] is not None + or f_false[idx] is None + and f_true[idx] is not None + ): warnings.warn( "In cond : Var '{}' or part of it is set differently in ifelse branchs, " "<{}, {}> in true branch and <{}, {}> in false branch. Set var to " "'None' in ifelse block might lead to error.".format( - seq_names[i], type(f_true[idx]), f_true[idx], - type(f_false[idx]), f_false[idx])) - - check_ret_none(to_sequence(true_output), to_sequence(false_output), - to_sequence(return_names)) + seq_names[i], + type(f_true[idx]), + f_true[idx], + type(f_false[idx]), + f_false[idx], + ) + ) + + check_ret_none( + to_sequence(true_output), + to_sequence(false_output), + to_sequence(return_names), + ) if is_dy2staic: true_output, false_output = change_none_to_undefinedvar( - true_output, false_output) + true_output, false_output + ) mask = cast(pred, dtype='int32') - merge_func = lambda name, false_var, true_var: select_input_with_buildin_type( - [false_var, true_var], mask, name) + merge_func = ( + lambda name, false_var, true_var: select_input_with_buildin_type( + [false_var, true_var], mask, name + ) + ) def merge_every_var_list(false_vars, true_vars, name): return map_structure(partial(merge_func, name), false_vars, true_vars) merged_output = list( - map(merge_every_var_list, to_sequence(false_output), - to_sequence(true_output), to_sequence(return_names))) + map( + merge_every_var_list, + to_sequence(false_output), + to_sequence(true_output), + to_sequence(return_names), + ) + ) merged_output = pack_sequence_as(false_output, flatten(merged_output)) return merged_output @@ -2721,7 +2936,8 @@ def change_none_to_undefinedvar(nest1, nest2): from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar def map_fn(x): - if x is None: return UndefinedVar("padding") + if x is None: + return UndefinedVar("padding") return x nest1_out = pack_sequence_as(nest1, list(map(map_fn, flatten(nest1)))) @@ -2730,56 +2946,81 @@ def change_none_to_undefinedvar(nest1, nest2): def expand_undefined_var(nest1, nest2, names): - """ TODO: make this function recursively. - nest1: Var1, (UndefinedVar, [1,2,3]) - nest2: Var2, ([1,2,3,4], UndefinedVar) - In this case, we should not expand recursively. + """TODO: make this function recursively. + nest1: Var1, (UndefinedVar, [1,2,3]) + nest2: Var2, ([1,2,3,4], UndefinedVar) + In this case, we should not expand recursively. """ from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar - from paddle.fluid.dygraph.dygraph_to_static.return_transformer import RETURN_VALUE_PREFIX + from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + RETURN_VALUE_PREFIX, + ) def pack_undefined_var_as(seq): - return pack_sequence_as(seq, - [UndefinedVar("padding") for i in flatten(seq)]) + return pack_sequence_as( + seq, [UndefinedVar("padding") for i in flatten(seq)] + ) def map_fn(n1, n2, name, order): - if not name.startswith(RETURN_VALUE_PREFIX) and (isinstance( - n1, UndefinedVar) or n1 is None): + if not name.startswith(RETURN_VALUE_PREFIX) and ( + isinstance(n1, UndefinedVar) or n1 is None + ): if n1 is None and n2 is not None: if order == 0: warnings.warn( "In cond : Var '{}' or part of it is set differently in ifelse branchs, " "<{}, {}> in true branch and <{}, {}> in false branch. Set var to " "'None' in ifelse block might lead to error.".format( - name, type(n1), n1, type(n2), n2)) + name, type(n1), n1, type(n2), n2 + ) + ) else: warnings.warn( "In cond : Var '{}' or part of it is set differently in ifelse branchs, " "<{}, {}> in true branch and <{}, {}> in false branch. Set var to " "'None' in ifelse block might lead to error.".format( - name, type(n2), n2, type(n1), n1)) + name, type(n2), n2, type(n1), n1 + ) + ) return pack_undefined_var_as(n2) return n1 nest1_out = list( - map(map_fn, to_sequence(nest1), to_sequence(nest2), to_sequence(names), - [0 for i in to_sequence(names)])) + map( + map_fn, + to_sequence(nest1), + to_sequence(nest2), + to_sequence(names), + [0 for i in to_sequence(names)], + ) + ) nest2_out = list( - map(map_fn, to_sequence(nest2), to_sequence(nest1), to_sequence(names), - [1 for i in to_sequence(names)])) - if not is_sequence(nest1): nest1_out = nest1_out[0] - if not is_sequence(nest2): nest2_out = nest2_out[0] + map( + map_fn, + to_sequence(nest2), + to_sequence(nest1), + to_sequence(names), + [1 for i in to_sequence(names)], + ) + ) + if not is_sequence(nest1): + nest1_out = nest1_out[0] + if not is_sequence(nest2): + nest2_out = nest2_out[0] return nest1_out, nest2_out def _error_message(what, arg_name, op_name, right_value, error_value): - error_message = "{what} of '{arg_name}' in {op_name} must be " \ + error_message = ( + "{what} of '{arg_name}' in {op_name} must be " "{right_value}, but received: {error_value}.".format( - what=what, - arg_name=arg_name, - op_name=op_name, - right_value=right_value, - error_value=error_value) + what=what, + arg_name=arg_name, + op_name=op_name, + right_value=right_value, + error_value=error_value, + ) + ) return error_message @@ -2860,24 +3101,42 @@ def case(pred_fn_pairs, default=None, name=None): for pred_fn in pred_fn_pairs: if not isinstance(pred_fn, tuple): raise TypeError( - _error_message("The elements' type", "pred_fn_pairs", - "case", tuple, type(pred_fn))) + _error_message( + "The elements' type", + "pred_fn_pairs", + "case", + tuple, + type(pred_fn), + ) + ) if len(pred_fn) != 2: raise TypeError( - _error_message("The tuple's size", "pred_fn_pairs", "case", - "2", - str(len(pred_fn)) + "-tuple")) + _error_message( + "The tuple's size", + "pred_fn_pairs", + "case", + "2", + str(len(pred_fn)) + "-tuple", + ) + ) pred, fn = pred_fn if not isinstance(pred, Variable): raise TypeError( - _error_message("The pred's type", "pred_fn_pairs", "case", - "boolean Variable", type(pred))) + _error_message( + "The pred's type", + "pred_fn_pairs", + "case", + "boolean Variable", + type(pred), + ) + ) if not callable(fn): raise TypeError( "The fn for {} of pred_fn_pairs in Op(case) must" - " be callable.".format(pred.name)) + " be callable.".format(pred.name) + ) if default is None: default_index = len(pred_fn_pairs) - 1 # pick the last one @@ -2979,8 +3238,11 @@ class Switch(object): raise ValueError("case should be called inside with") check_variable_and_dtype( - condition, 'condition', ['bool'], - 'the member function case of fluid.layers.Switch') + condition, + 'condition', + ['bool'], + 'the member function case of fluid.layers.Switch', + ) if len(self.pre_not_conditions) == 0: cond_block = ConditionalBlock([condition], is_scalar_condition=True) @@ -2989,12 +3251,14 @@ class Switch(object): else: pre_cond_num = len(self.pre_not_conditions) pre_not_cond = self.pre_not_conditions[pre_cond_num - 1] - new_not_cond = logical_and(x=pre_not_cond, - y=logical_not(x=condition)) + new_not_cond = logical_and( + x=pre_not_cond, y=logical_not(x=condition) + ) self.pre_not_conditions.append(new_not_cond) cond_block = ConditionalBlock( [logical_and(x=pre_not_cond, y=condition)], - is_scalar_condition=True) + is_scalar_condition=True, + ) return ConditionalBlockGuard(cond_block) @@ -3004,7 +3268,8 @@ class Switch(object): raise ValueError("there should be at least one condition") cond_block = ConditionalBlock( [self.pre_not_conditions[pre_cond_num - 1]], - is_scalar_condition=True) + is_scalar_condition=True, + ) return ConditionalBlockGuard(cond_block) def __enter__(self): @@ -3024,7 +3289,6 @@ class Switch(object): class IfElseBlockGuard(object): - def __init__(self, is_true, ifelse): if not isinstance(ifelse, IfElse): raise TypeError("ifelse must be an instance of IfElse class") @@ -3045,7 +3309,11 @@ class IfElseBlockGuard(object): self.cond_block = self.cond_block.block() def __enter__(self): - self.ie.status = IfElse.IN_IF_ELSE_TRUE_BLOCKS if self.is_true else IfElse.IN_IF_ELSE_FALSE_BLOCKS + self.ie.status = ( + IfElse.IN_IF_ELSE_TRUE_BLOCKS + if self.is_true + else IfElse.IN_IF_ELSE_FALSE_BLOCKS + ) self.cond_block.__enter__() def __exit__(self, exc_type, exc_val, exc_tb): @@ -3132,6 +3400,7 @@ class IfElse(object): There is a ``call ()`` function inside the object, that is, by calling ``output = ie ()``, all the outputs inside the block of False are fused as the whole output, the output type is a list, and the type of each element in the list is Variable. """ + OUT_IF_ELSE_BLOCKS = 0 IN_IF_ELSE_TRUE_BLOCKS = 1 IN_IF_ELSE_FALSE_BLOCKS = 2 @@ -3153,24 +3422,27 @@ class IfElse(object): if id(x) not in self.input_table: parent_block = self._parent_block() out_true = parent_block.create_var( - name=unique_name.generate_with_ignorable_key('ifelse_input' + - self.helper.name), - dtype=x.dtype) + name=unique_name.generate_with_ignorable_key( + 'ifelse_input' + self.helper.name + ), + dtype=x.dtype, + ) out_false = parent_block.create_var( - name=unique_name.generate_with_ignorable_key('ifelse_input' + - self.helper.name), - dtype=x.dtype) - parent_block.append_op(type='split_lod_tensor', - inputs={ - 'X': x, - 'Mask': self.cond, - }, - outputs={ - 'OutTrue': out_true, - 'OutFalse': out_false - }, - attrs={'level': 0}) + name=unique_name.generate_with_ignorable_key( + 'ifelse_input' + self.helper.name + ), + dtype=x.dtype, + ) + parent_block.append_op( + type='split_lod_tensor', + inputs={ + 'X': x, + 'Mask': self.cond, + }, + outputs={'OutTrue': out_true, 'OutFalse': out_false}, + attrs={'level': 0}, + ) self.input_table[id(x)] = (out_true, out_false) else: out_true, out_false = self.input_table[id(x)] @@ -3194,17 +3466,21 @@ class IfElse(object): if self.status == self.OUT_IF_ELSE_BLOCKS: raise ValueError("output can only be invoked in the sub-block") - out_table = self.output_table[1 if self.status == - self.IN_IF_ELSE_TRUE_BLOCKS else 0] + out_table = self.output_table[ + 1 if self.status == self.IN_IF_ELSE_TRUE_BLOCKS else 0 + ] parent_block = self._parent_block() for each_out in outs: - check_type(each_out, "each output", Variable, - "fluid.layers.IfElse.output") + check_type( + each_out, "each output", Variable, "fluid.layers.IfElse.output" + ) # create outside tensor outside_out = parent_block.create_var( - name=unique_name.generate_with_ignorable_key("_".join( - [self.helper.name, 'output'])), - dtype=each_out.dtype) + name=unique_name.generate_with_ignorable_key( + "_".join([self.helper.name, 'output']) + ), + dtype=each_out.dtype, + ) out_table.append(outside_out) # assign local var to outside @@ -3215,8 +3491,9 @@ class IfElse(object): raise ValueError("IfElse::__call__ must be out of sub-block") false_len, true_len = list(map(len, self.output_table)) if false_len == 0 and true_len == 0: - raise ValueError("Must invoke true_block/false_block before " - "__call__") + raise ValueError( + "Must invoke true_block/false_block before " "__call__" + ) elif false_len != true_len and false_len != 0 and true_len != 0: raise ValueError("The output side must be same") elif false_len == 0 or true_len == 0: @@ -3227,11 +3504,14 @@ class IfElse(object): rlist = [] for false_var, true_var in zip(*self.output_table): rlist.append( - merge_lod_tensor(in_true=true_var, - in_false=false_var, - mask=self.cond, - x=self.cond, - level=0)) + merge_lod_tensor( + in_true=true_var, + in_false=false_var, + mask=self.cond, + x=self.cond, + level=0, + ) + ) return rlist @@ -3302,6 +3582,7 @@ class DynamicRNN(object): # Get RNN's result of the last time step last = fluid.layers.sequence_last_step(out) """ + BEFORE_RNN = 0 IN_RNN = 1 AFTER_RNN = 2 @@ -3419,39 +3700,44 @@ class DynamicRNN(object): if self.lod_rank_table is None: self.lod_rank_table = parent_block.create_var( name=unique_name.generate('lod_rank_table'), - type=core.VarDesc.VarType.LOD_RANK_TABLE) + type=core.VarDesc.VarType.LOD_RANK_TABLE, + ) self.lod_rank_table.stop_gradient = True - parent_block.append_op(type='lod_rank_table', - inputs={"X": x}, - outputs={"Out": self.lod_rank_table}, - attrs={"level": level}) + parent_block.append_op( + type='lod_rank_table', + inputs={"X": x}, + outputs={"Out": self.lod_rank_table}, + attrs={"level": level}, + ) self.max_seq_len = parent_block.create_var( name=unique_name.generate('dynamic_rnn_max_seq_len'), - dtype='int64') + dtype='int64', + ) self.max_seq_len.stop_gradient = False - parent_block.append_op(type='max_sequence_len', - inputs={'RankTable': self.lod_rank_table}, - outputs={"Out": self.max_seq_len}) + parent_block.append_op( + type='max_sequence_len', + inputs={'RankTable': self.lod_rank_table}, + outputs={"Out": self.max_seq_len}, + ) self.cond.stop_gradient = True - parent_block.append_op(type='less_than', - inputs={ - 'X': self.step_idx, - 'Y': self.max_seq_len - }, - outputs={'Out': self.cond}, - attrs={'force_cpu': True}) + parent_block.append_op( + type='less_than', + inputs={'X': self.step_idx, 'Y': self.max_seq_len}, + outputs={'Out': self.cond}, + attrs={'force_cpu': True}, + ) input_array = parent_block.create_var( name=unique_name.generate('dynamic_rnn_input_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=x.dtype) + dtype=x.dtype, + ) self.input_array.append((input_array, x.dtype)) - parent_block.append_op(type='lod_tensor_to_array', - inputs={ - 'X': x, - 'RankTable': self.lod_rank_table - }, - outputs={'Out': input_array}) + parent_block.append_op( + type='lod_tensor_to_array', + inputs={'X': x, 'RankTable': self.lod_rank_table}, + outputs={'Out': input_array}, + ) return array_read(array=input_array, i=self.step_idx) def static_input(self, x): @@ -3584,18 +3870,19 @@ class DynamicRNN(object): check_type(x, 'x', Variable, 'fluid.layers.DynamicRNN.static_input()') if self.lod_rank_table is None: raise RuntimeError( - "static_input() must be called after step_input().") + "static_input() must be called after step_input()." + ) parent_block = self._parent_block_() x_reordered = parent_block.create_var( name=unique_name.generate("dynamic_rnn_static_input_reordered"), type=core.VarDesc.VarType.LOD_TENSOR, - dtype=x.dtype) - parent_block.append_op(type='reorder_lod_tensor_by_rank', - inputs={ - 'X': [x], - 'RankTable': [self.lod_rank_table] - }, - outputs={'Out': [x_reordered]}) + dtype=x.dtype, + ) + parent_block.append_op( + type='reorder_lod_tensor_by_rank', + inputs={'X': [x], 'RankTable': [self.lod_rank_table]}, + outputs={'Out': [x_reordered]}, + ) return shrink_memory(x_reordered, self.step_idx, self.lod_rank_table) @signature_safe_contextmanager @@ -3610,10 +3897,9 @@ class DynamicRNN(object): """ if self.status != DynamicRNN.BEFORE_RNN: raise ValueError("rnn.block() can only be invoke once") - self.step_idx = fill_constant(shape=[1], - dtype='int64', - value=0, - force_cpu=True) + self.step_idx = fill_constant( + shape=[1], dtype='int64', value=0, force_cpu=True + ) self.step_idx.stop_gradient = False self.status = DynamicRNN.IN_RNN with self.while_op.block(): @@ -3623,15 +3909,18 @@ class DynamicRNN(object): for new_mem, mem_array in self.mem_link: array_write(x=new_mem, i=self.step_idx, array=mem_array) - less_than(x=self.step_idx, - y=self.max_seq_len, - force_cpu=True, - cond=self.cond) + less_than( + x=self.step_idx, + y=self.max_seq_len, + force_cpu=True, + cond=self.cond, + ) self.status = DynamicRNN.AFTER_RNN for each_array in self.output_array: self.outputs.append( - array_to_lod_tensor(x=each_array, table=self.lod_rank_table)) + array_to_lod_tensor(x=each_array, table=self.lod_rank_table) + ) def __call__(self, *args, **kwargs): """ @@ -3647,19 +3936,25 @@ class DynamicRNN(object): ValueError: When :code:`__call__()` is called before :code:`block()` . """ if self.status != DynamicRNN.AFTER_RNN: - raise ValueError(("Output of the dynamic RNN can only be visited " - "outside the rnn block.")) + raise ValueError( + ( + "Output of the dynamic RNN can only be visited " + "outside the rnn block." + ) + ) if len(self.outputs) == 1: return self.outputs[0] else: return self.outputs - def memory(self, - init=None, - shape=None, - value=0.0, - need_reorder=False, - dtype='float32'): + def memory( + self, + init=None, + shape=None, + value=0.0, + need_reorder=False, + dtype='float32', + ): r""" Create a memory Variable for DynamicRNN to deliver data cross time steps. It can be initialized by an existing Tensor or a constant Tensor of given @@ -3748,11 +4043,16 @@ class DynamicRNN(object): self._assert_in_rnn_block_('memory') self._init_zero_idx_() if shape is not None: - check_type(shape, 'shape', (list, tuple), - 'fluid.layers.DynamicRNN.memory()') + check_type( + shape, + 'shape', + (list, tuple), + 'fluid.layers.DynamicRNN.memory()', + ) if init is not None: - check_type(init, 'init', Variable, - 'fluid.layers.DynamicRNN.memory()') + check_type( + init, 'init', Variable, 'fluid.layers.DynamicRNN.memory()' + ) parent_block = self._parent_block_() init_tensor = init if need_reorder == True: @@ -3760,32 +4060,36 @@ class DynamicRNN(object): raise ValueError( 'If set need_reorder to True, make sure step_input be ' 'invoked before ' - 'memory(init=init, need_reordered=True, ...).') + 'memory(init=init, need_reordered=True, ...).' + ) init_reordered = parent_block.create_var( name=unique_name.generate('dynamic_rnn_mem_init_reordered'), type=core.VarDesc.VarType.LOD_TENSOR, - dtype=init.dtype) - parent_block.append_op(type='reorder_lod_tensor_by_rank', - inputs={ - 'X': [init_tensor], - 'RankTable': [self.lod_rank_table] - }, - outputs={'Out': [init_reordered]}) + dtype=init.dtype, + ) + parent_block.append_op( + type='reorder_lod_tensor_by_rank', + inputs={ + 'X': [init_tensor], + 'RankTable': [self.lod_rank_table], + }, + outputs={'Out': [init_reordered]}, + ) init_tensor = init_reordered mem_array = parent_block.create_var( name=unique_name.generate('dynamic_rnn_mem_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=init.dtype) - parent_block.append_op(type='write_to_array', - inputs={ - 'X': init_tensor, - 'I': self.zero_idx - }, - outputs={'Out': mem_array}) + dtype=init.dtype, + ) + parent_block.append_op( + type='write_to_array', + inputs={'X': init_tensor, 'I': self.zero_idx}, + outputs={'Out': mem_array}, + ) retv = array_read(array=mem_array, i=self.step_idx) - retv = shrink_memory(x=retv, - i=self.step_idx, - table=self.lod_rank_table) + retv = shrink_memory( + x=retv, i=self.step_idx, table=self.lod_rank_table + ) self.mem_dict[retv.name] = mem_array return retv else: @@ -3795,24 +4099,27 @@ class DynamicRNN(object): ) parent_block = self._parent_block_() init = parent_block.create_var( - name=unique_name.generate('mem_init'), dtype=dtype) + name=unique_name.generate('mem_init'), dtype=dtype + ) arr, dtype = self.input_array[0] - in0 = parent_block.create_var(name=unique_name.generate('in0'), - dtype=dtype) - parent_block.append_op(type='read_from_array', - inputs={ - 'X': [arr], - 'I': [self.zero_idx] - }, - outputs={'Out': [in0]}) - parent_block.append_op(type='fill_constant_batch_size_like', - inputs={'Input': [in0]}, - outputs={'Out': [init]}, - attrs={ - 'shape': [-1] + shape, - 'value': float(value), - 'dtype': init.dtype - }) + in0 = parent_block.create_var( + name=unique_name.generate('in0'), dtype=dtype + ) + parent_block.append_op( + type='read_from_array', + inputs={'X': [arr], 'I': [self.zero_idx]}, + outputs={'Out': [in0]}, + ) + parent_block.append_op( + type='fill_constant_batch_size_like', + inputs={'Input': [in0]}, + outputs={'Out': [init]}, + attrs={ + 'shape': [-1] + shape, + 'value': float(value), + 'dtype': init.dtype, + }, + ) return self.memory(init=init) def update_memory(self, ex_mem, new_mem): @@ -3834,10 +4141,18 @@ class DynamicRNN(object): ValueError: When :code:`update_memory()` is called before :code:`step_input()` . """ self._assert_in_rnn_block_('update_memory') - check_type(ex_mem, 'ex_mem', Variable, - 'fluid.layers.DynamicRNN.update_memory()') - check_type(new_mem, 'new_mem', Variable, - 'fluid.layers.DynamicRNN.update_memory()') + check_type( + ex_mem, + 'ex_mem', + Variable, + 'fluid.layers.DynamicRNN.update_memory()', + ) + check_type( + new_mem, + 'new_mem', + Variable, + 'fluid.layers.DynamicRNN.update_memory()', + ) mem_array = self.mem_dict.get(ex_mem.name, None) if mem_array is None: @@ -3864,13 +4179,16 @@ class DynamicRNN(object): self._assert_in_rnn_block_('output') parent_block = self._parent_block_() for each in outputs: - check_type(each, "outputs", Variable, - "fluid.layers.DynamicRNN.output") + check_type( + each, "outputs", Variable, "fluid.layers.DynamicRNN.output" + ) outside_array = parent_block.create_var( - name=unique_name.generate_with_ignorable_key("_".join( - [self.helper.name, "output_array", each.name])), + name=unique_name.generate_with_ignorable_key( + "_".join([self.helper.name, "output_array", each.name]) + ), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=each.dtype) + dtype=each.dtype, + ) array_write(x=each, i=self.step_idx, array=outside_array) self.output_array.append(outside_array) @@ -3878,16 +4196,19 @@ class DynamicRNN(object): if self.zero_idx is None: parent_block = self._parent_block_() self.zero_idx = parent_block.create_var( - name=unique_name.generate('zero_idx'), dtype='int64') - parent_block.append_op(type='fill_constant', - inputs={}, - outputs={'Out': [self.zero_idx]}, - attrs={ - 'shape': [1], - 'dtype': self.zero_idx.dtype, - 'value': float(0), - 'force_cpu': True - }) + name=unique_name.generate('zero_idx'), dtype='int64' + ) + parent_block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [self.zero_idx]}, + attrs={ + 'shape': [1], + 'dtype': self.zero_idx.dtype, + 'value': float(0), + 'force_cpu': True, + }, + ) def _parent_block_(self): prog = self.helper.main_program @@ -3900,7 +4221,8 @@ class DynamicRNN(object): def _assert_in_rnn_block_(self, method): if self.status != DynamicRNN.IN_RNN: raise ValueError( - "{0} can only be invoked inside rnn block.".format(method)) + "{0} can only be invoked inside rnn block.".format(method) + ) def switch_case(branch_index, branch_fns, default=None, name=None): @@ -3977,44 +4299,71 @@ def switch_case(branch_index, branch_fns, default=None, name=None): def _check_args(branch_index, branch_fns, default): - check_variable_and_dtype(branch_index, 'branch_index', - ['uint8', 'int32', 'int64'], 'switch_case') + check_variable_and_dtype( + branch_index, + 'branch_index', + ['uint8', 'int32', 'int64'], + 'switch_case', + ) if convert_dtype(branch_index.dtype) != "int64": branch_index = cast(branch_index, "int64") check_type(branch_fns, 'branch_fns', (list, tuple, dict), 'switch_case') - branch_fns = branch_fns.items() if isinstance(branch_fns, - dict) else branch_fns + branch_fns = ( + branch_fns.items() if isinstance(branch_fns, dict) else branch_fns + ) - branch_fns = list(enumerate(branch_fns)) if all( - callable(fn) for fn in branch_fns) else branch_fns + branch_fns = ( + list(enumerate(branch_fns)) + if all(callable(fn) for fn in branch_fns) + else branch_fns + ) keys_of_fns = [] for index_fn_pair in branch_fns: if not isinstance(index_fn_pair, tuple): raise TypeError( - _error_message("The elements' type", "branch_fns", - "switch_case", tuple, type(branch_fns))) + _error_message( + "The elements' type", + "branch_fns", + "switch_case", + tuple, + type(branch_fns), + ) + ) if len(index_fn_pair) != 2: raise TypeError( - _error_message("The tuple's size", "branch_fns", - "switch_case", "2", - str(len(index_fn_pair)) + "-tuple")) + _error_message( + "The tuple's size", + "branch_fns", + "switch_case", + "2", + str(len(index_fn_pair)) + "-tuple", + ) + ) key, fn = index_fn_pair if not isinstance(key, int): raise TypeError( - _error_message("The key's type", "branch_fns", - "switch_case", int, type(key))) + _error_message( + "The key's type", + "branch_fns", + "switch_case", + int, + type(key), + ) + ) if key in keys_of_fns: raise ValueError( - "The key in 'branch_fns' must be unique, but '{}' appears more than once." - .format(key)) + "The key in 'branch_fns' must be unique, but '{}' appears more than once.".format( + key + ) + ) else: keys_of_fns.append(key) @@ -4022,7 +4371,12 @@ def switch_case(branch_index, branch_fns, default=None, name=None): raise TypeError( _error_message( "The type of function for key {}".format(key), - "branch_fns", "switch_case", "callable", type(fn))) + "branch_fns", + "switch_case", + "callable", + type(fn), + ) + ) if default is None: default = sorted(branch_fns)[-1][1] @@ -4073,20 +4427,20 @@ def reorder_lod_tensor_by_rank(x, rank_table): """ check_type(x, 'x', (Variable), 'reorder_lod_tensor_by_rank') - check_type(rank_table, 'rank_table', (Variable), - 'reorder_lod_tensor_by_rank') + check_type( + rank_table, 'rank_table', (Variable), 'reorder_lod_tensor_by_rank' + ) if rank_table.type != core.VarDesc.VarType.LOD_RANK_TABLE: raise TypeError("The type of rank_table should be LOD_RANK_TABLE.") helper = LayerHelper('reorder_lod_tensor_by_rank', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reorder_lod_tensor_by_rank', - inputs={ - 'X': [x], - 'RankTable': [rank_table] - }, - outputs={'Out': [out]}) + helper.append_op( + type='reorder_lod_tensor_by_rank', + inputs={'X': [x], 'RankTable': [rank_table]}, + outputs={'Out': [out]}, + ) return out @@ -4125,14 +4479,15 @@ def is_empty(x, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.is_empty(x) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'is_empty') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'is_empty' + ) check_type(name, "name", (str, type(None)), "is_empty") helper = LayerHelper("is_empty", **locals()) cond = helper.create_variable_for_type_inference(dtype='bool') cond.stop_gradient = True - helper.append_op(type='is_empty', - inputs={'X': [x]}, - outputs={'Out': [cond]}) + helper.append_op( + type='is_empty', inputs={'X': [x]}, outputs={'Out': [cond]} + ) return cond diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 80754898e72..d5774592374 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -31,7 +31,12 @@ from ..data_feeder import check_variable_and_dtype, check_type, check_dtype import math import numpy as np from functools import reduce -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from paddle.utils import deprecated from paddle import _C_ops, _legacy_C_ops from ..framework import in_dygraph_mode @@ -68,17 +73,19 @@ __all__ = [ ] -def retinanet_target_assign(bbox_pred, - cls_logits, - anchor_box, - anchor_var, - gt_boxes, - gt_labels, - is_crowd, - im_info, - num_classes=1, - positive_overlap=0.5, - negative_overlap=0.4): +def retinanet_target_assign( + bbox_pred, + cls_logits, + anchor_box, + anchor_var, + gt_boxes, + gt_labels, + is_crowd, + im_info, + num_classes=1, + positive_overlap=0.5, + negative_overlap=0.4, +): r""" **Target Assign Layer for the detector RetinaNet.** @@ -245,22 +252,42 @@ def retinanet_target_assign(bbox_pred, """ - check_variable_and_dtype(bbox_pred, 'bbox_pred', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(cls_logits, 'cls_logits', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(anchor_box, 'anchor_box', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(anchor_var, 'anchor_var', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(gt_boxes, 'gt_boxes', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(gt_labels, 'gt_labels', ['int32'], - 'retinanet_target_assign') - check_variable_and_dtype(is_crowd, 'is_crowd', ['int32'], - 'retinanet_target_assign') - check_variable_and_dtype(im_info, 'im_info', ['float32', 'float64'], - 'retinanet_target_assign') + check_variable_and_dtype( + bbox_pred, + 'bbox_pred', + ['float32', 'float64'], + 'retinanet_target_assign', + ) + check_variable_and_dtype( + cls_logits, + 'cls_logits', + ['float32', 'float64'], + 'retinanet_target_assign', + ) + check_variable_and_dtype( + anchor_box, + 'anchor_box', + ['float32', 'float64'], + 'retinanet_target_assign', + ) + check_variable_and_dtype( + anchor_var, + 'anchor_var', + ['float32', 'float64'], + 'retinanet_target_assign', + ) + check_variable_and_dtype( + gt_boxes, 'gt_boxes', ['float32', 'float64'], 'retinanet_target_assign' + ) + check_variable_and_dtype( + gt_labels, 'gt_labels', ['int32'], 'retinanet_target_assign' + ) + check_variable_and_dtype( + is_crowd, 'is_crowd', ['int32'], 'retinanet_target_assign' + ) + check_variable_and_dtype( + im_info, 'im_info', ['float32', 'float64'], 'retinanet_target_assign' + ) helper = LayerHelper('retinanet_target_assign', **locals()) # Assign target label to anchors @@ -268,30 +295,34 @@ def retinanet_target_assign(bbox_pred, score_index = helper.create_variable_for_type_inference(dtype='int32') target_label = helper.create_variable_for_type_inference(dtype='int32') target_bbox = helper.create_variable_for_type_inference( - dtype=anchor_box.dtype) + dtype=anchor_box.dtype + ) bbox_inside_weight = helper.create_variable_for_type_inference( - dtype=anchor_box.dtype) + dtype=anchor_box.dtype + ) fg_num = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type="retinanet_target_assign", - inputs={ - 'Anchor': anchor_box, - 'GtBoxes': gt_boxes, - 'GtLabels': gt_labels, - 'IsCrowd': is_crowd, - 'ImInfo': im_info - }, - outputs={ - 'LocationIndex': loc_index, - 'ScoreIndex': score_index, - 'TargetLabel': target_label, - 'TargetBBox': target_bbox, - 'BBoxInsideWeight': bbox_inside_weight, - 'ForegroundNumber': fg_num - }, - attrs={ - 'positive_overlap': positive_overlap, - 'negative_overlap': negative_overlap - }) + helper.append_op( + type="retinanet_target_assign", + inputs={ + 'Anchor': anchor_box, + 'GtBoxes': gt_boxes, + 'GtLabels': gt_labels, + 'IsCrowd': is_crowd, + 'ImInfo': im_info, + }, + outputs={ + 'LocationIndex': loc_index, + 'ScoreIndex': score_index, + 'TargetLabel': target_label, + 'TargetBBox': target_bbox, + 'BBoxInsideWeight': bbox_inside_weight, + 'ForegroundNumber': fg_num, + }, + attrs={ + 'positive_overlap': positive_overlap, + 'negative_overlap': negative_overlap, + }, + ) loc_index.stop_gradient = True score_index.stop_gradient = True @@ -305,22 +336,31 @@ def retinanet_target_assign(bbox_pred, predicted_cls_logits = nn.gather(cls_logits, score_index) predicted_bbox_pred = nn.gather(bbox_pred, loc_index) - return predicted_cls_logits, predicted_bbox_pred, target_label, target_bbox, bbox_inside_weight, fg_num - - -def rpn_target_assign(bbox_pred, - cls_logits, - anchor_box, - anchor_var, - gt_boxes, - is_crowd, - im_info, - rpn_batch_size_per_im=256, - rpn_straddle_thresh=0.0, - rpn_fg_fraction=0.5, - rpn_positive_overlap=0.7, - rpn_negative_overlap=0.3, - use_random=True): + return ( + predicted_cls_logits, + predicted_bbox_pred, + target_label, + target_bbox, + bbox_inside_weight, + fg_num, + ) + + +def rpn_target_assign( + bbox_pred, + cls_logits, + anchor_box, + anchor_var, + gt_boxes, + is_crowd, + im_info, + rpn_batch_size_per_im=256, + rpn_straddle_thresh=0.0, + rpn_fg_fraction=0.5, + rpn_positive_overlap=0.7, + rpn_negative_overlap=0.3, + use_random=True, +): """ **Target Assign Layer for region proposal network (RPN) in Faster-RCNN detection.** @@ -410,51 +450,62 @@ def rpn_target_assign(bbox_pred, helper = LayerHelper('rpn_target_assign', **locals()) - check_variable_and_dtype(bbox_pred, 'bbox_pred', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(cls_logits, 'cls_logits', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(anchor_box, 'anchor_box', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(anchor_var, 'anchor_var', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(gt_boxes, 'gt_boxes', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(is_crowd, 'is_crowd', ['int32'], - 'rpn_target_assign') - check_variable_and_dtype(im_info, 'im_info', ['float32', 'float64'], - 'rpn_target_assign') + check_variable_and_dtype( + bbox_pred, 'bbox_pred', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + cls_logits, 'cls_logits', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + anchor_box, 'anchor_box', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + anchor_var, 'anchor_var', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + gt_boxes, 'gt_boxes', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + is_crowd, 'is_crowd', ['int32'], 'rpn_target_assign' + ) + check_variable_and_dtype( + im_info, 'im_info', ['float32', 'float64'], 'rpn_target_assign' + ) # Assign target label to anchors loc_index = helper.create_variable_for_type_inference(dtype='int32') score_index = helper.create_variable_for_type_inference(dtype='int32') target_label = helper.create_variable_for_type_inference(dtype='int32') target_bbox = helper.create_variable_for_type_inference( - dtype=anchor_box.dtype) + dtype=anchor_box.dtype + ) bbox_inside_weight = helper.create_variable_for_type_inference( - dtype=anchor_box.dtype) - helper.append_op(type="rpn_target_assign", - inputs={ - 'Anchor': anchor_box, - 'GtBoxes': gt_boxes, - 'IsCrowd': is_crowd, - 'ImInfo': im_info - }, - outputs={ - 'LocationIndex': loc_index, - 'ScoreIndex': score_index, - 'TargetLabel': target_label, - 'TargetBBox': target_bbox, - 'BBoxInsideWeight': bbox_inside_weight - }, - attrs={ - 'rpn_batch_size_per_im': rpn_batch_size_per_im, - 'rpn_straddle_thresh': rpn_straddle_thresh, - 'rpn_positive_overlap': rpn_positive_overlap, - 'rpn_negative_overlap': rpn_negative_overlap, - 'rpn_fg_fraction': rpn_fg_fraction, - 'use_random': use_random - }) + dtype=anchor_box.dtype + ) + helper.append_op( + type="rpn_target_assign", + inputs={ + 'Anchor': anchor_box, + 'GtBoxes': gt_boxes, + 'IsCrowd': is_crowd, + 'ImInfo': im_info, + }, + outputs={ + 'LocationIndex': loc_index, + 'ScoreIndex': score_index, + 'TargetLabel': target_label, + 'TargetBBox': target_bbox, + 'BBoxInsideWeight': bbox_inside_weight, + }, + attrs={ + 'rpn_batch_size_per_im': rpn_batch_size_per_im, + 'rpn_straddle_thresh': rpn_straddle_thresh, + 'rpn_positive_overlap': rpn_positive_overlap, + 'rpn_negative_overlap': rpn_negative_overlap, + 'rpn_fg_fraction': rpn_fg_fraction, + 'use_random': use_random, + }, + ) loc_index.stop_gradient = True score_index.stop_gradient = True @@ -467,7 +518,13 @@ def rpn_target_assign(bbox_pred, predicted_cls_logits = nn.gather(cls_logits, score_index) predicted_bbox_pred = nn.gather(bbox_pred, loc_index) - return predicted_cls_logits, predicted_bbox_pred, target_label, target_bbox, bbox_inside_weight + return ( + predicted_cls_logits, + predicted_bbox_pred, + target_label, + target_bbox, + bbox_inside_weight, + ) def sigmoid_focal_loss(x, label, fg_num, gamma=2.0, alpha=0.25): @@ -597,8 +654,9 @@ def sigmoid_focal_loss(x, label, fg_num, gamma=2.0, alpha=0.25): print(outs) """ - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'sigmoid_focal_loss') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'sigmoid_focal_loss' + ) check_variable_and_dtype(label, 'label', ['int32'], 'sigmoid_focal_loss') check_variable_and_dtype(fg_num, 'fg_num', ['int32'], 'sigmoid_focal_loss') @@ -606,31 +664,28 @@ def sigmoid_focal_loss(x, label, fg_num, gamma=2.0, alpha=0.25): out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="sigmoid_focal_loss", - inputs={ - "X": x, - "Label": label, - "FgNum": fg_num - }, - attrs={ - "gamma": gamma, - 'alpha': alpha - }, - outputs={"Out": out}) + helper.append_op( + type="sigmoid_focal_loss", + inputs={"X": x, "Label": label, "FgNum": fg_num}, + attrs={"gamma": gamma, 'alpha': alpha}, + outputs={"Out": out}, + ) return out -def detection_output(loc, - scores, - prior_box, - prior_box_var, - background_label=0, - nms_threshold=0.3, - nms_top_k=400, - keep_top_k=200, - score_threshold=0.01, - nms_eta=1.0, - return_index=False): +def detection_output( + loc, + scores, + prior_box, + prior_box_var, + background_label=0, + nms_threshold=0.3, + nms_top_k=400, + keep_top_k=200, + score_threshold=0.01, + nms_eta=1.0, + return_index=False, +): """ Given the regression locations, classification confidences and prior boxes, @@ -715,50 +770,48 @@ def detection_output(loc, return_index=True) """ helper = LayerHelper("detection_output", **locals()) - decoded_box = box_coder(prior_box=prior_box, - prior_box_var=prior_box_var, - target_box=loc, - code_type='decode_center_size') + decoded_box = box_coder( + prior_box=prior_box, + prior_box_var=prior_box_var, + target_box=loc, + code_type='decode_center_size', + ) scores = nn.softmax(input=scores) scores = nn.transpose(scores, perm=[0, 2, 1]) scores.stop_gradient = True nmsed_outs = helper.create_variable_for_type_inference( - dtype=decoded_box.dtype) + dtype=decoded_box.dtype + ) if return_index: index = helper.create_variable_for_type_inference(dtype='int') - helper.append_op(type="multiclass_nms2", - inputs={ - 'Scores': scores, - 'BBoxes': decoded_box - }, - outputs={ - 'Out': nmsed_outs, - 'Index': index - }, - attrs={ - 'background_label': 0, - 'nms_threshold': nms_threshold, - 'nms_top_k': nms_top_k, - 'keep_top_k': keep_top_k, - 'score_threshold': score_threshold, - 'nms_eta': 1.0, - }) + helper.append_op( + type="multiclass_nms2", + inputs={'Scores': scores, 'BBoxes': decoded_box}, + outputs={'Out': nmsed_outs, 'Index': index}, + attrs={ + 'background_label': 0, + 'nms_threshold': nms_threshold, + 'nms_top_k': nms_top_k, + 'keep_top_k': keep_top_k, + 'score_threshold': score_threshold, + 'nms_eta': 1.0, + }, + ) index.stop_gradient = True else: - helper.append_op(type="multiclass_nms", - inputs={ - 'Scores': scores, - 'BBoxes': decoded_box - }, - outputs={'Out': nmsed_outs}, - attrs={ - 'background_label': 0, - 'nms_threshold': nms_threshold, - 'nms_top_k': nms_top_k, - 'keep_top_k': keep_top_k, - 'score_threshold': score_threshold, - 'nms_eta': 1.0, - }) + helper.append_op( + type="multiclass_nms", + inputs={'Scores': scores, 'BBoxes': decoded_box}, + outputs={'Out': nmsed_outs}, + attrs={ + 'background_label': 0, + 'nms_threshold': nms_threshold, + 'nms_top_k': nms_top_k, + 'keep_top_k': keep_top_k, + 'score_threshold': score_threshold, + 'nms_eta': 1.0, + }, + ) nmsed_outs.stop_gradient = True if return_index: return nmsed_outs, index @@ -768,9 +821,9 @@ def detection_output(loc, @templatedoc() def iou_similarity(x, y, box_normalized=True, name=None): """ - :alias_main: paddle.nn.functional.iou_similarity - :alias: paddle.nn.functional.iou_similarity,paddle.nn.functional.loss.iou_similarity - :old_api: paddle.fluid.layers.iou_similarity + :alias_main: paddle.nn.functional.iou_similarity + :alias: paddle.nn.functional.iou_similarity,paddle.nn.functional.loss.iou_similarity + :old_api: paddle.fluid.layers.iou_similarity ${comment} @@ -810,24 +863,25 @@ def iou_similarity(x, y, box_normalized=True, name=None): helper = LayerHelper("iou_similarity", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="iou_similarity", - inputs={ - "X": x, - "Y": y - }, - attrs={"box_normalized": box_normalized}, - outputs={"Out": out}) + helper.append_op( + type="iou_similarity", + inputs={"X": x, "Y": y}, + attrs={"box_normalized": box_normalized}, + outputs={"Out": out}, + ) return out @templatedoc() -def box_coder(prior_box, - prior_box_var, - target_box, - code_type="encode_center_size", - box_normalized=True, - name=None, - axis=0): +def box_coder( + prior_box, + prior_box_var, + target_box, + code_type="encode_center_size", + box_normalized=True, + name=None, + axis=0, +): r""" **Box Coder Layer** @@ -942,33 +996,49 @@ def box_coder(prior_box, box_normalized=False, axis=1) """ - check_variable_and_dtype(prior_box, 'prior_box', ['float32', 'float64'], - 'box_coder') - check_variable_and_dtype(target_box, 'target_box', ['float32', 'float64'], - 'box_coder') + check_variable_and_dtype( + prior_box, 'prior_box', ['float32', 'float64'], 'box_coder' + ) + check_variable_and_dtype( + target_box, 'target_box', ['float32', 'float64'], 'box_coder' + ) if in_dygraph_mode(): if isinstance(prior_box_var, Variable): - box_coder_op = _C_ops.box_coder(prior_box, prior_box_var, - target_box, code_type, - box_normalized, axis, []) + box_coder_op = _C_ops.box_coder( + prior_box, + prior_box_var, + target_box, + code_type, + box_normalized, + axis, + [], + ) elif isinstance(prior_box_var, list): - box_coder_op = _C_ops.box_coder(prior_box, None, target_box, - code_type, box_normalized, axis, - prior_box_var) + box_coder_op = _C_ops.box_coder( + prior_box, + None, + target_box, + code_type, + box_normalized, + axis, + prior_box_var, + ) else: raise TypeError( - "Input variance of box_coder must be Variable or lisz") + "Input variance of box_coder must be Variable or lisz" + ) return box_coder_op helper = LayerHelper("box_coder", **locals()) output_box = helper.create_variable_for_type_inference( - dtype=prior_box.dtype) + dtype=prior_box.dtype + ) inputs = {"PriorBox": prior_box, "TargetBox": target_box} attrs = { "code_type": code_type, "box_normalized": box_normalized, - "axis": axis + "axis": axis, } if isinstance(prior_box_var, Variable): inputs['PriorBoxVar'] = prior_box_var @@ -976,10 +1046,12 @@ def box_coder(prior_box, attrs['variance'] = prior_box_var else: raise TypeError("Input variance of box_coder must be Variable or lisz") - helper.append_op(type="box_coder", - inputs=inputs, - attrs=attrs, - outputs={"OutputBox": output_box}) + helper.append_op( + type="box_coder", + inputs=inputs, + attrs=attrs, + outputs={"OutputBox": output_box}, + ) return output_box @@ -1004,32 +1076,37 @@ def polygon_box_transform(input, name=None): input = fluid.data(name='input', shape=[4, 10, 5, 5], dtype='float32') out = fluid.layers.polygon_box_transform(input) """ - check_variable_and_dtype(input, "input", ['float32', 'float64'], - 'polygon_box_transform') + check_variable_and_dtype( + input, "input", ['float32', 'float64'], 'polygon_box_transform' + ) helper = LayerHelper("polygon_box_transform", **locals()) output = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="polygon_box_transform", - inputs={"Input": input}, - attrs={}, - outputs={"Output": output}) + helper.append_op( + type="polygon_box_transform", + inputs={"Input": input}, + attrs={}, + outputs={"Output": output}, + ) return output @deprecated(since="2.0.0", update_to="paddle.vision.ops.yolo_loss") @templatedoc(op_type="yolov3_loss") -def yolov3_loss(x, - gt_box, - gt_label, - anchors, - anchor_mask, - class_num, - ignore_thresh, - downsample_ratio, - gt_score=None, - use_label_smooth=True, - name=None, - scale_x_y=1.): +def yolov3_loss( + x, + gt_box, + gt_label, + anchors, + anchor_mask, + class_num, + ignore_thresh, + downsample_ratio, + gt_score=None, + use_label_smooth=True, + name=None, + scale_x_y=1.0, +): """ ${comment} @@ -1105,18 +1182,33 @@ def yolov3_loss(x, raise TypeError("Attr class_num of yolov3_loss must be an integer") if not isinstance(ignore_thresh, float): raise TypeError( - "Attr ignore_thresh of yolov3_loss must be a float number") + "Attr ignore_thresh of yolov3_loss must be a float number" + ) if not isinstance(use_label_smooth, bool): raise TypeError( - "Attr use_label_smooth of yolov3_loss must be a bool value") + "Attr use_label_smooth of yolov3_loss must be a bool value" + ) if _non_static_mode(): - attrs = ("anchors", anchors, "anchor_mask", anchor_mask, "class_num", - class_num, "ignore_thresh", ignore_thresh, "downsample_ratio", - downsample_ratio, "use_label_smooth", use_label_smooth, - "scale_x_y", scale_x_y) - loss, _, _ = _legacy_C_ops.yolov3_loss(x, gt_box, gt_label, gt_score, - *attrs) + attrs = ( + "anchors", + anchors, + "anchor_mask", + anchor_mask, + "class_num", + class_num, + "ignore_thresh", + ignore_thresh, + "downsample_ratio", + downsample_ratio, + "use_label_smooth", + use_label_smooth, + "scale_x_y", + scale_x_y, + ) + loss, _, _ = _legacy_C_ops.yolov3_loss( + x, gt_box, gt_label, gt_score, *attrs + ) return loss helper = LayerHelper('yolov3_loss', **locals()) @@ -1142,30 +1234,34 @@ def yolov3_loss(x, "scale_x_y": scale_x_y, } - helper.append_op(type='yolov3_loss', - inputs=inputs, - outputs={ - 'Loss': loss, - 'ObjectnessMask': objectness_mask, - 'GTMatchMask': gt_match_mask - }, - attrs=attrs) + helper.append_op( + type='yolov3_loss', + inputs=inputs, + outputs={ + 'Loss': loss, + 'ObjectnessMask': objectness_mask, + 'GTMatchMask': gt_match_mask, + }, + attrs=attrs, + ) return loss @deprecated(since="2.0.0", update_to="paddle.vision.ops.yolo_box") @templatedoc(op_type="yolo_box") -def yolo_box(x, - img_size, - anchors, - class_num, - conf_thresh, - downsample_ratio, - clip_bbox=True, - name=None, - scale_x_y=1., - iou_aware=False, - iou_aware_factor=0.5): +def yolo_box( + x, + img_size, + anchors, + class_num, + conf_thresh, + downsample_ratio, + clip_bbox=True, + name=None, + scale_x_y=1.0, + iou_aware=False, + iou_aware_factor=0.5, +): """ ${comment} @@ -1233,33 +1329,37 @@ def yolo_box(x, "clip_bbox": clip_bbox, "scale_x_y": scale_x_y, "iou_aware": iou_aware, - "iou_aware_factor": iou_aware_factor + "iou_aware_factor": iou_aware_factor, } - helper.append_op(type='yolo_box', - inputs={ - "X": x, - "ImgSize": img_size, - }, - outputs={ - 'Boxes': boxes, - 'Scores': scores, - }, - attrs=attrs) + helper.append_op( + type='yolo_box', + inputs={ + "X": x, + "ImgSize": img_size, + }, + outputs={ + 'Boxes': boxes, + 'Scores': scores, + }, + attrs=attrs, + ) return boxes, scores @templatedoc() -def detection_map(detect_res, - label, - class_num, - background_label=0, - overlap_threshold=0.3, - evaluate_difficult=True, - has_state=None, - input_states=None, - out_states=None, - ap_version='integral'): +def detection_map( + detect_res, + label, + class_num, + background_label=0, + overlap_threshold=0.3, + evaluate_difficult=True, + has_state=None, + input_states=None, + out_states=None, + ap_version='integral', +): """ ${comment} @@ -1307,45 +1407,49 @@ def detection_map(detect_res, return helper.create_variable_for_type_inference(dtype=type) map_out = __create_var('float32') - accum_pos_count_out = out_states[ - 0] if out_states is not None else __create_var('int32') - accum_true_pos_out = out_states[ - 1] if out_states is not None else __create_var('float32') - accum_false_pos_out = out_states[ - 2] if out_states is not None else __create_var('float32') + accum_pos_count_out = ( + out_states[0] if out_states is not None else __create_var('int32') + ) + accum_true_pos_out = ( + out_states[1] if out_states is not None else __create_var('float32') + ) + accum_false_pos_out = ( + out_states[2] if out_states is not None else __create_var('float32') + ) pos_count = input_states[0] if input_states is not None else None true_pos = input_states[1] if input_states is not None else None false_pos = input_states[2] if input_states is not None else None - helper.append_op(type="detection_map", - inputs={ - 'Label': label, - 'DetectRes': detect_res, - 'HasState': has_state, - 'PosCount': pos_count, - 'TruePos': true_pos, - 'FalsePos': false_pos - }, - outputs={ - 'MAP': map_out, - 'AccumPosCount': accum_pos_count_out, - 'AccumTruePos': accum_true_pos_out, - 'AccumFalsePos': accum_false_pos_out - }, - attrs={ - 'overlap_threshold': overlap_threshold, - 'evaluate_difficult': evaluate_difficult, - 'ap_type': ap_version, - 'class_num': class_num, - }) + helper.append_op( + type="detection_map", + inputs={ + 'Label': label, + 'DetectRes': detect_res, + 'HasState': has_state, + 'PosCount': pos_count, + 'TruePos': true_pos, + 'FalsePos': false_pos, + }, + outputs={ + 'MAP': map_out, + 'AccumPosCount': accum_pos_count_out, + 'AccumTruePos': accum_true_pos_out, + 'AccumFalsePos': accum_false_pos_out, + }, + attrs={ + 'overlap_threshold': overlap_threshold, + 'evaluate_difficult': evaluate_difficult, + 'ap_type': ap_version, + 'class_num': class_num, + }, + ) return map_out -def bipartite_match(dist_matrix, - match_type=None, - dist_threshold=None, - name=None): +def bipartite_match( + dist_matrix, match_type=None, dist_threshold=None, name=None +): """ This operator implements a greedy bipartite matching algorithm, which is @@ -1417,25 +1521,30 @@ def bipartite_match(dist_matrix, helper = LayerHelper('bipartite_match', **locals()) match_indices = helper.create_variable_for_type_inference(dtype='int32') match_distance = helper.create_variable_for_type_inference( - dtype=dist_matrix.dtype) - helper.append_op(type='bipartite_match', - inputs={'DistMat': dist_matrix}, - attrs={ - 'match_type': match_type, - 'dist_threshold': dist_threshold, - }, - outputs={ - 'ColToRowMatchIndices': match_indices, - 'ColToRowMatchDist': match_distance - }) + dtype=dist_matrix.dtype + ) + helper.append_op( + type='bipartite_match', + inputs={'DistMat': dist_matrix}, + attrs={ + 'match_type': match_type, + 'dist_threshold': dist_threshold, + }, + outputs={ + 'ColToRowMatchIndices': match_indices, + 'ColToRowMatchDist': match_distance, + }, + ) return match_indices, match_distance -def target_assign(input, - matched_indices, - negative_indices=None, - mismatch_value=None, - name=None): +def target_assign( + input, + matched_indices, + negative_indices=None, + mismatch_value=None, + name=None, +): """ This operator can be, for given the target bounding boxes or labels, @@ -1524,36 +1633,37 @@ def target_assign(input, helper = LayerHelper('target_assign', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) out_weight = helper.create_variable_for_type_inference(dtype='float32') - helper.append_op(type='target_assign', - inputs={ - 'X': input, - 'MatchIndices': matched_indices, - 'NegIndices': negative_indices - }, - outputs={ - 'Out': out, - 'OutWeight': out_weight - }, - attrs={'mismatch_value': mismatch_value}) + helper.append_op( + type='target_assign', + inputs={ + 'X': input, + 'MatchIndices': matched_indices, + 'NegIndices': negative_indices, + }, + outputs={'Out': out, 'OutWeight': out_weight}, + attrs={'mismatch_value': mismatch_value}, + ) return out, out_weight -def ssd_loss(location, - confidence, - gt_box, - gt_label, - prior_box, - prior_box_var=None, - background_label=0, - overlap_threshold=0.5, - neg_pos_ratio=3.0, - neg_overlap=0.5, - loc_loss_weight=1.0, - conf_loss_weight=1.0, - match_type='per_prediction', - mining_type='max_negative', - normalize=True, - sample_size=None): +def ssd_loss( + location, + confidence, + gt_box, + gt_label, + prior_box, + prior_box_var=None, + background_label=0, + overlap_threshold=0.5, + neg_pos_ratio=3.0, + neg_overlap=0.5, + loc_loss_weight=1.0, + conf_loss_weight=1.0, + match_type='per_prediction', + mining_type='max_negative', + normalize=True, + sample_size=None, +): r""" :alias_main: paddle.nn.functional.ssd_loss :alias: paddle.nn.functional.ssd_loss,paddle.nn.functional.loss.ssd_loss @@ -1685,17 +1795,19 @@ def ssd_loss(location, # 1.1 Compute IOU similarity between ground-truth boxes and prior boxes. iou = iou_similarity(x=gt_box, y=prior_box) # 1.2 Compute matched bounding box by bipartite matching algorithm. - matched_indices, matched_dist = bipartite_match(iou, match_type, - overlap_threshold) + matched_indices, matched_dist = bipartite_match( + iou, match_type, overlap_threshold + ) # 2. Compute confidence for mining hard examples # 2.1. Get the target label based on matched indices - gt_label = nn.reshape(x=gt_label, - shape=(len(gt_label.shape) - 1) * (0, ) + (-1, 1)) + gt_label = nn.reshape( + x=gt_label, shape=(len(gt_label.shape) - 1) * (0,) + (-1, 1) + ) gt_label.stop_gradient = True - target_label, _ = target_assign(gt_label, - matched_indices, - mismatch_value=background_label) + target_label, _ = target_assign( + gt_label, matched_indices, mismatch_value=background_label + ) # 2.2. Compute confidence loss. # Reshape confidence to 2D tensor. confidence = __reshape_to_2d(confidence) @@ -1708,47 +1820,54 @@ def ssd_loss(location, actual_shape.stop_gradient = True # shape=(-1, 0) is set for compile-time, the correct shape is set by # actual_shape in runtime. - conf_loss = nn.reshape(x=conf_loss, - shape=(-1, 0), - actual_shape=actual_shape) + conf_loss = nn.reshape( + x=conf_loss, shape=(-1, 0), actual_shape=actual_shape + ) conf_loss.stop_gradient = True neg_indices = helper.create_variable_for_type_inference(dtype='int32') dtype = matched_indices.dtype updated_matched_indices = helper.create_variable_for_type_inference( - dtype=dtype) - helper.append_op(type='mine_hard_examples', - inputs={ - 'ClsLoss': conf_loss, - 'LocLoss': None, - 'MatchIndices': matched_indices, - 'MatchDist': matched_dist, - }, - outputs={ - 'NegIndices': neg_indices, - 'UpdatedMatchIndices': updated_matched_indices - }, - attrs={ - 'neg_pos_ratio': neg_pos_ratio, - 'neg_dist_threshold': neg_overlap, - 'mining_type': mining_type, - 'sample_size': sample_size, - }) + dtype=dtype + ) + helper.append_op( + type='mine_hard_examples', + inputs={ + 'ClsLoss': conf_loss, + 'LocLoss': None, + 'MatchIndices': matched_indices, + 'MatchDist': matched_dist, + }, + outputs={ + 'NegIndices': neg_indices, + 'UpdatedMatchIndices': updated_matched_indices, + }, + attrs={ + 'neg_pos_ratio': neg_pos_ratio, + 'neg_dist_threshold': neg_overlap, + 'mining_type': mining_type, + 'sample_size': sample_size, + }, + ) # 4. Assign classification and regression targets # 4.1. Encoded bbox according to the prior boxes. - encoded_bbox = box_coder(prior_box=prior_box, - prior_box_var=prior_box_var, - target_box=gt_box, - code_type='encode_center_size') + encoded_bbox = box_coder( + prior_box=prior_box, + prior_box_var=prior_box_var, + target_box=gt_box, + code_type='encode_center_size', + ) # 4.2. Assign regression targets target_bbox, target_loc_weight = target_assign( - encoded_bbox, updated_matched_indices, mismatch_value=background_label) + encoded_bbox, updated_matched_indices, mismatch_value=background_label + ) # 4.3. Assign classification targets target_label, target_conf_weight = target_assign( gt_label, updated_matched_indices, negative_indices=neg_indices, - mismatch_value=background_label) + mismatch_value=background_label, + ) # 5. Compute loss. # 5.1 Compute confidence loss. @@ -1794,7 +1913,7 @@ def prior_box( image, min_sizes, max_sizes=None, - aspect_ratios=[1.], + aspect_ratios=[1.0], variance=[0.1, 0.1, 0.2, 0.2], flip=False, clip=False, @@ -1840,66 +1959,66 @@ def prior_box( Tuple: A tuple with two Variable (boxes, variances) boxes(Variable): the output prior boxes of PriorBox. - 4-D tensor, the layout is [H, W, num_priors, 4]. + 4-D tensor, the layout is [H, W, num_priors, 4]. H is the height of input, W is the width of input, num_priors is the total box count of each position of input. variances(Variable): the expanded variances of PriorBox. - 4-D tensor, the layput is [H, W, num_priors, 4]. + 4-D tensor, the layput is [H, W, num_priors, 4]. H is the height of input, W is the width of input num_priors is the total box count of each position of input Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import numpy as np + #declarative mode + import paddle.fluid as fluid + import numpy as np import paddle paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,9]) - image = fluid.data(name="image", shape=[None,3,9,12]) - box, var = fluid.layers.prior_box( + input = fluid.data(name="input", shape=[None,3,6,9]) + image = fluid.data(name="image", shape=[None,3,9,12]) + box, var = fluid.layers.prior_box( input=input, image=image, - min_sizes=[100.], + min_sizes=[100.], clip=True, flip=True) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - # prepare a batch of data - input_data = np.random.rand(1,3,6,9).astype("float32") - image_data = np.random.rand(1,3,9,12).astype("float32") + # prepare a batch of data + input_data = np.random.rand(1,3,6,9).astype("float32") + image_data = np.random.rand(1,3,9,12).astype("float32") - box_out, var_out = exe.run(fluid.default_main_program(), + box_out, var_out = exe.run(fluid.default_main_program(), feed={"input":input_data,"image":image_data}, fetch_list=[box,var], return_numpy=True) - # print(box_out.shape) - # (6, 9, 1, 4) - # print(var_out.shape) - # (6, 9, 1, 4) - - # imperative mode - import paddle.fluid.dygraph as dg - - with dg.guard(place) as g: - input = dg.to_variable(input_data) - image = dg.to_variable(image_data) - box, var = fluid.layers.prior_box( - input=input, - image=image, - min_sizes=[100.], - clip=True, - flip=True) - # print(box.shape) - # [6L, 9L, 1L, 4L] + # print(box_out.shape) + # (6, 9, 1, 4) + # print(var_out.shape) + # (6, 9, 1, 4) + + # imperative mode + import paddle.fluid.dygraph as dg + + with dg.guard(place) as g: + input = dg.to_variable(input_data) + image = dg.to_variable(image_data) + box, var = fluid.layers.prior_box( + input=input, + image=image, + min_sizes=[100.], + clip=True, + flip=True) + # print(box.shape) + # [6L, 9L, 1L, 4L] # print(var.shape) - # [6L, 9L, 1L, 4L] + # [6L, 9L, 1L, 4L] """ @@ -1907,25 +2026,38 @@ def prior_box( step_w, step_h = steps if max_sizes == None: max_sizes = [] - return _C_ops.prior_box(input, image, min_sizes, aspect_ratios, - variance, max_sizes, flip, clip, step_w, step_h, - offset, min_max_aspect_ratios_order) + return _C_ops.prior_box( + input, + image, + min_sizes, + aspect_ratios, + variance, + max_sizes, + flip, + clip, + step_w, + step_h, + offset, + min_max_aspect_ratios_order, + ) helper = LayerHelper("prior_box", **locals()) dtype = helper.input_dtype() - check_variable_and_dtype(input, 'input', - ['uint8', 'int8', 'float32', 'float64'], - 'prior_box') + check_variable_and_dtype( + input, 'input', ['uint8', 'int8', 'float32', 'float64'], 'prior_box' + ) def _is_list_or_tuple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if not _is_list_or_tuple_(min_sizes): min_sizes = [min_sizes] if not _is_list_or_tuple_(aspect_ratios): aspect_ratios = [aspect_ratios] if not (_is_list_or_tuple_(steps) and len(steps) == 2): - raise ValueError('steps should be a list or tuple ', - 'with length 2, (step_width, step_height).') + raise ValueError( + 'steps should be a list or tuple ', + 'with length 2, (step_width, step_height).', + ) min_sizes = list(map(float, min_sizes)) aspect_ratios = list(map(float, aspect_ratios)) @@ -1940,7 +2072,7 @@ def prior_box( 'step_w': steps[0], 'step_h': steps[1], 'offset': offset, - 'min_max_aspect_ratios_order': min_max_aspect_ratios_order + 'min_max_aspect_ratios_order': min_max_aspect_ratios_order, } if max_sizes is not None and len(max_sizes) > 0 and max_sizes[0] > 0: if not _is_list_or_tuple_(max_sizes): @@ -1951,14 +2083,8 @@ def prior_box( var = helper.create_variable_for_type_inference(dtype) helper.append_op( type="prior_box", - inputs={ - "Input": input, - "Image": image - }, - outputs={ - "Boxes": box, - "Variances": var - }, + inputs={"Input": input, "Image": image}, + outputs={"Boxes": box, "Variances": var}, attrs=attrs, ) box.stop_gradient = True @@ -1966,17 +2092,19 @@ def prior_box( return box, var -def density_prior_box(input, - image, - densities=None, - fixed_sizes=None, - fixed_ratios=None, - variance=[0.1, 0.1, 0.2, 0.2], - clip=False, - steps=[0.0, 0.0], - offset=0.5, - flatten_to_2d=False, - name=None): +def density_prior_box( + input, + image, + densities=None, + fixed_sizes=None, + fixed_ratios=None, + variance=[0.1, 0.1, 0.2, 0.2], + clip=False, + steps=[0.0, 0.0], + offset=0.5, + flatten_to_2d=False, + name=None, +): r""" This op generates density prior boxes for SSD(Single Shot MultiBox Detector) @@ -2100,11 +2228,12 @@ def density_prior_box(input, """ helper = LayerHelper("density_prior_box", **locals()) dtype = helper.input_dtype() - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'density_prior_box') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'density_prior_box' + ) def _is_list_or_tuple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) check_type(densities, 'densities', (list, tuple), 'density_prior_box') check_type(fixed_sizes, 'fixed_sizes', (list, tuple), 'density_prior_box') @@ -2113,8 +2242,10 @@ def density_prior_box(input, raise ValueError('densities and fixed_sizes length should be euqal.') if not (_is_list_or_tuple_(steps) and len(steps) == 2): - raise ValueError('steps should be a list or tuple ', - 'with length 2, (step_width, step_height).') + raise ValueError( + 'steps should be a list or tuple ', + 'with length 2, (step_width, step_height).', + ) densities = list(map(int, densities)) fixed_sizes = list(map(float, fixed_sizes)) @@ -2136,14 +2267,8 @@ def density_prior_box(input, var = helper.create_variable_for_type_inference(dtype) helper.append_op( type="density_prior_box", - inputs={ - "Input": input, - "Image": image - }, - outputs={ - "Boxes": box, - "Variances": var - }, + inputs={"Input": input, "Image": image}, + outputs={"Boxes": box, "Variances": var}, attrs=attrs, ) box.stop_gradient = True @@ -2152,29 +2277,31 @@ def density_prior_box(input, @static_only -def multi_box_head(inputs, - image, - base_size, - num_classes, - aspect_ratios, - min_ratio=None, - max_ratio=None, - min_sizes=None, - max_sizes=None, - steps=None, - step_w=None, - step_h=None, - offset=0.5, - variance=[0.1, 0.1, 0.2, 0.2], - flip=True, - clip=False, - kernel_size=1, - pad=0, - stride=1, - name=None, - min_max_aspect_ratios_order=False): +def multi_box_head( + inputs, + image, + base_size, + num_classes, + aspect_ratios, + min_ratio=None, + max_ratio=None, + min_sizes=None, + max_sizes=None, + steps=None, + step_w=None, + step_h=None, + offset=0.5, + variance=[0.1, 0.1, 0.2, 0.2], + flip=True, + clip=False, + kernel_size=1, + pad=0, + stride=1, + name=None, + min_max_aspect_ratios_order=False, +): """ - :api_attr: Static Graph + :api_attr: Static Graph Base on SSD ((Single Shot MultiBox Detector) algorithm, generate prior boxes, regression location and classification confidence on multiple input feature @@ -2319,7 +2446,7 @@ def multi_box_head(inputs, return out def _is_list_or_tuple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) def _is_list_or_tuple_and_equal(data, length, err_info): if not (_is_list_or_tuple_(data) and len(data) == length): @@ -2338,31 +2465,39 @@ def multi_box_head(inputs, max_sizes = [] step = int(math.floor(((max_ratio - min_ratio)) / (num_layer - 2))) for ratio in range(min_ratio, max_ratio + 1, step): - min_sizes.append(base_size * ratio / 100.) - max_sizes.append(base_size * (ratio + step) / 100.) - min_sizes = [base_size * .10] + min_sizes - max_sizes = [base_size * .20] + max_sizes + min_sizes.append(base_size * ratio / 100.0) + max_sizes.append(base_size * (ratio + step) / 100.0) + min_sizes = [base_size * 0.10] + min_sizes + max_sizes = [base_size * 0.20] + max_sizes if aspect_ratios: _is_list_or_tuple_and_equal( - aspect_ratios, num_layer, + aspect_ratios, + num_layer, 'aspect_ratios should be list or tuple, and the length of inputs ' - 'and aspect_ratios should be the same.') + 'and aspect_ratios should be the same.', + ) if step_h is not None: _is_list_or_tuple_and_equal( - step_h, num_layer, + step_h, + num_layer, 'step_h should be list or tuple, and the length of inputs and ' - 'step_h should be the same.') + 'step_h should be the same.', + ) if step_w is not None: _is_list_or_tuple_and_equal( - step_w, num_layer, + step_w, + num_layer, 'step_w should be list or tuple, and the length of inputs and ' - 'step_w should be the same.') + 'step_w should be the same.', + ) if steps is not None: _is_list_or_tuple_and_equal( - steps, num_layer, + steps, + num_layer, 'steps should be list or tuple, and the length of inputs and ' - 'step_w should be the same.') + 'step_w should be the same.', + ) step_w = steps step_h = steps @@ -2386,9 +2521,20 @@ def multi_box_head(inputs, aspect_ratio = [aspect_ratio] step = [step_w[i] if step_w else 0.0, step_h[i] if step_w else 0.0] - box, var = prior_box(input, image, min_size, max_size, aspect_ratio, - variance, flip, clip, step, offset, None, - min_max_aspect_ratios_order) + box, var = prior_box( + input, + image, + min_size, + max_size, + aspect_ratio, + variance, + flip, + clip, + step, + offset, + None, + min_max_aspect_ratios_order, + ) box_results.append(box) var_results.append(var) @@ -2397,11 +2543,13 @@ def multi_box_head(inputs, # get loc num_loc_output = num_boxes * 4 - mbox_loc = nn.conv2d(input=input, - num_filters=num_loc_output, - filter_size=kernel_size, - padding=pad, - stride=stride) + mbox_loc = nn.conv2d( + input=input, + num_filters=num_loc_output, + filter_size=kernel_size, + padding=pad, + stride=stride, + ) mbox_loc = nn.transpose(mbox_loc, perm=[0, 2, 3, 1]) mbox_loc_flatten = nn.flatten(mbox_loc, axis=1) @@ -2409,11 +2557,13 @@ def multi_box_head(inputs, # get conf num_conf_output = num_boxes * num_classes - conf_loc = nn.conv2d(input=input, - num_filters=num_conf_output, - filter_size=kernel_size, - padding=pad, - stride=stride) + conf_loc = nn.conv2d( + input=input, + num_filters=num_conf_output, + filter_size=kernel_size, + padding=pad, + stride=stride, + ) conf_loc = nn.transpose(conf_loc, perm=[0, 2, 3, 1]) conf_loc_flatten = nn.flatten(conf_loc, axis=1) mbox_confs.append(conf_loc_flatten) @@ -2435,21 +2585,24 @@ def multi_box_head(inputs, mbox_locs_concat = tensor.concat(mbox_locs, axis=1) mbox_locs_concat = nn.reshape(mbox_locs_concat, shape=[0, -1, 4]) mbox_confs_concat = tensor.concat(mbox_confs, axis=1) - mbox_confs_concat = nn.reshape(mbox_confs_concat, - shape=[0, -1, num_classes]) + mbox_confs_concat = nn.reshape( + mbox_confs_concat, shape=[0, -1, num_classes] + ) box.stop_gradient = True var.stop_gradient = True return mbox_locs_concat, mbox_confs_concat, box, var -def anchor_generator(input, - anchor_sizes=None, - aspect_ratios=None, - variance=[0.1, 0.1, 0.2, 0.2], - stride=None, - offset=0.5, - name=None): +def anchor_generator( + input, + anchor_sizes=None, + aspect_ratios=None, + variance=[0.1, 0.1, 0.2, 0.2], + stride=None, + offset=0.5, + name=None, +): """ **Anchor generator operator** @@ -2513,15 +2666,17 @@ def anchor_generator(input, dtype = helper.input_dtype() def _is_list_or_tuple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if not _is_list_or_tuple_(anchor_sizes): anchor_sizes = [anchor_sizes] if not _is_list_or_tuple_(aspect_ratios): aspect_ratios = [aspect_ratios] if not (_is_list_or_tuple_(stride) and len(stride) == 2): - raise ValueError('stride should be a list or tuple ', - 'with length 2, (stride_width, stride_height).') + raise ValueError( + 'stride should be a list or tuple ', + 'with length 2, (stride_width, stride_height).', + ) anchor_sizes = list(map(float, anchor_sizes)) aspect_ratios = list(map(float, aspect_ratios)) @@ -2532,7 +2687,7 @@ def anchor_generator(input, 'aspect_ratios': aspect_ratios, 'variances': variance, 'stride': stride, - 'offset': offset + 'offset': offset, } anchor = helper.create_variable_for_type_inference(dtype) @@ -2540,10 +2695,7 @@ def anchor_generator(input, helper.append_op( type="anchor_generator", inputs={"Input": input}, - outputs={ - "Anchors": anchor, - "Variances": var - }, + outputs={"Anchors": anchor, "Variances": var}, attrs=attrs, ) anchor.stop_gradient = True @@ -2551,12 +2703,14 @@ def anchor_generator(input, return anchor, var -def roi_perspective_transform(input, - rois, - transformed_height, - transformed_width, - spatial_scale=1.0, - name=None): +def roi_perspective_transform( + input, + rois, + transformed_height, + transformed_width, + spatial_scale=1.0, + name=None, +): """ **The** `rois` **of this op should be a LoDTensor.** @@ -2606,16 +2760,24 @@ def roi_perspective_transform(input, rois = fluid.data(name='rois', shape=[None, 8], lod_level=1, dtype='float32') out, mask, transform_matrix = fluid.layers.roi_perspective_transform(x, rois, 7, 7, 1.0) """ - check_variable_and_dtype(input, 'input', ['float32'], - 'roi_perspective_transform') - check_variable_and_dtype(rois, 'rois', ['float32'], - 'roi_perspective_transform') - check_type(transformed_height, 'transformed_height', int, - 'roi_perspective_transform') - check_type(transformed_width, 'transformed_width', int, - 'roi_perspective_transform') - check_type(spatial_scale, 'spatial_scale', float, - 'roi_perspective_transform') + check_variable_and_dtype( + input, 'input', ['float32'], 'roi_perspective_transform' + ) + check_variable_and_dtype( + rois, 'rois', ['float32'], 'roi_perspective_transform' + ) + check_type( + transformed_height, + 'transformed_height', + int, + 'roi_perspective_transform', + ) + check_type( + transformed_width, 'transformed_width', int, 'roi_perspective_transform' + ) + check_type( + spatial_scale, 'spatial_scale', float, 'roi_perspective_transform' + ) helper = LayerHelper('roi_perspective_transform', **locals()) dtype = helper.input_dtype() @@ -2624,43 +2786,44 @@ def roi_perspective_transform(input, transform_matrix = helper.create_variable_for_type_inference(dtype) out2in_idx = helper.create_variable_for_type_inference(dtype="int32") out2in_w = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="roi_perspective_transform", - inputs={ - "X": input, - "ROIs": rois - }, - outputs={ - "Out": out, - "Out2InIdx": out2in_idx, - "Out2InWeights": out2in_w, - "Mask": mask, - "TransformMatrix": transform_matrix - }, - attrs={ - "transformed_height": transformed_height, - "transformed_width": transformed_width, - "spatial_scale": spatial_scale - }) + helper.append_op( + type="roi_perspective_transform", + inputs={"X": input, "ROIs": rois}, + outputs={ + "Out": out, + "Out2InIdx": out2in_idx, + "Out2InWeights": out2in_w, + "Mask": mask, + "TransformMatrix": transform_matrix, + }, + attrs={ + "transformed_height": transformed_height, + "transformed_width": transformed_width, + "spatial_scale": spatial_scale, + }, + ) return out, mask, transform_matrix -def generate_proposal_labels(rpn_rois, - gt_classes, - is_crowd, - gt_boxes, - im_info, - batch_size_per_im=256, - fg_fraction=0.25, - fg_thresh=0.25, - bg_thresh_hi=0.5, - bg_thresh_lo=0.0, - bbox_reg_weights=[0.1, 0.1, 0.2, 0.2], - class_nums=None, - use_random=True, - is_cls_agnostic=False, - is_cascade_rcnn=False, - max_overlap=None, - return_max_overlap=False): +def generate_proposal_labels( + rpn_rois, + gt_classes, + is_crowd, + gt_boxes, + im_info, + batch_size_per_im=256, + fg_fraction=0.25, + fg_thresh=0.25, + bg_thresh_hi=0.5, + bg_thresh_lo=0.0, + bbox_reg_weights=[0.1, 0.1, 0.2, 0.2], + class_nums=None, + use_random=True, + is_cls_agnostic=False, + is_cascade_rcnn=False, + max_overlap=None, + return_max_overlap=False, +): """ **Generate Proposal Labels of Faster-RCNN** @@ -2730,26 +2893,36 @@ def generate_proposal_labels(rpn_rois, helper = LayerHelper('generate_proposal_labels', **locals()) - check_variable_and_dtype(rpn_rois, 'rpn_rois', ['float32', 'float64'], - 'generate_proposal_labels') - check_variable_and_dtype(gt_classes, 'gt_classes', ['int32'], - 'generate_proposal_labels') - check_variable_and_dtype(is_crowd, 'is_crowd', ['int32'], - 'generate_proposal_labels') + check_variable_and_dtype( + rpn_rois, 'rpn_rois', ['float32', 'float64'], 'generate_proposal_labels' + ) + check_variable_and_dtype( + gt_classes, 'gt_classes', ['int32'], 'generate_proposal_labels' + ) + check_variable_and_dtype( + is_crowd, 'is_crowd', ['int32'], 'generate_proposal_labels' + ) if is_cascade_rcnn: - assert max_overlap is not None, "Input max_overlap of generate_proposal_labels should not be None if is_cascade_rcnn is True" + assert ( + max_overlap is not None + ), "Input max_overlap of generate_proposal_labels should not be None if is_cascade_rcnn is True" rois = helper.create_variable_for_type_inference(dtype=rpn_rois.dtype) labels_int32 = helper.create_variable_for_type_inference( - dtype=gt_classes.dtype) + dtype=gt_classes.dtype + ) bbox_targets = helper.create_variable_for_type_inference( - dtype=rpn_rois.dtype) + dtype=rpn_rois.dtype + ) bbox_inside_weights = helper.create_variable_for_type_inference( - dtype=rpn_rois.dtype) + dtype=rpn_rois.dtype + ) bbox_outside_weights = helper.create_variable_for_type_inference( - dtype=rpn_rois.dtype) + dtype=rpn_rois.dtype + ) max_overlap_with_gt = helper.create_variable_for_type_inference( - dtype=rpn_rois.dtype) + dtype=rpn_rois.dtype + ) inputs = { 'RpnRois': rpn_rois, @@ -2760,28 +2933,30 @@ def generate_proposal_labels(rpn_rois, } if max_overlap is not None: inputs['MaxOverlap'] = max_overlap - helper.append_op(type="generate_proposal_labels", - inputs=inputs, - outputs={ - 'Rois': rois, - 'LabelsInt32': labels_int32, - 'BboxTargets': bbox_targets, - 'BboxInsideWeights': bbox_inside_weights, - 'BboxOutsideWeights': bbox_outside_weights, - 'MaxOverlapWithGT': max_overlap_with_gt - }, - attrs={ - 'batch_size_per_im': batch_size_per_im, - 'fg_fraction': fg_fraction, - 'fg_thresh': fg_thresh, - 'bg_thresh_hi': bg_thresh_hi, - 'bg_thresh_lo': bg_thresh_lo, - 'bbox_reg_weights': bbox_reg_weights, - 'class_nums': class_nums, - 'use_random': use_random, - 'is_cls_agnostic': is_cls_agnostic, - 'is_cascade_rcnn': is_cascade_rcnn - }) + helper.append_op( + type="generate_proposal_labels", + inputs=inputs, + outputs={ + 'Rois': rois, + 'LabelsInt32': labels_int32, + 'BboxTargets': bbox_targets, + 'BboxInsideWeights': bbox_inside_weights, + 'BboxOutsideWeights': bbox_outside_weights, + 'MaxOverlapWithGT': max_overlap_with_gt, + }, + attrs={ + 'batch_size_per_im': batch_size_per_im, + 'fg_fraction': fg_fraction, + 'fg_thresh': fg_thresh, + 'bg_thresh_hi': bg_thresh_hi, + 'bg_thresh_lo': bg_thresh_lo, + 'bbox_reg_weights': bbox_reg_weights, + 'class_nums': class_nums, + 'use_random': use_random, + 'is_cls_agnostic': is_cls_agnostic, + 'is_cascade_rcnn': is_cascade_rcnn, + }, + ) rois.stop_gradient = True labels_int32.stop_gradient = True @@ -2791,12 +2966,33 @@ def generate_proposal_labels(rpn_rois, max_overlap_with_gt.stop_gradient = True if return_max_overlap: - return rois, labels_int32, bbox_targets, bbox_inside_weights, bbox_outside_weights, max_overlap_with_gt - return rois, labels_int32, bbox_targets, bbox_inside_weights, bbox_outside_weights + return ( + rois, + labels_int32, + bbox_targets, + bbox_inside_weights, + bbox_outside_weights, + max_overlap_with_gt, + ) + return ( + rois, + labels_int32, + bbox_targets, + bbox_inside_weights, + bbox_outside_weights, + ) -def generate_mask_labels(im_info, gt_classes, is_crowd, gt_segms, rois, - labels_int32, num_classes, resolution): +def generate_mask_labels( + im_info, + gt_classes, + is_crowd, + gt_segms, + rois, + labels_int32, + num_classes, + resolution, +): r""" **Generate Mask Labels for Mask-RCNN** @@ -2912,28 +3108,29 @@ def generate_mask_labels(im_info, gt_classes, is_crowd, gt_segms, rois, mask_rois = helper.create_variable_for_type_inference(dtype=rois.dtype) roi_has_mask_int32 = helper.create_variable_for_type_inference( - dtype=gt_classes.dtype) + dtype=gt_classes.dtype + ) mask_int32 = helper.create_variable_for_type_inference( - dtype=gt_classes.dtype) - - helper.append_op(type="generate_mask_labels", - inputs={ - 'ImInfo': im_info, - 'GtClasses': gt_classes, - 'IsCrowd': is_crowd, - 'GtSegms': gt_segms, - 'Rois': rois, - 'LabelsInt32': labels_int32 - }, - outputs={ - 'MaskRois': mask_rois, - 'RoiHasMaskInt32': roi_has_mask_int32, - 'MaskInt32': mask_int32 - }, - attrs={ - 'num_classes': num_classes, - 'resolution': resolution - }) + dtype=gt_classes.dtype + ) + + helper.append_op( + type="generate_mask_labels", + inputs={ + 'ImInfo': im_info, + 'GtClasses': gt_classes, + 'IsCrowd': is_crowd, + 'GtSegms': gt_segms, + 'Rois': rois, + 'LabelsInt32': labels_int32, + }, + outputs={ + 'MaskRois': mask_rois, + 'RoiHasMaskInt32': roi_has_mask_int32, + 'MaskInt32': mask_int32, + }, + attrs={'num_classes': num_classes, 'resolution': resolution}, + ) mask_rois.stop_gradient = True roi_has_mask_int32.stop_gradient = True @@ -2942,18 +3139,20 @@ def generate_mask_labels(im_info, gt_classes, is_crowd, gt_segms, rois, return mask_rois, roi_has_mask_int32, mask_int32 -def generate_proposals(scores, - bbox_deltas, - im_info, - anchors, - variances, - pre_nms_top_n=6000, - post_nms_top_n=1000, - nms_thresh=0.5, - min_size=0.1, - eta=1.0, - return_rois_num=False, - name=None): +def generate_proposals( + scores, + bbox_deltas, + im_info, + anchors, + variances, + pre_nms_top_n=6000, + post_nms_top_n=1000, + nms_thresh=0.5, + min_size=0.1, + eta=1.0, + return_rois_num=False, + name=None, +): """ **Generate proposal Faster-RCNN** @@ -3031,18 +3230,20 @@ def generate_proposals(scores, im_info, anchors, variances) """ - return paddle.vision.ops.generate_proposals(scores=scores, - bbox_deltas=bbox_deltas, - img_size=im_info[:2], - anchors=anchors, - variances=variances, - pre_nms_top_n=pre_nms_top_n, - post_nms_top_n=post_nms_top_n, - nms_thresh=nms_thresh, - min_size=min_size, - eta=eta, - return_rois_num=return_rois_num, - name=name) + return paddle.vision.ops.generate_proposals( + scores=scores, + bbox_deltas=bbox_deltas, + img_size=im_info[:2], + anchors=anchors, + variances=variances, + pre_nms_top_n=pre_nms_top_n, + post_nms_top_n=post_nms_top_n, + nms_thresh=nms_thresh, + min_size=min_size, + eta=eta, + return_rois_num=return_rois_num, + name=name, + ) def box_clip(input, im_info, name=None): @@ -3097,8 +3298,9 @@ def box_clip(input, im_info, name=None): """ check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'box_clip') - check_variable_and_dtype(im_info, 'im_info', ['float32', 'float64'], - 'box_clip') + check_variable_and_dtype( + im_info, 'im_info', ['float32', 'float64'], 'box_clip' + ) helper = LayerHelper("box_clip", **locals()) output = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -3108,15 +3310,17 @@ def box_clip(input, im_info, name=None): return output -def retinanet_detection_output(bboxes, - scores, - anchors, - im_info, - score_threshold=0.05, - nms_top_k=1000, - keep_top_k=100, - nms_threshold=0.3, - nms_eta=1.0): +def retinanet_detection_output( + bboxes, + scores, + anchors, + im_info, + score_threshold=0.05, + nms_top_k=1000, + keep_top_k=100, + nms_threshold=0.3, + nms_eta=1.0, +): """ **Detection Output Layer for the detector RetinaNet.** @@ -3225,54 +3429,69 @@ def retinanet_detection_output(bboxes, check_type(bboxes, 'bboxes', (list), 'retinanet_detection_output') for i, bbox in enumerate(bboxes): - check_variable_and_dtype(bbox, 'bbox{}'.format(i), - ['float32', 'float64'], - 'retinanet_detection_output') + check_variable_and_dtype( + bbox, + 'bbox{}'.format(i), + ['float32', 'float64'], + 'retinanet_detection_output', + ) check_type(scores, 'scores', (list), 'retinanet_detection_output') for i, score in enumerate(scores): - check_variable_and_dtype(score, 'score{}'.format(i), - ['float32', 'float64'], - 'retinanet_detection_output') + check_variable_and_dtype( + score, + 'score{}'.format(i), + ['float32', 'float64'], + 'retinanet_detection_output', + ) check_type(anchors, 'anchors', (list), 'retinanet_detection_output') for i, anchor in enumerate(anchors): - check_variable_and_dtype(anchor, 'anchor{}'.format(i), - ['float32', 'float64'], - 'retinanet_detection_output') - check_variable_and_dtype(im_info, 'im_info', ['float32', 'float64'], - 'retinanet_detection_output') + check_variable_and_dtype( + anchor, + 'anchor{}'.format(i), + ['float32', 'float64'], + 'retinanet_detection_output', + ) + check_variable_and_dtype( + im_info, 'im_info', ['float32', 'float64'], 'retinanet_detection_output' + ) helper = LayerHelper('retinanet_detection_output', **locals()) output = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('scores')) - helper.append_op(type="retinanet_detection_output", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores, - 'Anchors': anchors, - 'ImInfo': im_info - }, - attrs={ - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': 1., - }, - outputs={'Out': output}) + dtype=helper.input_dtype('scores') + ) + helper.append_op( + type="retinanet_detection_output", + inputs={ + 'BBoxes': bboxes, + 'Scores': scores, + 'Anchors': anchors, + 'ImInfo': im_info, + }, + attrs={ + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': 1.0, + }, + outputs={'Out': output}, + ) output.stop_gradient = True return output -def multiclass_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=0, - name=None): +def multiclass_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=0, + name=None, +): """ **Multiclass NMS** @@ -3384,10 +3603,12 @@ def multiclass_nms(bboxes, keep_top_k=200, normalized=False) """ - check_variable_and_dtype(bboxes, 'BBoxes', ['float32', 'float64'], - 'multiclass_nms') - check_variable_and_dtype(scores, 'Scores', ['float32', 'float64'], - 'multiclass_nms') + check_variable_and_dtype( + bboxes, 'BBoxes', ['float32', 'float64'], 'multiclass_nms' + ) + check_variable_and_dtype( + scores, 'Scores', ['float32', 'float64'], 'multiclass_nms' + ) check_type(score_threshold, 'score_threshold', float, 'multicalss_nms') check_type(nms_top_k, 'nums_top_k', int, 'multiclass_nms') check_type(keep_top_k, 'keep_top_k', int, 'mutliclass_nms') @@ -3398,36 +3619,37 @@ def multiclass_nms(bboxes, helper = LayerHelper('multiclass_nms', **locals()) output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) - helper.append_op(type="multiclass_nms", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'nms_eta': nms_eta, - 'keep_top_k': keep_top_k, - 'normalized': normalized - }, - outputs={'Out': output}) + helper.append_op( + type="multiclass_nms", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'nms_eta': nms_eta, + 'keep_top_k': keep_top_k, + 'normalized': normalized, + }, + outputs={'Out': output}, + ) output.stop_gradient = True return output -def locality_aware_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=-1, - name=None): +def locality_aware_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=-1, + name=None, +): """ **Local Aware NMS** @@ -3506,10 +3728,12 @@ def locality_aware_nms(bboxes, keep_top_k=200, normalized=False) """ - check_variable_and_dtype(bboxes, 'bboxes', ['float32', 'float64'], - 'locality_aware_nms') - check_variable_and_dtype(scores, 'scores', ['float32', 'float64'], - 'locality_aware_nms') + check_variable_and_dtype( + bboxes, 'bboxes', ['float32', 'float64'], 'locality_aware_nms' + ) + check_variable_and_dtype( + scores, 'scores', ['float32', 'float64'], 'locality_aware_nms' + ) check_type(background_label, 'background_label', int, 'locality_aware_nms') check_type(score_threshold, 'score_threshold', float, 'locality_aware_nms') check_type(nms_top_k, 'nms_top_k', int, 'locality_aware_nms') @@ -3520,47 +3744,49 @@ def locality_aware_nms(bboxes, shape = scores.shape assert len(shape) == 3, "dim size of scores must be 3" - assert shape[ - 1] == 1, "locality_aware_nms only support one class, Tensor score shape must be [N, 1, M]" + assert ( + shape[1] == 1 + ), "locality_aware_nms only support one class, Tensor score shape must be [N, 1, M]" helper = LayerHelper('locality_aware_nms', **locals()) output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) out = {'Out': output} - helper.append_op(type="locality_aware_nms", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'nms_eta': nms_eta, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs={'Out': output}) + helper.append_op( + type="locality_aware_nms", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'nms_eta': nms_eta, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs={'Out': output}, + ) output.stop_gradient = True return output -def matrix_nms(bboxes, - scores, - score_threshold, - post_threshold, - nms_top_k, - keep_top_k, - use_gaussian=False, - gaussian_sigma=2., - background_label=0, - normalized=True, - return_index=False, - name=None): +def matrix_nms( + bboxes, + scores, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + use_gaussian=False, + gaussian_sigma=2.0, + background_label=0, + normalized=True, + return_index=False, + name=None, +): """ **Matrix NMS** @@ -3637,8 +3863,16 @@ def matrix_nms(bboxes, normalized=False) """ if in_dygraph_mode(): - attrs = (score_threshold, nms_top_k, keep_top_k, post_threshold, - use_gaussian, gaussian_sigma, background_label, normalized) + attrs = ( + score_threshold, + nms_top_k, + keep_top_k, + post_threshold, + use_gaussian, + gaussian_sigma, + background_label, + normalized, + ) out, index = _C_ops.matrix_nms(bboxes, scores, *attrs) if return_index: @@ -3646,10 +3880,12 @@ def matrix_nms(bboxes, else: return out - check_variable_and_dtype(bboxes, 'BBoxes', ['float32', 'float64'], - 'matrix_nms') - check_variable_and_dtype(scores, 'Scores', ['float32', 'float64'], - 'matrix_nms') + check_variable_and_dtype( + bboxes, 'BBoxes', ['float32', 'float64'], 'matrix_nms' + ) + check_variable_and_dtype( + scores, 'Scores', ['float32', 'float64'], 'matrix_nms' + ) check_type(score_threshold, 'score_threshold', float, 'matrix_nms') check_type(post_threshold, 'post_threshold', float, 'matrix_nms') check_type(nms_top_k, 'nums_top_k', int, 'matrix_nms') @@ -3662,25 +3898,21 @@ def matrix_nms(bboxes, helper = LayerHelper('matrix_nms', **locals()) output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) index = helper.create_variable_for_type_inference(dtype='int') - helper.append_op(type="matrix_nms", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'score_threshold': score_threshold, - 'post_threshold': post_threshold, - 'nms_top_k': nms_top_k, - 'keep_top_k': keep_top_k, - 'use_gaussian': use_gaussian, - 'gaussian_sigma': gaussian_sigma, - 'background_label': background_label, - 'normalized': normalized - }, - outputs={ - 'Out': output, - 'Index': index - }) + helper.append_op( + type="matrix_nms", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'score_threshold': score_threshold, + 'post_threshold': post_threshold, + 'nms_top_k': nms_top_k, + 'keep_top_k': keep_top_k, + 'use_gaussian': use_gaussian, + 'gaussian_sigma': gaussian_sigma, + 'background_label': background_label, + 'normalized': normalized, + }, + outputs={'Out': output, 'Index': index}, + ) output.stop_gradient = True if return_index: @@ -3689,13 +3921,15 @@ def matrix_nms(bboxes, return output -def distribute_fpn_proposals(fpn_rois, - min_level, - max_level, - refer_level, - refer_scale, - rois_num=None, - name=None): +def distribute_fpn_proposals( + fpn_rois, + min_level, + max_level, + refer_level, + refer_scale, + rois_num=None, + name=None, +): r""" **This op only takes LoDTensor as input.** In Feature Pyramid Networks @@ -3763,22 +3997,21 @@ def distribute_fpn_proposals(fpn_rois, refer_level=4, refer_scale=224) """ - return paddle.vision.ops.distribute_fpn_proposals(fpn_rois=fpn_rois, - min_level=min_level, - max_level=max_level, - refer_level=refer_level, - refer_scale=refer_scale, - rois_num=rois_num, - name=name) + return paddle.vision.ops.distribute_fpn_proposals( + fpn_rois=fpn_rois, + min_level=min_level, + max_level=max_level, + refer_level=refer_level, + refer_scale=refer_scale, + rois_num=rois_num, + name=name, + ) @templatedoc() -def box_decoder_and_assign(prior_box, - prior_box_var, - target_box, - box_score, - box_clip, - name=None): +def box_decoder_and_assign( + prior_box, prior_box_var, target_box, box_score, box_clip, name=None +): """ ${comment} @@ -3818,41 +4051,53 @@ def box_decoder_and_assign(prior_box, pb, pbv, loc, scores, 4.135) """ - check_variable_and_dtype(prior_box, 'prior_box', ['float32', 'float64'], - 'box_decoder_and_assign') - check_variable_and_dtype(target_box, 'target_box', ['float32', 'float64'], - 'box_decoder_and_assign') - check_variable_and_dtype(box_score, 'box_score', ['float32', 'float64'], - 'box_decoder_and_assign') + check_variable_and_dtype( + prior_box, 'prior_box', ['float32', 'float64'], 'box_decoder_and_assign' + ) + check_variable_and_dtype( + target_box, + 'target_box', + ['float32', 'float64'], + 'box_decoder_and_assign', + ) + check_variable_and_dtype( + box_score, 'box_score', ['float32', 'float64'], 'box_decoder_and_assign' + ) helper = LayerHelper("box_decoder_and_assign", **locals()) decoded_box = helper.create_variable_for_type_inference( - dtype=prior_box.dtype) + dtype=prior_box.dtype + ) output_assign_box = helper.create_variable_for_type_inference( - dtype=prior_box.dtype) - - helper.append_op(type="box_decoder_and_assign", - inputs={ - "PriorBox": prior_box, - "PriorBoxVar": prior_box_var, - "TargetBox": target_box, - "BoxScore": box_score - }, - attrs={"box_clip": box_clip}, - outputs={ - "DecodeBox": decoded_box, - "OutputAssignBox": output_assign_box - }) + dtype=prior_box.dtype + ) + + helper.append_op( + type="box_decoder_and_assign", + inputs={ + "PriorBox": prior_box, + "PriorBoxVar": prior_box_var, + "TargetBox": target_box, + "BoxScore": box_score, + }, + attrs={"box_clip": box_clip}, + outputs={ + "DecodeBox": decoded_box, + "OutputAssignBox": output_assign_box, + }, + ) return decoded_box, output_assign_box -def collect_fpn_proposals(multi_rois, - multi_scores, - min_level, - max_level, - post_nms_top_n, - rois_num_per_level=None, - name=None): +def collect_fpn_proposals( + multi_rois, + multi_scores, + min_level, + max_level, + post_nms_top_n, + rois_num_per_level=None, + name=None, +): """ **This OP only supports LoDTensor as input**. Concat multi-level RoIs @@ -3922,17 +4167,21 @@ def collect_fpn_proposals(multi_rois, input_scores = multi_scores[:num_lvl] if _non_static_mode(): - assert rois_num_per_level is not None, "rois_num_per_level should not be None in dygraph mode." + assert ( + rois_num_per_level is not None + ), "rois_num_per_level should not be None in dygraph mode." attrs = ('post_nms_topN', post_nms_top_n) output_rois, rois_num = _legacy_C_ops.collect_fpn_proposals( - input_rois, input_scores, rois_num_per_level, *attrs) + input_rois, input_scores, rois_num_per_level, *attrs + ) check_type(multi_rois, 'multi_rois', list, 'collect_fpn_proposals') check_type(multi_scores, 'multi_scores', list, 'collect_fpn_proposals') helper = LayerHelper('collect_fpn_proposals', **locals()) dtype = helper.input_dtype('multi_rois') - check_dtype(dtype, 'multi_rois', ['float32', 'float64'], - 'collect_fpn_proposals') + check_dtype( + dtype, 'multi_rois', ['float32', 'float64'], 'collect_fpn_proposals' + ) output_rois = helper.create_variable_for_type_inference(dtype) output_rois.stop_gradient = True @@ -3946,10 +4195,12 @@ def collect_fpn_proposals(multi_rois, rois_num = helper.create_variable_for_type_inference(dtype='int32') rois_num.stop_gradient = True outputs['RoisNum'] = rois_num - helper.append_op(type='collect_fpn_proposals', - inputs=inputs, - outputs=outputs, - attrs={'post_nms_topN': post_nms_top_n}) + helper.append_op( + type='collect_fpn_proposals', + inputs=inputs, + outputs=outputs, + attrs={'post_nms_topN': post_nms_top_n}, + ) if rois_num_per_level is not None: return output_rois, rois_num return output_rois diff --git a/python/paddle/fluid/layers/device.py b/python/paddle/fluid/layers/device.py index a5c2e8c3e20..ac352ef52b1 100644 --- a/python/paddle/fluid/layers/device.py +++ b/python/paddle/fluid/layers/device.py @@ -28,15 +28,16 @@ __all__ = [] def get_places(device_count=None, device_type=None): helper = LayerHelper('get_places', **locals()) out_places = helper.create_variable( - name=unique_name.generate_with_ignorable_key(helper.name + ".out")) + name=unique_name.generate_with_ignorable_key(helper.name + ".out") + ) attrs = dict() if device_count is not None: attrs['device_count'] = int(device_count) if device_type is not None: attrs['device_type'] = str(device_type) - helper.append_op(type='get_places', - outputs={"Out": [out_places]}, - attrs=attrs) + helper.append_op( + type='get_places', outputs={"Out": [out_places]}, attrs=attrs + ) return out_places diff --git a/python/paddle/fluid/layers/distributions.py b/python/paddle/fluid/layers/distributions.py index e7763463232..f955b9299c7 100644 --- a/python/paddle/fluid/layers/distributions.py +++ b/python/paddle/fluid/layers/distributions.py @@ -20,7 +20,12 @@ import math import numpy as np import warnings -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) __all__ = ['Uniform', 'Normal', 'Categorical', 'MultivariateNormalDiag'] @@ -64,7 +69,8 @@ class Distribution(object): if is_variable and is_number: raise ValueError( - 'if one argument is Variable, all arguments should be Variable') + 'if one argument is Variable, all arguments should be Variable' + ) return is_variable @@ -79,7 +85,7 @@ class Distribution(object): """ numpy_args = [] variable_args = [] - tmp = 0. + tmp = 0.0 for arg in args: valid_arg = False @@ -87,7 +93,9 @@ class Distribution(object): if isinstance(arg, cls): valid_arg = True break - assert valid_arg, "type of input args must be float, list, numpy.ndarray or Variable." + assert ( + valid_arg + ), "type of input args must be float, list, numpy.ndarray or Variable." if isinstance(arg, float): arg = np.zeros(1) + arg arg_np = np.array(arg) @@ -175,10 +183,12 @@ class Uniform(Distribution): """ def __init__(self, low, high): - check_type(low, 'low', (float, np.ndarray, tensor.Variable, list), - 'Uniform') - check_type(high, 'high', (float, np.ndarray, tensor.Variable, list), - 'Uniform') + check_type( + low, 'low', (float, np.ndarray, tensor.Variable, list), 'Uniform' + ) + check_type( + high, 'high', (float, np.ndarray, tensor.Variable, list), 'Uniform' + ) self.all_arg_is_float = False self.batch_size_unknown = False @@ -209,17 +219,26 @@ class Uniform(Distribution): if self.batch_size_unknown: output_shape = shape + batch_shape zero_tmp = tensor.fill_constant_batch_size_like( - self.low + self.high, batch_shape + shape, self.low.dtype, 0.) + self.low + self.high, batch_shape + shape, self.low.dtype, 0.0 + ) uniform_random_tmp = nn.uniform_random_batch_size_like( - zero_tmp, zero_tmp.shape, min=0., max=1., seed=seed) - output = uniform_random_tmp * (zero_tmp + self.high - - self.low) + self.low + zero_tmp, zero_tmp.shape, min=0.0, max=1.0, seed=seed + ) + output = ( + uniform_random_tmp * (zero_tmp + self.high - self.low) + + self.low + ) return nn.reshape(output, output_shape) else: output_shape = shape + batch_shape - output = nn.uniform_random(output_shape, seed=seed) * ( - tensor.zeros(output_shape, dtype=self.low.dtype) + - (self.high - self.low)) + self.low + output = ( + nn.uniform_random(output_shape, seed=seed) + * ( + tensor.zeros(output_shape, dtype=self.low.dtype) + + (self.high - self.low) + ) + + self.low + ) if self.all_arg_is_float: return nn.reshape(output, shape) else: @@ -235,8 +254,9 @@ class Uniform(Distribution): Variable: log probability.The data type is same with value. """ - check_variable_and_dtype(value, 'value', ['float32', 'float64'], - 'log_prob') + check_variable_and_dtype( + value, 'value', ['float32', 'float64'], 'log_prob' + ) lb_bool = control_flow.less_than(self.low, value) ub_bool = control_flow.less_than(value, self.high) @@ -317,10 +337,12 @@ class Normal(Distribution): """ def __init__(self, loc, scale): - check_type(loc, 'loc', (float, np.ndarray, tensor.Variable, list), - 'Normal') - check_type(scale, 'scale', (float, np.ndarray, tensor.Variable, list), - 'Normal') + check_type( + loc, 'loc', (float, np.ndarray, tensor.Variable, list), 'Normal' + ) + check_type( + scale, 'scale', (float, np.ndarray, tensor.Variable, list), 'Normal' + ) self.batch_size_unknown = False self.all_arg_is_float = False @@ -353,18 +375,24 @@ class Normal(Distribution): if self.batch_size_unknown: output_shape = shape + batch_shape zero_tmp = tensor.fill_constant_batch_size_like( - self.loc + self.scale, batch_shape + shape, self.loc.dtype, 0.) + self.loc + self.scale, batch_shape + shape, self.loc.dtype, 0.0 + ) zero_tmp_shape = nn.shape(zero_tmp) - normal_random_tmp = nn.gaussian_random(zero_tmp_shape, - mean=0., - std=1., - seed=seed) + normal_random_tmp = nn.gaussian_random( + zero_tmp_shape, mean=0.0, std=1.0, seed=seed + ) output = normal_random_tmp * (zero_tmp + self.scale) + self.loc return nn.reshape(output, output_shape) else: output_shape = shape + batch_shape - output = nn.gaussian_random(output_shape, mean=0., std=1., seed=seed) * \ - (tensor.zeros(output_shape, dtype=self.loc.dtype) + self.scale) + self.loc + output = ( + nn.gaussian_random(output_shape, mean=0.0, std=1.0, seed=seed) + * ( + tensor.zeros(output_shape, dtype=self.loc.dtype) + + self.scale + ) + + self.loc + ) if self.all_arg_is_float: return nn.reshape(output, shape) else: @@ -378,11 +406,12 @@ class Normal(Distribution): """ batch_shape = list((self.loc + self.scale).shape) - zero_tmp = tensor.fill_constant_batch_size_like(self.loc + self.scale, - batch_shape, - self.loc.dtype, 0.) - return 0.5 + 0.5 * math.log(2 * math.pi) + nn.log( - (self.scale + zero_tmp)) + zero_tmp = tensor.fill_constant_batch_size_like( + self.loc + self.scale, batch_shape, self.loc.dtype, 0.0 + ) + return ( + 0.5 + 0.5 * math.log(2 * math.pi) + nn.log((self.scale + zero_tmp)) + ) def log_prob(self, value): """Log probability density/mass function. @@ -394,14 +423,17 @@ class Normal(Distribution): Variable: log probability.The data type is same with value. """ - check_variable_and_dtype(value, 'value', ['float32', 'float64'], - 'log_prob') + check_variable_and_dtype( + value, 'value', ['float32', 'float64'], 'log_prob' + ) var = self.scale * self.scale log_scale = nn.log(self.scale) - return -1. * ((value - self.loc) * - (value - self.loc)) / (2. * var) - log_scale - math.log( - math.sqrt(2. * math.pi)) + return ( + -1.0 * ((value - self.loc) * (value - self.loc)) / (2.0 * var) + - log_scale + - math.log(math.sqrt(2.0 * math.pi)) + ) def kl_divergence(self, other): """The KL-divergence between two normal distributions. @@ -417,10 +449,10 @@ class Normal(Distribution): check_type(other, 'other', Normal, 'kl_divergence') var_ratio = self.scale / other.scale - var_ratio = (var_ratio * var_ratio) + var_ratio = var_ratio * var_ratio t1 = (self.loc - other.loc) / other.scale - t1 = (t1 * t1) - return 0.5 * (var_ratio + t1 - 1. - nn.log(var_ratio)) + t1 = t1 * t1 + return 0.5 * (var_ratio + t1 - 1.0 - nn.log(var_ratio)) class Categorical(Distribution): @@ -477,8 +509,9 @@ class Categorical(Distribution): Args: logits(list|numpy.ndarray|Variable): The logits input of categorical distribution. The data type is float32. """ - check_type(logits, 'logits', (np.ndarray, tensor.Variable, list), - 'Categorical') + check_type( + logits, 'logits', (np.ndarray, tensor.Variable, list), 'Categorical' + ) if self._validate_args(logits): self.logits = logits @@ -499,7 +532,8 @@ class Categorical(Distribution): logits = self.logits - nn.reduce_max(self.logits, dim=-1, keep_dim=True) other_logits = other.logits - nn.reduce_max( - other.logits, dim=-1, keep_dim=True) + other.logits, dim=-1, keep_dim=True + ) e_logits = ops.exp(logits) other_e_logits = ops.exp(other_logits) z = nn.reduce_sum(e_logits, dim=-1, keep_dim=True) @@ -508,7 +542,8 @@ class Categorical(Distribution): kl = nn.reduce_sum( prob * (logits - nn.log(z) - other_logits + nn.log(other_z)), dim=-1, - keep_dim=True) + keep_dim=True, + ) return kl @@ -524,7 +559,8 @@ class Categorical(Distribution): z = nn.reduce_sum(e_logits, dim=-1, keep_dim=True) prob = e_logits / z entropy = -1.0 * nn.reduce_sum( - prob * (logits - nn.log(z)), dim=-1, keep_dim=True) + prob * (logits - nn.log(z)), dim=-1, keep_dim=True + ) return entropy @@ -598,10 +634,18 @@ class MultivariateNormalDiag(Distribution): """ def __init__(self, loc, scale): - check_type(loc, 'loc', (np.ndarray, tensor.Variable, list), - 'MultivariateNormalDiag') - check_type(scale, 'scale', (np.ndarray, tensor.Variable, list), - 'MultivariateNormalDiag') + check_type( + loc, + 'loc', + (np.ndarray, tensor.Variable, list), + 'MultivariateNormalDiag', + ) + check_type( + scale, + 'scale', + (np.ndarray, tensor.Variable, list), + 'MultivariateNormalDiag', + ) if self._validate_args(loc, scale): self.loc = loc @@ -614,7 +658,8 @@ class MultivariateNormalDiag(Distribution): batch_shape = list(value.shape) one_all = tensor.ones(shape=batch_shape, dtype=self.loc.dtype) one_diag = tensor.diag( - tensor.ones(shape=[batch_shape[0]], dtype=self.loc.dtype)) + tensor.ones(shape=[batch_shape[0]], dtype=self.loc.dtype) + ) det_diag = nn.reduce_prod(value + one_all - one_diag) return det_diag @@ -624,7 +669,8 @@ class MultivariateNormalDiag(Distribution): batch_shape = list(value.shape) one_all = tensor.ones(shape=batch_shape, dtype=self.loc.dtype) one_diag = tensor.diag( - tensor.ones(shape=[batch_shape[0]], dtype=self.loc.dtype)) + tensor.ones(shape=[batch_shape[0]], dtype=self.loc.dtype) + ) inv_diag = nn.elementwise_pow(value, (one_all - 2 * one_diag)) return inv_diag @@ -636,8 +682,10 @@ class MultivariateNormalDiag(Distribution): Variable: Shannon entropy of Multivariate Normal distribution. The data type is float32. """ - entropy = 0.5 * (self.scale.shape[0] * (1.0 + math.log(2 * math.pi)) + - nn.log(self._det(self.scale))) + entropy = 0.5 * ( + self.scale.shape[0] * (1.0 + math.log(2 * math.pi)) + + nn.log(self._det(self.scale)) + ) return entropy @@ -654,8 +702,9 @@ class MultivariateNormalDiag(Distribution): check_type(other, 'other', MultivariateNormalDiag, 'kl_divergence') tr_cov_matmul = nn.reduce_sum(self._inv(other.scale) * self.scale) - loc_matmul_cov = nn.matmul((other.loc - self.loc), - self._inv(other.scale)) + loc_matmul_cov = nn.matmul( + (other.loc - self.loc), self._inv(other.scale) + ) tri_matmul = nn.matmul(loc_matmul_cov, (other.loc - self.loc)) k = list(self.scale.shape)[0] ln_cov = nn.log(self._det(other.scale)) - nn.log(self._det(self.scale)) diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 5c62dc6d61d..86654bbf669 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -23,30 +23,46 @@ from .control_flow import BlockGuard from .layer_function_generator import templatedoc from .. import core from ..executor import global_scope -from ..framework import convert_np_dtype_to_dtype_, default_main_program, \ - default_startup_program, program_guard, Program, Variable +from ..framework import ( + convert_np_dtype_to_dtype_, + default_main_program, + default_startup_program, + program_guard, + Program, + Variable, +) from ..layer_helper import LayerHelper from ..unique_name import generate as unique_name import logging from ..data_feeder import check_dtype, check_type from paddle.fluid.framework import static_only -from ..framework import _get_paddle_place, _current_expected_place, _set_expected_place +from ..framework import ( + _get_paddle_place, + _current_expected_place, + _set_expected_place, +) __all__ = [ - 'data', 'read_file', 'double_buffer', 'py_reader', - 'create_py_reader_by_data', 'load' + 'data', + 'read_file', + 'double_buffer', + 'py_reader', + 'create_py_reader_by_data', + 'load', ] @static_only -def data(name, - shape, - append_batch_size=True, - dtype='float32', - lod_level=0, - type=core.VarDesc.VarType.LOD_TENSOR, - stop_gradient=True): +def data( + name, + shape, + append_batch_size=True, + dtype='float32', + lod_level=0, + type=core.VarDesc.VarType.LOD_TENSOR, + stop_gradient=True, +): """ **Data Layer** @@ -124,13 +140,15 @@ def data(name, if append_batch_size: shape = [-1] + shape # append batch size as -1 - data_var = helper.create_global_variable(name=name, - shape=shape, - dtype=dtype, - type=type, - stop_gradient=stop_gradient, - lod_level=lod_level, - is_data=True) + data_var = helper.create_global_variable( + name=name, + shape=shape, + dtype=dtype, + type=type, + stop_gradient=stop_gradient, + lod_level=lod_level, + is_data=True, + ) return data_var @@ -245,13 +263,13 @@ class ListenAndServ(object): attrs={ 'endpoint': self.endpoint, 'Fanin': self.fan_in, - 'optimize_blocks': - [current_block - ], # did not support multiple optimize blocks in layers - 'distributed_mode': - DistributedMode.SYNC, # did not support async now in layers - 'grad_to_block_id': [""] - }) + 'optimize_blocks': [ + current_block + ], # did not support multiple optimize blocks in layers + 'distributed_mode': DistributedMode.SYNC, # did not support async now in layers + 'grad_to_block_id': [""], + }, + ) def Send(endpoints, send_vars, dummy_output=None, sync=True): @@ -266,14 +284,14 @@ def Send(endpoints, send_vars, dummy_output=None, sync=True): sync (bool): whether to wait the request finish """ - assert (type(send_vars) == list) + assert type(send_vars) == list if dummy_output is None: dummy_output = [] elif isinstance(dummy_output, Variable): dummy_output = [dummy_output] - assert (type(dummy_output) == list) + assert type(dummy_output) == list epmap = endpoints.split(",") endpoints = list(set(epmap)) @@ -281,22 +299,23 @@ def Send(endpoints, send_vars, dummy_output=None, sync=True): helper = LayerHelper("Send", **locals()) rpc_op_role_name = core.op_proto_and_checker_maker.kOpRoleAttrName() - helper.append_op(type="send", - inputs={"X": send_vars}, - outputs={"Out": dummy_output}, - attrs={ - "endpoints": - endpoints, - "epmap": - epmap, - rpc_op_role_name: - core.op_proto_and_checker_maker.OpRole.RPC - }) + helper.append_op( + type="send", + inputs={"X": send_vars}, + outputs={"Out": dummy_output}, + attrs={ + "endpoints": endpoints, + "epmap": epmap, + rpc_op_role_name: core.op_proto_and_checker_maker.OpRole.RPC, + }, + ) if sync: - helper.append_op(type="send_barrier", - inputs={"X": dummy_output}, - outputs={"Out": []}, - attrs={"endpoints": endpoints}) + helper.append_op( + type="send_barrier", + inputs={"X": dummy_output}, + outputs={"Out": []}, + attrs={"endpoints": endpoints}, + ) def Recv(endpoints, get_vars, dummy_input=None, sync=True): @@ -312,35 +331,35 @@ def Recv(endpoints, get_vars, dummy_input=None, sync=True): Returns: list: list of received variables """ - assert (type(get_vars) == list) + assert type(get_vars) == list if dummy_input is None: dummy_input = [] elif isinstance(dummy_input, Variable): dummy_input = [dummy_input] - assert (type(dummy_input) == list) + assert type(dummy_input) == list epmap = endpoints.split(",") endpoints = list(set(epmap)) helper = LayerHelper("Recv", **locals()) - helper.append_op(type="recv", - inputs={"X": dummy_input}, - outputs={"Out": get_vars}, - attrs={ - "endpoints": endpoints, - "epmap": epmap - }) + helper.append_op( + type="recv", + inputs={"X": dummy_input}, + outputs={"Out": get_vars}, + attrs={"endpoints": endpoints, "epmap": epmap}, + ) if sync: - helper.append_op(type="fetch_barrier", - outputs={"Out": get_vars}, - attrs={"endpoints": endpoints}) + helper.append_op( + type="fetch_barrier", + outputs={"Out": get_vars}, + attrs={"endpoints": endpoints}, + ) return get_vars def monkey_patch_reader_methods(reader): - def __get_reader__(): scope = global_scope() var = scope.find_var(reader.name) @@ -381,24 +400,30 @@ def _copy_reader_create_op_(block, op): for arg_name in arg_names: new_output_map[param_name].append(block.var(arg_name)) - new_op = block.append_op(type=op.type, - inputs=new_input_map, - outputs=new_output_map, - attrs=op.all_attrs()) + new_op = block.append_op( + type=op.type, + inputs=new_input_map, + outputs=new_output_map, + attrs=op.all_attrs(), + ) return new_op -def _py_reader(capacity, - shapes, - dtypes, - lod_levels=None, - name=None, - use_double_buffer=True, - feed_list=None): +def _py_reader( + capacity, + shapes, + dtypes, + lod_levels=None, + name=None, + use_double_buffer=True, + feed_list=None, +): if feed_list is not None: if not isinstance(feed_list, list): - raise TypeError("feed_list should be a list of Variable" - " instead of " + str(type(feed_list))) + raise TypeError( + "feed_list should be a list of Variable" + " instead of " + str(type(feed_list)) + ) lod_levels = [] dtypes = [] shape_concat = [] @@ -440,22 +465,25 @@ def _py_reader(capacity, startup_blk = default_startup_program().current_block() startup_var = startup_blk.create_var(name=reader_name) - startup_blk.append_op(type='create_py_reader', - inputs={'blocking_queue': [queue_name]}, - outputs={'Out': [startup_var]}, - attrs={ - 'shape_concat': shape_concat, - 'lod_levels': lod_levels, - 'dtypes': dtype_int, - 'need_check_feed': need_check_feed, - 'ranks': ranks - }) + startup_blk.append_op( + type='create_py_reader', + inputs={'blocking_queue': [queue_name]}, + outputs={'Out': [startup_var]}, + attrs={ + 'shape_concat': shape_concat, + 'lod_levels': lod_levels, + 'dtypes': dtype_int, + 'need_check_feed': need_check_feed, + 'ranks': ranks, + }, + ) startup_var.desc.set_dtypes(dtypes) startup_var.persistable = True - main_prog_var = _copy_reader_var_(default_main_program().current_block(), - startup_var) + main_prog_var = _copy_reader_var_( + default_main_program().current_block(), startup_var + ) reader = monkey_patch_reader_methods(main_prog_var) if use_double_buffer: @@ -473,7 +501,6 @@ def _py_reader(capacity, reader.exited = False def start_provide_thread(func): - def __provider_thread__(legacy_expected_place): try: # See _DataLoaderIterSingleProcess._thread_loop() for why set expected place here. @@ -501,8 +528,9 @@ def _py_reader(capacity, logging.warn('Your decorated reader has raised an exception!') six.reraise(*sys.exc_info()) - reader.thread = threading.Thread(target=__provider_thread__, - args=(_current_expected_place(), )) + reader.thread = threading.Thread( + target=__provider_thread__, args=(_current_expected_place(),) + ) reader.thread.daemon = True reader.thread.start() @@ -518,17 +546,22 @@ def _py_reader(capacity, for dtype, shape, lod_level in zip(dtypes, shapes, lod_levels): name = str(counter) actual_feed_list.append( - data(name=name, - dtype=dtype, - shape=shape, - lod_level=lod_level)) + data( + name=name, + dtype=dtype, + shape=shape, + lod_level=lod_level, + ) + ) counter += 1 data_names = [feed_data.name for feed_data in actual_feed_list] - feeder = DataFeeder(feed_list=actual_feed_list, - place=core.CPUPlace()) - paddle_reader = feeder.decorate_reader(paddle_reader, - multi_devices=False) + feeder = DataFeeder( + feed_list=actual_feed_list, place=core.CPUPlace() + ) + paddle_reader = feeder.decorate_reader( + paddle_reader, multi_devices=False + ) def __tensor_provider__(): for slots in paddle_reader(): @@ -557,14 +590,11 @@ def _py_reader(capacity, return reader -def py_reader(capacity, - shapes, - dtypes, - lod_levels=None, - name=None, - use_double_buffer=True): +def py_reader( + capacity, shapes, dtypes, lod_levels=None, name=None, use_double_buffer=True +): """ - :api_attr: Static Graph + :api_attr: Static Graph Create a Python reader for data feeding in Python @@ -718,21 +748,23 @@ def py_reader(capacity, """ logging.warn( 'paddle.fluid.layers.py_reader() may be deprecated in the near future. ' - 'Please use paddle.fluid.io.DataLoader.from_generator() instead.') - return _py_reader(capacity=capacity, - shapes=shapes, - dtypes=dtypes, - lod_levels=lod_levels, - name=name, - use_double_buffer=use_double_buffer) - - -def create_py_reader_by_data(capacity, - feed_list, - name=None, - use_double_buffer=True): + 'Please use paddle.fluid.io.DataLoader.from_generator() instead.' + ) + return _py_reader( + capacity=capacity, + shapes=shapes, + dtypes=dtypes, + lod_levels=lod_levels, + name=name, + use_double_buffer=use_double_buffer, + ) + + +def create_py_reader_by_data( + capacity, feed_list, name=None, use_double_buffer=True +): """ - :api_attr: Static Graph + :api_attr: Static Graph The OP creates a Python reader for data feeding in Python, it is similar to :ref:`api_fluid_layers_py_reader` except that it can read data from @@ -799,24 +831,29 @@ def create_py_reader_by_data(capacity, """ logging.warn( 'paddle.fluid.layers.create_py_reader_by_data() may be deprecated in the near future. ' - 'Please use paddle.fluid.io.DataLoader.from_generator() instead.') - return _py_reader(capacity=capacity, - shapes=None, - dtypes=None, - lod_levels=None, - name=name, - use_double_buffer=use_double_buffer, - feed_list=feed_list) + 'Please use paddle.fluid.io.DataLoader.from_generator() instead.' + ) + return _py_reader( + capacity=capacity, + shapes=None, + dtypes=None, + lod_levels=None, + name=name, + use_double_buffer=use_double_buffer, + feed_list=feed_list, + ) def __create_shared_decorated_reader__(op_type, reader, attrs): var_name = unique_name(op_type) startup_blk = default_startup_program().current_block() startup_var = startup_blk.create_var(name=var_name) - startop_op = startup_blk.append_op(type=op_type, - inputs={'UnderlyingReader': reader}, - outputs={'Out': [startup_var]}, - attrs=attrs) + startop_op = startup_blk.append_op( + type=op_type, + inputs={'UnderlyingReader': reader}, + outputs={'Out': [startup_var]}, + attrs=attrs, + ) startup_var.persistable = True main_prog_block = default_main_program().current_block() main_prog_var = _copy_reader_var_(main_prog_block, startup_var) @@ -828,10 +865,12 @@ def __create_unshared_decorated_reader__(op_type, reader, attrs, name=None): new_reader_name = name if name is not None else unique_name(op_type) main_blk = default_main_program().current_block() new_reader = main_blk.create_var(name=new_reader_name) - main_blk.append_op(type=op_type, - inputs={'UnderlyingReader': reader}, - outputs={'Out': [new_reader]}, - attrs=attrs) + main_blk.append_op( + type=op_type, + inputs={'UnderlyingReader': reader}, + outputs={'Out': [new_reader]}, + attrs=attrs, + ) return monkey_patch_reader_methods(new_reader) @@ -864,15 +903,14 @@ def double_buffer(reader, place=None, name=None): if place is not None: attrs['place'] = str(_get_paddle_place(place)).upper() - return __create_unshared_decorated_reader__('create_double_buffer_reader', - reader, - attrs, - name=name) + return __create_unshared_decorated_reader__( + 'create_double_buffer_reader', reader, attrs, name=name + ) def read_file(reader): """ - :api_attr: Static Graph + :api_attr: Static Graph Execute the given reader and get data via it. @@ -898,13 +936,14 @@ def read_file(reader): """ helper = LayerHelper('read_file') out = [ - helper.create_variable_for_type_inference(stop_gradient=True, - dtype='float32') + helper.create_variable_for_type_inference( + stop_gradient=True, dtype='float32' + ) for _ in range(len(reader.desc.shapes())) ] - helper.append_op(type='read', - inputs={'Reader': [reader]}, - outputs={'Out': out}) + helper.append_op( + type='read', inputs={'Reader': [reader]}, outputs={'Out': out} + ) if len(out) == 1: return out[0] else: diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index 3beb3bae352..b75e628a37a 100755 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -19,15 +19,26 @@ import string from io import StringIO from ..proto import framework_pb2 -from ..framework import OpProtoHolder, Variable, core, convert_np_dtype_to_dtype_, _non_static_mode, in_dygraph_mode, _in_legacy_dygraph +from ..framework import ( + OpProtoHolder, + Variable, + core, + convert_np_dtype_to_dtype_, + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, +) from ..layer_helper import LayerHelper from ..data_feeder import check_variable_and_dtype from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'generate_layer_fn', 'generate_activation_fn', 'generate_inplace_fn', - 'autodoc', 'templatedoc' + 'generate_layer_fn', + 'generate_activation_fn', + 'generate_inplace_fn', + 'autodoc', + 'templatedoc', ] @@ -57,16 +68,16 @@ _two_bang_pattern_ = re.compile(r"!!([^!]+)!!") def escape_math(text): - #return _two_bang_pattern_.sub( + # return _two_bang_pattern_.sub( # r'$$\1$$', # _single_dollar_pattern_.sub(r':math:\n`\1`', # _two_dollar_pattern_.sub(r"!!\1!!", text))) return _two_dollar_pattern_.sub(r':math:`\1`', text) -def _generate_doc_string_(op_proto, - additional_args_lines=None, - skip_attrs_set=None): +def _generate_doc_string_( + op_proto, additional_args_lines=None, skip_attrs_set=None +): """ Generate docstring by OpProto @@ -146,23 +157,30 @@ def generate_layer_fn(op_type): """ op_proto = OpProtoHolder.instance().get_op_proto(op_type) - not_intermediate_outputs = \ - [output for output in op_proto.outputs if not output.intermediate] - intermediate_outputs = \ - [output for output in op_proto.outputs if output.intermediate] + not_intermediate_outputs = [ + output for output in op_proto.outputs if not output.intermediate + ] + intermediate_outputs = [ + output for output in op_proto.outputs if output.intermediate + ] if len(not_intermediate_outputs) != 1: - raise ValueError("Only one non intermediate output operator can be", - "automatically generated. {0}".format(op_type)) + raise ValueError( + "Only one non intermediate output operator can be", + "automatically generated. {0}".format(op_type), + ) if not_intermediate_outputs[0].duplicable: raise ValueError( - "Only non duplicable op can be automatically generated.") + "Only non duplicable op can be automatically generated." + ) for output in intermediate_outputs: if output.duplicable: - raise ValueError("The op can be automatically generated only when ", - "all intermediate ops are not duplicable.") + raise ValueError( + "The op can be automatically generated only when ", + "all intermediate ops are not duplicable.", + ) o_name = not_intermediate_outputs[0].name intermediate_output_names = [output.name for output in intermediate_outputs] @@ -187,14 +205,17 @@ def generate_layer_fn(op_type): for each in val: if not isinstance(each, Variable): raise ValueError( - "input of {0} must be variable".format(op_type)) + "input of {0} must be variable".format(op_type) + ) if dtype is None: dtype = each.dtype elif dtype != each.dtype: raise ValueError( "operator {0} must input same dtype. {1} vs {2}".format( - op_type, dtype, each.dtype)) + op_type, dtype, each.dtype + ) + ) if dtype is None: arg_dtype = kwargs.get("dtype") @@ -226,8 +247,11 @@ def generate_layer_fn(op_type): outputs = dict() out = kwargs.pop(_convert_(o_name), []) if out: - out_var = out[0] if (isinstance(out, list) - or isinstance(out, tuple)) else out + out_var = ( + out[0] + if (isinstance(out, list) or isinstance(out, tuple)) + else out + ) else: out_var = helper.create_variable_for_type_inference(dtype=dtype) outputs[o_name] = [out_var] @@ -235,10 +259,9 @@ def generate_layer_fn(op_type): outputs[name] = [ helper.create_variable_for_type_inference(dtype=dtype) ] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=kwargs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=kwargs + ) return helper.append_activation(out_var) func.__name__ = op_type @@ -269,14 +292,25 @@ def generate_activation_fn(op_type): return op(x) if op_type not in ["abs", "exp", "square"]: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - op_type) + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], op_type + ) else: # abs exp square ops support dtype(int32, int64, float16, float32, float64) - check_variable_and_dtype(x, 'x', [ - 'int32', 'int64', 'float16', 'float32', 'float64', 'complex64', - 'complex128' - ], op_type) + check_variable_and_dtype( + x, + 'x', + [ + 'int32', + 'int64', + 'float16', + 'float32', + 'float64', + 'complex64', + 'complex128', + ], + op_type, + ) helper = LayerHelper(op_type, **locals()) @@ -289,7 +323,8 @@ def generate_activation_fn(op_type): op_proto, additional_args_lines=[ "name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`." - ]) + ], + ) return func @@ -309,24 +344,31 @@ def generate_inplace_fn(inplace_op_type): op = getattr(_legacy_C_ops, inplace_op_type) return op(x) warnings.warn( - "In static mode, {}() is the same as {}() and does not perform inplace operation." - .format(inplace_op_type, origin_op_type)) + "In static mode, {}() is the same as {}() and does not perform inplace operation.".format( + inplace_op_type, origin_op_type + ) + ) return generate_activation_fn(origin_op_type)(x, name) func.__name__ = inplace_op_type func.__doc__ = """ Inplace version of ``{0}`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_fluid_layers_{1}`. -""".format(origin_op_type, origin_op_type) +""".format( + origin_op_type, origin_op_type + ) return func def autodoc(comment=""): - def __impl__(func): - func.__doc__ = _generate_doc_string_( - OpProtoHolder.instance().get_op_proto(func.__name__)) + comment + func.__doc__ = ( + _generate_doc_string_( + OpProtoHolder.instance().get_op_proto(func.__name__) + ) + + comment + ) return func return __impl__ @@ -371,18 +413,21 @@ def templatedoc(op_type=None): for each_input in op_proto.inputs: input_name = _convert_(each_input.name) args["{0}_comment".format(input_name)] = trim_ending_dot( - each_input.comment) + each_input.comment + ) args["{0}_type".format(input_name)] = "Variable" for each_attr in op_proto.attrs: input_name = _convert_(each_attr.name) args["{0}_comment".format(input_name)] = trim_ending_dot( - each_attr.comment) + each_attr.comment + ) args["{0}_type".format(input_name)] = _type_to_str_(each_attr.type) for each_opt in op_proto.outputs: output_name = _convert_(each_opt.name) args["{0}_comment".format(output_name)] = trim_ending_dot( - each_opt.comment) + each_opt.comment + ) args["{0}_type".format(output_name)] = "Variable" func.__doc__ = tmpl.substitute(args) return func diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index 3b1147979e1..737d9e75e11 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -34,16 +34,22 @@ from ..dygraph import learning_rate_scheduler as imperate_lr from ..data_feeder import check_variable_and_dtype, check_type __all__ = [ - 'exponential_decay', 'natural_exp_decay', 'inverse_time_decay', - 'polynomial_decay', 'piecewise_decay', 'noam_decay', 'cosine_decay', - 'linear_lr_warmup' + 'exponential_decay', + 'natural_exp_decay', + 'inverse_time_decay', + 'polynomial_decay', + 'piecewise_decay', + 'noam_decay', + 'cosine_decay', + 'linear_lr_warmup', ] def _decay_step_counter(begin=0): # the first global step is zero in learning rate decay global_step = nn.autoincreased_step_counter( - counter_name='@LR_DECAY_COUNTER@', begin=begin, step=1) + counter_name='@LR_DECAY_COUNTER@', begin=begin, step=1 + ) global_step = tensor.cast(global_step, 'float32') return global_step @@ -94,17 +100,18 @@ def noam_decay(d_model, warmup_steps, learning_rate=1.0): """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.NoamDecay(d_model, - warmup_steps, - learning_rate=learning_rate) + decay = imperate_lr.NoamDecay( + d_model, warmup_steps, learning_rate=learning_rate + ) return decay else: global_step = _decay_step_counter(1) a = global_step**-0.5 b = (warmup_steps**-1.5) * global_step - lr_value = learning_rate * (d_model**-0.5) * nn.elementwise_min( - a, b) + lr_value = ( + learning_rate * (d_model**-0.5) * nn.elementwise_min(a, b) + ) return lr_value @@ -147,17 +154,18 @@ def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False): paddle.enable_static() base_lr = 0.1 sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.ExponentialDecay(learning_rate, decay_steps, - decay_rate, staircase) + decay = imperate_lr.ExponentialDecay( + learning_rate, decay_steps, decay_rate, staircase + ) return decay else: global_step = _decay_step_counter() @@ -173,52 +181,53 @@ def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False): def natural_exp_decay(learning_rate, decay_steps, decay_rate, staircase=False): """ -Applies natural exponential decay to the initial learning rate. + Applies natural exponential decay to the initial learning rate. - When training a model, it is often recommended to lower the learning rate as the - training progresses. By using this function, the learning rate will be decayed by - natural exponential power 'decay_rate' every 'decay_steps' steps. + When training a model, it is often recommended to lower the learning rate as the + training progresses. By using this function, the learning rate will be decayed by + natural exponential power 'decay_rate' every 'decay_steps' steps. - Decayed learning rate calculates as follows: + Decayed learning rate calculates as follows: - >>> if not staircase: - >>> decayed_learning_rate = learning_rate * exp(- decay_rate * (global_step / decay_steps)) - >>> else: - >>> decayed_learning_rate = learning_rate * exp(- decay_rate * floor(global_step / decay_steps)) + >>> if not staircase: + >>> decayed_learning_rate = learning_rate * exp(- decay_rate * (global_step / decay_steps)) + >>> else: + >>> decayed_learning_rate = learning_rate * exp(- decay_rate * floor(global_step / decay_steps)) - Args: - learning_rate(Variable|float): The initial learning rate. It should be a Variable - or a float - decay_steps(int): The learning rate decay steps. See the decay computation above. - decay_rate(float): The learning rate decay rate. See the decay computation above. - staircase(bool): If True, decay the learning rate at discrete intervals, which - means the learning rate will be decayed by natural exponential power - `decay_rate` every `decay_steps`. If False, learning rate will be - decayed continuously and following the formula above. Default: False + Args: + learning_rate(Variable|float): The initial learning rate. It should be a Variable + or a float + decay_steps(int): The learning rate decay steps. See the decay computation above. + decay_rate(float): The learning rate decay rate. See the decay computation above. + staircase(bool): If True, decay the learning rate at discrete intervals, which + means the learning rate will be decayed by natural exponential power + `decay_rate` every `decay_steps`. If False, learning rate will be + decayed continuously and following the formula above. Default: False - Returns: - The decayed learning rate. The data type is float32. + Returns: + The decayed learning rate. The data type is float32. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - import paddle.fluid as fluid - import paddle + import paddle.fluid as fluid + import paddle - paddle.enable_static() - base_lr = 0.1 - sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.natural_exp_decay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + paddle.enable_static() + base_lr = 0.1 + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=fluid.layers.natural_exp_decay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.NaturalExpDecay(learning_rate, decay_steps, - decay_rate, staircase) + decay = imperate_lr.NaturalExpDecay( + learning_rate, decay_steps, decay_rate, staircase + ) return decay else: global_step = _decay_step_counter() @@ -268,16 +277,17 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False): paddle.enable_static() base_lr = 0.1 sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.inverse_time_decay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.inverse_time_decay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.InverseTimeDecay(learning_rate, decay_steps, - decay_rate, staircase) + decay = imperate_lr.InverseTimeDecay( + learning_rate, decay_steps, decay_rate, staircase + ) return decay else: global_step = _decay_step_counter() @@ -291,11 +301,9 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False): return decayed_lr -def polynomial_decay(learning_rate, - decay_steps, - end_learning_rate=0.0001, - power=1.0, - cycle=False): +def polynomial_decay( + learning_rate, decay_steps, end_learning_rate=0.0001, power=1.0, cycle=False +): """ Applies polynomial decay to the initial learning rate. @@ -332,74 +340,77 @@ def polynomial_decay(learning_rate, """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.PolynomialDecay(learning_rate, decay_steps, - end_learning_rate, power, cycle) + decay = imperate_lr.PolynomialDecay( + learning_rate, decay_steps, end_learning_rate, power, cycle + ) return decay else: global_step = _decay_step_counter() if cycle: div_res = ops.ceil(global_step / decay_steps) - zero_var = tensor.fill_constant(shape=[1], - dtype='float32', - value=0.0) - one_var = tensor.fill_constant(shape=[1], - dtype='float32', - value=1.0) + zero_var = tensor.fill_constant( + shape=[1], dtype='float32', value=0.0 + ) + one_var = tensor.fill_constant( + shape=[1], dtype='float32', value=1.0 + ) with control_flow.Switch() as switch: with switch.case(global_step == zero_var): tensor.assign(input=one_var, output=div_res) decay_steps = decay_steps * div_res else: - decay_steps_var = tensor.fill_constant(shape=[1], - dtype='float32', - value=float(decay_steps)) - global_step = nn.elementwise_min(x=global_step, - y=decay_steps_var) - - decayed_lr = (learning_rate - end_learning_rate) * \ - ((1 - global_step / decay_steps) ** power) + end_learning_rate + decay_steps_var = tensor.fill_constant( + shape=[1], dtype='float32', value=float(decay_steps) + ) + global_step = nn.elementwise_min( + x=global_step, y=decay_steps_var + ) + + decayed_lr = (learning_rate - end_learning_rate) * ( + (1 - global_step / decay_steps) ** power + ) + end_learning_rate return decayed_lr def piecewise_decay(boundaries, values): """ -Applies piecewise decay to the initial learning rate. - - The algorithm can be described as the code below. + Applies piecewise decay to the initial learning rate. - .. code-block:: text + The algorithm can be described as the code below. - boundaries = [10000, 20000] - values = [1.0, 0.5, 0.1] - if step < 10000: - learning_rate = 1.0 - elif 10000 <= step < 20000: - learning_rate = 0.5 - else: - learning_rate = 0.1 - Args: - boundaries: A list of steps numbers. - values: A list of learning rate values that will be picked during - different step boundaries. + .. code-block:: text - Returns: - The decayed learning rate. - - Examples: - .. code-block:: python - - import paddle.fluid as fluid - import paddle - paddle.enable_static() boundaries = [10000, 20000] values = [1.0, 0.5, 0.1] - optimizer = fluid.optimizer.Momentum( - momentum=0.9, - learning_rate=fluid.layers.piecewise_decay(boundaries=boundaries, values=values), - regularization=fluid.regularizer.L2Decay(1e-4)) + if step < 10000: + learning_rate = 1.0 + elif 10000 <= step < 20000: + learning_rate = 0.5 + else: + learning_rate = 0.1 + Args: + boundaries: A list of steps numbers. + values: A list of learning rate values that will be picked during + different step boundaries. + + Returns: + The decayed learning rate. + + Examples: + .. code-block:: python + + import paddle.fluid as fluid + import paddle + paddle.enable_static() + boundaries = [10000, 20000] + values = [1.0, 0.5, 0.1] + optimizer = fluid.optimizer.Momentum( + momentum=0.9, + learning_rate=fluid.layers.piecewise_decay(boundaries=boundaries, values=values), + regularization=fluid.regularizer.L2Decay(1e-4)) """ @@ -413,29 +424,36 @@ Applies piecewise decay to the initial learning rate. else: global_step = _decay_step_counter() - lr = tensor.create_global_var(shape=[1], - value=0.0, - dtype='float32', - persistable=True, - name="learning_rate") + lr = tensor.create_global_var( + shape=[1], + value=0.0, + dtype='float32', + persistable=True, + name="learning_rate", + ) with control_flow.Switch() as switch: for i in range(len(boundaries)): - boundary_val = tensor.fill_constant(shape=[1], - dtype='float32', - value=float( - boundaries[i]), - force_cpu=True) + boundary_val = tensor.fill_constant( + shape=[1], + dtype='float32', + value=float(boundaries[i]), + force_cpu=True, + ) with switch.case(global_step < boundary_val): - tensor.fill_constant(shape=[1], - dtype="float32", - value=float(values[i]), - out=lr) + tensor.fill_constant( + shape=[1], + dtype="float32", + value=float(values[i]), + out=lr, + ) with switch.default(): - tensor.fill_constant(shape=[1], - dtype="float32", - value=float(values[len(values) - 1]), - out=lr) + tensor.fill_constant( + shape=[1], + dtype="float32", + value=float(values[len(values) - 1]), + out=lr, + ) return lr @@ -469,20 +487,25 @@ def cosine_decay(learning_rate, step_each_epoch, epochs): lr = fluid.layers.cosine_decay( learning_rate = base_lr, step_each_epoch=10000, epochs=120) """ - check_type(learning_rate, 'learning_rate', (float, tensor.Variable), - 'cosine_decay') + check_type( + learning_rate, 'learning_rate', (float, tensor.Variable), 'cosine_decay' + ) with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.CosineDecay(learning_rate, step_each_epoch, - epochs) + decay = imperate_lr.CosineDecay( + learning_rate, step_each_epoch, epochs + ) return decay else: global_step = _decay_step_counter() cur_epoch = ops.floor(global_step / step_each_epoch) - decayed_lr = learning_rate * 0.5 * ( - ops.cos(cur_epoch * math.pi / epochs) + 1) + decayed_lr = ( + learning_rate + * 0.5 + * (ops.cos(cur_epoch * math.pi / epochs) + 1) + ) return decayed_lr @@ -551,26 +574,31 @@ def linear_lr_warmup(learning_rate, warmup_steps, start_lr, end_lr): with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - lr = imperate_lr.LinearLrWarmup(learning_rate, warmup_steps, - start_lr, end_lr) + lr = imperate_lr.LinearLrWarmup( + learning_rate, warmup_steps, start_lr, end_lr + ) return lr else: - lr = tensor.create_global_var(shape=[1], - value=0.0, - dtype=dtype, - persistable=True, - name="learning_rate_warmup") + lr = tensor.create_global_var( + shape=[1], + value=0.0, + dtype=dtype, + persistable=True, + name="learning_rate_warmup", + ) global_step = _decay_step_counter() with control_flow.Switch() as switch: with switch.case(global_step < warmup_steps): - decayed_lr = start_lr + linear_step * (global_step / - float(warmup_steps)) + decayed_lr = start_lr + linear_step * ( + global_step / float(warmup_steps) + ) tensor.assign(decayed_lr, lr) with switch.default(): if not isinstance(learning_rate, Variable): learning_rate = tensor.fill_constant( - shape=[1], dtype=dtype, value=float(learning_rate)) + shape=[1], dtype=dtype, value=float(learning_rate) + ) tensor.assign(learning_rate, lr) return lr diff --git a/python/paddle/fluid/layers/loss.py b/python/paddle/fluid/layers/loss.py index 23835d7e7a0..6dbbb7338a7 100644 --- a/python/paddle/fluid/layers/loss.py +++ b/python/paddle/fluid/layers/loss.py @@ -19,7 +19,13 @@ from paddle.utils import deprecated from . import nn from .layer_function_generator import templatedoc from ..layer_helper import LayerHelper -from ..framework import Variable, _non_static_mode, static_only, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Variable, + _non_static_mode, + static_only, + _in_legacy_dygraph, + in_dygraph_mode, +) from .. import core from ..data_feeder import check_variable_and_dtype, check_type from ..param_attr import ParamAttr @@ -52,12 +58,9 @@ __all__ = [ kIgnoreIndex = -100 -def center_loss(input, - label, - num_classes, - alpha, - param_attr, - update_center=True): +def center_loss( + input, label, num_classes, alpha, param_attr, update_center=True +): r""" :api_attr: Static Graph @@ -107,20 +110,22 @@ def center_loss(input, """ helper = LayerHelper('center_loss', **locals()) dtype = helper.input_dtype() - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'center_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'center_loss' + ) check_variable_and_dtype(label, 'label', ['int32', 'int64'], 'center_loss') centers_shape = [num_classes, input.shape[1]] - centers_param = helper.create_parameter(attr=param_attr, - shape=centers_shape, - dtype=dtype) + centers_param = helper.create_parameter( + attr=param_attr, shape=centers_shape, dtype=dtype + ) centers_param.stop_gradient = True if isinstance(alpha, Variable): alpha_param = alpha - check_variable_and_dtype(alpha, 'alpha', ['float32', 'float64'], - 'center_loss') + check_variable_and_dtype( + alpha, 'alpha', ['float32', 'float64'], 'center_loss' + ) else: assert isinstance(alpha, float) alpha_param = helper.create_variable( @@ -130,26 +135,26 @@ def center_loss(input, type=core.VarDesc.VarType.LOD_TENSOR, persistable=True, stop_gradient=True, - initializer=Constant(alpha)) + initializer=Constant(alpha), + ) centersdiff = helper.create_variable_for_type_inference(dtype=input.dtype) loss = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='center_loss', - inputs={ - 'X': [input], - 'Label': [label], - 'Centers': [centers_param], - 'CenterUpdateRate': [alpha_param] - }, - outputs={ - 'SampleCenterDiff': [centersdiff], - 'Loss': [loss], - 'CentersOut': [centers_param] - }, - attrs={ - 'cluster_num': num_classes, - 'need_update': update_center - }) + helper.append_op( + type='center_loss', + inputs={ + 'X': [input], + 'Label': [label], + 'Centers': [centers_param], + 'CenterUpdateRate': [alpha_param], + }, + outputs={ + 'SampleCenterDiff': [centersdiff], + 'Loss': [loss], + 'CentersOut': [centers_param], + }, + attrs={'cluster_num': num_classes, 'need_update': update_center}, + ) return loss @@ -195,22 +200,22 @@ def bpr_loss(input, label, name=None): """ helper = LayerHelper('bpr_loss', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'bpr_loss') - helper.append_op(type='bpr_loss', - inputs={ - 'X': [input], - 'Label': [label] - }, - outputs={'Y': [out]}) + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'bpr_loss' + ) + helper.append_op( + type='bpr_loss', + inputs={'X': [input], 'Label': [label]}, + outputs={'Y': [out]}, + ) return out def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex): r""" :alias_main: paddle.nn.functional.cross_entropy - :alias: paddle.nn.functional.cross_entropy,paddle.nn.functional.loss.cross_entropy - :old_api: paddle.fluid.layers.cross_entropy + :alias: paddle.nn.functional.cross_entropy,paddle.nn.functional.loss.cross_entropy + :old_api: paddle.fluid.layers.cross_entropy This operator computes the cross entropy between input and label. It supports both hard-label and and soft-label cross entropy computation. @@ -264,46 +269,46 @@ def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex): return cross_entropy2(input, label, ignore_index) if _non_static_mode(): - return _legacy_C_ops.cross_entropy(input, label, "soft_label", - soft_label, "ignore_index", - ignore_index) + return _legacy_C_ops.cross_entropy( + input, label, "soft_label", soft_label, "ignore_index", ignore_index + ) inputs = {'X': [input], 'Label': [label]} attrs = {"soft_label": soft_label, "ignore_index": ignore_index} - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'cross_entropy') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'cross_entropy' + ) helper = LayerHelper('cross_entropy', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='cross_entropy', - inputs=inputs, - outputs={'Y': [out]}, - attrs=attrs) + helper.append_op( + type='cross_entropy', inputs=inputs, outputs={'Y': [out]}, attrs=attrs + ) return out def cross_entropy2(input, label, ignore_index=kIgnoreIndex): if _non_static_mode(): - loss, _, _ = _legacy_C_ops.cross_entropy2(input, label, 'ignore_index', - ignore_index) + loss, _, _ = _legacy_C_ops.cross_entropy2( + input, label, 'ignore_index', ignore_index + ) return loss inputs = {'X': [input], 'Label': [label]} attrs = {'ignore_index': ignore_index} - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'cross_entropy2') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'cross_entropy2' + ) helper = LayerHelper('cross_entropy2', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) xshape = helper.create_variable_for_type_inference(dtype=input.dtype) match_x = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='cross_entropy2', - inputs=inputs, - outputs={ - 'Y': [out], - 'MatchX': [match_x], - 'XShape': [xshape] - }, - attrs=attrs) + helper.append_op( + type='cross_entropy2', + inputs=inputs, + outputs={'Y': [out], 'MatchX': [match_x], 'XShape': [xshape]}, + attrs=attrs, + ) return out @@ -342,12 +347,14 @@ def square_error_cost(input, label): return paddle.nn.functional.square_error_cost(input, label) -def edit_distance(input, - label, - normalized=True, - ignored_tokens=None, - input_length=None, - label_length=None): +def edit_distance( + input, + label, + normalized=True, + ignored_tokens=None, + input_length=None, + label_length=None, +): """ This op computes the edit distances, also called Levenshtein distance, between a batch of hypothesis strings and their references. It measures how dissimilar two strings are by counting @@ -382,7 +389,7 @@ def edit_distance(input, NOTE: This Api is different from fluid.metrics.EditDistance Returns: - Tuple: + Tuple: distance(Tensor): edit distance result, its data type is float32, and its shape is (batch_size, 1). sequence_num(Tensor): sequence number, its data type is float32, and its shape is (1,). @@ -415,17 +422,19 @@ def edit_distance(input, # [4] """ - return paddle.nn.functional.loss.edit_distance(input, label, normalized, - ignored_tokens, input_length, - label_length) + return paddle.nn.functional.loss.edit_distance( + input, label, normalized, ignored_tokens, input_length, label_length + ) -def warpctc(input, - label, - blank=0, - norm_by_times=False, - input_length=None, - label_length=None): +def warpctc( + input, + label, + blank=0, + norm_by_times=False, + input_length=None, + label_length=None, +): """ An operator integrating the open source Warp-CTC library (https://github.com/baidu-research/warp-ctc) @@ -548,8 +557,9 @@ def warpctc(input, raise ValueError( "input_length and label_length must not be None in dygraph mode!" ) - loss_out = _C_ops.warpctc(input, label, input_length, label_length, - blank, norm_by_times) + loss_out = _C_ops.warpctc( + input, label, input_length, label_length, blank, norm_by_times + ) return loss_out if _non_static_mode(): if input_length is None or label_length is None: @@ -572,26 +582,27 @@ def warpctc(input, check_variable_and_dtype(label, 'label', ['int32'], "warpctc") this_inputs = {'Logits': [input], 'Label': [label]} if input_length is not None and label_length is not None: - check_variable_and_dtype(input_length, 'LogitsLength', ['int64'], - "warpctc") - check_variable_and_dtype(label_length, 'LabelLength', ['int64'], - "warpctc") + check_variable_and_dtype( + input_length, 'LogitsLength', ['int64'], "warpctc" + ) + check_variable_and_dtype( + label_length, 'LabelLength', ['int64'], "warpctc" + ) this_inputs['LogitsLength'] = [input_length] this_inputs['LabelLength'] = [label_length] loss_out = helper.create_variable_for_type_inference(dtype=input.dtype) grad_out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='warpctc', - inputs=this_inputs, - outputs={ - 'WarpCTCGrad': [grad_out], - 'Loss': [loss_out] - }, - attrs={ - 'blank': blank, - 'norm_by_times': norm_by_times, - }) + helper.append_op( + type='warpctc', + inputs=this_inputs, + outputs={'WarpCTCGrad': [grad_out], 'Loss': [loss_out]}, + attrs={ + 'blank': blank, + 'norm_by_times': norm_by_times, + }, + ) return loss_out @@ -600,18 +611,20 @@ def warpctc(input, # the type is often "Variable", and arguments may vary. @static_only @templatedoc(op_type="nce") -def nce(input, - label, - num_total_classes, - sample_weight=None, - param_attr=None, - bias_attr=None, - num_neg_samples=None, - name=None, - sampler="uniform", - custom_dist=None, - seed=0, - is_sparse=False): +def nce( + input, + label, + num_total_classes, + sample_weight=None, + param_attr=None, + bias_attr=None, + num_neg_samples=None, + name=None, + sampler="uniform", + custom_dist=None, + seed=0, + is_sparse=False, +): """ :api_attr: Static Graph @@ -696,16 +709,20 @@ def nce(input, dim = input.shape[1] num_true_class = label.shape[1] - w = helper.create_parameter(attr=helper.param_attr, - shape=[num_total_classes, dim], - is_bias=False, - dtype=input.dtype) + w = helper.create_parameter( + attr=helper.param_attr, + shape=[num_total_classes, dim], + is_bias=False, + dtype=input.dtype, + ) inputs = {} if helper.bias_attr: - b = helper.create_parameter(attr=helper.bias_attr, - shape=[num_total_classes, 1], - is_bias=True, - dtype=input.dtype) + b = helper.create_parameter( + attr=helper.bias_attr, + shape=[num_total_classes, 1], + is_bias=True, + dtype=input.dtype, + ) inputs['Bias'] = b cost = helper.create_variable_for_type_inference(dtype=input.dtype) sample_logits = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -770,16 +787,20 @@ def nce(input, attr=ParamAttr(), shape=numpy_array.shape, dtype=numpy_array.dtype, - default_initializer=NumpyArrayInitializer(numpy_array)) + default_initializer=NumpyArrayInitializer(numpy_array), + ) ret.stop_gradient = True return ret inputs['CustomDistProbs'] = _init_by_numpy_array( - np.array(custom_dist).astype('float32')) + np.array(custom_dist).astype('float32') + ) inputs['CustomDistAlias'] = _init_by_numpy_array( - np.array(alias_).astype('int32')) + np.array(alias_).astype('int32') + ) inputs['CustomDistAliasProbs'] = _init_by_numpy_array( - np.array(alias_probs_).astype('float32')) + np.array(alias_probs_).astype('float32') + ) sampler = 2 else: raise Exception("Unsupported sampler type.") @@ -800,30 +821,34 @@ def nce(input, 'seed': seed, 'sampler': sampler, 'is_sparse': is_sparse, - 'remote_prefetch': remote_prefetch + 'remote_prefetch': remote_prefetch, } - helper.append_op(type='nce', - inputs=inputs, - outputs={ - 'Cost': cost, - 'SampleLogits': sample_logits, - 'SampleLabels': sample_labels - }, - attrs=attrs) + helper.append_op( + type='nce', + inputs=inputs, + outputs={ + 'Cost': cost, + 'SampleLogits': sample_logits, + 'SampleLabels': sample_labels, + }, + attrs=attrs, + ) return cost / (num_neg_samples + 1) -def hsigmoid(input, - label, - num_classes, - param_attr=None, - bias_attr=None, - name=None, - path_table=None, - path_code=None, - is_custom=False, - is_sparse=False): +def hsigmoid( + input, + label, + num_classes, + param_attr=None, + bias_attr=None, + name=None, + path_table=None, + path_code=None, + is_custom=False, + is_sparse=False, +): """ :api_attr: Static Graph @@ -906,16 +931,19 @@ def hsigmoid(input, dim = input.shape[1] if ((num_classes is None) or (num_classes < 2)) and (not is_custom): raise ValueError( - "num_classes must not be less than 2 with default tree") + "num_classes must not be less than 2 with default tree" + ) if (not is_custom) and (is_sparse): print("Sparse mode should not be used without custom tree") is_sparse = False - if (not is_custom) and ((path_table is not None) or - (path_code is not None)): + if (not is_custom) and ( + (path_table is not None) or (path_code is not None) + ): raise ValueError( - "only num_classes should be passed without custom tree") + "only num_classes should be passed without custom tree" + ) if (is_custom) and (path_code is None): raise ValueError("path_code should not be None with custom tree") @@ -932,59 +960,67 @@ def hsigmoid(input, "With sparse mode, if your models has only small parameter prefetch may cause speed down" ) if not is_custom: - weights = helper.create_parameter(attr=helper.param_attr, - shape=[num_classes - 1, dim], - is_bias=False, - dtype=input.dtype) + weights = helper.create_parameter( + attr=helper.param_attr, + shape=[num_classes - 1, dim], + is_bias=False, + dtype=input.dtype, + ) else: - weights = helper.create_parameter(attr=helper.param_attr, - shape=[num_classes, dim], - is_bias=False, - dtype=input.dtype) + weights = helper.create_parameter( + attr=helper.param_attr, + shape=[num_classes, dim], + is_bias=False, + dtype=input.dtype, + ) inputs = { "X": input, "W": weights, "PathTable": path_table, "PathCode": path_code, - "Label": label + "Label": label, } if helper.bias_attr: if not is_custom: - bias = helper.create_parameter(attr=helper.bias_attr, - shape=[num_classes - 1, 1], - is_bias=True, - dtype=input.dtype) + bias = helper.create_parameter( + attr=helper.bias_attr, + shape=[num_classes - 1, 1], + is_bias=True, + dtype=input.dtype, + ) inputs['Bias'] = bias else: - bias = helper.create_parameter(attr=helper.bias_attr, - shape=[num_classes, 1], - is_bias=True, - dtype=input.dtype) + bias = helper.create_parameter( + attr=helper.bias_attr, + shape=[num_classes, 1], + is_bias=True, + dtype=input.dtype, + ) inputs['Bias'] = bias - helper.append_op(type="hierarchical_sigmoid", - inputs=inputs, - outputs={ - "Out": out, - "PreOut": pre_out, - "W_Out": weights - }, - attrs={ - "num_classes": num_classes, - "is_sparse": is_sparse, - "remote_prefetch": remote_prefetch - }) + helper.append_op( + type="hierarchical_sigmoid", + inputs=inputs, + outputs={"Out": out, "PreOut": pre_out, "W_Out": weights}, + attrs={ + "num_classes": num_classes, + "is_sparse": is_sparse, + "remote_prefetch": remote_prefetch, + }, + ) return out -def sampled_softmax_with_cross_entropy(logits, - label, - num_samples, - num_true=1, - remove_accidental_hits=True, - use_customized_samples=False, - customized_samples=None, - customized_probabilities=None, - seed=0): +def sampled_softmax_with_cross_entropy( + logits, + label, + num_samples, + num_true=1, + remove_accidental_hits=True, + use_customized_samples=False, + customized_samples=None, + customized_probabilities=None, + seed=0, +): """ **Sampled Softmax With Cross Entropy Operator.** @@ -1052,90 +1088,121 @@ def sampled_softmax_with_cross_entropy(logits, logits=fc, label=label, num_samples=25) """ if _non_static_mode(): - sample_logits_attrs = ('use_customized_samples', use_customized_samples, - 'uniq', True, 'remove_accidental_hits', - remove_accidental_hits, 'num_samples', - num_samples, 'seed', seed) - _, _, _, _, sampled_logits_out, sampled_label_out = _legacy_C_ops.sample_logits( - logits, label, *sample_logits_attrs) + sample_logits_attrs = ( + 'use_customized_samples', + use_customized_samples, + 'uniq', + True, + 'remove_accidental_hits', + remove_accidental_hits, + 'num_samples', + num_samples, + 'seed', + seed, + ) + ( + _, + _, + _, + _, + sampled_logits_out, + sampled_label_out, + ) = _legacy_C_ops.sample_logits(logits, label, *sample_logits_attrs) depth = num_samples + 1 - sampled_softlabel_out = _legacy_C_ops.one_hot(sampled_label_out, - 'depth', depth) + sampled_softlabel_out = _legacy_C_ops.one_hot( + sampled_label_out, 'depth', depth + ) - softmax_with_cross_entropy_attrs = ('soft_label', True, - 'numeric_stable_mode', False) + softmax_with_cross_entropy_attrs = ( + 'soft_label', + True, + 'numeric_stable_mode', + False, + ) _, loss = _legacy_C_ops.softmax_with_cross_entropy( - sampled_logits_out, sampled_softlabel_out, - *softmax_with_cross_entropy_attrs) + sampled_logits_out, + sampled_softlabel_out, + *softmax_with_cross_entropy_attrs + ) return loss / num_true helper = LayerHelper('sample_logits', **locals()) - samples = customized_samples if use_customized_samples else helper.create_variable_for_type_inference( - dtype='int64') - probabilities = customized_probabilities if use_customized_samples else helper.create_variable_for_type_inference( - dtype=logits.dtype) - sampled_logits \ - = helper.create_variable_for_type_inference(dtype=logits.dtype) + samples = ( + customized_samples + if use_customized_samples + else helper.create_variable_for_type_inference(dtype='int64') + ) + probabilities = ( + customized_probabilities + if use_customized_samples + else helper.create_variable_for_type_inference(dtype=logits.dtype) + ) + sampled_logits = helper.create_variable_for_type_inference( + dtype=logits.dtype + ) sampled_label = helper.create_variable_for_type_inference(dtype='int64') sampled_softlabel = helper.create_variable_for_type_inference( - dtype=logits.dtype) + dtype=logits.dtype + ) logits_dim = helper.create_variable_for_type_inference(dtype=logits.dtype) labels_dim = helper.create_variable_for_type_inference(dtype=label.type) - helper.append_op(type='sample_logits', - inputs={ - 'Logits': logits, - 'Labels': label, - 'CustomizedSamples': customized_samples, - 'CustomizedProbabilities': customized_probabilities - }, - outputs={ - 'Samples': samples, - 'Probabilities': probabilities, - 'SampledLabels': sampled_label, - 'SampledLogits': sampled_logits, - 'LogitsDim': logits_dim, - 'LabelsDim': labels_dim - }, - attrs={ - 'use_customized_samples': use_customized_samples, - 'uniq': True, - 'remove_accidental_hits': remove_accidental_hits, - 'num_samples': num_samples, - 'seed': seed - }) + helper.append_op( + type='sample_logits', + inputs={ + 'Logits': logits, + 'Labels': label, + 'CustomizedSamples': customized_samples, + 'CustomizedProbabilities': customized_probabilities, + }, + outputs={ + 'Samples': samples, + 'Probabilities': probabilities, + 'SampledLabels': sampled_label, + 'SampledLogits': sampled_logits, + 'LogitsDim': logits_dim, + 'LabelsDim': labels_dim, + }, + attrs={ + 'use_customized_samples': use_customized_samples, + 'uniq': True, + 'remove_accidental_hits': remove_accidental_hits, + 'num_samples': num_samples, + 'seed': seed, + }, + ) loss = helper.create_variable_for_type_inference(dtype=logits.dtype) softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) - helper.append_op(type='one_hot', - inputs={'X': sampled_label}, - attrs={'depth': num_samples + 1}, - outputs={'Out': sampled_softlabel}) - - helper.append_op(type='softmax_with_cross_entropy', - inputs={ - 'Logits': sampled_logits, - 'Label': sampled_softlabel - }, - outputs={ - 'Softmax': softmax, - 'Loss': loss - }, - attrs={ - 'soft_label': True, - 'ignore_index': False, - 'numeric_stable_mode': False - }) + helper.append_op( + type='one_hot', + inputs={'X': sampled_label}, + attrs={'depth': num_samples + 1}, + outputs={'Out': sampled_softlabel}, + ) + + helper.append_op( + type='softmax_with_cross_entropy', + inputs={'Logits': sampled_logits, 'Label': sampled_softlabel}, + outputs={'Softmax': softmax, 'Loss': loss}, + attrs={ + 'soft_label': True, + 'ignore_index': False, + 'numeric_stable_mode': False, + }, + ) return loss / num_true -def softmax_with_cross_entropy(logits, - label, - soft_label=False, - ignore_index=kIgnoreIndex, - numeric_stable_mode=True, - return_softmax=False, - axis=-1): +def softmax_with_cross_entropy( + logits, + label, + soft_label=False, + ignore_index=kIgnoreIndex, + numeric_stable_mode=True, + return_softmax=False, + axis=-1, +): r""" This operator implements the cross entropy loss function with softmax. This function @@ -1231,8 +1298,14 @@ def softmax_with_cross_entropy(logits, print(out) """ return paddle.nn.functional.loss.fluid_softmax_with_cross_entropy( - logits, label, soft_label, ignore_index, numeric_stable_mode, - return_softmax, axis) + logits, + label, + soft_label, + ignore_index, + numeric_stable_mode, + return_softmax, + axis, + ) def identity_loss(x, reduction="none"): @@ -1285,10 +1358,9 @@ def identity_loss(x, reduction="none"): helper = LayerHelper('identity_loss', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="identity_loss", - inputs={"X": x}, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type="identity_loss", inputs={"X": x}, outputs={"Out": out}, attrs=attrs + ) return out @@ -1346,13 +1418,11 @@ def rank_loss(label, left, right, name=None): out = helper.create_variable_for_type_inference("float32") - helper.append_op(type='rank_loss', - inputs={ - "Label": label, - "Left": left, - "Right": right - }, - outputs={'Out': out}) + helper.append_op( + type='rank_loss', + inputs={"Label": label, "Left": left, "Right": right}, + outputs={'Out': out}, + ) return out @@ -1397,26 +1467,19 @@ def margin_rank_loss(label, left, right, margin=0.1, name=None): check_variable_and_dtype(label, 'right', ['float32'], 'margin_rank_loss') out = helper.create_variable_for_type_inference(left.dtype) act = helper.create_variable_for_type_inference(left.dtype) - helper.append_op(type='margin_rank_loss', - inputs={ - "Label": label, - "X1": left, - "X2": right - }, - outputs={ - 'Out': out, - 'Activated': act - }, - attrs={'margin': margin}) + helper.append_op( + type='margin_rank_loss', + inputs={"Label": label, "X1": left, "X2": right}, + outputs={'Out': out, 'Activated': act}, + attrs={'margin': margin}, + ) return out @templatedoc() -def sigmoid_cross_entropy_with_logits(x, - label, - ignore_index=kIgnoreIndex, - name=None, - normalize=False): +def sigmoid_cross_entropy_with_logits( + x, label, ignore_index=kIgnoreIndex, name=None, normalize=False +): """ ${comment} @@ -1453,32 +1516,32 @@ def sigmoid_cross_entropy_with_logits(x, """ if in_dygraph_mode(): - return _C_ops.sigmoid_cross_entropy_with_logits(x, label, normalize, - int(ignore_index)) - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'sigmoid_cross_entropy_with_logits') + return _C_ops.sigmoid_cross_entropy_with_logits( + x, label, normalize, int(ignore_index) + ) + check_variable_and_dtype( + x, + 'input', + ['float16', 'float32', 'float64'], + 'sigmoid_cross_entropy_with_logits', + ) helper = LayerHelper("sigmoid_cross_entropy_with_logits", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="sigmoid_cross_entropy_with_logits", - inputs={ - "X": x, - "Label": label - }, - attrs={ - "ignore_index": ignore_index, - 'normalize': normalize - }, - outputs={"Out": out}) + helper.append_op( + type="sigmoid_cross_entropy_with_logits", + inputs={"X": x, "Label": label}, + attrs={"ignore_index": ignore_index, 'normalize': normalize}, + outputs={"Out": out}, + ) return out -def teacher_student_sigmoid_loss(input, - label, - soft_max_up_bound=15.0, - soft_max_lower_bound=-15.0): +def teacher_student_sigmoid_loss( + input, label, soft_max_up_bound=15.0, soft_max_lower_bound=-15.0 +): """ **Teacher Student Log Loss Layer** @@ -1517,22 +1580,30 @@ def teacher_student_sigmoid_loss(input, cost = fluid.layers.teacher_student_sigmoid_loss(input=similarity, label=label) """ - check_variable_and_dtype(input, "input", - ['float32', 'float64', 'int32', 'int64'], - 'teacher_student_sigmoid_loss') - check_variable_and_dtype(label, "label", - ['float32', 'float64', 'int32', 'int64'], - 'teacher_student_sigmoid_loss') + check_variable_and_dtype( + input, + "input", + ['float32', 'float64', 'int32', 'int64'], + 'teacher_student_sigmoid_loss', + ) + check_variable_and_dtype( + label, + "label", + ['float32', 'float64', 'int32', 'int64'], + 'teacher_student_sigmoid_loss', + ) helper = LayerHelper('teacher_student_sigmoid_loss', **locals()) out = helper.create_variable(dtype=input.dtype) helper.append_op( type='teacher_student_sigmoid_loss', - inputs={'X': [input], - 'Label': [label]}, + inputs={'X': [input], 'Label': [label]}, outputs={'Y': [out]}, - attrs={"soft_max_lower_bound": float(soft_max_lower_bound), \ - "soft_max_up_bound": float(soft_max_up_bound)}) + attrs={ + "soft_max_lower_bound": float(soft_max_lower_bound), + "soft_max_up_bound": float(soft_max_up_bound), + }, + ) return out @@ -1587,23 +1658,22 @@ def huber_loss(input, label, delta): return out helper = LayerHelper('huber_loss', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'huber_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'huber_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'huber_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'huber_loss' + ) residual = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='huber_loss', - inputs={ - 'X': input, - 'Y': label - }, - outputs={ - 'Out': out, - 'Residual': residual - }, - attrs={'delta': delta}) + helper.append_op( + type='huber_loss', + inputs={'X': input, 'Y': label}, + outputs={'Out': out, 'Residual': residual}, + attrs={'delta': delta}, + ) return out @@ -1653,17 +1723,17 @@ def kldiv_loss(x, target, reduction='mean', name=None): """ helper = LayerHelper('kldiv_loss', **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'kldiv_loss') - check_variable_and_dtype(target, 'target', ['float32', 'float64'], - 'kldiv_loss') + check_variable_and_dtype( + target, 'target', ['float32', 'float64'], 'kldiv_loss' + ) check_type(reduction, 'reduction', str, 'kldiv_loss') loss = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='kldiv_loss', - inputs={ - 'X': x, - 'Target': target - }, - outputs={'Loss': loss}, - attrs={'reduction': reduction}) + helper.append_op( + type='kldiv_loss', + inputs={'X': x, 'Target': target}, + outputs={'Loss': loss}, + attrs={'reduction': reduction}, + ) return loss diff --git a/python/paddle/fluid/layers/math_op_patch.py b/python/paddle/fluid/layers/math_op_patch.py index a0fd6782db6..fc71321f147 100644 --- a/python/paddle/fluid/layers/math_op_patch.py +++ b/python/paddle/fluid/layers/math_op_patch.py @@ -53,14 +53,13 @@ EXPRESSION_MAP = { "__lt__": "A < B", "__le__": "A <= B", "__gt__": "A > B", - "__ge__": "A >= B" + "__ge__": "A >= B", } _already_patch_variable = False def monkey_patch_variable(): - def unique_tmp_name(): return unique_name.generate("tmp") @@ -85,15 +84,17 @@ def monkey_patch_variable(): def create_tensor(block, value, dtype, shape): value = float(value) var = create_new_tmp_var(block, dtype) - block.append_op(type="fill_constant", - outputs={'Out': [var]}, - attrs={ - 'dtype': var.dtype, - 'shape': shape, - 'value': value, - 'force_cpu': False - }, - stop_gradient=True) + block.append_op( + type="fill_constant", + outputs={'Out': [var]}, + attrs={ + 'dtype': var.dtype, + 'shape': shape, + 'value': value, + 'force_cpu': False, + }, + stop_gradient=True, + ) var.stop_gradient = True return var @@ -117,16 +118,18 @@ def monkey_patch_variable(): else: out_shape.append(d) assert batch_dim != -1 - block.append_op(type='fill_constant_batch_size_like', - outputs={'Out': [var]}, - inputs={'Input': [ref_var]}, - attrs={ - 'shape': out_shape, - 'value': value, - 'input_dim_idx': batch_dim, - 'output_dim_idx': batch_dim - }, - stop_gradient=True) + block.append_op( + type='fill_constant_batch_size_like', + outputs={'Out': [var]}, + inputs={'Input': [ref_var]}, + attrs={ + 'shape': out_shape, + 'value': value, + 'input_dim_idx': batch_dim, + 'output_dim_idx': batch_dim, + }, + stop_gradient=True, + ) var.stop_gradient = True return var @@ -208,37 +211,40 @@ def monkey_patch_variable(): """ block = current_block(self) out = create_new_tmp_var(block, dtype) - block.append_op(type="cast", - inputs={"X": [self]}, - outputs={"Out": [out]}, - attrs={ - "in_dtype": self.dtype, - "out_dtype": out.dtype - }) + block.append_op( + type="cast", + inputs={"X": [self]}, + outputs={"Out": [out]}, + attrs={"in_dtype": self.dtype, "out_dtype": out.dtype}, + ) out.stop_gradient = self.stop_gradient return out @static_only def append(self, var): """ - **Notes**: - **The type variable must be LoD Tensor Array. + **Notes**: + **The type variable must be LoD Tensor Array. """ if not isinstance(var, Variable): if in_declarative_mode(): - """ in dy2static mode, x may be tensorable values such as int, float, np.array - """ + """in dy2static mode, x may be tensorable values such as int, float, np.array""" from paddle.tensor.creation import to_tensor + var = to_tensor(var) else: raise TypeError( - "Required input var should be Variable, but received {}". - format(type(var))) + "Required input var should be Variable, but received {}".format( + type(var) + ) + ) if self.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: raise TypeError( - "Only Variable with VarType.LOD_TENSOR_ARRAY support `append` method, but received type: {}" - .format(self.type)) + "Only Variable with VarType.LOD_TENSOR_ARRAY support `append` method, but received type: {}".format( + self.type + ) + ) array_write(x=var, i=array_length(self), array=self) @static_only @@ -249,8 +255,10 @@ def monkey_patch_variable(): """ if len(self.shape) > 1: raise TypeError( - "Required input var should be 1-D Variable, but received {}". - format(self.shape)) + "Required input var should be 1-D Variable, but received {}".format( + self.shape + ) + ) return self @static_only @@ -266,23 +274,27 @@ def monkey_patch_variable(): Returns: Variable: self[index] """ - from paddle.fluid.dygraph.dygraph_to_static.convert_operators import _run_paddle_pop + from paddle.fluid.dygraph.dygraph_to_static.convert_operators import ( + _run_paddle_pop, + ) + if self.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: raise TypeError( - "Only Variable with VarType.LOD_TENSOR_ARRAY support `append` method, but received type: {}" - .format(self.type)) + "Only Variable with VarType.LOD_TENSOR_ARRAY support `append` method, but received type: {}".format( + self.type + ) + ) return _run_paddle_pop(self, *args) def _scalar_op_(var, scale, bias): block = current_block(var) out = create_new_tmp_var(block, var.dtype) - block.append_op(type="scale", - inputs={"X": [var]}, - outputs={"Out": [out]}, - attrs={ - "scale": scale, - "bias": bias - }) + block.append_op( + type="scale", + inputs={"X": [var]}, + outputs={"Out": [out]}, + attrs={"scale": scale, "bias": bias}, + ) return out def _neg_(var): @@ -325,11 +337,9 @@ def monkey_patch_variable(): def _scalar_div_(var, value): return _scalar_op_(var, 1.0 / value, 0.0) - def _binary_creator_(method_name, - op_type, - reverse=False, - scalar_method=None): - + def _binary_creator_( + method_name, op_type, reverse=False, scalar_method=None + ): def __impl__(self, other_var): # 1. scalar exists cases # we need combine the tensor.dtype and scalar.dtype, cast correct object @@ -352,7 +362,10 @@ def monkey_patch_variable(): # so the calculation result here and the calculation result of numpy are # different after 6 decimal point. If necessary, we can also use float64 here. # torch's behavior here is consistent with ours - if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_: + if ( + op_type == 'elementwise_div' + and self.dtype in _supported_int_dtype_ + ): self = astype(self, 'float32') # here use `scale` replace `elementwise` to get better performance # but only +, -, *, / can use this method @@ -372,18 +385,21 @@ def monkey_patch_variable(): has_batch_size = True break if not has_batch_size: - other_var = create_tensor(current_block(self), - other_var, - dtype=lhs_dtype, - shape=self.shape) + other_var = create_tensor( + current_block(self), + other_var, + dtype=lhs_dtype, + shape=self.shape, + ) else: other_var = create_tensor_with_batchsize( - self, other_var, lhs_dtype) + self, other_var, lhs_dtype + ) else: # add fill_op to current_block - other_var = create_scalar(current_block(self), - value=other_var, - dtype=lhs_dtype) + other_var = create_scalar( + current_block(self), value=other_var, dtype=lhs_dtype + ) # 3. unify right var type to left var rhs_dtype = safe_get_dtype(other_var) @@ -409,16 +425,22 @@ def monkey_patch_variable(): "%s:%s\nThe behavior of expression %s has been unified with %s(X, Y, axis=-1) from Paddle 2.0. " "If your code works well in the older versions but crashes in this version, try to use " "%s(X, Y, axis=0) instead of %s. This transitional warning will be dropped in the future." - % (file_name, line_num, EXPRESSION_MAP[method_name], - op_type, op_type, EXPRESSION_MAP[method_name]), - category=DeprecationWarning) - current_block(self).append_op(type=op_type, - inputs={ - 'X': [self], - 'Y': [other_var] - }, - outputs={'Out': out}, - attrs={'axis': axis}) + % ( + file_name, + line_num, + EXPRESSION_MAP[method_name], + op_type, + op_type, + EXPRESSION_MAP[method_name], + ), + category=DeprecationWarning, + ) + current_block(self).append_op( + type=op_type, + inputs={'X': [self], 'Y': [other_var]}, + outputs={'Out': out}, + attrs={'axis': axis}, + ) return out comment = OpProtoHolder.instance().get_op_proto(op_type).comment @@ -431,35 +453,43 @@ def monkey_patch_variable(): Returns: Variable - """.format(comment) + """.format( + comment + ) __impl__.__name__ = method_name return __impl__ def values(var): block = current_block(var) out = create_new_tmp_var(block, var.dtype) - block.append_op(type="sparse_values", - inputs={"x": [var]}, - outputs={"out": [out]}, - attrs={}) + block.append_op( + type="sparse_values", + inputs={"x": [var]}, + outputs={"out": [out]}, + attrs={}, + ) return out def indices(var): block = current_block(var) out = create_new_tmp_var(block, var.dtype) - block.append_op(type="sparse_indices", - inputs={"x": [var]}, - outputs={"out": [out]}, - attrs={}) + block.append_op( + type="sparse_indices", + inputs={"x": [var]}, + outputs={"out": [out]}, + attrs={}, + ) return out def to_dense(var): block = current_block(var) out = create_new_tmp_var(block, var.dtype) - block.append_op(type="sparse_to_dense", - inputs={"x": [var]}, - outputs={"out": [out]}, - attrs={}) + block.append_op( + type="sparse_to_dense", + inputs={"x": [var]}, + outputs={"out": [out]}, + attrs={}, + ) return out variable_methods = [ @@ -475,39 +505,78 @@ def monkey_patch_variable(): ('dim', lambda x: len(x.shape)), ('ndimension', lambda x: len(x.shape)), ('ndim', _ndim_), - ('__add__', - _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_)), + ( + '__add__', + _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_), + ), # a+b == b+a. Do not need to reverse explicitly - ('__radd__', - _binary_creator_('__radd__', 'elementwise_add', False, _scalar_add_)), - ('__sub__', - _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_)), - ('__rsub__', - _binary_creator_('__rsub__', 'elementwise_sub', True, _scalar_rsub_)), - ('__mul__', - _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_)), + ( + '__radd__', + _binary_creator_( + '__radd__', 'elementwise_add', False, _scalar_add_ + ), + ), + ( + '__sub__', + _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_), + ), + ( + '__rsub__', + _binary_creator_( + '__rsub__', 'elementwise_sub', True, _scalar_rsub_ + ), + ), + ( + '__mul__', + _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_), + ), # a*b == b*a. Do not need to reverse explicitly - ('__rmul__', - _binary_creator_('__rmul__', 'elementwise_mul', False, _scalar_mul_)), - ('__div__', - _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_)), - ('__truediv__', - _binary_creator_('__truediv__', 'elementwise_div', False, - _scalar_div_)), - ('__rdiv__', _binary_creator_('__rdiv__', 'elementwise_div', True, - None)), - ('__rtruediv__', - _binary_creator_('__rtruediv__', 'elementwise_div', True, None)), - ('__pow__', _binary_creator_('__pow__', 'elementwise_pow', False, - None)), - ('__rpow__', _binary_creator_('__rpow__', 'elementwise_pow', True, - None)), - ('__floordiv__', - _binary_creator_('__floordiv__', 'elementwise_floordiv', False, None)), - ('__mod__', _binary_creator_('__mod__', 'elementwise_mod', False, - None)), - ('__matmul__', _binary_creator_('__matmul__', "matmul_v2", False, - None)), + ( + '__rmul__', + _binary_creator_( + '__rmul__', 'elementwise_mul', False, _scalar_mul_ + ), + ), + ( + '__div__', + _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_), + ), + ( + '__truediv__', + _binary_creator_( + '__truediv__', 'elementwise_div', False, _scalar_div_ + ), + ), + ( + '__rdiv__', + _binary_creator_('__rdiv__', 'elementwise_div', True, None), + ), + ( + '__rtruediv__', + _binary_creator_('__rtruediv__', 'elementwise_div', True, None), + ), + ( + '__pow__', + _binary_creator_('__pow__', 'elementwise_pow', False, None), + ), + ( + '__rpow__', + _binary_creator_('__rpow__', 'elementwise_pow', True, None), + ), + ( + '__floordiv__', + _binary_creator_( + '__floordiv__', 'elementwise_floordiv', False, None + ), + ), + ( + '__mod__', + _binary_creator_('__mod__', 'elementwise_mod', False, None), + ), + ( + '__matmul__', + _binary_creator_('__matmul__', "matmul_v2", False, None), + ), # for logical compare ('__eq__', _binary_creator_('__eq__', 'equal', False, None)), ('__ne__', _binary_creator_('__ne__', 'not_equal', False, None)), @@ -528,13 +597,17 @@ def monkey_patch_variable(): setattr(Variable, method_name, method_impl) else: import paddle.tensor + for method_name in paddle.tensor.tensor_method_func: - if hasattr(Variable, method_name): continue + if hasattr(Variable, method_name): + continue method_impl = getattr(paddle.tensor, method_name, None) - if method_impl: setattr(Variable, method_name, method_impl) + if method_impl: + setattr(Variable, method_name, method_impl) for magic_method, origin_method in paddle.tensor.magic_method_func: impl = getattr(paddle.tensor, origin_method, None) - if impl: setattr(Variable, magic_method, impl) + if impl: + setattr(Variable, magic_method, impl) _already_patch_variable = True diff --git a/python/paddle/fluid/layers/metric_op.py b/python/paddle/fluid/layers/metric_op.py index 36679f13b5a..8a63b55089e 100755 --- a/python/paddle/fluid/layers/metric_op.py +++ b/python/paddle/fluid/layers/metric_op.py @@ -18,7 +18,13 @@ All layers just related to metric. import warnings from ..layer_helper import LayerHelper from ..initializer import Normal, Constant -from ..framework import Variable, _non_static_mode, _varbase_creator, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Variable, + _non_static_mode, + _varbase_creator, + _in_legacy_dygraph, + in_dygraph_mode, +) from .. import core from ..param_attr import ParamAttr from . import nn @@ -74,15 +80,18 @@ def accuracy(input, label, k=1, correct=None, total=None): total = _varbase_creator(dtype="int32") _k = k.numpy().item(0) if isinstance(k, Variable) else k - topk_out, topk_indices = _legacy_C_ops.top_k_v2(input, 'k', _k, - 'sorted', False) - _acc, _, _ = _legacy_C_ops.accuracy(topk_out, topk_indices, label, - correct, total) + topk_out, topk_indices = _legacy_C_ops.top_k_v2( + input, 'k', _k, 'sorted', False + ) + _acc, _, _ = _legacy_C_ops.accuracy( + topk_out, topk_indices, label, correct, total + ) return _acc helper = LayerHelper("accuracy", **locals()) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'accuracy') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'accuracy' + ) topk_out = helper.create_variable_for_type_inference(dtype=input.dtype) topk_indices = helper.create_variable_for_type_inference(dtype="int64") inputs = {"X": [input]} @@ -91,39 +100,38 @@ def accuracy(input, label, k=1, correct=None, total=None): else: attrs = {'k': k} attrs['sorted'] = False - helper.append_op(type="top_k_v2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": [topk_out], - "Indices": [topk_indices] - }) + helper.append_op( + type="top_k_v2", + inputs=inputs, + attrs=attrs, + outputs={"Out": [topk_out], "Indices": [topk_indices]}, + ) acc_out = helper.create_variable_for_type_inference(dtype="float32") if correct is None: correct = helper.create_variable_for_type_inference(dtype="int32") if total is None: total = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type="accuracy", - inputs={ - "Out": [topk_out], - "Indices": [topk_indices], - "Label": [label] - }, - outputs={ - "Accuracy": [acc_out], - "Correct": [correct], - "Total": [total], - }) + helper.append_op( + type="accuracy", + inputs={"Out": [topk_out], "Indices": [topk_indices], "Label": [label]}, + outputs={ + "Accuracy": [acc_out], + "Correct": [correct], + "Total": [total], + }, + ) return acc_out -def auc(input, - label, - curve='ROC', - num_thresholds=2**12 - 1, - topk=1, - slide_steps=1, - ins_tag_weight=None): +def auc( + input, + label, + curve='ROC', + num_thresholds=2**12 - 1, + topk=1, + slide_steps=1, + ins_tag_weight=None, +): """ **Area Under the Curve (AUC) Layer** @@ -214,13 +222,14 @@ def auc(input, helper = LayerHelper("auc", **locals()) if ins_tag_weight is None: - ins_tag_weight = tensor.fill_constant(shape=[1, 1], - dtype="float32", - value=1.0) + ins_tag_weight = tensor.fill_constant( + shape=[1, 1], dtype="float32", value=1.0 + ) check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'auc') check_variable_and_dtype(label, 'label', ['int32', 'int64'], 'auc') - check_variable_and_dtype(ins_tag_weight, 'ins_tag_weight', - ['float32', 'float64'], 'auc') + check_variable_and_dtype( + ins_tag_weight, 'ins_tag_weight', ['float32', 'float64'], 'auc' + ) auc_out = helper.create_variable_for_type_inference(dtype="float64") batch_auc_out = helper.create_variable_for_type_inference(dtype="float64") # make tp, tn, fp, fn persistable, so that can accumulate all batches. @@ -234,62 +243,71 @@ def auc(input, batch_stat_pos = helper.create_global_variable( persistable=True, dtype='int64', - shape=[(1 + slide_steps) * (num_thresholds + 1) + 1]) + shape=[(1 + slide_steps) * (num_thresholds + 1) + 1], + ) batch_stat_neg = helper.create_global_variable( persistable=True, dtype='int64', - shape=[(1 + slide_steps) * (num_thresholds + 1) + 1]) + shape=[(1 + slide_steps) * (num_thresholds + 1) + 1], + ) # for global auc # Needn't maintain the batch id - stat_pos = helper.create_global_variable(persistable=True, - dtype='int64', - shape=[1, num_thresholds + 1]) - stat_neg = helper.create_global_variable(persistable=True, - dtype='int64', - shape=[1, num_thresholds + 1]) + stat_pos = helper.create_global_variable( + persistable=True, dtype='int64', shape=[1, num_thresholds + 1] + ) + stat_neg = helper.create_global_variable( + persistable=True, dtype='int64', shape=[1, num_thresholds + 1] + ) for var in [batch_stat_pos, batch_stat_neg, stat_pos, stat_neg]: - helper.set_variable_initializer(var, Constant(value=0.0, - force_cpu=False)) + helper.set_variable_initializer( + var, Constant(value=0.0, force_cpu=False) + ) - #"InsTagWeight": [ins_tag_weight] + # "InsTagWeight": [ins_tag_weight] # Batch AUC - helper.append_op(type="auc", - inputs={ - "Predict": [input], - "Label": [label], - "StatPos": [batch_stat_pos], - "StatNeg": [batch_stat_neg] - }, - attrs={ - "curve": curve, - "num_thresholds": num_thresholds, - "slide_steps": slide_steps - }, - outputs={ - "AUC": [batch_auc_out], - "StatPosOut": [batch_stat_pos], - "StatNegOut": [batch_stat_neg] - }) + helper.append_op( + type="auc", + inputs={ + "Predict": [input], + "Label": [label], + "StatPos": [batch_stat_pos], + "StatNeg": [batch_stat_neg], + }, + attrs={ + "curve": curve, + "num_thresholds": num_thresholds, + "slide_steps": slide_steps, + }, + outputs={ + "AUC": [batch_auc_out], + "StatPosOut": [batch_stat_pos], + "StatNegOut": [batch_stat_neg], + }, + ) # Global AUC - helper.append_op(type="auc", - inputs={ - "Predict": [input], - "Label": [label], - "StatPos": [stat_pos], - "StatNeg": [stat_neg] - }, - attrs={ - "curve": curve, - "num_thresholds": num_thresholds, - "slide_steps": 0 - }, - outputs={ - "AUC": [auc_out], - "StatPosOut": [stat_pos], - "StatNegOut": [stat_neg] - }) - return auc_out, batch_auc_out, [ - batch_stat_pos, batch_stat_neg, stat_pos, stat_neg - ] + helper.append_op( + type="auc", + inputs={ + "Predict": [input], + "Label": [label], + "StatPos": [stat_pos], + "StatNeg": [stat_neg], + }, + attrs={ + "curve": curve, + "num_thresholds": num_thresholds, + "slide_steps": 0, + }, + outputs={ + "AUC": [auc_out], + "StatPosOut": [stat_pos], + "StatNegOut": [stat_neg], + }, + ) + return ( + auc_out, + batch_auc_out, + [batch_stat_pos, batch_stat_neg, stat_pos, stat_neg], + ) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 38d1b30f0a4..9f7cbb11411 100755 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -25,18 +25,39 @@ import paddle from ..layer_helper import LayerHelper from paddle.fluid.framework import _in_legacy_dygraph from ..initializer import Normal, Constant, NumpyArrayInitializer -from ..framework import Variable, OpProtoHolder, _non_static_mode, dygraph_only, _dygraph_tracer, default_main_program, _varbase_creator, static_only, _global_flags, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Variable, + OpProtoHolder, + _non_static_mode, + dygraph_only, + _dygraph_tracer, + default_main_program, + _varbase_creator, + static_only, + _global_flags, + _in_legacy_dygraph, + in_dygraph_mode, +) from ..framework import _current_expected_place from .. import dygraph_utils from ..param_attr import ParamAttr -from .layer_function_generator import autodoc, templatedoc, _generate_doc_string_ +from .layer_function_generator import ( + autodoc, + templatedoc, + _generate_doc_string_, +) from .tensor import concat, assign, fill_constant, zeros, tensor_array_to_tensor from . import utils from .. import unique_name from functools import reduce from .. import core from ...utils import deprecated -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) import paddle from paddle.utils import deprecated from paddle import _C_ops, _legacy_C_ops @@ -219,8 +240,10 @@ def _get_reduce_dim(dim, input): dim = [dim] else: raise TypeError( - "The type of dim must be int, list, tuple or range, but received {}" - .format(type(axis))) + "The type of dim must be int, list, tuple or range, but received {}".format( + type(axis) + ) + ) if dim is None: dim = [] if dim == [] or len(dim) == len(input.shape): @@ -232,13 +255,9 @@ def _get_reduce_dim(dim, input): @dygraph_only -def _elementwise_op_in_dygraph(x, - y, - axis=-1, - act=None, - use_mkldnn=False, - op_name=None): - +def _elementwise_op_in_dygraph( + x, y, axis=-1, act=None, use_mkldnn=False, op_name=None +): def is_inplace(op_name): return op_name[-1] == "_" @@ -249,24 +268,27 @@ def _elementwise_op_in_dygraph(x, if in_dygraph_mode(): op = getattr( _C_ops, - OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name) + OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name, + ) out = op(x, y) if _in_legacy_dygraph(): op = getattr(_legacy_C_ops, op_name) out = op(x, y, 'axis', axis, 'use_mkldnn', use_mkldnn) - return dygraph_utils._append_activation_in_dygraph(out, - act, - use_mkldnn=use_mkldnn) - - -def fc(input, - size, - num_flatten_dims=1, - param_attr=None, - bias_attr=None, - act=None, - name=None): + return dygraph_utils._append_activation_in_dygraph( + out, act, use_mkldnn=use_mkldnn + ) + + +def fc( + input, + size, + num_flatten_dims=1, + param_attr=None, + bias_attr=None, + act=None, + name=None, +): r""" :api_attr: Static Graph @@ -384,8 +406,9 @@ def fc(input, for i, input_x in enumerate(input): check_type(input_x, 'input[' + str(i) + ']', Variable, 'fc') dtype = helper.input_dtype() - check_dtype(dtype, 'input', ['float16', 'uint16', 'float32', 'float64'], - 'fc') + check_dtype( + dtype, 'input', ['float16', 'uint16', 'float32', 'float64'], 'fc' + ) mul_results = [] for input_var, param_attr in helper.iter_inputs_and_params(): input_shape = input_var.shape @@ -395,31 +418,28 @@ def fc(input, reduce(lambda a, b: a * b, input_shape[num_flatten_dims:], 1) ] + [size] - w = helper.create_parameter(attr=param_attr, - shape=param_shape, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=param_attr, shape=param_shape, dtype=dtype, is_bias=False + ) tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="mul", - inputs={ - "X": input_var, - "Y": w - }, - outputs={"Out": tmp}, - attrs={ - "x_num_col_dims": num_flatten_dims, - "y_num_col_dims": 1 - }) + helper.append_op( + type="mul", + inputs={"X": input_var, "Y": w}, + outputs={"Out": tmp}, + attrs={"x_num_col_dims": num_flatten_dims, "y_num_col_dims": 1}, + ) mul_results.append(tmp) if len(mul_results) == 1: pre_bias = mul_results[0] else: pre_bias = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="sum", - inputs={"X": mul_results}, - outputs={"Out": pre_bias}, - attrs={"use_mkldnn": False}) + helper.append_op( + type="sum", + inputs={"X": mul_results}, + outputs={"Out": pre_bias}, + attrs={"use_mkldnn": False}, + ) # add bias pre_activation = helper.append_bias_op(pre_bias, dim_start=num_flatten_dims) # add activation @@ -427,13 +447,15 @@ def fc(input, @deprecated(since="2.0.0", update_to="paddle.nn.functional.embedding") -def embedding(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): +def embedding( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): r""" :api_attr: Static Graph @@ -546,10 +568,15 @@ def embedding(input, """ helper = LayerHelper('embedding', **locals()) - check_variable_and_dtype(input, 'input', ['int64'], - 'fluid.layers.embedding') - check_dtype(dtype, 'dtype', ['uint16', 'float16', 'float32', 'float64'], - 'fluid.layers.embedding') + check_variable_and_dtype( + input, 'input', ['int64'], 'fluid.layers.embedding' + ) + check_dtype( + dtype, + 'dtype', + ['uint16', 'float16', 'float32', 'float64'], + 'fluid.layers.embedding', + ) if is_distributed: is_distributed = False @@ -559,37 +586,42 @@ def embedding(input, remote_prefetch = True if is_sparse else False - w = helper.create_parameter(attr=helper.param_attr, - shape=size, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False + ) tmp = helper.create_variable_for_type_inference(dtype) - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) - helper.append_op(type='lookup_table', - inputs={ - 'Ids': input, - 'W': w - }, - outputs={'Out': tmp}, - attrs={ - 'is_sparse': is_sparse, - 'is_distributed': is_distributed, - 'remote_prefetch': remote_prefetch, - 'padding_idx': padding_idx - }) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) + helper.append_op( + type='lookup_table', + inputs={'Ids': input, 'W': w}, + outputs={'Out': tmp}, + attrs={ + 'is_sparse': is_sparse, + 'is_distributed': is_distributed, + 'remote_prefetch': remote_prefetch, + 'padding_idx': padding_idx, + }, + ) return tmp -def _pull_sparse(input, - size, - table_id, - accessor_class, - name="embedding", - ctr_label_name="", - padding_id=0, - dtype='float32', - scale_sparse_grad=True): +def _pull_sparse( + input, + size, + table_id, + accessor_class, + name="embedding", + ctr_label_name="", + padding_id=0, + dtype='float32', + scale_sparse_grad=True, +): r""" **Pull Fleet Sparse Layer** @@ -636,35 +668,34 @@ def _pull_sparse(input, 'ScaleSparseGrad': scale_sparse_grad, 'InputNames': input_names, # this is only for compatible with embedding op - 'is_distributed': True + 'is_distributed': True, } # this is only for compatible with embedding op - w, _ = helper.create_or_get_global_variable(name=name, - shape=[size], - dtype=dtype, - is_bias=False, - persistable=True) - helper.append_op(type='pull_sparse', - inputs={ - 'Ids': inputs, - 'W': w - }, - outputs={'Out': outs}, - attrs=attrs) + w, _ = helper.create_or_get_global_variable( + name=name, shape=[size], dtype=dtype, is_bias=False, persistable=True + ) + helper.append_op( + type='pull_sparse', + inputs={'Ids': inputs, 'W': w}, + outputs={'Out': outs}, + attrs=attrs, + ) if len(outs) == 1: return outs[0] return outs -def _pull_sparse_v2(input, - size, - table_id, - accessor_class, - name="embedding", - ctr_label_name="", - padding_id=0, - dtype='float32', - scale_sparse_grad=True): +def _pull_sparse_v2( + input, + size, + table_id, + accessor_class, + name="embedding", + ctr_label_name="", + padding_id=0, + dtype='float32', + scale_sparse_grad=True, +): r""" **Pull Fleet Sparse Layer** @@ -711,31 +742,26 @@ def _pull_sparse_v2(input, 'ScaleSparseGrad': scale_sparse_grad, 'InputNames': input_names, # this is only for compatible with embedding op - 'is_distributed': True + 'is_distributed': True, } # this is only for compatible with embedding op - w, _ = helper.create_or_get_global_variable(name=name, - shape=[size], - dtype=dtype, - is_bias=False, - persistable=True) - helper.append_op(type='pull_sparse_v2', - inputs={ - 'Ids': inputs, - 'W': w - }, - outputs={'Out': outs}, - attrs=attrs) + w, _ = helper.create_or_get_global_variable( + name=name, shape=[size], dtype=dtype, is_bias=False, persistable=True + ) + helper.append_op( + type='pull_sparse_v2', + inputs={'Ids': inputs, 'W': w}, + outputs={'Out': outs}, + attrs=attrs, + ) if len(outs) == 1: return outs[0] return outs -def _pull_gpups_sparse(input, - size, - dtype='float32', - is_distributed=False, - is_sparse=False): +def _pull_gpups_sparse( + input, size, dtype='float32', is_distributed=False, is_sparse=False +): r""" **Pull GpuPS Sparse Layer** @@ -769,39 +795,36 @@ def _pull_gpups_sparse(input, helper = LayerHelper('pull_gpups_sparse', **locals()) if dtype != 'float32': raise ValueError( - "GpuPS only support float type embedding now, and your type is: " + - dtype) + "GpuPS only support float type embedding now, and your type is: " + + dtype + ) helper.input_dtype() inputs = helper.multiple_input() outs = [ helper.create_variable_for_type_inference(dtype) for i in range(len(inputs)) ] - w = helper.create_parameter(attr=helper.param_attr, - shape=[size[0]], - dtype=dtype, - is_bias=False) - helper.append_op(type='pull_gpups_sparse', - inputs={ - 'Ids': inputs, - 'W': w - }, - outputs={'Out': outs}, - attrs={ - 'size': size, - 'is_distributed': is_distributed, - 'is_sparse': is_sparse - }) + w = helper.create_parameter( + attr=helper.param_attr, shape=[size[0]], dtype=dtype, is_bias=False + ) + helper.append_op( + type='pull_gpups_sparse', + inputs={'Ids': inputs, 'W': w}, + outputs={'Out': outs}, + attrs={ + 'size': size, + 'is_distributed': is_distributed, + 'is_sparse': is_sparse, + }, + ) if len(outs) == 1: return outs[0] return outs -def _pull_box_sparse(input, - size, - dtype='float32', - is_distributed=False, - is_sparse=False): +def _pull_box_sparse( + input, size, dtype='float32', is_distributed=False, is_sparse=False +): r""" **Pull Box Sparse Layer** @@ -831,29 +854,28 @@ def _pull_box_sparse(input, helper = LayerHelper('pull_box_sparse', **locals()) if dtype != 'float32': raise ValueError( - "BoxPS only support float type embedding now, and your type is: " + - dtype) + "BoxPS only support float type embedding now, and your type is: " + + dtype + ) helper.input_dtype() inputs = helper.multiple_input() outs = [ helper.create_variable_for_type_inference(dtype) for i in range(len(inputs)) ] - w = helper.create_parameter(attr=helper.param_attr, - shape=[size], - dtype=dtype, - is_bias=False) - helper.append_op(type='pull_box_sparse', - inputs={ - 'Ids': inputs, - 'W': w - }, - outputs={'Out': outs}, - attrs={ - 'size': size, - 'is_distributed': is_distributed, - 'is_sparse': is_sparse - }) + w = helper.create_parameter( + attr=helper.param_attr, shape=[size], dtype=dtype, is_bias=False + ) + helper.append_op( + type='pull_box_sparse', + inputs={'Ids': inputs, 'W': w}, + outputs={'Out': outs}, + attrs={ + 'size': size, + 'is_distributed': is_distributed, + 'is_sparse': is_sparse, + }, + ) if len(outs) == 1: return outs[0] return outs @@ -949,37 +971,46 @@ def linear_chain_crf(input, label, param_attr=None, length=None): print(transition) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'linear_chain_crf') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'linear_chain_crf' + ) check_variable_and_dtype(label, 'label', ['int64'], 'linear_chain_crf') helper = LayerHelper('linear_chain_crf', **locals()) size = input.shape[2] if length else input.shape[1] - transition = helper.create_parameter(attr=helper.param_attr, - shape=[size + 2, size], - dtype=helper.input_dtype()) + transition = helper.create_parameter( + attr=helper.param_attr, + shape=[size + 2, size], + dtype=helper.input_dtype(), + ) alpha = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) emission_exps = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) transition_exps = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) log_likelihood = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) this_inputs = { "Emission": [input], "Transition": transition, - "Label": [label] + "Label": [label], } if length: this_inputs['Length'] = [length] - helper.append_op(type='linear_chain_crf', - inputs=this_inputs, - outputs={ - "Alpha": [alpha], - "EmissionExps": [emission_exps], - "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood - }) + helper.append_op( + type='linear_chain_crf', + inputs=this_inputs, + outputs={ + "Alpha": [alpha], + "EmissionExps": [emission_exps], + "TransitionExps": transition_exps, + "LogLikelihood": log_likelihood, + }, + ) return log_likelihood @@ -1035,18 +1066,22 @@ def crf_decoding(input, param_attr, label=None, length=None): crf_decode = paddle.static.nn.crf_decoding(input=emission, length=length, param_attr=paddle.ParamAttr(name="crfw_pad")) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'crf_decoding') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'crf_decoding' + ) helper = LayerHelper('crf_decoding', **locals()) transition = helper.get_parameter(param_attr.name) viterbi_path = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) + dtype=core.VarDesc.VarType.INT64 + ) inputs = {"Emission": [input], "Transition": transition, "Label": label} if length: inputs['Length'] = length - helper.append_op(type='crf_decoding', - inputs=inputs, - outputs={"ViterbiPath": [viterbi_path]}) + helper.append_op( + type='crf_decoding', + inputs=inputs, + outputs={"ViterbiPath": [viterbi_path]}, + ) return viterbi_path @@ -1080,26 +1115,23 @@ def cos_sim(X, Y): out = helper.create_variable_for_type_inference(dtype=X.dtype) xnorm = helper.create_variable_for_type_inference(dtype=X.dtype) ynorm = helper.create_variable_for_type_inference(dtype=X.dtype) - helper.append_op(type='cos_sim', - inputs={ - 'X': [X], - 'Y': [Y] - }, - outputs={ - 'Out': [out], - 'XNorm': [xnorm], - 'YNorm': [ynorm] - }) + helper.append_op( + type='cos_sim', + inputs={'X': [X], 'Y': [Y]}, + outputs={'Out': [out], 'XNorm': [xnorm], 'YNorm': [ynorm]}, + ) return out @deprecated(since="2.0.0", update_to="paddle.nn.functional.dropout") -def dropout(x, - dropout_prob, - is_test=None, - seed=None, - name=None, - dropout_implementation="downgrade_in_infer"): +def dropout( + x, + dropout_prob, + is_test=None, + seed=None, + name=None, + dropout_implementation="downgrade_in_infer", +): """ Computes dropout. @@ -1157,23 +1189,32 @@ def dropout(x, """ if not isinstance(dropout_prob, (float, int, Variable)): raise TypeError( - "dropout_prob argument should be a number(int|float) or Variable") + "dropout_prob argument should be a number(int|float) or Variable" + ) # fast return for p == 0 if isinstance(dropout_prob, (int, float)) and dropout_prob == 0: return x if _non_static_mode(): - if (seed is None - or seed == 0) and default_main_program().random_seed != 0: + if ( + seed is None or seed == 0 + ) and default_main_program().random_seed != 0: seed = default_main_program().random_seed if is_test is None: is_test = not _dygraph_tracer()._train_mode - out, mask = _legacy_C_ops.dropout(x, 'dropout_prob', dropout_prob, - 'is_test', is_test, 'fix_seed', seed - is not None, 'seed', - seed if seed is not None else 0, - 'dropout_implementation', - dropout_implementation) + out, mask = _legacy_C_ops.dropout( + x, + 'dropout_prob', + dropout_prob, + 'is_test', + is_test, + 'fix_seed', + seed is not None, + 'seed', + seed if seed is not None else 0, + 'dropout_implementation', + dropout_implementation, + ) return out def get_attrs(prog, dropout_prob, is_test, seed): @@ -1181,8 +1222,10 @@ def dropout(x, seed = prog.random_seed if isinstance(dropout_prob, Variable) and not dropout_prob.shape != [1]: raise TypeError( - "Required dropout_prob.shape == [1] if type(dropout_prob) is Variable, but received dropout_prob.shape = {}" - .format(dropout_prob.shape)) + "Required dropout_prob.shape == [1] if type(dropout_prob) is Variable, but received dropout_prob.shape = {}".format( + dropout_prob.shape + ) + ) attrs = { 'dropout_prob': dropout_prob, 'is_test': is_test, @@ -1193,32 +1236,35 @@ def dropout(x, return attrs helper = LayerHelper('dropout', **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'dropout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'dropout' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) mask = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) attrs = get_attrs(helper.main_program, dropout_prob, is_test, seed) - helper.append_op(type='dropout', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs=attrs) + helper.append_op( + type='dropout', + inputs={'X': [x]}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs=attrs, + ) return out @templatedoc() -def chunk_eval(input, - label, - chunk_scheme, - num_chunk_types, - excluded_chunk_types=None, - seq_length=None): +def chunk_eval( + input, + label, + chunk_scheme, + num_chunk_types, + excluded_chunk_types=None, + seq_length=None, +): r""" This operator computes the precision, recall and F1-score for chunk detection. It is often used in sequence tagging tasks, such as Named Entity Recognition(NER). @@ -1337,30 +1383,39 @@ def chunk_eval(input, num_infer_chunks = helper.create_variable_for_type_inference(dtype="int64") num_label_chunks = helper.create_variable_for_type_inference(dtype="int64") num_correct_chunks = helper.create_variable_for_type_inference( - dtype="int64") + dtype="int64" + ) this_input = {"Inference": [input], "Label": [label]} if seq_length is not None: this_input["SeqLength"] = [seq_length] - helper.append_op(type="chunk_eval", - inputs=this_input, - outputs={ - "Precision": [precision], - "Recall": [recall], - "F1-Score": [f1_score], - "NumInferChunks": [num_infer_chunks], - "NumLabelChunks": [num_label_chunks], - "NumCorrectChunks": [num_correct_chunks] - }, - attrs={ - "num_chunk_types": num_chunk_types, - "chunk_scheme": chunk_scheme, - "excluded_chunk_types": excluded_chunk_types or [] - }) - return (precision, recall, f1_score, num_infer_chunks, num_label_chunks, - num_correct_chunks) + helper.append_op( + type="chunk_eval", + inputs=this_input, + outputs={ + "Precision": [precision], + "Recall": [recall], + "F1-Score": [f1_score], + "NumInferChunks": [num_infer_chunks], + "NumLabelChunks": [num_label_chunks], + "NumCorrectChunks": [num_correct_chunks], + }, + attrs={ + "num_chunk_types": num_chunk_types, + "chunk_scheme": chunk_scheme, + "excluded_chunk_types": excluded_chunk_types or [], + }, + ) + return ( + precision, + recall, + f1_score, + num_infer_chunks, + num_label_chunks, + num_correct_chunks, + ) @deprecated(since="2.0.0", update_to="paddle.nn.functional.softmax") @@ -1481,38 +1536,44 @@ def softmax(input, use_cudnn=True, name=None, axis=-1): return _C_ops.softmax(input, axis) if _non_static_mode(): - return _legacy_C_ops.softmax(input, 'axis', axis, 'use_cudnn', - use_cudnn) + return _legacy_C_ops.softmax( + input, 'axis', axis, 'use_cudnn', use_cudnn + ) inputs = {"X": [input]} attrs = {"axis": axis, "use_cudnn": use_cudnn} helper = LayerHelper('softmax', **locals()) - check_variable_and_dtype(input, 'input/x', - ['float16', 'float32', 'float64'], 'softmax') + check_variable_and_dtype( + input, 'input/x', ['float16', 'float32', 'float64'], 'softmax' + ) dtype = helper.input_dtype() softmax_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="softmax", - inputs={"X": input}, - outputs={"Out": softmax_out}, - attrs=attrs) + helper.append_op( + type="softmax", + inputs={"X": input}, + outputs={"Out": softmax_out}, + attrs=attrs, + ) return softmax_out -def conv2d(input, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - name=None, - data_format="NCHW"): +def conv2d( + input, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None, + data_format="NCHW", +): r""" :api_attr: Static Graph @@ -1648,27 +1709,34 @@ def conv2d(input, print(conv2d.shape) # [-1, 2, 30, 30] """ - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'conv2d') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'conv2d' + ) if len(input.shape) != 4: - raise ValueError("Input size should be 4, " - "but received {}".format(len(input.shape))) + raise ValueError( + "Input size should be 4, " + "but received {}".format(len(input.shape)) + ) num_channels = input.shape[1] if not isinstance(use_cudnn, bool): - raise ValueError("Attr(use_cudnn) should be True or False. Received " - "Attr(use_cudnn): %s. " % str(use_cudnn)) + raise ValueError( + "Attr(use_cudnn) should be True or False. Received " + "Attr(use_cudnn): %s. " % str(use_cudnn) + ) if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) - channel_last = (data_format == "NHWC") + channel_last = data_format == "NHWC" num_channels = input.shape[3] if channel_last else input.shape[1] if num_channels < 0: raise ValueError( "The channel dimmention of the input(%s) should be defined. " - "Received: %s." % (str(input.shape), str(num_channels))) + "Received: %s." % (str(input.shape), str(num_channels)) + ) assert param_attr is not False, "param_attr should not be False here." if groups is None: @@ -1676,27 +1744,35 @@ def conv2d(input, elif groups <= 0: raise ValueError( "the groups of input must be greater than 0, " - "but received the groups of input is {}".format(groups)) + "but received the groups of input is {}".format(groups) + ) else: if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, input.shape, groups)) + ", the groups is {}".format(num_channels, input.shape, groups) + ) num_filter_channels = num_channels // groups l_type = 'conv2d' - if (num_channels == groups and num_filters % num_channels == 0 - and not use_cudnn): + if ( + num_channels == groups + and num_filters % num_channels == 0 + and not use_cudnn + ): l_type = 'depthwise_conv2d' - if (num_channels == groups and num_filters % num_channels == 0 - and core.is_compiled_with_rocm()): + if ( + num_channels == groups + and num_filters % num_channels == 0 + and core.is_compiled_with_rocm() + ): l_type = 'depthwise_conv2d' # NPU only supports depthwise_conv2d when "input_channel = output_channel = groups" if core.is_compiled_with_npu(): - if (num_channels == groups and num_channels == num_filters): + if num_channels == groups and num_channels == num_filters: l_type = 'depthwise_conv2d' else: l_type = 'conv2d' @@ -1710,7 +1786,6 @@ def conv2d(input, # padding def _update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -1721,14 +1796,16 @@ def conv2d(input, if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:4] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NHWC"): if not (padding[0] == [0, 0] and padding[3] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:3] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 4, 'padding') @@ -1745,8 +1822,9 @@ def conv2d(input, padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." % - str(padding)) + "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." + % str(padding) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0, 0] @@ -1764,39 +1842,47 @@ def conv2d(input, raise ValueError( "Invalid filter number, excepted number is larger than 0, but" " received {}, please check the input shape and " - "filter size.".format(filter_elem_num)) - std = (2.0 / filter_elem_num)**0.5 + "filter size.".format(filter_elem_num) + ) + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) filter_param = helper.create_parameter( attr=helper.param_attr, shape=filter_shape, dtype=dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) pre_bias = helper.create_variable_for_type_inference(dtype) - if (core.is_compiled_with_cuda() and paddle.fluid.get_flags( - "FLAGS_conv2d_disable_cudnn")["FLAGS_conv2d_disable_cudnn"]): + if ( + core.is_compiled_with_cuda() + and paddle.fluid.get_flags("FLAGS_conv2d_disable_cudnn")[ + "FLAGS_conv2d_disable_cudnn" + ] + ): use_cudnn = False - helper.append_op(type=l_type, - inputs={ - 'Input': input, - 'Filter': filter_param, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': stride, - 'paddings': padding, - 'dilations': dilation, - 'groups': groups, - 'use_cudnn': use_cudnn, - 'use_mkldnn': False, - 'fuse_relu_before_depthwise_conv': False, - "padding_algorithm": padding_algorithm, - "data_format": data_format, - }) + helper.append_op( + type=l_type, + inputs={ + 'Input': input, + 'Filter': filter_param, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'use_mkldnn': False, + 'fuse_relu_before_depthwise_conv': False, + "padding_algorithm": padding_algorithm, + "data_format": data_format, + }, + ) if data_format == 'NCHW': pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -1806,19 +1892,21 @@ def conv2d(input, return helper.append_activation(pre_act) -def conv3d(input, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - name=None, - data_format="NCDHW"): +def conv3d( + input, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None, + data_format="NCDHW", +): r""" :api_attr: Static Graph @@ -1961,37 +2049,46 @@ def conv3d(input, dtype = helper.input_dtype() if not isinstance(use_cudnn, bool): - raise ValueError("Attr(use_cudnn) should be True or False. Received " - "Attr(use_cudnn): %s. " % str(use_cudnn)) + raise ValueError( + "Attr(use_cudnn) should be True or False. Received " + "Attr(use_cudnn): %s. " % str(use_cudnn) + ) if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" if len(input.shape) != 5: raise ValueError( - "Input should be 5D tensor, but received input with the shape of {}" - .format(input.shape)) + "Input should be 5D tensor, but received input with the shape of {}".format( + input.shape + ) + ) num_channels = input.shape[4] if channel_last else input.shape[1] if num_channels < 0: raise ValueError( "The channel dimmention of the input(%s) should be defined. " - "Received: %s." % (str(input.shape), str(num_channels))) + "Received: %s." % (str(input.shape), str(num_channels)) + ) if groups is None: num_filter_channels = num_channels elif groups <= 0: raise ValueError( - "the groups of conv3d should be greater than 0. Received groups: {}" - .format(groups)) + "the groups of conv3d should be greater than 0. Received groups: {}".format( + groups + ) + ) else: if num_channels % groups != 0: raise ValueError( "The number of input channels must be divisible by Attr(groups). " - "Received: number of channels(%s), groups(%s)." % - (str(num_channels), str(groups))) + "Received: number of channels(%s), groups(%s)." + % (str(num_channels), str(groups)) + ) num_filter_channels = num_channels // groups filter_size = utils.convert_to_list(filter_size, 3, 'filter_size') @@ -1999,7 +2096,6 @@ def conv3d(input, dilation = utils.convert_to_list(dilation, 3, 'dilation') def _update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -2010,14 +2106,16 @@ def conv3d(input, if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:5] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NDHWC"): if not (padding[0] == [0, 0] and padding[4] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:4] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 6, 'padding') @@ -2037,8 +2135,9 @@ def conv3d(input, padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." % - str(padding)) + "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." + % str(padding) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0, 0, 0] @@ -2052,41 +2151,46 @@ def conv3d(input, filter_shape = [num_filters, num_filter_channels] + filter_size def _get_default_param_initializer(): - filter_elem_num = filter_size[0] * filter_size[1] * filter_size[ - 2] * num_channels + filter_elem_num = ( + filter_size[0] * filter_size[1] * filter_size[2] * num_channels + ) if filter_elem_num <= 0: raise ValueError( "Invalid filter number, excepted number is larger than 0, but" " received {}, please check the input shape and " - "filter size.".format(filter_elem_num)) + "filter size.".format(filter_elem_num) + ) - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) filter_param = helper.create_parameter( attr=helper.param_attr, shape=filter_shape, dtype=dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) pre_bias = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=l_type, - inputs={ - 'Input': input, - 'Filter': filter_param, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': stride, - 'paddings': padding, - 'dilations': dilation, - 'groups': groups, - 'use_cudnn': use_cudnn, - 'use_mkldnn': False, - "padding_algorithm": padding_algorithm, - "data_format": data_format, - }) + helper.append_op( + type=l_type, + inputs={ + 'Input': input, + 'Filter': filter_param, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'use_mkldnn': False, + "padding_algorithm": padding_algorithm, + "data_format": data_format, + }, + ) if data_format == 'NCDHW': pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -2097,17 +2201,19 @@ def conv3d(input, @templatedoc() -def pool2d(input, - pool_size=-1, - pool_type="max", - pool_stride=1, - pool_padding=0, - global_pooling=False, - use_cudnn=True, - ceil_mode=False, - name=None, - exclusive=True, - data_format="NCHW"): +def pool2d( + input, + pool_size=-1, + pool_type="max", + pool_stride=1, + pool_padding=0, + global_pooling=False, + use_cudnn=True, + ceil_mode=False, + name=None, + exclusive=True, + data_format="NCHW", +): """ ${comment} @@ -2218,27 +2324,31 @@ def pool2d(input, if pool_type not in ["max", "avg"]: raise ValueError( "Unknown Attr(pool_type): '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if global_pooling is False and pool_size == -1: raise ValueError( "When Attr(global_pooling) is False, Attr(pool_size) must be passed " - "and be a valid value. Received pool_size: %s." % str(pool_size)) + "and be a valid value. Received pool_size: %s." % str(pool_size) + ) if not isinstance(use_cudnn, bool): - raise TypeError("Attr(use_cudnn) should be True or False. Received " - "Attr(use_cudnn): %s." % str(use_cudnn)) + raise TypeError( + "Attr(use_cudnn) should be True or False. Received " + "Attr(use_cudnn): %s." % str(use_cudnn) + ) if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) pool_size = utils.convert_to_list(pool_size, 2, 'pool_size') pool_stride = utils.convert_to_list(pool_stride, 2, 'pool_stride') def update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -2249,14 +2359,16 @@ def pool2d(input, if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero pool_padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:4] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NHWC"): if not (padding[0] == [0, 0] and padding[3] == [0, 0]): raise ValueError( "Non-zero pool_padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:3] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 4, 'padding') @@ -2274,61 +2386,77 @@ def pool2d(input, if pool_padding not in ["SAME", "VALID"]: raise ValueError( "Unknown Attr(pool_padding): '%s'. It can only be 'SAME' or 'VALID'." - % str(pool_padding)) + % str(pool_padding) + ) if pool_padding == "VALID": padding_algorithm = "VALID" pool_padding = [0, 0] if ceil_mode != False: raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode) must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif pool_padding == "SAME": padding_algorithm = "SAME" pool_padding = [0, 0] pool_padding = update_padding(pool_padding, data_format) if in_dygraph_mode(): - return _C_ops.pool2d(input, pool_size, pool_stride, pool_padding, - ceil_mode, exclusive, data_format, pool_type, - global_pooling, False, padding_algorithm, - use_cudnn) + return _C_ops.pool2d( + input, + pool_size, + pool_stride, + pool_padding, + ceil_mode, + exclusive, + data_format, + pool_type, + global_pooling, + False, + padding_algorithm, + use_cudnn, + ) op_type = 'pool2d' helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype() pool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={"X": input}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "global_pooling": global_pooling, - "strides": pool_stride, - "paddings": pool_padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": use_cudnn, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "global_pooling": global_pooling, + "strides": pool_stride, + "paddings": pool_padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": use_cudnn, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) return pool_out @templatedoc() -def pool3d(input, - pool_size=-1, - pool_type="max", - pool_stride=1, - pool_padding=0, - global_pooling=False, - use_cudnn=True, - ceil_mode=False, - name=None, - exclusive=True, - data_format="NCDHW"): +def pool3d( + input, + pool_size=-1, + pool_type="max", + pool_stride=1, + pool_padding=0, + global_pooling=False, + use_cudnn=True, + ceil_mode=False, + name=None, + exclusive=True, + data_format="NCDHW", +): """ ${comment} @@ -2445,28 +2573,32 @@ def pool3d(input, if pool_type not in ["max", "avg"]: raise ValueError( "Unknown Attr(pool_type): '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if global_pooling is False and pool_size == -1: raise ValueError( "When Attr(global_pooling) is False, Attr(pool_size) must be passed " - "and be a valid value. Received Attr(pool_size): %s." % - str(pool_size)) + "and be a valid value. Received Attr(pool_size): %s." + % str(pool_size) + ) if not isinstance(use_cudnn, bool): - raise TypeError("Attr(use_cudnn) should be True or False. Received " - "Attr(use_cudnn): %s. " % str(use_cudnn)) + raise TypeError( + "Attr(use_cudnn) should be True or False. Received " + "Attr(use_cudnn): %s. " % str(use_cudnn) + ) if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s" % str(data_format)) + "Attr(data_format): %s" % str(data_format) + ) pool_size = utils.convert_to_list(pool_size, 3, 'pool_size') pool_stride = utils.convert_to_list(pool_stride, 3, 'pool_stride') def update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, (list, tuple)): return True @@ -2477,14 +2609,16 @@ def pool3d(input, if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero pool_padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:5] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NDHWC"): if not (padding[0] == [0, 0] and padding[4] == [0, 0]): raise ValueError( "Non-zero pool_padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:4] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 6, 'padding') @@ -2506,14 +2640,16 @@ def pool3d(input, if pool_padding not in ["SAME", "VALID"]: raise ValueError( "Unknown Attr(pool_padding): '%s'. It can only be 'SAME' or 'VALID'." - % str(pool_padding)) + % str(pool_padding) + ) if pool_padding == "VALID": padding_algorithm = "VALID" pool_padding = [0, 0, 0] if ceil_mode != False: raise ValueError( "When Attr(pool_padding) is \"VALID\", ceil_mode must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif pool_padding == "SAME": padding_algorithm = "SAME" pool_padding = [0, 0, 0] @@ -2525,33 +2661,33 @@ def pool3d(input, dtype = helper.input_dtype() pool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={"X": input}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "global_pooling": global_pooling, - "strides": pool_stride, - "paddings": pool_padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": use_cudnn, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "global_pooling": global_pooling, + "strides": pool_stride, + "paddings": pool_padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": use_cudnn, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) return pool_out @deprecated(since="2.0.0") @templatedoc(op_type="pool2d") -def adaptive_pool2d(input, - pool_size, - pool_type="max", - require_index=False, - name=None): +def adaptive_pool2d( + input, pool_size, pool_type="max", require_index=False, name=None +): r""" This operation calculates the output based on the input, pool_size, @@ -2648,19 +2784,24 @@ def adaptive_pool2d(input, pool_type='max') """ check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'adaptive_pool2d') + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'adaptive_pool2d', + ) check_type(pool_type, 'pool_type', str, 'adaptive_pool2d') check_type(pool_size, 'pool_size', (int, list, tuple), 'adaptive_pool2d') check_type(require_index, 'require_index', bool, 'adaptive_pool2d') if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if pool_type == "avg" and require_index: raise ValueError( - "invalid setting 'require_index' true when 'pool_type' is 'avg'.") + "invalid setting 'require_index' true when 'pool_type' is 'avg'." + ) pool_size = utils.convert_to_list(pool_size, 2, 'pool_size') @@ -2678,25 +2819,25 @@ def adaptive_pool2d(input, mask = helper.create_variable_for_type_inference(dtype) outputs["Mask"] = mask - helper.append_op(type=l_type, - inputs={"X": input}, - outputs=outputs, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": input}, + outputs=outputs, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "adaptive": True, + }, + ) return (pool_out, mask) if require_index else pool_out @deprecated(since="2.0.0") @templatedoc(op_type="pool3d") -def adaptive_pool3d(input, - pool_size, - pool_type="max", - require_index=False, - name=None): +def adaptive_pool3d( + input, pool_size, pool_type="max", require_index=False, name=None +): r""" This operation calculates the output based on the input, pool_size, @@ -2807,19 +2948,24 @@ def adaptive_pool3d(input, pool_type='max') """ check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'adaptive_pool3d') + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'adaptive_pool3d', + ) check_type(pool_type, 'pool_type', str, 'adaptive_pool3d') check_type(pool_size, 'pool_size', (int, list, tuple), 'adaptive_pool3d') check_type(require_index, 'require_index', bool, 'adaptive_pool3d') if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if pool_type == "avg" and require_index: raise ValueError( - "invalid setting 'require_index' true when 'pool_type' is 'avg'.") + "invalid setting 'require_index' true when 'pool_type' is 'avg'." + ) pool_size = utils.convert_to_list(pool_size, 3, 'pool_size') @@ -2837,32 +2983,36 @@ def adaptive_pool3d(input, mask = helper.create_variable_for_type_inference(dtype) outputs["Mask"] = mask - helper.append_op(type=l_type, - inputs={"X": input}, - outputs=outputs, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": input}, + outputs=outputs, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "adaptive": True, + }, + ) return (pool_out, mask) if require_index else pool_out -def batch_norm(input, - act=None, - is_test=False, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - data_layout='NCHW', - in_place=False, - name=None, - moving_mean_name=None, - moving_variance_name=None, - do_model_average_for_mean_and_var=True, - use_global_stats=False): +def batch_norm( + input, + act=None, + is_test=False, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + data_layout='NCHW', + in_place=False, + name=None, + moving_mean_name=None, + moving_variance_name=None, + do_model_average_for_mean_and_var=True, + use_global_stats=False, +): r""" :api_attr: Static Graph @@ -2977,11 +3127,14 @@ def batch_norm(input, print(hidden2.shape) # [3, 200] """ - assert bias_attr is not False, "bias_attr should not be False in batch_norm." + assert ( + bias_attr is not False + ), "bias_attr should not be False in batch_norm." helper = LayerHelper('batch_norm', **locals()) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'batch_norm') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'batch_norm' + ) dtype = helper.input_dtype() # use fp32 for bn parameter @@ -3000,31 +3153,38 @@ def batch_norm(input, param_shape = [channel_num] # create parameter - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True) - - mean = helper.create_parameter(attr=ParamAttr( - name=moving_mean_name, - initializer=Constant(0.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=dtype) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True + ) + + mean = helper.create_parameter( + attr=ParamAttr( + name=moving_mean_name, + initializer=Constant(0.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=dtype, + ) mean.stop_gradient = True - variance = helper.create_parameter(attr=ParamAttr( - name=moving_variance_name, - initializer=Constant(1.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=dtype) + variance = helper.create_parameter( + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=dtype, + ) variance.stop_gradient = True # create output @@ -3044,38 +3204,81 @@ def batch_norm(input, attrs_ = () if attrs_has_momentum: - attrs_ = ('momentum', momentum, 'epsilon', epsilon, 'is_test', - is_test, 'data_layout', data_layout, 'use_mkldnn', False, - 'fuse_with_relu', False, 'use_global_stats', - use_global_stats) + attrs_ = ( + 'momentum', + momentum, + 'epsilon', + epsilon, + 'is_test', + is_test, + 'data_layout', + data_layout, + 'use_mkldnn', + False, + 'fuse_with_relu', + False, + 'use_global_stats', + use_global_stats, + ) else: - attrs_ = ('epsilon', epsilon, 'is_test', is_test, 'data_layout', - data_layout, 'use_mkldnn', False, 'fuse_with_relu', False, - 'use_global_stats', use_global_stats) + attrs_ = ( + 'epsilon', + epsilon, + 'is_test', + is_test, + 'data_layout', + data_layout, + 'use_mkldnn', + False, + 'fuse_with_relu', + False, + 'use_global_stats', + use_global_stats, + ) if inputs_has_MomemtumTensor: batch_norm_out, _, _, _, _, _ = _legacy_C_ops.batch_norm( - input, scale, bias, mean, variance, momentum, mean_out, - variance_out, *attrs_) + input, + scale, + bias, + mean, + variance, + momentum, + mean_out, + variance_out, + *attrs_, + ) else: batch_norm_out, _, _, _, _, _ = _legacy_C_ops.batch_norm( - input, scale, bias, mean, variance, None, mean_out, - variance_out, *attrs_) + input, + scale, + bias, + mean, + variance, + None, + mean_out, + variance_out, + *attrs_, + ) - return dygraph_utils._append_activation_in_dygraph(batch_norm_out, - act=act, - use_mkldnn=False) + return dygraph_utils._append_activation_in_dygraph( + batch_norm_out, act=act, use_mkldnn=False + ) - saved_mean = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) reserve_space = None if not is_test: reserve_space = helper.create_variable_for_type_inference( - dtype=helper.input_dtype(), stop_gradient=True) + dtype=helper.input_dtype(), stop_gradient=True + ) - batch_norm_out = input if in_place else \ - helper.create_variable_for_type_inference(dtype) + batch_norm_out = ( + input if in_place else helper.create_variable_for_type_inference(dtype) + ) inputs = { "X": input, @@ -3084,7 +3287,7 @@ def batch_norm(input, "Mean": mean, "Variance": variance, "MeanOut": mean_out, - "VarianceOut": variance_out + "VarianceOut": variance_out, } attrs = { "epsilon": epsilon, @@ -3092,7 +3295,7 @@ def batch_norm(input, "data_layout": data_layout, "use_mkldnn": False, "fuse_with_relu": False, - "use_global_stats": use_global_stats + "use_global_stats": use_global_stats, } if isinstance(momentum, Variable): inputs['MomemtumTensor'] = momentum @@ -3104,33 +3307,34 @@ def batch_norm(input, "MeanOut": mean_out, "VarianceOut": variance_out, "SavedMean": saved_mean, - "SavedVariance": saved_variance + "SavedVariance": saved_variance, } if reserve_space is not None: outputs["ReserveSpace"] = reserve_space - helper.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return helper.append_activation(batch_norm_out) -def inplace_abn(input, - act=None, - is_test=False, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - data_layout='NCHW', - name=None, - moving_mean_name=None, - moving_variance_name=None, - do_model_average_for_mean_and_var=True, - use_global_stats=False, - act_alpha=1.0): +def inplace_abn( + input, + act=None, + is_test=False, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + data_layout='NCHW', + name=None, + moving_mean_name=None, + moving_variance_name=None, + do_model_average_for_mean_and_var=True, + use_global_stats=False, + act_alpha=1.0, +): r""" **In-place Activation Batch Normalization Layer** @@ -3164,14 +3368,14 @@ def inplace_abn(input, numerical stability. Default is 1e-5. param_attr(ParamAttr|None): The parameter attribute for Parameter `scale` of inplace_abn. If it is set to None or one attribute of ParamAttr, inplace_abn - will create ParamAttr as param_attr, the name of scale can be set in ParamAttr. - If the Initializer of the param_attr is not set, the parameter is initialized - with Xavier. Default: None. + will create ParamAttr as param_attr, the name of scale can be set in ParamAttr. + If the Initializer of the param_attr is not set, the parameter is initialized + with Xavier. Default: None. bias_attr(ParamAttr|None): The parameter attribute for the bias of inplace_abn. If it is set to None or one attribute of ParamAttr, inplace_abn - will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. - If the Initializer of the bias_attr is not set, the bias is initialized zero. - Default: None. + will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. + If the Initializer of the bias_attr is not set, the bias is initialized zero. + Default: None. data_layout (str, optional): Specify the data format of the input, and the data format of the output will be consistent with that of the input. An optional string from: `"NCHW"`, `"NHWC"`. The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of: @@ -3209,14 +3413,18 @@ def inplace_abn(input, hidden3 = fluid.layers.inplace_abn(input=hidden2, act='leaky_relu', act_alpha=0.2) """ - assert act in [None, 'identity', 'leaky_relu', 'elu'], \ - "inplace_abn only support act as None, 'identity', " \ + assert act in [None, 'identity', 'leaky_relu', 'elu'], ( + "inplace_abn only support act as None, 'identity', " "'leaky_relu', 'elu' currently" - assert bias_attr is not False, "bias_attr should not be False in inplace_abn." + ) + assert ( + bias_attr is not False + ), "bias_attr should not be False in inplace_abn." helper = LayerHelper('inplace_abn', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'inplace_abn') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'inplace_abn' + ) dtype = helper.input_dtype() input_shape = input.shape @@ -3231,31 +3439,38 @@ def inplace_abn(input, param_shape = [channel_num] # create parameter - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True) - - mean = helper.create_parameter(attr=ParamAttr( - name=moving_mean_name, - initializer=Constant(0.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=dtype) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True + ) + + mean = helper.create_parameter( + attr=ParamAttr( + name=moving_mean_name, + initializer=Constant(0.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=dtype, + ) mean.stop_gradient = True - variance = helper.create_parameter(attr=ParamAttr( - name=moving_variance_name, - initializer=Constant(1.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=dtype) + variance = helper.create_parameter( + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=dtype, + ) variance.stop_gradient = True # create output @@ -3277,39 +3492,88 @@ def inplace_abn(input, attrs__ = () if attrs_has_momentum: - attrs__ = ('momentum', momentum, 'epsilon', epsilon, 'is_test', - is_test, 'data_layout', data_layout, 'use_mkldnn', False, - 'fuse_with_relu', False, 'use_global_stats', - use_global_stats, 'activation', act, 'alpha', act_alpha) + attrs__ = ( + 'momentum', + momentum, + 'epsilon', + epsilon, + 'is_test', + is_test, + 'data_layout', + data_layout, + 'use_mkldnn', + False, + 'fuse_with_relu', + False, + 'use_global_stats', + use_global_stats, + 'activation', + act, + 'alpha', + act_alpha, + ) else: - attrs__ = ('epsilon', epsilon, 'is_test', is_test, 'data_layout', - data_layout, 'use_mkldnn', False, 'fuse_with_relu', - False, 'use_global_stats', use_global_stats, - 'activation', act, 'alpha', act_alpha) + attrs__ = ( + 'epsilon', + epsilon, + 'is_test', + is_test, + 'data_layout', + data_layout, + 'use_mkldnn', + False, + 'fuse_with_relu', + False, + 'use_global_stats', + use_global_stats, + 'activation', + act, + 'alpha', + act_alpha, + ) if inputs_has_MomemtumTensor: batch_norm_out, _, _, _, _, _ = _legacy_C_ops.inplace_abn_( - input, scale, bias, mean, variance, momentum, mean_out, - variance_out, *attrs__) + input, + scale, + bias, + mean, + variance, + momentum, + mean_out, + variance_out, + *attrs__, + ) return batch_norm_out else: batch_norm_out, _, _, _, _, _ = _legacy_C_ops.inplace_abn_( - input, scale, bias, mean, variance, None, mean_out, - variance_out, *attrs__) + input, + scale, + bias, + mean, + variance, + None, + mean_out, + variance_out, + *attrs__, + ) return batch_norm_out - saved_mean = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) reserve_space = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) inputs = { "X": input, "Scale": scale, "Bias": bias, "Mean": mean, - "Variance": variance + "Variance": variance, } attrs = { "epsilon": epsilon, @@ -3330,24 +3594,21 @@ def inplace_abn(input, "MeanOut": mean_out, "VarianceOut": variance_out, "SavedMean": saved_mean, - "SavedVariance": saved_variance + "SavedVariance": saved_variance, } if reserve_space is not None: outputs["ReserveSpace"] = reserve_space - helper.append_op(type="inplace_abn", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="inplace_abn", inputs=inputs, outputs=outputs, attrs=attrs + ) return batch_norm_out -def instance_norm(input, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - name=None): +def instance_norm( + input, epsilon=1e-05, param_attr=None, bias_attr=None, name=None +): r""" :api_attr: Static Graph @@ -3411,10 +3672,13 @@ def instance_norm(input, hidden1 = paddle.static.nn.fc(x, size=200) hidden2 = paddle.static.nn.instance_norm(hidden1) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'instance_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'instance_norm' + ) if param_attr is False: - assert bias_attr is False, "param_attr and bias_attr must be set to Fasle at the same time in instance_norm" + assert ( + bias_attr is False + ), "param_attr and bias_attr must be set to Fasle at the same time in instance_norm" helper = LayerHelper('instance_norm', **locals()) dtype = helper.input_dtype() @@ -3426,29 +3690,37 @@ def instance_norm(input, input_shape = input.shape if len(input.shape) < 2 or len(input.shape) > 5: raise ValueError( - 'expected 2D or 3D or 4D or 5D input (got {}D input, input shape is: {})' - .format(len(input.shape), input_shape)) + 'expected 2D or 3D or 4D or 5D input (got {}D input, input shape is: {})'.format( + len(input.shape), input_shape + ) + ) channel_num = input_shape[1] param_shape = [channel_num] if param_attr != False and bias_attr != False: # create parameter - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True, - default_initializer=Constant(0.0)) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) + bias = helper.create_parameter( + attr=helper.bias_attr, + shape=param_shape, + dtype=dtype, + is_bias=True, + default_initializer=Constant(0.0), + ) # create output - saved_mean = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) instance_norm_out = helper.create_variable_for_type_inference(dtype) @@ -3457,35 +3729,39 @@ def instance_norm(input, inputs["Scale"] = scale inputs["Bias"] = bias - helper.append_op(type="instance_norm", - inputs=inputs, - outputs={ - "Y": instance_norm_out, - "SavedMean": saved_mean, - "SavedVariance": saved_variance - }, - attrs={ - "epsilon": epsilon, - }) + helper.append_op( + type="instance_norm", + inputs=inputs, + outputs={ + "Y": instance_norm_out, + "SavedMean": saved_mean, + "SavedVariance": saved_variance, + }, + attrs={ + "epsilon": epsilon, + }, + ) return instance_norm_out @static_only -def data_norm(input, - act=None, - epsilon=1e-05, - param_attr=None, - data_layout='NCHW', - in_place=False, - name=None, - moving_mean_name=None, - moving_variance_name=None, - do_model_average_for_mean_and_var=True, - slot_dim=-1, - sync_stats=False, - summary_decay_rate=0.9999999, - enable_scale_and_shift=False): +def data_norm( + input, + act=None, + epsilon=1e-05, + param_attr=None, + data_layout='NCHW', + in_place=False, + name=None, + moving_mean_name=None, + moving_variance_name=None, + do_model_average_for_mean_and_var=True, + slot_dim=-1, + sync_stats=False, + summary_decay_rate=0.9999999, + enable_scale_and_shift=False, +): r""" :api_attr: Static Graph @@ -3583,39 +3859,54 @@ def data_norm(input, if name == None: name = "dn" if enable_scale_and_shift: - scale_w = helper.create_parameter(attr=ParamAttr( - name=name + '.scale_w', - initializer=Constant(value=float(scale_w_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) - bias = helper.create_parameter(attr=ParamAttr( - name=name + '.bias', - initializer=Constant(value=float(bias_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) + scale_w = helper.create_parameter( + attr=ParamAttr( + name=name + '.scale_w', + initializer=Constant(value=float(scale_w_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) + bias = helper.create_parameter( + attr=ParamAttr( + name=name + '.bias', + initializer=Constant(value=float(bias_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) # create parameter - batch_size = helper.create_parameter(attr=ParamAttr( - name=name + '.batch_size', - initializer=Constant(value=float(batch_size_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) - - batch_sum = helper.create_parameter(attr=ParamAttr( - name=name + '.batch_sum', - initializer=Constant(value=float(batch_sum_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) - - batch_square_sum = helper.create_parameter(attr=ParamAttr( - name=name + '.batch_square_sum', - initializer=Constant(value=float(batch_square_sum_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) + batch_size = helper.create_parameter( + attr=ParamAttr( + name=name + '.batch_size', + initializer=Constant(value=float(batch_size_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) + + batch_sum = helper.create_parameter( + attr=ParamAttr( + name=name + '.batch_sum', + initializer=Constant(value=float(batch_sum_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) + + batch_square_sum = helper.create_parameter( + attr=ParamAttr( + name=name + '.batch_square_sum', + initializer=Constant(value=float(batch_square_sum_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) means = helper.create_variable(dtype=dtype, stop_gradient=True) scales = helper.create_variable(dtype=dtype, stop_gradient=True) @@ -3626,7 +3917,7 @@ def data_norm(input, "X": input, "BatchSize": batch_size, "BatchSum": batch_sum, - "BatchSquareSum": batch_square_sum + "BatchSquareSum": batch_square_sum, } attrs = { "epsilon": epsilon, @@ -3641,31 +3932,35 @@ def data_norm(input, if enable_scale_and_shift: inputs["scale_w"] = scale_w inputs["bias"] = bias - helper.append_op(type="data_norm", - inputs=inputs, - outputs={ - "Y": data_norm_out, - "Means": means, - "Scales": scales, - "BatchSize": batch_size, - "BatchSum": batch_sum, - "BatchSquareSum": batch_square_sum - }, - attrs=attrs) + helper.append_op( + type="data_norm", + inputs=inputs, + outputs={ + "Y": data_norm_out, + "Means": means, + "Scales": scales, + "BatchSize": batch_size, + "BatchSum": batch_sum, + "BatchSquareSum": batch_square_sum, + }, + attrs=attrs, + ) return helper.append_activation(data_norm_out) @templatedoc() -def layer_norm(input, - scale=True, - shift=True, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None, - name=None): +def layer_norm( + input, + scale=True, + shift=True, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + name=None, +): r""" :api_attr: Static Graph @@ -3728,11 +4023,13 @@ def layer_norm(input, output = paddle.static.nn.layer_norm(input=x, begin_norm_axis=1) print(output.shape) # [8, 32, 32] """ - assert _non_static_mode( - ) is not True, "please use LayerNorm instead of layer_norm in dygraph mode!" + assert ( + _non_static_mode() is not True + ), "please use LayerNorm instead of layer_norm in dygraph mode!" helper = LayerHelper('layer_norm', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'layer_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'layer_norm' + ) dtype = helper.input_dtype() # create intput and parameters @@ -3740,57 +4037,65 @@ def layer_norm(input, input_shape = input.shape param_shape = [reduce(lambda x, y: x * y, input_shape[begin_norm_axis:])] if scale: - assert param_attr is not False, "param_attr should not be False when using scale." - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) + assert ( + param_attr is not False + ), "param_attr should not be False when using scale." + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) inputs['Scale'] = scale else: if param_attr: warnings.warn("param_attr is only available with scale is True.") if shift: - assert bias_attr is not False, "bias_attr should not be False when using shift." - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True) + assert ( + bias_attr is not False + ), "bias_attr should not be False when using shift." + bias = helper.create_parameter( + attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True + ) inputs['Bias'] = bias else: if bias_attr: warnings.warn("bias_attr is only available with shift is True.") # create output - mean_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) - variance_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + mean_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) + variance_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) layer_norm_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="layer_norm", - inputs=inputs, - outputs={ - "Y": layer_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis - }) + helper.append_op( + type="layer_norm", + inputs=inputs, + outputs={ + "Y": layer_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}, + ) return helper.append_activation(layer_norm_out) @templatedoc() -def group_norm(input, - groups, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None, - data_layout='NCHW', - name=None): +def group_norm( + input, + groups, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + data_layout='NCHW', + name=None, +): """ :api_attr: Static Graph @@ -3835,8 +4140,9 @@ def group_norm(input, """ helper = LayerHelper('group_norm', **locals()) dtype = helper.input_dtype() - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'group_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'group_norm' + ) # create intput and parameters inputs = {'X': input} input_shape = input.shape @@ -3847,20 +4153,23 @@ def group_norm(input, if data_layout != 'NCHW' and data_layout != 'NHWC': raise ValueError( "Param(data_layout) of Op(fluid.layers.group_norm) got wrong value: received " - + data_layout + " but only NCHW or NHWC supported.") + + data_layout + + " but only NCHW or NHWC supported." + ) channel_num = input_shape[1] if data_layout == 'NCHW' else input_shape[-1] param_shape = [channel_num] if param_attr: - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) inputs['Scale'] = scale if bias_attr: - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True + ) inputs['Bias'] = bias # create output @@ -3868,18 +4177,20 @@ def group_norm(input, variance_out = helper.create_variable(dtype=dtype, stop_gradient=True) group_norm_out = helper.create_variable(dtype=dtype) - helper.append_op(type="group_norm", - inputs=inputs, - outputs={ - "Y": group_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": epsilon, - "groups": groups, - "data_layout": data_layout - }) + helper.append_op( + type="group_norm", + inputs=inputs, + outputs={ + "Y": group_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={ + "epsilon": epsilon, + "groups": groups, + "data_layout": data_layout, + }, + ) return helper.append_activation(group_norm_out) @@ -3948,8 +4259,9 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None): print(x.shape) # [2, 8, 32, 32] """ helper = LayerHelper('spectral_norm', **locals()) - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'spectral_norm') + check_variable_and_dtype( + weight, 'weight', ['float32', 'float64'], 'spectral_norm' + ) check_type(dim, 'dim', int, 'spectral_norm') check_type(power_iters, 'power_iters', int, 'spectral_norm') check_type(eps, 'eps', float, 'spectral_norm') @@ -3958,21 +4270,27 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None): # create intput and parameters input_shape = weight.shape assert weight.numel() > 0, "Any dimension of input cannot be equal to 0." - assert dim < len(input_shape), ("The input `dim` should be less than the " - "rank of `weight`, but received dim=" - "{}".format(dim)) + assert dim < len(input_shape), ( + "The input `dim` should be less than the " + "rank of `weight`, but received dim=" + "{}".format(dim) + ) h = input_shape[dim] w = np.prod(input_shape) // h - u = helper.create_parameter(attr=ParamAttr(), - shape=[h], - dtype=dtype, - default_initializer=Normal(0., 1.)) + u = helper.create_parameter( + attr=ParamAttr(), + shape=[h], + dtype=dtype, + default_initializer=Normal(0.0, 1.0), + ) u.stop_gradient = True - v = helper.create_parameter(attr=ParamAttr(), - shape=[w], - dtype=dtype, - default_initializer=Normal(0., 1.)) + v = helper.create_parameter( + attr=ParamAttr(), + shape=[w], + dtype=dtype, + default_initializer=Normal(0.0, 1.0), + ) v.stop_gradient = True if in_dygraph_mode(): @@ -3985,34 +4303,38 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None): # create output out = helper.create_variable(dtype=dtype) - helper.append_op(type="spectral_norm", - inputs=inputs, - outputs={ - "Out": out, - }, - attrs={ - "dim": dim, - "power_iters": power_iters, - "eps": eps, - }) + helper.append_op( + type="spectral_norm", + inputs=inputs, + outputs={ + "Out": out, + }, + attrs={ + "dim": dim, + "power_iters": power_iters, + "eps": eps, + }, + ) return out -def conv2d_transpose(input, - num_filters, - output_size=None, - filter_size=None, - padding=0, - stride=1, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - name=None, - data_format='NCHW'): +def conv2d_transpose( + input, + num_filters, + output_size=None, + filter_size=None, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None, + data_format='NCHW', +): r""" :api_attr: Static Graph @@ -4168,20 +4490,29 @@ def conv2d_transpose(input, conv2d_transpose = paddle.static.nn.conv2d_transpose(input=data, num_filters=2, filter_size=3) print(conv2d_transpose.shape) # [-1, 2, 34, 34] """ - assert param_attr is not False, "param_attr should not be False in conv2d_transpose." + assert ( + param_attr is not False + ), "param_attr should not be False in conv2d_transpose." if len(input.shape) != 4: - raise ValueError("Input size should be 4, " - "but received {}".format(len(input.shape))) + raise ValueError( + "Input size should be 4, " + "but received {}".format(len(input.shape)) + ) if data_format not in ['NCHW', 'NHWC']: raise ValueError( "Attr(data_format) of Op(fluid.layers.conv2d_transpose) got wrong value: received " - + data_format + " but only NCHW or NHWC supported.") + + data_format + + " but only NCHW or NHWC supported." + ) input_channel = input.shape[1] if data_format == 'NCHW' else input.shape[-1] op_type = 'conv2d_transpose' - if (input_channel == groups and num_filters == input_channel - and not use_cudnn): + if ( + input_channel == groups + and num_filters == input_channel + and not use_cudnn + ): op_type = 'depthwise_conv2d_transpose' helper = LayerHelper(op_type, **locals()) @@ -4195,7 +4526,6 @@ def conv2d_transpose(input, raise ValueError("use_cudnn should be True or False") def _update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -4206,14 +4536,16 @@ def conv2d_transpose(input, if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:4] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NHWC"): if not (padding[0] == [0, 0] and padding[3] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:3] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 4, 'padding') @@ -4227,8 +4559,9 @@ def conv2d_transpose(input, padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." % - str(padding)) + "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." + % str(padding) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0, 0, 0, 0] @@ -4248,44 +4581,63 @@ def conv2d_transpose(input, elif isinstance(output_size, int): output_size = utils.convert_to_list(output_size, 2, 'output_size') elif isinstance(output_size, Variable): - check_dtype(output_size.dtype, 'output_size', ['int32', 'int64'], - 'conv2d_transpose') - if len(output_size.shape) == 1 and (output_size.shape[0] == 1 - or output_size.shape[0] == 2): + check_dtype( + output_size.dtype, + 'output_size', + ['int32', 'int64'], + 'conv2d_transpose', + ) + if len(output_size.shape) == 1 and ( + output_size.shape[0] == 1 or output_size.shape[0] == 2 + ): if output_size.shape[0] == 1: output_size = [output_size, output_size] else: raise ValueError("output_size must contain one or two integers.") else: raise ValueError( - "output_size should be int, list[int] or tuple[int] or Tensor") + "output_size should be int, list[int] or tuple[int] or Tensor" + ) if filter_size is None: if output_size is []: raise ValueError("output_size must be set when filter_size is None") if not _non_static_mode(): - if isinstance(output_size, - Variable) or utils._contain_var(output_size): + if isinstance(output_size, Variable) or utils._contain_var( + output_size + ): raise ValueError( "filter_size should not be None when output_size is Variable or contain Variable in static mode." ) else: output_size = utils.convert_shape_to_list(output_size) if len(output_size) == 1: - output_size = utils.convert_to_list(output_size[0], 2, - 'output_size') + output_size = utils.convert_to_list( + output_size[0], 2, 'output_size' + ) h_in = input.shape[2] if data_format == 'NCHW' else input.shape[1] w_in = input.shape[3] if data_format == 'NCHW' else input.shape[2] - filter_size_h = (output_size[0] - (h_in - 1) * stride[0] + padding[0] + - padding[1] - 1) // dilation[0] + 1 - filter_size_w = (output_size[1] - (w_in - 1) * stride[1] + padding[2] + - padding[3] - 1) // dilation[1] + 1 + filter_size_h = ( + output_size[0] + - (h_in - 1) * stride[0] + + padding[0] + + padding[1] + - 1 + ) // dilation[0] + 1 + filter_size_w = ( + output_size[1] + - (w_in - 1) * stride[1] + + padding[2] + + padding[3] + - 1 + ) // dilation[1] + 1 filter_size = [filter_size_h, filter_size_w] else: - filter_size = utils.convert_to_list(filter_size, 2, - 'conv2d_transpose.filter_size') + filter_size = utils.convert_to_list( + filter_size, 2, 'conv2d_transpose.filter_size' + ) if len(padding) == 4 and utils._is_symmetric_padding(padding, 2): padding = [padding[0], padding[2]] @@ -4295,31 +4647,31 @@ def conv2d_transpose(input, elif groups <= 0: raise ValueError( "the groups of input must be greater than 0, " - "but received the groups of input is {}".format(groups)) + "but received the groups of input is {}".format(groups) + ) filter_shape = [input_channel, num_filters // groups] + filter_size - img_filter = helper.create_parameter(dtype=input.dtype, - shape=filter_shape, - attr=helper.param_attr) + img_filter = helper.create_parameter( + dtype=input.dtype, shape=filter_shape, attr=helper.param_attr + ) pre_bias = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type=op_type, - inputs={ - 'Input': [input], - 'Filter': [img_filter] - }, - outputs={'Output': pre_bias}, - attrs={ - 'output_size': output_size, - 'strides': stride, - 'paddings': padding, - 'padding_algorithm': padding_algorithm, - 'dilations': dilation, - 'groups': groups, - 'use_cudnn': use_cudnn, - 'data_format': data_format - }) + helper.append_op( + type=op_type, + inputs={'Input': [input], 'Filter': [img_filter]}, + outputs={'Output': pre_bias}, + attrs={ + 'output_size': output_size, + 'strides': stride, + 'paddings': padding, + 'padding_algorithm': padding_algorithm, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'data_format': data_format, + }, + ) if data_format == 'NCHW': pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -4329,20 +4681,22 @@ def conv2d_transpose(input, return out -def conv3d_transpose(input, - num_filters, - output_size=None, - filter_size=None, - padding=0, - stride=1, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - name=None, - data_format='NCDHW'): +def conv3d_transpose( + input, + num_filters, + output_size=None, + filter_size=None, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None, + data_format='NCDHW', +): r""" :api_attr: Static Graph @@ -4506,11 +4860,15 @@ def conv3d_transpose(input, output = exe.run(feed={"data": x}, fetch_list=[res]) print(output) """ - assert param_attr is not False, "param_attr should not be False in conv3d_transpose." + assert ( + param_attr is not False + ), "param_attr should not be False in conv3d_transpose." if data_format not in ['NCDHW', 'NDHWC']: raise ValueError( "Param(data_format) of Op(fluid.layers.conv3d_transpose) got wrong value: received " - + data_format + " but only NCDHW or NDHWC supported.") + + data_format + + " but only NCDHW or NDHWC supported." + ) l_type = "conv3d_transpose" helper = LayerHelper(l_type, **locals()) @@ -4518,9 +4876,13 @@ def conv3d_transpose(input, raise TypeError("Input of conv3d_transpose must be Variable") if len(input.shape) != 5: raise ValueError( - "Input should be 5D tensor, but received input with the shape of {}" - .format(input.shape)) - input_channel = input.shape[1] if data_format == 'NCDHW' else input.shape[-1] + "Input should be 5D tensor, but received input with the shape of {}".format( + input.shape + ) + ) + input_channel = ( + input.shape[1] if data_format == 'NCDHW' else input.shape[-1] + ) stride = utils.convert_to_list(stride, 3, 'stride') dilation = utils.convert_to_list(dilation, 3, 'dilation') @@ -4529,7 +4891,6 @@ def conv3d_transpose(input, raise ValueError("use_cudnn should be True or False") def _update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -4540,14 +4901,16 @@ def conv3d_transpose(input, if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:5] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NDHWC"): if not (padding[0] == [0, 0] and padding[4] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:4] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 6, 'padding') @@ -4558,8 +4921,12 @@ def conv3d_transpose(input, else: padding = utils.convert_to_list(padding, 3, 'padding') padding = [ - padding[0], padding[0], padding[1], padding[1], padding[2], - padding[2] + padding[0], + padding[0], + padding[1], + padding[1], + padding[2], + padding[2], ] return padding @@ -4568,8 +4935,9 @@ def conv3d_transpose(input, padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." % - str(padding)) + "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." + % str(padding) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0, 0, 0, 0, 0, 0] @@ -4589,16 +4957,32 @@ def conv3d_transpose(input, h_in = input.shape[3] if data_format == 'NCDHW' else input.shape[2] w_in = input.shape[4] if data_format == 'NCDHW' else input.shape[3] - filter_size_d = (output_size[0] - (d_in - 1) * stride[0] + padding[0] + - padding[1] - 1) // dilation[0] + 1 - filter_size_h = (output_size[1] - (h_in - 1) * stride[1] + padding[2] + - padding[3] - 1) // dilation[1] + 1 - filter_size_w = (output_size[2] - (w_in - 1) * stride[2] + padding[4] + - padding[5] - 1) // dilation[2] + 1 + filter_size_d = ( + output_size[0] + - (d_in - 1) * stride[0] + + padding[0] + + padding[1] + - 1 + ) // dilation[0] + 1 + filter_size_h = ( + output_size[1] + - (h_in - 1) * stride[1] + + padding[2] + + padding[3] + - 1 + ) // dilation[1] + 1 + filter_size_w = ( + output_size[2] + - (w_in - 1) * stride[2] + + padding[4] + + padding[5] + - 1 + ) // dilation[2] + 1 filter_size = [filter_size_d, filter_size_h, filter_size_w] else: - filter_size = utils.convert_to_list(filter_size, 3, - 'conv3d_transpose.filter_size') + filter_size = utils.convert_to_list( + filter_size, 3, 'conv3d_transpose.filter_size' + ) if len(padding) == 6 and utils._is_symmetric_padding(padding, 3): padding = [padding[0], padding[2], padding[4]] @@ -4613,18 +4997,22 @@ def conv3d_transpose(input, groups = 1 if groups is None else groups if groups <= 0: raise ValueError( - "the groups of conv3d_transpose should be greater than 0. Received groups: {}" - .format(groups)) + "the groups of conv3d_transpose should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_filters % groups != 0: raise ValueError( "Attr(num_filters) must be divisible by groups," "Received: Attr(num_filters) is {}, the groups is {}".format( - num_filters, groups)) + num_filters, groups + ) + ) filter_shape = [input_channel, num_filters // groups] + filter_size - img_filter = helper.create_parameter(dtype=input.dtype, - shape=filter_shape, - attr=helper.param_attr) + img_filter = helper.create_parameter( + dtype=input.dtype, shape=filter_shape, attr=helper.param_attr + ) if data_format == 'NCDHW': data_format = 'NCHW' @@ -4632,22 +5020,21 @@ def conv3d_transpose(input, data_format = 'NHWC' pre_bias = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type=l_type, - inputs={ - 'Input': [input], - 'Filter': [img_filter] - }, - outputs={'Output': pre_bias}, - attrs={ - 'output_size': output_size, - 'strides': stride, - 'paddings': padding, - 'padding_algorithm': padding_algorithm, - 'dilations': dilation, - 'groups': groups, - 'use_cudnn': use_cudnn, - 'data_format': data_format - }) + helper.append_op( + type=l_type, + inputs={'Input': [input], 'Filter': [img_filter]}, + outputs={'Output': pre_bias}, + attrs={ + 'output_size': output_size, + 'strides': stride, + 'paddings': padding, + 'padding_algorithm': padding_algorithm, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'data_format': data_format, + }, + ) if data_format == 'NCHW': pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -4717,18 +5104,24 @@ def reduce_sum(input, dim=None, keep_dim=False, name=None): if in_dygraph_mode(): return _C_ops.sum(input, dim, None, keep_dim) elif _in_legacy_dygraph(): - return _legacy_C_ops.reduce_sum(input, 'dim', dim, 'keep_dim', keep_dim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_sum( + input, 'dim', dim, 'keep_dim', keep_dim, 'reduce_all', reduce_all + ) attrs = {'dim': dim, 'keep_dim': keep_dim, 'reduce_all': reduce_all} check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'reduce_sum') + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'reduce_sum', + ) helper = LayerHelper('reduce_sum', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='reduce_sum', - inputs={'X': input}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_sum', + inputs={'X': input}, + outputs={'Out': out}, + attrs=attrs, + ) return out @@ -4846,18 +5239,18 @@ def reduce_max(input, dim=None, keep_dim=False, name=None): if in_dygraph_mode(): return _C_ops.max(input, dim if dim != None else [], keep_dim) - helper.append_op(type='reduce_max', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_max', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -4918,18 +5311,18 @@ def reduce_min(input, dim=None, keep_dim=False, name=None): if in_dygraph_mode(): return _C_ops.min(input, dim if dim != None else [], keep_dim) - helper.append_op(type='reduce_min', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_min', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -4990,30 +5383,37 @@ def reduce_prod(input, dim=None, keep_dim=False, name=None): dim = [dim] else: raise TypeError( - "The type of axis must be int, list or tuple, but received {}". - format(type(dim))) + "The type of axis must be int, list or tuple, but received {}".format( + type(dim) + ) + ) if in_dygraph_mode(): return _C_ops.reduce_prod( - input, dim if dim != None and dim != [] else [0], keep_dim, True if - dim == None or dim == [] or len(dim) == len(input.shape) else False) + input, + dim if dim != None and dim != [] else [0], + keep_dim, + True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + ) helper = LayerHelper('reduce_prod', **locals()) - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'reduce_prod') + check_variable_and_dtype( + input, 'input', ['float32', 'float64', 'int32', 'int64'], 'reduce_prod' + ) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='reduce_prod', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_prod', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -5070,18 +5470,18 @@ def reduce_all(input, dim=None, keep_dim=False, name=None): check_variable_and_dtype(input, 'input', ('bool'), 'reduce_all') helper = LayerHelper('reduce_all', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='reduce_all', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_all', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -5133,18 +5533,18 @@ def reduce_any(input, dim=None, keep_dim=False, name=None): out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) if dim is not None and not isinstance(dim, list): dim = [dim] - helper.append_op(type='reduce_any', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_any', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -5219,15 +5619,17 @@ def split(input, num_or_sections, dim=-1, name=None): if utils._contain_var(num_or_sections): for index, item in enumerate(num_or_sections): if isinstance(item, Variable): - num_or_sections[index] = num_or_sections[index].numpy( - )[0] + num_or_sections[index] = num_or_sections[index].numpy()[ + 0 + ] attrs += ('sections', list(num_or_sections)) else: attrs += ('sections', list(num_or_sections)) else: raise TypeError( "The type of 'num_or_sections' in split must be int, list or tuple in imperative mode, but " - "received %s." % (type(num_or_sections))) + "received %s." % (type(num_or_sections)) + ) if in_dygraph_mode(): if isinstance(num_or_sections, int): return _C_ops.split_with_num(input, num_or_sections, dim) @@ -5239,8 +5641,11 @@ def split(input, num_or_sections, dim=-1, name=None): return out check_variable_and_dtype( - input, 'input', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], 'split') + input, + 'input', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'split', + ) check_type(num_or_sections, 'num_or_sections', (list, int, tuple), 'split') check_type(dim, 'dim', (int, Variable), 'split') if isinstance(dim, Variable): @@ -5260,19 +5665,18 @@ def split(input, num_or_sections, dim=-1, name=None): dim_size.stop_gradient = True tensor_list.append(dim_size) else: - assert (isinstance(dim_size, int)) + assert isinstance(dim_size, int) if dim_size == -1: assert unk_dim_idx == -1, ( "Only one value of 'num_or_section' in split can " - "be -1. But received num_or_section[%d] is also -1." % - idx) + "be -1. But received num_or_section[%d] is also -1." + % idx + ) unk_dim_idx = idx temp_out = helper.create_variable_for_type_inference('int32') - fill_constant([1], - 'int32', - dim_size, - force_cpu=True, - out=temp_out) + fill_constant( + [1], 'int32', dim_size, force_cpu=True, out=temp_out + ) tensor_list.append(temp_out) return tensor_list @@ -5287,31 +5691,37 @@ def split(input, num_or_sections, dim=-1, name=None): if isinstance(num_or_sections, int): assert num_or_sections > 1, 'num_or_sections must be more than 1.' if isinstance(dim, int) and input_shape[dim] > 0: - assert input_shape[dim] % num_or_sections ==0, \ - "The input's size along the split dimension " \ - "must be evenly divisible by Attr(num_or_sections). " \ - "But %d is not evenly divisible by %d. " % (num_or_sections,input_shape[dim]) + assert input_shape[dim] % num_or_sections == 0, ( + "The input's size along the split dimension " + "must be evenly divisible by Attr(num_or_sections). " + "But %d is not evenly divisible by %d. " + % (num_or_sections, input_shape[dim]) + ) num = num_or_sections else: if isinstance(dim, int) and input_shape[dim] > 0: - assert len(num_or_sections) <= input_shape[ - dim], 'len(num_or_sections) must not be more than input.shape[dim].' + assert ( + len(num_or_sections) <= input_shape[dim] + ), 'len(num_or_sections) must not be more than input.shape[dim].' num = len(num_or_sections) attrs['sections'] = list( - map(lambda ele: -1 - if isinstance(ele, Variable) else ele, num_or_sections)) + map( + lambda ele: -1 if isinstance(ele, Variable) else ele, + num_or_sections, + ) + ) if utils._contain_var(num_or_sections): inputs['SectionsTensorList'] = _get_SectionsTensorList( - num_or_sections) + num_or_sections + ) outs = [ helper.create_variable_for_type_inference(dtype=helper.input_dtype()) for i in range(num) ] - helper.append_op(type='split', - inputs=inputs, - outputs={'Out': outs}, - attrs=attrs) + helper.append_op( + type='split', inputs=inputs, outputs={'Out': outs}, attrs=attrs + ) return outs @@ -5362,8 +5772,9 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): if in_dygraph_mode(): out, _ = _C_ops.norm(x, 1 if axis is None else axis, epsilon, False) elif _in_legacy_dygraph(): - _, out = _legacy_C_ops.norm(x, 'axis', 1 if axis is None else axis, - 'epsilon', epsilon) + _, out = _legacy_C_ops.norm( + x, 'axis', 1 if axis is None else axis, 'epsilon', epsilon + ) return out check_variable_and_dtype(x, "X", ("float16", "float32", "float64"), "norm") @@ -5371,16 +5782,15 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): helper = LayerHelper("l2_normalize", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) norm = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="norm", - inputs={"X": x}, - outputs={ - "Out": out, - "Norm": norm - }, - attrs={ - "axis": 1 if axis is None else axis, - "epsilon": epsilon, - }) + helper.append_op( + type="norm", + inputs={"X": x}, + outputs={"Out": out, "Norm": norm}, + attrs={ + "axis": 1 if axis is None else axis, + "epsilon": epsilon, + }, + ) return out @@ -5461,16 +5871,25 @@ def matmul(x, y, transpose_x=False, transpose_y=False, alpha=1.0, name=None): """ if _non_static_mode(): out = _varbase_creator(dtype=x.dtype) - _legacy_C_ops.matmul(x, y, out, 'transpose_X', transpose_x, - 'transpose_Y', transpose_y, 'alpha', float(alpha)) + _legacy_C_ops.matmul( + x, + y, + out, + 'transpose_X', + transpose_x, + 'transpose_Y', + transpose_y, + 'alpha', + float(alpha), + ) return out def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): - check_variable_and_dtype(val, name, - ['float16', 'float32', 'float64'], - 'matmul') + check_variable_and_dtype( + val, name, ['float16', 'float32', 'float64'], 'matmul' + ) x_shape = list(x.shape) y_shape = list(y.shape) if len(x_shape) == 1: @@ -5484,11 +5903,12 @@ def matmul(x, y, transpose_x=False, transpose_y=False, alpha=1.0, name=None): if transpose_y: y_shape[-2], y_shape[-1] = y_shape[-1], y_shape[-2] if x_shape[-1] != y_shape[-2]: - assert (x_shape[-1] == -1) or (y_shape[-2] == -1), \ - "After performing an optional transpose, Input X's width should be " \ - "equal to Y's width for multiplication " \ - "prerequisites. But received X's shape: %s, Y's shape: %s\n" % \ - (x_shape, y_shape) + assert (x_shape[-1] == -1) or (y_shape[-2] == -1), ( + "After performing an optional transpose, Input X's width should be " + "equal to Y's width for multiplication " + "prerequisites. But received X's shape: %s, Y's shape: %s\n" + % (x_shape, y_shape) + ) if len(y_shape) > 2 and len(x_shape) > 2: for i, dim_x in enumerate(x_shape[:-2]): @@ -5500,7 +5920,8 @@ def matmul(x, y, transpose_x=False, transpose_y=False, alpha=1.0, name=None): "When the matrix is larger than 2 dimensions, the higher " "dimensional values of the two matrices need to be equal. " "But received x_shape[%d] != y_shape[%d]. X's shape: %s, " - "Y's shape: %s.\n" % (i, i, x_shape, y_shape)) + "Y's shape: %s.\n" % (i, i, x_shape, y_shape) + ) attrs = { 'transpose_X': transpose_x, @@ -5512,21 +5933,20 @@ def matmul(x, y, transpose_x=False, transpose_y=False, alpha=1.0, name=None): helper = LayerHelper('matmul', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='matmul', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='matmul', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs=attrs, + ) return out def topk(input, k, name=None): """ :alias_main: paddle.topk - :alias: paddle.topk,paddle.tensor.topk,paddle.tensor.search.topk - :old_api: paddle.fluid.layers.topk + :alias: paddle.topk,paddle.tensor.topk,paddle.tensor.search.topk + :old_api: paddle.fluid.layers.topk This OP is used to find values and indices of the k largest entries for the last dimension. @@ -5611,23 +6031,20 @@ def topk(input, k, name=None): values = helper.create_variable_for_type_inference(dtype=input.dtype) indices = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="top_k", - inputs=inputs, - outputs={ - "Out": [values], - "Indices": [indices] - }, - attrs=attrs) + helper.append_op( + type="top_k", + inputs=inputs, + outputs={"Out": [values], "Indices": [indices]}, + attrs=attrs, + ) values.stop_gradient = True indices.stop_gradient = True return values, indices -def ctc_greedy_decoder(input, - blank, - input_length=None, - padding_value=0, - name=None): +def ctc_greedy_decoder( + input, blank, input_length=None, padding_value=0, name=None +): r""" This op is used to decode sequences by greedy policy by the following steps: @@ -5753,8 +6170,9 @@ def ctc_greedy_decoder(input, input_length=x_pad_len) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'ctc_greedy_decoder') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'ctc_greedy_decoder' + ) helper = LayerHelper("ctc_greedy_decoder", **locals()) _, topk_indices = topk(input, k=1) @@ -5763,32 +6181,27 @@ def ctc_greedy_decoder(input, ctc_out = helper.create_variable_for_type_inference(dtype="int64") if input_length is None: - helper.append_op(type="ctc_align", - inputs={"Input": [topk_indices]}, - outputs={"Output": [ctc_out]}, - attrs={ - "merge_repeated": True, - "blank": blank - }) + helper.append_op( + type="ctc_align", + inputs={"Input": [topk_indices]}, + outputs={"Output": [ctc_out]}, + attrs={"merge_repeated": True, "blank": blank}, + ) return ctc_out else: ctc_out_len = helper.create_variable_for_type_inference(dtype="int64") ctc_input = squeeze(topk_indices, [2]) - helper.append_op(type="ctc_align", - inputs={ - "Input": [ctc_input], - "InputLength": [input_length] - }, - outputs={ - "Output": [ctc_out], - "OutputLength": [ctc_out_len] - }, - attrs={ - "merge_repeated": True, - "blank": blank, - "padding_value": padding_value - }) + helper.append_op( + type="ctc_align", + inputs={"Input": [ctc_input], "InputLength": [input_length]}, + outputs={"Output": [ctc_out], "OutputLength": [ctc_out_len]}, + attrs={ + "merge_repeated": True, + "blank": blank, + "padding_value": padding_value, + }, + ) return ctc_out, ctc_out_len @@ -5849,10 +6262,21 @@ def transpose(x, perm, name=None): out, _ = _legacy_C_ops.transpose2(x, 'axis', perm) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'transpose') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'transpose', + ) check_type(perm, 'perm', (list, tuple), 'transpose') if isinstance(perm, tuple): perm = list(perm) @@ -5861,34 +6285,37 @@ def transpose(x, perm, name=None): "Input(perm) is the permutation of dimensions of Input(x), " "its length should be equal to dimensions of Input(x), " "but received dimension of Input(x) is %s, " - "the length of Input(perm) is %s." % (len(x.shape), len(perm))) + "the length of Input(perm) is %s." % (len(x.shape), len(perm)) + ) for idx, dim in enumerate(perm): if dim >= len(x.shape): raise ValueError( "Each element in Input(perm) should be less than Input(x)'s dimension, " "but %d-th element in Input(perm) is %d which exceeds Input(x)'s " - "dimension %d." % (idx, perm[idx], len(x.shape))) + "dimension %d." % (idx, perm[idx], len(x.shape)) + ) helper = LayerHelper('transpose', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='transpose2', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'XShape': [x_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [x]}, + outputs={'Out': [out], 'XShape': [x_shape]}, + attrs={'axis': perm}, + ) return out -def im2sequence(input, - filter_size=1, - stride=1, - padding=0, - input_image_size=None, - out_stride=1, - name=None): +def im2sequence( + input, + filter_size=1, + stride=1, + padding=0, + input_image_size=None, + out_stride=1, + name=None, +): r""" :api_attr: Static Graph @@ -6002,8 +6429,9 @@ def im2sequence(input, """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." check_variable_and_dtype(input, 'input', ['float32'], 'im2sequence') @@ -6025,10 +6453,9 @@ def im2sequence(input, attrs["out_stride"] = out_stride helper = LayerHelper('im2sequence', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='im2sequence', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='im2sequence', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -6068,16 +6495,15 @@ def row_conv(input, future_context_size, param_attr=None, act=None): check_variable_and_dtype(input, 'input', ['float32'], 'row_conv') dtype = helper.input_dtype() filter_shape = [future_context_size + 1, input.shape[-1]] - filter_param = helper.create_parameter(attr=helper.param_attr, - shape=filter_shape, - dtype=dtype) + filter_param = helper.create_parameter( + attr=helper.param_attr, shape=filter_shape, dtype=dtype + ) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='row_conv', - inputs={ - 'X': [input], - 'Filter': [filter_param] - }, - outputs={'Out': [out]}) + helper.append_op( + type='row_conv', + inputs={'X': [input], 'Filter': [filter_param]}, + outputs={'Out': [out]}, + ) return helper.append_activation(out) @@ -6143,20 +6569,23 @@ def multiplex(inputs, index, name=None): check_type(inputs, 'inputs', (list), 'multiplex') if len(inputs) < 2: raise ValueError( - "inputs should be a list object with at least 2 elements.") + "inputs should be a list object with at least 2 elements." + ) for id, x in enumerate(inputs): - check_variable_and_dtype(x, 'input[' + str(id) + ']', - ['float32', 'float64', 'int32', 'int64'], - 'multiplex') + check_variable_and_dtype( + x, + 'input[' + str(id) + ']', + ['float32', 'float64', 'int32', 'int64'], + 'multiplex', + ) check_variable_and_dtype(index, "index", ['int32', 'int64'], 'multiplex') out = helper.create_variable_for_type_inference(inputs[0].dtype) - helper.append_op(type='multiplex', - inputs={ - 'X': inputs, - 'Ids': index - }, - outputs={'Out': [out]}) + helper.append_op( + type='multiplex', + inputs={'X': inputs, 'Ids': index}, + outputs={'Out': [out]}, + ) return out @@ -6223,18 +6652,17 @@ def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None): diff = helper.create_variable_for_type_inference(dtype=x.dtype) loss = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='smooth_l1_loss', - inputs={ - 'X': x, - 'Y': y, - 'InsideWeight': inside_weight, - 'OutsideWeight': outside_weight - }, - outputs={ - 'Diff': diff, - 'Out': loss - }, - attrs={'sigma': sigma if sigma is not None else 1.0}) + helper.append_op( + type='smooth_l1_loss', + inputs={ + 'X': x, + 'Y': y, + 'InsideWeight': inside_weight, + 'OutsideWeight': outside_weight, + }, + outputs={'Diff': diff, 'Out': loss}, + attrs={'sigma': sigma if sigma is not None else 1.0}, + ) return loss @@ -6326,10 +6754,12 @@ def one_hot(input, depth, allow_out_of_range=False): if isinstance(depth, Variable): depth = depth.numpy() assert depth.shape == ( - 1, ), "depth of type Variable should have shape [1]" + 1, + ), "depth of type Variable should have shape [1]" depth = depth.item(0) - out = _legacy_C_ops.one_hot(input, 'depth', depth, 'allow_out_of_range', - allow_out_of_range) + out = _legacy_C_ops.one_hot( + input, 'depth', depth, 'allow_out_of_range', allow_out_of_range + ) out.stop_gradient = True return out @@ -6346,10 +6776,9 @@ def one_hot(input, depth, allow_out_of_range=False): depth.stop_gradient = True inputs = {'X': input, 'depth_tensor': depth} attrs = {'allow_out_of_range': allow_out_of_range} - helper.append_op(type="one_hot", - inputs=inputs, - attrs=attrs, - outputs={'Out': one_hot_out}) + helper.append_op( + type="one_hot", inputs=inputs, attrs=attrs, outputs={'Out': one_hot_out} + ) one_hot_out.stop_gradient = True return one_hot_out @@ -6387,16 +6816,18 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): dtype='int64', shape=[1], persistable=True, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) if is_new_var: - helper.set_variable_initializer(counter, - initializer=Constant(value=begin - 1, - force_cpu=True)) + helper.set_variable_initializer( + counter, initializer=Constant(value=begin - 1, force_cpu=True) + ) helper.main_program.global_block()._prepend_op( type='increment', inputs={'X': [counter]}, outputs={'Out': [counter]}, - attrs={'step': float(step)}) + attrs={'step': float(step)}, + ) counter.stop_gradient = True return counter @@ -6500,7 +6931,7 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): """ if in_dygraph_mode(): tmp_tensor_type = core.eager.Tensor - #TODO(zhiqiu): enable inplace in dygraph mode. + # TODO(zhiqiu): enable inplace in dygraph mode. if inplace: warnings.warn( "Inplace on reshape is not allowed and will be discarded in dygraph mode currently." @@ -6518,7 +6949,8 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return dygraph_utils._append_activation_in_dygraph(out, act) else: @@ -6540,14 +6972,26 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return dygraph_utils._append_activation_in_dygraph(out, act) - check_variable_and_dtype(x, 'x', [ - 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', 'bool', - 'uint16' - ], 'reshape') + check_variable_and_dtype( + x, + 'x', + [ + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'bool', + 'uint16', + ], + 'reshape', + ) check_type(shape, 'shape', (list, tuple, Variable), 'reshape') check_type(actual_shape, 'actual_shape', (Variable, type(None)), 'reshape') @@ -6571,20 +7015,23 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): "\t# z.shape is [-1, -1, 4]\n\n" " If your target shape in Reshape represents dynamic shape, " "please turn it into a Tensor under @to_static. See above example for details." - % dim_idx) + % dim_idx + ) unk_dim_idx = dim_idx elif dim_size == 0: assert dim_idx < len(x.shape), ( "The index of 0 in `shape` must be less than " "the input tensor X's dimensions. " - "But received shape[%d] = 0, X's dimensions = %d." % - (dim_idx, len(x.shape))) + "But received shape[%d] = 0, X's dimensions = %d." + % (dim_idx, len(x.shape)) + ) else: assert dim_size > 0, ( "Each dimension value of 'shape' in reshape must not " "be negative except one unknown dimension. " - "But received shape[%d] = %s." % - (dim_idx, str(dim_size))) + "But received shape[%d] = %s." + % (dim_idx, str(dim_size)) + ) return attrs_shape inputs = {"X": x} @@ -6593,8 +7040,10 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): shape.stop_gradient = True inputs["Shape"] = shape elif isinstance(shape, (list, tuple)): - assert len(shape) > 0, ("The size of 'shape' in reshape can't be zero, " - "but received %s." % len(shape)) + assert len(shape) > 0, ( + "The size of 'shape' in reshape can't be zero, " + "but received %s." % len(shape) + ) attrs["shape"] = get_attr_shape(shape) if utils._contain_var(shape): inputs['ShapeTensor'] = utils._convert_to_tensor_list(shape) @@ -6602,16 +7051,18 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): actual_shape.stop_gradient = True inputs["Shape"] = actual_shape - out = x if inplace else helper.create_variable_for_type_inference( - dtype=x.dtype) + out = ( + x + if inplace + else helper.create_variable_for_type_inference(dtype=x.dtype) + ) x_shape = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="reshape2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="reshape2", + inputs=inputs, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return helper.append_activation(out) @@ -6676,10 +7127,22 @@ def squeeze(input, axes, name=None): return out helper = LayerHelper("squeeze", **locals()) - check_variable_and_dtype(input, 'input', [ - 'float16', 'float32', 'float64', 'bool', 'int8', 'int32', 'int64', - 'complex64', 'complex128' - ], 'squeeze') + check_variable_and_dtype( + input, + 'input', + [ + 'float16', + 'float32', + 'float64', + 'bool', + 'int8', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'squeeze', + ) check_type(axes, 'axis/axes', (list, tuple, Variable), 'squeeze') attrs = {} @@ -6693,13 +7156,12 @@ def squeeze(input, axes, name=None): attrs["axes"] = axes out = helper.create_variable_for_type_inference(dtype=input.dtype) x_shape = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="squeeze2", - inputs={"X": input}, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="squeeze2", + inputs={"X": input}, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return out @@ -6749,18 +7211,23 @@ def unsqueeze(input, axes, name=None): return _C_ops.unsqueeze(input, axes) check_type(axes, 'axis/axes', (int, list, tuple, Variable), 'unsqueeze') - check_variable_and_dtype(input, 'input', [ - 'float16', - 'float32', - 'float64', - 'bool', - 'int8', - 'int16', - 'int32', - 'int64', - 'complex64', - 'complex128', - ], 'unsqueeze') + check_variable_and_dtype( + input, + 'input', + [ + 'float16', + 'float32', + 'float64', + 'bool', + 'int8', + 'int16', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'unsqueeze', + ) helper = LayerHelper("unsqueeze2", **locals()) inputs = {"X": input} attrs = {} @@ -6778,13 +7245,12 @@ def unsqueeze(input, axes, name=None): out = helper.create_variable_for_type_inference(dtype=input.dtype) x_shape = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="unsqueeze2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="unsqueeze2", + inputs=inputs, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return out @@ -6870,24 +7336,24 @@ def lod_reset(x, y=None, target_lod=None): y = fluid.layers.data(name='y', shape=[10, 20], lod_level=2) out = fluid.layers.lod_reset(x=x, y=y) """ - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'lod_reset') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'lod_reset' + ) helper = LayerHelper("lod_reset", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) if y is not None: check_type(y, 'y', (Variable), 'lod_reset') - #TODO: check y.lod_level = 0 dtype - helper.append_op(type="lod_reset", - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}) + # TODO: check y.lod_level = 0 dtype + helper.append_op( + type="lod_reset", inputs={'X': x, 'Y': y}, outputs={'Out': out} + ) elif target_lod is not None: - helper.append_op(type="lod_reset", - inputs={'X': x}, - attrs={'target_lod': target_lod}, - outputs={'Out': out}) + helper.append_op( + type="lod_reset", + inputs={'X': x}, + attrs={'target_lod': target_lod}, + outputs={'Out': out}, + ) else: raise ValueError("y and target_lod should not be both none.") return out @@ -6941,8 +7407,9 @@ def lod_append(x, level): if (not isinstance(level, Iterable)) and (not isinstance(level, Variable)): raise ValueError("Input(level) must be list, tuple or Variable.") - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'lod_append') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'lod_append' + ) helper = LayerHelper("lod_append", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) @@ -6952,27 +7419,22 @@ def lod_append(x, level): if isinstance(level, Variable): inputs['Y'] = level - #TODO: check y.lod_level = 0 dtype + # TODO: check y.lod_level = 0 dtype else: attrs['target_lod'] = level - helper.append_op(type="lod_reset", - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + helper.append_op( + type="lod_reset", inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out -def lrn(input, - n=5, - k=1.0, - alpha=1e-4, - beta=0.75, - name=None, - data_format='NCHW'): +def lrn( + input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None, data_format='NCHW' +): r""" :alias_main: paddle.nn.functional.lrn - :alias: paddle.nn.functional.lrn,paddle.nn.functional.norm.lrn - :old_api: paddle.fluid.layers.lrn + :alias: paddle.nn.functional.lrn,paddle.nn.functional.norm.lrn + :old_api: paddle.fluid.layers.lrn This operator implements the Local Response Normalization Layer. This layer performs a type of "lateral inhibition" by normalizing over local input regions. @@ -7030,38 +7492,44 @@ def lrn(input, if dims != 4: raise ValueError( - "Input's dimension size of Op(lrn) must be 4, but received %d." % - (dims)) + "Input's dimension size of Op(lrn) must be 4, but received %d." + % (dims) + ) if data_format not in ['NCHW', 'NHWC']: raise ValueError( - "Attr(data_format) of Op(lrn) got wrong value: received " + - data_format + " but only NCHW or NHWC supported.") + "Attr(data_format) of Op(lrn) got wrong value: received " + + data_format + + " but only NCHW or NHWC supported." + ) - mid_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + mid_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) lrn_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="lrn", - inputs={"X": input}, - outputs={ - "Out": lrn_out, - "MidOut": mid_out, - }, - attrs={ - "n": n, - "k": k, - "alpha": alpha, - "beta": beta, - "data_format": data_format - }) + helper.append_op( + type="lrn", + inputs={"X": input}, + outputs={ + "Out": lrn_out, + "MidOut": mid_out, + }, + attrs={ + "n": n, + "k": k, + "alpha": alpha, + "beta": beta, + "data_format": data_format, + }, + ) return lrn_out -def pad(x, paddings, pad_value=0., name=None): +def pad(x, paddings, pad_value=0.0, name=None): r""" :alias_main: paddle.nn.functional.pad - :alias: paddle.nn.functional.pad,paddle.nn.functional.common.pad - :old_api: paddle.fluid.layers.pad + :alias: paddle.nn.functional.pad,paddle.nn.functional.common.pad + :old_api: paddle.fluid.layers.pad This op will pad a tensor with a constant value given by :attr:`pad_value`, and the padded shape is specified by :attr:`paddings`. @@ -7112,10 +7580,20 @@ def pad(x, paddings, pad_value=0., name=None): x = fluid.data(name='data', shape=[300, 300], dtype='float32') out = fluid.layers.pad(x=x, paddings=[0, 1, 1, 2], pad_value=0.) """ - check_variable_and_dtype(x, 'x', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], "pad") + check_variable_and_dtype( + x, + 'x', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + "pad", + ) check_type(pad_value, 'pad_value', (float, int, Variable), 'pad') if isinstance(pad_value, int): @@ -7124,17 +7602,16 @@ def pad(x, paddings, pad_value=0., name=None): helper = LayerHelper('pad', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'paddings': paddings, - 'pad_value': pad_value - }) + helper.append_op( + type='pad', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'paddings': paddings, 'pad_value': pad_value}, + ) return out -def pad_constant_like(x, y, pad_value=0., name=None): +def pad_constant_like(x, y, pad_value=0.0, name=None): r""" Pad :attr:`y` with :attr:`pad_value`, the number of values padded to the edges of each axis is specified by the difference of the shape @@ -7214,31 +7691,29 @@ def pad_constant_like(x, y, pad_value=0., name=None): # out is a rank 4 tensor variable, and out.shape = [2, 3 ,2 , 3] """ check_type(x, 'x', (Variable), 'pad_constant_like') - check_variable_and_dtype(y, 'y', ['float32', 'float64', 'int32', 'int64'], - "pad_constant_like") + check_variable_and_dtype( + y, 'y', ['float32', 'float64', 'int32', 'int64'], "pad_constant_like" + ) helper = LayerHelper('pad_constant_like', **locals()) dtype = helper.input_dtype(input_param_name='y') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad_constant_like', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={'pad_value': float(pad_value)}) + helper.append_op( + type='pad_constant_like', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={'pad_value': float(pad_value)}, + ) return out -def label_smooth(label, - prior_dist=None, - epsilon=0.1, - dtype="float32", - name=None): +def label_smooth( + label, prior_dist=None, epsilon=0.1, dtype="float32", name=None +): r""" :alias_main: paddle.nn.functional.label_smooth - :alias: paddle.nn.functional.label_smooth,paddle.nn.functional.common.label_smooth - :old_api: paddle.fluid.layers.label_smooth + :alias: paddle.nn.functional.label_smooth,paddle.nn.functional.common.label_smooth + :old_api: paddle.fluid.layers.label_smooth Label smoothing is a mechanism to regularize the classifier layer and is called label-smoothing regularization (LSR). @@ -7295,37 +7770,42 @@ def label_smooth(label, if in_dygraph_mode(): return _C_ops.label_smooth(label, prior_dist, float(epsilon)) - if epsilon > 1. or epsilon < 0.: + if epsilon > 1.0 or epsilon < 0.0: raise ValueError("The value of epsilon must be between 0 and 1.") if _non_static_mode(): - return _legacy_C_ops.label_smooth(label, prior_dist, 'epsilon', - float(epsilon)) + return _legacy_C_ops.label_smooth( + label, prior_dist, 'epsilon', float(epsilon) + ) - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'label_smooth') + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'label_smooth' + ) helper = LayerHelper("label_smooth", **locals()) label.stop_gradient = True smooth_label = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="label_smooth", - inputs={ - "X": label, - "PriorDist": prior_dist - } if prior_dist else {"X": label}, - outputs={"Out": smooth_label}, - attrs={"epsilon": float(epsilon)}) + helper.append_op( + type="label_smooth", + inputs={"X": label, "PriorDist": prior_dist} + if prior_dist + else {"X": label}, + outputs={"Out": smooth_label}, + attrs={"epsilon": float(epsilon)}, + ) return smooth_label @templatedoc() -def roi_pool(input, - rois, - pooled_height=1, - pooled_width=1, - spatial_scale=1.0, - rois_num=None, - name=None): +def roi_pool( + input, + rois, + pooled_height=1, + pooled_width=1, + spatial_scale=1.0, + rois_num=None, + name=None, +): """ This operator implements the roi_pooling layer. @@ -7391,10 +7871,20 @@ def roi_pool(input, print(np.array(out).shape) # (2, 1, 1, 1) """ if _non_static_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." pool_out, argmaxes = _legacy_C_ops.roi_pool( - input, rois, rois_num, "pooled_height", pooled_height, - "pooled_width", pooled_width, "spatial_scale", spatial_scale) + input, + rois, + rois_num, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + "spatial_scale", + spatial_scale, + ) return pool_out, argmaxes check_variable_and_dtype(input, 'input', ['float32'], 'roi_pool') @@ -7410,29 +7900,30 @@ def roi_pool(input, } if rois_num is not None: inputs['RoisNum'] = rois_num - helper.append_op(type="roi_pool", - inputs=inputs, - outputs={ - "Out": pool_out, - "Argmax": argmaxes - }, - attrs={ - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "spatial_scale": spatial_scale - }) + helper.append_op( + type="roi_pool", + inputs=inputs, + outputs={"Out": pool_out, "Argmax": argmaxes}, + attrs={ + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "spatial_scale": spatial_scale, + }, + ) return pool_out @templatedoc() -def roi_align(input, - rois, - pooled_height=1, - pooled_width=1, - spatial_scale=1.0, - sampling_ratio=-1, - rois_num=None, - name=None): +def roi_align( + input, + rois, + pooled_height=1, + pooled_width=1, + spatial_scale=1.0, + sampling_ratio=-1, + rois_num=None, + name=None, +): """ ${comment} @@ -7480,21 +7971,41 @@ def roi_align(input, rois_num=rois_num) """ if in_dygraph_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." - return _C_ops.roi_align(input, rois, rois_num, pooled_height, - pooled_width, spatial_scale, sampling_ratio, - False) + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." + return _C_ops.roi_align( + input, + rois, + rois_num, + pooled_height, + pooled_width, + spatial_scale, + sampling_ratio, + False, + ) if _in_legacy_dygraph(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." - align_out = _legacy_C_ops.roi_align(input, rois, rois_num, - "pooled_height", pooled_height, - "pooled_width", pooled_width, - "spatial_scale", spatial_scale, - "sampling_ratio", sampling_ratio) + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." + align_out = _legacy_C_ops.roi_align( + input, + rois, + rois_num, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + "spatial_scale", + spatial_scale, + "sampling_ratio", + sampling_ratio, + ) return align_out - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'roi_align') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'roi_align' + ) check_variable_and_dtype(rois, 'rois', ['float32', 'float64'], 'roi_align') helper = LayerHelper('roi_align', **locals()) dtype = helper.input_dtype() @@ -7505,15 +8016,17 @@ def roi_align(input, } if rois_num is not None: inputs['RoisNum'] = rois_num - helper.append_op(type="roi_align", - inputs=inputs, - outputs={"Out": align_out}, - attrs={ - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "spatial_scale": spatial_scale, - "sampling_ratio": sampling_ratio - }) + helper.append_op( + type="roi_align", + inputs=inputs, + outputs={"Out": align_out}, + attrs={ + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "spatial_scale": spatial_scale, + "sampling_ratio": sampling_ratio, + }, + ) return align_out @@ -7559,21 +8072,22 @@ def dice_loss(input, label, epsilon=0.00001, name=None): predictions = F.softmax(x) loss = F.dice_loss(input=predictions, label=label) """ - return paddle.nn.functional.dice_loss(input, - label, - epsilon=epsilon, - name=name) - - -def image_resize(input, - out_shape=None, - scale=None, - name=None, - resample='BILINEAR', - actual_shape=None, - align_corners=True, - align_mode=1, - data_format='NCHW'): + return paddle.nn.functional.dice_loss( + input, label, epsilon=epsilon, name=name + ) + + +def image_resize( + input, + out_shape=None, + scale=None, + name=None, + resample='BILINEAR', + actual_shape=None, + align_corners=True, + align_mode=1, + data_format='NCHW', +): """ This op resizes a batch of images. @@ -7813,65 +8327,65 @@ def image_resize(input, Examples: .. code-block:: python - #declarative mode - import paddle - import paddle.fluid as fluid - import numpy as np - paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,10]) + #declarative mode + import paddle + import paddle.fluid as fluid + import numpy as np + paddle.enable_static() + input = fluid.data(name="input", shape=[None,3,6,10]) - #1 - output = fluid.layers.image_resize(input=input,out_shape=[12,12]) + #1 + output = fluid.layers.image_resize(input=input,out_shape=[12,12]) - #2 - #x = np.array([2]).astype("int32") - #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") - #fluid.layers.assign(input=x, output=dim1) - #output = fluid.layers.image_resize(input=input,out_shape=[12,dim1]) + #2 + #x = np.array([2]).astype("int32") + #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") + #fluid.layers.assign(input=x, output=dim1) + #output = fluid.layers.image_resize(input=input,out_shape=[12,dim1]) - #3 - #x = np.array([3,12]).astype("int32") - #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") - #fluid.layers.assign(input=x, output=shape_tensor) - #output = fluid.layers.image_resize(input=input,out_shape=shape_tensor) + #3 + #x = np.array([3,12]).astype("int32") + #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") + #fluid.layers.assign(input=x, output=shape_tensor) + #output = fluid.layers.image_resize(input=input,out_shape=shape_tensor) - #4 - #x = np.array([0.5]).astype("float32") - #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") - #fluid.layers.assign(x,scale_tensor) - #output = fluid.layers.image_resize(input=input,scale=scale_tensor) + #4 + #x = np.array([0.5]).astype("float32") + #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") + #fluid.layers.assign(x,scale_tensor) + #output = fluid.layers.image_resize(input=input,scale=scale_tensor) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,3,6,10).astype("float32") + input_data = np.random.rand(2,3,6,10).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - #1 - # (2, 3, 12, 12) - #2 - # (2, 3, 12, 2) - #3 - # (2, 3, 3, 12) - #4 - # (2, 3, 3, 5) + #1 + # (2, 3, 12, 12) + #2 + # (2, 3, 12, 2) + #3 + # (2, 3, 3, 12) + #4 + # (2, 3, 3, 5) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.image_resize(input=input, out_shape=[12,12]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.image_resize(input=input, out_shape=[12,12]) + print(output.shape) - # [2L, 3L, 12L, 12L] + # [2L, 3L, 12L, 12L] """ resample_methods = { @@ -7885,7 +8399,8 @@ def image_resize(input, if resample not in resample_methods: raise ValueError( "The 'resample' of image_resize can only be 'LINEAR', 'BILINEAR', 'TRILINEAR' " - "or 'NEAREST' currently.") + "or 'NEAREST' currently." + ) resample_type = resample_methods[resample] if resample == 'LINEAR' and len(input.shape) != 3: @@ -7907,19 +8422,25 @@ def image_resize(input, if len(input.shape) == 3 and data_format not in ['NCW', 'NWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCW` or `NWC` supported for 3-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCW` or `NWC` supported for 3-D input." + ) elif len(input.shape) == 4 and data_format not in ['NCHW', 'NHWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCHW` or `NHWC` supported for 4-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCHW` or `NHWC` supported for 4-D input." + ) elif len(input.shape) == 5 and data_format not in ['NCDHW', 'NDHWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCDHW` or `NDHWC` supported for 5-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCDHW` or `NDHWC` supported for 5-D input." + ) def _is_list_or_turple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if data_format == 'NCHW' or data_format == 'NCDHW' or data_format == 'NCW': data_layout = 'NCHW' @@ -7934,7 +8455,7 @@ def image_resize(input, "interp_method": resample_type, "align_corners": align_corners, "align_mode": align_mode, - "data_layout": data_layout + "data_layout": data_layout, } if out_shape is not None: @@ -7952,16 +8473,17 @@ def image_resize(input, out_shape[i] = dim.numpy()[0] if not (_is_list_or_turple_(out_shape)): raise TypeError( - "out_shape should be a list or tuple or Variable.") + "out_shape should be a list or tuple or Variable." + ) # Validate the shape contain_var = False for dim_idx, dim_size in enumerate(out_shape): if isinstance(dim_size, Variable): contain_var = True continue - assert dim_size > 0, ( - "Each dimension size given in out_shape must be greater than 0." - ) + assert ( + dim_size > 0 + ), "Each dimension size given in out_shape must be greater than 0." if contain_var: new_size_tensor = [] @@ -7972,22 +8494,22 @@ def image_resize(input, new_size_tensor.append(dim) size_list.append(-1) else: - assert (isinstance(dim, int)) + assert isinstance(dim, int) temp_out = helper.create_variable_for_type_inference( - 'int32') - fill_constant([1], - 'int32', - dim, - force_cpu=True, - out=temp_out) + 'int32' + ) + fill_constant( + [1], 'int32', dim, force_cpu=True, out=temp_out + ) new_size_tensor.append(temp_out) size_list.append(dim) inputs['SizeTensor'] = new_size_tensor if len(input.shape) == 3: if len(out_shape) != 1: - raise ValueError("out_shape length should be 1 for " - "input 3-D tensor.") + raise ValueError( + "out_shape length should be 1 for " "input 3-D tensor." + ) if contain_var: attrs['out_w'] = size_list[0] else: @@ -7995,8 +8517,9 @@ def image_resize(input, attrs['out_w'] = out_shape[0] elif len(input.shape) == 4: if len(out_shape) != 2: - raise ValueError("out_shape length should be 2 for " - "input 4-D tensor.") + raise ValueError( + "out_shape length should be 2 for " "input 4-D tensor." + ) if contain_var: attrs['out_h'] = size_list[0] attrs['out_w'] = size_list[1] @@ -8006,8 +8529,9 @@ def image_resize(input, attrs['out_w'] = out_shape[1] if len(input.shape) == 5: if len(out_shape) != 3: - raise ValueError("out_shape length should be 3 for " - "input 5-D tensor.") + raise ValueError( + "out_shape length should be 3 for " "input 5-D tensor." + ) if contain_var: attrs['out_d'] = size_list[0] attrs['out_h'] = size_list[1] @@ -8030,7 +8554,8 @@ def image_resize(input, attrs['scale'] = float(scale) else: raise TypeError( - "Attr(scale)'s type should be float, int or Variable.") + "Attr(scale)'s type should be float, int or Variable." + ) if isinstance(actual_shape, Variable): warnings.warn( @@ -8062,22 +8587,26 @@ def image_resize(input, return out out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='{}_interp'.format(resample_type), - inputs=inputs, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type='{}_interp'.format(resample_type), + inputs=inputs, + outputs={"Out": out}, + attrs=attrs, + ) return out @templatedoc(op_type="linear_interp") -def resize_linear(input, - out_shape=None, - scale=None, - name=None, - actual_shape=None, - align_corners=True, - align_mode=1, - data_format='NCW'): +def resize_linear( + input, + out_shape=None, + scale=None, + name=None, + actual_shape=None, + align_corners=True, + align_mode=1, + data_format='NCW', +): """ This op resizes the input by performing linear interpolation based on given output shape which specified by actual_shape, out_shape and scale @@ -8153,58 +8682,69 @@ def resize_linear(input, For more information, please refer to :ref:`api_guide_Name` Returns: - Variable: 3-D tensor(NCW or NWC). + Variable: 3-D tensor(NCW or NWC). Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import numpy as np - input = fluid.data(name="input", shape=[None,3,100]) + #declarative mode + import paddle.fluid as fluid + import numpy as np + input = fluid.data(name="input", shape=[None,3,100]) - output = fluid.layers.resize_linear(input=input,out_shape=[50,]) + output = fluid.layers.resize_linear(input=input,out_shape=[50,]) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(1,3,100).astype("float32") + input_data = np.random.rand(1,3,100).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - # (1, 3, 50) + # (1, 3, 50) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.resize_linear(input=input, out_shape=[50,]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.resize_linear(input=input, out_shape=[50,]) + print(output.shape) - # [1L, 3L, 50L] + # [1L, 3L, 50L] """ - return image_resize(input, out_shape, scale, name, 'LINEAR', actual_shape, - align_corners, align_mode, data_format) + return image_resize( + input, + out_shape, + scale, + name, + 'LINEAR', + actual_shape, + align_corners, + align_mode, + data_format, + ) @templatedoc(op_type="bilinear_interp") -def resize_bilinear(input, - out_shape=None, - scale=None, - name=None, - actual_shape=None, - align_corners=True, - align_mode=1, - data_format='NCHW'): +def resize_bilinear( + input, + out_shape=None, + scale=None, + name=None, + actual_shape=None, + align_corners=True, + align_mode=1, + data_format='NCHW', +): """ This op resizes the input by performing bilinear interpolation based on given @@ -8291,86 +8831,97 @@ def resize_bilinear(input, name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` Returns: - Variable: 4-D tensor(NCHW or NHWC). + Variable: 4-D tensor(NCHW or NHWC). Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import numpy as np - import paddle - paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,10]) + #declarative mode + import paddle.fluid as fluid + import numpy as np + import paddle + paddle.enable_static() + input = fluid.data(name="input", shape=[None,3,6,10]) - #1 - output = fluid.layers.resize_bilinear(input=input,out_shape=[12,12]) + #1 + output = fluid.layers.resize_bilinear(input=input,out_shape=[12,12]) - #2 - #x = np.array([2]).astype("int32") - #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") - #fluid.layers.assign(input=x, output=dim1) - #output = fluid.layers.resize_bilinear(input=input,out_shape=[12,dim1]) + #2 + #x = np.array([2]).astype("int32") + #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") + #fluid.layers.assign(input=x, output=dim1) + #output = fluid.layers.resize_bilinear(input=input,out_shape=[12,dim1]) - #3 - #x = np.array([3,12]).astype("int32") - #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") - #fluid.layers.assign(input=x, output=shape_tensor) - #output = fluid.layers.resize_bilinear(input=input,out_shape=shape_tensor) + #3 + #x = np.array([3,12]).astype("int32") + #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") + #fluid.layers.assign(input=x, output=shape_tensor) + #output = fluid.layers.resize_bilinear(input=input,out_shape=shape_tensor) - #4 - #x = np.array([0.5]).astype("float32") - #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") - #fluid.layers.assign(x,scale_tensor) - #output = fluid.layers.resize_bilinear(input=input,scale=scale_tensor) + #4 + #x = np.array([0.5]).astype("float32") + #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") + #fluid.layers.assign(x,scale_tensor) + #output = fluid.layers.resize_bilinear(input=input,scale=scale_tensor) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,3,6,10).astype("float32") + input_data = np.random.rand(2,3,6,10).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - #1 - # (2, 3, 12, 12) - #2 - # (2, 3, 12, 2) - #3 - # (2, 3, 3, 12) - #4 - # (2, 3, 3, 5) + #1 + # (2, 3, 12, 12) + #2 + # (2, 3, 12, 2) + #3 + # (2, 3, 3, 12) + #4 + # (2, 3, 3, 5) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.resize_bilinear(input=input, out_shape=[12,12]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.resize_bilinear(input=input, out_shape=[12,12]) + print(output.shape) - # [2L, 3L, 12L, 12L] + # [2L, 3L, 12L, 12L] """ - return image_resize(input, out_shape, scale, name, 'BILINEAR', actual_shape, - align_corners, align_mode, data_format) + return image_resize( + input, + out_shape, + scale, + name, + 'BILINEAR', + actual_shape, + align_corners, + align_mode, + data_format, + ) @templatedoc(op_type="trilinear_interp") -def resize_trilinear(input, - out_shape=None, - scale=None, - name=None, - actual_shape=None, - align_corners=True, - align_mode=1, - data_format='NCDHW'): +def resize_trilinear( + input, + out_shape=None, + scale=None, + name=None, + actual_shape=None, + align_corners=True, + align_mode=1, + data_format='NCDHW', +): """ This op resizes the input by performing trilinear interpolation based on given @@ -8462,82 +9013,93 @@ def resize_trilinear(input, Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import paddle - import numpy as np - paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,8,10]) + #declarative mode + import paddle.fluid as fluid + import paddle + import numpy as np + paddle.enable_static() + input = fluid.data(name="input", shape=[None,3,6,8,10]) - #1 - output = fluid.layers.resize_trilinear(input=input,out_shape=[12,12,12]) + #1 + output = fluid.layers.resize_trilinear(input=input,out_shape=[12,12,12]) - #2 - #x = np.array([2]).astype("int32") - #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") - #fluid.layers.assign(input=x, output=dim1) - #output = fluid.layers.resize_trilinear(input=input,out_shape=[12,dim1,4]) + #2 + #x = np.array([2]).astype("int32") + #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") + #fluid.layers.assign(input=x, output=dim1) + #output = fluid.layers.resize_trilinear(input=input,out_shape=[12,dim1,4]) - #3 - #x = np.array([3,12,12]).astype("int32") - #shape_tensor = fluid.data(name="shape_tensor", shape=[3], dtype="int32") - #fluid.layers.assign(input=x, output=shape_tensor) - #output = fluid.layers.resize_trilinear(input=input,out_shape=shape_tensor) + #3 + #x = np.array([3,12,12]).astype("int32") + #shape_tensor = fluid.data(name="shape_tensor", shape=[3], dtype="int32") + #fluid.layers.assign(input=x, output=shape_tensor) + #output = fluid.layers.resize_trilinear(input=input,out_shape=shape_tensor) - #4 - #x = np.array([0.5]).astype("float32") - #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") - #fluid.layers.assign(x,scale_tensor) - #output = fluid.layers.resize_trilinear(input=input,scale=scale_tensor) + #4 + #x = np.array([0.5]).astype("float32") + #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") + #fluid.layers.assign(x,scale_tensor) + #output = fluid.layers.resize_trilinear(input=input,scale=scale_tensor) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,3,6,8,10).astype("float32") + input_data = np.random.rand(2,3,6,8,10).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - #1 - # (2, 3, 12, 12, 12) - #2 - # (2, 3, 12, 2, 4) - #3 - # (2, 3, 3, 12, 12) - #4 - # (2, 3, 3, 4, 5) + #1 + # (2, 3, 12, 12, 12) + #2 + # (2, 3, 12, 2, 4) + #3 + # (2, 3, 3, 12, 12) + #4 + # (2, 3, 3, 4, 5) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.resize_trilinear(input=input, out_shape=[12,12,12]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.resize_trilinear(input=input, out_shape=[12,12,12]) + print(output.shape) - # [2L, 3L, 12L, 12L, 12L] + # [2L, 3L, 12L, 12L, 12L] """ - return image_resize(input, out_shape, scale, name, 'TRILINEAR', - actual_shape, align_corners, align_mode, data_format) + return image_resize( + input, + out_shape, + scale, + name, + 'TRILINEAR', + actual_shape, + align_corners, + align_mode, + data_format, + ) @templatedoc(op_type="nearest_interp") -def resize_nearest(input, - out_shape=None, - scale=None, - name=None, - actual_shape=None, - align_corners=True, - data_format='NCHW'): +def resize_nearest( + input, + out_shape=None, + scale=None, + name=None, + actual_shape=None, + align_corners=True, + data_format='NCHW', +): """ This op resizes the input by performing nearest neighbor interpolation in both the @@ -8593,7 +9155,7 @@ def resize_nearest(input, And :attr:`out_shape` has a higher priority than :attr:`scale`. Default: None. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` - actual_shape(Variable): An optional input to specify output shape + actual_shape(Variable): An optional input to specify output shape dynamically. If provided, image resize according to this given shape rather than :attr:`out_shape` and :attr:`scale` specifying @@ -8613,85 +9175,87 @@ def resize_nearest(input, `[batch_size, input_channels, input_height, input_width]`. Returns: - Variable: 4-D tensor(NCHW or NHWC). + Variable: 4-D tensor(NCHW or NHWC). Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import numpy as np - import paddle - paddle.enable_static() + #declarative mode + import paddle.fluid as fluid + import numpy as np + import paddle + paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,10]) + input = fluid.data(name="input", shape=[None,3,6,10]) - #1 - output = fluid.layers.resize_nearest(input=input,out_shape=[12,12]) + #1 + output = fluid.layers.resize_nearest(input=input,out_shape=[12,12]) - #2 - #x = np.array([2]).astype("int32") - #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") - #fluid.layers.assign(input=x, output=dim1) - #output = fluid.layers.resize_nearest(input=input,out_shape=[12,dim1]) + #2 + #x = np.array([2]).astype("int32") + #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") + #fluid.layers.assign(input=x, output=dim1) + #output = fluid.layers.resize_nearest(input=input,out_shape=[12,dim1]) - #3 - #x = np.array([3,12]).astype("int32") - #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") - #fluid.layers.assign(input=x, output=shape_tensor) - #output = fluid.layers.resize_nearest(input=input,out_shape=shape_tensor) + #3 + #x = np.array([3,12]).astype("int32") + #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") + #fluid.layers.assign(input=x, output=shape_tensor) + #output = fluid.layers.resize_nearest(input=input,out_shape=shape_tensor) - #4 - #x = np.array([0.5]).astype("float32") - #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") - #fluid.layers.assign(x,scale_tensor) - #output = fluid.layers.resize_nearest(input=input,scale=scale_tensor) + #4 + #x = np.array([0.5]).astype("float32") + #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") + #fluid.layers.assign(x,scale_tensor) + #output = fluid.layers.resize_nearest(input=input,scale=scale_tensor) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,3,6,10).astype("float32") + input_data = np.random.rand(2,3,6,10).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - #1 - # (2, 3, 12, 12) - #2 - # (2, 3, 12, 2) - #3 - # (2, 3, 3, 12) - #4 - # (2, 3, 3, 5) + #1 + # (2, 3, 12, 12) + #2 + # (2, 3, 12, 2) + #3 + # (2, 3, 3, 12) + #4 + # (2, 3, 3, 5) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.resize_nearest(input=input, out_shape=[12,12]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.resize_nearest(input=input, out_shape=[12,12]) + print(output.shape) - # [2L, 3L, 12L, 12L] + # [2L, 3L, 12L, 12L] """ - return image_resize(input, - out_shape, - scale, - name, - 'NEAREST', - actual_shape, - align_corners, - align_mode=1, - data_format=data_format) + return image_resize( + input, + out_shape, + scale, + name, + 'NEAREST', + actual_shape, + align_corners, + align_mode=1, + data_format=data_format, + ) def image_resize_short(input, out_short_len, resample='BILINEAR'): @@ -8719,15 +9283,18 @@ def image_resize_short(input, out_short_len, resample='BILINEAR'): in_shape = input.shape if len(in_shape) != 4: raise ValueError( - "The rank of input must be 4 (num_batches, channels, in_h, in_w).") + "The rank of input must be 4 (num_batches, channels, in_h, in_w)." + ) hw = in_shape[2:4] short_idx = hw.index(min(hw)) long_idx = 1 - short_idx out_shape = list(hw) out_shape[short_idx] = out_short_len out_shape[long_idx] = int( - float(out_shape[long_idx]) * - (float(out_short_len) / float(hw[short_idx])) + 0.5) + float(out_shape[long_idx]) + * (float(out_short_len) / float(hw[short_idx])) + + 0.5 + ) return image_resize(input=input, out_shape=out_shape, resample=resample) @@ -8766,8 +9333,8 @@ def gather(input, index, overwrite=True): index (Tensor): The index input tensor with rank=1. Data type is int32 or int64. overwrite (bool, optional): The mode that updating the grad when has same index. If True, use the overwrite mode to update the grad of the same index, - if False, use the accumulate mode to update the grad of the same index. - Default value is True. + if False, use the accumulate mode to update the grad of the same index. + Default value is True. Returns: output (Tensor): The output is a tensor with the same rank as input. @@ -8788,19 +9355,21 @@ def gather(input, index, overwrite=True): return _legacy_C_ops.gather(input, index, None, 'overwrite', overwrite) check_variable_and_dtype( - input, 'x', - ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], 'gather') + input, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], + 'gather', + ) check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather') helper = LayerHelper('gather', **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="gather", - inputs={ - "X": input, - "Index": index - }, - outputs={"Out": out}, - attrs={'overwrite': overwrite}) + helper.append_op( + type="gather", + inputs={"X": input, "Index": index}, + outputs={"Out": out}, + attrs={'overwrite': overwrite}, + ) return out @@ -8885,18 +9454,20 @@ def gather_nd(input, index, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.gather_nd(input, index) check_variable_and_dtype( - input, 'input', - ['bool', 'float32', 'float64', 'int16', 'int32', 'int64'], 'gather_np') + input, + 'input', + ['bool', 'float32', 'float64', 'int16', 'int32', 'int64'], + 'gather_np', + ) check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather_np') helper = LayerHelper('gather_nd', **locals()) dtype = helper.input_dtype() output = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="gather_nd", - inputs={ - "X": input, - "Index": index - }, - outputs={"Out": output}) + helper.append_op( + type="gather_nd", + inputs={"X": input, "Index": index}, + outputs={"Out": output}, + ) return output @@ -8904,8 +9475,8 @@ def gather_nd(input, index, name=None): def scatter(input, index, updates, name=None, overwrite=True): """ :alias_main: paddle.scatter - :alias: paddle.scatter,paddle.tensor.scatter,paddle.tensor.manipulation.scatter - :old_api: paddle.fluid.layers.scatter + :alias: paddle.scatter,paddle.tensor.scatter,paddle.tensor.manipulation.scatter + :old_api: paddle.fluid.layers.scatter **Scatter Layer** @@ -8944,8 +9515,8 @@ def scatter(input, index, updates, name=None, overwrite=True): name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` . overwrite (bool): The mode that updating the output when there are same indices. If True, use the overwrite mode to update the output of the same index, - if False, use the accumulate mode to update the output of the same index. - Default value is True. + if False, use the accumulate mode to update the output of the same index. + Default value is True. Returns: Variable(Tensor|LoDTensor): The output is a Tensor with the same shape as input. @@ -8981,14 +9552,12 @@ def scatter(input, index, updates, name=None, overwrite=True): helper = LayerHelper('scatter', **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="scatter", - inputs={ - "X": input, - "Ids": index, - "Updates": updates - }, - attrs={'overwrite': overwrite}, - outputs={"Out": out}) + helper.append_op( + type="scatter", + inputs={"X": input, "Ids": index, "Updates": updates}, + attrs={'overwrite': overwrite}, + outputs={"Out": out}, + ) return out @@ -9073,13 +9642,11 @@ def scatter_nd_add(ref, index, updates, name=None): helper = LayerHelper('scatter_nd_add', **locals()) dtype = helper.input_dtype(input_param_name='ref') output = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="scatter_nd_add", - inputs={ - "X": ref, - "Index": index, - "Updates": updates - }, - outputs={"Out": output}) + helper.append_op( + type="scatter_nd_add", + inputs={"X": ref, "Index": index, "Updates": updates}, + outputs={"Out": output}, + ) return output @@ -9156,9 +9723,9 @@ def random_crop(x, shape, seed=None): """ helper = LayerHelper("random_crop", **locals()) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'uint8', 'int16', 'int32'], - 'random_crop') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'uint8', 'int16', 'int32'], 'random_crop' + ) check_type(shape, 'shape', (list, Variable), 'random_crop') dtype = x.dtype out = helper.create_variable_for_type_inference(dtype) @@ -9170,19 +9737,16 @@ def random_crop(x, shape, seed=None): seed = helper.create_variable( name=unique_name.generate("random_crop_seed"), dtype="int64", - persistable=True) + persistable=True, + ) elif not isinstance(seed, Variable): raise ValueError("'seed' must be a Variable or an int.") - helper.append_op(type="random_crop", - inputs={ - "X": x, - "Seed": seed - }, - outputs={ - "Out": out, - "SeedOut": seed - }, - attrs=op_attrs) + helper.append_op( + type="random_crop", + inputs={"X": x, "Seed": seed}, + outputs={"Out": out, "SeedOut": seed}, + attrs=op_attrs, + ) return out @@ -9253,8 +9817,7 @@ def relu(x, name=None): out1 = fluid.layers.relu(x1) print(out1.numpy()) # [[0. 0. ] - # [1. 2.6]] -""" + # [1. 2.6]]""" if in_dygraph_mode(): return _C_ops.relu(x) @@ -9267,9 +9830,9 @@ def relu(x, name=None): helper = LayerHelper('relu', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="relu", - inputs={"X": helper.input('x')}, - outputs={"Out": out}) + helper.append_op( + type="relu", inputs={"X": helper.input('x')}, outputs={"Out": out} + ) return out @@ -9339,10 +9902,9 @@ def selu(x, scale=None, alpha=None, name=None): if alpha is not None: attrs["alpha"] = alpha - helper.append_op(type="selu", - inputs={"X": x}, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type="selu", inputs={"X": x}, outputs={"Out": out}, attrs=attrs + ) return out @@ -9393,23 +9955,23 @@ def mean_iou(input, label, num_classes): return _legacy_C_ops.mean_iou(input, label, 'num_classes', num_classes) helper = LayerHelper('mean_iou', **locals()) - check_variable_and_dtype(input, 'Predictions', ['int32', 'int64'], - 'mean_iou') + check_variable_and_dtype( + input, 'Predictions', ['int32', 'int64'], 'mean_iou' + ) check_variable_and_dtype(label, 'Labels', ['int32', 'int64'], 'mean_iou') out_mean_iou = helper.create_variable_for_type_inference(dtype='float32') out_wrong = helper.create_variable_for_type_inference(dtype='int32') out_correct = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type="mean_iou", - inputs={ - "Predictions": input, - "Labels": label - }, - outputs={ - "OutMeanIou": out_mean_iou, - "OutWrong": out_wrong, - "OutCorrect": out_correct - }, - attrs={"num_classes": num_classes}) + helper.append_op( + type="mean_iou", + inputs={"Predictions": input, "Labels": label}, + outputs={ + "OutMeanIou": out_mean_iou, + "OutWrong": out_wrong, + "OutCorrect": out_correct, + }, + attrs={"num_classes": num_classes}, + ) return out_mean_iou, out_wrong, out_correct @@ -9506,10 +10068,12 @@ def crop(x, shape=None, offsets=None, name=None): else: attrs['offsets'] = offsets - helper.append_op(type='crop', - inputs=ipts, - outputs={'Out': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='crop', + inputs=ipts, + outputs={'Out': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out @@ -9602,11 +10166,13 @@ def crop_tensor(x, shape=None, offsets=None, name=None): """ helper = LayerHelper('crop_tensor', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'crop_tensor') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'crop_tensor' + ) check_type(shape, 'shape', (list, tuple, Variable), 'crop_tensor') - check_type(offsets, 'offsets', (list, tuple, Variable, type(None)), - 'crop_tensor') + check_type( + offsets, 'offsets', (list, tuple, Variable, type(None)), 'crop_tensor' + ) if offsets is None: offsets = [0] * len(x.shape) @@ -9619,25 +10185,30 @@ def crop_tensor(x, shape=None, offsets=None, name=None): if not isinstance(shape_val, int): raise TypeError( "Attr(shape)'s dtype of Op(crop_tensor) should be int32, but received: %s." - % type(shape_val)) + % type(shape_val) + ) if shape_val == 0: raise ValueError( "Attr(shape) of Op(crop_tensor) should not be zero, but received: %s." - % str(shape_val)) + % str(shape_val) + ) if shape_val < -1: raise ValueError( "When the element in Attr(shape) of Op(crop_tensor) is negative, only -1 is supported, but received: %s." - % str(shape_val)) + % str(shape_val) + ) def _attr_offsets_check(offset_val): if not isinstance(offset_val, int): raise TypeError( "Attr(offsets)'s dtype of Op(crop_tensor) should be int32, but received: %s." - % type(offset_val)) + % type(offset_val) + ) if offset_val < 0: raise ValueError( "Attr(offsets) of Op(crop_tensor) should be greater or equal to zero, but received: %s." - % str(offset_val)) + % str(offset_val) + ) if isinstance(offsets, Variable): offsets.stop_gradient = True @@ -9678,11 +10249,9 @@ def crop_tensor(x, shape=None, offsets=None, name=None): else: _attr_shape_check(dim_size) temp_out = helper.create_variable_for_type_inference('int32') - fill_constant([1], - 'int32', - dim_size, - force_cpu=True, - out=temp_out) + fill_constant( + [1], 'int32', dim_size, force_cpu=True, out=temp_out + ) new_shape_tensor.append(temp_out) shape_attr.append(dim_size) ipts['ShapeTensor'] = new_shape_tensor @@ -9692,18 +10261,20 @@ def crop_tensor(x, shape=None, offsets=None, name=None): _attr_shape_check(dim_size) attrs['shape'] = shape - helper.append_op(type='crop_tensor', - inputs=ipts, - outputs={'Out': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='crop_tensor', + inputs=ipts, + outputs={'Out': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out def affine_grid(theta, out_shape, name=None): """ :alias_main: paddle.nn.functional.affine_grid - :alias: paddle.nn.functional.affine_grid,paddle.nn.functional.vision.affine_grid - :old_api: paddle.fluid.layers.affine_grid + :alias: paddle.nn.functional.affine_grid,paddle.nn.functional.vision.affine_grid + :old_api: paddle.fluid.layers.affine_grid It generates a grid of (x,y) coordinates using the parameters of the affine transformation that correspond to a set of points where @@ -9746,11 +10317,15 @@ def affine_grid(theta, out_shape, name=None): """ helper = LayerHelper('affine_grid') - check_variable_and_dtype(theta, 'theta', ['float32', 'float64'], - 'affine_grid') - - if not (isinstance(out_shape, list) or isinstance(out_shape, tuple) or \ - isinstance(out_shape, Variable)): + check_variable_and_dtype( + theta, 'theta', ['float32', 'float64'], 'affine_grid' + ) + + if not ( + isinstance(out_shape, list) + or isinstance(out_shape, tuple) + or isinstance(out_shape, Variable) + ): raise ValueError("The out_shape should be a list, tuple or Variable.") if not isinstance(theta, Variable): @@ -9761,27 +10336,32 @@ def affine_grid(theta, out_shape, name=None): attrs = {} if isinstance(out_shape, Variable): ipts['OutputShape'] = out_shape - check_variable_and_dtype(out_shape, 'out_shape', ['int32'], - 'affine_grid') + check_variable_and_dtype( + out_shape, 'out_shape', ['int32'], 'affine_grid' + ) else: attrs['output_shape'] = out_shape if core.is_compiled_with_rocm(): # ROCM platform do not have MIOPEN kernel for affine_grid attrs['use_cudnn'] = False - helper.append_op(type='affine_grid', - inputs=ipts, - outputs={'Output': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='affine_grid', + inputs=ipts, + outputs={'Output': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out -def pad2d(input, - paddings=[0, 0, 0, 0], - mode='constant', - pad_value=0.0, - data_format="NCHW", - name=None): +def pad2d( + input, + paddings=[0, 0, 0, 0], + mode='constant', + pad_value=0.0, + data_format="NCHW", + name=None, +): """ Pad 2-d images according to 'paddings' and 'mode'. @@ -9795,10 +10375,10 @@ def pad2d(input, Otherwise, it is a 1-D Tensor with shape [4]. Data type is int32. Default is [0, 0, 0, 0]. mode (str): Three modes: 'constant' (default), 'reflect', 'edge' . - When in 'constant' mode, this op uses a constant value to pad the input tensor. - When in 'reflect' mode, uses reflection of the input boundaries to pad the input tensor. - When in 'edge' mode, uses input boundaries to pad the input tensor. - Default is 'constant' + When in 'constant' mode, this op uses a constant value to pad the input tensor. + When in 'reflect' mode, uses reflection of the input boundaries to pad the input tensor. + When in 'edge' mode, uses input boundaries to pad the input tensor. + Default is 'constant' pad_value (float32): The value to fill the padded areas in 'constant' mode . Default is 0.0 data_format (str): An string from: "NHWC", "NCHW". Specify the data format of the input data. @@ -9869,15 +10449,29 @@ def pad2d(input, # [2. 1. 2. 3. 2.]]]] """ if _non_static_mode(): - _paddings = paddings.numpy().tolist() if isinstance( - paddings, Variable) else paddings - return _legacy_C_ops.pad2d(input, 'mode', mode, 'pad_value', pad_value, - 'data_format', data_format, 'paddings', - _paddings) + _paddings = ( + paddings.numpy().tolist() + if isinstance(paddings, Variable) + else paddings + ) + return _legacy_C_ops.pad2d( + input, + 'mode', + mode, + 'pad_value', + pad_value, + 'data_format', + data_format, + 'paddings', + _paddings, + ) check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - "pad2d") + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + "pad2d", + ) attrs = {'mode': mode, 'pad_value': pad_value, 'data_format': data_format} inputs = {'X': [input]} @@ -9889,16 +10483,18 @@ def pad2d(input, helper = LayerHelper('pad2d', **locals()) - assert mode in ['reflect', 'edge', 'constant' - ], "mode should be one of constant, reflect, edge." + assert mode in [ + 'reflect', + 'edge', + 'constant', + ], "mode should be one of constant, reflect, edge." dtype = helper.input_dtype(input_param_name='input') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad2d', - inputs=inputs, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type='pad2d', inputs=inputs, outputs={"Out": out}, attrs=attrs + ) return out @@ -9907,8 +10503,8 @@ def pad2d(input, def elu(x, alpha=1.0, name=None): """ :alias_main: paddle.nn.functional.elu - :alias: paddle.nn.functional.elu,paddle.nn.functional.activation.elu - :old_api: paddle.fluid.layers.elu + :alias: paddle.nn.functional.elu,paddle.nn.functional.activation.elu + :old_api: paddle.fluid.layers.elu ${comment} Args: @@ -9937,10 +10533,12 @@ def elu(x, alpha=1.0, name=None): helper = LayerHelper('elu', **locals()) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'elu') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='elu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'alpha': alpha}) + helper.append_op( + type='elu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'alpha': alpha}, + ) return out @@ -9978,13 +10576,15 @@ def relu6(x, threshold=6.0, name=None): helper = LayerHelper('relu6', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='relu6', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'threshold': threshold, - 'use_mkldnn': _global_flags()["FLAGS_use_mkldnn"] - }) + helper.append_op( + type='relu6', + inputs={'X': x}, + outputs={'Out': out}, + attrs={ + 'threshold': threshold, + 'use_mkldnn': _global_flags()["FLAGS_use_mkldnn"], + }, + ) return out @@ -10021,7 +10621,8 @@ def pow(x, factor=1.0, name=None): # y_2 is x^{3.0} """ check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], 'pow') + x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], 'pow' + ) helper = LayerHelper('pow', **locals()) inputs = {'X': x} @@ -10034,10 +10635,9 @@ def pow(x, factor=1.0, name=None): attrs['factor'] = factor out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='pow', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='pow', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -10077,13 +10677,12 @@ def stanh(x, scale_a=0.67, scale_b=1.7159, name=None): helper = LayerHelper('stanh', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='stanh', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'scale_a': scale_a, - 'scale_b': scale_b - }) + helper.append_op( + type='stanh', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'scale_a': scale_a, 'scale_b': scale_b}, + ) return out @@ -10116,18 +10715,18 @@ def hard_sigmoid(x, slope=0.2, offset=0.5, name=None): if _non_static_mode(): return _legacy_C_ops.hard_sigmoid(x, 'slope', slope, 'offset', offset) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hard_sigmoid') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hard_sigmoid' + ) helper = LayerHelper('hard_sigmoid', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='hard_sigmoid', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'slope': slope, - 'offset': offset - }) + helper.append_op( + type='hard_sigmoid', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'slope': slope, 'offset': offset}, + ) return out @@ -10135,8 +10734,8 @@ def hard_sigmoid(x, slope=0.2, offset=0.5, name=None): def swish(x, beta=1.0, name=None): r""" :alias_main: paddle.nn.functional.swish - :alias: paddle.nn.functional.swish,paddle.nn.functional.activation.swish - :old_api: paddle.fluid.layers.swish + :alias: paddle.nn.functional.swish,paddle.nn.functional.activation.swish + :old_api: paddle.fluid.layers.swish Elementwise swish activation function. See `Searching for Activation Functions `_ for more details. @@ -10208,10 +10807,12 @@ def swish(x, beta=1.0, name=None): helper = LayerHelper('swish', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='swish', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'slope': beta}) + helper.append_op( + type='swish', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'slope': beta}, + ) return out @@ -10272,52 +10873,57 @@ def prelu(x, mode, param_attr=None, data_format="NCHW", name=None): if mode == 'channel': true_data_format = [ - 'NC', 'NCL', 'NCHW', 'NCDHW', 'NLC', 'NHWC', 'NDHWC' + 'NC', + 'NCL', + 'NCHW', + 'NCDHW', + 'NLC', + 'NHWC', + 'NDHWC', ] if data_format not in true_data_format: raise ValueError( "data_format must be one of 'NC', 'NCL', 'NCHW', 'NCDHW', " - "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format)) + "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format) + ) data_format = 'NCHW' if data_format[1] == 'C' else 'NHWC' - assert len( - x.shape - ) >= 2, "The size of input shape should be equal or larger than 2 in prelu() when mode is 'channel'" - #NOTE(zhiqiu): The alpha_shape should be [1, channel] + [1] * len(x.shape[2:]). + assert ( + len(x.shape) >= 2 + ), "The size of input shape should be equal or larger than 2 in prelu() when mode is 'channel'" + # NOTE(zhiqiu): The alpha_shape should be [1, channel] + [1] * len(x.shape[2:]). # To be consistent with Prelu, it is simplified. - #NOTE(zhiqiu): Revert shape to [1, channel, 1, 1] for compatibility with saved model of old version. - #NOTE(GuoxiaWang): support NHWC data format + # NOTE(zhiqiu): Revert shape to [1, channel, 1, 1] for compatibility with saved model of old version. + # NOTE(GuoxiaWang): support NHWC data format if data_format == 'NHWC': alpha_shape = [1, 1, 1, x.shape[-1]] else: alpha_shape = [1, x.shape[1], 1, 1] elif mode == 'element': - assert len( - x.shape - ) >= 1, "The size of input shape should be equal or larger than 1 in prelu() when mode is 'element'" + assert ( + len(x.shape) >= 1 + ), "The size of input shape should be equal or larger than 1 in prelu() when mode is 'element'" alpha_shape = [1] + list(x.shape)[1:] dtype = helper.input_dtype(input_param_name='x') - alpha = helper.create_parameter(attr=helper.param_attr, - shape=alpha_shape, - dtype=dtype, - is_bias=False, - default_initializer=Constant(0.25)) + alpha = helper.create_parameter( + attr=helper.param_attr, + shape=alpha_shape, + dtype=dtype, + is_bias=False, + default_initializer=Constant(0.25), + ) if in_dygraph_mode(): return _C_ops.prelu(x, alpha, data_format, mode) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="prelu", - inputs={ - "X": x, - 'Alpha': alpha - }, - attrs={ - "mode": mode, - "data_format": data_format - }, - outputs={"Out": out}) + helper.append_op( + type="prelu", + inputs={"X": x, 'Alpha': alpha}, + attrs={"mode": mode, "data_format": data_format}, + outputs={"Out": out}, + ) return out @@ -10358,13 +10964,12 @@ def brelu(x, t_min=0.0, t_max=24.0, name=None): helper = LayerHelper('brelu', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='brelu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 't_min': t_min, - 't_max': t_max - }) + helper.append_op( + type='brelu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'t_min': t_min, 't_max': t_max}, + ) return out @@ -10431,15 +11036,18 @@ def soft_relu(x, threshold=40.0, name=None): res = exe.run(fluid.default_main_program(), feed={'x':img}, fetch_list=[output]) print(res) # [array([[0.6931472, 1.3132616], [2.126928 , 3.0485873]], dtype=float32)] """ - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'soft_relu') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'soft_relu' + ) helper = LayerHelper('soft_relu', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='soft_relu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='soft_relu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out @@ -10508,8 +11116,11 @@ def flatten(x, axis=1, name=None): # out shape is [16, 3] """ check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int8', 'int32', 'int64', 'uint8'], - 'flatten') + x, + 'x', + ['float32', 'float64', 'int8', 'int32', 'int64', 'uint8'], + 'flatten', + ) if _non_static_mode(): return _legacy_C_ops.flatten2(x, 'axis', axis)[0] @@ -10523,13 +11134,12 @@ def flatten(x, axis=1, name=None): out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='flatten2', - inputs={"X": x}, - outputs={ - 'Out': out, - 'XShape': x_shape - }, - attrs={"axis": axis}) + helper.append_op( + type='flatten2', + inputs={"X": x}, + outputs={'Out': out, 'XShape': x_shape}, + attrs={"axis": axis}, + ) return out @@ -10623,42 +11233,53 @@ def stack(x, axis=0, name=None): if not isinstance(x, list) and not isinstance(x, tuple): # NOTE:(zhiqiu) Only support Variable as input if the Variable is a LOD_TENSOR_ARRAY create by create_array, array_write, array_read, etc. # In that case, Variable is array of tensors indeed. - if isinstance(x, Variable) and x.desc.type( - ) == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + isinstance(x, Variable) + and x.desc.type() == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): x = [x] else: raise TypeError( - "The type of '%s' in %s must be %s, but received %s" % - ('x', 'stack', 'list[Tensor], tuple[Tensor] or TensorArray', - type(x))) + "The type of '%s' in %s must be %s, but received %s" + % ( + 'x', + 'stack', + 'list[Tensor], tuple[Tensor] or TensorArray', + type(x), + ) + ) helper = LayerHelper('stack', **locals()) out = helper.create_variable_for_type_inference(x[0].dtype) if x[0].desc.type() == core.VarDesc.VarType.LOD_TENSOR_ARRAY: - assert len(x) == 1, "If the elements of 'x' in stack are Variable(LoDTensorArray), " \ - "number of the elements must be 1, but received %s." % len(x) + assert len(x) == 1, ( + "If the elements of 'x' in stack are Variable(LoDTensorArray), " + "number of the elements must be 1, but received %s." % len(x) + ) out_index = helper.create_variable_for_type_inference(dtype="int32") for i in x: - check_variable_and_dtype(i, 'x', \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'stack') - - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': x[0]}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': True - }) + check_variable_and_dtype( + i, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'stack', + ) + + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': x[0]}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': True}, + ) else: - helper.append_op(type='stack', - inputs={'X': x}, - outputs={'Y': out}, - attrs={'axis': axis}) + helper.append_op( + type='stack', + inputs={'X': x}, + outputs={'Y': out}, + attrs={'axis': axis}, + ) return out @@ -10722,21 +11343,12 @@ def filter_by_instag(ins, ins_tag, filter_tag, is_lod, out_val_if_empty=0): out = helper.create_variable_for_type_inference(dtype=ins.dtype) loss_weight = helper.create_variable_for_type_inference(dtype=np.float64) mmap = helper.create_variable_for_type_inference(dtype=ins_tag.dtype) - helper.append_op(type='filter_by_instag', - inputs={ - 'Ins': ins, - 'Ins_tag': ins_tag, - 'Filter_tag': filter_tag - }, - outputs={ - 'Out': out, - 'LossWeight': loss_weight, - 'IndexMap': mmap - }, - attrs={ - 'is_lod': is_lod, - 'out_val_if_empty': out_val_if_empty - }) + helper.append_op( + type='filter_by_instag', + inputs={'Ins': ins, 'Ins_tag': ins_tag, 'Filter_tag': filter_tag}, + outputs={'Out': out, 'LossWeight': loss_weight, 'IndexMap': mmap}, + attrs={'is_lod': is_lod, 'out_val_if_empty': out_val_if_empty}, + ) return [out, loss_weight] @@ -10744,8 +11356,8 @@ def filter_by_instag(ins, ins_tag, filter_tag, is_lod, out_val_if_empty=0): def unstack(x, axis=0, num=None): """ :alias_main: paddle.unstack - :alias: paddle.unstack,paddle.tensor.unstack,paddle.tensor.manipulation.unstack - :old_api: paddle.fluid.layers.unstack + :alias: paddle.unstack,paddle.tensor.unstack,paddle.tensor.manipulation.unstack + :old_api: paddle.fluid.layers.unstack **UnStack Layer** @@ -10794,13 +11406,12 @@ def unstack(x, axis=0, num=None): for _ in range(num): outs.append(helper.create_variable_for_type_inference(x.dtype)) - helper.append_op(type='unstack', - inputs={'X': [x]}, - outputs={'Y': outs}, - attrs={ - 'axis': axis, - 'num': num - }) + helper.append_op( + type='unstack', + inputs={'X': [x]}, + outputs={'Y': outs}, + attrs={'axis': axis, 'num': num}, + ) return outs @@ -10808,8 +11419,8 @@ def unstack(x, axis=0, num=None): def expand(x, expand_times, name=None): """ :alias_main: paddle.expand - :alias: paddle.expand,paddle.tensor.expand,paddle.tensor.manipulation.expand - :old_api: paddle.fluid.layers.expand + :alias: paddle.expand,paddle.tensor.expand,paddle.tensor.manipulation.expand + :old_api: paddle.fluid.layers.expand This operation tiles ``x`` multiple times according to the parameter ``expand_times``. The times number for each dimension of ``x`` is set by the parameter ``expand_times``. @@ -10883,12 +11494,16 @@ def expand(x, expand_times, name=None): inputs = {"X": [x]} attrs = {} check_variable_and_dtype( - x, 'x', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'expand') + x, + 'x', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'expand', + ) check_type(expand_times, 'expand_times', (list, tuple, Variable), 'expand') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == True: raise ValueError( - "expand op bool date type must set the stop_gradient to be False") + "expand op bool date type must set the stop_gradient to be False" + ) helper = LayerHelper('expand', input=x, **locals()) @@ -10899,8 +11514,9 @@ def expand(x, expand_times, name=None): attrs_expand_times.append(-1) else: attrs_expand_times.append(times) - assert times > 0, ( - "Each element given in expand_times must not be negative.") + assert ( + times > 0 + ), "Each element given in expand_times must not be negative." return attrs_expand_times if isinstance(expand_times, Variable): @@ -10910,14 +11526,14 @@ def expand(x, expand_times, name=None): attrs['expand_times'] = get_attr_expand_times(expand_times) if utils._contain_var(expand_times): inputs['expand_times_tensor'] = utils._convert_to_tensor_list( - expand_times) + expand_times + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='expand', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='expand', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -10925,8 +11541,8 @@ def expand(x, expand_times, name=None): def expand_as(x, target_tensor, name=None): """ :alias_main: paddle.expand_as - :alias: paddle.expand_as,paddle.tensor.expand_as,paddle.tensor.manipulation.expand_as - :old_api: paddle.fluid.layers.expand_as + :alias: paddle.expand_as,paddle.tensor.expand_as,paddle.tensor.manipulation.expand_as + :old_api: paddle.fluid.layers.expand_as expand_as operator tiles to the input by given expand tensor. You should set expand tensor for each dimension by providing tensor 'target_tensor'. The rank of X @@ -10992,12 +11608,15 @@ def expand_as(x, target_tensor, name=None): if _non_static_mode(): return _legacy_C_ops.expand_as(x, target_tensor) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'int32', 'int64', 'bool'], - 'expand_as') - check_variable_and_dtype(target_tensor, 'target_tensor', - ['float32', 'float64', 'int32', 'int64', 'bool'], - 'expand_as') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64', 'bool'], 'expand_as' + ) + check_variable_and_dtype( + target_tensor, + 'target_tensor', + ['float32', 'float64', 'int32', 'int64', 'bool'], + 'expand_as', + ) helper = LayerHelper('expand_as', input=x, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) @@ -11011,14 +11630,16 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ @deprecated(since='1.8.0', update_to="paddle.uniform") @templatedoc() -def uniform_random_batch_size_like(input, - shape, - dtype='float32', - input_dim_idx=0, - output_dim_idx=0, - min=-1.0, - max=1.0, - seed=0): +def uniform_random_batch_size_like( + input, + shape, + dtype='float32', + input_dim_idx=0, + output_dim_idx=0, + min=-1.0, + max=1.0, + seed=0, +): """ This OP initializes a variable with random values sampled from a uniform distribution in the range [min, max). The input_dim_idx used to get the input dimension value which will be used to resize the output dimension. @@ -11081,39 +11702,46 @@ def uniform_random_batch_size_like(input, """ - check_variable_and_dtype(input, 'Input', ("float32", 'float64', "uint16"), - 'uniform_random_batch_size_like') + check_variable_and_dtype( + input, + 'Input', + ("float32", 'float64', "uint16"), + 'uniform_random_batch_size_like', + ) check_type(shape, 'shape', (list, tuple), 'uniform_random_batch_size_like') - check_dtype(dtype, 'dtype', ('float32', 'float64', "uint16"), - 'uniform_random_batch_size_like') + check_dtype( + dtype, + 'dtype', + ('float32', 'float64', "uint16"), + 'uniform_random_batch_size_like', + ) helper = LayerHelper('uniform_random_batch_size_like', **locals()) out = helper.create_variable_for_type_inference(dtype) c_dtype = convert_np_dtype_to_dtype_(dtype) - helper.append_op(type='uniform_random_batch_size_like', - inputs={'Input': input}, - outputs={'Out': out}, - attrs={ - 'shape': shape, - 'input_dim_idx': input_dim_idx, - 'output_dim_idx': output_dim_idx, - 'min': min, - 'max': max, - 'seed': seed, - 'dtype': c_dtype - }) + helper.append_op( + type='uniform_random_batch_size_like', + inputs={'Input': input}, + outputs={'Out': out}, + attrs={ + 'shape': shape, + 'input_dim_idx': input_dim_idx, + 'output_dim_idx': output_dim_idx, + 'min': min, + 'max': max, + 'seed': seed, + 'dtype': c_dtype, + }, + ) return out @deprecated(since="2.0.0", update_to="paddle.normal") @templatedoc() -def gaussian_random(shape, - mean=0.0, - std=1.0, - seed=0, - dtype='float32', - name=None): +def gaussian_random( + shape, mean=0.0, std=1.0, seed=0, dtype='float32', name=None +): """ This OP returns a Tensor filled with random values sampled from a Gaussian distribution, with ``shape`` and ``dtype``. @@ -11212,15 +11840,24 @@ def gaussian_random(shape, if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) place = _current_expected_place() - return _C_ops.gaussian_random(shape, float(mean), float(std), seed, - dtype, place) + return _C_ops.gaussian_random( + shape, float(mean), float(std), seed, dtype, place + ) if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.gaussian_random('shape', shape, - 'mean', float(mean), 'std', - float(std), 'seed', seed, 'dtype', - dtype) + return _legacy_C_ops.gaussian_random( + 'shape', + shape, + 'mean', + float(mean), + 'std', + float(std), + 'seed', + seed, + 'dtype', + dtype, + ) check_type(shape, 'shape', (list, tuple, Variable), 'gaussian_random/randn') check_dtype(dtype, 'dtype', ['float32', 'float64'], 'gaussian_random/randn') @@ -11231,19 +11868,17 @@ def gaussian_random(shape, 'std': std, 'seed': seed, 'dtype': dtype, - 'use_mkldnn': False + 'use_mkldnn': False, } - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='gaussian_random/randn') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='gaussian_random/randn' + ) helper = LayerHelper('gaussian_random', **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='gaussian_random', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='gaussian_random', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -11277,28 +11912,28 @@ def sampling_id(x, min=0.0, max=1.0, seed=0, dtype='float32'): helper = LayerHelper('sampling_id', **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='sampling_id', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'min': min, - 'max': max, - 'seed': seed - }) + helper.append_op( + type='sampling_id', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'min': min, 'max': max, 'seed': seed}, + ) return out @deprecated(since='1.8.0', update_to="paddle.normal") @templatedoc() -def gaussian_random_batch_size_like(input, - shape, - input_dim_idx=0, - output_dim_idx=0, - mean=0.0, - std=1.0, - seed=0, - dtype='float32'): +def gaussian_random_batch_size_like( + input, + shape, + input_dim_idx=0, + output_dim_idx=0, + mean=0.0, + std=1.0, + seed=0, + dtype='float32', +): """ ${comment} @@ -11329,26 +11964,40 @@ def gaussian_random_batch_size_like(input, """ helper = LayerHelper('gaussian_random_batch_size_like', **locals()) - check_type(input, 'input', (Variable), - 'fluid.layers.gaussian_random_batch_size_like') - check_type(shape, 'shape', (list, tuple), - 'fluid.layers.gaussian_random_batch_size_like') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'int'], - 'fluid.layers.gaussian_random_batch_size_like') + check_type( + input, + 'input', + (Variable), + 'fluid.layers.gaussian_random_batch_size_like', + ) + check_type( + shape, + 'shape', + (list, tuple), + 'fluid.layers.gaussian_random_batch_size_like', + ) + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'int'], + 'fluid.layers.gaussian_random_batch_size_like', + ) out = helper.create_variable_for_type_inference(dtype) c_dtype = convert_np_dtype_to_dtype_(dtype) - helper.append_op(type='gaussian_random_batch_size_like', - inputs={'Input': input}, - outputs={'Out': out}, - attrs={ - 'shape': shape, - 'input_dim_idx': input_dim_idx, - 'output_dim_idx': output_dim_idx, - 'mean': mean, - 'std': std, - 'seed': seed, - 'dtype': c_dtype - }) + helper.append_op( + type='gaussian_random_batch_size_like', + inputs={'Input': input}, + outputs={'Out': out}, + attrs={ + 'shape': shape, + 'input_dim_idx': input_dim_idx, + 'output_dim_idx': output_dim_idx, + 'mean': mean, + 'std': std, + 'seed': seed, + 'dtype': c_dtype, + }, + ) return out @@ -11507,7 +12156,8 @@ def slice(input, axes, starts, ends): axes = list(axes) if len(axes) == 0: raise ValueError( - "Input axes should not be an empty list/tuple.") + "Input axes should not be an empty list/tuple." + ) for i in range(len(axes)): if axes[i] < 0: axes[i] = max(0, axes[i] + len(input.shape)) @@ -11516,8 +12166,10 @@ def slice(input, axes, starts, ends): else: raise ValueError( - "Input axes must be a python list or tuple, but reveived {}". - format(type(axes))) + "Input axes must be a python list or tuple, but reveived {}".format( + type(axes) + ) + ) infer_flags = list(1 for i in range(len(axes))) @@ -11525,7 +12177,8 @@ def slice(input, axes, starts, ends): if isinstance(starts, (list, tuple)): starts = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in starts ] elif isinstance(starts, tmp_tensor_type): @@ -11535,7 +12188,9 @@ def slice(input, axes, starts, ends): if isinstance(ends, (list, tuple)): ends = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item for item in ends + if isinstance(item, tmp_tensor_type) + else item + for item in ends ] attrs += ('ends', ends) elif isinstance(ends, tmp_tensor_type): @@ -11553,7 +12208,8 @@ def slice(input, axes, starts, ends): axes = list(axes) if len(axes) == 0: raise ValueError( - "Input axes should not be an empty list/tuple.") + "Input axes should not be an empty list/tuple." + ) for i in range(len(axes)): if axes[i] < 0: axes[i] = max(0, axes[i] + len(input.shape)) @@ -11562,8 +12218,10 @@ def slice(input, axes, starts, ends): else: raise ValueError( - "Input axes must be a python list or tuple, but reveived {}" - .format(type(axes))) + "Input axes must be a python list or tuple, but reveived {}".format( + type(axes) + ) + ) infer_flags = list(1 for i in range(len(axes))) @@ -11572,7 +12230,8 @@ def slice(input, axes, starts, ends): if isinstance(starts, (list, tuple)): starts = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in starts ] attrs += ('starts', starts) @@ -11584,7 +12243,8 @@ def slice(input, axes, starts, ends): if isinstance(ends, (list, tuple)): ends = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in ends ] attrs += ('ends', ends) @@ -11593,16 +12253,27 @@ def slice(input, axes, starts, ends): ends_tensor.stop_gradient = True infer_flags = list(-1 for i in range(len(axes))) - return _legacy_C_ops.slice(input, starts_tensor, ends_tensor, None, - None, 'axes', axes, 'infer_flags', - infer_flags, *attrs) + return _legacy_C_ops.slice( + input, + starts_tensor, + ends_tensor, + None, + None, + 'axes', + axes, + 'infer_flags', + infer_flags, + *attrs, + ) if not isinstance(starts, (list, tuple, Variable)): raise ValueError( - "Input starts must be an Variable, python list or tuple.") + "Input starts must be an Variable, python list or tuple." + ) if not isinstance(ends, (list, tuple, Variable)): raise ValueError( - "Input ends must be an Variable, python list or tuple.") + "Input ends must be an Variable, python list or tuple." + ) helper = LayerHelper('slice', **locals()) @@ -11649,11 +12320,11 @@ def slice(input, axes, starts, ends): # infer_flags attrs['infer_flags'] = infer_flags out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('input')) - helper.append_op(type='slice', - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + dtype=helper.input_dtype('input') + ) + helper.append_op( + type='slice', inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out @@ -11662,8 +12333,8 @@ def slice(input, axes, starts, ends): def strided_slice(input, axes, starts, ends, strides): """ :alias_main: paddle.strided_slice - :alias: paddle.strided_slice,paddle.tensor.strided_slice,paddle.tensor.manipulation.strided_slice - :old_api: paddle.fluid.layers.strided_slice + :alias: paddle.strided_slice,paddle.tensor.strided_slice,paddle.tensor.manipulation.strided_slice + :old_api: paddle.fluid.layers.strided_slice This operator produces a slice of ``input`` along multiple axes. Similar to numpy: https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html @@ -11764,9 +12435,12 @@ def strided_slice(input, axes, starts, ends, strides): helper = LayerHelper('strided_slice', **locals()) - check_variable_and_dtype(input, 'input', - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'strided_slice') + check_variable_and_dtype( + input, + 'input', + ['bool', 'float32', 'float64', 'int32', 'int64'], + 'strided_slice', + ) check_type(axes, 'axes', (list, tuple), 'strided_slice') check_type(starts, 'starts', (list, tuple, Variable), 'strided_slice') check_type(ends, 'ends', (list, tuple, Variable), 'strided_slice') @@ -11774,8 +12448,9 @@ def strided_slice(input, axes, starts, ends, strides): def check_list_elements_dtype(list_input, input_name): if isinstance(list_input, Variable): - check_dtype(list_input.dtype, input_name, ['int32'], - 'strided_slice') + check_dtype( + list_input.dtype, input_name, ['int32'], 'strided_slice' + ) else: for i, var in enumerate(list_input): var_name = input_name + '[' + str(i) + ']' @@ -11794,7 +12469,7 @@ def strided_slice(input, axes, starts, ends, strides): dim.stop_gradient = True new_list_tensor.append(dim) else: - assert (isinstance(dim, int)) + assert isinstance(dim, int) temp_out = helper.create_variable_for_type_inference('int32') fill_constant([1], 'int32', dim, force_cpu=True, out=temp_out) new_list_tensor.append(temp_out) @@ -11811,7 +12486,7 @@ def strided_slice(input, axes, starts, ends, strides): 'starts': starts, 'ends': ends, 'strides': strides, - 'infer_flags': infer_flags + 'infer_flags': infer_flags, } else: # starts @@ -11866,11 +12541,11 @@ def strided_slice(input, axes, starts, ends, strides): attrs['strides'] = strides attrs['infer_flags'] = infer_flags out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('input')) - helper.append_op(type='strided_slice', - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + dtype=helper.input_dtype('input') + ) + helper.append_op( + type='strided_slice', inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out @@ -11878,8 +12553,8 @@ def strided_slice(input, axes, starts, ends, strides): def shape(input): """ :alias_main: paddle.shape - :alias: paddle.shape,paddle.tensor.shape,paddle.tensor.attribute.shape - :old_api: paddle.fluid.layers.shape + :alias: paddle.shape,paddle.tensor.shape,paddle.tensor.attribute.shape + :old_api: paddle.fluid.layers.shape **Shape Layer** @@ -11937,16 +12612,29 @@ def shape(input): out.stop_gradient = True return out - check_variable_and_dtype(input, 'input', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'shape') + check_variable_and_dtype( + input, + 'input', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'shape', + ) helper = LayerHelper('shape', **locals()) out = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type='shape', - inputs={'Input': input}, - outputs={'Out': out}, - stop_gradient=True) + helper.append_op( + type='shape', + inputs={'Input': input}, + outputs={'Out': out}, + stop_gradient=True, + ) return out @@ -12014,8 +12702,11 @@ def size(input): return _legacy_C_ops.size(input) check_variable_and_dtype( - input, 'input', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], "size") + input, + 'input', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + "size", + ) helper = LayerHelper('size', **locals()) out = helper.create_variable_for_type_inference(dtype='int64') helper.append_op(type='size', inputs={'Input': input}, outputs={'Out': out}) @@ -12031,27 +12722,29 @@ def _elementwise_op(helper): assert x is not None, 'x cannot be None in {}'.format(op_type) assert y is not None, 'y cannot be None in {}'.format(op_type) check_variable_and_dtype( - x, 'x', ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'], - op_type) + x, + 'x', + ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'], + op_type, + ) check_variable_and_dtype( - y, 'y', ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'], - op_type) + y, + 'y', + ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'], + op_type, + ) axis = helper.kwargs.get('axis', -1) use_mkldnn = helper.kwargs.get('use_mkldnn', False) name = helper.kwargs.get('name', None) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=op_type, - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={ - 'axis': axis, - 'use_mkldnn': use_mkldnn - }) + helper.append_op( + type=op_type, + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={'axis': axis, 'use_mkldnn': use_mkldnn}, + ) return helper.append_activation(out) @@ -12107,15 +12800,33 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): return dygraph_utils._append_activation_in_dygraph(out) if _non_static_mode(): _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale - out = _legacy_C_ops.scale(x, 'scale', float(_scale), 'bias', - float(bias), 'bias_after_scale', - bias_after_scale) + out = _legacy_C_ops.scale( + x, + 'scale', + float(_scale), + 'bias', + float(bias), + 'bias_after_scale', + bias_after_scale, + ) return dygraph_utils._append_activation_in_dygraph(out) - check_variable_and_dtype(x, "x", [ - 'float16', 'uint16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8' - ], "scale") + check_variable_and_dtype( + x, + "x", + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + ], + "scale", + ) inputs = {'X': [x]} attrs = { 'bias': float(bias), @@ -12128,91 +12839,90 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): helper = LayerHelper('scale', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='scale', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='scale', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return helper.append_activation(out) def elementwise_add(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_add(x, y) - # z = x + y + import paddle.fluid as fluid + import numpy as np + import paddle + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_add(x, y) + # z = x + y - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # [3., 8., 6.] + print(z_value) # [3., 8., 6.] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_add(x, y, axis=1) - # z = x + y + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_add(x, y, axis=1) + # z = x + y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + print(z_value) # z.shape=[2,3,4,5] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), - "y": np.random.randint(1, 5, size=[5]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[5], dtype='float32') - z = fluid.layers.elementwise_add(x, y, axis=3) - # z = x + y + def gen_data(): + return { + "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), + "y": np.random.randint(1, 5, size=[5]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[5], dtype='float32') + z = fluid.layers.elementwise_add(x, y, axis=3) + # z = x + y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) + print(z_value) # z.shape=[2,3,4,5] """ if _non_static_mode(): @@ -12222,7 +12932,8 @@ Examples: axis=axis, act=act, op_name='elementwise_add', - use_mkldnn=_global_flags()["FLAGS_use_mkldnn"]) + use_mkldnn=_global_flags()["FLAGS_use_mkldnn"], + ) return _elementwise_op(LayerHelper('elementwise_add', **locals())) @@ -12231,90 +12942,88 @@ Examples: def elementwise_div(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_div(x, y) - # z = x / y + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_div(x, y) + # z = x / y - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # [2., 0.6, 2.] + print(z_value) # [2., 0.6, 2.] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_div(x, y, axis=1) - # z = x / y + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_div(x, y, axis=1) + # z = x / y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + print(z_value) # z.shape=[2,3,4,5] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), - "y": np.random.randint(1, 5, size=[5]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[5], dtype='float32') - z = fluid.layers.elementwise_div(x, y, axis=3) - # z = x / y + def gen_data(): + return { + "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), + "y": np.random.randint(1, 5, size=[5]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[5], dtype='float32') + z = fluid.layers.elementwise_div(x, y, axis=3) + # z = x / y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) + print(z_value) # z.shape=[2,3,4,5] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_div') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_div' + ) return _elementwise_op(LayerHelper('elementwise_div', **locals())) @@ -12322,90 +13031,88 @@ Examples: def elementwise_sub(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_sub(x, y) - # z = x - y + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_sub(x, y) + # z = x - y - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # [1., -2., 2.] + print(z_value) # [1., -2., 2.] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_sub(x, y, axis=1) - # z = x - y + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_sub(x, y, axis=1) + # z = x - y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + print(z_value) # z.shape=[2,3,4,5] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), - "y": np.random.randint(1, 5, size=[5]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[5], dtype='float32') - z = fluid.layers.elementwise_sub(x, y, axis=3) - # z = x - y + def gen_data(): + return { + "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), + "y": np.random.randint(1, 5, size=[5]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[5], dtype='float32') + z = fluid.layers.elementwise_sub(x, y, axis=3) + # z = x - y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) + print(z_value) # z.shape=[2,3,4,5] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_sub') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_sub' + ) return _elementwise_op(LayerHelper('elementwise_sub', **locals())) @@ -12414,222 +13121,216 @@ Examples: def elementwise_mul(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_mul(x, y) - # z = x * y + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_mul(x, y) + # z = x * y - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # [2., 15., 8.] + print(z_value) # [2., 15., 8.] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_mul(x, y, axis=1) - # z = x * y + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_mul(x, y, axis=1) + # z = x * y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + print(z_value) # z.shape=[2,3,4,5] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), - "y": np.random.randint(1, 5, size=[5]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[5], dtype='float32') - z = fluid.layers.elementwise_mul(x, y, axis=3) - # z = x * y + def gen_data(): + return { + "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), + "y": np.random.randint(1, 5, size=[5]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[5], dtype='float32') + z = fluid.layers.elementwise_mul(x, y, axis=3) + # z = x * y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) + print(z_value) # z.shape=[2,3,4,5] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_mul') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_mul' + ) return _elementwise_op(LayerHelper('elementwise_mul', **locals())) def elementwise_max(x, y, axis=-1, act=None, name=None): """ - :alias_main: paddle.elementwise_max - :alias: paddle.elementwise_max,paddle.tensor.elementwise_max,paddle.tensor.math.elementwise_max - :old_api: paddle.fluid.layers.elementwise_max + :alias_main: paddle.elementwise_max + :alias: paddle.elementwise_max,paddle.tensor.elementwise_max,paddle.tensor.math.elementwise_max + :old_api: paddle.fluid.layers.elementwise_max -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_max(x, y) + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_max(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[2, 5, 4] + print(z_value) #[2, 5, 4] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_max(x, y, axis=1) + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_max(x, y, axis=1) - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value)#[[[[1., 1., 1., 1., 1.] .... [1., 1., 1., 1., 1.]]]] + print(z_value)#[[[[1., 1., 1., 1., 1.] .... [1., 1., 1., 1., 1.]]]] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_max') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_max' + ) return _elementwise_op(LayerHelper('elementwise_max', **locals())) def elementwise_min(x, y, axis=-1, act=None, name=None): """ - :alias_main: paddle.elementwise_min - :alias: paddle.elementwise_min,paddle.tensor.elementwise_min,paddle.tensor.math.elementwise_min - :old_api: paddle.fluid.layers.elementwise_min + :alias_main: paddle.elementwise_min + :alias: paddle.elementwise_min,paddle.tensor.elementwise_min,paddle.tensor.math.elementwise_min + :old_api: paddle.fluid.layers.elementwise_min -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_min(x, y) + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_min(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[1, 3, 2] + print(z_value) #[1, 3, 2] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_min(x, y, axis=1) + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_min(x, y, axis=1) - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value)#[[[[0., 0., 0., 0., 0.] .... [0., 0., 0., 0., 0.]]]] + print(z_value)#[[[[0., 0., 0., 0., 0.] .... [0., 0., 0., 0., 0.]]]] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_min') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_min' + ) return _elementwise_op(LayerHelper('elementwise_min', **locals())) @@ -12637,37 +13338,35 @@ Examples: def elementwise_pow(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_pow(x, y) + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_pow(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[2, 243, 16] + print(z_value) #[2, 243, 16] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_pow') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_pow' + ) return _elementwise_op(LayerHelper('elementwise_pow', **locals())) @@ -12675,37 +13374,35 @@ Examples: def elementwise_mod(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([10, 15, 8]).astype('int32'), - "y": np.array([3, 6, 5]).astype('int32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='int32') - y = fluid.data(name="y", shape=[3], dtype='int32') - z = fluid.layers.elementwise_mod(x, y) + def gen_data(): + return { + "x": np.array([10, 15, 8]).astype('int32'), + "y": np.array([3, 6, 5]).astype('int32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='int32') + y = fluid.data(name="y", shape=[3], dtype='int32') + z = fluid.layers.elementwise_mod(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[1, 3, 3] + print(z_value) #[1, 3, 3] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_mod') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_mod' + ) return _elementwise_op(LayerHelper('elementwise_mod', **locals())) @@ -12714,78 +13411,89 @@ Examples: def elementwise_floordiv(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([10, 15, 8]).astype('int32'), - "y": np.array([3, 7, 5]).astype('int32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='int32') - y = fluid.data(name="y", shape=[3], dtype='int32') - z = fluid.layers.elementwise_floordiv(x, y) + def gen_data(): + return { + "x": np.array([10, 15, 8]).astype('int32'), + "y": np.array([3, 7, 5]).astype('int32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='int32') + y = fluid.data(name="y", shape=[3], dtype='int32') + z = fluid.layers.elementwise_floordiv(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[3, 2, 1] + print(z_value) #[3, 2, 1] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_floordiv') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_floordiv' + ) return _elementwise_op(LayerHelper('elementwise_floordiv', **locals())) for func in [ - elementwise_add, - elementwise_div, - elementwise_sub, - elementwise_mul, - elementwise_max, - elementwise_pow, - elementwise_min, - elementwise_mod, - elementwise_floordiv, + elementwise_add, + elementwise_div, + elementwise_sub, + elementwise_mul, + elementwise_max, + elementwise_pow, + elementwise_min, + elementwise_mod, + elementwise_floordiv, ]: op_proto = OpProtoHolder.instance().get_op_proto(func.__name__) # insert the c++ doc string on top of python doc string - func.__doc__ = _generate_doc_string_( - op_proto, - additional_args_lines=[ - "axis (int32, optional): If X.dimension != Y.dimension, \ + func.__doc__ = ( + _generate_doc_string_( + op_proto, + additional_args_lines=[ + "axis (int32, optional): If X.dimension != Y.dimension, \ Y.dimension must be a subsequence of x.dimension. \ And axis is the start dimension index for broadcasting Y onto X. ", - "act (string, optional): Activation applied to the output. \ + "act (string, optional): Activation applied to the output. \ Default is None. Details: :ref:`api_guide_activations_en` ", - "name (string, optional): Name of the output. \ + "name (string, optional): Name of the output. \ Default is None. It's used to print debug info for developers. Details: \ - :ref:`api_guide_Name` " - ], - skip_attrs_set={ - "x_data_format", "y_data_format", "axis", "use_quantizer", - "mkldnn_data_type", "Scale_x", "Scale_y", "Scale_out" - }) + """\n""" + str(func.__doc__) + :ref:`api_guide_Name` ", + ], + skip_attrs_set={ + "x_data_format", + "y_data_format", + "axis", + "use_quantizer", + "mkldnn_data_type", + "Scale_x", + "Scale_y", + "Scale_out", + }, + ) + + """\n""" + + str(func.__doc__) + ) doc_list = func.__doc__.splitlines() for idx, val in enumerate(doc_list): - if val.startswith("Warning: ") and val.endswith( - " instead." - ) and "and will be removed in future versions." in val: + if ( + val.startswith("Warning: ") + and val.endswith(" instead.") + and "and will be removed in future versions." in val + ): doc_list.insert(0, doc_list.pop(idx)) func.__doc__ = "\n" + "\n".join(i for i in doc_list) break @@ -12796,9 +13504,12 @@ for func in []: op_proto, additional_args_lines=[ "act (basestring|None): Activation applied to the output.", - "name (basestring|None): Name of the output." - ]) - func.__doc__ = func.__doc__ + """ + "name (basestring|None): Name of the output.", + ], + ) + func.__doc__ = ( + func.__doc__ + + """ Examples: .. code-block:: python @@ -12833,8 +13544,16 @@ Examples: x5 = fluid.layers.data(name="x5", shape=[2, 3, 4, 5], dtype='float32') y5 = fluid.layers.data(name="y5", shape=[2], dtype='float32') z5 = fluid.layers.%s(x5, y5, axis=0) - """ % (func.__name__, func.__name__, func.__name__, func.__name__, - func.__name__, func.__name__) + """ + % ( + func.__name__, + func.__name__, + func.__name__, + func.__name__, + func.__name__, + func.__name__, + ) + ) def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): @@ -12845,14 +13564,18 @@ def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): else: return op(x) check_variable_and_dtype( - x, "x", + x, + "x", ["bool", "int8", "int16", "int32", "int64", "float32", "float64"], - op_name) + op_name, + ) if y is not None: check_variable_and_dtype( - y, "y", + y, + "y", ["bool", "int8", "int16", "int32", "int64", "float32", "float64"], - op_name) + op_name, + ) if out is not None: check_type(out, "out", Variable, op_name) @@ -12861,18 +13584,16 @@ def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): if binary_op and x.dtype != y.dtype: raise ValueError( "(InvalidArgument) The DataType of %s Op's Variable must be consistent, but received %s and %s." - % (op_name, x.dtype, y.dtype)) + % (op_name, x.dtype, y.dtype) + ) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) if binary_op: - helper.append_op(type=op_name, - inputs={ - "X": x, - "Y": y - }, - outputs={"Out": out}) + helper.append_op( + type=op_name, inputs={"X": x, "Y": y}, outputs={"Out": out} + ) else: helper.append_op(type=op_name, inputs={"X": x}, outputs={"Out": out}) @@ -12914,12 +13635,9 @@ def logical_and(x, y, out=None, name=None): if in_dygraph_mode(): return _C_ops.logical_and(x, y) - return _logical_op(op_name="logical_and", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_and", x=x, y=y, name=name, out=out, binary_op=True + ) def logical_or(x, y, out=None, name=None): @@ -12959,12 +13677,9 @@ def logical_or(x, y, out=None, name=None): """ if in_dygraph_mode(): return _C_ops.logical_or(x, y) - return _logical_op(op_name="logical_or", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_or", x=x, y=y, name=name, out=out, binary_op=True + ) def logical_xor(x, y, out=None, name=None): @@ -13005,12 +13720,9 @@ def logical_xor(x, y, out=None, name=None): if in_dygraph_mode(): return _C_ops.logical_xor(x, y) - return _logical_op(op_name="logical_xor", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_xor", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -13043,18 +13755,15 @@ def logical_not(x, out=None, name=None): """ if in_dygraph_mode(): return _C_ops.logical_not(x) - return _logical_op(op_name="logical_not", - x=x, - y=None, - name=name, - out=out, - binary_op=False) + return _logical_op( + op_name="logical_not", x=x, y=None, name=name, out=out, binary_op=False + ) @templatedoc() def clip(x, min, max, name=None): """ - :old_api: paddle.fluid.layers.clip + :old_api: paddle.fluid.layers.clip ${comment} @@ -13085,21 +13794,20 @@ def clip(x, min, max, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'clip') if name is None: - name = unique_name.generate_with_ignorable_key(".".join( - [helper.name, 'tmp'])) - - out = helper.create_variable(type=x.type, - name=name, - dtype=x.dtype, - persistable=False) - - helper.append_op(type="clip", - inputs={"X": x}, - attrs={ - "min": min, - "max": max - }, - outputs={"Out": out}) + name = unique_name.generate_with_ignorable_key( + ".".join([helper.name, 'tmp']) + ) + + out = helper.create_variable( + type=x.type, name=name, dtype=x.dtype, persistable=False + ) + + helper.append_op( + type="clip", + inputs={"X": x}, + attrs={"min": min, "max": max}, + outputs={"Out": out}, + ) return out @@ -13143,18 +13851,20 @@ def clip_by_norm(x, max_norm, name=None): check_type(max_norm, 'max_norm', (float), 'clip_by_norm') if name is None: - name = unique_name.generate_with_ignorable_key(".".join( - [helper.name, 'tmp'])) + name = unique_name.generate_with_ignorable_key( + ".".join([helper.name, 'tmp']) + ) - out = helper.create_variable(type=x.type, - name=name, - dtype=x.dtype, - persistable=False) + out = helper.create_variable( + type=x.type, name=name, dtype=x.dtype, persistable=False + ) - helper.append_op(type="clip_by_norm", - inputs={"X": x}, - attrs={"max_norm": max_norm}, - outputs={"Out": out}) + helper.append_op( + type="clip_by_norm", + inputs={"X": x}, + attrs={"max_norm": max_norm}, + outputs={"Out": out}, + ) return out @@ -13193,10 +13903,9 @@ def mean(x, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'mean') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="mean", - inputs={"X": x}, - attrs={}, - outputs={"Out": out}) + helper.append_op( + type="mean", inputs={"X": x}, attrs={}, outputs={"Out": out} + ) return out @@ -13231,10 +13940,12 @@ def merge_selected_rows(x, name=None): helper = LayerHelper("merge_selected_rows", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="merge_selected_rows", - inputs={"X": x}, - attrs={}, - outputs={"Out": out}) + helper.append_op( + type="merge_selected_rows", + inputs={"X": x}, + attrs={}, + outputs={"Out": out}, + ) return out @@ -13274,8 +13985,14 @@ def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None): """ if _non_static_mode(): - return _legacy_C_ops.mul(x, y, 'x_num_col_dims', x_num_col_dims, - 'y_num_col_dims', y_num_col_dims) + return _legacy_C_ops.mul( + x, + y, + 'x_num_col_dims', + x_num_col_dims, + 'y_num_col_dims', + y_num_col_dims, + ) inputs = {"X": [x], "Y": [y]} attrs = {"x_num_col_dims": x_num_col_dims, "y_num_col_dims": y_num_col_dims} @@ -13284,13 +14001,9 @@ def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None): check_variable_and_dtype(y, 'y', ['float16', 'float32', 'float64'], 'mul') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="mul", - inputs={ - "X": x, - "Y": y - }, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="mul", inputs={"X": x, "Y": y}, attrs=attrs, outputs={"Out": out} + ) return out @@ -13420,24 +14133,27 @@ def space_to_depth(x, blocksize, name=None): if not (isinstance(blocksize, int)): raise ValueError("blocksize must be a python Int") - check_variable_and_dtype(x, 'x', \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'space_to_depth') + check_variable_and_dtype( + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'space_to_depth', + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="space_to_depth", - inputs={"X": x}, - attrs={"blocksize": blocksize}, - outputs={"Out": out}) + helper.append_op( + type="space_to_depth", + inputs={"X": x}, + attrs={"blocksize": blocksize}, + outputs={"Out": out}, + ) return out -def affine_channel(x, - scale=None, - bias=None, - data_layout='NCHW', - name=None, - act=None): +def affine_channel( + x, scale=None, bias=None, data_layout='NCHW', name=None, act=None +): """ Applies a separate affine transformation to each channel of the input. @@ -13504,14 +14220,12 @@ def affine_channel(x, check_type(bias, 'bias', (Variable, type(None)), 'affine_channel') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="affine_channel", - inputs={ - "X": x, - 'Scale': scale, - 'Bias': bias - }, - attrs={"data_layout": data_layout}, - outputs={"Out": out}) + helper.append_op( + type="affine_channel", + inputs={"X": x, 'Scale': scale, 'Bias': bias}, + attrs={"data_layout": data_layout}, + outputs={"Out": out}, + ) return helper.append_activation(out) @@ -13610,8 +14324,9 @@ def similarity_focus(input, axis, indexes, name=None): """ helper = LayerHelper('similarity_focus', **locals()) # check attrs - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - "similarity_focus") + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], "similarity_focus" + ) check_type(axis, 'axis', int, "similarity_focus") check_type(indexes, 'indexes', list, "similarity_focus") if axis != 1 and axis != 2 and axis != 3: @@ -13620,13 +14335,12 @@ def similarity_focus(input, axis, indexes, name=None): raise ValueError("indexes can not be empty.") out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='similarity_focus', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - "axis": axis, - "indexes": indexes - }) + helper.append_op( + type='similarity_focus', + inputs={'X': input}, + outputs={'Out': out}, + attrs={"axis": axis, "indexes": indexes}, + ) return out @@ -13681,15 +14395,15 @@ def hash(input, hash_size, num_hash=1, name=None): check_type(hash_size, 'hash_size', int, 'hash') check_type(num_hash, 'num_hash', int, 'hash') helper = LayerHelper('hash', **locals()) - out = helper.create_variable_for_type_inference(helper.input_dtype(), - stop_gradient=True) - helper.append_op(type='hash', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'num_hash': num_hash, - 'mod_by': hash_size - }) + out = helper.create_variable_for_type_inference( + helper.input_dtype(), stop_gradient=True + ) + helper.append_op( + type='hash', + inputs={'X': input}, + outputs={'Out': out}, + attrs={'num_hash': num_hash, 'mod_by': hash_size}, + ) return out @@ -13783,8 +14497,9 @@ def grid_sampler(x, grid, name=None): helper = LayerHelper("grid_sampler", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'grid_sampler') - check_variable_and_dtype(grid, 'grid', ['float32', 'float64'], - 'grid_sampler') + check_variable_and_dtype( + grid, 'grid', ['float32', 'float64'], 'grid_sampler' + ) if not isinstance(x, Variable): return ValueError("The x should be a Variable") @@ -13796,10 +14511,9 @@ def grid_sampler(x, grid, name=None): attrs = {'use_cudnn': False} if core.is_compiled_with_rocm() else {} - helper.append_op(type='grid_sampler', - inputs=ipts, - outputs={'Output': out}, - attrs=attrs) + helper.append_op( + type='grid_sampler', inputs=ipts, outputs={'Output': out}, attrs=attrs + ) return out @@ -13892,33 +14606,30 @@ def add_position_encoding(input, alpha, beta, name=None): """ if _non_static_mode(): - return _legacy_C_ops.add_position_encoding(input, "alpha", alpha, - "beta", beta) + return _legacy_C_ops.add_position_encoding( + input, "alpha", alpha, "beta", beta + ) helper = LayerHelper('add_position_encoding', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - "add_position_encoding") + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], "add_position_encoding" + ) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type="add_position_encoding", - inputs={"X": input}, - outputs={"Out": out}, - attrs={ - "alpha": alpha, - "beta": beta - }) + helper.append_op( + type="add_position_encoding", + inputs={"X": input}, + outputs={"Out": out}, + attrs={"alpha": alpha, "beta": beta}, + ) return out -def bilinear_tensor_product(x, - y, - size, - act=None, - name=None, - param_attr=None, - bias_attr=None): +def bilinear_tensor_product( + x, y, size, act=None, name=None, param_attr=None, bias_attr=None +): r""" :api_attr: Static Graph @@ -13969,23 +14680,21 @@ def bilinear_tensor_product(x, param_shape = [size, x.shape[1], y.shape[1]] - w = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=param_shape, dtype=dtype, is_bias=False + ) out = helper.create_variable_for_type_inference(dtype=dtype) inputs = {"X": x, "Y": y, "Weight": w} if helper.bias_attr: bias_size = [1, size] - bias = helper.create_parameter(attr=helper.bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) inputs["Bias"] = bias - helper.append_op(type="bilinear_tensor_product", - inputs=inputs, - outputs={"Out": out}) + helper.append_op( + type="bilinear_tensor_product", inputs=inputs, outputs={"Out": out} + ) # add activation return helper.append_activation(out) @@ -14035,10 +14744,12 @@ def get_tensor_from_selected_rows(x, name=None): ) helper = LayerHelper('get_tensor_from_selected_rows', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='get_tensor_from_selected_rows', - inputs={'X': x}, - outputs={'Out': out}, - attrs={}) + helper.append_op( + type='get_tensor_from_selected_rows', + inputs={'X': x}, + outputs={'Out': out}, + attrs={}, + ) return out @@ -14108,10 +14819,12 @@ def shuffle_channel(x, group, name=None): if not isinstance(group, int): raise TypeError("group must be int type") - helper.append_op(type="shuffle_channel", - inputs={"X": x}, - outputs={"Out": out}, - attrs={"group": group}) + helper.append_op( + type="shuffle_channel", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"group": group}, + ) return out @@ -14149,8 +14862,9 @@ def temporal_shift(x, seg_num, shift_ratio=0.25, name=None, data_format="NCHW"): input = paddle.randn([6, 4, 2, 2]) out = F.temporal_shift(x=input, seg_num=2, shift_ratio=0.2) """ - return paddle.nn.functional.temporal_shift(x, seg_num, shift_ratio, name, - data_format) + return paddle.nn.functional.temporal_shift( + x, seg_num, shift_ratio, name, data_format + ) class PyFuncRegistry(object): @@ -14209,7 +14923,7 @@ class PyFuncRegistry(object): func_ret = self._func(*args[idx:], **kwargs) if not isinstance(func_ret, (list, tuple)): - func_ret = (func_ret, ) + func_ret = (func_ret,) ret = [] for each_ret in func_ret: @@ -14425,11 +15139,13 @@ def py_func(func, x, out, backward_func=None, skip_vars_in_backward_input=None): out_list = out else: raise TypeError( - 'Output must be Variable/list(Variable)/tuple(Variable)') + 'Output must be Variable/list(Variable)/tuple(Variable)' + ) fwd_func_id = PyFuncRegistry(func).id - bwd_func_id = PyFuncRegistry( - backward_func).id if backward_func is not None else -1 + bwd_func_id = ( + PyFuncRegistry(backward_func).id if backward_func is not None else -1 + ) for each_out in out_list: if len(each_out.shape) == 0: @@ -14449,18 +15165,22 @@ def py_func(func, x, out, backward_func=None, skip_vars_in_backward_input=None): for v in skip_vars_in_backward_input: if not v.name in fwd_in_out: raise ValueError( - 'Variable {} is not found in forward inputs and outputs'. - format(v.name)) + 'Variable {} is not found in forward inputs and outputs'.format( + v.name + ) + ) backward_skip_vars.add(v.name) - helper.append_op(type='py_func', - inputs={'X': x}, - outputs={'Out': out_list}, - attrs={ - 'forward_callable_id': fwd_func_id, - 'backward_callable_id': bwd_func_id, - 'backward_skip_vars': list(backward_skip_vars) - }) + helper.append_op( + type='py_func', + inputs={'X': x}, + outputs={'Out': out_list}, + attrs={ + 'forward_callable_id': fwd_func_id, + 'backward_callable_id': bwd_func_id, + 'backward_skip_vars': list(backward_skip_vars), + }, + ) return out @@ -14470,13 +15190,15 @@ py_func.registered_func_num = PyFuncRegistry.registered_func_num @templatedoc() -def psroi_pool(input, - rois, - output_channels, - spatial_scale, - pooled_height, - pooled_width, - name=None): +def psroi_pool( + input, + rois, + output_channels, + spatial_scale, + pooled_height, + pooled_width, + name=None, +): """ ${comment} @@ -14524,29 +15246,30 @@ def psroi_pool(input, raise TypeError("pooled_width must be int type") dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='psroi_pool', - inputs={ - 'X': input, - 'ROIs': rois - }, - outputs={'Out': out}, - attrs={ - 'output_channels': output_channels, - 'spatial_scale': spatial_scale, - 'pooled_height': pooled_height, - 'pooled_width': pooled_width - }) + helper.append_op( + type='psroi_pool', + inputs={'X': input, 'ROIs': rois}, + outputs={'Out': out}, + attrs={ + 'output_channels': output_channels, + 'spatial_scale': spatial_scale, + 'pooled_height': pooled_height, + 'pooled_width': pooled_width, + }, + ) return out @templatedoc() -def prroi_pool(input, - rois, - spatial_scale=1.0, - pooled_height=1, - pooled_width=1, - batch_roi_nums=None, - name=None): +def prroi_pool( + input, + rois, + spatial_scale=1.0, + pooled_height=1, + pooled_width=1, + batch_roi_nums=None, + name=None, +): """ The precise roi pooling implementation for paddle. Reference: https://arxiv.org/pdf/1807.11590.pdf @@ -14609,14 +15332,16 @@ def prroi_pool(input, inputs_op = {'X': input, 'ROIs': rois} if batch_roi_nums is not None: inputs_op['BatchRoINums'] = batch_roi_nums - helper.append_op(type='prroi_pool', - inputs=inputs_op, - outputs={'Out': out}, - attrs={ - 'spatial_scale': spatial_scale, - 'pooled_height': pooled_height, - 'pooled_width': pooled_width - }) + helper.append_op( + type='prroi_pool', + inputs=inputs_op, + outputs={'Out': out}, + attrs={ + 'spatial_scale': spatial_scale, + 'pooled_height': pooled_height, + 'pooled_width': pooled_width, + }, + ) return out @@ -14645,23 +15370,23 @@ def pixel_shuffle(x, upscale_factor): Examples: .. code-block:: python - # declarative mode - import paddle.fluid as fluid - import numpy as np - input = fluid.data(name="input", shape=[2,9,4,4]) - output = fluid.layers.pixel_shuffle(x=input, upscale_factor=3) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + # declarative mode + import paddle.fluid as fluid + import numpy as np + input = fluid.data(name="input", shape=[2,9,4,4]) + output = fluid.layers.pixel_shuffle(x=input, upscale_factor=3) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,9,4,4).astype("float32") - output_data = exe.run(fluid.default_main_program(), + input_data = np.random.rand(2,9,4,4).astype("float32") + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - # print(output.shape) - # (2L, 1L, 12L, 12L) + # print(output.shape) + # (2L, 1L, 12L, 12L) """ @@ -14673,10 +15398,12 @@ def pixel_shuffle(x, upscale_factor): if not isinstance(upscale_factor, int): raise TypeError("upscale factor must be int type") - helper.append_op(type="pixel_shuffle", - inputs={"X": x}, - outputs={"Out": out}, - attrs={"upscale_factor": upscale_factor}) + helper.append_op( + type="pixel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"upscale_factor": upscale_factor}, + ) return out @@ -14726,8 +15453,9 @@ def fsp_matrix(x, y): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'fsp_matrix') check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'fsp_matrix') helper = LayerHelper('fsp_matrix', **locals()) - out = helper.create_variable_for_type_inference(dtype=helper.input_dtype( - input_param_name='x')) + out = helper.create_variable_for_type_inference( + dtype=helper.input_dtype(input_param_name='x') + ) helper.append_op(type='fsp', inputs={'X': x, 'Y': y}, outputs={'Out': out}) return out @@ -14777,15 +15505,15 @@ def continuous_value_model(input, cvm, use_cvm=True): """ helper = LayerHelper('cvm', **locals()) out = helper.create_variable(dtype=input.dtype) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'cvm') - helper.append_op(type='cvm', - inputs={ - 'X': [input], - 'CVM': [cvm] - }, - outputs={'Y': [out]}, - attrs={"use_cvm": use_cvm}) + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'cvm' + ) + helper.append_op( + type='cvm', + inputs={'X': [input], 'CVM': [cvm]}, + outputs={'Y': [out]}, + attrs={"use_cvm": use_cvm}, + ) return out @@ -14831,11 +15559,14 @@ def where(condition): helper = LayerHelper("where_index", **locals()) out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) - - helper.append_op(type='where_index', - inputs={'Condition': condition}, - outputs={'Out': [out]}) + dtype=core.VarDesc.VarType.INT64 + ) + + helper.append_op( + type='where_index', + inputs={'Condition': condition}, + outputs={'Out': [out]}, + ) return out @@ -14894,21 +15625,21 @@ def unique(x, dtype='int32'): out, index = fluid.layers.unique(x) # out is [2, 3, 1, 5]; index is [0, 1, 1, 2, 3, 1] """ - check_variable_and_dtype(x, "x", ['float32', 'float64', 'int32', 'int64'], - "unique") + check_variable_and_dtype( + x, "x", ['float32', 'float64', 'int32', 'int64'], "unique" + ) helper = LayerHelper("unique", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) index = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='unique', - inputs={'X': x}, - attrs={'dtype': convert_np_dtype_to_dtype_(dtype)}, - outputs={ - 'Out': [out], - 'Index': [index] - }) + helper.append_op( + type='unique', + inputs={'X': x}, + attrs={'dtype': convert_np_dtype_to_dtype_(dtype)}, + outputs={'Out': [out], 'Index': [index]}, + ) return out, index @@ -14941,11 +15672,13 @@ def unique_with_counts(x, dtype='int32'): # count is [1, 3, 1, 1] # x.shape=(6,) out.shape=(4,), index.shape=(6,), count.shape=(4,) """ - check_variable_and_dtype(x, "x", ['float32', 'float64', 'int32', 'int64'], - "unique_with_counts") + check_variable_and_dtype( + x, "x", ['float32', 'float64', 'int32', 'int64'], "unique_with_counts" + ) if not (dtype == 'int32' or dtype == 'int64'): raise TypeError( - "Op unique_with_counts, index dtype must be int32 or int64") + "Op unique_with_counts, index dtype must be int32 or int64" + ) if x is None or len(x.shape) != 1: raise ValueError( @@ -14960,33 +15693,33 @@ def unique_with_counts(x, dtype='int32'): count = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='unique_with_counts', - inputs={'X': x}, - attrs={'dtype': convert_np_dtype_to_dtype_(dtype)}, - outputs={ - 'Out': [out], - 'Index': [index], - 'Count': [count] - }) + helper.append_op( + type='unique_with_counts', + inputs={'X': x}, + attrs={'dtype': convert_np_dtype_to_dtype_(dtype)}, + outputs={'Out': [out], 'Index': [index], 'Count': [count]}, + ) return out, index, count -def deformable_conv(input, - offset, - mask, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - deformable_groups=None, - im2col_step=None, - param_attr=None, - bias_attr=None, - modulated=True, - name=None): +def deformable_conv( + input, + offset, + mask, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + deformable_groups=None, + im2col_step=None, + param_attr=None, + bias_attr=None, + modulated=True, + name=None, +): r""" :api_attr: Static Graph @@ -15117,10 +15850,12 @@ def deformable_conv(input, num_filters=2, filter_size=filter_size, padding=1, modulated=False) """ - check_variable_and_dtype(input, "input", ['float32', 'float64'], - 'deformable_conv') - check_variable_and_dtype(offset, "offset", ['float32', 'float64'], - 'deformable_conv') + check_variable_and_dtype( + input, "input", ['float32', 'float64'], 'deformable_conv' + ) + check_variable_and_dtype( + offset, "offset", ['float32', 'float64'], 'deformable_conv' + ) check_type(mask, 'mask', (Variable, type(None)), 'deformable_conv') num_channels = input.shape[1] @@ -15155,52 +15890,58 @@ def deformable_conv(input, raise ValueError( "Invalid filter number, excepted number is larger than 0, but" " received {}, please check the input shape and " - "filter size.".format(filter_elem_num)) - std = (2.0 / filter_elem_num)**0.5 + "filter size.".format(filter_elem_num) + ) + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) filter_param = helper.create_parameter( attr=helper.param_attr, shape=filter_shape, dtype=dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) pre_bias = helper.create_variable_for_type_inference(dtype) if modulated: - helper.append_op(type='deformable_conv', - inputs={ - 'Input': input, - 'Filter': filter_param, - 'Offset': offset, - 'Mask': mask, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': stride, - 'paddings': padding, - 'dilations': dilation, - 'groups': groups, - 'deformable_groups': deformable_groups, - 'im2col_step': im2col_step, - }) + helper.append_op( + type='deformable_conv', + inputs={ + 'Input': input, + 'Filter': filter_param, + 'Offset': offset, + 'Mask': mask, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'deformable_groups': deformable_groups, + 'im2col_step': im2col_step, + }, + ) else: - helper.append_op(type='deformable_conv_v1', - inputs={ - 'Input': input, - 'Filter': filter_param, - 'Offset': offset, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': stride, - 'paddings': padding, - 'dilations': dilation, - 'groups': groups, - 'deformable_groups': deformable_groups, - 'im2col_step': im2col_step, - }) + helper.append_op( + type='deformable_conv_v1', + inputs={ + 'Input': input, + 'Filter': filter_param, + 'Offset': offset, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'deformable_groups': deformable_groups, + 'im2col_step': im2col_step, + }, + ) output = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) return output @@ -15276,23 +16017,26 @@ def unfold(x, kernel_sizes, strides=1, paddings=0, dilations=1, name=None): y = F.unfold(x, [3, 3], 1, 1, 1) """ - return paddle.nn.functional.unfold(x, kernel_sizes, strides, paddings, - dilations, name) - - -def deformable_roi_pooling(input, - rois, - trans, - no_trans=False, - spatial_scale=1.0, - group_size=[1, 1], - pooled_height=1, - pooled_width=1, - part_size=None, - sample_per_part=1, - trans_std=0.1, - position_sensitive=False, - name=None): + return paddle.nn.functional.unfold( + x, kernel_sizes, strides, paddings, dilations, name + ) + + +def deformable_roi_pooling( + input, + rois, + trans, + no_trans=False, + spatial_scale=1.0, + group_size=[1, 1], + pooled_height=1, + pooled_width=1, + part_size=None, + sample_per_part=1, + trans_std=0.1, + position_sensitive=False, + name=None, +): r""" Deformable ROI Pooling Layer @@ -15397,17 +16141,22 @@ def deformable_roi_pooling(input, position_sensitive=False) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'deformable_roi_pooling') - check_variable_and_dtype(rois, 'rois', ['float32', 'float64'], - 'deformable_roi_pooling') - check_variable_and_dtype(trans, 'trans', ['float32', 'float64'], - 'deformable_roi_pooling') - check_type(group_size, 'group_size', (list, tuple), - 'deformable_roi_pooling') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'deformable_roi_pooling' + ) + check_variable_and_dtype( + rois, 'rois', ['float32', 'float64'], 'deformable_roi_pooling' + ) + check_variable_and_dtype( + trans, 'trans', ['float32', 'float64'], 'deformable_roi_pooling' + ) + check_type( + group_size, 'group_size', (list, tuple), 'deformable_roi_pooling' + ) if part_size is not None: - check_type(part_size, 'part_size', (list, tuple), - 'deformable_roi_pooling') + check_type( + part_size, 'part_size', (list, tuple), 'deformable_roi_pooling' + ) input_channels = input.shape[1] if position_sensitive == False: @@ -15425,27 +16174,22 @@ def deformable_roi_pooling(input, dtype = helper.input_dtype() output = helper.create_variable_for_type_inference(dtype) top_count = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type="deformable_psroi_pooling", - inputs={ - "Input": input, - "ROIs": rois, - "Trans": trans - }, - outputs={ - "Output": output, - "TopCount": top_count - }, - attrs={ - "no_trans": no_trans, - "spatial_scale": spatial_scale, - "output_dim": output_channels, - "group_size": group_size, - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "part_size": part_size, - "sample_per_part": sample_per_part, - "trans_std": trans_std - }) + helper.append_op( + type="deformable_psroi_pooling", + inputs={"Input": input, "ROIs": rois, "Trans": trans}, + outputs={"Output": output, "TopCount": top_count}, + attrs={ + "no_trans": no_trans, + "spatial_scale": spatial_scale, + "output_dim": output_channels, + "group_size": group_size, + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "part_size": part_size, + "sample_per_part": sample_per_part, + "trans_std": trans_std, + }, + ) return output @@ -15497,27 +16241,31 @@ def shard_index(input, index_num, nshards, shard_id, ignore_value=-1): # [[-1], [1]] """ if in_dygraph_mode(): - return _C_ops.shard_index(input, index_num, nshards, shard_id, - ignore_value) + return _C_ops.shard_index( + input, index_num, nshards, shard_id, ignore_value + ) check_variable_and_dtype(input, 'input', ['int64', 'int32'], 'shard_index') op_type = 'shard_index' helper = LayerHelper(op_type, **locals()) if shard_id < 0 or shard_id >= nshards: - raise ValueError('The shard_id(%d) should be in [0, %d)' % - (shard_id, nshards)) + raise ValueError( + 'The shard_id(%d) should be in [0, %d)' % (shard_id, nshards) + ) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type=op_type, - inputs={'X': [input]}, - outputs={'Out': out}, - attrs={ - 'index_num': index_num, - 'nshards': nshards, - 'shard_id': shard_id, - 'ignore_value': ignore_value - }, - stop_gradient=True) + helper.append_op( + type=op_type, + inputs={'X': [input]}, + outputs={'Out': out}, + attrs={ + 'index_num': index_num, + 'nshards': nshards, + 'shard_id': shard_id, + 'ignore_value': ignore_value, + }, + stop_gradient=True, + ) return out @@ -15572,22 +16320,22 @@ def hard_swish(x, threshold=6.0, scale=6.0, offset=3.0, name=None): print(out) # [[0.66666667, 1.66666667,3., 4.]] """ if _non_static_mode(): - return _legacy_C_ops.hard_swish(x, 'threshold', threshold, 'scale', - scale, 'offset', offset) + return _legacy_C_ops.hard_swish( + x, 'threshold', threshold, 'scale', scale, 'offset', offset + ) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hard_swish') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hard_swish' + ) helper = LayerHelper('hard_swish', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='hard_swish', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'threshold': threshold, - 'scale': scale, - 'offset': offset - }) + helper.append_op( + type='hard_swish', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold, 'scale': scale, 'offset': offset}, + ) return out @@ -15658,15 +16406,20 @@ def mish(x, threshold=20, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'mish') check_type(threshold, 'threshold', (float, int), 'mish') - assert threshold > 0, "threshold of mish should be greater than 0, " \ - "but got {}".format(threshold) + assert ( + threshold > 0 + ), "threshold of mish should be greater than 0, " "but got {}".format( + threshold + ) helper = LayerHelper('mish', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='mish', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='mish', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out @@ -15736,12 +16489,9 @@ def gather_tree(ids, parents): @deprecated(since="2.0.0", update_to="paddle.uniform") @templatedoc() -def uniform_random(shape, - dtype='float32', - min=-1.0, - max=1.0, - seed=0, - name=None): +def uniform_random( + shape, dtype='float32', min=-1.0, max=1.0, seed=0, name=None +): """ This OP returns a Tensor filled with random values sampled from a uniform distribution in the range [``min``, ``max``), with ``shape`` and ``dtype``. @@ -15821,34 +16571,47 @@ def uniform_random(shape, if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) - return _C_ops.uniform_random(shape, dtype, float(min), float(max), seed, - _current_expected_place()) + return _C_ops.uniform_random( + shape, + dtype, + float(min), + float(max), + seed, + _current_expected_place(), + ) elif _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.uniform_random('shape', - shape, 'min', float(min), 'max', - float(max), 'seed', seed, 'dtype', - dtype) + return _legacy_C_ops.uniform_random( + 'shape', + shape, + 'min', + float(min), + 'max', + float(max), + 'seed', + seed, + 'dtype', + dtype, + ) check_type(shape, 'shape', (list, tuple, Variable), 'uniform_random/rand') - check_dtype(dtype, 'dtype', ('float32', 'float64', 'uint16'), - 'uniform_random/rand') + check_dtype( + dtype, 'dtype', ('float32', 'float64', 'uint16'), 'uniform_random/rand' + ) check_type(min, 'min', (float, int, Variable), 'uniform_random/rand') check_type(max, 'max', (float, int, Variable), 'uniform_random/rand') inputs = dict() attrs = {'seed': seed, 'min': min, 'max': max, 'dtype': dtype} - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='uniform_random/rand') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='uniform_random/rand' + ) helper = LayerHelper("uniform_random", **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="uniform_random", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="uniform_random", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) utils.try_set_static_shape_tensor(out, shape) return out @@ -15884,11 +16647,13 @@ def unbind(input, axis=0): helper = LayerHelper("unbind", **locals()) check_type(input, 'input', (Variable), 'unbind') dtype = helper.input_dtype() - check_dtype(dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], - 'unbind') + check_dtype( + dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], 'unbind' + ) if not isinstance(axis, (int)): - raise TypeError("The type of 'axis' must be int, but received %s." % - (type(axis))) + raise TypeError( + "The type of 'axis' must be int, but received %s." % (type(axis)) + ) if isinstance(axis, np.generic): axis = np.asscalar(axis) input_shape = input.shape @@ -15899,8 +16664,10 @@ def unbind(input, axis=0): for i in range(num) ] - helper.append_op(type="unbind", - inputs={"X": input}, - outputs={"Out": outs}, - attrs={"axis": axis}) + helper.append_op( + type="unbind", + inputs={"X": input}, + outputs={"Out": outs}, + attrs={"axis": axis}, + ) return outs diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index b1b4a1a55f9..61e12a98a01 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -13,17 +13,27 @@ # limitations under the License. import os -from .layer_function_generator import generate_layer_fn, generate_activation_fn, generate_inplace_fn, add_sample_code +from .layer_function_generator import ( + generate_layer_fn, + generate_activation_fn, + generate_inplace_fn, + add_sample_code, +) from .. import core from ..framework import convert_np_dtype_to_dtype_, Variable, in_dygraph_mode -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from paddle.utils import deprecated from paddle import _C_ops, _legacy_C_ops import paddle __deprecated_func_name__ = { 'tanh_shrink': 'tanhshrink', - 'logsigmoid': 'log_sigmoid' + 'logsigmoid': 'log_sigmoid', } __activations_noattr__ = [ @@ -36,9 +46,28 @@ __activations_noattr__ = [ ] __unary_func__ = [ - 'exp', 'expm1', 'atan', 'sqrt', 'rsqrt', 'abs', 'ceil', 'floor', 'cos', - 'tan', 'acos', 'sin', 'sinh', 'asin', 'cosh', 'round', 'reciprocal', - 'square', 'acosh', 'asinh', 'atanh', 'lgamma' + 'exp', + 'expm1', + 'atan', + 'sqrt', + 'rsqrt', + 'abs', + 'ceil', + 'floor', + 'cos', + 'tan', + 'acos', + 'sin', + 'sinh', + 'asin', + 'cosh', + 'round', + 'reciprocal', + 'square', + 'acosh', + 'asinh', + 'atanh', + 'lgamma', ] __inplace_unary_func__ = [ @@ -80,8 +109,9 @@ for _OP in set(__activations_noattr__): if _OP in __deprecated_func_name__: _new_OP = __deprecated_func_name__[_OP] _func = generate_activation_fn(_OP) - _func = deprecated(since="2.0.0", - update_to="paddle.nn.functional.%s" % (_new_OP))(_func) + _func = deprecated( + since="2.0.0", update_to="paddle.nn.functional.%s" % (_new_OP) + )(_func) globals()[_OP] = _func for _OP in set(__unary_func__): @@ -101,7 +131,8 @@ for _OP in set(__inplace_unary_func__): globals()[_OP] = _func add_sample_code( - globals()["sigmoid"], r""" + globals()["sigmoid"], + r""" Examples: .. code-block:: python @@ -113,10 +144,12 @@ Examples: print(out) # [0.40131234 0.450166 0.52497919 0.57444252] -""") +""", +) add_sample_code( - globals()["silu"], r""" + globals()["silu"], + r""" Examples: .. code-block:: python @@ -128,10 +161,12 @@ Examples: print(out) # [ 0.7310586 1.7615942 2.8577224, 3.9280552 ] -""") +""", +) add_sample_code( - globals()["logsigmoid"], r""" + globals()["logsigmoid"], + r""" Examples: .. code-block:: python @@ -143,10 +178,12 @@ Examples: print(out) # [-0.91301525 -0.79813887 -0.64439666 -0.55435524] -""") +""", +) add_sample_code( - globals()["exp"], r""" + globals()["exp"], + r""" Examples: .. code-block:: python @@ -157,10 +194,12 @@ Examples: print(out) # [0.67032005 0.81873075 1.10517092 1.34985881] -""") +""", +) add_sample_code( - globals()["expm1"], r""" + globals()["expm1"], + r""" Examples: .. code-block:: python @@ -171,10 +210,12 @@ Examples: print(out) # [-0.32967997, -0.18126924, 0.10517092, 0.34985882] -""") +""", +) add_sample_code( - globals()["tanh"], r""" + globals()["tanh"], + r""" Examples: .. code-block:: python @@ -185,10 +226,12 @@ Examples: print(out) # [-0.37994896 -0.19737532 0.09966799 0.29131261] -""") +""", +) add_sample_code( - globals()["atan"], r""" + globals()["atan"], + r""" Examples: .. code-block:: python @@ -199,10 +242,12 @@ Examples: print(out) # [-0.38050638 -0.19739556 0.09966865 0.29145679] -""") +""", +) add_sample_code( - globals()["tanh_shrink"], r""" + globals()["tanh_shrink"], + r""" Examples: .. code-block:: python @@ -214,10 +259,12 @@ Examples: print(out) # [-0.020051, -0.00262468, 0.000332005, 0.00868739] -""") +""", +) add_sample_code( - globals()["sqrt"], r""" + globals()["sqrt"], + r""" Examples: .. code-block:: python @@ -228,10 +275,12 @@ Examples: print(out) # [0.31622777 0.4472136 0.54772256 0.63245553] -""") +""", +) add_sample_code( - globals()["rsqrt"], r""" + globals()["rsqrt"], + r""" Examples: .. code-block:: python @@ -242,10 +291,12 @@ Examples: print(out) # [3.16227766 2.23606798 1.82574186 1.58113883] -""") +""", +) add_sample_code( - globals()["abs"], r""" + globals()["abs"], + r""" Examples: .. code-block:: python @@ -256,10 +307,12 @@ Examples: print(out) # [0.4 0.2 0.1 0.3] -""") +""", +) add_sample_code( - globals()["ceil"], r""" + globals()["ceil"], + r""" Examples: .. code-block:: python @@ -270,10 +323,12 @@ Examples: print(out) # [-0. -0. 1. 1.] -""") +""", +) add_sample_code( - globals()["floor"], r""" + globals()["floor"], + r""" Examples: .. code-block:: python @@ -284,10 +339,12 @@ Examples: print(out) # [-1. -1. 0. 0.] -""") +""", +) add_sample_code( - globals()["cos"], r""" + globals()["cos"], + r""" Examples: .. code-block:: python @@ -298,10 +355,12 @@ Examples: print(out) # [0.92106099 0.98006658 0.99500417 0.95533649] -""") +""", +) add_sample_code( - globals()["tan"], r""" + globals()["tan"], + r""" Examples: .. code-block:: python @@ -312,10 +371,12 @@ Examples: print(out) # [-0.42279324, -0.20271005, 0.10033467, 0.30933627] -""") +""", +) add_sample_code( - globals()["acos"], r""" + globals()["acos"], + r""" Examples: .. code-block:: python @@ -326,10 +387,12 @@ Examples: print(out) # [1.98231317 1.77215425 1.47062891 1.26610367] -""") +""", +) add_sample_code( - globals()["sin"], r""" + globals()["sin"], + r""" Examples: .. code-block:: python @@ -340,10 +403,12 @@ Examples: print(out) # [-0.38941834 -0.19866933 0.09983342 0.29552021] -""") +""", +) add_sample_code( - globals()["asin"], r""" + globals()["asin"], + r""" Examples: .. code-block:: python @@ -354,10 +419,12 @@ Examples: print(out) # [-0.41151685 -0.20135792 0.10016742 0.30469265] -""") +""", +) add_sample_code( - globals()["cosh"], r""" + globals()["cosh"], + r""" Examples: .. code-block:: python @@ -368,10 +435,12 @@ Examples: print(out) # [1.08107237 1.02006676 1.00500417 1.04533851] -""") +""", +) add_sample_code( - globals()["sinh"], r""" + globals()["sinh"], + r""" Examples: .. code-block:: python @@ -382,10 +451,12 @@ Examples: print(out) # [-0.41075233 -0.201336 0.10016675 0.30452029] -""") +""", +) add_sample_code( - globals()["asinh"], r""" + globals()["asinh"], + r""" Examples: .. code-block:: python @@ -396,10 +467,12 @@ Examples: print(out) # [-0.39003533, -0.19869010, 0.09983408, 0.29567307] -""") +""", +) add_sample_code( - globals()["acosh"], r""" + globals()["acosh"], + r""" Examples: .. code-block:: python @@ -410,10 +483,12 @@ Examples: print(out) # [0. , 1.76274729, 2.06343699, 2.29243159] -""") +""", +) add_sample_code( - globals()["atanh"], r""" + globals()["atanh"], + r""" Examples: .. code-block:: python @@ -424,10 +499,12 @@ Examples: print(out) # [-0.42364895, -0.20273256, 0.10033535, 0.30951962] -""") +""", +) add_sample_code( - globals()["round"], r""" + globals()["round"], + r""" Examples: .. code-block:: python @@ -438,10 +515,12 @@ Examples: print(out) # [-1. -0. 1. 2.] -""") +""", +) add_sample_code( - globals()["reciprocal"], r""" + globals()["reciprocal"], + r""" Examples: .. code-block:: python @@ -452,10 +531,12 @@ Examples: print(out) # [-2.5 -5. 10. 3.33333333] -""") +""", +) add_sample_code( - globals()["square"], r""" + globals()["square"], + r""" Examples: .. code-block:: python @@ -466,7 +547,8 @@ Examples: print(out) # [0.16 0.04 0.01 0.09] -""") +""", +) _softplus_ = generate_layer_fn('softplus') @@ -515,7 +597,8 @@ Examples: """ add_sample_code( - globals()["softsign"], r""" + globals()["softsign"], + r""" Examples: .. code-block:: python @@ -527,14 +610,16 @@ Examples: print(out) # [-0.285714, -0.166667, 0.0909091, 0.230769] -""") +""", +) _softshrink_ = generate_layer_fn('softshrink') def softshrink(x, alpha=None): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softshrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softshrink' + ) locals_var = locals().copy() kwargs = dict() @@ -582,8 +667,9 @@ _hard_shrink_ = generate_layer_fn('hard_shrink') @deprecated(since="2.0.0", update_to="paddle.nn.functional.hardshrink") def hard_shrink(x, threshold=None): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hard_shrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hard_shrink' + ) locals_var = locals().copy() kwargs = dict() @@ -593,20 +679,25 @@ def hard_shrink(x, threshold=None): return _hard_shrink_(**kwargs) -hard_shrink.__doc__ = _hard_shrink_.__doc__ + """ +hard_shrink.__doc__ = ( + _hard_shrink_.__doc__ + + """ Examples: >>> import paddle.fluid as fluid >>> data = fluid.layers.data(name="input", shape=[784]) >>> result = fluid.layers.hard_shrink(x=data, threshold=0.3) """ +) _cum_sum_ = generate_layer_fn('cumsum') -@deprecated(since="2.0.0", - update_to="paddle.cumsum", - reason="New APIs for Paddle 2.0 are coming.") +@deprecated( + since="2.0.0", + update_to="paddle.cumsum", + reason="New APIs for Paddle 2.0 are coming.", +) def cumsum(x, axis=None, exclusive=None, reverse=None): check_type(x, 'x', (Variable), 'cumsum') locals_var = locals().copy() @@ -645,8 +736,9 @@ _thresholded_relu_ = generate_layer_fn('thresholded_relu') def thresholded_relu(x, threshold=None): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'thresholded_relu') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'thresholded_relu' + ) locals_var = locals().copy() kwargs = dict() diff --git a/python/paddle/fluid/layers/rnn.py b/python/paddle/fluid/layers/rnn.py index 087cffcfe3b..f2ab7da4c84 100644 --- a/python/paddle/fluid/layers/rnn.py +++ b/python/paddle/fluid/layers/rnn.py @@ -31,6 +31,7 @@ from ..layer_helper import LayerHelper from ..framework import _non_static_mode from ..param_attr import ParamAttr from ..data_feeder import check_variable_and_dtype, check_type, check_dtype + try: from collections.abc import Sequence except: @@ -63,7 +64,7 @@ __all__ = [ class RNNCell(object): """ - :api_attr: Static Graph + :api_attr: Static Graph RNNCell is the base class for abstraction representing the calculations mapping the input and state to the output and new state. It is suitable to @@ -95,12 +96,14 @@ class RNNCell(object): def __call__(self, inputs, states, **kwargs): return self.call(inputs, states, **kwargs) - def get_initial_states(self, - batch_ref, - shape=None, - dtype='float32', - init_value=0, - batch_dim_idx=0): + def get_initial_states( + self, + batch_ref, + shape=None, + dtype='float32', + init_value=0, + batch_dim_idx=0, + ): r""" Generate initialized states according to provided shape, data type and value. @@ -126,9 +129,12 @@ class RNNCell(object): Variable: tensor variable[s] packed in the same structure provided \ by shape, representing the initialized states. """ - check_variable_and_dtype(batch_ref, 'batch_ref', - ['float32', 'float64', 'int32', 'int64'], - 'RNNCell') + check_variable_and_dtype( + batch_ref, + 'batch_ref', + ['float32', 'float64', 'int32', 'int64'], + 'RNNCell', + ) check_type(shape, 'shape', (list, tuple, type(None), int), 'RNNCell') if isinstance(shape, (list, tuple)): shapes = map_structure(lambda x: x, shape) @@ -144,17 +150,17 @@ class RNNCell(object): def _is_shape_sequence(seq): """For shape, list/tuple of integer is the finest-grained objection""" - if (isinstance(seq, list) or isinstance(seq, tuple)): - if reduce(lambda flag, x: isinstance(x, int) and flag, seq, - True): + if isinstance(seq, list) or isinstance(seq, tuple): + if reduce( + lambda flag, x: isinstance(x, int) and flag, seq, True + ): return False # TODO: Add check for the illegal if isinstance(seq, dict): return True - return (isinstance(seq, Sequence) and not isinstance(seq, str)) + return isinstance(seq, Sequence) and not isinstance(seq, str) class Shape(object): - def __init__(self, shape): self.shape = shape if shape[0] == -1 else ([-1] + list(shape)) @@ -180,7 +186,11 @@ class RNNCell(object): shape=shape.shape, dtype=dtype, value=init_value, - input_dim_idx=batch_dim_idx), states_shapes, states_dtypes) + input_dim_idx=batch_dim_idx, + ), + states_shapes, + states_dtypes, + ) return init_states @property @@ -196,7 +206,8 @@ class RNNCell(object): `get_initial_states`. """ raise NotImplementedError( - "Please add implementaion for `state_shape` in the used cell.") + "Please add implementaion for `state_shape` in the used cell." + ) @property def state_dtype(self): @@ -211,12 +222,13 @@ class RNNCell(object): `get_initial_states`. """ raise NotImplementedError( - "Please add implementaion for `state_dtype` in the used cell.") + "Please add implementaion for `state_dtype` in the used cell." + ) class GRUCell(RNNCell): r""" - :api_attr: Static Graph + :api_attr: Static Graph Gated Recurrent Unit cell. It is a wrapper for `fluid.contrib.layers.rnn_impl.BasicGRUUnit` to make it adapt to RNNCell. @@ -244,14 +256,16 @@ class GRUCell(RNNCell): cell = layers.GRUCell(hidden_size=256) """ - def __init__(self, - hidden_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - dtype="float32", - name="GRUCell"): + def __init__( + self, + hidden_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + dtype="float32", + name="GRUCell", + ): """ Constructor of GRUCell. @@ -272,9 +286,16 @@ class GRUCell(RNNCell): check_dtype(dtype, 'dtype', ['float32', 'float64'], 'GRUCell') self.hidden_size = hidden_size from .. import contrib # TODO: resolve recurrent import + self.gru_unit = contrib.layers.rnn_impl.BasicGRUUnit( - name, hidden_size, param_attr, bias_attr, gate_activation, - activation, dtype) + name, + hidden_size, + param_attr, + bias_attr, + gate_activation, + activation, + dtype, + ) def call(self, inputs, states): r""" @@ -295,10 +316,12 @@ class GRUCell(RNNCell): tensor is same as that of `states`. """ - check_variable_and_dtype(inputs, 'inputs', ['float32', 'float64'], - 'GRUCell') - check_variable_and_dtype(states, 'states', ['float32', 'float64'], - 'GRUCell') + check_variable_and_dtype( + inputs, 'inputs', ['float32', 'float64'], 'GRUCell' + ) + check_variable_and_dtype( + states, 'states', ['float32', 'float64'], 'GRUCell' + ) new_hidden = self.gru_unit(inputs, states) return new_hidden, new_hidden @@ -314,7 +337,7 @@ class GRUCell(RNNCell): class LSTMCell(RNNCell): r""" - :api_attr: Static Graph + :api_attr: Static Graph Long-Short Term Memory cell. It is a wrapper for `fluid.contrib.layers.rnn_impl.BasicLSTMUnit` to make it adapt to RNNCell. @@ -343,15 +366,17 @@ class LSTMCell(RNNCell): cell = layers.LSTMCell(hidden_size=256) """ - def __init__(self, - hidden_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - dtype="float32", - name="LSTMCell"): + def __init__( + self, + hidden_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + dtype="float32", + name="LSTMCell", + ): """ Constructor of LSTMCell. @@ -375,9 +400,17 @@ class LSTMCell(RNNCell): check_dtype(dtype, 'dtype', ['float32', 'float64'], 'LSTMCell') self.hidden_size = hidden_size from .. import contrib # TODO: resolve recurrent import + self.lstm_unit = contrib.layers.rnn_impl.BasicLSTMUnit( - name, hidden_size, param_attr, bias_attr, gate_activation, - activation, forget_bias, dtype) + name, + hidden_size, + param_attr, + bias_attr, + gate_activation, + activation, + forget_bias, + dtype, + ) def call(self, inputs, states): r""" @@ -400,13 +433,18 @@ class LSTMCell(RNNCell): tensors all is same as that of `states`. """ - check_variable_and_dtype(inputs, 'inputs', ['float32', 'float64'], - 'LSTMCell') + check_variable_and_dtype( + inputs, 'inputs', ['float32', 'float64'], 'LSTMCell' + ) check_type(states, 'states', list, 'LSTMCell') if isinstance(states, list): for i, state in enumerate(states): - check_variable_and_dtype(state, 'state[' + str(i) + ']', - ['float32', 'float64'], 'LSTMCell') + check_variable_and_dtype( + state, + 'state[' + str(i) + ']', + ['float32', 'float64'], + 'LSTMCell', + ) pre_hidden, pre_cell = states new_hidden, new_cell = self.lstm_unit(inputs, pre_hidden, pre_cell) @@ -422,13 +460,15 @@ class LSTMCell(RNNCell): return [[self.hidden_size], [self.hidden_size]] -def rnn(cell, - inputs, - initial_states=None, - sequence_length=None, - time_major=False, - is_reverse=False, - **kwargs): +def rnn( + cell, + inputs, + initial_states=None, + sequence_length=None, + time_major=False, + is_reverse=False, + **kwargs +): """ rnn creates a recurrent neural network specified by RNNCell `cell`, which performs :code:`cell.call()` (for dygraph mode :code:`cell.forward`) @@ -483,15 +523,28 @@ def rnn(cell, """ if _non_static_mode(): - return _rnn_dynamic_graph(cell, inputs, initial_states, sequence_length, - time_major, is_reverse, **kwargs) + return _rnn_dynamic_graph( + cell, + inputs, + initial_states, + sequence_length, + time_major, + is_reverse, + **kwargs + ) else: - return _rnn_static_graph(cell, inputs, initial_states, sequence_length, - time_major, is_reverse, **kwargs) + return _rnn_static_graph( + cell, + inputs, + initial_states, + sequence_length, + time_major, + is_reverse, + **kwargs + ) class ArrayWrapper(object): - def __init__(self, x): self.array = [x] @@ -505,8 +558,9 @@ class ArrayWrapper(object): def _maybe_copy(state, new_state, step_mask): """update rnn state or just pass the old state through""" - new_state = nn.elementwise_mul(new_state, step_mask, axis=0) \ - + nn.elementwise_mul(state, (1 - step_mask), axis=0) + new_state = nn.elementwise_mul( + new_state, step_mask, axis=0 + ) + nn.elementwise_mul(state, (1 - step_mask), axis=0) return new_state @@ -515,34 +569,40 @@ def _transpose_batch_time(x): return nn.transpose(x, perm) -def _rnn_dynamic_graph(cell, - inputs, - initial_states=None, - sequence_length=None, - time_major=False, - is_reverse=False, - **kwargs): +def _rnn_dynamic_graph( + cell, + inputs, + initial_states=None, + sequence_length=None, + time_major=False, + is_reverse=False, + **kwargs +): time_step_index = 0 if time_major else 1 flat_inputs = flatten(inputs) time_steps = flat_inputs[0].shape[time_step_index] if initial_states is None: initial_states = cell.get_initial_states( - batch_ref=inputs, batch_dim_idx=1 if time_major else 0) + batch_ref=inputs, batch_dim_idx=1 if time_major else 0 + ) if not time_major: inputs = map_structure(_transpose_batch_time, inputs) if sequence_length is not None: - mask = sequence_lod.sequence_mask(sequence_length, - maxlen=time_steps, - dtype=inputs.dtype) + mask = sequence_lod.sequence_mask( + sequence_length, maxlen=time_steps, dtype=inputs.dtype + ) mask = nn.transpose(mask, [1, 0]) if is_reverse: inputs = map_structure(lambda x: tensor.reverse(x, axis=[0]), inputs) - mask = tensor.reverse(mask, axis=[0]) \ - if sequence_length is not None else None + mask = ( + tensor.reverse(mask, axis=[0]) + if sequence_length is not None + else None + ) states = initial_states outputs = [] @@ -550,42 +610,56 @@ def _rnn_dynamic_graph(cell, step_inputs = map_structure(lambda x: x[i], inputs) step_outputs, new_states = cell(step_inputs, states, **kwargs) if sequence_length is not None: - new_states = map_structure(partial(_maybe_copy, step_mask=mask[i]), - states, new_states) + new_states = map_structure( + partial(_maybe_copy, step_mask=mask[i]), states, new_states + ) states = new_states - outputs = map_structure(lambda x: ArrayWrapper(x), - step_outputs) if i == 0 else map_structure( - lambda x, x_array: x_array.append(x), - step_outputs, outputs) + outputs = ( + map_structure(lambda x: ArrayWrapper(x), step_outputs) + if i == 0 + else map_structure( + lambda x, x_array: x_array.append(x), step_outputs, outputs + ) + ) final_outputs = map_structure( - lambda x: nn.stack(x.array, axis=time_step_index), outputs) + lambda x: nn.stack(x.array, axis=time_step_index), outputs + ) if is_reverse: final_outputs = map_structure( - lambda x: tensor.reverse(x, axis=time_step_index), final_outputs) + lambda x: tensor.reverse(x, axis=time_step_index), final_outputs + ) final_states = new_states return final_outputs, final_states -def _rnn_static_graph(cell, - inputs, - initial_states=None, - sequence_length=None, - time_major=False, - is_reverse=False, - **kwargs): +def _rnn_static_graph( + cell, + inputs, + initial_states=None, + sequence_length=None, + time_major=False, + is_reverse=False, + **kwargs +): check_type(inputs, 'inputs', (Variable, list, tuple), 'rnn') if isinstance(inputs, (list, tuple)): for i, input_x in enumerate(inputs): - check_variable_and_dtype(input_x, 'inputs[' + str(i) + ']', - ['float32', 'float64'], 'rnn') - check_type(initial_states, 'initial_states', - (Variable, list, tuple, type(None)), 'rnn') + check_variable_and_dtype( + input_x, 'inputs[' + str(i) + ']', ['float32', 'float64'], 'rnn' + ) + check_type( + initial_states, + 'initial_states', + (Variable, list, tuple, type(None)), + 'rnn', + ) - check_type(sequence_length, 'sequence_length', (Variable, type(None)), - 'rnn') + check_type( + sequence_length, 'sequence_length', (Variable, type(None)), 'rnn' + ) def _switch_grad(x, stop=False): x.stop_gradient = stop @@ -593,7 +667,8 @@ def _rnn_static_graph(cell, if initial_states is None: initial_states = cell.get_initial_states( - batch_ref=inputs, batch_dim_idx=1 if time_major else 0) + batch_ref=inputs, batch_dim_idx=1 if time_major else 0 + ) initial_states = map_structure(_switch_grad, initial_states) if not time_major: @@ -604,7 +679,8 @@ def _rnn_static_graph(cell, mask = sequence_lod.sequence_mask( sequence_length, maxlen=max_seq_len, - dtype=flatten(initial_states)[0].dtype) + dtype=flatten(initial_states)[0].dtype, + ) mask = nn.transpose(mask, [1, 0]) if is_reverse: inputs = map_structure(lambda x: tensor.reverse(x, axis=[0]), inputs) @@ -621,7 +697,8 @@ def _rnn_static_graph(cell, if sequence_length: step_mask = rnn.step_input(mask) new_states = map_structure( - partial(_maybe_copy, step_mask=step_mask), states, new_states) + partial(_maybe_copy, step_mask=step_mask), states, new_states + ) map_structure(rnn.update_memory, states, new_states) flat_outputs = flatten(outputs) @@ -629,14 +706,15 @@ def _rnn_static_graph(cell, map_structure(rnn.step_output, new_states) rnn_out = rnn() - final_outputs = rnn_out[:len(flat_outputs)] + final_outputs = rnn_out[: len(flat_outputs)] final_outputs = pack_sequence_as(outputs, final_outputs) - final_states = map_structure(lambda x: x[-1], rnn_out[len(flat_outputs):]) + final_states = map_structure(lambda x: x[-1], rnn_out[len(flat_outputs) :]) final_states = pack_sequence_as(new_states, final_states) if is_reverse: - final_outputs = map_structure(lambda x: tensor.reverse(x, axis=[0]), - final_outputs) + final_outputs = map_structure( + lambda x: tensor.reverse(x, axis=[0]), final_outputs + ) if not time_major: final_outputs = map_structure(_transpose_batch_time, final_outputs) @@ -644,13 +722,15 @@ def _rnn_static_graph(cell, return (final_outputs, final_states) -def birnn(cell_fw, - cell_bw, - inputs, - initial_states=None, - sequence_length=None, - time_major=False, - **kwargs): +def birnn( + cell_fw, + cell_bw, + inputs, + initial_states=None, + sequence_length=None, + time_major=False, + **kwargs +): """ birnn creates a bidirectional recurrent neural network specified by RNNCell `cell_fw` and `cell_bw`, which performs :code:`cell.call()` @@ -709,28 +789,35 @@ def birnn(cell_fw, """ if initial_states is None: states_fw = cell_fw.get_initial_states( - batch_ref=inputs, batch_dim_idx=1 if time_major else 0) + batch_ref=inputs, batch_dim_idx=1 if time_major else 0 + ) states_bw = cell_fw.get_initial_states( - batch_ref=inputs, batch_dim_idx=1 if time_major else 0) + batch_ref=inputs, batch_dim_idx=1 if time_major else 0 + ) else: states_fw, states_bw = initial_states - outputs_fw, states_fw = rnn(cell_fw, - inputs, - states_fw, - sequence_length, - time_major=time_major, - **kwargs) - - outputs_bw, states_bw = rnn(cell_bw, - inputs, - states_bw, - sequence_length, - time_major=time_major, - is_reverse=True, - **kwargs) - - outputs = map_structure(lambda x, y: tensor.concat([x, y], -1), outputs_fw, - outputs_bw) + outputs_fw, states_fw = rnn( + cell_fw, + inputs, + states_fw, + sequence_length, + time_major=time_major, + **kwargs + ) + + outputs_bw, states_bw = rnn( + cell_bw, + inputs, + states_bw, + sequence_length, + time_major=time_major, + is_reverse=True, + **kwargs + ) + + outputs = map_structure( + lambda x, y: tensor.concat([x, y], -1), outputs_fw, outputs_bw + ) final_states = (states_fw, states_bw) return outputs, final_states @@ -738,7 +825,7 @@ def birnn(cell_fw, class Decoder(object): """ - :api_attr: Static Graph + :api_attr: Static Graph Decoder is the base class for any decoder instance used in `dynamic_decode`. It provides interface for output generation for one time step, which can be @@ -890,13 +977,15 @@ class BeamSearchDecoder(Decoder): """ - def __init__(self, - cell, - start_token, - end_token, - beam_size, - embedding_fn=None, - output_fn=None): + def __init__( + self, + cell, + start_token, + end_token, + beam_size, + embedding_fn=None, + output_fn=None, + ): """ Constructor of BeamSearchDecoder. @@ -939,22 +1028,23 @@ class BeamSearchDecoder(Decoder): Variable: A tensor with shape `[batch_size * beam_size, ...]`, whose \ data type is same as `x`. """ - check_type(x, 'x', (Variable), - 'BeamSearchDecoder.tile_beam_merge_with_batch') + check_type( + x, 'x', (Variable), 'BeamSearchDecoder.tile_beam_merge_with_batch' + ) x = nn.unsqueeze(x, [1]) # [batch_size, 1, ...] expand_times = [1] * len(x.shape) expand_times[1] = beam_size x = paddle.tile(x, expand_times) # [batch_size, beam_size, ...] - x = nn.transpose(x, - list(range(2, len(x.shape))) + - [0, 1]) # [..., batch_size, beam_size] + x = nn.transpose( + x, list(range(2, len(x.shape))) + [0, 1] + ) # [..., batch_size, beam_size] # use 0 to copy to avoid wrong shape - x = nn.reshape(x, shape=[0] * (len(x.shape) - 2) + - [-1]) # [..., batch_size * beam_size] + x = nn.reshape( + x, shape=[0] * (len(x.shape) - 2) + [-1] + ) # [..., batch_size * beam_size] x = nn.transpose( - x, [len(x.shape) - 1] + - list(range(0, - len(x.shape) - 1))) # [batch_size * beam_size, ...] + x, [len(x.shape) - 1] + list(range(0, len(x.shape) - 1)) + ) # [batch_size * beam_size, ...] return x def _split_batch_beams(self, x): @@ -1032,14 +1122,16 @@ class BeamSearchDecoder(Decoder): replaced with a tensor with all probability on the EOS token. """ check_type(probs, 'probs', (Variable), 'BeamSearchDecoder._mask_probs') - check_type(finished, 'finished', (Variable), - 'BeamSearchDecoder._mask_probs') + check_type( + finished, 'finished', (Variable), 'BeamSearchDecoder._mask_probs' + ) # TODO: use where_op finished = tensor.cast(finished, dtype=probs.dtype) probs = nn.elementwise_mul( paddle.tile(nn.unsqueeze(finished, [2]), [1, 1, self.vocab_size]), self.noend_mask_tensor, - axis=-1) - nn.elementwise_mul(probs, (finished - 1), axis=0) + axis=-1, + ) - nn.elementwise_mul(probs, (finished - 1), axis=0) return probs def _gather(self, x, indices, batch_size): @@ -1059,37 +1151,48 @@ class BeamSearchDecoder(Decoder): """ check_type(x, 'x', (Variable), 'BeamSearchDecoder._gather') check_type(indices, 'indices', (Variable), 'BeamSearchDecoder._gather') - check_type(batch_size, 'batch_size', (Variable), - 'BeamSearchDecoder._gather') + check_type( + batch_size, 'batch_size', (Variable), 'BeamSearchDecoder._gather' + ) # TODO: compatibility of int32 and int64 - batch_size = tensor.cast( - batch_size, - indices.dtype) if batch_size.dtype != indices.dtype else batch_size + batch_size = ( + tensor.cast(batch_size, indices.dtype) + if batch_size.dtype != indices.dtype + else batch_size + ) batch_size.stop_gradient = True # TODO: remove this batch_pos = paddle.tile( - nn.unsqueeze(tensor.range(0, batch_size, 1, dtype=indices.dtype), - [1]), [1, self.beam_size]) + nn.unsqueeze( + tensor.range(0, batch_size, 1, dtype=indices.dtype), [1] + ), + [1, self.beam_size], + ) topk_coordinates = nn.stack([batch_pos, indices], axis=2) topk_coordinates.stop_gradient = True return nn.gather_nd(x, topk_coordinates) class OutputWrapper( - collections.namedtuple("OutputWrapper", - ("scores", "predicted_ids", "parent_ids"))): + collections.namedtuple( + "OutputWrapper", ("scores", "predicted_ids", "parent_ids") + ) + ): """ The structure for the returned value `outputs` of `decoder.step`. A namedtuple includes scores, predicted_ids, parent_ids as fields. """ + pass class StateWrapper( - collections.namedtuple( - "StateWrapper", - ("cell_states", "log_probs", "finished", "lengths"))): + collections.namedtuple( + "StateWrapper", ("cell_states", "log_probs", "finished", "lengths") + ) + ): """ The structure for the argument `states` of `decoder.step`. A namedtuple includes cell_states, log_probs, finished, lengths as fields. """ + pass def initialize(self, initial_cell_states): @@ -1117,22 +1220,30 @@ class BeamSearchDecoder(Decoder): state = flatten(initial_cell_states)[0] self.batch_size = nn.shape(state)[0] - self.start_token_tensor = tensor.fill_constant(shape=[1], - dtype="int64", - value=self.start_token) - self.end_token_tensor = tensor.fill_constant(shape=[1], - dtype="int64", - value=self.end_token) - - init_cell_states = map_structure(self._expand_to_beam_size, - initial_cell_states) - init_inputs = paddle.full(shape=[self.batch_size, self.beam_size], - fill_value=self.start_token_tensor, - dtype=self.start_token_tensor.dtype) + self.start_token_tensor = tensor.fill_constant( + shape=[1], dtype="int64", value=self.start_token + ) + self.end_token_tensor = tensor.fill_constant( + shape=[1], dtype="int64", value=self.end_token + ) + + init_cell_states = map_structure( + self._expand_to_beam_size, initial_cell_states + ) + init_inputs = paddle.full( + shape=[self.batch_size, self.beam_size], + fill_value=self.start_token_tensor, + dtype=self.start_token_tensor.dtype, + ) log_probs = paddle.tile( tensor.assign( - np.array([[0.] + [-self.kinf] * (self.beam_size - 1)], - dtype="float32")), [self.batch_size, 1]) + np.array( + [[0.0] + [-self.kinf] * (self.beam_size - 1)], + dtype="float32", + ) + ), + [self.batch_size, 1], + ) if paddle.get_default_dtype() == "float64": log_probs = tensor.cast(log_probs, "float64") # TODO: remove the restriction of force_cpu @@ -1141,13 +1252,19 @@ class BeamSearchDecoder(Decoder): shape=[-1, self.beam_size], dtype="bool", value=False, - force_cpu=True) + force_cpu=True, + ) init_lengths = tensor.zeros_like(init_inputs) - init_inputs = self.embedding_fn( - init_inputs) if self.embedding_fn else init_inputs - return init_inputs, self.StateWrapper(init_cell_states, log_probs, - init_finished, - init_lengths), init_finished + init_inputs = ( + self.embedding_fn(init_inputs) if self.embedding_fn else init_inputs + ) + return ( + init_inputs, + self.StateWrapper( + init_cell_states, log_probs, init_finished, init_lengths + ), + init_finished, + ) def _beam_search_step(self, time, logits, next_cell_states, beam_state): r""" @@ -1178,50 +1295,61 @@ class BeamSearchDecoder(Decoder): """ self.vocab_size = logits.shape[-1] - self.vocab_size_tensor = tensor.fill_constant(shape=[1], - dtype="int64", - value=self.vocab_size) + self.vocab_size_tensor = tensor.fill_constant( + shape=[1], dtype="int64", value=self.vocab_size + ) noend_array = [-self.kinf] * self.vocab_size noend_array[self.end_token] = 0 self.noend_mask_tensor = tensor.assign(np.array(noend_array, "float32")) if paddle.get_default_dtype() == "float64": - self.noend_mask_tensor = tensor.cast(self.noend_mask_tensor, - "float64") + self.noend_mask_tensor = tensor.cast( + self.noend_mask_tensor, "float64" + ) step_log_probs = nn.log(nn.softmax(logits)) step_log_probs = self._mask_probs(step_log_probs, beam_state.finished) - log_probs = nn.elementwise_add(x=step_log_probs, - y=beam_state.log_probs, - axis=0) + log_probs = nn.elementwise_add( + x=step_log_probs, y=beam_state.log_probs, axis=0 + ) # TODO: length penalty scores = log_probs scores = nn.reshape(scores, [-1, self.beam_size * self.vocab_size]) # TODO: add grad for topk then this beam search can be used to train topk_scores, topk_indices = paddle.topk(x=scores, k=self.beam_size) - beam_indices = nn.elementwise_floordiv(topk_indices, - self.vocab_size_tensor) + beam_indices = nn.elementwise_floordiv( + topk_indices, self.vocab_size_tensor + ) token_indices = nn.elementwise_mod(topk_indices, self.vocab_size_tensor) next_log_probs = self._gather( nn.reshape(log_probs, [-1, self.beam_size * self.vocab_size]), - topk_indices, self.batch_size) + topk_indices, + self.batch_size, + ) next_cell_states = map_structure( lambda x: self._gather(x, beam_indices, self.batch_size), - next_cell_states) - next_finished = self._gather(beam_state.finished, beam_indices, - self.batch_size) - next_lengths = self._gather(beam_state.lengths, beam_indices, - self.batch_size) - next_lengths = next_lengths + tensor.cast(nn.logical_not(next_finished), - beam_state.lengths.dtype) + next_cell_states, + ) + next_finished = self._gather( + beam_state.finished, beam_indices, self.batch_size + ) + next_lengths = self._gather( + beam_state.lengths, beam_indices, self.batch_size + ) + next_lengths = next_lengths + tensor.cast( + nn.logical_not(next_finished), beam_state.lengths.dtype + ) next_finished = control_flow.logical_or( next_finished, - control_flow.equal(token_indices, self.end_token_tensor)) - - beam_search_output = self.OutputWrapper(topk_scores, token_indices, - beam_indices) - beam_search_state = self.StateWrapper(next_cell_states, next_log_probs, - next_finished, next_lengths) + control_flow.equal(token_indices, self.end_token_tensor), + ) + + beam_search_output = self.OutputWrapper( + topk_scores, token_indices, beam_indices + ) + beam_search_state = self.StateWrapper( + next_cell_states, next_log_probs, next_finished, next_lengths + ) return beam_search_output, beam_search_state def step(self, time, inputs, states, **kwargs): @@ -1254,11 +1382,13 @@ class BeamSearchDecoder(Decoder): """ inputs = map_structure(self._merge_batch_beams, inputs) cell_states = map_structure(self._merge_batch_beams, states.cell_states) - cell_outputs, next_cell_states = self.cell(inputs, cell_states, - **kwargs) + cell_outputs, next_cell_states = self.cell( + inputs, cell_states, **kwargs + ) cell_outputs = map_structure(self._split_batch_beams, cell_outputs) - next_cell_states = map_structure(self._split_batch_beams, - next_cell_states) + next_cell_states = map_structure( + self._split_batch_beams, next_cell_states + ) if self.output_fn is not None: cell_outputs = self.output_fn(cell_outputs) @@ -1267,12 +1397,14 @@ class BeamSearchDecoder(Decoder): time=time, logits=cell_outputs, next_cell_states=next_cell_states, - beam_state=states) + beam_state=states, + ) finished = beam_search_state.finished sample_ids = beam_search_output.predicted_ids sample_ids.stop_gradient = True - next_inputs = self.embedding_fn( - sample_ids) if self.embedding_fn else sample_ids + next_inputs = ( + self.embedding_fn(sample_ids) if self.embedding_fn else sample_ids + ) return (beam_search_output, beam_search_state, next_inputs, finished) @@ -1300,8 +1432,9 @@ class BeamSearchDecoder(Decoder): `[time_step, batch_size, beam_size]`. `final_states` is the same \ as the input argument `final_states`. """ - predicted_ids = nn.gather_tree(outputs.predicted_ids, - outputs.parent_ids) + predicted_ids = nn.gather_tree( + outputs.predicted_ids, outputs.parent_ids + ) # TODO: use FinalBeamSearchDecoderOutput as output return predicted_ids, final_states @@ -1319,15 +1452,16 @@ class BeamSearchDecoder(Decoder): return True -def _dynamic_decode_imperative(decoder, - inits=None, - max_step_num=None, - output_time_major=False, - impute_finished=False, - is_test=False, - return_length=False, - **kwargs): - +def _dynamic_decode_imperative( + decoder, + inits=None, + max_step_num=None, + output_time_major=False, + impute_finished=False, + is_test=False, + return_length=False, + **kwargs +): def _maybe_copy(state, new_state, step_mask): # TODO: use where_op state_dtype = state.dtype @@ -1340,28 +1474,30 @@ def _dynamic_decode_imperative(decoder, # to sum(bool) error. step_mask.stop_gradient = True new_state = nn.elementwise_mul( - state, step_mask, axis=0) - nn.elementwise_mul(new_state, - (step_mask - 1), - axis=0) + state, step_mask, axis=0 + ) - nn.elementwise_mul(new_state, (step_mask - 1), axis=0) if convert_dtype(state_dtype) in ["bool"]: new_state = tensor.cast(new_state, dtype=state_dtype) return new_state initial_inputs, initial_states, initial_finished = decoder.initialize(inits) - inputs, states, finished = (initial_inputs, initial_states, - initial_finished) + inputs, states, finished = ( + initial_inputs, + initial_states, + initial_finished, + ) cond = control_flow.logical_not((nn.reduce_all(initial_finished))) sequence_lengths = tensor.cast(tensor.zeros_like(initial_finished), "int64") outputs = None step_idx = 0 - step_idx_tensor = tensor.fill_constant(shape=[1], - dtype="int64", - value=step_idx) + step_idx_tensor = tensor.fill_constant( + shape=[1], dtype="int64", value=step_idx + ) while cond.numpy(): - (step_outputs, next_states, next_inputs, - next_finished) = decoder.step(step_idx_tensor, inputs, states, - **kwargs) + (step_outputs, next_states, next_inputs, next_finished) = decoder.step( + step_idx_tensor, inputs, states, **kwargs + ) if not decoder.tracks_own_finished: # BeamSearchDecoder would track it own finished, since # beams would be reordered and the finished status of each @@ -1373,26 +1509,37 @@ def _dynamic_decode_imperative(decoder, tensor.assign(next_finished, finished) next_sequence_lengths = nn.elementwise_add( sequence_lengths, - tensor.cast(control_flow.logical_not(finished), - sequence_lengths.dtype)) + tensor.cast( + control_flow.logical_not(finished), sequence_lengths.dtype + ), + ) if impute_finished: # rectify the states for the finished. next_states = map_structure( - lambda x, y: _maybe_copy(x, y, finished), states, - next_states) + lambda x, y: _maybe_copy(x, y, finished), + states, + next_states, + ) else: warnings.warn( "`next_states` has no `lengths` attribute, the returned `sequence_lengths` would be all zeros." ) if not hasattr(next_states, "lengths") else None - next_sequence_lengths = getattr(next_states, "lengths", - sequence_lengths) - - outputs = map_structure( - lambda x: ArrayWrapper(x), - step_outputs) if step_idx == 0 else map_structure( - lambda x, x_array: x_array.append(x), step_outputs, outputs) - inputs, states, finished, sequence_lengths = (next_inputs, next_states, - next_finished, - next_sequence_lengths) + next_sequence_lengths = getattr( + next_states, "lengths", sequence_lengths + ) + + outputs = ( + map_structure(lambda x: ArrayWrapper(x), step_outputs) + if step_idx == 0 + else map_structure( + lambda x, x_array: x_array.append(x), step_outputs, outputs + ) + ) + inputs, states, finished, sequence_lengths = ( + next_inputs, + next_states, + next_finished, + next_sequence_lengths, + ) control_flow.increment(x=step_idx_tensor, value=1.0, in_place=True) step_idx += 1 @@ -1405,42 +1552,49 @@ def _dynamic_decode_imperative(decoder, final_states = states try: - final_outputs, final_states = decoder.finalize(final_outputs, - final_states, - sequence_lengths) + final_outputs, final_states = decoder.finalize( + final_outputs, final_states, sequence_lengths + ) except NotImplementedError: pass if not output_time_major: final_outputs = map_structure( lambda x: nn.transpose(x, [1, 0] + list(range(2, len(x.shape)))), - final_outputs) + final_outputs, + ) - return (final_outputs, final_states, - sequence_lengths) if return_length else (final_outputs, - final_states) + return ( + (final_outputs, final_states, sequence_lengths) + if return_length + else (final_outputs, final_states) + ) -def _dynamic_decode_declarative(decoder, - inits=None, - max_step_num=None, - output_time_major=False, - impute_finished=False, - is_test=False, - return_length=False, - **kwargs): +def _dynamic_decode_declarative( + decoder, + inits=None, + max_step_num=None, + output_time_major=False, + impute_finished=False, + is_test=False, + return_length=False, + **kwargs +): initial_inputs, initial_states, initial_finished = decoder.initialize(inits) - global_inputs, global_states, global_finished = (initial_inputs, - initial_states, - initial_finished) + global_inputs, global_states, global_finished = ( + initial_inputs, + initial_states, + initial_finished, + ) global_finished.stop_gradient = True step_idx = tensor.fill_constant(shape=[1], dtype="int64", value=0) cond = control_flow.logical_not((nn.reduce_all(initial_finished))) if max_step_num is not None: - max_step_num = tensor.fill_constant(shape=[1], - dtype="int64", - value=max_step_num) + max_step_num = tensor.fill_constant( + shape=[1], dtype="int64", value=max_step_num + ) while_op = control_flow.While(cond, is_test=is_test) sequence_lengths = tensor.cast(tensor.zeros_like(initial_finished), "int64") @@ -1453,9 +1607,11 @@ def _dynamic_decode_declarative(decoder, else: # inputs and states of all steps must be saved for backward and training inputs_arrays = map_structure( - lambda x: control_flow.array_write(x, step_idx), initial_inputs) + lambda x: control_flow.array_write(x, step_idx), initial_inputs + ) states_arrays = map_structure( - lambda x: control_flow.array_write(x, step_idx), initial_states) + lambda x: control_flow.array_write(x, step_idx), initial_states + ) def _maybe_copy(state, new_state, step_mask): # TODO: use where_op @@ -1469,9 +1625,8 @@ def _dynamic_decode_declarative(decoder, # to sum(bool) error. step_mask.stop_gradient = True new_state = nn.elementwise_mul( - state, step_mask, axis=0) - nn.elementwise_mul(new_state, - (step_mask - 1), - axis=0) + state, step_mask, axis=0 + ) - nn.elementwise_mul(new_state, (step_mask - 1), axis=0) if convert_dtype(state_dtype) in ["bool"]: new_state = tensor.cast(new_state, dtype=state_dtype) return new_state @@ -1481,8 +1636,9 @@ def _dynamic_decode_declarative(decoder, def _create_array_out_of_while(dtype): current_block_idx = default_main_program().current_block_idx - default_main_program().current_block_idx = default_main_program( - ).current_block().parent_idx + default_main_program().current_block_idx = ( + default_main_program().current_block().parent_idx + ) tensor_array = control_flow.create_array(dtype) default_main_program().current_block_idx = current_block_idx return tensor_array @@ -1492,23 +1648,30 @@ def _dynamic_decode_declarative(decoder, if not is_test: inputs = map_structure( lambda array: control_flow.array_read(array, step_idx), - inputs_arrays) + inputs_arrays, + ) states = map_structure( lambda array: control_flow.array_read(array, step_idx), - states_arrays) - (outputs, next_states, next_inputs, - next_finished) = decoder.step(step_idx, inputs, states, **kwargs) + states_arrays, + ) + (outputs, next_states, next_inputs, next_finished) = decoder.step( + step_idx, inputs, states, **kwargs + ) if not decoder.tracks_own_finished: # BeamSearchDecoder would track it own finished, since beams would # be reordered and the finished status of each entry might change. # Otherwise, perform logical OR which would not change the already # finished. - next_finished = control_flow.logical_or(next_finished, - global_finished) + next_finished = control_flow.logical_or( + next_finished, global_finished + ) next_sequence_lengths = nn.elementwise_add( sequence_lengths, - tensor.cast(control_flow.logical_not(global_finished), - sequence_lengths.dtype)) + tensor.cast( + control_flow.logical_not(global_finished), + sequence_lengths.dtype, + ), + ) if impute_finished: # rectify the states for the finished. next_states = map_structure( lambda x, y: _maybe_copy(x, y, global_finished), @@ -1519,16 +1682,22 @@ def _dynamic_decode_declarative(decoder, warnings.warn( "`next_states` has no `lengths` attribute, the returned `sequence_lengths` would be all zeros." ) if not hasattr(next_states, "lengths") else None - next_sequence_lengths = getattr(next_states, "lengths", - sequence_lengths) + next_sequence_lengths = getattr( + next_states, "lengths", sequence_lengths + ) # create tensor array in global block after dtype[s] of outputs can be got outputs_arrays = map_structure( - lambda x: _create_array_out_of_while(x.dtype), outputs) + lambda x: _create_array_out_of_while(x.dtype), outputs + ) map_structure( lambda x, x_array: control_flow.array_write( - x, i=step_idx, array=x_array), outputs, outputs_arrays) + x, i=step_idx, array=x_array + ), + outputs, + outputs_arrays, + ) control_flow.increment(x=step_idx, value=1.0, in_place=True) # update the global_finished first, since it might be also in states of # decoder, which otherwise would write a stale finished status to array @@ -1540,50 +1709,68 @@ def _dynamic_decode_declarative(decoder, else: map_structure( lambda x, x_array: control_flow.array_write( - x, i=step_idx, array=x_array), next_inputs, inputs_arrays) + x, i=step_idx, array=x_array + ), + next_inputs, + inputs_arrays, + ) map_structure( lambda x, x_array: control_flow.array_write( - x, i=step_idx, array=x_array), next_states, states_arrays) + x, i=step_idx, array=x_array + ), + next_states, + states_arrays, + ) if max_step_num is not None: control_flow.logical_and( control_flow.logical_not(nn.reduce_all(global_finished)), - control_flow.less_equal(step_idx, max_step_num), cond) + control_flow.less_equal(step_idx, max_step_num), + cond, + ) else: control_flow.logical_not(nn.reduce_all(global_finished), cond) final_outputs = map_structure( lambda array: tensor.tensor_array_to_tensor( - array, axis=0, use_stack=True)[0], outputs_arrays) + array, axis=0, use_stack=True + )[0], + outputs_arrays, + ) if is_test: final_states = global_states else: final_states = map_structure( lambda array: control_flow.array_read(array, step_idx), - states_arrays) + states_arrays, + ) try: - final_outputs, final_states = decoder.finalize(final_outputs, - final_states, - sequence_lengths) + final_outputs, final_states = decoder.finalize( + final_outputs, final_states, sequence_lengths + ) except NotImplementedError: pass if not output_time_major: final_outputs = map_structure(_transpose_batch_time, final_outputs) - return (final_outputs, final_states, - sequence_lengths) if return_length else (final_outputs, - final_states) + return ( + (final_outputs, final_states, sequence_lengths) + if return_length + else (final_outputs, final_states) + ) -def dynamic_decode(decoder, - inits=None, - max_step_num=None, - output_time_major=False, - impute_finished=False, - is_test=False, - return_length=False, - **kwargs): +def dynamic_decode( + decoder, + inits=None, + max_step_num=None, + output_time_major=False, + impute_finished=False, + is_test=False, + return_length=False, + **kwargs +): r""" Dynamic decoding performs :code:`decoder.step()` repeatedly until the returned Tensor indicating finished status contains all True values or the number of @@ -1659,13 +1846,27 @@ def dynamic_decode(decoder, max_step_num=10) """ if _non_static_mode(): - return _dynamic_decode_imperative(decoder, inits, max_step_num, - output_time_major, impute_finished, - is_test, return_length, **kwargs) + return _dynamic_decode_imperative( + decoder, + inits, + max_step_num, + output_time_major, + impute_finished, + is_test, + return_length, + **kwargs + ) else: - return _dynamic_decode_declarative(decoder, inits, max_step_num, - output_time_major, impute_finished, - is_test, return_length, **kwargs) + return _dynamic_decode_declarative( + decoder, + inits, + max_step_num, + output_time_major, + impute_finished, + is_test, + return_length, + **kwargs + ) class DecodeHelper(object): @@ -1792,11 +1993,14 @@ class TrainingHelper(DecodeHelper): # extend inputs to avoid to slice out of range in `next_inputs` # may be easier and have better performance than condition_op self.inputs_ = map_structure( - lambda x: nn.pad(x, - paddings=([0, 1] + [0, 0] * (len(x.shape) - 1)) - if time_major else ([0, 0, 0, 1] + [0, 0] * - (len(x.shape) - 2))), - self.inputs) + lambda x: nn.pad( + x, + paddings=([0, 1] + [0, 0] * (len(x.shape) - 1)) + if time_major + else ([0, 0, 0, 1] + [0, 0] * (len(x.shape) - 2)), + ), + self.inputs, + ) def initialize(self): r""" @@ -1813,12 +2017,14 @@ class TrainingHelper(DecodeHelper): """ init_finished = control_flow.equal( self.sequence_length, - tensor.fill_constant(shape=[1], - dtype=self.sequence_length.dtype, - value=0)) + tensor.fill_constant( + shape=[1], dtype=self.sequence_length.dtype, value=0 + ), + ) # TODO: support zero length init_inputs = map_structure( - lambda x: x[0] if self.time_major else x[:, 0], self.inputs) + lambda x: x[0] if self.time_major else x[:, 0], self.inputs + ) return init_inputs, init_finished def sample(self, time, outputs, states): @@ -1872,8 +2078,11 @@ class TrainingHelper(DecodeHelper): shape `[batch_size]`. """ # TODO: compatibility of int32 and int64 - time = tensor.cast(time, "int32") if convert_dtype( - time.dtype) not in ["int32"] else time + time = ( + tensor.cast(time, "int32") + if convert_dtype(time.dtype) not in ["int32"] + else time + ) if self.sequence_length.dtype != time.dtype: self.sequence_length = tensor.cast(self.sequence_length, time.dtype) next_time = time + 1 @@ -1881,11 +2090,12 @@ class TrainingHelper(DecodeHelper): def _slice(x): # TODO: use Variable.__getitem__ axes = [0 if self.time_major else 1] - return nn.squeeze(nn.slice(x, - axes=axes, - starts=[next_time], - ends=[next_time + 1]), - axes=axes) + return nn.squeeze( + nn.slice( + x, axes=axes, starts=[next_time], ends=[next_time + 1] + ), + axes=axes, + ) next_inputs = map_structure(_slice, self.inputs_) return finished, next_inputs, states @@ -1943,9 +2153,9 @@ class GreedyEmbeddingHelper(DecodeHelper): """ self.embedding_fn = embedding_fn self.start_tokens = start_tokens - self.end_token = tensor.fill_constant(shape=[1], - dtype="int64", - value=end_token) + self.end_token = tensor.fill_constant( + shape=[1], dtype="int64", value=end_token + ) def initialize(self): r""" @@ -1966,7 +2176,8 @@ class GreedyEmbeddingHelper(DecodeHelper): shape=[-1], dtype="bool", value=False, - force_cpu=True) + force_cpu=True, + ) init_inputs = self.embedding_fn(self.start_tokens) return init_inputs, init_finished @@ -2054,12 +2265,14 @@ class SampleEmbeddingHelper(GreedyEmbeddingHelper): decoder=decoder, inits=decoder_cell.get_initial_states(encoder_output)) """ - def __init__(self, - embedding_fn, - start_tokens, - end_token, - softmax_temperature=None, - seed=None): + def __init__( + self, + embedding_fn, + start_tokens, + end_token, + softmax_temperature=None, + seed=None, + ): r""" Constructor of SampleEmbeddingHelper. @@ -2086,11 +2299,16 @@ class SampleEmbeddingHelper(GreedyEmbeddingHelper): structure of) tensor variable[s], and `finished` is a tensor with \ bool data type. """ - super(SampleEmbeddingHelper, self).__init__(embedding_fn, start_tokens, - end_token) - self.softmax_temperature = tensor.fill_constant( - shape=[1], dtype="float32", value=softmax_temperature - ) if softmax_temperature is not None else None + super(SampleEmbeddingHelper, self).__init__( + embedding_fn, start_tokens, end_token + ) + self.softmax_temperature = ( + tensor.fill_constant( + shape=[1], dtype="float32", value=softmax_temperature + ) + if softmax_temperature is not None + else None + ) self.seed = seed def sample(self, time, outputs, states): @@ -2112,16 +2330,19 @@ class SampleEmbeddingHelper(GreedyEmbeddingHelper): Variable: An `int64` tensor with shape `[batch_size]`, representing \ the sampled ids. """ - logits = (outputs / self.softmax_temperature - ) if self.softmax_temperature is not None else outputs + logits = ( + (outputs / self.softmax_temperature) + if self.softmax_temperature is not None + else outputs + ) probs = nn.softmax(logits) # TODO: remove this stop_gradient. The stop_gradient of sample_ids can # not pass to probs, since sampling_id op does not have corresponding # grad op and thus can not pass. probs.stop_gradient = True - sample_ids = nn.sampling_id(probs, - seed=self.seed, - dtype=self.start_tokens.dtype) + sample_ids = nn.sampling_id( + probs, seed=self.seed, dtype=self.start_tokens.dtype + ) return sample_ids @@ -2201,12 +2422,13 @@ class BasicDecoder(Decoder): return initial_inputs, initial_cell_states, initial_finished class OutputWrapper( - collections.namedtuple("OutputWrapper", - ("cell_outputs", "sample_ids"))): + collections.namedtuple("OutputWrapper", ("cell_outputs", "sample_ids")) + ): """ The structure for the returned value `outputs` of `decoder.step`. A namedtuple includes cell_outputs, sample_ids as fields. """ + pass def step(self, time, inputs, states, **kwargs): @@ -2248,32 +2470,35 @@ class BasicDecoder(Decoder): cell_outputs, cell_states = self.cell(inputs, states, **kwargs) if self.output_fn is not None: cell_outputs = self.output_fn(cell_outputs) - sample_ids = self.helper.sample(time=time, - outputs=cell_outputs, - states=cell_states) + sample_ids = self.helper.sample( + time=time, outputs=cell_outputs, states=cell_states + ) sample_ids.stop_gradient = True - (finished, next_inputs, - next_states) = self.helper.next_inputs(time=time, - outputs=cell_outputs, - states=cell_states, - sample_ids=sample_ids) + (finished, next_inputs, next_states) = self.helper.next_inputs( + time=time, + outputs=cell_outputs, + states=cell_states, + sample_ids=sample_ids, + ) outputs = self.OutputWrapper(cell_outputs, sample_ids) return (outputs, next_states, next_inputs, finished) -def dynamic_lstm(input, - size, - h_0=None, - c_0=None, - param_attr=None, - bias_attr=None, - use_peepholes=True, - is_reverse=False, - gate_activation='sigmoid', - cell_activation='tanh', - candidate_activation='tanh', - dtype='float32', - name=None): +def dynamic_lstm( + input, + size, + h_0=None, + c_0=None, + param_attr=None, + bias_attr=None, + use_peepholes=True, + is_reverse=False, + gate_activation='sigmoid', + cell_activation='tanh', + candidate_activation='tanh', + dtype='float32', + name=None, +): r""" :api_attr: Static Graph @@ -2373,35 +2598,40 @@ def dynamic_lstm(input, forward.shape # (-1, 512) cell.shape # (-1, 512) """ - assert _non_static_mode( - ) is not True, "please use lstm instead of dynamic_lstm in dygraph mode!" - assert bias_attr is not False, "bias_attr should not be False in dynamic_lstm." - - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'dynamic_lstm') + assert ( + _non_static_mode() is not True + ), "please use lstm instead of dynamic_lstm in dygraph mode!" + assert ( + bias_attr is not False + ), "bias_attr should not be False in dynamic_lstm." + + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'dynamic_lstm' + ) check_type(h_0, 'h_0', (Variable, type(None)), 'dynamic_lstm') if isinstance(h_0, Variable): - check_variable_and_dtype(h_0, 'h_0', ['float32', 'float64'], - 'dynamic_lstm') + check_variable_and_dtype( + h_0, 'h_0', ['float32', 'float64'], 'dynamic_lstm' + ) check_type(c_0, 'c_0', (Variable, type(None)), 'dynamic_lstm') if isinstance(c_0, Variable): - check_variable_and_dtype(c_0, 'c_0', ['float32', 'float64'], - 'dynamic_lstm') + check_variable_and_dtype( + c_0, 'c_0', ['float32', 'float64'], 'dynamic_lstm' + ) helper = LayerHelper('lstm', **locals()) size = size // 4 - weight = helper.create_parameter(attr=helper.param_attr, - shape=[size, 4 * size], - dtype=dtype) + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 4 * size], dtype=dtype + ) bias_size = [1, 7 * size] if not use_peepholes: bias_size[1] = 4 * size - bias = helper.create_parameter(attr=helper.bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) hidden = helper.create_variable_for_type_inference(dtype) cell = helper.create_variable_for_type_inference(dtype) @@ -2410,47 +2640,55 @@ def dynamic_lstm(input, inputs = {'Input': input, 'Weight': weight, 'Bias': bias} batch_size = input.shape[0] if h_0: - assert h_0.shape == (batch_size, size), \ + assert h_0.shape == (batch_size, size), ( 'The shape of h0 should be (batch_size, %d)' % size + ) inputs['H0'] = h_0 if c_0: - assert c_0.shape == (batch_size, size), \ + assert c_0.shape == (batch_size, size), ( 'The shape of c0 should be (batch_size, %d)' % size + ) inputs['C0'] = c_0 - helper.append_op(type='lstm', - inputs=inputs, - outputs={ - 'Hidden': hidden, - 'Cell': cell, - 'BatchGate': batch_gate, - 'BatchCellPreAct': batch_cell_pre_act - }, - attrs={ - 'use_peepholes': use_peepholes, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'cell_activation': cell_activation, - 'candidate_activation': candidate_activation - }) + helper.append_op( + type='lstm', + inputs=inputs, + outputs={ + 'Hidden': hidden, + 'Cell': cell, + 'BatchGate': batch_gate, + 'BatchCellPreAct': batch_cell_pre_act, + }, + attrs={ + 'use_peepholes': use_peepholes, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation, + }, + ) return hidden, cell -@deprecated(since='2.0.0', - update_to='paddle.nn.LSTM', - reason="This API may occur CUDNN errors.") -def lstm(input, - init_h, - init_c, - max_len, - hidden_size, - num_layers, - dropout_prob=0.0, - is_bidirec=False, - is_test=False, - name=None, - default_initializer=None, - seed=-1): +@deprecated( + since='2.0.0', + update_to='paddle.nn.LSTM', + reason="This API may occur CUDNN errors.", +) +def lstm( + input, + init_h, + init_c, + max_len, + hidden_size, + num_layers, + dropout_prob=0.0, + is_bidirec=False, + is_test=False, + name=None, + default_initializer=None, + seed=-1, +): r""" :api_attr: Static Graph @@ -2577,63 +2815,71 @@ def lstm(input, weight_size += input_weight_size + hidden_weight_size weight_size += hidden_size * 8 * num_dirrection - weight = helper.create_parameter(attr=helper.param_attr, - shape=[weight_size], - dtype=dtype, - default_initializer=default_initializer) + weight = helper.create_parameter( + attr=helper.param_attr, + shape=[weight_size], + dtype=dtype, + default_initializer=default_initializer, + ) out = helper.create_variable_for_type_inference(dtype) last_h = helper.create_variable_for_type_inference(dtype) last_c = helper.create_variable_for_type_inference(dtype) reserve = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) state_out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) state_out.persistable = True - helper.append_op(type='cudnn_lstm', - inputs={ - 'Input': input, - 'InitH': init_h, - 'InitC': init_c, - 'W': weight, - }, - outputs={ - 'Out': out, - 'LastH': last_h, - 'LastC': last_c, - 'Reserve': reserve, - 'StateOut': state_out, - }, - attrs={ - 'is_bidirec': is_bidirec, - 'input_size': input_size, - 'hidden_size': hidden_size, - 'num_layers': num_layers, - 'is_test': is_test, - 'dropout_prob': dropout_prob, - 'seed': seed, - }) + helper.append_op( + type='cudnn_lstm', + inputs={ + 'Input': input, + 'InitH': init_h, + 'InitC': init_c, + 'W': weight, + }, + outputs={ + 'Out': out, + 'LastH': last_h, + 'LastC': last_c, + 'Reserve': reserve, + 'StateOut': state_out, + }, + attrs={ + 'is_bidirec': is_bidirec, + 'input_size': input_size, + 'hidden_size': hidden_size, + 'num_layers': num_layers, + 'is_test': is_test, + 'dropout_prob': dropout_prob, + 'seed': seed, + }, + ) return out, last_h, last_c -def dynamic_lstmp(input, - size, - proj_size, - param_attr=None, - bias_attr=None, - use_peepholes=True, - is_reverse=False, - gate_activation='sigmoid', - cell_activation='tanh', - candidate_activation='tanh', - proj_activation='tanh', - dtype='float32', - name=None, - h_0=None, - c_0=None, - cell_clip=None, - proj_clip=None): +def dynamic_lstmp( + input, + size, + proj_size, + param_attr=None, + bias_attr=None, + use_peepholes=True, + is_reverse=False, + gate_activation='sigmoid', + cell_activation='tanh', + candidate_activation='tanh', + proj_activation='tanh', + dtype='float32', + name=None, + h_0=None, + c_0=None, + cell_clip=None, + proj_clip=None, +): r""" :api_attr: Static Graph @@ -2756,39 +3002,44 @@ def dynamic_lstmp(input, last_c.shape # (-1, 512) """ - assert _non_static_mode( - ) is not True, "please use lstm instead of dynamic_lstmp in dygraph mode!" + assert ( + _non_static_mode() is not True + ), "please use lstm instead of dynamic_lstmp in dygraph mode!" - assert bias_attr is not False, "bias_attr should not be False in dynamic_lstmp." + assert ( + bias_attr is not False + ), "bias_attr should not be False in dynamic_lstmp." - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'dynamic_lstmp') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'dynamic_lstmp' + ) check_type(h_0, 'h_0', (Variable, type(None)), 'dynamic_lstmp') if isinstance(h_0, Variable): - check_variable_and_dtype(h_0, 'h_0', ['float32', 'float64'], - 'dynamic_lstmp') + check_variable_and_dtype( + h_0, 'h_0', ['float32', 'float64'], 'dynamic_lstmp' + ) check_type(c_0, 'c_0', (Variable, type(None)), 'dynamic_lstmp') if isinstance(c_0, Variable): - check_variable_and_dtype(c_0, 'c_0', ['float32', 'float64'], - 'dynamic_lstmp') + check_variable_and_dtype( + c_0, 'c_0', ['float32', 'float64'], 'dynamic_lstmp' + ) helper = LayerHelper('lstmp', **locals()) size = size // 4 - weight = helper.create_parameter(attr=helper.param_attr, - shape=[proj_size, 4 * size], - dtype=dtype) - proj_weight = helper.create_parameter(attr=helper.param_attr, - shape=[size, proj_size], - dtype=dtype) + weight = helper.create_parameter( + attr=helper.param_attr, shape=[proj_size, 4 * size], dtype=dtype + ) + proj_weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, proj_size], dtype=dtype + ) bias_size = [1, 7 * size] if not use_peepholes: bias_size[1] = 4 * size - bias = helper.create_parameter(attr=helper.bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) projection = helper.create_variable_for_type_inference(dtype) cell = helper.create_variable_for_type_inference(dtype) @@ -2800,16 +3051,18 @@ def dynamic_lstmp(input, 'Input': input, 'Weight': weight, 'ProjWeight': proj_weight, - 'Bias': bias + 'Bias': bias, } batch_size = input.shape[0] if h_0: - assert h_0.shape == (batch_size, proj_size), \ + assert h_0.shape == (batch_size, proj_size), ( 'The shape of h0 should be (batch_size, %d)' % proj_size + ) inputs['H0'] = h_0 if c_0: - assert c_0.shape == (batch_size, size), \ + assert c_0.shape == (batch_size, size), ( 'The shape of c0 should be (batch_size, %d)' % size + ) inputs['C0'] = c_0 if cell_clip: @@ -2817,37 +3070,41 @@ def dynamic_lstmp(input, if proj_clip: assert proj_clip >= 0, "proj_clip should not be negative." - helper.append_op(type='lstmp', - inputs=inputs, - outputs={ - 'Projection': projection, - 'Cell': cell, - 'BatchHidden': batch_hidden, - 'BatchGate': batch_gate, - 'BatchCellPreAct': batch_cell_pre_act - }, - attrs={ - 'use_peepholes': use_peepholes, - 'cell_clip': cell_clip, - 'proj_clip': proj_clip, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'cell_activation': cell_activation, - 'candidate_activation': candidate_activation, - 'proj_activation': proj_activation - }) + helper.append_op( + type='lstmp', + inputs=inputs, + outputs={ + 'Projection': projection, + 'Cell': cell, + 'BatchHidden': batch_hidden, + 'BatchGate': batch_gate, + 'BatchCellPreAct': batch_cell_pre_act, + }, + attrs={ + 'use_peepholes': use_peepholes, + 'cell_clip': cell_clip, + 'proj_clip': proj_clip, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation, + 'proj_activation': proj_activation, + }, + ) return projection, cell -def dynamic_gru(input, - size, - param_attr=None, - bias_attr=None, - is_reverse=False, - gate_activation='sigmoid', - candidate_activation='tanh', - h_0=None, - origin_mode=False): +def dynamic_gru( + input, + size, + param_attr=None, + bias_attr=None, + is_reverse=False, + gate_activation='sigmoid', + candidate_activation='tanh', + h_0=None, + origin_mode=False, +): r""" :api_attr: Static Graph @@ -2953,33 +3210,35 @@ def dynamic_gru(input, hidden = fluid.layers.dynamic_gru(input=x, size=hidden_dim) """ - assert _non_static_mode( - ) is not True, "please use gru instead of dynamic_gru in dygraph mode!" + assert ( + _non_static_mode() is not True + ), "please use gru instead of dynamic_gru in dygraph mode!" - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'dynamic_gru') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'dynamic_gru' + ) check_type(h_0, 'h_0', (Variable, type(None)), 'dynamic_gru') if isinstance(h_0, Variable): - check_variable_and_dtype(h_0, 'h_0', ['float32', 'float64'], - 'dynamic_gru') + check_variable_and_dtype( + h_0, 'h_0', ['float32', 'float64'], 'dynamic_gru' + ) helper = LayerHelper('gru', **locals()) dtype = helper.input_dtype() - weight = helper.create_parameter(attr=helper.param_attr, - shape=[size, 3 * size], - dtype=dtype) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=[1, 3 * size], - dtype=dtype, - is_bias=True) + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype + ) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=[1, 3 * size], dtype=dtype, is_bias=True + ) batch_size = input.shape[0] inputs = {'Input': input, 'Weight': weight, 'Bias': bias} if h_0: - assert h_0.shape == ( - batch_size, - size), 'The shape of h0 should be(batch_size, %d)' % size + assert h_0.shape == (batch_size, size), ( + 'The shape of h0 should be(batch_size, %d)' % size + ) inputs['H0'] = h_0 hidden = helper.create_variable_for_type_inference(dtype) @@ -2987,31 +3246,35 @@ def dynamic_gru(input, batch_reset_hidden_prev = helper.create_variable_for_type_inference(dtype) batch_hidden = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='gru', - inputs=inputs, - outputs={ - 'Hidden': hidden, - 'BatchGate': batch_gate, - 'BatchResetHiddenPrev': batch_reset_hidden_prev, - 'BatchHidden': batch_hidden - }, - attrs={ - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'activation': candidate_activation, - 'origin_mode': origin_mode - }) + helper.append_op( + type='gru', + inputs=inputs, + outputs={ + 'Hidden': hidden, + 'BatchGate': batch_gate, + 'BatchResetHiddenPrev': batch_reset_hidden_prev, + 'BatchHidden': batch_hidden, + }, + attrs={ + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'activation': candidate_activation, + 'origin_mode': origin_mode, + }, + ) return hidden -def gru_unit(input, - hidden, - size, - param_attr=None, - bias_attr=None, - activation='tanh', - gate_activation='sigmoid', - origin_mode=False): +def gru_unit( + input, + hidden, + size, + param_attr=None, + bias_attr=None, + activation='tanh', + gate_activation='sigmoid', + origin_mode=False, +): r""" :api_attr: Static Graph @@ -3113,8 +3376,9 @@ def gru_unit(input, """ check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'gru_unit') - check_variable_and_dtype(hidden, 'hidden', ['float32', 'float64'], - 'gru_unit') + check_variable_and_dtype( + hidden, 'hidden', ['float32', 'float64'], 'gru_unit' + ) check_type(size, 'size', (int), 'gru_unit') activation_dict = dict( identity=0, @@ -3130,9 +3394,9 @@ def gru_unit(input, size = size // 3 # create weight - weight = helper.create_parameter(attr=helper.param_attr, - shape=[size, 3 * size], - dtype=dtype) + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype + ) gate = helper.create_variable_for_type_inference(dtype) reset_hidden_pre = helper.create_variable_for_type_inference(dtype) @@ -3141,10 +3405,9 @@ def gru_unit(input, # create bias if helper.bias_attr: bias_size = [1, 3 * size] - bias = helper.create_parameter(attr=helper.bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) inputs['Bias'] = bias helper.append_op( @@ -3158,22 +3421,25 @@ def gru_unit(input, attrs={ 'activation': 2, # tanh 'gate_activation': 1, # sigmoid - 'origin_mode': origin_mode - }) + 'origin_mode': origin_mode, + }, + ) return updated_hidden, reset_hidden_pre, gate -def beam_search(pre_ids, - pre_scores, - ids, - scores, - beam_size, - end_id, - level=0, - is_accumulated=True, - name=None, - return_parent_idx=False): +def beam_search( + pre_ids, + pre_scores, + ids, + scores, + beam_size, + end_id, + level=0, + is_accumulated=True, + name=None, + return_parent_idx=False, +): r""" Beam search is a classical algorithm for selecting candidate words in a @@ -3280,11 +3546,13 @@ def beam_search(pre_ids, end_id=end_id) """ check_variable_and_dtype(pre_ids, 'pre_ids', ['int64'], 'beam_search') - check_variable_and_dtype(pre_scores, 'pre_scores', ['float32', 'float64'], - 'beam_search') + check_variable_and_dtype( + pre_scores, 'pre_scores', ['float32', 'float64'], 'beam_search' + ) check_type(ids, 'ids', (Variable, type(None)), 'beam_search') - check_variable_and_dtype(scores, 'scores', ['float32', 'float64'], - 'beam_search') + check_variable_and_dtype( + scores, 'scores', ['float32', 'float64'], 'beam_search' + ) helper = LayerHelper('beam_search', **locals()) score_type = pre_scores.dtype id_type = pre_ids.dtype @@ -3294,7 +3562,8 @@ def beam_search(pre_ids, inputs["ids"] = ids selected_scores = helper.create_variable_for_type_inference( - dtype=score_type) + dtype=score_type + ) selected_ids = helper.create_variable_for_type_inference(dtype=id_type) # parent_idx is a tensor used to gather cell states at the next time # step. Though lod in selected_ids can also be used to gather by @@ -3308,7 +3577,7 @@ def beam_search(pre_ids, outputs={ 'selected_ids': selected_ids, 'selected_scores': selected_scores, - 'parent_idx': parent_idx + 'parent_idx': parent_idx, }, attrs={ # TODO(ChunweiYan) to assure other value support @@ -3316,7 +3585,8 @@ def beam_search(pre_ids, 'beam_size': beam_size, 'end_id': end_id, 'is_accumulated': is_accumulated, - }) + }, + ) if return_parent_idx: return selected_ids, selected_scores, parent_idx else: @@ -3381,37 +3651,37 @@ def beam_search_decode(ids, scores, beam_size, end_id, name=None): ids, scores, beam_size=5, end_id=0) """ check_variable_and_dtype(ids, 'ids', ['int64'], 'beam_search_encode') - check_variable_and_dtype(scores, 'scores', ['float32'], - 'beam_search_encode') + check_variable_and_dtype( + scores, 'scores', ['float32'], 'beam_search_encode' + ) helper = LayerHelper('beam_search_decode', **locals()) sentence_ids = helper.create_variable_for_type_inference(dtype=ids.dtype) sentence_scores = helper.create_variable_for_type_inference( - dtype=scores.dtype) - - helper.append_op(type="beam_search_decode", - inputs={ - "Ids": ids, - "Scores": scores - }, - outputs={ - "SentenceIds": sentence_ids, - "SentenceScores": sentence_scores - }, - attrs={ - "beam_size": beam_size, - "end_id": end_id - }) + dtype=scores.dtype + ) + + helper.append_op( + type="beam_search_decode", + inputs={"Ids": ids, "Scores": scores}, + outputs={ + "SentenceIds": sentence_ids, + "SentenceScores": sentence_scores, + }, + attrs={"beam_size": beam_size, "end_id": end_id}, + ) return sentence_ids, sentence_scores -def lstm_unit(x_t, - hidden_t_prev, - cell_t_prev, - forget_bias=0.0, - param_attr=None, - bias_attr=None, - name=None): +def lstm_unit( + x_t, + hidden_t_prev, + cell_t_prev, + forget_bias=0.0, + param_attr=None, + bias_attr=None, + name=None, +): r""" :api_attr: Static Graph @@ -3496,10 +3766,12 @@ def lstm_unit(x_t, """ helper = LayerHelper('lstm_unit', **locals()) check_variable_and_dtype(x_t, 'x_t', ['float32', 'float64'], 'lstm_unit') - check_variable_and_dtype(hidden_t_prev, 'hidden_t_prev', - ['float32', 'float64'], 'lstm_unit') - check_variable_and_dtype(cell_t_prev, 'cell_t_prev', ['float32', 'float64'], - 'lstm_unit') + check_variable_and_dtype( + hidden_t_prev, 'hidden_t_prev', ['float32', 'float64'], 'lstm_unit' + ) + check_variable_and_dtype( + cell_t_prev, 'cell_t_prev', ['float32', 'float64'], 'lstm_unit' + ) if len(x_t.shape) != 2: raise ValueError("Rank of x_t must be 2.") @@ -3509,37 +3781,41 @@ def lstm_unit(x_t, if len(cell_t_prev.shape) != 2: raise ValueError("Rank of cell_t_prev must be 2.") - if x_t.shape[0] != hidden_t_prev.shape[0] or x_t.shape[ - 0] != cell_t_prev.shape[0]: - raise ValueError("The 1st dimensions of x_t, hidden_t_prev and " - "cell_t_prev must be the same.") + if ( + x_t.shape[0] != hidden_t_prev.shape[0] + or x_t.shape[0] != cell_t_prev.shape[0] + ): + raise ValueError( + "The 1st dimensions of x_t, hidden_t_prev and " + "cell_t_prev must be the same." + ) if hidden_t_prev.shape[1] != cell_t_prev.shape[1]: - raise ValueError("The 2nd dimensions of hidden_t_prev and " - "cell_t_prev must be the same.") + raise ValueError( + "The 2nd dimensions of hidden_t_prev and " + "cell_t_prev must be the same." + ) if bias_attr is None: bias_attr = ParamAttr() size = cell_t_prev.shape[1] concat_out = nn.concat(input=[x_t, hidden_t_prev], axis=1) - fc_out = nn.fc(input=concat_out, - size=4 * size, - param_attr=param_attr, - bias_attr=bias_attr) + fc_out = nn.fc( + input=concat_out, + size=4 * size, + param_attr=param_attr, + bias_attr=bias_attr, + ) dtype = x_t.dtype c = helper.create_variable_for_type_inference(dtype) h = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='lstm_unit', - inputs={ - "X": fc_out, - "C_prev": cell_t_prev - }, - outputs={ - "C": c, - "H": h - }, - attrs={"forget_bias": forget_bias}) + helper.append_op( + type='lstm_unit', + inputs={"X": fc_out, "C_prev": cell_t_prev}, + outputs={"C": c, "H": h}, + attrs={"forget_bias": forget_bias}, + ) return h, c diff --git a/python/paddle/fluid/layers/sequence_lod.py b/python/paddle/fluid/layers/sequence_lod.py index 5deac6ff952..1ea5e2cf5e5 100644 --- a/python/paddle/fluid/layers/sequence_lod.py +++ b/python/paddle/fluid/layers/sequence_lod.py @@ -14,7 +14,14 @@ import paddle from .layer_function_generator import templatedoc -from ..framework import core, Variable, _non_static_mode, in_dygraph_mode, _in_legacy_dygraph, convert_np_dtype_to_dtype_ +from ..framework import ( + core, + Variable, + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, + convert_np_dtype_to_dtype_, +) from ..layer_helper import LayerHelper from ..data_feeder import check_variable_and_dtype, check_type, check_dtype from ..core import VarDesc @@ -41,20 +48,22 @@ __all__ = [ @templatedoc() -def sequence_conv(input, - num_filters, - filter_size=3, - filter_stride=1, - padding=True, - padding_start=None, - bias_attr=None, - param_attr=None, - act=None, - name=None): +def sequence_conv( + input, + num_filters, + filter_size=3, + filter_stride=1, + padding=True, + padding_start=None, + bias_attr=None, + param_attr=None, + act=None, + name=None, +): r""" Note: - Only receives LoDTensor as input. If your input is Tensor, please use conv2d Op.(fluid.layers.** :ref:`api_fluid_layers_conv2d` ). + Only receives LoDTensor as input. If your input is Tensor, please use conv2d Op.(fluid.layers.** :ref:`api_fluid_layers_conv2d` ). This operator receives input sequences with variable length and other convolutional configuration parameters(num_filters, filter_size) to apply the convolution operation. @@ -146,31 +155,35 @@ def sequence_conv(input, x_conved = paddle.static.nn.sequence_conv(input=x, num_filters=2, filter_size=3, padding_start=-1) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_conv') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_conv' + ) helper = LayerHelper('sequence_conv', **locals()) dtype = helper.input_dtype() filter_shape = [filter_size * input.shape[1], num_filters] - filter_param = helper.create_parameter(attr=helper.param_attr, - shape=filter_shape, - dtype=dtype) + filter_param = helper.create_parameter( + attr=helper.param_attr, shape=filter_shape, dtype=dtype + ) pre_bias = helper.create_variable_for_type_inference(dtype) if padding_start is None: padding_start = -int(filter_size // 2) - helper.append_op(type='sequence_conv', - inputs={ - 'X': [input], - 'Filter': [filter_param], - }, - outputs={"Out": pre_bias}, - attrs={ - 'contextStride': filter_stride, - 'contextStart': padding_start, - 'contextLength': filter_size, - }) + helper.append_op( + type='sequence_conv', + inputs={ + 'X': [input], + 'Filter': [filter_param], + }, + outputs={"Out": pre_bias}, + attrs={ + 'contextStride': filter_stride, + 'contextStart': padding_start, + 'contextLength': filter_size, + }, + ) pre_act = helper.append_bias_op(pre_bias) return helper.append_activation(pre_act) @@ -244,17 +257,21 @@ def sequence_softmax(input, use_cudnn=False, name=None): dtype='float32', lod_level=1) x_sequence_softmax_2 = paddle.static.nn.sequence_softmax(input=y) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_softmax', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_softmax') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_softmax' + ) dtype = helper.input_dtype() softmax_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="sequence_softmax", - inputs={"X": input}, - outputs={"Out": softmax_out}, - attrs={"use_cudnn": use_cudnn}) + helper.append_op( + type="sequence_softmax", + inputs={"X": input}, + outputs={"Out": softmax_out}, + attrs={"use_cudnn": use_cudnn}, + ) return softmax_out @@ -345,26 +362,27 @@ def sequence_pool(input, pool_type, is_test=False, pad_value=0.0): last_x = paddle.static.nn.sequence_pool(input=x, pool_type='last') first_x = paddle.static.nn.sequence_pool(input=x, pool_type='first') """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_pool') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_pool' + ) helper = LayerHelper('sequence_pool', **locals()) dtype = helper.input_dtype() pool_out = helper.create_variable_for_type_inference(dtype) max_index = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="sequence_pool", - inputs={"X": input}, - outputs={ - "Out": pool_out, - "MaxIndex": max_index - }, - attrs={ - "pooltype": pool_type.upper(), - "is_test": is_test, - "pad_value": pad_value - }) + helper.append_op( + type="sequence_pool", + inputs={"X": input}, + outputs={"Out": pool_out, "MaxIndex": max_index}, + attrs={ + "pooltype": pool_type.upper(), + "is_test": is_test, + "pad_value": pad_value, + }, + ) # when pool_type is max, variable max_index is initialized, # so we stop the gradient explicitly here @@ -422,20 +440,24 @@ def sequence_concat(input, name=None): y = paddle.static.data(name='y', shape=[-1, 10], dtype='float32', lod_level=1) out = paddle.static.nn.sequence_concat(input=[x, y]) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_concat', **locals()) check_type(input, 'input', list, 'fluid.layers.sequence_concat') for i, input_x in enumerate(input): - check_variable_and_dtype(input_x, 'input[' + str(i) + ']', - ['int64', 'float32', 'float64'], - 'fluid.layers.sequence_concat') + check_variable_and_dtype( + input_x, + 'input[' + str(i) + ']', + ['int64', 'float32', 'float64'], + 'fluid.layers.sequence_concat', + ) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='sequence_concat', - inputs={'X': input}, - outputs={'Out': [out]}) + helper.append_op( + type='sequence_concat', inputs={'X': input}, outputs={'Out': [out]} + ) return out @@ -491,8 +513,9 @@ def sequence_first_step(input): x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1) x_first_step = paddle.static.nn.sequence_first_step(input=x) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_first_step') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_first_step' + ) return sequence_pool(input=input, pool_type="first") @@ -549,8 +572,9 @@ def sequence_last_step(input): x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1) x_last_step = paddle.static.nn.sequence_last_step(input=x) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_last_step') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_last_step' + ) return sequence_pool(input=input, pool_type="last") @@ -615,17 +639,23 @@ def sequence_slice(input, offset, length, name=None): subseqs = paddle.static.nn.sequence_slice(input=seqs, offset=offset, length=length) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper("sequence_slice", **locals()) - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'sequence_slice') - check_variable_and_dtype(offset, 'offset', ['int32', 'int64'], - 'sequence_slice') - check_variable_and_dtype(length, 'length', ['int32', 'int64'], - 'sequence_slice') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64', 'int32', 'int64'], + 'sequence_slice', + ) + check_variable_and_dtype( + offset, 'offset', ['int32', 'int64'], 'sequence_slice' + ) + check_variable_and_dtype( + length, 'length', ['int32', 'int64'], 'sequence_slice' + ) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) @@ -633,13 +663,11 @@ def sequence_slice(input, offset, length, name=None): offset.stop_gradient = True length.stop_gradient = True - helper.append_op(type="sequence_slice", - inputs={ - "X": input, - "Offset": offset, - "Length": length - }, - outputs={"Out": out}) + helper.append_op( + type="sequence_slice", + inputs={"X": input, "Offset": offset, "Length": length}, + outputs={"Out": out}, + ) return out @@ -765,20 +793,21 @@ def sequence_expand(x, y, ref_level=-1, name=None): # dtype: float # data: [1 2 1 2 3 4 3 4] """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'sequence_expand') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'sequence_expand' + ) helper = LayerHelper('sequence_expand', **locals()) dtype = helper.input_dtype(input_param_name='x') tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='sequence_expand', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': tmp}, - attrs={'ref_level': ref_level}) + helper.append_op( + type='sequence_expand', + inputs={'X': x, 'Y': y}, + outputs={'Out': tmp}, + attrs={'ref_level': ref_level}, + ) return tmp @@ -886,20 +915,19 @@ def sequence_expand_as(x, y, name=None): # dtype: float # data: [1 1 1 2 2 2 3 4] """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'sequence_expand_as') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'sequence_expand_as' + ) check_type(y, 'y', Variable, 'sequence_expand_as') helper = LayerHelper('sequence_expand_as', **locals()) dtype = helper.input_dtype(input_param_name='x') tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='sequence_expand_as', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': tmp}) + helper.append_op( + type='sequence_expand_as', inputs={'X': x, 'Y': y}, outputs={'Out': tmp} + ) return tmp @@ -990,14 +1018,22 @@ def sequence_pad(x, pad_value, maxlen=None, name=None): out = paddle.static.nn.sequence_pad(x=x, pad_value=pad_value) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_pad', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'fluid.layers.sequence_pad') - check_variable_and_dtype(pad_value, 'pad_value', - ['float32', 'float64', 'int32', 'int64'], - 'fluid.layers.sequence_pad') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'fluid.layers.sequence_pad', + ) + check_variable_and_dtype( + pad_value, + 'pad_value', + ['float32', 'float64', 'int32', 'int64'], + 'fluid.layers.sequence_pad', + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) length = helper.create_variable_for_type_inference(VarDesc.VarType.INT64) @@ -1007,16 +1043,12 @@ def sequence_pad(x, pad_value, maxlen=None, name=None): if maxlen is None: maxlen = -1 - helper.append_op(type='sequence_pad', - inputs={ - 'X': x, - 'PadValue': pad_value - }, - outputs={ - 'Out': out, - 'Length': length - }, - attrs={'padded_length': maxlen}) + helper.append_op( + type='sequence_pad', + inputs={'X': x, 'PadValue': pad_value}, + outputs={'Out': out, 'Length': length}, + attrs={'padded_length': maxlen}, + ) return out, length @@ -1030,22 +1062,22 @@ def sequence_unpad(x, length, name=None): .. code-block:: text - Case 1: + Case 1: - Given input Variable **x**: - x.data = [[ 1.0, 2.0, 3.0, 4.0, 5.0], - [ 6.0, 7.0, 8.0, 9.0, 10.0], - [11.0, 12.0, 13.0, 14.0, 15.0]], + Given input Variable **x**: + x.data = [[ 1.0, 2.0, 3.0, 4.0, 5.0], + [ 6.0, 7.0, 8.0, 9.0, 10.0], + [11.0, 12.0, 13.0, 14.0, 15.0]], - in which there are 3 sequences padded to length 5, and the actual length - specified by input Variable **length**: + in which there are 3 sequences padded to length 5, and the actual length + specified by input Variable **length**: - length.data = [2, 3, 4], + length.data = [2, 3, 4], - after unpadding, the output Variable will be: + after unpadding, the output Variable will be: - out.data = [[1.0, 2.0, 6.0, 7.0, 8.0, 11.0, 12.0, 13.0, 14.0]] - out.lod = [[0, 2, 5, 9]] + out.data = [[1.0, 2.0, 6.0, 7.0, 8.0, 11.0, 12.0, 13.0, 14.0]] + out.lod = [[0, 2, 5, 9]] Args: x(Variable): A Tensor which contains padding data, and its shape size can not be less than 2. @@ -1075,24 +1107,29 @@ def sequence_unpad(x, length, name=None): unpad_data = paddle.static.nn.sequence_unpad(x=pad_data, length=len) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_unpad', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'fluid.layers.sequence_unpad') - check_variable_and_dtype(length, 'length', ['int64'], - 'fluid.layers.sequence_unpad') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'fluid.layers.sequence_unpad', + ) + check_variable_and_dtype( + length, 'length', ['int64'], 'fluid.layers.sequence_unpad' + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) length.stop_gradient = True - helper.append_op(type='sequence_unpad', - inputs={ - 'X': x, - 'Length': length - }, - outputs={'Out': out}) + helper.append_op( + type='sequence_unpad', + inputs={'X': x, 'Length': length}, + outputs={'Out': out}, + ) return out @@ -1145,17 +1182,23 @@ def sequence_reshape(input, new_dim): x = paddle.static.data(name='x', shape=[None, 16], dtype='float32', lod_level=1) x_reshaped = paddle.static.nn.sequence_reshape(input=x, new_dim=4) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_reshape', **locals()) - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'fluid.layers.sequence_reshape') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64', 'int32', 'int64'], + 'fluid.layers.sequence_reshape', + ) out = helper.create_variable_for_type_inference(helper.input_dtype()) - helper.append_op(type='sequence_reshape', - inputs={'X': [input]}, - outputs={'Out': [out]}, - attrs={'new_dim': new_dim}) + helper.append_op( + type='sequence_reshape', + inputs={'X': [input]}, + outputs={'Out': [out]}, + attrs={'new_dim': new_dim}, + ) return out @@ -1224,28 +1267,34 @@ def sequence_scatter(input, index, updates, name=None): output = paddle.static.nn.sequence_scatter(input, index, updates) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_scatter', **locals()) - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'sequence_scatter') - check_variable_and_dtype(index, 'index', ['int32', 'int64'], - 'sequence_scatter') - check_variable_and_dtype(updates, 'updates', - ['float32', 'float64', 'int32', 'int64'], - 'sequence_scatter') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64', 'int32', 'int64'], + 'sequence_scatter', + ) + check_variable_and_dtype( + index, 'index', ['int32', 'int64'], 'sequence_scatter' + ) + check_variable_and_dtype( + updates, + 'updates', + ['float32', 'float64', 'int32', 'int64'], + 'sequence_scatter', + ) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="sequence_scatter", - inputs={ - "X": input, - "Ids": index, - "Updates": updates - }, - outputs={"Out": out}) + helper.append_op( + type="sequence_scatter", + inputs={"X": input, "Ids": index, "Updates": updates}, + outputs={"Out": out}, + ) return out @@ -1300,20 +1349,22 @@ def sequence_enumerate(input, win_size, pad_value=0, name=None): x = paddle.static.data(name='x', shape=[-1, 1], dtype='int32', lod_level=1) out = paddle.static.nn.sequence_enumerate(input=x, win_size=3, pad_value=0) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(input, 'input', ['int32', 'int64'], - 'sequence_enumerate') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + input, 'input', ['int32', 'int64'], 'sequence_enumerate' + ) helper = LayerHelper('sequence_enumerate', **locals()) - out = helper.create_variable_for_type_inference(helper.input_dtype(), - stop_gradient=True) - helper.append_op(type='sequence_enumerate', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'win_size': win_size, - 'pad_value': pad_value - }) + out = helper.create_variable_for_type_inference( + helper.input_dtype(), stop_gradient=True + ) + helper.append_op( + type='sequence_enumerate', + inputs={'X': input}, + outputs={'Out': out}, + attrs={'win_size': win_size, 'pad_value': pad_value}, + ) return out @@ -1427,16 +1478,22 @@ def sequence_reverse(x, name=None): x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1) x_reversed = paddle.static.nn.sequence_reverse(x) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper("sequence_reverse", **locals()) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'int8', 'int32', 'int64'], - 'fluid.layers.sequence_reverse') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int8', 'int32', 'int64'], + 'fluid.layers.sequence_reverse', + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="sequence_reverse", - inputs={"X": x}, - outputs={"Y": out}, - attrs=dict()) + helper.append_op( + type="sequence_reverse", + inputs={"X": x}, + outputs={"Y": out}, + attrs=dict(), + ) return out diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 5e82bef749b..a282fae027f 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -19,14 +19,28 @@ import warnings from ..layer_helper import LayerHelper from ..param_attr import ParamAttr from ..initializer import Initializer -from ..framework import _current_expected_place, convert_np_dtype_to_dtype_, _non_static_mode, _varbase_creator, device_guard, _in_legacy_dygraph, in_dygraph_mode, _get_paddle_place +from ..framework import ( + _current_expected_place, + convert_np_dtype_to_dtype_, + _non_static_mode, + _varbase_creator, + device_guard, + _in_legacy_dygraph, + in_dygraph_mode, + _get_paddle_place, +) from ..framework import Variable from ..initializer import Constant from ..core import VarDesc from .. import core from .layer_function_generator import templatedoc from . import utils -from ..data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from ..data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from paddle.utils import deprecated from .utils import check_shape @@ -83,24 +97,32 @@ def create_tensor(dtype, name=None, persistable=False): import paddle.fluid as fluid tensor = fluid.layers.create_tensor(dtype='float32') """ - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'int8', 'int32', 'int32', - 'int64' - ], 'create_tensor') + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int32', + 'int32', + 'int64', + ], + 'create_tensor', + ) helper = LayerHelper("create_tensor", **locals()) - return helper.create_variable(name=helper.name, - dtype=dtype, - persistable=persistable) + return helper.create_variable( + name=helper.name, dtype=dtype, persistable=persistable + ) -def create_parameter(shape, - dtype, - name=None, - attr=None, - is_bias=False, - default_initializer=None): +def create_parameter( + shape, dtype, name=None, attr=None, is_bias=False, default_initializer=None +): """ - :api_attr: Static Graph + :api_attr: Static Graph This function creates a parameter. The parameter is a learnable variable, which can have gradient, and can be optimized. @@ -132,31 +154,55 @@ def create_parameter(shape, """ check_type(shape, 'shape', (list, tuple, numpy.ndarray), 'create_parameter') for item in shape: - check_type(item, 'item of shape', - (int, numpy.uint8, numpy.int8, numpy.int16, numpy.int32, - numpy.int64), 'create_parameter') - - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8' - ], 'create_parameter') + check_type( + item, + 'item of shape', + ( + int, + numpy.uint8, + numpy.int8, + numpy.int16, + numpy.int32, + numpy.int64, + ), + 'create_parameter', + ) + + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + ], + 'create_parameter', + ) check_type(attr, 'attr', (type(None), ParamAttr), 'create_parameter') - check_type(default_initializer, 'default_initializer', - (type(None), Initializer), 'create_parameter') + check_type( + default_initializer, + 'default_initializer', + (type(None), Initializer), + 'create_parameter', + ) helper = LayerHelper("create_parameter", **locals()) if attr is None: attr = ParamAttr(name=name) - return helper.create_parameter(attr, shape, convert_dtype(dtype), is_bias, - default_initializer) + return helper.create_parameter( + attr, shape, convert_dtype(dtype), is_bias, default_initializer + ) -def create_global_var(shape, - value, - dtype, - persistable=False, - force_cpu=False, - name=None): +def create_global_var( + shape, value, dtype, persistable=False, force_cpu=False, name=None +): """ This function creates a new tensor variable with value in the global block(block 0). @@ -183,35 +229,53 @@ def create_global_var(shape, var = paddle.static.create_global_var(shape=[2,3], value=1.0, dtype='float32', persistable=True, force_cpu=True, name='new_var') """ - check_type(shape, 'shape', (list, tuple, numpy.ndarray), - 'create_global_var') + check_type( + shape, 'shape', (list, tuple, numpy.ndarray), 'create_global_var' + ) for item in shape: - check_type(item, 'item of shape', - (int, numpy.uint8, numpy.int8, numpy.int16, numpy.int32, - numpy.int64), 'create_global_var') - - check_dtype(dtype, 'dtype', [ - 'bool', - 'float16', - 'float32', - 'float64', - 'int8', - 'int16', - 'int32', - 'int64', - 'uint8', - 'uint16', - ], 'create_global_var') + check_type( + item, + 'item of shape', + ( + int, + numpy.uint8, + numpy.int8, + numpy.int16, + numpy.int32, + numpy.int64, + ), + 'create_global_var', + ) + + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'create_global_var', + ) helper = LayerHelper("global_var", **locals()) - var = helper.create_global_variable(dtype=dtype, - shape=shape, - persistable=persistable, - name=name, - stop_gradient=True) - helper.set_variable_initializer(var, - initializer=Constant(value=float(value), - force_cpu=force_cpu)) + var = helper.create_global_variable( + dtype=dtype, + shape=shape, + persistable=persistable, + name=name, + stop_gradient=True, + ) + helper.set_variable_initializer( + var, initializer=Constant(value=float(value), force_cpu=force_cpu) + ) return var @@ -251,25 +315,50 @@ def cast(x, dtype): out = _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', - 'uint8', 'uint16' - ], 'cast') - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8', 'uint16' - ], 'cast') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'cast', + ) + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'cast', + ) helper = LayerHelper('cast', **locals()) out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=x.stop_gradient) - helper.append_op(type='cast', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': out.dtype - }) + dtype=dtype, stop_gradient=x.stop_gradient + ) + helper.append_op( + type='cast', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'in_dtype': x.dtype, 'out_dtype': out.dtype}, + ) return out @@ -344,9 +433,11 @@ def concat(input, axis=0, name=None): if not isinstance(input, Variable): for id, x in enumerate(input): check_variable_and_dtype( - x, 'input[' + str(id) + ']', + x, + 'input[' + str(id) + ']', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'concat') + 'concat', + ) if x.dtype != input[0].dtype: raise TypeError( "All the Tensors in the input must have the same data type." @@ -357,8 +448,11 @@ def concat(input, axis=0, name=None): if isinstance(axis, Variable): check_dtype( - axis.dtype, 'axis', ['int32', 'int64'], 'concat', - "The data type of axis must be int32 or int64 when axis is a Tensor" + axis.dtype, + 'axis', + ['int32', 'int64'], + 'concat', + "The data type of axis must be int32 or int64 when axis is a Tensor", ) helper = LayerHelper('concat', **locals()) @@ -369,19 +463,17 @@ def concat(input, axis=0, name=None): # This feature is supported for Dynamic-to-Static, because after transformed, the type of inputs[0] # is LOD_TENSOR_ARRAY in some scenarios. And this feature can be used in static mode. - assert len(input) == 1, "If the elements of 'input' in concat are Variable(LoDTensorArray), " \ - "number of the elements must be 1, but received %s." % len(input) + assert len(input) == 1, ( + "If the elements of 'input' in concat are Variable(LoDTensorArray), " + "number of the elements must be 1, but received %s." % len(input) + ) out_index = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': input[0]}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': False - }) + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': input[0]}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': False}, + ) else: inputs = {'X': input} attrs = {} @@ -389,10 +481,9 @@ def concat(input, axis=0, name=None): axis.stop_gradient = True attrs['axis'] = axis - helper.append_op(type='concat', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='concat', inputs=inputs, outputs={'Out': [out]}, attrs=attrs + ) return out @@ -478,33 +569,36 @@ def tensor_array_to_tensor(input, axis=1, name=None, use_stack=False): """ if _non_static_mode(): assert isinstance( - input, list), "The 'input' in tensor_array_to_tensor must be list" + input, list + ), "The 'input' in tensor_array_to_tensor must be list" from .nn import stack, concat from ..dygraph import to_variable + op = stack if use_stack else concat res = op(input, axis=axis) sizes = to_variable( - numpy.array(list(map(lambda x: int(x.shape[axis]), input)))) + numpy.array(list(map(lambda x: int(x.shape[axis]), input))) + ) return res, sizes check_type(input, 'input', (list, Variable), 'tensor_array_to_tensor') if isinstance(input, list): for i, input_x in enumerate(input): - check_type(input_x, 'input[' + str(i) + ']', Variable, - 'tensor_array_to_tensor') + check_type( + input_x, + 'input[' + str(i) + ']', + Variable, + 'tensor_array_to_tensor', + ) helper = LayerHelper('tensor_array_to_tensor', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) out_index = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': input}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': use_stack - }) + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': input}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': use_stack}, + ) return out, out_index @@ -561,25 +655,36 @@ def sums(input, out=None): check_type(input, 'input', (Variable, tuple, list), 'sums') if isinstance(input, list) or isinstance(input, tuple): for input_section in input: - check_variable_and_dtype(input_section, "input", \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'sums') + check_variable_and_dtype( + input_section, + "input", + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'sums', + ) else: - check_variable_and_dtype(input, "input", \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'sums') + check_variable_and_dtype( + input, + "input", + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'sums', + ) helper = LayerHelper('sum', **locals()) if out is None: out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) else: - check_variable_and_dtype(out, "out", - ['float32', 'float64', 'int32', 'int64'], - 'sums') - - helper.append_op(type='sum', - inputs={'X': input}, - outputs={'Out': out}, - attrs={'use_mkldnn': False}) + check_variable_and_dtype( + out, "out", ['float32', 'float64', 'int32', 'int64'], 'sums' + ) + + helper.append_op( + type='sum', + inputs={'X': input}, + outputs={'Out': out}, + attrs={'use_mkldnn': False}, + ) return out @@ -614,9 +719,12 @@ def assign(input, output=None): result3 = paddle.assign(np.array([[2.5, 2.5], [2.5, 2.5], [2.5, 2.5]], dtype='float32')) # result3 = [[2.5, 2.5], [2.5, 2.5], [2.5, 2.5]] """ helper = LayerHelper('assign', **locals()) - check_type(input, 'input', - (Variable, numpy.ndarray, list, tuple, float, int, bool), - 'assign') + check_type( + input, + 'input', + (Variable, numpy.ndarray, list, tuple, float, int, bool), + 'assign', + ) is_inplace = True if output is not None else False if numpy.isscalar(input) and not isinstance(input, str): @@ -642,16 +750,29 @@ def assign(input, output=None): output = core.eager.Tensor() _legacy_C_ops.assign(input, output) else: - check_dtype(input.dtype, 'input', [ - 'float16', 'uint16', 'float32', 'float64', 'int32', 'int64', - 'uint8', 'bool' - ], 'assign', '(When the type of input in assign is Variable.)') + check_dtype( + input.dtype, + 'input', + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'bool', + ], + 'assign', + '(When the type of input in assign is Variable.)', + ) if output is None: output = helper.create_variable_for_type_inference( - dtype=input.dtype) - helper.append_op(type='assign', - inputs={'X': [input]}, - outputs={'Out': [output]}) + dtype=input.dtype + ) + helper.append_op( + type='assign', inputs={'X': [input]}, outputs={'Out': [output]} + ) elif isinstance(input, numpy.ndarray): # Not support [var, var, ...] currently. if len(input.shape) > 0 and any(isinstance(x, Variable) for x in input): @@ -665,7 +786,8 @@ def assign(input, output=None): warnings.warn( "paddle.assign doesn't support float64 input now due " "to current platform protobuf data limitation, we convert " - "it to float32") + "it to float32" + ) dtype = VarDesc.VarType.FP32 if dtype == VarDesc.VarType.BOOL: value_name = "bool_values" @@ -683,31 +805,49 @@ def assign(input, output=None): raise TypeError( "When the type of 'input' in assign is numpy.ndarray, " "the data type of 'input' must be bool, float32, int32 or int64, but " - "received %s." % convert_dtype(dtype)) + "received %s." % convert_dtype(dtype) + ) if input.size > 1024 * 1024: - raise ValueError("The size of input is too big. Please consider " - "saving it to file and 'load_op' to load it") + raise ValueError( + "The size of input is too big. Please consider " + "saving it to file and 'load_op' to load it" + ) if in_dygraph_mode(): if output is None: output = zeros(list(input.shape), dtype) - _C_ops.assign_value_(output, list(input.shape), dtype, values, - _current_expected_place()) + _C_ops.assign_value_( + output, + list(input.shape), + dtype, + values, + _current_expected_place(), + ) elif _in_legacy_dygraph(): if output is None: output = core.VarBase() - _legacy_C_ops.assign_value(output, 'shape', list(input.shape), - 'dtype', dtype, value_name, values) + _legacy_C_ops.assign_value( + output, + 'shape', + list(input.shape), + 'dtype', + dtype, + value_name, + values, + ) else: if output is None: output = helper.create_variable_for_type_inference( - dtype=input.dtype) - helper.append_op(type='assign_value', - outputs={'Out': [output]}, - attrs={ - 'dtype': dtype, - 'shape': list(input.shape), - value_name: values - }) + dtype=input.dtype + ) + helper.append_op( + type='assign_value', + outputs={'Out': [output]}, + attrs={ + 'dtype': dtype, + 'shape': list(input.shape), + value_name: values, + }, + ) if is_inplace and _non_static_mode(): output._bump_inplace_version() @@ -783,7 +923,11 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None, name=None): shape = list( map( lambda x: x.numpy().flat[0] - if isinstance(x, Variable) else x, shape)) + if isinstance(x, Variable) + else x, + shape, + ) + ) break if not isinstance(dtype, core.VarDesc.VarType): @@ -811,9 +955,19 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None, name=None): else: attrs['str_value'] = str(float(value.numpy().item(0))) - _legacy_C_ops.fill_constant(out, 'value', float(value), 'force_cpu', - force_cpu, 'dtype', out.dtype, 'str_value', - attrs['str_value'], 'shape', shape) + _legacy_C_ops.fill_constant( + out, + 'value', + float(value), + 'force_cpu', + force_cpu, + 'dtype', + out.dtype, + 'str_value', + attrs['str_value'], + 'shape', + shape, + ) out.stop_gradient = True return out @@ -825,43 +979,60 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None, name=None): inputs['ValueTensor'] = value check_shape(shape) - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'uint8', 'int16', 'int32', - 'int64', 'complex64', 'complex128' - ], 'fill_constant') + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'uint8', + 'int16', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'fill_constant', + ) check_type(shape, 'shape', (Variable, list, tuple), 'fill_constant') if out is not None: - check_variable_and_dtype(out, 'out', [convert_dtype(dtype)], - 'fill_constant') + check_variable_and_dtype( + out, 'out', [convert_dtype(dtype)], 'fill_constant' + ) helper = LayerHelper("fill_constant", **locals()) - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='fill_constant') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='fill_constant' + ) if out is None: out = helper.create_variable_for_type_inference(dtype=dtype) attrs['dtype'] = out.dtype - helper.append_op(type='fill_constant', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs, - stop_gradient=True) + helper.append_op( + type='fill_constant', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + stop_gradient=True, + ) out.stop_gradient = True return out @deprecated(since='1.8.0', update_to="paddle.fluid.layers.fill_constant") @templatedoc() -def fill_constant_batch_size_like(input, - shape, - dtype, - value, - input_dim_idx=0, - output_dim_idx=0, - force_cpu=False): +def fill_constant_batch_size_like( + input, + shape, + dtype, + value, + input_dim_idx=0, + output_dim_idx=0, + force_cpu=False, +): """ This OP creates a Tesnor according the shape and dtype, and initializes the Tensor with the constants provided in ``value``. When the input is LoDTensor @@ -903,8 +1074,9 @@ def fill_constant_batch_size_like(input, place = _current_expected_place() if force_cpu: place = core.CPUPlace() - out = _C_ops.full_batch_size_like(input, shape, dtype, value, - input_dim_idx, output_dim_idx, place) + out = _C_ops.full_batch_size_like( + input, shape, dtype, value, input_dim_idx, output_dim_idx, place + ) out.stop_gradient = True return out @@ -916,25 +1088,27 @@ def fill_constant_batch_size_like(input, 'value': float(value), 'input_dim_idx': input_dim_idx, 'output_dim_idx': output_dim_idx, - 'force_cpu': force_cpu + 'force_cpu': force_cpu, } if convert_dtype(dtype) in ['int64', 'int32']: attrs['str_value'] = str(int(value)) else: attrs['str_value'] = str(float(value)) - helper.append_op(type='fill_constant_batch_size_like', - inputs={'Input': input}, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='fill_constant_batch_size_like', + inputs={'Input': input}, + outputs={'Out': [out]}, + attrs=attrs, + ) out.stop_gradient = True return out def argmin(x, axis=0): """ - :alias_main: paddle.argmin - :alias: paddle.argmin,paddle.tensor.argmin,paddle.tensor.search.argmin - :old_api: paddle.fluid.layers.argmin + :alias_main: paddle.argmin + :alias: paddle.argmin,paddle.tensor.argmin,paddle.tensor.search.argmin + :old_api: paddle.fluid.layers.argmin **argmin** @@ -984,14 +1158,19 @@ def argmin(x, axis=0): # [1 0 2]] """ check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'uint8', 'int16', 'int32', 'int64'], - 'argmin') + x, + 'x', + ['float32', 'float64', 'uint8', 'int16', 'int32', 'int64'], + 'argmin', + ) helper = LayerHelper("arg_min", **locals()) out = helper.create_variable_for_type_inference(VarDesc.VarType.INT64) - helper.append_op(type='arg_min', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs={'axis': axis}) + helper.append_op( + type='arg_min', + inputs={'X': x}, + outputs={'Out': [out]}, + attrs={'axis': axis}, + ) out.stop_gradient = True return out @@ -1046,23 +1225,28 @@ def argmax(x, axis=0): # [0 3 1]] """ check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'uint8', 'int16', 'int32', 'int64'], - 'argmax') + x, + 'x', + ['float32', 'float64', 'uint8', 'int16', 'int32', 'int64'], + 'argmax', + ) helper = LayerHelper("arg_max", **locals()) out = helper.create_variable_for_type_inference(VarDesc.VarType.INT64) - helper.append_op(type='arg_max', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs={'axis': axis}) + helper.append_op( + type='arg_max', + inputs={'X': x}, + outputs={'Out': [out]}, + attrs={'axis': axis}, + ) out.stop_gradient = True return out def argsort(input, axis=-1, descending=False, name=None): """ - :alias_main: paddle.argsort - :alias: paddle.argsort,paddle.tensor.argsort,paddle.tensor.search.argsort - :old_api: paddle.fluid.layers.argsort + :alias_main: paddle.argsort + :alias: paddle.argsort,paddle.tensor.argsort,paddle.tensor.search.argsort + :old_api: paddle.fluid.layers.argsort This OP sorts the input along the given axis, and returns sorted output data Varibale and its corresponding index Variable with the same shape as @@ -1133,23 +1317,24 @@ def argsort(input, axis=-1, descending=False, name=None): # [5. 7. 7. 9.]]] """ check_variable_and_dtype( - input, 'input', - ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], 'argsort') + input, + 'input', + ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], + 'argsort', + ) helper = LayerHelper("argsort", **locals()) - out = helper.create_variable_for_type_inference(dtype=input.dtype, - stop_gradient=True) - ids = helper.create_variable_for_type_inference(VarDesc.VarType.INT64, - stop_gradient=True) - helper.append_op(type='argsort', - inputs={'X': input}, - outputs={ - 'Out': out, - 'Indices': ids - }, - attrs={ - 'axis': axis, - 'descending': descending - }) + out = helper.create_variable_for_type_inference( + dtype=input.dtype, stop_gradient=True + ) + ids = helper.create_variable_for_type_inference( + VarDesc.VarType.INT64, stop_gradient=True + ) + helper.append_op( + type='argsort', + inputs={'X': input}, + outputs={'Out': out, 'Indices': ids}, + attrs={'axis': axis, 'descending': descending}, + ) return out, ids @@ -1215,9 +1400,9 @@ def zeros(shape, dtype, force_cpu=False, name=None): def reverse(x, axis): """ - :alias_main: paddle.reverse - :alias: paddle.reverse,paddle.tensor.reverse,paddle.tensor.manipulation.reverse - :old_api: paddle.fluid.layers.reverse + :alias_main: paddle.reverse + :alias: paddle.reverse,paddle.tensor.reverse,paddle.tensor.manipulation.reverse + :old_api: paddle.fluid.layers.reverse The OP reverses the tensor :attr:`x` along the given :attr:`axis`. @@ -1275,9 +1460,9 @@ def reverse(x, axis): reversed_tensor_array = fluid.layers.reverse(tensor_array, 0) # {[[3, 4, 5]], [[0, 1, 2]]} """ - check_variable_and_dtype(x, 'x', - ('float32', 'float64', 'int32', 'int64', 'uint8'), - 'reverse') + check_variable_and_dtype( + x, 'x', ('float32', 'float64', 'int32', 'int64', 'uint8'), 'reverse' + ) check_type(axis, 'axis', (int, tuple, list, Variable), 'reverse') if isinstance(axis, int): axis = [axis] @@ -1285,10 +1470,12 @@ def reverse(x, axis): return _C_ops.reverse(x, axis) helper = LayerHelper("reverse", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reverse', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs={'axis': axis}) + helper.append_op( + type='reverse', + inputs={'X': x}, + outputs={'Out': [out]}, + attrs={'axis': axis}, + ) return out @@ -1304,13 +1491,12 @@ def save(x, file_path, overwrite=True): error will be thrown. """ helper = LayerHelper("save", **locals()) - helper.append_op(type="save", - inputs={"input": x}, - outputs={}, - args={ - "file_path": file_path, - "overwrite": overwrite - }) + helper.append_op( + type="save", + inputs={"input": x}, + outputs={}, + args={"file_path": file_path, "overwrite": overwrite}, + ) def save_combine(x, file_path, overwrite=True): @@ -1342,13 +1528,12 @@ def save_combine(x, file_path, overwrite=True): normed = fluid.layers.save_combine([v1, v2], file_path="output") """ helper = LayerHelper("save_combine", **locals()) - helper.append_op(type="save_combine", - inputs={"input": x}, - outputs={}, - args={ - "file_path": file_path, - "overwrite": overwrite - }) + helper.append_op( + type="save_combine", + inputs={"input": x}, + outputs={}, + args={"file_path": file_path, "overwrite": overwrite}, + ) def load_combine(out, file_path): @@ -1360,10 +1545,12 @@ def load_combine(out, file_path): file_path(str): The path of the disk file. """ helper = LayerHelper("load_combine", **locals()) - helper.append_op(type="load_combine", - inputs={}, - output={"Out": out}, - args={"file_path": file_path}) + helper.append_op( + type="load_combine", + inputs={}, + output={"Out": out}, + args={"file_path": file_path}, + ) def has_inf(x): @@ -1447,8 +1634,9 @@ def isfinite(x): print(y) """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "isfinite") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "isfinite" + ) helper = LayerHelper("isfinite", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') @@ -1506,8 +1694,11 @@ def range(start, end, step, dtype, name=None): """ out_shape = None - if not isinstance(start, Variable) and not isinstance( - end, Variable) and not isinstance(step, Variable): + if ( + not isinstance(start, Variable) + and not isinstance(end, Variable) + and not isinstance(step, Variable) + ): out_shape = [int(math.ceil((end - start) / step))] if not isinstance(dtype, core.VarDesc.VarType): @@ -1539,17 +1730,16 @@ def range(start, end, step, dtype, name=None): out.stop_gradient = True return out - check_dtype(dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], - 'range/arange') + check_dtype( + dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], 'range/arange' + ) helper = LayerHelper('range', **locals()) out = helper.create_variable_for_type_inference(dtype, shape=out_shape) - helper.append_op(type='range', - inputs={ - 'Start': start, - 'End': end, - 'Step': step - }, - outputs={'Out': out}) + helper.append_op( + type='range', + inputs={'Start': start, 'End': end, 'Step': step}, + outputs={'Out': out}, + ) out.stop_gradient = True if out_shape is not None: out.desc.set_shape(out_shape) @@ -1604,52 +1794,70 @@ def linspace(start, stop, num, dtype=None, name=None): with device_guard("cpu"): tensor_num = fill_constant([1], 'int32', num) if in_dygraph_mode(): - return _C_ops.linspace(tensor_start, tensor_stop, tensor_num, dtype, - _current_expected_place()) + return _C_ops.linspace( + tensor_start, + tensor_stop, + tensor_num, + dtype, + _current_expected_place(), + ) if _in_legacy_dygraph(): - return _legacy_C_ops.linspace(tensor_start, tensor_stop, tensor_num, - 'dtype', dtype) + return _legacy_C_ops.linspace( + tensor_start, tensor_stop, tensor_num, 'dtype', dtype + ) helper = LayerHelper("linspace", **locals()) start_dtype = convert_dtype(tensor_start.dtype) stop_dtype = convert_dtype(tensor_stop.dtype) out_dtype = convert_dtype(dtype) if isinstance(start, Variable): - check_dtype(start.dtype, 'start', - ['float32', 'float64', 'int32', 'int64'], 'linspace') + check_dtype( + start.dtype, + 'start', + ['float32', 'float64', 'int32', 'int64'], + 'linspace', + ) else: check_type(start, 'start', (int, float), 'linspace') if isinstance(stop, Variable): - check_dtype(stop.dtype, 'stop', - ['float32', 'float64', 'int32', 'int64'], 'linspace') + check_dtype( + stop.dtype, + 'stop', + ['float32', 'float64', 'int32', 'int64'], + 'linspace', + ) else: check_type(stop, 'stop', (int, float), 'linspace') if isinstance(num, Variable): check_dtype(num.dtype, 'num', ['int32'], 'linspace') - check_dtype(dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], - 'linspace') - if ((stop_dtype == "float64" or start_dtype == "float64") - and out_dtype in ["float32", "int32"]) or ( - (stop_dtype == "int64" or start_dtype == "int64") - and out_dtype == "int32"): + check_dtype( + dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], 'linspace' + ) + if ( + (stop_dtype == "float64" or start_dtype == "float64") + and out_dtype in ["float32", "int32"] + ) or ( + (stop_dtype == "int64" or start_dtype == "int64") + and out_dtype == "int32" + ): raise ValueError( "The dtype of start/stop is {}/{} but the attr(dtype) of linspace is {}, " - "which may cause data type overflows. Please reset attr(dtype) of linspace." - .format(start_dtype, stop_dtype, dtype)) + "which may cause data type overflows. Please reset attr(dtype) of linspace.".format( + start_dtype, stop_dtype, dtype + ) + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='linspace', - inputs={ - 'Start': tensor_start, - 'Stop': tensor_stop, - 'Num': tensor_num - }, - attrs={'dtype': dtype}, - outputs={'Out': [out]}) + helper.append_op( + type='linspace', + inputs={'Start': tensor_start, 'Stop': tensor_stop, 'Num': tensor_num}, + attrs={'dtype': dtype}, + outputs={'Out': [out]}, + ) if isinstance(num, int): - out.desc.set_shape((num, )) + out.desc.set_shape((num,)) return out @@ -1679,23 +1887,25 @@ def zeros_like(x, out=None): data = fluid.layers.zeros_like(x) # [0.0, 0.0, 0.0] """ - check_variable_and_dtype(x, "x", - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'zeros_like') + check_variable_and_dtype( + x, "x", ['bool', 'float32', 'float64', 'int32', 'int64'], 'zeros_like' + ) helper = LayerHelper("zeros_like", **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: check_variable_and_dtype( - out, "out", ['bool', 'float32', 'float64', 'int32', 'int64'], - 'zeros_like') - helper.append_op(type='fill_any_like', - inputs={'X': [x]}, - attrs={ - 'value': 0, - "dtype": x.dtype - }, - outputs={'Out': [out]}) + out, + "out", + ['bool', 'float32', 'float64', 'int32', 'int64'], + 'zeros_like', + ) + helper.append_op( + type='fill_any_like', + inputs={'X': [x]}, + attrs={'value': 0, "dtype": x.dtype}, + outputs={'Out': [out]}, + ) out.stop_gradient = True return out @@ -1732,8 +1942,12 @@ def diag(diagonal): """ check_type(diagonal, 'diagonal', (Variable, numpy.ndarray), 'diag') - check_dtype(diagonal.dtype, 'diagonal', - ['float32', 'float64', 'int32', 'int64'], 'diag') + check_dtype( + diagonal.dtype, + 'diagonal', + ['float32', 'float64', 'int32', 'int64'], + 'diag', + ) helper = LayerHelper("diag", **locals()) if not isinstance(diagonal, Variable): @@ -1741,19 +1955,17 @@ def diag(diagonal): out = helper.create_variable_for_type_inference(dtype=diagonal.dtype) - helper.append_op(type='diag', - inputs={'Diagonal': [diagonal]}, - outputs={'Out': [out]}) + helper.append_op( + type='diag', inputs={'Diagonal': [diagonal]}, outputs={'Out': [out]} + ) out.stop_gradient = True return out -def eye(num_rows, - num_columns=None, - batch_shape=None, - dtype='float32', - name=None): +def eye( + num_rows, num_columns=None, batch_shape=None, dtype='float32', name=None +): """ This function constructs a or a batch of 2-D tensor with ones on the diagonal and zeros elsewhere. @@ -1806,25 +2018,33 @@ def eye(num_rows, num_columns = num_rows if in_dygraph_mode(): - out = _C_ops.eye(num_rows, num_columns, dtype, - _current_expected_place()) + out = _C_ops.eye( + num_rows, num_columns, dtype, _current_expected_place() + ) elif _in_legacy_dygraph(): - out = _legacy_C_ops.eye('dtype', dtype, 'num_rows', num_rows, - 'num_columns', num_columns) + out = _legacy_C_ops.eye( + 'dtype', dtype, 'num_rows', num_rows, 'num_columns', num_columns + ) else: helper = LayerHelper("eye", **locals()) - check_dtype(dtype, 'dtype', - ['float16', 'float32', 'float64', 'int32', 'int64'], 'eye') + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'eye', + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='eye', - inputs={}, - outputs={'Out': [out]}, - attrs={ - 'num_rows': num_rows, - 'num_columns': num_columns, - 'dtype': dtype - }, - stop_gradient=True) + helper.append_op( + type='eye', + inputs={}, + outputs={'Out': [out]}, + attrs={ + 'num_rows': num_rows, + 'num_columns': num_columns, + 'dtype': dtype, + }, + stop_gradient=True, + ) if batch_shape is not None: re_shape = [1] * len(batch_shape) @@ -1836,11 +2056,12 @@ def eye(num_rows, if not isinstance(batch_shape, list): raise TypeError("batch_shape should be a list") - for batch_val in (batch_shape): + for batch_val in batch_shape: if batch_val <= 0: raise TypeError("batch_shape should be a positive int list") from .nn import reshape, expand + out = reshape(x=out, shape=re_shape) out = expand(x=out, expand_times=expand_times) @@ -1871,25 +2092,31 @@ def ones_like(x, out=None): data = fluid.layers.ones_like(x) # [1.0, 1.0, 1.0] """ - check_variable_and_dtype(x, "x", - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'ones_like') + check_variable_and_dtype( + x, "x", ['bool', 'float32', 'float64', 'int32', 'int64'], 'ones_like' + ) helper = LayerHelper("ones_like", **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: check_variable_and_dtype( - out, "out", ['bool', 'float32', 'float64', 'int32', 'int64'], - 'ones_like') - helper.append_op(type='fill_any_like', - inputs={'X': [x]}, - attrs={'value': 1.0}, - outputs={'Out': [out]}) + out, + "out", + ['bool', 'float32', 'float64', 'int32', 'int64'], + 'ones_like', + ) + helper.append_op( + type='fill_any_like', + inputs={'X': [x]}, + attrs={'value': 1.0}, + outputs={'Out': [out]}, + ) return out @deprecated(since="2.0.0", update_to="paddle.triu") def triu(input, diagonal=0, name=None): import paddle + return paddle.tensor.triu(x=input, diagonal=diagonal, name=name) diff --git a/python/paddle/fluid/layers/utils.py b/python/paddle/fluid/layers/utils.py index 28d00161b26..e96e105c141 100644 --- a/python/paddle/fluid/layers/utils.py +++ b/python/paddle/fluid/layers/utils.py @@ -17,9 +17,15 @@ import copy import six import numpy as np from ..framework import Block, Variable, _non_static_mode -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from ..layer_helper import LayerHelper from sys import version_info + try: from collections.abc import Sequence except: @@ -54,25 +60,45 @@ def convert_to_list(value, n, name, dtype=int): try: value_list = list(value) except TypeError: - raise ValueError("The " + name + - "'s type must be list or tuple. Received: " + - str(value)) + raise ValueError( + "The " + + name + + "'s type must be list or tuple. Received: " + + str(value) + ) if len(value_list) != n: - raise ValueError("The " + name + "'s length must be " + str(n) + - ". Received: " + str(value)) + raise ValueError( + "The " + + name + + "'s length must be " + + str(n) + + ". Received: " + + str(value) + ) for single_value in value_list: - assert not isinstance( - single_value, Variable - ), "Required numerical type with '%s', but received Tensor." % dtype + assert not isinstance(single_value, Variable), ( + "Required numerical type with '%s', but received Tensor." + % dtype + ) try: dtype(single_value) except (ValueError, TypeError): - raise ValueError("The " + name + - "'s type must be a list or tuple of " + - str(n) + " " + str(dtype) + " . Received: " + - str(value) + " " - "including element " + str(single_value) + - " of type" + " " + str(type(single_value))) + raise ValueError( + "The " + + name + + "'s type must be a list or tuple of " + + str(n) + + " " + + str(dtype) + + " . Received: " + + str(value) + + " " + "including element " + + str(single_value) + + " of type" + + " " + + str(type(single_value)) + ) return value_list @@ -82,7 +108,7 @@ def is_sequence(seq): """ if isinstance(seq, dict): return True - return (isinstance(seq, Sequence) and not isinstance(seq, str)) + return isinstance(seq, Sequence) and not isinstance(seq, str) def _hash_with_id(*args): @@ -91,7 +117,7 @@ def _hash_with_id(*args): """ assert len(args) > 0 info = tuple([id(v) for v in args]) - return hash(info) & 0xfffffff + return hash(info) & 0xFFFFFFF def _sorted(dict_): @@ -136,9 +162,9 @@ def to_sequence(nest): def flatten(nest): """ - :alias_main: paddle.flatten - :alias: paddle.flatten,paddle.tensor.flatten,paddle.tensor.manipulation.flatten - :old_api: paddle.fluid.layers.flatten + :alias_main: paddle.flatten + :alias: paddle.flatten,paddle.tensor.flatten,paddle.tensor.manipulation.flatten + :old_api: paddle.fluid.layers.flatten Traverse all entries in the nested structure and put them into an list. """ @@ -160,9 +186,12 @@ def _sequence_like(instance, args): # corresponding `OrderedDict` to pack it back). result = dict(zip(_sorted(instance), args)) return type(instance)((key, result[key]) for key in instance.keys()) - elif (isinstance(instance, tuple) and hasattr(instance, "_fields") - and isinstance(instance._fields, Sequence) - and all(isinstance(f, str) for f in instance._fields)): + elif ( + isinstance(instance, tuple) + and hasattr(instance, "_fields") + and isinstance(instance._fields, Sequence) + and all(isinstance(f, str) for f in instance._fields) + ): # This is a namedtuple return type(instance)(*args) else: @@ -195,15 +224,22 @@ def pack_sequence_as(structure, flat_sequence): if not is_sequence(structure): if len(flat_sequence) != 1: raise ValueError( - "Structure is a scalar but len(flat_sequence) == %d > 1" % - len(flat_sequence)) + "Structure is a scalar but len(flat_sequence) == %d > 1" + % len(flat_sequence) + ) return flat_sequence[0] flat_structure = flatten(structure) if len(flat_structure) != len(flat_sequence): raise ValueError( "Could not pack sequence. Structure had %d elements, but flat_sequence " - "had %d elements. Structure: %s, flat_sequence: %s." % - (len(flat_structure), len(flat_sequence), structure, flat_sequence)) + "had %d elements. Structure: %s, flat_sequence: %s." + % ( + len(flat_structure), + len(flat_sequence), + structure, + flat_sequence, + ) + ) _, packed = _packed_nest_with_indices(structure, flat_sequence, 0) return _sequence_like(structure, packed) @@ -243,7 +279,8 @@ def _recursive_assert_same_structure(nest1, nest2, check_types): if is_sequence_nest1 != is_sequence(nest2): raise ValueError( "The two structures don't have the same nested structure.\n\n" - "First structure: %s\n\nSecond structure: %s." % (nest1, nest2)) + "First structure: %s\n\nSecond structure: %s." % (nest1, nest2) + ) if not is_sequence_nest1: return # finished checking if check_types: @@ -252,16 +289,19 @@ def _recursive_assert_same_structure(nest1, nest2, check_types): if type_nest1 != type_nest2: raise TypeError( "The two structures don't have the same sequence type. First " - "structure has type %s, while second structure has type %s." % - (type_nest1, type_nest2)) + "structure has type %s, while second structure has type %s." + % (type_nest1, type_nest2) + ) if isinstance(nest1, dict): keys1 = set(nest1.keys()) keys2 = set(nest2.keys()) if keys1 != keys2: raise ValueError( "The two dictionaries don't have the same set of keys. First " - "structure has keys {}, while second structure has keys {}." - .format(keys1, keys2)) + "structure has keys {}, while second structure has keys {}.".format( + keys1, keys2 + ) + ) nest1_as_sequence = [n for n in _yield_value(nest1)] nest2_as_sequence = [n for n in _yield_value(nest2)] for n1, n2 in zip(nest1_as_sequence, nest2_as_sequence): @@ -269,16 +309,16 @@ def _recursive_assert_same_structure(nest1, nest2, check_types): def padding_to_same_structure(nest1, nest2, obj=None): - def _padding_to_same_structure_single(value, obj): - def change_none_to_obj(x): - if x is None: return obj + if x is None: + return obj return x if is_sequence(value): value = pack_sequence_as( - value, [change_none_to_obj(item) for item in flatten(value)]) + value, [change_none_to_obj(item) for item in flatten(value)] + ) else: value = change_none_to_obj(value) return value @@ -295,10 +335,12 @@ def assert_same_structure(nest1, nest2, check_types=True): len_nest1 = len(flatten(nest1)) if is_sequence(nest1) else 1 len_nest2 = len(flatten(nest2)) if is_sequence(nest2) else 1 if len_nest1 != len_nest2: - raise ValueError("The two structures don't have the same number of " - "elements.\n\nFirst structure (%i elements): %s\n\n" - "Second structure (%i elements): %s" % - (len_nest1, nest1, len_nest2, nest2)) + raise ValueError( + "The two structures don't have the same number of " + "elements.\n\nFirst structure (%i elements): %s\n\n" + "Second structure (%i elements): %s" + % (len_nest1, nest1, len_nest2, nest2) + ) _recursive_assert_same_structure(nest1, nest2, check_types) @@ -343,9 +385,12 @@ def get_shape_tensor_inputs(inputs, attrs, shape, op_type): if isinstance(dim, Variable): dim.stop_gradient = True check_dtype( - dim.dtype, 'shape[' + str(idx) + ']', ['int32', 'int64'], + dim.dtype, + 'shape[' + str(idx) + ']', + ['int32', 'int64'], op_type, - '(When type of shape in' + op_type + 'is list or tuple.)') + '(When type of shape in' + op_type + 'is list or tuple.)', + ) if convert_dtype(dim.dtype) == 'int64': dim = cast(x=dim, dtype='int32') shape_tensor_list.append(dim) @@ -356,9 +401,14 @@ def get_shape_tensor_inputs(inputs, attrs, shape, op_type): if isinstance(shape, Variable): shape.stop_gradient = True - check_dtype(shape.dtype, 'shape', ['int32', 'int64'], 'fill_constant', - '(When type of shape in' + op_type + ' is Variable.)') - if (convert_dtype(shape.dtype) == 'int64'): + check_dtype( + shape.dtype, + 'shape', + ['int32', 'int64'], + 'fill_constant', + '(When type of shape in' + op_type + ' is Variable.)', + ) + if convert_dtype(shape.dtype) == 'int64': shape = cast(shape, 'int32') inputs["ShapeTensor"] = shape elif isinstance(shape, (list, tuple)): @@ -374,6 +424,7 @@ def _convert_to_tensor_list(old_list, dtype="int32"): Converts all elements of a list to Variable. """ from .tensor import fill_constant + new_list_tensor = [] for ele in old_list: @@ -393,8 +444,11 @@ def convert_shape_to_list(shape): """ if isinstance(shape, (list, tuple)): shape = list( - map(lambda x: x.numpy().flat[0] - if isinstance(x, Variable) else x, shape)) + map( + lambda x: x.numpy().flat[0] if isinstance(x, Variable) else x, + shape, + ) + ) else: shape = shape.numpy().astype(int).tolist() return shape @@ -462,7 +516,8 @@ def try_get_constant_shape_from_tensor(shape_tensor): generate_op = shape_tensor.op if generate_op.type == 'shape': var = shape_tensor.block.vars[ - generate_op.input_arg_names[0]] + generate_op.input_arg_names[0] + ] return var.shape except: return None @@ -476,9 +531,11 @@ def get_inputs_outputs_in_block(block): created in this block. """ assert isinstance( - block, - Block), "input non-Block argument for get_inputs_outputs_in_block." - assert block.parent_idx != -1, "input block should be a sub-block, not main block." + block, Block + ), "input non-Block argument for get_inputs_outputs_in_block." + assert ( + block.parent_idx != -1 + ), "input block should be a sub-block, not main block." # Find input/output var names of all ops in block inner_inputs = set() diff --git a/python/paddle/fluid/lazy_init.py b/python/paddle/fluid/lazy_init.py index 8fc175ae3ef..d2118259d03 100644 --- a/python/paddle/fluid/lazy_init.py +++ b/python/paddle/fluid/lazy_init.py @@ -36,7 +36,8 @@ class LazyInitHelper(object): """ if self._state: return - assert framework._non_static_mode( + assert ( + framework._non_static_mode() ), "LazyInit.enable() is only available in dygraph mode." self._state = True @@ -56,7 +57,8 @@ class LazyInitHelper(object): dygraph mode into static mode. """ self.enable() - if self._in_guard: return + if self._in_guard: + return self._tracer = framework._dygraph_tracer_ framework._dygraph_tracer_ = None self._in_guard = True @@ -66,7 +68,8 @@ class LazyInitHelper(object): Exit from lazy mode and recover _dygraph_tracer_. """ self.disable() - if not self._in_guard: return + if not self._in_guard: + return assert self._tracer is not None framework._dygraph_tracer_ = self._tracer self._tracer = None diff --git a/python/paddle/fluid/lod_tensor.py b/python/paddle/fluid/lod_tensor.py index 1b1e74f1fec..d099536fcd7 100644 --- a/python/paddle/fluid/lod_tensor.py +++ b/python/paddle/fluid/lod_tensor.py @@ -72,10 +72,12 @@ def create_lod_tensor(data, recursive_seq_lens, place): elif isinstance(data, list): # dtype and shape are not important here, # we only want to reuse code of DataToLoDTensorConverter - converter = DataToLoDTensorConverter(place=place, - lod_level=len(recursive_seq_lens), - shape=[], - dtype=core.VarDesc.VarType.FP32) + converter = DataToLoDTensorConverter( + place=place, + lod_level=len(recursive_seq_lens), + shape=[], + dtype=core.VarDesc.VarType.FP32, + ) new_recursive_seq_lens = [] for seq in data: @@ -91,7 +93,7 @@ def create_lod_tensor(data, recursive_seq_lens, place): # FIXME(zjl): the original logic of create_lod_tensor would append # 1 to the shape. Maybe it is not a right way? Currently, we only # follow the previous logic - arr = arr.reshape(arr.shape + (1, )) + arr = arr.reshape(arr.shape + (1,)) tensor = core.LoDTensor() tensor.set(arr, place) tensor.set_recursive_sequence_lengths(recursive_seq_lens) @@ -100,18 +102,21 @@ def create_lod_tensor(data, recursive_seq_lens, place): tensor = core.LoDTensor() tensor.set(data, place) tensor.set_recursive_sequence_lengths(recursive_seq_lens) - assert tensor.has_valid_recursive_sequence_lengths( + assert ( + tensor.has_valid_recursive_sequence_lengths() ), "the provided lod info is invalid" return tensor else: raise TypeError( - "data should be either a LoDTensor, a Numpy array or a list") + "data should be either a LoDTensor, a Numpy array or a list" + ) -def create_random_int_lodtensor(recursive_seq_lens, base_shape, place, low, - high): +def create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low, high +): """ - :api_attr: Static Graph + :api_attr: Static Graph Create a LoDTensor containing random integers. diff --git a/python/paddle/fluid/memory_analysis.py b/python/paddle/fluid/memory_analysis.py index de9a260ada8..9d33927b98b 100644 --- a/python/paddle/fluid/memory_analysis.py +++ b/python/paddle/fluid/memory_analysis.py @@ -19,8 +19,9 @@ import numpy as np def get_var_and_memory_size(block, var_name, batch_size=None): var = block._find_var_recursive(var_name) assert var is not None, "Variable {} cannot be found".format(var_name) - assert var.type == core.VarDesc.VarType.LOD_TENSOR, "Variable {} is not Tensor".format( - var_name) + assert ( + var.type == core.VarDesc.VarType.LOD_TENSOR + ), "Variable {} is not Tensor".format(var_name) shape = list(var.shape) if not shape: return var, 0 @@ -31,8 +32,9 @@ def get_var_and_memory_size(block, var_name, batch_size=None): assert not has_none shape[i] = batch_size has_none = True - assert all([s >= 0 - for s in shape]), "shape {} is not deterministic".format(shape) + assert all([s >= 0 for s in shape]), "shape {} is not deterministic".format( + shape + ) mem_size = int(np.prod(shape)) * core.size_of_dtype(var.dtype) return var, mem_size @@ -46,7 +48,9 @@ def pre_allocate_memory(size, place): # NOTE: does not consider inplace yet. def get_max_memory_info(program, batch_size=None): - assert program.num_blocks == 1, "only support to analysis program with only one block" + assert ( + program.num_blocks == 1 + ), "only support to analysis program with only one block" cur_tmp_mem = 0 max_tmp_mem = 0 max_persistable_mem = 0 diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index f14692f79eb..8f15508e699 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -43,8 +43,12 @@ def _is_numpy_(var): def _is_number_(var): - return isinstance(var, int) or isinstance(var, np.int64) or isinstance( - var, float) or (isinstance(var, np.ndarray) and var.shape == (1, )) + return ( + isinstance(var, int) + or isinstance(var, np.int64) + or isinstance(var, float) + or (isinstance(var, np.ndarray) and var.shape == (1,)) + ) def _is_number_or_matrix_(var): @@ -116,13 +120,14 @@ class MetricBase(object): """ states = { attr: value - for attr, value in self.__dict__.items() if not attr.startswith("_") + for attr, value in self.__dict__.items() + if not attr.startswith("_") } for attr, value in states.items(): if isinstance(value, int): setattr(self, attr, 0) elif isinstance(value, float): - setattr(self, attr, .0) + setattr(self, attr, 0.0) elif isinstance(value, (np.ndarray, np.generic)): setattr(self, attr, np.zeros_like(value)) else: @@ -144,7 +149,8 @@ class MetricBase(object): """ states = { attr: value - for attr, value in self.__dict__.items() if not attr.startswith("_") + for attr, value in self.__dict__.items() + if not attr.startswith("_") } config = {} config.update({"name": self._name, "states": copy.deepcopy(states)}) @@ -170,7 +176,8 @@ class MetricBase(object): """ raise NotImplementedError( - "Should not use it directly, please extend it.") + "Should not use it directly, please extend it." + ) def eval(self): """ @@ -187,7 +194,8 @@ class MetricBase(object): float|list(float)|numpy.array: the metrics via Python. """ raise NotImplementedError( - "Should not use it directly, please extend it.") + "Should not use it directly, please extend it." + ) class CompositeMetric(MetricBase): @@ -340,7 +348,7 @@ class Precision(MetricBase): float: Results of the calculated Precision. Scalar output with float dtype. """ ap = self.tp + self.fp - return float(self.tp) / ap if ap != 0 else .0 + return float(self.tp) / ap if ap != 0 else 0.0 class Recall(MetricBase): @@ -423,7 +431,7 @@ class Recall(MetricBase): float: results of the calculated Recall. Scalar output with float dtype. """ recall = self.tp + self.fn - return float(self.tp) / recall if recall != 0 else .0 + return float(self.tp) / recall if recall != 0 else 0.0 class Accuracy(MetricBase): @@ -465,8 +473,8 @@ class Accuracy(MetricBase): def __init__(self, name=None): super(Accuracy, self).__init__(name) - self.value = .0 - self.weight = .0 + self.value = 0.0 + self.weight = 0.0 def update(self, value, weight): r""" @@ -482,7 +490,8 @@ class Accuracy(MetricBase): """ if not _is_number_or_matrix_(value): raise ValueError( - "The 'value' must be a number(int, float) or a numpy ndarray.") + "The 'value' must be a number(int, float) or a numpy ndarray." + ) if not _is_number_(weight): raise ValueError("The 'weight' must be a number(int, float).") if _is_number_(weight) and weight < 0: @@ -499,8 +508,10 @@ class Accuracy(MetricBase): """ if self.weight == 0: - raise ValueError("There is no data in Accuracy Metrics. \ - Please check layers.accuracy output has added to Accuracy.") + raise ValueError( + "There is no data in Accuracy Metrics. \ + Please check layers.accuracy output has added to Accuracy." + ) return self.value / self.weight @@ -592,13 +603,21 @@ class ChunkEvaluator(MetricBase): float: mean precision, recall and f1 score. """ - precision = float( - self.num_correct_chunks - ) / self.num_infer_chunks if self.num_infer_chunks else 0 - recall = float(self.num_correct_chunks - ) / self.num_label_chunks if self.num_label_chunks else 0 - f1_score = float(2 * precision * recall) / ( - precision + recall) if self.num_correct_chunks else 0 + precision = ( + float(self.num_correct_chunks) / self.num_infer_chunks + if self.num_infer_chunks + else 0 + ) + recall = ( + float(self.num_correct_chunks) / self.num_label_chunks + if self.num_label_chunks + else 0 + ) + f1_score = ( + float(2 * precision * recall) / (precision + recall) + if self.num_correct_chunks + else 0 + ) return precision, recall, f1_score @@ -653,7 +672,7 @@ class EditDistance(MetricBase): def __init__(self, name): super(EditDistance, self).__init__(name) - self.total_distance = .0 + self.total_distance = 0.0 self.seq_num = 0 self.instance_error = 0 @@ -789,11 +808,14 @@ class Auc(MetricBase): tot_neg_prev = tot_neg tot_pos += self._stat_pos[idx] tot_neg += self._stat_neg[idx] - auc += self.trapezoid_area(tot_neg, tot_neg_prev, tot_pos, - tot_pos_prev) + auc += self.trapezoid_area( + tot_neg, tot_neg_prev, tot_pos, tot_pos_prev + ) idx -= 1 - return auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + return ( + auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + ) class DetectionMAP(object): @@ -879,16 +901,18 @@ class DetectionMAP(object): """ - def __init__(self, - input, - gt_label, - gt_box, - gt_difficult=None, - class_num=None, - background_label=0, - overlap_threshold=0.5, - evaluate_difficult=True, - ap_version='integral'): + def __init__( + self, + input, + gt_label, + gt_box, + gt_difficult=None, + class_num=None, + background_label=0, + overlap_threshold=0.5, + evaluate_difficult=True, + ap_version='integral', + ): self.helper = LayerHelper('map_eval') gt_label = layers.cast(x=gt_label, dtype=gt_box.dtype) @@ -899,30 +923,36 @@ class DetectionMAP(object): label = layers.concat([gt_label, gt_box], axis=1) # calculate mean average precision (mAP) of current mini-batch - map = detection.detection_map(input, - label, - class_num, - background_label, - overlap_threshold=overlap_threshold, - evaluate_difficult=evaluate_difficult, - ap_version=ap_version) + map = detection.detection_map( + input, + label, + class_num, + background_label, + overlap_threshold=overlap_threshold, + evaluate_difficult=evaluate_difficult, + ap_version=ap_version, + ) states = [] states.append( - self._create_state(dtype='int32', - shape=None, - suffix='accum_pos_count')) + self._create_state( + dtype='int32', shape=None, suffix='accum_pos_count' + ) + ) states.append( - self._create_state(dtype='float32', - shape=None, - suffix='accum_true_pos')) + self._create_state( + dtype='float32', shape=None, suffix='accum_true_pos' + ) + ) states.append( - self._create_state(dtype='float32', - shape=None, - suffix='accum_false_pos')) + self._create_state( + dtype='float32', shape=None, suffix='accum_false_pos' + ) + ) var = self._create_state(dtype='int32', shape=[1], suffix='has_state') - self.helper.set_variable_initializer(var, - initializer=Constant(value=int(0))) + self.helper.set_variable_initializer( + var, initializer=Constant(value=int(0)) + ) self.has_state = var # calculate accumulative mAP @@ -936,12 +966,15 @@ class DetectionMAP(object): has_state=self.has_state, input_states=states, out_states=states, - ap_version=ap_version) + ap_version=ap_version, + ) - layers.fill_constant(shape=self.has_state.shape, - value=1, - dtype=self.has_state.dtype, - out=self.has_state) + layers.fill_constant( + shape=self.has_state.shape, + value=1, + dtype=self.has_state.dtype, + out=self.has_state, + ) self.cur_map = map self.accum_map = accum_map @@ -955,11 +988,12 @@ class DetectionMAP(object): shape(tuple|list): the shape of state Returns: State variable """ - state = self.helper.create_variable(name="_".join( - [unique_name.generate(self.helper.name), suffix]), - persistable=True, - dtype=dtype, - shape=shape) + state = self.helper.create_variable( + name="_".join([unique_name.generate(self.helper.name), suffix]), + persistable=True, + dtype=dtype, + shape=shape, + ) return state def get_map_var(self): @@ -981,19 +1015,20 @@ class DetectionMAP(object): def _clone_var_(block, var): assert isinstance(var, Variable) - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=var.persistable) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=var.persistable, + ) if reset_program is None: reset_program = Program() with program_guard(main_program=reset_program): var = _clone_var_(reset_program.current_block(), self.has_state) - layers.fill_constant(shape=var.shape, - value=0, - dtype=var.dtype, - out=var) + layers.fill_constant( + shape=var.shape, value=0, dtype=var.dtype, out=var + ) executor.run(reset_program) diff --git a/python/paddle/fluid/multiprocess_utils.py b/python/paddle/fluid/multiprocess_utils.py index 73bba0069cd..deca95457e6 100644 --- a/python/paddle/fluid/multiprocess_utils.py +++ b/python/paddle/fluid/multiprocess_utils.py @@ -23,7 +23,7 @@ import queue # multi-process worker check indices queue interval, avoid # hanging in subprocess data loading -MP_STATUS_CHECK_INTERVAL = 5. +MP_STATUS_CHECK_INTERVAL = 5.0 # NOTE: [ mmap files clear ] If there is still data in the multiprocess queue when the main process finishes reading, # the data in the queue needs to be popped. Then the LoDTensor read by the main process @@ -56,7 +56,7 @@ def _cleanup_mmap(): # NOTE used for register a function to be executed at interpreter exit. -class CleanupFuncRegistrar(): +class CleanupFuncRegistrar: # Record the cleanup functions that have been executed _executed_func_set = set() # Record the cleanup functions that have been registered @@ -64,7 +64,6 @@ class CleanupFuncRegistrar(): @classmethod def register(cls, function, signals=[]): - def _func_exectuor(): if function not in cls._executed_func_set: try: @@ -93,8 +92,10 @@ class CleanupFuncRegistrar(): for sig in signals: orig_handler = signal.signal(sig, _signal_handler) if orig_handler not in (signal.SIG_DFL, signal.SIG_IGN): - if (sig == signal.SIGINT - and orig_handler is signal.default_int_handler): + if ( + sig == signal.SIGINT + and orig_handler is signal.default_int_handler + ): continue if orig_handler not in cls._registered_func_set: atexit.register(orig_handler) diff --git a/python/paddle/fluid/net_drawer.py b/python/paddle/fluid/net_drawer.py index b6a4c28b499..cf5cbf60ea3 100644 --- a/python/paddle/fluid/net_drawer.py +++ b/python/paddle/fluid/net_drawer.py @@ -30,16 +30,19 @@ except ImportError: 'Cannot import graphviz, which is required for drawing a network. This ' 'can usually be installed in python with "pip install graphviz". Also, ' 'pydot requires graphviz to convert dot files to pdf: in ubuntu, this ' - 'can usually be installed with "sudo apt-get install graphviz".') - print('net_drawer will not run correctly. Please install the correct ' - 'dependencies.') + 'can usually be installed with "sudo apt-get install graphviz".' + ) + print( + 'net_drawer will not run correctly. Please install the correct ' + 'dependencies.' + ) exit(0) OP_STYLE = { 'shape': 'oval', 'color': '#0F9D58', 'style': 'filled', - 'fontcolor': '#FFFFFF' + 'fontcolor': '#FFFFFF', } VAR_STYLE = {} @@ -52,7 +55,6 @@ GRAPH_ID = 0 def unique_id(): - def generator(): GRAPH_ID += 1 return GRAPH_ID @@ -85,7 +87,8 @@ def parse_graph(program, graph, var_dict, **kwargs): temp_id = 0 proto = framework_pb2.ProgramDesc.FromString( - program.desc.serialize_to_string()) + program.desc.serialize_to_string() + ) for block in proto.blocks: for op in block.ops: op.type = op.type + "_" + str(temp_id) @@ -113,12 +116,14 @@ def draw_graph(startup_program, main_program, **kwargs): filename = kwargs.get("filename") if filename == None: filename = str(graph_id) + ".gv" - g = Graph(name=str(graph_id), - filename=filename, - graph_attr=GRAPH_STYLE, - node_attr=OP_STYLE, - edge_attr=VAR_STYLE, - **kwargs) + g = Graph( + name=str(graph_id), + filename=filename, + graph_attr=GRAPH_STYLE, + node_attr=OP_STYLE, + edge_attr=VAR_STYLE, + **kwargs + ) var_dict = {} parse_graph(startup_program, g, var_dict) diff --git a/python/paddle/fluid/nets.py b/python/paddle/fluid/nets.py index a61487ee71d..bfff9c12e92 100644 --- a/python/paddle/fluid/nets.py +++ b/python/paddle/fluid/nets.py @@ -25,24 +25,26 @@ __all__ = [ ] -def simple_img_conv_pool(input, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - param_attr=None, - bias_attr=None, - act=None, - use_cudnn=True): +def simple_img_conv_pool( + input, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + param_attr=None, + bias_attr=None, + act=None, + use_cudnn=True, +): r""" - :api_attr: Static Graph + :api_attr: Static Graph The simple_img_conv_pool api is composed of :ref:`api_fluid_layers_conv2d` and :ref:`api_fluid_layers_pool2d` . @@ -115,42 +117,48 @@ def simple_img_conv_pool(input, pool_stride=2, act="relu") """ - conv_out = layers.conv2d(input=input, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=param_attr, - bias_attr=bias_attr, - act=act, - use_cudnn=use_cudnn) - - pool_out = layers.pool2d(input=conv_out, - pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + conv_out = layers.conv2d( + input=input, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=param_attr, + bias_attr=bias_attr, + act=act, + use_cudnn=use_cudnn, + ) + + pool_out = layers.pool2d( + input=conv_out, + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) return pool_out -def img_conv_group(input, - conv_num_filter, - pool_size, - conv_padding=1, - conv_filter_size=3, - conv_act=None, - param_attr=None, - conv_with_batchnorm=False, - conv_batchnorm_drop_rate=0.0, - pool_stride=1, - pool_type="max", - use_cudnn=True): +def img_conv_group( + input, + conv_num_filter, + pool_size, + conv_padding=1, + conv_filter_size=3, + conv_act=None, + param_attr=None, + conv_with_batchnorm=False, + conv_batchnorm_drop_rate=0.0, + pool_stride=1, + pool_type="max", + use_cudnn=True, +): """ - :api_attr: Static Graph + :api_attr: Static Graph The Image Convolution Group is composed of Convolution2d, BatchNorm, DropOut, and Pool2D. According to the input arguments, img_conv_group will do serials of @@ -210,8 +218,9 @@ def img_conv_group(input, pool_stride=2) """ tmp = input - assert isinstance(conv_num_filter, list) or \ - isinstance(conv_num_filter, tuple) + assert isinstance(conv_num_filter, list) or isinstance( + conv_num_filter, tuple + ) def __extend_list__(obj): if not hasattr(obj, '__len__'): @@ -231,13 +240,15 @@ def img_conv_group(input, if conv_with_batchnorm[i]: local_conv_act = None - tmp = layers.conv2d(input=tmp, - num_filters=conv_num_filter[i], - filter_size=conv_filter_size[i], - padding=conv_padding[i], - param_attr=param_attr[i], - act=local_conv_act, - use_cudnn=use_cudnn) + tmp = layers.conv2d( + input=tmp, + num_filters=conv_num_filter[i], + filter_size=conv_filter_size[i], + padding=conv_padding[i], + param_attr=param_attr[i], + act=local_conv_act, + use_cudnn=use_cudnn, + ) if conv_with_batchnorm[i]: tmp = layers.batch_norm(input=tmp, act=conv_act) @@ -245,23 +256,27 @@ def img_conv_group(input, if abs(drop_rate) > 1e-5: tmp = layers.dropout(x=tmp, dropout_prob=drop_rate) - pool_out = layers.pool2d(input=tmp, - pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - use_cudnn=use_cudnn) + pool_out = layers.pool2d( + input=tmp, + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + use_cudnn=use_cudnn, + ) return pool_out -def sequence_conv_pool(input, - num_filters, - filter_size, - param_attr=None, - act="sigmoid", - pool_type="max", - bias_attr=None): +def sequence_conv_pool( + input, + num_filters, + filter_size, + param_attr=None, + act="sigmoid", + pool_type="max", + bias_attr=None, +): """ - :api_attr: Static Graph + :api_attr: Static Graph **This api takes input as an LoDTensor. If input is a Tensor, please use** :ref:`api_fluid_nets_simple_img_conv_pool` **instead** @@ -315,12 +330,14 @@ def sequence_conv_pool(input, """ check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'input') - conv_out = layers.sequence_conv(input=input, - num_filters=num_filters, - filter_size=filter_size, - param_attr=param_attr, - bias_attr=bias_attr, - act=act) + conv_out = layers.sequence_conv( + input=input, + num_filters=num_filters, + filter_size=filter_size, + param_attr=param_attr, + bias_attr=bias_attr, + act=act, + ) pool_out = layers.sequence_pool(input=conv_out, pool_type=pool_type) return pool_out @@ -329,7 +346,7 @@ def sequence_conv_pool(input, @deprecated(since="2.0.0", update_to="paddle.nn.functional.glu") def glu(input, dim=-1): r""" - :api_attr: Static Graph + :api_attr: Static Graph The Gated Linear Units(GLU) composed by :ref:`api_fluid_layers_split` , :ref:`api_fluid_layers_sigmoid` and :ref:`api_fluid_layers_elementwise_mul` . @@ -366,19 +383,18 @@ def glu(input, dim=-1): # shape of output: [-1, 3, 3, 9] output = fluid.nets.glu(input=data, dim=1) """ - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - "glu") + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], "glu" + ) a, b = layers.split(input, num_or_sections=2, dim=dim) act_b = layers.sigmoid(x=b) out = layers.elementwise_mul(x=a, y=act_b) return out -def scaled_dot_product_attention(queries, - keys, - values, - num_heads=1, - dropout_rate=0.): +def scaled_dot_product_attention( + queries, keys, values, num_heads=1, dropout_rate=0.0 +): r""" :api_attr: Static Graph @@ -449,46 +465,63 @@ def scaled_dot_product_attention(queries, contexts = fluid.nets.scaled_dot_product_attention(queries, keys, values) contexts.shape # [3, 5, 10] """ - check_variable_and_dtype(queries, 'queries', ['float32', 'float64'], - "scaled_dot_product_attention") - check_variable_and_dtype(keys, 'keys', ['float32', 'float64'], - "scaled_dot_product_attention") - check_variable_and_dtype(values, 'values', ['float32', 'float64'], - "scaled_dot_product_attention") + check_variable_and_dtype( + queries, + 'queries', + ['float32', 'float64'], + "scaled_dot_product_attention", + ) + check_variable_and_dtype( + keys, 'keys', ['float32', 'float64'], "scaled_dot_product_attention" + ) + check_variable_and_dtype( + values, 'values', ['float32', 'float64'], "scaled_dot_product_attention" + ) if not (queries.dtype == keys.dtype == values.dtype): raise TypeError( "The dtype of keys, values and queries should be the same." "But received queries.dtype = %s, " - " keys.dtype = %s, values.dtype) = %s." % - (convert_dtype(queries.dtype), convert_dtype( - keys.dtype), convert_dtype(values.dtype))) + " keys.dtype = %s, values.dtype) = %s." + % ( + convert_dtype(queries.dtype), + convert_dtype(keys.dtype), + convert_dtype(values.dtype), + ) + ) if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): raise ValueError( "Inputs queries, keys and values should all be 3-D tensors." "But received len(queries.shape) = %d, " - "len(keys.shape) = %d, len(values.shape) = %d." % - (len(queries.shape), len(keys.shape), len(values.shape))) + "len(keys.shape) = %d, len(values.shape) = %d." + % (len(queries.shape), len(keys.shape), len(values.shape)) + ) if queries.shape[-1] != keys.shape[-1]: raise ValueError( "The hidden size of queries and keys should be the same." "But received queries' hidden size = %d and keys' hidden size = %d." - % (queries.shape[-1], keys.shape[-1])) + % (queries.shape[-1], keys.shape[-1]) + ) if keys.shape[-2] != values.shape[-2]: raise ValueError( "The max sequence length in value batch and in key batch " "should be the same. But received max sequence length in value batch " - "= %d, in key batch = %d." % (values.shape[-2], keys.shape[-2])) + "= %d, in key batch = %d." % (values.shape[-2], keys.shape[-2]) + ) if keys.shape[-1] % num_heads != 0: - raise ValueError("The hidden size of keys (%d) must be divisible " - "by the number of attention heads (%d)." % - (keys.shape[-1], num_heads)) + raise ValueError( + "The hidden size of keys (%d) must be divisible " + "by the number of attention heads (%d)." + % (keys.shape[-1], num_heads) + ) if values.shape[-1] % num_heads != 0: - raise ValueError("The hidden size of values (%d) must be divisible " - "by the number of attention heads (%d)." % - (values.shape[-1], num_heads)) + raise ValueError( + "The hidden size of values (%d) must be divisible " + "by the number of attention heads (%d)." + % (values.shape[-1], num_heads) + ) def __compute_qkv(queries, keys, values, num_heads): """ @@ -535,9 +568,10 @@ def scaled_dot_product_attention(queries, # reshape the 3-D input: [batch_size, max_sequence_length, hidden_dim] # into a 4-D output: # [batch_size, max_sequence_length, num_heads, hidden_size_per_head]. - reshaped = layers.reshape(x=x, - shape=list(x.shape[:-1]) + - [num_heads, hidden_size // num_heads]) + reshaped = layers.reshape( + x=x, + shape=list(x.shape[:-1]) + [num_heads, hidden_size // num_heads], + ) # permute the dimensions into: # [batch_size, num_heads, max_sequence_len, hidden_size_per_head] @@ -557,17 +591,25 @@ def scaled_dot_product_attention(queries, [bs, max_sequence_length, num_heads * hidden_dim]. """ - if len(x.shape) == 3: return x + if len(x.shape) == 3: + return x if len(x.shape) != 4: raise ValueError("Input(x) should be a 4-D Tensor.") trans_x = layers.transpose(x, perm=[0, 2, 1, 3]) - return layers.reshape(x=trans_x, - shape=list( - map(int, [ - trans_x.shape[0], trans_x.shape[1], - trans_x.shape[2] * trans_x.shape[3] - ]))) + return layers.reshape( + x=trans_x, + shape=list( + map( + int, + [ + trans_x.shape[0], + trans_x.shape[1], + trans_x.shape[2] * trans_x.shape[3], + ], + ) + ), + ) q, k, v = __compute_qkv(queries, keys, values, num_heads) @@ -579,13 +621,15 @@ def scaled_dot_product_attention(queries, scaled_q = layers.scale(x=q, scale=key_dim_per_head**-0.5) product = layers.matmul(x=scaled_q, y=k, transpose_y=True) - weights = layers.reshape(x=layers.reshape(x=product, - shape=[-1, product.shape[-1]], - act="softmax"), - shape=product.shape) + weights = layers.reshape( + x=layers.reshape( + x=product, shape=[-1, product.shape[-1]], act="softmax" + ), + shape=product.shape, + ) if dropout_rate: - weights = layers.dropout(weights, - dropout_prob=dropout_rate, - is_test=False) + weights = layers.dropout( + weights, dropout_prob=dropout_rate, is_test=False + ) ctx_multiheads = layers.matmul(weights, v) return __combine_heads(ctx_multiheads) diff --git a/python/paddle/fluid/op.py b/python/paddle/fluid/op.py index bdcdad1218e..c8b118127cc 100644 --- a/python/paddle/fluid/op.py +++ b/python/paddle/fluid/op.py @@ -47,7 +47,8 @@ class OpDescCreationMethod(object): def __init__(self, op_proto): if not isinstance(op_proto, framework_pb2.OpProto): raise TypeError( - "Type of op_proto should be OpProto in PaddlePaddle.") + "Type of op_proto should be OpProto in PaddlePaddle." + ) self.__op_proto__ = op_proto self.__extra_attrs__ = core.get_op_extra_attrs(op_proto.type) @@ -67,8 +68,9 @@ class OpDescCreationMethod(object): if not input_parameter.duplicable and len(input_arguments) > 1: raise ValueError( - "Input %s expects only one input, but %d are given." % - (input_parameter.name, len(input_arguments))) + "Input %s expects only one input, but %d are given." + % (input_parameter.name, len(input_arguments)) + ) ipt = op_desc.inputs.add() ipt.parameter = input_parameter.name @@ -81,8 +83,9 @@ class OpDescCreationMethod(object): if not output_parameter.duplicable and len(output_arguments) > 1: raise ValueError( - "Output %s expects only one output, but %d are given." % - (output_parameter.name, len(output_arguments))) + "Output %s expects only one output, but %d are given." + % (output_parameter.name, len(output_arguments)) + ) out = op_desc.outputs.add() out.parameter = output_parameter.name @@ -126,13 +129,14 @@ class OpDescCreationMethod(object): new_attr.float64 = user_defined_attr else: raise NotImplementedError( - "A not supported attribute type: %s." % - (str(attr.type))) + "A not supported attribute type: %s." % (str(attr.type)) + ) for attr_name, defalut_val in self.__extra_attrs__.items(): user_defined_attr = kwargs.get(attr_name, None) if user_defined_attr is not None: attr_type = int( - core.get_attrtibute_type(op_desc.type, attr_name)) + core.get_attrtibute_type(op_desc.type, attr_name) + ) new_attr = op_desc.attrs.add() new_attr.name = attr_name new_attr.type = attr_type @@ -160,8 +164,8 @@ class OpDescCreationMethod(object): new_attr.longs.extend(user_defined_attr) else: raise NotImplementedError( - "A not supported attribute type: %s." % - (str(attr_type))) + "A not supported attribute type: %s." % (str(attr_type)) + ) return op_desc @@ -178,7 +182,6 @@ class OpDescCreationMethod(object): class OpInfo(object): - def __init__(self, name, method, inputs, outputs, attrs, extra_attrs): self.name = name self.method = method @@ -200,18 +203,17 @@ def create_op_creation_method(op_proto): extra_attrs_map = core.get_op_extra_attrs(op_proto.type) - return OpInfo(method=__impl__, - name=op_proto.type, - inputs=[(var.name, var.duplicable) - for var in op_proto.inputs], - outputs=[(var.name, var.duplicable) - for var in op_proto.outputs], - attrs=[attr.name for attr in op_proto.attrs], - extra_attrs=[item for item in extra_attrs_map.keys()]) + return OpInfo( + method=__impl__, + name=op_proto.type, + inputs=[(var.name, var.duplicable) for var in op_proto.inputs], + outputs=[(var.name, var.duplicable) for var in op_proto.outputs], + attrs=[attr.name for attr in op_proto.attrs], + extra_attrs=[item for item in extra_attrs_map.keys()], + ) class OperatorFactory(object): - def __init__(self): self.op_methods = dict() @@ -224,13 +226,15 @@ class OperatorFactory(object): if len(args) != 0: raise ValueError( "Except the argument \"type\"," - "all of the other arguments should be keyword arguments.") + "all of the other arguments should be keyword arguments." + ) t = kwargs.pop("type") else: if len(args) != 1: raise ValueError( "Except the argument \"type\"," - "all of the other arguments should be keyword arguments.") + "all of the other arguments should be keyword arguments." + ) t = args[0] return self.get_op_info(t).method(**kwargs) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 4471eb9dcbc..ce58c0cfe02 100755 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -19,20 +19,42 @@ from collections import defaultdict import paddle from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table -from paddle.fluid.framework import Program, Variable, Parameter, name_scope, default_main_program, default_startup_program, device_guard +from paddle.fluid.framework import ( + Program, + Variable, + Parameter, + name_scope, + default_main_program, + default_startup_program, + device_guard, +) from . import framework from . import layers from . import unique_name -from .backward import append_backward, _some_in_set_, _append_grad_suffix_, _get_no_grad_set_name -from .clip import GradientClipBase, GradientClipByNorm, error_clip_callback, append_gradient_clip_ops, ClipGradByGlobalNorm +from .backward import ( + append_backward, + _some_in_set_, + _append_grad_suffix_, + _get_no_grad_set_name, +) +from .clip import ( + GradientClipBase, + GradientClipByNorm, + error_clip_callback, + append_gradient_clip_ops, + ClipGradByGlobalNorm, +) from .framework import program_guard from .initializer import Constant from .layer_helper import LayerHelper from .layers import ops from .dygraph import base as imperative_base from .dygraph import no_grad -from .dygraph.learning_rate_scheduler import LearningRateDecay, _LearningRateEpochDecay +from .dygraph.learning_rate_scheduler import ( + LearningRateDecay, + _LearningRateEpochDecay, +) from paddle.fluid import core from paddle.fluid.layers import tensor from functools import reduce @@ -41,16 +63,40 @@ from .wrapped_decorator import signature_safe_contextmanager from .. import compat as cpt import warnings from paddle import _C_ops, _legacy_C_ops -from ..fluid.framework import _in_legacy_dygraph, in_dygraph_mode, _current_expected_place +from ..fluid.framework import ( + _in_legacy_dygraph, + in_dygraph_mode, + _current_expected_place, +) __all__ = [ - 'SGD', 'Momentum', 'Adagrad', 'Adam', 'Adamax', 'Dpsgd', 'DecayedAdagrad', - 'Ftrl', 'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer', - 'AdamOptimizer', 'AdamaxOptimizer', 'DpsgdOptimizer', - 'DecayedAdagradOptimizer', 'RMSPropOptimizer', 'FtrlOptimizer', 'Adadelta', - 'AdadeltaOptimizer', 'ModelAverage', 'LarsMomentum', - 'LarsMomentumOptimizer', 'LambOptimizer', 'ExponentialMovingAverage', - 'PipelineOptimizer', 'LookaheadOptimizer', 'RecomputeOptimizer' + 'SGD', + 'Momentum', + 'Adagrad', + 'Adam', + 'Adamax', + 'Dpsgd', + 'DecayedAdagrad', + 'Ftrl', + 'SGDOptimizer', + 'MomentumOptimizer', + 'AdagradOptimizer', + 'AdamOptimizer', + 'AdamaxOptimizer', + 'DpsgdOptimizer', + 'DecayedAdagradOptimizer', + 'RMSPropOptimizer', + 'FtrlOptimizer', + 'Adadelta', + 'AdadeltaOptimizer', + 'ModelAverage', + 'LarsMomentum', + 'LarsMomentumOptimizer', + 'LambOptimizer', + 'ExponentialMovingAverage', + 'PipelineOptimizer', + 'LookaheadOptimizer', + 'RecomputeOptimizer', ] @@ -63,14 +109,16 @@ class Optimizer(object): """ @imperative_base.no_grad - def __init__(self, - learning_rate, - parameter_list=None, - regularization=None, - grad_clip=None, - flatten_param_grads=False, - align_size=-1, - name=None): + def __init__( + self, + learning_rate, + parameter_list=None, + regularization=None, + grad_clip=None, + flatten_param_grads=False, + align_size=-1, + name=None, + ): """ Args: flatten_param_grads (bool, optional): Whether to flatten all the parameters and grads. @@ -79,15 +127,19 @@ class Optimizer(object): """ # Because of the loop import, so place it in the function body from paddle.optimizer.lr import LRScheduler - self._parameter_list = list( - parameter_list) if parameter_list is not None else None + + self._parameter_list = ( + list(parameter_list) if parameter_list is not None else None + ) self._name = name if framework._non_static_mode(): - if not isinstance(learning_rate, - (float, LearningRateDecay, LRScheduler)): + if not isinstance( + learning_rate, (float, LearningRateDecay, LRScheduler) + ): raise TypeError( "learning rate should be float or LRScheduler, got %s here" - % type(learning_rate)) + % type(learning_rate) + ) if self._parameter_list is None: raise AttributeError( "parameter_list argument given to the Optimizer should not be None in dygraph mode." @@ -98,14 +150,17 @@ class Optimizer(object): logging.info( "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % regularization.__str__()) + % regularization.__str__() + ) break else: - if not isinstance(learning_rate, - (float, framework.Variable, LRScheduler)): + if not isinstance( + learning_rate, (float, framework.Variable, LRScheduler) + ): raise TypeError( "learning rate should be float or LRScheduler, got %s here" - % type(learning_rate)) + % type(learning_rate) + ) if grad_clip is not None: if not isinstance(grad_clip, GradientClipBase): @@ -128,7 +183,8 @@ class Optimizer(object): self._learning_rate_map = dict() if isinstance(self._learning_rate, framework.Variable): self._learning_rate_map[ - framework.default_main_program()] = self._learning_rate + framework.default_main_program() + ] = self._learning_rate # Dictionary of accumulators. Some optimizer subclasses need to # allocate and manage extra variables associated with the parameters # to train. These variables are called accumulators. @@ -169,6 +225,7 @@ class Optimizer(object): ''' from paddle.optimizer.lr import LRScheduler + state_dict = {} for k, v in self._accumulators.items(): for para_name, var_tmp in v.items(): @@ -184,14 +241,13 @@ class Optimizer(object): if not isinstance(self._learning_rate, _LearningRateEpochDecay): var_tmp = None - var_temp = framework._varbase_creator(None, - name='global_step', - dtype='int32') + var_temp = framework._varbase_creator( + None, name='global_step', dtype='int32' + ) - tensor.fill_constant([1], - "int32", - self._learning_rate.step_num, - out=var_temp) + tensor.fill_constant( + [1], "int32", self._learning_rate.step_num, out=var_temp + ) state_dict['global_step'] = var_temp return state_dict @@ -230,6 +286,7 @@ class Optimizer(object): para_state_dict, opti_state_dict = fluid.load_dygraph("paddle_dy") ''' from paddle.optimizer.lr import LRScheduler + if isinstance(self._learning_rate, LRScheduler): self._learning_rate.set_dict(state_dict["LR_Scheduler"]) @@ -237,25 +294,33 @@ class Optimizer(object): self._learning_rate.set_dict(state_dict["LR_Scheduler"]) if not isinstance(self._learning_rate, _LearningRateEpochDecay): - assert 'global_step' in state_dict, \ - 'Global step not in state dict, Dygraph use LearningRateDecay, global_step must in state_dict' + assert ( + 'global_step' in state_dict + ), 'Global step not in state dict, Dygraph use LearningRateDecay, global_step must in state_dict' global_step = state_dict['global_step'] if isinstance(global_step, Variable): step_np = global_step step_np = np.array(step_np.value().get_tensor()) - assert step_np.shape == (1,), \ - "global step shape is (1,), the shape is {}".format( step_np.shape ) + assert step_np.shape == ( + 1, + ), "global step shape is (1,), the shape is {}".format( + step_np.shape + ) self._learning_rate.step_num = int(step_np[0]) elif isinstance(global_step, np.ndarray): - assert global_step.shape == (1,), \ - "global step shape is (1,), the shape is {}".format( global_step.shape ) + assert global_step.shape == ( + 1, + ), "global step shape is (1,), the shape is {}".format( + global_step.shape + ) self._learning_rate.step_num = global_step[0] else: raise RuntimeError( "Type not supprt, value in state dict must be [VarBase, Variable, numpy], the type is ", - type(global_step)) + type(global_step), + ) def _load_state_para(state_dict, param): var = param.value() @@ -269,29 +334,36 @@ class Optimizer(object): elif isinstance(load_para, np.ndarray): load_para_np = load_para else: - raise RuntimeError("State dict type {} not supprt".format( - str(type(load_para)))) + raise RuntimeError( + "State dict type {} not supprt".format(str(type(load_para))) + ) - assert model_np.shape == load_para_np.shape, \ - "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( - param.name, model_np.shape, load_para_np.shape) + assert ( + model_np.shape == load_para_np.shape + ), "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( + param.name, model_np.shape, load_para_np.shape + ) - assert model_np.dtype == load_para_np.dtype, \ - "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( - param.name, model_np.dtype, load_para_np.dtype) + assert ( + model_np.dtype == load_para_np.dtype + ), "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( + param.name, model_np.dtype, load_para_np.dtype + ) tensor.set(load_para_np, framework._current_expected_place()) self._accumulators_holder = state_dict for k, v in self._accumulators.items(): for para_name, var_tmp in v.items(): - assert var_tmp.name in state_dict, \ - "optimizer variable {} not found".format( var_tmp.name ) + assert ( + var_tmp.name in state_dict + ), "optimizer variable {} not found".format(var_tmp.name) _load_state_para(state_dict, var_tmp) for k, v in self._global_accumulators.items(): - assert v.name in state_dict, \ - "optimizer variable {} not found".format( v.name ) + assert ( + v.name in state_dict + ), "optimizer variable {} not found".format(v.name) _load_state_para(state_dict, v) # [aliases] Compatible with old method names @@ -311,6 +383,7 @@ class Optimizer(object): def _create_global_learning_rate(self): from paddle.optimizer.lr import LRScheduler + if isinstance(self._learning_rate, LRScheduler): lr_var = self._global_learning_rate() # only create global lr_var once @@ -322,16 +395,19 @@ class Optimizer(object): shape=[1], persistable=True, stop_gradient=True, - dtype='float32' if self._dtype is None else self._dtype) + dtype='float32' if self._dtype is None else self._dtype, + ) main_prog = framework.default_main_program() main_prog.lr_sheduler = self._learning_rate main_prog.lr_var = lr_var self._learning_rate_map[ - framework.default_main_program()] = lr_var + framework.default_main_program() + ] = lr_var lr_value = float(self._learning_rate()) self.helper.set_variable_initializer( - lr_var, initializer=Constant(value=lr_value)) + lr_var, initializer=Constant(value=lr_value) + ) return if imperative_base.enabled(): @@ -342,17 +418,20 @@ class Optimizer(object): if isinstance(lr, framework.Variable): return else: - self._learning_rate_map[framework.default_main_program( - )] = layers.create_global_var( + self._learning_rate_map[ + framework.default_main_program() + ] = layers.create_global_var( name=unique_name.generate("learning_rate"), shape=[1], value=float(self._learning_rate), dtype='float32' if self._dtype is None else self._dtype, - persistable=True) + persistable=True, + ) # get learning rate Variable from LearningRateDecay elif isinstance(self._learning_rate, LearningRateDecay): self._learning_rate_map[ - framework.default_main_program()] = self._learning_rate() + framework.default_main_program() + ] = self._learning_rate() else: raise TypeError( "optimizer's learning rate must be float or LearningRateDecay" @@ -371,12 +450,14 @@ class Optimizer(object): # create learning rate in the current main program self._learning_rate_map[ - framework.default_main_program()] = layers.create_global_var( - name=unique_name.generate("learning_rate"), - shape=[1], - value=float(self._learning_rate), - dtype='float32' if self._dtype is None else self._dtype, - persistable=True) + framework.default_main_program() + ] = layers.create_global_var( + name=unique_name.generate("learning_rate"), + shape=[1], + value=float(self._learning_rate), + dtype='float32' if self._dtype is None else self._dtype, + persistable=True, + ) @framework.dygraph_only def set_lr(self, value): @@ -431,7 +512,8 @@ class Optimizer(object): if not isinstance(value, (framework.Variable, float)): raise TypeError( "The type of 'value' in optimizer.set_lr must be (float, Variable), but received %s." - % (type(value))) + % (type(value)) + ) if isinstance(self._learning_rate, LearningRateDecay): raise RuntimeError( "optimizer's learning rate can't be LearningRateDecay when invoke this API, because this will lead to conflict." @@ -442,28 +524,42 @@ class Optimizer(object): if current_lr is not None: if in_dygraph_mode(): place = _current_expected_place() - _C_ops.full_(current_lr, list(current_lr.shape), - float(value), current_lr.dtype, place) + _C_ops.full_( + current_lr, + list(current_lr.shape), + float(value), + current_lr.dtype, + place, + ) elif _in_legacy_dygraph(): - _legacy_C_ops.fill_constant(current_lr, 'value', - float(value), 'dtype', - current_lr.dtype, 'shape', - list(current_lr.shape)) + _legacy_C_ops.fill_constant( + current_lr, + 'value', + float(value), + 'dtype', + current_lr.dtype, + 'shape', + list(current_lr.shape), + ) else: - global_block = framework.default_main_program( - ).global_block() - global_block.append_op(type='fill_constant', - outputs={'Out': [current_lr]}, - attrs={ - 'dtype': current_lr.dtype, - 'shape': list(current_lr.shape), - 'value': float(value) - }, - stop_gradient=True) + global_block = ( + framework.default_main_program().global_block() + ) + global_block.append_op( + type='fill_constant', + outputs={'Out': [current_lr]}, + attrs={ + 'dtype': current_lr.dtype, + 'shape': list(current_lr.shape), + 'value': float(value), + }, + stop_gradient=True, + ) else: - assert len(value.shape) == 1 and value.shape[ - 0] == 1, "optimizer's learning rate must be 1-D Tensor with shape[1]" + assert ( + len(value.shape) == 1 and value.shape[0] == 1 + ), "optimizer's learning rate must be 1-D Tensor with shape[1]" self._learning_rate_map[framework.default_main_program()] = value @framework.dygraph_only @@ -540,8 +636,7 @@ class Optimizer(object): return self._learning_rate_map.get(program, None) def _append_optimize_op(self, block, param_and_grad): - """ append optimize operator to block and return all the added optimize_op - """ + """append optimize operator to block and return all the added optimize_op""" raise NotImplementedError() def _create_param_lr(self, param_and_grad): @@ -555,8 +650,8 @@ class Optimizer(object): return self._global_learning_rate() else: with default_main_program()._lr_schedule_guard( - is_with_opt=True), framework.name_scope( - 'scale_with_param_lr'): + is_with_opt=True + ), framework.name_scope('scale_with_param_lr'): return self._global_learning_rate() * param_lr def _create_accumulators(self, block, parameters): @@ -581,14 +676,16 @@ class Optimizer(object): """ pass - def _add_accumulator(self, - name, - param, - dtype=None, - fill_value=0.0, - shape=None, - type=None, - device=None): + def _add_accumulator( + self, + name, + param, + dtype=None, + fill_value=0.0, + shape=None, + type=None, + device=None, + ): """Utility function to add an accumulator for a parameter Args: @@ -600,13 +697,17 @@ class Optimizer(object): """ if self._name is not None: name = self._name + "_" + name - if (name in self._accumulators - and param.name in self._accumulators[name]): + if ( + name in self._accumulators + and param.name in self._accumulators[name] + ): if framework._non_static_mode(): return self._accumulators[name][param.name] raise Exception( "Accumulator {} already exists for parameter {}".format( - name, param.name)) + name, param.name + ) + ) if shape == None: shape = param.shape assert isinstance(self.helper, LayerHelper) @@ -620,32 +721,39 @@ class Optimizer(object): persistable=True, dtype=dtype or param.dtype, type=core.VarDesc.VarType.LOD_TENSOR - if framework._non_static_mode() else - (param.type if type is None else type), + if framework._non_static_mode() + else (param.type if type is None else type), shape=shape, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) if device is None: device = self._get_device_for_param(param.name) with device_guard(device): self.helper.set_variable_initializer( - var, initializer=Constant(value=float(fill_value))) + var, initializer=Constant(value=float(fill_value)) + ) if framework._non_static_mode(): if len(self._accumulators_holder) > 0: - assert var_name in self._accumulators_holder, \ - "Optimizer set error, {} should in state dict".format( var_name ) + assert ( + var_name in self._accumulators_holder + ), "Optimizer set error, {} should in state dict".format( + var_name + ) var.set_value(self._accumulators_holder[var_name]) self._accumulators[name][param.name] = var return var - def _add_global_accumulator(self, - name, - dtype=None, - fill_value=0.0, - shape=None, - type=None, - device=None): + def _add_global_accumulator( + self, + name, + dtype=None, + fill_value=0.0, + shape=None, + type=None, + device=None, + ): """Utility function to add a global accumulator for all parameters in the model Args: @@ -659,7 +767,7 @@ class Optimizer(object): """ if self._name is not None: name = self._name + "_" + name - if (name in self._global_accumulators): + if name in self._global_accumulators: if framework._non_static_mode(): return self._global_accumulators[name] raise Exception("Global accumulator {} already exists".format(name)) @@ -677,17 +785,22 @@ class Optimizer(object): dtype=dtype if dtype else self._dtype, type=type, shape=shape, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) if device is None: device = 'cpu' with device_guard(device): self.helper.set_variable_initializer( - var, initializer=Constant(value=float(fill_value))) + var, initializer=Constant(value=float(fill_value)) + ) if framework._non_static_mode(): if len(self._accumulators_holder) > 0: - assert var_name in self._accumulators_holder, \ - "Optimizer set error, {} should in state dict".format( var_name ) + assert ( + var_name in self._accumulators_holder + ), "Optimizer set error, {} should in state dict".format( + var_name + ) var.set_value(self._accumulators_holder[var_name]) self._global_accumulators[name] = var @@ -705,11 +818,15 @@ class Optimizer(object): """ if self._name is not None: name = self._name + "_" + name - if (name not in self._accumulators - or param.name not in self._accumulators[name]): + if ( + name not in self._accumulators + or param.name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, param.name)) + name, param.name + ) + ) return self._accumulators[name][param.name] def _get_global_accumulator(self, name): @@ -723,7 +840,7 @@ class Optimizer(object): """ if self._name is not None: name = self._name + "_" + name - if (name not in self._global_accumulators): + if name not in self._global_accumulators: raise Exception("Global accumulator {} does not exist".format(name)) return self._global_accumulators[name] @@ -732,13 +849,15 @@ class Optimizer(object): if param_and_grad[0].trainable is True: param_name = param_and_grad[0].name ops = target_block.ops - device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) for op in ops: input_arg_names = op.input_arg_names if param_name in input_arg_names: self._param_device_map[param_name] = op.attr( - device_attr_name) + device_attr_name + ) break def _get_device_for_param(self, param_name): @@ -775,17 +894,19 @@ class Optimizer(object): target_block = global_block current_block = framework.default_main_program().current_block() if current_block.idx != global_block.idx: - assert current_block.backward_block_idx != -1, \ - "current block is not global_block, but it doesn't have backward block." + assert ( + current_block.backward_block_idx != -1 + ), "current block is not global_block, but it doesn't have backward block." target_block = framework.default_main_program().blocks[ - current_block.backward_block_idx] + current_block.backward_block_idx + ] start = len(target_block.ops) self._update_param_device_map(parameters_and_grads, target_block) self._create_accumulators( - target_block, - [p[0] for p in parameters_and_grads if p[0].trainable]) + target_block, [p[0] for p in parameters_and_grads if p[0].trainable] + ) self._create_global_learning_rate() if framework._non_static_mode(): @@ -799,13 +920,16 @@ class Optimizer(object): if param_and_grad[1] is None: continue with param_and_grad[0].block.program._optimized_guard( - param_and_grad), name_scope("optimizer"): + param_and_grad + ), name_scope("optimizer"): if param_and_grad[0].trainable is True: device = self._get_device_for_param( - param_and_grad[0].name) + param_and_grad[0].name + ) with device_guard(device): optimize_op = self._append_optimize_op( - target_block, param_and_grad) + target_block, param_and_grad + ) # Get custom finish ops for subclasses # FIXME: Need to fix this once we figure out how to handle dependencies @@ -834,7 +958,8 @@ class Optimizer(object): if p.name == table_name: if table_param is not None: raise RuntimeError( - "multi dist table var found, only support one now!") + "multi dist table var found, only support one now!" + ) table_param = p table_grad = g else: @@ -842,8 +967,9 @@ class Optimizer(object): sgd_op = None if table_param is not None: param_and_grad = [table_param, table_grad] - with table_param.block.program._optimized_guard(param_and_grad), \ - framework.name_scope("optimizer"): + with table_param.block.program._optimized_guard( + param_and_grad + ), framework.name_scope("optimizer"): self._create_global_learning_rate() # create the optimize op sgd_op = global_block.append_op( @@ -851,17 +977,20 @@ class Optimizer(object): inputs={ "Param": table_param, "Grad": table_grad, - "LearningRate": self._create_param_lr(param_and_grad) + "LearningRate": self._create_param_lr(param_and_grad), }, - outputs={"ParamOut": param_and_grad[0]}) + outputs={"ParamOut": param_and_grad[0]}, + ) return new_param_grads, (table_param, table_grad), sgd_op - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ The first part of ``minimize``, do auto-diff to append backward operations for the current program. @@ -897,8 +1026,9 @@ class Optimizer(object): self._dtype = loss.dtype if framework._non_static_mode(): - parameter_list = parameter_list if parameter_list \ - else self._parameter_list + parameter_list = ( + parameter_list if parameter_list else self._parameter_list + ) params_grads = [] for param in parameter_list: @@ -912,29 +1042,36 @@ class Optimizer(object): if callbacks is None: callbacks = [error_clip_callback] else: - assert (isinstance(callbacks, list)) + assert isinstance(callbacks, list) program = loss.block.program - assert len(loss.shape) == 1 and loss.shape[0] == 1, \ - "The loss.shape should be (1L,), but the current loss.shape is {}. " \ + assert len(loss.shape) == 1 and loss.shape[0] == 1, ( + "The loss.shape should be (1L,), but the current loss.shape is {}. " "Maybe that you should call paddle.mean to process the current loss.".format( - loss.shape) - parameter_list = parameter_list if parameter_list \ - else self._parameter_list + loss.shape + ) + ) + parameter_list = ( + parameter_list if parameter_list else self._parameter_list + ) with program_guard(program, startup_program): - params_grads = append_backward(loss, parameter_list, - act_no_grad_set, callbacks) + params_grads = append_backward( + loss, parameter_list, act_no_grad_set, callbacks + ) return params_grads def _create_regularization_of_grad(self, param, grad, regularization=None): - """ Create and add backward regularization Operators + """Create and add backward regularization Operators Function helper of append_regularization_ops. """ # If no gradient or no regularization is specified, then we don't need to do anything if grad is None or ( - (not hasattr(param, 'regularizer') or - (hasattr(param, 'regularizer') and param.regularizer is None)) - and regularization is None): + ( + not hasattr(param, 'regularizer') + or (hasattr(param, 'regularizer') and param.regularizer is None) + ) + and regularization is None + ): return grad regularization_term = None if hasattr(param, 'regularizer') and param.regularizer is not None: @@ -959,7 +1096,8 @@ class Optimizer(object): dtype=param.dtype, shape=param.shape, lod_level=param.lod_level, - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) inputs = {"X": [grad, regularization_term]} outputs = {"Out": [new_grad]} @@ -967,9 +1105,9 @@ class Optimizer(object): return new_grad - def append_regularization_ops(self, - parameters_and_grads, - regularization=None): + def append_regularization_ops( + self, parameters_and_grads, regularization=None + ): r"""Create and add backward regularization Operators Creates and adds backward regularization operators in the BlockDesc. @@ -994,23 +1132,28 @@ class Optimizer(object): if framework._non_static_mode(): for param, grad in parameters_and_grads: new_grad = self._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) params_and_grads.append((param, new_grad)) else: repeate_regularizer = False with framework.name_scope('regularization'): for param, grad in parameters_and_grads: - if not repeate_regularizer and getattr( - param, 'regularizer', - None) is not None and regularization is not None: + if ( + not repeate_regularizer + and getattr(param, 'regularizer', None) is not None + and regularization is not None + ): repeate_regularizer = True logging.info( "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % regularization.__str__()) + % regularization.__str__() + ) with param.block.program._optimized_guard([param, grad]): new_grad = self._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) params_and_grads.append((param, new_grad)) return params_and_grads @@ -1021,11 +1164,14 @@ class Optimizer(object): if g is None: continue g.persistable = True - if getattr(p, 'need_clip', True) is False or getattr( - p, 'regularizer', None) is not None: + if ( + getattr(p, 'need_clip', True) is False + or getattr(p, 'regularizer', None) is not None + ): warnings.warn( "flatten_param_grads=True will be discarded since paramter '{}''s need_clip is False or " - "the regularizer is set".format(p.name)) + "the regularizer is set".format(p.name) + ) self._flatten_param_grads = False return params_grads @@ -1040,7 +1186,8 @@ class Optimizer(object): persistable=True, dtype=need_flatten_params[0].dtype, shape=[np.sum(shape)], - belong_to_optimizer=True) + belong_to_optimizer=True, + ) flatten_param.trainable = True flatten_param.optimize_attr = need_flatten_params[0].optimize_attr @@ -1051,41 +1198,48 @@ class Optimizer(object): persistable=True, dtype=need_flatten_grads[0].dtype, shape=[np.sum(shape)], - belong_to_optimizer=True) + belong_to_optimizer=True, + ) with program_guard(default_main_program()): - block.append_op(type="coalesce_tensor", - inputs={"Input": need_flatten_params}, - outputs={ - "Output": need_flatten_params, - "FusedOutput": flatten_param - }, - attrs={ - "copy_data": True, - "use_align": True, - "align_size": self._align_size, - "dtype": need_flatten_params[0].dtype - }) - - block.append_op(type="coalesce_tensor", - inputs={"Input": need_flatten_grads}, - outputs={ - "Output": need_flatten_grads, - "FusedOutput": flatten_grad - }, - attrs={ - "copy_data": True, - "use_align": True, - "align_size": self._align_size, - "dtype": need_flatten_grads[0].dtype - }) + block.append_op( + type="coalesce_tensor", + inputs={"Input": need_flatten_params}, + outputs={ + "Output": need_flatten_params, + "FusedOutput": flatten_param, + }, + attrs={ + "copy_data": True, + "use_align": True, + "align_size": self._align_size, + "dtype": need_flatten_params[0].dtype, + }, + ) + + block.append_op( + type="coalesce_tensor", + inputs={"Input": need_flatten_grads}, + outputs={ + "Output": need_flatten_grads, + "FusedOutput": flatten_grad, + }, + attrs={ + "copy_data": True, + "use_align": True, + "align_size": self._align_size, + "dtype": need_flatten_grads[0].dtype, + }, + ) - #NOTE(zhiqiu): the initializer should be set after coalesce_tensor op, + # NOTE(zhiqiu): the initializer should be set after coalesce_tensor op, # so the shape of flatten_param and flatten_grad will be inferred. - self.helper.set_variable_initializer(flatten_param, - initializer=Constant(0.0)) - self.helper.set_variable_initializer(flatten_grad, - initializer=Constant(0.0)) + self.helper.set_variable_initializer( + flatten_param, initializer=Constant(0.0) + ) + self.helper.set_variable_initializer( + flatten_grad, initializer=Constant(0.0) + ) return [(flatten_param, flatten_grad)] @@ -1115,8 +1269,9 @@ class Optimizer(object): # NOTE(zhiqiu): currently, only support ClipGradByGlobalNorm and without regularization. if self._flatten_param_grads and self.regularization is None: - if self._grad_clip == None or isinstance(self._grad_clip, - ClipGradByGlobalNorm): + if self._grad_clip == None or isinstance( + self._grad_clip, ClipGradByGlobalNorm + ): params_grads = self.flatten_param_grads(params_grads) # 'optimizer(grad_clip)' or 'set_gradient_clip' @@ -1126,8 +1281,9 @@ class Optimizer(object): params_grads = append_gradient_clip_ops(params_grads) # Add regularization if any - params_grads = self.append_regularization_ops(params_grads, - self.regularization) + params_grads = self.append_regularization_ops( + params_grads, self.regularization + ) optimize_ops = self._create_optimization_pass(params_grads) return optimize_ops @@ -1145,12 +1301,15 @@ class Optimizer(object): list: A list of operators appended to the current program. """ if framework._non_static_mode(): - with program_guard(framework.default_main_program(), - framework.default_startup_program()): + with program_guard( + framework.default_main_program(), + framework.default_startup_program(), + ): if self._grad_clip is not None: params_grads = self._grad_clip(params_grads) params_grads = self.append_regularization_ops( - params_grads, self.regularization) + params_grads, self.regularization + ) optimize_ops = self._create_optimization_pass(params_grads) else: program = loss.block.program @@ -1162,7 +1321,8 @@ class Optimizer(object): no_grad_set = _get_no_grad_set_name(no_grad_set) parameters = loss.block.program.global_block().all_parameters() param_no_trainable = set( - [param.name for param in parameters if param.trainable is False]) + [param.name for param in parameters if param.trainable is False] + ) # If the parameter is no trainable, it should not have a gradient. no_grad_set.update(param_no_trainable) @@ -1202,11 +1362,9 @@ class Optimizer(object): p.clear_gradient() @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ Add operations to minimize ``loss`` by updating ``parameter_list``. @@ -1234,17 +1392,20 @@ class Optimizer(object): """ assert isinstance(loss, Variable), "The loss should be an Variable." - parameter_list = parameter_list if parameter_list \ - else self._parameter_list + parameter_list = ( + parameter_list if parameter_list else self._parameter_list + ) - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + optimize_ops = self.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads @@ -1305,19 +1466,23 @@ class SGDOptimizer(Optimizer): """ - def __init__(self, - learning_rate, - parameter_list=None, - regularization=None, - grad_clip=None, - multi_precision=False, - name=None): + def __init__( + self, + learning_rate, + parameter_list=None, + regularization=None, + grad_clip=None, + multi_precision=False, + name=None, + ): assert learning_rate is not None - super(SGDOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(SGDOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "sgd" self._use_mkldnn = False self._multi_precision = multi_precision @@ -1331,19 +1496,23 @@ class SGDOptimizer(Optimizer): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -1357,7 +1526,10 @@ class SGDOptimizer(Optimizer): if self._multi_precision and p.dtype == core.VarDesc.VarType.FP16: master_p = self._create_master_weight(p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Adam optimizer." @@ -1366,19 +1538,35 @@ class SGDOptimizer(Optimizer): @no_grad def _append_optimize_op(self, block, param_and_grad): - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) lr = self._create_param_lr(param_and_grad) if in_dygraph_mode(): - _C_ops.sgd_(param_and_grad[0], lr, param_and_grad[1], master_weight, - find_master) + _C_ops.sgd_( + param_and_grad[0], + lr, + param_and_grad[1], + master_weight, + find_master, + ) return None if _in_legacy_dygraph(): - _legacy_C_ops.sgd(param_and_grad[0], lr, param_and_grad[1], - master_weight, param_and_grad[0], master_weight) + _legacy_C_ops.sgd( + param_and_grad[0], + lr, + param_and_grad[1], + master_weight, + param_and_grad[0], + master_weight, + ) return None assert isinstance(block, framework.Block) @@ -1386,7 +1574,7 @@ class SGDOptimizer(Optimizer): inputs = { "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": lr + "LearningRate": lr, } outputs = {"ParamOut": param_and_grad[0]} @@ -1397,11 +1585,13 @@ class SGDOptimizer(Optimizer): inputs["MasterParam"] = master_weight outputs["MasterParamOut"] = master_weight - sgd_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + sgd_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return sgd_op @@ -1478,21 +1668,25 @@ class MomentumOptimizer(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate, - momentum, - parameter_list=None, - use_nesterov=False, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + momentum, + parameter_list=None, + use_nesterov=False, + regularization=None, + grad_clip=None, + name=None, + ): assert learning_rate is not None assert momentum is not None - super(MomentumOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(MomentumOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "momentum" self._momentum = momentum self._use_nesterov = bool(use_nesterov) @@ -1506,15 +1700,26 @@ class MomentumOptimizer(Optimizer): def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - velocity_acc = self._get_accumulator(self._velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._velocity_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) master_weight = None if framework._non_static_mode(): _, _, _ = _legacy_C_ops.momentum( - param_and_grad[0], param_and_grad[1], velocity_acc, lr, - master_weight, param_and_grad[0], velocity_acc, master_weight, - 'mu', self._momentum, 'use_nesterov', self._use_nesterov) + param_and_grad[0], + param_and_grad[1], + velocity_acc, + lr, + master_weight, + param_and_grad[0], + velocity_acc, + master_weight, + 'mu', + self._momentum, + 'use_nesterov', + self._use_nesterov, + ) return None attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov} @@ -1522,19 +1727,21 @@ class MomentumOptimizer(Optimizer): "Param": [param_and_grad[0]], "Grad": [param_and_grad[1]], "Velocity": [velocity_acc], - "LearningRate": [lr] + "LearningRate": [lr], } outputs = { "ParamOut": [param_and_grad[0]], - "VelocityOut": [velocity_acc] + "VelocityOut": [velocity_acc], } # create the momentum optimize op - momentum_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + momentum_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return momentum_op @@ -1607,32 +1814,36 @@ class DGCMomentumOptimizer(Optimizer): _u_velocity_acc_str = "_dgc_u_" _v_velocity_acc_str = "_dgc_v_" - def __init__(self, - learning_rate, - momentum, - rampup_begin_step, - rampup_step=1, - sparsity=[0.999], - parameter_list=None, - use_nesterov=False, - num_trainers=None, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + momentum, + rampup_begin_step, + rampup_step=1, + sparsity=[0.999], + parameter_list=None, + use_nesterov=False, + num_trainers=None, + regularization=None, + grad_clip=None, + name=None, + ): if framework._non_static_mode(): raise Exception("In dygraph, don't support DGCMomentumOptimizer.") - assert core.is_compiled_with_cuda(), \ - "Paddle is not compiled with CUDA. DGC is only support GPU for now." + assert ( + core.is_compiled_with_cuda() + ), "Paddle is not compiled with CUDA. DGC is only support GPU for now." assert learning_rate is not None assert momentum is not None - super(DGCMomentumOptimizer, - self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(DGCMomentumOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "dgc_momentum" self._momentum = momentum self._use_nesterov = bool(use_nesterov) @@ -1651,17 +1862,20 @@ class DGCMomentumOptimizer(Optimizer): raise TypeError( "The type of grad_clip should be 'GradientClipByNorm', because DGCMomentumOptimizer only support GradientClipByNorm" ) - assert isinstance( - num_trainers, int - ), "The type of num_trainers should be 'int', but received %s" % type( - num_trainers) - assert num_trainers > 0, "The value of num_trainers should be greater than 0!" + assert isinstance(num_trainers, int), ( + "The type of num_trainers should be 'int', but received %s" + % type(num_trainers) + ) + assert ( + num_trainers > 0 + ), "The value of num_trainers should be greater than 0!" self._num_trainers = num_trainers self._dgc_clip_norm = grad_clip.clip_norm * (num_trainers**-0.5) self.regular_type, self.regular_coeff = self._get_regularization_param( - self.regularization) + self.regularization + ) def _get_regularization_param(self, regularization): regular_type = 0 @@ -1670,6 +1884,7 @@ class DGCMomentumOptimizer(Optimizer): if regularization is not None: regular_coeff = regularization._regularization_coeff from .regularizer import L1Decay, L2Decay + if isinstance(regularization, L1Decay): regular_type = 1 elif isinstance(regularization, L2Decay): @@ -1680,17 +1895,20 @@ class DGCMomentumOptimizer(Optimizer): def _is_use_dgc(self, param_var, grad_var): var_numel = abs(reduce(lambda x, y: x * y, param_var.shape)) - if var_numel < 16384 or \ - param_var.type == core.VarDesc.VarType.SELECTED_ROWS or \ - grad_var.type == core.VarDesc.VarType.SELECTED_ROWS or \ - param_var.dtype != core.VarDesc.VarType.FP32 : + if ( + var_numel < 16384 + or param_var.type == core.VarDesc.VarType.SELECTED_ROWS + or grad_var.type == core.VarDesc.VarType.SELECTED_ROWS + or param_var.dtype != core.VarDesc.VarType.FP32 + ): return False return True def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - velocity_acc = self._get_accumulator(self._u_velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._u_velocity_acc_str, param_and_grad[0] + ) assert velocity_acc is not None inputs = { @@ -1709,36 +1927,42 @@ class DGCMomentumOptimizer(Optimizer): type = "momentum" else: type = "dgc_momentum" - inputs.update({ - "current_step": self._global_step_var, - "nranks": self._nranks_var - }) + inputs.update( + { + "current_step": self._global_step_var, + "nranks": self._nranks_var, + } + ) outputs.update({'Grad_out': param_and_grad[1]}) attrs.update({"rampup_begin_step": float(self._rampup_begin_step)}) # create the dgc momentum optimize op - dgc_momentum_op = block.append_op(type=type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + dgc_momentum_op = block.append_op( + type=type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return dgc_momentum_op def _add_auto_increment_var(self, counter_name, begin, step=1): helper = LayerHelper('global_step_counter') counter, is_new_var = helper.create_or_get_global_variable( - name=counter_name, dtype='float32', shape=[1], persistable=True) + name=counter_name, dtype='float32', shape=[1], persistable=True + ) if is_new_var: - helper.set_variable_initializer(counter, - initializer=Constant( - value=float(begin - 1), - force_cpu=True)) + helper.set_variable_initializer( + counter, + initializer=Constant(value=float(begin - 1), force_cpu=True), + ) helper.main_program.global_block()._prepend_op( type='increment', inputs={'X': [counter]}, outputs={'Out': [counter]}, attrs={'step': float(step)}, - stop_gradient=True) + stop_gradient=True, + ) counter.stop_gradient = True return counter @@ -1746,12 +1970,13 @@ class DGCMomentumOptimizer(Optimizer): def _add_nranks_var(self, name, value=-1): helper = LayerHelper('global_step_counter') counter, is_new_var = helper.create_or_get_global_variable( - name=name, dtype='float32', shape=[1], persistable=True) + name=name, dtype='float32', shape=[1], persistable=True + ) if is_new_var: - helper.set_variable_initializer(counter, - initializer=Constant( - value=float(value), - force_cpu=True)) + helper.set_variable_initializer( + counter, + initializer=Constant(value=float(value), force_cpu=True), + ) counter.stop_gradient = True return counter @@ -1762,10 +1987,12 @@ class DGCMomentumOptimizer(Optimizer): # step counter self._global_step_var = self._add_auto_increment_var( - counter_name=core.dgc.kDGCCounterName(), begin=0) + counter_name=core.dgc.kDGCCounterName(), begin=0 + ) - self._nranks_var = self._add_nranks_var(name=core.dgc.kDGCNRanksName(), - value=-1) + self._nranks_var = self._add_nranks_var( + name=core.dgc.kDGCNRanksName(), value=-1 + ) # rampup begin step var for all_reduce_op_handle self._rampup_begin_step_var = tensor.create_global_var( @@ -1774,7 +2001,8 @@ class DGCMomentumOptimizer(Optimizer): persistable=True, name=core.dgc.kDGCRampUpBeginStepName(), value=self._rampup_begin_step * 1.0, - force_cpu=True) + force_cpu=True, + ) self.helper = LayerHelper(self.__class__.__name__) @@ -1787,29 +2015,32 @@ class DGCMomentumOptimizer(Optimizer): v_var = self._add_accumulator(self._v_velocity_acc_str, param_var) - k_var = tensor.create_global_var(shape=[1], - dtype=param_var.dtype, - persistable=True, - name=param_var.name + - core.dgc.kDGCKName(), - value=0.0, - force_cpu=True) - - encoded_var = tensor.create_global_var(shape=[1], - dtype=param_var.dtype, - persistable=True, - name=param_var.name + - core.dgc.kDGCEncodedName(), - value=0.0, - force_cpu=False) - - gather_var = tensor.create_global_var(shape=[1], - dtype=param_var.dtype, - persistable=True, - name=param_var.name + - core.dgc.kDGCGatherName(), - value=0.0, - force_cpu=False) + k_var = tensor.create_global_var( + shape=[1], + dtype=param_var.dtype, + persistable=True, + name=param_var.name + core.dgc.kDGCKName(), + value=0.0, + force_cpu=True, + ) + + encoded_var = tensor.create_global_var( + shape=[1], + dtype=param_var.dtype, + persistable=True, + name=param_var.name + core.dgc.kDGCEncodedName(), + value=0.0, + force_cpu=False, + ) + + gather_var = tensor.create_global_var( + shape=[1], + dtype=param_var.dtype, + persistable=True, + name=param_var.name + core.dgc.kDGCGatherName(), + value=0.0, + force_cpu=False, + ) # del back oprolevarname op_maker = core.op_proto_and_checker_maker @@ -1832,14 +2063,23 @@ class DGCMomentumOptimizer(Optimizer): clip_var = grad_var if self._dgc_clip_norm is not None: clip_var = self._append_clip_norm(grad_var, self._dgc_clip_norm) - self._dgc_op(param_var, clip_var, grad_var, u_var, v_var, k_var, - encoded_var, gather_var) + self._dgc_op( + param_var, + clip_var, + grad_var, + u_var, + v_var, + k_var, + encoded_var, + gather_var, + ) def _is_the_backward_op(self, op): op_maker = core.op_proto_and_checker_maker backward = core.op_proto_and_checker_maker.OpRole.Backward - if op_maker.kOpRoleVarAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(backward): + if op_maker.kOpRoleVarAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(backward): return True return False @@ -1849,34 +2089,42 @@ class DGCMomentumOptimizer(Optimizer): helper = LayerHelper("dgc_clip_by_norm_op", **args) if name is None: - name = unique_name.generate_with_ignorable_key(".".join( - [helper.name, 'tmp'])) - - out = helper.create_variable(type=x.type, - name=name, - dtype=x.dtype, - persistable=False) - - helper.append_op(type="dgc_clip_by_norm", - inputs={ - "X": x, - "current_step": self._global_step_var - }, - attrs={ - "max_norm": max_norm, - "rampup_begin_step": float(self._rampup_begin_step) - }, - outputs={"Out": out}) + name = unique_name.generate_with_ignorable_key( + ".".join([helper.name, 'tmp']) + ) + + out = helper.create_variable( + type=x.type, name=name, dtype=x.dtype, persistable=False + ) + + helper.append_op( + type="dgc_clip_by_norm", + inputs={"X": x, "current_step": self._global_step_var}, + attrs={ + "max_norm": max_norm, + "rampup_begin_step": float(self._rampup_begin_step), + }, + outputs={"Out": out}, + ) return out def _append_clip_norm(self, grad_var, clip_norm): with grad_var.block.program._backward_role_guard(): - return self._clip_by_norm(x=grad_var, - max_norm=clip_norm, - name=grad_var.name) + return self._clip_by_norm( + x=grad_var, max_norm=clip_norm, name=grad_var.name + ) - def _dgc_op(self, param_var, clip_var, grad_var, u_var, v_var, k_var, - encoded_var, gather_var): + def _dgc_op( + self, + param_var, + clip_var, + grad_var, + u_var, + v_var, + k_var, + encoded_var, + gather_var, + ): block = framework.default_main_program().global_block() op_maker = core.op_proto_and_checker_maker @@ -1885,47 +2133,44 @@ class DGCMomentumOptimizer(Optimizer): # The regularizer of the Parameters have higher priority if param_var.regularizer is not None: regular_type, regular_coeff = self._get_regularization_param( - param_var.regularizer) - - dgc_op = block.append_op(type="dgc", - inputs={ - "U": u_var, - "V": v_var, - "Grad": clip_var, - "Param": param_var, - "current_step": self._global_step_var, - "nranks": self._nranks_var, - }, - outputs={ - "U_out": u_var, - "V_out": v_var, - "EncodeGrad": encoded_var, - "k": k_var, - "Grad_out": grad_var, - "GatherBuff": gather_var, - }, - attrs={ - "m": - self._momentum, - "sparsity": - self._sparsity, - "use_nesterov": - self._use_nesterov, - "rampup_begin_step": - float(self._rampup_begin_step), - "rampup_step": - float(self._rampup_step), - "regular_coeff": - float(regular_coeff), - "regular_type": - int(regular_type), - }, - stop_gradient=True) + param_var.regularizer + ) + + dgc_op = block.append_op( + type="dgc", + inputs={ + "U": u_var, + "V": v_var, + "Grad": clip_var, + "Param": param_var, + "current_step": self._global_step_var, + "nranks": self._nranks_var, + }, + outputs={ + "U_out": u_var, + "V_out": v_var, + "EncodeGrad": encoded_var, + "k": k_var, + "Grad_out": grad_var, + "GatherBuff": gather_var, + }, + attrs={ + "m": self._momentum, + "sparsity": self._sparsity, + "use_nesterov": self._use_nesterov, + "rampup_begin_step": float(self._rampup_begin_step), + "rampup_step": float(self._rampup_step), + "regular_coeff": float(regular_coeff), + "regular_type": int(regular_type), + }, + stop_gradient=True, + ) backward = op_maker.OpRole.Backward dgc_op._set_attr(op_maker.kOpRoleAttrName(), backward) - dgc_op._set_attr(op_maker.kOpRoleVarAttrName(), - [param_var.name, grad_var.name]) + dgc_op._set_attr( + op_maker.kOpRoleVarAttrName(), [param_var.name, grad_var.name] + ) @imperative_base.no_grad def apply_gradients(self, params_grads): @@ -1935,8 +2180,11 @@ class DGCMomentumOptimizer(Optimizer): self._append_dgc_ops(params_grads) params_grads = sorted(params_grads, key=lambda x: x[0].name) - params_grads, table_param_and_grad, table_optimize_op = \ - self._process_distribute_lookuptable(params_grads) + ( + params_grads, + table_param_and_grad, + table_optimize_op, + ) = self._process_distribute_lookuptable(params_grads) not_dgc_params_grads = [] dgc_params_grads = [] @@ -1952,10 +2200,12 @@ class DGCMomentumOptimizer(Optimizer): not_dgc_params_grads = self._grad_clip(not_dgc_params_grads) else: not_dgc_params_grads = append_gradient_clip_ops( - not_dgc_params_grads) + not_dgc_params_grads + ) not_dgc_params_grads = self.append_regularization_ops( - not_dgc_params_grads, self.regularization) + not_dgc_params_grads, self.regularization + ) params_grads = not_dgc_params_grads + dgc_params_grads params_grads = sorted(params_grads, key=lambda x: x[0].name) @@ -2031,27 +2281,30 @@ class LarsMomentumOptimizer(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate, - momentum, - lars_coeff=0.001, - lars_weight_decay=0.0005, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None, - exclude_from_weight_decay=None, - epsilon=0, - multi_precision=False, - rescale_grad=1.0): + def __init__( + self, + learning_rate, + momentum, + lars_coeff=0.001, + lars_weight_decay=0.0005, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + exclude_from_weight_decay=None, + epsilon=0, + multi_precision=False, + rescale_grad=1.0, + ): assert learning_rate is not None assert momentum is not None - super(LarsMomentumOptimizer, - self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(LarsMomentumOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "lars_momentum" self._momentum = momentum self._lars_coeff = float(lars_coeff) @@ -2073,19 +2326,23 @@ class LarsMomentumOptimizer(Optimizer): var_name = param.name + '_fp32_master' var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -2099,15 +2356,22 @@ class LarsMomentumOptimizer(Optimizer): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _create_accumulators(self, block, parameters): @@ -2118,7 +2382,10 @@ class LarsMomentumOptimizer(Optimizer): master_p = self._create_master_weight(p) self._add_accumulator(self._velocity_acc_str, master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Lars optimizer." @@ -2135,14 +2402,20 @@ class LarsMomentumOptimizer(Optimizer): _lars_weight_decay = 0.0 break - velocity_acc = self._get_accumulator(self._velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._velocity_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) attrs = { "mu": self._momentum, @@ -2150,14 +2423,14 @@ class LarsMomentumOptimizer(Optimizer): "lars_weight_decay": [_lars_weight_decay], "multi_precision": find_master, "epsilon": self._epsilon, - "rescale_grad": self._rescale_grad + "rescale_grad": self._rescale_grad, } inputs = { "Param": param_and_grad[0], "Grad": param_and_grad[1], "Velocity": velocity_acc, - "LearningRate": lr + "LearningRate": lr, } outputs = {"ParamOut": param_and_grad[0], "VelocityOut": velocity_acc} @@ -2168,18 +2441,34 @@ class LarsMomentumOptimizer(Optimizer): if framework._non_static_mode(): tmp, tmp2 = _legacy_C_ops.lars_momentum( - [param_and_grad[0]], [param_and_grad[1]], [velocity_acc], [lr], - [param_and_grad[0]], [velocity_acc], "mu", self._momentum, - "lars_coeff", self._lars_coeff, "lars_weight_decay", - [_lars_weight_decay], "multi_precision", find_master, "epsilon", - self._epsilon, "rescale_grad", self._rescale_grad) + [param_and_grad[0]], + [param_and_grad[1]], + [velocity_acc], + [lr], + [param_and_grad[0]], + [velocity_acc], + "mu", + self._momentum, + "lars_coeff", + self._lars_coeff, + "lars_weight_decay", + [_lars_weight_decay], + "multi_precision", + find_master, + "epsilon", + self._epsilon, + "rescale_grad", + self._rescale_grad, + ) else: # create the momentum optimize op - momentum_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + momentum_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return momentum_op @@ -2249,21 +2538,25 @@ class AdagradOptimizer(Optimizer): """ _moment_acc_str = "moment" - def __init__(self, - learning_rate, - epsilon=1.0e-6, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None, - initial_accumulator_value=0.0): + def __init__( + self, + learning_rate, + epsilon=1.0e-6, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + initial_accumulator_value=0.0, + ): assert learning_rate is not None assert epsilon is not None - super(AdagradOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(AdagradOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "adagrad" self._epsilon = epsilon self.initial_accumulator_value = initial_accumulator_value @@ -2272,26 +2565,38 @@ class AdagradOptimizer(Optimizer): assert isinstance(block, framework.Block) for p in parameters: - self._add_accumulator(self._moment_acc_str, - p, - fill_value=self.initial_accumulator_value) + self._add_accumulator( + self._moment_acc_str, + p, + fill_value=self.initial_accumulator_value, + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - moment_acc = self._get_accumulator(self._moment_acc_str, - param_and_grad[0]) + moment_acc = self._get_accumulator( + self._moment_acc_str, param_and_grad[0] + ) if in_dygraph_mode(): - _C_ops.adagrad_(param_and_grad[0], param_and_grad[1], moment_acc, - self._create_param_lr(param_and_grad), - self._epsilon) + _C_ops.adagrad_( + param_and_grad[0], + param_and_grad[1], + moment_acc, + self._create_param_lr(param_and_grad), + self._epsilon, + ) return None elif _in_legacy_dygraph(): - _legacy_C_ops.adagrad(param_and_grad[0], param_and_grad[1], - moment_acc, - self._create_param_lr(param_and_grad), - param_and_grad[0], moment_acc, "epsilon", - self._epsilon) + _legacy_C_ops.adagrad( + param_and_grad[0], + param_and_grad[1], + moment_acc, + self._create_param_lr(param_and_grad), + param_and_grad[0], + moment_acc, + "epsilon", + self._epsilon, + ) return None else: # Create the adagrad optimizer op @@ -2301,14 +2606,15 @@ class AdagradOptimizer(Optimizer): "Param": param_and_grad[0], "Grad": param_and_grad[1], "Moment": moment_acc, - "LearningRate": self._create_param_lr(param_and_grad) + "LearningRate": self._create_param_lr(param_and_grad), }, outputs={ "ParamOut": param_and_grad[0], - "MomentOut": moment_acc + "MomentOut": moment_acc, }, attrs={"epsilon": self._epsilon}, - stop_gradient=True) + stop_gradient=True, + ) return adagrad_op @@ -2476,31 +2782,34 @@ class AdamOptimizer(Optimizer): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None, - lazy_mode=False, - use_global_beta_pow=False, - flatten_param_grads=False, - align_size=-1): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + lazy_mode=False, + use_global_beta_pow=False, + flatten_param_grads=False, + align_size=-1, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(AdamOptimizer, - self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - flatten_param_grads=flatten_param_grads, - align_size=align_size, - name=name) + super(AdamOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + flatten_param_grads=flatten_param_grads, + align_size=align_size, + name=name, + ) self.type = "adam" self._beta1 = beta1 self._beta2 = beta2 @@ -2519,65 +2828,109 @@ class AdamOptimizer(Optimizer): self._add_accumulator( name=self._beta1_pow_acc_str, param=p, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_accumulator( name=self._beta2_pow_acc_str, param=p, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) if self._use_global_beta_pow: self._add_global_accumulator( name=self._beta1_pow_acc_str, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_global_accumulator( name=self._beta2_pow_acc_str, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) if self._use_global_beta_pow: beta1_pow_acc = self._get_global_accumulator( - self._beta1_pow_acc_str) + self._beta1_pow_acc_str + ) beta2_pow_acc = self._get_global_accumulator( - self._beta2_pow_acc_str) + self._beta2_pow_acc_str + ) else: - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) # create the adam optimize op if framework._non_static_mode(): - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) master_weight = None _, _, _, _, _, _ = _legacy_C_ops.adam( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, param_and_grad[0], - moment1, moment2, beta1_pow_acc, beta2_pow_acc, master_weight, - 'epsilon', self._epsilon, 'lazy_mode', self._lazy_mode, - 'min_row_size_to_use_multithread', 1000, 'beta1', _beta1, - 'beta2', _beta2, 'use_global_beta_pow', - self._use_global_beta_pow) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'epsilon', + self._epsilon, + 'lazy_mode', + self._lazy_mode, + 'min_row_size_to_use_multithread', + 1000, + 'beta1', + _beta1, + 'beta2', + _beta2, + 'use_global_beta_pow', + self._use_global_beta_pow, + ) return None @@ -2588,7 +2941,7 @@ class AdamOptimizer(Optimizer): "Moment1": [moment1], "Moment2": [moment2], "Beta1Pow": [beta1_pow_acc], - "Beta2Pow": [beta2_pow_acc] + "Beta2Pow": [beta2_pow_acc], } # Pass found_inf to adam, to skip update for not only param, but also momentum and beta_pow @@ -2607,7 +2960,7 @@ class AdamOptimizer(Optimizer): attrs = { "lazy_mode": self._lazy_mode, "min_row_size_to_use_multithread": 1000, - 'use_global_beta_pow': self._use_global_beta_pow + 'use_global_beta_pow': self._use_global_beta_pow, } if isinstance(self._beta1, Variable): @@ -2623,23 +2976,26 @@ class AdamOptimizer(Optimizer): else: attrs['epsilon'] = self._epsilon - adam_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + adam_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return adam_op def _finish_update(self, block, parameters_and_grads): - r"""Update beta1_pow and beta2_pow accumulator - """ + r"""Update beta1_pow and beta2_pow accumulator""" assert isinstance(block, framework.Block) if self._use_global_beta_pow: beta1_pow_acc = self._get_global_accumulator( - self._beta1_pow_acc_str) + self._beta1_pow_acc_str + ) beta2_pow_acc = self._get_global_accumulator( - self._beta2_pow_acc_str) + self._beta2_pow_acc_str + ) with block.program._optimized_guard([]): inputs = {"X": beta1_pow_acc} @@ -2648,18 +3004,22 @@ class AdamOptimizer(Optimizer): if isinstance(self._beta1, Variable): inputs["Y"] = self._beta1 # use elementwise_mul for better performance - block.append_op(type="elementwise_mul", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + block.append_op( + type="elementwise_mul", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) else: attrs['scale'] = self._beta1 - block.append_op(type="scale", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + block.append_op( + type="scale", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) inputs = {"X": beta2_pow_acc} outputs = {"Out": beta2_pow_acc} @@ -2667,18 +3027,22 @@ class AdamOptimizer(Optimizer): if isinstance(self._beta2, Variable): inputs["Y"] = self._beta2 # use elementwise_mul for better performance - block.append_op(type="elementwise_mul", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + block.append_op( + type="elementwise_mul", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) else: attrs['scale'] = self._beta2 - block.append_op(type="scale", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + block.append_op( + type="scale", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) class AdamaxOptimizer(Optimizer): @@ -2766,24 +3130,28 @@ class AdamaxOptimizer(Optimizer): _inf_norm_acc_str = "inf_norm" _beta1_pow_acc_str = "beta1_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(AdamaxOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(AdamaxOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "adamax" self._beta1 = beta1 self._beta2 = beta2 @@ -2794,31 +3162,54 @@ class AdamaxOptimizer(Optimizer): for p in parameters: self._add_accumulator(self._moment_acc_str, p) self._add_accumulator(self._inf_norm_acc_str, p) - self._add_accumulator(name=self._beta1_pow_acc_str, - param=p, - fill_value=self._beta1, - shape=[1]) + self._add_accumulator( + name=self._beta1_pow_acc_str, + param=p, + fill_value=self._beta1, + shape=[1], + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) moment = self._get_accumulator(self._moment_acc_str, param_and_grad[0]) - inf_norm = self._get_accumulator(self._inf_norm_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) + inf_norm = self._get_accumulator( + self._inf_norm_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) if framework.in_dygraph_mode(): - _C_ops.adamax_(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), moment, - inf_norm, beta1_pow_acc, self._beta1, self._beta2, - self._epsilon) + _C_ops.adamax_( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + moment, + inf_norm, + beta1_pow_acc, + self._beta1, + self._beta2, + self._epsilon, + ) elif framework._in_legacy_dygraph(): - _legacy_C_ops.adamax(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), moment, - inf_norm, beta1_pow_acc, param_and_grad[0], - moment, inf_norm, "beta1", self._beta1, - "beta2", self._beta2, "epsilon", self._epsilon) + _legacy_C_ops.adamax( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + moment, + inf_norm, + beta1_pow_acc, + param_and_grad[0], + moment, + inf_norm, + "beta1", + self._beta1, + "beta2", + self._beta2, + "epsilon", + self._epsilon, + ) else: # create the adamax optimize op adamax_op = block.append_op( @@ -2829,47 +3220,53 @@ class AdamaxOptimizer(Optimizer): "LearningRate": self._create_param_lr(param_and_grad), "Moment": moment, "InfNorm": inf_norm, - "Beta1Pow": beta1_pow_acc + "Beta1Pow": beta1_pow_acc, }, outputs={ "ParamOut": param_and_grad[0], "MomentOut": moment, - "InfNormOut": inf_norm + "InfNormOut": inf_norm, }, attrs={ "beta1": self._beta1, "beta2": self._beta2, - "epsilon": self._epsilon + "epsilon": self._epsilon, }, - stop_gradient=True) + stop_gradient=True, + ) return adamax_op def _finish_update(self, block, parameters_and_grads): - """Update Beta1 Power accumulator - """ + """Update Beta1 Power accumulator""" assert isinstance(block, framework.Block) for param, grad in parameters_and_grads: if grad is None or param.trainable is False: continue - with param.block.program._optimized_guard([param, grad - ]), name_scope('adamx'): - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param) + with param.block.program._optimized_guard( + [param, grad] + ), name_scope('adamx'): + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param + ) if framework._non_static_mode(): if framework.in_dygraph_mode(): - tmp = _C_ops.scale(beta1_pow_acc, self._beta1, 0.0, - True) + tmp = _C_ops.scale( + beta1_pow_acc, self._beta1, 0.0, True + ) else: - tmp = _legacy_C_ops.scale(beta1_pow_acc, "scale", - self._beta1) + tmp = _legacy_C_ops.scale( + beta1_pow_acc, "scale", self._beta1 + ) beta1_pow_acc.copy_(tmp, False) else: - block.append_op(type="scale", - inputs={"X": beta1_pow_acc}, - outputs={"Out": beta1_pow_acc}, - attrs={"scale": self._beta1}, - stop_gradient=True) + block.append_op( + type="scale", + inputs={"X": beta1_pow_acc}, + outputs={"Out": beta1_pow_acc}, + attrs={"scale": self._beta1}, + stop_gradient=True, + ) class DpsgdOptimizer(Optimizer): @@ -2917,18 +3314,21 @@ class DpsgdOptimizer(Optimizer): Currently, DpsgdOptimizer doesn't support sparse parameter optimization. """ - def __init__(self, - learning_rate=0.001, - clip=0.9, - batch_size=0.999, - sigma=1e-8, - parameter_list=None): + def __init__( + self, + learning_rate=0.001, + clip=0.9, + batch_size=0.999, + sigma=1e-8, + parameter_list=None, + ): assert learning_rate is not None assert clip is not None assert batch_size is not None assert sigma is not None - super(DpsgdOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list) + super(DpsgdOptimizer, self).__init__( + learning_rate=learning_rate, parameter_list=parameter_list + ) self.type = "dpsgd" self._clip = clip self._batch_size = batch_size @@ -2949,29 +3349,37 @@ class DpsgdOptimizer(Optimizer): self._seed = 0 if framework._non_static_mode(): - _legacy_C_ops.dpsgd(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), - param_and_grad[0], "clip", self._clip, - "batch_size", self._batch_size, "sigma", - self._sigma, "seed", self._seed) + _legacy_C_ops.dpsgd( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + param_and_grad[0], + "clip", + self._clip, + "batch_size", + self._batch_size, + "sigma", + self._sigma, + "seed", + self._seed, + ) else: - dpsgd_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "LearningRate": - self._create_param_lr(param_and_grad) - }, - outputs={"ParamOut": param_and_grad[0]}, - attrs={ - "clip": self._clip, - "batch_size": self._batch_size, - "sigma": self._sigma, - "seed": self._seed - }, - stop_gradient=True) + dpsgd_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "LearningRate": self._create_param_lr(param_and_grad), + }, + outputs={"ParamOut": param_and_grad[0]}, + attrs={ + "clip": self._clip, + "batch_size": self._batch_size, + "sigma": self._sigma, + "seed": self._seed, + }, + stop_gradient=True, + ) return dpsgd_op @@ -3034,24 +3442,27 @@ class DecayedAdagradOptimizer(Optimizer): """ _moment_acc_str = "moment" - def __init__(self, - learning_rate, - decay=0.95, - epsilon=1.0e-6, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + decay=0.95, + epsilon=1.0e-6, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): assert learning_rate is not None assert decay is not None assert epsilon is not None - super(DecayedAdagradOptimizer, - self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(DecayedAdagradOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "decayed_adagrad" self._decay = decay self._epsilon = epsilon @@ -3065,16 +3476,23 @@ class DecayedAdagradOptimizer(Optimizer): def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - moment_acc = self._get_accumulator(self._moment_acc_str, - param_and_grad[0]) + moment_acc = self._get_accumulator( + self._moment_acc_str, param_and_grad[0] + ) if framework._non_static_mode(): - _legacy_C_ops.decayed_adagrad(param_and_grad[0], param_and_grad[1], - moment_acc, - self._create_param_lr(param_and_grad), - param_and_grad[0], moment_acc, - "epsilon", self._epsilon, "decay", - self._decay) + _legacy_C_ops.decayed_adagrad( + param_and_grad[0], + param_and_grad[1], + moment_acc, + self._create_param_lr(param_and_grad), + param_and_grad[0], + moment_acc, + "epsilon", + self._epsilon, + "decay", + self._decay, + ) else: # Create the decayed adagrad optimizer op decayed_adagrad_op = block.append_op( @@ -3083,17 +3501,15 @@ class DecayedAdagradOptimizer(Optimizer): "Param": param_and_grad[0], "Grad": param_and_grad[1], "Moment": moment_acc, - "LearningRate": self._create_param_lr(param_and_grad) + "LearningRate": self._create_param_lr(param_and_grad), }, outputs={ "ParamOut": param_and_grad[0], - "MomentOut": moment_acc + "MomentOut": moment_acc, }, - attrs={ - "epsilon": self._epsilon, - "decay": self._decay - }, - stop_gradient=True) + attrs={"epsilon": self._epsilon, "decay": self._decay}, + stop_gradient=True, + ) return decayed_adagrad_op @@ -3155,25 +3571,29 @@ class AdadeltaOptimizer(Optimizer): _avg_squared_grad_acc_str = "_avg_squared_grad" _avg_squared_update_acc_str = "_avg_squared_update" - def __init__(self, - learning_rate, - epsilon=1.0e-6, - rho=0.95, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + epsilon=1.0e-6, + rho=0.95, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set.") if epsilon is None: raise ValueError("epsilon is not set.") if rho is None: raise ValueError("rho is not set.") - super(AdadeltaOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(AdadeltaOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "adadelta" self._epsilon = epsilon self._rho = rho @@ -3191,46 +3611,53 @@ class AdadeltaOptimizer(Optimizer): raise TypeError("block is not instance of framework.Block.") avg_squared_grad_acc = self._get_accumulator( - self._avg_squared_grad_acc_str, param_and_grad[0]) + self._avg_squared_grad_acc_str, param_and_grad[0] + ) avg_squared_update_acc = self._get_accumulator( - self._avg_squared_update_acc_str, param_and_grad[0]) + self._avg_squared_update_acc_str, param_and_grad[0] + ) if framework.in_dygraph_mode(): - _C_ops.adadelta_(param_and_grad[0], param_and_grad[1], - avg_squared_grad_acc, avg_squared_update_acc, - self._rho, self._epsilon) + _C_ops.adadelta_( + param_and_grad[0], + param_and_grad[1], + avg_squared_grad_acc, + avg_squared_update_acc, + self._rho, + self._epsilon, + ) elif framework._in_legacy_dygraph(): - _legacy_C_ops.adadelta(param_and_grad[0], param_and_grad[1], - avg_squared_grad_acc, avg_squared_update_acc, - param_and_grad[0], avg_squared_grad_acc, - avg_squared_update_acc, "epsilon", - self._epsilon, "rho", self._rho) + _legacy_C_ops.adadelta( + param_and_grad[0], + param_and_grad[1], + avg_squared_grad_acc, + avg_squared_update_acc, + param_and_grad[0], + avg_squared_grad_acc, + avg_squared_update_acc, + "epsilon", + self._epsilon, + "rho", + self._rho, + ) else: # Create the adadelta optimizer op - adadelta_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "AvgSquaredGrad": - avg_squared_grad_acc, - "AvgSquaredUpdate": - avg_squared_update_acc - }, - outputs={ - "ParamOut": - param_and_grad[0], - "AvgSquaredGradOut": - avg_squared_grad_acc, - "AvgSquaredUpdateOut": - avg_squared_update_acc - }, - attrs={ - "epsilon": self._epsilon, - "rho": self._rho - }, - stop_gradient=True) + adadelta_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "AvgSquaredGrad": avg_squared_grad_acc, + "AvgSquaredUpdate": avg_squared_update_acc, + }, + outputs={ + "ParamOut": param_and_grad[0], + "AvgSquaredGradOut": avg_squared_grad_acc, + "AvgSquaredUpdateOut": avg_squared_update_acc, + }, + attrs={"epsilon": self._epsilon, "rho": self._rho}, + stop_gradient=True, + ) return adadelta_op @@ -3346,21 +3773,25 @@ class RMSPropOptimizer(Optimizer): _mean_square_acc_str = "mean_square" _mean_grad_acc_str = "mean_grad" - def __init__(self, - learning_rate, - rho=0.95, - epsilon=1.0e-6, - momentum=0.0, - centered=False, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): - super(RMSPropOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + def __init__( + self, + learning_rate, + rho=0.95, + epsilon=1.0e-6, + momentum=0.0, + centered=False, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): + super(RMSPropOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) if learning_rate is None: raise ValueError("learning_rate is not set.") if rho is None: @@ -3389,27 +3820,49 @@ class RMSPropOptimizer(Optimizer): if not isinstance(block, framework.Block): raise TypeError("block is not instance of framework.Block.") - momentum_acc = self._get_accumulator(self._momentum_acc_str, - param_and_grad[0]) - mean_square_acc = self._get_accumulator(self._mean_square_acc_str, - param_and_grad[0]) - mean_grad_acc = self._get_accumulator(self._mean_grad_acc_str, - param_and_grad[0]) + momentum_acc = self._get_accumulator( + self._momentum_acc_str, param_and_grad[0] + ) + mean_square_acc = self._get_accumulator( + self._mean_square_acc_str, param_and_grad[0] + ) + mean_grad_acc = self._get_accumulator( + self._mean_grad_acc_str, param_and_grad[0] + ) if in_dygraph_mode(): - _C_ops.rmsprop_(param_and_grad[0], mean_square_acc, - param_and_grad[1], momentum_acc, - self._create_param_lr(param_and_grad), - mean_grad_acc, self._epsilon, self._rho, - self._momentum, self._centered) + _C_ops.rmsprop_( + param_and_grad[0], + mean_square_acc, + param_and_grad[1], + momentum_acc, + self._create_param_lr(param_and_grad), + mean_grad_acc, + self._epsilon, + self._rho, + self._momentum, + self._centered, + ) return None elif _in_legacy_dygraph(): - _legacy_C_ops.rmsprop(param_and_grad[0], mean_square_acc, - self._create_param_lr(param_and_grad), - param_and_grad[1], momentum_acc, - param_and_grad[0], momentum_acc, - mean_square_acc, mean_grad_acc, "epsilon", - self._epsilon, "decay", self._rho, "momentum", - self._momentum, "centered", self._centered) + _legacy_C_ops.rmsprop( + param_and_grad[0], + mean_square_acc, + self._create_param_lr(param_and_grad), + param_and_grad[1], + momentum_acc, + param_and_grad[0], + momentum_acc, + mean_square_acc, + mean_grad_acc, + "epsilon", + self._epsilon, + "decay", + self._rho, + "momentum", + self._momentum, + "centered", + self._centered, + ) return None else: rmsprop_op = block.append_op( @@ -3426,15 +3879,16 @@ class RMSPropOptimizer(Optimizer): "ParamOut": param_and_grad[0], "MomentOut": momentum_acc, "MeanSquareOut": mean_square_acc, - "MeanGradOut": mean_grad_acc + "MeanGradOut": mean_grad_acc, }, attrs={ "epsilon": self._epsilon, "decay": self._rho, "momentum": self._momentum, - "centered": self._centered + "centered": self._centered, }, - stop_gradient=True) + stop_gradient=True, + ) return rmsprop_op @@ -3537,20 +3991,24 @@ class FtrlOptimizer(Optimizer): _squared_acc_str = "squared" _linear_acc_str = "linear" - def __init__(self, - learning_rate, - l1=0.0, - l2=0.0, - lr_power=-0.5, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): - super(FtrlOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + def __init__( + self, + learning_rate, + l1=0.0, + l2=0.0, + lr_power=-0.5, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): + super(FtrlOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) if learning_rate is None: raise ValueError("learning_rate is not set.") @@ -3571,43 +4029,52 @@ class FtrlOptimizer(Optimizer): if not isinstance(block, framework.Block): raise TypeError("block is not instance of framework.Block.") - squared_acc = self._get_accumulator(self._squared_acc_str, - param_and_grad[0]) - linear_acc = self._get_accumulator(self._linear_acc_str, - param_and_grad[0]) + squared_acc = self._get_accumulator( + self._squared_acc_str, param_and_grad[0] + ) + linear_acc = self._get_accumulator( + self._linear_acc_str, param_and_grad[0] + ) if framework._non_static_mode(): - _legacy_C_ops.ftrl(param_and_grad[0], squared_acc, linear_acc, - param_and_grad[1], - self._create_param_lr(param_and_grad), - param_and_grad[0], squared_acc, linear_acc, "l1", - self._l1, "l2", self._l2, "lr_power", - self._lr_power) + _legacy_C_ops.ftrl( + param_and_grad[0], + squared_acc, + linear_acc, + param_and_grad[1], + self._create_param_lr(param_and_grad), + param_and_grad[0], + squared_acc, + linear_acc, + "l1", + self._l1, + "l2", + self._l2, + "lr_power", + self._lr_power, + ) else: - ftrl_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "SquaredAccumulator": - squared_acc, - "LinearAccumulator": - linear_acc, - "LearningRate": - self._create_param_lr(param_and_grad), - }, - outputs={ - "ParamOut": param_and_grad[0], - "SquaredAccumOut": squared_acc, - "LinearAccumOut": linear_acc - }, - attrs={ - "l1": self._l1, - "l2": self._l2, - "lr_power": self._lr_power - }, - stop_gradient=True) + ftrl_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "SquaredAccumulator": squared_acc, + "LinearAccumulator": linear_acc, + "LearningRate": self._create_param_lr(param_and_grad), + }, + outputs={ + "ParamOut": param_and_grad[0], + "SquaredAccumOut": squared_acc, + "LinearAccumOut": linear_acc, + }, + attrs={ + "l1": self._l1, + "l2": self._l2, + "lr_power": self._lr_power, + }, + stop_gradient=True, + ) return ftrl_op @@ -3690,30 +4157,34 @@ class LambOptimizer(AdamOptimizer): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - lamb_weight_decay=0.01, - beta1=0.9, - beta2=0.999, - epsilon=1e-6, - parameter_list=None, - regularization=None, - grad_clip=None, - exclude_from_weight_decay_fn=None, - name=None): + def __init__( + self, + learning_rate=0.001, + lamb_weight_decay=0.01, + beta1=0.9, + beta2=0.999, + epsilon=1e-6, + parameter_list=None, + regularization=None, + grad_clip=None, + exclude_from_weight_decay_fn=None, + name=None, + ): assert learning_rate is not None assert lamb_weight_decay is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(LambOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - name=name) + super(LambOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + name=name, + ) self.type = "lamb" self._weight_decay = lamb_weight_decay self._exclude_from_weight_decay_fn = exclude_from_weight_decay_fn @@ -3722,57 +4193,82 @@ class LambOptimizer(AdamOptimizer): assert isinstance(block, framework.Block) block.program._use_lamb = True - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - - if self._exclude_from_weight_decay_fn is not None \ - and self._exclude_from_weight_decay_fn(param_and_grad[0]): + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + + if ( + self._exclude_from_weight_decay_fn is not None + and self._exclude_from_weight_decay_fn(param_and_grad[0]) + ): weight_decay = 0.0 else: weight_decay = self._weight_decay lr = self._create_param_lr(param_and_grad) master_weight = None if framework._non_static_mode(): - _legacy_C_ops.lamb(param_and_grad[0], param_and_grad[1], lr, - moment1, moment2, beta1_pow_acc, beta2_pow_acc, - master_weight, param_and_grad[0], moment1, - moment2, beta1_pow_acc, beta2_pow_acc, - master_weight, 'beta1', self._beta1, 'beta2', - self._beta2, 'epsilon', self._epsilon, - 'weight_decay', weight_decay) + _legacy_C_ops.lamb( + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'beta1', + self._beta1, + 'beta2', + self._beta2, + 'epsilon', + self._epsilon, + 'weight_decay', + weight_decay, + ) return None # create the lamb optimize op - lamb_op = block.append_op(type=self.type, - inputs={ - "Param": param_and_grad[0], - "Grad": param_and_grad[1], - "LearningRate": lr, - "Moment1": moment1, - "Moment2": moment2, - "Beta1Pow": beta1_pow_acc, - "Beta2Pow": beta2_pow_acc - }, - outputs={ - "ParamOut": param_and_grad[0], - "Moment1Out": moment1, - "Moment2Out": moment2, - "Beta1PowOut": beta1_pow_acc, - "Beta2PowOut": beta2_pow_acc - }, - attrs={ - "beta1": self._beta1, - "beta2": self._beta2, - "epsilon": self._epsilon, - "weight_decay": weight_decay - }, - stop_gradient=True) + lamb_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "LearningRate": lr, + "Moment1": moment1, + "Moment2": moment2, + "Beta1Pow": beta1_pow_acc, + "Beta2Pow": beta2_pow_acc, + }, + outputs={ + "ParamOut": param_and_grad[0], + "Moment1Out": moment1, + "Moment2Out": moment2, + "Beta1PowOut": beta1_pow_acc, + "Beta2PowOut": beta2_pow_acc, + }, + attrs={ + "beta1": self._beta1, + "beta2": self._beta2, + "epsilon": self._epsilon, + "weight_decay": weight_decay, + }, + stop_gradient=True, + ) return lamb_op @@ -3886,38 +4382,44 @@ class ModelAverage(Optimizer): fetch_list=[loss.name]) """ - def __init__(self, - average_window_rate, - min_average_window=10000, - max_average_window=10000, - regularization=None, - name=None): + def __init__( + self, + average_window_rate, + min_average_window=10000, + max_average_window=10000, + regularization=None, + name=None, + ): if framework._non_static_mode(): raise Exception("In dygraph, don't support ModelAverage.") - super(ModelAverage, self).__init__(0.0, - regularization=regularization, - name=name) + super(ModelAverage, self).__init__( + 0.0, regularization=regularization, name=name + ) self.average_window = average_window_rate self.min_average_window = min_average_window self.max_average_window = max_average_window self.params_grads = [] - for param in framework.default_main_program().global_block( - ).all_parameters(): + for param in ( + framework.default_main_program().global_block().all_parameters() + ): if param.do_model_average != False: grad = param.block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [param.name, 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join([param.name, 'tmp']) + ), dtype=param.dtype, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) self.params_grads.append((param, grad)) for param, grad in self.params_grads: if grad is None: continue with param.block.program._optimized_guard( - [param, grad]), name_scope('move_average'): + [param, grad] + ), name_scope('move_average'): self._append_average_accumulate_op(param) self.apply_program = Program() @@ -3939,20 +4441,25 @@ class ModelAverage(Optimizer): sum_2 = block._clone_variable(self._get_accumulator('sum_2', param)) sum_3 = block._clone_variable(self._get_accumulator('sum_3', param)) num_accumulates = block._clone_variable( - self._get_accumulator('num_accumulates', param)) + self._get_accumulator('num_accumulates', param) + ) old_num_accumulates = block._clone_variable( - self._get_accumulator('old_num_accumulates', param)) + self._get_accumulator('old_num_accumulates', param) + ) num_updates = block._clone_variable( - self._get_accumulator('num_updates', param)) + self._get_accumulator('num_updates', param) + ) # backup param value to grad layers.assign(input=param, output=grad) # param = (sum_1 + sum_2 + sum_3) / (num_accumulates + old_num_accumulates) tmp = layers.sum(x=[num_accumulates, old_num_accumulates]) sum = layers.sum(x=[sum_1, sum_2, sum_3]) tmp = layers.cast( - x=tmp, dtype='float32' if self._dtype == None else self._dtype) + x=tmp, dtype='float32' if self._dtype == None else self._dtype + ) sum = layers.cast( - x=sum, dtype='float32' if self._dtype == None else self._dtype) + x=sum, dtype='float32' if self._dtype == None else self._dtype + ) ops._elementwise_div(x=sum, y=tmp, out=param) def _add_average_restore_op(self, block, param_grad): @@ -3965,44 +4472,42 @@ class ModelAverage(Optimizer): sum_1 = self._add_accumulator('sum_1', param) sum_2 = self._add_accumulator('sum_2', param) sum_3 = self._add_accumulator('sum_3', param) - num_accumulates = self._add_accumulator('num_accumulates', - param, - dtype='int64', - shape=[1]) - old_num_accumulates = self._add_accumulator('old_num_accumulates', - param, - dtype='int64', - shape=[1]) - num_updates = self._add_accumulator('num_updates', - param, - dtype='int64', - shape=[1]) - - self.helper.append_op(type='average_accumulates', - inputs={ - "param": param, - "in_sum_1": sum_1, - "in_sum_2": sum_2, - "in_sum_3": sum_3, - "in_num_accumulates": num_accumulates, - "in_old_num_accumulates": old_num_accumulates, - "in_num_updates": num_updates - }, - outputs={ - "out_sum_1": sum_1, - "out_sum_2": sum_2, - "out_sum_3": sum_3, - "out_num_accumulates": num_accumulates, - "out_old_num_accumulates": - old_num_accumulates, - "out_num_updates": num_updates, - }, - attrs={ - "average_window": self.average_window, - "min_average_window": self.min_average_window, - "max_average_window": self.max_average_window, - }, - stop_gradient=True) + num_accumulates = self._add_accumulator( + 'num_accumulates', param, dtype='int64', shape=[1] + ) + old_num_accumulates = self._add_accumulator( + 'old_num_accumulates', param, dtype='int64', shape=[1] + ) + num_updates = self._add_accumulator( + 'num_updates', param, dtype='int64', shape=[1] + ) + + self.helper.append_op( + type='average_accumulates', + inputs={ + "param": param, + "in_sum_1": sum_1, + "in_sum_2": sum_2, + "in_sum_3": sum_3, + "in_num_accumulates": num_accumulates, + "in_old_num_accumulates": old_num_accumulates, + "in_num_updates": num_updates, + }, + outputs={ + "out_sum_1": sum_1, + "out_sum_2": sum_2, + "out_sum_3": sum_3, + "out_num_accumulates": num_accumulates, + "out_old_num_accumulates": old_num_accumulates, + "out_num_updates": num_updates, + }, + attrs={ + "average_window": self.average_window, + "min_average_window": self.min_average_window, + "max_average_window": self.max_average_window, + }, + stop_gradient=True, + ) @signature_safe_contextmanager def apply(self, executor, need_restore=True): @@ -4117,7 +4622,7 @@ class ModelAverage(Optimizer): class ExponentialMovingAverage(object): r""" - :api_attr: Static Graph + :api_attr: Static Graph Compute the moving average of parameters with exponential decay. Given a parameter :math:`\\theta`, its exponential moving average (EMA) @@ -4127,7 +4632,7 @@ class ExponentialMovingAverage(object): \\text{EMA}_0 & = 0 - \\text{EMA}_t & = \\text{decay} * \\text{EMA}_{t-1} + (1 - \\text{decay}) * \\theta_t + \\text{EMA}_t & = \\text{decay} * \\text{EMA}_{t-1} + (1 - \\text{decay}) * \\theta_t The average results calculated by **update()** method will be saved in temporary variables which are created and maintained by the object, and can @@ -4215,7 +4720,8 @@ class ExponentialMovingAverage(object): def __init__(self, decay=0.999, thres_steps=None, name=None): if framework._non_static_mode(): raise Exception( - "In dygraph, don't support ExponentialMovingAverage.") + "In dygraph, don't support ExponentialMovingAverage." + ) self._decay = decay self._thres_steps = thres_steps self._name = name if name is not None else '' @@ -4225,17 +4731,21 @@ class ExponentialMovingAverage(object): self._params_tmps = [] for param in default_main_program().global_block().all_parameters(): if param.do_model_average != False: - tmp = param.block.create_var(name=unique_name.generate(".".join( - [self._name + param.name, 'ema_tmp'])), - dtype=param.dtype, - persistable=False, - stop_gradient=True) + tmp = param.block.create_var( + name=unique_name.generate( + ".".join([self._name + param.name, 'ema_tmp']) + ), + dtype=param.dtype, + persistable=False, + stop_gradient=True, + ) self._params_tmps.append((param, tmp)) self._ema_vars = {} for param, tmp in self._params_tmps: - with param.block.program._optimized_guard( - [param, tmp]), name_scope('moving_average'): + with param.block.program._optimized_guard([param, tmp]), name_scope( + 'moving_average' + ): self._ema_vars[param.name] = self._create_ema_vars(param) self.apply_program = Program() @@ -4250,8 +4760,9 @@ class ExponentialMovingAverage(object): # bias correction with layers.control_flow.Switch() as switch: with switch.case(global_step > 0): - layers.assign(output=param, - input=ema / (1.0 - decay_pow)) + layers.assign( + output=param, input=ema / (1.0 - decay_pow) + ) with switch.default(): layers.assign(output=param, input=ema) @@ -4270,7 +4781,8 @@ class ExponentialMovingAverage(object): value=self._decay, dtype='float32', persistable=True, - name="scheduled_ema_decay_rate") + name="scheduled_ema_decay_rate", + ) if self._thres_steps is not None: decay_t = (self._thres_steps + 1.0) / (self._thres_steps + 10.0) @@ -4279,16 +4791,18 @@ class ExponentialMovingAverage(object): layers.tensor.assign(decay_t, decay_var) with switch.default(): layers.tensor.assign( - np.array([self._decay], dtype=np.float32), - decay_var) + np.array([self._decay], dtype=np.float32), decay_var + ) return decay_var def _get_decay_pow(self, block): - global_step = layers.create_global_var(name=self._step_counter_name, - shape=[1], - value=0, - dtype='int64', - persistable=True) + global_step = layers.create_global_var( + name=self._step_counter_name, + shape=[1], + value=0, + dtype='int64', + persistable=True, + ) global_step = layers.cast(global_step, "float32") decay_var = block._clone_variable(self._decay_var) decay_pow_acc = layers.elementwise_pow(decay_var, global_step) @@ -4300,7 +4814,8 @@ class ExponentialMovingAverage(object): shape=param.shape, value=0.0, dtype=param.dtype, - persistable=True) + persistable=True, + ) return param_ema @@ -4310,18 +4825,21 @@ class ExponentialMovingAverage(object): train program. """ global_step = layers.autoincreased_step_counter( - counter_name=self._step_counter_name) + counter_name=self._step_counter_name + ) param_master_emas = [] for param, tmp in self._params_tmps: - with param.block.program._optimized_guard( - [param, tmp]), name_scope('moving_average'): + with param.block.program._optimized_guard([param, tmp]), name_scope( + 'moving_average' + ): param_ema = self._ema_vars[param.name] if param.name + '.master' in self._ema_vars: master_ema = self._ema_vars[param.name + '.master'] param_master_emas.append([param_ema, master_ema]) else: ema_t = param_ema * self._decay_var + param * ( - 1 - self._decay_var) + 1 - self._decay_var + ) layers.assign(input=ema_t, output=param_ema) # for fp16 params @@ -4332,8 +4850,9 @@ class ExponentialMovingAverage(object): outputs={"Out": param_ema}, attrs={ "in_dtype": master_ema.dtype, - "out_dtype": param_ema.dtype - }) + "out_dtype": param_ema.dtype, + }, + ) @signature_safe_contextmanager def apply(self, executor, need_restore=True): @@ -4363,7 +4882,7 @@ class ExponentialMovingAverage(object): class PipelineOptimizer(object): """ - :api_attr: Static Graph + :api_attr: Static Graph Pipeline Optimizer: Make a program to run as pipeline, that is splitting a program into multiple sections (sub-programs) and each section run on a @@ -4426,14 +4945,19 @@ class PipelineOptimizer(object): self._device = "gpu" if framework._non_static_mode(): raise Exception("In dygraph, don't support PipelineOptimizer.") - valid_optimizers = (Optimizer, paddle.optimizer.Optimizer, - paddle.fluid.contrib.mixed_precision.decorator. - OptimizerWithMixedPrecision) + valid_optimizers = ( + Optimizer, + paddle.optimizer.Optimizer, + paddle.fluid.contrib.mixed_precision.decorator.OptimizerWithMixedPrecision, + ) if not isinstance(optimizer, valid_optimizers): - raise ValueError("The 'optimizer' parameter for " - "PipelineOptimizer must be an instance of " - "{}, but the given type is {}.".format( - valid_optimizers, type(optimizer))) + raise ValueError( + "The 'optimizer' parameter for " + "PipelineOptimizer must be an instance of " + "{}, but the given type is {}.".format( + valid_optimizers, type(optimizer) + ) + ) self._optimizer = optimizer # Get the original optimizer defined by users, such as SGD @@ -4441,11 +4965,13 @@ class PipelineOptimizer(object): while hasattr(self._origin_optimizer, "inner_opt"): self._origin_optimizer = self._origin_optimizer.inner_opt - assert num_microbatches >= 1, ( - "num_microbatches must be a positive value.") + assert ( + num_microbatches >= 1 + ), "num_microbatches must be a positive value." self._num_microbatches = num_microbatches - assert start_cpu_core_id >= 0, ( - "start_cpu_core_id must be a non-negative integer.") + assert ( + start_cpu_core_id >= 0 + ), "start_cpu_core_id must be a non-negative integer." self._start_cpu_core_id = start_cpu_core_id self._place_list = None op_maker = core.op_proto_and_checker_maker @@ -4473,41 +4999,47 @@ class PipelineOptimizer(object): if op.type == "reduce_any": # cast the bool var to int32 to use allreduce_max op temp_var_name = unique_name.generate(out_name + "_cast_int32") - temp_var = block.create_var(name=temp_var_name, - shape=[1], - dtype="int32") - block._insert_op(op_idx + 1 + offset, - type='cast', - inputs={'X': out_var}, - outputs={'Out': temp_var}, - attrs={ - 'in_dtype': out_var.dtype, - 'out_dtype': temp_var.dtype, - self._op_role_key: self._op_role.Optimize - }) + temp_var = block.create_var( + name=temp_var_name, shape=[1], dtype="int32" + ) + block._insert_op( + op_idx + 1 + offset, + type='cast', + inputs={'X': out_var}, + outputs={'Out': temp_var}, + attrs={ + 'in_dtype': out_var.dtype, + 'out_dtype': temp_var.dtype, + self._op_role_key: self._op_role.Optimize, + }, + ) offset += 1 block._insert_op( op_idx + 1 + offset, type='c_allreduce_max' - if op.type == "reduce_any" else 'c_allreduce_sum', + if op.type == "reduce_any" + else 'c_allreduce_sum', inputs={'X': temp_var if op.type == "reduce_any" else out_var}, outputs={'Out': temp_var if op.type == "reduce_any" else out_var}, attrs={ 'ring_id': self.global_ring_id, self._op_role_key: self._op_role.Optimize, - 'use_calc_stream': True - }) + 'use_calc_stream': True, + }, + ) offset += 1 if op.type == "reduce_any": - block._insert_op(op_idx + 1 + offset, - type='cast', - inputs={'X': temp_var}, - outputs={'Out': out_var}, - attrs={ - 'in_dtype': temp_var.dtype, - 'out_dtype': out_var.dtype, - self._op_role_key: self._op_role.Optimize - }) + block._insert_op( + op_idx + 1 + offset, + type='cast', + inputs={'X': temp_var}, + outputs={'Out': out_var}, + attrs={ + 'in_dtype': temp_var.dtype, + 'out_dtype': out_var.dtype, + self._op_role_key: self._op_role.Optimize, + }, + ) offset += 1 return offset @@ -4563,13 +5095,15 @@ class PipelineOptimizer(object): if var in used_var_set or "_blocking_queue" in var: continue used_var_set.add(var) - if block._find_var_recursive(str(var)): continue + if block._find_var_recursive(str(var)): + continue source_var = ori_block._var_recursive(str(var)) if source_var.type == core.VarDesc.VarType.READER: dest_var = block.create_var( name=var, type=core.VarDesc.VarType.READER, - persistable=source_var.persistable) + persistable=source_var.persistable, + ) elif isinstance(source_var, Parameter): dest_var = block.create_parameter( name=source_var.name, @@ -4581,14 +5115,16 @@ class PipelineOptimizer(object): trainable=source_var.trainable, optimize_attr=source_var.optimize_attr, regularizer=source_var.regularizer, - error_clip=source_var.error_clip) + error_clip=source_var.error_clip, + ) else: dest_var = block._clone_variable(source_var, False) self._clone_var_attr(dest_var, source_var) # When use with sharding, allreduce_sum and allreduce_max # used for global gradient clip and amp will be added by sharding. op_idx += 1 - if self.use_sharding or not should_insert: continue + if self.use_sharding or not should_insert: + continue inserted_ops = self._insert_allreduce_op(op_idx - 1, block) added_op_num += inserted_ops op_idx += inserted_ops @@ -4598,15 +5134,18 @@ class PipelineOptimizer(object): assert self._op_role_key in op.attr_names op_role = int(op.attr(self._op_role_key)) return op_role & int(self._op_role.Backward) and op_role & int( - self._op_role.Loss) + self._op_role.Loss + ) def _is_forward_op(self, op): - return self._op_role_key in op.attr_names and (int( - op.attr(self._op_role_key)) == int(self._op_role.Forward)) + return self._op_role_key in op.attr_names and ( + int(op.attr(self._op_role_key)) == int(self._op_role.Forward) + ) def _is_backward_op(self, op): return self._op_role_key in op.attr_names and ( - int(op.attr(self._op_role_key)) & int(self._op_role.Backward)) + int(op.attr(self._op_role_key)) & int(self._op_role.Backward) + ) def _is_loss_op(self, op): assert self._op_role_key in op.attr_names @@ -4614,11 +5153,15 @@ class PipelineOptimizer(object): def _is_optimize_op(self, op): return self._op_role_key in op.attr_names and ( - int(op.attr(self._op_role_key)) & int(self._op_role.Optimize)) + int(op.attr(self._op_role_key)) & int(self._op_role.Optimize) + ) def _is_update_op(self, op): - return 'Param' in op.input_names and 'Grad' in op.input_names and ( - "LearningRate" in op.input_names) + return ( + 'Param' in op.input_names + and 'Grad' in op.input_names + and ("LearningRate" in op.input_names) + ) def _split_program(self, main_program, devices): """ @@ -4665,10 +5208,11 @@ class PipelineOptimizer(object): get the real op_device attribute of the fill_constant as the device where the corresponding parameters on. """ - assert "beta1_pow_acc" in var_name or "beta2_pow_acc" in var_name, \ - 'For accumulators for Adam, the name must contain beta1_pow_acc ' \ + assert "beta1_pow_acc" in var_name or "beta2_pow_acc" in var_name, ( + 'For accumulators for Adam, the name must contain beta1_pow_acc ' 'or beta2_pow_acc.' - param_name = var_name[0:var_name.index('_beta')] + ) + param_name = var_name[0 : var_name.index('_beta')] device = self._param_device_map[param_name] return device @@ -4680,7 +5224,8 @@ class PipelineOptimizer(object): if device == "cpu": assert op.type == "fill_constant", ( "For ops in startup program with the op_device attribute " - "of cpu, they must be of type fill_constant.") + "of cpu, they must be of type fill_constant." + ) output_var = op.output_arg_names[0] device = self._get_op_device_for_startup_program(output_var) @@ -4689,7 +5234,8 @@ class PipelineOptimizer(object): else: # LR related ops device = None - if device and device_index != device_id: continue + if device and device_index != device_id: + continue op_desc = op.desc ap_op = new_startup_program.global_block().desc.append_op() ap_op.copy_from(op_desc) @@ -4709,7 +5255,8 @@ class PipelineOptimizer(object): var_name = var_name.replace('.cast_fp16', '') post_ops = self.input_var_to_op[var_name] - if post_ops == None: return None + if post_ops == None: + return None result_op = None for post_op, post_idx in reversed(post_ops): if post_idx > index: @@ -4723,7 +5270,8 @@ class PipelineOptimizer(object): variable named var_name. """ prev_ops = self.output_var_to_op[var_name] - if prev_ops == None: return None + if prev_ops == None: + return None result_op = None for prev_op, prev_idx in reversed(prev_ops): if prev_idx < index: @@ -4749,7 +5297,8 @@ class PipelineOptimizer(object): lod_level=ref_var.lod_level, persistable=ref_var.persistable, is_data=ref_var.is_data, - need_check_feed=ref_var.desc.need_check_feed()) + need_check_feed=ref_var.desc.need_check_feed(), + ) self._clone_var_attr(new_var, ref_var) return new_var @@ -4775,11 +5324,16 @@ class PipelineOptimizer(object): """ Get the op_device attribute of a op. """ - device = op.attr(self._op_device_key) \ - if op.has_attr(self._op_device_key) else None + device = ( + op.attr(self._op_device_key) + if op.has_attr(self._op_device_key) + else None + ) if device: - assert device[0:3] == 'gpu' or device[0:3] == 'npu', "Now, only gpu and npu devices are " \ + assert device[0:3] == 'gpu' or device[0:3] == 'npu', ( + "Now, only gpu and npu devices are " "supported in pipeline parallemism." + ) return device def _add_op_device_attr_for_op(self, op, idx, block): @@ -4798,40 +5352,48 @@ class PipelineOptimizer(object): elif op.type == "sum" and self._is_backward_op(op): # For sum ops that compute the sum of @RENAMED@ vars for name in op.desc.input_arg_names(): - assert '@RENAME@' in name, \ - "The op must be sum used to accumulate renamed vars." + assert ( + '@RENAME@' in name + ), "The op must be sum used to accumulate renamed vars." assert len(op.desc.output_arg_names()) == 1 out_name = op.desc.output_arg_names()[0] post_op = self._find_post_op(idx, out_name) assert post_op.has_attr( - 'op_device'), "{} has no op_device attr for var {}".format( - post_op.type, out_name) + 'op_device' + ), "{} has no op_device attr for var {}".format( + post_op.type, out_name + ) device = post_op.attr(self._op_device_key) assert device, "The post op must have op_device set." op._set_attr(self._op_device_key, device) - elif (op.type == "cast" - or op.type == "scale") and self._is_backward_op(op): + elif (op.type == "cast" or op.type == "scale") and self._is_backward_op( + op + ): prev_op = self._find_prev_op(idx, op.desc.input("X")[0]) op._set_attr(self._op_device_key, prev_op.attr(self._op_device_key)) elif op.type == "memcpy" and not self._is_optimize_op(op): # for checkpoint offloading - assert len(op.input_arg_names) == 1 and len( - op.output_arg_names) == 1 + assert ( + len(op.input_arg_names) == 1 and len(op.output_arg_names) == 1 + ) input_name = op.input_arg_names[0] output_name = op.output_arg_names[0] if '@Fetch' in output_name: post_op = self._find_post_op(idx, output_name) - op._set_attr(self._op_device_key, - post_op.attr(self._op_device_key)) + op._set_attr( + self._op_device_key, post_op.attr(self._op_device_key) + ) else: prev_op = self._find_prev_op(idx, op.desc.input("X")[0]) - op._set_attr(self._op_device_key, - prev_op.attr(self._op_device_key)) + op._set_attr( + self._op_device_key, prev_op.attr(self._op_device_key) + ) elif self._is_loss_op(op): # For loss * loss_scaling op added by AMP offset = 1 - while (not block.ops[idx + offset].has_attr(self._op_device_key) - or not block.ops[idx + offset].attr(self._op_device_key)): + while not block.ops[idx + offset].has_attr( + self._op_device_key + ) or not block.ops[idx + offset].attr(self._op_device_key): offset += 1 device = block.ops[idx + offset].attr(self._op_device_key) assert device, "Please put you program within device_guard scope." @@ -4847,19 +5409,26 @@ class PipelineOptimizer(object): elif self._is_gradient_clip_op(op) or self._is_regularization_op(op): # For gradient clip and regularization ops, we set their op_device # attribute to the device where their corresponding parameters on. - assert self._op_role_var_key in op.attr_names, "gradient_clip " \ + assert self._op_role_var_key in op.attr_names, ( + "gradient_clip " "and regularization ops must have op_role_var attribute." + ) op_role_var = op.attr(self._op_role_var_key) - assert len(op_role_var) == 2, "op_role_var for gradient_clip " \ + assert len(op_role_var) == 2, ( + "op_role_var for gradient_clip " "regularization ops must have two elements." + ) param_name = op_role_var[0] device = self._param_device_map[param_name] # For sum op added by global gradient clip, it must be # put on all devices - if (op.type == 'sum' or op.type == 'sqrt' - or op.type == 'fill_constant' - or op.type == 'elementwise_max' - or op.type == 'elementwise_div'): + if ( + op.type == 'sum' + or op.type == 'sqrt' + or op.type == 'fill_constant' + or op.type == 'elementwise_max' + or op.type == 'elementwise_div' + ): device = f"{self._device}:all" op._set_attr(self._op_device_key, device) elif op.type == "alloc_float_status" or op.type == "clear_float_status": @@ -4876,12 +5445,18 @@ class PipelineOptimizer(object): float_status_var.persistable = True else: other_known_ops = [ - 'update_loss_scaling', 'reduce_any', 'concat', 'sum', - 'check_finite_and_unscale', 'memcpy' + 'update_loss_scaling', + 'reduce_any', + 'concat', + 'sum', + 'check_finite_and_unscale', + 'memcpy', ] - assert op.type in other_known_ops, "For other ops without " \ - "op_device set, they must be one of {}, but it " \ + assert op.type in other_known_ops, ( + "For other ops without " + "op_device set, they must be one of {}, but it " "is {}".format(other_known_ops, op.type) + ) assert self._is_optimize_op(op) op._set_attr(self._op_device_key, f"{self._device}:all") @@ -4891,8 +5466,11 @@ class PipelineOptimizer(object): not that attribute set. """ for idx, op in enumerate(list(block.ops)): - if (op.type == "create_py_reader" or op.type == "read" - or op.type == "create_double_buffer_reader"): + if ( + op.type == "create_py_reader" + or op.type == "read" + or op.type == "create_double_buffer_reader" + ): # Copy read related ops to all section to make them exit # after each epoch. # We use "gpu:all" to represent the op should be put on all @@ -4901,7 +5479,8 @@ class PipelineOptimizer(object): op._set_attr(self._op_device_key, f"{self._device}:all") continue # op_device attribute has been set - if self._get_op_device_attr(op): continue + if self._get_op_device_attr(op): + continue self._add_op_device_attr_for_op(op, idx, block) def _check_validation(self, block): @@ -4922,33 +5501,42 @@ class PipelineOptimizer(object): ] for op in block.ops: if not op._has_kernel(op.type): - assert op.type == "conditional_block" and (op.attr( - self._op_role_key) == int(self._op_role.LRSched)), ( - "Now, the only supported op without kernel is " - "conditional_block, and its op role must be LRSched.") + assert op.type == "conditional_block" and ( + op.attr(self._op_role_key) == int(self._op_role.LRSched) + ), ( + "Now, the only supported op without kernel is " + "conditional_block, and its op role must be LRSched." + ) assert op.has_attr( - self._op_role_key), ("op ({}) has no {} attribute.".format( - op.type, self._op_role_key)) + self._op_role_key + ), "op ({}) has no {} attribute.".format(op.type, self._op_role_key) op_role = op.attr(self._op_role_key) - assert int(op_role) in valid_op_role_value, \ - "op_role {} for op {} must be one of {}".format( - op_role, - op.type, - valid_op_role_value) + assert ( + int(op_role) in valid_op_role_value + ), "op_role {} for op {} must be one of {}".format( + op_role, op.type, valid_op_role_value + ) assert op.has_attr( - self._op_device_key), ("op ({}) has no {} attribute.".format( - op.type, self._op_device_key)) + self._op_device_key + ), "op ({}) has no {} attribute.".format( + op.type, self._op_device_key + ) device = op.attr(self._op_device_key) - assert device, ("op_device attribute for op " - "{} has not been set.".format(op.type)) - if device == f"{self._device}:all": continue + assert ( + device + ), "op_device attribute for op " "{} has not been set.".format( + op.type + ) + if device == f"{self._device}:all": + continue dev_type = device.split(':')[0] assert dev_type == "gpu" or dev_type == 'npu', ( "Now only gpu and npu devices are supported " - "for pipeline parallelism.") + "for pipeline parallelism." + ) if device not in device_list: device_list.append(device) @@ -4971,16 +5559,18 @@ class PipelineOptimizer(object): break extra_index_info = { 'index': 0, - 'first_optimize_index': first_optimize_index + 'first_optimize_index': first_optimize_index, } for index, op in enumerate(list(block.ops)): cur_device = op.attr(self._op_device_key) - if cur_device == f"{self._device}:all": continue + if cur_device == f"{self._device}:all": + continue for var_name in op.input_arg_names: var = block.var(var_name) # skip data var - if var.is_data: continue + if var.is_data: + continue prev_device = None prev_op = self._find_prev_op(index, var_name) @@ -4990,13 +5580,15 @@ class PipelineOptimizer(object): prev_device = self._param_device_map[var_name] if not prev_device: - prev_device = prev_op.attr(self._op_device_key) \ - if prev_op else None + prev_device = ( + prev_op.attr(self._op_device_key) if prev_op else None + ) if prev_device is None or prev_device == f"{self._device}:all": continue - if prev_device == cur_device: continue + if prev_device == cur_device: + continue if var_name not in input_var_to_device: input_var_to_device[var_name] = [] @@ -5009,20 +5601,25 @@ class PipelineOptimizer(object): # check send/recv stage valid is_forward = self._is_forward_op(op) is_backward = self._is_backward_op(op) - assert is_forward or is_backward, \ - 'send/recv in pipeline should only be inserted in forward or backward,' \ + assert is_forward or is_backward, ( + 'send/recv in pipeline should only be inserted in forward or backward,' 'please check the op_role of op={}'.format(op) + ) if is_forward: - assert prev_id < cur_id, \ - "In forward, send/recv can only be passed forward, but now " \ + assert prev_id < cur_id, ( + "In forward, send/recv can only be passed forward, but now " "prev_stage={} great than cur_stage={}, please check op_device of op={}".format( - prev_id, cur_id, op) + prev_id, cur_id, op + ) + ) elif is_backward: - assert prev_id > cur_id, \ - "In backward, send/recv can only be passed backward, but now " \ + assert prev_id > cur_id, ( + "In backward, send/recv can only be passed backward, but now " "prev_stage={} less than cur_stage={}, please check op_device of op={}".format( - prev_id, cur_id, op) + prev_id, cur_id, op + ) + ) def _insert_send_recv(cur_id, prev_id): cur_dev = device_type + str(cur_id) @@ -5034,13 +5631,15 @@ class PipelineOptimizer(object): _insert_send_recv(cur_id - 1, prev_id) _insert_send_recv(cur_id, cur_id - 1) input_var_to_device[var_name].append( - (cur_dev, prev_dev)) + (cur_dev, prev_dev) + ) return elif cur_id - prev_id < -1: _insert_send_recv(cur_id + 1, prev_id) _insert_send_recv(cur_id, cur_id + 1) input_var_to_device[var_name].append( - (cur_dev, prev_dev)) + (cur_dev, prev_dev) + ) return assert abs(cur_id - prev_id) == 1 @@ -5069,12 +5668,16 @@ class PipelineOptimizer(object): self._op_role_key: op_role, 'use_calc_stream': True, 'peer': 1, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) extra_index_info['index'] += 1 var_shape = list(var.shape) - var_shape[0] = self.micro_batch_size if var_shape[ - 0] < 0 else var_shape[0] + var_shape[0] = ( + self.micro_batch_size + if var_shape[0] < 0 + else var_shape[0] + ) block._insert_op_without_sync( index=index + extra_index_info['index'], type='recv_v2', @@ -5086,17 +5689,22 @@ class PipelineOptimizer(object): self._op_role_key: op_role, 'use_calc_stream': True, 'peer': 0, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) extra_index_info['index'] += 1 elif self.schedule_mode == '1F1B': # 1F1B var_shape = list(var.shape) - var_shape[0] = self.micro_batch_size if var_shape[ - 0] < 0 else var_shape[0] + var_shape[0] = ( + self.micro_batch_size + if var_shape[0] < 0 + else var_shape[0] + ) numel = np.prod(var_shape) - use_mp = (self.mp_degree > 1) and (numel % - self.mp_degree == 0) + use_mp = (self.mp_degree > 1) and ( + numel % self.mp_degree == 0 + ) if 'subprog' in var.name: # For recompute, if the checkpoints var is layer_norm_6.tmp_2 @@ -5119,32 +5727,34 @@ class PipelineOptimizer(object): self._op_device_key: cur_dev, self._op_role_key: op_role, 'use_calc_stream': True, - }) + }, + ) extra_index_info['index'] += 1 return _check_stage(cur_id, prev_id) - block._insert_op_without_sync(index=index + - extra_index_info['index'], - type='c_sync_calc_stream', - inputs={'X': [var]}, - outputs={'Out': [var]}, - attrs={ - self._op_device_key: - prev_dev, - self._op_role_key: - op_role, - }) + block._insert_op_without_sync( + index=index + extra_index_info['index'], + type='c_sync_calc_stream', + inputs={'X': [var]}, + outputs={'Out': [var]}, + attrs={ + self._op_device_key: prev_dev, + self._op_role_key: op_role, + }, + ) extra_index_info['index'] += 1 prefix_name = var.name.split('@')[0] prefix_var = block.var(prefix_name) - is_param = True if isinstance(prefix_var, - Parameter) else False + is_param = ( + True if isinstance(prefix_var, Parameter) else False + ) block._insert_op_without_sync( index=index + extra_index_info['index'], type='send_v2' - if not use_mp or is_param else 'partial_send', + if not use_mp or is_param + else 'partial_send', inputs={'X': var}, attrs={ self._op_device_key: prev_dev, @@ -5155,12 +5765,14 @@ class PipelineOptimizer(object): # if send_v2, num&id attr is not in op_attrs, will not insert 'num': self.mp_degree, 'id': self.mp_rank, - }) + }, + ) extra_index_info['index'] += 1 insert_index = None if int(op_role) == int(self._op_role.Backward): insert_index = extra_index_info[ - 'first_optimize_index'] + 'first_optimize_index' + ] new_op_role = self._op_role.Optimize else: insert_index = index @@ -5174,14 +5786,16 @@ class PipelineOptimizer(object): self._op_device_key: prev_dev, self._op_role_key: new_op_role, 'ring_id': ring_id, - }) + }, + ) if int(op_role) == int(self._op_role.Forward): sync_comm_op._set_attr('pipeline_flag', '') extra_index_info['index'] += 1 block._insert_op_without_sync( index=index + extra_index_info['index'], type='recv_v2' - if not use_mp or is_param else 'partial_recv', + if not use_mp or is_param + else 'partial_recv', outputs={'Out': [var]}, attrs={ 'out_shape': var_shape, @@ -5194,7 +5808,8 @@ class PipelineOptimizer(object): # if recv_v2, num&id attr is not in op_attrs, will not insert 'num': self.mp_degree, 'id': self.mp_rank, - }) + }, + ) extra_index_info['index'] += 1 if use_mp and not is_param: block._insert_op_without_sync( @@ -5210,27 +5825,33 @@ class PipelineOptimizer(object): # if recv_v2, num&id attr is not in op_attrs, will not insert 'nranks': self.mp_degree, 'rank': self.mp_rank, - }) + }, + ) extra_index_info['index'] += 1 else: raise ValueError( "Now only 'F-then-B' and '1F1B' are supported." - "The given value is {}.".format(self.schedule_mode)) + "The given value is {}.".format(self.schedule_mode) + ) - _insert_send_recv(int(cur_device.split(':')[1]), - int(prev_device.split(':')[1])) + _insert_send_recv( + int(cur_device.split(':')[1]), + int(prev_device.split(':')[1]), + ) block._sync_with_cpp() def _insert_loss_scale(self, block): """ Scale the loss corresponding to number of micro-batches. """ - if self._num_microbatches == 1: return + if self._num_microbatches == 1: + return for index, op in reversed(tuple(enumerate(list(block.ops)))): if self._is_loss_grad_op(op): - assert op.type == 'fill_constant', \ - "loss_grad_op must be fill_constant op, " \ + assert op.type == 'fill_constant', ( + "loss_grad_op must be fill_constant op, " "but this op is {}".format(op.type) + ) assert op.has_attr('value') loss_scale = float(op.attr('value')) loss_scale = loss_scale / self._num_microbatches @@ -5239,24 +5860,25 @@ class PipelineOptimizer(object): def _rename_gradient_var_name(self, block): for index, op in enumerate(block.ops): - if not self._is_optimize_op(op): continue + if not self._is_optimize_op(op): + continue input_names = op.input_arg_names output_names = op.output_arg_names in_out_names = input_names + output_names - if op.type == 'cast' or op.type == "c_sync_comm_stream": continue + if op.type == 'cast' or op.type == "c_sync_comm_stream": + continue # append "MERGED" to the names of parameter gradients, # and mofify the op_role_var attribute (by rename_arg func). for name in in_out_names: - if not core.grad_var_suffix() in name: continue + if not core.grad_var_suffix() in name: + continue param_name = name.strip(core.grad_var_suffix()) new_grad_name = name + "@MERGED" self._rename_arg(op, name, new_grad_name) - def _accumulate_gradients(self, - block, - pp_allreduce_in_optimize=False, - strategy=None, - shard=None): + def _accumulate_gradients( + self, block, pp_allreduce_in_optimize=False, strategy=None, shard=None + ): """ Create a new merged gradient for each parameter and accumulate the corresponding gradient to it. @@ -5264,7 +5886,8 @@ class PipelineOptimizer(object): fp16_allreduce = strategy.fp16_allreduce if strategy else False if strategy and strategy.fuse_grad_merge: fused_gradient_names = self._accumulate_gradients_with_fuse( - block, fp16_allreduce, strategy.fuse_grad_size_in_MB, shard) + block, fp16_allreduce, strategy.fuse_grad_size_in_MB, shard + ) return fused_gradient_names merged_gradient_names = [] @@ -5288,22 +5911,30 @@ class PipelineOptimizer(object): # maybe have no optimize # if first_opt_op_idx == len(block.ops): return - if self._is_backward_op(op) and (self._op_role_var_key - in op.attr_names): + if self._is_backward_op(op) and ( + self._op_role_var_key in op.attr_names + ): op_role_var = op.attr(self._op_role_var_key) - if len(op_role_var) == 0: continue + if len(op_role_var) == 0: + continue assert len(op_role_var) % 2 == 0 for i in range(0, len(op_role_var), 2): offset = 0 param_name = op_role_var[i] - if not block.has_var(param_name): continue - if '@BroadCast' in param_name: continue + if not block.has_var(param_name): + continue + if '@BroadCast' in param_name: + continue param_grad_name = param_name + core.grad_var_suffix() merged_param_grad_name = param_grad_name + merged_suffix if not block.has_var(merged_param_grad_name): - self._create_var(block, block.vars[param_name], - merged_param_grad_name, dtype) + self._create_var( + block, + block.vars[param_name], + merged_param_grad_name, + dtype, + ) assert block.has_var(merged_param_grad_name) param_grad_var = block.var(param_grad_name) @@ -5315,22 +5946,19 @@ class PipelineOptimizer(object): inputs={}, outputs={'Out': [merged_param_grad_var]}, attrs={ - 'shape': - merged_param_grad_var.shape, - 'dtype': - merged_param_grad_var.dtype, - 'value': - float(0), + 'shape': merged_param_grad_var.shape, + 'dtype': merged_param_grad_var.dtype, + 'value': float(0), # a trick to run this op once per mini-batch - self._op_role_key: - self._op_role.Optimize.LRSched, - }) + self._op_role_key: self._op_role.Optimize.LRSched, + }, + ) offset += 1 grad_name = op_role_var[i + 1] grad_var = block.vars[grad_name] is_fp16_grad = 'cast_fp16' in grad_name - need_cast = (is_fp16_grad is not fp16_allreduce) + need_cast = is_fp16_grad is not fp16_allreduce if need_cast: # if fp16_allreduce: @@ -5339,20 +5967,20 @@ class PipelineOptimizer(object): # cast grad to fp32 to accumulate to merged gradient cast_grad_var_name = param_grad_name + '@TMP' cast_grad_var = self._create_var( - block, param_grad_var, cast_grad_var_name, dtype) + block, param_grad_var, cast_grad_var_name, dtype + ) cast_grad_var.persistable = False - block._insert_op(index=first_opt_op_idx + offset, - type='cast', - inputs={'X': grad_var}, - outputs={'Out': cast_grad_var}, - attrs={ - 'in_dtype': - grad_var.dtype, - 'out_dtype': - cast_grad_var.dtype, - self._op_role_key: - self._op_role.Backward, - }) + block._insert_op( + index=first_opt_op_idx + offset, + type='cast', + inputs={'X': grad_var}, + outputs={'Out': cast_grad_var}, + attrs={ + 'in_dtype': grad_var.dtype, + 'out_dtype': cast_grad_var.dtype, + self._op_role_key: self._op_role.Backward, + }, + ) offset += 1 grad_var = cast_grad_var @@ -5363,11 +5991,13 @@ class PipelineOptimizer(object): outputs={'Out': merged_param_grad_var}, attrs={ self._op_role_key: self._op_role.Backward, - }) + }, + ) offset += 1 merged_gradient_names.append(merged_param_grad_name) - if not fp16_allreduce: return merged_gradient_names + if not fp16_allreduce: + return merged_gradient_names first_opt_op_idx = None for index, op in reversed(tuple(enumerate(list(block.ops)))): @@ -5391,28 +6021,31 @@ class PipelineOptimizer(object): grad_var = block.var(grad_name) grad_var.persistable = False - block._insert_op(index=first_opt_op_idx, - type='cast', - inputs={'X': fp16_grad_var}, - outputs={'Out': grad_var}, - attrs={ - 'in_dtype': fp16_grad_var.dtype, - 'out_dtype': grad_var.dtype, - self._op_role_key: self._op_role.Optimize, - }) + block._insert_op( + index=first_opt_op_idx, + type='cast', + inputs={'X': fp16_grad_var}, + outputs={'Out': grad_var}, + attrs={ + 'in_dtype': fp16_grad_var.dtype, + 'out_dtype': grad_var.dtype, + self._op_role_key: self._op_role.Optimize, + }, + ) return merged_gradient_names - def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, - fused_size, grad_param_pairs, - first_opt_op_idx): + def _insert_accumulate_gradients_with_fuse( + self, main_block, fp16, fused_size, grad_param_pairs, first_opt_op_idx + ): grad_param_pairs = self._sort_grad_param_by_dtype( - main_block, grad_param_pairs) + main_block, grad_param_pairs + ) grad_param_segments = [] merged_suffix = '@MERGED@FP16' if fp16 else '@MERGED' dtype = paddle.float16 if fp16 else paddle.float32 - cur_size = 0. + cur_size = 0.0 last_dtype = None # split the grad based on dtype and fused size for grad, param in grad_param_pairs: @@ -5423,7 +6056,8 @@ class PipelineOptimizer(object): dtype=dtype, shape=real_grad.shape, persistable=True, - stop_gradient=False) + stop_gradient=False, + ) real_param = main_block.var(param) if hasattr(real_param, 'is_distributed'): merged_grad_var.is_distributed = real_param.is_distributed @@ -5431,13 +6065,16 @@ class PipelineOptimizer(object): # two strategies for splitting the grad # 1. the current segment's size reach the user defined grad_size_in_MB # 2. the upcoming grad holds different dtype compared with grads in current segment - if len(grad_param_segments) == 0 \ - or cur_size + tmp_size > fused_size \ - or real_grad.dtype != last_dtype: + if ( + len(grad_param_segments) == 0 + or cur_size + tmp_size > fused_size + or real_grad.dtype != last_dtype + ): grad_param_segments.append( - ([real_grad], [real_param], [merged_grad_var])) + ([real_grad], [real_param], [merged_grad_var]) + ) last_dtype = real_grad.dtype - cur_size = 0. + cur_size = 0.0 else: grad_param_segments[-1][0].append(real_grad) grad_param_segments[-1][1].append(real_param) @@ -5450,21 +6087,28 @@ class PipelineOptimizer(object): for grad_param_segment in grad_param_segments: grad_segment = grad_param_segment[0] merged_grad_segment = grad_param_segment[2] - fused_grad = main_block.create_var(name='FusedGrad_{}'.format( - grad_segment[0].name), - dtype=grad_segment[0].dtype, - persistable=False, - stop_gradient=False) + fused_grad = main_block.create_var( + name='FusedGrad_{}'.format(grad_segment[0].name), + dtype=grad_segment[0].dtype, + persistable=False, + stop_gradient=False, + ) # keep the '.cast_fp16' info in the fuse var name - fused_merged_grad_name_prefix = 'FusedMergedGrad.cast_fp16.' if \ - merged_grad_segment[0].dtype == paddle.float16 else 'FusedMergedGrad' - fused_merged_grad_name = fused_merged_grad_name_prefix + '_{}'.format( - merged_grad_segment[0].name) + fused_merged_grad_name_prefix = ( + 'FusedMergedGrad.cast_fp16.' + if merged_grad_segment[0].dtype == paddle.float16 + else 'FusedMergedGrad' + ) + fused_merged_grad_name = ( + fused_merged_grad_name_prefix + + '_{}'.format(merged_grad_segment[0].name) + ) fused_merged_grad = main_block.create_var( name=fused_merged_grad_name, dtype=merged_grad_segment[0].dtype, persistable=True, - stop_gradient=False) + stop_gradient=False, + ) fused_gradients.append(fused_grad) fused_merged_gradients.append(fused_merged_grad) @@ -5490,10 +6134,7 @@ class PipelineOptimizer(object): first_back_op_idx + offset, type="coalesce_tensor", inputs={"Input": params}, - outputs={ - "Output": grads, - "FusedOutput": fused_grad - }, + outputs={"Output": grads, "FusedOutput": fused_grad}, attrs={ # Explanation of user_defined_size_of_dtype: # In coalesce op, the align size is 256 bytes @@ -5517,7 +6158,8 @@ class PipelineOptimizer(object): # To avoid these problematic triggers, set constant is needed for npu "set_constant": core.is_compiled_with_npu(), "constant": float(0.0), - }) + }, + ) offset += 1 # For the gradient_merged_fused_var, given a init value during the coalesce op # this will remove a problematic fill_constant op. This op role of this coalesce @@ -5528,7 +6170,7 @@ class PipelineOptimizer(object): inputs={"Input": params}, outputs={ "Output": merged_grads, - "FusedOutput": fused_merged_grad + "FusedOutput": fused_merged_grad, }, attrs={ "user_defined_size_of_dtype": 2, @@ -5537,8 +6179,9 @@ class PipelineOptimizer(object): "copy_data": False, "use_align": True, "dtype": merged_grads[0].dtype, - self._op_role_key: self._op_role.Optimize.LRSched - }) + self._op_role_key: self._op_role.Optimize.LRSched, + }, + ) offset += 1 # insert gradient merge relating ops @@ -5548,25 +6191,28 @@ class PipelineOptimizer(object): fused_grad = fused_gradients[i] fused_merged_grad = fused_merged_gradients[i] is_fp16_grad = 'cast_fp16' in fused_grad.name - need_cast = (is_fp16_grad is not fp16) + need_cast = is_fp16_grad is not fp16 if need_cast: # for fp16 allreduce, cast fp32 grad to fp16 # for fp32 allreduce, cast fp16 grad to fp32 cast_grad_var_name = fused_grad.name + '@TMP' - cast_grad_var = main_block.create_var(name=cast_grad_var_name, - dtype=dtype, - persistable=False, - stop_gradient=False) - main_block._insert_op(index=first_opt_op_idx + offset, - type='cast', - inputs={'X': fused_grad}, - outputs={'Out': cast_grad_var}, - attrs={ - 'in_dtype': fused_grad.dtype, - 'out_dtype': cast_grad_var.dtype, - self._op_role_key: - self._op_role.Backward, - }) + cast_grad_var = main_block.create_var( + name=cast_grad_var_name, + dtype=dtype, + persistable=False, + stop_gradient=False, + ) + main_block._insert_op( + index=first_opt_op_idx + offset, + type='cast', + inputs={'X': fused_grad}, + outputs={'Out': cast_grad_var}, + attrs={ + 'in_dtype': fused_grad.dtype, + 'out_dtype': cast_grad_var.dtype, + self._op_role_key: self._op_role.Backward, + }, + ) offset += 1 fused_grad = cast_grad_var main_block._insert_op( @@ -5574,7 +6220,8 @@ class PipelineOptimizer(object): type='sum', inputs={'X': [fused_merged_grad, fused_grad]}, outputs={'Out': fused_merged_grad}, - attrs={self._op_role_key: self._op_role.Backward}) + attrs={self._op_role_key: self._op_role.Backward}, + ) offset += 1 if fp16: @@ -5585,21 +6232,24 @@ class PipelineOptimizer(object): assert main_block.has_var(fp16_grad_name) fp16_grad = main_block.var(fp16_grad_name) fp32_grad_name = param + core.grad_var_suffix() + '@MERGED' - fp32_grad = main_block.create_var(name=fp32_grad_name, - dtype=paddle.float32, - shape=real_grad.shape, - persistable=False, - stop_gradient=False) - main_block._insert_op(index=first_opt_op_idx + offset, - type='cast', - inputs={'X': fp16_grad}, - outputs={'Out': fp32_grad}, - attrs={ - 'in_dtype': paddle.float16, - 'out_dtype': paddle.float32, - self._op_role_key: - self._op_role.Optimize, - }) + fp32_grad = main_block.create_var( + name=fp32_grad_name, + dtype=paddle.float32, + shape=real_grad.shape, + persistable=False, + stop_gradient=False, + ) + main_block._insert_op( + index=first_opt_op_idx + offset, + type='cast', + inputs={'X': fp16_grad}, + outputs={'Out': fp32_grad}, + attrs={ + 'in_dtype': paddle.float16, + 'out_dtype': paddle.float32, + self._op_role_key: self._op_role.Optimize, + }, + ) offset += 1 # replace the var with it's name, which will be used for inserting allreduce @@ -5608,11 +6258,9 @@ class PipelineOptimizer(object): return fused_merged_gradients, first_opt_op_idx - def _accumulate_gradients_with_fuse(self, - main_block, - fp16, - fused_size, - shard=None): + def _accumulate_gradients_with_fuse( + self, main_block, fp16, fused_size, shard=None + ): first_opt_op_idx = None grad_param_pairs = [] # obtain all param/grad pairs that needed to be fused @@ -5632,8 +6280,9 @@ class PipelineOptimizer(object): if first_opt_op_idx == len(main_block.ops): return - if self._is_backward_op(op) and (self._op_role_var_key - in op.attr_names): + if self._is_backward_op(op) and ( + self._op_role_var_key in op.attr_names + ): op_role_var = op.attr(self._op_role_var_key) if len(op_role_var) == 0: continue @@ -5645,7 +6294,8 @@ class PipelineOptimizer(object): if '@BroadCast' in param_name: continue grad_param_pairs.append( - (op_role_var[i + 1], op_role_var[i])) + (op_role_var[i + 1], op_role_var[i]) + ) if len(grad_param_pairs) == 0: return @@ -5659,9 +6309,12 @@ class PipelineOptimizer(object): all_fused_merged_gradients = [] for pairs in device_to_pairs: - fused_merged_gradients, first_opt_op_idx = \ - self._insert_accumulate_gradients_with_fuse( - main_block, fp16, fused_size, pairs, first_opt_op_idx) + ( + fused_merged_gradients, + first_opt_op_idx, + ) = self._insert_accumulate_gradients_with_fuse( + main_block, fp16, fused_size, pairs, first_opt_op_idx + ) all_fused_merged_gradients += fused_merged_gradients main_block._sync_with_cpp() @@ -5697,8 +6350,12 @@ class PipelineOptimizer(object): core.VarDesc.VarType.UINT8: 1, } assert -1 not in var.shape - return reduce(lambda x, y: x * y, - var.shape) * dtype_to_size[var.dtype] / 1024.0 / 1024.0 + return ( + reduce(lambda x, y: x * y, var.shape) + * dtype_to_size[var.dtype] + / 1024.0 + / 1024.0 + ) def _add_sub_blocks(self, main_block, program_list): main_program = main_block.program @@ -5719,12 +6376,14 @@ class PipelineOptimizer(object): def _get_device_info(self, block): for op in block.ops: - if not op._has_kernel(op.type): continue + if not op._has_kernel(op.type): + continue op_device = op.attr(self._op_device_key) return op_device - def _process_persistable_vars_in_multi_sections(self, main_program, - startup_prog, program_list): + def _process_persistable_vars_in_multi_sections( + self, main_program, startup_prog, program_list + ): """ Special Case: process persistable vars that exist in multiple sections, e.g., shared weight @@ -5735,9 +6394,11 @@ class PipelineOptimizer(object): for prog in program_list: block = prog.block(0) for var_name in block.vars: - if var_name == "double_buffer_0": continue + if var_name == "double_buffer_0": + continue var = block.var(var_name) - if not var.persistable: continue + if not var.persistable: + continue if not var_name in var_info: var_info[var_name] = [] if not prog in var_info[var_name]: @@ -5753,23 +6414,30 @@ class PipelineOptimizer(object): for prog in var_info[var_name]: block = prog.block(0) for op in block.ops: - if op.type == "recv_v2" or op.type == "create_py_reader" or \ - op.type == "read" or op.type == "update_loss_scaling": + if ( + op.type == "recv_v2" + or op.type == "create_py_reader" + or op.type == "read" + or op.type == "update_loss_scaling" + ): continue # We have processed lr related vars if op.attr(self._op_role_key) == int( - self._op_role.Optimize.LRSched): + self._op_role.Optimize.LRSched + ): continue if var_name in op.desc.output_arg_names(): assert var_name not in write_info, ( "two sections write the same var({}): second " - "op {}.".format(var_name, op)) + "op {}.".format(var_name, op) + ) write_info[var_name] = prog break for var_name in var_info.keys(): # Case 1: read only variables, no special process - if not var_name in write_info: continue + if not var_name in write_info: + continue # Case 2: one write multiple reads write_prog = write_info[var_name] @@ -5778,7 +6446,8 @@ class PipelineOptimizer(object): write_dev_index = int(write_device.split(':')[1]) all_progs = var_info[var_name] for prog in all_progs: - if prog == write_prog: continue + if prog == write_prog: + continue read_block = prog.block(0) read_device = self._get_device_info(read_block) read_dev_index = int(read_device.split(':')[1]) @@ -5799,69 +6468,60 @@ class PipelineOptimizer(object): 'X': write_block.var(var_name), }, attrs={ - self._op_device_key: - write_device, - 'use_calc_stream': - False, + self._op_device_key: write_device, + 'use_calc_stream': False, # A trick to make the role LRSched to avoid copy every # microbatch - self._op_role_key: - self._op_role.LRSched, - 'peer': - read_dev_index, - 'ring_id': - ring_id - }) + self._op_role_key: self._op_role.LRSched, + 'peer': read_dev_index, + 'ring_id': ring_id, + }, + ) read_block._insert_op( index=0, type='recv_v2', outputs={'Out': [read_block.var(var_name)]}, attrs={ - 'out_shape': - read_block.var(var_name).shape, - 'dtype': - read_block.var(var_name).dtype, - self._op_device_key: - read_device, - 'use_calc_stream': - False, + 'out_shape': read_block.var(var_name).shape, + 'dtype': read_block.var(var_name).dtype, + self._op_device_key: read_device, + 'use_calc_stream': False, # A trick to make the role LRSched to avoid copy every # microbatch - self._op_role_key: - self._op_role.LRSched, - 'peer': - write_dev_index, - 'ring_id': - ring_id - }) + self._op_role_key: self._op_role.LRSched, + 'peer': write_dev_index, + 'ring_id': ring_id, + }, + ) read_block._insert_op( index=1, type='c_sync_comm_stream', inputs={'X': [read_block.var(var_name)]}, outputs={'Out': [read_block.var(var_name)]}, attrs={ - self._op_device_key: - read_device, + self._op_device_key: read_device, # A trick to make the role LRSched to avoid copy every # microbatch - self._op_role_key: - self._op_role.LRSched, - 'ring_id': - ring_id - }) + self._op_role_key: self._op_role.LRSched, + 'ring_id': ring_id, + }, + ) def _is_gradient_clip_op(self, op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip") def _is_regularization_op(self, op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/regularization") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/regularization") def _is_weight_decay_op(self, op): # in AdamW namescope is /optimizer_*/weight decay/ - return op.desc.has_attr("op_namescope") \ - and 'weight decay' in op.desc.attr("op_namescope") + return op.desc.has_attr( + "op_namescope" + ) and 'weight decay' in op.desc.attr("op_namescope") def _get_input_output_info(self, block): ''' @@ -5884,7 +6544,8 @@ class PipelineOptimizer(object): """ optimize forward send's sync_comm_stream schedule """ - if self.schedule_mode != '1F1B': return + if self.schedule_mode != '1F1B': + return block = program.block(0) @@ -5896,11 +6557,13 @@ class PipelineOptimizer(object): break # last pipeline stage - if backward_recv_index is None: return + if backward_recv_index is None: + return offset = 0 for index, op in enumerate(list(block.ops)): - if index >= backward_recv_index: break + if index >= backward_recv_index: + break if op.type == 'c_sync_comm_stream' and op.has_attr('pipeline_flag'): var_name = op.input_arg_names[0] var = block.var(var_name) @@ -5917,7 +6580,8 @@ class PipelineOptimizer(object): type='nop', inputs={'X': [var]}, outputs={'Out': [var]}, - attrs={self._op_role_key: self._op_role.Backward}) + attrs={self._op_role_key: self._op_role.Backward}, + ) block._sync_with_cpp() def _mv_head_recv(self, program): @@ -5933,10 +6597,17 @@ class PipelineOptimizer(object): insert_index = None op = program.global_block().ops[i] op_role = int(op.attr(self._op_role_key)) - if op_role == int( - self._op_role.Backward) and backward_insert_index is None: + if ( + op_role == int(self._op_role.Backward) + and backward_insert_index is None + ): backward_insert_index = i - if op.type != "partial_recv" and op.type != "partial_allgather" and op.type != "nop" and op.type != "recv_v2": + if ( + op.type != "partial_recv" + and op.type != "partial_allgather" + and op.type != "nop" + and op.type != "recv_v2" + ): continue if op_role == int(self._op_role.Forward): if i == forward_insert_index: @@ -5956,11 +6627,13 @@ class PipelineOptimizer(object): op_outputs = dict() for name in op.output_names: op_outputs[name] = op.output(name) - block._insert_op_without_sync(index=insert_index, - type=op.type, - inputs=op_inputs, - outputs=op_outputs, - attrs=op.all_attrs()) + block._insert_op_without_sync( + index=insert_index, + type=op.type, + inputs=op_inputs, + outputs=op_outputs, + attrs=op.all_attrs(), + ) block._remove_op(i + 1) if op_role == int(self._op_role.Forward): forward_insert_index += 1 @@ -5995,13 +6668,12 @@ class PipelineOptimizer(object): "However, some backward op don't need this var(NoNeedBufferVars), " "there will be no error at this time.\n" "So please check these persistable vars which changed in " - "forward and used in backward:\n{}".format(used_in_backward)) + "forward and used in backward:\n{}".format(used_in_backward) + ) - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): main_block = loss.block self.origin_main_block = main_block main_program = main_block.program @@ -6021,8 +6693,9 @@ class PipelineOptimizer(object): 'mp_rank', ] for key in required_keys: - assert key in pipeline_opt, \ - 'Please use pipeline with fleet to use {}.'.format(key) + assert ( + key in pipeline_opt + ), 'Please use pipeline with fleet to use {}.'.format(key) self.local_rank = pipeline_opt['local_rank'] self.schedule_mode = pipeline_opt['schedule_mode'] self.micro_batch_size = pipeline_opt['micro_batch_size'] @@ -6036,11 +6709,14 @@ class PipelineOptimizer(object): assert 0 <= self.mp_rank < self.mp_degree optimize_ops, params_grads = self._optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set) + loss, startup_program, parameter_list, no_grad_set + ) self._param_device_map = self._origin_optimizer._param_device_map - self.output_var_to_op, self.input_var_to_op = \ - self._get_input_output_info(main_block) + ( + self.output_var_to_op, + self.input_var_to_op, + ) = self._get_input_output_info(main_block) # Step1: add default op_device attribute for ops. self._add_op_device_attr(main_block) device_list = self._check_validation(main_block) @@ -6058,7 +6734,8 @@ class PipelineOptimizer(object): sorted_device_list = sorted(device_list, key=cmp_to_key(device_cmp)) assert sorted_device_list == device_list, ( "With pipeline parallelism, you must use gpu devices one after " - "another in the order of their ids.") + "another in the order of their ids." + ) # Step2: add send and recv ops between section boundaries self._insert_sendrecv_ops_for_boundaries(main_block) @@ -6074,7 +6751,8 @@ class PipelineOptimizer(object): assert self.local_rank < len(device_list), ( "Manually specified " "pipeline stage must be less than total number of pipeline " - "stages.") + "stages." + ) else: self.local_rank %= len(device_list) # Step3.5: optimize forward send sync_comm to overlap send and recv @@ -6099,7 +6777,8 @@ class PipelineOptimizer(object): # Step6: Split startup program new_startup_program = self._split_startup_program( - startup_program, self.local_rank) + startup_program, self.local_rank + ) startup_program._pipeline_opt = { "startup_program": new_startup_program, @@ -6141,12 +6820,18 @@ class PipelineOptimizer(object): "num_microbatches": self._num_microbatches, "start_cpu_core_id": self._start_cpu_core_id, } - return optimize_ops, params_grads, program_list, self._pipeline_pair, self._pp_ring_map + return ( + optimize_ops, + params_grads, + program_list, + self._pipeline_pair, + self._pp_ring_map, + ) class RecomputeOptimizer(Optimizer): """ - :api_attr: Static Graph + :api_attr: Static Graph Recompute Optimizer Wrapper @@ -6225,8 +6910,8 @@ class RecomputeOptimizer(Optimizer): checkpoints, list ), "_checkpoints should be a list of Variable or a list of String" for ckpt in checkpoints: - assert ( - isinstance(ckpt, str) or isinstance(ckpt, Variable) + assert isinstance(ckpt, str) or isinstance( + ckpt, Variable ), "_checkpoints should be a list of Variable or a list of String" self._checkpoints = checkpoints @@ -6237,7 +6922,7 @@ class RecomputeOptimizer(Optimizer): @framework.deprecate_stat_dict def load(self, state_dict): """ - :api_attr: Static Graph + :api_attr: Static Graph load function is not supported by Recompute Optimizer for now. :return: None @@ -6273,7 +6958,8 @@ class RecomputeOptimizer(Optimizer): print(cpt.get_exception_message(e)) """ raise NotImplementedError( - "load function is not supported by Recompute Optimizer for now") + "load function is not supported by Recompute Optimizer for now" + ) def apply_gradients(self, params_grads): """ @@ -6331,14 +7017,16 @@ class RecomputeOptimizer(Optimizer): shape=self.checkpoint_shape, dtype=self._main_program.global_block().var(varname).dtype, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) fetch_var = self._main_program.global_block().create_var( name=fetched_var_name, shape=self.checkpoint_shape, dtype=self._main_program.global_block().var(varname).dtype, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) return pinned_var_name, fetched_var_name @@ -6363,21 +7051,25 @@ class RecomputeOptimizer(Optimizer): shape=self.checkpoint_shape, dtype=self._main_program.global_block().var(var.name).dtype, persistable=False, - stop_gradient=True) - block.append_op(type='fill_constant', - outputs={'Out': varname}, - attrs={ - "shape": var.shape, - "dtype": var.dtype, - "value": 0.0, - "place_type": 2, - OP_ROLE_KEY: op_role, - }) + stop_gradient=True, + ) + block.append_op( + type='fill_constant', + outputs={'Out': varname}, + attrs={ + "shape": var.shape, + "dtype": var.dtype, + "value": 0.0, + "place_type": 2, + OP_ROLE_KEY: op_role, + }, + ) return - def _insert_async_memcpy_op(self, insert_idx, src_varname, dst_varname, - op_role, dst_place_type): + def _insert_async_memcpy_op( + self, insert_idx, src_varname, dst_varname, op_role, dst_place_type + ): OP_ROLE_KEY = core.op_proto_and_checker_maker.kOpRoleAttrName() self.block._insert_op_without_sync( insert_idx, @@ -6386,22 +7078,26 @@ class RecomputeOptimizer(Optimizer): outputs={ 'Out': [self._main_program.global_block().var(dst_varname)] }, - attrs={ - "dst_place_type": int(dst_place_type), - OP_ROLE_KEY: op_role - }) + attrs={"dst_place_type": int(dst_place_type), OP_ROLE_KEY: op_role}, + ) def _insert_fetch_op(self, idx, varname): - assert varname in self.checkpoint_name2pinned_name, "Try to fetch {} from Pinned Memory, but it is NOT a checkpoint".format( - varname) + assert ( + varname in self.checkpoint_name2pinned_name + ), "Try to fetch {} from Pinned Memory, but it is NOT a checkpoint".format( + varname + ) pinned_varname = self.checkpoint_name2pinned_name[varname] fetch_varname = self.checkpoint_name2fetch_name[varname] self._insert_async_memcpy_op(idx, pinned_varname, fetch_varname, 1, 1) def _insert_offload_op(self, idx, varname): - assert varname in self.checkpoint_name2pinned_name, "Try to offload {} to Pinned Memory, but it is NOT a checkpoint".format( - varname) + assert ( + varname in self.checkpoint_name2pinned_name + ), "Try to offload {} to Pinned Memory, but it is NOT a checkpoint".format( + varname + ) pinned_varname = self.checkpoint_name2pinned_name[varname] self._insert_async_memcpy_op(idx, varname, pinned_varname, 0, 2) @@ -6410,8 +7106,9 @@ class RecomputeOptimizer(Optimizer): pass def _record_fetch_op(self, idx): - assert len(self.un_fetch_checkpoint_names - ) > 0, "Could NOT found checkpoint to fetch" + assert ( + len(self.un_fetch_checkpoint_names) > 0 + ), "Could NOT found checkpoint to fetch" checkpoint_name = self.un_fetch_checkpoint_names.pop(-1) logging.debug("Record fetch [{}]".format(checkpoint_name)) self.idx2insertions[idx] = ("fetch", checkpoint_name) @@ -6420,14 +7117,18 @@ class RecomputeOptimizer(Optimizer): def _record_offload_op(self, idx, checkpoint_name): expected_checkpoint_name = self.un_offload_checkpoint_names.pop(0) - assert checkpoint_name == expected_checkpoint_name, "expected to offload [{}] but got [{}]".format( - expected_checkpoint_name, checkpoint_name) + assert ( + checkpoint_name == expected_checkpoint_name + ), "expected to offload [{}] but got [{}]".format( + expected_checkpoint_name, checkpoint_name + ) logging.debug("Record offload [{}]".format(checkpoint_name)) self.idx2insertions[idx] = ("offload", checkpoint_name) def _record_sync_op(self, idx, checkpoint_name): - assert checkpoint_name not in self.synced_checkpoints, "Try to sync the checkpoint [{}] twice".format( - checkpoint_name) + assert ( + checkpoint_name not in self.synced_checkpoints + ), "Try to sync the checkpoint [{}] twice".format(checkpoint_name) self.synced_checkpoints.add(checkpoint_name) logging.debug("Record offload sync [{}]".format(checkpoint_name)) self.idx2insertions[idx] = ("sync", checkpoint_name) @@ -6450,14 +7151,16 @@ class RecomputeOptimizer(Optimizer): break assert self.bw_strart_op_idx < len( - self.block.ops), "Could NOT found backword op in prog" + self.block.ops + ), "Could NOT found backword op in prog" # fetch second to last checkpoint at the beginning of BW fetched_checkpoint_varname = self._record_fetch_op( - self.bw_strart_op_idx) + self.bw_strart_op_idx + ) last_last_fetch_checkpoint = None - for i, op in enumerate(self.block.ops[self.bw_strart_op_idx:]): + for i, op in enumerate(self.block.ops[self.bw_strart_op_idx :]): idx = self.bw_strart_op_idx + i input_vars = op.desc.input_arg_names() @@ -6467,28 +7170,39 @@ class RecomputeOptimizer(Optimizer): # fetch the offloade checkpoint when the first usage of its previous one if self.checkpoint_usage_count[input_var] == 0: # TODO (JZ-LIANG) sync memcpy_stream if extra stream for memcpy - second_to_last_fetch_checkpoint = fetched_checkpoint_varname + second_to_last_fetch_checkpoint = ( + fetched_checkpoint_varname + ) # there is NO fetch ahead the first checkpoint if input_var != self.sorted_checkpoint_names[0]: - fetched_checkpoint_varname = self._record_fetch_op( - idx) + fetched_checkpoint_varname = ( + self._record_fetch_op(idx) + ) # should check the current used checkpoint is ths last fetch one - assert second_to_last_fetch_checkpoint == input_var, "Current recompute segment should use [{}] BUT got [{}]".format( - second_to_last_fetch_checkpoint, input_var) + assert ( + second_to_last_fetch_checkpoint == input_var + ), "Current recompute segment should use [{}] BUT got [{}]".format( + second_to_last_fetch_checkpoint, input_var + ) # rename self.block.ops[idx]._rename_input( input_var, - self.checkpoint_name2fetch_name[input_var]) + self.checkpoint_name2fetch_name[input_var], + ) self.checkpoint_usage_count[input_var] += 1 else: raise ValueError( "use checkpoint [{}] before fetch in BW".format( - input_var)) + input_var + ) + ) - assert len(self.un_fetch_checkpoint_names - ) == 0, "{} checkpoints have NOT been Recorded".format( - self.un_fetch_checkpoint_names) + assert ( + len(self.un_fetch_checkpoint_names) == 0 + ), "{} checkpoints have NOT been Recorded".format( + self.un_fetch_checkpoint_names + ) def _update_backward(self): if len(self.idx2insertions) == 0: @@ -6500,15 +7214,18 @@ class RecomputeOptimizer(Optimizer): if operation == "fetch": self._insert_fetch_op(op_idx, checkpoint_name) logging.debug( - "Insert [{}] fetch op.".format(checkpoint_name)) + "Insert [{}] fetch op.".format(checkpoint_name) + ) del self.idx2insertions[op_idx] elif operation == "sync": self._insert_sync_op(op_idx, checkpoint_name) logging.debug("Sync [{}] fetch op.".format(checkpoint_name)) self.block._sync_with_cpp() - assert len( - self.idx2insertions) == 0, "{} checkpoints left un-Fecthed".format( - [ele[1] for ele in self.idx2insertions.values()]) + assert ( + len(self.idx2insertions) == 0 + ), "{} checkpoints left un-Fecthed".format( + [ele[1] for ele in self.idx2insertions.values()] + ) def _parse_forward(self): @@ -6521,7 +7238,7 @@ class RecomputeOptimizer(Optimizer): for checkpoint_name in self.un_offload_checkpoint_names: self.checkpoint_usage_count_and_idx[checkpoint_name] = { 'count': 0, - 'idx': -1 + 'idx': -1, } self.synced_checkpoints = set() self.fw_strart_op_idx = len(self.block.ops) @@ -6531,11 +7248,13 @@ class RecomputeOptimizer(Optimizer): break assert self.fw_strart_op_idx < len( - self.block.ops), "Could NOT found Forward op in prog" + self.block.ops + ), "Could NOT found Forward op in prog" last_offload_checkpoint = None for i, op in enumerate( - self.block.ops[self.fw_strart_op_idx:self.bw_strart_op_idx]): + self.block.ops[self.fw_strart_op_idx : self.bw_strart_op_idx] + ): idx = self.fw_strart_op_idx + i output_vars = op.desc.output_arg_names() @@ -6543,91 +7262,129 @@ class RecomputeOptimizer(Optimizer): for output_var in output_vars: if output_var in need_offload_checkpoint_names: - assert len( - output_vars - ) == 1, "chekpoint should be the only Output of a certain op, but [{}] is from [{}]".format( - output_var, op) + assert ( + len(output_vars) == 1 + ), "chekpoint should be the only Output of a certain op, but [{}] is from [{}]".format( + output_var, op + ) if output_var in self.un_offload_checkpoint_names: # insert sync op if last checkpoint has not been sync if last_offload_checkpoint != None: - if self.checkpoint_usage_count_and_idx[ - last_offload_checkpoint]['count'] == 0: - self._record_sync_op(idx, - last_offload_checkpoint) + if ( + self.checkpoint_usage_count_and_idx[ + last_offload_checkpoint + ]['count'] + == 0 + ): + self._record_sync_op( + idx, last_offload_checkpoint + ) else: - last_usage_idx = self.checkpoint_usage_count_and_idx[ - last_offload_checkpoint]['idx'] - assert last_usage_idx > 0, "last_usage_idx of checkpoint [{}] should large than 0".format( - last_offload_checkpoint) - self._record_sync_op(last_usage_idx + 1, - last_offload_checkpoint) + last_usage_idx = ( + self.checkpoint_usage_count_and_idx[ + last_offload_checkpoint + ]['idx'] + ) + assert ( + last_usage_idx > 0 + ), "last_usage_idx of checkpoint [{}] should large than 0".format( + last_offload_checkpoint + ) + self._record_sync_op( + last_usage_idx + 1, last_offload_checkpoint + ) # insert offload op after the checkpoint's generation op self._record_offload_op(idx + 1, output_var) last_offload_checkpoint = output_var else: raise ValueError( - "There should be just ONE op that output checkpoint [{}]" - .format(output_var)) + "There should be just ONE op that output checkpoint [{}]".format( + output_var + ) + ) # need to sync the last need to offload checkpoint before the last checkpoint as output op if output_var == last_checkpoint: - assert len( - output_vars - ) == 1, "chekpoint should be the only Output of a certain op, but [{}] is from [{}]".format( - output_var, op) - assert last_offload_checkpoint == self.sorted_checkpoint_names[ - -2], "the last offload chekpoint before [{}] is suppose to be [{}], but got [{}]".format( - last_checkpoint, self.sorted_checkpoint_names[-2], - last_offload_checkpoint) + assert ( + len(output_vars) == 1 + ), "chekpoint should be the only Output of a certain op, but [{}] is from [{}]".format( + output_var, op + ) + assert ( + last_offload_checkpoint + == self.sorted_checkpoint_names[-2] + ), "the last offload chekpoint before [{}] is suppose to be [{}], but got [{}]".format( + last_checkpoint, + self.sorted_checkpoint_names[-2], + last_offload_checkpoint, + ) # sync if last checkpoint has not been sync - if self.checkpoint_usage_count_and_idx[ - last_offload_checkpoint]['idx'] == 0: + if ( + self.checkpoint_usage_count_and_idx[ + last_offload_checkpoint + ]['idx'] + == 0 + ): self._record_sync_op(idx, last_offload_checkpoint) else: last_usage_idx = self.checkpoint_usage_count_and_idx[ - last_offload_checkpoint]['idx'] - assert last_usage_idx > 0, "last_usage_idx of checkpoint [{}] should large than 0".format( - last_offload_checkpoint) - self._record_sync_op(last_usage_idx + 1, - last_offload_checkpoint) + last_offload_checkpoint + ]['idx'] + assert ( + last_usage_idx > 0 + ), "last_usage_idx of checkpoint [{}] should large than 0".format( + last_offload_checkpoint + ) + self._record_sync_op( + last_usage_idx + 1, last_offload_checkpoint + ) # record checkpoint usage for input_var in input_vars: if input_var in need_offload_checkpoint_names: - assert input_var not in self.synced_checkpoints, "checkpoint [{}] used after sync".format( - input_var) + assert ( + input_var not in self.synced_checkpoints + ), "checkpoint [{}] used after sync".format(input_var) self.checkpoint_usage_count_and_idx[input_var]['count'] += 1 self.checkpoint_usage_count_and_idx[input_var]['idx'] = idx - assert len(self.un_offload_checkpoint_names - ) == 0, "{} checkpoints have NOT been Recorded".format( - self.un_fetch_checkpoint_names) + assert ( + len(self.un_offload_checkpoint_names) == 0 + ), "{} checkpoints have NOT been Recorded".format( + self.un_fetch_checkpoint_names + ) assert len(self.synced_checkpoints) == len( need_offload_checkpoint_names ), "{} checkpoints have NOT been Recorded".format( - set(need_offload_checkpoint_names) - set(self.synced_checkpoints)) + set(need_offload_checkpoint_names) - set(self.synced_checkpoints) + ) def _update_forward(self): if len(self.idx2insertions) == 0: return for op_idx in reversed( - range(self.fw_strart_op_idx, self.bw_strart_op_idx)): + range(self.fw_strart_op_idx, self.bw_strart_op_idx) + ): if op_idx in self.idx2insertions: operation, checkpoint_name = self.idx2insertions[op_idx] if operation == "offload": self._insert_offload_op(op_idx, checkpoint_name) logging.debug( - "Insert [{}] offload op.".format(checkpoint_name)) + "Insert [{}] offload op.".format(checkpoint_name) + ) del self.idx2insertions[op_idx] elif operation == "sync": self._insert_sync_op(op_idx, checkpoint_name) logging.debug( - "Insert [{}] offload_sync op.".format(checkpoint_name)) + "Insert [{}] offload_sync op.".format(checkpoint_name) + ) del self.idx2insertions[op_idx] self.block._sync_with_cpp() - assert len(self.idx2insertions - ) == 0, "{} checkpoints left un-Offloaded".format( - [ele[1] for ele in self.idx2insertions.values()]) + assert ( + len(self.idx2insertions) == 0 + ), "{} checkpoints left un-Offloaded".format( + [ele[1] for ele in self.idx2insertions.values()] + ) def _check_offload_fetch(self): # TODO(JZ-LIANG) the single stream offload need no sync @@ -6647,21 +7404,28 @@ class RecomputeOptimizer(Optimizer): startup_program = paddle.static.default_startup_program() with program_guard(self._main_program, startup_program): - assert len(self.checkpoint_shape) > 0, ( - "checkpoints shape {} should be an non empty list like: [12, 512, 1024]" - .format(self.checkpoint_shape)) - assert all([ele > 0 for ele in self.checkpoint_shape]), ( - "all ele in checkpoints shape {} should be a determined integer larger than 0" - .format(self.checkpoint_shape)) + assert ( + len(self.checkpoint_shape) > 0 + ), "checkpoints shape {} should be an non empty list like: [12, 512, 1024]".format( + self.checkpoint_shape + ) + assert all( + [ele > 0 for ele in self.checkpoint_shape] + ), "all ele in checkpoints shape {} should be a determined integer larger than 0".format( + self.checkpoint_shape + ) self.checkpoint_name2pinned_name = dict() self.checkpoint_name2fetch_name = dict() for checkpoint_varname in self.sorted_checkpoint_names: pinned_var_name, fetch_var_name = self._creat_vars( - checkpoint_varname) + checkpoint_varname + ) self.checkpoint_name2pinned_name[ - checkpoint_varname] = pinned_var_name + checkpoint_varname + ] = pinned_var_name self.checkpoint_name2fetch_name[ - checkpoint_varname] = fetch_var_name + checkpoint_varname + ] = fetch_var_name self._append_fill_constant_ops(startup_program) # TODO (JZ-LIANG) to provide two offload stragtegy in future # step 2. parse & update FW: rename, offload, sync @@ -6675,12 +7439,14 @@ class RecomputeOptimizer(Optimizer): return - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ call append_backward with checkpoints. @@ -6722,12 +7488,14 @@ class RecomputeOptimizer(Optimizer): no_grad_set=None) print("Finished backward") """ - assert (self._checkpoints - is not None), "You should call _set_checkpoints first" + assert ( + self._checkpoints is not None + ), "You should call _set_checkpoints first" if framework._non_static_mode(): raise NotImplementedError( - "DyGraph current does not support recompute") + "DyGraph current does not support recompute" + ) self._dtype = loss.dtype program = loss.block.program @@ -6745,12 +7513,15 @@ class RecomputeOptimizer(Optimizer): loss, parameter_list, no_grad_set, - checkpoints=checkpoint_vars) + checkpoints=checkpoint_vars, + ) else: - params_grads = append_backward(loss, - parameter_list, - no_grad_set, - checkpoints=checkpoint_vars) + params_grads = append_backward( + loss, + parameter_list, + no_grad_set, + checkpoints=checkpoint_vars, + ) if self.enable_offload: self.sorted_checkpoint_names = sorted_checkpoint_names @@ -6797,39 +7568,43 @@ class RecomputeOptimizer(Optimizer): print("Finished apply_optimize") """ - func = self._optimizer.apply_optimize if hasattr( - self._optimizer, - 'apply_optimize') else self._optimizer._apply_optimize - return func(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + func = ( + self._optimizer.apply_optimize + if hasattr(self._optimizer, 'apply_optimize') + else self._optimizer._apply_optimize + ) + return func( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): assert isinstance(loss, Variable), "The loss should be an Variable." - assert (self._checkpoints - is not None), "You should call _set_checkpoints first" + assert ( + self._checkpoints is not None + ), "You should call _set_checkpoints first" if framework._non_static_mode(): raise NotImplementedError( - "DyGraph current does not support recompute") - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + "DyGraph current does not support recompute" + ) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + optimize_ops = self.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads class LookaheadOptimizer(object): r""" - :api_attr: Static Graph + :api_attr: Static Graph This implements the Lookahead optimizer of the paper : https://arxiv.org/abs/1907.08610. @@ -6843,7 +7618,7 @@ class LookaheadOptimizer(object): slow\_param_t &= slow\_param_{t-1} + \\alpha * (fast\_param_{t-1} - slow\_param_{t-1}) - fast\_param_t &= slow\_param_t + fast\_param_t &= slow\_param_t Args: inner_optimizer (Optimizer): The optimizer that update fast params step by step. @@ -6892,11 +7667,11 @@ class LookaheadOptimizer(object): if framework._non_static_mode(): raise Exception("In dygraph, don't support LookaheadOptimizer.") - assert (inner_optimizer is not None), "inner optimizer can not be None" + assert inner_optimizer is not None, "inner optimizer can not be None" assert ( 0.0 <= alpha <= 1.0 ), "alpha should be larger or equal to 0.0, and less or equal than 1.0" - assert (isinstance(k, int) and k > 0), "k should be a positive integer" + assert isinstance(k, int) and k > 0, "k should be a positive integer" self.inner_optimizer = inner_optimizer self.alpha = alpha @@ -6907,7 +7682,8 @@ class LookaheadOptimizer(object): # Apply inner optimizer to the main_program mini_out = self.inner_optimizer.minimize( - loss, startup_program=startup_program) + loss, startup_program=startup_program + ) # Get startup_program and main_program if startup_program is None: @@ -6919,58 +7695,68 @@ class LookaheadOptimizer(object): param_to_slow = {} for param in params: fast_var = main_block.var(param) - assert (fast_var is not None) - slow_var = main_block.create_var(name=param + "@SLOW", - shape=fast_var.shape, - dtype=fast_var.dtype, - persistable=True) + assert fast_var is not None + slow_var = main_block.create_var( + name=param + "@SLOW", + shape=fast_var.shape, + dtype=fast_var.dtype, + persistable=True, + ) param_to_slow[param] = slow_var # add some vars to the startup_program startup_block = startup_program.global_block() for param in params: fast_var = startup_block.var(param) - assert (fast_var is not None) - slow_var = startup_block.create_var(name=param + "@SLOW", - shape=fast_var.shape, - dtype=fast_var.dtype, - persistable=True) + assert fast_var is not None + slow_var = startup_block.create_var( + name=param + "@SLOW", + shape=fast_var.shape, + dtype=fast_var.dtype, + persistable=True, + ) - startup_block.append_op(type="assign", - inputs={"X": fast_var}, - outputs={"Out": slow_var}) + startup_block.append_op( + type="assign", inputs={"X": fast_var}, outputs={"Out": slow_var} + ) with framework.program_guard(main_block.program, startup_program): # Add Var k to main prog and startup prog - k = layers.create_global_var(name="lookahead_k", - shape=[1], - value=int(self.k), - dtype='int32', - persistable=True) + k = layers.create_global_var( + name="lookahead_k", + shape=[1], + value=int(self.k), + dtype='int32', + persistable=True, + ) # Add Var alpha to main prog and startup prog - alpha = layers.create_global_var(name="lookahead_alpha", - shape=[1], - value=float(self.alpha), - dtype='float32', - persistable=True) + alpha = layers.create_global_var( + name="lookahead_alpha", + shape=[1], + value=float(self.alpha), + dtype='float32', + persistable=True, + ) # Add Var step - step = layers.create_global_var(name="lookahead_step", - shape=[1], - value=int(0), - dtype='int32', - persistable=True) + step = layers.create_global_var( + name="lookahead_step", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + ) layers.increment(x=step, value=1.0, in_place=True) # lookahead - zero_var = layers.fill_constant(shape=[1], - dtype='float32', - value=0.0) + zero_var = layers.fill_constant( + shape=[1], dtype='float32', value=0.0 + ) - one_var = layers.fill_constant(shape=[1], - dtype='float32', - value=1.0) + one_var = layers.fill_constant( + shape=[1], dtype='float32', value=1.0 + ) mod = layers.elementwise_mod(step, k) with layers.control_flow.Switch() as switch: @@ -6986,8 +7772,9 @@ class LookaheadOptimizer(object): tmp_var = layers.elementwise_add( layers.elementwise_mul(fast_var, alpha), layers.elementwise_mul( - slow_var, - layers.elementwise_sub(one_var, alpha))) + slow_var, layers.elementwise_sub(one_var, alpha) + ), + ) layers.assign(input=tmp_var, output=slow_var) layers.assign(input=tmp_var, output=fast_var) with switch.default(): @@ -7057,11 +7844,13 @@ class GradientMergeOptimizer(object): raise Exception( "In dygraph, we don't support GradientMergeOptimizer." "You can do Gradient merge by yourself with k-times forward + backward, " - "and one-time optimizer.minimize()") + "and one-time optimizer.minimize()" + ) - assert (inner_optimizer is not None), "inner optimizer can not be None" - assert (isinstance(k_steps, int) - and k_steps > 0), "k_steps should be a positive integer" + assert inner_optimizer is not None, "inner optimizer can not be None" + assert ( + isinstance(k_steps, int) and k_steps > 0 + ), "k_steps should be a positive integer" self.inner_optimizer = inner_optimizer self.k_steps = k_steps @@ -7075,12 +7864,14 @@ class GradientMergeOptimizer(object): def _set_avg(self, avg): self.avg = avg - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): assert isinstance(loss, Variable), "The loss should be an Variable." assert ( parameter_list is None @@ -7090,7 +7881,8 @@ class GradientMergeOptimizer(object): ), "The no_grad_set should be None when using GradientMergeOptimizer" params_grads = self.inner_optimizer.backward( - loss, startup_program=startup_program) + loss, startup_program=startup_program + ) return params_grads def apply_optimize(self, loss, startup_program, params_grads): @@ -7102,26 +7894,33 @@ class GradientMergeOptimizer(object): def _is_the_backward_op(self, op): op_maker = core.op_proto_and_checker_maker backward = core.op_proto_and_checker_maker.OpRole.Backward - if op_maker.kOpRoleVarAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(backward): + if op_maker.kOpRoleVarAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(backward): return True return False def _remove_op_role_var(self, param, grad): op_maker = core.op_proto_and_checker_maker op = grad.op - assert self._is_the_backward_op(op), \ - 'grad.op={} is not the backward op which produces the grad={}' \ - .format(op, grad.name) + assert self._is_the_backward_op( + op + ), 'grad.op={} is not the backward op which produces the grad={}'.format( + op, grad.name + ) block = grad.block var_attr = op.all_attrs()[op_maker.kOpRoleVarAttrName()] - assert param.name in var_attr, \ - 'when using GradientMergeOptimizer, param={} must be in var_attr={}' \ - .format(param.name, var_attr) - assert grad.name in var_attr, \ - 'when using GradientMergeOptimizer, grad={} must be in var_attr={}' \ - .format(param.name, var_attr) + assert ( + param.name in var_attr + ), 'when using GradientMergeOptimizer, param={} must be in var_attr={}'.format( + param.name, var_attr + ) + assert ( + grad.name in var_attr + ), 'when using GradientMergeOptimizer, grad={} must be in var_attr={}'.format( + param.name, var_attr + ) # remove (param, grad) from op_role_var var_attr.remove(param.name) @@ -7148,53 +7947,54 @@ class GradientMergeOptimizer(object): def _get_gm_cond_var(self, main_block): # Add const var - k_step_var = layers.create_global_var(name="gradient_merge_k", - shape=[1], - value=int(self.k_steps), - dtype='int32', - persistable=True, - force_cpu=True) - - zero_var = layers.create_global_var(name="gradient_merge_zero", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + k_step_var = layers.create_global_var( + name="gradient_merge_k", + shape=[1], + value=int(self.k_steps), + dtype='int32', + persistable=True, + force_cpu=True, + ) + + zero_var = layers.create_global_var( + name="gradient_merge_zero", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) # Add step var & cond var - step_var = layers.create_global_var(name="gradient_merge_step", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + step_var = layers.create_global_var( + name="gradient_merge_step", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) - cond_var = main_block.create_var(name="gradient_merge_cond", - shape=[1], - dtype='bool') + cond_var = main_block.create_var( + name="gradient_merge_cond", shape=[1], dtype='bool' + ) with device_guard("cpu"): # step_var = (step_var + 1) % k_step layers.increment(x=step_var, value=1.0, in_place=True) - main_block.append_op(type='elementwise_mod', - inputs={ - 'X': step_var, - 'Y': k_step_var - }, - outputs={'Out': step_var}, - attrs={ - 'axis': -1, - 'use_mkldnn': False - }) + main_block.append_op( + type='elementwise_mod', + inputs={'X': step_var, 'Y': k_step_var}, + outputs={'Out': step_var}, + attrs={'axis': -1, 'use_mkldnn': False}, + ) # cond_var = (step_var == 0) - main_block.append_op(type='equal', - inputs={ - 'X': step_var, - 'Y': zero_var - }, - outputs={'Out': cond_var}) + main_block.append_op( + type='equal', + inputs={'X': step_var, 'Y': zero_var}, + outputs={'Out': cond_var}, + ) return cond_var @@ -7206,7 +8006,7 @@ class GradientMergeOptimizer(object): cond = self._get_gm_cond_var(main_block) - #TODO(mapingshuo) support sparse embedding + # TODO(mapingshuo) support sparse embedding # step1: remove grad.op's op_role_var for param, grad in params_grads: assert ( @@ -7225,41 +8025,41 @@ class GradientMergeOptimizer(object): for param, grad in params_grads: param_name = param.name param_var = main_block.var(param_name) - assert (param_var is not None) - gradient_merge_var = main_block.create_var(name=param_name + - "@GRAD@GradientMerge", - shape=param_var.shape, - dtype=param_var.dtype, - persistable=True) + assert param_var is not None + gradient_merge_var = main_block.create_var( + name=param_name + "@GRAD@GradientMerge", + shape=param_var.shape, + dtype=param_var.dtype, + persistable=True, + ) param_to_gradient_merge[param_name] = gradient_merge_var startup_gradient_merge_var = startup_block.create_var( name=param_name + "@GRAD@GradientMerge", shape=param_var.shape, dtype=param_var.dtype, - persistable=True) - startup_block.append_op(type="fill_constant", - outputs={"Out": startup_gradient_merge_var}, - attrs={ - "shape": param_var.shape, - "dtype": param_var.dtype, - "value": float(0), - }) + persistable=True, + ) + startup_block.append_op( + type="fill_constant", + outputs={"Out": startup_gradient_merge_var}, + attrs={ + "shape": param_var.shape, + "dtype": param_var.dtype, + "value": float(0), + }, + ) # grad_merge += grad new_grad_op = main_block.append_op( type="elementwise_add", - inputs={ - 'X': grad, - 'Y': gradient_merge_var - }, + inputs={'X': grad, 'Y': gradient_merge_var}, outputs={'Out': gradient_merge_var}, - attrs={ - 'axis': -1, - 'use_mkldnn': False - }) - self._add_gm_op_role_var(new_grad_op, param, gradient_merge_var, - cond) + attrs={'axis': -1, 'use_mkldnn': False}, + ) + self._add_gm_op_role_var( + new_grad_op, param, gradient_merge_var, cond + ) new_params_grads.append([param, gradient_merge_var]) def true_apply_gradient(): @@ -7273,16 +8073,19 @@ class GradientMergeOptimizer(object): if self.avg: for param, new_grad in new_params_grads: # grad /= k_steps - cur_block.append_op(type='scale', - inputs={'X': new_grad}, - outputs={'Out': new_grad}, - attrs={ - 'scale': 1.0 / self.k_steps, - 'bias': 0.0, - 'bias_after_scale': False - }) - new_grad.op._set_attr(op_maker.kOpRoleAttrName(), - op_maker.OpRole.Backward) + cur_block.append_op( + type='scale', + inputs={'X': new_grad}, + outputs={'Out': new_grad}, + attrs={ + 'scale': 1.0 / self.k_steps, + 'bias': 0.0, + 'bias_after_scale': False, + }, + ) + new_grad.op._set_attr( + op_maker.kOpRoleAttrName(), op_maker.OpRole.Backward + ) for param, new_grad in new_params_grads: # NOTE. regularization will append ops to grad.block, @@ -7292,36 +8095,40 @@ class GradientMergeOptimizer(object): new_grad.block = cur_block self._optimize_ops = self.inner_optimizer.apply_gradients( - new_params_grads) + new_params_grads + ) # clear gradient_merge_vars for param, new_grad in new_params_grads: - layers.fill_constant(shape=new_grad.shape, - dtype=new_grad.dtype, - value=0.0, - out=new_grad) - new_grad.op._set_attr(op_maker.kOpRoleAttrName(), - op_maker.OpRole.Optimize) + layers.fill_constant( + shape=new_grad.shape, + dtype=new_grad.dtype, + value=0.0, + out=new_grad, + ) + new_grad.op._set_attr( + op_maker.kOpRoleAttrName(), op_maker.OpRole.Optimize + ) # step3. apply gradient layers.cond(cond, true_fn=true_apply_gradient, false_fn=None) return self._optimize_ops - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): assert isinstance(loss, Variable), "The loss should be an Variable." - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + optimize_ops = self.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index fe8a2b2610b..c0a45b041e6 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -27,7 +27,7 @@ BuildStrategy = core.ParallelExecutor.BuildStrategy class ParallelExecutor(object): """ - :api_attr: Static Graph + :api_attr: Static Graph The ParallelExecutor is an upgraded version of :code:`paddle.static.Executor` that supports multi-node model training and testing based on the data-parallel mode. In data-parallel mode, @@ -147,16 +147,18 @@ class ParallelExecutor(object): """ - def __init__(self, - use_cuda, - loss_name=None, - main_program=None, - share_vars_from=None, - exec_strategy=None, - build_strategy=None, - num_trainers=1, - trainer_id=0, - scope=None): + def __init__( + self, + use_cuda, + loss_name=None, + main_program=None, + share_vars_from=None, + exec_strategy=None, + build_strategy=None, + num_trainers=1, + trainer_id=0, + scope=None, + ): if build_strategy is None: build_strategy = BuildStrategy() @@ -164,22 +166,28 @@ class ParallelExecutor(object): if num_trainers != 1 and build_strategy.num_trainers != num_trainers: sys.stderr.write( 'The value of build_strategy.num_trainers[%d] is overwritten ' - 'by the passed num_trainers[%d].\n' % - (build_strategy.num_trainers, num_trainers)) + 'by the passed num_trainers[%d].\n' + % (build_strategy.num_trainers, num_trainers) + ) build_strategy.num_trainers = num_trainers if trainer_id != 0 and build_strategy.trainer_id != trainer_id: sys.stderr.write( 'The value of build_strategy.trainer_id[%d] is overwritten ' - 'by the passed trainer_id[%d].\n' % - (build_strategy.trainer_id, trainer_id)) + 'by the passed trainer_id[%d].\n' + % (build_strategy.trainer_id, trainer_id) + ) build_strategy.trainer_id = trainer_id - self._places = framework.cuda_places( - ) if use_cuda else framework.cpu_places() + self._places = ( + framework.cuda_places() if use_cuda else framework.cpu_places() + ) self._scope = scope if scope is not None else executor.global_scope() - main_program = main_program if main_program is not None \ + main_program = ( + main_program + if main_program is not None else framework.default_main_program() + ) self._compiled_program = compiler.CompiledProgram(main_program) if share_vars_from: @@ -192,7 +200,9 @@ class ParallelExecutor(object): build_strategy=build_strategy, exec_strategy=exec_strategy, share_vars_from=share_vars_from._compiled_program - if share_vars_from else None) + if share_vars_from + else None, + ) self._place = core.CUDAPlace(0) if use_cuda else core.CPUPlace() self._exe = executor.Executor(self._place) @@ -299,11 +309,13 @@ class ParallelExecutor(object): fetch_list=[loss.name]) """ - return self._exe.run(program=self._compiled_program, - scope=self._scope, - feed=feed, - fetch_list=fetch_list, - return_numpy=return_numpy) + return self._exe.run( + program=self._compiled_program, + scope=self._scope, + feed=feed, + fetch_list=fetch_list, + return_numpy=return_numpy, + ) @property def device_count(self): @@ -363,14 +375,20 @@ class ParallelExecutor(object): parallel_exe.drop_local_exe_scopes() """ - check_type(self._compiled_program._executor, - "the Executor of compiled program", core.ParallelExecutor, - "ParallelExecutor.drop_local_exe_scopes") + check_type( + self._compiled_program._executor, + "the Executor of compiled program", + core.ParallelExecutor, + "ParallelExecutor.drop_local_exe_scopes", + ) self._compiled_program._executor.drop_local_exe_scopes() # This API is used to check whether DropLocalExeScopes can work. def _need_create_local_exe_scopes(self): - check_type(self._compiled_program._executor, - "the Executor of compiled program", core.ParallelExecutor, - "ParallelExecutor._need_create_local_exe_scopes") + check_type( + self._compiled_program._executor, + "the Executor of compiled program", + core.ParallelExecutor, + "ParallelExecutor._need_create_local_exe_scopes", + ) return self._compiled_program._executor._need_create_local_exe_scopes() diff --git a/python/paddle/fluid/param_attr.py b/python/paddle/fluid/param_attr.py index b43625759e3..ba7bead304b 100644 --- a/python/paddle/fluid/param_attr.py +++ b/python/paddle/fluid/param_attr.py @@ -71,24 +71,31 @@ class ParamAttr(object): paddle.nn.Linear(3, 4, weight_attr=weight_attr) """ - def __init__(self, - name=None, - initializer=None, - learning_rate=1.0, - regularizer=None, - trainable=True, - do_model_average=True, - need_clip=True): + def __init__( + self, + name=None, + initializer=None, + learning_rate=1.0, + regularizer=None, + trainable=True, + do_model_average=True, + need_clip=True, + ): check_type(name, "name", (str, type(None)), "ParamAttr") check_type(learning_rate, "learning_rate", (float, int), "ParamAttr") check_type(trainable, "trainable", (bool), "ParamAttr") check_type(do_model_average, "do_model_average", (bool), "ParamAttr") check_type(need_clip, "need_clip", (bool), "ParamAttr") - check_type(initializer, "initializer", (Initializer, type(None)), - "ParamAttr") - check_type(regularizer, "regularizer", - (WeightDecayRegularizer, type(None)), "ParamAttr") + check_type( + initializer, "initializer", (Initializer, type(None)), "ParamAttr" + ) + check_type( + regularizer, + "regularizer", + (WeightDecayRegularizer, type(None)), + "ParamAttr", + ) self.name = name if self.name == "": @@ -191,13 +198,11 @@ class ParamAttr(object): """ kwargs = { 'name': self.name, - 'optimize_attr': { - 'learning_rate': self.learning_rate - }, + 'optimize_attr': {'learning_rate': self.learning_rate}, 'regularizer': self.regularizer, 'trainable': self.trainable, 'do_model_average': self.do_model_average, - 'need_clip': self.need_clip + 'need_clip': self.need_clip, } if with_initializer: kwargs['initializer'] = self.initializer @@ -277,21 +282,24 @@ class WeightNormParamAttr(ParamAttr): # these paramters for inference. params_with_weight_norm = [] - def __init__(self, - dim=None, - name=None, - initializer=None, - learning_rate=1.0, - regularizer=None, - trainable=True, - do_model_average=False, - need_clip=True): - super(WeightNormParamAttr, - self).__init__(name=name, - initializer=initializer, - learning_rate=learning_rate, - regularizer=regularizer, - trainable=trainable, - do_model_average=do_model_average, - need_clip=need_clip) + def __init__( + self, + dim=None, + name=None, + initializer=None, + learning_rate=1.0, + regularizer=None, + trainable=True, + do_model_average=False, + need_clip=True, + ): + super(WeightNormParamAttr, self).__init__( + name=name, + initializer=initializer, + learning_rate=learning_rate, + regularizer=regularizer, + trainable=trainable, + do_model_average=do_model_average, + need_clip=need_clip, + ) self.dim = dim diff --git a/python/paddle/fluid/profiler.py b/python/paddle/fluid/profiler.py index 827c8b42a5b..6ebb2f9a088 100644 --- a/python/paddle/fluid/profiler.py +++ b/python/paddle/fluid/profiler.py @@ -20,8 +20,11 @@ import sys from paddle.utils.deprecated import deprecated __all__ = [ - 'cuda_profiler', 'reset_profiler', 'profiler', 'start_profiler', - 'stop_profiler' + 'cuda_profiler', + 'reset_profiler', + 'profiler', + 'start_profiler', + 'stop_profiler', ] NVPROF_CONFIG = [ @@ -39,8 +42,8 @@ NVPROF_CONFIG = [ since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) @signature_safe_contextmanager def cuda_profiler(output_file, output_mode=None, config=None): """ @@ -118,8 +121,8 @@ def npu_profiler(output_file, config=None): since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) def reset_profiler(): """ Clear the previous time record. It works for @@ -146,8 +149,8 @@ def reset_profiler(): since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) def start_profiler(state, tracer_option='Default'): """ Enable the profiler. Uers can use `fluid.profiler.start_profiler` and @@ -204,7 +207,8 @@ def start_profiler(state, tracer_option='Default'): if tracer_option not in ['Default', 'OpDetail', 'AllOpDetail']: raise ValueError( - "tracer option must be 'Default', 'OpDetail', 'AllOpDetail'.") + "tracer option must be 'Default', 'OpDetail', 'AllOpDetail'." + ) if tracer_option == "Default": prof_tracer_option = core.TracerOption.kDefault elif tracer_option == "OpDetail": @@ -220,8 +224,8 @@ def start_profiler(state, tracer_option='Default'): since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) def stop_profiler(sorted_key=None, profile_path='/tmp/profile'): """ Stop the profiler. Uers can use `fluid.profiler.start_profiler` and @@ -264,8 +268,10 @@ def stop_profiler(sorted_key=None, profile_path='/tmp/profile'): return sorted_key = 'default' if sorted_key is None else sorted_key if sorted_key not in ['default', 'calls', 'total', 'max', 'min', 'ave']: - raise ValueError("The sorted_key must be None or in 'calls', 'total', " - "'max', 'min' and 'ave'") + raise ValueError( + "The sorted_key must be None or in 'calls', 'total', " + "'max', 'min' and 'ave'" + ) key_map = { 'default': core.EventSortingKey.kDefault, 'calls': core.EventSortingKey.kCalls, @@ -283,13 +289,12 @@ def stop_profiler(sorted_key=None, profile_path='/tmp/profile'): since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) @signature_safe_contextmanager -def profiler(state, - sorted_key=None, - profile_path='/tmp/profile', - tracer_option='Default'): +def profiler( + state, sorted_key=None, profile_path='/tmp/profile', tracer_option='Default' +): """ The profiler interface. This profiler can be used to profile both CPU and GPU program. diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index 07dd03a2bc6..5c2e92ae458 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -21,14 +21,39 @@ import paddle import time import copy -from .framework import Program, Variable, program_guard, default_main_program, default_startup_program, _non_static_mode, cpu_places, _current_expected_place, _in_eager_without_dygraph_check +from .framework import ( + Program, + Variable, + program_guard, + default_main_program, + default_startup_program, + _non_static_mode, + cpu_places, + _current_expected_place, + _in_eager_without_dygraph_check, +) from .executor import global_scope from .data_feeder import DataFeeder, BatchedTensorProvider -from .multiprocess_utils import multiprocess_queue_set, CleanupFuncRegistrar, _cleanup_mmap, _cleanup, _set_SIGCHLD_handler +from .multiprocess_utils import ( + multiprocess_queue_set, + CleanupFuncRegistrar, + _cleanup_mmap, + _cleanup, + _set_SIGCHLD_handler, +) from .dataloader import BatchSampler, Dataset, IterableDataset, Subset -from .dataloader.dataloader_iter import _DataLoaderIterSingleProcess, _DataLoaderIterMultiProcess, _DatasetKind, default_collate_fn +from .dataloader.dataloader_iter import ( + _DataLoaderIterSingleProcess, + _DataLoaderIterMultiProcess, + _DatasetKind, + default_collate_fn, +) from .dataloader.batch_sampler import _InfiniteIterableSampler -from .layers.io import monkey_patch_reader_methods, _copy_reader_var_, double_buffer +from .layers.io import ( + monkey_patch_reader_methods, + _copy_reader_var_, + double_buffer, +) from .unique_name import UniqueNameGenerator from .framework import _get_paddle_place, _get_paddle_place_list from paddle.fluid.framework import _set_expected_place, _current_expected_place @@ -39,8 +64,10 @@ import warnings import os import multiprocessing import signal + # NOTE: queue has a different name in python2 and python3 import queue + # NOTE: [ avoid hanging & failed quickly ] These value is used in getting data from another process QUEUE_GET_TIMEOUT = 60 @@ -120,7 +147,6 @@ def _reader_process_loop(batch_reader, data_queue): class DataLoaderBase(object): - def __init__(self): self._places = None @@ -151,12 +177,12 @@ class DataLoaderBase(object): "this means the input data contains nested lists with different lengths. " "\n\t* Check the reader function passed to 'decorate_batch_generator'" " to locate the data causes this issue.\n\t* Please consider using " - "'fluid.create_lod_tensor' to convert it to a LoD-Tensor.") + "'fluid.create_lod_tensor' to convert it to a LoD-Tensor." + ) return arr class AuToTune(object): - def __init__(self, loader): self.loader = loader self.max_num_worker = multiprocessing.cpu_count() / 2 @@ -174,12 +200,14 @@ class AuToTune(object): # pick the best num_workers auto_tune_start = time.time() logging.debug("========= DataLoader Auto Tune =========") - logging.debug("User config for DataLoader: " + - str(self.loader.num_workers)) + logging.debug( + "User config for DataLoader: " + str(self.loader.num_workers) + ) best_num_workers = 0 min_cost = float("inf") - logging.debug("Tuning Range for num_workers: 0 ~ " + - str(self.max_num_worker)) + logging.debug( + "Tuning Range for num_workers: 0 ~ " + str(self.max_num_worker) + ) num_workers = 0 while num_workers < self.max_num_worker: auto_tune_loader.num_workers = num_workers @@ -188,25 +216,37 @@ class AuToTune(object): min_cost = avg_cost best_num_workers = num_workers else: - update_num = self.is_best(auto_tune_loader, best_num_workers, - min_cost, self.max_num_worker) + update_num = self.is_best( + auto_tune_loader, + best_num_workers, + min_cost, + self.max_num_worker, + ) if update_num == best_num_workers: break else: best_num_workers = update_num - logging.debug("num_workers: " + str(num_workers) + " avg_cost: " + - str(avg_cost)) + logging.debug( + "num_workers: " + + str(num_workers) + + " avg_cost: " + + str(avg_cost) + ) num_workers += 2 - logging.info("auto_tune dataLoader best_num_workers: " + - str(best_num_workers)) - logging.debug("AutoTuning Cost for DataLoader: " + - str(time.time() - auto_tune_start) + ' seconds') + logging.info( + "auto_tune dataLoader best_num_workers: " + str(best_num_workers) + ) + logging.debug( + "AutoTuning Cost for DataLoader: " + + str(time.time() - auto_tune_start) + + ' seconds' + ) # tune the default loader's num_workers return best_num_workers def need_autotune(self): - if (sys.platform == 'darwin' or sys.platform == 'win32'): + if sys.platform == 'darwin' or sys.platform == 'win32': return False else: return True @@ -219,8 +259,9 @@ class AuToTune(object): def get_autotune_loader(self): loader = copy.copy(self.loader) batch_size = self.loader.batch_sampler.batch_size - if isinstance(self.loader.batch_sampler, - paddle.io.DistributedBatchSampler): + if isinstance( + self.loader.batch_sampler, paddle.io.DistributedBatchSampler + ): dataset = self.loader.batch_sampler.dataset sub_dataset = self.get_sub_dataset(dataset, batch_size) loader.batch_sampler = paddle.io.DistributedBatchSampler( @@ -229,14 +270,16 @@ class AuToTune(object): num_replicas=self.loader.batch_sampler.nranks, rank=self.loader.batch_sampler.local_rank, shuffle=self.loader.batch_sampler.shuffle, - drop_last=self.loader.batch_sampler.drop_last) + drop_last=self.loader.batch_sampler.drop_last, + ) elif isinstance(self.loader.batch_sampler, paddle.io.BatchSampler): dataset = self.loader.batch_sampler.sampler.data_source sub_dataset = self.get_sub_dataset(dataset, batch_size) loader.batch_sampler = paddle.io.BatchSampler( dataset=sub_dataset, batch_size=batch_size, - drop_last=self.loader.batch_sampler.drop_last) + drop_last=self.loader.batch_sampler.drop_last, + ) else: loader = None return loader @@ -261,10 +304,14 @@ class AuToTune(object): while num_workers < num_work_boundary and step < 5: self.loader.num_workers = num_workers time = self.evaluate_reader_cost(reader) - logging.debug("for back num_workers: " + str(num_workers) + - " avg_cost: " + str(time)) + logging.debug( + "for back num_workers: " + + str(num_workers) + + " avg_cost: " + + str(time) + ) step += 1 - if (time < best_time * 0.70 * boundary): + if time < best_time * 0.70 * boundary: return num_workers else: num_workers += 1 @@ -442,23 +489,25 @@ class DataLoader(object): """ - def __init__(self, - dataset, - feed_list=None, - places=None, - return_list=True, - batch_sampler=None, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=None, - num_workers=0, - use_buffer_reader=True, - prefetch_factor=2, - use_shared_memory=True, - timeout=0, - worker_init_fn=None, - persistent_workers=False): + def __init__( + self, + dataset, + feed_list=None, + places=None, + return_list=True, + batch_sampler=None, + batch_size=1, + shuffle=False, + drop_last=False, + collate_fn=None, + num_workers=0, + use_buffer_reader=True, + prefetch_factor=2, + use_shared_memory=True, + timeout=0, + worker_init_fn=None, + persistent_workers=False, + ): self.return_list = return_list self.collate_fn = collate_fn self.use_buffer_reader = use_buffer_reader @@ -468,8 +517,9 @@ class DataLoader(object): self.dataset = dataset if not return_list and not _non_static_mode(): - assert feed_list is not None, \ - "feed_list should be set when return_list=False" + assert ( + feed_list is not None + ), "feed_list should be set when return_list=False" self.feed_list = feed_list if places is None: @@ -481,11 +531,13 @@ class DataLoader(object): self.places = _convert_places(places) assert num_workers >= 0, "num_workers should be a non-negative value" - if num_workers > 0 and (sys.platform == 'darwin' - or sys.platform == 'win32'): + if num_workers > 0 and ( + sys.platform == 'darwin' or sys.platform == 'win32' + ): warnings.warn( - "DataLoader with multi-process mode is not supported on MacOs and Windows currently." \ - " Please use signle-process mode with num_workers = 0 instead") + "DataLoader with multi-process mode is not supported on MacOs and Windows currently." + " Please use signle-process mode with num_workers = 0 instead" + ) num_workers = 0 self.num_workers = num_workers @@ -502,44 +554,53 @@ class DataLoader(object): self.dataset_kind = _DatasetKind.ITER if shuffle: raise ValueError( - "IterableDataset not support shuffle, but got shuffle={}". - format(shuffle)) + "IterableDataset not support shuffle, but got shuffle={}".format( + shuffle + ) + ) if batch_sampler is not None: raise ValueError( - "IterableDataset expect unspecified batch_sampler") + "IterableDataset expect unspecified batch_sampler" + ) else: self.dataset_kind = _DatasetKind.MAP if batch_sampler is not None: - assert batch_size == 1 and not shuffle and not drop_last, \ - "batch_size/shuffle/drop_last should not be set when " \ + assert batch_size == 1 and not shuffle and not drop_last, ( + "batch_size/shuffle/drop_last should not be set when " "batch_sampler is given" + ) self.batch_sampler = batch_sampler self.batch_size = None elif batch_size is None: self.batch_sampler = None self.batch_size = None else: - assert batch_size > 0, \ - "batch_size should be None or a positive value when " \ + assert batch_size > 0, ( + "batch_size should be None or a positive value when " "batch_sampler is not given" + ) self.batch_size = batch_size if isinstance(dataset, IterableDataset): self.batch_sampler = _InfiniteIterableSampler( - dataset, batch_size) + dataset, batch_size + ) else: - self.batch_sampler = BatchSampler(dataset=dataset, - batch_size=batch_size, - shuffle=shuffle, - drop_last=drop_last) + self.batch_sampler = BatchSampler( + dataset=dataset, + batch_size=batch_size, + shuffle=shuffle, + drop_last=drop_last, + ) self.drop_last = drop_last self.auto_collate_batch = self.batch_sampler is not None self.pin_memory = False if _non_static_mode(): - self.pin_memory = True if use_pinned_memory( - ) is None else use_pinned_memory() + self.pin_memory = ( + True if use_pinned_memory() is None else use_pinned_memory() + ) self._persistent_workers = persistent_workers self._iterator = None @@ -570,13 +631,15 @@ class DataLoader(object): return self.__iter__() @staticmethod - def from_generator(feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False, - use_multiprocess=False, - drop_last=True): + def from_generator( + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + use_multiprocess=False, + drop_last=True, + ): """ .. warning:: This API will be deprecated in the future, it is recommended to use @@ -888,12 +951,23 @@ class DataLoader(object): print(run_inference(drop_last=False)) # [1.0, 4.0, 9.0] """ if _non_static_mode(): - return DygraphGeneratorLoader(feed_list, capacity, - use_double_buffer, iterable, - return_list, use_multiprocess) + return DygraphGeneratorLoader( + feed_list, + capacity, + use_double_buffer, + iterable, + return_list, + use_multiprocess, + ) else: - return GeneratorLoader(feed_list, capacity, use_double_buffer, - iterable, return_list, drop_last) + return GeneratorLoader( + feed_list, + capacity, + use_double_buffer, + iterable, + return_list, + drop_last, + ) @staticmethod def from_dataset(dataset, places, drop_last=True): @@ -950,13 +1024,15 @@ class DygraphGeneratorLoader(DataLoaderBase): static graph GeneratorLoader, Separate implementation to keep code readable. """ - def __init__(self, - feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=False): + def __init__( + self, + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=False, + ): self._batch_reader = None self._places = None self._feed_list = feed_list @@ -979,8 +1055,9 @@ class DygraphGeneratorLoader(DataLoaderBase): # NOTE: the multiprocessing in different platform is incompatible, we will solve it later self._use_multiprocess = use_multiprocess - if self._use_multiprocess and (sys.platform == 'darwin' - or sys.platform == 'win32'): + if self._use_multiprocess and ( + sys.platform == 'darwin' or sys.platform == 'win32' + ): warnings.warn( "NOTE: DygraphGeneratorLoader with multiprocess mode is not currently supported on MacOs and Windows." ) @@ -999,8 +1076,9 @@ class DygraphGeneratorLoader(DataLoaderBase): # mode, this thread is used to get next batch data from self._batch_reader, then # push it into self._blocking_queue self._thread = None - self._pin_memory = True if use_pinned_memory( - ) is None else use_pinned_memory() + self._pin_memory = ( + True if use_pinned_memory() is None else use_pinned_memory() + ) @property def queue(self): @@ -1042,14 +1120,20 @@ class DygraphGeneratorLoader(DataLoaderBase): self._dtypes = [] self._need_check_feed = [] self._blocking_queue = core.init_lod_tensor_blocking_queue( - core.Variable(), self._capacity, False) + core.Variable(), self._capacity, False + ) self._reader = None - self._reader = core.create_py_reader(self.queue, self._var_names, - self._shapes, self._dtypes, - self._need_check_feed, - self._places, - self._use_double_buffer, True, - self._pin_memory) + self._reader = core.create_py_reader( + self.queue, + self._var_names, + self._shapes, + self._dtypes, + self._need_check_feed, + self._places, + self._use_double_buffer, + True, + self._pin_memory, + ) def _start(self): if self._use_multiprocess: @@ -1060,9 +1144,10 @@ class DygraphGeneratorLoader(DataLoaderBase): # add _data_queue into global queue set global multiprocess_queue_set multiprocess_queue_set.add(self._data_queue) - self._process = multiprocessing.Process(target=_reader_process_loop, - args=(self._batch_reader, - self._data_queue)) + self._process = multiprocessing.Process( + target=_reader_process_loop, + args=(self._batch_reader, self._data_queue), + ) self._process.daemon = True self._process.start() @@ -1079,13 +1164,15 @@ class DygraphGeneratorLoader(DataLoaderBase): self._thread_done_event = threading.Event() self._thread = threading.Thread( target=self._reader_thread_loop_for_multiprocess, - args=(_current_expected_place(), )) + args=(_current_expected_place(),), + ) self._thread.daemon = True self._thread.start() else: self._thread = threading.Thread( target=self._reader_thread_loop_for_singleprocess, - args=(_current_expected_place(), )) + args=(_current_expected_place(),), + ) self._thread.daemon = True self._thread.start() @@ -1097,8 +1184,9 @@ class DygraphGeneratorLoader(DataLoaderBase): def __iter__(self): assert self.iterable, "DataLoader is not iterable" - assert self._batch_reader is not None, \ - "Data source of DataLoader has not set yet" + assert ( + self._batch_reader is not None + ), "Data source of DataLoader has not set yet" self._init_iterable() self._start() @@ -1108,7 +1196,8 @@ class DygraphGeneratorLoader(DataLoaderBase): try: if _in_eager_without_dygraph_check(): return core.eager.read_next_tensor_list( - self._reader.read_next_list()[0]) + self._reader.read_next_list()[0] + ) else: return self._reader.read_next_var_list() except StopIteration: @@ -1193,23 +1282,22 @@ class DygraphGeneratorLoader(DataLoaderBase): self._blocking_queue.kill() self._thread = None logging.warning( - "DygraphDataLoader reader thread raised an exception.") + "DygraphDataLoader reader thread raised an exception." + ) six.reraise(*sys.exc_info()) - def set_sample_generator(self, - reader, - batch_size, - drop_last=True, - places=None): + def set_sample_generator( + self, reader, batch_size, drop_last=True, places=None + ): assert batch_size > 0, "batch_size must be larger than 0" if isinstance(places, (list, tuple)): places = _get_paddle_place_list(places) else: places = _get_paddle_place(places) - self.set_sample_list_generator(paddle.batch(reader, - batch_size=batch_size, - drop_last=drop_last), - places=places) + self.set_sample_list_generator( + paddle.batch(reader, batch_size=batch_size, drop_last=drop_last), + places=places, + ) return self def set_sample_list_generator(self, reader, places=None): @@ -1241,20 +1329,22 @@ class DygraphGeneratorLoader(DataLoaderBase): if places is None: places = _current_expected_place() self._places = _convert_places(places) - assert len(self._places) == 1, \ - "Number of places must be 1 in imperative mode" + assert ( + len(self._places) == 1 + ), "Number of places must be 1 in imperative mode" return self class GeneratorLoader(DataLoaderBase): - - def __init__(self, - feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False, - drop_last=True): + def __init__( + self, + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + drop_last=True, + ): self._tensor_reader = None self._places = None self._thread = None @@ -1291,14 +1381,20 @@ class GeneratorLoader(DataLoaderBase): v.desc.need_check_feed() for v in self._feed_list ] self._queue = core.init_lod_tensor_blocking_queue( - core.Variable(), self._capacity, self._keep_order) + core.Variable(), self._capacity, self._keep_order + ) self._reader = None - self._reader = core.create_py_reader(self.queue, self._var_names, - self._shapes, self._dtypes, - self._need_check_feed, - self._places, - self._use_double_buffer, - self._drop_last, False) + self._reader = core.create_py_reader( + self.queue, + self._var_names, + self._shapes, + self._dtypes, + self._need_check_feed, + self._places, + self._use_double_buffer, + self._drop_last, + False, + ) def _init_non_iterable(self): lod_levels = [] @@ -1317,13 +1413,15 @@ class GeneratorLoader(DataLoaderBase): need_check_feed.append(int(feed_data.desc.need_check_feed())) queue_name = data_loader_unique_name_generator( - 'lod_tensor_blocking_queue') + 'lod_tensor_blocking_queue' + ) reader_name = data_loader_unique_name_generator('create_py_reader') double_buffer_name = data_loader_unique_name_generator('double_buffer') var = global_scope().var(queue_name) self._queue = core.init_lod_tensor_blocking_queue( - var, self._capacity, self._keep_order) + var, self._capacity, self._keep_order + ) if self._keep_order: block = default_main_program().current_block() @@ -1333,16 +1431,18 @@ class GeneratorLoader(DataLoaderBase): reader_var = block.create_var(name=reader_name) dtype_int = [int(t) for t in dtypes] - block.append_op(type='create_py_reader', - inputs={'blocking_queue': [queue_name]}, - outputs={'Out': [reader_var]}, - attrs={ - 'shape_concat': shape_concat, - 'lod_levels': lod_levels, - 'dtypes': dtype_int, - 'need_check_feed': need_check_feed, - 'ranks': ranks - }) + block.append_op( + type='create_py_reader', + inputs={'blocking_queue': [queue_name]}, + outputs={'Out': [reader_var]}, + attrs={ + 'shape_concat': shape_concat, + 'lod_levels': lod_levels, + 'dtypes': dtype_int, + 'need_check_feed': need_check_feed, + 'ranks': ranks, + }, + ) reader_var.desc.set_dtypes(dtypes) reader_var.persistable = True @@ -1354,7 +1454,8 @@ class GeneratorLoader(DataLoaderBase): reader.reset = self._queue.reset else: main_prog_var = _copy_reader_var_( - default_main_program().current_block(), reader_var) + default_main_program().current_block(), reader_var + ) main_prog_var.stop_gradient = True main_prog_var.persistable = True @@ -1362,8 +1463,9 @@ class GeneratorLoader(DataLoaderBase): reader = monkey_patch_reader_methods(main_prog_var) if self._use_double_buffer: - double_buffer_reader = double_buffer(reader, - name=double_buffer_name) + double_buffer_reader = double_buffer( + reader, name=double_buffer_name + ) # we return a double buffer reader. However, the reset method comes from # py_reader. double_buffer_reader.reset = reader.reset @@ -1375,7 +1477,8 @@ class GeneratorLoader(DataLoaderBase): type='read', inputs={'Reader': [self._reader]}, outputs={'Out': self._feed_list}, - attrs={'drop_last': self._drop_last}) + attrs={'drop_last': self._drop_last}, + ) @property def queue(self): @@ -1387,8 +1490,9 @@ class GeneratorLoader(DataLoaderBase): def __iter__(self): assert self.iterable, "DataLoader is not iterable" - assert self._tensor_reader is not None, \ - "Data source of DataLoader has not set yet" + assert ( + self._tensor_reader is not None + ), "Data source of DataLoader has not set yet" self._init_iterable() self._start() @@ -1409,15 +1513,18 @@ class GeneratorLoader(DataLoaderBase): six.reraise(*sys.exc_info()) def start(self): - assert not self._iterable, "start() cannot be called when DataLoader is iterable" + assert ( + not self._iterable + ), "start() cannot be called when DataLoader is iterable" self._start() def reset(self): - assert not self._iterable, "reset() cannot be called when DataLoader is iterable" + assert ( + not self._iterable + ), "reset() cannot be called when DataLoader is iterable" self._reset() def _start(self): - def __thread_main__(legacy_expected_place): try: # See _DataLoaderIterSingleProcess._thread_loop() for why set expected place here. @@ -1450,8 +1557,9 @@ class GeneratorLoader(DataLoaderBase): logging.warning('Your reader has raised an exception!') six.reraise(*sys.exc_info()) - self._thread = threading.Thread(target=__thread_main__, - args=(_current_expected_place(), )) + self._thread = threading.Thread( + target=__thread_main__, args=(_current_expected_place(),) + ) self._thread.daemon = True self._thread.start() @@ -1465,11 +1573,9 @@ class GeneratorLoader(DataLoaderBase): self._exited = False self._reader.reset() - def set_sample_generator(self, - reader, - batch_size, - drop_last=True, - places=None): + def set_sample_generator( + self, reader, batch_size, drop_last=True, places=None + ): assert batch_size > 0, "batch_size must be larger than 0" if isinstance(places, (list, tuple)): places = _get_paddle_place_list(places) @@ -1482,16 +1588,20 @@ class GeneratorLoader(DataLoaderBase): break if has_lod: - self.set_sample_list_generator(paddle.batch(reader, - batch_size=batch_size, - drop_last=drop_last), - places=places) + self.set_sample_list_generator( + paddle.batch( + reader, batch_size=batch_size, drop_last=drop_last + ), + places=places, + ) else: - reader = BatchedTensorProvider(feed_list=self._feed_list, - place=core.CPUPlace(), - batch_size=batch_size, - generator=reader, - drop_last=drop_last) + reader = BatchedTensorProvider( + feed_list=self._feed_list, + place=core.CPUPlace(), + batch_size=batch_size, + generator=reader, + drop_last=drop_last, + ) self.set_batch_generator(reader, places=places) return self @@ -1501,8 +1611,9 @@ class GeneratorLoader(DataLoaderBase): else: places = _get_paddle_place(places) with program_guard(Program(), Program()): - feeder = DataFeeder(feed_list=self._feed_list, - place=core.CPUPlace()) + feeder = DataFeeder( + feed_list=self._feed_list, place=core.CPUPlace() + ) paddle_reader = feeder.decorate_reader(reader, multi_devices=False) def __tensor_reader_impl__(): @@ -1519,12 +1630,15 @@ class GeneratorLoader(DataLoaderBase): places = _get_paddle_place(places) self._tensor_reader = reader if self._iterable: - assert places is not None, "Places cannot be None when DataLoader is iterable" + assert ( + places is not None + ), "Places cannot be None when DataLoader is iterable" self._places = _convert_places(places) else: if places is not None: logging.info( - 'places would be ommited when DataLoader is not iterable') + 'places would be ommited when DataLoader is not iterable' + ) return self @@ -1696,15 +1810,17 @@ class PyReader(DataLoaderBase): relu = fluid.layers.relu(image) """ - def __init__(self, - feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False): - self._loader = DataLoader.from_generator(feed_list, capacity, - use_double_buffer, iterable, - return_list) + def __init__( + self, + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + ): + self._loader = DataLoader.from_generator( + feed_list, capacity, use_double_buffer, iterable, return_list + ) @property def queue(self): @@ -1725,8 +1841,8 @@ class PyReader(DataLoaderBase): Start the data feeding thread. Can only call when the reader object is not iterable. - Example: - .. code-block:: python + Example: + .. code-block:: python import paddle import paddle.fluid as fluid @@ -1754,7 +1870,7 @@ class PyReader(DataLoaderBase): reader.reset() break - ''' + ''' self._loader.start() def reset(self): @@ -1794,11 +1910,9 @@ class PyReader(DataLoaderBase): ''' self._loader.reset() - def decorate_sample_generator(self, - sample_generator, - batch_size, - drop_last=True, - places=None): + def decorate_sample_generator( + self, sample_generator, batch_size, drop_last=True, places=None + ): ''' Set the data source of the PyReader object. @@ -1861,8 +1975,9 @@ class PyReader(DataLoaderBase): executor.run(feed=data, fetch_list=[loss]) ''' - self._loader.set_sample_generator(sample_generator, batch_size, - drop_last, places) + self._loader.set_sample_generator( + sample_generator, batch_size, drop_last, places + ) def decorate_sample_list_generator(self, reader, places=None): ''' @@ -1987,11 +2102,12 @@ class PyReader(DataLoaderBase): class DatasetLoader(DataLoaderBase): - def __init__(self, dataset, places, drop_last): - assert isinstance(dataset, paddle.distributed.fleet.dataset.DatasetBase - ), "dataset must be type of DatasetBase" - assert not _non_static_mode( + assert isinstance( + dataset, paddle.distributed.fleet.dataset.DatasetBase + ), "dataset must be type of DatasetBase" + assert ( + not _non_static_mode() ), "DatasetLoader is not supported in dygraph mode yet" if isinstance(places, (list, tuple)): places = _get_paddle_place_list(places) @@ -2000,32 +2116,48 @@ class DatasetLoader(DataLoaderBase): thread_num = len(places) - assert len(dataset.filelist) >= thread_num, \ - "Filelist number of dataset {} must be not less than place number {}".format(len(dataset.filelist), thread_num) + assert ( + len(dataset.filelist) >= thread_num + ), "Filelist number of dataset {} must be not less than place number {}".format( + len(dataset.filelist), thread_num + ) if dataset.thread_num != 0 and dataset.thread_num != thread_num: logging.warn( 'thread_num {} which is set in Dataset is ignored'.format( - dataset.thread_num)) + dataset.thread_num + ) + ) dataset._set_thread(thread_num) - if isinstance(dataset, paddle.distributed.fleet.dataset.InMemoryDataset - ) and dataset.queue_num > thread_num: + if ( + isinstance( + dataset, paddle.distributed.fleet.dataset.InMemoryDataset + ) + and dataset.queue_num > thread_num + ): logging.warn( "queue_num {} which is set in Dataset is ignored".format( - dataset.queue_num)) + dataset.queue_num + ) + ) dataset._set_queue_num(thread_num) self._dataset = dataset use_slots = [ - slot.name for slot in dataset.proto_desc.multi_slot_desc.slots + slot.name + for slot in dataset.proto_desc.multi_slot_desc.slots if slot.is_used ] self._iterable_dataset = core.IterableDatasetWrapper( - dataset.dataset, use_slots, _convert_places(places), - dataset.proto_desc.batch_size, drop_last) + dataset.dataset, + use_slots, + _convert_places(places), + dataset.proto_desc.batch_size, + drop_last, + ) def __iter__(self): self._dataset._finish_to_run() diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py index ebf3230984e..00240a1d3f4 100644 --- a/python/paddle/fluid/regularizer.py +++ b/python/paddle/fluid/regularizer.py @@ -37,13 +37,11 @@ class WeightDecayRegularizer(object): pass def __call__(self, param, grad, block): - """Add corresponding weight decay operations to the network - """ + """Add corresponding weight decay operations to the network""" raise NotImplementedError() def __str__(self): - """Debug string - """ + """Debug string""" raise NotImplementedError() @@ -134,21 +132,25 @@ class L2DecayRegularizer(WeightDecayRegularizer): if framework._non_static_mode(): if framework.in_dygraph_mode(): - return _C_ops.scale(param, self._regularization_coeff, 0.0, - True) + return _C_ops.scale( + param, self._regularization_coeff, 0.0, True + ) else: - return _legacy_C_ops.scale(param, "scale", - self._regularization_coeff) + return _legacy_C_ops.scale( + param, "scale", self._regularization_coeff + ) else: - decay = block.create_var(dtype=param.dtype, - shape=param.shape, - lod_level=param.lod_level) + decay = block.create_var( + dtype=param.dtype, shape=param.shape, lod_level=param.lod_level + ) # Append Op to calculate decay - block.append_op(type='scale', - inputs={"X": param}, - outputs={"Out": decay}, - attrs={"scale": self._regularization_coeff}) + block.append_op( + type='scale', + inputs={"X": param}, + outputs={"Out": decay}, + attrs={"scale": self._regularization_coeff}, + ) return decay @@ -245,12 +247,12 @@ class L1DecayRegularizer(WeightDecayRegularizer): sign = block.create_var(dtype=param.dtype, shape=param.shape) decay = block.create_var(dtype=param.dtype, shape=param.shape) else: - sign = block.create_var(dtype=param.dtype, - shape=param.shape, - lod_level=param.lod_level) - decay = block.create_var(dtype=param.dtype, - shape=param.shape, - lod_level=param.lod_level) + sign = block.create_var( + dtype=param.dtype, shape=param.shape, lod_level=param.lod_level + ) + decay = block.create_var( + dtype=param.dtype, shape=param.shape, lod_level=param.lod_level + ) if in_dygraph_mode(): sign = _C_ops.sign(param) return _C_ops.scale(sign, self._regularization_coeff, 0.0, True) @@ -259,10 +261,12 @@ class L1DecayRegularizer(WeightDecayRegularizer): block.append_op(type='sign', inputs={"X": param}, outputs={"Out": sign}) # Append scale op to the output of sign op - block.append_op(type='scale', - inputs={"X": sign}, - outputs={"Out": decay}, - attrs={"scale": self._regularization_coeff}) + block.append_op( + type='scale', + inputs={"X": sign}, + outputs={"Out": decay}, + attrs={"scale": self._regularization_coeff}, + ) return decay diff --git a/python/paddle/fluid/tests/book/notest_understand_sentiment.py b/python/paddle/fluid/tests/book/notest_understand_sentiment.py index f8e49fafac4..02a17b9f92c 100644 --- a/python/paddle/fluid/tests/book/notest_understand_sentiment.py +++ b/python/paddle/fluid/tests/book/notest_understand_sentiment.py @@ -22,43 +22,41 @@ import sys import os -def convolution_net(data, - label, - input_dim, - class_dim=2, - emb_dim=32, - hid_dim=32): - emb = fluid.layers.embedding(input=data, - size=[input_dim, emb_dim], - is_sparse=True) - conv_3 = fluid.nets.sequence_conv_pool(input=emb, - num_filters=hid_dim, - filter_size=3, - act="tanh", - pool_type="sqrt") - conv_4 = fluid.nets.sequence_conv_pool(input=emb, - num_filters=hid_dim, - filter_size=4, - act="tanh", - pool_type="sqrt") - prediction = fluid.layers.fc(input=[conv_3, conv_4], - size=class_dim, - act="softmax") +def convolution_net( + data, label, input_dim, class_dim=2, emb_dim=32, hid_dim=32 +): + emb = fluid.layers.embedding( + input=data, size=[input_dim, emb_dim], is_sparse=True + ) + conv_3 = fluid.nets.sequence_conv_pool( + input=emb, + num_filters=hid_dim, + filter_size=3, + act="tanh", + pool_type="sqrt", + ) + conv_4 = fluid.nets.sequence_conv_pool( + input=emb, + num_filters=hid_dim, + filter_size=4, + act="tanh", + pool_type="sqrt", + ) + prediction = fluid.layers.fc( + input=[conv_3, conv_4], size=class_dim, act="softmax" + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) avg_cost = paddle.mean(cost) accuracy = fluid.layers.accuracy(input=prediction, label=label) return avg_cost, accuracy, prediction -def dyn_rnn_lstm(data, - label, - input_dim, - class_dim=2, - emb_dim=32, - lstm_size=128): - emb = fluid.layers.embedding(input=data, - size=[input_dim, emb_dim], - is_sparse=True) +def dyn_rnn_lstm( + data, label, input_dim, class_dim=2, emb_dim=32, lstm_size=128 +): + emb = fluid.layers.embedding( + input=data, size=[input_dim, emb_dim], is_sparse=True + ) sentence = fluid.layers.fc(input=emb, size=lstm_size, act='tanh') rnn = fluid.layers.DynamicRNN() @@ -73,13 +71,17 @@ def dyn_rnn_lstm(data, return gate0 + gate1 forget_gate = fluid.layers.sigmoid( - x=gate_common(word, prev_hidden, lstm_size)) + x=gate_common(word, prev_hidden, lstm_size) + ) input_gate = fluid.layers.sigmoid( - x=gate_common(word, prev_hidden, lstm_size)) + x=gate_common(word, prev_hidden, lstm_size) + ) output_gate = fluid.layers.sigmoid( - x=gate_common(word, prev_hidden, lstm_size)) + x=gate_common(word, prev_hidden, lstm_size) + ) cell_gate = fluid.layers.sigmoid( - x=gate_common(word, prev_hidden, lstm_size)) + x=gate_common(word, prev_hidden, lstm_size) + ) cell = forget_gate * prev_cell + input_gate * cell_gate hidden = output_gate * fluid.layers.tanh(x=cell) @@ -95,18 +97,14 @@ def dyn_rnn_lstm(data, return avg_cost, accuracy, prediction -def stacked_lstm_net(data, - label, - input_dim, - class_dim=2, - emb_dim=128, - hid_dim=512, - stacked_num=3): +def stacked_lstm_net( + data, label, input_dim, class_dim=2, emb_dim=128, hid_dim=512, stacked_num=3 +): assert stacked_num % 2 == 1 - emb = fluid.layers.embedding(input=data, - size=[input_dim, emb_dim], - is_sparse=True) + emb = fluid.layers.embedding( + input=data, size=[input_dim, emb_dim], is_sparse=True + ) # add bias attr # TODO(qijun) linear act @@ -117,54 +115,57 @@ def stacked_lstm_net(data, for i in range(2, stacked_num + 1): fc = fluid.layers.fc(input=inputs, size=hid_dim) - lstm, cell = fluid.layers.dynamic_lstm(input=fc, - size=hid_dim, - is_reverse=(i % 2) == 0) + lstm, cell = fluid.layers.dynamic_lstm( + input=fc, size=hid_dim, is_reverse=(i % 2) == 0 + ) inputs = [fc, lstm] fc_last = fluid.layers.sequence_pool(input=inputs[0], pool_type='max') lstm_last = fluid.layers.sequence_pool(input=inputs[1], pool_type='max') - prediction = fluid.layers.fc(input=[fc_last, lstm_last], - size=class_dim, - act='softmax') + prediction = fluid.layers.fc( + input=[fc_last, lstm_last], size=class_dim, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) avg_cost = paddle.mean(cost) accuracy = fluid.layers.accuracy(input=prediction, label=label) return avg_cost, accuracy, prediction -def train(word_dict, - net_method, - use_cuda, - parallel=False, - save_dirname=None, - is_local=True): +def train( + word_dict, + net_method, + use_cuda, + parallel=False, + save_dirname=None, + is_local=True, +): BATCH_SIZE = 128 PASS_NUM = 5 dict_dim = len(word_dict) class_dim = 2 - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") if not parallel: - cost, acc_out, prediction = net_method(data, - label, - input_dim=dict_dim, - class_dim=class_dim) + cost, acc_out, prediction = net_method( + data, label, input_dim=dict_dim, class_dim=class_dim + ) else: raise NotImplementedError() adagrad = fluid.optimizer.Adagrad(learning_rate=0.002) adagrad.minimize(cost) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.imdb.train(word_dict), buf_size=1000), - batch_size=BATCH_SIZE) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.imdb.train(word_dict), buf_size=1000 + ), + batch_size=BATCH_SIZE, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[data, label], place=place) @@ -174,19 +175,23 @@ def train(word_dict, for pass_id in range(PASS_NUM): for data in train_data(): - cost_val, acc_val = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[cost, acc_out]) + cost_val, acc_val = exe.run( + main_program, + feed=feeder.feed(data), + fetch_list=[cost, acc_out], + ) print("cost=" + str(cost_val) + " acc=" + str(acc_val)) if cost_val < 0.4 and acc_val > 0.8: if save_dirname is not None: - fluid.io.save_inference_model(save_dirname, ["words"], - prediction, exe) + fluid.io.save_inference_model( + save_dirname, ["words"], prediction, exe + ) return if math.isnan(float(cost_val)): sys.exit("got NaN loss, training failed.") - raise AssertionError("Cost is too large for {0}".format( - net_method.__name__)) + raise AssertionError( + "Cost is too large for {0}".format(net_method.__name__) + ) if is_local: train_loop(fluid.default_main_program()) @@ -205,8 +210,9 @@ def train(word_dict, t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -226,8 +232,11 @@ def infer(word_dict, use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) word_dict_len = len(word_dict) @@ -244,19 +253,19 @@ def infer(word_dict, use_cuda, save_dirname=None): recursive_seq_lens = [[3, 4, 2]] base_shape = [1] # The range of random integers is [low, high] - tensor_words = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) + tensor_words = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. assert feed_target_names[0] == "words" - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_words}, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_words}, + fetch_list=fetch_targets, + return_numpy=False, + ) print(results[0].recursive_sequence_lengths()) np_data = np.array(results[0]) print("Inference Shape: ", np_data.shape) @@ -267,16 +276,17 @@ def main(word_dict, net_method, use_cuda, parallel=False, save_dirname=None): if use_cuda and not fluid.core.is_compiled_with_cuda(): return - train(word_dict, - net_method, - use_cuda, - parallel=parallel, - save_dirname=save_dirname) + train( + word_dict, + net_method, + use_cuda, + parallel=parallel, + save_dirname=save_dirname, + ) infer(word_dict, use_cuda, save_dirname) class TestUnderstandSentiment(unittest.TestCase): - @classmethod def setUpClass(cls): cls.word_dict = paddle.dataset.imdb.word_dict() @@ -292,17 +302,21 @@ class TestUnderstandSentiment(unittest.TestCase): def test_conv_cpu(self): with self.new_program_scope(): - main(self.word_dict, - net_method=convolution_net, - use_cuda=False, - save_dirname="understand_sentiment_conv.inference.model") + main( + self.word_dict, + net_method=convolution_net, + use_cuda=False, + save_dirname="understand_sentiment_conv.inference.model", + ) def test_conv_cpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=convolution_net, - use_cuda=False, - parallel=True) + main( + self.word_dict, + net_method=convolution_net, + use_cuda=False, + parallel=True, + ) @unittest.skip(reason="make CI faster") def test_stacked_lstm_cpu(self): @@ -311,29 +325,35 @@ class TestUnderstandSentiment(unittest.TestCase): self.word_dict, net_method=stacked_lstm_net, use_cuda=False, - save_dirname="understand_sentiment_stacked_lstm.inference.model" + save_dirname="understand_sentiment_stacked_lstm.inference.model", ) def test_stacked_lstm_cpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=stacked_lstm_net, - use_cuda=False, - parallel=True) + main( + self.word_dict, + net_method=stacked_lstm_net, + use_cuda=False, + parallel=True, + ) def test_conv_gpu(self): with self.new_program_scope(): - main(self.word_dict, - net_method=convolution_net, - use_cuda=True, - save_dirname="understand_sentiment_conv.inference.model") + main( + self.word_dict, + net_method=convolution_net, + use_cuda=True, + save_dirname="understand_sentiment_conv.inference.model", + ) def test_conv_gpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=convolution_net, - use_cuda=True, - parallel=True) + main( + self.word_dict, + net_method=convolution_net, + use_cuda=True, + parallel=True, + ) @unittest.skip(reason="make CI faster") def test_stacked_lstm_gpu(self): @@ -342,30 +362,36 @@ class TestUnderstandSentiment(unittest.TestCase): self.word_dict, net_method=stacked_lstm_net, use_cuda=True, - save_dirname="understand_sentiment_stacked_lstm.inference.model" + save_dirname="understand_sentiment_stacked_lstm.inference.model", ) def test_stacked_lstm_gpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=stacked_lstm_net, - use_cuda=True, - parallel=True) + main( + self.word_dict, + net_method=stacked_lstm_net, + use_cuda=True, + parallel=True, + ) @unittest.skip(reason='make CI faster') def test_dynrnn_lstm_gpu(self): with self.new_program_scope(): - main(self.word_dict, - net_method=dyn_rnn_lstm, - use_cuda=True, - parallel=False) + main( + self.word_dict, + net_method=dyn_rnn_lstm, + use_cuda=True, + parallel=False, + ) def test_dynrnn_lstm_gpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=dyn_rnn_lstm, - use_cuda=True, - parallel=True) + main( + self.word_dict, + net_method=dyn_rnn_lstm, + use_cuda=True, + parallel=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/book/test_fit_a_line.py b/python/paddle/fluid/tests/book/test_fit_a_line.py index 788e8d12d83..338008c48d5 100644 --- a/python/paddle/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/test_fit_a_line.py @@ -32,7 +32,8 @@ def convert_uint16_to_float(in_list): in_list = numpy.asarray(in_list) out = numpy.vectorize( lambda x: struct.unpack('> 16)) + numpy.uint16(struct.unpack('> 16) + ) out = numpy.reshape(out, in_list.shape).view(numpy.uint16) return out @@ -73,15 +75,18 @@ def train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16): sgd_optimizer, amp_lists=amp.bf16.AutoMixedPrecisionListsBF16(), use_bf16_guard=False, - use_pure_bf16=pure_bf16) - sgd_optimizer.minimize(avg_cost, - startup_program=fluid.default_startup_program()) + use_pure_bf16=pure_bf16, + ) + sgd_optimizer.minimize( + avg_cost, startup_program=fluid.default_startup_program() + ) BATCH_SIZE = 20 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.uci_housing.train(), buf_size=500), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.uci_housing.train(), buf_size=500), + batch_size=BATCH_SIZE, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -91,29 +96,33 @@ def train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16): exe.run(fluid.default_startup_program()) test_prog = main_program.clone(for_test=True) if pure_bf16: - sgd_optimizer.amp_init(exe.place, - test_program=test_prog, - use_bf16_test=True) + sgd_optimizer.amp_init( + exe.place, test_program=test_prog, use_bf16_test=True + ) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): - avg_loss_value, = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + (avg_loss_value,) = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) if avg_loss_value.dtype == numpy.uint16: avg_loss_value = convert_uint16_to_float(avg_loss_value) if avg_loss_value[0] < 10.0: if save_dirname is not None: - paddle.static.save_inference_model(save_dirname, [x], - [y_predict], - exe, - clip_extra=False) + paddle.static.save_inference_model( + save_dirname, + [x], + [y_predict], + exe, + clip_extra=False, + ) return if math.isnan(float(avg_loss_value)): sys.exit("got NaN loss, training failed.") - raise AssertionError("Fit a line cost is too large, {0:2.2}".format( - avg_loss_value[0])) + raise AssertionError( + "Fit a line cost is too large, {0:2.2}".format(avg_loss_value[0]) + ) if is_local: train_loop(fluid.default_main_program()) @@ -132,8 +141,9 @@ def train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -153,30 +163,38 @@ def infer(use_cuda, save_dirname=None, use_bf16=False): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(save_dirname, exe) # The input's dimension should be 2-D and the second dim is 13 # The input data should be >= 0 batch_size = 10 - test_reader = paddle.batch(paddle.dataset.uci_housing.test(), - batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.uci_housing.test(), batch_size=batch_size + ) test_data = next(test_reader()) - test_feat = numpy.array([data[0] - for data in test_data]).astype("float32") + test_feat = numpy.array([data[0] for data in test_data]).astype( + "float32" + ) if use_bf16: test_feat = convert_float_to_uint16(test_feat) - test_label = numpy.array([data[1] - for data in test_data]).astype("float32") + test_label = numpy.array([data[1] for data in test_data]).astype( + "float32" + ) assert feed_target_names[0] == 'x' - results = exe.run(inference_program, - feed={feed_target_names[0]: numpy.array(test_feat)}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: numpy.array(test_feat)}, + fetch_list=fetch_targets, + ) if results[0].dtype == numpy.uint16: results[0] = convert_uint16_to_float(results[0]) print("infer shape: ", results[0].shape) @@ -201,7 +219,6 @@ def main(use_cuda, is_local=True, use_bf16=False, pure_bf16=False): class TestFitALineBase(unittest.TestCase): - @contextlib.contextmanager def program_scope_guard(self): prog = fluid.Program() @@ -213,7 +230,6 @@ class TestFitALineBase(unittest.TestCase): class TestFitALine(TestFitALineBase): - def test_cpu(self): with self.program_scope_guard(): main(use_cuda=False) @@ -223,10 +239,10 @@ class TestFitALine(TestFitALineBase): main(use_cuda=True) -@unittest.skipIf(not fluid.core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not fluid.core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestFitALineBF16(TestFitALineBase): - def test_bf16(self): with self.program_scope_guard(): main(use_cuda=False, use_bf16=True) diff --git a/python/paddle/fluid/tests/book/test_image_classification.py b/python/paddle/fluid/tests/book/test_image_classification.py index c8a4861370a..ac4dbaec226 100644 --- a/python/paddle/fluid/tests/book/test_image_classification.py +++ b/python/paddle/fluid/tests/book/test_image_classification.py @@ -26,21 +26,18 @@ paddle.enable_static() def resnet_cifar10(input, depth=32): - - def conv_bn_layer(input, - ch_out, - filter_size, - stride, - padding, - act='relu', - bias_attr=False): - tmp = fluid.layers.conv2d(input=input, - filter_size=filter_size, - num_filters=ch_out, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) + def conv_bn_layer( + input, ch_out, filter_size, stride, padding, act='relu', bias_attr=False + ): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr, + ) return fluid.layers.batch_norm(input=tmp, act=act) def shortcut(input, ch_in, ch_out, stride): @@ -63,33 +60,31 @@ def resnet_cifar10(input, depth=32): assert (depth - 2) % 6 == 0 n = (depth - 2) // 6 - conv1 = conv_bn_layer(input=input, - ch_out=16, - filter_size=3, - stride=1, - padding=1) + conv1 = conv_bn_layer( + input=input, ch_out=16, filter_size=3, stride=1, padding=1 + ) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) res2 = layer_warp(basicblock, res1, 16, 32, n, 2) res3 = layer_warp(basicblock, res2, 32, 64, n, 2) - pool = fluid.layers.pool2d(input=res3, - pool_size=8, - pool_type='avg', - pool_stride=1) + pool = fluid.layers.pool2d( + input=res3, pool_size=8, pool_type='avg', pool_stride=1 + ) return pool def vgg16_bn_drop(input): - def conv_block(input, num_filter, groups, dropouts): - return fluid.nets.img_conv_group(input=input, - pool_size=2, - pool_stride=2, - conv_num_filter=[num_filter] * groups, - conv_filter_size=3, - conv_act='relu', - conv_with_batchnorm=True, - conv_batchnorm_drop_rate=dropouts, - pool_type='max') + return fluid.nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max', + ) conv1 = conv_block(input, 64, 2, [0.3, 0]) conv2 = conv_block(conv1, 128, 2, [0.4, 0]) @@ -135,12 +130,16 @@ def train(net_type, use_cuda, save_dirname, is_local): BATCH_SIZE = 128 PASS_NUM = 1 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.cifar.train10(), buf_size=128 * 10), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.cifar.train10(), buf_size=128 * 10 + ), + batch_size=BATCH_SIZE, + ) - test_reader = paddle.batch(paddle.dataset.cifar.test10(), - batch_size=BATCH_SIZE) + test_reader = paddle.batch( + paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -157,9 +156,11 @@ def train(net_type, use_cuda, save_dirname, is_local): acc_list = [] avg_loss_list = [] for tid, test_data in enumerate(test_reader()): - loss_t, acc_t = exe.run(program=test_program, - feed=feeder.feed(test_data), - fetch_list=[avg_cost, acc]) + loss_t, acc_t = exe.run( + program=test_program, + feed=feeder.feed(test_data), + fetch_list=[avg_cost, acc], + ) if math.isnan(float(loss_t)): sys.exit("got NaN loss, training failed.") acc_list.append(float(acc_t)) @@ -170,13 +171,18 @@ def train(net_type, use_cuda, save_dirname, is_local): avg_loss_value = numpy.array(avg_loss_list).mean() print( - 'PassID {0:1}, BatchID {1:04}, Test Loss {2:2.2}, Acc {3:2.2}' - .format(pass_id, batch_id + 1, float(avg_loss_value), - float(acc_value))) + 'PassID {0:1}, BatchID {1:04}, Test Loss {2:2.2}, Acc {3:2.2}'.format( + pass_id, + batch_id + 1, + float(avg_loss_value), + float(acc_value), + ) + ) if acc_value > 0.01: # Low threshold for speeding up CI - fluid.io.save_inference_model(save_dirname, ["pixel"], - [predict], exe) + fluid.io.save_inference_model( + save_dirname, ["pixel"], [predict], exe + ) return if is_local: @@ -196,8 +202,9 @@ def train(net_type, use_cuda, save_dirname, is_local): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -217,8 +224,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # The input's dimension of conv should be 4-D or 5-D. # Use normilized image pixels as input data, which should be in the range [0, 1.0]. @@ -227,14 +237,21 @@ def infer(use_cuda, save_dirname=None): # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("infer results: ", results[0]) - fluid.io.save_inference_model(save_dirname, feed_target_names, - fetch_targets, exe, inference_program) + fluid.io.save_inference_model( + save_dirname, + feed_target_names, + fetch_targets, + exe, + inference_program, + ) def main(net_type, use_cuda, is_local=True): @@ -244,7 +261,8 @@ def main(net_type, use_cuda, is_local=True): # Directory for saving the trained model temp_dir = tempfile.TemporaryDirectory() save_dirname = os.path.join( - temp_dir.name, "image_classification_" + net_type + ".inference.model") + temp_dir.name, "image_classification_" + net_type + ".inference.model" + ) train(net_type, use_cuda, save_dirname, is_local) infer(use_cuda, save_dirname) @@ -252,7 +270,6 @@ def main(net_type, use_cuda, is_local=True): class TestImageClassification(unittest.TestCase): - def test_vgg_cuda(self): with self.scope_prog_guard(): main('vgg', use_cuda=True) diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index 16e4a282777..69585b64c89 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -50,26 +50,32 @@ def load_parameter(file_name, h, w): return np.fromfile(f, dtype=np.float32).reshape(h, w) -def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, - **ignored): +def db_lstm( + word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, **ignored +): # 8 features - predicate_embedding = fluid.layers.embedding(input=predicate, - size=[pred_dict_len, word_dim], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='vemb') - - mark_embedding = fluid.layers.embedding(input=mark, - size=[mark_dict_len, mark_dim], - dtype='float32', - is_sparse=IS_SPARSE) + predicate_embedding = fluid.layers.embedding( + input=predicate, + size=[pred_dict_len, word_dim], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='vemb', + ) + + mark_embedding = fluid.layers.embedding( + input=mark, + size=[mark_dict_len, mark_dim], + dtype='float32', + is_sparse=IS_SPARSE, + ) word_input = [word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2] emb_layers = [ - fluid.layers.embedding(size=[word_dict_len, word_dim], - input=x, - param_attr=fluid.ParamAttr(name=embedding_name, - trainable=False)) + fluid.layers.embedding( + size=[word_dict_len, word_dim], + input=x, + param_attr=fluid.ParamAttr(name=embedding_name, trainable=False), + ) for x in word_input ] emb_layers.append(predicate_embedding) @@ -81,141 +87,172 @@ def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, hidden_0 = fluid.layers.sums(input=hidden_0_layers) - lstm_0 = fluid.layers.dynamic_lstm(input=hidden_0, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid') + lstm_0 = fluid.layers.dynamic_lstm( + input=hidden_0, + size=hidden_dim, + candidate_activation='relu', + gate_activation='sigmoid', + cell_activation='sigmoid', + ) # stack L-LSTM and R-LSTM with direct edges input_tmp = [hidden_0, lstm_0] for i in range(1, depth): - mix_hidden = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=hidden_dim), - fluid.layers.fc(input=input_tmp[1], size=hidden_dim) - ]) - - lstm = fluid.layers.dynamic_lstm(input=mix_hidden, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid', - is_reverse=((i % 2) == 1)) + mix_hidden = fluid.layers.sums( + input=[ + fluid.layers.fc(input=input_tmp[0], size=hidden_dim), + fluid.layers.fc(input=input_tmp[1], size=hidden_dim), + ] + ) + + lstm = fluid.layers.dynamic_lstm( + input=mix_hidden, + size=hidden_dim, + candidate_activation='relu', + gate_activation='sigmoid', + cell_activation='sigmoid', + is_reverse=((i % 2) == 1), + ) input_tmp = [mix_hidden, lstm] - feature_out = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=label_dict_len, act='tanh'), - fluid.layers.fc(input=input_tmp[1], size=label_dict_len, act='tanh') - ]) + feature_out = fluid.layers.sums( + input=[ + fluid.layers.fc( + input=input_tmp[0], size=label_dict_len, act='tanh' + ), + fluid.layers.fc( + input=input_tmp[1], size=label_dict_len, act='tanh' + ), + ] + ) return feature_out def train(use_cuda, save_dirname=None, is_local=True): # define network topology - word = fluid.layers.data(name='word_data', - shape=[1], - dtype='int64', - lod_level=1) - predicate = fluid.layers.data(name='verb_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_n2 = fluid.layers.data(name='ctx_n2_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_n1 = fluid.layers.data(name='ctx_n1_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_0 = fluid.layers.data(name='ctx_0_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_p1 = fluid.layers.data(name='ctx_p1_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_p2 = fluid.layers.data(name='ctx_p2_data', - shape=[1], - dtype='int64', - lod_level=1) - mark = fluid.layers.data(name='mark_data', - shape=[1], - dtype='int64', - lod_level=1) + word = fluid.layers.data( + name='word_data', shape=[1], dtype='int64', lod_level=1 + ) + predicate = fluid.layers.data( + name='verb_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_n2 = fluid.layers.data( + name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_n1 = fluid.layers.data( + name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_0 = fluid.layers.data( + name='ctx_0_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_p1 = fluid.layers.data( + name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_p2 = fluid.layers.data( + name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1 + ) + mark = fluid.layers.data( + name='mark_data', shape=[1], dtype='int64', lod_level=1 + ) feature_out = db_lstm(**locals()) - target = fluid.layers.data(name='target', - shape=[1], - dtype='int64', - lod_level=1) - crf_cost = fluid.layers.linear_chain_crf(input=feature_out, - label=target, - param_attr=fluid.ParamAttr( - name='crfw', - learning_rate=mix_hidden_lr)) + target = fluid.layers.data( + name='target', shape=[1], dtype='int64', lod_level=1 + ) + crf_cost = fluid.layers.linear_chain_crf( + input=feature_out, + label=target, + param_attr=fluid.ParamAttr(name='crfw', learning_rate=mix_hidden_lr), + ) avg_cost = paddle.mean(crf_cost) # TODO(qiao) # check other optimizers and check why out will be NAN sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay(learning_rate=0.01, - decay_steps=100000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=0.01, + decay_steps=100000, + decay_rate=0.5, + staircase=True, + ) + ) sgd_optimizer.minimize(avg_cost) # TODO(qiao) # add dependency track and move this config before optimizer crf_decode = fluid.layers.crf_decoding( - input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) + input=feature_out, param_attr=fluid.ParamAttr(name='crfw') + ) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.conll05.test(), buf_size=8192), - batch_size=BATCH_SIZE) + train_data = paddle.batch( + paddle.reader.shuffle(paddle.dataset.conll05.test(), buf_size=8192), + batch_size=BATCH_SIZE, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - feeder = fluid.DataFeeder(feed_list=[ - word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, mark, target - ], - place=place) + feeder = fluid.DataFeeder( + feed_list=[ + word, + ctx_n2, + ctx_n1, + ctx_0, + ctx_p1, + ctx_p2, + predicate, + mark, + target, + ], + place=place, + ) exe = fluid.Executor(place) def train_loop(main_program): exe.run(fluid.default_startup_program()) - embedding_param = fluid.global_scope().find_var( - embedding_name).get_tensor() + embedding_param = ( + fluid.global_scope().find_var(embedding_name).get_tensor() + ) embedding_param.set( load_parameter(conll05.get_embedding(), word_dict_len, word_dim), - place) + place, + ) start_time = time.time() batch_id = 0 for pass_id in range(PASS_NUM): for data in train_data(): - cost = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + cost = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) cost = cost[0] if batch_id % 10 == 0: print("avg_cost:" + str(cost)) if batch_id != 0: - print("second per batch: " + - str((time.time() - start_time) / batch_id)) + print( + "second per batch: " + + str((time.time() - start_time) / batch_id) + ) # Set the threshold low to speed up the CI test if float(cost) < 80.0: if save_dirname is not None: # TODO(liuyiqun): Change the target to crf_decode fluid.io.save_inference_model( - save_dirname, [ - 'word_data', 'verb_data', 'ctx_n2_data', - 'ctx_n1_data', 'ctx_0_data', 'ctx_p1_data', - 'ctx_p2_data', 'mark_data' - ], [feature_out], exe) + save_dirname, + [ + 'word_data', + 'verb_data', + 'ctx_n2_data', + 'ctx_n1_data', + 'ctx_0_data', + 'ctx_p1_data', + 'ctx_p2_data', + 'mark_data', + ], + [feature_out], + exe, + ) return batch_id = batch_id + 1 @@ -241,8 +278,9 @@ def train(use_cuda, save_dirname=None, is_local=True): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -262,8 +300,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # Setup input by creating LoDTensor to represent sequence of words. # Here each word is the basic element of the LoDTensor and the shape of @@ -278,46 +319,30 @@ def infer(use_cuda, save_dirname=None): recursive_seq_lens = [[3, 4, 2]] base_shape = [1] # The range of random integers is [low, high] - word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - pred = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=pred_dict_len - 1) - ctx_n2 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - ctx_n1 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - ctx_0 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - ctx_p1 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - ctx_p2 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - mark = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=mark_dict_len - 1) + word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + pred = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=pred_dict_len - 1 + ) + ctx_n2 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + ctx_n1 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + ctx_0 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + ctx_p1 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + ctx_p2 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + mark = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=mark_dict_len - 1 + ) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. @@ -330,19 +355,21 @@ def infer(use_cuda, save_dirname=None): assert feed_target_names[6] == 'ctx_p2_data' assert feed_target_names[7] == 'mark_data' - results = exe.run(inference_program, - feed={ - feed_target_names[0]: word, - feed_target_names[1]: pred, - feed_target_names[2]: ctx_n2, - feed_target_names[3]: ctx_n1, - feed_target_names[4]: ctx_0, - feed_target_names[5]: ctx_p1, - feed_target_names[6]: ctx_p2, - feed_target_names[7]: mark - }, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={ + feed_target_names[0]: word, + feed_target_names[1]: pred, + feed_target_names[2]: ctx_n2, + feed_target_names[3]: ctx_n1, + feed_target_names[4]: ctx_0, + feed_target_names[5]: ctx_p1, + feed_target_names[6]: ctx_p2, + feed_target_names[7]: mark, + }, + fetch_list=fetch_targets, + return_numpy=False, + ) print(results[0].recursive_sequence_lengths()) np_data = np.array(results[0]) print("Inference Shape: ", np_data.shape) @@ -354,8 +381,9 @@ def main(use_cuda, is_local=True): temp_dir = tempfile.TemporaryDirectory() # Directory for saving the trained model - save_dirname = os.path.join(temp_dir.name, - "label_semantic_roles.inference.model") + save_dirname = os.path.join( + temp_dir.name, "label_semantic_roles.inference.model" + ) train(use_cuda, save_dirname, is_local) infer(use_cuda, save_dirname) @@ -364,7 +392,6 @@ def main(use_cuda, is_local=True): class TestLabelSemanticRoles(unittest.TestCase): - def test_cuda(self): with self.scope_prog_guard(): main(use_cuda=True) diff --git a/python/paddle/fluid/tests/book/test_machine_translation.py b/python/paddle/fluid/tests/book/test_machine_translation.py index d97b474d6ab..4ae6462f02f 100644 --- a/python/paddle/fluid/tests/book/test_machine_translation.py +++ b/python/paddle/fluid/tests/book/test_machine_translation.py @@ -40,15 +40,16 @@ decoder_size = hidden_dim def encoder(is_sparse): # encoder - src_word_id = pd.data(name="src_word_id", - shape=[1], - dtype='int64', - lod_level=1) - src_embedding = pd.embedding(input=src_word_id, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=is_sparse, - param_attr=fluid.ParamAttr(name='vemb')) + src_word_id = pd.data( + name="src_word_id", shape=[1], dtype='int64', lod_level=1 + ) + src_embedding = pd.embedding( + input=src_word_id, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=is_sparse, + param_attr=fluid.ParamAttr(name='vemb'), + ) fc1 = pd.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = pd.dynamic_lstm(input=fc1, size=hidden_dim * 4) @@ -58,27 +59,28 @@ def encoder(is_sparse): def decoder_train(context, is_sparse): # decoder - trg_language_word = pd.data(name="target_language_word", - shape=[1], - dtype='int64', - lod_level=1) - trg_embedding = pd.embedding(input=trg_language_word, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=is_sparse, - param_attr=fluid.ParamAttr(name='vemb')) + trg_language_word = pd.data( + name="target_language_word", shape=[1], dtype='int64', lod_level=1 + ) + trg_embedding = pd.embedding( + input=trg_language_word, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=is_sparse, + param_attr=fluid.ParamAttr(name='vemb'), + ) rnn = pd.DynamicRNN() with rnn.block(): current_word = rnn.step_input(trg_embedding) pre_state = rnn.memory(init=context) - current_state = pd.fc(input=[current_word, pre_state], - size=decoder_size, - act='tanh') + current_state = pd.fc( + input=[current_word, pre_state], size=decoder_size, act='tanh' + ) - current_score = pd.fc(input=current_state, - size=target_dict_dim, - act='softmax') + current_score = pd.fc( + input=current_state, size=target_dict_dim, act='softmax' + ) rnn.update_memory(pre_state, current_state) rnn.output(current_score) @@ -99,10 +101,9 @@ def decoder_decode(context, is_sparse): scores_array = pd.create_array('float32') init_ids = pd.data(name="init_ids", shape=[1], dtype="int64", lod_level=2) - init_scores = pd.data(name="init_scores", - shape=[1], - dtype="float32", - lod_level=2) + init_scores = pd.data( + name="init_scores", shape=[1], dtype="float32", lod_level=2 + ) pd.array_write(init_ids, array=ids_array, i=counter) pd.array_write(init_scores, array=scores_array, i=counter) @@ -118,32 +119,38 @@ def decoder_decode(context, is_sparse): # expand the recursive_sequence_lengths of pre_state to be the same with pre_score pre_state_expanded = pd.sequence_expand(pre_state, pre_score) - pre_ids_emb = pd.embedding(input=pre_ids, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=is_sparse) + pre_ids_emb = pd.embedding( + input=pre_ids, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=is_sparse, + ) # use rnn unit to update rnn - current_state = pd.fc(input=[pre_state_expanded, pre_ids_emb], - size=decoder_size, - act='tanh') + current_state = pd.fc( + input=[pre_state_expanded, pre_ids_emb], + size=decoder_size, + act='tanh', + ) current_state_with_lod = pd.lod_reset(x=current_state, y=pre_score) # use score to do beam search - current_score = pd.fc(input=current_state_with_lod, - size=target_dict_dim, - act='softmax') + current_score = pd.fc( + input=current_state_with_lod, size=target_dict_dim, act='softmax' + ) topk_scores, topk_indices = pd.topk(current_score, k=beam_size) # calculate accumulated scores after topk to reduce computation cost - accu_scores = pd.elementwise_add(x=pd.log(topk_scores), - y=pd.reshape(pre_score, shape=[-1]), - axis=0) - selected_ids, selected_scores = pd.beam_search(pre_ids, - pre_score, - topk_indices, - accu_scores, - beam_size, - end_id=10, - level=0) + accu_scores = pd.elementwise_add( + x=pd.log(topk_scores), y=pd.reshape(pre_score, shape=[-1]), axis=0 + ) + selected_ids, selected_scores = pd.beam_search( + pre_ids, + pre_score, + topk_indices, + accu_scores, + beam_size, + end_id=10, + level=0, + ) pd.increment(x=counter, value=1, in_place=True) @@ -159,7 +166,8 @@ def decoder_decode(context, is_sparse): pd.logical_and(x=length_cond, y=finish_cond, out=cond) translation_ids, translation_scores = pd.beam_search_decode( - ids=ids_array, scores=scores_array, beam_size=beam_size, end_id=10) + ids=ids_array, scores=scores_array, beam_size=beam_size, end_id=10 + ) # return init_ids, init_scores @@ -173,25 +181,31 @@ def train_main(use_cuda, is_sparse, is_local=True): context = encoder(is_sparse) rnn_out = decoder_train(context, is_sparse) - label = pd.data(name="target_language_next_word", - shape=[1], - dtype='int64', - lod_level=1) + label = pd.data( + name="target_language_next_word", shape=[1], dtype='int64', lod_level=1 + ) cost = pd.cross_entropy(input=rnn_out, label=label) avg_cost = pd.mean(cost) optimizer = fluid.optimizer.Adagrad( learning_rate=1e-4, regularization=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) optimizer.minimize(avg_cost) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) feed_order = [ - 'src_word_id', 'target_language_word', 'target_language_next_word' + 'src_word_id', + 'target_language_word', + 'target_language_next_word', ] exe = Executor(place) @@ -207,12 +221,18 @@ def train_main(use_cuda, is_sparse, is_local=True): batch_id = 0 for pass_id in range(1): for data in train_data(): - outs = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + outs = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) avg_cost_val = np.array(outs[0]) - print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + - " avg_cost=" + str(avg_cost_val)) + print( + 'pass_id=' + + str(pass_id) + + ' batch=' + + str(batch_id) + + " avg_cost=" + + str(avg_cost_val) + ) if batch_id > 3: break batch_id += 1 @@ -234,8 +254,9 @@ def train_main(use_cuda, is_sparse, is_local=True): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -254,21 +275,27 @@ def decode_main(use_cuda, is_sparse): exe.run(framework.default_startup_program()) init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64') - init_scores_data = np.array([1. for _ in range(batch_size)], - dtype='float32') + init_scores_data = np.array( + [1.0 for _ in range(batch_size)], dtype='float32' + ) init_ids_data = init_ids_data.reshape((batch_size, 1)) init_scores_data = init_scores_data.reshape((batch_size, 1)) init_recursive_seq_lens = [1] * batch_size init_recursive_seq_lens = [init_recursive_seq_lens, init_recursive_seq_lens] - init_ids = fluid.create_lod_tensor(init_ids_data, init_recursive_seq_lens, - place) - init_scores = fluid.create_lod_tensor(init_scores_data, - init_recursive_seq_lens, place) + init_ids = fluid.create_lod_tensor( + init_ids_data, init_recursive_seq_lens, place + ) + init_scores = fluid.create_lod_tensor( + init_scores_data, init_recursive_seq_lens, place + ) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) feed_order = ['src_word_id'] feed_list = [ @@ -286,7 +313,8 @@ def decode_main(use_cuda, is_sparse): framework.default_main_program(), feed=feed_dict, fetch_list=[translation_ids, translation_scores], - return_numpy=False) + return_numpy=False, + ) print(result_ids.recursive_sequence_lengths()) break @@ -306,8 +334,9 @@ def scope_prog_guard(): def inject_test_train(use_cuda, is_sparse): - f_name = 'test_{0}_{1}_train'.format('cuda' if use_cuda else 'cpu', - 'sparse' if is_sparse else 'dense') + f_name = 'test_{0}_{1}_train'.format( + 'cuda' if use_cuda else 'cpu', 'sparse' if is_sparse else 'dense' + ) def f(*args): with scope_prog_guard(): @@ -317,8 +346,9 @@ def inject_test_train(use_cuda, is_sparse): def inject_test_decode(use_cuda, is_sparse, decorator=None): - f_name = 'test_{0}_{1}_decode'.format('cuda' if use_cuda else 'cpu', - 'sparse' if is_sparse else 'dense') + f_name = 'test_{0}_{1}_decode'.format( + 'cuda' if use_cuda else 'cpu', 'sparse' if is_sparse else 'dense' + ) def f(*args): with scope_prog_guard(): @@ -340,11 +370,12 @@ for _use_cuda_ in (False, True): _decorator_ = None if _use_cuda_: _decorator_ = unittest.skip( - reason='Beam Search does not support CUDA!') + reason='Beam Search does not support CUDA!' + ) - inject_test_decode(is_sparse=_is_sparse_, - use_cuda=_use_cuda_, - decorator=_decorator_) + inject_test_decode( + is_sparse=_is_sparse_, use_cuda=_use_cuda_, decorator=_decorator_ + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/book/test_recognize_digits.py b/python/paddle/fluid/tests/book/test_recognize_digits.py index f2afcdefa06..fd99eb04d99 100644 --- a/python/paddle/fluid/tests/book/test_recognize_digits.py +++ b/python/paddle/fluid/tests/book/test_recognize_digits.py @@ -43,30 +43,36 @@ def mlp(img, label): def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) return loss_net(conv_pool_2, label) -def train(nn_type, - use_cuda, - parallel, - save_dirname=None, - save_full_dirname=None, - model_filename=None, - params_filename=None, - is_local=True): +def train( + nn_type, + use_cuda, + parallel, + save_dirname=None, + save_full_dirname=None, + model_filename=None, + params_filename=None, + is_local=True, +): if use_cuda and not fluid.core.is_compiled_with_cuda(): return img = fluid.layers.data(name='img', shape=[1, 28, 28], dtype='float32') @@ -91,11 +97,13 @@ def train(nn_type, exe = fluid.Executor(place) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=BATCH_SIZE) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=BATCH_SIZE, + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=BATCH_SIZE + ) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) def train_loop(main_program): @@ -113,7 +121,8 @@ def train(nn_type, acc_np, avg_loss_np = exe.run( program=test_program, feed=feeder.feed(test_data), - fetch_list=[acc, avg_loss]) + fetch_list=[acc, avg_loss], + ) acc_set.append(float(acc_np)) avg_loss_set.append(float(avg_loss_np)) # get test acc and loss @@ -123,23 +132,33 @@ def train(nn_type, # Smaller value to increase CI speed if save_dirname is not None: fluid.io.save_inference_model( - save_dirname, ["img"], [prediction], + save_dirname, + ["img"], + [prediction], exe, model_filename=model_filename, - params_filename=params_filename) + params_filename=params_filename, + ) if save_full_dirname is not None: fluid.io.save_inference_model( - save_full_dirname, [], [], + save_full_dirname, + [], + [], exe, model_filename=model_filename, params_filename=params_filename, - export_for_deployment=False) + export_for_deployment=False, + ) return else: print( - 'PassID {0:1}, BatchID {1:04}, Test Loss {2:2.2}, Acc {3:2.2}' - .format(pass_id, batch_id + 1, float(avg_loss_val), - float(acc_val))) + 'PassID {0:1}, BatchID {1:04}, Test Loss {2:2.2}, Acc {3:2.2}'.format( + pass_id, + batch_id + 1, + float(avg_loss_val), + float(acc_val), + ) + ) if math.isnan(float(avg_loss_val)): sys.exit("got NaN loss, training failed.") raise AssertionError("Loss of recognize digits is too large") @@ -161,18 +180,18 @@ def train(nn_type, t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": train_loop(t.get_trainer_program()) -def infer(use_cuda, - save_dirname=None, - model_filename=None, - params_filename=None): +def infer( + use_cuda, save_dirname=None, model_filename=None, params_filename=None +): if save_dirname is None: return @@ -185,22 +204,28 @@ def infer(use_cuda, # the feed_target_names (the names of variables that will be feeded # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe, - model_filename, - params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + save_dirname, exe, model_filename, params_filename + ) # The input's dimension of conv should be 4-D or 5-D. # Use normilized image pixels as input data, which should be in the range [-1.0, 1.0]. batch_size = 1 tensor_img = numpy.random.uniform( - -1.0, 1.0, [batch_size, 1, 28, 28]).astype("float32") + -1.0, 1.0, [batch_size, 1, 28, 28] + ).astype("float32") # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("infer results: ", results[0]) @@ -217,17 +242,21 @@ def main(use_cuda, parallel, nn_type, combine): params_filename = "__params_combined__" # call train() with is_local argument to run distributed train - train(nn_type=nn_type, - use_cuda=use_cuda, - parallel=parallel, - save_dirname=save_dirname, - save_full_dirname=save_full_dirname, - model_filename=model_filename, - params_filename=params_filename) - infer(use_cuda=use_cuda, - save_dirname=save_dirname, - model_filename=model_filename, - params_filename=params_filename) + train( + nn_type=nn_type, + use_cuda=use_cuda, + parallel=parallel, + save_dirname=save_dirname, + save_full_dirname=save_full_dirname, + model_filename=model_filename, + params_filename=params_filename, + ) + infer( + use_cuda=use_cuda, + save_dirname=save_dirname, + model_filename=model_filename, + params_filename=params_filename, + ) class TestRecognizeDigits(unittest.TestCase): @@ -235,7 +264,6 @@ class TestRecognizeDigits(unittest.TestCase): def inject_test_method(use_cuda, parallel, nn_type, combine): - def __impl__(self): prog = fluid.Program() startup_prog = fluid.Program() @@ -244,9 +272,12 @@ def inject_test_method(use_cuda, parallel, nn_type, combine): with fluid.program_guard(prog, startup_prog): main(use_cuda, parallel, nn_type, combine) - fn = 'test_{0}_{1}_{2}_{3}'.format(nn_type, 'cuda' if use_cuda else 'cpu', - 'parallel' if parallel else 'normal', - 'combine' if combine else 'separate') + fn = 'test_{0}_{1}_{2}_{3}'.format( + nn_type, + 'cuda' if use_cuda else 'cpu', + 'parallel' if parallel else 'normal', + 'combine' if combine else 'separate', + ) setattr(TestRecognizeDigits, fn, __impl__) @@ -255,7 +286,7 @@ def inject_all_tests(): for use_cuda in (False, True): if use_cuda and not core.is_compiled_with_cuda(): continue - for parallel in (False, ): + for parallel in (False,): for nn_type in ('mlp', 'conv'): inject_test_method(use_cuda, parallel, nn_type, True) diff --git a/python/paddle/fluid/tests/book/test_recommender_system.py b/python/paddle/fluid/tests/book/test_recommender_system.py index 3d9e14a7a18..132ee1b5214 100644 --- a/python/paddle/fluid/tests/book/test_recommender_system.py +++ b/python/paddle/fluid/tests/book/test_recommender_system.py @@ -40,11 +40,13 @@ def get_usr_combined_features(): uid = layers.data(name='user_id', shape=[1], dtype='int64') - usr_emb = layers.embedding(input=uid, - dtype='float32', - size=[USR_DICT_SIZE, 32], - param_attr='user_table', - is_sparse=IS_SPARSE) + usr_emb = layers.embedding( + input=uid, + dtype='float32', + size=[USR_DICT_SIZE, 32], + param_attr='user_table', + is_sparse=IS_SPARSE, + ) usr_fc = layers.fc(input=usr_emb, size=32) @@ -52,35 +54,42 @@ def get_usr_combined_features(): usr_gender_id = layers.data(name='gender_id', shape=[1], dtype='int64') - usr_gender_emb = layers.embedding(input=usr_gender_id, - size=[USR_GENDER_DICT_SIZE, 16], - param_attr='gender_table', - is_sparse=IS_SPARSE) + usr_gender_emb = layers.embedding( + input=usr_gender_id, + size=[USR_GENDER_DICT_SIZE, 16], + param_attr='gender_table', + is_sparse=IS_SPARSE, + ) usr_gender_fc = layers.fc(input=usr_gender_emb, size=16) USR_AGE_DICT_SIZE = len(paddle.dataset.movielens.age_table) usr_age_id = layers.data(name='age_id', shape=[1], dtype="int64") - usr_age_emb = layers.embedding(input=usr_age_id, - size=[USR_AGE_DICT_SIZE, 16], - is_sparse=IS_SPARSE, - param_attr='age_table') + usr_age_emb = layers.embedding( + input=usr_age_id, + size=[USR_AGE_DICT_SIZE, 16], + is_sparse=IS_SPARSE, + param_attr='age_table', + ) usr_age_fc = layers.fc(input=usr_age_emb, size=16) USR_JOB_DICT_SIZE = paddle.dataset.movielens.max_job_id() + 1 usr_job_id = layers.data(name='job_id', shape=[1], dtype="int64") - usr_job_emb = layers.embedding(input=usr_job_id, - size=[USR_JOB_DICT_SIZE, 16], - param_attr='job_table', - is_sparse=IS_SPARSE) + usr_job_emb = layers.embedding( + input=usr_job_id, + size=[USR_JOB_DICT_SIZE, 16], + param_attr='job_table', + is_sparse=IS_SPARSE, + ) usr_job_fc = layers.fc(input=usr_job_emb, size=16) concat_embed = layers.concat( - input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc], axis=1) + input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc], axis=1 + ) usr_combined_features = layers.fc(input=concat_embed, size=200, act="tanh") @@ -93,47 +102,51 @@ def get_mov_combined_features(): mov_id = layers.data(name='movie_id', shape=[1], dtype='int64') - mov_emb = layers.embedding(input=mov_id, - dtype='float32', - size=[MOV_DICT_SIZE, 32], - param_attr='movie_table', - is_sparse=IS_SPARSE) + mov_emb = layers.embedding( + input=mov_id, + dtype='float32', + size=[MOV_DICT_SIZE, 32], + param_attr='movie_table', + is_sparse=IS_SPARSE, + ) mov_fc = layers.fc(input=mov_emb, size=32) CATEGORY_DICT_SIZE = len(paddle.dataset.movielens.movie_categories()) - category_id = layers.data(name='category_id', - shape=[1], - dtype='int64', - lod_level=1) + category_id = layers.data( + name='category_id', shape=[1], dtype='int64', lod_level=1 + ) - mov_categories_emb = layers.embedding(input=category_id, - size=[CATEGORY_DICT_SIZE, 32], - is_sparse=IS_SPARSE) + mov_categories_emb = layers.embedding( + input=category_id, size=[CATEGORY_DICT_SIZE, 32], is_sparse=IS_SPARSE + ) - mov_categories_hidden = layers.sequence_pool(input=mov_categories_emb, - pool_type="sum") + mov_categories_hidden = layers.sequence_pool( + input=mov_categories_emb, pool_type="sum" + ) MOV_TITLE_DICT_SIZE = len(paddle.dataset.movielens.get_movie_title_dict()) - mov_title_id = layers.data(name='movie_title', - shape=[1], - dtype='int64', - lod_level=1) + mov_title_id = layers.data( + name='movie_title', shape=[1], dtype='int64', lod_level=1 + ) - mov_title_emb = layers.embedding(input=mov_title_id, - size=[MOV_TITLE_DICT_SIZE, 32], - is_sparse=IS_SPARSE) + mov_title_emb = layers.embedding( + input=mov_title_id, size=[MOV_TITLE_DICT_SIZE, 32], is_sparse=IS_SPARSE + ) - mov_title_conv = nets.sequence_conv_pool(input=mov_title_emb, - num_filters=32, - filter_size=3, - act="tanh", - pool_type="sum") + mov_title_conv = nets.sequence_conv_pool( + input=mov_title_emb, + num_filters=32, + filter_size=3, + act="tanh", + pool_type="sum", + ) concat_embed = layers.concat( - input=[mov_fc, mov_categories_hidden, mov_title_conv], axis=1) + input=[mov_fc, mov_categories_hidden, mov_title_conv], axis=1 + ) # FIXME(dzh) : need tanh operator mov_combined_features = layers.fc(input=concat_embed, size=200, act="tanh") @@ -169,15 +182,23 @@ def train(use_cuda, save_dirname, is_local=True): exe = Executor(place) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.movielens.train(), buf_size=8192), - batch_size=BATCH_SIZE) - test_reader = paddle.batch(paddle.dataset.movielens.test(), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.movielens.train(), buf_size=8192), + batch_size=BATCH_SIZE, + ) + test_reader = paddle.batch( + paddle.dataset.movielens.test(), batch_size=BATCH_SIZE + ) feed_order = [ - 'user_id', 'gender_id', 'age_id', 'job_id', 'movie_id', 'category_id', - 'movie_title', 'score' + 'user_id', + 'gender_id', + 'age_id', + 'job_id', + 'movie_id', + 'category_id', + 'movie_title', + 'score', ] def train_loop(main_program): @@ -192,16 +213,20 @@ def train(use_cuda, save_dirname, is_local=True): for pass_id in range(PASS_NUM): for batch_id, data in enumerate(train_reader()): # train a mini-batch - outs = exe.run(program=main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + outs = exe.run( + program=main_program, + feed=feeder.feed(data), + fetch_list=[avg_cost], + ) out = np.array(outs[0]) if (batch_id + 1) % 10 == 0: avg_cost_set = [] for test_data in test_reader(): - avg_cost_np = exe.run(program=test_program, - feed=feeder.feed(test_data), - fetch_list=[avg_cost]) + avg_cost_np = exe.run( + program=test_program, + feed=feeder.feed(test_data), + fetch_list=[avg_cost], + ) avg_cost_set.append(avg_cost_np[0]) break # test only 1 segment for speeding up CI @@ -211,10 +236,19 @@ def train(use_cuda, save_dirname, is_local=True): # if avg_cost less than 6.0, we think our code is good. if save_dirname is not None: fluid.io.save_inference_model( - save_dirname, [ - "user_id", "gender_id", "age_id", "job_id", - "movie_id", "category_id", "movie_title" - ], [scale_infer], exe) + save_dirname, + [ + "user_id", + "gender_id", + "age_id", + "job_id", + "movie_id", + "category_id", + "movie_title", + ], + [scale_infer], + exe, + ) return if math.isnan(float(out[0])): @@ -237,8 +271,9 @@ def train(use_cuda, save_dirname, is_local=True): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -258,8 +293,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # Use the first data from paddle.dataset.movielens.test() as input assert feed_target_names[0] == "user_id" @@ -288,27 +326,32 @@ def infer(use_cuda, save_dirname=None): assert feed_target_names[5] == "category_id" category_id = fluid.create_lod_tensor( - [np.array([10, 8, 9], dtype='int64')], [[3]], place) + [np.array([10, 8, 9], dtype='int64')], [[3]], place + ) assert feed_target_names[6] == "movie_title" movie_title = fluid.create_lod_tensor( - [np.array([1069, 4140, 2923, 710, 988], dtype='int64')], [[5]], - place) + [np.array([1069, 4140, 2923, 710, 988], dtype='int64')], + [[5]], + place, + ) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={ - feed_target_names[0]: user_id, - feed_target_names[1]: gender_id, - feed_target_names[2]: age_id, - feed_target_names[3]: job_id, - feed_target_names[4]: movie_id, - feed_target_names[5]: category_id, - feed_target_names[6]: movie_title - }, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={ + feed_target_names[0]: user_id, + feed_target_names[1]: gender_id, + feed_target_names[2]: age_id, + feed_target_names[3]: job_id, + feed_target_names[4]: movie_id, + feed_target_names[5]: category_id, + feed_target_names[6]: movie_title, + }, + fetch_list=fetch_targets, + return_numpy=False, + ) print("inferred score: ", np.array(results[0])) @@ -318,8 +361,9 @@ def main(use_cuda): # Directory for saving the inference model temp_dir = tempfile.TemporaryDirectory() - save_dirname = os.path.join(temp_dir.name, - "recommender_system.inference.model") + save_dirname = os.path.join( + temp_dir.name, "recommender_system.inference.model" + ) train(use_cuda, save_dirname) infer(use_cuda, save_dirname) diff --git a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py index b7de6f5da90..f896e244975 100644 --- a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py +++ b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py @@ -41,19 +41,23 @@ USE_PEEPHOLES = False def bi_lstm_encoder(input_seq, hidden_size): - input_forward_proj = fluid.layers.fc(input=input_seq, - size=hidden_size * 4, - bias_attr=True) - forward, _ = fluid.layers.dynamic_lstm(input=input_forward_proj, - size=hidden_size * 4, - use_peepholes=USE_PEEPHOLES) - input_backward_proj = fluid.layers.fc(input=input_seq, - size=hidden_size * 4, - bias_attr=True) - backward, _ = fluid.layers.dynamic_lstm(input=input_backward_proj, - size=hidden_size * 4, - is_reverse=True, - use_peepholes=USE_PEEPHOLES) + input_forward_proj = fluid.layers.fc( + input=input_seq, size=hidden_size * 4, bias_attr=True + ) + forward, _ = fluid.layers.dynamic_lstm( + input=input_forward_proj, + size=hidden_size * 4, + use_peepholes=USE_PEEPHOLES, + ) + input_backward_proj = fluid.layers.fc( + input=input_seq, size=hidden_size * 4, bias_attr=True + ) + backward, _ = fluid.layers.dynamic_lstm( + input=input_backward_proj, + size=hidden_size * 4, + is_reverse=True, + use_peepholes=USE_PEEPHOLES, + ) forward_last = fluid.layers.sequence_last_step(input=forward) backward_first = fluid.layers.sequence_first_step(input=backward) @@ -63,7 +67,6 @@ def bi_lstm_encoder(input_seq, hidden_size): # FIXME(peterzhang2029): Replace this function with the lstm_unit_op. def lstm_step(x_t, hidden_t_prev, cell_t_prev, size): - def linear(inputs): return fluid.layers.fc(input=inputs, size=size, bias_attr=True) @@ -72,26 +75,28 @@ def lstm_step(x_t, hidden_t_prev, cell_t_prev, size): output_gate = fluid.layers.sigmoid(x=linear([hidden_t_prev, x_t])) cell_tilde = fluid.layers.tanh(x=linear([hidden_t_prev, x_t])) - cell_t = fluid.layers.sums(input=[ - fluid.layers.elementwise_mul(x=forget_gate, y=cell_t_prev), - fluid.layers.elementwise_mul(x=input_gate, y=cell_tilde) - ]) + cell_t = fluid.layers.sums( + input=[ + fluid.layers.elementwise_mul(x=forget_gate, y=cell_t_prev), + fluid.layers.elementwise_mul(x=input_gate, y=cell_tilde), + ] + ) - hidden_t = fluid.layers.elementwise_mul(x=output_gate, - y=fluid.layers.tanh(x=cell_t)) + hidden_t = fluid.layers.elementwise_mul( + x=output_gate, y=fluid.layers.tanh(x=cell_t) + ) return hidden_t, cell_t -def lstm_decoder_without_attention(target_embedding, decoder_boot, context, - decoder_size): +def lstm_decoder_without_attention( + target_embedding, decoder_boot, context, decoder_size +): rnn = fluid.layers.DynamicRNN() cell_init = fluid.layers.fill_constant_batch_size_like( - input=decoder_boot, - value=0.0, - shape=[-1, decoder_size], - dtype='float32') + input=decoder_boot, value=0.0, shape=[-1, decoder_size], dtype='float32' + ) cell_init.stop_gradient = False with rnn.block(): @@ -100,15 +105,15 @@ def lstm_decoder_without_attention(target_embedding, decoder_boot, context, hidden_mem = rnn.memory(init=decoder_boot, need_reorder=True) cell_mem = rnn.memory(init=cell_init) - decoder_inputs = fluid.layers.concat(input=[context, current_word], - axis=1) + decoder_inputs = fluid.layers.concat( + input=[context, current_word], axis=1 + ) h, c = lstm_step(decoder_inputs, hidden_mem, cell_mem, decoder_size) rnn.update_memory(hidden_mem, h) rnn.update_memory(cell_mem, c) - out = fluid.layers.fc(input=h, - size=target_dict_dim, - bias_attr=True, - act='softmax') + out = fluid.layers.fc( + input=h, size=target_dict_dim, bias_attr=True, act='softmax' + ) rnn.output(out) return rnn() @@ -116,43 +121,44 @@ def lstm_decoder_without_attention(target_embedding, decoder_boot, context, def seq_to_seq_net(): """Construct a seq2seq network.""" - src_word_idx = fluid.layers.data(name='source_sequence', - shape=[1], - dtype='int64', - lod_level=1) + src_word_idx = fluid.layers.data( + name='source_sequence', shape=[1], dtype='int64', lod_level=1 + ) src_embedding = fluid.layers.embedding( input=src_word_idx, size=[source_dict_dim, embedding_dim], - dtype='float32') + dtype='float32', + ) src_forward_last, src_backward_first = bi_lstm_encoder( - input_seq=src_embedding, hidden_size=encoder_size) + input_seq=src_embedding, hidden_size=encoder_size + ) encoded_vector = fluid.layers.concat( - input=[src_forward_last, src_backward_first], axis=1) + input=[src_forward_last, src_backward_first], axis=1 + ) - decoder_boot = fluid.layers.fc(input=src_backward_first, - size=decoder_size, - bias_attr=False, - act='tanh') + decoder_boot = fluid.layers.fc( + input=src_backward_first, size=decoder_size, bias_attr=False, act='tanh' + ) - trg_word_idx = fluid.layers.data(name='target_sequence', - shape=[1], - dtype='int64', - lod_level=1) + trg_word_idx = fluid.layers.data( + name='target_sequence', shape=[1], dtype='int64', lod_level=1 + ) trg_embedding = fluid.layers.embedding( input=trg_word_idx, size=[target_dict_dim, embedding_dim], - dtype='float32') - - prediction = lstm_decoder_without_attention(trg_embedding, decoder_boot, - encoded_vector, decoder_size) - label = fluid.layers.data(name='label_sequence', - shape=[1], - dtype='int64', - lod_level=1) + dtype='float32', + ) + + prediction = lstm_decoder_without_attention( + trg_embedding, decoder_boot, encoded_vector, decoder_size + ) + label = fluid.layers.data( + name='label_sequence', shape=[1], dtype='int64', lod_level=1 + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) avg_cost = paddle.mean(cost) @@ -165,9 +171,12 @@ def train(use_cuda, save_dirname=None): optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) @@ -183,20 +192,31 @@ def train(use_cuda, save_dirname=None): batch_id = 0 for pass_id in range(2): for data in train_data(): - outs = exe.run(framework.default_main_program(), - feed=feeder.feed(data), - fetch_list=[avg_cost]) + outs = exe.run( + framework.default_main_program(), + feed=feeder.feed(data), + fetch_list=[avg_cost], + ) avg_cost_val = np.array(outs[0]) - print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + - " avg_cost=" + str(avg_cost_val)) + print( + 'pass_id=' + + str(pass_id) + + ' batch=' + + str(batch_id) + + " avg_cost=" + + str(avg_cost_val) + ) if math.isnan(float(avg_cost_val[0])): sys.exit("got NaN loss, training failed.") if batch_id > 3: if save_dirname is not None: fluid.io.save_inference_model( - save_dirname, ['source_sequence', 'target_sequence'], - [prediction], exe) + save_dirname, + ['source_sequence', 'target_sequence'], + [prediction], + exe, + ) return batch_id += 1 @@ -215,8 +235,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # Setup input by creating LoDTensor to represent sequence of words. # Here each word is the basic element of the LoDTensor and the shape of @@ -231,28 +254,26 @@ def infer(use_cuda, save_dirname=None): recursive_seq_lens = [[4, 6]] base_shape = [1] # The range of random integers is [low, high] - word_data = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=1) - trg_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=1) + word_data = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=1 + ) + trg_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=1 + ) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. assert feed_target_names[0] == 'source_sequence' assert feed_target_names[1] == 'target_sequence' - results = exe.run(inference_program, - feed={ - feed_target_names[0]: word_data, - feed_target_names[1]: trg_word, - }, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={ + feed_target_names[0]: word_data, + feed_target_names[1]: trg_word, + }, + fetch_list=fetch_targets, + return_numpy=False, + ) print(results[0].recursive_sequence_lengths()) np_data = np.array(results[0]) print("Inference shape: ", np_data.shape) @@ -265,8 +286,9 @@ def main(use_cuda): # Directory for saving the trained model temp_dir = tempfile.TemporaryDirectory() - save_dirname = os.path.join(temp_dir.name, - "rnn_encoder_decoder.inference.model") + save_dirname = os.path.join( + temp_dir.name, "rnn_encoder_decoder.inference.model" + ) train(use_cuda, save_dirname) infer(use_cuda, save_dirname) @@ -274,7 +296,6 @@ def main(use_cuda): class TestRnnEncoderDecoder(unittest.TestCase): - def test_cuda(self): with self.scope_prog_guard(): main(use_cuda=True) diff --git a/python/paddle/fluid/tests/book/test_word2vec_book.py b/python/paddle/fluid/tests/book/test_word2vec_book.py index ec7ab9cc837..69cef4021c9 100644 --- a/python/paddle/fluid/tests/book/test_word2vec_book.py +++ b/python/paddle/fluid/tests/book/test_word2vec_book.py @@ -33,17 +33,21 @@ def get_place(target): return fluid.CPUPlace() else: raise ValueError( - "Target `{0}` is not on the support list: `cuda`, `xpu` and `cpu`.". - format(target)) - - -def train(target, - is_sparse, - is_parallel, - save_dirname, - is_local=True, - use_bf16=False, - pure_bf16=False): + "Target `{0}` is not on the support list: `cuda`, `xpu` and `cpu`.".format( + target + ) + ) + + +def train( + target, + is_sparse, + is_parallel, + save_dirname, + is_local=True, + use_bf16=False, + pure_bf16=False, +): PASS_NUM = 100 EMBED_SIZE = 32 HIDDEN_SIZE = 256 @@ -52,35 +56,44 @@ def train(target, IS_SPARSE = is_sparse def __network__(words): - embed_first = fluid.layers.embedding(input=words[0], - size=[dict_size, EMBED_SIZE], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='shared_w') - embed_second = fluid.layers.embedding(input=words[1], - size=[dict_size, EMBED_SIZE], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='shared_w') - embed_third = fluid.layers.embedding(input=words[2], - size=[dict_size, EMBED_SIZE], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='shared_w') - embed_forth = fluid.layers.embedding(input=words[3], - size=[dict_size, EMBED_SIZE], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='shared_w') + embed_first = fluid.layers.embedding( + input=words[0], + size=[dict_size, EMBED_SIZE], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='shared_w', + ) + embed_second = fluid.layers.embedding( + input=words[1], + size=[dict_size, EMBED_SIZE], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='shared_w', + ) + embed_third = fluid.layers.embedding( + input=words[2], + size=[dict_size, EMBED_SIZE], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='shared_w', + ) + embed_forth = fluid.layers.embedding( + input=words[3], + size=[dict_size, EMBED_SIZE], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='shared_w', + ) concat_embed = fluid.layers.concat( - input=[embed_first, embed_second, embed_third, embed_forth], axis=1) - hidden1 = fluid.layers.fc(input=concat_embed, - size=HIDDEN_SIZE, - act='sigmoid') - predict_word = fluid.layers.fc(input=hidden1, - size=dict_size, - act='softmax') + input=[embed_first, embed_second, embed_third, embed_forth], axis=1 + ) + hidden1 = fluid.layers.fc( + input=concat_embed, size=HIDDEN_SIZE, act='sigmoid' + ) + predict_word = fluid.layers.fc( + input=hidden1, size=dict_size, act='softmax' + ) cost = fluid.layers.cross_entropy(input=predict_word, label=words[4]) avg_cost = paddle.mean(cost) return avg_cost, predict_word @@ -96,7 +109,8 @@ def train(target, if not is_parallel: avg_cost, predict_word = __network__( - [first_word, second_word, third_word, forth_word, next_word]) + [first_word, second_word, third_word, forth_word, next_word] + ) else: raise NotImplementedError() @@ -105,20 +119,24 @@ def train(target, sgd_optimizer = paddle.static.amp.bf16.decorate_bf16( sgd_optimizer, amp_lists=paddle.static.amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'softmax', 'concat'}, ), + custom_fp32_list={'softmax', 'concat'}, + ), use_bf16_guard=False, - use_pure_bf16=pure_bf16) + use_pure_bf16=pure_bf16, + ) sgd_optimizer.minimize(avg_cost, fluid.default_startup_program()) - train_reader = paddle.batch(paddle.dataset.imikolov.train(word_dict, N), - BATCH_SIZE) + train_reader = paddle.batch( + paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE + ) place = get_place(target) exe = fluid.Executor(place) feeder = fluid.DataFeeder( feed_list=[first_word, second_word, third_word, forth_word, next_word], - place=place) + place=place, + ) def train_loop(main_program): exe.run(fluid.default_startup_program()) @@ -127,15 +145,17 @@ def train(target, for pass_id in range(PASS_NUM): for data in train_reader(): - avg_cost_np = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + avg_cost_np = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) if avg_cost_np[0] < 5.0: if save_dirname is not None and not pure_bf16: fluid.io.save_inference_model( save_dirname, ['firstw', 'secondw', 'thirdw', 'forthw'], - [predict_word], exe) + [predict_word], + exe, + ) return if math.isnan(float(avg_cost_np[0])): sys.exit("got NaN loss, training failed.") @@ -159,8 +179,9 @@ def train(target, t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -179,8 +200,11 @@ def infer(target, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) word_dict = paddle.dataset.imikolov.build_dict() dict_size = len(word_dict) @@ -195,26 +219,18 @@ def infer(target, save_dirname=None): recursive_seq_lens = [[1]] base_shape = [1] # The range of random integers is [low, high] - first_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=dict_size - 1) - second_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=dict_size - 1) - third_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=dict_size - 1) - fourth_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=dict_size - 1) + first_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=dict_size - 1 + ) + second_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=dict_size - 1 + ) + third_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=dict_size - 1 + ) + fourth_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=dict_size - 1 + ) assert feed_target_names[0] == 'firstw' assert feed_target_names[1] == 'secondw' @@ -223,15 +239,17 @@ def infer(target, save_dirname=None): # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={ - feed_target_names[0]: first_word, - feed_target_names[1]: second_word, - feed_target_names[2]: third_word, - feed_target_names[3]: fourth_word - }, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={ + feed_target_names[0]: first_word, + feed_target_names[1]: second_word, + feed_target_names[2]: third_word, + feed_target_names[3]: fourth_word, + }, + fetch_list=fetch_targets, + return_numpy=False, + ) def to_infer_tensor(lod_tensor): infer_tensor = fluid.core.PaddleTensor() @@ -282,18 +300,26 @@ def main(target, is_sparse, is_parallel, use_bf16, pure_bf16): # so only inference is turned on. train("cpu", is_sparse, is_parallel, save_dirname) else: - train(target, - is_sparse, - is_parallel, - save_dirname, - use_bf16=use_bf16, - pure_bf16=pure_bf16) + train( + target, + is_sparse, + is_parallel, + save_dirname, + use_bf16=use_bf16, + pure_bf16=pure_bf16, + ) infer(target, save_dirname) temp_dir.cleanup() -FULL_TEST = os.getenv('FULL_TEST', - '0').lower() in ['true', '1', 't', 'y', 'yes', 'on'] +FULL_TEST = os.getenv('FULL_TEST', '0').lower() in [ + 'true', + '1', + 't', + 'y', + 'yes', + 'on', +] SKIP_REASON = "Only run minimum number of tests in CI server, to make CI faster" @@ -301,15 +327,15 @@ class W2VTest(unittest.TestCase): pass -def inject_test_method(target, - is_sparse, - is_parallel, - use_bf16=False, - pure_bf16=False): +def inject_test_method( + target, is_sparse, is_parallel, use_bf16=False, pure_bf16=False +): fn_name = "test_{0}_{1}_{2}{3}".format( - target, "sparse" if is_sparse else "dense", + target, + "sparse" if is_sparse else "dense", "parallel" if is_parallel else "normal", - "_purebf16" if pure_bf16 else "_bf16" if use_bf16 else "") + "_purebf16" if pure_bf16 else "_bf16" if use_bf16 else "", + ) def __impl__(*args, **kwargs): prog = fluid.Program() @@ -319,20 +345,22 @@ def inject_test_method(target, with fluid.program_guard(prog, startup_prog): main(target, is_sparse, is_parallel, use_bf16, pure_bf16) - if (not fluid.core.is_compiled_with_cuda() - or target == "cuda") and is_sparse: + if ( + not fluid.core.is_compiled_with_cuda() or target == "cuda" + ) and is_sparse: fn = __impl__ else: # skip the other test when on CI server - fn = unittest.skipUnless(condition=FULL_TEST, - reason=SKIP_REASON)(__impl__) + fn = unittest.skipUnless(condition=FULL_TEST, reason=SKIP_REASON)( + __impl__ + ) setattr(W2VTest, fn_name, fn) for target in ("cuda", "cpu", "xpu"): for is_sparse in (False, True): - for is_parallel in (False, ): + for is_parallel in (False,): inject_test_method(target, is_sparse, is_parallel) inject_test_method("cpu", False, False, True) inject_test_method("cpu", False, False, True, True) diff --git a/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_c_setup.py b/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_c_setup.py index 39d47d6f448..34110b6395c 100644 --- a/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_c_setup.py +++ b/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_c_setup.py @@ -24,7 +24,6 @@ from setuptools.command.build_ext import build_ext # cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid # for C/ObjC but not for C++ class BuildExt(build_ext): - def build_extensions(self): if '-Wstrict-prototypes' in self.compiler.compiler_so: self.compiler.compiler_so.remove('-Wstrict-prototypes') @@ -48,8 +47,9 @@ paddle_custom_kernel_include = [ os.path.join(site_packages_path, 'paddle', 'include'), ] # include path third_party -compile_third_party_path = os.path.join(os.environ['PADDLE_BINARY_DIR'], - 'third_party') +compile_third_party_path = os.path.join( + os.environ['PADDLE_BINARY_DIR'], 'third_party' +) paddle_custom_kernel_include += [ os.path.join(compile_third_party_path, 'install/gflags/include'), # gflags os.path.join(compile_third_party_path, 'install/glog/include'), # glog @@ -69,10 +69,13 @@ custom_kernel_dot_module = Extension( include_dirs=paddle_custom_kernel_include, library_dirs=paddle_custom_kernel_library_dir, libraries=libs, - extra_compile_args=paddle_extra_compile_args) + extra_compile_args=paddle_extra_compile_args, +) -setup(name='custom_kernel_dot_c', - version='1.0', - description='custom kernel fot compiling', - cmdclass={'build_ext': BuildExt}, - ext_modules=[custom_kernel_dot_module]) +setup( + name='custom_kernel_dot_c', + version='1.0', + description='custom kernel fot compiling', + cmdclass={'build_ext': BuildExt}, + ext_modules=[custom_kernel_dot_module], +) diff --git a/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_setup.py b/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_setup.py index 8b953ca8725..0ed5610cda4 100644 --- a/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_setup.py +++ b/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_setup.py @@ -24,7 +24,6 @@ from setuptools.command.build_ext import build_ext # cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid # for C/ObjC but not for C++ class BuildExt(build_ext): - def build_extensions(self): if '-Wstrict-prototypes' in self.compiler.compiler_so: self.compiler.compiler_so.remove('-Wstrict-prototypes') @@ -45,12 +44,15 @@ if core.is_compiled_with_npu(): # include path site_packages_path = site.getsitepackages() paddle_custom_kernel_include = list( - map(lambda path: os.path.join(path, 'paddle', 'include'), - site_packages_path)) + map( + lambda path: os.path.join(path, 'paddle', 'include'), site_packages_path + ) +) # include path third_party -compile_third_party_path = os.path.join(os.environ['PADDLE_BINARY_DIR'], - 'third_party') +compile_third_party_path = os.path.join( + os.environ['PADDLE_BINARY_DIR'], 'third_party' +) paddle_custom_kernel_include += [ os.path.join(compile_third_party_path, 'install/gflags/include'), # gflags os.path.join(compile_third_party_path, 'install/glog/include'), # glog @@ -58,7 +60,8 @@ paddle_custom_kernel_include += [ # libs path paddle_custom_kernel_library_dir = list( - map(lambda path: os.path.join(path, 'paddle', 'fluid'), site_packages_path)) + map(lambda path: os.path.join(path, 'paddle', 'fluid'), site_packages_path) +) # libs libs = [':libpaddle.so'] @@ -69,10 +72,13 @@ custom_kernel_dot_module = Extension( include_dirs=paddle_custom_kernel_include, library_dirs=paddle_custom_kernel_library_dir, libraries=libs, - extra_compile_args=paddle_extra_compile_args) + extra_compile_args=paddle_extra_compile_args, +) -setup(name='custom_kernel_dot', - version='1.0', - description='custom kernel fot compiling', - cmdclass={'build_ext': BuildExt}, - ext_modules=[custom_kernel_dot_module]) +setup( + name='custom_kernel_dot', + version='1.0', + description='custom kernel fot compiling', + cmdclass={'build_ext': BuildExt}, + ext_modules=[custom_kernel_dot_module], +) diff --git a/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_dot.py b/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_dot.py index 8badd5dfbde..6d941699dd9 100644 --- a/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_dot.py +++ b/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_dot.py @@ -20,14 +20,16 @@ import numpy as np # use dot as test case. class TestCustomKernelDot(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) # --inplace to place output so file to current dir - cmd = 'cd {} && {} custom_kernel_dot_setup.py build_ext --inplace'.format( - cur_dir, sys.executable) + cmd = ( + 'cd {} && {} custom_kernel_dot_setup.py build_ext --inplace'.format( + cur_dir, sys.executable + ) + ) os.system(cmd) def test_custom_kernel_dot_run(self): @@ -37,6 +39,7 @@ class TestCustomKernelDot(unittest.TestCase): result = np.sum(x_data * y_data, axis=1).reshape([2, 1]) import paddle + paddle.set_device('cpu') x = paddle.to_tensor(x_data) y = paddle.to_tensor(y_data) @@ -46,18 +49,20 @@ class TestCustomKernelDot(unittest.TestCase): out.numpy(), result, err_msg='custom kernel dot out: {},\n numpy dot out: {}'.format( - out.numpy(), result)) + out.numpy(), result + ), + ) class TestCustomKernelDotC(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) # --inplace to place output so file to current dir cmd = 'cd {} && {} custom_kernel_dot_c_setup.py build_ext --inplace'.format( - cur_dir, sys.executable) + cur_dir, sys.executable + ) os.system(cmd) def test_custom_kernel_dot_run(self): @@ -67,6 +72,7 @@ class TestCustomKernelDotC(unittest.TestCase): result = np.sum(x_data * y_data, axis=1).reshape([2, 1]) import paddle + paddle.set_device('cpu') x = paddle.to_tensor(x_data) y = paddle.to_tensor(y_data) @@ -76,7 +82,9 @@ class TestCustomKernelDotC(unittest.TestCase): out.numpy(), result, err_msg='custom kernel dot out: {},\n numpy dot out: {}'.format( - out.numpy(), result)) + out.numpy(), result + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_load.py b/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_load.py index ff7ff3e04a8..1ebeb6a4585 100644 --- a/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_load.py +++ b/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_load.py @@ -20,22 +20,25 @@ import numpy as np class TestCustomKernelLoad(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) # --inplace to place output so file to current dir - cmd = 'cd {} && {} custom_kernel_dot_setup.py build_ext --inplace'.format( - cur_dir, sys.executable) + cmd = ( + 'cd {} && {} custom_kernel_dot_setup.py build_ext --inplace'.format( + cur_dir, sys.executable + ) + ) os.system(cmd) # get paddle lib path and place so paddle_lib_path = '' - site_dirs = site.getsitepackages() if hasattr( - site, 'getsitepackages') else [ - x for x in sys.path if 'site-packages' in x - ] + site_dirs = ( + site.getsitepackages() + if hasattr(site, 'getsitepackages') + else [x for x in sys.path if 'site-packages' in x] + ) for site_dir in site_dirs: lib_dir = os.path.sep.join([site_dir, 'paddle', 'libs']) if os.path.exists(lib_dir): @@ -47,10 +50,12 @@ class TestCustomKernelLoad(unittest.TestCase): if os.path.exists(lib_dir): paddle_lib_path = lib_dir self.default_path = os.path.sep.join( - [paddle_lib_path, '..', '..', 'paddle-plugins']) + [paddle_lib_path, '..', '..', 'paddle-plugins'] + ) # copy so to default path - cmd = 'mkdir -p {} && cp ./*.so {}'.format(self.default_path, - self.default_path) + cmd = 'mkdir -p {} && cp ./*.so {}'.format( + self.default_path, self.default_path + ) os.system(cmd) # wait def test_custom_kernel_dot_load(self): @@ -60,6 +65,7 @@ class TestCustomKernelLoad(unittest.TestCase): result = np.sum(x_data * y_data, axis=1).reshape([2, 1]) import paddle + paddle.set_device('cpu') x = paddle.to_tensor(x_data) y = paddle.to_tensor(y_data) @@ -69,7 +75,9 @@ class TestCustomKernelLoad(unittest.TestCase): out.numpy(), result, err_msg='custom kernel dot out: {},\n numpy dot out: {}'.format( - out.numpy(), result)) + out.numpy(), result + ), + ) def tearDown(self): cmd = 'rm -rf {}'.format(self.default_path) diff --git a/python/paddle/fluid/tests/custom_op/custom_raw_op_kernel_op_setup.py b/python/paddle/fluid/tests/custom_op/custom_raw_op_kernel_op_setup.py index a1ae9279972..c29bf7014f8 100644 --- a/python/paddle/fluid/tests/custom_op/custom_raw_op_kernel_op_setup.py +++ b/python/paddle/fluid/tests/custom_op/custom_raw_op_kernel_op_setup.py @@ -41,9 +41,13 @@ if core.is_compiled_with_nccl(): macros.append(("THRUST_IGNORE_CUB_VERSION_CHECK", None)) include_dirs = list(paddle_includes) + [cwd] -setup(name=os.getenv("MODULE_NAME", "custom_raw_op_kernel_op_setup"), - ext_modules=extension(sources=sources, - include_dirs=include_dirs, - extra_compile_args=extra_compile_args, - _compile_dir=compile_dir, - define_macros=macros)) +setup( + name=os.getenv("MODULE_NAME", "custom_raw_op_kernel_op_setup"), + ext_modules=extension( + sources=sources, + include_dirs=include_dirs, + extra_compile_args=extra_compile_args, + _compile_dir=compile_dir, + define_macros=macros, + ), +) diff --git a/python/paddle/fluid/tests/custom_op/custom_relu_setup.py b/python/paddle/fluid/tests/custom_op/custom_relu_setup.py index 7dd3dd7aa7e..e71c5584e67 100644 --- a/python/paddle/fluid/tests/custom_op/custom_relu_setup.py +++ b/python/paddle/fluid/tests/custom_op/custom_relu_setup.py @@ -30,4 +30,6 @@ setup( sources=sources, # test for multi ops include_dirs=paddle_includes, extra_compile_args=extra_compile_args, - verbose=True)) + verbose=True, + ), +) diff --git a/python/paddle/fluid/tests/custom_op/test_check_abi.py b/python/paddle/fluid/tests/custom_op/test_check_abi.py index 39bcf1ae628..ab374123625 100644 --- a/python/paddle/fluid/tests/custom_op/test_check_abi.py +++ b/python/paddle/fluid/tests/custom_op/test_check_abi.py @@ -20,7 +20,6 @@ import paddle.utils.cpp_extension.extension_utils as utils class TestABIBase(unittest.TestCase): - def test_environ(self): compiler_list = ['gcc', 'cl'] for compiler in compiler_list: @@ -35,7 +34,6 @@ class TestABIBase(unittest.TestCase): class TestCheckCompiler(TestABIBase): - def test_expected_compiler(self): if utils.OS_NAME.startswith('linux'): gt = ['gcc', 'g++', 'gnu-c++', 'gnu-cc'] @@ -73,7 +71,8 @@ class TestCheckCompiler(TestABIBase): # check Compiler Compatibility WARNING self.assertTrue(len(error) == 1) self.assertTrue( - "Compiler Compatibility WARNING" in str(error[0].message)) + "Compiler Compatibility WARNING" in str(error[0].message) + ) def test_exception_windows(self): # clear environ @@ -86,8 +85,10 @@ class TestCheckCompiler(TestABIBase): self.assertFalse(flag) # check ABI Compatibility WARNING self.assertTrue(len(error) == 1) - self.assertTrue("Failed to check compiler version for" in str( - error[0].message)) + self.assertTrue( + "Failed to check compiler version for" + in str(error[0].message) + ) def test_exception_linux(self): # clear environ @@ -107,8 +108,10 @@ class TestCheckCompiler(TestABIBase): self.assertFalse(flag) # check ABI Compatibility WARNING self.assertTrue(len(error) == 1) - self.assertTrue("Failed to check compiler version for" in str( - error[0].message)) + self.assertTrue( + "Failed to check compiler version for" + in str(error[0].message) + ) # restore utils._expected_compiler_current_platform = raw_func @@ -137,7 +140,6 @@ class TestCheckCompiler(TestABIBase): class TestRunCMDException(unittest.TestCase): - def test_exception(self): for verbose in [True, False]: with self.assertRaisesRegexp(RuntimeError, "Failed to run command"): diff --git a/python/paddle/fluid/tests/custom_op/test_context_pool.py b/python/paddle/fluid/tests/custom_op/test_context_pool.py index 69b8b18559e..0c4829fb90f 100644 --- a/python/paddle/fluid/tests/custom_op/test_context_pool.py +++ b/python/paddle/fluid/tests/custom_op/test_context_pool.py @@ -25,7 +25,8 @@ from paddle.fluid.framework import _test_eager_guard # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\context_pool_jit\\context_pool_jit.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -37,11 +38,11 @@ custom_ops = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cflags extra_cuda_cflags=extra_nvcc_args, # test for cflags - verbose=True) + verbose=True, +) class TestContextPool(unittest.TestCase): - def setUp(self): self.devices = ['cpu'] if paddle.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/custom_op/test_custom_attrs_jit.py b/python/paddle/fluid/tests/custom_op/test_custom_attrs_jit.py index a0be75c0a41..7109d957384 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_attrs_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_attrs_jit.py @@ -25,7 +25,8 @@ from paddle.fluid.framework import _test_eager_guard # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\custom_attrs_jit\\custom_attrs_jit.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -37,11 +38,11 @@ custom_attrs = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cflags extra_cuda_cflags=extra_nvcc_args, # test for cflags - verbose=True) + verbose=True, +) class TestJitCustomAttrs(unittest.TestCase): - def setUp(self): paddle.set_device('cpu') # prepare test value @@ -58,11 +59,18 @@ class TestJitCustomAttrs(unittest.TestCase): def func_attr_value(self): x = paddle.ones([2, 2], dtype='float32') x.stop_gradient = False - out = custom_attrs.attr_test(x, self.bool_attr, self.int_attr, - self.float_attr, self.int64_attr, - self.str_attr, self.int_vec_attr, - self.float_vec_attr, self.int64_vec_attr, - self.str_vec_attr) + out = custom_attrs.attr_test( + x, + self.bool_attr, + self.int_attr, + self.float_attr, + self.int64_attr, + self.str_attr, + self.int_vec_attr, + self.float_vec_attr, + self.int64_vec_attr, + self.str_vec_attr, + ) out.stop_gradient = False out.backward() @@ -76,12 +84,18 @@ class TestJitCustomAttrs(unittest.TestCase): def func_const_attr_value(self): x = paddle.ones([2, 2], dtype='float32') x.stop_gradient = False - out = custom_attrs.const_attr_test(x, self.bool_attr, self.int_attr, - self.float_attr, self.int64_attr, - self.str_attr, self.int_vec_attr, - self.float_vec_attr, - self.int64_vec_attr, - self.str_vec_attr) + out = custom_attrs.const_attr_test( + x, + self.bool_attr, + self.int_attr, + self.float_attr, + self.int64_attr, + self.str_attr, + self.int_vec_attr, + self.float_vec_attr, + self.int64_vec_attr, + self.str_vec_attr, + ) out.stop_gradient = False out.backward() diff --git a/python/paddle/fluid/tests/custom_op/test_custom_concat.py b/python/paddle/fluid/tests/custom_op/test_custom_concat.py index ae3022411b1..a0744ce0870 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_concat.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_concat.py @@ -42,7 +42,8 @@ custom_ops = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) def concat_dynamic(func, dtype, np_inputs, axis_v, with_attr=False): @@ -85,29 +86,29 @@ def concat_static(func, dtype, np_inputs, axis_v, with_attr=False): if with_attr: feed_dict = { "x1": np_inputs[0].astype(dtype), - "x2": np_inputs[1].astype(dtype) + "x2": np_inputs[1].astype(dtype), } else: feed_dict = { "x1": np_inputs[0].astype(dtype), "x2": np_inputs[1].astype(dtype), - "axis": axis + "axis": axis, } out_v, x1_grad_v, x2_grad_v = exe.run( static.default_main_program(), feed=feed_dict, - fetch_list=[out.name, x1.name + "@GRAD", x2.name + "@GRAD"]) + fetch_list=[out.name, x1.name + "@GRAD", x2.name + "@GRAD"], + ) paddle.disable_static() return out_v, x1_grad_v, x2_grad_v class TestCustomConcatDynamicAxisJit(unittest.TestCase): - def setUp(self): self.dtypes = ['float32', 'float64', 'int32', 'int64'] self.np_inputs = [ np.array([[1, 2, 3], [4, 5, 6]]), - np.array([[11, 12, 13], [14, 15, 16]]) + np.array([[11, 12, 13], [14, 15, 16]]), ] self.axises = [0, 1] @@ -116,15 +117,19 @@ class TestCustomConcatDynamicAxisJit(unittest.TestCase): out, pd_out, err_msg='custom op {}: {},\n paddle api {}: {}'.format( - name, out, name, pd_out)) + name, out, name, pd_out + ), + ) def func_dynamic(self): for dtype in self.dtypes: for axis in self.axises: - out, grad_inputs = concat_dynamic(custom_ops.custom_concat, - dtype, self.np_inputs, axis) - pd_out, pd_grad_inputs = concat_dynamic(paddle.concat, dtype, - self.np_inputs, axis) + out, grad_inputs = concat_dynamic( + custom_ops.custom_concat, dtype, self.np_inputs, axis + ) + pd_out, pd_grad_inputs = concat_dynamic( + paddle.concat, dtype, self.np_inputs, axis + ) self.check_output(out, pd_out, "out") for x_grad, pd_x_grad in zip(grad_inputs, pd_grad_inputs): @@ -138,11 +143,12 @@ class TestCustomConcatDynamicAxisJit(unittest.TestCase): def test_static(self): for dtype in self.dtypes: for axis in self.axises: - out, x1_grad, x2_grad = concat_static(custom_ops.custom_concat, - dtype, self.np_inputs, - axis) + out, x1_grad, x2_grad = concat_static( + custom_ops.custom_concat, dtype, self.np_inputs, axis + ) pd_out, pd_x1_grad, pd_x2_grad = concat_static( - paddle.concat, dtype, self.np_inputs, axis) + paddle.concat, dtype, self.np_inputs, axis + ) self.check_output(out, pd_out, "out") self.check_output(x1_grad, pd_x1_grad, "x1_grad") @@ -152,11 +158,15 @@ class TestCustomConcatDynamicAxisJit(unittest.TestCase): for dtype in self.dtypes: for axis in self.axises: out, grad_inputs = concat_dynamic( - custom_ops.custom_concat_with_attr, dtype, self.np_inputs, - axis, True) - pd_out, pd_grad_inputs = concat_dynamic(paddle.concat, dtype, - self.np_inputs, axis, - True) + custom_ops.custom_concat_with_attr, + dtype, + self.np_inputs, + axis, + True, + ) + pd_out, pd_grad_inputs = concat_dynamic( + paddle.concat, dtype, self.np_inputs, axis, True + ) self.check_output(out, pd_out, "out") for x_grad, pd_x_grad in zip(grad_inputs, pd_grad_inputs): @@ -171,10 +181,15 @@ class TestCustomConcatDynamicAxisJit(unittest.TestCase): for dtype in self.dtypes: for axis in self.axises: out, x1_grad, x2_grad = concat_static( - custom_ops.custom_concat_with_attr, dtype, self.np_inputs, - axis, True) + custom_ops.custom_concat_with_attr, + dtype, + self.np_inputs, + axis, + True, + ) pd_out, pd_x1_grad, pd_x2_grad = concat_static( - paddle.concat, dtype, self.np_inputs, axis, True) + paddle.concat, dtype, self.np_inputs, axis, True + ) self.check_output(out, pd_out, "out") self.check_output(x1_grad, pd_x1_grad, "x1_grad") diff --git a/python/paddle/fluid/tests/custom_op/test_custom_conj.py b/python/paddle/fluid/tests/custom_op/test_custom_conj.py index a389a72df73..a9990facd79 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_conj.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_conj.py @@ -36,12 +36,15 @@ custom_ops = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) def is_complex(dtype): - return dtype == paddle.fluid.core.VarDesc.VarType.COMPLEX64 or \ - dtype == paddle.fluid.core.VarDesc.VarType.COMPLEX128 + return ( + dtype == paddle.fluid.core.VarDesc.VarType.COMPLEX64 + or dtype == paddle.fluid.core.VarDesc.VarType.COMPLEX128 + ) def to_complex(dtype): @@ -83,15 +86,16 @@ def conj_static(func, shape, dtype, np_input): exe = static.Executor() exe.run(static.default_startup_program()) - out_v, x_grad_v = exe.run(static.default_main_program(), - feed={"x": np_input}, - fetch_list=[out.name, x.name + "@GRAD"]) + out_v, x_grad_v = exe.run( + static.default_main_program(), + feed={"x": np_input}, + fetch_list=[out.name, x.name + "@GRAD"], + ) paddle.disable_static() return out_v, x_grad_v class TestCustomConjJit(unittest.TestCase): - def setUp(self): self.dtypes = ['float32', 'float64'] self.shape = [2, 20, 2, 3] @@ -101,7 +105,9 @@ class TestCustomConjJit(unittest.TestCase): out, pd_out, err_msg='custom op {}: {},\n paddle api {}: {}'.format( - name, out, name, pd_out)) + name, out, name, pd_out + ), + ) def run_dynamic(self, dtype, np_input): out, x_grad = conj_dynamic(custom_ops.custom_conj, dtype, np_input) @@ -111,10 +117,12 @@ class TestCustomConjJit(unittest.TestCase): self.check_output(x_grad, pd_x_grad, "x's grad") def run_static(self, dtype, np_input): - out, x_grad = conj_static(custom_ops.custom_conj, self.shape, dtype, - np_input) - pd_out, pd_x_grad = conj_static(paddle.conj, self.shape, dtype, - np_input) + out, x_grad = conj_static( + custom_ops.custom_conj, self.shape, dtype, np_input + ) + pd_out, pd_x_grad = conj_static( + paddle.conj, self.shape, dtype, np_input + ) self.check_output(out, pd_out, "out") self.check_output(x_grad, pd_x_grad, "x's grad") @@ -138,7 +146,8 @@ class TestCustomConjJit(unittest.TestCase): def test_complex_dynamic(self): for dtype in self.dtypes: np_input = np.random.random(self.shape).astype( - dtype) + 1j * np.random.random(self.shape).astype(dtype) + dtype + ) + 1j * np.random.random(self.shape).astype(dtype) self.run_dynamic(to_complex(dtype), np_input) diff --git a/python/paddle/fluid/tests/custom_op/test_custom_linear.py b/python/paddle/fluid/tests/custom_op/test_custom_linear.py index 3ae650ee949..bb7bc0358f4 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_linear.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_linear.py @@ -37,7 +37,8 @@ custom_ops = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) def linear_dynamic(func, device, dtype, np_x, np_weight, np_bias): @@ -56,9 +57,9 @@ def linear_static(func, device, dtype, np_x, np_weight, np_bias): with static.scope_guard(static.Scope()): with static.program_guard(static.Program()): x = static.data(name="x", shape=[None, np_x.shape[1]], dtype=dtype) - weight = static.data(name="weight", - shape=np_weight.shape, - dtype=dtype) + weight = static.data( + name="weight", shape=np_weight.shape, dtype=dtype + ) bias = static.data(name="bias", shape=np_bias.shape, dtype=dtype) x.stop_gradient = False weight.stop_gradient = False @@ -75,18 +76,20 @@ def linear_static(func, device, dtype, np_x, np_weight, np_bias): feed={ "x": np_x.astype(dtype), "weight": np_weight.astype(dtype), - "bias": np_bias.astype(dtype) + "bias": np_bias.astype(dtype), }, fetch_list=[ - out.name, x.name + "@GRAD", weight.name + "@GRAD", - bias.name + "@GRAD" - ]) + out.name, + x.name + "@GRAD", + weight.name + "@GRAD", + bias.name + "@GRAD", + ], + ) paddle.disable_static() return out_v, x_grad_v, weight_grad_v, bias_grad_v class TestCustomLinearJit(unittest.TestCase): - def setUp(self): self.dtypes = ['float32', 'float64'] self.devices = ['cpu'] @@ -101,36 +104,75 @@ class TestCustomLinearJit(unittest.TestCase): out, pd_out, err_msg='custom op {}: {},\n paddle api {}: {}'.format( - name, out, name, pd_out)) + name, out, name, pd_out + ), + ) def test_static(self): for device in self.devices: for dtype in self.dtypes: - phi_out, phi_x_grad, phi_weight_grad, phi_bias_grad = linear_static( - custom_ops.phi_linear, device, dtype, self.np_x, - self.np_weight, self.np_bias) + ( + phi_out, + phi_x_grad, + phi_weight_grad, + phi_bias_grad, + ) = linear_static( + custom_ops.phi_linear, + device, + dtype, + self.np_x, + self.np_weight, + self.np_bias, + ) pd_out, pd_x_grad, pd_weight_grad, pd_bias_grad = linear_static( - F.linear, device, dtype, self.np_x, self.np_weight, - self.np_bias) + F.linear, + device, + dtype, + self.np_x, + self.np_weight, + self.np_bias, + ) self.check_output(phi_out, pd_out, "out") self.check_output(phi_x_grad, pd_x_grad, "x_grad") - self.check_output(phi_weight_grad, pd_weight_grad, - "weight_grad") + self.check_output( + phi_weight_grad, pd_weight_grad, "weight_grad" + ) self.check_output(phi_bias_grad, pd_bias_grad, "bias_grad") def func_dynamic(self): for device in self.devices: for dtype in self.dtypes: - phi_out, phi_x_grad, phi_weight_grad, phi_bias_grad = linear_dynamic( - custom_ops.phi_linear, device, dtype, self.np_x, - self.np_weight, self.np_bias) - pd_out, pd_x_grad, pd_weight_grad, pd_bias_grad = linear_dynamic( - F.linear, device, dtype, self.np_x, self.np_weight, - self.np_bias) + ( + phi_out, + phi_x_grad, + phi_weight_grad, + phi_bias_grad, + ) = linear_dynamic( + custom_ops.phi_linear, + device, + dtype, + self.np_x, + self.np_weight, + self.np_bias, + ) + ( + pd_out, + pd_x_grad, + pd_weight_grad, + pd_bias_grad, + ) = linear_dynamic( + F.linear, + device, + dtype, + self.np_x, + self.np_weight, + self.np_bias, + ) self.check_output(phi_out, pd_out, "phi_out") self.check_output(phi_x_grad, pd_x_grad, "x_grad") - self.check_output(phi_weight_grad, pd_weight_grad, - "weight_grad") + self.check_output( + phi_weight_grad, pd_weight_grad, "weight_grad" + ) self.check_output(phi_bias_grad, pd_bias_grad, "bias_grad") def test_dynamic(self): diff --git a/python/paddle/fluid/tests/custom_op/test_custom_raw_op_kernel_op.py b/python/paddle/fluid/tests/custom_op/test_custom_raw_op_kernel_op.py index 3cd550c95f0..33f98c219d8 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_raw_op_kernel_op.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_raw_op_kernel_op.py @@ -36,8 +36,9 @@ def prepare_module_path(): else: site_dir = site.getsitepackages()[0] custom_egg_path = [x for x in os.listdir(site_dir) if MODULE_NAME in x] - assert len(custom_egg_path - ) == 1, "Matched egg number is %d." % len(custom_egg_path) + assert len(custom_egg_path) == 1, "Matched egg number is %d." % len( + custom_egg_path + ) sys.path.append(os.path.join(site_dir, custom_egg_path[0])) @@ -46,7 +47,6 @@ def prepare_module_path(): # temporarily. @unittest.skipIf(os.name == "nt", "Windows does not support yet.") class TestCustomRawReluOp(unittest.TestCase): - @classmethod def setUpClass(cls): path = os.path.dirname(os.path.abspath(__file__)) @@ -78,11 +78,14 @@ class TestCustomRawReluOp(unittest.TestCase): exe = paddle.static.Executor() exe.run(paddle.static.default_startup_program()) - x_np = np.random.uniform(low=-1.0, high=1.0, size=[2, - 3]).astype('float32') - y1_value, y2_value = exe.run(paddle.static.default_main_program(), - feed={x.name: x_np}, - fetch_list=[y1, y2]) + x_np = np.random.uniform(low=-1.0, high=1.0, size=[2, 3]).astype( + 'float32' + ) + y1_value, y2_value = exe.run( + paddle.static.default_main_program(), + feed={x.name: x_np}, + fetch_list=[y1, y2], + ) np.testing.assert_array_equal(y1_value, y2_value) paddle.disable_static() diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py index 762a16b61ff..f445410e906 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py @@ -28,7 +28,8 @@ from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\custom_relu_for_model_jit\\custom_relu_for_model_jit.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -47,7 +48,8 @@ custom_module = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) class Net(nn.Layer): @@ -59,7 +61,9 @@ class Net(nn.Layer): super(Net, self).__init__() self.fc1 = nn.Linear(in_dim, in_dim) self.fc2 = nn.Linear(in_dim, out_dim) - self.relu_act = custom_module.custom_relu if use_custom_op else nn.functional.relu + self.relu_act = ( + custom_module.custom_relu if use_custom_op else nn.functional.relu + ) def forward(self, x): out = self.fc1(x) @@ -73,7 +77,6 @@ class Net(nn.Layer): class TestDygraphModel(unittest.TestCase): - def tearDown(self): self.temp_dir.cleanup() @@ -85,8 +88,9 @@ class TestDygraphModel(unittest.TestCase): self.batch_num = 10 self.batch_size = 4 self.datas = [ - np.random.uniform( - size=[self.batch_size, self.in_dim]).astype('float32') + np.random.uniform(size=[self.batch_size, self.in_dim]).astype( + 'float32' + ) for i in range(self.batch_num) ] self.labels = [ @@ -100,14 +104,16 @@ class TestDygraphModel(unittest.TestCase): self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'infer_model') self.model_path_template = os.path.join( - self.model_save_dir, 'custom_relu_dygaph_model_{}.pdparams') + self.model_save_dir, 'custom_relu_dygaph_model_{}.pdparams' + ) self.model_dy2stat_path = os.path.join( - self.model_save_dir, 'infer_model/custom_relu_model_dy2sta') + self.model_save_dir, 'infer_model/custom_relu_model_dy2sta' + ) # for dy2stat - self.x_spec = paddle.static.InputSpec(shape=[None, self.in_dim], - dtype='float32', - name='x') + self.x_spec = paddle.static.InputSpec( + shape=[None, self.in_dim], dtype='float32', name='x' + ) def func_train_eval(self): for device in self.devices: @@ -120,24 +126,30 @@ class TestDygraphModel(unittest.TestCase): # open this when dy2stat is ready for eager if _in_legacy_dygraph(): custom_relu_dy2stat_train_out = self.train_model( - use_custom_op=True, dy2stat=True) # for to_static - np.testing.assert_array_equal(origin_relu_train_out, - custom_relu_dy2stat_train_out) + use_custom_op=True, dy2stat=True + ) # for to_static + np.testing.assert_array_equal( + origin_relu_train_out, custom_relu_dy2stat_train_out + ) - np.testing.assert_array_equal(origin_relu_train_out, - custom_relu_train_out) + np.testing.assert_array_equal( + origin_relu_train_out, custom_relu_train_out + ) # for eval origin_relu_eval_out = self.eval_model(use_custom_op=False) custom_relu_eval_out = self.eval_model(use_custom_op=True) if _in_legacy_dygraph(): custom_relu_dy2stat_eval_out = self.eval_model( - use_custom_op=True, dy2stat=True) # for to_static - np.testing.assert_array_equal(origin_relu_eval_out, - custom_relu_dy2stat_eval_out) + use_custom_op=True, dy2stat=True + ) # for to_static + np.testing.assert_array_equal( + origin_relu_eval_out, custom_relu_dy2stat_eval_out + ) - np.testing.assert_array_equal(origin_relu_eval_out, - custom_relu_eval_out) + np.testing.assert_array_equal( + origin_relu_eval_out, custom_relu_eval_out + ) def test_train_eval(self): with _test_eager_guard(): @@ -154,8 +166,9 @@ class TestDygraphModel(unittest.TestCase): if dy2stat: net = paddle.jit.to_static(net, input_spec=[self.x_spec]) mse_loss = paddle.nn.MSELoss() - sgd = paddle.optimizer.SGD(learning_rate=0.1, - parameters=net.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.1, parameters=net.parameters() + ) for batch_id in range(self.batch_num): x = paddle.to_tensor(self.datas[batch_id]) @@ -173,8 +186,9 @@ class TestDygraphModel(unittest.TestCase): if dy2stat: paddle.jit.save(net, self.model_dy2stat_path) else: - paddle.save(net.state_dict(), - self.model_path_template.format(use_custom_op)) + paddle.save( + net.state_dict(), self.model_path_template.format(use_custom_op) + ) return out.numpy() @@ -185,7 +199,8 @@ class TestDygraphModel(unittest.TestCase): net = paddle.jit.load(self.model_dy2stat_path) else: state_dict = paddle.load( - self.model_path_template.format(use_custom_op)) + self.model_path_template.format(use_custom_op) + ) net.set_state_dict(state_dict) sample_x = paddle.to_tensor(self.datas[0]) @@ -196,7 +211,6 @@ class TestDygraphModel(unittest.TestCase): class TestStaticModel(unittest.TestCase): - def setUp(self): self.seed = 2021 self.in_dim = 10 @@ -204,8 +218,9 @@ class TestStaticModel(unittest.TestCase): self.batch_num = 10 self.batch_size = 8 self.datas = [ - np.random.uniform( - size=[self.batch_size, self.in_dim]).astype('float32') + np.random.uniform(size=[self.batch_size, self.in_dim]).astype( + 'float32' + ) for i in range(self.batch_num) ] self.labels = [ @@ -219,7 +234,8 @@ class TestStaticModel(unittest.TestCase): self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'infer_model') self.model_path_template = os.path.join( - self.model_save_dir, 'custom_relu_static_model_{}_{}') + self.model_save_dir, 'custom_relu_static_model_{}_{}' + ) paddle.enable_static() @@ -230,38 +246,44 @@ class TestStaticModel(unittest.TestCase): def test_train_eval(self): for device in self.devices: # for train - original_relu_train_out = self.train_model(device, - use_custom_op=False) + original_relu_train_out = self.train_model( + device, use_custom_op=False + ) custom_relu_train_out = self.train_model(device, use_custom_op=True) # using PE - original_relu_train_pe_out = self.train_model(device, - use_custom_op=False, - use_pe=True) - custom_relu_train_pe_out = self.train_model(device, - use_custom_op=True, - use_pe=True) - - np.testing.assert_array_equal(original_relu_train_out, - custom_relu_train_out) - np.testing.assert_array_equal(original_relu_train_pe_out, - custom_relu_train_pe_out) + original_relu_train_pe_out = self.train_model( + device, use_custom_op=False, use_pe=True + ) + custom_relu_train_pe_out = self.train_model( + device, use_custom_op=True, use_pe=True + ) + + np.testing.assert_array_equal( + original_relu_train_out, custom_relu_train_out + ) + np.testing.assert_array_equal( + original_relu_train_pe_out, custom_relu_train_pe_out + ) # for eval - original_relu_eval_out = self.eval_model(device, - use_custom_op=False) + original_relu_eval_out = self.eval_model( + device, use_custom_op=False + ) custom_relu_eval_out = self.eval_model(device, use_custom_op=True) # using PE - original_relu_eval_pe_out = self.eval_model(device, - use_custom_op=False, - use_pe=True) - custom_relu_eval_pe_out = self.eval_model(device, - use_custom_op=True, - use_pe=True) - - np.testing.assert_array_equal(original_relu_eval_out, - custom_relu_eval_out) - np.testing.assert_array_equal(original_relu_eval_pe_out, - custom_relu_eval_pe_out) + original_relu_eval_pe_out = self.eval_model( + device, use_custom_op=False, use_pe=True + ) + custom_relu_eval_pe_out = self.eval_model( + device, use_custom_op=True, use_pe=True + ) + + np.testing.assert_array_equal( + original_relu_eval_out, custom_relu_eval_out + ) + np.testing.assert_array_equal( + original_relu_eval_pe_out, custom_relu_eval_pe_out + ) def train_model(self, device, use_custom_op=False, use_pe=False): # reset random seed @@ -271,14 +293,15 @@ class TestStaticModel(unittest.TestCase): paddle.set_device(device) with paddle.static.scope_guard(paddle.static.Scope()): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - x = paddle.static.data(shape=[None, self.in_dim], - name='x', - dtype='float32') - y = paddle.static.data(shape=[None, 1], - name='y', - dtype='float32') + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + x = paddle.static.data( + shape=[None, self.in_dim], name='x', dtype='float32' + ) + y = paddle.static.data( + shape=[None, 1], name='y', dtype='float32' + ) net = Net(self.in_dim, self.out_dim, use_custom_op) out = net(x) @@ -292,12 +315,14 @@ class TestStaticModel(unittest.TestCase): # For PE if use_pe: - places = paddle.static.cpu_places( - ) if device == 'cpu' else paddle.static.cuda_places() + places = ( + paddle.static.cpu_places() + if device == 'cpu' + else paddle.static.cuda_places() + ) main_program = paddle.static.CompiledProgram( - paddle.static.default_main_program( - )).with_data_parallel(loss_name=loss.name, - places=places) + paddle.static.default_main_program() + ).with_data_parallel(loss_name=loss.name, places=places) else: main_program = paddle.static.default_main_program() @@ -305,17 +330,19 @@ class TestStaticModel(unittest.TestCase): x_data = self.datas[batch_id] y_data = self.labels[batch_id] - res = exe.run(main_program, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[out]) + res = exe.run( + main_program, + feed={'x': x_data, 'y': y_data}, + fetch_list=[out], + ) # save model paddle.static.save_inference_model( - self.model_path_template.format(use_custom_op, use_pe), [x], - [out], exe) + self.model_path_template.format(use_custom_op, use_pe), + [x], + [out], + exe, + ) return res[0] @@ -326,15 +353,20 @@ class TestStaticModel(unittest.TestCase): with paddle.static.program_guard(paddle.static.Program()): exe = paddle.static.Executor() - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model( - self.model_path_template.format(use_custom_op, use_pe), - exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model( + self.model_path_template.format(use_custom_op, use_pe), exe + ) x_data = self.datas[0] - results = exe.run(inference_program, - feed={feed_target_names[0]: x_data}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: x_data}, + fetch_list=fetch_targets, + ) return results[0] diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py index b1255bc3879..00b2860987b 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py @@ -21,10 +21,12 @@ from paddle.utils.cpp_extension.extension_utils import run_cmd from utils import IS_MAC, extra_cc_args, extra_nvcc_args, paddle_includes from test_custom_relu_op_setup import custom_relu_dynamic, custom_relu_static from paddle.fluid.framework import _test_eager_guard + # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\custom_relu_module_jit\\custom_relu_module_jit.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -43,16 +45,17 @@ custom_module = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) class TestJITLoad(unittest.TestCase): - def setUp(self): self.custom_ops = [ - custom_module.custom_relu, custom_module.custom_relu_dup, + custom_module.custom_relu, + custom_module.custom_relu_dup, custom_module.custom_relu_no_x_in_backward, - custom_module.custom_relu_out + custom_module.custom_relu_out, ] self.dtypes = ['float32', 'float64'] if paddle.is_compiled_with_cuda(): @@ -69,13 +72,16 @@ class TestJITLoad(unittest.TestCase): x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: out = custom_relu_static(custom_op, device, dtype, x) - pd_out = custom_relu_static(custom_op, device, dtype, x, - False) + pd_out = custom_relu_static( + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) def func_dynamic(self): for device in self.devices: @@ -84,20 +90,26 @@ class TestJITLoad(unittest.TestCase): continue x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: - out, x_grad = custom_relu_dynamic(custom_op, device, dtype, - x) + out, x_grad = custom_relu_dynamic( + custom_op, device, dtype, x + ) pd_out, pd_x_grad = custom_relu_dynamic( - custom_op, device, dtype, x, False) + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) np.testing.assert_array_equal( x_grad, pd_x_grad, - err_msg='custom op x grad: {},\n paddle api x grad: {}'. - format(x_grad, pd_x_grad)) + err_msg='custom op x grad: {},\n paddle api x grad: {}'.format( + x_grad, pd_x_grad + ), + ) def test_dynamic(self): with _test_eager_guard(): @@ -141,7 +153,8 @@ class TestJITLoad(unittest.TestCase): extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, + ) custom_conj = custom_module.custom_conj self.assertIsNotNone(custom_conj) diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py index 32e34091d38..0f73b4a2124 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py @@ -41,12 +41,9 @@ def custom_relu_dynamic(func, device, dtype, np_x, use_func=True): return out.numpy(), t.grad.numpy() -def custom_relu_static(func, - device, - dtype, - np_x, - use_func=True, - test_infer=False): +def custom_relu_static( + func, device, dtype, np_x, use_func=True, test_infer=False +): paddle.enable_static() paddle.set_device(device) @@ -60,9 +57,11 @@ def custom_relu_static(func, exe = static.Executor() exe.run(static.default_startup_program()) # in static mode, x data has been covered by out - out_v = exe.run(static.default_main_program(), - feed={'X': np_x}, - fetch_list=[out.name]) + out_v = exe.run( + static.default_main_program(), + feed={'X': np_x}, + fetch_list=[out.name], + ) paddle.disable_static() return out_v @@ -85,11 +84,11 @@ def custom_relu_static_pe(func, device, dtype, np_x, use_func=True): # in static mode, x data has been covered by out compiled_prog = static.CompiledProgram( - static.default_main_program()).with_data_parallel( - loss_name=out.name, places=places) - out_v = exe.run(compiled_prog, - feed={'X': np_x}, - fetch_list=[out.name]) + static.default_main_program() + ).with_data_parallel(loss_name=out.name, places=places) + out_v = exe.run( + compiled_prog, feed={'X': np_x}, fetch_list=[out.name] + ) paddle.disable_static() return out_v @@ -101,9 +100,9 @@ def custom_relu_static_inference(func, device, np_data, np_label, path_prefix): with static.scope_guard(static.Scope()): with static.program_guard(static.Program()): # simple module - data = static.data(name='data', - shape=[None, 1, 28, 28], - dtype='float32') + data = static.data( + name='data', shape=[None, 1, 28, 28], dtype='float32' + ) label = static.data(name='label', shape=[None, 1], dtype='int64') hidden = static.nn.fc(data, size=128) @@ -122,23 +121,21 @@ def custom_relu_static_inference(func, device, np_data, np_label, path_prefix): # train for i in range(4): - avg_loss_v = exe.run(static.default_main_program(), - feed={ - 'data': np_data, - 'label': np_label - }, - fetch_list=[avg_loss]) + avg_loss_v = exe.run( + static.default_main_program(), + feed={'data': np_data, 'label': np_label}, + fetch_list=[avg_loss], + ) # save inference model static.save_inference_model(path_prefix, [data], [predict], exe) # get train predict value - predict_v = exe.run(static.default_main_program(), - feed={ - 'data': np_data, - 'label': np_label - }, - fetch_list=[predict]) + predict_v = exe.run( + static.default_main_program(), + feed={'data': np_data, 'label': np_label}, + fetch_list=[predict], + ) return predict_v @@ -151,10 +148,9 @@ def custom_relu_double_grad_dynamic(func, device, dtype, np_x, use_func=True): out = func(t) if use_func else paddle.nn.functional.relu(t) out.stop_gradient = False - dx = paddle.grad(outputs=[out], - inputs=[t], - create_graph=True, - retain_graph=True) + dx = paddle.grad( + outputs=[out], inputs=[t], create_graph=True, retain_graph=True + ) dx[0].backward() @@ -163,16 +159,17 @@ def custom_relu_double_grad_dynamic(func, device, dtype, np_x, use_func=True): class TestNewCustomOpSetUpInstall(unittest.TestCase): - def setUp(self): cur_dir = os.path.dirname(os.path.abspath(__file__)) # compile, install the custom op egg into site-packages under background if os.name == 'nt': cmd = 'cd /d {} && python custom_relu_setup.py install'.format( - cur_dir) + cur_dir + ) else: cmd = 'cd {} && {} custom_relu_setup.py install'.format( - cur_dir, sys.executable) + cur_dir, sys.executable + ) run_cmd(cmd) # NOTE(Aurelius84): Normally, it's no need to add following codes for users. @@ -188,16 +185,18 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): custom_egg_path = [ x for x in os.listdir(site_dir) if 'custom_relu_module_setup' in x ] - assert len(custom_egg_path - ) == 1, "Matched egg number is %d." % len(custom_egg_path) + assert len(custom_egg_path) == 1, "Matched egg number is %d." % len( + custom_egg_path + ) sys.path.append(os.path.join(site_dir, custom_egg_path[0])) # usage: import the package directly import custom_relu_module_setup + # `custom_relu_dup` is same as `custom_relu_dup` self.custom_ops = [ custom_relu_module_setup.custom_relu, - custom_relu_module_setup.custom_relu_dup + custom_relu_module_setup.custom_relu_dup, ] self.dtypes = ['float32', 'float64'] @@ -220,13 +219,16 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: out = custom_relu_static(custom_op, device, dtype, x) - pd_out = custom_relu_static(custom_op, device, dtype, x, - False) + pd_out = custom_relu_static( + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) def test_static_pe(self): for device in self.devices: @@ -236,13 +238,16 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: out = custom_relu_static_pe(custom_op, device, dtype, x) - pd_out = custom_relu_static_pe(custom_op, device, dtype, x, - False) + pd_out = custom_relu_static_pe( + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) def func_dynamic(self): for device in self.devices: @@ -251,20 +256,26 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): continue x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: - out, x_grad = custom_relu_dynamic(custom_op, device, dtype, - x) + out, x_grad = custom_relu_dynamic( + custom_op, device, dtype, x + ) pd_out, pd_x_grad = custom_relu_dynamic( - custom_op, device, dtype, x, False) + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) np.testing.assert_array_equal( x_grad, pd_x_grad, - err_msg='custom op x grad: {},\n paddle api x grad: {}'. - format(x_grad, pd_x_grad)) + err_msg='custom op x grad: {},\n paddle api x grad: {}'.format( + x_grad, pd_x_grad + ), + ) def test_dynamic(self): with _test_eager_guard(): @@ -277,22 +288,29 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): np_label = np.random.random((1, 1)).astype("int64") path_prefix = "custom_op_inference/custom_relu" for device in self.devices: - predict = custom_relu_static_inference(self.custom_ops[0], device, - np_data, np_label, - path_prefix) + predict = custom_relu_static_inference( + self.custom_ops[0], device, np_data, np_label, path_prefix + ) # load inference model with static.scope_guard(static.Scope()): exe = static.Executor() - [inference_program, feed_target_names, - fetch_targets] = static.load_inference_model(path_prefix, exe) - predict_infer = exe.run(inference_program, - feed={feed_target_names[0]: np_data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = static.load_inference_model(path_prefix, exe) + predict_infer = exe.run( + inference_program, + feed={feed_target_names[0]: np_data}, + fetch_list=fetch_targets, + ) np.testing.assert_array_equal( predict, predict_infer, - err_msg='custom op predict: {},\n custom op infer predict: {}' - .format(predict, predict_infer)) + err_msg='custom op predict: {},\n custom op infer predict: {}'.format( + predict, predict_infer + ), + ) paddle.disable_static() def test_static_save_and_run_inference_predictor(self): @@ -302,26 +320,32 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): path_prefix = "custom_op_inference/custom_relu" from paddle.inference import Config from paddle.inference import create_predictor + for device in self.devices: - predict = custom_relu_static_inference(self.custom_ops[0], device, - np_data, np_label, - path_prefix) + predict = custom_relu_static_inference( + self.custom_ops[0], device, np_data, np_label, path_prefix + ) # load inference model - config = Config(path_prefix + ".pdmodel", - path_prefix + ".pdiparams") + config = Config( + path_prefix + ".pdmodel", path_prefix + ".pdiparams" + ) predictor = create_predictor(config) input_tensor = predictor.get_input_handle( - predictor.get_input_names()[0]) + predictor.get_input_names()[0] + ) input_tensor.reshape(np_data.shape) input_tensor.copy_from_cpu(np_data.copy()) predictor.run() output_tensor = predictor.get_output_handle( - predictor.get_output_names()[0]) + predictor.get_output_names()[0] + ) predict_infer = output_tensor.copy_to_cpu() self.assertTrue( np.isclose(predict, predict_infer, rtol=5e-5).any(), "custom op predict: {},\n custom op infer predict: {}".format( - predict, predict_infer)) + predict, predict_infer + ), + ) paddle.disable_static() def test_func_double_grad_dynamic(self): @@ -331,33 +355,43 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): continue x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) out, dx_grad = custom_relu_double_grad_dynamic( - self.custom_ops[0], device, dtype, x) + self.custom_ops[0], device, dtype, x + ) pd_out, pd_dx_grad = custom_relu_double_grad_dynamic( - self.custom_ops[0], device, dtype, x, False) + self.custom_ops[0], device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, err_msg='custom op out: {},\n paddle api out: {}'.format( - out, pd_out)) + out, pd_out + ), + ) np.testing.assert_array_equal( dx_grad, pd_dx_grad, - err_msg='custom op dx grad: {},\n paddle api dx grad: {}'. - format(dx_grad, pd_dx_grad)) + err_msg='custom op dx grad: {},\n paddle api dx grad: {}'.format( + dx_grad, pd_dx_grad + ), + ) def test_with_dataloader(self): for device in self.devices: paddle.set_device(device) # data loader transform = Compose( - [Normalize(mean=[127.5], std=[127.5], data_format='CHW')]) - train_dataset = paddle.vision.datasets.MNIST(mode='train', - transform=transform) - train_loader = paddle.io.DataLoader(train_dataset, - batch_size=64, - shuffle=True, - drop_last=True, - num_workers=0) + [Normalize(mean=[127.5], std=[127.5], data_format='CHW')] + ) + train_dataset = paddle.vision.datasets.MNIST( + mode='train', transform=transform + ) + train_loader = paddle.io.DataLoader( + train_dataset, + batch_size=64, + shuffle=True, + drop_last=True, + num_workers=0, + ) for batch_id, (image, _) in enumerate(train_loader()): out = self.custom_ops[0](image) @@ -366,7 +400,9 @@ class TestNewCustomOpSetUpInstall(unittest.TestCase): out, pd_out, err_msg='custom op out: {},\n paddle api out: {}'.format( - out, pd_out)) + out, pd_out + ), + ) if batch_id == 5: break diff --git a/python/paddle/fluid/tests/custom_op/test_custom_simple_slice.py b/python/paddle/fluid/tests/custom_op/test_custom_simple_slice.py index e7d60fd4296..b67b09131fa 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_simple_slice.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_simple_slice.py @@ -25,7 +25,8 @@ from paddle.fluid.framework import _test_eager_guard # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\custom_simple_slice\\custom_simple_slice.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -36,11 +37,11 @@ custom_ops = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) class TestCustomSimpleSliceJit(unittest.TestCase): - def func_slice_output(self): np_x = np.random.random((5, 2)).astype("float32") x = paddle.to_tensor(np_x) @@ -49,8 +50,10 @@ class TestCustomSimpleSliceJit(unittest.TestCase): np.testing.assert_array_equal( custom_op_out, np_out, - err_msg='custom op: {},\n numpy: {}'.format(np_out, - custom_op_out.numpy())) + err_msg='custom op: {},\n numpy: {}'.format( + np_out, custom_op_out.numpy() + ), + ) def test_slice_output(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py b/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py index fb65422758f..d3fec61f4bd 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py @@ -36,7 +36,8 @@ custom_ops = load( extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) def custom_tanh_double_grad_dynamic(func, device, dtype, np_x): @@ -48,10 +49,9 @@ def custom_tanh_double_grad_dynamic(func, device, dtype, np_x): out = func(t) out.stop_gradient = False - dx = paddle.grad(outputs=[out], - inputs=[t], - create_graph=True, - retain_graph=True) + dx = paddle.grad( + outputs=[out], inputs=[t], create_graph=True, retain_graph=True + ) dx[0].backward() @@ -62,7 +62,6 @@ def custom_tanh_double_grad_dynamic(func, device, dtype, np_x): class TestCustomTanhDoubleGradJit(unittest.TestCase): - def setUp(self): paddle.set_device('cpu') self.dtypes = ['float32', 'float64'] @@ -73,27 +72,35 @@ class TestCustomTanhDoubleGradJit(unittest.TestCase): for dtype in self.dtypes: x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) out, dx_grad, dout = custom_tanh_double_grad_dynamic( - custom_ops.custom_tanh, device, dtype, x) + custom_ops.custom_tanh, device, dtype, x + ) pd_out, pd_dx_grad, pd_dout = custom_tanh_double_grad_dynamic( - paddle.tanh, device, dtype, x) + paddle.tanh, device, dtype, x + ) np.testing.assert_allclose( out, pd_out, rtol=1e-05, err_msg='custom op out: {},\n paddle api out: {}'.format( - out, pd_out)) + out, pd_out + ), + ) np.testing.assert_allclose( dx_grad, pd_dx_grad, rtol=1e-05, - err_msg='custom op dx grad: {},\n paddle api dx grad: {}'. - format(dx_grad, pd_dx_grad)) + err_msg='custom op dx grad: {},\n paddle api dx grad: {}'.format( + dx_grad, pd_dx_grad + ), + ) np.testing.assert_allclose( dout, pd_dout, rtol=1e-05, - err_msg='custom op out grad: {},\n paddle api out grad: {}'. - format(dout, pd_dout)) + err_msg='custom op out grad: {},\n paddle api out grad: {}'.format( + dout, pd_dout + ), + ) def test_func_double_grad_dynamic(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/custom_op/test_dispatch_jit.py b/python/paddle/fluid/tests/custom_op/test_dispatch_jit.py index ff5192db7aa..a5bbb405fa4 100644 --- a/python/paddle/fluid/tests/custom_op/test_dispatch_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_dispatch_jit.py @@ -20,6 +20,7 @@ from paddle.utils.cpp_extension import load, get_build_directory from utils import paddle_includes, extra_cc_args from paddle.utils.cpp_extension.extension_utils import run_cmd from paddle.fluid.framework import _test_eager_guard + # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\dispatch_op\\dispatch_op.pyd'.format(get_build_directory()) @@ -32,11 +33,11 @@ dispatch_op = load( sources=['dispatch_test_op.cc'], extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, - verbose=True) + verbose=True, +) class TestJitDispatch(unittest.TestCase): - def setUp(self): paddle.set_device('cpu') @@ -50,7 +51,8 @@ class TestJitDispatch(unittest.TestCase): np.testing.assert_array_equal( np_x, np_out, - err_msg='custom op x: {},\n custom op out: {}'.format(np_x, np_out)) + err_msg='custom op x: {},\n custom op out: {}'.format(np_x, np_out), + ) def run_dispatch_test(self, func, dtype): with _test_eager_guard(): @@ -69,32 +71,49 @@ class TestJitDispatch(unittest.TestCase): def test_dispatch_float_and_integer(self): dtypes = [ - "float32", "float64", "int32", "int64", "int8", "uint8", "int16" + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "int16", ] for dtype in dtypes: - self.run_dispatch_test(dispatch_op.dispatch_test_float_and_integer, - dtype) + self.run_dispatch_test( + dispatch_op.dispatch_test_float_and_integer, dtype + ) def test_dispatch_float_and_complex(self): dtypes = ["float32", "float64", "complex64", "complex128"] for dtype in dtypes: - self.run_dispatch_test(dispatch_op.dispatch_test_float_and_complex, - dtype) + self.run_dispatch_test( + dispatch_op.dispatch_test_float_and_complex, dtype + ) def test_dispatch_float_and_integer_and_complex(self): dtypes = [ - "float32", "float64", "int32", "int64", "int8", "uint8", "int16", - "complex64", "complex128" + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "int16", + "complex64", + "complex128", ] for dtype in dtypes: self.run_dispatch_test( - dispatch_op.dispatch_test_float_and_integer_and_complex, dtype) + dispatch_op.dispatch_test_float_and_integer_and_complex, dtype + ) def test_dispatch_float_and_half(self): dtypes = ["float32", "float64", "float16"] for dtype in dtypes: - self.run_dispatch_test(dispatch_op.dispatch_test_float_and_half, - dtype) + self.run_dispatch_test( + dispatch_op.dispatch_test_float_and_half, dtype + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/custom_op/test_multi_out_jit.py b/python/paddle/fluid/tests/custom_op/test_multi_out_jit.py index eb9b9e4677f..7b6c30cabac 100644 --- a/python/paddle/fluid/tests/custom_op/test_multi_out_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_multi_out_jit.py @@ -22,6 +22,7 @@ from paddle.utils.cpp_extension import load, get_build_directory from paddle.utils.cpp_extension.extension_utils import run_cmd from utils import paddle_includes, extra_cc_args from paddle.fluid.framework import _test_eager_guard + # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\multi_out_jit\\multi_out_jit.pyd'.format(get_build_directory()) @@ -35,11 +36,11 @@ multi_out_module = load( sources=['multi_out_test_op.cc'], extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cflags - verbose=True) + verbose=True, +) class TestMultiOutputDtypes(unittest.TestCase): - def setUp(self): self.custom_op = multi_out_module.multi_out self.dtypes = ['float32', 'float64'] @@ -56,9 +57,11 @@ class TestMultiOutputDtypes(unittest.TestCase): exe = paddle.static.Executor() exe.run(paddle.static.default_startup_program()) - res = exe.run(paddle.static.default_main_program(), - feed={'X': x_data}, - fetch_list=outs) + res = exe.run( + paddle.static.default_main_program(), + feed={'X': x_data}, + fetch_list=outs, + ) return res @@ -69,12 +72,14 @@ class TestMultiOutputDtypes(unittest.TestCase): one_int32 = one_int32.numpy() # Fake_float64 self.assertTrue('float64' in str(zero_float64.dtype)) - np.testing.assert_array_equal(zero_float64, - np.zeros([4, 8]).astype('float64')) + np.testing.assert_array_equal( + zero_float64, np.zeros([4, 8]).astype('float64') + ) # ZFake_int32 self.assertTrue('int32' in str(one_int32.dtype)) - np.testing.assert_array_equal(one_int32, - np.ones([4, 8]).astype('int32')) + np.testing.assert_array_equal( + one_int32, np.ones([4, 8]).astype('int32') + ) def test_static(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/custom_op/test_sysconfig.py b/python/paddle/fluid/tests/custom_op/test_sysconfig.py index 89ef36931f3..78c0cdf0316 100644 --- a/python/paddle/fluid/tests/custom_op/test_sysconfig.py +++ b/python/paddle/fluid/tests/custom_op/test_sysconfig.py @@ -18,7 +18,6 @@ import paddle class SysConfigTest(unittest.TestCase): - def test_include(self): inc_dir = paddle.sysconfig.get_include() inc_dirs = inc_dir.split(os.sep) diff --git a/python/paddle/fluid/tests/custom_op/utils.py b/python/paddle/fluid/tests/custom_op/utils.py index adce68cd0f9..c21a1bb1624 100644 --- a/python/paddle/fluid/tests/custom_op/utils.py +++ b/python/paddle/fluid/tests/custom_op/utils.py @@ -26,7 +26,7 @@ site_packages_path = get_python_lib() # PaddlePaddle whl. So here we specific `include_dirs` to avoid errors in CI. paddle_includes = [ os.path.join(site_packages_path, 'paddle', 'include'), - os.path.join(site_packages_path, 'paddle', 'include', 'third_party') + os.path.join(site_packages_path, 'paddle', 'include', 'third_party'), ] # Test for extra compile args diff --git a/python/paddle/fluid/tests/custom_runtime/custom_device_multi_process_collective.py b/python/paddle/fluid/tests/custom_runtime/custom_device_multi_process_collective.py index 21e98710426..b658b885176 100644 --- a/python/paddle/fluid/tests/custom_runtime/custom_device_multi_process_collective.py +++ b/python/paddle/fluid/tests/custom_runtime/custom_device_multi_process_collective.py @@ -27,12 +27,21 @@ def train(prefix): device_ids = os.getenv("PADDLE_WORLD_DEVICE_IDS") current_device_id = os.getenv("PADDLE_LOCAL_DEVICE_IDS") - details = "selected_accelerators:{} selected_custom_devices:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{} device_ids:{} device_id:{}"\ - .format(selected_accelerators, selected_custom_devices, worker_endpoints, trainers_num, current_endpoint,trainer_id,device_ids, current_device_id) + details = "selected_accelerators:{} selected_custom_devices:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{} device_ids:{} device_id:{}".format( + selected_accelerators, + selected_custom_devices, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + device_ids, + current_device_id, + ) print(details) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(details) diff --git a/python/paddle/fluid/tests/custom_runtime/process_group_xccl.py b/python/paddle/fluid/tests/custom_runtime/process_group_xccl.py index b830d0eccf5..201b2d3df86 100644 --- a/python/paddle/fluid/tests/custom_runtime/process_group_xccl.py +++ b/python/paddle/fluid/tests/custom_runtime/process_group_xccl.py @@ -29,15 +29,16 @@ def init_process_group(strategy=None): is_master = True if rank == 0 else False store = paddle.fluid.core.TCPStore("127.0.0.1", 6173, is_master, nranks) pg_group = core.ProcessGroupCustom( - store, rank, nranks, - paddle.CustomPlace(ParallelEnv().device_type, - ParallelEnv().device_id)) + store, + rank, + nranks, + paddle.CustomPlace(ParallelEnv().device_type, ParallelEnv().device_id), + ) return pg_group class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -50,8 +51,9 @@ class TestProcessGroupFp32(unittest.TestCase): def test_create_process_group_xccl(self): with _test_eager_guard(): - paddle.set_device('custom_cpu:%d' % - paddle.distributed.ParallelEnv().dev_id) + paddle.set_device( + 'custom_cpu:%d' % paddle.distributed.ParallelEnv().dev_id + ) pg = init_process_group() @@ -147,8 +149,9 @@ class TestProcessGroupFp32(unittest.TestCase): task.wait() # paddle.fluid.core._custom_device_synchronize("custom_cpu", -1) out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) # assert np.array_equal(tensor_x, out_1) # assert np.array_equal(tensor_y, out_2) print("test allgather api ok\n") @@ -163,10 +166,12 @@ class TestProcessGroupFp32(unittest.TestCase): tensor_y = paddle.to_tensor(y) tensor_out1 = paddle.to_tensor(out1) tensor_out2 = paddle.to_tensor(out2) - raw_tensor_x_2 = paddle.slice(tensor_x, [0], [self.shape[0] // 2], - [self.shape[0]]) - raw_tensor_y_1 = paddle.slice(tensor_y, [0], [0], - [self.shape[0] // 2]) + raw_tensor_x_2 = paddle.slice( + tensor_x, [0], [self.shape[0] // 2], [self.shape[0]] + ) + raw_tensor_y_1 = paddle.slice( + tensor_y, [0], [0], [self.shape[0] // 2] + ) if pg.rank() == 0: task = pg.alltoall(tensor_x, tensor_out1) task.wait() @@ -176,8 +181,9 @@ class TestProcessGroupFp32(unittest.TestCase): task = pg.alltoall(tensor_y, tensor_out2) task.wait() # paddle.fluid.core._custom_device_synchronize("custom_cpu", -1) - out1_2 = paddle.slice(tensor_out1, [0], [self.shape[0] // 2], - [self.shape[0]]) + out1_2 = paddle.slice( + tensor_out1, [0], [self.shape[0] // 2], [self.shape[0]] + ) out2_1 = paddle.slice(tensor_out2, [0], [0], [self.shape[0] // 2]) # if pg.rank() == 0: # assert np.array_equal(out1_2.numpy(), raw_tensor_y_1.numpy()) @@ -223,8 +229,9 @@ class TestProcessGroupFp32(unittest.TestCase): task.wait() # paddle.fluid.core._custom_device_synchronize("custom_cpu", -1) out1 = paddle.slice(tensor_x, [0], [0], [self.shape[0]]) - out2 = paddle.slice(tensor_x, [0], [self.shape[0]], - [self.shape[0] * 2]) + out2 = paddle.slice( + tensor_x, [0], [self.shape[0]], [self.shape[0] * 2] + ) # if pg.rank() == 0: # assert np.array_equal(tensor_y, out1) # else: diff --git a/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py b/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py index 1127352d85d..586e0e322ab 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py +++ b/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py @@ -20,19 +20,26 @@ import time import tempfile -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - eager_mode=True, - log_dir=None): - from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc # noqa: F401 +def start_local_trainers( + cluster, + pod, + training_script, + training_script_args, + eager_mode=True, + log_dir=None, +): + from paddle.distributed.utils.launch_utils import ( # noqa: F401 + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, + ) current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -41,8 +48,8 @@ def start_local_trainers(cluster, os.system("rm -rf log && mkdir -p log") for idx, t in enumerate(pod.trainers): proc_env = { - "FLAGS_selected_custom_cpus": - "%s" % ",".join([str(g) for g in t.gpus]), + "FLAGS_selected_custom_cpus": "%s" + % ",".join([str(g) for g in t.gpus]), "PADDLE_TRAINER_ID": "%d" % t.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), @@ -65,10 +72,9 @@ def start_local_trainers(cluster, print("start trainer proc:{} env:{}".format(cmd, proc_env)) fn = open("workerlog.%d" % idx, "a") - proc = subprocess.Popen(cmd.split(" "), - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd.split(" "), env=current_env, stdout=fn, stderr=fn + ) tp = TrainerProc() tp.proc = proc @@ -82,7 +88,12 @@ def start_local_trainers(cluster, def get_cluster_from_args(selected_gpus): - from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc # noqa: F401 + from paddle.distributed.utils.launch_utils import ( # noqa: F401 + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, + ) cluster_node_ips = '127.0.0.1' node_ip = '127.0.0.1' @@ -104,9 +115,13 @@ def get_cluster_from_args(selected_gpus): class TestMultipleCustomCPU(unittest.TestCase): - def run_mnist_2custom_cpu(self, target_file_name, eager_mode=True): - from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc # noqa: F401 + from paddle.distributed.utils.launch_utils import ( # noqa: F401 + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, + ) selected_devices = [0, 1] cluster = None @@ -114,11 +129,13 @@ class TestMultipleCustomCPU(unittest.TestCase): cluster, pod = get_cluster_from_args(selected_devices) - procs = start_local_trainers(cluster, - pod, - eager_mode=eager_mode, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + eager_mode=eager_mode, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_endpoints()) @@ -130,7 +147,6 @@ class TestMultipleCustomCPU(unittest.TestCase): class TestProcessGroup(TestMultipleCustomCPU): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -142,15 +158,18 @@ class TestProcessGroup(TestMultipleCustomCPU): && git checkout {} -b dev \ && cd backends/custom_cpu \ && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), - os.getenv('PLUGIN_TAG')) + self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + ) os.system(cmd) # set environment for loading and registering compiled custom kernels # only valid in current process os.environ['CUSTOM_DEVICE_ROOT'] = os.path.join( - cur_dir, '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( - self.temp_dir.name)) + cur_dir, + '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( + self.temp_dir.name + ), + ) os.environ['FLAGS_selected_custom_cpus'] = '0,1' os.environ['CUSTOM_CPU_VISIBLE_DEVICES'] = '0,1' os.environ['PADDLE_XCCL_BACKEND'] = 'custom_cpu' @@ -159,7 +178,12 @@ class TestProcessGroup(TestMultipleCustomCPU): self.temp_dir.cleanup() def test_process_group_xccl(self): - from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc # noqa: F401 + from paddle.distributed.utils.launch_utils import ( # noqa: F401 + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, + ) self.run_mnist_2custom_cpu('process_group_xccl.py') diff --git a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_plugin.py b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_plugin.py index f1cfa8ef739..5d0e5ccc475 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_plugin.py +++ b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_plugin.py @@ -20,7 +20,6 @@ import tempfile class TestCustomCPUPlugin(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -32,15 +31,18 @@ class TestCustomCPUPlugin(unittest.TestCase): && git checkout {} -b dev \ && cd backends/custom_cpu \ && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), - os.getenv('PLUGIN_TAG')) + self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + ) os.system(cmd) # set environment for loading and registering compiled custom kernels # only valid in current process os.environ['CUSTOM_DEVICE_ROOT'] = os.path.join( - cur_dir, '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( - self.temp_dir.name)) + cur_dir, + '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( + self.temp_dir.name + ), + ) def tearDown(self): self.temp_dir.cleanup() @@ -65,16 +67,18 @@ class TestCustomCPUPlugin(unittest.TestCase): paddle.set_device('custom_cpu') dataset = paddle.vision.datasets.MNIST( mode='test', - transform=paddle.vision.transforms.Compose([ - paddle.vision.transforms.CenterCrop(20), - paddle.vision.transforms.RandomResizedCrop(14), - paddle.vision.transforms.Normalize(), - paddle.vision.transforms.ToTensor() - ])) - loader = paddle.io.DataLoader(dataset, - batch_size=32, - num_workers=1, - shuffle=True) + transform=paddle.vision.transforms.Compose( + [ + paddle.vision.transforms.CenterCrop(20), + paddle.vision.transforms.RandomResizedCrop(14), + paddle.vision.transforms.Normalize(), + paddle.vision.transforms.ToTensor(), + ] + ), + ) + loader = paddle.io.DataLoader( + dataset, batch_size=32, num_workers=1, shuffle=True + ) for image, label in loader: self.assertTrue(image.place.is_custom_place()) self.assertTrue(label.place.is_custom_place()) @@ -84,13 +88,13 @@ class TestCustomCPUPlugin(unittest.TestCase): import paddle class MNIST(paddle.nn.Layer): - def __init__(self): super(MNIST, self).__init__() self.shape = 1 * 28 * 28 self.size = 10 self.output_weight = self.create_parameter( - [self.shape, self.size]) + [self.shape, self.size] + ) self.accuracy = paddle.metric.Accuracy() def forward(self, inputs, label=None): @@ -110,15 +114,17 @@ class TestCustomCPUPlugin(unittest.TestCase): dataset = paddle.vision.datasets.MNIST( mode='train', transform=paddle.vision.transforms.Compose( - [paddle.vision.transforms.ToTensor()])) - loader = paddle.io.DataLoader(dataset, - batch_size=64, - num_workers=1, - shuffle=True) + [paddle.vision.transforms.ToTensor()] + ), + ) + loader = paddle.io.DataLoader( + dataset, batch_size=64, num_workers=1, shuffle=True + ) mnist = MNIST() - sgd = paddle.optimizer.SGD(learning_rate=0.01, - parameters=mnist.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.01, parameters=mnist.parameters() + ) data = next(loader()) img = data[0] @@ -139,6 +145,7 @@ class TestCustomCPUPlugin(unittest.TestCase): grad = np.ones([2, 2]).astype("float32") import paddle + paddle.set_device('custom_cpu') paddle.device.get_available_device() x_tensor = paddle.to_tensor(x, stop_gradient=False) @@ -153,18 +160,21 @@ class TestCustomCPUPlugin(unittest.TestCase): def _test_eager_copy_to(self): import paddle + x = np.random.random([2, 2]).astype("float32") # cpu -> custom - cpu_tensor = paddle.to_tensor(x, - dtype='float32', - place=paddle.CPUPlace()) + cpu_tensor = paddle.to_tensor( + x, dtype='float32', place=paddle.CPUPlace() + ) custom_cpu_tensor = cpu_tensor._copy_to( - paddle.CustomPlace('custom_cpu', 0), True) + paddle.CustomPlace('custom_cpu', 0), True + ) np.testing.assert_array_equal(custom_cpu_tensor, x) self.assertTrue(custom_cpu_tensor.place.is_custom_place()) # custom -> custom another_custom_cpu_tensor = custom_cpu_tensor._copy_to( - paddle.CustomPlace('custom_cpu', 0), True) + paddle.CustomPlace('custom_cpu', 0), True + ) np.testing.assert_array_equal(another_custom_cpu_tensor, x) self.assertTrue(another_custom_cpu_tensor.place.is_custom_place()) # custom -> cpu @@ -173,13 +183,15 @@ class TestCustomCPUPlugin(unittest.TestCase): self.assertTrue(another_cpu_tensor.place.is_cpu_place()) # custom -> custom self another_custom_cpu_tensor = another_custom_cpu_tensor._copy_to( - paddle.CustomPlace('custom_cpu', 0), True) + paddle.CustomPlace('custom_cpu', 0), True + ) np.testing.assert_array_equal(another_custom_cpu_tensor, x) self.assertTrue(another_custom_cpu_tensor.place.is_custom_place()) def _test_fallback_kernel(self): # using (custom_cpu, add, int16) which is not registered import paddle + r = np.array([6, 6, 6], 'int16') x = paddle.to_tensor([5, 4, 3], 'int16') y = paddle.to_tensor([1, 2, 3], 'int16') @@ -188,8 +200,10 @@ class TestCustomCPUPlugin(unittest.TestCase): def _test_scalar(self): import paddle - data_1 = paddle.to_tensor([[[[1.0, 4.0, 5.0, 7.0], [3.0, 4.0, 5.0, - 6.0]]]]) + + data_1 = paddle.to_tensor( + [[[[1.0, 4.0, 5.0, 7.0], [3.0, 4.0, 5.0, 6.0]]]] + ) k_t = paddle.to_tensor([3], dtype="int32") value_1, indices_1 = paddle.topk(data_1, k=k_t) diff --git a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_profiler_plugin.py b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_profiler_plugin.py index 2e307fbb826..82c2ddbbe73 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_profiler_plugin.py +++ b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_profiler_plugin.py @@ -19,7 +19,6 @@ import tempfile class TestCustomCPUProfilerPlugin(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -31,15 +30,18 @@ class TestCustomCPUProfilerPlugin(unittest.TestCase): && git checkout {} -b dev \ && cd backends/custom_cpu \ && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), - os.getenv('PLUGIN_TAG')) + self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + ) os.system(cmd) # set environment for loading and registering compiled custom kernels # only valid in current process os.environ['CUSTOM_DEVICE_ROOT'] = os.path.join( - cur_dir, '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( - self.temp_dir.name)) + cur_dir, + '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( + self.temp_dir.name + ), + ) def tearDown(self): self.temp_dir.cleanup() @@ -47,6 +49,7 @@ class TestCustomCPUProfilerPlugin(unittest.TestCase): def test_custom_device(self): import paddle + with paddle.fluid.framework._test_eager_guard(): self._test_custom_profiler() @@ -57,9 +60,12 @@ class TestCustomCPUProfilerPlugin(unittest.TestCase): paddle.set_device('custom_cpu') x = paddle.to_tensor([1, 2, 3]) - p = profiler.Profiler(targets=[ - profiler.ProfilerTarget.CPU, profiler.ProfilerTarget.CUSTOM_DEVICE - ]) + p = profiler.Profiler( + targets=[ + profiler.ProfilerTarget.CPU, + profiler.ProfilerTarget.CUSTOM_DEVICE, + ] + ) p.start() for iter in range(10): x = x + 1 diff --git a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_to_static.py b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_to_static.py index e5e9638a0b5..2d8796152b5 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_to_static.py +++ b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_to_static.py @@ -35,8 +35,11 @@ def train_func_base(epoch_id, train_loader, model, cost, optimizer): loss.backward() optimizer.step() optimizer.clear_grad() - print("Epoch [{}/{}], Step [{}/{}], Loss: {}".format( - epoch_id + 1, EPOCH_NUM, batch_id + 1, total_step, loss.numpy())) + print( + "Epoch [{}/{}], Step [{}/{}], Loss: {}".format( + epoch_id + 1, EPOCH_NUM, batch_id + 1, total_step, loss.numpy() + ) + ) epoch_end = time.time() print( f"Epoch ID: {epoch_id+1}, FP32 train epoch time: {(epoch_end - epoch_start) * 1000} ms" @@ -51,8 +54,9 @@ def train_func_ampo1(epoch_id, train_loader, model, cost, optimizer, scaler): for batch_id, (images, labels) in enumerate(train_loader()): # forward with paddle.amp.auto_cast( - custom_black_list={"flatten_contiguous_range", "greater_than"}, - level='O1'): + custom_black_list={"flatten_contiguous_range", "greater_than"}, + level='O1', + ): outputs = model(images) loss = cost(outputs, labels) # backward and optimize @@ -60,8 +64,11 @@ def train_func_ampo1(epoch_id, train_loader, model, cost, optimizer, scaler): scaled.backward() scaler.minimize(optimizer, scaled) optimizer.clear_grad() - print("Epoch [{}/{}], Step [{}/{}], Loss: {}".format( - epoch_id + 1, EPOCH_NUM, batch_id + 1, total_step, loss.numpy())) + print( + "Epoch [{}/{}], Step [{}/{}], Loss: {}".format( + epoch_id + 1, EPOCH_NUM, batch_id + 1, total_step, loss.numpy() + ) + ) epoch_end = time.time() print( f"Epoch ID: {epoch_id+1}, AMPO1 train epoch time: {(epoch_end - epoch_start) * 1000} ms" @@ -90,7 +97,6 @@ def test_func(epoch_id, test_loader, model, cost): class TestCustomCPUPlugin(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -102,15 +108,18 @@ class TestCustomCPUPlugin(unittest.TestCase): && git checkout {} -b dev \ && cd backends/custom_cpu \ && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), - os.getenv('PLUGIN_TAG')) + self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + ) os.system(cmd) # set environment for loading and registering compiled custom kernels # only valid in current process os.environ['CUSTOM_DEVICE_ROOT'] = os.path.join( - cur_dir, '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( - self.temp_dir.name)) + cur_dir, + '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( + self.temp_dir.name + ), + ) def tearDown(self): self.temp_dir.cleanup() @@ -123,7 +132,6 @@ class TestCustomCPUPlugin(unittest.TestCase): import paddle class LeNet5(paddle.nn.Layer): - def __init__(self): super(LeNet5, self).__init__() self.fc = paddle.nn.Linear(in_features=1024, out_features=10) @@ -145,35 +153,44 @@ class TestCustomCPUPlugin(unittest.TestCase): # cost and optimizer cost = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) # convert to static model build_strategy = paddle.static.BuildStrategy() mnist = paddle.jit.to_static(model, build_strategy=build_strategy) # data loader - transform = paddle.vision.transforms.Compose([ - paddle.vision.transforms.Resize((32, 32)), - paddle.vision.transforms.ToTensor(), - paddle.vision.transforms.Normalize(mean=(0.1307, ), std=(0.3081, )) - ]) - train_dataset = paddle.vision.datasets.MNIST(mode='train', - transform=transform, - download=True) - test_dataset = paddle.vision.datasets.MNIST(mode='test', - transform=transform, - download=True) - train_loader = paddle.io.DataLoader(train_dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) - test_loader = paddle.io.DataLoader(test_dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + transform = paddle.vision.transforms.Compose( + [ + paddle.vision.transforms.Resize((32, 32)), + paddle.vision.transforms.ToTensor(), + paddle.vision.transforms.Normalize( + mean=(0.1307,), std=(0.3081,) + ), + ] + ) + train_dataset = paddle.vision.datasets.MNIST( + mode='train', transform=transform, download=True + ) + test_dataset = paddle.vision.datasets.MNIST( + mode='test', transform=transform, download=True + ) + train_loader = paddle.io.DataLoader( + train_dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) + test_loader = paddle.io.DataLoader( + test_dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) # train and eval for epoch_id in range(EPOCH_NUM): @@ -184,7 +201,6 @@ class TestCustomCPUPlugin(unittest.TestCase): import paddle class LeNet5(paddle.nn.Layer): - def __init__(self): super(LeNet5, self).__init__() self.fc = paddle.nn.Linear(in_features=1024, out_features=10) @@ -206,42 +222,52 @@ class TestCustomCPUPlugin(unittest.TestCase): # cost and optimizer cost = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) # convert to static model scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - model, optimizer = paddle.amp.decorate(models=model, - optimizers=optimizer, - level='O1') + model, optimizer = paddle.amp.decorate( + models=model, optimizers=optimizer, level='O1' + ) # data loader - transform = paddle.vision.transforms.Compose([ - paddle.vision.transforms.Resize((32, 32)), - paddle.vision.transforms.ToTensor(), - paddle.vision.transforms.Normalize(mean=(0.1307, ), std=(0.3081, )) - ]) - train_dataset = paddle.vision.datasets.MNIST(mode='train', - transform=transform, - download=True) - test_dataset = paddle.vision.datasets.MNIST(mode='test', - transform=transform, - download=True) - train_loader = paddle.io.DataLoader(train_dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) - test_loader = paddle.io.DataLoader(test_dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + transform = paddle.vision.transforms.Compose( + [ + paddle.vision.transforms.Resize((32, 32)), + paddle.vision.transforms.ToTensor(), + paddle.vision.transforms.Normalize( + mean=(0.1307,), std=(0.3081,) + ), + ] + ) + train_dataset = paddle.vision.datasets.MNIST( + mode='train', transform=transform, download=True + ) + test_dataset = paddle.vision.datasets.MNIST( + mode='test', transform=transform, download=True + ) + train_loader = paddle.io.DataLoader( + train_dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) + test_loader = paddle.io.DataLoader( + test_dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) # train and eval for epoch_id in range(EPOCH_NUM): - train_func_ampo1(epoch_id, train_loader, model, cost, optimizer, - scaler) + train_func_ampo1( + epoch_id, train_loader, model, cost, optimizer, scaler + ) test_func(epoch_id, test_loader, model, cost) diff --git a/python/paddle/fluid/tests/test_beam_search_decoder.py b/python/paddle/fluid/tests/test_beam_search_decoder.py index 66cd8e89d32..3a5c8604648 100644 --- a/python/paddle/fluid/tests/test_beam_search_decoder.py +++ b/python/paddle/fluid/tests/test_beam_search_decoder.py @@ -22,7 +22,12 @@ import paddle.fluid as fluid import paddle.fluid.framework as framework import paddle.fluid.layers as layers from paddle.fluid.executor import Executor -from paddle.fluid.contrib.decoder.beam_search_decoder import BeamSearchDecoder, InitState, StateCell, TrainingDecoder +from paddle.fluid.contrib.decoder.beam_search_decoder import ( + BeamSearchDecoder, + InitState, + StateCell, + TrainingDecoder, +) import unittest paddle.enable_static() @@ -43,14 +48,15 @@ beam_size = 2 def encoder(): # encoder - src_word = layers.data(name="src_word", - shape=[1], - dtype='int64', - lod_level=1) - src_embedding = layers.embedding(input=src_word, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=IS_SPARSE) + src_word = layers.data( + name="src_word", shape=[1], dtype='int64', lod_level=1 + ) + src_embedding = layers.embedding( + input=src_word, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=IS_SPARSE, + ) fc1 = layers.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = layers.dynamic_lstm(input=fc1, size=hidden_dim * 4) @@ -67,9 +73,9 @@ def decoder_state_cell(context): current_word = state_cell.get_input('x') prev_h = state_cell.get_state('h') # make sure lod of h heritted from prev_h - h = layers.fc(input=[prev_h, current_word], - size=decoder_size, - act='tanh') + h = layers.fc( + input=[prev_h, current_word], size=decoder_size, act='tanh' + ) state_cell.set_state('h', h) return state_cell @@ -77,23 +83,26 @@ def decoder_state_cell(context): def decoder_train(state_cell): # decoder - trg_language_word = layers.data(name="target_word", - shape=[1], - dtype='int64', - lod_level=1) - trg_embedding = layers.embedding(input=trg_language_word, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=IS_SPARSE) + trg_language_word = layers.data( + name="target_word", shape=[1], dtype='int64', lod_level=1 + ) + trg_embedding = layers.embedding( + input=trg_language_word, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=IS_SPARSE, + ) decoder = TrainingDecoder(state_cell) with decoder.block(): current_word = decoder.step_input(trg_embedding) decoder.state_cell.compute_state(inputs={'x': current_word}) - current_score = layers.fc(input=decoder.state_cell.get_state('h'), - size=target_dict_dim, - act='softmax') + current_score = layers.fc( + input=decoder.state_cell.get_state('h'), + size=target_dict_dim, + act='softmax', + ) decoder.state_cell.update_states() decoder.output(current_score) @@ -101,27 +110,27 @@ def decoder_train(state_cell): def decoder_decode(state_cell): - init_ids = layers.data(name="init_ids", - shape=[1], - dtype="int64", - lod_level=2) - init_scores = layers.data(name="init_scores", - shape=[1], - dtype="float32", - lod_level=2) - - decoder = BeamSearchDecoder(state_cell=state_cell, - init_ids=init_ids, - init_scores=init_scores, - target_dict_dim=target_dict_dim, - word_dim=word_dim, - input_var_dict={}, - topk_size=topk_size, - sparse_emb=IS_SPARSE, - max_len=max_length, - beam_size=beam_size, - end_id=1, - name=None) + init_ids = layers.data( + name="init_ids", shape=[1], dtype="int64", lod_level=2 + ) + init_scores = layers.data( + name="init_scores", shape=[1], dtype="float32", lod_level=2 + ) + + decoder = BeamSearchDecoder( + state_cell=state_cell, + init_ids=init_ids, + init_scores=init_scores, + target_dict_dim=target_dict_dim, + word_dim=word_dim, + input_var_dict={}, + topk_size=topk_size, + sparse_emb=IS_SPARSE, + max_len=max_length, + beam_size=beam_size, + end_id=1, + name=None, + ) decoder.decode() translation_ids, translation_scores = decoder() @@ -136,19 +145,21 @@ def train_main(use_cuda): context = encoder() state_cell = decoder_state_cell(context) rnn_out = decoder_train(state_cell) - label = layers.data(name="target_next_word", - shape=[1], - dtype='int64', - lod_level=1) + label = layers.data( + name="target_next_word", shape=[1], dtype='int64', lod_level=1 + ) cost = layers.cross_entropy(input=rnn_out, label=label) avg_cost = paddle.mean(x=cost) optimizer = fluid.optimizer.Adagrad(learning_rate=1e-3) optimizer.minimize(avg_cost) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) feed_order = ['src_word', 'target_word', 'target_next_word'] exe = Executor(place) @@ -163,12 +174,18 @@ def train_main(use_cuda): for pass_id in range(1): for batch_id, data in enumerate(train_reader()): - outs = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + outs = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) avg_cost_val = np.array(outs[0]) - print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + - " avg_cost=" + str(avg_cost_val)) + print( + 'pass_id=' + + str(pass_id) + + ' batch=' + + str(batch_id) + + " avg_cost=" + + str(avg_cost_val) + ) if batch_id > 3: break @@ -188,8 +205,9 @@ def decode_main(use_cuda): exe.run(framework.default_startup_program()) init_ids_data = np.array([0 for _ in range(batch_size)], dtype='int64') - init_scores_data = np.array([1. for _ in range(batch_size)], - dtype='float32') + init_scores_data = np.array( + [1.0 for _ in range(batch_size)], dtype='float32' + ) init_ids_data = init_ids_data.reshape((batch_size, 1)) init_scores_data = init_scores_data.reshape((batch_size, 1)) init_lod = [1] * batch_size @@ -198,9 +216,12 @@ def decode_main(use_cuda): init_ids = fluid.create_lod_tensor(init_ids_data, init_lod, place) init_scores = fluid.create_lod_tensor(init_scores_data, init_lod, place) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) feed_order = ['src_word'] feed_list = [ @@ -218,7 +239,8 @@ def decode_main(use_cuda): framework.default_main_program(), feed=feed_dict, fetch_list=[translation_ids, translation_scores], - return_numpy=False) + return_numpy=False, + ) print(result_ids.lod()) diff --git a/python/paddle/fluid/tests/test_data_feeder.py b/python/paddle/fluid/tests/test_data_feeder.py index 5f97a250d75..515bf44c755 100644 --- a/python/paddle/fluid/tests/test_data_feeder.py +++ b/python/paddle/fluid/tests/test_data_feeder.py @@ -20,7 +20,6 @@ paddle.enable_static() class TestDataFeeder(unittest.TestCase): - def test_lod_level_0_converter(self): img = fluid.layers.data(name='image', shape=[1, 28, 28]) label = fluid.layers.data(name='label', shape=[1], dtype='int64') @@ -41,45 +40,48 @@ class TestDataFeeder(unittest.TestCase): def test_lod_level_1_converter(self): # lod_level = 1 # each sentence has a different number of words - sentences = fluid.layers.data(name='sentences', - shape=[1], - dtype='int64', - lod_level=1) + sentences = fluid.layers.data( + name='sentences', shape=[1], dtype='int64', lod_level=1 + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') feeder = fluid.DataFeeder([sentences, label], fluid.CPUPlace()) # lod = [[0, 3, 5, 9]] # data = [[1, 2, 3], [4, 5], [6, 7, 8, 9]] # label = [1] * len(data) - result = feeder.feed([([1, 2, 3], [1]), ([4, 5], [1]), - ([6, 7, 8, 9], [1])]) + result = feeder.feed( + [([1, 2, 3], [1]), ([4, 5], [1]), ([6, 7, 8, 9], [1])] + ) self.assertEqual(result['sentences'].shape(), [9, 1]) self.assertEqual(result['label'].shape(), [3, 1]) - self.assertEqual(result['sentences'].recursive_sequence_lengths(), - [[3, 2, 4]]) + self.assertEqual( + result['sentences'].recursive_sequence_lengths(), [[3, 2, 4]] + ) self.assertEqual(result['label'].recursive_sequence_lengths(), []) def test_lod_level_2_converter(self): # lod_level = 2 # paragraphs -> sentences -> words - paragraphs = fluid.layers.data(name='paragraphs', - shape=[1], - dtype='int64', - lod_level=2) + paragraphs = fluid.layers.data( + name='paragraphs', shape=[1], dtype='int64', lod_level=2 + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') feeder = fluid.DataFeeder([paragraphs, label], fluid.CPUPlace()) # lod = [[0, 2, 3], [0, 3, 5, 9]] # data = [[[1, 2, 3], [4, 5]], [[6, 7, 8, 9]]] # label = [1] * len(data) - result = feeder.feed([([[1, 2, 3], [4, 5]], [1]), ([[6, 7, 8, - 9]], [1])]) + result = feeder.feed( + [([[1, 2, 3], [4, 5]], [1]), ([[6, 7, 8, 9]], [1])] + ) self.assertEqual(result['paragraphs'].shape(), [9, 1]) self.assertEqual(result['label'].shape(), [2, 1]) - self.assertEqual(result['paragraphs'].recursive_sequence_lengths(), - [[2, 1], [3, 2, 4]]) + self.assertEqual( + result['paragraphs'].recursive_sequence_lengths(), + [[2, 1], [3, 2, 4]], + ) self.assertEqual(result['label'].recursive_sequence_lengths(), []) diff --git a/python/paddle/fluid/tests/test_detection.py b/python/paddle/fluid/tests/test_detection.py index bd3b8df7385..baa80a73aa2 100644 --- a/python/paddle/fluid/tests/test_detection.py +++ b/python/paddle/fluid/tests/test_detection.py @@ -28,7 +28,6 @@ paddle.enable_static() class LayerTest(unittest.TestCase): - @classmethod def setUpClass(cls): cls.seed = 111 @@ -53,57 +52,66 @@ class LayerTest(unittest.TestCase): fluid.default_main_program().random_seed = self.seed yield - def get_static_graph_result(self, - feed, - fetch_list, - with_lod=False, - force_to_use_cpu=False): + def get_static_graph_result( + self, feed, fetch_list, with_lod=False, force_to_use_cpu=False + ): exe = fluid.Executor(self._get_place(force_to_use_cpu)) exe.run(fluid.default_startup_program()) - return exe.run(fluid.default_main_program(), - feed=feed, - fetch_list=fetch_list, - return_numpy=(not with_lod)) + return exe.run( + fluid.default_main_program(), + feed=feed, + fetch_list=fetch_list, + return_numpy=(not with_lod), + ) @contextlib.contextmanager def dynamic_graph(self, force_to_use_cpu=False): with fluid.dygraph.guard( - self._get_place(force_to_use_cpu=force_to_use_cpu)): + self._get_place(force_to_use_cpu=force_to_use_cpu) + ): fluid.default_startup_program().random_seed = self.seed fluid.default_main_program().random_seed = self.seed yield class TestDetection(unittest.TestCase): - def test_detection_output(self): program = Program() with program_guard(program): - pb = layers.data(name='prior_box', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - pbv = layers.data(name='prior_box_var', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - loc = layers.data(name='target_box', - shape=[2, 10, 4], - append_batch_size=False, - dtype='float32') - scores = layers.data(name='scores', - shape=[2, 10, 20], - append_batch_size=False, - dtype='float32') - out = layers.detection_output(scores=scores, - loc=loc, - prior_box=pb, - prior_box_var=pbv) - out2, index = layers.detection_output(scores=scores, - loc=loc, - prior_box=pb, - prior_box_var=pbv, - return_index=True) + pb = layers.data( + name='prior_box', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + pbv = layers.data( + name='prior_box_var', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + loc = layers.data( + name='target_box', + shape=[2, 10, 4], + append_batch_size=False, + dtype='float32', + ) + scores = layers.data( + name='scores', + shape=[2, 10, 20], + append_batch_size=False, + dtype='float32', + ) + out = layers.detection_output( + scores=scores, loc=loc, prior_box=pb, prior_box_var=pbv + ) + out2, index = layers.detection_output( + scores=scores, + loc=loc, + prior_box=pb, + prior_box_var=pbv, + return_index=True, + ) self.assertIsNotNone(out) self.assertIsNotNone(out2) self.assertIsNotNone(index) @@ -115,10 +123,12 @@ class TestDetection(unittest.TestCase): with program_guard(program): x = layers.data(name='x', shape=[4], dtype='float32') y = layers.data(name='z', shape=[4], dtype='float32', lod_level=1) - bcoder = layers.box_coder(prior_box=x, - prior_box_var=[0.1, 0.2, 0.1, 0.2], - target_box=y, - code_type='encode_center_size') + bcoder = layers.box_coder( + prior_box=x, + prior_box_var=[0.1, 0.2, 0.1, 0.2], + target_box=y, + code_type='encode_center_size', + ) self.assertIsNotNone(bcoder) print(str(program)) @@ -126,28 +136,30 @@ class TestDetection(unittest.TestCase): program = Program() with program_guard(program): x1 = fluid.data(name='x1', shape=[10, 4], dtype='int32') - y1 = fluid.data(name='y1', - shape=[10, 4], - dtype='float32', - lod_level=1) + y1 = fluid.data( + name='y1', shape=[10, 4], dtype='float32', lod_level=1 + ) x2 = fluid.data(name='x2', shape=[10, 4], dtype='float32') - y2 = fluid.data(name='y2', - shape=[10, 4], - dtype='int32', - lod_level=1) - - self.assertRaises(TypeError, - layers.box_coder, - prior_box=x1, - prior_box_var=[0.1, 0.2, 0.1, 0.2], - target_box=y1, - code_type='encode_center_size') - self.assertRaises(TypeError, - layers.box_coder, - prior_box=x2, - prior_box_var=[0.1, 0.2, 0.1, 0.2], - target_box=y2, - code_type='encode_center_size') + y2 = fluid.data( + name='y2', shape=[10, 4], dtype='int32', lod_level=1 + ) + + self.assertRaises( + TypeError, + layers.box_coder, + prior_box=x1, + prior_box_var=[0.1, 0.2, 0.1, 0.2], + target_box=y1, + code_type='encode_center_size', + ) + self.assertRaises( + TypeError, + layers.box_coder, + prior_box=x2, + prior_box_var=[0.1, 0.2, 0.1, 0.2], + target_box=y2, + code_type='encode_center_size', + ) def test_detection_api(self): program = Program() @@ -156,10 +168,12 @@ class TestDetection(unittest.TestCase): y = layers.data(name='y', shape=[4], dtype='float32') z = layers.data(name='z', shape=[4], dtype='float32', lod_level=1) iou = layers.iou_similarity(x=x, y=y) - bcoder = layers.box_coder(prior_box=x, - prior_box_var=y, - target_box=z, - code_type='encode_center_size') + bcoder = layers.box_coder( + prior_box=x, + prior_box_var=y, + target_box=z, + code_type='encode_center_size', + ) self.assertIsNotNone(iou) self.assertIsNotNone(bcoder) @@ -167,23 +181,21 @@ class TestDetection(unittest.TestCase): self.assertIsNotNone(matched_indices) self.assertIsNotNone(matched_dist) - gt = layers.data(name='gt', - shape=[1, 1], - dtype='int32', - lod_level=1) - trg, trg_weight = layers.target_assign(gt, - matched_indices, - mismatch_value=0) + gt = layers.data( + name='gt', shape=[1, 1], dtype='int32', lod_level=1 + ) + trg, trg_weight = layers.target_assign( + gt, matched_indices, mismatch_value=0 + ) self.assertIsNotNone(trg) self.assertIsNotNone(trg_weight) - gt2 = layers.data(name='gt2', - shape=[10, 4], - dtype='float32', - lod_level=1) - trg, trg_weight = layers.target_assign(gt2, - matched_indices, - mismatch_value=0) + gt2 = layers.data( + name='gt2', shape=[10, 4], dtype='float32', lod_level=1 + ) + trg, trg_weight = layers.target_assign( + gt2, matched_indices, mismatch_value=0 + ) self.assertIsNotNone(trg) self.assertIsNotNone(trg_weight) @@ -192,24 +204,26 @@ class TestDetection(unittest.TestCase): def test_ssd_loss(self): program = Program() with program_guard(program): - pb = layers.data(name='prior_box', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - pbv = layers.data(name='prior_box_var', - shape=[10, 4], - append_batch_size=False, - dtype='float32') + pb = layers.data( + name='prior_box', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + pbv = layers.data( + name='prior_box_var', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) loc = layers.data(name='target_box', shape=[10, 4], dtype='float32') scores = layers.data(name='scores', shape=[10, 21], dtype='float32') - gt_box = layers.data(name='gt_box', - shape=[4], - lod_level=1, - dtype='float32') - gt_label = layers.data(name='gt_label', - shape=[1], - lod_level=1, - dtype='int32') + gt_box = layers.data( + name='gt_box', shape=[4], lod_level=1, dtype='float32' + ) + gt_label = layers.data( + name='gt_label', shape=[1], lod_level=1, dtype='int32' + ) loss = layers.ssd_loss(loc, scores, gt_box, gt_label, pb, pbv) self.assertIsNotNone(loss) self.assertEqual(loss.shape[-1], 1) @@ -217,73 +231,75 @@ class TestDetection(unittest.TestCase): class TestPriorBox(unittest.TestCase): - def test_prior_box(self): program = Program() with program_guard(program): data_shape = [3, 224, 224] - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) conv1 = fluid.layers.conv2d(images, 3, 3, 2) - box, var = layers.prior_box(input=conv1, - image=images, - min_sizes=[100.0], - aspect_ratios=[1.], - flip=True, - clip=True) + box, var = layers.prior_box( + input=conv1, + image=images, + min_sizes=[100.0], + aspect_ratios=[1.0], + flip=True, + clip=True, + ) assert len(box.shape) == 4 assert box.shape == var.shape assert box.shape[3] == 4 class TestPriorBox2(unittest.TestCase): - def test_prior_box(self): program = Program() with program_guard(program): data_shape = [None, 3, None, None] images = fluid.data(name='pixel', shape=data_shape, dtype='float32') conv1 = fluid.layers.conv2d(images, 3, 3, 2) - box, var = layers.prior_box(input=conv1, - image=images, - min_sizes=[100.0], - aspect_ratios=[1.], - flip=True, - clip=True) + box, var = layers.prior_box( + input=conv1, + image=images, + min_sizes=[100.0], + aspect_ratios=[1.0], + flip=True, + clip=True, + ) assert len(box.shape) == 4 assert box.shape == var.shape assert box.shape[3] == 4 class TestDensityPriorBox(unittest.TestCase): - def test_density_prior_box(self): program = Program() with program_guard(program): data_shape = [3, 224, 224] - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) conv1 = fluid.layers.conv2d(images, 3, 3, 2) - box, var = layers.density_prior_box(input=conv1, - image=images, - densities=[3, 4], - fixed_sizes=[50., 60.], - fixed_ratios=[1.0], - clip=True) + box, var = layers.density_prior_box( + input=conv1, + image=images, + densities=[3, 4], + fixed_sizes=[50.0, 60.0], + fixed_ratios=[1.0], + clip=True, + ) assert len(box.shape) == 4 assert box.shape == var.shape assert box.shape[-1] == 4 class TestAnchorGenerator(unittest.TestCase): - def test_anchor_generator(self): data_shape = [3, 224, 224] - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) conv1 = fluid.layers.conv2d(images, 3, 3, 2) anchor, var = fluid.layers.anchor_generator( input=conv1, @@ -291,14 +307,14 @@ class TestAnchorGenerator(unittest.TestCase): aspect_ratios=[0.5, 1.0, 2.0], variance=[0.1, 0.1, 0.2, 0.2], stride=[16.0, 16.0], - offset=0.5) + offset=0.5, + ) assert len(anchor.shape) == 4 assert anchor.shape == var.shape assert anchor.shape[3] == 4 class TestGenerateProposalLabels(unittest.TestCase): - def check_out(self, outs): rois = outs[0] labels_int32 = outs[1] @@ -320,27 +336,22 @@ class TestGenerateProposalLabels(unittest.TestCase): def test_generate_proposal_labels(self): program = Program() with program_guard(program): - rpn_rois = fluid.data(name='rpn_rois', - shape=[4, 4], - dtype='float32', - lod_level=1) - gt_classes = fluid.data(name='gt_classes', - shape=[6], - dtype='int32', - lod_level=1) - is_crowd = fluid.data(name='is_crowd', - shape=[6], - dtype='int32', - lod_level=1) - gt_boxes = fluid.data(name='gt_boxes', - shape=[6, 4], - dtype='float32', - lod_level=1) + rpn_rois = fluid.data( + name='rpn_rois', shape=[4, 4], dtype='float32', lod_level=1 + ) + gt_classes = fluid.data( + name='gt_classes', shape=[6], dtype='int32', lod_level=1 + ) + is_crowd = fluid.data( + name='is_crowd', shape=[6], dtype='int32', lod_level=1 + ) + gt_boxes = fluid.data( + name='gt_boxes', shape=[6, 4], dtype='float32', lod_level=1 + ) im_info = fluid.data(name='im_info', shape=[1, 3], dtype='float32') - max_overlap = fluid.data(name='max_overlap', - shape=[4], - dtype='float32', - lod_level=1) + max_overlap = fluid.data( + name='max_overlap', shape=[4], dtype='float32', lod_level=1 + ) self.class_nums = 5 outs = fluid.layers.generate_proposal_labels( rpn_rois=rpn_rois, @@ -354,7 +365,8 @@ class TestGenerateProposalLabels(unittest.TestCase): bg_thresh_hi=0.5, bg_thresh_lo=0.0, bbox_reg_weights=[0.1, 0.1, 0.2, 0.2], - class_nums=self.class_nums) + class_nums=self.class_nums, + ) outs_1 = fluid.layers.generate_proposal_labels( rpn_rois=rpn_rois, gt_classes=gt_classes, @@ -370,7 +382,8 @@ class TestGenerateProposalLabels(unittest.TestCase): class_nums=self.class_nums, is_cascade_rcnn=True, max_overlap=max_overlap, - return_max_overlap=True) + return_max_overlap=True, + ) self.check_out(outs) self.check_out(outs_1) @@ -378,57 +391,69 @@ class TestGenerateProposalLabels(unittest.TestCase): class TestGenerateMaskLabels(unittest.TestCase): - def test_generate_mask_labels(self): program = Program() with program_guard(program): - im_info = layers.data(name='im_info', - shape=[1, 3], - dtype='float32', - lod_level=1, - append_batch_size=False) - gt_classes = layers.data(name='gt_classes', - shape=[2, 1], - dtype='int32', - lod_level=1, - append_batch_size=False) - is_crowd = layers.data(name='is_crowd', - shape=[2, 1], - dtype='int32', - lod_level=1, - append_batch_size=False) - gt_segms = layers.data(name='gt_segms', - shape=[20, 2], - dtype='float32', - lod_level=3, - append_batch_size=False) - rois = layers.data(name='rois', - shape=[4, 4], - dtype='float32', - lod_level=1, - append_batch_size=False) - labels_int32 = layers.data(name='labels_int32', - shape=[4, 1], - dtype='int32', - lod_level=1, - append_batch_size=False) + im_info = layers.data( + name='im_info', + shape=[1, 3], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + gt_classes = layers.data( + name='gt_classes', + shape=[2, 1], + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + is_crowd = layers.data( + name='is_crowd', + shape=[2, 1], + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + gt_segms = layers.data( + name='gt_segms', + shape=[20, 2], + dtype='float32', + lod_level=3, + append_batch_size=False, + ) + rois = layers.data( + name='rois', + shape=[4, 4], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + labels_int32 = layers.data( + name='labels_int32', + shape=[4, 1], + dtype='int32', + lod_level=1, + append_batch_size=False, + ) num_classes = 5 resolution = 14 - outs = fluid.layers.generate_mask_labels(im_info=im_info, - gt_classes=gt_classes, - is_crowd=is_crowd, - gt_segms=gt_segms, - rois=rois, - labels_int32=labels_int32, - num_classes=num_classes, - resolution=resolution) + outs = fluid.layers.generate_mask_labels( + im_info=im_info, + gt_classes=gt_classes, + is_crowd=is_crowd, + gt_segms=gt_segms, + rois=rois, + labels_int32=labels_int32, + num_classes=num_classes, + resolution=resolution, + ) mask_rois, roi_has_mask_int32, mask_int32 = outs assert mask_rois.shape[1] == 4 assert mask_int32.shape[1] == num_classes * resolution * resolution class TestMultiBoxHead(unittest.TestCase): - def test_multi_box_head(self): data_shape = [3, 224, 224] mbox_locs, mbox_confs, box, var = self.multi_box_head_output(data_shape) @@ -439,9 +464,9 @@ class TestMultiBoxHead(unittest.TestCase): assert mbox_locs.shape[1] == mbox_confs.shape[1] def multi_box_head_output(self, data_shape): - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) conv1 = fluid.layers.conv2d(images, 3, 3, 2) conv2 = fluid.layers.conv2d(conv1, 3, 3, 2) conv3 = fluid.layers.conv2d(conv2, 3, 3, 2) @@ -454,37 +479,47 @@ class TestMultiBoxHead(unittest.TestCase): num_classes=21, min_ratio=20, max_ratio=90, - aspect_ratios=[[2.], [2., 3.], [2., 3.], [2., 3.], [2.], [2.]], + aspect_ratios=[ + [2.0], + [2.0, 3.0], + [2.0, 3.0], + [2.0, 3.0], + [2.0], + [2.0], + ], base_size=300, offset=0.5, flip=True, - clip=True) + clip=True, + ) return mbox_locs, mbox_confs, box, var class TestDetectionMAP(unittest.TestCase): - def test_detection_map(self): program = Program() with program_guard(program): - detect_res = layers.data(name='detect_res', - shape=[10, 6], - append_batch_size=False, - dtype='float32') - label = layers.data(name='label', - shape=[10, 6], - append_batch_size=False, - dtype='float32') + detect_res = layers.data( + name='detect_res', + shape=[10, 6], + append_batch_size=False, + dtype='float32', + ) + label = layers.data( + name='label', + shape=[10, 6], + append_batch_size=False, + dtype='float32', + ) map_out = detection.detection_map(detect_res, label, 21) self.assertIsNotNone(map_out) - self.assertEqual(map_out.shape, (1, )) + self.assertEqual(map_out.shape, (1,)) print(str(program)) class TestRpnTargetAssign(unittest.TestCase): - def test_rpn_target_assign(self): program = Program() with program_guard(program): @@ -492,49 +527,62 @@ class TestRpnTargetAssign(unittest.TestCase): cls_logits_shape = [10, 50, 2] anchor_shape = [50, 4] - bbox_pred = layers.data(name='bbox_pred', - shape=bbox_pred_shape, - append_batch_size=False, - dtype='float32') - cls_logits = layers.data(name='cls_logits', - shape=cls_logits_shape, - append_batch_size=False, - dtype='float32') - anchor_box = layers.data(name='anchor_box', - shape=anchor_shape, - append_batch_size=False, - dtype='float32') - anchor_var = layers.data(name='anchor_var', - shape=anchor_shape, - append_batch_size=False, - dtype='float32') - gt_boxes = layers.data(name='gt_boxes', - shape=[4], - lod_level=1, - dtype='float32') - is_crowd = layers.data(name='is_crowd', - shape=[1, 10], - dtype='int32', - lod_level=1, - append_batch_size=False) - im_info = layers.data(name='im_info', - shape=[1, 3], - dtype='float32', - lod_level=1, - append_batch_size=False) - outs = layers.rpn_target_assign(bbox_pred=bbox_pred, - cls_logits=cls_logits, - anchor_box=anchor_box, - anchor_var=anchor_var, - gt_boxes=gt_boxes, - is_crowd=is_crowd, - im_info=im_info, - rpn_batch_size_per_im=256, - rpn_straddle_thresh=0.0, - rpn_fg_fraction=0.5, - rpn_positive_overlap=0.7, - rpn_negative_overlap=0.3, - use_random=False) + bbox_pred = layers.data( + name='bbox_pred', + shape=bbox_pred_shape, + append_batch_size=False, + dtype='float32', + ) + cls_logits = layers.data( + name='cls_logits', + shape=cls_logits_shape, + append_batch_size=False, + dtype='float32', + ) + anchor_box = layers.data( + name='anchor_box', + shape=anchor_shape, + append_batch_size=False, + dtype='float32', + ) + anchor_var = layers.data( + name='anchor_var', + shape=anchor_shape, + append_batch_size=False, + dtype='float32', + ) + gt_boxes = layers.data( + name='gt_boxes', shape=[4], lod_level=1, dtype='float32' + ) + is_crowd = layers.data( + name='is_crowd', + shape=[1, 10], + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + im_info = layers.data( + name='im_info', + shape=[1, 3], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + outs = layers.rpn_target_assign( + bbox_pred=bbox_pred, + cls_logits=cls_logits, + anchor_box=anchor_box, + anchor_var=anchor_var, + gt_boxes=gt_boxes, + is_crowd=is_crowd, + im_info=im_info, + rpn_batch_size_per_im=256, + rpn_straddle_thresh=0.0, + rpn_fg_fraction=0.5, + rpn_positive_overlap=0.7, + rpn_negative_overlap=0.3, + use_random=False, + ) pred_scores = outs[0] pred_loc = outs[1] tgt_lbl = outs[2] @@ -553,29 +601,29 @@ class TestRpnTargetAssign(unittest.TestCase): class TestGenerateProposals(LayerTest): - def test_generate_proposals(self): scores_np = np.random.rand(2, 3, 4, 4).astype('float32') bbox_deltas_np = np.random.rand(2, 12, 4, 4).astype('float32') im_info_np = np.array([[8, 8, 0.5], [6, 6, 0.5]]).astype('float32') - anchors_np = np.reshape(np.arange(4 * 4 * 3 * 4), - [4, 4, 3, 4]).astype('float32') + anchors_np = np.reshape(np.arange(4 * 4 * 3 * 4), [4, 4, 3, 4]).astype( + 'float32' + ) variances_np = np.ones((4, 4, 3, 4)).astype('float32') with self.static_graph(): - scores = fluid.data(name='scores', - shape=[2, 3, 4, 4], - dtype='float32') - bbox_deltas = fluid.data(name='bbox_deltas', - shape=[2, 12, 4, 4], - dtype='float32') + scores = fluid.data( + name='scores', shape=[2, 3, 4, 4], dtype='float32' + ) + bbox_deltas = fluid.data( + name='bbox_deltas', shape=[2, 12, 4, 4], dtype='float32' + ) im_info = fluid.data(name='im_info', shape=[2, 3], dtype='float32') - anchors = fluid.data(name='anchors', - shape=[4, 4, 3, 4], - dtype='float32') - variances = fluid.data(name='var', - shape=[4, 4, 3, 4], - dtype='float32') + anchors = fluid.data( + name='anchors', shape=[4, 4, 3, 4], dtype='float32' + ) + variances = fluid.data( + name='var', shape=[4, 4, 3, 4], dtype='float32' + ) rois, roi_probs, rois_num = fluid.layers.generate_proposals( scores, bbox_deltas, @@ -584,17 +632,23 @@ class TestGenerateProposals(LayerTest): variances, pre_nms_top_n=10, post_nms_top_n=5, - return_rois_num=True) - rois_stat, roi_probs_stat, rois_num_stat = self.get_static_graph_result( + return_rois_num=True, + ) + ( + rois_stat, + roi_probs_stat, + rois_num_stat, + ) = self.get_static_graph_result( feed={ 'scores': scores_np, 'bbox_deltas': bbox_deltas_np, 'im_info': im_info_np, 'anchors': anchors_np, - 'var': variances_np + 'var': variances_np, }, fetch_list=[rois, roi_probs, rois_num], - with_lod=False) + with_lod=False, + ) with self.dynamic_graph(): scores_dy = base.to_variable(scores_np) @@ -610,7 +664,8 @@ class TestGenerateProposals(LayerTest): variances_dy, pre_nms_top_n=10, post_nms_top_n=5, - return_rois_num=True) + return_rois_num=True, + ) rois_dy = rois.numpy() roi_probs_dy = roi_probs.numpy() rois_num_dy = rois_num.numpy() @@ -621,7 +676,6 @@ class TestGenerateProposals(LayerTest): class TestYoloDetection(unittest.TestCase): - def test_yolov3_loss(self): program = Program() with program_guard(program): @@ -629,14 +683,18 @@ class TestYoloDetection(unittest.TestCase): gt_box = layers.data(name='gt_box', shape=[10, 4], dtype='float32') gt_label = layers.data(name='gt_label', shape=[10], dtype='int32') gt_score = layers.data(name='gt_score', shape=[10], dtype='float32') - loss = layers.yolov3_loss(x, - gt_box, - gt_label, [10, 13, 30, 13], [0, 1], - 10, - 0.7, - 32, - gt_score=gt_score, - use_label_smooth=False) + loss = layers.yolov3_loss( + x, + gt_box, + gt_label, + [10, 13, 30, 13], + [0, 1], + 10, + 0.7, + 32, + gt_score=gt_score, + use_label_smooth=False, + ) self.assertIsNotNone(loss) @@ -645,8 +703,9 @@ class TestYoloDetection(unittest.TestCase): with program_guard(program): x = layers.data(name='x', shape=[30, 7, 7], dtype='float32') img_size = layers.data(name='img_size', shape=[2], dtype='int32') - boxes, scores = layers.yolo_box(x, img_size, [10, 13, 30, 13], 10, - 0.01, 32) + boxes, scores = layers.yolo_box( + x, img_size, [10, 13, 30, 13], 10, 0.01, 32 + ) self.assertIsNotNone(boxes) self.assertIsNotNone(scores) @@ -657,15 +716,19 @@ class TestYoloDetection(unittest.TestCase): gt_box = layers.data(name='gt_box', shape=[10, 4], dtype='float32') gt_label = layers.data(name='gt_label', shape=[10], dtype='int32') gt_score = layers.data(name='gt_score', shape=[10], dtype='float32') - loss = layers.yolov3_loss(x, - gt_box, - gt_label, [10, 13, 30, 13], [0, 1], - 10, - 0.7, - 32, - gt_score=gt_score, - use_label_smooth=False, - scale_x_y=1.2) + loss = layers.yolov3_loss( + x, + gt_box, + gt_label, + [10, 13, 30, 13], + [0, 1], + 10, + 0.7, + 32, + gt_score=gt_score, + use_label_smooth=False, + scale_x_y=1.2, + ) self.assertIsNotNone(loss) @@ -674,38 +737,32 @@ class TestYoloDetection(unittest.TestCase): with program_guard(program): x = layers.data(name='x', shape=[30, 7, 7], dtype='float32') img_size = layers.data(name='img_size', shape=[2], dtype='int32') - boxes, scores = layers.yolo_box(x, - img_size, [10, 13, 30, 13], - 10, - 0.01, - 32, - scale_x_y=1.2) + boxes, scores = layers.yolo_box( + x, img_size, [10, 13, 30, 13], 10, 0.01, 32, scale_x_y=1.2 + ) self.assertIsNotNone(boxes) self.assertIsNotNone(scores) class TestBoxClip(unittest.TestCase): - def test_box_clip(self): program = Program() with program_guard(program): - input_box = layers.data(name='input_box', - shape=[7, 4], - dtype='float32', - lod_level=1) + input_box = layers.data( + name='input_box', shape=[7, 4], dtype='float32', lod_level=1 + ) im_info = layers.data(name='im_info', shape=[3], dtype='float32') out = layers.box_clip(input_box, im_info) self.assertIsNotNone(out) class TestMulticlassNMS(unittest.TestCase): - def test_multiclass_nms(self): program = Program() with program_guard(program): - bboxes = layers.data(name='bboxes', - shape=[-1, 10, 4], - dtype='float32') + bboxes = layers.data( + name='bboxes', shape=[-1, 10, 4], dtype='float32' + ) scores = layers.data(name='scores', shape=[-1, 10], dtype='float32') output = layers.multiclass_nms(bboxes, scores, 0.3, 400, 200, 0.7) self.assertIsNotNone(output) @@ -713,57 +770,56 @@ class TestMulticlassNMS(unittest.TestCase): def test_multiclass_nms_error(self): program = Program() with program_guard(program): - bboxes1 = fluid.data(name='bboxes1', - shape=[10, 10, 4], - dtype='int32') - scores1 = fluid.data(name='scores1', - shape=[10, 10], - dtype='float32') - bboxes2 = fluid.data(name='bboxes2', - shape=[10, 10, 4], - dtype='float32') + bboxes1 = fluid.data( + name='bboxes1', shape=[10, 10, 4], dtype='int32' + ) + scores1 = fluid.data( + name='scores1', shape=[10, 10], dtype='float32' + ) + bboxes2 = fluid.data( + name='bboxes2', shape=[10, 10, 4], dtype='float32' + ) scores2 = fluid.data(name='scores2', shape=[10, 10], dtype='int32') - self.assertRaises(TypeError, - layers.multiclass_nms, - bboxes=bboxes1, - scores=scores1, - score_threshold=0.5, - nms_top_k=400, - keep_top_k=200) - self.assertRaises(TypeError, - layers.multiclass_nms, - bboxes=bboxes2, - scores=scores2, - score_threshold=0.5, - nms_top_k=400, - keep_top_k=200) + self.assertRaises( + TypeError, + layers.multiclass_nms, + bboxes=bboxes1, + scores=scores1, + score_threshold=0.5, + nms_top_k=400, + keep_top_k=200, + ) + self.assertRaises( + TypeError, + layers.multiclass_nms, + bboxes=bboxes2, + scores=scores2, + score_threshold=0.5, + nms_top_k=400, + keep_top_k=200, + ) class TestMulticlassNMS2(unittest.TestCase): - def test_multiclass_nms2(self): program = Program() with program_guard(program): - bboxes = layers.data(name='bboxes', - shape=[-1, 10, 4], - dtype='float32') + bboxes = layers.data( + name='bboxes', shape=[-1, 10, 4], dtype='float32' + ) scores = layers.data(name='scores', shape=[-1, 10], dtype='float32') - output = fluid.contrib.multiclass_nms2(bboxes, scores, 0.3, 400, - 200, 0.7) - output2, index = fluid.contrib.multiclass_nms2(bboxes, - scores, - 0.3, - 400, - 200, - 0.7, - return_index=True) + output = fluid.contrib.multiclass_nms2( + bboxes, scores, 0.3, 400, 200, 0.7 + ) + output2, index = fluid.contrib.multiclass_nms2( + bboxes, scores, 0.3, 400, 200, 0.7, return_index=True + ) self.assertIsNotNone(output) self.assertIsNotNone(output2) self.assertIsNotNone(index) class TestCollectFpnPropsals(LayerTest): - def test_collect_fpn_proposals(self): multi_bboxes_np = [] multi_scores_np = [] @@ -781,17 +837,21 @@ class TestCollectFpnPropsals(LayerTest): multi_scores = [] rois_num_per_level = [] for i in range(4): - bboxes = fluid.data(name='rois' + str(i), - shape=[5, 4], - dtype='float32', - lod_level=1) - scores = fluid.data(name='scores' + str(i), - shape=[5, 1], - dtype='float32', - lod_level=1) - rois_num = fluid.data(name='rois_num' + str(i), - shape=[None], - dtype='int32') + bboxes = fluid.data( + name='rois' + str(i), + shape=[5, 4], + dtype='float32', + lod_level=1, + ) + scores = fluid.data( + name='scores' + str(i), + shape=[5, 1], + dtype='float32', + lod_level=1, + ) + rois_num = fluid.data( + name='rois_num' + str(i), shape=[None], dtype='int32' + ) multi_bboxes.append(bboxes) multi_scores.append(scores) @@ -803,14 +863,16 @@ class TestCollectFpnPropsals(LayerTest): 2, 5, 10, - rois_num_per_level=rois_num_per_level) + rois_num_per_level=rois_num_per_level, + ) feed = {} for i in range(4): feed['rois' + str(i)] = multi_bboxes_np[i] feed['scores' + str(i)] = multi_scores_np[i] feed['rois_num' + str(i)] = rois_num_per_level_np[i] fpn_rois_stat, rois_num_stat = self.get_static_graph_result( - feed=feed, fetch_list=[fpn_rois, rois_num], with_lod=True) + feed=feed, fetch_list=[fpn_rois, rois_num], with_lod=True + ) fpn_rois_stat = np.array(fpn_rois_stat) rois_num_stat = np.array(rois_num_stat) @@ -831,7 +893,8 @@ class TestCollectFpnPropsals(LayerTest): 2, 5, 10, - rois_num_per_level=rois_num_per_level_dy) + rois_num_per_level=rois_num_per_level_dy, + ) fpn_rois_dy = fpn_rois_dy.numpy() rois_num_dy = rois_num_dy.numpy() @@ -839,72 +902,80 @@ class TestCollectFpnPropsals(LayerTest): np.testing.assert_array_equal(rois_num_stat, rois_num_dy) def test_collect_fpn_proposals_error(self): - def generate_input(bbox_type, score_type, name): multi_bboxes = [] multi_scores = [] for i in range(4): - bboxes = fluid.data(name='rois' + name + str(i), - shape=[10, 4], - dtype=bbox_type, - lod_level=1) - scores = fluid.data(name='scores' + name + str(i), - shape=[10, 1], - dtype=score_type, - lod_level=1) + bboxes = fluid.data( + name='rois' + name + str(i), + shape=[10, 4], + dtype=bbox_type, + lod_level=1, + ) + scores = fluid.data( + name='scores' + name + str(i), + shape=[10, 1], + dtype=score_type, + lod_level=1, + ) multi_bboxes.append(bboxes) multi_scores.append(scores) return multi_bboxes, multi_scores program = Program() with program_guard(program): - bbox1 = fluid.data(name='rois', - shape=[5, 10, 4], - dtype='float32', - lod_level=1) - score1 = fluid.data(name='scores', - shape=[5, 10, 1], - dtype='float32', - lod_level=1) + bbox1 = fluid.data( + name='rois', shape=[5, 10, 4], dtype='float32', lod_level=1 + ) + score1 = fluid.data( + name='scores', shape=[5, 10, 1], dtype='float32', lod_level=1 + ) bbox2, score2 = generate_input('int32', 'float32', '2') - self.assertRaises(TypeError, - layers.collect_fpn_proposals, - multi_rois=bbox1, - multi_scores=score1, - min_level=2, - max_level=5, - post_nms_top_n=2000) - self.assertRaises(TypeError, - layers.collect_fpn_proposals, - multi_rois=bbox2, - multi_scores=score2, - min_level=2, - max_level=5, - post_nms_top_n=2000) + self.assertRaises( + TypeError, + layers.collect_fpn_proposals, + multi_rois=bbox1, + multi_scores=score1, + min_level=2, + max_level=5, + post_nms_top_n=2000, + ) + self.assertRaises( + TypeError, + layers.collect_fpn_proposals, + multi_rois=bbox2, + multi_scores=score2, + min_level=2, + max_level=5, + post_nms_top_n=2000, + ) class TestDistributeFpnProposals(LayerTest): - def test_distribute_fpn_proposals(self): rois_np = np.random.rand(10, 4).astype('float32') rois_num_np = np.array([4, 6]).astype('int32') with self.static_graph(): rois = fluid.data(name='rois', shape=[10, 4], dtype='float32') rois_num = fluid.data(name='rois_num', shape=[None], dtype='int32') - multi_rois, restore_ind, rois_num_per_level = layers.distribute_fpn_proposals( + ( + multi_rois, + restore_ind, + rois_num_per_level, + ) = layers.distribute_fpn_proposals( fpn_rois=rois, min_level=2, max_level=5, refer_level=4, refer_scale=224, - rois_num=rois_num) + rois_num=rois_num, + ) fetch_list = multi_rois + [restore_ind] + rois_num_per_level - output_stat = self.get_static_graph_result(feed={ - 'rois': rois_np, - 'rois_num': rois_num_np - }, - fetch_list=fetch_list, - with_lod=True) + output_stat = self.get_static_graph_result( + feed={'rois': rois_np, 'rois_num': rois_num_np}, + fetch_list=fetch_list, + with_lod=True, + ) output_stat_np = [] for output in output_stat: output_np = np.array(output) @@ -914,13 +985,18 @@ class TestDistributeFpnProposals(LayerTest): with self.dynamic_graph(): rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - multi_rois_dy, restore_ind_dy, rois_num_per_level_dy = layers.distribute_fpn_proposals( + ( + multi_rois_dy, + restore_ind_dy, + rois_num_per_level_dy, + ) = layers.distribute_fpn_proposals( fpn_rois=rois_dy, min_level=2, max_level=5, refer_level=4, refer_scale=224, - rois_num=rois_num_dy) + rois_num=rois_num_dy, + ) print(type(multi_rois_dy)) output_dy = multi_rois_dy + [restore_ind_dy] + rois_num_per_level_dy output_dy_np = [] @@ -935,83 +1011,93 @@ class TestDistributeFpnProposals(LayerTest): def test_distribute_fpn_proposals_error(self): program = Program() with program_guard(program): - fpn_rois = fluid.data(name='data_error', - shape=[10, 4], - dtype='int32', - lod_level=1) - self.assertRaises(TypeError, - layers.distribute_fpn_proposals, - fpn_rois=fpn_rois, - min_level=2, - max_level=5, - refer_level=4, - refer_scale=224) + fpn_rois = fluid.data( + name='data_error', shape=[10, 4], dtype='int32', lod_level=1 + ) + self.assertRaises( + TypeError, + layers.distribute_fpn_proposals, + fpn_rois=fpn_rois, + min_level=2, + max_level=5, + refer_level=4, + refer_scale=224, + ) class TestBoxDecoderAndAssign(unittest.TestCase): - def test_box_decoder_and_assign(self): program = Program() with program_guard(program): pb = fluid.data(name='prior_box', shape=[None, 4], dtype='float32') pbv = fluid.data(name='prior_box_var', shape=[4], dtype='float32') - loc = fluid.data(name='target_box', - shape=[None, 4 * 81], - dtype='float32') - scores = fluid.data(name='scores', - shape=[None, 81], - dtype='float32') - decoded_box, output_assign_box = fluid.layers.box_decoder_and_assign( - pb, pbv, loc, scores, 4.135) + loc = fluid.data( + name='target_box', shape=[None, 4 * 81], dtype='float32' + ) + scores = fluid.data( + name='scores', shape=[None, 81], dtype='float32' + ) + ( + decoded_box, + output_assign_box, + ) = fluid.layers.box_decoder_and_assign(pb, pbv, loc, scores, 4.135) self.assertIsNotNone(decoded_box) self.assertIsNotNone(output_assign_box) def test_box_decoder_and_assign_error(self): - def generate_input(pb_type, pbv_type, loc_type, score_type, name): - pb = fluid.data(name='prior_box' + name, - shape=[None, 4], - dtype=pb_type) - pbv = fluid.data(name='prior_box_var' + name, - shape=[4], - dtype=pbv_type) - loc = fluid.data(name='target_box' + name, - shape=[None, 4 * 81], - dtype=loc_type) - scores = fluid.data(name='scores' + name, - shape=[None, 81], - dtype=score_type) + pb = fluid.data( + name='prior_box' + name, shape=[None, 4], dtype=pb_type + ) + pbv = fluid.data( + name='prior_box_var' + name, shape=[4], dtype=pbv_type + ) + loc = fluid.data( + name='target_box' + name, shape=[None, 4 * 81], dtype=loc_type + ) + scores = fluid.data( + name='scores' + name, shape=[None, 81], dtype=score_type + ) return pb, pbv, loc, scores program = Program() with program_guard(program): - pb1, pbv1, loc1, scores1 = generate_input('int32', 'float32', - 'float32', 'float32', '1') - pb2, pbv2, loc2, scores2 = generate_input('float32', 'float32', - 'int32', 'float32', '2') - pb3, pbv3, loc3, scores3 = generate_input('float32', 'float32', - 'float32', 'int32', '3') - self.assertRaises(TypeError, - layers.box_decoder_and_assign, - prior_box=pb1, - prior_box_var=pbv1, - target_box=loc1, - box_score=scores1, - box_clip=4.0) - self.assertRaises(TypeError, - layers.box_decoder_and_assign, - prior_box=pb2, - prior_box_var=pbv2, - target_box=loc2, - box_score=scores2, - box_clip=4.0) - self.assertRaises(TypeError, - layers.box_decoder_and_assign, - prior_box=pb3, - prior_box_var=pbv3, - target_box=loc3, - box_score=scores3, - box_clip=4.0) + pb1, pbv1, loc1, scores1 = generate_input( + 'int32', 'float32', 'float32', 'float32', '1' + ) + pb2, pbv2, loc2, scores2 = generate_input( + 'float32', 'float32', 'int32', 'float32', '2' + ) + pb3, pbv3, loc3, scores3 = generate_input( + 'float32', 'float32', 'float32', 'int32', '3' + ) + self.assertRaises( + TypeError, + layers.box_decoder_and_assign, + prior_box=pb1, + prior_box_var=pbv1, + target_box=loc1, + box_score=scores1, + box_clip=4.0, + ) + self.assertRaises( + TypeError, + layers.box_decoder_and_assign, + prior_box=pb2, + prior_box_var=pbv2, + target_box=loc2, + box_score=scores2, + box_clip=4.0, + ) + self.assertRaises( + TypeError, + layers.box_decoder_and_assign, + prior_box=pb3, + prior_box_var=pbv3, + target_box=loc3, + box_score=scores3, + box_clip=4.0, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/test_error_clip.py b/python/paddle/fluid/tests/test_error_clip.py index eac41b818c2..f6e1122e045 100644 --- a/python/paddle/fluid/tests/test_error_clip.py +++ b/python/paddle/fluid/tests/test_error_clip.py @@ -36,12 +36,14 @@ with fluid.program_guard(main_program=prog): prog_clip = prog.clone() prog_clip.block(0).var(hidden1.name)._set_error_clip( - fluid.clip.ErrorClipByValue(max=CLIP_MAX, min=CLIP_MIN)) + fluid.clip.ErrorClipByValue(max=CLIP_MAX, min=CLIP_MIN) +) avg_cost_clip = prog_clip.block(0).var(avg_cost.name) fluid.backward.append_backward(loss=avg_cost) -fluid.backward.append_backward(loss=avg_cost_clip, - callbacks=[fluid.clip.error_clip_callback]) +fluid.backward.append_backward( + loss=avg_cost_clip, callbacks=[fluid.clip.error_clip_callback] +) hidden1_grad = prog.block(0).var(hidden1.name + "@GRAD") hidden1_grad_clip = prog_clip.block(0).var(hidden1.name + "@GRAD") @@ -49,9 +51,10 @@ hidden1_grad_clip = prog_clip.block(0).var(hidden1.name + "@GRAD") hidden2_grad = prog.block(0).var(hidden2.name + "@GRAD") hidden2_grad_clip = prog_clip.block(0).var(hidden2.name + "@GRAD") -train_reader = paddle.batch(paddle.reader.shuffle(paddle.dataset.mnist.train(), - buf_size=8192), - batch_size=BATCH_SIZE) +train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=8192), + batch_size=BATCH_SIZE, +) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -63,15 +66,18 @@ for data in train_reader(): count += 1 if count > 5: break - out1, out2 = exe.run(prog, - feed=feeder.feed(data), - fetch_list=[hidden1_grad, hidden2_grad]) + out1, out2 = exe.run( + prog, feed=feeder.feed(data), fetch_list=[hidden1_grad, hidden2_grad] + ) out1_clip, out2_clip = exe.run( prog_clip, feed=feeder.feed(data), - fetch_list=[hidden1_grad_clip, hidden2_grad_clip]) - if not ((out1.clip(min=CLIP_MIN, max=CLIP_MAX) == out1_clip).all() and - (out2 == out2_clip).all()): + fetch_list=[hidden1_grad_clip, hidden2_grad_clip], + ) + if not ( + (out1.clip(min=CLIP_MIN, max=CLIP_MAX) == out1_clip).all() + and (out2 == out2_clip).all() + ): exit(1) exit(0) diff --git a/python/paddle/fluid/tests/test_if_else_op.py b/python/paddle/fluid/tests/test_if_else_op.py index cc9f4996410..aa4aca0724c 100644 --- a/python/paddle/fluid/tests/test_if_else_op.py +++ b/python/paddle/fluid/tests/test_if_else_op.py @@ -59,19 +59,19 @@ class TestMNISTIfElseOp(unittest.TestCase): prob = layers.fc(input=hidden, size=10, act='softmax') layers.assign(input=prob, output=false_out) - prob = merge_lod_tensor(in_true=true_out, - in_false=false_out, - mask=cond, - x=image) + prob = merge_lod_tensor( + in_true=true_out, in_false=false_out, mask=cond, x=image + ) loss = layers.cross_entropy(input=prob, label=label) avg_loss = paddle.mean(loss) optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=8192), - batch_size=10) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=8192), + batch_size=10, + ) place = core.CPUPlace() exe = Executor(place) @@ -84,12 +84,9 @@ class TestMNISTIfElseOp(unittest.TestCase): y_data = np.array([x[1] for x in data]).astype("int64") y_data = np.expand_dims(y_data, axis=1) - outs = exe.run(prog, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + outs = exe.run( + prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss] + ) print(outs[0]) if outs[0] < 1.0: return @@ -126,9 +123,10 @@ class TestMNISTIfElseOp(unittest.TestCase): optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=8192), - batch_size=200) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=8192), + batch_size=200, + ) place = core.CPUPlace() exe = Executor(place) @@ -141,12 +139,9 @@ class TestMNISTIfElseOp(unittest.TestCase): y_data = np.array([x[1] for x in data]).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) - outs = exe.run(prog, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + outs = exe.run( + prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss] + ) print(outs[0]) if outs[0] < 1.0: return @@ -154,7 +149,6 @@ class TestMNISTIfElseOp(unittest.TestCase): class TestIfElse(unittest.TestCase): - def set_test_case(self): # condiction is: self.data < self.cond_value self.cond_value = 0.5 @@ -174,9 +168,9 @@ class TestIfElse(unittest.TestCase): startup_prog = Program() with program_guard(prog, startup_prog): src = layers.data(name='data', shape=[1], dtype='float32') - cond = layers.fill_constant([1], - dtype='float32', - value=self.cond_value) + cond = layers.fill_constant( + [1], dtype='float32', value=self.cond_value + ) ifcond = layers.less_than(x=src, y=cond) ie = layers.IfElse(ifcond) with ie.true_block(): @@ -194,9 +188,11 @@ class TestIfElse(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) fetch_list = [out] - o1, = exe.run(fluid.default_main_program(), - feed={'data': self.data}, - fetch_list=[out]) + (o1,) = exe.run( + fluid.default_main_program(), + feed={'data': self.data}, + fetch_list=[out], + ) o2 = self.numpy_cal() np.testing.assert_allclose( @@ -216,31 +212,28 @@ class TestIfElse(unittest.TestCase): class TestIfElseTrueBranch(TestIfElse): - def set_test_case(self): # condiction is: self.data < self.cond_value - self.cond_value = 10. + self.cond_value = 10.0 self.data = np.random.rand(25, 1).astype(np.float32) class TestIfElseFalseBranch(TestIfElse): - def set_test_case(self): # condiction is: self.data < self.cond_value - self.cond_value = -10. + self.cond_value = -10.0 self.data = np.random.rand(25, 1).astype(np.float32) class TestIfElseError(unittest.TestCase): - def test_input_type_error(self): main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): src = layers.data(name='data', shape=[1], dtype='float32') - const_value = layers.fill_constant([1], - dtype='float32', - value=123.0) + const_value = layers.fill_constant( + [1], dtype='float32', value=123.0 + ) ifcond = layers.less_than(x=src, y=const_value) with self.assertRaises(TypeError): ie = layers.IfElse(set()) diff --git a/python/paddle/fluid/tests/test_lod_tensor.py b/python/paddle/fluid/tests/test_lod_tensor.py index 4650c3ec2df..b84a11dbdc6 100644 --- a/python/paddle/fluid/tests/test_lod_tensor.py +++ b/python/paddle/fluid/tests/test_lod_tensor.py @@ -14,28 +14,33 @@ import paddle.fluid as fluid import paddle.fluid.core as core -from paddle.fluid.lod_tensor import create_lod_tensor, create_random_int_lodtensor +from paddle.fluid.lod_tensor import ( + create_lod_tensor, + create_random_int_lodtensor, +) import numpy as np import unittest class TestLoDTensor(unittest.TestCase): - def test_pybind_recursive_seq_lens(self): tensor = fluid.LoDTensor() recursive_seq_lens = [] tensor.set_recursive_sequence_lengths(recursive_seq_lens) recursive_seq_lens = [[], [1], [3]] - self.assertRaises(Exception, tensor.set_recursive_sequence_lengths, - recursive_seq_lens) + self.assertRaises( + Exception, tensor.set_recursive_sequence_lengths, recursive_seq_lens + ) recursive_seq_lens = [[0], [2], [3]] - self.assertRaises(Exception, tensor.set_recursive_sequence_lengths, - recursive_seq_lens) + self.assertRaises( + Exception, tensor.set_recursive_sequence_lengths, recursive_seq_lens + ) recursive_seq_lens = [[1, 2, 3]] tensor.set_recursive_sequence_lengths(recursive_seq_lens) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) tensor.set(np.random.random([6, 1]), fluid.CPUPlace()) self.assertTrue(tensor.has_valid_recursive_sequence_lengths()) tensor.set(np.random.random([9, 1]), fluid.CPUPlace()) @@ -45,8 +50,9 @@ class TestLoDTensor(unittest.TestCase): # Moreover, last level's sum should be equal to the tensor height recursive_seq_lens = [[2, 3], [1, 3, 1, 2, 2]] tensor.set_recursive_sequence_lengths(recursive_seq_lens) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) tensor.set(np.random.random([8, 1]), fluid.CPUPlace()) self.assertFalse(tensor.has_valid_recursive_sequence_lengths()) recursive_seq_lens = [[2, 3], [1, 3, 1, 2, 1]] @@ -57,40 +63,54 @@ class TestLoDTensor(unittest.TestCase): def test_create_lod_tensor(self): # Create LoDTensor from a list - data = [[np.int64(1), np.int64(2), - np.int64(3)], [np.int64(3), np.int64(4)]] + data = [ + [np.int64(1), np.int64(2), np.int64(3)], + [np.int64(3), np.int64(4)], + ] wrong_recursive_seq_lens = [[2, 2]] correct_recursive_seq_lens = [[3, 2]] - self.assertRaises(AssertionError, create_lod_tensor, data, - wrong_recursive_seq_lens, fluid.CPUPlace()) - tensor = create_lod_tensor(data, correct_recursive_seq_lens, - fluid.CPUPlace()) - self.assertEqual(tensor.recursive_sequence_lengths(), - correct_recursive_seq_lens) + self.assertRaises( + AssertionError, + create_lod_tensor, + data, + wrong_recursive_seq_lens, + fluid.CPUPlace(), + ) + tensor = create_lod_tensor( + data, correct_recursive_seq_lens, fluid.CPUPlace() + ) + self.assertEqual( + tensor.recursive_sequence_lengths(), correct_recursive_seq_lens + ) self.assertEqual(tensor._dtype(), core.VarDesc.VarType.INT64) self.assertEqual(tensor.shape(), [5, 1]) np.testing.assert_array_equal( np.array(tensor), - np.array([1, 2, 3, 3, 4]).reshape(tensor.shape()).astype('int64')) + np.array([1, 2, 3, 3, 4]).reshape(tensor.shape()).astype('int64'), + ) # Create LoDTensor from numpy array data = np.random.random([10, 1]).astype('float64') recursive_seq_lens = [[2, 1], [3, 3, 4]] tensor = create_lod_tensor(data, recursive_seq_lens, fluid.CPUPlace()) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) self.assertEqual(tensor._dtype(), core.VarDesc.VarType.FP64) self.assertEqual(tensor.shape(), [10, 1]) np.testing.assert_array_equal(np.array(tensor), data) # Create LoDTensor from another LoDTensor, they are differnt instances new_recursive_seq_lens = [[2, 2, 1], [1, 2, 2, 3, 2]] - new_tensor = create_lod_tensor(tensor, new_recursive_seq_lens, - fluid.CPUPlace()) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) - self.assertEqual(new_tensor.recursive_sequence_lengths(), - new_recursive_seq_lens) + new_tensor = create_lod_tensor( + tensor, new_recursive_seq_lens, fluid.CPUPlace() + ) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) + self.assertEqual( + new_tensor.recursive_sequence_lengths(), new_recursive_seq_lens + ) def test_create_random_int_lodtensor(self): # The shape of a word, commonly used in speech and NLP problem, is [1] @@ -99,10 +119,12 @@ class TestLoDTensor(unittest.TestCase): dict_size = 10000 low = 0 high = dict_size - 1 - tensor = create_random_int_lodtensor(recursive_seq_lens, shape, - fluid.CPUPlace(), low, high) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) + tensor = create_random_int_lodtensor( + recursive_seq_lens, shape, fluid.CPUPlace(), low, high + ) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) self.assertEqual(tensor.shape(), [10, 1]) def test_print_lodtensor(self): @@ -111,43 +133,53 @@ class TestLoDTensor(unittest.TestCase): dict_size = 100 low = 0 high = dict_size - 1 - tensor = create_random_int_lodtensor(recursive_seq_lens, shape, - fluid.CPUPlace(), low, high) + tensor = create_random_int_lodtensor( + recursive_seq_lens, shape, fluid.CPUPlace(), low, high + ) print(tensor) self.assertTrue(isinstance(str(tensor), str)) if core.is_compiled_with_cuda(): - gtensor = create_random_int_lodtensor(recursive_seq_lens, shape, - fluid.CUDAPlace(0), low, high) + gtensor = create_random_int_lodtensor( + recursive_seq_lens, shape, fluid.CUDAPlace(0), low, high + ) print(gtensor) self.assertTrue(isinstance(str(gtensor), str)) def test_dlpack_support(self): tensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CPUPlace()) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CPUPlace(), + ) dltensor = tensor._to_dlpack() tensor_from_dlpack = fluid.core.from_dlpack(dltensor) self.assertTrue(isinstance(tensor_from_dlpack, fluid.core.Tensor)) np.testing.assert_array_equal( np.array(tensor_from_dlpack), - np.array([[1], [2], [3], [4]]).astype('int')) + np.array([[1], [2], [3], [4]]).astype('int'), + ) # when build with cuda if core.is_compiled_with_cuda(): gtensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CUDAPlace(0)) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CUDAPlace(0), + ) gdltensor = gtensor._to_dlpack() gtensor_from_dlpack = fluid.core.from_dlpack(gdltensor) self.assertTrue(isinstance(gtensor_from_dlpack, fluid.core.Tensor)) np.testing.assert_array_equal( np.array(gtensor_from_dlpack), - np.array([[1], [2], [3], [4]]).astype('int')) + np.array([[1], [2], [3], [4]]).astype('int'), + ) def test_as_type(self): tensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CPUPlace()) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CPUPlace(), + ) fp32_tensor = tensor._as_type(core.VarDesc.VarType.FP32) print(fp32_tensor) diff --git a/python/paddle/fluid/tests/test_python_operator_overriding.py b/python/paddle/fluid/tests/test_python_operator_overriding.py index 7fded67c9ba..562b8cef5ff 100644 --- a/python/paddle/fluid/tests/test_python_operator_overriding.py +++ b/python/paddle/fluid/tests/test_python_operator_overriding.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestPythonOperatorOverride(unittest.TestCase): - def check_result(self, fn, place, dtype): shape = [9, 10] @@ -33,27 +32,22 @@ class TestPythonOperatorOverride(unittest.TestCase): y_data = np.random.random(size=shape).astype(dtype) python_out = fn(x_data, y_data) - x_var = layers.create_global_var(name='x', - shape=shape, - value=0.0, - dtype=dtype, - persistable=True) - y_var = layers.create_global_var(name='y', - shape=shape, - value=0.0, - dtype=dtype, - persistable=True) + x_var = layers.create_global_var( + name='x', shape=shape, value=0.0, dtype=dtype, persistable=True + ) + y_var = layers.create_global_var( + name='y', shape=shape, value=0.0, dtype=dtype, persistable=True + ) out = fn(x_var, y_var) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fluid_out = exe.run(fluid.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[out]) + fluid_out = exe.run( + fluid.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[out], + ) np.testing.assert_array_equal(python_out, fluid_out[0]) @@ -79,8 +73,9 @@ class TestPythonOperatorOverride(unittest.TestCase): for place in places: for dtype in dtypes: for compare_fn in compare_fns: - with framework.program_guard(framework.Program(), - framework.Program()): + with framework.program_guard( + framework.Program(), framework.Program() + ): self.check_result(compare_fn, place, dtype) diff --git a/python/paddle/fluid/tests/test_sequential.py b/python/paddle/fluid/tests/test_sequential.py index 09cfbcdd7e3..7446bb83841 100644 --- a/python/paddle/fluid/tests/test_sequential.py +++ b/python/paddle/fluid/tests/test_sequential.py @@ -17,7 +17,6 @@ import paddle class TestDataFeeder(unittest.TestCase): - def test_lod_level_1_converter(self): sequential = paddle.nn.Sequential() diff --git a/python/paddle/fluid/tests/unittests/__init__.py b/python/paddle/fluid/tests/unittests/__init__.py index e427eb51247..126cbeb4938 100644 --- a/python/paddle/fluid/tests/unittests/__init__.py +++ b/python/paddle/fluid/tests/unittests/__init__.py @@ -17,8 +17,10 @@ # please refer to https://stackoverflow.com/questions/8953844/import-module-from-subfolder import os + if os.name == 'nt': import sys + dirname, filename = os.path.split(os.path.abspath(__file__)) sys.path.insert(0, dirname) print(sys.path) diff --git a/python/paddle/fluid/tests/unittests/ascend_group.py b/python/paddle/fluid/tests/unittests/ascend_group.py index 32b42a00fc6..b0928f40931 100644 --- a/python/paddle/fluid/tests/unittests/ascend_group.py +++ b/python/paddle/fluid/tests/unittests/ascend_group.py @@ -35,8 +35,9 @@ role = fleet.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) -def init_communicator(startup_program, main_program, current_endpoint, - endpoints, ring_id): +def init_communicator( + startup_program, main_program, current_endpoint, endpoints, ring_id +): nranks = len(endpoints) other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) @@ -44,55 +45,64 @@ def init_communicator(startup_program, main_program, current_endpoint, assert group_rank >= 0 block = startup_program.global_block() - nccl_id_var = block.create_var(name=unique_name.generate('nccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': group_rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - OP_ROLE_KEY: OpRole.Forward, - }) - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': group_rank, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward, - }) + nccl_id_var = block.create_var( + name=unique_name.generate('nccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': group_rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': group_rank, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) # add input op for test fill_var_name = "tensor@Filled" - fill_var = block.create_var(name=fill_var_name, - shape=[10, 10], - dtype='float32', - persistable=False, - stop_gradient=True) - block.append_op(type="fill_constant", - outputs={"Out": fill_var_name}, - attrs={ - "shape": [10, 10], - "dtype": fill_var.dtype, - "value": 1.0, - "place_type": 1 - }) + fill_var = block.create_var( + name=fill_var_name, + shape=[10, 10], + dtype='float32', + persistable=False, + stop_gradient=True, + ) + block.append_op( + type="fill_constant", + outputs={"Out": fill_var_name}, + attrs={ + "shape": [10, 10], + "dtype": fill_var.dtype, + "value": 1.0, + "place_type": 1, + }, + ) with fluid.program_guard(main_program): op_type = "c_allreduce_sum" data = fluid.layers.fill_constant(shape=[1], dtype='float32', value=2.5) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [data]}, - outputs={'Out': [data]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True - }) + helper.append_op( + type=op_type, + inputs={'X': [data]}, + outputs={'Out': [data]}, + attrs={'ring_id': ring_id, 'use_calc_stream': True}, + ) print("startup program:", startup_program) print("main program:", main_program) @@ -102,7 +112,7 @@ def train(world_endpoints, world_device_ids, local_device_ids, local_rank): startup_programs = [] main_programs = [] - #trainer_endpoints=["127.0.0.1:6071","127.0.0.1:6072","127.0.0.1:6073","127.0.0.1:6074"] + # trainer_endpoints=["127.0.0.1:6071","127.0.0.1:6072","127.0.0.1:6073","127.0.0.1:6074"] trainer_endpoints = world_endpoints groups = [[], [], []] groups[0] = [trainer_endpoints[0], trainer_endpoints[1]] @@ -132,10 +142,12 @@ def train(world_endpoints, world_device_ids, local_device_ids, local_rank): main_program = main_programs[local_rank] loss = Loss(Block(main_program)) optimizer = ascend_optimizer.AscendOptimizer(None, fetch_list=[]) - optimizer.minimize(loss, - startup_program, - auto_dp=True, - rank_table_file=os.getenv("RANK_TABLE_FILE", None)) + optimizer.minimize( + loss, + startup_program, + auto_dp=True, + rank_table_file=os.getenv("RANK_TABLE_FILE", None), + ) exe = paddle.static.Executor(paddle.CPUPlace()) exe.run(startup_program) diff --git a/python/paddle/fluid/tests/unittests/ascend_multi_process_collective.py b/python/paddle/fluid/tests/unittests/ascend_multi_process_collective.py index 1853e529d51..113c4286f35 100644 --- a/python/paddle/fluid/tests/unittests/ascend_multi_process_collective.py +++ b/python/paddle/fluid/tests/unittests/ascend_multi_process_collective.py @@ -27,12 +27,21 @@ def train(prefix): device_ids = os.getenv("PADDLE_WORLD_DEVICE_IDS") current_device_id = os.getenv("PADDLE_LOCAL_DEVICE_IDS") - details = "selected_accelerators:{} selected_npus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{} device_ids:{} device_id:{}"\ - .format(selected_accelerators, selected_npus, worker_endpoints, trainers_num, current_endpoint,trainer_id,device_ids, current_device_id) + details = "selected_accelerators:{} selected_npus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{} device_ids:{} device_id:{}".format( + selected_accelerators, + selected_npus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + device_ids, + current_device_id, + ) print(details) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(details) diff --git a/python/paddle/fluid/tests/unittests/asp/asp_pruning_base.py b/python/paddle/fluid/tests/unittests/asp/asp_pruning_base.py index 6e9603e6761..e21d5f598eb 100644 --- a/python/paddle/fluid/tests/unittests/asp/asp_pruning_base.py +++ b/python/paddle/fluid/tests/unittests/asp/asp_pruning_base.py @@ -24,21 +24,18 @@ paddle.enable_static() class TestASPHelperPruningBase(unittest.TestCase): - def setUp(self): self.main_program = fluid.Program() self.startup_program = fluid.Program() def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 32, 32], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 32, 32], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=4, - filter_size=3, - padding=2, - act="relu") + hidden = fluid.layers.conv2d( + input=img, num_filters=4, filter_size=3, padding=2, act="relu" + ) hidden = fluid.layers.fc(input=hidden, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') return img, label, prediction @@ -46,23 +43,26 @@ class TestASPHelperPruningBase(unittest.TestCase): with fluid.program_guard(self.main_program, self.startup_program): self.img, self.label, self.predict = build_model() - def run_inference_pruning_test(self, get_mask_gen_func, - get_mask_check_func): + def run_inference_pruning_test( + self, get_mask_gen_func, get_mask_check_func + ): place = paddle.CPUPlace() if core.is_compiled_with_cuda(): place = paddle.CUDAPlace(0) exe = fluid.Executor(place) - self.__pruning_and_checking(exe, place, get_mask_gen_func, - get_mask_check_func, False) + self.__pruning_and_checking( + exe, place, get_mask_gen_func, get_mask_check_func, False + ) def run_training_pruning_test(self, get_mask_gen_func, get_mask_check_func): with fluid.program_guard(self.main_program, self.startup_program): loss = paddle.mean( - fluid.layers.cross_entropy(input=self.predict, - label=self.label)) + fluid.layers.cross_entropy(input=self.predict, label=self.label) + ) optimizer = paddle.incubate.asp.decorate( - fluid.optimizer.SGD(learning_rate=0.01)) + fluid.optimizer.SGD(learning_rate=0.01) + ) optimizer.minimize(loss, self.startup_program) place = paddle.CPUPlace() @@ -70,19 +70,24 @@ class TestASPHelperPruningBase(unittest.TestCase): place = paddle.CUDAPlace(0) exe = fluid.Executor(place) - self.__pruning_and_checking(exe, place, get_mask_gen_func, - get_mask_check_func, True) + self.__pruning_and_checking( + exe, place, get_mask_gen_func, get_mask_check_func, True + ) - def __pruning_and_checking(self, exe, place, mask_func_name, - check_func_name, with_mask): + def __pruning_and_checking( + self, exe, place, mask_func_name, check_func_name, with_mask + ): exe.run(self.startup_program) - paddle.incubate.asp.prune_model(self.main_program, - mask_algo=mask_func_name, - with_mask=with_mask) + paddle.incubate.asp.prune_model( + self.main_program, mask_algo=mask_func_name, with_mask=with_mask + ) for param in self.main_program.global_block().all_parameters(): if ASPHelper._is_supported_layer(self.main_program, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( - mat.T, func_name=check_func_name, n=2, m=4)) + mat.T, func_name=check_func_name, n=2, m=4 + ) + ) diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py b/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py index fde68b39e7b..cdccd107142 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py @@ -19,12 +19,13 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.contrib import sparsity -from paddle.fluid.contrib.sparsity.supported_layer_list import supported_layers_and_prune_func_map +from paddle.fluid.contrib.sparsity.supported_layer_list import ( + supported_layers_and_prune_func_map, +) from paddle.fluid.dygraph.layers import Layer, _convert_camel_to_snake class MyOwnLayer(Layer): - def __init__(self): super(MyOwnLayer, self).__init__() @@ -47,7 +48,6 @@ def my_own_pruning(tensor, m, n, mask_algo, param_name): class TestASPAddSupportedLayer(unittest.TestCase): - def test_add_supported_layer_via_name(self): sparsity.add_supported_layer("test_supported_1") sparsity.add_supported_layer("test_supported_2", my_own_pruning) @@ -55,37 +55,41 @@ class TestASPAddSupportedLayer(unittest.TestCase): my_own_layer_name = _convert_camel_to_snake(MyOwnLayer.__name__) self.assertTrue( - "test_supported_1" in supported_layers_and_prune_func_map) + "test_supported_1" in supported_layers_and_prune_func_map + ) + self.assertTrue( + "test_supported_2" in supported_layers_and_prune_func_map + ) self.assertTrue( - "test_supported_2" in supported_layers_and_prune_func_map) + "test_supported_2" in supported_layers_and_prune_func_map + ) self.assertTrue( - "test_supported_2" in supported_layers_and_prune_func_map) - self.assertTrue(supported_layers_and_prune_func_map["test_supported_2"] - == my_own_pruning) + supported_layers_and_prune_func_map["test_supported_2"] + == my_own_pruning + ) self.assertTrue( - my_own_layer_name in supported_layers_and_prune_func_map) + my_own_layer_name in supported_layers_and_prune_func_map + ) class TestASPDynamicCustomerizedPruneFunc(unittest.TestCase): - def setUp(self): paddle.disable_static() class CustomerLayer(paddle.nn.Layer): - def __init__(self): super(CustomerLayer, self).__init__() - self.weight = self.create_parameter(shape=[32, 32], - attr=None, - dtype='float32', - is_bias=False) + self.weight = self.create_parameter( + shape=[32, 32], attr=None, dtype='float32', is_bias=False + ) self.linear1 = paddle.nn.Linear(32, 32) self.linear2 = paddle.nn.Linear(32, 10) def forward(self, input_): - hidden = paddle.nn.functional.linear(x=input_, - weight=self.weight) + hidden = paddle.nn.functional.linear( + x=input_, weight=self.weight + ) hidden = self.linear1(hidden) out = self.linear2(hidden) return out @@ -93,8 +97,11 @@ class TestASPDynamicCustomerizedPruneFunc(unittest.TestCase): sparsity.add_supported_layer(CustomerLayer, my_own_pruning) self.layer = CustomerLayer() - self.customer_prefix = paddle.fluid.dygraph.layers._convert_camel_to_snake( - CustomerLayer.__name__) + self.customer_prefix = ( + paddle.fluid.dygraph.layers._convert_camel_to_snake( + CustomerLayer.__name__ + ) + ) self.supported_layer_count_ref = 3 def test_inference_pruning(self): @@ -106,23 +113,28 @@ class TestASPDynamicCustomerizedPruneFunc(unittest.TestCase): mat = param.numpy() if sparsity.asp.ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): supported_layer_count += 1 - if (self.customer_prefix in param.name): + if self.customer_prefix in param.name: self.assertLessEqual( - np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) + np.sum(mat.flatten() - static_tensor.flatten()), 1e-4 + ) else: self.assertTrue( sparsity.check_sparsity( mat.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) def test_training_pruning(self): - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=self.layer.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=self.layer.parameters() + ) optimizer = sparsity.decorate(optimizer) sparsity.prune_model(self.layer, mask_algo="mask_1d", with_mask=True) @@ -132,37 +144,49 @@ class TestASPDynamicCustomerizedPruneFunc(unittest.TestCase): mat = param.numpy() if sparsity.asp.ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): - mat_mask = sparsity.asp.ASPHelper._get_program_asp_info( - paddle.static.default_main_program()).mask_vars[ - param.name].numpy() + mat_mask = ( + sparsity.asp.ASPHelper._get_program_asp_info( + paddle.static.default_main_program() + ) + .mask_vars[param.name] + .numpy() + ) supported_layer_count += 1 - if (self.customer_prefix in param.name): + if self.customer_prefix in param.name: self.assertLessEqual( - np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) + np.sum(mat.flatten() - static_tensor.flatten()), 1e-4 + ) self.assertLessEqual( - np.sum(mat_mask.flatten() - - static_tensor_mask.flatten()), 1e-4) + np.sum( + mat_mask.flatten() - static_tensor_mask.flatten() + ), + 1e-4, + ) else: self.assertTrue( sparsity.check_sparsity( mat.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertTrue( sparsity.check_sparsity( mat_mask.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) class TestASPStaticCustomerizedPruneFunc(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -172,23 +196,19 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase): self.customer_prefix = "customer_layer" def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 32, 32], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 32, 32], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=4, - filter_size=3, - padding=2, - act="relu") - hidden = fluid.layers.fc(input=hidden, - size=32, - act='relu', - name=self.customer_prefix) - hidden = fluid.layers.fc(input=hidden, - size=32, - act='relu', - name=self.customer_prefix) + hidden = fluid.layers.conv2d( + input=img, num_filters=4, filter_size=3, padding=2, act="relu" + ) + hidden = fluid.layers.fc( + input=hidden, size=32, act='relu', name=self.customer_prefix + ) + hidden = fluid.layers.fc( + input=hidden, size=32, act='relu', name=self.customer_prefix + ) hidden = fluid.layers.fc(input=hidden, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') return img, label, prediction @@ -207,89 +227,110 @@ class TestASPStaticCustomerizedPruneFunc(unittest.TestCase): def test_inference_pruning(self): self.exe.run(self.startup_program) - sparsity.prune_model(self.main_program, - mask_algo="mask_1d", - with_mask=False) + sparsity.prune_model( + self.main_program, mask_algo="mask_1d", with_mask=False + ) supported_layer_count = 0 for param in self.main_program.global_block().all_parameters(): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) if sparsity.asp.ASPHelper._is_supported_layer( - self.main_program, param.name): + self.main_program, param.name + ): supported_layer_count += 1 - if (self.customer_prefix in param.name): + if self.customer_prefix in param.name: self.assertLessEqual( - np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) + np.sum(mat.flatten() - static_tensor.flatten()), 1e-4 + ) else: - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( sparsity.check_sparsity( mat.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) def test_training_pruning(self): with fluid.program_guard(self.main_program, self.startup_program): loss = paddle.mean( - fluid.layers.cross_entropy(input=self.predict, - label=self.label)) + fluid.layers.cross_entropy(input=self.predict, label=self.label) + ) optimizer = sparsity.decorate( - fluid.optimizer.SGD(learning_rate=0.01)) + fluid.optimizer.SGD(learning_rate=0.01) + ) optimizer.minimize(loss, self.startup_program) self.exe.run(self.startup_program) - sparsity.prune_model(self.main_program, - mask_algo="mask_1d", - with_mask=True) + sparsity.prune_model( + self.main_program, mask_algo="mask_1d", with_mask=True + ) supported_layer_count = 0 for param in self.main_program.global_block().all_parameters(): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) if sparsity.asp.ASPHelper._is_supported_layer( - self.main_program, param.name): - mat_mask = np.array(fluid.global_scope().find_var( - sparsity.asp.ASPHelper._get_mask_name( - param.name)).get_tensor()) + self.main_program, param.name + ): + mat_mask = np.array( + fluid.global_scope() + .find_var(sparsity.asp.ASPHelper._get_mask_name(param.name)) + .get_tensor() + ) supported_layer_count += 1 - if (self.customer_prefix in param.name): + if self.customer_prefix in param.name: self.assertLessEqual( - np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) + np.sum(mat.flatten() - static_tensor.flatten()), 1e-4 + ) self.assertLessEqual( - np.sum(mat_mask.flatten() - - static_tensor_mask.flatten()), 1e-4) + np.sum( + mat_mask.flatten() - static_tensor_mask.flatten() + ), + 1e-4, + ) else: - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - sparsity.check_sparsity(mat.T, n=2, m=4)) + sparsity.check_sparsity(mat.T, n=2, m=4) + ) self.assertFalse( - sparsity.check_sparsity(mat_mask.T, n=2, m=4)) + sparsity.check_sparsity(mat_mask.T, n=2, m=4) + ) else: self.assertTrue( sparsity.check_sparsity( mat.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertTrue( sparsity.check_sparsity( mat_mask.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py index a501ed42794..389645139c5 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py @@ -21,13 +21,11 @@ import numpy as np class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() - self.conv1 = paddle.nn.Conv2D(in_channels=3, - out_channels=2, - kernel_size=3, - padding=2) + self.conv1 = paddle.nn.Conv2D( + in_channels=3, out_channels=2, kernel_size=3, padding=2 + ) self.linear1 = paddle.nn.Linear(1352, 32) self.linear2 = paddle.nn.Linear(32, 32) self.linear3 = paddle.nn.Linear(32, 10) @@ -42,7 +40,6 @@ class MyLayer(paddle.nn.Layer): class TestASPDynamicOptimize(unittest.TestCase): - def setUp(self): self.layer = MyLayer() @@ -52,41 +49,84 @@ class TestASPDynamicOptimize(unittest.TestCase): self.place = paddle.CUDAPlace(0) self.optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=self.layer.parameters()) + learning_rate=0.01, parameters=self.layer.parameters() + ) def test_is_supported_layers(self): program = paddle.static.default_main_program() names = [ - 'embedding_0.w_0', 'fack_layer_0.w_0', 'conv2d_0.w_0', - 'conv2d_0.b_0', 'conv2d_1.w_0', 'conv2d_1.b_0', 'fc_0.w_0', - 'fc_0.b_0', 'fc_1.w_0', 'fc_1.b_0', 'linear_2.w_0', 'linear_2.b_0' + 'embedding_0.w_0', + 'fack_layer_0.w_0', + 'conv2d_0.w_0', + 'conv2d_0.b_0', + 'conv2d_1.w_0', + 'conv2d_1.b_0', + 'fc_0.w_0', + 'fc_0.b_0', + 'fc_1.w_0', + 'fc_1.b_0', + 'linear_2.w_0', + 'linear_2.b_0', ] ref = [ - False, False, True, False, True, False, True, False, True, False, - True, False + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) paddle.incubate.asp.set_excluded_layers(['fc_1', 'conv2d_0']) ref = [ - False, False, False, False, True, False, True, False, False, False, - True, False + False, + False, + False, + False, + True, + False, + True, + False, + False, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) paddle.incubate.asp.reset_excluded_layers() ref = [ - False, False, True, False, True, False, True, False, True, False, - True, False + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) def test_decorate(self): param_names = [param.name for param in self.layer.parameters()] @@ -96,7 +136,8 @@ class TestASPDynamicOptimize(unittest.TestCase): for name in param_names: mask_var = ASPHelper._get_program_asp_info(program).mask_vars.get( - name, None) + name, None + ) if ASPHelper._is_supported_layer(program, name): self.assertTrue(mask_var is not None) else: @@ -107,14 +148,18 @@ class TestASPDynamicOptimize(unittest.TestCase): paddle.incubate.asp.prune_model(self.layer) - imgs = paddle.to_tensor(np.random.randn(32, 3, 24, 24), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(32, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(32, 3, 24, 24), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(32, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') @@ -126,34 +171,41 @@ class TestASPDynamicOptimize(unittest.TestCase): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) def test_asp_training_with_amp(self): self.optimizer = paddle.incubate.asp.decorate(self.optimizer) paddle.incubate.asp.prune_model(self.layer) - imgs = paddle.to_tensor(np.random.randn(32, 3, 24, 24), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(32, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(32, 3, 24, 24), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(32, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -168,21 +220,24 @@ class TestASPDynamicOptimize(unittest.TestCase): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py index b689a40f88f..b64763ebe7c 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py @@ -24,21 +24,18 @@ paddle.enable_static() class TestASPStaticOptimize(unittest.TestCase): - def setUp(self): self.main_program = fluid.Program() self.startup_program = fluid.Program() def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 24, 24], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 24, 24], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=4, - filter_size=3, - padding=2, - act="relu") + hidden = fluid.layers.conv2d( + input=img, num_filters=4, filter_size=3, padding=2, act="relu" + ) hidden = fluid.layers.fc(input=hidden, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') return img, label, prediction @@ -46,11 +43,11 @@ class TestASPStaticOptimize(unittest.TestCase): with fluid.program_guard(self.main_program, self.startup_program): self.img, self.label, predict = build_model() self.loss = paddle.mean( - fluid.layers.cross_entropy(input=predict, label=self.label)) + fluid.layers.cross_entropy(input=predict, label=self.label) + ) self.optimizer = fluid.optimizer.SGD(learning_rate=0.01) def test_get_not_ASP_relevant_vars(self): - def check_params(params, params_from_asp): if len(params_from_asp) != len(params): return False @@ -62,61 +59,112 @@ class TestASPStaticOptimize(unittest.TestCase): params = self.main_program.global_block().all_parameters() params_from_asp = ASPHelper._get_not_ASP_relevant_vars( - self.main_program) + self.main_program + ) self.assertTrue(check_params(params, params_from_asp)) with fluid.program_guard(self.main_program, self.startup_program): - ASPHelper._minimize(self.optimizer, self.loss, self.main_program, - self.startup_program) + ASPHelper._minimize( + self.optimizer, + self.loss, + self.main_program, + self.startup_program, + ) params_from_asp_after_opt = ASPHelper._get_not_ASP_relevant_vars( - self.main_program) + self.main_program + ) self.assertTrue(check_params(params, params_from_asp_after_opt)) def test_is_supported_layers(self): program = paddle.static.default_main_program() names = [ - 'embedding_0.w_0', 'fack_layer_0.w_0', 'conv2d_0.w_0', - 'conv2d_0.b_0', 'conv2d_1.w_0', 'conv2d_1.b_0', 'fc_0.w_0', - 'fc_0.b_0', 'fc_1.w_0', 'fc_1.b_0', 'linear_2.w_0', 'linear_2.b_0' + 'embedding_0.w_0', + 'fack_layer_0.w_0', + 'conv2d_0.w_0', + 'conv2d_0.b_0', + 'conv2d_1.w_0', + 'conv2d_1.b_0', + 'fc_0.w_0', + 'fc_0.b_0', + 'fc_1.w_0', + 'fc_1.b_0', + 'linear_2.w_0', + 'linear_2.b_0', ] ref = [ - False, False, True, False, True, False, True, False, True, False, - True, False + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) paddle.incubate.asp.set_excluded_layers(['fc_1', 'conv2d_0'], program) ref = [ - False, False, False, False, True, False, True, False, False, False, - True, False + False, + False, + False, + False, + True, + False, + True, + False, + False, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) paddle.incubate.asp.reset_excluded_layers(program) ref = [ - False, False, True, False, True, False, True, False, True, False, - True, False + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) def test_decorate(self): param_names = self.__get_param_names( - self.main_program.global_block().all_parameters()) + self.main_program.global_block().all_parameters() + ) with fluid.program_guard(self.main_program, self.startup_program): self.optimizer = paddle.incubate.asp.decorate(self.optimizer) self.optimizer.minimize(self.loss, self.startup_program) param_names_after_minimize = self.__get_param_names( - self.main_program.global_block().all_parameters()) + self.main_program.global_block().all_parameters() + ) - self.__check_mask_variables_and_ops(param_names, - param_names_after_minimize) + self.__check_mask_variables_and_ops( + param_names, param_names_after_minimize + ) def test_asp_training(self): with fluid.program_guard(self.main_program, self.startup_program): @@ -132,63 +180,77 @@ class TestASPStaticOptimize(unittest.TestCase): exe.run(self.startup_program) paddle.incubate.asp.prune_model(self.main_program) - data = (np.random.randn(32, 3, 24, - 24), np.random.randint(10, size=(32, 1))) + data = ( + np.random.randn(32, 3, 24, 24), + np.random.randint(10, size=(32, 1)), + ) exe.run(self.main_program, feed=feeder.feed([data])) for param in self.main_program.global_block().all_parameters(): if ASPHelper._is_supported_layer(self.main_program, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) def test_asp_training_with_amp(self): if core.is_compiled_with_cuda(): place = paddle.CUDAPlace(0) with fluid.program_guard(self.main_program, self.startup_program): - self.optimizer = fluid.contrib.mixed_precision.decorator.decorate( - self.optimizer) + self.optimizer = ( + fluid.contrib.mixed_precision.decorator.decorate( + self.optimizer + ) + ) self.optimizer = paddle.incubate.asp.decorate(self.optimizer) self.optimizer.minimize(self.loss, self.startup_program) exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=[self.img, self.label], - place=place) + feeder = fluid.DataFeeder( + feed_list=[self.img, self.label], place=place + ) exe.run(self.startup_program) paddle.incubate.asp.prune_model(self.main_program) - data = (np.random.randn(32, 3, 24, - 24), np.random.randint(10, size=(32, 1))) + data = ( + np.random.randn(32, 3, 24, 24), + np.random.randint(10, size=(32, 1)), + ) exe.run(self.main_program, feed=feeder.feed([data])) for param in self.main_program.global_block().all_parameters(): if ASPHelper._is_supported_layer(self.main_program, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) def __get_param_names(self, params): param_names = [] @@ -196,11 +258,15 @@ class TestASPStaticOptimize(unittest.TestCase): param_names.append(p.name) return param_names - def __check_mask_variables_and_ops(self, param_names, - param_names_after_minimize): + def __check_mask_variables_and_ops( + self, param_names, param_names_after_minimize + ): for n in param_names: - self.assertFalse(ASPHelper._is_supported_layer(self.main_program, n) and \ - ASPHelper._get_mask_name(n) not in param_names_after_minimize) + self.assertFalse( + ASPHelper._is_supported_layer(self.main_program, n) + and ASPHelper._get_mask_name(n) + not in param_names_after_minimize + ) mask_names = [] for n in param_names: @@ -209,8 +275,7 @@ class TestASPStaticOptimize(unittest.TestCase): masking_ops = [] for op in self.main_program.global_block().ops: - if op.type == 'elementwise_mul' and \ - op.input('Y')[0] in mask_names: + if op.type == 'elementwise_mul' and op.input('Y')[0] in mask_names: masking_ops.append(op.input('Y')[0]) self.assertTrue(len(masking_ops) == len(mask_names)) diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py index 74f27d74b21..ff86ba9e782 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py @@ -22,13 +22,11 @@ from paddle.fluid.contrib.sparsity.asp import ASPHelper class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() - self.conv1 = paddle.nn.Conv2D(in_channels=3, - out_channels=2, - kernel_size=3, - padding=2) + self.conv1 = paddle.nn.Conv2D( + in_channels=3, out_channels=2, kernel_size=3, padding=2 + ) self.linear1 = paddle.nn.Linear(1352, 32) self.linear2 = paddle.nn.Linear(32, 10) @@ -41,7 +39,6 @@ class MyLayer(paddle.nn.Layer): class TestASPDynamicPruningBase(unittest.TestCase): - def setUp(self): self.layer = MyLayer() @@ -49,72 +46,82 @@ class TestASPDynamicPruningBase(unittest.TestCase): if core.is_compiled_with_cuda(): place = paddle.CUDAPlace(0) - self.img = paddle.to_tensor(np.random.uniform(low=-0.5, - high=0.5, - size=(32, 3, 24, 24)), - dtype=np.float32, - place=place, - stop_gradient=False) + self.img = paddle.to_tensor( + np.random.uniform(low=-0.5, high=0.5, size=(32, 3, 24, 24)), + dtype=np.float32, + place=place, + stop_gradient=False, + ) self.set_config() def set_config(self): self.mask_gen_func = 'mask_1d' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + ) def test_inference_pruning(self): self.__pruning_and_checking(False) def test_training_pruning(self): - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=self.layer.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=self.layer.parameters() + ) optimizer = paddle.incubate.asp.decorate(optimizer) self.__pruning_and_checking(True) def __pruning_and_checking(self, with_mask): - paddle.incubate.asp.prune_model(self.layer, - mask_algo=self.mask_gen_func, - with_mask=with_mask) + paddle.incubate.asp.prune_model( + self.layer, mask_algo=self.mask_gen_func, with_mask=with_mask + ) for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( - mat.T, func_name=self.mask_check_func, n=2, m=4)) + mat.T, func_name=self.mask_check_func, n=2, m=4 + ) + ) class TestASPDynamicPruning1D(TestASPDynamicPruningBase): - def set_config(self): self.mask_gen_func = 'mask_1d' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + ) class TestASPDynamicPruning2DBest(TestASPDynamicPruningBase): - def set_config(self): self.mask_gen_func = 'mask_2d_best' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + ) class TestASPDynamicPruning2DGreedy(TestASPDynamicPruningBase): - def set_config(self): self.mask_gen_func = 'mask_2d_greedy' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py index 084fbeef2d3..f312797f43a 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py @@ -24,21 +24,18 @@ paddle.enable_static() class TestASPStaticPruningBase(unittest.TestCase): - def setUp(self): self.main_program = fluid.Program() self.startup_program = fluid.Program() def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 24, 24], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 24, 24], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=2, - filter_size=3, - padding=2, - act="relu") + hidden = fluid.layers.conv2d( + input=img, num_filters=2, filter_size=3, padding=2, act="relu" + ) hidden = fluid.layers.fc(input=hidden, size=32, act='softmax') hidden = fluid.layers.fc(input=hidden, size=3, act='softmax') prediction = fluid.layers.fc(input=hidden, size=3, act='softmax') @@ -51,7 +48,9 @@ class TestASPStaticPruningBase(unittest.TestCase): def set_config(self): self.mask_gen_func = 'mask_1d' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + ) def test_inference_pruning(self): place = paddle.CPUPlace() @@ -64,10 +63,11 @@ class TestASPStaticPruningBase(unittest.TestCase): def test_training_pruning(self): with fluid.program_guard(self.main_program, self.startup_program): loss = paddle.mean( - fluid.layers.cross_entropy(input=self.predict, - label=self.label)) + fluid.layers.cross_entropy(input=self.predict, label=self.label) + ) optimizer = paddle.incubate.asp.decorate( - fluid.optimizer.SGD(learning_rate=0.01)) + fluid.optimizer.SGD(learning_rate=0.01) + ) optimizer.minimize(loss, self.startup_program) place = paddle.CPUPlace() @@ -79,45 +79,52 @@ class TestASPStaticPruningBase(unittest.TestCase): def __pruning_and_checking(self, exe, place, with_mask): exe.run(self.startup_program) - paddle.incubate.asp.prune_model(self.main_program, - mask_algo=self.mask_gen_func, - with_mask=with_mask) + paddle.incubate.asp.prune_model( + self.main_program, mask_algo=self.mask_gen_func, with_mask=with_mask + ) for param in self.main_program.global_block().all_parameters(): if ASPHelper._is_supported_layer(self.main_program, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( - mat.T, func_name=self.mask_check_func, n=2, m=4)) + mat.T, func_name=self.mask_check_func, n=2, m=4 + ) + ) class TestASPStaticPruning1D(TestASPStaticPruningBase): - def set_config(self): self.mask_gen_func = 'mask_1d' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + ) class TestASPStaticPruning2DBest(TestASPStaticPruningBase): - def set_config(self): self.mask_gen_func = 'mask_2d_best' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + ) class TestASPStaticPruning2DGreedy(TestASPStaticPruningBase): - def set_config(self): self.mask_gen_func = 'mask_2d_greedy' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py b/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py index c059703f0cc..4347371c311 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py @@ -22,13 +22,11 @@ import numpy as np class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() - self.conv1 = paddle.nn.Conv2D(in_channels=3, - out_channels=4, - kernel_size=3, - padding=2) + self.conv1 = paddle.nn.Conv2D( + in_channels=3, out_channels=4, kernel_size=3, padding=2 + ) self.linear1 = paddle.nn.Linear(4624, 32) self.linear2 = paddle.nn.Linear(32, 32) self.linear3 = paddle.nn.Linear(32, 10) @@ -43,7 +41,6 @@ class MyLayer(paddle.nn.Layer): class TestASPDynamicOptimize(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -54,7 +51,8 @@ class TestASPDynamicOptimize(unittest.TestCase): self.place = paddle.CUDAPlace(0) self.optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=self.layer.parameters()) + learning_rate=0.01, parameters=self.layer.parameters() + ) self.optimizer = paddle.incubate.asp.decorate(self.optimizer) paddle.incubate.asp.prune_model(self.layer) @@ -67,11 +65,13 @@ class TestASPDynamicOptimize(unittest.TestCase): paddle.save(self.optimizer.state_dict(), opt_path) asp_info = ASPHelper._get_program_asp_info( - paddle.static.default_main_program()) + paddle.static.default_main_program() + ) for param_name in asp_info.mask_vars: mask = asp_info.mask_vars[param_name] asp_info.update_mask_vars( - param_name, paddle.ones(shape=mask.shape, dtype=mask.dtype)) + param_name, paddle.ones(shape=mask.shape, dtype=mask.dtype) + ) asp_info.update_masks(param_name, np.ones(shape=mask.shape)) net_state_dict = paddle.load(net_path) @@ -80,14 +80,18 @@ class TestASPDynamicOptimize(unittest.TestCase): self.layer.set_state_dict(net_state_dict) self.optimizer.set_state_dict(opt_state_dict) - imgs = paddle.to_tensor(np.random.randn(64, 3, 32, 32), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(64, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(64, 3, 32, 32), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(64, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') @@ -99,24 +103,26 @@ class TestASPDynamicOptimize(unittest.TestCase): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) class TestASPStaticOptimize(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -124,15 +130,13 @@ class TestASPStaticOptimize(unittest.TestCase): self.startup_program = fluid.Program() def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 32, 32], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 32, 32], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=4, - filter_size=3, - padding=2, - act="relu") + hidden = fluid.layers.conv2d( + input=img, num_filters=4, filter_size=3, padding=2, act="relu" + ) hidden = fluid.layers.fc(input=hidden, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') return img, label, prediction @@ -140,7 +144,8 @@ class TestASPStaticOptimize(unittest.TestCase): with fluid.program_guard(self.main_program, self.startup_program): self.img, self.label, predict = build_model() self.loss = paddle.mean( - fluid.layers.cross_entropy(input=predict, label=self.label)) + fluid.layers.cross_entropy(input=predict, label=self.label) + ) self.optimizer = fluid.optimizer.SGD(learning_rate=0.01) self.optimizer = paddle.incubate.asp.decorate(self.optimizer) self.optimizer.minimize(self.loss, self.startup_program) @@ -166,29 +171,35 @@ class TestASPStaticOptimize(unittest.TestCase): state_dict = paddle.load(param_path) prog.set_state_dict(state_dict) - feeder = fluid.DataFeeder(feed_list=[self.img, self.label], - place=self.place) + feeder = fluid.DataFeeder( + feed_list=[self.img, self.label], place=self.place + ) - data = (np.random.randn(64, 3, 32, - 32), np.random.randint(10, size=(64, 1))) + data = ( + np.random.randn(64, 3, 32, 32), + np.random.randint(10, size=(64, 1)), + ) self.exe.run(prog, feed=feeder.feed([data])) for param in prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_utils.py b/python/paddle/fluid/tests/unittests/asp/test_asp_utils.py index a94520662ef..a55ba05a75d 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_utils.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_utils.py @@ -20,33 +20,50 @@ import numpy as np class TestASPUtils(unittest.TestCase): - def test_get_check_method(self): self.assertEqual( paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method( - paddle.fluid.contrib.sparsity.MaskAlgo.MASK_1D), - paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D) + paddle.fluid.contrib.sparsity.MaskAlgo.MASK_1D + ), + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D, + ) self.assertEqual( paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method( - paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_GREEDY), - paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D) + paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_GREEDY + ), + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, + ) self.assertEqual( paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method( - paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_BEST), - paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D) + paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_BEST + ), + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, + ) def test_density(self): - x = np.array([[1.0, 1.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], - [1.0, 0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], - [0.0, 1.0, 0.0, 0.0, 1.0]]) + x = np.array( + [ + [1.0, 1.0, 1.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [1.0, 0.0, 0.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [0.0, 1.0, 0.0, 0.0, 1.0], + ] + ) self.assertEqual(paddle.incubate.asp.calculate_density(x), 0.56) x[:, 0] = 0.0 self.assertEqual(paddle.incubate.asp.calculate_density(x), 0.4) def test_check_mask_1d(self): - x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], - [1.0, 1.0, 0.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], - [0.0, 1.0, 0.0, 0.0, 1.0]]) + x = np.array( + [ + [1.0, 0.0, 0.0, 1.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [0.0, 1.0, 0.0, 0.0, 1.0], + ] + ) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_1d(x, 3, 4)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 5)) @@ -58,18 +75,26 @@ class TestASPUtils(unittest.TestCase): for _ in range(10): x = np.random.randint(10, size=(5, 5)) x = paddle.fluid.contrib.sparsity.get_mask_1d(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4) + ) x = np.random.randn(5, 4) x = paddle.fluid.contrib.sparsity.get_mask_1d(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4) + ) def test_check_mask_2d(self): - x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0, 1.0]]) + x = np.array( + [ + [1.0, 0.0, 0.0, 1.0, 1.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 1.0], + ] + ) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_2d(x, 3, 4)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 5)) @@ -81,28 +106,31 @@ class TestASPUtils(unittest.TestCase): for _ in range(10): x = np.random.randint(10, size=(5, 5)) x = paddle.fluid.contrib.sparsity.get_mask_2d_greedy(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4) + ) x = np.random.randn(5, 4) x = paddle.fluid.contrib.sparsity.get_mask_2d_greedy(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4) + ) def test_get_mask_2d_best(self): for _ in range(10): x = np.random.randint(10, size=(5, 5)) x = paddle.fluid.contrib.sparsity.get_mask_2d_best(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4) + ) x = np.random.randn(5, 4) x = paddle.fluid.contrib.sparsity.get_mask_2d_best(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4) + ) def test_threadsafe_valid_2d_patterns(self): - def get_reference(m=4, n=2): from itertools import permutations @@ -112,17 +140,20 @@ class TestASPUtils(unittest.TestCase): patterns = patterns + patterns patterns = np.asarray(list(set(permutations(patterns, m)))) - valid = ((patterns.sum(axis=1) <= n).sum( - axis=1) == m).nonzero()[0].reshape(-1) + valid = ( + ((patterns.sum(axis=1) <= n).sum(axis=1) == m) + .nonzero()[0] + .reshape(-1) + ) valid_patterns = np.empty((valid.shape[0], m, m)) valid_patterns[:] = patterns[valid[:]] return valid_patterns for _ in range(4): computing_thread = threading.Thread( - target=paddle.fluid.contrib.sparsity.utils. - _compute_valid_2d_patterns, - args=(2, 4)) + target=paddle.fluid.contrib.sparsity.utils._compute_valid_2d_patterns, + args=(2, 4), + ) computing_thread.start() time.sleep(3) patterns_map = paddle.fluid.contrib.sparsity.utils._valid_2d_patterns @@ -132,7 +163,8 @@ class TestASPUtils(unittest.TestCase): self.assertTrue(reference_key in patterns_map) self.assertTrue(len(patterns_map) == 1) self.assertTrue( - (reference_patterns == patterns_map[reference_key]).all()) + (reference_patterns == patterns_map[reference_key]).all() + ) def test_check_sparsity(self): for _ in range(10): @@ -173,49 +205,64 @@ class TestASPUtils(unittest.TestCase): mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D, n=2, - m=4), paddle.fluid.contrib.sparsity.check_mask_1d(mask, 2, 4)) + m=4, + ), + paddle.fluid.contrib.sparsity.check_mask_1d(mask, 2, 4), + ) mask = paddle.fluid.contrib.sparsity.get_mask_2d_best(x_2d, 2, 4) self.assertEqual( paddle.fluid.contrib.sparsity.check_sparsity( mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, n=2, - m=4), paddle.fluid.contrib.sparsity.check_mask_2d(mask, 2, 4)) + m=4, + ), + paddle.fluid.contrib.sparsity.check_mask_2d(mask, 2, 4), + ) def __test_1D_2D_sparse_mask_generation_methods(self, x): mask = paddle.fluid.contrib.sparsity.create_mask( x, func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_1D, n=2, - m=4) + m=4, + ) self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) mask = paddle.fluid.contrib.sparsity.create_mask( x, func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_GREEDY, n=2, - m=4) + m=4, + ) self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, n=2, - m=4)) + m=4, + ) + ) mask = paddle.fluid.contrib.sparsity.create_mask( x, func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_BEST, n=2, - m=4) + m=4, + ) self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, n=2, - m=4)) + m=4, + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py index d132b319f56..a2116283a33 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py @@ -29,7 +29,6 @@ else: class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() self.linear1 = paddle.nn.Linear(32, 32) @@ -42,7 +41,6 @@ class MyLayer(paddle.nn.Layer): class TestFleetWithASPDynamic(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -56,7 +54,8 @@ class TestFleetWithASPDynamic(unittest.TestCase): self.place = paddle.CUDAPlace(0) self.optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=self.layer.parameters()) + learning_rate=0.01, parameters=self.layer.parameters() + ) def test_with_asp(self): fleet.init(is_collective=True) @@ -67,14 +66,18 @@ class TestFleetWithASPDynamic(unittest.TestCase): self.optimizer = fleet.distributed_optimizer(self.optimizer) self.layer = fleet.distributed_model(self.layer) - imgs = paddle.to_tensor(np.random.randn(64, 32), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(64, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(64, 32), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(64, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') @@ -86,24 +89,26 @@ class TestFleetWithASPDynamic(unittest.TestCase): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) class TestFleetWithASPAMPDynamic(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -117,7 +122,8 @@ class TestFleetWithASPAMPDynamic(unittest.TestCase): self.place = paddle.CUDAPlace(0) self.optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=self.layer.parameters()) + learning_rate=0.01, parameters=self.layer.parameters() + ) def test_with_asp(self): fleet.init(is_collective=True) @@ -128,14 +134,18 @@ class TestFleetWithASPAMPDynamic(unittest.TestCase): self.optimizer = fleet.distributed_optimizer(self.optimizer) self.layer = fleet.distributed_model(self.layer) - imgs = paddle.to_tensor(np.random.randn(64, 32), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(64, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(64, 32), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(64, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -150,20 +160,23 @@ class TestFleetWithASPAMPDynamic(unittest.TestCase): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py index e5ff5f839e7..9e74e10fbc5 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py @@ -32,7 +32,6 @@ paddle.enable_static() class TestFleetWithASPSharding(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -49,9 +48,9 @@ class TestFleetWithASPSharding(unittest.TestCase): def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): - input_x = paddle.static.data(name="x", - shape=[-1, 32], - dtype='float32') + input_x = paddle.static.data( + name="x", shape=[-1, 32], dtype='float32' + ) input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64') fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh') @@ -71,7 +70,7 @@ class TestFleetWithASPSharding(unittest.TestCase): "sharding_degree": 8, "mp_degree": 1, "hybrid_dp": False, - "gradient_merge_acc_step": 1 + "gradient_merge_acc_step": 1, } dist_strategy.nccl_comm_num = 1 dist_strategy.asp = True @@ -80,18 +79,21 @@ class TestFleetWithASPSharding(unittest.TestCase): def test_with_asp_sharding(self): fleet.init(is_collective=True) train_prog, startup_prog = fluid.Program(), fluid.Program() - avg_cost, strategy, input_x, input_y = self.net(train_prog, - startup_prog) + avg_cost, strategy, input_x, input_y = self.net( + train_prog, startup_prog + ) with fluid.program_guard(train_prog, startup_prog): optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) if paddle.fluid.is_compiled_with_cuda(): place = fluid.CUDAPlace( - int(os.environ.get('FLAGS_selected_gpus', 0))) + int(os.environ.get('FLAGS_selected_gpus', 0)) + ) else: place = fluid.CPUPlace() @@ -106,20 +108,23 @@ class TestFleetWithASPSharding(unittest.TestCase): for param in train_prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(train_prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py index f2056101791..aa4b0208847 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py @@ -32,7 +32,6 @@ paddle.enable_static() class TestFleetWithASPStatic(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -41,9 +40,9 @@ class TestFleetWithASPStatic(unittest.TestCase): def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): - input_x = paddle.static.data(name="x", - shape=[-1, 32], - dtype='float32') + input_x = paddle.static.data( + name="x", shape=[-1, 32], dtype='float32' + ) input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64') fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh') @@ -58,17 +57,22 @@ class TestFleetWithASPStatic(unittest.TestCase): def test_with_asp(self): fleet.init(is_collective=True) train_prog, startup_prog = fluid.Program(), fluid.Program() - avg_cost, strategy, input_x, input_y = self.net(train_prog, - startup_prog) + avg_cost, strategy, input_x, input_y = self.net( + train_prog, startup_prog + ) with fluid.program_guard(train_prog, startup_prog): optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - place = fluid.CUDAPlace( - 0) if paddle.fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place) @@ -81,24 +85,26 @@ class TestFleetWithASPStatic(unittest.TestCase): for param in train_prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(train_prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) class TestFleetWithASPAMPStatic(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -107,9 +113,9 @@ class TestFleetWithASPAMPStatic(unittest.TestCase): def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): - input_x = paddle.static.data(name="x", - shape=[-1, 32], - dtype='float32') + input_x = paddle.static.data( + name="x", shape=[-1, 32], dtype='float32' + ) input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64') fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh') @@ -124,18 +130,23 @@ class TestFleetWithASPAMPStatic(unittest.TestCase): def test_with_asp_and_amp(self): fleet.init(is_collective=True) train_prog, startup_prog = fluid.Program(), fluid.Program() - avg_cost, strategy, input_x, input_y = self.net(train_prog, - startup_prog) + avg_cost, strategy, input_x, input_y = self.net( + train_prog, startup_prog + ) strategy.amp = True with fluid.program_guard(train_prog, startup_prog): optimizer = paddle.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - place = fluid.CUDAPlace( - 0) if paddle.fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place) @@ -150,41 +161,49 @@ class TestFleetWithASPAMPStatic(unittest.TestCase): for param in train_prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(train_prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) def test_with_asp_and_pure_fp16(self): fleet.init(is_collective=True) train_prog, startup_prog = fluid.Program(), fluid.Program() with paddle.static.amp.fp16_guard(): - avg_cost, strategy, \ - input_x, input_y = self.net(train_prog, - startup_prog) + avg_cost, strategy, input_x, input_y = self.net( + train_prog, startup_prog + ) strategy.amp = True strategy.amp_configs = {'use_pure_fp16': True} with fluid.program_guard(train_prog, startup_prog): with paddle.static.amp.fp16_guard(): optimizer = optimizer = paddle.optimizer.Momentum( - learning_rate=0.01, multi_precision=True) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + learning_rate=0.01, multi_precision=True + ) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - place = fluid.CUDAPlace( - 0) if paddle.fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place) @@ -199,20 +218,23 @@ class TestFleetWithASPAMPStatic(unittest.TestCase): for param in train_prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(train_prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_checkpoint_utils.py b/python/paddle/fluid/tests/unittests/auto_checkpoint_utils.py index 4303c56675b..a9eba03ae56 100644 --- a/python/paddle/fluid/tests/unittests/auto_checkpoint_utils.py +++ b/python/paddle/fluid/tests/unittests/auto_checkpoint_utils.py @@ -25,7 +25,7 @@ import numpy as np BATCH_NUM = 4 BATCH_SIZE = 1 -#IMAGE_SIZE = 128 +# IMAGE_SIZE = 128 CLASS_NUM = 2 USE_GPU = False # whether use GPU to run model @@ -47,7 +47,6 @@ def get_random_images_and_labels(image_shape, label_shape): def sample_list_generator_creator(): - def __reader__(): for _ in range(BATCH_NUM): sample_list = [] @@ -61,21 +60,17 @@ def sample_list_generator_creator(): class AutoCheckpointBase(unittest.TestCase): - - def _init_env(self, - exe, - main_prog, - startup_prog, - minimize=True, - iterable=True): - + def _init_env( + self, exe, main_prog, startup_prog, minimize=True, iterable=True + ): def simple_net(): image = fluid.data(name='image', shape=[-1, 4, 4], dtype='float32') label = fluid.data(name='label', shape=[-1, 1], dtype='int64') fc_tmp = fluid.layers.fc(image, size=CLASS_NUM) cross_entropy = fluid.layers.softmax_with_cross_entropy( - fc_tmp, label) + fc_tmp, label + ) loss = fluid.layers.reduce_mean(cross_entropy) sgd = fluid.optimizer.SGD(learning_rate=1e-3) if minimize: @@ -87,17 +82,20 @@ class AutoCheckpointBase(unittest.TestCase): if minimize: compiled = fluid.CompiledProgram(main_prog).with_data_parallel( - loss_name=loss.name) + loss_name=loss.name + ) else: compiled = None loader = fluid.io.DataLoader.from_generator( feed_list=[image, label], capacity=64, use_double_buffer=True, - iterable=iterable) + iterable=iterable, + ) - loader.set_sample_list_generator(sample_list_generator_creator(), - places[0]) + loader.set_sample_list_generator( + sample_list_generator_creator(), places[0] + ) if minimize: exe.run(startup_prog) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/amp_pass_unittest.py b/python/paddle/fluid/tests/unittests/auto_parallel/amp_pass_unittest.py index c00a3367a98..5d41e1cc484 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/amp_pass_unittest.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/amp_pass_unittest.py @@ -31,7 +31,9 @@ def apply_pass(use_amp=False, level=None): amp.enable = True amp.custom_white_list = ['softmax', 'layer_norm', 'gelu'] amp.custom_black_list = [ - 'c_softmax_with_cross_entropy', 'elementwise_div', 'reduce_sum' + 'c_softmax_with_cross_entropy', + 'elementwise_div', + 'reduce_sum', ] amp.init_loss_scaling = 32768 amp.use_fp16_guard = False @@ -47,7 +49,6 @@ def reset_prog(): class TestAMPPass(unittest.TestCase): - def setUp(self): self.rtol = 1e-5 self.atol = 1e-8 @@ -82,7 +83,9 @@ class TestAMPPass(unittest.TestCase): rtol=rtol or self.rtol, atol=atol or self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses)) + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) def test_amp_pass(self): # mp2 training diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_model.py b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_model.py index bccf0963ce1..2c03dab50ea 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_model.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_model.py @@ -36,39 +36,39 @@ def get_random_inputs_and_labels(input_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_size): batch_input, batch_label = get_random_inputs_and_labels( [batch_size, sequence_len, hidden_size], - [batch_size, sequence_len, 1]) + [batch_size, sequence_len, 1], + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -85,29 +85,34 @@ class MLPLayer(nn.Layer): def mlp_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) auto.shard_tensor(input, _global_process_mesh, [None, None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) loss = paddle.mean(error_cost) - loader = paddle.io.DataLoader.from_generator(feed_list=[input, label], - capacity=4 * batch_size, - iterable=True) + loader = paddle.io.DataLoader.from_generator( + feed_list=[input, label], capacity=4 * batch_size, iterable=True + ) return loss, train_program, start_program, loader @@ -125,17 +130,24 @@ def train(): train_program = static.Program() start_program = static.Program() loss, train_program, start_program, loader = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) places = static.cuda_places() loader.set_batch_generator(batch_generator_creator(), places=places) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_gpt_planner.py b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_gpt_planner.py index 3634f4258a0..06b5a808cf4 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_gpt_planner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_gpt_planner.py @@ -21,51 +21,58 @@ import numpy as np sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) -def get_gpt_model(train_program, start_program, place, batch_size, sequence_len, - vocab_size): +def get_gpt_model( + train_program, start_program, place, batch_size, sequence_len, vocab_size +): modeling.init_global() with static.program_guard(train_program, start_program): - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] - gpt = GPTModel(vocab_size=1000, - hidden_size=64, - num_hidden_layers=2, - num_attention_heads=8, - intermediate_size=256, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=8, + intermediate_size=256, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) @@ -102,17 +109,28 @@ def train(): sequence_len = 512 vocab_size = 1000 train_program, start_program, loss, gen_data = get_gpt_model( - train_program, start_program, place, batch_size, sequence_len, - vocab_size) - - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + train_program, + start_program, + place, + batch_size, + sequence_len, + vocab_size, + ) + + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) places = static.cuda_places() exe = paddle.static.Executor(places[0]) @@ -121,25 +139,29 @@ def train(): for step in range(10): tokens, position_ids, attention_mask, labels, loss_mask = gen_data() if loss.name in distributed_main_program.global_block().vars: - loss_print, = exe.run(distributed_main_program, - feed={ - "tokens": tokens, - "position_ids": position_ids, - "attention_mask": attention_mask, - "labels": labels, - "loss_mask": loss_mask - }, - fetch_list=[loss]) + (loss_print,) = exe.run( + distributed_main_program, + feed={ + "tokens": tokens, + "position_ids": position_ids, + "attention_mask": attention_mask, + "labels": labels, + "loss_mask": loss_mask, + }, + fetch_list=[loss], + ) print("step: %s, loss: %f" % (step, loss_print[0])) else: - exe.run(distributed_main_program, - feed={ - "tokens": tokens, - "position_ids": position_ids, - "attention_mask": attention_mask, - "labels": labels, - "loss_mask": loss_mask - }) + exe.run( + distributed_main_program, + feed={ + "tokens": tokens, + "position_ids": position_ids, + "attention_mask": attention_mask, + "labels": labels, + "loss_mask": loss_mask, + }, + ) print("step: %s, loss: %s" % (step, "None")) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_planner.py b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_planner.py index b40a61ed34c..3111611895a 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_planner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_planner.py @@ -17,12 +17,15 @@ import paddle.static as static from paddle.distributed import fleet from paddle.distributed.auto_parallel.cost import CostEstimator from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) def train(): from auto_parallel_relaunch_model import mlp_pretrain_forward from auto_parallel_relaunch_model import batch_generator_creator + dist_strategy = fleet.DistributedStrategy() # init parallel optimizer dist_strategy.auto_search = True @@ -30,17 +33,24 @@ def train(): train_program = static.Program() start_program = static.Program() loss, train_program, start_program, loader = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) # add cost estimator dist_context = get_default_distributed_context() @@ -51,11 +61,18 @@ def train(): dims_mapping = dist_op.dist_attr.get_input_dims_mapping(var_name) if dims_mapping is None: dist_op.dist_attr.set_input_dims_mapping( - var_name, [ - -1 for i in range( - len(train_program.global_block().vars[var_name]. - shape)) - ]) + var_name, + [ + -1 + for i in range( + len( + train_program.global_block() + .vars[var_name] + .shape + ) + ) + ], + ) cluster.gen_default_config_cluster(device_count=2) cost_estimator = CostEstimator(train_program, cluster) global_cost = cost_estimator.estimate(dist_context) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/clip_grad_by_global_norm.py b/python/paddle/fluid/tests/unittests/auto_parallel/clip_grad_by_global_norm.py index ba255fba279..d25a8bcd4c5 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/clip_grad_by_global_norm.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/clip_grad_by_global_norm.py @@ -61,7 +61,6 @@ def reset_prog(): class TestGradientClipByGlobalNorm(unittest.TestCase): - def setUp(self): self.batch_size = 2 self.batch_num = 1 @@ -94,9 +93,10 @@ class TestGradientClipByGlobalNorm(unittest.TestCase): sharding_p, rtol=1e-05, atol=1e-08, - err_msg= - 'gradient clip by global norm has wrong results!, \nu={}\nv={}\ndiff={}' - .format(dp_p, sharding_p, dp_p - sharding_p)) + err_msg='gradient clip by global norm has wrong results!, \nu={}\nv={}\ndiff={}'.format( + dp_p, sharding_p, dp_p - sharding_p + ), + ) def test_grad_clip(self): # dp2 training @@ -108,7 +108,8 @@ class TestGradientClipByGlobalNorm(unittest.TestCase): sharding_engine = self.get_engine(True) sharding_engine.fit(self.dataset, 3, batch_size=self.batch_size) sharding_param_values = get_parameter_value( - sharding_engine.main_program) + sharding_engine.main_program + ) self.check_result(dp_param_values, sharding_param_values) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/converter.py b/python/paddle/fluid/tests/unittests/auto_parallel/converter.py index 291dae6612c..5e0506c3785 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/converter.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/converter.py @@ -28,21 +28,21 @@ def test_convert(): tensor_name: { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [-1, -1] + "dims_mapping": [-1, -1], } } row_strategy = { tensor_name: { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [0, -1] + "dims_mapping": [0, -1], } } col_strategy = { tensor_name: { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [-1, 0] + "dims_mapping": [-1, 0], } } @@ -70,7 +70,7 @@ def test_convert(): new_name: { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [0, -1] + "dims_mapping": [0, -1], } } converter = Converter(tensor_dict, col_strategy, row_strategy) @@ -84,14 +84,14 @@ def test_convert(): "tensor_2": { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [-1, -1] + "dims_mapping": [-1, -1], } } row_strategy = { "tensor_2": { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [0, -1] + "dims_mapping": [0, -1], } } tensor_dict = {"tensor_2": [complete_tensor]} diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/engine_api.py b/python/paddle/fluid/tests/unittests/auto_parallel/engine_api.py index 38287e98a21..c09edf04425 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/engine_api.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/engine_api.py @@ -45,7 +45,6 @@ my_feed_vars = [] class MyDataset(Dataset): - def __init__(self, num_samples): super(MyDataset, self).__init__() self.num_samples = num_samples @@ -66,38 +65,38 @@ def get_random_inputs_and_labels(image_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_num): batch_input, batch_label = get_random_inputs_and_labels( - [batch_size, image_size], [batch_size, 1]) + [batch_size, image_size], [batch_size, 1] + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -121,16 +120,20 @@ class MLPLayer(nn.Layer): def train_high_level(fetch): global is_fetch is_fetch = fetch - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() strategy = auto.Strategy() @@ -142,11 +145,13 @@ def train_high_level(fetch): train_dataset = MyDataset(batch_num * batch_size) eval_dataset1 = MyDataset(5 * batch_size) - history = engine.fit(train_data=train_dataset, - epochs=2, - batch_size=batch_size, - valid_data=eval_dataset1, - log_freq=1) + history = engine.fit( + train_data=train_dataset, + epochs=2, + batch_size=batch_size, + valid_data=eval_dataset1, + log_freq=1, + ) # eval eval_dataset2 = MyDataset(batch_size) @@ -165,16 +170,20 @@ def train_high_level(fetch): def train_low_level(): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() strategy = auto.Strategy() @@ -189,18 +198,18 @@ def train_low_level(): # Build normal normal dataloader # train train_dataset = MyDataset(batch_num * batch_size) - train_dataloader = engine.dataloader(train_dataset, - batch_size=batch_size, - mode="train") + train_dataloader = engine.dataloader( + train_dataset, batch_size=batch_size, mode="train" + ) engine.prepare(mode="train") for data in train_dataloader: outs = engine.run(data, feed=feed_dict, mode="train") # eval eval_dataset2 = MyDataset(batch_size) - eval_dataloader = engine.dataloader(eval_dataset2, - batch_size=batch_size, - mode="eval") + eval_dataloader = engine.dataloader( + eval_dataset2, batch_size=batch_size, mode="eval" + ) engine.prepare(mode="eval") for data in eval_dataloader: outs = engine.run(data, feed=feed_dict, mode="eval") @@ -223,9 +232,9 @@ def train_low_level(): # Build dataloader from generator # train train_dataset = MyDataset(batch_num * batch_size) - train_dataloader = engine.dataloader_from_generator(train_dataset, - batch_size=batch_size, - mode="train") + train_dataloader = engine.dataloader_from_generator( + train_dataset, batch_size=batch_size, mode="train" + ) engine.prepare(mode="train") for data in train_dataloader: outs = engine.run(data, feed=feed_dict, mode="train") @@ -233,17 +242,18 @@ def train_low_level(): # eval engine.to_mode("eval") eval_dataset2 = MyDataset(batch_size) - eval_dataloader = engine.dataloader_from_generator(eval_dataset2, - batch_size=batch_size) + eval_dataloader = engine.dataloader_from_generator( + eval_dataset2, batch_size=batch_size + ) engine.prepare() for data in eval_dataloader: outs = engine.run(data, feed=feed_dict) # predict test_dataset = MyDataset(batch_size) - predict_dataloader = engine.dataloader_from_generator(test_dataset, - batch_size=batch_size, - mode="predict") + predict_dataloader = engine.dataloader_from_generator( + test_dataset, batch_size=batch_size, mode="predict" + ) engine.prepare(mode="predict") for data in predict_dataloader: outs = engine.run(data, feed=feed_dict, mode="predict") @@ -257,16 +267,20 @@ def train_low_level(): def train_builtin_data_vars(): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() strategy = auto.Strategy() @@ -284,9 +298,9 @@ def train_builtin_data_vars(): with static.program_guard(engine.main_program, engine.startup_program): feed_list = engine.inputs + engine.labels print(feed_list) - loader = paddle.io.DataLoader.from_generator(feed_list=feed_list, - capacity=4 * batch_size, - iterable=False) + loader = paddle.io.DataLoader.from_generator( + feed_list=feed_list, capacity=4 * batch_size, iterable=False + ) places = static.cuda_places() loader.set_batch_generator(batch_generator_creator(), places=places) @@ -297,36 +311,40 @@ def train_builtin_data_vars(): while True: engine.run() except paddle.fluid.core.EOFException: - loader.reset( - ) # call DataLoader.reset() after catching EOFException + loader.reset() # call DataLoader.reset() after catching EOFException def train_non_builtin_data_vars(): main_program = static.Program() startup_program = static.Program() - with static.program_guard(main_program, - startup_program), utils.unique_name.guard(): - input = static.data(name="input", - shape=[batch_size, image_size], - dtype='float32') + with static.program_guard( + main_program, startup_program + ), utils.unique_name.guard(): + input = static.data( + name="input", shape=[batch_size, image_size], dtype='float32' + ) label = static.data(name="label", shape=[batch_size, 1], dtype='int64') - loader = paddle.io.DataLoader.from_generator(feed_list=[input, label], - capacity=4 * batch_size, - iterable=False) + loader = paddle.io.DataLoader.from_generator( + feed_list=[input, label], capacity=4 * batch_size, iterable=False + ) places = static.cuda_places() loader.set_batch_generator(batch_generator_creator(), places=places) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() predict = mlp(input) loss_var = loss(predict, label) @@ -334,53 +352,58 @@ def train_non_builtin_data_vars(): strategy = auto.Strategy() strategy.auto_mode = "semi" - engine = auto.Engine(loss=loss_var, - optimizer=optimizer, - metrics=metric, - strategy=strategy) + engine = auto.Engine( + loss=loss_var, optimizer=optimizer, metrics=metric, strategy=strategy + ) # train engine.to_mode("train") - engine.prepare(inputs=[input], - labels=[label], - main_program=main_program, - startup_program=startup_program) + engine.prepare( + inputs=[input], + labels=[label], + main_program=main_program, + startup_program=startup_program, + ) for _ in range(epoch_num): loader.start() # call DataLoader.start() before each epoch starts try: while True: engine.run() except paddle.fluid.core.EOFException: - loader.reset( - ) # call DataLoader.reset() after catching EOFException + loader.reset() # call DataLoader.reset() after catching EOFException def get_cost(): main_program = static.default_main_program() startup_program = static.default_startup_program() - with static.program_guard(main_program, - startup_program), utils.unique_name.guard(): - input = static.data(name="input", - shape=[batch_size, image_size], - dtype='float32') + with static.program_guard( + main_program, startup_program + ), utils.unique_name.guard(): + input = static.data( + name="input", shape=[batch_size, image_size], dtype='float32' + ) label = static.data(name="label", shape=[batch_size, 1], dtype='int64') - loader = paddle.io.DataLoader.from_generator(feed_list=[input, label], - capacity=4 * batch_size, - iterable=False) + loader = paddle.io.DataLoader.from_generator( + feed_list=[input, label], capacity=4 * batch_size, iterable=False + ) places = static.cuda_places() loader.set_batch_generator(batch_generator_creator(), places=places) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() predict = mlp(input) loss_var = loss(predict, label) @@ -388,24 +411,27 @@ def get_cost(): strategy = auto.Strategy() strategy.auto_mode = "semi" - engine = auto.Engine(loss=loss_var, - optimizer=optimizer, - metrics=metric, - strategy=strategy) + engine = auto.Engine( + loss=loss_var, optimizer=optimizer, metrics=metric, strategy=strategy + ) engine.cost() def get_cost_by_spec(): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() strategy = auto.Strategy() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/engine_api_dp.py b/python/paddle/fluid/tests/unittests/auto_parallel/engine_api_dp.py index deb084a79fe..4a04f9c5a6a 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/engine_api_dp.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/engine_api_dp.py @@ -34,7 +34,6 @@ paddle.seed(44) class MyDataset(Dataset): - def __init__(self, num_samples): super(MyDataset, self).__init__() self.num_samples = num_samples @@ -49,27 +48,27 @@ class MyDataset(Dataset): class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -87,26 +86,28 @@ class MLPLayer(nn.Layer): def train(fetch): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) dist_strategy = auto.Strategy() dist_strategy.auto_mode = "semi" # init engine - engine = auto.Engine(mlp, - loss, - optimizer, - paddle.metric.Accuracy(), - strategy=dist_strategy) + engine = auto.Engine( + mlp, loss, optimizer, paddle.metric.Accuracy(), strategy=dist_strategy + ) # train train_dataset = MyDataset(batch_num * batch_size) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py b/python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py index 318773c71e0..178ba60f814 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py @@ -21,14 +21,17 @@ from paddle.distributed.fleet import auto sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) sequence_len = 512 vocab_size = 1000 class FakeDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples self.sequence_len = sequence_len @@ -40,8 +43,11 @@ class FakeDataset(paddle.io.Dataset): random.seed(2021) tokens = np.random.randint(self.vocab_size, size=self.sequence_len) position_ids = np.arange(self.sequence_len) - attention_mask = np.tril(np.ones(self.sequence_len)).reshape( - (1, self.sequence_len, self.sequence_len)).astype(np.float32) + attention_mask = ( + np.tril(np.ones(self.sequence_len)) + .reshape((1, self.sequence_len, self.sequence_len)) + .astype(np.float32) + ) labels = np.random.randint(self.vocab_size, size=self.sequence_len) loss_mask = np.ones(self.sequence_len).astype(np.float32) return tokens, position_ids, attention_mask, labels, loss_mask @@ -51,30 +57,32 @@ class FakeDataset(paddle.io.Dataset): def create_data_holder(batch_size): - tokens = paddle.static.InputSpec(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.InputSpec(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.InputSpec( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.InputSpec( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.InputSpec( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.InputSpec(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.InputSpec(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.InputSpec( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.InputSpec( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) return [tokens, position_ids, attention_mask], [labels, loss_mask] def generate_model(strategy): modeling.init_global() ranks = list(range(paddle.distributed.get_world_size())) - modeling._global_process_mesh = auto.ProcessMesh(mesh=ranks, - dim_names=["x"]) + modeling._global_process_mesh = auto.ProcessMesh( + mesh=ranks, dim_names=["x"] + ) if strategy == "serial": modeling._global_parallel_strategy = "serial" elif strategy == "mp": @@ -84,24 +92,25 @@ def generate_model(strategy): else: raise ValueError("Only support serial, mp2 and dp2.") - gpt = GPTModel(vocab_size=1000, - hidden_size=64, - num_hidden_layers=2, - num_attention_heads=8, - intermediate_size=256, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3) - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=8, + intermediate_size=256, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + ) + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) criterion = GPTPretrainingCriterion() return model, criterion diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/gradient_merge_pass_unittest.py b/python/paddle/fluid/tests/unittests/auto_parallel/gradient_merge_pass_unittest.py index bdce36be605..8ed91061fd4 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/gradient_merge_pass_unittest.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/gradient_merge_pass_unittest.py @@ -43,7 +43,6 @@ def reset_prog(): class TestGradientMergePass(unittest.TestCase): - def setUp(self): self.rtol = 1e-5 self.atol = 1e-8 @@ -78,23 +77,23 @@ class TestGradientMergePass(unittest.TestCase): rtol=self.rtol, atol=self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses)) + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) def test_gradient_merge_pass(self): # dp2 training dp_engine = self.get_engine() - history = dp_engine.fit(self.dataset, - 3, - batch_size=self.batch_size, - log_freq=1) + history = dp_engine.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) dp_losses = np.array(history.history["loss"]) # dp2 gradient merge training gm_engine = self.get_engine(True) - history = gm_engine.fit(self.dataset, - 3, - batch_size=self.batch_size, - log_freq=1) + history = gm_engine.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) gm_losses = np.array(history.history["loss"]) # avg_loss = 0 diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/high_order_grad.py b/python/paddle/fluid/tests/unittests/auto_parallel/high_order_grad.py index 8e148719a56..1ba31ad2e1d 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/high_order_grad.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/high_order_grad.py @@ -22,7 +22,6 @@ paddle.seed(1234) class FCNet: - def __init__(self, num_ins, num_outs, num_layers, hidden_size): self.num_ins = num_ins self.num_outs = num_outs @@ -43,12 +42,12 @@ class FCNet: lsize = self.hidden_size rsize = self.hidden_size - w = paddle.static.create_parameter(shape=[lsize, rsize], - dtype="float32", - is_bias=False) - b = paddle.static.create_parameter(shape=[rsize], - dtype="float32", - is_bias=True) + w = paddle.static.create_parameter( + shape=[lsize, rsize], dtype="float32", is_bias=False + ) + b = paddle.static.create_parameter( + shape=[rsize], dtype="float32", is_bias=True + ) self.weights.append(w) self.biases.append(b) @@ -62,13 +61,14 @@ class FCNet: class LaplaceModel(paddle.nn.Layer): - def __init__(self, num_ins=2, num_outs=1, num_layers=5, hidden_size=20): super(LaplaceModel, self).__init__() - self.net = FCNet(num_ins=num_ins, - num_outs=num_outs, - num_layers=num_layers, - hidden_size=hidden_size) + self.net = FCNet( + num_ins=num_ins, + num_outs=num_outs, + num_layers=num_layers, + hidden_size=hidden_size, + ) def forward(self, inputs, bc_index): inputs.stop_gradient = False @@ -82,7 +82,6 @@ class LaplaceModel(paddle.nn.Layer): class LaplaceDataset(paddle.io.Dataset): - def __init__(self, num_sample): self.num_sample = num_sample @@ -126,10 +125,9 @@ def main(): dist_strategy = auto.Strategy() dist_strategy.auto_mode = "semi" - engine = auto.Engine(laplace, - loss=loss_func, - optimizer=optimizer, - strategy=dist_strategy) + engine = auto.Engine( + laplace, loss=loss_func, optimizer=optimizer, strategy=dist_strategy + ) engine.fit(train_dataset, train_sample_split=2, batch_size=None) dist_context = engine.dist_context diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/iterable_dataset.py b/python/paddle/fluid/tests/unittests/auto_parallel/iterable_dataset.py index d5eb88bc6b5..b953471eef8 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/iterable_dataset.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/iterable_dataset.py @@ -36,7 +36,6 @@ paddle.seed(44) class MyDataset(paddle.io.IterableDataset): - def __init__(self, num_samples): self.num_samples = num_samples @@ -48,17 +47,18 @@ class MyDataset(paddle.io.IterableDataset): class MyDataset1(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples self.data = [] for i in range(self.num_samples): input1 = np.random.uniform(size=image_size).astype("float32") - label1 = np.array(np.random.randint(0, class_num - 1, - dtype="int64")) + label1 = np.array( + np.random.randint(0, class_num - 1, dtype="int64") + ) input2 = np.random.uniform(size=image_size).astype("float32") - label2 = np.array(np.random.randint(0, class_num - 1, - dtype="int64")) + label2 = np.array( + np.random.randint(0, class_num - 1, dtype="int64") + ) input = np.stack((input1, input2)) label = np.stack((label1, label2)) self.data.append((input, label)) @@ -71,27 +71,27 @@ class MyDataset1(paddle.io.Dataset): class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -108,27 +108,29 @@ class MLPLayer(nn.Layer): def train(fetch): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) dist_strategy = auto.Strategy() dist_strategy.auto_mode = "semi" dist_strategy.split_data = True # init engine - engine = auto.Engine(mlp, - loss, - optimizer, - paddle.metric.Accuracy(), - strategy=dist_strategy) + engine = auto.Engine( + mlp, loss, optimizer, paddle.metric.Accuracy(), strategy=dist_strategy + ) # train train_dataset = MyDataset(batch_num * batch_size) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/optimization_tuner_api.py b/python/paddle/fluid/tests/unittests/auto_parallel/optimization_tuner_api.py index 3cc24a538fe..b05e27d4c54 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/optimization_tuner_api.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/optimization_tuner_api.py @@ -31,27 +31,27 @@ paddle.seed(44) class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -68,16 +68,20 @@ class MLPLayer(nn.Layer): def train(fetch): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) dist_strategy = auto.Strategy() dist_strategy.auto_mode = "semi" @@ -97,15 +101,13 @@ def train(fetch): tuning.verbose = True dataset = MyDataset(batch_num * batch_size) - engine = auto.Engine(mlp, - loss, - optimizer, - paddle.metric.Accuracy(), - strategy=dist_strategy) + engine = auto.Engine( + mlp, loss, optimizer, paddle.metric.Accuracy(), strategy=dist_strategy + ) engine._tune(dataset, batch_size=batch_size) # check tuned - assert (engine._dist_contexts['train'].strategy.sharding.stage != 3) + assert engine._dist_contexts['train'].strategy.sharding.stage != 3 if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/recompute_pass_unittest.py b/python/paddle/fluid/tests/unittests/auto_parallel/recompute_pass_unittest.py index 9c17af12d73..1aa83f1a8c9 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/recompute_pass_unittest.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/recompute_pass_unittest.py @@ -38,7 +38,6 @@ def reset_prog(): class TestRecomputePass(unittest.TestCase): - def setUp(self): self.rtol = 1e-6 self.atol = 1e-8 @@ -73,7 +72,9 @@ class TestRecomputePass(unittest.TestCase): rtol=self.rtol, atol=self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses)) + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) def test_recompute_pass(self): # mp2 training diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/sharding_pass_unittest.py b/python/paddle/fluid/tests/unittests/auto_parallel/sharding_pass_unittest.py index a549cd8e6d8..80c2b74ade2 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/sharding_pass_unittest.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/sharding_pass_unittest.py @@ -43,7 +43,6 @@ def reset_prog(): class TestShardingPass(unittest.TestCase): - def setUp(self): self.rtol = 1e-6 self.atol = 1e-8 @@ -78,7 +77,9 @@ class TestShardingPass(unittest.TestCase): rtol=self.rtol, atol=self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses)) + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) def test_sharding_pass(self): # dp2 training @@ -88,25 +89,25 @@ class TestShardingPass(unittest.TestCase): # sharding2 stage1 training sharding1_engine = self.get_engine(True, 1) - history = sharding1_engine.fit(self.dataset, - 3, - batch_size=self.batch_size) + history = sharding1_engine.fit( + self.dataset, 3, batch_size=self.batch_size + ) sharding1_losses = np.array(history.history["loss"]) self.check_results(dp_losses, sharding1_losses) # sharding2 stage2 training sharding2_engine = self.get_engine(True, 2) - history = sharding2_engine.fit(self.dataset, - 3, - batch_size=self.batch_size) + history = sharding2_engine.fit( + self.dataset, 3, batch_size=self.batch_size + ) sharding2_losses = np.array(history.history["loss"]) self.check_results(dp_losses, sharding2_losses) # sharding2 stage3 training sharding3_engine = self.get_engine(True, 3) - history = sharding3_engine.fit(self.dataset, - 3, - batch_size=self.batch_size) + history = sharding3_engine.fit( + self.dataset, 3, batch_size=self.batch_size + ) sharding3_losses = np.array(history.history["loss"]) self.check_results(dp_losses, sharding3_losses) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_auto_parallel_relaunch.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_auto_parallel_relaunch.py index 53734e21d9a..42d309866c3 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_auto_parallel_relaunch.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_auto_parallel_relaunch.py @@ -93,7 +93,6 @@ mapping_josn = """ class TestAutoParallelReLaunch(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -101,10 +100,12 @@ class TestAutoParallelReLaunch(unittest.TestCase): self.temp_dir.cleanup() def test_relaunch(self): - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") - mapping_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_rank_mapping.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) + mapping_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_rank_mapping.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: @@ -115,20 +116,32 @@ class TestAutoParallelReLaunch(unittest.TestCase): json.dump(mapping_josn_object, mapping_josn_file) file_dir = os.path.dirname(os.path.abspath(__file__)) - launch_model_path = os.path.join(file_dir, - "auto_parallel_relaunch_model.py") + launch_model_path = os.path.join( + file_dir, "auto_parallel_relaunch_model.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] else: coverage_args = [] - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--log_dir", self.temp_dir.name, - "--cluster_topo_path", cluster_json_path, "--rank_mapping_path", - mapping_json_path, "--enable_auto_mapping", "True", - launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--log_dir", + self.temp_dir.name, + "--cluster_topo_path", + cluster_json_path, + "--rank_mapping_path", + mapping_json_path, + "--enable_auto_mapping", + "True", + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() self.assertEqual(process.returncode, 0) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_base_cost.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_base_cost.py index 745351f3d7e..b03f3408d1f 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_base_cost.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_base_cost.py @@ -28,10 +28,18 @@ from paddle.distributed.auto_parallel.dist_context import DistributedContext from paddle.distributed import fleet from paddle.distributed.auto_parallel.parallelizer import AutoParallelizer from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.cost.base_cost import build_comp_desc_from_dist_op -from paddle.distributed.auto_parallel.cost.base_cost import build_comm_desc_from_dist_op -from paddle.distributed.auto_parallel.cost.base_cost import build_comm_costs_from_descs -from paddle.distributed.auto_parallel.cost.base_cost import build_comp_costs_from_descs +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comp_desc_from_dist_op, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comm_desc_from_dist_op, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comm_costs_from_descs, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comp_costs_from_descs, +) from paddle.distributed.auto_parallel.cost.base_cost import build_dp_costs from paddle.distributed.auto_parallel.cost import AllreduceSumOpCost from paddle.distributed.auto_parallel.cost import _g_op_cost_factory @@ -39,33 +47,34 @@ from test_cluster import cluster_json paddle.enable_static() _global_parallel_strategy = "dp_mp_pp" -_global_process_mesh = auto.ProcessMesh([[[0, 1], [4, 5]], [[2, 3], [6, 7]]], - dim_names=["x", "y", "z"]) +_global_process_mesh = auto.ProcessMesh( + [[[0, 1], [4, 5]], [[2, 3], [6, 7]]], dim_names=["x", "y", "z"] +) PP_MESH_0 = auto.ProcessMesh([[0, 1], [4, 5]], dim_names=["x", "y"]) PP_MESH_1 = auto.ProcessMesh([[2, 3], [6, 7]], dim_names=["x", "y"]) class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -81,29 +90,33 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) fill_constant_out = paddle.fluid.layers.fill_constant_batch_size_like( - input=input, shape=[batch_size], value=1, dtype="int32") + input=input, shape=[batch_size], value=1, dtype="int32" + ) embedding = paddle.nn.Embedding(10, hidden_size, sparse=True) embedding_out = embedding(fill_constant_out) auto.shard_tensor(input, PP_MESH_0, ["x", None]) auto.shard_tensor(label, PP_MESH_1, ["x", None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(embedding_out) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -115,8 +128,9 @@ def mlp_forward(train_program, start_program): def get_prog(train_program, startup_program, dist_context, rank_id): global _global_process_mesh dist_context.process_mesh = _global_process_mesh - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -126,19 +140,21 @@ def get_prog(train_program, startup_program, dist_context, rank_id): # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) return train_program, startup_program, params_grads class TestBaseCost(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -147,8 +163,9 @@ class TestBaseCost(unittest.TestCase): def test_base_cost(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -160,7 +177,8 @@ class TestBaseCost(unittest.TestCase): dist_context = DistributedContext() rank_id = 2 train_program, startup_program, params_grads = get_prog( - train_program, startup_program, dist_context, rank_id) + train_program, startup_program, dist_context, rank_id + ) for op in train_program.global_block().ops: dist_op = dist_context.get_dist_op_for_program(op) @@ -171,13 +189,15 @@ class TestBaseCost(unittest.TestCase): var_names = None if op.input_arg_names: var_names = op.input_arg_names[0] - comm_descs = build_comm_desc_from_dist_op("c_allreduce_sum", - dist_op, - dist_context, - var_names, - attrs=None, - parallel_axis=0, - group_ranks=None) + comm_descs = build_comm_desc_from_dist_op( + "c_allreduce_sum", + dist_op, + dist_context, + var_names, + attrs=None, + parallel_axis=0, + group_ranks=None, + ) self.assertTrue(isinstance(comm_descs, dict) and comm_descs) comm_descs = build_comm_desc_from_dist_op( "c_allreduce_sum", @@ -186,22 +206,38 @@ class TestBaseCost(unittest.TestCase): var_names, attrs=None, parallel_axis=None, - group_ranks=processes) + group_ranks=processes, + ) self.assertTrue(isinstance(comm_descs, dict) and comm_descs) comm_costs = build_comm_costs_from_descs( - AllreduceSumOpCost, dist_context, processes, comm_descs, - cluster) + AllreduceSumOpCost, + dist_context, + processes, + comm_descs, + cluster, + ) self.assertTrue(comm_costs) comp_costs = build_comp_costs_from_descs( - _g_op_cost_factory[op.type], dist_context, processes, - comp_descs, cluster) + _g_op_cost_factory[op.type], + dist_context, + processes, + comp_descs, + cluster, + ) self.assertTrue(comp_costs) result = [] - build_dp_costs(result, dist_op, dist_context, var_names[0], - None, 0, cluster) + build_dp_costs( + result, + dist_op, + dist_context, + var_names[0], + None, + 0, + cluster, + ) self.assertTrue(result) # Remove unnecessary files diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster.py index 1aeed0c72ef..f5c4d53c060 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster.py @@ -1968,7 +1968,6 @@ multi_cluster_json = """{ class TestCluster(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -1977,8 +1976,9 @@ class TestCluster(unittest.TestCase): def test_single_machine(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster_single.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster_single.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: @@ -2003,8 +2003,9 @@ class TestCluster(unittest.TestCase): def test_multi_machine(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster_multi.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster_multi.json" + ) cluster_json_object = json.loads(multi_cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster_v2.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster_v2.py index acadc4842a2..832cdf75f92 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster_v2.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster_v2.py @@ -19,7 +19,6 @@ from paddle.distributed.auto_parallel.cluster_v2 import DeviceMesh class TestDeviceMesh(unittest.TestCase): - def test_device_mesh(self): name = "my_device_mesh" mesh = [[0, 1, 2], [3, 4, 5]] @@ -77,11 +76,11 @@ class TestDeviceMesh(unittest.TestCase): self.assertEqual(device_mesh.machine(0).device(2), dev2) self.assertEqual(device_mesh.machine(1).link(3, 4), link2) self.assertEqual( - device_mesh.machine(0).devices, - device_mesh.machine(0).devices) + device_mesh.machine(0).devices, device_mesh.machine(0).devices + ) self.assertEqual( - device_mesh.machine(0).links, - device_mesh.machine(0).links) + device_mesh.machine(0).links, device_mesh.machine(0).links + ) self.assertEqual(device_mesh.device_type, "GPU") self.assertEqual(device_mesh.devices, device_mesh.devices) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_comm_cost.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_comm_cost.py index 5744cf6d392..32b180ffdc1 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_comm_cost.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_comm_cost.py @@ -32,7 +32,6 @@ from test_cluster import cluster_json, multi_cluster_json class TestCommOpCost(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -41,8 +40,9 @@ class TestCommOpCost(unittest.TestCase): def test_comm_cost(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster0.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster0.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -55,54 +55,72 @@ class TestCommOpCost(unittest.TestCase): comm_context = CommContext(cluster) # Check AllreduceSumCost 128MB ring cost - allreduce_sum_op_desc = build_comm_desc("c_allreduce_sum", - [0, 1, 2, 3, 4, 5, 6, 7], - paddle.float32, - [1, 32 * (10**6)]) + allreduce_sum_op_desc = build_comm_desc( + "c_allreduce_sum", + [0, 1, 2, 3, 4, 5, 6, 7], + paddle.float32, + [1, 32 * (10**6)], + ) allreduce_sum_op_cost = AllreduceSumOpCost( - op_desc=allreduce_sum_op_desc, comm_context=comm_context) + op_desc=allreduce_sum_op_desc, comm_context=comm_context + ) # Check AllgatherOpCost cost - allgather_op_desc = build_comm_desc("c_allgather", - [0, 1, 2, 3, 4, 5, 6, 7], - paddle.float32, [1, 32 * (10**6)]) - allgather_op_cost = AllgatherOpCost(op_desc=allgather_op_desc, - comm_context=comm_context) + allgather_op_desc = build_comm_desc( + "c_allgather", + [0, 1, 2, 3, 4, 5, 6, 7], + paddle.float32, + [1, 32 * (10**6)], + ) + allgather_op_cost = AllgatherOpCost( + op_desc=allgather_op_desc, comm_context=comm_context + ) self.assertTrue(allgather_op_cost.time > 0) # Check BroadcastOpCost cost - broadcast_op_desc = build_comm_desc("c_broadcast", - [0, 1, 2, 3, 4, 5, 6, 7], - paddle.float32, [1, 32 * (10**6)]) - broadcast_op_cost = BroadcastOpCost(op_desc=broadcast_op_desc, - comm_context=comm_context) + broadcast_op_desc = build_comm_desc( + "c_broadcast", + [0, 1, 2, 3, 4, 5, 6, 7], + paddle.float32, + [1, 32 * (10**6)], + ) + broadcast_op_cost = BroadcastOpCost( + op_desc=broadcast_op_desc, comm_context=comm_context + ) self.assertTrue(broadcast_op_cost.time > 0) # Check SendOpCost cost - send_op_desc = build_comm_desc("send_v2", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - send_op_cost = SendOpCost(op_desc=send_op_desc, - comm_context=comm_context) + send_op_desc = build_comm_desc( + "send_v2", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + send_op_cost = SendOpCost( + op_desc=send_op_desc, comm_context=comm_context + ) self.assertTrue(send_op_cost.time > 0) # Check RecvOpCost cost - recv_op_desc = build_comm_desc("recv_v2", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - recv_op_cost = RecvOpCost(op_desc=recv_op_desc, - comm_context=comm_context) + recv_op_desc = build_comm_desc( + "recv_v2", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + recv_op_cost = RecvOpCost( + op_desc=recv_op_desc, comm_context=comm_context + ) self.assertTrue(recv_op_cost.time > 0) # Check IdentityOpCost cost - identity_op_desc = build_comm_desc("c_identity", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - identity_op_cost = IdentityOpCost(op_desc=identity_op_desc, - comm_context=comm_context) + identity_op_desc = build_comm_desc( + "c_identity", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + identity_op_cost = IdentityOpCost( + op_desc=identity_op_desc, comm_context=comm_context + ) self.assertTrue(identity_op_cost.time >= 0) def test_cross_machine_comm_cost(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster1.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster1.json" + ) cluster_json_object = json.loads(multi_cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -118,40 +136,53 @@ class TestCommOpCost(unittest.TestCase): allreduce_sum_op_desc = build_comm_desc( "c_allreduce_sum", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - paddle.float32, [1, 32 * (10**6)]) + paddle.float32, + [1, 32 * (10**6)], + ) allreduce_sum_op_cost = AllreduceSumOpCost( - op_desc=allreduce_sum_op_desc, comm_context=comm_context) + op_desc=allreduce_sum_op_desc, comm_context=comm_context + ) # Check AllgatherOpCost cost allgather_op_desc = build_comm_desc( "c_allgather", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - paddle.float32, [1, 32 * (10**6)]) - allgather_op_cost = AllgatherOpCost(op_desc=allgather_op_desc, - comm_context=comm_context) + paddle.float32, + [1, 32 * (10**6)], + ) + allgather_op_cost = AllgatherOpCost( + op_desc=allgather_op_desc, comm_context=comm_context + ) self.assertTrue(allgather_op_cost.time > 0) # Check BroadcastOpCost cost broadcast_op_desc = build_comm_desc( "c_broadcast", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - paddle.float32, [1, 32 * (10**6)]) - broadcast_op_cost = BroadcastOpCost(op_desc=broadcast_op_desc, - comm_context=comm_context) + paddle.float32, + [1, 32 * (10**6)], + ) + broadcast_op_cost = BroadcastOpCost( + op_desc=broadcast_op_desc, comm_context=comm_context + ) self.assertTrue(broadcast_op_cost.time > 0) # Check SendOpCost cost - send_op_desc = build_comm_desc("send_v2", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - send_op_cost = SendOpCost(op_desc=send_op_desc, - comm_context=comm_context) + send_op_desc = build_comm_desc( + "send_v2", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + send_op_cost = SendOpCost( + op_desc=send_op_desc, comm_context=comm_context + ) self.assertTrue(send_op_cost.time > 0) # Check RecvOpCost cost - recv_op_desc = build_comm_desc("recv_v2", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - recv_op_cost = RecvOpCost(op_desc=recv_op_desc, - comm_context=comm_context) + recv_op_desc = build_comm_desc( + "recv_v2", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + recv_op_cost = RecvOpCost( + op_desc=recv_op_desc, comm_context=comm_context + ) self.assertTrue(recv_op_cost.time > 0) # Remove unnecessary files diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_comp_cost.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_comp_cost.py index f57b7d86311..8520654f37e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_comp_cost.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_comp_cost.py @@ -20,56 +20,100 @@ from paddle.distributed.auto_parallel.cluster import Cluster from paddle.distributed.auto_parallel.cost.comp_op_cost import AssignOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import AssignValueOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import BeamSearchOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import BeamSearchDecodeOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + BeamSearchDecodeOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import CastOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import ConcatOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseAddOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseAddGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseDivOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseDivGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseMulOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseMulGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseSubOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseAddOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseAddGradOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseDivOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseDivGradOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseMulOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseMulGradOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseSubOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import EmbeddingOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import EmbeddingGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import FillConstantOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import FillConstantBatchSizeLikeOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + EmbeddingGradOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + FillConstantOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + FillConstantBatchSizeLikeOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import GatherOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import GeluOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import GeluGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import GreaterEqualOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + GreaterEqualOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import IncrementOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import IsEmptyOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LayerNormOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import LayerNormGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + LayerNormGradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import LessThanOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LogicalNotOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LogicalAndOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LodResetOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LogOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import LookupTableV2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import LookupTableV2GradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + LookupTableV2OpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + LookupTableV2GradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import MatmulOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import MatmulV2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import MatmulV2GradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + MatmulV2GradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import MemcpyOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import MulOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import MulGradOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import OneHotOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ReadFromArrayOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ReadFromArrayOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import ReduceSumOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ReduceSumGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ReduceSumGradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import Reshape2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import Reshape2GradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + Reshape2GradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import ReduceMeanOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ReduceMeanGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ReduceMeanGradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import SamplingIdOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import ScaleOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SliceOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SoftmaxOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SoftmaxGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import SoftmaxWithCrossEntropyOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import SoftmaxWithCrossEntropyGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + SoftmaxWithCrossEntropyOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + SoftmaxWithCrossEntropyGradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import SplitOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import Squeeze2OpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SquareOpCost @@ -77,18 +121,25 @@ from paddle.distributed.auto_parallel.cost.comp_op_cost import SquareGradOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SumOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import TopKOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import Transpose2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import Transpose2GradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + Transpose2GradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import Unsqueeze2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import WriteToArrayOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + WriteToArrayOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import DropoutGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import FusedSoftmaxMaskUpperTriangleOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import FusedSoftmaxMaskUpperTriangleGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + FusedSoftmaxMaskUpperTriangleOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + FusedSoftmaxMaskUpperTriangleGradOpCost, +) from test_cluster import cluster_json class TestCompOpCost(unittest.TestCase): - def test_comp_cost(self): # Build cluster file_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_conditional_block_reshard.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_conditional_block_reshard.py index 86371cbae64..e7ecdece944 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_conditional_block_reshard.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_conditional_block_reshard.py @@ -23,36 +23,44 @@ from paddle.distributed.fleet import auto class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=64, - intermediate_size=4 * 64, - initializer_range=0.02): + def __init__( + self, hidden_size=64, intermediate_size=4 * 64, initializer_range=0.02 + ): super(MLPLayer, self).__init__() self.norm = nn.LayerNorm(hidden_size, epsilon=1e-5) self.linear0 = nn.Linear( hidden_size, intermediate_size, - paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range)), - bias_attr=None) + paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ) + ), + bias_attr=None, + ) self.linear1 = nn.Linear( intermediate_size, hidden_size, - paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range)), - bias_attr=None) + paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ) + ), + bias_attr=None, + ) def forward(self, input): out = self.norm(input) - auto.shard_tensor(self.linear0.weight, auto.ProcessMesh([0, 1], "x"), - [None, "x"]) + auto.shard_tensor( + self.linear0.weight, auto.ProcessMesh([0, 1], "x"), [None, "x"] + ) out = self.linear0(out) out = F.gelu(out, approximate=True) - auto.shard_tensor(self.linear1.weight, auto.ProcessMesh([0, 1], "x"), - ["x", None]) + auto.shard_tensor( + self.linear1.weight, auto.ProcessMesh([0, 1], "x"), ["x", None] + ) out = self.linear1(out) if paddle.mean(out) < 2: @@ -75,7 +83,6 @@ def loss_fn(predict, label): class TestSubblock(unittest.TestCase): - def test_subblock(self): mlp = MLPLayer() @@ -87,9 +94,9 @@ class TestSubblock(unittest.TestCase): input_sepc = InputSpec([4, 64], 'float32', 'input') label_spec = InputSpec([4, 1], 'float32', 'label') - engine.prepare(inputs_spec=[input_sepc], - labels_spec=[label_spec], - mode="predict") + engine.prepare( + inputs_spec=[input_sepc], labels_spec=[label_spec], mode="predict" + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_converter.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_converter.py index 7c838513d79..4b8b964ed43 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_converter.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_converter.py @@ -21,7 +21,6 @@ from paddle.distributed.auto_parallel.converter import Converter class TestConverter(unittest.TestCase): - def test_converter(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "converter.py") @@ -32,10 +31,19 @@ class TestConverter(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() @@ -57,7 +65,7 @@ class TestConverter(unittest.TestCase): 'tmp_0': { "process_shape": [1], "process_group": [0], - "dims_mapping": [-1] + "dims_mapping": [-1], } } with self.assertRaises(TypeError): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_assign.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_assign.py index b21dd606d8c..39d146c2e7f 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_assign.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_assign.py @@ -26,8 +26,9 @@ def make_program(): x = paddle.static.data(name='x', shape=[4, 4, 8], dtype='float32') y = paddle.static.data(name='y', shape=[4, 4, 8], dtype='float32') - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["d"]), - [None, "d", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["d"]), [None, "d", None] + ) z = paddle.add(x, y) paddle.assign(x, output=z) @@ -48,14 +49,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistAssign(unittest.TestCase): - def test_dist_assign(self): dist_main_prog, dist_context = parallelizer(make_program, 0) @@ -72,9 +73,11 @@ class TestDistAssign(unittest.TestCase): dist_out = dist_context.get_dist_tensor_for_program(out_var) x_dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - x_name) + x_name + ) out_dims_mapping = dist_op.dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert x_dims_mapping == out_dims_mapping assert out_dims_mapping == dist_out.dist_attr.dims_mapping diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_attr_v2.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_attr_v2.py index 9977fb4654a..acff242fd8c 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_attr_v2.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_attr_v2.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestDistAttr(unittest.TestCase): - def test_tensor_dist_attr_ctor(self): train_program = static.Program() start_program = static.Program() @@ -40,8 +39,9 @@ class TestDistAttr(unittest.TestCase): dist_attr.dims_mapping = [0, -1] dist_attr.batch_dim = 1 dist_attr.dynamic_dims = [1, 1] - self.assertEqual(dist_attr.process_mesh, - ProcessMesh([[0, 1, 2], [3, 4, 5]])) + self.assertEqual( + dist_attr.process_mesh, ProcessMesh([[0, 1, 2], [3, 4, 5]]) + ) self.assertEqual(dist_attr.dims_mapping, [0, -1]) self.assertEqual(dist_attr.batch_dim, 1) self.assertEqual(dist_attr.dynamic_dims, [1, 1]) @@ -59,16 +59,18 @@ class TestDistAttr(unittest.TestCase): dist_attr.dims_mapping = [0, -1] dist_attr.batch_dim = 1 dist_attr.dynamic_dims = [1, 1] - self.assertEqual(input.dist_attr.process_mesh, - ProcessMesh([[0, 1, 2], [3, 4, 5]])) + self.assertEqual( + input.dist_attr.process_mesh, ProcessMesh([[0, 1, 2], [3, 4, 5]]) + ) self.assertEqual(input.dist_attr.dims_mapping, [0, -1]) self.assertEqual(input.dist_attr.batch_dim, 1) self.assertEqual(input.dist_attr.dynamic_dims, [1, 1]) self.assertTrue(input.dist_attr.verify()) input1.dist_attr = dist_attr - self.assertEqual(input1.dist_attr.process_mesh, - ProcessMesh([[0, 1, 2], [3, 4, 5]])) + self.assertEqual( + input1.dist_attr.process_mesh, ProcessMesh([[0, 1, 2], [3, 4, 5]]) + ) self.assertEqual(input1.dist_attr.dims_mapping, [0, -1]) self.assertEqual(input1.dist_attr.batch_dim, 1) self.assertEqual(input1.dist_attr.dynamic_dims, [1, 1]) @@ -100,19 +102,24 @@ class TestDistAttr(unittest.TestCase): op_dist_attr.set_output_dist_attr(output.name, output_dist_attr) self.assertEqual(op_dist_attr.process_mesh, process_mesh) self.assertEqual( - op_dist_attr.input_dist_attr(input.name).process_mesh, process_mesh) + op_dist_attr.input_dist_attr(input.name).process_mesh, process_mesh + ) self.assertEqual( - op_dist_attr.input_dist_attr(input1.name).process_mesh, - process_mesh) + op_dist_attr.input_dist_attr(input1.name).process_mesh, process_mesh + ) self.assertEqual( op_dist_attr.output_dist_attr(output.name).process_mesh, - process_mesh) + process_mesh, + ) self.assertEqual( - op_dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1]) + op_dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1] + ) self.assertEqual( - op_dist_attr.input_dist_attr(input1.name).dims_mapping, [-1, 1]) + op_dist_attr.input_dist_attr(input1.name).dims_mapping, [-1, 1] + ) self.assertEqual( - op_dist_attr.output_dist_attr(output.name).dims_mapping, [0, 1]) + op_dist_attr.output_dist_attr(output.name).dims_mapping, [0, 1] + ) self.assertTrue(op_dist_attr.verify()) self.assertTrue(str(op_dist_attr), str(op_dist_attr)) @@ -164,19 +171,23 @@ class TestDistAttr(unittest.TestCase): self.assertEqual(op.desc.dist_attr.process_mesh, process_mesh) self.assertEqual( - op.dist_attr.input_dist_attr(input.name).process_mesh, process_mesh) + op.dist_attr.input_dist_attr(input.name).process_mesh, process_mesh + ) self.assertEqual( - op.dist_attr.input_dist_attr(input1.name).process_mesh, - process_mesh) + op.dist_attr.input_dist_attr(input1.name).process_mesh, process_mesh + ) self.assertEqual( - op.dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1]) + op.dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1] + ) self.assertEqual( - op.dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1]) + op.dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1] + ) self.assertEqual( - op.desc.dist_attr.input_dist_attr(input1.name).dims_mapping, - [-1, 1]) + op.desc.dist_attr.input_dist_attr(input1.name).dims_mapping, [-1, 1] + ) self.assertEqual( - op.dist_attr.output_dist_attr(output.name).dims_mapping, [0, 1]) + op.dist_attr.output_dist_attr(output.name).dims_mapping, [0, 1] + ) self.assertTrue(op.desc.dist_attr.verify()) self.assertTrue(str(op_dist_attr), str(op_dist_attr)) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_context.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_context.py index ea24f654795..c872f9e9d0c 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_context.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_context.py @@ -32,7 +32,7 @@ hidden_size = 1024 sequence_len = 512 _g_process_mesh = [ auto.ProcessMesh([0, 1], dim_names=["x"]), - auto.ProcessMesh([2, 3], dim_names=["x"]) + auto.ProcessMesh([2, 3], dim_names=["x"]), ] @@ -43,41 +43,45 @@ def get_random_inputs_and_labels(input_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_size): batch_input, batch_label = get_random_inputs_and_labels( [batch_size, sequence_len, hidden_size], - [batch_size, sequence_len, 1]) + [batch_size, sequence_len, 1], + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): out = self.norm(input) @@ -99,78 +103,106 @@ def get_program(): start_program = static.Program() with static.program_guard(train_program, start_program): # input - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) data_holder = [input, label] # dataloader - dataloader = paddle.io.DataLoader.from_generator(feed_list=data_holder, - capacity=4 * - batch_size, - iterable=False) - dataloader.set_batch_generator(batch_generator_creator(), - places=paddle.static.cuda_places()) + dataloader = paddle.io.DataLoader.from_generator( + feed_list=data_holder, capacity=4 * batch_size, iterable=False + ) + dataloader.set_batch_generator( + batch_generator_creator(), places=paddle.static.cuda_places() + ) # data dist_attr auto.shard_tensor(input, _g_process_mesh[0], ["x", None, None]) auto.shard_tensor(label, _g_process_mesh[0], ["x", None, None]) - mlp_start = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_start = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_start(input) - mlp_mid = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_mid = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_mid(pred) - mlp_end = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_end = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_end(pred) error_cost = paddle.nn.functional.square_error_cost(pred, label) loss = paddle.mean(error_cost) - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) feed_vars = {"inputs": [input], "labels": [label]} fetch_vars = {"loss": [loss]} - return train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars + return ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) class TestDistributedContext(unittest.TestCase): - def test_backup_restore(self): - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program( + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program() + dist_context = DistributedContext( + train_program, start_program, optimizer, loss, feed_vars, fetch_vars ) - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars) dist_context.initialize() dist_context._backup(serial=True, dist=True) - dist_context._restore(serial=True, - serial_mode="to_backup", - dist=True, - dist_mode="to_backup") + dist_context._restore( + serial=True, + serial_mode="to_backup", + dist=True, + dist_mode="to_backup", + ) dist_context._backup(serial=True, dist=True) - dist_context._restore(serial=True, - serial_mode="to_original", - dist=True, - dist_mode="to_original") + dist_context._restore( + serial=True, + serial_mode="to_original", + dist=True, + dist_mode="to_original", + ) dist_context._backup(serial=True, dist=True) dist_context._restore(serial=True, dist=True, dist_mode="to_default") @@ -179,25 +211,44 @@ class TestDistributedContext(unittest.TestCase): dist_context._restore(serial=True, dist=True, dist_mode="to_nothing") def test_deepcopy(self): - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program( + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program() + dist_context = DistributedContext( + train_program, start_program, optimizer, loss, feed_vars, fetch_vars ) - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars) dist_context.initialize() copy_dist_context = copy.deepcopy(dist_context) copy_list = [ - "_original_serial_main_program", "_original_serial_startup_program", \ - "_serial_main_program", "_serial_startup_program", "_serial_graph", \ - "_dist_main_programs", "_dist_startup_programs", \ - "_serial_ordered_nodes", "_serial_ordered_tensor_nodes", \ - "_serial_ordered_op_nodes", "_original_serial_loss", \ - "_original_serial_feed_vars", "_original_serial_fetch_vars", \ - "_serial_loss", "_serial_feed_vars", "_serial_fetch_vars", "_serial_optimizer", \ - "_backup_serial_main_program_stack", "_backup_serial_startup_program_stack", \ - "_pass_context"] + "_original_serial_main_program", + "_original_serial_startup_program", + "_serial_main_program", + "_serial_startup_program", + "_serial_graph", + "_dist_main_programs", + "_dist_startup_programs", + "_serial_ordered_nodes", + "_serial_ordered_tensor_nodes", + "_serial_ordered_op_nodes", + "_original_serial_loss", + "_original_serial_feed_vars", + "_original_serial_fetch_vars", + "_serial_loss", + "_serial_feed_vars", + "_serial_fetch_vars", + "_serial_optimizer", + "_backup_serial_main_program_stack", + "_backup_serial_startup_program_stack", + "_pass_context", + ] for i in range(len(copy_list)): copy_obj = "copy_dist_context." + copy_list[i] diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_embedding.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_embedding.py index 034a2cf1892..c302cb7e817 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_embedding.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_embedding.py @@ -27,36 +27,41 @@ def make_program_lookup_table_v1_mp_dp(): block = main_program.global_block() with paddle.static.program_guard(main_program, start_program): - src_ids = paddle.static.data(name='src_ids', - shape=[12, 512, 1], - dtype='int64') + src_ids = paddle.static.data( + name='src_ids', shape=[12, 512, 1], dtype='int64' + ) src_ids.stop_gradient = True emb_out = paddle.fluid.layers.embedding( input=src_ids, size=[64, 128], param_attr=paddle.fluid.ParamAttr(name="emb_weight"), dtype="float32", - is_sparse=False) + is_sparse=False, + ) loss = paddle.fluid.layers.reduce_mean(emb_out) auto.shard_tensor( - src_ids, auto.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), - ["x", None, None]) + src_ids, + auto.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), + ["x", None, None], + ) emb_weight = block.vars["emb_weight"] auto.shard_tensor( - emb_weight, auto.ProcessMesh([[0, 1], [2, 3]], - dim_names=["x", "y"]), ["y", None]) + emb_weight, + auto.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), + ["y", None], + ) return main_program, start_program, loss class TestDistPNorm(unittest.TestCase): - def test_lookup_table_v1_mp_dp(self): for rank in range(4): dist_main_prog, dist_context = parallelizer( - make_program_lookup_table_v1_mp_dp, rank) + make_program_lookup_table_v1_mp_dp, rank + ) ops = dist_main_prog.global_block().ops op_types = [] @@ -64,9 +69,16 @@ class TestDistPNorm(unittest.TestCase): op_types.append(op.type) assert op_types == [ - 'reshape2', 'c_embedding', 'c_allreduce_sum', 'reduce_mean', - 'fill_constant', 'reduce_mean_grad', 'c_identity', - 'c_embedding_grad', 'c_allreduce_sum', 'scale' + 'reshape2', + 'c_embedding', + 'c_allreduce_sum', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'c_identity', + 'c_embedding_grad', + 'c_allreduce_sum', + 'scale', ] diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_matmul.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_matmul.py index 6ca0619800e..c53113ae76c 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_matmul.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_matmul.py @@ -84,10 +84,9 @@ def matmul_dp2mp2(init_x, init_y, trans_x, trans_y): y = init_y(trans_y) x.stop_gradient = False y.stop_gradient = False - out = paddle.fluid.layers.matmul(x, - y, - transpose_x=trans_x, - transpose_y=trans_y) + out = paddle.fluid.layers.matmul( + x, y, transpose_x=trans_x, transpose_y=trans_y + ) loss = paddle.mean(out) return main_program, start_program, loss @@ -123,19 +122,23 @@ def parallelizer(program_func, *args, **kwargs): dist_context.block_state.parse_backward_blocks(main_program) partitioner = Partitioner(dist_context, 0) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistMatmul(unittest.TestCase): - def check_col_program(self, main_program, dist_ctx): # [0, -1] * [-1, 1] --> [0, 1] ref_ops = [ - "c_identity", "matmul", "reduce_mean", "fill_constant", - "reduce_mean_grad", "matmul_grad" + "c_identity", + "matmul", + "reduce_mean", + "fill_constant", + "reduce_mean_grad", + "matmul_grad", ] ops = [] block = main_program.global_block() @@ -148,10 +151,12 @@ class TestDistMatmul(unittest.TestCase): assert op_dist_attr.impl_idx == 0 assert op_dist_attr.impl_type == "matmul" out_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert out_dims_mapping == [0, 1] tensor_dist_attr = dist_ctx.get_tensor_dist_attr_for_program( - out_var) + out_var + ) assert tensor_dist_attr.dims_mapping == [0, 1] if op.type == "matmul_grad": op_dist_attr = dist_ctx.get_op_dist_attr_for_program(op) @@ -163,8 +168,12 @@ class TestDistMatmul(unittest.TestCase): def check_row_program(self, main_program, dist_ctx): # [0, -1, 1] * [1, -1] --> [0, -1, -1] ref_ops = [ - "matmul", "c_allreduce_sum", "reduce_mean", "fill_constant", - "reduce_mean_grad", "matmul_grad" + "matmul", + "c_allreduce_sum", + "reduce_mean", + "fill_constant", + "reduce_mean_grad", + "matmul_grad", ] ops = [] block = main_program.global_block() @@ -177,10 +186,12 @@ class TestDistMatmul(unittest.TestCase): assert op_dist_attr.impl_idx == 1 assert op_dist_attr.impl_type == "matmul" out_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert out_dims_mapping == [0, -1, -1] tensor_dist_attr = dist_ctx.get_tensor_dist_attr_for_program( - out_var) + out_var + ) assert tensor_dist_attr.dims_mapping == [0, -1, -1] if op.type == "matmul_grad": op_dist_attr = dist_ctx.get_op_dist_attr_for_program(op) @@ -190,10 +201,10 @@ class TestDistMatmul(unittest.TestCase): class TestDistMatmulCol(TestDistMatmul): - def init(self, trans_x, trans_y): - dist_main_prog, dist_ctx = parallelizer(matmul_dp2mp2, init_x_col, - init_y_col, trans_x, trans_y) + dist_main_prog, dist_ctx = parallelizer( + matmul_dp2mp2, init_x_col, init_y_col, trans_x, trans_y + ) return dist_main_prog, dist_ctx def test_matmul_col(self): @@ -214,10 +225,10 @@ class TestDistMatmulCol(TestDistMatmul): class TestDistMatmulRow(TestDistMatmul): - def init(self, trans_x, trans_y): - dist_main_prog, dist_ctx = parallelizer(matmul_dp2mp2, init_x_row, - init_y_row, trans_x, trans_y) + dist_main_prog, dist_ctx = parallelizer( + matmul_dp2mp2, init_x_row, init_y_row, trans_x, trans_y + ) return dist_main_prog, dist_ctx def test_matmul_row(self): @@ -238,12 +249,15 @@ class TestDistMatmulRow(TestDistMatmul): class TestDistMatmulV2(unittest.TestCase): - def check_col_program(self, main_program, dist_ctx): # [0, -1] * [-1, 1] --> [0, 1] ref_ops = [ - "c_identity", "matmul_v2", "reduce_mean", "fill_constant", - "reduce_mean_grad", "matmul_v2_grad" + "c_identity", + "matmul_v2", + "reduce_mean", + "fill_constant", + "reduce_mean_grad", + "matmul_v2_grad", ] ops = [] block = main_program.global_block() @@ -256,10 +270,12 @@ class TestDistMatmulV2(unittest.TestCase): assert op_dist_attr.impl_idx == 0 assert op_dist_attr.impl_type == "matmul_v2" out_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert out_dims_mapping == [0, 1] tensor_dist_attr = dist_ctx.get_tensor_dist_attr_for_program( - out_var) + out_var + ) assert tensor_dist_attr.dims_mapping == [0, 1] if op.type == "matmul_v2_grad": op_dist_attr = dist_ctx.get_op_dist_attr_for_program(op) @@ -271,8 +287,12 @@ class TestDistMatmulV2(unittest.TestCase): def check_row_program(self, main_program, dist_ctx): # [0, -1, 1] * [1, -1] --> [0, -1, -1] ref_ops = [ - "matmul_v2", "c_allreduce_sum", "reduce_mean", "fill_constant", - "reduce_mean_grad", "matmul_v2_grad" + "matmul_v2", + "c_allreduce_sum", + "reduce_mean", + "fill_constant", + "reduce_mean_grad", + "matmul_v2_grad", ] ops = [] block = main_program.global_block() @@ -285,10 +305,12 @@ class TestDistMatmulV2(unittest.TestCase): assert op_dist_attr.impl_idx == 1 assert op_dist_attr.impl_type == "matmul_v2" out_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert out_dims_mapping == [0, -1, -1] tensor_dist_attr = dist_ctx.get_tensor_dist_attr_for_program( - out_var) + out_var + ) assert tensor_dist_attr.dims_mapping == [0, -1, -1] if op.type == "matmul_v2_grad": op_dist_attr = dist_ctx.get_op_dist_attr_for_program(op) @@ -298,10 +320,10 @@ class TestDistMatmulV2(unittest.TestCase): class TestDistMatmulV2Col(TestDistMatmulV2): - def init(self, trans_x, trans_y): - dist_main_prog, dist_ctx = parallelizer(matmulv2_dp2mp2, init_x_col, - init_y_col, trans_x, trans_y) + dist_main_prog, dist_ctx = parallelizer( + matmulv2_dp2mp2, init_x_col, init_y_col, trans_x, trans_y + ) return dist_main_prog, dist_ctx def test_matmul_col(self): @@ -322,10 +344,10 @@ class TestDistMatmulV2Col(TestDistMatmulV2): class TestDistMatmulV2Row(TestDistMatmulV2): - def init(self, trans_x, trans_y): - dist_main_prog, dist_ctx = parallelizer(matmulv2_dp2mp2, init_x_row, - init_y_row, trans_x, trans_y) + dist_main_prog, dist_ctx = parallelizer( + matmulv2_dp2mp2, init_x_row, init_y_row, trans_x, trans_y + ) return dist_main_prog, dist_ctx def test_matmul_row(self): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_op_cost.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_op_cost.py index d7bdbfd7774..517debaa588 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_op_cost.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_op_cost.py @@ -18,7 +18,10 @@ import copy import paddle from paddle.distributed.fleet import auto from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container, is_elementwise_op +from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, + is_elementwise_op, +) from paddle.fluid import program_guard from paddle.fluid.backward import append_backward @@ -42,7 +45,8 @@ def parallelizer(program_func, rank): # generate backward and complete backward with paddle.static.program_guard(main_program, startup_program): params_grads = append_backward( - loss, None, None, None, distop_context=dist_context.dist_op_context) + loss, None, None, None, distop_context=dist_context.dist_op_context + ) completer.complete_backward_annotation(main_program) dist_context.block_state.parse_backward_blocks(main_program) @@ -57,23 +61,23 @@ def parallelizer(program_func, rank): class TestDistOpCost(unittest.TestCase): - def test_dist_op_cost_part1(self): - def make_program(): main_program = paddle.static.Program() start_program = paddle.static.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 8], dtype='float32') x.stop_gradient = True - label = paddle.static.data(name="label", - shape=[4, 1], - dtype='float32') + label = paddle.static.data( + name="label", shape=[4, 1], dtype='float32' + ) label.stop_gradient = True - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None] + ) tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=x, shape=[2, 8], value=1, dtype='float32') + input=x, shape=[2, 8], value=1, dtype='float32' + ) weight_attr = paddle.ParamAttr() linear = paddle.nn.Linear(8, 1, weight_attr=weight_attr) linear_out = linear(x) @@ -89,43 +93,53 @@ class TestDistOpCost(unittest.TestCase): cluster = Cluster() cluster.gen_default_config_cluster(device_count=2) for idx, op in enumerate(ops): - if op.type != "matmul_v2" and op.type != "matmul_v2_grad" and op.type != "sgd": + if ( + op.type != "matmul_v2" + and op.type != "matmul_v2_grad" + and op.type != "sgd" + ): dist_op = dist_context.get_dist_op_for_program(op) op_dist_attr = dist_op.dist_attr processes = op_dist_attr.process_mesh.processes if is_elementwise_op(op.type): container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) else: container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, cluster + ) self.assertTrue(dist_op_cost) def test_dist_op_cost_part2(self): - def make_program(): main_program = paddle.static.Program() start_program = paddle.static.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4], dtype='float32') x.stop_gradient = True - label = paddle.static.data(name="label", - shape=[8, 1], - dtype='float32') + label = paddle.static.data( + name="label", shape=[8, 1], dtype='float32' + ) label.stop_gradient = True - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x"]) - - auto.shard_tensor(label, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x"] + ) + + auto.shard_tensor( + label, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) # embedding tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=x, shape=[4], value=1, dtype='int32') + input=x, shape=[4], value=1, dtype='int32' + ) embedding = paddle.nn.Embedding(10, 8) out = embedding(tmp) # row parallel embedding @@ -133,47 +147,64 @@ class TestDistOpCost(unittest.TestCase): if op.type == "lookup_table_v2": W = main_program.global_block().vars[op.input("W")[0]] auto.shard_tensor( - W, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) - out = paddle.fluid.layers.transpose(out, - [1, 0]) # [8, 2] [-1, 0] + W, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) + out = paddle.fluid.layers.transpose( + out, [1, 0] + ) # [8, 2] [-1, 0] # matmul param1 = paddle.fluid.layers.create_parameter( - [4, 8], paddle.float32) # [2, 8] [0, -1] - auto.shard_tensor(param1, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + [4, 8], paddle.float32 + ) # [2, 8] [0, -1] + auto.shard_tensor( + param1, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) param2 = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 4] [-1, 0] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, "x"]) - out1 = paddle.fluid.layers.matmul(out, - param1) # [8, 8] [-1, -1] + [8, 8], paddle.float32 + ) # [8, 4] [-1, 0] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, "x"], + ) + out1 = paddle.fluid.layers.matmul( + out, param1 + ) # [8, 8] [-1, -1] tmp_param = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 8] [-1, -1] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, None]) + [8, 8], paddle.float32 + ) # [8, 8] [-1, -1] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, None], + ) tmp_out = paddle.fluid.layers.matmul(out1, tmp_param) - out2 = paddle.fluid.layers.matmul(tmp_out, - param2) # [8, 4] [-1, 0] + out2 = paddle.fluid.layers.matmul( + tmp_out, param2 + ) # [8, 4] [-1, 0] - out8 = paddle.fluid.layers.transpose(out2, - [1, 0]) # [4, 8] [0, -1] + out8 = paddle.fluid.layers.transpose( + out2, [1, 0] + ) # [4, 8] [0, -1] # reshape out9 = paddle.reshape(out8, [8, 2, 4]) # [4, 2, 4] [0, -1, -1] tmp_reshape_out = paddle.reshape(out9, [8, 4, 2]) - out10 = paddle.reshape(tmp_reshape_out, - [8, 8]) # [4, 8] [0, -1] + out10 = paddle.reshape( + tmp_reshape_out, [8, 8] + ) # [4, 8] [0, -1] # softmax softmax = paddle.nn.Softmax() out11 = softmax(out10) error_cost = paddle.nn.functional.square_error_cost( - out11, label) + out11, label + ) loss = paddle.mean(error_cost) return main_program, start_program, loss @@ -187,37 +218,43 @@ class TestDistOpCost(unittest.TestCase): processes = op_dist_attr.process_mesh.processes if is_elementwise_op(op.type): container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) else: container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, cluster + ) self.assertTrue(dist_op_cost) def test_dist_op_cost_part3(self): - def make_program(): main_program = paddle.static.Program() start_program = paddle.static.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4], dtype='float32') x.stop_gradient = True - label = paddle.static.data(name="label", - shape=[8, 1], - dtype='float32') + label = paddle.static.data( + name="label", shape=[8, 1], dtype='float32' + ) label.stop_gradient = True - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x"]) - - auto.shard_tensor(label, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x"] + ) + + auto.shard_tensor( + label, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) # embedding tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=x, shape=[4], value=1, dtype='int32') + input=x, shape=[4], value=1, dtype='int32' + ) embedding = paddle.nn.Embedding(10, 8) out = embedding(tmp) # row parallel embedding @@ -225,46 +262,61 @@ class TestDistOpCost(unittest.TestCase): if op.type == "lookup_table_v2": W = main_program.global_block().vars[op.input("W")[0]] auto.shard_tensor( - W, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) - out = paddle.fluid.layers.transpose(out, - [1, 0]) # [8, 2] [-1, 0] + W, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) + out = paddle.fluid.layers.transpose( + out, [1, 0] + ) # [8, 2] [-1, 0] # matmul_v2 param1 = paddle.fluid.layers.create_parameter( - [4, 8], paddle.float32) # [2, 8] [0, -1] - auto.shard_tensor(param1, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + [4, 8], paddle.float32 + ) # [2, 8] [0, -1] + auto.shard_tensor( + param1, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) param2 = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 4] [-1, 0] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, "x"]) + [8, 8], paddle.float32 + ) # [8, 4] [-1, 0] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, "x"], + ) out1 = paddle.matmul(out, param1) # [8, 8] [-1, -1] tmp_param = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 8] [-1, -1] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, None]) + [8, 8], paddle.float32 + ) # [8, 8] [-1, -1] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, None], + ) tmp_out = paddle.matmul(out1, tmp_param) out2 = paddle.matmul(tmp_out, param2) # [8, 4] [-1, 0] - out8 = paddle.fluid.layers.transpose(out2, - [1, 0]) # [4, 8] [0, -1] + out8 = paddle.fluid.layers.transpose( + out2, [1, 0] + ) # [4, 8] [0, -1] # reshape out9 = paddle.reshape(out8, [8, 2, 4]) # [4, 2, 4] [0, -1, -1] tmp_reshape_out = paddle.reshape(out9, [8, 4, 2]) - out10 = paddle.reshape(tmp_reshape_out, - [8, 8]) # [4, 8] [0, -1] + out10 = paddle.reshape( + tmp_reshape_out, [8, 8] + ) # [4, 8] [0, -1] # softmax softmax = paddle.nn.Softmax() out11 = softmax(out10) error_cost = paddle.nn.functional.square_error_cost( - out11, label) + out11, label + ) loss = paddle.mean(error_cost) return main_program, start_program, loss @@ -278,36 +330,42 @@ class TestDistOpCost(unittest.TestCase): processes = op_dist_attr.process_mesh.processes if is_elementwise_op(op.type): container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) else: container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, cluster + ) self.assertTrue(dist_op_cost) def test_dist_op_cost_part4(self): - def make_program(): main_program = paddle.static.Program() start_program = paddle.static.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4], dtype='float32') x.stop_gradient = True - label = paddle.static.data(name="label", - shape=[8, 1], - dtype='float32') + label = paddle.static.data( + name="label", shape=[8, 1], dtype='float32' + ) label.stop_gradient = True - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x"]) - auto.shard_tensor(label, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x"] + ) + auto.shard_tensor( + label, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) # embedding tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=x, shape=[4], value=1, dtype='int32') + input=x, shape=[4], value=1, dtype='int32' + ) embedding = paddle.nn.Embedding(10, 8) out = embedding(tmp) # row parallel embedding @@ -315,48 +373,64 @@ class TestDistOpCost(unittest.TestCase): if op.type == "lookup_table_v2": W = main_program.global_block().vars[op.input("W")[0]] auto.shard_tensor( - W, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) - out = paddle.fluid.layers.transpose(out, - [1, 0]) # [8, 2] [-1, 0] + W, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) + out = paddle.fluid.layers.transpose( + out, [1, 0] + ) # [8, 2] [-1, 0] # mul param1 = paddle.fluid.layers.create_parameter( - [4, 8], paddle.float32) # [2, 8] [0, -1] - auto.shard_tensor(param1, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + [4, 8], paddle.float32 + ) # [2, 8] [0, -1] + auto.shard_tensor( + param1, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) param2 = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 4] [-1, 0] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, "x"]) + [8, 8], paddle.float32 + ) # [8, 4] [-1, 0] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, "x"], + ) out1 = paddle.fluid.layers.mul(out, param1) # [8, 8] [-1, -1] tmp_param = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 8] [-1, -1] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, None]) + [8, 8], paddle.float32 + ) # [8, 8] [-1, -1] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, None], + ) tmp_out = paddle.fluid.layers.mul(out1, tmp_param) - out2 = paddle.fluid.layers.mul(tmp_out, - param2) # [8, 4] [-1, 0] + out2 = paddle.fluid.layers.mul( + tmp_out, param2 + ) # [8, 4] [-1, 0] - out8 = paddle.fluid.layers.transpose(out2, - [1, 0]) # [4, 8] [0, -1] + out8 = paddle.fluid.layers.transpose( + out2, [1, 0] + ) # [4, 8] [0, -1] # reshape out9 = paddle.reshape(out8, [8, 2, 4]) # [4, 2, 4] [0, -1, -1] tmp_reshape_out = paddle.reshape(out9, [8, 4, 2]) - out10 = paddle.reshape(tmp_reshape_out, - [8, 8]) # [4, 8] [0, -1] + out10 = paddle.reshape( + tmp_reshape_out, [8, 8] + ) # [4, 8] [0, -1] # softmax softmax = paddle.nn.Softmax() out11 = softmax(out10) error_cost = paddle.nn.functional.square_error_cost( - out11, label) + out11, label + ) loss = paddle.mean(error_cost) return main_program, start_program, loss @@ -370,14 +444,17 @@ class TestDistOpCost(unittest.TestCase): processes = op_dist_attr.process_mesh.processes if is_elementwise_op(op.type): container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) else: container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, cluster + ) self.assertTrue(dist_op_cost) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_pnorm.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_pnorm.py index 9f54f2a774a..66ce40e4e33 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_pnorm.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_pnorm.py @@ -28,8 +28,9 @@ def make_program_dp2(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 5, 6], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) tmp_0 = paddle.norm(x, p=2) return main_program, start_program, tmp_0 @@ -40,8 +41,9 @@ def make_program_serial(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 5, 6], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0], dim_names=["x"]), - [None, None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0], dim_names=["x"]), [None, None, None] + ) tmp_0 = paddle.norm(x, p=2) return main_program, start_program, tmp_0 @@ -60,19 +62,20 @@ def parallelizer(program_func, rank): with program_guard(main_program, start_program): params_grads = append_backward( - loss, distop_context=dist_context.dist_op_context) + loss, distop_context=dist_context.dist_op_context + ) completer.complete_backward_annotation(main_program) dist_context.block_state.parse_backward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistPNorm(unittest.TestCase): - def test_dist_pnorm_dp2(self): for rank in range(2): @@ -102,7 +105,11 @@ class TestDistPNorm(unittest.TestCase): assert output_attr.dims_mapping[0] == 0 assert set(output_attr.dims_mapping[1:]) == set([-1]) assert op_types == [ - "c_allgather", "p_norm", "fill_constant", "p_norm_grad", "slice" + "c_allgather", + "p_norm", + "fill_constant", + "p_norm_grad", + "slice", ] def test_dist_pnorm_serial(self): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_reshape.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_reshape.py index 02797d690f2..035fd235624 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_reshape.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_reshape.py @@ -25,8 +25,9 @@ def make_program_dp2(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 4, 8], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) tmp_0 = paddle.reshape(x, shape=[0, 0, 4, 2]) tmp_1 = paddle.reshape(tmp_0, shape=[0, 0, 8]) @@ -47,14 +48,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistReshape(unittest.TestCase): - def test_dist_reshape_mp2(self): for rank in range(2): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_shape.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_shape.py index 5e18b7d90c5..767a4cbccb2 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_shape.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_shape.py @@ -25,8 +25,9 @@ def make_program(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 4, 8], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) shape = paddle.shape(x) return main_program, start_program @@ -44,14 +45,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistShape(unittest.TestCase): - def test_dist_shape(self): dist_main_prog, dist_context = parallelizer(make_program, 0) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_slice.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_slice.py index 89261e6c9bf..7e4d5eaee90 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_slice.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_slice.py @@ -24,8 +24,9 @@ def make_program_dp2(): start_program = paddle.fluid.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 5, 6], dtype='float32') - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) tmp_0 = x[0] tmp_1 = x[:, 0, :] @@ -39,8 +40,9 @@ def make_program_serial(): start_program = paddle.fluid.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 5, 6], dtype='float32') - auto.shard_tensor(x, auto.ProcessMesh([0], dim_names=["x"]), - [None, None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0], dim_names=["x"]), [None, None, None] + ) tmp_0 = x[0] tmp_1 = x[:, 0, :] @@ -64,14 +66,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistSlice(unittest.TestCase): - def test_dist_slice_dp2(self): for rank in range(2): dist_main_prog, dist_context = parallelizer(make_program_dp2, rank) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_split.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_split.py index 7d7c60f4bdd..18e79b8cf95 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_split.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_split.py @@ -25,8 +25,9 @@ def make_program_dp2(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 12, 16], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) out0, out1, out2 = paddle.split(x, num_or_sections=3, axis=1) return main_program, start_program @@ -44,14 +45,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistSplit(unittest.TestCase): - def test_dist_split_dp2(self): for rank in range(2): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api.py index c8aba02719c..880f0e582bb 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api.py @@ -20,7 +20,6 @@ import subprocess class TestEngineAPI(unittest.TestCase): - def test_engine_api(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "engine_api.py") @@ -31,10 +30,19 @@ class TestEngineAPI(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api_dp.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api_dp.py index fab8db0182e..b3dad0551c2 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api_dp.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api_dp.py @@ -20,7 +20,6 @@ import subprocess class TestEngineAPI(unittest.TestCase): - def test_engine_api(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "engine_api_dp.py") @@ -31,10 +30,19 @@ class TestEngineAPI(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_callbacks.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_callbacks.py index 9baaee353f7..5c7b2f1d177 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_callbacks.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_callbacks.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestCallbacks(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() @@ -51,14 +50,16 @@ class TestCallbacks(unittest.TestCase): engine = auto.Engine(LeNet(), strategy=strategy) engine.prepare(inputs_spec, mode="predict") - cbks = config_callbacks(engine=engine, - batch_size=128, - epochs=epochs, - steps=steps, - log_freq=freq, - verbose=self.verbose, - metrics=['loss', 'acc'], - save_dir=self.save_dir) + cbks = config_callbacks( + engine=engine, + batch_size=128, + epochs=epochs, + steps=steps, + log_freq=freq, + verbose=self.verbose, + metrics=['loss', 'acc'], + save_dir=self.save_dir, + ) cbks.on_begin('train') logs = {'loss': 50.341673, 'acc': 0.00256} @@ -119,7 +120,6 @@ class TestCallbacks(unittest.TestCase): class TestCallbacksEngine(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) @@ -136,37 +136,39 @@ class TestCallbacksEngine(unittest.TestCase): base_lr = 1e-3 boundaries = [5, 8] values = [base_lr * (0.1**i) for i in range(len(boundaries) + 1)] - lr = paddle.optimizer.lr.PiecewiseDecay(boundaries=boundaries, - values=values, - verbose=False) - optimizer = paddle.optimizer.Adam(learning_rate=lr, - parameters=model.parameters()) + lr = paddle.optimizer.lr.PiecewiseDecay( + boundaries=boundaries, values=values, verbose=False + ) + optimizer = paddle.optimizer.Adam( + learning_rate=lr, parameters=model.parameters() + ) auto.fetch(model.parameters()[0], "param0", logging=True) metrics = paddle.metric.Accuracy(topk=(1, 2)) self.engine = auto.Engine(model, loss, optimizer, metrics) def test_fit_eval(self): - history = self.engine.fit(train_data=self.train_dataset, - valid_data=self.test_dataset, - batch_size=128, - steps_per_epoch=60, - valid_steps=40, - log_freq=20, - save_dir=self.save_dir, - save_freq=1) + history = self.engine.fit( + train_data=self.train_dataset, + valid_data=self.test_dataset, + batch_size=128, + steps_per_epoch=60, + valid_steps=40, + log_freq=20, + save_dir=self.save_dir, + save_freq=1, + ) print(history.history) def test_eval(self): - self.engine.evaluate(valid_data=self.test_dataset, - batch_size=128, - steps=40, - log_freq=10) + self.engine.evaluate( + valid_data=self.test_dataset, batch_size=128, steps=40, log_freq=10 + ) def test_predict(self): logger_cbks = paddle.callbacks.ProgBarLogger() - self.engine.predict(test_data=self.test_dataset, - batch_size=128, - callbacks=[logger_cbks]) + self.engine.predict( + test_data=self.test_dataset, batch_size=128, callbacks=[logger_cbks] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_high_order_grad.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_high_order_grad.py index 7c3629182fa..7e2728e8e97 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_high_order_grad.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_high_order_grad.py @@ -20,7 +20,6 @@ import subprocess class TestHighOrderGrad(unittest.TestCase): - def test_dp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "high_order_grad.py") @@ -31,10 +30,19 @@ class TestHighOrderGrad(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_interface.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_interface.py index a36c142dc86..332cb134a54 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_interface.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_interface.py @@ -18,7 +18,9 @@ import paddle.nn as nn import paddle.nn.functional as F import paddle.static as static from paddle.distributed.fleet import auto -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) from paddle.distributed.auto_parallel.process_mesh import ProcessMesh paddle.enable_static() @@ -27,71 +29,83 @@ batch_size = 4 epoch_num = 10 hidden_size = 1024 sequence_len = 512 -process_mesh1 = ProcessMesh(mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], - dim_names=["x", "y"]) +process_mesh1 = ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["x", "y"] +) process_mesh2 = ProcessMesh(mesh=[0, 1, 2, 3], dim_names=["x"]) class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): auto.shard_tensor(self.linear0.weight, process_mesh1[0], [None, "y"]) - linear0 = auto.shard_op(self.linear0, process_mesh1, - [["y", None, None]], [[None, "x", None]]) + linear0 = auto.shard_op( + self.linear0, + process_mesh1, + [["y", None, None]], + [[None, "x", None]], + ) linear0_out = linear0(input) gelu = auto.shard_op(F.gelu, process_mesh1, [["y", "x", None], None]) gelu_out = gelu(linear0_out, approximate=True) auto.shard_tensor(self.linear1.weight, shard_spec=["y", None]) - linear1 = auto.shard_op(self.linear1, - process_mesh1[1], - out_shard_specs=[["y", None, None]]) + linear1 = auto.shard_op( + self.linear1, process_mesh1[1], out_shard_specs=[["y", None, None]] + ) linear1_out = linear1(gelu_out) return self.linear0, self.linear1, linear0_out, gelu_out, linear1_out class TestAutoParallelAPI(unittest.TestCase): - def test_api(self): # input - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) auto.shard_tensor(input, process_mesh1, ["x", None, None]) auto.shard_tensor(label, process_mesh1, ["y", None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) with ProcessMesh(process_mesh1.mesh, process_mesh1.dim_names): linear0, linear1, linear0_out, gelu_out, linear1_out = mlp(input) @@ -124,32 +138,42 @@ class TestAutoParallelAPI(unittest.TestCase): self.assertTrue(dist_input.dist_attr.is_annotated("dims_mapping")) dist_linear0_weight = default_dist_context.get_dist_tensor_for_program( - linear0.weight) - self.assertEqual(dist_linear0_weight.dist_attr.process_mesh, - process_mesh1[0]) + linear0.weight + ) + self.assertEqual( + dist_linear0_weight.dist_attr.process_mesh, process_mesh1[0] + ) self.assertEqual(dist_linear0_weight.dist_attr.dims_mapping, [-1, 0]) self.assertTrue( - dist_linear0_weight.dist_attr.is_annotated("process_mesh")) + dist_linear0_weight.dist_attr.is_annotated("process_mesh") + ) self.assertTrue( - dist_linear0_weight.dist_attr.is_annotated("dims_mapping")) + dist_linear0_weight.dist_attr.is_annotated("dims_mapping") + ) dist_linear1_weight = default_dist_context.get_dist_tensor_for_program( - linear1.weight) - self.assertEqual(dist_linear1_weight.dist_attr.process_mesh, - process_mesh1) + linear1.weight + ) + self.assertEqual( + dist_linear1_weight.dist_attr.process_mesh, process_mesh1 + ) self.assertEqual(dist_linear1_weight.dist_attr.dims_mapping, [1, -1]) self.assertTrue( - dist_linear1_weight.dist_attr.is_annotated("process_mesh")) + dist_linear1_weight.dist_attr.is_annotated("process_mesh") + ) self.assertTrue( - dist_linear1_weight.dist_attr.is_annotated("dims_mapping")) + dist_linear1_weight.dist_attr.is_annotated("dims_mapping") + ) dist_linear1_out = default_dist_context.get_dist_tensor_for_program( - linear1_out) + linear1_out + ) self.assertEqual(dist_linear1_out.dist_attr.process_mesh, process_mesh1) self.assertEqual(dist_linear1_out.dist_attr.dims_mapping, [-1, -1, -1]) self.assertTrue(dist_linear1_out.dist_attr.is_annotated("process_mesh")) self.assertFalse( - dist_linear1_out.dist_attr.is_annotated("dims_mapping")) + dist_linear1_out.dist_attr.is_annotated("dims_mapping") + ) dist_op = default_dist_context.get_dist_op_for_program(matmul0) self.assertEqual(dist_op.dist_attr.process_mesh, process_mesh1) @@ -167,7 +191,8 @@ class TestAutoParallelAPI(unittest.TestCase): self.assertEqual(dist_op.dist_attr.impl_type, "default") self.assertEqual(dist_op.dist_attr.impl_idx, 0) tensor_dist_attr = dist_op.dist_attr.get_output_dist_attr( - linear0_out.name) + linear0_out.name + ) self.assertEqual(tensor_dist_attr.process_mesh, process_mesh1) self.assertEqual(tensor_dist_attr.dims_mapping, [-1, 0, -1]) self.assertTrue(tensor_dist_attr.is_annotated("process_mesh")) @@ -180,7 +205,8 @@ class TestAutoParallelAPI(unittest.TestCase): self.assertEqual(dist_op.dist_attr.impl_idx, 0) self.assertTrue(dist_op.dist_attr.is_annotated("process_mesh")) tensor_dist_attr = dist_op.dist_attr.get_input_dist_attr( - linear0_out.name) + linear0_out.name + ) self.assertEqual(tensor_dist_attr.process_mesh, process_mesh1) self.assertEqual(tensor_dist_attr.dims_mapping, [1, 0, -1]) self.assertTrue(tensor_dist_attr.is_annotated("process_mesh")) @@ -208,7 +234,8 @@ class TestAutoParallelAPI(unittest.TestCase): self.assertEqual(dist_op.dist_attr.impl_idx, 0) self.assertTrue(dist_op.dist_attr.is_annotated("process_mesh")) tensor_dist_attr = dist_op.dist_attr.get_output_dist_attr( - linear1_out.name) + linear1_out.name + ) self.assertEqual(tensor_dist_attr.process_mesh, process_mesh1[1]) self.assertEqual(tensor_dist_attr.dims_mapping, [0, -1, -1]) self.assertTrue(tensor_dist_attr.is_annotated("process_mesh")) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_iterable_dataset.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_iterable_dataset.py index 8a7d9d57602..f431878ae20 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_iterable_dataset.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_iterable_dataset.py @@ -20,7 +20,6 @@ import subprocess class TestEngineAPI(unittest.TestCase): - def test_engine_api(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "iterable_dataset.py") @@ -31,10 +30,19 @@ class TestEngineAPI(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_lr_grad_clip.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_lr_grad_clip.py index fc5240a369d..49ff195a50f 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_lr_grad_clip.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_lr_grad_clip.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestEngineBase(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.batch_num = 5 @@ -35,15 +34,18 @@ class TestEngineBase(unittest.TestCase): self.init_engine() def init_model(self): - self.mlp = MLPLayer(hidden_size=self.hidden_size, - intermediate_size=4 * self.hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + self.mlp = MLPLayer( + hidden_size=self.hidden_size, + intermediate_size=4 * self.hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) self.loss = paddle.nn.CrossEntropyLoss() def init_optimizer(self): - self.optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - parameters=self.mlp.parameters()) + self.optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, parameters=self.mlp.parameters() + ) def init_dataset(self): self.dataset = MyDataset(self.batch_num * self.batch_size) @@ -52,17 +54,19 @@ class TestEngineBase(unittest.TestCase): # inputs = InputSpec([self.batch_size, self.hidden_size], 'float32', 'x') # labels = InputSpec([self.batch_size], 'int64', 'label') - self.engine = auto.Engine(model=self.mlp, - loss=self.loss, - optimizer=self.optimizer, - metrics=paddle.metric.Accuracy()) + self.engine = auto.Engine( + model=self.mlp, + loss=self.loss, + optimizer=self.optimizer, + metrics=paddle.metric.Accuracy(), + ) class TestLRScheduler(TestEngineBase): - def init_optimizer(self): scheduler = paddle.optimizer.lr.CosineAnnealingDecay( - learning_rate=0.00001, T_max=10) + learning_rate=0.00001, T_max=10 + ) self.optimizer = paddle.optimizer.SGD(learning_rate=scheduler) def test_lr_scheduler(self): @@ -73,11 +77,11 @@ class TestLRScheduler(TestEngineBase): class TestGradClipByGlobalNorm(TestEngineBase): - def init_optimizer(self): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - self.optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - grad_clip=clip) + self.optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, grad_clip=clip + ) def test_grad_clip(self): @@ -89,19 +93,20 @@ class TestGradClipByGlobalNorm(TestEngineBase): ops = self.engine.main_program.global_block().ops has_grad_clip = False for op in ops: - if op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip"): + if op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip"): has_grad_clip = True break assert has_grad_clip is True class TestGradClipByNorm(TestGradClipByGlobalNorm): - def init_optimizer(self): clip = paddle.nn.ClipGradByNorm(clip_norm=1.0) - self.optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - grad_clip=clip) + self.optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, grad_clip=clip + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_new_cost_model.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_new_cost_model.py index 59db55596af..963d16d239e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_new_cost_model.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_new_cost_model.py @@ -20,9 +20,15 @@ import tempfile import paddle import paddle.distributed.auto_parallel.cost as cost_model -from paddle.distributed.auto_parallel.cost.base_cost import build_comp_desc_from_op -from paddle.distributed.auto_parallel.cost.base_cost import build_comp_desc_str_for_predict -from paddle.distributed.auto_parallel.cost.base_cost import calc_time_by_modeling +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comp_desc_from_op, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comp_desc_str_for_predict, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + calc_time_by_modeling, +) from paddle.distributed.auto_parallel.cluster import Cluster from paddle.distributed.auto_parallel.cost import CommContext from test_cluster import cluster_json @@ -37,7 +43,6 @@ def check_cost(cost): class TestCost(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -60,7 +65,8 @@ class TestCost(unittest.TestCase): matmul_v2_op = op break matmul_v2_cost = cost_model._g_op_cost_factory["matmul_v2"]( - op=matmul_v2_op) + op=matmul_v2_op + ) desc = build_comp_desc_from_op(op=matmul_v2_op) desc_str = build_comp_desc_str_for_predict(desc) self.assertIsNotNone(desc_str) @@ -73,8 +79,9 @@ class TestCost(unittest.TestCase): def test_comm_cost(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -90,7 +97,8 @@ class TestCost(unittest.TestCase): desc["inputs"] = {"X": [(paddle.float32, [100, 200])]} desc["group_ranks"] = [0, 1] allreduce_cost = cost_model._g_op_cost_factory["c_allreduce_sum"]( - op_desc=desc, comm_context=CommContext(cluster)) + op_desc=desc, comm_context=CommContext(cluster) + ) self.assertTrue(check_cost(allreduce_cost.cost)) # Remove unnecessary files @@ -99,8 +107,9 @@ class TestCost(unittest.TestCase): def test_cost_estimator(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -108,8 +117,9 @@ class TestCost(unittest.TestCase): cluster.build_from_file(cluster_json_path) train_program = paddle.static.Program() - cost_estimator = cost_model.CostEstimator(train_program, - cluster=cluster) + cost_estimator = cost_model.CostEstimator( + train_program, cluster=cluster + ) self.assertIsNotNone(cost_estimator) # Remove unnecessary files diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_optimization_tuner_api.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_optimization_tuner_api.py index 68f2376a834..a8be4468333 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_optimization_tuner_api.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_optimization_tuner_api.py @@ -21,7 +21,6 @@ import subprocess class TestOptimizationTunerAPI(unittest.TestCase): - def test_engine_api(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "optimization_tuner_api.py") @@ -32,10 +31,19 @@ class TestOptimizationTunerAPI(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "launch", "--gpus", "0,1", "--log_dir", tmp_dir.name, - launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "launch", + "--gpus", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py index ab48e2838f9..ea3bd45c035 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py @@ -19,14 +19,21 @@ import paddle.static as static from paddle.distributed import fleet from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.dist_context import DistributedContext, set_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + DistributedContext, + set_default_distributed_context, +) from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner from paddle.distributed.auto_parallel.process_mesh import ProcessMesh import sys sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) paddle.enable_static() @@ -36,7 +43,7 @@ hidden_size = 1024 sequence_len = 512 _g_process_mesh = [ ProcessMesh([0, 1], dim_names=["x"]), - ProcessMesh([2, 3], dim_names=["x"]) + ProcessMesh([2, 3], dim_names=["x"]), ] @@ -59,76 +66,100 @@ def get_program_v3(): # ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]) # ] with static.program_guard(train_program, start_program): - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] - gpt = GPTModel(vocab_size=1000, - hidden_size=1024, - num_hidden_layers=2, - num_attention_heads=16, - intermediate_size=4 * 1024, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3, - pp_degree=1) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4 * 1024, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + pp_degree=1, + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) feed_vars = { "inputs": [tokens, position_ids, attention_mask, loss_mask], - "labels": [labels] + "labels": [labels], } fetch_vars = {"loss": [loss]} - return train_program, start_program, None, loss, optimizer, feed_vars, fetch_vars + return ( + train_program, + start_program, + None, + loss, + optimizer, + feed_vars, + fetch_vars, + ) class TestParallelTunerTrain(unittest.TestCase): - def test_tune_with_train(self): flag = False set_default_distributed_context(DistributedContext()) - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program_v3( - ) + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program_v3() cluster = Cluster() cluster.gen_default_config_cluster(node_count=1, device_count=8) - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars, cluster) + dist_context = DistributedContext( + train_program, + start_program, + optimizer, + loss, + feed_vars, + fetch_vars, + cluster, + ) dist_context.initialize() parallel_tuner = ParallelTuner(dist_context, max_trials=3, mode="train") parallel_tuner.tune() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py index 27833a6a185..4a7aa6fd208 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py @@ -19,7 +19,10 @@ import paddle.static as static from paddle.distributed import fleet from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.dist_context import DistributedContext, set_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + DistributedContext, + set_default_distributed_context, +) from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner from paddle.distributed.auto_parallel.process_mesh import ProcessMesh from paddle.distributed.auto_parallel.planner_v2 import Planner @@ -28,7 +31,11 @@ import sys sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) paddle.enable_static() @@ -38,7 +45,7 @@ hidden_size = 1024 sequence_len = 512 _g_process_mesh = [ ProcessMesh([0, 1], dim_names=["x"]), - ProcessMesh([2, 3], dim_names=["x"]) + ProcessMesh([2, 3], dim_names=["x"]), ] @@ -58,86 +65,111 @@ def get_program_v3(): modeling._global_parallel_strategy = "dp_mp_pp" modeling.DPMPPP_MESH_LIST = [ ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), - ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]) + ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]), ] with static.program_guard(train_program, start_program): - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] - gpt = GPTModel(vocab_size=1000, - hidden_size=1024, - num_hidden_layers=2, - num_attention_heads=16, - intermediate_size=4 * 1024, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3, - pp_degree=len(modeling.DPMPPP_MESH_LIST)) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4 * 1024, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + pp_degree=len(modeling.DPMPPP_MESH_LIST), + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) feed_vars = { "inputs": [tokens, position_ids, attention_mask, loss_mask], - "labels": [labels] + "labels": [labels], } fetch_vars = {"loss": [loss]} - return train_program, start_program, None, loss, optimizer, feed_vars, fetch_vars + return ( + train_program, + start_program, + None, + loss, + optimizer, + feed_vars, + fetch_vars, + ) class TestParallelTunerFull(unittest.TestCase): - def test_tune_with_planner(self): flag = False set_default_distributed_context(DistributedContext()) - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program_v3( - ) + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program_v3() cluster = Cluster() cluster.gen_default_config_cluster(node_count=1, device_count=8) strategy = Strategy() strategy.auto_mode = "full" - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars, cluster, strategy) + dist_context = DistributedContext( + train_program, + start_program, + optimizer, + loss, + feed_vars, + fetch_vars, + cluster, + strategy, + ) dist_context.initialize() planner = Planner("train", dist_context) - planner._parallel_tuner = ParallelTuner(planner._dist_context, - mode=planner._mode, - max_trials=3) + planner._parallel_tuner = ParallelTuner( + planner._dist_context, mode=planner._mode, max_trials=3 + ) planner.plan() flag = True self.assertTrue(flag) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py index 2d7a2c10579..30e7eabcc11 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py @@ -19,14 +19,21 @@ import paddle.static as static from paddle.distributed import fleet from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.dist_context import DistributedContext, set_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + DistributedContext, + set_default_distributed_context, +) from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner from paddle.distributed.auto_parallel.process_mesh import ProcessMesh import sys sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) paddle.enable_static() @@ -36,7 +43,7 @@ hidden_size = 1024 sequence_len = 512 _g_process_mesh = [ ProcessMesh([0, 1], dim_names=["x"]), - ProcessMesh([2, 3], dim_names=["x"]) + ProcessMesh([2, 3], dim_names=["x"]), ] @@ -56,84 +63,108 @@ def get_program_v3(): modeling._global_parallel_strategy = "dp_mp_pp" modeling.DPMPPP_MESH_LIST = [ ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), - ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]) + ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]), ] with static.program_guard(train_program, start_program): - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] - gpt = GPTModel(vocab_size=1000, - hidden_size=1024, - num_hidden_layers=2, - num_attention_heads=16, - intermediate_size=4 * 1024, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3, - pp_degree=len(modeling.DPMPPP_MESH_LIST)) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4 * 1024, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + pp_degree=len(modeling.DPMPPP_MESH_LIST), + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) feed_vars = { "inputs": [tokens, position_ids, attention_mask, loss_mask], - "labels": [labels] + "labels": [labels], } fetch_vars = {"loss": [loss]} - return train_program, start_program, None, loss, optimizer, feed_vars, fetch_vars + return ( + train_program, + start_program, + None, + loss, + optimizer, + feed_vars, + fetch_vars, + ) class TestParallelTunerPredict(unittest.TestCase): - def test_tune_predict(self): flag = False set_default_distributed_context(DistributedContext()) - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program_v3( - ) + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program_v3() cluster = Cluster() cluster.gen_default_config_cluster(node_count=1, device_count=8) - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars, cluster) + dist_context = DistributedContext( + train_program, + start_program, + optimizer, + loss, + feed_vars, + fetch_vars, + cluster, + ) dist_context.initialize() - parallel_tuner = ParallelTuner(dist_context, - max_trials=3, - mode="predict") + parallel_tuner = ParallelTuner( + dist_context, max_trials=3, mode="predict" + ) parallel_tuner.tune() flag = True diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_amp.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_amp.py index 09de1d18c1a..2d8f3c3a8e3 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_amp.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_amp.py @@ -20,7 +20,6 @@ import subprocess class TestAMPPass(unittest.TestCase): - def test_mp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "amp_pass_unittest.py") @@ -31,10 +30,19 @@ class TestAMPPass(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_grad_clip.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_grad_clip.py index bfc5eb18a06..6febed42428 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_grad_clip.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_grad_clip.py @@ -20,11 +20,11 @@ import subprocess class TestGradientClip(unittest.TestCase): - def test_dp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) - launch_model_path = os.path.join(file_dir, - "clip_grad_by_global_norm.py") + launch_model_path = os.path.join( + file_dir, "clip_grad_by_global_norm.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] @@ -32,10 +32,19 @@ class TestGradientClip(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_gradient_merge.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_gradient_merge.py index 7ce62fa1389..0ca2cae2e1b 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_gradient_merge.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_gradient_merge.py @@ -20,11 +20,11 @@ import subprocess class TestGradientMergePass(unittest.TestCase): - def test_dp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) - launch_model_path = os.path.join(file_dir, - "gradient_merge_pass_unittest.py") + launch_model_path = os.path.join( + file_dir, "gradient_merge_pass_unittest.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] @@ -32,10 +32,19 @@ class TestGradientMergePass(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_quantization.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_quantization.py index e8df938d622..1181f090cba 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_quantization.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_quantization.py @@ -34,7 +34,6 @@ def apply_pass(): class TestQuantizationPass(unittest.TestCase): - def test_qat_pass(self): batch_size = 8 diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_recompute.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_recompute.py index c6062faa260..ff873bec4f5 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_recompute.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_recompute.py @@ -20,7 +20,6 @@ import subprocess class TestRecomputePass(unittest.TestCase): - def test_mp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "recompute_pass_unittest.py") @@ -31,10 +30,19 @@ class TestRecomputePass(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_sharding.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_sharding.py index 31467b6d505..eab74b45130 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_sharding.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_sharding.py @@ -20,7 +20,6 @@ import subprocess class TestShardingPass(unittest.TestCase): - def test_dp2sharding2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "sharding_pass_unittest.py") @@ -31,10 +30,19 @@ class TestShardingPass(unittest.TestCase): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_prim_dist_op.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_prim_dist_op.py index 028d2b820a5..2c22f3fb1c1 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_prim_dist_op.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_prim_dist_op.py @@ -22,7 +22,10 @@ from paddle.distributed.fleet import auto from paddle.distributed.auto_parallel.completion import Completer from paddle.distributed.auto_parallel.partitioner import Partitioner from paddle.distributed.auto_parallel.utils import set_var_dist_attr -from paddle.distributed.auto_parallel.dist_context import DistributedContext, get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + DistributedContext, + get_default_distributed_context, +) paddle.enable_static() enable_prim() @@ -31,60 +34,68 @@ rank = 0 class TestPrimDistOp(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() self.layer_help = LayerHelper('TestPrimDistOp') - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_prog() def init_prog(self): # block = self.main_program.global_block() # block = self.main_program.global_block() - self.w = self.layer_help.create_parameter(dtype="float", - shape=[20], - attr=None) - self.w_grad = paddle.static.data(name='w_grad', - shape=[20], - dtype='float') + self.w = self.layer_help.create_parameter( + dtype="float", shape=[20], attr=None + ) + self.w_grad = paddle.static.data( + name='w_grad', shape=[20], dtype='float' + ) self.tmp1 = paddle.static.data(name='tmp1', shape=[20], dtype='float') self.tmp2 = paddle.static.data(name='tmp2', shape=[20], dtype='float') - self.batch_reduced = paddle.static.data(name='batch_reduced', - shape=[1], - dtype='float') + self.batch_reduced = paddle.static.data( + name='batch_reduced', shape=[1], dtype='float' + ) self.attrs = {} default_dist_context = get_default_distributed_context() _global_process_mesh = auto.ProcessMesh(list(range(nranks))) - tensor_dist_attr = set_var_dist_attr(default_dist_context, - self.tmp1, [-1], - _global_process_mesh, - mark_annotated=True) - tensor_dist_attr = set_var_dist_attr(default_dist_context, - self.tmp1, [-1], - _global_process_mesh, - mark_annotated=True) - - op = self.layer_help.append_op(type="add_p", - inputs={ - 'X': self.tmp1, - 'Y': self.w - }, - outputs={'Z': self.w_grad}, - attrs=self.attrs) - - op = self.layer_help.append_op(type="reduce_sum_p", - inputs={'X': self.tmp2}, - outputs={'Y': self.batch_reduced}, - attrs={"axis": [0]}) + tensor_dist_attr = set_var_dist_attr( + default_dist_context, + self.tmp1, + [-1], + _global_process_mesh, + mark_annotated=True, + ) + tensor_dist_attr = set_var_dist_attr( + default_dist_context, + self.tmp1, + [-1], + _global_process_mesh, + mark_annotated=True, + ) + + op = self.layer_help.append_op( + type="add_p", + inputs={'X': self.tmp1, 'Y': self.w}, + outputs={'Z': self.w_grad}, + attrs=self.attrs, + ) + + op = self.layer_help.append_op( + type="reduce_sum_p", + inputs={'X': self.tmp2}, + outputs={'Y': self.batch_reduced}, + attrs={"axis": [0]}, + ) def test_loss_and_grad_allreduce(self): - dist_context = DistributedContext(self.main_program, - self.startup_program) + dist_context = DistributedContext( + self.main_program, self.startup_program + ) completer = Completer(dist_context) completer.complete_prim_annotation(self.main_program) dist_context.block_state.parse_forward_blocks(self.main_program) @@ -95,7 +106,8 @@ class TestPrimDistOp(unittest.TestCase): dist_context.data_parallel_group = list(range(nranks)) partitioner = Partitioner(dist_context, rank) dist_main_prog, dist_startup_prog, _ = partitioner.partition( - self.main_program, self.startup_program, [(self.w, self.w_grad)]) + self.main_program, self.startup_program, [(self.w, self.w_grad)] + ) ops = dist_main_prog.global_block().ops self.assertTrue(ops[1].type == "c_allreduce_sum") diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh.py index 6b83c4da245..2e698efc416 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh.py @@ -19,7 +19,9 @@ import paddle.nn as nn import paddle.nn.functional as F import paddle.static as static from paddle.distributed.auto_parallel.process_mesh import ProcessMesh -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) paddle.enable_static() @@ -30,29 +32,33 @@ sequence_len = 512 class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): out = self.norm(input) @@ -63,7 +69,6 @@ class MLPLayer(nn.Layer): class TestProcessMesh(unittest.TestCase): - def test_construction(self): mesh = [[0, 1, 2], [3, 4, 5]] process_mesh = ProcessMesh(mesh, dim_names=["x", "y"]) @@ -106,17 +111,21 @@ class TestProcessMesh(unittest.TestCase): def test_context_manager(self): mesh = np.array([1, 2, 3, 4]) - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') - - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) + + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) with ProcessMesh(mesh, "d"): out = mlp(input) @@ -127,15 +136,18 @@ class TestProcessMesh(unittest.TestCase): for block in default_program.blocks: for tensor in block.vars.values(): dist_tensor = default_dist_context.get_dist_tensor_for_program( - tensor) + tensor + ) if dist_tensor is not None: - self.assertEqual(dist_tensor.dist_attr.process_mesh, - ProcessMesh(mesh)) + self.assertEqual( + dist_tensor.dist_attr.process_mesh, ProcessMesh(mesh) + ) for op in block.ops: dist_op = default_dist_context.get_dist_op_for_program(op) if dist_op is not None: - self.assertEqual(dist_op.dist_attr.process_mesh, - ProcessMesh(mesh)) + self.assertEqual( + dist_op.dist_attr.process_mesh, ProcessMesh(mesh) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh_v2.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh_v2.py index 3c58f9e8cd3..7a813e70305 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh_v2.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh_v2.py @@ -14,11 +14,13 @@ import unittest from paddle.distributed.auto_parallel.process_mesh_v2 import ( - ProcessMesh, compute_compatible_process_mesh, merge_process_mesh) + ProcessMesh, + compute_compatible_process_mesh, + merge_process_mesh, +) class TestProcessMesh(unittest.TestCase): - def test_process_mesh(self): mesh = [[0, 1, 2], [3, 4, 5]] mesh2 = [[0, 1], [2, 3]] @@ -41,34 +43,41 @@ class TestProcessMesh(unittest.TestCase): self.assertEqual(str(process_mesh), str(process_mesh)) def test_compute_compatible_process_mesh(self): - process_mesh1 = ProcessMesh([[0, 1, 2], [3, 4, 5]], - dim_names=["x", "y"]) + process_mesh1 = ProcessMesh( + [[0, 1, 2], [3, 4, 5]], dim_names=["x", "y"] + ) compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh1, None]) + [process_mesh1, None] + ) self.assertEqual(compatible_process_mesh, process_mesh1) compatible_process_mesh = compute_compatible_process_mesh( - [None, process_mesh1]) + [None, process_mesh1] + ) self.assertEqual(compatible_process_mesh, process_mesh1) process_mesh2 = ProcessMesh([[0, 1, 2], [3, 4, 5]]) compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh1, process_mesh2]) + [process_mesh1, process_mesh2] + ) self.assertEqual(compatible_process_mesh, process_mesh1) self.assertEqual(compatible_process_mesh, process_mesh2) process_mesh2 = ProcessMesh([[0, 1, 2, 3, 4, 5]]) compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh1, process_mesh2]) + [process_mesh1, process_mesh2] + ) self.assertEqual(compatible_process_mesh, process_mesh1) process_mesh2 = ProcessMesh([[0, 1, 2]]) compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh1, process_mesh2]) + [process_mesh1, process_mesh2] + ) self.assertEqual(compatible_process_mesh, process_mesh1) def test_merge_process_mesh(self): - process_mesh1 = ProcessMesh([[0, 1, 2], [3, 4, 5]], - dim_names=["x", "y"]) + process_mesh1 = ProcessMesh( + [[0, 1, 2], [3, 4, 5]], dim_names=["x", "y"] + ) merged_process_mesh = merge_process_mesh([process_mesh1, None]) print(merged_process_mesh) self.assertEqual(merged_process_mesh, ProcessMesh([0, 1, 2, 3, 4, 5])) @@ -85,8 +94,9 @@ class TestProcessMesh(unittest.TestCase): process_mesh2 = ProcessMesh([[6, 7]]) merged_process_mesh = merge_process_mesh([process_mesh1, process_mesh2]) - self.assertEqual(merged_process_mesh, - ProcessMesh([0, 1, 2, 3, 4, 5, 6, 7])) + self.assertEqual( + merged_process_mesh, ProcessMesh([0, 1, 2, 3, 4, 5, 6, 7]) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_recorder.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_recorder.py index d9594b95198..5f07a711768 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_recorder.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_recorder.py @@ -19,7 +19,6 @@ from paddle.distributed.auto_parallel.tuner import recorder as rd class TestRecorder(unittest.TestCase): - def test_register(self): recorder = rd.MetricsRecorder() recorder.register("metric") @@ -35,8 +34,9 @@ class TestRecorder(unittest.TestCase): recorder = rd.MetricsRecorder() recorder.update("metric", 4, 1000) self.assertEqual(recorder.records["metric"].direction, "min") - self.assertEqual(recorder.get_records("metric"), - [rd.MetricRecord(4, 1000)]) + self.assertEqual( + recorder.get_records("metric"), [rd.MetricRecord(4, 1000)] + ) def test_get_records(self): recorder = rd.MetricsRecorder() @@ -44,12 +44,15 @@ class TestRecorder(unittest.TestCase): recorder.update("metric", 2, step=1) recorder.update("metric", 3, step=2) recorder.update("metric", 4, step=3) - self.assertEqual(recorder.get_records("metric"), [ - rd.MetricRecord(1, 0), - rd.MetricRecord(2, 1), - rd.MetricRecord(3, 2), - rd.MetricRecord(4, 3), - ]) + self.assertEqual( + recorder.get_records("metric"), + [ + rd.MetricRecord(1, 0), + rd.MetricRecord(2, 1), + rd.MetricRecord(3, 2), + rd.MetricRecord(4, 3), + ], + ) def test_set_records(self): recorder = rd.MetricsRecorder() @@ -62,12 +65,15 @@ class TestRecorder(unittest.TestCase): rd.MetricRecord(4, 3), ], ) - self.assertEqual(recorder.get_records("metric"), [ - rd.MetricRecord(1, 0), - rd.MetricRecord(2, 1), - rd.MetricRecord(3, 2), - rd.MetricRecord(4, 3), - ]) + self.assertEqual( + recorder.get_records("metric"), + [ + rd.MetricRecord(1, 0), + rd.MetricRecord(2, 1), + rd.MetricRecord(3, 2), + rd.MetricRecord(4, 3), + ], + ) def test_get_best_value(self): recorder = rd.MetricsRecorder() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_gpt_planner.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_gpt_planner.py index bde0615a41d..0ffe09bfd1e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_gpt_planner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_gpt_planner.py @@ -21,7 +21,6 @@ import subprocess class TestPlannerReLaunch(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -31,10 +30,12 @@ class TestPlannerReLaunch(unittest.TestCase): def test_relaunch_with_planner(self): from test_auto_parallel_relaunch import cluster_json, mapping_josn - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") - mapping_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_rank_mapping.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) + mapping_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_rank_mapping.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: @@ -46,19 +47,31 @@ class TestPlannerReLaunch(unittest.TestCase): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join( - file_dir, "auto_parallel_relaunch_with_gpt_planner.py") + file_dir, "auto_parallel_relaunch_with_gpt_planner.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] else: coverage_args = [] - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--log_dir", self.temp_dir.name, - "--cluster_topo_path", cluster_json_path, "--rank_mapping_path", - mapping_json_path, "--enable_auto_mapping", "True", - launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--log_dir", + self.temp_dir.name, + "--cluster_topo_path", + cluster_json_path, + "--rank_mapping_path", + mapping_json_path, + "--enable_auto_mapping", + "True", + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() self.assertEqual(process.returncode, 0) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_planner.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_planner.py index 3c17b6d126e..b8469c1a407 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_planner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_planner.py @@ -21,7 +21,6 @@ import subprocess class TestPlannerReLaunch(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -31,10 +30,12 @@ class TestPlannerReLaunch(unittest.TestCase): def test_relaunch_with_planner(self): from test_auto_parallel_relaunch import cluster_json, mapping_josn - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") - mapping_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_rank_mapping.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) + mapping_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_rank_mapping.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: @@ -46,19 +47,31 @@ class TestPlannerReLaunch(unittest.TestCase): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join( - file_dir, "auto_parallel_relaunch_with_planner.py") + file_dir, "auto_parallel_relaunch_with_planner.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] else: coverage_args = [] - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--log_dir", self.temp_dir.name, - "--cluster_topo_path", cluster_json_path, "--rank_mapping_path", - mapping_json_path, "--enable_auto_mapping", "True", - launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--log_dir", + self.temp_dir.name, + "--cluster_topo_path", + cluster_json_path, + "--rank_mapping_path", + mapping_json_path, + "--enable_auto_mapping", + "True", + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() self.assertEqual(process.returncode, 0) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_strategy.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_strategy.py index 08cd1dee0e6..54b11c4934c 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_strategy.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_strategy.py @@ -18,7 +18,6 @@ from paddle.distributed.fleet import auto class TestStrategy(unittest.TestCase): - def test_default_config(self): strategy = auto.Strategy() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_to_static.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_to_static.py index e751ef64be2..3c8b71e7139 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_to_static.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_to_static.py @@ -34,7 +34,6 @@ class_num = 10 class MyDataset(Dataset): - def __init__(self, num_samples): super(MyDataset, self).__init__() self.num_samples = num_samples @@ -49,26 +48,26 @@ class MyDataset(Dataset): class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) - - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=None) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=None) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) + + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=None + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=None + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=None) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -85,17 +84,19 @@ class MLPLayer(nn.Layer): class TestWholeProgram(unittest.TestCase): - def test_apply_optimzier(self): paddle.disable_static() - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) metrics = paddle.metric.Accuracy() loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - parameters=mlp.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, parameters=mlp.parameters() + ) inputs = InputSpec([batch_size, hidden_size], 'float32', 'x') labels = InputSpec([batch_size], 'int64', 'label') @@ -114,7 +115,8 @@ class TestWholeProgram(unittest.TestCase): optimize_ops, _ = program_helper.apply_optimizer(optimizer) all_ops = program_helper.main_program.block(0).ops sgd_ops = [ - op for op in program_helper.main_program.block(0).ops + op + for op in program_helper.main_program.block(0).ops if op.type == 'sgd' ] self.assertEqual(len(all_ops), 37) @@ -124,16 +126,18 @@ class TestWholeProgram(unittest.TestCase): class TestToStatic(unittest.TestCase): - def test_to_static(self): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - parameters=mlp.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, parameters=mlp.parameters() + ) dataset = MyDataset(batch_num * batch_size) @@ -141,11 +145,13 @@ class TestToStatic(unittest.TestCase): # labels = InputSpec([batch_size], 'int64', 'label') assert _non_static_mode() == True - engine = auto.Engine(model=mlp, - loss=loss, - optimizer=optimizer, - metrics=paddle.metric.Accuracy(), - strategy=None) + engine = auto.Engine( + model=mlp, + loss=loss, + optimizer=optimizer, + metrics=paddle.metric.Accuracy(), + strategy=None, + ) engine.fit(dataset, batch_size=batch_size) engine.evaluate(dataset, batch_size=batch_size) engine.predict(dataset, batch_size=batch_size) @@ -153,14 +159,15 @@ class TestToStatic(unittest.TestCase): class TestLazyInit(unittest.TestCase): - def test_lazy_init(self): with LazyGuard(): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() metrics = paddle.metric.Accuracy() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_trial.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_trial.py index e39991fcaa5..fc52d1c394e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_trial.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_trial.py @@ -19,7 +19,6 @@ from paddle.distributed.auto_parallel.tuner import trial as tr class TestTiral(unittest.TestCase): - def test_trial(self): space = ts.TunableSpace() space.choice("choice", [0, 1, 2, 3], default=2) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_space.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_space.py index 58ff36aba09..badc90275fd 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_space.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_space.py @@ -18,7 +18,6 @@ from paddle.distributed.auto_parallel.tuner import tunable_space as ts class TestTunableSpace(unittest.TestCase): - def test_fixed(self): space = ts.TunableSpace() fixed = space.fixed("fixed", default=4) @@ -73,10 +72,9 @@ class TestTunableSpace(unittest.TestCase): def test_float_range(self): space = ts.TunableSpace() - float_range = space.float_range("float_range", - start=0.4, - stop=4.4, - default=2.0) + float_range = space.float_range( + "float_range", start=0.4, stop=4.4, default=2.0 + ) self.assertEqual(space.values["float_range"], 2.0) self.assertEqual(len(space.variables), 1) self.assertEqual(space.variables["float_range"].name, "float_range") diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_variable.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_variable.py index ce0a076c83e..641f7b4347e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_variable.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_variable.py @@ -18,7 +18,6 @@ from paddle.distributed.auto_parallel.tuner import tunable_variable as tv class TestTunableVariable(unittest.TestCase): - def test_fixed(self): fixed = tv.Fixed("fixed", True) fixed = tv.Fixed.from_state(fixed.get_state()) @@ -64,12 +63,9 @@ class TestTunableVariable(unittest.TestCase): self.assertIn(int_range.random(1234), [1, 2, 3, 4]) self.assertNotEqual(int_range.default, 4) - int_range = tv.IntRange("int_range", - start=1, - stop=8, - step=2, - default=3, - endpoint=True) + int_range = tv.IntRange( + "int_range", start=1, stop=8, step=2, default=3, endpoint=True + ) int_range = tv.IntRange.from_state(int_range.get_state()) self.assertEqual(int_range.default, 3) self.assertIn(int_range.random(), [1, 3, 5, 7]) @@ -77,10 +73,9 @@ class TestTunableVariable(unittest.TestCase): self.assertNotEqual(int_range.default, 2) def test_float_range(self): - float_range = tv.FloatRange("float_range", - start=0.4, - stop=4.4, - default=2.0) + float_range = tv.FloatRange( + "float_range", start=0.4, stop=4.4, default=2.0 + ) float_range = tv.FloatRange.from_state(float_range.get_state()) self.assertEqual(float_range.default, 2.0) self.assertGreaterEqual(float_range.random(), 0.4) @@ -88,12 +83,14 @@ class TestTunableVariable(unittest.TestCase): self.assertNotAlmostEqual(float_range.random(), 1) self.assertNotAlmostEqual(float_range.random(), 4.4) - float_range = tv.FloatRange("float_range", - start=0.4, - stop=8.4, - step=2.0, - default=3.0, - endpoint=True) + float_range = tv.FloatRange( + "float_range", + start=0.4, + stop=8.4, + step=2.0, + default=3.0, + endpoint=True, + ) float_range = tv.FloatRange.from_state(float_range.get_state()) self.assertEqual(float_range.default, 3.0) self.assertGreaterEqual(float_range.random(), 0.4) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_completion.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_completion.py index bba1e2ba6f1..9a8b9e1a8f3 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_completion.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_completion.py @@ -40,50 +40,56 @@ def get_random_inputs_and_labels(input_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_size): batch_input, batch_label = get_random_inputs_and_labels( [batch_size, sequence_len, hidden_size], - [batch_size, sequence_len, 1]) + [batch_size, sequence_len, 1], + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): out = self.norm(input) - auto.shard_tensor(self.linear0.weight, _g_process_mesh[:, 0], - [None, 'x']) + auto.shard_tensor( + self.linear0.weight, _g_process_mesh[:, 0], [None, 'x'] + ) out = self.linear0(out) out = F.gelu(out, approximate=True) - auto.shard_tensor(self.linear1.weight, _g_process_mesh[:, 1], - ['x', None]) + auto.shard_tensor( + self.linear1.weight, _g_process_mesh[:, 1], ['x', None] + ) out = self.linear1(out) return out @@ -95,15 +101,19 @@ def loop_cond(i, loop_len, input_array): def loop_body(i, loop_len, input_array): pre_input = paddle.tensor.array_read(array=input_array, i=i) - mlp_while0 = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) - - mlp_while1 = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_while0 = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) + + mlp_while1 = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) output = mlp_while0(pre_input) cur_pred = mlp_while1(output) @@ -128,41 +138,46 @@ def get_program(): loop_len = paddle.full(shape=[1], fill_value=epoch_num, dtype='int64') # input - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) data_holder = [input, label] # dataloader - dataloader = paddle.io.DataLoader.from_generator(feed_list=data_holder, - capacity=4 * - batch_size, - iterable=False) - dataloader.set_batch_generator(batch_generator_creator(), - places=paddle.static.cuda_places()) + dataloader = paddle.io.DataLoader.from_generator( + feed_list=data_holder, capacity=4 * batch_size, iterable=False + ) + dataloader.set_batch_generator( + batch_generator_creator(), places=paddle.static.cuda_places() + ) # data dist_attr auto.shard_tensor(input, _g_process_mesh[:, 0], [None, None, None]) auto.shard_tensor(label, _g_process_mesh[:, 0], [None, None, None]) - mlp_start = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_start = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_start(input) input_array = paddle.tensor.array_write(pred, i) i, loop_len, input_array = static.nn.while_loop( - cond=loop_cond, - body=loop_body, - loop_vars=[i, loop_len, input_array]) + cond=loop_cond, body=loop_body, loop_vars=[i, loop_len, input_array] + ) end_pred = paddle.tensor.array_read(array=input_array, i=i) - mlp_end = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_end = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_end(end_pred) error_cost = paddle.nn.functional.square_error_cost(pred, label) @@ -172,13 +187,13 @@ def get_program(): class TestMLP(unittest.TestCase): - def test_completer(self): train_program, start_program, dataloader, i, loss = get_program() dist_context = DistributedContext() completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) # print_program_with_dist_attr(complete_train_program, dist_context) def test_completer_by_dist_op(self): @@ -186,7 +201,8 @@ class TestMLP(unittest.TestCase): dist_context = DistributedContext() completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) complete_train_program = completer._complete_tensor_dist_attr_by_op() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_partition.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_partition.py index 2e3f1d93633..4bb9272748b 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_partition.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_partition.py @@ -25,7 +25,9 @@ from paddle.distributed import fleet from paddle.distributed.auto_parallel.completion import Completer from paddle.distributed.auto_parallel.partitioner import Partitioner from paddle.distributed.auto_parallel.utils import make_data_unshard -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) paddle.enable_static() @@ -43,41 +45,45 @@ def get_random_inputs_and_labels(input_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_size): batch_input, batch_label = get_random_inputs_and_labels( [batch_size, sequence_len, hidden_size], - [batch_size, sequence_len, 1]) + [batch_size, sequence_len, 1], + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): @@ -114,41 +120,46 @@ def get_program(): auto.shard_tensor(i, _g_process_mesh, [None]) # 循环次数 - loop_len = fluid.layers.fill_constant(shape=[1], - dtype='int64', - value=epoch_num) + loop_len = fluid.layers.fill_constant( + shape=[1], dtype='int64', value=epoch_num + ) auto.shard_tensor(loop_len, _g_process_mesh, [None]) # input - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) data_holder = [input, label] # dataloader - dataloader = paddle.io.DataLoader.from_generator(feed_list=data_holder, - capacity=4 * - batch_size, - iterable=False) - dataloader.set_batch_generator(batch_generator_creator(), - places=paddle.static.cuda_places()) + dataloader = paddle.io.DataLoader.from_generator( + feed_list=data_holder, capacity=4 * batch_size, iterable=False + ) + dataloader.set_batch_generator( + batch_generator_creator(), places=paddle.static.cuda_places() + ) # data dist_attr auto.shard_tensor(input, _g_process_mesh, [None, None, None]) auto.shard_tensor(label, _g_process_mesh, [None, None, None]) # fill constant bsz like tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=input, shape=[-1, 16, 0, 48], dtype='float32', value=0) + input=input, shape=[-1, 16, 0, 48], dtype='float32', value=0 + ) auto.shard_tensor(tmp, _g_process_mesh, [None, 'x', None, None]) # model - mlp_start = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_start = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_start(input) input_array = fluid.layers.array_write(pred, i) @@ -168,10 +179,12 @@ def get_program(): pre_input = fluid.layers.array_read(array=input_array, i=i) auto.shard_tensor(pre_input, _g_process_mesh, [None, None, None]) - mlp_while = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_while = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) cur_pred = mlp_while(pre_input) # 更新循环条件 @@ -182,10 +195,12 @@ def get_program(): end_pred = fluid.layers.array_read(array=input_array, i=i) auto.shard_tensor(end_pred, _g_process_mesh, [None, None, None]) - mlp_end = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_end = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_end(end_pred) error_cost = paddle.nn.functional.square_error_cost(pred, label) @@ -345,24 +360,25 @@ def partition(train_program, start_program, dist_context): rank = paddle.distributed.get_rank() partitioner = Partitioner(dist_context, rank) dist_main_prog, dist_startup_prog, _ = partitioner.partition( - train_program, start_program, []) + train_program, start_program, [] + ) return dist_main_prog, dist_startup_prog class TestMLP(unittest.TestCase): - def test_partitioner(self): train_program, start_program, dataloader, i, loss = get_program() dist_context = get_default_distributed_context() - train_program, start_program = completion(train_program, start_program, - dist_context) + train_program, start_program = completion( + train_program, start_program, dist_context + ) dist_context.block_state.parse_forward_blocks(train_program) - dist_main_prog, dist_startup_prog = partition(train_program, - start_program, - dist_context) + dist_main_prog, dist_startup_prog = partition( + train_program, start_program, dist_context + ) global_block_ops = dist_main_prog.blocks[0].ops fill_op = None diff --git a/python/paddle/fluid/tests/unittests/auto_parallel_autoconvert.py b/python/paddle/fluid/tests/unittests/auto_parallel_autoconvert.py index b8c64c0af54..f2ced04fe56 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel_autoconvert.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel_autoconvert.py @@ -26,9 +26,19 @@ from paddle.distributed.fleet import auto from paddle.distributed import fleet from paddle.fluid.initializer import NumpyArrayInitializer -from paddle.distributed.auto_parallel.utils import save_distributed_checkpoint, load_distributed_checkpoint, load_checkpoint_into_program -from paddle.distributed.auto_parallel.utils import get_dist_attr, merge_and_slice_parameter, load_parameter_into_program -from paddle.distributed.auto_parallel.dist_context import set_default_distributed_context +from paddle.distributed.auto_parallel.utils import ( + save_distributed_checkpoint, + load_distributed_checkpoint, + load_checkpoint_into_program, +) +from paddle.distributed.auto_parallel.utils import ( + get_dist_attr, + merge_and_slice_parameter, + load_parameter_into_program, +) +from paddle.distributed.auto_parallel.dist_context import ( + set_default_distributed_context, +) paddle.enable_static() _global_parallel_strategy = None @@ -38,11 +48,9 @@ PP_MESH_1 = None class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=64, - intermediate_size=4 * 64, - initializer_range=0.02): + def __init__( + self, hidden_size=64, intermediate_size=4 * 64, initializer_range=0.02 + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size @@ -52,14 +60,12 @@ class MLPLayer(nn.Layer): weight_attr0 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr0)) weight_attr1 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr1)) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -67,15 +73,19 @@ class MLPLayer(nn.Layer): auto.shard_tensor(self.linear0.weight, PP_MESH_0, [None, None]) auto.shard_tensor(self.linear1.weight, PP_MESH_1, [None, None]) elif _global_parallel_strategy == "mp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, "x"]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, "x"] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, None]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, None] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, None] + ) out = self.norm(input) out = self.linear0(out) @@ -85,16 +95,17 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program,start_program), \ - utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 64 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "pp": auto.shard_tensor(input, PP_MESH_0, [None, None]) @@ -104,9 +115,11 @@ def mlp_forward(train_program, start_program): elif _global_parallel_strategy == "mp": auto.shard_tensor(input, _global_process_mesh, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) loss = paddle.mean(error_cost) @@ -119,28 +132,35 @@ def get_distributed_program(): dist_strategy = fleet.DistributedStrategy() dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) optimizer = paddle.fluid.optimizer.SGDOptimizer(learning_rate=0.01) optimizer = fleet.distributed_optimizer(optimizer) _, _, dist_startup_prog, dist_main_prog = optimizer.minimize( - loss, startup_program) + loss, startup_program + ) return dist_main_prog, dist_startup_prog, loss class TestMLPAutoConvert(unittest.TestCase): - def setUp(self): paddle.seed(2021) random.seed(2021) np.random.seed(2021) def tearDown(self): - os.remove("./model_state_rank{}.pdmodel".format( - str(paddle.distributed.get_rank()))) - os.remove("./dist_attr_rank{}.pdattr".format( - str(paddle.distributed.get_rank()))) + os.remove( + "./model_state_rank{}.pdmodel".format( + str(paddle.distributed.get_rank()) + ) + ) + os.remove( + "./dist_attr_rank{}.pdattr".format( + str(paddle.distributed.get_rank()) + ) + ) def test_mlp_mp2pp(self): set_default_distributed_context(None) @@ -159,16 +179,18 @@ class TestMLPAutoConvert(unittest.TestCase): for step in range(20): if step == 10: - save_distributed_checkpoint(dist_main_prog, - ".", - dist_attr_path=".") - - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + save_distributed_checkpoint( + dist_main_prog, ".", dist_attr_path="." + ) + + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) last_res = res[0] set_default_distributed_context(None) @@ -179,50 +201,65 @@ class TestMLPAutoConvert(unittest.TestCase): global PP_MESH_1 PP_MESH_1 = auto.ProcessMesh(mesh=[1], dim_names=["pp1"]) - dist_main_prog_load, dist_start_prog_load, loss_load = get_distributed_program( - ) + ( + dist_main_prog_load, + dist_start_prog_load, + loss_load, + ) = get_distributed_program() place = paddle.set_device("gpu") exe = paddle.static.Executor(place) exe.run(dist_start_prog_load) ckpt_path = [ - "./model_state_rank0.pdmodel", "./model_state_rank1.pdmodel" + "./model_state_rank0.pdmodel", + "./model_state_rank1.pdmodel", ] dist_attr_path = [ - "./dist_attr_rank0.pdattr", "./dist_attr_rank1.pdattr" + "./dist_attr_rank0.pdattr", + "./dist_attr_rank1.pdattr", ] - load_checkpoint_into_program(ckpt_path, dist_attr_path, - dist_main_prog_load) + load_checkpoint_into_program( + ckpt_path, dist_attr_path, dist_main_prog_load + ) for step in range(10, 20): if paddle.distributed.get_rank() in [0]: - res = exe.run(dist_main_prog_load, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }) + res = exe.run( + dist_main_prog_load, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + ) else: - res = exe.run(dist_main_prog_load, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss_load]) + res = exe.run( + dist_main_prog_load, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss_load], + ) if paddle.distributed.get_rank() in [1]: self.assertEqual(last_res, res[0]) class TestMLPAutoConvert2(unittest.TestCase): - def setUp(self): paddle.seed(2021) random.seed(2021) np.random.seed(2021) def tearDown(self): - os.remove("./model_state_rank{}.pdmodel".format( - str(paddle.distributed.get_rank()))) - os.remove("./dist_attr_rank{}.pdattr".format( - str(paddle.distributed.get_rank()))) + os.remove( + "./model_state_rank{}.pdmodel".format( + str(paddle.distributed.get_rank()) + ) + ) + os.remove( + "./dist_attr_rank{}.pdattr".format( + str(paddle.distributed.get_rank()) + ) + ) def test_mlp_pp2mp(self): set_default_distributed_context(None) @@ -247,18 +284,22 @@ class TestMLPAutoConvert2(unittest.TestCase): save_distributed_checkpoint(dist_main_prog, ".", ".", add_info) if paddle.distributed.get_rank() in [0]: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + ) else: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) if paddle.distributed.get_rank() in [1]: last_res = res[0] @@ -266,41 +307,49 @@ class TestMLPAutoConvert2(unittest.TestCase): _global_parallel_strategy = "mp" _global_process_mesh = auto.ProcessMesh([0, 1], dim_names=["x"]) - dist_main_prog_load, dist_start_prog_load, loss_load = get_distributed_program( - ) + ( + dist_main_prog_load, + dist_start_prog_load, + loss_load, + ) = get_distributed_program() place = paddle.set_device("gpu") exe = paddle.static.Executor(place) exe.run(dist_start_prog_load) ckpt_path = [ - "./model_state_rank0.pdmodel", "./model_state_rank1.pdmodel" + "./model_state_rank0.pdmodel", + "./model_state_rank1.pdmodel", ] dist_attr_path = [ - "./dist_attr_rank0.pdattr", "./dist_attr_rank1.pdattr" + "./dist_attr_rank0.pdattr", + "./dist_attr_rank1.pdattr", ] param_dict, pre_dist_attr, add_info = load_distributed_checkpoint( - ckpt_path, dist_attr_path) + ckpt_path, dist_attr_path + ) batch = add_info["batch"] batch_size = add_info["batch_size"] start_index = batch * batch_size input = input[start_index:, :] label = label[start_index:, :] cur_dist_attr = get_dist_attr(dist_main_prog_load) - sliced_param_dict = merge_and_slice_parameter(param_dict, pre_dist_attr, - cur_dist_attr) + sliced_param_dict = merge_and_slice_parameter( + param_dict, pre_dist_attr, cur_dist_attr + ) load_parameter_into_program(sliced_param_dict, dist_main_prog_load) for step in range(10): - res = exe.run(dist_main_prog_load, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss_load]) + res = exe.run( + dist_main_prog_load, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss_load], + ) if paddle.distributed.get_rank() in [1]: self.assertEqual(last_res, res[0]) class TestMLPAutoConvertInvalid(unittest.TestCase): - def setUp(self): paddle.seed(2021) random.seed(2021) @@ -314,24 +363,32 @@ class TestMLPAutoConvertInvalid(unittest.TestCase): _global_process_mesh = auto.ProcessMesh([0, 1], dim_names=["x"]) dist_main_prog, _, _ = get_distributed_program() with self.assertRaises(TypeError): - save_distributed_checkpoint(dist_main_prog, [""], [""], - addition_info=[0]) + save_distributed_checkpoint( + dist_main_prog, [""], [""], addition_info=[0] + ) with self.assertRaises(ValueError): - save_distributed_checkpoint(dist_main_prog, [""], [""], - addition_info={"step": 0}) + save_distributed_checkpoint( + dist_main_prog, [""], [""], addition_info={"step": 0} + ) with self.assertRaises(ValueError): - save_distributed_checkpoint(dist_main_prog, [""], [""], - addition_info={"batch": 0.0}) + save_distributed_checkpoint( + dist_main_prog, [""], [""], addition_info={"batch": 0.0} + ) with self.assertRaises(ValueError): - load_checkpoint_into_program(["./model_state_rank.pdmodel"], - ["./dist_attr_rank.pdattr"], - dist_main_prog) + load_checkpoint_into_program( + ["./model_state_rank.pdmodel"], + ["./dist_attr_rank.pdattr"], + dist_main_prog, + ) with self.assertRaises(ValueError): - load_distributed_checkpoint(["./model_state_rank.pdmodel"], - ["./dist_attr_rank.pdattr"]) + load_distributed_checkpoint( + ["./model_state_rank.pdmodel"], ["./dist_attr_rank.pdattr"] + ) with self.assertRaises(TypeError): - load_distributed_checkpoint({"0": "./model_state_rank.pdmodel"}, - {"1": "./dist_attr_rank.pdattr"}) + load_distributed_checkpoint( + {"0": "./model_state_rank.pdmodel"}, + {"1": "./dist_attr_rank.pdattr"}, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel_data_unshard.py b/python/paddle/fluid/tests/unittests/auto_parallel_data_unshard.py index e08739039c8..53a1250f191 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel_data_unshard.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel_data_unshard.py @@ -28,9 +28,7 @@ paddle.distributed.init_parallel_env() class TestDataUnshard(unittest.TestCase): - def test_dp2pp1mp1(self): - def create_model(train_program, start_program): with paddle.static.program_guard(train_program, start_program): @@ -39,7 +37,8 @@ class TestDataUnshard(unittest.TestCase): label = paddle.static.data(name='label', shape=[2, 8]) weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=0.02)) + initializer=nn.initializer.Normal(mean=0.0, std=0.02) + ) linear0 = nn.Linear(8, 8, weight_attr) linear1 = nn.Linear(8, 8, weight_attr) @@ -52,7 +51,8 @@ class TestDataUnshard(unittest.TestCase): gelu_out = F.gelu(linear0_out) linear1_out = linear1(gelu_out) error_cost = paddle.nn.functional.square_error_cost( - linear1_out, label) + linear1_out, label + ) loss = paddle.mean(error_cost) return train_program, start_program, loss, input, label @@ -60,20 +60,27 @@ class TestDataUnshard(unittest.TestCase): start_program = paddle.static.Program() # serial program train_program, start_program, loss, input, label = create_model( - train_program, start_program) + train_program, start_program + ) dist_strategy = fleet.DistributedStrategy() dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) worker_index = paddle.distributed.get_rank() paddle.seed(worker_index + 2021) @@ -87,20 +94,20 @@ class TestDataUnshard(unittest.TestCase): input_data = np.array(range(2 * 8)).reshape([2, 8]).astype("float32") label_data = np.random.randint(0, 10, [2, 8]).astype("float32") - fetchs = [loss.name, 'split@RESHARD.tmp_0'] if worker_index == 0 else [ - loss.name, 'split@RESHARD.tmp_1' - ] - loss_np, shard_data_np = exe.run(distributed_main_program, - feed={ - "input": input_data, - "label": label_data - }, - fetch_list=fetchs) + fetchs = ( + [loss.name, 'split@RESHARD.tmp_0'] + if worker_index == 0 + else [loss.name, 'split@RESHARD.tmp_1'] + ) + loss_np, shard_data_np = exe.run( + distributed_main_program, + feed={"input": input_data, "label": label_data}, + fetch_list=fetchs, + ) desired = input_data[worker_index].reshape(shard_data_np.shape) np.testing.assert_allclose(shard_data_np, desired) def dp1pp1mp2(self): - def create_model(train_program, start_program): with paddle.static.program_guard(train_program, start_program): @@ -109,7 +116,8 @@ class TestDataUnshard(unittest.TestCase): label = paddle.static.data(name='label', shape=[8, 8]) weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=0.02)) + initializer=nn.initializer.Normal(mean=0.0, std=0.02) + ) linear0 = nn.Linear(8, 8, weight_attr) linear1 = nn.Linear(8, 8, weight_attr) @@ -124,7 +132,8 @@ class TestDataUnshard(unittest.TestCase): linear1_out = linear1(gelu_out) error_cost = paddle.nn.functional.square_error_cost( - linear1_out, label) + linear1_out, label + ) loss = paddle.mean(error_cost) return train_program, start_program, loss, input, label @@ -132,20 +141,27 @@ class TestDataUnshard(unittest.TestCase): start_program = paddle.static.Program() # serial program train_program, start_program, loss, input, label = create_model( - train_program, start_program) + train_program, start_program + ) dist_strategy = fleet.DistributedStrategy() dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) worker_index = paddle.distributed.get_rank() paddle.seed(worker_index + 2021) @@ -160,12 +176,11 @@ class TestDataUnshard(unittest.TestCase): label_data = np.random.randint(0, 10, [8, 8]).astype("float32") fetchs = [loss.name, 'input'] - loss_np, shard_data_np = exe.run(distributed_main_program, - feed={ - "input": input_data, - "label": label_data - }, - fetch_list=fetchs) + loss_np, shard_data_np = exe.run( + distributed_main_program, + feed={"input": input_data, "label": label_data}, + fetch_list=fetchs, + ) desired = input_data.reshape(shard_data_np.shape) np.testing.assert_allclose(shard_data_np, desired) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel_gpt_model.py b/python/paddle/fluid/tests/unittests/auto_parallel_gpt_model.py index 3940aa0170b..c4dc52ebfd2 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel_gpt_model.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel_gpt_model.py @@ -41,20 +41,23 @@ class MultiHeadAttention(nn.Layer): Multi-Head Attention performs multiple parallel attention to jointly attending to information from different representation subspaces. """ + Cache = collections.namedtuple("Cache", ["k", "v"]) StaticCache = collections.namedtuple("StaticCache", ["k", "v"]) - def __init__(self, - embed_dim, - num_heads, - dropout=0., - kdim=None, - vdim=None, - need_weights=False, - weight_attr=None, - bias_attr=None, - fuse=False, - mesh_idx=None): + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + kdim=None, + vdim=None, + need_weights=False, + weight_attr=None, + bias_attr=None, + fuse=False, + mesh_idx=None, + ): super(MultiHeadAttention, self).__init__() self.embed_dim = embed_dim self.kdim = kdim if kdim is not None else embed_dim @@ -65,36 +68,43 @@ class MultiHeadAttention(nn.Layer): self.fuse = fuse self.mesh_idx = mesh_idx self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" if self.fuse: assert self.kdim == embed_dim assert self.vdim == embed_dim - self.qkv_proj = nn.Linear(embed_dim, - 3 * embed_dim, - weight_attr, - bias_attr=bias_attr) + self.qkv_proj = nn.Linear( + embed_dim, 3 * embed_dim, weight_attr, bias_attr=bias_attr + ) else: - self.q_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr=weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - embed_dim, - weight_attr=weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - embed_dim, - weight_attr=weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr=weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + embed_dim, + embed_dim, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) + self.k_proj = nn.Linear( + self.kdim, + embed_dim, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) + self.v_proj = nn.Linear( + self.vdim, + embed_dim, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) + self.out_proj = nn.Linear( + embed_dim, embed_dim, weight_attr=weight_attr, bias_attr=bias_attr + ) def _fuse_prepare_qkv(self, query): mix_layer = self.qkv_proj(query) - mix_layer = paddle.reshape_(mix_layer, - [0, 0, self.num_heads, 3 * self.head_dim]) + mix_layer = paddle.reshape_( + mix_layer, [0, 0, self.num_heads, 3 * self.head_dim] + ) mix_layer = paddle.transpose(mix_layer, [0, 2, 1, 3]) q, k, v = paddle.split(mix_layer, num_or_sections=3, axis=-1) return q, k, v @@ -107,17 +117,21 @@ class MultiHeadAttention(nn.Layer): """ q = self.q_proj(query) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.q_proj.weight, _global_process_mesh, - [None, "x"]) + auto.shard_tensor( + self.q_proj.weight, _global_process_mesh, [None, "x"] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.q_proj.weight, _global_process_mesh, - [None, "y"]) + auto.shard_tensor( + self.q_proj.weight, _global_process_mesh, [None, "y"] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.q_proj.weight, MPPP_MESH_LIST[self.mesh_idx], - [None, "x"]) + auto.shard_tensor( + self.q_proj.weight, MPPP_MESH_LIST[self.mesh_idx], [None, "x"] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.q_proj.weight, - DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"]) + auto.shard_tensor( + self.q_proj.weight, DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"] + ) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) q = tensor.transpose(x=q, perm=[0, 2, 1, 3]) @@ -146,30 +160,38 @@ class MultiHeadAttention(nn.Layer): """ k = self.k_proj(key) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.k_proj.weight, _global_process_mesh, - [None, "x"]) + auto.shard_tensor( + self.k_proj.weight, _global_process_mesh, [None, "x"] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.k_proj.weight, _global_process_mesh, - [None, "y"]) + auto.shard_tensor( + self.k_proj.weight, _global_process_mesh, [None, "y"] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.k_proj.weight, MPPP_MESH_LIST[self.mesh_idx], - [None, "x"]) + auto.shard_tensor( + self.k_proj.weight, MPPP_MESH_LIST[self.mesh_idx], [None, "x"] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.k_proj.weight, - DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"]) + auto.shard_tensor( + self.k_proj.weight, DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"] + ) v = self.v_proj(value) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.v_proj.weight, _global_process_mesh, - [None, "x"]) + auto.shard_tensor( + self.v_proj.weight, _global_process_mesh, [None, "x"] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.v_proj.weight, _global_process_mesh, - [None, "y"]) + auto.shard_tensor( + self.v_proj.weight, _global_process_mesh, [None, "y"] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.v_proj.weight, MPPP_MESH_LIST[self.mesh_idx], - [None, "x"]) + auto.shard_tensor( + self.v_proj.weight, MPPP_MESH_LIST[self.mesh_idx], [None, "x"] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.v_proj.weight, - DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"]) + auto.shard_tensor( + self.v_proj.weight, DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"] + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) v = tensor.reshape(x=v, shape=[0, 0, self.num_heads, self.head_dim]) @@ -190,24 +212,22 @@ class MultiHeadAttention(nn.Layer): input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) v = layers.fill_constant_batch_size_like( input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) return self.Cache(k, v) else: # incremental_state with initial value, mainly for usage like UniLM return self.Cache(key, value) - def forward(self, - query, - key, - value, - attn_mask=None, - use_cache=False, - cache=None): + def forward( + self, query, key, value, attn_mask=None, use_cache=False, cache=None + ): """ Applies multi-head attention to map queries and a set of key-value pairs to outputs. @@ -221,20 +241,22 @@ class MultiHeadAttention(nn.Layer): else: q, k, v = self._prepare_qkv(query, key, value, use_cache, cache) else: - q, k, v, cache = self._prepare_qkv(query, key, value, use_cache, - cache) - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + q, k, v, cache = self._prepare_qkv( + query, key, value, use_cache, cache + ) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if attn_mask is not None: product = product + attn_mask weights = F.softmax(product) if self.dropout: - weights = F.dropout(weights, - self.dropout, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) # combine heads out = tensor.transpose(out, perm=[0, 2, 1, 3]) @@ -242,17 +264,23 @@ class MultiHeadAttention(nn.Layer): # project to output out = self.out_proj(out) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.out_proj.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.out_proj.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.out_proj.weight, _global_process_mesh, - ["y", None]) + auto.shard_tensor( + self.out_proj.weight, _global_process_mesh, ["y", None] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.out_proj.weight, - MPPP_MESH_LIST[self.mesh_idx], ["x", None]) + auto.shard_tensor( + self.out_proj.weight, MPPP_MESH_LIST[self.mesh_idx], ["x", None] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.out_proj.weight, - DPMPPP_MESH_LIST[self.mesh_idx], ["y", None]) + auto.shard_tensor( + self.out_proj.weight, + DPMPPP_MESH_LIST[self.mesh_idx], + ["y", None], + ) outs = [out] if self.need_weights: @@ -279,13 +307,15 @@ class TransformerDecoder(nn.Layer): raise ValueError("Only support LayerNorm") self.checkpoints = [] - def forward(self, - tgt, - memory, - tgt_mask=None, - memory_mask=None, - use_cache=False, - cache=None): + def forward( + self, + tgt, + memory, + tgt_mask=None, + memory_mask=None, + use_cache=False, + cache=None, + ): """ Applies a stack of N Transformer decoder layers on inputs. If `norm` is provided, also applies layer normalization on the output of last decoder @@ -295,134 +325,170 @@ class TransformerDecoder(nn.Layer): new_caches = [] self.checkpoints = [] if _global_parallel_strategy == "pp": - auto.shard_tensor(output, PP_MESH_LIST[0], - [None for i in range(len(output.shape))]) + auto.shard_tensor( + output, + PP_MESH_LIST[0], + [None for i in range(len(output.shape))], + ) if _global_parallel_strategy == "dp_pp": - auto.shard_tensor(output, DPPP_MESH_LIST[0], ["x"] + - [None for i in range(len(output.shape) - 1)]) + auto.shard_tensor( + output, + DPPP_MESH_LIST[0], + ["x"] + [None for i in range(len(output.shape) - 1)], + ) if _global_parallel_strategy == "mp_pp": - auto.shard_tensor(output, MPPP_MESH_LIST[0], - [None for i in range(len(output.shape))]) + auto.shard_tensor( + output, + MPPP_MESH_LIST[0], + [None for i in range(len(output.shape))], + ) if _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(output, DPMPPP_MESH_LIST[0], ["x"] + - [None for i in range(len(output.shape) - 1)]) + auto.shard_tensor( + output, + DPMPPP_MESH_LIST[0], + ["x"] + [None for i in range(len(output.shape) - 1)], + ) for i, mod in enumerate(self.layers): if cache is None: if use_cache: if _global_parallel_strategy == "pp": output, new_cache = auto.shard_op( - mod, PP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, PP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, PP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + PP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_pp": output, new_cache = auto.shard_op( - mod, DPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, DPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPPP_MESH_LIST[mod.mesh_idx], ["x"] + - [None for i in range(len(output.shape) - 1)]) + output, + DPPP_MESH_LIST[mod.mesh_idx], + ["x"] + + [None for i in range(len(output.shape) - 1)], + ) elif _global_parallel_strategy == "mp_pp": output, new_cache = auto.shard_op( - mod, MPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, MPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, MPPP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + MPPP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_mp_pp": output, new_cache = auto.shard_op( - mod, - DPMPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, use_cache, - cache) + mod, DPMPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPMPPP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + DPMPPP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) new_caches.append(new_cache) else: if _global_parallel_strategy == "pp": output = auto.shard_op(mod, PP_MESH_LIST[mod.mesh_idx])( - output, memory, tgt_mask, use_cache, cache) + output, memory, tgt_mask, use_cache, cache + ) auto.shard_tensor( - output, PP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + PP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_pp": output = auto.shard_op( - mod, DPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, DPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPPP_MESH_LIST[mod.mesh_idx], ["x"] + - [None for i in range(len(output.shape) - 1)]) + output, + DPPP_MESH_LIST[mod.mesh_idx], + ["x"] + + [None for i in range(len(output.shape) - 1)], + ) elif _global_parallel_strategy == "mp_pp": output = auto.shard_op( - mod, MPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, MPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, MPPP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + MPPP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_mp_pp": - output = auto.shard_op(mod, - DPMPPP_MESH_LIST[mod.mesh_idx])( - output, memory, tgt_mask, - use_cache, cache) + output = auto.shard_op( + mod, DPMPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPMPPP_MESH_LIST[mod.mesh_idx], ["x"] + - [None for i in range(len(output.shape) - 1)]) + output, + DPMPPP_MESH_LIST[mod.mesh_idx], + ["x"] + + [None for i in range(len(output.shape) - 1)], + ) else: - output = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) else: if _global_parallel_strategy == "pp": output, new_cache = auto.shard_op( - mod, - PP_MESH_LIST[mod.mesh_idx])(output, memory, tgt_mask, - use_cache, cache) - auto.shard_tensor(output, PP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + mod, PP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) + auto.shard_tensor( + output, + PP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_pp": output, new_cache = auto.shard_op( - mod, - DPPP_MESH_LIST[mod.mesh_idx])(output, memory, tgt_mask, - use_cache, cache) + mod, DPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPPP_MESH_LIST[mod.mesh_idx], - ["x"] + [None for i in range(len(output.shape) - 1)]) + output, + DPPP_MESH_LIST[mod.mesh_idx], + ["x"] + [None for i in range(len(output.shape) - 1)], + ) elif _global_parallel_strategy == "mp_pp": output, new_cache = auto.shard_op( - mod, - MPPP_MESH_LIST[mod.mesh_idx])(output, memory, tgt_mask, - use_cache, cache) - auto.shard_tensor(output, MPPP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + mod, MPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) + auto.shard_tensor( + output, + MPPP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_mp_pp": output, new_cache = auto.shard_op( - mod, DPMPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, DPMPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPMPPP_MESH_LIST[mod.mesh_idx], - ["x"] + [None for i in range(len(output.shape) - 1)]) + output, + DPMPPP_MESH_LIST[mod.mesh_idx], + ["x"] + [None for i in range(len(output.shape) - 1)], + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache[i]) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache[i], + ) new_caches.append(new_cache) self.checkpoints.append(output.name) if self.norm is not None: @@ -436,7 +502,7 @@ class TransformerDecoder(nn.Layer): produced by `TransformerDecoderLayer.gen_cache`. See `TransformerDecoderLayer.gen_cache` for more details. If `do_zip` is True, apply `zip` on these tuples to get a list with two elements. - """ + """ cache = [layer.gen_cache(memory) for layer in self.layers] if do_zip: cache = list(zip(*cache)) @@ -449,18 +515,20 @@ class TransformerDecoderLayer(nn.Layer): It contains multiheadattention and some linear layers. """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="gelu", - attn_dropout=None, - act_dropout=None, - normalize_before=True, - weight_attr=None, - bias_attr=None, - mesh_idx=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="gelu", + attn_dropout=None, + act_dropout=None, + normalize_before=True, + weight_attr=None, + bias_attr=None, + mesh_idx=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 @@ -471,20 +539,20 @@ class TransformerDecoderLayer(nn.Layer): self.normalize_before = normalize_before weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0], - mesh_idx=self.mesh_idx) - self.linear1 = nn.Linear(d_model, - dim_feedforward, - weight_attrs[2], - bias_attr=bias_attrs[2]) - self.linear2 = nn.Linear(dim_feedforward, - d_model, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + mesh_idx=self.mesh_idx, + ) + self.linear1 = nn.Linear( + d_model, dim_feedforward, weight_attrs[2], bias_attr=bias_attrs[2] + ) + self.linear2 = nn.Linear( + dim_feedforward, d_model, weight_attrs[2], bias_attr=bias_attrs[2] + ) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout1 = nn.Dropout(dropout, mode="upscale_in_train") @@ -498,8 +566,9 @@ class TransformerDecoderLayer(nn.Layer): if use_cache is False: tgt = self.self_attn(tgt, tgt, tgt, tgt_mask, use_cache, cache) else: - tgt, incremental_cache = self.self_attn(tgt, tgt, tgt, tgt_mask, - use_cache, cache) + tgt, incremental_cache = self.self_attn( + tgt, tgt, tgt, tgt_mask, use_cache, cache + ) tgt = residual + self.dropout1(tgt) if not self.normalize_before: tgt = self.norm1(tgt) @@ -507,40 +576,54 @@ class TransformerDecoderLayer(nn.Layer): if self.normalize_before: tgt = self.norm2(tgt) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, "x"]) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, "x"] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, "y"]) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, "y"] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.linear1.weight, - MPPP_MESH_LIST[self.mesh_idx], [None, "x"]) + auto.shard_tensor( + self.linear1.weight, MPPP_MESH_LIST[self.mesh_idx], [None, "x"] + ) if _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.linear1.weight, - DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"]) + auto.shard_tensor( + self.linear1.weight, + DPMPPP_MESH_LIST[self.mesh_idx], + [None, "y"], + ) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.linear2.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.linear2.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.linear2.weight, _global_process_mesh, - ["y", None]) + auto.shard_tensor( + self.linear2.weight, _global_process_mesh, ["y", None] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.linear2.weight, - MPPP_MESH_LIST[self.mesh_idx], ["x", None]) + auto.shard_tensor( + self.linear2.weight, MPPP_MESH_LIST[self.mesh_idx], ["x", None] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.linear2.weight, - DPMPPP_MESH_LIST[self.mesh_idx], ["y", None]) + auto.shard_tensor( + self.linear2.weight, + DPMPPP_MESH_LIST[self.mesh_idx], + ["y", None], + ) tgt = self.dropout2( - self.linear2(F.gelu(self.linear1(tgt), approximate=True))) + self.linear2(F.gelu(self.linear1(tgt), approximate=True)) + ) tgt = residual + tgt if not self.normalize_before: tgt = self.norm2(tgt) return tgt if use_cache is False else (tgt, incremental_cache) def gen_cache(self, memory): - incremental_cache = self.self_attn.gen_cache(memory, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + memory, type=self.self_attn.Cache + ) return incremental_cache @@ -549,26 +632,36 @@ class GPTEmbeddings(nn.Layer): Include embeddings from word, position and token_type embeddings """ - def __init__(self, - vocab_size, - hidden_size=768, - hidden_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02): + def __init__( + self, + vocab_size, + hidden_size=768, + hidden_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + ): super(GPTEmbeddings, self).__init__() self.word_embeddings = nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( max_position_embeddings, hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.dropout = nn.Dropout(hidden_dropout_prob) def forward(self, input_ids, position_ids=None): @@ -578,17 +671,21 @@ class GPTEmbeddings(nn.Layer): position_ids = seq_length - ones input_embedings = self.word_embeddings(input_ids) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.word_embeddings.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.word_embeddings.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.word_embeddings.weight, _global_process_mesh, - ["y", None]) + auto.shard_tensor( + self.word_embeddings.weight, _global_process_mesh, ["y", None] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.word_embeddings.weight, MPPP_MESH_LIST[0], - ["x", None]) + auto.shard_tensor( + self.word_embeddings.weight, MPPP_MESH_LIST[0], ["x", None] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.word_embeddings.weight, DPMPPP_MESH_LIST[0], - ["y", None]) + auto.shard_tensor( + self.word_embeddings.weight, DPMPPP_MESH_LIST[0], ["y", None] + ) position_embeddings = self.position_embeddings(position_ids) embeddings = input_embedings + position_embeddings @@ -601,36 +698,42 @@ class GPTModel(nn.Layer): The base model of gpt. """ - def __init__(self, - vocab_size=50304, - hidden_size=1024, - num_hidden_layers=24, - num_attention_heads=16, - intermediate_size=4096, - hidden_act="gelu", - hidden_dropout_prob=0., - attention_probs_dropout_prob=0., - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3, - pp_degree=None): + def __init__( + self, + vocab_size=50304, + hidden_size=1024, + num_hidden_layers=24, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + pp_degree=None, + ): super(GPTModel, self).__init__() self.pad_token_id = pad_token_id self.initializer_range = initializer_range self.hidden_size = hidden_size self.vocab_size = vocab_size self.layer_per_stage = None - self.pipline_mode = (pp_degree is not None and pp_degree > 1) + self.pipline_mode = pp_degree is not None and pp_degree > 1 if self.pipline_mode: self.layer_per_stage = num_hidden_layers // pp_degree - self.embeddings = GPTEmbeddings(vocab_size, hidden_size, - hidden_dropout_prob, - max_position_embeddings, - type_vocab_size, self.initializer_range) + self.embeddings = GPTEmbeddings( + vocab_size, + hidden_size, + hidden_dropout_prob, + max_position_embeddings, + type_vocab_size, + self.initializer_range, + ) decoder_layers = nn.LayerList() for i in range(num_hidden_layers): mesh_index = None @@ -638,59 +741,82 @@ class GPTModel(nn.Layer): if self.layer_per_stage is not None: mesh_index = i // self.layer_per_stage decoder_layers.append( - DecoderLayer(d_model=hidden_size, - nhead=num_attention_heads, - dim_feedforward=intermediate_size, - dropout=hidden_dropout_prob, - activation=hidden_act, - attn_dropout=attention_probs_dropout_prob, - act_dropout=hidden_dropout_prob, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)), - bias_attr=None, - mesh_idx=mesh_index)) + DecoderLayer( + d_model=hidden_size, + nhead=num_attention_heads, + dim_feedforward=intermediate_size, + dropout=hidden_dropout_prob, + activation=hidden_act, + attn_dropout=attention_probs_dropout_prob, + act_dropout=hidden_dropout_prob, + weight_attr=paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ), + bias_attr=None, + mesh_idx=mesh_index, + ) + ) Decoder = TransformerDecoder - self.decoder = Decoder(decoder_layers, - num_hidden_layers, - norm="LayerNorm", - hidden_size=hidden_size) + self.decoder = Decoder( + decoder_layers, + num_hidden_layers, + norm="LayerNorm", + hidden_size=hidden_size, + ) self.checkpoints = [] - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - use_cache=False, - cache=None): + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + use_cache=False, + cache=None, + ): self.checkpoints = [] if position_ids is None: past_length = 0 if cache is not None: past_length = paddle.shape(cache[0].k)[-2] - position_ids = paddle.arange(past_length, - paddle.shape(input_ids)[-1] + - past_length, - dtype='int64') + position_ids = paddle.arange( + past_length, + paddle.shape(input_ids)[-1] + past_length, + dtype='int64', + ) position_ids = position_ids.unsqueeze(0) position_ids = paddle.fluid.layers.expand_as( - position_ids, input_ids) - embedding_output = self.embeddings(input_ids=input_ids, - position_ids=position_ids) + position_ids, input_ids + ) + embedding_output = self.embeddings( + input_ids=input_ids, position_ids=position_ids + ) if _global_parallel_strategy == "pp": - auto.shard_tensor(input_ids, PP_MESH_LIST[0], - [None for i in range(len(input_ids.shape))]) + auto.shard_tensor( + input_ids, + PP_MESH_LIST[0], + [None for i in range(len(input_ids.shape))], + ) if _global_parallel_strategy == "dp_pp": - auto.shard_tensor(input_ids, DPPP_MESH_LIST[0], ["x"] + - [None for i in range(len(input_ids.shape) - 1)]) + auto.shard_tensor( + input_ids, + DPPP_MESH_LIST[0], + ["x"] + [None for i in range(len(input_ids.shape) - 1)], + ) if _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(input_ids, DPMPPP_MESH_LIST[0], ["x"] + - [None for i in range(len(input_ids.shape) - 1)]) - encoder_outputs = self.decoder(embedding_output, - memory=None, - tgt_mask=attention_mask, - use_cache=use_cache, - cache=cache) + auto.shard_tensor( + input_ids, + DPMPPP_MESH_LIST[0], + ["x"] + [None for i in range(len(input_ids.shape) - 1)], + ) + encoder_outputs = self.decoder( + embedding_output, + memory=None, + tgt_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) self.checkpoints.extend(self.decoder.checkpoints) return encoder_outputs @@ -711,22 +837,26 @@ class GPTForPretraining(nn.Layer): super(GPTForPretraining, self).__init__() self.gpt = gpt - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - masked_positions=None, - use_cache=False, - cache=None): + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + masked_positions=None, + use_cache=False, + cache=None, + ): input_ids.stop_gradient = True position_ids.stop_gradient = True attention_mask.stop_gradient = True - outputs = self.gpt(input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - use_cache=use_cache, - cache=cache) + outputs = self.gpt( + input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) if use_cache: encoder_outputs, cached_kvs = outputs[:2] else: @@ -766,8 +896,9 @@ class GPTForPretraining(nn.Layer): w_dims_mapping = ["y"] + [None for i in range(len(w.shape) - 1)] if mesh: - matmul = auto.shard_op(paddle.matmul, mesh, - [x_dims_mapping, w_dims_mapping, None]) + matmul = auto.shard_op( + paddle.matmul, mesh, [x_dims_mapping, w_dims_mapping, None] + ) logits = matmul(x, w, transpose_y=True) else: logits = paddle.matmul(x, w, transpose_y=True) @@ -795,26 +926,31 @@ class GPTPretrainingCriterion(nn.Layer): mesh = None if _global_parallel_strategy == "dp": mesh = _global_process_mesh - dims_mapping = ["x" - ] + [None for i in range(len(loss_mask.shape) - 1)] + dims_mapping = ["x"] + [ + None for i in range(len(loss_mask.shape) - 1) + ] elif _global_parallel_strategy == "dp_mp": mesh = _global_process_mesh - dims_mapping = ["x" - ] + [None for i in range(len(loss_mask.shape) - 1)] + dims_mapping = ["x"] + [ + None for i in range(len(loss_mask.shape) - 1) + ] elif _global_parallel_strategy == "dp_pp": mesh = DPPP_MESH_LIST[-1] - dims_mapping = ["x" - ] + [None for i in range(len(loss_mask.shape) - 1)] + dims_mapping = ["x"] + [ + None for i in range(len(loss_mask.shape) - 1) + ] elif _global_parallel_strategy == "dp_mp_pp": mesh = DPMPPP_MESH_LIST[-1] - dims_mapping = ["x" - ] + [None for i in range(len(loss_mask.shape) - 1)] + dims_mapping = ["x"] + [ + None for i in range(len(loss_mask.shape) - 1) + ] if mesh: auto.shard_tensor(loss_mask, mesh, dims_mapping) - masked_lm_loss = self.loss_func(prediction_scores, - masked_lm_labels.unsqueeze(2)) + masked_lm_loss = self.loss_func( + prediction_scores, masked_lm_labels.unsqueeze(2) + ) loss_mask = loss_mask.reshape([-1]) masked_lm_loss = paddle.sum(masked_lm_loss.reshape([-1]) * loss_mask) total_loss = masked_lm_loss / loss_mask.sum() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel_save_load.py b/python/paddle/fluid/tests/unittests/auto_parallel_save_load.py index 0fc773cc9bc..ce8bc690a95 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel_save_load.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel_save_load.py @@ -27,7 +27,10 @@ from paddle.distributed.fleet import auto from paddle.distributed import fleet from paddle.fluid.initializer import NumpyArrayInitializer -from paddle.distributed.auto_parallel.utils import save_distributed_checkpoint, load_checkpoint_into_program +from paddle.distributed.auto_parallel.utils import ( + save_distributed_checkpoint, + load_checkpoint_into_program, +) paddle.enable_static() _global_parallel_strategy = None @@ -37,11 +40,9 @@ PP_MESH_1 = None class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=64, - intermediate_size=4 * 64, - initializer_range=0.02): + def __init__( + self, hidden_size=64, intermediate_size=4 * 64, initializer_range=0.02 + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size @@ -50,14 +51,12 @@ class MLPLayer(nn.Layer): weight_attr = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr)) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -65,15 +64,19 @@ class MLPLayer(nn.Layer): auto.shard_tensor(self.linear0.weight, PP_MESH_0, [None, None]) auto.shard_tensor(self.linear1.weight, PP_MESH_1, [None, None]) elif _global_parallel_strategy == "mp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, "x"]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, "x"] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, None]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, None] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, None] + ) out = self.norm(input) out = self.linear0(out) @@ -84,17 +87,18 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program,start_program), \ - utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 64 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "pp": auto.shard_tensor(input, PP_MESH_0, [None, None]) @@ -104,9 +108,11 @@ def mlp_forward(train_program, start_program): elif _global_parallel_strategy == "mp": auto.shard_tensor(input, _global_process_mesh, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -123,19 +129,20 @@ def get_distributed_program(): dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) optimizer = paddle.fluid.optimizer.SGDOptimizer(learning_rate=0.01) optimizer = fleet.distributed_optimizer(optimizer) _, _, dist_startup_prog, dist_main_prog = optimizer.minimize( - loss, startup_program) + loss, startup_program + ) return dist_main_prog, dist_startup_prog, loss class TestMLPSaveLoad(unittest.TestCase): - def setUp(self): paddle.seed(2021) random.seed(2021) @@ -160,30 +167,34 @@ class TestMLPSaveLoad(unittest.TestCase): os.makedirs(path, exist_ok=True) save_distributed_checkpoint(dist_main_prog, path, path) - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) last_res = res[0] ckpt_path = [ "./output_dp0/model_state_rank0.pdmodel", - "./output_dp1/model_state_rank1.pdmodel" + "./output_dp1/model_state_rank1.pdmodel", ] dist_attr_path = [ "./output_dp0/dist_attr_rank0.pdattr", - "./output_dp1/dist_attr_rank1.pdattr" + "./output_dp1/dist_attr_rank1.pdattr", ] load_checkpoint_into_program(ckpt_path, dist_attr_path, dist_main_prog) for step in range(10, 20): - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) self.assertEqual(last_res, res[0]) shutil.rmtree("./output_dp{}".format(paddle.distributed.get_rank())) @@ -208,30 +219,34 @@ class TestMLPSaveLoad(unittest.TestCase): os.makedirs(path, exist_ok=True) save_distributed_checkpoint(dist_main_prog, path, path) - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) last_res = res[0] ckpt_path = [ "./output_mp0/model_state_rank0.pdmodel", - "./output_mp1/model_state_rank1.pdmodel" + "./output_mp1/model_state_rank1.pdmodel", ] dist_attr_path = [ "./output_mp0/dist_attr_rank0.pdattr", - "./output_mp1/dist_attr_rank1.pdattr" + "./output_mp1/dist_attr_rank1.pdattr", ] load_checkpoint_into_program(ckpt_path, dist_attr_path, dist_main_prog) for step in range(10, 20): - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) self.assertEqual(last_res, res[0]) shutil.rmtree("./output_mp{}".format(paddle.distributed.get_rank())) @@ -261,45 +276,53 @@ class TestMLPSaveLoad(unittest.TestCase): save_distributed_checkpoint(dist_main_prog, path, path) if paddle.distributed.get_rank() in [0]: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + ) else: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) if paddle.distributed.get_rank() in [1]: last_res = res[0] ckpt_path = [ "./output_pp0/model_state_rank0.pdmodel", - "./output_pp1/model_state_rank1.pdmodel" + "./output_pp1/model_state_rank1.pdmodel", ] dist_attr_path = [ "./output_pp0/dist_attr_rank0.pdattr", - "./output_pp1/dist_attr_rank1.pdattr" + "./output_pp1/dist_attr_rank1.pdattr", ] load_checkpoint_into_program(ckpt_path, dist_attr_path, dist_main_prog) for step in range(10, 20): if paddle.distributed.get_rank() in [0]: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + ) else: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) if paddle.distributed.get_rank() in [1]: self.assertEqual(last_res, res[0]) diff --git a/python/paddle/fluid/tests/unittests/autograd/config.py b/python/paddle/fluid/tests/unittests/autograd/config.py index 311ca49d395..ff2d64a43bb 100644 --- a/python/paddle/fluid/tests/unittests/autograd/config.py +++ b/python/paddle/fluid/tests/unittests/autograd/config.py @@ -23,27 +23,11 @@ DEFAULT_DTYPE = 'float64' # derivative. It's a empirical value provided by Paddle Science team. TOLERANCE = { "float32": { - "first_order_grad": { - "rtol": 1e-3, - "atol": 1e-3, - "eps": 1e-4 - }, - "second_order_grad": { - "rtol": 1e-2, - "atol": 1e-2, - "eps": 1e-2 - } + "first_order_grad": {"rtol": 1e-3, "atol": 1e-3, "eps": 1e-4}, + "second_order_grad": {"rtol": 1e-2, "atol": 1e-2, "eps": 1e-2}, }, "float64": { - "first_order_grad": { - "rtol": 1e-7, - "atol": 1e-7, - "eps": 1e-7 - }, - "second_order_grad": { - "rtol": 1e-5, - "atol": 1e-5, - "eps": 1e-5 - } - } + "first_order_grad": {"rtol": 1e-7, "atol": 1e-7, "eps": 1e-7}, + "second_order_grad": {"rtol": 1e-5, "atol": 1e-5, "eps": 1e-5}, + }, } diff --git a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_dynamic.py b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_dynamic.py index 054edf2ea28..81f1bdede6f 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_dynamic.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_dynamic.py @@ -34,7 +34,6 @@ def make_v(f, inputs): class TestAutogradFunctional(unittest.TestCase): - @classmethod def setUpClass(cls): cls.RAW_INPUTS = { @@ -52,8 +51,9 @@ class TestAutogradFunctional(unittest.TestCase): def gen_input(self, inp, stop_gradient=False): if isinstance(inp, paddle.Tensor): return inp - return paddle.to_tensor(self.RAW_INPUTS[inp], - stop_gradient=stop_gradient) + return paddle.to_tensor( + self.RAW_INPUTS[inp], stop_gradient=stop_gradient + ) def gen_inputs(self, inputs): if isinstance(inputs, list): @@ -62,13 +62,9 @@ class TestAutogradFunctional(unittest.TestCase): inputs = [self.gen_input(inputs)] return inputs - def gen_test_pairs(self, - func, - inputs, - v=None, - create_graph=False, - allow_unused=False): - + def gen_test_pairs( + self, func, inputs, v=None, create_graph=False, allow_unused=False + ): def vjp_test(): nonlocal v xs = self.gen_inputs(inputs) @@ -86,27 +82,27 @@ class TestAutogradFunctional(unittest.TestCase): v = self.gen_inputs(v) outputs = func(*xs) if v is not None: - inputs_grad = paddle.grad(outputs, - xs, - v, - create_graph=create_graph, - allow_unused=allow_unused) + inputs_grad = paddle.grad( + outputs, + xs, + v, + create_graph=create_graph, + allow_unused=allow_unused, + ) else: - inputs_grad = paddle.grad(outputs, - xs, - create_graph=create_graph, - allow_unused=allow_unused) + inputs_grad = paddle.grad( + outputs, + xs, + create_graph=create_graph, + allow_unused=allow_unused, + ) return outputs, inputs_grad return vjp_test, grad_test - def gen_jvp_tests(self, - func, - inputs, - v=None, - create_graph=False, - allow_unused=False): - + def gen_jvp_tests( + self, func, inputs, v=None, create_graph=False, allow_unused=False + ): def jvp_test(): nonlocal v xs = self.gen_inputs(inputs) @@ -117,13 +113,15 @@ class TestAutogradFunctional(unittest.TestCase): xs, v, create_graph=create_graph, - allow_unused=allow_unused) + allow_unused=allow_unused, + ) else: outputs, outputs_grad = paddle.incubate.autograd.jvp( func, xs, create_graph=create_graph, - allow_unused=allow_unused) + allow_unused=allow_unused, + ) return outputs, outputs_grad return jvp_test @@ -144,7 +142,6 @@ class TestAutogradFunctional(unittest.TestCase): class TestVJP(TestAutogradFunctional): - def func_vjp_i1o1(self): test_cases = [ [reduce, 'A'], # noqa @@ -222,20 +219,29 @@ class TestVJP(TestAutogradFunctional): def test_input_single_tensor(self): self.assertIsInstance( paddle.incubate.autograd.vjp(paddle.tanh, paddle.rand((3, 4)))[1], - paddle.fluid.framework.Variable) + paddle.fluid.framework.Variable, + ) @utils.place(config.DEVICES) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'expected_exception'), - (('v_shape_not_equal_ys', utils.square, np.random.rand(3), - np.random.rand(1), RuntimeError), )) + ( + ( + 'v_shape_not_equal_ys', + utils.square, + np.random.rand(3), + np.random.rand(1), + RuntimeError, + ), + ), +) class TestVJPException(unittest.TestCase): - def func_vjp(self): with self.assertRaises(self.expected_exception): - paddle.incubate.autograd.vjp(self.fun, paddle.to_tensor(self.xs), - paddle.to_tensor(self.v)) + paddle.incubate.autograd.vjp( + self.fun, paddle.to_tensor(self.xs), paddle.to_tensor(self.v) + ) def test_all_cases(self): with _test_eager_guard(): @@ -245,7 +251,8 @@ class TestVJPException(unittest.TestCase): def jac(grad_fn, f, inputs): assert grad_fn in [ - paddle.incubate.autograd.vjp, paddle.incubate.autograd.jvp + paddle.incubate.autograd.vjp, + paddle.incubate.autograd.jvp, ] if grad_fn is paddle.incubate.autograd.jvp: vs = [paddle.zeros_like(x) for x in inputs] @@ -274,7 +281,6 @@ def jac(grad_fn, f, inputs): class TestJVP(TestAutogradFunctional): - def func_jvp_i1o1(self): test_cases = [ [reduce, 'A'], # noqa @@ -330,36 +336,58 @@ class TestJVP(TestAutogradFunctional): @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'func', 'xs'), ( - ('1d_in_1d_out', utils.square, np.array([2., 3.])), - ('3d_in_3d_out', utils.square, np.random.rand(2, 3, 4)), - ('single_in_single_out', utils.square, np.random.rand(2, 3)), - ('multi_in_single_out', paddle.matmul, - (np.random.rand(2, 2), np.random.rand(2, 2))), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'func', 'xs'), + ( + ('1d_in_1d_out', utils.square, np.array([2.0, 3.0])), + ('3d_in_3d_out', utils.square, np.random.rand(2, 3, 4)), + ('single_in_single_out', utils.square, np.random.rand(2, 3)), + ( + 'multi_in_single_out', + paddle.matmul, + (np.random.rand(2, 2), np.random.rand(2, 2)), + ), + ), +) class TestJacobianNoBatch(unittest.TestCase): - def setUp(self): - self._dtype = self.xs[0].dtype if isinstance( - self.xs, typing.Sequence) else self.xs.dtype - self._eps = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("eps") - self._rtol = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("rtol") - self._atol = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("atol") + self._dtype = ( + self.xs[0].dtype + if isinstance(self.xs, typing.Sequence) + else self.xs.dtype + ) + self._eps = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("eps") + ) + self._rtol = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("rtol") + ) + self._atol = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("atol") + ) def func_jacobian(self): - xs = [paddle.to_tensor(x) for x in self.xs] if isinstance( - self.xs, typing.Sequence) else paddle.to_tensor(self.xs) + xs = ( + [paddle.to_tensor(x) for x in self.xs] + if isinstance(self.xs, typing.Sequence) + else paddle.to_tensor(self.xs) + ) self._actual = paddle.incubate.autograd.Jacobian(self.func, xs, False) self._expected = self._get_expected() Index = collections.namedtuple('Index', ('type', 'value')) - indexes = (Index('all', (slice(0, None, None), slice(0, None, None))), - Index('row', (0, slice(0, None, None))), - Index('col', (slice(0, None, None), 0)), - Index('multi-row', (slice(0, 2, 1), slice(0, None, None)))) + indexes = ( + Index('all', (slice(0, None, None), slice(0, None, None))), + Index('row', (0, slice(0, None, None))), + Index('col', (slice(0, None, None), 0)), + Index('multi-row', (slice(0, 2, 1), slice(0, None, None))), + ) self.assertEqual(self._actual[:].numpy().dtype, self._expected.dtype) for index in indexes: np.testing.assert_allclose( @@ -367,15 +395,18 @@ class TestJacobianNoBatch(unittest.TestCase): self._expected.__getitem__(index.value), rtol=self._rtol, atol=self._atol, - err_msg= - f'Testcase {index.type} index not passed, value is {index.value}' + err_msg=f'Testcase {index.type} index not passed, value is {index.value}', ) def _get_expected(self): - xs = [paddle.to_tensor(x) for x in self.xs] if isinstance( - self.xs, typing.Sequence) else paddle.to_tensor(self.xs) - jac = utils._compute_numerical_jacobian(self.func, xs, self._eps, - self._dtype) + xs = ( + [paddle.to_tensor(x) for x in self.xs] + if isinstance(self.xs, typing.Sequence) + else paddle.to_tensor(self.xs) + ) + jac = utils._compute_numerical_jacobian( + self.func, xs, self._eps, self._dtype + ) return utils._np_concat_matrix_sequence(jac, utils.MatrixFormat.NM) def test_all_cases(self): @@ -385,42 +416,71 @@ class TestJacobianNoBatch(unittest.TestCase): @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'func', 'xs'), ( - ('1d_in_1d_out', utils.square, np.array([[1., 2., 3.], [3., 4., 3.]])), - ('3d_in_3d_out', utils.square, np.random.rand(2, 3, 4)), - ('multi_in_single_out', utils.square, np.random.rand(2, 3)), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'func', 'xs'), + ( + ( + '1d_in_1d_out', + utils.square, + np.array([[1.0, 2.0, 3.0], [3.0, 4.0, 3.0]]), + ), + ('3d_in_3d_out', utils.square, np.random.rand(2, 3, 4)), + ('multi_in_single_out', utils.square, np.random.rand(2, 3)), + ), +) class TestJacobianBatchFirst(unittest.TestCase): - def setUp(self): - self._dtype = self.xs[0].dtype if isinstance( - self.xs, typing.Sequence) else self.xs.dtype - self._eps = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("eps") - self._rtol = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("rtol") - self._atol = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("atol") + self._dtype = ( + self.xs[0].dtype + if isinstance(self.xs, typing.Sequence) + else self.xs.dtype + ) + self._eps = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("eps") + ) + self._rtol = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("rtol") + ) + self._atol = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("atol") + ) def func_jacobian(self): - xs = [paddle.to_tensor(x) for x in self.xs] if isinstance( - self.xs, typing.Sequence) else paddle.to_tensor(self.xs) + xs = ( + [paddle.to_tensor(x) for x in self.xs] + if isinstance(self.xs, typing.Sequence) + else paddle.to_tensor(self.xs) + ) self._actual = paddle.incubate.autograd.Jacobian(self.func, xs, True) self._expected = self._get_expected() Index = collections.namedtuple('Index', ('type', 'value')) - indexes = (Index( - 'all', - (slice(0, None, None), slice(0, None, None), slice(0, None, None))), - Index('row', - (slice(0, None, None), 0, slice(0, None, None))), - Index('col', - (slice(0, None, None), slice(0, None, None), 0)), - Index('batch', (slice(0, 2, None), slice( - 0, None, None), slice(0, None, None))), - Index('multi_row', - (slice(0, 1, None), slice(0, 2, 1), slice( - 0, None, None)))) + indexes = ( + Index( + 'all', + ( + slice(0, None, None), + slice(0, None, None), + slice(0, None, None), + ), + ), + Index('row', (slice(0, None, None), 0, slice(0, None, None))), + Index('col', (slice(0, None, None), slice(0, None, None), 0)), + Index( + 'batch', + (slice(0, 2, None), slice(0, None, None), slice(0, None, None)), + ), + Index( + 'multi_row', + (slice(0, 1, None), slice(0, 2, 1), slice(0, None, None)), + ), + ) self.assertEqual(self._actual[:].numpy().dtype, self._expected.dtype) for index in indexes: np.testing.assert_allclose( @@ -428,18 +488,22 @@ class TestJacobianBatchFirst(unittest.TestCase): self._expected.__getitem__(index.value), rtol=self._rtol, atol=self._atol, - err_msg= - f'Testcase {index.type} index not passed, value is {index.value}' + err_msg=f'Testcase {index.type} index not passed, value is {index.value}', ) def _get_expected(self): - xs = [paddle.to_tensor(x) for x in self.xs] if isinstance( - self.xs, typing.Sequence) else paddle.to_tensor(self.xs) - jac = utils._compute_numerical_batch_jacobian(self.func, xs, self._eps, - self._dtype, False) + xs = ( + [paddle.to_tensor(x) for x in self.xs] + if isinstance(self.xs, typing.Sequence) + else paddle.to_tensor(self.xs) + ) + jac = utils._compute_numerical_batch_jacobian( + self.func, xs, self._eps, self._dtype, False + ) jac = utils._np_concat_matrix_sequence(jac, utils.MatrixFormat.NBM) - return utils._np_transpose_matrix_format(jac, utils.MatrixFormat.NBM, - utils.MatrixFormat.BNM) + return utils._np_transpose_matrix_format( + jac, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM + ) def test_all_cases(self): with _test_eager_guard(): @@ -448,64 +512,74 @@ class TestJacobianBatchFirst(unittest.TestCase): class TestHessianNoBatch(unittest.TestCase): - @classmethod def setUpClass(self): self.shape = (2, 2) self.dtype = 'float32' self.np_dtype = np.float32 - self.numerical_delta = config.TOLERANCE.get( - self.dtype).get("second_order_grad").get("eps") - self.rtol = config.TOLERANCE.get( - self.dtype).get("second_order_grad").get("rtol") - self.atol = config.TOLERANCE.get( - self.dtype).get("second_order_grad").get("atol") + self.numerical_delta = ( + config.TOLERANCE.get(self.dtype).get("second_order_grad").get("eps") + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype) + .get("second_order_grad") + .get("rtol") + ) + self.atol = ( + config.TOLERANCE.get(self.dtype) + .get("second_order_grad") + .get("atol") + ) self.x = paddle.rand(shape=self.shape, dtype=self.dtype) self.y = paddle.rand(shape=self.shape, dtype=self.dtype) def func_single_input(self): - def func(x): return paddle.sum(paddle.matmul(x, x)) numerical_hessian = utils._compute_numerical_hessian( - func, self.x, self.numerical_delta, self.np_dtype) + func, self.x, self.numerical_delta, self.np_dtype + ) numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian) self.x.stop_gradient = False hessian = paddle.incubate.autograd.Hessian(func, self.x) - np.testing.assert_allclose(hessian[:].numpy(), numerical_hessian, - self.rtol, self.atol) + np.testing.assert_allclose( + hessian[:].numpy(), numerical_hessian, self.rtol, self.atol + ) def func_multi_input(self): - def func(x, y): return paddle.sum(paddle.matmul(x, y)) numerical_hessian = utils._compute_numerical_hessian( - func, [self.x, self.y], self.numerical_delta, self.np_dtype) + func, [self.x, self.y], self.numerical_delta, self.np_dtype + ) numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian) self.x.stop_gradient = False self.y.stop_gradient = False hessian = paddle.incubate.autograd.Hessian(func, [self.x, self.y]) - np.testing.assert_allclose(hessian[:].numpy(), - numerical_hessian, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + hessian[:].numpy(), + numerical_hessian, + rtol=self.rtol, + atol=self.atol, + ) def func_allow_unused_true(self): - def func(x, y): return paddle.sum(paddle.matmul(x, x)) numerical_hessian = utils._compute_numerical_hessian( - func, [self.x, self.y], self.numerical_delta, self.np_dtype) + func, [self.x, self.y], self.numerical_delta, self.np_dtype + ) numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian) self.x.stop_gradient = False self.y.stop_gradient = False hessian = paddle.incubate.autograd.Hessian(func, [self.x, self.y]) - np.testing.assert_allclose(hessian[:].numpy(), numerical_hessian, - self.rtol, self.atol) + np.testing.assert_allclose( + hessian[:].numpy(), numerical_hessian, self.rtol, self.atol + ) def func_create_graph_true(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -514,17 +588,18 @@ class TestHessianNoBatch(unittest.TestCase): return paddle.sum(F.sigmoid(x)) numerical_hessian = utils._compute_numerical_hessian( - func, self.x, self.numerical_delta, self.np_dtype) + func, self.x, self.numerical_delta, self.np_dtype + ) numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian) self.x.stop_gradient = False hessian = paddle.incubate.autograd.Hessian(func, self.x) assert hessian[:].stop_gradient == False - np.testing.assert_allclose(hessian[:].numpy(), numerical_hessian, - self.rtol, self.atol) + np.testing.assert_allclose( + hessian[:].numpy(), numerical_hessian, self.rtol, self.atol + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_out_not_single(self): - def func(x): return x * x @@ -548,7 +623,6 @@ class TestHessianNoBatch(unittest.TestCase): class TestHessianBatchFirst(unittest.TestCase): - @classmethod def setUpClass(self): self.x_shape = (5, 2) @@ -557,108 +631,117 @@ class TestHessianBatchFirst(unittest.TestCase): self.nbatch, self.nrow = 5, 2 self.dtype = 'float32' self.np_dtype = np.float32 - self.numerical_delta = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('eps') - self.rtol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('rtol') - self.atol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('atol') + self.numerical_delta = ( + config.TOLERANCE.get(self.dtype).get('second_order_grad').get('eps') + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('rtol') + ) + self.atol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('atol') + ) self.x = paddle.rand(shape=self.x_shape, dtype=self.dtype) self.weight = paddle.rand(shape=self.weight_shape, dtype=self.dtype) self.y = paddle.rand(shape=self.y_shape, dtype=self.dtype) def func_single_input(self): - def func(x): return paddle.matmul(x * x, self.weight)[:, 0:1] expected = utils._compute_numerical_batch_hessian( - func, self.x, self.numerical_delta, self.np_dtype) + func, self.x, self.numerical_delta, self.np_dtype + ) H = paddle.incubate.autograd.Hessian(func, self.x, is_batched=True) - actual = utils._np_transpose_matrix_format(H[:].numpy(), - utils.MatrixFormat.BNM, - utils.MatrixFormat.NBM) + actual = utils._np_transpose_matrix_format( + H[:].numpy(), utils.MatrixFormat.BNM, utils.MatrixFormat.NBM + ) actual = actual.reshape((H.shape[1], -1)) np.testing.assert_allclose(actual, expected, self.rtol, self.atol) def func_multi_input(self): - def func(x, y): return paddle.matmul(x * x * y * y, self.weight)[:, 0:1] xs_len = 2 expected = utils._compute_numerical_batch_hessian( - func, [self.x, self.y], self.numerical_delta, self.np_dtype) + func, [self.x, self.y], self.numerical_delta, self.np_dtype + ) expected = np.reshape( np.array(expected), - (xs_len, xs_len, self.nrow, self.nbatch, self.nrow)) + (xs_len, xs_len, self.nrow, self.nbatch, self.nrow), + ) expected = [[n for n in row] for row in expected] expected = utils._np_concat_matrix_sequence(expected) self.x.stop_gradient = False self.y.stop_gradient = False - H = paddle.incubate.autograd.Hessian(func, [self.x, self.y], - is_batched=True) - actual = utils._np_transpose_matrix_format(H[:].numpy(), - utils.MatrixFormat.BNM, - utils.MatrixFormat.NBM) + H = paddle.incubate.autograd.Hessian( + func, [self.x, self.y], is_batched=True + ) + actual = utils._np_transpose_matrix_format( + H[:].numpy(), utils.MatrixFormat.BNM, utils.MatrixFormat.NBM + ) np.testing.assert_allclose(actual, expected, self.rtol, self.atol) def func_allow_unused(self): - def func(x, y): return paddle.matmul(x * x, self.weight)[:, 0:1] xs_len = 2 expected = utils._compute_numerical_batch_hessian( - func, [self.x, self.y], self.numerical_delta, self.np_dtype) + func, [self.x, self.y], self.numerical_delta, self.np_dtype + ) expected = np.reshape( np.array(expected), - (xs_len, xs_len, self.nrow, self.nbatch, self.nrow)) + (xs_len, xs_len, self.nrow, self.nbatch, self.nrow), + ) expected = [[n for n in row] for row in expected] expected = utils._np_concat_matrix_sequence(expected) - expected = utils._np_transpose_matrix_format(expected, - utils.MatrixFormat.NBM, - utils.MatrixFormat.BNM) + expected = utils._np_transpose_matrix_format( + expected, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM + ) - actual = paddle.incubate.autograd.Hessian(func, [self.x, self.y], - is_batched=True)[:] + actual = paddle.incubate.autograd.Hessian( + func, [self.x, self.y], is_batched=True + )[:] - np.testing.assert_allclose(actual, - expected, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + actual, expected, rtol=self.rtol, atol=self.atol + ) def func_stop_gradient(self): - def func(x): return paddle.matmul(x * x, self.weight)[:, 0:1] expected = utils._compute_numerical_batch_hessian( - func, self.x, self.numerical_delta, self.np_dtype) + func, self.x, self.numerical_delta, self.np_dtype + ) x = self.x.clone() x.stop_gradient = True H = paddle.incubate.autograd.Hessian(func, self.x, is_batched=True)[:] - actual = utils._np_transpose_matrix_format(H[:].numpy(), - utils.MatrixFormat.BNM, - utils.MatrixFormat.NBM) + actual = utils._np_transpose_matrix_format( + H[:].numpy(), utils.MatrixFormat.BNM, utils.MatrixFormat.NBM + ) actual = actual.reshape((H.shape[1], -1)) np.testing.assert_allclose(actual, expected, self.rtol, self.atol) def func_out_not_single(self): - def func(x): - return (x * x) + return x * x with self.assertRaises(RuntimeError): - paddle.incubate.autograd.Hessian(func, - paddle.ones((3, 3)), - is_batched=True) + paddle.incubate.autograd.Hessian( + func, paddle.ones((3, 3)), is_batched=True + ) def test_all_cases(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_prim.py b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_prim.py index de12fd1ef90..70fb6b1bbfa 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_prim.py @@ -22,23 +22,35 @@ import utils @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), ( - ('unary_float32', paddle.tanh, (np.random.rand(2, 3), ), 'float32'), - ('binary_float32', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float32'), - ('unary_float64', paddle.tanh, (np.random.rand(2, 3), ), 'float64'), - ('binary_float64', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float64'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), + ( + ('unary_float32', paddle.tanh, (np.random.rand(2, 3),), 'float32'), + ( + 'binary_float32', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float32', + ), + ('unary_float64', paddle.tanh, (np.random.rand(2, 3),), 'float64'), + ( + 'binary_float64', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float64', + ), + ), +) class TestJacobianPrim(unittest.TestCase): - @classmethod def setUpClass(cls): cls.args = [arg.astype(cls.dtype) for arg in cls.args] - cls._rtol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('rtol') - cls._atol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('atol') + cls._rtol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('rtol') + ) + cls._atol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('atol') + ) def setUp(self): paddle.enable_static() @@ -49,7 +61,6 @@ class TestJacobianPrim(unittest.TestCase): paddle.disable_static() def test_jacobian_prim(self): - def wrapper(fun, args): mp = paddle.static.Program() sp = paddle.static.Program() @@ -65,10 +76,11 @@ class TestJacobianPrim(unittest.TestCase): paddle.incubate.autograd.prim2orig() exe = paddle.static.Executor() exe.run(sp) - [jac] = exe.run(mp, - feed={f'arg{i}': arg - for i, arg in enumerate(args)}, - fetch_list=[jac]) + [jac] = exe.run( + mp, + feed={f'arg{i}': arg for i, arg in enumerate(args)}, + fetch_list=[jac], + ) return jac paddle.incubate.autograd.enable_prim() @@ -76,30 +88,41 @@ class TestJacobianPrim(unittest.TestCase): paddle.incubate.autograd.disable_prim() orig_jac = wrapper(self.fun, self.args) - np.testing.assert_allclose(orig_jac, - prim_jac, - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + orig_jac, prim_jac, rtol=self._rtol, atol=self._atol + ) @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), ( - ('unary_float32', paddle.tanh, (np.random.rand(1), ), 'float32'), - ('binary_float32', paddle.multiply, - (np.random.rand(1), np.random.rand(1)), 'float32'), - ('unary_float64', paddle.tanh, (np.random.rand(1), ), 'float64'), - ('binary_float64', paddle.multiply, - (np.random.rand(1), np.random.rand(1)), 'float64'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), + ( + ('unary_float32', paddle.tanh, (np.random.rand(1),), 'float32'), + ( + 'binary_float32', + paddle.multiply, + (np.random.rand(1), np.random.rand(1)), + 'float32', + ), + ('unary_float64', paddle.tanh, (np.random.rand(1),), 'float64'), + ( + 'binary_float64', + paddle.multiply, + (np.random.rand(1), np.random.rand(1)), + 'float64', + ), + ), +) class TestHessianPrim(unittest.TestCase): - @classmethod def setUpClass(cls): cls.args = [arg.astype(cls.dtype) for arg in cls.args] - cls._rtol = config.TOLERANCE.get( - cls.dtype).get('second_order_grad').get('rtol') - cls._atol = config.TOLERANCE.get( - cls.dtype).get('second_order_grad').get('atol') + cls._rtol = ( + config.TOLERANCE.get(cls.dtype).get('second_order_grad').get('rtol') + ) + cls._atol = ( + config.TOLERANCE.get(cls.dtype).get('second_order_grad').get('atol') + ) def setUp(self): paddle.enable_static() @@ -110,7 +133,6 @@ class TestHessianPrim(unittest.TestCase): paddle.disable_static() def test_jacobian_prim(self): - def wrapper(fun, args): mp = paddle.static.Program() sp = paddle.static.Program() @@ -126,11 +148,11 @@ class TestHessianPrim(unittest.TestCase): paddle.incubate.autograd.prim2orig() exe = paddle.static.Executor() exe.run(sp) - [hessian - ] = exe.run(mp, - feed={f'arg{i}': arg - for i, arg in enumerate(args)}, - fetch_list=[hessian]) + [hessian] = exe.run( + mp, + feed={f'arg{i}': arg for i, arg in enumerate(args)}, + fetch_list=[hessian], + ) return hessian paddle.incubate.autograd.enable_prim() @@ -138,30 +160,41 @@ class TestHessianPrim(unittest.TestCase): paddle.incubate.autograd.disable_prim() orig_jac = wrapper(self.fun, self.args) - np.testing.assert_allclose(orig_jac, - prim_jac, - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + orig_jac, prim_jac, rtol=self._rtol, atol=self._atol + ) @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), ( - ('unary_float32', paddle.tanh, (np.random.rand(2, 3), ), 'float32'), - ('binary_float32', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float32'), - ('unary_float64', paddle.tanh, (np.random.rand(2, 3), ), 'float64'), - ('binary_float64', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float64'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), + ( + ('unary_float32', paddle.tanh, (np.random.rand(2, 3),), 'float32'), + ( + 'binary_float32', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float32', + ), + ('unary_float64', paddle.tanh, (np.random.rand(2, 3),), 'float64'), + ( + 'binary_float64', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float64', + ), + ), +) class TestJvpPrim(unittest.TestCase): - @classmethod def setUpClass(cls): cls.args = [arg.astype(cls.dtype) for arg in cls.args] - cls._rtol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('rtol') - cls._atol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('atol') + cls._rtol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('rtol') + ) + cls._atol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('atol') + ) def setUp(self): paddle.enable_static() @@ -172,7 +205,6 @@ class TestJvpPrim(unittest.TestCase): paddle.disable_static() def test_jacobian_prim(self): - def wrapper(fun, args): mp = paddle.static.Program() sp = paddle.static.Program() @@ -190,9 +222,9 @@ class TestJvpPrim(unittest.TestCase): exe.run(sp) jvp_res = exe.run( mp, - feed={f'arg{i}': arg - for i, arg in enumerate(args)}, - fetch_list=[jvp_res]) + feed={f'arg{i}': arg for i, arg in enumerate(args)}, + fetch_list=[jvp_res], + ) return jvp_res paddle.incubate.autograd.enable_prim() @@ -200,30 +232,41 @@ class TestJvpPrim(unittest.TestCase): paddle.incubate.autograd.disable_prim() orig_jvp = wrapper(self.fun, self.args) - np.testing.assert_allclose(orig_jvp, - prim_jvp, - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + orig_jvp, prim_jvp, rtol=self._rtol, atol=self._atol + ) @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), ( - ('unary_float32', paddle.tanh, (np.random.rand(2, 3), ), 'float32'), - ('binary_float32', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float32'), - ('unary_float64', paddle.tanh, (np.random.rand(2, 3), ), 'float64'), - ('binary_float64', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float64'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), + ( + ('unary_float32', paddle.tanh, (np.random.rand(2, 3),), 'float32'), + ( + 'binary_float32', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float32', + ), + ('unary_float64', paddle.tanh, (np.random.rand(2, 3),), 'float64'), + ( + 'binary_float64', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float64', + ), + ), +) class TestVjpPrim(unittest.TestCase): - @classmethod def setUpClass(cls): cls.args = [arg.astype(cls.dtype) for arg in cls.args] - cls._rtol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('rtol') - cls._atol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('atol') + cls._rtol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('rtol') + ) + cls._atol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('atol') + ) def setUp(self): paddle.enable_static() @@ -234,7 +277,6 @@ class TestVjpPrim(unittest.TestCase): paddle.disable_static() def test_jacobian_prim(self): - def wrapper(fun, args): mp = paddle.static.Program() sp = paddle.static.Program() @@ -252,9 +294,9 @@ class TestVjpPrim(unittest.TestCase): exe.run(sp) vjp_res = exe.run( mp, - feed={f'arg{i}': arg - for i, arg in enumerate(args)}, - fetch_list=[vjp_res]) + feed={f'arg{i}': arg for i, arg in enumerate(args)}, + fetch_list=[vjp_res], + ) return vjp_res paddle.incubate.autograd.enable_prim() @@ -263,10 +305,9 @@ class TestVjpPrim(unittest.TestCase): orig_vjp = wrapper(self.fun, self.args) for orig, prim in zip(orig_vjp, prim_vjp): - np.testing.assert_allclose(orig, - prim, - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + orig, prim, rtol=self._rtol, atol=self._atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_static.py b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_static.py index 2102cd81ca7..fd3d7d9458a 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_static.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_static.py @@ -26,29 +26,71 @@ paddle.enable_static() @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'stop_gradient'), ( - ('tensor_input', utils.reduce, np.random.rand(2, 3), None, False), - ('tensor_sequence_input', utils.reduce, np.random.rand(2, 3), None, False), - ('v_not_none', utils.reduce, np.random.rand(2, - 3), np.random.rand(1), False), - ('xs_stop_gradient', utils.reduce, np.random.rand( - 2, 3), np.random.rand(1), True), - ('func_mutmul', utils.matmul, - (np.random.rand(3, 2), np.random.rand(2, 3)), None, False), - ('func_mul', utils.mul, - (np.random.rand(3, 3), np.random.rand(3, 3)), None, False), - ('func_out_two', utils.o2, - (np.random.rand(10), np.random.rand(10)), None, False), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'stop_gradient'), + ( + ('tensor_input', utils.reduce, np.random.rand(2, 3), None, False), + ( + 'tensor_sequence_input', + utils.reduce, + np.random.rand(2, 3), + None, + False, + ), + ( + 'v_not_none', + utils.reduce, + np.random.rand(2, 3), + np.random.rand(1), + False, + ), + ( + 'xs_stop_gradient', + utils.reduce, + np.random.rand(2, 3), + np.random.rand(1), + True, + ), + ( + 'func_mutmul', + utils.matmul, + (np.random.rand(3, 2), np.random.rand(2, 3)), + None, + False, + ), + ( + 'func_mul', + utils.mul, + (np.random.rand(3, 3), np.random.rand(3, 3)), + None, + False, + ), + ( + 'func_out_two', + utils.o2, + (np.random.rand(10), np.random.rand(10)), + None, + False, + ), + ), +) class TestVJP(unittest.TestCase): - def setUp(self): - self.dtype = str(self.xs[0].dtype) if isinstance( - self.xs, typing.Sequence) else str(self.xs.dtype) - self._rtol = config.TOLERANCE.get(str( - self.dtype)).get("first_order_grad").get("rtol") - self._atol = config.TOLERANCE.get(str( - self.dtype)).get("first_order_grad").get("atol") + self.dtype = ( + str(self.xs[0].dtype) + if isinstance(self.xs, typing.Sequence) + else str(self.xs.dtype) + ) + self._rtol = ( + config.TOLERANCE.get(str(self.dtype)) + .get("first_order_grad") + .get("rtol") + ) + self._atol = ( + config.TOLERANCE.get(str(self.dtype)) + .get("first_order_grad") + .get("atol") + ) def _vjp(self): exe = paddle.static.Executor() @@ -56,9 +98,11 @@ class TestVJP(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=self.stop_gradient) - ys, xs_grads = paddle.incubate.autograd.vjp(self.fun, static_xs, - static_v) + self.xs, self.v, stop_gradient=self.stop_gradient + ) + ys, xs_grads = paddle.incubate.autograd.vjp( + self.fun, static_xs, static_v + ) exe.run(sp) return exe.run(mp, feed=feed, fetch_list=[ys, xs_grads]) @@ -68,9 +112,13 @@ class TestVJP(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) xs_grads = paddle.static.gradients(ys, static_xs, static_v) exe.run(sp) return exe.run(mp, feed=feed, fetch_list=[ys, xs_grads]) @@ -80,19 +128,25 @@ class TestVJP(unittest.TestCase): expected = self._expected_vjp() self.assertEqual(len(actual), len(expected)) for i in range(len(actual)): - np.testing.assert_allclose(actual[i], - expected[i], - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + actual[i], expected[i], rtol=self._rtol, atol=self._atol + ) @utils.place(config.DEVICES) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'expected_exception'), - (('v_shape_not_equal_ys', utils.square, np.random.rand(3), - np.random.rand(1), RuntimeError), )) + ( + ( + 'v_shape_not_equal_ys', + utils.square, + np.random.rand(3), + np.random.rand(1), + RuntimeError, + ), + ), +) class TestVJPException(unittest.TestCase): - def setUp(self): self.exe = paddle.static.Executor() @@ -101,9 +155,11 @@ class TestVJPException(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v) - ys, xs_grads = paddle.incubate.autograd.vjp(self.fun, static_xs, - static_v) + self.xs, self.v + ) + ys, xs_grads = paddle.incubate.autograd.vjp( + self.fun, static_xs, static_v + ) self.exe.run(sp) return self.exe.run(mp, feed, fetch_list=[ys, xs_grads]) @@ -176,11 +232,11 @@ def make_tensors(inps): all_data_shapes = { - 'A': [[1., 2.]], - 'B': [[1., 2.], [2., 1.]], - 'C': [[2., 2.], [2., 1.]], - 'D': [[[2., 2.], [2., 1.]], [[1., 2.], [2., 1.]]], - 'E': [[[3., 4.], [2., 3.]], [[2., 1.], [1., 3.]]], + 'A': [[1.0, 2.0]], + 'B': [[1.0, 2.0], [2.0, 1.0]], + 'C': [[2.0, 2.0], [2.0, 1.0]], + 'D': [[[2.0, 2.0], [2.0, 1.0]], [[1.0, 2.0], [2.0, 1.0]]], + 'E': [[[3.0, 4.0], [2.0, 3.0]], [[2.0, 1.0], [1.0, 3.0]]], } @@ -190,7 +246,6 @@ def prepare_data(test, input_shapes, dtype): class TestJacobianFloat32(unittest.TestCase): - @classmethod def setUpClass(self): paddle.enable_static() @@ -201,8 +256,9 @@ class TestJacobianFloat32(unittest.TestCase): self.dtype = 'float32' self.np_dtype = np.float32 prepare_data(self, all_data_shapes, self.dtype) - self.eps = config.TOLERANCE.get( - self.dtype).get('first_order_grad').get('eps') + self.eps = ( + config.TOLERANCE.get(self.dtype).get('first_order_grad').get('eps') + ) # self.rtol = config.TOLERANCE.get(self.dtype).get('first_order_grad').get('rtol') # self.atol = config.TOLERANCE.get(self.dtype).get('first_order_grad').get('atol') # Do't use tolerance in config, which will cause this test case failed. @@ -227,17 +283,17 @@ class TestJacobianFloat32(unittest.TestCase): else: feeds = {'x': inps} pd_jacobians = exe.run(main, feed=feeds, fetch_list=[full_jacobian])[0] - np_jacobians = approx_jacobian(np_f, - inps, - self.dtype, - self.eps, - batch=batch) + np_jacobians = approx_jacobian( + np_f, inps, self.dtype, self.eps, batch=batch + ) if batch: np_jacobians = utils._np_transpose_matrix_format( - np_jacobians, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM) + np_jacobians, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM + ) - np.testing.assert_allclose(pd_jacobians, np_jacobians, self.rtol, - self.atol) + np.testing.assert_allclose( + pd_jacobians, np_jacobians, self.rtol, self.atol + ) def run_test_by_rows(self, pd_f, np_f, inps, batch=False): main = fluid.Program() @@ -261,8 +317,9 @@ class TestJacobianFloat32(unittest.TestCase): pd_jac = exe.run(main, feed=feeds, fetch_list=[rows]) np_jac = approx_jacobian(np_f, inps, self.dtype, self.eps, batch=batch) for i in range(nrow): - np.testing.assert_allclose(pd_jac[i], np_jac[i], self.rtol, - self.atol) + np.testing.assert_allclose( + pd_jac[i], np_jac[i], self.rtol, self.atol + ) def run_test_by_entries(self, pd_f, np_f, inps, batch=False): main = fluid.Program() @@ -293,7 +350,6 @@ class TestJacobianFloat32(unittest.TestCase): np.testing.assert_allclose(pd_entry, np_entry, self.rtol, self.atol) def test_square(self): - def pd_f(x): return paddle.multiply(x, x) @@ -305,7 +361,6 @@ class TestJacobianFloat32(unittest.TestCase): self.run_test_by_entries(pd_f, np_f, self.A) def test_mul(self): - def pd_f(x, y): return paddle.multiply(x, y) @@ -322,7 +377,6 @@ class TestJacobianFloat32(unittest.TestCase): self.run_test_by_entries(pd_f, np_f, [self.B, self.C]) def test_matmul(self): - def pd_f(x, y): return paddle.matmul(x, y) @@ -335,7 +389,6 @@ class TestJacobianFloat32(unittest.TestCase): self.run_test_by_entries(pd_f, np_f, [self.B, self.C]) def test_batch_matmul(self): - def pd_f(x, y): return paddle.matmul(x, y) @@ -349,7 +402,6 @@ class TestJacobianFloat32(unittest.TestCase): class TestJacobianFloat64(TestJacobianFloat32): - @classmethod def setUpClass(self): paddle.enable_static() @@ -359,16 +411,18 @@ class TestJacobianFloat64(TestJacobianFloat32): self.place = fluid.CPUPlace() self.dtype = 'float64' prepare_data(self, all_data_shapes, self.dtype) - self.eps = config.TOLERANCE.get( - self.dtype).get('first_order_grad').get('eps') - self.rtol = config.TOLERANCE.get( - self.dtype).get('first_order_grad').get('rtol') - self.atol = config.TOLERANCE.get( - self.dtype).get('first_order_grad').get('atol') + self.eps = ( + config.TOLERANCE.get(self.dtype).get('first_order_grad').get('eps') + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype).get('first_order_grad').get('rtol') + ) + self.atol = ( + config.TOLERANCE.get(self.dtype).get('first_order_grad').get('atol') + ) class TestHessianFloat32(unittest.TestCase): - @classmethod def setUpClass(self): paddle.enable_static() @@ -378,12 +432,19 @@ class TestHessianFloat32(unittest.TestCase): self.place = fluid.CPUPlace() self.dtype = 'float32' prepare_data(self, all_data_shapes, self.dtype) - self.eps = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('eps') - self.rtol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('rtol') - self.atol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('atol') + self.eps = ( + config.TOLERANCE.get(self.dtype).get('second_order_grad').get('eps') + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('rtol') + ) + self.atol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('atol') + ) def run_test_by_fullmatrix(self, pd_f, inps, np_hess, batch=False): main = fluid.Program() @@ -403,7 +464,6 @@ class TestHessianFloat32(unittest.TestCase): np.testing.assert_allclose(pd_hess, np_hess, self.rtol, self.atol) def test_square(self): - def pd_f(x): """Input is a square matrix.""" return paddle.matmul(x, x.T).flatten().sum() @@ -418,7 +478,6 @@ class TestHessianFloat32(unittest.TestCase): class TestHessianFloat64(TestHessianFloat32): - @classmethod def setUpClass(self): paddle.enable_static() @@ -428,12 +487,19 @@ class TestHessianFloat64(TestHessianFloat32): self.place = fluid.CPUPlace() self.dtype = 'float64' prepare_data(self, all_data_shapes, self.dtype) - self.eps = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('eps') - self.rtol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('rtol') - self.atol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('atol') + self.eps = ( + config.TOLERANCE.get(self.dtype).get('second_order_grad').get('eps') + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('rtol') + ) + self.atol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('atol') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py index f345f583945..e90d6871c30 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py @@ -24,14 +24,14 @@ paddle.enable_static() ############################ Test linearize rules ############################ class TestAddPJVPAndTranspose(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() self.layer_help = LayerHelper('TestPrim2Orig') - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_data() def init_data(self): @@ -41,8 +41,9 @@ class TestAddPJVPAndTranspose(unittest.TestCase): Y = paddle.static.data(name='Y', shape=[2, 2], dtype='float') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -67,12 +68,15 @@ class TestAddPJVPAndTranspose(unittest.TestCase): ] def test_op(self): - with paddle.static.program_guard(self.main_program, - self.startup_program): - op = self.layer_help.append_op(type=self.op_type, - inputs=self.prim_input, - outputs=self.prim_output, - attrs=self.prim_attrs) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + op = self.layer_help.append_op( + type=self.op_type, + inputs=self.prim_input, + outputs=self.prim_output, + attrs=self.prim_attrs, + ) jvp_out = _jvp(op, *self.jvp_args) jvp_out = flatten(jvp_out) @@ -91,7 +95,6 @@ class TestAddPJVPAndTranspose(unittest.TestCase): class TestSubPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'sub_p' @@ -99,8 +102,9 @@ class TestSubPJVPAndTranspose(TestAddPJVPAndTranspose): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='int64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -123,12 +127,11 @@ class TestSubPJVPAndTranspose(TestAddPJVPAndTranspose): 'sub_p', # transpose op: 'fill_constant_p', - 'sub_p' + 'sub_p', ] class TestMulPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'mul_p' @@ -136,8 +139,9 @@ class TestMulPJVPAndTranspose(TestAddPJVPAndTranspose): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='int64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -163,12 +167,11 @@ class TestMulPJVPAndTranspose(TestAddPJVPAndTranspose): 'mul_p', 'add_p', # transpose op: - 'mul_p' + 'mul_p', ] class TestDivPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'div_p' @@ -176,8 +179,9 @@ class TestDivPJVPAndTranspose(TestAddPJVPAndTranspose): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='int64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -205,12 +209,11 @@ class TestDivPJVPAndTranspose(TestAddPJVPAndTranspose): 'mul_p', 'sub_p', # transpose op: - 'div_p' + 'div_p', ] class TestSqrtPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'sqrt_p' @@ -219,14 +222,15 @@ class TestSqrtPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -242,7 +246,6 @@ class TestSqrtPJVPAndTranspose(TestAddPJVPAndTranspose): class TestRSqrtPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'rsqrt_p' @@ -251,14 +254,15 @@ class TestRSqrtPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -275,7 +279,6 @@ class TestRSqrtPJVPAndTranspose(TestAddPJVPAndTranspose): class TestTanhPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'tanh_p' @@ -284,14 +287,15 @@ class TestTanhPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -307,7 +311,6 @@ class TestTanhPJVPAndTranspose(TestAddPJVPAndTranspose): class TestSinPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'sin_p' @@ -316,14 +319,15 @@ class TestSinPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -337,7 +341,6 @@ class TestSinPJVPAndTranspose(TestAddPJVPAndTranspose): class TestCosPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'cos_p' @@ -346,14 +349,15 @@ class TestCosPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -369,7 +373,6 @@ class TestCosPJVPAndTranspose(TestAddPJVPAndTranspose): class TestExpPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'exp_p' @@ -378,14 +381,15 @@ class TestExpPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -398,7 +402,6 @@ class TestExpPJVPAndTranspose(TestAddPJVPAndTranspose): class TestErfPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'erf_p' @@ -407,14 +410,15 @@ class TestErfPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -434,7 +438,6 @@ class TestErfPJVPAndTranspose(TestAddPJVPAndTranspose): class TestAbsPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'abs_p' @@ -443,14 +446,15 @@ class TestAbsPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -467,7 +471,6 @@ class TestAbsPJVPAndTranspose(TestAddPJVPAndTranspose): class TestCastPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'cast_p' @@ -476,14 +479,15 @@ class TestCastPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'dtype': paddle.float64} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose @@ -498,12 +502,11 @@ class TestCastPJVPAndTranspose(TestAddPJVPAndTranspose): # jvp op: 'cast_p', # transpose op: - 'cast_p' + 'cast_p', ] class TestLogPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'log_p' @@ -512,14 +515,15 @@ class TestLogPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -532,7 +536,6 @@ class TestLogPJVPAndTranspose(TestAddPJVPAndTranspose): class TestReshapePJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'reshape_p' @@ -541,14 +544,15 @@ class TestReshapePJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'shape': [2, 32]} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[8, 8], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose @@ -570,7 +574,6 @@ class TestReshapePJVPAndTranspose(TestAddPJVPAndTranspose): class TestBroadcastPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'broadcast_p' @@ -579,21 +582,22 @@ class TestBroadcastPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'shape': [2, 10, 7]} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[10, 7], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose check_dot = lambda v: v is X - Y_BAR = paddle.static.data(name='Y_BAR', - shape=[2, 10, 7], - dtype='int64') + Y_BAR = paddle.static.data( + name='Y_BAR', shape=[2, 10, 7], dtype='int64' + ) self.transpose_args = (check_dot, Y_BAR) self.transpose_out_shape_map = { 0: X, @@ -606,12 +610,11 @@ class TestBroadcastPJVPAndTranspose(TestAddPJVPAndTranspose): 'broadcast_p', # transpose op: 'reduce_sum_p', - 'reshape_p' + 'reshape_p', ] class TestTransposePJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'transpose_p' @@ -620,23 +623,24 @@ class TestTransposePJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': [0, 2, 3, 1]} # Set JVP - X_DOT = paddle.static.data(name='X_DOT', - shape=[2, 3, 4, 5], - dtype='int64') - self.jvp_args = (X_DOT, ) + X_DOT = paddle.static.data( + name='X_DOT', shape=[2, 3, 4, 5], dtype='int64' + ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose check_dot = lambda v: v is X - Y_BAR = paddle.static.data(name='Y_BAR', - shape=[2, 4, 5, 3], - dtype='int64') + Y_BAR = paddle.static.data( + name='Y_BAR', shape=[2, 4, 5, 3], dtype='int64' + ) self.transpose_args = (check_dot, Y_BAR) self.transpose_out_shape_map = { 0: X, @@ -653,7 +657,6 @@ class TestTransposePJVPAndTranspose(TestAddPJVPAndTranspose): class TestSplitPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'split_p' @@ -664,16 +667,18 @@ class TestSplitPJVPAndTranspose(TestAddPJVPAndTranspose): self.prim_output = { 'YS': [ self.layer_help.create_variable_for_type_inference( - dtype=X.dtype) for i in range(4) + dtype=X.dtype + ) + for i in range(4) ] } self.prim_attrs = {'num_or_sections': [2, 3, 4, 1], 'axis': 2} # Set JVP - X_DOT = paddle.static.data(name='X_DOT', - shape=[2, 7, 10], - dtype='int64') - self.jvp_args = (X_DOT, ) + X_DOT = paddle.static.data( + name='X_DOT', shape=[2, 7, 10], dtype='int64' + ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = { 0: self.prim_output['YS'][0], 1: self.prim_output['YS'][1], @@ -705,7 +710,6 @@ class TestSplitPJVPAndTranspose(TestAddPJVPAndTranspose): class TestConcatPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'concat_p' @@ -716,8 +720,9 @@ class TestConcatPJVPAndTranspose(TestAddPJVPAndTranspose): 'XS': [X, Y, Z], } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': 1} @@ -727,14 +732,14 @@ class TestConcatPJVPAndTranspose(TestAddPJVPAndTranspose): paddle.static.data(name='X_DOT2', shape=[3, 2, 5], dtype='float64'), paddle.static.data(name='X_DOT3', shape=[3, 3, 5], dtype='float64'), ] - self.jvp_args = (XS_DOT, ) + self.jvp_args = (XS_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose check_dot = lambda v: v is X or v is Y or v is Z - Y_BAR = paddle.static.data(name='Y_BAR', - shape=[3, 14, 5], - dtype='float64') + Y_BAR = paddle.static.data( + name='Y_BAR', shape=[3, 14, 5], dtype='float64' + ) self.transpose_args = (check_dot, Y_BAR) self.transpose_out_shape_map = { 0: X, @@ -753,30 +758,30 @@ class TestConcatPJVPAndTranspose(TestAddPJVPAndTranspose): class TestReduceSumPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'reduce_sum_p' X = paddle.static.data(name='X', shape=[2, 3, 4, 5], dtype='float64') self.prim_input = {'X': X} self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': [2], 'keepdim': False} # Set JVP - X_DOT = paddle.static.data(name='X_DOT1', - shape=[2, 3, 4, 5], - dtype='float64') - self.jvp_args = (X_DOT, ) + X_DOT = paddle.static.data( + name='X_DOT1', shape=[2, 3, 4, 5], dtype='float64' + ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose check_dot = lambda v: v is X - Y_BAR = paddle.static.data(name='Y_BAR', - shape=[2, 3, 5], - dtype='float64') + Y_BAR = paddle.static.data( + name='Y_BAR', shape=[2, 3, 5], dtype='float64' + ) self.transpose_args = (check_dot, Y_BAR) self.transpose_out_shape_map = { 0: X, @@ -794,7 +799,6 @@ class TestReduceSumPJVPAndTranspose(TestAddPJVPAndTranspose): class TestMatmulPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'matmul_p' @@ -802,8 +806,9 @@ class TestMatmulPJVPAndTranspose(TestAddPJVPAndTranspose): Y = paddle.static.data(name='Y', shape=[3, 4], dtype='float64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -835,7 +840,6 @@ class TestMatmulPJVPAndTranspose(TestAddPJVPAndTranspose): class TestSliceSelectPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'slice_select_p' @@ -844,19 +848,20 @@ class TestSliceSelectPJVPAndTranspose(TestAddPJVPAndTranspose): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = { 'axis': [1], 'starts': [0], 'ends': [20], - 'strides': [2] + 'strides': [2], } # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[3, 20], dtype='float64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose @@ -879,7 +884,6 @@ class TestSliceSelectPJVPAndTranspose(TestAddPJVPAndTranspose): class TestSliceAssignPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'slice_assign_p' @@ -887,14 +891,15 @@ class TestSliceAssignPJVPAndTranspose(TestAddPJVPAndTranspose): Y = paddle.static.data(name='Y', shape=[3, 5], dtype='float64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = { 'axis': [1], 'starts': [0], 'ends': [10], - 'strides': [2] + 'strides': [2], } # Set JVP @@ -917,23 +922,23 @@ class TestSliceAssignPJVPAndTranspose(TestAddPJVPAndTranspose): # transpose op: 'slice_assign_p', 'slice_select_p', - 'fill_constant_p' + 'fill_constant_p', ] class TestGatherPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'gather_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') - IndexTensor = paddle.static.data(name='IndexTensor', - shape=[3], - dtype='int32') + IndexTensor = paddle.static.data( + name='IndexTensor', shape=[3], dtype='int32' + ) self.prim_input = {'X': X, 'IndexTensor': IndexTensor} self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': 1} @@ -965,19 +970,19 @@ class TestGatherPJVPAndTranspose(TestAddPJVPAndTranspose): class TestScatterAddPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'scatter_add_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') Y = paddle.static.data(name='Y', shape=[9, 3], dtype='float64') - IndexTensor = paddle.static.data(name='IndexTensor', - shape=[3], - dtype='int32') + IndexTensor = paddle.static.data( + name='IndexTensor', shape=[3], dtype='int32' + ) self.prim_input = {'X': X, 'Y': Y, 'IndexTensor': IndexTensor} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': 1} @@ -1001,12 +1006,11 @@ class TestScatterAddPJVPAndTranspose(TestAddPJVPAndTranspose): # transpose op: 'scatter_add_p', 'fill_constant_p', - 'gather_p' + 'gather_p', ] class TestSelectPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'select_p' @@ -1016,15 +1020,16 @@ class TestSelectPJVPAndTranspose(TestAddPJVPAndTranspose): self.prim_input = {'Condition': Cond, 'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP - Cond_DOT = paddle.static.data(name='Cond_DOT', - shape=[9, 5], - dtype='float64') + Cond_DOT = paddle.static.data( + name='Cond_DOT', shape=[9, 5], dtype='float64' + ) X_DOT = paddle.static.data(name='X_DOT', shape=[9, 5], dtype='float64') Y_DOT = paddle.static.data(name='Y_DOT', shape=[9, 5], dtype='float64') self.jvp_args = (Cond_DOT, X_DOT, Y_DOT) @@ -1051,7 +1056,6 @@ class TestSelectPJVPAndTranspose(TestAddPJVPAndTranspose): class TestEqPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'eq_p' @@ -1060,8 +1064,9 @@ class TestEqPJVPAndTranspose(TestAddPJVPAndTranspose): self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1081,7 +1086,6 @@ class TestEqPJVPAndTranspose(TestAddPJVPAndTranspose): class TestGtPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'gt_p' @@ -1090,8 +1094,9 @@ class TestGtPJVPAndTranspose(TestAddPJVPAndTranspose): self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1111,7 +1116,6 @@ class TestGtPJVPAndTranspose(TestAddPJVPAndTranspose): class TestGePJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'ge_p' @@ -1120,8 +1124,9 @@ class TestGePJVPAndTranspose(TestAddPJVPAndTranspose): self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1141,7 +1146,6 @@ class TestGePJVPAndTranspose(TestAddPJVPAndTranspose): class TestNePJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'ne_p' @@ -1150,8 +1154,9 @@ class TestNePJVPAndTranspose(TestAddPJVPAndTranspose): self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1171,7 +1176,6 @@ class TestNePJVPAndTranspose(TestAddPJVPAndTranspose): class TestPowPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'pow_p' @@ -1179,8 +1183,9 @@ class TestPowPJVPAndTranspose(TestAddPJVPAndTranspose): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='float32') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1211,7 +1216,6 @@ class TestPowPJVPAndTranspose(TestAddPJVPAndTranspose): class TestMaxPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'max_p' @@ -1219,8 +1223,9 @@ class TestMaxPJVPAndTranspose(TestAddPJVPAndTranspose): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='float32') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} diff --git a/python/paddle/fluid/tests/unittests/autograd/test_minimize.py b/python/paddle/fluid/tests/unittests/autograd/test_minimize.py index 10259802c69..e3de0f24e0d 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_minimize.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_minimize.py @@ -17,14 +17,16 @@ import unittest import numpy as np import paddle from paddle.incubate.autograd.primx import prim2orig -from paddle.incubate.autograd.utils import (disable_prim, enable_prim, - prim_enabled) +from paddle.incubate.autograd.utils import ( + disable_prim, + enable_prim, + prim_enabled, +) paddle.enable_static() class TestMinimize(unittest.TestCase): - def model(self, x, w, bias, opt): paddle.seed(0) place = paddle.CPUPlace() @@ -36,12 +38,12 @@ class TestMinimize(unittest.TestCase): with paddle.static.program_guard(main, startup): input_x = paddle.static.data('x', x.shape, dtype=x.dtype) input_x.stop_gradient = False - params_w = paddle.static.create_parameter(shape=w.shape, - dtype=w.dtype, - is_bias=False) - params_bias = paddle.static.create_parameter(shape=bias.shape, - dtype=bias.dtype, - is_bias=True) + params_w = paddle.static.create_parameter( + shape=w.shape, dtype=w.dtype, is_bias=False + ) + params_bias = paddle.static.create_parameter( + shape=bias.shape, dtype=bias.dtype, is_bias=True + ) y = paddle.tanh(paddle.matmul(input_x, params_w) + params_bias) loss = paddle.norm(y, p=2) opt = opt @@ -49,13 +51,9 @@ class TestMinimize(unittest.TestCase): if prim_enabled(): prim2orig(main.block(0)) exe.run(startup) - grads = exe.run(main, - feed={ - 'x': x, - 'w': w, - 'bias': bias - }, - fetch_list=grads) + grads = exe.run( + main, feed={'x': x, 'w': w, 'bias': bias}, fetch_list=grads + ) return grads def test_adam(self): diff --git a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py index 914ea38fa9a..69161d89a11 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py @@ -24,14 +24,14 @@ paddle.enable_static() ############################ Test orig2prim rules ############################ class TestElementWiseAddOrig2Prim(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() self.layer_help = LayerHelper('TestOrig2Prim') - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_data() def init_data(self): @@ -41,8 +41,9 @@ class TestElementWiseAddOrig2Prim(unittest.TestCase): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -52,12 +53,15 @@ class TestElementWiseAddOrig2Prim(unittest.TestCase): self.out_map = {0: self.output['Out']} def test_op(self): - with paddle.static.program_guard(self.main_program, - self.startup_program): - op = self.layer_help.append_op(type=self.op_type, - inputs=self.input, - outputs=self.output, - attrs=self.attrs) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + op = self.layer_help.append_op( + type=self.op_type, + inputs=self.input, + outputs=self.output, + attrs=self.attrs, + ) prim_out = _orig2prim(op, *self.orig2prim_args) all_ops = [op.type for op in self.main_program.block(0).ops] @@ -69,7 +73,6 @@ class TestElementWiseAddOrig2Prim(unittest.TestCase): class TestSqrtOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'sqrt' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -78,19 +81,19 @@ class TestSqrtOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['sqrt', 'sqrt_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestElementWiseMulOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_mul' X = paddle.static.data(name='X', shape=[8, 8], dtype='float') @@ -98,8 +101,9 @@ class TestElementWiseMulOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -110,7 +114,6 @@ class TestElementWiseMulOrig2Prim(TestElementWiseAddOrig2Prim): class TestElementWiseDivOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_div' X = paddle.static.data(name='X', shape=[8, 8], dtype='float') @@ -118,8 +121,9 @@ class TestElementWiseDivOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -130,7 +134,6 @@ class TestElementWiseDivOrig2Prim(TestElementWiseAddOrig2Prim): class TestMatmulV2Orig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'matmul_v2' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -138,8 +141,9 @@ class TestMatmulV2Orig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'trans_x': True, 'trans_y': True} @@ -149,7 +153,6 @@ class TestMatmulV2Orig2Prim(TestElementWiseAddOrig2Prim): class TestTanhOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'tanh' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -158,18 +161,18 @@ class TestTanhOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['tanh', 'tanh_p'] self.out_map = {0: self.output['Out']} class TestSinOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'sin' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -178,18 +181,18 @@ class TestSinOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['sin', 'sin_p'] self.out_map = {0: self.output['Out']} class TestCosOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'cos' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -198,18 +201,18 @@ class TestCosOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['cos', 'cos_p'] self.out_map = {0: self.output['Out']} class TestExpOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'exp' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -218,18 +221,18 @@ class TestExpOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['exp', 'exp_p'] self.out_map = {0: self.output['Out']} class TestErfOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'erf' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -238,18 +241,18 @@ class TestErfOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['erf', 'erf_p'] self.out_map = {0: self.output['Out']} class TestAbsOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'abs' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -258,18 +261,18 @@ class TestAbsOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['abs', 'abs_p'] self.out_map = {0: self.output['Out']} class TestLogOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'log' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -278,18 +281,18 @@ class TestLogOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['log', 'log_p'] self.out_map = {0: self.output['Out']} class TestReshape2Orig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'reshape2' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -298,10 +301,10 @@ class TestReshape2Orig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - X, - 'XShape': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': X, + 'XShape': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = {'shape': [6, 5]} @@ -316,7 +319,6 @@ class TestReshape2Orig2Prim(TestElementWiseAddOrig2Prim): class TestConcatOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'concat' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -326,8 +328,9 @@ class TestConcatOrig2Prim(TestElementWiseAddOrig2Prim): 'X': [X, Y], } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': 0} @@ -340,7 +343,6 @@ class TestConcatOrig2Prim(TestElementWiseAddOrig2Prim): class TestSliceOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'slice' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -349,8 +351,9 @@ class TestSliceOrig2Prim(TestElementWiseAddOrig2Prim): 'Input': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'axes': [0], @@ -364,7 +367,6 @@ class TestSliceOrig2Prim(TestElementWiseAddOrig2Prim): class TestFillZerosLikeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'fill_zeros_like' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -373,18 +375,18 @@ class TestFillZerosLikeOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['fill_zeros_like', 'fill_constant_p'] self.out_map = {0: self.output['Out']} class TestFillAnyLikeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'fill_any_like' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -393,18 +395,18 @@ class TestFillAnyLikeOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['fill_any_like', 'fill_constant_p'] self.out_map = {0: self.output['Out']} class TestFillAnyLikeOrig2Prim2(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'fill_any_like' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -413,18 +415,18 @@ class TestFillAnyLikeOrig2Prim2(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'dtype': paddle.float32, 'value': 5} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['fill_any_like', 'fill_constant_p'] self.out_map = {0: self.output['Out']} class TestSumOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'sum' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -432,18 +434,18 @@ class TestSumOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = ((X, Y), ) + self.orig2prim_args = ((X, Y),) self.all_ops = ['sum', 'add_p'] self.out_map = {0: self.output['Out']} class TestPNormOrig2Prim1(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'p_norm' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -452,21 +454,21 @@ class TestPNormOrig2Prim1(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'porder': 1, 'asvector': True, } - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['p_norm', 'reshape_p', 'abs_p', 'reduce_sum_p'] self.out_map = {0: self.output['Out']} class TestPNormOrig2Prim2(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'p_norm' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -475,23 +477,27 @@ class TestPNormOrig2Prim2(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'porder': 2, 'asvector': True, } - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'p_norm', 'reshape_p', 'sqrt_p', 'reduce_sum_p', 'mul_p' + 'p_norm', + 'reshape_p', + 'sqrt_p', + 'reduce_sum_p', + 'mul_p', ] self.out_map = {0: self.output['Out']} class TestIndexSelectOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'index_select' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -499,8 +505,9 @@ class TestIndexSelectOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Index': Index} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'dim': 0, @@ -515,7 +522,6 @@ class TestIndexSelectOrig2Prim(TestElementWiseAddOrig2Prim): class TestElementwiseSubOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_sub' X = paddle.static.data(name='X', shape=[5, 6], dtype='int32') @@ -523,8 +529,9 @@ class TestElementwiseSubOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'dim': 0, @@ -539,7 +546,6 @@ class TestElementwiseSubOrig2Prim(TestElementWiseAddOrig2Prim): class TestScaleOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'scale' X = paddle.static.data(name='X', shape=[10, 7], dtype='int32') @@ -548,8 +554,9 @@ class TestScaleOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'scale': 2.0, 'bias': 1.0, 'bias_after_scale': True} @@ -558,13 +565,16 @@ class TestScaleOrig2Prim(TestElementWiseAddOrig2Prim): X, ) self.all_ops = [ - 'scale', 'fill_constant_p', 'fill_constant_p', 'mul_p', 'add_p' + 'scale', + 'fill_constant_p', + 'fill_constant_p', + 'mul_p', + 'add_p', ] self.out_map = {0: self.output['Out']} class TestAssignOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'assign' X = paddle.static.data(name='X', shape=[10, 7], dtype='int32') @@ -573,18 +583,18 @@ class TestAssignOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['assign', 'fill_constant_p', 'add_p'] self.out_map = {0: self.output['Out']} class TestWhereOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'where' Cond = paddle.static.data(name='Condition', shape=[5, 6], dtype='bool') @@ -593,8 +603,9 @@ class TestWhereOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'Condition': Cond, 'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} self.orig2prim_args = (Cond, X, Y) @@ -603,7 +614,6 @@ class TestWhereOrig2Prim(TestElementWiseAddOrig2Prim): class TestEqualOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'equal' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -611,8 +621,9 @@ class TestEqualOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Out': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} self.orig2prim_args = (X, Y) @@ -622,7 +633,6 @@ class TestEqualOrig2Prim(TestElementWiseAddOrig2Prim): class TestNeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'not_equal' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -630,8 +640,9 @@ class TestNeOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Out': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} self.orig2prim_args = (X, Y) @@ -641,7 +652,6 @@ class TestNeOrig2Prim(TestElementWiseAddOrig2Prim): class TestGtOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'greater_than' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -649,8 +659,9 @@ class TestGtOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Out': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} self.orig2prim_args = (X, Y) @@ -660,7 +671,6 @@ class TestGtOrig2Prim(TestElementWiseAddOrig2Prim): class TestGeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'greater_equal' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -668,8 +678,9 @@ class TestGeOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Out': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} self.orig2prim_args = (X, Y) @@ -679,7 +690,6 @@ class TestGeOrig2Prim(TestElementWiseAddOrig2Prim): class TestPowOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_pow' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -687,8 +697,9 @@ class TestPowOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -699,7 +710,6 @@ class TestPowOrig2Prim(TestElementWiseAddOrig2Prim): class TestMaxOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_max' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -707,8 +717,9 @@ class TestMaxOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -719,97 +730,119 @@ class TestMaxOrig2Prim(TestElementWiseAddOrig2Prim): class TestGeluOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'gelu' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'approximate': False} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'gelu', 'add_p', 'erf_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'mul_p', 'mul_p', 'mul_p' + 'gelu', + 'add_p', + 'erf_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'mul_p', + 'mul_p', + 'mul_p', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestGeluApproximateOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'gelu' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'approximate': True} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'add_p', 'add_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'fill_constant_p', 'fill_constant_p', 'gelu', - 'mul_p', 'mul_p', 'mul_p', 'mul_p', 'pow_p', 'tanh_p' + 'add_p', + 'add_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'gelu', + 'mul_p', + 'mul_p', + 'mul_p', + 'mul_p', + 'pow_p', + 'tanh_p', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestDropoutOrig2PrimCase1(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'dropout' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Mask': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.uint8), - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype), + 'Mask': self.layer_help.create_variable_for_type_inference( + dtype=paddle.uint8 + ), + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = { 'dropout_prob': 0.5, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.orig2prim_args = (None, X) self.all_ops = [ - 'bernoulli_p', 'mul_p', 'fill_constant_p', 'div_p', 'cast_p', - 'dropout' + 'bernoulli_p', + 'mul_p', + 'fill_constant_p', + 'div_p', + 'cast_p', + 'dropout', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Mask'], 1: self.output['Out']} class TestDropoutOrig2PrimCase2(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'dropout' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Mask': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.uint8), - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype), + 'Mask': self.layer_help.create_variable_for_type_inference( + dtype=paddle.uint8 + ), + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = { 'dropout_prob': 0.5, 'is_test': False, - 'dropout_implementation': 'downgrade_in_infer' + 'dropout_implementation': 'downgrade_in_infer', } self.orig2prim_args = (None, X) @@ -819,23 +852,23 @@ class TestDropoutOrig2PrimCase2(TestElementWiseAddOrig2Prim): class TestDropoutOrig2PrimCase3(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'dropout' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Mask': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.uint8), - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype), + 'Mask': self.layer_help.create_variable_for_type_inference( + dtype=paddle.uint8 + ), + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = { 'dropout_prob': 0.5, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.orig2prim_args = (None, X) @@ -845,35 +878,38 @@ class TestDropoutOrig2PrimCase3(TestElementWiseAddOrig2Prim): class TestDropoutOrig2PrimCase4(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'dropout' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Mask': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.uint8), - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype), + 'Mask': self.layer_help.create_variable_for_type_inference( + dtype=paddle.uint8 + ), + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = { 'dropout_prob': 0.5, 'is_test': True, - 'dropout_implementation': 'downgrade_in_infer' + 'dropout_implementation': 'downgrade_in_infer', } self.orig2prim_args = (None, X) self.all_ops = [ - 'bernoulli_p', 'fill_constant_p', 'mul_p', 'cast_p', 'dropout' + 'bernoulli_p', + 'fill_constant_p', + 'mul_p', + 'cast_p', + 'dropout', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Mask'], 1: self.output['Out']} class TestReduceSumOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'reduce_sum' @@ -881,87 +917,90 @@ class TestReduceSumOrig2Prim(TestElementWiseAddOrig2Prim): self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [0, 1], 'keep_dim': False} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['reduce_sum', 'reduce_sum_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestReduceMeanOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'reduce_mean' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [0, 1], 'keep_dim': False} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'reduce_mean', 'reduce_sum_p', 'fill_constant_p', 'div_p' + 'reduce_mean', + 'reduce_sum_p', + 'fill_constant_p', + 'div_p', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestSizeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'size' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'Input': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.int64) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.int64 + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['size', 'fill_constant_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestCastOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'cast' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'in_dtype': X.dtype, 'out_dtype': paddle.float64} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['cast', 'cast_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestPowScalarOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'pow' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } - self.attrs = {'factor': 2.} + self.attrs = {'factor': 2.0} self.orig2prim_args = (None, X) self.all_ops = ['pow', 'pow_p', 'fill_constant_p'] # { prim_op_output_index: orig_op_output_var } @@ -969,25 +1008,24 @@ class TestPowScalarOrig2Prim(TestElementWiseAddOrig2Prim): class TestSquareOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'square' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['square', 'pow_p', 'fill_constant_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestRSqrtOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'rsqrt' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -996,19 +1034,19 @@ class TestRSqrtOrig2Prim(TestElementWiseAddOrig2Prim): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['rsqrt', 'rsqrt_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestBatchnormOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'batch_norm' x = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -1022,20 +1060,23 @@ class TestBatchnormOrig2Prim(TestElementWiseAddOrig2Prim): "Scale": [w], "Bias": [b], "Mean": [m], - "Variance": [v] + "Variance": [v], } saved_variance = self.layer_help.create_variable_for_type_inference( - dtype=x.dtype, stop_gradient=True) + dtype=x.dtype, stop_gradient=True + ) batch_norm_out = self.layer_help.create_variable_for_type_inference( - x.dtype) + x.dtype + ) saved_mean = self.layer_help.create_variable_for_type_inference( - dtype=x.dtype, stop_gradient=True) + dtype=x.dtype, stop_gradient=True + ) self.output = { "Y": [batch_norm_out], "MeanOut": [m], "VarianceOut": [v], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } self.attrs = { @@ -1050,31 +1091,60 @@ class TestBatchnormOrig2Prim(TestElementWiseAddOrig2Prim): } self.orig2prim_args = (b, m, None, w, v, x) self.all_ops = [ - 'add_p', 'add_p', 'add_p', 'add_p', 'batch_norm', 'broadcast_p', - 'broadcast_p', 'broadcast_p', 'broadcast_p', 'broadcast_p', 'div_p', - 'div_p', 'div_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'mul_p', 'mul_p', 'mul_p', 'mul_p', 'mul_p', - 'pow_p', 'reduce_sum_p', 'reduce_sum_p', 'reshape_p', 'reshape_p', - 'reshape_p', 'reshape_p', 'sqrt_p', 'sub_p', 'sub_p', 'sub_p', - 'sub_p' + 'add_p', + 'add_p', + 'add_p', + 'add_p', + 'batch_norm', + 'broadcast_p', + 'broadcast_p', + 'broadcast_p', + 'broadcast_p', + 'broadcast_p', + 'div_p', + 'div_p', + 'div_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'mul_p', + 'mul_p', + 'mul_p', + 'mul_p', + 'mul_p', + 'pow_p', + 'reduce_sum_p', + 'reduce_sum_p', + 'reshape_p', + 'reshape_p', + 'reshape_p', + 'reshape_p', + 'sqrt_p', + 'sub_p', + 'sub_p', + 'sub_p', + 'sub_p', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {} class TestFillConstantOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'fill_constant' - self.attrs = {'value': 1., 'shape': (2, 3), 'dtype': paddle.float32} + self.attrs = {'value': 1.0, 'shape': (2, 3), 'dtype': paddle.float32} self.input = {} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float32) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float32 + ) } self.orig2prim_args = (None, None, None) @@ -1084,14 +1154,13 @@ class TestFillConstantOrig2Prim(TestElementWiseAddOrig2Prim): class TestUniformRandomOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'uniform_random' self.input = {} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float32) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float32 + ) } self.attrs = {'shape': [1, 2]} @@ -1101,7 +1170,6 @@ class TestUniformRandomOrig2Prim(TestElementWiseAddOrig2Prim): class TestSigmoidOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'sigmoid' X = paddle.static.data(name='X', shape=[3], dtype='float32') @@ -1109,15 +1177,21 @@ class TestSigmoidOrig2Prim(TestElementWiseAddOrig2Prim): self.attrs = {} self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float32) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float32 + ) } - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'sigmoid', 'div_p', 'fill_constant_p', 'add_p', 'fill_constant_p', - 'exp_p', 'fill_constant_p', 'sub_p' + 'sigmoid', + 'div_p', + 'fill_constant_p', + 'add_p', + 'fill_constant_p', + 'exp_p', + 'fill_constant_p', + 'sub_p', ] self.out_map = {0: self.output['Out']} diff --git a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py index 13423690ed8..b864bdaa45f 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py @@ -24,14 +24,14 @@ paddle.enable_static() ############################ Test prim2orig rules ############################ class TestAddPPrim2Orig(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() self.layer_help = LayerHelper('TestPrim2Orig') - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_data() def init_data(self): @@ -41,8 +41,9 @@ class TestAddPPrim2Orig(unittest.TestCase): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -52,12 +53,15 @@ class TestAddPPrim2Orig(unittest.TestCase): self.out_map = {self.output['Z']: 0} def test_op(self): - with paddle.static.program_guard(self.main_program, - self.startup_program): - op = self.layer_help.append_op(type=self.op_type, - inputs=self.input, - outputs=self.output, - attrs=self.attrs) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + op = self.layer_help.append_op( + type=self.op_type, + inputs=self.input, + outputs=self.output, + attrs=self.attrs, + ) orig_out = _prim2orig(op, *self.prim2orig_args) all_ops = [op.type for op in self.main_program.block(0).ops] @@ -68,7 +72,6 @@ class TestAddPPrim2Orig(unittest.TestCase): class TestSubPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'sub_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -76,8 +79,9 @@ class TestSubPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -87,7 +91,6 @@ class TestSubPPrim2Orig(TestAddPPrim2Orig): class TestMulPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'mul_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -95,8 +98,9 @@ class TestMulPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -106,7 +110,6 @@ class TestMulPPrim2Orig(TestAddPPrim2Orig): class TestDivPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'div_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -114,8 +117,9 @@ class TestDivPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -125,7 +129,6 @@ class TestDivPPrim2Orig(TestAddPPrim2Orig): class TestSqrtPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'sqrt_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -134,18 +137,18 @@ class TestSqrtPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['sqrt_p', 'sqrt'] self.out_map = {self.output['Y']: 0} class TestTanhPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'tanh_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -154,18 +157,18 @@ class TestTanhPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['tanh_p', 'tanh'] self.out_map = {self.output['Y']: 0} class TestSinPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'sin_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -174,18 +177,18 @@ class TestSinPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['sin_p', 'sin'] self.out_map = {self.output['Y']: 0} class TestCosPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'cos_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -194,18 +197,18 @@ class TestCosPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['cos_p', 'cos'] self.out_map = {self.output['Y']: 0} class TestExpPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'exp_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -214,18 +217,18 @@ class TestExpPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['exp_p', 'exp'] self.out_map = {self.output['Y']: 0} class TestErfPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'erf_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -234,18 +237,18 @@ class TestErfPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['erf_p', 'erf'] self.out_map = {self.output['Y']: 0} class TestAbsPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'abs_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -254,18 +257,18 @@ class TestAbsPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['abs_p', 'abs'] self.out_map = {self.output['Y']: 0} class TestLogPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'log_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -274,18 +277,18 @@ class TestLogPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['log_p', 'log'] self.out_map = {self.output['Y']: 0} class TestReshapePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'reshape_p' X = paddle.static.data(name='X', shape=[2, 8], dtype='float64') @@ -294,18 +297,18 @@ class TestReshapePPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'shape': [4, 4]} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['reshape_p', 'reshape2'] self.out_map = {self.output['Y']: 0} class TestBroadcastPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'broadcast_p' X = paddle.static.data(name='X', shape=[2, 8], dtype='float64') @@ -314,18 +317,18 @@ class TestBroadcastPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'shape': [10, 2, 8]} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['broadcast_p', 'expand_v2'] self.out_map = {self.output['Y']: 0} class TestTransposePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'transpose_p' X = paddle.static.data(name='X', shape=[7, 8, 9, 10], dtype='float64') @@ -334,18 +337,18 @@ class TestTransposePPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [1, 2, 0, 3]} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['transpose_p', 'transpose2'] self.out_map = {self.output['Y']: 0} class TestSplitPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'split_p' X = paddle.static.data(name='X', shape=[3, 9, 5], dtype='float64') @@ -356,12 +359,14 @@ class TestSplitPPrim2Orig(TestAddPPrim2Orig): self.output = { 'YS': [ self.layer_help.create_variable_for_type_inference( - dtype=X.dtype) for i in range(3) + dtype=X.dtype + ) + for i in range(3) ] } self.attrs = {'num_or_sections': [2, 3, 4], 'axis': 1} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['split_p', 'split'] self.out_map = { self.output['YS'][0]: 0, @@ -371,7 +376,6 @@ class TestSplitPPrim2Orig(TestAddPPrim2Orig): class TestConcatPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'concat_p' X = paddle.static.data(name='X', shape=[3, 9, 5], dtype='float64') @@ -382,36 +386,36 @@ class TestConcatPPrim2Orig(TestAddPPrim2Orig): 'XS': [X, Y, Z], } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': 0} - self.prim2orig_args = ((X, Y, Z), ) + self.prim2orig_args = ((X, Y, Z),) self.all_ops = ['concat_p', 'concat'] self.out_map = {self.output['Y']: 0} class TestReducePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'reduce_sum_p' X = paddle.static.data(name='X', shape=[3, 9, 5], dtype='float64') self.input = {'X': X} self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [1], 'keepdim': True} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['reduce_sum_p', 'reduce_sum'] self.out_map = {self.output['Y']: 0} class TestMatmulPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'matmul_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') @@ -419,8 +423,9 @@ class TestMatmulPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -430,7 +435,6 @@ class TestMatmulPPrim2Orig(TestAddPPrim2Orig): class TestSliceSelectPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'slice_select_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') @@ -439,18 +443,18 @@ class TestSliceSelectPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [0], 'starts': [1], 'ends': [8], 'strides': [2]} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['slice_select_p', 'strided_slice'] self.out_map = {self.output['Y']: 0} class TestSliceAssignPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'slice_assign_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') @@ -458,8 +462,9 @@ class TestSliceAssignPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [1], 'starts': [0], 'ends': [3], 'strides': [1]} @@ -469,18 +474,18 @@ class TestSliceAssignPPrim2Orig(TestAddPPrim2Orig): class TestGatherPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'gather_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') - IndexTensor = paddle.static.data(name='IndexTensor', - shape=[3], - dtype='int32') + IndexTensor = paddle.static.data( + name='IndexTensor', shape=[3], dtype='int32' + ) self.input = {'X': X, 'IndexTensor': IndexTensor} self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'axis': 0, @@ -495,19 +500,19 @@ class TestGatherPPrim2Orig(TestAddPPrim2Orig): class TestScatterAddPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'scatter_add_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') Y = paddle.static.data(name='Y', shape=[3, 5], dtype='float64') - IndexTensor = paddle.static.data(name='IndexTensor', - shape=[3], - dtype='int32') + IndexTensor = paddle.static.data( + name='IndexTensor', shape=[3], dtype='int32' + ) self.input = {'X': X, 'Y': Y, 'IndexTensor': IndexTensor} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'axis': 0, @@ -515,20 +520,23 @@ class TestScatterAddPPrim2Orig(TestAddPPrim2Orig): self.prim2orig_args = (IndexTensor, X, Y) self.all_ops = [ - 'scatter_add_p', 'fill_any_like', 'scatter', 'elementwise_add' + 'scatter_add_p', + 'fill_any_like', + 'scatter', + 'elementwise_add', ] self.out_map = {self.output['Z']: 0} class TestFillConstantPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'fill_constant_p' self.input = {} self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(paddle.int32) + 'Y': self.layer_help.create_variable_for_type_inference( + paddle.int32 + ) } self.attrs = {'value': 10, 'shape': [5, 5], 'dtype': paddle.int32} @@ -538,7 +546,6 @@ class TestFillConstantPPrim2Orig(TestAddPPrim2Orig): class TestSelectPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'select_p' Cond = paddle.static.data(name='Condition', shape=[5, 6], dtype='bool') @@ -547,8 +554,9 @@ class TestSelectPPrim2Orig(TestAddPPrim2Orig): self.input = {'Condition': Cond, 'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} self.prim2orig_args = (Cond, X, Y) @@ -557,7 +565,6 @@ class TestSelectPPrim2Orig(TestAddPPrim2Orig): class TestEqPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'eq_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -565,8 +572,9 @@ class TestEqPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Z': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} @@ -576,7 +584,6 @@ class TestEqPPrim2Orig(TestAddPPrim2Orig): class TestNePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'ne_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -584,8 +591,9 @@ class TestNePPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Z': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} @@ -595,7 +603,6 @@ class TestNePPrim2Orig(TestAddPPrim2Orig): class TestGtPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'gt_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -603,8 +610,9 @@ class TestGtPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Z': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} @@ -614,7 +622,6 @@ class TestGtPPrim2Orig(TestAddPPrim2Orig): class TestGePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'ge_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -622,8 +629,9 @@ class TestGePPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Z': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} @@ -633,7 +641,6 @@ class TestGePPrim2Orig(TestAddPPrim2Orig): class TestPowPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'pow_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -641,8 +648,9 @@ class TestPowPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -652,7 +660,6 @@ class TestPowPPrim2Orig(TestAddPPrim2Orig): class TestMaxPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'max_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -660,8 +667,9 @@ class TestMaxPPrim2Orig(TestAddPPrim2Orig): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -671,15 +679,14 @@ class TestMaxPPrim2Orig(TestAddPPrim2Orig): class TestBernoulliPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'bernoulli_p' self.input = {} self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float64) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float64 + ) } self.attrs = {'shape': [7, 8], 'dtype': paddle.float64, 'p': 0.5} @@ -689,7 +696,6 @@ class TestBernoulliPPrim2Orig(TestAddPPrim2Orig): class TestCastPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'cast_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -698,18 +704,18 @@ class TestCastPPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'dtype': paddle.int64} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['cast_p', 'cast'] self.out_map = {self.output['Y']: 0} class TestRsqrtPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'rsqrt_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -718,33 +724,33 @@ class TestRsqrtPrim2Orig(TestAddPPrim2Orig): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['rsqrt_p', 'rsqrt'] self.out_map = {self.output['Y']: 0} class TestUniformRandomPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'uniform_random_p' self.input = {} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float64) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float64 + ) } self.attrs = { 'shape': [1, 2, 3], 'min': -1.0, 'max': 1.0, 'seed': 0, - 'dtype': paddle.float64 + 'dtype': paddle.float64, } self.prim2orig_args = () diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py index 2451ed4190c..7f1a06c9240 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py @@ -27,13 +27,30 @@ from paddle.incubate.autograd import primx @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'xs', 'dtype'), ( - ('uniform_random', - lambda: paddle.uniform([1, 2, 3], dtype='float32', min=0, max=1.0, seed=1), - (), 'int32'), ('sigmoid', paddle.nn.functional.sigmoid, - (np.random.rand(5, ), ), 'float32'))) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'xs', 'dtype'), + ( + ( + 'uniform_random', + lambda: paddle.uniform( + [1, 2, 3], dtype='float32', min=0, max=1.0, seed=1 + ), + (), + 'int32', + ), + ( + 'sigmoid', + paddle.nn.functional.sigmoid, + ( + np.random.rand( + 5, + ), + ), + 'float32', + ), + ), +) class TestFowardApi(unittest.TestCase): - @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) @@ -47,14 +64,14 @@ class TestFowardApi(unittest.TestCase): paddle.disable_static() def test_grad(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs = utils.gen_static_inputs_and_feed( - self.xs, stop_gradient=False) + self.xs, stop_gradient=False + ) out = self.fun(*static_xs) exe = paddle.static.Executor() exe.run(sp) @@ -68,7 +85,8 @@ class TestFowardApi(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs = utils.gen_static_inputs_and_feed( - self.xs, stop_gradient=False) + self.xs, stop_gradient=False + ) out = self.fun(*static_xs) primx.orig2prim(mp.block(0)) primx.prim2orig(mp.block(0)) @@ -86,18 +104,32 @@ class TestFowardApi(unittest.TestCase): @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), - (('dropout', paddle.nn.functional.dropout, - (np.random.rand(5000, 5000), ), None, 'float32'), )) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), + ( + ( + 'dropout', + paddle.nn.functional.dropout, + (np.random.rand(5000, 5000),), + None, + 'float32', + ), + ), +) class TestDropoutGrad(unittest.TestCase): - @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def setUp(self): paddle.enable_static() @@ -108,16 +140,17 @@ class TestDropoutGrad(unittest.TestCase): paddle.disable_static() def test_grad(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) + self.xs, self.v, stop_gradient=False + ) _, ys_grad = paddle.incubate.autograd.vjp( - self.fun, static_xs, static_v) + self.fun, static_xs, static_v + ) exe = paddle.static.Executor() exe.run(sp) out = exe.run(mp, feed=feed, fetch_list=ys_grad) @@ -130,9 +163,13 @@ class TestDropoutGrad(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() @@ -151,17 +188,30 @@ class TestDropoutGrad(unittest.TestCase): @utils.place(config.DEVICES) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), - (('matmul', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), )) + ( + ( + 'matmul', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + None, + 'float32', + ), + ), +) class TestWithoutProgramGuard(unittest.TestCase): - @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def setUp(self): paddle.enable_static() @@ -172,18 +222,22 @@ class TestWithoutProgramGuard(unittest.TestCase): paddle.disable_static() def test_forward_grad_without_program_guard(self): - def with_program_guard(): paddle.incubate.autograd.enable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.forward_grad( - ys, static_xs, static_v) + ys, static_xs, static_v + ) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() exe.run(sp) @@ -194,11 +248,16 @@ class TestWithoutProgramGuard(unittest.TestCase): def without_program_guard(): paddle.incubate.autograd.enable_prim() feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.forward_grad( - ys, static_xs, static_v) + ys, static_xs, static_v + ) sp = paddle.fluid.framework.default_startup_program() mp = paddle.fluid.framework.default_main_program() exe = paddle.static.Executor() @@ -210,22 +269,27 @@ class TestWithoutProgramGuard(unittest.TestCase): expected = with_program_guard() actual = without_program_guard() self.assertEqual(type(actual), type(expected)) - np.testing.assert_allclose(np.concatenate(actual), - np.concatenate(expected), - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + np.concatenate(actual), + np.concatenate(expected), + rtol=self._rtol, + atol=self._atol, + ) def test_grad_without_program_guard(self): - def with_program_guard(): paddle.incubate.autograd.enable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) xs_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() @@ -237,9 +301,13 @@ class TestWithoutProgramGuard(unittest.TestCase): def without_program_guard(): paddle.incubate.autograd.enable_prim() feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) xs_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) sp = paddle.fluid.framework.default_startup_program() mp = paddle.fluid.framework.default_main_program() @@ -253,43 +321,91 @@ class TestWithoutProgramGuard(unittest.TestCase): actual = without_program_guard() for i, j in zip(actual, expected): self.assertEqual(type(i), type(j)) - np.testing.assert_allclose(np.concatenate(i), - np.concatenate(j), - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + np.concatenate(i), + np.concatenate(j), + rtol=self._rtol, + atol=self._atol, + ) @utils.place(config.DEVICES) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), - (('matmul', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), - ('multiply', paddle.multiply, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), - ('add', paddle.add, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), - ('input_not_sequence', paddle.tanh, - (np.random.rand(5, 5), ), None, 'float64'), - ('input_gradients_not_none', paddle.matmul, - (np.random.rand(3, 3), np.random.rand(3, 3)), - (np.random.rand(3, 3), np.random.rand(3, 3)), 'float64'), - ('log', paddle.log, (np.random.rand(3, 4), ), None, 'float32'), - ('abs', paddle.abs, (np.random.uniform(-10, 10, - (10, 10)), ), None, 'float32'), - ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200), ), None, 'float32'), - ('sigmoid', paddle.nn.functional.sigmoid, - (np.random.rand(5, ), ), None, 'float32'))) + ( + ( + 'matmul', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + None, + 'float32', + ), + ( + 'multiply', + paddle.multiply, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float64', + ), + ( + 'add', + paddle.add, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float32', + ), + ( + 'input_not_sequence', + paddle.tanh, + (np.random.rand(5, 5),), + None, + 'float64', + ), + ( + 'input_gradients_not_none', + paddle.matmul, + (np.random.rand(3, 3), np.random.rand(3, 3)), + (np.random.rand(3, 3), np.random.rand(3, 3)), + 'float64', + ), + ('log', paddle.log, (np.random.rand(3, 4),), None, 'float32'), + ( + 'abs', + paddle.abs, + (np.random.uniform(-10, 10, (10, 10)),), + None, + 'float32', + ), + ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200),), None, 'float32'), + ( + 'sigmoid', + paddle.nn.functional.sigmoid, + ( + np.random.rand( + 5, + ), + ), + None, + 'float32', + ), + ), +) # paddle.where, paddle.pow, paddle.maximum has no double grad definition, # can not compute forward grad use double trick class TestForwardGrad(unittest.TestCase): - @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def setUp(self): paddle.enable_static() @@ -300,16 +416,17 @@ class TestForwardGrad(unittest.TestCase): paddle.disable_static() def test_forward_grad(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) + self.xs, self.v, stop_gradient=False + ) _, ys_grad = paddle.incubate.autograd.jvp( - self.fun, static_xs, static_v) + self.fun, static_xs, static_v + ) exe = paddle.static.Executor() exe.run(sp) out = exe.run(mp, feed=feed, fetch_list=ys_grad) @@ -322,11 +439,16 @@ class TestForwardGrad(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.forward_grad( - ys, static_xs, static_v) + ys, static_xs, static_v + ) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() exe.run(sp) @@ -337,10 +459,12 @@ class TestForwardGrad(unittest.TestCase): actual = actual() expected = expected() self.assertEqual(type(actual), type(expected)) - np.testing.assert_allclose(np.concatenate(actual), - np.concatenate(expected), - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + np.concatenate(actual), + np.concatenate(expected), + rtol=self._rtol, + atol=self._atol, + ) def test_prim_disabled(self): paddle.incubate.autograd.disable_prim() @@ -349,11 +473,16 @@ class TestForwardGrad(unittest.TestCase): with self.assertRaises(RuntimeError): with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.forward_grad( - ys, static_xs, static_v) + ys, static_xs, static_v + ) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() exe.run(sp) @@ -364,11 +493,13 @@ class TestForwardGrad(unittest.TestCase): paddle.incubate.autograd.enable_prim() with self.assertRaises(TypeError): paddle.incubate.autograd.forward_grad( - 1, paddle.static.data('inputs', shape=[1])) + 1, paddle.static.data('inputs', shape=[1]) + ) with self.assertRaises(TypeError): paddle.incubate.autograd.forward_grad( - paddle.static.data('targets', shape=[1]), 1) + paddle.static.data('targets', shape=[1]), 1 + ) paddle.incubate.autograd.disable_prim() @@ -379,107 +510,247 @@ where_wrap = lambda x, y: paddle.where(paddle.eye(3, 4) == 1, x, y) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), ( - ('matmul', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), - ('multiply', paddle.multiply, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), - ('div', paddle.divide, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), - ('add', paddle.add, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), - ('input_not_sequence', paddle.tanh, - (np.random.rand(5, 5), ), None, 'float64'), - ('input_gradients_not_none', paddle.matmul, - (np.random.rand(3, 3), np.random.rand(3, 3)), - (np.random.rand(3, 3), ), 'float64'), - ('sin', paddle.sin, (np.random.rand(100, 200), ), None, 'float32'), - ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200), ), None, 'float32'), - ('cos', paddle.cos, (np.random.rand(200, 90), ), None, 'float32'), - ('exp', paddle.exp, (np.random.rand(299, 320), ), None, 'float32'), + ( + 'matmul', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + None, + 'float32', + ), + ( + 'multiply', + paddle.multiply, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float64', + ), + ( + 'div', + paddle.divide, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float64', + ), + ( + 'add', + paddle.add, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float32', + ), + ( + 'input_not_sequence', + paddle.tanh, + (np.random.rand(5, 5),), + None, + 'float64', + ), + ( + 'input_gradients_not_none', + paddle.matmul, + (np.random.rand(3, 3), np.random.rand(3, 3)), + (np.random.rand(3, 3),), + 'float64', + ), + ('sin', paddle.sin, (np.random.rand(100, 200),), None, 'float32'), + ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200),), None, 'float32'), + ('cos', paddle.cos, (np.random.rand(200, 90),), None, 'float32'), + ('exp', paddle.exp, (np.random.rand(299, 320),), None, 'float32'), # In where op, grad of condition computed by paddle.static.gradients is None, # and paddle.incubate.autograd.grad will replace None with zeros while transpose # will just return None because cond_dot is unused, that is a diff. - ('select', where_wrap, - (np.random.rand(3, 4), np.random.rand(3, 4)), None, 'float32'), + ( + 'select', + where_wrap, + (np.random.rand(3, 4), np.random.rand(3, 4)), + None, + 'float32', + ), # pow_p and pow has diff when compute z_dot of 0^0 - ('pow', paddle.pow, - (np.array([1, 2, 3]), np.array([0, 2, 7])), None, 'float32'), + ( + 'pow', + paddle.pow, + (np.array([1, 2, 3]), np.array([0, 2, 7])), + None, + 'float32', + ), # To make max_p consistent with paddle.maximum, be sure x.grad = 0 and y.grad = 1 when x==y. - ('max', paddle.maximum, ( - np.array([1, 2, 3]), - np.array([2, 2, 2]), - ), None, 'float32'), - ('erf', paddle.erf, (np.random.rand(300, 288), ), None, 'float32'), - ('gelu', paddle.nn.functional.gelu, - (np.random.rand(200, 189), ), None, 'float32'), - ('gelu_approximate', lambda x: paddle.nn.functional.gelu(x, True), - (np.random.rand(200, 189), ), None, 'float32'), - ('sum', paddle.sum, (np.random.rand(200, 345), ), None, 'float32'), - ('sigmoid', paddle.nn.functional.sigmoid, - (np.random.rand(5, ), ), None, 'float32'), - ('sum_with_axis', lambda x: paddle.sum(x, axis=1), - (np.random.rand(200, 345), ), None, 'float32'), - ('sum_with_keepdim', lambda x: paddle.sum(x, keepdim=True), - (np.random.rand(200, 345), ), None, 'float32'), - ('mean', paddle.mean, (np.random.rand(200, 345), ), None, 'float32'), - ('mean_with_axis', lambda x: paddle.mean(x, axis=1), - (np.random.rand(200, 345), ), None, 'float32'), - ('mean_with_keepdim', lambda x: paddle.mean(x, keepdim=True), - (np.random.rand(200, 345), ), None, 'float32'), - ('mean_with_axis_keepdim', - lambda x: paddle.mean(x, axis=0, keepdim=True), - (np.random.rand(200, 345), ), None, 'float32'), - ('abs', paddle.abs, (np.random.uniform(-10, 10, - (200, 345)), ), None, 'float32'), - ('cast_float', lambda x: paddle.cast(x, paddle.float64), - (np.random.rand(10, 20), ), None, 'float32'), - ('cast_int', lambda x: paddle.cast(x, paddle.int32), - (np.random.rand(10, 20), ), None, 'float32'), - ('square', paddle.square, (np.random.rand(100), ), None, 'float32'), - ('pow_scalar', lambda x: paddle.pow(x, 2), - (np.random.rand(20, 30), ), None, 'float32'), - ('var', paddle.var, (np.random.rand(200, 324), ), None, 'float32'), - ('var_with_axis', lambda x: paddle.var(x, axis=1), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('var_without_unbiased', - lambda x: paddle.var(x, axis=1, unbiased=False), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('var_with_keepdim', lambda x: paddle.var(x, axis=1, keepdim=True), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('bn', lambda x, w, b: paddle.nn.functional.batch_norm( - x, paddle.ones((10, )), paddle.ones( - (10, )), w, b), (np.random.rand(10, 10), np.random.rand(10), - np.random.rand(10)), None, 'float32'), - ('bn_train', lambda x, w, b: paddle.nn.functional.batch_norm( - x, paddle.ones((10, )), paddle.ones((10, )), w, b, training=True), - (np.random.rand( - 10, 10), np.random.rand(10), np.random.rand(10)), None, 'float32'), - ('bn_nhwc', lambda x, w, b: paddle.nn.functional.batch_norm( - x, - paddle.ones((10, )) + 1, - paddle.ones((10, )), - w, - b, - training=True, - data_format='NHWC', - ), (np.random.rand( - 10, 10), np.random.rand(10), np.random.rand(10)), None, 'float32'), - ('bn_global_stat', - lambda x, w, b: paddle.nn.functional.batch_norm(x, - paddle.ones( - (10, )) + 3.2, - paddle.ones( - (10, )) + 6.7, - w, - b, - training=True, - data_format='NHWC', - use_global_stats=True), - (np.random.rand( - 10, 10), np.random.rand(10), np.random.rand(10)), None, 'float32'), - )) + ( + 'max', + paddle.maximum, + ( + np.array([1, 2, 3]), + np.array([2, 2, 2]), + ), + None, + 'float32', + ), + ('erf', paddle.erf, (np.random.rand(300, 288),), None, 'float32'), + ( + 'gelu', + paddle.nn.functional.gelu, + (np.random.rand(200, 189),), + None, + 'float32', + ), + ( + 'gelu_approximate', + lambda x: paddle.nn.functional.gelu(x, True), + (np.random.rand(200, 189),), + None, + 'float32', + ), + ('sum', paddle.sum, (np.random.rand(200, 345),), None, 'float32'), + ( + 'sigmoid', + paddle.nn.functional.sigmoid, + ( + np.random.rand( + 5, + ), + ), + None, + 'float32', + ), + ( + 'sum_with_axis', + lambda x: paddle.sum(x, axis=1), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ( + 'sum_with_keepdim', + lambda x: paddle.sum(x, keepdim=True), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ('mean', paddle.mean, (np.random.rand(200, 345),), None, 'float32'), + ( + 'mean_with_axis', + lambda x: paddle.mean(x, axis=1), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ( + 'mean_with_keepdim', + lambda x: paddle.mean(x, keepdim=True), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ( + 'mean_with_axis_keepdim', + lambda x: paddle.mean(x, axis=0, keepdim=True), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ( + 'abs', + paddle.abs, + (np.random.uniform(-10, 10, (200, 345)),), + None, + 'float32', + ), + ( + 'cast_float', + lambda x: paddle.cast(x, paddle.float64), + (np.random.rand(10, 20),), + None, + 'float32', + ), + ( + 'cast_int', + lambda x: paddle.cast(x, paddle.int32), + (np.random.rand(10, 20),), + None, + 'float32', + ), + ('square', paddle.square, (np.random.rand(100),), None, 'float32'), + ( + 'pow_scalar', + lambda x: paddle.pow(x, 2), + (np.random.rand(20, 30),), + None, + 'float32', + ), + ('var', paddle.var, (np.random.rand(200, 324),), None, 'float32'), + ( + 'var_with_axis', + lambda x: paddle.var(x, axis=1), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'var_without_unbiased', + lambda x: paddle.var(x, axis=1, unbiased=False), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'var_with_keepdim', + lambda x: paddle.var(x, axis=1, keepdim=True), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'bn', + lambda x, w, b: paddle.nn.functional.batch_norm( + x, paddle.ones((10,)), paddle.ones((10,)), w, b + ), + (np.random.rand(10, 10), np.random.rand(10), np.random.rand(10)), + None, + 'float32', + ), + ( + 'bn_train', + lambda x, w, b: paddle.nn.functional.batch_norm( + x, paddle.ones((10,)), paddle.ones((10,)), w, b, training=True + ), + (np.random.rand(10, 10), np.random.rand(10), np.random.rand(10)), + None, + 'float32', + ), + ( + 'bn_nhwc', + lambda x, w, b: paddle.nn.functional.batch_norm( + x, + paddle.ones((10,)) + 1, + paddle.ones((10,)), + w, + b, + training=True, + data_format='NHWC', + ), + (np.random.rand(10, 10), np.random.rand(10), np.random.rand(10)), + None, + 'float32', + ), + ( + 'bn_global_stat', + lambda x, w, b: paddle.nn.functional.batch_norm( + x, + paddle.ones((10,)) + 3.2, + paddle.ones((10,)) + 6.7, + w, + b, + training=True, + data_format='NHWC', + use_global_stats=True, + ), + (np.random.rand(10, 10), np.random.rand(10), np.random.rand(10)), + None, + 'float32', + ), + ), +) class TestGrad(unittest.TestCase): - def setUp(self): paddle.enable_static() paddle.incubate.autograd.enable_prim() @@ -491,22 +762,29 @@ class TestGrad(unittest.TestCase): @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def test_grad(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) + self.xs, self.v, stop_gradient=False + ) _, ys_grad = paddle.incubate.autograd.vjp( - self.fun, static_xs, static_v) + self.fun, static_xs, static_v + ) exe = paddle.static.Executor() exe.run(sp) out = exe.run(mp, feed=feed, fetch_list=ys_grad) @@ -519,9 +797,13 @@ class TestGrad(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() @@ -540,24 +822,29 @@ class TestGrad(unittest.TestCase): paddle.incubate.autograd.enable_prim() with self.assertRaises(TypeError): paddle.incubate.autograd.grad( - 1, paddle.static.data('inputs', shape=[1])) + 1, paddle.static.data('inputs', shape=[1]) + ) with self.assertRaises(TypeError): paddle.incubate.autograd.grad( - paddle.static.data('targets', shape=[1]), 1) + paddle.static.data('targets', shape=[1]), 1 + ) paddle.incubate.autograd.disable_prim() def test_disable_prim(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) exe = paddle.static.Executor() exe.run(sp) @@ -571,9 +858,13 @@ class TestGrad(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.static.gradients(ys, static_xs, static_v) exe = paddle.static.Executor() exe.run(sp) @@ -599,7 +890,7 @@ multiply_ag = lambda xs: xs[0] * xs[0] * xs[0] * xs[0] * xs[0] sin_ag = lambda xs: anp.sin(xs[0]) cos_ag = lambda xs: anp.cos(xs[0]) exp_ag = lambda xs: anp.exp(xs[0]) -pow_ag = lambda xs: xs[0]**xs[1] +pow_ag = lambda xs: xs[0] ** xs[1] log_ag = lambda xs: anp.log(xs[0]) erf_ag = lambda xs: ascipy.special.erf(xs[0]) sigmoid_ag = lambda xs: 1.0 / (1 + anp.exp(-xs[0])) @@ -616,28 +907,63 @@ def gelu_ag(x, approximate=False): @utils.place(config.DEVICES) @utils.parameterize( - (utils.TEST_CASE_NAME, 'fun_pd', 'fun_ag', 'xs', 'v', 'dtype'), ( - ('multiply', multiply_pd, multiply_ag, - (np.random.rand(3, 5), ), None, 'float32'), - ('sin', paddle.sin, sin_ag, (np.random.rand(2, 3), ), None, 'float32'), - ('cos', paddle.cos, cos_ag, (np.random.rand(3, 4), ), None, 'float32'), - ('exp', paddle.exp, exp_ag, (np.random.rand(2, 3), ), None, 'float32'), - ('pow', paddle.pow, pow_ag, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), - ('log', paddle.log, log_ag, (np.random.rand(3, 8), ), None, 'float32'), - ('erf', paddle.erf, erf_ag, - (np.random.rand(100, 200), ), None, 'float32'), - ('gelu', paddle.nn.functional.gelu, lambda xs: gelu_ag(xs[0]), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('gelu_approximate', - lambda x: paddle.nn.functional.gelu(x, approximate=True), - lambda xs: gelu_ag(xs[0], approximate=True), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('sigmoid', paddle.nn.functional.sigmoid, sigmoid_ag, - (np.random.rand(10, 20), ), None, 'float32'), - )) + (utils.TEST_CASE_NAME, 'fun_pd', 'fun_ag', 'xs', 'v', 'dtype'), + ( + ( + 'multiply', + multiply_pd, + multiply_ag, + (np.random.rand(3, 5),), + None, + 'float32', + ), + ('sin', paddle.sin, sin_ag, (np.random.rand(2, 3),), None, 'float32'), + ('cos', paddle.cos, cos_ag, (np.random.rand(3, 4),), None, 'float32'), + ('exp', paddle.exp, exp_ag, (np.random.rand(2, 3),), None, 'float32'), + ( + 'pow', + paddle.pow, + pow_ag, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float32', + ), + ('log', paddle.log, log_ag, (np.random.rand(3, 8),), None, 'float32'), + ( + 'erf', + paddle.erf, + erf_ag, + (np.random.rand(100, 200),), + None, + 'float32', + ), + ( + 'gelu', + paddle.nn.functional.gelu, + lambda xs: gelu_ag(xs[0]), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'gelu_approximate', + lambda x: paddle.nn.functional.gelu(x, approximate=True), + lambda xs: gelu_ag(xs[0], approximate=True), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'sigmoid', + paddle.nn.functional.sigmoid, + sigmoid_ag, + (np.random.rand(10, 20),), + None, + 'float32', + ), + ), +) class TestGradWithHigherOrder(unittest.TestCase): - def setUp(self): paddle.enable_static() paddle.incubate.autograd.enable_prim() @@ -649,13 +975,18 @@ class TestGradWithHigherOrder(unittest.TestCase): @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def test_grad(self): - def expected(): egrad = autograd.elementwise_grad grad_3 = egrad(egrad(egrad(self.fun_ag)))(self.xs) @@ -671,9 +1002,13 @@ class TestGradWithHigherOrder(unittest.TestCase): startup = paddle.static.Program() with paddle.static.program_guard(main, startup): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun_pd(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun_pd(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun_pd(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun_pd(static_xs) + ) grad1 = paddle_grad(ys, static_xs, static_v) grad2 = paddle_grad(grad1, static_xs, static_v) diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primops.py b/python/paddle/fluid/tests/unittests/autograd/test_primops.py index 677c064bb78..592229aa3dc 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primops.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primops.py @@ -27,8 +27,14 @@ paddle.enable_static() @utils.place(config.DEVICES) @utils.parameterize( - (utils.TEST_CASE_NAME, 'op', 'args', 'kwargs', 'expected_shape', - 'expected_dtype'), + ( + utils.TEST_CASE_NAME, + 'op', + 'args', + 'kwargs', + 'expected_shape', + 'expected_dtype', + ), ( ('add', primops.add, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), ('sub', primops.sub, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), @@ -43,73 +49,144 @@ paddle.enable_static() ('erf', primops.erf, randn(2, 3), {}, (2, 3), 'float64'), ('abs', primops.abs, randn(2, 3), {}, (2, 3), 'float64'), ('log', primops.log, randn(2, 3), {}, (2, 3), 'float64'), - ('cast', primops.cast, randn(2, 3), { - 'dtype': paddle.int64 - }, (2, 3), 'int64'), - ('reshape', primops.reshape, randn(2, 3), { - 'shape': (3, 2) - }, (3, 2), 'float64'), - ('broadcast', primops.broadcast, randn(2), { - 'shape': (3, 2) - }, (3, 2), 'float64'), - ('transpose', primops.transpose, randn(2, 3), { - 'axis': (1, 0) - }, (3, 2), 'float64'), - ('concat_axis0', primops.concat, ((randn(2, 3), randn(2, 3)), ), { - 'axis': 0 - }, (4, 3), 'float64'), - ('concat_axis1', primops.concat, ((randn(2, 3), randn(2, 3)), ), { - 'axis': 1 - }, (2, 6), 'float64'), - ('reduce_axis1', primops.reduce_sum, randn(2, 3), { - 'axis': (1, ) - }, (2, ), 'float64'), - ('reduce_axis01', primops.reduce_sum, randn(2, 3), { - 'axis': (0, 1) - }, (1, ), 'float64'), - ('split', primops.split, randn(2, 3), { - 'num_or_sections': [1, 2], - 'axis': 1 - }, ((2, 1), (2, 2)), ('float64', 'float64')), - ('matmul', primops.matmul, (randn(2, 3), randn(3, 2)), {}, - (2, 2), 'float64'), - ('slice_select', primops.slice_select, randn(3, 2), { - 'axis': [0], - 'starts': [0], - 'ends': [2], - 'strides': [1] - }, (2, 2), 'float64'), - ('slice_assign', primops.slice_assign, (randn(2, 3), randn(2, 2)), { - 'axis': [1], - 'starts': [1], - 'ends': [3], - 'strides': [1] - }, (2, 3), 'float64'), - ('gather', primops.gather, (randn(3, 2), randint(0, 2, - (5, ), np.int32)), { - 'axis': 0 - }, (5, 2), 'float64'), - ('scatter_add', primops.scatter_add, - (randn(3, 2), randn(5, 2), randint(0, 2, (5, ), np.int32)), { - 'axis': 0 - }, (3, 2), 'float64'), - ('fill_const', primops.fill_const, (), { - 'value': 10, - 'shape': (3, 2), - 'dtype': paddle.float32 - }, (3, 2), 'float32'), + ( + 'cast', + primops.cast, + randn(2, 3), + {'dtype': paddle.int64}, + (2, 3), + 'int64', + ), + ( + 'reshape', + primops.reshape, + randn(2, 3), + {'shape': (3, 2)}, + (3, 2), + 'float64', + ), + ( + 'broadcast', + primops.broadcast, + randn(2), + {'shape': (3, 2)}, + (3, 2), + 'float64', + ), + ( + 'transpose', + primops.transpose, + randn(2, 3), + {'axis': (1, 0)}, + (3, 2), + 'float64', + ), + ( + 'concat_axis0', + primops.concat, + ((randn(2, 3), randn(2, 3)),), + {'axis': 0}, + (4, 3), + 'float64', + ), + ( + 'concat_axis1', + primops.concat, + ((randn(2, 3), randn(2, 3)),), + {'axis': 1}, + (2, 6), + 'float64', + ), + ( + 'reduce_axis1', + primops.reduce_sum, + randn(2, 3), + {'axis': (1,)}, + (2,), + 'float64', + ), + ( + 'reduce_axis01', + primops.reduce_sum, + randn(2, 3), + {'axis': (0, 1)}, + (1,), + 'float64', + ), + ( + 'split', + primops.split, + randn(2, 3), + {'num_or_sections': [1, 2], 'axis': 1}, + ((2, 1), (2, 2)), + ('float64', 'float64'), + ), + ( + 'matmul', + primops.matmul, + (randn(2, 3), randn(3, 2)), + {}, + (2, 2), + 'float64', + ), + ( + 'slice_select', + primops.slice_select, + randn(3, 2), + {'axis': [0], 'starts': [0], 'ends': [2], 'strides': [1]}, + (2, 2), + 'float64', + ), + ( + 'slice_assign', + primops.slice_assign, + (randn(2, 3), randn(2, 2)), + {'axis': [1], 'starts': [1], 'ends': [3], 'strides': [1]}, + (2, 3), + 'float64', + ), + ( + 'gather', + primops.gather, + (randn(3, 2), randint(0, 2, (5,), np.int32)), + {'axis': 0}, + (5, 2), + 'float64', + ), + ( + 'scatter_add', + primops.scatter_add, + (randn(3, 2), randn(5, 2), randint(0, 2, (5,), np.int32)), + {'axis': 0}, + (3, 2), + 'float64', + ), + ( + 'fill_const', + primops.fill_const, + (), + {'value': 10, 'shape': (3, 2), 'dtype': paddle.float32}, + (3, 2), + 'float32', + ), ('neg', primops.neg, randn(2, 3), {}, (2, 3), 'float64'), - ('select', primops.select, - (randn(2, 3) > 0, randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), + ( + 'select', + primops.select, + (randn(2, 3) > 0, randn(2, 3), randn(2, 3)), + {}, + (2, 3), + 'float64', + ), ('eq', primops.eq, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'bool'), ('ne', primops.ne, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'bool'), ('gt', primops.gt, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'bool'), ('ge', primops.ge, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'bool'), ('pow', primops.pow, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), ('max', primops.max, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), - )) + ), +) class TestPrimops(unittest.TestCase): - @classmethod def setUpClass(cls): paddle.enable_static() @@ -136,15 +213,18 @@ class TestPrimops(unittest.TestCase): """convert numpy ndarray to paddle Variable recursively.""" return [ paddle.static.data(f'x{uuid.uuid4()}', v.shape, v.dtype) - if isinstance(v, np.ndarray) else self.arr2var(v) for v in arr + if isinstance(v, np.ndarray) + else self.arr2var(v) + for v in arr ] def _as_tuple(self, input): if isinstance(input, (tuple, list)) and len(input) == 0: return input if not isinstance(input, (tuple, list)) or all( - isinstance(i, int) for i in input): - return (input, ) + isinstance(i, int) for i in input + ): + return (input,) return input diff --git a/python/paddle/fluid/tests/unittests/autograd/test_transform.py b/python/paddle/fluid/tests/unittests/autograd/test_transform.py index f29b0d9b716..3498f217bc6 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_transform.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_transform.py @@ -22,13 +22,13 @@ paddle.enable_static() class TestAutoGradTransformForAdd(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_data() def init_data(self): @@ -36,13 +36,13 @@ class TestAutoGradTransformForAdd(unittest.TestCase): self.xs_shape_map = {0: (20, 40), 1: (20, 40)} # { output_index: output_shape } self.ys_shape_map = {0: (20, 40)} - X0 = paddle.static.data(name='X0', - shape=self.xs_shape_map[0], - dtype='float32') + X0 = paddle.static.data( + name='X0', shape=self.xs_shape_map[0], dtype='float32' + ) X0.stop_gradient = False - X1 = paddle.static.data(name='X1', - shape=self.xs_shape_map[1], - dtype='float32') + X1 = paddle.static.data( + name='X1', shape=self.xs_shape_map[1], dtype='float32' + ) X1.stop_gradient = False A = paddle.tanh(X0) @@ -94,28 +94,56 @@ class TestAutoGradTransformForAdd(unittest.TestCase): 'fill_constant_p', # transposed op 'mul_p', - 'mul_p' + 'mul_p', ] self.prim2orig_ops_with_blacklist = [ - 'tanh', 'tanh', 'add_p', 'fill_constant', 'fill_constant', - 'fill_constant', 'elementwise_mul', 'sub_p', 'fill_constant', - 'elementwise_mul', 'sub_p', 'fill_constant', 'elementwise_mul', - 'elementwise_mul', 'rsqrt', 'fill_constant', 'elementwise_div', - 'elementwise_div', 'elementwise_mul' + 'tanh', + 'tanh', + 'add_p', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'elementwise_mul', + 'sub_p', + 'fill_constant', + 'elementwise_mul', + 'sub_p', + 'fill_constant', + 'elementwise_mul', + 'elementwise_mul', + 'rsqrt', + 'fill_constant', + 'elementwise_div', + 'elementwise_div', + 'elementwise_mul', ] self.prim2orig_ops = [ - 'tanh', 'tanh', 'elementwise_add', 'fill_constant', 'fill_constant', - 'fill_constant', 'elementwise_mul', 'elementwise_sub', - 'fill_constant', 'elementwise_mul', 'elementwise_sub', - 'fill_constant', 'elementwise_mul', 'elementwise_mul', 'rsqrt', - 'fill_constant', 'elementwise_div', 'elementwise_div', - 'elementwise_mul' + 'tanh', + 'tanh', + 'elementwise_add', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'elementwise_mul', + 'elementwise_sub', + 'fill_constant', + 'elementwise_mul', + 'elementwise_sub', + 'fill_constant', + 'elementwise_mul', + 'elementwise_mul', + 'rsqrt', + 'fill_constant', + 'elementwise_div', + 'elementwise_div', + 'elementwise_mul', ] def test_run(self): # Must using with program_guard(), otherwise prim ops will append other block - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): ad = Transform(self.main_program.block(0)) orig_ops = [op.type for op in self.main_program.block(0).ops] self.assertEqual(sorted(orig_ops), sorted(self.orig_ops)) @@ -150,11 +178,13 @@ class TestAutoGradTransformForAdd(unittest.TestCase): self.assertEqual(flatten_ys_bar[k].shape, v) # Test prim2orig with blacklist - prim2orig(block=self.main_program.block(0), - blacklist=['add_p', 'sub_p']) + prim2orig( + block=self.main_program.block(0), blacklist=['add_p', 'sub_p'] + ) prim2orig_ops = [op.type for op in self.main_program.block(0).ops] - self.assertEqual(sorted(prim2orig_ops), - sorted(self.prim2orig_ops_with_blacklist)) + self.assertEqual( + sorted(prim2orig_ops), sorted(self.prim2orig_ops_with_blacklist) + ) # Test prim2orig prim2orig(block=self.main_program.block(0)) @@ -163,19 +193,18 @@ class TestAutoGradTransformForAdd(unittest.TestCase): class TestAutoGradTransformForMatmul(TestAutoGradTransformForAdd): - def init_data(self): # { input_index: input_shape } self.xs_shape_map = {0: (100, 2), 1: (5, 2)} # { output_index: output_shape } self.ys_shape_map = {0: (100, 5)} - X0 = paddle.static.data('X0', - shape=self.xs_shape_map[0], - dtype='float32') + X0 = paddle.static.data( + 'X0', shape=self.xs_shape_map[0], dtype='float32' + ) X0.stop_gradient = False - X1 = paddle.static.data('X1', - shape=self.xs_shape_map[1], - dtype='float32') + X1 = paddle.static.data( + 'X1', shape=self.xs_shape_map[1], dtype='float32' + ) X1.stop_gradient = False A = paddle.reshape(X1, [2, 5]) @@ -189,8 +218,13 @@ class TestAutoGradTransformForMatmul(TestAutoGradTransformForAdd): self.orig_ops = ['reshape2', 'scale', 'matmul_v2'] self.orig2prim_ops = [ - 'reshape_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'mul_p', 'add_p', 'matmul_p' + 'reshape_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'mul_p', + 'add_p', + 'matmul_p', ] self.linearize_ops = self.orig2prim_ops + [ # call fill_const() in linearize() function @@ -204,7 +238,7 @@ class TestAutoGradTransformForMatmul(TestAutoGradTransformForAdd): # 'add_p', 'matmul_p', 'matmul_p', - 'add_p' + 'add_p', ] self.transpose_ops = self.orig2prim_ops + [ # call fill_const() in transpose() function @@ -264,24 +298,23 @@ class TestAutoGradTransformForMatmul(TestAutoGradTransformForAdd): class TestAutoGradTransformForIndexSelect(TestAutoGradTransformForAdd): - def init_data(self): # { input_index: input_shape } - self.xs_shape_map = {0: (7, 8, 9), 1: (8, 1), 2: (7, 8, 9), 3: (3, )} + self.xs_shape_map = {0: (7, 8, 9), 1: (8, 1), 2: (7, 8, 9), 3: (3,)} # { output_index: output_shape } self.ys_shape_map = {0: (3, 16, 9)} - X0 = paddle.static.data('X0', - shape=self.xs_shape_map[0], - dtype='float32') + X0 = paddle.static.data( + 'X0', shape=self.xs_shape_map[0], dtype='float32' + ) X0.stop_gradient = False - X1 = paddle.static.data('X1', - shape=self.xs_shape_map[1], - dtype='float32') + X1 = paddle.static.data( + 'X1', shape=self.xs_shape_map[1], dtype='float32' + ) X1.stop_gradient = False - X2 = paddle.static.data('X2', - shape=self.xs_shape_map[2], - dtype='float32') + X2 = paddle.static.data( + 'X2', shape=self.xs_shape_map[2], dtype='float32' + ) X2.stop_gradient = False X3 = paddle.static.data('X3', shape=self.xs_shape_map[3], dtype='int32') X3.stop_gradient = False @@ -297,12 +330,23 @@ class TestAutoGradTransformForIndexSelect(TestAutoGradTransformForAdd): Y, ] self.orig_ops = [ - 'elementwise_add', 'p_norm', 'elementwise_sub', 'concat', - 'index_select' + 'elementwise_add', + 'p_norm', + 'elementwise_sub', + 'concat', + 'index_select', ] self.orig2prim_ops = [ - 'broadcast_p', 'add_p', 'reshape_p', 'mul_p', 'reduce_sum_p', - 'sqrt_p', 'broadcast_p', 'sub_p', 'concat_p', 'gather_p' + 'broadcast_p', + 'add_p', + 'reshape_p', + 'mul_p', + 'reduce_sum_p', + 'sqrt_p', + 'broadcast_p', + 'sub_p', + 'concat_p', + 'gather_p', ] self.linearize_ops = self.orig2prim_ops + [ # call fill_const() in linearize() function @@ -324,7 +368,7 @@ class TestAutoGradTransformForIndexSelect(TestAutoGradTransformForAdd): 'broadcast_p', 'sub_p', 'concat_p', - 'gather_p' + 'gather_p', ] self.transpose_ops = self.orig2prim_ops + [ # call fill_const() in transpose() function @@ -357,26 +401,81 @@ class TestAutoGradTransformForIndexSelect(TestAutoGradTransformForAdd): ] self.prim2orig_ops_with_blacklist = [ - 'expand_v2', 'add_p', 'reshape2', 'elementwise_mul', 'reduce_sum', - 'sqrt', 'expand_v2', 'sub_p', 'concat', 'gather', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'elementwise_mul', 'reduce_sum', 'reshape2', - 'reshape2', 'elementwise_mul', 'elementwise_mul', 'reshape2', - 'expand_v2', 'elementwise_div', 'reduce_sum', 'reshape2', - 'fill_constant', 'sub_p', 'split', 'fill_constant', 'fill_any_like', - 'add_p', 'scatter', 'elementwise_add', 'add_p' + 'expand_v2', + 'add_p', + 'reshape2', + 'elementwise_mul', + 'reduce_sum', + 'sqrt', + 'expand_v2', + 'sub_p', + 'concat', + 'gather', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'elementwise_mul', + 'reduce_sum', + 'reshape2', + 'reshape2', + 'elementwise_mul', + 'elementwise_mul', + 'reshape2', + 'expand_v2', + 'elementwise_div', + 'reduce_sum', + 'reshape2', + 'fill_constant', + 'sub_p', + 'split', + 'fill_constant', + 'fill_any_like', + 'add_p', + 'scatter', + 'elementwise_add', + 'add_p', ] self.prim2orig_ops = [ - 'expand_v2', 'elementwise_add', 'reshape2', 'elementwise_mul', - 'reduce_sum', 'sqrt', 'expand_v2', 'elementwise_sub', 'concat', - 'gather', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', - 'elementwise_mul', 'reduce_sum', 'reshape2', 'reshape2', - 'elementwise_mul', 'elementwise_mul', 'reshape2', 'expand_v2', - 'elementwise_div', 'reduce_sum', 'reshape2', 'fill_constant', - 'elementwise_sub', 'split', 'fill_constant', 'fill_any_like', - 'elementwise_add', 'scatter', 'elementwise_add', 'elementwise_add' + 'expand_v2', + 'elementwise_add', + 'reshape2', + 'elementwise_mul', + 'reduce_sum', + 'sqrt', + 'expand_v2', + 'elementwise_sub', + 'concat', + 'gather', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'elementwise_mul', + 'reduce_sum', + 'reshape2', + 'reshape2', + 'elementwise_mul', + 'elementwise_mul', + 'reshape2', + 'expand_v2', + 'elementwise_div', + 'reduce_sum', + 'reshape2', + 'fill_constant', + 'elementwise_sub', + 'split', + 'fill_constant', + 'fill_any_like', + 'elementwise_add', + 'scatter', + 'elementwise_add', + 'elementwise_add', ] diff --git a/python/paddle/fluid/tests/unittests/autograd/utils.py b/python/paddle/fluid/tests/unittests/autograd/utils.py index bbbce4eb22f..abcc71dd43f 100644 --- a/python/paddle/fluid/tests/unittests/autograd/utils.py +++ b/python/paddle/fluid/tests/unittests/autograd/utils.py @@ -32,20 +32,22 @@ def _product(t): def _get_item(t, idx): assert isinstance( - t, - paddle.fluid.framework.Variable), "The first argument t must be Tensor." - assert isinstance(idx, - int), "The second argument idx must be an int number." + t, paddle.fluid.framework.Variable + ), "The first argument t must be Tensor." + assert isinstance( + idx, int + ), "The second argument idx must be an int number." flat_t = paddle.reshape(t, [-1]) return flat_t.__getitem__(idx) def _set_item(t, idx, value): assert isinstance( - t, - paddle.fluid.framework.Variable), "The first argument t must be Tensor." - assert isinstance(idx, - int), "The second argument idx must be an int number." + t, paddle.fluid.framework.Variable + ), "The first argument t must be Tensor." + assert isinstance( + idx, int + ), "The second argument idx must be an int number." flat_t = paddle.reshape(t, [-1]) flat_t.__setitem__(idx, value) return paddle.reshape(flat_t, t.shape) @@ -60,8 +62,9 @@ def _compute_numerical_jacobian(func, xs, delta, np_dtype): for i in range(fout_size): jac_i = list([] for _ in range(fin_size)) for j in range(fin_size): - jac_i[j] = np.zeros((_product(ys[i].shape), _product(xs[j].shape)), - dtype=np_dtype) + jac_i[j] = np.zeros( + (_product(ys[i].shape), _product(xs[j].shape)), dtype=np_dtype + ) jacobian[i] = jac_i for j in range(fin_size): @@ -81,7 +84,7 @@ def _compute_numerical_jacobian(func, xs, delta, np_dtype): for p in range(_product(ys[i].shape)): y_pos = _get_item(ys_pos[i], p) y_neg = _get_item(ys_neg[i], p) - jacobian[i][j][p][q] = (y_pos - y_neg) / delta / 2. + jacobian[i][j][p][q] = (y_pos - y_neg) / delta / 2.0 return jacobian @@ -94,7 +97,8 @@ def _compute_numerical_hessian(func, xs, delta, np_dtype): hessian_i = list([] for _ in range(fin_size)) for j in range(fin_size): hessian_i[j] = np.zeros( - (_product(xs[i].shape), _product(xs[j].shape)), dtype=np_dtype) + (_product(xs[i].shape), _product(xs[j].shape)), dtype=np_dtype + ) hessian[i] = hessian_i for i in range(fin_size): @@ -105,15 +109,19 @@ def _compute_numerical_hessian(func, xs, delta, np_dtype): x_pos = orig + delta xs[j] = _set_item(xs[j], q, x_pos) jacobian_pos = _compute_numerical_jacobian( - func, xs, delta, np_dtype) + func, xs, delta, np_dtype + ) x_neg = orig - delta xs[j] = _set_item(xs[j], q, x_neg) jacobian_neg = _compute_numerical_jacobian( - func, xs, delta, np_dtype) + func, xs, delta, np_dtype + ) xs[j] = _set_item(xs[j], q, orig) hessian[i][j][p][q] = ( - jacobian_pos[0][i][0][p] - - jacobian_neg[0][i][0][p]) / delta / 2. + (jacobian_pos[0][i][0][p] - jacobian_neg[0][i][0][p]) + / delta + / 2.0 + ) return hessian @@ -125,11 +133,9 @@ def concat_to_matrix(xs, is_batched=False): return np.concatenate(rows, 1) if is_batched else np.concatenate(rows, 0) -def _compute_numerical_batch_jacobian(func, - xs, - delta, - np_dtype, - merge_batch=True): +def _compute_numerical_batch_jacobian( + func, xs, delta, np_dtype, merge_batch=True +): no_batch_jacobian = _compute_numerical_jacobian(func, xs, delta, np_dtype) xs = list(as_tensors(xs)) ys = list(as_tensors(func(*xs))) @@ -192,7 +198,8 @@ def _compute_numerical_batch_hessian(func, xs, delta, np_dtype): mid = len(hessian_res) // 2 for i in range(mid): hessian_result.append( - np.stack((hessian_res[i], hessian_res[mid + i]), axis=0)) + np.stack((hessian_res[i], hessian_res[mid + i]), axis=0) + ) return hessian_result @@ -205,8 +212,9 @@ def _compute_numerical_vjp(func, xs, v, delta, np_dtype): vjp = [np.zeros((_product(x.shape)), dtype=np_dtype) for x in xs] for j in range(len(xs)): for q in range(_product(xs[j].shape)): - vjp[j][q] = np.sum(jacobian[:, j, :, q].reshape(flat_v.shape) * - flat_v) + vjp[j][q] = np.sum( + jacobian[:, j, :, q].reshape(flat_v.shape) * flat_v + ) vjp = [vjp[j].reshape(xs[j].shape) for j in range(len(xs))] return vjp @@ -218,8 +226,9 @@ def _compute_numerical_vhp(func, xs, v, delta, np_dtype): vhp = [np.zeros((_product(x.shape)), dtype=np_dtype) for x in xs] for j in range(len(xs)): for q in range(_product(xs[j].shape)): - vhp[j][q] = np.sum(hessian[:, j, :, q].reshape(flat_v.shape) * - flat_v) + vhp[j][q] = np.sum( + hessian[:, j, :, q].reshape(flat_v.shape) * flat_v + ) vhp = [vhp[j].reshape(xs[j].shape) for j in range(len(xs))] return vhp @@ -256,7 +265,6 @@ def unuse(x, y): def nested(x): - def inner(y): return x * y @@ -286,8 +294,7 @@ def place(devices, key='place'): def decorate(cls): module = sys.modules[cls.__module__].__dict__ raw_classes = { - k: v - for k, v in module.items() if k.startswith(cls.__name__) + k: v for k, v in module.items() if k.startswith(cls.__name__) } for raw_name, raw_cls in raw_classes.items(): @@ -295,7 +302,7 @@ def place(devices, key='place'): test_cls = dict(raw_cls.__dict__) test_cls.update({key: d}) new_name = raw_name + '.' + d.__class__.__name__ - module[new_name] = type(new_name, (raw_cls, ), test_cls) + module[new_name] = type(new_name, (raw_cls,), test_cls) del module[raw_name] return cls @@ -324,10 +331,13 @@ def parameterize(fields, values=None): } test_cls.update(values) name = cls.__name__ + str(i) - name = name + '.' + \ - values.get('suffix') if values.get('suffix') else name + name = ( + name + '.' + values.get('suffix') + if values.get('suffix') + else name + ) - test_cls_module[name] = type(name, (cls, ), test_cls) + test_cls_module[name] = type(name, (cls,), test_cls) for m in list(cls.__dict__): if m.startswith("test"): @@ -375,8 +385,12 @@ def _np_concat_matrix_sequence(src, src_format=MatrixFormat.NM): else: return np.concatenate(xs, axis=1) - supported_format = (MatrixFormat.NBM, MatrixFormat.BNM, MatrixFormat.NMB, - MatrixFormat.NM) + supported_format = ( + MatrixFormat.NBM, + MatrixFormat.BNM, + MatrixFormat.NMB, + MatrixFormat.NM, + ) if src_format not in supported_format: raise ValueError( f"Supported Jacobian format is {supported_format}, but got {src_format}" diff --git a/python/paddle/fluid/tests/unittests/benchmark.py b/python/paddle/fluid/tests/unittests/benchmark.py index 3a27fca61ef..d6fa01e398c 100644 --- a/python/paddle/fluid/tests/unittests/benchmark.py +++ b/python/paddle/fluid/tests/unittests/benchmark.py @@ -20,7 +20,6 @@ from op_test import OpTest class BenchmarkSuite(OpTest): - def timeit_function(self, callback, iters, *args, **kwargs): assert iters != 0, "Iters should >= 1" start = time.time() @@ -30,20 +29,23 @@ class BenchmarkSuite(OpTest): return elapse / iters def _assert_cpu_gpu_same(self, cpu_outs, gpu_outs, fetch_list, atol): - for item_cpu_out, item_gpu_out, variable in zip(cpu_outs, gpu_outs, - fetch_list): + for item_cpu_out, item_gpu_out, variable in zip( + cpu_outs, gpu_outs, fetch_list + ): # the cpu version is baseline, expect gpu version keep same with cpu version. expect = item_cpu_out expect_t = np.array(item_cpu_out) actual = item_gpu_out actual_t = np.array(item_gpu_out) var_name = variable if isinstance(variable, str) else variable.name - np.testing.assert_allclose(actual_t, - expect_t, - rtol=1e-05, - atol=atol) - self.assertListEqual(actual.lod(), expect.lod(), - "Output (" + var_name + ") has different lod") + np.testing.assert_allclose( + actual_t, expect_t, rtol=1e-05, atol=atol + ) + self.assertListEqual( + actual.lod(), + expect.lod(), + "Output (" + var_name + ") has different lod", + ) def _get_input_names(self): inputs = [] @@ -83,18 +85,23 @@ class BenchmarkSuite(OpTest): for place in places: elapses.append(self.timeit_output_with_place(place, iters)) for place, elapse in zip(places, elapses): - print("One pass of ({2}_op) at {0} cost {1}".format( - str(place), elapse, self.op_type)) + print( + "One pass of ({2}_op) at {0} cost {1}".format( + str(place), elapse, self.op_type + ) + ) def timeit_grad_with_place(self, place, iters=100): inputs_to_check = self._get_input_names() output_names = self._get_output_names() - return self.timeit_function(self._get_gradient, - iters, - inputs_to_check, - place, - output_names, - no_grad_set=None) + return self.timeit_function( + self._get_gradient, + iters, + inputs_to_check, + place, + output_names, + no_grad_set=None, + ) def timeit_grad(self, iters=100): places = self._get_places() @@ -102,5 +109,8 @@ class BenchmarkSuite(OpTest): for place in places: elapses.append(self.timeit_grad_with_place(place, iters)) for place, elapse in zip(places, elapses): - print("One pass of ({2}_grad_op) at {0} cost {1}".format( - str(place), elapse, self.op_type)) + print( + "One pass of ({2}_grad_op) at {0} cost {1}".format( + str(place), elapse, self.op_type + ) + ) diff --git a/python/paddle/fluid/tests/unittests/benchmark_sum_op.py b/python/paddle/fluid/tests/unittests/benchmark_sum_op.py index d3d909e17e4..e8cb02f16cb 100644 --- a/python/paddle/fluid/tests/unittests/benchmark_sum_op.py +++ b/python/paddle/fluid/tests/unittests/benchmark_sum_op.py @@ -21,7 +21,6 @@ from benchmark import BenchmarkSuite class TestSumOp(BenchmarkSuite): - def setUp(self): self.op_type = "sum" self.customize_testcase() diff --git a/python/paddle/fluid/tests/unittests/c_embedding_op_base.py b/python/paddle/fluid/tests/unittests/c_embedding_op_base.py index ef3c942d2b0..e9b1ca4d8aa 100644 --- a/python/paddle/fluid/tests/unittests/c_embedding_op_base.py +++ b/python/paddle/fluid/tests/unittests/c_embedding_op_base.py @@ -32,7 +32,6 @@ def get_c_embedding(start, end, table, ids): class TestCEmbeddingCPU(OpTest): - def setUp(self): self.init_dtype() self.initcase() @@ -46,8 +45,9 @@ class TestCEmbeddingCPU(OpTest): def initcase(self): self.op_type = "c_embedding" table = np.random.random((17, 64)).astype(self.dtype) - ids = np.random.randint(low=0, high=17 * 2, - size=(2, 4)).astype(self.ids_dtype) + ids = np.random.randint(low=0, high=17 * 2, size=(2, 4)).astype( + self.ids_dtype + ) self.start_index = 10 self.end_index = self.start_index + 17 @@ -72,7 +72,6 @@ class TestCEmbeddingCPU(OpTest): class TestCEmbeddingOpBase(TestCEmbeddingCPU): - def setUp(self): self.init_dtype() self.initcase() @@ -106,7 +105,6 @@ class TestCEmbeddingOpBase(TestCEmbeddingCPU): class TestCEmbeddingOpFP32(TestCEmbeddingOpBase): - def setUp(self): self.init_dtype() self.initcase() @@ -114,8 +112,9 @@ class TestCEmbeddingOpFP32(TestCEmbeddingOpBase): def initcase(self): self.op_type = "c_embedding" table = np.random.random((17, 64)).astype(self.dtype) - ids = np.random.randint(low=0, high=17 * 2, - size=(2, 4)).astype(self.ids_dtype) + ids = np.random.randint(low=0, high=17 * 2, size=(2, 4)).astype( + self.ids_dtype + ) self.start_index = 10 ids[0][1] = 12 ids[0][2] = 12 diff --git a/python/paddle/fluid/tests/unittests/check_nan_inf_base.py b/python/paddle/fluid/tests/unittests/check_nan_inf_base.py index 1c4bfd387c4..e1319378728 100644 --- a/python/paddle/fluid/tests/unittests/check_nan_inf_base.py +++ b/python/paddle/fluid/tests/unittests/check_nan_inf_base.py @@ -30,8 +30,9 @@ np.random.seed(0) def generator(): batch_size = 5 for i in range(5): - curr_train_x = np.random.randint(batch_size, - size=(batch_size, 3)).astype("float32") + curr_train_x = np.random.randint( + batch_size, size=(batch_size, 3) + ).astype("float32") if i >= 2: curr_train_x[0, :] = np.nan curr_train_x[-1, :] = np.inf @@ -62,7 +63,8 @@ def net(): hidden = fluid.layers.fc(input=hidden, size=3, act=None) cost, y_predict = fluid.layers.softmax_with_cross_entropy( - hidden, y, return_softmax=True) + hidden, y, return_softmax=True + ) acc_top1 = fluid.layers.accuracy(input=y_predict, label=y, k=1) avg_cost = paddle.mean(cost) @@ -88,14 +90,15 @@ def check(use_cuda): for train_data, y_label in generator(): outs = exe.run( main, - feed={ - 'x': train_data, - 'y': y_label - }, - fetch_list=[y_predict.name, avg_cost.name, acc_top1.name]) + feed={'x': train_data, 'y': y_label}, + fetch_list=[y_predict.name, avg_cost.name, acc_top1.name], + ) step += 1 - print('iter={:.0f},cost={},acc1={}'.format( - step, outs[1][0], outs[2][0])) + print( + 'iter={:.0f},cost={},acc1={}'.format( + step, outs[1][0], outs[2][0] + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py b/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py index 9622ee4e9a9..233cfee3087 100644 --- a/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py +++ b/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py @@ -28,8 +28,9 @@ np.random.seed(0) def generator(): batch_size = 5 for i in range(5): - curr_train_x = np.random.randint(batch_size, - size=(batch_size, 3)).astype("float32") + curr_train_x = np.random.randint( + batch_size, size=(batch_size, 3) + ).astype("float32") if i >= 2: curr_train_x[0, :] = np.nan curr_train_x[-1, :] = np.inf @@ -42,7 +43,6 @@ def generator(): class TestLayer(nn.Layer): - def __init__(self): super(TestLayer, self).__init__() self.linear1 = nn.Linear(3, 400) @@ -82,8 +82,11 @@ def check(use_cuda): acc_top1 = paddle.metric.accuracy(input=y_pred, label=y, k=1) - print('iter={:.0f}, cost={}, acc1={}'.format(step, avg_cost.numpy(), - acc_top1.numpy())) + print( + 'iter={:.0f}, cost={}, acc1={}'.format( + step, avg_cost.numpy(), acc_top1.numpy() + ) + ) sgd.step() sgd.clear_grad() diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py index a69e6c59e0e..94be00887a2 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestCollectiveAllgatherAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -47,29 +46,30 @@ class TestCollectiveAllgatherAPI(test_base.TestCollectiveAPIRunnerBase): if args['backend'] == 'nccl': device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() elif args['backend'] == 'bkcl': device_id = int(os.getenv("FLAGS_selected_xpus", "0")) place = fluid.XPUPlace(device_id) else: place = fluid.CPUPlace() - indata = test_base.create_test_data(shape=(10, 1000), - dtype=args["dtype"], - seed=os.getpid()) - assert args[ - 'static_mode'] == 1, "collective_allgather_api only support static mode" - result = self.get_model(train_prog, - startup_prog, - rank, - dtype=args["dtype"]) + indata = test_base.create_test_data( + shape=(10, 1000), dtype=args["dtype"], seed=os.getpid() + ) + assert ( + args['static_mode'] == 1 + ), "collective_allgather_api only support static mode" + result = self.get_model( + train_prog, startup_prog, rank, dtype=args["dtype"] + ) exe = fluid.Executor(place) exe.run(startup_prog) fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=fetch_list) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=fetch_list + ) sys.stdout.buffer.write(pickle.dumps(out)) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api_dygraph.py index 4d5f82e2882..4477b663a58 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveAllgatherAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allgather_object_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_allgather_object_api_dygraph.py index 76610a84041..03771e30d7e 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allgather_object_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allgather_object_api_dygraph.py @@ -18,7 +18,6 @@ import test_collective_api_base as test_base class TestCollectiveAllgatherObjectAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py index 9c86afc5f4b..a28f8b175da 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py @@ -21,15 +21,14 @@ paddle.enable_static() class TestCollectiveAllreduceAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.all_reduce(tindata) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api_dygraph.py index 9bdbaa18177..30cb1e5a377 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveAllreduceAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_new_group_api.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_new_group_api.py index 5eb983c8905..b3258e858a2 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_new_group_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_new_group_api.py @@ -21,15 +21,14 @@ paddle.enable_static() class TestCollectiveAllreduceNewGroupAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) gp = paddle.distributed.new_group([0, 1]) paddle.distributed.all_reduce(tindata, group=gp, sync_op=True) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op.py index a88b73ce04a..69072d40aa8 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op.py @@ -22,30 +22,34 @@ paddle.enable_static() class TestCollectiveAllreduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = 0 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofallreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allreduce_sum", - inputs={'X': tindata}, - attrs={'ring_id': ring_id}, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allreduce_sum", + inputs={'X': tindata}, + attrs={'ring_id': ring_id}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op_wait.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op_wait.py index 10b867df69a..c700d49415b 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op_wait.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op_wait.py @@ -22,22 +22,22 @@ paddle.enable_static() class TestCollectiveAllreduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = 0 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofallreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) # tout = tin + tin - tin = tin if True: @@ -58,21 +58,27 @@ class TestCollectiveAllreduce(TestCollectiveRunnerBase): outputs={'Out': toutdata}, ) - main_prog.global_block().append_op(type='c_wait_compute', - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + main_prog.global_block().append_op( + type='c_wait_compute', + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) - main_prog.global_block().append_op(type="c_allreduce_sum", - inputs={'X': toutdata}, - attrs={'ring_id': ring_id}, - outputs={'Out': toutdata}, - attr={'use_calc_stream': False}) + main_prog.global_block().append_op( + type="c_allreduce_sum", + inputs={'X': toutdata}, + attrs={'ring_id': ring_id}, + outputs={'Out': toutdata}, + attr={'use_calc_stream': False}, + ) - main_prog.global_block().append_op(type="c_wait_comm", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + main_prog.global_block().append_op( + type="c_wait_comm", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) # tout = tin + tout - tin = tout if True: diff --git a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api.py b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api.py index 57fbc69429a..e3f23f0cd01 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api.py @@ -21,15 +21,14 @@ paddle.enable_static() class TestCollectiveAllToAllAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) tindata = paddle.split(tindata, 2, axis=0) tout_data = [] paddle.distributed.alltoall(tindata, tout_data) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api_dygraph.py index eb19cadb114..560fa8162c5 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveAllToAllAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single.py b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single.py index e0e0fc74d37..0c9a1253393 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single.py @@ -20,13 +20,14 @@ import paddle.distributed as dist class TestCollectiveAllToAllSingle(unittest.TestCase): - def setUp(self): - assert not paddle.distributed.is_initialized(), \ - "The distributed environment has not been initialized." + assert ( + not paddle.distributed.is_initialized() + ), "The distributed environment has not been initialized." dist.init_parallel_env() - assert paddle.distributed.is_initialized(), \ - "The distributed environment has been initialized." + assert ( + paddle.distributed.is_initialized() + ), "The distributed environment has been initialized." paddle.fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -38,7 +39,8 @@ class TestCollectiveAllToAllSingle(unittest.TestCase): input = paddle.ones([size, size], dtype='int64') * rank output = paddle.empty([size, size], dtype='int64') expected_output = paddle.concat( - [paddle.ones([1, size], dtype='int64') * i for i in range(size)]) + [paddle.ones([1, size], dtype='int64') * i for i in range(size)] + ) group = dist.new_group([0, 1]) dist.alltoall_single(input, output, group=group) @@ -52,18 +54,22 @@ class TestCollectiveAllToAllSingle(unittest.TestCase): input = paddle.ones([sum(in_split_sizes), size], dtype='float32') * rank output = paddle.empty([(rank + 1) * size, size], dtype='float32') - expected_output = paddle.concat([ - paddle.ones([rank + 1, size], dtype='float32') * i - for i in range(size) - ]) + expected_output = paddle.concat( + [ + paddle.ones([rank + 1, size], dtype='float32') * i + for i in range(size) + ] + ) group = dist.new_group([0, 1]) - task = dist.alltoall_single(input, - output, - in_split_sizes, - out_split_sizes, - sync_op=False, - group=group) + task = dist.alltoall_single( + input, + output, + in_split_sizes, + out_split_sizes, + sync_op=False, + group=group, + ) task.wait() np.testing.assert_allclose(output.numpy(), expected_output.numpy()) @@ -71,8 +77,9 @@ class TestCollectiveAllToAllSingle(unittest.TestCase): def tearDown(self): dist.destroy_process_group() - assert not paddle.distributed.is_initialized(), \ - "The distributed environment has been deinitialized." + assert ( + not paddle.distributed.is_initialized() + ), "The distributed environment has been deinitialized." if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single_api_dygraph.py index f66b3a74bfd..0b5bf5ddde5 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveAllToAllSingleAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_barrier_api.py b/python/paddle/fluid/tests/unittests/collective/collective_barrier_api.py index 43fcca0155b..95054f678a3 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_barrier_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_barrier_api.py @@ -20,7 +20,6 @@ paddle.enable_static() class TestCollectiveBarrierAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_batch_isend_irecv.py b/python/paddle/fluid/tests/unittests/collective/collective_batch_isend_irecv.py index 2226d302291..6757e61fd07 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_batch_isend_irecv.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_batch_isend_irecv.py @@ -20,7 +20,6 @@ import paddle.distributed as dist class TestCollectiveBatchIsendIrecv(unittest.TestCase): - def setUp(self): dist.init_parallel_env() paddle.fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -33,8 +32,9 @@ class TestCollectiveBatchIsendIrecv(unittest.TestCase): # paddle.tensor([1, 2]) # Rank-1 recv_t = paddle.empty(shape=[2], dtype=send_t.dtype) send_op = dist.P2POp(dist.isend, send_t, (rank + 1) % world_size) - recv_op = dist.P2POp(dist.irecv, recv_t, - (rank - 1 + world_size) % world_size) + recv_op = dist.P2POp( + dist.irecv, recv_t, (rank - 1 + world_size) % world_size + ) tasks = dist.batch_isend_irecv([send_op, recv_op]) for task in tasks: diff --git a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py index 0a14a536609..d7f07e440d8 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py @@ -21,15 +21,14 @@ paddle.enable_static() class TestCollectiveBroadcastAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.broadcast(tindata, src=1) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api_dygraph.py index 9004d27d561..82bc998a249 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveBroadcastAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_op.py b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_op.py index ff99792c942..902e49b2648 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveBroadcast(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ class TestCollectiveBroadcast(TestCollectiveRunnerBase): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofbroadcast", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_broadcast", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_broadcast", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root': rootid}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_concat_op.py b/python/paddle/fluid/tests/unittests/collective/collective_concat_op.py index 34d49aeb284..533357fc44d 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_concat_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_concat_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveConcat(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,23 +29,22 @@ class TestCollectiveConcat(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofconcat", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_concat", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'rank': self.rank, - 'nranks': nranks - }, - outputs={'Out': toutdata}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_concat", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'rank': self.rank, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_global_gather.py b/python/paddle/fluid/tests/unittests/collective/collective_global_gather.py index 4403e06681b..6aa268b50a3 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_global_gather.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_global_gather.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -38,19 +37,19 @@ class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): n_expert = 2 world_size = 2 tot_expert = n_expert * world_size - local_input_buf = paddle.static.data(name="local_input_buf", - shape=[-1, in_feat], - dtype="float32") - local_expert_count = paddle.static.data(name="local_expert_count", - shape=[tot_expert], - dtype="int64") - global_expert_count = paddle.static.data(name="global_expert_count", - shape=[tot_expert], - dtype="int64") + local_input_buf = paddle.static.data( + name="local_input_buf", shape=[-1, in_feat], dtype="float32" + ) + local_expert_count = paddle.static.data( + name="local_expert_count", shape=[tot_expert], dtype="int64" + ) + global_expert_count = paddle.static.data( + name="global_expert_count", shape=[tot_expert], dtype="int64" + ) - output = moe_utils.global_gather(local_input_buf, - local_expert_count, - global_expert_count) + output = moe_utils.global_gather( + local_input_buf, local_expert_count, global_expert_count + ) return [output] @@ -65,7 +64,8 @@ class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): if args['backend'] == 'nccl': device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() elif args['backend'] == 'bkcl': device_id = int(os.getenv("FLAGS_selected_xpus", "0")) place = fluid.XPUPlace(device_id) @@ -81,19 +81,22 @@ class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): # Call paddle.distributed.alltoall() under legacy dygraph _enable_legacy_dygraph() np.random.seed(os.getpid()) - local_expert_count = np.random.randint(1, 4, - size=tot_expert).astype("int64") + local_expert_count = np.random.randint(1, 4, size=tot_expert).astype( + "int64" + ) local_expert_count = paddle.to_tensor(local_expert_count) global_expert_count = [] - paddle.distributed.alltoall(paddle.split(local_expert_count, 2, axis=0), - global_expert_count) + paddle.distributed.alltoall( + paddle.split(local_expert_count, 2, axis=0), global_expert_count + ) global_expert_count = paddle.concat(global_expert_count, axis=0) global_expert_count = global_expert_count.numpy() local_expert_count = local_expert_count.numpy() fwd_expert_count = sum(global_expert_count) np.random.seed(os.getpid()) - local_input_buf = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) paddle.enable_static() if args['static_mode']: @@ -103,13 +106,15 @@ class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={ - 'local_expert_count': local_expert_count, - 'global_expert_count': global_expert_count, - 'local_input_buf': local_input_buf - }, - fetch_list=fetch_list) + out = exe.run( + train_prog, + feed={ + 'local_expert_count': local_expert_count, + 'global_expert_count': global_expert_count, + 'local_input_buf': local_input_buf, + }, + fetch_list=fetch_list, + ) sys.stdout.buffer.write(pickle.dumps(out)) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_global_gather_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_global_gather_dygraph.py index acba43268f2..fb8d831f045 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_global_gather_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_global_gather_dygraph.py @@ -21,7 +21,6 @@ import paddle.distributed.utils.moe_utils as moe_utils class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -34,22 +33,24 @@ class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): world_size = 2 tot_expert = n_expert * world_size local_expert_count = np.random.randint( - 1, 4, size=tot_expert).astype("int") + 1, 4, size=tot_expert + ).astype("int") local_expert_count = paddle.to_tensor(local_expert_count) global_expert_count = [] paddle.distributed.alltoall( - paddle.split(local_expert_count, 2, axis=0), - global_expert_count) + paddle.split(local_expert_count, 2, axis=0), global_expert_count + ) global_expert_count = paddle.concat(global_expert_count, axis=0) fwd_expert_count = sum(global_expert_count) np.random.seed(seed) - local_input_buf = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) local_input_buf = paddle.to_tensor(local_input_buf) local_input_buf.stop_gradient = False - output = moe_utils.global_gather(local_input_buf, - local_expert_count, - global_expert_count) + output = moe_utils.global_gather( + local_input_buf, local_expert_count, global_expert_count + ) output.stop_gradient = False c = output * output c.stop_gradient = False diff --git a/python/paddle/fluid/tests/unittests/collective/collective_global_scatter.py b/python/paddle/fluid/tests/unittests/collective/collective_global_scatter.py index a7103d3c641..7fb0dc6ff71 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_global_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_global_scatter.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -37,20 +36,20 @@ class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): n_expert = 2 world_size = 2 tot_expert = n_expert * world_size - local_input_buf = paddle.static.data(name="local_input_buf", - shape=[-1, in_feat], - dtype="float32") - local_expert_count = paddle.static.data(name="local_expert_count", - shape=[tot_expert], - dtype="int64") + local_input_buf = paddle.static.data( + name="local_input_buf", shape=[-1, in_feat], dtype="float32" + ) + local_expert_count = paddle.static.data( + name="local_expert_count", shape=[tot_expert], dtype="int64" + ) global_expert_count = [] paddle.distributed.alltoall( - paddle.split(local_expert_count, 2, axis=0), - global_expert_count) + paddle.split(local_expert_count, 2, axis=0), global_expert_count + ) global_expert_count = paddle.concat(global_expert_count, axis=0) - output = moe_utils.global_scatter(local_input_buf, - local_expert_count, - global_expert_count) + output = moe_utils.global_scatter( + local_input_buf, local_expert_count, global_expert_count + ) return [output] def run_trainer(self, args): @@ -64,7 +63,8 @@ class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): if args['backend'] == 'nccl': device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() elif args['backend'] == 'bkcl': device_id = int(os.getenv("FLAGS_selected_xpus", "0")) place = fluid.XPUPlace(device_id) @@ -75,11 +75,13 @@ class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): n_expert = 2 world_size = 2 tot_expert = n_expert * world_size - local_expert_count = np.random.randint(1, 4, - size=tot_expert).astype("int64") + local_expert_count = np.random.randint(1, 4, size=tot_expert).astype( + "int64" + ) fwd_expert_count = sum(local_expert_count) - local_input_buf = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) if args['static_mode']: result = self.get_model(train_prog, startup_prog, rank) exe = fluid.Executor(place) @@ -87,12 +89,14 @@ class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={ - 'local_expert_count': local_expert_count, - 'local_input_buf': local_input_buf - }, - fetch_list=fetch_list) + out = exe.run( + train_prog, + feed={ + 'local_expert_count': local_expert_count, + 'local_input_buf': local_input_buf, + }, + fetch_list=fetch_list, + ) sys.stdout.buffer.write(pickle.dumps(out)) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_global_scatter_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_global_scatter_dygraph.py index b90905d5d39..324f079ee8c 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_global_scatter_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_global_scatter_dygraph.py @@ -21,7 +21,6 @@ import paddle.distributed.utils.moe_utils as moe_utils class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -34,21 +33,23 @@ class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): world_size = 2 tot_expert = n_expert * world_size local_expert_count = np.random.randint( - 1, 4, size=tot_expert).astype("int") + 1, 4, size=tot_expert + ).astype("int") fwd_expert_count = sum(local_expert_count) - local_input_buf = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) local_expert_count = paddle.to_tensor(local_expert_count) local_input_buf = paddle.to_tensor(local_input_buf) global_expert_count = [] paddle.distributed.alltoall( - paddle.split(local_expert_count, 2, axis=0), - global_expert_count) + paddle.split(local_expert_count, 2, axis=0), global_expert_count + ) global_expert_count = paddle.concat(global_expert_count, axis=0) local_input_buf.stop_gradient = False - output = moe_utils.global_scatter(local_input_buf, - local_expert_count, - global_expert_count) + output = moe_utils.global_scatter( + local_input_buf, local_expert_count, global_expert_count + ) output.stop_gradient = False c = output * output c.backward() diff --git a/python/paddle/fluid/tests/unittests/collective/collective_identity_op.py b/python/paddle/fluid/tests/unittests/collective/collective_identity_op.py index 0486ea61642..8fc2449f4f2 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_identity_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_identity_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveIdentity(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,22 +29,22 @@ class TestCollectiveIdentity(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_identity", - inputs={'X': tindata}, - outputs={'Out': toutdata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_identity", + inputs={'X': tindata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_isend_irecv_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_isend_irecv_api_dygraph.py index 37a38b218c5..d76e48f5ba5 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_isend_irecv_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_isend_irecv_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveIsendIrecvAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py index 83f4e345862..81e6d7daa27 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py @@ -21,15 +21,14 @@ paddle.enable_static() class TestCollectiveReduceAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.reduce(tindata, dst=0) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api_dygraph.py index 5e9dfc8265e..5486e317d89 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveReduceAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_op.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_op.py index d30b845098c..a0b34247a8d 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveReduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ class TestCollectiveReduce(TestCollectiveRunnerBase): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_reduce_sum", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root_id': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_reduce_sum", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root_id': rootid}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_op_calc_stream.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_op_calc_stream.py index ab3fe725df8..c7fe247d455 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_op_calc_stream.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_op_calc_stream.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveReduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,27 +29,32 @@ class TestCollectiveReduce(TestCollectiveRunnerBase): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_reduce_sum", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'root_id': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_reduce_sum", + inputs={'X': tindata}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'root_id': rootid, + }, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter.py index d7c67e60a46..a8a7c67d96e 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter.py @@ -20,7 +20,6 @@ import paddle.distributed as dist class TestCollectiveReduceScatter(unittest.TestCase): - def setUp(self): dist.init_parallel_env() paddle.fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -76,9 +75,9 @@ class TestCollectiveReduceScatter(unittest.TestCase): # [1, 2, 3, 4] # Rank-1 output = paddle.empty(shape=[2], dtype=input.dtype) - task = paddle.distributed.collective._reduce_scatter_base(output, - input, - sync_op=False) + task = paddle.distributed.collective._reduce_scatter_base( + output, input, sync_op=False + ) task.wait() diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter_api_dygraph.py index c9df2459a78..617f9889873 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveReduceScatterAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/collective_scatter_api.py index f198ae467d5..ec2ca037bbf 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_scatter_api.py @@ -21,19 +21,20 @@ paddle.enable_static() class TestCollectiveScatterAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32', - append_batch_size=False) - toutdata = layers.fill_constant(shape=[5, 1000], - dtype='float32', - value=1.0) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float32', + append_batch_size=False, + ) + toutdata = layers.fill_constant( + shape=[5, 1000], dtype='float32', value=1.0 + ) tensor_list = None if rank == 1: tensor_list = paddle.split(tindata, 2, axis=0) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_scatter_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_scatter_api_dygraph.py index 8f27f84a32d..9061a1b5a4f 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_scatter_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_scatter_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveScatterAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -32,9 +31,9 @@ class TestCollectiveScatterAPI(test_base.TestCollectiveAPIRunnerBase): if rank == 0: dist.scatter(subdata1, src=1) else: - dist.scatter(subdata1, - tensor_list=[subdata1, subdata2], - src=1) + dist.scatter( + subdata1, tensor_list=[subdata1, subdata2], src=1 + ) return [subdata1.cast("float32").numpy()] else: tindata = paddle.to_tensor(indata) @@ -42,9 +41,9 @@ class TestCollectiveScatterAPI(test_base.TestCollectiveAPIRunnerBase): if rank == 0: dist.scatter(subdata1, src=1) else: - dist.scatter(subdata1, - tensor_list=[subdata1, subdata2], - src=1) + dist.scatter( + subdata1, tensor_list=[subdata1, subdata2], src=1 + ) return [subdata1.numpy()] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_scatter_op.py b/python/paddle/fluid/tests/unittests/collective/collective_scatter_op.py index 4a8a1acfe5f..29cfe40117b 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_scatter_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_scatter_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveScatter(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,27 +29,28 @@ class TestCollectiveScatter(TestCollectiveRunnerBase): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_scatter", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root': rootid, - 'nranks': 2 - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_scatter", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root': rootid, 'nranks': 2}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py index d22080e8cdf..b8c233051ee 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py @@ -21,16 +21,17 @@ paddle.enable_static() class TestCollectiveSendRecvAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32', - append_batch_size=False) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float32', + append_batch_size=False, + ) if rank == 0: paddle.distributed.send(tindata, dst=1) else: diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api_dygraph.py index b4bf24ffbfa..701c8c81582 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api_dygraph.py @@ -19,7 +19,6 @@ import test_collective_api_base as test_base class TestCollectiveSendRecvAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op.py index 6324e619f9e..708326a3f49 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op.py @@ -21,36 +21,40 @@ paddle.enable_static() class TestCollectiveSendRecv(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = self.global_ring_id with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float64', - append_batch_size=False) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float64', + append_batch_size=False, + ) if self.rank == 0: - main_prog.global_block().append_op(type="send_v2", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'peer': 1, - 'use_calc_stream': True - }) + main_prog.global_block().append_op( + type="send_v2", + inputs={'X': tindata}, + attrs={ + 'ring_id': ring_id, + 'peer': 1, + 'use_calc_stream': True, + }, + ) else: - main_prog.global_block().append_op(type="recv_v2", - outputs={'Out': tindata}, - attrs={ - 'peer': 0, - 'ring_id': ring_id, - 'dtype': tindata.dtype, - 'out_shape': - tindata.shape, - 'use_calc_stream': True, - }) + main_prog.global_block().append_op( + type="recv_v2", + outputs={'Out': tindata}, + attrs={ + 'peer': 0, + 'ring_id': ring_id, + 'dtype': tindata.dtype, + 'out_shape': tindata.shape, + 'use_calc_stream': True, + }, + ) return tindata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_array.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_array.py index ec12db6f5be..eea4ef4615f 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_array.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_array.py @@ -22,39 +22,46 @@ paddle.enable_static() class TestCollectiveSendRecv(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = self.global_ring_id with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float64', - append_batch_size=False) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float64', + append_batch_size=False, + ) if self.rank == 0: data1 = fluid.layers.assign( - np.array([[0, 1, 2]], dtype='float32')) + np.array([[0, 1, 2]], dtype='float32') + ) data2 = fluid.layers.assign( - np.array([[3, 4, 5]], dtype='float32')) + np.array([[3, 4, 5]], dtype='float32') + ) elif self.rank == 1: data1 = fluid.layers.assign( - np.array([[3, 4, 5]], dtype='float32')) + np.array([[3, 4, 5]], dtype='float32') + ) data2 = fluid.layers.assign( - np.array([[0, 1, 2]], dtype='float32')) + np.array([[0, 1, 2]], dtype='float32') + ) tensor_array = fluid.layers.create_array(dtype='float32') i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0) fluid.layers.array_write(data1, i, tensor_array) fluid.layers.array_write(data2, i + 1, tensor_array) if self.rank == 0: - main_prog.global_block().append_op(type="send_v2", - inputs={'X': tensor_array}, - attrs={ - 'ring_id': ring_id, - 'peer': 1, - 'use_calc_stream': True - }) + main_prog.global_block().append_op( + type="send_v2", + inputs={'X': tensor_array}, + attrs={ + 'ring_id': ring_id, + 'peer': 1, + 'use_calc_stream': True, + }, + ) else: main_prog.global_block().append_op( type="recv_v2", @@ -65,7 +72,8 @@ class TestCollectiveSendRecv(TestCollectiveRunnerBase): 'dtype': data1.dtype, 'out_shape': [1, 3], 'use_calc_stream': True, - }) + }, + ) return tensor_array diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_dynamic_shape.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_dynamic_shape.py index da6ef74c114..28804d1570f 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_dynamic_shape.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_dynamic_shape.py @@ -21,41 +21,46 @@ paddle.enable_static() class TestCollectiveSendRecvDynamicShape(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = self.global_ring_id with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float64', - append_batch_size=False) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float64', + append_batch_size=False, + ) if self.rank == 0: - main_prog.global_block().append_op(type="send_v2", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'peer': 1, - 'use_calc_stream': True, - 'dynamic_shape': True - }) + main_prog.global_block().append_op( + type="send_v2", + inputs={'X': tindata}, + attrs={ + 'ring_id': ring_id, + 'peer': 1, + 'use_calc_stream': True, + 'dynamic_shape': True, + }, + ) else: - main_prog.global_block().append_op(type="recv_v2", - outputs={'Out': tindata}, - attrs={ - 'peer': 0, - 'ring_id': ring_id, - 'dtype': tindata.dtype, - 'out_shape': - tindata.shape, - 'use_calc_stream': True, - 'dynamic_shape': True - }) + main_prog.global_block().append_op( + type="recv_v2", + outputs={'Out': tindata}, + attrs={ + 'peer': 0, + 'ring_id': ring_id, + 'dtype': tindata.dtype, + 'out_shape': tindata.shape, + 'use_calc_stream': True, + 'dynamic_shape': True, + }, + ) return tindata if __name__ == "__main__": - runtime_main(TestCollectiveSendRecvDynamicShape, "sendrecv_dynamic_shape", - 0) + runtime_main( + TestCollectiveSendRecvDynamicShape, "sendrecv_dynamic_shape", 0 + ) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_split_op.py b/python/paddle/fluid/tests/unittests/collective/collective_split_op.py index 2e71ccf3049..57a8aa4fcc5 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_split_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_split_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveAllGather(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,23 +29,22 @@ class TestCollectiveAllGather(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofsplit", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_split", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'rank': self.rank, - 'nranks': nranks - }, - outputs={'Out': toutdata}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_split", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'rank': self.rank, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/column_parallel_linear_api.py b/python/paddle/fluid/tests/unittests/collective/column_parallel_linear_api.py index 5f6178dbfd6..accecd08c10 100644 --- a/python/paddle/fluid/tests/unittests/collective/column_parallel_linear_api.py +++ b/python/paddle/fluid/tests/unittests/collective/column_parallel_linear_api.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestColumnParallelLinearAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -32,18 +31,22 @@ class TestColumnParallelLinearAPI(TestCollectiveAPIRunnerBase): np.random.seed(2020) np_array = np.random.rand(1000, 16) - data = paddle.static.data(name='tindata', - shape=[10, 1000], - dtype="float32") + data = paddle.static.data( + name='tindata', shape=[10, 1000], dtype="float32" + ) paddle.distributed.broadcast(data, src=0) if rank == 0: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[:, 0:8]), ) + np_array[:, 0:8] + ), + ) else: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[:, 8:16]), ) + np_array[:, 8:16] + ), + ) linear_out = paddle.distributed.split( data, diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_allgather_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_allgather_api_dygraph.py index f4ff1e5b7b3..a04e30dfae1 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_allgather_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_allgather_api_dygraph.py @@ -19,8 +19,7 @@ import paddle.distributed as dist import test_collective_api_base as test_collective_base -class StreamAllgatherTestCase(): - +class StreamAllgatherTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ class StreamAllgatherTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,47 +39,53 @@ class StreamAllgatherTestCase(): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) # case 1: pass an empty tensor list empty_tensor_list = [] - task = dist.stream.all_gather(empty_tensor_list, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.all_gather( + empty_tensor_list, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() - assert np.allclose(empty_tensor_list, - test_data_list, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + empty_tensor_list, test_data_list, rtol=1e-05, atol=1e-05 + ) # case 2: pass a pre-sized tensor list full_tensor_list = [paddle.empty_like(tensor) for _ in test_data_list] - task = dist.stream.all_gather(full_tensor_list, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.all_gather( + full_tensor_list, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() - assert np.allclose(full_tensor_list, - test_data_list, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + full_tensor_list, test_data_list, rtol=1e-05, atol=1e-05 + ) # case 3: pass a pre-sized tensor result_tensor = paddle.concat( - [paddle.to_tensor(data) for data in test_data_list]) + [paddle.to_tensor(data) for data in test_data_list] + ) out_tensor = paddle.empty_like(result_tensor) - task = dist.stream.all_gather(out_tensor, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.all_gather( + out_tensor, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() assert np.allclose(out_tensor, result_tensor, rtol=1e-05, atol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_allreduce_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_allreduce_api_dygraph.py index abf45520351..be8841345e2 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_allreduce_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_allreduce_api_dygraph.py @@ -19,8 +19,7 @@ import paddle.distributed as dist import test_collective_api_base as test_collective_base -class StreamAllReduceTestCase(): - +class StreamAllReduceTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ class StreamAllReduceTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,15 +39,16 @@ class StreamAllReduceTestCase(): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) - task = dist.stream.all_reduce(tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.all_reduce( + tensor, sync_op=self._sync_op, use_calc_stream=self._use_calc_stream + ) if not self._sync_op: task.wait() diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_api_dygraph.py index 8842bcd08b6..07693342c6b 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_api_dygraph.py @@ -19,8 +19,7 @@ import paddle.distributed as dist import test_collective_api_base as test_collective_base -class StreamAllToAllTestCase(): - +class StreamAllToAllTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ class StreamAllToAllTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,17 +39,23 @@ class StreamAllToAllTestCase(): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) nranks = len(test_data_list) data1 = test_data_list[0] data2 = test_data_list[1] result1 = np.vstack( - [data1[0:data1.shape[0] // 2, :], data2[0:data2.shape[0] // 2, :]]) + [ + data1[0 : data1.shape[0] // 2, :], + data2[0 : data2.shape[0] // 2, :], + ] + ) result2 = np.vstack( - [data1[data1.shape[0] // 2:, :], data2[data2.shape[0] // 2:, :]]) + [data1[data1.shape[0] // 2 :, :], data2[data2.shape[0] // 2 :, :]] + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) @@ -57,48 +63,52 @@ class StreamAllToAllTestCase(): # case 1: pass an empty tensor list empty_tensor_list = [] - task = dist.stream.alltoall(empty_tensor_list, [t1, t2], - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.alltoall( + empty_tensor_list, + [t1, t2], + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() result_tensor_list = np.vstack(empty_tensor_list) if rank == 0: - assert np.allclose(result_tensor_list, - result1, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + result_tensor_list, result1, rtol=1e-05, atol=1e-05 + ) else: - assert np.allclose(result_tensor_list, - result2, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + result_tensor_list, result2, rtol=1e-05, atol=1e-05 + ) # case 2: pass a pre-sized tensor list full_tensor_list = [paddle.empty_like(t1) for _ in test_data_list] - task = dist.stream.alltoall(full_tensor_list, [t1, t2], - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.alltoall( + full_tensor_list, + [t1, t2], + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() result_tensor_list = np.vstack(full_tensor_list) if rank == 0: - assert np.allclose(result_tensor_list, - result1, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + result_tensor_list, result1, rtol=1e-05, atol=1e-05 + ) else: - assert np.allclose(result_tensor_list, - result2, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + result_tensor_list, result2, rtol=1e-05, atol=1e-05 + ) # case 3: pass a pre-sized tensor out_tensor = paddle.empty_like(tensor) - task = dist.stream.alltoall(out_tensor, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.alltoall( + out_tensor, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_single_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_single_api_dygraph.py index bb36693b0fb..cb235dd9d11 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_single_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_single_api_dygraph.py @@ -19,8 +19,7 @@ import paddle.distributed as dist import test_collective_api_base as test_collective_base -class StreamAllToAllSingleTestCase(): - +class StreamAllToAllSingleTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ class StreamAllToAllSingleTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,17 +39,23 @@ class StreamAllToAllSingleTestCase(): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) nranks = len(test_data_list) data1 = paddle.to_tensor(test_data_list[0]) data2 = paddle.to_tensor(test_data_list[1]) result1 = np.vstack( - (data1[0:data1.shape[0] // 2, :], data2[0:data2.shape[0] // 2, :])) + ( + data1[0 : data1.shape[0] // 2, :], + data2[0 : data2.shape[0] // 2, :], + ) + ) result2 = np.vstack( - (data1[data1.shape[0] // 2:, :], data2[data2.shape[0] // 2:, :])) + (data1[data1.shape[0] // 2 :, :], data2[data2.shape[0] // 2 :, :]) + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) @@ -59,7 +65,8 @@ class StreamAllToAllSingleTestCase(): out_tensor, tensor, sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_broadcast_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_broadcast_api_dygraph.py index 487dfd6ae68..68d496a11e4 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_broadcast_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_broadcast_api_dygraph.py @@ -19,8 +19,7 @@ import paddle.distributed as dist import test_collective_api_base as test_collective_base -class StreamBroadcastTestCase(): - +class StreamBroadcastTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ class StreamBroadcastTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -38,12 +38,15 @@ class StreamBroadcastTestCase(): src_rank = 1 result = test_collective_base.create_test_data( - shape=self._shape, dtype=self._dtype, seed=self._seeds[src_rank]) + shape=self._shape, dtype=self._dtype, seed=self._seeds[src_rank] + ) tensor = paddle.to_tensor(result) - task = dist.stream.broadcast(tensor, - src=src_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.broadcast( + tensor, + src=src_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_api_dygraph.py index a487eac566a..444390d20c9 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_api_dygraph.py @@ -19,8 +19,7 @@ import paddle.distributed as dist import test_collective_api_base as test_collective_base -class StreamReduceTestCase(): - +class StreamReduceTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ class StreamReduceTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,16 +39,19 @@ class StreamReduceTestCase(): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) - task = dist.stream.reduce(tensor, - dst=1, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.reduce( + tensor, + dst=1, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() @@ -56,10 +59,9 @@ class StreamReduceTestCase(): if rank == 1: assert np.allclose(tensor, result, rtol=1e-05, atol=1e-05) else: - assert np.allclose(tensor, - test_data_list[rank], - rtol=1e-05, - atol=1e-05) + assert np.allclose( + tensor, test_data_list[rank], rtol=1e-05, atol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_scatter_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_scatter_api_dygraph.py index effaf1cb6c9..d65d99ba8e5 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_scatter_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_scatter_api_dygraph.py @@ -17,11 +17,12 @@ import numpy as np import paddle import paddle.distributed as dist import test_collective_api_base as test_collective_base -from paddle.distributed.communication.stream.reduce_scatter import _reduce_scatter_base +from paddle.distributed.communication.stream.reduce_scatter import ( + _reduce_scatter_base, +) -class StreamReduceScatterTestCase(): - +class StreamReduceScatterTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -31,7 +32,8 @@ class StreamReduceScatterTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -40,12 +42,13 @@ class StreamReduceScatterTestCase(): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) reduce_result = sum(test_data_list) - result1 = reduce_result[0:reduce_result.shape[0] // 2] - result2 = reduce_result[reduce_result.shape[0] // 2:] + result1 = reduce_result[0 : reduce_result.shape[0] // 2] + result2 = reduce_result[reduce_result.shape[0] // 2 :] rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) @@ -53,9 +56,12 @@ class StreamReduceScatterTestCase(): # case 1: pass a pre-sized tensor list t1, t2 = paddle.split(tensor, 2, axis=0) result_tensor = paddle.empty_like(t1) - task = dist.stream.reduce_scatter(result_tensor, [t1, t2], - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.reduce_scatter( + result_tensor, + [t1, t2], + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: @@ -65,10 +71,12 @@ class StreamReduceScatterTestCase(): # case 2: pass a pre-sized tensor result_tensor = paddle.empty_like(t1) - task = dist.stream.reduce_scatter(result_tensor, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.reduce_scatter( + result_tensor, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: @@ -78,10 +86,12 @@ class StreamReduceScatterTestCase(): # case 3: test the legacy API result_tensor = paddle.empty_like(t1) - task = _reduce_scatter_base(result_tensor, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = _reduce_scatter_base( + result_tensor, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_scatter_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_scatter_api_dygraph.py index 6060e5050ca..b4f01798a66 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_scatter_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_scatter_api_dygraph.py @@ -19,8 +19,7 @@ import paddle.distributed as dist import test_collective_api_base as test_collective_base -class StreamScatterTestCase(): - +class StreamScatterTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ class StreamScatterTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,24 +39,28 @@ class StreamScatterTestCase(): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) src_rank = 1 src_data = test_data_list[src_rank] - result1 = src_data[0:src_data.shape[0] // 2] - result2 = src_data[src_data.shape[0] // 2:] + result1 = src_data[0 : src_data.shape[0] // 2] + result2 = src_data[src_data.shape[0] // 2 :] rank = dist.get_rank() # case 1: pass a pre-sized tensor list tensor = paddle.to_tensor(test_data_list[rank]) t1, t2 = paddle.split(tensor, 2, axis=0) - task = dist.stream.scatter(t1, [t1, t2], - src=src_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.scatter( + t1, + [t1, t2], + src=src_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == src_rank: @@ -67,11 +71,13 @@ class StreamScatterTestCase(): # case 2: pass a pre-sized tensor tensor = paddle.to_tensor(src_data) t1 = paddle.empty_like(t1) - task = dist.stream.scatter(t1, - tensor, - src=src_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.scatter( + t1, + tensor, + src=src_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == src_rank: diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_sendrecv_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_sendrecv_api_dygraph.py index f9b2806fc1b..aa7891a7ab7 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_sendrecv_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_sendrecv_api_dygraph.py @@ -19,8 +19,7 @@ import paddle.distributed as dist import test_collective_api_base as test_collective_base -class StreamSendRecvTestCase(): - +class StreamSendRecvTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ class StreamSendRecvTestCase(): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,9 +39,10 @@ class StreamSendRecvTestCase(): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) src_rank = 0 dst_rank = 1 @@ -49,15 +50,19 @@ class StreamSendRecvTestCase(): rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) if rank == 0: - task = dist.stream.send(tensor, - dst=dst_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.send( + tensor, + dst=dst_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) else: - task = dist.stream.recv(tensor, - src=src_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.recv( + tensor, + src=src_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/auto_parallel_parallelizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/auto_parallel_parallelizer.py index 185e2c23fed..d87cacb2138 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/auto_parallel_parallelizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/auto_parallel_parallelizer.py @@ -30,27 +30,27 @@ _global_process_mesh = None class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -67,24 +67,29 @@ class MLPLayer(nn.Layer): def mlp_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) auto.shard_tensor(input, _global_process_mesh, [None, None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) predict = mlp(input) @@ -95,7 +100,6 @@ def mlp_pretrain_forward(train_program, start_program): class TestMLPAutoParallelizer(unittest.TestCase): - def test_mlp_serial(self): global _global_process_mesh @@ -114,17 +118,24 @@ class TestMLPAutoParallelizer(unittest.TestCase): train_program = static.Program() start_program = static.Program() loss, train_program, start_program = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) suffix = core.kAutoParallelSuffix() for block in distributed_main_program.blocks: for op in block.ops: diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/c_comm_init_op.py b/python/paddle/fluid/tests/unittests/collective/fleet/c_comm_init_op.py index 4c981156ebb..a90c7f1baae 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/c_comm_init_op.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/c_comm_init_op.py @@ -15,14 +15,15 @@ import unittest import os import paddle.fluid as fluid -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) import paddle paddle.enable_static() class TestCCommInitOp(unittest.TestCase): - def setUp(self): self.endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS").split(',') self.current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT") @@ -42,24 +43,29 @@ class TestCCommInitOp(unittest.TestCase): nccl_id_var = block.create_var( name=fluid.unique_name.generate('nccl_id'), persistable=True, - type=fluid.core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': self.rank, - 'endpoint': self.current_endpoint, - 'other_endpoints': self.other_endpoints - }) - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': self.nranks, - 'rank': self.rank, - 'ring_id': 0, - 'device_id': self.gpu_id - }) + type=fluid.core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': self.rank, + 'endpoint': self.current_endpoint, + 'other_endpoints': self.other_endpoints, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': self.nranks, + 'rank': self.rank, + 'ring_id': 0, + 'device_id': self.gpu_id, + }, + ) self.exe.run(program) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge.py b/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge.py index dc144f997d4..9e2e2da3876 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge.py @@ -26,7 +26,6 @@ fluid.default_main_program().random_seed = 1 class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -39,23 +38,33 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = fluid.optimizer.GradientMergeOptimizer(opt, 2) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge_raw_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge_raw_optimizer.py index ff31a7016a6..f4de5280d0f 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge_raw_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge_raw_optimizer.py @@ -23,7 +23,6 @@ from dist_mnist import cnn_model class TestDistMnistGradientMergeRawOptimizer(TestDistRunnerBase): - def get_model(self, batch_size=2, single_device=False): paddle.enable_static() paddle.seed(1) @@ -54,9 +53,9 @@ class TestDistMnistGradientMergeRawOptimizer(TestDistRunnerBase): strategy.without_graph_optimization = True fleet.init(is_collective=True, strategy=strategy) - image = paddle.static.data(name='image', - shape=[None, 1, 28, 28], - dtype="float32") + image = paddle.static.data( + name='image', shape=[None, 1, 28, 28], dtype="float32" + ) label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') predict = cnn_model(image) acc = paddle.metric.accuracy(predict, label) @@ -68,7 +67,8 @@ class TestDistMnistGradientMergeRawOptimizer(TestDistRunnerBase): optimizer = fluid.optimizer.GradientMergeOptimizer( optimizer, k_steps=strategy.gradient_merge_configs["k_steps"], - avg=strategy.gradient_merge_configs["avg"]) + avg=strategy.gradient_merge_configs["avg"], + ) world_size = 1 else: optimizer = fleet.distributed_optimizer(optimizer) @@ -88,10 +88,12 @@ class TestDistMnistGradientMergeRawOptimizer(TestDistRunnerBase): else: assert start_allreduce_idx == 1 - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) return test_program, cost, train_reader, test_reader, acc, predict diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py index bac023a8e7d..13450324934 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py @@ -21,7 +21,10 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Linear from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.sharding import group_sharded_parallel, save_group_sharded_model +from paddle.distributed.sharding import ( + group_sharded_parallel, + save_group_sharded_model, +) epoch = 10 paddle.seed(2022) @@ -33,7 +36,6 @@ batch_size = 100 class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -49,7 +51,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -62,45 +63,46 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_multi_precision, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_multi_precision) + multi_precision=use_multi_precision, + ) return optimizer -def train_mlp(model, - shard_level, - use_multi_precision, - output_dir, - amp_level='O1'): +def train_mlp( + model, shard_level, use_multi_precision, output_dir, amp_level='O1' +): group = paddle.distributed.new_group([0, 1]) - optimizer = optimizer_setting(model=model, - use_multi_precision=use_multi_precision) - model = paddle.amp.decorate(models=model, - level=amp_level, - save_dtype='float32') + optimizer = optimizer_setting( + model=model, use_multi_precision=use_multi_precision + ) + model = paddle.amp.decorate( + models=model, level=amp_level, save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) - model, optimizer, scaler = group_sharded_parallel(model=model, - optimizer=optimizer, - level=shard_level, - scaler=scaler) - - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + model, optimizer, scaler = group_sharded_parallel( + model=model, optimizer=optimizer, level=shard_level, scaler=scaler + ) + + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -111,8 +113,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level=amp_level): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if not use_multi_precision: @@ -138,44 +141,56 @@ def test_sharding_api(): output_dir = tempfile.mkdtemp() # fp16 - stage2_params = train_mlp(mlp1, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2') - stage3_params = train_mlp(mlp2, - shard_level="p_g_os", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2') + stage2_params = train_mlp( + mlp1, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + ) + stage3_params = train_mlp( + mlp2, + shard_level="p_g_os", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # AMP mlp3, mlp4 = MLP(), MLP() mlp3.set_state_dict(state_dict) mlp4.set_state_dict(state_dict) - stage2_params = train_mlp(mlp3, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O1') - stage3_params = train_mlp(mlp4, - shard_level="p_g_os", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O1') + stage2_params = train_mlp( + mlp3, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O1', + ) + stage3_params = train_mlp( + mlp4, + shard_level="p_g_os", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O1', + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) shutil.rmtree(output_dir) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py index f57dfd7c98d..2f07007c66a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py @@ -19,7 +19,10 @@ import paddle import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.sharding import group_sharded_parallel, save_group_sharded_model +from paddle.distributed.sharding import ( + group_sharded_parallel, + save_group_sharded_model, +) epoch = 10 paddle.seed(2022) @@ -31,7 +34,6 @@ batch_size = 100 class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -47,7 +49,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -60,47 +61,55 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_multi_precision, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_multi_precision) + multi_precision=use_multi_precision, + ) return optimizer -def train_mlp(model, - shard_level, - use_multi_precision, - output_dir, - amp_level='O1', - sync_buffers=False, - dp_group=None): - optimizer = optimizer_setting(model=model, - use_multi_precision=use_multi_precision) - model = paddle.amp.decorate(models=model, - level=amp_level, - save_dtype='float32') +def train_mlp( + model, + shard_level, + use_multi_precision, + output_dir, + amp_level='O1', + sync_buffers=False, + dp_group=None, +): + optimizer = optimizer_setting( + model=model, use_multi_precision=use_multi_precision + ) + model = paddle.amp.decorate( + models=model, level=amp_level, save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) - model, optimizer, scaler = group_sharded_parallel(model=model, - optimizer=optimizer, - level=shard_level, - scaler=scaler, - sync_buffers=sync_buffers, - dp_group=dp_group) - - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + model, optimizer, scaler = group_sharded_parallel( + model=model, + optimizer=optimizer, + level=shard_level, + scaler=scaler, + sync_buffers=sync_buffers, + dp_group=dp_group, + ) + + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -111,8 +120,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level=amp_level): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if not use_multi_precision: @@ -138,51 +148,64 @@ def test_sharding_api(): output_dir = tempfile.mkdtemp() - #test sharding + dp, just for test + # test sharding + dp, just for test dp_group = paddle.distributed.new_group( - list(range(paddle.distributed.get_world_size()))) - - stage2_dp_params = train_mlp(mlp1, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2', - sync_buffers=True, - dp_group=dp_group) + list(range(paddle.distributed.get_world_size())) + ) + + stage2_dp_params = train_mlp( + mlp1, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + sync_buffers=True, + dp_group=dp_group, + ) # fp16 - stage2_params = train_mlp(mlp1, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2') - stage3_params = train_mlp(mlp2, - shard_level="p_g_os", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2') + stage2_params = train_mlp( + mlp1, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + ) + stage3_params = train_mlp( + mlp2, + shard_level="p_g_os", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # AMP mlp3, mlp4 = MLP(), MLP() mlp3.set_state_dict(state_dict) mlp4.set_state_dict(state_dict) - stage2_params = train_mlp(mlp3, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O1') - stage3_params = train_mlp(mlp4, - shard_level="p_g_os", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O1') + stage2_params = train_mlp( + mlp3, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O1', + ) + stage3_params = train_mlp( + mlp4, + shard_level="p_g_os", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O1', + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py index 40767d6fa86..970f7a5b037 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py @@ -23,8 +23,12 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) seed = 2022 epoch = 2 @@ -35,7 +39,6 @@ paddle.seed(seed) class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -51,7 +54,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -63,42 +65,50 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters(), - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[ + { + "params": model.parameters(), + } + ] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - batch_size=100, - use_pure_fp16=False, - accumulate_grad=False, - opt_group=False, - save_model=False, - test_minimize=False): +def train_mlp( + model, + sharding_stage, + batch_size=100, + use_pure_fp16=False, + accumulate_grad=False, + opt_group=False, + save_model=False, + test_minimize=False, +): if sharding_stage != "dp": group = paddle.distributed.new_group([0, 1], backend="nccl") if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if sharding_stage == 2: optimizer = GroupShardedOptimizerStage2( - params=optimizer._parameter_list, optim=optimizer, group=group) + params=optimizer._parameter_list, optim=optimizer, group=group + ) - model = GroupShardedStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + model = GroupShardedStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) else: model = paddle.DataParallel(model) @@ -108,18 +118,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage2_optimizer.minimize() error ======") + "====== Find sharding_stage2_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: @@ -174,50 +187,50 @@ def test_dp_stage2(): mlp7.set_state_dict(state_dict) # DP VS stage2 - dp_params = train_mlp(mlp1, - sharding_stage="dp", - use_pure_fp16=False, - opt_group=False) - stage2_params = train_mlp(mlp2, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) + dp_params = train_mlp( + mlp1, sharding_stage="dp", use_pure_fp16=False, opt_group=False + ) + stage2_params = train_mlp( + mlp2, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # stage2 accumulate grad stage2_params = train_mlp(mlp3, sharding_stage=2, accumulate_grad=True) - stage2_accumulate_grad = train_mlp(mlp4, - sharding_stage=2, - batch_size=20, - accumulate_grad=True) + stage2_accumulate_grad = train_mlp( + mlp4, sharding_stage=2, batch_size=20, accumulate_grad=True + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage2_accumulate_grad[i].numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage2_accumulate_grad[i].numpy(), + rtol=1e-5, + atol=1e-5, + ) # stage2 param list VS param group - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=False, - opt_group=True) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=False, opt_group=True + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage2, optimizer_stage2 = train_mlp(mlp6, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage2, optimizer_stage2 = train_mlp( + mlp6, + sharding_stage=2, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage2.state_dict(), model_file) paddle.save(optimizer_stage2.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py index 7f16d926f53..56c90009da6 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py @@ -23,8 +23,12 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) seed = 2022 epoch = 2 @@ -35,7 +39,6 @@ paddle.seed(seed) class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -51,7 +54,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -63,42 +65,50 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters(), - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[ + { + "params": model.parameters(), + } + ] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - batch_size=100, - use_pure_fp16=False, - accumulate_grad=False, - opt_group=False, - save_model=False, - test_minimize=False): +def train_mlp( + model, + sharding_stage, + batch_size=100, + use_pure_fp16=False, + accumulate_grad=False, + opt_group=False, + save_model=False, + test_minimize=False, +): if sharding_stage != "dp": group = paddle.distributed.new_group([0, 1], backend="nccl") if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if sharding_stage == 2: origin_model = model optimizer = GroupShardedOptimizerStage2( - params=optimizer._parameter_list, optim=optimizer, group=group) - model = GroupShardedStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + params=optimizer._parameter_list, optim=optimizer, group=group + ) + model = GroupShardedStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) model._set_reduce_overlap(True) optimizer._set_broadcast_overlap(True, model) else: @@ -110,18 +120,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage2_optimizer.minimize() error ======") + "====== Find sharding_stage2_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: @@ -178,50 +191,50 @@ def test_dp_stage2(): mlp7.set_state_dict(state_dict) # DP VS stage2 - dp_params = train_mlp(mlp1, - sharding_stage="dp", - use_pure_fp16=False, - opt_group=False) - stage2_params = train_mlp(mlp2, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) + dp_params = train_mlp( + mlp1, sharding_stage="dp", use_pure_fp16=False, opt_group=False + ) + stage2_params = train_mlp( + mlp2, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # stage2 accumulate grad stage2_params = train_mlp(mlp3, sharding_stage=2, accumulate_grad=True) - stage2_accumulate_grad = train_mlp(mlp4, - sharding_stage=2, - batch_size=20, - accumulate_grad=True) + stage2_accumulate_grad = train_mlp( + mlp4, sharding_stage=2, batch_size=20, accumulate_grad=True + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage2_accumulate_grad[i].numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage2_accumulate_grad[i].numpy(), + rtol=1e-5, + atol=1e-5, + ) # stage2 param list VS param group - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=False, - opt_group=True) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=False, opt_group=True + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage2, optimizer_stage2 = train_mlp(mlp6, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage2, optimizer_stage2 = train_mlp( + mlp6, + sharding_stage=2, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage2.state_dict(), model_file) paddle.save(optimizer_stage2.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py index 3420114d3f9..710dc90fcf5 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py @@ -18,11 +18,21 @@ import numpy as np import paddle from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler - -from dygraph_group_sharded_stage2 import MLP, reader_decorator, optimizer_setting +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) + +from dygraph_group_sharded_stage2 import ( + MLP, + reader_decorator, + optimizer_setting, +) seed = 2021 epoch = 2 @@ -40,20 +50,22 @@ def train_mlp(model, offload=False): scaler = paddle.amp.GradScaler(init_loss_scaling=1024) scaler = GroupShardedScaler(scaler) - optimizer = GroupShardedOptimizerStage2(params=optimizer._parameter_list, - optim=optimizer, - offload=offload) + optimizer = GroupShardedOptimizerStage2( + params=optimizer._parameter_list, optim=optimizer, offload=offload + ) model = GroupShardedStage2(model, optimizer, buffer_max_size=2**21) - train_reader = paddle.batch(reader_decorator(linear_size), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(linear_size), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -66,8 +78,9 @@ def train_mlp(model, offload=False): with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) scaler.scale(avg_loss).backward() @@ -93,10 +106,12 @@ def test_sharding_stage2_offload(): mlp_offload_params = train_mlp(mlp_offload, offload=True) for i in range(len(mlp_params)): - np.testing.assert_allclose(mlp_params[i].numpy(), - mlp_offload_params[i].numpy(), - rtol=5e-3, - atol=5e-3) + np.testing.assert_allclose( + mlp_params[i].numpy(), + mlp_offload_params[i].numpy(), + rtol=5e-3, + atol=5e-3, + ) return diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py index fe629541376..d6f8ef5efb7 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py @@ -23,10 +23,18 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import GroupShardedStage3 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import ( + GroupShardedStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) epoch = 10 paddle.seed(2022) @@ -37,7 +45,6 @@ l2_decay = 1e-4 class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -53,7 +60,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -66,53 +72,58 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_pure_fp16) + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - use_pure_fp16=False, - accumulate_grad=False, - batch_size=100, - opt_group=False, - sync_comm=False, - test_minimize=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + use_pure_fp16=False, + accumulate_grad=False, + batch_size=100, + opt_group=False, + sync_comm=False, + test_minimize=False, + save_model=False, +): group = paddle.distributed.new_group([0, 1]) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = GroupShardedScaler(scaler) if sharding_stage == 2: optimizer = GroupShardedOptimizerStage2( - params=optimizer._parameter_list, optim=optimizer, group=group) - model = GroupShardedStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + params=optimizer._parameter_list, optim=optimizer, group=group + ) + model = GroupShardedStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) elif sharding_stage == 3: - model = GroupShardedStage3(model, - optimizer=optimizer, - group=group, - sync_comm=sync_comm, - segment_size=2**15) + model = GroupShardedStage3( + model, + optimizer=optimizer, + group=group, + sync_comm=sync_comm, + segment_size=2**15, + ) # check optimizer.minimize() error if test_minimize: @@ -120,18 +131,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage3_optimizer.minimize() error ======") + "====== Find sharding_stage3_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -142,8 +156,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if batch_size == 20: @@ -178,8 +193,19 @@ def train_mlp(model, def test_stage2_stage3(): paddle.distributed.init_parallel_env() - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = MLP( - ), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -193,78 +219,87 @@ def test_stage2_stage3(): mlp10.set_state_dict(state_dict) # fp32 - stage2_params = train_mlp(mlp1, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) - stage3_params = train_mlp(mlp2, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False) + stage2_params = train_mlp( + mlp1, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) + stage3_params = train_mlp( + mlp2, sharding_stage=3, use_pure_fp16=False, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-6, - atol=1e-6) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-6, + atol=1e-6, + ) # fp32 accumulate grad - stage3_params = train_mlp(mlp3, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - opt_group=True) - stage3_params_add = train_mlp(mlp4, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - batch_size=20, - opt_group=True) + stage3_params = train_mlp( + mlp3, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + opt_group=True, + ) + stage3_params_add = train_mlp( + mlp4, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + batch_size=20, + opt_group=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_add[i].numpy(), - rtol=1e-6, - atol=1e-4) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_add[i].numpy(), + rtol=1e-6, + atol=1e-4, + ) # fp16 - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=True, - opt_group=False) - stage3_params = train_mlp(mlp6, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=True, opt_group=False + ) + stage3_params = train_mlp( + mlp6, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # fp16 sync_comm - stage3_params = train_mlp(mlp7, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) - stage3_params_re = train_mlp(mlp8, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False, - sync_comm=True) + stage3_params = train_mlp( + mlp7, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) + stage3_params_re = train_mlp( + mlp8, + sharding_stage=3, + use_pure_fp16=True, + opt_group=False, + sync_comm=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_re[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + stage3_params[i].numpy(), stage3_params_re[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage3, optimizer_stage3 = train_mlp(mlp9, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage3, optimizer_stage3 = train_mlp( + mlp9, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage3.state_dict(), model_file) paddle.save(optimizer_stage3.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) @@ -274,11 +309,13 @@ def test_stage2_stage3(): shutil.rmtree(output_dir) # check optimizer.minimize() error - train_mlp(mlp10, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - test_minimize=True) + train_mlp( + mlp10, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + test_minimize=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py index 2c586b56946..0c63d2e1d8a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py @@ -20,8 +20,12 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import GroupShardedStage3 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import ( + GroupShardedStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) epoch = 10 paddle.seed(2022) @@ -32,7 +36,6 @@ l2_decay = 1e-4 class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -48,7 +51,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -60,48 +62,56 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters() - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[{"params": model.parameters()}] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - use_pure_fp16=False, - accumulate_grad=False, - offload=False, - batch_size=100, - convert2cpu=False): +def train_mlp( + model, + use_pure_fp16=False, + accumulate_grad=False, + offload=False, + batch_size=100, + convert2cpu=False, +): group = paddle.distributed.new_group([0, 1]) optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = GroupShardedScaler(scaler) - model = GroupShardedStage3(model, - optimizer=optimizer, - group=group, - offload=offload, - segment_size=2**15) - - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + model = GroupShardedStage3( + model, + optimizer=optimizer, + group=group, + offload=offload, + segment_size=2**15, + ) + + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -112,8 +122,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if accumulate_grad: @@ -147,8 +158,15 @@ def train_mlp(model, def test_stage3_offload(): paddle.distributed.init_parallel_env() - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6 = MLP(), MLP(), MLP(), MLP(), MLP( - ), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -161,36 +179,43 @@ def test_stage3_offload(): stage3_params = train_mlp(mlp1, use_pure_fp16=False) stage3_params_offload = train_mlp(mlp2, use_pure_fp16=False, offload=True) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-6, - atol=1e-8) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-6, + atol=1e-8, + ) # fp16 offload stage3_params = train_mlp(mlp3, use_pure_fp16=True) stage3_params_offload = train_mlp(mlp4, use_pure_fp16=True, offload=True) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-2, - atol=1e-2) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-2, + atol=1e-2, + ) # fp32 accumulate grad offload - stage3_params = train_mlp(mlp5, - use_pure_fp16=False, - batch_size=20, - accumulate_grad=True) - stage3_params_offload = train_mlp(mlp6, - use_pure_fp16=False, - accumulate_grad=True, - offload=True, - batch_size=20, - convert2cpu=True) + stage3_params = train_mlp( + mlp5, use_pure_fp16=False, batch_size=20, accumulate_grad=True + ) + stage3_params_offload = train_mlp( + mlp6, + use_pure_fp16=False, + accumulate_grad=True, + offload=True, + batch_size=20, + convert2cpu=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-6, - atol=1e-8) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-6, + atol=1e-8, + ) return diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_optimizer_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_optimizer_stage2.py index 2f5a73830d7..78dbd9b2ccd 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_optimizer_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_optimizer_stage2.py @@ -22,7 +22,9 @@ from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard from paddle.distributed.fleet.utils.internal_storage import GradStorage -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) base_lr = 0.1 momentum_rate = 0.9 @@ -34,7 +36,6 @@ class_dim = 102 class MLP(fluid.Layer): - def __init__(self, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -48,7 +49,6 @@ class MLP(fluid.Layer): def reader_decorator(): - def __reader__(): for _ in range(100): img = np.random.rand(10).astype('float32') @@ -63,7 +63,8 @@ def optimizer_setting(parameter_list=None): learning_rate=base_lr, momentum=momentum_rate, weight_decay=paddle.regularizer.L2Decay(l2_decay), - parameters=parameter_list) + parameters=parameter_list, + ) return optimizer @@ -74,18 +75,20 @@ def train_mlp(): mlp = MLP() optimizer = optimizer_setting(parameter_list=mlp.parameters()) - oss_optimizer = ShardingOptimizerStage2(params=mlp.parameters(), - optim=optimizer, - group=group) + oss_optimizer = ShardingOptimizerStage2( + params=mlp.parameters(), optim=optimizer, group=group + ) # cover grad_storage code trainable_param2align = dict() for p in mlp.parameters(): trainable_param2align[p.name] = 0 - grad_storage = GradStorage(10000, - dtype=paddle.float32, - device="gpu", - destination=0, - parm2align=trainable_param2align) + grad_storage = GradStorage( + 10000, + dtype=paddle.float32, + device="gpu", + destination=0, + parm2align=trainable_param2align, + ) for p in mlp.parameters(): grad_storage.can_add_grad_view(p, trainable_param2align[p.name]) grad_storage.add_grad(p, trainable_param2align[p.name]) @@ -93,15 +96,17 @@ def train_mlp(): grad_storage.rebuild() grad_storage.reset_checked_in() - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2.py index a025030af2d..2303efccd02 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2.py @@ -24,8 +24,12 @@ from paddle.fluid.dygraph.nn import Linear from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) seed = 2022 epoch = 2 @@ -36,7 +40,7 @@ strategy.hybrid_configs = { "dp_degree": 2, "mp_degree": 1, "pp_degree": 1, - "sharding_degree": 1 + "sharding_degree": 1, } np.random.seed(seed) @@ -44,7 +48,6 @@ paddle.seed(seed) class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -60,7 +63,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -72,58 +74,63 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters() - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[{"params": model.parameters()}] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - batch_size=100, - use_pure_fp16=False, - accumulate_grad=False, - opt_group=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + batch_size=100, + use_pure_fp16=False, + accumulate_grad=False, + opt_group=False, + save_model=False, +): if sharding_stage == "dp": hcg = fleet.get_hybrid_communicate_group() group = hcg.get_check_parallel_group() else: group = paddle.distributed.new_group([0, 1]) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if sharding_stage == 2: - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - group=group) - - model = ShardingStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), optim=optimizer, group=group + ) + + model = ShardingStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) else: optimizer = fleet.distributed_optimizer(optimizer) model = fleet.distributed_model(model) - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: @@ -175,50 +182,50 @@ def test_dp_stage2(): mlp6.set_state_dict(state_dict) # DP VS stage2 - dp_params = train_mlp(mlp1, - sharding_stage="dp", - use_pure_fp16=False, - opt_group=False) - stage2_params = train_mlp(mlp2, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) + dp_params = train_mlp( + mlp1, sharding_stage="dp", use_pure_fp16=False, opt_group=False + ) + stage2_params = train_mlp( + mlp2, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # stage2 accumulate grad stage2_params = train_mlp(mlp3, sharding_stage=2, accumulate_grad=True) - stage2_accumulate_grad = train_mlp(mlp4, - sharding_stage=2, - batch_size=20, - accumulate_grad=True) + stage2_accumulate_grad = train_mlp( + mlp4, sharding_stage=2, batch_size=20, accumulate_grad=True + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage2_accumulate_grad[i].numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage2_accumulate_grad[i].numpy(), + rtol=1e-5, + atol=1e-5, + ) # stage2 param list VS param group - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=False, - opt_group=True) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=False, opt_group=True + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage2, optimizer_stage2 = train_mlp(mlp6, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage2, optimizer_stage2 = train_mlp( + mlp6, + sharding_stage=2, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage2.state_dict(), model_file) paddle.save(optimizer_stage2.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2_offload.py index 7469591c9ed..ce795de881c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2_offload.py @@ -19,9 +19,15 @@ import paddle from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ( + ShardingScaler, +) from dygraph_sharding_stage2 import MLP, reader_decorator, optimizer_setting @@ -35,7 +41,7 @@ strategy.hybrid_configs = { "dp_degree": 2, "mp_degree": 1, "pp_degree": 1, - "sharding_degree": 1 + "sharding_degree": 1, } np.random.seed(seed) @@ -49,20 +55,22 @@ def train_mlp(model, offload=False): scaler = paddle.amp.GradScaler(init_loss_scaling=1024) scaler = ShardingScaler(scaler) - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - offload=offload) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), optim=optimizer, offload=offload + ) model = ShardingStage2(model, optimizer, buffer_max_size=2**21) - train_reader = paddle.batch(reader_decorator(linear_size), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(linear_size), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -75,8 +83,9 @@ def train_mlp(model, offload=False): with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) scaler.scale(avg_loss).backward() @@ -101,10 +110,12 @@ def test_sharding_stage2_offload(): mlp_offload_params = train_mlp(mlp_offload, offload=True) for i in range(len(mlp_params)): - np.testing.assert_allclose(mlp_params[i].numpy(), - mlp_offload_params[i].numpy(), - rtol=5e-3, - atol=5e-3) + np.testing.assert_allclose( + mlp_params[i].numpy(), + mlp_offload_params[i].numpy(), + rtol=5e-3, + atol=5e-3, + ) return diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3.py index 4e9bc7a1cac..c5d58934060 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3.py @@ -24,10 +24,18 @@ from paddle.fluid.dygraph.nn import Linear from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ShardingStage3 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ( + ShardingStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ( + ShardingScaler, +) epoch = 10 paddle.seed(2021) @@ -38,7 +46,6 @@ l2_decay = 1e-4 class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -54,7 +61,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -67,53 +73,54 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_pure_fp16) + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - use_pure_fp16=False, - accumulate_grad=False, - batch_size=100, - opt_group=False, - sync_comm=False, - test_minimize=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + use_pure_fp16=False, + accumulate_grad=False, + batch_size=100, + opt_group=False, + sync_comm=False, + test_minimize=False, + save_model=False, +): group = paddle.distributed.new_group([0, 1]) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = ShardingScaler(scaler) if sharding_stage == 2: - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - group=group) - model = ShardingStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), optim=optimizer, group=group + ) + model = ShardingStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) elif sharding_stage == 3: - model = ShardingStage3(model, - optimizer=optimizer, - group=group, - sync_comm=sync_comm) + model = ShardingStage3( + model, optimizer=optimizer, group=group, sync_comm=sync_comm + ) # check optimizer.minimize() error if test_minimize: @@ -121,18 +128,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage3_optimizer.minimize() error ======") + "====== Find sharding_stage3_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -143,8 +153,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if batch_size == 20: @@ -178,8 +189,19 @@ def train_mlp(model, def test_stage2_stage3(): - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = MLP( - ), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -193,78 +215,87 @@ def test_stage2_stage3(): mlp10.set_state_dict(state_dict) # fp32 - stage2_params = train_mlp(mlp1, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) - stage3_params = train_mlp(mlp2, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False) + stage2_params = train_mlp( + mlp1, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) + stage3_params = train_mlp( + mlp2, sharding_stage=3, use_pure_fp16=False, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-6, - atol=1e-6) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-6, + atol=1e-6, + ) # fp32 accumulate grad - stage3_params = train_mlp(mlp3, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - opt_group=True) - stage3_params_add = train_mlp(mlp4, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - batch_size=20, - opt_group=True) + stage3_params = train_mlp( + mlp3, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + opt_group=True, + ) + stage3_params_add = train_mlp( + mlp4, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + batch_size=20, + opt_group=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_add[i].numpy(), - rtol=1e-6, - atol=1e-4) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_add[i].numpy(), + rtol=1e-6, + atol=1e-4, + ) # fp16 - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=True, - opt_group=False) - stage3_params = train_mlp(mlp6, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=True, opt_group=False + ) + stage3_params = train_mlp( + mlp6, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # fp16 sync_comm - stage3_params = train_mlp(mlp7, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) - stage3_params_re = train_mlp(mlp8, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False, - sync_comm=True) + stage3_params = train_mlp( + mlp7, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) + stage3_params_re = train_mlp( + mlp8, + sharding_stage=3, + use_pure_fp16=True, + opt_group=False, + sync_comm=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_re[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + stage3_params[i].numpy(), stage3_params_re[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage3, optimizer_stage3 = train_mlp(mlp9, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage3, optimizer_stage3 = train_mlp( + mlp9, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage3.state_dict(), model_file) paddle.save(optimizer_stage3.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) @@ -274,11 +305,13 @@ def test_stage2_stage3(): shutil.rmtree(output_dir) # check optimizer.minimize() error - train_mlp(mlp10, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - test_minimize=True) + train_mlp( + mlp10, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + test_minimize=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3_offload.py index d56ec33694b..f7eee14046d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3_offload.py @@ -21,8 +21,12 @@ from paddle.fluid.dygraph.nn import Linear from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ShardingStage3 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ( + ShardingStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ( + ShardingScaler, +) epoch = 10 paddle.seed(2022) @@ -33,7 +37,6 @@ l2_decay = 1e-4 class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -49,7 +52,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -61,47 +63,52 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters() - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[{"params": model.parameters()}] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - use_pure_fp16=False, - accumulate_grad=False, - offload=False, - batch_size=100, - convert2cpu=False): +def train_mlp( + model, + use_pure_fp16=False, + accumulate_grad=False, + offload=False, + batch_size=100, + convert2cpu=False, +): group = paddle.distributed.new_group([0, 1]) optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = ShardingScaler(scaler) - model = ShardingStage3(model, - optimizer=optimizer, - group=group, - offload=offload) - - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + model = ShardingStage3( + model, optimizer=optimizer, group=group, offload=offload + ) + + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -112,8 +119,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if accumulate_grad: @@ -146,8 +154,15 @@ def train_mlp(model, def test_stage3_offload(): - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6 = MLP(), MLP(), MLP(), MLP(), MLP( - ), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -160,36 +175,43 @@ def test_stage3_offload(): stage3_params = train_mlp(mlp1, use_pure_fp16=False) stage3_params_offload = train_mlp(mlp2, use_pure_fp16=False, offload=True) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-6, - atol=1e-8) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-6, + atol=1e-8, + ) # fp16 offload stage3_params = train_mlp(mlp3, use_pure_fp16=True) stage3_params_offload = train_mlp(mlp4, use_pure_fp16=True, offload=True) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-2, - atol=1e-2) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-2, + atol=1e-2, + ) # fp32 accumulate grad offload - stage3_params = train_mlp(mlp5, - use_pure_fp16=False, - batch_size=20, - accumulate_grad=True) - stage3_params_offload = train_mlp(mlp6, - use_pure_fp16=False, - accumulate_grad=True, - offload=True, - batch_size=20, - convert2cpu=True) + stage3_params = train_mlp( + mlp5, use_pure_fp16=False, batch_size=20, accumulate_grad=True + ) + stage3_params_offload = train_mlp( + mlp6, + use_pure_fp16=False, + accumulate_grad=True, + offload=True, + batch_size=20, + convert2cpu=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-6, - atol=1e-8) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-6, + atol=1e-8, + ) return diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_communicate_group.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_communicate_group.py index 0e81b08a7eb..0c6bf6e9dec 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_communicate_group.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_communicate_group.py @@ -18,11 +18,11 @@ from paddle.distributed import fleet class TestNewGroupAPI(object): - def __init__(self): paddle.distributed.init_parallel_env() - topo = fleet.CommunicateTopology(["data", "model", "sharding", "pipe"], - [2, 1, 1, 1]) + topo = fleet.CommunicateTopology( + ["data", "model", "sharding", "pipe"], [2, 1, 1, 1] + ) self.hcg = fleet.HybridCommunicateGroup(topo) d1 = np.array([1, 2, 3]) @@ -49,10 +49,13 @@ class TestNewGroupAPI(object): tmp = np.array([0, 0, 0]) result = paddle.to_tensor(tmp) - paddle.distributed.scatter(result, [self.tensor2, self.tensor1], - src=dp_src_rank, - group=dp_gp, - sync_op=True) + paddle.distributed.scatter( + result, + [self.tensor2, self.tensor1], + src=dp_src_rank, + group=dp_gp, + sync_op=True, + ) if dp_rank == 0: assert np.array_equal(result, self.tensor2) elif dp_rank == 1: @@ -63,13 +66,13 @@ class TestNewGroupAPI(object): assert np.array_equal(result, self.tensor1) print("test broadcast api ok") - paddle.distributed.reduce(result, - dst=dp_src_rank, - group=dp_gp, - sync_op=True) + paddle.distributed.reduce( + result, dst=dp_src_rank, group=dp_gp, sync_op=True + ) if dp_rank == 0: - assert np.array_equal(result, paddle.add(self.tensor1, - self.tensor1)) + assert np.array_equal( + result, paddle.add(self.tensor1, self.tensor1) + ) elif dp_rank == 1: assert np.array_equal(result, self.tensor1) print("test reduce api ok") @@ -77,7 +80,8 @@ class TestNewGroupAPI(object): paddle.distributed.all_reduce(result, sync_op=True) assert np.array_equal( result, - paddle.add(paddle.add(self.tensor1, self.tensor1), self.tensor1)) + paddle.add(paddle.add(self.tensor1, self.tensor1), self.tensor1), + ) print("test all_reduce api ok") paddle.distributed.wait(result, dp_gp, use_calc_stream=True) @@ -85,10 +89,9 @@ class TestNewGroupAPI(object): print("test wait api ok") result = [] - paddle.distributed.all_gather(result, - self.tensor1, - group=dp_gp, - sync_op=True) + paddle.distributed.all_gather( + result, self.tensor1, group=dp_gp, sync_op=True + ) assert np.array_equal(result[0], self.tensor1) assert np.array_equal(result[1], self.tensor1) print("test all_gather api ok") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_inference_helper.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_inference_helper.py index 83b0aaf76c8..e746f30fa57 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_inference_helper.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_inference_helper.py @@ -19,7 +19,9 @@ import paddle import numpy as np import paddle.fluid.layers as layers import paddle.distributed.fleet as fleet -from paddle.distributed.fleet.utils.hybrid_parallel_inference import HybridParallelInferenceHelper +from paddle.distributed.fleet.utils.hybrid_parallel_inference import ( + HybridParallelInferenceHelper, +) paddle.enable_static() @@ -40,7 +42,6 @@ def numpy_while(x, w1=1.0, w2=2.0, max_len=2): class TestHybridParallelInferenceHelperClass(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() fleet.init(is_collective=True, strategy=strategy) @@ -59,29 +60,27 @@ class TestHybridParallelInferenceHelperClass(unittest.TestCase): with paddle.static.program_guard(main_program, startup_program): with paddle.fluid.device_guard(f'{device}:0'): - X = paddle.static.data(name='X', - shape=[None, 2], - dtype='float32') + X = paddle.static.data( + name='X', shape=[None, 2], dtype='float32' + ) with paddle.fluid.device_guard(f'{device}:all'): - max_len = layers.fill_constant(shape=[1], - dtype="int64", - value=2, - force_cpu=False, - name="n") - step_idx = layers.fill_constant(shape=[1], - dtype="int64", - value=0, - force_cpu=False, - name="i") + max_len = layers.fill_constant( + shape=[1], dtype="int64", value=2, force_cpu=False, name="n" + ) + step_idx = layers.fill_constant( + shape=[1], dtype="int64", value=0, force_cpu=False, name="i" + ) data = layers.array_write(X, step_idx) - cond_int = layers.fill_constant(shape=[1], - dtype="int64", - value=0, - force_cpu=False, - name="cond_int") + cond_int = layers.fill_constant( + shape=[1], + dtype="int64", + value=0, + force_cpu=False, + name="cond_int", + ) print(cond_int.shape) cond = layers.less_than(x=step_idx, y=max_len) while_op = layers.While(cond, is_test=True) @@ -94,20 +93,26 @@ class TestHybridParallelInferenceHelperClass(unittest.TestCase): with paddle.fluid.device_guard(f'{device}:0'): param_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(1.0)) - weight1 = paddle.static.create_parameter(shape=[2, 5], - dtype='float32', - attr=param_attr, - is_bias=False) + initializer=paddle.nn.initializer.Constant(1.0) + ) + weight1 = paddle.static.create_parameter( + shape=[2, 5], + dtype='float32', + attr=param_attr, + is_bias=False, + ) hidden1 = paddle.matmul(input, weight1) with paddle.fluid.device_guard(f'{device}:1'): param_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(2.0)) - weight2 = paddle.static.create_parameter(shape=[5, 2], - dtype='float32', - attr=param_attr, - is_bias=False) + initializer=paddle.nn.initializer.Constant(2.0) + ) + weight2 = paddle.static.create_parameter( + shape=[5, 2], + dtype='float32', + attr=param_attr, + is_bias=False, + ) hidden2 = paddle.matmul(hidden1, weight2) layers.array_write(hidden2, i=step_idx, array=data) @@ -136,18 +141,20 @@ class TestHybridParallelInferenceHelperClass(unittest.TestCase): num_pp=2, init_comm=nranks > 1, ) - helper.gen_infer_program(['array_write_0.out'], ['cond_int.tmp_0'], - debug=True) + helper.gen_infer_program( + ['array_write_0.out'], ['cond_int.tmp_0'], debug=True + ) exe = paddle.static.Executor(paddle.CUDAPlace(dev_id)) exe.run(startup_program) for step in range(2): - init_data = np.random.uniform(low=0.0, high=1.0, - size=[2, 2]).astype('float32') - [res] = exe.run(main_program, - feed={"X": init_data}, - fetch_list=[out]) + init_data = np.random.uniform( + low=0.0, high=1.0, size=[2, 2] + ).astype('float32') + [res] = exe.run( + main_program, feed={"X": init_data}, fetch_list=[out] + ) res_np = numpy_while(init_data) assert len(res) == len(res_np) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_amp.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_amp.py index 4906a1c86ca..172753b2fbc 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_amp.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_amp.py @@ -19,22 +19,22 @@ import unittest class TestMPClipGrad(TestDistMPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(2.0) - scheduler = paddle.optimizer.lr.ExponentialDecay(learning_rate=0.001, - gamma=0.999, - verbose=True) - optimizer = paddle.optimizer.SGD(scheduler, - grad_clip=grad_clip, - parameters=[{ - 'params': - model.parameters(), - 'weight_decay': - 0.001, - 'learning_rate': - 0.1 - }]) + scheduler = paddle.optimizer.lr.ExponentialDecay( + learning_rate=0.001, gamma=0.999, verbose=True + ) + optimizer = paddle.optimizer.SGD( + scheduler, + grad_clip=grad_clip, + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + } + ], + ) return optimizer def train_batch(self, batch, model, optimizer, is_mp): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_clip_grad.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_clip_grad.py index c9bff4a261d..91d9789bfa0 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_clip_grad.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_clip_grad.py @@ -16,20 +16,19 @@ import paddle from hybrid_parallel_mp_model import TestDistMPTraning import unittest -#log = logging.getLogger("HybridParallel") -#log.setLevel(logging.WARNING) +# log = logging.getLogger("HybridParallel") +# log.setLevel(logging.WARNING) class TestMPClipGrad(TestDistMPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(2.0) - scheduler = paddle.optimizer.lr.ExponentialDecay(learning_rate=0.001, - gamma=0.999, - verbose=True) - optimizer = paddle.optimizer.SGD(scheduler, - grad_clip=grad_clip, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.ExponentialDecay( + learning_rate=0.001, gamma=0.999, verbose=True + ) + optimizer = paddle.optimizer.SGD( + scheduler, grad_clip=grad_clip, parameters=model.parameters() + ) return optimizer diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_fp16.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_fp16.py index 611e85ed4a8..6a28316c1f4 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_fp16.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_fp16.py @@ -19,20 +19,18 @@ import unittest class TestMPFP16(TestDistMPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(1.0) - scheduler = paddle.optimizer.lr.ExponentialDecay(learning_rate=0.001, - gamma=0.999, - verbose=True) - optimizer = paddle.optimizer.SGD(scheduler, - grad_clip=grad_clip, - parameters=model.parameters()) - - model, optimizer = paddle.amp.decorate(models=model, - optimizers=optimizer, - level='O2', - save_dtype='float32') + scheduler = paddle.optimizer.lr.ExponentialDecay( + learning_rate=0.001, gamma=0.999, verbose=True + ) + optimizer = paddle.optimizer.SGD( + scheduler, grad_clip=grad_clip, parameters=model.parameters() + ) + + model, optimizer = paddle.amp.decorate( + models=model, optimizers=optimizer, level='O2', save_dtype='float32' + ) return optimizer diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_layers.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_layers.py index 5f20209da4a..9a391771271 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_layers.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_layers.py @@ -31,7 +31,6 @@ def set_random_seed(seed): class ColumnLinearNet(fluid.dygraph.Layer): - def __init__(self, input_size, output_size, global_dtype): super(ColumnLinearNet, self).__init__() self.parallel_linear = fleet.meta_parallel.ColumnParallelLinear( @@ -40,7 +39,8 @@ class ColumnLinearNet(fluid.dygraph.Layer): weight_attr=None, has_bias=True, gather_output=True, - name="test_column_linear") + name="test_column_linear", + ) def forward(self, x): output = self.parallel_linear(x) @@ -48,7 +48,6 @@ class ColumnLinearNet(fluid.dygraph.Layer): class RowLinearNet(fluid.dygraph.Layer): - def __init__(self, input_size, output_size): super(RowLinearNet, self).__init__() self.parallel_linear = fleet.meta_parallel.RowParallelLinear( @@ -56,7 +55,8 @@ class RowLinearNet(fluid.dygraph.Layer): out_features=output_size, has_bias=True, input_is_parallel=False, - name="test_row_linear") + name="test_row_linear", + ) def forward(self, x): output = self.parallel_linear(x) @@ -64,11 +64,11 @@ class RowLinearNet(fluid.dygraph.Layer): class EmbeddingNet(fluid.dygraph.Layer): - def __init__(self, vocab_size, hidden_size): super(EmbeddingNet, self).__init__() self.embedding = fleet.meta_parallel.VocabParallelEmbedding( - vocab_size, hidden_size) + vocab_size, hidden_size + ) def forward(self, x): output = self.embedding(x) @@ -76,19 +76,22 @@ class EmbeddingNet(fluid.dygraph.Layer): class SimpleMatmul(fluid.dygraph.Layer): - def __init__(self, weight, output_size, global_dtype): super(SimpleMatmul, self).__init__() self.weight = paddle.create_parameter( shape=weight.shape, dtype=global_dtype, attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Assign(weight))) + initializer=paddle.nn.initializer.Assign(weight) + ), + ) self.bias = self.create_parameter( shape=[output_size], dtype=global_dtype, attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) def forward(self, x): output = paddle.matmul(x, self.weight) + self.bias @@ -96,7 +99,6 @@ class SimpleMatmul(fluid.dygraph.Layer): class SimpleEmbedding(fluid.dygraph.Layer): - def __init__(self, vocab_size, hidden_size, weight): super(SimpleEmbedding, self).__init__() self.embedding = paddle.nn.Embedding( @@ -104,7 +106,9 @@ class SimpleEmbedding(fluid.dygraph.Layer): hidden_size, weight_attr=paddle.framework.ParamAttr( name="origin_embedding", - initializer=paddle.nn.initializer.Assign(weight))) + initializer=paddle.nn.initializer.Assign(weight), + ), + ) def forward(self, x): output = self.embedding(x) @@ -112,14 +116,13 @@ class SimpleEmbedding(fluid.dygraph.Layer): class TestDistTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": self.model_parallel_size, - "pp_degree": 1 + "pp_degree": 1, } fleet.init(is_collective=True, strategy=strategy) @@ -144,10 +147,12 @@ class TestDistTraning(unittest.TestCase): model_b = SimpleMatmul(integral_w, output_size, global_dtype) - optimizer_a = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_a.parameters()) - optimizer_b = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_b.parameters()) + optimizer_a = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_a.parameters() + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_b.parameters() + ) for idx in range(5): input = paddle.randn([batch_size, input_size], global_dtype) input.stop_gradient = True @@ -192,11 +197,13 @@ class TestDistTraning(unittest.TestCase): model_b = SimpleMatmul(integral_w, output_size, global_dtype) - optimizer_a = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_a.parameters()) + optimizer_a = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_a.parameters() + ) - optimizer_b = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_b.parameters()) + optimizer_b = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_b.parameters() + ) for idx in range(5): input = paddle.randn([batch_size, input_size], global_dtype) @@ -213,9 +220,9 @@ class TestDistTraning(unittest.TestCase): optimizer_a.step() optimizer_b.step() - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=5e-6) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-6 + ) def test_parallel_embedding(self): batch_size = 17 @@ -240,21 +247,25 @@ class TestDistTraning(unittest.TestCase): for idx in range(len(integral_w)): tmp = paddle.gather( integral_w[idx], - paddle.to_tensor(list(range(vocab_size_per_card)))) + paddle.to_tensor(list(range(vocab_size_per_card))), + ) result_w.append(tmp) integral_w = paddle.concat(result_w, axis=0) model_b = SimpleEmbedding(vocab_size, hidden_size, integral_w) - optimizer_a = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_a.parameters()) + optimizer_a = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_a.parameters() + ) - optimizer_b = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_b.parameters()) + optimizer_b = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_b.parameters() + ) for _ in range(5): - np_input_data = np.random.randint(0, vocab_size, - (batch_size, seq_length)) + np_input_data = np.random.randint( + 0, vocab_size, (batch_size, seq_length) + ) input_data = paddle.to_tensor(np_input_data, dtype="int32") output_a = model_a(input_data) @@ -292,21 +303,23 @@ class TestDistTraning(unittest.TestCase): np.random.seed(seed) for _ in range(5): - np_label = np.random.randint(0, vocab_size, - (batch_size, seq_length)) + np_label = np.random.randint( + 0, vocab_size, (batch_size, seq_length) + ) label = paddle.to_tensor(np_label, dtype="int64") data = paddle.randn( shape=[batch_size, seq_length, class_size_per_card], - dtype='float32') + dtype='float32', + ) data.stop_gradient = False check_group = dist.new_group(list(range(self.model_parallel_size))) integral_data = [] partial_data = data.clone().detach() - paddle.distributed.all_gather(integral_data, - partial_data, - group=check_group) + paddle.distributed.all_gather( + integral_data, partial_data, group=check_group + ) integral_data = paddle.concat(integral_data, axis=-1) integral_data = integral_data.detach().clone() integral_data.stop_gradient = False @@ -315,23 +328,23 @@ class TestDistTraning(unittest.TestCase): loss_b = model_b(integral_data, label).sum() / batch_size print("loss_a: ", loss_a.numpy(), "loss_b: ", loss_b.numpy()) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=1e-6) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=1e-6 + ) loss_a.backward() loss_b.backward() integral_grad = [] partial_grad = data.grad.clone().detach() - paddle.distributed.all_gather(integral_grad, - partial_grad, - group=check_group) + paddle.distributed.all_gather( + integral_grad, partial_grad, group=check_group + ) integral_grad = paddle.concat(integral_grad, axis=-1) - np.testing.assert_allclose(integral_data.grad.numpy(), - integral_grad.numpy(), - rtol=1e-6) + np.testing.assert_allclose( + integral_data.grad.numpy(), integral_grad.numpy(), rtol=1e-6 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py index 707bd95a490..b43d0663ac7 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py @@ -44,7 +44,8 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): if world_size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=model_parallel_group) + lm_output, group=model_parallel_group + ) logits = paddle.matmul(input_parallel, logit_weights, transpose_y=True) @@ -52,53 +53,69 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): return logits return paddle.distributed.collective._c_concat( - logits, group=model_parallel_group) + logits, group=model_parallel_group + ) else: logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits class SimpleMPNet(fluid.dygraph.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2, mp_id): + def __init__( + self, + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ): super(SimpleMPNet, self).__init__() if mp_id == 0: - init_fc1_data = np_fc1[:, :(inner_size // 2)] - init_fc2_data = np_fc2[:(inner_size // 2), :] + init_fc1_data = np_fc1[:, : (inner_size // 2)] + init_fc2_data = np_fc2[: (inner_size // 2), :] else: - init_fc1_data = np_fc1[:, (inner_size // 2):] - init_fc2_data = np_fc2[(inner_size // 2):, :] + init_fc1_data = np_fc1[:, (inner_size // 2) :] + init_fc2_data = np_fc2[(inner_size // 2) :, :] self.linear1 = fleet.meta_parallel.ColumnParallelLinear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc1_data)), + initializer=paddle.nn.initializer.Assign(init_fc1_data) + ), gather_output=False, - has_bias=True) + has_bias=True, + ) self.linear2 = fleet.meta_parallel.RowParallelLinear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc2_data)), + initializer=paddle.nn.initializer.Assign(init_fc2_data) + ), input_is_parallel=True, - has_bias=True) + has_bias=True, + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = fleet.meta_parallel.VocabParallelEmbedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=0.5)) + weight_attr=paddle.nn.initializer.Constant(value=0.5), + ) def forward(self, x): x = self.embedding(x) @@ -110,39 +127,49 @@ class SimpleMPNet(fluid.dygraph.Layer): class SimpleDPNet(fluid.dygraph.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2): + def __init__( + self, vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ): super(SimpleDPNet, self).__init__() self.linear1 = paddle.nn.Linear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc1)), + initializer=paddle.nn.initializer.Assign(np_fc1) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear2 = paddle.nn.Linear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc2)), + initializer=paddle.nn.initializer.Assign(np_fc2) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = paddle.nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=0.5)) + weight_attr=paddle.nn.initializer.Constant(value=0.5), + ) def forward(self, x): x = self.embedding(x) @@ -154,7 +181,6 @@ class SimpleDPNet(fluid.dygraph.Layer): class TestDistMPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 @@ -162,7 +188,7 @@ class TestDistMPTraning(unittest.TestCase): strategy.hybrid_configs = { "dp_degree": self.data_parallel_size, "mp_degree": self.model_parallel_size, - "pp_degree": 1 + "pp_degree": 1, } fleet.init(is_collective=True, strategy=strategy) @@ -175,8 +201,9 @@ class TestDistMPTraning(unittest.TestCase): return loss def build_optimizer(self, model): - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return optimizer def build_model_optimizer(self): @@ -190,34 +217,50 @@ class TestDistMPTraning(unittest.TestCase): np_fc1 = np.random.random_sample((hidden_size, inner_size)) np_fc2 = np.random.random_sample((inner_size, hidden_size)) - model_a = SimpleMPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2, mp_id) + model_a = SimpleMPNet( + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ) optimizer_a = self.build_optimizer(model_a) model_a = fleet.distributed_model(model_a) optimizer_a = fleet.distributed_optimizer(optimizer_a) - model_b = SimpleDPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2) + model_b = SimpleDPNet( + vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ) optimizer_b = self.build_optimizer(model_b) return model_a, optimizer_a, model_b, optimizer_b def test_mp_model(self): - model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( - ) + ( + model_a, + optimizer_a, + model_b, + optimizer_b, + ) = self.build_model_optimizer() for _ in range(5): - np_data = np.random.randint(0, vocab_size, ( - batch_size, - seq_length, - )) + np_data = np.random.randint( + 0, + vocab_size, + ( + batch_size, + seq_length, + ), + ) batch = paddle.to_tensor(np_data) loss_a = self.train_batch(batch, model_a, optimizer_a, True) loss_b = self.train_batch(batch, model_b, optimizer_b, False) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=1e-6) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=1e-6 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_random.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_random.py index f779b3298b8..96e76b37c7a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_random.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_random.py @@ -20,14 +20,13 @@ import paddle.distributed.fleet as fleet class TestDistTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": self.model_parallel_size, - "pp_degree": 1 + "pp_degree": 1, } fleet.init(is_collective=True, strategy=strategy) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_amp.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_amp.py index 41bd79b58b5..af4e68be8f6 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_amp.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_amp.py @@ -34,7 +34,6 @@ micro_batch_size = 2 class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -47,7 +46,7 @@ class TestDistPPTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -61,14 +60,16 @@ class TestDistPPTraning(unittest.TestCase): grad_clip = paddle.nn.ClipGradByGlobalNorm(1.0) - #construct model a + # construct model a model_a = AlexNet(10) - scheduler_a = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, - grad_clip=grad_clip, - parameters=model_a.parameters()) + scheduler_a = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer_a = paddle.optimizer.SGD( + learning_rate=scheduler_a, + grad_clip=grad_clip, + parameters=model_a.parameters(), + ) scaler_a = paddle.amp.GradScaler(init_loss_scaling=2**5) @@ -79,12 +80,14 @@ class TestDistPPTraning(unittest.TestCase): # construct model b model_b = AlexNetPipeDesc(num_stages=self.pipeline_parallel_size) - scheduler_b = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, - grad_clip=grad_clip, - parameters=model_b.parameters()) + scheduler_b = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=scheduler_b, + grad_clip=grad_clip, + parameters=model_b.parameters(), + ) model_b = fleet.distributed_model(model_b) optimizer_b = fleet.distributed_optimizer(optimizer_b) scaler_b = paddle.amp.GradScaler(init_loss_scaling=2**5) @@ -94,15 +97,21 @@ class TestDistPPTraning(unittest.TestCase): param.set_value(parameters[idx + pp_id * (param_len // 2)]) # construct reader - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size, drop_last=True + ) for step_id, data in enumerate(train_reader()): - x_data = np.array([x[0] for x in data]).astype('float32').reshape( - batch_size, 1, 28, 28) - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + x_data = ( + np.array([x[0] for x in data]) + .astype('float32') + .reshape(batch_size, 1, 28, 28) + ) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) img.stop_gradient = True @@ -120,15 +129,14 @@ class TestDistPPTraning(unittest.TestCase): scheduler_a.step() with paddle.amp.auto_cast(): - loss_b = model_b.train_batch([img, label], - optimizer_b, - scheduler_b, - scaler=scaler_b) + loss_b = model_b.train_batch( + [img, label], optimizer_b, scheduler_b, scaler=scaler_b + ) print("loss: ", loss_a.numpy(), loss_b.numpy()) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=5e-5) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_clip_grad.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_clip_grad.py index 8f1d8e930e2..379a5f6417a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_clip_grad.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_clip_grad.py @@ -18,31 +18,30 @@ from hybrid_parallel_pp_alexnet import TestDistPPTraning class TestPPClipGrad(TestDistPPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(0.5) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - grad_clip=grad_clip, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, + grad_clip=grad_clip, + parameters=model.parameters(), + ) return scheduler, optimizer class TestPPClipGradParamGroup(TestDistPPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(0.5) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.Momentum(learning_rate=scheduler, - grad_clip=grad_clip, - parameters=[{ - "params": - model.parameters() - }]) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.Momentum( + learning_rate=scheduler, + grad_clip=grad_clip, + parameters=[{"params": model.parameters()}], + ) return scheduler, optimizer diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py index d55f989054c..1c14eb7f67c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py @@ -40,29 +40,29 @@ hidden_size = 8 class SimpleNet(Layer): - def __init__(self): super(SimpleNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) self.softmax_weight = self.create_parameter( - shape=[hidden_size, vocab_size]) - self.softmax_bias = self.create_parameter(shape=[vocab_size], - is_bias=False) + shape=[hidden_size, vocab_size] + ) + self.softmax_bias = self.create_parameter( + shape=[vocab_size], is_bias=False + ) def forward(self, x1, x2, y1): x_emb = self.word_embeddings(x1) fc = fluid.layers.matmul(x_emb, self.softmax_weight) fc = fluid.layers.elementwise_add(fc, self.softmax_bias) projection = fluid.layers.reshape(fc, shape=[-1, vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=y1, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1, soft_label=False + ) return loss.mean() class EmbeddingNet(Layer): - def __init__(self): super(EmbeddingNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) @@ -74,11 +74,11 @@ class EmbeddingNet(Layer): class MatmulNet(Layer): - def __init__(self): super(MatmulNet, self).__init__() self.softmax_weight = self.create_parameter( - shape=[hidden_size, vocab_size]) + shape=[hidden_size, vocab_size] + ) def forward(self, args): x1, x2 = args @@ -88,7 +88,6 @@ class MatmulNet(Layer): class BiasNet(Layer): - def __init__(self): super(BiasNet, self).__init__() self.softmax_bias = self.create_parameter(shape=[vocab_size]) @@ -101,20 +100,18 @@ class BiasNet(Layer): class LossNet(Layer): - def __init__(self): super(LossNet, self).__init__() def forward(self, args, y1): projection, x2 = args - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=y1[0], - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1[0], soft_label=False + ) return loss.mean() class SimpleNetPipe(Layer): - def __init__(self): super(SimpleNetPipe, self).__init__() self.features = Sequential(EmbeddingNet(), MatmulNet(), BiasNet()) @@ -125,7 +122,6 @@ class SimpleNetPipe(Layer): class TestDistEmbeddingTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -138,7 +134,7 @@ class TestDistEmbeddingTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -150,22 +146,28 @@ class TestDistEmbeddingTraning(unittest.TestCase): rank_id = dist.get_rank() set_random_seed(1024, dp_id, rank_id) - #construct model a + # construct model a model_a = SimpleNet() scheduler_a = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) - optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, - parameters=model_a.parameters()) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True + ) + optimizer_a = paddle.optimizer.SGD( + learning_rate=scheduler_a, parameters=model_a.parameters() + ) init_net = SimpleNetPipe() - model_b = PipelineLayer(layers=init_net.to_layers(), - num_stages=self.pipeline_parallel_size, - loss_fn=LossNet()) + model_b = PipelineLayer( + layers=init_net.to_layers(), + num_stages=self.pipeline_parallel_size, + loss_fn=LossNet(), + ) scheduler_b = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) - optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, - parameters=model_b.parameters()) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=scheduler_b, parameters=model_b.parameters() + ) model_b = fleet.distributed_model(model_b) optimizer_b = fleet.distributed_optimizer(optimizer_b) @@ -202,8 +204,9 @@ class TestDistEmbeddingTraning(unittest.TestCase): optimizer_a.clear_grad() scheduler_a.step() - loss_b = model_b.train_batch([(x1, x2), (y1, )], optimizer_b, - scheduler_b) + loss_b = model_b.train_batch( + [(x1, x2), (y1,)], optimizer_b, scheduler_b + ) print("loss", loss_a.numpy(), loss_b.numpy()) np.testing.assert_allclose(loss_a.numpy(), loss_b.numpy()) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_fp16.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_fp16.py index 06813a340eb..b0429167036 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_fp16.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_fp16.py @@ -34,7 +34,6 @@ micro_batch_size = 2 class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -47,7 +46,7 @@ class TestDistPPTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -61,25 +60,29 @@ class TestDistPPTraning(unittest.TestCase): grad_clip = paddle.nn.ClipGradByGlobalNorm(1.0) - #construct model a + # construct model a model_a = AlexNet(10) - scheduler_a = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, - grad_clip=grad_clip, - parameters=model_a.parameters()) + scheduler_a = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer_a = paddle.optimizer.SGD( + learning_rate=scheduler_a, + grad_clip=grad_clip, + parameters=model_a.parameters(), + ) scaler_a = paddle.amp.GradScaler(init_loss_scaling=2**5) # construct model b model_b = AlexNetPipeDesc(num_stages=self.pipeline_parallel_size) - scheduler_b = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, - grad_clip=grad_clip, - parameters=model_b.parameters()) + scheduler_b = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=scheduler_b, + grad_clip=grad_clip, + parameters=model_b.parameters(), + ) param_len = len(model_a.parameters()) parameters = [] @@ -89,14 +92,18 @@ class TestDistPPTraning(unittest.TestCase): for idx, param in enumerate(model_b.parameters()): param.set_value(parameters[idx + pp_id * (param_len // 2)]) - model_a, optimizer_a = paddle.amp.decorate(models=model_a, - optimizers=optimizer_a, - level='O2', - save_dtype='float32') - model_b, optimizer_b = paddle.amp.decorate(models=model_b, - optimizers=optimizer_b, - level='O2', - save_dtype='float32') + model_a, optimizer_a = paddle.amp.decorate( + models=model_a, + optimizers=optimizer_a, + level='O2', + save_dtype='float32', + ) + model_b, optimizer_b = paddle.amp.decorate( + models=model_b, + optimizers=optimizer_b, + level='O2', + save_dtype='float32', + ) model_b = fleet.distributed_model(model_b) optimizer_b = fleet.distributed_optimizer(optimizer_b) @@ -104,15 +111,21 @@ class TestDistPPTraning(unittest.TestCase): scaler_b = fleet.distributed_scaler(scaler_b) # construct reader - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size, drop_last=True + ) for step_id, data in enumerate(train_reader()): - x_data = np.array([x[0] for x in data]).astype('float32').reshape( - batch_size, 1, 28, 28) - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + x_data = ( + np.array([x[0] for x in data]) + .astype('float32') + .reshape(batch_size, 1, 28, 28) + ) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) img.stop_gradient = True @@ -129,15 +142,14 @@ class TestDistPPTraning(unittest.TestCase): scheduler_a.step() with paddle.amp.auto_cast(enable=True, level='O2'): - loss_b = model_b.train_batch([img, label], - optimizer_b, - scheduler_b, - scaler=scaler_b) + loss_b = model_b.train_batch( + [img, label], optimizer_b, scheduler_b, scaler=scaler_b + ) print("loss: ", loss_a.numpy(), loss_b.numpy()) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=5e-3) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-3 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_layer_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_layer_with_virtual_stage.py index 73807abdc18..f23df47ae4b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_layer_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_layer_with_virtual_stage.py @@ -17,12 +17,15 @@ import paddle from paddle.distributed import fleet import paddle.nn as nn from paddle.fluid.dygraph.layers import Layer -from paddle.distributed.fleet.meta_parallel import LayerDesc, PipelineLayer, PipelineParallelWithInterleave +from paddle.distributed.fleet.meta_parallel import ( + LayerDesc, + PipelineLayer, + PipelineParallelWithInterleave, +) import paddle.nn.functional as F class ReshapeHelp(Layer): - def __init__(self, shape): super(ReshapeHelp, self).__init__() self.shape = shape @@ -32,7 +35,6 @@ class ReshapeHelp(Layer): class MLPForVirtualStageLayerTest(PipelineLayer): - def __init__(self, num_classes=10, **kwargs): self.num_classes = num_classes decs = [ @@ -45,21 +47,19 @@ class MLPForVirtualStageLayerTest(PipelineLayer): LayerDesc(nn.Linear, 2, self.num_classes), LayerDesc(nn.Linear, self.num_classes, 2), ] - super(MLPForVirtualStageLayerTest, - self).__init__(layers=decs, - loss_fn=nn.CrossEntropyLoss(), - **kwargs) + super(MLPForVirtualStageLayerTest, self).__init__( + layers=decs, loss_fn=nn.CrossEntropyLoss(), **kwargs + ) class TestPipeLayerAPI(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.pipeline_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": self.pipeline_parallel_size + "pp_degree": self.pipeline_parallel_size, } fleet.init(is_collective=True, strategy=strategy) self.rank = fleet.worker_index() @@ -74,8 +74,9 @@ class TestPipeLayerAPI(unittest.TestCase): recompute_ctx={ "mp_group": self.hcg.get_model_parallel_group(), "offload": False, - "partition": False - }) + "partition": False, + }, + ) assert len(pipe_model.parameters()) > 0 model_chunks = pipe_model.get_model_chunks() assert model_chunks is not None @@ -84,14 +85,14 @@ class TestPipeLayerAPI(unittest.TestCase): optimizer = paddle.optimizer.SGD(parameters=pipe_model.parameters()) try: - model_chunks[0](paddle.to_tensor([1., 2.])) + model_chunks[0](paddle.to_tensor([1.0, 2.0])) raise NotImplementedError except PermissionError: pass # fake call for the forward function of virtual pipeline layer for i in range(len(model_chunks)): - out = pipe_model(paddle.to_tensor([1., 2.]), chunk_id=i) + out = pipe_model(paddle.to_tensor([1.0, 2.0]), chunk_id=i) assert list(out.shape) == [2] out = F.relu(out) loss = paddle.mean(out) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_recompute.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_recompute.py index fde0572d3d8..e9443b511e0 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_recompute.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_recompute.py @@ -42,7 +42,6 @@ dim_feedforward = 4 * d_model class EmbeddingNet(Layer): - def __init__(self): super(EmbeddingNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) @@ -56,7 +55,6 @@ class EmbeddingNet(Layer): class TransformerNet(Layer): - def __init__(self): super(TransformerNet, self).__init__() self.linear1 = nn.Linear(d_model, dim_feedforward) @@ -72,7 +70,9 @@ class TransformerNet(Layer): q = self.q_proj(x) k = self.k_proj(x) v = self.v_proj(x) - product = layers.matmul(x=q, y=k, transpose_y=True, alpha=d_model**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=d_model**-0.5 + ) weights = F.softmax(product) weights = F.dropout(weights, 0.2) @@ -86,20 +86,17 @@ class TransformerNet(Layer): class EmbeddingPipe(EmbeddingNet): - def forward(self, x): return super().forward(x) class TransformerNetPipe(TransformerNet): - def forward(self, x): output = super().forward(x) return output class CriterionPipe(Layer): - def __init__(self): super(CriterionPipe, self).__init__() @@ -109,7 +106,6 @@ class CriterionPipe(Layer): class ModelPipe(PipelineLayer): - def __init__(self, hcg): self.descs = [] self.descs.append(LayerDesc(EmbeddingPipe)) @@ -118,20 +114,21 @@ class ModelPipe(PipelineLayer): for x in range(2): self.descs.append(LayerDesc(TransformerNetPipe)) - super().__init__(layers=self.descs, - loss_fn=CriterionPipe(), - topology=self.hcg.topology(), - seg_method="layer:TransformerNetPipe", - recompute_interval=1, - recompute_ctx={ - "mp_group": self.hcg.get_model_parallel_group(), - "offload": False, - "partition": False - }) + super().__init__( + layers=self.descs, + loss_fn=CriterionPipe(), + topology=self.hcg.topology(), + seg_method="layer:TransformerNetPipe", + recompute_interval=1, + recompute_ctx={ + "mp_group": self.hcg.get_model_parallel_group(), + "offload": False, + "partition": False, + }, + ) class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -144,7 +141,7 @@ class TestDistPPTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -158,11 +155,12 @@ class TestDistPPTraning(unittest.TestCase): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(hcg) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load.py index 5d52093a86f..b7990ff0237 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load.py @@ -29,7 +29,6 @@ vocab_size = 128 class TestDistPPSaveLoadTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -42,7 +41,7 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -56,11 +55,12 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(topology) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) @@ -74,14 +74,16 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): loss = model.train_batch([x, x], optimizer, scheduler) model._layers.save_state_dict(output_dir) - paddle.save(optimizer.state_dict(), - os.path.join(output_dir, "model_state.pdopt")) + paddle.save( + optimizer.state_dict(), + os.path.join(output_dir, "model_state.pdopt"), + ) # construct data test_steps = 5 - np_data = np.random.randint(0, - vocab_size, - size=[test_steps, batch_size, length]) + np_data = np.random.randint( + 0, vocab_size, size=[test_steps, batch_size, length] + ) origin_loss = [] for step_id in range(5): @@ -101,8 +103,12 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): x = paddle.to_tensor(x_data) x.stop_gradient = True loss = model.train_batch([x, x], optimizer, scheduler) - print("origin loss: ", origin_loss[step_id], "current loss: ", - loss.numpy()) + print( + "origin loss: ", + origin_loss[step_id], + "current loss: ", + loss.numpy(), + ) np.testing.assert_allclose(loss.numpy(), origin_loss[step_id]) # finally, remove the model/optimizer path diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load_with_virtual_stage.py index f71e2511a20..dfe4ecae403 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load_with_virtual_stage.py @@ -20,7 +20,10 @@ import shutil import tempfile import paddle.distributed as dist import paddle.distributed.fleet as fleet -from hybrid_parallel_pp_transformer_with_virtual_stage import ModelPipe, set_random_seed +from hybrid_parallel_pp_transformer_with_virtual_stage import ( + ModelPipe, + set_random_seed, +) batch_size = 8 length = 8 @@ -29,7 +32,6 @@ vocab_size = 128 class TestDistPPSaveLoadTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -42,7 +44,7 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -56,11 +58,12 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(topology) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) @@ -74,14 +77,16 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): loss = model.train_batch([x, x], optimizer, scheduler) model._layers.save_state_dict(output_dir) - paddle.save(optimizer.state_dict(), - os.path.join(output_dir, "model_state.pdopt")) + paddle.save( + optimizer.state_dict(), + os.path.join(output_dir, "model_state.pdopt"), + ) # construct data test_steps = 5 - np_data = np.random.randint(0, - vocab_size, - size=[test_steps, batch_size, length]) + np_data = np.random.randint( + 0, vocab_size, size=[test_steps, batch_size, length] + ) origin_loss = [] for step_id in range(5): @@ -101,8 +106,12 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): x = paddle.to_tensor(x_data) x.stop_gradient = True loss = model.train_batch([x, x], optimizer, scheduler) - print("origin loss: ", origin_loss[step_id], "current loss: ", - loss.numpy()) + print( + "origin loss: ", + origin_loss[step_id], + "current loss: ", + loss.numpy(), + ) np.testing.assert_allclose(loss.numpy(), origin_loss[step_id]) # finally, remove the model/optimizer path diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py index de0475018d2..e1f4aa1762e 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py @@ -42,15 +42,15 @@ dim_feedforward = 4 * d_model class EmbeddingNet(Layer): - def __init__(self): super(EmbeddingNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) self.position_embeddings = nn.Embedding(vocab_size, hidden_size) def forward(self, x): - attention_mask = paddle.tensor.triu((paddle.ones( - (length, length), dtype="float32") * -1e9), 1) + attention_mask = paddle.tensor.triu( + (paddle.ones((length, length), dtype="float32") * -1e9), 1 + ) no_used = paddle.ones((3, 3), dtype="int32") @@ -65,7 +65,6 @@ class EmbeddingNet(Layer): class TransformerNet(Layer): - def __init__(self): super(TransformerNet, self).__init__() self.linear1 = nn.Linear(d_model, dim_feedforward) @@ -81,7 +80,9 @@ class TransformerNet(Layer): q = self.q_proj(x) k = self.k_proj(x) v = self.v_proj(x) - product = layers.matmul(x=q, y=k, transpose_y=True, alpha=d_model**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=d_model**-0.5 + ) weights = F.softmax(product + mask) # TODO(shenliang03) For save/load in PipeLineParallel, can’t support dropout temporarily. @@ -96,13 +97,11 @@ class TransformerNet(Layer): class EmbeddingPipe(EmbeddingNet): - def forward(self, x): return super().forward(x) class TransformerNetPipe(TransformerNet): - def forward(self, args): x, mask, no_used, p_emb = args[0], args[1], args[2], args[3] @@ -113,7 +112,6 @@ class TransformerNetPipe(TransformerNet): class CriterionPipe(Layer): - def __init__(self): super(CriterionPipe, self).__init__() @@ -123,7 +121,6 @@ class CriterionPipe(Layer): class ModelPipe(PipelineLayer): - def __init__(self, topology): self.descs = [] self.descs.append(LayerDesc(EmbeddingPipe)) @@ -133,14 +130,15 @@ class ModelPipe(PipelineLayer): self.descs.append(lambda x: x[0]) - super().__init__(layers=self.descs, - loss_fn=CriterionPipe(), - topology=topology, - seg_method="layer:TransformerNetPipe") + super().__init__( + layers=self.descs, + loss_fn=CriterionPipe(), + topology=topology, + seg_method="layer:TransformerNetPipe", + ) class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -153,7 +151,7 @@ class TestDistPPTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -167,11 +165,12 @@ class TestDistPPTraning(unittest.TestCase): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(topology) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py index edad5e5bc99..89e0410e2ca 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py @@ -43,15 +43,15 @@ dim_feedforward = 4 * d_model class EmbeddingNet(Layer): - def __init__(self): super(EmbeddingNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) self.position_embeddings = nn.Embedding(vocab_size, hidden_size) def forward(self, x): - attention_mask = paddle.tensor.triu((paddle.ones( - (length, length), dtype="float32") * -1e9), 1) + attention_mask = paddle.tensor.triu( + (paddle.ones((length, length), dtype="float32") * -1e9), 1 + ) no_used = paddle.ones((3, 3), dtype="int32") @@ -66,7 +66,6 @@ class EmbeddingNet(Layer): class TransformerNet(Layer): - def __init__(self): super(TransformerNet, self).__init__() self.linear1 = nn.Linear(d_model, dim_feedforward) @@ -82,7 +81,9 @@ class TransformerNet(Layer): q = self.q_proj(x) k = self.k_proj(x) v = self.v_proj(x) - product = layers.matmul(x=q, y=k, transpose_y=True, alpha=d_model**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=d_model**-0.5 + ) weights = F.softmax(product + mask) tgt = layers.matmul(weights, v) @@ -95,13 +96,11 @@ class TransformerNet(Layer): class EmbeddingPipe(EmbeddingNet): - def forward(self, x): return super().forward(x) class TransformerNetPipe(TransformerNet): - def forward(self, args): x, mask, no_used, p_emb = args[0], args[1], args[2], args[3] @@ -112,7 +111,6 @@ class TransformerNetPipe(TransformerNet): class CriterionPipe(Layer): - def __init__(self): super(CriterionPipe, self).__init__() @@ -122,7 +120,6 @@ class CriterionPipe(Layer): class ModelPipe(PipelineLayer): - def __init__(self, topology): self.descs = [] self.descs.append(LayerDesc(EmbeddingPipe)) @@ -137,11 +134,11 @@ class ModelPipe(PipelineLayer): loss_fn=CriterionPipe(), topology=topology, num_virtual_pipeline_stages=num_virtual_pipeline_stages, - seg_method="layer:TransformerNetPipe") + seg_method="layer:TransformerNetPipe", + ) class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -154,7 +151,7 @@ class TestDistPPTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -168,11 +165,12 @@ class TestDistPPTraning(unittest.TestCase): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(topology) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_qat.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_qat.py index 03e35799349..80e7b8b4e69 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_qat.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_qat.py @@ -79,7 +79,8 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): if world_size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=model_parallel_group) + lm_output, group=model_parallel_group + ) logits = paddle.matmul(input_parallel, logit_weights, transpose_y=True) @@ -87,22 +88,23 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): return logits return paddle.distributed.collective._c_concat( - logits, group=model_parallel_group) + logits, group=model_parallel_group + ) else: logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits class PACT(nn.Layer): - def __init__(self, init_value=20): super(PACT, self).__init__() alpha_attr = paddle.ParamAttr( name=self.full_name() + ".pact", - initializer=paddle.nn.initializer.Constant(value=init_value)) - self.alpha = self.create_parameter(shape=[1], - attr=alpha_attr, - dtype='float32') + initializer=paddle.nn.initializer.Constant(value=init_value), + ) + self.alpha = self.create_parameter( + shape=[1], attr=alpha_attr, dtype='float32' + ) def forward(self, x): out_left = paddle.nn.functional.relu(x - self.alpha) @@ -112,46 +114,61 @@ class PACT(nn.Layer): class SimpleMPNet(nn.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2, mp_id): + def __init__( + self, + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ): super(SimpleMPNet, self).__init__() if mp_id == 0: - init_fc1_data = np_fc1[:, :(inner_size // 2)] - init_fc2_data = np_fc2[:(inner_size // 2), :] + init_fc1_data = np_fc1[:, : (inner_size // 2)] + init_fc2_data = np_fc2[: (inner_size // 2), :] else: - init_fc1_data = np_fc1[:, (inner_size // 2):] - init_fc2_data = np_fc2[(inner_size // 2):, :] + init_fc1_data = np_fc1[:, (inner_size // 2) :] + init_fc2_data = np_fc2[(inner_size // 2) :, :] self.linear1 = fleet.meta_parallel.ColumnParallelLinear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc1_data)), + initializer=paddle.nn.initializer.Assign(init_fc1_data) + ), gather_output=False, - has_bias=True) + has_bias=True, + ) self.linear2 = fleet.meta_parallel.RowParallelLinear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc2_data)), + initializer=paddle.nn.initializer.Assign(init_fc2_data) + ), input_is_parallel=True, - has_bias=True) + has_bias=True, + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = fleet.meta_parallel.VocabParallelEmbedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=1.)) + weight_attr=paddle.nn.initializer.Constant(value=1.0), + ) def forward(self, x): x = self.embedding(x) @@ -163,53 +180,62 @@ class SimpleMPNet(nn.Layer): class SimpleDPNet(nn.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2): + def __init__( + self, vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ): super(SimpleDPNet, self).__init__() self.linear1 = paddle.nn.Linear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc1)), + initializer=paddle.nn.initializer.Assign(np_fc1) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear2 = paddle.nn.Linear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc2)), + initializer=paddle.nn.initializer.Assign(np_fc2) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = paddle.nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=1.)) + weight_attr=paddle.nn.initializer.Constant(value=1.0), + ) def forward(self, x): x = self.embedding(x) x = self.linear1(x) x = self.linear2(x) x = self.linear3(x) - x = paddle.matmul(x, - get_attr(self.embedding, "weight"), - transpose_y=True) + x = paddle.matmul( + x, get_attr(self.embedding, "weight"), transpose_y=True + ) return x class TestDistMPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 @@ -217,7 +243,7 @@ class TestDistMPTraning(unittest.TestCase): strategy.hybrid_configs = { "dp_degree": self.data_parallel_size, "mp_degree": self.model_parallel_size, - "pp_degree": 1 + "pp_degree": 1, } fleet.init(is_collective=True, strategy=strategy) self.onnx_format = False @@ -234,14 +260,14 @@ class TestDistMPTraning(unittest.TestCase): return loss def build_optimizer(self, model): - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return optimizer - def build_model_optimizer(self, - weight_quantize_type, - activation_quantize_type, - use_pact=False): + def build_model_optimizer( + self, weight_quantize_type, activation_quantize_type, use_pact=False + ): hcg = fleet.get_hybrid_communicate_group() word_size = hcg.get_model_parallel_world_size() mp_id = hcg.get_model_parallel_rank() @@ -251,23 +277,32 @@ class TestDistMPTraning(unittest.TestCase): weight_quantize_type=weight_quantize_type, activation_quantize_type=activation_quantize_type, fuse_conv_bn=self.fuse_conv_bn, - act_preprocess_layer=PACT if use_pact else None) + act_preprocess_layer=PACT if use_pact else None, + ) set_random_seed(1024, dp_id, rank_id) np_fc1 = np.ones((hidden_size, inner_size)) np_fc2 = np.ones( - (inner_size, - hidden_size)) #np.random.random_sample((inner_size, hidden_size)) + (inner_size, hidden_size) + ) # np.random.random_sample((inner_size, hidden_size)) - model_a = SimpleMPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2, mp_id) + model_a = SimpleMPNet( + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ) model_a = imperative_qat.quantize(model_a) optimizer_a = self.build_optimizer(model_a) model_a = fleet.distributed_model(model_a) optimizer_a = fleet.distributed_optimizer(optimizer_a) - model_b = SimpleDPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2) + model_b = SimpleDPNet( + vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ) model_b = imperative_qat.quantize(model_b) optimizer_b = self.build_optimizer(model_b) @@ -277,21 +312,27 @@ class TestDistMPTraning(unittest.TestCase): for epoch in range(5): - np_data = np.random.randint(0, vocab_size, ( - batch_size, - seq_length, - )) + np_data = np.random.randint( + 0, + vocab_size, + ( + batch_size, + seq_length, + ), + ) batch = paddle.to_tensor(np_data) loss_a = self.train_batch(batch, model_a, optimizer_a, True) loss_b = self.train_batch(batch, model_b, optimizer_b, False) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=1e-6) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=1e-6 + ) def test_mp_model_1(self): - if not fluid.core.is_compiled_with_cuda( - ) or fluid.core.get_cuda_device_count() == 0: + if ( + not fluid.core.is_compiled_with_cuda() + or fluid.core.get_cuda_device_count() == 0 + ): return selected_gpus = get_gpus('0,1') cluster = None @@ -299,12 +340,15 @@ class TestDistMPTraning(unittest.TestCase): model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( weight_quantize_type='abs_max', - activation_quantize_type='moving_average_abs_max') + activation_quantize_type='moving_average_abs_max', + ) self.train(model_a, optimizer_a, model_b, optimizer_b) def test_mp_model_2(self): - if not fluid.core.is_compiled_with_cuda( - ) or fluid.core.get_cuda_device_count() == 0: + if ( + not fluid.core.is_compiled_with_cuda() + or fluid.core.get_cuda_device_count() == 0 + ): return selected_gpus = get_gpus('0,1') cluster = None @@ -313,7 +357,8 @@ class TestDistMPTraning(unittest.TestCase): model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( weight_quantize_type='channel_wise_abs_max', activation_quantize_type='moving_average_abs_max', - use_pact=True) + use_pact=True, + ) self.train(model_a, optimizer_a, model_b, optimizer_b) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_sharding_model.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_sharding_model.py index 96a7825dc80..147ba1e494c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_sharding_model.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_sharding_model.py @@ -18,7 +18,9 @@ import random import paddle.distributed as dist import paddle.fluid as fluid import paddle.distributed.fleet as fleet -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.dygraph_sharding_optimizer import DygraphShardingOptimizer +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.dygraph_sharding_optimizer import ( + DygraphShardingOptimizer, +) import unittest vocab_size = 20 @@ -38,7 +40,8 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): if world_size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=model_parallel_group) + lm_output, group=model_parallel_group + ) logits = paddle.matmul(input_parallel, logit_weights, transpose_y=True) @@ -46,53 +49,69 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): return logits return paddle.distributed.collective._c_concat( - logits, group=model_parallel_group) + logits, group=model_parallel_group + ) else: logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits class SimpleMPNet(fluid.dygraph.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2, mp_id): + def __init__( + self, + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ): super(SimpleMPNet, self).__init__() if mp_id == 0: - init_fc1_data = np_fc1[:, :(inner_size // 2)] - init_fc2_data = np_fc2[:(inner_size // 2), :] + init_fc1_data = np_fc1[:, : (inner_size // 2)] + init_fc2_data = np_fc2[: (inner_size // 2), :] else: - init_fc1_data = np_fc1[:, (inner_size // 2):] - init_fc2_data = np_fc2[(inner_size // 2):, :] + init_fc1_data = np_fc1[:, (inner_size // 2) :] + init_fc2_data = np_fc2[(inner_size // 2) :, :] self.linear1 = fleet.meta_parallel.ColumnParallelLinear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc1_data)), + initializer=paddle.nn.initializer.Assign(init_fc1_data) + ), gather_output=False, - has_bias=True) + has_bias=True, + ) self.linear2 = fleet.meta_parallel.RowParallelLinear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc2_data)), + initializer=paddle.nn.initializer.Assign(init_fc2_data) + ), input_is_parallel=True, - has_bias=True) + has_bias=True, + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = fleet.meta_parallel.VocabParallelEmbedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=0.5)) + weight_attr=paddle.nn.initializer.Constant(value=0.5), + ) def forward(self, x): x = self.embedding(x) @@ -104,39 +123,49 @@ class SimpleMPNet(fluid.dygraph.Layer): class SimpleDPNet(fluid.dygraph.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2): + def __init__( + self, vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ): super(SimpleDPNet, self).__init__() self.linear1 = paddle.nn.Linear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc1)), + initializer=paddle.nn.initializer.Assign(np_fc1) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear2 = paddle.nn.Linear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc2)), + initializer=paddle.nn.initializer.Assign(np_fc2) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = paddle.nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=0.5)) + weight_attr=paddle.nn.initializer.Constant(value=0.5), + ) def forward(self, x): x = self.embedding(x) @@ -148,7 +177,6 @@ class SimpleDPNet(fluid.dygraph.Layer): class TestDistMPTraning(unittest.TestCase): - def setUp(self): random.seed(2021) np.random.seed(2021) @@ -163,10 +191,15 @@ class TestDistMPTraning(unittest.TestCase): } fleet.init(is_collective=True, strategy=self.strategy) self.data = [ - np.random.randint(0, vocab_size, ( - batch_size, - seq_length, - )) for _ in range(STEPS) + np.random.randint( + 0, + vocab_size, + ( + batch_size, + seq_length, + ), + ) + for _ in range(STEPS) ] def train_batch(self, batch, model, optimizer): @@ -178,11 +211,9 @@ class TestDistMPTraning(unittest.TestCase): optimizer.clear_grad() return loss - def build_optimizer(self, - model, - strategy=None, - is_sharding=True, - Optimizer="adam"): + def build_optimizer( + self, model, strategy=None, is_sharding=True, Optimizer="adam" + ): clip = paddle.nn.ClipGradByGlobalNorm(0.5) if Optimizer == "adam": if is_sharding: @@ -193,13 +224,15 @@ class TestDistMPTraning(unittest.TestCase): inner_optimizer_class=paddle.optimizer.AdamW, learning_rate=0.001, weight_decay=0.00001, - grad_clip=clip) + grad_clip=clip, + ) else: optimizer = paddle.optimizer.AdamW( parameters=model.parameters(), learning_rate=0.001, weight_decay=0.00001, - grad_clip=clip) + grad_clip=clip, + ) else: if is_sharding: optimizer = DygraphShardingOptimizer( @@ -208,12 +241,14 @@ class TestDistMPTraning(unittest.TestCase): params=model.parameters(), inner_optimizer_class=paddle.optimizer.Momentum, learning_rate=0.001, - grad_clip=clip) + grad_clip=clip, + ) else: optimizer = paddle.optimizer.Momentum( learning_rate=0.001, parameters=model.parameters(), - grad_clip=clip) + grad_clip=clip, + ) return optimizer def build_model_optimizer(self, Optimizer="adam"): @@ -226,37 +261,48 @@ class TestDistMPTraning(unittest.TestCase): np_fc1 = np.random.random_sample((hidden_size, inner_size)) np_fc2 = np.random.random_sample((inner_size, hidden_size)) - model_a = SimpleDPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2) - optimizer_a = self.build_optimizer(model_a, - strategy=self.strategy, - is_sharding=True, - Optimizer=Optimizer) + model_a = SimpleDPNet( + vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ) + optimizer_a = self.build_optimizer( + model_a, + strategy=self.strategy, + is_sharding=True, + Optimizer=Optimizer, + ) model_a = fleet.distributed_model(model_a) optimizer_a = fleet.distributed_optimizer(optimizer_a) - model_b = SimpleDPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2) - optimizer_b = self.build_optimizer(model_b, - strategy=self.strategy, - is_sharding=False, - Optimizer=Optimizer) + model_b = SimpleDPNet( + vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ) + optimizer_b = self.build_optimizer( + model_b, + strategy=self.strategy, + is_sharding=False, + Optimizer=Optimizer, + ) return model_a, optimizer_a, model_b, optimizer_b def sharding_model(self, Optimizer, sharded_accumulators): model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( - Optimizer=Optimizer) + Optimizer=Optimizer + ) self.assertTrue( - isinstance(optimizer_a._inner_opt, DygraphShardingOptimizer)) + isinstance(optimizer_a._inner_opt, DygraphShardingOptimizer) + ) for idx in range(STEPS): if idx == 2 and paddle.distributed.get_rank() == 0: self.assertTrue( - set(optimizer_a._inner_opt._inner_optimizer.state_dict(). - keys()) == sharded_accumulators) + set( + optimizer_a._inner_opt._inner_optimizer.state_dict().keys() + ) + == sharded_accumulators + ) if paddle.distributed.get_rank() == 0: batch_sharding = paddle.to_tensor(self.data[idx][:2]) @@ -268,31 +314,49 @@ class TestDistMPTraning(unittest.TestCase): loss_b = self.train_batch(batch_single, model_b, optimizer_b) for j in range(len(model_a.parameters())): - np.testing.assert_allclose(model_a.parameters()[j].numpy(), - model_b.parameters()[j].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + model_a.parameters()[j].numpy(), + model_b.parameters()[j].numpy(), + rtol=1e-6, + ) def test_sharding_adam(self): - sharded_accumulators = set([ - 'linear_0.w_0_moment1_0', 'linear_1.b_0_moment1_0', - 'linear_2.b_0_moment1_0', 'embedding_0.w_0_moment1_0', - 'linear_0.w_0_moment2_0', 'linear_1.b_0_moment2_0', - 'linear_2.b_0_moment2_0', 'embedding_0.w_0_moment2_0', - 'linear_0.w_0_beta1_pow_acc_0', 'linear_1.b_0_beta1_pow_acc_0', - 'linear_2.b_0_beta1_pow_acc_0', 'embedding_0.w_0_beta1_pow_acc_0', - 'linear_0.w_0_beta2_pow_acc_0', 'linear_1.b_0_beta2_pow_acc_0', - 'linear_2.b_0_beta2_pow_acc_0', 'embedding_0.w_0_beta2_pow_acc_0' - ]) - self.sharding_model(Optimizer="adam", - sharded_accumulators=sharded_accumulators) + sharded_accumulators = set( + [ + 'linear_0.w_0_moment1_0', + 'linear_1.b_0_moment1_0', + 'linear_2.b_0_moment1_0', + 'embedding_0.w_0_moment1_0', + 'linear_0.w_0_moment2_0', + 'linear_1.b_0_moment2_0', + 'linear_2.b_0_moment2_0', + 'embedding_0.w_0_moment2_0', + 'linear_0.w_0_beta1_pow_acc_0', + 'linear_1.b_0_beta1_pow_acc_0', + 'linear_2.b_0_beta1_pow_acc_0', + 'embedding_0.w_0_beta1_pow_acc_0', + 'linear_0.w_0_beta2_pow_acc_0', + 'linear_1.b_0_beta2_pow_acc_0', + 'linear_2.b_0_beta2_pow_acc_0', + 'embedding_0.w_0_beta2_pow_acc_0', + ] + ) + self.sharding_model( + Optimizer="adam", sharded_accumulators=sharded_accumulators + ) def test_sharding_momentum(self): - sharded_accumulators = set([ - 'linear_6.w_0_velocity_0', 'linear_7.b_0_velocity_0', - 'linear_8.b_0_velocity_0', 'embedding_2.w_0_velocity_0' - ]) - self.sharding_model(Optimizer="Momentum", - sharded_accumulators=sharded_accumulators) + sharded_accumulators = set( + [ + 'linear_6.w_0_velocity_0', + 'linear_7.b_0_velocity_0', + 'linear_8.b_0_velocity_0', + 'embedding_2.w_0_velocity_0', + ] + ) + self.sharding_model( + Optimizer="Momentum", sharded_accumulators=sharded_accumulators + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_shared_weight.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_shared_weight.py index 1696b966d7c..1176d70504b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_shared_weight.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_shared_weight.py @@ -44,15 +44,16 @@ hidden_size = 16 class SimpleNet(Layer): - def __init__(self): super(SimpleNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) self.softmax_weight = self.create_parameter( - shape=[hidden_size, vocab_size]) - self.softmax_bias = self.create_parameter(shape=[vocab_size], - is_bias=False) + shape=[hidden_size, vocab_size] + ) + self.softmax_bias = self.create_parameter( + shape=[vocab_size], is_bias=False + ) def forward(self, x1, x2, y1): x_emb = self.word_embeddings(x1) @@ -62,14 +63,13 @@ class SimpleNet(Layer): projection = paddle.matmul(projection, self.word_embeddings.weight) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=y1, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1, soft_label=False + ) return loss.mean() class EmbeddingPipe(Layer): - def __init__(self): super(EmbeddingPipe, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) @@ -85,11 +85,11 @@ class EmbeddingPipe(Layer): class MatmulNet(Layer): - def __init__(self): super(MatmulNet, self).__init__() self.softmax_weight = self.create_parameter( - shape=[hidden_size, vocab_size]) + shape=[hidden_size, vocab_size] + ) def forward(self, args): x1, x2 = args @@ -99,7 +99,6 @@ class MatmulNet(Layer): class BiasNet(Layer): - def __init__(self): super(BiasNet, self).__init__() self.softmax_bias = self.create_parameter(shape=[vocab_size]) @@ -112,26 +111,25 @@ class BiasNet(Layer): class LossNet(Layer): - def __init__(self): super(LossNet, self).__init__() def forward(self, args, y1): projection = args - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=y1[0], - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1[0], soft_label=False + ) return loss.mean() class SimpleNetPipe(PipelineLayer): - def __init__(self, **kwargs): self.descs = [] self.descs.append( - SharedLayerDesc('embed', - EmbeddingPipe, - shared_weight_attr='embedding_weight')) + SharedLayerDesc( + 'embed', EmbeddingPipe, shared_weight_attr='embedding_weight' + ) + ) self.descs.append(LayerDesc(MatmulNet)) self.descs.append(LayerDesc(BiasNet)) @@ -140,18 +138,20 @@ class SimpleNetPipe(PipelineLayer): return paddle.matmul(output[0], embedding.embedding_weight) self.descs.append( - SharedLayerDesc('embed', - EmbeddingPipe, - forward_func=_logits_helper, - shared_weight_attr='embedding_weight')) + SharedLayerDesc( + 'embed', + EmbeddingPipe, + forward_func=_logits_helper, + shared_weight_attr='embedding_weight', + ) + ) - super(SimpleNetPipe, self).__init__(layers=self.descs, - loss_fn=LossNet(), - **kwargs) + super(SimpleNetPipe, self).__init__( + layers=self.descs, loss_fn=LossNet(), **kwargs + ) class TestDistEmbeddingTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -164,7 +164,7 @@ class TestDistEmbeddingTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -176,19 +176,23 @@ class TestDistEmbeddingTraning(unittest.TestCase): rank_id = dist.get_rank() set_random_seed(1024, dp_id, rank_id) - #construct model a + # construct model a model_a = SimpleNet() scheduler_a = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) - optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, - parameters=model_a.parameters()) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True + ) + optimizer_a = paddle.optimizer.SGD( + learning_rate=scheduler_a, parameters=model_a.parameters() + ) model_b = SimpleNetPipe(topology=hcg.topology()) scheduler_b = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) - optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, - parameters=model_b.parameters()) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=scheduler_b, parameters=model_b.parameters() + ) model_b = fleet.distributed_model(model_b) optimizer_b = fleet.distributed_optimizer(optimizer_b) @@ -228,8 +232,9 @@ class TestDistEmbeddingTraning(unittest.TestCase): optimizer_a.clear_grad() scheduler_a.step() - loss_b = model_b.train_batch([(x1, x2), (y1, )], optimizer_b, - scheduler_b) + loss_b = model_b.train_batch( + [(x1, x2), (y1,)], optimizer_b, scheduler_b + ) print("loss", loss_a.numpy(), loss_b.numpy()) np.testing.assert_allclose(loss_a.numpy(), loss_b.numpy()) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/new_group.py b/python/paddle/fluid/tests/unittests/collective/fleet/new_group.py index b6e537681b2..9fa469ea5e2 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/new_group.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/new_group.py @@ -17,7 +17,6 @@ import paddle class TestNewGroupAPI(object): - def __init__(self): paddle.distributed.init_parallel_env() d1 = np.array([1, 2, 3]) @@ -32,10 +31,9 @@ class TestNewGroupAPI(object): tmp = np.array([0, 0, 0]) result = paddle.to_tensor(tmp) - paddle.distributed.scatter(result, [self.tensor2, self.tensor1], - src=0, - group=gp, - sync_op=True) + paddle.distributed.scatter( + result, [self.tensor2, self.tensor1], src=0, group=gp, sync_op=True + ) if gp.rank == 0: assert np.array_equal(result, self.tensor2) elif gp.rank == 1: @@ -48,8 +46,9 @@ class TestNewGroupAPI(object): paddle.distributed.reduce(result, dst=0, group=gp, sync_op=True) if gp.rank == 0: - assert np.array_equal(result, paddle.add(self.tensor1, - self.tensor1)) + assert np.array_equal( + result, paddle.add(self.tensor1, self.tensor1) + ) elif gp.rank == 1: assert np.array_equal(result, self.tensor1) print("test reduce api ok") @@ -57,7 +56,8 @@ class TestNewGroupAPI(object): paddle.distributed.all_reduce(result, sync_op=True) assert np.array_equal( result, - paddle.add(paddle.add(self.tensor1, self.tensor1), self.tensor1)) + paddle.add(paddle.add(self.tensor1, self.tensor1), self.tensor1), + ) print("test all_reduce api ok") paddle.distributed.wait(result, gp, use_calc_stream=True) @@ -65,10 +65,9 @@ class TestNewGroupAPI(object): print("test wait api ok") result = [] - paddle.distributed.all_gather(result, - self.tensor1, - group=gp, - sync_op=True) + paddle.distributed.all_gather( + result, self.tensor1, group=gp, sync_op=True + ) assert np.array_equal(result[0], self.tensor1) assert np.array_equal(result[1], self.tensor1) print("test all_gather api ok") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_class_center_sample.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_class_center_sample.py index ef7f5440987..bc75ede1d7e 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_class_center_sample.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_class_center_sample.py @@ -37,10 +37,13 @@ def class_center_sample_numpy(label, classes_list, num_samples): unique_label_per_device = [] for i in range(nranks): - index = np.logical_and(unique_label >= class_interval[i], - unique_label < class_interval[i + 1]) - pos_class_center_per_device.append(unique_label[index] - - class_interval[i]) + index = np.logical_and( + unique_label >= class_interval[i], + unique_label < class_interval[i + 1], + ) + pos_class_center_per_device.append( + unique_label[index] - class_interval[i] + ) unique_label_per_device.append(unique_label[index]) num_samples_per_device = [] @@ -50,8 +53,9 @@ def class_center_sample_numpy(label, classes_list, num_samples): remapped_dict = {} for i in range(nranks): - for idx, v in enumerate(unique_label_per_device[i], - sampled_class_interval[i]): + for idx, v in enumerate( + unique_label_per_device[i], sampled_class_interval[i] + ): remapped_dict[v] = idx remapped_label = [] @@ -62,7 +66,6 @@ def class_center_sample_numpy(label, classes_list, num_samples): class TestParallelClassCenterSampleOp(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() fleet.init(is_collective=True, strategy=strategy) @@ -83,24 +86,35 @@ class TestParallelClassCenterSampleOp(unittest.TestCase): for dtype in ('int32', 'int64'): for _ in range(5): - classes_list = np.random.randint(10, 15, (nranks, )) + classes_list = np.random.randint(10, 15, (nranks,)) num_class = np.sum(classes_list) - np_label = np.random.randint(0, - num_class, (batch_size, ), - dtype=dtype) + np_label = np.random.randint( + 0, num_class, (batch_size,), dtype=dtype + ) label = paddle.to_tensor(np_label, dtype=dtype) - np_remapped_label, np_sampled_class_center_per_device = class_center_sample_numpy( - np_label, classes_list, num_samples) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, classes_list[rank_id], num_samples) - np.testing.assert_allclose(remapped_label.numpy(), - np_remapped_label) + ( + np_remapped_label, + np_sampled_class_center_per_device, + ) = class_center_sample_numpy( + np_label, classes_list, num_samples + ) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, classes_list[rank_id], num_samples + ) + np.testing.assert_allclose( + remapped_label.numpy(), np_remapped_label + ) np_sampled_class_index = np_sampled_class_center_per_device[ - rank_id] + rank_id + ] np.testing.assert_allclose( - sampled_class_index.numpy()[:len(np_sampled_class_index)], - np_sampled_class_index) + sampled_class_index.numpy()[: len(np_sampled_class_index)], + np_sampled_class_index, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_different.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_different.py index 24a29f2a711..203396b0fd0 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_different.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_different.py @@ -25,14 +25,15 @@ np.random.seed(2021) class SimpleNet(fluid.Layer): - def __init__(self, hidden_size, vocab_size, is_sparse=False): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size - self.embedding = Embedding(size=[self.vocab_size, self.hidden_size], - dtype='float32', - is_sparse=is_sparse) + self.embedding = Embedding( + size=[self.vocab_size, self.hidden_size], + dtype='float32', + is_sparse=is_sparse, + ) self.lin_a = paddle.nn.Linear(self.hidden_size, self.vocab_size) self.lin_b = paddle.nn.Linear(self.vocab_size, 1) @@ -57,10 +58,9 @@ class SimpleNet(fluid.Layer): projection = paddle.reshape(projection, shape=[-1, 1]) output = paddle.gather(projection, emb_mask_inds) target = paddle.gather(label, emb_mask_inds) - loss_box = F.smooth_l1_loss(output, - target, - reduction='sum', - delta=1.0) + loss_box = F.smooth_l1_loss( + output, target, reduction='sum', delta=1.0 + ) loss_box = loss_box / len(conf) return loss_box @@ -72,36 +72,60 @@ batch_num = 2000 hidden_size = 5 vocab_size = 100 -conf_dataset = [[0], [0], [0], [0], [1], [0], [1], [0], [0], [1], [0], [1], [1], - [1], [1], [1], [1], [1], [1], [1], [1], [0], [0], [1]] +conf_dataset = [ + [0], + [0], + [0], + [0], + [1], + [0], + [1], + [0], + [0], + [1], + [0], + [1], + [1], + [1], + [1], + [1], + [1], + [1], + [1], + [1], + [1], + [0], + [0], + [1], +] def fake_sample_reader(): - def __reader__(): for i in range(batch_num): x_data = np.random.randint(0, vocab_size) - y_data = np.random.random_sample((1, )).astype('float32') - conf_data = np.array( - conf_dataset[i % len(conf_dataset)]).astype('int64') + y_data = np.random.random_sample((1,)).astype('float32') + conf_data = np.array(conf_dataset[i % len(conf_dataset)]).astype( + 'int64' + ) yield x_data, y_data, conf_data return __reader__ class TestSimpleNet(TestParallelDyGraphRunnerBase): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - is_sparse=False) + model = SimpleNet( + hidden_size=hidden_size, vocab_size=vocab_size, is_sparse=False + ) - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_same.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_same.py index 8b9aaf52943..f9097fc8461 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_same.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_same.py @@ -29,15 +29,18 @@ batch_num = 1000 class SimpleNet(fluid.Layer): - def __init__(self): super(SimpleNet, self).__init__() - self.net_a = paddle.nn.Sequential(paddle.nn.Linear(10, 20), - paddle.nn.Linear(20, 20), - paddle.nn.Linear(20, 5)) - self.net_b = paddle.nn.Sequential(paddle.nn.Linear(10, 20), - paddle.nn.Linear(20, 20), - paddle.nn.Linear(20, 5)) + self.net_a = paddle.nn.Sequential( + paddle.nn.Linear(10, 20), + paddle.nn.Linear(20, 20), + paddle.nn.Linear(20, 5), + ) + self.net_b = paddle.nn.Sequential( + paddle.nn.Linear(10, 20), + paddle.nn.Linear(20, 20), + paddle.nn.Linear(20, 5), + ) self.net_unused = Linear(10, 20) self.step = 0 @@ -51,24 +54,23 @@ class SimpleNet(fluid.Layer): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ class TestSimpleNet(TestParallelDyGraphRunnerBase): - def get_model(self): model = SimpleNet() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync.py index b00639202a6..633cdbcb0af 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync.py @@ -20,7 +20,12 @@ import paddle import paddle.fluid as fluid import paddle.distributed as dist from paddle.fluid.dygraph.nn import Linear -from test_dist_base import print_to_err, print_to_out, runtime_main, TestParallelDyGraphRunnerBase +from test_dist_base import ( + print_to_err, + print_to_out, + runtime_main, + TestParallelDyGraphRunnerBase, +) seed = 90 RUN_STEP = 20 @@ -29,7 +34,6 @@ batch_num = 1000 class SimpleNet(fluid.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.net_a = Linear(input_dim=10, output_dim=20) @@ -44,14 +48,14 @@ class SimpleNet(fluid.Layer): class TestNoSync(TestParallelDyGraphRunnerBase): - def get_model(self): model = SimpleNet() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): @@ -67,7 +71,7 @@ class TestNoSync(TestParallelDyGraphRunnerBase): device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace(device_id) else: - assert ("Only support CUDAPlace for now.") + assert "Only support CUDAPlace for now." with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = seed @@ -80,9 +84,11 @@ class TestNoSync(TestParallelDyGraphRunnerBase): dist.init_parallel_env() print_to_err( type(self).__name__, - "begin to prepare context in dygraph with nccl2") + "begin to prepare context in dygraph with nccl2", + ) model = paddle.DataParallel( - model, find_unused_parameters=args.find_unused_parameters) + model, find_unused_parameters=args.find_unused_parameters + ) print_to_err(type(self).__name__, "model built in dygraph") out_losses = self.model_train(args, model, opt, train_reader) print_to_out(out_losses) @@ -109,7 +115,8 @@ class TestNoSync(TestParallelDyGraphRunnerBase): model, train_reader, opt = self.get_model() if args.update_method in ["nccl2", "gloo"]: model = paddle.DataParallel( - model, find_unused_parameters=args.find_unused_parameters) + model, find_unused_parameters=args.find_unused_parameters + ) out_losses = self.model_train(args, model, opt, train_reader) print_to_out(out_losses) @@ -139,10 +146,9 @@ class TestNoSync(TestParallelDyGraphRunnerBase): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_control_flow.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_control_flow.py index afbb4cf0122..efe27d461ad 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_control_flow.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_control_flow.py @@ -27,7 +27,6 @@ batch_num = 1000 class SimpleNetControlFlow(fluid.Layer): - def __init__(self): super(SimpleNetControlFlow, self).__init__() self.net_a = Linear(input_dim=10, output_dim=20) @@ -46,14 +45,14 @@ class SimpleNetControlFlow(fluid.Layer): class TestNoSyncControlFlow(TestNoSync): - def get_model(self): model = SimpleNetControlFlow() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): @@ -66,10 +65,9 @@ class TestNoSyncControlFlow(TestNoSync): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_gradient_check.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_gradient_check.py index 51f5a3650ce..43c36ab491b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_gradient_check.py @@ -29,17 +29,19 @@ out_dim = 20 class SimpleNet(fluid.Layer): - def __init__(self, train_id): super(SimpleNet, self).__init__() - self.w1 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") - self.w2 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") + self.w1 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) + self.w2 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) self.share_net = Linear(out_dim, 1) - self.unused_param = self.create_parameter(shape=[out_dim, in_dim], - dtype="float32") + self.unused_param = self.create_parameter( + shape=[out_dim, in_dim], dtype="float32" + ) # just for test sync_params_buffers self.register_buffer("queue", paddle.randn([10, 5])) @@ -49,9 +51,10 @@ class SimpleNet(fluid.Layer): self.trainer_id = train_id def forward(self, x): - is_use = (paddle.equal_all( - x, paddle.ones(shape=(batch, in_dim))).numpy()[0] - and self.trainer_id == 1) + is_use = ( + paddle.equal_all(x, paddle.ones(shape=(batch, in_dim))).numpy()[0] + and self.trainer_id == 1 + ) if is_use: tmp = paddle.matmul(x, self.w1) @@ -62,7 +65,6 @@ class SimpleNet(fluid.Layer): class TestDistTraning(unittest.TestCase): - def test_multiple_gpus(self): self.trainer_id = dist.get_rank() dist.init_parallel_env() @@ -85,11 +87,13 @@ class TestDistTraning(unittest.TestCase): if step_id % 5 != 0: with model_a.no_sync(): - self.dp_layer(step_id, model_a, model_b, random_input, - ones_input) + self.dp_layer( + step_id, model_a, model_b, random_input, ones_input + ) else: - self.dp_layer(step_id, model_a, model_b, random_input, - ones_input) + self.dp_layer( + step_id, model_a, model_b, random_input, ones_input + ) self.check_gradient(model_a.parameters()) self.check_gradient(model_b.parameters()) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_unused_params.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_unused_params.py index bd8e598baf1..38804f11b2a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_unused_params.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_unused_params.py @@ -27,7 +27,6 @@ batch_num = 1000 class SimpleNetUnusedParam(fluid.Layer): - def __init__(self): super(SimpleNetUnusedParam, self).__init__() self.net_a = Linear(input_dim=10, output_dim=20) @@ -45,14 +44,14 @@ class SimpleNetUnusedParam(fluid.Layer): class TestNoSyncUnusedParam(TestNoSync): - def get_model(self): model = SimpleNetUnusedParam() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): @@ -65,10 +64,9 @@ class TestNoSyncUnusedParam(TestNoSync): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py index 09878cec229..0a0759f4380 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py @@ -33,12 +33,12 @@ train_parameters = { "name": "cosine_decay", "batch_size": batch_size, "epochs": [40, 80, 100], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "batch_size": batch_size, "lr": 0.0125, "total_images": 6149, - "num_epochs": 200 + "num_epochs": 200, } @@ -56,42 +56,47 @@ def optimizer_setting(params, parameter_list=None): num_epochs = params["num_epochs"] if fluid._non_static_mode(): optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.cosine_decay(learning_rate=lr, - step_each_epoch=step, - epochs=num_epochs), + learning_rate=fluid.layers.cosine_decay( + learning_rate=lr, step_each_epoch=step, epochs=num_epochs + ), momentum=momentum_rate, regularization=fluid.regularizer.L2Decay(l2_decay), - parameter_list=parameter_list) + parameter_list=parameter_list, + ) else: optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.cosine_decay(learning_rate=lr, - step_each_epoch=step, - epochs=num_epochs), + learning_rate=fluid.layers.cosine_decay( + learning_rate=lr, step_each_epoch=step, epochs=num_epochs + ), momentum=momentum_rate, - regularization=fluid.regularizer.L2Decay(l2_decay)) + regularization=fluid.regularizer.L2Decay(l2_decay), + ) return optimizer class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=False) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False, + ) # disable BatchNorm in multi-card. disable LayerNorm because of complex input_shape # self._batch_norm = BatchNorm(num_filters, act=act) @@ -104,7 +109,6 @@ class ConvBNLayer(fluid.dygraph.Layer): class SqueezeExcitation(fluid.dygraph.Layer): - def __init__(self, num_channels, reduction_ratio): super(SqueezeExcitation, self).__init__() @@ -115,15 +119,19 @@ class SqueezeExcitation(fluid.dygraph.Layer): num_channels, num_channels // reduction_ratio, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), - act='relu') + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + act='relu', + ) stdv = 1.0 / math.sqrt(num_channels / 16.0 * 1.0) self._excitation = Linear( num_channels // reduction_ratio, num_channels, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), - act='sigmoid') + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + act='sigmoid', + ) def forward(self, input): y = self._pool(input) @@ -135,39 +143,49 @@ class SqueezeExcitation(fluid.dygraph.Layer): class BottleneckBlock(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - cardinality, - reduction_ratio, - shortcut=True): + def __init__( + self, + num_channels, + num_filters, + stride, + cardinality, + reduction_ratio, + shortcut=True, + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act="relu") - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - groups=cardinality, - act="relu") - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 2, - filter_size=1, - act=None) - - self.scale = SqueezeExcitation(num_channels=num_filters * 2, - reduction_ratio=reduction_ratio) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act="relu", + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + act="relu", + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 2, + filter_size=1, + act=None, + ) + + self.scale = SqueezeExcitation( + num_channels=num_filters * 2, reduction_ratio=reduction_ratio + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 2, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 2, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -189,67 +207,76 @@ class BottleneckBlock(fluid.dygraph.Layer): class SeResNeXt(fluid.dygraph.Layer): - def __init__(self, layers=50, class_dim=102): super(SeResNeXt, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 101: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 23, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 152: cardinality = 64 reduction_ratio = 16 depth = [3, 8, 36, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - self.conv1 = ConvBNLayer(num_channels=64, - num_filters=64, - filter_size=3, - stride=1, - act='relu') - self.conv2 = ConvBNLayer(num_channels=64, - num_filters=128, - filter_size=3, - stride=1, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=3, + stride=2, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=64, + num_filters=64, + filter_size=3, + stride=1, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=64, + num_filters=128, + filter_size=3, + stride=1, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] num_channels = 64 @@ -258,19 +285,22 @@ class SeResNeXt(fluid.dygraph.Layer): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=cardinality, - reduction_ratio=reduction_ratio, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio, + shortcut=shortcut, + ), + ) num_channels = bottleneck_block._num_channels_out self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) stdv = 1.0 / math.sqrt(2048 * 1.0) self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 2 * 1 * 1 @@ -279,7 +309,9 @@ class SeResNeXt(fluid.dygraph.Layer): self.pool2d_avg_output, class_dim, param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) def forward(self, inputs): if self.layers == 50 or self.layers == 101: @@ -300,20 +332,23 @@ class SeResNeXt(fluid.dygraph.Layer): class TestSeResNeXt(TestParallelDyGraphRunnerBase): - def get_model(self): model = SeResNeXt() - train_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=train_parameters["batch_size"], - drop_last=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), + batch_size=train_parameters["batch_size"], + drop_last=True, + ) + optimizer = optimizer_setting( + train_parameters, parameter_list=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, opt, data): bs = len(data) - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') + dy_x_data = np.array([x[0].reshape(3, 224, 224) for x in data]).astype( + 'float32' + ) dy_x_data = dy_x_data / 255.0 y_data = np.array([x[1] for x in data]).astype('int64').reshape(bs, 1) img = to_variable(dy_x_data) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_sync_batch_norm.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_sync_batch_norm.py index 6d1f100252c..475eb1fa6d1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_sync_batch_norm.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_sync_batch_norm.py @@ -23,37 +23,42 @@ from test_dist_base import runtime_main, TestParallelDyGraphRunnerBase class TestLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(TestLayer, self).__init__() - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) self._sync_batch_norm = SyncBatchNorm(num_filters) - self._conv2 = Conv2D(in_channels=num_filters, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv2 = Conv2D( + in_channels=num_filters, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) - self._sync_batch_norm2 = SyncBatchNorm(num_filters, - weight_attr=False, - bias_attr=False) + self._sync_batch_norm2 = SyncBatchNorm( + num_filters, weight_attr=False, bias_attr=False + ) def forward(self, inputs): y = self._conv(inputs) @@ -65,20 +70,23 @@ class TestLayer(fluid.dygraph.Layer): class TestSyncBatchNorm(TestParallelDyGraphRunnerBase): - def get_model(self): model = TestLayer(3, 64, 7) - train_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=32, - drop_last=True) - opt = fluid.optimizer.Adam(learning_rate=1e-3, - parameter_list=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), + batch_size=32, + drop_last=True, + ) + opt = fluid.optimizer.Adam( + learning_rate=1e-3, parameter_list=model.parameters() + ) return model, train_reader, opt def run_one_loop(self, model, opt, data): batch_size = len(data) - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') + dy_x_data = np.array([x[0].reshape(3, 224, 224) for x in data]).astype( + 'float32' + ) img = to_variable(dy_x_data) img.stop_gradient = False diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_transformer.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_transformer.py index afe7799632d..97805222f80 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_transformer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_transformer.py @@ -16,10 +16,17 @@ import numpy as np import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph import Embedding, LayerNorm, Linear, to_variable, Layer +from paddle.fluid.dygraph import ( + Embedding, + LayerNorm, + Linear, + to_variable, + Layer, +) from paddle.optimizer.lr import NoamDecay from test_dist_base import runtime_main, TestParallelDyGraphRunnerBase + """ Note(chenweihang): To compare loss of single-card and multi-card in our dist test framework, two parameters need to be adjusted: @@ -33,6 +40,7 @@ class TrainTaskConfig(object): """ TrainTaskConfig """ + # the epoch number to train. pass_num = 20 # the number of sequences contained in a mini-batch. @@ -117,12 +125,17 @@ input_descs = { # encoder. # The actual data shape of src_slf_attn_bias is: # [batch_size, n_head, max_src_len_in_batch, max_src_len_in_batch] - "src_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "src_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # The actual data shape of trg_word is: # [batch_size, max_trg_len_in_batch, 1] - "trg_word": [(batch_size, seq_len, 1), "int64", - 2], # lod_level is only used in fast decoder. + "trg_word": [ + (batch_size, seq_len, 1), + "int64", + 2, + ], # lod_level is only used in fast decoder. # The actual data shape of trg_pos is: # [batch_size, max_trg_len_in_batch, 1] "trg_pos": [(batch_size, seq_len, 1), "int64"], @@ -130,14 +143,18 @@ input_descs = { # subsequent words in the decoder. # The actual data shape of trg_slf_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_trg_len_in_batch] - "trg_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used to remove attention weights on paddings of the source # input in the encoder-decoder attention. # The actual data shape of trg_src_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_src_len_in_batch] - "trg_src_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_src_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used in independent decoder program for inference. # The actual data shape of enc_output is: # [batch_size, max_src_len_in_batch, d_model] @@ -153,7 +170,7 @@ input_descs = { "init_score": [(batch_size, 1), "float32", 2], # This input is used in beam-search decoder for the first gather # (cell states updation) - "init_idx": [(batch_size, ), "int32"], + "init_idx": [(batch_size,), "int32"], } # Names of word embedding table which might be reused for weight sharing. @@ -200,26 +217,30 @@ def position_encoding_init(n_position, d_pos_vec): channels = d_pos_vec position = np.arange(n_position) num_timescales = channels // 2 - log_timescale_increment = (np.log(float(1e4) / float(1)) / - (num_timescales - 1)) - inv_timescales = np.exp( - np.arange(num_timescales)) * -log_timescale_increment + log_timescale_increment = np.log(float(1e4) / float(1)) / ( + num_timescales - 1 + ) + inv_timescales = ( + np.exp(np.arange(num_timescales)) * -log_timescale_increment + ) scaled_time = np.expand_dims(position, 1) * np.expand_dims( - inv_timescales, 0) + inv_timescales, 0 + ) signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1) signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant') position_enc = signal return position_enc.astype("float32") -pos_inp1 = position_encoding_init(ModelHyperParams.max_length, - ModelHyperParams.d_model) -pos_inp2 = position_encoding_init(ModelHyperParams.max_length, - ModelHyperParams.d_model) +pos_inp1 = position_encoding_init( + ModelHyperParams.max_length, ModelHyperParams.d_model +) +pos_inp2 = position_encoding_init( + ModelHyperParams.max_length, ModelHyperParams.d_model +) class PrePostProcessLayer(Layer): - def __init__(self, d_model, process_cmd, shape_len=None): super(PrePostProcessLayer, self).__init__() for cmd in process_cmd: @@ -227,11 +248,14 @@ class PrePostProcessLayer(Layer): self._layer_norm = LayerNorm( normalized_shape=d_model, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.)), + initializer=fluid.initializer.Constant(1.0) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.))) + initializer=fluid.initializer.Constant(0.0) + ), + ) - def forward(self, prev_out, out, process_cmd, dropout_rate=0.): + def forward(self, prev_out, out, process_cmd, dropout_rate=0.0): for cmd in process_cmd: if cmd == "a": # add residual connection out = out + prev_out if prev_out is not None else out @@ -243,12 +267,12 @@ class PrePostProcessLayer(Layer): out, dropout_prob=dropout_rate, seed=ModelHyperParams.dropout_seed, - is_test=False) + is_test=False, + ) return out class PositionwiseFeedForwardLayer(Layer): - def __init__(self, d_inner_hid, d_hid, dropout_rate): super(PositionwiseFeedForwardLayer, self).__init__() self._i2h = Linear(d_hid, d_inner_hid, act="relu") @@ -258,25 +282,28 @@ class PositionwiseFeedForwardLayer(Layer): def forward(self, x): hidden = self._i2h(x) if self._dropout_rate: - hidden = fluid.layers.dropout(hidden, - dropout_prob=self._dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) + hidden = fluid.layers.dropout( + hidden, + dropout_prob=self._dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) out = self._h2o(hidden) return out class MultiHeadAttentionLayer(Layer): - - def __init__(self, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - cache=None, - gather_idx=None, - static_kv=False): + def __init__( + self, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, + cache=None, + gather_idx=None, + static_kv=False, + ): super(MultiHeadAttentionLayer, self).__init__() self._n_head = n_head self._d_key = d_key @@ -299,20 +326,25 @@ class MultiHeadAttentionLayer(Layer): # split head reshaped_q = fluid.layers.reshape( - x=q, shape=[0, 0, self._n_head, self._d_key], inplace=False) + x=q, shape=[0, 0, self._n_head, self._d_key], inplace=False + ) transpose_q = fluid.layers.transpose(x=reshaped_q, perm=[0, 2, 1, 3]) reshaped_k = fluid.layers.reshape( - x=k, shape=[0, 0, self._n_head, self._d_key], inplace=False) + x=k, shape=[0, 0, self._n_head, self._d_key], inplace=False + ) transpose_k = fluid.layers.transpose(x=reshaped_k, perm=[0, 2, 1, 3]) reshaped_v = fluid.layers.reshape( - x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False) + x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False + ) transpose_v = fluid.layers.transpose(x=reshaped_v, perm=[0, 2, 1, 3]) # scale dot product attention - product = fluid.layers.matmul(x=transpose_q, - y=transpose_k, - transpose_y=True, - alpha=self._d_model**-0.5) + product = fluid.layers.matmul( + x=transpose_q, + y=transpose_k, + transpose_y=True, + alpha=self._d_model**-0.5, + ) if attn_bias is not None: product += attn_bias weights = fluid.layers.softmax(product) @@ -321,7 +353,8 @@ class MultiHeadAttentionLayer(Layer): weights, dropout_prob=self._dropout_rate, seed=ModelHyperParams.dropout_seed, - is_test=False) + is_test=False, + ) out = fluid.layers.matmul(weights_droped, transpose_v) else: out = fluid.layers.matmul(weights, transpose_v) @@ -333,7 +366,8 @@ class MultiHeadAttentionLayer(Layer): final_out = fluid.layers.reshape( x=trans_x, shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], - inplace=False) + inplace=False, + ) # fc to output proj_out = self._proj_fc(final_out) @@ -341,118 +375,150 @@ class MultiHeadAttentionLayer(Layer): class EncoderSubLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderSubLayer, self).__init__() self._preprocess_cmd = preprocess_cmd self._postprocess_cmd = postprocess_cmd self._prepostprocess_dropout = prepostprocess_dropout - self._preprocess_layer = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + self._preprocess_layer = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) self._multihead_attention_layer = MultiHeadAttentionLayer( - d_key, d_value, d_model, n_head, attention_dropout) - self._postprocess_layer = PrePostProcessLayer(d_model, - self._postprocess_cmd, - None) - self._preprocess_layer2 = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + d_key, d_value, d_model, n_head, attention_dropout + ) + self._postprocess_layer = PrePostProcessLayer( + d_model, self._postprocess_cmd, None + ) + self._preprocess_layer2 = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) self._positionwise_feed_forward = PositionwiseFeedForwardLayer( - d_inner_hid, d_model, relu_dropout) - self._postprocess_layer2 = PrePostProcessLayer(d_model, - self._postprocess_cmd, - None) + d_inner_hid, d_model, relu_dropout + ) + self._postprocess_layer2 = PrePostProcessLayer( + d_model, self._postprocess_cmd, None + ) def forward(self, enc_input, attn_bias): pre_process_multihead = self._preprocess_layer( - None, enc_input, self._preprocess_cmd, self._prepostprocess_dropout) - attn_output = self._multihead_attention_layer(pre_process_multihead, - None, None, attn_bias) - attn_output = self._postprocess_layer(enc_input, attn_output, - self._postprocess_cmd, - self._prepostprocess_dropout) + None, enc_input, self._preprocess_cmd, self._prepostprocess_dropout + ) + attn_output = self._multihead_attention_layer( + pre_process_multihead, None, None, attn_bias + ) + attn_output = self._postprocess_layer( + enc_input, + attn_output, + self._postprocess_cmd, + self._prepostprocess_dropout, + ) pre_process2_output = self._preprocess_layer2( - None, attn_output, self._preprocess_cmd, - self._prepostprocess_dropout) + None, + attn_output, + self._preprocess_cmd, + self._prepostprocess_dropout, + ) ffd_output = self._positionwise_feed_forward(pre_process2_output) - return self._postprocess_layer2(attn_output, ffd_output, - self._postprocess_cmd, - self._prepostprocess_dropout) + return self._postprocess_layer2( + attn_output, + ffd_output, + self._postprocess_cmd, + self._prepostprocess_dropout, + ) class EncoderLayer(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderLayer, self).__init__() self._preprocess_cmd = preprocess_cmd self._encoder_sublayers = list() self._prepostprocess_dropout = prepostprocess_dropout self._n_layer = n_layer - self._preprocess_layer = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + self._preprocess_layer = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) for i in range(n_layer): self._encoder_sublayers.append( self.add_sublayer( 'esl_%d' % i, - EncoderSubLayer(n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd))) + EncoderSubLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ), + ) + ) def forward(self, enc_input, attn_bias): for i in range(self._n_layer): enc_output = self._encoder_sublayers[i](enc_input, attn_bias) enc_input = enc_output - return self._preprocess_layer(None, enc_output, self._preprocess_cmd, - self._prepostprocess_dropout) + return self._preprocess_layer( + None, enc_output, self._preprocess_cmd, self._prepostprocess_dropout + ) class PrepareEncoderDecoderLayer(Layer): - - def __init__(self, - src_vocab_size, - src_emb_dim, - src_max_len, - dropout_rate, - is_sparse=False, - word_emb_param_name=None, - pos_enc_param_name=None): + def __init__( + self, + src_vocab_size, + src_emb_dim, + src_max_len, + dropout_rate, + is_sparse=False, + word_emb_param_name=None, + pos_enc_param_name=None, + ): super(PrepareEncoderDecoderLayer, self).__init__() self._src_max_len = src_max_len self._src_emb_dim = src_emb_dim self._src_vocab_size = src_vocab_size self._dropout_rate = dropout_rate - self._input_emb = Embedding(size=[src_vocab_size, src_emb_dim], - is_sparse=is_sparse, - padding_idx=0, - param_attr=fluid.ParamAttr( - name=word_emb_param_name, - initializer=fluid.initializer.Normal( - 0., src_emb_dim**-0.5))) + self._input_emb = Embedding( + size=[src_vocab_size, src_emb_dim], + is_sparse=is_sparse, + padding_idx=0, + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0.0, src_emb_dim**-0.5), + ), + ) if pos_enc_param_name is pos_enc_param_names[0]: pos_inp = pos_inp1 @@ -464,41 +530,50 @@ class PrepareEncoderDecoderLayer(Layer): param_attr=fluid.ParamAttr( name=pos_enc_param_name, initializer=fluid.initializer.NumpyArrayInitializer(pos_inp), - trainable=False)) + trainable=False, + ), + ) def forward(self, src_word, src_pos): src_word_emb = self._input_emb(src_word) - src_word_emb = fluid.layers.scale(x=src_word_emb, - scale=self._src_emb_dim**0.5) + src_word_emb = fluid.layers.scale( + x=src_word_emb, scale=self._src_emb_dim**0.5 + ) # # TODO change this to fit dynamic length input src_pos_emb = self._pos_emb(src_pos) src_pos_emb.stop_gradient = True enc_input = src_word_emb + src_pos_emb - return fluid.layers.dropout( - enc_input, - dropout_prob=self._dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) if self._dropout_rate else enc_input + return ( + fluid.layers.dropout( + enc_input, + dropout_prob=self._dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) + if self._dropout_rate + else enc_input + ) class WrapEncoderLayer(Layer): - - def __init__(self, - src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=False): + def __init__( + self, + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=False, + ): """ The wrapper assembles together all needed layers for the encoder. """ @@ -511,11 +586,21 @@ class WrapEncoderLayer(Layer): prepostprocess_dropout, is_sparse=is_sparse, word_emb_param_name=word_emb_param_names[0], - pos_enc_param_name=pos_enc_param_names[0]) - self._encoder = EncoderLayer(n_layer, n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd) + pos_enc_param_name=pos_enc_param_names[0], + ) + self._encoder = EncoderLayer( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ) def forward(self, enc_inputs): src_word, src_pos, src_slf_attn_bias = enc_inputs @@ -525,26 +610,28 @@ class WrapEncoderLayer(Layer): class DecoderSubLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None, - gather_idx=None): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + cache=None, + gather_idx=None, + ): super(DecoderSubLayer, self).__init__() self._postprocess_cmd = postprocess_cmd self._preprocess_cmd = preprocess_cmd self._prepostprcess_dropout = prepostprocess_dropout - self._pre_process_layer = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + self._pre_process_layer = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._multihead_attention_layer = MultiHeadAttentionLayer( d_key, d_value, @@ -552,11 +639,14 @@ class DecoderSubLayer(Layer): n_head, attention_dropout, cache=cache, - gather_idx=gather_idx) - self._post_process_layer = PrePostProcessLayer(d_model, postprocess_cmd, - None) - self._pre_process_layer2 = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + gather_idx=gather_idx, + ) + self._post_process_layer = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) + self._pre_process_layer2 = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._multihead_attention_layer2 = MultiHeadAttentionLayer( d_key, d_value, @@ -565,63 +655,86 @@ class DecoderSubLayer(Layer): attention_dropout, cache=cache, gather_idx=gather_idx, - static_kv=True) - self._post_process_layer2 = PrePostProcessLayer(d_model, - postprocess_cmd, None) - self._pre_process_layer3 = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + static_kv=True, + ) + self._post_process_layer2 = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) + self._pre_process_layer3 = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._positionwise_feed_forward_layer = PositionwiseFeedForwardLayer( - d_inner_hid, d_model, relu_dropout) - self._post_process_layer3 = PrePostProcessLayer(d_model, - postprocess_cmd, None) + d_inner_hid, d_model, relu_dropout + ) + self._post_process_layer3 = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) def forward(self, dec_input, enc_output, slf_attn_bias, dec_enc_attn_bias): - pre_process_rlt = self._pre_process_layer(None, dec_input, - self._preprocess_cmd, - self._prepostprcess_dropout) + pre_process_rlt = self._pre_process_layer( + None, dec_input, self._preprocess_cmd, self._prepostprcess_dropout + ) slf_attn_output = self._multihead_attention_layer( - pre_process_rlt, None, None, slf_attn_bias) + pre_process_rlt, None, None, slf_attn_bias + ) slf_attn_output_pp = self._post_process_layer( - dec_input, slf_attn_output, self._postprocess_cmd, - self._prepostprcess_dropout) - pre_process_rlt2 = self._pre_process_layer2(None, slf_attn_output_pp, - self._preprocess_cmd, - self._prepostprcess_dropout) + dec_input, + slf_attn_output, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) + pre_process_rlt2 = self._pre_process_layer2( + None, + slf_attn_output_pp, + self._preprocess_cmd, + self._prepostprcess_dropout, + ) enc_attn_output_pp = self._multihead_attention_layer2( - pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias) - enc_attn_output = self._post_process_layer2(slf_attn_output_pp, - enc_attn_output_pp, - self._postprocess_cmd, - self._prepostprcess_dropout) - pre_process_rlt3 = self._pre_process_layer3(None, enc_attn_output, - self._preprocess_cmd, - self._prepostprcess_dropout) + pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias + ) + enc_attn_output = self._post_process_layer2( + slf_attn_output_pp, + enc_attn_output_pp, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) + pre_process_rlt3 = self._pre_process_layer3( + None, + enc_attn_output, + self._preprocess_cmd, + self._prepostprcess_dropout, + ) ffd_output = self._positionwise_feed_forward_layer(pre_process_rlt3) - dec_output = self._post_process_layer3(enc_attn_output, ffd_output, - self._postprocess_cmd, - self._prepostprcess_dropout) + dec_output = self._post_process_layer3( + enc_attn_output, + ffd_output, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) return dec_output class DecoderLayer(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=None, - gather_idx=None): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + caches=None, + gather_idx=None, + ): super(DecoderLayer, self).__init__() - self._pre_process_layer = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + self._pre_process_layer = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._decoder_sub_layers = list() self._n_layer = n_layer self._preprocess_cmd = preprocess_cmd @@ -630,53 +743,62 @@ class DecoderLayer(Layer): self._decoder_sub_layers.append( self.add_sublayer( 'dsl_%d' % i, - DecoderSubLayer(n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None if caches is None else caches[i], - gather_idx=gather_idx))) - - def forward(self, dec_input, enc_output, dec_slf_attn_bias, - dec_enc_attn_bias): + DecoderSubLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + cache=None if caches is None else caches[i], + gather_idx=gather_idx, + ), + ) + ) + + def forward( + self, dec_input, enc_output, dec_slf_attn_bias, dec_enc_attn_bias + ): for i in range(self._n_layer): - tmp_dec_output = self._decoder_sub_layers[i](dec_input, enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias) + tmp_dec_output = self._decoder_sub_layers[i]( + dec_input, enc_output, dec_slf_attn_bias, dec_enc_attn_bias + ) dec_input = tmp_dec_output - dec_output = self._pre_process_layer(None, tmp_dec_output, - self._preprocess_cmd, - self._prepostprocess_dropout) + dec_output = self._pre_process_layer( + None, + tmp_dec_output, + self._preprocess_cmd, + self._prepostprocess_dropout, + ) return dec_output class WrapDecoderLayer(Layer): - - def __init__(self, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - caches=None, - gather_idx=None, - is_sparse=False): + def __init__( + self, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + caches=None, + gather_idx=None, + is_sparse=False, + ): """ The wrapper assembles together all needed layers for the encoder. """ @@ -689,20 +811,23 @@ class WrapDecoderLayer(Layer): prepostprocess_dropout, is_sparse=is_sparse, word_emb_param_name=word_emb_param_names[1], - pos_enc_param_name=pos_enc_param_names[1]) - self._decoder_layer = DecoderLayer(n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=caches, - gather_idx=gather_idx) + pos_enc_param_name=pos_enc_param_names[1], + ) + self._decoder_layer = DecoderLayer( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + caches=caches, + gather_idx=gather_idx, + ) self._weight_sharing = weight_sharing if not weight_sharing: self._fc = Linear(d_model, trg_vocab_size, bias_attr=False) @@ -710,17 +835,20 @@ class WrapDecoderLayer(Layer): def forward(self, dec_inputs=None, enc_output=None): trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias = dec_inputs dec_input = self._prepare_decoder_layer(trg_word, trg_pos) - dec_output = self._decoder_layer(dec_input, enc_output, - trg_slf_attn_bias, trg_src_attn_bias) + dec_output = self._decoder_layer( + dec_input, enc_output, trg_slf_attn_bias, trg_src_attn_bias + ) dec_output_reshape = fluid.layers.reshape( - dec_output, shape=[-1, dec_output.shape[-1]], inplace=False) + dec_output, shape=[-1, dec_output.shape[-1]], inplace=False + ) if self._weight_sharing: predict = fluid.layers.matmul( x=dec_output_reshape, y=self._prepare_decoder_layer._input_emb.weight, - transpose_y=True) + transpose_y=True, + ) else: predict = self._fc(dec_output_reshape) @@ -732,81 +860,91 @@ class WrapDecoderLayer(Layer): class TransFormer(Layer): - - def __init__(self, - src_vocab_size, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - label_smooth_eps, - use_py_reader=False, - is_test=False, - is_sparse=False): + def __init__( + self, + src_vocab_size, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + label_smooth_eps, + use_py_reader=False, + is_test=False, + is_sparse=False, + ): super(TransFormer, self).__init__() self._label_smooth_eps = label_smooth_eps self._trg_vocab_size = trg_vocab_size if weight_sharing: - assert src_vocab_size == trg_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) - self._wrap_encoder_layer = WrapEncoderLayer(src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=is_sparse) - self._wrap_decoder_layer = WrapDecoderLayer(trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=is_sparse) + assert ( + src_vocab_size == trg_vocab_size + ), "Vocabularies in source and target should be same for weight sharing." + self._wrap_encoder_layer = WrapEncoderLayer( + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=is_sparse, + ) + self._wrap_decoder_layer = WrapDecoderLayer( + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=is_sparse, + ) if weight_sharing: - self._wrap_decoder_layer._prepare_decoder_layer._input_emb.weight = self._wrap_encoder_layer._prepare_encoder_layer._input_emb.weight + self._wrap_decoder_layer._prepare_decoder_layer._input_emb.weight = ( + self._wrap_encoder_layer._prepare_encoder_layer._input_emb.weight + ) def forward(self, enc_inputs, dec_inputs, label, weights): enc_output = self._wrap_encoder_layer(enc_inputs) predict = self._wrap_decoder_layer(dec_inputs, enc_output) if self._label_smooth_eps: label_out = fluid.layers.label_smooth( - label=fluid.layers.one_hot(input=label, - depth=self._trg_vocab_size), - epsilon=self._label_smooth_eps) + label=fluid.layers.one_hot( + input=label, depth=self._trg_vocab_size + ), + epsilon=self._label_smooth_eps, + ) cost = fluid.layers.softmax_with_cross_entropy( logits=predict, label=label_out, - soft_label=True if self._label_smooth_eps else False) + soft_label=True if self._label_smooth_eps else False, + ) weighted_cost = cost * weights sum_cost = fluid.layers.reduce_sum(weighted_cost) token_num = fluid.layers.reduce_sum(weights) @@ -820,47 +958,54 @@ batch_num = 5 def fake_data_reader(): - def __reader__(): iteration = TrainTaskConfig.batch_size * batch_num for _ in range(iteration): # random data np.random.seed = 90 - src_word_np = np.arange(1, seq_len + 1).reshape([seq_len - ]).astype('int64') - src_pos_np = np.random.randint(1, - seq_len, - size=(seq_len), - dtype='int64') - src_slf_attn_bias_np = np.random.randn(ModelHyperParams.n_head, - seq_len, - seq_len).astype('float32') - - trg_word_np = np.arange(1, seq_len + 1).reshape([seq_len - ]).astype('int64') - trg_pos_np = np.random.randint(1, - seq_len, - size=(seq_len), - dtype='int64') - trg_slf_attn_bias_np = np.random.randn(ModelHyperParams.n_head, - seq_len, - seq_len).astype('float32') - trg_src_attn_bias_np = np.random.randn(ModelHyperParams.n_head, - seq_len, - seq_len).astype('float32') - - lbl_word_np = np.random.randint(1, - ModelHyperParams.src_vocab_size - 1, - size=(seq_len, 1), - dtype='int64') + src_word_np = ( + np.arange(1, seq_len + 1).reshape([seq_len]).astype('int64') + ) + src_pos_np = np.random.randint( + 1, seq_len, size=(seq_len), dtype='int64' + ) + src_slf_attn_bias_np = np.random.randn( + ModelHyperParams.n_head, seq_len, seq_len + ).astype('float32') + + trg_word_np = ( + np.arange(1, seq_len + 1).reshape([seq_len]).astype('int64') + ) + trg_pos_np = np.random.randint( + 1, seq_len, size=(seq_len), dtype='int64' + ) + trg_slf_attn_bias_np = np.random.randn( + ModelHyperParams.n_head, seq_len, seq_len + ).astype('float32') + trg_src_attn_bias_np = np.random.randn( + ModelHyperParams.n_head, seq_len, seq_len + ).astype('float32') + + lbl_word_np = np.random.randint( + 1, + ModelHyperParams.src_vocab_size - 1, + size=(seq_len, 1), + dtype='int64', + ) # Note(chenweihang): weight will introduce diff, so use constant here lbl_weight_np = np.ones((seq_len, 1)).astype('int64') data_inputs = [ - src_word_np, src_pos_np, src_slf_attn_bias_np, trg_word_np, - trg_pos_np, trg_slf_attn_bias_np, trg_src_attn_bias_np, - lbl_word_np, lbl_weight_np + src_word_np, + src_pos_np, + src_slf_attn_bias_np, + trg_word_np, + trg_pos_np, + trg_slf_attn_bias_np, + trg_src_attn_bias_np, + lbl_word_np, + lbl_weight_np, ] yield data_inputs @@ -884,20 +1029,30 @@ def np_to_variable(data): lbl_weight_np = lbl_weight_np.reshape(batch_size * seq_len, 1) data_inputs = [ - src_word_np, src_pos_np, src_slf_attn_bias_np, trg_word_np, trg_pos_np, - trg_slf_attn_bias_np, trg_src_attn_bias_np, lbl_word_np, lbl_weight_np + src_word_np, + src_pos_np, + src_slf_attn_bias_np, + trg_word_np, + trg_pos_np, + trg_slf_attn_bias_np, + trg_src_attn_bias_np, + lbl_word_np, + lbl_weight_np, ] var_inputs = [] - for i, field in enumerate(encoder_data_input_fields + - decoder_data_input_fields[:-1] + - label_data_input_fields): + for i, field in enumerate( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + ): var_inputs.append(to_variable(data_inputs[i], name=field)) - enc_inputs = var_inputs[0:len(encoder_data_input_fields)] - dec_inputs = var_inputs[len(encoder_data_input_fields - ):len(encoder_data_input_fields) + - len(decoder_data_input_fields[:-1])] + enc_inputs = var_inputs[0 : len(encoder_data_input_fields)] + dec_inputs = var_inputs[ + len(encoder_data_input_fields) : len(encoder_data_input_fields) + + len(decoder_data_input_fields[:-1]) + ] label = var_inputs[-2] weights = var_inputs[-1] @@ -908,38 +1063,45 @@ naive_optimize = True class TestTransformer(TestParallelDyGraphRunnerBase): - def get_model(self): - model = TransFormer(ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - is_sparse=True) - train_reader = paddle.batch(fake_data_reader(), - TrainTaskConfig.batch_size) + model = TransFormer( + ModelHyperParams.src_vocab_size, + ModelHyperParams.trg_vocab_size, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.prepostprocess_dropout, + ModelHyperParams.attention_dropout, + ModelHyperParams.relu_dropout, + ModelHyperParams.preprocess_cmd, + ModelHyperParams.postprocess_cmd, + ModelHyperParams.weight_sharing, + TrainTaskConfig.label_smooth_eps, + is_sparse=True, + ) + train_reader = paddle.batch( + fake_data_reader(), TrainTaskConfig.batch_size + ) if naive_optimize: - optimizer = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=model.parameters()) + optimizer = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=model.parameters() + ) else: - optimizer = fluid.optimizer.Adam(learning_rate=NoamDecay( - ModelHyperParams.d_model, TrainTaskConfig.warmup_steps, - TrainTaskConfig.learning_rate), - beta1=TrainTaskConfig.beta1, - beta2=TrainTaskConfig.beta2, - epsilon=TrainTaskConfig.eps, - parameter_list=model.parameters()) + optimizer = fluid.optimizer.Adam( + learning_rate=NoamDecay( + ModelHyperParams.d_model, + TrainTaskConfig.warmup_steps, + TrainTaskConfig.learning_rate, + ), + beta1=TrainTaskConfig.beta1, + beta2=TrainTaskConfig.beta2, + epsilon=TrainTaskConfig.eps, + parameter_list=model.parameters(), + ) return model, train_reader, optimizer @@ -947,7 +1109,8 @@ class TestTransformer(TestParallelDyGraphRunnerBase): enc_inputs, dec_inputs, label, weights = np_to_variable(batch) dy_sum_cost, dy_avg_cost, dy_predict, dy_token_num = model( - enc_inputs, dec_inputs, label, weights) + enc_inputs, dec_inputs, label, weights + ) return dy_avg_cost diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_margin_cross_entropy.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_margin_cross_entropy.py index 9d852785f6d..343cce6f4bb 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_margin_cross_entropy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_margin_cross_entropy.py @@ -30,7 +30,6 @@ def set_random_seed(seed): class TestParallelMarginSoftmaxCrossEntropyOp(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() fleet.init(is_collective=True, strategy=strategy) @@ -60,57 +59,71 @@ class TestParallelMarginSoftmaxCrossEntropyOp(unittest.TestCase): num_class = np.sum(num_class_per_card) for margin1, margin2, margin3, scale in zip( - margin1s, margin2s, margin3s, scales): + margin1s, margin2s, margin3s, scales + ): for _ in range(5): - np_label = np.random.randint(0, num_class, - (batch_size, )) + np_label = np.random.randint( + 0, num_class, (batch_size,) + ) label = paddle.to_tensor(np_label, dtype="int64") - input = paddle.randn(shape=[batch_size, feature_length], - dtype=dtype) + input = paddle.randn( + shape=[batch_size, feature_length], dtype=dtype + ) input.stop_gradient = False input_l2 = paddle.sqrt( - paddle.sum(paddle.square(input), - axis=1, - keepdim=True)) + paddle.sum( + paddle.square(input), axis=1, keepdim=True + ) + ) norm_input = paddle.divide(input, input_l2) weight = paddle.randn( shape=[feature_length, num_class_per_card[rank_id]], - dtype=dtype) + dtype=dtype, + ) weight.stop_gradient = False weight_l2 = paddle.sqrt( - paddle.sum(paddle.square(weight), - axis=0, - keepdim=True)) + paddle.sum( + paddle.square(weight), axis=0, keepdim=True + ) + ) norm_weight = paddle.divide(weight, weight_l2) data = paddle.matmul(norm_input, norm_weight) data.stop_gradient = False - sta = np.sum( - num_class_per_card[:rank_id]) if rank_id > 0 else 0 - end = np.sum(num_class_per_card[:rank_id + 1]) - - integral_data = np.zeros((batch_size, num_class), - dtype=dtype) - integral_data[:, - sta:end] = data.clone().detach().numpy() - integral_data = paddle.to_tensor(integral_data, - dtype=dtype) + sta = ( + np.sum(num_class_per_card[:rank_id]) + if rank_id > 0 + else 0 + ) + end = np.sum(num_class_per_card[: rank_id + 1]) + + integral_data = np.zeros( + (batch_size, num_class), dtype=dtype + ) + integral_data[:, sta:end] = ( + data.clone().detach().numpy() + ) + integral_data = paddle.to_tensor( + integral_data, dtype=dtype + ) paddle.distributed.all_reduce( integral_data, op=paddle.distributed.ReduceOp.SUM, - group=check_group) + group=check_group, + ) integral_data = integral_data.detach().clone() integral_data.stop_gradient = False # add arcface margin to logit theta = paddle.acos(integral_data) one_hot_label = paddle.nn.functional.one_hot( - label, num_classes=num_class) + label, num_classes=num_class + ) one_hot_label.stop_gradient = False if margin1 != 1.0: @@ -123,7 +136,10 @@ class TestParallelMarginSoftmaxCrossEntropyOp(unittest.TestCase): diff = one_hot_label * (margin_cos - integral_data) arc_data = (integral_data + diff) * scale - loss_a, softmax_a = paddle.nn.functional.margin_cross_entropy( + ( + loss_a, + softmax_a, + ) = paddle.nn.functional.margin_cross_entropy( data, label, margin1=margin1, @@ -132,54 +148,69 @@ class TestParallelMarginSoftmaxCrossEntropyOp(unittest.TestCase): scale=scale, group=check_group, return_softmax=True, - reduction=None) - loss_b, softmax_b = paddle.nn.functional.softmax_with_cross_entropy( + reduction=None, + ) + ( + loss_b, + softmax_b, + ) = paddle.nn.functional.softmax_with_cross_entropy( logits=arc_data, label=paddle.reshape(label, (-1, 1)), - return_softmax=True) - - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=1e-5, - atol=1e-7) - - integral_prob = np.zeros((batch_size, num_class), - dtype=dtype) - integral_prob[:, sta:end] = softmax_a.clone().detach( - ).numpy() - integral_prob = paddle.to_tensor(integral_prob, - dtype=dtype) + return_softmax=True, + ) + + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=1e-5, atol=1e-7 + ) + + integral_prob = np.zeros( + (batch_size, num_class), dtype=dtype + ) + integral_prob[:, sta:end] = ( + softmax_a.clone().detach().numpy() + ) + integral_prob = paddle.to_tensor( + integral_prob, dtype=dtype + ) paddle.distributed.all_reduce( integral_prob, op=paddle.distributed.ReduceOp.SUM, - group=check_group) + group=check_group, + ) integral_prob = integral_prob.detach().clone() integral_prob.stop_gradient = False - np.testing.assert_allclose(integral_prob.numpy(), - softmax_b.numpy(), - rtol=1e-5, - atol=1e-6) + np.testing.assert_allclose( + integral_prob.numpy(), + softmax_b.numpy(), + rtol=1e-5, + atol=1e-6, + ) loss_a = loss_a.sum() / batch_size loss_b = loss_b.sum() / batch_size loss_a.backward() loss_b.backward() - integral_grad = np.zeros((batch_size, num_class), - dtype=dtype) + integral_grad = np.zeros( + (batch_size, num_class), dtype=dtype + ) integral_grad[:, sta:end] = data.grad.clone().detach() - integral_grad = paddle.to_tensor(integral_grad, - dtype=dtype) + integral_grad = paddle.to_tensor( + integral_grad, dtype=dtype + ) paddle.distributed.all_reduce( integral_grad, op=paddle.distributed.ReduceOp.SUM, - group=check_group) - - np.testing.assert_allclose(integral_data.grad.numpy(), - integral_grad.numpy(), - rtol=1e-5, - atol=1e-7) + group=check_group, + ) + + np.testing.assert_allclose( + integral_data.grad.numpy(), + integral_grad.numpy(), + rtol=1e-5, + atol=1e-7, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist.py b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist.py index bd4e761a7a1..3e173c790ea 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist.py @@ -36,8 +36,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -45,13 +47,15 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 with fluid.device_guard("gpu:1"): predict = fluid.layers.fc( @@ -59,26 +63,29 @@ def cnn_model(data): size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) # To cover @RENAMED@GRADIENT predict2 = fluid.layers.fc( input=conv_pool_1, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) predict += predict2 return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data with fluid.device_guard("gpu:0"): - images = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype=DTYPE) + images = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype=DTYPE + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') if dist_strategy: @@ -86,7 +93,8 @@ class TestDistMnist2x2(TestDistRunnerBase): feed_list=[images, label], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) # Train program predict = cnn_model(images) with fluid.device_guard("gpu:1"): @@ -96,9 +104,9 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator with fluid.device_guard("gpu:1"): batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() base_lr = self.lr @@ -110,15 +118,18 @@ class TestDistMnist2x2(TestDistRunnerBase): opt = paddle.optimizer.AdamW( learning_rate=lr_val, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) acc_steps = 2 # accumulated steps for pipeline if dist_strategy: # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) fleet.init(is_collective=True) strategy = fleet.DistributedStrategy() strategy.pipeline = True @@ -126,23 +137,41 @@ class TestDistMnist2x2(TestDistRunnerBase): strategy.pipeline_configs = { 'micro_batch_size': batch_size, 'schedule_mode': '1F1B', - 'accumulate_steps': acc_steps + 'accumulate_steps': acc_steps, } - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size * acc_steps) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size * acc_steps) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps + ) if dist_strategy: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + data_loader, + ) else: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_multi_device.py b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_multi_device.py index 1933b55f023..e7f8a82f166 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_multi_device.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_multi_device.py @@ -36,8 +36,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -45,13 +47,15 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 with fluid.device_guard("gpu:1"): predict = fluid.layers.fc( @@ -59,26 +63,29 @@ def cnn_model(data): size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) # To cover @RENAMED@GRADIENT predict2 = fluid.layers.fc( input=conv_pool_1, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) predict += predict2 return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data with fluid.device_guard("gpu:0"): - images = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype=DTYPE) + images = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype=DTYPE + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') if dist_strategy: @@ -86,7 +93,8 @@ class TestDistMnist2x2(TestDistRunnerBase): feed_list=[images, label], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) # Train program predict = cnn_model(images) with fluid.device_guard("gpu:1"): @@ -96,9 +104,9 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator with fluid.device_guard("gpu:1"): batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() base_lr = self.lr @@ -110,15 +118,18 @@ class TestDistMnist2x2(TestDistRunnerBase): opt = fluid.optimizer.Momentum( learning_rate=lr_val, momentum=0.9, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) acc_steps = 2 # accumulated steps for pipeline if dist_strategy: # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) fleet.init(is_collective=True) strategy = fleet.DistributedStrategy() strategy.pipeline = True @@ -126,23 +137,41 @@ class TestDistMnist2x2(TestDistRunnerBase): strategy.pipeline_configs = { 'micro_batch_size': batch_size, 'schedule_mode': 'F-then-B', - 'accumulate_steps': acc_steps + 'accumulate_steps': acc_steps, } - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size * acc_steps) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size * acc_steps) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps + ) if dist_strategy: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + data_loader, + ) else: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_one_device.py b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_one_device.py index 33964a6400e..899cc24fdba 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_one_device.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_one_device.py @@ -36,8 +36,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -45,34 +47,37 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data device_id = 0 if dist_strategy: fleet.init(is_collective=True) with fluid.device_guard("gpu:0"): - images = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype=DTYPE) + images = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype=DTYPE + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') if dist_strategy: @@ -80,7 +85,8 @@ class TestDistMnist2x2(TestDistRunnerBase): feed_list=[images, label], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) # Train program predict = cnn_model(images) with fluid.device_guard("gpu:0"): @@ -90,9 +96,9 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator with fluid.device_guard("gpu:0"): batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() base_lr = self.lr @@ -104,28 +110,46 @@ class TestDistMnist2x2(TestDistRunnerBase): opt = fluid.optimizer.Momentum(learning_rate=lr_val, momentum=0.9) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) if dist_strategy: strategy = fleet.DistributedStrategy() strategy.pipeline = True strategy.pipeline_configs = { 'schedule_mode': 'F-then-B', - 'micro_batch_size': batch_size + 'micro_batch_size': batch_size, } - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) if dist_strategy: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + data_loader, + ) else: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_col.py b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_col.py index 40ec79b9844..cd874ae42b0 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_col.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_col.py @@ -27,60 +27,63 @@ IN_SIZE = 2 * MODEL_PARALLEL_SIZE OUT_SIZE = 2 * MODEL_PARALLEL_SIZE # Fix seed for test -#fluid.default_startup_program().random_seed = 1 -#fluid.default_main_program().random_seed = 1 +# fluid.default_startup_program().random_seed = 1 +# fluid.default_main_program().random_seed = 1 def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr def create_model(data, rank): np.random.seed(2021) np_weight = np.random.uniform(-1, 1, size=(IN_SIZE, OUT_SIZE)).astype(DTYPE) - np_bias = np.random.uniform(-1, 1, size=(OUT_SIZE, )).astype(DTYPE) + np_bias = np.random.uniform(-1, 1, size=(OUT_SIZE,)).astype(DTYPE) if rank is not None: start_col = 0 if rank == 0 else OUT_SIZE // 2 - np_weight_part = np_weight[:, start_col:start_col + OUT_SIZE // 2] - np_bias_part = np_bias[start_col:start_col + OUT_SIZE // 2] + np_weight_part = np_weight[:, start_col : start_col + OUT_SIZE // 2] + np_bias_part = np_bias[start_col : start_col + OUT_SIZE // 2] weight_attr, bias_attr = get_param_attr(np_weight_part, np_bias_part) - result = paddle.distributed.split(data, - size=(IN_SIZE, OUT_SIZE), - operation='linear', - axis=1, - num_partitions=MODEL_PARALLEL_SIZE, - weight_attr=weight_attr, - bias_attr=bias_attr) + result = paddle.distributed.split( + data, + size=(IN_SIZE, OUT_SIZE), + operation='linear', + axis=1, + num_partitions=MODEL_PARALLEL_SIZE, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) else: weight_attr, bias_attr = get_param_attr(np_weight, np_bias) - result = fluid.layers.fc(data, - size=OUT_SIZE, - param_attr=weight_attr, - bias_attr=bias_attr) + result = fluid.layers.fc( + data, size=OUT_SIZE, param_attr=weight_attr, bias_attr=bias_attr + ) predict = paddle.sum(result) return predict class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data - data_in = fluid.data(name='data_in', - shape=[batch_size, IN_SIZE], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, IN_SIZE], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -93,8 +96,9 @@ class TestModelParallel(TestDistRunnerBase): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_row.py b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_row.py index c14dd021729..44b0858712a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_row.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_row.py @@ -27,61 +27,67 @@ IN_SIZE = 2 * MODEL_PARALLEL_SIZE OUT_SIZE = 2 * MODEL_PARALLEL_SIZE # Fix seed for test -#fluid.default_startup_program().random_seed = 1 -#fluid.default_main_program().random_seed = 1 +# fluid.default_startup_program().random_seed = 1 +# fluid.default_main_program().random_seed = 1 def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr def create_model(data, rank): np.random.seed(2021) np_weight = np.random.uniform(-1, 1, size=(IN_SIZE, OUT_SIZE)).astype(DTYPE) - np_bias = np.random.uniform(-1, 1, size=(OUT_SIZE, )).astype(DTYPE) + np_bias = np.random.uniform(-1, 1, size=(OUT_SIZE,)).astype(DTYPE) if rank is not None: start_row = 0 if rank == 0 else IN_SIZE // 2 - np_weight_part = np_weight[start_row:start_row + IN_SIZE // 2, :] + np_weight_part = np_weight[start_row : start_row + IN_SIZE // 2, :] weight_attr, bias_attr = get_param_attr(np_weight_part, np_bias) - result = paddle.distributed.split(data, - size=(IN_SIZE, OUT_SIZE), - operation='linear', - axis=0, - num_partitions=MODEL_PARALLEL_SIZE, - weight_attr=weight_attr, - bias_attr=bias_attr) + result = paddle.distributed.split( + data, + size=(IN_SIZE, OUT_SIZE), + operation='linear', + axis=0, + num_partitions=MODEL_PARALLEL_SIZE, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) else: weight_attr, bias_attr = get_param_attr(np_weight, np_bias) result = fluid.layers.fc( data, size=OUT_SIZE, param_attr=paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(np_weight)), - bias_attr=bias_attr) + initializer=fluid.initializer.NumpyArrayInitializer(np_weight) + ), + bias_attr=bias_attr, + ) predict = paddle.sum(result) return predict class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data - data_in = fluid.data(name='data_in', - shape=[batch_size, IN_SIZE], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, IN_SIZE], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -94,8 +100,9 @@ class TestModelParallel(TestDistRunnerBase): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_embedding.py b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_embedding.py index f147190a5d2..832cad58eb8 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_embedding.py @@ -27,8 +27,8 @@ IN_SIZE = 2 * MODEL_PARALLEL_SIZE OUT_SIZE = 2 * MODEL_PARALLEL_SIZE # Fix seed for test -#fluid.default_startup_program().random_seed = 1 -#fluid.default_main_program().random_seed = 1 +# fluid.default_startup_program().random_seed = 1 +# fluid.default_main_program().random_seed = 1 def create_model(data, rank): @@ -36,15 +36,18 @@ def create_model(data, rank): np_weight = np.random.uniform(-1, 1, size=(IN_SIZE, OUT_SIZE)).astype(DTYPE) if rank is not None: start_row = 0 if rank == 0 else IN_SIZE // 2 - np_weight_part = np_weight[start_row:start_row + IN_SIZE // 2, :] + np_weight_part = np_weight[start_row : start_row + IN_SIZE // 2, :] result = paddle.distributed.split( data, size=(IN_SIZE, OUT_SIZE), operation='linear', axis=0, num_partitions=MODEL_PARALLEL_SIZE, - weight_attr=paddle.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(np_weight_part)), + weight_attr=paddle.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + np_weight_part + ) + ), bias_attr=False, ) else: @@ -52,7 +55,8 @@ def create_model(data, rank): data, size=OUT_SIZE, param_attr=paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(np_weight)), + initializer=fluid.initializer.NumpyArrayInitializer(np_weight) + ), bias_attr=False, ) @@ -61,19 +65,19 @@ def create_model(data, rank): class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data - data_in = fluid.data(name='data_in', - shape=[batch_size, IN_SIZE], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, IN_SIZE], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -86,8 +90,9 @@ class TestModelParallel(TestDistRunnerBase): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint.py index 3a06bc29d0f..cb61e2c9a8a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint.py @@ -20,14 +20,16 @@ from paddle.distributed.fleet.utils.fs import LocalFS, HDFSClient import paddle.fluid.incubate.checkpoint.auto_checkpoint as acp from paddle.fluid.incubate.checkpoint.checkpoint_saver import PaddleModel -from paddle.fluid.tests.unittests.auto_checkpoint_utils import AutoCheckpointBase, get_logger +from paddle.fluid.tests.unittests.auto_checkpoint_utils import ( + AutoCheckpointBase, + get_logger, +) paddle.enable_static() logger = get_logger() class AutoCheckPointACLBase(AutoCheckpointBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -44,7 +46,7 @@ class AutoCheckPointACLBase(AutoCheckpointBase): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) @@ -66,7 +68,8 @@ class AutoCheckPointACLBase(AutoCheckpointBase): logger.info("begin _run_normal") compiled, data_loader, optimizer, loss, image, label = self._init_env( - exe, main_prog, startup_prog) + exe, main_prog, startup_prog + ) for i in range(3): self.assertEqual(acp._get_train_epoch_range(), None) self.assertEqual(acp.g_acp_type, None) @@ -91,8 +94,9 @@ class AutoCheckPointACLBase(AutoCheckpointBase): logger.info("begin _not_use_train") exe, main_prog, startup_prog = self._generate() - compiled, data_loader, optimizer, loss, image, label = \ - self._init_env(exe, main_prog, startup_prog) + compiled, data_loader, optimizer, loss, image, label = self._init_env( + exe, main_prog, startup_prog + ) epochs = [] for i in acp.train_epoch_range(3, 0): @@ -111,8 +115,9 @@ class AutoCheckPointACLBase(AutoCheckpointBase): exe, main_prog, startup_prog = self._generate() - compiled, data_loader, optimizer, loss, image, label = \ - self._init_env(exe, main_prog, startup_prog) + compiled, data_loader, optimizer, loss, image, label = self._init_env( + exe, main_prog, startup_prog + ) o = None i = 0 @@ -149,7 +154,8 @@ class AutoCheckPointACLBase(AutoCheckpointBase): fs.delete(save_dir) compiled, data_loader, optimizer, loss, image, label = self._init_env( - exe, main_prog, startup_prog) + exe, main_prog, startup_prog + ) o = None i = 0 @@ -195,7 +201,6 @@ class AutoCheckPointACLBase(AutoCheckpointBase): class AutoCheckpointTest(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -212,7 +217,7 @@ class AutoCheckpointTest(AutoCheckPointACLBase): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_0", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_0", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint1.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint1.py index 2db7b1e8f80..583f8443062 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint1.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint1.py @@ -24,7 +24,6 @@ logger = get_logger() class AutoCheckpointTest1(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -41,7 +40,7 @@ class AutoCheckpointTest1(AutoCheckPointACLBase): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_1", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_1", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint2.py index 96b538ee9b8..3c0cc24c6ef 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint2.py @@ -24,7 +24,6 @@ logger = get_logger() class AutoCheckpointTest2(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -41,7 +40,7 @@ class AutoCheckpointTest2(AutoCheckPointACLBase): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_2", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_2", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint3.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint3.py index 0e75f8e0c47..beb19fa0b1e 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint3.py @@ -24,7 +24,6 @@ logger = get_logger() class AutoCheckpointTest3(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -41,7 +40,7 @@ class AutoCheckpointTest3(AutoCheckPointACLBase): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_3", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_3", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_dist_basic.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_dist_basic.py index 331f6b10450..702c3eb24a3 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_dist_basic.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_dist_basic.py @@ -30,7 +30,6 @@ logger = get_logger() class AutoCheckpointTestDist(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -47,7 +46,7 @@ class AutoCheckpointTestDist(AutoCheckPointACLBase): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_dist_basic", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_dist_basic", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) @@ -65,8 +64,9 @@ class AutoCheckpointTestDist(AutoCheckPointACLBase): # basic exe, main_prog, startup_prog = self._generate() - compiled, data_loader, optimizer, loss, image, label = \ - self._init_env(exe, main_prog, startup_prog, minimize=False) + compiled, data_loader, optimizer, loss, image, label = self._init_env( + exe, main_prog, startup_prog, minimize=False + ) # fleet os.environ["TRAINING_ROLE"] = "TRAINER" @@ -91,9 +91,9 @@ class AutoCheckpointTestDist(AutoCheckPointACLBase): logger.info("_run_save_0 name:{} epoch_no:{}".format(o.name, i)) for data in data_loader(): - fetch = exe.run(fleet.main_program, - feed=data, - fetch_list=[loss]) + fetch = exe.run( + fleet.main_program, feed=data, fetch_list=[loss] + ) self.assertEqual(len(o._exe_status), 1) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_multiple.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_multiple.py index 61e0733aa84..6bb59c5d2aa 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_multiple.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_multiple.py @@ -27,7 +27,6 @@ logger = get_logger() class AutoCheckpointTestMul(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -44,7 +43,7 @@ class AutoCheckpointTestMul(AutoCheckPointACLBase): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_dist_multiple", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_dist_multiple", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) @@ -62,11 +61,23 @@ class AutoCheckpointTestMul(AutoCheckPointACLBase): exe, main_prog1, startup_prog1 = self._generate() _, main_prog2, startup_prog2 = self._generate() - compiled1, data_loader1, optimizer1, loss1, image1, label1 = \ - self._init_env(exe, main_prog1, startup_prog1) - - compiled2, data_loader2, optimizer2, loss2, image2, label2 = \ - self._init_env(exe, main_prog2, startup_prog2) + ( + compiled1, + data_loader1, + optimizer1, + loss1, + image1, + label1, + ) = self._init_env(exe, main_prog1, startup_prog1) + + ( + compiled2, + data_loader2, + optimizer2, + loss2, + image2, + label2, + ) = self._init_env(exe, main_prog2, startup_prog2) o = None epochs = [] diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_half_async.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_half_async.py index ab5bd396d2c..fb27d860880 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_half_async.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_half_async.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestCommunicatorHalfAsyncEnd2End(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None) @@ -38,7 +37,6 @@ class TestCommunicatorHalfAsyncEnd2End(unittest.TestCase): return avg_cost, x, y def fake_reader(self): - def reader(): for i in range(10000): x = numpy.random.random((1, 13)).astype('float32') @@ -74,9 +72,11 @@ class TestCommunicatorHalfAsyncEnd2End(unittest.TestCase): feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) for batch_id, data in enumerate(train_reader()): - exe.run(paddle.static.default_main_program(), - feed=feeder.feed(data), - fetch_list=[]) + exe.run( + paddle.static.default_main_program(), + feed=feeder.feed(data), + fetch_list=[], + ) fleet.stop_worker() @@ -89,9 +89,11 @@ class TestCommunicatorHalfAsyncEnd2End(unittest.TestCase): role = role_maker.UserDefinedRoleMaker( current_id=0, role=role_maker.Role.WORKER - if training_role == "TRAINER" else role_maker.Role.SERVER, + if training_role == "TRAINER" + else role_maker.Role.SERVER, worker_num=1, - server_endpoints=["127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6002"], + ) if training_role == "TRAINER": self.run_trainer(role, strategy) @@ -137,9 +139,11 @@ half_run_server.run_ut() _python = sys.executable ps_cmd = "{} {}".format(_python, server_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) os.environ["http_proxy"] = "" os.environ["https_proxy"] = "" diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_sync.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_sync.py index 16cba174192..f938079beb2 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_sync.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_sync.py @@ -27,7 +27,6 @@ import paddle.distributed.fleet as fleet class TestCommunicator(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[1], dtype='float32') y = fluid.layers.data(name='y', shape=[1], dtype='float32') @@ -43,8 +42,9 @@ class TestCommunicator(unittest.TestCase): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" fleet.init(role_maker.PaddleCloudRoleMaker()) avg_cost = self.net() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_momentum_op.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_momentum_op.py index 0bc6f6daeca..763df5bf73d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_momentum_op.py @@ -20,7 +20,6 @@ import paddle.fluid as fluid class TestDGCMomentumOp1(unittest.TestCase): - def get_tensor(self, name, value, place=None): tensor = self.scope.var(name).get_tensor() tensor.set(value, self.place if place is None else place) @@ -49,13 +48,17 @@ class TestDGCMomentumOp1(unittest.TestCase): self.param_name, self.param_tensor = self.get_tensor('Param', param) self.grad_name, self.grad_tensor = self.get_tensor('Grad', grad) self.velocity_name, self.velocity_tensor = self.get_tensor( - 'Velocity', velocity) + 'Velocity', velocity + ) self.learning_rate_name, self.learning_rate_tensor = self.get_tensor( - 'LearningRate', learning_rate) + 'LearningRate', learning_rate + ) self.current_step_name, self.current_step_tensor = self.get_tensor( - 'current_step', current_step, core.CPUPlace()) + 'current_step', current_step, core.CPUPlace() + ) self.nranks_name, self.nranks_tensor = self.get_tensor( - 'nranks', nranks, core.CPUPlace()) + 'nranks', nranks, core.CPUPlace() + ) self.kwargs = { # inputs @@ -65,12 +68,10 @@ class TestDGCMomentumOp1(unittest.TestCase): 'LearningRate': self.learning_rate_name, 'current_step': self.current_step_name, 'nranks': self.nranks_name, - # attrs 'mu': mu, 'use_nesterov': use_nesterov, 'rampup_begin_step': rampup_begin_step, - # outputs 'ParamOut': self.param_name, 'VelocityOut': self.velocity_name, @@ -79,8 +80,9 @@ class TestDGCMomentumOp1(unittest.TestCase): velocity_out = mu * velocity + grad / nranks if use_nesterov: - param_out = param - grad * learning_rate - \ - velocity_out * mu * learning_rate + param_out = ( + param - grad * learning_rate - velocity_out * mu * learning_rate + ) else: param_out = param - learning_rate * velocity_out @@ -89,7 +91,7 @@ class TestDGCMomentumOp1(unittest.TestCase): self.outputs = { 'ParamOut': param_out, 'VelocityOut': velocity_out, - 'SGDOut': sgd_out + 'SGDOut': sgd_out, } def check(self, actual_t, expect_t, place, out_name, atol=1e-5): @@ -98,8 +100,16 @@ class TestDGCMomentumOp1(unittest.TestCase): expect_t, rtol=1e-05, atol=atol, - err_msg='Output (' + out_name + ') has diff at ' + str(place) + - '\nExpect ' + str(expect_t) + '\n' + 'But Got' + str(actual_t)) + err_msg='Output (' + + out_name + + ') has diff at ' + + str(place) + + '\nExpect ' + + str(expect_t) + + '\n' + + 'But Got' + + str(actual_t), + ) def check_momentum_step(self, place): self.setup(place=place) @@ -107,11 +117,19 @@ class TestDGCMomentumOp1(unittest.TestCase): dgc_momentum_op = Operator(self.op_type, **self.kwargs) dgc_momentum_op.run(self.scope, self.place) - self.check(np.array(self.param_tensor), self.outputs['ParamOut'], - self.place, self.param_name) + self.check( + np.array(self.param_tensor), + self.outputs['ParamOut'], + self.place, + self.param_name, + ) - self.check(np.array(self.velocity_tensor), self.outputs['VelocityOut'], - self.place, self.velocity_name) + self.check( + np.array(self.velocity_tensor), + self.outputs['VelocityOut'], + self.place, + self.velocity_name, + ) def check_sgd_step(self, place): self.setup(place=place, step=15.0) @@ -119,8 +137,12 @@ class TestDGCMomentumOp1(unittest.TestCase): dgc_momentum_op = Operator(self.op_type, **self.kwargs) dgc_momentum_op.run(self.scope, self.place) - self.check(np.array(self.param_tensor), self.outputs['SGDOut'], - self.place, self.param_name) + self.check( + np.array(self.param_tensor), + self.outputs['SGDOut'], + self.place, + self.param_name, + ) def test_cuda_place(self): if not core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_op.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_op.py index 78d22325cf8..6df74578aac 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_op.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_op.py @@ -23,7 +23,6 @@ g_array_size = 102400 class TestDGCOp(unittest.TestCase): - def setup(self, place, array_size=g_array_size): size = array_size np.random.seed(5) # fix seed @@ -72,20 +71,23 @@ class TestDGCOp(unittest.TestCase): self.param_tensor.set(self.param, place) self.current_step_tensor = self.scope.var( - self.current_step_name).get_tensor() + self.current_step_name + ).get_tensor() self.current_step_tensor.set(self.current_step, core.CPUPlace()) self.nranks_tensor = self.scope.var(self.nranks_name).get_tensor() self.nranks_tensor.set(self.nranks, core.CPUPlace()) self.encode_grad_tensor = self.scope.var( - self.encode_grad_name).get_tensor() + self.encode_grad_name + ).get_tensor() self.k_tensor = self.scope.var(self.k_name).get_tensor() self.k_tensor.set(self.k, core.CPUPlace()) self.gather_buff_tensor = self.scope.var( - self.gather_buff_name).get_tensor() + self.gather_buff_name + ).get_tensor() def check(self, actual_t, expect_t, place, out_name, atol=1e-5): np.testing.assert_allclose( @@ -93,8 +95,16 @@ class TestDGCOp(unittest.TestCase): expect_t, rtol=1e-05, atol=atol, - err_msg='Output (' + out_name + ') has diff at ' + str(place) + - '\nExpect ' + str(expect_t) + '\n' + 'But Got' + str(actual_t)) + err_msg='Output (' + + out_name + + ') has diff at ' + + str(place) + + '\nExpect ' + + str(expect_t) + + '\n' + + 'But Got' + + str(actual_t), + ) def test_run_and_check(self): self.setup(place=core.CUDAPlace(0)) @@ -106,7 +116,6 @@ class TestDGCOp(unittest.TestCase): 'Param': self.param_name, 'current_step': self.current_step_name, 'nranks': self.nranks_name, - # outputs 'U_out': self.u_name, 'V_out': self.v_name, @@ -114,7 +123,6 @@ class TestDGCOp(unittest.TestCase): 'Grad_out': self.grad_name, 'k': self.k_name, 'GatherBuff': self.gather_buff_name, - # attrs 'm': 0.9, 'sparsity': [0.75, 0.9375, 0.984375, 0.996, 0.999], @@ -127,7 +135,7 @@ class TestDGCOp(unittest.TestCase): dgc_op = Operator('dgc', **kwargs) - #atol = 1e-6 + # atol = 1e-6 dgc_op.run(self.scope, self.place) u_out = np.array(self.u_tensor) @@ -144,7 +152,7 @@ class TestDGCOp(unittest.TestCase): self.assertEqual(k, int(g_array_size * 0.25)) index = encode_grad_out[0:k].view(dtype=np.int32) - value = encode_grad_out[k:2 * k] + value = encode_grad_out[k : 2 * k] acl = 1e-7 diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py index 50f1b389a0a..0da05a377b3 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py @@ -24,20 +24,20 @@ paddle.enable_static() class TestDGCMomentumOptimizer(unittest.TestCase): - class MockDGCMomentum(optimizer.DGCMomentumOptimizer): - def get_accumulators(self): return self._accumulators def get_velocity_str(self): return self._u_velocity_acc_str - def check_dgc_momentum_optimizer(self, - dims=[5, 10, 8], - name="momentum", - regularization=None, - use_recompute=False): + def check_dgc_momentum_optimizer( + self, + dims=[5, 10, 8], + name="momentum", + regularization=None, + use_recompute=False, + ): init_program = framework.Program() program = framework.Program() block = program.global_block() @@ -47,23 +47,25 @@ class TestDGCMomentumOptimizer(unittest.TestCase): lod_level=0, name="mul.x", optimize_attr={'learning_rate': 1.1}, - regularizer=None if regularization is not None else - regularizer.L2DecayRegularizer(2e-4)) - mul_y = block.create_var(dtype="float32", - shape=[dims[1], dims[2]], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[dims[0], dims[2]], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) + regularizer=None + if regularization is not None + else regularizer.L2DecayRegularizer(2e-4), + ) + mul_y = block.create_var( + dtype="float32", shape=[dims[1], dims[2]], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", + shape=[dims[0], dims[2]], + lod_level=0, + name="mul.out", + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) learning_rate = 0.01 dgc_momentum_optimizer = self.MockDGCMomentum( @@ -72,33 +74,40 @@ class TestDGCMomentumOptimizer(unittest.TestCase): rampup_begin_step=0, num_trainers=2, regularization=regularization, - grad_clip=clip.GradientClipByNorm(1.0)) + grad_clip=clip.GradientClipByNorm(1.0), + ) if use_recompute: dgc_momentum_optimizer = optimizer.RecomputeOptimizer( - dgc_momentum_optimizer) + dgc_momentum_optimizer + ) dgc_momentum_optimizer._set_checkpoints([]) - dgc_momentum_optimizer.get_accumulators = dgc_momentum_optimizer._optimizer.get_accumulators - dgc_momentum_optimizer.get_velocity_str = dgc_momentum_optimizer._optimizer.get_velocity_str - - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + dgc_momentum_optimizer.get_accumulators = ( + dgc_momentum_optimizer._optimizer.get_accumulators + ) + dgc_momentum_optimizer.get_velocity_str = ( + dgc_momentum_optimizer._optimizer.get_velocity_str + ) + + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) # params_grads = append_backward(mean_out) params_grads = dgc_momentum_optimizer.backward( - mean_out, startup_program=init_program) + mean_out, startup_program=init_program + ) with framework.program_guard(program, init_program): opts = dgc_momentum_optimizer.apply_gradients(params_grads) accumulator_count = 1 if name == "momentum" else 2 self.assertEqual(len(params_grads), 1) - self.assertEqual(len(dgc_momentum_optimizer.get_accumulators()), - accumulator_count) + self.assertEqual( + len(dgc_momentum_optimizer.get_accumulators()), accumulator_count + ) self.assertEqual(len(opts), 2) sgd_op = opts[-1] @@ -109,7 +118,8 @@ class TestDGCMomentumOptimizer(unittest.TestCase): accumulators = dgc_momentum_optimizer.get_accumulators() self.assertEqual(len(accumulators), accumulator_count) self.assertTrue( - dgc_momentum_optimizer.get_velocity_str() in accumulators) + dgc_momentum_optimizer.get_velocity_str() in accumulators + ) velocity_acc = accumulators[dgc_momentum_optimizer.get_velocity_str()] self.assertEqual(len(velocity_acc), 1) self.assertTrue(mul_x.name in velocity_acc) @@ -139,22 +149,26 @@ class TestDGCMomentumOptimizer(unittest.TestCase): momentum=0.2, rampup_begin_step=0, num_trainers=2, - grad_clip=clip.GradientClipByGlobalNorm(1.0)) + grad_clip=clip.GradientClipByGlobalNorm(1.0), + ) def test_momentum_without_dgc(self): self.check_dgc_momentum_optimizer( - regularization=regularizer.L1Decay(1e-4)) + regularization=regularizer.L1Decay(1e-4) + ) def test_momentum_with_dgc(self): # 16 * 1024 = 16384, use dgc momentum self.check_dgc_momentum_optimizer( dims=[16, 1024, 8], name="dgc_momentum", - regularization=regularizer.L2Decay(1e-4)) + regularization=regularizer.L2Decay(1e-4), + ) # check param.regularizer in dgc - self.check_dgc_momentum_optimizer(dims=[16, 1024, 8], - name="dgc_momentum") + self.check_dgc_momentum_optimizer( + dims=[16, 1024, 8], name="dgc_momentum" + ) def test_momentum_with_dgc_recompute(self): # 16 * 1024 = 16384, use dgc momentum @@ -162,7 +176,8 @@ class TestDGCMomentumOptimizer(unittest.TestCase): dims=[16, 1024, 8], name="dgc_momentum", regularization=regularizer.L2Decay(1e-4), - use_recompute=True) + use_recompute=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_dgc_nccl.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_dgc_nccl.py index 8a4fdfcd07b..7a2b2425f24 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_dgc_nccl.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_dgc_nccl.py @@ -27,7 +27,11 @@ def count_of_sparse_all_reduce_calls(file_name): # NOTE(Aurelius84): The log file contains some binary contents that causes error # while `grep`. So we add `-a` to fix it. # -a, --text equivalent to --binary-files=text, make binaries equivalent to text. - cmd = 'grep -a sparse_all_reduce_op_handle ' + file_name + ' | grep in_numel | wc -l' + cmd = ( + 'grep -a sparse_all_reduce_op_handle ' + + file_name + + ' | grep in_numel | wc -l' + ) child = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) result = child.communicate()[0] print('test_info: result = ' + str(result)) @@ -37,7 +41,6 @@ def count_of_sparse_all_reduce_calls(file_name): class TestDistMnistNCCL2DGC(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -47,17 +50,22 @@ class TestDistMnistNCCL2DGC(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place(os.path.abspath("../../dist_mnist.py"), - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + os.path.abspath("../../dist_mnist.py"), + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) def tearDown(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - log_file = os.path.join(self.temp_dir.name, - 'test_dist_mnist_dgc_nccl_tr0_err.log') + log_file = os.path.join( + self.temp_dir.name, 'test_dist_mnist_dgc_nccl_tr0_err.log' + ) result = count_of_sparse_all_reduce_calls(log_file) # only 1 layer use dgc now, run_step=5, rampup_begin_step=2, so 1 * (5 - 2) = 3 @@ -68,7 +76,6 @@ class TestDistMnistNCCL2DGC(TestDistBase): class TestDistMnistNCCL2DGCMultiCards(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -78,19 +85,23 @@ class TestDistMnistNCCL2DGCMultiCards(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place_multi_cards( os.path.abspath("../../dist_mnist.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) def tearDown(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): log_file = os.path.join( self.temp_dir.name, - 'test_dist_mnist_dgc_nccl_dgc_2cards_local.log') + 'test_dist_mnist_dgc_nccl_dgc_2cards_local.log', + ) result = count_of_sparse_all_reduce_calls(log_file) # same as above, but use two cards self.assertEqual(result, 6) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_gradient_merge.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_gradient_merge.py index 54c9d6b26c3..c0b978560e1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_gradient_merge.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_gradient_merge.py @@ -21,7 +21,6 @@ flag_name = os.path.splitext(__file__)[0] class TestDistMnistGradMerge(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -29,14 +28,15 @@ class TestDistMnistGradMerge(TestDistBase): def test_dist_train(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist_gradient_merge.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist_gradient_merge.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestDistMnistGradMergeNoFuse(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -45,14 +45,15 @@ class TestDistMnistGradMergeNoFuse(TestDistBase): def test_dist_train(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist_gradient_merge.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name + "_no_fuse") + self.check_with_place( + "dist_mnist_gradient_merge.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name + "_no_fuse", + ) class TestDistMnistGradMergeRawOptimizerBase(TestDistBase): - def _setup_config(self): self._use_reader_alloc = False self._nccl2_mode = True @@ -66,19 +67,21 @@ class TestDistMnistGradMergeRawOptimizerBase(TestDistBase): if fluid.core.is_compiled_with_cuda(): avg = str(self.enable_avg()) log_name = flag_name + "_raw_optimizer_gm_avg_" + avg - self.check_with_place("dist_mnist_gradient_merge_raw_optimizer.py", - delta=1e-5, - check_error_log=True, - log_name=log_name, - need_envs={ - 'FLAGS_apply_pass_to_program': '1', - 'enable_gm_avg': avg, - }) + self.check_with_place( + "dist_mnist_gradient_merge_raw_optimizer.py", + delta=1e-5, + check_error_log=True, + log_name=log_name, + need_envs={ + 'FLAGS_apply_pass_to_program': '1', + 'enable_gm_avg': avg, + }, + ) class TestDistMnistGradMergeRawOptimizerAvg( - TestDistMnistGradMergeRawOptimizerBase): - + TestDistMnistGradMergeRawOptimizerBase +): def enable_avg(self): return True diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_se_resnext_dgc.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_se_resnext_dgc.py index 28936e36845..7020e63248e 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_se_resnext_dgc.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_se_resnext_dgc.py @@ -20,7 +20,6 @@ flag_name = os.path.splitext(__file__)[0] class TestDistSeResnetNCCL2DGC(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -31,11 +30,14 @@ class TestDistSeResnetNCCL2DGC(TestDistBase): @unittest.skip(reason="Skip unstable ci") def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place(os.path.abspath("../../dist_se_resnext.py"), - delta=30, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + os.path.abspath("../../dist_se_resnext.py"), + delta=30, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_distributed_strategy.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_distributed_strategy.py index 92a6715a642..140a910c336 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_distributed_strategy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_distributed_strategy.py @@ -15,15 +15,21 @@ import unittest import paddle import paddle.fluid as fluid -from paddle.fluid.transpiler.distribute_transpiler import DistributeTranspilerConfig, ServerRuntimeConfig -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet +from paddle.fluid.transpiler.distribute_transpiler import ( + DistributeTranspilerConfig, + ServerRuntimeConfig, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) import paddle.fluid.incubate.fleet.base.role_maker as role_maker import os class TestStrategyFactor(unittest.TestCase): - def test_sync_strategy(self): os.environ['CPU_NUM'] = "2" strategy = StrategyFactory.create_sync_strategy() @@ -48,19 +54,23 @@ class TestStrategyFactor(unittest.TestCase): # test set_program_config exception program_config_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_program_config, - program_config_dict) + self.assertRaises( + Exception, strategy.set_program_config, program_config_dict + ) program_config_illegal = None - self.assertRaises(Exception, strategy.set_program_config, - program_config_illegal) + self.assertRaises( + Exception, strategy.set_program_config, program_config_illegal + ) trainer_runtime_config = strategy.get_trainer_runtime_config() trainer_runtime_config.runtime_configs[ - 'communicator_send_queue_size'] = '50' + 'communicator_send_queue_size' + ] = '50' runtime_configs = trainer_runtime_config.get_communicator_flags() self.assertIn('communicator_send_queue_size', runtime_configs) - self.assertNotIn('communicator_independent_recv_thread', - runtime_configs) + self.assertNotIn( + 'communicator_independent_recv_thread', runtime_configs + ) self.assertEqual(runtime_configs['communicator_send_queue_size'], '2') def test_geo_strategy(self): @@ -87,19 +97,22 @@ class TestStrategyFactor(unittest.TestCase): # test set_build_strategy exception build_strategy_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_build_strategy, - build_strategy_dict) + self.assertRaises( + Exception, strategy.set_build_strategy, build_strategy_dict + ) build_strategy_illegal = None - self.assertRaises(Exception, strategy.set_build_strategy, - build_strategy_illegal) + self.assertRaises( + Exception, strategy.set_build_strategy, build_strategy_illegal + ) os.environ["CPU_NUM"] = '100' trainer_runtime_config = strategy.get_trainer_runtime_config() runtime_configs = trainer_runtime_config.get_communicator_flags() self.assertIn('communicator_thread_pool_size', runtime_configs) self.assertIn('communicator_send_wait_times', runtime_configs) - self.assertNotIn('communicator_independent_recv_thread', - runtime_configs) + self.assertNotIn( + 'communicator_independent_recv_thread', runtime_configs + ) def test_async_strategy(self): os.environ["CPU_NUM"] = '100' @@ -111,28 +124,40 @@ class TestStrategyFactor(unittest.TestCase): trainer_runtime_config = strategy.get_trainer_runtime_config() self.assertEqual( - trainer_runtime_config. - runtime_configs['communicator_send_queue_size'], '100') + trainer_runtime_config.runtime_configs[ + 'communicator_send_queue_size' + ], + '100', + ) # test set_trainer_runtime_config using dict trainer_runtime_config_dict = dict() trainer_runtime_config_dict['communicator_send_queue_size'] = '20' strategy.set_trainer_runtime_config(trainer_runtime_config_dict) trainer_runtime_config = strategy.get_trainer_runtime_config() - trainer_communicator_flags = trainer_runtime_config.get_communicator_flags( + trainer_communicator_flags = ( + trainer_runtime_config.get_communicator_flags() + ) + self.assertIn( + 'communicator_send_queue_size', trainer_communicator_flags ) - self.assertIn('communicator_send_queue_size', - trainer_communicator_flags) self.assertEqual( - trainer_communicator_flags['communicator_send_queue_size'], '20') + trainer_communicator_flags['communicator_send_queue_size'], '20' + ) # test set_trainer_runtime_config exception trainer_runtime_config_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_trainer_runtime_config, - trainer_runtime_config_dict) + self.assertRaises( + Exception, + strategy.set_trainer_runtime_config, + trainer_runtime_config_dict, + ) trainer_runtime_config_illegal = None - self.assertRaises(Exception, strategy.set_trainer_runtime_config, - trainer_runtime_config_illegal) + self.assertRaises( + Exception, + strategy.set_trainer_runtime_config, + trainer_runtime_config_illegal, + ) # test set_execute_strategy using fluid.ExecutionStrategy exec_strategy_class = fluid.ExecutionStrategy() @@ -150,11 +175,13 @@ class TestStrategyFactor(unittest.TestCase): # test set_execute_strategy exception exec_strategy_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_execute_strategy, - exec_strategy_dict) + self.assertRaises( + Exception, strategy.set_execute_strategy, exec_strategy_dict + ) exec_strategy_illegal = None - self.assertRaises(Exception, strategy.set_execute_strategy, - exec_strategy_illegal) + self.assertRaises( + Exception, strategy.set_execute_strategy, exec_strategy_illegal + ) def test_half_async_strategy(self): strategy = StrategyFactory.create_half_async_strategy() @@ -178,31 +205,39 @@ class TestStrategyFactor(unittest.TestCase): # test set_server_runtime_config exception server_runtime_config_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_server_runtime_config, - server_runtime_config_dict) + self.assertRaises( + Exception, + strategy.set_server_runtime_config, + server_runtime_config_dict, + ) server_runtime_config_illegal = None - self.assertRaises(Exception, strategy.set_server_runtime_config, - server_runtime_config_illegal) + self.assertRaises( + Exception, + strategy.set_server_runtime_config, + server_runtime_config_illegal, + ) os.environ["CPU_NUM"] = '100' trainer_runtime_config = strategy.get_trainer_runtime_config() trainer_runtime_config.runtime_configs[ - 'communicator_send_queue_size'] = '50' + 'communicator_send_queue_size' + ] = '50' runtime_configs = trainer_runtime_config.get_communicator_flags() self.assertIn('communicator_send_queue_size', runtime_configs) - self.assertNotIn('communicator_independent_recv_thread', - runtime_configs) + self.assertNotIn( + 'communicator_independent_recv_thread', runtime_configs + ) self.assertEqual(runtime_configs['communicator_send_queue_size'], '100') class TestCreateDefaultStrategy(unittest.TestCase): - def test_default_strategy(self): role = role_maker.UserDefinedRoleMaker( current_id=0, role=role_maker.Role.WORKER, worker_num=2, - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) def type_error_optimizer(): @@ -213,13 +248,13 @@ class TestCreateDefaultStrategy(unittest.TestCase): class TestHalfAsyncStrategy(unittest.TestCase): - def test_half_async_strategy(self): role = role_maker.UserDefinedRoleMaker( current_id=0, role=role_maker.Role.WORKER, worker_num=2, - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) half_async_config = DistributeTranspilerConfig() @@ -233,7 +268,6 @@ class TestHalfAsyncStrategy(unittest.TestCase): class TestDebugInfo(unittest.TestCase): - def test_debug_info(self): x = fluid.layers.data(name='x', shape=[1], dtype='float32') y = fluid.layers.data(name='y', shape=[1], dtype='float32') @@ -245,16 +279,19 @@ class TestDebugInfo(unittest.TestCase): current_id=0, role=role_maker.Role.WORKER, worker_num=2, - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) optimizer = fluid.optimizer.SGD(0.0001) strategy = StrategyFactory.create_sync_strategy() - strategy.set_debug_opt({ - "dump_param": ["fc_0.tmp_0"], - "dump_fields": ["fc_0.tmp_0", "fc_0.tmp_0@GRAD"], - "dump_fields_path": "dump_text/" - }) + strategy.set_debug_opt( + { + "dump_param": ["fc_0.tmp_0"], + "dump_fields": ["fc_0.tmp_0", "fc_0.tmp_0@GRAD"], + "dump_fields_path": "dump_text/", + } + ) optimizer = fleet.distributed_optimizer(optimizer, strategy) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute.py index afd619a390b..9cadbf4d7b8 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute.py @@ -24,35 +24,41 @@ from paddle.incubate.distributed.fleet import recompute_sequential def get_fc_block(block_idx, input_size, is_last=False): block_name = "block_" + str(block_idx) block = paddle.nn.Sequential( - (block_name + "_fc_0", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_0", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_dropout", paddle.nn.Dropout(p=0.5)), (block_name + "_relu_1", paddle.nn.ReLU()), - (block_name + "_fc_1", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_1", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_relu_2", paddle.nn.ReLU()), ) if is_last: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, 1, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, 1, bias_attr=False), + ) # add sublayer else: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, - input_size, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ) # add sublayer return block class Naive_fc_net(paddle.nn.Layer): - - def __init__(self, - input_size=10, - recompute_blocks=[1, 3], - use_fleet_sq=False, - segments=1, - use_raw_recompute=False, - recompute_kwargs={}): + def __init__( + self, + input_size=10, + recompute_blocks=[1, 3], + use_fleet_sq=False, + segments=1, + use_raw_recompute=False, + recompute_kwargs={}, + ): super(Naive_fc_net, self).__init__() self.recompute_blocks = recompute_blocks self.recompute_kwargs = recompute_kwargs @@ -67,28 +73,37 @@ class Naive_fc_net(paddle.nn.Layer): self.runfunc4 = get_fc_block(4, input_size, is_last=True) if self.use_fleet_sq and not use_raw_recompute: - self.runfuncs = paddle.nn.Sequential(self.runfunc0, self.runfunc1, - self.runfunc2, self.runfunc3, - self.runfunc4) + self.runfuncs = paddle.nn.Sequential( + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, + ) self.layers = [ - self.runfunc0, self.runfunc1, self.runfunc2, self.runfunc3, - self.runfunc4 + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, ] # default segments = 2 if use_raw_recompute: self.layers = [ paddle.nn.Sequential(self.runfunc0, self.runfunc1), - paddle.nn.Sequential(self.runfunc2, self.runfunc3, - self.runfunc4) + paddle.nn.Sequential( + self.runfunc2, self.runfunc3, self.runfunc4 + ), ] def forward(self, inputs): if self.use_fleet_sq and not self.use_raw_recompute: - return recompute_sequential({"segments": self.segments}, - self.runfuncs, inputs) + return recompute_sequential( + {"segments": self.segments}, self.runfuncs, inputs + ) if self.use_raw_recompute: inputs = recompute(self.layers[0], inputs) @@ -96,36 +111,42 @@ class Naive_fc_net(paddle.nn.Layer): for i in range(len(self.layers)): if i in self.recompute_blocks: - inputs = recompute(self.layers[i], inputs, - **self.recompute_kwargs) + inputs = recompute( + self.layers[i], inputs, **self.recompute_kwargs + ) else: inputs = self.layers[i](inputs) return inputs -def run_model(recompute_block=[], - recompute_kwargs={}, - use_fleet_sq=False, - use_raw_recompute=False, - segments=1, - enable_autocast=False, - pure_fp16=False): +def run_model( + recompute_block=[], + recompute_kwargs={}, + use_fleet_sq=False, + use_raw_recompute=False, + segments=1, + enable_autocast=False, + pure_fp16=False, +): gen = paddle.seed(10) gen.manual_seed(10) np.random.seed(10) random.seed(10) batch_size, input_size = 1, 10 - model = Naive_fc_net(input_size, - recompute_blocks=recompute_block, - use_fleet_sq=use_fleet_sq, - use_raw_recompute=use_raw_recompute, - segments=segments, - recompute_kwargs=recompute_kwargs) + model = Naive_fc_net( + input_size, + recompute_blocks=recompute_block, + use_fleet_sq=use_fleet_sq, + use_raw_recompute=use_raw_recompute, + segments=segments, + recompute_kwargs=recompute_kwargs, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) if enable_autocast: scaler = paddle.amp.GradScaler() @@ -158,9 +179,7 @@ def run_model(recompute_block=[], class TestPyLayer(unittest.TestCase): - def test_base_case(self, enable_autocast=False, pure_fp16=False): - def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): self.assertEqual(loss_ref, loss) self.assertEqual(param_ref, param) @@ -170,43 +189,56 @@ class TestPyLayer(unittest.TestCase): loss_ref, param_ref, grad_ref = run_model( recompute_block=[], enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # recompute second block - loss, param, grad = run_model(recompute_block=[1], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute fourth block - loss, param, grad = run_model(recompute_block=[3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second to fourth block - loss, param, grad = run_model(recompute_block=[1, 2, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 2, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second & fourth block - loss, param, grad = run_model(recompute_block=[1, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second & fourth block using fleet - loss, param, grad = run_model(recompute_block=[1, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute using recompute_sequential, segments=1 - loss, param, grad = run_model(recompute_block=[], - use_fleet_sq=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[], + use_fleet_sq=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with base recompute, and segments=2 @@ -214,14 +246,17 @@ class TestPyLayer(unittest.TestCase): recompute_block=[], enable_autocast=enable_autocast, use_raw_recompute=True, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # recompute using recompute_sequential, segments=2 - loss, param, grad = run_model(recompute_block=[], - use_fleet_sq=True, - segments=2, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[], + use_fleet_sq=True, + segments=2, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) def test_fc_net_with_dropout(self): @@ -237,8 +272,9 @@ class TestPyLayer(unittest.TestCase): paddle.set_device("gpu") kwargs = {"is_test": False} with self.assertRaises(TypeError): - loss_ref, param_ref, grad_ref = run_model(recompute_block=[2], - recompute_kwargs=kwargs) + loss_ref, param_ref, grad_ref = run_model( + recompute_block=[2], recompute_kwargs=kwargs + ) def test_recompute_cpu_rng(self): paddle.set_device("cpu") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py index d69c52144f5..9683f681451 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py @@ -27,35 +27,41 @@ import random def get_fc_block(block_idx, input_size, is_last=False): block_name = "block_" + str(block_idx) block = paddle.nn.Sequential( - (block_name + "_fc_0", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_0", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_dropout", paddle.nn.Dropout(p=0.5)), (block_name + "_relu_1", paddle.nn.ReLU()), - (block_name + "_fc_1", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_1", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_relu_2", paddle.nn.ReLU()), ) if is_last: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, 1, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, 1, bias_attr=False), + ) # add sublayer else: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, - input_size, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ) # add sublayer return block class Naive_fc_net(paddle.nn.Layer): - - def __init__(self, - input_size=10, - recompute_blocks=[1, 3], - use_fleet_sq=False, - segments=1, - use_raw_recompute=False, - recompute_kwargs={}): + def __init__( + self, + input_size=10, + recompute_blocks=[1, 3], + use_fleet_sq=False, + segments=1, + use_raw_recompute=False, + recompute_kwargs={}, + ): super(Naive_fc_net, self).__init__() self.recompute_blocks = recompute_blocks self.recompute_kwargs = recompute_kwargs @@ -70,28 +76,37 @@ class Naive_fc_net(paddle.nn.Layer): self.runfunc4 = get_fc_block(4, input_size, is_last=True) if self.use_fleet_sq and not use_raw_recompute: - self.runfuncs = paddle.nn.Sequential(self.runfunc0, self.runfunc1, - self.runfunc2, self.runfunc3, - self.runfunc4) + self.runfuncs = paddle.nn.Sequential( + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, + ) self.layers = [ - self.runfunc0, self.runfunc1, self.runfunc2, self.runfunc3, - self.runfunc4 + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, ] # default segments = 2 if use_raw_recompute: self.layers = [ paddle.nn.Sequential(self.runfunc0, self.runfunc1), - paddle.nn.Sequential(self.runfunc2, self.runfunc3, - self.runfunc4) + paddle.nn.Sequential( + self.runfunc2, self.runfunc3, self.runfunc4 + ), ] def forward(self, inputs): if self.use_fleet_sq and not self.use_raw_recompute: return paddle.incubate.distributed.fleet.recompute_sequential( - {"segments": self.segments}, self.runfuncs, inputs) + {"segments": self.segments}, self.runfuncs, inputs + ) if self.use_raw_recompute: inputs = recompute(self.layers[0], inputs) @@ -99,36 +114,42 @@ class Naive_fc_net(paddle.nn.Layer): for i in range(len(self.layers)): if i in self.recompute_blocks: - inputs = recompute(self.layers[i], inputs, - **self.recompute_kwargs) + inputs = recompute( + self.layers[i], inputs, **self.recompute_kwargs + ) else: inputs = self.layers[i](inputs) return inputs -def run_model(recompute_block=[], - recompute_kwargs={}, - use_fleet_sq=False, - use_raw_recompute=False, - segments=1, - enable_autocast=False, - pure_fp16=False): +def run_model( + recompute_block=[], + recompute_kwargs={}, + use_fleet_sq=False, + use_raw_recompute=False, + segments=1, + enable_autocast=False, + pure_fp16=False, +): gen = paddle.seed(10) gen.manual_seed(10) np.random.seed(10) random.seed(10) batch_size, input_size = 1, 10 - model = Naive_fc_net(input_size, - recompute_blocks=recompute_block, - use_fleet_sq=use_fleet_sq, - use_raw_recompute=use_raw_recompute, - segments=segments, - recompute_kwargs=recompute_kwargs) + model = Naive_fc_net( + input_size, + recompute_blocks=recompute_block, + use_fleet_sq=use_fleet_sq, + use_raw_recompute=use_raw_recompute, + segments=segments, + recompute_kwargs=recompute_kwargs, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) if enable_autocast: scaler = paddle.amp.GradScaler() @@ -161,9 +182,7 @@ def run_model(recompute_block=[], class TestPyLayer(unittest.TestCase): - def test_base_case(self, enable_autocast=False, pure_fp16=False): - def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): self.assertEqual(loss_ref, loss) self.assertEqual(param_ref, param) @@ -173,37 +192,48 @@ class TestPyLayer(unittest.TestCase): loss_ref, param_ref, grad_ref = run_model( recompute_block=[], enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # recompute second block - loss, param, grad = run_model(recompute_block=[1], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute fourth block - loss, param, grad = run_model(recompute_block=[3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second to fourth block - loss, param, grad = run_model(recompute_block=[1, 2, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 2, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second & fourth block - loss, param, grad = run_model(recompute_block=[1, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute_sequential with segments=1 using fleet - loss, param, grad = run_model(recompute_block=[], - use_fleet_sq=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[], + use_fleet_sq=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with base recompute, and segments=2 @@ -211,14 +241,17 @@ class TestPyLayer(unittest.TestCase): recompute_block=[], enable_autocast=enable_autocast, use_raw_recompute=True, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # recompute using paddle.incubate.distributed.fleet.recompute_sequential, segments=2 - loss, param, grad = run_model(recompute_block=[], - use_fleet_sq=True, - segments=2, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[], + use_fleet_sq=True, + segments=2, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) def test_fc_net_with_dropout(self): @@ -228,7 +261,8 @@ class TestPyLayer(unittest.TestCase): loss_ref, param_ref, grad_ref = run_model( recompute_block=[2], recompute_kwargs={"preserve_rng_state": False}, - enable_autocast=True) + enable_autocast=True, + ) def test_fc_net_with_amp(self): self.test_base_case(enable_autocast=True) @@ -240,8 +274,9 @@ class TestPyLayer(unittest.TestCase): paddle.set_device("gpu") kwargs = {"is_test": False} with self.assertRaises(TypeError): - loss_ref, param_ref, grad_ref = run_model(recompute_block=[2], - recompute_kwargs=kwargs) + loss_ref, param_ref, grad_ref = run_model( + recompute_block=[2], recompute_kwargs=kwargs + ) def test_recompute_cpu_rng(self): paddle.set_device("cpu") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py index f4d3f9728ef..8087fc98b6a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py @@ -21,8 +21,9 @@ class TestDygraphShardingOptimizerStage2(TestMultipleGpus): # check sharding logic as well as the accuracy with single mode def test_dygraph_sharding_optimizer_stage2(self): - self.run_mnist_2gpu('dygraph_sharding_optimizer_stage2.py', - eager_mode=False) + self.run_mnist_2gpu( + 'dygraph_sharding_optimizer_stage2.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py index 07a9b1296e4..0dd1d4961b3 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py @@ -27,8 +27,9 @@ class TestDygraphShardingStage2(TestMultipleGpus): def test_dygraph_sharding_stage2_offload(self): self.run_mnist_2gpu('dygraph_group_sharded_stage2_offload.py') - self.run_mnist_2gpu('dygraph_sharding_stage2_offload.py', - eager_mode=False) + self.run_mnist_2gpu( + 'dygraph_sharding_stage2_offload.py', eager_mode=False + ) def test_dygraph_sharding_stage2_with_comm_overlap(self): self.run_mnist_2gpu('dygraph_group_sharded_stage2_comm_overlap.py') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py index 82e7b8e12bf..ea4edbd3493 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py @@ -28,8 +28,9 @@ class TestDygraphShardingStage3(TestMultipleGpus): self.run_mnist_2gpu('dygraph_sharding_stage3.py', eager_mode=False) def test_dygraph_sharding_stage3_offload(self): - self.run_mnist_2gpu('dygraph_sharding_stage3_offload.py', - eager_mode=False) + self.run_mnist_2gpu( + 'dygraph_sharding_stage3_offload.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py index a7df64c1d92..83e09ba98e0 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py @@ -26,23 +26,22 @@ paddle.enable_static() def gen_data(): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def mlp(input_x, input_y, hid_dim=128, label_dim=2): fc_1 = paddle.static.nn.fc(x=input_x, size=hid_dim, activation='tanh') fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim, activation='tanh') - prediction = paddle.static.nn.fc(x=[fc_2], - size=label_dim, - activation='softmax') + prediction = paddle.static.nn.fc( + x=[fc_2], size=label_dim, activation='softmax' + ) cost = F.cross_entropy(input=prediction, label=input_y) avg_cost = paddle.mean(x=cost) return avg_cost class TestFleetAMPInit(unittest.TestCase): - def test_fleet_amp_init(self): if not fluid.core.is_compiled_with_cuda(): return @@ -54,19 +53,20 @@ class TestFleetAMPInit(unittest.TestCase): fleet.init(role) with paddle.static.program_guard(main_program, startup_program): - input_x = paddle.static.data(name="x", - shape=[None, 32], - dtype='float32') - input_y = paddle.static.data(name="y", - shape=[None, 1], - dtype='int64') + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32' + ) + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64' + ) cost = mlp(input_x, input_y) optimizer = paddle.optimizer.Momentum( learning_rate=0.001, momentum=0.9, weight_decay=fluid.regularizer.L2Decay(1e-4), - multi_precision=True) + multi_precision=True, + ) optimizer = paddle.static.amp.decorate(optimizer) optimizer = fleet.distributed_optimizer(optimizer) @@ -82,9 +82,9 @@ class TestFleetAMPInit(unittest.TestCase): step = 1 for i in range(step): - cost_val = exe.run(program=main_program, - feed=gen_data(), - fetch_list=[cost.name]) + cost_val = exe.run( + program=main_program, feed=gen_data(), fetch_list=[cost.name] + ) def test_fleet_amp_meta_optimizer_init(self): if not fluid.core.is_compiled_with_cuda(): @@ -97,19 +97,20 @@ class TestFleetAMPInit(unittest.TestCase): fleet.init(role) with paddle.static.program_guard(main_program, startup_program): - input_x = paddle.static.data(name="x", - shape=[None, 32], - dtype='float32') - input_y = paddle.static.data(name="y", - shape=[None, 1], - dtype='int64') + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32' + ) + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64' + ) cost = mlp(input_x, input_y) optimizer = paddle.optimizer.Momentum( learning_rate=0.001, momentum=0.9, weight_decay=fluid.regularizer.L2Decay(1e-4), - multi_precision=True) + multi_precision=True, + ) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.amp = True @@ -131,9 +132,9 @@ class TestFleetAMPInit(unittest.TestCase): step = 3 for i in range(step): - cost_val = exe.run(program=main_program, - feed=gen_data(), - fetch_list=[cost.name]) + cost_val = exe.run( + program=main_program, feed=gen_data(), fetch_list=[cost.name] + ) print(cost_val) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_meta_optimizer.py index 7eb235a5499..b2afb900c97 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_meta_optimizer.py @@ -24,14 +24,14 @@ paddle.enable_static() class TestFleetAMPOptimizer(TestFleetMetaOptimizer): - def test_amp_optimizer_backward(self): - """ test amp optimizer backward """ + """test amp optimizer backward""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = AMPOptimizer(opt) self.set_strategy(strategy, 'amp') @@ -44,12 +44,13 @@ class TestFleetAMPOptimizer(TestFleetMetaOptimizer): self.assertNotIn('check_finite_and_unscale', ops) def test_amp_optimizer_backward_gradients(self): - """ test amp optimizer backward + gradients""" + """test amp optimizer backward + gradients""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = AMPOptimizer(opt) self.set_strategy(strategy, 'amp') @@ -64,12 +65,13 @@ class TestFleetAMPOptimizer(TestFleetMetaOptimizer): self.assertIn('check_finite_and_unscale', ops) def test_amp_optimizer_backward_optimize(self): - """ test amp optimizer backward + optimizer """ + """test amp optimizer backward + optimizer""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = AMPOptimizer(opt) self.set_strategy(strategy, 'amp') @@ -83,7 +85,7 @@ class TestFleetAMPOptimizer(TestFleetMetaOptimizer): self.assertIn('check_finite_and_unscale', ops) def test_amp_optimizer(self): - """ test amp """ + """test amp""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -94,7 +96,7 @@ class TestFleetAMPOptimizer(TestFleetMetaOptimizer): self.assertIn('check_finite_and_unscale', ops) def test_pure_fp16_optimizer(self): - """ test pure fp16 """ + """test pure fp16""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'pure_fp16') @@ -109,7 +111,7 @@ class TestFleetAMPOptimizer(TestFleetMetaOptimizer): self.assertIn('check_finite_and_unscale', ops) def test_amp_distributed_optimizer(self): - """ test amp when distributed """ + """test amp when distributed""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -125,7 +127,7 @@ class TestFleetAMPOptimizer(TestFleetMetaOptimizer): self.assertEqual(check_count, len(train_prog.all_parameters())) def test_amp_recompute_optimizer(self): - """ test amp + recompute """ + """test amp + recompute""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -145,7 +147,7 @@ class TestFleetAMPOptimizer(TestFleetMetaOptimizer): self.assertIn('subprog', ''.join(outs)) def test_amp_recompute_lars_optimizer(self): - """ test amp + recompute """ + """test amp + recompute""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_checkpoint.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_checkpoint.py index ef091343204..a9c92849b01 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_checkpoint.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_checkpoint.py @@ -26,7 +26,6 @@ from paddle.fluid.incubate.checkpoint.checkpoint_saver import CheckpointSaver class FleetTest(unittest.TestCase): - def _test_checkpoint(self, fs, dir_path): file_name = "persistables" @@ -39,8 +38,9 @@ class FleetTest(unittest.TestCase): image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32') label = fluid.data(name='label', shape=[None, 1], dtype='int64') - feeder = fluid.DataFeeder(feed_list=[image, label], - place=fluid.CPUPlace()) + feeder = fluid.DataFeeder( + feed_list=[image, label], place=fluid.CPUPlace() + ) predict = fluid.layers.fc(input=image, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=predict, label=label) avg_loss = paddle.mean(loss) @@ -54,26 +54,24 @@ class FleetTest(unittest.TestCase): status = ExeTrainStatus() status.epoch_no = 2 - _, n1 = fleet.save_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status, - fs=fs) + _, n1 = fleet.save_checkpoint( + exe, dir_path, trainer_id=0, train_status=status, fs=fs + ) status2 = ExeTrainStatus() - fleet.load_checkpoint(exe, - dir_path, - trainer_id=0, - fs=fs, - train_status=status2) + fleet.load_checkpoint( + exe, dir_path, trainer_id=0, fs=fs, train_status=status2 + ) self.assertEqual(status2, status) - _, n2 = fleet.save_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status, - fs=fs, - remain_all_checkpoint=False) + _, n2 = fleet.save_checkpoint( + exe, + dir_path, + trainer_id=0, + train_status=status, + fs=fs, + remain_all_checkpoint=False, + ) self.assertEqual(n2, n1 + 1) c = CheckpointSaver(fs) @@ -82,36 +80,42 @@ class FleetTest(unittest.TestCase): # unnormal # test remain_all_checkpoint - fleet.save_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status, - fs=fs, - remain_all_checkpoint=False) + fleet.save_checkpoint( + exe, + dir_path, + trainer_id=0, + train_status=status, + fs=fs, + remain_all_checkpoint=False, + ) # can't save under a file fs = LocalFS() cache_path = "./.load_cache" fs.touch(cache_path) try: - fleet.save_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status, - fs=fs, - cache_path=cache_path) + fleet.save_checkpoint( + exe, + dir_path, + trainer_id=0, + train_status=status, + fs=fs, + cache_path=cache_path, + ) self.assertFalse(True) except: pass # can't load under a file try: - fleet.load_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status2, - fs=fs, - cache_path=cache_path) + fleet.load_checkpoint( + exe, + dir_path, + trainer_id=0, + train_status=status2, + fs=fs, + cache_path=cache_path, + ) self.assertFalse(True) except: pass diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_dgc_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_dgc_meta_optimizer.py index 522b563bc56..e75771e5c00 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_dgc_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_dgc_meta_optimizer.py @@ -25,15 +25,15 @@ paddle.enable_static() class TestFleetDGCOptimizer(TestFleetMetaOptimizer): - def test_dgc_optimizer_backward(self): - """ test dgc optimizer backward """ + """test dgc optimizer backward""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'dgc') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) dgc_opt = DGCOptimizer(opt) role = role_maker.PaddleCloudRoleMaker(is_collective=True) dgc_opt._set_basic_info(avg_cost, role, opt, strategy) @@ -43,13 +43,14 @@ class TestFleetDGCOptimizer(TestFleetMetaOptimizer): self.assertNotIn('dgc', ops) def test_dgc_optimizer_gradients(self): - """ test dgc optimizer backward + gradients """ + """test dgc optimizer backward + gradients""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'dgc') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) dgc_opt = DGCOptimizer(opt) role = role_maker.PaddleCloudRoleMaker(is_collective=True) dgc_opt._set_basic_info(avg_cost, role, opt, strategy) @@ -62,13 +63,14 @@ class TestFleetDGCOptimizer(TestFleetMetaOptimizer): self.assertIn('dgc_momentum', ops) def test_dgc_optimizer_optimize(self): - """ test dgc optimizer backward + optimize """ + """test dgc optimizer backward + optimize""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'dgc') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) dgc_opt = DGCOptimizer(opt) role = role_maker.PaddleCloudRoleMaker(is_collective=True) dgc_opt._set_basic_info(avg_cost, role, opt, strategy) @@ -130,8 +132,8 @@ class TestFleetDGCOptimizer(TestFleetMetaOptimizer): self.assertIn('subprog', ''.join(outs)) def test_amp_recompute_lars_dgc_not_apply_optimizer(self): - """ test amp + recompute + lars + dgc, - amp -/-> dgc, max_path is amp-->recompute-->lars + """test amp + recompute + lars + dgc, + amp -/-> dgc, max_path is amp-->recompute-->lars """ train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py index d2b57d9a8d0..51105cbcb10 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py @@ -17,7 +17,6 @@ import paddle class TestStrategyConfig(unittest.TestCase): - def test_amp(self): strategy = paddle.distributed.fleet.DistributedStrategy() strategy.amp = True @@ -35,7 +34,7 @@ class TestStrategyConfig(unittest.TestCase): "incr_every_n_steps": 1000, "incr_ratio": 2.0, "use_dynamic_loss_scaling": True, - "decr_ratio": 0.5 + "decr_ratio": 0.5, } strategy.amp_configs = configs self.assertEqual(strategy.amp_configs["init_loss_scaling"], 32768) @@ -78,7 +77,7 @@ class TestStrategyConfig(unittest.TestCase): strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": 2, - "pp_degree": 4 + "pp_degree": 4, } self.assertEqual(strategy.hybrid_configs["dp_degree"], 1) self.assertEqual(strategy.hybrid_configs["mp_degree"], 2) @@ -259,19 +258,26 @@ class TestStrategyConfig(unittest.TestCase): strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = { - "table_parameters.emb.accessor.embed_sgd_param.adagrad.learning_rate": - 0.05, + "table_parameters.emb.accessor.embed_sgd_param.adagrad.learning_rate": 0.05, "table_parameters.emb.accessor.table_accessor_save_param.num": 2, - "table_parameters.emb.accessor.table_accessor_save_param.param": - [1, 2] + "table_parameters.emb.accessor.table_accessor_save_param.param": [ + 1, + 2, + ], } strategy.sparse_table_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adagrad. - learning_rate, 0.05) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adagrad.learning_rate, + 0.05, + ) self.assertEqual( - strategy.sparse_table_configs[0].accessor. - table_accessor_save_param[0].param, 1) + strategy.sparse_table_configs[0] + .accessor.table_accessor_save_param[0] + .param, + 1, + ) strategy.adam_d2sum = True self.assertEqual(strategy.adam_d2sum, True) @@ -279,7 +285,7 @@ class TestStrategyConfig(unittest.TestCase): "uri": "123", "user": "456", "passwd": "789", - "hadoop_bin": "hadoop" + "hadoop_bin": "hadoop", } self.assertEqual(strategy.fs_client_param.user, "456") @@ -289,68 +295,90 @@ class TestStrategyConfig(unittest.TestCase): configs['emb'] = {"sparse_optimizer": "adagrad"} strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adagrad. - learning_rate, 0.05) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adagrad.learning_rate, + 0.05, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = {"sparse_optimizer": "naive"} strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.naive. - learning_rate, 0.05) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.naive.learning_rate, + 0.05, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = {"sparse_optimizer": "adam"} strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adam. - beta1_decay_rate, 0.9) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adam.beta1_decay_rate, + 0.9, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = { "sparse_accessor_class": "DownpourUnitAccessor", - "embed_sparse_optimizer": "std_adagrad" + "embed_sparse_optimizer": "std_adagrad", } strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.ctr_accessor_param. - show_scale, False) + strategy.sparse_table_configs[ + 0 + ].accessor.ctr_accessor_param.show_scale, + False, + ) self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adagrad. - initial_range, 0) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adagrad.initial_range, + 0, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = { "sparse_accessor_class": "DownpourCtrDoubleAccessor", - "embed_sparse_optimizer": "std_adagrad" + "embed_sparse_optimizer": "std_adagrad", } strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adagrad. - initial_range, 0.0001) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adagrad.initial_range, + 0.0001, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = {"sparse_optimizer": "shared_adam"} strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adam. - beta1_decay_rate, 0.9) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adam.beta1_decay_rate, + 0.9, + ) def test_trainer_desc_configs(self): strategy = paddle.distributed.fleet.DistributedStrategy() configs = { "dump_fields_path": "dump_data", "dump_fields": ["xxx", "yyy"], - "dump_param": ['zzz'] + "dump_param": ['zzz'], } strategy.trainer_desc_configs = configs - self.assertEqual(strategy.trainer_desc_configs["dump_fields_path"], - "dump_data") + self.assertEqual( + strategy.trainer_desc_configs["dump_fields_path"], "dump_data" + ) self.assertEqual(len(strategy.trainer_desc_configs["dump_fields"]), 2) self.assertEqual(len(strategy.trainer_desc_configs["dump_param"]), 1) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_fp16_allreduce_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_fp16_allreduce_meta_optimizer.py index 363843dd5e8..e170c9567ef 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_fp16_allreduce_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_fp16_allreduce_meta_optimizer.py @@ -23,27 +23,27 @@ paddle.enable_static() class TestFleetFP16CompressOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" def net(self, main_prog, startup_prog, dtype='float32'): with fluid.program_guard(main_prog, startup_prog): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype=dtype) - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype=dtype + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -62,7 +62,8 @@ class TestFleetFP16CompressOptimizer(unittest.TestCase): ops = [op.type for op in avg_cost.block.ops] cast_out = [ - op.output('Out')[0] for op in avg_cost.block.ops + op.output('Out')[0] + for op in avg_cost.block.ops if op.type == 'cast' ] diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_gradient_merge_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_gradient_merge_meta_optimizer.py index 99bc614b14e..459a3233148 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_gradient_merge_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_gradient_merge_meta_optimizer.py @@ -21,9 +21,10 @@ paddle.enable_static() class TestFleetGradientMergeMetaOptimizer(TestFleetMetaOptimizer): - def test_gradient_merge_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'gradient_merge') @@ -33,7 +34,9 @@ class TestFleetGradientMergeMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('@GradientMerge', ''.join(vars)) def test_recom_gm_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'gradient_merge') @@ -45,7 +48,9 @@ class TestFleetGradientMergeMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('subprog', ''.join(vars)) def test_gm_amp_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'gradient_merge') @@ -57,7 +62,9 @@ class TestFleetGradientMergeMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('cast', ''.join(vars)) def test_gm_pure_fp16_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'gradient_merge') @@ -67,8 +74,9 @@ class TestFleetGradientMergeMetaOptimizer(TestFleetMetaOptimizer): params = train_prog.all_parameters() for param in train_prog.all_parameters(): - self.assertEqual(param.dtype, - paddle.fluid.core.VarDesc.VarType.FP16) + self.assertEqual( + param.dtype, paddle.fluid.core.VarDesc.VarType.FP16 + ) vars = [x.name for x in train_prog.list_vars()] self.assertIn('@GradientMerge', ''.join(vars)) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py index bc6a554f84d..a146800d0cf 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): - def setUp(self): try: self._dist_ut_port_0 = int(os.environ["PADDLE_DIST_UT_PORT"]) @@ -34,58 +33,53 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): port_a = self._dist_ut_port_0 port_b = self._dist_ut_port_1 node_a = { - "PADDLE_TRAINER_ID": - "0", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_a), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "0", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } node_b = { - "PADDLE_TRAINER_ID": - "1", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_b), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "1", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } def node_func(): import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) @@ -102,60 +96,55 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): port_b = self._dist_ut_port_1 + 2 node_a = { - "PADDLE_TRAINER_ID": - "0", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_a), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "0", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } node_b = { - "PADDLE_TRAINER_ID": - "1", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_b), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "1", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } def node_func(): import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.nccl_comm_num = 2 strategy.sync_nccl_allreduce = True optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) exe.run(paddle.fluid.default_startup_program()) @@ -165,7 +154,7 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): def gen_data(): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } for i in range(10): @@ -182,58 +171,53 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): port_a = self._dist_ut_port_0 + 4 port_b = self._dist_ut_port_1 + 4 node_a = { - "PADDLE_TRAINER_ID": - "0", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_a), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "0", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } node_b = { - "PADDLE_TRAINER_ID": - "1", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_b), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "1", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } def node_func(): import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) @@ -249,60 +233,55 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): port_a = self._dist_ut_port_0 + 6 port_b = self._dist_ut_port_1 + 6 node_a = { - "PADDLE_TRAINER_ID": - "0", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_a), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "0", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } node_b = { - "PADDLE_TRAINER_ID": - "1", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_b), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "1", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } def node_func(): import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.nccl_comm_num = 2 strategy.sync_nccl_allreduce = True optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) exe.run(paddle.fluid.default_startup_program()) @@ -312,7 +291,7 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): def gen_data(): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } for i in range(10): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py index 3d30baba9a3..0a75eeac8cb 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py @@ -21,7 +21,6 @@ from launch_function_helper import launch_func class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): - def test_graph_execution_optimizer(self): node_a = { "PADDLE_TRAINER_ID": "0", @@ -30,7 +29,7 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:36001,127.0.0.1:36002", "http_proxy": "", "https_proxy": "", - "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1" + "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1", } node_b = { @@ -40,34 +39,36 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:36001,127.0.0.1:36002", "http_proxy": "", "https_proxy": "", - "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1" + "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1", } def node_func(): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.nccl_comm_num = 2 strategy.sync_nccl_allreduce = True optimizer = paddle.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) exe.run(paddle.fluid.default_startup_program()) @@ -77,7 +78,7 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): def gen_data(): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } for i in range(5): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_hybrid_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_hybrid_meta_optimizer.py index 3062812223d..5933b3a96a4 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_hybrid_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_hybrid_meta_optimizer.py @@ -24,11 +24,11 @@ paddle.enable_static() class TestFleetHybridOptimizer(TestFleetMetaOptimizer): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "3" - os.environ["PADDLE_TRAINER_ENDPOINTS"] = \ - "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" + os.environ[ + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" # pre-assigned ring id self.mp_ring_id = 0 self.sharding_ring_id = 1 @@ -65,40 +65,122 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'softmax', 'cross_entropy2', 'reduce_mean', - 'fill_constant', 'reduce_mean_grad', 'cross_entropy_grad2', - 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_sync_comm_stream', 'momentum', 'momentum', - 'momentum', 'momentum', 'momentum', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.dp_ring_id, created_ring_ids) @@ -107,16 +189,20 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group pp_group_waiting_prots = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and \ - op.desc.output_arg_names()[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_prots = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_prots, ['127.0.0.1:36003']) # check correctness of sharding group dp_group_waiting_ports = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" \ - and op.desc.output_arg_names()[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) @@ -147,34 +233,109 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'softmax', 'cross_entropy2', 'reduce_mean', - 'fill_constant', 'reduce_mean_grad', 'cross_entropy_grad2', - 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'coalesce_tensor', 'c_reduce_sum', 'coalesce_tensor', - 'c_reduce_sum', 'c_sync_comm_stream', 'momentum', 'momentum', - 'momentum', 'momentum', 'momentum', 'coalesce_tensor', - 'c_broadcast', 'coalesce_tensor', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'coalesce_tensor', + 'c_reduce_sum', + 'coalesce_tensor', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'coalesce_tensor', + 'c_broadcast', + 'coalesce_tensor', + 'c_broadcast', + ], + ) def test_opt_sharding_with_pp_amp_gclip(self): train_prog, startup_prog = static.Program(), static.Program() @@ -194,11 +355,9 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): strategy.fuse_grad_size_in_MB = 32 clip = paddle.fluid.clip.GradientClipByGlobalNorm(1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] self.debug_program(train_prog, startup_prog) @@ -211,46 +370,169 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'softmax', - 'cast', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'fill_constant', 'elementwise_mul_grad', 'reduce_mean_grad', - 'cross_entropy_grad2', 'cast', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'c_sync_calc_stream', - 'send_v2', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'coalesce_tensor', 'c_reduce_sum', 'coalesce_tensor', - 'c_reduce_sum', 'c_sync_comm_stream', 'check_finite_and_unscale', - 'cast', 'c_allreduce_max', 'c_allreduce_max', 'cast', - 'update_loss_scaling', 'squared_l2_norm', 'squared_l2_norm', - 'squared_l2_norm', 'squared_l2_norm', 'squared_l2_norm', 'sum', - 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', 'fill_constant', - 'elementwise_max', 'elementwise_div', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'coalesce_tensor', 'c_broadcast', 'coalesce_tensor', - 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'coalesce_tensor', + 'c_reduce_sum', + 'coalesce_tensor', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'coalesce_tensor', + 'c_broadcast', + 'coalesce_tensor', + 'c_broadcast', + ], + ) def test_opt_sharding_with_pp_amp_gclip_fuse_gm(self): train_prog, startup_prog = static.Program(), static.Program() @@ -271,11 +553,9 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): strategy.fuse_grad_merge = True clip = paddle.fluid.clip.GradientClipByGlobalNorm(1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] self.debug_program(train_prog, startup_prog) @@ -288,43 +568,151 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'softmax', - 'cast', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'cast', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'c_sync_calc_stream', - 'send_v2', 'cast', 'sum', 'cast', 'sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_sync_comm_stream', 'check_finite_and_unscale', - 'cast', 'c_allreduce_max', 'c_allreduce_max', 'cast', - 'update_loss_scaling', 'squared_l2_norm', 'squared_l2_norm', - 'squared_l2_norm', 'squared_l2_norm', 'squared_l2_norm', 'sum', - 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', 'fill_constant', - 'elementwise_max', 'elementwise_div', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'coalesce_tensor', 'c_broadcast', 'coalesce_tensor', - 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'cast', + 'sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'coalesce_tensor', + 'c_broadcast', + 'coalesce_tensor', + 'c_broadcast', + ], + ) def test_opt_sharding_with_pp_amp_ckp_fuse_gm_optcast(self): train_prog, startup_prog = static.Program(), static.Program() @@ -337,8 +725,12 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): } strategy.recompute = True strategy.recompute_configs = { - "checkpoints": - ["fc_0.tmp_2", "fc_1.tmp_2", "fc_2.tmp_2", "fc_3.tmp_2"] + "checkpoints": [ + "fc_0.tmp_2", + "fc_1.tmp_2", + "fc_2.tmp_2", + "fc_3.tmp_2", + ] } strategy.sharding = True @@ -368,49 +760,158 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', - 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', - 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', - 'cast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', - 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', 'mul', - 'elementwise_add', 'cast', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'coalesce_tensor', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'coalesce_tensor', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'cast', 'elementwise_add_grad', 'cast', - 'mul_grad', 'cast', 'tanh_grad', 'cast', 'elementwise_add_grad', - 'mul_grad', 'cast', 'tanh_grad', 'cast', 'elementwise_add_grad', - 'mul_grad', 'cast', 'cast', 'mul', 'elementwise_add', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'c_sync_calc_stream', 'send_v2', 'cast', 'sum', 'sum', 'cast', - 'sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'c_allreduce_max', 'cast', 'update_loss_scaling', - 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'cast', - 'momentum', 'momentum', 'cast', 'coalesce_tensor', 'c_broadcast', - 'c_broadcast', 'coalesce_tensor', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'cast', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'sum', + 'cast', + 'sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'momentum', + 'cast', + 'coalesce_tensor', + 'c_broadcast', + 'c_broadcast', + 'coalesce_tensor', + 'c_broadcast', + ], + ) class TestFleetHybridOptimizerBoundary(TestFleetMetaOptimizer): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "3" - os.environ["PADDLE_TRAINER_ENDPOINTS"] = \ - "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" + os.environ[ + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" # pre-assigned ring id self.mp_ring_id = 0 self.sharding_ring_id = 1 @@ -440,11 +941,9 @@ class TestFleetHybridOptimizerBoundary(TestFleetMetaOptimizer): strategy.fuse_grad_size_in_MB = 32 clip = paddle.fluid.clip.GradientClipByGlobalNorm(1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] self.debug_program(train_prog, startup_prog) @@ -461,32 +960,74 @@ class TestFleetHybridOptimizerBoundary(TestFleetMetaOptimizer): if is_loss_grad_op(op): self.assertEqual(op.type, 'fill_constant') self.assertTrue(op.has_attr('value')) - scale = strategy.pipeline_configs[ - 'accumulate_steps'] * strategy.sharding_configs['dp_degree'] + scale = ( + strategy.pipeline_configs['accumulate_steps'] + * strategy.sharding_configs['dp_degree'] + ) loss_scale = 1.0 / scale self.assertAlmostEqual(float(op.attr('value')), loss_scale) # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'matmul', 'cast', 'reduce_mean', - 'elementwise_mul', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cast', 'matmul_grad', 'c_sync_calc_stream', - 'send_v2', 'fill_constant', 'cast', 'sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'c_allreduce_max', 'cast', 'update_loss_scaling', - 'fill_constant', 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', - 'fill_constant', 'elementwise_max', 'elementwise_div', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'matmul', + 'cast', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cast', + 'matmul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'fill_constant', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'c_broadcast', + ], + ) def test_opt_sharding_with_pp_amp_gclip_boundary_card1(self): - """ test optimizer sharding without parameter in card0 """ + """test optimizer sharding without parameter in card0""" os.environ["PADDLE_TRAINER_ID"] = "1" train_prog, startup_prog = static.Program(), static.Program() avg_cost, strategy = self.boundary_net(train_prog, startup_prog) @@ -504,11 +1045,9 @@ class TestFleetHybridOptimizerBoundary(TestFleetMetaOptimizer): strategy.fuse_grad_size_in_MB = 32 clip = paddle.fluid.clip.GradientClipByGlobalNorm(1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] self.debug_program(train_prog, startup_prog) @@ -521,24 +1060,67 @@ class TestFleetHybridOptimizerBoundary(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'matmul', 'cast', 'reduce_mean', - 'elementwise_mul', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cast', 'matmul_grad', 'c_sync_calc_stream', - 'send_v2', 'fill_constant', 'cast', 'sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'c_allreduce_max', 'cast', 'update_loss_scaling', - 'squared_l2_norm', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', - 'sqrt', 'fill_constant', 'elementwise_max', 'elementwise_div', - 'elementwise_mul', 'momentum', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'matmul', + 'cast', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cast', + 'matmul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'squared_l2_norm', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'momentum', + 'c_broadcast', + ], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lamb_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lamb_meta_optimizer.py index 1c20d2e45be..a8fa8c7107b 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lamb_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lamb_meta_optimizer.py @@ -23,31 +23,32 @@ paddle.enable_static() class TestFleetLambMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): with fluid.unique_name.guard(): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - - fc_1 = paddle.fluid.layers.fc(input=input_x, - size=64, - act='tanh') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + + fc_1 = paddle.fluid.layers.fc( + input=input_x, size=64, act='tanh' + ) fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -78,8 +79,9 @@ class TestFleetLambMetaOptimizer(unittest.TestCase): startup_prog = fluid.Program() train_prog = fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.1, - momentum=0.9) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.1, momentum=0.9 + ) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) @@ -101,7 +103,8 @@ class TestFleetLambMetaOptimizer(unittest.TestCase): optimizer.minimize(avg_cost) ops_without_wd = [ - op for op in avg_cost.block.ops + op + for op in avg_cost.block.ops if op.type == 'lamb' and op.attr('op_role_var')[0].endswith('.b_0') ] for op in ops_without_wd: @@ -110,16 +113,17 @@ class TestFleetLambMetaOptimizer(unittest.TestCase): def test_lamb_apply_with_amp(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lars_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lars_meta_optimizer.py index b560cdaa66e..0a7298631cf 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lars_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lars_meta_optimizer.py @@ -23,31 +23,32 @@ paddle.enable_static() class TestFleetLarsMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): with fluid.unique_name.guard(): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - - fc_1 = paddle.fluid.layers.fc(input=input_x, - size=64, - act='tanh') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + + fc_1 = paddle.fluid.layers.fc( + input=input_x, size=64, act='tanh' + ) fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -67,8 +68,9 @@ class TestFleetLarsMetaOptimizer(unittest.TestCase): startup_prog = fluid.Program() train_prog = fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.01, - momentum=0.9) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9 + ) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) @@ -94,16 +96,21 @@ class TestFleetLarsMetaOptimizer(unittest.TestCase): startup_prog = fluid.Program() train_prog = fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.01, - momentum=0.9) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9 + ) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) ops_without_wd = [ - op for op in avg_cost.block.ops - if op.type == 'lars_momentum' and ("batch_norm" in op.attr( - 'op_role_var')[0] or ".b" in op.attr('op_role_var')[0]) + op + for op in avg_cost.block.ops + if op.type == 'lars_momentum' + and ( + "batch_norm" in op.attr('op_role_var')[0] + or ".b" in op.attr('op_role_var')[0] + ) ] for op in ops_without_wd: self.assertEqual(op.attr('lars_weight_decay')[0], 0) @@ -111,16 +118,17 @@ class TestFleetLarsMetaOptimizer(unittest.TestCase): def test_lars_apply_with_amp(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -143,8 +151,9 @@ class TestFleetLarsMetaOptimizer(unittest.TestCase): "exclude_from_weight_decay": ["batch_norm", ".b"], } - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.01, - momentum=0.9) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9 + ) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_localsgd_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_localsgd_meta_optimizer.py index 2765b8c3eae..a3711c6f841 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_localsgd_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_localsgd_meta_optimizer.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestFleetLocalSGDMetaOptimizer(TestFleetMetaOptimizer): - def test_localsgd_optimizer(self): train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) @@ -32,7 +31,8 @@ class TestFleetLocalSGDMetaOptimizer(TestFleetMetaOptimizer): ops = [op.type for op in avg_cost.block.ops] outs = [ - ''.join(op.output('Out')) for op in avg_cost.block.ops + ''.join(op.output('Out')) + for op in avg_cost.block.ops if op.type == 'conditional_block' ] @@ -48,7 +48,8 @@ class TestFleetLocalSGDMetaOptimizer(TestFleetMetaOptimizer): ops = [op.type for op in avg_cost.block.ops] outs = [ - ''.join(op.output('Out')) for op in avg_cost.block.ops + ''.join(op.output('Out')) + for op in avg_cost.block.ops if op.type == 'conditional_block' ] @@ -61,7 +62,6 @@ class TestFleetLocalSGDMetaOptimizer(TestFleetMetaOptimizer): class TestFleetAdaptiveLocalSGDMetaOptimizer(TestFleetMetaOptimizer): - def test_adaptive_localsgd_optimizer(self): train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) @@ -70,7 +70,8 @@ class TestFleetAdaptiveLocalSGDMetaOptimizer(TestFleetMetaOptimizer): ops = [op.type for op in avg_cost.block.ops] outs = [ - ''.join(op.output('Out')) for op in avg_cost.block.ops + ''.join(op.output('Out')) + for op in avg_cost.block.ops if op.type == 'conditional_block' ] @@ -86,7 +87,8 @@ class TestFleetAdaptiveLocalSGDMetaOptimizer(TestFleetMetaOptimizer): ops = [op.type for op in avg_cost.block.ops] outs = [ - ''.join(op.output('Out')) for op in avg_cost.block.ops + ''.join(op.output('Out')) + for op in avg_cost.block.ops if op.type == 'conditional_block' ] diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_log.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_log.py index 985905bdc81..93027dfb8b7 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_log.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_log.py @@ -19,7 +19,6 @@ import unittest class TestFleetLog(unittest.TestCase): - def setUp(self): fleet.init(log_level="DEBUG") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_meta_optimizer_base.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_meta_optimizer_base.py index 0c8db67dcb5..f127e072dd1 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_meta_optimizer_base.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_meta_optimizer_base.py @@ -17,40 +17,44 @@ import paddle from paddle import fluid import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker -from paddle.distributed.fleet.meta_optimizers.meta_optimizer_base import MetaOptimizerBase +from paddle.distributed.fleet.meta_optimizers.meta_optimizer_base import ( + MetaOptimizerBase, +) class TestFleetMetaOptimizerBase(unittest.TestCase): - def net(main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): with fluid.unique_name.guard(): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - - fc_1 = paddle.fluid.layers.fc(input=input_x, - size=64, - act='tanh') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + + fc_1 = paddle.fluid.layers.fc( + input=input_x, size=64, act='tanh' + ) fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) opt = MetaOptimizerBase(optimizer) opt_ops, params_grads = opt.minimize(avg_cost) - opt.apply_optimize(avg_cost, - paddle.static.default_startup_program(), - params_grads) + opt.apply_optimize( + avg_cost, + paddle.static.default_startup_program(), + params_grads, + ) return None net(fluid.default_startup_program(), fluid.default_main_program()) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer.py index 279a2e21f70..bee864e1f24 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer.py @@ -24,23 +24,23 @@ paddle.enable_static() class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def net(self): with static.device_guard("gpu:0"): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - input_z = paddle.fluid.layers.data(name="z", - shape=[1], - dtype="float32") + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + input_z = paddle.fluid.layers.data( + name="z", shape=[1], dtype="float32" + ) with static.device_guard("gpu:all"): input_z = input_z * 1.0 input_z.stop_gradient = True @@ -52,11 +52,12 @@ class TestFleetMetaOptimizer(unittest.TestCase): # for pipeline check_pipeline_persist_var coverage fc_2.persistable = True fc_2 = fc_2 * input_z - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) return avg_cost @@ -68,7 +69,7 @@ class TestFleetMetaOptimizer(unittest.TestCase): strategy.pipeline = True strategy.pipeline_configs = { 'micro_batch_size': 1, - 'accumulate_steps': 2 + 'accumulate_steps': 2, } train_prog, startup_prog = static.Program(), static.Program() @@ -77,12 +78,13 @@ class TestFleetMetaOptimizer(unittest.TestCase): avg_cost = self.net() optimizer = paddle.fluid.optimizer.Adam(0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) def test_pipeline_amp_optimizer(self): - """ test pipeline& with device:all """ + """test pipeline& with device:all""" role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) @@ -91,7 +93,7 @@ class TestFleetMetaOptimizer(unittest.TestCase): strategy.pipeline = True strategy.pipeline_configs = { 'micro_batch_size': 1, - 'accumulate_steps': 2 + 'accumulate_steps': 2, } train_prog, startup_prog = static.Program(), static.Program() @@ -100,8 +102,9 @@ class TestFleetMetaOptimizer(unittest.TestCase): avg_cost = self.net() optimizer = paddle.fluid.optimizer.Adam(0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) ops = train_prog._pipeline_opt['section_program'].global_block().ops diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer_with_recompute.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer_with_recompute.py index c45c81c35b4..8ce94b61a63 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer_with_recompute.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer_with_recompute.py @@ -20,24 +20,25 @@ paddle.enable_static() class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def test_pipeline_optimizer(self): import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) with paddle.fluid.device_guard("gpu:0"): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') fc_3 = paddle.fluid.layers.fc(input=fc_2, size=64, act='tanh') @@ -47,11 +48,12 @@ class TestFleetMetaOptimizer(unittest.TestCase): with paddle.fluid.device_guard("gpu:1"): fc_7 = paddle.fluid.layers.fc(input=fc_6, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_7], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_7], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -59,7 +61,7 @@ class TestFleetMetaOptimizer(unittest.TestCase): strategy.pipeline_configs = { 'micro_batch_size': 1, 'accumulate_steps': 2, - 'schedule_mode': '1F1B' + 'schedule_mode': '1F1B', } checkpoints = ['fc_5.tmp_0', 'fc_7.tmp_0'] @@ -67,7 +69,7 @@ class TestFleetMetaOptimizer(unittest.TestCase): strategy.recompute_configs = { "checkpoints": checkpoints, "enable_offload": False, - "checkpoint_shape": [] + "checkpoint_shape": [], } optimizer = paddle.fluid.optimizer.Adam(0.01) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_private_function.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_private_function.py index a4854c695f8..94c8437167e 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_private_function.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_private_function.py @@ -18,11 +18,10 @@ import threading class TestFleetPrivateFunction(unittest.TestCase): - def test_wait_port(self): - def init_server(port): import time + time.sleep(5) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(("127.0.0.1", port)) @@ -33,10 +32,11 @@ class TestFleetPrivateFunction(unittest.TestCase): c.close() break - thr = threading.Thread(target=init_server, args=(9292, )) + thr = threading.Thread(target=init_server, args=(9292,)) thr.start() import paddle.distributed.fleet as fleet + ep = ["127.0.0.1:9292"] fleet.base.private_helper_function.wait_server_ready(ep) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_raw_program_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_raw_program_meta_optimizer.py index 3fde52958d3..e985415d943 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_raw_program_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_raw_program_meta_optimizer.py @@ -20,27 +20,29 @@ paddle.enable_static() class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def test_pipeline_optimizer(self): import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_recompute_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_recompute_meta_optimizer.py index 0045f7311de..9de35f8ca35 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_recompute_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_recompute_meta_optimizer.py @@ -22,15 +22,15 @@ paddle.enable_static() class TestFleetRecomputeMetaOptimizer(TestFleetMetaOptimizer): - def test_recompute_optimizer_backward(self): - """ test recompute optimizer backward """ + """test recompute optimizer backward""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'recompute') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = RecomputeOptimizer(opt) opt.user_defined_strategy = strategy params_grads = opt.backward(avg_cost, startup_prog) @@ -41,13 +41,14 @@ class TestFleetRecomputeMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('subprog', ''.join(outs)) def test_recompute_optimizer_backward_gradients(self): - """ test recompute optimizer backward + gradients """ + """test recompute optimizer backward + gradients""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'recompute') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = RecomputeOptimizer(opt) opt.user_defined_strategy = strategy params_grads = opt.backward(avg_cost, startup_prog) @@ -60,13 +61,14 @@ class TestFleetRecomputeMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('subprog', ''.join(outs)) def test_recompute_optimizer_backward_optimize(self): - """ test recompute optimizer backward + optimize """ + """test recompute optimizer backward + optimize""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'recompute') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = RecomputeOptimizer(opt) opt.user_defined_strategy = strategy params_grads = opt.backward(avg_cost, startup_prog) @@ -126,7 +128,8 @@ class TestFleetRecomputeMetaOptimizer(TestFleetMetaOptimizer): self.optimizer(avg_cost, strategy, train_prog, startup_prog) ops = [op.type for op in avg_cost.block.ops] outs = [ - op.output('Out')[0] for op in avg_cost.block.ops + op.output('Out')[0] + for op in avg_cost.block.ops if op.type == 'memcpy' ] self.assertIn('memcpy', ops) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_rolemaker_new.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_rolemaker_new.py index 18ef1f78633..18c2487b66a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_rolemaker_new.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_rolemaker_new.py @@ -59,9 +59,11 @@ class TestCloudRoleMaker(unittest.TestCase): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ["POD_IP"] = "127.0.0.1" def test_tr_rolemaker(self): @@ -103,8 +105,9 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" - ro = role_maker.PaddleCloudRoleMaker(is_collective=False, - init_gloo=False) + ro = role_maker.PaddleCloudRoleMaker( + is_collective=False, init_gloo=False + ) self.assertEqual(ro._server_index(), 0) self.assertFalse(ro._is_worker()) self.assertTrue(ro._is_server()) @@ -139,7 +142,8 @@ class TestUserDefinedRoleMaker(unittest.TestCase): server_endpoints=["127.0.0.1:36001", "127.0.0.1:36001"], role=role_maker.Role.SERVER, current_id=0, - worker_num=2) + worker_num=2, + ) self.assertEqual(ro._server_num(), 2) ro._generate_role() self.assertTrue(ro._is_server()) @@ -152,7 +156,8 @@ class TestUserDefinedRoleMaker(unittest.TestCase): server_endpoints=["127.0.0.1:36001", "127.0.0.1:36001"], role=role_maker.Role.WORKER, current_id=0, - worker_num=2) + worker_num=2, + ) self.assertIn("127.0.0.1:36001", ro._get_pserver_endpoints()) self.assertTrue(ro._is_worker()) @@ -160,7 +165,6 @@ class TestUserDefinedRoleMaker(unittest.TestCase): class TestGlooWithCloudRoleMaker(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINERS_NUM"] = "1" os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001" @@ -287,6 +291,7 @@ class TestGlooWithCloudRoleMaker(unittest.TestCase): role = role_maker.PaddleCloudRoleMaker(is_collective=True) role._generate_role() import time + time.sleep(3) def test_fs_gloo5(self): @@ -443,8 +448,9 @@ class TestGlooWithCloudRoleMaker(unittest.TestCase): x = paddle.fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = paddle.fluid.layers.fc(input=x, size=1, act=None) y = paddle.fluid.layers.data(name='y', shape=[1], dtype='float32') - cost = paddle.fluid.layers.square_error_cost(input=y_predict, - label=y) + cost = paddle.fluid.layers.square_error_cost( + input=y_predict, label=y + ) avg_cost = paddle.mean(cost) return avg_cost diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py index 6c14b49278c..57a199c1333 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py @@ -25,9 +25,10 @@ paddle.enable_static() class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): - def test_sharding_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -40,28 +41,74 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('@BroadCast', ''.join(vars)) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0" - ])) - - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'momentum', 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_amp_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -77,33 +124,101 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('check_finite_and_unscale', ops) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0", - "loss_scaling_0", "num_bad_steps_0", "num_good_steps_0" - ])) - - self.assertEqual(ops, [ - 'cast', 'cast', 'cast', 'fill_constant', 'fill_constant', - 'fill_constant', 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_sync_comm_stream', 'cast', 'mul', 'elementwise_add', 'cast', - 'tanh', 'cast', 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', - 'mul', 'elementwise_add', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'cast', 'cast', 'cast', - 'check_finite_and_unscale', 'cast', 'c_allreduce_max', 'cast', - 'update_loss_scaling', 'momentum', 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + "loss_scaling_0", + "num_bad_steps_0", + "num_good_steps_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'cast', + 'cast', + 'cast', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'cast', + 'cast', + 'cast', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_recompute_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -120,29 +235,78 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('subprog', ''.join(vars)) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0" - ])) - - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'mul', 'elementwise_add', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'mul', 'elementwise_add', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'momentum', 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'mul', + 'elementwise_add', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'mul', + 'elementwise_add', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_amp_recompute_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -163,36 +327,115 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0", - "loss_scaling_0", "num_bad_steps_0", "num_good_steps_0" - ])) - self.assertEqual(ops, [ - 'cast', 'cast', 'cast', 'cast', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'cast', 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', 'mul', - 'elementwise_add', 'cast', 'tanh', 'cast', 'mul', 'elementwise_add', - 'softmax', 'cast', 'cross_entropy2', 'reduce_mean', - 'elementwise_mul', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'cast', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'cast', 'cast', 'mul', - 'elementwise_add', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'mul', - 'elementwise_add', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_sync_comm_stream', 'cast', - 'cast', 'cast', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + "loss_scaling_0", + "num_bad_steps_0", + "num_good_steps_0", + ] + ), + ) + self.assertEqual( + ops, + [ + 'cast', + 'cast', + 'cast', + 'cast', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'cast', + 'cast', + 'cast', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_amp_asp_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -212,44 +455,115 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): self.assertEqual( set(parameters), - set([ - 'fc_2.b_0', 'num_good_steps_0', 'fc_2.w_0', 'loss_scaling_0', - 'num_bad_steps_0', 'fc_2.w_0_velocity_0', 'fc_2.w_0.asp_mask', - 'learning_rate_0', 'fc_1.b_0', 'fc_1.w_0.asp_mask', - 'fc_0.w_0.asp_mask', 'fc_1.b_0_velocity_0', - 'fc_2.b_0_velocity_0' - ])) - self.assertEqual(ops, [ - 'cast', 'cast', 'cast', 'fill_constant', 'fill_constant', - 'fill_constant', 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_sync_comm_stream', 'cast', 'mul', 'elementwise_add', 'cast', - 'tanh', 'cast', 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', - 'mul', 'elementwise_add', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'cast', 'cast', 'cast', - 'check_finite_and_unscale', 'cast', 'c_allreduce_max', 'cast', - 'update_loss_scaling', 'momentum', 'momentum', 'momentum', - 'elementwise_mul' - ]) + set( + [ + 'fc_2.b_0', + 'num_good_steps_0', + 'fc_2.w_0', + 'loss_scaling_0', + 'num_bad_steps_0', + 'fc_2.w_0_velocity_0', + 'fc_2.w_0.asp_mask', + 'learning_rate_0', + 'fc_1.b_0', + 'fc_1.w_0.asp_mask', + 'fc_0.w_0.asp_mask', + 'fc_1.b_0_velocity_0', + 'fc_2.b_0_velocity_0', + ] + ), + ) + self.assertEqual( + ops, + [ + 'cast', + 'cast', + 'cast', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'cast', + 'cast', + 'cast', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + 'elementwise_mul', + ], + ) def test_sharding_weight_decay(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') regularization = paddle.fluid.regularizer.L2Decay(0.0001) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - regularization=regularization) + self.optimizer( + avg_cost, + strategy, + train_prog, + startup_prog, + regularization=regularization, + ) parameters = [ x.name for x in train_prog.list_vars() if x.persistable == True ] @@ -258,38 +572,87 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('@BroadCast', ''.join(vars)) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0" - ])) - - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'scale', 'sum', 'scale', 'sum', 'scale', - 'sum', 'momentum', 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'scale', + 'sum', + 'scale', + 'sum', + 'scale', + 'sum', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_gradient_clip(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') clip = paddle.fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) parameters = [ x.name for x in train_prog.list_vars() if x.persistable == True ] @@ -298,32 +661,86 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): self.assertIn('@BroadCast', ''.join(vars)) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0" - ])) - - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'squared_l2_norm', 'squared_l2_norm', - 'squared_l2_norm', 'sum', 'c_allreduce_sum', 'sqrt', - 'fill_constant', 'elementwise_max', 'elementwise_div', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', 'momentum', - 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_clone_for_test(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -334,21 +751,41 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): sharding.utils.add_sync_comm(test_prog, 1) ops = [op.type for op in test_prog.global_block().ops] - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', 'reduce_mean' - ]) + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + ], + ) class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "3" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" # pre-assigned ring id self.mp_ring_id = 0 @@ -359,7 +796,9 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): def test_sharding_with_mp(self): # NOTE(JZ-LIANG) MP parallelism need user to build model with MP API - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, _ = self.net(train_prog, startup_prog) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -371,7 +810,7 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): "sharding_degree": 2, "hybrid_dp": False, "gradient_merge_acc_step": 1, - "mp_degree": 2 + "mp_degree": 2, } self.optimizer(avg_cost, strategy, train_prog, startup_prog) startup_prog_ops = startup_prog.global_block().ops @@ -379,7 +818,8 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # should has ring id for MP created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.mp_ring_id, created_ring_ids) @@ -387,8 +827,10 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of MP group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) @@ -396,14 +838,18 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_sharding_hybrid_dp(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, _ = self.net(train_prog, startup_prog) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -416,7 +862,7 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): "dp_degree": 2, "hybrid_dp": True, "gradient_merge_acc_step": 1, - "mp_degree": 1 + "mp_degree": 1, } strategy.fuse_all_reduce_ops = False @@ -426,7 +872,8 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check ring id for outter dp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.dp_ring_id, created_ring_ids) @@ -434,8 +881,10 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) @@ -443,8 +892,10 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of dp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) @@ -453,32 +904,74 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): if is_loss_grad_op(op): self.assertEqual(op.type, 'fill_constant') self.assertTrue(op.has_attr('value')) - scale = strategy.sharding_configs[ - 'sharding_degree'] * strategy.sharding_configs['dp_degree'] + scale = ( + strategy.sharding_configs['sharding_degree'] + * strategy.sharding_configs['dp_degree'] + ) loss_scale = 1.0 / scale self.assertAlmostEqual(float(op.attr('value')), loss_scale) # check program (allreudce) ops = [op.type for op in main_prog_ops] - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'c_allreduce_sum', 'c_allreduce_sum', - 'c_allreduce_sum', 'c_sync_comm_stream', 'momentum', 'momentum', - 'momentum' - ]) + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_hybrid_dp_gm(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, _ = self.net(train_prog, startup_prog) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -491,7 +984,7 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): "dp_degree": 2, "hybrid_dp": True, "gradient_merge_acc_step": 4, - "mp_degree": 1 + "mp_degree": 1, } self.optimizer(avg_cost, strategy, train_prog, startup_prog) startup_prog_ops = startup_prog.global_block().ops @@ -499,7 +992,8 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check ring id for outter dp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.dp_ring_id, created_ring_ids) @@ -507,8 +1001,10 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) @@ -516,35 +1012,87 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of dp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) # check program fw_bw_ops = [op.type for op in train_prog.blocks[0].ops] opt_ops = [op.type for op in train_prog.blocks[2].ops] - self.assertEqual(fw_bw_ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'elementwise_add', 'elementwise_add', - 'elementwise_add', 'increment', 'elementwise_mod', 'equal', - 'conditional_block' - ]) - self.assertEqual(opt_ops, [ - 'c_allreduce_sum', 'c_allreduce_sum', 'c_allreduce_sum', 'scale', - 'scale', 'scale', 'momentum', 'momentum', 'momentum', - 'fill_constant', 'fill_constant', 'fill_constant' - ]) + self.assertEqual( + fw_bw_ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'elementwise_add', + 'elementwise_add', + 'elementwise_add', + 'increment', + 'elementwise_mod', + 'equal', + 'conditional_block', + ], + ) + self.assertEqual( + opt_ops, + [ + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'scale', + 'scale', + 'scale', + 'momentum', + 'momentum', + 'momentum', + 'fill_constant', + 'fill_constant', + 'fill_constant', + ], + ) # # check loss scale for gradient merge scale_ = -1 @@ -554,7 +1102,9 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): self.assertEqual(scale_, 0.25) def test_sharding_with_pp(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.sharding = True @@ -565,7 +1115,7 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): "hybrid_dp": False, "gradient_merge_acc_step": 4, "mp_degree": 1, - "pp_degree": 2 + "pp_degree": 2, } strategy.pipeline = True strategy.pipeline_configs = { @@ -585,39 +1135,112 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] print(startup_prog_op_types) # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init' - ]) - - self.assertEqual(main_prog_op_types, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_sync_comm_stream', 'recv_v2', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'softmax', - 'cross_entropy2', 'reduce_mean', 'fill_constant', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_sync_comm_stream', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'c_sync_comm_stream', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.sharding_ring_id, created_ring_ids) @@ -626,8 +1249,10 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) @@ -635,14 +1260,18 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_sharding_dp_with_allreduce_fuse(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, _ = self.net(train_prog, startup_prog) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -655,7 +1284,7 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): "dp_degree": 2, "hybrid_dp": True, "gradient_merge_acc_step": 1, - "mp_degree": 1 + "mp_degree": 1, } strategy.fuse_all_reduce_ops = True strategy.fuse_grad_size_in_MB = 2 @@ -672,7 +1301,9 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): assert 'FusedGrad' in op.input_arg_names[0] def test_hybrid_with_mp_pp_amp_gclip(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -690,11 +1321,9 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): "accumulate_steps": 4, } clip = paddle.fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] @@ -706,45 +1335,166 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'partial_recv', 'partial_allgather', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'c_sync_calc_stream', 'partial_send', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'c_sync_comm_stream', - 'check_finite_and_unscale', 'cast', 'c_allreduce_max', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'fill_constant', - 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', 'fill_constant', - 'elementwise_max', 'elementwise_div', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'partial_recv', + 'partial_allgather', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'partial_send', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'fill_constant', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) # pp + mp, partial send recv self.assertIn('partial_recv', main_prog_op_types) @@ -759,7 +1509,8 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.mp_ring_id, created_ring_ids) @@ -768,8 +1519,10 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): mp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(mp_group_waiting_ports, ['127.0.0.1:36003']) @@ -777,14 +1530,18 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_mp_pp_amp_gclip_for_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -802,12 +1559,14 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): "accumulate_steps": 4, } clip = paddle.fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip, - name="adamw") + self.optimizer( + avg_cost, + strategy, + train_prog, + startup_prog, + grad_clip=clip, + name="adamw", + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] @@ -819,51 +1578,191 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'partial_recv', 'partial_allgather', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'c_sync_calc_stream', 'partial_send', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'c_sync_comm_stream', - 'check_finite_and_unscale', 'cast', 'c_allreduce_max', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'memcpy', - 'fill_constant', 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', - 'fill_constant', 'elementwise_max', 'elementwise_div', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'adamw', 'adamw', 'adamw', - 'adamw', 'adamw', 'adamw', 'adamw', 'adamw' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'partial_recv', + 'partial_allgather', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'partial_send', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'memcpy', + 'fill_constant', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + ], + ) # pp + mp, partial send recv self.assertIn('partial_recv', main_prog_op_types) @@ -878,7 +1777,8 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.mp_ring_id, created_ring_ids) @@ -887,8 +1787,10 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): mp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(mp_group_waiting_ports, ['127.0.0.1:36003']) @@ -896,14 +1798,18 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_fp16allreduce(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -936,46 +1842,145 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', - 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', 'elementwise_add', - 'softmax', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'fill_constant', 'elementwise_mul_grad', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'cast', 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'cast', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'coalesce_tensor', 'c_allreduce_sum', - 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'coalesce_tensor', + 'c_allreduce_sum', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -983,22 +1988,28 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_sharding_pp_amp_fp16allreduce_in_optimize(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1033,14 +2044,33 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # ring: sharding, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + ], + ) # FIXME(wangxi): some bug in sharding+pp with pp_allreduce_in_optimize # self.assertEqual(main_prog_op_types, []) @@ -1050,7 +2080,8 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.sharding_ring_id, created_ring_ids) @@ -1058,22 +2089,28 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of sharding group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_cast(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1107,48 +2144,152 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'cast', - 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', - 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', - 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'cast', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'softmax_grad', 'elementwise_add_grad', 'cast', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'cast', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'coalesce_tensor', 'c_allreduce_sum', - 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'cast', 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'cast', - 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'momentum', - 'cast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'coalesce_tensor', + 'c_allreduce_sum', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'momentum', + 'cast', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -1156,22 +2297,28 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_offload(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1205,51 +2352,173 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'cast', 'memcpy', - 'c_broadcast', 'cast', 'memcpy', 'c_broadcast', 'cast', 'memcpy', - 'c_broadcast', 'cast', 'memcpy', 'c_broadcast', 'cast', 'memcpy', - 'c_broadcast', 'cast', 'memcpy', 'c_broadcast', 'cast', 'memcpy', - 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'cast', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'softmax_grad', 'elementwise_add_grad', 'cast', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'cast', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'coalesce_tensor', 'c_allreduce_sum', - 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'memcpy', - 'momentum', 'cast', 'memcpy', 'memcpy', 'momentum', 'cast', - 'memcpy', 'memcpy', 'momentum', 'cast', 'memcpy', 'memcpy', - 'momentum', 'cast', 'memcpy', 'memcpy', 'momentum', 'cast', - 'memcpy', 'memcpy', 'momentum', 'cast', 'memcpy', 'momentum', - 'memcpy', 'momentum', 'cast', 'memcpy' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'coalesce_tensor', + 'c_allreduce_sum', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'momentum', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -1257,23 +2526,30 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_cast_with_gradient_fuse( - self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + self, + ): + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1308,46 +2584,142 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'cast', - 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', - 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', - 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'cast', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'coalesce_tensor', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'fill_constant', 'elementwise_mul_grad', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'cast', 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'sum', 'cast', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', 'cast', - 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'cast', 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'cast', - 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'momentum', - 'cast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'sum', + 'cast', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'momentum', + 'cast', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -1355,22 +2727,28 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_with_gradient_fuse(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1403,44 +2781,127 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', - 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', 'elementwise_add', - 'softmax', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'cast', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'cast', 'sum', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -1448,22 +2909,28 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_with_gradient_fuse_and_avg_after_sum(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1481,11 +2948,11 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): strategy.pipeline_configs = { "schedule_mode": "1F1B", "micro_batch_size": 2, - "accumulate_steps": 4 + "accumulate_steps": 4, } strategy.gradient_scale_configs = { 'scale_strategy': 'avg', - 'scale_gradient': True + 'scale_gradient': True, } strategy.fuse_grad_merge = True self.optimizer(avg_cost, strategy, train_prog, startup_prog) @@ -1499,40 +2966,125 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): startup_prog_op_types = [op.type for op in startup_prog_ops] main_prog_op_types = [op.type for op in main_prog_ops] - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', - 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', 'elementwise_add', - 'softmax', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'cast', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'cast', 'sum', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', - 'c_sync_comm_stream', 'scale', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'scale', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_hybrid_with_pp_dp_with_gradient_fuse_and_avg_after_sum(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.sharding = True @@ -1546,11 +3098,11 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): strategy.pipeline_configs = { "schedule_mode": "1F1B", "micro_batch_size": 2, - "accumulate_steps": 4 + "accumulate_steps": 4, } strategy.gradient_scale_configs = { 'scale_strategy': 'avg', - 'scale_gradient': True + 'scale_gradient': True, } strategy.fuse_grad_merge = True self.optimizer(avg_cost, strategy, train_prog, startup_prog) @@ -1564,36 +3116,103 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): startup_prog_op_types = [op.type for op in startup_prog_ops] main_prog_op_types = [op.type for op in main_prog_ops] - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'softmax', 'cross_entropy2', 'reduce_mean', - 'coalesce_tensor', 'coalesce_tensor', 'fill_constant', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'sum', 'c_allreduce_sum', 'c_sync_comm_stream', 'scale', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'scale', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_hybrid_with_pp_dp_with_amp_no_dynamic_gradient_fuse_and_avg_after_sum( - self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + self, + ): + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.sharding = True @@ -1606,17 +3225,17 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): strategy.amp = True strategy.amp_configs = { 'custom_black_varnames': ['fc_6.b_0'], - 'use_dynamic_loss_scaling': False + 'use_dynamic_loss_scaling': False, } strategy.pipeline = True strategy.pipeline_configs = { "schedule_mode": "1F1B", "micro_batch_size": 2, - "accumulate_steps": 4 + "accumulate_steps": 4, } strategy.gradient_scale_configs = { 'scale_strategy': 'avg', - 'scale_gradient': True + 'scale_gradient': True, } strategy.fuse_grad_merge = True self.optimizer(avg_cost, strategy, train_prog, startup_prog) @@ -1630,35 +3249,115 @@ class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): startup_prog_op_types = [op.type for op in startup_prog_ops] main_prog_op_types = [op.type for op in main_prog_ops] - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', - 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', 'elementwise_add', - 'softmax', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'cast', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'cast', 'sum', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', - 'c_sync_comm_stream', 'scale', 'scale', 'check_finite_and_unscale', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'scale', + 'scale', + 'check_finite_and_unscale', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_static_mp_layers.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_static_mp_layers.py index 19f926748cf..fe583a4cbc6 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_static_mp_layers.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_static_mp_layers.py @@ -23,7 +23,6 @@ paddle.enable_static() class ColumnLinearNet(fluid.dygraph.Layer): - def __init__(self, input_size, output_size): super(ColumnLinearNet, self).__init__() self.parallel_linear = fleet.meta_parallel.ColumnParallelLinear( @@ -32,7 +31,8 @@ class ColumnLinearNet(fluid.dygraph.Layer): weight_attr=None, has_bias=True, gather_output=True, - name="test_column_linear") + name="test_column_linear", + ) def forward(self, x): output = self.parallel_linear(x) @@ -40,7 +40,6 @@ class ColumnLinearNet(fluid.dygraph.Layer): class RowLinearNet(fluid.dygraph.Layer): - def __init__(self, input_size, output_size): super(RowLinearNet, self).__init__() self.parallel_linear = fleet.meta_parallel.RowParallelLinear( @@ -48,7 +47,8 @@ class RowLinearNet(fluid.dygraph.Layer): out_features=output_size, has_bias=True, input_is_parallel=False, - name="test_row_linear") + name="test_row_linear", + ) def forward(self, x): output = self.parallel_linear(x) @@ -56,11 +56,11 @@ class RowLinearNet(fluid.dygraph.Layer): class EmbeddingNet(fluid.dygraph.Layer): - def __init__(self, vocab_size, hidden_size): super(EmbeddingNet, self).__init__() self.embedding = fleet.meta_parallel.VocabParallelEmbedding( - vocab_size, hidden_size) + vocab_size, hidden_size + ) def forward(self, x): output = self.embedding(x) @@ -68,11 +68,11 @@ class EmbeddingNet(fluid.dygraph.Layer): class TestDistTraning(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "2" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 @@ -97,19 +97,22 @@ class TestDistTraning(unittest.TestCase): x = paddle.static.data(name='x', shape=[None, input_size]) y = model_a(x) - #print(main_program) + # print(main_program) ops = main_program.global_block().ops ops = [op.type for op in ops] self.assertEqual( - ops, ['c_identity', 'matmul_v2', 'elementwise_add', 'c_concat']) + ops, ['c_identity', 'matmul_v2', 'elementwise_add', 'c_concat'] + ) weight = model_a.parallel_linear.weight bias = model_a.parallel_linear.bias self.assertEqual( weight.shape, - (input_size, output_size // self.model_parallel_size)) - self.assertEqual(bias.shape, - (output_size // self.model_parallel_size, )) + (input_size, output_size // self.model_parallel_size), + ) + self.assertEqual( + bias.shape, (output_size // self.model_parallel_size,) + ) def test_row_parallel_layer(self): main_program, startup_program = self.get_program() @@ -120,19 +123,21 @@ class TestDistTraning(unittest.TestCase): x = paddle.static.data(name='x', shape=[None, input_size]) y = model_a(x) - #print(main_program) + # print(main_program) ops = main_program.global_block().ops ops = [op.type for op in ops] self.assertEqual( ops, - ['c_split', 'matmul_v2', 'c_allreduce_sum', 'elementwise_add']) + ['c_split', 'matmul_v2', 'c_allreduce_sum', 'elementwise_add'], + ) weight = model_a.parallel_linear.weight bias = model_a.parallel_linear.bias self.assertEqual( weight.shape, - (input_size // self.model_parallel_size, output_size)) - self.assertEqual(bias.shape, (output_size, )) + (input_size // self.model_parallel_size, output_size), + ) + self.assertEqual(bias.shape, (output_size,)) def test_parallel_embedding(self): main_program, startup_program = self.get_program() @@ -143,12 +148,12 @@ class TestDistTraning(unittest.TestCase): # model_a model_a = EmbeddingNet(vocab_size, hidden_size) - x = paddle.static.data(name='x', - shape=[None, seq_len], - dtype='int64') + x = paddle.static.data( + name='x', shape=[None, seq_len], dtype='int64' + ) y = model_a(x) - #print(main_program) + # print(main_program) ops = main_program.global_block().ops ops = [op.type for op in ops] self.assertEqual(ops, ['c_embedding', 'c_allreduce_sum']) @@ -156,7 +161,8 @@ class TestDistTraning(unittest.TestCase): weight = model_a.embedding.weight self.assertEqual( weight.shape, - (vocab_size // self.model_parallel_size, hidden_size)) + (vocab_size // self.model_parallel_size, hidden_size), + ) def test_parallel_cross_entropy(self): main_program, startup_program = self.get_program() @@ -170,17 +176,19 @@ class TestDistTraning(unittest.TestCase): model_a = fleet.meta_parallel.ParallelCrossEntropy() x = paddle.static.data( - name='x', shape=[batch_size, seq_length, class_size_per_card]) - label = paddle.static.data(name='label', - shape=[batch_size, seq_length], - dtype='int64') + name='x', shape=[batch_size, seq_length, class_size_per_card] + ) + label = paddle.static.data( + name='label', shape=[batch_size, seq_length], dtype='int64' + ) loss_a = model_a(x, label) - #print(main_program) + # print(main_program) ops = main_program.global_block().ops ops = [op.type for op in ops] - self.assertEqual(ops, - ['unsqueeze2', 'c_softmax_with_cross_entropy']) + self.assertEqual( + ops, ['unsqueeze2', 'c_softmax_with_cross_entropy'] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_utils.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_utils.py index d7266cf29ab..6fc6a6d9a2f 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_utils.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_utils.py @@ -31,8 +31,9 @@ class TestFleetUtils(unittest.TestCase): train_dir = os.path.join("fleet_util_data", "train_program") def download_files(self): - path = download(self.proto_data_url, self.module_name, - self.proto_data_md5) + path = download( + self.proto_data_url, self.module_name, self.proto_data_md5 + ) print('data is downloaded at ' + path) tar = tarfile.open(path) unzip_folder = tempfile.mkdtemp() @@ -50,29 +51,35 @@ class TestFleetUtils(unittest.TestCase): text_program = "pruned_main_program.pbtxt" binary_program = "pruned_main_program.bin" fleet_util = FleetUtil() - text_to_binary = fleet_util.program_type_trans(program_dir, - text_program, True) - binary_to_text = fleet_util.program_type_trans(program_dir, - binary_program, False) + text_to_binary = fleet_util.program_type_trans( + program_dir, text_program, True + ) + binary_to_text = fleet_util.program_type_trans( + program_dir, binary_program, False + ) self.assertTrue( - os.path.exists(os.path.join(program_dir, text_to_binary))) + os.path.exists(os.path.join(program_dir, text_to_binary)) + ) self.assertTrue( - os.path.exists(os.path.join(program_dir, binary_to_text))) + os.path.exists(os.path.join(program_dir, binary_to_text)) + ) def test_parse_program_proto(self): data_dir = self.download_files() parse_program_file_path = os.path.join( - data_dir, os.path.join(self.pruned_dir, - "pruned_main_program.pbtxt")) + data_dir, os.path.join(self.pruned_dir, "pruned_main_program.pbtxt") + ) is_text_parse_program = True parse_output_dir = os.path.join(data_dir, self.pruned_dir) fleet_util = FleetUtil() - fleet_util.parse_program_proto(parse_program_file_path, - is_text_parse_program, parse_output_dir) + fleet_util.parse_program_proto( + parse_program_file_path, is_text_parse_program, parse_output_dir + ) ops_log = os.path.join(parse_output_dir, "ops.log") vars_log = os.path.join(parse_output_dir, "vars_all.log") - vars_persistable = os.path.join(parse_output_dir, - "vars_persistable.log") + vars_persistable = os.path.join( + parse_output_dir, "vars_persistable.log" + ) self.assertTrue(os.path.exists(ops_log)) self.assertTrue(os.path.exists(vars_log)) self.assertTrue(os.path.exists(vars_persistable)) @@ -89,7 +96,7 @@ class TestFleetUtils(unittest.TestCase): feed_config.feeded_vars_types = [np.float32, np.float32] feed_config.feeded_vars_filelist = [ os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_1")), - os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_2")) + os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_2")), ] fetch_config = config() @@ -106,7 +113,9 @@ class TestFleetUtils(unittest.TestCase): fleet_util = FleetUtil() # test saved var's shape - conf.dump_program_filename = "pruned_main_program.save_var_shape_not_match" + conf.dump_program_filename = ( + "pruned_main_program.save_var_shape_not_match" + ) self.assertRaises(Exception, fleet_util.check_vars_and_dump, conf) # test program.proto without feed_op and fetch_op @@ -114,10 +123,13 @@ class TestFleetUtils(unittest.TestCase): results = fleet_util.check_vars_and_dump(conf) self.assertTrue(len(results) == 1) np.testing.assert_array_almost_equal( - results[0], np.array([[3.0590223e-07]], dtype=np.float32)) + results[0], np.array([[3.0590223e-07]], dtype=np.float32) + ) # test feed_var's shape - conf.dump_program_filename = "pruned_main_program.feed_var_shape_not_match" + conf.dump_program_filename = ( + "pruned_main_program.feed_var_shape_not_match" + ) self.assertRaises(Exception, fleet_util.check_vars_and_dump, conf) # test correct case with feed_vars_filelist @@ -125,7 +137,8 @@ class TestFleetUtils(unittest.TestCase): results = fleet_util.check_vars_and_dump(conf) self.assertTrue(len(results) == 1) np.testing.assert_array_almost_equal( - results[0], np.array([[3.0590223e-07]], dtype=np.float32)) + results[0], np.array([[3.0590223e-07]], dtype=np.float32) + ) # test correct case without feed_vars_filelist conf.feed_config.feeded_vars_filelist = None @@ -145,14 +158,17 @@ class TestFleetUtils(unittest.TestCase): conf = config() conf.train_prog_path = os.path.join( - data_dir, os.path.join(self.train_dir, "join_main_program.pbtxt")) + data_dir, os.path.join(self.train_dir, "join_main_program.pbtxt") + ) conf.is_text_train_program = True # test not match conf.pruned_prog_path = os.path.join( data_dir, - os.path.join(self.pruned_dir, - "pruned_main_program.save_var_shape_not_match")) + os.path.join( + self.pruned_dir, "pruned_main_program.save_var_shape_not_match" + ), + ) conf.is_text_pruned_program = True conf.draw = False fleet_util = FleetUtil() @@ -161,8 +177,8 @@ class TestFleetUtils(unittest.TestCase): # test match conf.pruned_prog_path = os.path.join( - data_dir, os.path.join(self.pruned_dir, - "pruned_main_program.pbtxt")) + data_dir, os.path.join(self.pruned_dir, "pruned_main_program.pbtxt") + ) if sys.platform == 'win32' or sys.platform == 'sys.platform': conf.draw = False else: @@ -177,29 +193,39 @@ class TestFleetUtils(unittest.TestCase): else: data_dir = self.download_files() program_path = os.path.join( - data_dir, os.path.join(self.train_dir, - "join_main_program.pbtxt")) + data_dir, + os.path.join(self.train_dir, "join_main_program.pbtxt"), + ) is_text = True program = utils.load_program(program_path, is_text) output_dir = os.path.join(data_dir, self.train_dir) output_filename_1 = "draw_prog_1" output_filename_2 = "draw_prog_2" fleet_util = FleetUtil() - fleet_util.draw_from_program_file(program_path, is_text, output_dir, - output_filename_1) + fleet_util.draw_from_program_file( + program_path, is_text, output_dir, output_filename_1 + ) fleet_util.draw_from_program(program, output_dir, output_filename_2) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename_1 + ".dot"))) + os.path.join(output_dir, output_filename_1 + ".dot") + ) + ) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename_1 + ".pdf"))) + os.path.join(output_dir, output_filename_1 + ".pdf") + ) + ) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename_2 + ".dot"))) + os.path.join(output_dir, output_filename_2 + ".dot") + ) + ) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename_2 + ".pdf"))) + os.path.join(output_dir, output_filename_2 + ".pdf") + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py index 694cfe0cacc..b141b1ed65b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py @@ -23,12 +23,10 @@ java_home = os.environ["JAVA_HOME"] class FSTest1(FSTestBase): - def test_timeout(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=6 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=6 * 1000, sleep_inter=100 + ) src = "hdfs_test_timeout" dst = "new_hdfs_test_timeout" fs.delete(dst) @@ -40,7 +38,8 @@ class FSTest1(FSTestBase): try: fs.mv(src, dst, test_exists=False) self.assertFalse( - 1, "can't execute cmd:{} output:{}".format(cmd, output)) + 1, "can't execute cmd:{} output:{}".format(cmd, output) + ) except FSTimeOut as e: print("execute mv {} to {} timeout".format(src, dst)) @@ -49,10 +48,9 @@ class FSTest1(FSTestBase): print("second mv ret:{} output:{}".format(ret, output)) def test_is_dir(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=6 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=6 * 1000, sleep_inter=100 + ) self.assertFalse(fs.is_dir("./test_hdfs.py")) s = """ java.io.IOException: Input/output error @@ -67,23 +65,24 @@ java.io.IOException: Input/output error at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:79) at org.apache.hadoop.fs.FsShell.main(FsShell.java:2353) - """ + """ # fmt: off, avoid remove tabs in string print("split lines:", s.splitlines()) self.assertTrue(fs._test_match(s.splitlines()) != None) def test_config(self): config = {"fs.default.name": "hdfs://xxx", "hadoop.job.ugi": "ugi"} - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - config, - time_out=6 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", + config, + time_out=6 * 1000, + sleep_inter=100, + ) def test_exists(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=6 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=6 * 1000, sleep_inter=100 + ) self.assertFalse(fs.is_exist(os.path.abspath("./xxxx"))) self.assertFalse(fs.is_dir(os.path.abspath("./xxxx"))) self.assertTrue(fs.is_dir(os.path.abspath("./xxx/.."))) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs2.py index 682036a681a..787a7c93344 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs2.py @@ -22,12 +22,10 @@ java_home = os.environ["JAVA_HOME"] class FSTest2(FSTestBase): - def test_hdfs(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=5 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=5 * 1000, sleep_inter=100 + ) self._test_rm(fs) self._test_touch(fs) self._test_dirs(fs) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs3.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs3.py index 6bf304dc1e7..9aa0f30e99c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs3.py @@ -22,12 +22,10 @@ java_home = os.environ["JAVA_HOME"] class FSTest3(FSTestBase): - def test_hdfs(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=5 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=5 * 1000, sleep_inter=100 + ) self._test_mkdirs(fs) self._test_list_dir(fs) self._test_try_upload(fs) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_hybrid_parallel_inference_helper.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_hybrid_parallel_inference_helper.py index 06b3ec02de0..b0d07f8e92b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_hybrid_parallel_inference_helper.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_hybrid_parallel_inference_helper.py @@ -18,7 +18,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestHybridParallelInferenceHelper(TestMultipleGpus): - def test_hybrid_parallel_inference_helper(self): self.run_mnist_2gpu('hybrid_parallel_inference_helper.py') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py index 026fa8fb9d4..916a21359a4 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py @@ -32,31 +32,33 @@ if fluid.core.is_compiled_with_cuda(): class SimpleConv(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(SimpleConv, self).__init__() - self._conv = fluid.dygraph.Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=None, - use_cudnn=True) + self._conv = fluid.dygraph.Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=None, + use_cudnn=True, + ) def forward(self, inputs): return self._conv(inputs) class TestAutoCast(unittest.TestCase): - def amp_guard_white_op(self): data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') with fluid.dygraph.guard(): @@ -93,30 +95,37 @@ class TestAutoCast(unittest.TestCase): tracer = fluid.framework._dygraph_tracer() base_white_list = fluid.dygraph.amp.auto_cast.WHITE_LIST base_black_list = fluid.dygraph.amp.auto_cast.BLACK_LIST - with fluid.dygraph.amp_guard(custom_white_list=["log"], - custom_black_list=["conv2d"]): + with fluid.dygraph.amp_guard( + custom_white_list=["log"], custom_black_list=["conv2d"] + ): white_list, black_list = tracer._get_amp_op_list() self.assertTrue( - set(white_list) == (set(base_white_list) | {"log"}) - - {"conv2d"}) + set(white_list) + == (set(base_white_list) | {"log"}) - {"conv2d"} + ) self.assertTrue( - set(black_list) == (set(base_black_list) - {"log"}) - | {"conv2d"}) + set(black_list) + == (set(base_black_list) - {"log"}) | {"conv2d"} + ) base_white_list = fluid.dygraph.amp.auto_cast.PURE_FP16_WHITE_LIST base_black_list = fluid.dygraph.amp.auto_cast.PURE_FP16_BLACK_LIST - with fluid.dygraph.amp_guard(custom_white_list=["log"], - custom_black_list=["conv2d"], - level='O2'): + with fluid.dygraph.amp_guard( + custom_white_list=["log"], + custom_black_list=["conv2d"], + level='O2', + ): white_list, black_list = tracer._get_amp_op_list() self.assertTrue( - set(white_list) == (set(base_white_list) | {"log"}) - - {"conv2d"}) + set(white_list) + == (set(base_white_list) | {"log"}) - {"conv2d"} + ) self.assertTrue( - set(black_list) == (set(base_black_list) - {"log"}) - | {"conv2d"}) + set(black_list) + == (set(base_black_list) - {"log"}) | {"conv2d"} + ) def test_custom_op_list(self): self.custom_op_list() @@ -126,13 +135,16 @@ class TestAutoCast(unittest.TestCase): def func(): with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - with fluid.dygraph.amp_guard(custom_white_list=["conv2d"], - custom_black_list=["conv2d"]): + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + with fluid.dygraph.amp_guard( + custom_white_list=["conv2d"], custom_black_list=["conv2d"] + ): inp = fluid.dygraph.to_variable(inp_np) out = model(inp) @@ -149,35 +161,34 @@ class TestAutoCast(unittest.TestCase): with fluid.dygraph.amp_guard(True): out_amp_fp16 = conv2d(data) out_amp_fp32 = paddle.expand_as( - out_amp_fp16, - out_amp_fp16) # expand_as_v2 has no fp16 kernel + out_amp_fp16, out_amp_fp16 + ) # expand_as_v2 has no fp16 kernel with fluid.dygraph.amp_guard(True, level='O2'): out_purefp16_fp16 = conv2d(data) out_purefp16_fp32 = paddle.expand_as( - out_purefp16_fp16, - out_purefp16_fp16) # expand_as_v2 has no fp16 kernel + out_purefp16_fp16, out_purefp16_fp16 + ) # expand_as_v2 has no fp16 kernel self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32) self.assertTrue(out_amp_fp16.dtype == fluid.core.VarDesc.VarType.FP16) self.assertTrue(out_amp_fp32.dtype == fluid.core.VarDesc.VarType.FP32) self.assertTrue( - out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16) + out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16 + ) self.assertTrue( - out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32) + out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32 + ) def test_amp_guard_upsupported_fp16_op(self): self.amp_guard_upsupported_fp16_op() def mode_exception(self): - def func(): data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') with fluid.dygraph.guard(): - conv2d = fluid.dygraph.Conv2D(3, - 2, - 3, - bias_attr=False, - act=None) + conv2d = fluid.dygraph.Conv2D( + 3, 2, 3, bias_attr=False, act=None + ) data = fluid.dygraph.to_variable(data) with fluid.dygraph.amp_guard(level='O'): out = conv2d(data) @@ -189,15 +200,14 @@ class TestAutoCast(unittest.TestCase): class TestAmpScaler(unittest.TestCase): - def scale(self): with fluid.dygraph.guard(): data = paddle.rand([10, 1024]) scaler = paddle.fluid.dygraph.AmpScaler(init_loss_scaling=1024) scaled_data = scaler.scale(data) self.assertEqual( - np.array_equal(scaled_data.numpy(), - data.numpy() * 1024), True) + np.array_equal(scaled_data.numpy(), data.numpy() * 1024), True + ) def test_scale(self): self.scale() @@ -209,13 +219,16 @@ class TestAmpScaler(unittest.TestCase): paddle.seed(10) paddle.framework.random._manual_program_seed(10) with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters()) + learning_rate=0.01, parameter_list=model.parameters() + ) scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -226,7 +239,8 @@ class TestAmpScaler(unittest.TestCase): scaled_loss = scaler.scale(loss) scaled_loss.backward() optimize_ops, params_grads = scaler.minimize( - optimizer, scaled_loss) + optimizer, scaled_loss + ) else: print('use no scaler') loss.backward() @@ -236,19 +250,25 @@ class TestAmpScaler(unittest.TestCase): outs_with_scaler = run_simple_conv(inp_np, use_scaler=True) outs_no_scaler = run_simple_conv(inp_np, use_scaler=False) - self.assertEqual(outs_with_scaler[0], - []) # optimize_ops is [] in dygraph mode - self.assertEqual(outs_no_scaler[0], - []) # optimize_ops is [] in dygraph mode + self.assertEqual( + outs_with_scaler[0], [] + ) # optimize_ops is [] in dygraph mode + self.assertEqual( + outs_no_scaler[0], [] + ) # optimize_ops is [] in dygraph mode for i in range(len(outs_with_scaler[1])): # check each grad - np.testing.assert_allclose(outs_with_scaler[1][i][1].numpy(), - outs_no_scaler[1][i][1].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[1][i][1].numpy(), + outs_no_scaler[1][i][1].numpy(), + rtol=1e-05, + ) # check each parameter - np.testing.assert_allclose(outs_with_scaler[1][i][0].numpy(), - outs_no_scaler[1][i][0].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[1][i][0].numpy(), + outs_no_scaler[1][i][0].numpy(), + rtol=1e-05, + ) def test_minimize(self): self.minimize() @@ -260,13 +280,16 @@ class TestAmpScaler(unittest.TestCase): paddle.seed(10) paddle.framework.random._manual_program_seed(10) with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -289,9 +312,11 @@ class TestAmpScaler(unittest.TestCase): for i in range(len(outs_with_scaler)): # check each parameter - np.testing.assert_allclose(outs_with_scaler[i].numpy(), - outs_no_scaler[i].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[i].numpy(), + outs_no_scaler[i].numpy(), + rtol=1e-05, + ) def test_step(self): self.step() @@ -300,16 +325,19 @@ class TestAmpScaler(unittest.TestCase): inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32) inp_np[0][1][2][3] = np.nan with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) params_init = {} for param in model.parameters(): params_init[param.name] = param.numpy() optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters()) + learning_rate=0.01, parameter_list=model.parameters() + ) scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -322,18 +350,19 @@ class TestAmpScaler(unittest.TestCase): for param in model.parameters(): # param not update when tensor contains nan or inf - np.testing.assert_array_equal(param.numpy(), - params_init[param.name]) + np.testing.assert_array_equal( + param.numpy(), params_init[param.name] + ) def test_nan_inf(self): self.nan_inf() def step_update_exception(self): - def func1(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -347,8 +376,9 @@ class TestAmpScaler(unittest.TestCase): def func2(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -362,8 +392,9 @@ class TestAmpScaler(unittest.TestCase): def func3(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -380,13 +411,15 @@ class TestAmpScaler(unittest.TestCase): def test_get_and_set(self): with fluid.dygraph.guard(): - scaler = paddle.amp.GradScaler(enable=True, - init_loss_scaling=1024, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - use_dynamic_loss_scaling=True) + scaler = paddle.amp.GradScaler( + enable=True, + init_loss_scaling=1024, + incr_ratio=2.0, + decr_ratio=0.5, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=2, + use_dynamic_loss_scaling=True, + ) self.assertEqual(scaler.is_enable() == True, True) self.assertEqual(scaler.get_init_loss_scaling() == 1024, True) self.assertEqual(scaler.get_incr_ratio() == 2.0, True) @@ -407,13 +440,15 @@ class TestAmpScaler(unittest.TestCase): def test_state_dict_and_load_state_dict(self): with fluid.dygraph.guard(): - scaler1 = paddle.amp.GradScaler(enable=True, - init_loss_scaling=14, - incr_ratio=233.0, - decr_ratio=0.523, - incr_every_n_steps=1090, - decr_every_n_nan_or_inf=20, - use_dynamic_loss_scaling=True) + scaler1 = paddle.amp.GradScaler( + enable=True, + init_loss_scaling=14, + incr_ratio=233.0, + decr_ratio=0.523, + incr_every_n_steps=1090, + decr_every_n_nan_or_inf=20, + use_dynamic_loss_scaling=True, + ) scaler_state = scaler1.state_dict() scaler2 = paddle.amp.GradScaler(enable=True) scaler2.load_state_dict(scaler_state) @@ -428,7 +463,6 @@ class TestAmpScaler(unittest.TestCase): self.assertEqual(scaler3.is_enable() == False, True) def test_state_dict_and_load_state_dict_error(self): - def test_error(): state_empty = {} scaler = paddle.amp.GradScaler(enable=True) @@ -438,7 +472,6 @@ class TestAmpScaler(unittest.TestCase): def reader_decorator(reader): - def __reader__(): for item in reader(): img = np.array(item[0]).astype('float32').reshape(3, 224, 224) @@ -449,11 +482,9 @@ def reader_decorator(reader): class TestGradScalerStateDict(unittest.TestCase): - - def train_resnet(self, - enable_amp=True, - use_data_loader=True, - use_save_load=True): + def train_resnet( + self, enable_amp=True, use_data_loader=True, use_save_load=True + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -463,30 +494,35 @@ class TestGradScalerStateDict(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader @@ -496,13 +532,19 @@ class TestGradScalerStateDict(unittest.TestCase): if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -525,8 +567,9 @@ class TestGradScalerStateDict(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -543,43 +586,37 @@ class TestGradScalerStateDict(unittest.TestCase): return dy_out, dy_param_value, dy_grad_value def test_with_state_dict(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_use_state_dict = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=True) - out_no_state_dict = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=False) + out_use_state_dict = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=True + ) + out_no_state_dict = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=False + ) print('save_load:', out_use_state_dict[0], out_no_state_dict[0]) - np.testing.assert_allclose(out_use_state_dict[0], - out_no_state_dict[0], - rtol=1e-05) + np.testing.assert_allclose( + out_use_state_dict[0], out_no_state_dict[0], rtol=1e-05 + ) func_isinstance() class TestAmpDecorator(unittest.TestCase): - def test_mode_exception(self): - def func(): with fluid.dygraph.guard(): model = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) opt = paddle.optimizer.SGD(parameters=model.parameters()) - model, opt = paddle.amp.decorate(models=model, - optimizers=opt, - level='O') + model, opt = paddle.amp.decorate( + models=model, optimizers=opt, level='O' + ) self.assertRaises(ValueError, func) def test_input_type_exception(self): - def test_error_model(): - class MyModel(object): - def __init__(self): print("A fake Model") @@ -598,9 +635,7 @@ class TestAmpDecorator(unittest.TestCase): self.assertRaises(RuntimeError, test_error_distributed_model) def test_error_optimizer(): - class MyOptimizer(object): - def __init__(self): print("A fake Optimizer") @@ -613,48 +648,59 @@ class TestAmpDecorator(unittest.TestCase): def test_set_master_weight(self): model1 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt1 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model1.parameters(), - multi_precision=True) + opt1 = paddle.optimizer.Adam( + learning_rate=0.0001, + parameters=model1.parameters(), + multi_precision=True, + ) model2 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt2 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model2.parameters(), - multi_precision=False) - - model1, opt1 = paddle.amp.decorate(models=model1, - optimizers=opt1, - level='O2', - master_weight=None) + opt2 = paddle.optimizer.Adam( + learning_rate=0.0001, + parameters=model2.parameters(), + multi_precision=False, + ) + + model1, opt1 = paddle.amp.decorate( + models=model1, optimizers=opt1, level='O2', master_weight=None + ) self.assertEqual(opt1._multi_precision, True) - models, opt2 = paddle.amp.decorate(models=[model1, model2], - optimizers=opt2, - level='O2', - master_weight=None) + models, opt2 = paddle.amp.decorate( + models=[model1, model2], + optimizers=opt2, + level='O2', + master_weight=None, + ) self.assertEqual(opt2._multi_precision, True) model3 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt3 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model3.parameters()) + opt3 = paddle.optimizer.Adam( + learning_rate=0.0001, parameters=model3.parameters() + ) model4 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt4 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model4.parameters()) - - model3, opts = paddle.amp.decorate(models=model3, - optimizers=[opt3, opt4], - level='O2', - master_weight=True) + opt4 = paddle.optimizer.Adam( + learning_rate=0.0001, parameters=model4.parameters() + ) + + model3, opts = paddle.amp.decorate( + models=model3, + optimizers=[opt3, opt4], + level='O2', + master_weight=True, + ) self.assertEqual(opts[0]._multi_precision, True) self.assertEqual(opts[1]._multi_precision, True) models = [model3, model4] optimizers = [opt3, opt4] - models, optimizers = paddle.amp.decorate(models=models, - optimizers=optimizers, - level='O2', - master_weight=False) + models, optimizers = paddle.amp.decorate( + models=models, + optimizers=optimizers, + level='O2', + master_weight=False, + ) self.assertEqual(optimizers[0]._multi_precision, False) self.assertEqual(optimizers[1]._multi_precision, False) @@ -689,20 +735,19 @@ class TestAmpDecorator(unittest.TestCase): buffer = paddle.to_tensor(np.array([5]).astype("int32")) model.register_buffer("buffer_name", buffer, persistable=True) model = paddle.amp.decorate(models=model, level='O2') - self.assertEqual((model._buffers["buffer_name"].dtype == paddle.int32), - True) + self.assertEqual( + (model._buffers["buffer_name"].dtype == paddle.int32), True + ) class TestStateDictHookForAMP(unittest.TestCase): - def test_state_dict_hook(self): - def func_isinstance(): paddle.seed(100) model = paddle.nn.Linear(2, 4) - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) param_value_ori = {} for param in model.parameters(): param_value_ori[param.name] = param.numpy() @@ -723,7 +768,6 @@ class TestStateDictHookForAMP(unittest.TestCase): class TestPureFp16SaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -731,22 +775,19 @@ class TestPureFp16SaveLoad(unittest.TestCase): self.temp_dir.cleanup() def test_save_dtype_exception(self): - def func(): paddle.disable_static() model = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) opt = paddle.optimizer.SGD(parameters=model.parameters()) - paddle.amp.decorate(models=model, - optimizers=opt, - level='O2', - save_dtype='int') + paddle.amp.decorate( + models=model, optimizers=opt, level='O2', save_dtype='int' + ) self.assertRaises(ValueError, func) - def train_resnet(self, - enable_amp=True, - use_data_loader=True, - use_save_load=True): + def train_resnet( + self, enable_amp=True, use_data_loader=True, use_save_load=True + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -756,38 +797,45 @@ class TestPureFp16SaveLoad(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader if enable_amp: - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, + optimizers=optimizer, + level='O2', + save_dtype='float32', + ) for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: @@ -795,13 +843,19 @@ class TestPureFp16SaveLoad(unittest.TestCase): if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -825,8 +879,9 @@ class TestPureFp16SaveLoad(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -839,7 +894,7 @@ class TestPureFp16SaveLoad(unittest.TestCase): obj = { 'model': resnet.state_dict(), 'opt': optimizer.state_dict(), - 'scaler': scaler.state_dict() + 'scaler': scaler.state_dict(), } path = os.path.join(self.temp_dir.name, 'model.pdparams') paddle.save(obj, path) @@ -847,39 +902,40 @@ class TestPureFp16SaveLoad(unittest.TestCase): obj_load = paddle.load(path) resnet = ResNet(use_cudnn=True) optimizer = optimizer_setting( - train_parameters, parameter_list=resnet.parameters()) + train_parameters, parameter_list=resnet.parameters() + ) resnet.set_state_dict(obj_load['model']) optimizer.set_state_dict(obj_load['opt']) scaler.load_state_dict(obj_load['scaler']) - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, + optimizers=optimizer, + level='O2', + save_dtype='float32', + ) if use_data_loader: train_reader._reset() return dy_out, dy_param_value, dy_grad_value def test_with_save_load(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_use_save_load = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=True) - out_no_save_load = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=False) + out_use_save_load = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=True + ) + out_no_save_load = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=False + ) print('save_load:', out_use_save_load[0], out_no_save_load[0]) - np.testing.assert_allclose(out_use_save_load[0], - out_no_save_load[0], - rtol=1e-05) + np.testing.assert_allclose( + out_use_save_load[0], out_no_save_load[0], rtol=1e-05 + ) func_isinstance() class TestPureFp16InferenceSaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -895,21 +951,20 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, - (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( + 'int64' + ) return image, label def __len__(self): return self.num_samples class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -920,10 +975,12 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): def train(layer, loader, loss_fn, opt): for epoch_id in range(EPOCH_NUM): for batch_id, (image, label) in enumerate(loader()): - with paddle.amp.auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2'): + with paddle.amp.auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O2', + ): out = layer(image) loss = loss_fn(out, label) loss.backward() @@ -932,27 +989,31 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): # train layer = LinearNet() - adam = paddle.optimizer.Adam(learning_rate=0.001, - parameters=layer.parameters(), - multi_precision=True) + adam = paddle.optimizer.Adam( + learning_rate=0.001, + parameters=layer.parameters(), + multi_precision=True, + ) loss_fn = nn.CrossEntropyLoss() - layer, adam = paddle.amp.decorate(models=layer, - optimizers=adam, - save_dtype='float32') + layer, adam = paddle.amp.decorate( + models=layer, optimizers=adam, save_dtype='float32' + ) dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) train(layer, loader, loss_fn, adam) # save path = os.path.join(self.temp_dir.name, 'example_model/linear') - paddle.jit.save(layer, - path, - input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')]) + paddle.jit.save( + layer, path, input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')] + ) # jit.load loaded_layer = paddle.jit.load(path) @@ -966,12 +1027,17 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): # load_inference_model paddle.enable_static() exe = paddle.static.Executor() - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.load_inference_model(path, exe)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(path, exe) tensor_img = x - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("pred.numpy()", pred.numpy()) print("result", results[0]) np.testing.assert_array_equal(pred.numpy(), results[0]) @@ -986,11 +1052,13 @@ class TestResnet2(unittest.TestCase): Use paddle-2.0 API """ - def train_resnet(self, - enable_amp=True, - level='O1', - use_data_loader=False, - use_param_group=False): + def train_resnet( + self, + enable_amp=True, + level='O1', + use_data_loader=False, + use_param_group=False, + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -1014,43 +1082,42 @@ class TestResnet2(unittest.TestCase): # NOTE(zhiqiu): The Membership test operations(in / not in) calls "is" and "equal", # see details: https://docs.python.org/3/reference/expressions.html#membership-test-operations. # So do not use other_params = [p for p in resnet.parameters() if p not in conv_params] - optimizer = paddle.optimizer.Momentum(parameters=[{ - 'params': - conv_params, - 'learning_rate': - 0.01 - }, { - 'params': - other_params, - 'learning_rate': - 0.001 - }], - multi_precision=True) + optimizer = paddle.optimizer.Momentum( + parameters=[ + {'params': conv_params, 'learning_rate': 0.01}, + {'params': other_params, 'learning_rate': 0.001}, + ], + multi_precision=True, + ) else: optimizer = paddle.optimizer.SGD(parameters=resnet.parameters()) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader @@ -1063,13 +1130,19 @@ class TestResnet2(unittest.TestCase): if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -1094,8 +1167,9 @@ class TestResnet2(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -1107,70 +1181,65 @@ class TestResnet2(unittest.TestCase): return dy_out, dy_param_value, dy_grad_value def test_resnet(self): - def func_isinstance(): with fluid.dygraph.guard(): out_fp32 = self.train_resnet(enable_amp=False) out_amp = self.train_resnet(enable_amp=True) out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() def test_with_data_loader(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False, - use_data_loader=True) - out_amp = self.train_resnet(enable_amp=True, - use_data_loader=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, - use_data_loader=True, - level='O2') + out_fp32 = self.train_resnet( + enable_amp=False, use_data_loader=True + ) + out_amp = self.train_resnet( + enable_amp=True, use_data_loader=True + ) + out_pure_fp16 = self.train_resnet( + enable_amp=True, use_data_loader=True, level='O2' + ) print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() def test_param_group(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False, - use_data_loader=True, - use_param_group=True) - out_amp = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_param_group=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_param_group=True, - level='O2') + out_fp32 = self.train_resnet( + enable_amp=False, use_data_loader=True, use_param_group=True + ) + out_amp = self.train_resnet( + enable_amp=True, use_data_loader=True, use_param_group=True + ) + out_pure_fp16 = self.train_resnet( + enable_amp=True, + use_data_loader=True, + use_param_group=True, + level='O2', + ) print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() @@ -1191,42 +1260,54 @@ class TestResnet(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) optimizer = paddle.optimizer.Momentum( - parameters=resnet.parameters(), multi_precision=True) + parameters=resnet.parameters(), multi_precision=True + ) np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.fluid.dygraph.AmpScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.fluid.dygraph.AmpScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if enable_amp and (level == 'O2'): resnet, optimizer = paddle.fluid.dygraph.amp_decorate( - models=resnet, optimizers=optimizer, level='O2') + models=resnet, optimizers=optimizer, level='O2' + ) for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: break - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(dy_x_data) label = fluid.dygraph.to_variable(y_data) label.stop_gradient = True - with paddle.fluid.dygraph.amp_guard(enable=enable_amp, - level=level): + with paddle.fluid.dygraph.amp_guard( + enable=enable_amp, level=level + ): out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -1243,9 +1324,11 @@ class TestResnet(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -1256,31 +1339,27 @@ class TestResnet(unittest.TestCase): return dy_out, dy_param_value, dy_grad_value def test_resnet(self): - def func_isinstance(): out_fp32 = self.train_resnet(enable_amp=False) out_amp = self.train_resnet(enable_amp=True) out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=0.01) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=0.01 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.1 + ) func_isinstance() class TestLayerNormFp16(unittest.TestCase): - r''' layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 + r'''layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 and other params are fp32. ''' def test_layer_norm_fp16(self): - def func_isinstance(): if fluid.is_compiled_with_cuda(): with fluid.dygraph.guard(fluid.CUDAPlace(0)): @@ -1290,7 +1369,8 @@ class TestLayerNormFp16(unittest.TestCase): out = layer_norm(x) self.assertTrue( - out.dtype == fluid.core.VarDesc.VarType.FP16) + out.dtype == fluid.core.VarDesc.VarType.FP16 + ) func_isinstance() @@ -1298,7 +1378,8 @@ class TestLayerNormFp16(unittest.TestCase): @unittest.skipIf( paddle.is_compiled_with_cuda() and not core.is_bfloat16_supported(core.CUDAPlace(0)), - "skip bf16 test if cuda is in use but bf16 is not supported by gpu arch.") + "skip bf16 test if cuda is in use but bf16 is not supported by gpu arch.", +) class TestBf16(unittest.TestCase): ''' test amp for BF16 @@ -1306,43 +1387,37 @@ class TestBf16(unittest.TestCase): def train(self, enable_amp=True, amp_level='O1'): paddle.seed(100) - input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = paddle.nn.Conv2D(4, 6, (3, 3)) if amp_level == 'O2': - conv = paddle.amp.decorate(models=conv, - level=amp_level, - dtype='bfloat16') - with paddle.amp.auto_cast(enable=enable_amp, - level=amp_level, - dtype='bfloat16'): + conv = paddle.amp.decorate( + models=conv, level=amp_level, dtype='bfloat16' + ) + with paddle.amp.auto_cast( + enable=enable_amp, level=amp_level, dtype='bfloat16' + ): output = conv(input) output = output.cast('float32') return output.numpy() def test_bf16(self): - def func_isinstance(): out_fp32 = self.train(enable_amp=False) out_bf16_O1 = self.train(enable_amp=True, amp_level='O1') out_bf16_O2 = self.train(enable_amp=True, amp_level='O2') - np.testing.assert_allclose(out_fp32, - out_bf16_O1, - rtol=0.001, - atol=0.1) - np.testing.assert_allclose(out_fp32, - out_bf16_O2, - rtol=0.001, - atol=0.1) + np.testing.assert_allclose( + out_fp32, out_bf16_O1, rtol=0.001, atol=0.1 + ) + np.testing.assert_allclose( + out_fp32, out_bf16_O2, rtol=0.001, atol=0.1 + ) func_isinstance() class TestAmpWithPyLyer(unittest.TestCase): - def test_pylayer(self): - class MyMM(PyLayer): - @staticmethod def forward(ctx, a, b): ctx.save_for_backward(a, b) @@ -1367,9 +1442,7 @@ class TestAmpWithPyLyer(unittest.TestCase): class TestAmpWithHook(unittest.TestCase): - def test_hook_change_dtype(self): - def func_isinstance(): with paddle.fluid.dygraph.guard(): v = paddle.rand([3, 3]) @@ -1390,7 +1463,6 @@ class TestAmpWithHook(unittest.TestCase): func_isinstance() def test_hook_change_place(self): - def func_isinstance(): with paddle.fluid.dygraph.guard(): v = paddle.rand([3, 3]) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py index 7ff434987a9..83c9462a89e 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py @@ -31,31 +31,33 @@ if fluid.core.is_compiled_with_cuda(): class SimpleConv(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(SimpleConv, self).__init__() - self._conv = fluid.dygraph.Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=None, - use_cudnn=True) + self._conv = fluid.dygraph.Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=None, + use_cudnn=True, + ) def forward(self, inputs): return self._conv(inputs) class TestAutoCast(unittest.TestCase): - def amp_guard_white_op(self): data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') with fluid.dygraph.guard(): @@ -92,30 +94,37 @@ class TestAutoCast(unittest.TestCase): tracer = fluid.framework._dygraph_tracer() base_white_list = fluid.dygraph.amp.auto_cast.WHITE_LIST base_black_list = fluid.dygraph.amp.auto_cast.BLACK_LIST - with fluid.dygraph.amp_guard(custom_white_list=["log"], - custom_black_list=["conv2d"]): + with fluid.dygraph.amp_guard( + custom_white_list=["log"], custom_black_list=["conv2d"] + ): white_list, black_list = tracer._get_amp_op_list() self.assertTrue( - set(white_list) == (set(base_white_list) | {"log"}) - - {"conv2d"}) + set(white_list) + == (set(base_white_list) | {"log"}) - {"conv2d"} + ) self.assertTrue( - set(black_list) == (set(base_black_list) - {"log"}) - | {"conv2d"}) + set(black_list) + == (set(base_black_list) - {"log"}) | {"conv2d"} + ) base_white_list = fluid.dygraph.amp.auto_cast.PURE_FP16_WHITE_LIST base_black_list = fluid.dygraph.amp.auto_cast.PURE_FP16_BLACK_LIST - with fluid.dygraph.amp_guard(custom_white_list=["log"], - custom_black_list=["conv2d"], - level='O2'): + with fluid.dygraph.amp_guard( + custom_white_list=["log"], + custom_black_list=["conv2d"], + level='O2', + ): white_list, black_list = tracer._get_amp_op_list() self.assertTrue( - set(white_list) == (set(base_white_list) | {"log"}) - - {"conv2d"}) + set(white_list) + == (set(base_white_list) | {"log"}) - {"conv2d"} + ) self.assertTrue( - set(black_list) == (set(base_black_list) - {"log"}) - | {"conv2d"}) + set(black_list) + == (set(base_black_list) - {"log"}) | {"conv2d"} + ) def test_custom_op_list(self): self.custom_op_list() @@ -125,13 +134,16 @@ class TestAutoCast(unittest.TestCase): def func(): with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - with fluid.dygraph.amp_guard(custom_white_list=["conv2d"], - custom_black_list=["conv2d"]): + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + with fluid.dygraph.amp_guard( + custom_white_list=["conv2d"], custom_black_list=["conv2d"] + ): inp = fluid.dygraph.to_variable(inp_np) out = model(inp) @@ -148,35 +160,34 @@ class TestAutoCast(unittest.TestCase): with fluid.dygraph.amp_guard(True): out_amp_fp16 = conv2d(data) out_amp_fp32 = paddle.expand_as( - out_amp_fp16, - out_amp_fp16) # expand_as_v2 has no fp16 kernel + out_amp_fp16, out_amp_fp16 + ) # expand_as_v2 has no fp16 kernel with fluid.dygraph.amp_guard(True, level='O2'): out_purefp16_fp16 = conv2d(data) out_purefp16_fp32 = paddle.expand_as( - out_purefp16_fp16, - out_purefp16_fp16) # expand_as_v2 has no fp16 kernel + out_purefp16_fp16, out_purefp16_fp16 + ) # expand_as_v2 has no fp16 kernel self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32) self.assertTrue(out_amp_fp16.dtype == fluid.core.VarDesc.VarType.FP16) self.assertTrue(out_amp_fp32.dtype == fluid.core.VarDesc.VarType.FP32) self.assertTrue( - out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16) + out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16 + ) self.assertTrue( - out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32) + out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32 + ) def test_amp_guard_upsupported_fp16_op(self): self.amp_guard_upsupported_fp16_op() def mode_exception(self): - def func(): data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') with fluid.dygraph.guard(): - conv2d = fluid.dygraph.Conv2D(3, - 2, - 3, - bias_attr=False, - act=None) + conv2d = fluid.dygraph.Conv2D( + 3, 2, 3, bias_attr=False, act=None + ) data = fluid.dygraph.to_variable(data) with fluid.dygraph.amp_guard(level='O'): out = conv2d(data) @@ -188,15 +199,14 @@ class TestAutoCast(unittest.TestCase): class TestAmpScaler(unittest.TestCase): - def scale(self): with fluid.dygraph.guard(): data = paddle.rand([10, 1024]) scaler = paddle.fluid.dygraph.AmpScaler(init_loss_scaling=1024) scaled_data = scaler.scale(data) self.assertEqual( - np.array_equal(scaled_data.numpy(), - data.numpy() * 1024), True) + np.array_equal(scaled_data.numpy(), data.numpy() * 1024), True + ) def test_scale(self): self.scale() @@ -208,13 +218,16 @@ class TestAmpScaler(unittest.TestCase): paddle.seed(10) paddle.framework.random._manual_program_seed(10) with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters()) + learning_rate=0.01, parameter_list=model.parameters() + ) scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -225,7 +238,8 @@ class TestAmpScaler(unittest.TestCase): scaled_loss = scaler.scale(loss) scaled_loss.backward() optimize_ops, params_grads = scaler.minimize( - optimizer, scaled_loss) + optimizer, scaled_loss + ) else: print('use no scaler') loss.backward() @@ -235,19 +249,25 @@ class TestAmpScaler(unittest.TestCase): outs_with_scaler = run_simple_conv(inp_np, use_scaler=True) outs_no_scaler = run_simple_conv(inp_np, use_scaler=False) - self.assertEqual(outs_with_scaler[0], - []) # optimize_ops is [] in dygraph mode - self.assertEqual(outs_no_scaler[0], - []) # optimize_ops is [] in dygraph mode + self.assertEqual( + outs_with_scaler[0], [] + ) # optimize_ops is [] in dygraph mode + self.assertEqual( + outs_no_scaler[0], [] + ) # optimize_ops is [] in dygraph mode for i in range(len(outs_with_scaler[1])): # check each grad - np.testing.assert_allclose(outs_with_scaler[1][i][1].numpy(), - outs_no_scaler[1][i][1].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[1][i][1].numpy(), + outs_no_scaler[1][i][1].numpy(), + rtol=1e-05, + ) # check each parameter - np.testing.assert_allclose(outs_with_scaler[1][i][0].numpy(), - outs_no_scaler[1][i][0].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[1][i][0].numpy(), + outs_no_scaler[1][i][0].numpy(), + rtol=1e-05, + ) def test_minimize(self): self.minimize() @@ -259,13 +279,16 @@ class TestAmpScaler(unittest.TestCase): paddle.seed(10) paddle.framework.random._manual_program_seed(10) with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -288,9 +311,11 @@ class TestAmpScaler(unittest.TestCase): for i in range(len(outs_with_scaler)): # check each parameter - np.testing.assert_allclose(outs_with_scaler[i].numpy(), - outs_no_scaler[i].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[i].numpy(), + outs_no_scaler[i].numpy(), + rtol=1e-05, + ) def test_step(self): self.step() @@ -299,16 +324,19 @@ class TestAmpScaler(unittest.TestCase): inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32) inp_np[0][1][2][3] = np.nan with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) params_init = {} for param in model.parameters(): params_init[param.name] = param.numpy() optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters()) + learning_rate=0.01, parameter_list=model.parameters() + ) scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -321,18 +349,19 @@ class TestAmpScaler(unittest.TestCase): for param in model.parameters(): # param not update when tensor contains nan or inf - np.testing.assert_array_equal(param.numpy(), - params_init[param.name]) + np.testing.assert_array_equal( + param.numpy(), params_init[param.name] + ) def test_nan_inf(self): self.nan_inf() def step_update_exception(self): - def func1(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -346,8 +375,9 @@ class TestAmpScaler(unittest.TestCase): def func2(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -361,8 +391,9 @@ class TestAmpScaler(unittest.TestCase): def func3(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -379,13 +410,15 @@ class TestAmpScaler(unittest.TestCase): def test_get_and_set(self): with fluid.dygraph.guard(): - scaler = paddle.amp.GradScaler(enable=True, - init_loss_scaling=1024, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - use_dynamic_loss_scaling=True) + scaler = paddle.amp.GradScaler( + enable=True, + init_loss_scaling=1024, + incr_ratio=2.0, + decr_ratio=0.5, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=2, + use_dynamic_loss_scaling=True, + ) self.assertEqual(scaler.is_enable() == True, True) self.assertEqual(scaler.get_init_loss_scaling() == 1024, True) self.assertEqual(scaler.get_incr_ratio() == 2.0, True) @@ -406,13 +439,15 @@ class TestAmpScaler(unittest.TestCase): def test_state_dict_and_load_state_dict(self): with fluid.dygraph.guard(): - scaler1 = paddle.amp.GradScaler(enable=True, - init_loss_scaling=14, - incr_ratio=233.0, - decr_ratio=0.523, - incr_every_n_steps=1090, - decr_every_n_nan_or_inf=20, - use_dynamic_loss_scaling=True) + scaler1 = paddle.amp.GradScaler( + enable=True, + init_loss_scaling=14, + incr_ratio=233.0, + decr_ratio=0.523, + incr_every_n_steps=1090, + decr_every_n_nan_or_inf=20, + use_dynamic_loss_scaling=True, + ) scaler_state = scaler1.state_dict() scaler2 = paddle.amp.GradScaler(enable=True) scaler2.load_state_dict(scaler_state) @@ -427,7 +462,6 @@ class TestAmpScaler(unittest.TestCase): self.assertEqual(scaler3.is_enable() == False, True) def test_state_dict_and_load_state_dict_error(self): - def test_error(): state_empty = {} scaler = paddle.amp.GradScaler(enable=True) @@ -437,7 +471,6 @@ class TestAmpScaler(unittest.TestCase): def reader_decorator(reader): - def __reader__(): for item in reader(): img = np.array(item[0]).astype('float32').reshape(3, 224, 224) @@ -448,11 +481,9 @@ def reader_decorator(reader): class TestGradScalerStateDict(unittest.TestCase): - - def train_resnet(self, - enable_amp=True, - use_data_loader=True, - use_save_load=True): + def train_resnet( + self, enable_amp=True, use_data_loader=True, use_save_load=True + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -462,30 +493,35 @@ class TestGradScalerStateDict(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader @@ -495,13 +531,19 @@ class TestGradScalerStateDict(unittest.TestCase): if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -524,8 +566,9 @@ class TestGradScalerStateDict(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -542,43 +585,37 @@ class TestGradScalerStateDict(unittest.TestCase): return dy_out, dy_param_value, dy_grad_value def test_with_state_dict(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_use_state_dict = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=True) - out_no_state_dict = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=False) + out_use_state_dict = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=True + ) + out_no_state_dict = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=False + ) print('save_load:', out_use_state_dict[0], out_no_state_dict[0]) - np.testing.assert_allclose(out_use_state_dict[0], - out_no_state_dict[0], - rtol=1e-05) + np.testing.assert_allclose( + out_use_state_dict[0], out_no_state_dict[0], rtol=1e-05 + ) func_isinstance() class TestAmpDecorator(unittest.TestCase): - def test_mode_exception(self): - def func(): with fluid.dygraph.guard(): model = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) opt = paddle.optimizer.SGD(parameters=model.parameters()) - model, opt = paddle.amp.decorate(models=model, - optimizers=opt, - level='O') + model, opt = paddle.amp.decorate( + models=model, optimizers=opt, level='O' + ) self.assertRaises(ValueError, func) def test_input_type_exception(self): - def test_error_model(): - class MyModel(object): - def __init__(self): print("A fake Model") @@ -597,9 +634,7 @@ class TestAmpDecorator(unittest.TestCase): self.assertRaises(RuntimeError, test_error_distributed_model) def test_error_optimizer(): - class MyOptimizer(object): - def __init__(self): print("A fake Optimizer") @@ -612,48 +647,59 @@ class TestAmpDecorator(unittest.TestCase): def test_set_master_weight(self): model1 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt1 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model1.parameters(), - multi_precision=True) + opt1 = paddle.optimizer.Adam( + learning_rate=0.0001, + parameters=model1.parameters(), + multi_precision=True, + ) model2 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt2 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model2.parameters(), - multi_precision=False) - - model1, opt1 = paddle.amp.decorate(models=model1, - optimizers=opt1, - level='O2', - master_weight=None) + opt2 = paddle.optimizer.Adam( + learning_rate=0.0001, + parameters=model2.parameters(), + multi_precision=False, + ) + + model1, opt1 = paddle.amp.decorate( + models=model1, optimizers=opt1, level='O2', master_weight=None + ) self.assertEqual(opt1._multi_precision, True) - models, opt2 = paddle.amp.decorate(models=[model1, model2], - optimizers=opt2, - level='O2', - master_weight=None) + models, opt2 = paddle.amp.decorate( + models=[model1, model2], + optimizers=opt2, + level='O2', + master_weight=None, + ) self.assertEqual(opt2._multi_precision, True) model3 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt3 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model3.parameters()) + opt3 = paddle.optimizer.Adam( + learning_rate=0.0001, parameters=model3.parameters() + ) model4 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt4 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model4.parameters()) - - model3, opts = paddle.amp.decorate(models=model3, - optimizers=[opt3, opt4], - level='O2', - master_weight=True) + opt4 = paddle.optimizer.Adam( + learning_rate=0.0001, parameters=model4.parameters() + ) + + model3, opts = paddle.amp.decorate( + models=model3, + optimizers=[opt3, opt4], + level='O2', + master_weight=True, + ) self.assertEqual(opts[0]._multi_precision, True) self.assertEqual(opts[1]._multi_precision, True) models = [model3, model4] optimizers = [opt3, opt4] - models, optimizers = paddle.amp.decorate(models=models, - optimizers=optimizers, - level='O2', - master_weight=False) + models, optimizers = paddle.amp.decorate( + models=models, + optimizers=optimizers, + level='O2', + master_weight=False, + ) self.assertEqual(optimizers[0]._multi_precision, False) self.assertEqual(optimizers[1]._multi_precision, False) @@ -685,15 +731,13 @@ class TestAmpDecorator(unittest.TestCase): class TestStateDictHookForAMP(unittest.TestCase): - def test_state_dict_hook(self): - def func_isinstance(): paddle.seed(100) model = paddle.nn.Linear(2, 4) - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) param_value_ori = {} for param in model.parameters(): param_value_ori[param.name] = param.numpy() @@ -714,7 +758,6 @@ class TestStateDictHookForAMP(unittest.TestCase): class TestPureFp16SaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -722,22 +765,19 @@ class TestPureFp16SaveLoad(unittest.TestCase): self.temp_dir.cleanup() def test_save_dtype_exception(self): - def func(): paddle.disable_static() model = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) opt = paddle.optimizer.SGD(parameters=model.parameters()) - paddle.amp.decorate(models=model, - optimizers=opt, - level='O2', - save_dtype='int') + paddle.amp.decorate( + models=model, optimizers=opt, level='O2', save_dtype='int' + ) self.assertRaises(ValueError, func) - def train_resnet(self, - enable_amp=True, - use_data_loader=True, - use_save_load=True): + def train_resnet( + self, enable_amp=True, use_data_loader=True, use_save_load=True + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -747,38 +787,45 @@ class TestPureFp16SaveLoad(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader if enable_amp: - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, + optimizers=optimizer, + level='O2', + save_dtype='float32', + ) for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: @@ -786,13 +833,19 @@ class TestPureFp16SaveLoad(unittest.TestCase): if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -816,8 +869,9 @@ class TestPureFp16SaveLoad(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -830,7 +884,7 @@ class TestPureFp16SaveLoad(unittest.TestCase): obj = { 'model': resnet.state_dict(), 'opt': optimizer.state_dict(), - 'scaler': scaler.state_dict() + 'scaler': scaler.state_dict(), } path = os.path.join(self.temp_dir.name, 'model.pdparams') paddle.save(obj, path) @@ -838,39 +892,40 @@ class TestPureFp16SaveLoad(unittest.TestCase): obj_load = paddle.load(path) resnet = ResNet(use_cudnn=True) optimizer = optimizer_setting( - train_parameters, parameter_list=resnet.parameters()) + train_parameters, parameter_list=resnet.parameters() + ) resnet.set_state_dict(obj_load['model']) optimizer.set_state_dict(obj_load['opt']) scaler.load_state_dict(obj_load['scaler']) - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, + optimizers=optimizer, + level='O2', + save_dtype='float32', + ) if use_data_loader: train_reader._reset() return dy_out, dy_param_value, dy_grad_value def test_with_save_load(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_use_save_load = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=True) - out_no_save_load = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=False) + out_use_save_load = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=True + ) + out_no_save_load = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=False + ) print('save_load:', out_use_save_load[0], out_no_save_load[0]) - np.testing.assert_allclose(out_use_save_load[0], - out_no_save_load[0], - rtol=1e-05) + np.testing.assert_allclose( + out_use_save_load[0], out_no_save_load[0], rtol=1e-05 + ) func_isinstance() class TestPureFp16InferenceSaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -886,21 +941,20 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, - (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( + 'int64' + ) return image, label def __len__(self): return self.num_samples class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -911,10 +965,12 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): def train(layer, loader, loss_fn, opt): for epoch_id in range(EPOCH_NUM): for batch_id, (image, label) in enumerate(loader()): - with paddle.amp.auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2'): + with paddle.amp.auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O2', + ): out = layer(image) loss = loss_fn(out, label) loss.backward() @@ -923,27 +979,31 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): # train layer = LinearNet() - adam = paddle.optimizer.Adam(learning_rate=0.001, - parameters=layer.parameters(), - multi_precision=True) + adam = paddle.optimizer.Adam( + learning_rate=0.001, + parameters=layer.parameters(), + multi_precision=True, + ) loss_fn = nn.CrossEntropyLoss() - layer, adam = paddle.amp.decorate(models=layer, - optimizers=adam, - save_dtype='float32') + layer, adam = paddle.amp.decorate( + models=layer, optimizers=adam, save_dtype='float32' + ) dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) train(layer, loader, loss_fn, adam) # save path = os.path.join(self.temp_dir.name, 'example_model/linear') - paddle.jit.save(layer, - path, - input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')]) + paddle.jit.save( + layer, path, input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')] + ) # jit.load loaded_layer = paddle.jit.load(path) @@ -957,12 +1017,17 @@ class TestPureFp16InferenceSaveLoad(unittest.TestCase): # load_inference_model paddle.enable_static() exe = paddle.static.Executor() - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.load_inference_model(path, exe)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(path, exe) tensor_img = x - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("pred.numpy()", pred.numpy()) print("result", results[0]) np.testing.assert_array_equal(pred.numpy(), results[0]) @@ -977,11 +1042,13 @@ class TestResnet2(unittest.TestCase): Use paddle-2.0 API """ - def train_resnet(self, - enable_amp=True, - level='O1', - use_data_loader=False, - use_param_group=False): + def train_resnet( + self, + enable_amp=True, + level='O1', + use_data_loader=False, + use_param_group=False, + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -1005,43 +1072,42 @@ class TestResnet2(unittest.TestCase): # NOTE(zhiqiu): The Membership test operations(in / not in) calls "is" and "equal", # see details: https://docs.python.org/3/reference/expressions.html#membership-test-operations. # So do not use other_params = [p for p in resnet.parameters() if p not in conv_params] - optimizer = paddle.optimizer.Momentum(parameters=[{ - 'params': - conv_params, - 'learning_rate': - 0.01 - }, { - 'params': - other_params, - 'learning_rate': - 0.001 - }], - multi_precision=True) + optimizer = paddle.optimizer.Momentum( + parameters=[ + {'params': conv_params, 'learning_rate': 0.01}, + {'params': other_params, 'learning_rate': 0.001}, + ], + multi_precision=True, + ) else: optimizer = paddle.optimizer.SGD(parameters=resnet.parameters()) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader @@ -1054,13 +1120,19 @@ class TestResnet2(unittest.TestCase): if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -1085,8 +1157,9 @@ class TestResnet2(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -1098,70 +1171,65 @@ class TestResnet2(unittest.TestCase): return dy_out, dy_param_value, dy_grad_value def test_resnet(self): - def func_isinstance(): with fluid.dygraph.guard(): out_fp32 = self.train_resnet(enable_amp=False) out_amp = self.train_resnet(enable_amp=True) out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() def test_with_data_loader(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False, - use_data_loader=True) - out_amp = self.train_resnet(enable_amp=True, - use_data_loader=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, - use_data_loader=True, - level='O2') + out_fp32 = self.train_resnet( + enable_amp=False, use_data_loader=True + ) + out_amp = self.train_resnet( + enable_amp=True, use_data_loader=True + ) + out_pure_fp16 = self.train_resnet( + enable_amp=True, use_data_loader=True, level='O2' + ) print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() def test_param_group(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False, - use_data_loader=True, - use_param_group=True) - out_amp = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_param_group=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_param_group=True, - level='O2') + out_fp32 = self.train_resnet( + enable_amp=False, use_data_loader=True, use_param_group=True + ) + out_amp = self.train_resnet( + enable_amp=True, use_data_loader=True, use_param_group=True + ) + out_pure_fp16 = self.train_resnet( + enable_amp=True, + use_data_loader=True, + use_param_group=True, + level='O2', + ) print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() @@ -1182,42 +1250,54 @@ class TestResnet(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) optimizer = paddle.optimizer.Momentum( - parameters=resnet.parameters(), multi_precision=True) + parameters=resnet.parameters(), multi_precision=True + ) np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.fluid.dygraph.AmpScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.fluid.dygraph.AmpScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if enable_amp and (level == 'O2'): resnet, optimizer = paddle.fluid.dygraph.amp_decorate( - models=resnet, optimizers=optimizer, level='O2') + models=resnet, optimizers=optimizer, level='O2' + ) for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: break - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(dy_x_data) label = fluid.dygraph.to_variable(y_data) label.stop_gradient = True - with paddle.fluid.dygraph.amp_guard(enable=enable_amp, - level=level): + with paddle.fluid.dygraph.amp_guard( + enable=enable_amp, level=level + ): out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -1234,9 +1314,11 @@ class TestResnet(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -1247,31 +1329,27 @@ class TestResnet(unittest.TestCase): return dy_out, dy_param_value, dy_grad_value def test_resnet(self): - def func_isinstance(): out_fp32 = self.train_resnet(enable_amp=False) out_amp = self.train_resnet(enable_amp=True) out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=0.01) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=0.01 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.1 + ) func_isinstance() class TestLayerNormFp16(unittest.TestCase): - r''' layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 + r'''layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 and other params are fp32. ''' def test_layer_norm_fp16(self): - def func_isinstance(): if fluid.is_compiled_with_cuda(): with fluid.dygraph.guard(fluid.CUDAPlace(0)): @@ -1281,7 +1359,8 @@ class TestLayerNormFp16(unittest.TestCase): out = layer_norm(x) self.assertTrue( - out.dtype == fluid.core.VarDesc.VarType.FP16) + out.dtype == fluid.core.VarDesc.VarType.FP16 + ) func_isinstance() @@ -1293,45 +1372,41 @@ class TestBf16(unittest.TestCase): def train(self, enable_amp=True, amp_level='O1'): paddle.seed(100) - input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = paddle.nn.Conv2D(4, 6, (3, 3)) if amp_level == 'O2': - conv = paddle.amp.decorate(models=conv, - level=amp_level, - dtype='bfloat16') - with paddle.amp.auto_cast(enable=enable_amp, - level=amp_level, - dtype='bfloat16'): + conv = paddle.amp.decorate( + models=conv, level=amp_level, dtype='bfloat16' + ) + with paddle.amp.auto_cast( + enable=enable_amp, level=amp_level, dtype='bfloat16' + ): output = conv(input) output = output.cast('float32') return output.numpy() def test_bf16(self): - def func_isinstance(): - if fluid.core.is_compiled_with_cuda( - ) and fluid.core.is_bfloat16_supported(paddle.CUDAPlace(0)): + if ( + fluid.core.is_compiled_with_cuda() + and fluid.core.is_bfloat16_supported(paddle.CUDAPlace(0)) + ): out_fp32 = self.train(enable_amp=False) out_bf16_O1 = self.train(enable_amp=True, amp_level='O1') out_bf16_O2 = self.train(enable_amp=True, amp_level='O2') - np.testing.assert_allclose(out_fp32, - out_bf16_O1, - rtol=0.001, - atol=0.1) - np.testing.assert_allclose(out_fp32, - out_bf16_O2, - rtol=0.001, - atol=0.1) + np.testing.assert_allclose( + out_fp32, out_bf16_O1, rtol=0.001, atol=0.1 + ) + np.testing.assert_allclose( + out_fp32, out_bf16_O2, rtol=0.001, atol=0.1 + ) func_isinstance() class TestAmpWithPyLyer(unittest.TestCase): - def test_pylayer(self): - class MyMM(PyLayer): - @staticmethod def forward(ctx, a, b): ctx.save_for_backward(a, b) @@ -1356,9 +1431,7 @@ class TestAmpWithPyLyer(unittest.TestCase): class TestAmpWithHook(unittest.TestCase): - def test_hook_change_dtype(self): - def func_isinstance(): with paddle.fluid.dygraph.guard(): v = paddle.rand([3, 3]) @@ -1379,7 +1452,6 @@ class TestAmpWithHook(unittest.TestCase): func_isinstance() def test_hook_change_place(self): - def func_isinstance(): with paddle.fluid.dygraph.guard(): v = paddle.rand([3, 3]) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_ir_pass_pipeline.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_ir_pass_pipeline.py index 1b445f8f987..7d11c03a1f1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_ir_pass_pipeline.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_ir_pass_pipeline.py @@ -17,7 +17,6 @@ import test_pipeline class TestPipelineWithIRPass(test_pipeline.TestPipeline): - def need_envs(self): return {'FLAGS_apply_pass_to_program': '1'} diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_mixed_precision.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_mixed_precision.py index 37fcf7c6509..46f12447b1d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_mixed_precision.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_mixed_precision.py @@ -22,7 +22,6 @@ paddle.enable_static() class SimpleNet(nn.Layer): - def __init__(self, input_size, output_size): super(SimpleNet, self).__init__() self.linear1 = nn.Linear(input_size, output_size) @@ -36,16 +35,15 @@ class SimpleNet(nn.Layer): x = self.linear1(x) # currently, paddle's relu may hide nan/inf, relu(nan) = 0, relu(inf)= inf # so, do not use it here. - #x = self.relu1(x) + # x = self.relu1(x) x = self.linear2(x) - #x = self.relu2(x) + # x = self.relu2(x) x = self.linear3(x) return x class AMPTest(unittest.TestCase): - def setUp(self): self.place = paddle.CUDAPlace(0) @@ -61,10 +59,11 @@ class AMPTest(unittest.TestCase): loss = mse(out, label) opt = paddle.fluid.optimizer.Adam( - learning_rate=0.0001, parameter_list=model.parameters()) # 定义优化器 - opt = paddle.static.amp.decorate(opt, - init_loss_scaling=128.0, - use_dynamic_loss_scaling=True) + learning_rate=0.0001, parameter_list=model.parameters() + ) # 定义优化器 + opt = paddle.static.amp.decorate( + opt, init_loss_scaling=128.0, use_dynamic_loss_scaling=True + ) opt.minimize(loss) return model, loss, opt @@ -79,11 +78,17 @@ class AMPTest(unittest.TestCase): model, loss, opt = self.net() weight = model.linear1.weight moment1 = opt._optimizer._get_accumulator( - opt._optimizer._moment1_acc_str, weight) + opt._optimizer._moment1_acc_str, weight + ) beta_pow1 = opt._optimizer._get_accumulator( - opt._optimizer._beta1_pow_acc_str, weight) + opt._optimizer._beta1_pow_acc_str, weight + ) fetch_list = [ - loss, weight, moment1, beta_pow1, 'find_infinite_scale.tmp_0' + loss, + weight, + moment1, + beta_pow1, + 'find_infinite_scale.tmp_0', ] exe = paddle.static.Executor(self.place) @@ -98,20 +103,24 @@ class AMPTest(unittest.TestCase): ] weight_, moment1_, beta_pow1_ = exe.run( - startup_prog, fetch_list=[weight, moment1, beta_pow1]) - pre_weight_, pre_moment1_, pre_beta_pow1_ = weight_, moment1_, beta_pow1_ + startup_prog, fetch_list=[weight, moment1, beta_pow1] + ) + pre_weight_, pre_moment1_, pre_beta_pow1_ = ( + weight_, + moment1_, + beta_pow1_, + ) for i in range(nums_batch): if i % 2: train_data[i][10] = np.inf loss_, weight_, moment1_, beta_pow1_, found_inf = exe.run( main_prog, - feed={ - "X": train_data[i], - "Y": labels[i] - }, - fetch_list=fetch_list) - print(loss_, weight_[0][0], moment1_[0][0], beta_pow1_, - found_inf) + feed={"X": train_data[i], "Y": labels[i]}, + fetch_list=fetch_list, + ) + print( + loss_, weight_[0][0], moment1_[0][0], beta_pow1_, found_inf + ) if i % 2: self.assertTrue(found_inf) np.testing.assert_array_equal(weight_, pre_weight_) @@ -122,7 +131,11 @@ class AMPTest(unittest.TestCase): self.assertFalse(np.array_equal(weight_, pre_weight_)) self.assertFalse(np.array_equal(moment1_, pre_moment1_)) self.assertFalse(np.array_equal(beta_pow1_, pre_beta_pow1_)) - pre_weight_, pre_moment1_, pre_beta_pow1_ = weight_, moment1_, beta_pow1_ + pre_weight_, pre_moment1_, pre_beta_pow1_ = ( + weight_, + moment1_, + beta_pow1_, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py index 0d22628c8aa..a583afda701 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py @@ -19,7 +19,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestParallelClassCenterSample(TestMultipleGpus): - def test_parallel_class_center_sample(self): self.run_mnist_2gpu('parallel_class_center_sample.py') self.run_mnist_2gpu('parallel_class_center_sample.py', eager_mode=False) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_control_flow.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_control_flow.py index c991818a5fb..9bbccc25e73 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_control_flow.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_control_flow.py @@ -22,7 +22,6 @@ flag_name = os.path.splitext(__file__)[0] class TestDygraphControlFlowSame(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -31,14 +30,15 @@ class TestDygraphControlFlowSame(TestDistBase): def test_net(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_control_flow_same.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_control_flow_same.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestFleetDygraphControlFlowSame(TestDygraphControlFlowSame): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -48,7 +48,6 @@ class TestFleetDygraphControlFlowSame(TestDygraphControlFlowSame): class TestFleetDygraphControlFlowSameAccGrad(TestDygraphControlFlowSame): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -58,7 +57,6 @@ class TestFleetDygraphControlFlowSameAccGrad(TestDygraphControlFlowSame): class TestDygraphControlFlowDiff(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -67,14 +65,15 @@ class TestDygraphControlFlowDiff(TestDistBase): def test_net(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_control_flow_different.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_control_flow_different.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestFleetDygraphControlFlowDiff(TestDygraphControlFlowDiff): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -84,7 +83,6 @@ class TestFleetDygraphControlFlowDiff(TestDygraphControlFlowDiff): class TestFleetDygraphControlFlowDiffAccGrad(TestDygraphControlFlowDiff): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mnist.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mnist.py index 9d5fc508727..a0933061bce 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mnist.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mnist.py @@ -25,7 +25,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphMnist(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -38,14 +37,14 @@ class TestParallelDygraphMnist(TestDistBase): os.path.abspath("../../parallel_dygraph_mnist.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) -#TODO(liuyuhui): Multi-Card Baidu Kunlun XPU training exist accuracy problems -#it is difficult to find out immediately where the problem is, -#and we will work with frameworkers' help to fix it. +# TODO(liuyuhui): Multi-Card Baidu Kunlun XPU training exist accuracy problems +# it is difficult to find out immediately where the problem is, +# and we will work with frameworkers' help to fix it. class TestParallelDygraphMnistXPU(TestDistBase): - def _setup_config(self): self._sync_mode = False self._bkcl_mode = True @@ -58,18 +57,17 @@ class TestParallelDygraphMnistXPU(TestDistBase): os.path.abspath("../../parallel_dygraph_mnist.py"), delta=1e-4, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestParallelDygraphMnistSpawn(TestDistSpawnRunner): - def test_mnist_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): self.check_dist_result_with_spawn(test_class=TestMnist, delta=1e-5) class TestParallelDygraphMnistAccGrad(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -84,11 +82,11 @@ class TestParallelDygraphMnistAccGrad(TestDistBase): os.path.abspath("../../parallel_dygraph_mnist.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestFleetDygraphMnistXPU(TestDistBase): - def _setup_config(self): self._sync_mode = False self._bkcl_mode = True @@ -102,7 +100,8 @@ class TestFleetDygraphMnistXPU(TestDistBase): os.path.abspath("../../parallel_dygraph_mnist.py"), delta=1e-4, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py index 21aa82db3dc..aeee6a46676 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py @@ -19,7 +19,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestModelParallelLayer(TestMultipleGpus): - def test_hybrid_parallel_mp_layer(self): self.run_mnist_2gpu('hybrid_parallel_mp_layers.py') self.run_mnist_2gpu('hybrid_parallel_mp_layers.py', eager_mode=False) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync.py index 8a1c1e7a0c5..c44af13dc4e 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync.py @@ -27,7 +27,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphNoSync(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -36,14 +35,15 @@ class TestParallelDygraphNoSync(TestDistBase): def test_no_sync(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_no_sync.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_no_sync.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphNoSyncUnusedParam(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -52,14 +52,15 @@ class TestParallelDygraphNoSyncUnusedParam(TestDistBase): def test_no_sync_ununsed_param(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_no_sync_unused_params.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_no_sync_unused_params.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphNoSyncControlFlow(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -68,39 +69,40 @@ class TestParallelDygraphNoSyncControlFlow(TestDistBase): def test_no_sync_control_flow(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_no_sync_control_flow.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_no_sync_control_flow.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphNoSyncSpawn(TestDistSpawnRunner): - def test_no_sync_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): self.check_dist_result_with_spawn(test_class=TestNoSync, delta=1e-5) class TestParallelDygraphNoSyncUnusedParamSpawn(TestDistSpawnRunner): - def _args_config(self, args): args.find_unused_parameters = True def test_no_sync_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): - self.check_dist_result_with_spawn(test_class=TestNoSyncUnusedParam, - delta=1e-5) + self.check_dist_result_with_spawn( + test_class=TestNoSyncUnusedParam, delta=1e-5 + ) class TestParallelDygraphNoSyncControlFlowSpawn(TestDistSpawnRunner): - def _args_config(self, args): args.find_unused_parameters = True def test_no_sync_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): - self.check_dist_result_with_spawn(test_class=TestNoSyncControlFlow, - delta=1e-5) + self.check_dist_result_with_spawn( + test_class=TestNoSyncControlFlow, delta=1e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py index d170cd0ed54..e6467e07815 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py @@ -19,11 +19,11 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestDataParallelLayer(TestMultipleGpus): - def test_parallel_dygraph_dataparallel_no_sync(self): self.run_mnist_2gpu('parallel_dygraph_no_sync_gradient_check.py') - self.run_mnist_2gpu('parallel_dygraph_no_sync_gradient_check.py', - eager_mode=False) + self.run_mnist_2gpu( + 'parallel_dygraph_no_sync_gradient_check.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py index ebd1096fdc2..f45104de32c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py @@ -19,13 +19,14 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestHybridPipeParallel(TestMultipleGpus): - def test_hybrid_parallel_pp_layer(self): self.run_mnist_2gpu( - os.path.abspath('../../hybrid_parallel_pp_layer.py')) + os.path.abspath('../../hybrid_parallel_pp_layer.py') + ) self.run_mnist_2gpu( os.path.abspath('../../hybrid_parallel_pp_layer.py'), - eager_mode=False) + eager_mode=False, + ) def test_hybrid_parallel_pp_tuple_inputs(self): self.run_mnist_2gpu('hybrid_parallel_pp_embedding.py') @@ -33,8 +34,9 @@ class TestHybridPipeParallel(TestMultipleGpus): def test_hybrid_parallel_shared_weight(self): self.run_mnist_2gpu('hybrid_parallel_shared_weight.py') - self.run_mnist_2gpu('hybrid_parallel_shared_weight.py', - eager_mode=False) + self.run_mnist_2gpu( + 'hybrid_parallel_shared_weight.py', eager_mode=False + ) def test_pipeline_parallel_amp(self): self.run_mnist_2gpu('hybrid_parallel_pp_amp.py') @@ -46,8 +48,9 @@ class TestHybridPipeParallel(TestMultipleGpus): def test_hybrid_parallel_transformer(self): self.run_mnist_2gpu('hybrid_parallel_pp_transformer.py') - self.run_mnist_2gpu('hybrid_parallel_pp_transformer.py', - eager_mode=False) + self.run_mnist_2gpu( + 'hybrid_parallel_pp_transformer.py', eager_mode=False + ) def test_hybrid_parallel_save_load(self): self.run_mnist_2gpu('hybrid_parallel_pp_save_load.py') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py index 7af7d6dba1a..d09daa0764c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py @@ -19,17 +19,18 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestHybridPipeParallelWithVirtualStage(TestMultipleGpus): - def test_hybrid_parallel_pp_layer_with_virtual_stage(self): self.run_mnist_2gpu('hybrid_parallel_pp_layer_with_virtual_stage.py') def test_hybrid_parallel_pp_transformer_with_virtual_stage(self): self.run_mnist_2gpu( - 'hybrid_parallel_pp_transformer_with_virtual_stage.py') + 'hybrid_parallel_pp_transformer_with_virtual_stage.py' + ) def test_hybrid_parallel_save_load_with_virtual_stage(self): self.run_mnist_2gpu( - 'hybrid_parallel_pp_save_load_with_virtual_stage.py') + 'hybrid_parallel_pp_save_load_with_virtual_stage.py' + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py index 842f7166118..4ea7ebc9162 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py @@ -19,7 +19,12 @@ import copy import os import subprocess -from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc +from paddle.distributed.utils.launch_utils import ( + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, +) def get_cluster_from_args(selected_gpus): @@ -47,17 +52,19 @@ def get_gpus(selected_gpus): return selected_gpus -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - eager_mode=True, - log_dir=None): +def start_local_trainers( + cluster, + pod, + training_script, + training_script_args, + eager_mode=True, + log_dir=None, +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -68,7 +75,7 @@ def start_local_trainers(cluster, "PADDLE_TRAINER_ID": "%d" % t.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } if not eager_mode: @@ -101,10 +108,11 @@ def start_local_trainers(cluster, class TestMultipleGpus(unittest.TestCase): - def run_2gpu(self, target_file_name, eager_mode=True): - if not fluid.core.is_compiled_with_cuda( - ) or fluid.core.get_cuda_device_count() == 0: + if ( + not fluid.core.is_compiled_with_cuda() + or fluid.core.get_cuda_device_count() == 0 + ): return selected_gpus = get_gpus('0,1') @@ -113,11 +121,13 @@ class TestMultipleGpus(unittest.TestCase): cluster, pod = get_cluster_from_args(selected_gpus) - procs = start_local_trainers(cluster, - pod, - eager_mode=eager_mode, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + eager_mode=eager_mode, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_endpoints()) @@ -129,7 +139,6 @@ class TestMultipleGpus(unittest.TestCase): class TestDataParallelQAT(TestMultipleGpus): - def test_multiple_gpus_qat(self): self.run_2gpu('hybrid_parallel_qat.py') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_se_resnext.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_se_resnext.py index 0bdbb20787e..68fb9468f4b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_se_resnext.py @@ -25,7 +25,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphSeResNeXt(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -33,18 +32,20 @@ class TestParallelDygraphSeResNeXt(TestDistBase): def test_se_resnext(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_se_resnext.py", - delta=0.01, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_se_resnext.py", + delta=0.01, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSeResNeXtSpawn(TestDistSpawnRunner): - def test_se_resnext_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): - self.check_dist_result_with_spawn(test_class=TestSeResNeXt, - delta=0.01) + self.check_dist_result_with_spawn( + test_class=TestSeResNeXt, delta=0.01 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py index f6c6e3378e4..d0b17e86257 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py @@ -23,8 +23,9 @@ class TestHybridParallel(TestMultipleGpus): # check sharding logic as well as the accuracy with single mode def test_hybrid_parallel_sharding_logic(self): self.run_mnist_2gpu('hybrid_parallel_sharding_model.py') - self.run_mnist_2gpu('hybrid_parallel_sharding_model.py', - eager_mode=False) + self.run_mnist_2gpu( + 'hybrid_parallel_sharding_model.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding.py index 2f8ea8feddd..b1d304350cf 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding.py @@ -25,7 +25,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphSparseEmdedding(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -37,11 +36,11 @@ class TestParallelDygraphSparseEmdedding(TestDistBase): os.path.abspath("../../parallel_dygraph_sparse_embedding.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestParallelDygraphSparseEmdeddingFP64(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -49,19 +48,22 @@ class TestParallelDygraphSparseEmdeddingFP64(TestDistBase): def test_sparse_embedding_fp64(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place(os.path.abspath( - "../../parallel_dygraph_sparse_embedding_fp64.py"), - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + os.path.abspath( + "../../parallel_dygraph_sparse_embedding_fp64.py" + ), + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSparseEmdeddingSpawn(TestDistSpawnRunner): - def test_sparse_embedding_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): - self.check_dist_result_with_spawn(test_class=TestSparseEmbedding, - delta=1e-5) + self.check_dist_result_with_spawn( + test_class=TestSparseEmbedding, delta=1e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding_over_height.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding_over_height.py index b082da1b910..b82bce4e1e1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding_over_height.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding_over_height.py @@ -19,13 +19,14 @@ import unittest import paddle.fluid as fluid from test_dist_base import TestDistBase from spawn_runner_base import TestDistSpawnRunner -from parallel_dygraph_sparse_embedding_over_height import TestSparseEmbeddingOverHeight +from parallel_dygraph_sparse_embedding_over_height import ( + TestSparseEmbeddingOverHeight, +) flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphSparseEmdeddingOverHeight(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -33,19 +34,22 @@ class TestParallelDygraphSparseEmdeddingOverHeight(TestDistBase): def test_sparse_embedding(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place(os.path.abspath( - "../../parallel_dygraph_sparse_embedding_over_height.py"), - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + os.path.abspath( + "../../parallel_dygraph_sparse_embedding_over_height.py" + ), + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSparseEmdeddingOverHeightSpawn(TestDistSpawnRunner): - def test_sparse_embedding_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): self.check_dist_result_with_spawn( - test_class=TestSparseEmbeddingOverHeight, delta=1e-5) + test_class=TestSparseEmbeddingOverHeight, delta=1e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sync_batch_norm.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sync_batch_norm.py index 26e62bc75ec..1dd0ae27849 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sync_batch_norm.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sync_batch_norm.py @@ -22,7 +22,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphMnist(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -30,10 +29,12 @@ class TestParallelDygraphMnist(TestDistBase): def test_mnist(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_sync_batch_norm.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_sync_batch_norm.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py index c2b1b17fc52..c01d6ed6f08 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py @@ -19,7 +19,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestHybridParallel(TestMultipleGpus): - def test_hybrid_parallel_mp_random(self): self.run_mnist_2gpu('hybrid_parallel_mp_random.py') self.run_mnist_2gpu('hybrid_parallel_mp_random.py', eager_mode=False) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_transformer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_transformer.py index 8d12dfdce58..58ad40894fe 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_transformer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_transformer.py @@ -22,7 +22,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphTransformer(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -30,14 +29,15 @@ class TestParallelDygraphTransformer(TestDistBase): def test_transformer(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_transformer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphTransformerAccGrad(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -47,10 +47,12 @@ class TestParallelDygraphTransformerAccGrad(TestDistBase): def test_transformer(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_transformer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_unused_variables.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_unused_variables.py index 0a740b13c4e..a36eae2ccda 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_unused_variables.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_unused_variables.py @@ -25,7 +25,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphUnusedVar(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -37,11 +36,11 @@ class TestParallelDygraphUnusedVar(TestDistBase): os.path.abspath("../../parallel_dygraph_unused_variables.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestFleetDygraphUnusedVar(TestParallelDygraphUnusedVar): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -50,15 +49,14 @@ class TestFleetDygraphUnusedVar(TestParallelDygraphUnusedVar): class TestSparseEmbeddingUnusedVarsSpawn(TestDistSpawnRunner): - def test_mnist_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): self.check_dist_result_with_spawn( - test_class=TestSparseEmbeddingUnusedVars, delta=1e-5) + test_class=TestSparseEmbeddingUnusedVars, delta=1e-5 + ) class TestParallelDygraphNoVar(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -70,11 +68,11 @@ class TestParallelDygraphNoVar(TestDistBase): os.path.abspath("../../parallel_dygraph_none_var.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestParallelDygraphSharedUnusedVariables(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -86,7 +84,8 @@ class TestParallelDygraphSharedUnusedVariables(TestDistBase): os.path.abspath("../../parallel_dygraph_shared_unused_var.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py index b983be50f73..b4f7cfb39d8 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py @@ -19,11 +19,11 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestParallelMarginSoftmaxWithCrossEntropy(TestMultipleGpus): - def test_parallel_margin_cross_entropy(self): self.run_mnist_2gpu('parallel_margin_cross_entropy.py') - self.run_mnist_2gpu('parallel_margin_cross_entropy.py', - eager_mode=False) + self.run_mnist_2gpu( + 'parallel_margin_cross_entropy.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_pipeline.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_pipeline.py index e9fd934f286..a495ac13c77 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_pipeline.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_pipeline.py @@ -24,7 +24,6 @@ flag_name = os.path.splitext(__file__)[0] class TestPipeline(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -41,19 +40,23 @@ class TestPipeline(TestDistBase): # Now pipeline only gets the loss value of the last # microbatch, so it is not consistable with the # non-pipeline one. - self.check_with_place("pipeline_mnist.py", - delta=1e0, - check_error_log=True, - log_name=flag_name, - need_envs=self.need_envs()) + self.check_with_place( + "pipeline_mnist.py", + delta=1e0, + check_error_log=True, + log_name=flag_name, + need_envs=self.need_envs(), + ) def test_dist_train_multi_device(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("pipeline_mnist_multi_device.py", - check_error_log=True, - delta=1e0, - log_name=flag_name, - need_envs=self.need_envs()) + self.check_with_place( + "pipeline_mnist_multi_device.py", + check_error_log=True, + delta=1e0, + log_name=flag_name, + need_envs=self.need_envs(), + ) def test_dist_train_one_device(self): if fluid.core.is_compiled_with_cuda(): @@ -61,7 +64,8 @@ class TestPipeline(TestDistBase): "pipeline_mnist_one_device.py", check_error_log=True, log_name=flag_name, - need_envs={"PADDLE_MANUAL_PIPELINE_STAGE": "0"}) + need_envs={"PADDLE_MANUAL_PIPELINE_STAGE": "0"}, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_recv_save_op.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_recv_save_op.py index 927b02d74eb..c75aadd450a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_recv_save_op.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_recv_save_op.py @@ -46,20 +46,18 @@ def run_pserver(pserver_id): param.set(param_array, place) optimize_block = program._create_block(program.global_block().idx) - program.global_block().append_op(type="listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs={ - "optimize_blocks": - [optimize_block], - "endpoint": - '127.0.0.1:0', - "Fanin": - 1, - "distributed_mode": - DistributedMode.SYNC, - "grad_to_block_id": [] - }) + program.global_block().append_op( + type="listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs={ + "optimize_blocks": [optimize_block], + "endpoint": '127.0.0.1:0', + "Fanin": 1, + "distributed_mode": DistributedMode.SYNC, + "grad_to_block_id": [], + }, + ) exe = fluid.Executor(place) exe.run(program) @@ -67,12 +65,11 @@ def run_pserver(pserver_id): @unittest.skip("do not need currently") class TestListenAndServOp(unittest.TestCase): - def setUp(self): self.ps_timeout = 5 def _start_pserver(self, pserver_id, pserver_func): - p = Process(target=pserver_func, args=(pserver_id, )) + p = Process(target=pserver_func, args=(pserver_id,)) p.daemon = True p.start() return p @@ -104,15 +101,17 @@ class TestListenAndServOp(unittest.TestCase): emaps = ['127.0.0.1:' + str(port0), '127.0.0.1:' + str(port1)] # create and run recv and save operator - remote_recv_op = Operator("recv_save", - trainer_id=0, - shape=[10, 8], - slice_shapes=["5,8", "5,8"], - slice_varnames=["table", "table"], - remote_varnames=['table', 'table'], - is_sparse=False, - endpoints=emaps, - file_path=model_file) + remote_recv_op = Operator( + "recv_save", + trainer_id=0, + shape=[10, 8], + slice_shapes=["5,8", "5,8"], + slice_varnames=["table", "table"], + remote_varnames=['table', 'table'], + is_sparse=False, + endpoints=emaps, + file_path=model_file, + ) remote_recv_op.run(scope, place) @@ -125,44 +124,53 @@ class TestListenAndServOp(unittest.TestCase): type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[10, 8], dtype="float32", - persistable=True) + persistable=True, + ) slice0 = load_block.create_var( name="var.slice0", type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[3, 8], dtype="float32", - persistable=True) + persistable=True, + ) slice1 = load_block.create_var( name="var.slice1", type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[5, 8], dtype="float32", - persistable=True) - - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [origin]}, - attrs={'file_path': model_file}) - - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [slice0]}, - attrs={ - 'file_path': model_file, - 'seek': 2 * 8, - 'shape': slice0.shape - }) - - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [slice1]}, - attrs={ - 'file_path': model_file, - 'seek': 5 * 8, - 'shape': slice1.shape - }) + persistable=True, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [origin]}, + attrs={'file_path': model_file}, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice0]}, + attrs={ + 'file_path': model_file, + 'seek': 2 * 8, + 'shape': slice0.shape, + }, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice1]}, + attrs={ + 'file_path': model_file, + 'seek': 5 * 8, + 'shape': slice1.shape, + }, + ) exe = fluid.Executor(place=fluid.CPUPlace()) exe.run(load_prog) @@ -179,8 +187,9 @@ class TestListenAndServOp(unittest.TestCase): np.testing.assert_equal(origin[5:10], slice1) def _save_by_io_persistables(self, place, port0, port1, dirname, var_name): - self._run_nce_op_two_pserver(place, port0, port1, - os.path.join(dirname, var_name)) + self._run_nce_op_two_pserver( + place, port0, port1, os.path.join(dirname, var_name) + ) def test_recv_save_op_remote(self): # run pserver on CPU in sync mode @@ -198,8 +207,9 @@ class TestListenAndServOp(unittest.TestCase): param_name = "table" for place in places: - self._save_by_io_persistables(place, port0, port1, param_dir, - param_name) + self._save_by_io_persistables( + place, port0, port1, param_dir, param_name + ) # raise SIGTERM to pserver os.kill(p0.pid, signal.SIGINT) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_rnn_dp.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_rnn_dp.py index 8976f7919d2..9d116d74b25 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_rnn_dp.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_rnn_dp.py @@ -24,27 +24,30 @@ paddle.enable_static() class RNNEncoder(nn.Layer): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - dropout=0.0, - pooling_type=None, - **kwargs): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + dropout=0.0, + pooling_type=None, + **kwargs + ): super().__init__() self._input_size = input_size self._hidden_size = hidden_size self._direction = direction self._pooling_type = pooling_type - self.rnn_layer = nn.SimpleRNN(input_size=input_size, - hidden_size=hidden_size, - num_layers=num_layers, - direction=direction, - dropout=dropout, - **kwargs) + self.rnn_layer = nn.SimpleRNN( + input_size=input_size, + hidden_size=hidden_size, + num_layers=num_layers, + direction=direction, + dropout=dropout, + **kwargs + ) def get_input_dim(self): return self._input_size @@ -57,34 +60,40 @@ class RNNEncoder(nn.Layer): def forward(self, inputs, sequence_length): encoded_text, last_hidden = self.rnn_layer( - inputs, sequence_length=sequence_length) + inputs, sequence_length=sequence_length + ) output = paddle.max(encoded_text, axis=1) return output class RNNModel(nn.Layer): - - def __init__(self, - vocab_size, - num_classes, - emb_dim=128, - padding_idx=0, - rnn_hidden_size=198, - direction='forward', - rnn_layers=1, - dropout_rate=0.0, - pooling_type=None, - fc_hidden_size=96): + def __init__( + self, + vocab_size, + num_classes, + emb_dim=128, + padding_idx=0, + rnn_hidden_size=198, + direction='forward', + rnn_layers=1, + dropout_rate=0.0, + pooling_type=None, + fc_hidden_size=96, + ): super().__init__() - self.embedder = nn.Embedding(num_embeddings=vocab_size, - embedding_dim=emb_dim, - padding_idx=padding_idx) - self.rnn_encoder = RNNEncoder(emb_dim, - rnn_hidden_size, - num_layers=rnn_layers, - direction=direction, - dropout=dropout_rate, - pooling_type=pooling_type) + self.embedder = nn.Embedding( + num_embeddings=vocab_size, + embedding_dim=emb_dim, + padding_idx=padding_idx, + ) + self.rnn_encoder = RNNEncoder( + emb_dim, + rnn_hidden_size, + num_layers=rnn_layers, + direction=direction, + dropout=dropout_rate, + pooling_type=pooling_type, + ) self.fc = nn.Linear(self.rnn_encoder.get_output_dim(), fc_hidden_size) self.output_layer = nn.Linear(fc_hidden_size, num_classes) @@ -97,26 +106,30 @@ class RNNModel(nn.Layer): def rnn_pretrain_forward(train_program, start_program, topo=None): - with static.program_guard(train_program, - start_program), paddle.utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), paddle.utils.unique_name.guard(): batch_size = 1 - tokens = static.data(name="tokens", - shape=[batch_size, -1], - dtype="int64") + tokens = static.data( + name="tokens", shape=[batch_size, -1], dtype="int64" + ) seq_len = static.data(name="ids", shape=[batch_size], dtype="int64") labels = static.data(name="labels", shape=[batch_size], dtype="int64") data_holders = [tokens, seq_len, labels] vocab_size = 10 num_classes = 2 pad_token_id = 0 - model = RNNModel(vocab_size, - num_classes, - direction='forward', - padding_idx=pad_token_id, - pooling_type='max') - - optimizer = paddle.optimizer.Adam(parameters=model.parameters(), - learning_rate=0.001) + model = RNNModel( + vocab_size, + num_classes, + direction='forward', + padding_idx=pad_token_id, + pooling_type='max', + ) + + optimizer = paddle.optimizer.Adam( + parameters=model.parameters(), learning_rate=0.001 + ) criterion = paddle.nn.CrossEntropyLoss() preds = model(tokens, seq_len) loss = criterion(preds, labels) @@ -125,23 +138,30 @@ def rnn_pretrain_forward(train_program, start_program, topo=None): class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def test_rnn_raw_optimizer(self): import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) train_program = static.Program() start_program = static.Program() - train_program, start_program, loss, optimizer, data_holders = \ - rnn_pretrain_forward(train_program, start_program) + ( + train_program, + start_program, + loss, + optimizer, + data_holders, + ) = rnn_pretrain_forward(train_program, start_program) with paddle.static.program_guard( - train_program, start_program), paddle.utils.unique_name.guard(): + train_program, start_program + ), paddle.utils.unique_name.guard(): strategy = fleet.DistributedStrategy() strategy.without_graph_optimization = True strategy.fuse_all_reduce_ops = True diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_static_model_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_static_model_parallel.py index 06a9b056822..6f51af27952 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_static_model_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_static_model_parallel.py @@ -23,7 +23,6 @@ flag_name = os.path.splitext(__file__)[0] class TestStaticModelParallel(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,27 +32,36 @@ class TestStaticModelParallel(TestDistBase): def test_dist_static_model_parallel(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_by_row.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_by_row.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) def test_dist_static_model_parallel2(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_by_col.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_by_col.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) def test_dist_static_model_parallel3(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_embedding.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_embedding.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_tcp_store.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_tcp_store.py index ff5cff2c9cb..01a39bc6241 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_tcp_store.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_tcp_store.py @@ -18,7 +18,6 @@ import os class TestTCPStore(unittest.TestCase): - def test_tcp_store(self): dist_port = int(os.getenv("PADDLE_DIST_UT_PORT", 6170)) print("get dist_port:", dist_port) diff --git a/python/paddle/fluid/tests/unittests/collective/init_process_group.py b/python/paddle/fluid/tests/unittests/collective/init_process_group.py index dfb9dcf556c..f45e4004831 100644 --- a/python/paddle/fluid/tests/unittests/collective/init_process_group.py +++ b/python/paddle/fluid/tests/unittests/collective/init_process_group.py @@ -19,7 +19,6 @@ from paddle.fluid.framework import _test_eager_guard class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): self.config() diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/common.py b/python/paddle/fluid/tests/unittests/collective/multinode/common.py index cb2ff2ea09e..7849b6575c0 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/common.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/common.py @@ -17,8 +17,8 @@ from paddle.distributed import fleet def init_parallel_env(mode, global_batch_size, seed=1024): ''' - Args: - mode:(str) DP1-MP1-PP1-SH1-O1 + Args: + mode:(str) DP1-MP1-PP1-SH1-O1 ''' def parse_mode(mode): @@ -42,7 +42,7 @@ def init_parallel_env(mode, global_batch_size, seed=1024): "dp_degree": DP, "mp_degree": MP, "pp_degree": PP, - "sharding_degree": SH + "sharding_degree": SH, } accumulate_steps = 1 @@ -50,7 +50,7 @@ def init_parallel_env(mode, global_batch_size, seed=1024): if PP > 1: strategy.pipeline_configs = { "accumulate_steps": accumulate_steps, - "micro_batch_size": global_batch_size // DP // accumulate_steps + "micro_batch_size": global_batch_size // DP // accumulate_steps, } # set control in tensor parallel diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py index 5583041db8b..bb904b3d6bd 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py @@ -12,31 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from test_collective_multi_nodes import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_multi_nodes import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) class TestDygrapgHybridDP(TestCollectiveAPIRunnerBase): - def __init__(self): pass def check_pass(self, *args, **kwargs): from common import init_parallel_env import paddle + hcg = init_parallel_env("DP16-MP1-PP1-SH1-O1", 2) import numpy as np + dp_group = hcg.get_data_parallel_group() np.random.seed(1024) data = np.random.random((10 * dp_group.nranks, 100)).reshape( - (dp_group.nranks, -1, 100)) + (dp_group.nranks, -1, 100) + ) data_part = paddle.to_tensor(data[dp_group.rank]) paddle.distributed.collective.all_reduce(data_part) data_reduced = data_part data_sumed = np.sum(data, axis=0) - assert np.allclose(data_sumed, - data_reduced.numpy(), - rtol=1e-8, - atol=1e-8) + assert np.allclose( + data_sumed, data_reduced.numpy(), rtol=1e-8, atol=1e-8 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py index 6de0954b0e7..055e3e24fbd 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py @@ -15,7 +15,10 @@ import numpy as np import paddle import paddle.distributed.fleet as fleet -from test_collective_multi_nodes import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_multi_nodes import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) from paddle import nn import numpy as np @@ -28,15 +31,14 @@ def weight_init(mp, shape, col=True, seed=1024): else: if col: step = shape[1] // mp.nranks - _w = w[:, mp.rank * step:mp.rank * step + step] + _w = w[:, mp.rank * step : mp.rank * step + step] else: step = shape[0] // mp.nranks - _w = w[mp.rank * step:mp.rank * step + step, :] + _w = w[mp.rank * step : mp.rank * step + step, :] return paddle.fluid.initializer.NumpyArrayInitializer(_w) class Criterion(nn.Layer): - def __init__(self): super(Criterion, self).__init__() self.loss_func = nn.MSELoss(reduction="mean") @@ -47,7 +49,6 @@ class Criterion(nn.Layer): class ModelPipeline(fleet.meta_parallel.PipelineLayer): - def __init__(self, hcg): paddle.seed(1024) dp_linear = nn.Linear(32, 128) @@ -62,35 +63,38 @@ class ModelPipeline(fleet.meta_parallel.PipelineLayer): 512, weight_attr=weight_init(mp, (128, 512), True, 1204 + i), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( 512, 128, weight_attr=weight_init(mp, (512, 128), False, 2012 + i), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(128, - 512, - weight_attr=weight_init( - None, (128, 512), True, 1204 + i)) - mp_linear_2 = nn.Linear(512, - 128, - weight_attr=weight_init( - None, (512, 128), True, 2012 + i)) + mp_linear_1 = nn.Linear( + 128, + 512, + weight_attr=weight_init(None, (128, 512), True, 1204 + i), + ) + mp_linear_2 = nn.Linear( + 512, + 128, + weight_attr=weight_init(None, (512, 128), True, 2012 + i), + ) act = nn.ReLU6() layer_seq = nn.Sequential(mp_linear_1, mp_linear_2, act) self.layers_pp.append(layer_seq) out = nn.Linear(128, 32) self.layers_pp.append(out) - super(ModelPipeline, self).__init__(layers=self.layers_pp, - loss_fn=Criterion(), - topology=self.topology) + super(ModelPipeline, self).__init__( + layers=self.layers_pp, loss_fn=Criterion(), topology=self.topology + ) class Model(nn.Layer): - def __init__(self, hcg): super(Model, self).__init__() paddle.seed(1024) @@ -105,22 +109,26 @@ class Model(nn.Layer): 512, weight_attr=weight_init(mp, (128, 512), True, 1204 + i), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( 512, 128, weight_attr=weight_init(mp, (512, 128), False, 2012 + i), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(128, - 512, - weight_attr=weight_init( - None, (128, 512), True, 1204 + i)) - mp_linear_2 = nn.Linear(512, - 128, - weight_attr=weight_init( - None, (512, 128), True, 2012 + i)) + mp_linear_1 = nn.Linear( + 128, + 512, + weight_attr=weight_init(None, (128, 512), True, 1204 + i), + ) + mp_linear_2 = nn.Linear( + 512, + 128, + weight_attr=weight_init(None, (512, 128), True, 2012 + i), + ) act = nn.ReLU6() layer_seq = nn.Sequential(mp_linear_1, mp_linear_2, act) self.layers_pp.append(layer_seq) @@ -134,7 +142,6 @@ class Model(nn.Layer): class TestDygrapgHybridDPPPMP(TestCollectiveAPIRunnerBase): - def __init__(self): pass @@ -143,9 +150,11 @@ class TestDygrapgHybridDPPPMP(TestCollectiveAPIRunnerBase): from common import init_parallel_env import paddle from paddle.distributed import fleet + hcg = init_parallel_env("DP4-MP2-PP2-SH1-O1", 64) pp_degree = hcg.get_pipe_parallel_world_size() import numpy as np + crit = Criterion() if pp_degree <= 1: model = Model(hcg) @@ -154,10 +163,12 @@ class TestDygrapgHybridDPPPMP(TestCollectiveAPIRunnerBase): model_base = Model(None) - optimizer = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) optimizer_base = paddle.optimizer.Adam( - learning_rate=0.01, parameters=model_base.parameters()) + learning_rate=0.01, parameters=model_base.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py index 0c4eddffeec..04eacbc7fc4 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py @@ -15,7 +15,10 @@ import numpy as np import paddle import paddle.distributed.fleet as fleet -from test_collective_multi_nodes import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_multi_nodes import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) from paddle import nn import numpy as np @@ -28,15 +31,14 @@ def weight_init(mp, shape, col=True, seed=1024): else: if col: step = shape[1] // mp.nranks - _w = w[:, mp.rank * step:mp.rank * step + step] + _w = w[:, mp.rank * step : mp.rank * step + step] else: step = shape[0] // mp.nranks - _w = w[mp.rank * step:mp.rank * step + step, :] + _w = w[mp.rank * step : mp.rank * step + step, :] return paddle.fluid.initializer.NumpyArrayInitializer(_w) class Criterion(nn.Layer): - def __init__(self): super(Criterion, self).__init__() self.loss_func = nn.MSELoss(reduction="mean") @@ -47,7 +49,6 @@ class Criterion(nn.Layer): class ModelPipeline(fleet.meta_parallel.PipelineLayer): - def __init__(self, hcg): paddle.seed(1024) dp_linear = nn.Linear(32, 128) @@ -62,35 +63,38 @@ class ModelPipeline(fleet.meta_parallel.PipelineLayer): 512, weight_attr=weight_init(mp, (128, 512), True, 1204 + i), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( 512, 128, weight_attr=weight_init(mp, (512, 128), False, 2012 + i), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(128, - 512, - weight_attr=weight_init( - None, (128, 512), True, 1204 + i)) - mp_linear_2 = nn.Linear(512, - 128, - weight_attr=weight_init( - None, (512, 128), True, 2012 + i)) + mp_linear_1 = nn.Linear( + 128, + 512, + weight_attr=weight_init(None, (128, 512), True, 1204 + i), + ) + mp_linear_2 = nn.Linear( + 512, + 128, + weight_attr=weight_init(None, (512, 128), True, 2012 + i), + ) act = nn.ReLU6() layer_seq = nn.Sequential(mp_linear_1, mp_linear_2, act) self.layers_pp.append(layer_seq) out = nn.Linear(128, 32) self.layers_pp.append(out) - super(ModelPipeline, self).__init__(layers=self.layers_pp, - loss_fn=Criterion(), - topology=self.topology) + super(ModelPipeline, self).__init__( + layers=self.layers_pp, loss_fn=Criterion(), topology=self.topology + ) class Model(nn.Layer): - def __init__(self, hcg): super(Model, self).__init__() paddle.seed(1024) @@ -105,22 +109,26 @@ class Model(nn.Layer): 512, weight_attr=weight_init(mp, (128, 512), True, 1204 + i), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( 512, 128, weight_attr=weight_init(mp, (512, 128), False, 2012 + i), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(128, - 512, - weight_attr=weight_init( - None, (128, 512), True, 1204 + i)) - mp_linear_2 = nn.Linear(512, - 128, - weight_attr=weight_init( - None, (512, 128), True, 2012 + i)) + mp_linear_1 = nn.Linear( + 128, + 512, + weight_attr=weight_init(None, (128, 512), True, 1204 + i), + ) + mp_linear_2 = nn.Linear( + 512, + 128, + weight_attr=weight_init(None, (512, 128), True, 2012 + i), + ) act = nn.ReLU6() layer_seq = nn.Sequential(mp_linear_1, mp_linear_2, act) self.layers_pp.append(layer_seq) @@ -134,7 +142,6 @@ class Model(nn.Layer): class TestDygraphHybridFp16(TestCollectiveAPIRunnerBase): - def __init__(self): pass @@ -143,9 +150,11 @@ class TestDygraphHybridFp16(TestCollectiveAPIRunnerBase): from common import init_parallel_env import paddle from paddle.distributed import fleet + hcg = init_parallel_env("DP4-MP2-PP2-SH1-O1", 64) pp_degree = hcg.get_pipe_parallel_world_size() import numpy as np + crit = Criterion() if pp_degree <= 1: model = Model(hcg) @@ -154,17 +163,20 @@ class TestDygraphHybridFp16(TestCollectiveAPIRunnerBase): model_base = Model(None) - optimizer = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters(), - multi_precision=True) + optimizer = paddle.optimizer.Adam( + learning_rate=0.01, + parameters=model.parameters(), + multi_precision=True, + ) optimizer_base = paddle.optimizer.Adam( - learning_rate=0.01, parameters=model_base.parameters()) + learning_rate=0.01, parameters=model_base.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=4096) scaler = fleet.distributed_scaler(scaler) - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) @@ -177,9 +189,9 @@ class TestDygraphHybridFp16(TestCollectiveAPIRunnerBase): for _ in range(2): if pp_degree > 1: with paddle.amp.auto_cast(True, level='O2'): - loss = model.train_batch([x, y], - optimizer=optimizer, - scaler=scaler) + loss = model.train_batch( + [x, y], optimizer=optimizer, scaler=scaler + ) else: with paddle.amp.auto_cast(True, level='O2'): output = model(x) diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py index 16e6d1602f3..f0441f4eab8 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py @@ -15,7 +15,10 @@ import numpy as np import paddle import paddle.distributed.fleet as fleet -from test_collective_multi_nodes import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_multi_nodes import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) from paddle import nn import numpy as np @@ -30,15 +33,14 @@ def weight_init(mp, shape, col=True, seed=1024): else: if col: step = shape[1] // mp.nranks - _w = w[:, mp.rank * step:mp.rank * step + step] + _w = w[:, mp.rank * step : mp.rank * step + step] else: step = shape[0] // mp.nranks - _w = w[mp.rank * step:mp.rank * step + step, :] + _w = w[mp.rank * step : mp.rank * step + step, :] return paddle.fluid.initializer.NumpyArrayInitializer(_w) class Criterion(nn.Layer): - def __init__(self): super(Criterion, self).__init__() self.loss_func = nn.MSELoss(reduction="mean") @@ -49,7 +51,6 @@ class Criterion(nn.Layer): class RecomputeMatmulBlock(nn.Layer): - def __init__(self, mp, seed, m, n, k): super(RecomputeMatmulBlock, self).__init__() self.mp = mp @@ -59,22 +60,22 @@ class RecomputeMatmulBlock(nn.Layer): n, weight_attr=weight_init(mp, (m, n), True, seed), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( n, k, weight_attr=weight_init(mp, (n, k), False, seed + 1), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(m, - n, - weight_attr=weight_init( - None, (m, n), True, seed)) - mp_linear_2 = nn.Linear(n, - k, - weight_attr=weight_init( - None, (n, k), True, seed + 1)) + mp_linear_1 = nn.Linear( + m, n, weight_attr=weight_init(None, (m, n), True, seed) + ) + mp_linear_2 = nn.Linear( + n, k, weight_attr=weight_init(None, (n, k), True, seed + 1) + ) self.layers = nn.Sequential(mp_linear_1, mp_linear_2) def forward(self, x): @@ -88,7 +89,6 @@ RecomputeBlock = RecomputeMatmulBlock class ModelPipeline(fleet.meta_parallel.PipelineLayer): - def __init__(self, hcg): paddle.seed(1024) dp_linear = nn.Linear(32, 64) @@ -104,13 +104,12 @@ class ModelPipeline(fleet.meta_parallel.PipelineLayer): out = nn.Linear(64, 32) self.layers_pp.append(out) - super(ModelPipeline, self).__init__(layers=self.layers_pp, - loss_fn=Criterion(), - topology=self.topology) + super(ModelPipeline, self).__init__( + layers=self.layers_pp, loss_fn=Criterion(), topology=self.topology + ) class Model(nn.Layer): - def __init__(self, hcg): super(Model, self).__init__() paddle.seed(1024) @@ -133,7 +132,6 @@ class Model(nn.Layer): class TestDygrapgHybridRecompute(TestCollectiveAPIRunnerBase): - def __init__(self): pass @@ -142,9 +140,11 @@ class TestDygrapgHybridRecompute(TestCollectiveAPIRunnerBase): from common import init_parallel_env import paddle from paddle.distributed import fleet + hcg = init_parallel_env("DP4-MP2-PP2-SH1-O1", 64) pp_degree = hcg.get_pipe_parallel_world_size() import numpy as np + crit = Criterion() if pp_degree <= 1: model = Model(hcg) @@ -153,10 +153,12 @@ class TestDygrapgHybridRecompute(TestCollectiveAPIRunnerBase): model_base = Model(None) - optimizer = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) optimizer_base = paddle.optimizer.Adam( - learning_rate=0.01, parameters=model_base.parameters()) + learning_rate=0.01, parameters=model_base.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py index 9c432eefa8a..081cd04afb3 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py @@ -23,10 +23,18 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import GroupShardedStage3 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import ( + GroupShardedStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) epoch = 10 paddle.seed(2022) @@ -37,7 +45,6 @@ l2_decay = 1e-4 class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -57,7 +64,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -70,54 +76,60 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_pure_fp16) + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - use_pure_fp16=False, - accumulate_grad=False, - batch_size=100, - opt_group=False, - sync_comm=False, - test_minimize=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + use_pure_fp16=False, + accumulate_grad=False, + batch_size=100, + opt_group=False, + sync_comm=False, + test_minimize=False, + save_model=False, +): group = paddle.distributed.new_group( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = GroupShardedScaler(scaler) if sharding_stage == 2: optimizer = GroupShardedOptimizerStage2( - params=optimizer._parameter_list, optim=optimizer, group=group) - model = GroupShardedStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + params=optimizer._parameter_list, optim=optimizer, group=group + ) + model = GroupShardedStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) elif sharding_stage == 3: - model = GroupShardedStage3(model, - optimizer=optimizer, - group=group, - sync_comm=sync_comm, - segment_size=2**15) + model = GroupShardedStage3( + model, + optimizer=optimizer, + group=group, + sync_comm=sync_comm, + segment_size=2**15, + ) # check optimizer.minimize() error if test_minimize: @@ -125,18 +137,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage3_optimizer.minimize() error ======") + "====== Find sharding_stage3_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -147,8 +162,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if batch_size == 20: @@ -183,8 +199,19 @@ def train_mlp(model, def test_stage2_stage3(): paddle.distributed.init_parallel_env() - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = MLP( - ), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -198,79 +225,88 @@ def test_stage2_stage3(): mlp10.set_state_dict(state_dict) # fp32 - stage2_params = train_mlp(mlp1, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) - stage3_params = train_mlp(mlp2, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False) + stage2_params = train_mlp( + mlp1, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) + stage3_params = train_mlp( + mlp2, sharding_stage=3, use_pure_fp16=False, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-6, - atol=1e-6) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-6, + atol=1e-6, + ) # fp32 accumulate grad - stage3_params = train_mlp(mlp3, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - opt_group=True) - stage3_params_add = train_mlp(mlp4, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - batch_size=20, - opt_group=True) + stage3_params = train_mlp( + mlp3, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + opt_group=True, + ) + stage3_params_add = train_mlp( + mlp4, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + batch_size=20, + opt_group=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_add[i].numpy(), - rtol=1e-6, - atol=1e-4) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_add[i].numpy(), + rtol=1e-6, + atol=1e-4, + ) # fp16 - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=True, - opt_group=False) - stage3_params = train_mlp(mlp6, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=True, opt_group=False + ) + stage3_params = train_mlp( + mlp6, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # fp16 sync_comm - stage3_params = train_mlp(mlp7, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) - stage3_params_re = train_mlp(mlp8, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False, - sync_comm=True) + stage3_params = train_mlp( + mlp7, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) + stage3_params_re = train_mlp( + mlp8, + sharding_stage=3, + use_pure_fp16=True, + opt_group=False, + sync_comm=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_re[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + stage3_params[i].numpy(), stage3_params_re[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() try: model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage3, optimizer_stage3 = train_mlp(mlp9, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage3, optimizer_stage3 = train_mlp( + mlp9, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage3.state_dict(), model_file) paddle.save(optimizer_stage3.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) @@ -284,11 +320,13 @@ def test_stage2_stage3(): shutil.rmtree(output_dir) # check optimizer.minimize() error - train_mlp(mlp10, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - test_minimize=True) + train_mlp( + mlp10, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + test_minimize=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_sharding_stage2.py b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_sharding_stage2.py index affb6d31f2a..eeff08c3557 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_sharding_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_sharding_stage2.py @@ -24,8 +24,12 @@ from paddle.fluid.dygraph.nn import Linear from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) seed = 2022 epoch = 2 @@ -36,7 +40,7 @@ strategy.hybrid_configs = { "dp_degree": 16, "mp_degree": 1, "pp_degree": 1, - "sharding_degree": 1 + "sharding_degree": 1, } np.random.seed(seed) @@ -44,7 +48,6 @@ paddle.seed(seed) class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -64,7 +67,6 @@ class MLP(fluid.Layer): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -76,59 +78,65 @@ def reader_decorator(linear_size=1000): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters() - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[{"params": model.parameters()}] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - batch_size=100, - use_pure_fp16=False, - accumulate_grad=False, - opt_group=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + batch_size=100, + use_pure_fp16=False, + accumulate_grad=False, + opt_group=False, + save_model=False, +): if sharding_stage == "dp": hcg = fleet.get_hybrid_communicate_group() group = hcg.get_check_parallel_group() else: group = paddle.distributed.new_group( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if sharding_stage == 2: - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - group=group) - - model = ShardingStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), optim=optimizer, group=group + ) + + model = ShardingStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) else: optimizer = fleet.distributed_optimizer(optimizer) model = fleet.distributed_model(model) - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: @@ -180,53 +188,51 @@ def test_dp_stage2(): mlp6.set_state_dict(state_dict) # DP VS stage2 - dp_params = train_mlp(mlp1, - sharding_stage="dp", - use_pure_fp16=False, - opt_group=False) - stage2_params = train_mlp(mlp2, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) + dp_params = train_mlp( + mlp1, sharding_stage="dp", use_pure_fp16=False, opt_group=False + ) + stage2_params = train_mlp( + mlp2, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6, - atol=5e-4) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6, atol=5e-4 + ) # stage2 accumulate grad stage2_params = train_mlp(mlp3, sharding_stage=2, accumulate_grad=True) - stage2_accumulate_grad = train_mlp(mlp4, - sharding_stage=2, - batch_size=20, - accumulate_grad=True) + stage2_accumulate_grad = train_mlp( + mlp4, sharding_stage=2, batch_size=20, accumulate_grad=True + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage2_accumulate_grad[i].numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage2_accumulate_grad[i].numpy(), + rtol=1e-5, + atol=1e-5, + ) # stage2 param list VS param group - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=False, - opt_group=True) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=False, opt_group=True + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6, - atol=5e-4) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6, atol=5e-4 + ) # save/load model output_dir = tempfile.mkdtemp() try: model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage2, optimizer_stage2 = train_mlp(mlp6, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage2, optimizer_stage2 = train_mlp( + mlp6, + sharding_stage=2, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage2.state_dict(), model_file) paddle.save(optimizer_stage2.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py index c7ad96c061e..868585f81a0 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py @@ -20,10 +20,10 @@ import tempfile class TestCollectiveAPIRunnerBase(object): - def check_pass(self, *args, **kwargs): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def run_trainer(self, *args, **kwargs): self.check_pass(*args, **kwargs) @@ -37,7 +37,6 @@ def runtime_main(test_class, col_type=None): class TestDistBase(unittest.TestCase): - def setUp(self): self._trainers = 4 self._init_env() @@ -46,15 +45,17 @@ class TestDistBase(unittest.TestCase): self._python_interp = sys.executable self.temp_dir = tempfile.TemporaryDirectory() - def check_with_place(self, - model_file, - backend="nccl", - static_mode=False, - check_error_log=False, - need_envs={}, - eager_mode=True, - args=[], - kwargs={}): + def check_with_place( + self, + model_file, + backend="nccl", + static_mode=False, + check_error_log=False, + need_envs={}, + eager_mode=True, + args=[], + kwargs={}, + ): required_envs = { "FLAGS_fraction_of_gpu_memory_to_use": "0.15", "FLAGS_eager_delete_tensor_gb": "0.0", @@ -68,7 +69,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_WITH_GLOO": "0", "BACKEND": backend, "PADDLE_DISTRI_BACKEND": backend, - "PADDLE_USE_GPU": "1" + "PADDLE_USE_GPU": "1", } required_envs.update(need_envs) if check_error_log: @@ -87,15 +88,18 @@ class TestDistBase(unittest.TestCase): filted_envs = dict() for k in envs.keys(): if "PADDLE_" == k[:7] and k not in [ - "PADDLE_NNODES", "PADDLE_MASTER" + "PADDLE_NNODES", + "PADDLE_MASTER", ]: continue filted_envs[k] = envs[k] - launcher = subprocess.Popen(run_cluster_process.strip().split(), - stdout=sys.stderr, - stderr=sys.stdout, - env=filted_envs) + launcher = subprocess.Popen( + run_cluster_process.strip().split(), + stdout=sys.stderr, + stderr=sys.stdout, + env=filted_envs, + ) launcher.communicate(timeout=240) if launcher.poll() is None: diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py index 05124e0ddd5..80973d73555 100755 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py @@ -22,15 +22,14 @@ import os class TestDYgraphDPMode(TestDistBase): - def setUp(self): self._trainers = 16 self._init_env() def test_col_parallel_linear(self): - self.check_with_place("dygraph_hybrid_dp.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "dygraph_hybrid_dp.py", backend="nccl", need_envs=os.environ + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py index aa2a01454ae..1435dd8ef0e 100755 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py @@ -22,25 +22,24 @@ import os class TestDYgraphHybrid(TestDistBase): - def setUp(self): self._trainers = 16 self._init_env() def test_hybrid_dpppmp(self): - self.check_with_place("dygraph_hybrid_dpppmp.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "dygraph_hybrid_dpppmp.py", backend="nccl", need_envs=os.environ + ) def test_hybrid_recompute(self): - self.check_with_place("dygraph_hybrid_recompute.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "dygraph_hybrid_recompute.py", backend="nccl", need_envs=os.environ + ) def test_hybrid_fp16(self): - self.check_with_place("dygraph_hybrid_fp16.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "dygraph_hybrid_fp16.py", backend="nccl", need_envs=os.environ + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py index f0da628a4c8..d1689330d4a 100755 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py @@ -22,20 +22,23 @@ import os class TestDYgrapShardingDP(TestDistBase): - def setUp(self): self._trainers = 16 self._init_env() def test_hybrid_sharding_stage2(self): - self.check_with_place("mn_dygraph_sharding_stage2.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "mn_dygraph_sharding_stage2.py", + backend="nccl", + need_envs=os.environ, + ) def test_hybrid_sharding_stage3(self): - self.check_with_place("mn_dygraph_group_sharded_stage3.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "mn_dygraph_group_sharded_stage3.py", + backend="nccl", + need_envs=os.environ, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/orthogonal_strategy.py b/python/paddle/fluid/tests/unittests/collective/orthogonal_strategy.py index 5dbc624b79b..7d39b677233 100644 --- a/python/paddle/fluid/tests/unittests/collective/orthogonal_strategy.py +++ b/python/paddle/fluid/tests/unittests/collective/orthogonal_strategy.py @@ -14,27 +14,37 @@ import unittest import paddle.distributed as dist -from paddle.distributed.fleet.base.strategy_group import DPGroup, ShardingGroup, MPGroup, PPGroup +from paddle.distributed.fleet.base.strategy_group import ( + DPGroup, + ShardingGroup, + MPGroup, + PPGroup, +) from paddle.distributed.fleet.base.orthogonal_strategy import OrthogonalStrategy class TestOrthogonalStrategyAPI(unittest.TestCase): - def setUp(self): self._num_of_ranks = 2 dist.init_parallel_env() self._global_rank = dist.get_rank() self._strategy = OrthogonalStrategy( - [("dp", 2, DPGroup), ("mp", 1, MPGroup), - ("sharding", 1, ShardingGroup), ("pp", 1, PPGroup)], - fused_strategy_dict={"checkness": ["mp", "sharding", "pp"]}) + [ + ("dp", 2, DPGroup), + ("mp", 1, MPGroup), + ("sharding", 1, ShardingGroup), + ("pp", 1, PPGroup), + ], + fused_strategy_dict={"checkness": ["mp", "sharding", "pp"]}, + ) def test_orthogonal_strategy(self): dp_group = self._strategy.strategy_group("dp") self.assertEqual(dp_group.world_size, self._num_of_ranks) self.assertEqual(dp_group.group.nranks, self._num_of_ranks) - self.assertEqual(self._strategy.rank_in_strategy("dp"), - self._global_rank) + self.assertEqual( + self._strategy.rank_in_strategy("dp"), self._global_rank + ) fused_group = self._strategy.fused_strategy_group("checkness") self.assertEqual(fused_group.world_size, 1) diff --git a/python/paddle/fluid/tests/unittests/collective/parallel_embedding_api.py b/python/paddle/fluid/tests/unittests/collective/parallel_embedding_api.py index 8b6ab746ce8..0c118441b65 100644 --- a/python/paddle/fluid/tests/unittests/collective/parallel_embedding_api.py +++ b/python/paddle/fluid/tests/unittests/collective/parallel_embedding_api.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestParallelEmbeddingAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -36,24 +35,30 @@ class TestParallelEmbeddingAPI(TestCollectiveAPIRunnerBase): paddle.seed(2020) data_in = paddle.randint(0, size[0], shape=(10, 4)) - data = paddle.static.data(name='tindata', - shape=[10, 1000], - dtype="float32") + data = paddle.static.data( + name='tindata', shape=[10, 1000], dtype="float32" + ) per_part_size = size[0] // 2 if rank == 0: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[0:per_part_size, :]), ) + np_array[0:per_part_size, :] + ), + ) else: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[per_part_size:size[0], :]), ) + np_array[per_part_size : size[0], :] + ), + ) - emb_out = paddle.distributed.split(data_in, - size, - operation="embedding", - num_partitions=2, - weight_attr=param_attr) + emb_out = paddle.distributed.split( + data_in, + size, + operation="embedding", + num_partitions=2, + weight_attr=param_attr, + ) return [data_in, emb_out] diff --git a/python/paddle/fluid/tests/unittests/collective/process_group_gloo.py b/python/paddle/fluid/tests/unittests/collective/process_group_gloo.py index 4e678ef955e..3ace5179913 100644 --- a/python/paddle/fluid/tests/unittests/collective/process_group_gloo.py +++ b/python/paddle/fluid/tests/unittests/collective/process_group_gloo.py @@ -24,7 +24,6 @@ from paddle.fluid.dygraph.parallel import ParallelEnv class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -40,8 +39,9 @@ class TestProcessGroupFp32(unittest.TestCase): nranks = ParallelEnv().nranks rank = ParallelEnv().local_rank is_master = True if rank == 0 else False - store = paddle.fluid.core.TCPStore("127.0.0.1", 6272, is_master, - nranks, 30) + store = paddle.fluid.core.TCPStore( + "127.0.0.1", 6272, is_master, nranks, 30 + ) place = paddle.fluid.core.CPUPlace() pg = paddle.fluid.core.ProcessGroupGloo(store, rank, nranks, place) @@ -135,8 +135,9 @@ class TestProcessGroupFp32(unittest.TestCase): task = pg.all_gather(tensor_y, tensor_out) task.wait() out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) assert np.array_equal(tensor_x, out_1) assert np.array_equal(tensor_y, out_2) print("test allgather api ok\n") @@ -175,8 +176,9 @@ class TestProcessGroupFp32(unittest.TestCase): task = pg.scatter(tensor_x, tensor_y, 0) task.wait() out1 = paddle.slice(tensor_x, [0], [0], [self.shape[0]]) - out2 = paddle.slice(tensor_x, [0], [self.shape[0]], - [self.shape[0] * 2]) + out2 = paddle.slice( + tensor_x, [0], [self.shape[0]], [self.shape[0] * 2] + ) if pg.rank() == 0: assert np.array_equal(tensor_y, out1) else: diff --git a/python/paddle/fluid/tests/unittests/collective/process_group_mpi.py b/python/paddle/fluid/tests/unittests/collective/process_group_mpi.py index f3ca19ebe7e..998e323bfb1 100644 --- a/python/paddle/fluid/tests/unittests/collective/process_group_mpi.py +++ b/python/paddle/fluid/tests/unittests/collective/process_group_mpi.py @@ -42,12 +42,14 @@ def init_process_group(strategy=None): place = core.CPUPlace() _set_expected_place(place) - group = Group(rank, - world_size, - id=0, - ranks=list(range(world_size)), - pg=pg, - name=_default_group_name) + group = Group( + rank, + world_size, + id=0, + ranks=list(range(world_size)), + pg=pg, + name=_default_group_name, + ) _set_group_map_by_name(_default_group_name, group) _set_group_map(gid, group) _set_group_map_backend(group, "mpi") @@ -84,15 +86,15 @@ def test_allreduce_max(pg, shape, dtype): max_result = paddle.maximum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.MAX, - use_calc_stream=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.MAX, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, max_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.MAX, - use_calc_stream=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.MAX, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_y, max_result) print("test allreduce max api ok") @@ -109,15 +111,15 @@ def test_allreduce_min(pg, shape, dtype): min_result = paddle.minimum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.MIN, - use_calc_stream=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.MIN, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, min_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.MIN, - use_calc_stream=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.MIN, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_y, min_result) print("test allreduce min api ok") @@ -134,15 +136,15 @@ def test_allreduce_prod(pg, shape, dtype): prod_result = np.multiply(x, y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.PROD, - use_calc_stream=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.PROD, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, prod_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.PROD, - use_calc_stream=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.PROD, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_y, prod_result) print("test allreduce prod api ok") @@ -196,7 +198,7 @@ def test_allgather(pg, shape, dtype): else: tensor_out_list = [ paddle.empty_like(tensor_x), - paddle.empty_like(tensor_x) + paddle.empty_like(tensor_x), ] task = dist.all_gather(tensor_out_list, tensor_y, use_calc_stream=False) tensor_out = paddle.concat(tensor_out_list) @@ -309,17 +311,15 @@ def test_reduce_max(pg, shape, dtype): max_result = paddle.maximum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.MAX, - use_calc_stream=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.MAX, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, max_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.MAX, - use_calc_stream=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.MAX, use_calc_stream=False + ) task.wait() print("test reduce max api ok") @@ -335,17 +335,15 @@ def test_reduce_min(pg, shape, dtype): min_result = paddle.minimum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.MIN, - use_calc_stream=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.MIN, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, min_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.MIN, - use_calc_stream=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.MIN, use_calc_stream=False + ) task.wait() print("test reduce min api ok") @@ -361,17 +359,15 @@ def test_reduce_prod(pg, shape, dtype): prod_result = np.multiply(x, y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.PROD, - use_calc_stream=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.PROD, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, prod_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.PROD, - use_calc_stream=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.PROD, use_calc_stream=False + ) task.wait() print("test reduce prod api ok") @@ -436,7 +432,6 @@ def test_send_recv(pg, sub_group, shape, dtype): class TestProcessGroup(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) diff --git a/python/paddle/fluid/tests/unittests/collective/process_group_nccl.py b/python/paddle/fluid/tests/unittests/collective/process_group_nccl.py index aa3a9575f42..ff949d8f14c 100644 --- a/python/paddle/fluid/tests/unittests/collective/process_group_nccl.py +++ b/python/paddle/fluid/tests/unittests/collective/process_group_nccl.py @@ -32,7 +32,6 @@ def init_process_group(strategy=None): class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -45,8 +44,9 @@ class TestProcessGroupFp32(unittest.TestCase): def test_create_process_group_nccl(self): with _test_eager_guard(): - paddle.set_device('gpu:%d' % - paddle.distributed.ParallelEnv().dev_id) + paddle.set_device( + 'gpu:%d' % paddle.distributed.ParallelEnv().dev_id + ) pg = init_process_group() print("rank:", pg.rank(), "size:", pg.size(), "name:", pg.name()) @@ -81,15 +81,15 @@ class TestProcessGroupFp32(unittest.TestCase): max_result = paddle.maximum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.MAX, - sync_op=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.MAX, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, max_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.MAX, - sync_op=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.MAX, sync_op=False + ) task.wait() assert np.array_equal(tensor_y, max_result) @@ -106,15 +106,15 @@ class TestProcessGroupFp32(unittest.TestCase): min_result = paddle.minimum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.MIN, - sync_op=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.MIN, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, min_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.MIN, - sync_op=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.MIN, sync_op=False + ) task.wait() assert np.array_equal(tensor_y, min_result) @@ -131,15 +131,15 @@ class TestProcessGroupFp32(unittest.TestCase): prod_result = np.multiply(x, y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.PROD, - sync_op=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.PROD, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, prod_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.PROD, - sync_op=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.PROD, sync_op=False + ) task.wait() assert np.array_equal(tensor_y, prod_result) @@ -196,14 +196,15 @@ class TestProcessGroupFp32(unittest.TestCase): else: tensor_out_list = [ paddle.empty_like(tensor_x), - paddle.empty_like(tensor_x) + paddle.empty_like(tensor_x), ] task = dist.all_gather(tensor_out_list, tensor_y, sync_op=False) paddle.device.cuda.synchronize() tensor_out = paddle.concat(tensor_out_list) out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) assert np.array_equal(tensor_x, out_1) assert np.array_equal(tensor_y, out_2) print("test allgather api ok\n") @@ -219,8 +220,9 @@ class TestProcessGroupFp32(unittest.TestCase): paddle.device.cuda.synchronize() tensor_out = paddle.concat(tensor_out_list) out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) assert np.array_equal(tensor_x, out_1) assert np.array_equal(tensor_y, out_2) print("test allgather api2 ok\n") @@ -235,10 +237,12 @@ class TestProcessGroupFp32(unittest.TestCase): tensor_y = paddle.to_tensor(y) tensor_out1 = paddle.to_tensor(out1) tensor_out2 = paddle.to_tensor(out2) - raw_tensor_x_2 = paddle.slice(tensor_x, [0], [self.shape[0] // 2], - [self.shape[0]]) - raw_tensor_y_1 = paddle.slice(tensor_y, [0], [0], - [self.shape[0] // 2]) + raw_tensor_x_2 = paddle.slice( + tensor_x, [0], [self.shape[0] // 2], [self.shape[0]] + ) + raw_tensor_y_1 = paddle.slice( + tensor_y, [0], [0], [self.shape[0] // 2] + ) if pg.rank() == 0: task = pg.alltoall(tensor_x, tensor_out1) task.wait() @@ -250,8 +254,9 @@ class TestProcessGroupFp32(unittest.TestCase): task = dist.alltoall([in_1, in_2], out_tensor_list) paddle.device.cuda.synchronize() tensor_out2 = paddle.concat(out_tensor_list) - out1_2 = paddle.slice(tensor_out1, [0], [self.shape[0] // 2], - [self.shape[0]]) + out1_2 = paddle.slice( + tensor_out1, [0], [self.shape[0] // 2], [self.shape[0]] + ) out2_1 = paddle.slice(tensor_out2, [0], [0], [self.shape[0] // 2]) if pg.rank() == 0: assert np.array_equal(out1_2.numpy(), raw_tensor_y_1.numpy()) @@ -267,10 +272,12 @@ class TestProcessGroupFp32(unittest.TestCase): tensor_y = paddle.to_tensor(y) tensor_out1 = paddle.to_tensor(out1) tensor_out2 = paddle.to_tensor(out2) - raw_tensor_x_2 = paddle.slice(tensor_x, [0], [self.shape[0] // 2], - [self.shape[0]]) - raw_tensor_y_1 = paddle.slice(tensor_y, [0], [0], - [self.shape[0] // 2]) + raw_tensor_x_2 = paddle.slice( + tensor_x, [0], [self.shape[0] // 2], [self.shape[0]] + ) + raw_tensor_y_1 = paddle.slice( + tensor_y, [0], [0], [self.shape[0] // 2] + ) if pg.rank() == 0: task = pg.alltoall(tensor_x, tensor_out1) task.wait() @@ -282,8 +289,9 @@ class TestProcessGroupFp32(unittest.TestCase): task = dist.alltoall([in_1, in_2], out_tensor_list) paddle.device.cuda.synchronize() tensor_out2 = paddle.concat(out_tensor_list) - out1_2 = paddle.slice(tensor_out1, [0], [self.shape[0] // 2], - [self.shape[0]]) + out1_2 = paddle.slice( + tensor_out1, [0], [self.shape[0] // 2], [self.shape[0]] + ) out2_1 = paddle.slice(tensor_out2, [0], [0], [self.shape[0] // 2]) if pg.rank() == 0: assert np.array_equal(out1_2.numpy(), raw_tensor_y_1.numpy()) @@ -321,17 +329,15 @@ class TestProcessGroupFp32(unittest.TestCase): max_result = paddle.maximum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.MAX, - sync_op=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.MAX, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, max_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.MAX, - sync_op=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.MAX, sync_op=False + ) task.wait() print("test reduce max api ok") @@ -347,17 +353,15 @@ class TestProcessGroupFp32(unittest.TestCase): min_result = paddle.minimum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.MIN, - sync_op=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.MIN, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, min_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.MIN, - sync_op=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.MIN, sync_op=False + ) task.wait() print("test reduce min api ok") @@ -373,17 +377,15 @@ class TestProcessGroupFp32(unittest.TestCase): prod_result = np.multiply(x, y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.PROD, - sync_op=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.PROD, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, prod_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.PROD, - sync_op=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.PROD, sync_op=False + ) task.wait() print("test reduce prod api ok") @@ -398,7 +400,7 @@ class TestProcessGroupFp32(unittest.TestCase): if pg.rank() == 0: in_1, in_2 = paddle.split(tensor_x, 2) task = dist.scatter(tensor_y, [in_1, in_2], 0, sync_op=True) - #task.wait() + # task.wait() paddle.device.cuda.synchronize() # rank 1 else: @@ -406,8 +408,9 @@ class TestProcessGroupFp32(unittest.TestCase): task.wait() paddle.device.cuda.synchronize() out1 = paddle.slice(tensor_x, [0], [0], [self.shape[0]]) - out2 = paddle.slice(tensor_x, [0], [self.shape[0]], - [self.shape[0] * 2]) + out2 = paddle.slice( + tensor_x, [0], [self.shape[0]], [self.shape[0] * 2] + ) if pg.rank() == 0: assert np.array_equal(tensor_y, out1) else: @@ -450,7 +453,6 @@ class TestProcessGroupFp32(unittest.TestCase): class TestProcessGroupFp16(TestProcessGroupFp32): - def setUp(self): paddle.seed(2022) random.seed(2022) diff --git a/python/paddle/fluid/tests/unittests/collective/row_parallel_linear_api.py b/python/paddle/fluid/tests/unittests/collective/row_parallel_linear_api.py index 854ef9cb69e..e20f57f39ae 100644 --- a/python/paddle/fluid/tests/unittests/collective/row_parallel_linear_api.py +++ b/python/paddle/fluid/tests/unittests/collective/row_parallel_linear_api.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestRowParallelLinearAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -32,19 +31,23 @@ class TestRowParallelLinearAPI(TestCollectiveAPIRunnerBase): np.random.seed(2020) np_array = np.random.rand(1000, 16) - data = paddle.static.data(name='tindata', - shape=[10, 1000], - dtype="float32") + data = paddle.static.data( + name='tindata', shape=[10, 1000], dtype="float32" + ) paddle.distributed.broadcast(data, src=0) data = paddle.split(data, 2, axis=1)[rank] if rank == 0: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[0:500, :]), ) + np_array[0:500, :] + ), + ) else: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[500:1000, :]), ) + np_array[500:1000, :] + ), + ) linear_out = paddle.distributed.split( data, diff --git a/python/paddle/fluid/tests/unittests/collective/strategy_group.py b/python/paddle/fluid/tests/unittests/collective/strategy_group.py index 247a232aecc..75d2995d62a 100644 --- a/python/paddle/fluid/tests/unittests/collective/strategy_group.py +++ b/python/paddle/fluid/tests/unittests/collective/strategy_group.py @@ -16,7 +16,13 @@ import unittest import numpy as np import paddle import paddle.distributed as dist -from paddle.distributed.fleet.base.strategy_group import StrategyGroupBase, DPGroup, MPGroup, PPGroup, ShardingGroup +from paddle.distributed.fleet.base.strategy_group import ( + StrategyGroupBase, + DPGroup, + MPGroup, + PPGroup, + ShardingGroup, +) def _check_using_all_reduce(group): @@ -39,7 +45,6 @@ def _check_using_recv(group, src): class TestStrategyGroupAPI(unittest.TestCase): - def setUp(self): self._num_of_ranks = 2 self._list_of_rank = [[0, 1]] @@ -74,7 +79,12 @@ class TestStrategyGroupAPI(unittest.TestCase): def test_pipeline_parallel_group(self): pp_group = PPGroup(self._list_of_rank) - send_next_group, send_prev_group, recv_next_group, recv_prev_group = pp_group.p2p_groups + ( + send_next_group, + send_prev_group, + recv_next_group, + recv_prev_group, + ) = pp_group.p2p_groups if self._global_rank == 0: self.assertEqual(pp_group.rank_of_next_stage, 1) self.assertEqual(pp_group.rank_of_prev_stage, 1) diff --git a/python/paddle/fluid/tests/unittests/collective/test_allreduce.py b/python/paddle/fluid/tests/unittests/collective/test_allreduce.py index d137f0f3ec4..19763579b1a 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_allreduce.py +++ b/python/paddle/fluid/tests/unittests/collective/test_allreduce.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestAllReduceOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_broadcast.py b/python/paddle/fluid/tests/unittests/collective/test_broadcast.py index ff171d084d3..8aa7be89d15 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_broadcast.py +++ b/python/paddle/fluid/tests/unittests/collective/test_broadcast.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestCBroadcastOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_c_concat.py b/python/paddle/fluid/tests/unittests/collective/test_c_concat.py index bc241c1e556..c28e313744a 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_c_concat.py +++ b/python/paddle/fluid/tests/unittests/collective/test_c_concat.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestConcatOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_c_identity.py b/python/paddle/fluid/tests/unittests/collective/test_c_identity.py index 936e2961c57..585c0aff0e4 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_c_identity.py +++ b/python/paddle/fluid/tests/unittests/collective/test_c_identity.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestIdentityOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_c_split.py b/python/paddle/fluid/tests/unittests/collective/test_c_split.py index 43dc766ad24..b75398ca6bd 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_c_split.py +++ b/python/paddle/fluid/tests/unittests/collective/test_c_split.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestSplitOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_api.py index 9040564ce12..f0e9b3bba2f 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_api.py @@ -21,59 +21,96 @@ paddle.enable_static() class TestCollectiveAllgatherAPI(TestDistBase): - def _setup_config(self): pass def test_allgather_nccl(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "complex64", "complex128" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "complex64", + "complex128", ] for dtype in dtypes_to_test: - self.check_with_place("collective_allgather_api.py", - "allgather", - "nccl", - dtype=dtype) + self.check_with_place( + "collective_allgather_api.py", "allgather", "nccl", dtype=dtype + ) def test_allgather_gloo(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "complex64", "complex128" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "complex64", + "complex128", ] for dtype in dtypes_to_test: - self.check_with_place("collective_allgather_api.py", - "allgather", - "gloo", - "3", - dtype=dtype) + self.check_with_place( + "collective_allgather_api.py", + "allgather", + "gloo", + "3", + dtype=dtype, + ) def test_allgatther_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "complex64", "complex128" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "complex64", + "complex128", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_allgather_api_dygraph.py", - "allgather", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_allgather_api_dygraph.py", + "allgather", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_allgather_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16", "complex64", "complex128" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", + "complex64", + "complex128", ] for dtype in dtypes_to_test: - self.check_with_place("collective_allgather_api_dygraph.py", - "allgather", - "gloo", - "3", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_allgather_api_dygraph.py", + "allgather", + "gloo", + "3", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_object_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_object_api.py index 90539e89c0b..4a174010c51 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_object_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_object_api.py @@ -17,35 +17,42 @@ import test_collective_api_base as test_base class TestCollectiveAllgatherObjectAPI(test_base.TestDistBase): - def _setup_config(self): pass def test_allgather_nccl(self): - self.check_with_place("collective_allgather_object_api_dygraph.py", - "allgather_object", - "nccl", - static_mode="0", - dtype="pylist") - self.check_with_place("collective_allgather_object_api_dygraph.py", - "allgather_object", - "nccl", - static_mode="0", - dtype="pydict") + self.check_with_place( + "collective_allgather_object_api_dygraph.py", + "allgather_object", + "nccl", + static_mode="0", + dtype="pylist", + ) + self.check_with_place( + "collective_allgather_object_api_dygraph.py", + "allgather_object", + "nccl", + static_mode="0", + dtype="pydict", + ) def test_allgather_gloo_dygraph(self): - self.check_with_place("collective_allgather_object_api_dygraph.py", - "allgather_object", - "gloo", - "3", - static_mode="0", - dtype="pylist") - self.check_with_place("collective_allgather_object_api_dygraph.py", - "allgather_object", - "gloo", - "3", - static_mode="0", - dtype="pydict") + self.check_with_place( + "collective_allgather_object_api_dygraph.py", + "allgather_object", + "gloo", + "3", + static_mode="0", + dtype="pylist", + ) + self.check_with_place( + "collective_allgather_object_api_dygraph.py", + "allgather_object", + "gloo", + "3", + static_mode="0", + dtype="pydict", + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_allreduce_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_allreduce_api.py index a5080f78bce..6b2b9c0ade8 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_allreduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_allreduce_api.py @@ -21,50 +21,69 @@ paddle.enable_static() class TestCollectiveAllreduceAPI(TestDistBase): - def _setup_config(self): pass def test_allreduce_nccl(self): if paddle.fluid.core.is_compiled_with_cuda(): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "nccl") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "nccl" + ) def test_allreduce_bkcl(self): if paddle.fluid.core.is_compiled_with_xpu(): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "bkcl") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "bkcl" + ) def test_allreduce_gloo(self): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "gloo", "2") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "gloo", "2" + ) def test_allreduce_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_allreduce_api_dygraph.py", - "allreduce", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_allreduce_api_dygraph.py", + "allreduce", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_allreduce_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", ] for dtype in dtypes_to_test: - self.check_with_place("collective_allreduce_api_dygraph.py", - "allreduce", - "gloo", - "2", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_allreduce_api_dygraph.py", + "allreduce", + "gloo", + "2", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_api.py index 1edb06ae512..acbbdec2ca8 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_api.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestCollectiveAllToAllAPI(TestDistBase): - def _setup_config(self): pass @@ -30,17 +29,25 @@ class TestCollectiveAllToAllAPI(TestDistBase): def test_alltoall_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_alltoall_api_dygraph.py", - "alltoall", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_alltoall_api_dygraph.py", + "alltoall", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py index 9ef0a7639e1..6915f07e8fa 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py @@ -19,7 +19,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestCollectiveAllToAllSingle(TestMultipleGpus): - def test_collective_alltoall_single(self): self.run_mnist_2gpu('collective_alltoall_single.py', eager_mode=True) diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single_api.py index e3ef3f302f3..26917d8aba5 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single_api.py @@ -17,23 +17,30 @@ import test_collective_api_base as test_base class TestCollectiveAllToAllSingleAPI(test_base.TestDistBase): - def _setup_config(self): pass def test_alltooall_single_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_alltoall_single_api_dygraph.py", - "alltoall", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_alltoall_single_api_dygraph.py", + "alltoall", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_barrier_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_barrier_api.py index 8a883ce989c..daa23ff7a57 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_barrier_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_barrier_api.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestCollectiveBarrierAPI(TestDistBase): - def _setup_config(self): pass @@ -29,8 +28,9 @@ class TestCollectiveBarrierAPI(TestDistBase): self.check_with_place("collective_barrier_api.py", "barrier", "nccl") def test_barrier_gloo(self): - self.check_with_place("collective_barrier_api.py", "barrier", "gloo", - "5") + self.check_with_place( + "collective_barrier_api.py", "barrier", "gloo", "5" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py b/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py index e9fad9aa1c1..892e813660e 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py @@ -19,7 +19,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestCollectiveBatchIsendIrecv(TestMultipleGpus): - def test_collective_batch_isend_irecv(self): self.run_mnist_2gpu('collective_batch_isend_irecv.py', eager_mode=True) diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_broadcast_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_broadcast_api.py index 8f4e747b622..045d1eb3176 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_broadcast_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_broadcast_api.py @@ -21,44 +21,62 @@ paddle.enable_static() class TestCollectiveBroadcastAPI(TestDistBase): - def _setup_config(self): pass def test_broadcast_nccl(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "nccl") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "nccl" + ) def test_broadcast_gloo(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "gloo", "0") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "gloo", "0" + ) def test_broadcast_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_broadcast_api_dygraph.py", - "broadcast", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_broadcast_api_dygraph.py", + "broadcast", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_broadcast_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", ] for dtype in dtypes_to_test: - self.check_with_place("collective_broadcast_api_dygraph.py", - "broadcast", - "gloo", - "0", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_broadcast_api_dygraph.py", + "broadcast", + "gloo", + "0", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_cpu_barrier_with_gloo.py b/python/paddle/fluid/tests/unittests/collective/test_collective_cpu_barrier_with_gloo.py index 157e7106308..6049ccbd5d1 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_cpu_barrier_with_gloo.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_cpu_barrier_with_gloo.py @@ -26,12 +26,11 @@ paddle.enable_static() class CollectiveCPUBarrierWithGlooTest(unittest.TestCase): - def find_free_port(self): - def _free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -43,8 +42,9 @@ class CollectiveCPUBarrierWithGlooTest(unittest.TestCase): def barrier_func(self, id, rank_num, server_endpoint, out_dict, sleep_time): try: - paddle.distributed.gloo_init_parallel_env(id, rank_num, - server_endpoint) + paddle.distributed.gloo_init_parallel_env( + id, rank_num, server_endpoint + ) # 1st barrier # Run barrier to synchronize processes after starting paddle.distributed.gloo_barrier() @@ -52,7 +52,7 @@ class CollectiveCPUBarrierWithGlooTest(unittest.TestCase): # Let rank 0 sleep for one second and check that all processes # saw that artificial delay through the barrier start = time.time() - if (id == 0): + if id == 0: time.sleep(sleep_time) paddle.distributed.gloo_barrier() end = time.time() @@ -66,8 +66,9 @@ class CollectiveCPUBarrierWithGlooTest(unittest.TestCase): try: main_prog = fluid.Program() startup_prog = fluid.Program() - paddle.distributed.gloo_init_parallel_env(id, rank_num, - server_endpoint) + paddle.distributed.gloo_init_parallel_env( + id, rank_num, server_endpoint + ) place = fluid.CPUPlace() with fluid.program_guard(main_prog, startup_prog): paddle.distributed.barrier() @@ -77,7 +78,7 @@ class CollectiveCPUBarrierWithGlooTest(unittest.TestCase): # Let rank 0 sleep for one second and check that all processes # saw that artificial delay through the barrier start = time.time() - if (id == 0): + if id == 0: time.sleep(sleep_time) exe.run(main_prog) end = time.time() @@ -97,9 +98,10 @@ class CollectiveCPUBarrierWithGlooTest(unittest.TestCase): procs_out_dict = manager.dict() jobs = [] for id in range(num_of_ranks): - p = multiprocessing.Process(target=self.barrier_func, - args=(id, num_of_ranks, ep_str, - procs_out_dict, sleep_time)) + p = multiprocessing.Process( + target=self.barrier_func, + args=(id, num_of_ranks, ep_str, procs_out_dict, sleep_time), + ) jobs.append(p) p.start() for proc in jobs: @@ -117,9 +119,10 @@ class CollectiveCPUBarrierWithGlooTest(unittest.TestCase): procs_out_dict = manager.dict() jobs = [] for id in range(num_of_ranks): - p = multiprocessing.Process(target=self.barrier_op, - args=(id, num_of_ranks, ep_str, - procs_out_dict, sleep_time)) + p = multiprocessing.Process( + target=self.barrier_op, + args=(id, num_of_ranks, ep_str, procs_out_dict, sleep_time), + ) jobs.append(p) p.start() for proc in jobs: diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_global_gather.py b/python/paddle/fluid/tests/unittests/collective/test_collective_global_gather.py index 85406dd00bb..cc9dbfc10e3 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_global_gather.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_global_gather.py @@ -19,28 +19,32 @@ from test_collective_api_base import TestDistBase class TestCollectiveGlobalGatherAPI(TestDistBase): - def _setup_config(self): pass def test_global_gather_nccl(self): paddle.enable_static() - self.check_with_place("collective_global_gather.py", "global_gather", - "nccl") + self.check_with_place( + "collective_global_gather.py", "global_gather", "nccl" + ) def test_global_gather_nccl_dygraph(self): - self.check_with_place("collective_global_gather_dygraph.py", - "global_gather", - "nccl", - static_mode="0", - eager_mode=False) + self.check_with_place( + "collective_global_gather_dygraph.py", + "global_gather", + "nccl", + static_mode="0", + eager_mode=False, + ) def test_global_gather_nccl_dygraph_eager(self): - self.check_with_place("collective_global_gather_dygraph.py", - "global_gather", - "nccl", - static_mode="0", - eager_mode=True) + self.check_with_place( + "collective_global_gather_dygraph.py", + "global_gather", + "nccl", + static_mode="0", + eager_mode=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_global_scatter.py b/python/paddle/fluid/tests/unittests/collective/test_collective_global_scatter.py index c2668c62047..2b83f136c1e 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_global_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_global_scatter.py @@ -19,28 +19,32 @@ from test_collective_api_base import TestDistBase class TestCollectiveSelectScatterAPI(TestDistBase): - def _setup_config(self): pass def test_global_scatter_nccl(self): paddle.enable_static() - self.check_with_place("collective_global_scatter.py", "global_scatter", - "nccl") + self.check_with_place( + "collective_global_scatter.py", "global_scatter", "nccl" + ) def test_global_scatter_nccl_dygraph(self): - self.check_with_place("collective_global_scatter_dygraph.py", - "global_scatter", - "nccl", - static_mode="0", - eager_mode=False) + self.check_with_place( + "collective_global_scatter_dygraph.py", + "global_scatter", + "nccl", + static_mode="0", + eager_mode=False, + ) def test_global_scatter_nccl_dygraph_eager(self): - self.check_with_place("collective_global_scatter_dygraph.py", - "global_scatter", - "nccl", - static_mode="0", - eager_mode=True) + self.check_with_place( + "collective_global_scatter_dygraph.py", + "global_scatter", + "nccl", + static_mode="0", + eager_mode=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_isend_irecv_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_isend_irecv_api.py index 2b0727cae0c..d313fc5a35e 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_isend_irecv_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_isend_irecv_api.py @@ -17,23 +17,30 @@ import test_collective_api_base as test_base class TestCollectiveIsendIrecvAPI(test_base.TestDistBase): - def _setup_config(self): pass def test_isend_irecv_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_isend_irecv_api_dygraph.py", - "sendrecv", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_isend_irecv_api_dygraph.py", + "sendrecv", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_optimizer.py b/python/paddle/fluid/tests/unittests/collective/test_collective_optimizer.py index 3e4920f8f4f..98c4b10466d 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_optimizer.py @@ -26,11 +26,13 @@ import unittest import paddle.fluid as fluid -from paddle.fluid.incubate.fleet.collective import CollectiveOptimizer, DistributedStrategy +from paddle.fluid.incubate.fleet.collective import ( + CollectiveOptimizer, + DistributedStrategy, +) class CollectiveOptimizerTest(unittest.TestCase): - def test_ds_as_None(self): optimizer = fluid.optimizer.AdamOptimizer() dist_optimizer = CollectiveOptimizer(optimizer, strategy=None) @@ -40,8 +42,9 @@ class CollectiveOptimizerTest(unittest.TestCase): dist_strategy = DistributedStrategy() dist_strategy.forward_recompute = True dist_strategy.recompute_checkpoints = "NoneListTest" - self.assertRaises(ValueError, CollectiveOptimizer, optimizer, - dist_strategy) + self.assertRaises( + ValueError, CollectiveOptimizer, optimizer, dist_strategy + ) dist_strategy.recompute_checkpoints = [] dist_optimizer = CollectiveOptimizer(optimizer, dist_strategy) self.assertRaises(ValueError, dist_optimizer.minimize, None) @@ -58,7 +61,8 @@ class CollectiveOptimizerTest(unittest.TestCase): def test_amp_strategy(self): optimizer = fluid.optimizer.AdamOptimizer() optimizer = fluid.contrib.mixed_precision.decorate( - optimizer, init_loss_scaling=1.0, use_dynamic_loss_scaling=True) + optimizer, init_loss_scaling=1.0, use_dynamic_loss_scaling=True + ) dist_strategy = DistributedStrategy() dist_strategy.use_amp = True dist_optimizer = CollectiveOptimizer(optimizer, strategy=dist_strategy) diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_process_group.py b/python/paddle/fluid/tests/unittests/collective/test_collective_process_group.py index 8fe1090e1ff..fdefe6bda4c 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_process_group.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_process_group.py @@ -17,7 +17,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestProcessGroup(TestMultipleGpus): - def test_process_group_nccl(self): self.run_mnist_2gpu('process_group_nccl.py') diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce.py index e5cc5441097..96033e2e307 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestCReduceOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_api.py index 35bff97f916..ae946d99750 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_api.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestCollectiveReduceAPI(TestDistBase): - def _setup_config(self): pass @@ -38,30 +37,47 @@ class TestCollectiveReduceAPI(TestDistBase): def test_reduce_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_reduce_api_dygraph.py", - "reduce", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_reduce_api_dygraph.py", + "reduce", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_reduce_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", ] for dtype in dtypes_to_test: - self.check_with_place("collective_reduce_api_dygraph.py", - "reduce", - "gloo", - "1", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_reduce_api_dygraph.py", + "reduce", + "gloo", + "1", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py index a9151e14665..634a7ecee0c 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py @@ -19,7 +19,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestCollectiveReduceScatter(TestMultipleGpus): - def test_collective_reduce_scatter(self): self.run_mnist_2gpu('collective_reduce_scatter.py', eager_mode=True) diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter_api.py index 669478f58a3..26dfbd2fe13 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter_api.py @@ -17,23 +17,30 @@ import test_collective_api_base as test_base class TestCollectiveReduceScatterAPI(test_base.TestDistBase): - def _setup_config(self): pass def test_reduce_scatter_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_reduce_scatter_api_dygraph.py", - "reduce_scatter", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_reduce_scatter_api_dygraph.py", + "reduce_scatter", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_scatter.py b/python/paddle/fluid/tests/unittests/collective/test_collective_scatter.py index e4d26459967..1f36e5928ed 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_scatter.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestCScatterOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_scatter_api.py index ab7de7975fe..f421e2ccccf 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_scatter_api.py @@ -21,43 +21,60 @@ paddle.enable_static() class TestCollectiveScatterAPI(TestDistBase): - def _setup_config(self): pass def test_scatter_gloo(self): - self.check_with_place("collective_scatter_api.py", "scatter", "gloo", - "4") + self.check_with_place( + "collective_scatter_api.py", "scatter", "gloo", "4" + ) def test_scatter_nccl(self): self.check_with_place("collective_scatter_api.py", "scatter", "nccl") def test_scatter_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_scatter_api_dygraph.py", - "scatter", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_scatter_api_dygraph.py", + "scatter", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_scatter_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", ] for dtype in dtypes_to_test: - self.check_with_place("collective_scatter_api_dygraph.py", - "scatter", - "gloo", - "4", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_scatter_api_dygraph.py", + "scatter", + "gloo", + "4", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv.py b/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv.py index 8f1467cf652..aa558a73a20 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestSendRecvOp(TestDistBase): - def _setup_config(self): pass @@ -29,12 +28,14 @@ class TestSendRecvOp(TestDistBase): self.check_with_place("collective_sendrecv_op.py", "sendrecv") def test_sendrecv_dynamic_shape(self): - self.check_with_place("collective_sendrecv_op_dynamic_shape.py", - "sendrecv_dynamic_shape") + self.check_with_place( + "collective_sendrecv_op_dynamic_shape.py", "sendrecv_dynamic_shape" + ) def test_sendrecv_array(self): - self.check_with_place("collective_sendrecv_op_array.py", - "sendrecv_array") + self.check_with_place( + "collective_sendrecv_op_array.py", "sendrecv_array" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv_api.py index 3db6df5d46e..72bd44666ff 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv_api.py @@ -21,28 +21,35 @@ paddle.enable_static() class TestCollectiveSendRecvAPI(TestDistBase): - def _setup_config(self): pass - #def test_sendrecv_nccl(self): + # def test_sendrecv_nccl(self): # if paddle.fluid.core.is_compiled_with_cuda(): # self.check_with_place("collective_sendrecv_api.py", "sendrecv", # "nccl") def test_sendrecv_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_sendrecv_api_dygraph.py", - "sendrecv", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_sendrecv_api_dygraph.py", + "sendrecv", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_split_col_linear.py b/python/paddle/fluid/tests/unittests/collective/test_collective_split_col_linear.py index c2983d3ea79..ee5b334bf2f 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_split_col_linear.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_split_col_linear.py @@ -21,13 +21,13 @@ paddle.enable_static() class TestColParallelLinearAPI(TestDistBase): - def _setup_config(self): pass def test_col_parallel_linear(self): - self.check_with_place("column_parallel_linear_api.py", - "column_parallel_linear", "nccl") + self.check_with_place( + "column_parallel_linear_api.py", "column_parallel_linear", "nccl" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding.py b/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding.py index fbab3e68cea..5b7e6dc7d73 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding.py @@ -21,13 +21,13 @@ paddle.enable_static() class TestParallelEmbeddingAPI(TestDistBase): - def _setup_config(self): pass def test_parallel_embedding(self): - self.check_with_place("parallel_embedding_api.py", "parallel_embedding", - "nccl") + self.check_with_place( + "parallel_embedding_api.py", "parallel_embedding", "nccl" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding_none_divisible.py b/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding_none_divisible.py index b5888390bca..4cf5a59f153 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding_none_divisible.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding_none_divisible.py @@ -20,15 +20,14 @@ paddle.enable_static() class TestCollectiveSplitAssert(unittest.TestCase): - def network(self): fleet.init() - data = paddle.static.data(name='tindata', - shape=[10, 1000], - dtype="float32") - emb_out = paddle.distributed.split(data, (7, 8), - operation="embedding", - num_partitions=2) + data = paddle.static.data( + name='tindata', shape=[10, 1000], dtype="float32" + ) + emb_out = paddle.distributed.split( + data, (7, 8), operation="embedding", num_partitions=2 + ) def test_assert(self): with self.assertRaises(AssertionError): diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_split_row_linear.py b/python/paddle/fluid/tests/unittests/collective/test_collective_split_row_linear.py index c89b1cc7a06..8253208e8e5 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_split_row_linear.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_split_row_linear.py @@ -21,13 +21,13 @@ paddle.enable_static() class TestRowParallelLinearAPI(TestDistBase): - def _setup_config(self): pass def test_row_parallel_linear(self): - self.check_with_place("row_parallel_linear_api.py", - "row_parallel_linear", "nccl") + self.check_with_place( + "row_parallel_linear_api.py", "row_parallel_linear", "nccl" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_wait.py b/python/paddle/fluid/tests/unittests/collective/test_collective_wait.py index cf4bcd98fa0..934279b7622 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_wait.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_wait.py @@ -21,14 +21,13 @@ paddle.enable_static() class TestCWaitOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_wait(self): - self.check_with_place("collective_allreduce_op_wait.py", - "allreduce", - check_error_log=True) + self.check_with_place( + "collective_allreduce_op_wait.py", "allreduce", check_error_log=True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_api_base.py b/python/paddle/fluid/tests/unittests/collective/test_communication_api_base.py index 22edac00155..ee4601972f0 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_api_base.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_api_base.py @@ -22,7 +22,6 @@ import shutil class CommunicationTestDistBase(unittest.TestCase): - def setUp(self, save_log_dir=None, num_of_devices=2, timeout=120): self._python_interp = sys.executable self._save_log_dir = save_log_dir @@ -41,18 +40,24 @@ class CommunicationTestDistBase(unittest.TestCase): start_command_list = start_command.strip().split() try: - self._launcher = subprocess.run(start_command_list, - env=runtime_envs, - timeout=self._timeout, - check=True) + self._launcher = subprocess.run( + start_command_list, + env=runtime_envs, + timeout=self._timeout, + check=True, + ) except subprocess.TimeoutExpired as err: raise TimeoutError( - "Timeout while running command {}, try to set a longer period, {} is not enough." - .format(err.cmd, err.timeout)) + "Timeout while running command {}, try to set a longer period, {} is not enough.".format( + err.cmd, err.timeout + ) + ) except subprocess.CalledProcessError as err: raise RuntimeError( - "Error occurs when running this test case. The return code of command {} is {}" - .format(err.cmd, err.returncode)) + "Error occurs when running this test case. The return code of command {} is {}".format( + err.cmd, err.returncode + ) + ) def tearDown(self): if self._save_log_dir: @@ -63,7 +68,8 @@ class CommunicationTestDistBase(unittest.TestCase): shutil.copytree(self._log_dir.name, dir_name) else: raise RuntimeError( - "Directory {} exists, failed to save log.".format(dir_name)) + "Directory {} exists, failed to save log.".format(dir_name) + ) def gen_product_envs_list(default_envs, changeable_envs): diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allgather_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allgather_api.py index 306248a0335..d9d22dca3d1 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allgather_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allgather_api.py @@ -17,29 +17,32 @@ import test_communication_api_base as test_base class TestCommunicationStreamAllgatherAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamAllgatherAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamAllgatherAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_allgather_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_allgather_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_allgather_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamAllgatherAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allreduce_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allreduce_api.py index 7fe30a5d50f..8823466b282 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allreduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allreduce_api.py @@ -17,29 +17,32 @@ import test_communication_api_base as test_base class TestCommunicationStreamAllreduceAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamAllreduceAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamAllreduceAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_allreduce_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_allreduce_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_allreduce_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamAllreduceAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_api.py index b6094d69321..0169825e114 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_api.py @@ -17,29 +17,32 @@ import test_communication_api_base as test_base class TestCommunicationStreamAllToAllAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamAllToAllAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamAllToAllAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_alltoall_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_alltoall_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_alltoall_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamAllToAllAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_single_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_single_api.py index 81836dbc131..6a322b51137 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_single_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_single_api.py @@ -17,31 +17,34 @@ import test_communication_api_base as test_base class TestCommunicationStreamAllToAllSingleAPI( - test_base.CommunicationTestDistBase): - + test_base.CommunicationTestDistBase +): def setUp(self): - super(TestCommunicationStreamAllToAllSingleAPI, - self).setUp(num_of_devices=2, timeout=120) + super(TestCommunicationStreamAllToAllSingleAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_alltoall_single_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue self.run_test_case( "communication_stream_alltoall_single_api_dygraph.py", - user_defined_envs=envs) + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamAllToAllSingleAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_broadcast_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_broadcast_api.py index 693ef95a204..0bdf7e6b904 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_broadcast_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_broadcast_api.py @@ -17,29 +17,32 @@ import test_communication_api_base as test_base class TestCommunicationStreamBroadcastAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamBroadcastAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamBroadcastAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_broadcast_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_broadcast_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_broadcast_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamBroadcastAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_api.py index e0b50f6056f..0fc539ba1a7 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_api.py @@ -17,29 +17,32 @@ import test_communication_api_base as test_base class TestCommunicationStreamReduceAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamReduceAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamReduceAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_reduce_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_reduce_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_reduce_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamReduceAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_scatter_api.py index 1f6522d4362..bb8b9237622 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_scatter_api.py @@ -17,31 +17,34 @@ import test_communication_api_base as test_base class TestCommunicationStreamReduceScatterAPI( - test_base.CommunicationTestDistBase): - + test_base.CommunicationTestDistBase +): def setUp(self): - super(TestCommunicationStreamReduceScatterAPI, - self).setUp(num_of_devices=2, timeout=120) + super(TestCommunicationStreamReduceScatterAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_reduce_scatter_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue self.run_test_case( "communication_stream_reduce_scatter_api_dygraph.py", - user_defined_envs=envs) + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamReduceScatterAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_scatter_api.py index 2de99b40e73..5219085bf3b 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_scatter_api.py @@ -17,29 +17,32 @@ import test_communication_api_base as test_base class TestCommunicationStreamScatterAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamScatterAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamScatterAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_reduce_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_scatter_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_scatter_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamScatterAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_sendrecv_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_sendrecv_api.py index b32ceb2a952..5bd668aa4b8 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_sendrecv_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_sendrecv_api.py @@ -17,29 +17,32 @@ import test_communication_api_base as test_base class TestCommunicationStreamSendRecvAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamSendRecvAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamSendRecvAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_sendrecv_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_sendrecv_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_sendrecv_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamSendRecvAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_eager_dist_api.py b/python/paddle/fluid/tests/unittests/collective/test_eager_dist_api.py index 8fe1090e1ff..fdefe6bda4c 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_eager_dist_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_eager_dist_api.py @@ -17,7 +17,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestProcessGroup(TestMultipleGpus): - def test_process_group_nccl(self): self.run_mnist_2gpu('process_group_nccl.py') diff --git a/python/paddle/fluid/tests/unittests/collective/test_gen_nccl_id_op.py b/python/paddle/fluid/tests/unittests/collective/test_gen_nccl_id_op.py index 80afdd393f0..4656c97e670 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_gen_nccl_id_op.py +++ b/python/paddle/fluid/tests/unittests/collective/test_gen_nccl_id_op.py @@ -34,30 +34,35 @@ def run_gen_ncc_id(attr): with paddle.static.program_guard(main_program, startup_program): nccl_id_var = startup_program.global_block().create_var( - name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) for i in range(1, nccl_comm_num): startup_program.global_block().create_var( name="NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) if use_hallreduce: for i in range(0, nccl_comm_num): startup_program.global_block().create_var( name="Hierarchical_inter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().create_var( name="Hierarchical_exter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_nccl_id", inputs={}, outputs={"NCCLID": nccl_id_var}, - attrs=attr) + attrs=attr, + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -65,7 +70,6 @@ def run_gen_ncc_id(attr): class TestGenNcclIdOp(unittest.TestCase): - def setUp(self): try: self._dist_ut_port_0 = int(os.environ["PADDLE_DIST_UT_PORT"]) @@ -98,7 +102,7 @@ class TestGenNcclIdOp(unittest.TestCase): for i in range(nranks): attr['trainer_id'] = i # NOTE: multiprocessing cannot be covered by coverage - p = Process(target=run_gen_ncc_id, args=(attr, )) + p = Process(target=run_gen_ncc_id, args=(attr,)) p.start() procs.append(p) diff --git a/python/paddle/fluid/tests/unittests/collective/test_new_group_api.py b/python/paddle/fluid/tests/unittests/collective/test_new_group_api.py index dfa0fb2402f..e70f7073c09 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_new_group_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_new_group_api.py @@ -21,13 +21,13 @@ paddle.enable_static() class TestCollectiveAllreduceAPI(TestDistBase): - def _setup_config(self): pass def test_allreduce_nccl(self): - self.check_with_place("collective_allreduce_new_group_api.py", - "allreduce", "nccl") + self.check_with_place( + "collective_allreduce_new_group_api.py", "allreduce", "nccl" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/world_size_and_rank.py b/python/paddle/fluid/tests/unittests/collective/world_size_and_rank.py index 5916370f753..f6ae8444ef4 100644 --- a/python/paddle/fluid/tests/unittests/collective/world_size_and_rank.py +++ b/python/paddle/fluid/tests/unittests/collective/world_size_and_rank.py @@ -17,7 +17,6 @@ import paddle.distributed as dist class TestWorldSizeAndRankAPI(unittest.TestCase): - def setUp(self): self._num_of_ranks = 2 self._subgroup_ranks = [0, 1] @@ -33,9 +32,11 @@ class TestWorldSizeAndRankAPI(unittest.TestCase): self.assertEqual(dist.get_world_size(self._subgroup), world_size) def test_given_group_rank(self): - rank = self._subgroup_ranks.index( - self._global_rank - ) if self._global_rank in self._subgroup_ranks else -1 + rank = ( + self._subgroup_ranks.index(self._global_rank) + if self._global_rank in self._subgroup_ranks + else -1 + ) self.assertEqual(dist.get_rank(self._subgroup), rank) diff --git a/python/paddle/fluid/tests/unittests/collective_allgather_op.py b/python/paddle/fluid/tests/unittests/collective_allgather_op.py index e897e163885..ab39b834fdb 100644 --- a/python/paddle/fluid/tests/unittests/collective_allgather_op.py +++ b/python/paddle/fluid/tests/unittests/collective_allgather_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveAllGather(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ class TestCollectiveAllGather(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allgather", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allgather", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective_reducescatter.py b/python/paddle/fluid/tests/unittests/collective_reducescatter.py index 480f9aeb380..53f53046321 100644 --- a/python/paddle/fluid/tests/unittests/collective_reducescatter.py +++ b/python/paddle/fluid/tests/unittests/collective_reducescatter.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestCollectiveReduceScatter(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -29,9 +28,9 @@ class TestCollectiveReduceScatter(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = fluid.layers.collective._c_reducescatter(tindata, nranks) toutdata = fluid.layers.collective._c_sync_comm_stream(toutdata, 0) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective_reducescatter_op.py b/python/paddle/fluid/tests/unittests/collective_reducescatter_op.py index 042817f8c3f..1605c16183c 100644 --- a/python/paddle/fluid/tests/unittests/collective_reducescatter_op.py +++ b/python/paddle/fluid/tests/unittests/collective_reducescatter_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveReduceScatter(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ class TestCollectiveReduceScatter(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofrs", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_reducescatter", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_reducescatter", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/ctr_dataset_reader.py b/python/paddle/fluid/tests/unittests/ctr_dataset_reader.py index 981fe0f0ef5..5cc96cf1c4e 100644 --- a/python/paddle/fluid/tests/unittests/ctr_dataset_reader.py +++ b/python/paddle/fluid/tests/unittests/ctr_dataset_reader.py @@ -61,12 +61,10 @@ def load_lr_input_record(sent): class CtrReader(object): - def __init__(self): pass def _reader_creator(self, filelist): - def get_rand(low=0.0, high=1.0): return random.random() @@ -85,9 +83,7 @@ class CtrReader(object): class DatasetCtrReader(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def get_rand(low=0.0, high=1.0): return random.random() @@ -97,9 +93,10 @@ class DatasetCtrReader(fleet.MultiSlotDataGenerator): dnn_input = load_dnn_input_record(fs[0]) lr_input = load_lr_input_record(fs[1]) click = [int(fs[2])] - yield ("dnn_data", dnn_input), \ - ("lr_data", lr_input), \ - ("click", click) + yield ("dnn_data", dnn_input), ("lr_data", lr_input), ( + "click", + click, + ) return iter @@ -115,7 +112,9 @@ def prepare_data(): lines = f.readlines() err_info = "wrong meta format" assert len(lines) == 2, err_info - assert 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1], err_info + assert ( + 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1] + ), err_info res = map(int, [_.split(':')[1] for _ in lines]) res = list(res) dnn_input_dim = res[0] @@ -126,10 +125,9 @@ def prepare_data(): return dnn_input_dim, lr_input_dim, train_file_path -def gen_fake_line(dnn_data_num=7, - dnn_data_range=1e5, - lr_data_num=5, - lr_data_range=1e5): +def gen_fake_line( + dnn_data_num=7, dnn_data_range=1e5, lr_data_num=5, lr_data_range=1e5 +): line = "" # for deep data @@ -188,16 +186,17 @@ def prepare_fake_data(file_nums=4, file_lines=500): warnings.warn("Fake data write in {}".format(file_dir)) for file_index in range(file_nums): with open( - os.path.join(file_dir, - "ctr_train_data_part_{}".format(file_index)), - 'w+') as fin: + os.path.join(file_dir, "ctr_train_data_part_{}".format(file_index)), + 'w+', + ) as fin: file_str = "" file_str += gen_zero_line() for line_index in range(file_lines - 1): file_str += gen_fake_line() fin.write(file_str) warnings.warn( - "Write done ctr_train_data_part_{}".format(file_index)) + "Write done ctr_train_data_part_{}".format(file_index) + ) file_list = [os.path.join(file_dir, x) for x in os.listdir(file_dir)] assert len(file_list) == file_nums diff --git a/python/paddle/fluid/tests/unittests/decorator_helper.py b/python/paddle/fluid/tests/unittests/decorator_helper.py index 0d7887d9189..d1165e2a919 100644 --- a/python/paddle/fluid/tests/unittests/decorator_helper.py +++ b/python/paddle/fluid/tests/unittests/decorator_helper.py @@ -18,9 +18,7 @@ __all__ = ['many_times', 'prog_scope'] def many_times(times): - def __impl__(fn): - def __fn__(*args, **kwargs): for _ in range(times): fn(*args, **kwargs) @@ -31,9 +29,7 @@ def many_times(times): def prog_scope(): - def __impl__(fn): - def __fn__(*args, **kwargs): prog = fluid.Program() startup_prog = fluid.Program() diff --git a/python/paddle/fluid/tests/unittests/detected_gpu.py b/python/paddle/fluid/tests/unittests/detected_gpu.py index 47287278ed9..e77c6f5c8ff 100644 --- a/python/paddle/fluid/tests/unittests/detected_gpu.py +++ b/python/paddle/fluid/tests/unittests/detected_gpu.py @@ -18,8 +18,10 @@ import paddle.fluid as fluid print("compile with cuda:", fluid.core.is_compiled_with_cuda()) print("get_cuda_device_count:", fluid.core.get_cuda_device_count()) -if fluid.core.is_compiled_with_cuda( -) and fluid.core.get_cuda_device_count() > 0: +if ( + fluid.core.is_compiled_with_cuda() + and fluid.core.get_cuda_device_count() > 0 +): sys.exit(0) else: sys.exit(1) diff --git a/python/paddle/fluid/tests/unittests/dist_allreduce_op.py b/python/paddle/fluid/tests/unittests/dist_allreduce_op.py index 49ef5d64bc7..af769963158 100644 --- a/python/paddle/fluid/tests/unittests/dist_allreduce_op.py +++ b/python/paddle/fluid/tests/unittests/dist_allreduce_op.py @@ -35,8 +35,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -44,25 +46,28 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, single_device=False): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -75,17 +80,19 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) # Optimization # TODO(typhoonzero): fix distributed adam optimizer @@ -104,7 +111,14 @@ class TestDistMnist2x2(TestDistRunnerBase): data_parallel_param_grads.append([p, grad_reduce]) opt.apply_gradients(data_parallel_param_grads) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_ctr.py b/python/paddle/fluid/tests/unittests/dist_ctr.py index b5578eca6ad..ab10ca2d639 100644 --- a/python/paddle/fluid/tests/unittests/dist_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_ctr.py @@ -28,26 +28,31 @@ fluid.default_main_program().random_seed = 1 class TestDistCTR2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): dnn_input_dim, lr_input_dim = dist_ctr_reader.load_data_meta() """ network definition """ - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) # build dnn model dnn_layer_dims = [128, 64, 32, 1] @@ -57,10 +62,13 @@ class TestDistCTR2x2(TestDistRunnerBase): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=IS_SPARSE) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=IS_SPARSE, + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): fc = fluid.layers.fc( @@ -68,8 +76,10 @@ class TestDistCTR2x2(TestDistRunnerBase): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -79,16 +89,19 @@ class TestDistCTR2x2(TestDistRunnerBase): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=IS_SPARSE) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=IS_SPARSE, + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) predict = fluid.layers.fc(input=merge_layer, size=2, act='softmax') acc = fluid.layers.accuracy(input=predict, label=label) - auc_var, batch_auc_var, auc_states = fluid.layers.auc(input=predict, - label=label) + auc_var, batch_auc_var, auc_states = fluid.layers.auc( + input=predict, label=label + ) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(x=cost) @@ -98,24 +111,35 @@ class TestDistCTR2x2(TestDistRunnerBase): use_l2_decay = bool(os.getenv('USE_L2_DECAY', 0)) if use_l2_decay: regularization = fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-1) + regularization_coeff=1e-1 + ) use_lr_decay = bool(os.getenv('LR_DECAY', 0)) lr = 0.0001 if use_lr_decay: - lr = fluid.layers.exponential_decay(learning_rate=0.0001, - decay_steps=10000, - decay_rate=0.999, - staircase=True) - - sgd_optimizer = fluid.optimizer.SGD(learning_rate=lr, - regularization=regularization) + lr = fluid.layers.exponential_decay( + learning_rate=0.0001, + decay_steps=10000, + decay_rate=0.999, + staircase=True, + ) + + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=lr, regularization=regularization + ) sgd_optimizer.minimize(avg_cost) dataset = dist_ctr_reader.Dataset() train_reader = paddle.batch(dataset.train(), batch_size=batch_size) test_reader = paddle.batch(dataset.test(), batch_size=batch_size) - return inference_program, avg_cost, train_reader, test_reader, None, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + None, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_ctr_reader.py b/python/paddle/fluid/tests/unittests/dist_ctr_reader.py index 4bc231e4eaf..fafb5f19c5f 100644 --- a/python/paddle/fluid/tests/unittests/dist_ctr_reader.py +++ b/python/paddle/fluid/tests/unittests/dist_ctr_reader.py @@ -109,7 +109,6 @@ feeding_index = {'dnn_input': 0, 'lr_input': 1, 'click': 2} class Dataset(object): - def train(self): ''' Load trainset. @@ -164,7 +163,9 @@ def load_data_meta(): lines = read_data('data.meta.txt') err_info = "wrong meta format" assert len(lines) == 2, err_info - assert 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1], err_info + assert ( + 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1] + ), err_info res = map(int, [_.split(':')[1] for _ in lines]) res = list(res) logger.info('dnn input dim: %d' % res[0]) diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py index 53ef00064bf..dd01938556f 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py @@ -35,7 +35,6 @@ fluid.default_main_program().random_seed = 1 def fake_ctr_reader(): - def reader(): for _ in range(1000): deep = np.random.random_integers(0, 1e5 - 1, size=16).tolist() @@ -63,38 +62,47 @@ class TestDistCTR2x2(FleetDistRunnerBase): """ dnn_input_dim, lr_input_dim = int(1e5), int(1e5) - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] if args.reader == "pyreader": if is_train: - self.reader = fluid.io.PyReader(feed_list=datas, - capacity=64, - iterable=False, - use_double_buffer=False) + self.reader = fluid.io.PyReader( + feed_list=datas, + capacity=64, + iterable=False, + use_double_buffer=False, + ) else: - self.test_reader = fluid.io.PyReader(feed_list=datas, - capacity=64, - iterable=False, - use_double_buffer=False) - - -# build dnn model + self.test_reader = fluid.io.PyReader( + feed_list=datas, + capacity=64, + iterable=False, + use_double_buffer=False, + ) + + # build dnn model dnn_layer_dims = [128, 128, 64, 32, 1] dnn_embedding = fluid.layers.embedding( is_distributed=False, @@ -102,11 +110,14 @@ class TestDistCTR2x2(FleetDistRunnerBase): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), + initializer=fluid.initializer.Constant(value=0.01), + ), is_sparse=True, - padding_idx=0) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + padding_idx=0, + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): fc = fluid.layers.fc( @@ -114,8 +125,10 @@ class TestDistCTR2x2(FleetDistRunnerBase): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -125,9 +138,11 @@ class TestDistCTR2x2(FleetDistRunnerBase): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), + initializer=fluid.initializer.Constant(value=0.01), + ), is_sparse=True, - padding_idx=0) + padding_idx=0, + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) @@ -135,8 +150,9 @@ class TestDistCTR2x2(FleetDistRunnerBase): predict = fluid.layers.fc(input=merge_layer, size=2, act='softmax') acc = fluid.layers.accuracy(input=predict, label=label) - auc_var, batch_auc_var, auc_states = fluid.layers.auc(input=predict, - label=label) + auc_var, batch_auc_var, auc_states = fluid.layers.auc( + input=predict, label=label + ) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(x=cost) @@ -176,11 +192,14 @@ class TestDistCTR2x2(FleetDistRunnerBase): try: while True: batch_idx += 1 - loss_val = exe.run(program=paddle.static.default_main_program(), - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=paddle.static.default_main_program(), + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) message = "TEST ---> batch_idx: {} loss: {}\n".format( - batch_idx, loss_val) + batch_idx, loss_val + ) fleet.util.print_on_rank(message, 0) except fluid.core.EOFException: self.test_reader.reset() @@ -208,8 +227,10 @@ class TestDistCTR2x2(FleetDistRunnerBase): try: pass_start = time.time() while True: - loss_val = exe.run(program=fluid.default_main_program(), - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=fluid.default_main_program(), + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) # TODO(randomly fail) # reduce_output = fleet.util.all_reduce( @@ -217,7 +238,8 @@ class TestDistCTR2x2(FleetDistRunnerBase): # loss_all_trainer = fleet.util.all_gather(float(loss_val)) # loss_val = float(reduce_output) / len(loss_all_trainer) message = "TRAIN ---> pass: {} loss: {}\n".format( - epoch_id, loss_val) + epoch_id, loss_val + ) fleet.util.print_on_rank(message, 0) pass_time = time.time() - pass_start @@ -229,9 +251,9 @@ class TestDistCTR2x2(FleetDistRunnerBase): fleet.save_persistables(exe, dirname=dirname) model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, model_dir, [feed.name for feed in self.feeds], self.avg_cost + ) if fleet.is_first_worker(): self.check_model_right(model_dir) shutil.rmtree(model_dir) @@ -251,29 +273,36 @@ class TestDistCTR2x2(FleetDistRunnerBase): dataset = paddle.distributed.QueueDataset() pipe_command = 'python ctr_dataset_reader.py' - dataset.init(batch_size=batch_size, - use_var=self.feeds, - pipe_command=pipe_command, - thread_num=thread_num) + dataset.init( + batch_size=batch_size, + use_var=self.feeds, + pipe_command=pipe_command, + thread_num=thread_num, + ) dataset.set_filelist(filelist) for epoch_id in range(1): pass_start = time.time() dataset.set_filelist(filelist) - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) pass_time = time.time() - pass_start if os.getenv("SAVE_MODEL") == "1": model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, + model_dir, + [feed.name for feed in self.feeds], + self.avg_cost, + ) if fleet.is_first_worker(): self.check_model_right(model_dir) shutil.rmtree(model_dir) @@ -311,20 +340,25 @@ class TestDistCTR2x2(FleetDistRunnerBase): for epoch_id in range(1): pass_start = time.time() - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) pass_time = time.time() - pass_start dataset.release_memory() if os.getenv("SAVE_MODEL") == "1": model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, + model_dir, + [feed.name for feed in self.feeds], + self.avg_cost, + ) fleet.load_inference_model(model_dir, mode=0) if fleet.is_first_worker(): self.check_model_right(model_dir) @@ -341,9 +375,12 @@ class TestDistCTR2x2(FleetDistRunnerBase): dense_param_dirname = os.getenv("SAVE_DENSE_PARAM_DIRNAME", None) if dense_param_dirname: - fleet.save_dense_params(exe, dense_param_dirname, - fluid.global_scope(), - fluid.default_main_program()) + fleet.save_dense_params( + exe, + dense_param_dirname, + fluid.global_scope(), + fluid.default_main_program(), + ) save_one_table_dirname = os.getenv("SAVE_ONE_TABLE_DIRNAME", None) if save_one_table_dirname: @@ -355,5 +392,6 @@ class TestDistCTR2x2(FleetDistRunnerBase): fleet.save_persistables(exe, patch_dirname, None, 5) fleet.check_save_pre_patch_done() + if __name__ == "__main__": runtime_main(TestDistCTR2x2) diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_ctr_ps_gpu.py b/python/paddle/fluid/tests/unittests/dist_fleet_ctr_ps_gpu.py index 13ec262697e..62b41a3f455 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_ctr_ps_gpu.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_ctr_ps_gpu.py @@ -70,15 +70,19 @@ class TestDistGpuPsCTR2x2(TestDistCTR2x2): try: pass_start = time.time() while True: - loss_val = exe.run(program=fleet.main_program, - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=fleet.main_program, + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) - reduce_output = fleet.util.all_reduce(np.array(loss_val), - mode="sum") + reduce_output = fleet.util.all_reduce( + np.array(loss_val), mode="sum" + ) loss_all_trainer = fleet.util.all_gather(float(loss_val)) loss_val = float(reduce_output) / len(loss_all_trainer) message = "TRAIN ---> pass: {} loss: {}\n".format( - epoch_id, loss_val) + epoch_id, loss_val + ) fleet.util.print_on_rank(message, 0) pass_time = time.time() - pass_start @@ -86,9 +90,9 @@ class TestDistGpuPsCTR2x2(TestDistCTR2x2): self.reader.reset() model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, model_dir, [feed.name for feed in self.feeds], self.avg_cost + ) if fleet.is_first_worker(): self.check_model_right(model_dir) if fleet.is_first_worker(): @@ -96,8 +100,11 @@ class TestDistGpuPsCTR2x2(TestDistCTR2x2): shutil.rmtree(model_dir) def do_dataset_training(self, fleet): - dnn_input_dim, lr_input_dim, train_file_path = ctr_dataset_reader.prepare_data( - ) + ( + dnn_input_dim, + lr_input_dim, + train_file_path, + ) = ctr_dataset_reader.prepare_data() device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace(device_id) @@ -125,19 +132,24 @@ class TestDistGpuPsCTR2x2(TestDistCTR2x2): for epoch_id in range(1): pass_start = time.time() dataset.set_filelist(filelist) - exe.train_from_dataset(program=fleet.main_program, - dataset=dataset, - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fleet.main_program, + dataset=dataset, + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) pass_time = time.time() - pass_start if os.getenv("SAVE_MODEL") == "1": model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, + model_dir, + [feed.name for feed in self.feeds], + self.avg_cost, + ) if fleet.is_first_worker(): self.check_model_right(model_dir) if fleet.is_first_worker(): diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_debug_gloo.py b/python/paddle/fluid/tests/unittests/dist_fleet_debug_gloo.py index 82f7a2bc672..5b0c66538bb 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_debug_gloo.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_debug_gloo.py @@ -13,21 +13,24 @@ # limitations under the License. import logging -#import paddle.fluid.incubate.fleet.base.role_maker as role_maker + +# import paddle.fluid.incubate.fleet.base.role_maker as role_maker import paddle.distributed.fleet.base.role_maker as role_maker -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger("fluid") logger.setLevel(logging.INFO) -#role = role_maker.GeneralRoleMaker( -#init_timeout_seconds=100, -#run_timeout_seconds=100, -#http_ip_port="127.0.0.1:26001") +# role = role_maker.GeneralRoleMaker( +# init_timeout_seconds=100, +# run_timeout_seconds=100, +# http_ip_port="127.0.0.1:26001") -#role = role_maker.PaddleCloudRoleMaker(http_ip_port="127.0.0.1:26001") +# role = role_maker.PaddleCloudRoleMaker(http_ip_port="127.0.0.1:26001") -#role = role_maker.GeneralRoleMaker(path="./tmp4") +# role = role_maker.GeneralRoleMaker(path="./tmp4") logger.info("Begin") res = [0, 0] @@ -37,7 +40,7 @@ role = role_maker.PaddleCloudRoleMaker(path="./tmp4") fleet.init(role) print("init wancheng") # -#if fleet.is_worker(): +# if fleet.is_worker(): # import time # time.sleep(3) @@ -49,7 +52,7 @@ if fleet.worker_index() == 0: elif fleet.worker_index() == 1: role._all_reduce(role._node_type_comm, b) -#logger.info(res) -#print("res ", res) +# logger.info(res) +# print("res ", res) -#role._barrier_all() +# role._barrier_all() diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py index c4e78ba16e0..5f3aaf4efdc 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py @@ -49,21 +49,27 @@ class TestHeterPipelinePsCTR2x2(FleetDistHeterRunnerBase): dnn_input_dim, lr_input_dim = int(1e5), int(1e5) with fluid.device_guard("cpu"): - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="float32", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="float32", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] @@ -75,10 +81,13 @@ class TestHeterPipelinePsCTR2x2(FleetDistHeterRunnerBase): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool # build lr model @@ -88,10 +97,13 @@ class TestHeterPipelinePsCTR2x2(FleetDistHeterRunnerBase): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) - lr_pool = fluid.layers.sequence_pool(input=lr_embbding, - pool_type="sum") + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) + lr_pool = fluid.layers.sequence_pool( + input=lr_embbding, pool_type="sum" + ) with fluid.device_guard("gpu"): for i, dim in enumerate(dnn_layer_dims[1:]): @@ -100,8 +112,10 @@ class TestHeterPipelinePsCTR2x2(FleetDistHeterRunnerBase): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc with fluid.device_guard("cpu"): @@ -135,8 +149,9 @@ class TestHeterPipelinePsCTR2x2(FleetDistHeterRunnerBase): train_file_list = ctr_dataset_reader.prepare_fake_data() exe = fluid.Executor(fluid.CPUPlace()) - real_program = fluid.default_main_program( - )._heter_pipeline_opt["section_program"] + real_program = fluid.default_main_program()._heter_pipeline_opt[ + "section_program" + ] print(real_program) exe.run(fluid.default_startup_program()) @@ -161,12 +176,14 @@ class TestHeterPipelinePsCTR2x2(FleetDistHeterRunnerBase): for epoch_id in range(1): pass_start = time.time() dataset.set_filelist(filelist) - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) pass_time = time.time() - pass_start print("do_dataset_training done. using time {}".format(pass_time)) exe.close() @@ -176,24 +193,27 @@ class TestHeterPipelinePsCTR2x2(FleetDistHeterRunnerBase): exe = fluid.Executor() exe.run(fluid.default_startup_program()) fleet.init_worker() - real_program = fluid.default_main_program( - )._heter_pipeline_opt["section_program"] + real_program = fluid.default_main_program()._heter_pipeline_opt[ + "section_program" + ] print(real_program) thread_num = int(os.getenv("CPU_NUM", 2)) batch_size = 128 pass_start = time.time() - exe.train_from_dataset(program=fluid.default_main_program(), - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fluid.default_main_program(), + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) exe.close() pass_time = time.time() - pass_start print("do_dataset_heter_training done. using time {}".format(pass_time)) - #for epoch_id in range(1): + # for epoch_id in range(1): # pass_start = time.time() # dataset.set_filelist(filelist) # exe.train_from_dataset( diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py index 54c37822257..fabf762f63b 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py @@ -37,8 +37,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -46,25 +48,28 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestFleetMetaOptimizerPrecision(TestDistRunnerBase): - def get_model(self, batch_size=2, single_device=False): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -77,17 +82,19 @@ class TestFleetMetaOptimizerPrecision(TestDistRunnerBase): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) test_program = fluid.default_main_program().clone(for_test=True) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) optimizer = paddle.fluid.optimizer.Adam(0.01) if single_device: @@ -97,11 +104,19 @@ class TestFleetMetaOptimizerPrecision(TestDistRunnerBase): fleet.init(role) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.without_graph_optimization = True - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - return test_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer_fuse_allreduce.py b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer_fuse_allreduce.py index 5db0377b36a..cf6a1415bcb 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer_fuse_allreduce.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer_fuse_allreduce.py @@ -37,8 +37,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -46,25 +48,28 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestFleetMetaOptimizerFuseAllReducePrecision(TestDistRunnerBase): - def get_model(self, batch_size=2, single_device=False): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -77,17 +82,19 @@ class TestFleetMetaOptimizerFuseAllReducePrecision(TestDistRunnerBase): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) test_program = fluid.default_main_program().clone(for_test=True) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) optimizer = paddle.fluid.optimizer.Adam(0.01) if single_device: @@ -100,11 +107,19 @@ class TestFleetMetaOptimizerFuseAllReducePrecision(TestDistRunnerBase): strategy.fuse_all_reduce_ops = True strategy._calc_comm_same_stream = False strategy.fuse_grad_size_in_num = 8 - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - return test_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_simnet_bow.py b/python/paddle/fluid/tests/unittests/dist_fleet_simnet_bow.py index 84687999d9c..f9d926ad1c7 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_simnet_bow.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_simnet_bow.py @@ -41,7 +41,6 @@ fluid.default_main_program().random_seed = 1 def fake_simnet_reader(): - def reader(): for _ in range(1000): q = np.random.random_integers(0, 1500 - 1, size=1).tolist() @@ -57,62 +56,66 @@ def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost -def train_network(batch_size, - is_distributed=False, - is_sparse=False, - is_self_contained_lr=False, - is_pyreader=False): +def train_network( + batch_size, + is_distributed=False, + is_sparse=False, + is_self_contained_lr=False, + is_pyreader=False, +): # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) datas = [q, label, pt, nt] reader = None if is_pyreader: - reader = fluid.io.PyReader(feed_list=datas, - capacity=64, - iterable=False, - use_double_buffer=False) + reader = fluid.io.PyReader( + feed_list=datas, + capacity=64, + iterable=False, + use_double_buffer=False, + ) # embedding q_emb = fluid.embedding( @@ -120,8 +123,10 @@ def train_network(batch_size, is_distributed=is_distributed, size=[dict_dim, emb_dim], param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01), name="__emb__"), - is_sparse=is_sparse) + initializer=fluid.initializer.Constant(value=0.01), name="__emb__" + ), + is_sparse=is_sparse, + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -133,7 +138,8 @@ def train_network(batch_size, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr), + learning_rate=base_lr, + ), ) # embedding @@ -144,8 +150,10 @@ def train_network(batch_size, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -155,8 +163,10 @@ def train_network(batch_size, input=pt_ss, size=hid_dim, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01), name="__fc__"), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + initializer=fluid.initializer.Constant(value=0.01), name="__fc__" + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # embedding nt_emb = fluid.embedding( @@ -164,8 +174,10 @@ def train_network(batch_size, is_distributed=is_distributed, size=[dict_dim, emb_dim], param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01), name="__emb__"), - is_sparse=is_sparse) + initializer=fluid.initializer.Constant(value=0.01), name="__emb__" + ), + is_sparse=is_sparse, + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -175,8 +187,10 @@ def train_network(batch_size, input=nt_ss, size=hid_dim, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01), name="__fc__"), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + initializer=fluid.initializer.Constant(value=0.01), name="__fc__" + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -192,9 +206,13 @@ class TestDistSimnetBow2x2(FleetDistRunnerBase): """ def net(self, args, batch_size=4, lr=0.01): - avg_cost, _, predict, self.reader = \ - train_network(batch_size=batch_size, is_distributed=False, - is_sparse=True, is_self_contained_lr=False, is_pyreader=(args.reader == "pyreader")) + avg_cost, _, predict, self.reader = train_network( + batch_size=batch_size, + is_distributed=False, + is_sparse=True, + is_self_contained_lr=False, + is_pyreader=(args.reader == "pyreader"), + ) self.avg_cost = avg_cost self.predict = predict @@ -229,11 +247,14 @@ class TestDistSimnetBow2x2(FleetDistRunnerBase): try: pass_start = time.time() while True: - loss_val = exe.run(program=fluid.default_main_program(), - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=fluid.default_main_program(), + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) message = "TRAIN ---> pass: {} loss: {}\n".format( - epoch_id, loss_val) + epoch_id, loss_val + ) fleet.util.print_on_rank(message, 0) pass_time = time.time() - pass_start diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_sparse_embedding_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_sparse_embedding_ctr.py index c6a976b089b..c1410348e0f 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_sparse_embedding_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_sparse_embedding_ctr.py @@ -26,7 +26,6 @@ from test_dist_fleet_base import runtime_main, FleetDistRunnerBase def fake_ctr_reader(): - def reader(): for _ in range(1000): deep = np.random.random_integers(0, 1e10, size=16).tolist() @@ -54,29 +53,37 @@ class TestDistCTR2x2(FleetDistRunnerBase): """ dnn_input_dim, lr_input_dim = 10, 10 - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] if args.reader == "pyreader": - self.reader = fluid.io.PyReader(feed_list=datas, - capacity=64, - iterable=False, - use_double_buffer=False) + self.reader = fluid.io.PyReader( + feed_list=datas, + capacity=64, + iterable=False, + use_double_buffer=False, + ) # build dnn model initializer = int(os.getenv("INITIALIZER", "0")) @@ -98,9 +105,11 @@ class TestDistCTR2x2(FleetDistRunnerBase): size=[dnn_input_dim, dnn_layer_dims[0]], is_test=inference, entry=entry, - param_attr=fluid.ParamAttr(name="deep_embedding", initializer=init)) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + param_attr=fluid.ParamAttr(name="deep_embedding", initializer=init), + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): fc = fluid.layers.fc( @@ -108,8 +117,10 @@ class TestDistCTR2x2(FleetDistRunnerBase): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -120,7 +131,9 @@ class TestDistCTR2x2(FleetDistRunnerBase): entry=entry, param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01), + ), + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) @@ -159,19 +172,27 @@ class TestDistCTR2x2(FleetDistRunnerBase): self.reader.start() try: while True: - loss_val = exe.run(program=fluid.default_main_program(), - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=fluid.default_main_program(), + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) - print("TRAIN ---> pass: {} loss: {}\n".format( - epoch_id, loss_val)) + print( + "TRAIN ---> pass: {} loss: {}\n".format( + epoch_id, loss_val + ) + ) except fluid.core.EOFException: self.reader.reset() model_dir = os.getenv("MODEL_DIR", None) if model_dir: - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, + model_dir, + [feed.name for feed in self.feeds], + self.avg_cost, + ) fleet.load_model(model_dir, mode=1) diff --git a/python/paddle/fluid/tests/unittests/dist_mnist.py b/python/paddle/fluid/tests/unittests/dist_mnist.py index 1a0f61411a3..856ac1b930b 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist.py @@ -36,8 +36,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -45,25 +47,28 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -76,9 +81,9 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization @@ -88,24 +93,34 @@ class TestDistMnist2x2(TestDistRunnerBase): if not use_dgc: opt = fluid.optimizer.Momentum(learning_rate=self.lr, momentum=0.9) else: - opt = fluid.optimizer.DGCMomentumOptimizer(learning_rate=self.lr, - momentum=0.9, - rampup_begin_step=2) + opt = fluid.optimizer.DGCMomentumOptimizer( + learning_rate=self.lr, momentum=0.9, rampup_begin_step=2 + ) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=dist_strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=dist_strategy + ) _, param_grads = dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_mnist_batch_merge.py b/python/paddle/fluid/tests/unittests/dist_mnist_batch_merge.py index 67981130fc2..1e022a1c445 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist_batch_merge.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist_batch_merge.py @@ -35,7 +35,6 @@ def test_merge_reader(repeat_batch_size=8): class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -48,9 +47,9 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization @@ -58,10 +57,18 @@ class TestDistMnist2x2(TestDistRunnerBase): # Reader train_reader = paddle.batch(test_merge_reader, batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_mnist_fp16_allreduce.py b/python/paddle/fluid/tests/unittests/dist_mnist_fp16_allreduce.py index c9d37247b77..c0d92d01e6c 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist_fp16_allreduce.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist_fp16_allreduce.py @@ -14,7 +14,9 @@ import paddle import paddle.fluid as fluid -from paddle.distributed.fleet.meta_optimizers import FP16AllReduceOptimizer as FP16AllReduce +from paddle.distributed.fleet.meta_optimizers import ( + FP16AllReduceOptimizer as FP16AllReduce, +) from test_dist_base import TestDistRunnerBase, runtime_main from dist_mnist import cnn_model @@ -27,7 +29,6 @@ fluid.default_main_program().random_seed = 1 class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -40,23 +41,33 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = FP16AllReduce(opt) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_mnist_lars.py b/python/paddle/fluid/tests/unittests/dist_mnist_lars.py index b93dfb0594e..c117efb84f8 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist_lars.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist_lars.py @@ -26,7 +26,6 @@ fluid.default_main_program().random_seed = 1 class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -39,22 +38,32 @@ class TestDistMnist2x2(TestDistRunnerBase): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization - opt = fluid.optimizer.LarsMomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.LarsMomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_save_load.py b/python/paddle/fluid/tests/unittests/dist_save_load.py index 8d0083aaffe..eb36010ea6f 100644 --- a/python/paddle/fluid/tests/unittests/dist_save_load.py +++ b/python/paddle/fluid/tests/unittests/dist_save_load.py @@ -28,9 +28,7 @@ from dist_simnet_bow import TestDistSimnetBow2x2, DATA_URL, DATA_MD5 class TestDistSaveLoad2x2(TestDistSimnetBow2x2): - def _load_persistable_vars(self, executor, dirname, program): - def _is_checkpoint_var(var): """ the checkpoint will not save or load all the variables. @@ -38,9 +36,11 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): : param var(Variable) """ - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.RAW: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.RAW + ): return False # @GRAD are named for gradient variables, checkpoint will not save it. if "@GRAD" in var.name: @@ -58,21 +58,30 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): return var.persistable - io.load_vars(executor, - dirname=dirname, - main_program=program, - predicate=_is_checkpoint_var, - filename=None) + io.load_vars( + executor, + dirname=dirname, + main_program=program, + predicate=_is_checkpoint_var, + filename=None, + ) def run_pserver(self, args): self.get_model(batch_size=2) # NOTE: pserver should not call memory optimize - t = self.get_transpiler(args.trainer_id, fluid.default_main_program(), - args.endpoints, args.trainers, args.sync_mode, - False, args.current_endpoint) + t = self.get_transpiler( + args.trainer_id, + fluid.default_main_program(), + args.endpoints, + args.trainers, + args.sync_mode, + False, + args.current_endpoint, + ) pserver_prog = t.get_pserver_program(args.current_endpoint) - startup_prog = t.get_startup_program(args.current_endpoint, - pserver_prog) + startup_prog = t.get_startup_program( + args.current_endpoint, pserver_prog + ) need_load = bool(int(os.getenv("LOAD", "0"))) model_dir = os.getenv("MODEL_DIR", "") @@ -87,14 +96,23 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): exe.run(pserver_prog) def run_trainer(self, args): - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=2) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=2) if args.update_method == "pserver": - t = self.get_transpiler(args.trainer_id, - fluid.default_main_program(), - args.endpoints, args.trainers, - args.sync_mode) + t = self.get_transpiler( + args.trainer_id, + fluid.default_main_program(), + args.endpoints, + args.trainers, + args.sync_mode, + ) trainer_prog = t.get_trainer_program() else: @@ -114,17 +132,24 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): build_stra = fluid.BuildStrategy() if args.use_reduce: - build_stra.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + build_stra.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) else: - build_stra.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce + build_stra.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.AllReduce + ) - exe = fluid.ParallelExecutor(args.use_cuda, - loss_name=avg_cost.name, - exec_strategy=strategy, - build_strategy=build_stra) + exe = fluid.ParallelExecutor( + args.use_cuda, + loss_name=avg_cost.name, + exec_strategy=strategy, + build_strategy=build_stra, + ) feed_var_list = [ - var for var in trainer_prog.global_block().vars.values() + var + for var in trainer_prog.global_block().vars.values() if var.is_data ] @@ -149,13 +174,15 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): if save_mode == "LOCAL": if need_save: for _ in range(RUN_STEP): - loss, = exe.run(fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) + (loss,) = exe.run( + fetch_list=[avg_cost.name], feed=feeder.feed(get_data()) + ) if need_save and model_dir: io.save_persistables(startup_exe, model_dir, trainer_prog) var = np.array( - fluid.global_scope().find_var('__fc_b__').get_tensor()) + fluid.global_scope().find_var('__fc_b__').get_tensor() + ) sys.stdout.buffer.write(pickle.dumps(np.ravel(var).tolist())) elif save_mode == "DIST": @@ -163,18 +190,26 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): loss = None if need_save: for idx in range(8): - loss, = exe.run(fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) - if need_save and model_dir and idx == skip_steps and args.trainer_id == 0: - io.save_persistables(startup_exe, model_dir, - trainer_prog) + (loss,) = exe.run( + fetch_list=[avg_cost.name], feed=feeder.feed(get_data()) + ) + if ( + need_save + and model_dir + and idx == skip_steps + and args.trainer_id == 0 + ): + io.save_persistables( + startup_exe, model_dir, trainer_prog + ) else: for idx in range(8): data = get_data() if idx <= skip_steps: continue - loss, = exe.run(fetch_list=[avg_cost.name], - feed=feeder.feed(data)) + (loss,) = exe.run( + fetch_list=[avg_cost.name], feed=feeder.feed(data) + ) sys.stdout.buffer.write(pickle.dumps(loss.tolist())) else: raise Exception("save_mode must be LOCAL or DIST") diff --git a/python/paddle/fluid/tests/unittests/dist_se_resnext.py b/python/paddle/fluid/tests/unittests/dist_se_resnext.py index 5061bb85956..0d8ed873f03 100644 --- a/python/paddle/fluid/tests/unittests/dist_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/dist_se_resnext.py @@ -31,13 +31,12 @@ train_parameters = { "learning_strategy": { "name": "piecewise_decay", "epochs": [30, 60, 90], - "steps": [0.1, 0.01, 0.001, 0.0001] - } + "steps": [0.1, 0.01, 0.001, 0.0001], + }, } -class SE_ResNeXt(): - +class SE_ResNeXt: def __init__(self, layers=50): self.params = train_parameters self.layers = layers @@ -45,64 +44,65 @@ class SE_ResNeXt(): def net(self, input, class_dim=1000): layers = self.layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] - conv = self.conv_bn_layer(input=input, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - conv = fluid.layers.pool2d(input=conv, - pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + conv = self.conv_bn_layer( + input=input, num_filters=64, filter_size=7, stride=2, act='relu' + ) + conv = fluid.layers.pool2d( + input=conv, + pool_size=3, + pool_stride=2, + pool_padding=1, + pool_type='max', + ) elif layers == 101: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 23, 3] num_filters = [128, 256, 512, 1024] - conv = self.conv_bn_layer(input=input, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - conv = fluid.layers.pool2d(input=conv, - pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + conv = self.conv_bn_layer( + input=input, num_filters=64, filter_size=7, stride=2, act='relu' + ) + conv = fluid.layers.pool2d( + input=conv, + pool_size=3, + pool_stride=2, + pool_padding=1, + pool_type='max', + ) elif layers == 152: cardinality = 64 reduction_ratio = 16 depth = [3, 8, 36, 3] num_filters = [128, 256, 512, 1024] - conv = self.conv_bn_layer(input=input, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - conv = self.conv_bn_layer(input=conv, - num_filters=64, - filter_size=3, - stride=1, - act='relu') - conv = self.conv_bn_layer(input=conv, - num_filters=128, - filter_size=3, - stride=1, - act='relu') + conv = self.conv_bn_layer( + input=input, num_filters=64, filter_size=3, stride=2, act='relu' + ) + conv = self.conv_bn_layer( + input=conv, num_filters=64, filter_size=3, stride=1, act='relu' + ) + conv = self.conv_bn_layer( + input=conv, num_filters=128, filter_size=3, stride=1, act='relu' + ) conv = fluid.layers.pool2d( - input=conv, pool_size=3, pool_stride=2, pool_padding=1, \ - pool_type='max') + input=conv, + pool_size=3, + pool_stride=2, + pool_padding=1, + pool_type='max', + ) for block in range(len(depth)): for i in range(depth[block]): @@ -111,20 +111,22 @@ class SE_ResNeXt(): num_filters=num_filters[block], stride=2 if i == 0 and block != 0 else 1, cardinality=cardinality, - reduction_ratio=reduction_ratio) + reduction_ratio=reduction_ratio, + ) - pool = fluid.layers.pool2d(input=conv, - pool_size=7, - pool_type='avg', - global_pooling=True) + pool = fluid.layers.pool2d( + input=conv, pool_size=7, pool_type='avg', global_pooling=True + ) drop = fluid.layers.dropout(x=pool, dropout_prob=0.2) stdv = 1.0 / math.sqrt(drop.shape[1] * 1.0) out = fluid.layers.fc( input=drop, size=class_dim, act='softmax', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + ) return out def shortcut(self, input, ch_out, stride): @@ -135,37 +137,36 @@ class SE_ResNeXt(): else: return input - def bottleneck_block(self, input, num_filters, stride, cardinality, - reduction_ratio): - conv0 = self.conv_bn_layer(input=input, - num_filters=num_filters, - filter_size=1, - act='relu') - conv1 = self.conv_bn_layer(input=conv0, - num_filters=num_filters, - filter_size=3, - stride=stride, - groups=cardinality, - act='relu') - conv2 = self.conv_bn_layer(input=conv1, - num_filters=num_filters * 2, - filter_size=1, - act=None) - scale = self.squeeze_excitation(input=conv2, - num_channels=num_filters * 2, - reduction_ratio=reduction_ratio) + def bottleneck_block( + self, input, num_filters, stride, cardinality, reduction_ratio + ): + conv0 = self.conv_bn_layer( + input=input, num_filters=num_filters, filter_size=1, act='relu' + ) + conv1 = self.conv_bn_layer( + input=conv0, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + act='relu', + ) + conv2 = self.conv_bn_layer( + input=conv1, num_filters=num_filters * 2, filter_size=1, act=None + ) + scale = self.squeeze_excitation( + input=conv2, + num_channels=num_filters * 2, + reduction_ratio=reduction_ratio, + ) short = self.shortcut(input, num_filters * 2, stride) return fluid.layers.elementwise_add(x=short, y=scale, act='relu') - def conv_bn_layer(self, - input, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def conv_bn_layer( + self, input, num_filters, filter_size, stride=1, groups=1, act=None + ): conv = fluid.layers.conv2d( input=input, num_filters=num_filters, @@ -175,41 +176,45 @@ class SE_ResNeXt(): groups=groups, act=None, # avoid pserver CPU init differs from GPU - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - bias_attr=False) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + bias_attr=False, + ) return fluid.layers.batch_norm(input=conv, act=act) def squeeze_excitation(self, input, num_channels, reduction_ratio): - pool = fluid.layers.pool2d(input=input, - pool_size=0, - pool_type='avg', - global_pooling=True) + pool = fluid.layers.pool2d( + input=input, pool_size=0, pool_type='avg', global_pooling=True + ) stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0) squeeze = fluid.layers.fc( input=pool, size=num_channels // reduction_ratio, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - act='relu') + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + act='relu', + ) stdv = 1.0 / math.sqrt(squeeze.shape[1] * 1.0) excitation = fluid.layers.fc( input=squeeze, size=num_channels, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - act='sigmoid') + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + act='sigmoid', + ) scale = fluid.layers.elementwise_mul(x=input, y=excitation, axis=0) return scale class DistSeResneXt2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False): # Input data - image = fluid.layers.data(name="data", - shape=[3, 224, 224], - dtype='float32') + image = fluid.layers.data( + name="data", shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name="int64", shape=[1], dtype='int64') # Train program @@ -235,24 +240,30 @@ class DistSeResneXt2x2(TestDistRunnerBase): if not use_dgc: optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.piecewise_decay(boundaries=bd, - values=lr), + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr + ), momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) else: optimizer = fluid.optimizer.DGCMomentumOptimizer( - learning_rate=fluid.layers.piecewise_decay(boundaries=bd, - values=lr), + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr + ), momentum=0.9, rampup_begin_step=0, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) optimizer.minimize(avg_cost) # Reader - train_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), batch_size=batch_size + ) return test_program, avg_cost, train_reader, test_reader, acc_top1, out diff --git a/python/paddle/fluid/tests/unittests/dist_sharding_save.py b/python/paddle/fluid/tests/unittests/dist_sharding_save.py index b4733b61af8..d816d1b93ab 100755 --- a/python/paddle/fluid/tests/unittests/dist_sharding_save.py +++ b/python/paddle/fluid/tests/unittests/dist_sharding_save.py @@ -37,20 +37,21 @@ def runtime_main(): fleet.init(role) with fluid.program_guard(train_prog, startup_prog): with fluid.unique_name.guard(): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -61,10 +62,12 @@ def runtime_main(): "sharding_degree": 2, } - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.01, - momentum=0.9) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9 + ) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) # execution @@ -73,17 +76,16 @@ def runtime_main(): exe = fluid.Executor(place) exe.run(startup_prog) dirname = "./ut_sharding_save_model" - sharding.utils.save_persistables(exe, - dirname, - main_program=train_prog, - filename=None) + sharding.utils.save_persistables( + exe, dirname, main_program=train_prog, filename=None + ) out_losses = [] sys.stdout.buffer.write(pickle.dumps(out_losses)) if __name__ == "__main__": - #NOTE(liangjianzhong): dist unittest should be imlpement using runtime_main in test_dist_base.py + # NOTE(liangjianzhong): dist unittest should be imlpement using runtime_main in test_dist_base.py # but the runtime_main in test_dist_base.py use the fleet, DistributedStrategy from # paddle.fluid.incubate.fleet.collective which is not support by sharding (paddle.distributed.fleet). # this should be update in future. diff --git a/python/paddle/fluid/tests/unittests/dist_text_classification.py b/python/paddle/fluid/tests/unittests/dist_text_classification.py index c8315161f0f..417ff66e0cb 100644 --- a/python/paddle/fluid/tests/unittests/dist_text_classification.py +++ b/python/paddle/fluid/tests/unittests/dist_text_classification.py @@ -43,19 +43,23 @@ def get_worddict(dict_path): return word_dict, dict_dim -def conv_net(input, - dict_dim, - emb_dim=128, - window_size=3, - num_filters=128, - fc0_dim=96, - class_dim=2): +def conv_net( + input, + dict_dim, + emb_dim=128, + window_size=3, + num_filters=128, + fc0_dim=96, + class_dim=2, +): emb = fluid.layers.embedding( input=input, size=[dict_dim, emb_dim], is_sparse=False, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_3 = fluid.nets.sequence_conv_pool( input=emb, @@ -63,30 +67,35 @@ def conv_net(input, filter_size=window_size, act="tanh", pool_type="max", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) fc_0 = fluid.layers.fc( input=[conv_3], size=fc0_dim, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) prediction = fluid.layers.fc( input=[fc_0], size=class_dim, act="softmax", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return prediction def inference_network(dict_dim): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) out = conv_net(data, dict_dim) return out @@ -107,17 +116,16 @@ def get_optimizer(learning_rate): class TestDistTextClassification2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): - vocab = os.path.join(paddle.dataset.common.DATA_HOME, - "text_classification", "imdb.vocab") + vocab = os.path.join( + paddle.dataset.common.DATA_HOME, "text_classification", "imdb.vocab" + ) word_dict, dict_dim = get_worddict(vocab) # Input data - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') # Train program @@ -134,7 +142,14 @@ class TestDistTextClassification2x2(TestDistRunnerBase): # Reader train_reader, test_reader = get_reader(word_dict, batch_size) - return inference_program, avg_cost, train_reader, test_reader, acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + acc, + predict, + ) def tokenize(pattern): @@ -143,8 +158,10 @@ def tokenize(pattern): """ with tarfile.open( - paddle.dataset.common.download(DATA_URL, 'text_classification', - DATA_MD5)) as tarf: + paddle.dataset.common.download( + DATA_URL, 'text_classification', DATA_MD5 + ) + ) as tarf: # Note that we should use tarfile.next(), which does # sequential access of member files, other than # tarfile.extractfile, which does random access and might @@ -154,7 +171,8 @@ def tokenize(pattern): if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. yield tarf.extractfile(tf).read().rstrip(b'\n\r').translate( - None, string.punctuation.encode('latin-1')).lower().split() + None, string.punctuation.encode('latin-1') + ).lower().split() tf = tarf.next() @@ -188,8 +206,11 @@ def train(word_idx): :return: Training reader creator :rtype: callable """ - return reader_creator(re.compile(r"train/pos/.*\.txt$"), - re.compile(r"train/neg/.*\.txt$"), word_idx) + return reader_creator( + re.compile(r"train/pos/.*\.txt$"), + re.compile(r"train/neg/.*\.txt$"), + word_idx, + ) def test(word_idx): @@ -204,8 +225,11 @@ def test(word_idx): :return: Test reader creator :rtype: callable """ - return reader_creator(re.compile(r"test/pos/.*\.txt$"), - re.compile(r"test/neg/.*\.txt$"), word_idx) + return reader_creator( + re.compile(r"test/pos/.*\.txt$"), + re.compile(r"test/neg/.*\.txt$"), + word_idx, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_transformer.py b/python/paddle/fluid/tests/unittests/dist_transformer.py index 23bcd1957e0..4765ee7d82b 100644 --- a/python/paddle/fluid/tests/unittests/dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/dist_transformer.py @@ -35,7 +35,7 @@ fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 -#from transformer_config import ModelHyperParams, TrainTaskConfig, merge_cfg_from_list +# from transformer_config import ModelHyperParams, TrainTaskConfig, merge_cfg_from_list class TrainTaskConfig(object): # only support GPU currently use_gpu = True @@ -72,7 +72,8 @@ class TrainTaskConfig(object): check_acc = True data_path = expanduser("~") + ( - "/.cache/paddle/dataset/test_dist_transformer/") + "/.cache/paddle/dataset/test_dist_transformer/" + ) src_vocab_fpath = data_path + "vocab.bpe.32000" trg_vocab_fpath = data_path + "vocab.bpe.32000" train_file_pattern = data_path + "train.tok.clean.bpe.32000.en-de" @@ -179,12 +180,17 @@ input_descs = { # encoder. # The actual data shape of src_slf_attn_bias is: # [batch_size, n_head, max_src_len_in_batch, max_src_len_in_batch] - "src_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "src_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # The actual data shape of trg_word is: # [batch_size * max_trg_len_in_batch, 1] - "trg_word": [(batch_size, seq_len, 1), "int64", - 2], # lod_level is only used in fast decoder. + "trg_word": [ + (batch_size, seq_len, 1), + "int64", + 2, + ], # lod_level is only used in fast decoder. # The actual data shape of trg_pos is: # [batch_size * max_trg_len_in_batch, 1] "trg_pos": [(batch_size, seq_len, 1), "int64"], @@ -192,14 +198,18 @@ input_descs = { # subsequent words in the decoder. # The actual data shape of trg_slf_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_trg_len_in_batch] - "trg_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used to remove attention weights on paddings of the source # input in the encoder-decoder attention. # The actual data shape of trg_src_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_src_len_in_batch] - "trg_src_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_src_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used in independent decoder program for inference. # The actual data shape of enc_output is: # [batch_size, max_src_len_in_batch, d_model] @@ -212,8 +222,8 @@ input_descs = { # [batch_size * max_trg_len_in_batch, 1] "lbl_weight": [(batch_size * seq_len, 1), "float32"], # These inputs are used to change the shape tensor in beam-search decoder. - "trg_slf_attn_pre_softmax_shape_delta": [(2, ), "int32"], - "trg_slf_attn_post_softmax_shape_delta": [(4, ), "int32"], + "trg_slf_attn_pre_softmax_shape_delta": [(2,), "int32"], + "trg_slf_attn_post_softmax_shape_delta": [(4,), "int32"], "init_score": [(batch_size, 1), "float32"], } @@ -257,7 +267,7 @@ fast_decoder_data_input_fields = ( # "trg_slf_attn_post_softmax_shape_delta", ) -#from optim import LearningRateScheduler +# from optim import LearningRateScheduler class LearningRateScheduler(object): """ Wrapper for learning rate scheduling as described in the Transformer paper. @@ -265,12 +275,14 @@ class LearningRateScheduler(object): learning rate will be fed into the main_program as input data. """ - def __init__(self, - d_model, - warmup_steps, - learning_rate=0.001, - current_steps=0, - name="learning_rate"): + def __init__( + self, + d_model, + warmup_steps, + learning_rate=0.001, + current_steps=0, + name="learning_rate", + ): self.current_steps = current_steps self.warmup_steps = warmup_steps self.d_model = d_model @@ -280,68 +292,91 @@ class LearningRateScheduler(object): shape=[1], value=float(learning_rate), dtype="float32", - persistable=True) + persistable=True, + ) def update_learning_rate(self): self.current_steps += 1 - lr_value = np.power(self.d_model, -0.5) * np.min([ - np.power(self.current_steps, -0.5), - np.power(self.warmup_steps, -1.5) * self.current_steps - ]) * self.static_lr + lr_value = ( + np.power(self.d_model, -0.5) + * np.min( + [ + np.power(self.current_steps, -0.5), + np.power(self.warmup_steps, -1.5) * self.current_steps, + ] + ) + * self.static_lr + ) return np.array([lr_value], dtype="float32") -#from transformer_train import train_loop -def pad_batch_data(insts, - pad_idx, - n_head, - is_target=False, - is_label=False, - return_attn_bias=True, - return_max_len=True, - return_num_token=False): +# from transformer_train import train_loop +def pad_batch_data( + insts, + pad_idx, + n_head, + is_target=False, + is_label=False, + return_attn_bias=True, + return_max_len=True, + return_num_token=False, +): """ Pad the instances to the max sequence length in batch, and generate the corresponding position data and attention bias. """ return_list = [] max_len = max(len(inst) for inst in insts) - num_token = functools.reduce(lambda x, y: x + y, - [len(inst) - for inst in insts]) if return_num_token else 0 + num_token = ( + functools.reduce(lambda x, y: x + y, [len(inst) for inst in insts]) + if return_num_token + else 0 + ) # Any token included in dict can be used to pad, since the paddings' loss # will be masked out by weights and make no effect on parameter gradients. inst_data = np.array( - [inst + [pad_idx] * (max_len - len(inst)) for inst in insts]) + [inst + [pad_idx] * (max_len - len(inst)) for inst in insts] + ) return_list += [inst_data.astype("int64").reshape([-1, 1])] if is_label: # label weight - inst_weight = np.array([[1.] * len(inst) + [0.] * (max_len - len(inst)) - for inst in insts]) + inst_weight = np.array( + [ + [1.0] * len(inst) + [0.0] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_weight.astype("float32").reshape([-1, 1])] else: # position data - inst_pos = np.array([ - list(range(1, - len(inst) + 1)) + [0] * (max_len - len(inst)) - for inst in insts - ]) + inst_pos = np.array( + [ + list(range(1, len(inst) + 1)) + [0] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_pos.astype("int64").reshape([-1, 1])] if return_attn_bias: if is_target: # This is used to avoid attention on paddings and subsequent # words. slf_attn_bias_data = np.ones((inst_data.shape[0], max_len, max_len)) - slf_attn_bias_data = np.triu(slf_attn_bias_data, - 1).reshape([-1, 1, max_len, max_len]) - slf_attn_bias_data = np.tile(slf_attn_bias_data, - [1, n_head, 1, 1]) * [-1e9] + slf_attn_bias_data = np.triu(slf_attn_bias_data, 1).reshape( + [-1, 1, max_len, max_len] + ) + slf_attn_bias_data = np.tile( + slf_attn_bias_data, [1, n_head, 1, 1] + ) * [-1e9] else: # This is used to avoid attention on paddings. - slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] * - (max_len - len(inst)) - for inst in insts]) + slf_attn_bias_data = np.array( + [ + [0] * len(inst) + [-1e9] * (max_len - len(inst)) + for inst in insts + ] + ) slf_attn_bias_data = np.tile( slf_attn_bias_data.reshape([-1, 1, 1, max_len]), - [1, n_head, max_len, 1]) + [1, n_head, max_len, 1], + ) return_list += [slf_attn_bias_data.astype("float32")] if return_max_len: return_list += [max_len] @@ -350,22 +385,26 @@ def pad_batch_data(insts, return return_list if len(return_list) > 1 else return_list[0] -def prepare_batch_input(insts, data_input_names, src_pad_idx, trg_pad_idx, - n_head, d_model): +def prepare_batch_input( + insts, data_input_names, src_pad_idx, trg_pad_idx, n_head, d_model +): """ Put all padded data needed by training into a dict. """ src_word, src_pos, src_slf_attn_bias, src_max_len = pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False) + [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False + ) src_word = src_word.reshape(-1, src_max_len, 1) src_pos = src_pos.reshape(-1, src_max_len, 1) trg_word, trg_pos, trg_slf_attn_bias, trg_max_len = pad_batch_data( - [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True) + [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True + ) trg_word = trg_word.reshape(-1, trg_max_len, 1) trg_pos = trg_pos.reshape(-1, trg_max_len, 1) - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, trg_max_len, 1]).astype("float32") + trg_src_attn_bias = np.tile( + src_slf_attn_bias[:, :, ::src_max_len, :], [1, 1, trg_max_len, 1] + ).astype("float32") lbl_word, lbl_weight, num_token = pad_batch_data( [inst[2] for inst in insts], @@ -375,14 +414,27 @@ def prepare_batch_input(insts, data_input_names, src_pad_idx, trg_pad_idx, is_label=True, return_attn_bias=False, return_max_len=False, - return_num_token=True) + return_num_token=True, + ) data_input_dict = dict( list( - zip(data_input_names, [ - src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight - ]))) + zip( + data_input_names, + [ + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + lbl_word, + lbl_weight, + ], + ) + ) + ) return data_input_dict, np.asarray([num_token], dtype="float32") @@ -407,7 +459,7 @@ def read_multiple(reader, count, clip_last=True): if len(data) > count: inst_num_per_part = len(data) // count yield [ - data[inst_num_per_part * i:inst_num_per_part * (i + 1)] + data[inst_num_per_part * i : inst_num_per_part * (i + 1)] for i in range(count) ] @@ -423,21 +475,28 @@ def split_data(data, num_part): data = data[0] inst_num_per_part = len(data) // num_part return [ - data[inst_num_per_part * i:inst_num_per_part * (i + 1)] + data[inst_num_per_part * i : inst_num_per_part * (i + 1)] for i in range(num_part) ] -def test_context(test_program, avg_cost, train_exe, dev_count, data_input_names, - sum_cost, token_num): +def test_context( + test_program, + avg_cost, + train_exe, + dev_count, + data_input_names, + sum_cost, + token_num, +): val_data = DataReader( src_vocab_fpath=TrainTaskConfig.src_vocab_fpath, trg_vocab_fpath=TrainTaskConfig.trg_vocab_fpath, fpattern=TrainTaskConfig.val_file_pattern, token_delimiter=TrainTaskConfig.token_delimiter, use_token_batch=TrainTaskConfig.use_token_batch, - batch_size=TrainTaskConfig.batch_size * - (1 if TrainTaskConfig.use_token_batch else dev_count), + batch_size=TrainTaskConfig.batch_size + * (1 if TrainTaskConfig.use_token_batch else dev_count), pool_size=TrainTaskConfig.pool_size, sort_type=TrainTaskConfig.sort_type, start_mark=TrainTaskConfig.special_token[0], @@ -447,37 +506,47 @@ def test_context(test_program, avg_cost, train_exe, dev_count, data_input_names, max_length=ModelHyperParams.max_length - 2, clip_last_batch=False, shuffle=False, - shuffle_batch=False) + shuffle_batch=False, + ) build_strategy = fluid.BuildStrategy() strategy = fluid.ExecutionStrategy() strategy.num_threads = 1 - test_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, - main_program=test_program, - share_vars_from=train_exe, - build_strategy=build_strategy, - exec_strategy=strategy) + test_exe = fluid.ParallelExecutor( + use_cuda=TrainTaskConfig.use_gpu, + main_program=test_program, + share_vars_from=train_exe, + build_strategy=build_strategy, + exec_strategy=strategy, + ) def test(exe=test_exe): test_total_cost = 0 test_total_token = 0 test_data = read_multiple( reader=val_data.batch_generator, - count=dev_count if TrainTaskConfig.use_token_batch else 1) + count=dev_count if TrainTaskConfig.use_token_batch else 1, + ) for batch_id, data in enumerate(test_data()): feed_list = [] for place_id, data_buffer in enumerate( - split_data(data, num_part=dev_count)): + split_data(data, num_part=dev_count) + ): data_input_dict, _ = prepare_batch_input( - data_buffer, data_input_names, ModelHyperParams.eos_idx, - ModelHyperParams.eos_idx, ModelHyperParams.n_head, - ModelHyperParams.d_model) + data_buffer, + data_input_names, + ModelHyperParams.eos_idx, + ModelHyperParams.eos_idx, + ModelHyperParams.n_head, + ModelHyperParams.d_model, + ) feed_list.append(data_input_dict) - outs = exe.run(feed=feed_list, - fetch_list=[sum_cost.name, token_num.name]) + outs = exe.run( + feed=feed_list, fetch_list=[sum_cost.name, token_num.name] + ) sum_cost_val, token_num_val = np.array(outs[0]), np.array(outs[1]) test_total_cost += sum_cost_val.sum() test_total_token += token_num_val.sum() @@ -488,8 +557,17 @@ def test_context(test_program, avg_cost, train_exe, dev_count, data_input_names, return test -def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, - token_num, predict, test_program): +def train_loop( + exe, + train_progm, + dev_count, + sum_cost, + avg_cost, + lr_scheduler, + token_num, + predict, + test_program, +): # Initialize the parameters. if TrainTaskConfig.ckpt_path: lr_scheduler.current_steps = TrainTaskConfig.start_step @@ -502,8 +580,8 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, fpattern=TrainTaskConfig.train_file_pattern, token_delimiter=TrainTaskConfig.token_delimiter, use_token_batch=TrainTaskConfig.use_token_batch, - batch_size=TrainTaskConfig.batch_size * - (1 if TrainTaskConfig.use_token_batch else dev_count), + batch_size=TrainTaskConfig.batch_size + * (1 if TrainTaskConfig.use_token_batch else dev_count), pool_size=TrainTaskConfig.pool_size, sort_type=TrainTaskConfig.sort_type, shuffle=TrainTaskConfig.shuffle, @@ -513,39 +591,60 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, unk_mark=TrainTaskConfig.special_token[2], # count start and end tokens out max_length=ModelHyperParams.max_length - 2, - clip_last_batch=False) + clip_last_batch=False, + ) train_data = read_multiple( reader=train_data.batch_generator, - count=dev_count if TrainTaskConfig.use_token_batch else 1) + count=dev_count if TrainTaskConfig.use_token_batch else 1, + ) build_strategy = fluid.BuildStrategy() # Since the token number differs among devices, customize gradient scale to # use token average cost among multi-devices. and the gradient scale is # `1 / token_number` for average cost. - build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized + build_strategy.gradient_scale_strategy = ( + fluid.BuildStrategy.GradientScaleStrategy.Customized + ) strategy = fluid.ExecutionStrategy() strategy.num_threads = 1 - train_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, - loss_name=sum_cost.name, - main_program=train_progm, - build_strategy=build_strategy, - exec_strategy=strategy) + train_exe = fluid.ParallelExecutor( + use_cuda=TrainTaskConfig.use_gpu, + loss_name=sum_cost.name, + main_program=train_progm, + build_strategy=build_strategy, + exec_strategy=strategy, + ) - data_input_names = encoder_data_input_fields + decoder_data_input_fields[: - -1] + label_data_input_fields + data_input_names = ( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + ) if TrainTaskConfig.val_file_pattern is not None: - test = test_context(test_program, avg_cost, train_exe, dev_count, - data_input_names, sum_cost, token_num) + test = test_context( + test_program, + avg_cost, + train_exe, + dev_count, + data_input_names, + sum_cost, + token_num, + ) # the best cross-entropy value with label smoothing - loss_normalizer = -((1. - TrainTaskConfig.label_smooth_eps) * np.log( - (1. - TrainTaskConfig.label_smooth_eps)) + - TrainTaskConfig.label_smooth_eps * - np.log(TrainTaskConfig.label_smooth_eps / - (ModelHyperParams.trg_vocab_size - 1) + 1e-20)) + loss_normalizer = -( + (1.0 - TrainTaskConfig.label_smooth_eps) + * np.log((1.0 - TrainTaskConfig.label_smooth_eps)) + + TrainTaskConfig.label_smooth_eps + * np.log( + TrainTaskConfig.label_smooth_eps + / (ModelHyperParams.trg_vocab_size - 1) + + 1e-20 + ) + ) init = False for pass_id in range(TrainTaskConfig.pass_num): pass_start_time = time.time() @@ -560,36 +659,44 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, lr_rate = lr_scheduler.update_learning_rate() for place_id, data_buffer in enumerate( - split_data(data, num_part=dev_count)): + split_data(data, num_part=dev_count) + ): data_input_dict, num_token = prepare_batch_input( - data_buffer, data_input_names, ModelHyperParams.eos_idx, - ModelHyperParams.eos_idx, ModelHyperParams.n_head, - ModelHyperParams.d_model) + data_buffer, + data_input_names, + ModelHyperParams.eos_idx, + ModelHyperParams.eos_idx, + ModelHyperParams.n_head, + ModelHyperParams.d_model, + ) total_num_token += num_token feed_kv_pairs = list(data_input_dict.items()) if TrainTaskConfig.local: feed_kv_pairs += list( - {lr_scheduler.learning_rate.name: lr_rate}.items()) + {lr_scheduler.learning_rate.name: lr_rate}.items() + ) feed_list.append(dict(feed_kv_pairs)) if not init: for pos_enc_param_name in pos_enc_param_names: pos_enc = position_encoding_init( ModelHyperParams.max_length + 1, - ModelHyperParams.d_model) + ModelHyperParams.d_model, + ) feed_list[place_id][pos_enc_param_name] = pos_enc if not TrainTaskConfig.check_acc: for feed_dict in feed_list: - feed_dict[sum_cost.name + "@GRAD"] = 1. / total_num_token + feed_dict[sum_cost.name + "@GRAD"] = 1.0 / total_num_token else: b = 100 * TrainTaskConfig.batch_size a = np.asarray([b], dtype="float32") for feed_dict in feed_list: - feed_dict[sum_cost.name + "@GRAD"] = 1. / a + feed_dict[sum_cost.name + "@GRAD"] = 1.0 / a - outs = train_exe.run(fetch_list=[sum_cost.name, token_num.name], - feed=feed_list) + outs = train_exe.run( + fetch_list=[sum_cost.name, token_num.name], feed=feed_list + ) sum_cost_val, token_num_val = np.array(outs[0]), np.array(outs[1]) total_sum_cost = sum_cost_val.sum() @@ -603,10 +710,10 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, val_avg_cost, val_ppl = test() print("[%f]" % val_avg_cost) else: - assert (False) + assert False -#import transformer_reader as reader +# import transformer_reader as reader class SortType(object): GLOBAL = 'global' POOL = 'pool' @@ -614,7 +721,6 @@ class SortType(object): class Converter(object): - def __init__(self, vocab, beg, end, unk, delimiter): self._vocab = vocab self._beg = beg @@ -623,14 +729,17 @@ class Converter(object): self._delimiter = delimiter def __call__(self, sentence): - return [self._beg] + [ - self._vocab.get(w, self._unk) - for w in sentence.split(self._delimiter) - ] + [self._end] + return ( + [self._beg] + + [ + self._vocab.get(w, self._unk) + for w in sentence.split(self._delimiter) + ] + + [self._end] + ) class ComposedConverter(object): - def __init__(self, converters): self._converters = converters @@ -642,7 +751,6 @@ class ComposedConverter(object): class SentenceBatchCreator(object): - def __init__(self, batch_size): self.batch = [] self._batch_size = batch_size @@ -656,7 +764,6 @@ class SentenceBatchCreator(object): class TokenBatchCreator(object): - def __init__(self, batch_size): self.batch = [] self.max_len = -1 @@ -676,7 +783,6 @@ class TokenBatchCreator(object): class SampleInfo(object): - def __init__(self, i, max_len, min_len): self.i = i self.min_len = min_len @@ -684,7 +790,6 @@ class SampleInfo(object): class MinMaxFilter(object): - def __init__(self, max_len, min_len, underlying_creator): self._min_len = min_len self._max_len = max_len @@ -774,26 +879,28 @@ class DataReader(object): :type seed: int """ - def __init__(self, - src_vocab_fpath, - trg_vocab_fpath, - fpattern, - batch_size, - pool_size, - sort_type=SortType.GLOBAL, - clip_last_batch=True, - tar_fname=None, - min_length=0, - max_length=100, - shuffle=True, - shuffle_batch=False, - use_token_batch=False, - field_delimiter="\t", - token_delimiter=" ", - start_mark="", - end_mark="", - unk_mark="", - seed=0): + def __init__( + self, + src_vocab_fpath, + trg_vocab_fpath, + fpattern, + batch_size, + pool_size, + sort_type=SortType.GLOBAL, + clip_last_batch=True, + tar_fname=None, + min_length=0, + max_length=100, + shuffle=True, + shuffle_batch=False, + use_token_batch=False, + field_delimiter="\t", + token_delimiter=" ", + start_mark="", + end_mark="", + unk_mark="", + seed=0, + ): self._src_vocab = self.load_dict(src_vocab_fpath) self._only_src = True if trg_vocab_fpath is not None: @@ -810,26 +917,33 @@ class DataReader(object): self._max_length = max_length self._field_delimiter = field_delimiter self._token_delimiter = token_delimiter - self.load_src_trg_ids(end_mark, fpattern, start_mark, tar_fname, - unk_mark) + self.load_src_trg_ids( + end_mark, fpattern, start_mark, tar_fname, unk_mark + ) self._random = random.Random(x=seed) - def load_src_trg_ids(self, end_mark, fpattern, start_mark, tar_fname, - unk_mark): + def load_src_trg_ids( + self, end_mark, fpattern, start_mark, tar_fname, unk_mark + ): converters = [ - Converter(vocab=self._src_vocab, - beg=self._src_vocab[start_mark], - end=self._src_vocab[end_mark], - unk=self._src_vocab[unk_mark], - delimiter=self._token_delimiter) + Converter( + vocab=self._src_vocab, + beg=self._src_vocab[start_mark], + end=self._src_vocab[end_mark], + unk=self._src_vocab[unk_mark], + delimiter=self._token_delimiter, + ) ] if not self._only_src: converters.append( - Converter(vocab=self._trg_vocab, - beg=self._trg_vocab[start_mark], - end=self._trg_vocab[end_mark], - unk=self._trg_vocab[unk_mark], - delimiter=self._token_delimiter)) + Converter( + vocab=self._trg_vocab, + beg=self._trg_vocab[start_mark], + end=self._trg_vocab[end_mark], + unk=self._trg_vocab[unk_mark], + delimiter=self._token_delimiter, + ) + ) converters = ComposedConverter(converters) @@ -857,9 +971,9 @@ class DataReader(object): for line in f.extractfile(tar_fname): line = line.decode() fields = line.strip("\n").split(self._field_delimiter) - if (not self._only_src - and len(fields) == 2) or (self._only_src - and len(fields) == 1): + if (not self._only_src and len(fields) == 2) or ( + self._only_src and len(fields) == 1 + ): yield fields else: for fpath in fpaths: @@ -870,9 +984,9 @@ class DataReader(object): for line in f: line = line.decode() fields = line.strip("\n").split(self._field_delimiter) - if (not self._only_src - and len(fields) == 2) or (self._only_src - and len(fields) == 1): + if (not self._only_src and len(fields) == 2) or ( + self._only_src and len(fields) == 1 + ): yield fields @staticmethod @@ -890,9 +1004,9 @@ class DataReader(object): def batch_generator(self): # global sort or global shuffle if self._sort_type == SortType.GLOBAL: - infos = sorted(self._sample_infos, - key=lambda x: x.max_len, - reverse=True) + infos = sorted( + self._sample_infos, key=lambda x: x.max_len, reverse=True + ) else: if self._shuffle: infos = self._sample_infos @@ -902,16 +1016,20 @@ class DataReader(object): if self._sort_type == SortType.POOL: for i in range(0, len(infos), self._pool_size): - infos[i:i + self._pool_size] = sorted( - infos[i:i + self._pool_size], key=lambda x: x.max_len) + infos[i : i + self._pool_size] = sorted( + infos[i : i + self._pool_size], key=lambda x: x.max_len + ) # concat batch batches = [] - batch_creator = TokenBatchCreator( - self._batch_size - ) if self._use_token_batch else SentenceBatchCreator(self._batch_size) - batch_creator = MinMaxFilter(self._max_length, self._min_length, - batch_creator) + batch_creator = ( + TokenBatchCreator(self._batch_size) + if self._use_token_batch + else SentenceBatchCreator(self._batch_size) + ) + batch_creator = MinMaxFilter( + self._max_length, self._min_length, batch_creator + ) for info in infos: batch = batch_creator.append(info) @@ -930,34 +1048,49 @@ class DataReader(object): if self._only_src: yield [[self._src_seq_ids[idx]] for idx in batch_ids] else: - yield [(self._src_seq_ids[idx], self._trg_seq_ids[idx][:-1], - self._trg_seq_ids[idx][1:]) for idx in batch_ids] + yield [ + ( + self._src_seq_ids[idx], + self._trg_seq_ids[idx][:-1], + self._trg_seq_ids[idx][1:], + ) + for idx in batch_ids + ] -#from transformer_model import transformer +# from transformer_model import transformer def position_encoding_init(n_position, d_pos_vec): """ Generate the initial values for the sinusoid position encoding table. """ - position_enc = np.array([[ - pos / np.power(10000, 2 * (j // 2) / d_pos_vec) - for j in range(d_pos_vec) - ] if pos != 0 else np.zeros(d_pos_vec) for pos in range(n_position)]) + position_enc = np.array( + [ + [ + pos / np.power(10000, 2 * (j // 2) / d_pos_vec) + for j in range(d_pos_vec) + ] + if pos != 0 + else np.zeros(d_pos_vec) + for pos in range(n_position) + ] + ) position_enc[1:, 0::2] = np.sin(position_enc[1:, 0::2]) # dim 2i position_enc[1:, 1::2] = np.cos(position_enc[1:, 1::2]) # dim 2i+1 return position_enc.astype("float32") -def multi_head_attention(queries, - keys, - values, - attn_bias, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - cache=None): +def multi_head_attention( + queries, + keys, + values, + attn_bias, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, + cache=None, +): """ Multi-Head Attention. Note that attn_bias is added to the logit before computing softmax activiation to mask certain selected positions so that @@ -965,27 +1098,34 @@ def multi_head_attention(queries, """ if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): raise ValueError( - "Inputs: queries, keys and values should all be 3-D tensors.") + "Inputs: queries, keys and values should all be 3-D tensors." + ) def __compute_qkv(queries, keys, values, n_head, d_key, d_value): """ Add linear projection to queries, keys, and values. """ - q = layers.fc(input=queries, - size=d_key * n_head, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) - k = layers.fc(input=keys, - size=d_key * n_head, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) - v = layers.fc(input=values, - size=d_value * n_head, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) + q = layers.fc( + input=queries, + size=d_key * n_head, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) + k = layers.fc( + input=keys, + size=d_key * n_head, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) + v = layers.fc( + input=values, + size=d_value * n_head, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) return q, k, v def __split_heads(x, n_head): @@ -1001,8 +1141,9 @@ def multi_head_attention(queries, hidden_size = x.shape[-1] # The value 0 in shape attr means copying the corresponding dimension # size of the input as the output dimension size. - reshaped = layers.reshape(x=x, - shape=[0, 0, n_head, hidden_size // n_head]) + reshaped = layers.reshape( + x=x, shape=[0, 0, n_head, hidden_size // n_head] + ) # permute the dimensions into: # [batch_size, n_head, max_sequence_len, hidden_size_per_head] @@ -1013,7 +1154,8 @@ def multi_head_attention(queries, Transpose and then reshape the last two dimensions of input tensor x so that it becomes one dimension, which is reverse to __split_heads. """ - if len(x.shape) == 3: return x + if len(x.shape) == 3: + return x if len(x.shape) != 4: raise ValueError("Input(x) should be a 4-D Tensor.") @@ -1022,7 +1164,8 @@ def multi_head_attention(queries, # size of the input as the output dimension size. return layers.reshape( x=trans_x, - shape=list(map(int, [0, 0, trans_x.shape[2] * trans_x.shape[3]]))) + shape=list(map(int, [0, 0, trans_x.shape[2] * trans_x.shape[3]])), + ) def scaled_dot_product_attention(q, k, v, attn_bias, d_model, dropout_rate): """ @@ -1034,10 +1177,12 @@ def multi_head_attention(queries, product += attn_bias weights = layers.softmax(product) if dropout_rate: - weights = layers.dropout(weights, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) + weights = layers.dropout( + weights, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) out = layers.matmul(weights, v) return out @@ -1051,17 +1196,20 @@ def multi_head_attention(queries, k = __split_heads(k, n_head) v = __split_heads(v, n_head) - ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_model, - dropout_rate) + ctx_multiheads = scaled_dot_product_attention( + q, k, v, attn_bias, d_model, dropout_rate + ) out = __combine_heads(ctx_multiheads) # Project back to the model size. - proj_out = layers.fc(input=out, - size=d_model, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) + proj_out = layers.fc( + input=out, + size=d_model, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) return proj_out @@ -1071,21 +1219,25 @@ def positionwise_feed_forward(x, d_inner_hid, d_hid): This module consists of two linear transformations with a ReLU activation in between, which is applied to each position separately and identically. """ - hidden = layers.fc(input=x, - size=d_inner_hid, - num_flatten_dims=2, - act="relu", - param_attr=const_para_attr, - bias_attr=const_bias_attr) - out = layers.fc(input=hidden, - size=d_hid, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) + hidden = layers.fc( + input=x, + size=d_inner_hid, + num_flatten_dims=2, + act="relu", + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) + out = layers.fc( + input=hidden, + size=d_hid, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) return out -def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.): +def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.0): """ Add residual connection, layer normalization and droput to the out tensor optionally according to the value of process_cmd. @@ -1096,16 +1248,20 @@ def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.): if cmd == "a": # add residual connection out = out + prev_out if prev_out else out elif cmd == "n": # add layer normalization - out = layers.layer_norm(out, - begin_norm_axis=len(out.shape) - 1, - param_attr=fluid.initializer.Constant(1.), - bias_attr=fluid.initializer.Constant(0.)) + out = layers.layer_norm( + out, + begin_norm_axis=len(out.shape) - 1, + param_attr=fluid.initializer.Constant(1.0), + bias_attr=fluid.initializer.Constant(0.0), + ) elif cmd == "d": # add dropout if dropout_rate: - out = layers.dropout(out, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) + out = layers.dropout( + out, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) return out @@ -1113,14 +1269,16 @@ pre_process_layer = partial(pre_post_process_layer, None) post_process_layer = pre_post_process_layer -def prepare_encoder(src_word, - src_pos, - src_vocab_size, - src_emb_dim, - src_max_len, - dropout_rate=0., - word_emb_param_name=None, - pos_enc_param_name=None): +def prepare_encoder( + src_word, + src_pos, + src_vocab_size, + src_emb_dim, + src_max_len, + dropout_rate=0.0, + word_emb_param_name=None, + pos_enc_param_name=None, +): """Add word embeddings and position encodings. The output tensor has a shape of: [batch_size, max_src_length_in_batch, d_model]. @@ -1132,14 +1290,18 @@ def prepare_encoder(src_word, size=[src_vocab_size, src_emb_dim], param_attr=fluid.ParamAttr( name=word_emb_param_name, - initializer=fluid.initializer.ConstantInitializer(0.001))) + initializer=fluid.initializer.ConstantInitializer(0.001), + ), + ) else: src_word_emb = layers.embedding( src_word, size=[src_vocab_size, src_emb_dim], - param_attr=fluid.ParamAttr(name=word_emb_param_name, - initializer=fluid.initializer.Normal( - 0., src_emb_dim**-0.5))) + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0.0, src_emb_dim**-0.5), + ), + ) src_word_emb = layers.scale(x=src_word_emb, scale=src_emb_dim**0.5) src_pos_enc = layers.embedding( @@ -1148,76 +1310,109 @@ def prepare_encoder(src_word, param_attr=fluid.ParamAttr( name=pos_enc_param_name, trainable=False, - initializer=fluid.initializer.ConstantInitializer(0.001))) + initializer=fluid.initializer.ConstantInitializer(0.001), + ), + ) src_pos_enc.stop_gradient = True enc_input = src_word_emb + src_pos_enc - return layers.dropout(enc_input, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) if dropout_rate else enc_input - - -prepare_encoder = partial(prepare_encoder, - pos_enc_param_name=pos_enc_param_names[0]) -prepare_decoder = partial(prepare_encoder, - pos_enc_param_name=pos_enc_param_names[1]) - - -def encoder_layer(enc_input, - attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0.): + return ( + layers.dropout( + enc_input, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) + if dropout_rate + else enc_input + ) + + +prepare_encoder = partial( + prepare_encoder, pos_enc_param_name=pos_enc_param_names[0] +) +prepare_decoder = partial( + prepare_encoder, pos_enc_param_name=pos_enc_param_names[1] +) + + +def encoder_layer( + enc_input, + attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): """The encoder layers that can be stacked to form a deep encoder. This module consits of a multi-head (self) attention followed by position-wise feed-forward networks and both the two components companied with the post_process_layer to add residual connection, layer normalization and droput. """ - attn_output = multi_head_attention(enc_input, enc_input, enc_input, - attn_bias, d_key, d_value, d_model, - n_head, dropout_rate) - attn_output = post_process_layer(enc_input, attn_output, "dan", - dropout_rate) + attn_output = multi_head_attention( + enc_input, + enc_input, + enc_input, + attn_bias, + d_key, + d_value, + d_model, + n_head, + dropout_rate, + ) + attn_output = post_process_layer( + enc_input, attn_output, "dan", dropout_rate + ) ffd_output = positionwise_feed_forward(attn_output, d_inner_hid, d_model) return post_process_layer(attn_output, ffd_output, "dan", dropout_rate) -def encoder(enc_input, +def encoder( + enc_input, + attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): + """ + The encoder is composed of a stack of identical layers returned by calling + encoder_layer. + """ + for i in range(n_layer): + enc_output = encoder_layer( + enc_input, attn_bias, - n_layer, n_head, d_key, d_value, d_model, d_inner_hid, - dropout_rate=0.): - """ - The encoder is composed of a stack of identical layers returned by calling - encoder_layer. - """ - for i in range(n_layer): - enc_output = encoder_layer(enc_input, attn_bias, n_head, d_key, d_value, - d_model, d_inner_hid, dropout_rate) + dropout_rate, + ) enc_input = enc_output return enc_output -def decoder_layer(dec_input, - enc_output, - slf_attn_bias, - dec_enc_attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0., - cache=None): - """ The layer to be stacked in decoder part. +def decoder_layer( + dec_input, + enc_output, + slf_attn_bias, + dec_enc_attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, + cache=None, +): + """The layer to be stacked in decoder part. The structure of this module is similar to that in the encoder part except a multi-head attention is added to implement encoder-decoder attention. """ @@ -1270,18 +1465,20 @@ def decoder_layer(dec_input, return dec_output -def decoder(dec_input, - enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0., - caches=None): +def decoder( + dec_input, + enc_output, + dec_slf_attn_bias, + dec_enc_attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, + caches=None, +): """ The decoder is composed of a stack of identical decoder_layer layers. """ @@ -1290,17 +1487,19 @@ def decoder(dec_input, if caches is not None: cache = caches[i] - dec_output = decoder_layer(dec_input, - enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - cache=cache) + dec_output = decoder_layer( + dec_input, + enc_output, + dec_slf_attn_bias, + dec_enc_attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + cache=cache, + ) dec_input = dec_output return dec_output @@ -1311,12 +1510,15 @@ def make_all_inputs(input_fields): """ inputs = [] for input_field in input_fields: - input_var = layers.data(name=input_field, - shape=input_descs[input_field][0], - dtype=input_descs[input_field][1], - lod_level=input_descs[input_field][2] - if len(input_descs[input_field]) == 3 else 0, - append_batch_size=False) + input_var = layers.data( + name=input_field, + shape=input_descs[input_field][0], + dtype=input_descs[input_field][1], + lod_level=input_descs[input_field][2] + if len(input_descs[input_field]) == 3 + else 0, + append_batch_size=False, + ) inputs.append(input_var) return inputs @@ -1336,9 +1538,9 @@ def transformer( label_smooth_eps, ): if weight_sharing: - assert src_vocab_size == src_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) + assert ( + src_vocab_size == src_vocab_size + ), "Vocabularies in source and target should be same for weight sharing." enc_inputs = make_all_inputs(encoder_data_input_fields) enc_output = wrap_encoder( @@ -1376,14 +1578,16 @@ def transformer( # cancel padding index in calculating the loss. label, weights = make_all_inputs(label_data_input_fields) if label_smooth_eps: - label = layers.label_smooth(label=layers.one_hot(input=label, - depth=trg_vocab_size), - epsilon=label_smooth_eps) + label = layers.label_smooth( + label=layers.one_hot(input=label, depth=trg_vocab_size), + epsilon=label_smooth_eps, + ) cost = layers.softmax_with_cross_entropy( logits=layers.reshape(predict, shape=[-1, trg_vocab_size]), label=label, - soft_label=True if label_smooth_eps else False) + soft_label=True if label_smooth_eps else False, + ) weighted_cost = cost * weights sum_cost = layers.reduce_sum(weighted_cost) token_num = layers.reduce_sum(weights) @@ -1392,95 +1596,122 @@ def transformer( return sum_cost, avg_cost, predict, token_num -def wrap_encoder(src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - weight_sharing, - enc_inputs=None): +def wrap_encoder( + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + weight_sharing, + enc_inputs=None, +): """ The wrapper assembles together all needed layers for the encoder. """ if enc_inputs is None: # This is used to implement independent encoder program in inference. - src_word, src_pos, src_slf_attn_bias = \ - make_all_inputs(encoder_data_input_fields) + src_word, src_pos, src_slf_attn_bias = make_all_inputs( + encoder_data_input_fields + ) else: - src_word, src_pos, src_slf_attn_bias = \ - enc_inputs - enc_input = prepare_encoder(src_word, - src_pos, - src_vocab_size, - d_model, - max_length, - dropout_rate, - word_emb_param_name=word_emb_param_names[0]) - enc_output = encoder(enc_input, src_slf_attn_bias, n_layer, n_head, d_key, - d_value, d_model, d_inner_hid, dropout_rate) + src_word, src_pos, src_slf_attn_bias = enc_inputs + enc_input = prepare_encoder( + src_word, + src_pos, + src_vocab_size, + d_model, + max_length, + dropout_rate, + word_emb_param_name=word_emb_param_names[0], + ) + enc_output = encoder( + enc_input, + src_slf_attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + ) return enc_output -def wrap_decoder(trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - weight_sharing, - dec_inputs=None, - enc_output=None, - caches=None): +def wrap_decoder( + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + weight_sharing, + dec_inputs=None, + enc_output=None, + caches=None, +): """ The wrapper assembles together all needed layers for the decoder. """ if dec_inputs is None: # This is used to implement independent decoder program in inference. - trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias, \ - enc_output = make_all_inputs( - decoder_data_input_fields) + ( + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + enc_output, + ) = make_all_inputs(decoder_data_input_fields) else: trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias = dec_inputs - dec_input = prepare_decoder(trg_word, - trg_pos, - trg_vocab_size, - d_model, - max_length, - dropout_rate, - word_emb_param_name=word_emb_param_names[0] - if weight_sharing else word_emb_param_names[1]) - dec_output = decoder(dec_input, - enc_output, - trg_slf_attn_bias, - trg_src_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - caches=caches) + dec_input = prepare_decoder( + trg_word, + trg_pos, + trg_vocab_size, + d_model, + max_length, + dropout_rate, + word_emb_param_name=word_emb_param_names[0] + if weight_sharing + else word_emb_param_names[1], + ) + dec_output = decoder( + dec_input, + enc_output, + trg_slf_attn_bias, + trg_src_attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + caches=caches, + ) # Return logits for training and probs for inference. if weight_sharing: - predict = layers.matmul(x=dec_output, - y=fluid.framework._get_var( - word_emb_param_names[0]), - transpose_y=True) + predict = layers.matmul( + x=dec_output, + y=fluid.framework._get_var(word_emb_param_names[0]), + transpose_y=True, + ) else: - predict = layers.fc(input=dec_output, - size=trg_vocab_size, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) + predict = layers.fc( + input=dec_output, + size=trg_vocab_size, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) if dec_inputs is None: predict = layers.softmax(predict) return predict @@ -1506,86 +1737,108 @@ def fast_decode( Use beam search to decode. Caches will be used to store states of history steps which can make the decoding faster. """ - enc_output = wrap_encoder(src_vocab_size, max_in_len, n_layer, n_head, - d_key, d_value, d_model, d_inner_hid, - dropout_rate, weight_sharing) - start_tokens, init_scores, trg_src_attn_bias = \ - make_all_inputs(fast_decoder_data_input_fields ) + enc_output = wrap_encoder( + src_vocab_size, + max_in_len, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + weight_sharing, + ) + start_tokens, init_scores, trg_src_attn_bias = make_all_inputs( + fast_decoder_data_input_fields + ) def beam_search(): - max_len = layers.fill_constant(shape=[1], - dtype=start_tokens.dtype, - value=max_out_len) - step_idx = layers.fill_constant(shape=[1], - dtype=start_tokens.dtype, - value=0) + max_len = layers.fill_constant( + shape=[1], dtype=start_tokens.dtype, value=max_out_len + ) + step_idx = layers.fill_constant( + shape=[1], dtype=start_tokens.dtype, value=0 + ) cond = layers.less_than(x=step_idx, y=max_len) while_op = layers.While(cond) # array states will be stored for each step. - ids = layers.array_write(layers.reshape(start_tokens, (-1, 1)), - step_idx) + ids = layers.array_write( + layers.reshape(start_tokens, (-1, 1)), step_idx + ) scores = layers.array_write(init_scores, step_idx) # cell states will be overwrited at each step. # caches contains states of history steps to reduce redundant # computation in decoder. - caches = [{ - "k": - layers.fill_constant_batch_size_like(input=start_tokens, - shape=[-1, 0, d_model], - dtype=enc_output.dtype, - value=0), - "v": - layers.fill_constant_batch_size_like(input=start_tokens, - shape=[-1, 0, d_model], - dtype=enc_output.dtype, - value=0) - } for i in range(n_layer)] + caches = [ + { + "k": layers.fill_constant_batch_size_like( + input=start_tokens, + shape=[-1, 0, d_model], + dtype=enc_output.dtype, + value=0, + ), + "v": layers.fill_constant_batch_size_like( + input=start_tokens, + shape=[-1, 0, d_model], + dtype=enc_output.dtype, + value=0, + ), + } + for i in range(n_layer) + ] with while_op.block(): pre_ids = layers.array_read(array=ids, i=step_idx) pre_ids = layers.reshape(pre_ids, (-1, 1, 1)) pre_scores = layers.array_read(array=scores, i=step_idx) # sequence_expand can gather sequences according to lod thus can be # used in beam search to sift states corresponding to selected ids. - pre_src_attn_bias = layers.sequence_expand(x=trg_src_attn_bias, - y=pre_scores) + pre_src_attn_bias = layers.sequence_expand( + x=trg_src_attn_bias, y=pre_scores + ) pre_enc_output = layers.sequence_expand(x=enc_output, y=pre_scores) - pre_caches = [{ - "k": - layers.sequence_expand(x=cache["k"], y=pre_scores), - "v": - layers.sequence_expand(x=cache["v"], y=pre_scores), - } for cache in caches] + pre_caches = [ + { + "k": layers.sequence_expand(x=cache["k"], y=pre_scores), + "v": layers.sequence_expand(x=cache["v"], y=pre_scores), + } + for cache in caches + ] pre_pos = layers.elementwise_mul( x=layers.fill_constant_batch_size_like( - input= - pre_enc_output, # can't use pre_ids here since it has lod + input=pre_enc_output, # can't use pre_ids here since it has lod value=1, shape=[-1, 1, 1], - dtype=pre_ids.dtype), + dtype=pre_ids.dtype, + ), y=layers.increment(x=step_idx, value=1.0, in_place=False), - axis=0) - logits = wrap_decoder(trg_vocab_size, - max_in_len, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - weight_sharing, - dec_inputs=(pre_ids, pre_pos, None, - pre_src_attn_bias), - enc_output=pre_enc_output, - caches=pre_caches) + axis=0, + ) + logits = wrap_decoder( + trg_vocab_size, + max_in_len, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + weight_sharing, + dec_inputs=(pre_ids, pre_pos, None, pre_src_attn_bias), + enc_output=pre_enc_output, + caches=pre_caches, + ) logits = layers.reshape(logits, (-1, trg_vocab_size)) topk_scores, topk_indices = layers.topk( - input=layers.softmax(logits), k=beam_size) - accu_scores = layers.elementwise_add(x=layers.log(topk_scores), - y=layers.reshape(pre_scores, - shape=[-1]), - axis=0) + input=layers.softmax(logits), k=beam_size + ) + accu_scores = layers.elementwise_add( + x=layers.log(topk_scores), + y=layers.reshape(pre_scores, shape=[-1]), + axis=0, + ) # beam_search op uses lod to distinguish branches. topk_indices = layers.lod_reset(topk_indices, pre_ids) selected_ids, selected_scores = layers.beam_search( @@ -1594,7 +1847,8 @@ def fast_decode( ids=topk_indices, scores=accu_scores, beam_size=beam_size, - end_id=eos_idx) + end_id=eos_idx, + ) layers.increment(x=step_idx, value=1.0, in_place=True) # update states @@ -1610,7 +1864,8 @@ def fast_decode( layers.logical_and(x=length_cond, y=finish_cond, out=cond) finished_ids, finished_scores = layers.beam_search_decode( - ids, scores, beam_size=beam_size, end_id=eos_idx) + ids, scores, beam_size=beam_size, end_id=eos_idx + ) return finished_ids, finished_scores finished_ids, finished_scores = beam_search() @@ -1619,16 +1874,25 @@ def fast_decode( def get_model(is_dist, is_async): sum_cost, avg_cost, predict, token_num = transformer( - ModelHyperParams.src_vocab_size, ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, ModelHyperParams.n_layer, - ModelHyperParams.n_head, ModelHyperParams.d_key, - ModelHyperParams.d_value, ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, ModelHyperParams.dropout, - ModelHyperParams.weight_sharing, TrainTaskConfig.label_smooth_eps) - - local_lr_scheduler = LearningRateScheduler(ModelHyperParams.d_model, - TrainTaskConfig.warmup_steps, - TrainTaskConfig.learning_rate) + ModelHyperParams.src_vocab_size, + ModelHyperParams.trg_vocab_size, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.dropout, + ModelHyperParams.weight_sharing, + TrainTaskConfig.label_smooth_eps, + ) + + local_lr_scheduler = LearningRateScheduler( + ModelHyperParams.d_model, + TrainTaskConfig.warmup_steps, + TrainTaskConfig.learning_rate, + ) # Context to do validation. test_program = fluid.default_main_program().clone(for_test=True) @@ -1637,24 +1901,33 @@ def get_model(is_dist, is_async): learning_rate=local_lr_scheduler.learning_rate, beta1=TrainTaskConfig.beta1, beta2=TrainTaskConfig.beta2, - epsilon=TrainTaskConfig.eps) + epsilon=TrainTaskConfig.eps, + ) optimizer.minimize(sum_cost) elif is_async: optimizer = fluid.optimizer.SGD(0.003) optimizer.minimize(sum_cost) else: - lr_decay = fluid.layers\ - .learning_rate_scheduler\ - .noam_decay(ModelHyperParams.d_model, - TrainTaskConfig.warmup_steps) - - optimizer = fluid.optimizer.Adam(learning_rate=lr_decay, - beta1=TrainTaskConfig.beta1, - beta2=TrainTaskConfig.beta2, - epsilon=TrainTaskConfig.eps) + lr_decay = fluid.layers.learning_rate_scheduler.noam_decay( + ModelHyperParams.d_model, TrainTaskConfig.warmup_steps + ) + + optimizer = fluid.optimizer.Adam( + learning_rate=lr_decay, + beta1=TrainTaskConfig.beta1, + beta2=TrainTaskConfig.beta2, + epsilon=TrainTaskConfig.eps, + ) optimizer.minimize(sum_cost) - return sum_cost, avg_cost, predict, token_num, local_lr_scheduler, test_program + return ( + sum_cost, + avg_cost, + predict, + token_num, + local_lr_scheduler, + test_program, + ) def update_args(): @@ -1662,24 +1935,33 @@ def update_args(): trg_dict = DataReader.load_dict(TrainTaskConfig.trg_vocab_fpath) dict_args = [ "src_vocab_size", - str(len(src_dict)), "trg_vocab_size", - str(len(trg_dict)), "bos_idx", - str(src_dict[TrainTaskConfig.special_token[0]]), "eos_idx", - str(src_dict[TrainTaskConfig.special_token[1]]), "unk_idx", - str(src_dict[TrainTaskConfig.special_token[2]]) + str(len(src_dict)), + "trg_vocab_size", + str(len(trg_dict)), + "bos_idx", + str(src_dict[TrainTaskConfig.special_token[0]]), + "eos_idx", + str(src_dict[TrainTaskConfig.special_token[1]]), + "unk_idx", + str(src_dict[TrainTaskConfig.special_token[2]]), ] merge_cfg_from_list(dict_args, [TrainTaskConfig, ModelHyperParams]) class DistTransformer2x2(TestDistRunnerBase): - def run_pserver(self, args): get_model(True, not args.sync_mode) - t = self.get_transpiler(args.trainer_id, fluid.default_main_program(), - args.endpoints, args.trainers, args.sync_mode) + t = self.get_transpiler( + args.trainer_id, + fluid.default_main_program(), + args.endpoints, + args.trainers, + args.sync_mode, + ) pserver_prog = t.get_pserver_program(args.current_endpoint) - startup_prog = t.get_startup_program(args.current_endpoint, - pserver_prog) + startup_prog = t.get_startup_program( + args.current_endpoint, pserver_prog + ) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -1688,18 +1970,31 @@ class DistTransformer2x2(TestDistRunnerBase): def run_trainer(self, args): TrainTaskConfig.use_gpu = args.use_cuda - sum_cost, avg_cost, predict, token_num, local_lr_scheduler, test_program = get_model( - args.is_dist, not args.sync_mode) + ( + sum_cost, + avg_cost, + predict, + token_num, + local_lr_scheduler, + test_program, + ) = get_model(args.is_dist, not args.sync_mode) if args.is_dist: - t = self.get_transpiler(args.trainer_id, - fluid.default_main_program(), - args.endpoints, args.trainers, - args.sync_mode) + t = self.get_transpiler( + args.trainer_id, + fluid.default_main_program(), + args.endpoints, + args.trainers, + args.sync_mode, + ) trainer_prog = t.get_trainer_program() TrainTaskConfig.batch_size = 10 - TrainTaskConfig.train_file_pattern = TrainTaskConfig.data_path + "train.tok.clean.bpe.32000.en-de.train_{}".format( - args.trainer_id) + TrainTaskConfig.train_file_pattern = ( + TrainTaskConfig.data_path + + "train.tok.clean.bpe.32000.en-de.train_{}".format( + args.trainer_id + ) + ) else: TrainTaskConfig.batch_size = 20 trainer_prog = fluid.default_main_program() @@ -1713,8 +2008,17 @@ class DistTransformer2x2(TestDistRunnerBase): TrainTaskConfig.local = not args.is_dist - train_loop(startup_exe, trainer_prog, 1, sum_cost, avg_cost, - local_lr_scheduler, token_num, predict, test_program) + train_loop( + startup_exe, + trainer_prog, + 1, + sum_cost, + avg_cost, + local_lr_scheduler, + token_num, + predict, + test_program, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_word2vec.py b/python/paddle/fluid/tests/unittests/dist_word2vec.py index ca322d30cdc..0941dabcc5e 100644 --- a/python/paddle/fluid/tests/unittests/dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/dist_word2vec.py @@ -28,7 +28,6 @@ fluid.default_main_program().random_seed = 1 class TestDistWord2vec2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): BATCH_SIZE = batch_size @@ -40,7 +39,9 @@ class TestDistWord2vec2x2(TestDistRunnerBase): is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr( name='shared_w', - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1), + ), + ) embed_second = fluid.layers.embedding( input=words[1], size=[dict_size, EMBED_SIZE], @@ -48,7 +49,9 @@ class TestDistWord2vec2x2(TestDistRunnerBase): is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr( name='shared_w', - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1), + ), + ) embed_third = fluid.layers.embedding( input=words[2], size=[dict_size, EMBED_SIZE], @@ -56,7 +59,9 @@ class TestDistWord2vec2x2(TestDistRunnerBase): is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr( name='shared_w', - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1), + ), + ) embed_forth = fluid.layers.embedding( input=words[3], size=[dict_size, EMBED_SIZE], @@ -64,25 +69,33 @@ class TestDistWord2vec2x2(TestDistRunnerBase): is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr( name='shared_w', - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1), + ), + ) concat_embed = fluid.layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], - axis=1) + axis=1, + ) hidden1 = fluid.layers.fc( input=concat_embed, size=HIDDEN_SIZE, act='sigmoid', param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1) + ), + ) predict_word = fluid.layers.fc( input=hidden1, size=dict_size, act='softmax', param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.1))) - cost = fluid.layers.cross_entropy(input=predict_word, - label=words[4]) + initializer=fluid.initializer.Constant(value=0.1) + ), + ) + cost = fluid.layers.cross_entropy( + input=predict_word, label=words[4] + ) avg_cost = paddle.mean(cost) return avg_cost, predict_word @@ -90,30 +103,41 @@ class TestDistWord2vec2x2(TestDistRunnerBase): dict_size = len(word_dict) first_word = fluid.layers.data(name='firstw', shape=[1], dtype='int64') - second_word = fluid.layers.data(name='secondw', - shape=[1], - dtype='int64') + second_word = fluid.layers.data( + name='secondw', shape=[1], dtype='int64' + ) third_word = fluid.layers.data(name='thirdw', shape=[1], dtype='int64') forth_word = fluid.layers.data(name='forthw', shape=[1], dtype='int64') next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64') avg_cost, predict_word = __network__( - [first_word, second_word, third_word, forth_word, next_word]) + [first_word, second_word, third_word, forth_word, next_word] + ) inference_program = paddle.fluid.default_main_program().clone() sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) sgd_optimizer.minimize(avg_cost) - train_reader = paddle.batch(paddle.dataset.imikolov.train(word_dict, N), - BATCH_SIZE) - test_reader = paddle.batch(paddle.dataset.imikolov.test(word_dict, N), - BATCH_SIZE) + train_reader = paddle.batch( + paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE + ) + test_reader = paddle.batch( + paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE + ) - return inference_program, avg_cost, train_reader, test_reader, None, predict_word + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + None, + predict_word, + ) if __name__ == "__main__": import os + os.environ['CPU_NUM'] = '1' os.environ['USE_CUDA'] = "FALSE" runtime_main(TestDistWord2vec2x2) diff --git a/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py b/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py index 230272a1cff..b2ef0976d62 100644 --- a/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py @@ -73,7 +73,6 @@ def prune_fwd_bwd_ops(program, start_idx): class GradClipDecorator(ClipGradBase): - def __init__(self, clip, clip_after_allreduce): self.clip = clip self.clip_after_allreduce = clip_after_allreduce @@ -89,17 +88,18 @@ class GradClipDecorator(ClipGradBase): scale = 1.0 / world_size # scale = 1.0 for p, g in params_grads: - block.append_op(type='c_allreduce_sum', - inputs={'X': [g]}, - outputs={'Out': [g]}, - attrs={ - 'ring_id': 0, - 'use_calc_stream': True - }) - block.append_op(type='scale', - inputs={'X': [g]}, - outputs={'Out': [g]}, - attrs={'scale': scale}) + block.append_op( + type='c_allreduce_sum', + inputs={'X': [g]}, + outputs={'Out': [g]}, + attrs={'ring_id': 0, 'use_calc_stream': True}, + ) + block.append_op( + type='scale', + inputs={'X': [g]}, + outputs={'Out': [g]}, + attrs={'scale': scale}, + ) def _static_clip(self, params_grads): if self.clip_after_allreduce: @@ -112,7 +112,6 @@ class GradClipDecorator(ClipGradBase): class IdentityGradClip(ClipGradBase): - def _dygraph_clip(self, params_grads): return params_grads @@ -129,12 +128,14 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): with paddle.static.program_guard(main, startup): with paddle.fluid.unique_name.guard(): with paddle.static.amp.fp16_guard(): - image = paddle.static.data(name='image', - shape=[None, 3, 224, 224], - dtype=paddle.float32) - label = paddle.static.data(name='label', - shape=[None, 1], - dtype=paddle.int64) + image = paddle.static.data( + name='image', + shape=[None, 3, 224, 224], + dtype=paddle.float32, + ) + label = paddle.static.data( + name='label', shape=[None, 1], dtype=paddle.int64 + ) model = resnet() pred = model(image) loss_fn = paddle.nn.loss.CrossEntropyLoss() @@ -160,18 +161,24 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): kwargs.pop('clip_after_allreduce', None) kwargs.pop('alignment', None) kwargs.pop('use_master_acc_grad', None) - base_clip = grad_clip if grad_clip is not None else IdentityGradClip( + base_clip = ( + grad_clip if grad_clip is not None else IdentityGradClip() + ) + kwargs['grad_clip'] = GradClipDecorator( + base_clip, clip_after_allreduce ) - kwargs['grad_clip'] = GradClipDecorator(base_clip, - clip_after_allreduce) kwargs.pop('gradient_accumulation_steps', None) optimizer = optimizer_class(**kwargs) get_parameter = optimizer._get_parameter amp_list = paddle.static.amp.AutoMixedPrecisionLists( custom_white_list=[ - 'batch_norm', 'batch_norm_grad', 'conv2d', 'conv2d_grad' - ]) + 'batch_norm', + 'batch_norm_grad', + 'conv2d', + 'conv2d_grad', + ] + ) if use_fp16: if not use_distributed_lamb: optimizer._multi_precision = True @@ -182,14 +189,16 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): init_loss_scaling=1.0, use_dynamic_loss_scaling=False, use_pure_fp16=use_fp16, - use_fp16_guard=use_fp16) + use_fp16_guard=use_fp16, + ) amp_init = optimizer.amp_init else: amp_init = None if gm_steps > 1 and not use_distributed_lamb: optimizer = paddle.fluid.optimizer.GradientMergeOptimizer( - optimizer, k_steps=gm_steps, avg=False) + optimizer, k_steps=gm_steps, avg=False + ) params_grads = optimizer.backward(loss, startup) op_num = len(main.global_block().ops) @@ -222,8 +231,9 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): def reader(): for _ in range(6): - yield dict([(grad.name, gen_random_grad_tensor(grad)) - for grad in grads]) + yield dict( + [(grad.name, gen_random_grad_tensor(grad)) for grad in grads] + ) scope = paddle.static.Scope() fetch_list = params @@ -253,7 +263,6 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): class TestDistributedFusedLamb(unittest.TestCase): - @classmethod def setUpClass(cls): if not paddle.is_compiled_with_cuda(): @@ -266,28 +275,28 @@ class TestDistributedFusedLamb(unittest.TestCase): def config(self): clip_after_allreduce = bool( - distutils.util.strtobool(os.getenv('CLIP_AFTER_ALLREDUCE', 'True'))) + distutils.util.strtobool(os.getenv('CLIP_AFTER_ALLREDUCE', 'True')) + ) max_global_norm = float(os.getenv('MAX_GLOBAL_NORM', -1.0)) gm_steps = int(os.getenv('GRADIENT_MERGE_STEPS', 1)) use_master_acc_grad = bool(int(os.getenv('USE_MASTER_ACC_GRAD', '1'))) - print('clip_after_allreduce = {}, max_global_norm = {}'.format( - clip_after_allreduce, max_global_norm)) + print( + 'clip_after_allreduce = {}, max_global_norm = {}'.format( + clip_after_allreduce, max_global_norm + ) + ) return { - 'clip_after_allreduce': - clip_after_allreduce, - 'gradient_accumulation_steps': - gm_steps, - 'grad_clip': - paddle.nn.ClipGradByGlobalNorm(max_global_norm) - if max_global_norm > 0 else None, - 'use_master_acc_grad': - use_master_acc_grad, + 'clip_after_allreduce': clip_after_allreduce, + 'gradient_accumulation_steps': gm_steps, + 'grad_clip': paddle.nn.ClipGradByGlobalNorm(max_global_norm) + if max_global_norm > 0 + else None, + 'use_master_acc_grad': use_master_acc_grad, } - def run_main(self, - use_fp16, - use_master_param_norm=True, - use_master_acc_grad=True): + def run_main( + self, use_fp16, use_master_param_norm=True, use_master_acc_grad=True + ): if not paddle.is_compiled_with_cuda(): return @@ -316,7 +325,8 @@ class TestDistributedFusedLamb(unittest.TestCase): for ret1, ret2 in zip(result1, result2): max_diff = np.max(np.abs(ret1 - ret2)) msg = 'max_diff = {} atol = {} when use_fp16 = {} , use_master_param_norm = {}'.format( - max_diff, atol, use_fp16, use_master_param_norm) + max_diff, atol, use_fp16, use_master_param_norm + ) self.assertTrue(max_diff < atol, msg) print(msg) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/auto_parallel_pass_test_base.py b/python/paddle/fluid/tests/unittests/distributed_passes/auto_parallel_pass_test_base.py index 4028bae08a4..5b9bd4faf38 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/auto_parallel_pass_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/auto_parallel_pass_test_base.py @@ -25,11 +25,14 @@ from paddle.distributed.fleet import auto sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) class AutoPallelPassTestBase(DistPassTestBase): - def setUp(self): paddle.enable_static() seed = int(os.environ.get('SEED', -1)) @@ -60,14 +63,12 @@ class AutoPallelPassTestBase(DistPassTestBase): fleet.init(is_collective=True, strategy=dist_strategy) def check_main(self, gpus=None, **kwargs): - no_pass_rets = self._distributed_launch(model=None, - apply_pass=False, - gpus=gpus, - **kwargs) - pass_rets = self._distributed_launch(model=None, - apply_pass=True, - gpus=gpus, - **kwargs) + no_pass_rets = self._distributed_launch( + model=None, apply_pass=False, gpus=gpus, **kwargs + ) + pass_rets = self._distributed_launch( + model=None, apply_pass=True, gpus=gpus, **kwargs + ) self.check_results(no_pass_rets, pass_rets) def _run_gpu_main(self, model, apply_pass, dump_file, **kwargs): @@ -78,12 +79,18 @@ class AutoPallelPassTestBase(DistPassTestBase): self.apply_passes() else: self.apply_no_passes() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): with paddle.static.scope_guard(scope): with paddle.fluid.unique_name.guard(): - main_prog, startup_prog, inputs, outputs, data_loader = self.get_model( - place, **kwargs) + ( + main_prog, + startup_prog, + inputs, + outputs, + data_loader, + ) = self.get_model(place, **kwargs) inputs = self._to_var_names(inputs) outputs = self._to_var_names(outputs) @@ -98,8 +105,9 @@ class AutoPallelPassTestBase(DistPassTestBase): fetch_values = exe.run(main_prog, fetch_list=outputs) if paddle.distributed.get_rank() == 0: output_dict = OrderedDict(zip(outputs, fetch_values)) - print('batch {}, outputs {}'.format( - batch_id, output_dict)) + print( + 'batch {}, outputs {}'.format(batch_id, output_dict) + ) all_fetch_values.append(fetch_values) batch_id += 1 except paddle.fluid.core.EOFException: @@ -108,9 +116,9 @@ class AutoPallelPassTestBase(DistPassTestBase): with open(dump_file, "wb") as f: pickle.dump(all_fetch_values, f) - def get_gpt_model(self, strategy, place, batch_size, sequence_len, - vocab_size, **kwargs): - + def get_gpt_model( + self, strategy, place, batch_size, sequence_len, vocab_size, **kwargs + ): def gen_data(): np.random.seed(2021) for _ in range(10): @@ -121,14 +129,19 @@ class AutoPallelPassTestBase(DistPassTestBase): loss_mask = [] for _ in range(batch_size): tokens.append( - np.random.randint(vocab_size, - size=sequence_len).astype("int64")) + np.random.randint(vocab_size, size=sequence_len).astype( + "int64" + ) + ) position_ids.append(np.arange(sequence_len).astype("int64")) attention_mask.append( - [np.tril(np.ones(sequence_len)).astype("float32")]) + [np.tril(np.ones(sequence_len)).astype("float32")] + ) labels.append( - np.random.randint(vocab_size, - size=sequence_len).astype("int64")) + np.random.randint(vocab_size, size=sequence_len).astype( + "int64" + ) + ) loss_mask.append(np.ones(sequence_len).astype("float32")) yield tokens, position_ids, attention_mask, labels, loss_mask @@ -136,65 +149,74 @@ class AutoPallelPassTestBase(DistPassTestBase): modeling.init_global() if strategy == "dp": modeling._global_parallel_strategy = "dp" - modeling._global_process_mesh = auto.ProcessMesh(mesh=[0, 1], - dim_names=["x"]) + modeling._global_process_mesh = auto.ProcessMesh( + mesh=[0, 1], dim_names=["x"] + ) elif strategy == "mp": modeling._global_parallel_strategy = "mp" - modeling._global_process_mesh = auto.ProcessMesh(mesh=[0, 1], - dim_names=["x"]) + modeling._global_process_mesh = auto.ProcessMesh( + mesh=[0, 1], dim_names=["x"] + ) else: raise ValueError("'get_gpt_model' only support dp and mp.") - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] data_loader = paddle.fluid.io.DataLoader.from_generator( - feed_list=data_holder, capacity=70, iterable=False) + feed_list=data_holder, capacity=70, iterable=False + ) data_loader.set_batch_generator(gen_data, paddle.static.cuda_places()) if modeling._global_parallel_strategy == "dp": - auto.shard_tensor(tokens, modeling._global_process_mesh, - ["x", None]) + auto.shard_tensor( + tokens, modeling._global_process_mesh, ["x", None] + ) elif modeling._global_parallel_strategy == "pp": auto.shard_tensor(tokens, modeling.PP_MESH_LIST[0], [None, None]) - auto.shard_tensor(attention_mask, modeling.PP_MESH_LIST[0], - [None, None, None, None]) - - gpt = GPTModel(vocab_size=1000, - hidden_size=64, - num_hidden_layers=2, - num_attention_heads=8, - intermediate_size=256, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + auto.shard_tensor( + attention_mask, + modeling.PP_MESH_LIST[0], + [None, None, None, None], + ) + + gpt = GPTModel( + vocab_size=1000, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=8, + intermediate_size=256, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) @@ -202,17 +224,26 @@ class AutoPallelPassTestBase(DistPassTestBase): clip = paddle.nn.ClipGradByNorm(clip_norm=1.0) if kwargs.get('optimizer', None) == "LarsMomentum": optimizer = paddle.fluid.optimizer.LarsMomentumOptimizer( - learning_rate=0.001, momentum=0.9) + learning_rate=0.001, momentum=0.9 + ) else: - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=clip) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=clip, + ) optimizer = fleet.distributed_optimizer(optimizer) startup_program = paddle.static.default_startup_program() _, _, dist_startup_prog, dist_main_prog = optimizer.minimize( - loss, startup_program) - - return dist_main_prog, dist_startup_prog, data_holder, [loss - ], data_loader + loss, startup_program + ) + + return ( + dist_main_prog, + dist_startup_prog, + data_holder, + [loss], + data_loader, + ) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/check_pass_conflict_example.py b/python/paddle/fluid/tests/unittests/distributed_passes/check_pass_conflict_example.py index ffb8ea8e381..1223fbc0c19 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/check_pass_conflict_example.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/check_pass_conflict_example.py @@ -19,7 +19,6 @@ from model_zoo import resnet_model class CheckPassConflictTest1(PassConflictChecker): - def pass_config(self): return [ new_pass("fuse_all_reduce", {"max_memory_size": 1024 * 1024}), @@ -31,7 +30,6 @@ class CheckPassConflictTest1(PassConflictChecker): class CheckPassConflictTest2(PassConflictChecker): - def pass_config(self): return [ new_pass("fuse_elewise_add_act"), diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/dist_pass_test_base.py b/python/paddle/fluid/tests/unittests/distributed_passes/dist_pass_test_base.py index 6b0d53d2d00..5f62faeb422 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/dist_pass_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/dist_pass_test_base.py @@ -42,7 +42,7 @@ def prepare_python_path_and_return_module(path): python_path = dirname os.environ[env_name] = python_path print('GLOG_v=', os.environ.get('GLOG_v', None), flush=1) - return filename[:-len(py_suffix)] + return filename[: -len(py_suffix)] def remove_path_if_exists(path): @@ -57,7 +57,6 @@ def remove_path_if_exists(path): # NOTE: only support GPU now class DistPassTestBase(unittest.TestCase): - def setUp(self): paddle.enable_static() if paddle.is_compiled_with_cuda(): @@ -86,30 +85,31 @@ class DistPassTestBase(unittest.TestCase): raise NotImplementedError() def check_main(self, model=None, gpus=None, **kwargs): - pass_rets = self._distributed_launch(model=model, - apply_pass=True, - gpus=gpus, - **kwargs) - no_pass_rets = self._distributed_launch(model=model, - apply_pass=False, - gpus=gpus, - **kwargs) + pass_rets = self._distributed_launch( + model=model, apply_pass=True, gpus=gpus, **kwargs + ) + no_pass_rets = self._distributed_launch( + model=model, apply_pass=False, gpus=gpus, **kwargs + ) self.check_results(no_pass_rets, pass_rets) def check_results(self, no_pass_rets, pass_rets): self.assertEqual(len(no_pass_rets), len(pass_rets)) for no_pass_ret, pass_ret in zip(no_pass_rets, pass_rets): self.assertEqual(len(no_pass_ret), len(pass_ret)) - for i, (out_var_no_pass, - out_var_pass) in enumerate(zip(no_pass_ret, pass_ret)): + for i, (out_var_no_pass, out_var_pass) in enumerate( + zip(no_pass_ret, pass_ret) + ): if out_var_no_pass is None: self.assertTrue(out_var_pass is None) else: - np.testing.assert_allclose(out_var_no_pass, - out_var_pass, - rtol=self.rtol, - atol=self.atol, - equal_nan=self.equal_nan) + np.testing.assert_allclose( + out_var_no_pass, + out_var_pass, + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, + ) @classmethod def _to_var_names(cls, names_or_vars): @@ -129,12 +129,14 @@ class DistPassTestBase(unittest.TestCase): scope = paddle.static.Scope() if model is None: model = self.get_model - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): with paddle.static.scope_guard(scope): with paddle.fluid.unique_name.guard(): main_prog, startup_prog, inputs, outputs, reader = model( - place, **kwargs) + place, **kwargs + ) inputs = self._to_var_names(inputs) outputs = self._to_var_names(outputs) if apply_pass: @@ -146,7 +148,8 @@ class DistPassTestBase(unittest.TestCase): exe.run(startup_prog) for batch_id, input_data in enumerate(reader()): assert len(input_data) == len(inputs), "{} vs {}".format( - len(input_data), len(inputs)) + len(input_data), len(inputs) + ) feed = dict(zip(inputs, input_data)) fetch_values = exe.run(main_prog, feed=feed, fetch_list=outputs) if paddle.distributed.get_rank() == 0: @@ -204,26 +207,30 @@ class DistPassTestBase(unittest.TestCase): with open(model_dump_file, 'wb') as f: pickle.dump(model, f) - cmd = [ - sys.executable, - "-u", - ] + coverage_args + [ - "-m", - "launch", - "--log_dir", - output_dir, - "--gpus", - gpus, - os.path.join(file_dir, "pass_run_main.py"), - "--file_path", - inspect.getfile(type(self)), - "--class_name", - type(self).__name__, - "--input_file", - input_dump_file, - "--output_dir", - output_dir, - ] + cmd = ( + [ + sys.executable, + "-u", + ] + + coverage_args + + [ + "-m", + "launch", + "--log_dir", + output_dir, + "--gpus", + gpus, + os.path.join(file_dir, "pass_run_main.py"), + "--file_path", + inspect.getfile(type(self)), + "--class_name", + type(self).__name__, + "--input_file", + input_dump_file, + "--output_dir", + output_dir, + ] + ) if apply_pass: cmd += ["--apply_pass"] if model is not None: @@ -232,17 +239,22 @@ class DistPassTestBase(unittest.TestCase): prepare_python_path_and_return_module(__file__) exitcode = os.system(' '.join(cmd)) self.assertEqual( - exitcode, 0, - "Pass test failed with apply_pass = {}, please view log in {}". - format(apply_pass, output_dir)) + exitcode, + 0, + "Pass test failed with apply_pass = {}, please view log in {}".format( + apply_pass, output_dir + ), + ) results = [] for i in range(num_gpus): dump_file = '{0}/{1}.bin'.format(output_dir, i) self.assertTrue( os.path.exists(dump_file), - "Pass test failed with apply_pass = {}, please view log in {}" - .format(apply_pass, output_dir)) + "Pass test failed with apply_pass = {}, please view log in {}".format( + apply_pass, output_dir + ), + ) with open(dump_file, "rb") as f: results.append(pickle.load(f)) return results @@ -252,7 +264,6 @@ class DistPassTestBase(unittest.TestCase): class PassConflictChecker(DistPassTestBase): - def setUp(self): os.environ['DEBUG'] = '1' # to save the debug directory super(PassConflictChecker, self).setUp() @@ -270,14 +281,20 @@ class PassConflictChecker(DistPassTestBase): auto_pass_manager = PassManager(passes, auto_solve_conflict=True) new_passes = auto_pass_manager.passes self.assertEqual( - len(passes), len(new_passes), + len(passes), + len(new_passes), "After solving conflicts, the left passes are: {}".format( - auto_pass_manager.names)) + auto_pass_manager.names + ), + ) for i, (p1, p2) in enumerate(zip(passes, new_passes)): self.assertEqual( - id(p1), id(p2), - "After solving conflicts, the {}-th pass is different: {} vs {}" - .format(i, p1.name, p2.name)) + id(p1), + id(p2), + "After solving conflicts, the {}-th pass is different: {} vs {}".format( + i, p1.name, p2.name + ), + ) auto_pass_manager.apply([main_prog], [startup_prog]) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/model_zoo.py b/python/paddle/fluid/tests/unittests/distributed_passes/model_zoo.py index 9a48d117bb1..a0bff06e29c 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/model_zoo.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/model_zoo.py @@ -28,16 +28,15 @@ def get_seed_from_env(): return int(os.environ.get("SEED", 0)) -def resnet_model(place, - batch_size, - image_shape=[3, 224, 224], - num_classes=1000): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') - label = paddle.static.data(shape=[batch_size, 1], - dtype='int64', - name='label') +def resnet_model( + place, batch_size, image_shape=[3, 224, 224], num_classes=1000 +): + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) + label = paddle.static.data( + shape=[batch_size, 1], dtype='int64', name='label' + ) model = resnet(pretrained=False) loss_fn = nn.loss.CrossEntropyLoss() pred_out = model(image) @@ -58,9 +57,9 @@ def resnet_model(place, np.random.seed(seed + rank) for _ in range(10): image_np = np.random.random(size=image.shape).astype('float32') - label_np = np.random.randint(low=0, - high=num_classes, - size=label.shape).astype('int64') + label_np = np.random.randint( + low=0, high=num_classes, size=label.shape + ).astype('int64') yield image_np, label_np main_program = paddle.static.default_main_program() @@ -69,12 +68,12 @@ def resnet_model(place, def simple_net(place, batch_size, image_shape=[784], num_classes=10): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') - label = paddle.static.data(shape=[batch_size, 1], - dtype='int64', - name='label') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) + label = paddle.static.data( + shape=[batch_size, 1], dtype='int64', name='label' + ) linears = [nn.Linear(784, 784) for _ in range(3)] hidden = image for linear in linears: @@ -98,9 +97,9 @@ def simple_net(place, batch_size, image_shape=[784], num_classes=10): np.random.seed(seed + rank) for _ in range(10): image_np = np.random.random(size=image.shape).astype('float32') - label_np = np.random.randint(low=0, - high=num_classes, - size=label.shape).astype('int64') + label_np = np.random.randint( + low=0, high=num_classes, size=label.shape + ).astype('int64') yield image_np, label_np main_program = paddle.static.default_main_program() diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/pass_run_main.py b/python/paddle/fluid/tests/unittests/distributed_passes/pass_run_main.py index cd98d39a09e..cfea9a05c4e 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/pass_run_main.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/pass_run_main.py @@ -18,35 +18,43 @@ import pickle import importlib import os from paddle.distributed.fleet.launch_utils import run_with_coverage -from dist_pass_test_base import prepare_python_path_and_return_module, DistPassTestBase +from dist_pass_test_base import ( + prepare_python_path_and_return_module, + DistPassTestBase, +) def parse_args(): parser = argparse.ArgumentParser( - description='arguments for distributed pass tests') + description='arguments for distributed pass tests' + ) parser.add_argument('--file_path', type=str, help='The test file path.') parser.add_argument( '--class_name', type=str, - help= - 'The test class name. It is the class name that inherits the DistPassTestBase class.' + help='The test class name. It is the class name that inherits the DistPassTestBase class.', + ) + parser.add_argument( + '--apply_pass', + default=False, + action="store_true", + help='Whether to apply distributed passes.', ) - parser.add_argument('--apply_pass', - default=False, - action="store_true", - help='Whether to apply distributed passes.') parser.add_argument( '--input_file', type=str, - help='The input file which contains the dumped input arguments.') + help='The input file which contains the dumped input arguments.', + ) parser.add_argument( '--output_dir', type=str, - help='The output directory to save the logs and output results.') + help='The output directory to save the logs and output results.', + ) parser.add_argument( '--model_file', type=str, - help='The input model file which contains the dumped model function.') + help='The input model file which contains the dumped model function.', + ) return parser.parse_args() diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py b/python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py index bd2fa6b0198..87f275c21c6 100755 --- a/python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py @@ -17,11 +17,12 @@ import sys import shlex import unittest from paddle.fluid.tests.unittests.distributed_passes.dist_pass_test_base import ( # noqa: F401 - prepare_python_path_and_return_module, remove_path_if_exists) + prepare_python_path_and_return_module, + remove_path_if_exists, +) class PsPassTestBase(unittest.TestCase): - def init(self): self.config = {} self.config['ps_mode_config'] = "" @@ -46,30 +47,43 @@ class PsPassTestBase(unittest.TestCase): if ps_mode == "cpu-ps" or ps_mode == 'heter-ps': os.environ['WITH_DISTRIBUTE'] = 'ON' - cmd = [ - sys.executable, - "-u", - ] + [ - "-m", "launch", "--log_dir", self.config['log_dir'], - "--worker_num", self.config['worker_num'], "--server_num", - self.config['server_num'] + cmd = [sys.executable, "-u",] + [ + "-m", + "launch", + "--log_dir", + self.config['log_dir'], + "--worker_num", + self.config['worker_num'], + "--server_num", + self.config['server_num'], ] if ps_mode == 'heter-ps': os.environ['FLAGS_START_PORT'] = '12004' cmd += [ - '--heter_worker_num', self.config['heter_worker_num'], - '--heter_devices', self.config['heter_devices'] + '--heter_worker_num', + self.config['heter_worker_num'], + '--heter_devices', + self.config['heter_devices'], ] cmd += [ - "../ps/ps_dnn_trainer.py", "-m", self.config['ps_mode_config'], - "--run_minimize", self.config['run_minimize'], - "--run_single_pass", self.config['run_single_pass'], - "--run_the_one_ps", self.config['run_the_one_ps'], - "--debug_new_pass", self.config['debug_new_pass'], - "--debug_new_minimize", self.config['debug_new_minimize'], - "--applied_pass_name", self.config['applied_pass_name'], - "--debug_the_one_ps", self.config['debug_the_one_ps'] + "../ps/ps_dnn_trainer.py", + "-m", + self.config['ps_mode_config'], + "--run_minimize", + self.config['run_minimize'], + "--run_single_pass", + self.config['run_single_pass'], + "--run_the_one_ps", + self.config['run_the_one_ps'], + "--debug_new_pass", + self.config['debug_new_pass'], + "--debug_new_minimize", + self.config['debug_new_minimize'], + "--applied_pass_name", + self.config['applied_pass_name'], + "--debug_the_one_ps", + self.config['debug_the_one_ps'], ] elif ps_mode == "gpu-ps": os.environ['FLAGS_LAUNCH_BARRIER'] = '0' @@ -87,15 +101,25 @@ class PsPassTestBase(unittest.TestCase): os.environ['PADDLE_TRAINER_ID'] = '0' cmd = [ - sys.executable, "-u", "../ps/ps_dnn_trainer.py", "-m", - self.config['ps_mode_config'], "--run_minimize", - self.config['run_minimize'], "--run_single_pass", - self.config['run_single_pass'], "--run_the_one_ps", - self.config['run_the_one_ps'], "--debug_new_pass", - self.config['debug_new_pass'], "--debug_new_minimize", - self.config['debug_new_minimize'], "--applied_pass_name", - self.config['applied_pass_name'], "--debug_the_one_ps", - self.config['debug_the_one_ps'] + sys.executable, + "-u", + "../ps/ps_dnn_trainer.py", + "-m", + self.config['ps_mode_config'], + "--run_minimize", + self.config['run_minimize'], + "--run_single_pass", + self.config['run_single_pass'], + "--run_the_one_ps", + self.config['run_the_one_ps'], + "--debug_new_pass", + self.config['debug_new_pass'], + "--debug_new_minimize", + self.config['debug_new_minimize'], + "--applied_pass_name", + self.config['applied_pass_name'], + "--debug_the_one_ps", + self.config['debug_the_one_ps'], ] cmd = [shlex.quote(c) for c in cmd] diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py index 0d404d1732d..e4173e11dbc 100755 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py @@ -22,7 +22,6 @@ from auto_parallel_pass_test_base import AutoPallelPassTestBase class TestAMPPass(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -51,14 +50,14 @@ class TestAMPPass(AutoPallelPassTestBase): fleet.init(is_collective=True, strategy=dist_strategy) def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model("mp", place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + "mp", place, batch_size, sequence_len, vocab_size + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py index 7da95ddf8d2..530d4b2661e 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py @@ -19,7 +19,9 @@ import numpy as np import unittest import paddle import paddle.distributed.fleet as fleet -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) from paddle.distributed.passes import PassContext, new_pass from auto_parallel_pass_test_base import AutoPallelPassTestBase @@ -27,7 +29,6 @@ sys.path.append("..") class TestDataParallelPassWithScale1(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -55,24 +56,29 @@ class TestDataParallelPassWithScale1(AutoPallelPassTestBase): self._apply_pass = False def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) # test scaling with fillconstant def get_model(self, place, batch_size, sequence_len, vocab_size): - dist_main_prog, dist_startup_prog, data_holder, [ - loss - ], gen_data = self.get_gpt_model('dp', place, batch_size, sequence_len, - vocab_size) + ( + dist_main_prog, + dist_startup_prog, + data_holder, + [loss], + gen_data, + ) = self.get_gpt_model( + 'dp', place, batch_size, sequence_len, vocab_size + ) if self._apply_pass: config = {} config["dist_context"] = get_default_distributed_context() config["global_rank"] = paddle.distributed.get_rank() - dp_pass = new_pass("auto_parallel_data_parallel_optimization", - config) + dp_pass = new_pass( + "auto_parallel_data_parallel_optimization", config + ) dp_pass.apply([dist_main_prog], [dist_startup_prog], PassContext()) return dist_main_prog, dist_startup_prog, data_holder, [loss], gen_data @@ -83,20 +89,27 @@ class TestDataParallelPassWithScale2(TestDataParallelPassWithScale1): # test scaling with optimizer rescale_grad def get_model(self, place, batch_size, sequence_len, vocab_size): - dist_main_prog, dist_startup_prog, data_holder, [ - loss - ], gen_data = self.get_gpt_model('dp', - place, - batch_size, - sequence_len, - vocab_size, - optimizer='LarsMomentum') + ( + dist_main_prog, + dist_startup_prog, + data_holder, + [loss], + gen_data, + ) = self.get_gpt_model( + 'dp', + place, + batch_size, + sequence_len, + vocab_size, + optimizer='LarsMomentum', + ) if self._apply_pass: config = {} config["dist_context"] = get_default_distributed_context() config["global_rank"] = paddle.distributed.get_rank() - dp_pass = new_pass("auto_parallel_data_parallel_optimization", - config) + dp_pass = new_pass( + "auto_parallel_data_parallel_optimization", config + ) dp_pass.apply([dist_main_prog], [dist_startup_prog], PassContext()) return dist_main_prog, dist_startup_prog, data_holder, [loss], gen_data diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py index 2dfe965fc01..b3b2857e292 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py @@ -22,7 +22,6 @@ from auto_parallel_pass_test_base import AutoPallelPassTestBase class TestPF16Pass(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -42,29 +41,28 @@ class TestPF16Pass(AutoPallelPassTestBase): 'layer_norm', 'gelu', ], - "custom_black_list": - ['c_softmax_with_cross_entropy', 'elementwise_div', 'reduce_sum'], - "init_loss_scaling": - 32768, - "use_dynamic_loss_scaling": - True, - "use_pure_fp16": - True, - "use_fp16_guard": - False + "custom_black_list": [ + 'c_softmax_with_cross_entropy', + 'elementwise_div', + 'reduce_sum', + ], + "init_loss_scaling": 32768, + "use_dynamic_loss_scaling": True, + "use_pure_fp16": True, + "use_fp16_guard": False, } dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model("mp", place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + "mp", place, batch_size, sequence_len, vocab_size + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py index 3b6eb35937e..ddb4c3cea05 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py @@ -33,11 +33,9 @@ paddle.enable_static() class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=128, - intermediate_size=4 * 128, - initializer_range=0.02): + def __init__( + self, hidden_size=128, intermediate_size=4 * 128, initializer_range=0.02 + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size @@ -47,30 +45,24 @@ class MLPLayer(nn.Layer): weight_attr0 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr0)) weight_attr1 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr1)) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) - self.linear2 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear3 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) - self.linear4 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear5 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) + self.linear2 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear3 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) + self.linear4 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear5 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) self.norm0 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) @@ -94,11 +86,14 @@ class MLPLayer(nn.Layer): def mlp_forward(input, label, hidden_size): - auto.shard_tensor(input, auto.ProcessMesh([0], dim_names=["x"]), - [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + auto.shard_tensor( + input, auto.ProcessMesh([0], dim_names=["x"]), [None, None] + ) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) loss = paddle.mean(error_cost) @@ -106,7 +101,6 @@ def mlp_forward(input, label, hidden_size): class TestGradientMergePass(AutoPallelPassTestBase): - def init(self): paddle.seed(2022) random.seed(2022) @@ -120,18 +114,22 @@ class TestGradientMergePass(AutoPallelPassTestBase): fleet.init(is_collective=True, strategy=dist_strategy) def test_result(self): - no_pass_rets = self._distributed_launch(model=None, - apply_pass=False, - gpus=[0], - batch_size=32, - hidden_size=128, - max_step=2) - pass_rets = self._distributed_launch(model=None, - apply_pass=True, - gpus=[0], - batch_size=8, - hidden_size=128, - max_step=8) + no_pass_rets = self._distributed_launch( + model=None, + apply_pass=False, + gpus=[0], + batch_size=32, + hidden_size=128, + max_step=2, + ) + pass_rets = self._distributed_launch( + model=None, + apply_pass=True, + gpus=[0], + batch_size=8, + hidden_size=128, + max_step=8, + ) # avg loss for gradient_merge pass avg_loss = 0 pass_avg_ret_list = [] @@ -146,49 +144,63 @@ class TestGradientMergePass(AutoPallelPassTestBase): for no_pass_ret, pass_ret in zip(no_pass_rets[0], pass_avg_ret_list): print(f"no_pass_ret={no_pass_ret}, pass_ret={pass_ret}") self.assertTrue( - np.isclose(no_pass_ret, - pass_ret, - rtol=self.rtol, - atol=self.atol, - equal_nan=self.equal_nan)) + np.isclose( + no_pass_ret, + pass_ret, + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, + ) + ) def get_model(self, place, batch_size, hidden_size, max_step): - def gen_data(): for i in range(max_step): - x_data = input_data[i * batch_size:(i + 1) * batch_size, :] - y_data = label_data[i * batch_size:(i + 1) * batch_size, :] + x_data = input_data[i * batch_size : (i + 1) * batch_size, :] + y_data = label_data[i * batch_size : (i + 1) * batch_size, :] yield x_data, y_data train_program = static.Program() startup_program = static.Program() - with static.program_guard(train_program, startup_program), \ - utils.unique_name.guard(): - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + with static.program_guard( + train_program, startup_program + ), utils.unique_name.guard(): + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) input.stop_gradient = False data_holder = [input, label] data_loader = paddle.fluid.io.DataLoader.from_generator( - feed_list=data_holder, capacity=70, iterable=False) - data_loader.set_batch_generator(gen_data, - paddle.static.cuda_places()) + feed_list=data_holder, capacity=70, iterable=False + ) + data_loader.set_batch_generator( + gen_data, paddle.static.cuda_places() + ) loss = mlp_forward(input, label, hidden_size) optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.01) optimizer = fleet.distributed_optimizer(optimizer) - _, self._params_grads, dist_startup_prog, dist_main_prog = optimizer.minimize( - loss, startup_program) + ( + _, + self._params_grads, + dist_startup_prog, + dist_main_prog, + ) = optimizer.minimize(loss, startup_program) input_data = np.random.random(size=(128, hidden_size)).astype('float32') label_data = np.random.random(size=(128, 1)).astype('float32') - return dist_main_prog, dist_startup_prog, [input, - label], [loss], data_loader + return ( + dist_main_prog, + dist_startup_prog, + [input, label], + [loss], + data_loader, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py index fa5fd193aba..ddea56b2a14 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py @@ -22,7 +22,6 @@ from auto_parallel_pass_test_base import AutoPallelPassTestBase class TestRecomputePass(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -42,21 +41,21 @@ class TestRecomputePass(AutoPallelPassTestBase): fleet.init(is_collective=True, strategy=dist_strategy) def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model("mp", place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + "mp", place, batch_size, sequence_len, vocab_size + ) class TestRecomputePassDP(TestRecomputePass): - def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model("dp", place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + "dp", place, batch_size, sequence_len, vocab_size + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py index 21a9a81aa08..83e4667a1a9 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py @@ -25,7 +25,6 @@ sys.path.append("..") class TestShardingPass(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -55,14 +54,14 @@ class TestShardingPass(AutoPallelPassTestBase): fleet.init(is_collective=True, strategy=dist_strategy) def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model('dp', place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + 'dp', place, batch_size, sequence_len, vocab_size + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_adam_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_adam_pass.py index c1cdd0b7336..b570786f20c 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_adam_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_adam_pass.py @@ -25,7 +25,6 @@ paddle.enable_static() class DemoNet(nn.Layer): - def __init__(self): super(DemoNet, self).__init__() @@ -42,15 +41,14 @@ class DemoNet(nn.Layer): class TestFuseAdamPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = DemoNet() pred_out = model(image) @@ -86,10 +84,14 @@ class TestFuseAdamPass(DistPassTestBase): for op in main_prog.global_block().ops: op_type.append(op.type) if op.type == "adam": - self.assertTrue("@FUSEDVAR@_adam_Param_batch_norm2d_0.b_0" in - op.input("Param")) - self.assertTrue("@FUSEDVAR@_adam_Grad_batch_norm2d_0.b_0@GRAD" - in op.input("Grad")) + self.assertTrue( + "@FUSEDVAR@_adam_Param_batch_norm2d_0.b_0" + in op.input("Param") + ) + self.assertTrue( + "@FUSEDVAR@_adam_Grad_batch_norm2d_0.b_0@GRAD" + in op.input("Grad") + ) self.assertTrue("coalesce_tensor" in op_type) def test_fuse_adam(self): diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_all_reduce_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_all_reduce_pass.py index 06cd2ac6da4..46754a1193c 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_all_reduce_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_all_reduce_pass.py @@ -19,16 +19,17 @@ from model_zoo import resnet_model class TestFuseAllReducePass(DistPassTestBase): - def init(self): self.atol = 0.0 self.rtol = 0.0 def apply_passes(self, main_prog, startup_prog): - pass_manager = PassManager([ - new_pass("fuse_elewise_add_act"), - new_pass("fuse_all_reduce", {"max_memory_size": 1024 * 1024}) - ]) + pass_manager = PassManager( + [ + new_pass("fuse_elewise_add_act"), + new_pass("fuse_all_reduce", {"max_memory_size": 1024 * 1024}), + ] + ) pass_manager.apply([main_prog], [startup_prog]) print(pass_manager.names) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_act_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_act_pass.py index 12aeae57b09..cf00549e52f 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_act_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_act_pass.py @@ -25,7 +25,6 @@ paddle.enable_static() class BatchNormActNet(nn.Layer): - def __init__(self): super(BatchNormActNet, self).__init__() @@ -42,15 +41,14 @@ class BatchNormActNet(nn.Layer): class TestFuseBatchNormActPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = BatchNormActNet() pred_out = model(image) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_add_act_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_add_act_pass.py index 72dfd60d4a0..715dad4dc5b 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_add_act_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_add_act_pass.py @@ -25,7 +25,6 @@ paddle.enable_static() class BatchNormAddActNet(nn.Layer): - def __init__(self): super(BatchNormAddActNet, self).__init__() @@ -46,15 +45,14 @@ class BatchNormAddActNet(nn.Layer): class TestFuseBatchNormAddActPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = BatchNormAddActNet() pred_out = model(image) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_momentum_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_momentum_pass.py index c725a9b9156..4642be5839e 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_momentum_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_momentum_pass.py @@ -25,7 +25,6 @@ paddle.enable_static() class DemoNet(nn.Layer): - def __init__(self): super(DemoNet, self).__init__() @@ -42,15 +41,14 @@ class DemoNet(nn.Layer): class TestFuseAdamPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = DemoNet() pred_out = model(image) @@ -86,11 +84,14 @@ class TestFuseAdamPass(DistPassTestBase): for op in main_prog.global_block().ops: op_type.append(op.type) if op.type == "momentum": - self.assertTrue("@FUSEDVAR@_momentum_Param_batch_norm2d_0.b_0" - in op.input("Param")) self.assertTrue( - "@FUSEDVAR@_momentum_Grad_batch_norm2d_0.b_0@GRAD" in - op.input("Grad")) + "@FUSEDVAR@_momentum_Param_batch_norm2d_0.b_0" + in op.input("Param") + ) + self.assertTrue( + "@FUSEDVAR@_momentum_Grad_batch_norm2d_0.b_0@GRAD" + in op.input("Grad") + ) self.assertTrue("coalesce_tensor" in op_type) def test_fuse_adam(self): diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_relu_depthwise_conv_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_relu_depthwise_conv_pass.py index 61897b37ea7..c2c1404a967 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_relu_depthwise_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_relu_depthwise_conv_pass.py @@ -25,7 +25,6 @@ paddle.enable_static() class ReluDepthwiseConvNet(nn.Layer): - def __init__(self): super(ReluDepthwiseConvNet, self).__init__() @@ -42,15 +41,14 @@ class ReluDepthwiseConvNet(nn.Layer): class TestFuseReluDepthwiseConvPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[3, 224, 224]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = ReluDepthwiseConvNet() pred_out = model(image) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_sgd_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_sgd_pass.py index ab752c60ed3..68ca472f6d4 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_sgd_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_sgd_pass.py @@ -25,7 +25,6 @@ paddle.enable_static() class DemoNet(nn.Layer): - def __init__(self): super(DemoNet, self).__init__() @@ -42,15 +41,14 @@ class DemoNet(nn.Layer): class TestFuseAdamPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = DemoNet() pred_out = model(image) @@ -86,10 +84,14 @@ class TestFuseAdamPass(DistPassTestBase): for op in main_prog.global_block().ops: op_type.append(op.type) if op.type == "sgd": - self.assertTrue("@FUSEDVAR@_sgd_Param_batch_norm2d_0.b_0" in - op.input("Param")) - self.assertTrue("@FUSEDVAR@_sgd_Grad_batch_norm2d_0.b_0@GRAD" in - op.input("Grad")) + self.assertTrue( + "@FUSEDVAR@_sgd_Param_batch_norm2d_0.b_0" + in op.input("Param") + ) + self.assertTrue( + "@FUSEDVAR@_sgd_Grad_batch_norm2d_0.b_0@GRAD" + in op.input("Grad") + ) self.assertTrue("coalesce_tensor" in op_type) def test_fuse_adam(self): diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_inplace_addto_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_inplace_addto_pass.py index 0431c53c11c..e6f2adfee53 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_inplace_addto_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_inplace_addto_pass.py @@ -25,7 +25,6 @@ paddle.enable_static() class DemoNet(nn.Layer): - def __init__(self): super(DemoNet, self).__init__() @@ -43,16 +42,15 @@ class DemoNet(nn.Layer): class TestInplaceAddtoPass(DistPassTestBase): - def init(self): self.atol = 0.0 self.rtol = 0.0 paddle.fluid.set_flags({"FLAGS_max_inplace_grad_add": 8}) def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = DemoNet() pred_out = model(image) @@ -83,7 +81,8 @@ class TestInplaceAddtoPass(DistPassTestBase): def apply_passes(self, main_prog, startup_prog): pass_manager = PassManager( - [new_pass("inplace_addto_op", {"use_cuda": True})]) + [new_pass("inplace_addto_op", {"use_cuda": True})] + ) pass_manager.apply([main_prog], [startup_prog]) print(pass_manager.names) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_server_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_server_pass.py index e4844b664c2..b0725c2bf30 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_server_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_server_pass.py @@ -17,7 +17,6 @@ from ps_pass_test_base import PsPassTestBase class TestPsServerPass(PsPassTestBase): - def init(self): pass diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_trainer_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_trainer_pass.py index 9b7f7820c8f..d71a5b07b30 100755 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_trainer_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_trainer_pass.py @@ -19,7 +19,6 @@ from paddle.distributed.ps.utils.public import logger, ps_log_root_dir class TestPsTrainerPass(PsPassTestBase): - def setUp(self): pass @@ -142,7 +141,7 @@ class TestPsTrainerPass(PsPassTestBase): self.config['debug_new_minimize'] = '0' self.config['log_dir'] = ps_log_root_dir + "gpubox_log_old_minimize" remove_path_if_exists(self.config['log_dir']) - #self.ps_launch("gpu-ps") + # self.ps_launch("gpu-ps") self.config['debug_new_minimize'] = '1' self.config['log_dir'] = ps_log_root_dir + "gpubox_log_new_minimize" @@ -163,19 +162,25 @@ class TestPsTrainerPass(PsPassTestBase): self.config['applied_pass_name'] = "append_send_ops_pass" self.config['debug_new_pass'] = '0' - self.config['log_dir'] = ps_log_root_dir + "log_old_" + self.config[ - 'applied_pass_name'] + self.config['log_dir'] = ( + ps_log_root_dir + "log_old_" + self.config['applied_pass_name'] + ) remove_path_if_exists(self.config['log_dir']) self.ps_launch("cpu-ps") self.config['debug_new_pass'] = '1' - self.config['log_dir'] = ps_log_root_dir + "log_new_" + self.config[ - 'applied_pass_name'] + self.config['log_dir'] = ( + ps_log_root_dir + "log_new_" + self.config['applied_pass_name'] + ) remove_path_if_exists(self.config['log_dir']) self.ps_launch("cpu-ps") - file1 = './ps_log/async_append_send_ops_pass_debug:_0_worker_main.prototxt' - file2 = './ps_log/async_append_send_ops_pass_debug:_1_worker_main.prototxt' + file1 = ( + './ps_log/async_append_send_ops_pass_debug:_0_worker_main.prototxt' + ) + file2 = ( + './ps_log/async_append_send_ops_pass_debug:_1_worker_main.prototxt' + ) if self.check(file1, file2): logger.info('test_append_send_ops_pass passed!') else: diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_white_lists.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_white_lists.py index 645fa38099d..307b41d027c 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_white_lists.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_white_lists.py @@ -13,12 +13,17 @@ # limitations under the License. import unittest -from paddle.distributed.passes.pass_base import register_pass, PassBase, new_pass -from paddle.distributed.passes.pass_base import _make_rule_from_white_lists_dict as make_white_lists_rule +from paddle.distributed.passes.pass_base import ( + register_pass, + PassBase, + new_pass, +) +from paddle.distributed.passes.pass_base import ( + _make_rule_from_white_lists_dict as make_white_lists_rule, +) class TestConcretePass(PassBase): - def __init__(self): super(TestConcretePass, self).__init__() @@ -34,41 +39,35 @@ class TestConcretePass(PassBase): @register_pass("A") class A(TestConcretePass): - def __init__(self): super(A, self).__init__() @register_pass("B") class B(TestConcretePass): - def __init__(self): super(B, self).__init__() @register_pass("C") class C(TestConcretePass): - def __init__(self): super(C, self).__init__() @register_pass("D") class D(TestConcretePass): - def __init__(self): super(D, self).__init__() @register_pass("E") class E(TestConcretePass): - def __init__(self): super(E, self).__init__() class TestMakeWhiteListsRule(unittest.TestCase): - def test_main(self): before_white_lists = {"A": ["B", "C"]} after_white_lists = {"D": ["C"]} diff --git a/python/paddle/fluid/tests/unittests/distribution/config.py b/python/paddle/fluid/tests/unittests/distribution/config.py index aee76250e5d..29a27890bad 100644 --- a/python/paddle/fluid/tests/unittests/distribution/config.py +++ b/python/paddle/fluid/tests/unittests/distribution/config.py @@ -26,6 +26,6 @@ RTOL = { 'float32': 1e-03, 'complex64': 1e-3, 'float64': 1e-5, - 'complex128': 1e-5 + 'complex128': 1e-5, } ATOL = {'float32': 0.0, 'complex64': 0, 'float64': 0.0, 'complex128': 0} diff --git a/python/paddle/fluid/tests/unittests/distribution/mock_data.py b/python/paddle/fluid/tests/unittests/distribution/mock_data.py index 60299505f21..db08fe1c886 100644 --- a/python/paddle/fluid/tests/unittests/distribution/mock_data.py +++ b/python/paddle/fluid/tests/unittests/distribution/mock_data.py @@ -17,8 +17,9 @@ import paddle class Exponential(paddle.distribution.ExponentialFamily): """mock exponential distribution, which support computing entropy and - kl use bregman divergence + kl use bregman divergence """ + _mean_carrier_measure = 0 def __init__(self, rate): @@ -34,15 +35,14 @@ class Exponential(paddle.distribution.ExponentialFamily): @property def _natural_parameters(self): - return (-self._rate, ) + return (-self._rate,) def _log_normalizer(self, x): return -paddle.log(-x) class DummyExpFamily(paddle.distribution.ExponentialFamily): - """dummy class extend from exponential family - """ + """dummy class extend from exponential family""" def __init__(self, *args): pass @@ -52,7 +52,7 @@ class DummyExpFamily(paddle.distribution.ExponentialFamily): @property def _natural_parameters(self): - return (1.0, ) + return (1.0,) def _log_normalizer(self, x): return -paddle.log(-x) diff --git a/python/paddle/fluid/tests/unittests/distribution/parameterize.py b/python/paddle/fluid/tests/unittests/distribution/parameterize.py index 72c9ac03325..9c3341e34ce 100644 --- a/python/paddle/fluid/tests/unittests/distribution/parameterize.py +++ b/python/paddle/fluid/tests/unittests/distribution/parameterize.py @@ -25,16 +25,14 @@ TEST_CASE_NAME = 'suffix' def xrand(shape=(10, 10, 10), dtype=config.DEFAULT_DTYPE, min=1.0, max=10.0): - return ((np.random.rand(*shape).astype(dtype)) * (max - min) + min) + return (np.random.rand(*shape).astype(dtype)) * (max - min) + min def place(devices, key='place'): - def decorate(cls): module = sys.modules[cls.__module__].__dict__ raw_classes = { - k: v - for k, v in module.items() if k.startswith(cls.__name__) + k: v for k, v in module.items() if k.startswith(cls.__name__) } for raw_name, raw_cls in raw_classes.items(): @@ -42,7 +40,7 @@ def place(devices, key='place'): test_cls = dict(raw_cls.__dict__) test_cls.update({key: d}) new_name = raw_name + '.' + d.__class__.__name__ - module[new_name] = type(new_name, (raw_cls, ), test_cls) + module[new_name] = type(new_name, (raw_cls,), test_cls) del module[raw_name] return cls @@ -61,7 +59,7 @@ def parameterize_cls(fields, values=None): name = cls.__name__ + str(k) name = name + '.' + v.get('suffix') if v.get('suffix') else name - test_cls_module[name] = type(name, (cls, ), test_cls) + test_cls_module[name] = type(name, (cls,), test_cls) for m in list(cls.__dict__): if m.startswith("test"): @@ -71,10 +69,9 @@ def parameterize_cls(fields, values=None): return decorate -def parameterize_func(input, - name_func=None, - doc_func=None, - skip_on_empty=False): +def parameterize_func( + input, name_func=None, doc_func=None, skip_on_empty=False +): doc_func = doc_func or default_doc_func name_func = name_func or default_name_func @@ -88,13 +85,15 @@ def parameterize_func(input, raise ValueError( "Parameters iterable is empty (hint: use " "`parameterized.expand([], skip_on_empty=True)` to skip " - "this test when the input is empty)") + "this test when the input is empty)" + ) return wraps(f)(skip_on_empty_helper) digits = len(str(len(parameters) - 1)) for num, p in enumerate(parameters): - name = name_func(f, "{num:0>{digits}}".format(digits=digits, - num=num), p) + name = name_func( + f, "{num:0>{digits}}".format(digits=digits, num=num), p + ) # If the original function has patches applied by 'mock.patch', # re-construct all patches on the just former decoration layer # of param_as_standalone_func so as not to share @@ -113,9 +112,7 @@ def parameterize_func(input, def reapply_patches_if_need(func): - def dummy_wrapper(orgfunc): - @wraps(orgfunc) def dummy_func(*args, **kwargs): return orgfunc(*args, **kwargs) @@ -138,7 +135,7 @@ def delete_patches_if_need(func): def default_name_func(func, num, p): base_name = func.__name__ - name_suffix = "_%s" % (num, ) + name_suffix = "_%s" % (num,) if len(p.args) > 0 and isinstance(p.args[0], str): name_suffix += "_" + to_safe_name(p.args[0]) @@ -167,7 +164,6 @@ def default_doc_func(func, num, p): def param_as_standalone_func(p, func, name): - @functools.wraps(func) def standalone_func(*a): return func(*(a + p.args), **p.kwargs) @@ -209,36 +205,35 @@ _param = collections.namedtuple("param", "args kwargs") class param(_param): - def __new__(cls, *args, **kwargs): return _param.__new__(cls, args, kwargs) @classmethod def explicit(cls, args=None, kwargs=None): - """ Creates a ``param`` by explicitly specifying ``args`` and - ``kwargs``:: - >>> param.explicit([1,2,3]) - param(*(1, 2, 3)) - >>> param.explicit(kwargs={"foo": 42}) - param(*(), **{"foo": "42"}) - """ + """Creates a ``param`` by explicitly specifying ``args`` and + ``kwargs``:: + >>> param.explicit([1,2,3]) + param(*(1, 2, 3)) + >>> param.explicit(kwargs={"foo": 42}) + param(*(), **{"foo": "42"}) + """ args = args or () kwargs = kwargs or {} return cls(*args, **kwargs) @classmethod def from_decorator(cls, args): - """ Returns an instance of ``param()`` for ``@parameterized`` argument - ``args``:: - >>> param.from_decorator((42, )) - param(args=(42, ), kwargs={}) - >>> param.from_decorator("foo") - param(args=("foo", ), kwargs={}) - """ + """Returns an instance of ``param()`` for ``@parameterized`` argument + ``args``:: + >>> param.from_decorator((42, )) + param(args=(42, ), kwargs={}) + >>> param.from_decorator("foo") + param(args=("foo", ), kwargs={}) + """ if isinstance(args, param): return args elif isinstance(args, str): - args = (args, ) + args = (args,) try: return cls(*args) except TypeError as e: @@ -246,7 +241,8 @@ class param(_param): raise raise TypeError( "Parameters must be tuples, but %r is not (hint: use '(%r, )')" - % (args, args), ) + % (args, args), + ) def __repr__(self): return "param(*%r, **%r)" % self diff --git a/python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py b/python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py index 90209d9529b..9b1183a96eb 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py @@ -30,7 +30,7 @@ class TestDirichletOp(OpTest): def setUp(self): self.op_type = "dirichlet" - self.alpha = np.array((1., 2.)) + self.alpha = np.array((1.0, 2.0)) self.sample_shape = (100000, 2) self.inputs = {'Alpha': np.broadcast_to(self.alpha, self.sample_shape)} @@ -47,5 +47,7 @@ class TestDirichletOp(OpTest): scipy.stats.kstest( outs[0][:, 0], # scipy dirichlet have not cdf, use beta to replace it. - scipy.stats.beta(a=self.alpha[0], b=self.alpha[1]).cdf)[0], - 0.01) + scipy.stats.beta(a=self.alpha[0], b=self.alpha[1]).cdf, + )[0], + 0.01, + ) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution.py index 30024c10fee..8d592683aba 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution.py @@ -25,8 +25,7 @@ import parameterize paddle.enable_static() -class DistributionNumpy(): - +class DistributionNumpy: def sample(self): raise NotImplementedError @@ -44,9 +43,8 @@ class DistributionNumpy(): class DistributionTestName(unittest.TestCase): - def get_prefix(self, string): - return (string.split('.')[0]) + return string.split('.')[0] def test_normal_name(self): name = 'test_normal' @@ -136,15 +134,18 @@ class DistributionTestName(unittest.TestCase): @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'batch_shape', 'event_shape'), - [('test-tuple', (10, 20), (10, 20)), - ('test-list', [100, 100], [100, 200, 300]), - ('test-null-eventshape', (100, 100), ())]) + [ + ('test-tuple', (10, 20), (10, 20)), + ('test-list', [100, 100], [100, 200, 300]), + ('test-null-eventshape', (100, 100), ()), + ], +) class TestDistributionShape(unittest.TestCase): - def setUp(self): paddle.disable_static() self.dist = paddle.distribution.Distribution( - batch_shape=self.batch_shape, event_shape=self.event_shape) + batch_shape=self.batch_shape, event_shape=self.event_shape + ) def tearDown(self): paddle.enable_static() @@ -162,15 +163,15 @@ class TestDistributionShape(unittest.TestCase): self.dist.prob(paddle.to_tensor(parameterize.xrand())) def test_extend_shape(self): - shapes = [(34, 20), (56, ), ()] + shapes = [(34, 20), (56,), ()] for shape in shapes: self.assertTrue( self.dist._extend_shape(shape), - shape + self.dist.batch_shape + self.dist.event_shape) + shape + self.dist.batch_shape + self.dist.event_shape, + ) class TestDistributionException(unittest.TestCase): - def setUp(self): self._d = paddle.distribution.Distribution() diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta.py index 18315440a21..d6febc7ac4d 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta.py @@ -25,11 +25,15 @@ np.random.seed(2022) @place(DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'alpha', 'beta'), - [('test-scale', 1.0, 2.0), ('test-tensor', xrand(), xrand()), - ('test-broadcast', xrand((2, 1)), xrand((2, 5)))]) +@parameterize_cls( + (TEST_CASE_NAME, 'alpha', 'beta'), + [ + ('test-scale', 1.0, 2.0), + ('test-tensor', xrand(), xrand()), + ('test-broadcast', xrand((2, 1)), xrand((2, 5))), + ], +) class TestBeta(unittest.TestCase): - def setUp(self): # scale no need convert to tensor for scale input unittest alpha, beta = self.alpha, self.beta @@ -46,7 +50,8 @@ class TestBeta(unittest.TestCase): self._paddle_beta.mean, scipy.stats.beta.mean(self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_variance(self): with paddle.fluid.dygraph.guard(self.place): @@ -54,7 +59,8 @@ class TestBeta(unittest.TestCase): self._paddle_beta.variance, scipy.stats.beta.var(self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_prob(self): value = [np.random.rand(*self._paddle_beta.alpha.shape)] @@ -65,7 +71,8 @@ class TestBeta(unittest.TestCase): self._paddle_beta.prob(paddle.to_tensor(v)), scipy.stats.beta.pdf(v, self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_log_prob(self): value = [np.random.rand(*self._paddle_beta.alpha.shape)] @@ -76,7 +83,8 @@ class TestBeta(unittest.TestCase): self._paddle_beta.log_prob(paddle.to_tensor(v)), scipy.stats.beta.logpdf(v, self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_entropy(self): with paddle.fluid.dygraph.guard(self.place): @@ -84,23 +92,26 @@ class TestBeta(unittest.TestCase): self._paddle_beta.entropy(), scipy.stats.beta.entropy(self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_sample_shape(self): cases = [ { 'input': [], - 'expect': [] + paddle.squeeze(self._paddle_beta.alpha).shape + 'expect': [] + paddle.squeeze(self._paddle_beta.alpha).shape, }, { 'input': [2, 3], - 'expect': [2, 3] + paddle.squeeze(self._paddle_beta.alpha).shape + 'expect': [2, 3] + + paddle.squeeze(self._paddle_beta.alpha).shape, }, ] for case in cases: self.assertTrue( - self._paddle_beta.sample(case.get('input')).shape == case.get( - 'expect')) + self._paddle_beta.sample(case.get('input')).shape + == case.get('expect') + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta_static.py index 8976293e36c..d6976f8b685 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta_static.py @@ -27,100 +27,119 @@ paddle.enable_static() @param.place(config.DEVICES) -@param.parameterize_cls((param.TEST_CASE_NAME, 'alpha', 'beta'), - [('test-tensor', xrand((10, 10)), xrand((10, 10))), - ('test-broadcast', xrand((2, 1)), xrand((2, 5))), - ('test-larger-data', xrand((10, 20)), xrand( - (10, 20)))]) +@param.parameterize_cls( + (param.TEST_CASE_NAME, 'alpha', 'beta'), + [ + ('test-tensor', xrand((10, 10)), xrand((10, 10))), + ('test-broadcast', xrand((2, 1)), xrand((2, 5))), + ('test-larger-data', xrand((10, 20)), xrand((10, 20))), + ], +) class TestBeta(unittest.TestCase): - def setUp(self): self.program = paddle.static.Program() self.executor = paddle.static.Executor(self.place) with paddle.static.program_guard(self.program): # scale no need convert to tensor for scale input unittest - alpha = paddle.static.data('alpha', self.alpha.shape, - self.alpha.dtype) + alpha = paddle.static.data( + 'alpha', self.alpha.shape, self.alpha.dtype + ) beta = paddle.static.data('beta', self.beta.shape, self.beta.dtype) self._paddle_beta = paddle.distribution.Beta(alpha, beta) self.feeds = {'alpha': self.alpha, 'beta': self.beta} def test_mean(self): with paddle.static.program_guard(self.program): - [mean] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_beta.mean]) - np.testing.assert_allclose(mean, - scipy.stats.beta.mean( - self.alpha, self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [mean] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_beta.mean], + ) + np.testing.assert_allclose( + mean, + scipy.stats.beta.mean(self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_variance(self): with paddle.static.program_guard(self.program): - [variance - ] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_beta.variance]) - np.testing.assert_allclose(variance, - scipy.stats.beta.var( - self.alpha, self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [variance] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_beta.variance], + ) + np.testing.assert_allclose( + variance, + scipy.stats.beta.var(self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_prob(self): with paddle.static.program_guard(self.program): - value = paddle.static.data('value', self._paddle_beta.alpha.shape, - self._paddle_beta.alpha.dtype) + value = paddle.static.data( + 'value', + self._paddle_beta.alpha.shape, + self._paddle_beta.alpha.dtype, + ) prob = self._paddle_beta.prob(value) random_number = np.random.rand(*self._paddle_beta.alpha.shape) feeds = dict(self.feeds, value=random_number) - [prob] = self.executor.run(self.program, - feed=feeds, - fetch_list=[prob]) - np.testing.assert_allclose(prob, - scipy.stats.beta.pdf( - random_number, self.alpha, - self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [prob] = self.executor.run( + self.program, feed=feeds, fetch_list=[prob] + ) + np.testing.assert_allclose( + prob, + scipy.stats.beta.pdf(random_number, self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_log_prob(self): with paddle.static.program_guard(self.program): - value = paddle.static.data('value', self._paddle_beta.alpha.shape, - self._paddle_beta.alpha.dtype) + value = paddle.static.data( + 'value', + self._paddle_beta.alpha.shape, + self._paddle_beta.alpha.dtype, + ) prob = self._paddle_beta.log_prob(value) random_number = np.random.rand(*self._paddle_beta.alpha.shape) feeds = dict(self.feeds, value=random_number) - [prob] = self.executor.run(self.program, - feed=feeds, - fetch_list=[prob]) - np.testing.assert_allclose(prob, - scipy.stats.beta.logpdf( - random_number, self.alpha, - self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [prob] = self.executor.run( + self.program, feed=feeds, fetch_list=[prob] + ) + np.testing.assert_allclose( + prob, + scipy.stats.beta.logpdf(random_number, self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_entropy(self): with paddle.static.program_guard(self.program): - [entropy - ] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_beta.entropy()]) - np.testing.assert_allclose(entropy, - scipy.stats.beta.entropy( - self.alpha, self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [entropy] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_beta.entropy()], + ) + np.testing.assert_allclose( + entropy, + scipy.stats.beta.entropy(self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_sample(self): with paddle.static.program_guard(self.program): - [data] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=self._paddle_beta.sample()) - self.assertTrue(data.shape, - np.broadcast_arrays(self.alpha, self.beta)[0].shape) + [data] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=self._paddle_beta.sample(), + ) + self.assertTrue( + data.shape, np.broadcast_arrays(self.alpha, self.beta)[0].shape + ) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_categorical.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_categorical.py index 4be500b6084..0ccfb1e0769 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_categorical.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_categorical.py @@ -26,7 +26,6 @@ np.random.seed(2022) class CategoricalNumpy(DistributionNumpy): - def __init__(self, logits): self.logits = np.array(logits).astype('float32') @@ -35,25 +34,26 @@ class CategoricalNumpy(DistributionNumpy): e_logits = np.exp(logits) z = np.sum(e_logits, axis=-1, keepdims=True) prob = e_logits / z - return -1. * np.sum(prob * (logits - np.log(z)), axis=-1) + return -1.0 * np.sum(prob * (logits - np.log(z)), axis=-1) def kl_divergence(self, other): logits = self.logits - np.max(self.logits, axis=-1, keepdims=True) other_logits = other.logits - np.max( - other.logits, axis=-1, keepdims=True) + other.logits, axis=-1, keepdims=True + ) e_logits = np.exp(logits) other_e_logits = np.exp(other_logits) z = np.sum(e_logits, axis=-1, keepdims=True) other_z = np.sum(other_e_logits, axis=-1, keepdims=True) prob = e_logits / z - return np.sum(prob * - (logits - np.log(z) - other_logits + np.log(other_z)), - axis=-1, - keepdims=True) + return np.sum( + prob * (logits - np.log(z) - other_logits + np.log(other_z)), + axis=-1, + keepdims=True, + ) class CategoricalTest(unittest.TestCase): - def setUp(self, use_gpu=False, batch_size=3, dims=5): self.use_gpu = use_gpu if not use_gpu: @@ -79,8 +79,9 @@ class CategoricalTest(unittest.TestCase): # input logtis is 2-D Tensor # value used in probs and log_prob method is 1-D Tensor self.logits_np = np.random.rand(batch_size, dims).astype('float32') - self.other_logits_np = np.random.rand(batch_size, - dims).astype('float32') + self.other_logits_np = np.random.rand(batch_size, dims).astype( + 'float32' + ) self.value_np = np.array([2, 1, 3]).astype('int64') self.logits_shape = [batch_size, dims] @@ -103,15 +104,15 @@ class CategoricalTest(unittest.TestCase): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.logits_static = fluid.data(name='logits', - shape=self.logits_shape, - dtype='float32') - self.other_logits_static = fluid.data(name='other_logits', - shape=self.logits_shape, - dtype='float32') - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.logits_static = fluid.data( + name='logits', shape=self.logits_shape, dtype='float32' + ) + self.other_logits_static = fluid.data( + name='other_logits', shape=self.logits_shape, dtype='float32' + ) + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) def get_numpy_selected_probs(self, probability): np_probs = np.zeros(self.dist_shape + self.value_shape) @@ -124,36 +125,33 @@ class CategoricalTest(unittest.TestCase): sample, entropy, kl, probs, log_prob = fetch_list log_tolerance = 1e-4 - np.testing.assert_equal(sample.shape, - self.sample_shape + self.dist_shape) + np.testing.assert_equal( + sample.shape, self.sample_shape + self.dist_shape + ) np_categorical = CategoricalNumpy(self.logits_np) np_other_categorical = CategoricalNumpy(self.other_logits_np) np_entropy = np_categorical.entropy() np_kl = np_categorical.kl_divergence(np_other_categorical) - np.testing.assert_allclose(entropy, - np_entropy, - rtol=log_tolerance, - atol=log_tolerance) - np.testing.assert_allclose(kl, - np_kl, - rtol=log_tolerance, - atol=log_tolerance) + np.testing.assert_allclose( + entropy, np_entropy, rtol=log_tolerance, atol=log_tolerance + ) + np.testing.assert_allclose( + kl, np_kl, rtol=log_tolerance, atol=log_tolerance + ) sum_dist = np.sum(self.logits_np, axis=-1, keepdims=True) probability = self.logits_np / sum_dist np_probs = self.get_numpy_selected_probs(probability) np_log_prob = np.log(np_probs) - np.testing.assert_allclose(probs, - np_probs, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(log_prob, - np_log_prob, - rtol=tolerance, - atol=tolerance) + np.testing.assert_allclose( + probs, np_probs, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + log_prob, np_log_prob, rtol=tolerance, atol=tolerance + ) def test_categorical_distribution_dygraph(self, tolerance=1e-6): paddle.disable_static(self.place) @@ -186,25 +184,25 @@ class CategoricalTest(unittest.TestCase): feed_vars = { 'logits': self.logits_np, 'other_logits': self.other_logits_np, - 'value': self.value_np + 'value': self.value_np, } self.executor.run(fluid.default_startup_program()) - fetch_list = self.executor.run(program=self.test_program, - feed=feed_vars, - fetch_list=fetch_list) + fetch_list = self.executor.run( + program=self.test_program, feed=feed_vars, fetch_list=fetch_list + ) self.compare_with_numpy(fetch_list) class CategoricalTest2(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 2-D Tensor with dtype Float64 # value used in probs and log_prob method is 1-D Tensor self.logits_np = np.random.rand(batch_size, dims).astype('float64') - self.other_logits_np = np.random.rand(batch_size, - dims).astype('float64') + self.other_logits_np = np.random.rand(batch_size, dims).astype( + 'float64' + ) self.value_np = np.array([2, 1, 3]).astype('int64') self.logits_shape = [batch_size, dims] @@ -214,19 +212,18 @@ class CategoricalTest2(CategoricalTest): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.logits_static = fluid.data(name='logits', - shape=self.logits_shape, - dtype='float64') - self.other_logits_static = fluid.data(name='other_logits', - shape=self.logits_shape, - dtype='float64') - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.logits_static = fluid.data( + name='logits', shape=self.logits_shape, dtype='float64' + ) + self.other_logits_static = fluid.data( + name='other_logits', shape=self.logits_shape, dtype='float64' + ) + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) class CategoricalTest3(CategoricalTest): - def init_dynamic_data(self, batch_size, dims): # input logtis is 2-D numpy.ndarray with dtype Float32 # value used in probs and log_prob method is 1-D Tensor @@ -238,19 +235,19 @@ class CategoricalTest3(CategoricalTest): with fluid.program_guard(self.test_program): self.logits_static = self.logits_np self.other_logits_static = self.other_logits_np - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) class CategoricalTest4(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 2-D numpy.ndarray with dtype Float64 # value used in probs and log_prob method is 1-D Tensor self.logits_np = np.random.rand(batch_size, dims).astype('float64') - self.other_logits_np = np.random.rand(batch_size, - dims).astype('float64') + self.other_logits_np = np.random.rand(batch_size, dims).astype( + 'float64' + ) self.value_np = np.array([2, 1, 3]).astype('int64') self.logits_shape = [batch_size, dims] @@ -267,14 +264,13 @@ class CategoricalTest4(CategoricalTest): with fluid.program_guard(self.test_program): self.logits_static = self.logits_np self.other_logits_static = self.other_logits_np - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) # test shape of logits and value used in probs and log_prob method class CategoricalTest5(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 1-D Tensor # value used in probs and log_prob method is 1-D Tensor @@ -295,7 +291,6 @@ class CategoricalTest5(CategoricalTest): class CategoricalTest6(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 2-D Tensor # value used in probs and log_prob method has the same number of batches with input @@ -317,7 +312,6 @@ class CategoricalTest6(CategoricalTest): class CategoricalTest7(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 3-D Tensor # value used in probs and log_prob method has the same number of distribuions with input @@ -340,7 +334,6 @@ class CategoricalTest7(CategoricalTest): class CategoricalTest8(CategoricalTest): - def init_dynamic_data(self, batch_size, dims): # input logtis is 2-D list # value used in probs and log_prob method is 1-D Tensor @@ -352,13 +345,12 @@ class CategoricalTest8(CategoricalTest): with fluid.program_guard(self.test_program): self.logits_static = self.logits_np.tolist() self.other_logits_static = self.other_logits_np.tolist() - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) class CategoricalTest9(CategoricalTest): - def init_dynamic_data(self, batch_size, dims): # input logtis is 2-D tuple # value used in probs and log_prob method is 1-D Tensor @@ -370,13 +362,12 @@ class CategoricalTest9(CategoricalTest): with fluid.program_guard(self.test_program): self.logits_static = tuple(self.logits_np.tolist()) self.other_logits_static = tuple(self.other_logits_np.tolist()) - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) class DistributionTestError(unittest.TestCase): - def test_distribution_error(self): distribution = Distribution() @@ -384,13 +375,15 @@ class DistributionTestError(unittest.TestCase): self.assertRaises(NotImplementedError, distribution.entropy) normal = Normal(0.0, 1.0) - self.assertRaises(NotImplementedError, distribution.kl_divergence, - normal) + self.assertRaises( + NotImplementedError, distribution.kl_divergence, normal + ) value_npdata = np.array([0.8], dtype="float32") value_tensor = layers.create_tensor(dtype="float32") - self.assertRaises(NotImplementedError, distribution.log_prob, - value_tensor) + self.assertRaises( + NotImplementedError, distribution.log_prob, value_tensor + ) self.assertRaises(NotImplementedError, distribution.probs, value_tensor) def test_normal_error(self): @@ -456,8 +449,9 @@ class DistributionTestError(unittest.TestCase): categorical_other = Uniform(1.0, 2.0) # type of other must be an instance of Categorical - self.assertRaises(TypeError, categorical.kl_divergence, - categorical_other) + self.assertRaises( + TypeError, categorical.kl_divergence, categorical_other + ) def test_shape_not_match_error(): # shape of value must match shape of logits diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_constraint.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_constraint.py index cbf0431d9f3..3ef352d54bf 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_constraint.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_constraint.py @@ -23,10 +23,10 @@ import parameterize as param np.random.seed(2022) -@param.param_cls((param.TEST_CASE_NAME, 'value'), - [('NotImplement', np.random.rand(2, 3))]) +@param.param_cls( + (param.TEST_CASE_NAME, 'value'), [('NotImplement', np.random.rand(2, 3))] +) class TestConstraint(unittest.TestCase): - def setUp(self): self._constraint = constraint.Constraint() @@ -35,10 +35,10 @@ class TestConstraint(unittest.TestCase): self._constraint(self.value) -@param.param_cls((param.TEST_CASE_NAME, 'value', 'expect'), - [('real', 1., True)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'value', 'expect'), [('real', 1.0, True)] +) class TestReal(unittest.TestCase): - def setUp(self): self._constraint = constraint.Real() @@ -46,10 +46,11 @@ class TestReal(unittest.TestCase): self.assertEqual(self._constraint(self.value), self.expect) -@param.param_cls((param.TEST_CASE_NAME, 'lower', 'upper', 'value', 'expect'), - [('in_range', 0, 1, 0.5, True), ('out_range', 0, 1, 2, False)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'lower', 'upper', 'value', 'expect'), + [('in_range', 0, 1, 0.5, True), ('out_range', 0, 1, 2, False)], +) class TestRange(unittest.TestCase): - def setUp(self): self._constraint = constraint.Range(self.lower, self.upper) @@ -57,10 +58,11 @@ class TestRange(unittest.TestCase): self.assertEqual(self._constraint(self.value), self.expect) -@param.param_cls((param.TEST_CASE_NAME, 'value', 'expect'), - [('positive', 1, True), ('negative', -1, False)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'value', 'expect'), + [('positive', 1, True), ('negative', -1, False)], +) class TestPositive(unittest.TestCase): - def setUp(self): self._constraint = constraint.Positive() @@ -68,11 +70,14 @@ class TestPositive(unittest.TestCase): self.assertEqual(self._constraint(self.value), self.expect) -@param.param_cls((param.TEST_CASE_NAME, 'value', 'expect'), - [('simplex', paddle.to_tensor([0.5, 0.5]), True), - ('non_simplex', paddle.to_tensor([-0.5, 0.5]), False)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'value', 'expect'), + [ + ('simplex', paddle.to_tensor([0.5, 0.5]), True), + ('non_simplex', paddle.to_tensor([-0.5, 0.5]), False), + ], +) class TestSimplex(unittest.TestCase): - def setUp(self): self._constraint = constraint.Simplex() diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet.py index cb5534edcd5..399fe66b0a0 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet.py @@ -28,14 +28,15 @@ np.random.seed(2022) @param.param_cls( (param.TEST_CASE_NAME, 'concentration'), [ - ('test-one-dim', param.xrand((89, ))), + ('test-one-dim', param.xrand((89,))), # ('test-multi-dim', config.xrand((10, 20, 30))) - ]) + ], +) class TestDirichlet(unittest.TestCase): - def setUp(self): self._paddle_diric = paddle.distribution.Dirichlet( - paddle.to_tensor(self.concentration)) + paddle.to_tensor(self.concentration) + ) def test_mean(self): with paddle.fluid.dygraph.guard(self.place): @@ -43,7 +44,8 @@ class TestDirichlet(unittest.TestCase): self._paddle_diric.mean, scipy.stats.dirichlet.mean(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_variance(self): with paddle.fluid.dygraph.guard(self.place): @@ -51,7 +53,8 @@ class TestDirichlet(unittest.TestCase): self._paddle_diric.variance, scipy.stats.dirichlet.var(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_prob(self): value = [np.random.rand(*self.concentration.shape)] @@ -63,7 +66,8 @@ class TestDirichlet(unittest.TestCase): self._paddle_diric.prob(paddle.to_tensor(v)), scipy.stats.dirichlet.pdf(v, self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_log_prob(self): value = [np.random.rand(*self.concentration.shape)] @@ -75,7 +79,8 @@ class TestDirichlet(unittest.TestCase): self._paddle_diric.log_prob(paddle.to_tensor(v)), scipy.stats.dirichlet.logpdf(v, self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_entropy(self): with paddle.fluid.dygraph.guard(self.place): @@ -83,28 +88,35 @@ class TestDirichlet(unittest.TestCase): self._paddle_diric.entropy(), scipy.stats.dirichlet.entropy(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_natural_parameters(self): self.assertTrue( - isinstance(self._paddle_diric._natural_parameters, tuple)) + isinstance(self._paddle_diric._natural_parameters, tuple) + ) def test_log_normalizer(self): self.assertTrue( np.all( self._paddle_diric._log_normalizer( - paddle.to_tensor(param.xrand((100, 100, - 100)))).numpy() < 0.0)) + paddle.to_tensor(param.xrand((100, 100, 100))) + ).numpy() + < 0.0 + ) + ) @param.place(DEVICES) - @param.param_cls((param.TEST_CASE_NAME, 'concentration'), - [('test-zero-dim', np.array(1.0))]) + @param.param_cls( + (param.TEST_CASE_NAME, 'concentration'), + [('test-zero-dim', np.array(1.0))], + ) class TestDirichletException(unittest.TestCase): - def TestInit(self): with self.assertRaises(ValueError): - paddle.distribution.Dirichlet(paddle.squeeze( - self.concentration)) + paddle.distribution.Dirichlet( + paddle.squeeze(self.concentration) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet_static.py index 8b0d847ad16..edf66a26e02 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet_static.py @@ -26,82 +26,97 @@ paddle.enable_static() @place(DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'concentration'), - [('test-one-dim', np.random.rand(89) + 5.0)]) +@parameterize_cls( + (TEST_CASE_NAME, 'concentration'), + [('test-one-dim', np.random.rand(89) + 5.0)], +) class TestDirichlet(unittest.TestCase): - def setUp(self): self.program = paddle.static.Program() self.executor = paddle.static.Executor() with paddle.static.program_guard(self.program): - conc = paddle.static.data('conc', self.concentration.shape, - self.concentration.dtype) + conc = paddle.static.data( + 'conc', self.concentration.shape, self.concentration.dtype + ) self._paddle_diric = paddle.distribution.Dirichlet(conc) self.feeds = {'conc': self.concentration} def test_mean(self): with paddle.static.program_guard(self.program): - [out] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_diric.mean]) + [out] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_diric.mean], + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.mean(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_variance(self): with paddle.static.program_guard(self.program): - [out] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_diric.variance]) + [out] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_diric.variance], + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.var(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_prob(self): with paddle.static.program_guard(self.program): random_number = np.random.rand(*self.concentration.shape) random_number = random_number / random_number.sum() feeds = dict(self.feeds, value=random_number) - value = paddle.static.data('value', random_number.shape, - random_number.dtype) + value = paddle.static.data( + 'value', random_number.shape, random_number.dtype + ) out = self._paddle_diric.prob(value) - [out] = self.executor.run(self.program, - feed=feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.program, feed=feeds, fetch_list=[out] + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.pdf(random_number, self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_log_prob(self): with paddle.static.program_guard(self.program): random_number = np.random.rand(*self.concentration.shape) random_number = random_number / random_number.sum() feeds = dict(self.feeds, value=random_number) - value = paddle.static.data('value', random_number.shape, - random_number.dtype) + value = paddle.static.data( + 'value', random_number.shape, random_number.dtype + ) out = self._paddle_diric.log_prob(value) - [out] = self.executor.run(self.program, - feed=feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.program, feed=feeds, fetch_list=[out] + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.logpdf(random_number, self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_entropy(self): with paddle.static.program_guard(self.program): - [out] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_diric.entropy()]) + [out] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_diric.entropy()], + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.entropy(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily.py index ca54c978ca4..001696385e0 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily.py @@ -27,30 +27,46 @@ np.random.seed(2022) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'dist'), - [('test-mock-exp', - mock.Exponential( - rate=paddle.rand([100, 200, 99], dtype=config.DEFAULT_DTYPE)))]) + [ + ( + 'test-mock-exp', + mock.Exponential( + rate=paddle.rand([100, 200, 99], dtype=config.DEFAULT_DTYPE) + ), + ) + ], +) class TestExponentialFamily(unittest.TestCase): - def test_entropy(self): np.testing.assert_allclose( self.dist.entropy(), paddle.distribution.ExponentialFamily.entropy(self.dist), rtol=config.RTOL.get(config.DEFAULT_DTYPE), - atol=config.ATOL.get(config.DEFAULT_DTYPE)) + atol=config.ATOL.get(config.DEFAULT_DTYPE), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( (config.TEST_CASE_NAME, 'dist'), - [('test-dummy', mock.DummyExpFamily(0.5, 0.5)), - ('test-dirichlet', - paddle.distribution.Dirichlet(paddle.to_tensor(parameterize.xrand()))), - ('test-beta', - paddle.distribution.Beta(paddle.to_tensor(parameterize.xrand()), - paddle.to_tensor(parameterize.xrand())))]) + [ + ('test-dummy', mock.DummyExpFamily(0.5, 0.5)), + ( + 'test-dirichlet', + paddle.distribution.Dirichlet( + paddle.to_tensor(parameterize.xrand()) + ), + ), + ( + 'test-beta', + paddle.distribution.Beta( + paddle.to_tensor(parameterize.xrand()), + paddle.to_tensor(parameterize.xrand()), + ), + ), + ], +) class TestExponentialFamilyException(unittest.TestCase): - def test_entropy_exception(self): with self.assertRaises(NotImplementedError): paddle.distribution.ExponentialFamily.entropy(self.dist) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily_static.py index 2d1493a4e52..7c2fe5b5a4d 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily_static.py @@ -27,7 +27,6 @@ paddle.enable_static() @parameterize.place(config.DEVICES) class TestExponentialFamily(unittest.TestCase): - def setUp(self): self.program = paddle.static.Program() self.executor = paddle.static.Executor() @@ -45,17 +44,21 @@ class TestExponentialFamily(unittest.TestCase): fetch_list=[ self.mock_dist.entropy(), paddle.distribution.ExponentialFamily.entropy( - self.mock_dist) - ]) + self.mock_dist + ), + ], + ) np.testing.assert_allclose( out1, out2, rtol=config.RTOL.get(config.DEFAULT_DTYPE), - atol=config.ATOL.get(config.DEFAULT_DTYPE)) + atol=config.ATOL.get(config.DEFAULT_DTYPE), + ) def test_entropy_exception(self): with paddle.static.program_guard(self.program): with self.assertRaises(NotImplementedError): paddle.distribution.ExponentialFamily.entropy( - mock.DummyExpFamily(0.5, 0.5)) + mock.DummyExpFamily(0.5, 0.5) + ) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel.py index 3f3cfd8e8bb..3aa7c5e6436 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel.py @@ -24,47 +24,58 @@ from paddle.distribution.gumbel import Gumbel @parameterize.place(config.DEVICES) -@parameterize.parameterize_cls((parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('one-dim', parameterize.xrand((4, )), parameterize.xrand((4, ))), - ('multi-dim', parameterize.xrand((5, 3)), parameterize.xrand((5, 3))), -]) +@parameterize.parameterize_cls( + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('one-dim', parameterize.xrand((4,)), parameterize.xrand((4,))), + ('multi-dim', parameterize.xrand((5, 3)), parameterize.xrand((5, 3))), + ], +) class TestGumbel(unittest.TestCase): - def setUp(self): - self._dist = Gumbel(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(self.scale)) + self._dist = Gumbel( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) def test_mean(self): mean = self._dist.mean self.assertEqual(mean.numpy().dtype, self._np_mean().dtype) - np.testing.assert_allclose(mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): var = self._dist.variance self.assertEqual(var.numpy().dtype, self._np_variance().dtype) - np.testing.assert_allclose(var, - self._np_variance(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + var, + self._np_variance(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_stddev(self): stddev = self._dist.stddev self.assertEqual(stddev.numpy().dtype, self._np_stddev().dtype) - np.testing.assert_allclose(stddev, - self._np_stddev(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + stddev, + self._np_stddev(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): entropy = self._dist.entropy() self.assertEqual(entropy.numpy().dtype, self._np_entropy().dtype) - np.testing.assert_allclose(entropy, - self._np_entropy(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + entropy, + self._np_entropy(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_sample(self): @@ -73,16 +84,18 @@ class TestGumbel(unittest.TestCase): sample_values = samples.numpy() self.assertEqual(sample_values.dtype, self.scale.dtype) - np.testing.assert_allclose(sample_values.mean(axis=0), - scipy.stats.gumbel_r.mean(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.loc.dtype))) - np.testing.assert_allclose(sample_values.var(axis=0), - scipy.stats.gumbel_r.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + sample_values.mean(axis=0), + scipy.stats.gumbel_r.mean(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.loc.dtype)), + ) + np.testing.assert_allclose( + sample_values.var(axis=0), + scipy.stats.gumbel_r.var(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_rsample(self): @@ -91,16 +104,18 @@ class TestGumbel(unittest.TestCase): sample_values = samples.numpy() self.assertEqual(sample_values.dtype, self.scale.dtype) - np.testing.assert_allclose(sample_values.mean(axis=0), - scipy.stats.gumbel_r.mean(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.loc.dtype))) - np.testing.assert_allclose(sample_values.var(axis=0), - scipy.stats.gumbel_r.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + sample_values.mean(axis=0), + scipy.stats.gumbel_r.mean(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.loc.dtype)), + ) + np.testing.assert_allclose( + sample_values.var(axis=0), + scipy.stats.gumbel_r.var(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.loc.dtype)), + ) def _np_mean(self): return self.loc + self.scale * np.euler_gamma @@ -110,7 +125,8 @@ class TestGumbel(unittest.TestCase): def _np_variance(self): return np.divide( - np.multiply(np.power(self.scale, 2), np.power(np.pi, 2)), 6) + np.multiply(np.power(self.scale, 2), np.power(np.pi, 2)), 6 + ) def _np_entropy(self): return np.log(self.scale) + 1 + np.euler_gamma @@ -118,39 +134,52 @@ class TestGumbel(unittest.TestCase): @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), [ - ('value-float', np.array([0.1, 0.4]), np.array([1., 4. - ]), np.array([3., 7.])), + (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), + [ + ( + 'value-float', + np.array([0.1, 0.4]), + np.array([1.0, 4.0]), + np.array([3.0, 7.0]), + ), ('value-int', np.array([0.1, 0.4]), np.array([1, 4]), np.array([3, 7])), - ('value-multi-dim', np.array([0.1, 0.4]), np.array( - [1, 4]), np.array([[5., 4], [6, 2]])), - ]) + ( + 'value-multi-dim', + np.array([0.1, 0.4]), + np.array([1, 4]), + np.array([[5.0, 4], [6, 2]]), + ), + ], +) class TestGumbelPDF(unittest.TestCase): - def setUp(self): - self._dist = Gumbel(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(self.scale)) + self._dist = Gumbel( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) def test_prob(self): np.testing.assert_allclose( self._dist.prob(paddle.to_tensor(self.value)), scipy.stats.gumbel_r.pdf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_log_prob(self): np.testing.assert_allclose( self._dist.log_prob(paddle.to_tensor(self.value)), scipy.stats.gumbel_r.logpdf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_cdf(self): - np.testing.assert_allclose(self._dist.cdf(paddle.to_tensor(self.value)), - scipy.stats.gumbel_r.cdf( - self.value, self.loc, self.scale), - rtol=0.02, - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self._dist.cdf(paddle.to_tensor(self.value)), + scipy.stats.gumbel_r.cdf(self.value, self.loc, self.scale), + rtol=0.02, + atol=config.ATOL.get(str(self.loc.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel_static.py index 1c39da5fccf..7e0176dee56 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel_static.py @@ -27,20 +27,23 @@ paddle.enable_static() @parameterize.place(config.DEVICES) -@parameterize.parameterize_cls((parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('one-dim', parameterize.xrand((4, )), parameterize.xrand((4, ))), - ('multi-dim', parameterize.xrand((5, 3)), parameterize.xrand((5, 3))), -]) +@parameterize.parameterize_cls( + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('one-dim', parameterize.xrand((4,)), parameterize.xrand((4,))), + ('multi-dim', parameterize.xrand((5, 3)), parameterize.xrand((5, 3))), + ], +) class TestGumbel(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) self._dist = Gumbel(loc=loc, scale=scale) self.sample_shape = [50000] mean = self._dist.mean @@ -52,50 +55,63 @@ class TestGumbel(unittest.TestCase): self.feeds = {'loc': self.loc, 'scale': self.scale} executor.run(startup_program) - [self.mean, self.var, self.stddev, self.entropy, - self.samples] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [ + self.mean, + self.var, + self.stddev, + self.entropy, + self.samples, + ] = executor.run(main_program, feed=self.feeds, fetch_list=fetch_list) def test_mean(self): self.assertEqual(str(self.mean.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): self.assertEqual(str(self.var.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.var, - self._np_variance(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.var, + self._np_variance(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_stddev(self): self.assertEqual( - str(self.stddev.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.stddev, - self._np_stddev(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + str(self.stddev.dtype).split('.')[-1], self.scale.dtype + ) + np.testing.assert_allclose( + self.stddev, + self._np_stddev(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): self.assertEqual( - str(self.entropy.dtype).split('.')[-1], self.scale.dtype) + str(self.entropy.dtype).split('.')[-1], self.scale.dtype + ) def test_sample(self): self.assertEqual(self.samples.dtype, self.scale.dtype) - np.testing.assert_allclose(self.samples.mean(axis=0), - scipy.stats.gumbel_r.mean(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.scale.dtype))) - np.testing.assert_allclose(self.samples.var(axis=0), - scipy.stats.gumbel_r.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.samples.mean(axis=0), + scipy.stats.gumbel_r.mean(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.scale.dtype)), + ) + np.testing.assert_allclose( + self.samples.var(axis=0), + scipy.stats.gumbel_r.var(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.scale.dtype)), + ) def _np_mean(self): return self.loc + self.scale * np.euler_gamma @@ -105,7 +121,8 @@ class TestGumbel(unittest.TestCase): def _np_variance(self): return np.divide( - np.multiply(np.power(self.scale, 2), np.power(np.pi, 2)), 6) + np.multiply(np.power(self.scale, 2), np.power(np.pi, 2)), 6 + ) def _np_entropy(self): return np.log(self.scale) + 1 + np.euler_gamma @@ -113,15 +130,24 @@ class TestGumbel(unittest.TestCase): @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), [ - ('value-float', np.array([0.1, 0.4]), np.array([1., 4. - ]), np.array([3., 7.])), + (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), + [ + ( + 'value-float', + np.array([0.1, 0.4]), + np.array([1.0, 4.0]), + np.array([3.0, 7.0]), + ), ('value-int', np.array([0.1, 0.4]), np.array([1, 4]), np.array([3, 7])), - ('value-multi-dim', np.array([0.1, 0.4]), np.array( - [1, 4]), np.array([[5., 4], [6, 2]])), - ]) + ( + 'value-multi-dim', + np.array([0.1, 0.4]), + np.array([1, 4]), + np.array([[5.0, 4], [6, 2]]), + ), + ], +) class TestGumbelPDF(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() @@ -129,10 +155,12 @@ class TestGumbelPDF(unittest.TestCase): with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) - value = paddle.static.data('value', self.value.shape, - self.value.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) + value = paddle.static.data( + 'value', self.value.shape, self.value.dtype + ) self._dist = Gumbel(loc=loc, scale=scale) prob = self._dist.prob(value) log_prob = self._dist.log_prob(value) @@ -141,31 +169,33 @@ class TestGumbelPDF(unittest.TestCase): self.feeds = {'loc': self.loc, 'scale': self.scale, 'value': self.value} executor.run(startup_program) - [self.prob, self.log_prob, - self.cdf] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.prob, self.log_prob, self.cdf] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_prob(self): - np.testing.assert_allclose(self.prob, - scipy.stats.gumbel_r.pdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.prob, + scipy.stats.gumbel_r.pdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_log_prob(self): - np.testing.assert_allclose(self.log_prob, - scipy.stats.gumbel_r.logpdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.log_prob, + scipy.stats.gumbel_r.logpdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_cdf(self): - np.testing.assert_allclose(self.cdf, - scipy.stats.gumbel_r.cdf( - self.value, self.loc, self.scale), - rtol=0.3, - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.cdf, + scipy.stats.gumbel_r.cdf(self.value, self.loc, self.scale), + rtol=0.3, + atol=config.ATOL.get(str(self.loc.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent.py index eef5b5a6e44..c4c83ecb12b 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent.py @@ -25,35 +25,45 @@ np.random.seed(2022) @param.place(config.DEVICES) @param.param_cls( (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank'), - [('base_beta', - paddle.distribution.Beta(paddle.rand([1, 2]), paddle.rand([1, 2])), 1)]) + [ + ( + 'base_beta', + paddle.distribution.Beta(paddle.rand([1, 2]), paddle.rand([1, 2])), + 1, + ) + ], +) class TestIndependent(unittest.TestCase): - def setUp(self): - self._t = paddle.distribution.Independent(self.base, - self.reinterpreted_batch_rank) + self._t = paddle.distribution.Independent( + self.base, self.reinterpreted_batch_rank + ) def test_mean(self): np.testing.assert_allclose( self.base.mean, self._t.mean, rtol=config.RTOL.get(str(self.base.alpha.numpy().dtype)), - atol=config.ATOL.get(str(self.base.alpha.numpy().dtype))) + atol=config.ATOL.get(str(self.base.alpha.numpy().dtype)), + ) def test_variance(self): np.testing.assert_allclose( self.base.variance, self._t.variance, rtol=config.RTOL.get(str(self.base.alpha.numpy().dtype)), - atol=config.ATOL.get(str(self.base.alpha.numpy().dtype))) + atol=config.ATOL.get(str(self.base.alpha.numpy().dtype)), + ) def test_entropy(self): np.testing.assert_allclose( - self._np_sum_rightmost(self.base.entropy().numpy(), - self.reinterpreted_batch_rank), + self._np_sum_rightmost( + self.base.entropy().numpy(), self.reinterpreted_batch_rank + ), self._t.entropy(), rtol=config.RTOL.get(str(self.base.alpha.numpy().dtype)), - atol=config.ATOL.get(str(self.base.alpha.numpy().dtype))) + atol=config.ATOL.get(str(self.base.alpha.numpy().dtype)), + ) def _np_sum_rightmost(self, value, n): return np.sum(value, tuple(range(-n, 0))) if n > 0 else value @@ -63,10 +73,12 @@ class TestIndependent(unittest.TestCase): np.testing.assert_allclose( self._np_sum_rightmost( self.base.log_prob(paddle.to_tensor(value)).numpy(), - self.reinterpreted_batch_rank), + self.reinterpreted_batch_rank, + ), self._t.log_prob(paddle.to_tensor(value)).numpy(), rtol=config.RTOL.get(str(self.base.alpha.numpy().dtype)), - atol=config.ATOL.get(str(self.base.alpha.numpy().dtype))) + atol=config.ATOL.get(str(self.base.alpha.numpy().dtype)), + ) # TODO(cxxly): Add Kolmogorov-Smirnov test for sample result. def test_sample(self): @@ -79,16 +91,28 @@ class TestIndependent(unittest.TestCase): @param.place(config.DEVICES) @param.param_cls( - (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', - 'expected_exception'), - [('base_not_transform', '', 1, TypeError), - ('rank_less_than_zero', paddle.distribution.Transform(), -1, ValueError)]) + ( + param.TEST_CASE_NAME, + 'base', + 'reinterpreted_batch_rank', + 'expected_exception', + ), + [ + ('base_not_transform', '', 1, TypeError), + ( + 'rank_less_than_zero', + paddle.distribution.Transform(), + -1, + ValueError, + ), + ], +) class TestIndependentException(unittest.TestCase): - def test_init(self): with self.assertRaises(self.expected_exception): paddle.distribution.IndependentTransform( - self.base, self.reinterpreted_batch_rank) + self.base, self.reinterpreted_batch_rank + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent_static.py index 47bf1021e1a..06881bdcdc2 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent_static.py @@ -26,10 +26,17 @@ paddle.enable_static() @param.place(config.DEVICES) @param.param_cls( (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'alpha', 'beta'), - [('base_beta', paddle.distribution.Beta, 1, np.random.rand( - 1, 2), np.random.rand(1, 2))]) + [ + ( + 'base_beta', + paddle.distribution.Beta, + 1, + np.random.rand(1, 2), + np.random.rand(1, 2), + ) + ], +) class TestIndependent(unittest.TestCase): - def setUp(self): value = np.random.rand(1) self.dtype = value.dtype @@ -37,12 +44,14 @@ class TestIndependent(unittest.TestCase): sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): - alpha = paddle.static.data('alpha', self.alpha.shape, - self.alpha.dtype) + alpha = paddle.static.data( + 'alpha', self.alpha.shape, self.alpha.dtype + ) beta = paddle.static.data('beta', self.beta.shape, self.beta.dtype) self.base = self.base(alpha, beta) - t = paddle.distribution.Independent(self.base, - self.reinterpreted_batch_rank) + t = paddle.distribution.Independent( + self.base, self.reinterpreted_batch_rank + ) mean = t.mean variance = t.variance entropy = t.entropy() @@ -55,50 +64,69 @@ class TestIndependent(unittest.TestCase): base_log_prob = self.base.log_prob(static_value) fetch_list = [ - mean, variance, entropy, log_prob, base_mean, base_variance, - base_entropy, base_log_prob + mean, + variance, + entropy, + log_prob, + base_mean, + base_variance, + base_entropy, + base_log_prob, ] exe.run(sp) [ - self.mean, self.variance, self.entropy, self.log_prob, - self.base_mean, self.base_variance, self.base_entropy, - self.base_log_prob - ] = exe.run(mp, - feed={ - 'value': value, - 'alpha': self.alpha, - 'beta': self.beta - }, - fetch_list=fetch_list) + self.mean, + self.variance, + self.entropy, + self.log_prob, + self.base_mean, + self.base_variance, + self.base_entropy, + self.base_log_prob, + ] = exe.run( + mp, + feed={'value': value, 'alpha': self.alpha, 'beta': self.beta}, + fetch_list=fetch_list, + ) def test_mean(self): - np.testing.assert_allclose(self.mean, - self.base_mean, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self.mean, + self.base_mean, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) def test_variance(self): - np.testing.assert_allclose(self.variance, - self.base_variance, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self.variance, + self.base_variance, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) def test_entropy(self): - np.testing.assert_allclose(self._np_sum_rightmost( - self.base_entropy, self.reinterpreted_batch_rank), - self.entropy, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self._np_sum_rightmost( + self.base_entropy, self.reinterpreted_batch_rank + ), + self.entropy, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) def _np_sum_rightmost(self, value, n): return np.sum(value, tuple(range(-n, 0))) if n > 0 else value def test_log_prob(self): - np.testing.assert_allclose(self._np_sum_rightmost( - self.base_log_prob, self.reinterpreted_batch_rank), - self.log_prob, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self._np_sum_rightmost( + self.base_log_prob, self.reinterpreted_batch_rank + ), + self.log_prob, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace.py index 867fc1846a3..f93e3fac71a 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace.py @@ -23,42 +23,47 @@ import parameterize @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('one-dim', parameterize.xrand((2, )),\ - parameterize.xrand((2, ))), - ('multi-dim', parameterize.xrand((5, 5)),\ - parameterize.xrand((5, 5))), - ]) + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('one-dim', parameterize.xrand((2,)), parameterize.xrand((2,))), + ('multi-dim', parameterize.xrand((5, 5)), parameterize.xrand((5, 5))), + ], +) class TestLaplace(unittest.TestCase): - def setUp(self): - self._dist = paddle.distribution.Laplace(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(\ - self.scale)) + self._dist = paddle.distribution.Laplace( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) def test_mean(self): mean = self._dist.mean self.assertEqual(mean.numpy().dtype, self.scale.dtype) - np.testing.assert_allclose(mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): var = self._dist.variance self.assertEqual(var.numpy().dtype, self.scale.dtype) - np.testing.assert_allclose(var, - self._np_variance(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + var, + self._np_variance(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_stddev(self): stddev = self._dist.stddev self.assertEqual(stddev.numpy().dtype, self.scale.dtype) - np.testing.assert_allclose(stddev, - self._np_stddev(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + stddev, + self._np_stddev(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): entropy = self._dist.entropy() @@ -66,27 +71,30 @@ class TestLaplace(unittest.TestCase): def test_sample(self): - sample_shape = (50000, ) + sample_shape = (50000,) samples = self._dist.sample(sample_shape) sample_values = samples.numpy() self.assertEqual(samples.numpy().dtype, self.scale.dtype) - self.assertEqual(tuple(samples.shape), - tuple(self._dist._extend_shape(sample_shape))) + self.assertEqual( + tuple(samples.shape), tuple(self._dist._extend_shape(sample_shape)) + ) self.assertEqual(samples.shape, list(sample_shape + self.loc.shape)) self.assertEqual(sample_values.shape, sample_shape + self.loc.shape) - np.testing.assert_allclose(sample_values.mean(axis=0), - scipy.stats.laplace.mean(self.loc, - scale=self.scale), - rtol=0.2, - atol=0.) - np.testing.assert_allclose(sample_values.var(axis=0), - scipy.stats.laplace.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=0.) + np.testing.assert_allclose( + sample_values.mean(axis=0), + scipy.stats.laplace.mean(self.loc, scale=self.scale), + rtol=0.2, + atol=0.0, + ) + np.testing.assert_allclose( + sample_values.var(axis=0), + scipy.stats.laplace.var(self.loc, scale=self.scale), + rtol=0.1, + atol=0.0, + ) def _np_mean(self): return self.loc @@ -103,18 +111,20 @@ class TestLaplace(unittest.TestCase): @parameterize.place(config.DEVICES) -@parameterize.parameterize_cls((parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('float', 1., 2.), - ('int', 3, 4), -]) +@parameterize.parameterize_cls( + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('float', 1.0, 2.0), + ('int', 3, 4), + ], +) class TestLaplaceKS(unittest.TestCase): - def setUp(self): self._dist = paddle.distribution.Laplace(loc=self.loc, scale=self.scale) def test_sample(self): - sample_shape = (20000, ) + sample_shape = (20000,) samples = self._dist.sample(sample_shape) sample_values = samples.numpy() self.assertTrue(self._kstest(self.loc, self.scale, sample_values)) @@ -122,8 +132,8 @@ class TestLaplaceKS(unittest.TestCase): def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. ks, p_value = scipy.stats.kstest( - samples, - scipy.stats.laplace(loc, scale=scale).cdf) + samples, scipy.stats.laplace(loc, scale=scale).cdf + ) return ks < 0.02 @@ -131,78 +141,101 @@ class TestLaplaceKS(unittest.TestCase): @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), [ - ('value-float', np.array([0.2, 0.3]),\ - np.array([2., 3.]), np.array([2., 5.])), - ('value-int', np.array([0.2, 0.3]),\ - np.array([2., 3.]), np.array([2, 5])), - ('value-multi-dim', np.array([0.2, 0.3]), np.array([2., 3.]),\ - np.array([[4., 6], [8, 2]])), - ]) + ( + 'value-float', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([2.0, 5.0]), + ), + ( + 'value-int', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([2, 5]), + ), + ( + 'value-multi-dim', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([[4.0, 6], [8, 2]]), + ), + ], +) class TestLaplacePDF(unittest.TestCase): - def setUp(self): - self._dist = paddle.distribution.Laplace(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(\ - self.scale)) + self._dist = paddle.distribution.Laplace( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) def test_prob(self): np.testing.assert_allclose( self._dist.prob(paddle.to_tensor(self.value)), scipy.stats.laplace.pdf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_log_prob(self): np.testing.assert_allclose( self._dist.log_prob(paddle.to_tensor(self.value)), scipy.stats.laplace.logpdf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_cdf(self): - np.testing.assert_allclose(self._dist.cdf(paddle.to_tensor(self.value)), - scipy.stats.laplace.cdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self._dist.cdf(paddle.to_tensor(self.value)), + scipy.stats.laplace.cdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_icdf(self): np.testing.assert_allclose( self._dist.icdf(paddle.to_tensor(self.value)), scipy.stats.laplace.ppf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc1', 'scale1',\ - 'loc2', 'scale2'), [ - ('kl', np.array([0.0]), np.array([1.0]), \ - np.array([1.0]), np.array([0.5])) - ]) + (parameterize.TEST_CASE_NAME, 'loc1', 'scale1', 'loc2', 'scale2'), + [ + ( + 'kl', + np.array([0.0]), + np.array([1.0]), + np.array([1.0]), + np.array([0.5]), + ) + ], +) class TestLaplaceAndLaplaceKL(unittest.TestCase): - def setUp(self): - self._dist_1 = paddle.distribution.Laplace(loc=paddle.to_tensor(self.loc1), - scale=paddle.to_tensor(\ - self.scale1)) - self._dist_2 = paddle.distribution.Laplace(loc=paddle.to_tensor(self.loc2), - scale=paddle.to_tensor(\ - self.scale2)) + self._dist_1 = paddle.distribution.Laplace( + loc=paddle.to_tensor(self.loc1), scale=paddle.to_tensor(self.scale1) + ) + self._dist_2 = paddle.distribution.Laplace( + loc=paddle.to_tensor(self.loc2), scale=paddle.to_tensor(self.scale2) + ) def test_kl_divergence(self): - np.testing.assert_allclose(paddle.distribution.kl_divergence( - self._dist_1, self._dist_2), - self._np_kl(), - atol=0, - rtol=0.50) + np.testing.assert_allclose( + paddle.distribution.kl_divergence(self._dist_1, self._dist_2), + self._np_kl(), + atol=0, + rtol=0.50, + ) def _np_kl(self): - x = np.linspace(scipy.stats.laplace.ppf(0.01),\ - scipy.stats.laplace.ppf(0.99), 1000) - d1 = scipy.stats.laplace.pdf(x, loc=0., scale=1.) - d2 = scipy.stats.laplace.pdf(x, loc=1., scale=0.5) + x = np.linspace( + scipy.stats.laplace.ppf(0.01), scipy.stats.laplace.ppf(0.99), 1000 + ) + d1 = scipy.stats.laplace.pdf(x, loc=0.0, scale=1.0) + d2 = scipy.stats.laplace.pdf(x, loc=1.0, scale=0.5) return scipy.stats.entropy(d1, d2) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace_static.py index 62fe225849c..7c8ecf1609b 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace_static.py @@ -26,24 +26,24 @@ paddle.enable_static() @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('one-dim', parameterize.xrand((2, )),\ - parameterize.xrand((2, ))), - ('multi-dim', parameterize.xrand((5, 5)),\ - parameterize.xrand((5, 5))), - ]) + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('one-dim', parameterize.xrand((2,)), parameterize.xrand((2,))), + ('multi-dim', parameterize.xrand((5, 5)), parameterize.xrand((5, 5))), + ], +) class TestLaplace(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) self._dist = paddle.distribution.Laplace(loc=loc, scale=scale) - self.sample_shape = (50000, ) + self.sample_shape = (50000,) mean = self._dist.mean var = self._dist.variance stddev = self._dist.stddev @@ -53,56 +53,71 @@ class TestLaplace(unittest.TestCase): self.feeds = {'loc': self.loc, 'scale': self.scale} executor.run(startup_program) - [self.mean, self.var, self.stddev, self.entropy, - self.samples] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [ + self.mean, + self.var, + self.stddev, + self.entropy, + self.samples, + ] = executor.run(main_program, feed=self.feeds, fetch_list=fetch_list) def test_mean(self): self.assertEqual(str(self.mean.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): self.assertEqual(str(self.var.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.var, - self._np_variance(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.var, + self._np_variance(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_stddev(self): self.assertEqual( - str(self.stddev.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.stddev, - self._np_stddev(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + str(self.stddev.dtype).split('.')[-1], self.scale.dtype + ) + np.testing.assert_allclose( + self.stddev, + self._np_stddev(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): self.assertEqual( - str(self.entropy.dtype).split('.')[-1], self.scale.dtype) + str(self.entropy.dtype).split('.')[-1], self.scale.dtype + ) def test_sample(self): self.assertEqual(self.samples.dtype, self.scale.dtype) - self.assertEqual(tuple(self.samples.shape), - tuple(self._dist._extend_shape(self.sample_shape))) + self.assertEqual( + tuple(self.samples.shape), + tuple(self._dist._extend_shape(self.sample_shape)), + ) self.assertEqual(self.samples.shape, self.sample_shape + self.loc.shape) self.assertEqual(self.samples.shape, self.sample_shape + self.loc.shape) - np.testing.assert_allclose(self.samples.mean(axis=0), - scipy.stats.laplace.mean(self.loc, - scale=self.scale), - rtol=0.2, - atol=0.) - np.testing.assert_allclose(self.samples.var(axis=0), - scipy.stats.laplace.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=0.) + np.testing.assert_allclose( + self.samples.mean(axis=0), + scipy.stats.laplace.mean(self.loc, scale=self.scale), + rtol=0.2, + atol=0.0, + ) + np.testing.assert_allclose( + self.samples.var(axis=0), + scipy.stats.laplace.var(self.loc, scale=self.scale), + rtol=0.1, + atol=0.0, + ) def _np_mean(self): return self.loc @@ -122,15 +137,27 @@ class TestLaplace(unittest.TestCase): @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), [ - ('value-float', np.array([0.2, 0.3]),\ - np.array([2., 3.]), np.array([2., 5.])), - ('value-int', np.array([0.2, 0.3]),\ - np.array([2., 3.]), np.array([2, 5])), - ('value-multi-dim', np.array([0.2, 0.3]), np.array([2., 3.]),\ - np.array([[4., 6], [8, 2]])), - ]) + ( + 'value-float', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([2.0, 5.0]), + ), + ( + 'value-int', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([2, 5]), + ), + ( + 'value-multi-dim', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([[4.0, 6], [8, 2]]), + ), + ], +) class TestLaplacePDF(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() @@ -138,10 +165,12 @@ class TestLaplacePDF(unittest.TestCase): with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) - value = paddle.static.data('value', self.value.shape, - self.value.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) + value = paddle.static.data( + 'value', self.value.shape, self.value.dtype + ) self._dist = paddle.distribution.Laplace(loc=loc, scale=scale) prob = self._dist.prob(value) log_prob = self._dist.log_prob(value) @@ -151,49 +180,57 @@ class TestLaplacePDF(unittest.TestCase): self.feeds = {'loc': self.loc, 'scale': self.scale, 'value': self.value} executor.run(startup_program) - [self.prob, self.log_prob, self.cdf, - self.icdf] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.prob, self.log_prob, self.cdf, self.icdf] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_prob(self): - np.testing.assert_allclose(self.prob, - scipy.stats.laplace.pdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.prob, + scipy.stats.laplace.pdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_log_prob(self): - np.testing.assert_allclose(self.log_prob, - scipy.stats.laplace.logpdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.log_prob, + scipy.stats.laplace.logpdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_cdf(self): - np.testing.assert_allclose(self.cdf, - scipy.stats.laplace.cdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.cdf, + scipy.stats.laplace.cdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_icdf(self): - np.testing.assert_allclose(self.icdf, - scipy.stats.laplace.ppf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.icdf, + scipy.stats.laplace.ppf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc1', 'scale1',\ - 'loc2', 'scale2'), [ - ('kl', np.array([0.0]), np.array([1.0]), \ - np.array([1.0]), np.array([0.5])) - ]) + (parameterize.TEST_CASE_NAME, 'loc1', 'scale1', 'loc2', 'scale2'), + [ + ( + 'kl', + np.array([0.0]), + np.array([1.0]), + np.array([1.0]), + np.array([0.5]), + ) + ], +) class TestLaplaceAndLaplaceKL(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() @@ -201,34 +238,37 @@ class TestLaplaceAndLaplaceKL(unittest.TestCase): with paddle.static.program_guard(self.mp, self.sp): loc1 = paddle.static.data('loc1', self.loc1.shape, self.loc1.dtype) - scale1 = paddle.static.data('scale1', self.scale1.shape, - self.scale1.dtype) + scale1 = paddle.static.data( + 'scale1', self.scale1.shape, self.scale1.dtype + ) loc2 = paddle.static.data('loc2', self.loc2.shape, self.loc2.dtype) - scale2 = paddle.static.data('scale2', self.scale2.shape, - self.scale2.dtype) + scale2 = paddle.static.data( + 'scale2', self.scale2.shape, self.scale2.dtype + ) self._dist_1 = paddle.distribution.Laplace(loc=loc1, scale=scale1) self._dist_2 = paddle.distribution.Laplace(loc=loc2, scale=scale2) self.feeds = { 'loc1': self.loc1, 'scale1': self.scale1, 'loc2': self.loc2, - 'scale2': self.scale2 + 'scale2': self.scale2, } def test_kl_divergence(self): with paddle.static.program_guard(self.mp, self.sp): out = paddle.distribution.kl_divergence(self._dist_1, self._dist_2) self.executor.run(self.sp) - [out] = self.executor.run(self.mp, - feed=self.feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.mp, feed=self.feeds, fetch_list=[out] + ) np.testing.assert_allclose(out, self._np_kl(), atol=0, rtol=0.50) def _np_kl(self): - x = np.linspace(scipy.stats.laplace.ppf(0.01),\ - scipy.stats.laplace.ppf(0.99), 1000) - d1 = scipy.stats.laplace.pdf(x, loc=0., scale=1.) - d2 = scipy.stats.laplace.pdf(x, loc=1., scale=0.5) + x = np.linspace( + scipy.stats.laplace.ppf(0.01), scipy.stats.laplace.ppf(0.99), 1000 + ) + d1 = scipy.stats.laplace.pdf(x, loc=0.0, scale=1.0) + d2 = scipy.stats.laplace.pdf(x, loc=1.0, scale=0.5) return scipy.stats.entropy(d1, d2) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal.py index a7c97047505..be51e1ecf0a 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal.py @@ -28,7 +28,6 @@ from test_distribution import DistributionNumpy class LogNormalNumpy(DistributionNumpy): - def __init__(self, loc, scale): self.loc = np.array(loc) self.scale = np.array(scale) @@ -49,67 +48,86 @@ class LogNormalNumpy(DistributionNumpy): def log_prob(self, value): var = self.scale * self.scale log_scale = np.log(self.scale) - return -( - (np.log(value) - self.loc) * - (np.log(value) - self.loc)) / (2. * var) - log_scale - math.log( - math.sqrt(2. * math.pi)) - np.log(value) + return ( + -((np.log(value) - self.loc) * (np.log(value) - self.loc)) + / (2.0 * var) + - log_scale + - math.log(math.sqrt(2.0 * math.pi)) + - np.log(value) + ) def probs(self, value): var = self.scale * self.scale return np.exp( - -1. * ((np.log(value) - self.loc) * (np.log(value) - self.loc)) / - (2. * var)) / (math.sqrt(2 * math.pi) * self.scale * value) + -1.0 + * ((np.log(value) - self.loc) * (np.log(value) - self.loc)) + / (2.0 * var) + ) / (math.sqrt(2 * math.pi) * self.scale * value) def entropy(self): - return 0.5 + self.loc + 0.5 * np.log( - np.array(2. * math.pi).astype(self.loc.dtype)) + np.log(self.scale) + return ( + 0.5 + + self.loc + + 0.5 * np.log(np.array(2.0 * math.pi).astype(self.loc.dtype)) + + np.log(self.scale) + ) def kl_divergence(self, other): - var_ratio = (self.scale / other.scale) + var_ratio = self.scale / other.scale var_ratio = var_ratio * var_ratio - t1 = ((self.loc - other.loc) / other.scale) - t1 = (t1 * t1) + t1 = (self.loc - other.loc) / other.scale + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale', 'value'), - [('one-dim', xrand((2, )), xrand((2, )), xrand((2, ))), - ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3)))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale', 'value'), + [ + ('one-dim', xrand((2,)), xrand((2,)), xrand((2,))), + ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3))), + ], +) class LogNormalTest(unittest.TestCase): - def setUp(self): paddle.disable_static() - self.paddle_lognormal = LogNormal(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(self.scale)) + self.paddle_lognormal = LogNormal( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) self.np_lognormal = LogNormalNumpy(self.loc, self.scale) def test_mean(self): mean = self.paddle_lognormal.mean np_mean = self.np_lognormal.mean self.assertEqual(mean.numpy().dtype, np_mean.dtype) - np.testing.assert_allclose(mean, - np_mean, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + mean, + np_mean, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): var = self.paddle_lognormal.variance np_var = self.np_lognormal.variance self.assertEqual(var.numpy().dtype, np_var.dtype) - np.testing.assert_allclose(var, - np_var, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + var, + np_var, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): entropy = self.paddle_lognormal.entropy() np_entropy = self.np_lognormal.entropy() self.assertEqual(entropy.numpy().dtype, np_entropy.dtype) - np.testing.assert_allclose(entropy, - np_entropy, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + entropy, + np_entropy, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_probs(self): with paddle.fluid.dygraph.guard(self.place): @@ -119,94 +137,110 @@ class LogNormalTest(unittest.TestCase): probs, np_probs, rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_log_prob(self): with paddle.fluid.dygraph.guard(self.place): log_prob = self.paddle_lognormal.log_prob( - paddle.to_tensor(self.value)) + paddle.to_tensor(self.value) + ) np_log_prob = self.np_lognormal.log_prob(self.value) np.testing.assert_allclose( log_prob, np_log_prob, rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + atol=config.ATOL.get(str(self.scale.dtype)), + ) @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand( - (4, )), xrand((4, ), min=0, max=1))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale'), + [('sample', xrand((4,)), xrand((4,), min=0, max=1))], +) class TestLogNormalSample(unittest.TestCase): - def setUp(self): paddle.disable_static() self.paddle_lognormal = LogNormal(loc=self.loc, scale=self.scale) n = 100000 - self.sample_shape = (n, ) - self.rsample_shape = (n, ) + self.sample_shape = (n,) + self.rsample_shape = (n,) self.samples = self.paddle_lognormal.sample(self.sample_shape) self.rsamples = self.paddle_lognormal.rsample(self.rsample_shape) def test_sample(self): samples_mean = self.samples.mean(axis=0) samples_var = self.samples.var(axis=0) - np.testing.assert_allclose(samples_mean, - self.paddle_lognormal.mean, - rtol=0.1, - atol=0) - np.testing.assert_allclose(samples_var, - self.paddle_lognormal.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + samples_mean, self.paddle_lognormal.mean, rtol=0.1, atol=0 + ) + np.testing.assert_allclose( + samples_var, self.paddle_lognormal.variance, rtol=0.1, atol=0 + ) rsamples_mean = self.rsamples.mean(axis=0) rsamples_var = self.rsamples.var(axis=0) - np.testing.assert_allclose(rsamples_mean, - self.paddle_lognormal.mean, - rtol=0.1, - atol=0) - np.testing.assert_allclose(rsamples_var, - self.paddle_lognormal.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + rsamples_mean, self.paddle_lognormal.mean, rtol=0.1, atol=0 + ) + np.testing.assert_allclose( + rsamples_var, self.paddle_lognormal.variance, rtol=0.1, atol=0 + ) batch_shape = (self.loc + self.scale).shape - self.assertEqual(self.samples.shape, - list(self.sample_shape + batch_shape)) - self.assertEqual(self.rsamples.shape, - list(self.rsample_shape + batch_shape)) + self.assertEqual( + self.samples.shape, list(self.sample_shape + batch_shape) + ) + self.assertEqual( + self.rsamples.shape, list(self.rsample_shape + batch_shape) + ) for i in range(len(self.scale)): self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.samples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.samples[:, i]) + ) self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i]) + ) def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. ks, _ = scipy.stats.kstest( - samples, - scipy.stats.lognorm(s=scale, scale=np.exp(loc)).cdf) + samples, scipy.stats.lognorm(s=scale, scale=np.exp(loc)).cdf + ) return ks < 0.02 @place(config.DEVICES) @parameterize_cls( (TEST_CASE_NAME, 'loc1', 'scale1', 'loc2', 'scale2'), - [('one-dim', xrand((2, )), xrand((2, )), xrand((2, )), xrand((2, ))), - ('multi-dim', xrand((2, 2)), xrand((2, 2)), xrand((2, 2)), xrand((2, 2)))]) + [ + ('one-dim', xrand((2,)), xrand((2,)), xrand((2,)), xrand((2,))), + ( + 'multi-dim', + xrand((2, 2)), + xrand((2, 2)), + xrand((2, 2)), + xrand((2, 2)), + ), + ], +) class TestLogNormalKL(unittest.TestCase): - def setUp(self): paddle.disable_static() - self.ln_a = LogNormal(loc=paddle.to_tensor(self.loc1), - scale=paddle.to_tensor(self.scale1)) - self.ln_b = LogNormal(loc=paddle.to_tensor(self.loc2), - scale=paddle.to_tensor(self.scale2)) - self.normal_a = Normal(loc=paddle.to_tensor(self.loc1), - scale=paddle.to_tensor(self.scale1)) - self.normal_b = Normal(loc=paddle.to_tensor(self.loc2), - scale=paddle.to_tensor(self.scale2)) + self.ln_a = LogNormal( + loc=paddle.to_tensor(self.loc1), scale=paddle.to_tensor(self.scale1) + ) + self.ln_b = LogNormal( + loc=paddle.to_tensor(self.loc2), scale=paddle.to_tensor(self.scale2) + ) + self.normal_a = Normal( + loc=paddle.to_tensor(self.loc1), scale=paddle.to_tensor(self.scale1) + ) + self.normal_b = Normal( + loc=paddle.to_tensor(self.loc2), scale=paddle.to_tensor(self.scale2) + ) def test_kl_divergence(self): kl0 = self.ln_a.kl_divergence(self.ln_b) @@ -216,28 +250,34 @@ class TestLogNormalKL(unittest.TestCase): self.assertEqual(tuple(kl0.shape), self.scale1.shape) self.assertEqual(tuple(kl1.shape), self.scale1.shape) - np.testing.assert_allclose(kl0, - kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) - np.testing.assert_allclose(kl1, - kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) - np.testing.assert_allclose(kl_normal, - kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) + np.testing.assert_allclose( + kl0, + kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) + np.testing.assert_allclose( + kl1, + kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) + np.testing.assert_allclose( + kl_normal, + kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) def _kl(self, dist1, dist2): loc1 = np.array(dist1.loc) loc2 = np.array(dist2.loc) scale1 = np.array(dist1.scale) scale2 = np.array(dist2.scale) - var_ratio = (scale1 / scale2) + var_ratio = scale1 / scale2 var_ratio = var_ratio * var_ratio - t1 = ((loc1 - loc2) / scale2) - t1 = (t1 * t1) + t1 = (loc1 - loc2) / scale2 + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal_static.py index 75a9e497f34..9e5cab2da37 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal_static.py @@ -27,11 +27,14 @@ from test_distribution_lognormal import LogNormalNumpy @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale', 'value'), - [('one-dim', xrand((2, )), xrand((2, )), xrand((2, ))), - ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3)))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale', 'value'), + [ + ('one-dim', xrand((2,)), xrand((2,)), xrand((2,))), + ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3))), + ], +) class TestLogNormal(unittest.TestCase): - def setUp(self): paddle.enable_static() startup_program = paddle.static.Program() @@ -39,10 +42,12 @@ class TestLogNormal(unittest.TestCase): executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) - value = paddle.static.data('value', self.value.shape, - self.value.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) + value = paddle.static.data( + 'value', self.value.shape, self.value.dtype + ) self.paddle_lognormal = LogNormal(loc=loc, scale=scale) self.np_lognormal = LogNormalNumpy(loc=self.loc, scale=self.scale) mean = self.paddle_lognormal.mean @@ -54,56 +59,71 @@ class TestLogNormal(unittest.TestCase): self.feeds = {'loc': self.loc, 'scale': self.scale, 'value': self.value} executor.run(startup_program) - [self.mean, self.var, self.entropy, self.probs, - self.log_prob] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [ + self.mean, + self.var, + self.entropy, + self.probs, + self.log_prob, + ] = executor.run(main_program, feed=self.feeds, fetch_list=fetch_list) def test_mean(self): np_mean = self.np_lognormal.mean self.assertEqual(str(self.mean.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.mean, - np_mean, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.mean, + np_mean, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_var(self): np_var = self.np_lognormal.variance self.assertEqual(str(self.var.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.var, - np_var, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.var, + np_var, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): np_entropy = self.np_lognormal.entropy() self.assertEqual( - str(self.entropy.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.entropy, - np_entropy, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + str(self.entropy.dtype).split('.')[-1], self.scale.dtype + ) + np.testing.assert_allclose( + self.entropy, + np_entropy, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_probs(self): np_probs = self.np_lognormal.probs(self.value) - np.testing.assert_allclose(self.probs, - np_probs, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.probs, + np_probs, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_log_prob(self): np_log_prob = self.np_lognormal.log_prob(self.value) - np.testing.assert_allclose(self.log_prob, - np_log_prob, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.log_prob, + np_log_prob, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand( - (4, )), xrand((4, ), min=0, max=1))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale'), + [('sample', xrand((4,)), xrand((4,), min=0, max=1))], +) class TestLogNormalSample(unittest.TestCase): - def setUp(self): paddle.enable_static() startup_program = paddle.static.Program() @@ -111,11 +131,12 @@ class TestLogNormalSample(unittest.TestCase): executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) n = 100000 - self.sample_shape = (n, ) - self.rsample_shape = (n, ) + self.sample_shape = (n,) + self.rsample_shape = (n,) self.paddle_lognormal = LogNormal(loc=loc, scale=scale) mean = self.paddle_lognormal.mean variance = self.paddle_lognormal.variance @@ -125,10 +146,9 @@ class TestLogNormalSample(unittest.TestCase): self.feeds = {'loc': self.loc, 'scale': self.scale} executor.run(startup_program) - [self.mean, self.variance, self.samples, - self.rsamples] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.mean, self.variance, self.samples, self.rsamples] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_sample(self): samples_mean = self.samples.mean(axis=0) @@ -139,10 +159,9 @@ class TestLogNormalSample(unittest.TestCase): rsamples_mean = self.rsamples.mean(axis=0) rsamples_var = self.rsamples.var(axis=0) np.testing.assert_allclose(rsamples_mean, self.mean, rtol=0.1, atol=0) - np.testing.assert_allclose(rsamples_var, - self.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + rsamples_var, self.variance, rtol=0.1, atol=0 + ) batch_shape = (self.loc + self.scale).shape self.assertEqual(self.samples.shape, self.sample_shape + batch_shape) @@ -150,25 +169,35 @@ class TestLogNormalSample(unittest.TestCase): for i in range(len(self.scale)): self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.samples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.samples[:, i]) + ) self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i]) + ) def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. ks, _ = scipy.stats.kstest( - samples, - scipy.stats.lognorm(s=scale, scale=np.exp(loc)).cdf) + samples, scipy.stats.lognorm(s=scale, scale=np.exp(loc)).cdf + ) return ks < 0.02 @place(config.DEVICES) @parameterize_cls( (TEST_CASE_NAME, 'loc1', 'scale1', 'loc2', 'scale2'), - [('one-dim', xrand((2, )), xrand((2, )), xrand((2, )), xrand((2, ))), - ('multi-dim', xrand((2, 2)), xrand((2, 2)), xrand((2, 2)), xrand((2, 2)))]) + [ + ('one-dim', xrand((2,)), xrand((2,)), xrand((2,)), xrand((2,))), + ( + 'multi-dim', + xrand((2, 2)), + xrand((2, 2)), + xrand((2, 2)), + xrand((2, 2)), + ), + ], +) class TestLogNormalKL(unittest.TestCase): - def setUp(self): paddle.enable_static() startup_program = paddle.static.Program() @@ -176,11 +205,13 @@ class TestLogNormalKL(unittest.TestCase): executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc1 = paddle.static.data('loc1', self.loc1.shape, self.loc1.dtype) - scale1 = paddle.static.data('scale1', self.scale1.shape, - self.scale1.dtype) + scale1 = paddle.static.data( + 'scale1', self.scale1.shape, self.scale1.dtype + ) loc2 = paddle.static.data('loc2', self.loc2.shape, self.loc2.dtype) - scale2 = paddle.static.data('scale2', self.scale2.shape, - self.scale2.dtype) + scale2 = paddle.static.data( + 'scale2', self.scale2.shape, self.scale2.dtype + ) self.ln_a = LogNormal(loc=loc1, scale=scale1) self.ln_b = LogNormal(loc=loc2, scale=scale2) @@ -197,40 +228,45 @@ class TestLogNormalKL(unittest.TestCase): 'loc1': self.loc1, 'scale1': self.scale1, 'loc2': self.loc2, - 'scale2': self.scale2 + 'scale2': self.scale2, } executor.run(startup_program) - [self.kl0, self.kl1, self.kl_normal, - self.kl_formula] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.kl0, self.kl1, self.kl_normal, self.kl_formula] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_kl_divergence(self): - np.testing.assert_allclose(self.kl0, - self.kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) - - np.testing.assert_allclose(self.kl1, - self.kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) - - np.testing.assert_allclose(self.kl_normal, - self.kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) + np.testing.assert_allclose( + self.kl0, + self.kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) + + np.testing.assert_allclose( + self.kl1, + self.kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) + + np.testing.assert_allclose( + self.kl_normal, + self.kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) def _kl(self, dist1, dist2): loc1 = dist1.loc loc2 = dist2.loc - scale1 = (dist1.scale) - scale2 = (dist2.scale) - var_ratio = (scale1 / scale2) + scale1 = dist1.scale + scale2 = dist2.scale + var_ratio = scale1 / scale2 var_ratio = var_ratio * var_ratio - t1 = ((loc1 - loc2) / scale2) - t1 = (t1 * t1) + t1 = (loc1 - loc2) / scale2 + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial.py index 0bec1c5a58c..2c18469936c 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial.py @@ -24,41 +24,49 @@ import parameterize @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), [ - ('one-dim', 10, parameterize.xrand((3, ))), + (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), + [ + ('one-dim', 10, parameterize.xrand((3,))), ('multi-dim', 9, parameterize.xrand((10, 20))), ('prob-sum-one', 10, np.array([0.5, 0.2, 0.3])), - ('prob-sum-non-one', 10, np.array([2., 3., 5.])), - ]) + ('prob-sum-non-one', 10, np.array([2.0, 3.0, 5.0])), + ], +) class TestMultinomial(unittest.TestCase): - def setUp(self): self._dist = paddle.distribution.Multinomial( - total_count=self.total_count, probs=paddle.to_tensor(self.probs)) + total_count=self.total_count, probs=paddle.to_tensor(self.probs) + ) def test_mean(self): mean = self._dist.mean self.assertEqual(mean.numpy().dtype, self.probs.dtype) - np.testing.assert_allclose(mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_variance(self): var = self._dist.variance self.assertEqual(var.numpy().dtype, self.probs.dtype) - np.testing.assert_allclose(var, - self._np_variance(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + var, + self._np_variance(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_entropy(self): entropy = self._dist.entropy() self.assertEqual(entropy.numpy().dtype, self.probs.dtype) - np.testing.assert_allclose(entropy, - self._np_entropy(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + entropy, + self._np_entropy(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_sample(self): sample_shape = () @@ -66,26 +74,28 @@ class TestMultinomial(unittest.TestCase): self.assertEqual(samples.numpy().dtype, self.probs.dtype) self.assertEqual( tuple(samples.shape), - sample_shape + self._dist.batch_shape + self._dist.event_shape) + sample_shape + self._dist.batch_shape + self._dist.event_shape, + ) - sample_shape = (6, ) + sample_shape = (6,) samples = self._dist.sample(sample_shape) self.assertEqual(samples.numpy().dtype, self.probs.dtype) self.assertEqual( tuple(samples.shape), - sample_shape + self._dist.batch_shape + self._dist.event_shape) + sample_shape + self._dist.batch_shape + self._dist.event_shape, + ) self.assertTrue( - np.all(samples.sum(-1).numpy() == self._dist.total_count)) + np.all(samples.sum(-1).numpy() == self._dist.total_count) + ) - sample_shape = (5000, ) + sample_shape = (5000,) samples = self._dist.sample(sample_shape) sample_mean = samples.mean(axis=0) # Tolerance value 0.2 is empirical value which is consistent with # TensorFlow - np.testing.assert_allclose(sample_mean, - self._dist.mean, - atol=0, - rtol=0.20) + np.testing.assert_allclose( + sample_mean, self._dist.mean, atol=0, rtol=0.20 + ) def _np_variance(self): probs = self.probs / self.probs.sum(-1, keepdims=True) @@ -104,40 +114,54 @@ class TestMultinomial(unittest.TestCase): @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'total_count', 'probs', 'value'), [ - ('value-float', 10, np.array([0.2, 0.3, 0.5]), np.array([2., 3., 5.])), + ( + 'value-float', + 10, + np.array([0.2, 0.3, 0.5]), + np.array([2.0, 3.0, 5.0]), + ), ('value-int', 10, np.array([0.2, 0.3, 0.5]), np.array([2, 3, 5])), - ('value-multi-dim', 10, np.array([[0.3, 0.7], [0.5, 0.5] - ]), np.array([[4., 6], [8, 2]])), + ( + 'value-multi-dim', + 10, + np.array([[0.3, 0.7], [0.5, 0.5]]), + np.array([[4.0, 6], [8, 2]]), + ), # ('value-sum-non-n', 10, np.array([0.5, 0.2, 0.3]), np.array([4,5,2])), - ]) + ], +) class TestMultinomialPmf(unittest.TestCase): - def setUp(self): self._dist = paddle.distribution.Multinomial( - total_count=self.total_count, probs=paddle.to_tensor(self.probs)) + total_count=self.total_count, probs=paddle.to_tensor(self.probs) + ) def test_prob(self): np.testing.assert_allclose( self._dist.prob(paddle.to_tensor(self.value)), - scipy.stats.multinomial.pmf(self.value, self.total_count, - self.probs), + scipy.stats.multinomial.pmf( + self.value, self.total_count, self.probs + ), rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + atol=config.ATOL.get(str(self.probs.dtype)), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (config.TEST_CASE_NAME, 'total_count', 'probs'), [ + (config.TEST_CASE_NAME, 'total_count', 'probs'), + [ ('total_count_le_one', 0, np.array([0.3, 0.7])), ('total_count_float', np.array([0.3, 0.7])), ('probs_zero_dim', np.array(0)), - ]) + ], +) class TestMultinomialException(unittest.TestCase): - def TestInit(self): with self.assertRaises(ValueError): - paddle.distribution.Multinomial(self.total_count, - paddle.to_tensor(self.probs)) + paddle.distribution.Multinomial( + self.total_count, paddle.to_tensor(self.probs) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial_static.py index 56341d7fc0e..90efb815dea 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial_static.py @@ -26,61 +26,74 @@ paddle.enable_static() @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), [ - ('one-dim', 5, parameterize.xrand((3, ))), + (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), + [ + ('one-dim', 5, parameterize.xrand((3,))), ('multi-dim', 9, parameterize.xrand((2, 3))), ('prob-sum-one', 5, np.array([0.5, 0.2, 0.3])), - ('prob-sum-non-one', 5, np.array([2., 3., 5.])), - ]) + ('prob-sum-non-one', 5, np.array([2.0, 3.0, 5.0])), + ], +) class TestMultinomial(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): - probs = paddle.static.data('probs', self.probs.shape, - self.probs.dtype) + probs = paddle.static.data( + 'probs', self.probs.shape, self.probs.dtype + ) dist = paddle.distribution.Multinomial(self.total_count, probs) mean = dist.mean var = dist.variance entropy = dist.entropy() - mini_samples = dist.sample(shape=(6, )) - large_samples = dist.sample(shape=(5000, )) + mini_samples = dist.sample(shape=(6,)) + large_samples = dist.sample(shape=(5000,)) fetch_list = [mean, var, entropy, mini_samples, large_samples] feed = {'probs': self.probs} executor.run(startup_program) [ - self.mean, self.var, self.entropy, self.mini_samples, - self.large_samples + self.mean, + self.var, + self.entropy, + self.mini_samples, + self.large_samples, ] = executor.run(main_program, feed=feed, fetch_list=fetch_list) def test_mean(self): self.assertEqual(str(self.mean.dtype).split('.')[-1], self.probs.dtype) - np.testing.assert_allclose(self.mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + self.mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_variance(self): self.assertEqual(str(self.var.dtype).split('.')[-1], self.probs.dtype) - np.testing.assert_allclose(self.var, - self._np_variance(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + self.var, + self._np_variance(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_entropy(self): self.assertEqual( - str(self.entropy.dtype).split('.')[-1], self.probs.dtype) - np.testing.assert_allclose(self.entropy, - self._np_entropy(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + str(self.entropy.dtype).split('.')[-1], self.probs.dtype + ) + np.testing.assert_allclose( + self.entropy, + self._np_entropy(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_sample(self): self.assertEqual( - str(self.mini_samples.dtype).split('.')[-1], self.probs.dtype) + str(self.mini_samples.dtype).split('.')[-1], self.probs.dtype + ) self.assertTrue(np.all(self.mini_samples.sum(-1) == self.total_count)) sample_mean = self.large_samples.mean(axis=0) @@ -103,60 +116,75 @@ class TestMultinomial(unittest.TestCase): @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'total_count', 'probs', 'value'), [ - ('value-float', 5, np.array([0.2, 0.3, 0.5]), np.array([1., 1., 3.])), + ( + 'value-float', + 5, + np.array([0.2, 0.3, 0.5]), + np.array([1.0, 1.0, 3.0]), + ), ('value-int', 5, np.array([0.2, 0.3, 0.5]), np.array([2, 2, 1])), - ('value-multi-dim', 5, np.array([[0.3, 0.7], [0.5, 0.5] - ]), np.array([[1., 4.], [2., 3.]])), + ( + 'value-multi-dim', + 5, + np.array([[0.3, 0.7], [0.5, 0.5]]), + np.array([[1.0, 4.0], [2.0, 3.0]]), + ), # ('value-sum-non-n', 10, np.array([0.5, 0.2, 0.3]), np.array([4,5,2])), - ]) + ], +) class TestMultinomialPmf(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): - probs = paddle.static.data('probs', self.probs.shape, - self.probs.dtype) - value = paddle.static.data('value', self.value.shape, - self.value.dtype) + probs = paddle.static.data( + 'probs', self.probs.shape, self.probs.dtype + ) + value = paddle.static.data( + 'value', self.value.shape, self.value.dtype + ) dist = paddle.distribution.Multinomial(self.total_count, probs) pmf = dist.prob(value) feed = {'probs': self.probs, 'value': self.value} fetch_list = [pmf] executor.run(startup_program) - [self.pmf] = executor.run(main_program, - feed=feed, - fetch_list=fetch_list) + [self.pmf] = executor.run( + main_program, feed=feed, fetch_list=fetch_list + ) def test_prob(self): - np.testing.assert_allclose(self.pmf, - scipy.stats.multinomial.pmf( - self.value, self.total_count, - self.probs), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + self.pmf, + scipy.stats.multinomial.pmf( + self.value, self.total_count, self.probs + ), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), [ + (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), + [ ('total_count_le_one', 0, np.array([0.3, 0.7])), ('total_count_float', np.array([0.3, 0.7])), ('probs_zero_dim', np.array(0)), - ]) + ], +) class TestMultinomialException(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() self.main_program = paddle.static.Program() self.executor = paddle.static.Executor(self.place) with paddle.static.program_guard(self.main_program, startup_program): - probs = paddle.static.data('probs', self.probs.shape, - self.probs.dtype) + probs = paddle.static.data( + 'probs', self.probs.shape, self.probs.dtype + ) dist = paddle.distribution.Multinomial(self.total_count, probs) self.feed = {'probs': self.probs} diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_normal.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_normal.py index d9040559955..3f75a8c78c2 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_normal.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_normal.py @@ -29,7 +29,6 @@ np.random.seed(2022) class NormalNumpy(DistributionNumpy): - def __init__(self, loc, scale): self.loc = np.array(loc) self.scale = np.array(scale) @@ -44,29 +43,34 @@ class NormalNumpy(DistributionNumpy): def log_prob(self, value): var = self.scale * self.scale log_scale = np.log(self.scale) - return -((value - self.loc) * - (value - self.loc)) / (2. * var) - log_scale - math.log( - math.sqrt(2. * math.pi)) + return ( + -((value - self.loc) * (value - self.loc)) / (2.0 * var) + - log_scale + - math.log(math.sqrt(2.0 * math.pi)) + ) def probs(self, value): var = self.scale * self.scale - return np.exp(-1. * ((value - self.loc) * (value - self.loc)) / - (2. * var)) / (math.sqrt(2 * math.pi) * self.scale) + return np.exp( + -1.0 * ((value - self.loc) * (value - self.loc)) / (2.0 * var) + ) / (math.sqrt(2 * math.pi) * self.scale) def entropy(self): - return 0.5 + 0.5 * np.log( - np.array(2. * math.pi).astype(self.loc.dtype)) + np.log(self.scale) + return ( + 0.5 + + 0.5 * np.log(np.array(2.0 * math.pi).astype(self.loc.dtype)) + + np.log(self.scale) + ) def kl_divergence(self, other): - var_ratio = (self.scale / other.scale) + var_ratio = self.scale / other.scale var_ratio = var_ratio * var_ratio - t1 = ((self.loc - other.loc) / other.scale) - t1 = (t1 * t1) + t1 = (self.loc - other.loc) / other.scale + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) class NormalTest(unittest.TestCase): - def setUp(self, use_gpu=False, batch_size=2, dims=3): self.use_gpu = use_gpu if not use_gpu: @@ -112,9 +116,9 @@ class NormalTest(unittest.TestCase): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[], dtype='float32' + ) def compare_with_numpy(self, fetch_list, sample_shape=7, tolerance=1e-6): sample, entropy, log_prob, probs, kl = fetch_list @@ -134,22 +138,18 @@ class NormalTest(unittest.TestCase): # So set the tolerance from 1e-6 to 1e-4. log_tolerance = 1e-4 np.testing.assert_equal(sample.shape, np_sample.shape) - np.testing.assert_allclose(entropy, - np_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(log_prob, - np_lp, - rtol=log_tolerance, - atol=log_tolerance) - np.testing.assert_allclose(probs, - np_p, - rtol=log_tolerance, - atol=log_tolerance) - np.testing.assert_allclose(kl, - np_kl, - rtol=log_tolerance, - atol=log_tolerance) + np.testing.assert_allclose( + entropy, np_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + log_prob, np_lp, rtol=log_tolerance, atol=log_tolerance + ) + np.testing.assert_allclose( + probs, np_p, rtol=log_tolerance, atol=log_tolerance + ) + np.testing.assert_allclose( + kl, np_kl, rtol=log_tolerance, atol=log_tolerance + ) def test_normal_distribution_dygraph(self, sample_shape=7, tolerance=1e-6): paddle.disable_static(self.place) @@ -174,8 +174,9 @@ class NormalTest(unittest.TestCase): entropy = normal.entropy() log_prob = normal.log_prob(self.static_values) probs = normal.probs(self.static_values) - other_normal = Normal(self.static_other_loc, - self.static_other_scale) + other_normal = Normal( + self.static_other_loc, self.static_other_scale + ) kl = normal.kl_divergence(other_normal) fetch_list = [sample, entropy, log_prob, probs, kl] @@ -185,19 +186,18 @@ class NormalTest(unittest.TestCase): 'scale': self.scale_np, 'values': self.values_np, 'other_loc': self.other_loc_np, - 'other_scale': self.other_scale_np + 'other_scale': self.other_scale_np, } self.executor.run(fluid.default_startup_program()) - fetch_list = self.executor.run(program=self.test_program, - feed=feed_vars, - fetch_list=fetch_list) + fetch_list = self.executor.run( + program=self.test_program, feed=feed_vars, fetch_list=fetch_list + ) self.compare_with_numpy(fetch_list) class NormalTest2(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc ans scale are 'int' self.loc_np = int((np.random.ranf() - 0.5) * 8) @@ -213,7 +213,6 @@ class NormalTest2(NormalTest): class NormalTest3(NormalTest): - def init_numpy_data(self, batch_size, dims): # test broadcast: loc is float, scale is numpy.ndarray with dtype 'float32'. self.loc_np = (np.random.ranf() - 0.5) * 4 @@ -223,11 +222,13 @@ class NormalTest3(NormalTest): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = (np.random.ranf() - 0.5) * 4 - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) def init_static_data(self, batch_size, dims): self.static_loc = self.loc_np @@ -235,13 +236,12 @@ class NormalTest3(NormalTest): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class NormalTest4(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are numpy.ndarray with dtype 'float32'. self.loc_np = np.random.randn(batch_size, dims).astype('float32') @@ -251,11 +251,13 @@ class NormalTest4(NormalTest): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float32') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) def init_static_data(self, batch_size, dims): self.static_loc = self.loc_np @@ -263,13 +265,12 @@ class NormalTest4(NormalTest): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class NormalTest5(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are numpy.ndarray with dtype 'float64'. self.loc_np = np.random.randn(batch_size, dims).astype('float64') @@ -279,11 +280,13 @@ class NormalTest5(NormalTest): self.values_np = np.random.randn(batch_size, dims).astype('float64') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float64') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) def init_dynamic_data(self, batch_size, dims): self.dynamic_loc = self.loc_np @@ -298,13 +301,12 @@ class NormalTest5(NormalTest): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float64') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float64' + ) class NormalTest6(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are Tensor with dtype 'VarType.FP32'. self.loc_np = np.random.randn(batch_size, dims).astype('float32') @@ -314,11 +316,13 @@ class NormalTest6(NormalTest): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float32') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) def init_dynamic_data(self, batch_size, dims): self.dynamic_loc = paddle.to_tensor(self.loc_np) @@ -329,25 +333,24 @@ class NormalTest6(NormalTest): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_loc = layers.data(name='loc', - shape=[dims], - dtype='float32') - self.static_scale = layers.data(name='scale', - shape=[dims], - dtype='float32') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') - self.static_other_loc = layers.data(name='other_loc', - shape=[dims], - dtype='float32') - self.static_other_scale = layers.data(name='other_scale', - shape=[dims], - dtype='float32') + self.static_loc = layers.data( + name='loc', shape=[dims], dtype='float32' + ) + self.static_scale = layers.data( + name='scale', shape=[dims], dtype='float32' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) + self.static_other_loc = layers.data( + name='other_loc', shape=[dims], dtype='float32' + ) + self.static_other_scale = layers.data( + name='other_scale', shape=[dims], dtype='float32' + ) class NormalTest7(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are Tensor with dtype 'VarType.FP64'. self.loc_np = np.random.randn(batch_size, dims).astype('float64') @@ -357,42 +360,45 @@ class NormalTest7(NormalTest): self.values_np = np.random.randn(batch_size, dims).astype('float64') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float64') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) def init_dynamic_data(self, batch_size, dims): self.dynamic_loc = paddle.to_tensor(self.loc_np, dtype='float64') self.dynamic_scale = paddle.to_tensor(self.scale_np, dtype='float64') self.dynamic_values = paddle.to_tensor(self.values_np, dtype='float64') - self.dynamic_other_loc = paddle.to_tensor(self.other_loc_np, - dtype='float64') - self.dynamic_other_scale = paddle.to_tensor(self.other_scale_np, - dtype='float64') + self.dynamic_other_loc = paddle.to_tensor( + self.other_loc_np, dtype='float64' + ) + self.dynamic_other_scale = paddle.to_tensor( + self.other_scale_np, dtype='float64' + ) def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_loc = layers.data(name='loc', - shape=[dims], - dtype='float64') - self.static_scale = layers.data(name='scale', - shape=[dims], - dtype='float64') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float64') - self.static_other_loc = layers.data(name='other_loc', - shape=[dims], - dtype='float64') - self.static_other_scale = layers.data(name='other_scale', - shape=[dims], - dtype='float64') + self.static_loc = layers.data( + name='loc', shape=[dims], dtype='float64' + ) + self.static_scale = layers.data( + name='scale', shape=[dims], dtype='float64' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float64' + ) + self.static_other_loc = layers.data( + name='other_loc', shape=[dims], dtype='float64' + ) + self.static_other_scale = layers.data( + name='other_scale', shape=[dims], dtype='float64' + ) class NormalTest8(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are Tensor with dtype 'VarType.FP64'. value's dtype is 'VarType.FP32'. self.loc_np = np.random.randn(batch_size, dims).astype('float64') @@ -402,59 +408,66 @@ class NormalTest8(NormalTest): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float64') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) def init_dynamic_data(self, batch_size, dims): self.dynamic_loc = paddle.to_tensor(self.loc_np, dtype='float64') self.dynamic_scale = paddle.to_tensor(self.scale_np, dtype='float64') self.dynamic_values = paddle.to_tensor(self.values_np) - self.dynamic_other_loc = paddle.to_tensor(self.other_loc_np, - dtype='float64') - self.dynamic_other_scale = paddle.to_tensor(self.other_scale_np, - dtype='float64') + self.dynamic_other_loc = paddle.to_tensor( + self.other_loc_np, dtype='float64' + ) + self.dynamic_other_scale = paddle.to_tensor( + self.other_scale_np, dtype='float64' + ) def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_loc = layers.data(name='loc', - shape=[dims], - dtype='float64') - self.static_scale = layers.data(name='scale', - shape=[dims], - dtype='float64') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') - self.static_other_loc = layers.data(name='other_loc', - shape=[dims], - dtype='float64') - self.static_other_scale = layers.data(name='other_scale', - shape=[dims], - dtype='float64') + self.static_loc = layers.data( + name='loc', shape=[dims], dtype='float64' + ) + self.static_scale = layers.data( + name='scale', shape=[dims], dtype='float64' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) + self.static_other_loc = layers.data( + name='other_loc', shape=[dims], dtype='float64' + ) + self.static_other_scale = layers.data( + name='other_scale', shape=[dims], dtype='float64' + ) class NormalTest9(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are list. - self.loc_np = np.random.randn(batch_size, - dims).astype('float32').tolist() + self.loc_np = ( + np.random.randn(batch_size, dims).astype('float32').tolist() + ) self.scale_np = np.random.randn(batch_size, dims).astype('float32') while not np.all(self.scale_np > 0): self.scale_np = np.random.randn(batch_size, dims).astype('float32') self.scale_np = self.scale_np.tolist() self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence - self.other_loc_np = np.random.randn(batch_size, - dims).astype('float32').tolist() - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_loc_np = ( + np.random.randn(batch_size, dims).astype('float32').tolist() + ) + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.other_scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) self.other_scale_np = self.other_scale_np.tolist() def init_static_data(self, batch_size, dims): @@ -463,17 +476,17 @@ class NormalTest9(NormalTest): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class NormalTest10(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are tuple. self.loc_np = tuple( - np.random.randn(batch_size, dims).astype('float32').tolist()) + np.random.randn(batch_size, dims).astype('float32').tolist() + ) self.scale_np = np.random.randn(batch_size, dims).astype('float32') while not np.all(self.scale_np > 0): self.scale_np = np.random.randn(batch_size, dims).astype('float32') @@ -481,12 +494,15 @@ class NormalTest10(NormalTest): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = tuple( - np.random.randn(batch_size, dims).astype('float32').tolist()) - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + np.random.randn(batch_size, dims).astype('float32').tolist() + ) + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.other_scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) self.other_scale_np = tuple(self.other_scale_np.tolist()) def init_static_data(self, batch_size, dims): @@ -495,72 +511,73 @@ class NormalTest10(NormalTest): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand( - (4, )), xrand((4, )))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand((4,)), xrand((4,)))] +) class TestNormalSampleDygraph(unittest.TestCase): - def setUp(self): paddle.disable_static() self.paddle_normal = Normal(loc=self.loc, scale=self.scale) n = 100000 - self.sample_shape = (n, ) - self.rsample_shape = (n, ) + self.sample_shape = (n,) + self.rsample_shape = (n,) self.samples = self.paddle_normal.sample(self.sample_shape) self.rsamples = self.paddle_normal.rsample(self.rsample_shape) def test_sample(self): samples_mean = self.samples.mean(axis=0) samples_var = self.samples.var(axis=0) - np.testing.assert_allclose(samples_mean, - self.paddle_normal.mean, - rtol=0.1, - atol=0) - np.testing.assert_allclose(samples_var, - self.paddle_normal.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + samples_mean, self.paddle_normal.mean, rtol=0.1, atol=0 + ) + np.testing.assert_allclose( + samples_var, self.paddle_normal.variance, rtol=0.1, atol=0 + ) rsamples_mean = self.rsamples.mean(axis=0) rsamples_var = self.rsamples.var(axis=0) - np.testing.assert_allclose(rsamples_mean, - self.paddle_normal.mean, - rtol=0.1, - atol=0) - np.testing.assert_allclose(rsamples_var, - self.paddle_normal.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + rsamples_mean, self.paddle_normal.mean, rtol=0.1, atol=0 + ) + np.testing.assert_allclose( + rsamples_var, self.paddle_normal.variance, rtol=0.1, atol=0 + ) batch_shape = (self.loc + self.scale).shape - self.assertEqual(self.samples.shape, - list(self.sample_shape + batch_shape)) - self.assertEqual(self.rsamples.shape, - list(self.rsample_shape + batch_shape)) + self.assertEqual( + self.samples.shape, list(self.sample_shape + batch_shape) + ) + self.assertEqual( + self.rsamples.shape, list(self.rsample_shape + batch_shape) + ) for i in range(len(self.scale)): self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.samples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.samples[:, i]) + ) self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i]) + ) def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. - ks, _ = scipy.stats.kstest(samples, - scipy.stats.norm(loc=loc, scale=scale).cdf) + ks, _ = scipy.stats.kstest( + samples, scipy.stats.norm(loc=loc, scale=scale).cdf + ) return ks < 0.02 @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand( - (4, )), xrand((4, )))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand((4,)), xrand((4,)))] +) class TestNormalSampleStaic(unittest.TestCase): - def setUp(self): paddle.enable_static() startup_program = paddle.static.Program() @@ -568,11 +585,12 @@ class TestNormalSampleStaic(unittest.TestCase): executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) n = 100000 - self.sample_shape = (n, ) - self.rsample_shape = (n, ) + self.sample_shape = (n,) + self.rsample_shape = (n,) self.paddle_normal = Normal(loc=loc, scale=scale) mean = self.paddle_normal.mean variance = self.paddle_normal.variance @@ -582,10 +600,9 @@ class TestNormalSampleStaic(unittest.TestCase): self.feeds = {'loc': self.loc, 'scale': self.scale} executor.run(startup_program) - [self.mean, self.variance, self.samples, - self.rsamples] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.mean, self.variance, self.samples, self.rsamples] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_sample(self): samples_mean = self.samples.mean(axis=0) @@ -596,10 +613,9 @@ class TestNormalSampleStaic(unittest.TestCase): rsamples_mean = self.rsamples.mean(axis=0) rsamples_var = self.rsamples.var(axis=0) np.testing.assert_allclose(rsamples_mean, self.mean, rtol=0.1, atol=0) - np.testing.assert_allclose(rsamples_var, - self.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + rsamples_var, self.variance, rtol=0.1, atol=0 + ) batch_shape = (self.loc + self.scale).shape self.assertEqual(self.samples.shape, self.sample_shape + batch_shape) @@ -607,14 +623,17 @@ class TestNormalSampleStaic(unittest.TestCase): for i in range(len(self.scale)): self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.samples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.samples[:, i]) + ) self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i]) + ) def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. - ks, _ = scipy.stats.kstest(samples, - scipy.stats.norm(loc=loc, scale=scale).cdf) + ks, _ = scipy.stats.kstest( + samples, scipy.stats.norm(loc=loc, scale=scale).cdf + ) return ks < 0.02 diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform.py index 4f997d40bb0..f3f89068b35 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform.py @@ -28,22 +28,33 @@ paddle.seed(2022) @param.place(config.DEVICES) class TestTransform(unittest.TestCase): - def setUp(self): self._t = transform.Transform() - @param.param_func([(paddle.distribution.Distribution(), - paddle.distribution.TransformedDistribution), - (paddle.distribution.ExpTransform(), - paddle.distribution.ChainTransform)]) + @param.param_func( + [ + ( + paddle.distribution.Distribution(), + paddle.distribution.TransformedDistribution, + ), + ( + paddle.distribution.ExpTransform(), + paddle.distribution.ChainTransform, + ), + ] + ) def test_call(self, input, expected_type): t = transform.Transform() self.assertIsInstance(t(input), expected_type) - @param.param_func([(transform.Type.BIJECTION, True), - (transform.Type.INJECTION, True), - (transform.Type.SURJECTION, False), - (transform.Type.OTHER, False)]) + @param.param_func( + [ + (transform.Type.BIJECTION, True), + (transform.Type.INJECTION, True), + (transform.Type.SURJECTION, False), + (transform.Type.OTHER, False), + ] + ) def test_is_injective(self, type, expected): transform.Transform._type = type self.assertEqual(self._t._is_injective(), expected) @@ -54,26 +65,30 @@ class TestTransform(unittest.TestCase): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Real)) - @param.param_func([(0, TypeError), (paddle.rand( - (2, 3)), NotImplementedError)]) + @param.param_func( + [(0, TypeError), (paddle.rand((2, 3)), NotImplementedError)] + ) def test_forward(self, input, expected): with self.assertRaises(expected): self._t.forward(input) - @param.param_func([(0, TypeError), (paddle.rand( - (2, 3)), NotImplementedError)]) + @param.param_func( + [(0, TypeError), (paddle.rand((2, 3)), NotImplementedError)] + ) def test_inverse(self, input, expected): with self.assertRaises(expected): self._t.inverse(input) - @param.param_func([(0, TypeError), (paddle.rand( - (2, 3)), NotImplementedError)]) + @param.param_func( + [(0, TypeError), (paddle.rand((2, 3)), NotImplementedError)] + ) def test_forward_log_det_jacobian(self, input, expected): with self.assertRaises(expected): self._t.forward_log_det_jacobian(input) - @param.param_func([(0, TypeError), (paddle.rand( - (2, 3)), NotImplementedError)]) + @param.param_func( + [(0, TypeError), (paddle.rand((2, 3)), NotImplementedError)] + ) def test_inverse_log_det_jacobian(self, input, expected): with self.assertRaises(expected): self._t.inverse_log_det_jacobian(input) @@ -91,7 +106,6 @@ class TestTransform(unittest.TestCase): @param.place(config.DEVICES) class TestAbsTransform(unittest.TestCase): - def setUp(self): self._t = transform.AbsTransform() @@ -108,48 +122,66 @@ class TestAbsTransform(unittest.TestCase): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([-1., 1., 0.]), np.array([1., 1., 0.])), - (np.array([[1., -1., -0.1], [-3., -0.1, 0]]), - np.array([[1., 1., 0.1], [3., 0.1, 0]]))]) + @param.param_func( + [ + (np.array([-1.0, 1.0, 0.0]), np.array([1.0, 1.0, 0.0])), + ( + np.array([[1.0, -1.0, -0.1], [-3.0, -0.1, 0]]), + np.array([[1.0, 1.0, 0.1], [3.0, 0.1, 0]]), + ), + ] + ) def test_forward(self, input, expected): - np.testing.assert_allclose(self._t.forward( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) - @param.param_func([(np.array(1.), (-np.array(1.), np.array(1.)))]) + @param.param_func([(np.array(1.0), (-np.array(1.0), np.array(1.0)))]) def test_inverse(self, input, expected): actual0, actual1 = self._t.inverse(paddle.to_tensor(input)) expected0, expected1 = expected - np.testing.assert_allclose(actual0.numpy(), - expected0, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - np.testing.assert_allclose(actual1.numpy(), - expected1, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + actual0.numpy(), + expected0, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + np.testing.assert_allclose( + actual1.numpy(), + expected1, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def test_forward_log_det_jacobian(self): with self.assertRaises(NotImplementedError): - self._t.forward_log_det_jacobian(paddle.rand((10, ))) + self._t.forward_log_det_jacobian(paddle.rand((10,))) - @param.param_func([ - (np.array(1.), (np.array(0.), np.array(0.))), - ]) + @param.param_func( + [ + (np.array(1.0), (np.array(0.0), np.array(0.0))), + ] + ) def test_inverse_log_det_jacobian(self, input, expected): actual0, actual1 = self._t.inverse_log_det_jacobian( - paddle.to_tensor(input)) + paddle.to_tensor(input) + ) expected0, expected1 = expected - np.testing.assert_allclose(actual0.numpy(), - expected0, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - np.testing.assert_allclose(actual1.numpy(), - expected1, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + actual0.numpy(), + expected0, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + np.testing.assert_allclose( + actual1.numpy(), + expected1, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -161,20 +193,25 @@ class TestAbsTransform(unittest.TestCase): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'loc', 'scale'), [ - ('normal', np.random.rand(8, 10), np.random.rand(8, 10)), - ('broadcast', np.random.rand(2, 10), np.random.rand(10)), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('normal', np.random.rand(8, 10), np.random.rand(8, 10)), + ('broadcast', np.random.rand(2, 10), np.random.rand(10)), + ], +) class TestAffineTransform(unittest.TestCase): - def setUp(self): - self._t = transform.AffineTransform(paddle.to_tensor(self.loc), - paddle.to_tensor(self.scale)) + self._t = transform.AffineTransform( + paddle.to_tensor(self.loc), paddle.to_tensor(self.scale) + ) - @param.param_func([ - (paddle.rand([1]), 0, TypeError), - (0, paddle.rand([1]), TypeError), - ]) + @param.param_func( + [ + (paddle.rand([1]), 0, TypeError), + (0, paddle.rand([1]), TypeError), + ] + ) def test_init_exception(self, loc, scale, exc): with self.assertRaises(exc): paddle.distribution.AffineTransform(loc, scale) @@ -204,7 +241,8 @@ class TestAffineTransform(unittest.TestCase): self._t.forward(paddle.to_tensor(x)).numpy(), self._np_forward(x), rtol=config.RTOL.get(str(self._t.loc.numpy().dtype)), - atol=config.ATOL.get(str(self._t.loc.numpy().dtype))) + atol=config.ATOL.get(str(self._t.loc.numpy().dtype)), + ) def test_inverse(self): y = np.random.random(self.loc.shape) @@ -212,7 +250,8 @@ class TestAffineTransform(unittest.TestCase): self._t.inverse(paddle.to_tensor(y)).numpy(), self._np_inverse(y), rtol=config.RTOL.get(str(self._t.loc.numpy().dtype)), - atol=config.ATOL.get(str(self._t.loc.numpy().dtype))) + atol=config.ATOL.get(str(self._t.loc.numpy().dtype)), + ) def _np_forward(self, x): return self.loc + self.scale * x @@ -232,7 +271,8 @@ class TestAffineTransform(unittest.TestCase): self._t.inverse_log_det_jacobian(paddle.to_tensor(y)).numpy(), self._np_inverse_jacobian(y), rtol=config.RTOL.get(str(self._t.loc.numpy().dtype)), - atol=config.ATOL.get(str(self._t.loc.numpy().dtype))) + atol=config.ATOL.get(str(self._t.loc.numpy().dtype)), + ) def test_forward_log_det_jacobian(self): x = np.random.random(self.scale.shape) @@ -240,24 +280,26 @@ class TestAffineTransform(unittest.TestCase): self._t.forward_log_det_jacobian(paddle.to_tensor(x)).numpy(), self._np_forward_jacobian(x), rtol=config.RTOL.get(str(self._t.loc.numpy().dtype)), - atol=config.ATOL.get(str(self._t.loc.numpy().dtype))) + atol=config.ATOL.get(str(self._t.loc.numpy().dtype)), + ) def test_forward_shape(self): shape = self.loc.shape self.assertEqual( tuple(self._t.forward_shape(shape)), - np.broadcast(np.random.random(shape), self.loc, self.scale).shape) + np.broadcast(np.random.random(shape), self.loc, self.scale).shape, + ) def test_inverse_shape(self): shape = self.scale.shape self.assertEqual( tuple(self._t.forward_shape(shape)), - np.broadcast(np.random.random(shape), self.loc, self.scale).shape) + np.broadcast(np.random.random(shape), self.loc, self.scale).shape, + ) @param.place(config.DEVICES) class TestExpTransform(unittest.TestCase): - def setUp(self): self._t = transform.ExpTransform() @@ -274,48 +316,73 @@ class TestExpTransform(unittest.TestCase): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([0., 1., 2., - 3.]), np.exp(np.array([0., 1., 2., 3.]))), - (np.array([[0., 1., 2., 3.], [-5., 6., 7., 8.]]), - np.exp(np.array([[0., 1., 2., 3.], [-5., 6., 7., - 8.]])))]) + @param.param_func( + [ + ( + np.array([0.0, 1.0, 2.0, 3.0]), + np.exp(np.array([0.0, 1.0, 2.0, 3.0])), + ), + ( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]), + np.exp(np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]])), + ), + ] + ) def test_forward(self, input, expected): - np.testing.assert_allclose(self._t.forward( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), np.log(np.array([1., 2., 3.]))), - (np.array([[1., 2., 3.], [6., 7., 8.]]), - np.log(np.array([[1., 2., 3.], [6., 7., 8.]])))]) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]), np.log(np.array([1.0, 2.0, 3.0]))), + ( + np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]), + np.log(np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]])), + ), + ] + ) def test_inverse(self, input, expected): - np.testing.assert_allclose(self._t.inverse( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + np.testing.assert_allclose( + self._t.inverse(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_forward_log_det_jacobian(self, input): - np.testing.assert_allclose(self._t.forward_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward_log_det_jacobian(paddle.to_tensor(input)).numpy(), + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, x): return x - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_inverse_log_det_jacobian(self, input): - np.testing.assert_allclose(self._t.inverse_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse_log_det_jacobian(paddle.to_tensor(input)).numpy(), + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_inverse_jacobian(self, y): return -self._np_forward_jacobian(np.log(y)) @@ -331,127 +398,248 @@ class TestExpTransform(unittest.TestCase): @param.place(config.DEVICES) class TestChainTransform(unittest.TestCase): - - @param.param_func([(paddle.distribution.Transform, TypeError), - ([0], TypeError)]) + @param.param_func( + [(paddle.distribution.Transform, TypeError), ([0], TypeError)] + ) def test_init_exception(self, transforms, exception): with self.assertRaises(exception): paddle.distribution.ChainTransform(transforms) - @param.param_func(((transform.ChainTransform( - (transform.AbsTransform(), - transform.AffineTransform(paddle.rand([1]), paddle.rand([1])))), - False), (transform.ChainTransform(( - transform.AffineTransform(paddle.rand([1]), - paddle.rand([1])), - transform.ExpTransform(), - )), True))) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.AbsTransform(), + transform.AffineTransform( + paddle.rand([1]), paddle.rand([1]) + ), + ) + ), + False, + ), + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.rand([1]), paddle.rand([1]) + ), + transform.ExpTransform(), + ) + ), + True, + ), + ) + ) def test_is_injective(self, chain, expected): self.assertEqual(chain._is_injective(), expected) - @param.param_func(((transform.ChainTransform( - (transform.IndependentTransform(transform.ExpTransform(), 1), - transform.IndependentTransform(transform.ExpTransform(), 10), - transform.IndependentTransform(transform.ExpTransform(), 8))), - variable.Independent(variable.real, 10)), )) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.IndependentTransform( + transform.ExpTransform(), 1 + ), + transform.IndependentTransform( + transform.ExpTransform(), 10 + ), + transform.IndependentTransform( + transform.ExpTransform(), 8 + ), + ) + ), + variable.Independent(variable.real, 10), + ), + ) + ) def test_domain(self, input, expected): self.assertIsInstance(input._domain, type(expected)) self.assertEqual(input._domain.event_rank, expected.event_rank) self.assertEqual(input._domain.is_discrete, expected.is_discrete) - @param.param_func(((transform.ChainTransform( - (transform.IndependentTransform(transform.ExpTransform(), 9), - transform.IndependentTransform(transform.ExpTransform(), 4), - transform.IndependentTransform(transform.ExpTransform(), 5))), - variable.Independent(variable.real, 9)), )) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.IndependentTransform( + transform.ExpTransform(), 9 + ), + transform.IndependentTransform( + transform.ExpTransform(), 4 + ), + transform.IndependentTransform( + transform.ExpTransform(), 5 + ), + ) + ), + variable.Independent(variable.real, 9), + ), + ) + ) def test_codomain(self, input, expected): self.assertIsInstance(input._codomain, variable.Independent) self.assertEqual(input._codomain.event_rank, expected.event_rank) self.assertEqual(input._codomain.is_discrete, expected.is_discrete) - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(1.0)), - transform.ExpTransform())), np.array([0., 1., 2., 3.]), - np.exp(np.array([0., 1., 2., 3.]) * 1.0)), - (transform.ChainTransform( - (transform.ExpTransform(), transform.TanhTransform())), - np.array([[0., -1., 2., -3.], [-5., 6., 7., -8.]]), - np.tanh(np.exp(np.array([[0., -1., 2., -3.], [-5., 6., 7., -8.]])))) - ]) + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(1.0) + ), + transform.ExpTransform(), + ) + ), + np.array([0.0, 1.0, 2.0, 3.0]), + np.exp(np.array([0.0, 1.0, 2.0, 3.0]) * 1.0), + ), + ( + transform.ChainTransform( + (transform.ExpTransform(), transform.TanhTransform()) + ), + np.array([[0.0, -1.0, 2.0, -3.0], [-5.0, 6.0, 7.0, -8.0]]), + np.tanh( + np.exp( + np.array( + [[0.0, -1.0, 2.0, -3.0], [-5.0, 6.0, 7.0, -8.0]] + ) + ) + ), + ), + ] + ) def test_forward(self, chain, input, expected): - np.testing.assert_allclose(chain.forward( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(-1.0)), - transform.ExpTransform())), np.array([0., 1., 2., 3.]), - np.log(np.array([0., 1., 2., 3.])) / (-1.0)), - (transform.ChainTransform( - (transform.ExpTransform(), transform.TanhTransform())), - np.array([[0., 1., 2., 3.], [5., 6., 7., 8.]]), - np.log(np.arctanh(np.array([[0., 1., 2., 3.], [5., 6., 7., 8.]])))) - ]) + np.testing.assert_allclose( + chain.forward(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(-1.0) + ), + transform.ExpTransform(), + ) + ), + np.array([0.0, 1.0, 2.0, 3.0]), + np.log(np.array([0.0, 1.0, 2.0, 3.0])) / (-1.0), + ), + ( + transform.ChainTransform( + (transform.ExpTransform(), transform.TanhTransform()) + ), + np.array([[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0]]), + np.log( + np.arctanh( + np.array([[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0]]) + ) + ), + ), + ] + ) def test_inverse(self, chain, input, expected): - np.testing.assert_allclose(chain.inverse( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(-1.0)), - transform.PowerTransform(paddle.to_tensor(2.0)))), - np.array([1., 2., 3.]), np.log(2. * np.array([1., 2., 3.]))), - ]) + np.testing.assert_allclose( + chain.inverse(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(-1.0) + ), + transform.PowerTransform(paddle.to_tensor(2.0)), + ) + ), + np.array([1.0, 2.0, 3.0]), + np.log(2.0 * np.array([1.0, 2.0, 3.0])), + ), + ] + ) def test_forward_log_det_jacobian(self, chain, input, expected): - np.testing.assert_allclose(chain.forward_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(-1.0)), - transform.ExpTransform())), (2, 3, 5), (2, 3, 5)), - ]) + np.testing.assert_allclose( + chain.forward_log_det_jacobian(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(-1.0) + ), + transform.ExpTransform(), + ) + ), + (2, 3, 5), + (2, 3, 5), + ), + ] + ) def test_forward_shape(self, chain, shape, expected_shape): self.assertEqual(chain.forward_shape(shape), expected_shape) - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(-1.0)), - transform.ExpTransform())), (2, 3, 5), (2, 3, 5)), - ]) + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(-1.0) + ), + transform.ExpTransform(), + ) + ), + (2, 3, 5), + (2, 3, 5), + ), + ] + ) def test_inverse_shape(self, chain, shape, expected_shape): self.assertEqual(chain.inverse_shape(shape), expected_shape) @param.place(config.DEVICES) @param.param_cls( - (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'x'), [ - ('rank-over-zero', transform.ExpTransform(), 2, np.random.rand(2, 3, - 3)), - ]) + (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'x'), + [ + ( + 'rank-over-zero', + transform.ExpTransform(), + 2, + np.random.rand(2, 3, 3), + ), + ], +) class TestIndependentTransform(unittest.TestCase): - def setUp(self): - self._t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) + self._t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) - @param.param_func([(0, 0, TypeError), - (paddle.distribution.Transform(), -1, ValueError)]) + @param.param_func( + [(0, 0, TypeError), (paddle.distribution.Transform(), -1, ValueError)] + ) def test_init_exception(self, base, rank, exc): with self.assertRaises(exc): paddle.distribution.IndependentTransform(base, rank) @@ -463,42 +651,52 @@ class TestIndependentTransform(unittest.TestCase): self.assertTrue(isinstance(self._t._domain, variable.Independent)) self.assertEqual( self._t._domain.event_rank, - self.base._domain.event_rank + self.reinterpreted_batch_rank) - self.assertEqual(self._t._domain.is_discrete, - self.base._domain.is_discrete) + self.base._domain.event_rank + self.reinterpreted_batch_rank, + ) + self.assertEqual( + self._t._domain.is_discrete, self.base._domain.is_discrete + ) def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Independent)) self.assertEqual( self._t._codomain.event_rank, - self.base._codomain.event_rank + self.reinterpreted_batch_rank) - self.assertEqual(self._t._codomain.is_discrete, - self.base._codomain.is_discrete) + self.base._codomain.event_rank + self.reinterpreted_batch_rank, + ) + self.assertEqual( + self._t._codomain.is_discrete, self.base._codomain.is_discrete + ) def test_forward(self): np.testing.assert_allclose( self._t.forward(paddle.to_tensor(self.x)).numpy(), self.base.forward(paddle.to_tensor(self.x)).numpy(), rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + atol=config.ATOL.get(str(self.x.dtype)), + ) def test_inverse(self): np.testing.assert_allclose( self._t.inverse(paddle.to_tensor(self.x)).numpy(), self.base.inverse(paddle.to_tensor(self.x)).numpy(), rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + atol=config.ATOL.get(str(self.x.dtype)), + ) def test_forward_log_det_jacobian(self): actual = self._t.forward_log_det_jacobian(paddle.to_tensor(self.x)) - self.assertEqual(tuple(actual.shape), - self.x.shape[:-self.reinterpreted_batch_rank]) - expected = self.base.forward_log_det_jacobian(paddle.to_tensor( - self.x)).sum(list(range(-self.reinterpreted_batch_rank, 0))) - np.testing.assert_allclose(actual.numpy(), - expected.numpy(), - rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + self.assertEqual( + tuple(actual.shape), self.x.shape[: -self.reinterpreted_batch_rank] + ) + expected = self.base.forward_log_det_jacobian( + paddle.to_tensor(self.x) + ).sum(list(range(-self.reinterpreted_batch_rank, 0))) + np.testing.assert_allclose( + actual.numpy(), + expected.numpy(), + rtol=config.RTOL.get(str(self.x.dtype)), + atol=config.ATOL.get(str(self.x.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -511,13 +709,12 @@ class TestIndependentTransform(unittest.TestCase): @param.place(config.DEVICES) class TestPowerTransform(unittest.TestCase): - def setUp(self): - self._t = transform.PowerTransform(paddle.to_tensor(2.)) + self._t = transform.PowerTransform(paddle.to_tensor(2.0)) def test_init(self): with self.assertRaises(TypeError): - transform.PowerTransform(1.) + transform.PowerTransform(1.0) def test_is_injective(self): self.assertTrue(self._t._is_injective()) @@ -532,34 +729,50 @@ class TestPowerTransform(unittest.TestCase): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([2.]), np.array([0., -1., 2.]), - np.power(np.array([0., -1., 2.]), 2.)), - (np.array([[0.], [3.]]), np.array([[1., 0.], [5., 6.]]), - np.power(np.array([[1., 0.], [5., 6.]]), - np.array([[0.], [3.]])))]) + @param.param_func( + [ + ( + np.array([2.0]), + np.array([0.0, -1.0, 2.0]), + np.power(np.array([0.0, -1.0, 2.0]), 2.0), + ), + ( + np.array([[0.0], [3.0]]), + np.array([[1.0, 0.0], [5.0, 6.0]]), + np.power( + np.array([[1.0, 0.0], [5.0, 6.0]]), np.array([[0.0], [3.0]]) + ), + ), + ] + ) def test_forward(self, power, x, y): t = transform.PowerTransform(paddle.to_tensor(power)) - np.testing.assert_allclose(t.forward(paddle.to_tensor(x)).numpy(), - y, - rtol=config.RTOL.get(str(x.dtype)), - atol=config.ATOL.get(str(x.dtype))) + np.testing.assert_allclose( + t.forward(paddle.to_tensor(x)).numpy(), + y, + rtol=config.RTOL.get(str(x.dtype)), + atol=config.ATOL.get(str(x.dtype)), + ) - @param.param_func([(np.array([2.]), np.array([4.]), np.array([2.]))]) + @param.param_func([(np.array([2.0]), np.array([4.0]), np.array([2.0]))]) def test_inverse(self, power, y, x): t = transform.PowerTransform(paddle.to_tensor(power)) - np.testing.assert_allclose(t.inverse(paddle.to_tensor(y)).numpy(), - x, - rtol=config.RTOL.get(str(x.dtype)), - atol=config.ATOL.get(str(x.dtype))) + np.testing.assert_allclose( + t.inverse(paddle.to_tensor(y)).numpy(), + x, + rtol=config.RTOL.get(str(x.dtype)), + atol=config.ATOL.get(str(x.dtype)), + ) - @param.param_func(((np.array([2.]), np.array([3., 1.4, 0.8])), )) + @param.param_func(((np.array([2.0]), np.array([3.0, 1.4, 0.8])),)) def test_forward_log_det_jacobian(self, power, x): t = transform.PowerTransform(paddle.to_tensor(power)) - np.testing.assert_allclose(t.forward_log_det_jacobian( - paddle.to_tensor(x)).numpy(), - self._np_forward_jacobian(power, x), - rtol=config.RTOL.get(str(x.dtype)), - atol=config.ATOL.get(str(x.dtype))) + np.testing.assert_allclose( + t.forward_log_det_jacobian(paddle.to_tensor(x)).numpy(), + self._np_forward_jacobian(power, x), + rtol=config.RTOL.get(str(x.dtype)), + atol=config.ATOL.get(str(x.dtype)), + ) def _np_forward_jacobian(self, alpha, x): return np.abs(np.log(alpha * np.power(x, alpha - 1))) @@ -575,7 +788,6 @@ class TestPowerTransform(unittest.TestCase): @param.place(config.DEVICES) class TestTanhTransform(unittest.TestCase): - def setUp(self): self._t = transform.TanhTransform() @@ -594,57 +806,83 @@ class TestTanhTransform(unittest.TestCase): self.assertEqual(self._t._codomain._constraint._lower, -1) self.assertEqual(self._t._codomain._constraint._upper, 1) - @param.param_func([(np.array([0., 1., 2., - 3.]), np.tanh(np.array([0., 1., 2., 3.]))), - (np.array([[0., 1., 2., 3.], [-5., 6., 7., 8.]]), - np.tanh(np.array([[0., 1., 2., 3.], [-5., 6., 7., - 8.]])))]) + @param.param_func( + [ + ( + np.array([0.0, 1.0, 2.0, 3.0]), + np.tanh(np.array([0.0, 1.0, 2.0, 3.0])), + ), + ( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]), + np.tanh( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]) + ), + ), + ] + ) def test_forward(self, input, expected): - np.testing.assert_allclose(self._t.forward( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., - 3.]), np.arctanh(np.array([1., 2., 3.]))), - (np.array([[1., 2., 3.], [6., 7., 8.]]), - np.arctanh(np.array([[1., 2., 3.], [6., 7., 8.]])))]) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]), np.arctanh(np.array([1.0, 2.0, 3.0]))), + ( + np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]), + np.arctanh(np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]])), + ), + ] + ) def test_inverse(self, input, expected): - np.testing.assert_allclose(self._t.inverse( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + np.testing.assert_allclose( + self._t.inverse(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_forward_log_det_jacobian(self, input): - np.testing.assert_allclose(self._t.forward_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward_log_det_jacobian(paddle.to_tensor(input)).numpy(), + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, x): - return 2. * (np.log(2.) - x - self._np_softplus(-2. * x)) + return 2.0 * (np.log(2.0) - x - self._np_softplus(-2.0 * x)) - def _np_softplus(self, x, beta=1., threshold=20.): + def _np_softplus(self, x, beta=1.0, threshold=20.0): if np.any(beta * x > threshold): return x - return 1. / beta * np.log1p(np.exp(beta * x)) + return 1.0 / beta * np.log1p(np.exp(beta * x)) def _np_inverse_jacobian(self, y): return -self._np_forward_jacobian(np.arctanh(y)) - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_inverse_log_det_jacobian(self, input): - np.testing.assert_allclose(self._t.inverse_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse_log_det_jacobian(paddle.to_tensor(input)).numpy(), + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -656,20 +894,24 @@ class TestTanhTransform(unittest.TestCase): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'in_event_shape', 'out_event_shape'), [ - ('regular_shape', (2, 3), (3, 2)), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'in_event_shape', 'out_event_shape'), + [ + ('regular_shape', (2, 3), (3, 2)), + ], +) class TestReshapeTransform(unittest.TestCase): - def setUp(self): - self._t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + self._t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) @param.param_func([(0, 0, TypeError), ((1, 2), (1, 3), ValueError)]) def test_init_exception(self, in_event_shape, out_event_shape, exc): with self.assertRaises(exc): - paddle.distribution.ReshapeTransform(in_event_shape, - out_event_shape) + paddle.distribution.ReshapeTransform( + in_event_shape, out_event_shape + ) def test_is_injective(self): self.assertTrue(self._t._is_injective()) @@ -682,24 +924,30 @@ class TestReshapeTransform(unittest.TestCase): def test_forward(self): x = paddle.ones(self.in_event_shape) - np.testing.assert_allclose(self._t.forward(x), - paddle.ones(self.out_event_shape), - rtol=config.RTOL.get(str(x.numpy().dtype)), - atol=config.ATOL.get(str(x.numpy().dtype))) + np.testing.assert_allclose( + self._t.forward(x), + paddle.ones(self.out_event_shape), + rtol=config.RTOL.get(str(x.numpy().dtype)), + atol=config.ATOL.get(str(x.numpy().dtype)), + ) def test_inverse(self): x = paddle.ones(self.out_event_shape) - np.testing.assert_allclose(self._t.inverse(x).numpy(), - paddle.ones(self.in_event_shape).numpy(), - rtol=config.RTOL.get(str(x.numpy().dtype)), - atol=config.ATOL.get(str(x.numpy().dtype))) + np.testing.assert_allclose( + self._t.inverse(x).numpy(), + paddle.ones(self.in_event_shape).numpy(), + rtol=config.RTOL.get(str(x.numpy().dtype)), + atol=config.ATOL.get(str(x.numpy().dtype)), + ) def test_forward_log_det_jacobian(self): x = paddle.ones(self.in_event_shape) - np.testing.assert_allclose(self._t.forward_log_det_jacobian(x).numpy(), - paddle.zeros([1]).numpy(), - rtol=config.RTOL.get(str(x.numpy().dtype)), - atol=config.ATOL.get(str(x.numpy().dtype))) + np.testing.assert_allclose( + self._t.forward_log_det_jacobian(x).numpy(), + paddle.zeros([1]).numpy(), + rtol=config.RTOL.get(str(x.numpy().dtype)), + atol=config.ATOL.get(str(x.numpy().dtype)), + ) def test_in_event_shape(self): self.assertEqual(self._t.in_event_shape, self.in_event_shape) @@ -718,14 +966,13 @@ class TestReshapeTransform(unittest.TestCase): self._t.inverse_shape(shape) -def _np_softplus(x, beta=1., threshold=20.): +def _np_softplus(x, beta=1.0, threshold=20.0): if np.any(beta * x > threshold): return x - return 1. / beta * np.log1p(np.exp(beta * x)) + return 1.0 / beta * np.log1p(np.exp(beta * x)) class TestSigmoidTransform(unittest.TestCase): - def setUp(self): self._t = transform.SigmoidTransform() @@ -738,32 +985,43 @@ class TestSigmoidTransform(unittest.TestCase): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Variable)) - @param.param_func(((np.ones( - (5, 10)), 1 / (1 + np.exp(-np.ones((5, 10))))), )) + @param.param_func( + ((np.ones((5, 10)), 1 / (1 + np.exp(-np.ones((5, 10))))),) + ) def test_forward(self, input, expected): - np.testing.assert_allclose(self._t.forward(paddle.to_tensor(input)), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func( - ((np.ones(10), np.log(np.ones(10)) - np.log1p(-np.ones(10))), )) + ((np.ones(10), np.log(np.ones(10)) - np.log1p(-np.ones(10))),) + ) def test_inverse(self, input, expected): - np.testing.assert_allclose(self._t.inverse( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func( - ((np.ones(10), - -_np_softplus(-np.ones(10)) - _np_softplus(np.ones(10))), )) + ( + ( + np.ones(10), + -_np_softplus(-np.ones(10)) - _np_softplus(np.ones(10)), + ), + ) + ) def test_forward_log_det_jacobian(self, input, expected): - np.testing.assert_allclose(self._t.forward_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward_log_det_jacobian(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -775,7 +1033,6 @@ class TestSigmoidTransform(unittest.TestCase): class TestSoftmaxTransform(unittest.TestCase): - def setUp(self): self._t = transform.SoftmaxTransform() @@ -788,19 +1045,23 @@ class TestSoftmaxTransform(unittest.TestCase): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Variable)) - @param.param_func(((np.random.random((5, 10)), ), )) + @param.param_func(((np.random.random((5, 10)),),)) def test_forward(self, input): - np.testing.assert_allclose(self._t.forward(paddle.to_tensor(input)), - self._np_forward(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)), + self._np_forward(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) - @param.param_func(((np.random.random(10), ), )) + @param.param_func(((np.random.random(10),),)) def test_inverse(self, input): - np.testing.assert_allclose(self._t.inverse(paddle.to_tensor(input)), - self._np_inverse(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse(paddle.to_tensor(input)), + self._np_inverse(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward(self, x): x = np.exp(x - np.max(x, -1, keepdims=True)[0]) @@ -833,7 +1094,6 @@ class TestSoftmaxTransform(unittest.TestCase): class TestStickBreakingTransform(unittest.TestCase): - def setUp(self): self._t = transform.StickBreakingTransform() @@ -846,13 +1106,14 @@ class TestStickBreakingTransform(unittest.TestCase): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Variable)) - @param.param_func(((np.random.random((10)), ), )) + @param.param_func(((np.random.random((10)),),)) def test_forward(self, input): - np.testing.assert_allclose(self._t.inverse( - self._t.forward(paddle.to_tensor(input))), - input, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse(self._t.forward(paddle.to_tensor(input))), + input, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((2, 3, 5), (2, 3, 6))]) def test_forward_shape(self, shape, expected_shape): @@ -862,19 +1123,22 @@ class TestStickBreakingTransform(unittest.TestCase): def test_inverse_shape(self, shape, expected_shape): self.assertEqual(self._t.inverse_shape(shape), expected_shape) - @param.param_func(((np.random.random((10)), ), )) + @param.param_func(((np.random.random((10)),),)) def test_forward_log_det_jacobian(self, x): self.assertEqual( - self._t.forward_log_det_jacobian(paddle.to_tensor(x)).shape, [1]) + self._t.forward_log_det_jacobian(paddle.to_tensor(x)).shape, [1] + ) # Todo @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'transforms', 'axis'), [ - ('simple_one_transform', [transform.ExpTransform()], 0), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'transforms', 'axis'), + [ + ('simple_one_transform', [transform.ExpTransform()], 0), + ], +) class TestStackTransform(unittest.TestCase): - def setUp(self): self._t = transform.StackTransform(self.transforms, self.axis) @@ -887,25 +1151,42 @@ class TestStackTransform(unittest.TestCase): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Stack)) - @param.param_func([(np.array([[0., 1., 2., 3.]]), ), - (np.array([[-5., 6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([[0.0, 1.0, 2.0, 3.0]]),), + (np.array([[-5.0, 6.0, 7.0, 8.0]]),), + ] + ) def test_forward(self, input): - self.assertEqual(tuple(self._t.forward(paddle.to_tensor(input)).shape), - input.shape) + self.assertEqual( + tuple(self._t.forward(paddle.to_tensor(input)).shape), input.shape + ) - @param.param_func([(np.array([[1., 2., 3.]]), ), - (np.array([[6., 7., 8.]], ), )]) + @param.param_func( + [ + (np.array([[1.0, 2.0, 3.0]]),), + ( + np.array( + [[6.0, 7.0, 8.0]], + ), + ), + ] + ) def test_inverse(self, input): - self.assertEqual(tuple(self._t.inverse(paddle.to_tensor(input)).shape), - input.shape) + self.assertEqual( + tuple(self._t.inverse(paddle.to_tensor(input)).shape), input.shape + ) - @param.param_func([(np.array([[1., 2., 3.]]), ), (np.array([[6., 7., - 8.]]), )]) + @param.param_func( + [(np.array([[1.0, 2.0, 3.0]]),), (np.array([[6.0, 7.0, 8.0]]),)] + ) def test_forward_log_det_jacobian(self, input): self.assertEqual( tuple( - self._t.forward_log_det_jacobian( - paddle.to_tensor(input)).shape), input.shape) + self._t.forward_log_det_jacobian(paddle.to_tensor(input)).shape + ), + input.shape, + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -918,9 +1199,13 @@ class TestStackTransform(unittest.TestCase): def test_axis(self): self.assertEqual(self._t.axis, self.axis) - @param.param_func([(0, 0, TypeError), ([0], 0, TypeError), - ([paddle.distribution.ExpTransform()], 'axis', TypeError) - ]) + @param.param_func( + [ + (0, 0, TypeError), + ([0], 0, TypeError), + ([paddle.distribution.ExpTransform()], 'axis', TypeError), + ] + ) def test_init_exception(self, transforms, axis, exc): with self.assertRaises(exc): paddle.distribution.StackTransform(transforms, axis) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform_static.py index 4af17de382e..c5131468faa 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform_static.py @@ -28,14 +28,17 @@ paddle.enable_static() @param.place(config.DEVICES) class TestTransform(unittest.TestCase): - def setUp(self): self._t = transform.Transform() - @param.param_func([(transform.Type.BIJECTION, True), - (transform.Type.INJECTION, True), - (transform.Type.SURJECTION, False), - (transform.Type.OTHER, False)]) + @param.param_func( + [ + (transform.Type.BIJECTION, True), + (transform.Type.INJECTION, True), + (transform.Type.SURJECTION, False), + (transform.Type.OTHER, False), + ] + ) def test_is_injective(self, type, expected): transform.Transform._type = type self.assertEqual(self._t._is_injective(), expected) @@ -46,8 +49,12 @@ class TestTransform(unittest.TestCase): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Real)) - @param.param_func([(np.array(0), NotImplementedError), - (np.random.random((2, 3)), NotImplementedError)]) + @param.param_func( + [ + (np.array(0), NotImplementedError), + (np.random.random((2, 3)), NotImplementedError), + ] + ) def test_forward(self, input, expected): with self.assertRaises(expected): exe = paddle.static.Executor() @@ -55,14 +62,19 @@ class TestTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.Transform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.forward(static_input) exe.run(sp) exe.run(mp, feed={'input': input}, fetch_list=[output]) - @param.param_func([(np.array(0), NotImplementedError), - (np.random.random((2, 3)), NotImplementedError)]) + @param.param_func( + [ + (np.array(0), NotImplementedError), + (np.random.random((2, 3)), NotImplementedError), + ] + ) def test_inverse(self, input, expected): with self.assertRaises(expected): exe = paddle.static.Executor() @@ -70,14 +82,19 @@ class TestTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.Transform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.inverse(static_input) exe.run(sp) exe.run(mp, feed={'input': input}, fetch_list=[output]) - @param.param_func([(np.array(0), NotImplementedError), - (paddle.rand((2, 3)), NotImplementedError)]) + @param.param_func( + [ + (np.array(0), NotImplementedError), + (paddle.rand((2, 3)), NotImplementedError), + ] + ) def test_forward_log_det_jacobian(self, input, expected): with self.assertRaises(expected): exe = paddle.static.Executor() @@ -85,14 +102,19 @@ class TestTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.Transform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.forward_log_det_jacobian(static_input) exe.run(sp) exe.run(mp, feed={'input': input}, fetch_list=[output]) - @param.param_func([(np.array(0), NotImplementedError), - (paddle.rand((2, 3)), NotImplementedError)]) + @param.param_func( + [ + (np.array(0), NotImplementedError), + (paddle.rand((2, 3)), NotImplementedError), + ] + ) def test_inverse_log_det_jacobian(self, input, expected): with self.assertRaises(expected): exe = paddle.static.Executor() @@ -100,8 +122,9 @@ class TestTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.Transform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.inverse_log_det_jacobian(static_input) exe.run(sp) exe.run(mp, feed={'input': input}, fetch_list=[output]) @@ -119,7 +142,6 @@ class TestTransform(unittest.TestCase): @param.place(config.DEVICES) class TestAbsTransform(unittest.TestCase): - def setUp(self): self._t = transform.AbsTransform() @@ -136,9 +158,15 @@ class TestAbsTransform(unittest.TestCase): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([-1., 1., 0.]), np.array([1., 1., 0.])), - (np.array([[1., -1., -0.1], [-3., -0.1, 0]]), - np.array([[1., 1., 0.1], [3., 0.1, 0]]))]) + @param.param_func( + [ + (np.array([-1.0, 1.0, 0.0]), np.array([1.0, 1.0, 0.0])), + ( + np.array([[1.0, -1.0, -0.1], [-3.0, -0.1, 0]]), + np.array([[1.0, 1.0, 0.1], [3.0, 0.1, 0]]), + ), + ] + ) def test_forward(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -149,12 +177,14 @@ class TestAbsTransform(unittest.TestCase): output = t.forward(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1.]), (-np.array([1.]), np.array([1.])))]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func([(np.array([1.0]), (-np.array([1.0]), np.array([1.0])))]) def test_inverse(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -164,36 +194,43 @@ class TestAbsTransform(unittest.TestCase): static_input = paddle.static.data('input', input.shape, input.dtype) actual0, actual1 = t.inverse(static_input) exe.run(sp) - [actual0, actual1] = exe.run(mp, - feed={'input': input}, - fetch_list=[actual0, actual1]) + [actual0, actual1] = exe.run( + mp, feed={'input': input}, fetch_list=[actual0, actual1] + ) expected0, expected1 = expected - np.testing.assert_allclose(actual0, - expected0, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - np.testing.assert_allclose(actual1, - expected1, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + actual0, + expected0, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + np.testing.assert_allclose( + actual1, + expected1, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def test_forward_log_det_jacobian(self): - input = np.random.random((10, )) + input = np.random.random((10,)) with self.assertRaises(NotImplementedError): exe = paddle.static.Executor() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.AbsTransform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.forward_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - @param.param_func([ - (np.array([1.]), (np.array([0.]), np.array([0.]))), - ]) + @param.param_func( + [ + (np.array([1.0]), (np.array([0.0]), np.array([0.0]))), + ] + ) def test_inverse_log_det_jacobian(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -203,18 +240,22 @@ class TestAbsTransform(unittest.TestCase): static_input = paddle.static.data('input', input.shape, input.dtype) actual0, actual1 = t.inverse_log_det_jacobian(static_input) exe.run(sp) - [actual0, actual1] = exe.run(mp, - feed={'input': input}, - fetch_list=[actual0, actual1]) + [actual0, actual1] = exe.run( + mp, feed={'input': input}, fetch_list=[actual0, actual1] + ) expected0, expected1 = expected - np.testing.assert_allclose(actual0, - expected0, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - np.testing.assert_allclose(actual1, - expected1, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + actual0, + expected0, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + np.testing.assert_allclose( + actual1, + expected1, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -226,19 +267,22 @@ class TestAbsTransform(unittest.TestCase): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'loc', 'scale'), [ - ('normal', np.random.rand(8, 10), np.random.rand(8, 10)), - ('broadcast', np.random.rand(2, 10), np.random.rand(10)), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('normal', np.random.rand(8, 10), np.random.rand(8, 10)), + ('broadcast', np.random.rand(2, 10), np.random.rand(10)), + ], +) class TestAffineTransform(unittest.TestCase): - def setUp(self): sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) self._t = transform.AffineTransform(loc, scale) def test_is_injective(self): @@ -261,24 +305,26 @@ class TestAffineTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) t = transform.AffineTransform(loc, scale) - static_input = paddle.static.data('input', self.loc.shape, - self.loc.dtype) + static_input = paddle.static.data( + 'input', self.loc.shape, self.loc.dtype + ) output = t.forward(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'loc': self.loc, - 'scale': self.scale - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward(input), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + [output] = exe.run( + mp, + feed={'input': input, 'loc': self.loc, 'scale': self.scale}, + fetch_list=[output], + ) + np.testing.assert_allclose( + output, + self._np_forward(input), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_inverse(self): input = np.random.random(self.loc.shape) @@ -287,24 +333,26 @@ class TestAffineTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) t = transform.AffineTransform(loc, scale) - static_input = paddle.static.data('input', self.loc.shape, - self.loc.dtype) + static_input = paddle.static.data( + 'input', self.loc.shape, self.loc.dtype + ) output = t.inverse(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'loc': self.loc, - 'scale': self.scale - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_inverse(input), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + [output] = exe.run( + mp, + feed={'input': input, 'loc': self.loc, 'scale': self.scale}, + fetch_list=[output], + ) + np.testing.assert_allclose( + output, + self._np_inverse(input), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def _np_forward(self, x): return self.loc + self.scale * x @@ -325,23 +373,24 @@ class TestAffineTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) t = transform.AffineTransform(loc, scale) static_input = paddle.static.data('input', input.shape, input.dtype) output = t.inverse_log_det_jacobian(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'loc': self.loc, - 'scale': self.scale - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + [output] = exe.run( + mp, + feed={'input': input, 'loc': self.loc, 'scale': self.scale}, + fetch_list=[output], + ) + np.testing.assert_allclose( + output, + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_forward_log_det_jacobian(self): input = np.random.random(self.scale.shape) @@ -350,40 +399,42 @@ class TestAffineTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) t = transform.AffineTransform(loc, scale) static_input = paddle.static.data('input', input.shape, input.dtype) output = t.forward_log_det_jacobian(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'loc': self.loc, - 'scale': self.scale - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + [output] = exe.run( + mp, + feed={'input': input, 'loc': self.loc, 'scale': self.scale}, + fetch_list=[output], + ) + np.testing.assert_allclose( + output, + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_forward_shape(self): shape = self.loc.shape self.assertEqual( tuple(self._t.forward_shape(shape)), - np.broadcast(np.random.random(shape), self.loc, self.scale).shape) + np.broadcast(np.random.random(shape), self.loc, self.scale).shape, + ) def test_inverse_shape(self): shape = self.scale.shape self.assertEqual( tuple(self._t.forward_shape(shape)), - np.broadcast(np.random.random(shape), self.loc, self.scale).shape) + np.broadcast(np.random.random(shape), self.loc, self.scale).shape, + ) @param.place(config.DEVICES) class TestExpTransform(unittest.TestCase): - def setUp(self): self._t = transform.ExpTransform() @@ -400,11 +451,18 @@ class TestExpTransform(unittest.TestCase): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([0., 1., 2., - 3.]), np.exp(np.array([0., 1., 2., 3.]))), - (np.array([[0., 1., 2., 3.], [-5., 6., 7., 8.]]), - np.exp(np.array([[0., 1., 2., 3.], [-5., 6., 7., - 8.]])))]) + @param.param_func( + [ + ( + np.array([0.0, 1.0, 2.0, 3.0]), + np.exp(np.array([0.0, 1.0, 2.0, 3.0])), + ), + ( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]), + np.exp(np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]])), + ), + ] + ) def test_forward(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -415,14 +473,22 @@ class TestExpTransform(unittest.TestCase): output = t.forward(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), np.log(np.array([1., 2., 3.]))), - (np.array([[1., 2., 3.], [6., 7., 8.]]), - np.log(np.array([[1., 2., 3.], [6., 7., 8.]])))]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]), np.log(np.array([1.0, 2.0, 3.0]))), + ( + np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]), + np.log(np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]])), + ), + ] + ) def test_inverse(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -433,13 +499,19 @@ class TestExpTransform(unittest.TestCase): output = t.inverse(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_forward_log_det_jacobian(self, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -450,16 +522,22 @@ class TestExpTransform(unittest.TestCase): output = t.forward_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + output, + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, x): return x - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_inverse_log_det_jacobian(self, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -470,10 +548,12 @@ class TestExpTransform(unittest.TestCase): output = t.inverse_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + output, + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_inverse_jacobian(self, y): return -self._np_forward_jacobian(np.log(y)) @@ -489,44 +569,102 @@ class TestExpTransform(unittest.TestCase): @param.place(config.DEVICES) class TestChainTransform(unittest.TestCase): - - @param.param_func(((transform.ChainTransform( - (transform.AbsTransform(), - transform.AffineTransform(paddle.rand([1]), paddle.rand([1])))), - False), (transform.ChainTransform(( - transform.AffineTransform(paddle.rand([1]), - paddle.rand([1])), - transform.ExpTransform(), - )), True))) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.AbsTransform(), + transform.AffineTransform( + paddle.rand([1]), paddle.rand([1]) + ), + ) + ), + False, + ), + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.rand([1]), paddle.rand([1]) + ), + transform.ExpTransform(), + ) + ), + True, + ), + ) + ) def test_is_injective(self, chain, expected): self.assertEqual(chain._is_injective(), expected) - @param.param_func(((transform.ChainTransform( - (transform.IndependentTransform(transform.ExpTransform(), 1), - transform.IndependentTransform(transform.ExpTransform(), 10), - transform.IndependentTransform(transform.ExpTransform(), 8))), - variable.Independent(variable.real, 10)), )) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.IndependentTransform( + transform.ExpTransform(), 1 + ), + transform.IndependentTransform( + transform.ExpTransform(), 10 + ), + transform.IndependentTransform( + transform.ExpTransform(), 8 + ), + ) + ), + variable.Independent(variable.real, 10), + ), + ) + ) def test_domain(self, input, expected): self.assertIsInstance(input._domain, type(expected)) self.assertEqual(input._domain.event_rank, expected.event_rank) self.assertEqual(input._domain.is_discrete, expected.is_discrete) - @param.param_func(((transform.ChainTransform( - (transform.IndependentTransform(transform.ExpTransform(), 9), - transform.IndependentTransform(transform.ExpTransform(), 4), - transform.IndependentTransform(transform.ExpTransform(), 5))), - variable.Independent(variable.real, 9)), )) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.IndependentTransform( + transform.ExpTransform(), 9 + ), + transform.IndependentTransform( + transform.ExpTransform(), 4 + ), + transform.IndependentTransform( + transform.ExpTransform(), 5 + ), + ) + ), + variable.Independent(variable.real, 9), + ), + ) + ) def test_codomain(self, input, expected): self.assertIsInstance(input._codomain, variable.Independent) self.assertEqual(input._codomain.event_rank, expected.event_rank) self.assertEqual(input._codomain.is_discrete, expected.is_discrete) - @param.param_func([ - (transform.ChainTransform( - (transform.ExpTransform(), transform.TanhTransform())), - np.array([[0., -1., 2., -3.], [-5., 6., 7., -8.]]), - np.tanh(np.exp(np.array([[0., -1., 2., -3.], [-5., 6., 7., -8.]])))) - ]) + @param.param_func( + [ + ( + transform.ChainTransform( + (transform.ExpTransform(), transform.TanhTransform()) + ), + np.array([[0.0, -1.0, 2.0, -3.0], [-5.0, 6.0, 7.0, -8.0]]), + np.tanh( + np.exp( + np.array( + [[0.0, -1.0, 2.0, -3.0], [-5.0, 6.0, 7.0, -8.0]] + ) + ) + ), + ) + ] + ) def test_forward(self, chain, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -537,17 +675,28 @@ class TestChainTransform(unittest.TestCase): output = t.forward(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.ExpTransform(), transform.TanhTransform())), - np.array([[0., 1., 2., 3.], [5., 6., 7., 8.]]), - np.log(np.arctanh(np.array([[0., 1., 2., 3.], [5., 6., 7., 8.]])))) - ]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + (transform.ExpTransform(), transform.TanhTransform()) + ), + np.array([[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0]]), + np.log( + np.arctanh( + np.array([[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0]]) + ) + ), + ) + ] + ) def test_inverse(self, chain, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -558,41 +707,69 @@ class TestChainTransform(unittest.TestCase): output = t.inverse(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.full([1], 0.0), - paddle.full([1], -1.0)), - transform.ExpTransform())), (2, 3, 5), (2, 3, 5)), - ]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.full([1], 0.0), paddle.full([1], -1.0) + ), + transform.ExpTransform(), + ) + ), + (2, 3, 5), + (2, 3, 5), + ), + ] + ) def test_forward_shape(self, chain, shape, expected_shape): self.assertEqual(chain.forward_shape(shape), expected_shape) - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.full([1], 0.0), - paddle.full([1], -1.0)), - transform.ExpTransform())), (2, 3, 5), (2, 3, 5)), - ]) + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.full([1], 0.0), paddle.full([1], -1.0) + ), + transform.ExpTransform(), + ) + ), + (2, 3, 5), + (2, 3, 5), + ), + ] + ) def test_inverse_shape(self, chain, shape, expected_shape): self.assertEqual(chain.forward_shape(shape), expected_shape) @param.place(config.DEVICES) @param.param_cls( - (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'x'), [ - ('rank-over-zero', transform.ExpTransform(), 2, np.random.rand(2, 3, - 3)), - ]) + (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'x'), + [ + ( + 'rank-over-zero', + transform.ExpTransform(), + 2, + np.random.rand(2, 3, 3), + ), + ], +) class TestIndependentTransform(unittest.TestCase): - def setUp(self): - self._t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) + self._t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) def test_is_injective(self): self.assertEqual(self._t._is_injective(), self.base._is_injective()) @@ -601,81 +778,98 @@ class TestIndependentTransform(unittest.TestCase): self.assertTrue(isinstance(self._t._domain, variable.Independent)) self.assertEqual( self._t._domain.event_rank, - self.base._domain.event_rank + self.reinterpreted_batch_rank) - self.assertEqual(self._t._domain.is_discrete, - self.base._domain.is_discrete) + self.base._domain.event_rank + self.reinterpreted_batch_rank, + ) + self.assertEqual( + self._t._domain.is_discrete, self.base._domain.is_discrete + ) def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Independent)) self.assertEqual( self._t._codomain.event_rank, - self.base._codomain.event_rank + self.reinterpreted_batch_rank) - self.assertEqual(self._t._codomain.is_discrete, - self.base._codomain.is_discrete) + self.base._codomain.event_rank + self.reinterpreted_batch_rank, + ) + self.assertEqual( + self._t._codomain.is_discrete, self.base._codomain.is_discrete + ) def test_forward(self): exe = paddle.static.Executor() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): - t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) - static_input = paddle.static.data('input', self.x.shape, - self.x.dtype) + t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) + static_input = paddle.static.data( + 'input', self.x.shape, self.x.dtype + ) output = t.forward(static_input) expected = self.base.forward(static_input) exe.run(sp) - [output, expected] = exe.run(mp, - feed={'input': self.x}, - fetch_list=[output, expected]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + [output, expected] = exe.run( + mp, feed={'input': self.x}, fetch_list=[output, expected] + ) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(self.x.dtype)), + atol=config.ATOL.get(str(self.x.dtype)), + ) def test_inverse(self): exe = paddle.static.Executor() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): - t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) - static_input = paddle.static.data('input', self.x.shape, - self.x.dtype) + t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) + static_input = paddle.static.data( + 'input', self.x.shape, self.x.dtype + ) output = t.inverse(static_input) expected = self.base.inverse(static_input) exe.run(sp) - [output, expected] = exe.run(mp, - feed={'input': self.x}, - fetch_list=[output, expected]) - np.testing.assert_allclose(expected, - output, - rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + [output, expected] = exe.run( + mp, feed={'input': self.x}, fetch_list=[output, expected] + ) + np.testing.assert_allclose( + expected, + output, + rtol=config.RTOL.get(str(self.x.dtype)), + atol=config.ATOL.get(str(self.x.dtype)), + ) def test_forward_log_det_jacobian(self): exe = paddle.static.Executor() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): - t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) - static_input = paddle.static.data('input', self.x.shape, - self.x.dtype) + t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) + static_input = paddle.static.data( + 'input', self.x.shape, self.x.dtype + ) output = t.forward_log_det_jacobian(static_input) expected = self.base.forward_log_det_jacobian( - static_input.sum(list(range(-self.reinterpreted_batch_rank, - 0)))) + static_input.sum(list(range(-self.reinterpreted_batch_rank, 0))) + ) exe.run(sp) - [actual, expected] = exe.run(mp, - feed={'input': self.x}, - fetch_list=[output, expected]) - self.assertEqual(tuple(actual.shape), - self.x.shape[:-self.reinterpreted_batch_rank]) - np.testing.assert_allclose(actual, - expected, - rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + [actual, expected] = exe.run( + mp, feed={'input': self.x}, fetch_list=[output, expected] + ) + self.assertEqual( + tuple(actual.shape), self.x.shape[: -self.reinterpreted_batch_rank] + ) + np.testing.assert_allclose( + actual, + expected, + rtol=config.RTOL.get(str(self.x.dtype)), + atol=config.ATOL.get(str(self.x.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -688,13 +882,12 @@ class TestIndependentTransform(unittest.TestCase): @param.place(config.DEVICES) class TestPowerTransform(unittest.TestCase): - def setUp(self): - self._t = transform.PowerTransform(paddle.full([1], 2.)) + self._t = transform.PowerTransform(paddle.full([1], 2.0)) def test_init(self): with self.assertRaises(TypeError): - transform.PowerTransform(1.) + transform.PowerTransform(1.0) def test_is_injective(self): self.assertTrue(self._t._is_injective()) @@ -709,11 +902,22 @@ class TestPowerTransform(unittest.TestCase): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([2.]), np.array([0., -1., 2.]), - np.power(np.array([0., -1., 2.]), 2.)), - (np.array([[0.], [3.]]), np.array([[1., 0.], [5., 6.]]), - np.power(np.array([[1., 0.], [5., 6.]]), - np.array([[0.], [3.]])))]) + @param.param_func( + [ + ( + np.array([2.0]), + np.array([0.0, -1.0, 2.0]), + np.power(np.array([0.0, -1.0, 2.0]), 2.0), + ), + ( + np.array([[0.0], [3.0]]), + np.array([[1.0, 0.0], [5.0, 6.0]]), + np.power( + np.array([[1.0, 0.0], [5.0, 6.0]]), np.array([[0.0], [3.0]]) + ), + ), + ] + ) def test_forward(self, power, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -724,18 +928,17 @@ class TestPowerTransform(unittest.TestCase): static_input = paddle.static.data('input', input.shape, input.dtype) output = t.forward(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'power': power - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([2.]), np.array([4.]), np.array([2.]))]) + [output] = exe.run( + mp, feed={'input': input, 'power': power}, fetch_list=[output] + ) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func([(np.array([2.0]), np.array([4.0]), np.array([2.0]))]) def test_inverse(self, power, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -746,18 +949,17 @@ class TestPowerTransform(unittest.TestCase): static_input = paddle.static.data('input', input.shape, input.dtype) output = t.inverse(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'power': power - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func(((np.array([2.]), np.array([3., 1.4, 0.8])), )) + [output] = exe.run( + mp, feed={'input': input, 'power': power}, fetch_list=[output] + ) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func(((np.array([2.0]), np.array([3.0, 1.4, 0.8])),)) def test_forward_log_det_jacobian(self, power, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -768,16 +970,15 @@ class TestPowerTransform(unittest.TestCase): static_input = paddle.static.data('input', input.shape, input.dtype) output = t.forward_log_det_jacobian(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'power': power - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward_jacobian(power, input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + [output] = exe.run( + mp, feed={'input': input, 'power': power}, fetch_list=[output] + ) + np.testing.assert_allclose( + output, + self._np_forward_jacobian(power, input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, alpha, x): return np.abs(np.log(alpha * np.power(x, alpha - 1))) @@ -793,7 +994,6 @@ class TestPowerTransform(unittest.TestCase): @param.place(config.DEVICES) class TestTanhTransform(unittest.TestCase): - def setUp(self): self._t = transform.TanhTransform() @@ -812,11 +1012,20 @@ class TestTanhTransform(unittest.TestCase): self.assertEqual(self._t._codomain._constraint._lower, -1) self.assertEqual(self._t._codomain._constraint._upper, 1) - @param.param_func([(np.array([0., 1., 2., - 3.]), np.tanh(np.array([0., 1., 2., 3.]))), - (np.array([[0., 1., 2., 3.], [-5., 6., 7., 8.]]), - np.tanh(np.array([[0., 1., 2., 3.], [-5., 6., 7., - 8.]])))]) + @param.param_func( + [ + ( + np.array([0.0, 1.0, 2.0, 3.0]), + np.tanh(np.array([0.0, 1.0, 2.0, 3.0])), + ), + ( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]), + np.tanh( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]) + ), + ), + ] + ) def test_forward(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -827,15 +1036,22 @@ class TestTanhTransform(unittest.TestCase): output = t.forward(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., - 3.]), np.arctanh(np.array([1., 2., 3.]))), - (np.array([[1., 2., 3.], [6., 7., 8.]]), - np.arctanh(np.array([[1., 2., 3.], [6., 7., 8.]])))]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]), np.arctanh(np.array([1.0, 2.0, 3.0]))), + ( + np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]), + np.arctanh(np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]])), + ), + ] + ) def test_inverse(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -846,13 +1062,19 @@ class TestTanhTransform(unittest.TestCase): output = t.inverse(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_forward_log_det_jacobian(self, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -863,24 +1085,30 @@ class TestTanhTransform(unittest.TestCase): output = t.forward_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + output, + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, x): - return 2. * (np.log(2.) - x - self._np_softplus(-2. * x)) + return 2.0 * (np.log(2.0) - x - self._np_softplus(-2.0 * x)) - def _np_softplus(self, x, beta=1., threshold=20.): + def _np_softplus(self, x, beta=1.0, threshold=20.0): if np.any(beta * x > threshold): return x - return 1. / beta * np.log1p(np.exp(beta * x)) + return 1.0 / beta * np.log1p(np.exp(beta * x)) def _np_inverse_jacobian(self, y): return -self._np_forward_jacobian(np.arctanh(y)) - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_inverse_log_det_jacobian(self, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -891,10 +1119,12 @@ class TestTanhTransform(unittest.TestCase): output = t.inverse_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + output, + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -906,14 +1136,17 @@ class TestTanhTransform(unittest.TestCase): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'in_event_shape', 'out_event_shape'), [ - ('regular_shape', (2, 3), (3, 2)), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'in_event_shape', 'out_event_shape'), + [ + ('regular_shape', (2, 3), (3, 2)), + ], +) class TestReshapeTransform(unittest.TestCase): - def setUp(self): - self._t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + self._t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) def test_is_injective(self): self.assertTrue(self._t._is_injective()) @@ -930,16 +1163,19 @@ class TestReshapeTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): x = paddle.ones(self.in_event_shape) - t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) output = self._t.forward(x) exe.run(sp) [output] = exe.run(mp, feed={}, fetch_list=[output]) expected = np.ones(self.out_event_shape) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(expected.dtype)), - atol=config.ATOL.get(str(expected.dtype))) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(expected.dtype)), + atol=config.ATOL.get(str(expected.dtype)), + ) def test_inverse(self): exe = paddle.static.Executor() @@ -947,17 +1183,20 @@ class TestReshapeTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): x = paddle.ones(self.out_event_shape) - t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) output = self._t.inverse(x) exe.run(sp) [output] = exe.run(mp, feed={}, fetch_list=[output]) expected = np.ones(self.in_event_shape) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(expected.dtype)), - atol=config.ATOL.get(str(expected.dtype))) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(expected.dtype)), + atol=config.ATOL.get(str(expected.dtype)), + ) def test_forward_log_det_jacobian(self): exe = paddle.static.Executor() @@ -965,16 +1204,19 @@ class TestReshapeTransform(unittest.TestCase): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): x = paddle.ones(self.in_event_shape) - t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) output = self._t.forward_log_det_jacobian(x) exe.run(sp) [output] = exe.run(mp, feed={}, fetch_list=[output]) expected = np.zeros([1]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(expected.dtype)), - atol=config.ATOL.get(str(expected.dtype))) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(expected.dtype)), + atol=config.ATOL.get(str(expected.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution.py index c1aebf7e0d6..3c38c8cb4df 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution.py @@ -21,14 +21,21 @@ import parameterize as param @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'base', 'transforms'), - [('base_normal', paddle.distribution.Normal( - 0., 1.), [paddle.distribution.ExpTransform()])]) +@param.param_cls( + (param.TEST_CASE_NAME, 'base', 'transforms'), + [ + ( + 'base_normal', + paddle.distribution.Normal(0.0, 1.0), + [paddle.distribution.ExpTransform()], + ) + ], +) class TestIndependent(unittest.TestCase): - def setUp(self): self._t = paddle.distribution.TransformedDistribution( - self.base, self.transforms) + self.base, self.transforms + ) def _np_sum_rightmost(self, value, n): return np.sum(value, tuple(range(-n, 0))) if n > 0 else value @@ -39,7 +46,8 @@ class TestIndependent(unittest.TestCase): self.simple_log_prob(value, self.base, self.transforms), self._t.log_prob(value), rtol=config.RTOL.get(str(value.numpy().dtype)), - atol=config.ATOL.get(str(value.numpy().dtype))) + atol=config.ATOL.get(str(value.numpy().dtype)), + ) def simple_log_prob(self, value, base, transforms): log_prob = 0.0 diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution_static.py index b9f54498f64..f9c7ec6ef74 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution_static.py @@ -23,15 +23,21 @@ paddle.enable_static() @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'base', 'transforms'), - [('base_normal', paddle.distribution.Normal, - [paddle.distribution.ExpTransform()])]) +@param.param_cls( + (param.TEST_CASE_NAME, 'base', 'transforms'), + [ + ( + 'base_normal', + paddle.distribution.Normal, + [paddle.distribution.ExpTransform()], + ) + ], +) class TestIndependent(unittest.TestCase): - def setUp(self): value = np.array([0.5]) - loc = np.array([0.]) - scale = np.array([1.]) + loc = np.array([0.0]) + scale = np.array([1.0]) shape = [5, 10, 8] self.dtype = value.dtype exe = paddle.static.Executor() @@ -43,28 +49,32 @@ class TestIndependent(unittest.TestCase): static_scale = paddle.static.data('scale', scale.shape, scale.dtype) self.base = self.base(static_loc, static_scale) self._t = paddle.distribution.TransformedDistribution( - self.base, self.transforms) + self.base, self.transforms + ) actual_log_prob = self._t.log_prob(static_value) expected_log_prob = self.transformed_log_prob( - static_value, self.base, self.transforms) + static_value, self.base, self.transforms + ) sample_data = self._t.sample(shape) exe.run(sp) - [self.actual_log_prob, self.expected_log_prob, - self.sample_data] = exe.run( - mp, - feed={ - 'value': value, - 'loc': loc, - 'scale': scale - }, - fetch_list=[actual_log_prob, expected_log_prob, sample_data]) + [ + self.actual_log_prob, + self.expected_log_prob, + self.sample_data, + ] = exe.run( + mp, + feed={'value': value, 'loc': loc, 'scale': scale}, + fetch_list=[actual_log_prob, expected_log_prob, sample_data], + ) def test_log_prob(self): - np.testing.assert_allclose(self.actual_log_prob, - self.expected_log_prob, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self.actual_log_prob, + self.expected_log_prob, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) def transformed_log_prob(self, value, base, transforms): log_prob = 0.0 diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_uniform.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_uniform.py index 6aafa9cb356..522e9bebce4 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_uniform.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_uniform.py @@ -27,7 +27,6 @@ np.random.seed(2022) class UniformNumpy(DistributionNumpy): - def __init__(self, low, high): self.low = np.array(low) self.high = np.array(high) @@ -37,8 +36,9 @@ class UniformNumpy(DistributionNumpy): def sample(self, shape): shape = tuple(shape) + (self.low + self.high).shape - return self.low + (np.random.uniform(size=shape) * - (self.high - self.low)) + return self.low + ( + np.random.uniform(size=shape) * (self.high - self.low) + ) def log_prob(self, value): lb = np.less(self.low, value).astype(self.low.dtype) @@ -55,7 +55,6 @@ class UniformNumpy(DistributionNumpy): class UniformTest(unittest.TestCase): - def setUp(self, use_gpu=False, batch_size=5, dims=6): self.use_gpu = use_gpu if not use_gpu: @@ -90,9 +89,9 @@ class UniformTest(unittest.TestCase): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[], dtype='float32' + ) def compare_with_numpy(self, fetch_list, sample_shape=7, tolerance=1e-6): sample, entropy, log_prob, probs = fetch_list @@ -104,14 +103,12 @@ class UniformTest(unittest.TestCase): np_p = np_uniform.probs(self.values_np) np.testing.assert_equal(sample.shape, np_sample.shape) - np.testing.assert_allclose(entropy, - np_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(log_prob, - np_lp, - rtol=tolerance, - atol=tolerance) + np.testing.assert_allclose( + entropy, np_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + log_prob, np_lp, rtol=tolerance, atol=tolerance + ) np.testing.assert_allclose(probs, np_p, rtol=tolerance, atol=tolerance) def test_uniform_distribution_static(self, sample_shape=7, tolerance=1e-6): @@ -127,13 +124,13 @@ class UniformTest(unittest.TestCase): feed_vars = { 'low': self.low_np, 'high': self.high_np, - 'values': self.values_np + 'values': self.values_np, } self.executor.run(fluid.default_startup_program()) - fetch_list = self.executor.run(program=self.test_program, - feed=feed_vars, - fetch_list=fetch_list) + fetch_list = self.executor.run( + program=self.test_program, feed=feed_vars, fetch_list=fetch_list + ) self.compare_with_numpy(fetch_list) @@ -157,7 +154,6 @@ class UniformTest(unittest.TestCase): class UniformTest2(UniformTest): - def init_numpy_data(self, batch_size, dims): # low ans high are 'int' self.low_np = int(np.random.uniform(-2, 1)) @@ -166,48 +162,48 @@ class UniformTest2(UniformTest): class UniformTest3(UniformTest): - def init_numpy_data(self, batch_size, dims): # test broadcast: low is float, high is numpy.ndarray with dtype 'float32'. self.low_np = np.random.uniform(-2, 1) - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float32') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float32' + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest4(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are numpy.ndarray with dtype 'float32'. self.low_np = np.random.randn(batch_size, dims).astype('float32') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float32') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float32' + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest5(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are numpy.ndarray with dtype 'float64'. self.low_np = np.random.randn(batch_size, dims).astype('float64') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float64') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float64' + ) self.values_np = np.random.randn(batch_size, dims).astype('float64') def init_dynamic_data(self, batch_size, dims): @@ -219,18 +215,18 @@ class UniformTest5(UniformTest): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float64') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float64' + ) class UniformTest6(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are Tensor with dtype 'VarType.FP32'. self.low_np = np.random.randn(batch_size, dims).astype('float32') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float32') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float32' + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_dynamic_data(self, batch_size, dims): @@ -240,24 +236,24 @@ class UniformTest6(UniformTest): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_low = layers.data(name='low', - shape=[dims], - dtype='float32') - self.static_high = layers.data(name='high', - shape=[dims], - dtype='float32') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_low = layers.data( + name='low', shape=[dims], dtype='float32' + ) + self.static_high = layers.data( + name='high', shape=[dims], dtype='float32' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest7(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are Tensor with dtype 'VarType.FP64'. self.low_np = np.random.randn(batch_size, dims).astype('float64') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float64') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float64' + ) self.values_np = np.random.randn(batch_size, dims).astype('float64') def init_dynamic_data(self, batch_size, dims): @@ -267,24 +263,24 @@ class UniformTest7(UniformTest): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_low = layers.data(name='low', - shape=[dims], - dtype='float64') - self.static_high = layers.data(name='high', - shape=[dims], - dtype='float64') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float64') + self.static_low = layers.data( + name='low', shape=[dims], dtype='float64' + ) + self.static_high = layers.data( + name='high', shape=[dims], dtype='float64' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float64' + ) class UniformTest8(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are Tensor with dtype 'VarType.FP64'. value's dtype is 'VarType.FP32'. self.low_np = np.random.randn(batch_size, dims).astype('float64') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float64') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float64' + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_dynamic_data(self, batch_size, dims): @@ -294,77 +290,81 @@ class UniformTest8(UniformTest): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_low = layers.data(name='low', - shape=[dims], - dtype='float64') - self.static_high = layers.data(name='high', - shape=[dims], - dtype='float64') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_low = layers.data( + name='low', shape=[dims], dtype='float64' + ) + self.static_high = layers.data( + name='high', shape=[dims], dtype='float64' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest9(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are numpy.ndarray with dtype 'float32'. # high < low. self.low_np = np.random.randn(batch_size, dims).astype('float32') - self.high_np = np.random.uniform(-10.0, -5.0, - (batch_size, dims)).astype('float32') + self.high_np = np.random.uniform( + -10.0, -5.0, (batch_size, dims) + ).astype('float32') self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest10(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are list. - self.low_np = np.random.randn(batch_size, - dims).astype('float32').tolist() - self.high_np = np.random.uniform( - 5.0, 15.0, (batch_size, dims)).astype('float32').tolist() + self.low_np = ( + np.random.randn(batch_size, dims).astype('float32').tolist() + ) + self.high_np = ( + np.random.uniform(5.0, 15.0, (batch_size, dims)) + .astype('float32') + .tolist() + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest11(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are tuple. self.low_np = tuple( - np.random.randn(batch_size, dims).astype('float32').tolist()) + np.random.randn(batch_size, dims).astype('float32').tolist() + ) self.high_np = tuple( - np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float32').tolist()) + np.random.uniform(5.0, 15.0, (batch_size, dims)) + .astype('float32') + .tolist() + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTestSample(unittest.TestCase): - def setUp(self): self.init_param() @@ -382,7 +382,6 @@ class UniformTestSample(unittest.TestCase): class UniformTestSample2(UniformTestSample): - def init_param(self): self.low = -5.0 self.high = 2.0 diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_variable.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_variable.py index 982a47f5e18..ead60fa9740 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_variable.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_variable.py @@ -25,47 +25,52 @@ paddle.seed(2022) @param.param_cls( (param.TEST_CASE_NAME, 'is_discrete', 'event_rank', 'constraint'), - [('NotImplement', False, 0, constraint.Constraint())]) + [('NotImplement', False, 0, constraint.Constraint())], +) class TestVariable(unittest.TestCase): - def setUp(self): - self._var = variable.Variable(self.is_discrete, self.event_rank, - self.constraint) + self._var = variable.Variable( + self.is_discrete, self.event_rank, self.constraint + ) - @param.param_func([(1, )]) + @param.param_func([(1,)]) def test_costraint(self, value): with self.assertRaises(NotImplementedError): self._var.constraint(value) -@param.param_cls((param.TEST_CASE_NAME, 'base', 'rank'), - [('real_base', variable.real, 10)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'base', 'rank'), [('real_base', variable.real, 10)] +) class TestIndependent(unittest.TestCase): - def setUp(self): self._var = variable.Independent(self.base, self.rank) - @param.param_func([ - (paddle.rand([2, 3, 4]), ValueError), - ]) + @param.param_func( + [ + (paddle.rand([2, 3, 4]), ValueError), + ] + ) def test_costraint(self, value, expect): with self.assertRaises(expect): self._var.constraint(value) -@param.param_cls((param.TEST_CASE_NAME, 'vars', 'axis'), - [('real_base', [variable.real], 10)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'vars', 'axis'), [('real_base', [variable.real], 10)] +) class TestStack(unittest.TestCase): - def setUp(self): self._var = variable.Stack(self.vars, self.axis) def test_is_discrete(self): self.assertEqual(self._var.is_discrete, False) - @param.param_func([ - (paddle.rand([2, 3, 4]), ValueError), - ]) + @param.param_func( + [ + (paddle.rand([2, 3, 4]), ValueError), + ] + ) def test_costraint(self, value, expect): with self.assertRaises(expect): self._var.constraint(value) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_kl.py b/python/paddle/fluid/tests/unittests/distribution/test_kl.py index 582a9f5942f..5ce6d025422 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_kl.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_kl.py @@ -30,19 +30,26 @@ paddle.set_default_dtype('float64') @param.place(config.DEVICES) -@param.parameterize_cls((param.TEST_CASE_NAME, 'a1', 'b1', 'a2', 'b2'), [ - ('test_regular_input', 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random((4, 5)) + 1e-4), -]) +@param.parameterize_cls( + (param.TEST_CASE_NAME, 'a1', 'b1', 'a2', 'b2'), + [ + ( + 'test_regular_input', + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + ), + ], +) class TestKLBetaBeta(unittest.TestCase): - def setUp(self): - self.p = paddle.distribution.Beta(paddle.to_tensor(self.a1), - paddle.to_tensor(self.b1)) - self.q = paddle.distribution.Beta(paddle.to_tensor(self.a2), - paddle.to_tensor(self.b2)) + self.p = paddle.distribution.Beta( + paddle.to_tensor(self.a1), paddle.to_tensor(self.b1) + ) + self.q = paddle.distribution.Beta( + paddle.to_tensor(self.a2), paddle.to_tensor(self.b2) + ) def test_kl_divergence(self): with paddle.fluid.dygraph.guard(self.place): @@ -50,22 +57,31 @@ class TestKLBetaBeta(unittest.TestCase): paddle.distribution.kl_divergence(self.p, self.q), self.scipy_kl_beta_beta(self.a1, self.b1, self.a2, self.b2), rtol=config.RTOL.get(str(self.a1.dtype)), - atol=config.ATOL.get(str(self.a1.dtype))) + atol=config.ATOL.get(str(self.a1.dtype)), + ) def scipy_kl_beta_beta(self, a1, b1, a2, b2): - return (scipy.special.betaln(a2, b2) - scipy.special.betaln(a1, b1) + - (a1 - a2) * scipy.special.digamma(a1) + - (b1 - b2) * scipy.special.digamma(b1) + - (a2 - a1 + b2 - b1) * scipy.special.digamma(a1 + b1)) + return ( + scipy.special.betaln(a2, b2) + - scipy.special.betaln(a1, b1) + + (a1 - a2) * scipy.special.digamma(a1) + + (b1 - b2) * scipy.special.digamma(b1) + + (a2 - a1 + b2 - b1) * scipy.special.digamma(a1 + b1) + ) @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'conc1', 'conc2'), [ - ('test-regular-input', np.random.random( - (5, 7, 8, 10)), np.random.random((5, 7, 8, 10))), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'conc1', 'conc2'), + [ + ( + 'test-regular-input', + np.random.random((5, 7, 8, 10)), + np.random.random((5, 7, 8, 10)), + ), + ], +) class TestKLDirichletDirichlet(unittest.TestCase): - def setUp(self): self.p = paddle.distribution.Dirichlet(paddle.to_tensor(self.conc1)) self.q = paddle.distribution.Dirichlet(paddle.to_tensor(self.conc2)) @@ -76,17 +92,25 @@ class TestKLDirichletDirichlet(unittest.TestCase): paddle.distribution.kl_divergence(self.p, self.q), self.scipy_kl_diric_diric(self.conc1, self.conc2), rtol=config.RTOL.get(str(self.conc1.dtype)), - atol=config.ATOL.get(str(self.conc1.dtype))) + atol=config.ATOL.get(str(self.conc1.dtype)), + ) def scipy_kl_diric_diric(self, conc1, conc2): return ( - scipy.special.gammaln(np.sum(conc1, -1)) - - scipy.special.gammaln(np.sum(conc2, -1)) - np.sum( - scipy.special.gammaln(conc1) - scipy.special.gammaln(conc2), -1) + scipy.special.gammaln(np.sum(conc1, -1)) + - scipy.special.gammaln(np.sum(conc2, -1)) + - np.sum( + scipy.special.gammaln(conc1) - scipy.special.gammaln(conc2), -1 + ) + np.sum( - (conc1 - conc2) * - (scipy.special.digamma(conc1) - - scipy.special.digamma(np.sum(conc1, -1, keepdims=True))), -1)) + (conc1 - conc2) + * ( + scipy.special.digamma(conc1) + - scipy.special.digamma(np.sum(conc1, -1, keepdims=True)) + ), + -1, + ) + ) class DummyDistribution(paddle.distribution.Distribution): @@ -94,10 +118,11 @@ class DummyDistribution(paddle.distribution.Distribution): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'p', 'q'), - [('test-unregister', DummyDistribution(), DummyDistribution)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'p', 'q'), + [('test-unregister', DummyDistribution(), DummyDistribution)], +) class TestDispatch(unittest.TestCase): - def test_dispatch_with_unregister(self): with self.assertRaises(NotImplementedError): paddle.distribution.kl_divergence(self.p, self.q) @@ -106,18 +131,27 @@ class TestDispatch(unittest.TestCase): @param.place(config.DEVICES) @param.param_cls( (param.TEST_CASE_NAME, 'p', 'q'), - [('test-diff-dist', mock.Exponential(paddle.rand((100, 200, 100)) + 1.0), - mock.Exponential(paddle.rand((100, 200, 100)) + 2.0)), - ('test-same-dist', mock.Exponential( - paddle.to_tensor(1.0)), mock.Exponential(paddle.to_tensor(1.0)))]) + [ + ( + 'test-diff-dist', + mock.Exponential(paddle.rand((100, 200, 100)) + 1.0), + mock.Exponential(paddle.rand((100, 200, 100)) + 2.0), + ), + ( + 'test-same-dist', + mock.Exponential(paddle.to_tensor(1.0)), + mock.Exponential(paddle.to_tensor(1.0)), + ), + ], +) class TestKLExpfamilyExpFamily(unittest.TestCase): - def test_kl_expfamily_expfamily(self): - np.testing.assert_allclose(paddle.distribution.kl_divergence( - self.p, self.q), - kl._kl_expfamily_expfamily(self.p, self.q), - rtol=config.RTOL.get(config.DEFAULT_DTYPE), - atol=config.ATOL.get(config.DEFAULT_DTYPE)) + np.testing.assert_allclose( + paddle.distribution.kl_divergence(self.p, self.q), + kl._kl_expfamily_expfamily(self.p, self.q), + rtol=config.RTOL.get(config.DEFAULT_DTYPE), + atol=config.ATOL.get(config.DEFAULT_DTYPE), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_kl_static.py b/python/paddle/fluid/tests/unittests/distribution/test_kl_static.py index f5e354e85de..ea1f8a4aab2 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_kl_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_kl_static.py @@ -30,14 +30,19 @@ paddle.enable_static() @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'a1', 'b1', 'a2', 'b2'), [ - ('test_regular_input', 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random((4, 5)) + 1e-4), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'a1', 'b1', 'a2', 'b2'), + [ + ( + 'test_regular_input', + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + ), + ], +) class TestKLBetaBeta(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() @@ -55,46 +60,57 @@ class TestKLBetaBeta(unittest.TestCase): 'a1': self.a1, 'b1': self.b1, 'a2': self.a2, - 'b2': self.b2 + 'b2': self.b2, } def test_kl_divergence(self): with paddle.static.program_guard(self.mp, self.sp): out = paddle.distribution.kl_divergence(self.p, self.q) self.executor.run(self.sp) - [out] = self.executor.run(self.mp, - feed=self.feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.mp, feed=self.feeds, fetch_list=[out] + ) - np.testing.assert_allclose(out, - self.scipy_kl_beta_beta( - self.a1, self.b1, self.a2, self.b2), - rtol=config.RTOL.get(str(self.a1.dtype)), - atol=config.ATOL.get(str(self.a1.dtype))) + np.testing.assert_allclose( + out, + self.scipy_kl_beta_beta(self.a1, self.b1, self.a2, self.b2), + rtol=config.RTOL.get(str(self.a1.dtype)), + atol=config.ATOL.get(str(self.a1.dtype)), + ) def scipy_kl_beta_beta(self, a1, b1, a2, b2): - return (scipy.special.betaln(a2, b2) - scipy.special.betaln(a1, b1) + - (a1 - a2) * scipy.special.digamma(a1) + - (b1 - b2) * scipy.special.digamma(b1) + - (a2 - a1 + b2 - b1) * scipy.special.digamma(a1 + b1)) + return ( + scipy.special.betaln(a2, b2) + - scipy.special.betaln(a1, b1) + + (a1 - a2) * scipy.special.digamma(a1) + + (b1 - b2) * scipy.special.digamma(b1) + + (a2 - a1 + b2 - b1) * scipy.special.digamma(a1 + b1) + ) @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'conc1', 'conc2'), [ - ('test-regular-input', np.random.random( - (5, 7, 8, 10)), np.random.random((5, 7, 8, 10))), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'conc1', 'conc2'), + [ + ( + 'test-regular-input', + np.random.random((5, 7, 8, 10)), + np.random.random((5, 7, 8, 10)), + ), + ], +) class TestKLDirichletDirichlet(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() self.executor = paddle.static.Executor(self.place) with paddle.static.program_guard(self.mp, self.sp): - conc1 = paddle.static.data('conc1', self.conc1.shape, - self.conc1.dtype) - conc2 = paddle.static.data('conc2', self.conc2.shape, - self.conc2.dtype) + conc1 = paddle.static.data( + 'conc1', self.conc1.shape, self.conc1.dtype + ) + conc2 = paddle.static.data( + 'conc2', self.conc2.shape, self.conc2.dtype + ) self.p = paddle.distribution.Dirichlet(conc1) self.q = paddle.distribution.Dirichlet(conc2) self.feeds = {'conc1': self.conc1, 'conc2': self.conc2} @@ -104,24 +120,32 @@ class TestKLDirichletDirichlet(unittest.TestCase): with paddle.static.program_guard(self.mp, self.sp): out = paddle.distribution.kl_divergence(self.p, self.q) self.executor.run(self.sp) - [out] = self.executor.run(self.mp, - feed=self.feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.mp, feed=self.feeds, fetch_list=[out] + ) np.testing.assert_allclose( out, self.scipy_kl_diric_diric(self.conc1, self.conc2), rtol=config.RTOL.get(str(self.conc1.dtype)), - atol=config.ATOL.get(str(self.conc1.dtype))) + atol=config.ATOL.get(str(self.conc1.dtype)), + ) def scipy_kl_diric_diric(self, conc1, conc2): return ( - scipy.special.gammaln(np.sum(conc1, -1)) - - scipy.special.gammaln(np.sum(conc2, -1)) - np.sum( - scipy.special.gammaln(conc1) - scipy.special.gammaln(conc2), -1) + scipy.special.gammaln(np.sum(conc1, -1)) + - scipy.special.gammaln(np.sum(conc2, -1)) + - np.sum( + scipy.special.gammaln(conc1) - scipy.special.gammaln(conc2), -1 + ) + np.sum( - (conc1 - conc2) * - (scipy.special.digamma(conc1) - - scipy.special.digamma(np.sum(conc1, -1, keepdims=True))), -1)) + (conc1 - conc2) + * ( + scipy.special.digamma(conc1) + - scipy.special.digamma(np.sum(conc1, -1, keepdims=True)) + ), + -1, + ) + ) class DummyDistribution(paddle.distribution.Distribution): @@ -129,10 +153,10 @@ class DummyDistribution(paddle.distribution.Distribution): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'p', 'q'), - [('test-dispatch-exception')]) +@param.param_cls( + (param.TEST_CASE_NAME, 'p', 'q'), [('test-dispatch-exception')] +) class TestDispatch(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() @@ -150,23 +174,29 @@ class TestDispatch(unittest.TestCase): @param.place(config.DEVICES) -@param.param_cls((config.TEST_CASE_NAME, 'rate1', 'rate2'), - [('test-diff-dist', np.random.rand(100, 200, 100) + 1.0, - np.random.rand(100, 200, 100) + 2.0), - ('test-same-dist', np.array([1.0]), np.array([1.0]))]) +@param.param_cls( + (config.TEST_CASE_NAME, 'rate1', 'rate2'), + [ + ( + 'test-diff-dist', + np.random.rand(100, 200, 100) + 1.0, + np.random.rand(100, 200, 100) + 2.0, + ), + ('test-same-dist', np.array([1.0]), np.array([1.0])), + ], +) class TestKLExpfamilyExpFamily(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() self.executor = paddle.static.Executor(self.place) with paddle.static.program_guard(self.mp, self.sp): - rate1 = paddle.static.data('rate1', - shape=self.rate1.shape, - dtype=self.rate1.dtype) - rate2 = paddle.static.data('rate2', - shape=self.rate2.shape, - dtype=self.rate2.dtype) + rate1 = paddle.static.data( + 'rate1', shape=self.rate1.shape, dtype=self.rate1.dtype + ) + rate2 = paddle.static.data( + 'rate2', shape=self.rate2.shape, dtype=self.rate2.dtype + ) self.p = mock.Exponential(rate1) self.q = mock.Exponential(rate2) self.feeds = {'rate1': self.rate1, 'rate2': self.rate2} @@ -176,15 +206,16 @@ class TestKLExpfamilyExpFamily(unittest.TestCase): out1 = paddle.distribution.kl_divergence(self.p, self.q) out2 = kl._kl_expfamily_expfamily(self.p, self.q) self.executor.run(self.sp) - [out1, out2] = self.executor.run(self.mp, - feed=self.feeds, - fetch_list=[out1, out2]) + [out1, out2] = self.executor.run( + self.mp, feed=self.feeds, fetch_list=[out1, out2] + ) np.testing.assert_allclose( out1, out2, rtol=config.RTOL.get(config.DEFAULT_DTYPE), - atol=config.ATOL.get(config.DEFAULT_DTYPE)) + atol=config.ATOL.get(config.DEFAULT_DTYPE), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_fleet_api.py b/python/paddle/fluid/tests/unittests/dygraph_fleet_api.py index bf6139ffae6..a2834516d71 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_fleet_api.py +++ b/python/paddle/fluid/tests/unittests/dygraph_fleet_api.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestDygraphFleetAPI(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -35,12 +34,14 @@ class TestDygraphFleetAPI(unittest.TestCase): def test_dygraph_fleet_api(self): import paddle.distributed.fleet as fleet import paddle.distributed as dist + strategy = fleet.DistributedStrategy() strategy.amp = True strategy.recompute = True fleet.init(is_collective=True, strategy=strategy) - net = paddle.nn.Sequential(paddle.nn.Linear(10, 1), - paddle.nn.Linear(1, 2)) + net = paddle.nn.Sequential( + paddle.nn.Linear(10, 1), paddle.nn.Linear(1, 2) + ) net = dist.fleet.distributed_model(net) data = np.random.uniform(-1, 1, [30, 10]).astype('float32') data = paddle.to_tensor(data) diff --git a/python/paddle/fluid/tests/unittests/dygraph_recompute_hybrid.py b/python/paddle/fluid/tests/unittests/dygraph_recompute_hybrid.py index 105621498fe..7dbd6b6d7ef 100755 --- a/python/paddle/fluid/tests/unittests/dygraph_recompute_hybrid.py +++ b/python/paddle/fluid/tests/unittests/dygraph_recompute_hybrid.py @@ -24,34 +24,40 @@ from paddle.distributed import fleet def get_fc_block(block_idx, input_size, is_last=False): block_name = "block_" + str(block_idx) block = paddle.nn.Sequential( - (block_name + "_fc_0", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_0", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_dropout", paddle.nn.Dropout(p=0.5)), (block_name + "_relu_1", paddle.nn.ReLU()), - (block_name + "_fc_1", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_1", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_relu_2", paddle.nn.ReLU()), ) if is_last: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, 1, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, 1, bias_attr=False), + ) # add sublayer else: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, - input_size, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ) # add sublayer return block class Naive_fc_net(paddle.nn.Layer): - - def __init__(self, - input_size=10, - recompute_blocks=[1, 3], - offload=False, - partition=False, - recompute_kwargs={}): + def __init__( + self, + input_size=10, + recompute_blocks=[1, 3], + offload=False, + partition=False, + recompute_kwargs={}, + ): super(Naive_fc_net, self).__init__() self.recompute_blocks = recompute_blocks self.recompute_kwargs = recompute_kwargs @@ -65,8 +71,11 @@ class Naive_fc_net(paddle.nn.Layer): self.runfunc4 = get_fc_block(4, input_size, is_last=True) self.layers = [ - self.runfunc0, self.runfunc1, self.runfunc2, self.runfunc3, - self.runfunc4 + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, ] def forward(self, inputs): @@ -76,34 +85,43 @@ class Naive_fc_net(paddle.nn.Layer): { "mp_group": fleet.fleet._hcg.get_model_parallel_group(), "offload": self.offload, - "partition": self.partition - }, self.layers[i], inputs, **self.recompute_kwargs) + "partition": self.partition, + }, + self.layers[i], + inputs, + **self.recompute_kwargs + ) else: inputs = self.layers[i](inputs) return inputs -def run_model(recompute_block=[], - recompute_kwargs={}, - offload=False, - partition=False, - enable_autocast=False, - pure_fp16=False): +def run_model( + recompute_block=[], + recompute_kwargs={}, + offload=False, + partition=False, + enable_autocast=False, + pure_fp16=False, +): gen = paddle.seed(10) gen.manual_seed(10) np.random.seed(10) random.seed(10) batch_size, input_size = 1, 10 - model = Naive_fc_net(input_size, - recompute_blocks=recompute_block, - offload=offload, - partition=partition, - recompute_kwargs=recompute_kwargs) + model = Naive_fc_net( + input_size, + recompute_blocks=recompute_block, + offload=offload, + partition=partition, + recompute_kwargs=recompute_kwargs, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) @@ -140,7 +158,6 @@ def run_model(recompute_block=[], class TestPyLayer(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 @@ -154,7 +171,6 @@ class TestPyLayer(unittest.TestCase): fleet.init(is_collective=True, strategy=strategy) def test_base_case(self, enable_autocast=False, pure_fp16=False): - def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): self.assertEqual(loss_ref, loss) self.assertEqual(param_ref, param) @@ -164,34 +180,43 @@ class TestPyLayer(unittest.TestCase): loss_ref, param_ref, grad_ref = run_model( recompute_block=[], enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # with recompute, offload=False, partition=False - loss, param, grad = run_model(recompute_block=[1, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with recompute, offload=True, partition=False - loss, param, grad = run_model(recompute_block=[1, 2, 3], - offload=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 2, 3], + offload=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with recompute, offload=False, partition=True - loss, param, grad = run_model(recompute_block=[1], - partition=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1], + partition=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with recompute, offload=True, partition=True - loss, param, grad = run_model(recompute_block=[1, 3, 4], - offload=True, - partition=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3, 4], + offload=True, + partition=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) def test_fc_net_with_dropout(self): @@ -207,8 +232,9 @@ class TestPyLayer(unittest.TestCase): paddle.set_device("gpu") kwargs = {"is_test": False} with self.assertRaises(TypeError): - loss_ref, param_ref, grad_ref = run_model(recompute_block=[2], - recompute_kwargs=kwargs) + loss_ref, param_ref, grad_ref = run_model( + recompute_block=[2], recompute_kwargs=kwargs + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_dygraph_model.py index 60c2a4ef067..d9ccc587f7c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_dygraph_model.py @@ -21,89 +21,105 @@ from transformer_dygraph_model import MultiHeadAttention, PrePostProcessLayer class PositionwiseFeedForwardLayer(Layer): - - def __init__(self, - hidden_act, - d_inner_hid, - d_model, - dropout_rate, - param_initializer=None, - name=""): + def __init__( + self, + hidden_act, + d_inner_hid, + d_model, + dropout_rate, + param_initializer=None, + name="", + ): super(PositionwiseFeedForwardLayer, self).__init__() - self._i2h = Linear(input_dim=d_model, - output_dim=d_inner_hid, - param_attr=fluid.ParamAttr( - name=name + '_fc_0.w_0', - initializer=param_initializer), - bias_attr=name + '_fc_0.b_0', - act=hidden_act) - - self._h2o = Linear(input_dim=d_inner_hid, - output_dim=d_model, - param_attr=fluid.ParamAttr( - name=name + '_fc_1.w_0', - initializer=param_initializer), - bias_attr=name + '_fc_1.b_0') + self._i2h = Linear( + input_dim=d_model, + output_dim=d_inner_hid, + param_attr=fluid.ParamAttr( + name=name + '_fc_0.w_0', initializer=param_initializer + ), + bias_attr=name + '_fc_0.b_0', + act=hidden_act, + ) + + self._h2o = Linear( + input_dim=d_inner_hid, + output_dim=d_model, + param_attr=fluid.ParamAttr( + name=name + '_fc_1.w_0', initializer=param_initializer + ), + bias_attr=name + '_fc_1.b_0', + ) self._dropout_rate = dropout_rate def forward(self, x): hidden = self._i2h(x) if self._dropout_rate: - hidden = fluid.layers.dropout(hidden, - dropout_prob=self._dropout_rate, - is_test=False) + hidden = fluid.layers.dropout( + hidden, dropout_prob=self._dropout_rate, is_test=False + ) out = self._h2o(hidden) return out class EncoderSubLayer(Layer): - - def __init__(self, - hidden_act, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da", - param_initializer=None, - name=""): + def __init__( + self, + hidden_act, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + param_initializer=None, + name="", + ): super(EncoderSubLayer, self).__init__() self.name = name self._preprocess_cmd = preprocess_cmd self._postprocess_cmd = postprocess_cmd self._prepostprocess_dropout = prepostprocess_dropout - self._preprocess_layer = PrePostProcessLayer(self._preprocess_cmd, - d_model, - prepostprocess_dropout) + self._preprocess_layer = PrePostProcessLayer( + self._preprocess_cmd, d_model, prepostprocess_dropout + ) self._multihead_attention_layer = MultiHeadAttention( - d_key, d_value, d_model, n_head, attention_dropout, - param_initializer) + d_key, + d_value, + d_model, + n_head, + attention_dropout, + param_initializer, + ) self._postprocess_layer = PrePostProcessLayer( - self._postprocess_cmd, d_model, self._prepostprocess_dropout) + self._postprocess_cmd, d_model, self._prepostprocess_dropout + ) self._preprocess_layer2 = PrePostProcessLayer( - self._preprocess_cmd, d_model, self._prepostprocess_dropout) + self._preprocess_cmd, d_model, self._prepostprocess_dropout + ) self._positionwise_feed_forward = PositionwiseFeedForwardLayer( hidden_act, d_inner_hid, d_model, relu_dropout, param_initializer, - name=name + "_ffn") + name=name + "_ffn", + ) self._postprocess_layer2 = PrePostProcessLayer( - self._postprocess_cmd, d_model, self._prepostprocess_dropout) + self._postprocess_cmd, d_model, self._prepostprocess_dropout + ) def forward(self, enc_input, attn_bias): pre_process_multihead = self._preprocess_layer(enc_input) - attn_output = self._multihead_attention_layer(pre_process_multihead, - None, None, attn_bias) + attn_output = self._multihead_attention_layer( + pre_process_multihead, None, None, attn_bias + ) attn_output = self._postprocess_layer(attn_output, enc_input) pre_process2_output = self._preprocess_layer2(attn_output) ffd_output = self._positionwise_feed_forward(pre_process2_output) @@ -111,22 +127,23 @@ class EncoderSubLayer(Layer): class EncoderLayer(Layer): - - def __init__(self, - hidden_act, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da", - param_initializer=None, - name=""): + def __init__( + self, + hidden_act, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + param_initializer=None, + name="", + ): super(EncoderLayer, self).__init__() self._preprocess_cmd = preprocess_cmd @@ -135,25 +152,30 @@ class EncoderLayer(Layer): self._n_layer = n_layer self._hidden_act = hidden_act self._preprocess_layer = PrePostProcessLayer( - self._preprocess_cmd, 3, self._prepostprocess_dropout) + self._preprocess_cmd, 3, self._prepostprocess_dropout + ) for i in range(n_layer): self._encoder_sublayers.append( self.add_sublayer( 'esl_%d' % i, - EncoderSubLayer(hidden_act, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - param_initializer, - name=name + '_layer_' + str(i)))) + EncoderSubLayer( + hidden_act, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + param_initializer, + name=name + '_layer_' + str(i), + ), + ) + ) def forward(self, enc_input, attn_bias): for i in range(self._n_layer): @@ -164,7 +186,6 @@ class EncoderLayer(Layer): class BertModelLayer(Layer): - def __init__(self, config, return_pooled_out=True, use_fp16=False): super(BertModelLayer, self).__init__() @@ -185,36 +206,46 @@ class BertModelLayer(Layer): self._dtype = "float16" if use_fp16 else "float32" self._param_initializer = fluid.initializer.TruncatedNormal( - scale=config['initializer_range']) + scale=config['initializer_range'] + ) - self._src_emb = Embedding(size=[self._voc_size, self._emb_size], - param_attr=fluid.ParamAttr( - name=self._word_emb_name, - initializer=self._param_initializer), - dtype=self._dtype) + self._src_emb = Embedding( + size=[self._voc_size, self._emb_size], + param_attr=fluid.ParamAttr( + name=self._word_emb_name, initializer=self._param_initializer + ), + dtype=self._dtype, + ) self._pos_emb = Embedding( size=[self._max_position_seq_len, self._emb_size], - param_attr=fluid.ParamAttr(name=self._pos_emb_name, - initializer=self._param_initializer), - dtype=self._dtype) - - self._sent_emb = Embedding(size=[self._sent_types, self._emb_size], - param_attr=fluid.ParamAttr( - name=self._sent_emb_name, - initializer=self._param_initializer), - dtype=self._dtype) - - self.pooled_fc = Linear(input_dim=self._emb_size, - output_dim=self._emb_size, - param_attr=fluid.ParamAttr( - name="pooled_fc.w_0", - initializer=self._param_initializer), - bias_attr="pooled_fc.b_0", - act="tanh") + param_attr=fluid.ParamAttr( + name=self._pos_emb_name, initializer=self._param_initializer + ), + dtype=self._dtype, + ) + + self._sent_emb = Embedding( + size=[self._sent_types, self._emb_size], + param_attr=fluid.ParamAttr( + name=self._sent_emb_name, initializer=self._param_initializer + ), + dtype=self._dtype, + ) + + self.pooled_fc = Linear( + input_dim=self._emb_size, + output_dim=self._emb_size, + param_attr=fluid.ParamAttr( + name="pooled_fc.w_0", initializer=self._param_initializer + ), + bias_attr="pooled_fc.b_0", + act="tanh", + ) self.pre_process_layer = PrePostProcessLayer( - "nd", self._emb_size, self._prepostprocess_dropout) + "nd", self._emb_size, self._prepostprocess_dropout + ) self._encoder = EncoderLayer( hidden_act=self._hidden_act, @@ -229,7 +260,8 @@ class BertModelLayer(Layer): relu_dropout=0, preprocess_cmd="", postprocess_cmd="dan", - param_initializer=self._param_initializer) + param_initializer=self._param_initializer, + ) def forward(self, src_ids, position_ids, sentence_ids, input_mask): src_emb = self._src_emb(src_ids) @@ -241,16 +273,15 @@ class BertModelLayer(Layer): emb_out = self.pre_process_layer(emb_out) - self_attn_mask = fluid.layers.matmul(x=input_mask, - y=input_mask, - transpose_y=True) - self_attn_mask = fluid.layers.scale(x=self_attn_mask, - scale=10000.0, - bias=-1.0, - bias_after_scale=False) - n_head_self_attn_mask = fluid.layers.stack(x=[self_attn_mask] * - self._n_head, - axis=1) + self_attn_mask = fluid.layers.matmul( + x=input_mask, y=input_mask, transpose_y=True + ) + self_attn_mask = fluid.layers.scale( + x=self_attn_mask, scale=10000.0, bias=-1.0, bias_after_scale=False + ) + n_head_self_attn_mask = fluid.layers.stack( + x=[self_attn_mask] * self._n_head, axis=1 + ) n_head_self_attn_mask.stop_gradient = True enc_output = self._encoder(emb_out, n_head_self_attn_mask) @@ -258,26 +289,27 @@ class BertModelLayer(Layer): # TODO(zhhsplendid): uncomment this in next PR which we support various # length of early return # - #if not self.return_pooled_out: + # if not self.return_pooled_out: # return enc_output - next_sent_feat = fluid.layers.slice(input=enc_output, - axes=[1], - starts=[0], - ends=[1]) + next_sent_feat = fluid.layers.slice( + input=enc_output, axes=[1], starts=[0], ends=[1] + ) next_sent_feat = self.pooled_fc(next_sent_feat) - next_sent_feat = fluid.layers.reshape(next_sent_feat, - shape=[-1, self._emb_size]) + next_sent_feat = fluid.layers.reshape( + next_sent_feat, shape=[-1, self._emb_size] + ) return enc_output, next_sent_feat class PretrainModelLayer(Layer): - - def __init__(self, - config, - return_pooled_out=True, - weight_sharing=False, - use_fp16=False): + def __init__( + self, + config, + return_pooled_out=True, + weight_sharing=False, + use_fp16=False, + ): super(PretrainModelLayer, self).__init__() self.config = config self._voc_size = config['vocab_size'] @@ -287,84 +319,113 @@ class PretrainModelLayer(Layer): self._word_emb_name = "word_embedding" self._param_initializer = fluid.initializer.TruncatedNormal( - scale=config['initializer_range']) + scale=config['initializer_range'] + ) self._weight_sharing = weight_sharing self.use_fp16 = use_fp16 self._dtype = "float16" if use_fp16 else "float32" - self.bert_layer = BertModelLayer(config=self.config, - return_pooled_out=True, - use_fp16=self.use_fp16) + self.bert_layer = BertModelLayer( + config=self.config, return_pooled_out=True, use_fp16=self.use_fp16 + ) self.pre_process_layer = PrePostProcessLayer( - "n", self._emb_size, self._prepostprocess_dropout) - - self.pooled_fc = Linear(input_dim=self._emb_size, - output_dim=self._emb_size, - param_attr=fluid.ParamAttr( - name="mask_lm_trans_fc.w_0", - initializer=self._param_initializer), - bias_attr="mask_lm_trans_fc.b_0", - act="tanh") + "n", self._emb_size, self._prepostprocess_dropout + ) + + self.pooled_fc = Linear( + input_dim=self._emb_size, + output_dim=self._emb_size, + param_attr=fluid.ParamAttr( + name="mask_lm_trans_fc.w_0", initializer=self._param_initializer + ), + bias_attr="mask_lm_trans_fc.b_0", + act="tanh", + ) self.mask_lm_out_bias_attr = fluid.ParamAttr( name="mask_lm_out_fc.b_0", - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0), + ) if not self._weight_sharing: - self.out_fc = Linear(input_dim=self._emb_size, - output_dim=self._voc_size, - param_attr=fluid.ParamAttr( - name="mask_lm_out_fc.w_0", - initializer=self._param_initializer), - bias_attr=self.mask_lm_out_bias_attr) + self.out_fc = Linear( + input_dim=self._emb_size, + output_dim=self._voc_size, + param_attr=fluid.ParamAttr( + name="mask_lm_out_fc.w_0", + initializer=self._param_initializer, + ), + bias_attr=self.mask_lm_out_bias_attr, + ) else: self.fc_create_params = self.create_parameter( shape=[self._voc_size], dtype=self._dtype, attr=self.mask_lm_out_bias_attr, - is_bias=True) - - self.next_sent_fc = Linear(input_dim=self._emb_size, - output_dim=2, - param_attr=fluid.ParamAttr( - name="next_sent_fc.w_0", - initializer=self._param_initializer), - bias_attr="next_sent_fc.b_0") + is_bias=True, + ) + + self.next_sent_fc = Linear( + input_dim=self._emb_size, + output_dim=2, + param_attr=fluid.ParamAttr( + name="next_sent_fc.w_0", initializer=self._param_initializer + ), + bias_attr="next_sent_fc.b_0", + ) @declarative - def forward(self, src_ids, position_ids, sentence_ids, input_mask, - mask_label, mask_pos, labels): + def forward( + self, + src_ids, + position_ids, + sentence_ids, + input_mask, + mask_label, + mask_pos, + labels, + ): mask_pos = fluid.layers.cast(x=mask_pos, dtype='int32') - enc_output, next_sent_feat = self.bert_layer(src_ids, position_ids, - sentence_ids, input_mask) - reshaped_emb_out = fluid.layers.reshape(x=enc_output, - shape=[-1, self._emb_size]) + enc_output, next_sent_feat = self.bert_layer( + src_ids, position_ids, sentence_ids, input_mask + ) + reshaped_emb_out = fluid.layers.reshape( + x=enc_output, shape=[-1, self._emb_size] + ) mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos) mask_trans_feat = self.pooled_fc(mask_feat) mask_trans_feat = self.pre_process_layer(mask_trans_feat) if self._weight_sharing: - fc_out = fluid.layers.matmul(x=mask_trans_feat, - y=self.bert_layer._src_emb._w, - transpose_y=True) + fc_out = fluid.layers.matmul( + x=mask_trans_feat, + y=self.bert_layer._src_emb._w, + transpose_y=True, + ) fc_out += self.fc_create_params else: fc_out = self.out_fc(mask_trans_feat) - mask_lm_loss = fluid.layers.softmax_with_cross_entropy(logits=fc_out, - label=mask_label) + mask_lm_loss = fluid.layers.softmax_with_cross_entropy( + logits=fc_out, label=mask_label + ) mean_mask_lm_loss = paddle.mean(mask_lm_loss) next_sent_fc_out = self.next_sent_fc(next_sent_feat) - next_sent_loss, next_sent_softmax = fluid.layers.softmax_with_cross_entropy( - logits=next_sent_fc_out, label=labels, return_softmax=True) + ( + next_sent_loss, + next_sent_softmax, + ) = fluid.layers.softmax_with_cross_entropy( + logits=next_sent_fc_out, label=labels, return_softmax=True + ) - next_sent_acc = fluid.layers.accuracy(input=next_sent_softmax, - label=labels) + next_sent_acc = fluid.layers.accuracy( + input=next_sent_softmax, label=labels + ) mean_next_sent_loss = paddle.mean(next_sent_loss) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_utils.py index 3fb23049f1b..0cc22162d20 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_utils.py @@ -35,7 +35,7 @@ def get_bert_config(): "pooler_size_per_head": 8, "pooler_type": "first_token_transform", "type_vocab_size": 2, - "vocab_size": 21128 + "vocab_size": 21128, } return bert_config @@ -87,8 +87,9 @@ def mask(batch_tokens, total_token_num, vocab_size, CLS=1, SEP=2, MASK=3): # ensure at least mask one word in a sentence while not mask_flag: - token_index = int(self_random.randint(1, high=len(sent) - 1, - size=1)) + token_index = int( + self_random.randint(1, high=len(sent) - 1, size=1) + ) if sent[token_index] != SEP and sent[token_index] != CLS: mask_label.append(sent[token_index]) sent[token_index] = MASK @@ -99,12 +100,14 @@ def mask(batch_tokens, total_token_num, vocab_size, CLS=1, SEP=2, MASK=3): return batch_tokens, mask_label, mask_pos -def pad_batch_data(insts, - pad_idx=0, - return_pos=False, - return_input_mask=False, - return_max_len=False, - return_num_token=False): +def pad_batch_data( + insts, + pad_idx=0, + return_pos=False, + return_input_mask=False, + return_max_len=False, + return_num_token=False, +): """ Pad the instances to the max sequence length in batch, and generate the corresponding position data and input mask. @@ -114,24 +117,27 @@ def pad_batch_data(insts, # Any token included in dict can be used to pad, since the paddings' loss # will be masked out by weights and make no effect on parameter gradients. - inst_data = np.array([ - list(inst) + list([pad_idx] * (max_len - len(inst))) for inst in insts - ]) + inst_data = np.array( + [list(inst) + list([pad_idx] * (max_len - len(inst))) for inst in insts] + ) return_list += [inst_data.astype("int64").reshape([-1, max_len])] # position data if return_pos: - inst_pos = np.array([ - list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) - for inst in insts - ]) + inst_pos = np.array( + [ + list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_pos.astype("int64").reshape([-1, max_len])] if return_input_mask: # This is used to avoid attention on paddings. input_mask_data = np.array( - [[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts]) + [[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts] + ) input_mask_data = np.expand_dims(input_mask_data, axis=-1) return_list += [input_mask_data.astype("float32")] @@ -147,16 +153,18 @@ def pad_batch_data(insts, return return_list if len(return_list) > 1 else return_list[0] -def prepare_batch_data(insts, - total_token_num, - voc_size=0, - pad_id=None, - cls_id=None, - sep_id=None, - mask_id=None, - return_input_mask=True, - return_max_len=True, - return_num_token=False): +def prepare_batch_data( + insts, + total_token_num, + voc_size=0, + pad_id=None, + cls_id=None, + sep_id=None, + mask_id=None, + return_input_mask=True, + return_max_len=True, + return_num_token=False, +): """ 1. generate Tensor of data 2. generate Tensor of position @@ -175,30 +183,38 @@ def prepare_batch_data(insts, # First step: do mask without padding if mask_id >= 0: - out, mask_label, mask_pos = mask(batch_src_ids, - total_token_num, - vocab_size=voc_size, - CLS=cls_id, - SEP=sep_id, - MASK=mask_id) + out, mask_label, mask_pos = mask( + batch_src_ids, + total_token_num, + vocab_size=voc_size, + CLS=cls_id, + SEP=sep_id, + MASK=mask_id, + ) else: out = batch_src_ids # Second step: padding - src_id, self_input_mask = pad_batch_data(out, - pad_idx=pad_id, - return_input_mask=True) - pos_id = pad_batch_data(batch_pos_ids, - pad_idx=pad_id, - return_pos=False, - return_input_mask=False) - sent_id = pad_batch_data(batch_sent_ids, - pad_idx=pad_id, - return_pos=False, - return_input_mask=False) + src_id, self_input_mask = pad_batch_data( + out, pad_idx=pad_id, return_input_mask=True + ) + pos_id = pad_batch_data( + batch_pos_ids, pad_idx=pad_id, return_pos=False, return_input_mask=False + ) + sent_id = pad_batch_data( + batch_sent_ids, + pad_idx=pad_id, + return_pos=False, + return_input_mask=False, + ) if mask_id >= 0: return_list = [ - src_id, pos_id, sent_id, self_input_mask, mask_label, mask_pos + src_id, + pos_id, + sent_id, + self_input_mask, + mask_label, + mask_pos, ] + labels_list else: return_list = [src_id, pos_id, sent_id, self_input_mask] + labels_list @@ -208,16 +224,17 @@ def prepare_batch_data(insts, class DataReader(object): - - def __init__(self, - batch_size=4096, - in_tokens=True, - max_seq_len=512, - shuffle_files=False, - epoch=100, - voc_size=0, - is_test=False, - generate_neg_sample=False): + def __init__( + self, + batch_size=4096, + in_tokens=True, + max_seq_len=512, + shuffle_files=False, + epoch=100, + voc_size=0, + is_test=False, + generate_neg_sample=False, + ): self.batch_size = batch_size self.in_tokens = in_tokens @@ -237,8 +254,10 @@ class DataReader(object): self.is_test = is_test self.generate_neg_sample = generate_neg_sample if self.in_tokens: - assert self.batch_size >= self.max_seq_len, "The number of " \ - "tokens in batch should not be smaller than max seq length." + assert self.batch_size >= self.max_seq_len, ( + "The number of " + "tokens in batch should not be smaller than max seq length." + ) if self.is_test: self.epoch = 1 @@ -253,21 +272,22 @@ class DataReader(object): sent0_len = self_random.randint(50, 100) sent1_len = self_random.randint(50, 100) - token_ids = [1] \ - + [self_random.randint(0, 10000) for i in range(sent0_len-1)] \ - + [self_random.randint(0, 10000) for i in range(sent1_len-1)] \ - + [2] + token_ids = ( + [1] + + [self_random.randint(0, 10000) for i in range(sent0_len - 1)] + + [self_random.randint(0, 10000) for i in range(sent1_len - 1)] + + [2] + ) - sent_ids = [0 for i in range(sent0_len) - ] + [1 for i in range(sent1_len)] + sent_ids = [0 for i in range(sent0_len)] + [ + 1 for i in range(sent1_len) + ] pos_ids = [i for i in range(sent0_len + sent1_len)] label = 1 yield token_ids, sent_ids, pos_ids, label def data_generator(self): - def wrapper(): - def reader(): for epoch in range(self.epoch): self.current_epoch = epoch + 1 @@ -291,25 +311,30 @@ class DataReader(object): total_token_num += len(token_ids) else: yield batch, total_token_num - batch, total_token_num, max_len = [ - parsed_line - ], len(token_ids), len(token_ids) + batch, total_token_num, max_len = ( + [parsed_line], + len(token_ids), + len(token_ids), + ) if len(batch) > 0: yield batch, total_token_num for batch_data, total_token_num in batch_reader( - reader, self.batch_size, self.in_tokens): - yield prepare_batch_data(batch_data, - total_token_num, - voc_size=self.voc_size, - pad_id=self.pad_id, - cls_id=self.cls_id, - sep_id=self.sep_id, - mask_id=self.mask_id, - return_input_mask=True, - return_max_len=False, - return_num_token=False) + reader, self.batch_size, self.in_tokens + ): + yield prepare_batch_data( + batch_data, + total_token_num, + voc_size=self.voc_size, + pad_id=self.pad_id, + cls_id=self.cls_id, + sep_id=self.sep_id, + mask_id=self.mask_id, + return_input_mask=True, + return_max_len=False, + return_num_token=False, + ) return wrapper @@ -324,11 +349,13 @@ class ModelHyperParams(object): def get_feed_data_reader(bert_config): args = ModelHyperParams() - data_reader = DataReader(batch_size=args.batch_size, - in_tokens=args.in_tokens, - voc_size=bert_config['vocab_size'], - epoch=args.epoch, - max_seq_len=args.max_seq_len, - generate_neg_sample=args.generate_neg_sample) + data_reader = DataReader( + batch_size=args.batch_size, + in_tokens=args.in_tokens, + voc_size=bert_config['vocab_size'], + epoch=args.epoch, + max_seq_len=args.max_seq_len, + generate_neg_sample=args.generate_neg_sample, + ) return data_reader diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/darknet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/darknet.py index d188c693baa..ea72368fedb 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/darknet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/darknet.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import paddle.fluid as fluid from paddle.fluid.param_attr import ParamAttr @@ -20,35 +20,44 @@ from paddle.fluid.dygraph.nn import Conv2D, BatchNorm class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - ch_in, - ch_out, - filter_size=3, - stride=1, - groups=1, - padding=0, - act="leaky", - is_test=True): + def __init__( + self, + ch_in, + ch_out, + filter_size=3, + stride=1, + groups=1, + padding=0, + act="leaky", + is_test=True, + ): super(ConvBNLayer, self).__init__() - self.conv = Conv2D(num_channels=ch_in, - num_filters=ch_out, - filter_size=filter_size, - stride=stride, - padding=padding, - groups=groups, - param_attr=ParamAttr( - initializer=fluid.initializer.Normal(0., 0.02)), - bias_attr=False, - act=None) + self.conv = Conv2D( + num_channels=ch_in, + num_filters=ch_out, + filter_size=filter_size, + stride=stride, + padding=padding, + groups=groups, + param_attr=ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02) + ), + bias_attr=False, + act=None, + ) self.batch_norm = BatchNorm( num_channels=ch_out, is_test=is_test, - param_attr=ParamAttr(initializer=fluid.initializer.Normal(0., 0.02), - regularizer=L2Decay(0.)), - bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), - regularizer=L2Decay(0.))) + param_attr=ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02), + regularizer=L2Decay(0.0), + ), + bias_attr=ParamAttr( + initializer=fluid.initializer.Constant(0.0), + regularizer=L2Decay(0.0), + ), + ) self.act = act @@ -61,23 +70,20 @@ class ConvBNLayer(fluid.dygraph.Layer): class DownSample(fluid.dygraph.Layer): - - def __init__(self, - ch_in, - ch_out, - filter_size=3, - stride=2, - padding=1, - is_test=True): + def __init__( + self, ch_in, ch_out, filter_size=3, stride=2, padding=1, is_test=True + ): super(DownSample, self).__init__() - self.conv_bn_layer = ConvBNLayer(ch_in=ch_in, - ch_out=ch_out, - filter_size=filter_size, - stride=stride, - padding=padding, - is_test=is_test) + self.conv_bn_layer = ConvBNLayer( + ch_in=ch_in, + ch_out=ch_out, + filter_size=filter_size, + stride=stride, + padding=padding, + is_test=is_test, + ) self.ch_out = ch_out def forward(self, inputs): @@ -86,22 +92,25 @@ class DownSample(fluid.dygraph.Layer): class BasicBlock(fluid.dygraph.Layer): - def __init__(self, ch_in, ch_out, is_test=True): super(BasicBlock, self).__init__() - self.conv1 = ConvBNLayer(ch_in=ch_in, - ch_out=ch_out, - filter_size=1, - stride=1, - padding=0, - is_test=is_test) - self.conv2 = ConvBNLayer(ch_in=ch_out, - ch_out=ch_out * 2, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) + self.conv1 = ConvBNLayer( + ch_in=ch_in, + ch_out=ch_out, + filter_size=1, + stride=1, + padding=0, + is_test=is_test, + ) + self.conv2 = ConvBNLayer( + ch_in=ch_out, + ch_out=ch_out * 2, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) def forward(self, inputs): conv1 = self.conv1(inputs) @@ -111,7 +120,6 @@ class BasicBlock(fluid.dygraph.Layer): class LayerWarp(fluid.dygraph.Layer): - def __init__(self, ch_in, ch_out, count, is_test=True): super(LayerWarp, self).__init__() @@ -120,7 +128,8 @@ class LayerWarp(fluid.dygraph.Layer): for i in range(1, count): res_out = self.add_sublayer( "basic_block_%d" % (i), - BasicBlock(ch_out * 2, ch_out, is_test=is_test)) + BasicBlock(ch_out * 2, ch_out, is_test=is_test), + ) self.res_out_list.append(res_out) self.ch_out = ch_out @@ -135,18 +144,19 @@ DarkNet_cfg = {53: ([1, 2, 8, 8, 4])} class DarkNet53_conv_body(fluid.dygraph.Layer): - def __init__(self, ch_in=3, is_test=True): super(DarkNet53_conv_body, self).__init__() self.stages = DarkNet_cfg[53] self.stages = self.stages[0:5] - self.conv0 = ConvBNLayer(ch_in=ch_in, - ch_out=32, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) + self.conv0 = ConvBNLayer( + ch_in=ch_in, + ch_out=32, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) self.downsample0 = DownSample(ch_in=32, ch_out=32 * 2, is_test=is_test) self.darknet53_conv_block_list = [] @@ -155,14 +165,18 @@ class DarkNet53_conv_body(fluid.dygraph.Layer): for i, stage in enumerate(self.stages): conv_block = self.add_sublayer( "stage_%d" % (i), - LayerWarp(int(ch_in[i]), 32 * (2**i), stage, is_test=is_test)) + LayerWarp(int(ch_in[i]), 32 * (2**i), stage, is_test=is_test), + ) self.darknet53_conv_block_list.append(conv_block) for i in range(len(self.stages) - 1): downsample = self.add_sublayer( "stage_%d_downsample" % i, - DownSample(ch_in=32 * (2**(i + 1)), - ch_out=32 * (2**(i + 2)), - is_test=is_test)) + DownSample( + ch_in=32 * (2 ** (i + 1)), + ch_out=32 * (2 ** (i + 2)), + is_test=is_test, + ), + ) self.downsample_list.append(downsample) def forward(self, inputs): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/decos.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/decos.py index ee566fbcb33..73bbfffcabf 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/decos.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/decos.py @@ -18,7 +18,6 @@ from functools import wraps def deco1(fun): - @wraps(fun) def inner(*args, **kwargs): print('in decos.deco1, added 1') @@ -30,9 +29,7 @@ def deco1(fun): def deco2(x=0): - def inner_deco(func): - @wraps(func) def inner(*args, **kwargs): print('in decos.deco2, added {}'.format(x)) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py index d8f2a379d61..bbd0f25c05f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py @@ -135,7 +135,8 @@ def dyfunc_with_if_else_early_return2(): def dyfunc_with_if_else_with_list_geneator(x): if 10 > 5: y = paddle.add_n( - [paddle.full(shape=[2], fill_value=v) for v in range(5)]) + [paddle.full(shape=[2], fill_value=v) for v in range(5)] + ) else: y = x return y @@ -162,9 +163,9 @@ def nested_if_else(x_v): if paddle.mean(y).numpy()[0] < batch_size: y = fluid.layers.abs(y) else: - tmp = fluid.layers.fill_constant(y.shape, - dtype='float32', - value=-1) + tmp = fluid.layers.fill_constant( + y.shape, dtype='float32', value=-1 + ) y = y - tmp else: y = x_v - bias @@ -180,15 +181,15 @@ def nested_if_else_2(x): x_shape_0 = x.shape[0] if x_shape_0 < 1: if fluid.layers.shape(y).numpy()[0] < 1: - res = fluid.layers.fill_constant(value=2, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=2, shape=x.shape, dtype="int32" + ) # `z` is a new var here. z = y + 1 else: - res = fluid.layers.fill_constant(value=3, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=3, shape=x.shape, dtype="int32" + ) else: res = x return res @@ -213,33 +214,35 @@ def nested_if_else_3(x): else: y_shape = fluid.layers.shape(y) if y_shape.numpy()[0] < 1: - res = fluid.layers.fill_constant(value=2, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=2, shape=x.shape, dtype="int32" + ) # `z` is created in above code block. z = y + 1 else: - res = fluid.layers.fill_constant(value=3, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=3, shape=x.shape, dtype="int32" + ) # `out` is a new var. out = x + 1 return res class NetWithControlFlowIf(fluid.dygraph.Layer): - def __init__(self, hidden_dim=16): super(NetWithControlFlowIf, self).__init__() self.hidden_dim = hidden_dim self.fc = fluid.dygraph.Linear( input_dim=hidden_dim, output_dim=5, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) - self.alpha = 10. + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) + self.alpha = 10.0 self.constant_vars = {} @paddle.jit.to_static @@ -247,18 +250,21 @@ class NetWithControlFlowIf(fluid.dygraph.Layer): hidden_dim = input.shape[-1] if hidden_dim != self.hidden_dim: raise ValueError( - "hidden_dim {} of input is not equal to FC.weight[0]: {}". - format(hidden_dim, self.hidden_dim)) - - self.constant_vars['bias'] = fluid.layers.fill_constant([5], - dtype='float32', - value=1) + "hidden_dim {} of input is not equal to FC.weight[0]: {}".format( + hidden_dim, self.hidden_dim + ) + ) + + self.constant_vars['bias'] = fluid.layers.fill_constant( + [5], dtype='float32', value=1 + ) # Control flow `if` statement fc_out = self.fc(input) if paddle.mean(fc_out).numpy()[0] < 0: y = fc_out + self.constant_vars['bias'] self.constant_vars['w'] = fluid.layers.fill_constant( - [5], dtype='float32', value=10) + [5], dtype='float32', value=10 + ) if y.numpy()[0] < self.alpha: # Create new var, but is not used. x = 10 @@ -268,12 +274,13 @@ class NetWithControlFlowIf(fluid.dygraph.Layer): if y.numpy()[-1] < self.alpha: # Modify variable of class self.constant_vars['w'] = fluid.layers.fill_constant( - [hidden_dim], dtype='float32', value=9) + [hidden_dim], dtype='float32', value=9 + ) y = fluid.layers.abs(y) else: - tmp = fluid.layers.fill_constant(y.shape, - dtype='float32', - value=-1) + tmp = fluid.layers.fill_constant( + y.shape, dtype='float32', value=-1 + ) y = y - tmp else: y = fc_out - self.constant_vars['bias'] @@ -284,8 +291,12 @@ class NetWithControlFlowIf(fluid.dygraph.Layer): def if_with_and_or(x_v, label=None): batch_size = fluid.layers.shape(x_v) - if x_v is not None and (paddle.mean(x_v).numpy()[0] > 0 or label - is not None) and batch_size[0] > 1 and True: + if ( + x_v is not None + and (paddle.mean(x_v).numpy()[0] > 0 or label is not None) + and batch_size[0] > 1 + and True + ): x_v = x_v - 1 else: x_v = x_v + 1 @@ -317,8 +328,12 @@ def if_with_and_or_2(x, y=None): def if_with_and_or_3(x, y=None): batch_size = fluid.layers.shape(x) mean_res = paddle.mean(x) - if x is not None and batch_size[0] > 1 and y is not None and mean_res.numpy( - )[0] > 0: + if ( + x is not None + and batch_size[0] > 1 + and y is not None + and mean_res.numpy()[0] > 0 + ): x = x + 1 if mean_res.numpy()[0] > 0 and (x is not None and batch_size[0] > 1) and y: x = x - 1 @@ -328,19 +343,19 @@ def if_with_and_or_3(x, y=None): def if_with_and_or_4(x, y=None): batch_size = fluid.layers.shape(x) mean_res = paddle.mean(x) - if (x is not None and batch_size[0] > 1) or (y is not None - and mean_res.numpy()[0] > 0): + if (x is not None and batch_size[0] > 1) or ( + y is not None and mean_res.numpy()[0] > 0 + ): x = x + 1 - if (x is not None or batch_size[0] > 1) and (y is not None - or mean_res.numpy()[0] > 0): + if (x is not None or batch_size[0] > 1) and ( + y is not None or mean_res.numpy()[0] > 0 + ): x = x - 1 return x def if_with_class_var(x, y=None): - class Foo(object): - def __init__(self): self.a = 1 self.b = 2 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/predictor_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/predictor_utils.py index 2759177e978..554cf95e824 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/predictor_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/predictor_utils.py @@ -43,7 +43,8 @@ class PredictorTools(object): if os.path.exists(os.path.join(self.model_path, self.params_file)): config = AnalysisConfig( os.path.join(self.model_path, self.model_file), - os.path.join(self.model_path, self.params_file)) + os.path.join(self.model_path, self.params_file), + ) else: config = AnalysisConfig(os.path.join(self.model_path)) @@ -73,7 +74,7 @@ class PredictorTools(object): tensor_shapes = predictor.get_input_tensor_shape() names = predictor.get_input_names() for i, name in enumerate(names): - #assert name in self.feeds_var, '{} not in feeded dict'.format(name) + # assert name in self.feeds_var, '{} not in feeded dict'.format(name) shape = tensor_shapes[name] tensor = predictor.get_input_tensor(name) feed_data = self.feeds_var[i] diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_dygraph_model.py index 5af9b2ab94b..266bcf4e7b7 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_dygraph_model.py @@ -24,24 +24,26 @@ from paddle.fluid.dygraph.jit import declarative from paddle.fluid.dygraph.nn import Embedding from seq2seq_utils import Seq2SeqModelHyperParams as args -INF = 1. * 1e5 +INF = 1.0 * 1e5 alpha = 0.6 -uniform_initializer = lambda x: fluid.initializer.UniformInitializer(low=-x, - high=x) +uniform_initializer = lambda x: fluid.initializer.UniformInitializer( + low=-x, high=x +) zero_constant = fluid.initializer.Constant(0.0) class BasicLSTMUnit(Layer): - - def __init__(self, - hidden_size, - input_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - dtype='float32'): + def __init__( + self, + hidden_size, + input_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + dtype='float32', + ): super(BasicLSTMUnit, self).__init__(dtype) self._hiden_size = hidden_size @@ -56,12 +58,15 @@ class BasicLSTMUnit(Layer): self._weight = self.create_parameter( attr=self._param_attr, shape=[self._input_size + self._hiden_size, 4 * self._hiden_size], - dtype=self._dtype) + dtype=self._dtype, + ) - self._bias = self.create_parameter(attr=self._bias_attr, - shape=[4 * self._hiden_size], - dtype=self._dtype, - is_bias=True) + self._bias = self.create_parameter( + attr=self._bias_attr, + shape=[4 * self._hiden_size], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input, pre_hidden, pre_cell): concat_input_hidden = layers.concat([input, pre_hidden], 1) @@ -70,9 +75,11 @@ class BasicLSTMUnit(Layer): gate_input = layers.elementwise_add(gate_input, self._bias) i, j, f, o = layers.split(gate_input, num_or_sections=4, dim=-1) new_cell = layers.elementwise_add( - layers.elementwise_mul(pre_cell, - layers.sigmoid(f + self._forget_bias)), - layers.elementwise_mul(layers.sigmoid(i), layers.tanh(j))) + layers.elementwise_mul( + pre_cell, layers.sigmoid(f + self._forget_bias) + ), + layers.elementwise_mul(layers.sigmoid(i), layers.tanh(j)), + ) new_hidden = layers.tanh(new_cell) * layers.sigmoid(o) @@ -80,20 +87,21 @@ class BasicLSTMUnit(Layer): class BaseModel(fluid.dygraph.Layer): - - def __init__(self, - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=1, - init_scale=0.1, - dropout=None, - beam_size=1, - beam_start_token=1, - beam_end_token=2, - beam_max_step_num=2, - mode='train'): + def __init__( + self, + hidden_size, + src_vocab_size, + tar_vocab_size, + batch_size, + num_layers=1, + init_scale=0.1, + dropout=None, + beam_size=1, + beam_start_token=1, + beam_end_token=2, + beam_max_step_num=2, + mode='train', + ): super(BaseModel, self).__init__() self.hidden_size = hidden_size self.src_vocab_size = src_vocab_size @@ -116,40 +124,54 @@ class BaseModel(fluid.dygraph.Layer): self.src_embeder = Embedding( size=[self.src_vocab_size, self.hidden_size], param_attr=fluid.ParamAttr( - initializer=uniform_initializer(init_scale))) + initializer=uniform_initializer(init_scale) + ), + ) self.tar_embeder = Embedding( size=[self.tar_vocab_size, self.hidden_size], is_sparse=False, param_attr=fluid.ParamAttr( - initializer=uniform_initializer(init_scale))) + initializer=uniform_initializer(init_scale) + ), + ) self.enc_units = [] for i in range(num_layers): self.enc_units.append( self.add_sublayer( "enc_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size, - param_attr=param_attr, - bias_attr=bias_attr, - forget_bias=forget_bias))) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size, + param_attr=param_attr, + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) self.dec_units = [] for i in range(num_layers): self.dec_units.append( self.add_sublayer( "dec_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size, - param_attr=param_attr, - bias_attr=bias_attr, - forget_bias=forget_bias))) - - self.fc = fluid.dygraph.nn.Linear(self.hidden_size, - self.tar_vocab_size, - param_attr=param_attr, - bias_attr=False) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size, + param_attr=param_attr, + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) + + self.fc = fluid.dygraph.nn.Linear( + self.hidden_size, + self.tar_vocab_size, + param_attr=param_attr, + bias_attr=False, + ) def _transpose_batch_time(self, x): return fluid.layers.transpose(x, [1, 0] + list(range(2, len(x.shape)))) @@ -168,8 +190,9 @@ class BaseModel(fluid.dygraph.Layer): return x def _real_state(self, state, new_state, step_mask): - new_state = fluid.layers.elementwise_mul(new_state, step_mask, axis=0) - \ - fluid.layers.elementwise_mul(state, (step_mask - 1), axis=0) + new_state = fluid.layers.elementwise_mul( + new_state, step_mask, axis=0 + ) - fluid.layers.elementwise_mul(state, (step_mask - 1), axis=0) return new_state def _gather(self, x, indices, batch_pos): @@ -187,26 +210,28 @@ class BaseModel(fluid.dygraph.Layer): # NOTE: modify model code about `enc_hidden` and `enc_cell` to transforme dygraph code successfully. # Because nested list can't be transformed now. enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) zero = fluid.layers.zeros(shape=[1], dtype="int64") enc_hidden = fluid.layers.create_array(dtype="float32") enc_cell = fluid.layers.create_array(dtype="float32") for i in range(self.num_layers): index = zero + i - enc_hidden = fluid.layers.array_write(enc_hidden_0, - index, - array=enc_hidden) - enc_cell = fluid.layers.array_write(enc_cell_0, - index, - array=enc_cell) + enc_hidden = fluid.layers.array_write( + enc_hidden_0, index, array=enc_hidden + ) + enc_cell = fluid.layers.array_write( + enc_cell_0, index, array=enc_cell + ) max_seq_len = src_emb.shape[0] - enc_len_mask = fluid.layers.sequence_mask(src_sequence_length, - maxlen=max_seq_len, - dtype="float32") + enc_len_mask = fluid.layers.sequence_mask( + src_sequence_length, maxlen=max_seq_len, dtype="float32" + ) enc_len_mask = fluid.layers.transpose(enc_len_mask, [1, 0]) # TODO: Because diff exits if call while_loop in static graph. @@ -218,21 +243,24 @@ class BaseModel(fluid.dygraph.Layer): step_mask = enc_len_mask[k] new_enc_hidden, new_enc_cell = [], [] for i in range(self.num_layers): - enc_new_hidden, enc_new_cell = self.enc_units[i](enc_step_input, - enc_hidden[i], - enc_cell[i]) + enc_new_hidden, enc_new_cell = self.enc_units[i]( + enc_step_input, enc_hidden[i], enc_cell[i] + ) if self.dropout != None and self.dropout > 0.0: enc_step_input = fluid.layers.dropout( enc_new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: enc_step_input = enc_new_hidden new_enc_hidden.append( - self._real_state(enc_hidden[i], enc_new_hidden, step_mask)) + self._real_state(enc_hidden[i], enc_new_hidden, step_mask) + ) new_enc_cell.append( - self._real_state(enc_cell[i], enc_new_cell, step_mask)) + self._real_state(enc_cell[i], enc_new_cell, step_mask) + ) enc_hidden, enc_cell = new_enc_hidden, new_enc_cell @@ -245,30 +273,31 @@ class BaseModel(fluid.dygraph.Layer): step_input = tar_emb[j] new_dec_hidden, new_dec_cell = [], [] for i in range(self.num_layers): - new_hidden, new_cell = self.dec_units[i](step_input, - dec_hidden[i], - dec_cell[i]) + new_hidden, new_cell = self.dec_units[i]( + step_input, dec_hidden[i], dec_cell[i] + ) new_dec_hidden.append(new_hidden) new_dec_cell.append(new_cell) if self.dropout != None and self.dropout > 0.0: step_input = fluid.layers.dropout( new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: step_input = new_hidden dec_output.append(step_input) dec_output = fluid.layers.stack(dec_output) dec_output = self.fc(self._transpose_batch_time(dec_output)) - loss = fluid.layers.softmax_with_cross_entropy(logits=dec_output, - label=label, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=dec_output, label=label, soft_label=False + ) loss = fluid.layers.squeeze(loss, axes=[2]) max_tar_seq_len = fluid.layers.shape(tar)[1] - tar_mask = fluid.layers.sequence_mask(tar_sequence_length, - maxlen=max_tar_seq_len, - dtype='float32') + tar_mask = fluid.layers.sequence_mask( + tar_sequence_length, maxlen=max_tar_seq_len, dtype='float32' + ) loss = loss * tar_mask loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -283,26 +312,28 @@ class BaseModel(fluid.dygraph.Layer): src_emb = self.src_embeder(self._transpose_batch_time(src)) enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) zero = fluid.layers.zeros(shape=[1], dtype="int64") enc_hidden = fluid.layers.create_array(dtype="float32") enc_cell = fluid.layers.create_array(dtype="float32") for j in range(self.num_layers): index = zero + j - enc_hidden = fluid.layers.array_write(enc_hidden_0, - index, - array=enc_hidden) - enc_cell = fluid.layers.array_write(enc_cell_0, - index, - array=enc_cell) + enc_hidden = fluid.layers.array_write( + enc_hidden_0, index, array=enc_hidden + ) + enc_cell = fluid.layers.array_write( + enc_cell_0, index, array=enc_cell + ) max_seq_len = src_emb.shape[0] - enc_len_mask = fluid.layers.sequence_mask(src_sequence_length, - maxlen=max_seq_len, - dtype="float32") + enc_len_mask = fluid.layers.sequence_mask( + src_sequence_length, maxlen=max_seq_len, dtype="float32" + ) enc_len_mask = fluid.layers.transpose(enc_len_mask, [1, 0]) for k in range(args.max_seq_len): @@ -312,40 +343,50 @@ class BaseModel(fluid.dygraph.Layer): new_enc_hidden, new_enc_cell = [], [] for i in range(self.num_layers): - enc_new_hidden, enc_new_cell = self.enc_units[i](enc_step_input, - enc_hidden[i], - enc_cell[i]) + enc_new_hidden, enc_new_cell = self.enc_units[i]( + enc_step_input, enc_hidden[i], enc_cell[i] + ) if self.dropout != None and self.dropout > 0.0: enc_step_input = fluid.layers.dropout( enc_new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: enc_step_input = enc_new_hidden new_enc_hidden.append( - self._real_state(enc_hidden[i], enc_new_hidden, step_mask)) + self._real_state(enc_hidden[i], enc_new_hidden, step_mask) + ) new_enc_cell.append( - self._real_state(enc_cell[i], enc_new_cell, step_mask)) + self._real_state(enc_cell[i], enc_new_cell, step_mask) + ) enc_hidden, enc_cell = new_enc_hidden, new_enc_cell # beam search batch_beam_shape = (self.batch_size, self.beam_size) - vocab_size_tensor = to_variable(np.full( - (1), self.tar_vocab_size)).astype("int64") + vocab_size_tensor = to_variable( + np.full((1), self.tar_vocab_size) + ).astype("int64") start_token_tensor = to_variable( - np.full(batch_beam_shape, self.beam_start_token, dtype='int64')) + np.full(batch_beam_shape, self.beam_start_token, dtype='int64') + ) end_token_tensor = to_variable( - np.full(batch_beam_shape, self.beam_end_token, dtype='int64')) + np.full(batch_beam_shape, self.beam_end_token, dtype='int64') + ) step_input = self.tar_embeder(start_token_tensor) beam_finished = to_variable( - np.full(batch_beam_shape, 0, dtype='float32')) + np.full(batch_beam_shape, 0, dtype='float32') + ) beam_state_log_probs = to_variable( - np.array([[0.] + [-self.kinf] * (self.beam_size - 1)], - dtype="float32")) - beam_state_log_probs = fluid.layers.expand(beam_state_log_probs, - [self.batch_size, 1]) + np.array( + [[0.0] + [-self.kinf] * (self.beam_size - 1)], dtype="float32" + ) + ) + beam_state_log_probs = fluid.layers.expand( + beam_state_log_probs, [self.batch_size, 1] + ) dec_hidden, dec_cell = enc_hidden, enc_cell dec_hidden = [self._expand_to_beam_size(ele) for ele in dec_hidden] dec_cell = [self._expand_to_beam_size(ele) for ele in dec_cell] @@ -353,7 +394,10 @@ class BaseModel(fluid.dygraph.Layer): batch_pos = fluid.layers.expand( fluid.layers.unsqueeze( to_variable(np.arange(0, self.batch_size, 1, dtype="int64")), - [1]), [1, self.beam_size]) + [1], + ), + [1, self.beam_size], + ) predicted_ids = [] parent_ids = [] @@ -369,16 +413,17 @@ class BaseModel(fluid.dygraph.Layer): dec_cell = [self._merge_batch_beams(state) for state in dec_cell] for i in range(self.num_layers): - new_hidden, new_cell = self.dec_units[i](step_input, - dec_hidden[i], - dec_cell[i]) + new_hidden, new_cell = self.dec_units[i]( + step_input, dec_hidden[i], dec_cell[i] + ) new_dec_hidden.append(new_hidden) new_dec_cell.append(new_cell) if self.dropout != None and self.dropout > 0.0: step_input = fluid.layers.dropout( new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: step_input = new_hidden @@ -386,28 +431,40 @@ class BaseModel(fluid.dygraph.Layer): cell_outputs = self.fc(cell_outputs) step_log_probs = fluid.layers.log( - fluid.layers.softmax(cell_outputs)) + fluid.layers.softmax(cell_outputs) + ) noend_array = [-self.kinf] * self.tar_vocab_size noend_array[self.beam_end_token] = 0 noend_mask_tensor = to_variable( - np.array(noend_array, dtype='float32')) + np.array(noend_array, dtype='float32') + ) step_log_probs = fluid.layers.elementwise_mul( - fluid.layers.expand(fluid.layers.unsqueeze(beam_finished, [2]), [1, 1, self.tar_vocab_size]), - noend_mask_tensor, axis=-1) - \ - fluid.layers.elementwise_mul(step_log_probs, (beam_finished - 1), axis=0) - log_probs = fluid.layers.elementwise_add(x=step_log_probs, - y=beam_state_log_probs, - axis=0) + fluid.layers.expand( + fluid.layers.unsqueeze(beam_finished, [2]), + [1, 1, self.tar_vocab_size], + ), + noend_mask_tensor, + axis=-1, + ) - fluid.layers.elementwise_mul( + step_log_probs, (beam_finished - 1), axis=0 + ) + log_probs = fluid.layers.elementwise_add( + x=step_log_probs, y=beam_state_log_probs, axis=0 + ) scores = fluid.layers.reshape( - log_probs, [-1, self.beam_size * self.tar_vocab_size]) - topk_scores, topk_indices = fluid.layers.topk(input=scores, - k=self.beam_size) + log_probs, [-1, self.beam_size * self.tar_vocab_size] + ) + topk_scores, topk_indices = fluid.layers.topk( + input=scores, k=self.beam_size + ) beam_indices = fluid.layers.elementwise_floordiv( - topk_indices, vocab_size_tensor) + topk_indices, vocab_size_tensor + ) token_indices = fluid.layers.elementwise_mod( - topk_indices, vocab_size_tensor) + topk_indices, vocab_size_tensor + ) next_log_probs = self._gather(scores, topk_indices, batch_pos) x = 0 @@ -434,7 +491,8 @@ class BaseModel(fluid.dygraph.Layer): next_finished = fluid.layers.cast(next_finished, "bool") next_finished = fluid.layers.logical_or( next_finished, - fluid.layers.equal(token_indices, end_token_tensor)) + fluid.layers.equal(token_indices, end_token_tensor), + ) next_finished = fluid.layers.cast(next_finished, "float32") dec_hidden, dec_cell = new_dec_hidden, new_dec_cell @@ -452,20 +510,21 @@ class BaseModel(fluid.dygraph.Layer): class AttentionModel(fluid.dygraph.Layer): - - def __init__(self, - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=1, - init_scale=0.1, - dropout=None, - beam_size=1, - beam_start_token=1, - beam_end_token=2, - beam_max_step_num=2, - mode='train'): + def __init__( + self, + hidden_size, + src_vocab_size, + tar_vocab_size, + batch_size, + num_layers=1, + init_scale=0.1, + dropout=None, + beam_size=1, + beam_start_token=1, + beam_end_token=2, + beam_max_step_num=2, + mode='train', + ): super(AttentionModel, self).__init__() self.hidden_size = hidden_size self.src_vocab_size = src_vocab_size @@ -489,25 +548,33 @@ class AttentionModel(fluid.dygraph.Layer): size=[self.src_vocab_size, self.hidden_size], param_attr=fluid.ParamAttr( name='source_embedding', - initializer=uniform_initializer(init_scale))) + initializer=uniform_initializer(init_scale), + ), + ) self.tar_embeder = Embedding( size=[self.tar_vocab_size, self.hidden_size], is_sparse=False, param_attr=fluid.ParamAttr( name='target_embedding', - initializer=uniform_initializer(init_scale))) + initializer=uniform_initializer(init_scale), + ), + ) self.enc_units = [] for i in range(num_layers): self.enc_units.append( self.add_sublayer( "enc_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size, - param_attr=param_attr, - bias_attr=bias_attr, - forget_bias=forget_bias))) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size, + param_attr=param_attr, + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) self.dec_units = [] for i in range(num_layers): @@ -515,50 +582,67 @@ class AttentionModel(fluid.dygraph.Layer): self.dec_units.append( self.add_sublayer( "dec_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size * 2, - param_attr=ParamAttr( - name="dec_units_%d" % i, - initializer=uniform_initializer( - self.init_scale)), - bias_attr=bias_attr, - forget_bias=forget_bias))) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size * 2, + param_attr=ParamAttr( + name="dec_units_%d" % i, + initializer=uniform_initializer( + self.init_scale + ), + ), + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) else: self.dec_units.append( self.add_sublayer( "dec_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size, - param_attr=ParamAttr( - name="dec_units_%d" % i, - initializer=uniform_initializer( - self.init_scale)), - bias_attr=bias_attr, - forget_bias=forget_bias))) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size, + param_attr=ParamAttr( + name="dec_units_%d" % i, + initializer=uniform_initializer( + self.init_scale + ), + ), + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) self.attn_fc = fluid.dygraph.nn.Linear( self.hidden_size, self.hidden_size, - param_attr=ParamAttr(name="self_attn_fc", - initializer=uniform_initializer( - self.init_scale)), - bias_attr=False) + param_attr=ParamAttr( + name="self_attn_fc", + initializer=uniform_initializer(self.init_scale), + ), + bias_attr=False, + ) self.concat_fc = fluid.dygraph.nn.Linear( 2 * self.hidden_size, self.hidden_size, - param_attr=ParamAttr(name="self_concat_fc", - initializer=uniform_initializer( - self.init_scale)), - bias_attr=False) - - self.fc = fluid.dygraph.nn.Linear(self.hidden_size, - self.tar_vocab_size, - param_attr=ParamAttr( - name="self_fc", - initializer=uniform_initializer( - self.init_scale)), - bias_attr=False) + param_attr=ParamAttr( + name="self_concat_fc", + initializer=uniform_initializer(self.init_scale), + ), + bias_attr=False, + ) + + self.fc = fluid.dygraph.nn.Linear( + self.hidden_size, + self.tar_vocab_size, + param_attr=ParamAttr( + name="self_fc", initializer=uniform_initializer(self.init_scale) + ), + bias_attr=False, + ) def _transpose_batch_time(self, x): return fluid.layers.transpose(x, [1, 0] + list(range(2, len(x.shape)))) @@ -571,16 +655,16 @@ class AttentionModel(fluid.dygraph.Layer): expand_times = [1] * len(x.shape) expand_times[1] = self.beam_size x = fluid.layers.expand(x, expand_times) # [batch_size, beam_size, ...] - x = fluid.layers.transpose(x, - list(range(2, len(x.shape))) + - [0, 1]) # [..., batch_size, beam_size] + x = fluid.layers.transpose( + x, list(range(2, len(x.shape))) + [0, 1] + ) # [..., batch_size, beam_size] # use 0 to copy to avoid wrong shape - x = fluid.layers.reshape(x, shape=[0] * (len(x.shape) - 2) + - [-1]) # [..., batch_size * beam_size] + x = fluid.layers.reshape( + x, shape=[0] * (len(x.shape) - 2) + [-1] + ) # [..., batch_size * beam_size] x = fluid.layers.transpose( - x, [len(x.shape) - 1] + - list(range(0, - len(x.shape) - 1))) # [batch_size * beam_size, ...] + x, [len(x.shape) - 1] + list(range(0, len(x.shape) - 1)) + ) # [batch_size * beam_size, ...] return x def _split_batch_beams(self, x): @@ -594,8 +678,9 @@ class AttentionModel(fluid.dygraph.Layer): return x def _real_state(self, state, new_state, step_mask): - new_state = fluid.layers.elementwise_mul(new_state, step_mask, axis=0) - \ - fluid.layers.elementwise_mul(state, (step_mask - 1), axis=0) + new_state = fluid.layers.elementwise_mul( + new_state, step_mask, axis=0 + ) - fluid.layers.elementwise_mul(state, (step_mask - 1), axis=0) return new_state def _gather(self, x, indices, batch_pos): @@ -635,29 +720,31 @@ class AttentionModel(fluid.dygraph.Layer): # NOTE: modify model code about `enc_hidden` and `enc_cell` to transforme dygraph code successfully. # Because nested list can't be transformed now. enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) enc_hidden_0.stop_gradient = True enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) enc_hidden_0.stop_gradient = True zero = fluid.layers.zeros(shape=[1], dtype="int64") enc_hidden = fluid.layers.create_array(dtype="float32") enc_cell = fluid.layers.create_array(dtype="float32") for i in range(self.num_layers): index = zero + i - enc_hidden = fluid.layers.array_write(enc_hidden_0, - index, - array=enc_hidden) - enc_cell = fluid.layers.array_write(enc_cell_0, - index, - array=enc_cell) + enc_hidden = fluid.layers.array_write( + enc_hidden_0, index, array=enc_hidden + ) + enc_cell = fluid.layers.array_write( + enc_cell_0, index, array=enc_cell + ) max_seq_len = src_emb.shape[0] - enc_len_mask = fluid.layers.sequence_mask(src_sequence_length, - maxlen=max_seq_len, - dtype="float32") - enc_padding_mask = (enc_len_mask - 1.0) + enc_len_mask = fluid.layers.sequence_mask( + src_sequence_length, maxlen=max_seq_len, dtype="float32" + ) + enc_padding_mask = enc_len_mask - 1.0 enc_len_mask = fluid.layers.transpose(enc_len_mask, [1, 0]) enc_outputs = [] @@ -670,21 +757,24 @@ class AttentionModel(fluid.dygraph.Layer): step_mask = enc_len_mask[k] new_enc_hidden, new_enc_cell = [], [] for i in range(self.num_layers): - enc_new_hidden, enc_new_cell = self.enc_units[i](enc_step_input, - enc_hidden[i], - enc_cell[i]) + enc_new_hidden, enc_new_cell = self.enc_units[i]( + enc_step_input, enc_hidden[i], enc_cell[i] + ) if self.dropout != None and self.dropout > 0.0: enc_step_input = fluid.layers.dropout( enc_new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: enc_step_input = enc_new_hidden new_enc_hidden.append( - self._real_state(enc_hidden[i], enc_new_hidden, step_mask)) + self._real_state(enc_hidden[i], enc_new_hidden, step_mask) + ) new_enc_cell.append( - self._real_state(enc_cell[i], enc_new_cell, step_mask)) + self._real_state(enc_cell[i], enc_new_cell, step_mask) + ) enc_outputs.append(enc_step_input) enc_hidden, enc_cell = new_enc_hidden, new_enc_cell @@ -693,7 +783,8 @@ class AttentionModel(fluid.dygraph.Layer): # train input_feed = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) # NOTE: set stop_gradient here, otherwise grad var is null input_feed.stop_gradient = True dec_hidden, dec_cell = enc_hidden, enc_cell @@ -707,16 +798,17 @@ class AttentionModel(fluid.dygraph.Layer): step_input = fluid.layers.concat([step_input, input_feed], 1) new_dec_hidden, new_dec_cell = [], [] for i in range(self.num_layers): - new_hidden, new_cell = self.dec_units[i](step_input, - dec_hidden[i], - dec_cell[i]) + new_hidden, new_cell = self.dec_units[i]( + step_input, dec_hidden[i], dec_cell[i] + ) new_dec_hidden.append(new_hidden) new_dec_cell.append(new_cell) if self.dropout != None and self.dropout > 0.0: step_input = fluid.layers.dropout( new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: step_input = new_hidden dec_att = self.attention(step_input, enc_outputs, enc_padding_mask) @@ -729,14 +821,14 @@ class AttentionModel(fluid.dygraph.Layer): dec_output = fluid.layers.stack(dec_output) dec_output = self.fc(self._transpose_batch_time(dec_output)) - loss = fluid.layers.softmax_with_cross_entropy(logits=dec_output, - label=label, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=dec_output, label=label, soft_label=False + ) loss = fluid.layers.squeeze(loss, axes=[2]) max_tar_seq_len = fluid.layers.shape(tar)[1] - tar_mask = fluid.layers.sequence_mask(tar_sequence_length, - maxlen=max_tar_seq_len, - dtype='float32') + tar_mask = fluid.layers.sequence_mask( + tar_sequence_length, maxlen=max_tar_seq_len, dtype='float32' + ) loss = loss * tar_mask loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_utils.py index 23e25a974eb..44a55c79c92 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_utils.py @@ -40,7 +40,7 @@ def get_data_iter(batch_size, mode='train', cache_num=20): mask = np.zeros((bs), dtype='int32') for i, ele in enumerate(data): - ids[i, :len(ele)] = ele + ids[i, : len(ele)] = ele if not source: mask[i] = len(ele) - 1 else: @@ -61,7 +61,7 @@ def get_data_iter(batch_size, mode='train', cache_num=20): new_cache = sorted(b_src, key=lambda k: len(k[0])) for i in range(cache_num): - batch_data = new_cache[i * batch_size:(i + 1) * batch_size] + batch_data = new_cache[i * batch_size : (i + 1) * batch_size] src_cache = [w[0] for w in batch_data] tar_cache = [w[1] for w in batch_data] src_ids, src_mask = to_pad_np(src_cache, source=True) @@ -83,7 +83,7 @@ def get_data_iter(batch_size, mode='train', cache_num=20): for i in range(cache_num): batch_end = min(len(new_cache), (i + 1) * batch_size) - batch_data = new_cache[i * batch_size:batch_end] + batch_data = new_cache[i * batch_size : batch_end] src_cache = [w[0] for w in batch_data] tar_cache = [w[1] for w in batch_data] src_ids, src_mask = to_pad_np(src_cache, source=True) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py index a18da2173fe..b9a32c6e52a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py @@ -41,12 +41,14 @@ class EmbeddingLayer(object): """ # TODO(huihuangzheng): The original code set the is_sparse=True, but it # causes crush in dy2stat. Set it to True after fixing it. - emb = Embedding(size=[self.dict_size, self.emb_dim], - is_sparse=True, - padding_idx=self.padding_idx, - param_attr=attr.ParamAttr( - name=self.name, - initializer=fluid.initializer.Xavier())) + emb = Embedding( + size=[self.dict_size, self.emb_dim], + is_sparse=True, + padding_idx=self.padding_idx, + param_attr=attr.ParamAttr( + name=self.name, initializer=fluid.initializer.Xavier() + ), + ) return emb @@ -68,10 +70,12 @@ class FCLayer(object): """ operation """ - fc = FC(size=self.fc_dim, - param_attr=attr.ParamAttr(name="%s.w" % self.name), - bias_attr=attr.ParamAttr(name="%s.b" % self.name), - act=self.act) + fc = FC( + size=self.fc_dim, + param_attr=attr.ParamAttr(name="%s.w" % self.name), + bias_attr=attr.ParamAttr(name="%s.b" % self.name), + act=self.act, + ) return fc @@ -307,14 +311,16 @@ class FC(Layer): conv = fc(data) """ - def __init__(self, - size, - num_flatten_dims=1, - param_attr=None, - bias_attr=None, - act=None, - is_test=False, - dtype="float32"): + def __init__( + self, + size, + num_flatten_dims=1, + param_attr=None, + bias_attr=None, + act=None, + is_test=False, + dtype="float32", + ): super(FC, self).__init__(dtype) self._size = size @@ -328,27 +334,32 @@ class FC(Layer): def _build_once(self, input): i = 0 for inp, param in self._helper.iter_inputs_and_params( - input, self._param_attr): + input, self._param_attr + ): input_shape = inp.shape param_shape = [ - reduce(lambda a, b: a * b, input_shape[self._num_flatten_dims:], - 1) + reduce( + lambda a, b: a * b, input_shape[self._num_flatten_dims :], 1 + ) ] + [self._size] self.__w.append( self.add_parameter( '_w%d' % i, - self.create_parameter(attr=param, - shape=param_shape, - dtype=self._dtype, - is_bias=False))) + self.create_parameter( + attr=param, + shape=param_shape, + dtype=self._dtype, + is_bias=False, + ), + ) + ) i += 1 size = list([self._size]) - self._b = self.create_parameter(attr=self._bias_attr, - shape=size, - dtype=self._dtype, - is_bias=True) + self._b = self.create_parameter( + attr=self._bias_attr, shape=size, dtype=self._dtype, is_bias=True + ) # TODO(songyouwei): We should remove _w property @property @@ -384,18 +395,18 @@ class FC(Layer): mul_results = list() i = 0 for inp, param in self._helper.iter_inputs_and_params( - input, self._param_attr): + input, self._param_attr + ): tmp = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="mul", - inputs={ - "X": inp, - "Y": self.__w[i] - }, - outputs={"Out": tmp}, - attrs={ - "x_num_col_dims": self._num_flatten_dims, - "y_num_col_dims": 1 - }) + self._helper.append_op( + type="mul", + inputs={"X": inp, "Y": self.__w[i]}, + outputs={"Out": tmp}, + attrs={ + "x_num_col_dims": self._num_flatten_dims, + "y_num_col_dims": 1, + }, + ) i += 1 mul_results.append(tmp) @@ -403,22 +414,25 @@ class FC(Layer): pre_bias = mul_results[0] else: pre_bias = self._helper.create_variable_for_type_inference( - self._dtype) - self._helper.append_op(type="sum", - inputs={"X": mul_results}, - outputs={"Out": pre_bias}, - attrs={"use_mkldnn": False}) + self._dtype + ) + self._helper.append_op( + type="sum", + inputs={"X": mul_results}, + outputs={"Out": pre_bias}, + attrs={"use_mkldnn": False}, + ) if self._b is not None: pre_activation = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self._b] - }, - outputs={'Out': [pre_activation]}, - attrs={'axis': self._num_flatten_dims}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self._b]}, + outputs={'Out': [pre_activation]}, + attrs={'axis': self._num_flatten_dims}, + ) else: pre_activation = pre_bias # Currently, we don't support inplace in dygraph mode @@ -450,7 +464,10 @@ class HingeLoss(object): constant.ops(neg, neg.shape, "float32", 0.0), elementwise_add.ops( elementwise_sub.ops(neg, pos), - constant.ops(neg, neg.shape, "float32", self.margin)))) + constant.ops(neg, neg.shape, "float32", self.margin), + ), + ) + ) return loss @@ -469,8 +486,9 @@ class BOW(Layer): self.emb_dim = conf_dict["net"]["emb_dim"] self.bow_dim = conf_dict["net"]["bow_dim"] self.seq_len = conf_dict["seq_len"] - self.emb_layer = EmbeddingLayer(self.dict_size, self.emb_dim, - "emb").ops() + self.emb_layer = EmbeddingLayer( + self.dict_size, self.emb_dim, "emb" + ).ops() self.bow_layer = Linear(self.bow_dim, self.bow_dim) self.bow_layer_po = FCLayer(self.bow_dim, None, "fc").ops() self.softmax_layer = FCLayer(2, "softmax", "cos_sim").ops() @@ -484,10 +502,12 @@ class BOW(Layer): # embedding layer left_emb = self.emb_layer(left) right_emb = self.emb_layer(right) - left_emb = fluid.layers.reshape(left_emb, - shape=[-1, self.seq_len, self.bow_dim]) - right_emb = fluid.layers.reshape(right_emb, - shape=[-1, self.seq_len, self.bow_dim]) + left_emb = fluid.layers.reshape( + left_emb, shape=[-1, self.seq_len, self.bow_dim] + ) + right_emb = fluid.layers.reshape( + right_emb, shape=[-1, self.seq_len, self.bow_dim] + ) bow_left = fluid.layers.reduce_sum(left_emb, dim=1) bow_right = fluid.layers.reduce_sum(right_emb, dim=1) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py index b82a7b663bd..cc8f90594e4 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py @@ -43,7 +43,9 @@ class EmbeddingLayer(object): padding_idx=self.padding_idx, param_attr=paddle.ParamAttr( name=self.name, - initializer=paddle.nn.initializer.XavierUniform())) + initializer=paddle.nn.initializer.XavierUniform(), + ), + ) return emb @@ -65,10 +67,12 @@ class FCLayer(object): """ operation """ - fc = FC(size=self.fc_dim, - param_attr=paddle.ParamAttr(name="%s.w" % self.name), - bias_attr=paddle.ParamAttr(name="%s.b" % self.name), - act=self.act) + fc = FC( + size=self.fc_dim, + param_attr=paddle.ParamAttr(name="%s.w" % self.name), + bias_attr=paddle.ParamAttr(name="%s.b" % self.name), + act=self.act, + ) return fc @@ -293,14 +297,16 @@ class FC(paddle.nn.Layer): """ - def __init__(self, - size, - num_flatten_dims=1, - param_attr=None, - bias_attr=None, - act=None, - is_test=False, - dtype="float32"): + def __init__( + self, + size, + num_flatten_dims=1, + param_attr=None, + bias_attr=None, + act=None, + is_test=False, + dtype="float32", + ): super(FC, self).__init__(dtype) self._size = size @@ -314,27 +320,32 @@ class FC(paddle.nn.Layer): def _build_once(self, input): i = 0 for inp, param in self._helper.iter_inputs_and_params( - input, self._param_attr): + input, self._param_attr + ): input_shape = inp.shape param_shape = [ - reduce(lambda a, b: a * b, input_shape[self._num_flatten_dims:], - 1) + reduce( + lambda a, b: a * b, input_shape[self._num_flatten_dims :], 1 + ) ] + [self._size] self.__w.append( self.add_parameter( '_w%d' % i, - self.create_parameter(attr=param, - shape=param_shape, - dtype=self._dtype, - is_bias=False))) + self.create_parameter( + attr=param, + shape=param_shape, + dtype=self._dtype, + is_bias=False, + ), + ) + ) i += 1 size = list([self._size]) - self._b = self.create_parameter(attr=self._bias_attr, - shape=size, - dtype=self._dtype, - is_bias=True) + self._b = self.create_parameter( + attr=self._bias_attr, shape=size, dtype=self._dtype, is_bias=True + ) # TODO(songyouwei): We should remove _w property @property @@ -370,18 +381,18 @@ class FC(paddle.nn.Layer): mul_results = list() i = 0 for inp, param in self._helper.iter_inputs_and_params( - input, self._param_attr): + input, self._param_attr + ): tmp = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="mul", - inputs={ - "X": inp, - "Y": self.__w[i] - }, - outputs={"Out": tmp}, - attrs={ - "x_num_col_dims": self._num_flatten_dims, - "y_num_col_dims": 1 - }) + self._helper.append_op( + type="mul", + inputs={"X": inp, "Y": self.__w[i]}, + outputs={"Out": tmp}, + attrs={ + "x_num_col_dims": self._num_flatten_dims, + "y_num_col_dims": 1, + }, + ) i += 1 mul_results.append(tmp) @@ -389,22 +400,25 @@ class FC(paddle.nn.Layer): pre_bias = mul_results[0] else: pre_bias = self._helper.create_variable_for_type_inference( - self._dtype) - self._helper.append_op(type="sum", - inputs={"X": mul_results}, - outputs={"Out": pre_bias}, - attrs={"use_mkldnn": False}) + self._dtype + ) + self._helper.append_op( + type="sum", + inputs={"X": mul_results}, + outputs={"Out": pre_bias}, + attrs={"use_mkldnn": False}, + ) if self._b is not None: pre_activation = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self._b] - }, - outputs={'Out': [pre_activation]}, - attrs={'axis': self._num_flatten_dims}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self._b]}, + outputs={'Out': [pre_activation]}, + attrs={'axis': self._num_flatten_dims}, + ) else: pre_activation = pre_bias # Currently, we don't support inplace in dygraph mode @@ -436,7 +450,10 @@ class HingeLoss(object): constant.ops(neg, neg.shape, "float32", 0.0), elementwise_add.ops( elementwise_sub.ops(neg, pos), - constant.ops(neg, neg.shape, "float32", self.margin)))) + constant.ops(neg, neg.shape, "float32", self.margin), + ), + ) + ) return loss @@ -455,10 +472,12 @@ class BOW(paddle.nn.Layer): self.emb_dim = conf_dict["net"]["emb_dim"] self.bow_dim = conf_dict["net"]["bow_dim"] self.seq_len = conf_dict["seq_len"] - self.emb_layer = EmbeddingLayer(self.dict_size, self.emb_dim, - "emb").ops() - self.bow_layer = paddle.nn.Linear(in_features=self.bow_dim, - out_features=self.bow_dim) + self.emb_layer = EmbeddingLayer( + self.dict_size, self.emb_dim, "emb" + ).ops() + self.bow_layer = paddle.nn.Linear( + in_features=self.bow_dim, out_features=self.bow_dim + ) self.bow_layer_po = FCLayer(self.bow_dim, None, "fc").ops() self.softmax_layer = FCLayer(2, "softmax", "cos_sim").ops() @@ -471,10 +490,12 @@ class BOW(paddle.nn.Layer): # embedding layer left_emb = self.emb_layer(left) right_emb = self.emb_layer(right) - left_emb = paddle.reshape(left_emb, - shape=[-1, self.seq_len, self.bow_dim]) - right_emb = paddle.reshape(right_emb, - shape=[-1, self.seq_len, self.bow_dim]) + left_emb = paddle.reshape( + left_emb, shape=[-1, self.seq_len, self.bow_dim] + ) + right_emb = paddle.reshape( + right_emb, shape=[-1, self.seq_len, self.bow_dim] + ) bow_left = paddle.fluid.layers.reduce_sum(left_emb, dim=1) bow_right = paddle.fluid.layers.reduce_sum(right_emb, dim=1) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_assert.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_assert.py index 05f11c090de..c975a50bffd 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_assert.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_assert.py @@ -33,7 +33,6 @@ def dyfunc_assert_non_variable(x=True): class TestAssertVariable(unittest.TestCase): - def _run(self, func, x, with_exception, to_static): ProgramTranslator().enable(to_static) if with_exception: @@ -49,28 +48,28 @@ class TestAssertVariable(unittest.TestCase): self._run(func, x, with_exception, False) def test_non_variable(self): - self._run_dy_static(dyfunc_assert_non_variable, - x=False, - with_exception=True) - self._run_dy_static(dyfunc_assert_non_variable, - x=True, - with_exception=False) + self._run_dy_static( + dyfunc_assert_non_variable, x=False, with_exception=True + ) + self._run_dy_static( + dyfunc_assert_non_variable, x=True, with_exception=False + ) def test_bool_variable(self): - self._run_dy_static(dyfunc_assert_variable, - x=numpy.array([False]), - with_exception=True) - self._run_dy_static(dyfunc_assert_variable, - x=numpy.array([True]), - with_exception=False) + self._run_dy_static( + dyfunc_assert_variable, x=numpy.array([False]), with_exception=True + ) + self._run_dy_static( + dyfunc_assert_variable, x=numpy.array([True]), with_exception=False + ) def test_int_variable(self): - self._run_dy_static(dyfunc_assert_variable, - x=numpy.array([0]), - with_exception=True) - self._run_dy_static(dyfunc_assert_variable, - x=numpy.array([1]), - with_exception=False) + self._run_dy_static( + dyfunc_assert_variable, x=numpy.array([0]), with_exception=True + ) + self._run_dy_static( + dyfunc_assert_variable, x=numpy.array([1]), with_exception=False + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py index dbbbae2732e..8315caf3e32 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py @@ -21,7 +21,11 @@ import paddle import paddle.fluid as fluid from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func -from ifelse_simple_func import dyfunc_with_if_else, dyfunc_with_if_else2, nested_if_else +from ifelse_simple_func import ( + dyfunc_with_if_else, + dyfunc_with_if_else2, + nested_if_else, +) class TestAST2Func(unittest.TestCase): @@ -37,7 +41,6 @@ class TestAST2Func(unittest.TestCase): return transformed_func def test_ast2func(self): - def func(x, y): return x + y @@ -55,7 +58,6 @@ class TestAST2Func(unittest.TestCase): self.assertTrue((true_ret == test_ret).all()) def test_ast2func_static(self): - def func(x): y = fluid.layers.relu(x) loss = paddle.mean(y) @@ -74,8 +76,9 @@ class TestAST2Func(unittest.TestCase): def test_ast2func_error(self): with self.assertRaises(Exception) as e: self.assertRaises(TypeError, ast_to_func("x = a + b", 'foo')) - self.assertTrue("Type of ast_root should be gast.AST or ast.AST" in str( - e.exception)) + self.assertTrue( + "Type of ast_root should be gast.AST or ast.AST" in str(e.exception) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py index 9074c84e086..656d2768dca 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py @@ -70,16 +70,22 @@ def dyfunc_bool_to_tensor(x): class TestDygraphBasicApi_ToVariable(unittest.TestCase): - def setUp(self): self.input = np.ones(5).astype("int32") self.test_funcs = [ - dyfunc_to_tensor, dyfunc_bool_to_tensor, dyfunc_int_to_tensor, - dyfunc_float_to_tensor, dyfunc_to_variable, dyfunc_to_variable_2, - dyfunc_to_variable_3 + dyfunc_to_tensor, + dyfunc_bool_to_tensor, + dyfunc_int_to_tensor, + dyfunc_float_to_tensor, + dyfunc_to_variable, + dyfunc_to_variable_2, + dyfunc_to_variable_3, ] - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def get_dygraph_output(self): with fluid.dygraph.guard(): @@ -111,13 +117,18 @@ def dyfunc_BilinearTensorProduct(layer1, layer2): input1_dim=5, input2_dim=4, output_dim=1000, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) - res = bilinearTensorProduct(fluid.dygraph.base.to_variable(layer1), - fluid.dygraph.base.to_variable(layer2)) + res = bilinearTensorProduct( + fluid.dygraph.base.to_variable(layer1), + fluid.dygraph.base.to_variable(layer2), + ) return res @@ -126,10 +137,12 @@ def dyfunc_Conv2D(input): num_channels=3, num_filters=2, filter_size=3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) res = conv2d(input) return res @@ -140,10 +153,12 @@ def dyfunc_Conv3D(input): num_channels=3, num_filters=2, filter_size=3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) res = conv3d(input) return res @@ -155,10 +170,12 @@ def dyfunc_Conv2DTranspose(input): num_filters=12, filter_size=12, use_cudnn=False, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) ret = conv2dTranspose(input) return ret @@ -170,10 +187,12 @@ def dyfunc_Conv3DTranspose(input): num_filters=12, filter_size=12, use_cudnn=False, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) ret = conv3dTranspose(input) return ret @@ -184,24 +203,24 @@ def dyfunc_Linear(input): input_dim=10, output_dim=5, act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) res = fc(input) return res def dyfunc_Pool2D(input): - fluid.dygraph.Pool2D(pool_size=2, - pool_type='avg', - pool_stride=1, - global_pooling=False) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='avg', - pool_stride=1, - global_pooling=False) + fluid.dygraph.Pool2D( + pool_size=2, pool_type='avg', pool_stride=1, global_pooling=False + ) + pool2d = fluid.dygraph.Pool2D( + pool_size=2, pool_type='avg', pool_stride=1, global_pooling=False + ) res = pool2d(input) return res @@ -209,7 +228,8 @@ def dyfunc_Pool2D(input): def dyfunc_Prelu(input): prelu0 = fluid.PRelu( mode='all', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0))) + param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0)), + ) res = prelu0(input=input) return res @@ -252,7 +272,6 @@ class TestDygraphBasicApi(unittest.TestCase): class TestDygraphBasicApi_BilinearTensorProduct(TestDygraphBasicApi): - def setUp(self): self.input1 = np.random.random((5, 5)).astype('float32') self.input2 = np.random.random((5, 4)).astype('float32') @@ -271,8 +290,9 @@ class TestDygraphBasicApi_BilinearTensorProduct(TestDygraphBasicApi): main_program = fluid.Program() main_program.random_seed = SEED with fluid.program_guard(main_program, startup_program): - static_out = dygraph_to_static_func(self.dygraph_func)(self.input1, - self.input2) + static_out = dygraph_to_static_func(self.dygraph_func)( + self.input1, self.input2 + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) @@ -281,42 +301,36 @@ class TestDygraphBasicApi_BilinearTensorProduct(TestDygraphBasicApi): class TestDygraphBasicApi_Conv2D(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((1, 3, 3, 5)).astype('float32') self.dygraph_func = dyfunc_Conv2D class TestDygraphBasicApi_Conv3D(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((1, 3, 3, 3, 5)).astype('float32') self.dygraph_func = dyfunc_Conv3D class TestDygraphBasicApi_Conv2DTranspose(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((5, 3, 32, 32)).astype('float32') self.dygraph_func = dyfunc_Conv2DTranspose class TestDygraphBasicApi_Conv3DTranspose(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((5, 3, 12, 32, 32)).astype('float32') self.dygraph_func = dyfunc_Conv3DTranspose class TestDygraphBasicApi_Linear(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((4, 3, 10)).astype('float32') self.dygraph_func = dyfunc_Linear class TestDygraphBasicApi_Prelu(TestDygraphBasicApi): - def setUp(self): self.input = np.ones([5, 20, 10, 10]).astype('float32') self.dygraph_func = dyfunc_Prelu @@ -325,39 +339,36 @@ class TestDygraphBasicApi_Prelu(TestDygraphBasicApi): # 2. test Apis that inherit from LearningRateDecay def dyfunc_CosineDecay(): base_lr = 0.1 - CosineDecay = fluid.dygraph.CosineDecay(learning_rate=base_lr, - step_each_epoch=10000, - epochs=120) + CosineDecay = fluid.dygraph.CosineDecay( + learning_rate=base_lr, step_each_epoch=10000, epochs=120 + ) lr = CosineDecay() return lr def dyfunc_ExponentialDecay(): base_lr = 0.1 - exponential_decay = fluid.dygraph.ExponentialDecay(learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True) + exponential_decay = fluid.dygraph.ExponentialDecay( + learning_rate=base_lr, decay_steps=10000, decay_rate=0.5, staircase=True + ) lr = exponential_decay() return lr def dyfunc_InverseTimeDecay(): base_lr = 0.1 - inverse_time_decay = fluid.dygraph.InverseTimeDecay(learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True) + inverse_time_decay = fluid.dygraph.InverseTimeDecay( + learning_rate=base_lr, decay_steps=10000, decay_rate=0.5, staircase=True + ) lr = inverse_time_decay() return lr def dyfunc_NaturalExpDecay(): base_lr = 0.1 - natural_exp_decay = fluid.dygraph.NaturalExpDecay(learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True) + natural_exp_decay = fluid.dygraph.NaturalExpDecay( + learning_rate=base_lr, decay_steps=10000, decay_rate=0.5, staircase=True + ) lr = natural_exp_decay() return lr @@ -386,7 +397,6 @@ def dyfunc_PolynomialDecay(): class TestDygraphBasicApi_CosineDecay(unittest.TestCase): - def setUp(self): self.dygraph_func = dyfunc_CosineDecay @@ -417,43 +427,38 @@ class TestDygraphBasicApi_CosineDecay(unittest.TestCase): class TestDygraphBasicApi_ExponentialDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_ExponentialDecay class TestDygraphBasicApi_InverseTimeDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_InverseTimeDecay class TestDygraphBasicApi_NaturalExpDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_NaturalExpDecay class TestDygraphBasicApi_NoamDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_NoamDecay class TestDygraphBasicApi_PiecewiseDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_PiecewiseDecay class TestDygraphBasicApi_PolynomialDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_PolynomialDecay def _dygraph_fn(): import paddle.fluid as fluid + x = np.random.random((1, 3)).astype('float32') with fluid.dygraph.guard(): fluid.dygraph.to_variable(x) @@ -461,7 +466,6 @@ def _dygraph_fn(): class TestDygraphApiRecognition(unittest.TestCase): - def setUp(self): self.src = inspect.getsource(_dygraph_fn) self.root = gast.parse(self.src) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py index 1929df30fe3..7fb9eca2606 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py @@ -29,15 +29,15 @@ from bert_utils import get_bert_config, get_feed_data_reader from predictor_utils import PredictorTools program_translator = ProgramTranslator() -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) SEED = 2020 STEP_NUM = 10 PRINT_STEP = 2 class TestBert(unittest.TestCase): - def setUp(self): self.bert_config = get_bert_config() self.data_reader = get_feed_data_reader(self.bert_config) @@ -46,8 +46,9 @@ class TestBert(unittest.TestCase): self.model_save_prefix = os.path.join(self.model_save_dir, 'bert') self.model_filename = 'bert' + INFER_MODEL_SUFFIX self.params_filename = 'bert' + INFER_PARAMS_SUFFIX - self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, - 'bert.dygraph') + self.dy_state_dict_save_path = os.path.join( + self.temp_dir.name, 'bert.dygraph' + ) def tearDown(self): self.temp_dir.cleanup() @@ -57,20 +58,30 @@ class TestBert(unittest.TestCase): fluid.default_main_program().random_seed = SEED fluid.default_startup_program().random_seed = SEED - data_loader = fluid.io.DataLoader.from_generator(capacity=50, - iterable=True) - data_loader.set_batch_generator(data_reader.data_generator(), - places=place) + data_loader = fluid.io.DataLoader.from_generator( + capacity=50, iterable=True + ) + data_loader.set_batch_generator( + data_reader.data_generator(), places=place + ) - bert = PretrainModelLayer(config=bert_config, - weight_sharing=False, - use_fp16=False) + bert = PretrainModelLayer( + config=bert_config, weight_sharing=False, use_fp16=False + ) optimizer = fluid.optimizer.Adam(parameter_list=bert.parameters()) step_idx = 0 speed_list = [] for input_data in data_loader(): - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = input_data + ( + src_ids, + pos_ids, + sent_ids, + input_mask, + mask_label, + mask_pos, + labels, + ) = input_data next_sent_acc, mask_lm_loss, total_loss = bert( src_ids=src_ids, position_ids=pos_ids, @@ -78,7 +89,8 @@ class TestBert(unittest.TestCase): input_mask=input_mask, mask_label=mask_label, mask_pos=mask_pos, - labels=labels) + labels=labels, + ) total_loss.backward() optimizer.minimize(total_loss) bert.clear_gradients() @@ -89,15 +101,18 @@ class TestBert(unittest.TestCase): if step_idx % PRINT_STEP == 0: if step_idx == 0: - print("Step: %d, loss: %f, ppl: %f, next_sent_acc: %f" % - (step_idx, loss, ppl, acc)) + print( + "Step: %d, loss: %f, ppl: %f, next_sent_acc: %f" + % (step_idx, loss, ppl, acc) + ) avg_batch_time = time.time() else: speed = PRINT_STEP / (time.time() - avg_batch_time) speed_list.append(speed) print( "Step: %d, loss: %f, ppl: %f, next_sent_acc: %f, speed: %.3f steps/s" - % (step_idx, loss, ppl, acc, speed)) + % (step_idx, loss, ppl, acc, speed) + ) avg_batch_time = time.time() step_idx += 1 @@ -105,8 +120,9 @@ class TestBert(unittest.TestCase): if to_static: fluid.dygraph.jit.save(bert, self.model_save_prefix) else: - fluid.dygraph.save_dygraph(bert.state_dict(), - self.dy_state_dict_save_path) + fluid.dygraph.save_dygraph( + bert.state_dict(), self.dy_state_dict_save_path + ) break return loss, ppl @@ -122,38 +138,56 @@ class TestBert(unittest.TestCase): paddle.enable_static() exe = fluid.Executor(place) # load inference model - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - pred_res = exe.run(inference_program, - feed=dict(zip(feed_target_names, data)), - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + pred_res = exe.run( + inference_program, + feed=dict(zip(feed_target_names, data)), + fetch_list=fetch_targets, + ) return pred_res def predict_dygraph(self, bert_config, data): program_translator.enable(False) with fluid.dygraph.guard(place): - bert = PretrainModelLayer(config=bert_config, - weight_sharing=False, - use_fp16=False) + bert = PretrainModelLayer( + config=bert_config, weight_sharing=False, use_fp16=False + ) model_dict, _ = fluid.dygraph.load_dygraph( - self.dy_state_dict_save_path) + self.dy_state_dict_save_path + ) bert.set_dict(model_dict) bert.eval() input_vars = [fluid.dygraph.to_variable(x) for x in data] - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = input_vars - pred_res = bert(src_ids=src_ids, - position_ids=pos_ids, - sentence_ids=sent_ids, - input_mask=input_mask, - mask_label=mask_label, - mask_pos=mask_pos, - labels=labels) + ( + src_ids, + pos_ids, + sent_ids, + input_mask, + mask_label, + mask_pos, + labels, + ) = input_vars + pred_res = bert( + src_ids=src_ids, + position_ids=pos_ids, + sentence_ids=sent_ids, + input_mask=input_mask, + mask_label=mask_label, + mask_pos=mask_pos, + labels=labels, + ) pred_res = [var.numpy() for var in pred_res] return pred_res @@ -163,24 +197,42 @@ class TestBert(unittest.TestCase): bert = fluid.dygraph.jit.load(self.model_save_prefix) bert.eval() - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = data - pred_res = bert(src_ids, pos_ids, sent_ids, input_mask, mask_label, - mask_pos, labels) + ( + src_ids, + pos_ids, + sent_ids, + input_mask, + mask_label, + mask_pos, + labels, + ) = data + pred_res = bert( + src_ids, + pos_ids, + sent_ids, + input_mask, + mask_label, + mask_pos, + labels, + ) pred_res = [var.numpy() for var in pred_res] return pred_res def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, data) + output = PredictorTools( + self.model_save_dir, self.model_filename, self.params_filename, data + ) out = output() return out def test_train(self): - static_loss, static_ppl = self.train_static(self.bert_config, - self.data_reader) - dygraph_loss, dygraph_ppl = self.train_dygraph(self.bert_config, - self.data_reader) + static_loss, static_ppl = self.train_static( + self.bert_config, self.data_reader + ) + dygraph_loss, dygraph_ppl = self.train_dygraph( + self.bert_config, self.data_reader + ) np.testing.assert_allclose(static_loss, dygraph_loss, rtol=1e-05) np.testing.assert_allclose(static_ppl, dygraph_ppl, rtol=1e-05) @@ -194,29 +246,38 @@ class TestBert(unittest.TestCase): predictor_pred_res = self.predict_analysis_inference(data) for dy_res, st_res, dy_jit_res, predictor_res in zip( - dygraph_pred_res, static_pred_res, dygraph_jit_pred_res, - predictor_pred_res): + dygraph_pred_res, + static_pred_res, + dygraph_jit_pred_res, + predictor_pred_res, + ): np.testing.assert_allclose( st_res, dy_res, rtol=1e-05, err_msg='dygraph_res: {},\n static_res: {}'.format( dy_res[~np.isclose(st_res, dy_res)], - st_res[~np.isclose(st_res, dy_res)])) + st_res[~np.isclose(st_res, dy_res)], + ), + ) np.testing.assert_allclose( st_res, dy_jit_res, rtol=1e-05, err_msg='dygraph_jit_res: {},\n static_res: {}'.format( dy_jit_res[~np.isclose(st_res, dy_jit_res)], - st_res[~np.isclose(st_res, dy_jit_res)])) + st_res[~np.isclose(st_res, dy_jit_res)], + ), + ) np.testing.assert_allclose( st_res, predictor_res, rtol=1e-05, err_msg='dygraph_jit_res: {},\n static_res: {}'.format( predictor_res[~np.isclose(st_res, predictor_res)], - st_res[~np.isclose(st_res, predictor_res)])) + st_res[~np.isclose(st_res, predictor_res)], + ), + ) break diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bmn.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bmn.py index e54c6274502..0f25b6ec71a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bmn.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bmn.py @@ -38,9 +38,10 @@ if fluid.is_compiled_with_cuda(): fluid.set_flags({'FLAGS_cudnn_deterministic': True}) -def get_interp1d_mask(tscale, dscale, prop_boundary_ratio, num_sample, - num_sample_perbin): - """ generate sample mask for each point in Boundary-Matching Map """ +def get_interp1d_mask( + tscale, dscale, prop_boundary_ratio, num_sample, num_sample_perbin +): + """generate sample mask for each point in Boundary-Matching Map""" mask_mat = [] for start_index in range(tscale): mask_mat_vector = [] @@ -51,9 +52,13 @@ def get_interp1d_mask(tscale, dscale, prop_boundary_ratio, num_sample, center_len = float(p_xmax - p_xmin) + 1 sample_xmin = p_xmin - center_len * prop_boundary_ratio sample_xmax = p_xmax + center_len * prop_boundary_ratio - p_mask = _get_interp1d_bin_mask(sample_xmin, sample_xmax, - tscale, num_sample, - num_sample_perbin) + p_mask = _get_interp1d_bin_mask( + sample_xmin, + sample_xmax, + tscale, + num_sample, + num_sample_perbin, + ) else: p_mask = np.zeros([tscale, num_sample]) mask_mat_vector.append(p_mask) @@ -66,9 +71,10 @@ def get_interp1d_mask(tscale, dscale, prop_boundary_ratio, num_sample, return sample_mask -def _get_interp1d_bin_mask(seg_xmin, seg_xmax, tscale, num_sample, - num_sample_perbin): - """ generate sample mask for a boundary-matching pair """ +def _get_interp1d_bin_mask( + seg_xmin, seg_xmax, tscale, num_sample, num_sample_perbin +): + """generate sample mask for a boundary-matching pair""" plen = float(seg_xmax - seg_xmin) plen_sample = plen / (num_sample * num_sample_perbin - 1.0) total_samples = [ @@ -77,8 +83,9 @@ def _get_interp1d_bin_mask(seg_xmin, seg_xmax, tscale, num_sample, ] p_mask = [] for idx in range(num_sample): - bin_samples = total_samples[idx * num_sample_perbin:(idx + 1) * - num_sample_perbin] + bin_samples = total_samples[ + idx * num_sample_perbin : (idx + 1) * num_sample_perbin + ] bin_vector = np.zeros([tscale]) for sample in bin_samples: sample_upper = math.ceil(sample) @@ -94,34 +101,39 @@ def _get_interp1d_bin_mask(seg_xmin, seg_xmax, tscale, num_sample, class Conv1D(fluid.dygraph.Layer): - - def __init__(self, - prefix, - num_channels=256, - num_filters=256, - size_k=3, - padding=1, - groups=1, - act="relu"): + def __init__( + self, + prefix, + num_channels=256, + num_filters=256, + size_k=3, + padding=1, + groups=1, + act="relu", + ): super(Conv1D, self).__init__() fan_in = num_channels * size_k * 1 - k = 1. / math.sqrt(fan_in) - param_attr = ParamAttr(name=prefix + "_w", - initializer=fluid.initializer.Uniform(low=-k, - high=k)) - bias_attr = ParamAttr(name=prefix + "_b", - initializer=fluid.initializer.Uniform(low=-k, - high=k)) - - self._conv2d = fluid.dygraph.Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=(1, size_k), - stride=1, - padding=(0, padding), - groups=groups, - act=act, - param_attr=param_attr, - bias_attr=bias_attr) + k = 1.0 / math.sqrt(fan_in) + param_attr = ParamAttr( + name=prefix + "_w", + initializer=fluid.initializer.Uniform(low=-k, high=k), + ) + bias_attr = ParamAttr( + name=prefix + "_b", + initializer=fluid.initializer.Uniform(low=-k, high=k), + ) + + self._conv2d = fluid.dygraph.Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=(1, size_k), + stride=1, + padding=(0, padding), + groups=groups, + act=act, + param_attr=param_attr, + bias_attr=bias_attr, + ) def forward(self, x): x = fluid.layers.unsqueeze(input=x, axes=[2]) @@ -131,7 +143,6 @@ class Conv1D(fluid.dygraph.Layer): class BMN(fluid.dygraph.Layer): - def __init__(self, cfg): super(BMN, self).__init__() @@ -146,55 +157,65 @@ class BMN(fluid.dygraph.Layer): self.hidden_dim_3d = 512 # Base Module - self.b_conv1 = Conv1D(prefix="Base_1", - num_channels=cfg.feat_dim, - num_filters=self.hidden_dim_1d, - size_k=3, - padding=1, - groups=4, - act="relu") - self.b_conv2 = Conv1D(prefix="Base_2", - num_filters=self.hidden_dim_1d, - size_k=3, - padding=1, - groups=4, - act="relu") + self.b_conv1 = Conv1D( + prefix="Base_1", + num_channels=cfg.feat_dim, + num_filters=self.hidden_dim_1d, + size_k=3, + padding=1, + groups=4, + act="relu", + ) + self.b_conv2 = Conv1D( + prefix="Base_2", + num_filters=self.hidden_dim_1d, + size_k=3, + padding=1, + groups=4, + act="relu", + ) # Temporal Evaluation Module - self.ts_conv1 = Conv1D(prefix="TEM_s1", - num_filters=self.hidden_dim_1d, - size_k=3, - padding=1, - groups=4, - act="relu") - self.ts_conv2 = Conv1D(prefix="TEM_s2", - num_filters=1, - size_k=1, - padding=0, - act="sigmoid") - self.te_conv1 = Conv1D(prefix="TEM_e1", - num_filters=self.hidden_dim_1d, - size_k=3, - padding=1, - groups=4, - act="relu") - self.te_conv2 = Conv1D(prefix="TEM_e2", - num_filters=1, - size_k=1, - padding=0, - act="sigmoid") - - #Proposal Evaluation Module - self.p_conv1 = Conv1D(prefix="PEM_1d", - num_filters=self.hidden_dim_2d, - size_k=3, - padding=1, - act="relu") + self.ts_conv1 = Conv1D( + prefix="TEM_s1", + num_filters=self.hidden_dim_1d, + size_k=3, + padding=1, + groups=4, + act="relu", + ) + self.ts_conv2 = Conv1D( + prefix="TEM_s2", num_filters=1, size_k=1, padding=0, act="sigmoid" + ) + self.te_conv1 = Conv1D( + prefix="TEM_e1", + num_filters=self.hidden_dim_1d, + size_k=3, + padding=1, + groups=4, + act="relu", + ) + self.te_conv2 = Conv1D( + prefix="TEM_e2", num_filters=1, size_k=1, padding=0, act="sigmoid" + ) + + # Proposal Evaluation Module + self.p_conv1 = Conv1D( + prefix="PEM_1d", + num_filters=self.hidden_dim_2d, + size_k=3, + padding=1, + act="relu", + ) # init to speed up - sample_mask = get_interp1d_mask(self.tscale, self.dscale, - self.prop_boundary_ratio, - self.num_sample, self.num_sample_perbin) + sample_mask = get_interp1d_mask( + self.tscale, + self.dscale, + self.prop_boundary_ratio, + self.num_sample, + self.num_sample_perbin, + ) self.sample_mask = fluid.dygraph.base.to_variable(sample_mask) self.sample_mask.stop_gradient = True @@ -206,7 +227,8 @@ class BMN(fluid.dygraph.Layer): padding=0, act="relu", param_attr=ParamAttr(name="PEM_3d1_w"), - bias_attr=ParamAttr(name="PEM_3d1_b")) + bias_attr=ParamAttr(name="PEM_3d1_b"), + ) self.p_conv2d1 = fluid.dygraph.Conv2D( num_channels=512, @@ -216,7 +238,8 @@ class BMN(fluid.dygraph.Layer): padding=0, act="relu", param_attr=ParamAttr(name="PEM_2d1_w"), - bias_attr=ParamAttr(name="PEM_2d1_b")) + bias_attr=ParamAttr(name="PEM_2d1_b"), + ) self.p_conv2d2 = fluid.dygraph.Conv2D( num_channels=128, num_filters=self.hidden_dim_2d, @@ -225,7 +248,8 @@ class BMN(fluid.dygraph.Layer): padding=1, act="relu", param_attr=ParamAttr(name="PEM_2d2_w"), - bias_attr=ParamAttr(name="PEM_2d2_b")) + bias_attr=ParamAttr(name="PEM_2d2_b"), + ) self.p_conv2d3 = fluid.dygraph.Conv2D( num_channels=128, num_filters=self.hidden_dim_2d, @@ -234,7 +258,8 @@ class BMN(fluid.dygraph.Layer): padding=1, act="relu", param_attr=ParamAttr(name="PEM_2d3_w"), - bias_attr=ParamAttr(name="PEM_2d3_b")) + bias_attr=ParamAttr(name="PEM_2d3_b"), + ) self.p_conv2d4 = fluid.dygraph.Conv2D( num_channels=128, num_filters=2, @@ -243,7 +268,8 @@ class BMN(fluid.dygraph.Layer): padding=0, act="sigmoid", param_attr=ParamAttr(name="PEM_2d4_w"), - bias_attr=ParamAttr(name="PEM_2d4_b")) + bias_attr=ParamAttr(name="PEM_2d4_b"), + ) @to_static def forward(self, x): @@ -263,8 +289,9 @@ class BMN(fluid.dygraph.Layer): xp = self.p_conv1(x) # BM layer xp = fluid.layers.matmul(xp, self.sample_mask) - xp = fluid.layers.reshape(xp, - shape=[0, 0, -1, self.dscale, self.tscale]) + xp = fluid.layers.reshape( + xp, shape=[0, 0, -1, self.dscale, self.tscale] + ) xp = self.p_conv3d1(xp) xp = fluid.layers.squeeze(xp, axes=[2]) @@ -275,51 +302,54 @@ class BMN(fluid.dygraph.Layer): return xp, xs, xe -def bmn_loss_func(pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, - cfg): - +def bmn_loss_func( + pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, cfg +): def _get_mask(cfg): dscale = cfg.dscale tscale = cfg.tscale bm_mask = [] for idx in range(dscale): - mask_vector = [1 for i in range(tscale - idx) - ] + [0 for i in range(idx)] + mask_vector = [1 for i in range(tscale - idx)] + [ + 0 for i in range(idx) + ] bm_mask.append(mask_vector) bm_mask = np.array(bm_mask, dtype=np.float32) - self_bm_mask = fluid.layers.create_global_var(shape=[dscale, tscale], - value=0, - dtype=DATATYPE, - persistable=True) + self_bm_mask = fluid.layers.create_global_var( + shape=[dscale, tscale], value=0, dtype=DATATYPE, persistable=True + ) fluid.layers.assign(bm_mask, self_bm_mask) self_bm_mask.stop_gradient = True return self_bm_mask def tem_loss_func(pred_start, pred_end, gt_start, gt_end): - def bi_loss(pred_score, gt_label): - pred_score = fluid.layers.reshape(x=pred_score, - shape=[-1], - inplace=False) - gt_label = fluid.layers.reshape(x=gt_label, - shape=[-1], - inplace=False) + pred_score = fluid.layers.reshape( + x=pred_score, shape=[-1], inplace=False + ) + gt_label = fluid.layers.reshape( + x=gt_label, shape=[-1], inplace=False + ) gt_label.stop_gradient = True pmask = fluid.layers.cast(x=(gt_label > 0.5), dtype=DATATYPE) - num_entries = fluid.layers.cast(fluid.layers.shape(pmask), - dtype=DATATYPE) - num_positive = fluid.layers.cast(fluid.layers.reduce_sum(pmask), - dtype=DATATYPE) + num_entries = fluid.layers.cast( + fluid.layers.shape(pmask), dtype=DATATYPE + ) + num_positive = fluid.layers.cast( + fluid.layers.reduce_sum(pmask), dtype=DATATYPE + ) ratio = num_entries / num_positive coef_0 = 0.5 * ratio / (ratio - 1) coef_1 = 0.5 * ratio epsilon = 0.000001 # temp = fluid.layers.log(pred_score + epsilon) loss_pos = fluid.layers.elementwise_mul( - fluid.layers.log(pred_score + epsilon), pmask) + fluid.layers.log(pred_score + epsilon), pmask + ) loss_pos = coef_1 * fluid.layers.reduce_mean(loss_pos) loss_neg = fluid.layers.elementwise_mul( - fluid.layers.log(1.0 - pred_score + epsilon), (1.0 - pmask)) + fluid.layers.log(1.0 - pred_score + epsilon), (1.0 - pmask) + ) loss_neg = coef_0 * fluid.layers.reduce_mean(loss_neg) loss = -1 * (loss_pos + loss_neg) return loss @@ -336,39 +366,47 @@ def bmn_loss_func(pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, u_hmask = fluid.layers.cast(x=gt_iou_map > 0.7, dtype=DATATYPE) u_mmask = fluid.layers.logical_and(gt_iou_map <= 0.7, gt_iou_map > 0.3) u_mmask = fluid.layers.cast(x=u_mmask, dtype=DATATYPE) - u_lmask = fluid.layers.logical_and(gt_iou_map <= 0.3, gt_iou_map >= 0.) + u_lmask = fluid.layers.logical_and(gt_iou_map <= 0.3, gt_iou_map >= 0.0) u_lmask = fluid.layers.cast(x=u_lmask, dtype=DATATYPE) u_lmask = fluid.layers.elementwise_mul(u_lmask, mask) - num_h = fluid.layers.cast(fluid.layers.reduce_sum(u_hmask), - dtype=DATATYPE) - num_m = fluid.layers.cast(fluid.layers.reduce_sum(u_mmask), - dtype=DATATYPE) - num_l = fluid.layers.cast(fluid.layers.reduce_sum(u_lmask), - dtype=DATATYPE) + num_h = fluid.layers.cast( + fluid.layers.reduce_sum(u_hmask), dtype=DATATYPE + ) + num_m = fluid.layers.cast( + fluid.layers.reduce_sum(u_mmask), dtype=DATATYPE + ) + num_l = fluid.layers.cast( + fluid.layers.reduce_sum(u_lmask), dtype=DATATYPE + ) r_m = num_h / num_m u_smmask = fluid.layers.assign( local_random.uniform( - 0., 1., - [gt_iou_map.shape[1], gt_iou_map.shape[2]]).astype(DATATYPE)) + 0.0, 1.0, [gt_iou_map.shape[1], gt_iou_map.shape[2]] + ).astype(DATATYPE) + ) u_smmask = fluid.layers.elementwise_mul(u_mmask, u_smmask) - u_smmask = fluid.layers.cast(x=(u_smmask > (1. - r_m)), dtype=DATATYPE) + u_smmask = fluid.layers.cast(x=(u_smmask > (1.0 - r_m)), dtype=DATATYPE) r_l = num_h / num_l u_slmask = fluid.layers.assign( local_random.uniform( - 0., 1., - [gt_iou_map.shape[1], gt_iou_map.shape[2]]).astype(DATATYPE)) + 0.0, 1.0, [gt_iou_map.shape[1], gt_iou_map.shape[2]] + ).astype(DATATYPE) + ) u_slmask = fluid.layers.elementwise_mul(u_lmask, u_slmask) - u_slmask = fluid.layers.cast(x=(u_slmask > (1. - r_l)), dtype=DATATYPE) + u_slmask = fluid.layers.cast(x=(u_slmask > (1.0 - r_l)), dtype=DATATYPE) weights = u_hmask + u_smmask + u_slmask weights.stop_gradient = True loss = fluid.layers.square_error_cost(pred_score, gt_iou_map) loss = fluid.layers.elementwise_mul(loss, weights) - loss = 0.5 * fluid.layers.reduce_sum(loss) / fluid.layers.reduce_sum( - weights) + loss = ( + 0.5 + * fluid.layers.reduce_sum(loss) + / fluid.layers.reduce_sum(weights) + ) return loss @@ -386,24 +424,22 @@ def bmn_loss_func(pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, coef_1 = 0.5 * ratio epsilon = 0.000001 loss_pos = fluid.layers.elementwise_mul( - fluid.layers.log(pred_score + epsilon), pmask) + fluid.layers.log(pred_score + epsilon), pmask + ) loss_pos = coef_1 * fluid.layers.reduce_sum(loss_pos) loss_neg = fluid.layers.elementwise_mul( - fluid.layers.log(1.0 - pred_score + epsilon), nmask) + fluid.layers.log(1.0 - pred_score + epsilon), nmask + ) loss_neg = coef_0 * fluid.layers.reduce_sum(loss_neg) loss = -1 * (loss_pos + loss_neg) / num_entries return loss - pred_bm_reg = fluid.layers.squeeze(fluid.layers.slice(pred_bm, - axes=[1], - starts=[0], - ends=[1]), - axes=[1]) - pred_bm_cls = fluid.layers.squeeze(fluid.layers.slice(pred_bm, - axes=[1], - starts=[1], - ends=[2]), - axes=[1]) + pred_bm_reg = fluid.layers.squeeze( + fluid.layers.slice(pred_bm, axes=[1], starts=[0], ends=[1]), axes=[1] + ) + pred_bm_cls = fluid.layers.squeeze( + fluid.layers.slice(pred_bm, axes=[1], starts=[1], ends=[2]), axes=[1] + ) bm_mask = _get_mask(cfg) @@ -446,36 +482,35 @@ def optimizer(cfg, parameter_list): fluid.layers.piecewise_decay(boundaries=bd, values=lr), parameter_list=parameter_list, regularization=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=l2_weight_decay)) + regularization_coeff=l2_weight_decay + ), + ) return optimizer def fake_data_reader(args, mode='train'): - def iou_with_anchors(anchors_min, anchors_max, box_min, box_max): - """Compute jaccard score between a box and the anchors. - """ + """Compute jaccard score between a box and the anchors.""" len_anchors = anchors_max - anchors_min int_xmin = np.maximum(anchors_min, box_min) int_xmax = np.minimum(anchors_max, box_max) - inter_len = np.maximum(int_xmax - int_xmin, 0.) + inter_len = np.maximum(int_xmax - int_xmin, 0.0) union_len = len_anchors - inter_len + box_max - box_min jaccard = np.divide(inter_len, union_len) return jaccard def ioa_with_anchors(anchors_min, anchors_max, box_min, box_max): - """Compute intersection between score a box and the anchors. - """ + """Compute intersection between score a box and the anchors.""" len_anchors = anchors_max - anchors_min int_xmin = np.maximum(anchors_min, box_min) int_xmax = np.minimum(anchors_max, box_max) - inter_len = np.maximum(int_xmax - int_xmin, 0.) + inter_len = np.maximum(int_xmax - int_xmin, 0.0) scores = np.divide(inter_len, len_anchors) return scores def get_match_map(tscale): match_map = [] - tgap = 1. / tscale + tgap = 1.0 / tscale for idx in range(tscale): tmp_match_window = [] xmin = tgap * idx @@ -499,17 +534,21 @@ def fake_data_reader(args, mode='train'): gt_bbox = [] gt_iou_map = [] for idx in range(label_num): - duration = local_random.uniform(video_second * 0.4, - video_second * 0.8) - start_t = local_random.uniform(0.1 * video_second, - video_second - duration) + duration = local_random.uniform( + video_second * 0.4, video_second * 0.8 + ) + start_t = local_random.uniform( + 0.1 * video_second, video_second - duration + ) tmp_start = max(min(1, start_t / video_second), 0) tmp_end = max(min(1, (start_t + duration) / video_second), 0) gt_bbox.append([tmp_start, tmp_end]) - tmp_gt_iou_map = iou_with_anchors(match_map[:, 0], match_map[:, 1], - tmp_start, tmp_end) - tmp_gt_iou_map = np.reshape(tmp_gt_iou_map, - [args.dscale, args.tscale]) + tmp_gt_iou_map = iou_with_anchors( + match_map[:, 0], match_map[:, 1], tmp_start, tmp_end + ) + tmp_gt_iou_map = np.reshape( + tmp_gt_iou_map, [args.dscale, args.tscale] + ) gt_iou_map.append(tmp_gt_iou_map) gt_iou_map = np.array(gt_iou_map) gt_iou_map = np.max(gt_iou_map, axis=0) @@ -517,25 +556,38 @@ def fake_data_reader(args, mode='train'): gt_bbox = np.array(gt_bbox) gt_xmins = gt_bbox[:, 0] gt_xmaxs = gt_bbox[:, 1] - gt_len_small = 3. / args.tscale + gt_len_small = 3.0 / args.tscale gt_start_bboxs = np.stack( - (gt_xmins - gt_len_small / 2, gt_xmins + gt_len_small / 2), axis=1) + (gt_xmins - gt_len_small / 2, gt_xmins + gt_len_small / 2), axis=1 + ) gt_end_bboxs = np.stack( - (gt_xmaxs - gt_len_small / 2, gt_xmaxs + gt_len_small / 2), axis=1) + (gt_xmaxs - gt_len_small / 2, gt_xmaxs + gt_len_small / 2), axis=1 + ) match_score_start = [] for jdx in range(len(anchor_xmin)): match_score_start.append( np.max( - ioa_with_anchors(anchor_xmin[jdx], anchor_xmax[jdx], - gt_start_bboxs[:, 0], gt_start_bboxs[:, - 1]))) + ioa_with_anchors( + anchor_xmin[jdx], + anchor_xmax[jdx], + gt_start_bboxs[:, 0], + gt_start_bboxs[:, 1], + ) + ) + ) match_score_end = [] for jdx in range(len(anchor_xmin)): match_score_end.append( np.max( - ioa_with_anchors(anchor_xmin[jdx], anchor_xmax[jdx], - gt_end_bboxs[:, 0], gt_end_bboxs[:, 1]))) + ioa_with_anchors( + anchor_xmin[jdx], + anchor_xmax[jdx], + gt_end_bboxs[:, 0], + gt_end_bboxs[:, 1], + ) + ) + ) gt_start = np.array(match_score_start) gt_end = np.array(match_score_end) @@ -548,18 +600,22 @@ def fake_data_reader(args, mode='train'): for video_idx in range(iter_num): video_feat = local_random.random_sample( - [args.feat_dim, args.tscale]).astype('float32') + [args.feat_dim, args.tscale] + ).astype('float32') gt_iou_map, gt_start, gt_end = get_video_label( - match_map, anchor_xmin, anchor_xmax) + match_map, anchor_xmin, anchor_xmax + ) if mode == 'train' or mode == 'valid': batch_out.append((video_feat, gt_iou_map, gt_start, gt_end)) elif mode == 'test': batch_out.append( - (video_feat, gt_iou_map, gt_start, gt_end, video_idx)) + (video_feat, gt_iou_map, gt_start, gt_end, video_idx) + ) else: raise NotImplementedError( - 'mode {} not implemented'.format(mode)) + 'mode {} not implemented'.format(mode) + ) if len(batch_out) == args.batch_size: yield batch_out batch_out = [] @@ -589,20 +645,26 @@ def val_bmn(model, args): pred_bm, pred_start, pred_end = model(x_data) loss, tem_loss, pem_reg_loss, pem_cls_loss = bmn_loss_func( - pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, args) + pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, args + ) avg_loss = paddle.mean(loss) loss_data += [ avg_loss.numpy()[0], tem_loss.numpy()[0], pem_reg_loss.numpy()[0], - pem_cls_loss.numpy()[0] + pem_cls_loss.numpy()[0], ] - print('[VALID] iter {} '.format(batch_id) - + '\tLoss = {}, \ttem_loss = {}, \tpem_reg_loss = {}, \tpem_cls_loss = {}'.format( - '%f' % avg_loss.numpy()[0], '%f' % tem_loss.numpy()[0], \ - '%f' % pem_reg_loss.numpy()[0], '%f' % pem_cls_loss.numpy()[0])) + print( + '[VALID] iter {} '.format(batch_id) + + '\tLoss = {}, \ttem_loss = {}, \tpem_reg_loss = {}, \tpem_cls_loss = {}'.format( + '%f' % avg_loss.numpy()[0], + '%f' % tem_loss.numpy()[0], + '%f' % pem_reg_loss.numpy()[0], + '%f' % pem_cls_loss.numpy()[0], + ) + ) if batch_id == args.valid_batch_num: break @@ -610,11 +672,13 @@ def val_bmn(model, args): class TestTrain(unittest.TestCase): - def setUp(self): self.args = Args() - self.place = fluid.CPUPlace() if not fluid.is_compiled_with_cuda() \ + self.place = ( + fluid.CPUPlace() + if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0) + ) self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'inference') @@ -643,14 +707,18 @@ class TestTrain(unittest.TestCase): for epoch in range(args.epoch): for batch_id, data in enumerate(train_reader()): - video_feat = np.array([item[0] - for item in data]).astype(DATATYPE) - gt_iou_map = np.array([item[1] - for item in data]).astype(DATATYPE) - gt_start = np.array([item[2] - for item in data]).astype(DATATYPE) - gt_end = np.array([item[3] - for item in data]).astype(DATATYPE) + video_feat = np.array([item[0] for item in data]).astype( + DATATYPE + ) + gt_iou_map = np.array([item[1] for item in data]).astype( + DATATYPE + ) + gt_start = np.array([item[2] for item in data]).astype( + DATATYPE + ) + gt_end = np.array([item[3] for item in data]).astype( + DATATYPE + ) x_data = to_variable(video_feat) gt_iou_map = to_variable(gt_iou_map) @@ -663,8 +731,14 @@ class TestTrain(unittest.TestCase): pred_bm, pred_start, pred_end = bmn(x_data) loss, tem_loss, pem_reg_loss, pem_cls_loss = bmn_loss_func( - pred_bm, pred_start, pred_end, gt_iou_map, gt_start, - gt_end, args) + pred_bm, + pred_start, + pred_end, + gt_iou_map, + gt_start, + gt_end, + args, + ) avg_loss = paddle.mean(loss) avg_loss.backward() @@ -675,15 +749,21 @@ class TestTrain(unittest.TestCase): avg_loss.numpy()[0], tem_loss.numpy()[0], pem_reg_loss.numpy()[0], - pem_cls_loss.numpy()[0] + pem_cls_loss.numpy()[0], ] - if args.log_interval > 0 and (batch_id % args.log_interval - == 0): - print('[TRAIN] Epoch {}, iter {} '.format(epoch, batch_id) - + '\tLoss = {}, \ttem_loss = {}, \tpem_reg_loss = {}, \tpem_cls_loss = {}'.format( - '%f' % avg_loss.numpy()[0], '%f' % tem_loss.numpy()[0], \ - '%f' % pem_reg_loss.numpy()[0], '%f' % pem_cls_loss.numpy()[0])) + if args.log_interval > 0 and ( + batch_id % args.log_interval == 0 + ): + print( + '[TRAIN] Epoch {}, iter {} '.format(epoch, batch_id) + + '\tLoss = {}, \ttem_loss = {}, \tpem_reg_loss = {}, \tpem_cls_loss = {}'.format( + '%f' % avg_loss.numpy()[0], + '%f' % tem_loss.numpy()[0], + '%f' % pem_reg_loss.numpy()[0], + '%f' % pem_cls_loss.numpy()[0], + ) + ) # validation if batch_id % args.valid_interval == 0 and batch_id > 0: @@ -696,8 +776,9 @@ class TestTrain(unittest.TestCase): if to_static: fluid.dygraph.jit.save(bmn, self.model_save_prefix) else: - fluid.dygraph.save_dygraph(bmn.state_dict(), - self.dy_param_path) + fluid.dygraph.save_dygraph( + bmn.state_dict(), self.dy_param_path + ) break return np.array(loss_data) @@ -711,8 +792,10 @@ class TestTrain(unittest.TestCase): rtol=1e-05, err_msg='dygraph_res: {},\n static_res: {}'.format( dygraph_res[~np.isclose(dygraph_res, static_res)], - static_res[~np.isclose(dygraph_res, static_res)]), - atol=1e-8) + static_res[~np.isclose(dygraph_res, static_res)], + ), + atol=1e-8, + ) # Prediction needs trained models, so put `test_predict` at last of `test_train` self.verify_predict() @@ -729,32 +812,41 @@ class TestTrain(unittest.TestCase): predictor_pred_res = self.predict_analysis_inference(video_data) for dy_res, st_res, dy_jit_res, predictor_res in zip( - dygraph_pred_res, static_pred_res, dygraph_jit_pred_res, - predictor_pred_res): + dygraph_pred_res, + static_pred_res, + dygraph_jit_pred_res, + predictor_pred_res, + ): np.testing.assert_allclose( st_res, dy_res, rtol=1e-05, err_msg='dygraph_res: {},\n static_res: {}'.format( dy_res[~np.isclose(st_res, dy_res)], - st_res[~np.isclose(st_res, dy_res)]), - atol=1e-8) + st_res[~np.isclose(st_res, dy_res)], + ), + atol=1e-8, + ) np.testing.assert_allclose( st_res, dy_jit_res, rtol=1e-05, err_msg='dygraph_jit_res: {},\n static_res: {}'.format( dy_jit_res[~np.isclose(st_res, dy_jit_res)], - st_res[~np.isclose(st_res, dy_jit_res)]), - atol=1e-8) + st_res[~np.isclose(st_res, dy_jit_res)], + ), + atol=1e-8, + ) np.testing.assert_allclose( st_res, predictor_res, rtol=1e-05, err_msg='dygraph_jit_res: {},\n static_res: {}'.format( predictor_res[~np.isclose(st_res, predictor_res)], - st_res[~np.isclose(st_res, predictor_res)]), - atol=1e-8) + st_res[~np.isclose(st_res, predictor_res)], + ), + atol=1e-8, + ) break def predict_dygraph(self, data): @@ -776,14 +868,21 @@ class TestTrain(unittest.TestCase): paddle.enable_static() exe = fluid.Executor(self.place) # load inference model - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res @@ -799,8 +898,12 @@ class TestTrain(unittest.TestCase): return pred_res def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, [data]) + output = PredictorTools( + self.model_save_dir, + self.model_filename, + self.params_filename, + [data], + ) out = output() return out diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py index b5840bc45e3..45b149617d2 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py @@ -17,7 +17,9 @@ import numpy as np import paddle import paddle.fluid as fluid from paddle.fluid.dygraph.jit import declarative -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException SEED = 2020 @@ -25,7 +27,6 @@ np.random.seed(SEED) class TestDy2staticException(unittest.TestCase): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = None @@ -159,9 +160,7 @@ def test_for_in_else(x): def while_loop_class_var(x): - class Foo(object): - def __init__(self): self.a = 3 self.b = 4 @@ -205,11 +204,13 @@ def test_optim_break_in_while(x): class TestContinueInFor(unittest.TestCase): - def setUp(self): self.input = np.zeros((1)).astype('int64') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_dygraph_func() def init_dygraph_func(self): @@ -233,59 +234,52 @@ class TestContinueInFor(unittest.TestCase): static_res, rtol=1e-05, err_msg='dygraph res is {}\nstatic_res is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestContinueInForAtEnd(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_continue_in_for_at_end class TestBreakInFor(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_break_in_for class TestBreakInForAtEnd(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_break_in_for_at_end class TestBreakContinueInFor(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_break_continue_in_for class TestForInElse(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_for_in_else class TestContinueInWhile(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_continue_in_while class TestBreakInWhile(TestContinueInWhile): - def init_dygraph_func(self): self.dygraph_func = test_break_in_while class TestWhileLoopClassVar(TestContinueInWhile): - def init_dygraph_func(self): self.dygraph_func = while_loop_class_var class TestOptimBreakInFor(TestDy2staticException): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = test_optim_break_in_for @@ -293,7 +287,6 @@ class TestOptimBreakInFor(TestDy2staticException): class TestOptimBreakInWhile(TestContinueInWhile): - def init_dygraph_func(self): self.dygraph_func = test_optim_break_in_while diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py index f23d9ba783c..7df7c633312 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py @@ -26,7 +26,6 @@ from test_fetch_feed import Pool2D, Linear class TestCacheProgram(unittest.TestCase): - def setUp(self): self.batch_num = 5 self.dygraph_class = Pool2D @@ -44,26 +43,35 @@ class TestCacheProgram(unittest.TestCase): cur_out = out # Check forward ops prev_ops = cur_ops - cur_ops = Counter([ - op.type for op in fluid.default_main_program().block(0).ops - ]) + cur_ops = Counter( + [ + op.type + for op in fluid.default_main_program().block(0).ops + ] + ) if batch_id > 0: - prev_out_numpy = prev_out[0].numpy() if isinstance( - prev_out, (tuple, list)) else prev_out.numpy() - cur_out_numpy = cur_out[0].numpy() if isinstance( - cur_out, (tuple, list)) else cur_out.numpy() + prev_out_numpy = ( + prev_out[0].numpy() + if isinstance(prev_out, (tuple, list)) + else prev_out.numpy() + ) + cur_out_numpy = ( + cur_out[0].numpy() + if isinstance(cur_out, (tuple, list)) + else cur_out.numpy() + ) np.testing.assert_allclose( prev_out_numpy, cur_out_numpy, rtol=1e-05, - err_msg= - 'Output in previous batch is {}\n Output in current batch is \n{}' - .format(prev_out_numpy, cur_out_numpy)) + err_msg='Output in previous batch is {}\n Output in current batch is \n{}'.format( + prev_out_numpy, cur_out_numpy + ), + ) self.assertEqual(prev_ops, cur_ops) class TestCacheProgram2(TestCacheProgram): - def setUp(self): self.batch_num = 5 self.dygraph_class = Linear @@ -71,7 +79,6 @@ class TestCacheProgram2(TestCacheProgram): class TestCacheProgramWithOptimizer(unittest.TestCase): - def setUp(self): self.dygraph_class = Linear self.data = np.random.random((4, 10)).astype('float32') @@ -90,7 +97,8 @@ class TestCacheProgramWithOptimizer(unittest.TestCase): with fluid.dygraph.guard(fluid.CPUPlace()): dygraph_net = self.dygraph_class() adam = fluid.optimizer.AdamOptimizer( - learning_rate=0.001, parameter_list=dygraph_net.parameters()) + learning_rate=0.001, parameter_list=dygraph_net.parameters() + ) loss_data = [] for batch_id in range(self.batch_num): input = fluid.dygraph.to_variable(self.data) @@ -111,7 +119,9 @@ class TestCacheProgramWithOptimizer(unittest.TestCase): static_loss, rtol=1e-05, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) def simple_func(x): @@ -121,7 +131,6 @@ def simple_func(x): class TestConvertWithCache(unittest.TestCase): - def test_cache(self): static_func = convert_to_static(simple_func) # Get transformed function from cache. @@ -152,7 +161,6 @@ def sum_under_while(limit): class TestToOutputWithCache(unittest.TestCase): - def test_output(self): with fluid.dygraph.guard(): ret = sum_even_until_limit(80, 10) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cast.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cast.py index fc233ffbc02..17f598f0f17 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cast.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cast.py @@ -59,19 +59,23 @@ def test_mix_cast(x): class TestCastBase(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.prepare() self.set_func() def prepare(self): self.input_shape = (16, 32) self.input_dtype = 'float32' - self.input = np.random.binomial( - 4, 0.3, size=np.product(self.input_shape)).reshape( - self.input_shape).astype(self.input_dtype) + self.input = ( + np.random.binomial(4, 0.3, size=np.product(self.input_shape)) + .reshape(self.input_shape) + .astype(self.input_dtype) + ) self.cast_dtype = 'bool' def set_func(self): @@ -87,24 +91,29 @@ class TestCastBase(unittest.TestCase): self.assertTrue( res.dtype == self.cast_dtype, msg='The target dtype is {}, but the casted dtype is {}.'.format( - self.cast_dtype, res.dtype)) + self.cast_dtype, res.dtype + ), + ) ref_val = self.input.astype(self.cast_dtype) np.testing.assert_allclose( res, ref_val, rtol=1e-05, err_msg='The casted value is {}.\nThe correct value is {}.'.format( - res, ref_val)) + res, ref_val + ), + ) class TestIntCast(TestCastBase): - def prepare(self): - self.input_shape = (1, ) + self.input_shape = (1,) self.input_dtype = 'float32' - self.input = np.random.normal( - loc=6, scale=10, size=np.product(self.input_shape)).reshape( - self.input_shape).astype(self.input_dtype) + self.input = ( + np.random.normal(loc=6, scale=10, size=np.product(self.input_shape)) + .reshape(self.input_shape) + .astype(self.input_dtype) + ) self.cast_dtype = 'int32' def set_func(self): @@ -112,13 +121,14 @@ class TestIntCast(TestCastBase): class TestFloatCast(TestCastBase): - def prepare(self): self.input_shape = (8, 16) self.input_dtype = 'bool' - self.input = np.random.binomial( - 2, 0.5, size=np.product(self.input_shape)).reshape( - self.input_shape).astype(self.input_dtype) + self.input = ( + np.random.binomial(2, 0.5, size=np.product(self.input_shape)) + .reshape(self.input_shape) + .astype(self.input_dtype) + ) self.cast_dtype = 'float32' def set_func(self): @@ -126,13 +136,14 @@ class TestFloatCast(TestCastBase): class TestMixCast(TestCastBase): - def prepare(self): self.input_shape = (8, 32) self.input_dtype = 'float32' - self.input = np.random.normal( - loc=6, scale=10, size=np.product(self.input_shape)).reshape( - self.input_shape).astype(self.input_dtype) + self.input = ( + np.random.normal(loc=6, scale=10, size=np.product(self.input_shape)) + .reshape(self.input_shape) + .astype(self.input_dtype) + ) self.cast_int = 'int' self.cast_float = 'float32' self.cast_bool = 'bool' @@ -146,19 +157,26 @@ class TestMixCast(TestCastBase): self.assertTrue( res.dtype == self.cast_dtype, msg='The target dtype is {}, but the casted dtype is {}.'.format( - self.cast_dtype, res.dtype)) - ref_val = self.input.astype(self.cast_int).astype( - self.cast_float).astype(self.cast_bool).astype(self.cast_dtype) + self.cast_dtype, res.dtype + ), + ) + ref_val = ( + self.input.astype(self.cast_int) + .astype(self.cast_float) + .astype(self.cast_bool) + .astype(self.cast_dtype) + ) np.testing.assert_allclose( res, ref_val, rtol=1e-05, err_msg='The casted value is {}.\nThe correct value is {}.'.format( - res, ref_val)) + res, ref_val + ), + ) class TestNotVarCast(TestCastBase): - def prepare(self): self.input = 3.14 self.cast_dtype = 'int' @@ -173,7 +191,9 @@ class TestNotVarCast(TestCastBase): self.assertTrue( res == ref_val, msg='The casted value is {}.\nThe correct value is {}.'.format( - res, ref_val)) + res, ref_val + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py index 846576400d7..8712950e01b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py @@ -15,7 +15,9 @@ import unittest import paddle -from paddle.fluid.dygraph.dygraph_to_static.utils import FunctionNameLivenessAnalysis +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FunctionNameLivenessAnalysis, +) from paddle.utils import gast import inspect from numpy import append @@ -24,7 +26,6 @@ global_a = [] class JudgeVisitor(gast.NodeVisitor): - def __init__(self, ans, mod): self.ans = ans self.mod = mod @@ -34,27 +35,30 @@ class JudgeVisitor(gast.NodeVisitor): expected = self.ans.get(node.name, set()) exp_mod = self.mod.get(node.name, set()) assert scope.existed_vars() == expected, "Not Equals." - assert scope.modified_vars( - ) == exp_mod, "Not Equals in function:{} . expect {} , but get {}".format( - node.name, exp_mod, scope.modified_vars()) + assert ( + scope.modified_vars() == exp_mod + ), "Not Equals in function:{} . expect {} , but get {}".format( + node.name, exp_mod, scope.modified_vars() + ) self.generic_visit(node) class JudgePushPopVisitor(gast.NodeVisitor): - def __init__(self, push_pop_vars): self.pp_var = push_pop_vars def visit_FunctionDef(self, node): scope = node.pd_scope expected = self.pp_var.get(node.name, set()) - assert scope.push_pop_vars == expected, "Not Equals in function:{} . expect {} , but get {}".format( - node.name, expected, scope.push_pop_vars) + assert ( + scope.push_pop_vars == expected + ), "Not Equals in function:{} . expect {} , but get {}".format( + node.name, expected, scope.push_pop_vars + ) self.generic_visit(node) def test_normal_0(x): - def func(): if True: i = 1 @@ -103,8 +107,7 @@ def test_nonlocal(x, *args, **kargs): def test_push_pop_1(x, *args, **kargs): - """ push_pop_vars in main_function is : `l`, `k` - """ + """push_pop_vars in main_function is : `l`, `k`""" l = [] k = [] for i in range(10): @@ -114,8 +117,7 @@ def test_push_pop_1(x, *args, **kargs): def test_push_pop_2(x, *args, **kargs): - """ push_pop_vars in main_function is : `k` - """ + """push_pop_vars in main_function is : `k`""" l = [] k = [] @@ -128,10 +130,10 @@ def test_push_pop_2(x, *args, **kargs): def test_push_pop_3(x, *args, **kargs): - """ push_pop_vars in main_function is : `k` - NOTE: One may expect `k` and `l` because l - is nonlocal. Name bind analysis is - not implemented yet. + """push_pop_vars in main_function is : `k` + NOTE: One may expect `k` and `l` because l + is nonlocal. Name bind analysis is + not implemented yet. """ l = [] k = [] @@ -146,8 +148,7 @@ def test_push_pop_3(x, *args, **kargs): def test_push_pop_4(x, *args, **kargs): - """ push_pop_vars in main_function is : `k` - """ + """push_pop_vars in main_function is : `k`""" l = [] k = [] for i in range(10): @@ -160,20 +161,19 @@ def test_push_pop_4(x, *args, **kargs): class TestClosureAnalysis(unittest.TestCase): - def setUp(self): self.judge_type = "var and w_vars" self.init_dygraph_func() def init_dygraph_func(self): self.all_dygraph_funcs = [ - test_nonlocal, test_global, test_normal_0, test_normal_argument + test_nonlocal, + test_global, + test_normal_0, + test_normal_argument, ] self.answer = [ - { - 'func': set('k'), - 'test_nonlocal': set('i') - }, + {'func': set('k'), 'test_nonlocal': set('i')}, { 'func': set({'i'}), }, @@ -186,34 +186,27 @@ class TestClosureAnalysis(unittest.TestCase): ] self.modified_var = [ - { - 'func': set('ki'), - 'test_nonlocal': set('i') - }, - { - 'func': set({'i'}), - 'test_global': set({"t"}) - }, + {'func': set('ki'), 'test_nonlocal': set('i')}, + {'func': set({'i'}), 'test_global': set({"t"})}, { 'func': set('i'), }, - { - 'func': set('i'), - 'test_normal_argument': set('x') - }, + {'func': set('i'), 'test_normal_argument': set('x')}, ] def test_main(self): if self.judge_type == 'push_pop_vars': - for push_pop_vars, func in zip(self.push_pop_vars, - self.all_dygraph_funcs): + for push_pop_vars, func in zip( + self.push_pop_vars, self.all_dygraph_funcs + ): test_func = inspect.getsource(func) gast_root = gast.parse(test_func) name_visitor = FunctionNameLivenessAnalysis(gast_root) JudgePushPopVisitor(push_pop_vars).visit(gast_root) else: - for mod, ans, func in zip(self.modified_var, self.answer, - self.all_dygraph_funcs): + for mod, ans, func in zip( + self.modified_var, self.answer, self.all_dygraph_funcs + ): test_func = inspect.getsource(func) gast_root = gast.parse(test_func) name_visitor = FunctionNameLivenessAnalysis(gast_root) @@ -227,41 +220,48 @@ def TestClosureAnalysis_Attribute_func(): class TestClosureAnalysis_Attribute(TestClosureAnalysis): - def init_dygraph_func(self): self.all_dygraph_funcs = [TestClosureAnalysis_Attribute_func] self.answer = [{"TestClosureAnalysis_Attribute_func": set({'i'})}] - self.modified_var = [{ - "TestClosureAnalysis_Attribute_func": - set({'i', 'self.current.function'}) - }] + self.modified_var = [ + { + "TestClosureAnalysis_Attribute_func": set( + {'i', 'self.current.function'} + ) + } + ] class TestClosureAnalysis_PushPop(TestClosureAnalysis): - def init_dygraph_func(self): self.judge_type = "push_pop_vars" self.all_dygraph_funcs = [ - test_push_pop_1, test_push_pop_2, test_push_pop_3, test_push_pop_4 + test_push_pop_1, + test_push_pop_2, + test_push_pop_3, + test_push_pop_4, + ] + self.push_pop_vars = [ + { + "test_push_pop_1": set({'l', 'k'}), + }, + { + "test_push_pop_2": set({'k'}), + "func": set("l"), + }, + { + "test_push_pop_3": set({'k'}), + "func": set("l"), + }, + { + "test_push_pop_4": set({'k', 'l'}), + }, ] - self.push_pop_vars = [{ - "test_push_pop_1": set({'l', 'k'}), - }, { - "test_push_pop_2": set({'k'}), - "func": set("l"), - }, { - "test_push_pop_3": set({'k'}), - "func": set("l"), - }, { - "test_push_pop_4": set({'k', 'l'}), - }] class TestPushPopTrans(unittest.TestCase): - def test(self): - def vlist_of_dict(x): ma = {'a': []} for i in range(3): @@ -299,7 +299,6 @@ class TestPushPopTrans(unittest.TestCase): print(paddle.jit.to_static(vlist_of_dict)(x)) def test4(self): - def vlist_of_dict(x): a = np.array([1, 2, 3]) for i in range(3): @@ -311,7 +310,6 @@ class TestPushPopTrans(unittest.TestCase): print(paddle.jit.to_static(vlist_of_dict)(x)) def test5(self): - def vlist_of_dict(x): a = np.array([1, 2, 3]) for i in range(3): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py index 73f0262daa1..f22ca877616 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py @@ -20,14 +20,13 @@ import tempfile class BufferLayers(paddle.nn.Layer): - def __init__(self, out_channel): super(BufferLayers, self).__init__() self.out_channel = out_channel def forward(self, x): mean = paddle.mean(x) - if mean < 0.: + if mean < 0.0: x = x * self._mask() out = x - mean @@ -38,13 +37,13 @@ class BufferLayers(paddle.nn.Layer): class SequentialNet(paddle.nn.Layer): - def __init__(self, sub_layer, in_channel, out_channel): super(SequentialNet, self).__init__() self.layer = paddle.nn.Sequential( ('l1', paddle.nn.Linear(in_channel, in_channel)), ('l2', paddle.nn.Linear(in_channel, out_channel)), - ('l3', sub_layer(out_channel))) + ('l3', sub_layer(out_channel)), + ) def forward(self, x): out = self.layer(x) @@ -52,7 +51,6 @@ class SequentialNet(paddle.nn.Layer): class NestSequentialNet(paddle.nn.Layer): - def __init__(self): super().__init__() group1 = paddle.nn.Sequential( @@ -70,7 +68,6 @@ class NestSequentialNet(paddle.nn.Layer): class TestSequential(unittest.TestCase): - def setUp(self): paddle.set_device('cpu') self.seed = 2021 @@ -100,7 +97,8 @@ class TestSequential(unittest.TestCase): load_out, out, rtol=1e-05, - err_msg='load_out is {}\\st_out is {}'.format(load_out, out)) + err_msg='load_out is {}\\st_out is {}'.format(load_out, out), + ) return out @@ -113,7 +111,9 @@ class TestSequential(unittest.TestCase): st_out, rtol=1e-05, err_msg='dygraph_res is {}\nstatic_res is {}'.format( - dy_out, st_out)) + dy_out, st_out + ), + ) def _test_load(self, net, x): paddle.jit.save(net, self.model_path) @@ -123,11 +123,11 @@ class TestSequential(unittest.TestCase): class TestNestSequential(TestSequential): - def _init_config(self): self.net = NestSequentialNet() - self.model_path = os.path.join(self.temp_dir.name, - 'nested_sequential_net') + self.model_path = os.path.join( + self.temp_dir.name, 'nested_sequential_net' + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call.py index 2e0983ad5ee..241df1517e2 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call.py @@ -20,7 +20,9 @@ import numpy as np import paddle import paddle.fluid as fluid from paddle.fluid.dygraph import ProgramTranslator -from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import CONVERSION_OPTIONS +from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import ( + CONVERSION_OPTIONS, +) from test_program_translator import get_source_code import paddle.jit.dy2static as _jst @@ -64,7 +66,6 @@ def dyfunc_with_third_library_logging(x_v): class A: - @staticmethod def add(a, b): """ @@ -81,11 +82,13 @@ def dyfunc_with_staticmethod(x_v): class TestRecursiveCall1(unittest.TestCase): - def setUp(self): self.input = np.random.random([10, 16]).astype('float32') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_test_func() def init_test_func(self): @@ -111,24 +114,28 @@ class TestRecursiveCall1(unittest.TestCase): static_res, rtol=1e-05, err_msg='dygraph res is {}\nstatic_res is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) lambda_fun = lambda x: x class MyConvLayer(fluid.dygraph.Layer): - def __init__(self): super(MyConvLayer, self).__init__() self._conv = fluid.dygraph.Conv2D( num_channels=3, num_filters=2, filter_size=3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) @paddle.jit.to_static def forward(self, inputs): @@ -144,7 +151,6 @@ class MyConvLayer(fluid.dygraph.Layer): class MyLayer(fluid.dygraph.Layer): - def __init__(self): super(MyLayer, self).__init__() @@ -153,10 +159,13 @@ class MyLayer(fluid.dygraph.Layer): input_dim=5, output_dim=1, act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) @paddle.jit.to_static def forward(self, inputs): @@ -166,11 +175,13 @@ class MyLayer(fluid.dygraph.Layer): class TestRecursiveCall2(unittest.TestCase): - def setUp(self): self.input = np.random.random((1, 3, 3, 5)).astype('float32') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_func() def set_func(self): @@ -198,13 +209,11 @@ class TestRecursiveCall2(unittest.TestCase): class TestThirdPartyLibrary(TestRecursiveCall2): - def set_func(self): self.dygraph_func = dyfunc_with_third_library_logging class TestStaticMethod(TestRecursiveCall2): - def set_func(self): self.dygraph_func = dyfunc_with_staticmethod @@ -230,7 +239,6 @@ def func_convert_then_not_to_static(x): class TestClass(paddle.nn.Layer): - @paddle.jit.not_to_static def called_member(self, x): return paddle.sum(x) @@ -242,7 +250,6 @@ class TestClass(paddle.nn.Layer): class TestNotToConvert(TestRecursiveCall2): - def set_func(self): self.dygraph_func = func_not_to_static @@ -253,19 +260,16 @@ class TestNotToConvert(TestRecursiveCall2): class TestNotToConvert2(TestRecursiveCall2): - def set_func(self): self.dygraph_func = func_convert_then_not_to_static class TestNotToConvert3(TestRecursiveCall2): - def set_func(self): self.dygraph_func = TestClass() class TestDynamicToStaticCode(unittest.TestCase): - def setUp(self): self.set_func() self.set_answer_func() @@ -274,9 +278,7 @@ class TestDynamicToStaticCode(unittest.TestCase): self.func = func_not_to_static def set_answer_func(self): - - class StaticCode(): - + class StaticCode: @paddle.jit.not_to_static def func_not_to_static(x): res = func_sum(x) @@ -298,18 +300,17 @@ class TestDynamicToStaticCode(unittest.TestCase): answer_code, transformed_code, msg="\ntransformed_code : \n{}\nanswer_code : \n{}".format( - transformed_code, answer_code)) + transformed_code, answer_code + ), + ) class TestDynamicToStaticCode2(TestDynamicToStaticCode): - def set_func(self): self.func = func_convert_then_not_to_static def set_answer_func(self): - - class StaticCode(): - + class StaticCode: def func_convert_then_not_to_static(x): __return_value_0 = None y = _jst.Call(func_not_to_static)(x) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call_generator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call_generator.py index 9a417cfe585..a938d679158 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call_generator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call_generator.py @@ -24,14 +24,12 @@ def dyfunc_generator(): def main_func(): - """ Error will raise, but we only report a warning not intercept - """ + """Error will raise, but we only report a warning not intercept""" for i in dyfunc_generator(): print(i) class TestConvertGenerator(unittest.TestCase): - def test_raise_error(self): with self.assertRaises(Exception): to_static(main_func)() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_operators.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_operators.py index 65aec5ca8dd..284febe07cf 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_operators.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_operators.py @@ -18,14 +18,12 @@ import unittest class CallNotExist(paddle.nn.Layer): - def __call__(self): # call a non-exist API to trigger exception return paddle.nn.not_exist_api class ForwardNotExist(paddle.nn.Layer): - def forward(self): return 0 @@ -35,9 +33,7 @@ setattr(net, "forward", "A string so that convert forward will fail") class TestConvertCall(unittest.TestCase): - def test_class_exception(self): - @paddle.jit.to_static def call_not_exist(): net = CallNotExist() @@ -55,16 +51,17 @@ class TestConvertCall(unittest.TestCase): class TestConvertShapeCompare(unittest.TestCase): - def test_non_variable(self): - self.assertEqual(paddle.jit.dy2static.convert_shape_compare(1, "<", 2), - True) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "<=", 3), - True) + paddle.jit.dy2static.convert_shape_compare(1, "<", 2), True + ) + self.assertEqual( + paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "<=", 3), True + ) self.assertEqual( paddle.jit.dy2static.convert_shape_compare(1, ">", 2, "<=", 3), - False) + False, + ) def error_func(): """ @@ -73,77 +70,108 @@ class TestConvertShapeCompare(unittest.TestCase): raise ValueError("Used for test") self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, ">", 2, "<=", - lambda: error_func()), - False) + paddle.jit.dy2static.convert_shape_compare( + 1, ">", 2, "<=", lambda: error_func() + ), + False, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "in", - [1, 2, 3]), True) + paddle.jit.dy2static.convert_shape_compare( + 1, "<", 2, "in", [1, 2, 3] + ), + True, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "not in", - [1, 2, 3]), False) + paddle.jit.dy2static.convert_shape_compare( + 1, "<", 2, "not in", [1, 2, 3] + ), + False, + ) self.assertEqual( paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "is", 3), - False) + False, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "is not", - [1, 2, 3]), True) + paddle.jit.dy2static.convert_shape_compare( + 1, "<", 2, "is not", [1, 2, 3] + ), + True, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare([1, 2], "==", [1, 2], - "!=", [1, 2, 3]), True) + paddle.jit.dy2static.convert_shape_compare( + [1, 2], "==", [1, 2], "!=", [1, 2, 3] + ), + True, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare([1, 2], "!=", [1, 2, 3], - "==", [1, 2]), False) + paddle.jit.dy2static.convert_shape_compare( + [1, 2], "!=", [1, 2, 3], "==", [1, 2] + ), + False, + ) def test_variable(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.static.data(name='x', shape=[3, 2], dtype='float32') y = paddle.static.data(name='y', shape=[3, 2], dtype='float32') self.assertEqual( paddle.jit.dy2static.convert_shape_compare( - x, "is", x, "is not", y), True) + x, "is", x, "is not", y + ), + True, + ) self.assertEqual( paddle.jit.dy2static.convert_shape_compare( - x, "is not", x, "is not", y), False) + x, "is not", x, "is not", y + ), + False, + ) self.assertEqual( paddle.jit.dy2static.convert_shape_compare(x, "is", x, "is", y), - False) + False, + ) eq_out = paddle.jit.dy2static.convert_shape_compare(x, "==", y) not_eq_out = paddle.jit.dy2static.convert_shape_compare(x, "!=", y) long_eq_out = paddle.jit.dy2static.convert_shape_compare( - x, "==", x, "!=", y) - - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + x, "==", x, "!=", y + ) + + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - x_y_eq_out = exe.run(feed={ - "x": np.ones([3, 2]).astype(np.float32), - "y": np.ones([3, 2]).astype(np.float32) - }, - fetch_list=[eq_out, not_eq_out, long_eq_out]) - np.testing.assert_array_equal(np.array(x_y_eq_out), - np.array([[True], [False], [False]])) + x_y_eq_out = exe.run( + feed={ + "x": np.ones([3, 2]).astype(np.float32), + "y": np.ones([3, 2]).astype(np.float32), + }, + fetch_list=[eq_out, not_eq_out, long_eq_out], + ) + np.testing.assert_array_equal( + np.array(x_y_eq_out), np.array([[True], [False], [False]]) + ) set_a_zero = np.ones([3, 2]).astype(np.float32) set_a_zero[0][0] = 0.0 x_y_not_eq_out = exe.run( - feed={ - "x": np.ones([3, 2]).astype(np.float32), - "y": set_a_zero - }, - fetch_list=[eq_out, not_eq_out, long_eq_out]) - np.testing.assert_array_equal(np.array(x_y_not_eq_out), - np.array([[False], [True], [True]])) + feed={"x": np.ones([3, 2]).astype(np.float32), "y": set_a_zero}, + fetch_list=[eq_out, not_eq_out, long_eq_out], + ) + np.testing.assert_array_equal( + np.array(x_y_not_eq_out), np.array([[False], [True], [True]]) + ) paddle.disable_static() class ShapeLayer(paddle.nn.Layer): - def __init__(self): super(ShapeLayer, self).__init__() @@ -158,7 +186,6 @@ class ShapeLayer(paddle.nn.Layer): class TestChooseShapeAttrOrApiWithLayer(unittest.TestCase): - def test_tensor_shape(self): x = paddle.zeros(shape=[4, 1], dtype='float32') net = ShapeLayer() @@ -168,7 +195,6 @@ class TestChooseShapeAttrOrApiWithLayer(unittest.TestCase): class TestIfElseNoValue(unittest.TestCase): - def test_else_ret_none(self): input_x = paddle.to_tensor([[1, 2, 3], [4, 5, 6]]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cpu_cuda_to_tensor.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cpu_cuda_to_tensor.py index 53149f758ba..60f7f70ad4b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cpu_cuda_to_tensor.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cpu_cuda_to_tensor.py @@ -18,9 +18,7 @@ import numpy as np class TestCpuCuda(unittest.TestCase): - def test_cpu_cuda(self): - def func(x): x = paddle.to_tensor([1, 2, 3, 4]) x = x.cuda() @@ -33,9 +31,7 @@ class TestCpuCuda(unittest.TestCase): class TestToTensor(unittest.TestCase): - def test_to_tensor_with_variable_list(self): - def func(x): ones = paddle.to_tensor([1]) twos = paddle.to_tensor([2]) @@ -44,15 +40,15 @@ class TestToTensor(unittest.TestCase): x = paddle.to_tensor([3]) print(paddle.jit.to_static(func).code) - np.testing.assert_allclose(paddle.jit.to_static(func)(x).numpy(), - np.array([1, 2, 3, 4]), - rtol=1e-05) + np.testing.assert_allclose( + paddle.jit.to_static(func)(x).numpy(), + np.array([1, 2, 3, 4]), + rtol=1e-05, + ) class TestToTensor1(unittest.TestCase): - def test_to_tensor_with_variable_list(self): - def func(x): ones = paddle.to_tensor([1]) twos = paddle.to_tensor([2]) @@ -64,24 +60,26 @@ class TestToTensor1(unittest.TestCase): x = paddle.to_tensor([3]) print(paddle.jit.to_static(func).code) - np.testing.assert_allclose(paddle.jit.to_static(func)(x).numpy(), - np.array([1, 2, 3, 4]), - rtol=1e-05) + np.testing.assert_allclose( + paddle.jit.to_static(func)(x).numpy(), + np.array([1, 2, 3, 4]), + rtol=1e-05, + ) class TestToTensor2(unittest.TestCase): - def test_to_tensor_with_variable_list(self): - def func(x): x = paddle.to_tensor([[1], [2], [3], [4]]) return x x = paddle.to_tensor([3]) print(paddle.jit.to_static(func).code) - np.testing.assert_allclose(paddle.jit.to_static(func)(x).numpy(), - np.array([[1], [2], [3], [4]]), - rtol=1e-05) + np.testing.assert_allclose( + paddle.jit.to_static(func)(x).numpy(), + np.array([[1], [2], [3], [4]]), + rtol=1e-05, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py index c5dd0ac064b..03db8935079 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py @@ -30,6 +30,7 @@ import numpy as np from PIL import Image, ImageOps import os + # Use GPU:0 to elimate the influence of other tasks. os.environ["CUDA_VISIBLE_DEVICES"] = "1" @@ -61,19 +62,22 @@ program_translator = ProgramTranslator() class Cycle_Gan(fluid.dygraph.Layer): - def __init__(self, input_channel, istrain=True): super(Cycle_Gan, self).__init__() self.build_generator_resnet_9blocks_a = build_generator_resnet_9blocks( - input_channel) + input_channel + ) self.build_generator_resnet_9blocks_b = build_generator_resnet_9blocks( - input_channel) + input_channel + ) if istrain: self.build_gen_discriminator_a = build_gen_discriminator( - input_channel) + input_channel + ) self.build_gen_discriminator_b = build_gen_discriminator( - input_channel) + input_channel + ) @declarative def forward(self, input_A, input_B): @@ -86,9 +90,11 @@ class Cycle_Gan(fluid.dygraph.Layer): cyc_B = self.build_generator_resnet_9blocks_a(fake_A) diff_A = fluid.layers.abs( - fluid.layers.elementwise_sub(x=input_A, y=cyc_A)) + fluid.layers.elementwise_sub(x=input_A, y=cyc_A) + ) diff_B = fluid.layers.abs( - fluid.layers.elementwise_sub(x=input_B, y=cyc_B)) + fluid.layers.elementwise_sub(x=input_B, y=cyc_B) + ) cyc_A_loss = fluid.layers.reduce_mean(diff_A) * lambda_A cyc_B_loss = fluid.layers.reduce_mean(diff_B) * lambda_B cyc_loss = cyc_A_loss + cyc_B_loss @@ -100,17 +106,41 @@ class Cycle_Gan(fluid.dygraph.Layer): g_B_loss = fluid.layers.reduce_mean(fluid.layers.square(fake_rec_B - 1)) G = g_A_loss + g_B_loss idt_A = self.build_generator_resnet_9blocks_a(input_B) - idt_loss_A = fluid.layers.reduce_mean( - fluid.layers.abs(fluid.layers.elementwise_sub( - x=input_B, y=idt_A))) * lambda_B * lambda_identity + idt_loss_A = ( + fluid.layers.reduce_mean( + fluid.layers.abs( + fluid.layers.elementwise_sub(x=input_B, y=idt_A) + ) + ) + * lambda_B + * lambda_identity + ) idt_B = self.build_generator_resnet_9blocks_b(input_A) - idt_loss_B = fluid.layers.reduce_mean( - fluid.layers.abs(fluid.layers.elementwise_sub( - x=input_A, y=idt_B))) * lambda_A * lambda_identity + idt_loss_B = ( + fluid.layers.reduce_mean( + fluid.layers.abs( + fluid.layers.elementwise_sub(x=input_A, y=idt_B) + ) + ) + * lambda_A + * lambda_identity + ) idt_loss = fluid.layers.elementwise_add(idt_loss_A, idt_loss_B) g_loss = cyc_loss + G + idt_loss - return fake_A, fake_B, cyc_A, cyc_B, g_A_loss, g_B_loss, idt_loss_A, idt_loss_B, cyc_A_loss, cyc_B_loss, g_loss + return ( + fake_A, + fake_B, + cyc_A, + cyc_B, + g_A_loss, + g_B_loss, + idt_loss_A, + idt_loss_B, + cyc_A_loss, + cyc_B_loss, + g_loss, + ) @declarative def discriminatorA(self, input_A, input_B): @@ -134,23 +164,26 @@ class Cycle_Gan(fluid.dygraph.Layer): class build_resnet_block(fluid.dygraph.Layer): - def __init__(self, dim, use_bias=False): super(build_resnet_block, self).__init__() - self.conv0 = conv2d(num_channels=dim, - num_filters=dim, - filter_size=3, - stride=1, - stddev=0.02, - use_bias=False) - self.conv1 = conv2d(num_channels=dim, - num_filters=dim, - filter_size=3, - stride=1, - stddev=0.02, - relu=False, - use_bias=False) + self.conv0 = conv2d( + num_channels=dim, + num_filters=dim, + filter_size=3, + stride=1, + stddev=0.02, + use_bias=False, + ) + self.conv1 = conv2d( + num_channels=dim, + num_filters=dim, + filter_size=3, + stride=1, + stddev=0.02, + relu=False, + use_bias=False, + ) self.dim = dim def forward(self, inputs): @@ -163,33 +196,39 @@ class build_resnet_block(fluid.dygraph.Layer): class build_generator_resnet_9blocks(fluid.dygraph.Layer): - def __init__(self, input_channel): super(build_generator_resnet_9blocks, self).__init__() - self.conv0 = conv2d(num_channels=input_channel, - num_filters=32, - filter_size=7, - stride=1, - padding=0, - stddev=0.02) - self.conv1 = conv2d(num_channels=32, - num_filters=64, - filter_size=3, - stride=2, - padding=1, - stddev=0.02) - self.conv2 = conv2d(num_channels=64, - num_filters=128, - filter_size=3, - stride=2, - padding=1, - stddev=0.02) + self.conv0 = conv2d( + num_channels=input_channel, + num_filters=32, + filter_size=7, + stride=1, + padding=0, + stddev=0.02, + ) + self.conv1 = conv2d( + num_channels=32, + num_filters=64, + filter_size=3, + stride=2, + padding=1, + stddev=0.02, + ) + self.conv2 = conv2d( + num_channels=64, + num_filters=128, + filter_size=3, + stride=2, + padding=1, + stddev=0.02, + ) self.build_resnet_block_list = [] dim = 128 for i in range(9): - Build_Resnet_Block = self.add_sublayer("generator_%d" % (i + 1), - build_resnet_block(dim)) + Build_Resnet_Block = self.add_sublayer( + "generator_%d" % (i + 1), build_resnet_block(dim) + ) self.build_resnet_block_list.append(Build_Resnet_Block) self.deconv0 = DeConv2D( num_channels=dim, @@ -200,22 +239,26 @@ class build_generator_resnet_9blocks(fluid.dygraph.Layer): padding=[1, 1], outpadding=[0, 1, 0, 1], ) - self.deconv1 = DeConv2D(num_channels=32 * 2, - num_filters=32, - filter_size=3, - stride=2, - stddev=0.02, - padding=[1, 1], - outpadding=[0, 1, 0, 1]) - self.conv3 = conv2d(num_channels=32, - num_filters=input_channel, - filter_size=7, - stride=1, - stddev=0.02, - padding=0, - relu=False, - norm=False, - use_bias=True) + self.deconv1 = DeConv2D( + num_channels=32 * 2, + num_filters=32, + filter_size=3, + stride=2, + stddev=0.02, + padding=[1, 1], + outpadding=[0, 1, 0, 1], + ) + self.conv3 = conv2d( + num_channels=32, + num_filters=input_channel, + filter_size=7, + stride=1, + stddev=0.02, + padding=0, + relu=False, + norm=False, + use_bias=True, + ) def forward(self, inputs): pad_input = fluid.layers.pad2d(inputs, [3, 3, 3, 3], mode="reflect") @@ -233,49 +276,58 @@ class build_generator_resnet_9blocks(fluid.dygraph.Layer): class build_gen_discriminator(fluid.dygraph.Layer): - def __init__(self, input_channel): super(build_gen_discriminator, self).__init__() - self.conv0 = conv2d(num_channels=input_channel, - num_filters=64, - filter_size=4, - stride=2, - stddev=0.02, - padding=1, - norm=False, - use_bias=True, - relufactor=0.2) - self.conv1 = conv2d(num_channels=64, - num_filters=128, - filter_size=4, - stride=2, - stddev=0.02, - padding=1, - relufactor=0.2) - self.conv2 = conv2d(num_channels=128, - num_filters=IMAGE_SIZE, - filter_size=4, - stride=2, - stddev=0.02, - padding=1, - relufactor=0.2) - self.conv3 = conv2d(num_channels=IMAGE_SIZE, - num_filters=512, - filter_size=4, - stride=1, - stddev=0.02, - padding=1, - relufactor=0.2) - self.conv4 = conv2d(num_channels=512, - num_filters=1, - filter_size=4, - stride=1, - stddev=0.02, - padding=1, - norm=False, - relu=False, - use_bias=True) + self.conv0 = conv2d( + num_channels=input_channel, + num_filters=64, + filter_size=4, + stride=2, + stddev=0.02, + padding=1, + norm=False, + use_bias=True, + relufactor=0.2, + ) + self.conv1 = conv2d( + num_channels=64, + num_filters=128, + filter_size=4, + stride=2, + stddev=0.02, + padding=1, + relufactor=0.2, + ) + self.conv2 = conv2d( + num_channels=128, + num_filters=IMAGE_SIZE, + filter_size=4, + stride=2, + stddev=0.02, + padding=1, + relufactor=0.2, + ) + self.conv3 = conv2d( + num_channels=IMAGE_SIZE, + num_filters=512, + filter_size=4, + stride=1, + stddev=0.02, + padding=1, + relufactor=0.2, + ) + self.conv4 = conv2d( + num_channels=512, + num_filters=1, + filter_size=4, + stride=1, + stddev=0.02, + padding=1, + norm=False, + relu=False, + use_bias=True, + ) def forward(self, inputs): y = self.conv0(inputs) @@ -289,35 +341,42 @@ class build_gen_discriminator(fluid.dygraph.Layer): class conv2d(fluid.dygraph.Layer): """docstring for Conv2D""" - def __init__(self, - num_channels, - num_filters=64, - filter_size=7, - stride=1, - stddev=0.02, - padding=0, - norm=True, - relu=True, - relufactor=0.0, - use_bias=False): + def __init__( + self, + num_channels, + num_filters=64, + filter_size=7, + stride=1, + stddev=0.02, + padding=0, + norm=True, + relu=True, + relufactor=0.0, + use_bias=False, + ): super(conv2d, self).__init__() if use_bias == False: con_bias_attr = False else: con_bias_attr = fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.0)) - - self.conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=padding, - use_cudnn=use_cudnn, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=stddev)), - bias_attr=con_bias_attr) + initializer=fluid.initializer.Constant(0.0) + ) + + self.conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + use_cudnn=use_cudnn, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=stddev + ) + ), + bias_attr=con_bias_attr, + ) # Note(Aurelius84): The calculation of GPU kernel in BN is non-deterministic, # failure rate is 1/100 in Dev but seems incremental in CE platform. # If on GPU, we disable BN temporarily. @@ -328,10 +387,13 @@ class conv2d(fluid.dygraph.Layer): use_global_stats=True, # set True to use deterministic algorithm num_channels=num_filters, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.NormalInitializer(1.0, 0.02)), + initializer=fluid.initializer.NormalInitializer(1.0, 0.02) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.0)), - trainable_statistics=True) + initializer=fluid.initializer.Constant(0.0) + ), + trainable_statistics=True, + ) self.relufactor = relufactor self.use_bias = use_bias @@ -348,26 +410,28 @@ class conv2d(fluid.dygraph.Layer): class DeConv2D(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters=64, - filter_size=7, - stride=1, - stddev=0.02, - padding=[0, 0], - outpadding=[0, 0, 0, 0], - relu=True, - norm=True, - relufactor=0.0, - use_bias=False): + def __init__( + self, + num_channels, + num_filters=64, + filter_size=7, + stride=1, + stddev=0.02, + padding=[0, 0], + outpadding=[0, 0, 0, 0], + relu=True, + norm=True, + relufactor=0.0, + use_bias=False, + ): super(DeConv2D, self).__init__() if use_bias == False: de_bias_attr = False else: de_bias_attr = fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.0)) + initializer=fluid.initializer.Constant(0.0) + ) self._deconv = Conv2DTranspose( num_channels, @@ -377,9 +441,12 @@ class DeConv2D(fluid.dygraph.Layer): padding=padding, use_cudnn=use_cudnn, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.NormalInitializer(loc=0.0, - scale=stddev)), - bias_attr=de_bias_attr) + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=stddev + ) + ), + bias_attr=de_bias_attr, + ) if fluid.is_compiled_with_cuda(): norm = False if norm: @@ -387,10 +454,13 @@ class DeConv2D(fluid.dygraph.Layer): use_global_stats=True, # set True to use deterministic algorithm num_channels=num_filters, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.NormalInitializer(1.0, 0.02)), + initializer=fluid.initializer.NormalInitializer(1.0, 0.02) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.0)), - trainable_statistics=True) + initializer=fluid.initializer.Constant(0.0) + ), + trainable_statistics=True, + ) self.outpadding = outpadding self.relufactor = relufactor @@ -400,10 +470,9 @@ class DeConv2D(fluid.dygraph.Layer): def forward(self, inputs): conv = self._deconv(inputs) - conv = fluid.layers.pad2d(conv, - paddings=self.outpadding, - mode='constant', - pad_value=0.0) + conv = fluid.layers.pad2d( + conv, paddings=self.outpadding, mode='constant', pad_value=0.0 + ) if self.norm: conv = self.bn(conv) @@ -413,7 +482,6 @@ class DeConv2D(fluid.dygraph.Layer): class ImagePool(object): - def __init__(self, pool_size=50): self.pool = [] self.count = 0 @@ -436,11 +504,11 @@ class ImagePool(object): def reader_creater(): - def reader(): while True: fake_image = np.uint8( - np.random.random((IMAGE_SIZE + 30, IMAGE_SIZE + 30, 3)) * 255) + np.random.random((IMAGE_SIZE + 30, IMAGE_SIZE + 30, 3)) * 255 + ) image = Image.fromarray(fake_image) # Resize image = image.resize((286, 286), Image.BICUBIC) @@ -474,20 +542,29 @@ class Args(object): def optimizer_setting(parameters): lr = 0.0002 - optimizer = fluid.optimizer.Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=[ - 100 * step_per_epoch, 120 * step_per_epoch, 140 * step_per_epoch, - 160 * step_per_epoch, 180 * step_per_epoch - ], - values=[lr, lr * 0.8, lr * 0.6, lr * 0.4, lr * 0.2, lr * 0.1]), - parameter_list=parameters, - beta1=0.5) + optimizer = fluid.optimizer.Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=[ + 100 * step_per_epoch, + 120 * step_per_epoch, + 140 * step_per_epoch, + 160 * step_per_epoch, + 180 * step_per_epoch, + ], + values=[lr, lr * 0.8, lr * 0.6, lr * 0.4, lr * 0.2, lr * 0.1], + ), + parameter_list=parameters, + beta1=0.5, + ) return optimizer def train(args, to_static): - place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + ) program_translator.enable(to_static) @@ -507,8 +584,10 @@ def train(args, to_static): cycle_gan = Cycle_Gan(input_channel=data_shape[1], istrain=True) t_time = 0 - vars_G = cycle_gan.build_generator_resnet_9blocks_a.parameters( - ) + cycle_gan.build_generator_resnet_9blocks_b.parameters() + vars_G = ( + cycle_gan.build_generator_resnet_9blocks_a.parameters() + + cycle_gan.build_generator_resnet_9blocks_b.parameters() + ) vars_da = cycle_gan.build_gen_discriminator_a.parameters() vars_db = cycle_gan.build_gen_discriminator_b.parameters() @@ -525,17 +604,28 @@ def train(args, to_static): s_time = time.time() data_A = np.array( - [data_A[0].reshape(3, IMAGE_SIZE, - IMAGE_SIZE)]).astype("float32") + [data_A[0].reshape(3, IMAGE_SIZE, IMAGE_SIZE)] + ).astype("float32") data_B = np.array( - [data_B[0].reshape(3, IMAGE_SIZE, - IMAGE_SIZE)]).astype("float32") + [data_B[0].reshape(3, IMAGE_SIZE, IMAGE_SIZE)] + ).astype("float32") data_A = to_variable(data_A) data_B = to_variable(data_B) # optimize the g_A network - fake_A, fake_B, cyc_A, cyc_B, g_A_loss, g_B_loss, idt_loss_A, idt_loss_B, cyc_A_loss, cyc_B_loss, g_loss = cycle_gan( - data_A, data_B) + ( + fake_A, + fake_B, + cyc_A, + cyc_B, + g_A_loss, + g_B_loss, + idt_loss_A, + idt_loss_B, + cyc_A_loss, + cyc_B_loss, + g_loss, + ) = cycle_gan(data_A, data_B) g_loss.backward() optimizer1.minimize(g_loss) @@ -543,21 +633,24 @@ def train(args, to_static): fake_pool_B = B_pool.pool_image(fake_B).numpy() fake_pool_B = np.array( - [fake_pool_B[0].reshape(3, IMAGE_SIZE, - IMAGE_SIZE)]).astype("float32") + [fake_pool_B[0].reshape(3, IMAGE_SIZE, IMAGE_SIZE)] + ).astype("float32") fake_pool_B = to_variable(fake_pool_B) fake_pool_A = A_pool.pool_image(fake_A).numpy() fake_pool_A = np.array( - [fake_pool_A[0].reshape(3, IMAGE_SIZE, - IMAGE_SIZE)]).astype("float32") + [fake_pool_A[0].reshape(3, IMAGE_SIZE, IMAGE_SIZE)] + ).astype("float32") fake_pool_A = to_variable(fake_pool_A) # optimize the d_A network rec_B, fake_pool_rec_B = cycle_gan.discriminatorA( - data_B, fake_pool_B) - d_loss_A = (fluid.layers.square(fake_pool_rec_B) + - fluid.layers.square(rec_B - 1)) / 2.0 + data_B, fake_pool_B + ) + d_loss_A = ( + fluid.layers.square(fake_pool_rec_B) + + fluid.layers.square(rec_B - 1) + ) / 2.0 d_loss_A = fluid.layers.reduce_mean(d_loss_A) d_loss_A.backward() @@ -566,9 +659,12 @@ def train(args, to_static): # optimize the d_B network rec_A, fake_pool_rec_A = cycle_gan.discriminatorB( - data_A, fake_pool_A) - d_loss_B = (fluid.layers.square(fake_pool_rec_A) + - fluid.layers.square(rec_A - 1)) / 2.0 + data_A, fake_pool_A + ) + d_loss_B = ( + fluid.layers.square(fake_pool_rec_A) + + fluid.layers.square(rec_A - 1) + ) / 2.0 d_loss_B = fluid.layers.reduce_mean(d_loss_B) d_loss_B.backward() @@ -578,8 +674,15 @@ def train(args, to_static): # Log generator loss and discriminator loss cur_batch_loss = [ - g_loss, d_loss_A, d_loss_B, g_A_loss, cyc_A_loss, - idt_loss_A, g_B_loss, cyc_B_loss, idt_loss_B + g_loss, + d_loss_A, + d_loss_B, + g_A_loss, + cyc_A_loss, + idt_loss_A, + g_B_loss, + cyc_B_loss, + idt_loss_B, ] cur_batch_loss = [x.numpy()[0] for x in cur_batch_loss] @@ -587,8 +690,10 @@ def train(args, to_static): t_time += batch_time if batch_id % args.log_step == 0: print( - "batch: {}\t Batch_time_cost: {}\n g_loss: {}\t d_A_loss: {}\t d_B_loss:{}\n g_A_loss: {}\t g_A_cyc_loss: {}\t g_A_idt_loss: {}\n g_B_loss: {}\t g_B_cyc_loss: {}\t g_B_idt_loss: {}" - .format(batch_id, batch_time, *cur_batch_loss)) + "batch: {}\t Batch_time_cost: {}\n g_loss: {}\t d_A_loss: {}\t d_B_loss:{}\n g_A_loss: {}\t g_A_cyc_loss: {}\t g_A_idt_loss: {}\n g_B_loss: {}\t g_B_cyc_loss: {}\t g_B_idt_loss: {}".format( + batch_id, batch_time, *cur_batch_loss + ) + ) if batch_id > args.train_step: break @@ -598,7 +703,6 @@ def train(args, to_static): class TestCycleGANModel(unittest.TestCase): - def setUp(self): self.args = Args() @@ -617,8 +721,10 @@ class TestCycleGANModel(unittest.TestCase): if not fluid.is_compiled_with_cuda(): assert_func = np.array_equal - self.assertTrue(assert_func(dy_out, st_out), - msg="dy_out:\n {}\n st_out:\n{}".format(dy_out, st_out)) + self.assertTrue( + assert_func(dy_out, st_out), + msg="dy_out:\n {}\n st_out:\n{}".format(dy_out, st_out), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py index 947cfa35d43..6ea76d75925 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py @@ -19,8 +19,17 @@ import tempfile import paddle import paddle.fluid as fluid from paddle.static import InputSpec -from paddle.fluid.dygraph import to_variable, declarative, ProgramTranslator, Layer, jit -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ConcreteProgram, StaticFunction +from paddle.fluid.dygraph import ( + to_variable, + declarative, + ProgramTranslator, + Layer, + jit, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ConcreteProgram, + StaticFunction, +) from test_basic_api_transformation import dyfunc_to_variable @@ -28,7 +37,6 @@ program_trans = ProgramTranslator() class SimpleNet(Layer): - def __init__(self): super(SimpleNet, self).__init__() self.linear = fluid.dygraph.Linear(10, 3) @@ -55,10 +63,9 @@ class SimpleNet(Layer): z = z + int_val return z - @declarative(input_spec=[{ - 'x': InputSpec([None, 10]), - 'y': InputSpec([None, 10]) - }]) + @declarative( + input_spec=[{'x': InputSpec([None, 10]), 'y': InputSpec([None, 10])}] + ) def func_with_dict(self, d): x = d['x'] y = d['y'] @@ -66,12 +73,14 @@ class SimpleNet(Layer): return z - @declarative(input_spec=[[ - InputSpec([None]), { - 'x': InputSpec([None, 10]), - 'y': InputSpec([None, 10]) - } - ]]) + @declarative( + input_spec=[ + [ + InputSpec([None]), + {'x': InputSpec([None, 10]), 'y': InputSpec([None, 10])}, + ] + ] + ) def func_with_list_dict(self, dl): bias = dl[0] x = dl[1]['x'] @@ -84,7 +93,6 @@ class SimpleNet(Layer): class TestStaticFunctionInstance(unittest.TestCase): - def test_instance_same_class(self): with fluid.dygraph.guard(fluid.CPUPlace()): net_1 = SimpleNet() @@ -102,7 +110,6 @@ class TestStaticFunctionInstance(unittest.TestCase): class TestInputSpec(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_path = os.path.join(self.temp_dir.name, 'simple_net') @@ -114,7 +121,7 @@ class TestInputSpec(unittest.TestCase): with fluid.dygraph.guard(fluid.CPUPlace()): x = to_variable(np.ones([4, 10]).astype('float32')) y = to_variable(np.ones([4, 10]).astype('float32') * 2) - int_val = 4. + int_val = 4.0 net = SimpleNet() @@ -150,7 +157,7 @@ class TestInputSpec(unittest.TestCase): with fluid.dygraph.guard(fluid.CPUPlace()): x = to_variable(np.ones([4, 10]).astype('float32')) y = to_variable(np.ones([4, 10]).astype('float32') * 2) - int_val = 4. + int_val = 4.0 net = SimpleNet() @@ -160,26 +167,28 @@ class TestInputSpec(unittest.TestCase): # 2. requires len(input_spec) <= len(args) with self.assertRaises(ValueError): - net.add_func = declarative(net.add_func, - input_spec=[ - InputSpec([-1, 10]), - InputSpec([-1, 10]), - InputSpec([10]) - ]) + net.add_func = declarative( + net.add_func, + input_spec=[ + InputSpec([-1, 10]), + InputSpec([-1, 10]), + InputSpec([10]), + ], + ) net.add_func(x, y) def test_concrete_program(self): with fluid.dygraph.guard(fluid.CPUPlace()): x = to_variable(np.ones([4, 10]).astype('float32')) y = to_variable(np.ones([4, 10]).astype('float32') * 2) - int_val = 4. + int_val = 4.0 net = SimpleNet() # We can get concrete_program by specificing InputSpec information. Faking input is no need. net.add_func = declarative( net.add_func, - input_spec=[InputSpec([-1, 10]), - InputSpec([-1, 10], name='y')]) + input_spec=[InputSpec([-1, 10]), InputSpec([-1, 10], name='y')], + ) cp1 = net.add_func.concrete_program self.assertTrue(cp1.inputs[-1].shape == (-1, 10)) self.assertTrue(cp1.inputs[-1].name == 'y') @@ -187,10 +196,10 @@ class TestInputSpec(unittest.TestCase): # generate another program net.add_func = declarative( net.add_func, - input_spec=[InputSpec([10]), - InputSpec([10], name='label')]) + input_spec=[InputSpec([10]), InputSpec([10], name='label')], + ) cp2 = net.add_func.concrete_program - self.assertTrue(cp2.inputs[-1].shape == (10, )) + self.assertTrue(cp2.inputs[-1].shape == (10,)) self.assertTrue(cp2.inputs[-1].name == 'label') # Note(Aurelius84): New instance will be returned if we use `declarative(foo)` every time. # So number of cache program is 1. @@ -204,7 +213,6 @@ def foo_func(a, b, c=1, d=2): class TestDifferentInputSpecCacheProgram(unittest.TestCase): - def setUp(self): program_trans.enable(True) @@ -218,33 +226,33 @@ class TestDifferentInputSpecCacheProgram(unittest.TestCase): # [16, 10] + [10] (varbase) out_1 = foo(to_variable(x_data), to_variable(y_data)) - np.testing.assert_allclose(x_data + y_data, - out_1.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_data + y_data, out_1.numpy(), rtol=1e-05 + ) self.assertTrue(len(foo.program_cache) == 1) self.assertTrue(len(foo.program_cache.concrete_programs()) == 1) first_program = foo.program_cache.last() # [16, 10] + [10] (numpy) out_2 = foo(to_variable(x_data), y_data) - np.testing.assert_allclose(x_data + y_data, - out_2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_data + y_data, out_2.numpy(), rtol=1e-05 + ) self.assertTrue(len(foo.program_cache) == 1) # [16, 10] + [10] (numpy) out_3 = foo(to_variable(x_data), z_data) - np.testing.assert_allclose(x_data + z_data, - out_3.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_data + z_data, out_3.numpy(), rtol=1e-05 + ) # hit cache program self.assertTrue(len(foo.program_cache) == 1) # [16, 10] + [10] (numpy) with other different arguments (c=3) out_4 = foo(to_variable(x_data), z_data, 3) - np.testing.assert_allclose(x_data + z_data, - out_4.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_data + z_data, out_4.numpy(), rtol=1e-05 + ) # create a new program self.assertTrue(len(foo.program_cache) == 2) @@ -258,26 +266,29 @@ class TestDifferentInputSpecCacheProgram(unittest.TestCase): foo = declarative(foo_func) # 1. specific InputSpec for `x`/`y` - concrete_program_1 = foo.get_concrete_program(InputSpec([None, 10]), - InputSpec([10])) + concrete_program_1 = foo.get_concrete_program( + InputSpec([None, 10]), InputSpec([10]) + ) self.assertTrue(len(foo.program_cache) == 1) # 2. specific `c`/`d` explicitly with same default value - concrete_program_2 = foo.get_concrete_program(InputSpec([None, 10]), - InputSpec([10]), 1, 2) + concrete_program_2 = foo.get_concrete_program( + InputSpec([None, 10]), InputSpec([10]), 1, 2 + ) self.assertTrue(concrete_program_2 == concrete_program_1) self.assertTrue(len(foo.program_cache) == 1) # 3. specific `c` = 2 - concrete_program_3 = foo.get_concrete_program(InputSpec([None, 10]), - InputSpec([10]), - c=2) + concrete_program_3 = foo.get_concrete_program( + InputSpec([None, 10]), InputSpec([10]), c=2 + ) self.assertTrue(concrete_program_3 != concrete_program_1) self.assertTrue(len(foo.program_cache) == 2) # 4. specific x.shape = [10] - concrete_program_4 = foo.get_concrete_program(InputSpec([10]), - InputSpec([10])) + concrete_program_4 = foo.get_concrete_program( + InputSpec([10]), InputSpec([10]) + ) self.assertTrue(concrete_program_4 != concrete_program_1) self.assertTrue(len(foo.program_cache) == 3) @@ -287,19 +298,21 @@ class TestDifferentInputSpecCacheProgram(unittest.TestCase): # 6. specific unknown kwargs `e`=4 with self.assertRaises(TypeError): - concrete_program_5 = foo.get_concrete_program(InputSpec([10]), - InputSpec([10]), - e=4) + concrete_program_5 = foo.get_concrete_program( + InputSpec([10]), InputSpec([10]), e=4 + ) def test_concrete_program(self): with fluid.dygraph.guard(fluid.CPUPlace()): # usage 1 - foo_1 = paddle.jit.to_static(foo_func, - input_spec=[ - InputSpec([10], name='x'), - InputSpec([10], name='y') - ]) + foo_1 = paddle.jit.to_static( + foo_func, + input_spec=[ + InputSpec([10], name='x'), + InputSpec([10], name='y'), + ], + ) self.assertTrue(isinstance(foo_1.concrete_program, ConcreteProgram)) # usage 2 @@ -314,7 +327,6 @@ class TestDifferentInputSpecCacheProgram(unittest.TestCase): class TestInputDefaultName(unittest.TestCase): - def setUp(self): paddle.disable_static() self.net = SimpleNet() @@ -339,7 +351,6 @@ class TestInputDefaultName(unittest.TestCase): class TestDeclarativeAPI(unittest.TestCase): - def test_error(self): func = declarative(dyfunc_to_variable) @@ -358,7 +369,6 @@ class TestDeclarativeAPI(unittest.TestCase): class TestDecorateModelDirectly(unittest.TestCase): - def setUp(self): paddle.disable_static() program_trans.enable(True) @@ -385,27 +395,28 @@ class TestDecorateModelDirectly(unittest.TestCase): class TestErrorWithInitFromStaticMode(unittest.TestCase): - def test_raise_error(self): # disable imperative paddle.enable_static() net = SimpleNet() - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): net.forward.concrete_program - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): net.forward.inputs - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): net.forward.outputs class CallNonForwardFuncNet(paddle.nn.Layer): - def __init__(self): super(CallNonForwardFuncNet, self).__init__() self.sub = CallNonForwardFuncSubNet() @@ -416,7 +427,6 @@ class CallNonForwardFuncNet(paddle.nn.Layer): class CallNonForwardFuncSubNet(paddle.nn.Layer): - def __init__(self): super(CallNonForwardFuncSubNet, self).__init__() self.a = paddle.to_tensor([1, 2]) @@ -427,7 +437,6 @@ class CallNonForwardFuncSubNet(paddle.nn.Layer): class TestCallNonForwardFunc(unittest.TestCase): - def test_call_non_forward(self): paddle.disable_static() net = CallNonForwardFuncNet() @@ -437,7 +446,6 @@ class TestCallNonForwardFunc(unittest.TestCase): class SetBuffersNet1(paddle.nn.Layer): - def __init__(self): super(SetBuffersNet1, self).__init__() self.a = paddle.to_tensor([1]) @@ -449,7 +457,6 @@ class SetBuffersNet1(paddle.nn.Layer): class SetBuffersNet2(paddle.nn.Layer): - def __init__(self): super(SetBuffersNet2, self).__init__() self.b = paddle.to_tensor([2]) @@ -462,7 +469,6 @@ class SetBuffersNet2(paddle.nn.Layer): class TestSetBuffers(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_path = os.path.join(self.temp_dir.name, 'SetBuffersNet1') @@ -487,13 +493,11 @@ class TestSetBuffers(unittest.TestCase): class ClassNoInheritLayer: - def func(self, x): return x + 1 class TestClassNoInheritLayer(unittest.TestCase): - def test_to_static(self): paddle.disable_static() net = ClassNoInheritLayer() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_decorator_transform.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_decorator_transform.py index bda090fc4c9..2b73a1075e5 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_decorator_transform.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_decorator_transform.py @@ -22,12 +22,11 @@ from contextlib import contextmanager def deco1(func): - @wraps(func) def inner(*args, **kwargs): print('in deco1, added 1') _x = 2 - if (_x < 1): + if _x < 1: _x += 1 else: _x -= 1 @@ -39,7 +38,6 @@ def deco1(func): def deco2(fun): - @wraps(fun) def inner(*args, **kwargs): print('in deco2, added 2') @@ -51,9 +49,7 @@ def deco2(fun): def deco3(x=3): - def inner_deco(func): - @wraps(func) def inner(*args, **kwargs): print('in deco3, added {}'.format(x)) @@ -67,9 +63,7 @@ def deco3(x=3): def deco4(func=None, x=0): - def decorated(pyfunc): - @wraps(pyfunc) def inner_deco(*args, **kwargs): print('in deco4, added {}'.format(x)) @@ -187,7 +181,6 @@ def deco_with_paddle_api(): class TestDecoratorTransform(unittest.TestCase): - def test_deco_transform(self): outs = forward() np.testing.assert_allclose(outs[0], np.array(3), rtol=1e-05) @@ -206,9 +199,11 @@ class TestDecoratorTransform(unittest.TestCase): warn1() flag = False for warn in w: - if (issubclass(warn.category, UserWarning) - ) and "A context manager decorator is used" in str( - warn.message): + if ( + issubclass(warn.category, UserWarning) + ) and "A context manager decorator is used" in str( + warn.message + ): flag = True break self.assertTrue(flag) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_deepcopy.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_deepcopy.py index ecb2d97fa44..a4823be6430 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_deepcopy.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_deepcopy.py @@ -15,14 +15,15 @@ import unittest import paddle import numpy as np -from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + StaticFunction, +) from test_rollback import Net, foo from copy import deepcopy class TestDeepCopy(unittest.TestCase): - def test_net(self): net = Net() net = paddle.jit.to_static(net) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py index f6b71eb7265..fcb986045b8 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py @@ -18,32 +18,41 @@ import unittest import paddle import paddle.fluid as fluid from paddle.jit import to_static -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) -PLACE = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +PLACE = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) class SubNetWithDict(fluid.dygraph.Layer): - def __init__(self, hidden_size=16, output_size=16): super(SubNetWithDict, self).__init__() - init_weight = lambda x: fluid.ParamAttr(initializer=fluid.initializer. - Constant(x)) - - self.q_fc = fluid.dygraph.Linear(input_dim=hidden_size, - output_dim=output_size, - bias_attr=False, - param_attr=init_weight(0.6)) - self.k_fc = fluid.dygraph.Linear(input_dim=hidden_size, - output_dim=output_size, - bias_attr=False, - param_attr=init_weight(0.5)) - self.v_fc = fluid.dygraph.Linear(input_dim=hidden_size, - output_dim=output_size, - bias_attr=False, - param_attr=init_weight(0.2)) + init_weight = lambda x: fluid.ParamAttr( + initializer=fluid.initializer.Constant(x) + ) + + self.q_fc = fluid.dygraph.Linear( + input_dim=hidden_size, + output_dim=output_size, + bias_attr=False, + param_attr=init_weight(0.6), + ) + self.k_fc = fluid.dygraph.Linear( + input_dim=hidden_size, + output_dim=output_size, + bias_attr=False, + param_attr=init_weight(0.5), + ) + self.v_fc = fluid.dygraph.Linear( + input_dim=hidden_size, + output_dim=output_size, + bias_attr=False, + param_attr=init_weight(0.2), + ) def forward(self, input, cache=None): input = fluid.dygraph.to_variable(input) @@ -66,7 +75,6 @@ class SubNetWithDict(fluid.dygraph.Layer): class MainNetWithDict(fluid.dygraph.Layer): - def __init__(self, batch_size=64, hidden_size=16, output_size=16): super(MainNetWithDict, self).__init__() self.batch_size = batch_size @@ -78,16 +86,16 @@ class MainNetWithDict(fluid.dygraph.Layer): def forward(self, input, max_len=4): input = fluid.dygraph.to_variable(input) cache = { - "k": - fluid.layers.fill_constant( + "k": fluid.layers.fill_constant( shape=[self.batch_size, self.output_size], dtype='float32', - value=0), - "v": - fluid.layers.fill_constant( + value=0, + ), + "v": fluid.layers.fill_constant( shape=[self.batch_size, self.output_size], dtype='float32', - value=0), + value=0, + ), } # TODO(Aurelius84): The following code will be converted into: # max_len = layers.cond(layers.shape(input)[0] != max_len, @@ -165,11 +173,13 @@ def test_dic_pop_2(x): class TestDictPop(unittest.TestCase): - def setUp(self): self.input = np.random.random((3)).astype('int32') - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self._set_test_func() def _set_test_func(self): @@ -197,17 +207,17 @@ class TestDictPop(unittest.TestCase): static_res, rtol=1e-05, err_msg='dygraph result is {}\nstatic result is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestDictPop2(TestDictPop): - def _set_test_func(self): self.dygraph_func = test_dic_pop_2 class NetWithDictPop(paddle.nn.Layer): - def __init__(self): super(NetWithDictPop, self).__init__() @@ -224,7 +234,6 @@ class NetWithDictPop(paddle.nn.Layer): class TestDictPop3(TestNetWithDict): - def setUp(self): self.x = np.array([2, 2]).astype('float32') @@ -240,15 +249,16 @@ class TestDictPop3(TestNetWithDict): dygraph_result = self._run_dygraph() static_result = self._run_static() - self.assertTrue((dygraph_result == static_result).all(), - msg="dygraph result: {}\nstatic result: {}".format( - dygraph_result, static_result)) + self.assertTrue( + (dygraph_result == static_result).all(), + msg="dygraph result: {}\nstatic result: {}".format( + dygraph_result, static_result + ), + ) class TestDictCmpInFor(unittest.TestCase): - def test_with_for(self): - def func(): pos = [1, 3] neg = [-1, -3] @@ -257,15 +267,14 @@ class TestDictCmpInFor(unittest.TestCase): for (x, y) in zip(pos, neg): val = x - y dict_val.update( - {k: val + dict_val[k] - for k, v in dict_val.items()}) + {k: val + dict_val[k] for k, v in dict_val.items()} + ) return dict_val self.assertEqual(paddle.jit.to_static(func)()['minus'], 8) def test_with_for_enumerate(self): - def func(): pos = [1, 3] neg = [-1, -3] @@ -274,8 +283,8 @@ class TestDictCmpInFor(unittest.TestCase): for i, (x, y) in enumerate(zip(pos, neg)): val = x - y dict_val.update( - {k: val + dict_val[k] - for k, v in dict_val.items()}) + {k: val + dict_val[k] for k, v in dict_val.items()} + ) return dict_val diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_drop_path.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_drop_path.py index 201da70b017..c5eb3d7567e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_drop_path.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_drop_path.py @@ -26,7 +26,6 @@ def drop_path(x, training=False): class DropPath(paddle.nn.Layer): - def __init__(self): super(DropPath, self).__init__() @@ -36,7 +35,6 @@ class DropPath(paddle.nn.Layer): class TestTrainEval(unittest.TestCase): - def setUp(self): self.model = DropPath() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_duplicate_output.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_duplicate_output.py index 9534d1dc615..625051d9364 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_duplicate_output.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_duplicate_output.py @@ -26,14 +26,12 @@ else: class SimpleNet(paddle.nn.Layer): - def __init__(self): super().__init__() self._linear = paddle.nn.Linear(1, 1) def forward(self, x): - """ forward with duplicate outputs. - """ + """forward with duplicate outputs.""" x = self._linear(x) return x, x diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py index 4eada80aec8..6e7389cc132 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py @@ -23,13 +23,11 @@ SEED = 2020 class Pool2D(fluid.dygraph.Layer): - def __init__(self): super(Pool2D, self).__init__() - self.pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='avg', - pool_stride=1, - global_pooling=False) + self.pool2d = fluid.dygraph.Pool2D( + pool_size=2, pool_type='avg', pool_stride=1, global_pooling=False + ) @declarative def forward(self, x): @@ -42,17 +40,19 @@ class Pool2D(fluid.dygraph.Layer): class Linear(fluid.dygraph.Layer): - def __init__(self, input_dim=10, output_dim=5): super(Linear, self).__init__() self.fc = fluid.dygraph.Linear( input_dim, output_dim, act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) @declarative def forward(self, x): @@ -62,7 +62,6 @@ class Linear(fluid.dygraph.Layer): class TestPool2D(unittest.TestCase): - def setUp(self): self.dygraph_class = Pool2D self.data = np.random.random((1, 2, 4, 4)).astype('float32') @@ -95,11 +94,12 @@ class TestPool2D(unittest.TestCase): static_res, rtol=1e-05, err_msg='dygraph_res is {}\n static_res is \n{}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestLinear(TestPool2D): - def setUp(self): self.dygraph_class = Linear self.data = np.random.random((4, 10)).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_for_enumerate.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_for_enumerate.py index 6b399b5e9be..ff4e3e84330 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_for_enumerate.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_for_enumerate.py @@ -290,7 +290,6 @@ def for_tuple_as_enumerate_value(x_array): # 20. test for function in a class class ForwardContainsForLayer(paddle.nn.Layer): - def __init__(self): super(ForwardContainsForLayer, self).__init__() self.high = 5 @@ -326,8 +325,8 @@ def for_original_tuple(): # 23. for zip error @paddle.jit.to_static( - input_spec=[InputSpec(shape=[None, 10]), - InputSpec(shape=[None, 10])]) + input_spec=[InputSpec(shape=[None, 10]), InputSpec(shape=[None, 10])] +) def for_zip_error(x, y): for i, j in zip(x, y): a = i + j @@ -336,8 +335,8 @@ def for_zip_error(x, y): # 24. for zip @paddle.jit.to_static( - input_spec=[InputSpec(shape=[2, 10]), - InputSpec(shape=[2, 10])]) + input_spec=[InputSpec(shape=[2, 10]), InputSpec(shape=[2, 10])] +) def for_zip(x, y): for i, j in zip(x, y): a = i + j @@ -345,10 +344,12 @@ def for_zip(x, y): class TestTransformBase(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_input() self.set_test_func() @@ -357,7 +358,8 @@ class TestTransformBase(unittest.TestCase): def set_test_func(self): raise NotImplementedError( - "For Enumerate test should implement set_test_func") + "For Enumerate test should implement set_test_func" + ) def _run(self, to_static): program_translator.enable(to_static) @@ -372,22 +374,20 @@ class TestTransformBase(unittest.TestCase): class TestTransform(TestTransformBase): - def transformed_result_compare(self): dy_outs = self.get_dygraph_output() if not isinstance(dy_outs, (tuple, list)): - dy_outs = (dy_outs, ) + dy_outs = (dy_outs,) st_outs = self.get_static_output() if not isinstance(st_outs, (tuple, list)): - st_outs = (st_outs, ) + st_outs = (st_outs,) for x, y in zip(dy_outs, st_outs): np.testing.assert_allclose(x.numpy(), y.numpy(), rtol=1e-05) class TestTransformForOriginalList(TestTransform): - def _run(self, to_static): program_translator.enable(to_static) with fluid.dygraph.guard(): @@ -395,7 +395,6 @@ class TestTransformForOriginalList(TestTransform): class TestTransformError(TestTransformBase): - def transformed_error(self, etype): with self.assertRaises(etype): dy_out = self.get_dygraph_output() @@ -403,7 +402,6 @@ class TestTransformError(TestTransformBase): class TestForInRange(TestTransform): - def set_input(self): self.input = np.array([5]) @@ -415,7 +413,6 @@ class TestForInRange(TestTransform): class TestForIterList(TestTransform): - def set_test_func(self): self.dygraph_func = for_iter_list @@ -424,19 +421,16 @@ class TestForIterList(TestTransform): class TestForEnumerateSimple(TestForIterList): - def set_test_func(self): self.dygraph_func = for_enumerate_list class TestForInRangeWithBreak(TestForInRange): - def set_test_func(self): self.dygraph_func = for_in_range_with_break class TestForIterVarNumpy(TestTransform): - def set_input(self): self.input = np.array([1, 2, 3, 4, 5]) @@ -448,103 +442,86 @@ class TestForIterVarNumpy(TestTransform): class TestForEnumerateVarNumpy(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy class TestForEnumerateVarNumpyWithStart(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_start class TestForEnumerateVarNumpyWithBreak(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_break class TestForEnumerateVarNumpyWithContinue(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_continue class TestForEnumerateVarNumpyWithStartAndBreak(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_start_break class TestForEnumerateVarNumpyWithStartAndContinue(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_start_continue class TestForIterVar(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_iter_var class TestForIterVarIdx(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_iter_var_idx class TestForEnumerateVar(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var class TestForEnumerateVarWithNestedRange(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_with_nested_range class TestForIterVarList(TestForInRange): - def set_test_func(self): self.dygraph_func = for_iter_var_list class TestForEnumerateVarList(TestForInRange): - def set_test_func(self): self.dygraph_func = for_enumerate_var_list class TestForTupleAsIterVar(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_tuple_as_iter_var class TestForTupleAsEnumerateIter(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_tuple_as_enumerate_iter class TestForTupleAsEnumerateValue(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_tuple_as_enumerate_value class TestForwardContainsForLayer(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = ForwardContainsForLayer() class TestForOriginalList(TestTransformForOriginalList): - def set_test_func(self): self.dygraph_func = for_original_list @@ -553,7 +530,6 @@ class TestForOriginalList(TestTransformForOriginalList): class TestForOriginalTuple(TestTransformForOriginalList): - def set_test_func(self): self.dygraph_func = for_original_tuple @@ -562,7 +538,6 @@ class TestForOriginalTuple(TestTransformForOriginalList): class TestForZip(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py index 7a79eaf28db..a245b607f88 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py @@ -45,7 +45,6 @@ def decorated_call_decorated(x): class DoubleDecorated(object): - @classmethod @declarative def double_decorated_func1(self, x): @@ -58,20 +57,19 @@ class DoubleDecorated(object): class TestFullNameDecorator(unittest.TestCase): - def test_run_success(self): x = np.ones([1, 2]).astype("float32") answer = np.zeros([1, 2]).astype("float32") with fluid.dygraph.guard(): - np.testing.assert_allclose(dygraph_decorated_func(x).numpy(), - answer, - rtol=1e-05) - np.testing.assert_allclose(jit_decorated_func(x).numpy(), - answer, - rtol=1e-05) - np.testing.assert_allclose(decorated_call_decorated(x).numpy(), - answer, - rtol=1e-05) + np.testing.assert_allclose( + dygraph_decorated_func(x).numpy(), answer, rtol=1e-05 + ) + np.testing.assert_allclose( + jit_decorated_func(x).numpy(), answer, rtol=1e-05 + ) + np.testing.assert_allclose( + decorated_call_decorated(x).numpy(), answer, rtol=1e-05 + ) with self.assertRaises(NotImplementedError): DoubleDecorated().double_decorated_func1(x) with self.assertRaises(NotImplementedError): @@ -79,12 +77,13 @@ class TestFullNameDecorator(unittest.TestCase): class TestImportProgramTranslator(unittest.TestCase): - def test_diff_pkg_same_cls(self): dygraph_prog_trans = fluid.dygraph.ProgramTranslator() - dy_to_stat_prog_trans = fluid.dygraph.dygraph_to_static.ProgramTranslator( + dy_to_stat_prog_trans = ( + fluid.dygraph.dygraph_to_static.ProgramTranslator() ) - full_pkg_prog_trans = fluid.dygraph.dygraph_to_static.program_translator.ProgramTranslator( + full_pkg_prog_trans = ( + fluid.dygraph.dygraph_to_static.program_translator.ProgramTranslator() ) self.assertEqual(dygraph_prog_trans, dy_to_stat_prog_trans) self.assertEqual(dygraph_prog_trans, full_pkg_prog_trans) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py index 9fdb6e7c6d3..37bc3f5dc12 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestFunctionSpec(unittest.TestCase): - def test_constructor(self): foo_spec = FunctionSpec(foo_func) args_name = foo_spec.args_name @@ -51,11 +50,9 @@ class TestFunctionSpec(unittest.TestCase): self.assertTrue(len(kwargs) == 0) # case 2: foo(a=10, b=20, d=4) - args, kwargs = foo_spec.unified_args_and_kwargs([], { - 'a': 10, - 'b': 20, - 'd': 4 - }) + args, kwargs = foo_spec.unified_args_and_kwargs( + [], {'a': 10, 'b': 20, 'd': 4} + ) self.assertTupleEqual(args, (10, 20, 1, 4)) self.assertTrue(len(kwargs) == 0) @@ -83,7 +80,8 @@ class TestFunctionSpec(unittest.TestCase): # case 1 foo_spec = FunctionSpec(foo_func, input_spec=[a_spec, b_spec]) input_with_spec, _ = foo_spec.args_to_input_spec( - (a_tensor, b_tensor, 1, 2), {}) + (a_tensor, b_tensor, 1, 2), {} + ) self.assertTrue(len(input_with_spec) == 4) self.assertTrue(input_with_spec[0] == a_spec) # a @@ -93,8 +91,9 @@ class TestFunctionSpec(unittest.TestCase): # case 2 foo_spec = FunctionSpec(foo_func, input_spec=[a_spec]) - input_with_spec, _ = foo_spec.args_to_input_spec((a_tensor, b_tensor), - {}) + input_with_spec, _ = foo_spec.args_to_input_spec( + (a_tensor, b_tensor), {} + ) self.assertTrue(len(input_with_spec) == 2) self.assertTrue(input_with_spec[0] == a_spec) # a self.assertTupleEqual(input_with_spec[1].shape, (4, 10)) # b.shape @@ -104,14 +103,15 @@ class TestFunctionSpec(unittest.TestCase): # assert kwargs is None if set `input_spec` foo_spec = FunctionSpec(foo_func, input_spec=[a_spec]) with self.assertRaises(ValueError): - input_with_spec = foo_spec.args_to_input_spec((a_tensor, b_tensor), - {'c': 4}) + input_with_spec = foo_spec.args_to_input_spec( + (a_tensor, b_tensor), {'c': 4} + ) # case 4 # assert len(args) >= len(self._input_spec) foo_spec = FunctionSpec(foo_func, input_spec=[a_spec, b_spec]) with self.assertRaises(ValueError): - input_with_spec = foo_spec.args_to_input_spec((a_tensor, ), {}) + input_with_spec = foo_spec.args_to_input_spec((a_tensor,), {}) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grad.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grad.py index ae484bd72c6..38d93e3fba3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grad.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grad.py @@ -20,7 +20,6 @@ import tempfile class GradLayer(paddle.nn.Layer): - def __init__(self): super(GradLayer, self).__init__() @@ -33,7 +32,6 @@ class GradLayer(paddle.nn.Layer): class GradLinearLayer(paddle.nn.Layer): - def __init__(self): super(GradLinearLayer, self).__init__() self.linear = paddle.nn.Linear(5, 5, bias_attr=False) @@ -45,15 +43,13 @@ class GradLinearLayer(paddle.nn.Layer): for i in range(10): tmp = self.linear(tmp) out = tmp - dx = paddle.grad([out], [x], - None, - create_graph=True, - allow_unused=False)[0] + dx = paddle.grad( + [out], [x], None, create_graph=True, allow_unused=False + )[0] return dx class NoGradLinearLayer(paddle.nn.Layer): - def __init__(self): super(NoGradLinearLayer, self).__init__() self.linear = paddle.nn.Linear(5, 5, bias_attr=False) @@ -70,7 +66,6 @@ class NoGradLinearLayer(paddle.nn.Layer): class TestGrad(unittest.TestCase): - def setUp(self): self.func = GradLayer() self.x = paddle.ones(shape=[10, 2, 5], dtype='float32') @@ -90,17 +85,18 @@ class TestGrad(unittest.TestCase): class TestGradLinear(TestGrad): - def setUp(self): self.func = GradLinearLayer() self.x = paddle.ones(shape=[10, 2, 5], dtype='float32') self.x.stop_gradient = False self.temp_dir = tempfile.TemporaryDirectory() - self.infer_model_path = os.path.join(self.temp_dir.name, - 'double_grad_infer_model') - self.train_model_path = os.path.join(self.temp_dir.name, - 'double_grad_train_model') + self.infer_model_path = os.path.join( + self.temp_dir.name, 'double_grad_infer_model' + ) + self.train_model_path = os.path.join( + self.temp_dir.name, 'double_grad_train_model' + ) def tearDown(self): self.temp_dir.cleanup() @@ -118,9 +114,11 @@ class TestGradLinear(TestGrad): def test_save_train_program(self): grad_clip = paddle.nn.ClipGradByGlobalNorm(2.0) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - grad_clip=grad_clip, - parameters=self.func.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, + grad_clip=grad_clip, + parameters=self.func.parameters(), + ) for i in range(10): out = self.func(self.x) avg_loss = paddle.mean(paddle.abs(out - 1)) @@ -138,17 +136,18 @@ class TestGradLinear(TestGrad): class TestNoGradLinear(TestGradLinear): - def setUp(self): self.func = NoGradLinearLayer() self.x = paddle.ones(shape=[10, 2, 5], dtype='float32') self.x.stop_gradient = False self.temp_dir = tempfile.TemporaryDirectory() - self.infer_model_path = os.path.join(self.temp_dir.name, - 'no_grad_infer_model') - self.train_model_path = os.path.join(self.temp_dir.name, - 'no_grad_train_model') + self.infer_model_path = os.path.join( + self.temp_dir.name, 'no_grad_infer_model' + ) + self.train_model_path = os.path.join( + self.temp_dir.name, 'no_grad_train_model' + ) def tearDown(self): self.temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_gradient_aggregation.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_gradient_aggregation.py index 67232aa9a83..b30338ce0e4 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_gradient_aggregation.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_gradient_aggregation.py @@ -22,7 +22,6 @@ np.random.seed(SEED) class SimpleNet(paddle.nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.linear1 = paddle.nn.Linear(10, 3) @@ -32,25 +31,30 @@ class SimpleNet(paddle.nn.Layer): out1 = self.linear1(x) out2 = self.linear2(out1) return [out1, out2] # 梯度为0 - #return [out1] # 梯度正常 - #return [out2, out1] # 梯度正常 + # return [out1] # 梯度正常 + # return [out2, out1] # 梯度正常 class TestGradientAggregationInDy2Static(unittest.TestCase): - def test_to_static(self): - def simplenet_grad(inp, to_static=False): net = SimpleNet() - if to_static: net = paddle.jit.to_static(net) + if to_static: + net = paddle.jit.to_static(net) loss = net(inp) loss[0].backward() return net.linear1.weight.grad - inp = paddle.to_tensor(np.random.randn(10, )).astype("float32") - np.testing.assert_allclose(simplenet_grad(inp, True).numpy(), - simplenet_grad(inp, False).numpy(), - rtol=1e-05) + inp = paddle.to_tensor( + np.random.randn( + 10, + ) + ).astype("float32") + np.testing.assert_allclose( + simplenet_grad(inp, True).numpy(), + simplenet_grad(inp, False).numpy(), + rtol=1e-05, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grid_generator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grid_generator.py index caf349ecf8e..3f80130737f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grid_generator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grid_generator.py @@ -23,7 +23,6 @@ paddle.seed(2020) class GridGenerator(nn.Layer): - def __init__(self, in_channels, num_fiducial): super(GridGenerator, self).__init__() self.eps = 1e-6 @@ -32,15 +31,16 @@ class GridGenerator(nn.Layer): initializer = nn.initializer.Constant(value=0.0) param_attr = ParamAttr(learning_rate=0.0, initializer=initializer) bias_attr = ParamAttr(learning_rate=0.0, initializer=initializer) - self.fc = nn.Linear(in_channels, - 6, - weight_attr=param_attr, - bias_attr=bias_attr) - - @paddle.jit.to_static(input_spec=[ - paddle.static.InputSpec(shape=[None, 3, 32, 100], dtype='float32'), - paddle.static.InputSpec(shape=[32, 100], dtype='float32') - ]) + self.fc = nn.Linear( + in_channels, 6, weight_attr=param_attr, bias_attr=bias_attr + ) + + @paddle.jit.to_static( + input_spec=[ + paddle.static.InputSpec(shape=[None, 3, 32, 100], dtype='float32'), + paddle.static.InputSpec(shape=[32, 100], dtype='float32'), + ] + ) def forward(self, batch_C_prime, I_r_size): """ Generate the grid for the grid_sampler. @@ -54,7 +54,7 @@ class GridGenerator(nn.Layer): return C def build_C_paddle(self): - """ Return coordinates of fiducial points in I_r; C """ + """Return coordinates of fiducial points in I_r; C""" F = self.F ctrl_pts_x = paddle.linspace(-1.0, 1.0, int(F / 2)) ctrl_pts_y_top = -1 * paddle.ones([int(F / 2)]) @@ -68,16 +68,18 @@ class GridGenerator(nn.Layer): I_r_width, I_r_height = I_r_size I_r_grid_x = paddle.divide( (paddle.arange(-I_r_width, I_r_width, 2).astype('float32') + 1.0), - paddle.to_tensor(I_r_width).astype('float32')) + paddle.to_tensor(I_r_width).astype('float32'), + ) I_r_grid_y = paddle.divide( (paddle.arange(-I_r_height, I_r_height, 2).astype('float32') + 1.0), - paddle.to_tensor(I_r_height).astype('float32')) + paddle.to_tensor(I_r_height).astype('float32'), + ) P = paddle.stack(paddle.meshgrid(I_r_grid_x, I_r_grid_y), axis=2) P = paddle.transpose(P, perm=[1, 0, 2]) return P.reshape([-1, 2]) def build_inv_delta_C_paddle(self, C): - """ Return inv_delta_C which is needed to calculate T """ + """Return inv_delta_C which is needed to calculate T""" F = self.F hat_C = paddle.zeros((F, F), dtype='float32') for i in range(0, F): @@ -89,16 +91,19 @@ class GridGenerator(nn.Layer): hat_C[i, j] = r hat_C[j, i] = r hat_C = (hat_C**2) * paddle.log(hat_C) - delta_C = paddle.concat([ - paddle.concat([paddle.ones((F, 1)), C, hat_C], axis=1), - paddle.concat( - [paddle.zeros((2, 3)), - paddle.transpose(C, perm=[1, 0])], - axis=1), - paddle.concat([paddle.zeros( - (1, 3)), paddle.ones((1, F))], axis=1) - ], - axis=0) + delta_C = paddle.concat( + [ + paddle.concat([paddle.ones((F, 1)), C, hat_C], axis=1), + paddle.concat( + [paddle.zeros((2, 3)), paddle.transpose(C, perm=[1, 0])], + axis=1, + ), + paddle.concat( + [paddle.zeros((1, 3)), paddle.ones((1, F))], axis=1 + ), + ], + axis=0, + ) inv_delta_C = paddle.inverse(delta_C) return inv_delta_C @@ -111,8 +116,9 @@ class GridGenerator(nn.Layer): P_diff = P_tile - C_tile rbf_norm = paddle.norm(P_diff, p=2, axis=2, keepdim=False) - rbf = paddle.multiply(paddle.square(rbf_norm), - paddle.log(rbf_norm + eps)) + rbf = paddle.multiply( + paddle.square(rbf_norm), paddle.log(rbf_norm + eps) + ) P_hat = paddle.concat([paddle.ones((n, 1)), P, rbf], axis=1) return P_hat @@ -125,7 +131,6 @@ class GridGenerator(nn.Layer): class TestGridGenerator(unittest.TestCase): - def setUp(self): self.x = paddle.uniform(shape=[1, 20, 2], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py index d40c37f3fd6..da69243cbe9 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py @@ -17,11 +17,37 @@ import unittest import paddle from paddle.fluid.dygraph.jit import declarative -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException import paddle.fluid.core as core -from ifelse_simple_func import NetWithControlFlowIf, add_fn, dyfunc_empty_nonlocal, dyfunc_ifelse_ret_int1, dyfunc_ifelse_ret_int2, dyfunc_ifelse_ret_int3, dyfunc_ifelse_ret_int4, dyfunc_with_if_else, dyfunc_with_if_else2, dyfunc_with_if_else3, dyfunc_with_if_else_with_list_geneator, fluid, if_tensor_case, if_with_and_or, if_with_and_or_1, if_with_and_or_2, if_with_and_or_3, if_with_and_or_4, if_with_class_var, loss_fn, nested_if_else, nested_if_else_2, nested_if_else_3 +from ifelse_simple_func import ( + NetWithControlFlowIf, + add_fn, + dyfunc_empty_nonlocal, + dyfunc_ifelse_ret_int1, + dyfunc_ifelse_ret_int2, + dyfunc_ifelse_ret_int3, + dyfunc_ifelse_ret_int4, + dyfunc_with_if_else, + dyfunc_with_if_else2, + dyfunc_with_if_else3, + dyfunc_with_if_else_with_list_geneator, + fluid, + if_tensor_case, + if_with_and_or, + if_with_and_or_1, + if_with_and_or_2, + if_with_and_or_3, + if_with_and_or_4, + if_with_class_var, + loss_fn, + nested_if_else, + nested_if_else_2, + nested_if_else_3, +) np.random.seed(1) @@ -32,7 +58,6 @@ else: class TestDy2staticException(unittest.TestCase): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = None @@ -75,49 +100,42 @@ class TestDygraphIfElse(unittest.TestCase): class TestDygraphIfElse2(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_with_if_else2 class TestDygraphIfElse3(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_with_if_else3 class TestDygraphIfElse4(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_empty_nonlocal class TestDygraphIfElseWithListGenerator(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_with_if_else_with_list_geneator class TestDygraphNestedIfElse(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = nested_if_else class TestDygraphNestedIfElse2(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = nested_if_else_2 class TestDygraphNestedIfElse3(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = nested_if_else_3 @@ -150,7 +168,6 @@ def dyfunc_ifExp_with_while(x): class TestDygraphIfElse6(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_ifExp_with_while @@ -175,56 +192,48 @@ def dyfunc_ifExp(x): class TestDygraphIfElse7(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_ifExp class TestDygraphIfElseWithAndOr(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or class TestDygraphIfElseWithAndOr1(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_1 class TestDygraphIfElseWithAndOr2(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_2 class TestDygraphIfElseWithAndOr3(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_3 class TestDygraphIfElseWithAndOr4(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_4 class TestDygraphIfElseWithClassVar(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_class_var class TestDygraphIfTensor(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_tensor_case @@ -279,14 +288,12 @@ def call_external_func(x, label=None): class TestAst2FuncWithExternalFunc(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = call_external_func class NetWithExternalFunc(fluid.dygraph.Layer): - @declarative def forward(self, x, label=None): if paddle.mean(x) < 0: @@ -307,14 +314,12 @@ def softmax(x): class TestNetWithExternalFunc(TestDygraphIfElseNet): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.Net = NetWithExternalFunc class DiffModeNet1(paddle.nn.Layer): - def __init__(self, mode): super(DiffModeNet1, self).__init__() self.mode = mode @@ -331,7 +336,6 @@ class DiffModeNet1(paddle.nn.Layer): class DiffModeNet2(paddle.nn.Layer): - def __init__(self, mode): super(DiffModeNet2, self).__init__() self.mode = mode @@ -371,27 +375,28 @@ class TestDiffModeNet(unittest.TestCase): def test_train_mode(self): self.assertTrue( - (self._run(mode='train', - to_static=True) == self._run(mode='train', - to_static=False)).all()) + ( + self._run(mode='train', to_static=True) + == self._run(mode='train', to_static=False) + ).all() + ) def test_infer_mode(self): self.assertTrue( - (self._run(mode='infer', - to_static=True) == self._run(mode='infer', - to_static=False)).all()) + ( + self._run(mode='infer', to_static=True) + == self._run(mode='infer', to_static=False) + ).all() + ) class TestDiffModeNet2(TestDiffModeNet): - def init_net(self): self.Net = DiffModeNet2 class TestNewVarCreateInOneBranch(unittest.TestCase): - def test_var_used_in_another_for(self): - def case_func(training): # targets and targets_list is dynamically defined by training if training: @@ -414,7 +419,6 @@ class TestNewVarCreateInOneBranch(unittest.TestCase): class TestDy2StIfElseRetInt1(unittest.TestCase): - def setUp(self): self.x = np.random.random([5]).astype('float32') self.dyfunc = dyfunc_ifelse_ret_int1 @@ -433,7 +437,6 @@ class TestDy2StIfElseRetInt1(unittest.TestCase): class TestDy2StIfElseRetInt2(TestDy2staticException): - def setUp(self): self.x = np.random.random([5]).astype('float32') self.error = "Your if/else have different number of return value." @@ -441,7 +444,6 @@ class TestDy2StIfElseRetInt2(TestDy2staticException): class TestDy2StIfElseRetInt3(TestDy2StIfElseRetInt1): - def setUp(self): self.x = np.random.random([5]).astype('float32') self.dyfunc = dyfunc_ifelse_ret_int3 @@ -452,7 +454,6 @@ class TestDy2StIfElseRetInt3(TestDy2StIfElseRetInt1): class TestDy2StIfElseRetInt4(TestDy2StIfElseRetInt1): - def setUp(self): self.x = np.random.random([5]).astype('float32') self.dyfunc = dyfunc_ifelse_ret_int4 @@ -472,12 +473,11 @@ class TestDy2StIfElseRetInt4(TestDy2StIfElseRetInt1): class IfElseNet(paddle.nn.Layer): - def __init__(self): super(IfElseNet, self).__init__() - self.param = self.create_parameter(shape=[3, 2], - dtype='float32', - is_bias=False) + self.param = self.create_parameter( + shape=[3, 2], dtype='float32', is_bias=False + ) @paddle.jit.to_static def forward(self, a, b, c): @@ -493,7 +493,6 @@ class IfElseNet(paddle.nn.Layer): class TestDy2StIfElseBackward(unittest.TestCase): - def test_run_backward(self): a = paddle.randn((4, 3), dtype='float32') a.stop_gradient = False @@ -506,9 +505,9 @@ class TestDy2StIfElseBackward(unittest.TestCase): net.train() out = net(a, b, c) out.backward() - np.testing.assert_allclose((b + net.param).numpy(), - out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + (b + net.param).numpy(), out.numpy(), rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_isinstance.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_isinstance.py index dc3b4dd74a6..3b46602cac2 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_isinstance.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_isinstance.py @@ -31,13 +31,11 @@ import paddle.nn as nn class SimpleReturnLayer(nn.Layer): - def forward(self, x): return x class AddAttrLayer(nn.Layer): - def __init__(self): super(AddAttrLayer, self).__init__() self.attr = None @@ -48,21 +46,19 @@ class AddAttrLayer(nn.Layer): class IsInstanceLayer(nn.Layer): - def __init__(self, layer): super(IsInstanceLayer, self).__init__() self.layer = layer @paddle.jit.to_static def forward(self, x): - if isinstance(self.layer, (AddAttrLayer, )): + if isinstance(self.layer, (AddAttrLayer,)): self.layer.attr = x res = self.layer(x) return res class SequentialLayer(nn.Layer): - def __init__(self, layers): super(SequentialLayer, self).__init__() self.layers = nn.LayerList(layers) @@ -88,7 +84,6 @@ def train(model, to_static): class TestIsinstance(unittest.TestCase): - def test_isinstance_simple_return_layer(self): model = IsInstanceLayer(SimpleReturnLayer()) self._test_model(model) @@ -108,11 +103,12 @@ class TestIsinstance(unittest.TestCase): def _test_model(self, model): st_out = train(model, to_static=True) dy_out = train(model, to_static=False) - np.testing.assert_allclose(dy_out, - st_out, - rtol=1e-05, - err_msg='dy_out:\n {}\n st_out:\n{}'.format( - dy_out, st_out)) + np.testing.assert_allclose( + dy_out, + st_out, + rtol=1e-05, + err_msg='dy_out:\n {}\n st_out:\n{}'.format(dy_out, st_out), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_jit_property_save.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_jit_property_save.py index 481d30b8ba2..0d26ab51e1a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_jit_property_save.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_jit_property_save.py @@ -17,8 +17,7 @@ import paddle class TestPropertySave(unittest.TestCase): - """test jit property save - """ + """test jit property save""" def setUp(self): a = paddle.framework.core.Property() @@ -44,8 +43,7 @@ class TestPropertySave(unittest.TestCase): self.a.get_float(1) def test_set(self): - """test propety set. - """ + """test propety set.""" try: a = paddle.framework.core.Property() a.set_float('float', 10.0) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py index 7ffbfbb6533..15d18fe838a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py @@ -38,30 +38,33 @@ program_translator = ProgramTranslator() input_specs = [ paddle.static.InputSpec([None, None], 'int64'), paddle.static.InputSpec([None, None], 'int64'), - paddle.static.InputSpec([None], 'int64') + paddle.static.InputSpec([None], 'int64'), ] class DynamicGRU(fluid.dygraph.Layer): - - def __init__(self, - size, - h_0=None, - param_attr=None, - bias_attr=None, - is_reverse=False, - gate_activation='sigmoid', - candidate_activation='tanh', - origin_mode=False, - init_size=None): + def __init__( + self, + size, + h_0=None, + param_attr=None, + bias_attr=None, + is_reverse=False, + gate_activation='sigmoid', + candidate_activation='tanh', + origin_mode=False, + init_size=None, + ): super(DynamicGRU, self).__init__() - self.gru_unit = GRUUnit(size * 3, - param_attr=param_attr, - bias_attr=bias_attr, - activation=candidate_activation, - gate_activation=gate_activation, - origin_mode=origin_mode) + self.gru_unit = GRUUnit( + size * 3, + param_attr=param_attr, + bias_attr=bias_attr, + activation=candidate_activation, + gate_activation=gate_activation, + origin_mode=origin_mode, + ) self.size = size self.h_0 = h_0 @@ -81,15 +84,16 @@ class DynamicGRU(fluid.dygraph.Layer): j = i # input_ = inputs[:, j:j+1, :] # original code - input_ = fluid.layers.slice(inputs, - axes=[1], - starts=[j], - ends=[j + 1]) - input_ = fluid.layers.reshape(input_, [-1, input_.shape[2]], - inplace=False) + input_ = fluid.layers.slice( + inputs, axes=[1], starts=[j], ends=[j + 1] + ) + input_ = fluid.layers.reshape( + input_, [-1, input_.shape[2]], inplace=False + ) hidden, reset, gate = self.gru_unit(input_, hidden) - hidden_ = fluid.layers.reshape(hidden, [-1, 1, hidden.shape[1]], - inplace=False) + hidden_ = fluid.layers.reshape( + hidden, [-1, 1, hidden.shape[1]], inplace=False + ) res.append(hidden_) if self.is_reverse: @@ -99,7 +103,6 @@ class DynamicGRU(fluid.dygraph.Layer): class BiGRU(fluid.dygraph.Layer): - def __init__(self, input_dim, grnn_hidden_dim, init_bound, h_0=None): super(BiGRU, self).__init__() @@ -107,38 +110,54 @@ class BiGRU(fluid.dygraph.Layer): input_dim=input_dim, output_dim=grnn_hidden_dim * 3, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=-init_bound, - high=init_bound), + initializer=fluid.initializer.Uniform( + low=-init_bound, high=init_bound + ), regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) + regularization_coeff=1e-4 + ), + ), + ) self.gru = DynamicGRU( size=grnn_hidden_dim, h_0=h_0, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=-init_bound, - high=init_bound), + initializer=fluid.initializer.Uniform( + low=-init_bound, high=init_bound + ), regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) + regularization_coeff=1e-4 + ), + ), + ) self.pre_gru_r = Linear( input_dim=input_dim, output_dim=grnn_hidden_dim * 3, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=-init_bound, - high=init_bound), + initializer=fluid.initializer.Uniform( + low=-init_bound, high=init_bound + ), regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) + regularization_coeff=1e-4 + ), + ), + ) self.gru_r = DynamicGRU( size=grnn_hidden_dim, is_reverse=True, h_0=h_0, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=-init_bound, - high=init_bound), + initializer=fluid.initializer.Uniform( + low=-init_bound, high=init_bound + ), regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) + regularization_coeff=1e-4 + ), + ), + ) def forward(self, input_feature): res_pre_gru = self.pre_gru(input_feature) @@ -152,7 +171,6 @@ class BiGRU(fluid.dygraph.Layer): class LinearChainCRF(fluid.dygraph.Layer): - def __init__(self, param_attr, size=None, is_test=False, dtype='float32'): super(LinearChainCRF, self).__init__() @@ -163,7 +181,8 @@ class LinearChainCRF(fluid.dygraph.Layer): self._transition = self.create_parameter( attr=self._param_attr, shape=[self._size + 2, self._size], - dtype=self._dtype) + dtype=self._dtype, + ) @property def weight(self): @@ -176,41 +195,46 @@ class LinearChainCRF(fluid.dygraph.Layer): def forward(self, input, label, length=None): if _non_static_mode(): _, _, _, log_likelihood = _legacy_C_ops.linear_chain_crf( - input, self._transition, label, length, "is_test", - self._is_test) + input, self._transition, label, length, "is_test", self._is_test + ) return log_likelihood alpha = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) emission_exps = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) transition_exps = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) log_likelihood = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) this_inputs = { "Emission": [input], "Transition": self._transition, - "Label": [label] + "Label": [label], } if length is not None: this_inputs['Length'] = [length] - self._helper.append_op(type='linear_chain_crf', - inputs=this_inputs, - outputs={ - "Alpha": [alpha], - "EmissionExps": [emission_exps], - "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood - }, - attrs={ - "is_test": self._is_test, - }) + self._helper.append_op( + type='linear_chain_crf', + inputs=this_inputs, + outputs={ + "Alpha": [alpha], + "EmissionExps": [emission_exps], + "TransitionExps": transition_exps, + "LogLikelihood": log_likelihood, + }, + attrs={ + "is_test": self._is_test, + }, + ) return log_likelihood class CRFDecoding(fluid.dygraph.Layer): - def __init__(self, param_attr, size=None, is_test=False, dtype='float32'): super(CRFDecoding, self).__init__() @@ -221,7 +245,8 @@ class CRFDecoding(fluid.dygraph.Layer): self._transition = self.create_parameter( attr=self._param_attr, shape=[self._size + 2, self._size], - dtype=self._dtype) + dtype=self._dtype, + ) @property def weight(self): @@ -233,33 +258,35 @@ class CRFDecoding(fluid.dygraph.Layer): def forward(self, input, label=None, length=None): if _non_static_mode(): - return _legacy_C_ops.crf_decoding(input, self._transition, label, - length, "is_test", self._is_test) + return _legacy_C_ops.crf_decoding( + input, self._transition, label, length, "is_test", self._is_test + ) viterbi_path = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) this_inputs = { "Emission": [input], "Transition": self._transition, - "Label": label + "Label": label, } if length is not None: this_inputs['Length'] = [length] - self._helper.append_op(type='crf_decoding', - inputs=this_inputs, - outputs={"ViterbiPath": [viterbi_path]}, - attrs={ - "is_test": self._is_test, - }) + self._helper.append_op( + type='crf_decoding', + inputs=this_inputs, + outputs={"ViterbiPath": [viterbi_path]}, + attrs={ + "is_test": self._is_test, + }, + ) return viterbi_path class ChunkEval(fluid.dygraph.Layer): - - def __init__(self, - num_chunk_types, - chunk_scheme, - excluded_chunk_types=None): + def __init__( + self, num_chunk_types, chunk_scheme, excluded_chunk_types=None + ): super(ChunkEval, self).__init__() self.num_chunk_types = num_chunk_types self.chunk_scheme = chunk_scheme @@ -267,54 +294,69 @@ class ChunkEval(fluid.dygraph.Layer): def forward(self, input, label, seq_length=None): if _non_static_mode(): - return _legacy_C_ops.chunk_eval(input, label, seq_length, - "num_chunk_types", - self.num_chunk_types, - "chunk_scheme", self.chunk_scheme, - "excluded_chunk_types", - self.excluded_chunk_types or []) + return _legacy_C_ops.chunk_eval( + input, + label, + seq_length, + "num_chunk_types", + self.num_chunk_types, + "chunk_scheme", + self.chunk_scheme, + "excluded_chunk_types", + self.excluded_chunk_types or [], + ) precision = self._helper.create_variable_for_type_inference( - dtype="float32") + dtype="float32" + ) recall = self._helper.create_variable_for_type_inference( - dtype="float32") + dtype="float32" + ) f1_score = self._helper.create_variable_for_type_inference( - dtype="float32") + dtype="float32" + ) num_infer_chunks = self._helper.create_variable_for_type_inference( - dtype="int64") + dtype="int64" + ) num_label_chunks = self._helper.create_variable_for_type_inference( - dtype="int64") + dtype="int64" + ) num_correct_chunks = self._helper.create_variable_for_type_inference( - dtype="int64") + dtype="int64" + ) this_input = {"Inference": [input], "Label": [label]} if seq_length is not None: this_input["SeqLength"] = [seq_length] - self._helper.append_op(type='chunk_eval', - inputs=this_input, - outputs={ - "Precision": [precision], - "Recall": [recall], - "F1-Score": [f1_score], - "NumInferChunks": [num_infer_chunks], - "NumLabelChunks": [num_label_chunks], - "NumCorrectChunks": [num_correct_chunks] - }, - attrs={ - "num_chunk_types": - self.num_chunk_types, - "chunk_scheme": - self.chunk_scheme, - "excluded_chunk_types": - self.excluded_chunk_types or [] - }) - return (precision, recall, f1_score, num_infer_chunks, num_label_chunks, - num_correct_chunks) + self._helper.append_op( + type='chunk_eval', + inputs=this_input, + outputs={ + "Precision": [precision], + "Recall": [recall], + "F1-Score": [f1_score], + "NumInferChunks": [num_infer_chunks], + "NumLabelChunks": [num_label_chunks], + "NumCorrectChunks": [num_correct_chunks], + }, + attrs={ + "num_chunk_types": self.num_chunk_types, + "chunk_scheme": self.chunk_scheme, + "excluded_chunk_types": self.excluded_chunk_types or [], + }, + ) + return ( + precision, + recall, + f1_score, + num_infer_chunks, + num_label_chunks, + num_correct_chunks, + ) class LexNet(fluid.dygraph.Layer): - def __init__(self, args, length=None): super(LexNet, self).__init__() """ @@ -330,21 +372,26 @@ class LexNet(fluid.dygraph.Layer): self.vocab_size = args.vocab_size self.num_labels = args.num_labels self.grnn_hidden_dim = args.grnn_hidden_dim - self.emb_lr = args.emb_learning_rate if 'emb_learning_rate' in dir( - args) else 1.0 - self.crf_lr = args.emb_learning_rate if 'crf_learning_rate' in dir( - args) else 1.0 + self.emb_lr = ( + args.emb_learning_rate if 'emb_learning_rate' in dir(args) else 1.0 + ) + self.crf_lr = ( + args.emb_learning_rate if 'crf_learning_rate' in dir(args) else 1.0 + ) self.bigru_num = args.bigru_num self.init_bound = 0.1 self.word_embedding = Embedding( size=[self.vocab_size, self.word_emb_dim], dtype='float32', - param_attr=fluid.ParamAttr(learning_rate=self.emb_lr, - name="word_emb", - initializer=fluid.initializer.Uniform( - low=-self.init_bound, - high=self.init_bound))) + param_attr=fluid.ParamAttr( + learning_rate=self.emb_lr, + name="word_emb", + initializer=fluid.initializer.Uniform( + low=-self.init_bound, high=self.init_bound + ), + ), + ) h_0 = np.zeros((args.batch_size, self.grnn_hidden_dim), dtype="float32") h_0 = to_variable(h_0) @@ -355,34 +402,51 @@ class LexNet(fluid.dygraph.Layer): self.bigru_units.append( self.add_sublayer( "bigru_units%d" % i, - BiGRU(self.grnn_hidden_dim, - self.grnn_hidden_dim, - self.init_bound, - h_0=h_0))) + BiGRU( + self.grnn_hidden_dim, + self.grnn_hidden_dim, + self.init_bound, + h_0=h_0, + ), + ) + ) else: self.bigru_units.append( self.add_sublayer( "bigru_units%d" % i, - BiGRU(self.grnn_hidden_dim * 2, - self.grnn_hidden_dim, - self.init_bound, - h_0=h_0))) - - self.fc = Linear(input_dim=self.grnn_hidden_dim * 2, - output_dim=self.num_labels, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-self.init_bound, high=self.init_bound), - regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) - - self.linear_chain_crf = LinearChainCRF(param_attr=fluid.ParamAttr( - name='linear_chain_crfw', learning_rate=self.crf_lr), - size=self.num_labels) - - self.crf_decoding = CRFDecoding(param_attr=fluid.ParamAttr( - name='crfw', learning_rate=self.crf_lr), - size=self.num_labels) + BiGRU( + self.grnn_hidden_dim * 2, + self.grnn_hidden_dim, + self.init_bound, + h_0=h_0, + ), + ) + ) + + self.fc = Linear( + input_dim=self.grnn_hidden_dim * 2, + output_dim=self.num_labels, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=-self.init_bound, high=self.init_bound + ), + regularizer=fluid.regularizer.L2DecayRegularizer( + regularization_coeff=1e-4 + ), + ), + ) + + self.linear_chain_crf = LinearChainCRF( + param_attr=fluid.ParamAttr( + name='linear_chain_crfw', learning_rate=self.crf_lr + ), + size=self.num_labels, + ) + + self.crf_decoding = CRFDecoding( + param_attr=fluid.ParamAttr(name='crfw', learning_rate=self.crf_lr), + size=self.num_labels, + ) # share weight self.crf_decoding.weight = self.linear_chain_crf.weight @@ -400,9 +464,9 @@ class LexNet(fluid.dygraph.Layer): emission = self.fc(bigru_output) - crf_cost = self.linear_chain_crf(input=emission, - label=target, - length=length) + crf_cost = self.linear_chain_crf( + input=emission, label=target, length=length + ) avg_cost = paddle.mean(x=crf_cost) crf_decode = self.crf_decoding(input=emission, length=length) return avg_cost, crf_decode @@ -429,10 +493,16 @@ def get_random_input_data(batch_size, vocab_size, num_labels, max_seq_len=64): batch, init_lens = [], [] for i in range(iter_num * batch_size): cur_len = local_random.randint(3, max_seq_len) - word_ids = local_random.randint(0, vocab_size, - [cur_len]).astype('int64').tolist() - label_ids = local_random.randint( - 0, num_labels, [cur_len]).astype('int64').tolist() + word_ids = ( + local_random.randint(0, vocab_size, [cur_len]) + .astype('int64') + .tolist() + ) + label_ids = ( + local_random.randint(0, num_labels, [cur_len]) + .astype('int64') + .tolist() + ) batch.append((word_ids, label_ids)) init_lens.append(cur_len) if len(batch) == batch_size: @@ -457,9 +527,9 @@ def get_random_input_data(batch_size, vocab_size, num_labels, max_seq_len=64): def create_dataloader(reader, place): - data_loader = fluid.io.DataLoader.from_generator(capacity=16, - use_double_buffer=True, - iterable=True) + data_loader = fluid.io.DataLoader.from_generator( + capacity=16, use_double_buffer=True, iterable=True + ) data_loader.set_sample_list_generator(reader, places=place) @@ -467,11 +537,13 @@ def create_dataloader(reader, place): class TestLACModel(unittest.TestCase): - def setUp(self): self.args = Args() - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'inference') self.model_save_prefix = os.path.join(self.model_save_dir, 'lac') @@ -481,22 +553,28 @@ class TestLACModel(unittest.TestCase): def train(self, args, to_static): program_translator.enable(to_static) - place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - reader = get_random_input_data(args.batch_size, args.vocab_size, - args.num_labels) + reader = get_random_input_data( + args.batch_size, args.vocab_size, args.num_labels + ) train_loader = create_dataloader(reader, place) model = LexNet(args) optimizer = fluid.optimizer.AdamOptimizer( learning_rate=args.base_learning_rate, - parameter_list=model.parameters()) - chunk_eval = ChunkEval(int(math.ceil((args.num_labels - 1) / 2.0)), - "IOB") + parameter_list=model.parameters(), + ) + chunk_eval = ChunkEval( + int(math.ceil((args.num_labels - 1) / 2.0)), "IOB" + ) step = 0 chunk_evaluator = fluid.metrics.ChunkEvaluator() @@ -517,11 +595,16 @@ class TestLACModel(unittest.TestCase): end_time = time.time() if step % args.print_steps == 0: - (precision, recall, f1_score, num_infer_chunks, - num_label_chunks, - num_correct_chunks) = chunk_eval(input=crf_decode, - label=targets, - seq_length=length) + ( + precision, + recall, + f1_score, + num_infer_chunks, + num_label_chunks, + num_correct_chunks, + ) = chunk_eval( + input=crf_decode, label=targets, seq_length=length + ) outputs = [avg_cost, precision, recall, f1_score] avg_cost, precision, recall, f1_score = [ np.mean(x.numpy()) for x in outputs @@ -529,8 +612,15 @@ class TestLACModel(unittest.TestCase): print( "[train] step = %d, loss = %f, P: %f, R: %f, F1: %f, elapsed time %f" - % (step, avg_cost, precision, recall, f1_score, - end_time - start_time)) + % ( + step, + avg_cost, + precision, + recall, + f1_score, + end_time - start_time, + ) + ) step += 1 # save inference model @@ -539,10 +629,12 @@ class TestLACModel(unittest.TestCase): layer=model, path=self.model_save_prefix, input_spec=[input_specs[0], input_specs[-1]], - output_spec=[crf_decode]) + output_spec=[crf_decode], + ) else: - fluid.dygraph.save_dygraph(model.state_dict(), - self.dy_param_path) + fluid.dygraph.save_dygraph( + model.state_dict(), self.dy_param_path + ) return np.array(loss_data) @@ -554,14 +646,16 @@ class TestLACModel(unittest.TestCase): st_out, rtol=1e-05, err_msg='dygraph output:\n{},\nstatic output:\n {}.'.format( - dy_out, st_out)) + dy_out, st_out + ), + ) # Prediction needs trained models, so put `test_predict` at last of `test_train` # self.verify_predict() def verify_predict(self): - reader = get_random_input_data(self.args.batch_size, - self.args.vocab_size, - self.args.num_labels) + reader = get_random_input_data( + self.args.batch_size, self.args.vocab_size, self.args.num_labels + ) for batch in reader(): batch = [np.vstack(var) for var in zip(*batch)] dy_pre = self.predict_dygraph(batch) @@ -580,8 +674,9 @@ class TestLACModel(unittest.TestCase): model.set_dict(model_dict) model.eval() - _, pred_res = model(to_variable(words), to_variable(targets), - to_variable(length)) + _, pred_res = model( + to_variable(words), to_variable(targets), to_variable(length) + ) return pred_res.numpy() @@ -593,19 +688,23 @@ class TestLACModel(unittest.TestCase): paddle.enable_static() exe = fluid.Executor(self.place) # load inference model - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) words, targets, length = batch - pred_res = exe.run(inference_program, - feed={ - feed_target_names[0]: words, - feed_target_names[1]: length - }, - fetch_list=fetch_targets) + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: words, feed_target_names[1]: length}, + fetch_list=fetch_targets, + ) return pred_res[0] def predict_dygraph_jit(self, batch): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lambda.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lambda.py index 121954edf98..d0b45a050bc 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lambda.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lambda.py @@ -79,18 +79,23 @@ def call_lambda_with_ifExpr2(x): class TestLambda(unittest.TestCase): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.x = np.array([1, 3]).astype('float32') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_func() def init_func(self): self.dyfuncs = [ - call_lambda_as_func, call_lambda_directly, call_lambda_in_func, - call_lambda_with_ifExpr, call_lambda_with_ifExpr2 + call_lambda_as_func, + call_lambda_directly, + call_lambda_in_func, + call_lambda_with_ifExpr, + call_lambda_with_ifExpr2, ] def run_static(self, func): @@ -109,7 +114,8 @@ class TestLambda(unittest.TestCase): def test_ast_to_func(self): for func in self.dyfuncs: self.assertTrue( - (self.run_dygraph(func) == self.run_static(func)).all()) + (self.run_dygraph(func) == self.run_static(func)).all() + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_layer_hook.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_layer_hook.py index b2a23b7d86f..ad4aa8e173f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_layer_hook.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_layer_hook.py @@ -24,13 +24,14 @@ def forward_post_hook1(layer, input, output): def forward_pre_hook1(layer, input): - input_return = (input[0] * 2, ) + input_return = (input[0] * 2,) return input_return class SimpleNet(paddle.nn.Layer): - - def __init__(self, ): + def __init__( + self, + ): super(SimpleNet, self).__init__() self.fc1 = paddle.nn.Linear(10, 10) # sublayer1 register post hook @@ -53,7 +54,6 @@ class SimpleNet(paddle.nn.Layer): class TestNestLayerHook(unittest.TestCase): - def setUp(self): paddle.seed(2022) self.x = paddle.randn([4, 10]) @@ -90,12 +90,15 @@ class TestNestLayerHook(unittest.TestCase): dy_out, rtol=1e-05, err_msg='dygraph_res is {}\nstatic_res is {}'.format( - dy_out, st_out)) + dy_out, st_out + ), + ) np.testing.assert_allclose( st_out, load_out, rtol=1e-05, - err_msg='load_out is {}\nstatic_res is {}'.format(load_out, st_out)) + err_msg='load_out is {}\nstatic_res is {}'.format(load_out, st_out), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_len.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_len.py index c2a7fb2d1b9..01e5f1e1074 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_len.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_len.py @@ -42,10 +42,12 @@ def len_with_lod_tensor_array(x): class TestLen(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.x_data = np.random.random([10, 16]).astype('float32') self.init_func() @@ -70,7 +72,6 @@ class TestLen(unittest.TestCase): class TestLenWithTensorArray(TestLen): - def init_func(self): self.func = len_with_lod_tensor_array @@ -80,11 +81,13 @@ class TestLenWithTensorArray(TestLen): def len_with_selected_rows(place): block = fluid.default_main_program().global_block() # create selected_rows variable - var = block.create_var(name="X", - dtype="float32", - shape=[-1], - persistable=True, - type=fluid.core.VarDesc.VarType.SELECTED_ROWS) + var = block.create_var( + name="X", + dtype="float32", + shape=[-1], + persistable=True, + type=fluid.core.VarDesc.VarType.SELECTED_ROWS, + ) # y is Variable(SelectedRows) y = fluid.layers.merge_selected_rows(var) y_len = convert_call(len)(y) @@ -110,14 +113,17 @@ def len_with_selected_rows(place): class TestLenWithSelectedRows(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def test_len(self): selected_rows_var_len, var_tensor_len = len_with_selected_rows( - self.place) + self.place + ) self.assertEqual(selected_rows_var_len, var_tensor_len) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py index a1f681193e5..58ef815b82c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py @@ -42,7 +42,8 @@ def test_list_append_in_if(x): a.append(x) else: a.append( - fluid.layers.fill_constant(shape=[1, 2], value=9, dtype="int64")) + fluid.layers.fill_constant(shape=[1, 2], value=9, dtype="int64") + ) # TODO(Aurelius84): Currently, run_program_op doesn't support output LoDTensorArray. return a[0] @@ -98,9 +99,9 @@ def test_list_append_in_for_loop_with_concat(x, iter_num): def test_list_append_in_while_loop(x, iter_num): x = fluid.dygraph.to_variable(x) - iter_num = fluid.layers.fill_constant(shape=[1], - value=iter_num, - dtype="int32") + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) a = [] i = 0 while i < iter_num: @@ -111,9 +112,9 @@ def test_list_append_in_while_loop(x, iter_num): def test_list_append_in_while_loop_with_stack(x, iter_num): x = fluid.dygraph.to_variable(x) - iter_num = fluid.layers.fill_constant(shape=[1], - value=iter_num, - dtype="int32") + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) a = [] i = 0 while i < iter_num.numpy()[0]: @@ -180,9 +181,9 @@ def test_list_pop_in_for_loop(x, iter_num): def test_list_pop_in_while_loop(x, iter_num): x = fluid.dygraph.to_variable(x) - iter_num = fluid.layers.fill_constant(shape=[1], - value=iter_num, - dtype="int32") + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) a = [] b = [x] b.append(x) @@ -199,10 +200,12 @@ def test_list_pop_in_while_loop(x, iter_num): class TestListWithoutControlFlow(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_data() self.init_dygraph_func() @@ -252,24 +255,25 @@ class TestListWithoutControlFlow(unittest.TestCase): dy_res, rtol=1e-05, err_msg='dygraph_res is {}\nstatic_res is {}'.format( - dy_res, stat_res)) + dy_res, stat_res + ), + ) class TestListInIf(TestListWithoutControlFlow): - def init_dygraph_func(self): self.all_dygraph_funcs = [test_list_append_in_if] class TestListInWhileLoop(TestListWithoutControlFlow): - def init_data(self): self.input = np.random.random((3)).astype('int32') self.iter_num = 3 def init_dygraph_func(self): self.all_dygraph_funcs = [ - test_list_append_in_while_loop, test_list_pop_in_while_loop + test_list_append_in_while_loop, + test_list_pop_in_while_loop, ] def train(self, to_static=False): @@ -284,21 +288,19 @@ class TestListInWhileLoop(TestListWithoutControlFlow): class TestListInWhileLoopWithStack(TestListInWhileLoop): - def init_dygraph_func(self): self.all_dygraph_funcs = [test_list_append_in_while_loop_with_stack] class TestListInForLoop(TestListInWhileLoop): - def init_dygraph_func(self): self.all_dygraph_funcs = [ - test_list_append_in_for_loop, test_list_pop_in_for_loop + test_list_append_in_for_loop, + test_list_pop_in_for_loop, ] class TestListInForLoopWithConcat(TestListInWhileLoopWithStack): - def init_dygraph_func(self): self.all_dygraph_funcs = [ test_list_append_in_for_loop_with_concat, @@ -306,11 +308,10 @@ class TestListInForLoopWithConcat(TestListInWhileLoopWithStack): class TestListInForLoopWithSubscript(TestListWithoutControlFlow): - def init_dygraph_func(self): self.all_dygraph_funcs = [ test_list_append_in_for_subscript, - test_list_append_in_while_loop_subscript + test_list_append_in_while_loop_subscript, ] def init_data(self): @@ -318,7 +319,6 @@ class TestListInForLoopWithSubscript(TestListWithoutControlFlow): class ListWithCondNet(paddle.nn.Layer): - def __init__(self): super(ListWithCondNet, self).__init__() @@ -342,13 +342,12 @@ class ListWithCondNet(paddle.nn.Layer): class TestListWithCondGradInferVarType(unittest.TestCase): - def test_to_static(self): net = ListWithCondNet() x = paddle.to_tensor([2, 3, 4], dtype='float32') index = paddle.to_tensor([1]) res = net(x, index) - self.assertEqual(res[0], 48.) + self.assertEqual(res[0], 48.0) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py index d563758cdd6..cdc63ca353e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py @@ -26,7 +26,6 @@ from unittest import mock class TestLoggingUtils(unittest.TestCase): - def setUp(self): self.verbosity_level = 1 self.code_level = 3 @@ -50,19 +49,23 @@ class TestLoggingUtils(unittest.TestCase): def test_also_to_stdout(self): logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout = None self.assertEqual( - logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout, False) + logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout, False + ) paddle.jit.set_verbosity(also_to_stdout=False) self.assertEqual( - logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout, False) + logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout, False + ) logging_utils._TRANSLATOR_LOGGER.need_to_echo_node_to_stdout = None self.assertEqual( - logging_utils._TRANSLATOR_LOGGER.need_to_echo_code_to_stdout, False) + logging_utils._TRANSLATOR_LOGGER.need_to_echo_code_to_stdout, False + ) paddle.jit.set_code_level(also_to_stdout=True) self.assertEqual( - logging_utils._TRANSLATOR_LOGGER.need_to_echo_code_to_stdout, True) + logging_utils._TRANSLATOR_LOGGER.need_to_echo_code_to_stdout, True + ) with self.assertRaises(AssertionError): paddle.jit.set_verbosity(also_to_stdout=1) @@ -99,8 +102,9 @@ class TestLoggingUtils(unittest.TestCase): logging_utils.set_code_level(1, True) logging_utils.log_transformed_code(1, ast_code, "TestTransformer") logging_utils.set_code_level(logging_utils.LOG_AllTransformer, True) - logging_utils.log_transformed_code(logging_utils.LOG_AllTransformer, - ast_code, "TestTransformer") + logging_utils.log_transformed_code( + logging_utils.LOG_AllTransformer, ast_code, "TestTransformer" + ) def test_log_message(self): stream = io.StringIO() @@ -121,7 +125,8 @@ class TestLoggingUtils(unittest.TestCase): logging_utils.log(2, log_msg_2) result_msg = '\n'.join( - [warn_msg, error_msg, "(Level 1) " + log_msg_1, ""]) + [warn_msg, error_msg, "(Level 1) " + log_msg_1, ""] + ) self.assertEqual(result_msg, stream.getvalue()) def test_log_transformed_code(self): @@ -135,12 +140,14 @@ class TestLoggingUtils(unittest.TestCase): with mock.patch.object(sys, 'stdout', stream): paddle.jit.set_code_level(1) - logging_utils.log_transformed_code(1, ast_code, - "BasicApiTransformer") + logging_utils.log_transformed_code( + 1, ast_code, "BasicApiTransformer" + ) paddle.jit.set_code_level() - logging_utils.log_transformed_code(logging_utils.LOG_AllTransformer, - ast_code, "All Transformers") + logging_utils.log_transformed_code( + logging_utils.LOG_AllTransformer, ast_code, "All Transformers" + ) self.assertIn(source_code, stream.getvalue()) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py index a8804783904..4ab65c47660 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py @@ -23,7 +23,9 @@ import numpy as np import paddle import paddle.fluid as fluid from paddle.fluid.dygraph import ProgramTranslator -from paddle.fluid.dygraph.dygraph_to_static.logical_transformer import cmpop_node_to_str +from paddle.fluid.dygraph.dygraph_to_static.logical_transformer import ( + cmpop_node_to_str, +) program_translator = ProgramTranslator() @@ -171,16 +173,19 @@ def test_shape_not_equal(x): class TestLogicalBase(unittest.TestCase): - def setUp(self): self.input = np.array([3]).astype('int32') - self.place = paddle.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self._set_test_func() def _set_test_func(self): raise NotImplementedError( - "Method 'set_test_func' should be implemented.") + "Method 'set_test_func' should be implemented." + ) def _run(self, to_static): program_translator.enable(to_static) @@ -196,7 +201,6 @@ class TestLogicalBase(unittest.TestCase): class TestLogicalNot(TestLogicalBase): - def _set_test_func(self): self.dygraph_func = test_logical_not @@ -208,11 +212,12 @@ class TestLogicalNot(TestLogicalBase): static_res, rtol=1e-05, err_msg='dygraph result is {}\nstatic_result is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestLogicalNot2(TestLogicalBase): - def _set_test_func(self): self.dygraph_func = test_logical_not_2 @@ -224,55 +229,49 @@ class TestLogicalNot2(TestLogicalBase): static_res, rtol=1e-05, err_msg='dygraph result is {}\nstatic_result is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestLogicalAnd(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_and class TestLogicalAnd2(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_and_2 class TestLogicalOr(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_or class TestLogicalOr2(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_or_2 class TestLogicalNotAndOr(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_not_and_or class TestShapeEqual(TestLogicalNot): - def _set_test_func(self): self.input = np.ones([1, 2, 3]).astype('float32') self.dygraph_func = test_shape_equal class TestShapeNotEqual(TestLogicalNot): - def _set_test_func(self): self.input = np.ones([1, 2, 3]).astype('float32') self.dygraph_func = test_shape_not_equal class TestCmpopNodeToStr(unittest.TestCase): - def test_exception(self): with self.assertRaises(KeyError): cmpop_node_to_str(gast.Or()) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py index 478d573558e..6bc237f3781 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py @@ -63,12 +63,15 @@ def while_loop_dyfun_with_conflict_var(x): def while_loop_dyfunc_with_none(x): - i = fluid.dygraph.to_variable(x)\ - if x is not None \ - else fluid.dygraph.to_variable(x+1) + i = ( + fluid.dygraph.to_variable(x) + if x is not None + else fluid.dygraph.to_variable(x + 1) + ) # Use `to_variable` so that static analysis can analyze the type of X is Tensor x = fluid.dygraph.to_variable( - x) # TODO(liym27): Delete it if the type of parameter x can be resolved + x + ) # TODO(liym27): Delete it if the type of parameter x can be resolved flag = 1 while x < 10: i = i + x if flag is not None else x + i @@ -150,9 +153,7 @@ def while_loop_bool_op2(x): def while_loop_class_var(x): - class Foo(object): - def __init__(self): self.a = 3 self.b = 4 @@ -178,9 +179,7 @@ def loop_var_contains_property(x): def for_loop_class_var(max_len): - class Foo(object): - def __init__(self): self.a = 3 self.b = 4 @@ -189,9 +188,9 @@ def for_loop_class_var(max_len): foo = Foo() # Use `to_variable` so that static analysis can analyze the type of X is Tensor - max_len = fluid.layers.fill_constant(shape=[1], - value=max_len, - dtype="int32") + max_len = fluid.layers.fill_constant( + shape=[1], value=max_len, dtype="int32" + ) for i in range(max_len): foo.b = fluid.layers.zeros(shape=[1], dtype='float32') @@ -230,17 +229,18 @@ def for_loop_dufunc_with_listcomp(array): class TestNameVisitor(unittest.TestCase): - def setUp(self): self.loop_funcs = [ - while_loop_dyfunc, for_loop_dyfunc, while_loop_dyfunc_with_none, - for_loop_dufunc_with_listcomp + while_loop_dyfunc, + for_loop_dyfunc, + while_loop_dyfunc_with_none, + for_loop_dufunc_with_listcomp, ] self.loop_var_names = [ set(["i", "x"]), set(["i", "ret", "max_len"]), set(["i", "x"]), - set(["j", "array", "res", "x"]) + set(["j", "array", "res", "x"]), ] self.create_var_names = [set(), set(["ret"]), set(), set(["res", "x"])] @@ -254,8 +254,10 @@ class TestNameVisitor(unittest.TestCase): name_visitor = NameVisitor(gast_root) for node in gast.walk(gast_root): if isinstance(node, (gast.While, gast.For)): - loop_var_names, create_var_names = name_visitor.get_loop_var_names( - node) + ( + loop_var_names, + create_var_names, + ) = name_visitor.get_loop_var_names(node) self.assertEqual(loop_var_names, self.loop_var_names[i]) self.assertEqual(create_var_names, self.create_var_names[i]) @@ -268,34 +270,41 @@ class TestNameVisitor(unittest.TestCase): self.loop_var_names = [ set(["j", "two"]), set(["i", "three", "b"]), - set(["i"]) + set(["i"]), ] self.create_var_names = [set(), set(["b"]), set()] i = 0 for node in gast.walk(gast_root): if isinstance(node, (gast.While, gast.For)): - loop_var_names, create_var_names = name_visitor.get_loop_var_names( - node) + ( + loop_var_names, + create_var_names, + ) = name_visitor.get_loop_var_names(node) self.assertEqual( loop_var_names, self.loop_var_names[i], - msg="loop_var_names : {}, \nexpected loop_var_names : {}". - format(loop_var_names, self.loop_var_names[i])) + msg="loop_var_names : {}, \nexpected loop_var_names : {}".format( + loop_var_names, self.loop_var_names[i] + ), + ) self.assertEqual( create_var_names, self.create_var_names[i], - msg= - "i = {}\ncreate_var_names : {}, \nexpected create_var_names : {}" - .format(i, create_var_names, self.create_var_names[i])) + msg="i = {}\ncreate_var_names : {}, \nexpected create_var_names : {}".format( + i, create_var_names, self.create_var_names[i] + ), + ) i += 1 class TestTransformWhileLoop(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.x = np.zeros(shape=(1), dtype=np.int32) self._init_dyfunc() @@ -329,58 +338,52 @@ class TestTransformWhileLoop(unittest.TestCase): class TestTransformWhileLoopWithoutTensor(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_dyfunc_without_tensor class TestTransformWhileLoopWithConflicVar(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_dyfun_with_conflict_var class TestTransformWhileLoopWithNone(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_dyfunc_with_none class TestForBreakSingleReturn(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = for_break_single_return class TestWhileLoopBoolOp(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_bool_op class TestWhileLoopBoolOp2(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_bool_op2 class TestWhileLoopClassVar(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_class_var class TestLoopVarContainsProperty(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = loop_var_contains_property class TestTransformForLoop(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.len = 100 self._init_dyfunc() @@ -402,43 +405,37 @@ class TestTransformForLoop(unittest.TestCase): return ret.numpy() def test_ast_to_func(self): - np.testing.assert_allclose(self._run_dygraph(), - self._run_static(), - rtol=1e-05) + np.testing.assert_allclose( + self._run_dygraph(), self._run_static(), rtol=1e-05 + ) class TestTransformForLoop2(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_dyfunc2 class TestTransformForLoop3(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_dyfunc3 class TestTransformForLoop4(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_dyfunc4 class TestClassVarInForLoop(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_class_var class TestVarCreateInForLoop(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = var_create_in_for_loop class TestErrorInForLoop(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_dyfunc_not_support diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lstm.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lstm.py index d47445c05e9..b4bbfd789c0 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lstm.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lstm.py @@ -21,13 +21,11 @@ import tempfile class LSTMLayer(nn.Layer): - def __init__(self, in_channels, hidden_size): super(LSTMLayer, self).__init__() - self.cell = nn.LSTM(in_channels, - hidden_size, - direction='bidirectional', - num_layers=2) + self.cell = nn.LSTM( + in_channels, hidden_size, direction='bidirectional', num_layers=2 + ) def forward(self, x): x, _ = self.cell(x) @@ -35,7 +33,6 @@ class LSTMLayer(nn.Layer): class Net(nn.Layer): - def __init__(self, in_channels, hidden_size): super(Net, self).__init__() self.lstm = LSTMLayer(in_channels, hidden_size) @@ -46,7 +43,6 @@ class Net(nn.Layer): class TestLstm(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -79,15 +75,17 @@ class TestLstm(unittest.TestCase): x.stop_gradient = False dygraph_out = net(x) loss = paddle.mean(dygraph_out) - sgd = paddle.optimizer.SGD(learning_rate=0.001, - parameters=net.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.001, parameters=net.parameters() + ) loss.backward() sgd.step() # switch eval mode firstly net.eval() x = paddle.randn((2, 10, 12)) net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec(shape=[-1, 10, 12])]) + net, input_spec=[paddle.static.InputSpec(shape=[-1, 10, 12])] + ) model_path = os.path.join(self.temp_dir.name, 'simple_lstm') paddle.jit.save(net, model_path) @@ -101,7 +99,9 @@ class TestLstm(unittest.TestCase): static_out.numpy(), rtol=1e-05, err_msg='dygraph_out is {}\n static_out is \n{}'.format( - dygraph_out, static_out)) + dygraph_out, static_out + ), + ) # switch back into train mode. net.train() train_out = net(x) @@ -110,14 +110,15 @@ class TestLstm(unittest.TestCase): train_out.numpy(), rtol=1e-05, err_msg='dygraph_out is {}\n static_out is \n{}'.format( - dygraph_out, train_out)) + dygraph_out, train_out + ), + ) def test_save_without_training(self): self.test_save_in_eval(with_training=False) class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self.fc = nn.Linear(10, 12) @@ -131,7 +132,6 @@ class LinearNet(nn.Layer): class TestSaveInEvalMode(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -145,15 +145,17 @@ class TestSaveInEvalMode(unittest.TestCase): x.stop_gradient = False dygraph_out = net(x) loss = paddle.mean(dygraph_out) - sgd = paddle.optimizer.SGD(learning_rate=0.001, - parameters=net.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.001, parameters=net.parameters() + ) loss.backward() sgd.step() # switch eval mode firstly net.eval() # save directly net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec(shape=[-1, 10])]) + net, input_spec=[paddle.static.InputSpec(shape=[-1, 10])] + ) model_path = os.path.join(self.temp_dir.name, 'linear_net') paddle.jit.save(net, model_path) @@ -169,11 +171,12 @@ class TestSaveInEvalMode(unittest.TestCase): infer_out.numpy(), rtol=1e-05, err_msg='eval_out is {}\n infer_out is \n{}'.format( - eval_out, infer_out)) + eval_out, infer_out + ), + ) class TestEvalAfterSave(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -186,8 +189,9 @@ class TestEvalAfterSave(unittest.TestCase): x.stop_gradient = False dy_out = net(x) loss = paddle.mean(dy_out) - sgd = paddle.optimizer.SGD(learning_rate=0.001, - parameters=net.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.001, parameters=net.parameters() + ) loss.backward() sgd.step() x = paddle.randn((2, 10, 12)).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py index 7d407495402..bc4ce831142 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py @@ -37,44 +37,49 @@ if paddle.fluid.is_compiled_with_cuda(): class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=True, - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=True, + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=None, - bias_attr=None, - act=act, - use_cudnn=use_cudnn) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=None, + bias_attr=None, + act=act, + use_cudnn=use_cudnn, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -83,33 +88,30 @@ class SimpleImgConvPool(fluid.dygraph.Layer): class MNIST(fluid.dygraph.Layer): - def __init__(self): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(1, - 20, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_1 = SimpleImgConvPool( + 1, 20, 5, 2, 2, act="relu" + ) - self._simple_img_conv_pool_2 = SimpleImgConvPool(20, - 50, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_2 = SimpleImgConvPool( + 20, 50, 5, 2, 2, act="relu" + ) self.pool_2_shape = 50 * 4 * 4 SIZE = 10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 - self._fc = Linear(self.pool_2_shape, - 10, - param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=scale)), - act="softmax") + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 + self._fc = Linear( + self.pool_2_shape, + 10, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + act="softmax", + ) def forward(self, inputs, label=None): x = self.inference(inputs) @@ -131,15 +133,19 @@ class MNIST(fluid.dygraph.Layer): class TestMNIST(unittest.TestCase): - def setUp(self): self.epoch_num = 1 self.batch_size = 64 - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() - self.train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=self.batch_size, - drop_last=True) + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + self.train_reader = paddle.batch( + paddle.dataset.mnist.train(), + batch_size=self.batch_size, + drop_last=True, + ) self.temp_dir = tempfile.TemporaryDirectory() def tearDown(self): @@ -167,7 +173,9 @@ class TestMNISTWithToStatic(TestMNIST): static_loss, rtol=1e-05, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) with _test_eager_guard(): dygraph_loss = self.train_dygraph() static_loss = self.train_static() @@ -176,7 +184,9 @@ class TestMNISTWithToStatic(TestMNIST): static_loss, rtol=1e-05, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) def test_mnist_declarative_cpu_vs_mkldnn(self): dygraph_loss_cpu = self.train_dygraph() @@ -190,7 +200,9 @@ class TestMNISTWithToStatic(TestMNIST): dygraph_loss_mkldnn, rtol=1e-05, err_msg='cpu dygraph is {}\n mkldnn dygraph is \n{}'.format( - dygraph_loss_cpu, dygraph_loss_mkldnn)) + dygraph_loss_cpu, dygraph_loss_mkldnn + ), + ) def train(self, to_static=False): @@ -201,17 +213,21 @@ class TestMNISTWithToStatic(TestMNIST): mnist = MNIST() if to_static: mnist = paddle.jit.to_static(mnist) - adam = AdamOptimizer(learning_rate=0.001, - parameter_list=mnist.parameters()) + adam = AdamOptimizer( + learning_rate=0.001, parameter_list=mnist.parameters() + ) for epoch in range(self.epoch_num): start = time() for batch_id, data in enumerate(self.train_reader()): - dy_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -226,66 +242,87 @@ class TestMNISTWithToStatic(TestMNIST): mnist.clear_gradients() if batch_id % 10 == 0: print( - "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}" - .format(epoch, batch_id, avg_loss.numpy(), - acc.numpy(), - time() - start)) + "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}".format( + epoch, + batch_id, + avg_loss.numpy(), + acc.numpy(), + time() - start, + ) + ) start = time() if batch_id == 50: mnist.eval() prediction, acc, avg_loss = mnist(img, label) loss_data.append(avg_loss.numpy()[0]) # new save load check - self.check_jit_save_load(mnist, [dy_x_data], [img], - to_static, prediction) + self.check_jit_save_load( + mnist, [dy_x_data], [img], to_static, prediction + ) break return loss_data def check_jit_save_load(self, model, inputs, input_spec, to_static, gt_out): if to_static: infer_model_path = os.path.join( - self.temp_dir.name, 'test_mnist_inference_model_by_jit_save') + self.temp_dir.name, 'test_mnist_inference_model_by_jit_save' + ) model_save_dir = os.path.join(self.temp_dir.name, 'inference') model_save_prefix = os.path.join(model_save_dir, 'mnist') model_filename = "mnist" + INFER_MODEL_SUFFIX params_filename = "mnist" + INFER_PARAMS_SUFFIX - fluid.dygraph.jit.save(layer=model, - path=model_save_prefix, - input_spec=input_spec, - output_spec=[gt_out]) + fluid.dygraph.jit.save( + layer=model, + path=model_save_prefix, + input_spec=input_spec, + output_spec=[gt_out], + ) # load in static mode static_infer_out = self.jit_load_and_run_inference_static( - model_save_dir, model_filename, params_filename, inputs) - np.testing.assert_allclose(gt_out.numpy(), - static_infer_out, - rtol=1e-05) + model_save_dir, model_filename, params_filename, inputs + ) + np.testing.assert_allclose( + gt_out.numpy(), static_infer_out, rtol=1e-05 + ) # load in dygraph mode dygraph_infer_out = self.jit_load_and_run_inference_dygraph( - model_save_prefix, inputs) - np.testing.assert_allclose(gt_out.numpy(), - dygraph_infer_out, - rtol=1e-05) + model_save_prefix, inputs + ) + np.testing.assert_allclose( + gt_out.numpy(), dygraph_infer_out, rtol=1e-05 + ) # load in Paddle-Inference - predictor_infer_out = self.predictor_load_and_run_inference_analysis( - model_save_dir, model_filename, params_filename, inputs) - np.testing.assert_allclose(gt_out.numpy(), - predictor_infer_out, - rtol=1e-05) + predictor_infer_out = ( + self.predictor_load_and_run_inference_analysis( + model_save_dir, model_filename, params_filename, inputs + ) + ) + np.testing.assert_allclose( + gt_out.numpy(), predictor_infer_out, rtol=1e-05 + ) @switch_to_static_graph - def jit_load_and_run_inference_static(self, model_path, model_filename, - params_filename, inputs): + def jit_load_and_run_inference_static( + self, model_path, model_filename, params_filename, inputs + ): paddle.enable_static() exe = fluid.Executor(self.place) - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(dirname=model_path, - executor=exe, - model_filename=model_filename, - params_filename=params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + dirname=model_path, + executor=exe, + model_filename=model_filename, + params_filename=params_filename, + ) assert len(inputs) == len(feed_target_names) - results = exe.run(inference_program, - feed=dict(zip(feed_target_names, inputs)), - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed=dict(zip(feed_target_names, inputs)), + fetch_list=fetch_targets, + ) return np.array(results[0]) @@ -294,12 +331,13 @@ class TestMNISTWithToStatic(TestMNIST): pred = infer_net(inputs[0]) return pred.numpy() - def predictor_load_and_run_inference_analysis(self, model_path, - model_filename, - params_filename, inputs): - output = PredictorTools(model_path, model_filename, params_filename, - inputs) - out, = output() + def predictor_load_and_run_inference_analysis( + self, model_path, model_filename, params_filename, inputs + ): + output = PredictorTools( + model_path, model_filename, params_filename, inputs + ) + (out,) = output() return out diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_amp.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_amp.py index 56b55766af6..3610e72b953 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_amp.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_amp.py @@ -24,7 +24,6 @@ if paddle.fluid.is_compiled_with_cuda(): class TestAMP(TestMNIST): - def train_static(self): return self.train(to_static=True) @@ -43,7 +42,9 @@ class TestAMP(TestMNIST): rtol=1e-05, atol=0.001, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) def train(self, to_static=False): paddle.seed(SEED) @@ -53,8 +54,9 @@ class TestAMP(TestMNIST): print("Successfully to apply @to_static.") mnist = paddle.jit.to_static(mnist) - adam = AdamOptimizer(learning_rate=0.001, - parameter_list=mnist.parameters()) + adam = AdamOptimizer( + learning_rate=0.001, parameter_list=mnist.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -62,10 +64,14 @@ class TestAMP(TestMNIST): for epoch in range(self.epoch_num): start = time() for batch_id, data in enumerate(self.train_reader()): - dy_x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -83,9 +89,14 @@ class TestAMP(TestMNIST): mnist.clear_gradients() if batch_id % 10 == 0: print( - "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}" - .format(epoch, batch_id, avg_loss.numpy(), acc.numpy(), - time() - start)) + "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}".format( + epoch, + batch_id, + avg_loss.numpy(), + acc.numpy(), + time() - start, + ) + ) start = time() if batch_id == 50: break diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_pure_fp16.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_pure_fp16.py index beca9d3f933..8bdf8153742 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_pure_fp16.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_pure_fp16.py @@ -23,7 +23,6 @@ if paddle.fluid.is_compiled_with_cuda(): class TestPureFP16(TestMNIST): - def train_static(self): return self.train(to_static=True) @@ -41,7 +40,9 @@ class TestPureFP16(TestMNIST): rtol=1e-05, atol=0.001, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) def train(self, to_static=False): np.random.seed(SEED) @@ -58,33 +59,39 @@ class TestPureFP16(TestMNIST): build_strategy.enable_inplace = False mnist = paddle.jit.to_static(mnist, build_strategy=build_strategy) - optimizer = paddle.optimizer.Adam(learning_rate=0.001, - parameters=mnist.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.001, parameters=mnist.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - mnist, optimizer = paddle.amp.decorate(models=mnist, - optimizers=optimizer, - level='O2', - save_dtype='float32') + mnist, optimizer = paddle.amp.decorate( + models=mnist, optimizers=optimizer, level='O2', save_dtype='float32' + ) loss_data = [] for epoch in range(self.epoch_num): start = time() for batch_id, data in enumerate(self.train_reader()): - dy_x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) label.stop_gradient = True - with paddle.amp.auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2'): + with paddle.amp.auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O2', + ): prediction, acc, avg_loss = mnist(img, label=label) scaled = scaler.scale(avg_loss) @@ -96,9 +103,14 @@ class TestPureFP16(TestMNIST): mnist.clear_gradients() if batch_id % 2 == 0: print( - "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}" - .format(epoch, batch_id, avg_loss.numpy(), acc.numpy(), - time() - start)) + "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}".format( + epoch, + batch_id, + avg_loss.numpy(), + acc.numpy(), + time() - start, + ) + ) start = time() if batch_id == 10: break diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mobile_net.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mobile_net.py index ca9a78f5f30..e13a41db89a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mobile_net.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mobile_net.py @@ -39,32 +39,35 @@ program_translator = ProgramTranslator() class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - filter_size, - num_filters, - stride, - padding, - channels=None, - num_groups=1, - act='relu', - use_cudnn=True, - name=None): + def __init__( + self, + num_channels, + filter_size, + num_filters, + stride, + padding, + channels=None, + num_groups=1, + act='relu', + use_cudnn=True, + name=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=padding, - groups=num_groups, - act=None, - use_cudnn=use_cudnn, - param_attr=ParamAttr(initializer=MSRA(), - name=self.full_name() + - "_weights"), - bias_attr=False) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + groups=num_groups, + act=None, + use_cudnn=use_cudnn, + param_attr=ParamAttr( + initializer=MSRA(), name=self.full_name() + "_weights" + ), + bias_attr=False, + ) self._batch_norm = BatchNorm( num_filters, @@ -72,7 +75,8 @@ class ConvBNLayer(fluid.dygraph.Layer): param_attr=ParamAttr(name=self.full_name() + "_bn" + "_scale"), bias_attr=ParamAttr(name=self.full_name() + "_bn" + "_offset"), moving_mean_name=self.full_name() + "_bn" + '_mean', - moving_variance_name=self.full_name() + "_bn" + '_variance') + moving_variance_name=self.full_name() + "_bn" + '_variance', + ) def forward(self, inputs, if_act=False): y = self._conv(inputs) @@ -83,32 +87,35 @@ class ConvBNLayer(fluid.dygraph.Layer): class DepthwiseSeparable(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters1, - num_filters2, - num_groups, - stride, - scale, - name=None): + def __init__( + self, + num_channels, + num_filters1, + num_filters2, + num_groups, + stride, + scale, + name=None, + ): super(DepthwiseSeparable, self).__init__() - self._depthwise_conv = ConvBNLayer(num_channels=num_channels, - num_filters=int(num_filters1 * - scale), - filter_size=3, - stride=stride, - padding=1, - num_groups=int(num_groups * scale), - use_cudnn=True) + self._depthwise_conv = ConvBNLayer( + num_channels=num_channels, + num_filters=int(num_filters1 * scale), + filter_size=3, + stride=stride, + padding=1, + num_groups=int(num_groups * scale), + use_cudnn=True, + ) self._pointwise_conv = ConvBNLayer( num_channels=int(num_filters1 * scale), filter_size=1, num_filters=int(num_filters2 * scale), stride=1, - padding=0) + padding=0, + ) def forward(self, inputs): y = self._depthwise_conv(inputs) @@ -117,118 +124,148 @@ class DepthwiseSeparable(fluid.dygraph.Layer): class MobileNetV1(fluid.dygraph.Layer): - def __init__(self, scale=1.0, class_dim=1000): super(MobileNetV1, self).__init__() self.scale = scale self.dwsl = [] - self.conv1 = ConvBNLayer(num_channels=3, - filter_size=3, - channels=3, - num_filters=int(32 * scale), - stride=2, - padding=1) - - dws21 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 32 * scale), - num_filters1=32, - num_filters2=64, - num_groups=32, - stride=1, - scale=scale), - name="conv2_1") + self.conv1 = ConvBNLayer( + num_channels=3, + filter_size=3, + channels=3, + num_filters=int(32 * scale), + stride=2, + padding=1, + ) + + dws21 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(32 * scale), + num_filters1=32, + num_filters2=64, + num_groups=32, + stride=1, + scale=scale, + ), + name="conv2_1", + ) self.dwsl.append(dws21) - dws22 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 64 * scale), - num_filters1=64, - num_filters2=128, - num_groups=64, - stride=2, - scale=scale), - name="conv2_2") + dws22 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(64 * scale), + num_filters1=64, + num_filters2=128, + num_groups=64, + stride=2, + scale=scale, + ), + name="conv2_2", + ) self.dwsl.append(dws22) - dws31 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 128 * scale), - num_filters1=128, - num_filters2=128, - num_groups=128, - stride=1, - scale=scale), - name="conv3_1") + dws31 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(128 * scale), + num_filters1=128, + num_filters2=128, + num_groups=128, + stride=1, + scale=scale, + ), + name="conv3_1", + ) self.dwsl.append(dws31) - dws32 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 128 * scale), - num_filters1=128, - num_filters2=256, - num_groups=128, - stride=2, - scale=scale), - name="conv3_2") + dws32 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(128 * scale), + num_filters1=128, + num_filters2=256, + num_groups=128, + stride=2, + scale=scale, + ), + name="conv3_2", + ) self.dwsl.append(dws32) - dws41 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 256 * scale), - num_filters1=256, - num_filters2=256, - num_groups=256, - stride=1, - scale=scale), - name="conv4_1") + dws41 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(256 * scale), + num_filters1=256, + num_filters2=256, + num_groups=256, + stride=1, + scale=scale, + ), + name="conv4_1", + ) self.dwsl.append(dws41) - dws42 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 256 * scale), - num_filters1=256, - num_filters2=512, - num_groups=256, - stride=2, - scale=scale), - name="conv4_2") + dws42 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(256 * scale), + num_filters1=256, + num_filters2=512, + num_groups=256, + stride=2, + scale=scale, + ), + name="conv4_2", + ) self.dwsl.append(dws42) for i in range(5): - tmp = self.add_sublayer(sublayer=DepthwiseSeparable( + tmp = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(512 * scale), + num_filters1=512, + num_filters2=512, + num_groups=512, + stride=1, + scale=scale, + ), + name="conv5_" + str(i + 1), + ) + self.dwsl.append(tmp) + + dws56 = self.add_sublayer( + sublayer=DepthwiseSeparable( num_channels=int(512 * scale), num_filters1=512, - num_filters2=512, + num_filters2=1024, num_groups=512, - stride=1, - scale=scale), - name="conv5_" + str(i + 1)) - self.dwsl.append(tmp) - - dws56 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 512 * scale), - num_filters1=512, - num_filters2=1024, - num_groups=512, - stride=2, - scale=scale), - name="conv5_6") + stride=2, + scale=scale, + ), + name="conv5_6", + ) self.dwsl.append(dws56) - dws6 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 1024 * scale), - num_filters1=1024, - num_filters2=1024, - num_groups=1024, - stride=1, - scale=scale), - name="conv6") + dws6 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(1024 * scale), + num_filters1=1024, + num_filters2=1024, + num_groups=1024, + stride=1, + scale=scale, + ), + name="conv6", + ) self.dwsl.append(dws6) self.pool2d_avg = Pool2D(pool_type='avg', global_pooling=True) - self.out = Linear(int(1024 * scale), - class_dim, - param_attr=ParamAttr(initializer=MSRA(), - name=self.full_name() + - "fc7_weights"), - bias_attr=ParamAttr(name="fc7_offset")) + self.out = Linear( + int(1024 * scale), + class_dim, + param_attr=ParamAttr( + initializer=MSRA(), name=self.full_name() + "fc7_weights" + ), + bias_attr=ParamAttr(name="fc7_offset"), + ) @declarative def forward(self, inputs): @@ -242,7 +279,6 @@ class MobileNetV1(fluid.dygraph.Layer): class InvertedResidualUnit(fluid.dygraph.Layer): - def __init__( self, num_channels, @@ -255,30 +291,36 @@ class InvertedResidualUnit(fluid.dygraph.Layer): ): super(InvertedResidualUnit, self).__init__() num_expfilter = int(round(num_in_filter * expansion_factor)) - self._expand_conv = ConvBNLayer(num_channels=num_channels, - num_filters=num_expfilter, - filter_size=1, - stride=1, - padding=0, - act=None, - num_groups=1) - - self._bottleneck_conv = ConvBNLayer(num_channels=num_expfilter, - num_filters=num_expfilter, - filter_size=filter_size, - stride=stride, - padding=padding, - num_groups=num_expfilter, - act=None, - use_cudnn=True) - - self._linear_conv = ConvBNLayer(num_channels=num_expfilter, - num_filters=num_filters, - filter_size=1, - stride=1, - padding=0, - act=None, - num_groups=1) + self._expand_conv = ConvBNLayer( + num_channels=num_channels, + num_filters=num_expfilter, + filter_size=1, + stride=1, + padding=0, + act=None, + num_groups=1, + ) + + self._bottleneck_conv = ConvBNLayer( + num_channels=num_expfilter, + num_filters=num_expfilter, + filter_size=filter_size, + stride=stride, + padding=padding, + num_groups=num_expfilter, + act=None, + use_cudnn=True, + ) + + self._linear_conv = ConvBNLayer( + num_channels=num_expfilter, + num_filters=num_filters, + filter_size=1, + stride=1, + padding=0, + act=None, + num_groups=1, + ) def forward(self, inputs, ifshortcut): y = self._expand_conv(inputs, if_act=True) @@ -290,29 +332,33 @@ class InvertedResidualUnit(fluid.dygraph.Layer): class InvresiBlocks(fluid.dygraph.Layer): - def __init__(self, in_c, t, c, n, s): super(InvresiBlocks, self).__init__() - self._first_block = InvertedResidualUnit(num_channels=in_c, - num_in_filter=in_c, - num_filters=c, - stride=s, - filter_size=3, - padding=1, - expansion_factor=t) + self._first_block = InvertedResidualUnit( + num_channels=in_c, + num_in_filter=in_c, + num_filters=c, + stride=s, + filter_size=3, + padding=1, + expansion_factor=t, + ) self._inv_blocks = [] for i in range(1, n): - tmp = self.add_sublayer(sublayer=InvertedResidualUnit( - num_channels=c, - num_in_filter=c, - num_filters=c, - stride=1, - filter_size=3, - padding=1, - expansion_factor=t), - name=self.full_name() + "_" + str(i + 1)) + tmp = self.add_sublayer( + sublayer=InvertedResidualUnit( + num_channels=c, + num_in_filter=c, + num_filters=c, + stride=1, + filter_size=3, + padding=1, + expansion_factor=t, + ), + name=self.full_name() + "_" + str(i + 1), + ) self._inv_blocks.append(tmp) def forward(self, inputs): @@ -323,7 +369,6 @@ class InvresiBlocks(fluid.dygraph.Layer): class MobileNetV2(fluid.dygraph.Layer): - def __init__(self, class_dim=1000, scale=1.0): super(MobileNetV2, self).__init__() self.scale = scale @@ -339,48 +384,54 @@ class MobileNetV2(fluid.dygraph.Layer): (6, 320, 1, 1), ] - #1. conv1 - self._conv1 = ConvBNLayer(num_channels=3, - num_filters=int(32 * scale), - filter_size=3, - stride=2, - act=None, - padding=1) - - #2. bottleneck sequences + # 1. conv1 + self._conv1 = ConvBNLayer( + num_channels=3, + num_filters=int(32 * scale), + filter_size=3, + stride=2, + act=None, + padding=1, + ) + + # 2. bottleneck sequences self._invl = [] i = 1 in_c = int(32 * scale) for layer_setting in bottleneck_params_list: t, c, n, s = layer_setting i += 1 - tmp = self.add_sublayer(sublayer=InvresiBlocks(in_c=in_c, - t=t, - c=int(c * scale), - n=n, - s=s), - name='conv' + str(i)) + tmp = self.add_sublayer( + sublayer=InvresiBlocks( + in_c=in_c, t=t, c=int(c * scale), n=n, s=s + ), + name='conv' + str(i), + ) self._invl.append(tmp) in_c = int(c * scale) - #3. last_conv + # 3. last_conv self._out_c = int(1280 * scale) if scale > 1.0 else 1280 - self._conv9 = ConvBNLayer(num_channels=in_c, - num_filters=self._out_c, - filter_size=1, - stride=1, - act=None, - padding=0) - - #4. pool + self._conv9 = ConvBNLayer( + num_channels=in_c, + num_filters=self._out_c, + filter_size=1, + stride=1, + act=None, + padding=0, + ) + + # 4. pool self._pool2d_avg = Pool2D(pool_type='avg', global_pooling=True) - #5. fc + # 5. fc tmp_param = ParamAttr(name=self.full_name() + "fc10_weights") - self._fc = Linear(self._out_c, - class_dim, - param_attr=tmp_param, - bias_attr=ParamAttr(name="fc10_offset")) + self._fc = Linear( + self._out_c, + class_dim, + param_attr=tmp_param, + bias_attr=ParamAttr(name="fc10_offset"), + ) @declarative def forward(self, inputs): @@ -399,7 +450,8 @@ def create_optimizer(args, parameter_list): learning_rate=args.lr, momentum=args.momentum_rate, regularization=fluid.regularizer.L2Decay(args.l2_decay), - parameter_list=parameter_list) + parameter_list=parameter_list, + ) return optimizer @@ -430,8 +482,11 @@ class Args(object): class_dim = 50 print_step = 1 train_step = 10 - place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) model_save_dir = None model_save_prefix = None model_filename = None @@ -477,8 +532,9 @@ def train_mobilenet(args, to_static): t_end = time.time() softmax_out = fluid.layers.softmax(out, use_cudnn=False) - loss = fluid.layers.cross_entropy(input=softmax_out, - label=label) + loss = fluid.layers.cross_entropy( + input=softmax_out, label=label + ) avg_loss = paddle.mean(x=loss) acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) @@ -493,17 +549,29 @@ def train_mobilenet(args, to_static): t2 = time.time() train_batch_elapse = t2 - t1 if batch_id % args.print_step == 0: - print("epoch id: %d, batch step: %d, avg_loss %0.5f acc_top1 %0.5f acc_top5 %0.5f %2.4f sec net_t:%2.4f back_t:%2.4f read_t:%2.4f" % \ - (eop, batch_id, avg_loss.numpy(), acc_top1.numpy(), acc_top5.numpy(), train_batch_elapse, - t_end - t_start, t_end_back - t_start_back, t1 - t_last)) + print( + "epoch id: %d, batch step: %d, avg_loss %0.5f acc_top1 %0.5f acc_top5 %0.5f %2.4f sec net_t:%2.4f back_t:%2.4f read_t:%2.4f" + % ( + eop, + batch_id, + avg_loss.numpy(), + acc_top1.numpy(), + acc_top5.numpy(), + train_batch_elapse, + t_end - t_start, + t_end_back - t_start_back, + t1 - t_last, + ) + ) batch_id += 1 t_last = time.time() if batch_id > args.train_step: if to_static: fluid.dygraph.jit.save(net, args.model_save_prefix) else: - fluid.dygraph.save_dygraph(net.state_dict(), - args.dy_state_dict_save_path) + fluid.dygraph.save_dygraph( + net.state_dict(), args.dy_state_dict_save_path + ) break return np.array(loss_data) @@ -514,15 +582,22 @@ def predict_static(args, data): exe = fluid.Executor(args.place) # load inference model - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(args.model_save_dir, - executor=exe, - model_filename=args.model_filename, - params_filename=args.params_filename) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + args.model_save_dir, + executor=exe, + model_filename=args.model_filename, + params_filename=args.params_filename, + ) + + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res[0] @@ -554,51 +629,57 @@ def predict_dygraph_jit(args, data): def predict_analysis_inference(args, data): - output = PredictorTools(args.model_save_dir, args.model_filename, - args.params_filename, [data]) - out, = output() + output = PredictorTools( + args.model_save_dir, args.model_filename, args.params_filename, [data] + ) + (out,) = output() return out class TestMobileNet(unittest.TestCase): - def setUp(self): self.args = Args() self.temp_dir = tempfile.TemporaryDirectory() - self.args.model_save_dir = os.path.join(self.temp_dir.name, - "./inference") + self.args.model_save_dir = os.path.join( + self.temp_dir.name, "./inference" + ) def tearDown(self): self.temp_dir.cleanup() def train(self, model_name, to_static): self.args.model = model_name - self.args.model_save_prefix = os.path.join(self.temp_dir.name, - "./inference/" + model_name) + self.args.model_save_prefix = os.path.join( + self.temp_dir.name, "./inference/" + model_name + ) self.args.model_filename = model_name + INFER_MODEL_SUFFIX self.args.params_filename = model_name + INFER_PARAMS_SUFFIX self.args.dy_state_dict_save_path = os.path.join( - self.temp_dir.name, model_name + ".dygraph") + self.temp_dir.name, model_name + ".dygraph" + ) out = train_mobilenet(self.args, to_static) return out def assert_same_loss(self, model_name): dy_out = self.train(model_name, to_static=False) st_out = self.train(model_name, to_static=True) - np.testing.assert_allclose(dy_out, - st_out, - rtol=1e-05, - err_msg='dy_out: {}, st_out: {}'.format( - dy_out, st_out)) + np.testing.assert_allclose( + dy_out, + st_out, + rtol=1e-05, + err_msg='dy_out: {}, st_out: {}'.format(dy_out, st_out), + ) def assert_same_predict(self, model_name): self.args.model = model_name - self.args.model_save_prefix = os.path.join(self.temp_dir.name, - "./inference/" + model_name) + self.args.model_save_prefix = os.path.join( + self.temp_dir.name, "./inference/" + model_name + ) self.args.model_filename = model_name + INFER_MODEL_SUFFIX self.args.params_filename = model_name + INFER_PARAMS_SUFFIX self.args.dy_state_dict_save_path = os.path.join( - self.temp_dir.name, model_name + ".dygraph") + self.temp_dir.name, model_name + ".dygraph" + ) local_random = np.random.RandomState(SEED) image = local_random.random_sample([1, 3, 224, 224]).astype('float32') dy_pre = predict_dygraph(self.args, image) @@ -609,20 +690,25 @@ class TestMobileNet(unittest.TestCase): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) np.testing.assert_allclose( predictor_pre, st_pre, rtol=1e-05, atol=1e-05, err_msg='inference_pred_res:\n {}\n, st_pre: \n{}.'.format( - predictor_pre, st_pre)) + predictor_pre, st_pre + ), + ) def test_mobile_net(self): # MobileNet-V1 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_multi_forward.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_multi_forward.py index 3b81cd0749f..f320e9f010c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_multi_forward.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_multi_forward.py @@ -18,20 +18,20 @@ import unittest class MyLayer(paddle.nn.Layer): - def __init__(self): super().__init__() self.linear = paddle.nn.Linear(1, 1) - @paddle.jit.to_static(input_spec=[ - paddle.static.InputSpec(shape=[None, None], dtype=paddle.float32) - ]) + @paddle.jit.to_static( + input_spec=[ + paddle.static.InputSpec(shape=[None, None], dtype=paddle.float32) + ] + ) def forward(self, x): return self.linear(x) class TestBackward(unittest.TestCase): - def test_order_0(self): """ loss = 1 * w * 1 + 2 * w * 2 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_op_attr.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_op_attr.py index 9bdc6cfcb2c..8bfc7a83dfe 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_op_attr.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_op_attr.py @@ -19,7 +19,6 @@ from paddle.static import InputSpec class MySub(paddle.nn.Layer): - def __init__(self): super(MySub, self).__init__() @@ -28,7 +27,6 @@ class MySub(paddle.nn.Layer): class NetWithOpAttr(paddle.nn.Layer): - def __init__(self, in_num, out_num): super(NetWithOpAttr, self).__init__() @@ -44,7 +42,7 @@ class NetWithOpAttr(paddle.nn.Layer): @paddle.jit.to_static(input_spec=[InputSpec([10, 16])]) def with_cond(self, x): - if paddle.mean(x) > 0.: + if paddle.mean(x) > 0.0: out = self.linear(x) else: out = self.sub(x, x) @@ -53,7 +51,6 @@ class NetWithOpAttr(paddle.nn.Layer): class CheckOpAttr(unittest.TestCase): - def setUp(self): self.in_num = 16 self.out_num = 16 @@ -65,7 +62,7 @@ class CheckOpAttr(unittest.TestCase): "int_val": 10, "int_vals": [10, 20], "float_val": 3.8, - "float_vals": [3.8, -0.2] + "float_vals": [3.8, -0.2], } self.bn_attrs = {"bool_val": True, "bool_vals": [True, False]} self.sub_attrs = {"int_vals": [10, 20], "bool_vals": [True, False]} @@ -75,7 +72,7 @@ class CheckOpAttr(unittest.TestCase): 'elementwise_add': self.fc_attrs, 'batch_norm': self.bn_attrs, 'tanh': self.bn_attrs, - 'elementwise_sub': self.sub_attrs + 'elementwise_sub': self.sub_attrs, } def test_set_op_attrs(self): @@ -89,8 +86,9 @@ class CheckOpAttr(unittest.TestCase): self.assertEqual(len(net.linear._forward_pre_hooks), 1) self.assertEqual(len(net.linear._forward_post_hooks), 1) # to_static - net = paddle.jit.to_static(net, - input_spec=[InputSpec.from_tensor(self.x)]) + net = paddle.jit.to_static( + net, input_spec=[InputSpec.from_tensor(self.x)] + ) # assert attrs have be set. self.check_op_attrs(net.forward.concrete_program.main_program) @@ -103,7 +101,8 @@ class CheckOpAttr(unittest.TestCase): for cur_block in main_program.blocks: ops = cur_block.ops for op in ops: - if op.type not in self.infos: continue + if op.type not in self.infos: + continue for attr_name, expect_vals in self.infos[op.type].items(): op_vals = op.desc.attr(attr_name) if not isinstance(expect_vals, list): @@ -120,8 +119,9 @@ class CheckOpAttr(unittest.TestCase): def test_set_op_attrs_with_sub_block(self): net = NetWithOpAttr(self.in_num, self.out_num) # set attrs - net.linear._set_op_attrs({"int_vals": [0, - 0]}) # test overwrite behavior + net.linear._set_op_attrs( + {"int_vals": [0, 0]} + ) # test overwrite behavior net.linear._set_op_attrs(self.fc_attrs) net.bn._set_op_attrs(self.bn_attrs) net.sub._set_op_attrs(self.sub_attrs) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py index 8247787b515..90444ede53c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py @@ -20,7 +20,6 @@ from paddle.jit import to_static, ProgramTranslator class NetWithParameterList(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(NetWithParameterList, self).__init__() weight = self.create_parameter([in_size, out_size]) @@ -36,7 +35,6 @@ class NetWithParameterList(paddle.nn.Layer): class NetWithParameterListIter(NetWithParameterList): - def __init__(self, in_size, out_size): super(NetWithParameterListIter, self).__init__(in_size, out_size) @@ -51,7 +49,6 @@ class NetWithParameterListIter(NetWithParameterList): class TestParameterList(unittest.TestCase): - def setUp(self): self.seed = 2021 self.iter_num = 5 @@ -89,13 +86,14 @@ class TestParameterList(unittest.TestCase): class NetWithRawParamList(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(NetWithRawParamList, self).__init__() - weight = self.add_parameter('w', - self.create_parameter([in_size, out_size])) + weight = self.add_parameter( + 'w', self.create_parameter([in_size, out_size]) + ) bias = self.add_parameter( - 'b', self.create_parameter([out_size], is_bias=True)) + 'b', self.create_parameter([out_size], is_bias=True) + ) self.params = [weight] self.bias_dict = {'b': bias} @@ -108,7 +106,6 @@ class NetWithRawParamList(paddle.nn.Layer): class TestRawParameterList(unittest.TestCase): - def setUp(self): self.seed = 2021 self.iter_num = 5 @@ -142,7 +139,6 @@ class TestRawParameterList(unittest.TestCase): class NetWithSubLayerParamList(paddle.nn.Layer): - def __init__(self, sub_layer): super(NetWithSubLayerParamList, self).__init__() self.sub_layer = sub_layer @@ -158,7 +154,6 @@ class NetWithSubLayerParamList(paddle.nn.Layer): class TestSubLayerParameterList(TestRawParameterList): - def init_net(self): fc = paddle.nn.Linear(10, 3) self.net = NetWithSubLayerParamList(fc) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_params_no_grad.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_params_no_grad.py index f44faa642a0..24ff71b066d 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_params_no_grad.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_params_no_grad.py @@ -20,7 +20,6 @@ import unittest class Net(nn.Layer): - def __init__(self): super(Net, self).__init__() self.emb1 = nn.Embedding(100, 16) @@ -54,10 +53,11 @@ def train(): class TestParamsNoGrad(unittest.TestCase): - def test_two_card(self): - if paddle.is_compiled_with_cuda() and len( - paddle.static.cuda_places()) > 1: + if ( + paddle.is_compiled_with_cuda() + and len(paddle.static.cuda_places()) > 1 + ): dist.spawn(train, nprocs=2, gpus='0,1') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py index 72dca36d210..d11f4e9f15c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py @@ -54,7 +54,6 @@ def fake_data(shape): class TestWithNestedInput(unittest.TestCase): - def setUp(self): self.x = None self.y = None @@ -62,15 +61,14 @@ class TestWithNestedInput(unittest.TestCase): def fake_input(self): self.x = fake_data([10, 16]) self.y = [ - fake_data([10, 16]), "preprocess_cmd", 64, { - 'z': [fake_data([10, 12]), - fake_data([10, 12])], + fake_data([10, 16]), + "preprocess_cmd", + 64, + { + 'z': [fake_data([10, 12]), fake_data([10, 12])], 'c': fake_data([10, 10]), - 'd': { - 'da': 12, - 'dc': fake_data([10, 10]) - } - } + 'd': {'da': 12, 'dc': fake_data([10, 10])}, + }, ] def _run(self, to_static): @@ -92,7 +90,6 @@ class TestWithNestedInput(unittest.TestCase): class TestWithNestedOutput(unittest.TestCase): - def setUp(self): self.x = None self.y = None @@ -120,17 +117,17 @@ class TestWithNestedOutput(unittest.TestCase): self.assertTrue(len(dygraph_res) == len(static_res)) for dy_var, st_var in zip(dygraph_res, static_res): - if isinstance(dy_var, - (fluid.core.VarBase, fluid.core.eager.Tensor)): - np.testing.assert_allclose(dy_var.numpy(), - st_var.numpy(), - rtol=1e-05) + if isinstance( + dy_var, (fluid.core.VarBase, fluid.core.eager.Tensor) + ): + np.testing.assert_allclose( + dy_var.numpy(), st_var.numpy(), rtol=1e-05 + ) else: self.assertTrue(dy_var, st_var) class TestWithTrainAndEval(unittest.TestCase): - def test_switch_eval_and_train(self): program_translator = ProgramTranslator() @@ -142,25 +139,27 @@ class TestWithTrainAndEval(unittest.TestCase): _, train_partial_layer = linear_net.forward.program_cache.last()[-1] # check default mode is for training - self.assertEqual(train_partial_layer.program, - train_partial_layer._train_program) + self.assertEqual( + train_partial_layer.program, train_partial_layer._train_program + ) # switch to run test program after `eval()` linear_net.eval() linear_net(x) _, eval_partial_layer = linear_net.forward.program_cache.last()[-1] - self.assertEqual(eval_partial_layer.program, - eval_partial_layer._infer_program) + self.assertEqual( + eval_partial_layer.program, eval_partial_layer._infer_program + ) # switch back into training linear_net.train() linear_net(x) - self.assertEqual(train_partial_layer.program, - train_partial_layer._train_program) + self.assertEqual( + train_partial_layer.program, train_partial_layer._train_program + ) class TestWithNoGrad(unittest.TestCase): - def test_with_no_grad(self): with fluid.dygraph.guard(): linear_net = Linear() @@ -171,18 +170,19 @@ class TestWithNoGrad(unittest.TestCase): linear_net.train() linear_net(x) _, partial_layer = linear_net.forward.program_cache.last()[-1] - self.assertEqual(partial_layer.program, - partial_layer._train_program) + self.assertEqual( + partial_layer.program, partial_layer._train_program + ) class GPT2LMHeadModel(fluid.dygraph.Layer): - def __init__(self): super(GPT2LMHeadModel, self).__init__() self.embedding0 = paddle.nn.Embedding(20, 16) self.embedding1 = paddle.nn.Embedding(20, 32) self.lm_head_weight = paddle.to_tensor( - np.random.rand(2, 3).astype('float32')) + np.random.rand(2, 3).astype('float32') + ) @declarative def forward(self, x): @@ -192,7 +192,6 @@ class GPT2LMHeadModel(fluid.dygraph.Layer): class TestPruneUnusedParamInProgram(unittest.TestCase): - def test_prune(self): input_ids = np.array([[15, 11, 6, 3, 18, 13]]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_place.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_place.py index 96cffc32a19..58e8e7b6728 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_place.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_place.py @@ -17,7 +17,6 @@ import unittest class TestPlace(unittest.TestCase): - def test_place(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_print.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_print.py index 90737695aaf..3603cac199e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_print.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_print.py @@ -152,11 +152,13 @@ def dyfunc_print_continue_vars(x): class TestPrintBase(unittest.TestCase): - def setUp(self): self.input = numpy.ones(5).astype("int32") - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_test_func() def set_test_func(self): @@ -176,7 +178,6 @@ class TestPrintBase(unittest.TestCase): class TestPrintVariable(TestPrintBase): - def set_test_func(self): self.dygraph_func = dyfunc_print_variable @@ -186,37 +187,31 @@ class TestPrintVariable(TestPrintBase): class TestPrintNdArray(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_ndarray class TestPrintWithFormat(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_with_format class TestPrintWithFormat2(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_with_format2 class TestPrintWithIfElse(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_with_ifelse class TestPrintMultipleVar(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_multi_vars class TestPrintContinueVar(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_continue_vars diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py index 7bd415fdc8a..762836a937b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py @@ -26,7 +26,11 @@ from paddle.fluid.dygraph.jit import declarative from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code import paddle.jit.dy2static as _jst -from ifelse_simple_func import dyfunc_with_if_else, dyfunc_with_if_else_early_return1, dyfunc_with_if_else_early_return2 +from ifelse_simple_func import ( + dyfunc_with_if_else, + dyfunc_with_if_else_early_return1, + dyfunc_with_if_else_early_return2, +) np.random.seed(0) @@ -60,8 +64,7 @@ def get_source_code(func): return source_code -class StaticCode1(): - +class StaticCode1: def dyfunc_with_if_else(x_v, label=None): loss = _jst.UndefinedVar('loss') __return_1 = _jst.UndefinedVar('__return_1') @@ -70,11 +73,11 @@ class StaticCode1(): def get_args_0(): nonlocal x_v - return x_v, + return (x_v,) def set_args_0(__args): nonlocal x_v - x_v, = __args + (x_v,) = __args def true_fn_0(): nonlocal x_v @@ -86,12 +89,15 @@ class StaticCode1(): x_v = x_v + 1 return - _jst.IfElse(paddle.mean(x_v)[0] > 5, - true_fn_0, - false_fn_0, - get_args_0, - set_args_0, ('x_v', ), - push_pop_names=None) + _jst.IfElse( + paddle.mean(x_v)[0] > 5, + true_fn_0, + false_fn_0, + get_args_0, + set_args_0, + ('x_v',), + push_pop_names=None, + ) def get_args_1(): nonlocal __return_0, __return_1, __return_value_0, loss @@ -114,17 +120,19 @@ class StaticCode1(): __return_value_0 = x_v return - _jst.IfElse(label is not None, - true_fn_1, - false_fn_1, - get_args_1, - set_args_1, - ('__return_0', '__return_1', '__return_value_0', 'loss'), - push_pop_names=None) + _jst.IfElse( + label is not None, + true_fn_1, + false_fn_1, + get_args_1, + set_args_1, + ('__return_0', '__return_1', '__return_value_0', 'loss'), + push_pop_names=None, + ) return __return_value_0 -class StaticCode2(): +class StaticCode2: # TODO: Transform return statement def dyfunc_with_if_else(x_v, label=None): loss = _jst.UndefinedVar('loss') @@ -134,11 +142,11 @@ class StaticCode2(): def get_args_2(): nonlocal x_v - return x_v, + return (x_v,) def set_args_2(__args): nonlocal x_v - x_v, = __args + (x_v,) = __args def true_fn_2(): nonlocal x_v @@ -150,12 +158,15 @@ class StaticCode2(): x_v = x_v + 1 return - _jst.IfElse(paddle.mean(x_v)[0] > 5, - true_fn_2, - false_fn_2, - get_args_2, - set_args_2, ('x_v', ), - push_pop_names=None) + _jst.IfElse( + paddle.mean(x_v)[0] > 5, + true_fn_2, + false_fn_2, + get_args_2, + set_args_2, + ('x_v',), + push_pop_names=None, + ) def get_args_3(): nonlocal __return_2, __return_3, __return_value_1, loss @@ -178,18 +189,19 @@ class StaticCode2(): __return_value_1 = x_v return - _jst.IfElse(label is not None, - true_fn_3, - false_fn_3, - get_args_3, - set_args_3, - ('__return_2', '__return_3', '__return_value_1', 'loss'), - push_pop_names=None) + _jst.IfElse( + label is not None, + true_fn_3, + false_fn_3, + get_args_3, + set_args_3, + ('__return_2', '__return_3', '__return_value_1', 'loss'), + push_pop_names=None, + ) return __return_value_1 class NetWithError(fluid.dygraph.layers.Layer): - @declarative def forward(self, x): linear = fluid.dygraph.Linear(32, 64) @@ -198,7 +210,6 @@ class NetWithError(fluid.dygraph.layers.Layer): class TestDygraphToStaticCode(unittest.TestCase): - def setUp(self): # set to print all string diff when assertEqual fails self.maxDiff = None @@ -210,7 +221,8 @@ class TestDygraphToStaticCode(unittest.TestCase): answer = get_source_code(StaticCode1.dyfunc_with_if_else) self.assertEqual( answer.replace('\n', '').replace(' ', ''), - code.replace('\n', '').replace(' ', '')) + code.replace('\n', '').replace(' ', ''), + ) def test_program_translator(self): answer = get_source_code(StaticCode2.dyfunc_with_if_else) @@ -219,11 +231,11 @@ class TestDygraphToStaticCode(unittest.TestCase): print(code) self.assertEqual( answer.replace('\n', '').replace(' ', ''), - code.replace('\n', '').replace(' ', '')) + code.replace('\n', '').replace(' ', ''), + ) class TestEnableDeclarative(unittest.TestCase): - def setUp(self): self.x = np.random.randn(30, 10, 32).astype('float32') self.weight = np.random.randn(32, 64).astype('float32') @@ -240,16 +252,20 @@ class TestEnableDeclarative(unittest.TestCase): self.program_translator.enable(True) with fluid.dygraph.guard(): static_output = self.program_translator.get_output( - simple_func, self.x, self.weight) + simple_func, self.x, self.weight + ) self.program_translator.enable(False) with fluid.dygraph.guard(): dygraph_output = self.program_translator.get_output( - simple_func, self.x, self.weight) - np.testing.assert_allclose(static_output.numpy(), - dygraph_output.numpy(), - rtol=1e-05, - atol=1e-4) + simple_func, self.x, self.weight + ) + np.testing.assert_allclose( + static_output.numpy(), + dygraph_output.numpy(), + rtol=1e-05, + atol=1e-4, + ) def test_enable_disable_get_func(self): @@ -266,14 +282,18 @@ class TestEnableDeclarative(unittest.TestCase): self.assertTrue(callable(dygraph_func)) dygraph_output = dygraph_func(self.x, self.weight) self.assertTrue( - isinstance(dygraph_output, - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + dygraph_output, + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) def test_enable_disable_get_program(self): self.program_translator.enable(True) static_output = self.program_translator.get_program( - simple_func, self.x, self.weight) + simple_func, self.x, self.weight + ) self.assertTrue(isinstance(static_output, tuple)) self.assertEqual(len(static_output), 4) self.assertTrue(isinstance(static_output[0], fluid.Program)) @@ -288,10 +308,14 @@ class TestEnableDeclarative(unittest.TestCase): self.program_translator.enable(False) with fluid.dygraph.guard(): dygraph_output = self.program_translator.get_program( - simple_func, self.x, self.weight) + simple_func, self.x, self.weight + ) self.assertTrue( - isinstance(dygraph_output, - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + dygraph_output, + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) def test_enable_disable_declarative(self): @@ -302,14 +326,15 @@ class TestEnableDeclarative(unittest.TestCase): self.program_translator.enable(False) with fluid.dygraph.guard(): dygraph_output = decorated_simple_func(self.x, self.weight) - np.testing.assert_allclose(static_output.numpy(), - dygraph_output.numpy(), - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + static_output.numpy(), + dygraph_output.numpy(), + rtol=1e-05, + atol=1e-4, + ) class Net(fluid.dygraph.layers.Layer): - def __init__(self): super(Net, self).__init__() @@ -318,7 +343,6 @@ class Net(fluid.dygraph.layers.Layer): class TestErrorWithInitFromStaticMode(unittest.TestCase): - def setUp(self): self.program_translator = ProgramTranslator() self.x = np.random.randn(10, 32).astype('float32') @@ -329,17 +353,18 @@ class TestErrorWithInitFromStaticMode(unittest.TestCase): net = Net() self.program_translator.enable(True) - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): self.program_translator.get_output(net.forward, self.x) - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): self.program_translator.get_program(net.forward, self.x) class SwitchModeNet(paddle.nn.Layer): - def __init__(self): super(SwitchModeNet, self).__init__() @@ -358,7 +383,6 @@ def switch_mode_funciton(): class TestFunctionTrainEvalMode(unittest.TestCase): - def test_switch_mode(self): paddle.disable_static() switch_mode_funciton.eval() @@ -388,7 +412,6 @@ class TestFunctionTrainEvalMode(unittest.TestCase): class TestIfElseEarlyReturn(unittest.TestCase): - def test_ifelse_early_return1(self): answer = np.zeros([2, 2]) + 1 static_func = paddle.jit.to_static(dyfunc_with_if_else_early_return1) @@ -403,7 +426,6 @@ class TestIfElseEarlyReturn(unittest.TestCase): class TestRemoveCommentInDy2St(unittest.TestCase): - def func_with_comment(self): # Comment1 x = paddle.to_tensor([1, 2, 3]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py index b37d4af6b39..12150a3d5a0 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py @@ -32,13 +32,9 @@ program_translator = ProgramTranslator() class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -57,19 +53,26 @@ class SimpleLSTMRNN(fluid.Layer): weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -93,11 +96,12 @@ class SimpleLSTMRNN(fluid.Layer): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) hidden_array[k] = m cell_array[k] = c @@ -107,31 +111,36 @@ class SimpleLSTMRNN(fluid.Layer): step_input = fluid.layers.dropout( step_input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append(step_input) real_res = fluid.layers.concat(res, 1) real_res = fluid.layers.reshape( - real_res, [-1, self._num_steps, self._hidden_size]) + real_res, [-1, self._num_steps, self._hidden_size] + ) last_hidden = fluid.layers.concat(hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -139,11 +148,13 @@ class PtbModel(fluid.Layer): self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', @@ -151,19 +162,26 @@ class PtbModel(fluid.Layer): param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def build_once(self, input, label, init_hidden, init_cell): pass @@ -172,29 +190,34 @@ class PtbModel(fluid.Layer): def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -221,15 +244,18 @@ def train(place): with fluid.dygraph.guard(place): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - dropout=dropout) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + dropout=dropout, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=ptb_model.parameters() + ) for epoch_id in range(max_epoch): @@ -237,10 +263,12 @@ def train(place): iters = 0.0 total_sample = 0 - init_hidden_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) @@ -256,7 +284,8 @@ def train(place): y = to_variable(y_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) out_loss = dy_loss.numpy() dy_loss.backward() @@ -269,15 +298,21 @@ def train(place): if step_id % PRINT_STEP == 0: if step_id == 0: logging.info( - "epoch %d | step %d, loss %0.3f" % - (epoch_id, step_id, total_loss / total_sample)) + "epoch %d | step %d, loss %0.3f" + % (epoch_id, step_id, total_loss / total_sample) + ) avg_batch_time = time.time() else: speed = PRINT_STEP / (time.time() - avg_batch_time) logging.info( "epoch %d | step %d, loss %0.3f, speed %.3f steps/s" - % (epoch_id, step_id, total_loss / total_sample, - speed)) + % ( + epoch_id, + step_id, + total_loss / total_sample, + speed, + ) + ) avg_batch_time = time.time() return out_loss, last_hidden.numpy(), last_cell.numpy() @@ -294,10 +329,12 @@ def train_static(place): class TestPtb(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + ) def test_check_result(self): loss_1, hidden_1, cell_1 = train_static(self.place) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm_v2.py index ac3c40b599a..56b0a44bdb5 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm_v2.py @@ -26,13 +26,9 @@ program_translator = paddle.jit.ProgramTranslator() class SimpleLSTMRNN(paddle.nn.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -49,19 +45,28 @@ class SimpleLSTMRNN(paddle.nn.Layer): for i in range(self._num_layers): weight_1 = self.create_parameter( - attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Uniform( - low=-self._init_scale, high=self._init_scale)), + attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Uniform( + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=paddle.nn.initializer.Uniform( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( - attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Uniform( - low=-self._init_scale, high=self._init_scale)), + attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Uniform( + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=paddle.nn.initializer.Constant(0.0)) + default_initializer=paddle.nn.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -85,11 +90,12 @@ class SimpleLSTMRNN(paddle.nn.Layer): gate_input = paddle.matmul(x=nn, y=weight_1) gate_input = paddle.add(x=gate_input, y=bias) - i, j, f, o = paddle.split(x=gate_input, - num_or_sections=4, - axis=-1) + i, j, f, o = paddle.split( + x=gate_input, num_or_sections=4, axis=-1 + ) c = pre_cell * paddle.nn.functional.sigmoid( - f) + paddle.nn.functional.sigmoid(i) * paddle.tanh(j) + f + ) + paddle.nn.functional.sigmoid(i) * paddle.tanh(j) m = paddle.tanh(c) * paddle.nn.functional.sigmoid(o) hidden_array[k] = m cell_array[k] = c @@ -99,31 +105,36 @@ class SimpleLSTMRNN(paddle.nn.Layer): step_input = paddle.nn.functional.dropout( step_input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append(step_input) real_res = paddle.concat(x=res, axis=1) - real_res = paddle.reshape(real_res, - [-1, self._num_steps, self._hidden_size]) + real_res = paddle.reshape( + real_res, [-1, self._num_steps, self._hidden_size] + ) last_hidden = paddle.concat(x=hidden_array, axis=1) last_hidden = paddle.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = paddle.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = paddle.concat(x=cell_array, axis=1) last_cell = paddle.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = paddle.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(paddle.nn.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -131,31 +142,40 @@ class PtbModel(paddle.nn.Layer): self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = paddle.fluid.dygraph.nn.Embedding( size=[vocab_size, hidden_size], dtype='float32', is_sparse=False, param_attr=paddle.ParamAttr( name='embedding_para', - initializer=paddle.nn.initializer.Uniform(low=-init_scale, - high=init_scale))) + initializer=paddle.nn.initializer.Uniform( + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def build_once(self, input, label, init_hidden, init_cell): pass @@ -163,29 +183,35 @@ class PtbModel(paddle.nn.Layer): @paddle.jit.to_static def forward(self, input, label, init_hidden, init_cell): - init_h = paddle.reshape(init_hidden, - shape=[self.num_layers, -1, self.hidden_size]) + init_h = paddle.reshape( + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) - init_c = paddle.reshape(init_cell, - shape=[self.num_layers, -1, self.hidden_size]) + init_c = paddle.reshape( + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) - x_emb = paddle.reshape(x_emb, - shape=[-1, self.num_steps, self.hidden_size]) + x_emb = paddle.reshape( + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = paddle.nn.functional.dropout( x_emb, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) projection = paddle.matmul(x=rnn_out, y=self.softmax_weight) projection = paddle.add(x=projection, y=self.softmax_bias) loss = paddle.nn.functional.softmax_with_cross_entropy( - logits=projection, label=label, soft_label=False) + logits=projection, label=label, soft_label=False + ) loss = paddle.reshape(loss, shape=[-1, self.num_steps]) loss = paddle.mean(loss, axis=[0]) loss = paddle.fluid.layers.reduce_sum(loss) @@ -212,15 +238,18 @@ def train(place): paddle.disable_static(place) paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - dropout=dropout) - - sgd = paddle.optimizer.SGD(learning_rate=1e-3, - parameters=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + dropout=dropout, + ) + + sgd = paddle.optimizer.SGD( + learning_rate=1e-3, parameters=ptb_model.parameters() + ) for epoch_id in range(max_epoch): @@ -228,19 +257,19 @@ def train(place): iters = 0.0 total_sample = 0 - init_hidden_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') - - init_hidden = paddle.to_tensor(data=init_hidden_data, - dtype=None, - place=None, - stop_gradient=True) - init_cell = paddle.to_tensor(data=init_cell_data, - dtype=None, - place=None, - stop_gradient=True) + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) + + init_hidden = paddle.to_tensor( + data=init_hidden_data, dtype=None, place=None, stop_gradient=True + ) + init_cell = paddle.to_tensor( + data=init_cell_data, dtype=None, place=None, stop_gradient=True + ) for step_id in range(batch_num): x_data = np.arange(12).reshape(4, 3).astype('int64') y_data = np.arange(1, 13).reshape(4, 3).astype('int64') @@ -249,17 +278,16 @@ def train(place): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, num_steps, 1)) - x = paddle.to_tensor(data=x_data, - dtype=None, - place=None, - stop_gradient=True) - y = paddle.to_tensor(data=y_data, - dtype=None, - place=None, - stop_gradient=True) - - dy_loss, last_hidden, last_cell = ptb_model(x, y, init_hidden, - init_cell) + x = paddle.to_tensor( + data=x_data, dtype=None, place=None, stop_gradient=True + ) + y = paddle.to_tensor( + data=y_data, dtype=None, place=None, stop_gradient=True + ) + + dy_loss, last_hidden, last_cell = ptb_model( + x, y, init_hidden, init_cell + ) out_loss = dy_loss.numpy() dy_loss.backward() @@ -271,14 +299,17 @@ def train(place): total_sample += 1 if step_id % PRINT_STEP == 0: if step_id == 0: - logging.info("epoch %d | step %d, loss %0.3f" % - (epoch_id, step_id, total_loss / total_sample)) + logging.info( + "epoch %d | step %d, loss %0.3f" + % (epoch_id, step_id, total_loss / total_sample) + ) avg_batch_time = time.time() else: speed = PRINT_STEP / (time.time() - avg_batch_time) logging.info( - "epoch %d | step %d, loss %0.3f, speed %.3f steps/s" % - (epoch_id, step_id, total_loss / total_sample, speed)) + "epoch %d | step %d, loss %0.3f, speed %.3f steps/s" + % (epoch_id, step_id, total_loss / total_sample, speed) + ) avg_batch_time = time.time() ret = out_loss, last_hidden.numpy(), last_cell.numpy() @@ -297,10 +328,12 @@ def train_static(place): class TestPtb(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_check_result(self): loss_1, hidden_1, cell_1 = train_static(self.place) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_reinforcement_learning.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_reinforcement_learning.py index 99361fa3949..d4bcd98e644 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_reinforcement_learning.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_reinforcement_learning.py @@ -29,7 +29,6 @@ program_translator = ProgramTranslator() class Policy(Layer): - def __init__(self): super(Policy, self).__init__() @@ -74,17 +73,18 @@ def train(args, place, to_static): eps = np.finfo(np.float32).eps.item() optimizer = fluid.optimizer.AdamaxOptimizer( - learning_rate=1e-2, parameter_list=policy.parameters()) + learning_rate=1e-2, parameter_list=policy.parameters() + ) def get_mean_and_std(values=[]): - n = 0. - s = 0. + n = 0.0 + s = 0.0 for val in values: s += val n += 1 mean = s / n - std = 0. + std = 0.0 for val in values: std += (val - mean) * (val - mean) std /= n @@ -99,14 +99,14 @@ def train(args, place, to_static): while idx < len(probs) and sample > probs[idx]: sample -= probs[idx] idx += 1 - mask = [0.] * len(probs) - mask[idx] = 1. + mask = [0.0] * len(probs) + mask[idx] = 1.0 return idx, np.array([mask]).astype("float32") def choose_best_action(probs): idx = 0 if probs[0] > probs[1] else 1 - mask = [1., 0.] if idx == 0 else [0., 1.] + mask = [1.0, 0.0] if idx == 0 else [0.0, 1.0] return idx, np.array([mask]).astype("float32") @@ -189,9 +189,10 @@ def train(args, place, to_static): running_reward = 0.05 * ep_reward + (1 - 0.05) * running_reward if i_episode % args.log_interval == 0: print( - 'Episode {}\tLast reward: {:.2f}\tAverage reward: {:.2f}\t loss_probs: {}' - .format(i_episode, ep_reward, running_reward, - loss.numpy()[0])) + 'Episode {}\tLast reward: {:.2f}\tAverage reward: {:.2f}\t loss_probs: {}'.format( + i_episode, ep_reward, running_reward, loss.numpy()[0] + ) + ) if i_episode > args.train_step: break @@ -200,10 +201,12 @@ def train(args, place, to_static): class TestDeclarative(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + ) self.args = Args() def test_train(self): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py index 86d215faadd..a1f85bd3da3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py @@ -36,8 +36,9 @@ l2_decay = 1e-4 # NOTE: Reduce batch_size from 8 to 2 to avoid unittest timeout. batch_size = 2 epoch_num = 1 -place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ - else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) program_translator = ProgramTranslator() @@ -50,30 +51,34 @@ def optimizer_setting(parameter_list=None): learning_rate=base_lr, momentum=momentum_rate, regularization=fluid.regularizer.L2Decay(l2_decay), - parameter_list=parameter_list) + parameter_list=parameter_list, + ) return optimizer class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=False) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False, + ) self._batch_norm = BatchNorm(num_filters, act=act) @@ -85,29 +90,36 @@ class ConvBNLayer(fluid.dygraph.Layer): class BottleneckBlock(fluid.dygraph.Layer): - def __init__(self, num_channels, num_filters, stride, shortcut=True): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu') - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - act='relu') - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act=None) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act=None, + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -125,20 +137,23 @@ class BottleneckBlock(fluid.dygraph.Layer): y = fluid.layers.elementwise_add(x=short, y=conv2) - layer_helper = fluid.layer_helper.LayerHelper(self.full_name(), - act='relu') + layer_helper = fluid.layer_helper.LayerHelper( + self.full_name(), act='relu' + ) return layer_helper.append_activation(y) class ResNet(fluid.dygraph.Layer): - def __init__(self, layers=50, class_dim=102): super(ResNet, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: depth = [3, 4, 6, 3] @@ -149,15 +164,12 @@ class ResNet(fluid.dygraph.Layer): num_channels = [64, 256, 512, 1024] num_filters = [64, 128, 256, 512] - self.conv = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool2d_max = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv = ConvBNLayer( + num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu' + ) + self.pool2d_max = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] for block in range(len(depth)): @@ -165,17 +177,21 @@ class ResNet(fluid.dygraph.Layer): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels[block] - if i == 0 else num_filters[block] * 4, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels[block] + if i == 0 + else num_filters[block] * 4, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + shortcut=shortcut, + ), + ) self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 4 * 1 * 1 @@ -186,7 +202,9 @@ class ResNet(fluid.dygraph.Layer): class_dim, act='softmax', param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) def forward(self, inputs): y = self.conv(inputs) @@ -201,7 +219,6 @@ class ResNet(fluid.dygraph.Layer): def reader_decorator(reader): - def __reader__(): for item in reader(): img = np.array(item[0]).astype('float32').reshape(3, 224, 224) @@ -212,15 +229,15 @@ def reader_decorator(reader): class ResNetHelper: - def __init__(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'inference') self.model_save_prefix = os.path.join(self.model_save_dir, 'resnet') self.model_filename = 'resnet' + INFER_MODEL_SUFFIX self.params_filename = 'resnet' + INFER_PARAMS_SUFFIX - self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, - 'resnet.dygraph') + self.dy_state_dict_save_path = os.path.join( + self.temp_dir.name, 'resnet.dygraph' + ) def __del__(self): self.temp_dir.cleanup() @@ -234,18 +251,21 @@ class ResNetHelper: paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) - data_loader = fluid.io.DataLoader.from_generator(capacity=5, - iterable=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) + data_loader = fluid.io.DataLoader.from_generator( + capacity=5, iterable=True + ) data_loader.set_sample_list_generator(train_reader) resnet = ResNet() if to_static: - resnet = paddle.jit.to_static(resnet, - build_strategy=build_strategy) + resnet = paddle.jit.to_static( + resnet, build_strategy=build_strategy + ) optimizer = optimizer_setting(parameter_list=resnet.parameters()) for epoch in range(epoch_num): @@ -261,12 +281,12 @@ class ResNetHelper: pred = resnet(img) loss = fluid.layers.cross_entropy(input=pred, label=label) avg_loss = paddle.mean(x=loss) - acc_top1 = fluid.layers.accuracy(input=pred, - label=label, - k=1) - acc_top5 = fluid.layers.accuracy(input=pred, - label=label, - k=5) + acc_top1 = fluid.layers.accuracy( + input=pred, label=label, k=1 + ) + acc_top5 = fluid.layers.accuracy( + input=pred, label=label, k=5 + ) avg_loss.backward() optimizer.minimize(avg_loss) @@ -279,17 +299,27 @@ class ResNetHelper: end_time = time.time() if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + print( + "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" + % ( + epoch, + batch_id, + total_loss.numpy() / total_sample, + total_acc1.numpy() / total_sample, + total_acc5.numpy() / total_sample, + end_time - start_time, + ) + ) if batch_id == 10: if to_static: - fluid.dygraph.jit.save(resnet, - self.model_save_prefix) + fluid.dygraph.jit.save( + resnet, self.model_save_prefix + ) else: fluid.dygraph.save_dygraph( resnet.state_dict(), - self.dy_state_dict_save_path) + self.dy_state_dict_save_path, + ) # avoid dataloader throw abort signaal data_loader._reset() break @@ -302,7 +332,8 @@ class ResNetHelper: resnet = ResNet() model_dict, _ = fluid.dygraph.load_dygraph( - self.dy_state_dict_save_path) + self.dy_state_dict_save_path + ) resnet.set_dict(model_dict) resnet.eval() @@ -313,15 +344,22 @@ class ResNetHelper: def predict_static(self, data): paddle.enable_static() exe = fluid.Executor(place) - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res[0] @@ -335,14 +373,17 @@ class ResNetHelper: return pred_res.numpy() def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, [data]) - out, = output() + output = PredictorTools( + self.model_save_dir, + self.model_filename, + self.params_filename, + [data], + ) + (out,) = output() return out class TestResnet(unittest.TestCase): - def setUp(self): self.resnet_helper = ResNetHelper() @@ -360,19 +401,24 @@ class TestResnet(unittest.TestCase): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) np.testing.assert_allclose( predictor_pre, st_pre, rtol=1e-05, err_msg='predictor_pre:\n {}\n, st_pre: \n{}.'.format( - predictor_pre, st_pre)) + predictor_pre, st_pre + ), + ) def test_resnet(self): static_loss = self.train(to_static=True) @@ -382,7 +428,9 @@ class TestResnet(unittest.TestCase): dygraph_loss, rtol=1e-05, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) self.verify_predict() def test_in_static_mode_mkldnn(self): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_amp.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_amp.py index c535b7223db..aa53f663deb 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_amp.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_amp.py @@ -25,8 +25,9 @@ from test_resnet import ResNet, optimizer_setting, SEED # NOTE: Reduce batch_size from 8 to 2 to avoid unittest timeout. batch_size = 2 epoch_num = 1 -place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ - else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) program_translator = ProgramTranslator() @@ -58,10 +59,13 @@ def train(to_static, build_strategy=None): for batch_id in range(100): start_time = time.time() img = paddle.to_tensor( - np.random.random([batch_size, 3, 224, - 224]).astype('float32')) + np.random.random([batch_size, 3, 224, 224]).astype( + 'float32' + ) + ) label = paddle.to_tensor( - np.random.randint(0, 100, [batch_size, 1], dtype='int64')) + np.random.randint(0, 100, [batch_size, 1], dtype='int64') + ) img.stop_gradient = True label.stop_gradient = True @@ -87,9 +91,17 @@ def train(to_static, build_strategy=None): end_time = time.time() if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + print( + "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" + % ( + epoch, + batch_id, + total_loss.numpy() / total_sample, + total_acc1.numpy() / total_sample, + total_acc5.numpy() / total_sample, + end_time - start_time, + ) + ) if batch_id == 10: break @@ -97,7 +109,6 @@ def train(to_static, build_strategy=None): class TestResnet(unittest.TestCase): - def train(self, to_static): program_translator.enable(to_static) return train(to_static) @@ -110,7 +121,9 @@ class TestResnet(unittest.TestCase): dygraph_loss, rtol=1e-05, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_pure_fp16.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_pure_fp16.py index f4d724826ab..9951d67c2b1 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_pure_fp16.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_pure_fp16.py @@ -46,10 +46,9 @@ def train(to_static, build_strategy=None): optimizer = optimizer_setting(parameter_list=resnet.parameters()) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, optimizers=optimizer, level='O2', save_dtype='float32' + ) for epoch in range(epoch_num): loss_data = [] @@ -61,16 +60,20 @@ def train(to_static, build_strategy=None): for batch_id in range(100): start_time = time.time() img = paddle.to_tensor( - np.random.random([batch_size, 3, 224, 224]).astype('float32')) + np.random.random([batch_size, 3, 224, 224]).astype('float32') + ) label = paddle.to_tensor( - np.random.randint(0, 100, [batch_size, 1], dtype='int64')) + np.random.randint(0, 100, [batch_size, 1], dtype='int64') + ) img.stop_gradient = True label.stop_gradient = True - with paddle.amp.auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2'): + with paddle.amp.auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O2', + ): pred = resnet(img) loss = fluid.layers.cross_entropy(input=pred, label=label) avg_loss = paddle.mean(x=pred) @@ -90,9 +93,17 @@ def train(to_static, build_strategy=None): end_time = time.time() if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + print( + "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" + % ( + epoch, + batch_id, + total_loss.numpy() / total_sample, + total_acc1.numpy() / total_sample, + total_acc5.numpy() / total_sample, + end_time - start_time, + ) + ) if batch_id == 10: break @@ -100,7 +111,6 @@ def train(to_static, build_strategy=None): class TestResnet(unittest.TestCase): - def train(self, to_static): program_translator.enable(to_static) build_strategy = paddle.static.BuildStrategy() @@ -120,7 +130,9 @@ class TestResnet(unittest.TestCase): rtol=1e-05, atol=0.001, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py index 301da29d6d9..f4f2e815449 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py @@ -34,8 +34,9 @@ l2_decay = 1e-4 # NOTE: Reduce batch_size from 8 to 2 to avoid unittest timeout. batch_size = 2 epoch_num = 1 -place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ - else paddle.CPUPlace() +place = ( + paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() +) program_translator = paddle.jit.ProgramTranslator() @@ -48,29 +49,33 @@ def optimizer_setting(parameter_list=None): learning_rate=base_lr, momentum=momentum_rate, weight_decay=paddle.regularizer.L2Decay(l2_decay), - parameters=parameter_list) + parameters=parameter_list, + ) return optimizer class ConvBNLayer(paddle.nn.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = paddle.nn.Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv = paddle.nn.Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) self._batch_norm = paddle.nn.BatchNorm(num_filters, act=act) @@ -82,29 +87,36 @@ class ConvBNLayer(paddle.nn.Layer): class BottleneckBlock(paddle.nn.Layer): - def __init__(self, num_channels, num_filters, stride, shortcut=True): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu') - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - act='relu') - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act=None) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act=None, + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -122,20 +134,23 @@ class BottleneckBlock(paddle.nn.Layer): y = paddle.add(x=short, y=conv2) - layer_helper = paddle.fluid.layer_helper.LayerHelper(self.full_name(), - act='relu') + layer_helper = paddle.fluid.layer_helper.LayerHelper( + self.full_name(), act='relu' + ) return layer_helper.append_activation(y) class ResNet(paddle.nn.Layer): - def __init__(self, layers=50, class_dim=102): super(ResNet, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: depth = [3, 4, 6, 3] @@ -146,15 +161,12 @@ class ResNet(paddle.nn.Layer): num_channels = [64, 256, 512, 1024] num_filters = [64, 128, 256, 512] - self.conv = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool2d_max = paddle.fluid.dygraph.Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv = ConvBNLayer( + num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu' + ) + self.pool2d_max = paddle.fluid.dygraph.Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] for block in range(len(depth)): @@ -162,17 +174,21 @@ class ResNet(paddle.nn.Layer): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels[block] - if i == 0 else num_filters[block] * 4, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels[block] + if i == 0 + else num_filters[block] * 4, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + shortcut=shortcut, + ), + ) self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = paddle.fluid.dygraph.Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = paddle.fluid.dygraph.Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 4 * 1 * 1 @@ -182,7 +198,9 @@ class ResNet(paddle.nn.Layer): in_features=self.pool2d_avg_output, out_features=class_dim, weight_attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Uniform(-stdv, stdv))) + initializer=paddle.nn.initializer.Uniform(-stdv, stdv) + ), + ) @paddle.jit.to_static def forward(self, inputs): @@ -199,7 +217,6 @@ class ResNet(paddle.nn.Layer): def reader_decorator(reader): - def __reader__(): for item in reader(): img = np.array(item[0]).astype('float32').reshape(3, 224, 224) @@ -210,17 +227,22 @@ def reader_decorator(reader): class TestResnet(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, "./inference") - self.model_save_prefix = os.path.join(self.temp_dir.name, - "./inference/resnet_v2") - self.model_filename = "resnet_v2" + paddle.fluid.dygraph.io.INFER_MODEL_SUFFIX - self.params_filename = "resnet_v2" + paddle.fluid.dygraph.io.INFER_PARAMS_SUFFIX - self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, - "./resnet_v2.dygraph") + self.model_save_prefix = os.path.join( + self.temp_dir.name, "./inference/resnet_v2" + ) + self.model_filename = ( + "resnet_v2" + paddle.fluid.dygraph.io.INFER_MODEL_SUFFIX + ) + self.params_filename = ( + "resnet_v2" + paddle.fluid.dygraph.io.INFER_PARAMS_SUFFIX + ) + self.dy_state_dict_save_path = os.path.join( + self.temp_dir.name, "./resnet_v2.dygraph" + ) def tearDown(self): self.temp_dir.cleanup() @@ -234,12 +256,14 @@ class TestResnet(unittest.TestCase): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) - data_loader = paddle.io.DataLoader.from_generator(capacity=5, - iterable=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) + data_loader = paddle.io.DataLoader.from_generator( + capacity=5, iterable=True + ) data_loader.set_sample_list_generator(train_reader) resnet = ResNet() @@ -256,8 +280,9 @@ class TestResnet(unittest.TestCase): img, label = data pred = resnet(img) - loss = paddle.nn.functional.cross_entropy(input=pred, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=pred, label=label + ) avg_loss = paddle.mean(x=loss) acc_top1 = paddle.metric.accuracy(input=pred, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=pred, label=label, k=5) @@ -273,15 +298,24 @@ class TestResnet(unittest.TestCase): end_time = time.time() if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + print( + "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" + % ( + epoch, + batch_id, + total_loss.numpy() / total_sample, + total_acc1.numpy() / total_sample, + total_acc5.numpy() / total_sample, + end_time - start_time, + ) + ) if batch_id == 10: if to_static: paddle.jit.save(resnet, self.model_save_prefix) else: paddle.fluid.dygraph.save_dygraph( - resnet.state_dict(), self.dy_state_dict_save_path) + resnet.state_dict(), self.dy_state_dict_save_path + ) # avoid dataloader throw abort signaal data_loader._reset() break @@ -295,15 +329,16 @@ class TestResnet(unittest.TestCase): resnet = ResNet() model_dict, _ = paddle.fluid.dygraph.load_dygraph( - self.dy_state_dict_save_path) + self.dy_state_dict_save_path + ) resnet.set_dict(model_dict) resnet.eval() pred_res = resnet( - paddle.to_tensor(data=data, - dtype=None, - place=None, - stop_gradient=True)) + paddle.to_tensor( + data=data, dtype=None, place=None, stop_gradient=True + ) + ) ret = pred_res.numpy() paddle.enable_static() @@ -311,16 +346,22 @@ class TestResnet(unittest.TestCase): def predict_static(self, data): exe = paddle.static.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model( - self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res[0] @@ -336,9 +377,13 @@ class TestResnet(unittest.TestCase): return ret def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, [data]) - out, = output() + output = PredictorTools( + self.model_save_dir, + self.model_filename, + self.params_filename, + [data], + ) + (out,) = output() return out def train(self, to_static): @@ -355,19 +400,24 @@ class TestResnet(unittest.TestCase): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) np.testing.assert_allclose( predictor_pre, st_pre, rtol=1e-05, err_msg='predictor_pre:\n {}\n, st_pre: \n{}.'.format( - predictor_pre, st_pre)) + predictor_pre, st_pre + ), + ) def test_resnet(self): static_loss = self.train(to_static=True) @@ -377,7 +427,9 @@ class TestResnet(unittest.TestCase): dygraph_loss, rtol=1e-05, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) self.verify_predict() def test_in_static_mode_mkldnn(self): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py index e3323dd084a..a194cede9c0 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py @@ -168,7 +168,7 @@ def test_return_list_many_values(x): def test_return_tuple_one_value(x): x = fluid.dygraph.to_variable(x) x += 1 - return (x, ) + return (x,) @to_static @@ -250,7 +250,6 @@ def test_return_in_for_2(x): @to_static def test_return_nested(x): - def func(): rr = 0 if True: @@ -266,11 +265,13 @@ def test_return_nested(x): class TestReturnBase(unittest.TestCase): - def setUp(self): self.input = np.ones((1)).astype('int32') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_dygraph_func() self.program_translator = ProgramTranslator() @@ -294,9 +295,9 @@ class TestReturnBase(unittest.TestCase): self.assertTrue(isinstance(static_res, tuple)) self.assertEqual(len(dygraph_res), len(static_res)) for i in range(len(dygraph_res)): - np.testing.assert_allclose(dygraph_res[i], - static_res[i], - rtol=1e-05) + np.testing.assert_allclose( + dygraph_res[i], static_res[i], rtol=1e-05 + ) elif isinstance(dygraph_res, np.ndarray): np.testing.assert_allclose(dygraph_res, static_res, rtol=1e-05) else: @@ -311,134 +312,113 @@ class TestReturnBase(unittest.TestCase): class TestInsideFuncBase(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_inside_func_base class TestReturnIf(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_if class TestReturnOnlyIf(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_if_else_2 class TestReturnInFor(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_in_for class TestReturnInWhile(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_in_while class TestReturnIfDiff(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_diff_return class TestReturnIfElse(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_if_else class TestReturnInWhile2(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_in_while_2 self.error = "Found return statement in While or For body and loop" class TestReturnInFor2(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_in_for_2 self.error = "Found return statement in While or For body and loop" class TestRecursiveReturn(TestReturnBase): - def init_dygraph_func(self): self.input = self.input.astype(np.float32) self.dygraph_func = test_recursive_return class TestReturnDifferentLengthIfBody(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_different_length_if_body self.error = "Your if/else have different number of return value." class TestReturnDifferentLengthElse(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_different_length_else self.error = "Your if/else have different number of return value." class TestNoReturn(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_no_return class TestReturnNone(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_none self.error = "Your if/else have different number of return value." class TestReturnNoVariable(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_no_variable self.error = "Your if/else have different number of return value." class TestReturnListOneValue(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_list_one_value class TestReturnListManyValue(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_list_many_values class TestReturnTupleOneValue(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_tuple_one_value class TestReturnTupleManyValue(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_tuple_many_values class TestReturnNested(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_nested class TestReturnSpecial(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_without_paddle_cond diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py index f949e9c0da7..90c017ee9e7 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py @@ -16,11 +16,12 @@ import unittest import paddle import numpy as np from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + StaticFunction, +) class Net(paddle.nn.Layer): - def __init__(self): super(Net, self).__init__() self.sub = SubNet() @@ -38,7 +39,6 @@ class Net(paddle.nn.Layer): class SubNet(paddle.nn.Layer): - def __init__(self): super(SubNet, self).__init__() @@ -59,15 +59,14 @@ class SubNet(paddle.nn.Layer): def foo(x, flag=False): if flag: - out = x * 2. + out = x * 2.0 else: - out = x / 2. + out = x / 2.0 return out class TestRollBackPlainFunction(unittest.TestCase): - def setUp(self): paddle.set_device("cpu") @@ -86,7 +85,6 @@ class TestRollBackPlainFunction(unittest.TestCase): class TestRollBackNet(unittest.TestCase): - def setUp(self): paddle.set_device("cpu") @@ -118,8 +116,9 @@ class TestRollBackNet(unittest.TestCase): self.assertFalse(isinstance(net.infer, StaticFunction)) self.assertFalse("true_fn" in func_to_source_code(net.sub.forward)) dy_infer_out = net.infer(x) - np.testing.assert_array_equal(st_infer_out.numpy(), - dy_infer_out.numpy()) + np.testing.assert_array_equal( + st_infer_out.numpy(), dy_infer_out.numpy() + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py index 60aa64abe66..f1c3bd4c1b6 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py @@ -21,20 +21,22 @@ import paddle import paddle.fluid as fluid from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator from paddle.fluid.dygraph.jit import declarative -from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from +from paddle.fluid.dygraph.dygraph_to_static.partial_program import ( + partial_program_from, +) from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX SEED = 2020 np.random.seed(SEED) -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) program_translator = ProgramTranslator() class SimpleFcLayer(fluid.dygraph.Layer): - def __init__(self, fc_size): super(SimpleFcLayer, self).__init__() self._linear = fluid.dygraph.Linear(fc_size, fc_size) @@ -48,7 +50,6 @@ class SimpleFcLayer(fluid.dygraph.Layer): class TestDyToStaticSaveInferenceModel(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -64,8 +65,9 @@ class TestDyToStaticSaveInferenceModel(unittest.TestCase): x = fluid.dygraph.to_variable(x_data) layer = SimpleFcLayer(fc_size) - adam = fluid.optimizer.SGD(learning_rate=0.1, - parameter_list=layer.parameters()) + adam = fluid.optimizer.SGD( + learning_rate=0.1, parameter_list=layer.parameters() + ) for i in range(5): loss, pred = layer(x) @@ -74,65 +76,78 @@ class TestDyToStaticSaveInferenceModel(unittest.TestCase): layer.clear_gradients() # test for saving model in dygraph.guard infer_model_prefix = os.path.join( - self.temp_dir.name, "test_dy2stat_inference_in_guard/model") - infer_model_dir = os.path.join(self.temp_dir.name, - "test_dy2stat_inference_in_guard") - fluid.dygraph.jit.save(layer=layer, - path=infer_model_prefix, - input_spec=[x], - output_spec=[pred]) + self.temp_dir.name, "test_dy2stat_inference_in_guard/model" + ) + infer_model_dir = os.path.join( + self.temp_dir.name, "test_dy2stat_inference_in_guard" + ) + fluid.dygraph.jit.save( + layer=layer, + path=infer_model_prefix, + input_spec=[x], + output_spec=[pred], + ) # Check the correctness of the inference dygraph_out, _ = layer(x) self.check_save_inference_model(layer, [x_data], dygraph_out.numpy()) - self.check_save_inference_model(layer, [x_data], - dygraph_out.numpy(), - fetch=[loss]) - self.check_save_inference_model(layer, [x_data], - dygraph_out.numpy(), - feed=[x]) - - def check_save_inference_model(self, - model, - inputs, - gt_out, - feed=None, - fetch=None): + self.check_save_inference_model( + layer, [x_data], dygraph_out.numpy(), fetch=[loss] + ) + self.check_save_inference_model( + layer, [x_data], dygraph_out.numpy(), feed=[x] + ) + + def check_save_inference_model( + self, model, inputs, gt_out, feed=None, fetch=None + ): expected_persistable_vars = set([p.name for p in model.parameters()]) - infer_model_prefix = os.path.join(self.temp_dir.name, - "test_dy2stat_inference/model") - infer_model_dir = os.path.join(self.temp_dir.name, - "test_dy2stat_inference") + infer_model_prefix = os.path.join( + self.temp_dir.name, "test_dy2stat_inference/model" + ) + infer_model_dir = os.path.join( + self.temp_dir.name, "test_dy2stat_inference" + ) model_filename = "model" + INFER_MODEL_SUFFIX params_filename = "model" + INFER_PARAMS_SUFFIX - fluid.dygraph.jit.save(layer=model, - path=infer_model_prefix, - input_spec=feed if feed else None, - output_spec=fetch if fetch else None) + fluid.dygraph.jit.save( + layer=model, + path=infer_model_prefix, + input_spec=feed if feed else None, + output_spec=fetch if fetch else None, + ) # Check the correctness of the inference - infer_out = self.load_and_run_inference(infer_model_dir, model_filename, - params_filename, inputs) + infer_out = self.load_and_run_inference( + infer_model_dir, model_filename, params_filename, inputs + ) np.testing.assert_allclose(gt_out, infer_out, rtol=1e-05) - def load_and_run_inference(self, model_path, model_filename, - params_filename, inputs): + def load_and_run_inference( + self, model_path, model_filename, params_filename, inputs + ): paddle.enable_static() exe = fluid.Executor(place) - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(dirname=model_path, - executor=exe, - model_filename=model_filename, - params_filename=params_filename) - results = exe.run(inference_program, - feed=dict(zip(feed_target_names, inputs)), - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + dirname=model_path, + executor=exe, + model_filename=model_filename, + params_filename=params_filename, + ) + results = exe.run( + inference_program, + feed=dict(zip(feed_target_names, inputs)), + fetch_list=fetch_targets, + ) return np.array(results[0]) class TestPartialProgramRaiseError(unittest.TestCase): - def test_param_type(self): program_translator = ProgramTranslator() program_translator.enable(True) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py index 42be11182c0..0898b727304 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py @@ -25,16 +25,17 @@ from test_fetch_feed import Linear np.random.seed(2020) -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) class TestDyToStaticSaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "test_dy2stat_save_load") + self.model_path = os.path.join( + self.temp_dir.name, "test_dy2stat_save_load" + ) def tearDown(self): self.temp_dir.cleanup() @@ -49,8 +50,9 @@ class TestDyToStaticSaveLoad(unittest.TestCase): program_translator.enable(True) x = fluid.dygraph.to_variable(x_data) net = Linear(32, 64) - adam = AdamOptimizer(learning_rate=0.1, - parameter_list=net.parameters()) + adam = AdamOptimizer( + learning_rate=0.1, parameter_list=net.parameters() + ) for i in range(batch_num): static_out, static_loss = net(x) @@ -81,12 +83,12 @@ class TestDyToStaticSaveLoad(unittest.TestCase): program_translator.enable(False) dygraph_out, dygraph_loss = dygraph_net(x) - np.testing.assert_allclose(dygraph_out.numpy(), - static_out.numpy(), - rtol=1e-05) - np.testing.assert_allclose(dygraph_loss.numpy(), - static_loss.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + dygraph_out.numpy(), static_out.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + dygraph_loss.numpy(), static_loss.numpy(), rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py index 2a86c3f77a1..74f6bdb2d61 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py @@ -38,8 +38,9 @@ EPOCH_NUM = 1 PRINT_STEP = 2 STEP_NUM = 10 -place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ - else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) # Note: Set True to eliminate randomness. # 1. For one operation, cuDNN has several algorithms, @@ -52,7 +53,7 @@ train_parameters = { "name": "cosine_decay", "batch_size": BATCH_SIZE, "epochs": [40, 80, 100], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "lr": 0.0125, "total_images": 6149, @@ -78,35 +79,39 @@ def optimizer_setting(params, parameter_list): lr = params["lr"] num_epochs = params["num_epochs"] optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.cosine_decay(learning_rate=lr, - step_each_epoch=step, - epochs=num_epochs), + learning_rate=fluid.layers.cosine_decay( + learning_rate=lr, step_each_epoch=step, epochs=num_epochs + ), momentum=momentum_rate, regularization=fluid.regularizer.L2Decay(l2_decay), - parameter_list=parameter_list) + parameter_list=parameter_list, + ) return optimizer class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=False) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False, + ) self._batch_norm = BatchNorm(num_filters, act=act) @@ -118,7 +123,6 @@ class ConvBNLayer(fluid.dygraph.Layer): class SqueezeExcitation(fluid.dygraph.Layer): - def __init__(self, num_channels, reduction_ratio): super(SqueezeExcitation, self).__init__() @@ -129,15 +133,19 @@ class SqueezeExcitation(fluid.dygraph.Layer): num_channels, num_channels // reduction_ratio, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), - act='relu') + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + act='relu', + ) stdv = 1.0 / math.sqrt(num_channels / 16.0 * 1.0) self._excitation = Linear( num_channels // reduction_ratio, num_channels, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), - act='sigmoid') + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + act='sigmoid', + ) def forward(self, input): y = self._pool(input) @@ -149,39 +157,49 @@ class SqueezeExcitation(fluid.dygraph.Layer): class BottleneckBlock(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - cardinality, - reduction_ratio, - shortcut=True): + def __init__( + self, + num_channels, + num_filters, + stride, + cardinality, + reduction_ratio, + shortcut=True, + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act="relu") - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - groups=cardinality, - act="relu") - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 2, - filter_size=1, - act=None) - - self.scale = SqueezeExcitation(num_channels=num_filters * 2, - reduction_ratio=reduction_ratio) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act="relu", + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + act="relu", + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 2, + filter_size=1, + act=None, + ) + + self.scale = SqueezeExcitation( + num_channels=num_filters * 2, reduction_ratio=reduction_ratio + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 2, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 2, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -203,67 +221,76 @@ class BottleneckBlock(fluid.dygraph.Layer): class SeResNeXt(fluid.dygraph.Layer): - def __init__(self, layers=50, class_dim=102): super(SeResNeXt, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 101: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 23, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 152: cardinality = 64 reduction_ratio = 16 depth = [3, 8, 36, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - self.conv1 = ConvBNLayer(num_channels=64, - num_filters=64, - filter_size=3, - stride=1, - act='relu') - self.conv2 = ConvBNLayer(num_channels=64, - num_filters=128, - filter_size=3, - stride=1, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=3, + stride=2, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=64, + num_filters=64, + filter_size=3, + stride=1, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=64, + num_filters=128, + filter_size=3, + stride=1, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] num_channels = 64 @@ -274,19 +301,22 @@ class SeResNeXt(fluid.dygraph.Layer): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=cardinality, - reduction_ratio=reduction_ratio, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio, + shortcut=shortcut, + ), + ) num_channels = bottleneck_block._num_channels_out self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) stdv = 1.0 / math.sqrt(2048 * 1.0) self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 2 * 1 * 1 @@ -295,7 +325,9 @@ class SeResNeXt(fluid.dygraph.Layer): self.pool2d_avg_output, class_dim, param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) @declarative def forward(self, inputs, label): @@ -326,21 +358,23 @@ class SeResNeXt(fluid.dygraph.Layer): class TestSeResnet(unittest.TestCase): - def setUp(self): - self.train_reader = paddle.batch(paddle.dataset.flowers.train( - use_xmap=False, cycle=True), - batch_size=BATCH_SIZE, - drop_last=True) + self.train_reader = paddle.batch( + paddle.dataset.flowers.train(use_xmap=False, cycle=True), + batch_size=BATCH_SIZE, + drop_last=True, + ) self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, "inference") - self.model_save_prefix = os.path.join(self.temp_dir.name, - "inference/se_resnet") + self.model_save_prefix = os.path.join( + self.temp_dir.name, "inference/se_resnet" + ) self.model_filename = "se_resnet" + INFER_MODEL_SUFFIX self.params_filename = "se_resnet" + INFER_PARAMS_SUFFIX - self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, - "se_resnet.dygraph") + self.dy_state_dict_save_path = os.path.join( + self.temp_dir.name, "se_resnet.dygraph" + ) def tearDown(self): self.temp_dir.cleanup() @@ -355,8 +389,9 @@ class TestSeResnet(unittest.TestCase): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) se_resnext = SeResNeXt() - optimizer = optimizer_setting(train_parameters, - se_resnext.parameters()) + optimizer = optimizer_setting( + train_parameters, se_resnext.parameters() + ) for epoch_id in range(EPOCH_NUM): total_loss = 0.0 @@ -366,12 +401,14 @@ class TestSeResnet(unittest.TestCase): step_idx = 0 speed_list = [] for step_id, data in enumerate(train_reader()): - dy_x_data = np.array([ - x[0].reshape(3, 224, 224) for x in data - ]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape( - BATCH_SIZE, 1) + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(BATCH_SIZE, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -392,32 +429,54 @@ class TestSeResnet(unittest.TestCase): total_sample += 1 if step_id % PRINT_STEP == 0: if step_id == 0: - logging.info( "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f" % \ - ( epoch_id, step_id, total_loss / total_sample, \ - total_acc1 / total_sample, total_acc5 / total_sample)) + logging.info( + "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f" + % ( + epoch_id, + step_id, + total_loss / total_sample, + total_acc1 / total_sample, + total_acc5 / total_sample, + ) + ) avg_batch_time = time.time() else: speed = PRINT_STEP / (time.time() - avg_batch_time) speed_list.append(speed) - logging.info( "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, speed %.3f steps/s" % \ - ( epoch_id, step_id, total_loss / total_sample, \ - total_acc1 / total_sample, total_acc5 / total_sample, speed)) + logging.info( + "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, speed %.3f steps/s" + % ( + epoch_id, + step_id, + total_loss / total_sample, + total_acc1 / total_sample, + total_acc5 / total_sample, + speed, + ) + ) avg_batch_time = time.time() step_idx += 1 if step_idx == STEP_NUM: if to_static: - fluid.dygraph.jit.save(se_resnext, - self.model_save_prefix, - [img], - output_spec=[pred]) + fluid.dygraph.jit.save( + se_resnext, + self.model_save_prefix, + [img], + output_spec=[pred], + ) else: fluid.dygraph.save_dygraph( se_resnext.state_dict(), - self.dy_state_dict_save_path) + self.dy_state_dict_save_path, + ) break - return pred.numpy(), avg_loss.numpy(), acc_top1.numpy( - ), acc_top5.numpy() + return ( + pred.numpy(), + avg_loss.numpy(), + acc_top1.numpy(), + acc_top5.numpy(), + ) def predict_dygraph(self, data): program_translator = ProgramTranslator() @@ -426,7 +485,8 @@ class TestSeResnet(unittest.TestCase): se_resnext = SeResNeXt() model_dict, _ = fluid.dygraph.load_dygraph( - self.dy_state_dict_save_path) + self.dy_state_dict_save_path + ) se_resnext.set_dict(model_dict) se_resnext.eval() @@ -440,15 +500,22 @@ class TestSeResnet(unittest.TestCase): def predict_static(self, data): paddle.enable_static() exe = fluid.Executor(place) - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res[0] @@ -462,8 +529,12 @@ class TestSeResnet(unittest.TestCase): return pred_res.numpy() def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, [data]) + output = PredictorTools( + self.model_save_dir, + self.model_filename, + self.params_filename, + [data], + ) out = output() return out @@ -477,13 +548,16 @@ class TestSeResnet(unittest.TestCase): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) flat_st_pre = st_pre.flatten() flat_predictor_pre = np.array(predictor_pre).flatten() @@ -494,34 +568,42 @@ class TestSeResnet(unittest.TestCase): flat_st_pre[i], delta=1e-6, msg="predictor_pre:\n {}\n, st_pre: \n{}.".format( - flat_predictor_pre[i], flat_st_pre[i])) + flat_predictor_pre[i], flat_st_pre[i] + ), + ) def test_check_result(self): - pred_1, loss_1, acc1_1, acc5_1 = self.train(self.train_reader, - to_static=False) - pred_2, loss_2, acc1_2, acc5_2 = self.train(self.train_reader, - to_static=True) + pred_1, loss_1, acc1_1, acc5_1 = self.train( + self.train_reader, to_static=False + ) + pred_2, loss_2, acc1_2, acc5_2 = self.train( + self.train_reader, to_static=True + ) np.testing.assert_allclose( pred_1, pred_2, rtol=1e-05, - err_msg='static pred: {} \ndygraph pred: {}'.format(pred_1, pred_2)) + err_msg='static pred: {} \ndygraph pred: {}'.format(pred_1, pred_2), + ) np.testing.assert_allclose( loss_1, loss_2, rtol=1e-05, - err_msg='static loss: {} \ndygraph loss: {}'.format(loss_1, loss_2)) + err_msg='static loss: {} \ndygraph loss: {}'.format(loss_1, loss_2), + ) np.testing.assert_allclose( acc1_1, acc1_2, rtol=1e-05, - err_msg='static acc1: {} \ndygraph acc1: {}'.format(acc1_1, acc1_2)) + err_msg='static acc1: {} \ndygraph acc1: {}'.format(acc1_1, acc1_2), + ) np.testing.assert_allclose( acc5_1, acc5_2, rtol=1e-05, - err_msg='static acc5: {} \ndygraph acc5: {}'.format(acc5_1, acc5_2)) + err_msg='static acc5: {} \ndygraph acc5: {}'.format(acc5_1, acc5_2), + ) self.verify_predict() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_sentiment.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_sentiment.py index f581f45ca59..aab18c98ab4 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_sentiment.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_sentiment.py @@ -33,21 +33,24 @@ if fluid.is_compiled_with_cuda(): class SimpleConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - use_cudnn=True, - batch_size=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + use_cudnn=True, + batch_size=None, + ): super(SimpleConvPool, self).__init__() self.batch_size = batch_size - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - padding=[1, 1], - use_cudnn=use_cudnn, - act='tanh') + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + padding=[1, 1], + use_cudnn=use_cudnn, + act='tanh', + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -57,7 +60,6 @@ class SimpleConvPool(fluid.dygraph.Layer): class CNN(fluid.dygraph.Layer): - def __init__(self, dict_dim, batch_size, seq_len): super(CNN, self).__init__() self.dict_dim = dict_dim @@ -69,29 +71,37 @@ class CNN(fluid.dygraph.Layer): self.win_size = [3, self.hid_dim] self.batch_size = batch_size self.seq_len = seq_len - self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], - dtype='float32', - is_sparse=False) - self._simple_conv_pool_1 = SimpleConvPool(self.channels, - self.hid_dim, - self.win_size, - batch_size=self.batch_size) - self._fc1 = Linear(input_dim=self.hid_dim * self.seq_len, - output_dim=self.fc_hid_dim, - act="softmax") - self._fc_prediction = Linear(input_dim=self.fc_hid_dim, - output_dim=self.class_dim, - act="softmax") + self.embedding = Embedding( + size=[self.dict_dim + 1, self.emb_dim], + dtype='float32', + is_sparse=False, + ) + self._simple_conv_pool_1 = SimpleConvPool( + self.channels, + self.hid_dim, + self.win_size, + batch_size=self.batch_size, + ) + self._fc1 = Linear( + input_dim=self.hid_dim * self.seq_len, + output_dim=self.fc_hid_dim, + act="softmax", + ) + self._fc_prediction = Linear( + input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax" + ) @declarative def forward(self, inputs, label=None): emb = self.embedding(inputs) - o_np_mask = (fluid.layers.reshape(inputs, [-1, 1]) != - self.dict_dim).astype(dtype='float32') + o_np_mask = ( + fluid.layers.reshape(inputs, [-1, 1]) != self.dict_dim + ).astype(dtype='float32') mask_emb = fluid.layers.expand(o_np_mask, [1, self.hid_dim]) emb = emb * mask_emb emb = fluid.layers.reshape( - emb, shape=[-1, self.channels, self.seq_len, self.hid_dim]) + emb, shape=[-1, self.channels, self.seq_len, self.hid_dim] + ) conv_3 = self._simple_conv_pool_1(emb) fc_1 = self._fc1(conv_3) prediction = self._fc_prediction(fc_1) @@ -103,7 +113,6 @@ class CNN(fluid.dygraph.Layer): class BOW(fluid.dygraph.Layer): - def __init__(self, dict_dim, batch_size, seq_len): super(BOW, self).__init__() self.dict_dim = dict_dim @@ -113,24 +122,27 @@ class BOW(fluid.dygraph.Layer): self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len - self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], - dtype='float32', - is_sparse=False) - self._fc1 = Linear(input_dim=self.hid_dim, - output_dim=self.hid_dim, - act="tanh") - self._fc2 = Linear(input_dim=self.hid_dim, - output_dim=self.fc_hid_dim, - act="tanh") - self._fc_prediction = Linear(input_dim=self.fc_hid_dim, - output_dim=self.class_dim, - act="softmax") + self.embedding = Embedding( + size=[self.dict_dim + 1, self.emb_dim], + dtype='float32', + is_sparse=False, + ) + self._fc1 = Linear( + input_dim=self.hid_dim, output_dim=self.hid_dim, act="tanh" + ) + self._fc2 = Linear( + input_dim=self.hid_dim, output_dim=self.fc_hid_dim, act="tanh" + ) + self._fc_prediction = Linear( + input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax" + ) @declarative def forward(self, inputs, label=None): emb = self.embedding(inputs) - o_np_mask = (fluid.layers.reshape(inputs, [-1, 1]) != - self.dict_dim).astype(dtype='float32') + o_np_mask = ( + fluid.layers.reshape(inputs, [-1, 1]) != self.dict_dim + ).astype(dtype='float32') mask_emb = fluid.layers.expand(o_np_mask, [1, self.hid_dim]) emb = emb * mask_emb emb = fluid.layers.reshape(emb, shape=[-1, self.seq_len, self.hid_dim]) @@ -147,7 +159,6 @@ class BOW(fluid.dygraph.Layer): class GRU(fluid.dygraph.Layer): - def __init__(self, dict_dim, batch_size, seq_len): super(GRU, self).__init__() self.dict_dim = dict_dim @@ -157,30 +168,34 @@ class GRU(fluid.dygraph.Layer): self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len - self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], - dtype='float32', - param_attr=fluid.ParamAttr(learning_rate=30), - is_sparse=False) + self.embedding = Embedding( + size=[self.dict_dim + 1, self.emb_dim], + dtype='float32', + param_attr=fluid.ParamAttr(learning_rate=30), + is_sparse=False, + ) h_0 = np.zeros((self.batch_size, self.hid_dim), dtype="float32") h_0 = to_variable(h_0) self._fc1 = Linear(input_dim=self.hid_dim, output_dim=self.hid_dim * 3) - self._fc2 = Linear(input_dim=self.hid_dim, - output_dim=self.fc_hid_dim, - act="tanh") - self._fc_prediction = Linear(input_dim=self.fc_hid_dim, - output_dim=self.class_dim, - act="softmax") + self._fc2 = Linear( + input_dim=self.hid_dim, output_dim=self.fc_hid_dim, act="tanh" + ) + self._fc_prediction = Linear( + input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax" + ) self._gru = DynamicGRU(size=self.hid_dim, h_0=h_0) @declarative def forward(self, inputs, label=None): emb = self.embedding(inputs) - o_np_mask = (fluid.layers.reshape(inputs, [-1, 1]) != - self.dict_dim).astype('float32') + o_np_mask = ( + fluid.layers.reshape(inputs, [-1, 1]) != self.dict_dim + ).astype('float32') mask_emb = fluid.layers.expand(o_np_mask, [1, self.hid_dim]) emb = emb * mask_emb - emb = fluid.layers.reshape(emb, - shape=[self.batch_size, -1, self.hid_dim]) + emb = fluid.layers.reshape( + emb, shape=[self.batch_size, -1, self.hid_dim] + ) fc_1 = self._fc1(emb) gru_hidden = self._gru(fc_1) gru_hidden = fluid.layers.reduce_max(gru_hidden, dim=1) @@ -195,7 +210,6 @@ class GRU(fluid.dygraph.Layer): class BiGRU(fluid.dygraph.Layer): - def __init__(self, dict_dim, batch_size, seq_len): super(BiGRU, self).__init__() self.dict_dim = dict_dim @@ -205,42 +219,47 @@ class BiGRU(fluid.dygraph.Layer): self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len - self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], - dtype='float32', - param_attr=fluid.ParamAttr(learning_rate=30), - is_sparse=False) + self.embedding = Embedding( + size=[self.dict_dim + 1, self.emb_dim], + dtype='float32', + param_attr=fluid.ParamAttr(learning_rate=30), + is_sparse=False, + ) h_0 = np.zeros((self.batch_size, self.hid_dim), dtype="float32") h_0 = to_variable(h_0) self._fc1 = Linear(input_dim=self.hid_dim, output_dim=self.hid_dim * 3) - self._fc2 = Linear(input_dim=self.hid_dim * 2, - output_dim=self.fc_hid_dim, - act="tanh") - self._fc_prediction = Linear(input_dim=self.fc_hid_dim, - output_dim=self.class_dim, - act="softmax") - self._gru_forward = DynamicGRU(size=self.hid_dim, - h_0=h_0, - is_reverse=False) - self._gru_backward = DynamicGRU(size=self.hid_dim, - h_0=h_0, - is_reverse=True) + self._fc2 = Linear( + input_dim=self.hid_dim * 2, output_dim=self.fc_hid_dim, act="tanh" + ) + self._fc_prediction = Linear( + input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax" + ) + self._gru_forward = DynamicGRU( + size=self.hid_dim, h_0=h_0, is_reverse=False + ) + self._gru_backward = DynamicGRU( + size=self.hid_dim, h_0=h_0, is_reverse=True + ) @declarative def forward(self, inputs, label=None): emb = self.embedding(inputs) - o_np_mask = (fluid.layers.reshape(inputs, [-1, 1]) != - self.dict_dim).astype('float32') + o_np_mask = ( + fluid.layers.reshape(inputs, [-1, 1]) != self.dict_dim + ).astype('float32') mask_emb = fluid.layers.expand(o_np_mask, [1, self.hid_dim]) emb = emb * mask_emb - emb = fluid.layers.reshape(emb, - shape=[self.batch_size, -1, self.hid_dim]) + emb = fluid.layers.reshape( + emb, shape=[self.batch_size, -1, self.hid_dim] + ) fc_1 = self._fc1(emb) gru_forward = self._gru_forward(fc_1) gru_backward = self._gru_backward(fc_1) gru_forward_tanh = fluid.layers.tanh(gru_forward) gru_backward_tanh = fluid.layers.tanh(gru_backward) encoded_vector = fluid.layers.concat( - input=[gru_forward_tanh, gru_backward_tanh], axis=2) + input=[gru_forward_tanh, gru_backward_tanh], axis=2 + ) encoded_vector = fluid.layers.reduce_max(encoded_vector, dim=1) fc_2 = self._fc2(encoded_vector) prediction = self._fc_prediction(fc_2) @@ -261,11 +280,13 @@ def fake_data_reader(class_num, vocab_size, batch_size, padding_size): batch_data = [] while True: label = local_random.randint(0, class_num) - seq_len = local_random.randint(padding_size // 2, - int(padding_size * 1.2)) + seq_len = local_random.randint( + padding_size // 2, int(padding_size * 1.2) + ) word_ids = local_random.randint(0, vocab_size, [seq_len]).tolist() - word_ids = word_ids[:padding_size] + [vocab_size - ] * (padding_size - seq_len) + word_ids = word_ids[:padding_size] + [vocab_size] * ( + padding_size - seq_len + ) batch_data.append((word_ids, [label], seq_len)) if len(batch_data) == batch_size: yield batch_data @@ -287,16 +308,20 @@ class Args(object): def train(args, to_static): program_translator.enable(to_static) - place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): np.random.seed(SEED) paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - train_reader = fake_data_reader(args.class_num, args.vocab_size, - args.batch_size, args.padding_size) + train_reader = fake_data_reader( + args.class_num, args.vocab_size, args.batch_size, args.padding_size + ) train_loader = fluid.io.DataLoader.from_generator(capacity=24) train_loader.set_sample_list_generator(train_reader) @@ -309,7 +334,8 @@ def train(args, to_static): elif args.model_type == 'bigru_net': model = BiGRU(args.vocab_size, args.batch_size, args.padding_size) sgd_optimizer = fluid.optimizer.Adagrad( - learning_rate=args.lr, parameter_list=model.parameters()) + learning_rate=args.lr, parameter_list=model.parameters() + ) loss_data = [] for eop in range(args.epoch): @@ -333,9 +359,14 @@ def train(args, to_static): # used_time may be 0.0, cause zero division error if used_time < 1e-5: used_time = 1e-5 - print("step: %d, ave loss: %f, speed: %f steps/s" % - (batch_id, avg_cost.numpy()[0], - args.log_step / used_time)) + print( + "step: %d, ave loss: %f, speed: %f steps/s" + % ( + batch_id, + avg_cost.numpy()[0], + args.log_step / used_time, + ) + ) time_begin = time.time() if batch_id == args.train_step: @@ -345,7 +376,6 @@ def train(args, to_static): class TestSentiment(unittest.TestCase): - def setUp(self): self.args = Args() @@ -353,11 +383,12 @@ class TestSentiment(unittest.TestCase): self.args.model_type = model_type st_out = train(self.args, True) dy_out = train(self.args, False) - np.testing.assert_allclose(dy_out, - st_out, - rtol=1e-05, - err_msg='dy_out:\n {}\n st_out:\n {}'.format( - dy_out, st_out)) + np.testing.assert_allclose( + dy_out, + st_out, + rtol=1e-05, + err_msg='dy_out:\n {}\n st_out:\n {}'.format(dy_out, st_out), + ) def test_train(self): model_types = ['cnn_net', 'bow_net', 'gru_net', 'bigru_net'] diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_seq2seq.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_seq2seq.py index 7ed2d12f5a8..d4932b710c3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_seq2seq.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_seq2seq.py @@ -26,8 +26,9 @@ from seq2seq_dygraph_model import BaseModel, AttentionModel from seq2seq_utils import Seq2SeqModelHyperParams from seq2seq_utils import get_data_iter -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) program_translator = ProgramTranslator() STEP_NUM = 10 PRINT_STEP = 2 @@ -51,26 +52,32 @@ def train(args, attn_model=False): fluid.default_main_program().random_seed = 2020 if attn_model: - model = AttentionModel(args.hidden_size, - args.src_vocab_size, - args.tar_vocab_size, - args.batch_size, - num_layers=args.num_layers, - init_scale=args.init_scale, - dropout=args.dropout) + model = AttentionModel( + args.hidden_size, + args.src_vocab_size, + args.tar_vocab_size, + args.batch_size, + num_layers=args.num_layers, + init_scale=args.init_scale, + dropout=args.dropout, + ) else: - model = BaseModel(args.hidden_size, - args.src_vocab_size, - args.tar_vocab_size, - args.batch_size, - num_layers=args.num_layers, - init_scale=args.init_scale, - dropout=args.dropout) + model = BaseModel( + args.hidden_size, + args.src_vocab_size, + args.tar_vocab_size, + args.batch_size, + num_layers=args.num_layers, + init_scale=args.init_scale, + dropout=args.dropout, + ) gloabl_norm_clip = GradientClipByGlobalNorm(args.max_grad_norm) - optimizer = fluid.optimizer.SGD(args.learning_rate, - parameter_list=model.parameters(), - grad_clip=gloabl_norm_clip) + optimizer = fluid.optimizer.SGD( + args.learning_rate, + parameter_list=model.parameters(), + grad_clip=gloabl_norm_clip, + ) model.train() train_data_iter = get_data_iter(args.batch_size) @@ -96,8 +103,15 @@ def train(args, attn_model=False): if batch_id % PRINT_STEP == 0: print( "Batch:[%d]; Time: %.5f s; loss: %.5f; total_loss: %.5f; word num: %.5f; ppl: %.5f" - % (batch_id, batch_time, loss.numpy(), total_loss.numpy(), - word_count, np.exp(total_loss.numpy() / word_count))) + % ( + batch_id, + batch_time, + loss.numpy(), + total_loss.numpy(), + word_count, + np.exp(total_loss.numpy() / word_count), + ) + ) if attn_model: # NOTE: Please see code of AttentionModel. @@ -108,7 +122,9 @@ def train(args, attn_model=False): if batch_id + 1 >= STEP_NUM: break - model_path = args.attn_model_path if attn_model else args.base_model_path + model_path = ( + args.attn_model_path if attn_model else args.base_model_path + ) model_dir = os.path.join(model_path) if not os.path.exists(model_dir): @@ -121,27 +137,33 @@ def infer(args, attn_model=False): with fluid.dygraph.guard(place): if attn_model: - model = AttentionModel(args.hidden_size, - args.src_vocab_size, - args.tar_vocab_size, - args.batch_size, - beam_size=args.beam_size, - num_layers=args.num_layers, - init_scale=args.init_scale, - dropout=0.0, - mode='beam_search') + model = AttentionModel( + args.hidden_size, + args.src_vocab_size, + args.tar_vocab_size, + args.batch_size, + beam_size=args.beam_size, + num_layers=args.num_layers, + init_scale=args.init_scale, + dropout=0.0, + mode='beam_search', + ) else: - model = BaseModel(args.hidden_size, - args.src_vocab_size, - args.tar_vocab_size, - args.batch_size, - beam_size=args.beam_size, - num_layers=args.num_layers, - init_scale=args.init_scale, - dropout=0.0, - mode='beam_search') - - model_path = args.attn_model_path if attn_model else args.base_model_path + model = BaseModel( + args.hidden_size, + args.src_vocab_size, + args.tar_vocab_size, + args.batch_size, + beam_size=args.beam_size, + num_layers=args.num_layers, + init_scale=args.init_scale, + dropout=0.0, + mode='beam_search', + ) + + model_path = ( + args.attn_model_path if attn_model else args.base_model_path + ) state_dict, _ = fluid.dygraph.load_dygraph(model_path) model.set_dict(state_dict) model.eval() @@ -158,16 +180,18 @@ def infer(args, attn_model=False): class TestSeq2seq(unittest.TestCase): - def setUp(self): self.args = Seq2SeqModelHyperParams self.temp_dir = tempfile.TemporaryDirectory() - self.args.base_model_path = os.path.join(self.temp_dir.name, - self.args.base_model_path) - self.args.attn_model_path = os.path.join(self.temp_dir.name, - self.args.attn_model_path) - self.args.reload_model = os.path.join(self.temp_dir.name, - self.args.reload_model) + self.args.base_model_path = os.path.join( + self.temp_dir.name, self.args.base_model_path + ) + self.args.attn_model_path = os.path.join( + self.temp_dir.name, self.args.attn_model_path + ) + self.args.reload_model = os.path.join( + self.temp_dir.name, self.args.reload_model + ) def tearDown(self): self.temp_dir.cleanup() @@ -190,17 +214,23 @@ class TestSeq2seq(unittest.TestCase): dygraph_loss = self.run_dygraph(mode="train", attn_model=attn_model) static_loss = self.run_static(mode="train", attn_model=attn_model) result = np.allclose(dygraph_loss, static_loss) - self.assertTrue(result, - msg="\ndygraph_loss = {} \nstatic_loss = {}".format( - dygraph_loss, static_loss)) + self.assertTrue( + result, + msg="\ndygraph_loss = {} \nstatic_loss = {}".format( + dygraph_loss, static_loss + ), + ) def _test_predict(self, attn_model=False): pred_dygraph = self.run_dygraph(mode="test", attn_model=attn_model) pred_static = self.run_static(mode="test", attn_model=attn_model) result = np.allclose(pred_static, pred_dygraph) - self.assertTrue(result, - msg="\npred_dygraph = {} \npred_static = {}".format( - pred_dygraph, pred_static)) + self.assertTrue( + result, + msg="\npred_dygraph = {} \npred_static = {}".format( + pred_dygraph, pred_static + ), + ) def test_base_model(self): self._test_train(attn_model=False) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py index 0a5d567996d..405eb089d0c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py @@ -28,10 +28,10 @@ def setter(values): class TestGetterSetterHelper(unittest.TestCase): - def test_1(self): - helper = GetterSetterHelper(getter, setter, ['a', 'b', 'e'], - ['d', 'f', 'e']) + helper = GetterSetterHelper( + getter, setter, ['a', 'b', 'e'], ['d', 'f', 'e'] + ) print(helper.union()) expect_union = ['a', 'b', 'd', 'e', 'f'] assert helper.union() == expect_union diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py index 2bc344ae95a..466c6affcb8 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py @@ -36,22 +36,24 @@ def create_conf_dict(): def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("--batch_size", - type=int, - default=32, - help="Total examples' number in batch for training.") - parser.add_argument("--seq_len", - type=int, - default=32, - help="The length of each sentence.") - parser.add_argument("--epoch", - type=int, - default=1, - help="The number of training epoch.") - parser.add_argument("--fake_sample_size", - type=int, - default=128, - help="The number of samples of fake data.") + parser.add_argument( + "--batch_size", + type=int, + default=32, + help="Total examples' number in batch for training.", + ) + parser.add_argument( + "--seq_len", type=int, default=32, help="The length of each sentence." + ) + parser.add_argument( + "--epoch", type=int, default=1, help="The number of training epoch." + ) + parser.add_argument( + "--fake_sample_size", + type=int, + default=128, + help="The number of samples of fake data.", + ) args = parser.parse_args([]) return args @@ -72,7 +74,6 @@ vocab = fake_vocabulary() class FakeReaderProcessor(object): - def __init__(self, args, vocab): self.vocab = vocab self.seq_len = args.seq_len @@ -83,10 +84,10 @@ class FakeReaderProcessor(object): pos_title = query[:] neg_title = [26 - q for q in query] self.data_samples.append( - np.array([query, pos_title, neg_title]).astype(np.int64)) + np.array([query, pos_title, neg_title]).astype(np.int64) + ) def get_reader(self, mode, epoch=0): - def reader_with_pairwise(): if mode == "train": for i in range(self.sample_size): @@ -125,7 +126,8 @@ def train(conf_dict, to_static): beta1=0.9, beta2=0.999, epsilon=1e-08, - parameter_list=net.parameters()) + parameter_list=net.parameters(), + ) metric = fluid.metrics.Auc(name="auc") @@ -133,14 +135,14 @@ def train(conf_dict, to_static): losses = [] train_loader = fluid.io.DataLoader.from_generator( - capacity=16, - return_list=True, - iterable=True, - use_double_buffer=True) - get_train_examples = simnet_process.get_reader("train", - epoch=args.epoch) + capacity=16, return_list=True, iterable=True, use_double_buffer=True + ) + get_train_examples = simnet_process.get_reader( + "train", epoch=args.epoch + ) train_loader.set_sample_list_generator( - paddle.batch(get_train_examples, batch_size=args.batch_size), place) + paddle.batch(get_train_examples, batch_size=args.batch_size), place + ) for left, pos_right, neg_right in train_loader(): left = fluid.layers.reshape(left, shape=[-1, 1]) @@ -160,7 +162,6 @@ def train(conf_dict, to_static): class TestSimnet(unittest.TestCase): - def test_dygraph_static_same_loss(self): if fluid.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py index f2c72e9932e..bbb408e48c3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py @@ -34,22 +34,24 @@ def create_conf_dict(): def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("--batch_size", - type=int, - default=32, - help="Total examples' number in batch for training.") - parser.add_argument("--seq_len", - type=int, - default=32, - help="The length of each sentence.") - parser.add_argument("--epoch", - type=int, - default=1, - help="The number of training epoch.") - parser.add_argument("--fake_sample_size", - type=int, - default=128, - help="The number of samples of fake data.") + parser.add_argument( + "--batch_size", + type=int, + default=32, + help="Total examples' number in batch for training.", + ) + parser.add_argument( + "--seq_len", type=int, default=32, help="The length of each sentence." + ) + parser.add_argument( + "--epoch", type=int, default=1, help="The number of training epoch." + ) + parser.add_argument( + "--fake_sample_size", + type=int, + default=128, + help="The number of samples of fake data.", + ) args = parser.parse_args([]) return args @@ -70,7 +72,6 @@ vocab = fake_vocabulary() class FakeReaderProcessor(object): - def __init__(self, args, vocab): self.vocab = vocab self.seq_len = args.seq_len @@ -81,10 +82,10 @@ class FakeReaderProcessor(object): pos_title = query[:] neg_title = [26 - q for q in query] self.data_samples.append( - np.array([query, pos_title, neg_title]).astype(np.int64)) + np.array([query, pos_title, neg_title]).astype(np.int64) + ) def get_reader(self, mode, epoch=0): - def reader_with_pairwise(): if mode == "train": for i in range(self.sample_size): @@ -118,24 +119,26 @@ def train(conf_dict, to_static): net = BOW(conf_dict) loss = HingeLoss(conf_dict) - optimizer = paddle.optimizer.Adam(learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - parameters=net.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + parameters=net.parameters(), + ) metric = paddle.metric.Auc(name="auc") global_step = 0 losses = [] - train_loader = paddle.io.DataLoader.from_generator(capacity=16, - return_list=True, - iterable=True, - use_double_buffer=True) + train_loader = paddle.io.DataLoader.from_generator( + capacity=16, return_list=True, iterable=True, use_double_buffer=True + ) get_train_examples = simnet_process.get_reader("train", epoch=args.epoch) train_loader.set_sample_list_generator( - paddle.batch(get_train_examples, batch_size=args.batch_size), place) + paddle.batch(get_train_examples, batch_size=args.batch_size), place + ) for left, pos_right, neg_right in train_loader(): left = paddle.reshape(left, shape=[-1, 1]) @@ -156,7 +159,6 @@ def train(conf_dict, to_static): class TestSimnet(unittest.TestCase): - def test_dygraph_static_same_loss(self): if paddle.is_compiled_with_cuda(): paddle.fluid.set_flags({"FLAGS_cudnn_deterministic": True}) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py index 96bcc6d016e..ffd3e887933 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py @@ -97,7 +97,6 @@ def test_set_value(x): class LayerWithSetValue(paddle.nn.Layer): - def __init__(self, input_dim, hidden): super(LayerWithSetValue, self).__init__() self.linear = paddle.nn.Linear(input_dim, hidden) @@ -110,11 +109,13 @@ class LayerWithSetValue(paddle.nn.Layer): class TestSliceWithoutControlFlow(unittest.TestCase): - def setUp(self): self.init_input() - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.init_dygraph_func() paddle.disable_static() @@ -142,25 +143,21 @@ class TestSliceWithoutControlFlow(unittest.TestCase): class TestSliceInIf(TestSliceWithoutControlFlow): - def init_dygraph_func(self): self.dygraph_func = test_slice_in_if class TestSliceInWhileLoop(TestSliceWithoutControlFlow): - def init_dygraph_func(self): self.dygraph_func = test_slice_in_while_loop class TestSliceInForLoop(TestSliceWithoutControlFlow): - def init_dygraph_func(self): self.dygraph_func = test_slice_in_for_loop class TestSetValue(TestSliceWithoutControlFlow): - def init_input(self): self.input = np.full([3, 4, 5], 5).astype('float32') @@ -169,11 +166,11 @@ class TestSetValue(TestSliceWithoutControlFlow): class TestSetValueWithLayerAndSave(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "layer_use_set_value") + self.model_path = os.path.join( + self.temp_dir.name, "layer_use_set_value" + ) def tearDown(self): self.temp_dir.cleanup() @@ -182,10 +179,9 @@ class TestSetValueWithLayerAndSave(unittest.TestCase): prog_trans.enable(True) model = LayerWithSetValue(input_dim=10, hidden=1) x = paddle.full(shape=[5, 10], fill_value=5.0, dtype="float32") - paddle.jit.save(layer=model, - path=self.model_path, - input_spec=[x], - output_spec=None) + paddle.jit.save( + layer=model, path=self.model_path, input_spec=[x], output_spec=None + ) class TestSliceSupplementSpecialCase(unittest.TestCase): @@ -218,18 +214,20 @@ class TestSliceSupplementSpecialCase(unittest.TestCase): return inps[::2], inps[::-2] origin_result = func(inps) - sfunc = paddle.jit.to_static(func, - input_spec=[InputSpec(shape=[None, 4, 4])]) + sfunc = paddle.jit.to_static( + func, input_spec=[InputSpec(shape=[None, 4, 4])] + ) static_result = sfunc(inps) - np.testing.assert_array_equal(origin_result[0].numpy(), - static_result[0].numpy()) - np.testing.assert_array_equal(origin_result[1].numpy(), - static_result[1].numpy()) + np.testing.assert_array_equal( + origin_result[0].numpy(), static_result[0].numpy() + ) + np.testing.assert_array_equal( + origin_result[1].numpy(), static_result[1].numpy() + ) class TestPaddleStridedSlice(unittest.TestCase): - def test_compare_paddle_strided_slice_with_numpy(self): paddle.disable_static() array = np.arange(5) @@ -238,19 +236,21 @@ class TestPaddleStridedSlice(unittest.TestCase): s1 = 3 e1 = 1 stride1 = -2 - sl = paddle.strided_slice(pt, - axes=[ - 0, - ], - starts=[ - s1, - ], - ends=[ - e1, - ], - strides=[ - stride1, - ]) + sl = paddle.strided_slice( + pt, + axes=[ + 0, + ], + starts=[ + s1, + ], + ends=[ + e1, + ], + strides=[ + stride1, + ], + ) self.assertTrue(array[s1:e1:stride1], sl) @@ -259,27 +259,27 @@ class TestPaddleStridedSlice(unittest.TestCase): s2 = [8, -1] e2 = [1, -5] stride2 = [-2, -3] - sl = paddle.strided_slice(pt, - axes=[0, 1], - starts=s2, - ends=e2, - strides=stride2) + sl = paddle.strided_slice( + pt, axes=[0, 1], starts=s2, ends=e2, strides=stride2 + ) np.testing.assert_array_equal( - sl.numpy(), array[s2[0]:e2[0]:stride2[0], s2[1]:e2[1]:stride2[1]]) + sl.numpy(), + array[s2[0] : e2[0] : stride2[0], s2[1] : e2[1] : stride2[1]], + ) array = np.arange(6 * 7 * 8).reshape((6, 7, 8)) pt = paddle.to_tensor(array) s2 = [7, -1] e2 = [2, -5] stride2 = [-2, -3] - sl = paddle.strided_slice(pt, - axes=[0, 2], - starts=s2, - ends=e2, - strides=stride2) + sl = paddle.strided_slice( + pt, axes=[0, 2], starts=s2, ends=e2, strides=stride2 + ) - array_slice = array[s2[0]:e2[0]:stride2[0], ::, s2[1]:e2[1]:stride2[1]] + array_slice = array[ + s2[0] : e2[0] : stride2[0], ::, s2[1] : e2[1] : stride2[1] + ] np.testing.assert_array_equal(sl.numpy(), array_slice) @@ -289,7 +289,6 @@ def slice_zero_shape_tensor(x): class TestSliceZeroShapeTensor(unittest.TestCase): - def test_slice(self): paddle.disable_static() x = paddle.ones([0, 0, 0, 0]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py index cfbf47a318f..8ee2fcd0d59 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py @@ -18,7 +18,6 @@ import unittest class Net(Layer): - def __init__(self): super(Net, self).__init__() self.fc = paddle.nn.Linear(16, 3) @@ -36,7 +35,6 @@ class Net(Layer): class TestArgsSpecName(unittest.TestCase): - def read_from_dataset(self): self.x = paddle.randn([4, 2, 8]) self.y = paddle.randn([4, 2, 8]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py index 8f06b96122d..7a2e0e09b21 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py @@ -19,7 +19,10 @@ import paddle import paddle.fluid as fluid import unittest -from paddle.fluid.dygraph.dygraph_to_static import NodeVarType, StaticAnalysisVisitor +from paddle.fluid.dygraph.dygraph_to_static import ( + NodeVarType, + StaticAnalysisVisitor, +) def func_to_test1(a, b): @@ -71,7 +74,7 @@ result_var_type3 = { 'i': {NodeVarType.BOOLEAN}, 'j': {NodeVarType.UNKNOWN}, 'k': {NodeVarType.FLOAT}, - 'l': {NodeVarType.PADDLE_RETURN_TYPES} + 'l': {NodeVarType.PADDLE_RETURN_TYPES}, } @@ -87,12 +90,11 @@ result_var_type4 = { 'a': {NodeVarType.NUMPY_NDARRAY}, 'b': {NodeVarType.NUMPY_NDARRAY}, 'c': {NodeVarType.TENSOR}, - 'd': {NodeVarType.TENSOR} + 'd': {NodeVarType.TENSOR}, } def func_to_test5(): - def inner_int_func(): return 1 @@ -139,7 +141,7 @@ result_var_type6 = { 'i': {NodeVarType.INT}, 'x': {NodeVarType.INT}, 'y': {NodeVarType.INT}, - 'add': {NodeVarType.INT} + 'add': {NodeVarType.INT}, } @@ -156,21 +158,30 @@ result_var_type7 = { 'd': {NodeVarType.STRING}, 'e': {NodeVarType.PADDLE_RETURN_TYPES}, 'f': {NodeVarType.PADDLE_RETURN_TYPES}, - 'g': {NodeVarType.TENSOR} + 'g': {NodeVarType.TENSOR}, } test_funcs = [ - func_to_test1, func_to_test2, func_to_test3, func_to_test4, func_to_test5, - func_to_test6, func_to_test7 + func_to_test1, + func_to_test2, + func_to_test3, + func_to_test4, + func_to_test5, + func_to_test6, + func_to_test7, ] result_var_type = [ - result_var_type1, result_var_type2, result_var_type3, result_var_type4, - result_var_type5, result_var_type6, result_var_type7 + result_var_type1, + result_var_type2, + result_var_type3, + result_var_type4, + result_var_type5, + result_var_type6, + result_var_type7, ] class TestStaticAnalysis(unittest.TestCase): - def _check_wrapper(self, wrapper, node_to_wrapper_map): self.assertEqual(node_to_wrapper_map[wrapper.node], wrapper) if wrapper.parent is not None: diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_methods.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_methods.py index 3d88a03247c..ecda3427e7e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_methods.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_methods.py @@ -25,7 +25,6 @@ def tensor_clone(x): class TestTensorClone(unittest.TestCase): - def _run(self, to_static): prog_trans = paddle.jit.ProgramTranslator() prog_trans.enable(to_static) @@ -46,7 +45,6 @@ def tensor_numpy(x): class TestTensorDygraphOnlyMethodError(unittest.TestCase): - def _run(self, to_static): prog_trans = paddle.jit.ProgramTranslator() prog_trans.enable(to_static) @@ -68,7 +66,6 @@ def tensor_item(x): class TestTensorItem(unittest.TestCase): - def _run(self, to_static): prog_trans = paddle.jit.ProgramTranslator() prog_trans.enable(to_static) @@ -92,7 +89,6 @@ def tensor_size(x): class TestTensorSize(unittest.TestCase): - def _run(self, to_static): prog_trans = paddle.jit.ProgramTranslator() prog_trans.enable(to_static) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py index 55412fb0c49..5d06d1c694e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py @@ -109,13 +109,13 @@ def dyfunc_with_if_1(x): # `res.shape[0]` is transformed into # `paddle.jit.dy2static.convert_var_shape(res)[0]` if res.shape[0] > 1: - res = fluid.layers.fill_constant(value=2, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=2, shape=x.shape, dtype="int32" + ) else: - res = fluid.layers.fill_constant(value=3, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=3, shape=x.shape, dtype="int32" + ) return res @@ -231,11 +231,13 @@ def dyfunc_dict_assign_shape(): # 1. Basic tests without control flow class TestTensorShapeBasic(unittest.TestCase): - def setUp(self): self.input = np.ones(5).astype("int32") - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self._set_input_spec() self._set_expected_op_num() self.init_test_func() @@ -277,9 +279,11 @@ class TestTensorShapeBasic(unittest.TestCase): for block in program.blocks: self.shape_op_num += len( - [op for op in block.ops if op.type == "shape"]) + [op for op in block.ops if op.type == "shape"] + ) self.slice_op_num += len( - [op for op in block.ops if op.type == "slice"]) + [op for op in block.ops if op.type == "slice"] + ) def test_op_num(self): static_layer = paddle.jit.to_static(self.dygraph_func, self.input_spec) @@ -291,7 +295,6 @@ class TestTensorShapeBasic(unittest.TestCase): class TestTensorShapeBasic2(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_2 @@ -302,19 +305,16 @@ class TestTensorShapeBasic2(TestTensorShapeBasic): class TestTensorShapeBasic3(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_3 class TestTensorShapeBasic4(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_4 class TestTensorShapeBasic5(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_5 @@ -325,7 +325,6 @@ class TestTensorShapeBasic5(TestTensorShapeBasic): class TestTensorShapeBasic6(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_6 @@ -336,7 +335,6 @@ class TestTensorShapeBasic6(TestTensorShapeBasic): class TestTupleShape1(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((5, 7)).astype("int32") self.input_spec = [ @@ -351,7 +349,6 @@ class TestTupleShape1(TestTensorShapeBasic): class TestTupleShape2(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((5, 7)).astype("int32") self.input_spec = [ @@ -366,7 +363,6 @@ class TestTupleShape2(TestTensorShapeBasic): class TestTupleShape3(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((5, 7)).astype("int32") self.input_spec = [paddle.static.InputSpec(shape=[5, 7], dtype="int32")] @@ -379,7 +375,6 @@ class TestTupleShape3(TestTensorShapeBasic): class TestPaddleShapeApi(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((5, 7)).astype("int32") self.input_spec = [paddle.static.InputSpec(shape=[5, 7], dtype="int32")] @@ -393,7 +388,6 @@ class TestPaddleShapeApi(TestTensorShapeBasic): # 2. Tests with control flow if class TestTensorShapeInIf1(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_if_1 @@ -404,7 +398,6 @@ class TestTensorShapeInIf1(TestTensorShapeBasic): class TestTensorShapeInIf2(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_if_2 @@ -416,7 +409,6 @@ class TestTensorShapeInIf2(TestTensorShapeBasic): # 3. Tests with control flow for loop class TestTensorShapeInFor1(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_for_1 @@ -427,7 +419,6 @@ class TestTensorShapeInFor1(TestTensorShapeBasic): class TestTensorShapeInFor2(TestTensorShapeInFor1): - def init_test_func(self): self.dygraph_func = dyfunc_with_for_2 @@ -438,7 +429,6 @@ class TestTensorShapeInFor2(TestTensorShapeInFor1): class TestTensorShapeInFor3(TestTensorShapeInFor1): - def init_test_func(self): self.dygraph_func = dyfunc_with_for_3 @@ -450,7 +440,6 @@ class TestTensorShapeInFor3(TestTensorShapeInFor1): # 4. Tests with control flow while loop class TestTensorShapeInWhile1(TestTensorShapeInFor1): - def init_test_func(self): self.dygraph_func = dyfunc_with_while_1 @@ -461,7 +450,6 @@ class TestTensorShapeInWhile1(TestTensorShapeInFor1): class TestTensorShapeInWhile2(TestTensorShapeInFor1): - def init_test_func(self): self.dygraph_func = dyfunc_with_while_2 @@ -472,7 +460,6 @@ class TestTensorShapeInWhile2(TestTensorShapeInFor1): class TestTensorShapeInWhile3(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_while_3 @@ -483,7 +470,6 @@ class TestTensorShapeInWhile3(TestTensorShapeBasic): class TestTensorShapeInWhile4(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_while_4 @@ -495,7 +481,6 @@ class TestTensorShapeInWhile4(TestTensorShapeBasic): # 5. Test op num for negetive dim class TestOpNumBasicWithTensorShape(unittest.TestCase): - def setUp(self): self._set_input_spec() self._set_test_func() @@ -521,9 +506,11 @@ class TestOpNumBasicWithTensorShape(unittest.TestCase): for block in program.blocks: self.shape_op_num += len( - [op for op in block.ops if op.type == "shape"]) + [op for op in block.ops if op.type == "shape"] + ) self.slice_op_num += len( - [op for op in block.ops if op.type == "slice"]) + [op for op in block.ops if op.type == "slice"] + ) def test_op_num(self): static_layer = paddle.jit.to_static(self.dygraph_func, self.input_spec) @@ -536,7 +523,6 @@ class TestOpNumBasicWithTensorShape(unittest.TestCase): class TestOpNumBasicWithTensorShape4(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_tensor_shape_4 @@ -547,7 +533,6 @@ class TestOpNumBasicWithTensorShape4(TestOpNumBasicWithTensorShape): class TestOpNumWithTensorShapeTuple1(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_tuple_shape_1 @@ -558,7 +543,6 @@ class TestOpNumWithTensorShapeTuple1(TestOpNumBasicWithTensorShape): class TestOpNumWithTensorShapeInIf1(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_with_if_1 @@ -569,7 +553,6 @@ class TestOpNumWithTensorShapeInIf1(TestOpNumBasicWithTensorShape): class TestOpNumWithTensorShapeInFor1(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_with_for_1 @@ -580,7 +563,6 @@ class TestOpNumWithTensorShapeInFor1(TestOpNumBasicWithTensorShape): class TestOpNumWithTensorShapeInWhile1(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_with_while_1 @@ -591,7 +573,6 @@ class TestOpNumWithTensorShapeInWhile1(TestOpNumBasicWithTensorShape): class TestChangeShapeAfterAssign(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((2, 3)).astype("int32") self.input_spec = [ @@ -613,15 +594,14 @@ def dyfunc_with_static_convert_var_shape(x): else: # Test for correctly to find `batch_size__static_convert_var_shape_suffix_0` in # deeply nested scope. - res = fluid.layers.fill_constant(value=8, - shape=[batch_size], - dtype="int32") + res = fluid.layers.fill_constant( + value=8, shape=[batch_size], dtype="int32" + ) return res class TestFindStatiConvertVarShapeSuffixVar(unittest.TestCase): - def test(self): x_spec = paddle.static.InputSpec(shape=[None, 10]) func = paddle.jit.to_static(dyfunc_with_if_2, input_spec=[x_spec]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_to_tensor.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_to_tensor.py index af543a5d97b..2e8e9f06a1b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_to_tensor.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_to_tensor.py @@ -37,10 +37,9 @@ def case2(x): place = paddle.CUDAPlace(0) else: place = paddle.CPUPlace() - a = paddle.to_tensor([1.0, 2.0, 3.0], - place=place, - dtype="int64", - stop_gradient=False) + a = paddle.to_tensor( + [1.0, 2.0, 3.0], place=place, dtype="int64", stop_gradient=False + ) return a @@ -84,7 +83,6 @@ def case6(x): class TestToTensorReturnVal(unittest.TestCase): - def test_to_tensor_badreturn(self): paddle.disable_static() x = paddle.to_tensor([3]) @@ -133,7 +131,6 @@ class TestToTensorReturnVal(unittest.TestCase): class TestStatic(unittest.TestCase): - def test_static(self): paddle.enable_static() main_prog = Program() @@ -144,10 +141,12 @@ class TestStatic(unittest.TestCase): else: place = paddle.CPUPlace() - x = paddle.to_tensor(paddle.randn([5, 2]), - dtype='float64', - stop_gradient=False, - place=place) + x = paddle.to_tensor( + paddle.randn([5, 2]), + dtype='float64', + stop_gradient=False, + place=place, + ) out = paddle.static.nn.fc(x, 1) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py index f7b16c06cb8..d9eb890394f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py @@ -23,11 +23,16 @@ import paddle import paddle.fluid as fluid import transformer_util as util -from transformer_dygraph_model import CrossEntropyCriterion, Transformer, position_encoding_init +from transformer_dygraph_model import ( + CrossEntropyCriterion, + Transformer, + position_encoding_init, +) trainer_count = 1 -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) SEED = 10 STEP_NUM = 10 @@ -42,46 +47,73 @@ def train_static(args, batch_generator): with fluid.program_guard(train_prog, startup_prog): with fluid.unique_name.guard(): # define input and reader - input_field_names = util.encoder_data_input_fields + \ - util.decoder_data_input_fields[:-1] + util.label_data_input_fields + input_field_names = ( + util.encoder_data_input_fields + + util.decoder_data_input_fields[:-1] + + util.label_data_input_fields + ) input_descs = util.get_input_descs(args) - input_slots = [{ - "name": name, - "shape": input_descs[name][0], - "dtype": input_descs[name][1] - } for name in input_field_names] + input_slots = [ + { + "name": name, + "shape": input_descs[name][0], + "dtype": input_descs[name][1], + } + for name in input_field_names + ] input_field = util.InputField(input_slots) # Define DataLoader data_loader = fluid.io.DataLoader.from_generator( - input_field.feed_list, capacity=60) + input_field.feed_list, capacity=60 + ) data_loader.set_batch_generator(batch_generator, places=place) # define model transformer = Transformer( - args.src_vocab_size, args.trg_vocab_size, args.max_length + 1, - args.n_layer, args.n_head, args.d_key, args.d_value, - args.d_model, args.d_inner_hid, args.prepostprocess_dropout, - args.attention_dropout, args.relu_dropout, args.preprocess_cmd, - args.postprocess_cmd, args.weight_sharing, args.bos_idx, - args.eos_idx) + args.src_vocab_size, + args.trg_vocab_size, + args.max_length + 1, + args.n_layer, + args.n_head, + args.d_key, + args.d_value, + args.d_model, + args.d_inner_hid, + args.prepostprocess_dropout, + args.attention_dropout, + args.relu_dropout, + args.preprocess_cmd, + args.postprocess_cmd, + args.weight_sharing, + args.bos_idx, + args.eos_idx, + ) logits = transformer(*input_field.feed_list[:7]) # define loss criterion = CrossEntropyCriterion(args.label_smooth_eps) lbl_word, lbl_weight = input_field.feed_list[7:] - sum_cost, avg_cost, token_num = criterion(logits, lbl_word, - lbl_weight) + sum_cost, avg_cost, token_num = criterion( + logits, lbl_word, lbl_weight + ) # define optimizer learning_rate = fluid.layers.learning_rate_scheduler.noam_decay( - args.d_model, args.warmup_steps, args.learning_rate) - optimizer = fluid.optimizer.Adam(learning_rate=learning_rate, - beta1=args.beta1, - beta2=args.beta2, - epsilon=float(args.eps)) + args.d_model, args.warmup_steps, args.learning_rate + ) + optimizer = fluid.optimizer.Adam( + learning_rate=learning_rate, + beta1=args.beta1, + beta2=args.beta2, + epsilon=float(args.eps), + ) optimizer.minimize(avg_cost) # the best cross-entropy value with label smoothing - loss_normalizer = -((1. - args.label_smooth_eps) * np.log( - (1. - args.label_smooth_eps)) + args.label_smooth_eps * - np.log(args.label_smooth_eps / - (args.trg_vocab_size - 1) + 1e-20)) + loss_normalizer = -( + (1.0 - args.label_smooth_eps) + * np.log((1.0 - args.label_smooth_eps)) + + args.label_smooth_eps + * np.log( + args.label_smooth_eps / (args.trg_vocab_size - 1) + 1e-20 + ) + ) step_idx = 0 total_batch_num = 0 avg_loss = [] @@ -90,12 +122,15 @@ def train_static(args, batch_generator): for pass_id in range(args.epoch): batch_id = 0 for feed_dict in data_loader: - outs = exe.run(program=train_prog, - feed=feed_dict, - fetch_list=[sum_cost.name, token_num.name]) + outs = exe.run( + program=train_prog, + feed=feed_dict, + fetch_list=[sum_cost.name, token_num.name], + ) if step_idx % args.print_step == 0: sum_cost_val, token_num_val = np.array(outs[0]), np.array( - outs[1]) + outs[1] + ) total_sum_cost = sum_cost_val.sum() total_token_num = token_num_val.sum() total_avg_cost = total_sum_cost / total_token_num @@ -103,27 +138,40 @@ def train_static(args, batch_generator): if step_idx == 0: logging.info( "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f" % - (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]))) + "normalized loss: %f, ppl: %f" + % ( + step_idx, + pass_id, + batch_id, + total_avg_cost, + total_avg_cost - loss_normalizer, + np.exp([min(total_avg_cost, 100)]), + ) + ) avg_batch_time = time.time() else: logging.info( "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f, speed: %.2f steps/s" % - (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]), args.print_step / - (time.time() - avg_batch_time))) + "normalized loss: %f, ppl: %f, speed: %.2f steps/s" + % ( + step_idx, + pass_id, + batch_id, + total_avg_cost, + total_avg_cost - loss_normalizer, + np.exp([min(total_avg_cost, 100)]), + args.print_step / (time.time() - avg_batch_time), + ) + ) avg_batch_time = time.time() batch_id += 1 step_idx += 1 total_batch_num = total_batch_num + 1 if step_idx == STEP_NUM: if args.save_dygraph_model_path: - model_path = os.path.join(args.save_static_model_path, - "transformer") + model_path = os.path.join( + args.save_static_model_path, "transformer" + ) fluid.save(train_prog, model_path) break return np.array(avg_loss) @@ -139,30 +187,45 @@ def train_dygraph(args, batch_generator): train_loader.set_batch_generator(batch_generator, places=place) # define model transformer = Transformer( - args.src_vocab_size, args.trg_vocab_size, args.max_length + 1, - args.n_layer, args.n_head, args.d_key, args.d_value, args.d_model, - args.d_inner_hid, args.prepostprocess_dropout, - args.attention_dropout, args.relu_dropout, args.preprocess_cmd, - args.postprocess_cmd, args.weight_sharing, args.bos_idx, - args.eos_idx) + args.src_vocab_size, + args.trg_vocab_size, + args.max_length + 1, + args.n_layer, + args.n_head, + args.d_key, + args.d_value, + args.d_model, + args.d_inner_hid, + args.prepostprocess_dropout, + args.attention_dropout, + args.relu_dropout, + args.preprocess_cmd, + args.postprocess_cmd, + args.weight_sharing, + args.bos_idx, + args.eos_idx, + ) # define loss criterion = CrossEntropyCriterion(args.label_smooth_eps) # define optimizer learning_rate = fluid.layers.learning_rate_scheduler.noam_decay( - args.d_model, args.warmup_steps, args.learning_rate) + args.d_model, args.warmup_steps, args.learning_rate + ) # define optimizer optimizer = fluid.optimizer.Adam( learning_rate=learning_rate, beta1=args.beta1, beta2=args.beta2, epsilon=float(args.eps), - parameter_list=transformer.parameters()) + parameter_list=transformer.parameters(), + ) # the best cross-entropy value with label smoothing loss_normalizer = -( - (1. - args.label_smooth_eps) * np.log( - (1. - args.label_smooth_eps)) + - args.label_smooth_eps * np.log(args.label_smooth_eps / - (args.trg_vocab_size - 1) + 1e-20)) + (1.0 - args.label_smooth_eps) + * np.log((1.0 - args.label_smooth_eps)) + + args.label_smooth_eps + * np.log(args.label_smooth_eps / (args.trg_vocab_size - 1) + 1e-20) + ) ce_time = [] ce_ppl = [] avg_loss = [] @@ -171,14 +234,29 @@ def train_dygraph(args, batch_generator): pass_start_time = time.time() batch_id = 0 for input_data in train_loader(): - (src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, - lbl_weight) = input_data - logits = transformer(src_word, src_pos, src_slf_attn_bias, - trg_word, trg_pos, trg_slf_attn_bias, - trg_src_attn_bias) + ( + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + lbl_word, + lbl_weight, + ) = input_data + logits = transformer( + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + ) sum_cost, avg_cost, token_num = criterion( - logits, lbl_word, lbl_weight) + logits, lbl_word, lbl_weight + ) avg_cost.backward() optimizer.minimize(avg_cost) transformer.clear_gradients() @@ -188,20 +266,32 @@ def train_dygraph(args, batch_generator): if step_idx == 0: logging.info( "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f" % - (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]))) + "normalized loss: %f, ppl: %f" + % ( + step_idx, + pass_id, + batch_id, + total_avg_cost, + total_avg_cost - loss_normalizer, + np.exp([min(total_avg_cost, 100)]), + ) + ) avg_batch_time = time.time() else: logging.info( "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " "normalized loss: %f, ppl: %f, speed: %.2f steps/s" - % (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100) - ]), args.print_step / - (time.time() - avg_batch_time))) + % ( + step_idx, + pass_id, + batch_id, + total_avg_cost, + total_avg_cost - loss_normalizer, + np.exp([min(total_avg_cost, 100)]), + args.print_step + / (time.time() - avg_batch_time), + ) + ) ce_ppl.append(np.exp([min(total_avg_cost, 100)])) avg_batch_time = time.time() batch_id += 1 @@ -213,10 +303,12 @@ def train_dygraph(args, batch_generator): os.makedirs(model_dir) fluid.save_dygraph( transformer.state_dict(), - os.path.join(model_dir, "transformer")) + os.path.join(model_dir, "transformer"), + ) fluid.save_dygraph( optimizer.state_dict(), - os.path.join(model_dir, "transformer")) + os.path.join(model_dir, "transformer"), + ) break time_consumed = time.time() - pass_start_time ce_time.append(time_consumed) @@ -234,22 +326,37 @@ def predict_dygraph(args, batch_generator): # define model transformer = Transformer( - args.src_vocab_size, args.trg_vocab_size, args.max_length + 1, - args.n_layer, args.n_head, args.d_key, args.d_value, args.d_model, - args.d_inner_hid, args.prepostprocess_dropout, - args.attention_dropout, args.relu_dropout, args.preprocess_cmd, - args.postprocess_cmd, args.weight_sharing, args.bos_idx, - args.eos_idx) + args.src_vocab_size, + args.trg_vocab_size, + args.max_length + 1, + args.n_layer, + args.n_head, + args.d_key, + args.d_value, + args.d_model, + args.d_inner_hid, + args.prepostprocess_dropout, + args.attention_dropout, + args.relu_dropout, + args.preprocess_cmd, + args.postprocess_cmd, + args.weight_sharing, + args.bos_idx, + args.eos_idx, + ) # load the trained model model_dict, _ = util.load_dygraph( - os.path.join(args.save_dygraph_model_path, "transformer")) + os.path.join(args.save_dygraph_model_path, "transformer") + ) # to avoid a longer length than training, reset the size of position # encoding to max_length model_dict["encoder.pos_encoder.weight"] = position_encoding_init( - args.max_length + 1, args.d_model) + args.max_length + 1, args.d_model + ) model_dict["decoder.pos_encoder.weight"] = position_encoding_init( - args.max_length + 1, args.d_model) + args.max_length + 1, args.d_model + ) transformer.load_dict(model_dict) # set evaluate mode @@ -258,8 +365,13 @@ def predict_dygraph(args, batch_generator): step_idx = 0 speed_list = [] for input_data in test_loader(): - (src_word, src_pos, src_slf_attn_bias, trg_word, - trg_src_attn_bias) = input_data + ( + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_src_attn_bias, + ) = input_data seq_ids, seq_scores = transformer.beam_search( src_word, src_pos, @@ -269,28 +381,32 @@ def predict_dygraph(args, batch_generator): bos_id=args.bos_idx, eos_id=args.eos_idx, beam_size=args.beam_size, - max_len=args.max_out_len) + max_len=args.max_out_len, + ) seq_ids = seq_ids.numpy() seq_scores = seq_scores.numpy() if step_idx % args.print_step == 0: if step_idx == 0: logging.info( "Dygraph Predict: step_idx: %d, 1st seq_id: %d, 1st seq_score: %.2f" - % (step_idx, seq_ids[0][0][0], seq_scores[0][0])) + % (step_idx, seq_ids[0][0][0], seq_scores[0][0]) + ) avg_batch_time = time.time() else: speed = args.print_step / (time.time() - avg_batch_time) speed_list.append(speed) logging.info( "Dygraph Predict: step_idx: %d, 1st seq_id: %d, 1st seq_score: %.2f, speed: %.3f steps/s" - % (step_idx, seq_ids[0][0][0], seq_scores[0][0], speed)) + % (step_idx, seq_ids[0][0][0], seq_scores[0][0], speed) + ) avg_batch_time = time.time() step_idx += 1 if step_idx == STEP_NUM: break - logging.info("Dygraph Predict: avg_speed: %.4f steps/s" % - (np.mean(speed_list))) + logging.info( + "Dygraph Predict: avg_speed: %.4f steps/s" % (np.mean(speed_list)) + ) return seq_ids, seq_scores @@ -301,33 +417,53 @@ def predict_static(args, batch_generator): paddle.framework.random._manual_program_seed(SEED) # define input and reader - input_field_names = util.encoder_data_input_fields + util.fast_decoder_data_input_fields + input_field_names = ( + util.encoder_data_input_fields + util.fast_decoder_data_input_fields + ) input_descs = util.get_input_descs(args, 'test') - input_slots = [{ - "name": name, - "shape": input_descs[name][0], - "dtype": input_descs[name][1] - } for name in input_field_names] + input_slots = [ + { + "name": name, + "shape": input_descs[name][0], + "dtype": input_descs[name][1], + } + for name in input_field_names + ] input_field = util.InputField(input_slots) feed_list = input_field.feed_list - loader = fluid.io.DataLoader.from_generator(feed_list=feed_list, - capacity=10) + loader = fluid.io.DataLoader.from_generator( + feed_list=feed_list, capacity=10 + ) # define model transformer = Transformer( - args.src_vocab_size, args.trg_vocab_size, args.max_length + 1, - args.n_layer, args.n_head, args.d_key, args.d_value, args.d_model, - args.d_inner_hid, args.prepostprocess_dropout, - args.attention_dropout, args.relu_dropout, args.preprocess_cmd, - args.postprocess_cmd, args.weight_sharing, args.bos_idx, - args.eos_idx) - - out_ids, out_scores = transformer.beam_search(*feed_list, - bos_id=args.bos_idx, - eos_id=args.eos_idx, - beam_size=args.beam_size, - max_len=args.max_out_len) + args.src_vocab_size, + args.trg_vocab_size, + args.max_length + 1, + args.n_layer, + args.n_head, + args.d_key, + args.d_value, + args.d_model, + args.d_inner_hid, + args.prepostprocess_dropout, + args.attention_dropout, + args.relu_dropout, + args.preprocess_cmd, + args.postprocess_cmd, + args.weight_sharing, + args.bos_idx, + args.eos_idx, + ) + + out_ids, out_scores = transformer.beam_search( + *feed_list, + bos_id=args.bos_idx, + eos_id=args.eos_idx, + beam_size=args.beam_size, + max_len=args.max_out_len + ) # This is used here to set dropout to the test mode. test_prog = test_prog.clone(for_test=True) @@ -335,8 +471,9 @@ def predict_static(args, batch_generator): # define the executor and program for training exe = fluid.Executor(place) - util.load(test_prog, os.path.join(args.save_static_model_path, - "transformer"), exe) + util.load( + test_prog, os.path.join(args.save_static_model_path, "transformer"), exe + ) loader.set_batch_generator(batch_generator, places=place) @@ -347,32 +484,35 @@ def predict_static(args, batch_generator): test_prog, feed=feed_dict, fetch_list=[out_ids.name, out_scores.name], - return_numpy=True) + return_numpy=True, + ) if step_idx % args.print_step == 0: if step_idx == 0: logging.info( "Static Predict: step_idx: %d, 1st seq_id: %d, 1st seq_score: %.2f," - % (step_idx, seq_ids[0][0][0], seq_scores[0][0])) + % (step_idx, seq_ids[0][0][0], seq_scores[0][0]) + ) avg_batch_time = time.time() else: speed = args.print_step / (time.time() - avg_batch_time) speed_list.append(speed) logging.info( "Static Predict: step_idx: %d, 1st seq_id: %d, 1st seq_score: %.2f, speed: %.3f steps/s" - % (step_idx, seq_ids[0][0][0], seq_scores[0][0], speed)) + % (step_idx, seq_ids[0][0][0], seq_scores[0][0], speed) + ) avg_batch_time = time.time() step_idx += 1 if step_idx == STEP_NUM: break - logging.info("Static Predict: avg_speed: %.4f steps/s" % - (np.mean(speed_list))) + logging.info( + "Static Predict: avg_speed: %.4f steps/s" % (np.mean(speed_list)) + ) return seq_ids, seq_scores class TestTransformer(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -382,11 +522,14 @@ class TestTransformer(unittest.TestCase): def prepare(self, mode='train'): args = util.ModelHyperParams() args.save_dygraph_model_path = os.path.join( - self.temp_dir.name, args.save_dygraph_model_path) - args.save_static_model_path = os.path.join(self.temp_dir.name, - args.save_static_model_path) - args.inference_model_dir = os.path.join(self.temp_dir.name, - args.inference_model_dir) + self.temp_dir.name, args.save_dygraph_model_path + ) + args.save_static_model_path = os.path.join( + self.temp_dir.name, args.save_static_model_path + ) + args.inference_model_dir = os.path.join( + self.temp_dir.name, args.inference_model_dir + ) args.output_file = os.path.join(self.temp_dir.name, args.output_file) batch_generator = util.get_feed_data_reader(args, mode) return args, batch_generator @@ -395,9 +538,9 @@ class TestTransformer(unittest.TestCase): args, batch_generator = self.prepare(mode='train') static_avg_loss = train_static(args, batch_generator) dygraph_avg_loss = train_dygraph(args, batch_generator) - np.testing.assert_allclose(static_avg_loss, - dygraph_avg_loss, - rtol=1e-05) + np.testing.assert_allclose( + static_avg_loss, dygraph_avg_loss, rtol=1e-05 + ) def _test_predict(self): args, batch_generator = self.prepare(mode='test') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tsm.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tsm.py index 7561e7aa137..36232b627fd 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tsm.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tsm.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse import numpy as np @@ -30,43 +30,52 @@ np.random.seed(0) def parse_args(): parser = argparse.ArgumentParser("Paddle Video train script") - parser.add_argument('--config', - type=str, - default='tsm.yaml', - help='path to config file of model') - parser.add_argument('--use_gpu', - type=bool, - default=fluid.is_compiled_with_cuda(), - help='default use gpu.') + parser.add_argument( + '--config', + type=str, + default='tsm.yaml', + help='path to config file of model', + ) + parser.add_argument( + '--use_gpu', + type=bool, + default=fluid.is_compiled_with_cuda(), + help='default use gpu.', + ) args = parser.parse_args(['--config', 'tsm.yaml']) return args class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=None, - act=None, - param_attr=fluid.param_attr.ParamAttr(), - bias_attr=False) - - self._batch_norm = BatchNorm(num_filters, - act=act, - param_attr=fluid.param_attr.ParamAttr(), - bias_attr=fluid.param_attr.ParamAttr()) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=None, + act=None, + param_attr=fluid.param_attr.ParamAttr(), + bias_attr=False, + ) + + self._batch_norm = BatchNorm( + num_filters, + act=act, + param_attr=fluid.param_attr.ParamAttr(), + bias_attr=fluid.param_attr.ParamAttr(), + ) def forward(self, inputs): y = self._conv(inputs) @@ -76,34 +85,38 @@ class ConvBNLayer(fluid.dygraph.Layer): class BottleneckBlock(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - shortcut=True, - seg_num=8): + def __init__( + self, num_channels, num_filters, stride, shortcut=True, seg_num=8 + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu') - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - act='relu') - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act=None) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act=None, + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut self.seg_num = seg_num self._num_channels_out = int(num_filters * 4) @@ -122,7 +135,6 @@ class BottleneckBlock(fluid.dygraph.Layer): class TSM_ResNet(fluid.dygraph.Layer): - def __init__(self, name_scope, config, mode): super(TSM_ResNet, self).__init__(name_scope) @@ -130,8 +142,9 @@ class TSM_ResNet(fluid.dygraph.Layer): self.seg_num = config.MODEL.seg_num self.class_dim = config.MODEL.num_classes self.reshape_list = [ - config.MODEL.seglen * 3, config[mode.upper()]['target_size'], - config[mode.upper()]['target_size'] + config.MODEL.seglen * 3, + config[mode.upper()]['target_size'], + config[mode.upper()]['target_size'], ] if self.layers == 50: @@ -140,15 +153,12 @@ class TSM_ResNet(fluid.dygraph.Layer): raise NotImplementedError num_filters = [64, 128, 256, 512] - self.conv = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool2d_max = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv = ConvBNLayer( + num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu' + ) + self.pool2d_max = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] num_channels = 64 @@ -158,19 +168,23 @@ class TSM_ResNet(fluid.dygraph.Layer): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - shortcut=shortcut, - seg_num=self.seg_num)) + BottleneckBlock( + num_channels=num_channels, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + shortcut=shortcut, + seg_num=self.seg_num, + ), + ) num_channels = int(bottleneck_block._num_channels_out) self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) import math + stdv = 1.0 / math.sqrt(2048 * 1.0) self.out = Linear( @@ -178,9 +192,12 @@ class TSM_ResNet(fluid.dygraph.Layer): self.class_dim, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), bias_attr=fluid.param_attr.ParamAttr( - learning_rate=2.0, regularizer=fluid.regularizer.L2Decay(0.))) + learning_rate=2.0, regularizer=fluid.regularizer.L2Decay(0.0) + ), + ) @declarative def forward(self, inputs): @@ -199,7 +216,6 @@ class TSM_ResNet(fluid.dygraph.Layer): class FakeDataReader(object): - def __init__(self, mode, cfg): self.format = cfg.MODEL.format self.num_classes = cfg.MODEL.num_classes @@ -207,13 +223,18 @@ class FakeDataReader(object): self.seglen = cfg.MODEL.seglen self.target_size = cfg[mode.upper()]['target_size'] - self.img_mean = np.array(cfg.MODEL.image_mean).reshape( - [3, 1, 1]).astype(np.float32) - self.img_std = np.array(cfg.MODEL.image_std).reshape([3, 1, 1]).astype( - np.float32) - - self.batch_size = 1 if sys.platform == 'darwin' or os.name == 'nt' else cfg[ - mode.upper()]['batch_size'] + self.img_mean = ( + np.array(cfg.MODEL.image_mean).reshape([3, 1, 1]).astype(np.float32) + ) + self.img_std = ( + np.array(cfg.MODEL.image_std).reshape([3, 1, 1]).astype(np.float32) + ) + + self.batch_size = ( + 1 + if sys.platform == 'darwin' or os.name == 'nt' + else cfg[mode.upper()]['batch_size'] + ) self.generator_out = [] self.total_iter = 3 for i in range(self.total_iter): @@ -222,15 +243,20 @@ class FakeDataReader(object): label = np.int64(random.randint(0, self.num_classes - 1)) random_mean = self.img_mean[0][0][0] random_std = self.img_std[0][0][0] - imgs = np.random.normal(random_mean, random_std, [ - self.seg_num, self.seglen * 3, self.target_size, - self.target_size - ]).astype(np.float32) + imgs = np.random.normal( + random_mean, + random_std, + [ + self.seg_num, + self.seglen * 3, + self.target_size, + self.target_size, + ], + ).astype(np.float32) batch_out.append((imgs, label)) self.generator_out.append(batch_out) def create_reader(self): - def batch_reader(): for i in range(self.total_iter): yield self.generator_out[i] @@ -240,7 +266,9 @@ class FakeDataReader(object): def create_optimizer(cfg, params): total_videos = cfg.total_videos - batch_size = 1 if sys.platform == 'darwin' or os.name == 'nt' else cfg.batch_size + batch_size = ( + 1 if sys.platform == 'darwin' or os.name == 'nt' else cfg.batch_size + ) step = int(total_videos / batch_size + 1) bd = [e * step for e in cfg.decay_epochs] base_lr = cfg.learning_rate @@ -253,7 +281,8 @@ def create_optimizer(cfg, params): learning_rate=fluid.layers.piecewise_decay(boundaries=bd, values=lr), momentum=momentum, regularization=fluid.regularizer.L2Decay(l2_weight_decay), - parameter_list=params) + parameter_list=params, + ) return optimizer @@ -277,8 +306,9 @@ def train(args, fake_data_reader, to_static): video_model = TSM_ResNet("TSM", train_config, 'Train') - optimizer = create_optimizer(train_config.TRAIN, - video_model.parameters()) + optimizer = create_optimizer( + train_config.TRAIN, video_model.parameters() + ) train_reader = fake_data_reader.create_reader() @@ -297,16 +327,16 @@ def train(args, fake_data_reader, to_static): labels = to_variable(y_data) labels.stop_gradient = True outputs = video_model(imgs) - loss = fluid.layers.cross_entropy(input=outputs, - label=labels, - ignore_index=-1) + loss = fluid.layers.cross_entropy( + input=outputs, label=labels, ignore_index=-1 + ) avg_loss = paddle.mean(loss) - acc_top1 = fluid.layers.accuracy(input=outputs, - label=labels, - k=1) - acc_top5 = fluid.layers.accuracy(input=outputs, - label=labels, - k=5) + acc_top1 = fluid.layers.accuracy( + input=outputs, label=labels, k=1 + ) + acc_top5 = fluid.layers.accuracy( + input=outputs, label=labels, k=5 + ) avg_loss.backward() optimizer.minimize(avg_loss) @@ -317,26 +347,35 @@ def train(args, fake_data_reader, to_static): total_acc5 += acc_top5.numpy()[0] total_sample += 1 - print('TRAIN Epoch {}, iter {}, loss = {}, acc1 {}, acc5 {}'. - format(epoch, batch_id, - avg_loss.numpy()[0], - acc_top1.numpy()[0], - acc_top5.numpy()[0])) - ret.extend([ - avg_loss.numpy()[0], - acc_top1.numpy()[0], - acc_top5.numpy()[0] - ]) + print( + 'TRAIN Epoch {}, iter {}, loss = {}, acc1 {}, acc5 {}'.format( + epoch, + batch_id, + avg_loss.numpy()[0], + acc_top1.numpy()[0], + acc_top5.numpy()[0], + ) + ) + ret.extend( + [ + avg_loss.numpy()[0], + acc_top1.numpy()[0], + acc_top5.numpy()[0], + ] + ) print( - 'TRAIN End, Epoch {}, avg_loss= {}, avg_acc1= {}, avg_acc5= {}'. - format(epoch, total_loss / total_sample, - total_acc1 / total_sample, total_acc5 / total_sample)) + 'TRAIN End, Epoch {}, avg_loss= {}, avg_acc1= {}, avg_acc5= {}'.format( + epoch, + total_loss / total_sample, + total_acc1 / total_sample, + total_acc5 / total_sample, + ) + ) return ret class TestTsm(unittest.TestCase): - def test_dygraph_static_same_loss(self): if fluid.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typehint.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typehint.py index b8addd53d5b..206ab40d0bc 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typehint.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typehint.py @@ -32,10 +32,12 @@ def function(x: A) -> A: class TestTransformWhileLoop(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.x = np.zeros(shape=(1), dtype=np.int32) self._init_dyfunc() @@ -69,7 +71,6 @@ class TestTransformWhileLoop(unittest.TestCase): class TestTypeHint(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = function diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py index b55fb242199..e4dea9d6abf 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py @@ -20,7 +20,6 @@ from typing import Dict, List, Tuple class BaseLayer(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(BaseLayer, self).__init__() self._linear = paddle.nn.Linear(in_size, out_size) @@ -32,7 +31,6 @@ class BaseLayer(paddle.nn.Layer): class LinearNetWithTuple(BaseLayer): - def __init__(self, in_size, out_size): super(LinearNetWithTuple, self).__init__(in_size, out_size) @@ -42,7 +40,6 @@ class LinearNetWithTuple(BaseLayer): class LinearNetWithTuple2(BaseLayer): - def __init__(self, in_size, out_size): super(LinearNetWithTuple2, self).__init__(in_size, out_size) @@ -52,7 +49,6 @@ class LinearNetWithTuple2(BaseLayer): class LinearNetWithList(BaseLayer): - def __init__(self, in_size, out_size): super(LinearNetWithList, self).__init__(in_size, out_size) @@ -62,7 +58,6 @@ class LinearNetWithList(BaseLayer): class LinearNetWithDict(BaseLayer): - def __init__(self, in_size, out_size): super(LinearNetWithDict, self).__init__(in_size, out_size) @@ -72,7 +67,6 @@ class LinearNetWithDict(BaseLayer): class TestTyping(unittest.TestCase): - def setUp(self): self.in_num = 16 self.out_num = 16 @@ -105,7 +99,6 @@ class TestTyping(unittest.TestCase): class TestTypingTuple(TestTyping): - def build_net(self): return LinearNetWithTuple2(self.in_num, self.out_num) @@ -116,7 +109,6 @@ class TestTypingTuple(TestTyping): class TestTypingList(TestTyping): - def build_net(self): return LinearNetWithList(self.in_num, self.out_num) @@ -126,7 +118,6 @@ class TestTypingList(TestTyping): class TestTypingDict(TestTyping): - def build_net(self): return LinearNetWithDict(self.in_num, self.out_num) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py index 30300ddf2e4..4edb8150445 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py @@ -20,7 +20,6 @@ from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_func class TestIndexInList(unittest.TestCase): - def test_index_in_list(self): list_to_test = [1, 2, 3, 4, 5] self.assertEqual(index_in_list(list_to_test, 4), 3) @@ -37,8 +36,7 @@ def dyfunc_assign(input): [x, y] = m, n = z -class StaticCode(): - +class StaticCode: def dyfunc_assign(input): b = 1 a = b @@ -53,7 +51,6 @@ class StaticCode(): class TestIsPaddle(unittest.TestCase): - def fake_module(self): return types.ModuleType('paddlenlp') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py index 90177891693..353a5e8b797 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py @@ -15,29 +15,33 @@ import unittest from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_fill_constant_node +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_fill_constant_node, +) class TestVariableTransFunc(unittest.TestCase): - def test_create_fill_constant_node(self): node = create_fill_constant_node("a", 1.0) source = "a = paddle.full(shape=[1], dtype='float64', fill_value=1.0, name='a')" self.assertEqual( ast_to_source_code(node).replace('\n', '').replace(' ', ''), - source.replace(' ', '')) + source.replace(' ', ''), + ) node = create_fill_constant_node("b", True) source = "b = paddle.full(shape=[1], dtype='bool', fill_value=True, name='b')" self.assertEqual( ast_to_source_code(node).replace('\n', '').replace(' ', ''), - source.replace(' ', '')) + source.replace(' ', ''), + ) node = create_fill_constant_node("c", 4293) source = "c = paddle.full(shape=[1], dtype='int64', fill_value=4293, name='c')" self.assertEqual( ast_to_source_code(node).replace('\n', '').replace(' ', ''), - source.replace(' ', '')) + source.replace(' ', ''), + ) self.assertIsNone(create_fill_constant_node("e", None)) self.assertIsNone(create_fill_constant_node("e", [])) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_warning.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_warning.py index 846cb9e7aa1..a232c70fa71 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_warning.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_warning.py @@ -37,7 +37,6 @@ def false_fn(): class TestReturnNoneInIfelse(unittest.TestCase): - def test_dy2static_warning(self): paddle.disable_static() with warnings.catch_warnings(record=True) as w: @@ -46,9 +45,10 @@ class TestReturnNoneInIfelse(unittest.TestCase): flag = False for warn in w: if ( - issubclass(warn.category, UserWarning) + issubclass(warn.category, UserWarning) ) and "Set var to 'None' in ifelse block might lead to error." in str( - warn.message): + warn.message + ): flag = True break self.assertTrue(flag) @@ -63,9 +63,10 @@ class TestReturnNoneInIfelse(unittest.TestCase): flag = False for warn in w: if ( - issubclass(warn.category, UserWarning) + issubclass(warn.category, UserWarning) ) and "Set var to 'None' in ifelse block might lead to error." in str( - warn.message): + warn.message + ): flag = True break self.assertTrue(flag) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_word2vec.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_word2vec.py index fbabe1bc36b..a597e335187 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_word2vec.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_word2vec.py @@ -55,15 +55,15 @@ def build_dict(corpus, min_freq=3): word_freq_dict[word] = 0 word_freq_dict[word] += 1 - word_freq_dict = sorted(word_freq_dict.items(), - key=lambda x: x[1], - reverse=True) + word_freq_dict = sorted( + word_freq_dict.items(), key=lambda x: x[1], reverse=True + ) word2id_dict = dict() word2id_freq = dict() id2word_dict = dict() - word2id_freq[0] = 1. + word2id_freq[0] = 1.0 word2id_dict['[oov]'] = 0 id2word_dict[0] = '[oov]' @@ -85,8 +85,10 @@ word2id_freq, word2id_dict, id2word_dict = build_dict(corpus) vocab_size = len(word2id_freq) print("there are totoally %d different words in the corpus" % vocab_size) for _, (word, word_id) in zip(range(50), word2id_dict.items()): - print("word %s, its id %d, its word freq %d" % - (word, word_id, word2id_freq[word_id])) + print( + "word %s, its id %d, its word freq %d" + % (word, word_id, word2id_freq[word_id]) + ) def convert_corpus_to_id(corpus, word2id_dict): @@ -94,7 +96,9 @@ def convert_corpus_to_id(corpus, word2id_dict): for line in corpus: new_line = [ word2id_dict[word] - if word in word2id_dict else word2id_dict['[oov]'] for word in line + if word in word2id_dict + else word2id_dict['[oov]'] + for word in line ] new_corpus.append(new_line) return new_corpus @@ -104,10 +108,10 @@ corpus = convert_corpus_to_id(corpus, word2id_dict) def subsampling(corpus, word2id_freq): - def keep(word_id): return random.uniform(0, 1) < math.sqrt( - 1e-4 / word2id_freq[word_id] * len(corpus)) + 1e-4 / word2id_freq[word_id] * len(corpus) + ) new_corpus = [] for line in corpus: @@ -119,11 +123,13 @@ def subsampling(corpus, word2id_freq): corpus = subsampling(corpus, word2id_freq) -def build_data(corpus, - word2id_dict, - word2id_freq, - max_window_size=3, - negative_sample_num=10): +def build_data( + corpus, + word2id_dict, + word2id_freq, + max_window_size=3, + negative_sample_num=10, +): dataset = [] @@ -132,13 +138,15 @@ def build_data(corpus, window_size = random.randint(1, max_window_size) center_word = line[center_word_idx] - positive_word_range = (max(0, center_word_idx - window_size), - min( - len(line) - 1, - center_word_idx + window_size)) + positive_word_range = ( + max(0, center_word_idx - window_size), + min(len(line) - 1, center_word_idx + window_size), + ) positive_word_candidates = [ - line[idx] for idx in range(positive_word_range[0], - positive_word_range[1] + 1) + line[idx] + for idx in range( + positive_word_range[0], positive_word_range[1] + 1 + ) if idx != center_word_idx and line[idx] != line[center_word_idx] ] @@ -161,8 +169,10 @@ def build_data(corpus, dataset = build_data(corpus, word2id_dict, word2id_freq) for _, (center_word, target_word, label) in zip(range(50), dataset): - print("center_word %s, target %s, label %d" % - (id2word_dict[center_word], id2word_dict[target_word], label)) + print( + "center_word %s, target %s, label %d" + % (id2word_dict[center_word], id2word_dict[target_word], label) + ) def build_batch(dataset, batch_size, epoch_num): @@ -185,9 +195,14 @@ def build_batch(dataset, batch_size, epoch_num): if len(center_word_batch) == batch_size: yield np.array(center_word_batch).astype("int64"), np.array( - target_word_batch).astype("int64"), np.array( - label_batch).astype("float32"), np.array( - eval_word_batch).astype("int64") + target_word_batch + ).astype("int64"), np.array(label_batch).astype( + "float32" + ), np.array( + eval_word_batch + ).astype( + "int64" + ) center_word_batch = [] target_word_batch = [] label_batch = [] @@ -195,12 +210,15 @@ def build_batch(dataset, batch_size, epoch_num): if len(center_word_batch) > 0: yield np.array(center_word_batch).astype("int64"), np.array( - target_word_batch).astype("int64"), np.array(label_batch).astype( - "float32"), np.array(eval_word_batch).astype("int64") + target_word_batch + ).astype("int64"), np.array(label_batch).astype("float32"), np.array( + eval_word_batch + ).astype( + "int64" + ) class SkipGram(fluid.dygraph.Layer): - def __init__(self, name_scope, vocab_size, embedding_size, init_scale=0.1): super(SkipGram, self).__init__(name_scope) self.vocab_size = vocab_size @@ -213,7 +231,10 @@ class SkipGram(fluid.dygraph.Layer): name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-0.5 / self.embedding_size, - high=0.5 / self.embedding_size))) + high=0.5 / self.embedding_size, + ), + ), + ) self.embedding_out = Embedding( size=[self.vocab_size, self.embedding_size], @@ -222,7 +243,10 @@ class SkipGram(fluid.dygraph.Layer): name='embedding_out_para', initializer=fluid.initializer.UniformInitializer( low=-0.5 / self.embedding_size, - high=0.5 / self.embedding_size))) + high=0.5 / self.embedding_size, + ), + ), + ) @declarative def forward(self, center_words, target_words, label): @@ -231,8 +255,9 @@ class SkipGram(fluid.dygraph.Layer): # center_words_emb = [batch_size, embedding_size] # target_words_emb = [batch_size, embedding_size] - word_sim = fluid.layers.elementwise_mul(center_words_emb, - target_words_emb) + word_sim = fluid.layers.elementwise_mul( + center_words_emb, target_words_emb + ) word_sim = fluid.layers.reduce_sum(word_sim, dim=-1) pred = fluid.layers.sigmoid(word_sim) @@ -257,27 +282,34 @@ def train(to_static): random.seed(0) np.random.seed(0) - place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = 1000 fluid.default_main_program().random_seed = 1000 - skip_gram_model = SkipGram("skip_gram_model", vocab_size, - embedding_size) + skip_gram_model = SkipGram( + "skip_gram_model", vocab_size, embedding_size + ) adam = fluid.optimizer.AdamOptimizer( learning_rate=learning_rate, - parameter_list=skip_gram_model.parameters()) + parameter_list=skip_gram_model.parameters(), + ) step = 0 ret = [] for center_words, target_words, label, eval_words in build_batch( - dataset, batch_size, epoch_num): + dataset, batch_size, epoch_num + ): center_words_var = fluid.dygraph.to_variable(center_words) target_words_var = fluid.dygraph.to_variable(target_words) label_var = fluid.dygraph.to_variable(label) - pred, loss = skip_gram_model(center_words_var, target_words_var, - label_var) + pred, loss = skip_gram_model( + center_words_var, target_words_var, label_var + ) loss.backward() adam.minimize(loss) @@ -291,7 +323,6 @@ def train(to_static): class TestWord2Vec(unittest.TestCase): - def test_dygraph_static_same_loss(self): dygraph_loss = train(to_static=False) static_loss = train(to_static=True) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_yolov3.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_yolov3.py index 0fb791a3a28..eecd6806fa5 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_yolov3.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_yolov3.py @@ -47,27 +47,26 @@ class SmoothedValue(object): class FakeDataReader(object): - def __init__(self): self.generator_out = [] self.total_iter = cfg.max_iter for i in range(self.total_iter): batch_out = [] for j in range(cfg.batch_size): - img = np.random.normal(0.485, 0.229, - [3, cfg.input_size, cfg.input_size]) + img = np.random.normal( + 0.485, 0.229, [3, cfg.input_size, cfg.input_size] + ) point1 = cfg.input_size / 4 point2 = cfg.input_size / 2 gt_boxes = np.array([[point1, point1, point2, point2]]) - gt_labels = np.random.randint(low=0, - high=cfg.class_num, - size=[1]) + gt_labels = np.random.randint( + low=0, high=cfg.class_num, size=[1] + ) gt_scores = np.zeros([1]) batch_out.append([img, gt_boxes, gt_labels, gt_scores]) self.generator_out.append(batch_out) def reader(self): - def generator(): for i in range(self.total_iter): yield self.generator_out[i] @@ -97,9 +96,9 @@ def train(to_static): learning_rate = cfg.learning_rate values = [learning_rate * (gamma**i) for i in range(step_num + 1)] - lr = fluid.dygraph.PiecewiseDecay(boundaries=boundaries, - values=values, - begin=0) + lr = fluid.dygraph.PiecewiseDecay( + boundaries=boundaries, values=values, begin=0 + ) lr = fluid.layers.linear_lr_warmup( learning_rate=lr, @@ -112,7 +111,8 @@ def train(to_static): learning_rate=lr, regularization=fluid.regularizer.L2Decay(cfg.weight_decay), momentum=cfg.momentum, - parameter_list=model.parameters()) + parameter_list=model.parameters(), + ) start_time = time.time() snapshot_loss = 0 @@ -150,9 +150,13 @@ def train(to_static): snapshot_time += start_time - prev_start_time total_sample += 1 - print("Iter {:d}, loss {:.6f}, time {:.5f}".format( - iter_id, smoothed_loss.get_mean_value(), - start_time - prev_start_time)) + print( + "Iter {:d}, loss {:.6f}, time {:.5f}".format( + iter_id, + smoothed_loss.get_mean_value(), + start_time - prev_start_time, + ) + ) ret.append(smoothed_loss.get_mean_value()) loss.backward() @@ -164,14 +168,12 @@ def train(to_static): class TestYolov3(unittest.TestCase): - def test_dygraph_static_same_loss(self): dygraph_loss = train(to_static=False) static_loss = train(to_static=True) - np.testing.assert_allclose(dygraph_loss, - static_loss, - rtol=0.001, - atol=1e-05) + np.testing.assert_allclose( + dygraph_loss, static_loss, rtol=0.001, atol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_dygraph_model.py index 4c33f88469c..27139811c0a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_dygraph_model.py @@ -16,7 +16,13 @@ import numpy as np import paddle.fluid as fluid import paddle.fluid.layers as layers -from paddle.fluid.dygraph import Embedding, Layer, LayerNorm, Linear, to_variable +from paddle.fluid.dygraph import ( + Embedding, + Layer, + LayerNorm, + Linear, + to_variable, +) from paddle.fluid.dygraph.jit import dygraph_to_static_func from paddle.fluid.layers.utils import map_structure import paddle @@ -29,12 +35,15 @@ def position_encoding_init(n_position, d_pos_vec): channels = d_pos_vec position = np.arange(n_position) num_timescales = channels // 2 - log_timescale_increment = (np.log(float(1e4) / float(1)) / - (num_timescales - 1)) - inv_timescales = np.exp( - np.arange(num_timescales)) * -log_timescale_increment + log_timescale_increment = np.log(float(1e4) / float(1)) / ( + num_timescales - 1 + ) + inv_timescales = ( + np.exp(np.arange(num_timescales)) * -log_timescale_increment + ) scaled_time = np.expand_dims(position, 1) * np.expand_dims( - inv_timescales, 0) + inv_timescales, 0 + ) signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1) signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant') position_enc = signal @@ -42,7 +51,6 @@ def position_encoding_init(n_position, d_pos_vec): class PrePostProcessLayer(Layer): - def __init__(self, process_cmd, d_model, dropout_rate): super(PrePostProcessLayer, self).__init__() self.process_cmd = process_cmd @@ -53,18 +61,24 @@ class PrePostProcessLayer(Layer): elif cmd == "n": # add layer normalization self.functors.append( self.add_sublayer( - "layer_norm_%d" % - len([layer for layer in self.children()]), + "layer_norm_%d" + % len([layer for layer in self.children()]), LayerNorm( normalized_shape=d_model, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.)), + initializer=fluid.initializer.Constant(1.0) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.))))) + initializer=fluid.initializer.Constant(0.0) + ), + ), + ) + ) elif cmd == "d": # add dropout if dropout_rate: self.functors.append( - lambda x: layers.dropout(x, dropout_prob=dropout_rate)) + lambda x: layers.dropout(x, dropout_prob=dropout_rate) + ) def forward(self, x, residual=None): for i, cmd in enumerate(self.process_cmd): @@ -76,14 +90,15 @@ class PrePostProcessLayer(Layer): class MultiHeadAttention(Layer): - - def __init__(self, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - param_initializer=None): + def __init__( + self, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, + param_initializer=None, + ): super(MultiHeadAttention, self).__init__() self.n_head = n_head self.d_key = d_key @@ -94,22 +109,26 @@ class MultiHeadAttention(Layer): input_dim=d_model, output_dim=d_key * n_head, bias_attr=False, - param_attr=fluid.ParamAttr(initializer=param_initializer)) + param_attr=fluid.ParamAttr(initializer=param_initializer), + ) self.k_fc = Linear( input_dim=d_model, output_dim=d_key * n_head, bias_attr=False, - param_attr=fluid.ParamAttr(initializer=param_initializer)) + param_attr=fluid.ParamAttr(initializer=param_initializer), + ) self.v_fc = Linear( input_dim=d_model, output_dim=d_value * n_head, bias_attr=False, - param_attr=fluid.ParamAttr(initializer=param_initializer)) + param_attr=fluid.ParamAttr(initializer=param_initializer), + ) self.proj_fc = Linear( input_dim=d_value * n_head, output_dim=d_model, bias_attr=False, - param_attr=fluid.ParamAttr(initializer=param_initializer)) + param_attr=fluid.ParamAttr(initializer=param_initializer), + ) def forward(self, queries, keys, values, attn_bias, cache=None): # compute q ,k ,v @@ -132,10 +151,9 @@ class MultiHeadAttention(Layer): v = layers.concat([cache_v, v], axis=2) cache["k"], cache["v"] = k, v # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.d_model**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.d_model**-0.5 + ) if attn_bias is not None: product += attn_bias weights = layers.softmax(product) @@ -149,7 +167,6 @@ class MultiHeadAttention(Layer): class FFN(Layer): - def __init__(self, d_inner_hid, d_model, dropout_rate): super(FFN, self).__init__() self.dropout_rate = dropout_rate @@ -165,37 +182,44 @@ class FFN(Layer): class EncoderLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderLayer, self).__init__() - self.preprocesser1 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) - self.self_attn = MultiHeadAttention(d_key, d_value, d_model, n_head, - attention_dropout) - self.postprocesser1 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) + self.preprocesser1 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) + self.self_attn = MultiHeadAttention( + d_key, d_value, d_model, n_head, attention_dropout + ) + self.postprocesser1 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) - self.preprocesser2 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) + self.preprocesser2 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) self.ffn = FFN(d_inner_hid, d_model, relu_dropout) - self.postprocesser2 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) + self.postprocesser2 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) def forward(self, enc_input, attn_bias): - attn_output = self.self_attn(self.preprocesser1(enc_input), None, None, - attn_bias) + attn_output = self.self_attn( + self.preprocesser1(enc_input), None, None, attn_bias + ) attn_output = self.postprocesser1(attn_output, enc_input) ffn_output = self.ffn(self.preprocesser2(attn_output)) ffn_output = self.postprocesser2(ffn_output, attn_output) @@ -203,19 +227,20 @@ class EncoderLayer(Layer): class Encoder(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(Encoder, self).__init__() @@ -224,12 +249,23 @@ class Encoder(Layer): self.encoder_layers.append( self.add_sublayer( "layer_%d" % i, - EncoderLayer(n_head, d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, - relu_dropout, preprocess_cmd, - postprocess_cmd))) - self.processer = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) + EncoderLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ), + ) + ) + self.processer = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) def forward(self, enc_input, attn_bias): for encoder_layer in self.encoder_layers: @@ -240,14 +276,15 @@ class Encoder(Layer): class Embedder(Layer): - def __init__(self, vocab_size, emb_dim, bos_idx=0): super(Embedder, self).__init__() self.word_embedder = Embedding( size=[vocab_size, emb_dim], padding_idx=bos_idx, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Normal(0., emb_dim**-0.5))) + initializer=fluid.initializer.Normal(0.0, emb_dim**-0.5) + ), + ) def forward(self, word): word_emb = self.word_embedder(word) @@ -255,11 +292,23 @@ class Embedder(Layer): class WrapEncoder(Layer): - - def __init__(self, src_vocab_size, max_length, n_layer, n_head, d_key, - d_value, d_model, d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, preprocess_cmd, - postprocess_cmd, word_embedder): + def __init__( + self, + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + word_embedder, + ): super(WrapEncoder, self).__init__() self.emb_dropout = prepostprocess_dropout self.emb_dim = d_model @@ -268,12 +317,24 @@ class WrapEncoder(Layer): size=[max_length, self.emb_dim], param_attr=fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - position_encoding_init(max_length, self.emb_dim)), - trainable=False)) - self.encoder = Encoder(n_layer, n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, preprocess_cmd, - postprocess_cmd) + position_encoding_init(max_length, self.emb_dim) + ), + trainable=False, + ), + ) + self.encoder = Encoder( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ) def forward(self, src_word, src_pos, src_slf_attn_bias): word_emb = self.word_embedder(src_word) @@ -281,71 +342,96 @@ class WrapEncoder(Layer): pos_enc = self.pos_encoder(src_pos) pos_enc.stop_gradient = True emb = word_emb + pos_enc - enc_input = layers.dropout( - emb, - dropout_prob=self.emb_dropout, - ) if self.emb_dropout else emb + enc_input = ( + layers.dropout( + emb, + dropout_prob=self.emb_dropout, + ) + if self.emb_dropout + else emb + ) enc_output = self.encoder(enc_input, src_slf_attn_bias) return enc_output class DecoderLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(DecoderLayer, self).__init__() - self.preprocesser1 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) - self.self_attn = MultiHeadAttention(d_key, d_value, d_model, n_head, - attention_dropout) - self.postprocesser1 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) - self.preprocesser2 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) - self.cross_attn = MultiHeadAttention(d_key, d_value, d_model, n_head, - attention_dropout) - self.postprocesser2 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) - self.preprocesser3 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) + self.preprocesser1 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) + self.self_attn = MultiHeadAttention( + d_key, d_value, d_model, n_head, attention_dropout + ) + self.postprocesser1 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) + self.preprocesser2 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) + self.cross_attn = MultiHeadAttention( + d_key, d_value, d_model, n_head, attention_dropout + ) + self.postprocesser2 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) + self.preprocesser3 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) self.ffn = FFN(d_inner_hid, d_model, relu_dropout) - self.postprocesser3 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) + self.postprocesser3 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) - def forward(self, - dec_input, - enc_output, - self_attn_bias, - cross_attn_bias, - cache=None): - self_attn_output = self.self_attn(self.preprocesser1(dec_input), None, - None, self_attn_bias, cache) + def forward( + self, dec_input, enc_output, self_attn_bias, cross_attn_bias, cache=None + ): + self_attn_output = self.self_attn( + self.preprocesser1(dec_input), None, None, self_attn_bias, cache + ) self_attn_output = self.postprocesser1(self_attn_output, dec_input) cross_attn_output = self.cross_attn( - self.preprocesser2(self_attn_output), enc_output, enc_output, - cross_attn_bias) - cross_attn_output = self.postprocesser2(cross_attn_output, - self_attn_output) + self.preprocesser2(self_attn_output), + enc_output, + enc_output, + cross_attn_bias, + ) + cross_attn_output = self.postprocesser2( + cross_attn_output, self_attn_output + ) ffn_output = self.ffn(self.preprocesser3(cross_attn_output)) ffn_output = self.postprocesser3(ffn_output, cross_attn_output) return ffn_output class Decoder(Layer): - - def __init__(self, n_layer, n_head, d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ): super(Decoder, self).__init__() self.decoder_layers = list() @@ -353,33 +439,63 @@ class Decoder(Layer): self.decoder_layers.append( self.add_sublayer( "layer_%d" % i, - DecoderLayer(n_head, d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, - relu_dropout, preprocess_cmd, - postprocess_cmd))) - self.processer = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) - - def forward(self, + DecoderLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ), + ) + ) + self.processer = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) + + def forward( + self, + dec_input, + enc_output, + self_attn_bias, + cross_attn_bias, + caches=None, + ): + for i, decoder_layer in enumerate(self.decoder_layers): + dec_output = decoder_layer( dec_input, enc_output, self_attn_bias, cross_attn_bias, - caches=None): - for i, decoder_layer in enumerate(self.decoder_layers): - dec_output = decoder_layer(dec_input, enc_output, self_attn_bias, - cross_attn_bias, - None if caches is None else caches[i]) + None if caches is None else caches[i], + ) dec_input = dec_output return self.processer(dec_output) class WrapDecoder(Layer): - - def __init__(self, trg_vocab_size, max_length, n_layer, n_head, d_key, - d_value, d_model, d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, preprocess_cmd, - postprocess_cmd, share_input_output_embed, word_embedder): + def __init__( + self, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + share_input_output_embed, + word_embedder, + ): super(WrapDecoder, self).__init__() self.emb_dropout = prepostprocess_dropout @@ -389,40 +505,58 @@ class WrapDecoder(Layer): size=[max_length, self.emb_dim], param_attr=fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - position_encoding_init(max_length, self.emb_dim)), - trainable=False)) - self.decoder = Decoder(n_layer, n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, preprocess_cmd, - postprocess_cmd) + position_encoding_init(max_length, self.emb_dim) + ), + trainable=False, + ), + ) + self.decoder = Decoder( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ) if share_input_output_embed: - self.linear = lambda x: layers.matmul(x=x, - y=self.word_embedder. - word_embedder.weight, - transpose_y=True) + self.linear = lambda x: layers.matmul( + x=x, y=self.word_embedder.word_embedder.weight, transpose_y=True + ) else: - self.linear = Linear(input_dim=d_model, - output_dim=trg_vocab_size, - bias_attr=False) - - def forward(self, - trg_word, - trg_pos, - trg_slf_attn_bias, - trg_src_attn_bias, - enc_output, - caches=None): + self.linear = Linear( + input_dim=d_model, output_dim=trg_vocab_size, bias_attr=False + ) + + def forward( + self, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + enc_output, + caches=None, + ): word_emb = self.word_embedder(trg_word) word_emb = layers.scale(x=word_emb, scale=self.emb_dim**0.5) pos_enc = self.pos_encoder(trg_pos) pos_enc.stop_gradient = True emb = word_emb + pos_enc - dec_input = layers.dropout( - emb, - dropout_prob=self.emb_dropout, - ) if self.emb_dropout else emb - dec_output = self.decoder(dec_input, enc_output, trg_slf_attn_bias, - trg_src_attn_bias, caches) + dec_input = ( + layers.dropout( + emb, + dropout_prob=self.emb_dropout, + ) + if self.emb_dropout + else emb + ) + dec_output = self.decoder( + dec_input, enc_output, trg_slf_attn_bias, trg_src_attn_bias, caches + ) dec_output = layers.reshape( dec_output, shape=[-1, dec_output.shape[-1]], @@ -432,20 +566,21 @@ class WrapDecoder(Layer): class CrossEntropyCriterion(object): - def __init__(self, label_smooth_eps): self.label_smooth_eps = label_smooth_eps def __call__(self, predict, label, weights): if self.label_smooth_eps: - label_out = layers.label_smooth(label=layers.one_hot( - input=label, depth=predict.shape[-1]), - epsilon=self.label_smooth_eps) + label_out = layers.label_smooth( + label=layers.one_hot(input=label, depth=predict.shape[-1]), + epsilon=self.label_smooth_eps, + ) cost = layers.softmax_with_cross_entropy( logits=predict, label=label_out, - soft_label=True if self.label_smooth_eps else False) + soft_label=True if self.label_smooth_eps else False, + ) weighted_cost = cost * weights sum_cost = layers.reduce_sum(weighted_cost) token_num = layers.reduce_sum(weights) @@ -455,49 +590,72 @@ class CrossEntropyCriterion(object): class Transformer(Layer): - - def __init__(self, - src_vocab_size, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - bos_id=0, - eos_id=1): + def __init__( + self, + src_vocab_size, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + bos_id=0, + eos_id=1, + ): super(Transformer, self).__init__() - src_word_embedder = Embedder(vocab_size=src_vocab_size, - emb_dim=d_model, - bos_idx=bos_id) - self.encoder = WrapEncoder(src_vocab_size, max_length, n_layer, n_head, - d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, - relu_dropout, preprocess_cmd, - postprocess_cmd, src_word_embedder) + src_word_embedder = Embedder( + vocab_size=src_vocab_size, emb_dim=d_model, bos_idx=bos_id + ) + self.encoder = WrapEncoder( + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + src_word_embedder, + ) if weight_sharing: - assert src_vocab_size == trg_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) + assert ( + src_vocab_size == trg_vocab_size + ), "Vocabularies in source and target should be same for weight sharing." trg_word_embedder = src_word_embedder else: - trg_word_embedder = Embedder(vocab_size=trg_vocab_size, - emb_dim=d_model, - bos_idx=bos_id) - self.decoder = WrapDecoder(trg_vocab_size, max_length, n_layer, n_head, - d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, - relu_dropout, preprocess_cmd, - postprocess_cmd, weight_sharing, - trg_word_embedder) + trg_word_embedder = Embedder( + vocab_size=trg_vocab_size, emb_dim=d_model, bos_idx=bos_id + ) + self.decoder = WrapDecoder( + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + trg_word_embedder, + ) self.trg_vocab_size = trg_vocab_size self.n_layer = n_layer @@ -506,28 +664,39 @@ class Transformer(Layer): self.d_value = d_value @dygraph_to_static_func - def forward(self, src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias): + def forward( + self, + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + ): enc_output = self.encoder(src_word, src_pos, src_slf_attn_bias) - predict = self.decoder(trg_word, trg_pos, trg_slf_attn_bias, - trg_src_attn_bias, enc_output) + predict = self.decoder( + trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias, enc_output + ) return predict @dygraph_to_static_func - def beam_search(self, - src_word, - src_pos, - src_slf_attn_bias, - trg_word, - trg_src_attn_bias, - bos_id=0, - eos_id=1, - beam_size=4, - max_len=256): - + def beam_search( + self, + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_src_attn_bias, + bos_id=0, + eos_id=1, + beam_size=4, + max_len=256, + ): def expand_to_beam_size(tensor, beam_size): - tensor = layers.reshape(tensor, [tensor.shape[0], 1] + - list(tensor.shape[1:])) + tensor = layers.reshape( + tensor, [tensor.shape[0], 1] + list(tensor.shape[1:]) + ) tile_dims = [1] * len(tensor.shape) tile_dims[1] = beam_size return layers.expand(tensor, tile_dims) @@ -536,44 +705,60 @@ class Transformer(Layer): var_dim_in_state = 2 # count in beam dim tensor = layers.transpose( tensor, - list(range(var_dim_in_state, len(tensor.shape))) + - list(range(0, var_dim_in_state))) + list(range(var_dim_in_state, len(tensor.shape))) + + list(range(0, var_dim_in_state)), + ) - tensor = layers.reshape(tensor, [0] * - (len(tensor.shape) - var_dim_in_state) + - [batch_size * beam_size]) + tensor = layers.reshape( + tensor, + [0] * (len(tensor.shape) - var_dim_in_state) + + [batch_size * beam_size], + ) res = layers.transpose( tensor, list( - range((len(tensor.shape) + 1 - var_dim_in_state), - len(tensor.shape))) + - list(range(0, (len(tensor.shape) + 1 - var_dim_in_state)))) + range( + (len(tensor.shape) + 1 - var_dim_in_state), + len(tensor.shape), + ) + ) + + list(range(0, (len(tensor.shape) + 1 - var_dim_in_state))), + ) return res def split_batch_beams(tensor): var_dim_in_state = 1 tensor = layers.transpose( tensor, - list(range(var_dim_in_state, len(tensor.shape))) + - list(range(0, var_dim_in_state))) - tensor = layers.reshape(tensor, [0] * - (len(tensor.shape) - var_dim_in_state) + - [batch_size, beam_size]) + list(range(var_dim_in_state, len(tensor.shape))) + + list(range(0, var_dim_in_state)), + ) + tensor = layers.reshape( + tensor, + [0] * (len(tensor.shape) - var_dim_in_state) + + [batch_size, beam_size], + ) res = layers.transpose( tensor, list( - range((len(tensor.shape) - 1 - var_dim_in_state), - len(tensor.shape))) + - list(range(0, (len(tensor.shape) - 1 - var_dim_in_state)))) + range( + (len(tensor.shape) - 1 - var_dim_in_state), + len(tensor.shape), + ) + ) + + list(range(0, (len(tensor.shape) - 1 - var_dim_in_state))), + ) return res def mask_probs(probs, finished, noend_mask_tensor): finished = layers.cast(finished, dtype=probs.dtype) - probs = layers.elementwise_mul(layers.expand( - layers.unsqueeze(finished, [2]), [1, 1, self.trg_vocab_size]), - noend_mask_tensor, - axis=-1) - layers.elementwise_mul( - probs, (finished - 1), axis=0) + probs = layers.elementwise_mul( + layers.expand( + layers.unsqueeze(finished, [2]), [1, 1, self.trg_vocab_size] + ), + noend_mask_tensor, + axis=-1, + ) - layers.elementwise_mul(probs, (finished - 1), axis=0) return probs def gather(input, indices, batch_pos): @@ -585,87 +770,108 @@ class Transformer(Layer): batch_size = enc_output.shape[0] # constant number - inf = float(1. * 1e7) + inf = float(1.0 * 1e7) max_len = (enc_output.shape[1] + 20) if max_len is None else max_len - vocab_size_tensor = layers.fill_constant(shape=[1], - dtype="int64", - value=self.trg_vocab_size) + vocab_size_tensor = layers.fill_constant( + shape=[1], dtype="int64", value=self.trg_vocab_size + ) end_token_tensor = to_variable( - np.full([batch_size, beam_size], eos_id, dtype="int64")) + np.full([batch_size, beam_size], eos_id, dtype="int64") + ) noend_array = [-inf] * self.trg_vocab_size noend_array[eos_id] = 0 noend_mask_tensor = to_variable(np.array(noend_array, dtype="float32")) batch_pos = layers.expand( layers.unsqueeze( - to_variable(np.arange(0, batch_size, 1, dtype="int64")), [1]), - [1, beam_size]) + to_variable(np.arange(0, batch_size, 1, dtype="int64")), [1] + ), + [1, beam_size], + ) predict_ids = [] parent_ids = [] ### initialize states of beam search ### log_probs = to_variable( - np.array([[0.] + [-inf] * (beam_size - 1)] * batch_size, - dtype="float32")) + np.array( + [[0.0] + [-inf] * (beam_size - 1)] * batch_size, dtype="float32" + ) + ) - finished = to_variable(np.full([batch_size, beam_size], 0, - dtype="bool")) + finished = to_variable( + np.full([batch_size, beam_size], 0, dtype="bool") + ) - trg_word = layers.fill_constant(shape=[batch_size * beam_size, 1], - dtype="int64", - value=bos_id) + trg_word = layers.fill_constant( + shape=[batch_size * beam_size, 1], dtype="int64", value=bos_id + ) trg_src_attn_bias = merge_batch_beams( - expand_to_beam_size(trg_src_attn_bias, beam_size)) + expand_to_beam_size(trg_src_attn_bias, beam_size) + ) enc_output = merge_batch_beams( - expand_to_beam_size(enc_output, beam_size)) + expand_to_beam_size(enc_output, beam_size) + ) # init states (caches) for transformer, need to be updated according to selected beam - caches = [{ - "k": - layers.fill_constant( - shape=[batch_size, beam_size, self.n_head, 0, self.d_key], - dtype=enc_output.dtype, - value=0), - "v": - layers.fill_constant( - shape=[batch_size, beam_size, self.n_head, 0, self.d_value], - dtype=enc_output.dtype, - value=0), - } for i in range(self.n_layer)] + caches = [ + { + "k": layers.fill_constant( + shape=[batch_size, beam_size, self.n_head, 0, self.d_key], + dtype=enc_output.dtype, + value=0, + ), + "v": layers.fill_constant( + shape=[batch_size, beam_size, self.n_head, 0, self.d_value], + dtype=enc_output.dtype, + value=0, + ), + } + for i in range(self.n_layer) + ] for i in range(paddle.to_tensor(max_len)): - trg_pos = layers.fill_constant(shape=trg_word.shape, - dtype="int64", - value=i) - caches = map_structure(merge_batch_beams, - caches) # TODO: modified for dygraph2static - logits = self.decoder(trg_word, trg_pos, None, trg_src_attn_bias, - enc_output, caches) + trg_pos = layers.fill_constant( + shape=trg_word.shape, dtype="int64", value=i + ) + caches = map_structure( + merge_batch_beams, caches + ) # TODO: modified for dygraph2static + logits = self.decoder( + trg_word, trg_pos, None, trg_src_attn_bias, enc_output, caches + ) caches = map_structure(split_batch_beams, caches) step_log_probs = split_batch_beams( - fluid.layers.log(fluid.layers.softmax(logits))) - - step_log_probs = mask_probs(step_log_probs, finished, - noend_mask_tensor) - log_probs = layers.elementwise_add(x=step_log_probs, - y=log_probs, - axis=0) - log_probs = layers.reshape(log_probs, - [-1, beam_size * self.trg_vocab_size]) + fluid.layers.log(fluid.layers.softmax(logits)) + ) + + step_log_probs = mask_probs( + step_log_probs, finished, noend_mask_tensor + ) + log_probs = layers.elementwise_add( + x=step_log_probs, y=log_probs, axis=0 + ) + log_probs = layers.reshape( + log_probs, [-1, beam_size * self.trg_vocab_size] + ) scores = log_probs - topk_scores, topk_indices = fluid.layers.topk(input=scores, - k=beam_size) + topk_scores, topk_indices = fluid.layers.topk( + input=scores, k=beam_size + ) beam_indices = fluid.layers.elementwise_floordiv( - topk_indices, vocab_size_tensor) + topk_indices, vocab_size_tensor + ) token_indices = fluid.layers.elementwise_mod( - topk_indices, vocab_size_tensor) + topk_indices, vocab_size_tensor + ) # update states - caches = map_structure(lambda x: gather(x, beam_indices, batch_pos), - caches) + caches = map_structure( + lambda x: gather(x, beam_indices, batch_pos), caches + ) log_probs = gather(log_probs, topk_indices, batch_pos) finished = gather(finished, beam_indices, batch_pos) finished = layers.logical_or( - finished, layers.equal(token_indices, end_token_tensor)) + finished, layers.equal(token_indices, end_token_tensor) + ) trg_word = layers.reshape(token_indices, [-1, 1]) predict_ids.append(token_indices) @@ -677,7 +883,8 @@ class Transformer(Layer): predict_ids = layers.stack(predict_ids, axis=0) parent_ids = layers.stack(parent_ids, axis=0) finished_seq = layers.transpose( - layers.gather_tree(predict_ids, parent_ids), [1, 2, 0]) + layers.gather_tree(predict_ids, parent_ids), [1, 2, 0] + ) finished_scores = topk_scores return finished_seq, finished_scores diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py index 3b4d1438c5f..7e77f1ad033 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py @@ -31,36 +31,45 @@ def get_input_descs(args, mode="train"): input_descs_train = { "src_word": [(batch_size, seq_len), "int64", 2], "src_pos": [(batch_size, seq_len), "int64"], - "src_slf_attn_bias": [(batch_size, n_head, seq_len, seq_len), - "float32"], + "src_slf_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], "trg_word": [(batch_size, seq_len), "int64", 2], "trg_pos": [(batch_size, seq_len), "int64"], - "trg_slf_attn_bias": [(batch_size, n_head, seq_len, seq_len), - "float32"], - "trg_src_attn_bias": - [(batch_size, n_head, seq_len, seq_len), - "float32"], # TODO: 1 for predict, seq_len for train + "trg_slf_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], + "trg_src_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], # TODO: 1 for predict, seq_len for train "enc_output": [(batch_size, seq_len, d_model), "float32"], "lbl_word": [(None, 1), "int64"], "lbl_weight": [(None, 1), "float32"], "init_score": [(batch_size, 1), "float32", 2], - "init_idx": [(batch_size, ), "int32"], + "init_idx": [(batch_size,), "int32"], } input_descs_predict = { "src_word": [(batch_size, seq_len), "int64", 2], "src_pos": [(batch_size, seq_len), "int64"], - "src_slf_attn_bias": [(batch_size, n_head, seq_len, seq_len), - "float32"], + "src_slf_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], "trg_word": [(batch_size, seq_len), "int64", 2], "trg_pos": [(batch_size, seq_len), "int64"], - "trg_slf_attn_bias": [(batch_size, n_head, seq_len, seq_len), - "float32"], + "trg_slf_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], "trg_src_attn_bias": [(batch_size, n_head, 1, seq_len), "float32"], "enc_output": [(batch_size, seq_len, d_model), "float32"], "lbl_word": [(None, 1), "int64"], "lbl_weight": [(None, 1), "float32"], "init_score": [(batch_size, 1), "float32", 2], - "init_idx": [(batch_size, ), "int32"], + "init_idx": [(batch_size,), "int32"], } return input_descs_train if mode == "train" else input_descs_predict @@ -125,43 +134,58 @@ class ModelHyperParams(object): weight_sharing = True -def pad_batch_data(insts, - pad_idx, - n_head, - is_target=False, - is_label=False, - return_attn_bias=True, - return_max_len=True, - return_num_token=False): +def pad_batch_data( + insts, + pad_idx, + n_head, + is_target=False, + is_label=False, + return_attn_bias=True, + return_max_len=True, + return_num_token=False, +): return_list = [] max_len = max(len(inst) for inst in insts) inst_data = np.array( - [inst + [pad_idx] * (max_len - len(inst)) for inst in insts]) + [inst + [pad_idx] * (max_len - len(inst)) for inst in insts] + ) return_list += [inst_data.astype("int64").reshape([-1, 1])] if is_label: # label weight - inst_weight = np.array([[1.] * len(inst) + [0.] * (max_len - len(inst)) - for inst in insts]) + inst_weight = np.array( + [ + [1.0] * len(inst) + [0.0] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_weight.astype("float32").reshape([-1, 1])] else: # position data - inst_pos = np.array([ - list(range(0, len(inst))) + [0] * (max_len - len(inst)) - for inst in insts - ]) + inst_pos = np.array( + [ + list(range(0, len(inst))) + [0] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_pos.astype("int64").reshape([-1, 1])] if return_attn_bias: if is_target: slf_attn_bias_data = np.ones((inst_data.shape[0], max_len, max_len)) - slf_attn_bias_data = np.triu(slf_attn_bias_data, - 1).reshape([-1, 1, max_len, max_len]) - slf_attn_bias_data = np.tile(slf_attn_bias_data, - [1, n_head, 1, 1]) * [-1e9] + slf_attn_bias_data = np.triu(slf_attn_bias_data, 1).reshape( + [-1, 1, max_len, max_len] + ) + slf_attn_bias_data = np.tile( + slf_attn_bias_data, [1, n_head, 1, 1] + ) * [-1e9] else: - slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] * - (max_len - len(inst)) - for inst in insts]) + slf_attn_bias_data = np.array( + [ + [0] * len(inst) + [-1e9] * (max_len - len(inst)) + for inst in insts + ] + ) slf_attn_bias_data = np.tile( slf_attn_bias_data.reshape([-1, 1, 1, max_len]), - [1, n_head, max_len, 1]) + [1, n_head, max_len, 1], + ) return_list += [slf_attn_bias_data.astype("float32")] if return_max_len: return_list += [max_len] @@ -175,16 +199,19 @@ def pad_batch_data(insts, def prepare_train_input(insts, src_pad_idx, trg_pad_idx, n_head): src_word, src_pos, src_slf_attn_bias, src_max_len = pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False) + [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False + ) src_word = src_word.reshape(-1, src_max_len) src_pos = src_pos.reshape(-1, src_max_len) trg_word, trg_pos, trg_slf_attn_bias, trg_max_len = pad_batch_data( - [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True) + [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True + ) trg_word = trg_word.reshape(-1, trg_max_len) trg_pos = trg_pos.reshape(-1, trg_max_len) - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, trg_max_len, 1]).astype("float32") + trg_src_attn_bias = np.tile( + src_slf_attn_bias[:, :, ::src_max_len, :], [1, 1, trg_max_len, 1] + ).astype("float32") lbl_word, lbl_weight, num_token = pad_batch_data( [inst[2] for inst in insts], @@ -194,13 +221,21 @@ def prepare_train_input(insts, src_pad_idx, trg_pad_idx, n_head): is_label=True, return_attn_bias=False, return_max_len=False, - return_num_token=True) + return_num_token=True, + ) lbl_word = lbl_word.reshape(-1, 1) lbl_weight = lbl_weight.reshape(-1, 1) data_inputs = [ - src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + lbl_word, + lbl_weight, ] return data_inputs @@ -208,55 +243,66 @@ def prepare_train_input(insts, src_pad_idx, trg_pad_idx, n_head): def prepare_infer_input(insts, src_pad_idx, bos_idx, n_head): src_word, src_pos, src_slf_attn_bias, src_max_len = pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False) + [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False + ) # start tokens trg_word = np.asarray([[bos_idx]] * len(insts), dtype="int64") - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, 1, 1]).astype("float32") + trg_src_attn_bias = np.tile( + src_slf_attn_bias[:, :, ::src_max_len, :], [1, 1, 1, 1] + ).astype("float32") trg_word = trg_word.reshape(-1, 1) src_word = src_word.reshape(-1, src_max_len) src_pos = src_pos.reshape(-1, src_max_len) data_inputs = [ - src_word, src_pos, src_slf_attn_bias, trg_word, trg_src_attn_bias + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_src_attn_bias, ] return data_inputs def get_feed_data_reader(args, mode='train'): - def __for_train__(): - train_reader = paddle.batch(wmt16.train(args.src_vocab_size, - args.trg_vocab_size), - batch_size=args.batch_size) + train_reader = paddle.batch( + wmt16.train(args.src_vocab_size, args.trg_vocab_size), + batch_size=args.batch_size, + ) for batch in train_reader(): - tensors = prepare_train_input(batch, args.eos_idx, args.eos_idx, - args.n_head) + tensors = prepare_train_input( + batch, args.eos_idx, args.eos_idx, args.n_head + ) yield tensors def __for_test__(): - test_reader = paddle.batch(wmt16.test(args.src_vocab_size, - args.trg_vocab_size), - batch_size=args.batch_size) + test_reader = paddle.batch( + wmt16.test(args.src_vocab_size, args.trg_vocab_size), + batch_size=args.batch_size, + ) for batch in test_reader(): - tensors = prepare_infer_input(batch, args.eos_idx, args.eos_idx, - args.n_head) + tensors = prepare_infer_input( + batch, args.eos_idx, args.eos_idx, args.n_head + ) yield tensors return __for_train__ if mode == 'train' else __for_test__ class InputField(object): - def __init__(self, input_slots): self.feed_list = [] for slot in input_slots: self.feed_list.append( - fluid.layers.data(name=slot['name'], - shape=slot['shape'], - dtype=slot['dtype'], - lod_level=slot.get('lod_level', 0), - append_batch_size=False)) + fluid.layers.data( + name=slot['name'], + shape=slot['shape'], + dtype=slot['dtype'], + lod_level=slot.get('lod_level', 0), + append_batch_size=False, + ) + ) def load(program, model_path, executor=None, var_list=None): @@ -269,7 +315,8 @@ def load(program, model_path, executor=None, var_list=None): warnings.warn( "An UnicodeDecodeError is catched, which might be caused by loading " "a python2 saved model. Encoding of pickle.load would be set and " - "load again automatically.") + "load again automatically." + ) load_bak = pickle.load pickle.load = partial(load_bak, encoding="latin1") fluid.load(program, model_path, executor, var_list) @@ -282,16 +329,19 @@ def load_dygraph(model_path, keep_name_table=False): """ try: para_dict, opti_dict = fluid.load_dygraph( - model_path, keep_name_table=keep_name_table) + model_path, keep_name_table=keep_name_table + ) return para_dict, opti_dict except UnicodeDecodeError: warnings.warn( "An UnicodeDecodeError is catched, which might be caused by loading " "a python2 saved model. Encoding of pickle.load would be set and " - "load again automatically.") + "load again automatically." + ) load_bak = pickle.load pickle.load = partial(load_bak, encoding="latin1") para_dict, opti_dict = fluid.load_dygraph( - model_path, keep_name_table=keep_name_table) + model_path, keep_name_table=keep_name_table + ) pickle.load = load_bak return para_dict, opti_dict diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/tsm_config_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/tsm_config_utils.py index 3169311710e..1332e0cb86b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/tsm_config_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/tsm_config_utils.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import logging @@ -25,7 +25,6 @@ CONFIG_SECS = [ class AttrDict(dict): - def __getattr__(self, key): return self[key] @@ -39,6 +38,7 @@ class AttrDict(dict): def parse_config(cfg_file): """Load a config file into AttrDict""" import yaml + with open(cfg_file, 'r') as fopen: yaml_config = AttrDict(yaml.load(fopen, Loader=yaml.Loader)) create_attr_dict(yaml_config) @@ -47,6 +47,7 @@ def parse_config(cfg_file): def create_attr_dict(yaml_config): from ast import literal_eval + for key, value in yaml_config.items(): if type(value) is dict: yaml_config[key] = value = AttrDict(value) @@ -78,7 +79,8 @@ def merge_configs(cfg, sec, args_dict): def print_configs(cfg, mode): logger.info( - "---------------- {:>5} Arguments ----------------".format(mode)) + "---------------- {:>5} Arguments ----------------".format(mode) + ) for sec, sec_items in cfg.items(): logger.info("{}:".format(sec)) for k, v in sec_items.items(): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/yolov3.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/yolov3.py index e502d786430..0a50751b1e7 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/yolov3.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/yolov3.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import os import sys @@ -26,7 +26,6 @@ from darknet import ConvBNLayer class AttrDict(dict): - def __init__(self, *args, **kwargs): super(AttrDict, self).__init__(*args, **kwargs) @@ -78,12 +77,29 @@ cfg.pixel_means = [0.485, 0.456, 0.406] cfg.pixel_stds = [0.229, 0.224, 0.225] # anchors box weight and height cfg.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] # anchor mask of each yolo layer cfg.anchor_masks = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] # IoU threshold to ignore objectness loss of pred box -cfg.ignore_thresh = .7 +cfg.ignore_thresh = 0.7 # # SOLVER options # @@ -97,7 +113,7 @@ cfg.max_iter = 20 if fluid.is_compiled_with_cuda() else 1 cfg.no_mixup_iter = 10 if fluid.is_compiled_with_cuda() else 1 # warm up to learning rate cfg.warm_up_iter = 10 if fluid.is_compiled_with_cuda() else 1 -cfg.warm_up_factor = 0. +cfg.warm_up_factor = 0.0 # lr steps_with_decay cfg.lr_steps = [400000, 450000] cfg.lr_gamma = 0.1 @@ -115,49 +131,61 @@ cfg.class_num = 80 class YoloDetectionBlock(fluid.dygraph.Layer): - def __init__(self, ch_in, channel, is_test=True): super(YoloDetectionBlock, self).__init__() - assert channel % 2 == 0, \ - "channel {} cannot be divided by 2".format(channel) - - self.conv0 = ConvBNLayer(ch_in=ch_in, - ch_out=channel, - filter_size=1, - stride=1, - padding=0, - is_test=is_test) - self.conv1 = ConvBNLayer(ch_in=channel, - ch_out=channel * 2, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) - self.conv2 = ConvBNLayer(ch_in=channel * 2, - ch_out=channel, - filter_size=1, - stride=1, - padding=0, - is_test=is_test) - self.conv3 = ConvBNLayer(ch_in=channel, - ch_out=channel * 2, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) - self.route = ConvBNLayer(ch_in=channel * 2, - ch_out=channel, - filter_size=1, - stride=1, - padding=0, - is_test=is_test) - self.tip = ConvBNLayer(ch_in=channel, - ch_out=channel * 2, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) + assert channel % 2 == 0, "channel {} cannot be divided by 2".format( + channel + ) + + self.conv0 = ConvBNLayer( + ch_in=ch_in, + ch_out=channel, + filter_size=1, + stride=1, + padding=0, + is_test=is_test, + ) + self.conv1 = ConvBNLayer( + ch_in=channel, + ch_out=channel * 2, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) + self.conv2 = ConvBNLayer( + ch_in=channel * 2, + ch_out=channel, + filter_size=1, + stride=1, + padding=0, + is_test=is_test, + ) + self.conv3 = ConvBNLayer( + ch_in=channel, + ch_out=channel * 2, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) + self.route = ConvBNLayer( + ch_in=channel * 2, + ch_out=channel, + filter_size=1, + stride=1, + padding=0, + is_test=is_test, + ) + self.tip = ConvBNLayer( + ch_in=channel, + ch_out=channel * 2, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) def forward(self, inputs): out = self.conv0(inputs) @@ -170,7 +198,6 @@ class YoloDetectionBlock(fluid.dygraph.Layer): class Upsample(fluid.dygraph.Layer): - def __init__(self, scale=2): super(Upsample, self).__init__() self.scale = scale @@ -178,24 +205,22 @@ class Upsample(fluid.dygraph.Layer): def forward(self, inputs): # get dynamic upsample output shape shape_nchw = fluid.layers.shape(inputs) - shape_hw = fluid.layers.slice(shape_nchw, - axes=[0], - starts=[2], - ends=[4]) + shape_hw = fluid.layers.slice( + shape_nchw, axes=[0], starts=[2], ends=[4] + ) shape_hw.stop_gradient = True in_shape = fluid.layers.cast(shape_hw, dtype='int32') out_shape = in_shape * self.scale out_shape.stop_gradient = True # reisze by actual_shape - out = fluid.layers.resize_nearest(input=inputs, - scale=self.scale, - actual_shape=out_shape) + out = fluid.layers.resize_nearest( + input=inputs, scale=self.scale, actual_shape=out_shape + ) return out class YOLOv3(fluid.dygraph.Layer): - def __init__(self, ch_in, is_train=True, use_random=False): super(YOLOv3, self).__init__() @@ -210,47 +235,60 @@ class YOLOv3(fluid.dygraph.Layer): for i in range(3): yolo_block = self.add_sublayer( "yolo_detecton_block_%d" % (i), - YoloDetectionBlock(ch_in_list[i], - channel=512 // (2**i), - is_test=not self.is_train)) + YoloDetectionBlock( + ch_in_list[i], + channel=512 // (2**i), + is_test=not self.is_train, + ), + ) self.yolo_blocks.append(yolo_block) num_filters = len(cfg.anchor_masks[i]) * (cfg.class_num + 5) block_out = self.add_sublayer( "block_out_%d" % (i), - Conv2D(num_channels=1024 // (2**i), - num_filters=num_filters, - filter_size=1, - stride=1, - padding=0, - act=None, - param_attr=ParamAttr( - initializer=fluid.initializer.Normal(0., 0.02)), - bias_attr=ParamAttr( - initializer=fluid.initializer.Constant(0.0), - regularizer=L2Decay(0.)))) + Conv2D( + num_channels=1024 // (2**i), + num_filters=num_filters, + filter_size=1, + stride=1, + padding=0, + act=None, + param_attr=ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02) + ), + bias_attr=ParamAttr( + initializer=fluid.initializer.Constant(0.0), + regularizer=L2Decay(0.0), + ), + ), + ) self.block_outputs.append(block_out) if i < 2: route = self.add_sublayer( "route2_%d" % i, - ConvBNLayer(ch_in=512 // (2**i), - ch_out=256 // (2**i), - filter_size=1, - stride=1, - padding=0, - is_test=(not self.is_train))) + ConvBNLayer( + ch_in=512 // (2**i), + ch_out=256 // (2**i), + filter_size=1, + stride=1, + padding=0, + is_test=(not self.is_train), + ), + ) self.route_blocks_2.append(route) self.upsample = Upsample() @declarative - def forward(self, - inputs, - gtbox=None, - gtlabel=None, - gtscore=None, - im_id=None, - im_shape=None): + def forward( + self, + inputs, + gtbox=None, + gtlabel=None, + gtscore=None, + im_id=None, + im_shape=None, + ): self.outputs = [] self.boxes = [] self.scores = [] @@ -287,7 +325,8 @@ class YOLOv3(fluid.dygraph.Layer): class_num=cfg.class_num, ignore_thresh=cfg.ignore_thresh, downsample_ratio=self.downsample, - use_label_smooth=cfg.label_smooth) + use_label_smooth=cfg.label_smooth, + ) self.losses.append(fluid.layers.reduce_mean(loss)) else: @@ -302,10 +341,12 @@ class YOLOv3(fluid.dygraph.Layer): class_num=cfg.class_num, conf_thresh=cfg.valid_thresh, downsample_ratio=self.downsample, - name="yolo_box" + str(i)) + name="yolo_box" + str(i), + ) self.boxes.append(boxes) self.scores.append( - fluid.layers.transpose(scores, perm=[0, 2, 1])) + fluid.layers.transpose(scores, perm=[0, 2, 1]) + ) self.downsample //= 2 if not self.is_train: @@ -313,13 +354,15 @@ class YOLOv3(fluid.dygraph.Layer): yolo_boxes = fluid.layers.concat(self.boxes, axis=1) yolo_scores = fluid.layers.concat(self.scores, axis=2) - pred = fluid.layers.multiclass_nms(bboxes=yolo_boxes, - scores=yolo_scores, - score_threshold=cfg.valid_thresh, - nms_top_k=cfg.nms_topk, - keep_top_k=cfg.nms_posk, - nms_threshold=cfg.nms_thresh, - background_label=-1) + pred = fluid.layers.multiclass_nms( + bboxes=yolo_boxes, + scores=yolo_scores, + score_threshold=cfg.valid_thresh, + nms_top_k=cfg.nms_topk, + keep_top_k=cfg.nms_posk, + nms_threshold=cfg.nms_thresh, + background_label=-1, + ) return pred else: return sum(self.losses) diff --git a/python/paddle/fluid/tests/unittests/elastic_demo.py b/python/paddle/fluid/tests/unittests/elastic_demo.py index af26abd0d88..37a1c93bddd 100644 --- a/python/paddle/fluid/tests/unittests/elastic_demo.py +++ b/python/paddle/fluid/tests/unittests/elastic_demo.py @@ -15,10 +15,16 @@ import os, sys import time -sys.stderr.write("{}-DISTRIBUTED_TRAINER_ENDPOINTS={}\n".format( - os.environ['PADDLE_TRAINER_ID'], - os.environ['DISTRIBUTED_TRAINER_ENDPOINTS'])) -sys.stderr.write("{}-PADDLE_TRAINERS={}\n".format( - os.environ['PADDLE_TRAINER_ID'], os.environ['PADDLE_TRAINERS'])) +sys.stderr.write( + "{}-DISTRIBUTED_TRAINER_ENDPOINTS={}\n".format( + os.environ['PADDLE_TRAINER_ID'], + os.environ['DISTRIBUTED_TRAINER_ENDPOINTS'], + ) +) +sys.stderr.write( + "{}-PADDLE_TRAINERS={}\n".format( + os.environ['PADDLE_TRAINER_ID'], os.environ['PADDLE_TRAINERS'] + ) +) time.sleep(600) diff --git a/python/paddle/fluid/tests/unittests/fake_reader.py b/python/paddle/fluid/tests/unittests/fake_reader.py index b1e90d8b187..997e9f0fc73 100644 --- a/python/paddle/fluid/tests/unittests/fake_reader.py +++ b/python/paddle/fluid/tests/unittests/fake_reader.py @@ -15,23 +15,24 @@ import numpy as np -def fake_imdb_reader(word_dict_size, - sample_num, - lower_seq_len=100, - upper_seq_len=200, - class_dim=2): - +def fake_imdb_reader( + word_dict_size, + sample_num, + lower_seq_len=100, + upper_seq_len=200, + class_dim=2, +): def __reader__(): for _ in range(sample_num): - length = np.random.random_integers(low=lower_seq_len, - high=upper_seq_len, - size=[1])[0] - ids = np.random.random_integers(low=0, - high=word_dict_size - 1, - size=[length]).astype('int64') - label = np.random.random_integers(low=0, - high=class_dim - 1, - size=[1]).astype('int64')[0] + length = np.random.random_integers( + low=lower_seq_len, high=upper_seq_len, size=[1] + )[0] + ids = np.random.random_integers( + low=0, high=word_dict_size - 1, size=[length] + ).astype('int64') + label = np.random.random_integers( + low=0, high=class_dim - 1, size=[1] + ).astype('int64')[0] yield ids, label return __reader__ diff --git a/python/paddle/fluid/tests/unittests/feed_data_reader.py b/python/paddle/fluid/tests/unittests/feed_data_reader.py index 6d7e73ac3c0..9a1f15ddff5 100644 --- a/python/paddle/fluid/tests/unittests/feed_data_reader.py +++ b/python/paddle/fluid/tests/unittests/feed_data_reader.py @@ -17,7 +17,6 @@ from paddle.fluid.framework import Variable def cyclic_reader(reader): - def __reader__(): while True: for data in reader(): @@ -27,7 +26,6 @@ def cyclic_reader(reader): class FeedDataReader(object): - def __init__(self, feed_list, reader): self._feed_list = [] for var in feed_list: @@ -62,8 +60,11 @@ class FeedDataReader(object): if program._is_data_parallel: use_executor = False if program._places is None: - device_num = len(fluid.cuda_places()) if use_cuda else len( - fluid.cpu_places()) + device_num = ( + len(fluid.cuda_places()) + if use_cuda + else len(fluid.cpu_places()) + ) else: device_num = len(program._places) else: diff --git a/python/paddle/fluid/tests/unittests/fft/spectral_op_np.py b/python/paddle/fluid/tests/unittests/fft/spectral_op_np.py index 0dc958f54fc..65918a0cd59 100644 --- a/python/paddle/fluid/tests/unittests/fft/spectral_op_np.py +++ b/python/paddle/fluid/tests/unittests/fft/spectral_op_np.py @@ -34,8 +34,9 @@ def _get_norm_mode(norm, forward): def _get_inv_norm(n, norm_mode): - assert isinstance(norm_mode, - NormMode), "invalid norm_type {}".format(norm_mode) + assert isinstance(norm_mode, NormMode), "invalid norm_type {}".format( + norm_mode + ) if norm_mode == NormMode.none: return 1.0 if norm_mode == NormMode.by_sqrt_n: @@ -69,8 +70,9 @@ def _fftc2r(a, n=None, axis=-1, norm=None, forward=None): if n is None: n = (a.shape[axis] - 1) * 2 inv_norm = _get_inv_norm(n, norm) - output = _raw_fft(a.conj() if forward else a, n, axis, True, False, - inv_norm) + output = _raw_fft( + a.conj() if forward else a, n, axis, True, False, inv_norm + ) return output @@ -145,8 +147,10 @@ def _fft_fill_conj_grad(x, axes, length_to_double): last_fft_axis = axes[-1] shape = x.shape for multi_index in np.ndindex(*shape): - if 0 < multi_index[last_fft_axis] and multi_index[ - last_fft_axis] <= length_to_double: + if ( + 0 < multi_index[last_fft_axis] + and multi_index[last_fft_axis] <= length_to_double + ): x[multi_index] *= 2 return x diff --git a/python/paddle/fluid/tests/unittests/fft/test_fft.py b/python/paddle/fluid/tests/unittests/fft/test_fft.py index e6ea9865ae3..332a61c07fa 100644 --- a/python/paddle/fluid/tests/unittests/fft/test_fft.py +++ b/python/paddle/fluid/tests/unittests/fft/test_fft.py @@ -30,31 +30,28 @@ RTOL = { 'float32': 1e-03, 'complex64': 1e-3, 'float64': 1e-7, - 'complex128': 1e-7 + 'complex128': 1e-7, } ATOL = {'float32': 0.0, 'complex64': 0, 'float64': 0.0, 'complex128': 0} -def rand_x(dims=1, - dtype='float64', - min_dim_len=1, - max_dim_len=10, - complex=False): +def rand_x( + dims=1, dtype='float64', min_dim_len=1, max_dim_len=10, complex=False +): shape = [np.random.randint(min_dim_len, max_dim_len) for i in range(dims)] if complex: - return np.random.randn( - *shape).astype(dtype) + 1.j * np.random.randn(*shape).astype(dtype) + return np.random.randn(*shape).astype(dtype) + 1.0j * np.random.randn( + *shape + ).astype(dtype) else: return np.random.randn(*shape).astype(dtype) def place(devices, key='place'): - def decorate(cls): module = sys.modules[cls.__module__].__dict__ raw_classes = { - k: v - for k, v in module.items() if k.startswith(cls.__name__) + k: v for k, v in module.items() if k.startswith(cls.__name__) } for raw_name, raw_cls in raw_classes.items(): @@ -62,7 +59,7 @@ def place(devices, key='place'): test_cls = dict(raw_cls.__dict__) test_cls.update({key: d}) new_name = raw_name + '.' + d.__class__.__name__ - module[new_name] = type(new_name, (raw_cls, ), test_cls) + module[new_name] = type(new_name, (raw_cls,), test_cls) del module[raw_name] return cls @@ -82,7 +79,7 @@ def parameterize(fields, values=None): name = cls.__name__ + str(k) name = name + '.' + v.get('suffix') if v.get('suffix') else name - test_cls_module[name] = type(name, (cls, ), test_cls) + test_cls_module[name] = type(name, (cls,), test_cls) for m in list(cls.__dict__): if m.startswith("test"): @@ -95,68 +92,107 @@ def parameterize(fields, values=None): @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_x_complex', rand_x(5, complex=True), None, -1, 'backward'), - ('test_n_grater_input_length', rand_x(5, - max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5, complex=True), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ('test_x_complex', rand_x(5, complex=True), None, -1, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5, complex=True), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestFft(unittest.TestCase): - def test_fft(self): - """Test fft with norm condition - """ + """Test fft with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.fft(self.x, self.n, self.axis, - self.norm), - paddle.fft.fft(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.fft(self.x, self.n, self.axis, self.norm), + paddle.fft.fft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_x_complex', rand_x(5, complex=True), None, -1, 'backward'), - ('test_n_grater_input_length', rand_x(5, - max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5, complex=True), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ('test_x_complex', rand_x(5, complex=True), None, -1, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5, complex=True), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestIfft(unittest.TestCase): - def test_fft(self): - """Test ifft with norm condition - """ + """Test ifft with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.ifft(self.x, self.n, self.axis, - self.norm), - paddle.fft.ifft(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.ifft(self.x, self.n, self.axis, self.norm), + paddle.fft.ifft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestFftException(unittest.TestCase): - def test_fft(self): """Test fft with buoudary condition Test case include: @@ -166,55 +202,108 @@ class TestFftException(unittest.TestCase): - norm out of range """ with self.assertRaises(self.expect_exception): - paddle.fft.fft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.fft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_x_complex128', rand_x(5, complex=True), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5, complex=True), - (4, 4), (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_x_complex128', + rand_x(5, complex=True), + None, + (0, 1), + 'backward', + ), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5, complex=True), + (4, 4), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestFft2(unittest.TestCase): - def test_fft2(self): - """Test fft2 with norm condition - """ + """Test fft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.fft2(self.x, self.n, self.axis, - self.norm), - paddle.fft.fft2(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.fft2(self.x, self.n, self.axis, self.norm), + paddle.fft.fft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), None, ValueError), - ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), - ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), - ('test_n_len_not_equal_axis', rand_x(5, max_dim_len=5), 11, - (0, 1), 'backward', ValueError), - ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError)]) + [ + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + None, + ValueError, + ), + ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), + ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), + ( + 'test_n_len_not_equal_axis', + rand_x(5, max_dim_len=5), + 11, + (0, 1), + 'backward', + ValueError, + ), + ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestFft2Exception(unittest.TestCase): - def test_fft2(self): """Test fft2 with buoudary condition Test case include: @@ -227,267 +316,525 @@ class TestFft2Exception(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.fft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.fft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_x_complex128', rand_x(5, complex=True), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5, complex=True), - (3, 3), (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ('test_x_complex128', rand_x(5, complex=True), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5, complex=True), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestFftn(unittest.TestCase): - def test_fftn(self): - """Test fftn with norm condition - """ + """Test fftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.fftn(self.x, self.n, self.axis, - self.norm), - paddle.fft.fftn(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.fftn(self.x, self.n, self.axis, self.norm), + paddle.fft.fftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_x_complex128', rand_x(5, complex=True), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5, complex=True), - (3, 3), (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ('test_x_complex128', rand_x(5, complex=True), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5, complex=True), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestIFftn(unittest.TestCase): - def test_ifftn(self): - """Test ifftn with norm condition - """ + """Test ifftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.ifftn(self.x, self.n, self.axis, self.norm), - paddle.fft.ifftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.ifftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, -1, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 4, -1, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 2, -1, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, 1, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, 1, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + -1, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 4, + -1, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 2, + -1, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + 1, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + 1, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "ortho", + ), + ], +) class TestHfft(unittest.TestCase): - def test_hfft(self): - """Test hfft with norm condition - """ + """Test hfft with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.hfft(self.x, self.n, self.axis, - self.norm), - paddle.fft.hfft(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=1e-5, - atol=0) + np.testing.assert_allclose( + scipy.fft.hfft(self.x, self.n, self.axis, self.norm), + paddle.fft.hfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=1e-5, + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, -1, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 4, -1, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 2, -1, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + -1, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 4, + -1, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 2, + -1, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "ortho", + ), + ], +) class TestIrfft(unittest.TestCase): - def test_irfft(self): - """Test irfft with norm condition - """ + """Test irfft with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.irfft(self.x, self.n, self.axis, self.norm), - paddle.fft.irfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.irfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, None, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [4], None, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [2], None, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + None, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4], + None, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2], + None, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "ortho", + ), + ], +) class TestIrfftn(unittest.TestCase): - def test_irfftn(self): - """Test irfftn with norm condition - """ + """Test irfftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.irfftn(self.x, self.n, self.axis, self.norm), - paddle.fft.irfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.irfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, None, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [4], None, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [2], None, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + None, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4], + None, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2], + None, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "ortho", + ), + ], +) class TestHfftn(unittest.TestCase): - def test_hfftn(self): - """Test hfftn with norm condition - """ + """Test hfftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.hfftn(self.x, self.n, self.axis, self.norm), - paddle.fft.hfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.hfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, (-2, -1), "backward"), - ('test_with_s', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - [2, 2], (-2, -1), "backward", ValueError), - ('test_axis_not_last', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "backward"), - ('test_norm_forward', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "forward"), - ('test_norm_ortho', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + (-2, -1), + "backward", + ), + ( + 'test_with_s', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2, 2], + (-2, -1), + "backward", + ValueError, + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "ortho", + ), + ], +) class TestHfft2(unittest.TestCase): - def test_hfft2(self): - """Test hfft2 with norm condition - """ + """Test hfft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.hfft2(self.x, self.s, self.axis, self.norm), - paddle.fft.hfft2(paddle.to_tensor(self.x), self.s, self.axis, - self.norm), + paddle.fft.hfft2( + paddle.to_tensor(self.x), self.s, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, (-2, -1), "backward"), - ('test_n_equal_input_length', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (4, 6), - (-2, -1), "backward"), - ('test_axis_not_last', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "backward"), - ('test_norm_forward', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "forward"), - ('test_norm_ortho', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + (-2, -1), + "backward", + ), + ( + 'test_n_equal_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (4, 6), + (-2, -1), + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "ortho", + ), + ], +) class TestIrfft2(unittest.TestCase): - def test_irfft2(self): - """Test irfft2 with norm condition - """ + """Test irfft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.irfft2(self.x, self.s, self.axis, self.norm), - paddle.fft.irfft2(paddle.to_tensor(self.x), self.s, self.axis, - self.norm), + paddle.fft.irfft2( + paddle.to_tensor(self.x), self.s, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, -1, 'backward', RuntimeError), - ('test_n_nagative', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), -1, -1, 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4) + 1j * np.random.randn(4, 4), 0, -1, - 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (1, 2, 3), -1, 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, 10, 'backward', ValueError), - ('test_axis_with_array', np.random.randn(4) + 1j * np.random.randn(4), None, - (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, -1, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + -1, + 'backward', + RuntimeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + -1, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + 0, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2, 3), + -1, + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 10, + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + -1, + 'random', + ValueError, + ), + ], +) class TestHfftException(unittest.TestCase): - def test_hfft(self): """Test hfft with buoudary condition Test case include: @@ -500,27 +847,66 @@ class TestHfftException(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.hfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.hfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_n_nagative', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), -1, -1, 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4) + 1j * np.random.randn(4, 4), 0, -1, - 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (1, 2), -1, 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, 10, 'backward', ValueError), - ('test_axis_with_array', np.random.randn(4) + 1j * np.random.randn(4), None, - (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + -1, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + 0, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + -1, + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 10, + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfftException(unittest.TestCase): - def test_irfft(self): """ Test irfft with buoudary condition @@ -533,34 +919,84 @@ class TestIrfftException(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.irfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.irfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', RuntimeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, None, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (1, 2), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, -1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + RuntimeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + None, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + -1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestHfft2Exception(unittest.TestCase): - def test_hfft2(self): """ Test hfft2 with buoudary condition @@ -574,34 +1010,82 @@ class TestHfft2Exception(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.hfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.hfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_zero_point', - np.random.randn(4, 4, 1) + 1j * np.random.randn(4, 4, 1), None, - (-2, -1), "backward", ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (1, 2), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_zero_point', + np.random.randn(4, 4, 1) + 1j * np.random.randn(4, 4, 1), + None, + (-2, -1), + "backward", + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfft2Exception(unittest.TestCase): - def test_irfft2(self): """ Test irfft2 with buoudary condition @@ -615,34 +1099,84 @@ class TestIrfft2Exception(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.irfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.irfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', RuntimeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (10, 20), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + RuntimeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (10, 20), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestHfftnException(unittest.TestCase): - def test_hfftn(self): """Test hfftn with buoudary condition Test case include: @@ -655,31 +1189,74 @@ class TestHfftnException(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.hfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.hfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (10, 20), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (10, 20), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfftnException(unittest.TestCase): - def test_irfftn(self): """Test irfftn with buoudary condition Test case include: @@ -691,46 +1268,75 @@ class TestIrfftnException(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.irfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.irfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestRfft(unittest.TestCase): - def test_rfft(self): - """Test rfft with norm condition - """ + """Test rfft with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.rfft(self.x, self.n, self.axis, - self.norm), - paddle.fft.rfft(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.rfft(self.x, self.n, self.axis, self.norm), + paddle.fft.rfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestRfftException(unittest.TestCase): - def test_rfft(self): """Test rfft with buoudary condition Test case include: @@ -741,51 +1347,93 @@ class TestRfftException(unittest.TestCase): - the dimensions of n and axis are different """ with self.assertRaises(self.expect_exception): - paddle.fft.rfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.rfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (4, 4), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (4, 4), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestRfft2(unittest.TestCase): - def test_rfft2(self): - """Test rfft2 with norm condition - """ + """Test rfft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.rfft2(self.x, self.n, self.axis, self.norm), - paddle.fft.rfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.rfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), 'backward', RuntimeError), - ('test_x_1dim_tensor', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + 'backward', + RuntimeError, + ), + ('test_x_1dim_tensor', rand_x(1), None, (0, 1), 'backward', ValueError), + ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, (0, 1), 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestRfft2Exception(unittest.TestCase): - def test_rfft2(self): """Test rfft2 with buoudary condition Test case include: @@ -798,47 +1446,83 @@ class TestRfft2Exception(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.rfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.rfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (3, 3), - (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestRfftn(unittest.TestCase): - def test_rfftn(self): - """Test rfftn with norm condition - """ + """Test rfftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.rfftn(self.x, self.n, self.axis, self.norm), - paddle.fft.rfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.rfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x( - 4, complex=True), None, None, 'backward', RuntimeError), - ('test_n_nagative', rand_x(4), (-1, -1), (1, 2), 'backward', ValueError), - ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + RuntimeError, + ), + ( + 'test_n_nagative', + rand_x(4), + (-1, -1), + (1, 2), + 'backward', + ValueError, + ), + ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestRfftnException(unittest.TestCase): - def test_rfftn(self): """Test rfftn with buoudary condition Test case include: @@ -849,45 +1533,75 @@ class TestRfftnException(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.rfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.rfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestIhfft(unittest.TestCase): - def test_ihfft(self): - """Test ihfft with norm condition - """ + """Test ihfft with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.ihfft(self.x, self.n, self.axis, self.norm), - paddle.fft.ihfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.ihfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestIhfftException(unittest.TestCase): - def test_ihfft(self): """Test ihfft with buoudary condition Test case include: @@ -897,53 +1611,101 @@ class TestIhfftException(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.ihfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.ihfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (11, 11), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (1, 1), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (11, 11), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (1, 1), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestIhfft2(unittest.TestCase): - def test_ihfft2(self): - """Test ihfft2 with norm condition - """ + """Test ihfft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.ihfft2(self.x, self.n, self.axis, self.norm), - paddle.fft.ihfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.ihfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), None, ValueError), - ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), - ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), - ('test_n_len_not_equal_axis', rand_x(5, max_dim_len=5), 11, - (0, 1), 'backward', ValueError), - ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError)]) + [ + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + None, + ValueError, + ), + ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), + ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), + ( + 'test_n_len_not_equal_axis', + rand_x(5, max_dim_len=5), + 11, + (0, 1), + 'backward', + ValueError, + ), + ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestIhfft2Exception(unittest.TestCase): - def test_ihfft2(self): """Test ihfft2 with buoudary condition Test case include: @@ -956,46 +1718,75 @@ class TestIhfft2Exception(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.ihfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.ihfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (11, 11), - (0, 1), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (1, 1), - (0, 1), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (11, 11), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (1, 1), + (0, 1), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestIhfftn(unittest.TestCase): - def test_ihfftn(self): - """Test ihfftn with norm condition - """ + """Test ihfftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.ihfftn(self.x, self.n, self.axis, self.norm), - paddle.fft.ihfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.ihfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x( - 4, complex=True), None, None, 'backward', RuntimeError), - ('test_n_nagative', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + RuntimeError, + ), + ('test_n_nagative', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestIhfftnException(unittest.TestCase): - def test_ihfftn(self): """Test ihfftn with buoudary condition Test case include: @@ -1006,88 +1797,107 @@ class TestIhfftnException(unittest.TestCase): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.ihfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.ihfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'n', 'd', 'dtype'), [ - ('test_without_d', 20, 1, 'float32'), - ('test_with_d', 20, 0.5, 'float32'), -]) +@parameterize( + (TEST_CASE_NAME, 'n', 'd', 'dtype'), + [ + ('test_without_d', 20, 1, 'float32'), + ('test_with_d', 20, 0.5, 'float32'), + ], +) class TestFftFreq(unittest.TestCase): - def test_fftfreq(self): - """Test fftfreq with norm condition - """ + """Test fftfreq with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.fftfreq(self.n, self.d).astype(self.dtype), paddle.fft.fftfreq(self.n, self.d, self.dtype).numpy(), rtol=RTOL.get(str(self.dtype)), - atol=ATOL.get(str(self.dtype))) + atol=ATOL.get(str(self.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'n', 'd', 'dtype'), [ - ('test_without_d', 20, 1, 'float32'), - ('test_with_d', 20, 0.5, 'float32'), -]) +@parameterize( + (TEST_CASE_NAME, 'n', 'd', 'dtype'), + [ + ('test_without_d', 20, 1, 'float32'), + ('test_with_d', 20, 0.5, 'float32'), + ], +) class TestRfftFreq(unittest.TestCase): - def test_rfftfreq(self): - """Test rfftfreq with norm condition - """ + """Test rfftfreq with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.rfftfreq(self.n, self.d).astype(self.dtype), paddle.fft.rfftfreq(self.n, self.d, self.dtype).numpy(), rtol=RTOL.get(str(self.dtype)), - atol=ATOL.get(str(self.dtype))) + atol=ATOL.get(str(self.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'axes', 'dtype'), [ - ('test_1d', np.random.randn(10), (0, ), 'float64'), - ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), - ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), - ('test_2d_odd_with_all_axes', - np.random.randn(5, 5) + 1j * np.random.randn(5, 5), None, 'complex128'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'axes', 'dtype'), + [ + ('test_1d', np.random.randn(10), (0,), 'float64'), + ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), + ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), + ( + 'test_2d_odd_with_all_axes', + np.random.randn(5, 5) + 1j * np.random.randn(5, 5), + None, + 'complex128', + ), + ], +) class TestFftShift(unittest.TestCase): - def test_fftshift(self): - """Test fftshift with norm condition - """ + """Test fftshift with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.fftshift(self.x, self.axes), - paddle.fft.fftshift( - paddle.to_tensor(self.x), - self.axes).numpy(), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.fftshift(self.x, self.axes), + paddle.fft.fftshift( + paddle.to_tensor(self.x), self.axes + ).numpy(), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'axes'), - [('test_1d', np.random.randn(10), (0, ), 'float64'), - ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), - ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), - ('test_2d_odd_with_all_axes', - np.random.randn(5, 5) + 1j * np.random.randn(5, 5), None, 'complex128')]) + [ + ('test_1d', np.random.randn(10), (0,), 'float64'), + ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), + ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), + ( + 'test_2d_odd_with_all_axes', + np.random.randn(5, 5) + 1j * np.random.randn(5, 5), + None, + 'complex128', + ), + ], +) class TestIfftShift(unittest.TestCase): - def test_ifftshift(self): - """Test ifftshift with norm condition - """ + """Test ifftshift with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.ifftshift(self.x, self.axes), - paddle.fft.ifftshift( - paddle.to_tensor(self.x), - self.axes).numpy(), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.ifftshift(self.x, self.axes), + paddle.fft.ifftshift( + paddle.to_tensor(self.x), self.axes + ).numpy(), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/fft/test_fft_with_static_graph.py b/python/paddle/fluid/tests/unittests/fft/test_fft_with_static_graph.py index 7b286c5086c..35828ed1606 100644 --- a/python/paddle/fluid/tests/unittests/fft/test_fft_with_static_graph.py +++ b/python/paddle/fluid/tests/unittests/fft/test_fft_with_static_graph.py @@ -19,8 +19,15 @@ import numpy as np import paddle import scipy.fft -from test_fft import (ATOL, DEVICES, RTOL, TEST_CASE_NAME, parameterize, place, - rand_x) +from test_fft import ( + ATOL, + DEVICES, + RTOL, + TEST_CASE_NAME, + parameterize, + place, + rand_x, +) @contextlib.contextmanager @@ -40,70 +47,126 @@ def stgraph(func, place, x, n, axes, norm): @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_x_complex64', rand_x(5, np.float64, - complex=True), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_x_complex64', + rand_x(5, np.float64, complex=True), + None, + -1, + 'backward', + ), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestFft(unittest.TestCase): - def test_static_rfft(self): - with stgraph(paddle.fft.fft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.fft(self.x, self.n, self.axis, - self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.fft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.fft(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestFftException(unittest.TestCase): - def test_fft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.fft, self.place, self.x, self.n, self.axis, - self.norm) as y: + with stgraph( + paddle.fft.fft, self.place, self.x, self.n, self.axis, self.norm + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_x_complex128', rand_x(5, complex=True), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (4, 4), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_x_complex128', + rand_x(5, complex=True), + None, + (0, 1), + 'backward', + ), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (4, 4), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestFft2(unittest.TestCase): - def test_static_fft2(self): - with stgraph(paddle.fft.fft2, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.fft2(self.x, self.n, self.axis, - self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.fft2, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.fft2(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -114,274 +177,593 @@ class TestFft2(unittest.TestCase): ('test_x_1dim_tensor', rand_x(1), None, (0, 1), 'backward', ValueError), ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), ('test_n_zero', rand_x(2), 0, (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, - (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', - ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError) - ]) + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestFft2Exception(unittest.TestCase): - def test_static_fft2(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.fft2, self.place, self.x, self.n, self.axis, - self.norm) as y: + with stgraph( + paddle.fft.fft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_x_complex128', rand_x(5, np.float64, - complex=True), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (3, 3), - (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_x_complex128', + rand_x(5, np.float64, complex=True), + None, + None, + 'backward', + ), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestFftn(unittest.TestCase): - def test_static_fftn(self): - with stgraph(paddle.fft.fftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.fftn(self.x, self.n, self.axis, - self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x(4, - complex=True), None, None, 'backward', TypeError), - ('test_n_nagative', rand_x(4), (-1, -1), (1, 2), 'backward', ValueError), - ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) -class TestRfftnException(unittest.TestCase): + with stgraph( + paddle.fft.fftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.fftn(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + rand_x(4), + (-1, -1), + (1, 2), + 'backward', + ValueError, + ), + ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) +class TestRfftnException(unittest.TestCase): def test_static_rfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.rfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.rfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, -1, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 4, -1, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 2, -1, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, 1, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, 1, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + -1, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 4, + -1, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 2, + -1, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + 1, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + 1, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "ortho", + ), + ], +) class TestHfft(unittest.TestCase): - """Test hfft with norm condition - """ + """Test hfft with norm condition""" def test_hfft(self): - with stgraph(paddle.fft.hfft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.hfft(self.x, self.n, self.axis, - self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, -1, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 4, -1, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 2, -1, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "ortho"), -]) + with stgraph( + paddle.fft.hfft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.hfft(self.x, self.n, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + -1, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 4, + -1, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 2, + -1, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "ortho", + ), + ], +) class TestIrfft(unittest.TestCase): - """Test irfft with norm condition - """ + """Test irfft with norm condition""" def test_irfft(self): - with stgraph(paddle.fft.irfft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.irfft(self.x, self.n, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, None, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [4], None, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [2], None, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "ortho"), -]) + with stgraph( + paddle.fft.irfft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.irfft(self.x, self.n, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + None, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4], + None, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2], + None, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "ortho", + ), + ], +) class Testirfftn(unittest.TestCase): - """Test irfftn with norm condition - """ + """Test irfftn with norm condition""" def test_static_irfftn(self): - with stgraph(paddle.fft.irfftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.irfftn(self.x, self.n, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, None, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [4], None, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [2], None, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "ortho"), -]) + with stgraph( + paddle.fft.irfftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.irfftn(self.x, self.n, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + None, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4], + None, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2], + None, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "ortho", + ), + ], +) class Testhfftn(unittest.TestCase): - """Test hfftn with norm condition - """ + """Test hfftn with norm condition""" def test_static_hfftn(self): - with stgraph(paddle.fft.hfftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.hfftn(self.x, self.n, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, (-2, -1), "backward"), - ('test_n_grater_input_length', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), [4, 8], - (-2, -1), "backward"), - ('test_n_smaller_input_length', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), [2, 4], - (-2, -1), "backward"), - ('test_axis_not_last', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "backward"), - ('test_norm_forward', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "forward"), - ('test_norm_ortho', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "ortho"), -]) + with stgraph( + paddle.fft.hfftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.hfftn(self.x, self.n, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + (-2, -1), + "backward", + ), + ( + 'test_n_grater_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4, 8], + (-2, -1), + "backward", + ), + ( + 'test_n_smaller_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2, 4], + (-2, -1), + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "ortho", + ), + ], +) class Testhfft2(unittest.TestCase): - """Test hfft2 with norm condition - """ + """Test hfft2 with norm condition""" def test_static_hfft2(self): - with stgraph(paddle.fft.hfft2, self.place, self.x, self.s, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.hfft2(self.x, self.s, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, (-2, -1), "backward"), - ('test_n_equal_input_length', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (2, 4), - (-2, -1), "backward"), - ('test_axis_not_last', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "backward"), - ('test_norm_forward', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "forward"), - ('test_norm_ortho', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "ortho"), -]) + with stgraph( + paddle.fft.hfft2, self.place, self.x, self.s, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.hfft2(self.x, self.s, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + (-2, -1), + "backward", + ), + ( + 'test_n_equal_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (2, 4), + (-2, -1), + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "ortho", + ), + ], +) class TestIrfft2(unittest.TestCase): - """Test irfft2 with norm condition - """ + """Test irfft2 with norm condition""" def test_static_irfft2(self): - with stgraph(paddle.fft.irfft2, self.place, self.x, self.s, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.irfft2(self.x, self.s, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_input_dtype', np.random.randn(4, 4, - 4), None, -1, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, -1, 'backward', TypeError), - ('test_n_nagative', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), -1, -1, 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4) + 1j * np.random.randn(4, 4), 0, -1, - 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (1, 2, 3), -1, 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, 10, 'backward', ValueError), - ('test_axis_with_array', np.random.randn(4) + 1j * np.random.randn(4), None, - (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, -1, 'random', ValueError) -]) + with stgraph( + paddle.fft.irfft2, self.place, self.x, self.s, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.irfft2(self.x, self.s, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + -1, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + -1, + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + -1, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + 0, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2, 3), + -1, + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 10, + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + -1, + 'random', + ValueError, + ), + ], +) class TestHfftException(unittest.TestCase): '''Test hfft with buoudary condition Test case include: @@ -393,31 +775,89 @@ class TestHfftException(unittest.TestCase): def test_static_hfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.hfft, self.place, self.x, self.n, self.axis, - self.norm) as y: + with stgraph( + paddle.fft.hfft, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_input_dtype', np.random.randn(4, 4, - 4), None, -1, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, -1, 'backward', TypeError), - ('test_n_nagative', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), -1, -1, 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4) + 1j * np.random.randn(4, 4), 0, -1, - 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (1, 2), -1, 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, 10, 'backward', ValueError), - ('test_axis_with_array', np.random.randn(4) + 1j * np.random.randn(4), None, - (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + -1, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + -1, + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + -1, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + 0, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + -1, + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 10, + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfftException(unittest.TestCase): '''Test Irfft with buoudary condition Test case include: @@ -430,35 +870,97 @@ class TestIrfftException(unittest.TestCase): def test_static_irfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.irfft, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.irfft, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_input_dtype', np.random.randn( - 4, 4, 4), None, None, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', TypeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, None, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (1, 2), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, -1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + None, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + -1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestHfft2Exception(unittest.TestCase): '''Test hfft2 with buoudary condition Test case include: @@ -471,35 +973,97 @@ class TestHfft2Exception(unittest.TestCase): def test_static_hfft2(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.hfft2, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.hfft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_input_dtype', np.random.randn( - 4, 4, 4), None, None, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', TypeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (1, 2), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfft2Exception(unittest.TestCase): '''Test irfft2 with buoudary condition Test case include: @@ -512,35 +1076,97 @@ class TestIrfft2Exception(unittest.TestCase): def test_static_irfft2(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.irfft2, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.irfft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_input_dtype', np.random.randn( - 4, 4, 4), None, None, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', TypeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (10, 20), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (10, 20), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestHfftnException(unittest.TestCase): '''Test hfftn with buoudary condition Test case include: @@ -553,8 +1179,14 @@ class TestHfftnException(unittest.TestCase): def test_static_hfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.hfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.hfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @@ -562,29 +1194,75 @@ class TestHfftnException(unittest.TestCase): @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_input_dtype', np.random.randn( - 4, 4, 4), None, None, 'backward', TypeError), + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + None, + 'backward', + TypeError, + ), # ('test_bool_input', # (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4) # ).astype(np.bool_), None, (-2, -1), 'backward', ValueError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (0, 0), - (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (10, 20), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, - 1, 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError) - ]) + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (10, 20), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfftnException(unittest.TestCase): '''Test irfftn with buoudary condition Test case include: @@ -597,291 +1275,545 @@ class TestIrfftnException(unittest.TestCase): def test_static_irfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.irfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.irfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestRfft(unittest.TestCase): - def test_static_rfft(self): - with stgraph(paddle.fft.rfft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.rfft(self.x, self.n, self.axis, - self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.rfft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.rfft(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestRfftException(unittest.TestCase): - def test_rfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.rfft, self.place, self.x, self.n, self.axis, - self.norm) as y: + with stgraph( + paddle.fft.rfft, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (4, 4), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (4, 4), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestRfft2(unittest.TestCase): - def test_static_rfft2(self): - with stgraph(paddle.fft.rfft2, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.rfft2(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.rfft2, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.rfft2(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), 'backward', TypeError), + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + 'backward', + TypeError, + ), # ('test_x_not_tensor', [0, 1], None, (0, 1), 'backward', ValueError), ('test_x_1dim_tensor', rand_x(1), None, (0, 1), 'backward', ValueError), ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), ('test_n_zero', rand_x(2), 0, (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, - (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', - ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError) - ]) + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestRfft2Exception(unittest.TestCase): - def test_static_rfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.rfft2, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.rfft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (3, 3), - (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestRfftn(unittest.TestCase): - def test_static_rfft(self): - with stgraph(paddle.fft.rfftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.rfftn(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x(4, - complex=True), None, None, 'backward', TypeError), - ('test_n_nagative', rand_x(4), (-1, -1), (1, 2), 'backward', ValueError), - ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) -class TestRfftnException(unittest.TestCase): + with stgraph( + paddle.fft.rfftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.rfftn(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + rand_x(4), + (-1, -1), + (1, 2), + 'backward', + ValueError, + ), + ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) +class TestRfftnException(unittest.TestCase): def test_static_rfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.rfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.rfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestIhfft(unittest.TestCase): - def test_static_ihfft(self): - with stgraph(paddle.fft.ihfft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.ihfft(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.ihfft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.ihfft(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestIhfftException(unittest.TestCase): - def test_static_ihfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.ihfft, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.ihfft, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (11, 11), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (1, 1), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (11, 11), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (1, 1), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestIhfft2(unittest.TestCase): - def test_static_ihfft2(self): - with stgraph(paddle.fft.ihfft2, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.ihfft2(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.ihfft2, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.ihfft2(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), None, ValueError), + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + None, + ValueError, + ), # ('test_x_not_tensor', [0, 1], None, (0, 1), None, ValueError), ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), - ('test_n_len_not_equal_axis', rand_x(5, max_dim_len=5), 11, - (0, 1), 'backward', ValueError), + ( + 'test_n_len_not_equal_axis', + rand_x(5, max_dim_len=5), + 11, + (0, 1), + 'backward', + ValueError, + ), ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, - (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', - ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError) - ]) + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestIhfft2Exception(unittest.TestCase): - def test_static_ihfft2(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.ihfft2, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.ihfft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (11, 11), - (0, 1), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (1, 1), - (0, 1), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (11, 11), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (1, 1), + (0, 1), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestIhfftn(unittest.TestCase): - def test_static_ihfftn(self): - with stgraph(paddle.fft.ihfftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.ihfftn(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x(4, - complex=True), None, None, 'backward', TypeError), - ('test_n_nagative', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) -class TestIhfftnException(unittest.TestCase): + with stgraph( + paddle.fft.ihfftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.ihfftn(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + TypeError, + ), + ('test_n_nagative', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) +class TestIhfftnException(unittest.TestCase): def test_static_ihfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.ihfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.ihfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'axes', 'dtype'), [ - ('test_1d', np.random.randn(10), (0, ), 'float64'), - ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), - ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), - ('test_2d_odd_with_all_axes', - np.random.randn(5, 5) + 1j * np.random.randn(5, 5), None, 'complex128'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'axes', 'dtype'), + [ + ('test_1d', np.random.randn(10), (0,), 'float64'), + ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), + ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), + ( + 'test_2d_odd_with_all_axes', + np.random.randn(5, 5) + 1j * np.random.randn(5, 5), + None, + 'complex128', + ), + ], +) class TestFftShift(unittest.TestCase): - def test_fftshift(self): - """Test fftshift with norm condition - """ + """Test fftshift with norm condition""" paddle.enable_static() mp, sp = paddle.static.Program(), paddle.static.Program() with paddle.static.program_guard(mp, sp): @@ -898,16 +1830,21 @@ class TestFftShift(unittest.TestCase): @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'axes'), - [('test_1d', np.random.randn(10), (0, ), 'float64'), - ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), - ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), - ('test_2d_odd_with_all_axes', - np.random.randn(5, 5) + 1j * np.random.randn(5, 5), None, 'complex128')]) + [ + ('test_1d', np.random.randn(10), (0,), 'float64'), + ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), + ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), + ( + 'test_2d_odd_with_all_axes', + np.random.randn(5, 5) + 1j * np.random.randn(5, 5), + None, + 'complex128', + ), + ], +) class TestIfftShift(unittest.TestCase): - def test_ifftshift(self): - """Test ifftshift with norm condition - """ + """Test ifftshift with norm condition""" paddle.enable_static() mp, sp = paddle.static.Program(), paddle.static.Program() with paddle.static.program_guard(mp, sp): diff --git a/python/paddle/fluid/tests/unittests/fft/test_spectral_op.py b/python/paddle/fluid/tests/unittests/fft/test_spectral_op.py index bc362d1b7a6..64b6cc111db 100644 --- a/python/paddle/fluid/tests/unittests/fft/test_spectral_op.py +++ b/python/paddle/fluid/tests/unittests/fft/test_spectral_op.py @@ -17,7 +17,14 @@ import paddle import re import sys -from spectral_op_np import fft_c2c, fft_r2c, fft_c2r, fft_c2c_backward, fft_r2c_backward, fft_c2r_backward +from spectral_op_np import ( + fft_c2c, + fft_r2c, + fft_c2r, + fft_c2c_backward, + fft_r2c_backward, + fft_c2r_backward, +) from paddle import _C_ops sys.path.append("../") @@ -32,8 +39,11 @@ def parameterize(attrs, input_values=None): if isinstance(attrs, str): attrs = [attrs] - input_dicts = (attrs if input_values is None else - [dict(zip(attrs, vals)) for vals in input_values]) + input_dicts = ( + attrs + if input_values is None + else [dict(zip(attrs, vals)) for vals in input_values] + ) def decorator(base_class): test_class_module = sys.modules[base_class.__module__].__dict__ @@ -43,8 +53,7 @@ def parameterize(attrs, input_values=None): name = class_name(base_class, idx, input_dict) - test_class_module[name] = type(name, (base_class, ), - test_class_dict) + test_class_module[name] = type(name, (base_class,), test_class_dict) for method_name in list(base_class.__dict__): if method_name.startswith("test"): @@ -60,7 +69,8 @@ def to_safe_name(s): def class_name(cls, num, params_dict): suffix = to_safe_name( - next((v for v in params_dict.values() if isinstance(v, str)), "")) + next((v for v in params_dict.values() if isinstance(v, str)), "") + ) if TEST_CASE_NAME in params_dict: suffix = to_safe_name(params_dict["test_case"]) return "{}_{}{}".format(cls.__name__, num, suffix and "_" + suffix) @@ -80,20 +90,55 @@ def fft_c2r_python_api(x, axes, norm, forward, last_dim_size=0): @parameterize( (TEST_CASE_NAME, 'x', 'axes', 'norm', 'forward'), - [('test_axes_is_sqe_type', (np.random.random( - (12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), [0, 1], 'forward', True), - ('test_axis_not_last', (np.random.random( - (4, 8, 4)) + 1j * np.random.random( - (4, 8, 4))).astype(np.complex128), (0, 1), "backward", False), - ('test_norm_forward', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (0, ), "forward", False), - ('test_norm_backward', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (0, ), "backward", True), - ('test_norm_ortho', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (1, ), "ortho", True)]) + [ + ( + 'test_axes_is_sqe_type', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + [0, 1], + 'forward', + True, + ), + ( + 'test_axis_not_last', + ( + np.random.random((4, 8, 4)) + 1j * np.random.random((4, 8, 4)) + ).astype(np.complex128), + (0, 1), + "backward", + False, + ), + ( + 'test_norm_forward', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (0,), + "forward", + False, + ), + ( + 'test_norm_backward', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (0,), + "backward", + True, + ), + ( + 'test_norm_ortho', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (1,), + "ortho", + True, + ), + ], +) class TestFFTC2COp(OpTest): - def setUp(self): self.op_type = "fft_c2c" self.dtype = self.x.dtype @@ -105,90 +150,173 @@ class TestFFTC2COp(OpTest): self.attrs = { 'axes': self.axes, 'normalization': self.norm, - "forward": self.forward + "forward": self.forward, } self.outputs = {'Out': out} - self.out_grad = (np.random.random(self.x.shape) + - 1j * np.random.random(self.x.shape)).astype( - self.x.dtype) - self.x_grad = fft_c2c_backward(self.out_grad, self.axes, self.norm, - self.forward) + self.out_grad = ( + np.random.random(self.x.shape) + 1j * np.random.random(self.x.shape) + ).astype(self.x.dtype) + self.x_grad = fft_c2c_backward( + self.out_grad, self.axes, self.norm, self.forward + ) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad("X", - "Out", - user_defined_grads=[self.x_grad], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + "X", + "Out", + user_defined_grads=[self.x_grad], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) @parameterize( (TEST_CASE_NAME, 'x', 'axes', 'norm', 'forward', 'last_dim_size'), - [('test_axes_is_sqe_type', (np.random.random( - (12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), [0, 1], 'forward', True, 26), - ('test_axis_not_last', (np.random.random( - (4, 7, 4)) + 1j * np.random.random((4, 7, 4))).astype(np.complex128), - (0, 1), "backward", False, None), - ('test_norm_forward', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (0, ), "forward", False, 22), - ('test_norm_backward', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (0, ), "backward", True, 22), - ('test_norm_ortho', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (1, ), "ortho", True, 26)]) + [ + ( + 'test_axes_is_sqe_type', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + [0, 1], + 'forward', + True, + 26, + ), + ( + 'test_axis_not_last', + ( + np.random.random((4, 7, 4)) + 1j * np.random.random((4, 7, 4)) + ).astype(np.complex128), + (0, 1), + "backward", + False, + None, + ), + ( + 'test_norm_forward', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (0,), + "forward", + False, + 22, + ), + ( + 'test_norm_backward', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (0,), + "backward", + True, + 22, + ), + ( + 'test_norm_ortho', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (1,), + "ortho", + True, + 26, + ), + ], +) class TestFFTC2ROp(OpTest): - def setUp(self): self.op_type = "fft_c2r" self.dtype = self.x.dtype self.python_api = fft_c2r_python_api - out = fft_c2r(self.x, self.axes, self.norm, self.forward, - self.last_dim_size) + out = fft_c2r( + self.x, self.axes, self.norm, self.forward, self.last_dim_size + ) self.inputs = {'X': self.x} self.attrs = { "axes": self.axes, "normalization": self.norm, "forward": self.forward, - "last_dim_size": self.last_dim_size + "last_dim_size": self.last_dim_size, } self.outputs = {'Out': out} self.out_grad = np.random.random(out.shape).astype(out.dtype) - self.x_grad = fft_c2r_backward(self.x, self.out_grad, self.axes, - self.norm, self.forward, - self.last_dim_size) + self.x_grad = fft_c2r_backward( + self.x, + self.out_grad, + self.axes, + self.norm, + self.forward, + self.last_dim_size, + ) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(["X"], - "Out", - user_defined_grads=[self.x_grad], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + ["X"], + "Out", + user_defined_grads=[self.x_grad], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) @parameterize( (TEST_CASE_NAME, 'x', 'axes', 'norm', 'forward', 'onesided'), - [('test_axes_is_sqe_type', np.random.randn(12, 18).astype(np.float64), - (0, 1), 'forward', True, True), - ('test_axis_not_last', np.random.randn(4, 8, 4).astype(np.float64), - (0, 1), "backward", False, True), - ('test_norm_forward', np.random.randn(12, 18).astype(np.float64), - (0, 1), "forward", False, False), - ('test_norm_backward', np.random.randn(12, 18).astype(np.float64), - (0, ), "backward", True, False), - ('test_norm_ortho', np.random.randn(12, 18).astype(np.float64), - (1, ), "ortho", True, False)]) + [ + ( + 'test_axes_is_sqe_type', + np.random.randn(12, 18).astype(np.float64), + (0, 1), + 'forward', + True, + True, + ), + ( + 'test_axis_not_last', + np.random.randn(4, 8, 4).astype(np.float64), + (0, 1), + "backward", + False, + True, + ), + ( + 'test_norm_forward', + np.random.randn(12, 18).astype(np.float64), + (0, 1), + "forward", + False, + False, + ), + ( + 'test_norm_backward', + np.random.randn(12, 18).astype(np.float64), + (0,), + "backward", + True, + False, + ), + ( + 'test_norm_ortho', + np.random.randn(12, 18).astype(np.float64), + (1,), + "ortho", + True, + False, + ), + ], +) class TestFFTR2COp(OpTest): - def setUp(self): self.op_type = "fft_r2c" self.dtype = self.x.dtype @@ -201,20 +329,28 @@ class TestFFTR2COp(OpTest): 'axes': self.axes, 'normalization': self.norm, "forward": self.forward, - 'onesided': self.onesided + 'onesided': self.onesided, } self.outputs = {'Out': out} self.out_grad = np.random.random(out.shape).astype(out.dtype) - self.x_grad = fft_r2c_backward(self.x, self.out_grad, self.axes, - self.norm, self.forward, self.onesided) + self.x_grad = fft_r2c_backward( + self.x, + self.out_grad, + self.axes, + self.norm, + self.forward, + self.onesided, + ) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad("X", - "Out", - user_defined_grads=[self.x_grad], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + "X", + "Out", + user_defined_grads=[self.x_grad], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) diff --git a/python/paddle/fluid/tests/unittests/find_ports.py b/python/paddle/fluid/tests/unittests/find_ports.py index 5618d403448..868efe0ee8e 100644 --- a/python/paddle/fluid/tests/unittests/find_ports.py +++ b/python/paddle/fluid/tests/unittests/find_ports.py @@ -24,8 +24,7 @@ def train(): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "worker_endpoints:{}" \ - .format(worker_endpoints) + name = "worker_endpoints:{}".format(worker_endpoints) print(name) file_name = os.getenv("PADDLE_LAUNCH_LOG") diff --git a/python/paddle/fluid/tests/unittests/fleet_heter_ps_training.py b/python/paddle/fluid/tests/unittests/fleet_heter_ps_training.py index e99021f2822..08e1e1d7b24 100644 --- a/python/paddle/fluid/tests/unittests/fleet_heter_ps_training.py +++ b/python/paddle/fluid/tests/unittests/fleet_heter_ps_training.py @@ -30,32 +30,38 @@ def get_dataset(inputs): def net(batch_size=4, lr=0.01): """ - network definition - - Args: - batch_size(int): the size of mini-batch for training - lr(float): learning rate of training - Returns: - avg_cost: LoDTensor of cost. - """ + network definition + + Args: + batch_size(int): the size of mini-batch for training + lr(float): learning rate of training + Returns: + avg_cost: LoDTensor of cost. + """ dnn_input_dim, lr_input_dim = int(2), int(2) with fluid.device_guard("cpu"): - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="float32", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="float32", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] @@ -67,10 +73,13 @@ def net(batch_size=4, lr=0.01): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool # build lr model @@ -80,8 +89,10 @@ def net(batch_size=4, lr=0.01): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") with fluid.device_guard("gpu"): @@ -91,8 +102,10 @@ def net(batch_size=4, lr=0.01): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) @@ -124,22 +137,22 @@ dataset = get_dataset(feeds) if fleet.is_server(): pass - #fleet.init_server() - #fleet.run_server() + # fleet.init_server() + # fleet.run_server() elif fleet.is_heter_worker(): pass - #fleet.init_heter_worker() - #fleet.run_heter_worker(dataset=dataset) + # fleet.init_heter_worker() + # fleet.run_heter_worker(dataset=dataset) fleet.stop_worker() elif fleet.is_worker(): pass - #place = fluid.CPUPlace() - #exe = fluid.Executor(place) - #exe.run(fluid.default_startup_program()) - #fleet.init_worker() - #step = 1 - #for i in range(step): + # place = fluid.CPUPlace() + # exe = fluid.Executor(place) + # exe.run(fluid.default_startup_program()) + # fleet.init_worker() + # step = 1 + # for i in range(step): # exe.train_from_dataset( # program=fluid.default_main_program(), dataset=dataset, debug=False) - #exe.close() - #fleet.stop_worker() + # exe.close() + # fleet.stop_worker() diff --git a/python/paddle/fluid/tests/unittests/fleet_meta_optimizer_base.py b/python/paddle/fluid/tests/unittests/fleet_meta_optimizer_base.py index ebeeb1e272f..ee0f203881f 100755 --- a/python/paddle/fluid/tests/unittests/fleet_meta_optimizer_base.py +++ b/python/paddle/fluid/tests/unittests/fleet_meta_optimizer_base.py @@ -22,15 +22,16 @@ import paddle.distributed.fleet.base.role_maker as role_maker class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" self._debug = False def debug_program(self, main_prog, startup_prog): - if not self._debug: return + if not self._debug: + return main_prog_ops = main_prog.global_block().ops startup_prog_ops = startup_prog.global_block().ops @@ -38,8 +39,11 @@ class TestFleetMetaOptimizer(unittest.TestCase): main_prog_op_types = [op.type for op in main_prog_ops] startup_prog_op_types = [op.type for op in startup_prog_ops] - print("=== debug program and ops in func [{}] ===".format( - inspect.stack()[1].function)) + print( + "=== debug program and ops in func [{}] ===".format( + inspect.stack()[1].function + ) + ) print(main_prog) print(main_prog_op_types) print(startup_prog) @@ -50,29 +54,29 @@ class TestFleetMetaOptimizer(unittest.TestCase): with fluid.unique_name.guard(): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - - fc_1 = paddle.fluid.layers.fc(input=input_x, - size=64, - act='tanh') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + + fc_1 = paddle.fluid.layers.fc( + input=input_x, size=64, act='tanh' + ) fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() return avg_cost, strategy def pp_net(self, main_prog, startup_prog, pp_degree=2): - def fc_block(input_x): fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') @@ -84,23 +88,24 @@ class TestFleetMetaOptimizer(unittest.TestCase): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) with fluid.device_guard("gpu:0"): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) for stage_idx in range(pp_degree): with fluid.device_guard("gpu:" + str(stage_idx)): input_x = fc_block(input_x) with fluid.device_guard("gpu:" + str(pp_degree - 1)): - prediction = paddle.fluid.layers.fc(input=[input_x], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[input_x], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -120,14 +125,16 @@ class TestFleetMetaOptimizer(unittest.TestCase): strategy = fleet.DistributedStrategy() return avg_cost, strategy - def optimizer(self, - loss, - strategy, - train_prog, - startup_prog, - name='momentum', - regularization=None, - grad_clip=None): + def optimizer( + self, + loss, + strategy, + train_prog, + startup_prog, + name='momentum', + regularization=None, + grad_clip=None, + ): with fluid.program_guard(train_prog, startup_prog): with fluid.unique_name.guard(): if name == 'momentum': @@ -135,18 +142,23 @@ class TestFleetMetaOptimizer(unittest.TestCase): learning_rate=0.01, momentum=0.9, regularization=regularization, - grad_clip=grad_clip) + grad_clip=grad_clip, + ) elif name == 'adam': optimizer = paddle.fluid.optimizer.Adam( learning_rate=0.01, regularization=regularization, - grad_clip=grad_clip) + grad_clip=grad_clip, + ) elif name == 'adamw': - optimizer = paddle.optimizer.AdamW(learning_rate=0.01, - weight_decay=0.01, - grad_clip=grad_clip) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = paddle.optimizer.AdamW( + learning_rate=0.01, + weight_decay=0.01, + grad_clip=grad_clip, + ) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(loss) def set_strategy(self, strategy, name): @@ -182,7 +194,7 @@ class TestFleetMetaOptimizer(unittest.TestCase): strategy.dgc_configs = { "rampup_begin_step": 128, "rampup_step": 100, - "sparsity": [0.996, 0.999] + "sparsity": [0.996, 0.999], } elif name == 'recompute': strategy.recompute = True @@ -230,7 +242,7 @@ class TestFleetMetaOptimizer(unittest.TestCase): strategy.recompute_configs = { "checkpoints": ["fc_0.tmp_2", "fc_1.tmp_2"], "enable_offload": True, - "checkpoint_shape": [256] + "checkpoint_shape": [256], } elif name == "pipeline": strategy.pipeline = True diff --git a/python/paddle/fluid/tests/unittests/fleet_ps_training.py b/python/paddle/fluid/tests/unittests/fleet_ps_training.py index 65fa1ef935e..d14605a6179 100644 --- a/python/paddle/fluid/tests/unittests/fleet_ps_training.py +++ b/python/paddle/fluid/tests/unittests/fleet_ps_training.py @@ -15,7 +15,9 @@ import paddle.fluid as fluid from utils import gen_data from nets import mlp -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) from paddle.fluid.incubate.fleet.base import role_maker input_x = fluid.layers.data(name="x", shape=[32], dtype='float32') @@ -43,8 +45,10 @@ elif fleet.is_worker(): exe.run(fleet.startup_program) step = 1001 for i in range(step): - cost_val = exe.run(program=fleet.main_program, - feed=gen_data(), - fetch_list=[cost.name]) - print("worker_index: %d, step%d cost = %f" % - (fleet.worker_index(), i, cost_val[0])) + cost_val = exe.run( + program=fleet.main_program, feed=gen_data(), fetch_list=[cost.name] + ) + print( + "worker_index: %d, step%d cost = %f" + % (fleet.worker_index(), i, cost_val[0]) + ) diff --git a/python/paddle/fluid/tests/unittests/gradient_checker.py b/python/paddle/fluid/tests/unittests/gradient_checker.py index d928abe2ca9..6e200371852 100644 --- a/python/paddle/fluid/tests/unittests/gradient_checker.py +++ b/python/paddle/fluid/tests/unittests/gradient_checker.py @@ -21,6 +21,7 @@ import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.backward import _append_grad_suffix_, _as_list from paddle.fluid.framework import _test_eager_guard + try: from collections.abc import Sequence except: @@ -91,8 +92,11 @@ def make_jacobian(x, y_size, np_dtype): return np.zeros((_product(x.shape), y_size), dtype=np_dtype) elif isinstance(x, Sequence): jacobians = list( - filter(lambda t: t is not None, - (make_jacobian(item, y_size, np_dtype) for item in x))) + filter( + lambda t: t is not None, + (make_jacobian(item, y_size, np_dtype) for item in x), + ) + ) return jacobians else: None @@ -151,7 +155,7 @@ def _compute_numerical_jacobian(program, x, y, place, scope, delta): _set_item(x_t, i, orig, np_type) for j in range(len(y)): - jacobian[j][i, :] = (y_pos[j] - y_neg[j]) / delta / 2. + jacobian[j][i, :] = (y_pos[j] - y_neg[j]) / delta / 2.0 return jacobian @@ -180,10 +184,9 @@ def _compute_analytical_jacobian(program, x, y, place, scope): np_type = dtype_to_np_dtype(y.dtype) # create dy Variable in Program - dy = program.global_block().create_var(name=dy_name, - shape=y.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dy_name, shape=y.shape, dtype=np_type, persistable=True + ) # append backward dx = fluid.gradients(y, x, dy) @@ -213,23 +216,26 @@ def _compute_analytical_jacobian(program, x, y, place, scope): if dx_res[j] is not None: jacobian[dx_idx][:, i] = dx_res[j].flatten() else: - jacobian[dx_idx][:, i] = np.zeros(dx[dx_idx].shape, - dtype=np_type).flatten() + jacobian[dx_idx][:, i] = np.zeros( + dx[dx_idx].shape, dtype=np_type + ).flatten() _set_item(dy_t, i, 0, np_type) return jacobian -def grad_check(x, - y, - x_init=None, - place=None, - program=None, - eps=1e-6, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def grad_check( + x, + y, + x_init=None, + place=None, + program=None, + eps=1e-6, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check numerical and analytical gradients for dy/dx. Each Jacobian gradients is a 2-D array with shape [xi_size, yi_size]. @@ -279,8 +285,10 @@ def grad_check(x, # init inputs if x_init is not None if x_init: if len(x_init) != len(x): - raise ValueError('len(x_init) (=%d) is not the same' - ' as len(x) (= %d)' % (len(x_init), len(x))) + raise ValueError( + 'len(x_init) (=%d) is not the same' + ' as len(x) (= %d)' % (len(x_init), len(x)) + ) # init variable in main program for var, arr in zip(x, x_init): assert var.shape == arr.shape @@ -310,31 +318,37 @@ def grad_check(x, clone_x.append(b.var(xi.name)) break analytical.append( - _compute_analytical_jacobian(prog, clone_x, clone_y, place, scope)) + _compute_analytical_jacobian(prog, clone_x, clone_y, place, scope) + ) for i, (x_idx, y_idx) in enumerate( - product(*[range(len(x)), range(len(y))])): + product(*[range(len(x)), range(len(y))]) + ): a = analytical[y_idx][x_idx] n = numerical[x_idx][y_idx] if not np.allclose(a, n, rtol, atol): - msg = 'Jacobian mismatch for output %s ' \ - 'with respect to input %s on %s,\n' \ - 'numerical:%s\nanalytical:%s\n' \ - % (y[y_idx].name, x[x_idx].name, str(place), n, a) + msg = ( + 'Jacobian mismatch for output %s ' + 'with respect to input %s on %s,\n' + 'numerical:%s\nanalytical:%s\n' + % (y[y_idx].name, x[x_idx].name, str(place), n, a) + ) return fail_test(msg) return True -def double_grad_check(x, - y, - x_init=None, - y_grads=None, - place=None, - program=None, - eps=1e-6, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def double_grad_check( + x, + y, + x_init=None, + y_grads=None, + place=None, + program=None, + eps=1e-6, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check gradients of gradients. This function will append backward to the program before second order gradient check. @@ -375,10 +389,9 @@ def double_grad_check(x, for yi in y: dyi_name = _append_grad_suffix_(yi.name) np_type = dtype_to_np_dtype(yi.dtype) - dy = program.global_block().create_var(name=dyi_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True + ) dy.stop_gradient = False v = np.random.random(size=yi.shape).astype(np_type) set_var_in_scope(scope, place, dyi_name, v) @@ -407,17 +420,19 @@ def double_grad_check(x, # check triple grad and two outputs of the triple Kernel -def triple_grad_check(x, - y, - x_init=None, - y_grads=None, - x_grads_grads=None, - place=None, - program=None, - eps=1e-6, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def triple_grad_check( + x, + y, + x_init=None, + y_grads=None, + x_grads_grads=None, + place=None, + program=None, + eps=1e-6, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check triple gradients. This function will append backward to the program before third order gradient check. @@ -459,10 +474,9 @@ def triple_grad_check(x, for yi in y: dyi_name = _append_grad_suffix_(yi.name) np_type = dtype_to_np_dtype(yi.dtype) - dy = program.global_block().create_var(name=dyi_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True + ) dy.stop_gradient = False v = np.random.random(size=yi.shape).astype(np_type) set_var_in_scope(scope, place, dyi_name, v) @@ -484,10 +498,9 @@ def triple_grad_check(x, for dxi in target_grads: ddxi_name = _append_grad_suffix_(dxi.name) np_type = dtype_to_np_dtype(dxi.dtype) - ddx = program.global_block().create_var(name=ddxi_name, - shape=dxi.shape, - dtype=np_type, - persistable=True) + ddx = program.global_block().create_var( + name=ddxi_name, shape=dxi.shape, dtype=np_type, persistable=True + ) ddx.stop_gradient = False v = np.random.random(size=dxi.shape).astype(np_type) set_var_in_scope(scope, place, ddxi_name, v) @@ -507,30 +520,30 @@ def triple_grad_check(x, target_grads_grads = fluid.gradients(target_grads, x, x_grads_grads) # filter None in target_grads_grads for Dy/Dx may be None in kernel - filted = [(i, dyi) for i, dyi in enumerate(target_grads_grads) - if dyi is not None] + filted = [ + (i, dyi) for i, dyi in enumerate(target_grads_grads) if dyi is not None + ] filted_idx, filted_target_grads_grads = zip(*filted) x += x_grads_grads x_init += x_grads_grads_init # x <=> [x, dout, ddx] - grad_check(x=x, - y=filted_target_grads_grads, - x_init=x_init, - place=place, - program=program, - eps=eps, - atol=atol, - rtol=rtol) - - -def get_static_double_grad(x, - y, - x_init=None, - dy_init=None, - place=None, - program=None): + grad_check( + x=x, + y=filted_target_grads_grads, + x_init=x_init, + place=place, + program=program, + eps=eps, + atol=atol, + rtol=rtol, + ) + + +def get_static_double_grad( + x, y, x_init=None, dy_init=None, place=None, program=None +): """ Get Double Grad result of static graph. @@ -554,10 +567,9 @@ def get_static_double_grad(x, yi = y[i] dyi_name = _append_grad_suffix_(yi.name) np_type = dtype_to_np_dtype(yi.dtype) - dy = program.global_block().create_var(name=dyi_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True + ) dy.stop_gradient = False set_var_in_scope(scope, place, dyi_name, dy_init[i]) y_grads.append(dy) @@ -598,8 +610,10 @@ def get_static_double_grad(x, # init inputs if x_init is not None if x_init: if len(x_init) != len(x): - raise ValueError('len(x_init) (=%d) is not the same' - ' as len(x) (= %d)' % (len(x_init), len(x))) + raise ValueError( + 'len(x_init) (=%d) is not the same' + ' as len(x) (= %d)' % (len(x_init), len(x)) + ) # init variable in main program for var, arr in zip(x, x_init): assert var.shape == arr.shape @@ -611,10 +625,9 @@ def get_static_double_grad(x, np_type = dtype_to_np_dtype(yi.dtype) dy_name = _append_grad_suffix_(yi.name) # create dy Variable in Program - dy = program.global_block().create_var(name=dy_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dy_name, shape=yi.shape, dtype=np_type, persistable=True + ) # init dy tensor in scope value = np.ones(yi.shape, dtype=np_type) dy_t = set_var_in_scope(scope, place, dy_name, value) @@ -633,11 +646,9 @@ def get_static_double_grad(x, return ddx_res -def get_eager_double_grad(func, - x_init=None, - dy_init=None, - place=None, - return_mid_result=False): +def get_eager_double_grad( + func, x_init=None, dy_init=None, place=None, return_mid_result=False +): """ Get Double Grad result of dygraph. @@ -670,11 +681,13 @@ def get_eager_double_grad(func, dys.append(dy_tensor) # calculate first derivative outputs = func(inputs) - d_inputs = paddle.grad(outputs=outputs, - inputs=inputs, - grad_outputs=dys, - create_graph=True, - allow_unused=True) + d_inputs = paddle.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=dys, + create_graph=True, + allow_unused=True, + ) d_inputs = [d_input for d_input in d_inputs if d_input is not None] # calcluate second derivative @@ -691,29 +704,34 @@ def get_eager_double_grad(func, ddy.stop_gradient = False ddys.append(ddy) - dd_inputs = paddle.grad(outputs=d_inputs, - inputs=inputs, - grad_outputs=ddys, - create_graph=create_graph, - allow_unused=True) + dd_inputs = paddle.grad( + outputs=d_inputs, + inputs=inputs, + grad_outputs=ddys, + create_graph=create_graph, + allow_unused=True, + ) if return_mid_result: - return [dd_input for dd_input in dd_inputs - if dd_input is not None], inputs + ddys + return [ + dd_input for dd_input in dd_inputs if dd_input is not None + ], inputs + ddys else: return [ dd_input.numpy() for dd_input in dd_inputs if dd_input is not None ] -def double_grad_check_for_dygraph(func, - x, - y, - x_init=None, - place=None, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def double_grad_check_for_dygraph( + func, + x, + y, + x_init=None, + place=None, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check second order gradients of dygraph. This function will compare the second order gradients of dygraph and second order gradients of static graph @@ -755,34 +773,38 @@ def double_grad_check_for_dygraph(func, paddle.disable_static() with _test_eager_guard(): - eager_double_grad = get_eager_double_grad(func, x_init, y_grads_init, - place) + eager_double_grad = get_eager_double_grad( + func, x_init, y_grads_init, place + ) paddle.enable_static() - static_double_grad = get_static_double_grad(x, y, x_init, y_grads_init, - place) + static_double_grad = get_static_double_grad( + x, y, x_init, y_grads_init, place + ) if len(static_double_grad) != len(eager_double_grad): - msg = "The output grad tensor's number of static graph is different with dygraph, " \ + msg = ( + "The output grad tensor's number of static graph is different with dygraph, " "please check the python api unit test used." + ) raise RuntimeError(msg) for i in range(len(static_double_grad)): - if not np.allclose(static_double_grad[i], eager_double_grad[i], rtol, - atol): - msg = 'Check eager double result fail. Mismatch between static_graph double grad ' \ - 'and eager double grad on %s, the output double grad tensor\'s index is : %d \n' \ - 'static:%s\n eager:%s\n' \ + if not np.allclose( + static_double_grad[i], eager_double_grad[i], rtol, atol + ): + msg = ( + 'Check eager double result fail. Mismatch between static_graph double grad ' + 'and eager double grad on %s, the output double grad tensor\'s index is : %d \n' + 'static:%s\n eager:%s\n' % (str(place), i, static_double_grad[i], eager_double_grad[i]) + ) return fail_test(msg) -def get_static_triple_grad(x, - y, - x_init=None, - dy_init=None, - place=None, - program=None): +def get_static_triple_grad( + x, y, x_init=None, dy_init=None, place=None, program=None +): """ Get Triple Grad result of static graph. @@ -805,10 +827,9 @@ def get_static_triple_grad(x, yi = y[i] dyi_name = _append_grad_suffix_(yi.name) np_type = dtype_to_np_dtype(yi.dtype) - dy = program.global_block().create_var(name=dyi_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True + ) dy.stop_gradient = False set_var_in_scope(scope, place, dyi_name, dy_init[i]) y_grads.append(dy) @@ -828,19 +849,14 @@ def get_static_triple_grad(x, value = np.ones(dxi.shape, dtype=np_type) x_grads_grads_init.append(value) - return get_static_double_grad(x, - y, - x_init, - dy_init=x_grads_grads_init, - place=place, - program=program) + return get_static_double_grad( + x, y, x_init, dy_init=x_grads_grads_init, place=place, program=program + ) -def get_eager_triple_grad(func, - x_init=None, - dy_init=None, - place=None, - return_mid_result=False): +def get_eager_triple_grad( + func, x_init=None, dy_init=None, place=None, return_mid_result=False +): """ Get triple Grad result of dygraph. @@ -853,11 +869,9 @@ def get_eager_triple_grad(func, Returns: A list of numpy array that stores second derivative result calulated by dygraph """ - dd_y, dd_x = get_eager_double_grad(func, - x_init, - dy_init, - place, - return_mid_result=True) + dd_y, dd_x = get_eager_double_grad( + func, x_init, dy_init, place, return_mid_result=True + ) # calcluate third derivative dddys = [] @@ -866,23 +880,24 @@ def get_eager_triple_grad(func, dddy = paddle.ones(shape=dd_yi.shape, dtype=dd_yi.dtype) dddy.stop_gradient = False dddys.append(dddy) - ddd_inputs = paddle.grad(outputs=dd_y, - inputs=dd_x, - grad_outputs=dddys, - allow_unused=True) + ddd_inputs = paddle.grad( + outputs=dd_y, inputs=dd_x, grad_outputs=dddys, allow_unused=True + ) return [ ddd_input.numpy() for ddd_input in ddd_inputs if ddd_input is not None ] -def triple_grad_check_for_dygraph(func, - x, - y, - x_init=None, - place=None, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def triple_grad_check_for_dygraph( + func, + x, + y, + x_init=None, + place=None, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check third order gradients of dygraph. This function will compare the third order gradients of dygraph and third order gradients of static graph @@ -924,23 +939,30 @@ def triple_grad_check_for_dygraph(func, paddle.disable_static() with _test_eager_guard(): - eager_triple_grad = get_eager_triple_grad(func, x_init, y_grads_init, - place) + eager_triple_grad = get_eager_triple_grad( + func, x_init, y_grads_init, place + ) paddle.enable_static() - static_triple_grad = get_static_triple_grad(x, y, x_init, y_grads_init, - place) + static_triple_grad = get_static_triple_grad( + x, y, x_init, y_grads_init, place + ) if len(static_triple_grad) != len(eager_triple_grad): - msg = "The output grad tensor's number of static graph is different with dygraph, " \ + msg = ( + "The output grad tensor's number of static graph is different with dygraph, " "please check the python api unit test used." + ) raise RuntimeError(msg) for i in range(len(static_triple_grad)): - if not np.allclose(static_triple_grad[i], eager_triple_grad[i], rtol, - atol): - msg = 'Check eager double result fail. Mismatch between static_graph double grad ' \ - 'and eager double grad on %s, the output double grad tensor\'s index is : %d \n' \ - 'static:%s\n eager:%s\n' \ + if not np.allclose( + static_triple_grad[i], eager_triple_grad[i], rtol, atol + ): + msg = ( + 'Check eager double result fail. Mismatch between static_graph double grad ' + 'and eager double grad on %s, the output double grad tensor\'s index is : %d \n' + 'static:%s\n eager:%s\n' % (str(place), i, static_triple_grad[i], eager_triple_grad[i]) + ) return fail_test(msg) diff --git a/python/paddle/fluid/tests/unittests/hccl_tools.py b/python/paddle/fluid/tests/unittests/hccl_tools.py index f47fe9962fe..85011b0d237 100644 --- a/python/paddle/fluid/tests/unittests/hccl_tools.py +++ b/python/paddle/fluid/tests/unittests/hccl_tools.py @@ -48,24 +48,28 @@ def parse_args(): Examples: >>> parse_args() """ - parser = ArgumentParser(description="mindspore distributed training launch " - "helper utilty that will generate hccl" - " config file") + parser = ArgumentParser( + description="mindspore distributed training launch " + "helper utilty that will generate hccl" + " config file" + ) parser.add_argument( "--device_num", type=str, default="[0,8)", - help= - "The number of the Ascend accelerators used. please note that the Ascend accelerators" + help="The number of the Ascend accelerators used. please note that the Ascend accelerators" "used must be continuous, such [0,4) means to use four chips " "0,1,2,3; [0,1) means to use chip 0; The first four chips are" "a group, and the last four chips are a group. In addition to" "the [0,8) chips are allowed, other cross-group such as [3,6)" - "are prohibited.") - parser.add_argument("--visible_devices", - type=str, - default="0,1,2,3,4,5,6,7", - help="will use the visible devices sequentially") + "are prohibited.", + ) + parser.add_argument( + "--visible_devices", + type=str, + default="0,1,2,3,4,5,6,7", + help="will use the visible devices sequentially", + ) parser.add_argument("--server_ip", type=str, default="", help="server ip") args = parser.parse_args() return args @@ -108,20 +112,25 @@ def main(): first_num = int(args.device_num[1]) last_num = int(args.device_num[3]) if first_num < 0 or last_num > 8: - raise ValueError("device num {} must be in range [0,8] !".format( - args.device_num)) + raise ValueError( + "device num {} must be in range [0,8] !".format(args.device_num) + ) if first_num > last_num: raise ValueError( "First num {} of device num {} must less than last num {} !".format( - first_num, args.device_num, last_num)) + first_num, args.device_num, last_num + ) + ) if first_num < 4: if last_num > 4: if first_num == 0 and last_num == 8: pass else: raise ValueError( - "device num {} must be in the same group of [0,4] or [4,8] !" - .format(args.device_num)) + "device num {} must be in the same group of [0,4] or [4,8] !".format( + args.device_num + ) + ) device_num_list = list(range(first_num, last_num)) print("device_num_list:", device_num_list) @@ -146,26 +155,32 @@ def main(): device = { 'device_id': device_id, 'device_ip': device_ip, - 'rank_id': str(rank_id) + 'rank_id': str(rank_id), } - print('rank_id:{}, device_id:{}, device_ip:{}'.format( - rank_id, device_id, device_ip)) + print( + 'rank_id:{}, device_id:{}, device_ip:{}'.format( + rank_id, device_id, device_ip + ) + ) rank_id += 1 device_list.append(device) - hccn_table['server_list'].append({ - 'server_id': server_id, - 'device': device_list, - 'host_nic_ip': 'reserve' - }) + hccn_table['server_list'].append( + { + 'server_id': server_id, + 'device': device_list, + 'host_nic_ip': 'reserve', + } + ) hccn_table['status'] = 'completed' # save hccn_table to file table_path = os.getcwd() table_fn = os.path.join( table_path, - 'hccl_{}p_{}_{}.json'.format(len(device_num_list), - "".join(map(str, - device_num_list)), server_id)) + 'hccl_{}p_{}_{}.json'.format( + len(device_num_list), "".join(map(str, device_num_list)), server_id + ), + ) with open(table_fn, 'w') as table_fp: json.dump(hccn_table, table_fp, indent=4) sys.stdout.flush() diff --git a/python/paddle/fluid/tests/unittests/hdfs_test_utils.py b/python/paddle/fluid/tests/unittests/hdfs_test_utils.py index 0f87e560336..30d1c614937 100644 --- a/python/paddle/fluid/tests/unittests/hdfs_test_utils.py +++ b/python/paddle/fluid/tests/unittests/hdfs_test_utils.py @@ -15,13 +15,17 @@ import unittest import os -from paddle.distributed.fleet.utils.fs import FSFileExistsError, FSFileNotExistsError, HDFSClient, LocalFS +from paddle.distributed.fleet.utils.fs import ( + FSFileExistsError, + FSFileNotExistsError, + HDFSClient, + LocalFS, +) java_home = os.environ["JAVA_HOME"] class FSTestBase(unittest.TestCase): - def _test_dirs(self, fs): dir_path = os.path.abspath("./test_dir") fs.delete(dir_path) @@ -217,10 +221,12 @@ class FSTestBase(unittest.TestCase): pass def _test_list_dir(self, fs): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=15 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", + None, + time_out=15 * 1000, + sleep_inter=100, + ) fs.ls_dir("test_not_exists") def _test_touch(self, fs): diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py index 2b85788ae56..2e02163061f 100644 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py @@ -34,7 +34,6 @@ micro_batch_size = 2 class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -47,16 +46,17 @@ class TestDistPPTraning(unittest.TestCase): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) def build_optimizer(self, model): - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) return scheduler, optimizer def test_pp_model(self): @@ -67,7 +67,7 @@ class TestDistPPTraning(unittest.TestCase): rank_id = dist.get_rank() set_random_seed(1024, dp_id, rank_id) - #construct model a + # construct model a model_a = AlexNet(10) scheduler_a, optimizer_a = self.build_optimizer(model_a) @@ -87,15 +87,21 @@ class TestDistPPTraning(unittest.TestCase): param.set_value(parameters[idx + pp_id * (param_len // 2)]) # construct reader - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size, drop_last=True + ) for step_id, data in enumerate(train_reader()): - x_data = np.array([x[0] for x in data]).astype('float32').reshape( - batch_size, 1, 28, 28) - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + x_data = ( + np.array([x[0] for x in data]) + .astype('float32') + .reshape(batch_size, 1, 28, 28) + ) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) img.stop_gradient = True @@ -113,9 +119,9 @@ class TestDistPPTraning(unittest.TestCase): loss_b = model_b.train_batch([img, label], optimizer_b, scheduler_b) print("loss: ", loss_a.numpy(), loss_b.numpy()) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=5e-5) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py index 7cdce186cf8..13badaf9440 100644 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py @@ -23,7 +23,6 @@ import paddle.nn.functional as F class ReshapeHelp(Layer): - def __init__(self, shape): super(ReshapeHelp, self).__init__() self.shape = shape @@ -33,7 +32,6 @@ class ReshapeHelp(Layer): class AlexNet(Layer): - def __init__(self, num_classes=10): super(AlexNet, self).__init__() self.features = Sequential( @@ -64,7 +62,6 @@ class AlexNet(Layer): class AlexNetPipe(AlexNet): - def to_layers(self): feat = [self.features[i] for i in range(len(self.features))] loss_fn = [self.reshape_layer, self.classifier] @@ -73,7 +70,6 @@ class AlexNetPipe(AlexNet): class AlexNetPipeDesc(PipelineLayer): - def __init__(self, num_classes=10, **kwargs): self.num_classes = num_classes decs = [ @@ -93,20 +89,19 @@ class AlexNetPipeDesc(PipelineLayer): LayerDesc(ReshapeHelp, shape=[-1, 256]), LayerDesc(nn.Linear, 256, self.num_classes), # classifier ] - super(AlexNetPipeDesc, self).__init__(layers=decs, - loss_fn=nn.CrossEntropyLoss(), - **kwargs) + super(AlexNetPipeDesc, self).__init__( + layers=decs, loss_fn=nn.CrossEntropyLoss(), **kwargs + ) class TestPipeLayerAPI(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.pipeline_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": self.pipeline_parallel_size + "pp_degree": self.pipeline_parallel_size, } fleet.init(is_collective=True, strategy=strategy) self.hcg = fleet.get_hybrid_communicate_group() @@ -117,9 +112,11 @@ class TestPipeLayerAPI(unittest.TestCase): def test_pipelayer_sequential(self): init_net = AlexNetPipe() - pipe_model = PipelineLayer(layers=init_net.to_layers(), - num_stages=self.pipeline_parallel_size, - loss_fn=nn.CrossEntropyLoss()) + pipe_model = PipelineLayer( + layers=init_net.to_layers(), + num_stages=self.pipeline_parallel_size, + loss_fn=nn.CrossEntropyLoss(), + ) stage_id = self.hcg.get_stage_id() init_parameters = init_net.parameters() pipe_parameters = pipe_model.parameters() diff --git a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_controlflow.py b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_controlflow.py index 5482eda5446..39dee27fe4e 100644 --- a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_controlflow.py +++ b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_controlflow.py @@ -27,32 +27,28 @@ paddle.enable_static() # and new executor twice and check the result. # please override the _get_feeds() and build_prgram() class TestCompatibility(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.iter_run = 4 def _get_feed(self): - """ return the feeds - """ + """return the feeds""" return None def build_program(self): - def true_func(): - return layers.fill_constant(shape=[1, 2], dtype='int32', - value=1), layers.fill_constant( - shape=[2, 3], - dtype='bool', - value=True) + return layers.fill_constant( + shape=[1, 2], dtype='int32', value=1 + ), layers.fill_constant(shape=[2, 3], dtype='bool', value=True) def false_func(): - return layers.fill_constant(shape=[3, 4], dtype='float32', - value=3), layers.fill_constant( - shape=[4, 5], - dtype='int64', - value=2) + return layers.fill_constant( + shape=[3, 4], dtype='float32', value=3 + ), layers.fill_constant(shape=[4, 5], dtype='int64', value=2) main_program = Program() startup_program = Program() @@ -101,14 +97,11 @@ class TestCompatibility(unittest.TestCase): class TestWhile(TestCompatibility): - def _get_feed(self): - """ return the feeds - """ + """return the feeds""" return None def build_program(self): - def cond(i, ten): return i < ten @@ -119,10 +112,12 @@ class TestWhile(TestCompatibility): main_program = paddle.static.default_main_program() startup_program = paddle.static.default_startup_program() with paddle.static.program_guard(main_program, startup_program): - i = paddle.full(shape=[1], fill_value=0, - dtype='int64') # loop counter - ten = paddle.full(shape=[1], fill_value=10, - dtype='int64') # loop length + i = paddle.full( + shape=[1], fill_value=0, dtype='int64' + ) # loop counter + ten = paddle.full( + shape=[1], fill_value=10, dtype='int64' + ) # loop length i, ten = paddle.static.nn.while_loop(cond, body, [i, ten]) exe = paddle.static.Executor(paddle.CPUPlace()) diff --git a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_executor.py b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_executor.py index 9da058dfee6..f960ad6b049 100644 --- a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_executor.py +++ b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_executor.py @@ -30,10 +30,12 @@ paddle.enable_static() class TestDryRun(unittest.TestCase): - def setUp(self): - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.place = core.Place() self.place.set_place(place) @@ -57,7 +59,8 @@ class TestDryRun(unittest.TestCase): standaloneexecutor = StandaloneExecutor(self.place, main_program.desc) # test for cost_info cost_info = standaloneexecutor.dry_run( - scope, {"a": np.ones([2, 2], dtype="float32")}) + scope, {"a": np.ones([2, 2], dtype="float32")} + ) self.check_cost_info(cost_info) def check_cost_info(self, cost_info): @@ -103,11 +106,13 @@ def build_program(): class ExecutorStatisticsTestCase(unittest.TestCase): - def setUp(self): self.iter_n = 3 - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.perf_path = './perfstat' def test_parallel_executor_statistics(self): @@ -140,7 +145,8 @@ class ExecutorStatisticsTestCase(unittest.TestCase): with framework._enable_standalone_executor(enable): exe = paddle.static.Executor(self.place) helper_profiler = profiler.Profiler( - targets=[profiler.ProfilerTarget.CPU], scheduler=(1, 2)) + targets=[profiler.ProfilerTarget.CPU], scheduler=(1, 2) + ) helper_profiler.start() for i in range(self.iter_n): exe.run(main_program, fetch_list=fetch_list) @@ -157,11 +163,13 @@ class ExecutorStatisticsTestCase(unittest.TestCase): class MultiStreamModelTestCase(unittest.TestCase): - def setUp(self): self.iter_n = 2 - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def test_result(self): ground_truths = self.run_test(False) @@ -180,16 +188,19 @@ class MultiStreamModelTestCase(unittest.TestCase): outs = [] for i in range(self.iter_n): outs.append( - exe.run(main_program, scope=scope, fetch_list=fetch_list)) + exe.run(main_program, scope=scope, fetch_list=fetch_list) + ) print(outs) return outs class SwitchExecutorInterfaceWithFeed(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.iter_run = 2 def build_program(self, is_double=False): @@ -205,24 +216,27 @@ class SwitchExecutorInterfaceWithFeed(unittest.TestCase): return main_program, startup_program, [c] - def _run(self, - feed, - use_str=False, - is_double=False, - add_wrong_fetch=False, - use_compiled=False): + def _run( + self, + feed, + use_str=False, + is_double=False, + add_wrong_fetch=False, + use_compiled=False, + ): paddle.seed(2020) main_program, startup_program, fetch_vars = self.build_program( - is_double) + is_double + ) exe = paddle.static.Executor(self.place) exe.run(startup_program) if use_compiled: main_program = paddle.static.CompiledProgram( - main_program).with_data_parallel(fetch_vars[0].name, - places=[self.place]) + main_program + ).with_data_parallel(fetch_vars[0].name, places=[self.place]) if use_str: # test for fetch name fetch_vars = [x.name for x in fetch_vars] @@ -239,15 +253,13 @@ class SwitchExecutorInterfaceWithFeed(unittest.TestCase): def run_raw_executor(self, feed, use_compiled=False): with framework._enable_standalone_executor(False): # run construct program 1 - out1 = self._run(feed, - use_str=False, - is_double=False, - use_compiled=use_compiled) + out1 = self._run( + feed, use_str=False, is_double=False, use_compiled=use_compiled + ) # run construct program 2 with same executor - out2 = self._run(feed, - use_str=True, - is_double=True, - use_compiled=use_compiled) + out2 = self._run( + feed, use_str=True, is_double=True, use_compiled=use_compiled + ) return [out1, out2] @@ -303,7 +315,6 @@ class SwitchExecutorInterfaceWithFeed(unittest.TestCase): class TestException(unittest.TestCase): - def setUp(self): self.place = paddle.CPUPlace() self.fetch_vars = None @@ -315,10 +326,9 @@ class TestException(unittest.TestCase): w = paddle.rand([10, 3]) ids = paddle.static.data(name="id", shape=[5], dtype='int64') data = paddle.static.data(name="data", shape=[3], dtype='float32') - emb = paddle.nn.functional.embedding(x=ids, - weight=w, - sparse=False, - name="embedding") + emb = paddle.nn.functional.embedding( + x=ids, weight=w, sparse=False, name="embedding" + ) emb = emb + data return main_program, startup_program, emb @@ -342,43 +352,52 @@ class TestException(unittest.TestCase): return out def test_exception(self): - feed = [{ - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }, { - 'id': np.array([1, 2, 3, 4, 11]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }] + feed = [ + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + { + 'id': np.array([1, 2, 3, 4, 11]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + ] self.assertRaises(ValueError, self.run_new_executor, feed) def test_nan(self): flags = {'FLAGS_check_nan_inf': True, 'FLAGS_benchmark': True} paddle.fluid.set_flags(flags) - feed = [{ - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }, { - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }] + feed = [ + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + ] feed[1]['data'][0] = np.nan self.assertRaises(RuntimeError, self.run_new_executor, feed) def test_scope_find_temp_var(self): - feed = [{ - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }, { - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([2, 2, 2]).astype(np.float32), - }] + feed = [ + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([2, 2, 2]).astype(np.float32), + }, + ] self.run_new_executor(feed) - self.assertIsNone(paddle.static.global_scope().find_var( - self.fetch_vars.name)) + self.assertIsNone( + paddle.static.global_scope().find_var(self.fetch_vars.name) + ) class TestInplaceApiWithDataTransform(unittest.TestCase): - def test_increment(self): if paddle.fluid.core.is_compiled_with_cuda(): with paddle.fluid.device_guard("gpu:0"): @@ -389,8 +408,9 @@ class TestInplaceApiWithDataTransform(unittest.TestCase): with framework._enable_standalone_executor(): for i in range(10): - a, = exe.run(paddle.static.default_main_program(), - fetch_list=[x]) + (a,) = exe.run( + paddle.static.default_main_program(), fetch_list=[x] + ) self.assertEqual(a[0], 1) diff --git a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_multiply_write.py b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_multiply_write.py index 0a707d1683f..e5e44b3a28b 100644 --- a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_multiply_write.py +++ b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_multiply_write.py @@ -22,19 +22,17 @@ paddle.enable_static() class TestMultiplyWrite(TestCompatibility): - def _get_feed(self): - """ return the feeds - """ + """return the feeds""" return None def build_program(self): main_program = Program() startup_program = Program() with paddle.static.program_guard(main_program, startup_program): - out = paddle.full((1, ), 1) - inp1 = paddle.full((1, ), 2) - inp2 = paddle.full((1, ), 3) + out = paddle.full((1,), 1) + inp1 = paddle.full((1,), 2) + inp2 = paddle.full((1,), 3) paddle.fluid.layers.assign(inp1, out) paddle.fluid.layers.assign(inp2, out) diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/deprecated/test_custom_nllloss_ipu.py b/python/paddle/fluid/tests/unittests/ipu/custom_ops/deprecated/test_custom_nllloss_ipu.py index 9ae7b307ca5..55a9b1338ab 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/deprecated/test_custom_nllloss_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/deprecated/test_custom_nllloss_ipu.py @@ -22,22 +22,23 @@ import paddle.static from paddle.utils.cpp_extension import load sys.path.append( - os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__))))) + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) from op_test_ipu import IPUOpTest def load_custom_ops(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_nll_loss", - sources=[f"{cur_dir}/custom_nllloss.cc"], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], - extra_ldflags=['-lpopfloat']) + custom_ops = load( + name="custom_nll_loss", + sources=[f"{cur_dir}/custom_nllloss.cc"], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + extra_ldflags=['-lpopfloat'], + ) return custom_ops class TestBase(IPUOpTest): - def setUp(self): self.load_custom_ops() self.set_atol() @@ -75,12 +76,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = self.op(x, label, **self.op_attrs) out = paddle.mean(out) self.fetch_list = [out.name] @@ -96,7 +97,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_test_op(self): self.op = self.custom_ops.custom_nll_loss self.op_attrs = { diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_checkpointoutput_ipu.py b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_checkpointoutput_ipu.py index 698cef211db..866c69ffbdd 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_checkpointoutput_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_checkpointoutput_ipu.py @@ -27,16 +27,17 @@ from op_test_ipu import IPUOpTest def load_custom_ops(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="checkpointoutput", - sources=[ - f"{cur_dir}/custom_checkpointoutput.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx']) + custom_ops = load( + name="checkpointoutput", + sources=[ + f"{cur_dir}/custom_checkpointoutput.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + ) return custom_ops class TestCheckpointoutput(IPUOpTest): - def setUp(self): self.load_custom_ops() self.set_atol() @@ -66,9 +67,9 @@ class TestCheckpointoutput(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.add(x, x) x = self.op(x, **self.op_attrs) x = paddle.mean(x) diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_leaky_relu_ipu.py b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_leaky_relu_ipu.py index 70d51940865..bb54c0e40da 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_leaky_relu_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_leaky_relu_ipu.py @@ -29,17 +29,18 @@ from op_test_ipu import IPUOpTest, np_dtype_to_fluid_str def load_custom_ops(): # load custom ops cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_jit_ops", - sources=[ - f"{cur_dir}/leaky_relu_cpu.cc", - f"{cur_dir}/leaky_relu_ipu.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx']) + custom_ops = load( + name="custom_jit_ops", + sources=[ + f"{cur_dir}/leaky_relu_cpu.cc", + f"{cur_dir}/leaky_relu_ipu.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + ) return custom_ops class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,8 +50,9 @@ class TestBase(IPUOpTest): def set_feed(self): self.feed = { - "x": np.random.uniform(low=-2, high=2, size=[3, - 5]).astype('float32'), + "x": np.random.uniform(low=-2, high=2, size=[3, 5]).astype( + 'float32' + ), } def set_feed_attr(self): @@ -74,9 +76,11 @@ class TestBase(IPUOpTest): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) # custom op out = custom_ops.custom_leaky_relu(x, **self.attrs) fetch_list = [out.name] @@ -96,14 +100,16 @@ class TestBase(IPUOpTest): # add name mapping for paddle custom op and popart custom ops # `paddle_op` was defined in leaky_relu_cpu.cc # `popart_op`, `domain` and `version` was defined in leaky_relu_ipu.cc - ipu_strategy.add_custom_op(paddle_op="custom_leaky_relu", - popart_op="LeakyRelu", - domain='custom.ops', - version=1) + ipu_strategy.add_custom_op( + paddle_op="custom_leaky_relu", + popart_op="LeakyRelu", + domain='custom.ops', + version=1, + ) program = paddle.static.IpuCompiledProgram( - main_prog, scope=scope, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, scope=scope, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -114,10 +120,9 @@ class TestBase(IPUOpTest): res0 = self._test_base(False) res1 = self._test_base(True) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.assertTrue(res0.shape == res1.shape) diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_ops_ipu.py b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_ops_ipu.py index 744ea86663a..9dc405dca14 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_ops_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_ops_ipu.py @@ -28,56 +28,67 @@ from op_test_ipu import IPUOpTest # just load one custom-op for the data race issue under parallel mode def load_custom_detach(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_detach", - sources=[ - f"{cur_dir}/custom_detach.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], - extra_ldflags=['-lpopfloat']) + custom_ops = load( + name="custom_detach", + sources=[ + f"{cur_dir}/custom_detach.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + extra_ldflags=['-lpopfloat'], + ) return custom_ops def load_custom_identity(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_identity", - sources=[ - f"{cur_dir}/custom_identity.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], - extra_ldflags=['-lpopfloat']) + custom_ops = load( + name="custom_identity", + sources=[ + f"{cur_dir}/custom_identity.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + extra_ldflags=['-lpopfloat'], + ) return custom_ops def load_custom_nll(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_nll", - sources=[ - f"{cur_dir}/custom_nll.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], - extra_ldflags=['-lpopfloat']) + custom_ops = load( + name="custom_nll", + sources=[ + f"{cur_dir}/custom_nll.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + extra_ldflags=['-lpopfloat'], + ) return custom_ops def build_ipu_strategy(): ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.add_custom_op(paddle_op="custom_detach", - popart_op="Detach", - domain="ai.graphcore", - version=1) - ipu_strategy.add_custom_op(paddle_op="custom_identity", - popart_op="Identity", - domain="ai.onnx", - version=11) - ipu_strategy.add_custom_op(paddle_op="custom_nll", - popart_op="Nll", - domain="ai.graphcore", - version=1) + ipu_strategy.add_custom_op( + paddle_op="custom_detach", + popart_op="Detach", + domain="ai.graphcore", + version=1, + ) + ipu_strategy.add_custom_op( + paddle_op="custom_identity", + popart_op="Identity", + domain="ai.onnx", + version=11, + ) + ipu_strategy.add_custom_op( + paddle_op="custom_nll", + popart_op="Nll", + domain="ai.graphcore", + version=1, + ) return ipu_strategy class TestBase(IPUOpTest): - def setUp(self): self.load_custom_ops() self.set_atol() @@ -107,9 +118,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = self.op(x, **self.op_attrs) out = paddle.mean(out) self.fetch_list = [out.name] @@ -127,7 +138,6 @@ class TestBase(IPUOpTest): class TestIdentity(TestBase): - def load_custom_ops(self): self.custom_ops = load_custom_identity() @@ -137,7 +147,6 @@ class TestIdentity(TestBase): class TestNll(TestBase): - def load_custom_ops(self): self.custom_ops = load_custom_nll() @@ -159,12 +168,12 @@ class TestNll(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = self.op(x, label, **self.op_attrs) out = paddle.mean(out) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_data_parallel_ipu.py b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_data_parallel_ipu.py index 891aa501c50..dbeed8e4da5 100644 --- a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_data_parallel_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_data_parallel_ipu.py @@ -27,7 +27,6 @@ mpi_comm = None @unittest.skip('Disable distributed tests on auto CI.') class TestBase(IPUOpTest): - def set_attrs(self, enable_ipu, optimizer, log, onchip=False, rts=False): self.ipu_options = { "enable_pipelining": True, @@ -38,8 +37,8 @@ class TestBase(IPUOpTest): "replicated_graph_count": 2, "location_optimizer": { "on_chip": onchip, - "use_replicated_tensor_sharding": rts - } + "use_replicated_tensor_sharding": rts, + }, } self.cpu_bs = 16 @@ -63,19 +62,17 @@ class TestBase(IPUOpTest): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[bs, 3, 10, 10], - dtype='float32') + image = paddle.static.data( + name='image', shape=[bs, 3, 10, 10], dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0, stage=0): - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) with paddle.static.ipu_shard_guard(index=1, stage=1): - conv2 = paddle.static.nn.conv2d(conv1, - num_filters=3, - filter_size=3, - bias_attr=False) + conv2 = paddle.static.nn.conv2d( + conv1, num_filters=3, filter_size=3, bias_attr=False + ) # should consider influence of bs loss = paddle.mean(conv2) @@ -104,28 +101,35 @@ class TestBase(IPUOpTest): ipu_strategy.set_graph_config( num_ipus=2 * self.ipu_options['replicated_graph_count'], is_training=True, - enable_manual_shard=True) + enable_manual_shard=True, + ) ipu_strategy.set_options(self.ipu_options) - ipu_strategy.set_options({ - "enable_distribution": - True, - "enable_distributed_replicated_graphs": - True, - "global_replica_offset": - int(os.environ.get("PADDLE_TRAINER_ID")) * 2, - "global_replication_factor": - 4 - }) + ipu_strategy.set_options( + { + "enable_distribution": True, + "enable_distributed_replicated_graphs": True, + "global_replica_offset": int( + os.environ.get("PADDLE_TRAINER_ID") + ) + * 2, + "global_replication_factor": 4, + } + ) program = paddle.static.IpuCompiledProgram( - main_prog, ipu_strategy=ipu_strategy).compile( - feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) feed = { - "image": - np.tile(data, [ - self.ipu_options['replicated_graph_count'] * - self.ipu_options['batches_per_step'] * - self.ipu_options['accumulation_factor'], 1, 1, 1 - ]) + "image": np.tile( + data, + [ + self.ipu_options['replicated_graph_count'] + * self.ipu_options['batches_per_step'] + * self.ipu_options['accumulation_factor'], + 1, + 1, + 1, + ], + ) } else: @@ -176,11 +180,13 @@ if __name__ == "__main__": onchip = True if sys.argv[3] == "True" else False rts = True if sys.argv[4] == "True" else False test = TestBase() - test.set_attrs(enable_ipu=True, - optimizer=optimizer, - log=log, - onchip=onchip, - rts=rts) + test.set_attrs( + enable_ipu=True, + optimizer=optimizer, + log=log, + onchip=onchip, + rts=rts, + ) test.test() # Run cpu tests for compare elif len(sys.argv) == 3: diff --git a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_pod128_sample.py b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_pod128_sample.py index f81ed48f04f..a966f430822 100644 --- a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_pod128_sample.py +++ b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_pod128_sample.py @@ -71,43 +71,42 @@ def TestDistTraining(): exe.run(startup_prog) ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=64, - is_training=True, - enable_manual_shard=True) + ipu_strategy.set_graph_config( + num_ipus=64, is_training=True, enable_manual_shard=True + ) ipu_strategy.set_pipelining_config( enable_pipelining=True, batches_per_step=1, enable_gradient_accumulation=True, - accumulation_factor=4) - ipu_strategy.set_options({ - "enable_distribution": - True, - "enable_replicated_graphs": - True, - "replicated_graph_count": - 32, - "enable_distributed_replicated_graphs": - True, - "global_replica_offset": - # Paddle : int(os.environ.get("PADDLE_TRAINER_ID")) * 32 - # PopRun : int(os.environ.get("POPDIST_REPLICA_INDEX_OFFSET")) - int(os.environ.get("PADDLE_TRAINER_ID")) * 32, - "global_replication_factor": - 64, - "location_optimizer": { - "on_chip": False, - "use_replicated_tensor_sharding": True + accumulation_factor=4, + ) + ipu_strategy.set_options( + { + "enable_distribution": True, + "enable_replicated_graphs": True, + "replicated_graph_count": 32, + "enable_distributed_replicated_graphs": True, + "global_replica_offset": + # Paddle : int(os.environ.get("PADDLE_TRAINER_ID")) * 32 + # PopRun : int(os.environ.get("POPDIST_REPLICA_INDEX_OFFSET")) + int(os.environ.get("PADDLE_TRAINER_ID")) * 32, + "global_replication_factor": 64, + "location_optimizer": { + "on_chip": False, + "use_replicated_tensor_sharding": True, + }, } - }) + ) ipu_program = paddle.static.IpuCompiledProgram( - main_prog, ipu_strategy=ipu_strategy) + main_prog, ipu_strategy=ipu_strategy + ) program = ipu_program.compile(feed_list, fetch_list) for i in range(10): - res = exe.run(program, - feed={"x": input_data}, - fetch_list=fetch_list) + res = exe.run( + program, feed={"x": input_data}, fetch_list=fetch_list + ) print("index: {}, result: {}".format(i, res)) diff --git a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_sample.py b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_sample.py index d42977b5962..a3ffc6694a9 100644 --- a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_sample.py +++ b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_sample.py @@ -91,57 +91,73 @@ def Test(use_dist, file_name): if use_dist: ipu_strategy.set_graph_config(num_ipus=2, is_training=True) # Set distributed envs - ipu_strategy.set_options({ - "enable_distribution": - True, - "enable_replicated_graphs": - True, - "replicated_graph_count": - 2, - "enable_distributed_replicated_graphs": - True, - "global_replica_offset": - int(os.environ.get("PADDLE_TRAINER_ID")) * 2, - "global_replication_factor": - 4 - }) + ipu_strategy.set_options( + { + "enable_distribution": True, + "enable_replicated_graphs": True, + "replicated_graph_count": 2, + "enable_distributed_replicated_graphs": True, + "global_replica_offset": int( + os.environ.get("PADDLE_TRAINER_ID") + ) + * 2, + "global_replication_factor": 4, + } + ) else: ipu_strategy.set_graph_config(num_ipus=4, is_training=True) - ipu_strategy.set_options({ - "enable_replicated_graphs": True, - "replicated_graph_count": 4, - }) + ipu_strategy.set_options( + { + "enable_replicated_graphs": True, + "replicated_graph_count": 4, + } + ) ipu_program = paddle.static.IpuCompiledProgram( - main_prog, ipu_strategy=ipu_strategy) + main_prog, ipu_strategy=ipu_strategy + ) program = ipu_program.compile(feed_list, fetch_list) if use_dist: if os.environ.get("PADDLE_TRAINER_ID") == "0": - input_data = np.concatenate([ - np.array([[[1], [3]], [[2], [4]], - [[4], [127]]]).astype(np.int32), - np.array([[[1], [3]], [[2], [4]], - [[4], [127]]]).astype(np.int32) - ]) + input_data = np.concatenate( + [ + np.array( + [[[1], [3]], [[2], [4]], [[4], [127]]] + ).astype(np.int32), + np.array( + [[[1], [3]], [[2], [4]], [[4], [127]]] + ).astype(np.int32), + ] + ) else: - input_data = np.concatenate([ - np.array([[[8], [60]], [[50], [77]], - [[90], [13]]]).astype(np.int32), - np.array([[[8], [60]], [[50], [77]], - [[90], [13]]]).astype(np.int32) - ]) + input_data = np.concatenate( + [ + np.array( + [[[8], [60]], [[50], [77]], [[90], [13]]] + ).astype(np.int32), + np.array( + [[[8], [60]], [[50], [77]], [[90], [13]]] + ).astype(np.int32), + ] + ) else: - input_data = np.concatenate([ - np.array([[[1], [3]], [[2], [4]], - [[4], [127]]]).astype(np.int32), - np.array([[[1], [3]], [[2], [4]], - [[4], [127]]]).astype(np.int32), - np.array([[[8], [60]], [[50], [77]], - [[90], [13]]]).astype(np.int32), - np.array([[[8], [60]], [[50], [77]], - [[90], [13]]]).astype(np.int32) - ]) + input_data = np.concatenate( + [ + np.array([[[1], [3]], [[2], [4]], [[4], [127]]]).astype( + np.int32 + ), + np.array([[[1], [3]], [[2], [4]], [[4], [127]]]).astype( + np.int32 + ), + np.array( + [[[8], [60]], [[50], [77]], [[90], [13]]] + ).astype(np.int32), + np.array( + [[[8], [60]], [[50], [77]], [[90], [13]]] + ).astype(np.int32), + ] + ) feed_data = {"x": input_data} for step in range(10): diff --git a/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py b/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py index 9cb70d59d3e..d518dec3eb7 100644 --- a/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py @@ -55,7 +55,6 @@ class ExecutionMode(IntEnum): class IPUTest(unittest.TestCase): - @classmethod def setUpClass(cls): # Get random seeds @@ -84,10 +83,10 @@ class IPUTest(unittest.TestCase): return True -@unittest.skipIf(not paddle.is_compiled_with_ipu(), - "core is not compiled with IPU") +@unittest.skipIf( + not paddle.is_compiled_with_ipu(), "core is not compiled with IPU" +) class IPUD2STest(IPUTest): - @classmethod def setUpClass(cls): super().setUpClass() @@ -101,11 +100,11 @@ class IPUD2STest(IPUTest): paddle.framework.core.IpuBackend.get_instance().reset() -@unittest.skipIf(not paddle.is_compiled_with_ipu(), - "core is not compiled with IPU") +@unittest.skipIf( + not paddle.is_compiled_with_ipu(), "core is not compiled with IPU" +) class IPUOpTest(IPUTest): - """Base Class for single op unit tests using static graph on IPU. - """ + """Base Class for single op unit tests using static graph on IPU.""" @classmethod def setUpClass(cls): @@ -159,7 +158,6 @@ class IPUOpTest(IPUTest): # Decorator for static graph building def static_graph(builder): - def wrapper(self, *args, **kwargs): self.scope = paddle.static.Scope() self.main_prog = paddle.static.Program() @@ -168,9 +166,11 @@ class IPUOpTest(IPUTest): self.startup_prog.random_seed = self.SEED with paddle.static.scope_guard(self.scope): with paddle.utils.unique_name.guard( - paddle.utils.unique_name.generate('')): - with paddle.static.program_guard(self.main_prog, - self.startup_prog): + paddle.utils.unique_name.generate('') + ): + with paddle.static.program_guard( + self.main_prog, self.startup_prog + ): builder(self, *args, **kwargs) return wrapper @@ -181,11 +181,11 @@ class IPUOpTest(IPUTest): amp_list = paddle.static.amp.CustomOpLists() amp_list.unsupported_list = {'scale'} to_fp16_var_names = paddle.static.amp.cast_model_to_fp16( - main_program, amp_list, use_fp16_guard=False) + main_program, amp_list, use_fp16_guard=False + ) paddle.static.amp.cast_parameters_to_fp16( - paddle.CPUPlace(), - main_program, - to_fp16_var_names=to_fp16_var_names) + paddle.CPUPlace(), main_program, to_fp16_var_names=to_fp16_var_names + ) def run_op_test(self, exec_mode, ipu_strategy=None): # NOTE: some op has no inputs @@ -208,14 +208,12 @@ class IPUOpTest(IPUTest): # TODO(ipu) remove in the future version of popart # keep the log clean, no side effects for tests without profiling ipu_strategy.set_options( - {'engine_options': { - 'debug.retainDebugInformation': 'false' - }}) + {'engine_options': {'debug.retainDebugInformation': 'false'}} + ) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog @@ -226,9 +224,9 @@ class IPUOpTest(IPUTest): if self.is_training: result = [] for _ in range(self.epoch): - loss_res = exe.run(program, - feed=feed, - fetch_list=self.fetch_list) + loss_res = exe.run( + program, feed=feed, fetch_list=self.fetch_list + ) result.append(loss_res) else: result = exe.run(program, feed=feed, fetch_list=self.fetch_list) @@ -253,16 +251,16 @@ class IPUOpTest(IPUTest): for cpu_fp32_res, ipu_fp32_res in zip(cpu_fp32, ipu_fp32): cpu_fp32_res = np.asarray(cpu_fp32_res).astype(np.float32).flatten() ipu_fp32_res = np.asarray(ipu_fp32_res).astype(np.float32).flatten() - pass_check = np.allclose(ipu_fp32_res, - cpu_fp32_res, - rtol=self.rtol, - atol=self.atol) + pass_check = np.allclose( + ipu_fp32_res, cpu_fp32_res, rtol=self.rtol, atol=self.atol + ) if not pass_check: max_atol = np.abs(ipu_fp32_res - cpu_fp32_res).max() cpu_fp32_abs = np.abs(cpu_fp32_res) cpu_fp32_abs[cpu_fp32_abs == 0.0] = 1e-20 - max_rtol = (np.abs(ipu_fp32_res - cpu_fp32_res) / - cpu_fp32_abs).max() + max_rtol = ( + np.abs(ipu_fp32_res - cpu_fp32_res) / cpu_fp32_abs + ).max() raise AssertionError( f"ipu_fp32 check failed. max_atol is {max_atol}, max_rtol is {max_rtol}" ) @@ -274,22 +272,28 @@ class IPUOpTest(IPUTest): ipu_fp16 = output_dict[ExecutionMode.IPU_FP16] if len(cpu_fp32) != len(ipu_fp16): raise ValueError( - "different outputs number between ipu and cpu.") + "different outputs number between ipu and cpu." + ) for cpu_fp32_res, ipu_fp16_res in zip(cpu_fp32, ipu_fp16): - cpu_fp32_res = np.asarray(cpu_fp32_res).astype( - np.float32).flatten() - ipu_fp16_res = np.asarray(ipu_fp16_res).astype( - np.float32).flatten() - pass_check = np.allclose(ipu_fp16_res, - cpu_fp32_res, - rtol=self.rtol_fp16, - atol=self.atol_fp16) + cpu_fp32_res = ( + np.asarray(cpu_fp32_res).astype(np.float32).flatten() + ) + ipu_fp16_res = ( + np.asarray(ipu_fp16_res).astype(np.float32).flatten() + ) + pass_check = np.allclose( + ipu_fp16_res, + cpu_fp32_res, + rtol=self.rtol_fp16, + atol=self.atol_fp16, + ) if not pass_check: max_atol = np.abs(ipu_fp16_res - cpu_fp32_res).max() cpu_fp32_abs = np.abs(cpu_fp32_res) cpu_fp32_abs[cpu_fp32_abs == 0.0] = 1e-20 - max_rtol = (np.abs(ipu_fp16_res - cpu_fp32_res) / - cpu_fp32_abs).max() + max_rtol = ( + np.abs(ipu_fp16_res - cpu_fp32_res) / cpu_fp32_abs + ).max() raise AssertionError( f"ipu_fp16 check failed. max_atol is {max_atol}, max_rtol is {max_rtol}" ) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_activation_ops_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_activation_ops_ipu.py index 97ee7a45e00..672195469d2 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_activation_ops_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_activation_ops_ipu.py @@ -22,7 +22,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_test_op() @@ -47,9 +46,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = self.op(x, **self.op_attrs) self.fetch_list = [out.name] @@ -65,7 +64,6 @@ class TestBase(IPUOpTest): class TestBReluCase0(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[1, 3, 10, 10]) * 30 self.feed_fp32 = {'in_0': data.astype(np.float32)} @@ -78,35 +76,30 @@ class TestBReluCase0(TestBase): class TestBReluCase1(TestBReluCase0): - def set_test_op(self): self.op = paddle.fluid.layers.brelu self.op_attrs = {"t_min": 0.1, 't_max': 10.0} class TestEluCase1(TestBase): - def set_test_op(self): self.op = F.elu self.op_attrs = {"alpha": 0.3} class TestHardShrinkCase0(TestBase): - def set_test_op(self): self.op = F.hardshrink self.op_attrs = {} class TestHardSigmoidCase0(TestBase): - def set_test_op(self): self.op = F.hardsigmoid self.op_attrs = {} class TestHardSigmoidCase1(TestBase): - def set_test_op(self): self.op = F.hardsigmoid self.op_attrs = { @@ -116,126 +109,108 @@ class TestHardSigmoidCase1(TestBase): class TestHardSwishCase0(TestBase): - def set_test_op(self): self.op = F.hardswish self.op_attrs = {} class TestLeakyReluCase0(TestBase): - def set_test_op(self): self.op = F.leaky_relu self.op_attrs = {} class TestLeakyReluCase1(TestBase): - def set_test_op(self): self.op = F.leaky_relu self.op_attrs = {'negative_slope': 0.2333} class TestLog10Case0(TestBase): - def set_test_op(self): self.op = paddle.log10 self.op_attrs = {} class TestLog1pCase0(TestBase): - def set_test_op(self): self.op = paddle.log1p self.op_attrs = {} class TestLog2Case0(TestBase): - def set_test_op(self): self.op = paddle.log2 self.op_attrs = {} class TestLogSigmoidCase0(TestBase): - def set_test_op(self): self.op = F.log_sigmoid self.op_attrs = {} class TestLogSoftmaxCase0(TestBase): - def set_test_op(self): self.op = F.log_softmax self.op_attrs = {} class TestMishCase0(TestBase): - def set_test_op(self): self.op = F.mish self.op_attrs = {} class TestRelu6Case0(TestBase): - def set_test_op(self): self.op = F.relu6 self.op_attrs = {} class TestRsqrtCase0(TestBase): - def set_test_op(self): self.op = paddle.rsqrt self.op_attrs = {} class TestSeluCase0(TestBase): - def set_test_op(self): self.op = F.selu self.op_attrs = {} class TestSiluCase0(TestBase): - def set_test_op(self): self.op = F.silu self.op_attrs = {} class TestSoftShrinkCase0(TestBase): - def set_test_op(self): self.op = F.softshrink self.op_attrs = {} class TestSoftShrinkCase1(TestBase): - def set_test_op(self): self.op = F.softshrink self.op_attrs = {'threshold': 0.2333} class TestSquareCase0(TestBase): - def set_test_op(self): self.op = paddle.square self.op_attrs = {} class TestSwishCase0(TestBase): - def set_test_op(self): self.op = F.swish self.op_attrs = {} class TestTanhShrinkCase0(TestBase): - def set_atol(self): super().set_atol() self.atol = 1e-7 @@ -246,14 +221,12 @@ class TestTanhShrinkCase0(TestBase): class TestThresholdedReluCase0(TestBase): - def set_test_op(self): self.op = F.thresholded_relu self.op_attrs = {} class TestThresholdedReluCase1(TestBase): - def set_test_op(self): self.op = F.thresholded_relu self.op_attrs = {'threshold': 0.2333} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_affine_channel_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_affine_channel_op_ipu.py index 836b99099ff..0e76225c561 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_affine_channel_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_affine_channel_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,16 +47,18 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - data = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + data = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) input_scale = paddle.fluid.layers.create_parameter( - shape=[self.feed_shape[0][1]], dtype="float32") + shape=[self.feed_shape[0][1]], dtype="float32" + ) input_bias = paddle.fluid.layers.create_parameter( - shape=[self.feed_shape[0][1]], dtype="float32") - out = paddle.fluid.layers.affine_channel(data, - scale=input_scale, - bias=input_bias) + shape=[self.feed_shape[0][1]], dtype="float32" + ) + out = paddle.fluid.layers.affine_channel( + data, scale=input_scale, bias=input_bias + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -72,7 +73,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 4, 64, 64]) self.feed_fp32 = {'data': data.astype(np.float32)} @@ -81,7 +81,6 @@ class TestCase1(TestBase): @unittest.skip("Only support NCHW") class TestNHWC(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['data_layout'] = 'NHWC' diff --git a/python/paddle/fluid/tests/unittests/ipu/test_arg_max_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_arg_max_op_ipu.py index 078e744ae50..181ae2bfbc7 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_arg_max_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_arg_max_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.argmax(x, **self.attrs) self.fetch_list = [out.name] @@ -64,7 +63,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 0} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_arg_min_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_arg_min_op_ipu.py index 30c604901e8..9c1de175daf 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_arg_min_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_arg_min_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.argmin(x, **self.attrs) self.fetch_list = [out.name] @@ -64,7 +63,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 0} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_argsort_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_argsort_op_ipu.py index 3f19da43c71..0f6c28c82cd 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_argsort_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_argsort_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -47,9 +46,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out, _ = paddle.fluid.layers.argsort(x, **self.attrs) self.fetch_list = [out.name] @@ -67,7 +66,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'axis': 0, @@ -76,7 +74,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { 'axis': 1, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_assign_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_assign_op_ipu.py index 93cdaf018b4..08b40fc23c8 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_assign_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_assign_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -39,9 +38,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.assign(x) out = paddle.fluid.layers.elementwise_add(x, x) self.fetch_list = [out.name] @@ -58,7 +57,6 @@ class TestBase(IPUOpTest): class TestAssignFp32Value(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 3, 1]) self.feed_fp32 = {'in_0': data.astype(np.float32)} @@ -69,16 +67,15 @@ class TestAssignFp32Value(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) assign = paddle.assign(self.assign_fp32) out = paddle.fluid.layers.elementwise_add(x, assign) self.fetch_list = [out.name] class TestAssignBoolValue(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 3, 1]) self.feed_fp32 = {'in_0': data.astype(np.float32)} @@ -88,9 +85,9 @@ class TestAssignBoolValue(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.less_than(x, x) assign = paddle.assign(self.assign_bool) x = paddle.logical_and(x, assign) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_avg_shard_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_avg_shard_ipu.py index a3be5458ad8..1834104606e 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_avg_shard_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_avg_shard_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,25 +44,21 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) self.fetch_list = [x.name] def run_model(self, exec_mode): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_batch_norm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_batch_norm_op_ipu.py index 08e5049a790..f8a45e0bd2a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_batch_norm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_batch_norm_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -52,13 +51,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) x = paddle.fluid.layers.batch_norm(x, **self.attrs) self.fetch_list = [x.name] @@ -74,7 +72,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_atol(self): self.atol = 1e-6 self.rtol = 1e-6 @@ -89,7 +86,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_atol(self): self.atol = 1e-6 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_binary_cross_entropy_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_binary_cross_entropy_op_ipu.py index 113412b8341..caf28126d22 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_binary_cross_entropy_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_binary_cross_entropy_op_ipu.py @@ -22,7 +22,6 @@ import paddle.nn.functional as F class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -35,11 +34,11 @@ class TestBase(IPUOpTest): target = np.random.uniform(size=[3, 4, 2, 2]) self.feed_fp32 = { "x": x.astype(np.float32), - "target": target.astype(np.float32) + "target": target.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), - "target": target.astype(np.float16) + "target": target.astype(np.float16), } def set_feed_attr(self): @@ -53,12 +52,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - target = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + target = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = F.binary_cross_entropy(x, target, **self.attrs) self.fetch_list = [out.name] @@ -74,7 +73,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'reduction': 'sum', @@ -82,7 +80,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { 'reduction': 'none', diff --git a/python/paddle/fluid/tests/unittests/ipu/test_bmm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_bmm_op_ipu.py index 8ea20cebf07..dd1d99a8748 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_bmm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_bmm_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,12 +44,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.bmm(x, y, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_cast_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_cast_op_ipu.py index 675489d1e4c..45b191afed1 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_cast_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_cast_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,9 +47,11 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) out = paddle.cast(x, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +67,6 @@ class TestBase(IPUOpTest): class TestEnableFp16(TestBase): - @property def fp16_enabled(self): return True @@ -85,7 +85,6 @@ class TestEnableFp16(TestBase): class TestCase2(TestBase): - def set_atol(self): super().set_atol() self.atol = 1e-3 @@ -102,7 +101,6 @@ class TestCase2(TestBase): class TestCase3(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('float32'), @@ -114,7 +112,6 @@ class TestCase3(TestBase): class TestCase4(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('int32'), @@ -126,7 +123,6 @@ class TestCase4(TestBase): class TestCase5(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('float16'), @@ -138,7 +134,6 @@ class TestCase5(TestBase): class TestCase6(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('int32'), @@ -151,7 +146,6 @@ class TestCase6(TestBase): @unittest.skip('float64 is not supported') class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['dtype'] = 'float64' @@ -159,7 +153,6 @@ class TestCase7(TestBase): @unittest.skip('skip float16 to float32') class TestCase8(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('float16'), @@ -172,16 +165,15 @@ class TestCase8(TestBase): @unittest.skip('int32 to int8 is not supported') class TestCase9(TestBase): - def set_atol(self): super().set_atol() self.atol = 1 def set_data_feed(self): self.feed_fp32 = { - "x": - np.random.randint(low=1, high=100, size=[1, 3, 3, - 3]).astype('int32'), + "x": np.random.randint(low=1, high=100, size=[1, 3, 3, 3]).astype( + 'int32' + ), } def set_op_attrs(self): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_clip_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_clip_op_ipu.py index a221ad61767..a570024eb70 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_clip_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_clip_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,9 +47,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.clip(x, **self.attrs) self.fetch_list = [x.name] @@ -66,111 +65,96 @@ class TestBase(IPUOpTest): class TestNoMin(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['max'] = 3.4 class TestNoMax(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['min'] = 0.1 class TestNoMinNoMax(TestBase): - def set_op_attrs(self): self.attrs = {} class TestMinMaxTensor(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - min = paddle.fluid.layers.fill_constant(name="min", - shape=[1], - dtype='float32', - value=0.1) - max = paddle.fluid.layers.fill_constant(name="max", - shape=[1], - dtype='float32', - value=3.4) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + min = paddle.fluid.layers.fill_constant( + name="min", shape=[1], dtype='float32', value=0.1 + ) + max = paddle.fluid.layers.fill_constant( + name="max", shape=[1], dtype='float32', value=3.4 + ) x = paddle.clip(x, min=min, max=max) self.fetch_list = [x.name] class TestMinTensor(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - min = paddle.fluid.layers.fill_constant(name="min", - shape=[1], - dtype='float32', - value=0.1) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + min = paddle.fluid.layers.fill_constant( + name="min", shape=[1], dtype='float32', value=0.1 + ) x = paddle.clip(x, min=min) self.fetch_list = [x.name] class TestMaxTensor(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - max = paddle.fluid.layers.fill_constant(name="max", - shape=[1], - dtype='float32', - value=3.4) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + max = paddle.fluid.layers.fill_constant( + name="max", shape=[1], dtype='float32', value=3.4 + ) x = paddle.clip(x, max=max) self.fetch_list = [x.name] class TestCombine1(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - min = paddle.fluid.layers.fill_constant(name="min", - shape=[1], - dtype='float32', - value=0.1) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + min = paddle.fluid.layers.fill_constant( + name="min", shape=[1], dtype='float32', value=0.1 + ) x = paddle.clip(x, min=min, max=3.4) self.fetch_list = [x.name] class TestCombine2(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - max = paddle.fluid.layers.fill_constant(name="max", - shape=[1], - dtype='float32', - value=3.4) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + max = paddle.fluid.layers.fill_constant( + name="max", shape=[1], dtype='float32', value=3.4 + ) x = paddle.clip(x, min=0.1, max=max) self.fetch_list = [x.name] class TestIntInput(TestBase): - def set_feed(self): data = np.random.uniform(size=[5, 5]) self.feed_fp32 = {'x': data.astype(np.int32)} @@ -180,16 +164,15 @@ class TestIntInput(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int32' + ) x = paddle.clip(x, min=0.1, max=3.4) self.fetch_list = [x.name] class TestIntMinMax(TestBase): - def set_feed(self): data = np.random.uniform(size=[5, 5]) self.feed_fp32 = {'x': data.astype(np.int32)} @@ -199,17 +182,15 @@ class TestIntMinMax(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int32') - min = paddle.fluid.layers.fill_constant(name="min", - shape=[1], - dtype='int32', - value=1) - max = paddle.fluid.layers.fill_constant(name="max", - shape=[1], - dtype='int32', - value=3) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int32' + ) + min = paddle.fluid.layers.fill_constant( + name="min", shape=[1], dtype='int32', value=1 + ) + max = paddle.fluid.layers.fill_constant( + name="max", shape=[1], dtype='int32', value=3 + ) x = paddle.clip(x, min=min, max=max) self.fetch_list = [x.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_concat_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_concat_op_ipu.py index 733a5291cf5..588cae77fec 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_concat_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_concat_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ class TestBase(IPUOpTest): data2 = np.random.uniform(size=[1, 3, 10, 10]) self.feed_fp32 = { 'x': data1.astype(np.float32), - 'y': data2.astype(np.float32) + 'y': data2.astype(np.float32), } self.feed_fp16 = { 'x': data1.astype(np.float16), - 'y': data2.astype(np.float16) + 'y': data2.astype(np.float16), } def set_feed_attr(self): @@ -50,12 +49,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.concat([x, y], **self.attrs) self.fetch_list = [out.name] @@ -71,7 +70,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 1} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_conv2d_transpose_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_conv2d_transpose_op_ipu.py index 824b6b628de..af1e74bfd12 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_conv2d_transpose_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_conv2d_transpose_op_ipu.py @@ -21,7 +21,6 @@ from op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -52,9 +51,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.static.nn.conv2d_transpose(x, **self.attrs) self.fetch_list = [x.name] @@ -70,7 +69,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['stride'] = 2 @@ -78,7 +76,6 @@ class TestCase1(TestBase): @unittest.skip("Only support dilation=1") class TestCase2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['stride'] = 2 @@ -86,21 +83,18 @@ class TestCase2(TestBase): class TestCase3(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = 2 class TestCase4(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = "SAME" class TestCase5(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['stride'] = 2 @@ -108,14 +102,12 @@ class TestCase5(TestBase): class TestCase6(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = "VALID" class TestCase7(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = "VALID" @@ -123,7 +115,6 @@ class TestCase7(TestBase): class TestCase8(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['filter_size'] = 4 @@ -149,7 +140,6 @@ class TestCase10(TestBase): class TestCase11(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['groups'] = 3 @@ -157,17 +147,16 @@ class TestCase11(TestBase): # depthwise_conv2d_transpose Op class TestCase12(TestBase): - def set_feed(self): data = np.random.uniform(size=[1, 3, 10, 10]) weight = np.random.uniform(size=[3, 1, 3, 3]) self.feed_fp32 = { 'in_0': data.astype(np.float32), - 'in_1': weight.astype(np.float32) + 'in_1': weight.astype(np.float32), } self.feed_fp16 = { 'in_0': data.astype(np.float16), - 'in_1': weight.astype(np.float16) + 'in_1': weight.astype(np.float16), } self.feed_shape = [x.shape for x in self.feed_fp32.values()] self.feed_list = list(self.feed_fp32.keys()) @@ -178,12 +167,12 @@ class TestCase12(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - weight = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + weight = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) x = paddle.nn.functional.conv2d_transpose(x, weight, **self.attrs) self.fetch_list = [x.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_conv_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_conv_op_ipu.py index 076d51a5e67..89474001e78 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_conv_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_conv_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -53,9 +52,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.fluid.layers.conv2d(x, **self.attrs) self.fetch_list = [x.name] @@ -71,35 +70,30 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['num_filters'] = 1 class TestCase2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['filter_size'] = [3, 3] class TestCase2_1(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['filter_size'] = [3, 2] class TestCase3(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['stride'] = [2, 3] class TestCase4(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['dilation'] = [2, 2] @@ -113,21 +107,18 @@ class TestCase5(TestBase): class TestCase6(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = 2 class TestCase7(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = [2, 3] class TestCase8(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = [1, 2, 2, 3] @@ -135,17 +126,16 @@ class TestCase8(TestBase): # depthwise_conv2d Op class TestCase9(TestBase): - def set_feed(self): data = np.random.uniform(size=[1, 3, 10, 10]) weight = np.random.uniform(size=[3, 1, 3, 3]) self.feed_fp32 = { 'in_0': data.astype(np.float32), - 'in_1': weight.astype(np.float32) + 'in_1': weight.astype(np.float32), } self.feed_fp16 = { 'in_0': data.astype(np.float16), - 'in_1': weight.astype(np.float16) + 'in_1': weight.astype(np.float16), } self.feed_shape = [x.shape for x in self.feed_fp32.values()] self.feed_list = list(self.feed_fp32.keys()) @@ -156,12 +146,12 @@ class TestCase9(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - weight = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + weight = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) x = paddle.nn.functional.conv2d(x, weight, **self.attrs) self.fetch_list = [x.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_cross_entropy2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_cross_entropy2_op_ipu.py index 92cf442fe27..b7db271775d 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_cross_entropy2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_cross_entropy2_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ class TestBase(IPUOpTest): label = np.arange(3).reshape([3, 1]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } def set_feed_attr(self): @@ -52,20 +51,20 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) if on_ipu: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) else: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int64') - out = paddle.fluid.layers.cross_entropy(input=x, - label=label, - **self.attrs) + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int64' + ) + out = paddle.fluid.layers.cross_entropy( + input=x, label=label, **self.attrs + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -82,7 +81,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -91,23 +89,21 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[30, 70]) label = np.arange(30).reshape([30, 1]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } @unittest.skip("soft_label=True is not supported") class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = { 'soft_label': True, @@ -115,32 +111,30 @@ class TestCase3(TestBase): class TestCase4(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 5, 7]) label = np.random.randint(0, 7, [3, 5, 1], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 5, 6, 7]) label = np.random.randint(0, 7, [3, 5, 6], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_cumsum_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_cumsum_op_ipu.py index 5f859b064fe..d31a78e1230 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_cumsum_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_cumsum_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,9 +48,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) out = paddle.fluid.layers.cumsum(x, **self.attrs) self.fetch_list = [out.name] @@ -67,49 +66,44 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"exclusive": True, "reverse": False} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"exclusive": False, "reverse": True} class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"exclusive": True, "reverse": True} class TestCase4(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 128]) self.feed_fp32 = {"x": x.astype(np.int32)} @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="int32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="int32" + ) out = paddle.fluid.layers.cumsum(x, **self.attrs) self.fetch_list = [out.name] class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 128]) self.feed_fp32 = {"x": x.astype(np.int64)} @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="int64") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="int64" + ) out = paddle.fluid.layers.cumsum(x, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_data_norm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_data_norm_op_ipu.py index de84b94bb7d..9013a56ae1c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_data_norm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_data_norm_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,9 +39,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.static.nn.data_norm(input=x, **self.attrs) self.fetch_list = [x.name] @@ -58,15 +57,14 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"in_place": True} @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.static.nn.data_norm(input=x, **self.attrs) x = x + 1 self.fetch_list = [x.name] @@ -74,25 +72,21 @@ class TestCase1(TestBase): @unittest.skip("Do not support in_place=True when test single data_norm Op") class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"in_place": True} class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"data_layout": "NHWC"} class TestCase4(TestBase): - def set_op_attrs(self): self.attrs = {"epsilon": 0.001} class TestCase5(TestBase): - def set_op_attrs(self): self.attrs = {"do_model_average_for_mean_and_var": True} @@ -101,16 +95,12 @@ class TestCase6(TestBase): # If enable_scale_and_shift=True, it requires to set values of scale and bias in `param_attr` def set_op_attrs(self): self.attrs = { - "param_attr": { - "scale_w": 0.5, - "bias": 0.1 - }, - "enable_scale_and_shift": True + "param_attr": {"scale_w": 0.5, "bias": 0.1}, + "enable_scale_and_shift": True, } class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = { "param_attr": { @@ -118,9 +108,9 @@ class TestCase7(TestBase): "batch_sum": 0.1, "batch_square": 1e3, "scale_w": 0.5, - "bias": 0.1 + "bias": 0.1, }, - "enable_scale_and_shift": True + "enable_scale_and_shift": True, } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dist_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dist_op_ipu.py index 5f8db4faba7..a029f8a287b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dist_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dist_op_ipu.py @@ -21,7 +21,6 @@ from op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ class TestBase(IPUOpTest): data_y = np.random.uniform(size=[7, 1, 5]) self.feed_fp32 = { "x": data_x.astype(np.float32), - "y": data_y.astype(np.float32) + "y": data_y.astype(np.float32), } self.feed_fp16 = { "x": data_x.astype(np.float16), - "y": data_y.astype(np.float16) + "y": data_y.astype(np.float16), } def set_feed_attr(self): @@ -51,12 +50,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.dist(x, y, **self.attrs) self.fetch_list = [out.name] @@ -72,19 +71,16 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"p": 0} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"p": float("inf")} class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"p": float("-inf")} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dot_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dot_op_ipu.py index ed0c36f53eb..cf32eb09f5a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dot_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dot_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,12 +44,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.dot(x, y, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[6]) y = np.random.uniform(size=[6]) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dropout_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dropout_op_ipu.py index d104b39c292..d6b3813a6e6 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dropout_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dropout_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -42,14 +41,14 @@ class TestBase(IPUOpTest): self.attrs = { "dropout_prob": 0.5, "is_test": True, - "dropout_implementation": "downgrade_in_infer" + "dropout_implementation": "downgrade_in_infer", } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.fluid.layers.dropout(x, **self.attrs) out = paddle.fluid.layers.elementwise_add(x, x) self.fetch_list = [out.name] @@ -66,22 +65,20 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "dropout_prob": 0.5, "is_test": True, - "dropout_implementation": "upscale_in_train" + "dropout_implementation": "upscale_in_train", } class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "dropout_prob": 0.0, "is_test": False, - "dropout_implementation": "upscale_in_train" + "dropout_implementation": "upscale_in_train", } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dy2static_fp16_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dy2static_fp16_ipu.py index 098f3d6b2ba..3450978d427 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dy2static_fp16_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dy2static_fp16_ipu.py @@ -21,14 +21,12 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUD2STest class SimpleLayer(paddle.nn.Layer): - def __init__(self, use_ipu=False): super(SimpleLayer, self).__init__() self.use_ipu = use_ipu - self.conv = paddle.nn.Conv2D(in_channels=3, - out_channels=1, - kernel_size=2, - stride=1) + self.conv = paddle.nn.Conv2D( + in_channels=3, out_channels=1, kernel_size=2, stride=1 + ) def forward(self, x, target=None): x = self.conv(x) @@ -45,7 +43,6 @@ class SimpleLayer(paddle.nn.Layer): class TestBase(IPUD2STest): - def setUp(self): super().setUp() self.save_path = tempfile.TemporaryDirectory() @@ -59,28 +56,33 @@ class TestBase(IPUD2STest): np.random.seed(self.SEED) model = SimpleLayer(use_ipu) specs = [ - paddle.static.InputSpec(name="x", - shape=[32, 3, 10, 10], - dtype="float32"), + paddle.static.InputSpec( + name="x", shape=[32, 3, 10, 10], dtype="float32" + ), paddle.static.InputSpec(name="target", shape=[32], dtype="int64"), ] model = paddle.jit.to_static(model, input_spec=specs) - optim = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) data = paddle.uniform((32, 3, 10, 10), dtype='float32') label = paddle.randint(0, 10, shape=[32], dtype='int64') model_path = '{}/model_state_dict_{}.pdparams'.format( - self.save_path, 'ipu' if use_ipu else 'cpu') + self.save_path, 'ipu' if use_ipu else 'cpu' + ) optim_path = '{}/optim_state_dict_{}.pdopt'.format( - self.save_path, 'ipu' if use_ipu else 'cpu') + self.save_path, 'ipu' if use_ipu else 'cpu' + ) if use_ipu: paddle.set_device('ipu') ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False) + ipu_strategy.set_graph_config( + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ) ipu_strategy.set_precision_config(enable_fp16=True) ipu_strategy.set_optimizer(optim) data = data.astype(np.float16) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dy2static_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dy2static_ipu.py index 5a7d0836d6f..d8dd65646bf 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dy2static_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dy2static_ipu.py @@ -17,7 +17,9 @@ import unittest import numpy as np import paddle -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramCache +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramCache, +) from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUD2STest from paddle.jit import to_static from paddle.optimizer.lr import LRScheduler @@ -25,18 +27,18 @@ from functools import partial class SimpleLayer(paddle.nn.Layer): - - def __init__(self, - loss_op=None, - use_softmax=True, - use_reduction=True, - use_identity_loss=True): + def __init__( + self, + loss_op=None, + use_softmax=True, + use_reduction=True, + use_identity_loss=True, + ): super(SimpleLayer, self).__init__() self.loss_op = loss_op - self.conv = paddle.nn.Conv2D(in_channels=3, - out_channels=1, - kernel_size=2, - stride=1) + self.conv = paddle.nn.Conv2D( + in_channels=3, out_channels=1, kernel_size=2, stride=1 + ) self.use_softmax = use_softmax self.use_reduction = use_reduction self.use_identity_loss = use_identity_loss @@ -61,7 +63,6 @@ class SimpleLayer(paddle.nn.Layer): class TestBase(IPUD2STest): - def setUp(self): self.set_op_attrs() self.set_data_feed() @@ -74,25 +75,30 @@ class TestBase(IPUD2STest): self.label = paddle.randint(0, 10, shape=[8], dtype='int64') def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=not use_ipu, - use_identity_loss=use_ipu) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=not use_ipu, + use_identity_loss=use_ipu, + ) def _test(self, use_ipu=False): paddle.seed(self.SEED) np.random.seed(self.SEED) model = self.create_model(use_ipu) - optim = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) if use_ipu: paddle.set_device('ipu') ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False) + ipu_strategy.set_graph_config( + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ) ipu_strategy.set_optimizer(optim) epochs = 100 @@ -118,7 +124,6 @@ class TestBase(IPUD2STest): class TestSaveLoad(TestBase): - def setUp(self): super().setUp() self.save_path = tempfile.TemporaryDirectory() @@ -131,20 +136,25 @@ class TestSaveLoad(TestBase): paddle.seed(self.SEED) np.random.seed(self.SEED) model = self.create_model(use_ipu) - optim = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) model_path = '{}/model_state_dict_{}.pdparams'.format( - self.save_path, 'ipu' if use_ipu else 'cpu') + self.save_path, 'ipu' if use_ipu else 'cpu' + ) optim_path = '{}/optim_state_dict_{}.pdopt'.format( - self.save_path, 'ipu' if use_ipu else 'cpu') + self.save_path, 'ipu' if use_ipu else 'cpu' + ) if use_ipu: paddle.set_device('ipu') ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False) + ipu_strategy.set_graph_config( + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ) ipu_strategy.set_optimizer(optim) epochs = 100 @@ -182,7 +192,6 @@ class TestSaveLoad(TestBase): class TestPatch(IPUD2STest): - def setUp(cls): paddle.disable_static() @@ -201,16 +210,16 @@ class TestPatch(IPUD2STest): class TestWithoutIdentityLoss1(TestBase): - def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=True, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=True, + use_identity_loss=False, + ) class TestWithoutIdentityLoss2(TestBase): - def set_op_attrs(self): self.loss_op = paddle.fluid.layers.softmax_with_cross_entropy @@ -219,14 +228,15 @@ class TestWithoutIdentityLoss2(TestBase): self.label = paddle.randint(0, 10, shape=[8, 1], dtype='int64') def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=False, - use_reduction=True, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=False, + use_reduction=True, + use_identity_loss=False, + ) class TestWithoutIdentityLoss3(TestBase): - def set_op_attrs(self): self.loss_op = partial(paddle.fluid.layers.kldiv_loss, reduction="none") @@ -235,14 +245,15 @@ class TestWithoutIdentityLoss3(TestBase): self.label = paddle.rand(shape=[8, 81], dtype='float32') def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=True, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=True, + use_identity_loss=False, + ) class TestWithoutIdentityLoss4(TestBase): - def set_op_attrs(self): self.loss_op = paddle.nn.functional.binary_cross_entropy @@ -251,27 +262,31 @@ class TestWithoutIdentityLoss4(TestBase): self.label = paddle.rand(shape=[8, 81], dtype='float32') def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=False, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=False, + use_identity_loss=False, + ) class TestWithoutIdentityLoss5(TestBase): - def set_op_attrs(self): self.loss_op = paddle.fluid.layers.sigmoid_cross_entropy_with_logits def set_data_feed(self): self.data = paddle.uniform((8, 3, 10, 10), dtype='float32') - self.label = paddle.randint(0, 10, shape=[8, 81], - dtype='int64').astype('float32') + self.label = paddle.randint(0, 10, shape=[8, 81], dtype='int64').astype( + 'float32' + ) def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=True, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=True, + use_identity_loss=False, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_elemetwise_x_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_elemetwise_x_op_ipu.py index 9c35e43970e..8dcb3097c2f 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_elemetwise_x_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_elemetwise_x_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestMul(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,12 +42,12 @@ class TestMul(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = self.op(x, y, **self.attrs) self.fetch_list = [out.name] @@ -125,43 +124,36 @@ class TestMul(IPUOpTest): class TestAdd(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_add class TestSub(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_sub class TestDiv(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_div class TestMin(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_min class TestMax(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_max class TestPow(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_pow class TestMod(TestMul): - def set_atol(self): self.atol = 1e-7 self.rtol = 1e-5 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_equal_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_equal_op_ipu.py index 77a78a7cb78..e0ae7fa2467 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_equal_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_equal_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,12 +49,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.equal(x, y, **self.attrs) self.fetch_list = [out.name] @@ -71,7 +70,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): x = np.ones([1, 10]) y = np.ones([1, 10]) @@ -80,7 +78,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_data_feed(self): x = np.ones([1, 10]) y = np.arange(0, 10).reshape([1, 10]) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_eval_model_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_eval_model_ipu.py index f3b1d227d95..335a78c5982 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_eval_model_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_eval_model_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -57,25 +56,27 @@ class TestBase(IPUOpTest): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) weight_decay = self.attrs['weight_decay'] - opt = paddle.optimizer.SGD(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.SGD( + learning_rate=1e-1, weight_decay=weight_decay + ) if self.attrs['optimizer'] == 'adam': - opt = paddle.optimizer.Adam(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.Adam( + learning_rate=1e-1, weight_decay=weight_decay + ) elif self.attrs['optimizer'] == 'lamb': - opt = paddle.optimizer.Lamb(learning_rate=1e-1, - lamb_weight_decay=weight_decay) + opt = paddle.optimizer.Lamb( + learning_rate=1e-1, lamb_weight_decay=weight_decay + ) opt.minimize(loss) if run_ipu: @@ -92,8 +93,8 @@ class TestBase(IPUOpTest): ipu_strategy.set_graph_config(is_training=True) ipu_strategy.set_options({"runtime_options.enable_eval": True}) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -102,16 +103,17 @@ class TestBase(IPUOpTest): for epoch in range(200): if epoch == 100: ipu_strategy.set_options( - {"runtime_options.enable_eval": False}) - loss_res = exe.run(program, - feed=self.feed, - fetch_list=[loss]) + {"runtime_options.enable_eval": False} + ) + loss_res = exe.run( + program, feed=self.feed, fetch_list=[loss] + ) result.append(loss_res) else: for epoch in range(100): - loss_res = exe.run(program, - feed=self.feed, - fetch_list=[loss]) + loss_res = exe.run( + program, feed=self.feed, fetch_list=[loss] + ) result.append(loss_res) return np.array(result) @@ -120,10 +122,9 @@ class TestBase(IPUOpTest): ipu_loss = self._test_optimizer(True).flatten() cpu_loss = self._test_optimizer(False).flatten() self.assertTrue(ipu_loss[0] == ipu_loss[99]) - np.testing.assert_allclose(ipu_loss[100:], - cpu_loss, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + ipu_loss[100:], cpu_loss, rtol=1e-05, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_expand_as_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_expand_as_v2_op_ipu.py index ee68eba5e54..b66e1a6c5bd 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_expand_as_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_expand_as_v2_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -33,11 +32,11 @@ class TestBase(IPUOpTest): data_y = np.random.uniform(size=[2, 2, 3]) self.feed_fp32 = { 'x': data_x.astype(np.float32), - 'y': data_y.astype(np.float32) + 'y': data_y.astype(np.float32), } self.feed_fp16 = { 'x': data_x.astype(np.float16), - 'y': data_y.astype(np.float16) + 'y': data_y.astype(np.float16), } def set_feed_attr(self): @@ -47,12 +46,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype="float32" + ) out = paddle.expand_as(x, y) self.fetch_list = [out.name] @@ -68,33 +67,31 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): data_x = np.random.uniform(size=[2, 3]) data_y = np.random.uniform(size=[2, 4, 2, 3]) self.feed_fp32 = { 'x': data_x.astype(np.float32), - 'y': data_y.astype(np.float32) + 'y': data_y.astype(np.float32), } self.feed_fp16 = { 'x': data_x.astype(np.float16), - 'y': data_y.astype(np.float16) + 'y': data_y.astype(np.float16), } @unittest.skip("corresponding dimensions must have the same value.") class TestCase2(TestBase): - def set_data_feed(self): data_x = np.random.uniform(size=[2, 3]) data_y = np.random.uniform(size=[2, 4, 3, 3]) self.feed_fp32 = { 'x': data_x.astype(np.float32), - 'y': data_y.astype(np.float32) + 'y': data_y.astype(np.float32), } self.feed_fp16 = { 'x': data_x.astype(np.float16), - 'y': data_y.astype(np.float16) + 'y': data_y.astype(np.float16), } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_expand_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_expand_op_ipu.py index 843ec0438d7..784a6a41a41 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_expand_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_expand_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) out = paddle.fluid.layers.expand(x, **self.attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[2, 2]) self.feed_fp32 = {"x": x.astype(np.float32)} @@ -78,14 +76,15 @@ class TestCase1(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) expand_times = paddle.fluid.layers.fill_constant( - shape=[len(self.feed_shape[0])], dtype="int32", value=2) - out = paddle.fluid.layers.expand(x, - expand_times=expand_times, - **self.attrs) + shape=[len(self.feed_shape[0])], dtype="int32", value=2 + ) + out = paddle.fluid.layers.expand( + x, expand_times=expand_times, **self.attrs + ) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_expand_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_expand_v2_op_ipu.py index 5cb949a1943..cb98ed3253f 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_expand_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_expand_v2_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) out = paddle.expand(x, **self.attrs) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_attrs(self): self.attrs = {"shape": [5, 2, 2, 3]} class TestCase2(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 1, 3]) self.feed_fp32 = {'x': data.astype(np.float32)} @@ -80,14 +77,12 @@ class TestCase2(TestBase): @unittest.skip("corresponding dimensions must have the same value.") class TestCase3(TestBase): - def set_attrs(self): self.attrs = {"shape": [5, 2, 4, 3]} @unittest.skip("Do not support `shape` = Tensors.") class TestCase4(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[3, 3]) self.feed_fp32 = {'x': data.astype(np.float32)} @@ -95,9 +90,9 @@ class TestCase4(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) self.attrs = { 'name': 'y', 'shape': [3], diff --git a/python/paddle/fluid/tests/unittests/ipu/test_fill_any_like_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_fill_any_like_op_ipu.py index 74ecba6f18c..ae43689dfb4 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_fill_any_like_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_fill_any_like_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x_fill = paddle.full_like(x, **self.attrs) out = paddle.fluid.layers.elementwise_add(x_fill, x_fill) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {'fill_value': 3, 'dtype': 'int32'} class TestError(TestBase): - @IPUOpTest.static_graph def build_model(self): x = paddle.fluid.data('x', [-1, 3, 13], 'float32') @@ -82,8 +79,9 @@ class TestError(TestBase): def test_error(): self.run_op_test(IPUOpTest.ExecutionMode.IPU_FP32) - self.assertRaisesRegex(Exception, "Please check tensor shape setting", - test_error) + self.assertRaisesRegex( + Exception, "Please check tensor shape setting", test_error + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_fill_constant_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_fill_constant_op_ipu.py index 0e1e17cb65e..71dfb2d4097 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_fill_constant_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_fill_constant_op_ipu.py @@ -20,7 +20,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -63,7 +62,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'name': 'x', diff --git a/python/paddle/fluid/tests/unittests/ipu/test_flatten_contiguous_range_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_flatten_contiguous_range_op_ipu.py index 4723f753fb6..11bebbe36f7 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_flatten_contiguous_range_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_flatten_contiguous_range_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,9 +44,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.flatten(x=x, **self.attrs) self.fetch_list = [out.name] @@ -63,7 +62,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['start_axis'] = 0 @@ -71,7 +69,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['start_axis'] = 1 @@ -79,7 +76,6 @@ class TestCase2(TestBase): class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['start_axis'] = 1 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_flatten_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_flatten_op_ipu.py index d7c1da14e29..f363e40c5fd 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_flatten_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_flatten_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.flatten(x=x, **self.attrs) self.fetch_list = [out.name] @@ -62,14 +61,12 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['axis'] = 0 class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['axis'] = 2 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_flip_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_flip_op_ipu.py index 07e0acb60a1..abdb9d07b3a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_flip_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_flip_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -51,9 +50,11 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) x = paddle.flip(x, **self.attrs) self.fetch_list = [x.name] @@ -69,7 +70,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_feed(self): data = np.random.randint(0, 10, size=[3, 2, 2]) self.feed_fp32 = {'x': data.astype(np.int32)} @@ -77,7 +77,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_feed(self): data = np.random.randint(0, 2, size=[4, 3, 2, 2]) self.feed_fp32 = {'x': data.astype(np.bool)} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_fp16_support_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_fp16_support_ipu.py index 708dd0f4054..8433e45f46c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_fp16_support_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_fp16_support_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,22 +49,19 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - conv2 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + conv2 = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) add1 = conv1 + conv2 - conv3 = paddle.static.nn.conv2d(add1, - num_filters=8, - filter_size=8, - bias_attr=False) + conv3 = paddle.static.nn.conv2d( + add1, num_filters=8, filter_size=8, bias_attr=False + ) out = paddle.fluid.layers.relu(conv3, **self.attrs) self.fetch_list = [out.name] @@ -81,7 +77,6 @@ class TestBase(IPUOpTest): class TestIntInput(TestBase): - def set_data_feed(self): embedding = np.random.uniform(size=[10, 20]) indice = np.array([1, 3, 5]).astype(np.int32) @@ -96,12 +91,12 @@ class TestIntInput(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = paddle.fluid.layers.gather(x, index=y) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_gather_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_gather_op_ipu.py index 13a48e5a98f..5913db81f31 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_gather_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_gather_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,12 +43,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = paddle.fluid.layers.gather(x, index=y, **self.attrs) self.fetch_list = [out.name] @@ -65,7 +64,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[100]) y = np.array([1, 3, 5]) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_gelu_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_gelu_op_ipu.py index 2d14621d5fc..031fd257772 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_gelu_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_gelu_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.gelu(x, **self.attrs) self.fetch_list = [out.name] @@ -61,7 +60,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_atol(self): self.atol = 1e-10 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_gradient_clip_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_gradient_clip_ipu.py index b63d176ff27..d8b1aaf4c4a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_gradient_clip_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_gradient_clip_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -60,13 +59,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - image = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) self.fetch_list = [loss.name] @@ -74,20 +72,23 @@ class TestBase(IPUOpTest): # Only support ClipGradByGlobalNorm clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) if self.attrs['optimizer'] == 'sgd': - opt = paddle.optimizer.SGD(learning_rate=1e-1, - weight_decay=weight_decay, - grad_clip=clip) + opt = paddle.optimizer.SGD( + learning_rate=1e-1, weight_decay=weight_decay, grad_clip=clip + ) elif self.attrs['optimizer'] == 'adam': - opt = paddle.optimizer.Adam(learning_rate=1e-1, - weight_decay=weight_decay, - grad_clip=clip) + opt = paddle.optimizer.Adam( + learning_rate=1e-1, weight_decay=weight_decay, grad_clip=clip + ) elif self.attrs['optimizer'] == 'lamb': - opt = paddle.optimizer.Lamb(learning_rate=1e-1, - lamb_weight_decay=weight_decay, - grad_clip=clip) + opt = paddle.optimizer.Lamb( + learning_rate=1e-1, + lamb_weight_decay=weight_decay, + grad_clip=clip, + ) else: raise ValueError( - f"Not supported optimizer {self.attrs['optimizer']} for test") + f"Not supported optimizer {self.attrs['optimizer']} for test" + ) opt.minimize(loss) def run_model(self, exec_mode): @@ -102,7 +103,6 @@ class TestBase(IPUOpTest): class TestAdam(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -111,7 +111,6 @@ class TestAdam(TestBase): class TestLamb(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', diff --git a/python/paddle/fluid/tests/unittests/ipu/test_greater_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_greater_op_ipu.py index 4f2e9a1a94b..c48117dd401 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_greater_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_greater_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestGreaterThan(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -35,12 +34,12 @@ class TestGreaterThan(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = self.op(x, y, **self.attrs) self.fetch_list = [out.name] @@ -114,25 +113,21 @@ class TestGreaterThan(IPUOpTest): class TestLessThan(TestGreaterThan): - def set_test_op(self): self.op = paddle.fluid.layers.less_than class TestEqual(TestGreaterThan): - def set_test_op(self): self.op = paddle.fluid.layers.equal class TestGreaterEqual(TestGreaterThan): - def set_test_op(self): self.op = paddle.fluid.layers.greater_equal class TestLessEqual(TestGreaterThan): - def set_test_op(self): self.op = paddle.fluid.layers.less_equal diff --git a/python/paddle/fluid/tests/unittests/ipu/test_groupnorm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_groupnorm_op_ipu.py index 70f1a6a28e9..fbffe321b55 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_groupnorm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_groupnorm_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -53,30 +52,27 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if self.is_training: ch = self.feed_shape[0][1] - conv1 = paddle.static.nn.conv2d(x, - num_filters=ch, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + x, num_filters=ch, filter_size=3, bias_attr=False + ) scale = paddle.ParamAttr(trainable=True) bias = paddle.ParamAttr(trainable=True) - out = paddle.fluid.layers.nn.group_norm(conv1, - param_attr=scale, - bias_attr=bias, - **self.attrs) + out = paddle.fluid.layers.nn.group_norm( + conv1, param_attr=scale, bias_attr=bias, **self.attrs + ) loss = paddle.mean(out) adam = paddle.optimizer.Adam(learning_rate=1e-2) adam.minimize(loss) self.fetch_list = [loss.name] else: - out = paddle.fluid.layers.nn.group_norm(x, - param_attr=True, - bias_attr=True, - **self.attrs) + out = paddle.fluid.layers.nn.group_norm( + x, param_attr=True, bias_attr=True, **self.attrs + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -91,7 +87,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "groups": 4, @@ -101,7 +96,6 @@ class TestCase1(TestBase): class TestTrainCase1(TestBase): - def set_training(self): self.is_training = True self.epoch = 20 @@ -109,7 +103,6 @@ class TestTrainCase1(TestBase): @unittest.skipIf(IPUOpTest.use_ipumodel(), "skip for ipumodel") class TestTrainCase2(TestBase): - def set_atol(self): self.atol = 7e-4 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_huber_loss_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_huber_loss_op_ipu.py index 613fbd944c2..5030e368083 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_huber_loss_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_huber_loss_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ class TestBase(IPUOpTest): target = np.random.uniform(size=[3, 4, 2, 2]) self.feed_fp32 = { "x": x.astype(np.float32), - "target": target.astype(np.float32) + "target": target.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), - "target": target.astype(np.float16) + "target": target.astype(np.float16), } def set_feed_attr(self): @@ -52,12 +51,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - target = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + target = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.huber_loss(x, target, **self.attrs) self.fetch_list = [out.name] @@ -73,7 +72,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'delta': 0.5, @@ -81,7 +79,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { 'delta': 0.0, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_identity_loss_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_identity_loss_ipu.py index 53baf0de0ac..8e2d2110cf1 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_identity_loss_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_identity_loss_ipu.py @@ -20,14 +20,15 @@ import paddle.fluid as fluid import paddle.fluid.compiler as compiler import paddle.optimizer import paddle.static -from paddle.fluid.tests.unittests.ipu.op_test_ipu import (IPUOpTest, - np_dtype_to_fluid_str) +from paddle.fluid.tests.unittests.ipu.op_test_ipu import ( + IPUOpTest, + np_dtype_to_fluid_str, +) paddle.enable_static() class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -41,8 +42,9 @@ class TestBase(IPUOpTest): def set_feed(self): self.feed = { - "x": np.random.uniform(low=-2, high=2, size=[3, - 5]).astype('float32'), + "x": np.random.uniform(low=-2, high=2, size=[3, 5]).astype( + 'float32' + ), } def set_feed_attr(self): @@ -62,9 +64,11 @@ class TestBase(IPUOpTest): with fluid.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) out = self.op(x, reduction) fetch_list = [out.name] @@ -77,7 +81,8 @@ class TestBase(IPUOpTest): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(num_ipus=1, is_training=False) ipu_compiler = compiler.IpuCompiledProgram( - main_prog, ipu_strategy=ipu_strategy) + main_prog, ipu_strategy=ipu_strategy + ) program = ipu_compiler.compile(feed_list, fetch_list) ipu_res = exe.run(program, self.feed, fetch_list) @@ -92,10 +97,9 @@ class TestBase(IPUOpTest): # none cpu_res = self.feed['x'] - np.testing.assert_allclose(ipu_res[0], - cpu_res, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + ipu_res[0], cpu_res, rtol=1e-05, atol=self.atol + ) def test_base(self): # TODO: use string instead of int for reduction diff --git a/python/paddle/fluid/tests/unittests/ipu/test_inference_model_io_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_inference_model_io_ipu.py index 4a1aa8cba44..c370debfc4b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_inference_model_io_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_inference_model_io_ipu.py @@ -22,7 +22,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -60,19 +59,24 @@ class TestBase(IPUOpTest): startup_prog.random_seed = self.SEED generator = paddle.fluid.unique_name.UniqueNameGenerator() self.full_name = '/'.join( - [self.attrs['path'].name, self.attrs['model_name']]) + [self.attrs['path'].name, self.attrs['model_name']] + ) with paddle.fluid.unique_name.guard(generator): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False, - name='conv2d') + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype='float32', + ) + conv1 = paddle.static.nn.conv2d( + x, + num_filters=3, + filter_size=3, + bias_attr=False, + name='conv2d', + ) loss = paddle.mean(conv1) if self.attrs['is_training']: @@ -93,24 +97,22 @@ class TestBase(IPUOpTest): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config( - is_training=self.attrs['is_training']) + is_training=self.attrs['is_training'] + ) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, fetch_list) result = [] for i in range(self.attrs['steps']): - tmp = exe.run(program, - feed=self.feed, - fetch_list=fetch_list) + tmp = exe.run( + program, feed=self.feed, fetch_list=fetch_list + ) result.append(tmp) - paddle.static.save_inference_model(self.full_name, - x, - loss, - exe, - program=program.org_program) + paddle.static.save_inference_model( + self.full_name, x, loss, exe, program=program.org_program + ) def _test_load(self, run_ipu): if run_ipu: @@ -119,8 +121,11 @@ class TestBase(IPUOpTest): place = paddle.CPUPlace() exe = paddle.static.Executor(place) - [inference_program, feed_target_names, fetch_targets - ] = (paddle.static.load_inference_model(self.full_name, exe)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(self.full_name, exe) if run_ipu: feed_list = feed_target_names @@ -128,8 +133,8 @@ class TestBase(IPUOpTest): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=False) program = paddle.static.IpuCompiledProgram( - inference_program, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + inference_program, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = inference_program @@ -147,7 +152,6 @@ class TestBase(IPUOpTest): class TestAdam(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['steps'] = 100 @@ -158,7 +162,6 @@ class TestAdam(TestBase): class TestLamb(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['steps'] = 100 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_instancenorm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_instancenorm_op_ipu.py index b24e4be7ae7..3479899ebda 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_instancenorm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_instancenorm_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,31 +48,28 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if self.is_training: ch = self.feed_shape[0][1] - conv1 = paddle.static.nn.conv2d(x, - num_filters=ch, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + x, num_filters=ch, filter_size=3, bias_attr=False + ) scale = paddle.ParamAttr(trainable=True) bias = paddle.ParamAttr(trainable=True) - out = paddle.fluid.layers.nn.instance_norm(conv1, - param_attr=scale, - bias_attr=bias, - **self.attrs) + out = paddle.fluid.layers.nn.instance_norm( + conv1, param_attr=scale, bias_attr=bias, **self.attrs + ) loss = paddle.mean(out) adam = paddle.optimizer.Adam(learning_rate=1e-2) adam.minimize(loss) self.fetch_list = [loss.name] else: - out = paddle.fluid.layers.nn.instance_norm(x, - param_attr=True, - bias_attr=True, - **self.attrs) + out = paddle.fluid.layers.nn.instance_norm( + x, param_attr=True, bias_attr=True, **self.attrs + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -88,7 +84,6 @@ class TestBase(IPUOpTest): class TestTrainCase1(TestBase): - def set_training(self): self.is_training = True self.epoch = 10 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_interpolate_ops_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_interpolate_ops_ipu.py index 108e953659d..e31320585db 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_interpolate_ops_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_interpolate_ops_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,9 +44,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) out = paddle.nn.functional.interpolate(x, **self.attrs) self.fetch_list = [out.name] @@ -63,14 +62,12 @@ class TestBase(IPUOpTest): class TestCase0(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs["size"] = [3, 4] class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs["scale_factor"] = [2, 1] @@ -78,19 +75,16 @@ class TestCase1(TestBase): @unittest.skip("Only one of size or scale_factor should be defined") class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"size": [12, 12], "scale_factor": [2, 1]} class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"scale_factor": 2.5} class TestBilinear(TestBase): - @property def fp16_enabled(self): return False @@ -107,7 +101,6 @@ class TestBilinear(TestBase): # Take long time class TestBicubic(TestBase): - @property def fp16_enabled(self): return False @@ -124,7 +117,6 @@ class TestBicubic(TestBase): # Trilinear requires 5-D input class TestTrilinear(TestBase): - @property def fp16_enabled(self): return False @@ -144,13 +136,12 @@ class TestTrilinear(TestBase): self.attrs = { "size": [12, 12, 12], "mode": "trilinear", - "data_format": "NCDHW" + "data_format": "NCDHW", } # Linear requires 3-D input class TestLinear(TestBase): - @property def fp16_enabled(self): return False @@ -171,9 +162,9 @@ class TestLinear(TestBase): @unittest.skip( - "Transfer to Pool Op with 2-D ksize, now we only support 1-D ksize.") + "Transfer to Pool Op with 2-D ksize, now we only support 1-D ksize." +) class TestArea(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[2, 3, 6, 6]) self.feed_fp32 = {"x": x.astype(np.float32)} @@ -185,7 +176,6 @@ class TestArea(TestBase): # align_corners option can only be set with the interpolating modes: linear | bilinear | bicubic | trilinear class TestAlignCorners(TestBase): - @property def fp16_enabled(self): return False @@ -194,13 +184,12 @@ class TestAlignCorners(TestBase): self.attrs = { "size": [12, 12], "align_corners": True, - "mode": "bilinear" + "mode": "bilinear", } # class TestAlignMode(TestBase): - def set_op_attrs(self): self.attrs = {"size": [12, 12], "align_mode": 1} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_ipu_shard_api_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_ipu_shard_api_ipu.py index f04a3bb55e5..bb430fe35a9 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_ipu_shard_api_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_ipu_shard_api_ipu.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestIpuShard(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() @@ -59,14 +58,12 @@ class TestIpuShard(unittest.TestCase): def test_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [1, 2, 3, 1, 2, 1, 2] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestIpuPipeline(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() @@ -104,10 +101,9 @@ class TestIpuPipeline(unittest.TestCase): ipu_index_list = self._test() expected_ipu_index_list = [1, 2, 3, 1, 2, 1, 2] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_ipu_strategy_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_ipu_strategy_ipu.py index ed4b258b754..bd00f9c9a65 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_ipu_strategy_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_ipu_strategy_ipu.py @@ -21,13 +21,13 @@ paddle.enable_static() class TestIpuStrategy(unittest.TestCase): - def test_set_options(self): ipu_strategy = paddle.static.IpuStrategy() all_option_names = ipu_strategy._ipu_strategy.get_all_option_names() skip_options = [] skip_options.append( - 'mean_accumulation_and_replication_reduction_strategy') + 'mean_accumulation_and_replication_reduction_strategy' + ) skip_options.append('random_seed') for option_name in all_option_names: @@ -48,7 +48,9 @@ class TestIpuStrategy(unittest.TestCase): try: ipu_strategy.set_options({option_name: set_value}) new_value = ipu_strategy.get_option(option_name) - assert new_value == set_value, f"set {option_name} to {set_value} failed" + assert ( + new_value == set_value + ), f"set {option_name} to {set_value} failed" except: raise Exception(f"set {option_name} to {set_value} failed") @@ -70,21 +72,24 @@ class TestIpuStrategy(unittest.TestCase): options['engine_options'] = { 'debug.allowOutOfMemory': 'true', 'autoReport.directory': 'path', - 'autoReport.all': 'true' + 'autoReport.all': 'true', } options['random_seed'] = 1234 for k, v in options.items(): ipu_strategy.set_options({k: v}) - if (isinstance(v, list)): - assert v.sort() == ipu_strategy.get_option( - k).sort(), f"set {k} to {v} failed " + if isinstance(v, list): + assert ( + v.sort() == ipu_strategy.get_option(k).sort() + ), f"set {k} to {v} failed " else: assert v == ipu_strategy.get_option( - k), f"set {k} to {v} failed " + k + ), f"set {k} to {v} failed " # The custom logger need 2 int as inputs logger = lambda progress, total: print( - f"compile progrss: {progress}/{total}") + f"compile progrss: {progress}/{total}" + ) ipu_strategy.set_options({'compilation_progress_logger': logger}) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_kldiv_loss_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_kldiv_loss_op_ipu.py index ef3e173fded..d08df5399d9 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_kldiv_loss_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_kldiv_loss_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ class TestBase(IPUOpTest): target = np.random.uniform(size=[3, 4, 2, 2]) self.feed_fp32 = { "x": x.astype(np.float32), - "target": target.astype(np.float32) + "target": target.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), - "target": target.astype(np.float16) + "target": target.astype(np.float16), } def set_feed_attr(self): @@ -52,12 +51,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - target = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + target = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.kldiv_loss(x, target, **self.attrs) self.fetch_list = [out.name] @@ -73,7 +72,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'reduction': 'sum', @@ -81,7 +79,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { 'reduction': 'none', diff --git a/python/paddle/fluid/tests/unittests/ipu/test_layernorm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_layernorm_op_ipu.py index 9bf457d6f92..b6ca7c34714 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_layernorm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_layernorm_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -56,30 +55,27 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if self.is_training: ch = self.feed_shape[0][1] - conv1 = paddle.static.nn.conv2d(x, - num_filters=ch, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + x, num_filters=ch, filter_size=3, bias_attr=False + ) scale = paddle.ParamAttr(trainable=True) bias = paddle.ParamAttr(trainable=True) - out = paddle.fluid.layers.nn.layer_norm(conv1, - param_attr=scale, - bias_attr=bias, - **self.attrs) + out = paddle.fluid.layers.nn.layer_norm( + conv1, param_attr=scale, bias_attr=bias, **self.attrs + ) loss = paddle.mean(out) self.fetch_list = [loss.name] else: scale = self.attrs['scale'] bias = self.attrs['shift'] - out = paddle.fluid.layers.nn.layer_norm(x, - param_attr=scale, - bias_attr=bias, - **self.attrs) + out = paddle.fluid.layers.nn.layer_norm( + x, param_attr=scale, bias_attr=bias, **self.attrs + ) self.fetch_list = [out.name] if self.is_training: @@ -89,8 +85,9 @@ class TestBase(IPUOpTest): elif self.optimizer == 'adam': optimizer = paddle.optimizer.Adam(learning_rate=1e-2) elif self.optimizer == 'lamb': - optimizer = paddle.optimizer.Lamb(learning_rate=1e-2, - lamb_weight_decay=0.0) + optimizer = paddle.optimizer.Lamb( + learning_rate=1e-2, lamb_weight_decay=0.0 + ) if optimizer is not None: optimizer.minimize(loss) @@ -107,7 +104,6 @@ class TestBase(IPUOpTest): @unittest.skip('raise error') class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "scale": False, @@ -119,7 +115,6 @@ class TestCase1(TestBase): @unittest.skip('raise error') class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "scale": True, @@ -130,7 +125,6 @@ class TestCase2(TestBase): class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = { "scale": True, @@ -142,13 +136,12 @@ class TestCase3(TestBase): class TestTrainCase1(TestBase): - def set_op_attrs(self): self.attrs = { "scale": True, "shift": True, "begin_norm_axis": 1, - "epsilon": 1e-05 + "epsilon": 1e-05, } self.optimizer = 'sgd' @@ -162,7 +155,6 @@ class TestTrainCase1(TestBase): class TestTrainCase3(TestBase): - def set_atol(self): super().set_atol() self.atol = 5e-3 @@ -172,7 +164,7 @@ class TestTrainCase3(TestBase): "scale": True, "shift": True, "begin_norm_axis": 2, - "epsilon": 1e-05 + "epsilon": 1e-05, } self.optimizer = 'lamb' diff --git a/python/paddle/fluid/tests/unittests/ipu/test_logical_not_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_logical_not_op_ipu.py index d8eaa2f81bc..c75f6faa65b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_logical_not_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_logical_not_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,9 +39,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="bool") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="bool" + ) out = paddle.fluid.layers.logical_not(x) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_logical_x_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_logical_x_op_ipu.py index 79c22f47da5..a2d3817f7f5 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_logical_x_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_logical_x_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestLogicalAnd(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -39,12 +38,16 @@ class TestLogicalAnd(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype=self.feed_dtype[1]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) + y = paddle.static.data( + name=self.feed_list[1], + shape=self.feed_shape[1], + dtype=self.feed_dtype[1], + ) out = self.op(x, y, **self.attrs) self.fetch_list = [out.name] @@ -79,7 +82,6 @@ class TestLogicalAnd(IPUOpTest): class TestLogicalOr(TestLogicalAnd): - def set_test_op(self): self.op = paddle.fluid.layers.logical_or diff --git a/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_op_ipu.py index ffcf8a64f53..b2aa05fb734 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,14 +44,14 @@ class TestBase(IPUOpTest): "is_sparse": False, "is_distributed": False, "padding_idx": -1, - "dtype": 'float32' + "dtype": 'float32', } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int64') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int64' + ) out = paddle.fluid.layers.embedding(x, **self.attrs) if self.is_training: loss = paddle.mean(out) @@ -76,7 +75,6 @@ class TestBase(IPUOpTest): class TestTrainCase1(TestBase): - def set_atol(self): self.atol = 1e-7 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_v2_op_ipu.py index 2c8e7159cf2..4d8ca5bb037 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_v2_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,14 +44,14 @@ class TestBase(IPUOpTest): "embedding_dim": 16, "sparse": False, "padding_idx": -1, - "weight_attr": None + "weight_attr": None, } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int64') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int64' + ) embedding = paddle.nn.Embedding(**self.attrs) out = embedding(x) if self.is_training: @@ -77,7 +76,6 @@ class TestBase(IPUOpTest): class TestTrainCase1(TestBase): - def set_atol(self): self.atol = 1e-7 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_lr_sheduler_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_lr_sheduler_ipu.py index 137f3f61c53..7fffe224458 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_lr_sheduler_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_lr_sheduler_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class LR_New(LRScheduler): - def __init__(self, learning_rate=1e-5, last_epoch=-1, verbose=False): super(LR_New, self).__init__(learning_rate, last_epoch, verbose) @@ -32,16 +31,14 @@ class LR_New(LRScheduler): class TestConvNet(IPUOpTest): - @IPUOpTest.static_graph def build_model(self): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) opt = paddle.optimizer.Lamb(learning_rate=LR_New()) @@ -61,9 +58,8 @@ class TestConvNet(IPUOpTest): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=True) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog @@ -71,9 +67,9 @@ class TestConvNet(IPUOpTest): for _ in range(100): if hasattr(program, "lr_sheduler"): program.lr_sheduler.step() - loss_res = exe.run(program, - feed=self.feed, - fetch_list=self.fetch_list) + loss_res = exe.run( + program, feed=self.feed, fetch_list=self.fetch_list + ) result.append(loss_res) return np.array(result) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_margin_rank_loss_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_margin_rank_loss_op_ipu.py index e9964156a12..5861009fd85 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_margin_rank_loss_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_margin_rank_loss_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -55,15 +54,15 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self, on_ipu): - label = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - left = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') - right = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='float32') + label = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + left = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) + right = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='float32' + ) out = paddle.fluid.layers.margin_rank_loss(label, left, right) self.fetch_list = [out.name] @@ -79,7 +78,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'margin': 0.5, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_matmul_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_matmul_op_ipu.py index bf2af886959..ad53cfbae9c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_matmul_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_matmul_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,12 +48,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.matmul(x, y, **self.attrs) self.fetch_list = [out.name] @@ -71,7 +70,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, @@ -81,7 +79,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, @@ -97,7 +94,6 @@ class TestCase2(TestBase): class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[5, 4, 3, 2]) y = np.random.uniform(size=[5, 4, 2, 3]) @@ -107,7 +103,6 @@ class TestCase3(TestBase): class TestCase4(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 3, 2]) y = np.random.uniform(size=[4, 2, 3]) @@ -117,7 +112,6 @@ class TestCase4(TestBase): class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 2, 3]) y = np.random.uniform(size=[3, 2]) @@ -127,7 +121,6 @@ class TestCase5(TestBase): class TestCase6(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3]) @@ -137,7 +130,6 @@ class TestCase6(TestBase): @unittest.skip("not supported") class TestCase6_2(TestCase6): - def set_data_feed(self): x = np.random.uniform(size=[3]) @@ -153,7 +145,6 @@ class TestCase6_2(TestCase6): class TestCase7(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 3, 4, 5]) y = np.random.uniform(size=[1, 3, 4, 5]) @@ -166,7 +157,6 @@ class TestCase7(TestBase): class TestCase8(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 1]) y = np.random.uniform(size=[1, 2]) @@ -177,7 +167,6 @@ class TestCase8(TestBase): @unittest.skip("not supported") class TestCase8_2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3]) y = np.random.uniform(size=[2]) @@ -195,7 +184,6 @@ class TestCase8_2(TestBase): @unittest.skip("dim > 4 is not supported") class TestCase9(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[6, 5, 4, 2, 3]) @@ -204,7 +192,6 @@ class TestCase9(TestBase): class TestCase10(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_y": True, @@ -219,7 +206,6 @@ class TestCase10(TestBase): class TestCase11(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_matmul_serilize_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_matmul_serilize_ipu.py index 21aa7c4b992..6ee930f2475 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_matmul_serilize_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_matmul_serilize_ipu.py @@ -27,7 +27,6 @@ def set_serialize_factor(serialize_factor): class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -51,12 +50,16 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype=self.feed_dtype[1]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) + y = paddle.static.data( + name=self.feed_list[1], + shape=self.feed_shape[1], + dtype=self.feed_dtype[1], + ) # decrator maybe the best choice, but need to modify api out = paddle.matmul(x, y, **self.attrs) set_serialize_factor(4) @@ -75,8 +78,8 @@ class TestBase(IPUOpTest): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=self.is_training) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, self.fetch_list) else: program = self.main_prog result = exe.run(program, feed=self.feed, fetch_list=self.fetch_list) @@ -85,10 +88,9 @@ class TestBase(IPUOpTest): def test_base(self): res0 = self.run_model(False) res1 = self.run_model(True) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.assertTrue(res0.shape == res1.shape) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_matmul_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_matmul_v2_op_ipu.py index 37f575f64bd..c71395ed272 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_matmul_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_matmul_v2_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,12 +44,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.matmul(x, y, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, @@ -75,7 +73,6 @@ class TestCase1(TestBase): class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[5, 4, 2, 3]) y = np.random.uniform(size=[5, 4, 3, 2]) @@ -85,7 +82,6 @@ class TestCase3(TestBase): class TestCase4(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 2, 3]) y = np.random.uniform(size=[4, 3, 2]) @@ -95,7 +91,6 @@ class TestCase4(TestBase): class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 2, 3]) y = np.random.uniform(size=[3, 2]) @@ -105,7 +100,6 @@ class TestCase5(TestBase): class TestCase6(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3]) y = np.random.uniform(size=[3]) @@ -116,7 +110,6 @@ class TestCase6(TestBase): @unittest.skip("not supported") class TestCase6_2(TestCase6): - def set_data_feed(self): x = np.random.uniform(size=[3]) y = np.random.uniform(size=[3]) @@ -129,7 +122,6 @@ class TestCase6_2(TestCase6): class TestCase7(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 1]) y = np.random.uniform(size=[1, 2]) @@ -140,7 +132,6 @@ class TestCase7(TestBase): @unittest.skip("dim > 4 is not supported") class TestCase8(TestBase): - def set_data_feed(self): self.feed = { "x": np.random.uniform(size=[6, 5, 4, 2, 3]).astype('float32'), @@ -149,7 +140,6 @@ class TestCase8(TestBase): class TestCase9(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_y": True, @@ -164,7 +154,6 @@ class TestCase9(TestBase): class TestCase10(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_mean_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_mean_op_ipu.py index 0f60ed2485e..7c6b1e29738 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_mean_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_mean_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.mean(x) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_meshgrid_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_meshgrid_op_ipu.py index 56242fea367..d23f0eb58ef 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_meshgrid_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_meshgrid_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,11 +39,11 @@ class TestBase(IPUOpTest): data2 = np.random.uniform(size=[20]) self.feed_fp32 = { 'x': data1.astype(np.float32), - 'y': data2.astype(np.float32) + 'y': data2.astype(np.float32), } self.feed_fp16 = { 'x': data1.astype(np.float16), - 'y': data2.astype(np.float16) + 'y': data2.astype(np.float16), } def set_feed_attr(self): @@ -58,12 +57,16 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype=self.feed_dtype[1]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) + y = paddle.static.data( + name=self.feed_list[1], + shape=self.feed_shape[1], + dtype=self.feed_dtype[1], + ) r1, r2 = paddle.meshgrid(x, y) self.fetch_list = [r1.name, r2.name] @@ -81,7 +84,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_feed(self): data1 = np.random.uniform(size=[10]) data2 = np.random.uniform(size=[20]) @@ -89,41 +91,46 @@ class TestCase1(TestBase): self.feed_fp32 = { 'x': data1.astype(np.float32), 'y': data2.astype(np.float32), - 'z': data3.astype(np.float32) + 'z': data3.astype(np.float32), } self.feed_fp16 = { 'x': data1.astype(np.float16), 'y': data2.astype(np.float16), - 'z': data3.astype(np.float16) + 'z': data3.astype(np.float16), } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype=self.feed_dtype[1]) - z = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype=self.feed_dtype[2]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) + y = paddle.static.data( + name=self.feed_list[1], + shape=self.feed_shape[1], + dtype=self.feed_dtype[1], + ) + z = paddle.static.data( + name=self.feed_list[2], + shape=self.feed_shape[2], + dtype=self.feed_dtype[2], + ) r1, r2, r3 = paddle.meshgrid(x, y, z) self.fetch_list = [r1.name, r2.name, r3.name] class TestCase2(TestBase): - def set_feed(self): data1 = np.random.uniform(size=[100]) data2 = np.random.uniform(size=[200]) self.feed_fp32 = { 'x': data1.astype(np.int32), - 'y': data2.astype(np.int32) + 'y': data2.astype(np.int32), } self.feed_fp16 = { 'x': data1.astype(np.int32), - 'y': data2.astype(np.int32) + 'y': data2.astype(np.int32), } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_inference_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_inference_ipu.py index 7118466a521..ab62ec4e276 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_inference_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_inference_ipu.py @@ -22,7 +22,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -51,9 +50,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) # using fp32 x = paddle.static.nn.conv2d(input=x, num_filters=3, filter_size=3) @@ -77,7 +76,8 @@ class TestBase(IPUOpTest): amp_list = paddle.static.amp.CustomOpLists() amp_list.unsupported_list = {} to_fp16_var_names = paddle.static.amp.cast_model_to_fp16( - self.main_prog, amp_list, use_fp16_guard=True) + self.main_prog, amp_list, use_fp16_guard=True + ) if self.is_ipu_mode(exec_mode): place = paddle.CPUPlace() @@ -91,27 +91,29 @@ class TestBase(IPUOpTest): paddle.static.amp.cast_parameters_to_fp16( paddle.CPUPlace(), self.main_prog, - to_fp16_var_names=to_fp16_var_names) + to_fp16_var_names=to_fp16_var_names, + ) if self.is_ipu_mode(exec_mode): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config( is_training=False, num_ipus=self.num_ipus, - enable_manual_shard=self.enable_manual_shard) + enable_manual_shard=self.enable_manual_shard, + ) ipu_strategy.set_pipelining_config( enable_pipelining=self.enable_pipelining, - batches_per_step=self.batches_per_step) + batches_per_step=self.batches_per_step, + ) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog - result = exe.run(program, - feed=self.feed_fp32, - fetch_list=self.fetch_list) + result = exe.run( + program, feed=self.feed_fp32, fetch_list=self.fetch_list + ) self.output_dict[exec_mode] = result[0] def test(self): @@ -122,15 +124,14 @@ class TestBase(IPUOpTest): class TestPipline(TestBase): - @IPUOpTest.static_graph def build_model(self, exec_mode): feed_shape = list(self.feed_shape[0]) if self.is_ipu_mode(exec_mode): feed_shape[0] = 1 - x = paddle.static.data(name=self.feed_list[0], - shape=feed_shape, - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=feed_shape, dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0, stage=0): # using fp32 x = paddle.static.nn.conv2d(input=x, num_filters=3, filter_size=3) @@ -140,9 +141,9 @@ class TestPipline(TestBase): with paddle.static.ipu_shard_guard(index=1, stage=1): # using fp16 with paddle.static.amp.fp16_guard(): - x = paddle.static.nn.conv2d(input=x, - num_filters=6, - filter_size=3) + x = paddle.static.nn.conv2d( + input=x, num_filters=6, filter_size=3 + ) x = paddle.static.nn.batch_norm(x, act='relu') x = F.max_pool2d(x, kernel_size=2, stride=2) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_training_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_training_ipu.py index 51a0e91a29c..0ea8d5557f2 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_training_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_training_ipu.py @@ -22,7 +22,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -57,9 +56,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) # using fp32 x = paddle.static.nn.conv2d(input=x, num_filters=3, filter_size=3) @@ -87,7 +86,8 @@ class TestBase(IPUOpTest): amp_list = paddle.static.amp.CustomOpLists() amp_list.unsupported_list = {} to_fp16_var_names = paddle.static.amp.cast_model_to_fp16( - self.main_prog, amp_list) + self.main_prog, amp_list + ) if self.is_ipu_mode(exec_mode): place = paddle.CPUPlace() @@ -101,29 +101,31 @@ class TestBase(IPUOpTest): paddle.static.amp.cast_parameters_to_fp16( paddle.CPUPlace(), self.main_prog, - to_fp16_var_names=to_fp16_var_names) + to_fp16_var_names=to_fp16_var_names, + ) if self.is_ipu_mode(exec_mode): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config( is_training=self.is_training, num_ipus=self.num_ipus, - enable_manual_shard=self.enable_manual_shard) + enable_manual_shard=self.enable_manual_shard, + ) ipu_strategy.set_pipelining_config( enable_pipelining=self.enable_pipelining, - batches_per_step=self.batches_per_step) + batches_per_step=self.batches_per_step, + ) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog result = [] for _ in range(self.epoch): - out = exe.run(program, - feed=self.feed_fp32, - fetch_list=self.fetch_list) + out = exe.run( + program, feed=self.feed_fp32, fetch_list=self.fetch_list + ) result.append(out) self.output_dict[exec_mode] = result @@ -135,15 +137,14 @@ class TestBase(IPUOpTest): class TestPipline(TestBase): - @IPUOpTest.static_graph def build_model(self, exec_mode): feed_shape = list(self.feed_shape[0]) if self.is_ipu_mode(exec_mode): feed_shape[0] = 1 - x = paddle.static.data(name=self.feed_list[0], - shape=feed_shape, - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=feed_shape, dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0, stage=0): # using fp32 @@ -154,9 +155,9 @@ class TestPipline(TestBase): with paddle.static.ipu_shard_guard(index=1, stage=1): # using fp16 with paddle.static.amp.fp16_guard(): - x = paddle.static.nn.conv2d(input=x, - num_filters=6, - filter_size=3) + x = paddle.static.nn.conv2d( + input=x, num_filters=6, filter_size=3 + ) x = paddle.static.nn.batch_norm(x, act='relu') x = F.max_pool2d(x, kernel_size=2, stride=2) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_model_parallel_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_model_parallel_ipu.py index 55a6569b7dd..2592d0ddee5 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_model_parallel_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_model_parallel_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -59,19 +58,17 @@ class TestBase(IPUOpTest): bs = self.ipu_bs if run_ipu else self.cpu_bs with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[bs, 3, 10, 10], - dtype='float32') + image = paddle.static.data( + name='image', shape=[bs, 3, 10, 10], dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0): - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) with paddle.static.ipu_shard_guard(index=1): - conv2 = paddle.static.nn.conv2d(conv1, - num_filters=3, - filter_size=3, - bias_attr=False) + conv2 = paddle.static.nn.conv2d( + conv1, num_filters=3, filter_size=3, bias_attr=False + ) # should consider influence of bs loss = paddle.mean(conv2) @@ -101,11 +98,12 @@ class TestBase(IPUOpTest): ipu_strategy.set_graph_config( num_ipus=2 * self.ipu_options['replicated_graph_count'], is_training=self.is_training, - enable_manual_shard=True) + enable_manual_shard=True, + ) ipu_strategy.set_options(self.ipu_options) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -126,14 +124,12 @@ class TestBase(IPUOpTest): cpu_outputs = self._test_base(False) ipu_outputs = self._test_base(True) - np.testing.assert_allclose(cpu_outputs, - ipu_outputs, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + cpu_outputs, ipu_outputs, rtol=1e-05, atol=self.atol + ) class TestReplicaInference(TestBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -150,14 +146,13 @@ class TestReplicaInference(TestBase): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np_image} self.feed_ipu = { - "image": - np.tile(np_image, - [self.ipu_options['replicated_graph_count'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['replicated_graph_count'], 1, 1, 1] + ) } class TestReplicaCollectiveInference(TestBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -166,13 +161,11 @@ class TestReplicaCollectiveInference(TestBase): "accumulation_factor": 1, "enable_replicated_graphs": True, "replicated_graph_count": 2, - "accumulate_outer_fragment": { - 0: [] - }, + "accumulate_outer_fragment": {0: []}, "replicated_collectives_settings": { "prepare_schedule_for_merging_collectives": True, - "merge_all_reduce_collectives": True - } + "merge_all_reduce_collectives": True, + }, } self.cpu_bs = 1 self.ipu_bs = 1 @@ -181,14 +174,13 @@ class TestReplicaCollectiveInference(TestBase): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np_image} self.feed_ipu = { - "image": - np.tile(np_image, - [self.ipu_options['replicated_graph_count'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['replicated_graph_count'], 1, 1, 1] + ) } class TestPipelineInference(TestBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 2, @@ -205,13 +197,13 @@ class TestPipelineInference(TestBase): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np_image} self.feed_ipu = { - "image": - np.tile(np_image, [self.ipu_options['batches_per_step'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['batches_per_step'], 1, 1, 1] + ) } class TestTrainBase(TestBase): - def set_training(self): self.is_training = True self.epoch = 10 @@ -231,7 +223,6 @@ class TestTrainBase(TestBase): class TestReplicaTrain(TestTrainBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -239,7 +230,7 @@ class TestReplicaTrain(TestTrainBase): "enable_gradient_accumulation": False, "accumulation_factor": 1, "enable_replicated_graphs": True, - "replicated_graph_count": 2 + "replicated_graph_count": 2, } self.cpu_bs = 2 self.ipu_bs = 1 @@ -249,23 +240,21 @@ class TestReplicaTrain(TestTrainBase): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np.tile(np_image, [self.cpu_bs, 1, 1, 1])} self.feed_ipu = { - "image": - np.tile(np_image, - [self.ipu_options['replicated_graph_count'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['replicated_graph_count'], 1, 1, 1] + ) } def test(self): cpu_outputs = self._test_base(False) ipu_outputs = self._test_base(True)[::2] - np.testing.assert_allclose(cpu_outputs, - ipu_outputs, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + cpu_outputs, ipu_outputs, rtol=1e-05, atol=self.atol + ) class TestReplicaCollectiveTrain(TestTrainBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -274,13 +263,11 @@ class TestReplicaCollectiveTrain(TestTrainBase): "accumulation_factor": 1, "enable_replicated_graphs": True, "replicated_graph_count": 2, - "accumulate_outer_fragment": { - 0: [] - }, + "accumulate_outer_fragment": {0: []}, "replicated_collectives_settings": { "prepare_schedule_for_merging_collectives": True, - "merge_all_reduce_collectives": True - } + "merge_all_reduce_collectives": True, + }, } self.cpu_bs = 2 self.ipu_bs = 1 @@ -290,23 +277,21 @@ class TestReplicaCollectiveTrain(TestTrainBase): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np.tile(np_image, [self.cpu_bs, 1, 1, 1])} self.feed_ipu = { - "image": - np.tile(np_image, - [self.ipu_options['replicated_graph_count'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['replicated_graph_count'], 1, 1, 1] + ) } def test(self): cpu_outputs = self._test_base(False) ipu_outputs = self._test_base(True)[::2] - np.testing.assert_allclose(cpu_outputs, - ipu_outputs, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + cpu_outputs, ipu_outputs, rtol=1e-05, atol=self.atol + ) class TestPipelineTrain(TestTrainBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 3, @@ -323,22 +308,22 @@ class TestPipelineTrain(TestTrainBase): def set_data_feed(self): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np.tile(np_image, [self.cpu_bs, 1, 1, 1])} - bps_acc = self.ipu_options['batches_per_step'] * self.ipu_options[ - 'accumulation_factor'] + bps_acc = ( + self.ipu_options['batches_per_step'] + * self.ipu_options['accumulation_factor'] + ) self.feed_ipu = {"image": np.tile(np_image, [bps_acc, 1, 1, 1])} def test(self): cpu_outputs = self._test_base(False) ipu_outputs = self._test_base(True)[::3] - np.testing.assert_allclose(cpu_outputs, - ipu_outputs, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + cpu_outputs, ipu_outputs, rtol=1e-05, atol=self.atol + ) class TestAdamTrain(TestTrainBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -354,7 +339,6 @@ class TestAdamTrain(TestTrainBase): class TestAdamReplicaTrain(TestReplicaTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -370,7 +354,6 @@ class TestAdamReplicaTrain(TestReplicaTrain): class TestAdamPipelineTrain(TestPipelineTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 3, @@ -386,7 +369,6 @@ class TestAdamPipelineTrain(TestPipelineTrain): class TestAdamRecomputationTrain(TestPipelineTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 3, @@ -403,7 +385,6 @@ class TestAdamRecomputationTrain(TestPipelineTrain): class TestLambTrain(TestAdamTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -419,7 +400,6 @@ class TestLambTrain(TestAdamTrain): class TestLambReplicaTrain(TestAdamReplicaTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -435,7 +415,6 @@ class TestLambReplicaTrain(TestAdamReplicaTrain): class TestLambPipelineTrain(TestAdamPipelineTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 3, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_model_pipeline_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_model_pipeline_ipu.py index fb5f25619bf..6ee6b842332 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_model_pipeline_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_model_pipeline_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_training() self.set_data_feed() @@ -37,29 +36,28 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - image = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + image = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0): - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) with paddle.static.ipu_shard_guard(index=1): - conv2 = paddle.static.nn.conv2d(conv1, - num_filters=3, - filter_size=3, - bias_attr=False) + conv2 = paddle.static.nn.conv2d( + conv1, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv2) self.fetch_list = [loss.name] def run_model(self, exec_mode): ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=2, - is_training=False, - enable_manual_shard=True) - ipu_strategy.set_pipelining_config(enable_pipelining=True, - batches_per_step=2) + ipu_strategy.set_graph_config( + num_ipus=2, is_training=False, enable_manual_shard=True + ) + ipu_strategy.set_pipelining_config( + enable_pipelining=True, batches_per_step=2 + ) self.run_op_test(exec_mode, ipu_strategy=ipu_strategy) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_mul_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_mul_op_ipu.py index a5ace5f1bf1..c1c2116a413 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_mul_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_mul_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,12 +47,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.mul(x, y, **self.attrs) self.fetch_list = [out.name] @@ -69,7 +68,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 2, 5]) y = np.random.uniform(size=[5, 3]) @@ -84,7 +82,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 4, 2, 9]) y = np.random.uniform(size=[3, 6, 1, 2, 3]) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_not_equal_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_not_equal_op_ipu.py index c00a60775eb..465ed7d06b7 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_not_equal_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_not_equal_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,12 +49,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.not_equal(x, y, **self.attrs) self.fetch_list = [out.name] @@ -71,7 +70,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): x = np.ones([1, 10]) y = np.ones([1, 10]) @@ -80,7 +78,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_data_feed(self): x = np.ones([1, 10]) y = np.arange(0, 10).reshape([1, 10]) @@ -89,7 +86,6 @@ class TestCase2(TestBase): class TestScalar(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -116,10 +112,10 @@ class TestScalar(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - out = (x != 0.5) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + out = x != 0.5 self.fetch_list = [out.name] def run_model(self, exec_mode): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_one_hot_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_one_hot_op_ipu.py index fe5b658426e..53e8bab2adc 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_one_hot_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_one_hot_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int32' + ) out = paddle.fluid.layers.one_hot(x, **self.attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ class TestBase(IPUOpTest): @unittest.skip('does not support allow_out_of_range=True') class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"depth": 4, "allow_out_of_range": True} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_one_hot_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_one_hot_v2_op_ipu.py index e958cfd1f89..99294f9c5f1 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_one_hot_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_one_hot_v2_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int32' + ) out = paddle.fluid.input.one_hot(x, **self.attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ class TestBase(IPUOpTest): @unittest.skip('does not support allow_out_of_range=True') class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"depth": 4, "allow_out_of_range": True} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_optimizer_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_optimizer_ipu.py index f14b2174d39..d8b8116078b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_optimizer_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_optimizer_ipu.py @@ -20,7 +20,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -57,25 +56,27 @@ class TestBase(IPUOpTest): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) weight_decay = self.attrs['weight_decay'] - opt = paddle.optimizer.SGD(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.SGD( + learning_rate=1e-1, weight_decay=weight_decay + ) if self.attrs['optimizer'] == 'adam': - opt = paddle.optimizer.Adam(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.Adam( + learning_rate=1e-1, weight_decay=weight_decay + ) elif self.attrs['optimizer'] == 'lamb': - opt = paddle.optimizer.Lamb(learning_rate=1e-1, - lamb_weight_decay=weight_decay) + opt = paddle.optimizer.Lamb( + learning_rate=1e-1, lamb_weight_decay=weight_decay + ) opt.minimize(loss) if run_ipu: @@ -91,18 +92,23 @@ class TestBase(IPUOpTest): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=True) ipu_strategy.set_options( - {'loss_scaling': self.attrs["loss_scaling"]}) + {'loss_scaling': self.attrs["loss_scaling"]} + ) if "use_no_bias_optimizer" in self.attrs.keys(): - ipu_strategy.set_options({ - "use_no_bias_optimizer": - self.attrs["use_no_bias_optimizer"] - }) + ipu_strategy.set_options( + { + "use_no_bias_optimizer": self.attrs[ + "use_no_bias_optimizer" + ] + } + ) if "accl1_type" in self.attrs.keys(): ipu_strategy.set_options( - {"accl1_type": self.attrs["accl1_type"]}) + {"accl1_type": self.attrs["accl1_type"]} + ) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -118,15 +124,13 @@ class TestBase(IPUOpTest): ipu_loss = self._test_optimizer(True).flatten() cpu_loss = self._test_optimizer(False).flatten() - np.testing.assert_allclose(ipu_loss, - cpu_loss, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + ipu_loss, cpu_loss, rtol=1e-05, atol=self.atol + ) @unittest.skip('do not support L2 regularization') class TestSGD(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'sgd', @@ -137,7 +141,6 @@ class TestSGD(TestBase): @unittest.skip('do not support L2 regularization') class TestAdamCase1(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -147,7 +150,6 @@ class TestAdamCase1(TestBase): class TestAdamCase2(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -158,7 +160,6 @@ class TestAdamCase2(TestBase): @unittest.skip('cpu do not support AdamNoBias') class TestAdamNoBias(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -170,7 +171,6 @@ class TestAdamNoBias(TestBase): @unittest.skip('cpu do not support FLOAT16') class TestAdamCase3(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -182,7 +182,6 @@ class TestAdamCase3(TestBase): @unittest.skip('seems cpu output wrong') class TestLambCase1(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', @@ -193,7 +192,6 @@ class TestLambCase1(TestBase): @unittest.skip('seems cpu output wrong') class TestLamb(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', @@ -204,25 +202,23 @@ class TestLamb(TestBase): @unittest.skip('cpu do not support LambNoBias') class TestLambNoBias(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', "weight_decay": 0.1, "loss_scaling": 6.0, - "use_no_bias_optimizer": True + "use_no_bias_optimizer": True, } @unittest.skip('cpu do not support FLOAT16') class TestLambCase2(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', "weight_decay": 0.1, "loss_scaling": 6.0, - "accl1_type": "FLOAT16" + "accl1_type": "FLOAT16", } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_p_norm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_p_norm_op_ipu.py index bd6ff58751d..3cdcc509e4a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_p_norm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_p_norm_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,9 +39,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.nn.functional.normalize(x, **self.attrs) self.fetch_list = [x.name] @@ -58,13 +57,11 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 1} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"p": 3.5, "axis": 1, "epsilon": 1e-3} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_pad_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_pad_op_ipu.py index 741fc2189f0..325b95b6891 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_pad_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_pad_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,9 +39,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) pad = paddle.nn.functional.pad(x, **self.attrs) self.fetch_list = [pad.name] @@ -59,15 +58,14 @@ class TestBase(IPUOpTest): @unittest.skip("Do not support `pad` as a tensor") class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) const_attrs = { 'name': 'y', 'shape': [4], @@ -80,7 +78,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [2, 5], "data_format": "NCL"} @@ -93,7 +90,6 @@ class TestCase2(TestBase): class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [2, 5, 2, 3, 6, 3], "data_format": "NCDHW"} @@ -106,42 +102,36 @@ class TestCase3(TestBase): class TestCase4(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [2, 2, 1, 1], "mode": "reflect"} @unittest.skip("replicate mode is not supported") class TestCase5(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2, 3, 4], "mode": "replicate"} @unittest.skip("circular mode is not supported") class TestCase6(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2, 3, 4], "mode": "circular"} @unittest.skip("Only support NCL, NCHW, NCDHW") class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2], "data_format": "NLC"} @unittest.skip("Only support NCL, NCHW, NCDHW") class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2, 3, 4], "data_format": "NHWC"} @unittest.skip("Only support NCL, NCHW, NCDHW") class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2, 3, 4, 1, 3], "data_format": "NDHWC"} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_pool_avg_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_pool_avg_op_ipu.py index 8a2aa26f1c2..058e3b30a53 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_pool_avg_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_pool_avg_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -53,9 +52,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.pool2d(x, **self.attrs) self.fetch_list = [out.name] @@ -71,42 +70,36 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_size'] = 3 class TestCase1_2(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_size'] = [3, 1] class TestCase2(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_stride'] = 2 class TestCase2_2(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_stride'] = [2, 1] class TestCase3(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_padding'] = [1, 1] class TestCase3_2(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_padding'] = [1, 1, 2, 2] @@ -114,7 +107,6 @@ class TestCase3_2(TestBase): @unittest.skip('the results has a positional offset') class TestCase3_3(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_padding'] = [1, 2, 1, 1] @@ -122,7 +114,6 @@ class TestCase3_3(TestBase): @unittest.skip('paddle output has nan') class TestCase3_4(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_size'] = 1 @@ -130,40 +121,36 @@ class TestCase3_4(TestBase): class TestCase4(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['global_pooling'] = True class TestCase5(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['ceil_mode'] = True class TestCase6(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['exclusive'] = False class TestAdaptive(TestBase): - def set_op_attrs(self): self.attrs = { "pool_size": 1, "pool_type": 'avg', - "require_index": False + "require_index": False, } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.adaptive_pool2d(x, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_pool_max_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_pool_max_op_ipu.py index dca1103a0cd..aff790a775a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_pool_max_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_pool_max_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -53,9 +52,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.pool2d(x, **self.attrs) self.fetch_list = [out.name] @@ -71,42 +70,36 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_size'] = 3 class TestCase1_2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_size'] = [3, 1] class TestCase2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_stride'] = 2 class TestCase2_2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_stride'] = [2, 1] class TestCase3(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_padding'] = [1, 1] class TestCase3_2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_padding'] = [1, 1, 2, 2] @@ -114,7 +107,6 @@ class TestCase3_2(TestBase): @unittest.skip('auto_pad is not currently supported') class TestCase3_3(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_padding'] = 'VALID' @@ -122,47 +114,42 @@ class TestCase3_3(TestBase): @unittest.skip('auto_pad is not currently supported') class TestCase3_4(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_padding'] = 'SAME' class TestCase4(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['global_pooling'] = True class TestCase5(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['ceil_mode'] = True class TestCase6(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['exclusive'] = False class TestAdaptive(TestBase): - def set_op_attrs(self): self.attrs = { "pool_size": 1, "pool_type": 'max', - "require_index": False + "require_index": False, } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.adaptive_pool2d(x, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_pow_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_pow_op_ipu.py index 8355f5eefde..f61a8b8a24e 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_pow_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_pow_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.pow(x, **self.attrs) self.fetch_list = [out.name] @@ -62,18 +61,17 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): data1 = np.random.uniform(size=[1, 3, 2, 2]) data2 = np.array([2.0]) self.feed_fp32 = { "x": data1.astype(np.float32), - "y": data2.astype(np.float32) + "y": data2.astype(np.float32), } self.feed_fp16 = { "x": data1.astype(np.float16), - "y": data2.astype(np.float16) + "y": data2.astype(np.float16), } def set_op_attrs(self): @@ -81,12 +79,12 @@ class TestCase1(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - factor = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + factor = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.pow(x, factor=factor, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_prelu_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_prelu_op_ipu.py index b80560dccb3..a91b18d10ac 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_prelu_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_prelu_op_ipu.py @@ -22,7 +22,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -46,9 +45,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) array = np.random.uniform(size=[1]).astype(np.float32) result1 = paddle.zeros(shape=[1], dtype='float32') @@ -70,12 +69,11 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) array = np.random.uniform(size=[3]).astype(np.float32) result1 = paddle.zeros(shape=[3], dtype='float32') weight = paddle.assign(array, result1) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_print_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_print_op_ipu.py index 57935c8a657..70a2eba3c79 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_print_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_print_op_ipu.py @@ -22,7 +22,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest, IPUD2STest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,9 +48,11 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) out = paddle.fluid.layers.conv2d(x, num_filters=3, filter_size=3) out = paddle.fluid.layers.Print(out, **self.attrs) @@ -74,13 +75,11 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"message": "input_data"} class TestTrainCase1(TestBase): - def set_op_attrs(self): # "forward" : print forward # "backward" : print forward and backward @@ -94,7 +93,6 @@ class TestTrainCase1(TestBase): @unittest.skip("attrs are not supported") class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "first_n": 10, @@ -103,18 +101,16 @@ class TestCase2(TestBase): "print_tensor_type": True, "print_tensor_shape": True, "print_tensor_layout": True, - "print_tensor_lod": True + "print_tensor_lod": True, } class SimpleLayer(paddle.nn.Layer): - def __init__(self): super(SimpleLayer, self).__init__() - self.conv = paddle.nn.Conv2D(in_channels=3, - out_channels=1, - kernel_size=2, - stride=1) + self.conv = paddle.nn.Conv2D( + in_channels=3, out_channels=1, kernel_size=2, stride=1 + ) @to_static() def forward(self, x, target=None): @@ -130,7 +126,6 @@ class SimpleLayer(paddle.nn.Layer): class TestD2S(IPUD2STest): - def setUp(self): self.set_data_feed() @@ -142,16 +137,19 @@ class TestD2S(IPUD2STest): paddle.seed(self.SEED) np.random.seed(self.SEED) model = SimpleLayer() - optim = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) if use_ipu: paddle.set_device('ipu') ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False) + ipu_strategy.set_graph_config( + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ) ipu_strategy.set_optimizer(optim) result = [] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_rank_loss_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_rank_loss_op_ipu.py index ad3bbde1192..bebe0e22327 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_rank_loss_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_rank_loss_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,15 +48,15 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self, on_ipu): - label = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - left = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') - right = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='float32') + label = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + left = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) + right = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='float32' + ) out = paddle.fluid.layers.rank_loss(label, left, right) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_reduce_x_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_reduce_x_op_ipu.py index c78165f86e2..9139b1db69a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_reduce_x_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_reduce_x_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestMean(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -37,9 +36,9 @@ class TestMean(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = self.op(x, **self.attrs) self.fetch_list = [out.name] @@ -123,39 +122,34 @@ class TestMean(IPUOpTest): class TestMax(TestMean): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_max class TestMin(TestMean): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_min class TestProd(TestMean): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_prod class TestSum(TestMean): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_sum class TestLogsumexp(TestMean): - def set_test_op(self): self.op = paddle.logsumexp @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if 'dim' in self.attrs: self.attrs['axis'] = self.attrs['dim'] del self.attrs['dim'] @@ -167,7 +161,6 @@ class TestLogsumexp(TestMean): class TestAll(TestMean): - @property def fp16_enabled(self): return False @@ -184,9 +177,9 @@ class TestAll(TestMean): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='bool') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='bool' + ) out = self.op(x, **self.attrs) self.fetch_list = [out.name] @@ -195,7 +188,6 @@ class TestAll(TestMean): class TestAny(TestAll): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_any diff --git a/python/paddle/fluid/tests/unittests/ipu/test_reshape_inplace_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_reshape_inplace_op_ipu.py index 66358d83ee6..c6016d2ec13 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_reshape_inplace_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_reshape_inplace_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -47,9 +46,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) add = paddle.fluid.layers.elementwise_add(x, x) out = paddle.fluid.layers.reshape(add, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_attrs(self): self.attrs = { "shape": [-1, 0, 10], diff --git a/python/paddle/fluid/tests/unittests/ipu/test_reshape_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_reshape_op_ipu.py index 2da63cf7330..74bae31111b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_reshape_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_reshape_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,9 +44,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.reshape(x=x, **self.attrs) self.fetch_list = [out.name] @@ -63,7 +62,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['shape'] = [2, 3, -1, 2] @@ -71,7 +69,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['shape'] = [-1, 0, 3, 2] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_save_load_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_save_load_ipu.py index 1e8387ac228..feed2f739be 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_save_load_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_save_load_ipu.py @@ -24,7 +24,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -54,14 +53,14 @@ class TestBase(IPUOpTest): def build_model(self): generator = paddle.fluid.unique_name.UniqueNameGenerator() with paddle.fluid.unique_name.guard(generator): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False, - name='conv2d') + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype='float32', + ) + conv1 = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False, name='conv2d' + ) loss = paddle.mean(conv1) # apply optimizer self.optimizer().minimize(loss) @@ -83,7 +82,8 @@ class TestBase(IPUOpTest): ipu_strategy.set_precision_config(enable_fp16=True) IPUOpTest.cast_model_to_fp16(self.main_prog) ipu_compiler = paddle.static.IpuCompiledProgram( - self.main_prog, ipu_strategy=ipu_strategy) + self.main_prog, ipu_strategy=ipu_strategy + ) program = ipu_compiler.compile(self.feed_list, self.fetch_list) feed = self.feed_fp32 @@ -91,16 +91,19 @@ class TestBase(IPUOpTest): feed = self.feed_fp16 result = [] - run_steps = self.attrs['steps'] if save_otherwise_load \ + run_steps = ( + self.attrs['steps'] + if save_otherwise_load else self.attrs['steps'] - self.attrs['save_at_step'] + ) for i in range(run_steps): tmp = exe.run(program, feed=feed, fetch_list=self.fetch_list) - if save_otherwise_load and \ - i == self.attrs['save_at_step'] - 1: + if save_otherwise_load and i == self.attrs['save_at_step'] - 1: ipu_compiler._backend.weights_to_host() - paddle.static.save(self.main_prog, - self.attrs['model_path'].name) + paddle.static.save( + self.main_prog, self.attrs['model_path'].name + ) if save_otherwise_load and i >= self.attrs['save_at_step']: result.append(tmp) @@ -112,72 +115,61 @@ class TestBase(IPUOpTest): def test_base(self): res0 = self.run_model(IPUOpTest.ExecutionMode.IPU_FP32, True) res1 = self.run_model(IPUOpTest.ExecutionMode.IPU_FP32, False) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.attrs['model_path'].cleanup() class TestMomentum(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Momentum, learning_rate=1e-1) class TestAdam(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adam, learning_rate=1e-1) class TestLamb(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Lamb, learning_rate=1e-1) class TestAdamW(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.AdamW, learning_rate=1e-1) class TestAdamax(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adamax, learning_rate=1e-1) class TestAdagrad(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adagrad, learning_rate=1e-1) class TestAdadelta(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adagrad, learning_rate=1e-1) class TestRMSProp(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.RMSProp, learning_rate=1e-1) class TestCenteredRMSProp(TestBase): - def set_optimizer(self): - self.optimizer = partial(paddle.optimizer.RMSProp, - learning_rate=1e-1, - centered=True) + self.optimizer = partial( + paddle.optimizer.RMSProp, learning_rate=1e-1, centered=True + ) @unittest.skipIf(IPUOpTest.use_ipumodel(), "skip for ipumodel") class TestSGDFP16(TestBase): - def set_attrs(self): self.attrs = {} self.attrs['steps'] = 100 @@ -190,67 +182,57 @@ class TestSGDFP16(TestBase): def test_base(self): res0 = self.run_model(IPUOpTest.ExecutionMode.IPU_FP16, True) res1 = self.run_model(IPUOpTest.ExecutionMode.IPU_FP16, False) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.attrs['model_path'].cleanup() class TestMomentumFp16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Momentum, learning_rate=1e-1) class TestAdamFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adam, learning_rate=1e-1) class TestLambFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Lamb, learning_rate=1e-1) class TestAdamWFP16FP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.AdamW, learning_rate=1e-1) class TestAdamaxFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adamax, learning_rate=1e-1) class TestAdagradFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adagrad, learning_rate=1e-1) class TestAdadeltaFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adagrad, learning_rate=1e-1) class TestRMSPropFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.RMSProp, learning_rate=1e-1) class TestCenteredRMSPropFP16(TestSGDFP16): - def set_optimizer(self): - self.optimizer = partial(paddle.optimizer.RMSProp, - learning_rate=1e-1, - centered=True) + self.optimizer = partial( + paddle.optimizer.RMSProp, learning_rate=1e-1, centered=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_scale_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_scale_op_ipu.py index 296d365fea6..44ef0aa9fa4 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_scale_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_scale_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -52,9 +51,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.scale(x, **self.attrs) self.fetch_list = [out.name] @@ -70,7 +69,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "scale": 5.0, @@ -80,7 +78,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "scale": 1.0, @@ -90,7 +87,6 @@ class TestCase2(TestBase): class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = { "scale": 5.0, @@ -100,7 +96,6 @@ class TestCase3(TestBase): class TestCase4(TestBase): - def set_op_attrs(self): self.attrs = { "scale": 1.0, @@ -110,7 +105,6 @@ class TestCase4(TestBase): class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 3, 10, 10]) y = np.array([3.0]) @@ -125,12 +119,12 @@ class TestCase5(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.scale(x, scale=y, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_scaled_optimizer_state_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_scaled_optimizer_state_ipu.py index 3960be248ec..ca6d99ab2e5 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_scaled_optimizer_state_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_scaled_optimizer_state_ipu.py @@ -20,7 +20,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,26 +49,27 @@ class TestBase(IPUOpTest): self.attrs = { "optimizer": 'lamb', "weight_decay": 0.0, - "scaled_optimizer_state": True + "scaled_optimizer_state": True, } @IPUOpTest.static_graph def build_model(self): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) weight_decay = self.attrs['weight_decay'] - opt = paddle.optimizer.Adam(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.Adam( + learning_rate=1e-1, weight_decay=weight_decay + ) if self.attrs['optimizer'] == 'lamb': - opt = paddle.optimizer.Lamb(learning_rate=1e-1, - lamb_weight_decay=weight_decay) + opt = paddle.optimizer.Lamb( + learning_rate=1e-1, lamb_weight_decay=weight_decay + ) opt.minimize(loss) self.feed_list = [image.name] self.fetch_list = [loss.name] @@ -79,15 +79,21 @@ class TestBase(IPUOpTest): ipu_strategy.set_graph_config(is_training=self.is_training) if self.is_ipu_mode(exec_mode): if "use_no_bias_optimizer" in self.attrs.keys(): - ipu_strategy.set_options({ - "use_no_bias_optimizer": - self.attrs["use_no_bias_optimizer"] - }) + ipu_strategy.set_options( + { + "use_no_bias_optimizer": self.attrs[ + "use_no_bias_optimizer" + ] + } + ) if "scaled_optimizer_state" in self.attrs.keys(): - ipu_strategy.set_options({ - "scaled_optimizer_state": - self.attrs["scaled_optimizer_state"] - }) + ipu_strategy.set_options( + { + "scaled_optimizer_state": self.attrs[ + "scaled_optimizer_state" + ] + } + ) self.run_op_test(exec_mode, ipu_strategy=ipu_strategy) def test(self): @@ -99,12 +105,11 @@ class TestBase(IPUOpTest): class TestScaledAdam(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', "weight_decay": 0.0, - "scaled_optimizer_state": True + "scaled_optimizer_state": True, } def set_atol(self): @@ -115,25 +120,23 @@ class TestScaledAdam(TestBase): @unittest.skip('cpu do not support AdamNoBias') class TestScaledAdamNoBias(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', "weight_decay": 0.0, "use_no_bias_optimizer": True, - "scaled_optimizer_state": True + "scaled_optimizer_state": True, } @unittest.skip('cpu do not support LambNoBias') class TestScaledLambNoBias(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', "weight_decay": 0.0, "use_no_bias_optimizer": True, - "scaled_optimizer_state": True + "scaled_optimizer_state": True, } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_set_batch_size_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_set_batch_size_ipu.py index 9bce0b5df73..6cd27f24723 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_set_batch_size_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_set_batch_size_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,31 +48,28 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - conv2 = paddle.static.nn.conv2d(conv1, - num_filters=3, - filter_size=3, - bias_attr=False) - conv3 = paddle.static.nn.conv2d(conv2, - num_filters=3, - filter_size=3, - bias_attr=False) - conv4 = paddle.static.nn.conv2d(conv3, - num_filters=3, - filter_size=3, - bias_attr=False) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + conv2 = paddle.static.nn.conv2d( + conv1, num_filters=3, filter_size=3, bias_attr=False + ) + conv3 = paddle.static.nn.conv2d( + conv2, num_filters=3, filter_size=3, bias_attr=False + ) + conv4 = paddle.static.nn.conv2d( + conv3, num_filters=3, filter_size=3, bias_attr=False + ) self.fetch_list = [conv4.name] def run_model(self, exec_mode): ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(is_training=self.is_training, - micro_batch_size=2) + ipu_strategy.set_graph_config( + is_training=self.is_training, micro_batch_size=2 + ) self.run_op_test(exec_mode, ipu_strategy) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_set_ipu_shard_api.py b/python/paddle/fluid/tests/unittests/ipu/test_set_ipu_shard_api.py index 37423fda1fa..ccd0b6b0d2e 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_set_ipu_shard_api.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_set_ipu_shard_api.py @@ -23,7 +23,6 @@ paddle.enable_static() class SimpleNet(paddle.nn.Layer): - def __init__(self, input_size, output_size): super(SimpleNet, self).__init__() self.linear1 = nn.Linear(input_size, output_size) @@ -46,15 +45,14 @@ class SimpleNet(paddle.nn.Layer): class TestSetIpuShard(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model.linear1, index=1) @@ -74,22 +72,20 @@ class TestSetIpuShard(unittest.TestCase): ipu_index_list = self._test() expected_ipu_index_list = [1, 1, 2, 3, 3, 3, 4, 4] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestSetIpuPipeline(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model.linear1, stage=1) @@ -109,29 +105,27 @@ class TestSetIpuPipeline(unittest.TestCase): ipu_index_list = self._test() expected_ipu_index_list = [1, 1, 2, 3, 3, 3, 4, 4] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestSetIpuShardAndPipeline(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model.linear1, index=1, stage=2) set_ipu_shard(model.relu1, index=2, stage=3) - model.linear_relu2 = set_ipu_shard(model.linear_relu2, - index=3, - stage=4) + model.linear_relu2 = set_ipu_shard( + model.linear_relu2, index=3, stage=4 + ) model.linear3 = set_ipu_shard(model.linear3, index=4, stage=1) out = model(x) @@ -148,25 +142,38 @@ class TestSetIpuShardAndPipeline(unittest.TestCase): def test_set_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [ - 1, 1, 2, 3, 3, 3, 4, 4, 2, 2, 3, 4, 4, 4, 1, 1 + 1, + 1, + 2, + 3, + 3, + 3, + 4, + 4, + 2, + 2, + 3, + 4, + 4, + 4, + 1, + 1, ] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestSetIpuForModel(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model, index=1, stage=2) @@ -185,19 +192,31 @@ class TestSetIpuForModel(unittest.TestCase): def test_set_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [ - 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, ] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestSetIpuMixedModel(unittest.TestCase): - def setUp(self): - def linear_relu2_mixed(self, x): with paddle.static.ipu_shard_guard(index=2, stage=3): x = self.linear2(x) @@ -216,9 +235,9 @@ class TestSetIpuMixedModel(unittest.TestCase): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model.linear1, index=1, stage=2) @@ -239,13 +258,27 @@ class TestSetIpuMixedModel(unittest.TestCase): def test_set_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [ - 1, 1, 2, 2, 2, 3, 4, 4, 2, 2, 3, 3, 3, 4, 1, 1 + 1, + 1, + 2, + 2, + 2, + 3, + 4, + 4, + 2, + 2, + 3, + 3, + 3, + 4, + 1, + 1, ] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_share_data_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_share_data_op_ipu.py index 7d355858f13..59acdf92198 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_share_data_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_share_data_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -39,15 +38,15 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - self.main_prog.global_block().append_op(type="share_data", - inputs={"X": x}, - outputs={'Out': y}) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + self.main_prog.global_block().append_op( + type="share_data", inputs={"X": x}, outputs={'Out': y} + ) out = paddle.fluid.layers.elementwise_add(y, y) self.fetch_list = [out.name] @@ -63,7 +62,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 3, 1]) self.feed_fp32 = {'in_0': data.astype(np.float32)} @@ -74,15 +72,15 @@ class TestCase1(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - self.main_prog.global_block().append_op(type="share_data", - inputs={"X": x}, - outputs={'Out': y}) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + self.main_prog.global_block().append_op( + type="share_data", inputs={"X": x}, outputs={'Out': y} + ) out = paddle.fluid.layers.elementwise_add(x, y) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_sigmoid_cross_entropy_with_logits_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_sigmoid_cross_entropy_with_logits_op_ipu.py index 7a6cf28d443..c6b0f6a494d 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_sigmoid_cross_entropy_with_logits_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_sigmoid_cross_entropy_with_logits_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ class TestBase(IPUOpTest): label = np.arange(10).reshape([10]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.float32) + "label": label.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.float16) + "label": label.astype(np.float16), } def set_feed_attr(self): @@ -52,14 +51,15 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.sigmoid_cross_entropy_with_logits( - x, label, **self.attrs) + x, label, **self.attrs + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -74,7 +74,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'ignore_index': 1, @@ -82,7 +81,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_atol(self): # epsilon is added when normalize is True, use larger atol. self.atol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_slice_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_slice_op_ipu.py index 3bcbe417b98..6b5f6588bd3 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_slice_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_slice_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,9 +47,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.slice(x, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "axes": [0, 1], @@ -77,7 +75,6 @@ class TestCase1(TestBase): @unittest.skip('dynamic graph is not support on IPU') class TestCase2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 5, 6]) s = np.array([0, 0, 2]) @@ -85,12 +82,12 @@ class TestCase2(TestBase): self.feed_fp32 = { "x": x.astype(np.float32), "starts": s.astype(np.int32), - "ends": e.astype(np.int32) + "ends": e.astype(np.int32), } self.feed_fp16 = { "x": x.astype(np.float16), "starts": s.astype(np.int32), - "ends": e.astype(np.int32) + "ends": e.astype(np.int32), } def set_op_attrs(self): @@ -98,19 +95,18 @@ class TestCase2(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - starts = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') - ends = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='int32') - out = paddle.fluid.layers.slice(x, - starts=starts, - ends=ends, - **self.attrs) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + starts = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) + ends = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='int32' + ) + out = paddle.fluid.layers.slice( + x, starts=starts, ends=ends, **self.attrs + ) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_softmax_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_softmax_op_ipu.py index ebc05942b93..8c1dfa04275 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_softmax_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_softmax_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.softmax(x, **self.attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 2} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_softmax_with_cross_entropy_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_softmax_with_cross_entropy_op_ipu.py index d3084154a06..c5a789942aa 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_softmax_with_cross_entropy_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_softmax_with_cross_entropy_op_ipu.py @@ -22,7 +22,6 @@ import paddle.nn.functional as F class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -35,11 +34,11 @@ class TestBase(IPUOpTest): label = np.arange(3).reshape([3, 1]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } def set_feed_attr(self): @@ -53,17 +52,17 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) if on_ipu: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) else: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int64') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int64' + ) out = F.softmax_with_cross_entropy(x, label, **self.attrs) self.fetch_list = [out.name] @@ -81,7 +80,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -90,37 +88,34 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[30, 70]) label = np.arange(30).reshape([30, 1]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 5, 7]) label = np.random.randint(0, 7, [3, 5, 1], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } class TestCase4(TestBase): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -130,17 +125,17 @@ class TestCase4(TestBase): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) if on_ipu: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) else: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int64') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int64' + ) loss, softmax = F.softmax_with_cross_entropy(x, label, **self.attrs) self.fetch_list = [loss.name, softmax.name] @@ -158,7 +153,6 @@ class TestCase4(TestBase): class TestCase5(TestCase4): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -172,16 +166,15 @@ class TestCase5(TestCase4): label = np.random.randint(0, 5, [3, 1, 7, 11], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } class TestCase6(TestCase4): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -195,11 +188,11 @@ class TestCase6(TestCase4): label = np.random.randint(0, 7, [3, 5, 1, 9, 11], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_split_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_split_op_ipu.py index 8d8c5190692..dada2225edf 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_split_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_split_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.split(x, **self.attrs) self.fetch_list = [fetch.name for fetch in out] @@ -63,7 +62,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"num_or_sections": [2, 8], "axis": 2} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_squeeze_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_squeeze_op_ipu.py index 9039dfdb3f0..f3e312c863e 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_squeeze_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_squeeze_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.squeeze(x, **self.attrs) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axes": []} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"axes": [-2]} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_stack_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_stack_op_ipu.py index fa0a48081b4..1f1fbf6d789 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_stack_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_stack_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -36,12 +35,12 @@ class TestBase(IPUOpTest): self.feed_fp32 = { "x": x.astype(np.float32), "y": y.astype(np.float32), - "z": z.astype(np.float32) + "z": z.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), "y": y.astype(np.float16), - "z": z.astype(np.float16) + "z": z.astype(np.float16), } def set_feed_attr(self): @@ -54,15 +53,15 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') - z = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) + z = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='float32' + ) out = paddle.fluid.layers.stack([x, y, z], **self.attrs) self.fetch_list = [out.name] @@ -78,7 +77,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": -2} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_strided_slice_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_strided_slice_op_ipu.py index f41ad607426..5f1bed2938d 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_strided_slice_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_strided_slice_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,14 +43,14 @@ class TestBase(IPUOpTest): "axes": [1, 2, 3], "starts": [-3, 0, 2], "ends": [3, 2, 4], - "strides": [1, 1, 1] + "strides": [1, 1, 1], } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.strided_slice(x, **self.attrs) self.fetch_list = [out.name] @@ -67,7 +66,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 4]) self.feed_fp32 = {"in_0": data.astype(np.float32)} @@ -78,13 +76,12 @@ class TestCase1(TestBase): "axes": [0, 1], "starts": [1, 3], "ends": [2, 0], - "strides": [1, -1] + "strides": [1, -1], } @unittest.skip('Only strides of 1 or -1 are supported.') class TestCase2(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 4]) self.feed_fp32 = {"in_0": data.astype(np.float32)} @@ -95,13 +92,12 @@ class TestCase2(TestBase): "axes": [0, 1], "starts": [1, 3], "ends": [-1, 1000], - "strides": [1, 3] + "strides": [1, 3], } @unittest.skip('dynamic graph is not support on IPU') class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 5, 6]) s = np.array([0, 0, 2]) @@ -109,12 +105,12 @@ class TestCase3(TestBase): self.feed_fp32 = { "x": x.astype(np.float32), "starts": s.astype(np.int32), - "ends": e.astype(np.int32) + "ends": e.astype(np.int32), } self.feed_fp16 = { "x": x.astype(np.float16), "starts": s.astype(np.int32), - "ends": e.astype(np.int32) + "ends": e.astype(np.int32), } def set_op_attrs(self): @@ -122,19 +118,18 @@ class TestCase3(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - starts = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') - ends = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='int32') - out = paddle.fluid.layers.strided_slice(x, - starts=starts, - ends=ends, - **self.attrs) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + starts = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) + ends = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='int32' + ) + out = paddle.fluid.layers.strided_slice( + x, starts=starts, ends=ends, **self.attrs + ) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_sum_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_sum_op_ipu.py index 3c4f9ff80d5..9ed200626c1 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_sum_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_sum_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,12 +44,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.sum([x, y], **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 3, 2, 2]) y = np.random.uniform(size=[1, 3, 2, 2]) @@ -74,25 +72,25 @@ class TestCase1(TestBase): self.feed_fp32 = { "x": x.astype(np.float32), "y": y.astype(np.float32), - "z": z.astype(np.float32) + "z": z.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), "y": y.astype(np.float16), - "z": z.astype(np.float16) + "z": z.astype(np.float16), } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') - z = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) + z = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='float32' + ) out = paddle.fluid.layers.sum([x, y, z], **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_tile_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_tile_op_ipu.py index ecdbe20f75f..becacf6b8be 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_tile_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_tile_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -46,9 +45,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.tile(x, **self.attrs) self.fetch_list = [out.name] @@ -64,7 +63,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "repeat_times": [2, 3, 2], @@ -73,7 +71,6 @@ class TestCase1(TestBase): @unittest.skip('dynamic graph is not support on IPU') class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 5, 6]) r = np.array([3, 2, 4]) @@ -91,12 +88,12 @@ class TestCase3(TestBase): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - r = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + r = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = paddle.tile(x, repeat_times=r, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_topk_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_topk_op_ipu.py index 4194887ab2f..bfc31118b4e 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_topk_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_topk_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestTopKOp(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,17 +49,16 @@ class TestTopKOp(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if not self.use_k_as_const_variable: topk_values, topk_indices = self.op(x, **self.attrs) else: # !important, popart cannot accept non const tensor - K_t = paddle.fluid.layers.fill_constant(shape=[1], - dtype='int32', - value=self.k, - name="in_2") + K_t = paddle.fluid.layers.fill_constant( + shape=[1], dtype='int32', value=self.k, name="in_2" + ) topk_values, topk_indices = self.op(x, K_t, **self.attrs) self.fetch_list = [topk_values.name, topk_indices.name] @@ -83,14 +81,12 @@ class TestTopKOp(IPUOpTest): class TestCase2(TestTopKOp): - def set_test_op(self): self.op = paddle.topk @unittest.skip("Trying to get data as int64 but it is of type int32") class TestCase3(TestTopKOp): - def set_op_attrs(self): self.use_k_as_const_variable = True self.attrs = {} @@ -99,7 +95,6 @@ class TestCase3(TestTopKOp): @unittest.skip("Trying to get data as int64 but it is of type int32") class TestCase4(TestCase3): - def set_test_op(self): self.op = paddle.topk diff --git a/python/paddle/fluid/tests/unittests/ipu/test_transpose_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_transpose_op_ipu.py index e8f3cae3673..34e0457c2de 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_transpose_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_transpose_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.transpose(x, **self.attrs) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"perm": [0, 1, 2, 3]} class TestCase2(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[1, 2, 3, 4, 5]) self.feed_fp32 = {"x": data.astype(np.float32)} @@ -79,7 +76,6 @@ class TestCase2(TestBase): class TestCase_ZeroDim(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[]) self.feed_fp32 = {"x": data.astype(np.float32)} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_unary_ops_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_unary_ops_ipu.py index bbf0f7b6996..9670e1da633 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_unary_ops_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_unary_ops_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_test_op() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = self.op(x, **self.op_attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ class TestBase(IPUOpTest): class TestAcos(TestBase): - @property def fp16_enabled(self): return False @@ -77,63 +75,54 @@ class TestAcos(TestBase): class TestAsin(TestAcos): - def set_test_op(self): self.op = paddle.fluid.layers.asin self.op_attrs = {} class TestSinh(TestAcos): - def set_test_op(self): self.op = paddle.fluid.layers.sinh self.op_attrs = {} class TestAtan(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.atan self.op_attrs = {} class TestCeil(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.ceil self.op_attrs = {} class TestCos(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.cos self.op_attrs = {} class TestCosh(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.cosh self.op_attrs = {} class TestErf(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.erf self.op_attrs = {} class TestExp(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.exp self.op_attrs = {} class TestFloor(TestBase): - @property def fp16_enabled(self): return False @@ -144,84 +133,72 @@ class TestFloor(TestBase): class TestLog(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.log self.op_attrs = {} class TestReciprocal(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.reciprocal self.op_attrs = {} class TestRelu(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.relu self.op_attrs = {} class TestRound(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.round self.op_attrs = {} class TestSigmoid(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.sigmoid self.op_attrs = {} class TestSign(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.sign self.op_attrs = {} class TestSin(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.sin self.op_attrs = {} class TestSoftplus(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.softplus self.op_attrs = {} class TestSoftsign(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.softsign self.op_attrs = {} class TestSqrt(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.sqrt self.op_attrs = {} class TestTan(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.tan self.op_attrs = {} class TestTanh(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.tanh self.op_attrs = {} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_unsqueeze_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_unsqueeze_op_ipu.py index 3f3b9f4f890..d5f3c5913e4 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_unsqueeze_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_unsqueeze_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.unsqueeze(x, **self.attrs) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axes": -1} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"axes": [1, 2]} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_unstack_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_unstack_op_ipu.py index 46900011752..382a70ba012 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_unstack_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_unstack_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -46,9 +45,9 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.unstack(x, **self.attrs) self.fetch_list = [x.name for x in out] @@ -64,7 +63,6 @@ class TestBase(IPUOpTest): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "axis": 2, @@ -72,7 +70,6 @@ class TestCase1(TestBase): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "axis": 0, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_varname_inplace_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_varname_inplace_ipu.py index 56e7eef4cb2..e42e5b4d54e 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_varname_inplace_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_varname_inplace_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -54,9 +53,11 @@ class TestBase(IPUOpTest): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) add1 = paddle.fluid.layers.elementwise_add(x, x) reshape = paddle.fluid.layers.reshape(add1, **self.attrs) add2 = paddle.fluid.layers.elementwise_add(reshape, reshape) @@ -75,7 +76,8 @@ class TestBase(IPUOpTest): exe.run(startup_prog) scale1_out = main_prog.global_block().ops[4].output("Out")[0] main_prog.global_block().ops[4]._rename_output( - scale1_out, add2.name) + scale1_out, add2.name + ) main_prog.global_block().ops[5]._rename_input(scale1_out, add2.name) if run_ipu: @@ -83,8 +85,8 @@ class TestBase(IPUOpTest): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=self.is_training) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -95,10 +97,9 @@ class TestBase(IPUOpTest): res0 = self._test_base(True) res1 = self._test_base(False) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.assertTrue(res0.shape == res1.shape) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_warpctc_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_warpctc_op_ipu.py index d173ae77e5c..1161699893a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_warpctc_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_warpctc_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -39,20 +38,24 @@ class TestBase(IPUOpTest): self.max_seq_length = 5 self.max_label_length = 3 self.num_classes = 5 - self.logits_length = np.array([self.max_seq_length] * self.batch_size, - dtype=np.int64) - self.labels_length = np.array([self.max_label_length] * self.batch_size, - dtype=np.int64) + self.logits_length = np.array( + [self.max_seq_length] * self.batch_size, dtype=np.int64 + ) + self.labels_length = np.array( + [self.max_label_length] * self.batch_size, dtype=np.int64 + ) self.blank = self.num_classes - 1 self.norm_by_times = False logits = np.random.uniform( - 0.1, 1.0, [self.max_seq_length, self.batch_size, self.num_classes - ]).astype("float32") - labels = np.random.randint(0, - self.num_classes - 1, - [self.batch_size, self.max_label_length], - dtype="int32") + 0.1, 1.0, [self.max_seq_length, self.batch_size, self.num_classes] + ).astype("float32") + labels = np.random.randint( + 0, + self.num_classes - 1, + [self.batch_size, self.max_label_length], + dtype="int32", + ) self.feed_fp32 = { "Logits": logits, @@ -79,26 +82,28 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - data = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - logits = paddle.nn.Linear(self.num_classes, - self.num_classes, - bias_attr=False)(data) - labels = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') - input_length = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='int64') - label_length = paddle.static.data(name=self.feed_list[3], - shape=self.feed_shape[3], - dtype='int64') - out = paddle.fluid.layers.warpctc(logits, - labels, - input_length=input_length, - label_length=label_length, - **self.attrs) + data = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + logits = paddle.nn.Linear( + self.num_classes, self.num_classes, bias_attr=False + )(data) + labels = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) + input_length = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='int64' + ) + label_length = paddle.static.data( + name=self.feed_list[3], shape=self.feed_shape[3], dtype='int64' + ) + out = paddle.fluid.layers.warpctc( + logits, + labels, + input_length=input_length, + label_length=label_length, + **self.attrs + ) loss = paddle.mean(out) adam = paddle.optimizer.Adam(learning_rate=1e-2) adam.minimize(loss) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_weight_decay_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_weight_decay_ipu.py index 60594298321..613032c6262 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_weight_decay_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_weight_decay_ipu.py @@ -24,7 +24,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest @unittest.skipIf(IPUOpTest.use_ipumodel(), "skip for ipumodel") class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -57,7 +56,6 @@ class TestBase(IPUOpTest): } def _test_optimizer(self, run_ipu=True): - def exclude_fn(param): return param.name.endswith('.w_0') @@ -70,22 +68,23 @@ class TestBase(IPUOpTest): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) bias = paddle.fluid.layers.create_parameter( - shape=[1, 3, 10, 10], is_bias=True, dtype='float32') + shape=[1, 3, 10, 10], is_bias=True, dtype='float32' + ) add1 = image + bias - conv1 = paddle.static.nn.conv2d(add1, - num_filters=3, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + add1, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) opt = paddle.optimizer.Lamb( learning_rate=1e-3, lamb_weight_decay=self.attrs['weight_decay'], - exclude_from_weight_decay_fn=exclude_fn) + exclude_from_weight_decay_fn=exclude_fn, + ) opt.minimize(loss) if run_ipu: @@ -102,10 +101,11 @@ class TestBase(IPUOpTest): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=True) ipu_strategy.set_options( - {'loss_scaling': self.attrs["loss_scaling"]}) + {'loss_scaling': self.attrs["loss_scaling"]} + ) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -121,10 +121,9 @@ class TestBase(IPUOpTest): ipu_loss = self._test_optimizer(True).flatten() cpu_loss = self._test_optimizer(False).flatten() - np.testing.assert_allclose(ipu_loss, - cpu_loss, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + ipu_loss, cpu_loss, rtol=1e-05, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_weight_sharing_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_weight_sharing_ipu.py index 4fe9ef12733..8eb0edf4cfb 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_weight_sharing_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_weight_sharing_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestWeightSharing(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -51,24 +50,27 @@ class TestWeightSharing(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int64') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int64' + ) with paddle.static.ipu_shard_guard(index=0, stage=0): y = paddle.fluid.layers.embedding( input=x, size=[768, 768], dtype='float32', param_attr=paddle.fluid.ParamAttr(name='word_embedding'), - is_sparse=False) + is_sparse=False, + ) with paddle.static.ipu_shard_guard(index=1, stage=1): z = paddle.fluid.layers.fc( - input=y, size=768, param_attr=paddle.fluid.ParamAttr(name="fc")) + input=y, size=768, param_attr=paddle.fluid.ParamAttr(name="fc") + ) with paddle.static.ipu_shard_guard(index=0, stage=2): out = paddle.fluid.layers.matmul( x=z, y=self.main_prog.global_block().var('word_embedding'), - transpose_y=True) + transpose_y=True, + ) self.feed_list = [x.name] self.fetch_list = [out.name] @@ -82,15 +84,17 @@ class TestWeightSharing(IPUOpTest): exe.run(self.startup_prog) if run_ipu: ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=2, - is_training=self.is_training, - enable_manual_shard=True) - ipu_strategy.set_pipelining_config(enable_pipelining=True, - batches_per_step=3) + ipu_strategy.set_graph_config( + num_ipus=2, + is_training=self.is_training, + enable_manual_shard=True, + ) + ipu_strategy.set_pipelining_config( + enable_pipelining=True, batches_per_step=3 + ) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog @@ -102,10 +106,9 @@ class TestWeightSharing(IPUOpTest): res0 = self.run_model(False) res1 = self.run_model(True) - np.testing.assert_allclose(res0.flatten(), - res1[0].flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1[0].flatten(), rtol=1e-05, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_where_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_where_op_ipu.py index 091f5dc960d..15a815bf5a7 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_where_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_where_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ class TestBase(IPUOpTest): data_y = np.random.uniform(size=[4, 5, 6]) self.feed_fp32 = { "in_0": data_x.astype(np.float32), - "in_1": data_y.astype(np.float32) + "in_1": data_y.astype(np.float32), } self.feed_fp16 = { "in_0": data_x.astype(np.float16), - "in_1": data_y.astype(np.float16) + "in_1": data_y.astype(np.float16), } def set_feed_attr(self): @@ -51,12 +50,12 @@ class TestBase(IPUOpTest): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.where(x > 1, x, y) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_yolo_box_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_yolo_box_op_ipu.py index 3e19ca2b0c9..328e46d430c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_yolo_box_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_yolo_box_op_ipu.py @@ -21,7 +21,6 @@ from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUOpTest class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,14 +49,14 @@ class TestBase(IPUOpTest): "class_num": 80, "anchors": [10, 13, 16, 30, 33, 23], "conf_thresh": 0.01, - "downsample_ratio": 32 + "downsample_ratio": 32, } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) attrs = { 'name': 'img_size', 'shape': [1, 2], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py b/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py index ed191d55603..d76a1710129 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py @@ -24,7 +24,12 @@ import paddle from paddle.fluid.core import PassVersionChecker import paddle.inference as paddle_infer from typing import Optional, List, Callable, Dict, Any -from program_config import OpConfig, ProgramConfig, create_fake_model, create_quant_model +from program_config import ( + OpConfig, + ProgramConfig, + create_fake_model, + create_quant_model, +) import hypothesis from hypothesis import given, settings @@ -32,22 +37,28 @@ import hypothesis.strategies as st logging.basicConfig(level=logging.INFO, format="%(message)s") -settings.register_profile("ci", - max_examples=100, - suppress_health_check=hypothesis.HealthCheck.all(), - deadline=None, - print_blob=True, - derandomize=True, - report_multiple_bugs=False) -settings.register_profile("dev", - max_examples=1000, - suppress_health_check=hypothesis.HealthCheck.all(), - deadline=None, - print_blob=True, - derandomize=True, - report_multiple_bugs=False) -if float(os.getenv('TEST_NUM_PERCENT_CASES', default='1.0')) < 1 or \ - os.getenv('HYPOTHESIS_TEST_PROFILE', 'dev') == 'ci': +settings.register_profile( + "ci", + max_examples=100, + suppress_health_check=hypothesis.HealthCheck.all(), + deadline=None, + print_blob=True, + derandomize=True, + report_multiple_bugs=False, +) +settings.register_profile( + "dev", + max_examples=1000, + suppress_health_check=hypothesis.HealthCheck.all(), + deadline=None, + print_blob=True, + derandomize=True, + report_multiple_bugs=False, +) +if ( + float(os.getenv('TEST_NUM_PERCENT_CASES', default='1.0')) < 1 + or os.getenv('HYPOTHESIS_TEST_PROFILE', 'dev') == 'ci' +): settings.load_profile("ci") else: settings.load_profile("dev") @@ -69,15 +80,15 @@ SkipReasons = IgnoreReasons class AutoScanTest(unittest.TestCase): - def __init__(self, *args, **kwargs): np.random.seed(1024) paddle.enable_static() super(AutoScanTest, self).__init__(*args, **kwargs) self.ignore_cases = [] abs_dir = os.path.abspath(os.path.dirname(__file__)) - self.cache_dir = os.path.join(abs_dir, - str(self.__module__) + '_cache_dir') + self.cache_dir = os.path.join( + abs_dir, str(self.__module__) + '_cache_dir' + ) self.available_passes_in_framework = set() self.num_ran_programs = 0 self.num_invalid_programs = 0 @@ -97,23 +108,29 @@ class AutoScanTest(unittest.TestCase): raise NotImplementedError @abc.abstractmethod - def add_ignore_check_case(self, teller: [ - Callable[[ProgramConfig, paddle_infer.Config], bool] - ], reason: IgnoreReasons, note: str): + def add_ignore_check_case( + self, + teller: [Callable[[ProgramConfig, paddle_infer.Config], bool]], + reason: IgnoreReasons, + note: str, + ): self.ignore_cases.append((teller, reason, note)) def is_program_valid(self, program_config: ProgramConfig) -> bool: return True - def run_test_config(self, model, params, prog_config, pred_config, - feed_data) -> Dict[str, np.ndarray]: + def run_test_config( + self, model, params, prog_config, pred_config, feed_data + ) -> Dict[str, np.ndarray]: ''' Test a single case. ''' pred_config.set_model_buffer(model, len(model), params, len(params)) predictor = paddle_infer.create_predictor(pred_config) - self.available_passes_in_framework = self.available_passes_in_framework | set( - pred_config.pass_builder().all_passes()) + self.available_passes_in_framework = ( + self.available_passes_in_framework + | set(pred_config.pass_builder().all_passes()) + ) for name, _ in prog_config.inputs.items(): input_tensor = predictor.get_input_handle(name) input_tensor.copy_from_cpu(feed_data[name]['data']) @@ -121,20 +138,28 @@ class AutoScanTest(unittest.TestCase): input_tensor.set_lod(feed_data[name]['lod']) predictor.run() result = {} - for out_name, o_name in zip(prog_config.outputs, - predictor.get_output_names()): + for out_name, o_name in zip( + prog_config.outputs, predictor.get_output_names() + ): result[out_name] = predictor.get_output_handle(o_name).copy_to_cpu() return result @abc.abstractmethod - def assert_tensors_near(self, atol: float, rtol: float, - tensor: Dict[str, np.array], - baseline: Dict[str, np.array]): + def assert_tensors_near( + self, + atol: float, + rtol: float, + tensor: Dict[str, np.array], + baseline: Dict[str, np.array], + ): for key, arr in tensor.items(): self.assertTrue( baseline[key].shape == arr.shape, - "The output shapes are not equal, the baseline shape is " + - str(baseline[key].shape) + ', but got ' + str(arr.shape)) + "The output shapes are not equal, the baseline shape is " + + str(baseline[key].shape) + + ', but got ' + + str(arr.shape), + ) diff = abs(baseline[key] - arr) np.testing.assert_allclose( baseline[key], @@ -142,22 +167,28 @@ class AutoScanTest(unittest.TestCase): rtol=rtol, atol=atol, err_msg='Output has diff, Maximum absolute error: {}'.format( - np.amax(diff))) + np.amax(diff) + ), + ) @abc.abstractmethod def run_test(self, quant=False): raise NotImplementedError - def generate_op_config(self, ops_config: List[Dict[str, - Any]]) -> List[OpConfig]: + def generate_op_config( + self, ops_config: List[Dict[str, Any]] + ) -> List[OpConfig]: ops = [] for i in range(len(ops_config)): op_config = ops_config[i] ops.append( - OpConfig(type=op_config['op_type'], - inputs=op_config['op_inputs'], - outputs=op_config['op_outputs'], - attrs=op_config['op_attrs'])) + OpConfig( + type=op_config['op_type'], + inputs=op_config['op_inputs'], + outputs=op_config['op_outputs'], + attrs=op_config['op_attrs'], + ) + ) return ops @abc.abstractmethod @@ -173,11 +204,13 @@ class AutoScanTest(unittest.TestCase): logging.info("SUCCESS: " + msg) @abc.abstractmethod - def create_inference_config(self, - passes: Optional[List[str]] = None, - use_gpu: bool = False, - use_mkldnn: bool = False, - ir_optim: Optional[bool] = None): + def create_inference_config( + self, + passes: Optional[List[str]] = None, + use_gpu: bool = False, + use_mkldnn: bool = False, + ir_optim: Optional[bool] = None, + ): config = paddle_infer.Config() config.switch_ir_debug(True) config.set_optim_cache_dir(self.cache_dir) @@ -195,7 +228,6 @@ class AutoScanTest(unittest.TestCase): class MkldnnAutoScanTest(AutoScanTest): - def __init__(self, *args, **kwargs): super(MkldnnAutoScanTest, self).__init__(*args, **kwargs) @@ -215,7 +247,7 @@ class MkldnnAutoScanTest(AutoScanTest): for name, tensor_config in prog_config.inputs.items(): feed_data[name] = { 'data': tensor_config.data, - 'lod': tensor_config.lod + 'lod': tensor_config.lod, } results: List[Dict[str, np.ndarray]] = [] @@ -223,23 +255,31 @@ class MkldnnAutoScanTest(AutoScanTest): base_config = self.create_inference_config(ir_optim=False) logging.info('RUN program_config: ' + str(prog_config)) results.append( - self.run_test_config(model, params, prog_config, base_config, - feed_data)) + self.run_test_config( + model, params, prog_config, base_config, feed_data + ) + ) self.success_log('RUN_CPU_BASELINE done') - for pred_config, ( - atol, rtol) in self.sample_predictor_configs(prog_config): + for pred_config, (atol, rtol) in self.sample_predictor_configs( + prog_config + ): # skip info ignore_flag = False for ignore_info in self.ignore_cases: if ignore_info[0](prog_config, pred_config): ignore_flag = True - if ignore_info[ - 1] == IgnoreReasons.MKLDNN_ACCURACY_ERROR: + if ( + ignore_info[1] + == IgnoreReasons.MKLDNN_ACCURACY_ERROR + ): self.ignore_log( - "[MKLDNN_ACCURACY_ERROR] " + ignore_info[2] + - ' ' + ' vs ' + - self.inference_config_str(pred_config)) + "[MKLDNN_ACCURACY_ERROR] " + + ignore_info[2] + + ' ' + + ' vs ' + + self.inference_config_str(pred_config) + ) else: raise NotImplementedError break @@ -251,20 +291,26 @@ class MkldnnAutoScanTest(AutoScanTest): try: results.append( - self.run_test_config(model, params, prog_config, - pred_config, feed_data)) - self.assert_tensors_near(atol, rtol, results[-1], - results[0]) + self.run_test_config( + model, params, prog_config, pred_config, feed_data + ) + ) + self.assert_tensors_near( + atol, rtol, results[-1], results[0] + ) except Exception as e: self.fail_log( - self.inference_config_str(pred_config) + - '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) + self.inference_config_str(pred_config) + + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)) + ) if not ignore_flag: status = False continue - self.success_log('RUN predictor_config ' + - self.inference_config_str(pred_config) + - ' done') + self.success_log( + 'RUN predictor_config ' + + self.inference_config_str(pred_config) + + ' done' + ) self.assertTrue(status) @@ -278,7 +324,6 @@ class MkldnnAutoScanTest(AutoScanTest): class PassAutoScanTest(AutoScanTest): - def __init__(self, *args, **kwargs): super(PassAutoScanTest, self).__init__(*args, **kwargs) self.passes = [] @@ -299,13 +344,17 @@ class PassAutoScanTest(AutoScanTest): def assert_op_list(self, op_list_after_fusion): if not self.passes: raise ValueError( - "In PassAutoScan you should give a valid pass name.") - last_passed_program = os.path.join(self.cache_dir, - self.passes[-1] + ".pdmodel") + "In PassAutoScan you should give a valid pass name." + ) + last_passed_program = os.path.join( + self.cache_dir, self.passes[-1] + ".pdmodel" + ) if not os.path.exists(last_passed_program): raise ValueError( - "Cannot find file {}, please make sure that your pass name is correct" - .format(last_passed_program)) + "Cannot find file {}, please make sure that your pass name is correct".format( + last_passed_program + ) + ) model_bytes = paddle.static.load_from_file(last_passed_program) pg = paddle.static.deserialize_program(model_bytes) main_block = pg.desc.block(0) @@ -317,16 +366,19 @@ class PassAutoScanTest(AutoScanTest): self.assertTrue( op_list_after_fusion == after_op_list, "Expected operator list after fusion is {}, but now it's {}".format( - op_list_after_fusion, after_op_list), + op_list_after_fusion, after_op_list + ), ) - def run_and_statis(self, - quant=False, - max_examples=100, - reproduce=None, - min_success_num=25, - max_duration=180, - passes=None): + def run_and_statis( + self, + quant=False, + max_examples=100, + reproduce=None, + min_success_num=25, + max_duration=180, + passes=None, + ): if os.getenv('HYPOTHESIS_TEST_PROFILE', 'ci') == "dev": max_examples *= 10 min_success_num *= 10 @@ -343,7 +395,9 @@ class PassAutoScanTest(AutoScanTest): report_multiple_bugs=False, ) settings.load_profile("ci") - assert passes is not None, "Parameter of passes must be defined in function run_and_statis." + assert ( + passes is not None + ), "Parameter of passes must be defined in function run_and_statis." self.passes = passes self.add_ignore_pass_case() @@ -361,32 +415,44 @@ class PassAutoScanTest(AutoScanTest): logging.info("Start to running test of {}".format(type(self))) loop_func() logging.info( - "===================Statistical Information===================") - logging.info("Number of Generated Programs: {}".format( - self.num_ran_programs + self.num_invalid_programs)) - logging.info("Number of Invalid Programs: {}".format( - self.num_invalid_programs)) + "===================Statistical Information===================" + ) + logging.info( + "Number of Generated Programs: {}".format( + self.num_ran_programs + self.num_invalid_programs + ) + ) + logging.info( + "Number of Invalid Programs: {}".format(self.num_invalid_programs) + ) logging.info("Number of Ran Programs: {}".format(self.num_ran_programs)) logging.info("Number of Ignore Tests: {}".format(self.num_ignore_tests)) - successful_ran_programs = int(self.num_ran_programs - - self.num_ignore_tests / - max(self.num_predictor_kinds, 1)) + successful_ran_programs = int( + self.num_ran_programs + - self.num_ignore_tests / max(self.num_predictor_kinds, 1) + ) logging.info( - "Number of successfully ran programs approximately equal to {}". - format(successful_ran_programs)) + "Number of successfully ran programs approximately equal to {}".format( + successful_ran_programs + ) + ) if successful_ran_programs < min_success_num: logging.warning( "satisfied_programs = ran_programs - num_ignore_tests / num_predictor_kinds" ) logging.error( - "At least {} programs need to ran successfully, but now only about {} programs satisfied." - .format(min_success_num, successful_ran_programs)) + "At least {} programs need to ran successfully, but now only about {} programs satisfied.".format( + min_success_num, successful_ran_programs + ) + ) assert False used_time = time.time() - start_time if max_duration > 0 and used_time > max_duration: logging.error( - "The duration exceeds {} seconds, if this is necessary, try to set a larger number for parameter `max_duration`." - .format(max_duration)) + "The duration exceeds {} seconds, if this is necessary, try to set a larger number for parameter `max_duration`.".format( + max_duration + ) + ) assert False def run_test(self, quant=False, prog_configs=None): @@ -406,13 +472,16 @@ class PassAutoScanTest(AutoScanTest): for name, tensor_config in prog_config.inputs.items(): feed_data[name] = { 'data': tensor_config.data, - 'lod': tensor_config.lod + 'lod': tensor_config.lod, } logging.info('RUN program_config: ' + str(prog_config)) self.num_predictor_kinds = 0 - for pred_config, op_list, ( - atol, rtol) in self.sample_predictor_configs(prog_config): + for ( + pred_config, + op_list, + (atol, rtol), + ) in self.sample_predictor_configs(prog_config): self.num_predictor_kinds += 1 # skip info @@ -423,9 +492,12 @@ class PassAutoScanTest(AutoScanTest): self.num_ignore_tests += 1 if ignore_info[1] == IgnoreReasons.PASS_ACCURACY_ERROR: self.ignore_log( - "[PASS_ACCURACY_ERROR] " + ignore_info[2] + - ' ' + ' vs ' + - self.inference_config_str(pred_config)) + "[PASS_ACCURACY_ERROR] " + + ignore_info[2] + + ' ' + + ' vs ' + + self.inference_config_str(pred_config) + ) else: raise NotImplementedError break @@ -437,37 +509,44 @@ class PassAutoScanTest(AutoScanTest): # baseline: no ir_optim run base_config = self.create_inference_config( - ir_optim=False, use_gpu=pred_config.use_gpu()) + ir_optim=False, use_gpu=pred_config.use_gpu() + ) try: # baseline - base_result = self.run_test_config(model, params, - prog_config, base_config, - feed_data) - self.success_log('RUN_BASELINE ' + - self.inference_config_str(base_config) + - ' done') + base_result = self.run_test_config( + model, params, prog_config, base_config, feed_data + ) + self.success_log( + 'RUN_BASELINE ' + + self.inference_config_str(base_config) + + ' done' + ) if os.path.exists(self.cache_dir): shutil.rmtree(self.cache_dir) - pred_result = self.run_test_config(model, params, - prog_config, pred_config, - feed_data) - self.assert_tensors_near(atol, rtol, pred_result, - base_result) + pred_result = self.run_test_config( + model, params, prog_config, pred_config, feed_data + ) + self.assert_tensors_near( + atol, rtol, pred_result, base_result + ) if not ignore_flag: self.assert_op_list(op_list) except Exception as e: self.fail_log( - self.inference_config_str(pred_config) + - '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) + self.inference_config_str(pred_config) + + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)) + ) if not ignore_flag: status = False continue - self.success_log('RUN predictor_config ' + - self.inference_config_str(pred_config) + - ' done') + self.success_log( + 'RUN predictor_config ' + + self.inference_config_str(pred_config) + + ' done' + ) status = self.check_op_version() and status self.assertTrue(status) @@ -502,14 +581,20 @@ class PassAutoScanTest(AutoScanTest): class TrtLayerAutoScanTest(AutoScanTest): - class TensorRTParam: ''' TensorRT subgraph engine parameters. ''' - def __init__(self, workspace_size, max_batch_size, min_subgraph_size, - precision, use_static, use_calib_mode): + def __init__( + self, + workspace_size, + max_batch_size, + min_subgraph_size, + precision, + use_static, + use_calib_mode, + ): self.workspace_size = workspace_size self.max_batch_size = max_batch_size self.min_subgraph_size = min_subgraph_size @@ -519,11 +604,16 @@ class TrtLayerAutoScanTest(AutoScanTest): class DynamicShapeParam: ''' - Prepare TensorRT subgraph engine dynamic shape parameters. - ''' + Prepare TensorRT subgraph engine dynamic shape parameters. + ''' - def __init__(self, min_input_shape, max_input_shape, opt_input_shape, - disable_trt_plugin_fp16): + def __init__( + self, + min_input_shape, + max_input_shape, + opt_input_shape, + disable_trt_plugin_fp16, + ): self.min_input_shape = min_input_shape self.max_input_shape = max_input_shape self.opt_input_shape = opt_input_shape @@ -537,10 +627,12 @@ class TrtLayerAutoScanTest(AutoScanTest): min_subgraph_size=0, precision=paddle_infer.PrecisionType.Float32, use_static=True, - use_calib_mode=False) + use_calib_mode=False, + ) self.dynamic_shape = self.DynamicShapeParam({}, {}, {}, False) self.num_percent_cases = float( - os.getenv('TEST_NUM_PERCENT_CASES', default='1.0')) + os.getenv('TEST_NUM_PERCENT_CASES', default='1.0') + ) # Use a seperate random generator for skipping tests self.skip_rng = np.random.default_rng(int(time.strftime("%W"))) @@ -558,31 +650,43 @@ class TrtLayerAutoScanTest(AutoScanTest): min_subgraph_size=self.trt_param.min_subgraph_size, precision_mode=self.trt_param.precision, use_static=self.trt_param.use_static, - use_calib_mode=self.trt_param.use_calib_mode) + use_calib_mode=self.trt_param.use_calib_mode, + ) if self.dynamic_shape.min_input_shape and ( - self.dynamic_shape.min_input_shape.keys() == - self.dynamic_shape.max_input_shape.keys() == - self.dynamic_shape.opt_input_shape.keys()): + self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.max_input_shape.keys() + == self.dynamic_shape.opt_input_shape.keys() + ): config.set_trt_dynamic_shape_info( self.dynamic_shape.min_input_shape, self.dynamic_shape.max_input_shape, self.dynamic_shape.opt_input_shape, - self.dynamic_shape.disable_trt_plugin_fp16) + self.dynamic_shape.disable_trt_plugin_fp16, + ) return config - def assert_tensors_near(self, atol: float, rtol: float, - tensor: Dict[str, np.array], - baseline: Dict[str, np.array]): + def assert_tensors_near( + self, + atol: float, + rtol: float, + tensor: Dict[str, np.array], + baseline: Dict[str, np.array], + ): for key, arr in tensor.items(): self.assertEqual( - baseline[key].shape, arr.shape, - 'The output shapes are not equal, the baseline shape is ' + - str(baseline[key].shape) + ', but got ' + str(arr.shape)) + baseline[key].shape, + arr.shape, + 'The output shapes are not equal, the baseline shape is ' + + str(baseline[key].shape) + + ', but got ' + + str(arr.shape), + ) np.testing.assert_allclose(baseline[key], arr, rtol=rtol, atol=atol) def assert_op_size(self, trt_engine_num, paddle_op_num): last_passed_program = os.path.join( - self.cache_dir, 'transpose_flatten_concat_fuse_pass.pdmodel') + self.cache_dir, 'transpose_flatten_concat_fuse_pass.pdmodel' + ) model_bytes = paddle.static.load_from_file(last_passed_program) pg = paddle.static.deserialize_program(model_bytes) main_block = pg.desc.block(0) @@ -593,13 +697,19 @@ class TrtLayerAutoScanTest(AutoScanTest): trt_engine_size = sum(op_types) paddle_op_size = op_size - trt_engine_size self.assertEqual( - trt_engine_num, trt_engine_size, + trt_engine_num, + trt_engine_size, 'Expected trt_engine_num is {}, but got {}!'.format( - trt_engine_num, trt_engine_size)) + trt_engine_num, trt_engine_size + ), + ) self.assertEqual( - paddle_op_num, paddle_op_size, + paddle_op_num, + paddle_op_size, 'Expected paddle_op_num is {}, but got {}!'.format( - paddle_op_num, paddle_op_size)) + paddle_op_num, paddle_op_size + ), + ) def inference_config_str(self, config: paddle_infer.Config) -> str: dic = {} @@ -639,21 +749,26 @@ class TrtLayerAutoScanTest(AutoScanTest): for name, tensor_config in prog_config.inputs.items(): feed_data[name] = { 'data': tensor_config.data, - 'lod': tensor_config.lod + 'lod': tensor_config.lod, } results: List[Dict[str, np.ndarray]] = [] if not skip_baseline: - #baseline: gpu run + # baseline: gpu run logging.info('RUN program_config: ' + str(prog_config)) gpu_config = self.create_inference_config(use_trt=False) results.append( - self.run_test_config(model, params, prog_config, gpu_config, - feed_data)) + self.run_test_config( + model, params, prog_config, gpu_config, feed_data + ) + ) self.success_log('RUN_GPU_BASELINE done') - for pred_config, nodes_num, threshold in self.sample_predictor_configs( - prog_config): + for ( + pred_config, + nodes_num, + threshold, + ) in self.sample_predictor_configs(prog_config): if os.path.exists(self.cache_dir): shutil.rmtree(self.cache_dir) @@ -662,17 +777,24 @@ class TrtLayerAutoScanTest(AutoScanTest): atol = threshold rtol = 1e-8 elif isinstance(threshold, list) or isinstance( - threshold, tuple): + threshold, tuple + ): atol = threshold[0] rtol = threshold[1] else: raise NotImplementedError - if pred_config.tensorrt_precision_mode( - ) != paddle_infer.PrecisionType.Int8 and quant: + if ( + pred_config.tensorrt_precision_mode() + != paddle_infer.PrecisionType.Int8 + and quant + ): continue - if pred_config.tensorrt_precision_mode( - ) == paddle_infer.PrecisionType.Int8 and not quant: + if ( + pred_config.tensorrt_precision_mode() + == paddle_infer.PrecisionType.Int8 + and not quant + ): continue ignore_flag = False @@ -682,11 +804,15 @@ class TrtLayerAutoScanTest(AutoScanTest): if reason == IgnoreReasons.TRT_NOT_IMPLEMENTED: self.ignore_log( '[TRT_NOT_IMPLEMENTED] {} vs {}'.format( - note, - self.inference_config_str(pred_config))) + note, self.inference_config_str(pred_config) + ) + ) elif reason == IgnoreReasons.TRT_NOT_SUPPORT: - self.ignore_log('[TRT_NOT_SUPPORT] {} vs {}'.format( - note, self.inference_config_str(pred_config))) + self.ignore_log( + '[TRT_NOT_SUPPORT] {} vs {}'.format( + note, self.inference_config_str(pred_config) + ) + ) else: raise NotImplementedError break @@ -697,30 +823,45 @@ class TrtLayerAutoScanTest(AutoScanTest): try: pred_config_deserialize = paddle_infer.Config(pred_config) results.append( - self.run_test_config(model, params, prog_config, - pred_config, feed_data)) - self.assert_tensors_near(atol, rtol, results[-1], - results[0]) + self.run_test_config( + model, params, prog_config, pred_config, feed_data + ) + ) + self.assert_tensors_near( + atol, rtol, results[-1], results[0] + ) trt_engine_num, paddle_op_num = nodes_num self.assert_op_size(trt_engine_num, paddle_op_num) # deserialize test if trt_engine_num > 0: - self.run_test_config(model, params, prog_config, - pred_config_deserialize, feed_data) - - self.success_log('RUN predictor_config {} done'.format( - self.inference_config_str(pred_config))) + self.run_test_config( + model, + params, + prog_config, + pred_config_deserialize, + feed_data, + ) + + self.success_log( + 'RUN predictor_config {} done'.format( + self.inference_config_str(pred_config) + ) + ) except Exception as e: self.fail_log( - self.inference_config_str(pred_config) + - '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) + self.inference_config_str(pred_config) + + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)) + ) all_passes = False self.assertTrue(all_passes) # TODO(wilber): just for backward compatible - def add_skip_case(self, teller: [ - Callable[[ProgramConfig, paddle_infer.Config], bool] - ], reason: IgnoreReasons, note: str): + def add_skip_case( + self, + teller: [Callable[[ProgramConfig, paddle_infer.Config], bool]], + reason: IgnoreReasons, + note: str, + ): self.ignore_cases.append((teller, reason, note)) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py b/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py index 60980fbb732..cc493349037 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py @@ -24,7 +24,6 @@ from paddle.fluid.core import create_paddle_predictor class InferencePassTest(unittest.TestCase): - def __init__(self, methodName='runTest'): paddle.enable_static() super(InferencePassTest, self).__init__(methodName) @@ -49,24 +48,28 @@ class InferencePassTest(unittest.TestCase): def _get_place(self): return set([False, core.is_compiled_with_cuda()]) - def _save_models(self, dirname, feeded_var_names, target_vars, executor, - program, scope): + def _save_models( + self, dirname, feeded_var_names, target_vars, executor, program, scope + ): with fluid.scope_guard(scope): # save models as combined to ensure that # there won't be too many useless files # after finishing a couple of tests. - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, program) + fluid.io.save_inference_model( + dirname, feeded_var_names, target_vars, executor, program + ) def _get_paddle_outs(self, executor, program, scope): ''' Return PaddlePaddle outputs. ''' with fluid.scope_guard(scope): - outs = executor.run(program=program, - feed=self.feeds, - fetch_list=self.fetch_list, - return_numpy=False) + outs = executor.run( + program=program, + feed=self.feeds, + fetch_list=self.fetch_list, + return_numpy=False, + ) return outs def _get_inference_outs(self, config): @@ -95,10 +98,9 @@ class InferencePassTest(unittest.TestCase): return outs - def _get_analysis_config(self, - use_gpu=False, - use_trt=False, - use_mkldnn=False): + def _get_analysis_config( + self, use_gpu=False, use_trt=False, use_mkldnn=False + ): ''' Return a new object of AnalysisConfig. ''' @@ -116,19 +118,22 @@ class InferencePassTest(unittest.TestCase): self.trt_parameters.min_subgraph_size, self.trt_parameters.precision, self.trt_parameters.use_static, - self.trt_parameters.use_calib_mode) + self.trt_parameters.use_calib_mode, + ) if self.trt_parameters.use_inspector: config.enable_tensorrt_inspector() self.assertTrue( config.tensorrt_inspector_enabled(), - "The inspector option is not set correctly.") + "The inspector option is not set correctly.", + ) if self.dynamic_shape_params: config.set_trt_dynamic_shape_info( self.dynamic_shape_params.min_input_shape, self.dynamic_shape_params.max_input_shape, self.dynamic_shape_params.optim_input_shape, - self.dynamic_shape_params.disable_trt_plugin_fp16) + self.dynamic_shape_params.disable_trt_plugin_fp16, + ) if self.enable_tensorrt_varseqlen: config.enable_tensorrt_varseqlen() @@ -145,18 +150,16 @@ class InferencePassTest(unittest.TestCase): or disable TensorRT, enable MKLDNN or disable MKLDNN are all the same. ''' - self.assertFalse(self.feeds is None, - "The inputs of the model is None. ") + self.assertFalse( + self.feeds is None, "The inputs of the model is None. " + ) use_gpu = self._get_place() for place_ in use_gpu: self.check_output_with_option(place_, atol) - def check_output_with_option(self, - use_gpu, - atol=1e-5, - flatten=False, - quant=False, - rtol=1e-5): + def check_output_with_option( + self, use_gpu, atol=1e-5, flatten=False, quant=False, rtol=1e-5 + ): ''' Check whether calculating on CPU and GPU, enable TensorRT or disable TensorRT, enable MKLDNN or disable MKLDNN @@ -168,17 +171,26 @@ class InferencePassTest(unittest.TestCase): device = "GPU" if use_gpu else "CPU" with fluid.scope_guard(scope): executor.run(self.startup_program) - self._save_models(self.path, list(self.feeds.keys()), self.fetch_list, - executor, self.main_program, scope) + self._save_models( + self.path, + list(self.feeds.keys()), + self.fetch_list, + executor, + self.main_program, + scope, + ) paddle_outs = self._get_paddle_outs(executor, self.main_program, scope) inference_outs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu)) + self._get_analysis_config(use_gpu=use_gpu) + ) # Check whether the results calculated on CPU and on GPU are the same. self.assertTrue( len(paddle_outs) == len(inference_outs), - "The number of outputs is different between inference and training forward at {}" - .format(device)) + "The number of outputs is different between inference and training forward at {}".format( + device + ), + ) for out, inference_out in zip(paddle_outs, inference_outs): paddle_out = np.array(out) @@ -191,28 +203,35 @@ class InferencePassTest(unittest.TestCase): inference_out, rtol=1e-05, atol=atol, - err_msg= - 'Output has diff between inference and training forward at {} '. - format(device)) + err_msg='Output has diff between inference and training forward at {} '.format( + device + ), + ) # Check whether the trt results and the GPU results are the same. if use_gpu and self.enable_trt: tensorrt_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_trt=self.enable_trt)) + self._get_analysis_config( + use_gpu=use_gpu, use_trt=self.enable_trt + ) + ) if self.trt_parameters.use_static: - #deserialize + # deserialize tensorrt_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_trt=self.enable_trt)) + self._get_analysis_config( + use_gpu=use_gpu, use_trt=self.enable_trt + ) + ) self.assertTrue( len(tensorrt_outputs) == len(paddle_outs), - "The number of outputs is different between GPU and TensorRT. ") + "The number of outputs is different between GPU and TensorRT. ", + ) - for paddle_out, tensorrt_output in zip(paddle_outs, - tensorrt_outputs): + for paddle_out, tensorrt_output in zip( + paddle_outs, tensorrt_outputs + ): paddle_out = np.array(paddle_out) if flatten: paddle_out = paddle_out.flatten() @@ -223,17 +242,21 @@ class InferencePassTest(unittest.TestCase): paddle_out, rtol=rtol, atol=atol, - err_msg='Output has diff between GPU and TensorRT. ') + err_msg='Output has diff between GPU and TensorRT. ', + ) # Check whether the mkldnn results and the CPU results are the same. if (not use_gpu) and self.enable_mkldnn: mkldnn_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_mkldnn=self.enable_mkldnn)) + self._get_analysis_config( + use_gpu=use_gpu, use_mkldnn=self.enable_mkldnn + ) + ) self.assertTrue( len(paddle_outs) == len(mkldnn_outputs), - "The number of outputs is different between CPU and MKLDNN. ") + "The number of outputs is different between CPU and MKLDNN. ", + ) if self.enable_mkldnn_bfloat16: atol = 0.01 @@ -243,21 +266,24 @@ class InferencePassTest(unittest.TestCase): mkldnn_output, rtol=1e-05, atol=atol, - err_msg='Output has diff between CPU and MKLDNN. ') + err_msg='Output has diff between CPU and MKLDNN. ', + ) class TensorRTParam: ''' Prepare TensorRT subgraph engine parameters. ''' - def __init__(self, - workspace_size, - max_batch_size, - min_subgraph_size, - precision, - use_static, - use_calib_mode, - use_inspector=False): + def __init__( + self, + workspace_size, + max_batch_size, + min_subgraph_size, + precision, + use_static, + use_calib_mode, + use_inspector=False, + ): self.workspace_size = workspace_size self.max_batch_size = max_batch_size self.min_subgraph_size = min_subgraph_size @@ -271,8 +297,13 @@ class InferencePassTest(unittest.TestCase): Prepare TensorRT subgraph engine dynamic shape parameters. ''' - def __init__(self, min_input_shape, max_input_shape, optim_input_shape, - disable_trt_plugin_fp16): + def __init__( + self, + min_input_shape, + max_input_shape, + optim_input_shape, + disable_trt_plugin_fp16, + ): self.min_input_shape = min_input_shape self.max_input_shape = max_input_shape self.optim_input_shape = optim_input_shape diff --git a/python/paddle/fluid/tests/unittests/ir/inference/program_config.py b/python/paddle/fluid/tests/unittests/ir/inference/program_config.py index 5ad6f1c885c..74375d9367d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/program_config.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/program_config.py @@ -32,10 +32,12 @@ class TensorConfig: A config builder for a input or a weight. ''' - def __init__(self, - lod: Optional[List[List[int]]] = None, - data_gen: Optional[Callable[..., np.array]] = None, - shape: Optional[List[List[int]]] = None): + def __init__( + self, + lod: Optional[List[List[int]]] = None, + data_gen: Optional[Callable[..., np.array]] = None, + shape: Optional[List[List[int]]] = None, + ): ''' shape: The shape of the tensor. dtype: The data type of the tensor. @@ -48,7 +50,9 @@ class TensorConfig: self.dtype = data_gen().dtype self.shape = data_gen().shape else: - assert shape is not None, "While data_gen is not defined, shape must not be None" + assert ( + shape is not None + ), "While data_gen is not defined, shape must not be None" self.data = np.random.normal(0.0, 1.0, shape).astype(np.float32) self.shape = shape self.dtype = self.data.dtype @@ -64,16 +68,18 @@ class VarType(enum.Enum): class OpConfig: - ''' A config builder for generating a Op. ''' - - def __init__(self, - type: str, - inputs: Dict[str, List[str]], - outputs: Dict[str, List[str]], - attrs: Dict[str, Any] = None, - outputs_var_type: Dict[str, VarType] = None, - outputs_dtype: Dict[str, np.dtype] = None, - **kwargs): + '''A config builder for generating a Op.''' + + def __init__( + self, + type: str, + inputs: Dict[str, List[str]], + outputs: Dict[str, List[str]], + attrs: Dict[str, Any] = None, + outputs_var_type: Dict[str, VarType] = None, + outputs_dtype: Dict[str, np.dtype] = None, + **kwargs, + ): self.type = type self.inputs = inputs self.outputs = outputs @@ -91,26 +97,50 @@ class OpConfig: _OP_WITHOUT_KERNEL_SET = { - 'feed', 'fetch', 'recurrent', 'go', 'rnn_memory_helper_grad', - 'conditional_block', 'while', 'send', 'recv', 'listen_and_serv', - 'fl_listen_and_serv', 'ncclInit', 'select', 'checkpoint_notify', - 'gen_bkcl_id', 'c_gen_bkcl_id', 'gen_nccl_id', 'c_gen_nccl_id', - 'c_comm_init', 'c_sync_calc_stream', 'c_sync_comm_stream', - 'queue_generator', 'dequeue', 'enqueue', 'heter_listen_and_serv', - 'c_wait_comm', 'c_wait_compute', 'c_gen_hccl_id', 'c_comm_init_hccl', - 'copy_cross_scope' + 'feed', + 'fetch', + 'recurrent', + 'go', + 'rnn_memory_helper_grad', + 'conditional_block', + 'while', + 'send', + 'recv', + 'listen_and_serv', + 'fl_listen_and_serv', + 'ncclInit', + 'select', + 'checkpoint_notify', + 'gen_bkcl_id', + 'c_gen_bkcl_id', + 'gen_nccl_id', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_sync_calc_stream', + 'c_sync_comm_stream', + 'queue_generator', + 'dequeue', + 'enqueue', + 'heter_listen_and_serv', + 'c_wait_comm', + 'c_wait_compute', + 'c_gen_hccl_id', + 'c_comm_init_hccl', + 'copy_cross_scope', } class BlockConfig: - ''' A config builder for generating a Block. ''' - - def __init__(self, - ops: List[OpConfig], - vars: List[str], - vars_dtype: Dict[str, np.dtype] = None, - vars_var_type: Dict[str, VarType] = None, - vars_lod_level: Dict[str, int] = None): + '''A config builder for generating a Block.''' + + def __init__( + self, + ops: List[OpConfig], + vars: List[str], + vars_dtype: Dict[str, np.dtype] = None, + vars_var_type: Dict[str, VarType] = None, + vars_lod_level: Dict[str, int] = None, + ): self.ops = ops self.vars = vars self.vars_dtype = vars_dtype @@ -121,10 +151,14 @@ class BlockConfig: for name in self.vars: var_desc = block_desc.var(name.encode()) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) - if self.vars_lod_level is not None and name in self.vars_lod_level.keys( + if ( + self.vars_lod_level is not None + and name in self.vars_lod_level.keys() ): var_desc.set_lod_level(self.vars_lod_level[name]) - if self.vars_var_type is not None and name in self.vars_var_type.keys( + if ( + self.vars_var_type is not None + and name in self.vars_var_type.keys() ): if self.vars_var_type[name] == VarType.LOD_TENSOR_ARRAY: var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR_ARRAY) @@ -134,7 +168,8 @@ class BlockConfig: var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) if self.vars_dtype is not None and name in self.vars_dtype.keys(): var_desc.set_dtype( - convert_np_dtype_to_dtype_(self.vars_dtype[name])) + convert_np_dtype_to_dtype_(self.vars_dtype[name]) + ) for op_config in self.ops: op_desc = block_desc.append_op() @@ -150,22 +185,32 @@ class BlockConfig: continue var_desc = block_desc.var(v.encode()) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) - if op_config.outputs_var_type is not None and v in op_config.outputs_var_type.keys( + if ( + op_config.outputs_var_type is not None + and v in op_config.outputs_var_type.keys() ): - if op_config.outputs_var_type[ - v] == VarType.LOD_TENSOR_ARRAY: + if ( + op_config.outputs_var_type[v] + == VarType.LOD_TENSOR_ARRAY + ): var_desc.set_type( - core.VarDesc.VarType.LOD_TENSOR_ARRAY) - elif op_config.outputs_var_type[ - v] == VarType.STEP_SCOPES: + core.VarDesc.VarType.LOD_TENSOR_ARRAY + ) + elif ( + op_config.outputs_var_type[v] == VarType.STEP_SCOPES + ): var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES) continue var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) - if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys( + if ( + op_config.outputs_dtype is not None + and v in op_config.outputs_dtype.keys() ): var_desc.set_dtype( convert_np_dtype_to_dtype_( - op_config.outputs_dtype[v])) + op_config.outputs_dtype[v] + ) + ) if op_config.type not in _OP_WITHOUT_KERNEL_SET: op_desc.infer_var_type(block_desc) op_desc.infer_shape(block_desc) @@ -173,10 +218,15 @@ class BlockConfig: class ProgramConfig: - ''' A config builder for generating a Program. ''' - - def __init__(self, ops: List[OpConfig], weights: Dict[str, TensorConfig], - inputs: Dict[str, TensorConfig], outputs: List[str]): + '''A config builder for generating a Program.''' + + def __init__( + self, + ops: List[OpConfig], + weights: Dict[str, TensorConfig], + inputs: Dict[str, TensorConfig], + outputs: List[str], + ): self.ops = ops # if no weight need to save, we create a place_holder to help seriazlie params. if not weights: @@ -209,7 +259,7 @@ class ProgramConfig: def create_fake_model(program_config): - ''' Create a Paddle model(in memory) according to the given config. ''' + '''Create a Paddle model(in memory) according to the given config.''' paddle.enable_static() main_program_desc = core.ProgramDesc() util_program = fluid.Program() @@ -249,21 +299,22 @@ def create_fake_model(program_config): shape=tensor_config.shape, type=core.VarDesc.VarType.LOD_TENSOR, name=name, - initializer=NumpyArrayInitializer(tensor_config.data)) + initializer=NumpyArrayInitializer(tensor_config.data), + ) in_vars = [] for name in sorted(save_var_map.keys()): in_vars.append(save_var_map[name]) out_var = util_program.global_block().create_var( - type=core.VarDesc.VarType.RAW, name="out_var_0") + type=core.VarDesc.VarType.RAW, name="out_var_0" + ) out_var.desc.set_persistable(True) - util_program.global_block().append_op(type='save_combine', - inputs={'X': in_vars}, - outputs={'Y': out_var}, - attrs={ - 'file_path': '', - 'save_to_memory': True - }) + util_program.global_block().append_op( + type='save_combine', + inputs={'X': in_vars}, + outputs={'Y': out_var}, + attrs={'file_path': '', 'save_to_memory': True}, + ) for op_config in program_config.ops: op_desc = main_block_desc.append_op() op_desc.set_type(op_config.type) @@ -283,19 +334,26 @@ def create_fake_model(program_config): continue var_desc = main_block_desc.var(v.encode()) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) - if op_config.outputs_var_type is not None and v in op_config.outputs_var_type.keys( + if ( + op_config.outputs_var_type is not None + and v in op_config.outputs_var_type.keys() ): - if op_config.outputs_var_type[ - v] == VarType.LOD_TENSOR_ARRAY: + if ( + op_config.outputs_var_type[v] + == VarType.LOD_TENSOR_ARRAY + ): var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR_ARRAY) elif op_config.outputs_var_type[v] == VarType.STEP_SCOPES: var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES) continue var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) - if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys( + if ( + op_config.outputs_dtype is not None + and v in op_config.outputs_dtype.keys() ): var_desc.set_dtype( - convert_np_dtype_to_dtype_(op_config.outputs_dtype[v])) + convert_np_dtype_to_dtype_(op_config.outputs_dtype[v]) + ) if op_config.type not in _OP_WITHOUT_KERNEL_SET: op_desc.infer_var_type(main_block_desc) op_desc.infer_shape(main_block_desc) @@ -327,19 +385,26 @@ def create_fake_model(program_config): return model, params -def create_quant_model(model, - params, - activation_quantize_type='moving_average_abs_max', - weight_quantize_type='channel_wise_abs_max', - save=False): +def create_quant_model( + model, + params, + activation_quantize_type='moving_average_abs_max', + weight_quantize_type='channel_wise_abs_max', + save=False, +): place = paddle.CUDAPlace(0) scope = global_scope() exe = paddle.static.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model(path_prefix=None, - executor=exe, - model_filename=model, - params_filename=params) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model( + path_prefix=None, + executor=exe, + model_filename=model, + params_filename=params, + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) out_scale_op_list = [ @@ -431,11 +496,11 @@ def create_quant_model(model, def _get_op_output_var_names(op): """ """ - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." var_names = [] - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in op_real_in_out_name: return [] @@ -452,7 +517,8 @@ def create_quant_model(model, scope=scope, place=place, activation_quantize_type=activation_quantize_type, - weight_quantize_type=weight_quantize_type) + weight_quantize_type=weight_quantize_type, + ) transform_pass.apply(graph) op_nodes = graph.all_op_nodes() @@ -461,15 +527,18 @@ def create_quant_model(model, var_names = _get_op_output_var_names(op_node) for var_name in var_names: in_node = graph._find_node_by_name(op_node.outputs, var_name) - if in_node.dtype() not in \ - [core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]: + if in_node.dtype() not in [ + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, + ]: continue op_node.op()._set_attr("out_threshold", 3.0) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( - scope=scope, place=place, weight_quantize_type=weight_quantize_type) + scope=scope, place=place, weight_quantize_type=weight_quantize_type + ) freeze_pass.apply(graph) main_program = graph.to_program() @@ -487,18 +556,21 @@ def create_quant_model(model, tensor.set(np.ones(tensor.shape(), dtype=np.float32), place) if save: - fluid.io.save_inference_model('test_inference_model', - feed_target_names, - fetch_targets, - exe, - main_program=main_program) + fluid.io.save_inference_model( + 'test_inference_model', + feed_target_names, + fetch_targets, + exe, + main_program=main_program, + ) feed_vars = [ main_program.global_block().var(name) for name in feed_target_names ] - serialized_program = paddle.static.serialize_program(feed_vars, - fetch_targets, - program=main_program) + serialized_program = paddle.static.serialize_program( + feed_vars, fetch_targets, program=main_program + ) serialized_params = paddle.static.serialize_persistables( - feed_vars, fetch_targets, executor=exe, program=main_program) + feed_vars, fetch_targets, executor=exe, program=main_program + ) return serialized_program, serialized_params diff --git a/python/paddle/fluid/tests/unittests/ir/inference/quant_dequant_test.py b/python/paddle/fluid/tests/unittests/ir/inference/quant_dequant_test.py index c41e505332f..bdbff1a385d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/quant_dequant_test.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/quant_dequant_test.py @@ -24,14 +24,13 @@ from paddle.fluid.contrib.slim.quantization import QuantizationFreezePass from paddle.fluid.contrib.slim.quantization import OutScaleForTrainingPass from paddle.fluid.contrib.slim.quantization import OutScaleForInferencePass from paddle.fluid.contrib.slim.quantization import AddQuantDequantPass -from paddle.fluid import (core, Program, Variable) +from paddle.fluid import core, Program, Variable from paddle.fluid.io import prepend_feed_ops, append_fetch_ops from paddle.fluid.core import create_paddle_predictor from paddle.fluid.core import AnalysisConfig class QuantDequantTest(unittest.TestCase): - def __init__(self, methodName='runTest'): super(QuantDequantTest, self).__init__(methodName) paddle.enable_static() @@ -60,28 +59,34 @@ class QuantDequantTest(unittest.TestCase): def _normalize_program(self, program, feed_vars, fetch_vars): if not isinstance(program, Program): raise TypeError( - "program type must be `fluid.Program`, but received `%s`" % - type(program)) + "program type must be `fluid.Program`, but received `%s`" + % type(program) + ) if not isinstance(feed_vars, list): feed_vars = [feed_vars] if not all(isinstance(v, Variable) for v in feed_vars): raise TypeError( - "feed_vars type must be a Variable or a list of Variable.") + "feed_vars type must be a Variable or a list of Variable." + ) if not isinstance(fetch_vars, list): fetch_vars = [fetch_vars] if not all(isinstance(v, Variable) for v in fetch_vars): raise TypeError( - "fetch_vars type must be a Variable or a list of Variable.") + "fetch_vars type must be a Variable or a list of Variable." + ) # remind users to set auc_states to 0 if auc op were found. for op in program.global_block().ops: # clear device of Op - device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) op._set_attr(device_attr_name, "") if op.type == 'auc': - warnings.warn("Be sure that you have set auc states to 0 " - "before saving inference model.") + warnings.warn( + "Be sure that you have set auc states to 0 " + "before saving inference model." + ) break # serialize program @@ -98,7 +103,8 @@ class QuantDequantTest(unittest.TestCase): feed_var_names = [var.name for var in feed_vars] copy_program = copy_program._prune_with_input( - feeded_var_names=feed_var_names, targets=fetch_vars) + feeded_var_names=feed_var_names, targets=fetch_vars + ) copy_program = copy_program._inference_optimize(prune_read_op=True) fetch_var_names = [var.name for var in fetch_vars] prepend_feed_ops(copy_program, feed_var_names) @@ -106,25 +112,30 @@ class QuantDequantTest(unittest.TestCase): copy_program.desc._set_version() return copy_program - def _save_models(self, dirname, feeded_var_names, target_vars, executor, - program, scope): + def _save_models( + self, dirname, feeded_var_names, target_vars, executor, program, scope + ): with fluid.scope_guard(scope): - fluid.io.save_inference_model(dirname, - feeded_var_names, - target_vars, - executor, - program, - clip_extra=True) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + program, + clip_extra=True, + ) def _get_paddle_outs(self, feed, fetch_list, executor, program, scope): ''' Return PaddlePaddle outputs. ''' with fluid.scope_guard(scope): - outs = executor.run(program=program, - feed=feed, - fetch_list=fetch_list, - return_numpy=True) + outs = executor.run( + program=program, + feed=feed, + fetch_list=fetch_list, + return_numpy=True, + ) return outs def _get_inference_outs(self, config): @@ -152,10 +163,9 @@ class QuantDequantTest(unittest.TestCase): ] return outs - def _get_analysis_config(self, - use_gpu=False, - use_trt=False, - use_mkldnn=False): + def _get_analysis_config( + self, use_gpu=False, use_trt=False, use_mkldnn=False + ): ''' Return a new object of AnalysisConfig. ''' @@ -173,14 +183,16 @@ class QuantDequantTest(unittest.TestCase): self.trt_parameters.min_subgraph_size, self.trt_parameters.precision, self.trt_parameters.use_static, - self.trt_parameters.use_calib_mode) + self.trt_parameters.use_calib_mode, + ) if self.dynamic_shape_params: config.set_trt_dynamic_shape_info( self.dynamic_shape_params.min_input_shape, self.dynamic_shape_params.max_input_shape, self.dynamic_shape_params.optim_input_shape, - self.dynamic_shape_params.disable_trt_plugin_fp16) + self.dynamic_shape_params.disable_trt_plugin_fp16, + ) if self.enable_tensorrt_varseqlen: config.enable_tensorrt_varseqlen() @@ -191,12 +203,9 @@ class QuantDequantTest(unittest.TestCase): print('config summary:', config.summary()) return config - def check_output_with_option(self, - use_gpu, - atol=1e-5, - flatten=False, - quant=False, - rtol=1e-5): + def check_output_with_option( + self, use_gpu, atol=1e-5, flatten=False, quant=False, rtol=1e-5 + ): ''' Check whether calculating on CPU and GPU, enable TensorRT or disable TensorRT, enable MKLDNN or disable MKLDNN @@ -211,14 +220,16 @@ class QuantDequantTest(unittest.TestCase): executor.run(self.startup_program) executor.run(self.test_startup_program) main_graph = IrGraph(core.Graph(self.main_program.desc), for_test=False) - test_graph = IrGraph(core.Graph(self.test_main_program.desc), - for_test=True) + test_graph = IrGraph( + core.Graph(self.test_main_program.desc), for_test=True + ) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=self.activation_quantize_type, - weight_quantize_type=self.weight_quantize_type) + weight_quantize_type=self.weight_quantize_type, + ) transform_pass.apply(main_graph) transform_pass.apply(test_graph) @@ -237,17 +248,19 @@ class QuantDequantTest(unittest.TestCase): iters = 10 batch_size = 1 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) - feeder = fluid.DataFeeder(feed_list=[self.data, self.label], - place=place) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) + feeder = fluid.DataFeeder( + feed_list=[self.data, self.label], place=place + ) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = executor.run(binary, - feed=feeder.feed(data), - fetch_list=[self.loss]) + loss_v = executor.run( + binary, feed=feeder.feed(data), fetch_list=[self.loss] + ) scale_inference_pass = OutScaleForInferencePass(scope=scope) scale_inference_pass.apply(test_graph) @@ -256,29 +269,40 @@ class QuantDequantTest(unittest.TestCase): freeze_pass = QuantizationFreezePass( scope=scope, place=place, - weight_quantize_type=self.weight_quantize_type) + weight_quantize_type=self.weight_quantize_type, + ) freeze_pass.apply(test_graph) self.main_program = test_graph.to_program() with fluid.scope_guard(scope): - self.main_program = self._normalize_program(self.main_program, - self.data, - self.fetch_list) - - self._save_models(self.path, list(self.feeds.keys()), self.fetch_list, - executor, self.main_program, scope) + self.main_program = self._normalize_program( + self.main_program, self.data, self.fetch_list + ) - paddle_outs = self._get_paddle_outs(self.feeds, self.fetch_list, - executor, self.main_program, scope) + self._save_models( + self.path, + list(self.feeds.keys()), + self.fetch_list, + executor, + self.main_program, + scope, + ) + + paddle_outs = self._get_paddle_outs( + self.feeds, self.fetch_list, executor, self.main_program, scope + ) inference_outs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu)) + self._get_analysis_config(use_gpu=use_gpu) + ) # Check whether the results calculated on CPU and on GPU are the same. self.assertTrue( len(paddle_outs) == len(inference_outs), - "The number of outputs is different between inference and training forward at {}" - .format(device)) + "The number of outputs is different between inference and training forward at {}".format( + device + ), + ) for out, inference_out in zip(paddle_outs, inference_outs): paddle_out = np.array(out) @@ -292,28 +316,35 @@ class QuantDequantTest(unittest.TestCase): inference_out, rtol=1e-05, atol=atol, - err_msg= - 'Output has diff between inference and training forward at {} '. - format(device)) + err_msg='Output has diff between inference and training forward at {} '.format( + device + ), + ) # Check whether the trt results and the GPU results are the same. if use_gpu and self.enable_trt: tensorrt_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_trt=self.enable_trt)) + self._get_analysis_config( + use_gpu=use_gpu, use_trt=self.enable_trt + ) + ) if self.trt_parameters.use_static: - #deserialize + # deserialize tensorrt_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_trt=self.enable_trt)) + self._get_analysis_config( + use_gpu=use_gpu, use_trt=self.enable_trt + ) + ) self.assertTrue( len(tensorrt_outputs) == len(paddle_outs), - "The number of outputs is different between GPU and TensorRT. ") + "The number of outputs is different between GPU and TensorRT. ", + ) - for paddle_out, tensorrt_output in zip(paddle_outs, - tensorrt_outputs): + for paddle_out, tensorrt_output in zip( + paddle_outs, tensorrt_outputs + ): paddle_out = np.array(paddle_out) if flatten: @@ -325,17 +356,21 @@ class QuantDequantTest(unittest.TestCase): tensorrt_output, rtol=rtol, atol=atol, - err_msg='Output has diff between GPU and TensorRT. ') + err_msg='Output has diff between GPU and TensorRT. ', + ) # Check whether the mkldnn results and the CPU results are the same. if (not use_gpu) and self.enable_mkldnn: mkldnn_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_mkldnn=self.enable_mkldnn)) + self._get_analysis_config( + use_gpu=use_gpu, use_mkldnn=self.enable_mkldnn + ) + ) self.assertTrue( len(paddle_outs) == len(mkldnn_outputs), - "The number of outputs is different between CPU and MKLDNN. ") + "The number of outputs is different between CPU and MKLDNN. ", + ) if self.enable_mkldnn_bfloat16: atol = 0.01 @@ -345,15 +380,23 @@ class QuantDequantTest(unittest.TestCase): mkldnn_output, rtol=1e-05, atol=atol, - err_msg='Output has diff between CPU and MKLDNN. ') + err_msg='Output has diff between CPU and MKLDNN. ', + ) class TensorRTParam: ''' Prepare TensorRT subgraph engine parameters. ''' - def __init__(self, workspace_size, max_batch_size, min_subgraph_size, - precision, use_static, use_calib_mode): + def __init__( + self, + workspace_size, + max_batch_size, + min_subgraph_size, + precision, + use_static, + use_calib_mode, + ): self.workspace_size = workspace_size self.max_batch_size = max_batch_size self.min_subgraph_size = min_subgraph_size @@ -366,8 +409,13 @@ class QuantDequantTest(unittest.TestCase): Prepare TensorRT subgraph engine dynamic shape parameters. ''' - def __init__(self, min_input_shape, max_input_shape, optim_input_shape, - disable_trt_plugin_fp16): + def __init__( + self, + min_input_shape, + max_input_shape, + optim_input_shape, + disable_trt_plugin_fp16, + ): self.min_input_shape = min_input_shape self.max_input_shape = max_input_shape self.optim_input_shape = optim_input_shape diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_adaptive_pool2d_convert_global_pass_autoscan.py b/python/paddle/fluid/tests/unittests/ir/inference/test_adaptive_pool2d_convert_global_pass_autoscan.py index aa2d29d6137..c0814dc920b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_adaptive_pool2d_convert_global_pass_autoscan.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_adaptive_pool2d_convert_global_pass_autoscan.py @@ -21,55 +21,60 @@ import hypothesis.strategies as st class TestAdaptivePool2dConvertGlobalPass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_config(self, draw): x_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) pooling_type = draw(st.sampled_from(["max", "avg"])) - data_format = "NCHW" #trt support this format only + data_format = "NCHW" # trt support this format only strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) paddings = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) ceil_mode = draw(st.booleans()) exclusive = draw(st.booleans()) global_pooling = draw(st.booleans()) padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VAILD"])) - pool_op = OpConfig("pool2d", - inputs={"X": ["input_data"]}, - outputs={"Out": ["pool_output"]}, - ksize=[1, 1], - adaptive=True, - pooling_type=pooling_type, - data_format=data_format, - strides=strides, - paddings=paddings, - ceil_mode=ceil_mode, - global_pooling=global_pooling, - padding_algorithm=padding_algorithm, - exclusive=exclusive) + pool_op = OpConfig( + "pool2d", + inputs={"X": ["input_data"]}, + outputs={"Out": ["pool_output"]}, + ksize=[1, 1], + adaptive=True, + pooling_type=pooling_type, + data_format=data_format, + strides=strides, + paddings=paddings, + ceil_mode=ceil_mode, + global_pooling=global_pooling, + padding_algorithm=padding_algorithm, + exclusive=exclusive, + ) ops = [pool_op] - program_config = ProgramConfig(ops=ops, - weights={}, - inputs={ - "input_data": - TensorConfig(shape=x_shape), - }, - outputs=["pool_output"]) + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig(shape=x_shape), + }, + outputs=["pool_output"], + ) return program_config @@ -81,14 +86,17 @@ class TestAdaptivePool2dConvertGlobalPass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['pool2d'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["adaptive_pool2d_convert_global_pass"], - min_success_num=40) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["adaptive_pool2d_convert_global_pass"], + min_success_num=40, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py index d619438de4e..fdc995fee78 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py @@ -48,12 +48,36 @@ class TestConvActMkldnnFusePass(PassAutoScanTest): filter_shape = prog_config.weights["filter"].shape input_shape = prog_config.inputs["input_x"].shape if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -70,9 +94,10 @@ class TestConvActMkldnnFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=5, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=5, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=5, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -80,9 +105,10 @@ class TestConvActMkldnnFusePass(PassAutoScanTest): # 3. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -90,35 +116,41 @@ class TestConvActMkldnnFusePass(PassAutoScanTest): # 4. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 5. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) # 6. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=4, max_size=4 + ) + ) # 7. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 8. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 9. Generate legal input:ResidualData of conv2d res_shape = [] if draw(st.booleans()): res_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), + min_size=4, + max_size=4, + ) + ) # 10. Generate legal shape of input:bias of conv2d conv_bias_shape = [] @@ -135,58 +167,69 @@ class TestConvActMkldnnFusePass(PassAutoScanTest): } weights = { "filter": TensorConfig(shape=f_shape), - "conv_bias": TensorConfig(shape=conv_bias_shape) + "conv_bias": TensorConfig(shape=conv_bias_shape), } use_mkldnn = True else: inputs = { "Input": ["input_x"], "Filter": ["filter"], - "ResidualData": ["residualdata"] + "ResidualData": ["residualdata"], } weights = {"filter": TensorConfig(shape=f_shape)} use_mkldnn = False # 11. Generate legal act type of conv2d act_type = draw( - st.sampled_from(["relu", "leaky_relu", "relu6", "swish"])) - - conv2d_op = OpConfig("conv2d", - inputs=inputs, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format, - use_mkldnn=True) + st.sampled_from(["relu", "leaky_relu", "relu6", "swish"]) + ) + + conv2d_op = OpConfig( + "conv2d", + inputs=inputs, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + use_mkldnn=True, + ) # 11. Generate legal attr of act act_op = None self.passes = ["conv_activation_mkldnn_fuse_pass"] if act_type == "relu6": threshold = draw(st.floats(min_value=1.0, max_value=10.0)) - act_op = OpConfig("relu6", - inputs={"X": ["conv2d_out"]}, - outputs={"Out": ["relu_out"]}, - threshold=threshold) + act_op = OpConfig( + "relu6", + inputs={"X": ["conv2d_out"]}, + outputs={"Out": ["relu_out"]}, + threshold=threshold, + ) elif act_type == "leaky_relu": alpha = draw(st.floats(min_value=0.1, max_value=1.0)) - act_op = OpConfig("leaky_relu", - inputs={"X": ["conv2d_out"]}, - outputs={"Out": ["relu_out"]}, - alpha=alpha) + act_op = OpConfig( + "leaky_relu", + inputs={"X": ["conv2d_out"]}, + outputs={"Out": ["relu_out"]}, + alpha=alpha, + ) elif act_type == "relu": - act_op = OpConfig("relu", - inputs={"X": ["conv2d_out"]}, - outputs={"Out": ["relu_out"]}) + act_op = OpConfig( + "relu", + inputs={"X": ["conv2d_out"]}, + outputs={"Out": ["relu_out"]}, + ) elif act_type == "swish": beta = draw(st.floats(min_value=0.1, max_value=1.0)) - act_op = OpConfig("swish", - inputs={"X": ["conv2d_out"]}, - outputs={"Out": ["swish_out"]}, - beta=beta) + act_op = OpConfig( + "swish", + inputs={"X": ["conv2d_out"]}, + outputs={"Out": ["swish_out"]}, + beta=beta, + ) ops = [conv2d_op, act_op] @@ -195,7 +238,7 @@ class TestConvActMkldnnFusePass(PassAutoScanTest): weights=weights, inputs={ "input_x": TensorConfig(shape=x_shape), - "residualdata": TensorConfig(shape=res_shape) + "residualdata": TensorConfig(shape=res_shape), }, outputs=ops[-1].outputs["Out"], ) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bias_mkldnn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bias_mkldnn_fuse_pass.py index eb02c6e1c77..ec7c8db66d0 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bias_mkldnn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bias_mkldnn_fuse_pass.py @@ -48,12 +48,36 @@ class TestConvBiasMkldnnFusePass(PassAutoScanTest): filter_shape = prog_config.weights["filter"].shape input_shape = prog_config.inputs["input_x"].shape if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -70,9 +94,10 @@ class TestConvBiasMkldnnFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=5, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=5, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=5, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -80,9 +105,10 @@ class TestConvBiasMkldnnFusePass(PassAutoScanTest): # 3. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -90,27 +116,30 @@ class TestConvBiasMkldnnFusePass(PassAutoScanTest): # 4. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) # 5. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) # 6. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) # 7. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 8. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) # 9. Generate legal shape of input:bias of elementwise_add bias_shape = [f_shape[0]] @@ -137,7 +166,7 @@ class TestConvBiasMkldnnFusePass(PassAutoScanTest): weights = { "filter": TensorConfig(shape=f_shape), "bias": TensorConfig(shape=bias_shape), - "conv_bias": TensorConfig(shape=conv_bias_shape) + "conv_bias": TensorConfig(shape=conv_bias_shape), } use_mkldnn = True else: @@ -147,28 +176,29 @@ class TestConvBiasMkldnnFusePass(PassAutoScanTest): } weights = { "filter": TensorConfig(shape=f_shape), - "bias": TensorConfig(shape=bias_shape) + "bias": TensorConfig(shape=bias_shape), } use_mkldnn = False - conv2d_op = OpConfig("conv2d", - inputs=inputs, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format, - use_mkldnn=use_mkldnn) - - add_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv2d_out"], - "Y": ["bias"] - }, - outputs={"Out": ["add_out"]}, - axis=axis) + conv2d_op = OpConfig( + "conv2d", + inputs=inputs, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + use_mkldnn=use_mkldnn, + ) + + add_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv2d_out"], "Y": ["bias"]}, + outputs={"Out": ["add_out"]}, + axis=axis, + ) ops = [conv2d_op, add_op] @@ -176,13 +206,14 @@ class TestConvBiasMkldnnFusePass(PassAutoScanTest): ops=ops, weights=weights, inputs={"input_x": TensorConfig(shape=x_shape)}, - outputs=ops[-1].outputs["Out"]) + outputs=ops[-1].outputs["Out"], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=350, - passes=["conv_bias_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, max_examples=350, passes=["conv_bias_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bn_fuse_pass.py index 0657ba51cdc..e16fd8b10c2 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bn_fuse_pass.py @@ -23,7 +23,6 @@ import hypothesis.strategies as st class TestConvBnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -46,24 +45,29 @@ class TestConvBnFusePass(PassAutoScanTest): out_channel = groups * out_channel_factor batch_size = draw(st.integers(min_value=1, max_value=4)) dilations = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) paddings = draw( - st.lists(st.integers(min_value=0, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=2), min_size=2, max_size=2 + ) + ) strides = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) has_bias = draw(st.booleans()) use_mkldnn = draw(st.booleans()) epsilon = draw(st.floats(min_value=0.0, max_value=0.001)) - x_shape = [ - batch_size, in_channel, 64, 64 - ] if data_format == "NCHW" else [batch_size, 64, 64, in_channel] + x_shape = ( + [batch_size, in_channel, 64, 64] + if data_format == "NCHW" + else [batch_size, 64, 64, in_channel] + ) w_shape = [out_channel, filter_channel, filter_size, filter_size] scale_shape = [out_channel] bias_shape = [out_channel] @@ -91,41 +95,45 @@ class TestConvBnFusePass(PassAutoScanTest): def generate_bn_Var(): return np.random.random(var_shape).astype(np.float32) - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["conv2d_input"], - "Filter": ["conv2d_weight"], - }, - outputs={"Output": ["conv2d_out"]}, - data_format=data_format, - dilations=dilations, - padding_algorithm=padding_algorithm, - groups=groups, - paddings=paddings, - strides=strides, - use_mkldnn=use_mkldnn, - has_bias=has_bias, - is_test=True) - bn_op = OpConfig("batch_norm", - inputs={ - "X": ["conv2d_out"], - "Scale": ["batch_norm_Scale"], - "Bias": ["batch_norm_Bias"], - "Mean": ["batch_norm_Mean"], - "Variance": ["batch_norm_Variance"], - }, - outputs={ - "Y": ["batch_norm_Y"], - "MeanOut": ["batch_norm_Mean"], - "VarianceOut": ["batch_norm_Variance"], - "SavedMean": ["batch_norm_SavedMean"], - "SavedVariance": ["batch_norm_SavedVariance"], - "ReserveSpace": ["batch_norm_ReserveSpace"], - }, - epsilon=epsilon, - trainable_statistics=False, - data_layout=data_format, - is_test=True) + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["conv2d_input"], + "Filter": ["conv2d_weight"], + }, + outputs={"Output": ["conv2d_out"]}, + data_format=data_format, + dilations=dilations, + padding_algorithm=padding_algorithm, + groups=groups, + paddings=paddings, + strides=strides, + use_mkldnn=use_mkldnn, + has_bias=has_bias, + is_test=True, + ) + bn_op = OpConfig( + "batch_norm", + inputs={ + "X": ["conv2d_out"], + "Scale": ["batch_norm_Scale"], + "Bias": ["batch_norm_Bias"], + "Mean": ["batch_norm_Mean"], + "Variance": ["batch_norm_Variance"], + }, + outputs={ + "Y": ["batch_norm_Y"], + "MeanOut": ["batch_norm_Mean"], + "VarianceOut": ["batch_norm_Variance"], + "SavedMean": ["batch_norm_SavedMean"], + "SavedVariance": ["batch_norm_SavedVariance"], + "ReserveSpace": ["batch_norm_ReserveSpace"], + }, + epsilon=epsilon, + trainable_statistics=False, + data_layout=data_format, + is_test=True, + ) if has_bias == True: conv2d_op.inputs["Bias"] = ["conv2d_bias"] ops = [conv2d_op, bn_op] @@ -133,25 +141,25 @@ class TestConvBnFusePass(PassAutoScanTest): program_config = ProgramConfig( ops=ops, inputs={ - "conv2d_input": - TensorConfig(data_gen=partial(generate_conv2d_Input)), + "conv2d_input": TensorConfig( + data_gen=partial(generate_conv2d_Input) + ), }, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_conv2d_Filter)), - "batch_norm_Scale": - TensorConfig(data_gen=generate_bn_Scale), - "batch_norm_Bias": - TensorConfig(data_gen=generate_bn_Bias), - "batch_norm_Mean": - TensorConfig(data_gen=generate_bn_Mean), - "batch_norm_Variance": - TensorConfig(data_gen=generate_bn_Var), + "conv2d_weight": TensorConfig( + data_gen=partial(generate_conv2d_Filter) + ), + "batch_norm_Scale": TensorConfig(data_gen=generate_bn_Scale), + "batch_norm_Bias": TensorConfig(data_gen=generate_bn_Bias), + "batch_norm_Mean": TensorConfig(data_gen=generate_bn_Mean), + "batch_norm_Variance": TensorConfig(data_gen=generate_bn_Var), }, - outputs=["batch_norm_Y"]) + outputs=["batch_norm_Y"], + ) if has_bias == True: program_config.weights["conv2d_bias"] = TensorConfig( - data_gen=partial(generate_conv2d_Bias)) + data_gen=partial(generate_conv2d_Bias) + ) return program_config def sample_predictor_configs(self, program_config): @@ -174,34 +182,40 @@ class TestConvBnFusePass(PassAutoScanTest): min_subgraph_size=1, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) if program_config.ops[0].attrs['has_bias']: yield config, ['conv2d', 'elementwise_add'], (1e-5, 1e-5) else: # it will enter conv_elementwise_add_fuse_pass yield config, ['conv2d_fusion'], (1e-5, 1e-5) def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[0].attrs[ - 'data_format'] == "NHWC" and not predictor_config.mkldnn_enabled( - ): + if ( + program_config.ops[0].attrs['data_format'] == "NHWC" + and not predictor_config.mkldnn_enabled() + ): return True return False # mkldnn Output has diff with bias! def teller2(program_config, predictor_config): - return predictor_config.mkldnn_enabled( - ) and program_config.ops[0].attrs['has_bias'] == True + return ( + predictor_config.mkldnn_enabled() + and program_config.ops[0].attrs['has_bias'] == True + ) self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The output format of conv2d is wrong when data_format attribute is NHWC" + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The output format of conv2d is wrong when data_format attribute is NHWC", ) self.add_ignore_check_case( - teller2, IgnoreReasons.PASS_ACCURACY_ERROR, - "Currently mkldnn Output has diff with bias!") + teller2, + IgnoreReasons.PASS_ACCURACY_ERROR, + "Currently mkldnn Output has diff with bias!", + ) def test(self): self.run_and_statis( diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add2_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add2_act_fuse_pass.py index d08da522e1a..75d0e445f7a 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add2_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add2_act_fuse_pass.py @@ -24,21 +24,21 @@ os.environ['NVIDIA_TF32_OVERRIDE'] = '0' class TestConvElementwiseAdd2ActPass(PassAutoScanTest): r""" - x_var f_var(persistable) - \ / - conv2d - | - conv2d_var y_var(persistable) - \ / - elementwise_add - | - x1_var elementwise_add_out_var - \ / - elementwise_add - | - act - | - act_var + x_var f_var(persistable) + \ / + conv2d + | + conv2d_var y_var(persistable) + \ / + elementwise_add + | + x1_var elementwise_add_out_var + \ / + elementwise_add + | + act + | + act_var """ def sample_predictor_configs(self, program_config): @@ -58,16 +58,66 @@ class TestConvElementwiseAdd2ActPass(PassAutoScanTest): if data_format != "NCHW": return False if padding_algorithm == "VALID": - if int(((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1)) <= 0 or \ - int(((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1)) <= 0: + if ( + int( + ( + ( + input_shape[2] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) + ) + <= 0 + or int( + ( + ( + input_shape[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) + ) + <= 0 + ): return False if padding_algorithm == "EXPLICIT": - if int(((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1)) <= 0 or \ - int(((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1)) <= 0: + if ( + int( + ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) + ) + <= 0 + or int( + ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) + ) + <= 0 + ): return False if padding_algorithm == "SAME": - if int((input_shape[2] + strides[0] - 1) / strides[0]) <= 0 or int( - (input_shape[3] + strides[1] - 1) / strides[1]) <= 0: + if ( + int((input_shape[2] + strides[0] - 1) / strides[0]) <= 0 + or int((input_shape[3] + strides[1] - 1) / strides[1]) <= 0 + ): return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -89,9 +139,12 @@ class TestConvElementwiseAdd2ActPass(PassAutoScanTest): while is_not_valid: # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), + min_size=4, + max_size=4, + ) + ) x_shape[1] = draw(st.integers(min_value=1, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -99,9 +152,12 @@ class TestConvElementwiseAdd2ActPass(PassAutoScanTest): # 3. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=7), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=7), + min_size=4, + max_size=4, + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -109,51 +165,94 @@ class TestConvElementwiseAdd2ActPass(PassAutoScanTest): # 4. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), + min_size=2, + max_size=2, + ) + ) # 5. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + st.sampled_from(["EXPLICIT", "SAME", "VALID"]) + ) # 6. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), + min_size=4, + max_size=4, + ) + ) # 7. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 8. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), + min_size=2, + max_size=2, + ) + ) # 9. Generate legal elemntwise_add: X of conv2d bias_2_dict = dict() bias_2_dict[1] = [ - x_shape[0], f_shape[0], - int(((x_shape[2] + padding[0] + padding[1] - - (dilations[0] * (f_shape[2] - 1) + 1)) / strides[0] + 1)), - int(((x_shape[3] + padding[2] + padding[3] - - (dilations[1] * (f_shape[3] - 1) + 1)) / strides[1] + 1)) + x_shape[0], + f_shape[0], + int( + ( + ( + x_shape[2] + + padding[0] + + padding[1] + - (dilations[0] * (f_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) + ), + int( + ( + ( + x_shape[3] + + padding[2] + + padding[3] + - (dilations[1] * (f_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) + ), ] bias_2_dict[2] = [ - x_shape[0], f_shape[0], + x_shape[0], + f_shape[0], int((x_shape[2] + strides[0] - 1) / strides[0]), - int((x_shape[3] + strides[1] - 1) / strides[1]) + int((x_shape[3] + strides[1] - 1) / strides[1]), ] bias_2_dict[3] = [ - x_shape[0], f_shape[0], - int(((x_shape[2] - (dilations[0] * - (f_shape[2] - 1) + 1)) / strides[0] + 1)), - int(((x_shape[3] - (dilations[1] * - (f_shape[3] - 1) + 1)) / strides[1] + 1)) + x_shape[0], + f_shape[0], + int( + ( + (x_shape[2] - (dilations[0] * (f_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) + ), + int( + ( + (x_shape[3] - (dilations[1] * (f_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) + ), ] bias_index = 1 if padding_algorithm == "SAME": @@ -176,37 +275,34 @@ class TestConvElementwiseAdd2ActPass(PassAutoScanTest): # 12. Generate legal attr:axis of elementwise_add_2 axis_2 = -1 - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_x"], - "Filter": ["filter"] - }, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format) - add_1_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv2d_out"], - "Y": ["bias_1"] - }, - outputs={"Out": ["add_1_out"]}, - axis=axis_1) - - add_2_op = OpConfig("elementwise_add", - inputs={ - "X": ["bias_2"], - "Y": ["add_1_out"] - }, - outputs={"Out": ["add_out"]}, - axis=axis_2) - - relu_op = OpConfig("relu", - inputs={"X": ["add_out"]}, - outputs={"Out": ["relu_out"]}) + conv2d_op = OpConfig( + "conv2d", + inputs={"Input": ["input_x"], "Filter": ["filter"]}, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + ) + add_1_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv2d_out"], "Y": ["bias_1"]}, + outputs={"Out": ["add_1_out"]}, + axis=axis_1, + ) + + add_2_op = OpConfig( + "elementwise_add", + inputs={"X": ["bias_2"], "Y": ["add_1_out"]}, + outputs={"Out": ["add_out"]}, + axis=axis_2, + ) + + relu_op = OpConfig( + "relu", inputs={"X": ["add_out"]}, outputs={"Out": ["relu_out"]} + ) ops = [conv2d_op, add_1_op, add_2_op, relu_op] @@ -218,16 +314,18 @@ class TestConvElementwiseAdd2ActPass(PassAutoScanTest): }, inputs={ "input_x": TensorConfig(shape=x_shape), - "bias_2": TensorConfig(shape=bias_2_shape) + "bias_2": TensorConfig(shape=bias_2_shape), }, outputs=ops[-1].outputs["Out"], ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["conv_elementwise_add2_act_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["conv_elementwise_add2_act_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_act_fuse_pass.py index 7c1ae063ce8..ca349819519 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_act_fuse_pass.py @@ -55,12 +55,36 @@ class TestConvElementwiseAddActPass(PassAutoScanTest): if data_format != "NCHW": return False if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -77,9 +101,10 @@ class TestConvElementwiseAddActPass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=1, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -87,9 +112,10 @@ class TestConvElementwiseAddActPass(PassAutoScanTest): # 3. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=7), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=7), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -97,35 +123,41 @@ class TestConvElementwiseAddActPass(PassAutoScanTest): # 4. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 5. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) # 6. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=4, max_size=4 + ) + ) # 7. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 8. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 9. Generate legal input:ResidualData of conv2d res_shape = [] if draw(st.booleans()): res_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), + min_size=4, + max_size=4, + ) + ) # 10. Generate legal shape of input:bias of elementwise_add bias_shape = [f_shape[0]] @@ -133,30 +165,31 @@ class TestConvElementwiseAddActPass(PassAutoScanTest): # 11. Generate legal attr:axis of elementwise_add axis = 1 - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_x"], - "Filter": ["filter"], - "ResidualData": ["residualdata"] - }, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format) - add_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv2d_out"], - "Y": ["bias"] - }, - outputs={"Out": ["add_out"]}, - axis=axis) - - relu_op = OpConfig("relu", - inputs={"X": ["add_out"]}, - outputs={"Out": ["relu_out"]}) + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["input_x"], + "Filter": ["filter"], + "ResidualData": ["residualdata"], + }, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + ) + add_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv2d_out"], "Y": ["bias"]}, + outputs={"Out": ["add_out"]}, + axis=axis, + ) + + relu_op = OpConfig( + "relu", inputs={"X": ["add_out"]}, outputs={"Out": ["relu_out"]} + ) ops = [conv2d_op, add_op, relu_op] @@ -168,16 +201,18 @@ class TestConvElementwiseAddActPass(PassAutoScanTest): }, inputs={ "input_x": TensorConfig(shape=x_shape), - "residualdata": TensorConfig(shape=res_shape) + "residualdata": TensorConfig(shape=res_shape), }, outputs=ops[-1].outputs["Out"], ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=400, - passes=["conv_elementwise_add_act_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=400, + passes=["conv_elementwise_add_act_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_fuse_pass.py index 303c8017e95..6501d6c953b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_fuse_pass.py @@ -23,7 +23,6 @@ import hypothesis.strategies as st class TestConvEltwiseAddFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -46,21 +45,26 @@ class TestConvEltwiseAddFusePass(PassAutoScanTest): out_channel = groups * out_channel_factor batch_size = draw(st.integers(min_value=1, max_value=4)) dilations = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) paddings = draw( - st.lists(st.integers(min_value=0, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=2), min_size=2, max_size=2 + ) + ) strides = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) - x_shape = [ - batch_size, in_channel, 64, 64 - ] if data_format == "NCHW" else [batch_size, 64, 64, in_channel] + x_shape = ( + [batch_size, in_channel, 64, 64] + if data_format == "NCHW" + else [batch_size, 64, 64, in_channel] + ) w_shape = [out_channel, filter_channel, filter_size, filter_size] scale_shape = [out_channel] bias_shape = [out_channel] @@ -77,26 +81,27 @@ class TestConvEltwiseAddFusePass(PassAutoScanTest): def generate_scale_bias(): return np.random.random(bias_shape).astype(np.float32) - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_data"], - "Filter": ["conv2d_weight"], - }, - outputs={"Output": ["conv_output"]}, - data_format=data_format, - dilations=dilations, - padding_algorithm=padding_algorithm, - groups=groups, - paddings=paddings, - strides=strides, - is_test=True) - eltwise_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv_output"], - "Y": ["conv2d_bias"] - }, - outputs={"Out": ["elementwise_output"]}, - axis=axis) + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + outputs={"Output": ["conv_output"]}, + data_format=data_format, + dilations=dilations, + padding_algorithm=padding_algorithm, + groups=groups, + paddings=paddings, + strides=strides, + is_test=True, + ) + eltwise_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv_output"], "Y": ["conv2d_bias"]}, + outputs={"Out": ["elementwise_output"]}, + axis=axis, + ) ops = [conv2d_op, eltwise_op] program_config = ProgramConfig( @@ -105,12 +110,15 @@ class TestConvEltwiseAddFusePass(PassAutoScanTest): "input_data": TensorConfig(data_gen=partial(generate_input)), }, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight)), - "conv2d_bias": - TensorConfig(data_gen=partial(generate_scale_bias)), + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight) + ), + "conv2d_bias": TensorConfig( + data_gen=partial(generate_scale_bias) + ), }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config def sample_predictor_configs(self, program_config): @@ -125,7 +133,8 @@ class TestConvEltwiseAddFusePass(PassAutoScanTest): min_subgraph_size=1, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['conv2d_fusion'], (1e-4, 1e-4) def add_ignore_pass_case(self): @@ -137,10 +146,11 @@ class TestConvEltwiseAddFusePass(PassAutoScanTest): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, "The output format of conv2d is wrong when data_format attribute is NHWC, \ it will trigger Broadcast dimension mismatch bug \ - when data_format attribute is NHWC and axis of eltwise op is 1 for this pass." + when data_format attribute is NHWC and axis of eltwise op is 1 for this pass.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_eltwiseadd_bn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_eltwiseadd_bn_fuse_pass.py index a88150aecbf..9a25806c25f 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_eltwiseadd_bn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_eltwiseadd_bn_fuse_pass.py @@ -25,19 +25,19 @@ os.environ['NVIDIA_TF32_OVERRIDE'] = '0' class TestConvEltwiseaddBnFusePass(PassAutoScanTest): r""" - x_var f_var(persistable) - \ / - conv2d - | - conv2d_var bias_var(persistable) - \ / - elementwise_add - | - elementwise_add_var Scale(persistable) Bias(persistable) Mean(persistable) Variance(persistable) - | - batch_norm - | - Y MeanOut VarianceOut SavedMeanSavedVariance + x_var f_var(persistable) + \ / + conv2d + | + conv2d_var bias_var(persistable) + \ / + elementwise_add + | + elementwise_add_var Scale(persistable) Bias(persistable) Mean(persistable) Variance(persistable) + | + batch_norm + | + Y MeanOut VarianceOut SavedMeanSavedVariance """ def sample_predictor_configs(self, program_config): @@ -66,12 +66,36 @@ class TestConvEltwiseaddBnFusePass(PassAutoScanTest): if data_format != "NCHW": return False if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": @@ -105,9 +129,10 @@ class TestConvEltwiseaddBnFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=10, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=10, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=1, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -115,9 +140,10 @@ class TestConvEltwiseaddBnFusePass(PassAutoScanTest): # 2. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=7), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=7), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -125,35 +151,41 @@ class TestConvEltwiseaddBnFusePass(PassAutoScanTest): # 3. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 4. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) # 5. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=4, max_size=4 + ) + ) # 6. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 7. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 9. Generate legal input:ResidualData of conv2d res_shape = [] if draw(st.booleans()): res_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), + min_size=4, + max_size=4, + ) + ) # 10. Generate legal shape of input:bias of elementwise_add bias_shape = [f_shape[0]] @@ -177,51 +209,54 @@ class TestConvEltwiseaddBnFusePass(PassAutoScanTest): epsilon = draw(st.floats(min_value=0.00001, max_value=0.001)) def generate_batch_variance(): - return (0.1 + - (1.0 - 0.1) * np.random.random(bn_variance_shape)).astype( - np.float32) - - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_x"], - "Filter": ["filter"], - "ResidualData": ["residualdata"] - }, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format) - add_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv2d_out"], - "Y": ["bias"] - }, - outputs={"Out": ["add_out"]}, - axis=axis) - - bn_op = OpConfig("batch_norm", - inputs={ - "X": ["add_out"], - "Scale": ["scale_in"], - "Bias": ["bias_in"], - "Mean": ["mean_in"], - "Variance": ["variance_in"] - }, - outputs={ - "Y": ["y_out"], - "MeanOut": ["mean_in"], - "VarianceOut": ["variance_in"], - "SavedMean": ["SavedMean_out"], - "SavedVariance": ["SavedVariance_out"], - "ReserveSpace": ["ReserveSpace_out"] - }, - epsilon=epsilon, - is_test=True, - trainable_statistics=False, - data_layout=data_format) + return ( + 0.1 + (1.0 - 0.1) * np.random.random(bn_variance_shape) + ).astype(np.float32) + + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["input_x"], + "Filter": ["filter"], + "ResidualData": ["residualdata"], + }, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + ) + add_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv2d_out"], "Y": ["bias"]}, + outputs={"Out": ["add_out"]}, + axis=axis, + ) + + bn_op = OpConfig( + "batch_norm", + inputs={ + "X": ["add_out"], + "Scale": ["scale_in"], + "Bias": ["bias_in"], + "Mean": ["mean_in"], + "Variance": ["variance_in"], + }, + outputs={ + "Y": ["y_out"], + "MeanOut": ["mean_in"], + "VarianceOut": ["variance_in"], + "SavedMean": ["SavedMean_out"], + "SavedVariance": ["SavedVariance_out"], + "ReserveSpace": ["ReserveSpace_out"], + }, + epsilon=epsilon, + is_test=True, + trainable_statistics=False, + data_layout=data_format, + ) ops = [conv2d_op, add_op, bn_op] @@ -243,15 +278,18 @@ class TestConvEltwiseaddBnFusePass(PassAutoScanTest): }, inputs={ "input_x": TensorConfig(shape=x_shape), - "residualdata": TensorConfig(shape=res_shape) + "residualdata": TensorConfig(shape=res_shape), }, - outputs=outputs) + outputs=outputs, + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["conv_eltwiseadd_bn_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["conv_eltwiseadd_bn_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_bn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_bn_fuse_pass.py index 90d81b50471..3749479a6cc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_bn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_bn_fuse_pass.py @@ -34,10 +34,12 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): ''' def test(self): - self.run_and_statis(quant=False, - max_examples=150, - max_duration=250, - passes=["conv_transpose_bn_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=150, + max_duration=250, + passes=["conv_transpose_bn_fuse_pass"], + ) def sample_program_config(self, draw): # generate random number @@ -47,26 +49,31 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): random_input_dim2 = draw(st.integers(min_value=20, max_value=50)) random_groups = draw(st.integers(min_value=1, max_value=2)) random_dilations = draw( - st.lists(st.integers(min_value=1, max_value=3), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=3), min_size=2, max_size=2 + ) + ) random_strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_paddings = draw( - st.lists(st.integers(min_value=0, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=4), min_size=2, max_size=2 + ) + ) random_padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + st.sampled_from(["EXPLICIT", "SAME", "VALID"]) + ) random_data_layout = draw(st.sampled_from(["NCHW", "NHWC"])) random_use_mkldnn = draw(st.booleans()) random_output_size = [] random_filter = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_out_channel = draw(st.integers(min_value=10, max_value=25)) random_epsilon = draw(st.floats(min_value=0.0, max_value=0.001)) @@ -87,24 +94,24 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): return np.random.random(shape).astype(np.float32) def generate_batch_norm_Scale(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Bias(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Mean(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Variance(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) # define op conv2d_op = OpConfig( @@ -112,7 +119,7 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): inputs={ "Input": ["conv2d_Input"], "Filter": ["conv2d_Filter"], - #"Bias": ["conv2d_Bias"], + # "Bias": ["conv2d_Bias"], }, outputs={ "Output": ["conv2d_Out"], @@ -128,33 +135,34 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): 'output_padding': random_output_size, 'use_mkldnn': random_use_mkldnn, 'is_test': True, - }) - - batch_norm_op = OpConfig(type="batch_norm", - inputs={ - "X": ["conv2d_Out"], - "Scale": ["batch_norm_Scale"], - "Bias": ["batch_norm_Bias"], - "Mean": ["batch_norm_Mean"], - "Variance": ["batch_norm_Variance"], - }, - outputs={ - "Y": ["batch_norm_Y"], - "MeanOut": ["batch_norm_Mean"], - "VarianceOut": ["batch_norm_Variance"], - "SavedMean": ["batch_norm_SavedMean"], - "SavedVariance": - ["batch_norm_SavedVariance"], - "ReserveSpace": - ["batch_norm_ReserveSpace"], - }, - attrs={ - 'epsilon': random_epsilon, - 'is_test': True, - 'trainable_statistics': False, - 'data_layout': random_data_layout, - 'use_mkldnn': random_use_mkldnn, - }) + }, + ) + + batch_norm_op = OpConfig( + type="batch_norm", + inputs={ + "X": ["conv2d_Out"], + "Scale": ["batch_norm_Scale"], + "Bias": ["batch_norm_Bias"], + "Mean": ["batch_norm_Mean"], + "Variance": ["batch_norm_Variance"], + }, + outputs={ + "Y": ["batch_norm_Y"], + "MeanOut": ["batch_norm_Mean"], + "VarianceOut": ["batch_norm_Variance"], + "SavedMean": ["batch_norm_SavedMean"], + "SavedVariance": ["batch_norm_SavedVariance"], + "ReserveSpace": ["batch_norm_ReserveSpace"], + }, + attrs={ + 'epsilon': random_epsilon, + 'is_test': True, + 'trainable_statistics': False, + 'data_layout': random_data_layout, + 'use_mkldnn': random_use_mkldnn, + }, + ) # define model_net model_net = [conv2d_op, batch_norm_op] @@ -165,18 +173,22 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): "conv2d_Input": TensorConfig(data_gen=generate_conv2d_Input), }, weights={ - "conv2d_Filter": - TensorConfig(data_gen=generate_conv2d_Filter), - "batch_norm_Scale": - TensorConfig(data_gen=generate_batch_norm_Scale), - "batch_norm_Bias": - TensorConfig(data_gen=generate_batch_norm_Bias), - "batch_norm_Mean": - TensorConfig(data_gen=generate_batch_norm_Mean), - "batch_norm_Variance": - TensorConfig(data_gen=generate_batch_norm_Variance), + "conv2d_Filter": TensorConfig(data_gen=generate_conv2d_Filter), + "batch_norm_Scale": TensorConfig( + data_gen=generate_batch_norm_Scale + ), + "batch_norm_Bias": TensorConfig( + data_gen=generate_batch_norm_Bias + ), + "batch_norm_Mean": TensorConfig( + data_gen=generate_batch_norm_Mean + ), + "batch_norm_Variance": TensorConfig( + data_gen=generate_batch_norm_Variance + ), }, - outputs=["batch_norm_Y"]) + outputs=["batch_norm_Y"], + ) return program_config @@ -201,7 +213,6 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): return True def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): if program_config.ops[0].attrs['data_format'] == "NHWC": return True @@ -213,9 +224,13 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The output format of conv2d_transpose is wrong when data_format attribute is NHWC" + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The output format of conv2d_transpose is wrong when data_format attribute is NHWC", ) - self.add_ignore_check_case(teller2, IgnoreReasons.PASS_ACCURACY_ERROR, - "there is diff when group >1 in this pass") + self.add_ignore_check_case( + teller2, + IgnoreReasons.PASS_ACCURACY_ERROR, + "there is diff when group >1 in this pass", + ) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_eltwiseadd_bn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_eltwiseadd_bn_fuse_pass.py index 91d9b863e2f..8b5bfc91548 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_eltwiseadd_bn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_eltwiseadd_bn_fuse_pass.py @@ -38,10 +38,12 @@ class TestConvTransposeEltwiseaddBnFusePass(PassAutoScanTest): ''' def test(self): - self.run_and_statis(quant=False, - max_examples=150, - max_duration=250, - passes=["conv_transpose_eltwiseadd_bn_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=150, + max_duration=250, + passes=["conv_transpose_eltwiseadd_bn_fuse_pass"], + ) def sample_program_config(self, draw): # generate random number @@ -51,26 +53,31 @@ class TestConvTransposeEltwiseaddBnFusePass(PassAutoScanTest): random_input_dim2 = draw(st.integers(min_value=20, max_value=50)) random_groups = draw(st.integers(min_value=1, max_value=2)) random_dilations = draw( - st.lists(st.integers(min_value=1, max_value=3), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=3), min_size=2, max_size=2 + ) + ) random_strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_paddings = draw( - st.lists(st.integers(min_value=0, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=4), min_size=2, max_size=2 + ) + ) random_padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + st.sampled_from(["EXPLICIT", "SAME", "VALID"]) + ) random_data_layout = draw(st.sampled_from(["NCHW", "NHWC"])) random_use_mkldnn = draw(st.booleans()) random_output_size = [] random_filter = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_out_channel = draw(st.integers(min_value=20, max_value=25)) random_epsilon = draw(st.floats(min_value=0.0, max_value=0.001)) @@ -91,89 +98,93 @@ class TestConvTransposeEltwiseaddBnFusePass(PassAutoScanTest): return np.random.random(shape).astype(np.float32) def generate_elementwise_add_Y(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Scale(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Bias(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Mean(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Variance(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) # define op - conv2d_op = OpConfig(type="conv2d_transpose", - inputs={ - "Input": ["conv2d_Input"], - "Filter": ["conv2d_Filter"], - }, - outputs={ - "Output": ["conv2d_Out"], - }, - attrs={ - 'groups': random_groups, - 'dilations': random_dilations, - 'strides': random_strides, - 'paddings': random_paddings, - 'padding_algorithm': random_padding_algorithm, - 'data_format': random_data_layout, - 'output_size': random_output_size, - 'output_padding': random_output_size, - 'use_mkldnn': random_use_mkldnn, - 'is_test': True, - }) - - elementwise_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["conv2d_Out"], - "Y": ["elementwise_add_Y"], - }, - outputs={ - "Out": ["elementwise_add_Out"], - }, - attrs={ - 'axis': 1, - }) - - batch_norm_op = OpConfig(type="batch_norm", - inputs={ - "X": ["elementwise_add_Out"], - "Scale": ["batch_norm_Scale"], - "Bias": ["batch_norm_Bias"], - "Mean": ["batch_norm_Mean"], - "Variance": ["batch_norm_Variance"], - }, - outputs={ - "Y": ["batch_norm_Y"], - "MeanOut": ["batch_norm_Mean"], - "VarianceOut": ["batch_norm_Variance"], - "SavedMean": ["batch_norm_SavedMean"], - "SavedVariance": - ["batch_norm_SavedVariance"], - "ReserveSpace": - ["batch_norm_ReserveSpace"], - }, - attrs={ - 'epsilon': random_epsilon, - 'is_test': True, - 'trainable_statistics': False, - 'data_layout': random_data_layout, - 'use_mkldnn': random_use_mkldnn, - }) + conv2d_op = OpConfig( + type="conv2d_transpose", + inputs={ + "Input": ["conv2d_Input"], + "Filter": ["conv2d_Filter"], + }, + outputs={ + "Output": ["conv2d_Out"], + }, + attrs={ + 'groups': random_groups, + 'dilations': random_dilations, + 'strides': random_strides, + 'paddings': random_paddings, + 'padding_algorithm': random_padding_algorithm, + 'data_format': random_data_layout, + 'output_size': random_output_size, + 'output_padding': random_output_size, + 'use_mkldnn': random_use_mkldnn, + 'is_test': True, + }, + ) + + elementwise_op = OpConfig( + type="elementwise_add", + inputs={ + "X": ["conv2d_Out"], + "Y": ["elementwise_add_Y"], + }, + outputs={ + "Out": ["elementwise_add_Out"], + }, + attrs={ + 'axis': 1, + }, + ) + + batch_norm_op = OpConfig( + type="batch_norm", + inputs={ + "X": ["elementwise_add_Out"], + "Scale": ["batch_norm_Scale"], + "Bias": ["batch_norm_Bias"], + "Mean": ["batch_norm_Mean"], + "Variance": ["batch_norm_Variance"], + }, + outputs={ + "Y": ["batch_norm_Y"], + "MeanOut": ["batch_norm_Mean"], + "VarianceOut": ["batch_norm_Variance"], + "SavedMean": ["batch_norm_SavedMean"], + "SavedVariance": ["batch_norm_SavedVariance"], + "ReserveSpace": ["batch_norm_ReserveSpace"], + }, + attrs={ + 'epsilon': random_epsilon, + 'is_test': True, + 'trainable_statistics': False, + 'data_layout': random_data_layout, + 'use_mkldnn': random_use_mkldnn, + }, + ) # define model_net model_net = [conv2d_op, elementwise_op, batch_norm_op] @@ -185,20 +196,25 @@ class TestConvTransposeEltwiseaddBnFusePass(PassAutoScanTest): "conv2d_Input": TensorConfig(data_gen=generate_conv2d_Input), }, weights={ - "conv2d_Filter": - TensorConfig(data_gen=generate_conv2d_Filter), - "elementwise_add_Y": - TensorConfig(data_gen=generate_elementwise_add_Y), - "batch_norm_Scale": - TensorConfig(data_gen=generate_batch_norm_Scale), - "batch_norm_Bias": - TensorConfig(data_gen=generate_batch_norm_Bias), - "batch_norm_Mean": - TensorConfig(data_gen=generate_batch_norm_Mean), - "batch_norm_Variance": - TensorConfig(data_gen=generate_batch_norm_Variance), + "conv2d_Filter": TensorConfig(data_gen=generate_conv2d_Filter), + "elementwise_add_Y": TensorConfig( + data_gen=generate_elementwise_add_Y + ), + "batch_norm_Scale": TensorConfig( + data_gen=generate_batch_norm_Scale + ), + "batch_norm_Bias": TensorConfig( + data_gen=generate_batch_norm_Bias + ), + "batch_norm_Mean": TensorConfig( + data_gen=generate_batch_norm_Mean + ), + "batch_norm_Variance": TensorConfig( + data_gen=generate_batch_norm_Variance + ), }, - outputs=["batch_norm_Y"]) + outputs=["batch_norm_Y"], + ) return program_config @@ -223,7 +239,6 @@ class TestConvTransposeEltwiseaddBnFusePass(PassAutoScanTest): return True def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): if program_config.ops[0].attrs['data_format'] == "NHWC": return True @@ -235,8 +250,12 @@ class TestConvTransposeEltwiseaddBnFusePass(PassAutoScanTest): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The output format of conv2d_transpose is wrong when data_format attribute is NHWC" + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The output format of conv2d_transpose is wrong when data_format attribute is NHWC", + ) + self.add_ignore_check_case( + teller2, + IgnoreReasons.PASS_ACCURACY_ERROR, + "there is diff when group >1 in this pass", ) - self.add_ignore_check_case(teller2, IgnoreReasons.PASS_ACCURACY_ERROR, - "there is diff when group >1 in this pass") diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_delete_c_identity_op_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_delete_c_identity_op_pass.py index 7efe5e97180..b008c134f8e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_delete_c_identity_op_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_delete_c_identity_op_pass.py @@ -20,7 +20,6 @@ import hypothesis.strategies as st class TestDeleteCIdentityPass(PassAutoScanTest): - def sample_predictor_configs(self, program_config): config = self.create_trt_inference_config() config.enable_tensorrt_engine( @@ -29,28 +28,34 @@ class TestDeleteCIdentityPass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['relu'], (1e-5, 1e-5) def sample_program_config(self, draw): n = draw(st.integers(min_value=1, max_value=2)) - relu_op = OpConfig("relu", - inputs={"X": ["relu_x"]}, - outputs={"Out": ["relu_out"]}) - c_identity_op = OpConfig("c_identity", - inputs={"X": ["relu_out"]}, - outputs={"Out": ["id_out"]}) + relu_op = OpConfig( + "relu", inputs={"X": ["relu_x"]}, outputs={"Out": ["relu_out"]} + ) + c_identity_op = OpConfig( + "c_identity", + inputs={"X": ["relu_out"]}, + outputs={"Out": ["id_out"]}, + ) program_config = ProgramConfig( ops=[relu_op, c_identity_op], weights={}, inputs={"relu_x": TensorConfig(shape=[n])}, - outputs=["id_out"]) + outputs=["id_out"], + ) return program_config def test(self): - self.run_and_statis(max_examples=2, - min_success_num=2, - passes=["delete_c_identity_op_pass"]) + self.run_and_statis( + max_examples=2, + min_success_num=2, + passes=["delete_c_identity_op_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py index ce51cda6e45..3cbd48dea6d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py @@ -24,21 +24,21 @@ import hypothesis.strategies as st class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): r''' - in_var1 emb_var in_var2 emb_var in_var3 emb_var in_var emb_var - | | | | | | | | - lookup_table lookup_table lookup_table ... lookup_table - | | | | - lkt_var lkt_var lkt_var lkt_var - \ / | ... | - elementwise_add | | - \ / | - elementwise_add | - | | - elt_var / - \ / - elementwise_add - | - layer_norm + in_var1 emb_var in_var2 emb_var in_var3 emb_var in_var emb_var + | | | | | | | | + lookup_table lookup_table lookup_table ... lookup_table + | | | | + lkt_var lkt_var lkt_var lkt_var + \ / | ... | + elementwise_add | | + \ / | + elementwise_add | + | | + elt_var / + \ / + elementwise_add + | + layer_norm ''' def is_program_valid(self, program_config: ProgramConfig) -> bool: @@ -54,16 +54,20 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): if program_config.ops[3].attrs['axis'] not in [-1, 2]: return False - if not (program_config.ops[5].attrs['epsilon'] >= 0 - and program_config.ops[5].attrs['epsilon'] <= 0.001): + if not ( + program_config.ops[5].attrs['epsilon'] >= 0 + and program_config.ops[5].attrs['epsilon'] <= 0.001 + ): return False if program_config.ops[5].attrs['begin_norm_axis'] != 2: return False # input check - if program_config.weights['embedding_weight1'].shape[ - 1] != program_config.weights['layer_norm_scale'].shape[0]: + if ( + program_config.weights['embedding_weight1'].shape[1] + != program_config.weights['layer_norm_scale'].shape[0] + ): return False return True @@ -83,17 +87,17 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): def generate_input(attrs): if attrs[0]['op_type'] == 'lookup_table': - return np.random.randint(0, - attrs[3]['weight_size'][0], - size=(attrs[3]['batch_size'], - attrs[3]['input_dim'], - 1)).astype(np.int64) + return np.random.randint( + 0, + attrs[3]['weight_size'][0], + size=(attrs[3]['batch_size'], attrs[3]['input_dim'], 1), + ).astype(np.int64) else: - return np.random.randint(0, - attrs[3]['weight_size'][0], - size=(attrs[3]['batch_size'], - attrs[3]['input_dim'])).astype( - np.int64) + return np.random.randint( + 0, + attrs[3]['weight_size'][0], + size=(attrs[3]['batch_size'], attrs[3]['input_dim']), + ).astype(np.int64) def generate_weight1(attrs): # set embedding weight by attrs @@ -102,116 +106,128 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): def generate_weight2(attrs): # set layernorm weight by attrs if attrs[2]['begin_norm_axis'] == 1: - return np.random.random(attrs[3]['input_dim'] * - attrs[3]['weight_size'][1]).astype( - np.float32) + return np.random.random( + attrs[3]['input_dim'] * attrs[3]['weight_size'][1] + ).astype(np.float32) else: return np.random.random(attrs[3]['weight_size'][1]).astype( - np.float32) + np.float32 + ) - attrs = [{ - 'is_sparse': is_sparse, - 'is_distributed': is_distributed, - 'padding_idx': padding_idx, - 'op_type': op_type - }, { - 'axis': axis - }, { - 'begin_norm_axis': begin_norm_axis, - 'epsilon': epsilon - }, { - 'batch_size': batch_size, - 'input_dim': input_dim, - 'weight_size': weight_size - }] + attrs = [ + { + 'is_sparse': is_sparse, + 'is_distributed': is_distributed, + 'padding_idx': padding_idx, + 'op_type': op_type, + }, + {'axis': axis}, + {'begin_norm_axis': begin_norm_axis, 'epsilon': epsilon}, + { + 'batch_size': batch_size, + 'input_dim': input_dim, + 'weight_size': weight_size, + }, + ] - emb_op1 = OpConfig(type=attrs[0]['op_type'], - inputs={ - "Ids": ["input_data1"], - "W": ["embedding_weight1"] - }, - outputs={"Out": ["embedding_output1"]}, - attrs={ - 'is_sparse': attrs[0]['is_sparse'], - 'is_distributed': attrs[0]['is_distributed'], - 'padding_idx': attrs[0]['padding_idx'] - }) - emb_op2 = OpConfig(type=attrs[0]['op_type'], - inputs={ - "Ids": ["input_data2"], - "W": ["embedding_weight2"] - }, - outputs={"Out": ["embedding_output2"]}, - attrs={ - 'is_sparse': attrs[0]['is_sparse'], - 'is_distributed': attrs[0]['is_distributed'], - 'padding_idx': attrs[0]['padding_idx'] - }) - emb_op3 = OpConfig(type=attrs[0]['op_type'], - inputs={ - "Ids": ["input_data3"], - "W": ["embedding_weight3"] - }, - outputs={"Out": ["embedding_output3"]}, - attrs={ - 'is_sparse': attrs[0]['is_sparse'], - 'is_distributed': attrs[0]['is_distributed'], - 'padding_idx': attrs[0]['padding_idx'] - }) - add_op1 = OpConfig(type='elementwise_add', - inputs={ - "X": [emb_op2.outputs["Out"][0]], - "Y": [emb_op3.outputs["Out"][0]], - }, - outputs={"Out": ["elementwise_add_output1"]}, - attrs={"axis": attrs[1]['axis']}) - add_op2 = OpConfig(type='elementwise_add', - inputs={ - "X": [add_op1.outputs["Out"][0]], - "Y": [emb_op1.outputs["Out"][0]], - }, - outputs={"Out": ["elementwise_add_output2"]}, - attrs={"axis": attrs[1]['axis']}) - layer_norm_op = OpConfig(type='layer_norm', - inputs={ - "X": [add_op2.outputs["Out"][0]], - "Bias": ["layer_norm_bias"], - "Scale": ["layer_norm_scale"] - }, - outputs={ - "Y": ["layer_norm_output1"], - "Mean": ["layer_norm_output2"], - "Variance": ["layer_norm_output3"] - }, - attrs={ - 'begin_norm_axis': - attrs[2]['begin_norm_axis'], - 'epsilon': attrs[2]['epsilon'] - }) + emb_op1 = OpConfig( + type=attrs[0]['op_type'], + inputs={"Ids": ["input_data1"], "W": ["embedding_weight1"]}, + outputs={"Out": ["embedding_output1"]}, + attrs={ + 'is_sparse': attrs[0]['is_sparse'], + 'is_distributed': attrs[0]['is_distributed'], + 'padding_idx': attrs[0]['padding_idx'], + }, + ) + emb_op2 = OpConfig( + type=attrs[0]['op_type'], + inputs={"Ids": ["input_data2"], "W": ["embedding_weight2"]}, + outputs={"Out": ["embedding_output2"]}, + attrs={ + 'is_sparse': attrs[0]['is_sparse'], + 'is_distributed': attrs[0]['is_distributed'], + 'padding_idx': attrs[0]['padding_idx'], + }, + ) + emb_op3 = OpConfig( + type=attrs[0]['op_type'], + inputs={"Ids": ["input_data3"], "W": ["embedding_weight3"]}, + outputs={"Out": ["embedding_output3"]}, + attrs={ + 'is_sparse': attrs[0]['is_sparse'], + 'is_distributed': attrs[0]['is_distributed'], + 'padding_idx': attrs[0]['padding_idx'], + }, + ) + add_op1 = OpConfig( + type='elementwise_add', + inputs={ + "X": [emb_op2.outputs["Out"][0]], + "Y": [emb_op3.outputs["Out"][0]], + }, + outputs={"Out": ["elementwise_add_output1"]}, + attrs={"axis": attrs[1]['axis']}, + ) + add_op2 = OpConfig( + type='elementwise_add', + inputs={ + "X": [add_op1.outputs["Out"][0]], + "Y": [emb_op1.outputs["Out"][0]], + }, + outputs={"Out": ["elementwise_add_output2"]}, + attrs={"axis": attrs[1]['axis']}, + ) + layer_norm_op = OpConfig( + type='layer_norm', + inputs={ + "X": [add_op2.outputs["Out"][0]], + "Bias": ["layer_norm_bias"], + "Scale": ["layer_norm_scale"], + }, + outputs={ + "Y": ["layer_norm_output1"], + "Mean": ["layer_norm_output2"], + "Variance": ["layer_norm_output3"], + }, + attrs={ + 'begin_norm_axis': attrs[2]['begin_norm_axis'], + 'epsilon': attrs[2]['epsilon'], + }, + ) program_config = ProgramConfig( ops=[emb_op1, emb_op2, emb_op3, add_op1, add_op2, layer_norm_op], weights={ - "embedding_weight1": - TensorConfig(data_gen=partial(generate_weight1, attrs[3])), - "embedding_weight2": - TensorConfig(data_gen=partial(generate_weight1, attrs[3])), - "embedding_weight3": - TensorConfig(data_gen=partial(generate_weight1, attrs[3])), - "layer_norm_bias": - TensorConfig(data_gen=partial(generate_weight2, attrs)), - "layer_norm_scale": - TensorConfig(data_gen=partial(generate_weight2, attrs)) + "embedding_weight1": TensorConfig( + data_gen=partial(generate_weight1, attrs[3]) + ), + "embedding_weight2": TensorConfig( + data_gen=partial(generate_weight1, attrs[3]) + ), + "embedding_weight3": TensorConfig( + data_gen=partial(generate_weight1, attrs[3]) + ), + "layer_norm_bias": TensorConfig( + data_gen=partial(generate_weight2, attrs) + ), + "layer_norm_scale": TensorConfig( + data_gen=partial(generate_weight2, attrs) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, attrs)), - "input_data2": - TensorConfig(data_gen=partial(generate_input, attrs)), - "input_data3": - TensorConfig(data_gen=partial(generate_input, attrs)) + "input_data1": TensorConfig( + data_gen=partial(generate_input, attrs) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, attrs) + ), + "input_data3": TensorConfig( + data_gen=partial(generate_input, attrs) + ), }, - outputs=["layer_norm_output1"]) + outputs=["layer_norm_output1"], + ) return program_config @@ -227,7 +243,8 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5) # trt dynamic_shape config = self.create_trt_inference_config() @@ -237,61 +254,71 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) if program_config.ops[0].type == 'lookup_table': config.set_trt_dynamic_shape_info( { "input_data1": [1, 4, 1], "input_data2": [1, 4, 1], - "input_data3": [1, 4, 1] - }, { + "input_data3": [1, 4, 1], + }, + { "input_data1": [4, 512, 1], "input_data2": [4, 512, 1], - "input_data3": [4, 512, 1] - }, { + "input_data3": [4, 512, 1], + }, + { "input_data1": [2, 128, 1], "input_data2": [2, 128, 1], - "input_data3": [2, 128, 1] - }) + "input_data3": [2, 128, 1], + }, + ) else: config.set_trt_dynamic_shape_info( { "input_data1": [1, 4], "input_data2": [1, 4], - "input_data3": [1, 4] - }, { + "input_data3": [1, 4], + }, + { "input_data1": [4, 512], "input_data2": [4, 512], - "input_data3": [4, 512] - }, { + "input_data3": [4, 512], + }, + { "input_data1": [2, 128], "input_data2": [2, 128], - "input_data3": [2, 128] - }) + "input_data3": [2, 128], + }, + ) yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5) def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[3].attrs['axis'] in [ - -1, 2 - ] and program_config.ops[5].attrs[ - 'begin_norm_axis'] == 2 and program_config.weights[ - 'embedding_weight1'].shape in [(64, 32), (64, 64)]: + if ( + program_config.ops[3].attrs['axis'] in [-1, 2] + and program_config.ops[5].attrs['begin_norm_axis'] == 2 + and program_config.weights['embedding_weight1'].shape + in [(64, 32), (64, 64)] + ): return True return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass output has diff in a specific case. We need to fix it as soon as possible." + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass output has diff in a specific case. We need to fix it as soon as possible.", ) def test(self): # this fuse need to fix, now there's no program can ran successfully - self.run_and_statis(quant=False, - max_examples=50, - passes=["embedding_eltwise_layernorm_fuse_pass"], - min_success_num=0) + self.run_and_statis( + quant=False, + max_examples=50, + passes=["embedding_eltwise_layernorm_fuse_pass"], + min_success_num=0, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_elementwise_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_elementwise_layernorm_fuse_pass.py index b15fb88d73d..5074fd1d1b7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_elementwise_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_elementwise_layernorm_fuse_pass.py @@ -21,7 +21,6 @@ import hypothesis.strategies as st class FcElementLayernormFusePassDataGen: - def __init__(self, min_v, max_v, shape, dtype): self.min_v = min_v self.max_v = max_v @@ -29,8 +28,9 @@ class FcElementLayernormFusePassDataGen: self.dtype = dtype def __call__(self): - return np.random.normal(self.min_v, self.max_v, - self.shape).astype(self.dtype) + return np.random.normal(self.min_v, self.max_v, self.shape).astype( + self.dtype + ) class TestFCElementwiseLayerNormFusePass(PassAutoScanTest): @@ -56,18 +56,20 @@ class TestFCElementwiseLayerNormFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X of fc x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) x_shape = [2, 1] x_rank = len(x_shape) # 2. Generate attr:in_num_col_dims of fc in_num_col_dims = draw(st.integers(min_value=1, max_value=x_rank - 1)) # 3. Generate legal shape of input:W/bias of fc w_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) w_shape[0] = int(np.prod(x_shape[in_num_col_dims:])) w_shape = [1, 2] fc_bias_shape = [ @@ -84,17 +86,14 @@ class TestFCElementwiseLayerNormFusePass(PassAutoScanTest): axis = draw(st.integers(min_value=-1, max_value=0)) # 5. Generate legal shape of layer_norm begin_norm_axis = draw( - st.integers(min_value=1, max_value=len(fc_out_shape) - 1)) + st.integers(min_value=1, max_value=len(fc_out_shape) - 1) + ) layer_norm_shape = [int(np.prod(fc_out_shape[begin_norm_axis:]))] epsilon = 1e-5 fc_op = OpConfig( "fc", - inputs={ - "Input": ["fc_x"], - "W": ["fc_w"], - "Bias": ["fc_bias"] - }, + inputs={"Input": ["fc_x"], "W": ["fc_w"], "Bias": ["fc_bias"]}, outputs={"Out": ["fc_out"]}, in_num_col_dims=in_num_col_dims, padding_weights=False, @@ -104,43 +103,40 @@ class TestFCElementwiseLayerNormFusePass(PassAutoScanTest): ) add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["fc_out"], - "Y": ["add_bias"] - }, + inputs={"X": ["fc_out"], "Y": ["add_bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) - layer_norm_op = OpConfig("layer_norm", - inputs={ - "X": ["add_out"], - "Scale": ["scale"], - "Bias": ["layer_norm_bias"] - }, - outputs={ - "Y": ["layer_norm_out"], - "Mean": ["layer_norm_mean"], - "Variance": ["layer_norm_var"] - }, - begin_norm_axis=begin_norm_axis, - epsilon=epsilon) + layer_norm_op = OpConfig( + "layer_norm", + inputs={ + "X": ["add_out"], + "Scale": ["scale"], + "Bias": ["layer_norm_bias"], + }, + outputs={ + "Y": ["layer_norm_out"], + "Mean": ["layer_norm_mean"], + "Variance": ["layer_norm_var"], + }, + begin_norm_axis=begin_norm_axis, + epsilon=epsilon, + ) ops = [fc_op, add_op, layer_norm_op] program_config = ProgramConfig( ops=ops, weights={ - "fc_w": - TensorConfig(shape=w_shape), - "fc_bias": - TensorConfig(shape=fc_bias_shape), - "add_bias": - TensorConfig(shape=add_bias_shape), - "scale": - TensorConfig(shape=layer_norm_shape, - data_gen=FcElementLayernormFusePassDataGen( - 0.0, 0.5, layer_norm_shape, np.float32)), - "layer_norm_bias": - TensorConfig(shape=layer_norm_shape), + "fc_w": TensorConfig(shape=w_shape), + "fc_bias": TensorConfig(shape=fc_bias_shape), + "add_bias": TensorConfig(shape=add_bias_shape), + "scale": TensorConfig( + shape=layer_norm_shape, + data_gen=FcElementLayernormFusePassDataGen( + 0.0, 0.5, layer_norm_shape, np.float32 + ), + ), + "layer_norm_bias": TensorConfig(shape=layer_norm_shape), }, inputs={ "fc_x": TensorConfig(shape=x_shape), @@ -150,9 +146,11 @@ class TestFCElementwiseLayerNormFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["fc_elementwise_layernorm_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["fc_elementwise_layernorm_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py index 58494fba43c..73aa1738b84 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py @@ -50,7 +50,8 @@ class TestFcFusePass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['fc'], (1e-5, 1e-5) def add_ignore_pass_case(self): @@ -74,8 +75,10 @@ class TestFcFusePass(PassAutoScanTest): def teller2(program_config, predictor_config): # TODO fuse has bug while axis != -1 axis = program_config.ops[1].attrs["axis"] - if axis != -1 and axis != program_config.ops[0].attrs[ - "x_num_col_dims"]: + if ( + axis != -1 + and axis != program_config.ops[0].attrs["x_num_col_dims"] + ): return True return False @@ -102,18 +105,21 @@ class TestFcFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X of mul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=4 + ) + ) # 2. Generate attr:x_num_col_dims/y_num_col_dims of mul x_num_col_dims = draw( - st.integers(min_value=1, max_value=len(x_shape) - 1)) + st.integers(min_value=1, max_value=len(x_shape) - 1) + ) y_num_col_dims = 1 # 3. Generate legal shape of input:Y of mul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = int(np.prod(x_shape[x_num_col_dims:])) # 4. Generate legal attr:axis of elementwise_add mul_out_shape = x_shape[:x_num_col_dims] + y_shape[1:] @@ -122,12 +128,13 @@ class TestFcFusePass(PassAutoScanTest): if axis >= 0: max_bias_rank = x_num_col_dims + 1 - axis bias_rank = draw(st.integers(min_value=1, max_value=max_bias_rank)) - bias_shape = mul_out_shape[axis:axis + bias_rank] + bias_shape = mul_out_shape[axis : axis + bias_rank] else: max_bias_rank = 1 bias_rank = draw( - st.integers(min_value=1, max_value=len(mul_out_shape))) - bias_shape = mul_out_shape[-1 * bias_rank:] + st.integers(min_value=1, max_value=len(mul_out_shape)) + ) + bias_shape = mul_out_shape[-1 * bias_rank :] # 6. Random choose if use broadcast for elementwise_add, e.g [3, 4] -> [1, 4] if draw(st.booleans()): broadcast_dims = draw(st.integers(min_value=1, max_value=bias_rank)) @@ -147,28 +154,22 @@ class TestFcFusePass(PassAutoScanTest): # Use function `add_skip_pass_case` to ignore the programs even if they cause bug while runing mul_op = OpConfig( "mul", - inputs={ - "X": ["mul_x"], - "Y": ["mul_y"] - }, + inputs={"X": ["mul_x"], "Y": ["mul_y"]}, outputs={"Out": ["mul_out"]}, x_num_col_dims=x_num_col_dims, y_num_col_dims=y_num_col_dims, ) add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["mul_out"], - "Y": ["bias"] - }, + inputs={"X": ["mul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) ops = [mul_op, add_op] if has_relu: - relu_op = OpConfig("relu", - inputs={"X": ["add_out"]}, - outputs={"Out": ["relu_out"]}) + relu_op = OpConfig( + "relu", inputs={"X": ["add_out"]}, outputs={"Out": ["relu_out"]} + ) ops.append(relu_op) program_config = ProgramConfig( ops=ops, @@ -184,9 +185,9 @@ class TestFcFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=500, - passes=["fc_fuse_pass"]) + self.run_and_statis( + quant=False, max_examples=500, passes=["fc_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_gru_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_gru_fuse_pass.py index b722fb798c5..f0c2cbefc0c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_gru_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_gru_fuse_pass.py @@ -20,28 +20,29 @@ from paddle.fluid.core import PassVersionChecker class FcGruFusePassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): dict_dim, emb_dim = 128, 64 - data = fluid.data(name='step_data', - shape=[None], - dtype='int64', - lod_level=1) + data = fluid.data( + name='step_data', shape=[None], dtype='int64', lod_level=1 + ) emb = fluid.embedding(input=data, size=[dict_dim, emb_dim]) hidden_dim = 512 x = fluid.layers.fc(input=emb, size=hidden_dim * 3) - hidden = fluid.layers.dynamic_gru(input=x, - size=hidden_dim, - bias_attr=True, - origin_mode=False, - is_reverse=True) + hidden = fluid.layers.dynamic_gru( + input=x, + size=hidden_dim, + bias_attr=True, + origin_mode=False, + is_reverse=True, + ) batch = 16 lod_tensor = fluid.LoDTensor() lod_tensor.set( np.random.randint(0, dict_dim, size=[batch]).astype("int64"), - fluid.CPUPlace()) + fluid.CPUPlace(), + ) lod_tensor.set_lod([[0, batch]]) self.feeds = {"step_data": lod_tensor} self.fetch_list = [hidden] @@ -53,28 +54,29 @@ class FcGruFusePassTest(InferencePassTest): class MulGruFusePassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): dict_dim, emb_dim = 128, 64 - data = fluid.data(name='step_data', - shape=[None], - dtype='int64', - lod_level=1) + data = fluid.data( + name='step_data', shape=[None], dtype='int64', lod_level=1 + ) emb = fluid.embedding(input=data, size=[dict_dim, emb_dim]) hidden_dim = 512 x = fluid.layers.fc(input=emb, size=hidden_dim * 3, bias_attr=False) - hidden = fluid.layers.dynamic_gru(input=x, - size=hidden_dim, - bias_attr=True, - origin_mode=False, - is_reverse=True) + hidden = fluid.layers.dynamic_gru( + input=x, + size=hidden_dim, + bias_attr=True, + origin_mode=False, + is_reverse=True, + ) batch = 16 lod_tensor = fluid.LoDTensor() lod_tensor.set( np.random.randint(0, dict_dim, size=[batch]).astype("int64"), - fluid.CPUPlace()) + fluid.CPUPlace(), + ) lod_tensor.set_lod([[0, batch]]) self.feeds = {"step_data": lod_tensor} self.fetch_list = [hidden] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_lstm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_lstm_fuse_pass.py index 040cf05b02a..acef755d571 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_lstm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_lstm_fuse_pass.py @@ -20,26 +20,26 @@ from paddle.fluid.core import PassVersionChecker class MulLstmFusePassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): dict_dim, emb_dim = 128, 64 hidden_dim = 512 - data = fluid.data(name='data', - shape=[1], - dtype='int64', - lod_level=1) + data = fluid.data( + name='data', shape=[1], dtype='int64', lod_level=1 + ) emb = fluid.embedding(input=data, size=[dict_dim, emb_dim]) x = fluid.layers.fc(input=emb, size=hidden_dim * 4, bias_attr=False) - forward, cell = fluid.layers.dynamic_lstm(input=x, - size=hidden_dim * 4) + forward, cell = fluid.layers.dynamic_lstm( + input=x, size=hidden_dim * 4 + ) batch = 16 lod_tensor = fluid.LoDTensor() lod_tensor.set( np.random.randint(0, dict_dim, size=[batch]).astype("int64"), - fluid.CPUPlace()) + fluid.CPUPlace(), + ) lod_tensor.set_lod([[0, batch]]) self.feeds = {"data": lod_tensor} self.fetch_list = [forward, cell] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_flatten2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_flatten2_matmul_fuse_pass.py index 21e0843a5aa..2d84561b770 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_flatten2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_flatten2_matmul_fuse_pass.py @@ -44,9 +44,10 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape and attr of flatten2 x_shape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=10), min_size=4, max_size=4 + ) + ) # [a, b, c, d] => [a, b*c*d] flatten_axis = 1 flatten_shape = [x_shape[0], x_shape[1] * x_shape[2] * x_shape[3]] @@ -58,9 +59,10 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = flatten_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -82,17 +84,11 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): "X": ["flatten2_x"], }, axis=flatten_axis, - outputs={ - "Out": ["flatten2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["flatten2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["flatten2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["flatten2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -107,10 +103,7 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -143,10 +136,12 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - max_duration=1000, - passes=["gpu_cpu_flatten2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=50, + max_duration=1000, + passes=["gpu_cpu_flatten2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_identity_scale_clean_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_identity_scale_clean_pass.py index e97eac16528..2261ba416d7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_identity_scale_clean_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_identity_scale_clean_pass.py @@ -20,7 +20,6 @@ import hypothesis.strategies as st class TestIdentityScaleCleanPass(PassAutoScanTest): - def sample_predictor_configs(self, program_config): config = self.create_trt_inference_config() config.enable_tensorrt_engine( @@ -29,7 +28,8 @@ class TestIdentityScaleCleanPass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['relu'], (1e-5, 1e-5) def sample_program_config(self, draw): @@ -39,25 +39,29 @@ class TestIdentityScaleCleanPass(PassAutoScanTest): h = draw(st.integers(min_value=1, max_value=20)) w = draw(st.integers(min_value=1, max_value=20)) - relu_op = OpConfig("relu", - inputs={"X": ["relu_x"]}, - outputs={"Out": ["relu_out"]}) - scale_op = OpConfig("scale", - inputs={"X": ["relu_out"]}, - outputs={"Out": ["scale_out"]}, - bias=0., - scale=1., - bias_after_scale=True) + relu_op = OpConfig( + "relu", inputs={"X": ["relu_x"]}, outputs={"Out": ["relu_out"]} + ) + scale_op = OpConfig( + "scale", + inputs={"X": ["relu_out"]}, + outputs={"Out": ["scale_out"]}, + bias=0.0, + scale=1.0, + bias_after_scale=True, + ) program_config = ProgramConfig( ops=[relu_op, scale_op], weights={}, inputs={"relu_x": TensorConfig(shape=[n, c, h, w])}, - outputs=["scale_out"]) + outputs=["scale_out"], + ) return program_config def test(self): - self.run_and_statis(max_examples=25, - passes=["identity_scale_op_clean_pass"]) + self.run_and_statis( + max_examples=25, passes=["identity_scale_op_clean_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_layer_norm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_layer_norm_fuse_pass.py index 9b928436b59..3c6bf6a1ca9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_layer_norm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_layer_norm_fuse_pass.py @@ -76,15 +76,17 @@ class TestFcFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=4, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=4, max_size=5 + ) + ) x_shape_rank = len(x_shape) # 2. Generate attrs of reduce_mean keep_dim = draw(st.booleans()) reduce_all = False begin_norm_axis = draw( - st.integers(min_value=1, max_value=x_shape_rank - 1)) + st.integers(min_value=1, max_value=x_shape_rank - 1) + ) if begin_norm_axis == x_shape_rank - 1 and draw(st.booleans()): reduce_mean_dim = [-1] else: @@ -108,22 +110,30 @@ class TestFcFusePass(PassAutoScanTest): pow_axis = -1 def generate_pow_data(): - return np.array([ - 2, - ], dtype="float32") + return np.array( + [ + 2, + ], + dtype="float32", + ) # 5. Generate attrs of elementwise_add if keep_dim: add_axis = draw( - st.integers(min_value=-1, max_value=x_shape_rank - 1)) + st.integers(min_value=-1, max_value=x_shape_rank - 1) + ) else: add_axis = draw( - st.integers(min_value=-1, max_value=begin_norm_axis - 1)) + st.integers(min_value=-1, max_value=begin_norm_axis - 1) + ) def generate_epsilon_data(): - return np.array([ - 1e-5, - ], dtype="float32") + return np.array( + [ + 1e-5, + ], + dtype="float32", + ) # 6. Generate attrs of elementwise_div div_axis = 0 @@ -151,19 +161,13 @@ class TestFcFusePass(PassAutoScanTest): ) sub_op = OpConfig( "elementwise_sub", - inputs={ - "X": ["x"], - "Y": ["mean_out"] - }, + inputs={"X": ["x"], "Y": ["mean_out"]}, outputs={"Out": ["sub_out"]}, axis=sub_axis, ) pow_op = OpConfig( "elementwise_pow", - inputs={ - "X": ["sub_out"], - "Y": ["pow_y"] - }, + inputs={"X": ["sub_out"], "Y": ["pow_y"]}, outputs={"Out": ["pow_out"]}, axis=pow_axis, ) @@ -179,10 +183,7 @@ class TestFcFusePass(PassAutoScanTest): ) add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["mean_out2"], - "Y": ["epsilon_var"] - }, + inputs={"X": ["mean_out2"], "Y": ["epsilon_var"]}, outputs={"Out": ["add_out"]}, axis=add_axis, ) @@ -195,35 +196,33 @@ class TestFcFusePass(PassAutoScanTest): ) div_op = OpConfig( "elementwise_div", - inputs={ - "X": ["sub_out"], - "Y": ["sqrt_out"] - }, + inputs={"X": ["sub_out"], "Y": ["sqrt_out"]}, outputs={"Out": ["div_out"]}, axis=div_axis, ) mul_op = OpConfig( "elementwise_mul", - inputs={ - "X": ["div_out"], - "Y": ["gamma_var"] - }, + inputs={"X": ["div_out"], "Y": ["gamma_var"]}, outputs={"Out": ["mul_out"]}, axis=mul_axis, ) add_op2 = OpConfig( "elementwise_add", - inputs={ - "X": ["mul_out"], - "Y": ["beta_var"] - }, + inputs={"X": ["mul_out"], "Y": ["beta_var"]}, outputs={"Out": ["add_out2"]}, axis=add_axis2, ) ops = [ - mean_op1, sub_op, pow_op, mean_op2, add_op, sqrt_op, div_op, mul_op, - add_op2 + mean_op1, + sub_op, + pow_op, + mean_op2, + add_op, + sqrt_op, + div_op, + mul_op, + add_op2, ] program_config = ProgramConfig( diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_layernorm_shift_partition_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_layernorm_shift_partition_pass.py index aeaff113f58..56436710f33 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_layernorm_shift_partition_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_layernorm_shift_partition_pass.py @@ -49,14 +49,19 @@ class TestLayernormShiftPartitionPass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({ - "input_data": [1, 9, 96], - }, { - "input_data": [4, 3136, 768], - }, { - "input_data": [1, 784, 384], - }) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + { + "input_data": [1, 9, 96], + }, + { + "input_data": [4, 3136, 768], + }, + { + "input_data": [1, 784, 384], + }, + ) yield config, ['layernorm_shift_partition'], (1e-5, 1e-5) # trt dynamic_shape @@ -67,14 +72,19 @@ class TestLayernormShiftPartitionPass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({ - "input_data": [1, 9, 96], - }, { - "input_data": [4, 3136, 768], - }, { - "input_data": [1, 784, 384], - }) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + { + "input_data": [1, 9, 96], + }, + { + "input_data": [4, 3136, 768], + }, + { + "input_data": [1, 784, 384], + }, + ) yield config, ['layernorm_shift_partition'], (1e-3, 1e-3) def sample_program_config(self, draw): @@ -90,95 +100,112 @@ class TestLayernormShiftPartitionPass(PassAutoScanTest): def generate_input(attrs): return np.random.random( - [attrs[1]["batch_size"], - *attrs[1]["input_dim"]]).astype(np.float32) + [attrs[1]["batch_size"], *attrs[1]["input_dim"]] + ).astype(np.float32) def generate_weight(attrs): return np.random.random(attrs[1]['input_dim'][-1]).astype( - np.float32) + np.float32 + ) - attrs = [{ - 'begin_norm_axis': begin_norm_axis, - 'epsilon': epsilon, - }, { - 'batch_size': batch_size, - 'input_dim': [(window_size * move_shape)**2, dim], - }, { - 'axis': axis, - 'input_resolution': window_size * move_shape, - 'move_shape': move_shape, - 'window_size': window_size, - }] + attrs = [ + { + 'begin_norm_axis': begin_norm_axis, + 'epsilon': epsilon, + }, + { + 'batch_size': batch_size, + 'input_dim': [(window_size * move_shape) ** 2, dim], + }, + { + 'axis': axis, + 'input_resolution': window_size * move_shape, + 'move_shape': move_shape, + 'window_size': window_size, + }, + ] - layer_norm_op = OpConfig(type="layer_norm", - inputs={ - "X": ["input_data"], - "Bias": ["layer_norm_bias"], - "Scale": ["layer_norm_scale"] - }, - outputs={ - "Y": ["layer_norm_output1"], - "Mean": ["layer_norm_output2"], - "Variance": ["layer_norm_output3"] - }, - attrs={ - "begin_norm_axis": - attrs[0]["begin_norm_axis"], - "epsilon": attrs[0]["epsilon"], - }) - reshape_op2 = OpConfig(type="reshape2", - inputs={ - "X": ["layer_norm_output1"], - }, - outputs={ - "Out": ["reshape_output2"], - "XShape": ["reshape_output2_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["input_resolution"], - attrs[2]["input_resolution"], - attrs[1]["input_dim"][-1] - ] - }) - reshape_op3 = OpConfig(type="reshape2", - inputs={ - "X": ["reshape_output2"], - }, - outputs={ - "Out": ["reshape_output3"], - "XShape": ["reshape_output3_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["move_shape"], - attrs[2]["window_size"], - attrs[2]["move_shape"], - attrs[2]["window_size"], - attrs[1]["input_dim"][-1] - ] - }) - transpose_op4 = OpConfig(type='transpose2', - inputs={ - "X": ["reshape_output3"], - }, - outputs={"Out": ["transpose_output4"]}, - attrs={"axis": attrs[2]['axis']}) - reshape_op5 = OpConfig(type="reshape2", - inputs={ - "X": ["transpose_output4"], - }, - outputs={ - "Out": ["reshape_output5"], - "XShape": ["reshape_output5_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["window_size"], - attrs[2]["window_size"], - attrs[1]["input_dim"][-1] - ] - }) + layer_norm_op = OpConfig( + type="layer_norm", + inputs={ + "X": ["input_data"], + "Bias": ["layer_norm_bias"], + "Scale": ["layer_norm_scale"], + }, + outputs={ + "Y": ["layer_norm_output1"], + "Mean": ["layer_norm_output2"], + "Variance": ["layer_norm_output3"], + }, + attrs={ + "begin_norm_axis": attrs[0]["begin_norm_axis"], + "epsilon": attrs[0]["epsilon"], + }, + ) + reshape_op2 = OpConfig( + type="reshape2", + inputs={ + "X": ["layer_norm_output1"], + }, + outputs={ + "Out": ["reshape_output2"], + "XShape": ["reshape_output2_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["input_resolution"], + attrs[2]["input_resolution"], + attrs[1]["input_dim"][-1], + ] + }, + ) + reshape_op3 = OpConfig( + type="reshape2", + inputs={ + "X": ["reshape_output2"], + }, + outputs={ + "Out": ["reshape_output3"], + "XShape": ["reshape_output3_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["move_shape"], + attrs[2]["window_size"], + attrs[2]["move_shape"], + attrs[2]["window_size"], + attrs[1]["input_dim"][-1], + ] + }, + ) + transpose_op4 = OpConfig( + type='transpose2', + inputs={ + "X": ["reshape_output3"], + }, + outputs={"Out": ["transpose_output4"]}, + attrs={"axis": attrs[2]['axis']}, + ) + reshape_op5 = OpConfig( + type="reshape2", + inputs={ + "X": ["transpose_output4"], + }, + outputs={ + "Out": ["reshape_output5"], + "XShape": ["reshape_output5_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["window_size"], + attrs[2]["window_size"], + attrs[1]["input_dim"][-1], + ] + }, + ) reshape_op6 = OpConfig( type="reshape2", inputs={ @@ -189,35 +216,49 @@ class TestLayernormShiftPartitionPass(PassAutoScanTest): "XShape": ["reshape_output6_xshape"], }, attrs={ - 'shape': - [-1, attrs[2]["window_size"]**2, attrs[1]["input_dim"][-1]] - }) + 'shape': [ + -1, + attrs[2]["window_size"] ** 2, + attrs[1]["input_dim"][-1], + ] + }, + ) program_config = ProgramConfig( ops=[ - layer_norm_op, reshape_op2, reshape_op3, transpose_op4, - reshape_op5, reshape_op6 + layer_norm_op, + reshape_op2, + reshape_op3, + transpose_op4, + reshape_op5, + reshape_op6, ], weights={ - "layer_norm_bias": - TensorConfig(data_gen=partial(generate_weight, attrs)), - "layer_norm_scale": - TensorConfig(data_gen=partial(generate_weight, attrs)) + "layer_norm_bias": TensorConfig( + data_gen=partial(generate_weight, attrs) + ), + "layer_norm_scale": TensorConfig( + data_gen=partial(generate_weight, attrs) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, attrs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, attrs) + ), }, - outputs=["reshape_output6"]) + outputs=["reshape_output6"], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - passes=["layernorm_shift_partition_fuse_pass"], - max_duration=250, - min_success_num=50) + self.run_and_statis( + quant=False, + max_examples=50, + passes=["layernorm_shift_partition_fuse_pass"], + max_duration=250, + min_success_num=50, + ) class TestLayernormShiftPartition2Pass(PassAutoScanTest): @@ -248,14 +289,19 @@ class TestLayernormShiftPartition2Pass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({ - "input_data": [1, 9, 96], - }, { - "input_data": [4, 3136, 768], - }, { - "input_data": [1, 784, 384], - }) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + { + "input_data": [1, 9, 96], + }, + { + "input_data": [4, 3136, 768], + }, + { + "input_data": [1, 784, 384], + }, + ) yield config, ['layernorm_shift_partition'], (1e-5, 1e-5) # trt dynamic_shape @@ -266,14 +312,19 @@ class TestLayernormShiftPartition2Pass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({ - "input_data": [1, 9, 96], - }, { - "input_data": [4, 3136, 768], - }, { - "input_data": [1, 784, 384], - }) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + { + "input_data": [1, 9, 96], + }, + { + "input_data": [4, 3136, 768], + }, + { + "input_data": [1, 784, 384], + }, + ) yield config, ['layernorm_shift_partition'], (1e-3, 1e-3) def sample_program_config(self, draw): @@ -289,106 +340,124 @@ class TestLayernormShiftPartition2Pass(PassAutoScanTest): def generate_input(attrs): return np.random.random( - [attrs[1]["batch_size"], - *attrs[1]["input_dim"]]).astype(np.float32) + [attrs[1]["batch_size"], *attrs[1]["input_dim"]] + ).astype(np.float32) def generate_weight(attrs): return np.random.random(attrs[1]['input_dim'][-1]).astype( - np.float32) + np.float32 + ) - attrs = [{ - 'begin_norm_axis': begin_norm_axis, - 'epsilon': epsilon, - }, { - 'batch_size': batch_size, - 'input_dim': [(window_size * move_shape)**2, dim], - }, { - 'axis': axis, - 'input_resolution': window_size * move_shape, - 'move_shape': move_shape, - 'window_size': window_size, - }] + attrs = [ + { + 'begin_norm_axis': begin_norm_axis, + 'epsilon': epsilon, + }, + { + 'batch_size': batch_size, + 'input_dim': [(window_size * move_shape) ** 2, dim], + }, + { + 'axis': axis, + 'input_resolution': window_size * move_shape, + 'move_shape': move_shape, + 'window_size': window_size, + }, + ] - layer_norm_op = OpConfig(type="layer_norm", - inputs={ - "X": ["input_data"], - "Bias": ["layer_norm_bias"], - "Scale": ["layer_norm_scale"] - }, - outputs={ - "Y": ["layer_norm_output1"], - "Mean": ["layer_norm_output2"], - "Variance": ["layer_norm_output3"] - }, - attrs={ - "begin_norm_axis": - attrs[0]["begin_norm_axis"], - "epsilon": attrs[0]["epsilon"], - }) - reshape_op2 = OpConfig(type="reshape2", - inputs={ - "X": ["layer_norm_output1"], - }, - outputs={ - "Out": ["reshape_output2"], - "XShape": ["reshape_output2_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["input_resolution"], - attrs[2]["input_resolution"], - attrs[1]["input_dim"][-1] - ] - }) - roll_op1 = OpConfig(type="roll", - inputs={"X": ["reshape_output2"]}, - outputs={"Out": ["roll_output1"]}, - attrs={ - "axis": [1, 2], - "shifts": [ - -math.floor( - (attrs[2]["window_size"]) / 2.0), - -math.floor((attrs[2]["window_size"]) / 2.0) - ] - }) - reshape_op3 = OpConfig(type="reshape2", - inputs={ - "X": ["roll_output1"], - }, - outputs={ - "Out": ["reshape_output3"], - "XShape": ["reshape_output3_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["move_shape"], - attrs[2]["window_size"], - attrs[2]["move_shape"], - attrs[2]["window_size"], - attrs[1]["input_dim"][-1] - ] - }) - transpose_op4 = OpConfig(type='transpose2', - inputs={ - "X": ["reshape_output3"], - }, - outputs={"Out": ["transpose_output4"]}, - attrs={"axis": attrs[2]['axis']}) - reshape_op5 = OpConfig(type="reshape2", - inputs={ - "X": ["transpose_output4"], - }, - outputs={ - "Out": ["reshape_output5"], - "XShape": ["reshape_output5_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["window_size"], - attrs[2]["window_size"], - attrs[1]["input_dim"][-1] - ] - }) + layer_norm_op = OpConfig( + type="layer_norm", + inputs={ + "X": ["input_data"], + "Bias": ["layer_norm_bias"], + "Scale": ["layer_norm_scale"], + }, + outputs={ + "Y": ["layer_norm_output1"], + "Mean": ["layer_norm_output2"], + "Variance": ["layer_norm_output3"], + }, + attrs={ + "begin_norm_axis": attrs[0]["begin_norm_axis"], + "epsilon": attrs[0]["epsilon"], + }, + ) + reshape_op2 = OpConfig( + type="reshape2", + inputs={ + "X": ["layer_norm_output1"], + }, + outputs={ + "Out": ["reshape_output2"], + "XShape": ["reshape_output2_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["input_resolution"], + attrs[2]["input_resolution"], + attrs[1]["input_dim"][-1], + ] + }, + ) + roll_op1 = OpConfig( + type="roll", + inputs={"X": ["reshape_output2"]}, + outputs={"Out": ["roll_output1"]}, + attrs={ + "axis": [1, 2], + "shifts": [ + -math.floor((attrs[2]["window_size"]) / 2.0), + -math.floor((attrs[2]["window_size"]) / 2.0), + ], + }, + ) + reshape_op3 = OpConfig( + type="reshape2", + inputs={ + "X": ["roll_output1"], + }, + outputs={ + "Out": ["reshape_output3"], + "XShape": ["reshape_output3_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["move_shape"], + attrs[2]["window_size"], + attrs[2]["move_shape"], + attrs[2]["window_size"], + attrs[1]["input_dim"][-1], + ] + }, + ) + transpose_op4 = OpConfig( + type='transpose2', + inputs={ + "X": ["reshape_output3"], + }, + outputs={"Out": ["transpose_output4"]}, + attrs={"axis": attrs[2]['axis']}, + ) + reshape_op5 = OpConfig( + type="reshape2", + inputs={ + "X": ["transpose_output4"], + }, + outputs={ + "Out": ["reshape_output5"], + "XShape": ["reshape_output5_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["window_size"], + attrs[2]["window_size"], + attrs[1]["input_dim"][-1], + ] + }, + ) reshape_op6 = OpConfig( type="reshape2", inputs={ @@ -399,35 +468,50 @@ class TestLayernormShiftPartition2Pass(PassAutoScanTest): "XShape": ["reshape_output6_xshape"], }, attrs={ - 'shape': - [-1, attrs[2]["window_size"]**2, attrs[1]["input_dim"][-1]] - }) + 'shape': [ + -1, + attrs[2]["window_size"] ** 2, + attrs[1]["input_dim"][-1], + ] + }, + ) program_config = ProgramConfig( ops=[ - layer_norm_op, reshape_op2, roll_op1, reshape_op3, - transpose_op4, reshape_op5, reshape_op6 + layer_norm_op, + reshape_op2, + roll_op1, + reshape_op3, + transpose_op4, + reshape_op5, + reshape_op6, ], weights={ - "layer_norm_bias": - TensorConfig(data_gen=partial(generate_weight, attrs)), - "layer_norm_scale": - TensorConfig(data_gen=partial(generate_weight, attrs)) + "layer_norm_bias": TensorConfig( + data_gen=partial(generate_weight, attrs) + ), + "layer_norm_scale": TensorConfig( + data_gen=partial(generate_weight, attrs) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, attrs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, attrs) + ), }, - outputs=["reshape_output6"]) + outputs=["reshape_output6"], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - passes=["layernorm_shift_partition_fuse_pass"], - max_duration=250, - min_success_num=50) + self.run_and_statis( + quant=False, + max_examples=50, + passes=["layernorm_shift_partition_fuse_pass"], + max_duration=250, + min_success_num=50, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_to_mul_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_to_mul_pass.py index e6db1204897..138782f00e1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_to_mul_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_to_mul_pass.py @@ -21,9 +21,9 @@ import hypothesis.strategies as st class TestMapMatmulToMulPass(PassAutoScanTest): r""" - x_var y_var(persistable) - \ / - matmul + x_var y_var(persistable) + \ / + matmul """ def sample_predictor_configs(self, program_config): @@ -67,19 +67,23 @@ class TestMapMatmulToMulPass(PassAutoScanTest): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass error on TRT while shape of mul_x > 5.") + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass error on TRT while shape of mul_x > 5.", + ) def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[-1] alpha = 1.0 transpose_X = False @@ -87,10 +91,7 @@ class TestMapMatmulToMulPass(PassAutoScanTest): matmul_op = OpConfig( "matmul", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -121,10 +122,12 @@ class TestMapMatmulToMulPass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=["gpu_cpu_map_matmul_to_mul_pass"], - max_duration=180) + self.run_and_statis( + quant=False, + max_examples=100, + passes=["gpu_cpu_map_matmul_to_mul_pass"], + max_duration=180, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_matmul_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_matmul_pass.py index 38f86dc7170..d2dec606073 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_matmul_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_matmul_pass.py @@ -21,9 +21,9 @@ import hypothesis.strategies as st class TestMapMatmulToMulPass(PassAutoScanTest): r""" - x_var y_var(persistable) - \ / - matmul_v2 + x_var y_var(persistable) + \ / + matmul_v2 """ def sample_predictor_configs(self, program_config): @@ -63,19 +63,23 @@ class TestMapMatmulToMulPass(PassAutoScanTest): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass error on TRT while shape of mul_x > 5.") + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass error on TRT while shape of mul_x > 5.", + ) def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) transpose_X = draw(st.booleans()) transpose_Y = draw(st.booleans()) if transpose_X: @@ -89,15 +93,12 @@ class TestMapMatmulToMulPass(PassAutoScanTest): else: y_shape[0] = x_shape[-1] - y_shape = x_shape[0:len(x_shape) - 2] + y_shape + y_shape = x_shape[0 : len(x_shape) - 2] + y_shape alpha = 1.0 matmul_op = OpConfig( "matmul_v2", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, trans_x=transpose_X, @@ -128,9 +129,11 @@ class TestMapMatmulToMulPass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=["gpu_cpu_map_matmul_v2_to_matmul_pass"]) + self.run_and_statis( + quant=False, + max_examples=100, + passes=["gpu_cpu_map_matmul_v2_to_matmul_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_mul_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_mul_pass.py index d4c6db1c7f5..148610edad1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_mul_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_mul_pass.py @@ -21,9 +21,9 @@ import hypothesis.strategies as st class TestMapMatmulToMulPass(PassAutoScanTest): r""" - x_var y_var(persistable) - \ / - matmul_v2 + x_var y_var(persistable) + \ / + matmul_v2 """ def sample_predictor_configs(self, program_config): @@ -63,19 +63,23 @@ class TestMapMatmulToMulPass(PassAutoScanTest): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass error on TRT while shape of mul_x > 5.") + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass error on TRT while shape of mul_x > 5.", + ) def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[-1] alpha = 1.0 transpose_X = False @@ -83,10 +87,7 @@ class TestMapMatmulToMulPass(PassAutoScanTest): matmul_op = OpConfig( "matmul_v2", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, trans_x=transpose_X, @@ -117,9 +118,11 @@ class TestMapMatmulToMulPass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=["gpu_cpu_map_matmul_v2_to_mul_pass"]) + self.run_and_statis( + quant=False, + max_examples=100, + passes=["gpu_cpu_map_matmul_v2_to_mul_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_scale_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_scale_fuse_pass.py index 6db5468292c..2a60d62db3d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_scale_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_scale_fuse_pass.py @@ -21,11 +21,11 @@ import hypothesis.strategies as st class TestMatmulScaleFusePass(PassAutoScanTest): r""" - x_var y_var(persistable) - \ / - matmul - | - scale + x_var y_var(persistable) + \ / + matmul + | + scale """ def sample_predictor_configs(self, program_config): @@ -50,14 +50,18 @@ class TestMatmulScaleFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) x_shape_rank = len(x_shape) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=x_shape_rank, - max_size=x_shape_rank)) + st.lists( + st.integers(min_value=1, max_value=8), + min_size=x_shape_rank, + max_size=x_shape_rank, + ) + ) y_shape_rank = len(y_shape) y_shape[-2] = x_shape[-1] for i in range(y_shape_rank - 3, -1, -1): @@ -75,10 +79,7 @@ class TestMatmulScaleFusePass(PassAutoScanTest): matmul_op = OpConfig( "matmul", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, transpose_X=transpose_X, transpose_Y=transpose_Y, @@ -95,10 +96,7 @@ class TestMatmulScaleFusePass(PassAutoScanTest): if is_scale_tensor: scale_op = OpConfig( "scale", - inputs={ - "X": ["matmul_out"], - "ScaleTensor": ["scale_tensor"] - }, + inputs={"X": ["matmul_out"], "ScaleTensor": ["scale_tensor"]}, outputs={"Out": ["scale_out"]}, scale=scale_value, bias=0.0, @@ -122,7 +120,7 @@ class TestMatmulScaleFusePass(PassAutoScanTest): if is_scale_tensor: weights = { "matmul_y": TensorConfig(shape=y_shape), - "scale_tensor": TensorConfig(shape=scale_shape) + "scale_tensor": TensorConfig(shape=scale_shape), } inputs = { "matmul_x": TensorConfig(shape=x_shape), diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_v2_scale_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_v2_scale_fuse_pass.py index a9b48e7cf36..4b0c226e1a7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_v2_scale_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_v2_scale_fuse_pass.py @@ -21,13 +21,13 @@ import hypothesis.strategies as st class TestMatmulV2ScaleFusePass(PassAutoScanTest): r""" - x_var y_var(persistable) x_var y_var*scale(persistable) - \ / \ / - matmul_v2 matmul_v2 - | => | - scale scale_out - ｜ - scale_out + x_var y_var(persistable) x_var y_var*scale(persistable) + \ / \ / + matmul_v2 matmul_v2 + | => | + scale scale_out + ｜ + scale_out """ def sample_predictor_configs(self, program_config): @@ -44,14 +44,18 @@ class TestMatmulV2ScaleFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) x_shape_rank = len(x_shape) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=x_shape_rank, - max_size=x_shape_rank)) + st.lists( + st.integers(min_value=1, max_value=8), + min_size=x_shape_rank, + max_size=x_shape_rank, + ) + ) y_shape_rank = len(y_shape) y_shape[-2] = x_shape[-1] for i in range(y_shape_rank - 3, -1, -1): @@ -68,10 +72,7 @@ class TestMatmulV2ScaleFusePass(PassAutoScanTest): matmul_v2_op = OpConfig( "matmul_v2", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, trans_x=transpose_X, trans_y=transpose_Y, @@ -86,10 +87,7 @@ class TestMatmulV2ScaleFusePass(PassAutoScanTest): if is_scale_tensor: scale_op = OpConfig( "scale", - inputs={ - "X": ["matmul_out"], - "ScaleTensor": ["scale_tensor"] - }, + inputs={"X": ["matmul_out"], "ScaleTensor": ["scale_tensor"]}, outputs={"Out": ["scale_out"]}, scale=scale_value, bias=0.0, diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_merge_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_merge_layernorm_fuse_pass.py index cd25181f501..897e63e7cf5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_merge_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_merge_layernorm_fuse_pass.py @@ -47,10 +47,13 @@ class TestMergeLayernormFusePass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({"input_data": [1, 196, 96]}, - {"input_data": [4, 3136, 384]}, - {"input_data": [1, 3136, 96]}) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + {"input_data": [1, 196, 96]}, + {"input_data": [4, 3136, 384]}, + {"input_data": [1, 3136, 96]}, + ) yield config, ["merge_layernorm"], (1e-5, 1e-5) # trt dynamic_shape fp16 config = self.create_trt_inference_config() @@ -60,10 +63,13 @@ class TestMergeLayernormFusePass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({"input_data": [1, 196, 96]}, - {"input_data": [4, 3136, 384]}, - {"input_data": [1, 3136, 96]}) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + {"input_data": [1, 196, 96]}, + {"input_data": [4, 3136, 384]}, + {"input_data": [1, 3136, 96]}, + ) yield config, ["merge_layernorm"], (1e-3, 1e-3) def sample_program_config(self, draw): @@ -72,39 +78,45 @@ class TestMergeLayernormFusePass(PassAutoScanTest): input_n = draw(st.sampled_from([96, 192, 384])) layernorm_40_begin_norm_axis = 2 layernorm_40_epsilon = draw( - st.floats(min_value=0.0000001, max_value=0.001)) + st.floats(min_value=0.0000001, max_value=0.001) + ) def generate_input(attrs): - return np.random.random([ - attrs[3]['batch_size'], - attrs[3]['input_H_W'] * attrs[3]['input_H_W'], - attrs[3]['input_n'] - ]).astype(np.float32) + return np.random.random( + [ + attrs[3]['batch_size'], + attrs[3]['input_H_W'] * attrs[3]['input_H_W'], + attrs[3]['input_n'], + ] + ).astype(np.float32) def generate_weight(attrs): - return np.random.random([attrs[3]['input_n'] * 4 - ]).astype(np.float32) + return np.random.random([attrs[3]['input_n'] * 4]).astype( + np.float32 + ) - attrs = [{ - 'shape': [-1, input_H_W, input_H_W, input_n] - }, { - 'shape': [-1, int(input_H_W * input_H_W / 4), - int(input_n * 4)] - }, { - 'begin_norm_axis': layernorm_40_begin_norm_axis, - 'epsilon': layernorm_40_epsilon - }, { - 'batch_size': batch_size, - 'input_H_W': input_H_W, - 'input_n': input_n - }] - reshape2_00_op = OpConfig(type="reshape2", - inputs={'X': ['input_data']}, - outputs={ - 'Out': ['reshape2_00_out'], - 'XShape': ['reshape2_00_outxshape'] - }, - attrs={'shape': attrs[0]['shape']}) + attrs = [ + {'shape': [-1, input_H_W, input_H_W, input_n]}, + {'shape': [-1, int(input_H_W * input_H_W / 4), int(input_n * 4)]}, + { + 'begin_norm_axis': layernorm_40_begin_norm_axis, + 'epsilon': layernorm_40_epsilon, + }, + { + 'batch_size': batch_size, + 'input_H_W': input_H_W, + 'input_n': input_n, + }, + ] + reshape2_00_op = OpConfig( + type="reshape2", + inputs={'X': ['input_data']}, + outputs={ + 'Out': ['reshape2_00_out'], + 'XShape': ['reshape2_00_outxshape'], + }, + attrs={'shape': attrs[0]['shape']}, + ) strided_slice_10_op = OpConfig( type="strided_slice", inputs={'Input': ['reshape2_00_out']}, @@ -114,8 +126,9 @@ class TestMergeLayernormFusePass(PassAutoScanTest): 'starts': [0, 0], 'infer_flags': [1, 1], 'ends': [attrs[3]['input_H_W'], attrs[3]['input_H_W']], - 'strides': [2, 2] - }) + 'strides': [2, 2], + }, + ) strided_slice_11_op = OpConfig( type="strided_slice", inputs={'Input': ['reshape2_00_out']}, @@ -125,8 +138,9 @@ class TestMergeLayernormFusePass(PassAutoScanTest): 'starts': [1, 0], 'infer_flags': [1, 1], 'ends': [attrs[3]['input_H_W'], attrs[3]['input_H_W']], - 'strides': [2, 2] - }) + 'strides': [2, 2], + }, + ) strided_slice_12_op = OpConfig( type="strided_slice", inputs={'Input': ['reshape2_00_out']}, @@ -136,8 +150,9 @@ class TestMergeLayernormFusePass(PassAutoScanTest): 'starts': [0, 1], 'infer_flags': [1, 1], 'ends': [attrs[3]['input_H_W'], attrs[3]['input_H_W']], - 'strides': [2, 2] - }) + 'strides': [2, 2], + }, + ) strided_slice_13_op = OpConfig( type="strided_slice", inputs={'Input': ['reshape2_00_out']}, @@ -147,68 +162,84 @@ class TestMergeLayernormFusePass(PassAutoScanTest): 'starts': [1, 1], 'infer_flags': [1, 1], 'ends': [attrs[3]['input_H_W'], attrs[3]['input_H_W']], - 'strides': [2, 2] - }) - concat_20_op = OpConfig(type="concat", - inputs={ - 'X': [ - 'strided_slice_10_out', - 'strided_slice_11_out', - 'strided_slice_12_out', - 'strided_slice_13_out' - ] - }, - outputs={'Out': ['concat_20_out']}, - attrs={'axis': -1}) - reshape2_30_op = OpConfig(type='reshape2', - inputs={'X': ['concat_20_out']}, - outputs={ - 'Out': ['reshape2_30_Out'], - 'XShape': ['reshape2_30_XShape'] - }, - attrs={'shape': attrs[1]['shape']}) - layernorm_40_op = OpConfig(type='layer_norm', - inputs={ - 'X': ['reshape2_30_Out'], - 'Bias': ['layer_norm_bias'], - 'Scale': ['layer_norm_scale'] - }, - outputs={ - "Y": ["layer_norm_out"], - "Mean": ["layer_norm_outMean"], - "Variance": ["layer_norm_outVariance"] - }, - attrs={ - 'begin_norm_axis': - attrs[2]['begin_norm_axis'], - 'epsilon': - attrs[2]['epsilon'] - }) + 'strides': [2, 2], + }, + ) + concat_20_op = OpConfig( + type="concat", + inputs={ + 'X': [ + 'strided_slice_10_out', + 'strided_slice_11_out', + 'strided_slice_12_out', + 'strided_slice_13_out', + ] + }, + outputs={'Out': ['concat_20_out']}, + attrs={'axis': -1}, + ) + reshape2_30_op = OpConfig( + type='reshape2', + inputs={'X': ['concat_20_out']}, + outputs={ + 'Out': ['reshape2_30_Out'], + 'XShape': ['reshape2_30_XShape'], + }, + attrs={'shape': attrs[1]['shape']}, + ) + layernorm_40_op = OpConfig( + type='layer_norm', + inputs={ + 'X': ['reshape2_30_Out'], + 'Bias': ['layer_norm_bias'], + 'Scale': ['layer_norm_scale'], + }, + outputs={ + "Y": ["layer_norm_out"], + "Mean": ["layer_norm_outMean"], + "Variance": ["layer_norm_outVariance"], + }, + attrs={ + 'begin_norm_axis': attrs[2]['begin_norm_axis'], + 'epsilon': attrs[2]['epsilon'], + }, + ) program_config = ProgramConfig( ops=[ - reshape2_00_op, strided_slice_10_op, strided_slice_11_op, - strided_slice_12_op, strided_slice_13_op, concat_20_op, - reshape2_30_op, layernorm_40_op + reshape2_00_op, + strided_slice_10_op, + strided_slice_11_op, + strided_slice_12_op, + strided_slice_13_op, + concat_20_op, + reshape2_30_op, + layernorm_40_op, ], weights={ - 'layer_norm_bias': - TensorConfig(data_gen=partial(generate_weight, attrs)), - 'layer_norm_scale': - TensorConfig(data_gen=partial(generate_weight, attrs)) + 'layer_norm_bias': TensorConfig( + data_gen=partial(generate_weight, attrs) + ), + 'layer_norm_scale': TensorConfig( + data_gen=partial(generate_weight, attrs) + ), }, inputs={ - 'input_data': - TensorConfig(data_gen=partial(generate_input, attrs)) + 'input_data': TensorConfig( + data_gen=partial(generate_input, attrs) + ) }, - outputs=['layer_norm_out']) + outputs=['layer_norm_out'], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - passes=["merge_layernorm_fuse_pass"], - max_duration=250, - min_success_num=50) + self.run_and_statis( + quant=False, + max_examples=50, + passes=["merge_layernorm_fuse_pass"], + max_duration=250, + min_success_num=50, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_batch_norm_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_batch_norm_act_fuse_pass.py index 63bb9e7256d..85c3aa2cd1c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_batch_norm_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_batch_norm_act_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -55,40 +54,40 @@ class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): def generate_weight(): return np.random.random(channel).astype(np.float32) - batch_norm_op = OpConfig(type="batch_norm", - inputs={ - "X": ["input_data"], - "Bias": ["Bias"], - "Mean": ["Mean"], - "Scale": ["Scale"], - "Variance": ["Variance"] - }, - outputs={ - "Y": ["norm_output"], - "MeanOut": ["Mean"], - "VarianceOut": ["Variance"], - "SavedMean": ["SavedMean"], - "SavedVariance": ["SavedVariance"] - }, - attrs={ - "data_layout": data_layout, - "epsilon": epsilon, - "fuse_with_relu": fuse_with_relu, - "is_test": is_test, - "momentum": momentum, - "trainable_statistics": - trainable_statistics, - "use_global_stats": use_global_stats, - "use_mkldnn": use_mkldnn1 - }) - - relu_op = OpConfig(type="relu", - inputs={"X": ["norm_output"]}, - outputs={"Out": ["relu_output"]}, - attrs={ - "use_cudnn": use_cudnn, - "use_mkldnn": use_mkldnn2 - }) + batch_norm_op = OpConfig( + type="batch_norm", + inputs={ + "X": ["input_data"], + "Bias": ["Bias"], + "Mean": ["Mean"], + "Scale": ["Scale"], + "Variance": ["Variance"], + }, + outputs={ + "Y": ["norm_output"], + "MeanOut": ["Mean"], + "VarianceOut": ["Variance"], + "SavedMean": ["SavedMean"], + "SavedVariance": ["SavedVariance"], + }, + attrs={ + "data_layout": data_layout, + "epsilon": epsilon, + "fuse_with_relu": fuse_with_relu, + "is_test": is_test, + "momentum": momentum, + "trainable_statistics": trainable_statistics, + "use_global_stats": use_global_stats, + "use_mkldnn": use_mkldnn1, + }, + ) + + relu_op = OpConfig( + type="relu", + inputs={"X": ["norm_output"]}, + outputs={"Out": ["relu_output"]}, + attrs={"use_cudnn": use_cudnn, "use_mkldnn": use_mkldnn2}, + ) model_net = [batch_norm_op, relu_op] @@ -98,12 +97,13 @@ class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): "Bias": TensorConfig(data_gen=partial(generate_weight)), "Mean": TensorConfig(data_gen=partial(generate_weight)), "Scale": TensorConfig(data_gen=partial(generate_weight)), - "Variance": TensorConfig(data_gen=partial(generate_weight)) + "Variance": TensorConfig(data_gen=partial(generate_weight)), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["relu_output"]) + outputs=["relu_output"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_bias_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_bias_fuse_pass.py index 21d32349290..9bf4fa55b0c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_bias_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_bias_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestConv3dBiasMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,78 +38,80 @@ class TestConv3dBiasMkldnnFusePass(PassAutoScanTest): def generate_input1(attrs): if attrs[0]['data_format'] == "NCDHW": return np.random.random( - [attrs[2]['batch_size'], 48, 64, 32, 64]).astype(np.float32) + [attrs[2]['batch_size'], 48, 64, 32, 64] + ).astype(np.float32) else: return np.random.random( - [attrs[2]['batch_size'], 64, 32, 64, 48]).astype(np.float32) + [attrs[2]['batch_size'], 64, 32, 64, 48] + ).astype(np.float32) def generate_weight1(): - return np.random.random([16, int(48 / groups), 3, 3, - 3]).astype(np.float32) + return np.random.random([16, int(48 / groups), 3, 3, 3]).astype( + np.float32 + ) def generate_weight2(): return np.random.random([16]).astype(np.float32) - attrs = [{ - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - }, { - "axis": axis - }, { - 'batch_size': batch_size - }] - - ops_config = [{ - "op_type": "conv3d", - "op_inputs": { - "Input": ["input_data1"], - "Filter": ["conv_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": attrs[0]['data_format'], - "dilations": attrs[0]['dilations'], - "padding_algorithm": attrs[0]['padding_algorithm'], - "groups": attrs[0]['groups'], - "paddings": attrs[0]['paddings'], - "strides": attrs[0]['strides'], - "is_test": True - } - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["conv_output"], - "Y": ["elementwise_weight"] + attrs = [ + { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, }, - "op_outputs": { - "Out": ["elementwise_output"] + {"axis": axis}, + {'batch_size': batch_size}, + ] + + ops_config = [ + { + "op_type": "conv3d", + "op_inputs": { + "Input": ["input_data1"], + "Filter": ["conv_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": attrs[0]['data_format'], + "dilations": attrs[0]['dilations'], + "padding_algorithm": attrs[0]['padding_algorithm'], + "groups": attrs[0]['groups'], + "paddings": attrs[0]['paddings'], + "strides": attrs[0]['strides'], + "is_test": True, + }, }, - "op_attrs": { - 'axis': attrs[1]['axis'] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["conv_output"], + "Y": ["elementwise_weight"], + }, + "op_outputs": {"Out": ["elementwise_output"]}, + "op_attrs": {'axis': attrs[1]['axis']}, }, - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv_weight": - TensorConfig(data_gen=partial(generate_weight1)), - "elementwise_weight": - TensorConfig(data_gen=partial(generate_weight2)) + "conv_weight": TensorConfig(data_gen=partial(generate_weight1)), + "elementwise_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input1, attrs)) + "input_data1": TensorConfig( + data_gen=partial(generate_input1, attrs) + ) }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_op.py index 22454e55a13..f9ea557a53a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_op.py @@ -23,53 +23,54 @@ import hypothesis.strategies as st class TestMkldnnConv3dOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): if kwargs["data_format"] == "NCDHW": - return np.random.random([kwargs["batch_size"], 48, 64, 32, - 64]).astype(np.float32) + return np.random.random( + [kwargs["batch_size"], 48, 64, 32, 64] + ).astype(np.float32) else: - return np.random.random([kwargs["batch_size"], 64, 32, 64, - 48]).astype(np.float32) + return np.random.random( + [kwargs["batch_size"], 64, 32, 64, 48] + ).astype(np.float32) def generate_weight(*args, **kwargs): - return np.random.random([16, - int(48 / kwargs["groups"]), 3, 3, - 3]).astype(np.float32) + return np.random.random( + [16, int(48 / kwargs["groups"]), 3, 3, 3] + ).astype(np.float32) - conv3d_op = OpConfig(type="conv3d", - inputs={ - "Input": ["input_data"], - "Filter": ["conv_weight"] - }, - outputs={"Output": ["conv_output"]}, - attrs={ - "data_format": kwargs["data_format"], - "dilations": kwargs["dilations"], - "padding_algorithm": - kwargs["padding_algorithm"], - "groups": kwargs["groups"], - "paddings": kwargs["paddings"], - "strides": kwargs["strides"], - "is_test": True - }) + conv3d_op = OpConfig( + type="conv3d", + inputs={"Input": ["input_data"], "Filter": ["conv_weight"]}, + outputs={"Output": ["conv_output"]}, + attrs={ + "data_format": kwargs["data_format"], + "dilations": kwargs["dilations"], + "padding_algorithm": kwargs["padding_algorithm"], + "groups": kwargs["groups"], + "paddings": kwargs["paddings"], + "strides": kwargs["strides"], + "is_test": True, + }, + ) program_config = ProgramConfig( ops=[conv3d_op], weights={ - "conv_weight": - TensorConfig(data_gen=partial(generate_weight, *args, **kwargs)) + "conv_weight": TensorConfig( + data_gen=partial(generate_weight, *args, **kwargs) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)) + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ) }, - outputs=["conv_output"]) + outputs=["conv_output"], + ) yield program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py index fb70f2260a3..bc4d527be87 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py @@ -20,18 +20,19 @@ from paddle.fluid.core import PassVersionChecker class ConvActivationMkldnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") - conv_out = fluid.layers.conv2d(data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - bias_attr=self.conv_bias_attr, - act=self.act) + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) + conv_out = fluid.layers.conv2d( + data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + bias_attr=self.conv_bias_attr, + act=self.act, + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -55,7 +56,6 @@ class ConvActivationMkldnnFusePassTest(InferencePassTest): class ConvActivationMkldnnFusePassTest_1(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 @@ -64,7 +64,6 @@ class ConvActivationMkldnnFusePassTest_1(ConvActivationMkldnnFusePassTest): class ConvActivationMkldnnFusePassTest_2(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 3 self.conv_filter_size = 3 @@ -73,7 +72,6 @@ class ConvActivationMkldnnFusePassTest_2(ConvActivationMkldnnFusePassTest): class ConvActivationMkldnnFusePassTest_3(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 @@ -82,7 +80,6 @@ class ConvActivationMkldnnFusePassTest_3(ConvActivationMkldnnFusePassTest): class ConvActivationMkldnnFusePassTest_4(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 3 self.conv_filter_size = 3 @@ -91,7 +88,6 @@ class ConvActivationMkldnnFusePassTest_4(ConvActivationMkldnnFusePassTest): class ConvActivationMkldnnFusePassTest_5(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 @@ -100,7 +96,6 @@ class ConvActivationMkldnnFusePassTest_5(ConvActivationMkldnnFusePassTest): class ConvActivationMkldnnFusePassTest_6(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 @@ -109,7 +104,6 @@ class ConvActivationMkldnnFusePassTest_6(ConvActivationMkldnnFusePassTest): class ConvHardSigmoidOneDNNFusePassTest(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_affine_channel_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_affine_channel_fuse_pass.py index 94b9ac66a3d..ba6179a1ff4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_affine_channel_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_affine_channel_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestConvAffineChannelFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -38,22 +37,27 @@ class TestConvAffineChannelFusePass(PassAutoScanTest): out_channel = groups * out_channel_factor batch_size = draw(st.integers(min_value=1, max_value=4)) dilations = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) paddings = draw( - st.lists(st.integers(min_value=0, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=2), min_size=2, max_size=2 + ) + ) strides = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) has_bias = draw(st.booleans()) - x_shape = [ - batch_size, in_channel, 64, 64 - ] if data_format == "NCHW" else [batch_size, 64, 64, in_channel] + x_shape = ( + [batch_size, in_channel, 64, 64] + if data_format == "NCHW" + else [batch_size, 64, 64, in_channel] + ) w_shape = [out_channel, filter_channel, filter_size, filter_size] scale_shape = [out_channel] bias_shape = [out_channel] @@ -70,28 +74,32 @@ class TestConvAffineChannelFusePass(PassAutoScanTest): def generate_scale_bias(): return np.random.random(bias_shape).astype(np.float32) - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_data"], - "Filter": ["conv2d_weight"], - }, - outputs={"Output": ["conv_output"]}, - data_format=data_format, - dilations=dilations, - padding_algorithm=padding_algorithm, - groups=groups, - paddings=paddings, - strides=strides, - has_bias=has_bias, - is_test=True) - ac_op = OpConfig("affine_channel", - inputs={ - "X": ["conv_output"], - "Scale": ["affine_channel_scale"], - "Bias": ["affine_channel_bias"] - }, - outputs={"Out": ["affine_channel_ouput"]}, - data_layout=data_format) + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + outputs={"Output": ["conv_output"]}, + data_format=data_format, + dilations=dilations, + padding_algorithm=padding_algorithm, + groups=groups, + paddings=paddings, + strides=strides, + has_bias=has_bias, + is_test=True, + ) + ac_op = OpConfig( + "affine_channel", + inputs={ + "X": ["conv_output"], + "Scale": ["affine_channel_scale"], + "Bias": ["affine_channel_bias"], + }, + outputs={"Out": ["affine_channel_ouput"]}, + data_layout=data_format, + ) if has_bias == True: conv2d_op.inputs["Bias"] = ["conv2d_bias"] ops = [conv2d_op, ac_op] @@ -102,19 +110,23 @@ class TestConvAffineChannelFusePass(PassAutoScanTest): "input_data": TensorConfig(data_gen=partial(generate_input)), }, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight)), - "conv2d_bias": - TensorConfig(data_gen=partial(generate_bias)), - "affine_channel_scale": - TensorConfig(data_gen=partial(generate_scale_bias)), - "affine_channel_bias": - TensorConfig(data_gen=partial(generate_scale_bias)), + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight) + ), + "conv2d_bias": TensorConfig(data_gen=partial(generate_bias)), + "affine_channel_scale": TensorConfig( + data_gen=partial(generate_scale_bias) + ), + "affine_channel_bias": TensorConfig( + data_gen=partial(generate_scale_bias) + ), }, - outputs=["affine_channel_ouput"]) + outputs=["affine_channel_ouput"], + ) if has_bias == True: program_config.weights["conv2d_bias"] = TensorConfig( - data_gen=partial(generate_bias)) + data_gen=partial(generate_bias) + ) return program_config def sample_predictor_configs(self, program_config): @@ -131,18 +143,23 @@ class TestConvAffineChannelFusePass(PassAutoScanTest): # mkldnn Output has diff with bias! def teller2(program_config, predictor_config): - return predictor_config.mkldnn_enabled( - ) and program_config.ops[0].attrs['has_bias'] == True + return ( + predictor_config.mkldnn_enabled() + and program_config.ops[0].attrs['has_bias'] == True + ) self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, "The output format of conv2d is wrong when data_format attribute is NHWC, \ - because currently its fused op (Conv2DFusion) only supports data format of channel first (NCHW)." + because currently its fused op (Conv2DFusion) only supports data format of channel first (NCHW).", ) self.add_ignore_check_case( - teller2, IgnoreReasons.PASS_ACCURACY_ERROR, - "Currently mkldnn Output has diff with bias!") + teller2, + IgnoreReasons.PASS_ACCURACY_ERROR, + "Currently mkldnn Output has diff with bias!", + ) def test(self): self.run_and_statis( diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_bias_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_bias_fuse_pass.py index 7a1a04ca06f..6c667ac08db 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_bias_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_bias_fuse_pass.py @@ -19,22 +19,24 @@ import paddle.fluid as fluid from paddle.fluid.core import PassVersionChecker -#padding SAME +# padding SAME class ConvBiasMkldnnFusePassSamePadTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding="SAME", - bias_attr=param_attr) + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding="SAME", + bias_attr=param_attr, + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -46,25 +48,28 @@ class ConvBiasMkldnnFusePassSamePadTest(InferencePassTest): use_gpu = False self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible("conv_bias_mkldnn_fuse_pass")) + PassVersionChecker.IsCompatible("conv_bias_mkldnn_fuse_pass") + ) -#padding VALID +# padding VALID class ConvBiasMkldnnFusePassValidPadTest(ConvBiasMkldnnFusePassSamePadTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding="VALID", - bias_attr=param_attr) + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding="VALID", + bias_attr=param_attr, + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -73,22 +78,24 @@ class ConvBiasMkldnnFusePassValidPadTest(ConvBiasMkldnnFusePassSamePadTest): self.enable_mkldnn = True -#padding EXPLICT NUMBER +# padding EXPLICT NUMBER class ConvBiasMkldnnFusePassExplictPadTest(ConvBiasMkldnnFusePassSamePadTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding=[2, 4, 6, 8], - bias_attr=param_attr) + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding=[2, 4, 6, 8], + bias_attr=param_attr, + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -98,24 +105,26 @@ class ConvBiasMkldnnFusePassExplictPadTest(ConvBiasMkldnnFusePassSamePadTest): class ConvBiasMkldnnFusePassGroupTest(ConvBiasMkldnnFusePassSamePadTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding="VALID", - groups=3, - bias_attr=param_attr, - use_cudnn=False, - act="softmax", - data_format="NCHW") + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding="VALID", + groups=3, + bias_attr=param_attr, + use_cudnn=False, + act="softmax", + data_format="NCHW", + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -125,26 +134,29 @@ class ConvBiasMkldnnFusePassGroupTest(ConvBiasMkldnnFusePassSamePadTest): class ConvBiasMkldnnFusePassDialtionsGroupsTest( - ConvBiasMkldnnFusePassSamePadTest): - + ConvBiasMkldnnFusePassSamePadTest +): def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding="VALID", - dilation=2, - groups=3, - bias_attr=param_attr, - use_cudnn=False, - act="softmax", - data_format="NCHW") + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding="VALID", + dilation=2, + groups=3, + bias_attr=param_attr, + use_cudnn=False, + act="softmax", + data_format="NCHW", + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -154,20 +166,22 @@ class ConvBiasMkldnnFusePassDialtionsGroupsTest( class ConvTransposeMkldnnFusePassDialtionsGroupsTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[-1, 3, 5, 5], dtype="float32") param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d_transpose(input=data, - num_filters=3, - filter_size=3, - padding="SAME", - dilation=1, - bias_attr=param_attr, - use_cudnn=False) + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d_transpose( + input=data, + num_filters=3, + filter_size=3, + padding="SAME", + dilation=1, + bias_attr=param_attr, + use_cudnn=False, + ) self.feeds = {"data": np.random.random((1, 3, 5, 5)).astype("float32")} self.fetch_list = [conv_out] @@ -178,7 +192,9 @@ class ConvTransposeMkldnnFusePassDialtionsGroupsTest(InferencePassTest): self.check_output_with_option(use_gpu) self.assertTrue( PassVersionChecker.IsCompatible( - "conv_transpose_bias_mkldnn_fuse_pass")) + "conv_transpose_bias_mkldnn_fuse_pass" + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_concat_relu_mkldnn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_concat_relu_mkldnn_fuse_pass.py index a5d2738869f..2d76d2846e9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_concat_relu_mkldnn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_concat_relu_mkldnn_fuse_pass.py @@ -21,7 +21,6 @@ import hypothesis.strategies as st class TestConvConcatActivationMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): data_format = draw(st.sampled_from(['NCHW', 'NHWC'])) dilations = draw(st.sampled_from([[2, 2]])) @@ -31,11 +30,24 @@ class TestConvConcatActivationMkldnnFusePass(PassAutoScanTest): strides = draw(st.sampled_from([[1, 2]])) axis = draw(st.sampled_from([0])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'swish', 'mish', 'sqrt', 'hard_swish', - 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', - 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_data(input_type): if input_type == 'NCHW': @@ -43,94 +55,103 @@ class TestConvConcatActivationMkldnnFusePass(PassAutoScanTest): elif input_type == 'NHWC': return np.random.random([16, 64, 64, 48]).astype(np.float32) elif input_type == 'weights': - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - conv2d_op1 = OpConfig(type='conv2d', - inputs={ - 'Input': ['conv_input_1'], - 'Filter': ['conv_weights_1'] - }, - outputs={'Output': ['conv_output_1']}, - attrs={ - 'data_format': data_format, - 'dilations': dilations, - 'padding_algorithm': padding_algorithm, - 'groups': groups, - 'paddings': paddings, - 'strides': strides - }) - - conv2d_op2 = OpConfig(type='conv2d', - inputs={ - 'Input': ['conv_input_2'], - 'Filter': ['conv_weights_2'] - }, - outputs={'Output': ['conv_output_2']}, - attrs={ - 'data_format': data_format, - 'dilations': dilations, - 'padding_algorithm': padding_algorithm, - 'groups': groups, - 'paddings': paddings, - 'strides': strides - }) + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + conv2d_op1 = OpConfig( + type='conv2d', + inputs={'Input': ['conv_input_1'], 'Filter': ['conv_weights_1']}, + outputs={'Output': ['conv_output_1']}, + attrs={ + 'data_format': data_format, + 'dilations': dilations, + 'padding_algorithm': padding_algorithm, + 'groups': groups, + 'paddings': paddings, + 'strides': strides, + }, + ) + + conv2d_op2 = OpConfig( + type='conv2d', + inputs={'Input': ['conv_input_2'], 'Filter': ['conv_weights_2']}, + outputs={'Output': ['conv_output_2']}, + attrs={ + 'data_format': data_format, + 'dilations': dilations, + 'padding_algorithm': padding_algorithm, + 'groups': groups, + 'paddings': paddings, + 'strides': strides, + }, + ) - concat_op = OpConfig(type='concat', - inputs={'X': ['conv_output_1', 'conv_output_2']}, - outputs={'Out': ['concat_output']}, - attrs={'axis': axis}) + concat_op = OpConfig( + type='concat', + inputs={'X': ['conv_output_1', 'conv_output_2']}, + outputs={'Out': ['concat_output']}, + attrs={'axis': axis}, + ) if activation_type == 'relu6': - activation_op = OpConfig(activation_type, - inputs={'X': ['concat_output']}, - outputs={'Out': ['activation_output']}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['concat_output']}, + outputs={'Out': ['activation_output']}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0)), + ) elif activation_type == 'leaky_relu': - activation_op = OpConfig(activation_type, - inputs={'X': ['concat_output']}, - outputs={'Out': ['activation_output']}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['concat_output']}, + outputs={'Out': ['activation_output']}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == 'swish': - activation_op = OpConfig(activation_type, - inputs={'X': ['concat_output']}, - outputs={'Out': ['activation_output']}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['concat_output']}, + outputs={'Out': ['activation_output']}, + beta=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == 'clip': activation_op = OpConfig( activation_type, inputs={'X': ['concat_output']}, outputs={'Out': ['activation_output']}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={'X': ['concat_output']}, - outputs={'Out': ['activation_output']}) + activation_op = OpConfig( + activation_type, + inputs={'X': ['concat_output']}, + outputs={'Out': ['activation_output']}, + ) model_net = [conv2d_op1, conv2d_op2, concat_op, activation_op] program_config = ProgramConfig( ops=model_net, inputs={ - 'conv_input_1': - TensorConfig(data_gen=partial(generate_data, data_format)), - 'conv_input_2': - TensorConfig(data_gen=partial(generate_data, data_format)) + 'conv_input_1': TensorConfig( + data_gen=partial(generate_data, data_format) + ), + 'conv_input_2': TensorConfig( + data_gen=partial(generate_data, data_format) + ), }, weights={ - 'conv_weights_1': - TensorConfig(data_gen=partial(generate_data, 'weights')), - 'conv_weights_2': - TensorConfig(data_gen=partial(generate_data, 'weights')) + 'conv_weights_1': TensorConfig( + data_gen=partial(generate_data, 'weights') + ), + 'conv_weights_2': TensorConfig( + data_gen=partial(generate_data, 'weights') + ), }, - outputs=['activation_output']) + outputs=['activation_output'], + ) return program_config @@ -139,8 +160,9 @@ class TestConvConcatActivationMkldnnFusePass(PassAutoScanTest): yield config, ['conv2d', 'conv2d', 'concat'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=['conv_activation_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, passes=['conv_activation_mkldnn_fuse_pass'] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_elementwise_add_fuse_pass.py index 3d0ca34a5e0..edf8599442f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_elementwise_add_fuse_pass.py @@ -23,7 +23,6 @@ import hypothesis.strategies as st # the two inputs of elementwise_add are tensor class TestConvElementwiseAddMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -46,58 +45,60 @@ class TestConvElementwiseAddMkldnnFusePass(PassAutoScanTest): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([48, int(48 / groups), 3, - 3]).astype(np.float32) - - relu_op = OpConfig(type="relu", - inputs={"X": ["input_data"]}, - outputs={"Out": ["relu_out"]}, - attrs={}) - - conv2d_op1 = OpConfig(type="conv2d", - inputs={ - "Input": ["relu_out"], - "Filter": ["conv_weight1"] - }, - outputs={"Output": ["conv_output1"]}, - attrs={ - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - }) - - conv2d_op2 = OpConfig(type="conv2d", - inputs={ - "Input": ["input_data"], - "Filter": ["conv_weight2"] - }, - outputs={"Output": ["conv_output2"]}, - attrs={ - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - }) - - elt_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["conv_output1"], - "Y": ["conv_output2"] - }, - outputs={"Out": ["elementwise_output"]}, - attrs={'axis': axis}) + return np.random.random([48, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + relu_op = OpConfig( + type="relu", + inputs={"X": ["input_data"]}, + outputs={"Out": ["relu_out"]}, + attrs={}, + ) + + conv2d_op1 = OpConfig( + type="conv2d", + inputs={"Input": ["relu_out"], "Filter": ["conv_weight1"]}, + outputs={"Output": ["conv_output1"]}, + attrs={ + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, + ) + + conv2d_op2 = OpConfig( + type="conv2d", + inputs={"Input": ["input_data"], "Filter": ["conv_weight2"]}, + outputs={"Output": ["conv_output2"]}, + attrs={ + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, + ) + + elt_op = OpConfig( + type="elementwise_add", + inputs={"X": ["conv_output1"], "Y": ["conv_output2"]}, + outputs={"Out": ["elementwise_output"]}, + attrs={'axis': axis}, + ) model_net = [relu_op, conv2d_op1, conv2d_op2, elt_op] @@ -105,12 +106,13 @@ class TestConvElementwiseAddMkldnnFusePass(PassAutoScanTest): ops=model_net, weights={ "conv_weight1": TensorConfig(data_gen=partial(generate_weight)), - "conv_weight2": TensorConfig(data_gen=partial(generate_weight)) + "conv_weight2": TensorConfig(data_gen=partial(generate_weight)), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config @@ -119,8 +121,9 @@ class TestConvElementwiseAddMkldnnFusePass(PassAutoScanTest): yield config, ["relu", "conv2d", "conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_elementwise_add_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_elementwise_add_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py index 3abb582365b..8094f323a90 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py @@ -23,7 +23,6 @@ import hypothesis.strategies as st class TestConvGeluMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,45 +38,45 @@ class TestConvGeluMkldnnFusePass(PassAutoScanTest): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["input_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - } - }, { - "op_type": "gelu", - "op_inputs": { - "X": ["conv_output"] - }, - "op_outputs": { - "Out": ["gelu_output"] + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["input_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, }, - "op_attrs": { - "approximate": approximate, + { + "op_type": "gelu", + "op_inputs": {"X": ["conv_output"]}, + "op_outputs": {"Out": ["gelu_output"]}, + "op_attrs": { + "approximate": approximate, + }, }, - }] + ] ops = self.generate_op_config(ops_config) @@ -89,7 +88,8 @@ class TestConvGeluMkldnnFusePass(PassAutoScanTest): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["gelu_output"]) + outputs=["gelu_output"], + ) return program_config @@ -98,8 +98,9 @@ class TestConvGeluMkldnnFusePass(PassAutoScanTest): yield config, ["conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_activation_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py index 010d3e8f3aa..2b76ba180f5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestConvHardSigmoidMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,46 +38,43 @@ class TestConvHardSigmoidMkldnnFusePass(PassAutoScanTest): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["input_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - } - }, { - "op_type": "hard_sigmoid", - "op_inputs": { - "X": ["conv_output"] - }, - "op_outputs": { - "Out": ["sigmoid_output"] + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["input_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, }, - "op_attrs": { - "slope": slope, - "offset": offset + { + "op_type": "hard_sigmoid", + "op_inputs": {"X": ["conv_output"]}, + "op_outputs": {"Out": ["sigmoid_output"]}, + "op_attrs": {"slope": slope, "offset": offset}, }, - }] + ] ops = self.generate_op_config(ops_config) @@ -90,7 +86,8 @@ class TestConvHardSigmoidMkldnnFusePass(PassAutoScanTest): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["sigmoid_output"]) + outputs=["sigmoid_output"], + ) return program_config @@ -99,8 +96,9 @@ class TestConvHardSigmoidMkldnnFusePass(PassAutoScanTest): yield config, ["conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_activation_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py index 799d1cbaa1c..417fb95c72a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestConvHardSwishMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -40,47 +39,47 @@ class TestConvHardSwishMkldnnFusePass(PassAutoScanTest): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["input_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - } - }, { - "op_type": "hard_swish", - "op_inputs": { - "X": ["conv_output"] - }, - "op_outputs": { - "Out": ["swish_output"] + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["input_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, }, - "op_attrs": { - "threshold": threshold, - "scale": scale, - "offset": offset + { + "op_type": "hard_swish", + "op_inputs": {"X": ["conv_output"]}, + "op_outputs": {"Out": ["swish_output"]}, + "op_attrs": { + "threshold": threshold, + "scale": scale, + "offset": offset, + }, }, - }] + ] ops = self.generate_op_config(ops_config) @@ -92,7 +91,8 @@ class TestConvHardSwishMkldnnFusePass(PassAutoScanTest): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["swish_output"]) + outputs=["swish_output"], + ) return program_config @@ -101,8 +101,9 @@ class TestConvHardSwishMkldnnFusePass(PassAutoScanTest): yield config, ["conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_activation_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py index 6f5810cb802..93f58741ba8 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py @@ -21,7 +21,6 @@ import hypothesis.strategies as st class TestConvMishMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [op.attrs for op in program_config.ops] # If the problem has been fixed, the judgment @@ -42,43 +41,43 @@ class TestConvMishMkldnnFusePass(PassAutoScanTest): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["input_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - } - }, { - "op_type": "mish", - "op_inputs": { - "X": ["conv_output"] + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["input_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, }, - "op_outputs": { - "Out": ["mish_output"] + { + "op_type": "mish", + "op_inputs": {"X": ["conv_output"]}, + "op_outputs": {"Out": ["mish_output"]}, + "op_attrs": {}, }, - "op_attrs": {}, - }] + ] ops = self.generate_op_config(ops_config) @@ -90,7 +89,8 @@ class TestConvMishMkldnnFusePass(PassAutoScanTest): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["mish_output"]) + outputs=["mish_output"], + ) return program_config @@ -99,8 +99,9 @@ class TestConvMishMkldnnFusePass(PassAutoScanTest): yield config, ["conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_activation_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_transpose_bias_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_transpose_bias_fuse_pass.py index 2eebd529554..01de1243148 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_transpose_bias_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_transpose_bias_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestConvTransposeMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -47,11 +46,13 @@ class TestConvTransposeMkldnnFusePass(PassAutoScanTest): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 16, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 16, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 16]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 16]).astype( + np.float32 + ) def generate_weight1(): return np.random.random([16, 16, 3, 3]).astype(np.float32) @@ -59,46 +60,47 @@ class TestConvTransposeMkldnnFusePass(PassAutoScanTest): def generate_weight2(): return np.random.random([16 * groups]).astype(np.float32) - conv2d_op = OpConfig(type="conv2d_transpose", - inputs={ - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - outputs={"Output": ["conv_output"]}, - attrs={ - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides, - "output_size": [], - "output_padding": [], - "is_test": True - }) - - elt_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["conv_output"], - "Y": ["elementwise_weight"] - }, - outputs={"Out": ["elementwise_output"]}, - attrs={'axis': axis}) + conv2d_op = OpConfig( + type="conv2d_transpose", + inputs={"Input": ["input_data"], "Filter": ["conv2d_weight"]}, + outputs={"Output": ["conv_output"]}, + attrs={ + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + "output_size": [], + "output_padding": [], + "is_test": True, + }, + ) + + elt_op = OpConfig( + type="elementwise_add", + inputs={"X": ["conv_output"], "Y": ["elementwise_weight"]}, + outputs={"Out": ["elementwise_output"]}, + attrs={'axis': axis}, + ) model_net = [conv2d_op, elt_op] program_config = ProgramConfig( ops=model_net, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight1)), - "elementwise_weight": - TensorConfig(data_gen=partial(generate_weight2)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "elementwise_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config @@ -107,9 +109,11 @@ class TestConvTransposeMkldnnFusePass(PassAutoScanTest): yield config, ['conv2d_transpose'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_duration=300, - passes=["conv_transpose_bias_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, + max_duration=300, + passes=["conv_transpose_bias_mkldnn_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_cpu_bfloat16_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_cpu_bfloat16_pass.py index 95996f22a86..2cd9cbcb057 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_cpu_bfloat16_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_cpu_bfloat16_pass.py @@ -20,20 +20,20 @@ from paddle.fluid.core import PassVersionChecker class TestMKLDNNCpuBfloat16Pass(InferencePassTest): - def setUp(self): self.init_data() with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name='x', - shape=[-1] + self.shape_x, - dtype=self.d_type) + x = fluid.data( + name='x', shape=[-1] + self.shape_x, dtype=self.d_type + ) out = fluid.layers.transpose(x, perm=[0, 1, 2, 3]) out = fluid.layers.reshape(out, [0, 0, 0, 0]) out = fluid.layers.fc(out, size=1) self.feeds = { - "x": - np.random.random([self.bs] + self.shape_x).astype(self.d_type) + "x": np.random.random([self.bs] + self.shape_x).astype( + self.d_type + ) } self.fetch_list = [out] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_depthwise_conv_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_depthwise_conv_pass.py index 736c5fca7f8..3a9f520f1c7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_depthwise_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_depthwise_conv_pass.py @@ -42,24 +42,29 @@ class DepthwiseConvMKLDNNPass(PassAutoScanTest): random_groups = draw(st.integers(min_value=1, max_value=3)) random_dilations = draw( - st.lists(st.integers(min_value=1, max_value=3), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=3), min_size=2, max_size=2 + ) + ) random_strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_paddings = draw( - st.lists(st.integers(min_value=0, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=4), min_size=2, max_size=2 + ) + ) random_padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + st.sampled_from(["EXPLICIT", "SAME", "VALID"]) + ) random_data_layout = draw(st.sampled_from(["NCHW", "NHWC"])) random_filter = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) def generate_conv2d_Input(): shape = [random_input_dim1, random_input_dim2] @@ -78,23 +83,25 @@ class DepthwiseConvMKLDNNPass(PassAutoScanTest): return np.random.random(shape).astype(np.float32) # define op - conv2d_op = OpConfig(type="depthwise_conv2d", - inputs={ - "Input": ["conv2d_Input"], - "Filter": ["conv2d_Filter"], - }, - outputs={ - "Output": ["conv2d_Out"], - }, - attrs={ - 'groups': random_groups, - 'dilations': random_dilations, - 'strides': random_strides, - 'paddings': random_paddings, - 'padding_algorithm': random_padding_algorithm, - 'data_format': random_data_layout, - 'use_mkldnn': True, - }) + conv2d_op = OpConfig( + type="depthwise_conv2d", + inputs={ + "Input": ["conv2d_Input"], + "Filter": ["conv2d_Filter"], + }, + outputs={ + "Output": ["conv2d_Out"], + }, + attrs={ + 'groups': random_groups, + 'dilations': random_dilations, + 'strides': random_strides, + 'paddings': random_paddings, + 'padding_algorithm': random_padding_algorithm, + 'data_format': random_data_layout, + 'use_mkldnn': True, + }, + ) # define model_net model_net = [conv2d_op] @@ -108,7 +115,8 @@ class DepthwiseConvMKLDNNPass(PassAutoScanTest): weights={ "conv2d_Filter": TensorConfig(data_gen=generate_conv2d_Filter), }, - outputs=["conv2d_Out"]) + outputs=["conv2d_Out"], + ) return program_config @@ -128,13 +136,13 @@ class DepthwiseConvMKLDNNPass(PassAutoScanTest): return True def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): if program_config.ops[0].attrs['data_format'] == "NHWC": return True return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The output format of depthwise_conv2d is wrong when data_format attribute is NHWC" + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The output format of depthwise_conv2d is wrong when data_format attribute is NHWC", ) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py index 0063fea59a7..c9647ec60b5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py @@ -28,12 +28,12 @@ class ElementwiseActivationMkldnnFusePassTest(InferencePassTest): def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data_A = fluid.data(name="data_A", - shape=[-1, 3, 100, 100], - dtype="float32") - data_B = fluid.data(name="data_B", - shape=[-1, 3, 100, 100], - dtype="float32") + data_A = fluid.data( + name="data_A", shape=[-1, 3, 100, 100], dtype="float32" + ) + data_B = fluid.data( + name="data_B", shape=[-1, 3, 100, 100], dtype="float32" + ) elt_out = self.operand(data_A, data_B) if self.act is not None: if self.act_beta is not None: @@ -45,7 +45,7 @@ class ElementwiseActivationMkldnnFusePassTest(InferencePassTest): self.feeds = { "data_A": np.random.random((1, 3, 100, 100)).astype("float32"), - "data_B": np.random.random((1, 3, 100, 100)).astype("float32") + "data_B": np.random.random((1, 3, 100, 100)).astype("float32"), } self.fetch_list = [elt_out] self.enable_mkldnn = True @@ -63,24 +63,24 @@ class ElementwiseActivationMkldnnFusePassTest(InferencePassTest): class ElementwiseActivationMkldnnFusePassTest_Add_Relu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.relu class ElementwiseActivationMkldnnFusePassTest_Add_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.tanh class ElementwiseActivationMkldnnFusePassTest_Add_LeakyRelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act_alpha = 0.2 @@ -88,8 +88,8 @@ class ElementwiseActivationMkldnnFusePassTest_Add_LeakyRelu( class ElementwiseActivationMkldnnFusePassTest_Add_Swish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act_alpha = 4 @@ -97,32 +97,32 @@ class ElementwiseActivationMkldnnFusePassTest_Add_Swish( class ElementwiseActivationMkldnnFusePassTest_Add_HardSwish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.hard_swish class ElementwiseActivationMkldnnFusePassTest_Add_SQRT( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.sqrt class ElementwiseActivationMkldnnFusePassTest_Add_ABS( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.abs class ElementwiseActivationMkldnnFusePassTest_Add_Clip( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.clip @@ -131,16 +131,16 @@ class ElementwiseActivationMkldnnFusePassTest_Add_Clip( class ElementwiseActivationMkldnnFusePassTest_Add_Gelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.gelu class ElementwiseActivationMkldnnFusePassTest_Add_Gelu_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.gelu @@ -148,8 +148,8 @@ class ElementwiseActivationMkldnnFusePassTest_Add_Gelu_Tanh( class ElementwiseActivationMkldnnFusePassTest_Add_Relu6( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.relu6 @@ -157,32 +157,32 @@ class ElementwiseActivationMkldnnFusePassTest_Add_Relu6( class ElementwiseActivationMkldnnFusePassTest_Add_Sigmoid( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.sigmoid class ElementwiseActivationMkldnnFusePassTest_Sub_Relu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.relu class ElementwiseActivationMkldnnFusePassTest_Sub_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.tanh class ElementwiseActivationMkldnnFusePassTest_Sub_LeakyRelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act_alpha = 0.2 @@ -190,32 +190,32 @@ class ElementwiseActivationMkldnnFusePassTest_Sub_LeakyRelu( class ElementwiseActivationMkldnnFusePassTest_Sub_Swish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.swish class ElementwiseActivationMkldnnFusePassTest_Sub_HardSwish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.hard_swish class ElementwiseActivationMkldnnFusePassTest_Sub_ABS( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.abs class ElementwiseActivationMkldnnFusePassTest_Sub_Clip( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.clip @@ -224,16 +224,16 @@ class ElementwiseActivationMkldnnFusePassTest_Sub_Clip( class ElementwiseActivationMkldnnFusePassTest_Sub_Gelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.gelu class ElementwiseActivationMkldnnFusePassTest_Sub_Gelu_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.gelu @@ -241,8 +241,8 @@ class ElementwiseActivationMkldnnFusePassTest_Sub_Gelu_Tanh( class ElementwiseActivationMkldnnFusePassTest_Sub_Relu6( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.relu6 @@ -250,32 +250,32 @@ class ElementwiseActivationMkldnnFusePassTest_Sub_Relu6( class ElementwiseActivationMkldnnFusePassTest_Sub_Sigmoid( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.sigmoid class ElementwiseActivationMkldnnFusePassTest_Mul_Relu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.relu class ElementwiseActivationMkldnnFusePassTest_Mul_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.tanh class ElementwiseActivationMkldnnFusePassTest_Mul_LeakyRelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act_alpha = 0.2 @@ -283,40 +283,40 @@ class ElementwiseActivationMkldnnFusePassTest_Mul_LeakyRelu( class ElementwiseActivationMkldnnFusePassTest_Mul_Swish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.swish class ElementwiseActivationMkldnnFusePassTest_Mul_HardSwish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.hard_swish class ElementwiseActivationMkldnnFusePassTest_Mul_SQRT( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.sqrt class ElementwiseActivationMkldnnFusePassTest_Mul_ABS( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.abs class ElementwiseActivationMkldnnFusePassTest_Mul_Clip( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.clip @@ -325,16 +325,16 @@ class ElementwiseActivationMkldnnFusePassTest_Mul_Clip( class ElementwiseActivationMkldnnFusePassTest_Mul_Gelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.gelu class ElementwiseActivationMkldnnFusePassTest_Mul_Gelu_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.gelu @@ -342,8 +342,8 @@ class ElementwiseActivationMkldnnFusePassTest_Mul_Gelu_Tanh( class ElementwiseActivationMkldnnFusePassTest_Mul_Relu6( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.relu6 @@ -351,8 +351,8 @@ class ElementwiseActivationMkldnnFusePassTest_Mul_Relu6( class ElementwiseActivationMkldnnFusePassTest_Mul_Sigmoid( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.sigmoid diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py index feef212b1f6..8144e02afa0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestElementWiseAddReluFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -30,29 +29,24 @@ class TestElementWiseAddReluFusePass(PassAutoScanTest): batch_size = draw(st.integers(min_value=1, max_value=4)) def generate_input(): - return np.random.random([batch_size, 3, 100, - 100]).astype(np.float32) + return np.random.random([batch_size, 3, 100, 100]).astype( + np.float32 + ) - ops_config = [{ - "op_type": "elementwise_add", - "op_inputs": { - "X": ["A"], - "Y": ["B"] - }, - "op_outputs": { - "Out": ["add_output"] - }, - "op_attrs": {} - }, { - "op_type": "relu", - "op_inputs": { - "X": ["add_output"] + ops_config = [ + { + "op_type": "elementwise_add", + "op_inputs": {"X": ["A"], "Y": ["B"]}, + "op_outputs": {"Out": ["add_output"]}, + "op_attrs": {}, }, - "op_outputs": { - "Out": ["relu_output"] + { + "op_type": "relu", + "op_inputs": {"X": ["add_output"]}, + "op_outputs": {"Out": ["relu_output"]}, + "op_attrs": {}, }, - "op_attrs": {} - }] + ] ops = self.generate_op_config(ops_config) @@ -61,9 +55,10 @@ class TestElementWiseAddReluFusePass(PassAutoScanTest): weights={}, inputs={ "A": TensorConfig(data_gen=partial(generate_input)), - "B": TensorConfig(data_gen=partial(generate_input)) + "B": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["relu_output"]) + outputs=["relu_output"], + ) return program_config @@ -72,9 +67,9 @@ class TestElementWiseAddReluFusePass(PassAutoScanTest): yield config, ["elementwise_add"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["elt_act_mkldnn_fuse_pass"], - min_success_num=4) + self.run_and_statis( + quant=False, passes=["elt_act_mkldnn_fuse_pass"], min_success_num=4 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_act_fuse_pass.py index c438c28370e..4848a8c9770 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_act_fuse_pass.py @@ -25,13 +25,12 @@ enable_static() class FCGeluTanhOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) gelu_out = fluid.layers.gelu(fc_out, approximate=False) @@ -48,13 +47,12 @@ class FCGeluTanhOneDnnFusePassTest(InferencePassTest): class FCGeluErfOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) gelu_out = fluid.layers.gelu(fc_out, approximate=True) @@ -72,13 +70,12 @@ class FCGeluErfOneDnnFusePassTest(InferencePassTest): class FCTanhOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) tanh_out = fluid.layers.tanh(fc_out) @@ -96,13 +93,12 @@ class FCTanhOneDnnFusePassTest(InferencePassTest): class FCSigmoidOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) sigmoid_out = fluid.layers.sigmoid(fc_out) @@ -120,13 +116,12 @@ class FCSigmoidOneDnnFusePassTest(InferencePassTest): class FCHardSwishOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) hardswish_out = fluid.layers.hard_swish(fc_out) @@ -144,13 +139,12 @@ class FCHardSwishOneDnnFusePassTest(InferencePassTest): class FCMishOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) mish_out = fluid.layers.mish(fc_out) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_elementwise_add_fuse_pass.py index d69a7b8a8bc..85f877147ff 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_elementwise_add_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestFCElementwiseAddMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): axis = draw(st.sampled_from([-1, 0, 1])) fc_as_x = draw(st.sampled_from([True, False])) @@ -38,34 +37,40 @@ class TestFCElementwiseAddMkldnnFusePass(PassAutoScanTest): def generate_fc_bias(): return np.random.random([fc_wei]).astype(np.float32) - relu_op = OpConfig(type="relu", - inputs={"X": ["input_data"]}, - outputs={"Out": ["relu_out"]}, - attrs={}) - - fc_op = OpConfig(type="fc", - inputs={ - "Input": ["relu_out"], - "W": ["fc_weight"], - "Bias": ["fc_bias"] - }, - outputs={"Out": ["fc_output"]}, - attrs={ - "use_mkldnn": True, - "padding_weights": False, - "activation_type": "", - "in_num_col_dims": 1, - }) + relu_op = OpConfig( + type="relu", + inputs={"X": ["input_data"]}, + outputs={"Out": ["relu_out"]}, + attrs={}, + ) + + fc_op = OpConfig( + type="fc", + inputs={ + "Input": ["relu_out"], + "W": ["fc_weight"], + "Bias": ["fc_bias"], + }, + outputs={"Out": ["fc_output"]}, + attrs={ + "use_mkldnn": True, + "padding_weights": False, + "activation_type": "", + "in_num_col_dims": 1, + }, + ) if fc_as_x: inputs = {"X": ["fc_output"], "Y": ["input_data"]} else: inputs = {"X": ["input_data"], "Y": ["fc_output"]} - elt_add_op = OpConfig(type="elementwise_add", - inputs=inputs, - outputs={"Out": ["elementwise_output"]}, - attrs={'axis': axis}) + elt_add_op = OpConfig( + type="elementwise_add", + inputs=inputs, + outputs={"Out": ["elementwise_output"]}, + attrs={'axis': axis}, + ) model_net = [relu_op, fc_op, elt_add_op] @@ -78,7 +83,8 @@ class TestFCElementwiseAddMkldnnFusePass(PassAutoScanTest): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config @@ -87,8 +93,9 @@ class TestFCElementwiseAddMkldnnFusePass(PassAutoScanTest): yield config, ["relu", "fc"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["fc_elementwise_add_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["fc_elementwise_add_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_inplace_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_inplace_fuse_pass.py index 6b589821ba8..6dc911390b0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_inplace_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_inplace_fuse_pass.py @@ -21,22 +21,20 @@ from paddle.fluid.core import PassVersionChecker class MkldnnInplacePassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): paddle.enable_static() - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") - conv_out_1 = fluid.layers.conv2d(data, - num_filters=3, - filter_size=3, - bias_attr=False) + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) + conv_out_1 = fluid.layers.conv2d( + data, num_filters=3, filter_size=3, bias_attr=False + ) softmax_out = fluid.layers.softmax(conv_out_1) relu_out = fluid.layers.relu(conv_out_1) - eltwise_out = fluid.layers.elementwise_add(softmax_out, - relu_out, - axis=-1) + eltwise_out = fluid.layers.elementwise_add( + softmax_out, relu_out, axis=-1 + ) self.pass_name = 'mkldnn_inplace_pass' self.feeds = { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_int8_scale_calculation_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_int8_scale_calculation_pass.py index 3d2895cc619..7308f93d792 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_int8_scale_calculation_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_int8_scale_calculation_pass.py @@ -20,7 +20,6 @@ import hypothesis.strategies as st class TestInt8ScaleCalculationMkldnnPass(PassAutoScanTest): - def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_gpu=False) config.pass_builder().append_pass("int8_scale_calculation_mkldnn_pass") @@ -36,12 +35,36 @@ class TestInt8ScaleCalculationMkldnnPass(PassAutoScanTest): filter_shape = prog_config.weights["filter"].shape input_shape = prog_config.inputs["input_x"].shape if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -57,40 +80,45 @@ class TestInt8ScaleCalculationMkldnnPass(PassAutoScanTest): def sample_program_config(self, draw): x_shape = draw( - st.lists(st.integers(min_value=5, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=5, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=5, max_value=10)) data_format = draw(st.sampled_from(["NCHW", "NHWC"])) f_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: f_shape[1] = x_shape[3] strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) padding = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) groups = draw(st.integers(min_value=1, max_value=3)) dilations = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) bias_shape = [f_shape[0]] inputs = dict() @@ -116,17 +144,19 @@ class TestInt8ScaleCalculationMkldnnPass(PassAutoScanTest): "filter": TensorConfig(shape=f_shape), } - conv2d_op = OpConfig("conv2d", - inputs=inputs, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format, - use_mkldnn=use_mkldnn, - mkldnn_data_type="int8") + conv2d_op = OpConfig( + "conv2d", + inputs=inputs, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + use_mkldnn=use_mkldnn, + mkldnn_data_type="int8", + ) ops = [conv2d_op] @@ -134,13 +164,16 @@ class TestInt8ScaleCalculationMkldnnPass(PassAutoScanTest): ops=ops, weights=weights, inputs={"input_x": TensorConfig(shape=x_shape)}, - outputs=["conv2d_out"]) + outputs=["conv2d_out"], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=["int8_scale_calculation_mkldnn_pass"]) + self.run_and_statis( + quant=False, + max_examples=100, + passes=["int8_scale_calculation_mkldnn_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_log_softmax_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_log_softmax_op.py index 929863b42a7..814163d4ec3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_log_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_log_softmax_op.py @@ -22,28 +22,30 @@ import hypothesis.strategies as st class TestMKLDNNLogSoftmaxOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) - logsoftmax_op = OpConfig(type="log_softmax", - inputs={"X": ["input_data"]}, - outputs={"Out": ["output_data"]}, - attrs={"axis": kwargs['axis']}) + logsoftmax_op = OpConfig( + type="log_softmax", + inputs={"X": ["input_data"]}, + outputs={"Out": ["output_data"]}, + attrs={"axis": kwargs['axis']}, + ) program_config = ProgramConfig( ops=[logsoftmax_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -51,10 +53,12 @@ class TestMKLDNNLogSoftmaxOp(MkldnnAutoScanTest): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(axis=st.sampled_from([-2, -1, 0, 1]), - in_shape=st.lists(st.integers(min_value=2, max_value=5), - min_size=3, - max_size=5)) + @given( + axis=st.sampled_from([-2, -1, 0, 1]), + in_shape=st.lists( + st.integers(min_value=2, max_value=5), min_size=3, max_size=5 + ), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py index b894fc708b4..a90129394e1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -21,7 +21,6 @@ import hypothesis.strategies as st class TestMatmulActivationMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): transpose_X = draw(st.booleans()) transpose_Y = draw(st.booleans()) @@ -30,11 +29,24 @@ class TestMatmulActivationMkldnnFusePass(PassAutoScanTest): channel = draw(st.sampled_from([8])) input_dim = draw(st.sampled_from([32])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'swish', 'mish', 'sqrt', 'hard_swish', - 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', - 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_input(type): if transpose_X and transpose_Y: @@ -55,50 +67,52 @@ class TestMatmulActivationMkldnnFusePass(PassAutoScanTest): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type='matmul', - inputs={ - 'X': ['matmul_X'], - 'Y': ['matmul_Y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={ - 'transpose_X': transpose_X, - 'transpose_Y': transpose_Y, - 'alpha': alpha - }) + matmul_op = OpConfig( + type='matmul', + inputs={'X': ['matmul_X'], 'Y': ['matmul_Y']}, + outputs={'Out': ['matmul_output']}, + attrs={ + 'transpose_X': transpose_X, + 'transpose_Y': transpose_Y, + 'alpha': alpha, + }, + ) if activation_type == "relu6": - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0)), + ) elif activation_type == "leaky_relu": - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "swish": - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + beta=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "clip": activation_op = OpConfig( activation_type, inputs={"X": ["matmul_output"]}, outputs={"Out": ["activation_output"]}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + ) model_net = [matmul_op, activation_op] @@ -107,9 +121,10 @@ class TestMatmulActivationMkldnnFusePass(PassAutoScanTest): weights={}, inputs={ 'matmul_X': TensorConfig(data_gen=partial(generate_input, 'x')), - 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'y')) + 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'y')), }, - outputs=['activation_output']) + outputs=['activation_output'], + ) return program_config @@ -118,9 +133,11 @@ class TestMatmulActivationMkldnnFusePass(PassAutoScanTest): yield config, ['matmul'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=['matmul_activation_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, + max_examples=30, + passes=['matmul_activation_mkldnn_fuse_pass'], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_activation_fuse_pass.py index a27ed9dd9c9..035cd1d3800 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_activation_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestMatmulElementwiseAddActivationMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): axis = draw(st.sampled_from([-1, 0, 1])) matmul_as_x = draw(st.booleans()) @@ -30,71 +29,88 @@ class TestMatmulElementwiseAddActivationMkldnnFusePass(PassAutoScanTest): channel = draw(st.sampled_from([16, 32, 64])) input_dim = draw(st.sampled_from([16, 32, 64])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish', 'sqrt', - 'hard_swish', 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', - 'hard_sigmoid', 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'tanh', + 'sigmoid', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_input(): - return np.random.random([batch_size, channel, input_dim, - input_dim]).astype(np.float32) - - matmul_op = OpConfig(type='matmul', - inputs={ - 'X': ['matmul_x'], - 'Y': ['matmul_y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={ - 'use_mkldnn': True, - }) + return np.random.random( + [batch_size, channel, input_dim, input_dim] + ).astype(np.float32) + + matmul_op = OpConfig( + type='matmul', + inputs={'X': ['matmul_x'], 'Y': ['matmul_y']}, + outputs={'Out': ['matmul_output']}, + attrs={ + 'use_mkldnn': True, + }, + ) if matmul_as_x: inputs = {'X': ['matmul_output'], 'Y': ['elementwise_addend']} else: inputs = {'X': ['elementwise_addend'], 'Y': ['matmul_output']} - elt_add_op = OpConfig(type='elementwise_add', - inputs=inputs, - outputs={'Out': ['elementwise_add_output']}, - attrs={ - 'axis': axis, - 'use_mkldnn': True - }) + elt_add_op = OpConfig( + type='elementwise_add', + inputs=inputs, + outputs={'Out': ['elementwise_add_output']}, + attrs={'axis': axis, 'use_mkldnn': True}, + ) if activation_type == "relu6": - activation_op = OpConfig(activation_type, - inputs={"X": ["elementwise_add_output"]}, - outputs={"Out": ["activation_output"]}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["elementwise_add_output"]}, + outputs={"Out": ["activation_output"]}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0)), + ) elif activation_type == "leaky_relu": - activation_op = OpConfig(activation_type, - inputs={"X": ["elementwise_add_output"]}, - outputs={"Out": ["activation_output"]}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["elementwise_add_output"]}, + outputs={"Out": ["activation_output"]}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "swish": - activation_op = OpConfig(activation_type, - inputs={"X": ["elementwise_add_output"]}, - outputs={"Out": ["activation_output"]}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["elementwise_add_output"]}, + outputs={"Out": ["activation_output"]}, + beta=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "clip": activation_op = OpConfig( activation_type, inputs={"X": ["elementwise_add_output"]}, outputs={"Out": ["activation_output"]}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={"X": ["elementwise_add_output"]}, - outputs={"Out": ["activation_output"]}) + activation_op = OpConfig( + activation_type, + inputs={"X": ["elementwise_add_output"]}, + outputs={"Out": ["activation_output"]}, + ) model_net = [matmul_op, elt_add_op, activation_op] @@ -104,10 +120,12 @@ class TestMatmulElementwiseAddActivationMkldnnFusePass(PassAutoScanTest): inputs={ 'matmul_x': TensorConfig(data_gen=partial(generate_input)), 'matmul_y': TensorConfig(data_gen=partial(generate_input)), - 'elementwise_addend': - TensorConfig(data_gen=partial(generate_input)) + 'elementwise_addend': TensorConfig( + data_gen=partial(generate_input) + ), }, - outputs=['activation_output']) + outputs=['activation_output'], + ) return program_config @@ -116,16 +134,19 @@ class TestMatmulElementwiseAddActivationMkldnnFusePass(PassAutoScanTest): use_mkldnn=True, passes=[ 'matmul_elementwise_add_mkldnn_fuse_pass', - 'matmul_activation_mkldnn_fuse_pass' - ]) + 'matmul_activation_mkldnn_fuse_pass', + ], + ) yield config, ['matmul'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=[ - 'matmul_elementwise_add_mkldnn_fuse_pass', - 'matmul_activation_mkldnn_fuse_pass' - ]) + self.run_and_statis( + quant=False, + passes=[ + 'matmul_elementwise_add_mkldnn_fuse_pass', + 'matmul_activation_mkldnn_fuse_pass', + ], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_fuse_pass.py index 38c8985dbad..95b79db4630 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestMatmulElementwiseAddMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): axis = draw(st.sampled_from([-1, 0, 1])) matmul_as_x = draw(st.booleans()) @@ -31,31 +30,30 @@ class TestMatmulElementwiseAddMkldnnFusePass(PassAutoScanTest): input_dim = draw(st.sampled_from([16, 32, 64])) def generate_input(): - return np.random.random([batch_size, channel, input_dim, - input_dim]).astype(np.float32) - - matmul_op = OpConfig(type='matmul', - inputs={ - 'X': ['matmul_x'], - 'Y': ['matmul_y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={ - 'use_mkldnn': True, - }) + return np.random.random( + [batch_size, channel, input_dim, input_dim] + ).astype(np.float32) + + matmul_op = OpConfig( + type='matmul', + inputs={'X': ['matmul_x'], 'Y': ['matmul_y']}, + outputs={'Out': ['matmul_output']}, + attrs={ + 'use_mkldnn': True, + }, + ) if matmul_as_x: inputs = {'X': ['matmul_output'], 'Y': ['elementwise_addend']} else: inputs = {'X': ['elementwise_addend'], 'Y': ['matmul_output']} - elt_add_op = OpConfig(type='elementwise_add', - inputs=inputs, - outputs={'Out': ['elementwise_add_output']}, - attrs={ - 'axis': axis, - 'use_mkldnn': True - }) + elt_add_op = OpConfig( + type='elementwise_add', + inputs=inputs, + outputs={'Out': ['elementwise_add_output']}, + attrs={'axis': axis, 'use_mkldnn': True}, + ) model_net = [matmul_op, elt_add_op] @@ -65,21 +63,25 @@ class TestMatmulElementwiseAddMkldnnFusePass(PassAutoScanTest): inputs={ 'matmul_x': TensorConfig(data_gen=partial(generate_input)), 'matmul_y': TensorConfig(data_gen=partial(generate_input)), - 'elementwise_addend': - TensorConfig(data_gen=partial(generate_input)) + 'elementwise_addend': TensorConfig( + data_gen=partial(generate_input) + ), }, - outputs=['elementwise_add_output']) + outputs=['elementwise_add_output'], + ) return program_config def sample_predictor_configs(self, program_config): config = self.create_inference_config( - use_mkldnn=True, passes=['matmul_elementwise_add_mkldnn_fuse_pass']) + use_mkldnn=True, passes=['matmul_elementwise_add_mkldnn_fuse_pass'] + ) yield config, ['matmul'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=['matmul_elementwise_add_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, passes=['matmul_elementwise_add_mkldnn_fuse_pass'] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_op_output_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_op_output_fuse_pass.py index 1136867e553..064847dd7a0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_op_output_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_op_output_fuse_pass.py @@ -20,7 +20,6 @@ from inference_pass_test import InferencePassTest class TestMKLDNNMatmulFuseOp(InferencePassTest): - def init_data(self): self.bs = 8 self.d_type = np.float32 @@ -30,16 +29,17 @@ class TestMKLDNNMatmulFuseOp(InferencePassTest): def make_network(self): with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name='x', - shape=[-1] + self.shape_x, - dtype=self.d_type) - y = fluid.data(name='y', - shape=[-1] + self.shape_y, - dtype=self.d_type) + x = fluid.data( + name='x', shape=[-1] + self.shape_x, dtype=self.d_type + ) + y = fluid.data( + name='y', shape=[-1] + self.shape_y, dtype=self.d_type + ) out = fluid.layers.matmul(x, y) out = fluid.layers.transpose(out, perm=[0, 2, 1, 3]) out = fluid.layers.reshape( - out, [0, 0, self.shape_y[0] * self.shape_y[2]]) + out, [0, 0, self.shape_y[0] * self.shape_y[2]] + ) out = fluid.layers.fc(out, size=1) return out @@ -51,7 +51,7 @@ class TestMKLDNNMatmulFuseOp(InferencePassTest): def set_feeds(self, out): self.feeds = { "x": np.random.random([self.bs] + self.shape_x).astype(self.d_type), - "y": np.random.random([self.bs] + self.shape_y).astype(self.d_type) + "y": np.random.random([self.bs] + self.shape_y).astype(self.d_type), } self.fetch_list = [out] @@ -61,7 +61,6 @@ class TestMKLDNNMatmulFuseOp(InferencePassTest): class TestMKLDNNMatmulOtherDimsFuseOp(TestMKLDNNMatmulFuseOp): - def init_data(self): self.bs = 8 self.d_type = np.float32 @@ -71,15 +70,14 @@ class TestMKLDNNMatmulOtherDimsFuseOp(TestMKLDNNMatmulFuseOp): class TestMKLDNNMatmulOpNotFusedWrongTransposeAxis(TestMKLDNNMatmulFuseOp): - def make_network(self): with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name='x', - shape=[-1] + self.shape_x, - dtype=self.d_type) - y = fluid.data(name='y', - shape=[-1] + self.shape_y, - dtype=self.d_type) + x = fluid.data( + name='x', shape=[-1] + self.shape_x, dtype=self.d_type + ) + y = fluid.data( + name='y', shape=[-1] + self.shape_y, dtype=self.d_type + ) out = fluid.layers.matmul(x, y) out = fluid.layers.transpose(out, perm=[0, 1, 2, 3]) out = fluid.layers.reshape(out, [0, 0, 0, 0]) @@ -88,7 +86,6 @@ class TestMKLDNNMatmulOpNotFusedWrongTransposeAxis(TestMKLDNNMatmulFuseOp): class TestMKLDNNMatmulOpNotFusedBreakPattern(TestMKLDNNMatmulFuseOp): - def init_data(self): self.bs = 7 self.d_type = np.float32 @@ -98,18 +95,20 @@ class TestMKLDNNMatmulOpNotFusedBreakPattern(TestMKLDNNMatmulFuseOp): def make_network(self): with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name='x', - shape=[-1] + self.shape_x, - dtype=self.d_type) - y = fluid.data(name='y', - shape=[-1] + self.shape_y, - dtype=self.d_type) + x = fluid.data( + name='x', shape=[-1] + self.shape_x, dtype=self.d_type + ) + y = fluid.data( + name='y', shape=[-1] + self.shape_y, dtype=self.d_type + ) out = fluid.layers.matmul(x, y) out = fluid.layers.transpose(out, perm=[0, 2, 1, 3]) - out = fluid.layers.transpose(out, perm=[0, 1, 2, - 3]) # breaks pattern + out = fluid.layers.transpose( + out, perm=[0, 1, 2, 3] + ) # breaks pattern out = fluid.layers.reshape( - out, [0, 0, self.shape_y[0] * self.shape_y[2]]) + out, [0, 0, self.shape_y[0] * self.shape_y[2]] + ) out = fluid.layers.fc(out, size=1) return out diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_transpose_reshape_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_transpose_reshape_fuse_pass.py index 56825149dd2..6c9a9c31594 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_transpose_reshape_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_transpose_reshape_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestMatmulTransposeReshapeMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -63,39 +62,39 @@ class TestMatmulTransposeReshapeMkldnnFusePass(PassAutoScanTest): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type="matmul", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["matmul_output"]}, - attrs={ - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) - - transpose2_op = OpConfig(type="transpose2", - inputs={"X": ["matmul_output"]}, - outputs={ - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] - }, - attrs={'axis': axis}) - - reshape2_op = OpConfig(type="reshape2", - inputs={"X": ["transpose2_output"]}, - outputs={ - "Out": ["reshape2_output"], - "XShape": ["reshape2_xshape"] - }, - attrs={'shape': shape}) + matmul_op = OpConfig( + type="matmul", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["matmul_output"]}, + attrs={ + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) + + transpose2_op = OpConfig( + type="transpose2", + inputs={"X": ["matmul_output"]}, + outputs={ + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + attrs={'axis': axis}, + ) + + reshape2_op = OpConfig( + type="reshape2", + inputs={"X": ["transpose2_output"]}, + outputs={"Out": ["reshape2_output"], "XShape": ["reshape2_xshape"]}, + attrs={'shape': shape}, + ) model_net = [matmul_op, transpose2_op, reshape2_op] @@ -103,12 +102,15 @@ class TestMatmulTransposeReshapeMkldnnFusePass(PassAutoScanTest): ops=model_net, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, "x")), - "input_data2": - TensorConfig(data_gen=partial(generate_input, "y")) + "input_data1": TensorConfig( + data_gen=partial(generate_input, "x") + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, "y") + ), }, - outputs=["reshape2_output"]) + outputs=["reshape2_output"], + ) return program_config @@ -118,7 +120,8 @@ class TestMatmulTransposeReshapeMkldnnFusePass(PassAutoScanTest): def test(self): self.run_and_statis( - quant=False, passes=["matmul_transpose_reshape_mkldnn_fuse_pass"]) + quant=False, passes=["matmul_transpose_reshape_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_activation_fuse_pass.py index 153b81fa797..97b2c22b35e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_activation_fuse_pass.py @@ -21,7 +21,6 @@ import hypothesis.strategies as st class TestMatmulv2ActivationMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): transpose_X = draw(st.booleans()) transpose_Y = draw(st.booleans()) @@ -29,11 +28,24 @@ class TestMatmulv2ActivationMkldnnFusePass(PassAutoScanTest): channel = draw(st.sampled_from([16, 32, 64])) input_dim = draw(st.sampled_from([16, 32, 64])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'swish', 'mish', 'sqrt', 'hard_swish', - 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', - 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_input(type): broadcast_X = st.booleans() @@ -60,49 +72,48 @@ class TestMatmulv2ActivationMkldnnFusePass(PassAutoScanTest): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type='matmul_v2', - inputs={ - 'X': ['matmul_X'], - 'Y': ['matmul_Y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={ - 'trans_x': transpose_X, - 'trans_y': transpose_Y - }) + matmul_op = OpConfig( + type='matmul_v2', + inputs={'X': ['matmul_X'], 'Y': ['matmul_Y']}, + outputs={'Out': ['matmul_output']}, + attrs={'trans_x': transpose_X, 'trans_y': transpose_Y}, + ) if activation_type == 'relu6': - activation_op = OpConfig(activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0)), + ) elif activation_type == 'leaky_relu': - activation_op = OpConfig(activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == 'swish': - activation_op = OpConfig(activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + beta=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == 'clip': activation_op = OpConfig( activation_type, inputs={'X': ['matmul_output']}, outputs={'Out': ['activation_output']}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}) + activation_op = OpConfig( + activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + ) model_net = [matmul_op, activation_op] @@ -111,9 +122,10 @@ class TestMatmulv2ActivationMkldnnFusePass(PassAutoScanTest): weights={}, inputs={ 'matmul_X': TensorConfig(data_gen=partial(generate_input, 'X')), - 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'Y')) + 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'Y')), }, - outputs=['activation_output']) + outputs=['activation_output'], + ) return program_config @@ -122,9 +134,11 @@ class TestMatmulv2ActivationMkldnnFusePass(PassAutoScanTest): yield config, ['matmul_v2'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=['matmul_activation_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, + max_examples=30, + passes=['matmul_activation_mkldnn_fuse_pass'], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_elementwise_add_fuse_pass.py index 03f2867948e..783a38783e6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_elementwise_add_fuse_pass.py @@ -21,7 +21,6 @@ import hypothesis.strategies as st class TestMatmulV2ElementwiseAddMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): axis = draw(st.sampled_from([-1, 0, 1])) matmul_as_x = draw(st.booleans()) @@ -49,26 +48,24 @@ class TestMatmulV2ElementwiseAddMkldnnFusePass(PassAutoScanTest): shape_out = [batch_size, channel, input_dim_X, input_dim_Y] return np.random.random(shape_out).astype(np.float32) - matmul_op = OpConfig(type='matmul_v2', - inputs={ - 'X': ['matmul_X'], - 'Y': ['matmul_Y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={'use_mkldnn': True}) + matmul_op = OpConfig( + type='matmul_v2', + inputs={'X': ['matmul_X'], 'Y': ['matmul_Y']}, + outputs={'Out': ['matmul_output']}, + attrs={'use_mkldnn': True}, + ) if matmul_as_x: inputs = {'X': ['matmul_output'], 'Y': ['elementwise_addend']} else: inputs = {'X': ['elementwise_addend'], 'Y': ['matmul_output']} - elt_add_op = OpConfig(type='elementwise_add', - inputs=inputs, - outputs={'Out': ['elementwise_add_output']}, - attrs={ - 'axis': axis, - 'use_mkldnn': True - }) + elt_add_op = OpConfig( + type='elementwise_add', + inputs=inputs, + outputs={'Out': ['elementwise_add_output']}, + attrs={'axis': axis, 'use_mkldnn': True}, + ) model_net = [matmul_op, elt_add_op] @@ -76,14 +73,14 @@ class TestMatmulV2ElementwiseAddMkldnnFusePass(PassAutoScanTest): ops=model_net, weights={}, inputs={ - 'matmul_X': - TensorConfig(data_gen=partial(generate_input, 'X')), - 'matmul_Y': - TensorConfig(data_gen=partial(generate_input, 'Y')), - 'elementwise_addend': - TensorConfig(data_gen=partial(generate_input, 'ElAdd')) + 'matmul_X': TensorConfig(data_gen=partial(generate_input, 'X')), + 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'Y')), + 'elementwise_addend': TensorConfig( + data_gen=partial(generate_input, 'ElAdd') + ), }, - outputs=['elementwise_add_output']) + outputs=['elementwise_add_output'], + ) return program_config @@ -92,9 +89,11 @@ class TestMatmulV2ElementwiseAddMkldnnFusePass(PassAutoScanTest): yield config, ['matmul_v2'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=['matmul_elementwise_add_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, + max_examples=30, + passes=['matmul_elementwise_add_mkldnn_fuse_pass'], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_transpose_reshape_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_transpose_reshape_fuse_pass.py index 42f349cf4ae..317595d20a6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_transpose_reshape_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_transpose_reshape_fuse_pass.py @@ -22,18 +22,25 @@ import hypothesis.strategies as st class TestMatmulv2TransposeReshapeMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: - if program_config.inputs["input_data1"].shape[ - -4] != 1 and program_config.inputs["input_data2"].shape[-4] != 1: - if program_config.inputs["input_data1"].shape[ - -4] != program_config.inputs["input_data2"].shape[-4]: + if ( + program_config.inputs["input_data1"].shape[-4] != 1 + and program_config.inputs["input_data2"].shape[-4] != 1 + ): + if ( + program_config.inputs["input_data1"].shape[-4] + != program_config.inputs["input_data2"].shape[-4] + ): return False - if program_config.inputs["input_data1"].shape[ - -3] != 1 and program_config.inputs["input_data2"].shape[-3] != 1: - if program_config.inputs["input_data1"].shape[ - -3] != program_config.inputs["input_data2"].shape[-3]: + if ( + program_config.inputs["input_data1"].shape[-3] != 1 + and program_config.inputs["input_data2"].shape[-3] != 1 + ): + if ( + program_config.inputs["input_data1"].shape[-3] + != program_config.inputs["input_data2"].shape[-3] + ): return False attrs = [ @@ -76,38 +83,38 @@ class TestMatmulv2TransposeReshapeMkldnnFusePass(PassAutoScanTest): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type="matmul_v2", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["matmul_output"]}, - attrs={ - "trans_x": transpose_X, - "trans_y": transpose_Y, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) - - transpose2_op = OpConfig(type="transpose2", - inputs={"X": ["matmul_output"]}, - outputs={ - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] - }, - attrs={'axis': axis}) - - reshape2_op = OpConfig(type="reshape2", - inputs={"X": ["transpose2_output"]}, - outputs={ - "Out": ["reshape2_output"], - "XShape": ["reshape2_xshape"] - }, - attrs={'shape': shape}) + matmul_op = OpConfig( + type="matmul_v2", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["matmul_output"]}, + attrs={ + "trans_x": transpose_X, + "trans_y": transpose_Y, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) + + transpose2_op = OpConfig( + type="transpose2", + inputs={"X": ["matmul_output"]}, + outputs={ + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + attrs={'axis': axis}, + ) + + reshape2_op = OpConfig( + type="reshape2", + inputs={"X": ["transpose2_output"]}, + outputs={"Out": ["reshape2_output"], "XShape": ["reshape2_xshape"]}, + attrs={'shape': shape}, + ) model_net = [matmul_op, transpose2_op, reshape2_op] @@ -115,12 +122,15 @@ class TestMatmulv2TransposeReshapeMkldnnFusePass(PassAutoScanTest): ops=model_net, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, "x")), - "input_data2": - TensorConfig(data_gen=partial(generate_input, "y")) + "input_data1": TensorConfig( + data_gen=partial(generate_input, "x") + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, "y") + ), }, - outputs=["reshape2_output"]) + outputs=["reshape2_output"], + ) return program_config @@ -139,7 +149,8 @@ class TestMatmulv2TransposeReshapeMkldnnFusePass(PassAutoScanTest): def test(self): self.run_and_statis( - quant=False, passes=["matmul_transpose_reshape_mkldnn_fuse_pass"]) + quant=False, passes=["matmul_transpose_reshape_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmulv2_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmulv2_op.py index 06829c2d78b..29dee32df55 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmulv2_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmulv2_op.py @@ -23,25 +23,30 @@ import hypothesis.strategies as st class TestMkldnnMatmulv2Op(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: if len(program_config.inputs["input_data2"].shape) == 4: - if program_config.inputs["input_data1"].shape[ - -4] != 1 and program_config.inputs["input_data2"].shape[ - -4] != 1: - if program_config.inputs["input_data1"].shape[ - -4] != program_config.inputs["input_data2"].shape[-4]: + if ( + program_config.inputs["input_data1"].shape[-4] != 1 + and program_config.inputs["input_data2"].shape[-4] != 1 + ): + if ( + program_config.inputs["input_data1"].shape[-4] + != program_config.inputs["input_data2"].shape[-4] + ): return False - if program_config.inputs["input_data1"].shape[ - -3] != 1 and program_config.inputs["input_data2"].shape[-3] != 1: - if program_config.inputs["input_data1"].shape[ - -3] != program_config.inputs["input_data2"].shape[-3]: + if ( + program_config.inputs["input_data1"].shape[-3] != 1 + and program_config.inputs["input_data2"].shape[-3] != 1 + ): + if ( + program_config.inputs["input_data1"].shape[-3] + != program_config.inputs["input_data2"].shape[-3] + ): return False return True def sample_program_configs(self, *args, **kwargs): - def generate_input(type, *args, **kwargs): transpose_X = kwargs["transpose_X"] transpose_Y = kwargs["transpose_Y"] @@ -81,35 +86,35 @@ class TestMkldnnMatmulv2Op(MkldnnAutoScanTest): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type="matmul_v2", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["matmul_output"]}, - attrs={ - "trans_x": kwargs["transpose_X"], - "trans_y": kwargs["transpose_Y"], - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) + matmul_op = OpConfig( + type="matmul_v2", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["matmul_output"]}, + attrs={ + "trans_x": kwargs["transpose_X"], + "trans_y": kwargs["transpose_Y"], + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) program_config = ProgramConfig( ops=[matmul_op], weights={}, inputs={ - "input_data1": - TensorConfig( - data_gen=partial(generate_input, "x", *args, **kwargs)), - "input_data2": - TensorConfig( - data_gen=partial(generate_input, "y", *args, **kwargs)) + "input_data1": TensorConfig( + data_gen=partial(generate_input, "x", *args, **kwargs) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, "y", *args, **kwargs) + ), }, - outputs=["matmul_output"]) + outputs=["matmul_output"], + ) yield program_config @@ -117,14 +122,16 @@ class TestMkldnnMatmulv2Op(MkldnnAutoScanTest): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(transpose_X=st.booleans(), - transpose_Y=st.booleans(), - y_dim_len=st.sampled_from([3, 4]), - batch_size1=st.integers(min_value=1, max_value=4), - batch_size2=st.integers(min_value=1, max_value=4), - channel1=st.sampled_from([1, 16, 32, 64]), - channel2=st.sampled_from([1, 16, 32, 64]), - input_dim=st.sampled_from([16, 32, 64])) + @given( + transpose_X=st.booleans(), + transpose_Y=st.booleans(), + y_dim_len=st.sampled_from([3, 4]), + batch_size1=st.integers(min_value=1, max_value=4), + batch_size2=st.integers(min_value=1, max_value=4), + channel1=st.sampled_from([1, 16, 32, 64]), + channel2=st.sampled_from([1, 16, 32, 64]), + input_dim=st.sampled_from([16, 32, 64]), + ) def test(self, *args, **kwargs): self.run_test(*args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_mish_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_mish_op.py index 2b2759cc651..a6062cb526b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_mish_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_mish_op.py @@ -22,35 +22,39 @@ import hypothesis.strategies as st class TestMkldnnMishOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # if mode is channel, and in_shape is 1 rank - if len(program_config.inputs['input_data'].shape - ) == 1 and program_config.ops[0].attrs['mode'] == 'channel': + if ( + len(program_config.inputs['input_data'].shape) == 1 + and program_config.ops[0].attrs['mode'] == 'channel' + ): return False return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) - mish_op = OpConfig(type="mish", - inputs={"X": ["input_data"]}, - outputs={"Out": ["output_data"]}, - attrs={ - "mode": kwargs['mode'], - "data_format": kwargs['data_format'] - }) + mish_op = OpConfig( + type="mish", + inputs={"X": ["input_data"]}, + outputs={"Out": ["output_data"]}, + attrs={ + "mode": kwargs['mode'], + "data_format": kwargs['data_format'], + }, + ) program_config = ProgramConfig( ops=[mish_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -58,11 +62,13 @@ class TestMkldnnMishOp(MkldnnAutoScanTest): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(mode=st.sampled_from(['all', 'channel', 'element']), - data_format=st.sampled_from(['NCHW', 'NHWC']), - in_shape=st.lists(st.integers(min_value=1, max_value=32), - min_size=1, - max_size=4)) + @given( + mode=st.sampled_from(['all', 'channel', 'element']), + data_format=st.sampled_from(['NCHW', 'NHWC']), + in_shape=st.lists( + st.integers(min_value=1, max_value=32), min_size=1, max_size=4 + ), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad2d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad2d_op.py index e26cf2a8605..86fc1df8cc6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad2d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad2d_op.py @@ -22,29 +22,31 @@ import hypothesis.strategies as st class TestOneDNNPad2DOp(MkldnnAutoScanTest): - def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) - pad3d_op = OpConfig(type="pad2d", - inputs={"X": ["input_data"]}, - outputs={"Out": ["output_data"]}, - attrs={ - "mode": "constant", - "data_format": kwargs['data_format'], - "paddings": kwargs['paddings'], - }) + pad3d_op = OpConfig( + type="pad2d", + inputs={"X": ["input_data"]}, + outputs={"Out": ["output_data"]}, + attrs={ + "mode": "constant", + "data_format": kwargs['data_format'], + "paddings": kwargs['paddings'], + }, + ) program_config = ProgramConfig( ops=[pad3d_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -52,11 +54,15 @@ class TestOneDNNPad2DOp(MkldnnAutoScanTest): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(data_format=st.sampled_from(['NCHW', 'NHWC']), - in_shape=st.sampled_from([[2, 3, 4, 5], [1, 4, 1, 3], [4, 3, 2, 1], - [1, 1, 1, 1]]), - paddings=st.sampled_from([[0, 0, 0, 0], [1, 2, 0, 1], [2, 5, 11, 3], - [0, 5, 0, 1]])) + @given( + data_format=st.sampled_from(['NCHW', 'NHWC']), + in_shape=st.sampled_from( + [[2, 3, 4, 5], [1, 4, 1, 3], [4, 3, 2, 1], [1, 1, 1, 1]] + ), + paddings=st.sampled_from( + [[0, 0, 0, 0], [1, 2, 0, 1], [2, 5, 11, 3], [0, 5, 0, 1]] + ), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad3d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad3d_op.py index 8184bb149e1..cb4daa68653 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad3d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad3d_op.py @@ -22,37 +22,35 @@ import hypothesis.strategies as st class TestOneDNNPad3DOp(MkldnnAutoScanTest): - def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) def generate_paddings(): return np.random.randint(0, 4, size=(6)).astype(np.int32) - pad3d_op = OpConfig(type="pad3d", - inputs={ - "X": ["input_data"], - "Paddings": ["paddings_data"] - }, - outputs={"Out": ["output_data"]}, - attrs={ - "mode": "constant", - "data_format": kwargs['data_format'], - "paddings": kwargs['paddings'], - }) + pad3d_op = OpConfig( + type="pad3d", + inputs={"X": ["input_data"], "Paddings": ["paddings_data"]}, + outputs={"Out": ["output_data"]}, + attrs={ + "mode": "constant", + "data_format": kwargs['data_format'], + "paddings": kwargs['paddings'], + }, + ) program_config = ProgramConfig( ops=[pad3d_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), - "paddings_data": - TensorConfig(data_gen=generate_paddings) + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), + "paddings_data": TensorConfig(data_gen=generate_paddings), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -60,12 +58,21 @@ class TestOneDNNPad3DOp(MkldnnAutoScanTest): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(data_format=st.sampled_from(['NCDHW', 'NDHWC']), - use_paddings_tensor=st.sampled_from([True, False]), - in_shape=st.sampled_from([[2, 3, 4, 5, 6], [1, 4, 1, 3, 2], - [4, 3, 2, 1, 1], [1, 1, 1, 1, 1]]), - paddings=st.sampled_from([[0, 0, 0, 0, 0, 0], [1, 2, 0, 1, 2, 1], - [2, 5, 11, 3, 4, 3], [0, 5, 0, 1, 0, 2]])) + @given( + data_format=st.sampled_from(['NCDHW', 'NDHWC']), + use_paddings_tensor=st.sampled_from([True, False]), + in_shape=st.sampled_from( + [[2, 3, 4, 5, 6], [1, 4, 1, 3, 2], [4, 3, 2, 1, 1], [1, 1, 1, 1, 1]] + ), + paddings=st.sampled_from( + [ + [0, 0, 0, 0, 0, 0], + [1, 2, 0, 1, 2, 1], + [2, 5, 11, 3, 4, 3], + [0, 5, 0, 1, 0, 2], + ] + ), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_prelu_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_prelu_op.py index 646cf1170ee..5a9aabe1b9b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_prelu_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_prelu_op.py @@ -23,16 +23,16 @@ import hypothesis.strategies as st class TestMkldnnPreluOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # if mode is channel, and in_shape is 1 rank - if len(program_config.inputs['input_data'].shape - ) == 1 and program_config.ops[0].attrs['mode'] == 'channel': + if ( + len(program_config.inputs['input_data'].shape) == 1 + and program_config.ops[0].attrs['mode'] == 'channel' + ): return False return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) @@ -45,38 +45,42 @@ class TestMkldnnPreluOp(MkldnnAutoScanTest): return np.zeros((1)).astype(np.float32) if kwargs['data_format'] == 'NCHW': return np.random.random(kwargs['in_shape'][1]).astype( - np.float32) + np.float32 + ) else: return np.random.random(kwargs['in_shape'][-1]).astype( - np.float32) + np.float32 + ) else: if len(kwargs['in_shape']) <= 1: # not valid case, just return 0 return np.zeros((1)).astype(np.float32) return np.random.random(kwargs['in_shape']).astype(np.float32) - prelu_op = OpConfig(type="prelu", - inputs={ - "X": ["input_data"], - "Alpha": ["alpha_weight"] - }, - outputs={"Out": ["output_data"]}, - attrs={ - "mode": kwargs['mode'], - "data_format": kwargs['data_format'] - }) + prelu_op = OpConfig( + type="prelu", + inputs={"X": ["input_data"], "Alpha": ["alpha_weight"]}, + outputs={"Out": ["output_data"]}, + attrs={ + "mode": kwargs['mode'], + "data_format": kwargs['data_format'], + }, + ) program_config = ProgramConfig( ops=[prelu_op], weights={ - "alpha_weight": - TensorConfig(data_gen=partial(generate_alpha, *args, **kwargs)) + "alpha_weight": TensorConfig( + data_gen=partial(generate_alpha, *args, **kwargs) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -87,11 +91,13 @@ class TestMkldnnPreluOp(MkldnnAutoScanTest): def add_skip_pass_case(self): pass - @given(mode=st.sampled_from(['all', 'channel', 'element']), - data_format=st.sampled_from(['NCHW', 'NHWC']), - in_shape=st.lists(st.integers(min_value=1, max_value=32), - min_size=1, - max_size=4)) + @given( + mode=st.sampled_from(['all', 'channel', 'element']), + data_format=st.sampled_from(['NCHW', 'NHWC']), + in_shape=st.lists( + st.integers(min_value=1, max_value=32), min_size=1, max_size=4 + ), + ) def test(self, *args, **kwargs): self.add_skip_pass_case() self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_fuse_pass.py index 62615ce54c8..7147c412618 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_fuse_pass.py @@ -25,7 +25,6 @@ num = 32 * 64 class TestReshapeTransposeMatmulMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -35,8 +34,8 @@ class TestReshapeTransposeMatmulMkldnnFusePass(PassAutoScanTest): alpha = draw(st.floats(min_value=0.01, max_value=2)) axis = draw(st.sampled_from([[0, 2, 1, 3]])) shape = draw( - st.sampled_from([[0, 64, -1, 32], [0, 32, -1, 64], [-1, 32, 1, - 64]])) + st.sampled_from([[0, 64, -1, 32], [0, 32, -1, 64], [-1, 32, 1, 64]]) + ) batch_size = draw(st.integers(min_value=1, max_value=4)) channel = draw(st.integers(min_value=1, max_value=64)) input_dim = draw(st.sampled_from([32, 64])) @@ -65,8 +64,10 @@ class TestReshapeTransposeMatmulMkldnnFusePass(PassAutoScanTest): if attrs[2]['transpose_X'] and attrs[2]['transpose_Y']: shape_y = [ - matmul_shape[0], matmul_shape[1], matmul_shape[-1], - int(num / matmul_shape[-1]) + matmul_shape[0], + matmul_shape[1], + matmul_shape[-1], + int(num / matmul_shape[-1]), ] elif attrs[2]['transpose_X']: shape_y = matmul_shape @@ -74,70 +75,64 @@ class TestReshapeTransposeMatmulMkldnnFusePass(PassAutoScanTest): shape_y = matmul_shape else: shape_y = [ - matmul_shape[0], matmul_shape[1], matmul_shape[-1], - int(num / matmul_shape[-1]) + matmul_shape[0], + matmul_shape[1], + matmul_shape[-1], + int(num / matmul_shape[-1]), ] return np.random.random(shape_y).astype(np.float32) - attrs = [{ - "shape": shape - }, { - "axis": axis - }, { - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha - }, { - 'batch_size': batch_size, - 'channel': channel, - 'input_dim': input_dim - }] - - ops_config = [{ - "op_type": "reshape2", - "op_inputs": { - "X": ["input_data1"] - }, - "op_outputs": { - "Out": ["reshape2_output"], - "XShape": ["reshape2_xshape"] - }, - "op_attrs": { - 'shape': attrs[0]['shape'] - }, - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["reshape2_output"] + attrs = [ + {"shape": shape}, + {"axis": axis}, + { + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha, }, - "op_outputs": { - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] + { + 'batch_size': batch_size, + 'channel': channel, + 'input_dim': input_dim, }, - "op_attrs": { - 'axis': attrs[1]['axis'] + ] + + ops_config = [ + { + "op_type": "reshape2", + "op_inputs": {"X": ["input_data1"]}, + "op_outputs": { + "Out": ["reshape2_output"], + "XShape": ["reshape2_xshape"], + }, + "op_attrs": {'shape': attrs[0]['shape']}, }, - }, { - "op_type": "matmul", - "op_inputs": { - "X": ["transpose2_output"], - "Y": ["input_data2"] + { + "op_type": "transpose2", + "op_inputs": {"X": ["reshape2_output"]}, + "op_outputs": { + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + "op_attrs": {'axis': attrs[1]['axis']}, }, - "op_outputs": { - "Out": ["matmul_output"] + { + "op_type": "matmul", + "op_inputs": {"X": ["transpose2_output"], "Y": ["input_data2"]}, + "op_outputs": {"Out": ["matmul_output"]}, + "op_attrs": { + 'transpose_X': attrs[2]['transpose_X'], + 'transpose_Y': attrs[2]['transpose_Y'], + 'alpha': attrs[2]['alpha'], + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, }, - "op_attrs": { - 'transpose_X': attrs[2]['transpose_X'], - 'transpose_Y': attrs[2]['transpose_Y'], - 'alpha': attrs[2]['alpha'], - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - } - }] + ] ops = self.generate_op_config(ops_config) @@ -145,12 +140,15 @@ class TestReshapeTransposeMatmulMkldnnFusePass(PassAutoScanTest): ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input1, attrs)), - "input_data2": - TensorConfig(data_gen=partial(generate_input2, attrs)) + "input_data1": TensorConfig( + data_gen=partial(generate_input1, attrs) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input2, attrs) + ), }, - outputs=["matmul_output"]) + outputs=["matmul_output"], + ) return program_config @@ -160,7 +158,8 @@ class TestReshapeTransposeMatmulMkldnnFusePass(PassAutoScanTest): def test(self): self.run_and_statis( - quant=False, passes=["reshape_transpose_matmul_mkldnn_fuse_pass"]) + quant=False, passes=["reshape_transpose_matmul_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_v2_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_v2_fuse_pass.py index 9d6c0309e17..b62b3eaf517 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_v2_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_v2_fuse_pass.py @@ -21,24 +21,26 @@ from paddle.fluid.core import PassVersionChecker class TestReshapeTransposeMatmulV2OneDNNFusePass(InferencePassTest): - def setUp(self): self.set_params() self.tranpose_perm = [0, 2, 1, 3] self.pass_name = 'reshape_transpose_matmul_mkldnn_fuse_pass' with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=self.data_shape, - dtype="float32") - weight = fluid.layers.create_parameter(shape=self.weight_shape, - dtype="float32") + data = fluid.data( + name="data", shape=self.data_shape, dtype="float32" + ) + weight = fluid.layers.create_parameter( + shape=self.weight_shape, dtype="float32" + ) reshape = fluid.layers.reshape(data, shape=self.reshape_shape) transpose = fluid.layers.transpose(reshape, self.tranpose_perm) - matmul = paddle.matmul(transpose, - weight, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y) + matmul = paddle.matmul( + transpose, + weight, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + ) self.fetch_list = [matmul] self.enable_mkldnn = True @@ -60,8 +62,8 @@ class TestReshapeTransposeMatmulV2OneDNNFusePass(InferencePassTest): class TestReshapeTransposeMatmulV2OneDNNFusePassBroadcast( - TestReshapeTransposeMatmulV2OneDNNFusePass): - + TestReshapeTransposeMatmulV2OneDNNFusePass +): def set_params(self): self.data_shape = [2, 64, 16] self.weight_shape = [1, 2, 8, 64] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_scale_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_scale_matmul_fuse_pass.py index c12915c1eb1..109a56edbcb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_scale_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_scale_matmul_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -40,39 +39,55 @@ class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): def generate_input(attrs, type): if attrs[1]['transpose_X'] and attrs[1]['transpose_Y']: shape_x = [ - attrs[2]['batch_size'], attrs[2]['channel'], - attrs[2]['input_dim'], 32 + attrs[2]['batch_size'], + attrs[2]['channel'], + attrs[2]['input_dim'], + 32, ] shape_y = [ - attrs[2]['batch_size'], attrs[2]['channel'], 64, - attrs[2]['input_dim'] + attrs[2]['batch_size'], + attrs[2]['channel'], + 64, + attrs[2]['input_dim'], ] elif attrs[1]['transpose_X']: shape_x = [ - attrs[2]['batch_size'], attrs[2]['channel'], - attrs[2]['input_dim'], 32 + attrs[2]['batch_size'], + attrs[2]['channel'], + attrs[2]['input_dim'], + 32, ] shape_y = [ - attrs[2]['batch_size'], attrs[2]['channel'], - attrs[2]['input_dim'], 64 + attrs[2]['batch_size'], + attrs[2]['channel'], + attrs[2]['input_dim'], + 64, ] elif attrs[1]['transpose_Y']: shape_x = [ - attrs[2]['batch_size'], attrs[2]['channel'], 32, - attrs[2]['input_dim'] + attrs[2]['batch_size'], + attrs[2]['channel'], + 32, + attrs[2]['input_dim'], ] shape_y = [ - attrs[2]['batch_size'], attrs[2]['channel'], 8, - attrs[2]['input_dim'] + attrs[2]['batch_size'], + attrs[2]['channel'], + 8, + attrs[2]['input_dim'], ] else: shape_x = [ - attrs[2]['batch_size'], attrs[2]['channel'], 32, - attrs[2]['input_dim'] + attrs[2]['batch_size'], + attrs[2]['channel'], + 32, + attrs[2]['input_dim'], ] shape_y = [ - attrs[2]['batch_size'], attrs[2]['channel'], - attrs[2]['input_dim'], 16 + attrs[2]['batch_size'], + attrs[2]['channel'], + attrs[2]['input_dim'], + 16, ] if type == "x": @@ -80,54 +95,52 @@ class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): else: return np.random.random(shape_y).astype(np.float32) - attrs = [{ - "scale": scale, - "bias": bias, - "bias_after_scale": bias_after_scale - }, { - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha - }, { - 'batch_size': batch_size, - 'channel': channel, - 'input_dim': input_dim - }] - - ops_config = [{ - "op_type": "scale", - "op_inputs": { - "X": ["input_data1"] + attrs = [ + { + "scale": scale, + "bias": bias, + "bias_after_scale": bias_after_scale, }, - "op_outputs": { - "Out": ["scale_output"] + { + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha, }, - "op_attrs": { - "scale": attrs[0]['scale'], - "bias": attrs[0]['bias'], - "bias_after_scale": attrs[0]['bias_after_scale'] + { + 'batch_size': batch_size, + 'channel': channel, + 'input_dim': input_dim, }, - }, { - "op_type": "matmul", - "op_inputs": { - "X": ["scale_output"], - "Y": ["input_data2"] + ] + + ops_config = [ + { + "op_type": "scale", + "op_inputs": {"X": ["input_data1"]}, + "op_outputs": {"Out": ["scale_output"]}, + "op_attrs": { + "scale": attrs[0]['scale'], + "bias": attrs[0]['bias'], + "bias_after_scale": attrs[0]['bias_after_scale'], + }, }, - "op_outputs": { - "Out": ["matmul_output"] + { + "op_type": "matmul", + "op_inputs": {"X": ["scale_output"], "Y": ["input_data2"]}, + "op_outputs": {"Out": ["matmul_output"]}, + "op_attrs": { + 'transpose_X': attrs[1]['transpose_X'], + 'transpose_Y': attrs[1]['transpose_Y'], + 'alpha': attrs[1]['alpha'], + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, }, - "op_attrs": { - 'transpose_X': attrs[1]['transpose_X'], - 'transpose_Y': attrs[1]['transpose_Y'], - 'alpha': attrs[1]['alpha'], - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - } - }] + ] ops = self.generate_op_config(ops_config) @@ -135,12 +148,15 @@ class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, attrs, "x")), - "input_data2": - TensorConfig(data_gen=partial(generate_input, attrs, "y")) + "input_data1": TensorConfig( + data_gen=partial(generate_input, attrs, "x") + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, attrs, "y") + ), }, - outputs=["matmul_output"]) + outputs=["matmul_output"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shape_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shape_op.py index 92111062b12..15fae343729 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shape_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shape_op.py @@ -22,28 +22,31 @@ import hypothesis.strategies as st class TestMkldnnShapeOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype( - kwargs['in_dtype']) + kwargs['in_dtype'] + ) - shape_op = OpConfig(type="shape", - inputs={"Input": ["input_data"]}, - outputs={"Out": ["output_data"]}) + shape_op = OpConfig( + type="shape", + inputs={"Input": ["input_data"]}, + outputs={"Out": ["output_data"]}, + ) program_config = ProgramConfig( ops=[shape_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -51,10 +54,12 @@ class TestMkldnnShapeOp(MkldnnAutoScanTest): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(in_shape=st.lists(st.integers(min_value=1, max_value=3), - min_size=1, - max_size=6), - in_dtype=st.sampled_from([np.float32, np.uint16, np.int8, np.uint8])) + @given( + in_shape=st.lists( + st.integers(min_value=1, max_value=3), min_size=1, max_size=6 + ), + in_dtype=st.sampled_from([np.float32, np.uint16, np.int8, np.uint8]), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py index 423bb0f0dbb..b4d94c4e815 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py @@ -31,7 +31,6 @@ def product(input): class TestShuffleChannelMKLDNNDetectPass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: input_shape = program_config.inputs['input_data'].shape first_reshape2_shape = program_config.ops[0].attrs['shape'] @@ -43,18 +42,26 @@ class TestShuffleChannelMKLDNNDetectPass(PassAutoScanTest): img_w = input_shape[-1] if shape_prod != product(first_reshape2_shape) or shape_prod != product( - second_reshape2_shape): + second_reshape2_shape + ): return False - if len(input_shape) != 4 or len(first_reshape2_shape) != 5 or len( - second_reshape2_shape) != 4: + if ( + len(input_shape) != 4 + or len(first_reshape2_shape) != 5 + or len(second_reshape2_shape) != 4 + ): return False if transpose2_axis != [0, 2, 1, 3, 4]: return False - if first_reshape2_shape[-1] != img_w or first_reshape2_shape[ - -2] != img_h: + if ( + first_reshape2_shape[-1] != img_w + or first_reshape2_shape[-2] != img_h + ): return False - if second_reshape2_shape[-1] != img_w or second_reshape2_shape[ - -2] != img_h: + if ( + second_reshape2_shape[-1] != img_w + or second_reshape2_shape[-2] != img_h + ): return False return True @@ -62,10 +69,12 @@ class TestShuffleChannelMKLDNNDetectPass(PassAutoScanTest): def sample_program_config(self, draw): input_shape = draw(st.sampled_from([[128, 32, 32]])) first_reshape2_shape = draw( - st.sampled_from([[2, 64, 32, 32], [8, 16, 32, 32]])) + st.sampled_from([[2, 64, 32, 32], [8, 16, 32, 32]]) + ) transpose2_axis = draw(st.sampled_from([[0, 2, 1, 3, 4], [0, 2, 1, 3]])) second_reshape2_shape = draw( - st.sampled_from([[128, 32, 32], [128, 31, 32]])) + st.sampled_from([[128, 32, 32], [128, 31, 32]]) + ) batch_size = draw(st.integers(min_value=1, max_value=10)) input_shape.insert(0, batch_size) @@ -75,43 +84,37 @@ class TestShuffleChannelMKLDNNDetectPass(PassAutoScanTest): def generate_input(): return np.random.random(input_shape).astype(np.float32) - ops_config = [{ - "op_type": "reshape2", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["first_reshape2_output"], - "XShape": ["first_reshape2_xshape"] - }, - "op_attrs": { - 'shape': first_reshape2_shape - }, - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["first_reshape2_output"] - }, - "op_outputs": { - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] - }, - "op_attrs": { - 'axis': transpose2_axis + ops_config = [ + { + "op_type": "reshape2", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": { + "Out": ["first_reshape2_output"], + "XShape": ["first_reshape2_xshape"], + }, + "op_attrs": {'shape': first_reshape2_shape}, }, - }, { - "op_type": "reshape2", - "op_inputs": { - "X": ["transpose2_output"], + { + "op_type": "transpose2", + "op_inputs": {"X": ["first_reshape2_output"]}, + "op_outputs": { + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + "op_attrs": {'axis': transpose2_axis}, }, - "op_outputs": { - "Out": ["output_data"], - "XShape": ["second_reshape2_xshape"] + { + "op_type": "reshape2", + "op_inputs": { + "X": ["transpose2_output"], + }, + "op_outputs": { + "Out": ["output_data"], + "XShape": ["second_reshape2_xshape"], + }, + "op_attrs": {'shape': second_reshape2_shape}, }, - "op_attrs": { - 'shape': second_reshape2_shape - } - }] + ] ops = self.generate_op_config(ops_config) @@ -121,7 +124,8 @@ class TestShuffleChannelMKLDNNDetectPass(PassAutoScanTest): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["output_data"]) + outputs=["output_data"], + ) return program_config @@ -130,8 +134,9 @@ class TestShuffleChannelMKLDNNDetectPass(PassAutoScanTest): yield config, ["shuffle_channel"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["shuffle_channel_mkldnn_detect_pass"]) + self.run_and_statis( + quant=False, passes=["shuffle_channel_mkldnn_detect_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_op.py index d9050b58ee3..72dfe3f6561 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_op.py @@ -22,28 +22,30 @@ import hypothesis.strategies as st class TestMKLDNNShuffleChannelOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) - shuffle_channel_op = OpConfig(type="shuffle_channel", - inputs={"X": ["input_data"]}, - outputs={"Out": ["output_data"]}, - attrs={"group": kwargs['group']}) + shuffle_channel_op = OpConfig( + type="shuffle_channel", + inputs={"X": ["input_data"]}, + outputs={"Out": ["output_data"]}, + attrs={"group": kwargs['group']}, + ) program_config = ProgramConfig( ops=[shuffle_channel_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -51,8 +53,10 @@ class TestMKLDNNShuffleChannelOp(MkldnnAutoScanTest): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(group=st.sampled_from([1, 2, 8, 32, 128]), - in_shape=st.sampled_from([[5, 512, 2, 3], [2, 256, 5, 4]])) + @given( + group=st.sampled_from([1, 2, 8, 32, 128]), + in_shape=st.sampled_from([[5, 512, 2, 3], [2, 256, 5, 4]]), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py index 037234ccda1..67445e18444 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py @@ -28,17 +28,18 @@ class SoftplusActivationReluOneDNNFusePassTest(InferencePassTest): def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) softplus_out = fluid.layers.softplus(data) if self.fuse_beta is not None: - activation_out = self.fuse_activation(softplus_out, - self.fuse_alpha, - self.fuse_beta) + activation_out = self.fuse_activation( + softplus_out, self.fuse_alpha, self.fuse_beta + ) elif self.fuse_alpha is not None: - activation_out = self.fuse_activation(softplus_out, - self.fuse_alpha) + activation_out = self.fuse_activation( + softplus_out, self.fuse_alpha + ) else: activation_out = self.fuse_activation(softplus_out) @@ -60,52 +61,52 @@ class SoftplusActivationReluOneDNNFusePassTest(InferencePassTest): class SoftplusActivationTanhOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.tanh class SoftplusActivationLeakyReluOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.leaky_relu self.fuse_alpha = 0.3 class SoftplusActivationSwishOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.swish self.fuse_alpha = 3 class SoftplusActivationHardSwishOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.hard_swish class SoftplusActivationSqrtOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.hard_swish class SoftplusActivationAbsOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.abs class SoftplusActivationClipOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.clip self.fuse_alpha = 1.1 @@ -113,30 +114,30 @@ class SoftplusActivationClipOneDNNFusePassTest( class SoftplusActivationGeluErfOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.gelu class SoftplusActivationGeluTanhOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.gelu self.fuse_alpha = True # simulated "Approximate" attr class SoftplusActivationRelu6OneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.relu6 class SoftplusActivationSigmoidOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.sigmoid diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mul_gru_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mul_gru_fuse_pass.py index 54f6b60ff86..ad0ba4c1fc9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mul_gru_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mul_gru_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestMulGruFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -43,83 +42,89 @@ class TestMulGruFusePass(PassAutoScanTest): def generate_weight(shape): return np.full(shape, 0.0001).astype(np.float32) - im2sequence_op = OpConfig(type="im2sequence", - inputs={"X": ["input_data"]}, - outputs={"Out": ["seq_out"]}, - attrs={ - "kernels": [6, 1], - "out_stride": [1, 1], - "paddings": [0, 0, 0, 0], - "strides": [1, 1] - }) - - mul_op = OpConfig(type="mul", - inputs={ - "X": ["seq_out"], - "Y": ["mul_weight"] - }, - outputs={"Out": ["mul_out"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) + im2sequence_op = OpConfig( + type="im2sequence", + inputs={"X": ["input_data"]}, + outputs={"Out": ["seq_out"]}, + attrs={ + "kernels": [6, 1], + "out_stride": [1, 1], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, + ) + + mul_op = OpConfig( + type="mul", + inputs={"X": ["seq_out"], "Y": ["mul_weight"]}, + outputs={"Out": ["mul_out"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) if has_origin_mode: - gru_op = OpConfig(type="gru", - inputs={ - "Input": ["mul_out"], - "Weight": ["gru_weight"], - "Bias": ["gru_bias"] - }, - outputs={ - "BatchGate": ["batch_gate"], - "BatchHidden": ["batch_hidden"], - "BatchResetHiddenPrev": ["batch_reset"], - "Hidden": ["hidden"] - }, - attrs={ - 'activation': activation, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'is_test': True, - 'origin_mode': origin_mode - }) + gru_op = OpConfig( + type="gru", + inputs={ + "Input": ["mul_out"], + "Weight": ["gru_weight"], + "Bias": ["gru_bias"], + }, + outputs={ + "BatchGate": ["batch_gate"], + "BatchHidden": ["batch_hidden"], + "BatchResetHiddenPrev": ["batch_reset"], + "Hidden": ["hidden"], + }, + attrs={ + 'activation': activation, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'is_test': True, + 'origin_mode': origin_mode, + }, + ) else: - gru_op = OpConfig(type="gru", - inputs={ - "Input": ["mul_out"], - "Weight": ["gru_weight"], - "Bias": ["gru_bias"] - }, - outputs={ - "BatchGate": ["batch_gate"], - "BatchHidden": ["batch_hidden"], - "BatchResetHiddenPrev": ["batch_reset"], - "Hidden": ["hidden"] - }, - attrs={ - 'activation': activation, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'is_test': True - }) + gru_op = OpConfig( + type="gru", + inputs={ + "Input": ["mul_out"], + "Weight": ["gru_weight"], + "Bias": ["gru_bias"], + }, + outputs={ + "BatchGate": ["batch_gate"], + "BatchHidden": ["batch_hidden"], + "BatchResetHiddenPrev": ["batch_reset"], + "Hidden": ["hidden"], + }, + attrs={ + 'activation': activation, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'is_test': True, + }, + ) model_net = [im2sequence_op, mul_op, gru_op] program_config = ProgramConfig( ops=model_net, weights={ - "mul_weight": - TensorConfig(data_gen=partial(generate_weight, [768, 600])), - "gru_weight": - TensorConfig(data_gen=partial(generate_weight, [200, 600])), - "gru_bias": - TensorConfig(data_gen=partial(generate_weight, [1, 600])) + "mul_weight": TensorConfig( + data_gen=partial(generate_weight, [768, 600]) + ), + "gru_weight": TensorConfig( + data_gen=partial(generate_weight, [200, 600]) + ), + "gru_bias": TensorConfig( + data_gen=partial(generate_weight, [1, 600]) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["hidden"]) + outputs=["hidden"], + ) return program_config @@ -128,9 +133,9 @@ class TestMulGruFusePass(PassAutoScanTest): yield config, ["im2sequence", "fusion_gru"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_duration=300, - passes=["mul_gru_fuse_pass"]) + self.run_and_statis( + quant=False, max_duration=300, passes=["mul_gru_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mul_lstm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mul_lstm_fuse_pass.py index 2d65a7fad5e..3b6e72ecf43 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mul_lstm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mul_lstm_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestMulLstmFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -34,7 +33,8 @@ class TestMulLstmFusePass(PassAutoScanTest): gate_activation = draw(st.sampled_from(["sigmoid"])) cell_activation = draw(st.sampled_from(["tanh", "relu", "identity"])) candidate_activation = draw( - st.sampled_from(["tanh", "relu", "identity"])) + st.sampled_from(["tanh", "relu", "identity"]) + ) batch_size = draw(st.integers(min_value=1, max_value=40)) def generate_input(): @@ -44,47 +44,47 @@ class TestMulLstmFusePass(PassAutoScanTest): def generate_weight(shape): return np.full(shape, 0.0001).astype(np.float32) - im2sequence_op = OpConfig(type="im2sequence", - inputs={"X": ["input_data"]}, - outputs={"Out": ["seq_out"]}, - attrs={ - "kernels": [6, 1], - "out_stride": [1, 1], - "paddings": [0, 0, 0, 0], - "strides": [1, 1] - }) - - mul_op = OpConfig(type="mul", - inputs={ - "X": ["seq_out"], - "Y": ["mul_weight"] - }, - outputs={"Out": ["mul_out"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) - - lstm_op = OpConfig(type="lstm", - inputs={ - "Input": ["mul_out"], - "Weight": ["lstm_weight"], - "Bias": ["lstm_bias"] - }, - outputs={ - "Hidden": ["lstm_hidden"], - "Cell": ["lstm_cell"], - "BatchGate": ["lstm_gate"], - "BatchCellPreAct": ["lstm_batch_cell"] - }, - attrs={ - 'use_peepholes': use_peepholes, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'cell_activation': cell_activation, - 'candidate_activation': candidate_activation, - 'is_test': True - }) + im2sequence_op = OpConfig( + type="im2sequence", + inputs={"X": ["input_data"]}, + outputs={"Out": ["seq_out"]}, + attrs={ + "kernels": [6, 1], + "out_stride": [1, 1], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, + ) + + mul_op = OpConfig( + type="mul", + inputs={"X": ["seq_out"], "Y": ["mul_weight"]}, + outputs={"Out": ["mul_out"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) + + lstm_op = OpConfig( + type="lstm", + inputs={ + "Input": ["mul_out"], + "Weight": ["lstm_weight"], + "Bias": ["lstm_bias"], + }, + outputs={ + "Hidden": ["lstm_hidden"], + "Cell": ["lstm_cell"], + "BatchGate": ["lstm_gate"], + "BatchCellPreAct": ["lstm_batch_cell"], + }, + attrs={ + 'use_peepholes': use_peepholes, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation, + 'is_test': True, + }, + ) model_net = [im2sequence_op, mul_op, lstm_op] @@ -96,17 +96,21 @@ class TestMulLstmFusePass(PassAutoScanTest): program_config = ProgramConfig( ops=model_net, weights={ - "mul_weight": - TensorConfig(data_gen=partial(generate_weight, [768, 600])), - "lstm_weight": - TensorConfig(data_gen=partial(generate_weight, [150, 600])), - "lstm_bias": - TensorConfig(data_gen=partial(generate_weight, lstm_bias_shape)) + "mul_weight": TensorConfig( + data_gen=partial(generate_weight, [768, 600]) + ), + "lstm_weight": TensorConfig( + data_gen=partial(generate_weight, [150, 600]) + ), + "lstm_bias": TensorConfig( + data_gen=partial(generate_weight, lstm_bias_shape) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["lstm_hidden"]) + outputs=["lstm_hidden"], + ) return program_config @@ -115,9 +119,9 @@ class TestMulLstmFusePass(PassAutoScanTest): yield config, ["im2sequence", "fusion_lstm"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_duration=300, - passes=["mul_lstm_fuse_pass"]) + self.run_and_statis( + quant=False, max_duration=300, passes=["mul_lstm_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_multihead_matmul_fuse_pass_v3.py b/python/paddle/fluid/tests/unittests/ir/inference/test_multihead_matmul_fuse_pass_v3.py index c59da1a8913..25f1c551709 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_multihead_matmul_fuse_pass_v3.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_multihead_matmul_fuse_pass_v3.py @@ -20,171 +20,190 @@ import unittest class TestMultiheadMatmulFusePass(PassAutoScanTest): - def sample_predictor_configs(self, program_config): # gpu config = self.create_inference_config(use_gpu=True) yield config, ["multihead_matmul", "mul"], (1e-2, 1e-3) def sample_program_config(self, draw): - def generate_mul_input(): return np.random.random([1, 128, 768]).astype(np.float32) - 0.5 def generate_elewise_input(): return np.random.random([1, 12, 128, 128]).astype(np.float32) - mul_0 = OpConfig("mul", - inputs={ - "X": ["mul_x"], - "Y": ["mul_0_w"] - }, - outputs={"Out": ["mul_0_out"]}, - x_num_col_dims=2, - y_num_col_dims=1) - mul_1 = OpConfig("mul", - inputs={ - "X": ["mul_x"], - "Y": ["mul_1_w"] - }, - outputs={"Out": ["mul_1_out"]}, - x_num_col_dims=2, - y_num_col_dims=1) - mul_2 = OpConfig("mul", - inputs={ - "X": ["mul_x"], - "Y": ["mul_2_w"] - }, - outputs={"Out": ["mul_2_out"]}, - x_num_col_dims=2, - y_num_col_dims=1) - ele_0 = OpConfig("elementwise_add", - inputs={ - "X": [mul_0.outputs["Out"][0]], - "Y": ["ele_0_w"] - }, - outputs={"Out": ["ele_0_out"]}, - axis=-1) - ele_1 = OpConfig("elementwise_add", - inputs={ - "X": [mul_1.outputs["Out"][0]], - "Y": ["ele_1_w"] - }, - outputs={"Out": ["ele_1_out"]}, - axis=-1) - ele_2 = OpConfig("elementwise_add", - inputs={ - "X": [mul_2.outputs["Out"][0]], - "Y": ["ele_2_w"] - }, - outputs={"Out": ["ele_2_out"]}, - axis=-1) - reshape_0 = OpConfig("reshape2", - inputs={"X": [ele_0.outputs["Out"][0]]}, - outputs={ - "Out": ["reshape_0_out"], - "XShape": ["reshape_0_Xout"] - }, - shape=(1, 128, 12, 64)) - reshape_1 = OpConfig("reshape2", - inputs={"X": [ele_1.outputs["Out"][0]]}, - outputs={ - "Out": ["reshape_1_out"], - "XShape": ["reshape_1_Xout"] - }, - shape=(1, 128, 12, 64)) - reshape_2 = OpConfig("reshape2", - inputs={"X": [ele_2.outputs["Out"][0]]}, - outputs={ - "Out": ["reshape_2_out"], - "XShape": ["reshape_2_Xout"] - }, - shape=(1, 128, 12, 64)) - transpose_0 = OpConfig("transpose2", - inputs={"X": [reshape_0.outputs["Out"][0]]}, - outputs={"Out": ["transpose_0_out"]}, - axis=(0, 2, 1, 3)) - transpose_1 = OpConfig("transpose2", - inputs={"X": [reshape_1.outputs["Out"][0]]}, - outputs={"Out": ["transpose_1_out"]}, - axis=(0, 2, 3, 1)) - transpose_2 = OpConfig("transpose2", - inputs={"X": [reshape_2.outputs["Out"][0]]}, - outputs={"Out": ["transpose_2_out"]}, - axis=(0, 2, 1, 3)) - matmul_0 = OpConfig("matmul", - inputs={ - "X": [transpose_0.outputs["Out"][0]], - "Y": [transpose_1.outputs["Out"][0]] - }, - outputs={"Out": ["matmul_0_out"]}, - alpha=0.125, - transpose_X=False, - transpose_Y=False, - fused_reshape_Out=[], - fused_reshape_X=[], - fused_reshape_Y=[], - fused_transpose_Out=[], - fused_transpose_X=[], - fused_transpose_Y=[]) - ele_3 = OpConfig("elementwise_add", - inputs={ - "X": [matmul_0.outputs["Out"][0]], - "Y": ["eltadd_qk_b_var"] - }, - outputs={"Out": ["ele_3_out"]}, - axis=-1) - softmax_op = OpConfig("softmax", - inputs={"X": [ele_3.outputs["Out"][0]]}, - outputs={"Out": ["softmax_out"]}, - axis=3, - is_test=True) - matmul_1 = OpConfig("matmul", - inputs={ - "X": [softmax_op.outputs["Out"][0]], - "Y": [transpose_2.outputs["Out"][0]] - }, - outputs={"Out": ["matmul_1_out"]}, - alpha=1.0, - transpose_X=False, - transpose_Y=False, - fused_reshape_Out=[], - fused_reshape_X=[], - fused_reshape_Y=[], - fused_transpose_Out=[], - fused_transpose_X=[], - fused_transpose_Y=[]) - transpose_3 = OpConfig("transpose2", - inputs={"X": [matmul_1.outputs["Out"][0]]}, - outputs={"Out": ["transpose_3_out"]}, - axis=(0, 2, 1, 3)) - reshape_3 = OpConfig("reshape2", - inputs={"X": [transpose_3.outputs["Out"][0]]}, - outputs={ - "Out": ["reshape_3_out"], - "XShape": ["reshape_3_Xout"] - }, - shape=(1, 128, 768)) - mul_3 = OpConfig("mul", - inputs={ - "X": [reshape_3.outputs["Out"][0]], - "Y": ["mul_3_w"] - }, - outputs={"Out": ["mul_3_out"]}, - x_num_col_dims=2, - y_num_col_dims=1) + mul_0 = OpConfig( + "mul", + inputs={"X": ["mul_x"], "Y": ["mul_0_w"]}, + outputs={"Out": ["mul_0_out"]}, + x_num_col_dims=2, + y_num_col_dims=1, + ) + mul_1 = OpConfig( + "mul", + inputs={"X": ["mul_x"], "Y": ["mul_1_w"]}, + outputs={"Out": ["mul_1_out"]}, + x_num_col_dims=2, + y_num_col_dims=1, + ) + mul_2 = OpConfig( + "mul", + inputs={"X": ["mul_x"], "Y": ["mul_2_w"]}, + outputs={"Out": ["mul_2_out"]}, + x_num_col_dims=2, + y_num_col_dims=1, + ) + ele_0 = OpConfig( + "elementwise_add", + inputs={"X": [mul_0.outputs["Out"][0]], "Y": ["ele_0_w"]}, + outputs={"Out": ["ele_0_out"]}, + axis=-1, + ) + ele_1 = OpConfig( + "elementwise_add", + inputs={"X": [mul_1.outputs["Out"][0]], "Y": ["ele_1_w"]}, + outputs={"Out": ["ele_1_out"]}, + axis=-1, + ) + ele_2 = OpConfig( + "elementwise_add", + inputs={"X": [mul_2.outputs["Out"][0]], "Y": ["ele_2_w"]}, + outputs={"Out": ["ele_2_out"]}, + axis=-1, + ) + reshape_0 = OpConfig( + "reshape2", + inputs={"X": [ele_0.outputs["Out"][0]]}, + outputs={"Out": ["reshape_0_out"], "XShape": ["reshape_0_Xout"]}, + shape=(1, 128, 12, 64), + ) + reshape_1 = OpConfig( + "reshape2", + inputs={"X": [ele_1.outputs["Out"][0]]}, + outputs={"Out": ["reshape_1_out"], "XShape": ["reshape_1_Xout"]}, + shape=(1, 128, 12, 64), + ) + reshape_2 = OpConfig( + "reshape2", + inputs={"X": [ele_2.outputs["Out"][0]]}, + outputs={"Out": ["reshape_2_out"], "XShape": ["reshape_2_Xout"]}, + shape=(1, 128, 12, 64), + ) + transpose_0 = OpConfig( + "transpose2", + inputs={"X": [reshape_0.outputs["Out"][0]]}, + outputs={"Out": ["transpose_0_out"]}, + axis=(0, 2, 1, 3), + ) + transpose_1 = OpConfig( + "transpose2", + inputs={"X": [reshape_1.outputs["Out"][0]]}, + outputs={"Out": ["transpose_1_out"]}, + axis=(0, 2, 3, 1), + ) + transpose_2 = OpConfig( + "transpose2", + inputs={"X": [reshape_2.outputs["Out"][0]]}, + outputs={"Out": ["transpose_2_out"]}, + axis=(0, 2, 1, 3), + ) + matmul_0 = OpConfig( + "matmul", + inputs={ + "X": [transpose_0.outputs["Out"][0]], + "Y": [transpose_1.outputs["Out"][0]], + }, + outputs={"Out": ["matmul_0_out"]}, + alpha=0.125, + transpose_X=False, + transpose_Y=False, + fused_reshape_Out=[], + fused_reshape_X=[], + fused_reshape_Y=[], + fused_transpose_Out=[], + fused_transpose_X=[], + fused_transpose_Y=[], + ) + ele_3 = OpConfig( + "elementwise_add", + inputs={ + "X": [matmul_0.outputs["Out"][0]], + "Y": ["eltadd_qk_b_var"], + }, + outputs={"Out": ["ele_3_out"]}, + axis=-1, + ) + softmax_op = OpConfig( + "softmax", + inputs={"X": [ele_3.outputs["Out"][0]]}, + outputs={"Out": ["softmax_out"]}, + axis=3, + is_test=True, + ) + matmul_1 = OpConfig( + "matmul", + inputs={ + "X": [softmax_op.outputs["Out"][0]], + "Y": [transpose_2.outputs["Out"][0]], + }, + outputs={"Out": ["matmul_1_out"]}, + alpha=1.0, + transpose_X=False, + transpose_Y=False, + fused_reshape_Out=[], + fused_reshape_X=[], + fused_reshape_Y=[], + fused_transpose_Out=[], + fused_transpose_X=[], + fused_transpose_Y=[], + ) + transpose_3 = OpConfig( + "transpose2", + inputs={"X": [matmul_1.outputs["Out"][0]]}, + outputs={"Out": ["transpose_3_out"]}, + axis=(0, 2, 1, 3), + ) + reshape_3 = OpConfig( + "reshape2", + inputs={"X": [transpose_3.outputs["Out"][0]]}, + outputs={"Out": ["reshape_3_out"], "XShape": ["reshape_3_Xout"]}, + shape=(1, 128, 768), + ) + mul_3 = OpConfig( + "mul", + inputs={"X": [reshape_3.outputs["Out"][0]], "Y": ["mul_3_w"]}, + outputs={"Out": ["mul_3_out"]}, + x_num_col_dims=2, + y_num_col_dims=1, + ) ops = [ - mul_0, mul_1, mul_2, ele_0, ele_1, ele_2, reshape_0, reshape_1, - reshape_2, transpose_0, transpose_1, transpose_2, matmul_0, ele_3, - softmax_op, matmul_1, transpose_3, reshape_3, mul_3 + mul_0, + mul_1, + mul_2, + ele_0, + ele_1, + ele_2, + reshape_0, + reshape_1, + reshape_2, + transpose_0, + transpose_1, + transpose_2, + matmul_0, + ele_3, + softmax_op, + matmul_1, + transpose_3, + reshape_3, + mul_3, ] program_config = ProgramConfig( ops=ops, inputs={ - "mul_x": - TensorConfig(data_gen=partial(generate_mul_input)), - "eltadd_qk_b_var": - TensorConfig(data_gen=partial(generate_elewise_input)) + "mul_x": TensorConfig(data_gen=partial(generate_mul_input)), + "eltadd_qk_b_var": TensorConfig( + data_gen=partial(generate_elewise_input) + ), }, weights={ "mul_0_w": TensorConfig(shape=[768, 768]), @@ -193,16 +212,19 @@ class TestMultiheadMatmulFusePass(PassAutoScanTest): "mul_3_w": TensorConfig(shape=[768, 768]), "ele_0_w": TensorConfig(shape=[768]), "ele_1_w": TensorConfig(shape=[768]), - "ele_2_w": TensorConfig(shape=[768]) + "ele_2_w": TensorConfig(shape=[768]), }, - outputs=[ops[-1].outputs["Out"][0]]) + outputs=[ops[-1].outputs["Out"][0]], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - min_success_num=1, - passes=["multihead_matmul_fuse_pass_v3"]) + self.run_and_statis( + quant=False, + max_examples=100, + min_success_num=1, + passes=["multihead_matmul_fuse_pass_v3"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_fc_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_fc_activation_fuse_pass.py index 4e798a0ed57..770aac84ec0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_fc_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_fc_activation_fuse_pass.py @@ -21,32 +21,46 @@ import hypothesis.strategies as st class TestFCActivationOneDNNFusePass(PassAutoScanTest): - def sample_program_config(self, draw): fc_in = draw(st.sampled_from([32, 64])) fc_wei = draw(st.sampled_from([64])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'swish', 'mish', 'sqrt', 'hard_swish', - 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', - 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_input(shape): return np.random.random(shape).astype(np.float32) - fc_op = OpConfig(type="fc", - inputs={ - "Input": ["fc_input"], - "W": ["fc_weight"], - "Bias": ["fc_bias"] - }, - outputs={"Out": ["fc_output"]}, - attrs={ - "use_mkldnn": True, - "padding_weights": False, - "in_num_col_dims": 1, - }) + fc_op = OpConfig( + type="fc", + inputs={ + "Input": ["fc_input"], + "W": ["fc_weight"], + "Bias": ["fc_bias"], + }, + outputs={"Out": ["fc_output"]}, + attrs={ + "use_mkldnn": True, + "padding_weights": False, + "in_num_col_dims": 1, + }, + ) if activation_type == "clip": activation_op = OpConfig( @@ -54,58 +68,69 @@ class TestFCActivationOneDNNFusePass(PassAutoScanTest): inputs={"X": ["fc_output"]}, outputs={"Out": ["activation_output"]}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) elif activation_type == "gelu": - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}, - approximate=draw(st.booleans())) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + approximate=draw(st.booleans()), + ) elif activation_type == "leaky_relu": - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "relu6": - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}, - threshold=6) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + threshold=6, + ) elif activation_type == "swish": - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}, - beta=draw( - st.floats(min_value=0.1, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + beta=draw(st.floats(min_value=0.1, max_value=10.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + ) model_net = [fc_op, activation_op] program_config = ProgramConfig( ops=model_net, weights={ - "fc_weight": - TensorConfig( - data_gen=partial(generate_input, [fc_wei, fc_wei])), - "fc_bias": - TensorConfig(data_gen=partial(generate_input, [fc_wei])), + "fc_weight": TensorConfig( + data_gen=partial(generate_input, [fc_wei, fc_wei]) + ), + "fc_bias": TensorConfig( + data_gen=partial(generate_input, [fc_wei]) + ), }, inputs={ - "fc_input": - TensorConfig(data_gen=partial(generate_input, [fc_in, fc_wei])) + "fc_input": TensorConfig( + data_gen=partial(generate_input, [fc_in, fc_wei]) + ) }, - outputs=["activation_output"]) + outputs=["activation_output"], + ) return program_config def sample_predictor_configs(self, program_config): config = self.create_inference_config( - use_mkldnn=True, passes=["fc_act_mkldnn_fuse_pass"]) + use_mkldnn=True, passes=["fc_act_mkldnn_fuse_pass"] + ) yield config, ["fc"], (1e-5, 1e-5) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_repeated_fc_relu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_repeated_fc_relu_fuse_pass.py index f0cb8797f4e..1334407cfaf 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_repeated_fc_relu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_repeated_fc_relu_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestRepeatedFcReluFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,82 +38,77 @@ class TestRepeatedFcReluFusePass(PassAutoScanTest): def generate_weight(shape): return np.random.random(shape).astype(np.float32) - attrs = [{ - "x_col": x_col, - "y_col": y_col - }, { - "axis": axis - }, { - 'batch_size': batch_size, - 'dim': dim - }] - - mul_op1 = OpConfig(type="mul", - inputs={ - "X": ["input_data"], - "Y": ["mul1_weight"] - }, - outputs={"Out": ["mul1_output"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) - - elt_op1 = OpConfig(type="elementwise_add", - inputs={ - "X": ["mul1_output"], - "Y": ["elementwise1_weight"] - }, - outputs={"Out": ["elementwise1_output"]}, - attrs={"axis": axis}) - - relu_op1 = OpConfig(type="relu", - inputs={"X": ["elementwise1_output"]}, - outputs={"Out": ["relu1_output"]}, - attrs={}) - - mul_op2 = OpConfig(type="mul", - inputs={ - "X": ["relu1_output"], - "Y": ["mul2_weight"] - }, - outputs={"Out": ["mul2_output"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) - - elt_op2 = OpConfig(type="elementwise_add", - inputs={ - "X": ["mul2_output"], - "Y": ["elementwise2_weight"] - }, - outputs={"Out": ["elementwise2_output"]}, - attrs={"axis": axis}) - - relu_op2 = OpConfig(type="relu", - inputs={"X": ["elementwise2_output"]}, - outputs={"Out": ["relu2_output"]}, - attrs={}) + attrs = [ + {"x_col": x_col, "y_col": y_col}, + {"axis": axis}, + {'batch_size': batch_size, 'dim': dim}, + ] + + mul_op1 = OpConfig( + type="mul", + inputs={"X": ["input_data"], "Y": ["mul1_weight"]}, + outputs={"Out": ["mul1_output"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) + + elt_op1 = OpConfig( + type="elementwise_add", + inputs={"X": ["mul1_output"], "Y": ["elementwise1_weight"]}, + outputs={"Out": ["elementwise1_output"]}, + attrs={"axis": axis}, + ) + + relu_op1 = OpConfig( + type="relu", + inputs={"X": ["elementwise1_output"]}, + outputs={"Out": ["relu1_output"]}, + attrs={}, + ) + + mul_op2 = OpConfig( + type="mul", + inputs={"X": ["relu1_output"], "Y": ["mul2_weight"]}, + outputs={"Out": ["mul2_output"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) + + elt_op2 = OpConfig( + type="elementwise_add", + inputs={"X": ["mul2_output"], "Y": ["elementwise2_weight"]}, + outputs={"Out": ["elementwise2_output"]}, + attrs={"axis": axis}, + ) + + relu_op2 = OpConfig( + type="relu", + inputs={"X": ["elementwise2_output"]}, + outputs={"Out": ["relu2_output"]}, + attrs={}, + ) model_net = [mul_op1, elt_op1, relu_op1, mul_op2, elt_op2, relu_op2] program_config = ProgramConfig( ops=model_net, weights={ - "mul1_weight": - TensorConfig(data_gen=partial(generate_weight, [dim, 32])), - "mul2_weight": - TensorConfig(data_gen=partial(generate_weight, [32, 128])), - "elementwise1_weight": - TensorConfig(data_gen=partial(generate_weight, [32])), - "elementwise2_weight": - TensorConfig(data_gen=partial(generate_weight, [128])) + "mul1_weight": TensorConfig( + data_gen=partial(generate_weight, [dim, 32]) + ), + "mul2_weight": TensorConfig( + data_gen=partial(generate_weight, [32, 128]) + ), + "elementwise1_weight": TensorConfig( + data_gen=partial(generate_weight, [32]) + ), + "elementwise2_weight": TensorConfig( + data_gen=partial(generate_weight, [128]) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["relu2_output"]) + outputs=["relu2_output"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_reshape2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_reshape2_matmul_fuse_pass.py index 7de2491e693..8efe56ed222 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_reshape2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_reshape2_matmul_fuse_pass.py @@ -44,9 +44,10 @@ class TestReshape2MatmulFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape and attr of reshape2 reshape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=10), min_size=2, max_size=2 + ) + ) x_shape = reshape + [1, 1] # 2. Generate attr:transpose_X/transpose_Y/alpha of matmul @@ -56,9 +57,10 @@ class TestReshape2MatmulFusePass(PassAutoScanTest): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -91,17 +93,11 @@ class TestReshape2MatmulFusePass(PassAutoScanTest): "X": ["reshape2_x"], }, shape=reshape, - outputs={ - "Out": ["reshape2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["reshape2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["reshape2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["reshape2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -116,10 +112,7 @@ class TestReshape2MatmulFusePass(PassAutoScanTest): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -152,10 +145,12 @@ class TestReshape2MatmulFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - max_duration=1000, - passes=["gpu_cpu_reshape2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=50, + max_duration=1000, + passes=["gpu_cpu_reshape2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_seq_concat_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_seq_concat_fc_fuse_pass.py index 53dc65f84b5..52153788abe 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_seq_concat_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_seq_concat_fc_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestSeqConcatFcFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -44,83 +43,83 @@ class TestSeqConcatFcFusePass(PassAutoScanTest): def generate_weight(shape): return np.random.random(shape).astype(np.float32) - sequence_expand_op1 = OpConfig(type="sequence_expand", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["seq_exp1_out"]}, - attrs={"ref_level": ref_level}) - - sequence_expand_op2 = OpConfig(type="sequence_expand", - inputs={ - "X": ["input_data1"], - "Y": ["input_data3"] - }, - outputs={"Out": ["seq_exp2_out"]}, - attrs={"ref_level": ref_level}) + sequence_expand_op1 = OpConfig( + type="sequence_expand", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["seq_exp1_out"]}, + attrs={"ref_level": ref_level}, + ) + + sequence_expand_op2 = OpConfig( + type="sequence_expand", + inputs={"X": ["input_data1"], "Y": ["input_data3"]}, + outputs={"Out": ["seq_exp2_out"]}, + attrs={"ref_level": ref_level}, + ) concat_op = OpConfig( type="concat", inputs={"X": ["input_data1", "seq_exp1_out", "seq_exp2_out"]}, outputs={"Out": ["concat_output"]}, - attrs={'axis': axis1}) - - mul_op = OpConfig(type="mul", - inputs={ - "X": ["concat_output"], - "Y": ["mul_weight"] - }, - outputs={"Out": ["mul_out"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) - - elt_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["mul_out"], - "Y": ["elt_weight"] - }, - outputs={"Out": ["elt_out"]}, - attrs={"axis": axis2}) - - act_op = OpConfig(type=act_type, - inputs={"X": ["elt_out"]}, - outputs={"Out": ["act_out"]}, - attrs={ - "use_cudnn": use_cudnn, - "use_mkldnn": use_mkldnn - }) + attrs={'axis': axis1}, + ) + + mul_op = OpConfig( + type="mul", + inputs={"X": ["concat_output"], "Y": ["mul_weight"]}, + outputs={"Out": ["mul_out"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) + + elt_op = OpConfig( + type="elementwise_add", + inputs={"X": ["mul_out"], "Y": ["elt_weight"]}, + outputs={"Out": ["elt_out"]}, + attrs={"axis": axis2}, + ) + + act_op = OpConfig( + type=act_type, + inputs={"X": ["elt_out"]}, + outputs={"Out": ["act_out"]}, + attrs={"use_cudnn": use_cudnn, "use_mkldnn": use_mkldnn}, + ) model_net = [ - sequence_expand_op1, sequence_expand_op2, concat_op, mul_op, elt_op, - act_op + sequence_expand_op1, + sequence_expand_op2, + concat_op, + mul_op, + elt_op, + act_op, ] program_config = ProgramConfig( ops=model_net, weights={ - "mul_weight": - TensorConfig(data_gen=partial(generate_weight, [384, dim])), - "elt_weight": - TensorConfig(data_gen=partial(generate_weight, [dim])) + "mul_weight": TensorConfig( + data_gen=partial(generate_weight, [384, dim]) + ), + "elt_weight": TensorConfig( + data_gen=partial(generate_weight, [dim]) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, - [batch_size, 128]), - lod=[[0, 1]]), - "input_data2": - TensorConfig(data_gen=partial(generate_input, - [batch_size, 128]), - lod=[[0, 1]]), - "input_data3": - TensorConfig(data_gen=partial(generate_input, - [batch_size, 128]), - lod=[[0, 1]]) + "input_data1": TensorConfig( + data_gen=partial(generate_input, [batch_size, 128]), + lod=[[0, 1]], + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, [batch_size, 128]), + lod=[[0, 1]], + ), + "input_data3": TensorConfig( + data_gen=partial(generate_input, [batch_size, 128]), + lod=[[0, 1]], + ), }, - outputs=["act_out"]) + outputs=["act_out"], + ) return program_config @@ -129,15 +128,15 @@ class TestSeqConcatFcFusePass(PassAutoScanTest): yield config, ["fusion_seqexpand_concat_fc"], (1e-5, 1e-5) def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): if program_config.ops[-1].type == "relu": return True return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass output has diff in a specific case. We need to fix it as soon as possible." + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass output has diff in a specific case. We need to fix it as soon as possible.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_seqconv_eltadd_relu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_seqconv_eltadd_relu_fuse_pass.py index ee5bb892b94..08c95df169b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_seqconv_eltadd_relu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_seqconv_eltadd_relu_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestSeqconvEltaddReluFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -41,71 +40,80 @@ class TestSeqconvEltaddReluFusePass(PassAutoScanTest): def generate_weight(shape): return np.random.random(shape).astype(np.float32) - im2sequence_op = OpConfig(type="im2sequence", - inputs={"X": ["input_data"]}, - outputs={"Out": ["seq_out"]}, - attrs={ - "kernels": [6, 1], - "out_stride": [1, 1], - "paddings": [0, 0, 0, 0], - "strides": [1, 1] - }) - - sequence_conv_op = OpConfig(type="sequence_conv", - inputs={ - "X": ["seq_out"], - "Filter": ["conv_weight"] - }, - outputs={"Out": ["conv_out"]}, - attrs={ - "contextLength": contextLength, - "contextStart": contextStart, - "contextStride": contextStride, - "paddingTrainable": paddingTrainable - }) - - elementwise_add_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["conv_out"], - "Y": ["elt_weight"] - }, - outputs={"Out": ["elt_output"]}, - attrs={'axis': axis}) - - relu_op = OpConfig(type="relu", - inputs={"X": ["elt_output"]}, - outputs={"Out": ["relu_output"]}, - attrs={}) + im2sequence_op = OpConfig( + type="im2sequence", + inputs={"X": ["input_data"]}, + outputs={"Out": ["seq_out"]}, + attrs={ + "kernels": [6, 1], + "out_stride": [1, 1], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, + ) + + sequence_conv_op = OpConfig( + type="sequence_conv", + inputs={"X": ["seq_out"], "Filter": ["conv_weight"]}, + outputs={"Out": ["conv_out"]}, + attrs={ + "contextLength": contextLength, + "contextStart": contextStart, + "contextStride": contextStride, + "paddingTrainable": paddingTrainable, + }, + ) + + elementwise_add_op = OpConfig( + type="elementwise_add", + inputs={"X": ["conv_out"], "Y": ["elt_weight"]}, + outputs={"Out": ["elt_output"]}, + attrs={'axis': axis}, + ) + + relu_op = OpConfig( + type="relu", + inputs={"X": ["elt_output"]}, + outputs={"Out": ["relu_output"]}, + attrs={}, + ) model_net = [ - im2sequence_op, sequence_conv_op, elementwise_add_op, relu_op + im2sequence_op, + sequence_conv_op, + elementwise_add_op, + relu_op, ] program_config = ProgramConfig( ops=model_net, weights={ - "conv_weight": - TensorConfig( - data_gen=partial(generate_weight, [768 * - contextLength, 16])), - "elt_weight": - TensorConfig(data_gen=partial(generate_weight, [16])) + "conv_weight": TensorConfig( + data_gen=partial(generate_weight, [768 * contextLength, 16]) + ), + "elt_weight": TensorConfig( + data_gen=partial(generate_weight, [16]) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["relu_output"]) + outputs=["relu_output"], + ) return program_config def sample_predictor_configs(self, program_config): config = self.create_inference_config() - yield config, ["im2sequence", - "fusion_seqconv_eltadd_relu"], (1e-5, 1e-5) + yield config, ["im2sequence", "fusion_seqconv_eltadd_relu"], ( + 1e-5, + 1e-5, + ) def test(self): - self.run_and_statis(quant=False, - passes=["seqconv_eltadd_relu_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["seqconv_eltadd_relu_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_seqpool_cvm_concat_fuse_pass_py.py b/python/paddle/fluid/tests/unittests/ir/inference/test_seqpool_cvm_concat_fuse_pass_py.py index d768b1c23ff..08878c3cd40 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_seqpool_cvm_concat_fuse_pass_py.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_seqpool_cvm_concat_fuse_pass_py.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestSeqpoolCvmConcatFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -47,84 +46,88 @@ class TestSeqpoolCvmConcatFusePass(PassAutoScanTest): def generate_input3(): return np.random.random([1, 768]).astype(np.float32) - im2sequence_op = OpConfig(type="im2sequence", - inputs={"X": ["input_data1"]}, - outputs={"Out": ["seq_out"]}, - attrs={ - "kernels": [6, 1], - "out_stride": [1, 1], - "paddings": [0, 0, 0, 0], - "strides": [1, 1] - }) - - sequence_pool_op1 = OpConfig(type="sequence_pool", - inputs={"X": ["seq_out"]}, - outputs={ - "Out": ["seq_pool1_out"], - "MaxIndex": ["index1_out"] - }, - attrs={ - "is_test": is_test, - "pooltype": pooltype, - "pad_value": pad_value1 - }) - - sequence_pool_op2 = OpConfig(type="sequence_pool", - inputs={"X": ["seq_out"]}, - outputs={ - "Out": ["seq_pool2_out"], - "MaxIndex": ["index2_out"] - }, - attrs={ - "is_test": is_test, - "pooltype": pooltype, - "pad_value": pad_value2 - }) - - sequence_pool_op3 = OpConfig(type="sequence_pool", - inputs={"X": ["seq_out"]}, - outputs={ - "Out": ["seq_pool3_out"], - "MaxIndex": ["index3_out"] - }, - attrs={ - "is_test": is_test, - "pooltype": pooltype, - "pad_value": pad_value3 - }) - - cvm_op1 = OpConfig(type="cvm", - inputs={ - "X": ["seq_pool1_out"], - "CVM": ["input_data2"] - }, - outputs={"Y": ["cvm1_out"]}, - attrs={"use_cvm": use_cvm}) - - cvm_op2 = OpConfig(type="cvm", - inputs={ - "X": ["seq_pool2_out"], - "CVM": ["input_data2"] - }, - outputs={"Y": ["cvm2_out"]}, - attrs={"use_cvm": use_cvm}) - - cvm_op3 = OpConfig(type="cvm", - inputs={ - "X": ["seq_pool3_out"], - "CVM": ["input_data2"] - }, - outputs={"Y": ["cvm3_out"]}, - attrs={"use_cvm": use_cvm}) - - concat_op = OpConfig(type="concat", - inputs={"X": ["cvm1_out", "cvm2_out", "cvm3_out"]}, - outputs={"Out": ["concat_output"]}, - attrs={'axis': axis}) + im2sequence_op = OpConfig( + type="im2sequence", + inputs={"X": ["input_data1"]}, + outputs={"Out": ["seq_out"]}, + attrs={ + "kernels": [6, 1], + "out_stride": [1, 1], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, + ) + + sequence_pool_op1 = OpConfig( + type="sequence_pool", + inputs={"X": ["seq_out"]}, + outputs={"Out": ["seq_pool1_out"], "MaxIndex": ["index1_out"]}, + attrs={ + "is_test": is_test, + "pooltype": pooltype, + "pad_value": pad_value1, + }, + ) + + sequence_pool_op2 = OpConfig( + type="sequence_pool", + inputs={"X": ["seq_out"]}, + outputs={"Out": ["seq_pool2_out"], "MaxIndex": ["index2_out"]}, + attrs={ + "is_test": is_test, + "pooltype": pooltype, + "pad_value": pad_value2, + }, + ) + + sequence_pool_op3 = OpConfig( + type="sequence_pool", + inputs={"X": ["seq_out"]}, + outputs={"Out": ["seq_pool3_out"], "MaxIndex": ["index3_out"]}, + attrs={ + "is_test": is_test, + "pooltype": pooltype, + "pad_value": pad_value3, + }, + ) + + cvm_op1 = OpConfig( + type="cvm", + inputs={"X": ["seq_pool1_out"], "CVM": ["input_data2"]}, + outputs={"Y": ["cvm1_out"]}, + attrs={"use_cvm": use_cvm}, + ) + + cvm_op2 = OpConfig( + type="cvm", + inputs={"X": ["seq_pool2_out"], "CVM": ["input_data2"]}, + outputs={"Y": ["cvm2_out"]}, + attrs={"use_cvm": use_cvm}, + ) + + cvm_op3 = OpConfig( + type="cvm", + inputs={"X": ["seq_pool3_out"], "CVM": ["input_data2"]}, + outputs={"Y": ["cvm3_out"]}, + attrs={"use_cvm": use_cvm}, + ) + + concat_op = OpConfig( + type="concat", + inputs={"X": ["cvm1_out", "cvm2_out", "cvm3_out"]}, + outputs={"Out": ["concat_output"]}, + attrs={'axis': axis}, + ) model_net = [ - im2sequence_op, sequence_pool_op1, sequence_pool_op2, - sequence_pool_op3, cvm_op1, cvm_op2, cvm_op3, concat_op + im2sequence_op, + sequence_pool_op1, + sequence_pool_op2, + sequence_pool_op3, + cvm_op1, + cvm_op2, + cvm_op3, + concat_op, ] program_config = ProgramConfig( @@ -133,9 +136,10 @@ class TestSeqpoolCvmConcatFusePass(PassAutoScanTest): inputs={ "input_data1": TensorConfig(data_gen=partial(generate_input1)), "input_data2": TensorConfig(data_gen=partial(generate_input2)), - "input_data3": TensorConfig(data_gen=partial(generate_input3)) + "input_data3": TensorConfig(data_gen=partial(generate_input3)), }, - outputs=["concat_output"]) + outputs=["concat_output"], + ) return program_config @@ -144,8 +148,9 @@ class TestSeqpoolCvmConcatFusePass(PassAutoScanTest): yield config, ["im2sequence", "fusion_seqpool_cvm_concat"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["seqpool_cvm_concat_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["seqpool_cvm_concat_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_shuffle_channel_detect_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_shuffle_channel_detect_pass.py index 669b1906d2c..5aa03a57f4b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_shuffle_channel_detect_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_shuffle_channel_detect_pass.py @@ -23,7 +23,6 @@ import hypothesis.strategies as st class TestShuffleChannelDetectPass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -46,44 +45,52 @@ class TestShuffleChannelDetectPass(PassAutoScanTest): def generate_reshape2_Input(): return np.random.random(x_shape).astype(np.float32) - reshape2_op1 = OpConfig("reshape2", - inputs={ - "X": ["reshape2_input1"], - }, - outputs={ - "Out": ["reshape2_output1"], - "XShape": ["reshape2_xshape1"] - }, - shape=shape, - input_shape=x_shape) - transpose2_op = OpConfig("transpose2", - inputs={ - "X": ["reshape2_output1"], - }, - outputs={ - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] - }, - axis=axis_v) - reshape2_op2 = OpConfig("reshape2", - inputs={ - "X": ["transpose2_output"], - }, - outputs={ - "Out": ["reshape2_output2"], - "XShape": ["reshape2_xshape2"] - }, - shape=x_shape) + reshape2_op1 = OpConfig( + "reshape2", + inputs={ + "X": ["reshape2_input1"], + }, + outputs={ + "Out": ["reshape2_output1"], + "XShape": ["reshape2_xshape1"], + }, + shape=shape, + input_shape=x_shape, + ) + transpose2_op = OpConfig( + "transpose2", + inputs={ + "X": ["reshape2_output1"], + }, + outputs={ + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + axis=axis_v, + ) + reshape2_op2 = OpConfig( + "reshape2", + inputs={ + "X": ["transpose2_output"], + }, + outputs={ + "Out": ["reshape2_output2"], + "XShape": ["reshape2_xshape2"], + }, + shape=x_shape, + ) ops = [reshape2_op1, transpose2_op, reshape2_op2] program_config = ProgramConfig( ops=ops, inputs={ - "reshape2_input1": - TensorConfig(data_gen=partial(generate_reshape2_Input)), + "reshape2_input1": TensorConfig( + data_gen=partial(generate_reshape2_Input) + ), }, weights={}, - outputs=["reshape2_output2"]) + outputs=["reshape2_output2"], + ) return program_config def sample_predictor_configs(self, program_config): @@ -94,7 +101,8 @@ class TestShuffleChannelDetectPass(PassAutoScanTest): min_subgraph_size=1, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['shuffle_channel'], (1e-5, 1e-5) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_simplify_with_basic_ops_pass_autoscan.py b/python/paddle/fluid/tests/unittests/ir/inference/test_simplify_with_basic_ops_pass_autoscan.py index 6dab206551c..42c5a38a72a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_simplify_with_basic_ops_pass_autoscan.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_simplify_with_basic_ops_pass_autoscan.py @@ -21,47 +21,49 @@ import hypothesis.strategies as st class TestSimplifyWithBasicOpsPassUpscale(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_config(self, draw): - #scale = draw(st.floats(min_value=0.01, max_value=1.0)) - #bias = draw(st.floats(min_value=0.01, max_value=2.0)) - #bias_after_scale = draw(st.booleans()) + # scale = draw(st.floats(min_value=0.01, max_value=1.0)) + # bias = draw(st.floats(min_value=0.01, max_value=2.0)) + # bias_after_scale = draw(st.booleans()) fix_seed = draw(st.booleans()) dropout_implementation = "upscale_in_train" dropout_prob = draw(st.floats(min_value=0.0, max_value=1.0)) seed = draw(st.integers(min_value=0, max_value=512)) x_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=4 + ) + ) is_test = True - dropout_op = OpConfig("dropout", - inputs={"X": ["input_data"]}, - outputs={ - "Out": ["dropout_output"], - "Mask": ["mask"] - }, - fix_seed=fix_seed, - dropout_implementation=dropout_implementation, - dropout_prob=dropout_prob, - seed=seed, - is_test=is_test) - relu_op = OpConfig("relu", - inputs={"X": ["dropout_output"]}, - outputs={"Out": ["relu_out"]}) + dropout_op = OpConfig( + "dropout", + inputs={"X": ["input_data"]}, + outputs={"Out": ["dropout_output"], "Mask": ["mask"]}, + fix_seed=fix_seed, + dropout_implementation=dropout_implementation, + dropout_prob=dropout_prob, + seed=seed, + is_test=is_test, + ) + relu_op = OpConfig( + "relu", + inputs={"X": ["dropout_output"]}, + outputs={"Out": ["relu_out"]}, + ) ops = [dropout_op, relu_op] - program_config = ProgramConfig(ops=ops, - weights={}, - inputs={ - "input_data": - TensorConfig(shape=x_shape), - }, - outputs=["relu_out"]) + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig(shape=x_shape), + }, + outputs=["relu_out"], + ) return program_config @@ -77,18 +79,20 @@ class TestSimplifyWithBasicOpsPassUpscale(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['relu'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=["simplify_with_basic_ops_pass"], - min_success_num=30) + self.run_and_statis( + quant=False, + max_examples=30, + passes=["simplify_with_basic_ops_pass"], + min_success_num=30, + ) class TestSimplifyWithBasicOpsPassDowngrade(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -98,34 +102,37 @@ class TestSimplifyWithBasicOpsPassDowngrade(PassAutoScanTest): dropout_prob = draw(st.floats(min_value=0.0, max_value=1.0)) seed = draw(st.integers(min_value=0, max_value=512)) x_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=4 + ) + ) is_test = True - dropout_op = OpConfig("dropout", - inputs={"X": ["input_data"]}, - outputs={ - "Out": ["dropout_output"], - "Mask": ["mask"] - }, - fix_seed=fix_seed, - dropout_implementation=dropout_implementation, - dropout_prob=dropout_prob, - seed=seed, - is_test=is_test) - relu_op = OpConfig("relu", - inputs={"X": ["dropout_output"]}, - outputs={"Out": ["relu_out"]}) + dropout_op = OpConfig( + "dropout", + inputs={"X": ["input_data"]}, + outputs={"Out": ["dropout_output"], "Mask": ["mask"]}, + fix_seed=fix_seed, + dropout_implementation=dropout_implementation, + dropout_prob=dropout_prob, + seed=seed, + is_test=is_test, + ) + relu_op = OpConfig( + "relu", + inputs={"X": ["dropout_output"]}, + outputs={"Out": ["relu_out"]}, + ) ops = [dropout_op, relu_op] - program_config = ProgramConfig(ops=ops, - weights={}, - inputs={ - "input_data": - TensorConfig(shape=x_shape), - }, - outputs=["relu_out"]) + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig(shape=x_shape), + }, + outputs=["relu_out"], + ) return program_config @@ -141,14 +148,17 @@ class TestSimplifyWithBasicOpsPassDowngrade(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['scale', 'relu'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=["simplify_with_basic_ops_pass"], - min_success_num=30) + self.run_and_statis( + quant=False, + max_examples=30, + passes=["simplify_with_basic_ops_pass"], + min_success_num=30, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_squared_mat_sub_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_squared_mat_sub_fuse_pass.py index 93a6e0fa322..4e2287c9a65 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_squared_mat_sub_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_squared_mat_sub_fuse_pass.py @@ -22,7 +22,6 @@ import hypothesis.strategies as st class TestSquaredMatSubFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -49,110 +48,125 @@ class TestSquaredMatSubFusePass(PassAutoScanTest): else: return np.random.random(shape_y).astype(np.float32) - matmul_op1 = OpConfig(type="matmul", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["matmul1_output"]}, - attrs={ - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha1, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) - - square_op1 = OpConfig(type="square", - inputs={"X": ["matmul1_output"]}, - outputs={"Out": ["square1_output"]}, - attrs={}) - - square_op2 = OpConfig(type="square", - inputs={"X": ["input_data1"]}, - outputs={"Out": ["square2_output"]}, - attrs={}) - - square_op3 = OpConfig(type="square", - inputs={"X": ["input_data2"]}, - outputs={"Out": ["square3_output"]}, - attrs={}) - - matmul_op2 = OpConfig(type="matmul", - inputs={ - "X": ["square2_output"], - "Y": ["square3_output"] - }, - outputs={"Out": ["matmul2_output"]}, - attrs={ - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha2, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) - - elt_sub_op = OpConfig(type="elementwise_sub", - inputs={ - "X": ["square1_output"], - "Y": ["matmul2_output"] - }, - outputs={"Out": ["sub_out"]}, - attrs={"axis": axis1}) + matmul_op1 = OpConfig( + type="matmul", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["matmul1_output"]}, + attrs={ + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha1, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) + + square_op1 = OpConfig( + type="square", + inputs={"X": ["matmul1_output"]}, + outputs={"Out": ["square1_output"]}, + attrs={}, + ) + + square_op2 = OpConfig( + type="square", + inputs={"X": ["input_data1"]}, + outputs={"Out": ["square2_output"]}, + attrs={}, + ) + + square_op3 = OpConfig( + type="square", + inputs={"X": ["input_data2"]}, + outputs={"Out": ["square3_output"]}, + attrs={}, + ) + + matmul_op2 = OpConfig( + type="matmul", + inputs={"X": ["square2_output"], "Y": ["square3_output"]}, + outputs={"Out": ["matmul2_output"]}, + attrs={ + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha2, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) + + elt_sub_op = OpConfig( + type="elementwise_sub", + inputs={"X": ["square1_output"], "Y": ["matmul2_output"]}, + outputs={"Out": ["sub_out"]}, + attrs={"axis": axis1}, + ) if has_str_value: - fill_constant_op = OpConfig(type="fill_constant", - inputs={}, - outputs={"Out": ["constant_out"]}, - attrs={ - "dtype": 5, - "place_type": place_type, - "str_value": str_value, - "value": value, - "shape": shape - }) + fill_constant_op = OpConfig( + type="fill_constant", + inputs={}, + outputs={"Out": ["constant_out"]}, + attrs={ + "dtype": 5, + "place_type": place_type, + "str_value": str_value, + "value": value, + "shape": shape, + }, + ) else: - fill_constant_op = OpConfig(type="fill_constant", - inputs={}, - outputs={"Out": ["constant_out"]}, - attrs={ - "dtype": 5, - "place_type": place_type, - "value": value, - "shape": shape - }) - - elt_mul_op = OpConfig(type="elementwise_mul", - inputs={ - "X": ["sub_out"], - "Y": ["constant_out"] - }, - outputs={"Out": ["mul_out"]}, - attrs={"axis": axis2}) + fill_constant_op = OpConfig( + type="fill_constant", + inputs={}, + outputs={"Out": ["constant_out"]}, + attrs={ + "dtype": 5, + "place_type": place_type, + "value": value, + "shape": shape, + }, + ) + + elt_mul_op = OpConfig( + type="elementwise_mul", + inputs={"X": ["sub_out"], "Y": ["constant_out"]}, + outputs={"Out": ["mul_out"]}, + attrs={"axis": axis2}, + ) model_net = [ - matmul_op1, square_op1, square_op2, square_op3, matmul_op2, - elt_sub_op, fill_constant_op, elt_mul_op + matmul_op1, + square_op1, + square_op2, + square_op3, + matmul_op2, + elt_sub_op, + fill_constant_op, + elt_mul_op, ] program_config = ProgramConfig( ops=model_net, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, "x")), - "input_data2": - TensorConfig(data_gen=partial(generate_input, "y")) + "input_data1": TensorConfig( + data_gen=partial(generate_input, "x") + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, "y") + ), }, - outputs=["mul_out"]) + outputs=["mul_out"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_squeeze2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_squeeze2_matmul_fuse_pass.py index 8deee7c1517..e7ef132f1ab 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_squeeze2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_squeeze2_matmul_fuse_pass.py @@ -44,9 +44,10 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X of squeeze2 x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) # axes of squeeze2 == [2, 3] x_shape += [1, 1] axes = [2, 3] @@ -58,9 +59,10 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -93,17 +95,11 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): "X": ["squeeze2_x"], }, axes=axes, - outputs={ - "Out": ["squeeze2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["squeeze2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["squeeze2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["squeeze2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -118,10 +114,7 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -154,10 +147,12 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - max_duration=1000, - passes=["gpu_cpu_squeeze2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=50, + max_duration=1000, + passes=["gpu_cpu_squeeze2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_transpose_flatten_concat_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_transpose_flatten_concat_fuse_pass.py index 41145be9bca..acdff3d9a86 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_transpose_flatten_concat_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_transpose_flatten_concat_fuse_pass.py @@ -94,14 +94,18 @@ class TestTransposeFlattenConcatFusePass(PassAutoScanTest): if draw(st.booleans()): trans_axis[j], trans_axis[-1] = trans_axis[-1], trans_axis[j] # Generate axis of flatten - flatten_axis = draw(st.integers(min_value=0, - max_value=x_shape_rank - 1)) + flatten_axis = draw( + st.integers(min_value=0, max_value=x_shape_rank - 1) + ) for i in range(times): # Generate x_shape of transpose x_shape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=x_shape_rank, - max_size=x_shape_rank)) + st.lists( + st.integers(min_value=1, max_value=10), + min_size=x_shape_rank, + max_size=x_shape_rank, + ) + ) str_i = str(i) transpose_op = OpConfig( @@ -112,7 +116,7 @@ class TestTransposeFlattenConcatFusePass(PassAutoScanTest): axis=trans_axis, outputs={ "Out": ["trans_out" + str_i], - "XShape": ["trans_shape" + str_i] + "XShape": ["trans_shape" + str_i], }, ) ops.append(transpose_op) @@ -124,7 +128,7 @@ class TestTransposeFlattenConcatFusePass(PassAutoScanTest): axis=flatten_axis, outputs={ "Out": ["flatten2_out" + str_i], - "XShape": ["xshape" + str_i] + "XShape": ["xshape" + str_i], }, ) concat_input.append("flatten2_out" + str_i) @@ -152,9 +156,11 @@ class TestTransposeFlattenConcatFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["transpose_flatten_concat_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["transpose_flatten_concat_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_activation_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_activation_pass.py index 7ba824360ad..0f2a8a97430 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_activation_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_activation_pass.py @@ -24,18 +24,18 @@ from paddle.fluid.core import AnalysisConfig class TensorRTSubgraphPassActivationTest(InferencePassTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) def setUp(self): self.setUpTensorRTParam() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 32, 32], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 32, 32], dtype="float32" + ) act_out = self.append_act(data) out = fluid.layers.batch_norm(act_out, is_test=True) self.feeds = { @@ -51,298 +51,340 @@ class TensorRTSubgraphPassActivationTest(InferencePassTest): use_gpu = True if os.path.exists(self.path + "_opt_cache"): shutil.rmtree(self.path + "_opt_cache") - if self.trt_parameters.precision == AnalysisConfig.Precision.Float32: + if ( + self.trt_parameters.precision + == AnalysisConfig.Precision.Float32 + ): self.check_output_with_option(use_gpu) else: self.check_output_with_option(use_gpu, 1e-3) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLeakyReluTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.leaky_relu(x) class TensorRTSubgraphPassRelu6Test(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.relu6(x) class TensorRTSubgraphPassSoftMaxTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.softmax(x) class TensorRTSubgraphPassSigmoidTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.sigmoid(x) class TensorRTSubgraphPassHardSwishTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.hard_swish(x) class TensorRTSubgraphPassHardSigmoidTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.hard_sigmoid(x) -class TensorRTSubgraphPassHardSwishPluginTest(TensorRTSubgraphPassActivationTest - ): - +class TensorRTSubgraphPassHardSwishPluginTest( + TensorRTSubgraphPassActivationTest +): def append_act(self, x): return fluid.layers.hard_swish(x, threshold=4.0, scale=8.0) class TensorRTSubgraphPassClipTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.clip(x, 0, 1) class TensorRTSubgraphPassTanhTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.tanh(x) class TensorRTSubgraphPassSwishTest(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) def append_act(self, x): return fluid.layers.swish(x) class TensorRTSubgraphPassSwishFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) def append_act(self, x): return fluid.layers.swish(x) class TensorRTSubgraphPassDynamicSwishFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.swish(x) class TensorRTSubgraphPassMishTest(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) def append_act(self, x): return fluid.layers.mish(x) class TensorRTSubgraphPassMishFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) def append_act(self, x): return fluid.layers.mish(x) class TensorRTSubgraphPassDynamicMishFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.mish(x) class TensorRTSubgraphPassPreluAllTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluChannelTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.prelu(x, mode='channel') class TensorRTSubgraphPassPreluElementTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.prelu(x, mode='element') class TensorRTSubgraphPassPreluDynamicTest(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluFp16Test(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluFp16DynamicTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluFp16DynamicSerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassGeluTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.gelu(x) class TensorRTSubgraphPassGeluDynamicTest(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.gelu(x) class TensorRTSubgraphPassGeluFp16Test(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) def append_act(self, x): return fluid.layers.gelu(x) class TensorRTSubgraphPassGeluFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) def append_act(self, x): return fluid.layers.gelu(x) -class TensorRTSubgraphPassGeluFp16DynamicTest(TensorRTSubgraphPassActivationTest - ): - +class TensorRTSubgraphPassGeluFp16DynamicTest( + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.gelu(x) class TensorRTSubgraphPassGeluFp16DynamicSerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.gelu(x) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_affine_channel_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_affine_channel_op.py index eb2360668f5..69e73d43596 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_affine_channel_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_affine_channel_op.py @@ -23,7 +23,6 @@ from paddle.fluid.core import AnalysisConfig class TRTAffineChannelTest(InferencePassTest): - def setUp(self): self.bs = 2 self.channel = 8 @@ -38,7 +37,8 @@ class TRTAffineChannelTest(InferencePassTest): # set min_graph_size to 2, # because affine channel doesn't support nhwc format self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, 2, self.precision, self.serialize, False) + 1 << 30, self.bs, 2, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): if self.data_layout == 'NCHW': @@ -51,13 +51,16 @@ class TRTAffineChannelTest(InferencePassTest): scale = fluid.layers.create_parameter( shape=[self.channel], dtype='float32', - default_initializer=fluid.initializer.Constant(2.)) + default_initializer=fluid.initializer.Constant(2.0), + ) bias = fluid.layers.create_parameter( shape=[self.channel], dtype='float32', - default_initializer=fluid.initializer.Constant(.5)) + default_initializer=fluid.initializer.Constant(0.5), + ) affine_channel_out = fluid.layers.affine_channel( - data, scale=scale, bias=bias, data_layout=self.data_layout) + data, scale=scale, bias=bias, data_layout=self.data_layout + ) out = fluid.layers.batch_norm(affine_channel_out, is_test=True) shape[0] = self.bs @@ -74,7 +77,8 @@ class TRTAffineChannelTest(InferencePassTest): atol = 2e-2 self.check_output_with_option(use_gpu, atol, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build() @@ -82,30 +86,39 @@ class TRTAffineChannelTest(InferencePassTest): def run_test_all(self): precision_opt = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_opt = [False, True] if self.data_layout == 'NCHW': min_shape = [ - self.bs, self.channel, self.height // 2, self.width // 2 + self.bs, + self.channel, + self.height // 2, + self.width // 2, ] max_shape = [self.bs, self.channel, self.height * 2, self.width * 2] opt_shape = [self.bs, self.channel, self.height, self.width] if self.data_layout == 'NHWC': min_shape = [ - self.bs, self.height // 2, self.width // 2, self.channel + self.bs, + self.height // 2, + self.width // 2, + self.channel, ] max_shape = [self.bs, self.height * 2, self.width * 2, self.channel] opt_shape = [self.bs, self.height, self.width, self.channel] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( - {'in': min_shape}, {'in': max_shape}, {'in': opt_shape}, False) + {'in': min_shape}, {'in': max_shape}, {'in': opt_shape}, False + ) dynamic_shape_opt = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_opt, serialize_opt, dynamic_shape_opt): + precision_opt, serialize_opt, dynamic_shape_opt + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -126,7 +139,9 @@ class TRTAffineChannelTest(InferencePassTest): self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( {'in': [self.bs, self.channel, self.height // 2, self.width // 2]}, {'in': [self.bs, self.channel, self.height * 2, self.width * 2]}, - {'in': [self.bs, self.channel, self.height, self.width]}, False) + {'in': [self.bs, self.channel, self.height, self.width]}, + False, + ) self.run_test() def test_nchw_all(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_anchor_generator_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_anchor_generator_op.py index 7e22604a374..cd05c8528bc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_anchor_generator_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_anchor_generator_op.py @@ -22,40 +22,47 @@ from paddle.fluid.core import AnalysisConfig class TRTAnchorGeneratorBaseTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 16 self.height = 32 self.width = 32 - self.anchor_sizes = [64., 128., 256., 512.] - self.aspect_ratios = [.5, 1., 2.] - self.variance = [.1, .1, .2, .2] - self.stride = [8., 8.] + self.anchor_sizes = [64.0, 128.0, 256.0, 512.0] + self.aspect_ratios = [0.5, 1.0, 2.0] + self.variance = [0.1, 0.1, 0.2, 0.2] + self.stride = [8.0, 8.0] self.precision = AnalysisConfig.Precision.Float32 self.serialize = False self.enable_trt = True self.feeds = { - 'data': - np.random.random([self.bs, self.channel, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.height, self.width] + ).astype('float32'), } def build(self): min_graph_size = 3 if self.dynamic_shape_params is not None else 2 self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, min_graph_size, self.precision, self.serialize, - False) + 1 << 30, + self.bs, + min_graph_size, + self.precision, + self.serialize, + False, + ) with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=[-1, self.channel, self.height, self.width], - dtype='float32') + data = fluid.data( + name='data', + shape=[-1, self.channel, self.height, self.width], + dtype='float32', + ) anchor, var = fluid.layers.detection.anchor_generator( data, anchor_sizes=self.anchor_sizes, aspect_ratios=self.aspect_ratios, variance=self.variance, - stride=self.stride) + stride=self.stride, + ) if self.dynamic_shape_params is not None: anchor = fluid.layers.transpose(anchor, [2, 3, 0, 1]) out = fluid.layers.batch_norm(anchor, is_test=True) @@ -69,10 +76,17 @@ class TRTAnchorGeneratorBaseTest(InferencePassTest): def set_dynamic(self): self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( { - 'data': - [self.bs, self.channel, self.height // 2, self.width // 2] - }, {'data': [self.bs, self.channel, self.height, self.width]}, - {'data': [self.bs, self.channel, self.height, self.width]}, False) + 'data': [ + self.bs, + self.channel, + self.height // 2, + self.width // 2, + ] + }, + {'data': [self.bs, self.channel, self.height, self.width]}, + {'data': [self.bs, self.channel, self.height, self.width]}, + False, + ) def test_base(self): self.run_test() @@ -113,7 +127,8 @@ class TRTAnchorGeneratorBaseTest(InferencePassTest): atol = 1e-3 self.check_output_with_option(use_gpu, atol, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_c_allreduce_infer_script.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_c_allreduce_infer_script.py index e90157c928c..d2d34e34cbc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_c_allreduce_infer_script.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_c_allreduce_infer_script.py @@ -39,20 +39,25 @@ def run(op_type, precision): lod_level=data.lod_level, persistable=False, is_data=False, - initializer=paddle.nn.initializer.Constant(value=1.0)) - block.append_op(type=op_type, - inputs={'X': data}, - outputs={'Out': c_data}, - attrs={ - 'ring_id': 0, - 'use_calc_stream': True, - 'use_model_parallel': True - }) + initializer=paddle.nn.initializer.Constant(value=1.0), + ) + block.append_op( + type=op_type, + inputs={'X': data}, + outputs={'Out': c_data}, + attrs={ + 'ring_id': 0, + 'use_calc_stream': True, + 'use_model_parallel': True, + }, + ) out = paddle.static.nn.fc( x=c_data, size=1, weight_attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(value=0.5))) + initializer=paddle.nn.initializer.Constant(value=0.5) + ), + ) mean = paddle.mean(out) exe = paddle.static.Executor(paddle.CPUPlace()) exe.run(startup_program) @@ -68,12 +73,17 @@ def run(op_type, precision): dist_config.enable_dist_model(True) with tempfile.TemporaryDirectory(prefix="allreduce_") as tmpdir: - paddle.static.save_inference_model(os.path.join(tmpdir, "model"), - [data], [mean], - exe, - program=main_program) - config = Config(os.path.join(tmpdir, "model.pdmodel"), - os.path.join(tmpdir, "model.pdiparams")) + paddle.static.save_inference_model( + os.path.join(tmpdir, "model"), + [data], + [mean], + exe, + program=main_program, + ) + config = Config( + os.path.join(tmpdir, "model.pdmodel"), + os.path.join(tmpdir, "model.pdiparams"), + ) config.enable_memory_optim() config.enable_use_gpu(1000, fleet.worker_index()) config.set_dist_config(dist_config) @@ -82,11 +92,14 @@ def run(op_type, precision): max_batch_size=1, min_subgraph_size=1, precision_mode=PrecisionType.Half - if precision == "fp16" else PrecisionType.Int8, + if precision == "fp16" + else PrecisionType.Int8, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({"data": [3, 4]}, {"data": [3, 4]}, - {"data": [3, 4]}) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + {"data": [3, 4]}, {"data": [3, 4]}, {"data": [3, 4]} + ) predictor = create_predictor(config) input_names = predictor.get_input_names() input_tensor = predictor.get_input_handle("data") diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_op.py index 088b548f018..70e62f9eaf6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_op.py @@ -22,29 +22,31 @@ from paddle.fluid.core import AnalysisConfig class TensorRTSubgraphPassConv3dTest(InferencePassTest): - def setUp(self): self.init_params() self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 6, 32, 32], - dtype="float32") - conv_out = fluid.layers.conv3d(input=data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - stride=self.stride, - act=None) + data = fluid.data( + name="data", shape=[-1, 3, 6, 32, 32], dtype="float32" + ) + conv_out = fluid.layers.conv3d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + stride=self.stride, + act=None, + ) self.feeds = { "data": np.random.random([1, 3, 6, 32, 32]).astype("float32"), } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassConv3dTest.TensorRTParam( - 1 << 30, 32, 1, self.precision, self.use_static, False) + 1 << 30, 32, 1, self.precision, self.use_static, False + ) self.fetch_list = [conv_out] def init_params(self): @@ -65,12 +67,13 @@ class TensorRTSubgraphPassConv3dTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) - + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) -class TensorRTSubgraphPassConv3dValidPaddingTest(TensorRTSubgraphPassConv3dTest - ): +class TensorRTSubgraphPassConv3dValidPaddingTest( + TensorRTSubgraphPassConv3dTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -79,7 +82,6 @@ class TensorRTSubgraphPassConv3dValidPaddingTest(TensorRTSubgraphPassConv3dTest class TensorRTSubgraphPassConv3dSamePaddingTest(TensorRTSubgraphPassConv3dTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -88,7 +90,6 @@ class TensorRTSubgraphPassConv3dSamePaddingTest(TensorRTSubgraphPassConv3dTest): class TensorRTSubgraphPassConv3dPaddingTest(TensorRTSubgraphPassConv3dTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -97,7 +98,6 @@ class TensorRTSubgraphPassConv3dPaddingTest(TensorRTSubgraphPassConv3dTest): class TensorRTSubgraphPassConv3dStrideTest(TensorRTSubgraphPassConv3dTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -107,39 +107,49 @@ class TensorRTSubgraphPassConv3dStrideTest(TensorRTSubgraphPassConv3dTest): class DynamicShapeTensorRTSubgraphPassConv3dTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, -1, -1, -1], - dtype="float32") - conv_out = fluid.layers.conv3d(input=data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - stride=self.stride, - act=None) + data = fluid.data( + name="data", shape=[-1, 6, -1, -1, -1], dtype="float32" + ) + conv_out = fluid.layers.conv3d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + stride=self.stride, + act=None, + ) self.feeds = { "data": np.random.random([1, 6, 32, 32, 8]).astype("float32"), } self.enable_trt = True - self.trt_parameters = DynamicShapeTensorRTSubgraphPassConv3dTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = DynamicShapeTensorRTSubgraphPassConv3dTest.DynamicShapeParam( - { - "data": [1, 6, 8, 8, 8], - "conv3d_0.tmp_0": [1, 6, 8, 8, 4], - }, { - "data": [32, 6, 32, 32, 8], - "conv3d_0.tmp_0": [32, 6, 32, 32, 8], - }, { - "data": [16, 6, 16, 16, 8], - "conv3d_0.tmp_0": [16, 6, 16, 16, 8], - }, False) + self.trt_parameters = ( + DynamicShapeTensorRTSubgraphPassConv3dTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) + self.dynamic_shape_params = ( + DynamicShapeTensorRTSubgraphPassConv3dTest.DynamicShapeParam( + { + "data": [1, 6, 8, 8, 8], + "conv3d_0.tmp_0": [1, 6, 8, 8, 4], + }, + { + "data": [32, 6, 32, 32, 8], + "conv3d_0.tmp_0": [32, 6, 32, 32, 8], + }, + { + "data": [16, 6, 16, 16, 8], + "conv3d_0.tmp_0": [16, 6, 16, 16, 8], + }, + False, + ) + ) self.fetch_list = [conv_out] def set_params(self): @@ -155,7 +165,8 @@ class DynamicShapeTensorRTSubgraphPassConv3dTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_transpose_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_transpose_op.py index 0cd4bd1a8e9..9beabe55052 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_transpose_op.py @@ -22,13 +22,12 @@ from paddle.fluid.core import AnalysisConfig class TensorRTSubgraphPassConv3dTransposeTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 4, 4, 32, 32], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 4, 4, 32, 32], dtype="float32" + ) conv_out = fluid.layers.conv3d_transpose( input=data, num_filters=self.conv_num_filters, @@ -38,13 +37,17 @@ class TensorRTSubgraphPassConv3dTransposeTest(InferencePassTest): bias_attr=False, use_cudnn=self.use_cudnn, stride=1, - act=None) + act=None, + ) self.feeds = { "data": np.random.random([1, 4, 4, 32, 32]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassConv3dTransposeTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TensorRTSubgraphPassConv3dTransposeTest.TensorRTParam( + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [conv_out] def set_params(self): @@ -59,12 +62,13 @@ class TensorRTSubgraphPassConv3dTransposeTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassConv3dTransposeSamePaddingTest( - TensorRTSubgraphPassConv3dTransposeTest): - + TensorRTSubgraphPassConv3dTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -74,8 +78,8 @@ class TensorRTSubgraphPassConv3dTransposeSamePaddingTest( class TensorRTSubgraphPassConv3dTransposeMultigroupTest( - TensorRTSubgraphPassConv3dTransposeTest): - + TensorRTSubgraphPassConv3dTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -85,13 +89,12 @@ class TensorRTSubgraphPassConv3dTransposeMultigroupTest( class DynamicShapeTensorRTSubgraphPassConv3dTransposeTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, -1, -1, -1], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, -1, -1, -1], dtype="float32" + ) conv_out = fluid.layers.conv3d_transpose( input=data, num_filters=self.conv_num_filters, @@ -101,24 +104,32 @@ class DynamicShapeTensorRTSubgraphPassConv3dTransposeTest(InferencePassTest): bias_attr=False, use_cudnn=self.use_cudnn, stride=self.stride, - act=None) + act=None, + ) self.feeds = { "data": np.random.random([1, 6, 32, 32, 8]).astype("float32"), } self.enable_trt = True - self.trt_parameters = DynamicShapeTensorRTSubgraphPassConv3dTransposeTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + DynamicShapeTensorRTSubgraphPassConv3dTransposeTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.dynamic_shape_params = DynamicShapeTensorRTSubgraphPassConv3dTransposeTest.DynamicShapeParam( { "data": [1, 6, 8, 8, 8], "conv3d_transpose_0.tmp_0": [1, 6, 8, 8, 1], - }, { + }, + { "data": [32, 6, 32, 32, 8], "conv3d_transpose_0.tmp_0": [32, 6, 64, 64, 16], - }, { + }, + { "data": [16, 6, 16, 16, 8], "conv3d_transpose_0.tmp_0": [16, 6, 16, 16, 8], - }, False) + }, + False, + ) self.fetch_list = [conv_out] def set_params(self): @@ -134,7 +145,8 @@ class DynamicShapeTensorRTSubgraphPassConv3dTransposeTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py index 5e163bce407..90dbed96f90 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py @@ -25,27 +25,29 @@ os.environ['NVIDIA_TF32_OVERRIDE'] = '0' class TensorRTSubgraphPassConvTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") - conv_out = fluid.layers.conv2d(input=data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - act=None) + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + act=None, + ) self.feeds = { "data": np.random.random([1, 6, 64, 64]).astype("float32"), } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassConvTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [conv_out] def set_params(self): @@ -60,11 +62,11 @@ class TensorRTSubgraphPassConvTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassConvValidPaddingTest(TensorRTSubgraphPassConvTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -74,7 +76,6 @@ class TensorRTSubgraphPassConvValidPaddingTest(TensorRTSubgraphPassConvTest): class TensorRTSubgraphPassConvSamePaddingTest(InferencePassTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -84,7 +85,6 @@ class TensorRTSubgraphPassConvSamePaddingTest(InferencePassTest): class TensorRTSubgraphPassDepthwiseConvTest(TensorRTSubgraphPassConvTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -94,7 +94,6 @@ class TensorRTSubgraphPassDepthwiseConvTest(TensorRTSubgraphPassConvTest): class TensorRTSubgraphPassDepthwiseConv2Test(TensorRTSubgraphPassConvTest): - def set_params(self): self.conv_num_filters = 12 self.conv_filter_size = 6 @@ -104,13 +103,12 @@ class TensorRTSubgraphPassDepthwiseConv2Test(TensorRTSubgraphPassConvTest): class TensorRTSubgraphPassConvTransposeTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) conv_out = fluid.layers.conv2d_transpose( input=data, num_filters=self.conv_num_filters, @@ -119,13 +117,17 @@ class TensorRTSubgraphPassConvTransposeTest(InferencePassTest): padding=self.conv_padding, bias_attr=False, use_cudnn=self.use_cudnn, - act=None) + act=None, + ) self.feeds = { "data": np.random.random([1, 6, 64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassConvTransposeTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TensorRTSubgraphPassConvTransposeTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [conv_out] def set_params(self): @@ -140,12 +142,13 @@ class TensorRTSubgraphPassConvTransposeTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassConvTransposeValidPaddingTest( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -155,8 +158,8 @@ class TensorRTSubgraphPassConvTransposeValidPaddingTest( class TensorRTSubgraphPassConvTransposeSamePaddingTest( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -166,8 +169,8 @@ class TensorRTSubgraphPassConvTransposeSamePaddingTest( class TensorRTSubgraphPassConvTransposeMultiGroupTest( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -177,8 +180,8 @@ class TensorRTSubgraphPassConvTransposeMultiGroupTest( class TensorRTSubgraphPassConvTranspose2Test( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 12 self.conv_filter_size = 4 @@ -188,8 +191,8 @@ class TensorRTSubgraphPassConvTranspose2Test( class TensorRTSubgraphPassDepthwiseConvTransposeTest( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 4 @@ -199,42 +202,52 @@ class TensorRTSubgraphPassDepthwiseConvTransposeTest( class DynamicShapeTensorRTSubgraphPassConvTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, -1, -1], - dtype="float32") - conv_out = fluid.layers.conv2d(input=data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - stride=self.stride, - act=None) + data = fluid.data( + name="data", shape=[-1, 6, -1, -1], dtype="float32" + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + stride=self.stride, + act=None, + ) self.feeds = { "data": np.random.random([32, 6, 64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = DynamicShapeTensorRTSubgraphPassConvTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = DynamicShapeTensorRTSubgraphPassConvTest.DynamicShapeParam( - { - "conv2d_0.tmp_0": [1, 6, 8, 8], - "data": [1, 6, 8, 8], - "depthwise_conv2d_0.tmp_0": [1, 6, 8, 8] - }, { - "conv2d_0.tmp_0": [32, 6, 64, 64], - "data": [32, 6, 64, 64], - "depthwise_conv2d_0.tmp_0": [32, 6, 64, 64] - }, { - "conv2d_0.tmp_0": [16, 6, 16, 16], - "data": [16, 6, 16, 16], - "depthwise_conv2d_0.tmp_0": [16, 6, 16, 16] - }, False) + self.trt_parameters = ( + DynamicShapeTensorRTSubgraphPassConvTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) + self.dynamic_shape_params = ( + DynamicShapeTensorRTSubgraphPassConvTest.DynamicShapeParam( + { + "conv2d_0.tmp_0": [1, 6, 8, 8], + "data": [1, 6, 8, 8], + "depthwise_conv2d_0.tmp_0": [1, 6, 8, 8], + }, + { + "conv2d_0.tmp_0": [32, 6, 64, 64], + "data": [32, 6, 64, 64], + "depthwise_conv2d_0.tmp_0": [32, 6, 64, 64], + }, + { + "conv2d_0.tmp_0": [16, 6, 16, 16], + "data": [16, 6, 16, 16], + "depthwise_conv2d_0.tmp_0": [16, 6, 16, 16], + }, + False, + ) + ) self.fetch_list = [conv_out] def set_params(self): @@ -250,12 +263,13 @@ class DynamicShapeTensorRTSubgraphPassConvTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class DynamicShapeTensorRTSubgraphPassDepthwiseConvTransposeTest( - DynamicShapeTensorRTSubgraphPassConvTest): - + DynamicShapeTensorRTSubgraphPassConvTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_quant_dequant_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_quant_dequant_pass.py index fbaf6944be9..f19de2a3bb3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_quant_dequant_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_quant_dequant_pass.py @@ -23,25 +23,26 @@ from paddle.fluid.core import AnalysisConfig class QuantDequantTensorRTSubgraphPassConvTest(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) data_reshape = fluid.layers.reshape(self.data, shape=[1, 4, 14, 14]) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') label_shape = fluid.layers.reshape(self.label, shape=[1, 1, 1]) - conv_out = fluid.layers.conv2d(input=data_reshape, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - act=None) + conv_out = fluid.layers.conv2d( + input=data_reshape, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + act=None, + ) if self.conv_padding == [1, 1]: cout = fluid.layers.reshape(conv_out, shape=[1, 1, 10816]) elif self.conv_padding == 'VALID': @@ -58,21 +59,25 @@ class QuantDequantTensorRTSubgraphPassConvTest(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = QuantDequantTensorRTSubgraphPassConvTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + self.trt_parameters = ( + QuantDequantTensorRTSubgraphPassConvTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -86,17 +91,17 @@ class QuantDequantTensorRTSubgraphPassConvTest(QuantDequantTest): def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1e-1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class QuantDequantTensorRTSubgraphPassConvValidPaddingTest( - QuantDequantTensorRTSubgraphPassConvTest): - + QuantDequantTensorRTSubgraphPassConvTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -106,8 +111,8 @@ class QuantDequantTensorRTSubgraphPassConvValidPaddingTest( class QuantDequantTensorRTSubgraphPassConvSamePaddingTest( - QuantDequantTensorRTSubgraphPassConvTest): - + QuantDequantTensorRTSubgraphPassConvTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -117,8 +122,8 @@ class QuantDequantTensorRTSubgraphPassConvSamePaddingTest( class QuantDequantTensorRTSubgraphPassDWConvTest( - QuantDequantTensorRTSubgraphPassConvTest): - + QuantDequantTensorRTSubgraphPassConvTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -128,25 +133,26 @@ class QuantDequantTensorRTSubgraphPassDWConvTest( class DynamicShapeQuantDequantTensorRTSubgraphPassConvTest(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) data_reshape = fluid.layers.reshape(self.data, shape=[1, 4, 14, 14]) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') label_shape = fluid.layers.reshape(self.label, shape=[1, 1, 1]) - conv_out = fluid.layers.conv2d(input=data_reshape, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - act=None) + conv_out = fluid.layers.conv2d( + input=data_reshape, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + act=None, + ) cout = fluid.layers.reshape(conv_out, shape=[1, 1, 10816]) result = fluid.layers.relu(cout) loss = fluid.layers.cross_entropy(input=result, label=label_shape) @@ -156,41 +162,49 @@ class DynamicShapeQuantDequantTensorRTSubgraphPassConvTest(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = DynamicShapeQuantDequantTensorRTSubgraphPassConvTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + self.trt_parameters = ( + DynamicShapeQuantDequantTensorRTSubgraphPassConvTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) self.dynamic_shape_params = DynamicShapeQuantDequantTensorRTSubgraphPassConvTest.DynamicShapeParam( { "conv2d_0.tmp_0": [1, 4, 14, 14], "data": [1, 28, 28], "depthwise_conv2d_0.tmp_0": [1, 4, 14, 14], "reshape2_0.tmp_0": [1, 4, 14, 14], - "reshape2_2.tmp_0": [1, 1, 10816] - }, { + "reshape2_2.tmp_0": [1, 1, 10816], + }, + { "conv2d_0.tmp_0": [4, 4, 14, 14], "data": [4, 28, 28], "depthwise_conv2d_0.tmp_0": [4, 4, 14, 14], "reshape2_0.tmp_0": [4, 4, 14, 14], - "reshape2_2.tmp_0": [1, 1, 43264] - }, { + "reshape2_2.tmp_0": [1, 1, 43264], + }, + { "conv2d_0.tmp_0": [1, 4, 14, 14], "data": [1, 28, 28], "depthwise_conv2d_0.tmp_0": [1, 4, 14, 14], "reshape2_0.tmp_0": [1, 4, 14, 14], - "reshape2_2.tmp_0": [1, 1, 10816] - }, False) + "reshape2_2.tmp_0": [1, 1, 10816], + }, + False, + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -204,23 +218,22 @@ class DynamicShapeQuantDequantTensorRTSubgraphPassConvTest(QuantDequantTest): def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1e-1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class QuantDequantTensorRTSubgraphPassConvTransposeTest(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) data_reshape = fluid.layers.reshape(self.data, shape=[1, 4, 14, 14]) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') label_shape = fluid.layers.reshape(self.label, shape=[1, 1, 1]) @@ -232,7 +245,8 @@ class QuantDequantTensorRTSubgraphPassConvTransposeTest(QuantDequantTest): padding=self.conv_padding, bias_attr=False, use_cudnn=self.use_cudnn, - act=None) + act=None, + ) if self.conv_padding == [1, 1]: cout = fluid.layers.reshape(conv_out, shape=[1, 1, 14400]) elif self.conv_padding == 'VALID': @@ -249,21 +263,25 @@ class QuantDequantTensorRTSubgraphPassConvTransposeTest(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = QuantDequantTensorRTSubgraphPassConvTransposeTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + self.trt_parameters = ( + QuantDequantTensorRTSubgraphPassConvTransposeTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -277,17 +295,17 @@ class QuantDequantTensorRTSubgraphPassConvTransposeTest(QuantDequantTest): def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1e-1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class QuantDequantTensorRTSubgraphPassConvTransValidPaddingTest( - QuantDequantTensorRTSubgraphPassConvTransposeTest): - + QuantDequantTensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -297,8 +315,8 @@ class QuantDequantTensorRTSubgraphPassConvTransValidPaddingTest( class QuantDequantTensorRTSubgraphPassConvTransSamePaddingTest( - QuantDequantTensorRTSubgraphPassConvTransposeTest): - + QuantDequantTensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -308,8 +326,8 @@ class QuantDequantTensorRTSubgraphPassConvTransSamePaddingTest( class QuantDequantTensorRTSubgraphPassTransDWConvTest( - QuantDequantTensorRTSubgraphPassConvTransposeTest): - + QuantDequantTensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_activation.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_activation.py index 49367f78e2d..c9bf048a044 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_activation.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_activation.py @@ -22,12 +22,10 @@ from typing import Any, Dict, List class TrtConvertActivationTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): if dims == 1: return np.random.random([32]).astype(np.float32) @@ -41,11 +39,19 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): for dims in [1, 2, 3, 4]: for batch in [1, 4]: for op_type in [ - "relu", "sigmoid", "tanh", "relu6", "elu", "selu", - "softsign", "stanh", "thresholded_relu", "softplus" + "relu", + "sigmoid", + "tanh", + "relu6", + "elu", + "selu", + "softsign", + "stanh", + "thresholded_relu", + "softplus", ]: # few samples to reduce time - #for beta in [-0.2, 0.5, 0.67, 3]: + # for beta in [-0.2, 0.5, 0.67, 3]: # for alpha in [-0.2, 0.5, 0.67, 3]: for beta in [0.67]: for alpha in [0.67]: @@ -62,33 +68,34 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): if op_type == "softplus": dics = [{"beta": beta}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dims, batch, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -131,19 +138,23 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_affine_channel.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_affine_channel.py index 34b033b4b7c..a9ba3b11f81 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_affine_channel.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_affine_channel.py @@ -22,12 +22,10 @@ from typing import Any, Dict, List class TrtConvertAffineChannelTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch, dims, attrs: List[Dict[str, Any]]): if dims == 2: return np.ones([batch, 64]).astype(np.float32) @@ -50,42 +48,45 @@ class TrtConvertAffineChannelTest(TrtLayerAutoScanTest): self.dims = dims dics = [{"data_layout": data_layout}] - ops_config = [{ - "op_type": "affine_channel", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale"], - "Bias": ["bias"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "affine_channel", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale"], + "Bias": ["bias"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "scale": - TensorConfig( - data_gen=partial(generate_weight1, dims, dics)), - "bias": - TensorConfig( - data_gen=partial(generate_weight1, dims, dics)) + "scale": TensorConfig( + data_gen=partial(generate_weight1, dims, dics) + ), + "bias": TensorConfig( + data_gen=partial(generate_weight1, dims, dics) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, dims, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, batch, dims, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 2: self.dynamic_shape.min_input_shape = {"input_data": [1, 32]} @@ -132,19 +133,23 @@ class TrtConvertAffineChannelTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_anchor_generator.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_anchor_generator.py index b3e58c204f3..32cb6b64514 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_anchor_generator.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_anchor_generator.py @@ -22,60 +22,66 @@ from typing import Any, Dict, List class TrtConvertAnchorGeneratorTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch, attrs: List[Dict[str, Any]]): return np.random.random([batch, 3, 64, 64]).astype(np.float32) for batch in [1, 2, 4]: for anchor_sizes in [[64.0, 128.0, 256.0, 512.0]]: for aspect_ratios in [[0.5, 1, 2], [0.4, 1.2, 3]]: - for variances in [[1.0, 1.0, 1.0, 1.0], - [0.5, 1.0, 0.5, 1.0]]: + for variances in [ + [1.0, 1.0, 1.0, 1.0], + [0.5, 1.0, 0.5, 1.0], + ]: for stride in [[16.0, 16.0], [16.0, 32.0]]: for offset in [0.5, 0.8]: - dics = [{ - "anchor_sizes": anchor_sizes, - "aspect_ratios": aspect_ratios, - "variances": variances, - "stride": stride, - "offset": offset - }] - - ops_config = [{ - "op_type": "anchor_generator", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Anchors": ["output_anchors"], - "Variances": ["output_variances"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "anchor_sizes": anchor_sizes, + "aspect_ratios": aspect_ratios, + "variances": variances, + "stride": stride, + "offset": offset, + } + ] + + ops_config = [ + { + "op_type": "anchor_generator", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": { + "Anchors": ["output_anchors"], + "Variances": ["output_variances"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, batch, dics + ) + ) }, outputs=[ - "output_anchors", "output_variances" - ]) + "output_anchors", + "output_variances", + ], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -100,19 +106,23 @@ class TrtConvertAnchorGeneratorTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_arg_max.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_arg_max.py index 3d8d492e891..a1913257146 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_arg_max.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_arg_max.py @@ -22,7 +22,6 @@ from typing import List class TrtConvertArgMaxTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: input_shape = program_config.inputs["arg_max_input"].shape axis = program_config.ops[0].attrs["axis"] @@ -33,7 +32,6 @@ class TrtConvertArgMaxTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input(rank, batch): dims = [batch] for i in range(rank - 1): @@ -48,36 +46,37 @@ class TrtConvertArgMaxTest(TrtLayerAutoScanTest): self.rank = rank flatten = False dtype = 2 - ops_config = [{ - "op_type": "arg_max", - "op_inputs": { - "X": ["arg_max_input"] - }, - "op_outputs": { - "Out": ["arg_max_out"] - }, - "op_attrs": { - "axis": axis, - "keepdims": keepdims, - "flatten": flatten, - "dtype": dtype + ops_config = [ + { + "op_type": "arg_max", + "op_inputs": {"X": ["arg_max_input"]}, + "op_outputs": {"Out": ["arg_max_out"]}, + "op_attrs": { + "axis": axis, + "keepdims": keepdims, + "flatten": flatten, + "dtype": dtype, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "arg_max_input": - TensorConfig(data_gen=partial( - generate_input, rank, batch)) + "arg_max_input": TensorConfig( + data_gen=partial( + generate_input, rank, batch + ) + ) }, - outputs=["arg_max_out"]) + outputs=["arg_max_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.rank == 3: self.dynamic_shape.min_input_shape = { @@ -117,19 +116,23 @@ class TrtConvertArgMaxTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_batch_norm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_batch_norm.py index 88d13c062a5..806ead9bc19 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_batch_norm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_batch_norm.py @@ -22,12 +22,10 @@ from typing import Any, Dict, List class TrtConvertBatchNormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: if attrs[0]['data_layout'] == "NCHW": @@ -62,88 +60,119 @@ class TrtConvertBatchNormTest(TrtLayerAutoScanTest): for momentum in [0.9, 0.8]: self.num_input = num_input self.dims = dims - dics = [{ - "epsilon": epsilon, - "data_layout": data_layout, - "momentum": momentum, - "is_test": True, - "trainable_statistics": False - }, {}] - dics_intput = [{ - "X": ["batch_norm_input"], - "Bias": ["Bias"], - "Mean": ["Mean"], - "Scale": ["Scale"], - "Variance": ["Variance"], - "MomentumTensor": ["MomentumTensor"] - }, { - "X": ["batch_norm_input"], - "Bias": ["Bias"], - "Mean": ["Mean"], - "Scale": ["Scale"], - "Variance": ["Variance"] - }] - dics_intputs = [{ - "Bias": - TensorConfig(data_gen=partial( - generate_bias, dics, batch)), - "Mean": - TensorConfig(data_gen=partial( - generate_mean, dics, batch)), - "Scale": - TensorConfig(data_gen=partial( - generate_scale, dics, batch)), - "Variance": - TensorConfig(data_gen=partial( - generate_variance, dics, batch)), - "MomentumTensor": - TensorConfig(data_gen=partial( - generate_MomentumTensor, dics, batch)), - }, { - "Bias": - TensorConfig(data_gen=partial( - generate_bias, dics, batch)), - "Mean": - TensorConfig(data_gen=partial( - generate_mean, dics, batch)), - "Scale": - TensorConfig(data_gen=partial( - generate_scale, dics, batch)), - "Variance": - TensorConfig(data_gen=partial( - generate_variance, dics, batch)) - }] - ops_config = [{ - "op_type": - "batch_norm", - "op_inputs": - dics_intput[num_input], - "op_outputs": { - "Y": ["batch_norm_out"], - "MeanOut": ["Mean"], - "VarianceOut": ["Variance"], - "SavedMean": ["SavedMean"], - "SavedVariance": ["SavedVariance"] + dics = [ + { + "epsilon": epsilon, + "data_layout": data_layout, + "momentum": momentum, + "is_test": True, + "trainable_statistics": False, + }, + {}, + ] + dics_intput = [ + { + "X": ["batch_norm_input"], + "Bias": ["Bias"], + "Mean": ["Mean"], + "Scale": ["Scale"], + "Variance": ["Variance"], + "MomentumTensor": ["MomentumTensor"], + }, + { + "X": ["batch_norm_input"], + "Bias": ["Bias"], + "Mean": ["Mean"], + "Scale": ["Scale"], + "Variance": ["Variance"], }, - "op_attrs": - dics[0] - }] + ] + dics_intputs = [ + { + "Bias": TensorConfig( + data_gen=partial( + generate_bias, dics, batch + ) + ), + "Mean": TensorConfig( + data_gen=partial( + generate_mean, dics, batch + ) + ), + "Scale": TensorConfig( + data_gen=partial( + generate_scale, dics, batch + ) + ), + "Variance": TensorConfig( + data_gen=partial( + generate_variance, dics, batch + ) + ), + "MomentumTensor": TensorConfig( + data_gen=partial( + generate_MomentumTensor, + dics, + batch, + ) + ), + }, + { + "Bias": TensorConfig( + data_gen=partial( + generate_bias, dics, batch + ) + ), + "Mean": TensorConfig( + data_gen=partial( + generate_mean, dics, batch + ) + ), + "Scale": TensorConfig( + data_gen=partial( + generate_scale, dics, batch + ) + ), + "Variance": TensorConfig( + data_gen=partial( + generate_variance, dics, batch + ) + ), + }, + ] + ops_config = [ + { + "op_type": "batch_norm", + "op_inputs": dics_intput[num_input], + "op_outputs": { + "Y": ["batch_norm_out"], + "MeanOut": ["Mean"], + "VarianceOut": ["Variance"], + "SavedMean": ["SavedMean"], + "SavedVariance": ["SavedVariance"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights=dics_intputs[num_input], inputs={ - "batch_norm_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)) + "batch_norm_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ) }, - outputs=["batch_norm_out"]) + outputs=["batch_norm_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: if attrs[0]['data_layout'] == "NCHW": @@ -202,29 +231,35 @@ class TrtConvertBatchNormTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(program_config.weights) == 5: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT MomentumTensor NOT SUPPORT") + self.add_skip_case( + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "INPUT MomentumTensor NOT SUPPORT", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bilinear_interp_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bilinear_interp_v2.py index 83559d4fe86..5015e7e36be 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bilinear_interp_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bilinear_interp_v2.py @@ -22,7 +22,6 @@ import unittest class TrtConvertBilinearInterpV2Test(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -33,13 +32,13 @@ class TrtConvertBilinearInterpV2Test(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 3, 64, 64]).astype(np.float32) def generate_input2(attrs: List[Dict[str, Any]]): - return np.random.uniform(low=0.5, high=6.0, - size=(2)).astype("float32") + return np.random.uniform(low=0.5, high=6.0, size=(2)).astype( + "float32" + ) for data_layout in ["NCHW", "NHWC"]: for scale_y in [2.0, -1.0, 0.0]: @@ -47,48 +46,55 @@ class TrtConvertBilinearInterpV2Test(TrtLayerAutoScanTest): scale = [scale_y, scale_x] for out_h in [32, 64, 128, 192]: for out_w in [32, 64]: - dics = [{ - "data_layout": data_layout, - "interp_method": "bilinear", - "align_corners": False, - "align_mode": 0, - "scale": scale, - "out_h": out_h, - "out_w": out_w - }] - - ops_config = [{ - "op_type": "bilinear_interp_v2", - "op_inputs": { - "X": ["input_data"], - "Scale": ["input_scale"] - }, - "op_outputs": { - "Out": ["bilinear_interp_v2_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_layout": data_layout, + "interp_method": "bilinear", + "align_corners": False, + "align_mode": 0, + "scale": scale, + "out_h": out_h, + "out_w": out_w, + } + ] + + ops_config = [ + { + "op_type": "bilinear_interp_v2", + "op_inputs": { + "X": ["input_data"], + "Scale": ["input_scale"], + }, + "op_outputs": { + "Out": [ + "bilinear_interp_v2_output_data" + ] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "input_scale": - TensorConfig( - data_gen=partial(generate_input2, dics)) + "input_scale": TensorConfig( + data_gen=partial(generate_input2, dics) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["bilinear_interp_v2_output_data"]) + outputs=["bilinear_interp_v2_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 64, 64]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -110,19 +116,23 @@ class TrtConvertBilinearInterpV2Test(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bmm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bmm.py index 54a2cffe3ad..6a7aedc9c9e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bmm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bmm.py @@ -23,9 +23,7 @@ import os class TrtConvertBmmTest_dynamic(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -35,49 +33,50 @@ class TrtConvertBmmTest_dynamic(TrtLayerAutoScanTest): input1_shape = [batch, 350, 75] input2_shape = [batch, 75, 25] dics = [{}] - ops_config = [{ - "op_type": "bmm", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "bmm", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial(generate_input, input1_shape) + ), + "input2_data": TensorConfig( + data_gen=partial(generate_input, input2_shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input1_data": [10, 350, 75], - "input2_data": [10, 75, 25] + "input2_data": [10, 75, 25], } self.dynamic_shape.max_input_shape = { "input1_data": [100, 350, 75], - "input2_data": [100, 75, 25] + "input2_data": [100, 75, 25], } self.dynamic_shape.opt_input_shape = { "input1_data": [15, 350, 75], - "input2_data": [15, 75, 25] + "input2_data": [15, 75, 25], } def clear_dynamic_shape(): @@ -98,25 +97,29 @@ class TrtConvertBmmTest_dynamic(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # The output has little diff between gpu and trt in CI-Windows-Inference tol_fp32 = 1e-4 tol_half = 1e-4 - if (os.name == 'nt'): + if os.name == 'nt': tol_fp32 = 1e-2 tol_half = 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), tol_fp32 + attrs, True + ), tol_fp32 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), tol_half + attrs, True + ), tol_half def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_c_allreduce.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_c_allreduce.py index a9f1964e49a..964234220cf 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_c_allreduce.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_c_allreduce.py @@ -19,14 +19,13 @@ import paddle class TestDistTRT(unittest.TestCase): - def setUp(self): self.init_case() self.script = "test_trt_c_allreduce_infer_script.py" def init_case(self): self.op_type = "c_allreduce_sum" - self.target_value = 4. + self.target_value = 4.0 self.precision = "fp16" def test_run(self): @@ -45,14 +44,13 @@ class TestDistTRT(unittest.TestCase): class TestMin(TestDistTRT): - def init_case(self): self.op_type = "c_allreduce_min" - self.target_value = 2. + self.target_value = 2.0 self.precision = "int8" -#class TestMax(TestDistTRT): +# class TestMax(TestDistTRT): # # def init_case(self): # self.op_type = "c_allreduce_max" @@ -60,7 +58,7 @@ class TestMin(TestDistTRT): # self.precision = "fp16" # # -#class TestProd(TestDistTRT): +# class TestProd(TestDistTRT): # # def init_case(self): # self.op_type = "c_allreduce_prod" diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_cast.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_cast.py index 5c915db7250..76b46313f95 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_cast.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_cast.py @@ -22,7 +22,6 @@ from typing import List class TrtConvertCastTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -31,14 +30,13 @@ class TrtConvertCastTest(TrtLayerAutoScanTest): return False if attrs[0]['in_dtype'] in [4, 5] and attrs[0]['out_dtype'] == 4: return False - if attrs[0]['in_dtype'] not in [ - 2, 4, 5 - ] or attrs[0]['out_dtype'] not in [2, 4, 5]: + if attrs[0]['in_dtype'] not in [2, 4, 5] or attrs[0][ + 'out_dtype' + ] not in [2, 4, 5]: return False return True def sample_program_configs(self): - def generate_input(type): if type == 0: return np.ones([1, 3, 64, 64]).astype(np.bool) @@ -53,32 +51,32 @@ class TrtConvertCastTest(TrtLayerAutoScanTest): for out_dtype in [0, 2, 4, 5, 6]: dics = [{"in_dtype": in_dtype, "out_dtype": out_dtype}] - ops_config = [{ - "op_type": "cast", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["cast_output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "cast", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["cast_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, in_dtype)) + "input_data": TensorConfig( + data_gen=partial(generate_input, in_dtype) + ) }, - outputs=["cast_output_data"]) + outputs=["cast_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 64, 64]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -100,19 +98,23 @@ class TrtConvertCastTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_clip.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_clip.py index ad1dce6cdb2..18d5adb284b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_clip.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_clip.py @@ -22,12 +22,10 @@ import unittest class TrtConvertClipTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([32]).astype(np.float32) @@ -46,52 +44,52 @@ class TrtConvertClipTest(TrtLayerAutoScanTest): for dims in [1, 2, 3, 4]: for batch in [1, 4]: - for op_inputs in [{ - "X": ["input_data"] - }, { - "X": ["input_data"], - "Min": ["Min_"], - "Max": ["Max_"] - }]: + for op_inputs in [ + {"X": ["input_data"]}, + {"X": ["input_data"], "Min": ["Min_"], "Max": ["Max_"]}, + ]: self.input_num = len(op_inputs) self.dims = dims - dics = [{ - "min": np.random.uniform(1, 10), - "max": np.random.uniform(10, 20) - }, { - "op_inputs": op_inputs - }] - ops_config = [{ - "op_type": "clip", - "op_inputs": op_inputs, - "op_outputs": { - "Out": ["output_data"] + dics = [ + { + "min": np.random.uniform(1, 10), + "max": np.random.uniform(10, 20), }, - "op_attrs": dics[0] - }] + {"op_inputs": op_inputs}, + ] + ops_config = [ + { + "op_type": "clip", + "op_inputs": op_inputs, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "Min_": - TensorConfig( - data_gen=partial(generate_weight1, dics)), - "Max_": - TensorConfig( - data_gen=partial(generate_weight2, dics)) + "Min_": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + "Max_": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dims, batch, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs(self, program_config): - def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -135,19 +133,23 @@ class TrtConvertClipTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_concat.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_concat.py index af4d479f8f2..46a7b82ef4d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_concat.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_concat.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertConcatTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,14 +30,13 @@ class TrtConvertConcatTest(TrtLayerAutoScanTest): attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if len(inputs['concat_input1'].shape) <= attrs[0]['axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -79,58 +77,83 @@ class TrtConvertConcatTest(TrtLayerAutoScanTest): self.num_input = num_input self.dims = dims dics = [{"axis": axis}, {}] - dics_intput = [{ - "X": - ["concat_input1", "concat_input2", "concat_input3"], - "AxisTensor": ["AxisTensor"], - }, { - "X": - ["concat_input1", "concat_input2", "concat_input3"] - }] - dics_inputs = [{ - "concat_input1": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)), - "concat_input2": - TensorConfig( - data_gen=partial(generate_input2, dics, batch)), - "concat_input3": - TensorConfig( - data_gen=partial(generate_input3, dics, batch)), - "AxisTensor": - TensorConfig( - data_gen=partial(generate_weight1, dics)) - }, { - "concat_input1": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)), - "concat_input2": - TensorConfig( - data_gen=partial(generate_input2, dics, batch)), - "concat_input3": - TensorConfig( - data_gen=partial(generate_input3, dics, batch)) - }] - ops_config = [{ - "op_type": "concat", - "op_inputs": dics_intput[num_input], - "op_outputs": { - "Out": ["concat_output"] + dics_intput = [ + { + "X": [ + "concat_input1", + "concat_input2", + "concat_input3", + ], + "AxisTensor": ["AxisTensor"], + }, + { + "X": [ + "concat_input1", + "concat_input2", + "concat_input3", + ] + }, + ] + dics_inputs = [ + { + "concat_input1": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "concat_input2": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), + "concat_input3": TensorConfig( + data_gen=partial( + generate_input3, dics, batch + ) + ), + "AxisTensor": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + }, + { + "concat_input1": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "concat_input2": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), + "concat_input3": TensorConfig( + data_gen=partial( + generate_input3, dics, batch + ) + ), }, - "op_attrs": dics[0] - }] + ] + ops_config = [ + { + "op_type": "concat", + "op_inputs": dics_intput[num_input], + "op_outputs": {"Out": ["concat_output"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs=dics_inputs[num_input], - outputs=["concat_output"]) + outputs=["concat_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.num_input == 0: if self.dims == 4: @@ -138,76 +161,76 @@ class TrtConvertConcatTest(TrtLayerAutoScanTest): "concat_input1": [1, 3, 24, 24], "concat_input2": [1, 3, 24, 24], "concat_input3": [1, 3, 24, 24], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 3, 48, 48], "concat_input2": [4, 3, 48, 48], "concat_input3": [4, 3, 48, 48], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 3, 24, 24], "concat_input2": [1, 3, 24, 24], "concat_input3": [1, 3, 24, 24], - "AxisTensor": [1] + "AxisTensor": [1], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "concat_input1": [1, 3, 24], "concat_input2": [1, 3, 24], "concat_input3": [1, 3, 24], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 12, 48], "concat_input2": [4, 12, 48], "concat_input3": [4, 12, 48], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 3, 24], "concat_input2": [1, 3, 24], "concat_input3": [1, 3, 24], - "AxisTensor": [1] + "AxisTensor": [1], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "concat_input1": [1, 24], "concat_input2": [1, 24], "concat_input3": [1, 24], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 48], "concat_input2": [4, 48], "concat_input3": [4, 48], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 24], "concat_input2": [1, 24], "concat_input3": [1, 24], - "AxisTensor": [1] + "AxisTensor": [1], } elif self.dims == 1: self.dynamic_shape.min_input_shape = { "concat_input1": [24], "concat_input2": [24], "concat_input3": [24], - "AxisTensor": [0] + "AxisTensor": [0], } self.dynamic_shape.max_input_shape = { "concat_input1": [48], "concat_input2": [48], "concat_input3": [48], - "AxisTensor": [0] + "AxisTensor": [0], } self.dynamic_shape.opt_input_shape = { "concat_input1": [24], "concat_input2": [24], "concat_input3": [24], - "AxisTensor": [0] + "AxisTensor": [0], } elif self.num_input == 1: if self.dims == 4: @@ -219,60 +242,60 @@ class TrtConvertConcatTest(TrtLayerAutoScanTest): self.dynamic_shape.max_input_shape = { "concat_input1": [4, 3, 48, 48], "concat_input2": [4, 3, 48, 48], - "concat_input3": [4, 3, 48, 48] + "concat_input3": [4, 3, 48, 48], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 3, 24, 24], "concat_input2": [1, 3, 24, 24], - "concat_input3": [1, 3, 24, 24] + "concat_input3": [1, 3, 24, 24], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "concat_input1": [1, 3, 24], "concat_input2": [1, 3, 24], - "concat_input3": [1, 3, 24] + "concat_input3": [1, 3, 24], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 12, 48], "concat_input2": [4, 12, 48], - "concat_input3": [4, 12, 48] + "concat_input3": [4, 12, 48], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 3, 24], "concat_input2": [1, 3, 24], - "concat_input3": [1, 3, 24] + "concat_input3": [1, 3, 24], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "concat_input1": [1, 24], "concat_input2": [1, 24], - "concat_input3": [1, 24] + "concat_input3": [1, 24], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 48], "concat_input2": [4, 48], - "concat_input3": [4, 48] + "concat_input3": [4, 48], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 24], "concat_input2": [1, 24], - "concat_input3": [1, 24] + "concat_input3": [1, 24], } elif self.dims == 1: self.dynamic_shape.min_input_shape = { "concat_input1": [24], "concat_input2": [24], - "concat_input3": [24] + "concat_input3": [24], } self.dynamic_shape.max_input_shape = { "concat_input1": [48], "concat_input2": [48], - "concat_input3": [48] + "concat_input3": [48], } self.dynamic_shape.opt_input_shape = { "concat_input1": [24], "concat_input2": [24], - "concat_input3": [24] + "concat_input3": [24], } def clear_dynamic_shape(): @@ -296,29 +319,33 @@ class TrtConvertConcatTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(program_config.inputs) == 4: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT AxisTensor NOT SUPPORT") + self.add_skip_case( + teller1, SkipReasons.TRT_NOT_SUPPORT, "INPUT AxisTensor NOT SUPPORT" + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d.py index 0682477357e..364d8a1b6bb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d.py @@ -23,7 +23,6 @@ from program_config import TensorConfig, ProgramConfig class TrtConvertConv2dTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,14 +30,17 @@ class TrtConvertConv2dTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[0] * 10 < 7000: if attrs[0]['padding_algorithm'] == 'SAME' and ( - attrs[0]['strides'][0] > 1 or attrs[0]['strides'][1] > 1): + attrs[0]['strides'][0] > 1 or attrs[0]['strides'][1] > 1 + ): return False return True @@ -47,8 +49,12 @@ class TrtConvertConv2dTest(TrtLayerAutoScanTest): self.trt_param.workspace_size = 1073741824 def generate_input1(batch, attrs: List[Dict[str, Any]]): - return np.ones([batch, attrs[0]['groups'] * 3, 64, 64]).astype( - np.float32) / 4 + return ( + np.ones([batch, attrs[0]['groups'] * 3, 64, 64]).astype( + np.float32 + ) + / 4 + ) def generate_weight1(attrs: List[Dict[str, Any]]): return np.random.random([9, 3, 3, 3]).astype(np.float32) - 0.5 @@ -71,73 +77,82 @@ class TrtConvertConv2dTest(TrtLayerAutoScanTest): data_format_options, ] - for batch, strides, paddings, groups, padding_algorithm, dilations, data_format in itertools.product( - *configurations): - - attrs = [{ - "data_fromat": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides, - "data_format": data_format, - }, {}] - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["conv_output_data"] + for ( + batch, + strides, + paddings, + groups, + padding_algorithm, + dilations, + data_format, + ) in itertools.product(*configurations): + + attrs = [ + { + "data_fromat": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + "data_format": data_format, }, - "op_attrs": attrs[0] - }, { - "op_type": "relu", - "op_inputs": { - "X": ["conv_output_data"] + {}, + ] + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": {"Output": ["conv_output_data"]}, + "op_attrs": attrs[0], }, - "op_outputs": { - "Out": ["output_data"] + { + "op_type": "relu", + "op_inputs": {"X": ["conv_output_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": attrs[1], }, - "op_attrs": attrs[1] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight1, attrs)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1, attrs) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, attrs)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, attrs) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): input_groups = attrs[0]['groups'] * 3 self.dynamic_shape.min_input_shape = { "input_data": [1, input_groups, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, input_groups, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, input_groups, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } def clear_dynamic_shape(): @@ -156,25 +171,31 @@ class TrtConvertConv2dTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-2, 1e-2) + attrs, False + ), (1e-2, 1e-2) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-2, 1e-2) + attrs, True + ), (1e-2, 1e-2) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_fusion.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_fusion.py index 420b3a5cfaa..ab3af8e24aa 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_fusion.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_fusion.py @@ -23,7 +23,6 @@ from trt_layer_auto_scan_test import TrtLayerAutoScanTest class TrtConvertConv2dFusionTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,8 +30,10 @@ class TrtConvertConv2dFusionTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False if attrs[0]['groups'] <= 1: @@ -41,7 +42,8 @@ class TrtConvertConv2dFusionTest(TrtLayerAutoScanTest): ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[0] * 10 < 7000: if attrs[0]['padding_algorithm'] == 'SAME' and ( - attrs[0]['strides'][0] > 1 or attrs[0]['strides'][1] > 1): + attrs[0]['strides'][0] > 1 or attrs[0]['strides'][1] > 1 + ): return False return True @@ -50,8 +52,9 @@ class TrtConvertConv2dFusionTest(TrtLayerAutoScanTest): self.trt_param.workspace_size = 1073741824 def generate_input1(batch, attrs: List[Dict[str, Any]]): - return np.ones([batch, attrs[0]['groups'] * 3, 64, - 64]).astype(np.float32) + return np.ones([batch, attrs[0]['groups'] * 3, 64, 64]).astype( + np.float32 + ) def generate_weight1(attrs: List[Dict[str, Any]]): return np.random.random([24, 3, 3, 3]).astype(np.float32) @@ -77,77 +80,87 @@ class TrtConvertConv2dFusionTest(TrtLayerAutoScanTest): data_format_options, ] - for (batch, strides, paddings, groups, padding_algorithm, dilations, - data_format) in product(*configurations): - - attrs = [{ - "strides": strides, - "paddings": paddings, - "groups": groups, - "padding_algorithm": padding_algorithm, - "dilations": dilations, - "data_format": data_format, - }, { - "axis": 1 - }] - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["conv_output_data"] + for ( + batch, + strides, + paddings, + groups, + padding_algorithm, + dilations, + data_format, + ) in product(*configurations): + + attrs = [ + { + "strides": strides, + "paddings": paddings, + "groups": groups, + "padding_algorithm": padding_algorithm, + "dilations": dilations, + "data_format": data_format, }, - "op_attrs": attrs[0] - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["conv_output_data"], - "Y": ["elementwise_weight"] + {"axis": 1}, + ] + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": {"Output": ["conv_output_data"]}, + "op_attrs": attrs[0], }, - "op_outputs": { - "Out": ["output_data"] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["conv_output_data"], + "Y": ["elementwise_weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": attrs[1], }, - "op_attrs": attrs[1] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight1, attrs)), - "elementwise_weight": - TensorConfig(data_gen=partial(generate_weight2, attrs)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1, attrs) + ), + "elementwise_weight": TensorConfig( + data_gen=partial(generate_weight2, attrs) + ), }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, attrs)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, attrs) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): input_groups = attrs[0]['groups'] * 3 self.dynamic_shape.min_input_shape = { "input_data": [1, input_groups, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [2, input_groups, 64, 64], - "output_data": [2, 24, 64, 64] + "output_data": [2, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, input_groups, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } def clear_dynamic_shape(): @@ -166,25 +179,31 @@ class TrtConvertConv2dFusionTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_transpose.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_transpose.py index 3103a105de2..bb0f3cfc3a8 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_transpose.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_transpose.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertConv2dTransposeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -30,8 +29,10 @@ class TrtConvertConv2dTransposeTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False if inputs['input_data'].shape[1] != weights['conv2d_weight'].shape[0]: @@ -54,12 +55,13 @@ class TrtConvertConv2dTransposeTest(TrtLayerAutoScanTest): def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): if attrs[0]['groups'] == 1: - return np.random.random([num_channels, num_channels, 3, - 3]).astype(np.float32) + return np.random.random( + [num_channels, num_channels, 3, 3] + ).astype(np.float32) else: return np.random.random( - [num_channels, int(num_channels / 2), 3, - 3]).astype(np.float32) + [num_channels, int(num_channels / 2), 3, 3] + ).astype(np.float32) for num_channels in [2, 4, 6]: for batch in [1, 4]: @@ -67,99 +69,113 @@ class TrtConvertConv2dTransposeTest(TrtLayerAutoScanTest): for paddings in [[0, 3], [1, 2, 3, 4]]: for groups in [2]: for padding_algorithm in [ - 'EXPLICIT', 'SAME', 'VALID' + 'EXPLICIT', + 'SAME', + 'VALID', ]: for dilations in [[2, 2], [1, 2]]: for data_format in ['NCHW']: self.num_channels = num_channels - dics = [{ - "data_fromat": data_format, - "dilations": dilations, - "padding_algorithm": - padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides, - "data_format": data_format, - "output_size": [], - "output_padding": [] - }] - - ops_config = [{ - "op_type": "conv2d_transpose", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_fromat": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + "data_format": data_format, + "output_size": [], + "output_padding": [], + } + ] + + ops_config = [ + { + "op_type": "conv2d_transpose", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": { + "Output": ["output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config( - ops_config) + ops_config + ) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial( - generate_weight1, - num_channels, dics)) + "conv2d_weight": TensorConfig( + data_gen=partial( + generate_weight1, + num_channels, + dics, + ) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, - num_channels, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, + batch, + num_channels, + dics, + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.num_channels == 2: self.dynamic_shape.min_input_shape = { "input_data": [1, 2, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, 2, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, 2, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } elif self.num_channels == 4: self.dynamic_shape.min_input_shape = { "input_data": [1, 4, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, 4, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, 4, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } else: self.dynamic_shape.min_input_shape = { "input_data": [1, 6, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, 6, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, 6, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } def clear_dynamic_shape(): @@ -178,10 +194,12 @@ class TrtConvertConv2dTransposeTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # self.trt_param.precision = paddle_infer.PrecisionType.Int8 # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, False), (1e-5, 1e-5) @@ -190,24 +208,26 @@ class TrtConvertConv2dTransposeTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) # self.trt_param.precision = paddle_infer.PrecisionType.Int8 # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, True), (1e-5, 1e-5) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.trt_param.precision == paddle_infer.PrecisionType.Int8: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "When precisionType is int8 without relu op, output is different between Trt and Paddle." + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "When precisionType is int8 without relu op, output is different between Trt and Paddle.", ) def test(self): @@ -221,7 +241,6 @@ class TrtConvertConv2dTransposeTest(TrtLayerAutoScanTest): # Special case class TrtConvertConv2dTransposeTest2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[2] * 10 < 7000: @@ -241,49 +260,52 @@ class TrtConvertConv2dTransposeTest2(TrtLayerAutoScanTest): batch = 1 self.num_channels = num_channels - dics = [{ - "data_fromat": 'NCHW', - "dilations": [1, 1], - "padding_algorithm": 'EXPLICIT', - "groups": 1, - "paddings": [1, 1], - "strides": [2, 2], - "output_padding": [1, 1], - "output_size": [], - }] - - ops_config = [{ - "op_type": "conv2d_transpose", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_fromat": 'NCHW', + "dilations": [1, 1], + "padding_algorithm": 'EXPLICIT', + "groups": 1, + "paddings": [1, 1], + "strides": [2, 2], + "output_padding": [1, 1], + "output_size": [], + } + ] + + ops_config = [ + { + "op_type": "conv2d_transpose", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig( - data_gen=partial(generate_weight1, num_channels, dics)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1, num_channels, dics) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, batch, - num_channels, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, num_channels, dics) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 128, 20, 30], @@ -311,19 +333,23 @@ class TrtConvertConv2dTransposeTest2(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-4 + attrs, False + ), 1e-4 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e0, 1e-3) + attrs, False + ), (1e0, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-4 + attrs, True + ), 1e-4 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e0, 1e-3) + attrs, True + ), (1e0, 1e-3) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv3d_transpose.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv3d_transpose.py index a19405cdec9..d0edb4fba0e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv3d_transpose.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv3d_transpose.py @@ -23,7 +23,6 @@ from typing import Any, Dict, List # Special case class TrtConvertConv3dTransposeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[2] * 10 < 8400: @@ -37,56 +36,60 @@ class TrtConvertConv3dTransposeTest(TrtLayerAutoScanTest): return np.ones([batch, num_channels, 4, 20, 30]).astype(np.float32) def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): - return np.random.random([num_channels, 64, 3, 3, - 3]).astype(np.float32) + return np.random.random([num_channels, 64, 3, 3, 3]).astype( + np.float32 + ) num_channels = 128 batch = 1 # in_channels self.num_channels = num_channels - dics = [{ - "data_fromat": 'NCHW', - "dilations": [1, 1, 1], - "padding_algorithm": 'EXPLICIT', - "groups": 1, - "paddings": [1, 1, 1], - "strides": [2, 2, 2], - "output_padding": [1, 1, 1], - "output_size": [], - }] - - ops_config = [{ - "op_type": "conv3d_transpose", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv3d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_fromat": 'NCHW', + "dilations": [1, 1, 1], + "padding_algorithm": 'EXPLICIT', + "groups": 1, + "paddings": [1, 1, 1], + "strides": [2, 2, 2], + "output_padding": [1, 1, 1], + "output_size": [], + } + ] + + ops_config = [ + { + "op_type": "conv3d_transpose", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv3d_weight"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv3d_weight": - TensorConfig( - data_gen=partial(generate_weight1, num_channels, dics)) + "conv3d_weight": TensorConfig( + data_gen=partial(generate_weight1, num_channels, dics) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, batch, - num_channels, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, num_channels, dics) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 128, 4, 20, 30], @@ -114,13 +117,15 @@ class TrtConvertConv3dTransposeTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_deformable_conv.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_deformable_conv.py index f50f1704875..e8fb7142b4d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_deformable_conv.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_deformable_conv.py @@ -22,7 +22,6 @@ import unittest class TrtConvertDeformableConvTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -30,114 +29,158 @@ class TrtConvertDeformableConvTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['filter_data'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['filter_data'].shape[1] * attrs[0]['groups'] + ): return False return True def sample_program_configs(self): - - def compute_output_size(input_size: List[int], kernel_sizes: List[int], - attrs: List[Dict[str, Any]]): + def compute_output_size( + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): strides = attrs[0]['strides'] paddings = attrs[0]['paddings'] dilations = attrs[0]['dilations'] output_size = [] - for i, k, s, p, d in zip(input_size, kernel_sizes, strides, - paddings, dilations): + for i, k, s, p, d in zip( + input_size, kernel_sizes, strides, paddings, dilations + ): k = d * (k - 1) + 1 output_size.append((i + 2 * p - k) // s + 1) return output_size - def generate_input1(batch: int, input_size: List[int], - kernel_sizes: List[int], attrs: List[Dict[str, - Any]]): + def generate_input1( + batch: int, + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): return np.random.random([batch, 3] + input_size).astype(np.float32) - def generate_offset1(batch: int, input_size: List[int], - kernel_sizes: List[int], attrs: List[Dict[str, - Any]]): + def generate_offset1( + batch: int, + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): output_size = compute_output_size(input_size, kernel_sizes, attrs) - return np.random.random([batch, 2 * np.prod(kernel_sizes)] + - output_size).astype(np.float32) - - def generate_mask1(batch: int, input_size: List[int], - kernel_sizes: List[int], attrs: List[Dict[str, - Any]]): + return np.random.random( + [batch, 2 * np.prod(kernel_sizes)] + output_size + ).astype(np.float32) + + def generate_mask1( + batch: int, + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): output_size = compute_output_size(input_size, kernel_sizes, attrs) - return np.random.random([batch, np.prod(kernel_sizes)] + - output_size).astype(np.float32) - - def generate_filter1(batch: int, input_size: List[int], - kernel_sizes: List[int], attrs: List[Dict[str, - Any]]): + return np.random.random( + [batch, np.prod(kernel_sizes)] + output_size + ).astype(np.float32) + + def generate_filter1( + batch: int, + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): return np.random.random([6, 3] + kernel_sizes).astype(np.float32) for batch in [ - 1, + 1, ]: for input_size in [[32, 32]]: for kernel_sizes in [[3, 3]]: for strides in [[1, 1], [2, 2]]: for paddings in [[1, 1], [0, 2]]: for groups in [ - 1, + 1, ]: for dilations in [[1, 1], [2, 2]]: - dics = [{ - "strides": strides, - "paddings": paddings, - "groups": groups, - "dilations": dilations, - "deformable_groups": 1, - "im2col_step": 1 - }] - - ops_config = [{ - "op_type": "deformable_conv", - "op_inputs": { - "Input": ["input_data"], - "Offset": ["offset_data"], - "Mask": ["mask_data"], - "Filter": ["filter_data"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "strides": strides, + "paddings": paddings, + "groups": groups, + "dilations": dilations, + "deformable_groups": 1, + "im2col_step": 1, + } + ] + + ops_config = [ + { + "op_type": "deformable_conv", + "op_inputs": { + "Input": ["input_data"], + "Offset": ["offset_data"], + "Mask": ["mask_data"], + "Filter": ["filter_data"], + }, + "op_outputs": { + "Output": ["output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "filter_data": - TensorConfig(data_gen=partial( - generate_filter1, batch, input_size, - kernel_sizes, dics)) + "filter_data": TensorConfig( + data_gen=partial( + generate_filter1, + batch, + input_size, + kernel_sizes, + dics, + ) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, input_size, - kernel_sizes, dics)), - "offset_data": - TensorConfig(data_gen=partial( - generate_offset1, batch, input_size, - kernel_sizes, dics)), - "mask_data": - TensorConfig(data_gen=partial( - generate_mask1, batch, input_size, - kernel_sizes, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, + batch, + input_size, + kernel_sizes, + dics, + ) + ), + "offset_data": TensorConfig( + data_gen=partial( + generate_offset1, + batch, + input_size, + kernel_sizes, + dics, + ) + ), + "mask_data": TensorConfig( + data_gen=partial( + generate_mask1, + batch, + input_size, + kernel_sizes, + dics, + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def clear_dynamic_shape(): self.dynamic_shape.min_input_shape = {} self.dynamic_shape.max_input_shape = {} @@ -158,7 +201,8 @@ class TrtConvertDeformableConvTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 def test(self): self.trt_param.workspace_size = 1 << 28 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d.py index 90afa7fc1f0..8fd03ae830a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d.py @@ -23,7 +23,6 @@ import paddle.inference as paddle_infer class TrtConvertDepthwiseConv2dTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,8 +30,10 @@ class TrtConvertDepthwiseConv2dTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False return True @@ -65,61 +66,72 @@ class TrtConvertDepthwiseConv2dTest(TrtLayerAutoScanTest): data_format_options, ] - for (batch, strides, paddings, groups, padding_algorithm, dilations, - data_format) in itertools.product(*configurations): - attrs = [{ - "strides": strides, - "paddings": paddings, - "groups": groups, - "padding_algorithm": padding_algorithm, - "dilations": dilations, - "data_fromat": data_format, - }] - - ops_config = [{ - "op_type": "depthwise_conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": attrs[0] - }] + for ( + batch, + strides, + paddings, + groups, + padding_algorithm, + dilations, + data_format, + ) in itertools.product(*configurations): + attrs = [ + { + "strides": strides, + "paddings": paddings, + "groups": groups, + "padding_algorithm": padding_algorithm, + "dilations": dilations, + "data_fromat": data_format, + } + ] + + ops_config = [ + { + "op_type": "depthwise_conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": attrs[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight1, attrs)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1, attrs) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, attrs)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, attrs) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): groups = attrs[0]['groups'] self.dynamic_shape.min_input_shape = { "input_data": [1, groups, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, groups, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, groups, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } def clear_dynamic_shape(): @@ -139,35 +151,44 @@ class TrtConvertDepthwiseConv2dTest(TrtLayerAutoScanTest): self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num(), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) self.trt_param.precision = paddle_infer.PrecisionType.Int8 - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num(), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) self.trt_param.precision = paddle_infer.PrecisionType.Int8 - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[0].attrs[ - 'padding_algorithm'] == "SAME" or program_config.ops[ - 0].attrs['padding_algorithm'] == "VALID": + if ( + program_config.ops[0].attrs['padding_algorithm'] == "SAME" + or program_config.ops[0].attrs['padding_algorithm'] == "VALID" + ): return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "When padding_algorithm is 'SAME' or 'VALID', Trt dose not support. In this case, trt build error is caused by scale op." + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "When padding_algorithm is 'SAME' or 'VALID', Trt dose not support. In this case, trt build error is caused by scale op.", ) def teller2(program_config, predictor_config): @@ -176,8 +197,9 @@ class TrtConvertDepthwiseConv2dTest(TrtLayerAutoScanTest): return False self.add_skip_case( - teller2, SkipReasons.TRT_NOT_IMPLEMENTED, - "When precisionType is int8 without relu op, output is different between Trt and Paddle." + teller2, + SkipReasons.TRT_NOT_IMPLEMENTED, + "When precisionType is int8 without relu op, output is different between Trt and Paddle.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py index 417849ded55..345d546ae64 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py @@ -22,7 +22,6 @@ import unittest class TrtConvertDepthwiseConv2dTransposeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -30,8 +29,10 @@ class TrtConvertDepthwiseConv2dTransposeTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False if inputs['input_data'].shape[1] != weights['conv2d_weight'].shape[1]: @@ -53,12 +54,14 @@ class TrtConvertDepthwiseConv2dTransposeTest(TrtLayerAutoScanTest): self.trt_param.workspace_size = 1073741824 def generate_input1(batch, attrs: List[Dict[str, Any]]): - return np.ones([batch, attrs[0]['groups'], 64, - 64]).astype(np.float32) + return np.ones([batch, attrs[0]['groups'], 64, 64]).astype( + np.float32 + ) def generate_weight1(attrs: List[Dict[str, Any]]): - return np.random.random([attrs[0]['groups'], 1, 3, - 3]).astype(np.float32) + return np.random.random([attrs[0]['groups'], 1, 3, 3]).astype( + np.float32 + ) for batch in [1, 2, 4]: for strides in [[1, 1], [2, 2], [1, 2]]: @@ -68,62 +71,71 @@ class TrtConvertDepthwiseConv2dTransposeTest(TrtLayerAutoScanTest): for dilations in [[1, 1], [2, 2], [1, 2]]: for data_format in ['NCHW']: - dics = [{ - "data_fromat": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides, - "data_format": data_format, - "output_size": [], - "output_padding": [] - }] - - ops_config = [{ - "op_type": "conv2d_transpose", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_fromat": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + "data_format": data_format, + "output_size": [], + "output_padding": [], + } + ] + + ops_config = [ + { + "op_type": "conv2d_transpose", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": { + "Output": ["output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial( - generate_weight1, dics)) + "conv2d_weight": TensorConfig( + data_gen=partial( + generate_weight1, dics + ) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, batch, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, attrs[0]['groups'], 32, 32], - "output_data": [1, attrs[0]['groups'], 32, 32] + "output_data": [1, attrs[0]['groups'], 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, attrs[0]['groups'], 64, 64], - "output_data": [4, attrs[0]['groups'], 64, 64] + "output_data": [4, attrs[0]['groups'], 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, attrs[0]['groups'], 64, 64], - "output_data": [1, attrs[0]['groups'], 64, 64] + "output_data": [1, attrs[0]['groups'], 64, 64], } def clear_dynamic_shape(): @@ -142,10 +154,12 @@ class TrtConvertDepthwiseConv2dTransposeTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # self.trt_param.precision = paddle_infer.PrecisionType.Int8 # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, False), (1e-5, 1e-5) @@ -154,24 +168,26 @@ class TrtConvertDepthwiseConv2dTransposeTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) # self.trt_param.precision = paddle_infer.PrecisionType.Int8 # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, True), (1e-5, 1e-5) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.trt_param.precision == paddle_infer.PrecisionType.Int8: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "When precisionType is int8 without relu op, output is different between Trt and Paddle." + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "When precisionType is int8 without relu op, output is different between Trt and Paddle.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_dropout.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_dropout.py index 3f843c773cb..91b8380d7d6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_dropout.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_dropout.py @@ -22,12 +22,10 @@ from typing import Any, Dict, List class TrtConvertDropoutTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([64]).astype(np.float32) @@ -42,47 +40,57 @@ class TrtConvertDropoutTest(TrtLayerAutoScanTest): for batch in [1, 2, 4]: for fix_seed in [False, True]: for dropout_implementation in [ - "downgrade_in_infer", "upscale_in_train" + "downgrade_in_infer", + "upscale_in_train", ]: for dropout_prob in [np.random.random()]: for seed in [0, 64, 128, 512]: self.dims = dims - dics = [{ - "fix_seed": fix_seed, - "dropout_implementation": - dropout_implementation, - "dropout_prob": dropout_prob, - "seed": seed, - "is_test": True - }] - - ops_config = [{ - "op_type": "dropout", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["dropout_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "fix_seed": fix_seed, + "dropout_implementation": dropout_implementation, + "dropout_prob": dropout_prob, + "seed": seed, + "is_test": True, + } + ] + + ops_config = [ + { + "op_type": "dropout", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": { + "Out": ["dropout_output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, + dims, + batch, + dics, + ) + ) }, - outputs=["dropout_output_data"]) + outputs=["dropout_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -128,19 +136,23 @@ class TrtConvertDropoutTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_elementwise.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_elementwise.py index f9d7f0b9079..36990554755 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_elementwise.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_elementwise.py @@ -24,12 +24,10 @@ from typing import List # This is the special test case with weight including batch dimension # I don't want to mess up the code written by others, so I wrote a class specifically class TrtConvertElementwiseTest_one_input_special_case0(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -39,44 +37,50 @@ class TrtConvertElementwiseTest_one_input_special_case0(TrtLayerAutoScanTest): for batch in [1, 4]: for shape in [[batch, 32, 16, 32]]: for op_type in [ - "elementwise_add", "elementwise_mul", "elementwise_sub", - "elementwise_div", "elementwise_pow", "elementwise_min", - "elementwise_max" + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in [-1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"], - "Y": ["weight"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data"], + "Y": ["weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "weight": - TensorConfig(data_gen=partial(generate_weight)) + "weight": TensorConfig( + data_gen=partial(generate_weight) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)), + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. if self.dims == 4: @@ -106,19 +110,23 @@ class TrtConvertElementwiseTest_one_input_special_case0(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-5, 1e-5) + attrs, True + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass @@ -130,12 +138,10 @@ class TrtConvertElementwiseTest_one_input_special_case0(TrtLayerAutoScanTest): # This is the special test case class TrtConvertElementwiseTest_one_input_special_case1(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -144,44 +150,47 @@ class TrtConvertElementwiseTest_one_input_special_case1(TrtLayerAutoScanTest): for shape in [[32]]: for op_type in [ - "elementwise_add", "elementwise_mul", "elementwise_sub", - "elementwise_div", "elementwise_pow", "elementwise_min", - "elementwise_max" + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in [-1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"], - "Y": ["weight"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": {"X": ["input_data"], "Y": ["weight"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "weight": - TensorConfig(data_gen=partial(generate_weight)) + "weight": TensorConfig( + data_gen=partial(generate_weight) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)), + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [32]} self.dynamic_shape.max_input_shape = {"input_data": [64]} @@ -205,19 +214,23 @@ class TrtConvertElementwiseTest_one_input_special_case1(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-5, 1e-5) + attrs, True + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass @@ -228,12 +241,10 @@ class TrtConvertElementwiseTest_one_input_special_case1(TrtLayerAutoScanTest): class TrtConvertElementwiseTest_one_input(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -241,47 +252,57 @@ class TrtConvertElementwiseTest_one_input(TrtLayerAutoScanTest): return np.random.randn(32).astype(np.float32) for batch in [1, 4]: - for shape in [[32], [batch, 32], [batch, 32, 32], - [batch, 32, 16, 32]]: + for shape in [ + [32], + [batch, 32], + [batch, 32, 32], + [batch, 32, 16, 32], + ]: for op_type in [ - "elementwise_add", "elementwise_mul", "elementwise_sub", - "elementwise_div", "elementwise_pow", "elementwise_min", - "elementwise_max" + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in [-1 if len(shape) == 1 else 1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"], - "Y": ["weight"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data"], + "Y": ["weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "weight": - TensorConfig(data_gen=partial(generate_weight)) + "weight": TensorConfig( + data_gen=partial(generate_weight) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)), + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. if self.dims == 1: @@ -325,19 +346,23 @@ class TrtConvertElementwiseTest_one_input(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-5, 1e-5) + attrs, True + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass @@ -348,108 +373,112 @@ class TrtConvertElementwiseTest_one_input(TrtLayerAutoScanTest): class TrtConvertElementwiseTest_two_input_without_broadcast( - TrtLayerAutoScanTest): - + TrtLayerAutoScanTest +): def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) for shape in [[4], [4, 32], [2, 64, 32], [1, 8, 16, 32]]: for op_type in [ - "elementwise_add", "elementwise_mul", "elementwise_sub", - "elementwise_div", "elementwise_pow", "elementwise_min", - "elementwise_max" + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in [0, -1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data1"], - "Y": ["input_data2"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data1"], + "Y": ["input_data2"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig( - data_gen=partial(generate_input, shape)), - "input_data2": - TensorConfig( - data_gen=partial(generate_input, shape)) + "input_data1": TensorConfig( + data_gen=partial(generate_input, shape) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = { "input_data1": [1], - "input_data2": [1] + "input_data2": [1], } self.dynamic_shape.max_input_shape = { "input_data1": [128], - "input_data2": [128] + "input_data2": [128], } self.dynamic_shape.opt_input_shape = { "input_data1": [32], - "input_data2": [32] + "input_data2": [32], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "input_data1": [1, 4], - "input_data2": [1, 4] + "input_data2": [1, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [128, 256], - "input_data2": [128, 256] + "input_data2": [128, 256], } self.dynamic_shape.opt_input_shape = { "input_data1": [32, 64], - "input_data2": [32, 64] + "input_data2": [32, 64], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "input_data1": [1, 4, 4], - "input_data2": [1, 4, 4] + "input_data2": [1, 4, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [128, 128, 256], - "input_data2": [128, 128, 256] + "input_data2": [128, 128, 256], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 64, 64], - "input_data2": [2, 64, 64] + "input_data2": [2, 64, 64], } elif self.dims == 4: self.dynamic_shape.min_input_shape = { "input_data1": [1, 4, 4, 4], - "input_data2": [1, 4, 4, 4] + "input_data2": [1, 4, 4, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [8, 128, 64, 128], - "input_data2": [8, 128, 64, 128] + "input_data2": [8, 128, 64, 128], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 64, 32, 32], - "input_data2": [2, 64, 32, 32] + "input_data2": [2, 64, 32, 32], } def clear_dynamic_shape(): @@ -470,10 +499,12 @@ class TrtConvertElementwiseTest_two_input_without_broadcast( clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) @@ -491,7 +522,6 @@ class TrtConvertElementwiseTest_two_input_without_broadcast( class TrtConvertElementwiseTest_two_input_with_broadcast(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs if len(inputs['input_data1'].shape) != len(inputs['input_data2'].shape): @@ -500,7 +530,6 @@ class TrtConvertElementwiseTest_two_input_with_broadcast(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -512,8 +541,12 @@ class TrtConvertElementwiseTest_two_input_with_broadcast(TrtLayerAutoScanTest): input2_shape5_list = [[32], [2, 1, 32], [4, 1, 1, 32]] input2_shape6_list = [[1, 32], [1, 32], [1, 1, 1, 32]] input2_shape_list = [ - input2_shape1_list, input2_shape2_list, input2_shape3_list, - input2_shape4_list, input2_shape5_list, input2_shape6_list + input2_shape1_list, + input2_shape2_list, + input2_shape3_list, + input2_shape4_list, + input2_shape5_list, + input2_shape6_list, ] axis1_list = [[-1], [1, -1], [1, -1]] axis2_list = [[-1], [0], [0]] @@ -522,8 +555,12 @@ class TrtConvertElementwiseTest_two_input_with_broadcast(TrtLayerAutoScanTest): axis5_list = [[-1, 1], [-1, 0], [-1, 0]] axis6_list = [[-1, 0], [-1, 1], [-1, 0]] axis_list = [ - axis1_list, axis2_list, axis3_list, axis4_list, axis5_list, - axis6_list + axis1_list, + axis2_list, + axis3_list, + axis4_list, + axis5_list, + axis6_list, ] for i in range(3): @@ -531,66 +568,75 @@ class TrtConvertElementwiseTest_two_input_with_broadcast(TrtLayerAutoScanTest): for j in range(6): input2_shape = input2_shape_list[j][i] for op_type in [ - "elementwise_add", - "elementwise_mul", - "elementwise_sub", - "elementwise_div", - "elementwise_pow", - "elementwise_min", - "elementwise_max", + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in axis_list[j][i]: self.shape1 = input1_shape self.shape2 = input2_shape dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data1"], - "Y": ["input_data2"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data1"], + "Y": ["input_data2"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial( - generate_input, input1_shape)), - "input_data2": - TensorConfig(data_gen=partial( - generate_input, input2_shape)) + "input_data1": TensorConfig( + data_gen=partial( + generate_input, input1_shape + ) + ), + "input_data2": TensorConfig( + data_gen=partial( + generate_input, input2_shape + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): - max_shape = [[128], [128, 128], [128, 128, 128], - [128, 128, 128, 128]] + max_shape = [ + [128], + [128, 128], + [128, 128, 128], + [128, 128, 128, 128], + ] min_shape = [[1], [1, 1], [1, 1, 1], [1, 1, 1, 1]] opt_shape = [[32], [32, 32], [32, 32, 32], [32, 32, 32, 32]] self.dynamic_shape.min_input_shape = { "input_data1": min_shape[len(self.shape1) - 1], - "input_data2": min_shape[len(self.shape2) - 1] + "input_data2": min_shape[len(self.shape2) - 1], } self.dynamic_shape.max_input_shape = { "input_data1": max_shape[len(self.shape1) - 1], - "input_data2": max_shape[len(self.shape2) - 1] + "input_data2": max_shape[len(self.shape2) - 1], } self.dynamic_shape.opt_input_shape = { "input_data1": opt_shape[len(self.shape1) - 1], - "input_data2": opt_shape[len(self.shape2) - 1] + "input_data2": opt_shape[len(self.shape2) - 1], } def clear_dynamic_shape(): @@ -626,12 +672,10 @@ class TrtConvertElementwiseTest_two_input_with_broadcast(TrtLayerAutoScanTest): class TrtConvertElementwiseTest_one_input_corner_case(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -640,52 +684,58 @@ class TrtConvertElementwiseTest_one_input_corner_case(TrtLayerAutoScanTest): return np.random.rand(32).astype(np.float32) for batch in [1, 2, 4]: - for shape in [[32], [batch, 32], [batch, 32, 32], - [batch, 32, 16, 32]]: + for shape in [ + [32], + [batch, 32], + [batch, 32, 32], + [batch, 32, 16, 32], + ]: for op_type in [ - "elementwise_add", - "elementwise_mul", - "elementwise_sub", - "elementwise_div", - "elementwise_pow", - "elementwise_min", - "elementwise_max", + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: self.op_type = op_type for axis in [-1 if len(shape) == 1 else 1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["weight"], - "Y": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["weight"], + "Y": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "weight": - TensorConfig(data_gen=partial(generate_weight)) + "weight": TensorConfig( + data_gen=partial(generate_weight) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)), + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. if self.dims == 1: diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_emb_eltwise_layernorm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_emb_eltwise_layernorm.py index 12b80e09a55..c47992b52af 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_emb_eltwise_layernorm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_emb_eltwise_layernorm.py @@ -22,15 +22,14 @@ import unittest class TrtConvertEmbEltwiseLayernormTest1(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch, input_size): - return np.random.randint(0, 7, size=(batch, input_size, - 1)).astype(np.int64) + return np.random.randint(0, 7, size=(batch, input_size, 1)).astype( + np.int64 + ) def generate_weight1(size11, size2): return np.random.randn(size11, size2).astype(np.float32) @@ -56,180 +55,228 @@ class TrtConvertEmbEltwiseLayernormTest1(TrtLayerAutoScanTest): for axis1 in [0, -1]: for axis2 in [0, -1]: for type in [ - "lookup_table", - "lookup_table_v2" + "lookup_table", + "lookup_table_v2", ]: - dics = [{ - "is_sparse": False, - "is_distributed": False, - "padding_idx": -1, - "is_test": True - }, { - "is_sparse": False, - "is_distributed": False, - "padding_idx": -1, - }, { - "axis": axis1 - }, { - "axis": axis2 - }, { - "begin_norm_axis": norm_axis, - "epsilon": epsilon - }] - ops_config = [{ - "op_type": - type, - "op_inputs": { - "Ids": ["input_data1"], - "W": ["embedding1_weight"] - }, - "op_outputs": { - "Out": - ["embedding1_output"] + dics = [ + { + "is_sparse": False, + "is_distributed": False, + "padding_idx": -1, + "is_test": True, }, - "op_attrs": - dics[0] if type - == "lookup_table" else dics[1] - }, { - "op_type": - type, - "op_inputs": { - "Ids": ["input_data2"], - "W": ["embedding2_weight"] + { + "is_sparse": False, + "is_distributed": False, + "padding_idx": -1, }, - "op_outputs": { - "Out": - ["embedding2_output"] + {"axis": axis1}, + {"axis": axis2}, + { + "begin_norm_axis": norm_axis, + "epsilon": epsilon, }, - "op_attrs": - dics[0] if type - == "lookup_table" else dics[1] - }, { - "op_type": - type, - "op_inputs": { - "Ids": ["input_data3"], - "W": ["embedding3_weight"] + ] + ops_config = [ + { + "op_type": type, + "op_inputs": { + "Ids": ["input_data1"], + "W": [ + "embedding1_weight" + ], + }, + "op_outputs": { + "Out": [ + "embedding1_output" + ] + }, + "op_attrs": dics[0] + if type == "lookup_table" + else dics[1], }, - "op_outputs": { - "Out": - ["embedding3_output"] + { + "op_type": type, + "op_inputs": { + "Ids": ["input_data2"], + "W": [ + "embedding2_weight" + ], + }, + "op_outputs": { + "Out": [ + "embedding2_output" + ] + }, + "op_attrs": dics[0] + if type == "lookup_table" + else dics[1], }, - "op_attrs": - dics[0] if type - == "lookup_table" else dics[1] - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["embedding2_output"], - "Y": ["embedding3_output"] + { + "op_type": type, + "op_inputs": { + "Ids": ["input_data3"], + "W": [ + "embedding3_weight" + ], + }, + "op_outputs": { + "Out": [ + "embedding3_output" + ] + }, + "op_attrs": dics[0] + if type == "lookup_table" + else dics[1], }, - "op_outputs": { - "Out": - ["elementwise_add1_output"] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": [ + "embedding2_output" + ], + "Y": [ + "embedding3_output" + ], + }, + "op_outputs": { + "Out": [ + "elementwise_add1_output" + ] + }, + "op_attrs": dics[2], }, - "op_attrs": dics[2] - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": - ["elementwise_add1_output"], - "Y": ["embedding1_output"] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": [ + "elementwise_add1_output" + ], + "Y": [ + "embedding1_output" + ], + }, + "op_outputs": { + "Out": [ + "elementwise_add2_output" + ] + }, + "op_attrs": dics[3], }, - "op_outputs": { - "Out": - ["elementwise_add2_output"] + { + "op_type": "layer_norm", + "op_inputs": { + "X": [ + "elementwise_add2_output" + ], + "Bias": [ + "layer_norm_bias" + ], + "Scale": [ + "layer_norm_scale" + ], + }, + "op_outputs": { + "Y": [ + "layer_norm_output1" + ], + "Mean": [ + "layer_norm_output2" + ], + "Variance": [ + "layer_norm_output3" + ], + }, + "op_attrs": dics[4], }, - "op_attrs": dics[3] - }, { - "op_type": "layer_norm", - "op_inputs": { - "X": - ["elementwise_add2_output"], - "Bias": ["layer_norm_bias"], - "Scale": - ["layer_norm_scale"] - }, - "op_outputs": { - "Y": ["layer_norm_output1"], - "Mean": - ["layer_norm_output2"], - "Variance": - ["layer_norm_output3"] - }, - "op_attrs": dics[4] - }] + ] ops = self.generate_op_config( - ops_config) + ops_config + ) program_config = ProgramConfig( ops=ops, weights={ - "embedding1_weight": - TensorConfig( + "embedding1_weight": TensorConfig( data_gen=partial( generate_weight1, - size11, size2)), - "embedding2_weight": - TensorConfig( + size11, + size2, + ) + ), + "embedding2_weight": TensorConfig( data_gen=partial( generate_weight2, - size12, size2)), - "embedding3_weight": - TensorConfig( + size12, + size2, + ) + ), + "embedding3_weight": TensorConfig( data_gen=partial( generate_weight3, - size13, size2)), - "layer_norm_bias": - TensorConfig( + size13, + size2, + ) + ), + "layer_norm_bias": TensorConfig( data_gen=partial( generate_weight4, - size2)), - "layer_norm_scale": - TensorConfig( + size2, + ) + ), + "layer_norm_scale": TensorConfig( data_gen=partial( generate_weight4, - size2)) + size2, + ) + ), }, inputs={ - "input_data1": - TensorConfig( + "input_data1": TensorConfig( data_gen=partial( generate_input, - batch, input_size)), - "input_data2": - TensorConfig( + batch, + input_size, + ) + ), + "input_data2": TensorConfig( data_gen=partial( generate_input, - batch, input_size)), - "input_data3": - TensorConfig( + batch, + input_size, + ) + ), + "input_data3": TensorConfig( data_gen=partial( generate_input, - batch, input_size)) + batch, + input_size, + ) + ), }, - outputs=["layer_norm_output1"]) + outputs=["layer_norm_output1"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data1": [1, 4, 1], "input_data2": [1, 4, 1], - "input_data3": [1, 4, 1] + "input_data3": [1, 4, 1], } self.dynamic_shape.max_input_shape = { "input_data1": [4, 512, 1], "input_data2": [4, 512, 1], - "input_data3": [4, 512, 1] + "input_data3": [4, 512, 1], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 128, 1], "input_data2": [2, 128, 1], - "input_data3": [2, 128, 1] + "input_data3": [2, 128, 1], } def clear_dynamic_shape(): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_equal.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_equal.py index 4ca0da9532b..ee73586e6e9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_equal.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_equal.py @@ -22,7 +22,6 @@ from typing import List class TrtConvertElementwiseTest_one_input_corner_case(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -35,7 +34,6 @@ class TrtConvertElementwiseTest_one_input_corner_case(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -44,86 +42,84 @@ class TrtConvertElementwiseTest_one_input_corner_case(TrtLayerAutoScanTest): for axis in [-1 if len(shape) == 1 else 1]: self.dims = len(shape) dics = [{"axis": axis}, {"in_dtype": 0, "out_dtype": 5}] - ops_config = [{ - "op_type": "equal", - "op_inputs": { - "X": ["input_data1"], - "Y": ["input_data2"] - }, - "op_outputs": { - "Out": ["compare_output_data"] + ops_config = [ + { + "op_type": "equal", + "op_inputs": { + "X": ["input_data1"], + "Y": ["input_data2"], + }, + "op_outputs": {"Out": ["compare_output_data"]}, + "op_attrs": dics[0], }, - "op_attrs": dics[0] - }, { - "op_type": "cast", - "op_inputs": { - "X": ["compare_output_data"] + { + "op_type": "cast", + "op_inputs": {"X": ["compare_output_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[1], }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[1] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig( - data_gen=partial(generate_input, shape)), - "input_data2": - TensorConfig( - data_gen=partial(generate_input, shape)) + "input_data1": TensorConfig( + data_gen=partial(generate_input, shape) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. if self.dims == 2: self.dynamic_shape.min_input_shape = { "input_data1": [1, 1], - "input_data2": [1, 1] + "input_data2": [1, 1], } self.dynamic_shape.max_input_shape = { "input_data1": [4, 1], - "input_data2": [4, 1] + "input_data2": [4, 1], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 1], - "input_data2": [2, 1] + "input_data2": [2, 1], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "input_data1": [1, 1, 4], - "input_data2": [1, 1, 4] + "input_data2": [1, 1, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [4, 1, 256], - "input_data2": [1, 1, 256] + "input_data2": [1, 1, 256], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 1, 16], - "input_data2": [2, 1, 16] + "input_data2": [2, 1, 16], } elif self.dims == 4: self.dynamic_shape.min_input_shape = { "input_data1": [1, 1, 4, 4], - "input_data2": [1, 1, 4, 4] + "input_data2": [1, 1, 4, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [4, 1, 128, 256], - "input_data2": [4, 1, 128, 256] + "input_data2": [4, 1, 128, 256], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 1, 32, 16], - "input_data2": [2, 1, 32, 16] + "input_data2": [2, 1, 32, 16], } def clear_dynamic_shape(): @@ -144,19 +140,23 @@ class TrtConvertElementwiseTest_one_input_corner_case(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_expand_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_expand_v2.py index 781264531d6..6eb02281035 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_expand_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_expand_v2.py @@ -22,7 +22,6 @@ import unittest class TrtConvertExpandV2Test(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -35,7 +34,6 @@ class TrtConvertExpandV2Test(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: self.input_shape = [1, 1, 4, 6] @@ -69,29 +67,31 @@ class TrtConvertExpandV2Test(TrtLayerAutoScanTest): self.dims = dims dics_intput = [{"X": ["expand_v2_input"]}] - ops_config = [{ - "op_type": "expand_v2", - "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["expand_v2_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "expand_v2", + "op_inputs": dics_intput[0], + "op_outputs": {"Out": ["expand_v2_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "expand_v2_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "expand_v2_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["expand_v2_out"]) + outputs=["expand_v2_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -146,19 +146,23 @@ class TrtConvertExpandV2Test(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass @@ -169,7 +173,6 @@ class TrtConvertExpandV2Test(TrtLayerAutoScanTest): class TrtConvertExpandV2Test2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -177,7 +180,6 @@ class TrtConvertExpandV2Test2(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 1: self.input_shape = [1] @@ -192,18 +194,13 @@ class TrtConvertExpandV2Test2(TrtLayerAutoScanTest): ] self.dims = dims dics_intput = [ - { - "X": ["expand_v2_input"], - "Shape": ["shapeT1_data"] - }, + {"X": ["expand_v2_input"], "Shape": ["shapeT1_data"]}, ] ops_config = [ { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT1_data"] - }, + "op_outputs": {"Out": ["shapeT1_data"]}, "op_attrs": { "dtype": 2, "str_value": "10", @@ -213,10 +210,8 @@ class TrtConvertExpandV2Test2(TrtLayerAutoScanTest): { "op_type": "expand_v2", "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["expand_v2_out"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["expand_v2_out"]}, + "op_attrs": dics[0], }, ] ops = self.generate_op_config(ops_config) @@ -224,16 +219,18 @@ class TrtConvertExpandV2Test2(TrtLayerAutoScanTest): ops=ops, weights={}, inputs={ - "expand_v2_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "expand_v2_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["expand_v2_out"]) + outputs=["expand_v2_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 1: self.dynamic_shape.min_input_shape = {"expand_v2_input": [1]} @@ -262,7 +259,6 @@ class TrtConvertExpandV2Test2(TrtLayerAutoScanTest): class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -270,7 +266,6 @@ class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: self.input_shape = [1, 1, 4, 6] @@ -291,18 +286,18 @@ class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): { "X": ["expand_v2_input"], "expand_shapes_tensor": [ - "shapeT1_data", "shapeT2_data", "shapeT3_data", - "shapeT4_data" - ] + "shapeT1_data", + "shapeT2_data", + "shapeT3_data", + "shapeT4_data", + ], }, ] ops_config = [ { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT1_data"] - }, + "op_outputs": {"Out": ["shapeT1_data"]}, "op_attrs": { "dtype": 2, "str_value": "10", @@ -312,9 +307,7 @@ class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT2_data"] - }, + "op_outputs": {"Out": ["shapeT2_data"]}, "op_attrs": { "dtype": 2, "str_value": "12", @@ -324,9 +317,7 @@ class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT3_data"] - }, + "op_outputs": {"Out": ["shapeT3_data"]}, "op_attrs": { "dtype": 2, "str_value": "4", @@ -336,9 +327,7 @@ class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT4_data"] - }, + "op_outputs": {"Out": ["shapeT4_data"]}, "op_attrs": { "dtype": 2, "str_value": "6", @@ -348,10 +337,8 @@ class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): { "op_type": "expand_v2", "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["expand_v2_out"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["expand_v2_out"]}, + "op_attrs": dics[0], }, ] ops = self.generate_op_config(ops_config) @@ -359,16 +346,18 @@ class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): ops=ops, weights={}, inputs={ - "expand_v2_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "expand_v2_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["expand_v2_out"]) + outputs=["expand_v2_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fc.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fc.py index 97045ee8a54..04763c98588 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fc.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fc.py @@ -23,10 +23,9 @@ import os class TrtConvertFcTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # The output has diff between gpu and trt in CI windows - if (os.name == 'nt'): + if os.name == 'nt': return False return True @@ -34,12 +33,14 @@ class TrtConvertFcTest(TrtLayerAutoScanTest): self.trt_param.workspace_size = 1073741824 def generate_input1(batch, attrs: List[Dict[str, Any]]): - return np.random.random([batch, 3, 64, (int)(attrs[0]["m"] / 2), - 2]).astype(np.float32) + return np.random.random( + [batch, 3, 64, (int)(attrs[0]["m"] / 2), 2] + ).astype(np.float32) def generate_w(batch, attrs: List[Dict[str, Any]]): - return np.random.random([attrs[0]["m"], - attrs[0]["n"]]).astype(np.float32) + return np.random.random([attrs[0]["m"], attrs[0]["n"]]).astype( + np.float32 + ) def generate_bias(batch, attrs: List[Dict[str, Any]]): return np.random.random([attrs[0]["n"]]).astype(np.float32) @@ -53,7 +54,7 @@ class TrtConvertFcTest(TrtLayerAutoScanTest): "m": m, "n": n, }, - {} + {}, ] ops_config = [ @@ -62,12 +63,10 @@ class TrtConvertFcTest(TrtLayerAutoScanTest): "op_inputs": { "Input": ["input_data"], "W": ["w_data"], - "Bias": ["bias_data"] - }, - "op_outputs": { - "Out": ["output_data"] + "Bias": ["bias_data"], }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], }, ] @@ -76,24 +75,26 @@ class TrtConvertFcTest(TrtLayerAutoScanTest): program_config = ProgramConfig( ops=ops, weights={ - "w_data": - TensorConfig(data_gen=partial(generate_w, batch, dics)), - "bias_data": - TensorConfig( - data_gen=partial(generate_bias, batch, dics)) + "w_data": TensorConfig( + data_gen=partial(generate_w, batch, dics) + ), + "bias_data": TensorConfig( + data_gen=partial(generate_bias, batch, dics) + ), }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, dics)), + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, dics) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 3, 32, 16, 2], @@ -121,19 +122,23 @@ class TrtConvertFcTest(TrtLayerAutoScanTest): # clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() @@ -143,10 +148,9 @@ class TrtConvertFcTest(TrtLayerAutoScanTest): class TrtConvertFcTest2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # The output has diff between gpu and trt in CI windows - if (os.name == 'nt'): + if os.name == 'nt': return False return True @@ -157,8 +161,9 @@ class TrtConvertFcTest2(TrtLayerAutoScanTest): return np.random.random([batch, 3, 64, 14]).astype(np.float32) def generate_w(batch, attrs: List[Dict[str, Any]]): - return np.random.random([attrs[0]["m"], - attrs[0]["n"]]).astype(np.float32) + return np.random.random([attrs[0]["m"], attrs[0]["n"]]).astype( + np.float32 + ) def generate_bias(batch, attrs: List[Dict[str, Any]]): return np.random.random([attrs[0]["n"]]).astype(np.float32) @@ -172,7 +177,7 @@ class TrtConvertFcTest2(TrtLayerAutoScanTest): "m": m, "n": n, }, - {} + {}, ] ops_config = [ @@ -181,12 +186,10 @@ class TrtConvertFcTest2(TrtLayerAutoScanTest): "op_inputs": { "Input": ["input_data"], "W": ["w_data"], - "Bias": ["bias_data"] + "Bias": ["bias_data"], }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], }, ] @@ -195,24 +198,26 @@ class TrtConvertFcTest2(TrtLayerAutoScanTest): program_config = ProgramConfig( ops=ops, weights={ - "w_data": - TensorConfig(data_gen=partial(generate_w, batch, dics)), - "bias_data": - TensorConfig( - data_gen=partial(generate_bias, batch, dics)) + "w_data": TensorConfig( + data_gen=partial(generate_w, batch, dics) + ), + "bias_data": TensorConfig( + data_gen=partial(generate_bias, batch, dics) + ), }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, dics)), + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, dics) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): self.dynamic_shape.min_input_shape = { "input_data": [1, 3, 32, 14], @@ -277,7 +282,7 @@ class TrtConvertFcTest3(TrtLayerAutoScanTest): "m": m, "n": n, }, - {} + {}, ] ops_config = [ @@ -286,12 +291,10 @@ class TrtConvertFcTest3(TrtLayerAutoScanTest): "op_inputs": { "Input": ["input_data"], "W": ["w_data"], - "Bias": ["bias_data"] + "Bias": ["bias_data"], }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], }, ] @@ -300,24 +303,26 @@ class TrtConvertFcTest3(TrtLayerAutoScanTest): program_config = ProgramConfig( ops=ops, weights={ - "w_data": - TensorConfig(data_gen=partial(generate_w, batch, dics)), - "bias_data": - TensorConfig( - data_gen=partial(generate_bias, batch, dics)) + "w_data": TensorConfig( + data_gen=partial(generate_w, batch, dics) + ), + "bias_data": TensorConfig( + data_gen=partial(generate_bias, batch, dics) + ), }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, dics)), + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, dics) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): self.dynamic_shape.min_input_shape = { "input_data": [1, 14, 1, 2], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fill_constant.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fill_constant.py index 5e7132d74c4..6e22f5db13c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fill_constant.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fill_constant.py @@ -22,12 +22,10 @@ from typing import Any, Dict, List class TrtConvertSplitTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_value_data(attrs: List[Dict[str, Any]]): return np.array([1]).astype(np.int32) @@ -47,21 +45,28 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): str_value = str_value else: str_value = "" - dics = [{ - "str_value": str_value, - "value": value, - "shape": shape, - "dtype": dtype - }, { - "axis": -1 - }] - dics_intput = [{ - "ValueTensor": ["value_data"] - }, { - "ShapeTensor": ["shape_data"], - }, { - "ShapeTensorList": ["shapeT1_data", "shapeT2_data"], - }, {}] + dics = [ + { + "str_value": str_value, + "value": value, + "shape": shape, + "dtype": dtype, + }, + {"axis": -1}, + ] + dics_intput = [ + {"ValueTensor": ["value_data"]}, + { + "ShapeTensor": ["shape_data"], + }, + { + "ShapeTensorList": [ + "shapeT1_data", + "shapeT2_data", + ], + }, + {}, + ] ops_config = [ { "op_type": "fill_constant", @@ -69,7 +74,7 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): "op_outputs": { "Out": ["out_data"], }, - "op_attrs": dics[0] + "op_attrs": dics[0], }, ] @@ -81,26 +86,31 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): ops=ops, weights={}, inputs={ - "value_data": - TensorConfig(data_gen=partial( - generate_value_data, dics)), - "shape_data": - TensorConfig(data_gen=partial( - generate_shape_data, dics)), - "shapeT1_data": - TensorConfig(data_gen=partial( - generate_shapelist_data, dics)), - "shapeT2_data": - TensorConfig(data_gen=partial( - generate_shapelist_data, dics)), + "value_data": TensorConfig( + data_gen=partial(generate_value_data, dics) + ), + "shape_data": TensorConfig( + data_gen=partial(generate_shape_data, dics) + ), + "shapeT1_data": TensorConfig( + data_gen=partial( + generate_shapelist_data, dics + ) + ), + "shapeT2_data": TensorConfig( + data_gen=partial( + generate_shapelist_data, dics + ) + ), }, - outputs=["out_data"]) + outputs=["out_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.input_shape = [1, 1] max_shape = list(self.input_shape) @@ -118,7 +128,7 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): self.dynamic_shape.opt_input_shape = {} def generate_trt_nodes_num(attrs, dynamic_shape): - if (self.num_input < 3): + if self.num_input < 3: return 0, 6 return 1, 5 @@ -131,10 +141,12 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten.py index 314457e25e9..4027b8e1f48 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten.py @@ -22,12 +22,10 @@ from typing import List class TrtConvertFlattenTest_dim_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([batch, 32]).astype(np.float32) @@ -39,34 +37,35 @@ class TrtConvertFlattenTest_dim_2(TrtLayerAutoScanTest): else: op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } dics = [{"axis": axis}] - ops_config = [{ - "op_type": "flatten", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "flatten", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 8]} self.dynamic_shape.max_input_shape = {"input_data": [4, 64]} @@ -100,31 +99,33 @@ class TrtConvertFlattenTest_dim_2(TrtLayerAutoScanTest): # for static_shape clear_dynamic_shape() yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() class TrtConvertFlattenTest_dim_3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([batch, 32, 64]).astype(np.float32) @@ -136,34 +137,35 @@ class TrtConvertFlattenTest_dim_3(TrtLayerAutoScanTest): else: op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } dics = [{"axis": axis}] - ops_config = [{ - "op_type": "flatten", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "flatten", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 8, 8]} self.dynamic_shape.max_input_shape = {"input_data": [4, 32, 64]} @@ -198,31 +200,33 @@ class TrtConvertFlattenTest_dim_3(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() class TrtConvertFlattenTest_dim_4(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([batch, 8, 8, 8]).astype(np.float32) @@ -234,34 +238,35 @@ class TrtConvertFlattenTest_dim_4(TrtLayerAutoScanTest): else: op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } dics = [{"axis": axis}] - ops_config = [{ - "op_type": "flatten", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "flatten", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 4, 4, 4]} self.dynamic_shape.max_input_shape = {"input_data": [4, 32, 32, 32]} @@ -296,31 +301,33 @@ class TrtConvertFlattenTest_dim_4(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() class TrtConvertFlattenTest_dim_5(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([batch, 8, 8, 8]).astype(np.float32) @@ -332,34 +339,35 @@ class TrtConvertFlattenTest_dim_5(TrtLayerAutoScanTest): else: op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } dics = [{"axis": axis}] - ops_config = [{ - "op_type": "flatten", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "flatten", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 4, 4, 4]} self.dynamic_shape.max_input_shape = {"input_data": [4, 16, 16, 8]} @@ -394,19 +402,23 @@ class TrtConvertFlattenTest_dim_5(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten_contiguous_range.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten_contiguous_range.py index e39d8bf10e2..fb27f8e630e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten_contiguous_range.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten_contiguous_range.py @@ -22,12 +22,10 @@ from typing import List class TrtConvertFlattenContiguousRangeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([2, batch, 4, 8, 3]).astype(np.float32) @@ -37,35 +35,36 @@ class TrtConvertFlattenContiguousRangeTest(TrtLayerAutoScanTest): type = "flatten_contiguous_range" op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } - ops_config = [{ - "op_type": type, - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": { - "start_axis": start_axis, - "stop_axis": stop_axis, + ops_config = [ + { + "op_type": type, + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": { + "start_axis": start_axis, + "stop_axis": stop_axis, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [2, 1, 4, 8, 3]} self.dynamic_shape.max_input_shape = {"input_data": [2, 4, 4, 8, 3]} @@ -97,19 +96,23 @@ class TrtConvertFlattenContiguousRangeTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fused_token_prune.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fused_token_prune.py index 287f0af0823..5130660257c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fused_token_prune.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fused_token_prune.py @@ -22,7 +22,6 @@ import unittest class TrtConvertFusedTokenPruneTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -40,65 +39,70 @@ class TrtConvertFusedTokenPruneTest(TrtLayerAutoScanTest): for keep_first_token in [True, False]: for keep_order in [True, False]: - dics = [{ - "keep_first_token": keep_first_token, - "keep_order": keep_order - }] - ops_config = [{ - "op_type": "fused_token_prune", - "op_inputs": { - "Attn": ["attn"], - "X": ["x"], - "Mask": ["mask"], - "NewMask": ["new_mask"] - }, - "op_outputs": { - "SlimmedX": ["slimmed_x"], - "CLSInds": ["cls_inds"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "keep_first_token": keep_first_token, + "keep_order": keep_order, + } + ] + ops_config = [ + { + "op_type": "fused_token_prune", + "op_inputs": { + "Attn": ["attn"], + "X": ["x"], + "Mask": ["mask"], + "NewMask": ["new_mask"], + }, + "op_outputs": { + "SlimmedX": ["slimmed_x"], + "CLSInds": ["cls_inds"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "attn": - TensorConfig( - data_gen=partial(generate_attn_or_mask, dics)), - "x": - TensorConfig(data_gen=partial(generate_x, dics)), - "mask": - TensorConfig( - data_gen=partial(generate_attn_or_mask, dics)), - "new_mask": - TensorConfig(data_gen=partial(generate_new_mask, dics)) + "attn": TensorConfig( + data_gen=partial(generate_attn_or_mask, dics) + ), + "x": TensorConfig(data_gen=partial(generate_x, dics)), + "mask": TensorConfig( + data_gen=partial(generate_attn_or_mask, dics) + ), + "new_mask": TensorConfig( + data_gen=partial(generate_new_mask, dics) + ), }, - outputs=["slimmed_x", "cls_inds"]) + outputs=["slimmed_x", "cls_inds"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "attn": [4, 12, 64, 64], "x": [4, 64, 76], "mask": [4, 12, 64, 64], - "new_mask": [4, 12, 32, 32] + "new_mask": [4, 12, 32, 32], } self.dynamic_shape.max_input_shape = { "attn": [4, 12, 64, 64], "x": [4, 64, 76], "mask": [4, 12, 64, 64], - "new_mask": [4, 12, 32, 32] + "new_mask": [4, 12, 32, 32], } self.dynamic_shape.opt_input_shape = { "attn": [4, 12, 64, 64], "x": [4, 64, 76], "mask": [4, 12, 64, 64], - "new_mask": [4, 12, 32, 32] + "new_mask": [4, 12, 32, 32], } def clear_dynamic_shape(): @@ -116,10 +120,12 @@ class TrtConvertFusedTokenPruneTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-2, 1e-2) + attrs, True + ), (1e-2, 1e-2) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-1, 1e-2) + attrs, True + ), (1e-1, 1e-2) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather.py index d74cad22928..ca2984fa187 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather.py @@ -22,7 +22,6 @@ import unittest class TrtConvertGatherTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -34,7 +33,6 @@ class TrtConvertGatherTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(shape): return np.random.random(shape).astype(np.float32) @@ -51,112 +49,126 @@ class TrtConvertGatherTest(TrtLayerAutoScanTest): for index in [[1, 4], [4, 8]]: for axis in [0, 1, 2, 3]: for overwrite in [True, False]: - for input in [{ - "X": ["input_data"], - "Index": ["index_data"] - }, { + for input in [ + {"X": ["input_data"], "Index": ["index_data"]}, + { "X": ["input_data"], "Index": ["index_data"], - "Axis": ["axis_data"] - }]: + "Axis": ["axis_data"], + }, + ]: for index_type_int32 in [True, False]: self.shape = shape self.axis = axis self.input_num = len(input) self.index_type_int32 = index_type_int32 dics = [{"overwrite": overwrite, "axis": axis}] - ops_config = [{ - "op_type": "gather", - "op_inputs": input, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "gather", + "op_inputs": input, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, shape)), - "index_data": - TensorConfig(data_gen=partial( - generate_input2 - if index_type_int32 == - True else generate_input4, index)), - } if len(input) == 2 else { - "input_data": - TensorConfig(data_gen=partial( - generate_input1, shape)), - "index_data": - TensorConfig(data_gen=partial( - generate_input2, index)), - "axis_data": - TensorConfig(data_gen=partial( - generate_input3, axis)), + "input_data": TensorConfig( + data_gen=partial( + generate_input1, shape + ) + ), + "index_data": TensorConfig( + data_gen=partial( + generate_input2 + if index_type_int32 == True + else generate_input4, + index, + ) + ), + } + if len(input) == 2 + else { + "input_data": TensorConfig( + data_gen=partial( + generate_input1, shape + ) + ), + "index_data": TensorConfig( + data_gen=partial( + generate_input2, index + ) + ), + "axis_data": TensorConfig( + data_gen=partial( + generate_input3, axis + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if len(self.shape) == 1: self.dynamic_shape.min_input_shape = { "input_data": [4], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [128], - "index_data": [4] + "index_data": [4], } self.dynamic_shape.opt_input_shape = { "input_data": [16], - "index_data": [2] + "index_data": [2], } elif len(self.shape) == 2: self.dynamic_shape.min_input_shape = { "input_data": [2, 4], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [256, 256], - "index_data": [4] + "index_data": [4], } self.dynamic_shape.opt_input_shape = { "input_data": [64, 32], - "index_data": [2] + "index_data": [2], } elif len(self.shape) == 3: self.dynamic_shape.min_input_shape = { "input_data": [2, 4, 4], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [128, 256, 256], - "index_data": [4] + "index_data": [4], } self.dynamic_shape.opt_input_shape = { "input_data": [16, 64, 32], - "index_data": [2] + "index_data": [2], } elif len(self.shape) == 4: self.dynamic_shape.min_input_shape = { "input_data": [2, 4, 4, 2], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [128, 256, 64, 128], - "index_data": [4] + "index_data": [4], } self.dynamic_shape.opt_input_shape = { "input_data": [16, 64, 16, 32], - "index_data": [2] + "index_data": [2], } def clear_dynamic_shape(): @@ -181,10 +193,12 @@ class TrtConvertGatherTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-5 + False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-3 + False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) @@ -200,14 +214,17 @@ class TrtConvertGatherTest(TrtLayerAutoScanTest): def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0: inputs = program_config.inputs - if len(inputs['input_data'].shape) == 1 or len( - inputs['index_data'].shape) == 1: + if ( + len(inputs['input_data'].shape) == 1 + or len(inputs['index_data'].shape) == 1 + ): return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Need to repair the case: trt reshape out failed for dynamic shape mode when inputs' dims==1. under trt7.0 " + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Need to repair the case: trt reshape out failed for dynamic shape mode when inputs' dims==1. under trt7.0 ", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather_nd.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather_nd.py index ea392357114..6ac8314ec77 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather_nd.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather_nd.py @@ -23,7 +23,6 @@ import os class TrtConvertGatherNdTest_dim_4_1(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # The output has diff between gpu and trt in CI windows # if ( and self.trt_param.precision == paddle_infer.PrecisionType.Half): @@ -31,54 +30,53 @@ class TrtConvertGatherNdTest_dim_4_1(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32, 64, 64]).astype(np.float32) def generate_input2(): return np.ones([1]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) for i in range(10): program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), - "index_data": - TensorConfig(data_gen=partial(generate_input2)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), + "index_data": TensorConfig( + data_gen=partial(generate_input2) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 8, 8, 8], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [4, 32, 64, 64], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 32, 64, 64], - "index_data": [1] + "index_data": [1], } def clear_dynamic_shape(): @@ -105,15 +103,16 @@ class TrtConvertGatherNdTest_dim_4_1(TrtLayerAutoScanTest): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -121,29 +120,24 @@ class TrtConvertGatherNdTest_dim_4_1(TrtLayerAutoScanTest): class TrtConvertGatherNdTest_dim_4_1_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32, 64, 64]).astype(np.float32) def generate_input2(): return np.array([1, 2]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -153,25 +147,26 @@ class TrtConvertGatherNdTest_dim_4_1_2(TrtLayerAutoScanTest): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 8, 8, 8], - "index_data": [2] + "index_data": [2], } self.dynamic_shape.max_input_shape = { "input_data": [4, 32, 64, 64], - "index_data": [2] + "index_data": [2], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 32, 64, 64], - "index_data": [2] + "index_data": [2], } def clear_dynamic_shape(): @@ -198,15 +193,16 @@ class TrtConvertGatherNdTest_dim_4_1_2(TrtLayerAutoScanTest): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -214,29 +210,24 @@ class TrtConvertGatherNdTest_dim_4_1_2(TrtLayerAutoScanTest): class TrtConvertGatherNdTest_dim_4_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32, 64, 64]).astype(np.float32) def generate_input2(): return np.ones([2, 2]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -246,25 +237,26 @@ class TrtConvertGatherNdTest_dim_4_2(TrtLayerAutoScanTest): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 8, 8, 8], - "index_data": [2, 2] + "index_data": [2, 2], } self.dynamic_shape.max_input_shape = { "input_data": [4, 32, 64, 64], - "index_data": [2, 2] + "index_data": [2, 2], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 32, 64, 64], - "index_data": [2, 2] + "index_data": [2, 2], } def clear_dynamic_shape(): @@ -291,15 +283,16 @@ class TrtConvertGatherNdTest_dim_4_2(TrtLayerAutoScanTest): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -307,29 +300,24 @@ class TrtConvertGatherNdTest_dim_4_2(TrtLayerAutoScanTest): class TrtConvertGatherNdTest_dim_4_3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32, 64, 64]).astype(np.float32) def generate_input2(): return np.ones([2, 2, 4]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -339,25 +327,26 @@ class TrtConvertGatherNdTest_dim_4_3(TrtLayerAutoScanTest): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 8, 8, 8], - "index_data": [2, 2, 4] + "index_data": [2, 2, 4], } self.dynamic_shape.max_input_shape = { "input_data": [4, 32, 64, 64], - "index_data": [2, 2, 4] + "index_data": [2, 2, 4], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 32, 64, 64], - "index_data": [2, 2, 4] + "index_data": [2, 2, 4], } def clear_dynamic_shape(): @@ -384,15 +373,16 @@ class TrtConvertGatherNdTest_dim_4_3(TrtLayerAutoScanTest): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -400,29 +390,24 @@ class TrtConvertGatherNdTest_dim_4_3(TrtLayerAutoScanTest): class TrtConvertGatherNdTest_dim_2_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32]).astype(np.float32) def generate_input2(): return np.array([[0, 3], [1, 9]]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -432,25 +417,26 @@ class TrtConvertGatherNdTest_dim_2_2(TrtLayerAutoScanTest): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 4], - "index_data": [2, 2] + "index_data": [2, 2], } self.dynamic_shape.max_input_shape = { "input_data": [4, 64], - "index_data": [2, 2] + "index_data": [2, 2], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 8], - "index_data": [2, 2] + "index_data": [2, 2], } def clear_dynamic_shape(): @@ -477,15 +463,16 @@ class TrtConvertGatherNdTest_dim_2_2(TrtLayerAutoScanTest): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -493,30 +480,26 @@ class TrtConvertGatherNdTest_dim_2_2(TrtLayerAutoScanTest): class TrtConvertGatherNdTest_dim_3_3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([16, 32, 256]).astype(np.float32) def generate_input2(): - return np.array([[[2, 5], [3, 8]], [[0, 2], [0, - 3]]]).astype(np.int32) - - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + return np.array([[[2, 5], [3, 8]], [[0, 2], [0, 3]]]).astype( + np.int32 + ) + + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -526,25 +509,26 @@ class TrtConvertGatherNdTest_dim_3_3(TrtLayerAutoScanTest): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 4, 4], - "index_data": [1, 1, 1] + "index_data": [1, 1, 1], } self.dynamic_shape.max_input_shape = { "input_data": [16, 64, 512], - "index_data": [4, 2, 4] + "index_data": [4, 2, 4], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 8, 64], - "index_data": [2, 2, 2] + "index_data": [2, 2, 2], } def clear_dynamic_shape(): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gelu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gelu.py index 4f8e5c6356a..d6d2f876361 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gelu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gelu.py @@ -22,12 +22,10 @@ import unittest class TrtConvertGeluTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([32]).astype(np.float32) @@ -43,33 +41,32 @@ class TrtConvertGeluTest(TrtLayerAutoScanTest): self.dims = dims dics = [{"approximate": approximate}] - ops_config = [{ - "op_type": "gelu", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "gelu", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dims, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dims, dics) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -123,19 +120,23 @@ class TrtConvertGeluTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_grid_sampler.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_grid_sampler.py index e04c096d949..36b0c1638cb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_grid_sampler.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_grid_sampler.py @@ -22,29 +22,27 @@ import unittest class TrtConvertGridSampler(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([1, 3, 32, 32]).astype(np.float32) def generate_input2(): return np.random.random([1, 3, 3, 2]).astype(np.float32) - ops_config = [{ - "op_type": "grid_sampler", - "op_inputs": { - "X": ["input_data"], - "Grid": ["grid_data"], - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "grid_sampler", + "op_inputs": { + "X": ["input_data"], + "Grid": ["grid_data"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) for i in range(10): @@ -52,30 +50,33 @@ class TrtConvertGridSampler(TrtLayerAutoScanTest): ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), - "grid_data": - TensorConfig(data_gen=partial(generate_input2)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), + "grid_data": TensorConfig( + data_gen=partial(generate_input2) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 3, 32, 32], - "grid_data": [1, 3, 3, 2] + "grid_data": [1, 3, 3, 2], } self.dynamic_shape.max_input_shape = { "input_data": [1, 3, 64, 64], - "grid_data": [1, 3, 4, 4] + "grid_data": [1, 3, 4, 4], } self.dynamic_shape.opt_input_shape = { "input_data": [1, 3, 32, 32], - "grid_data": [1, 3, 3, 2] + "grid_data": [1, 3, 3, 2], } def clear_dynamic_shape(): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_group_norm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_group_norm.py index 0c8e6eeb87b..18f40934172 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_group_norm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_group_norm.py @@ -22,7 +22,6 @@ import unittest class TrtConvertGroupNormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -36,7 +35,6 @@ class TrtConvertGroupNormTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input(attrs: List[Dict[str, Any]], batch): if attrs[0]['data_layout'] == 'NCHW': return np.random.random([batch, 32, 64, 64]).astype(np.float32) @@ -53,47 +51,56 @@ class TrtConvertGroupNormTest(TrtLayerAutoScanTest): for group in [1, 4, 32, -1]: for epsilon in [0.0001, 0.0007, -1, 1]: for data_layout in ['NCHW']: - dics = [{ - "epsilon": epsilon, - "groups": group, - "data_layout": data_layout - }] - ops_config = [{ - "op_type": "group_norm", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale_weight"], - "Bias": ["bias_weight"] - }, - "op_outputs": { - "Y": ["y_output"], - "Mean": ["mean_output"], - "Variance": ["variance_output"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "epsilon": epsilon, + "groups": group, + "data_layout": data_layout, + } + ] + ops_config = [ + { + "op_type": "group_norm", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale_weight"], + "Bias": ["bias_weight"], + }, + "op_outputs": { + "Y": ["y_output"], + "Mean": ["mean_output"], + "Variance": ["variance_output"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "scale_weight": - TensorConfig(data_gen=partial(generate_scale)), - "bias_weight": - TensorConfig(data_gen=partial(generate_bias)) + "scale_weight": TensorConfig( + data_gen=partial(generate_scale) + ), + "bias_weight": TensorConfig( + data_gen=partial(generate_bias) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input, dics, batch)) + "input_data": TensorConfig( + data_gen=partial( + generate_input, dics, batch + ) + ) }, - outputs=["y_output"]) + outputs=["y_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 16, 16, 16]} self.dynamic_shape.max_input_shape = { @@ -117,19 +124,23 @@ class TrtConvertGroupNormTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_sigmoid.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_sigmoid.py index 0a1699139c9..3c8cd8948f4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_sigmoid.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_sigmoid.py @@ -22,12 +22,10 @@ import unittest class TrtConvertHardSigmoidTest_dim_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -37,33 +35,34 @@ class TrtConvertHardSigmoidTest_dim_2(TrtLayerAutoScanTest): for slope in [0.1, 0.5]: for offset in [0.2, 0.7]: dics = [{"slope": slope, "offset": offset}] - ops_config = [{ - "op_type": "hard_sigmoid", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "hard_sigmoid", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)) + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.input_dim == 2: self.dynamic_shape.min_input_shape = {"input_data": [1, 8]} diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_swish.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_swish.py index b014d6fa2b0..4180c5b0830 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_swish.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_swish.py @@ -22,7 +22,6 @@ import unittest class TrtConvertHardSwishTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -36,46 +35,46 @@ class TrtConvertHardSwishTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 3, 32, 32]).astype(np.float32) for threshold in [6.0, 7.0, 100.0, 0.0, -1.0]: for scale in [5.0, 7.0, -1.0, 0.0, 100.0]: for offset in [3.0, 5.0, -1.0, 0.0, 100.0]: - dics = [{ - "threshold": threshold, - "scale": scale, - "offset": offset - }] - - ops_config = [{ - "op_type": "hard_swish", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["hard_swish_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "threshold": threshold, + "scale": scale, + "offset": offset, + } + ] + + ops_config = [ + { + "op_type": "hard_swish", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["hard_swish_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["hard_swish_output_data"]) + outputs=["hard_swish_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 16, 16]} self.dynamic_shape.max_input_shape = {"input_data": [2, 3, 32, 32]} @@ -97,19 +96,23 @@ class TrtConvertHardSwishTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_instance_norm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_instance_norm.py index 135858fec77..b3be8dfdc66 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_instance_norm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_instance_norm.py @@ -24,7 +24,6 @@ import os class TrtConvertInstanceNormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -36,7 +35,6 @@ class TrtConvertInstanceNormTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], shape_input): return np.random.random(shape_input).astype(np.float32) @@ -44,48 +42,60 @@ class TrtConvertInstanceNormTest(TrtLayerAutoScanTest): return np.random.random(shape_input[1]).astype(np.float32) for batch in [1, 2, 4]: - for shape_input in [[batch, 16], [batch, 32, 64], - [batch, 16, 32, 64]]: + for shape_input in [ + [batch, 16], + [batch, 32, 64], + [batch, 16, 32, 64], + ]: self.in_dim = len(shape_input) for epsilon in [0.0005, -1, 1]: dics = [{"epsilon": epsilon}] - ops_config = [{ - "op_type": "instance_norm", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale_data"], - "Bias": ["bias_data"] - }, - "op_outputs": { - "Y": ["y_data"], - "SavedMean": ["saved_mean_data"], - "SavedVariance": ["saved_variance_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "instance_norm", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale_data"], + "Bias": ["bias_data"], + }, + "op_outputs": { + "Y": ["y_data"], + "SavedMean": ["saved_mean_data"], + "SavedVariance": ["saved_variance_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "bias_data": - TensorConfig(data_gen=partial( - generate_input2, dics, shape_input)), - "scale_data": - TensorConfig(data_gen=partial( - generate_input2, dics, shape_input)) + "bias_data": TensorConfig( + data_gen=partial( + generate_input2, dics, shape_input + ) + ), + "scale_data": TensorConfig( + data_gen=partial( + generate_input2, dics, shape_input + ) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dics, shape_input)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dics, shape_input + ) + ) }, - outputs=["y_data"]) + outputs=["y_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.in_dim == 2: self.dynamic_shape.min_input_shape = {"input_data": [1, 4]} @@ -128,30 +138,35 @@ class TrtConvertInstanceNormTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller2(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller2, SkipReasons.TRT_NOT_SUPPORT, - "The output has diff between gpu and trt in Windows.") + teller2, + SkipReasons.TRT_NOT_SUPPORT, + "The output has diff between gpu and trt in Windows.", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_inverse.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_inverse.py index 390eff3e1e8..6ccb00d1a0f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_inverse.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_inverse.py @@ -22,41 +22,41 @@ import unittest class TrtConvertInverse(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([32, 32]).astype(np.float32) - ops_config = [{ - "op_type": "inverse", - "op_inputs": { - "Input": ["input_data"], - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "inverse", + "op_inputs": { + "Input": ["input_data"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) for i in range(10): program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 1], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_layer_norm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_layer_norm.py index f6e4dec9538..2d52f74be38 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_layer_norm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_layer_norm.py @@ -23,7 +23,6 @@ import unittest class TrtConvertLayerNormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -34,13 +33,13 @@ class TrtConvertLayerNormTest(TrtLayerAutoScanTest): if attrs[0]['epsilon'] < 0 or attrs[0]['epsilon'] > 0.001: return False if attrs[0]['begin_norm_axis'] <= 0 or attrs[0]['begin_norm_axis'] >= ( - len(inputs['input_data'].shape) - 1): + len(inputs['input_data'].shape) - 1 + ): return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], shape_input): return np.ones(shape_input).astype(np.float32) @@ -53,49 +52,52 @@ class TrtConvertLayerNormTest(TrtLayerAutoScanTest): for epsilon in [0.0005, -1, 1]: for begin_norm_axis in [1, 0, -1, 2, 3]: - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis - }, {}] - - ops_config = [{ - "op_type": "layer_norm", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale_data"], - "Bias": ["bias_data"] - }, - "op_outputs": { - "Y": ["y_data"], - "Mean": ["saved_mean_data"], - "Variance": ["saved_variance_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + {"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}, + {}, + ] + + ops_config = [ + { + "op_type": "layer_norm", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale_data"], + "Bias": ["bias_data"], + }, + "op_outputs": { + "Y": ["y_data"], + "Mean": ["saved_mean_data"], + "Variance": ["saved_variance_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) shape_input = [1, 3, 64, 64] program_config = ProgramConfig( ops=ops, weights={ - "bias_data": - TensorConfig(data_gen=partial(generate_input2, dics, - shape_input)), - "scale_data": - TensorConfig(data_gen=partial(generate_input2, dics, - shape_input)) + "bias_data": TensorConfig( + data_gen=partial(generate_input2, dics, shape_input) + ), + "scale_data": TensorConfig( + data_gen=partial(generate_input2, dics, shape_input) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics, - shape_input)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics, shape_input) + ) }, - outputs=["y_data"]) + outputs=["y_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -108,7 +110,7 @@ class TrtConvertLayerNormTest(TrtLayerAutoScanTest): def generate_trt_nodes_num(attrs, dynamic_shape): inputs = program_config.inputs - #if not dynamic_shape: + # if not dynamic_shape: # if attrs[0]["begin_norm_axis"] >= len(inputs["input_data"].shape) - 1: # print ("iiiiiii") # return 0, 3 @@ -122,26 +124,29 @@ class TrtConvertLayerNormTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() class TrtConvertLayerNormTest_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -152,13 +157,13 @@ class TrtConvertLayerNormTest_2(TrtLayerAutoScanTest): if attrs[0]['epsilon'] < 0 or attrs[0]['epsilon'] > 0.001: return False if attrs[0]['begin_norm_axis'] <= 0 or attrs[0]['begin_norm_axis'] >= ( - len(inputs['input_data'].shape) - 1): + len(inputs['input_data'].shape) - 1 + ): return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], shape_input): return np.ones(shape_input).astype(np.float32) @@ -171,49 +176,52 @@ class TrtConvertLayerNormTest_2(TrtLayerAutoScanTest): for epsilon in [0.0005, -1, 1]: for begin_norm_axis in [1, 0, -1, 2, 3]: - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis - }, {}] - - ops_config = [{ - "op_type": "layer_norm", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale_data"], - "Bias": ["bias_data"] - }, - "op_outputs": { - "Y": ["y_data"], - "Mean": ["saved_mean_data"], - "Variance": ["saved_variance_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + {"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}, + {}, + ] + + ops_config = [ + { + "op_type": "layer_norm", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale_data"], + "Bias": ["bias_data"], + }, + "op_outputs": { + "Y": ["y_data"], + "Mean": ["saved_mean_data"], + "Variance": ["saved_variance_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) shape_input = [2, 64, 3, 3] program_config = ProgramConfig( ops=ops, weights={ - "bias_data": - TensorConfig(data_gen=partial(generate_input2, dics, - shape_input)), - "scale_data": - TensorConfig(data_gen=partial(generate_input2, dics, - shape_input)) + "bias_data": TensorConfig( + data_gen=partial(generate_input2, dics, shape_input) + ), + "scale_data": TensorConfig( + data_gen=partial(generate_input2, dics, shape_input) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics, - shape_input)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics, shape_input) + ) }, - outputs=["y_data"]) + outputs=["y_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 64, 3, 3]} self.dynamic_shape.max_input_shape = {"input_data": [4, 64, 3, 3]} @@ -226,7 +234,7 @@ class TrtConvertLayerNormTest_2(TrtLayerAutoScanTest): def generate_trt_nodes_num(attrs, dynamic_shape): inputs = program_config.inputs - #if not dynamic_shape: + # if not dynamic_shape: # if attrs[0]["begin_norm_axis"] >= len(inputs["input_data"].shape) - 1: # print ("iiiiiii") # return 0, 3 @@ -240,19 +248,23 @@ class TrtConvertLayerNormTest_2(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_leaky_relu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_leaky_relu.py index 2742a536de9..93f7115ae4f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_leaky_relu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_leaky_relu.py @@ -23,12 +23,10 @@ import unittest class TrtConvertLeakyReluTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(shape): return np.random.random(shape).astype(np.float32) @@ -37,32 +35,35 @@ class TrtConvertLeakyReluTest(TrtLayerAutoScanTest): self.input_dim = len(shape) for alpha in [0.02, 1.0, 100.0, -1.0, 0.0]: dics = [{"alpha": alpha}] - ops_config = [{ - "op_type": "leaky_relu", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["y_data"], - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "leaky_relu", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": { + "Out": ["y_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, shape)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, shape) + ) }, - outputs=["y_data"]) + outputs=["y_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.input_dim == 2: self.dynamic_shape.min_input_shape = {"input_data": [1, 8]} @@ -101,25 +102,31 @@ class TrtConvertLeakyReluTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul.py index b4895d805bc..9175c9b4ae1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul.py @@ -22,12 +22,10 @@ import unittest class TrtConvertMatmulTest_static(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -47,48 +45,55 @@ class TrtConvertMatmulTest_static(TrtLayerAutoScanTest): input1_shape = [batch, 32, 6] input2_shape = [batch, 6, 11] for alpha in [0.3, 1.0]: - dics = [{ - "transpose_X": trans_x, - "transpose_Y": trans_y, - "alpha": alpha, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }] - ops_config = [{ - "op_type": "matmul", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "transpose_X": trans_x, + "transpose_Y": trans_y, + "alpha": alpha, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + } + ] + ops_config = [ + { + "op_type": "matmul", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig(data_gen=partial( - generate_input, input1_shape)), - "input2_data": - TensorConfig(data_gen=partial( - generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial( + generate_input, input1_shape + ) + ), + "input2_data": TensorConfig( + data_gen=partial( + generate_input, input2_shape + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): pass @@ -109,12 +114,10 @@ class TrtConvertMatmulTest_static(TrtLayerAutoScanTest): class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -133,60 +136,63 @@ class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): # input1_shape = [batch, 32, 6] # input2_shape = [batch, 6, 11] for alpha in [0.3, 1.0]: - dics = [{ - "transpose_X": trans_x, - "transpose_Y": trans_y, - "alpha": alpha, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }] - ops_config = [{ - "op_type": "matmul", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "transpose_X": trans_x, + "transpose_Y": trans_y, + "alpha": alpha, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + } + ] + ops_config = [ + { + "op_type": "matmul", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial(generate_input, input1_shape) + ), + "input2_data": TensorConfig( + data_gen=partial(generate_input, input2_shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input1_data": [1, 4, 4], - "input2_data": [1, 4, 4] + "input2_data": [1, 4, 4], } self.dynamic_shape.max_input_shape = { "input1_data": [16, 4, 4], - "input2_data": [16, 4, 4] + "input2_data": [16, 4, 4], } self.dynamic_shape.opt_input_shape = { "input1_data": [8, 4, 4], - "input2_data": [8, 4, 4] + "input2_data": [8, 4, 4], } attrs = [ diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 524262244d3..82d9143a098 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -23,9 +23,7 @@ import os class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -34,53 +32,56 @@ class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): for trans_y in [False]: input1_shape = [batch, 64, 350, 75] input2_shape = [75, 25] - dics = [{ - "trans_x": trans_x, - "trans_y": trans_y, - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "trans_x": trans_x, + "trans_y": trans_y, + } + ] + ops_config = [ + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial(generate_input, input1_shape) + ), + "input2_data": TensorConfig( + data_gen=partial(generate_input, input2_shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input1_data": [10, 64, 350, 75], - "input2_data": [75, 25] + "input2_data": [75, 25], } self.dynamic_shape.max_input_shape = { "input1_data": [100, 64, 350, 75], - "input2_data": [75, 25] + "input2_data": [75, 25], } self.dynamic_shape.opt_input_shape = { "input1_data": [15, 64, 350, 75], - "input2_data": [75, 25] + "input2_data": [75, 25], } attrs = [ @@ -90,7 +91,7 @@ class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): # The output has little diff between gpu and trt in CI-Windows-Inference tol_fp32 = 1e-5 tol_half = 1e-5 - if (os.name == 'nt'): + if os.name == 'nt': tol_fp32 = 1e-3 tol_half = 1e-3 # for dynamic_shape @@ -109,9 +110,7 @@ class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): class TrtConvertMatmulTest_dynamic2(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -120,53 +119,56 @@ class TrtConvertMatmulTest_dynamic2(TrtLayerAutoScanTest): for trans_y in [False]: input1_shape = [60, 40] input2_shape = [batch, 40, 90] - dics = [{ - "trans_x": trans_x, - "trans_y": trans_y, - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "trans_x": trans_x, + "trans_y": trans_y, + } + ] + ops_config = [ + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial(generate_input, input1_shape) + ), + "input2_data": TensorConfig( + data_gen=partial(generate_input, input2_shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input1_data": [60, 40], - "input2_data": [10, 40, 90] + "input2_data": [10, 40, 90], } self.dynamic_shape.max_input_shape = { "input1_data": [60, 40], - "input2_data": [20, 40, 90] + "input2_data": [20, 40, 90], } self.dynamic_shape.opt_input_shape = { "input1_data": [60, 40], - "input2_data": [15, 40, 90] + "input2_data": [15, 40, 90], } attrs = [ @@ -175,7 +177,7 @@ class TrtConvertMatmulTest_dynamic2(TrtLayerAutoScanTest): # The output has little diff between gpu and trt in CI-Windows-Inference tol_fp32 = 1e-5 tol_half = 1e-5 - if (os.name == 'nt'): + if os.name == 'nt': tol_fp32 = 1e-3 tol_half = 1e-3 # for dynamic_shape @@ -194,9 +196,7 @@ class TrtConvertMatmulTest_dynamic2(TrtLayerAutoScanTest): class TrtConvertMatmulTest_dynamic3(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -219,86 +219,95 @@ class TrtConvertMatmulTest_dynamic3(TrtLayerAutoScanTest): elif case == 2: input1_shape = [50] input2_shape = [50] - if (case == 0 or case == 1): - dics = [{ - "trans_x": False, - "trans_y": False, - }] - elif (case == 2): - dics = [{ - "trans_x": trans_x, - "trans_y": trans_y, - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + if case == 0 or case == 1: + dics = [ + { + "trans_x": False, + "trans_y": False, + } + ] + elif case == 2: + dics = [ + { + "trans_x": trans_x, + "trans_y": trans_y, + } + ] + ops_config = [ + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig(data_gen=partial( - generate_input, input1_shape)), - "input2_data": - TensorConfig(data_gen=partial( - generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial( + generate_input, input1_shape + ) + ), + "input2_data": TensorConfig( + data_gen=partial( + generate_input, input2_shape + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): - if (self.case == 0): + if self.case == 0: self.dynamic_shape.min_input_shape = { "input1_data": [20, 50], - "input2_data": [50] + "input2_data": [50], } self.dynamic_shape.max_input_shape = { "input1_data": [30, 50], - "input2_data": [50] + "input2_data": [50], } self.dynamic_shape.opt_input_shape = { "input1_data": [25, 50], - "input2_data": [50] + "input2_data": [50], } - elif (self.case == 1): + elif self.case == 1: self.dynamic_shape.min_input_shape = { "input2_data": [50, 20], - "input1_data": [50] + "input1_data": [50], } self.dynamic_shape.max_input_shape = { "input2_data": [50, 30], - "input1_data": [50] + "input1_data": [50], } self.dynamic_shape.opt_input_shape = { "input2_data": [50, 25], - "input1_data": [50] + "input1_data": [50], } - elif (self.case == 2): + elif self.case == 2: self.dynamic_shape.min_input_shape = { "input2_data": [30], - "input1_data": [50] + "input1_data": [50], } self.dynamic_shape.max_input_shape = { "input2_data": [50], - "input1_data": [50] + "input1_data": [50], } self.dynamic_shape.opt_input_shape = { "input2_data": [50], - "input1_data": [50] + "input1_data": [50], } generate_dynamic_shape() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_mish.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_mish.py index d6d62967cfb..baac4943f7c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_mish.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_mish.py @@ -21,12 +21,10 @@ import unittest class TrtConvertMishTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch, dim1, dim2, dim3): shape = [batch] if dim1 != 0: @@ -51,35 +49,36 @@ class TrtConvertMishTest(TrtLayerAutoScanTest): if dim1 == 0 and dim2 == 0 and dim3 != 0: continue - ops_config = [{ - "op_type": "mish", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["mish_output_data"] - }, - "op_attrs": { - "threshold": thre + ops_config = [ + { + "op_type": "mish", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["mish_output_data"]}, + "op_attrs": {"threshold": thre}, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input, batch, dim1, dim2, - dim3)) + "input_data": TensorConfig( + data_gen=partial( + generate_input, + batch, + dim1, + dim2, + dim3, + ) + ) }, - outputs=["mish_output_data"]) + outputs=["mish_output_data"], + ) yield program_config def sample_predictor_configs(self, program_config): - def generate_dynamic_shape(attrs): if self.dim1 == 0: self.dynamic_shape.min_input_shape = { @@ -139,29 +138,35 @@ class TrtConvertMishTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.dim1 == 0 and self.dim2 == 0 and self.dim3 == 0: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "Trt does not support 1-dimensional input.") + self.add_skip_case( + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Trt does not support 1-dimensional input.", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms.py index e52eb4d1ce9..2d98b339d2a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms.py @@ -22,7 +22,6 @@ import unittest class TrtConvertMulticlassNMSTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,17 +38,21 @@ class TrtConvertMulticlassNMSTest(TrtLayerAutoScanTest): min_subgraph_size=self.trt_param.min_subgraph_size, precision_mode=self.trt_param.precision, use_static=self.trt_param.use_static, - use_calib_mode=self.trt_param.use_calib_mode) - if len(self.dynamic_shape.min_input_shape - ) != 0 and self.dynamic_shape.min_input_shape.keys( - ) == self.dynamic_shape.max_input_shape.keys( - ) and self.dynamic_shape.min_input_shape.keys( - ) == self.dynamic_shape.opt_input_shape.keys(): + use_calib_mode=self.trt_param.use_calib_mode, + ) + if ( + len(self.dynamic_shape.min_input_shape) != 0 + and self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.max_input_shape.keys() + and self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.opt_input_shape.keys() + ): config.set_trt_dynamic_shape_info( self.dynamic_shape.min_input_shape, self.dynamic_shape.max_input_shape, self.dynamic_shape.opt_input_shape, - self.dynamic_shape.disable_trt_plugin_fp16) + self.dynamic_shape.disable_trt_plugin_fp16, + ) return config else: config = paddle_infer.Config() @@ -59,15 +62,15 @@ class TrtConvertMulticlassNMSTest(TrtLayerAutoScanTest): return config def sample_program_configs(self): - def generate_boxes(batch, num_boxes): - return np.arange(batch * num_boxes * 4, - dtype=np.float32).reshape([batch, num_boxes, 4]) + return np.arange(batch * num_boxes * 4, dtype=np.float32).reshape( + [batch, num_boxes, 4] + ) def generate_scores(batch, num_boxes, num_classes): - return np.arange(batch * num_classes * num_boxes, - dtype=np.float32).reshape( - [batch, num_classes, num_boxes]) + return np.arange( + batch * num_classes * num_boxes, dtype=np.float32 + ).reshape([batch, num_classes, num_boxes]) # return np.random.rand(batch, num_classes, num_boxes).astype(np.float32) for batch in [1, 2]: @@ -76,46 +79,55 @@ class TrtConvertMulticlassNMSTest(TrtLayerAutoScanTest): for num_boxes, num_classes in [[80, 100], [40, 200], [20, 400]]: self.num_boxes, self.num_classes = num_boxes, num_classes for score_threshold in [ - 0.01, + 0.01, ]: - ops_config = [{ - "op_type": "multiclass_nms", - "op_inputs": { - "BBoxes": ["input_bboxes"], - "Scores": ["input_scores"], - }, - "op_outputs": { - "Out": ["nms_output_boxes"], - }, - "op_attrs": { - "background_label": -1, - "score_threshold": score_threshold, - "nms_top_k": num_boxes, - "keep_top_k": num_boxes, - "nms_threshold": 0.3, - "normalized": False, - "nms_eta": nms_eta + ops_config = [ + { + "op_type": "multiclass_nms", + "op_inputs": { + "BBoxes": ["input_bboxes"], + "Scores": ["input_scores"], + }, + "op_outputs": { + "Out": ["nms_output_boxes"], + }, + "op_attrs": { + "background_label": -1, + "score_threshold": score_threshold, + "nms_top_k": num_boxes, + "keep_top_k": num_boxes, + "nms_threshold": 0.3, + "normalized": False, + "nms_eta": nms_eta, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_bboxes": - TensorConfig(data_gen=partial( - generate_boxes, batch, num_boxes)), - "input_scores": - TensorConfig( - data_gen=partial(generate_scores, batch, - num_boxes, num_classes)) + "input_bboxes": TensorConfig( + data_gen=partial( + generate_boxes, batch, num_boxes + ) + ), + "input_scores": TensorConfig( + data_gen=partial( + generate_scores, + batch, + num_boxes, + num_classes, + ) + ), }, - outputs=["nms_output_boxes"]) + outputs=["nms_output_boxes"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The last dim of input_bboxes should be static. self.dynamic_shape.min_input_shape = { @@ -147,39 +159,53 @@ class TrtConvertMulticlassNMSTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 # self.trt_param.precision = paddle_infer.PrecisionType.Half # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, True), (1e-2, 1e-2) - def assert_tensors_near(self, atol: float, rtol: float, - tensor: Dict[str, np.array], - baseline: Dict[str, np.array]): + def assert_tensors_near( + self, + atol: float, + rtol: float, + tensor: Dict[str, np.array], + baseline: Dict[str, np.array], + ): # the order of tensorrt outputs are not consistent with paddle for key, arr in tensor.items(): if key == "nms_output_boxes": basline_arr = np.array( - sorted(baseline[key].reshape((-1, 6)), - key=lambda i: [i[0], i[1]])) + sorted( + baseline[key].reshape((-1, 6)), + key=lambda i: [i[0], i[1]], + ) + ) arr = np.array( - sorted(arr.reshape((-1, 6)), key=lambda i: [i[0], i[1]])) + sorted(arr.reshape((-1, 6)), key=lambda i: [i[0], i[1]]) + ) else: basline_arr = np.array(baseline[key].reshape((-1, 1))) arr = np.array(arr.reshape((-1, 1))) self.assertTrue( basline_arr.shape == arr.shape, - "The output shapes are not equal, the baseline shape is " + - str(basline_arr.shape) + ', but got ' + str(arr.shape)) + "The output shapes are not equal, the baseline shape is " + + str(basline_arr.shape) + + ', but got ' + + str(arr.shape), + ) diff = abs(basline_arr - arr) np.testing.assert_allclose( basline_arr, @@ -187,7 +213,9 @@ class TrtConvertMulticlassNMSTest(TrtLayerAutoScanTest): rtol=rtol, atol=atol, err_msg='Output has diff, Maximum absolute error: {}'.format( - np.amax(diff))) + np.amax(diff) + ), + ) def assert_op_size(self, trt_engine_num, paddle_op_num): # tensorrt op num is not consistent with paddle diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms3.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms3.py index 7b743b00b3d..3a04993f2cb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms3.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms3.py @@ -22,7 +22,6 @@ import unittest class TrtConvertMulticlassNMS3Test(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,17 +38,21 @@ class TrtConvertMulticlassNMS3Test(TrtLayerAutoScanTest): min_subgraph_size=self.trt_param.min_subgraph_size, precision_mode=self.trt_param.precision, use_static=self.trt_param.use_static, - use_calib_mode=self.trt_param.use_calib_mode) - if len(self.dynamic_shape.min_input_shape - ) != 0 and self.dynamic_shape.min_input_shape.keys( - ) == self.dynamic_shape.max_input_shape.keys( - ) and self.dynamic_shape.min_input_shape.keys( - ) == self.dynamic_shape.opt_input_shape.keys(): + use_calib_mode=self.trt_param.use_calib_mode, + ) + if ( + len(self.dynamic_shape.min_input_shape) != 0 + and self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.max_input_shape.keys() + and self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.opt_input_shape.keys() + ): config.set_trt_dynamic_shape_info( self.dynamic_shape.min_input_shape, self.dynamic_shape.max_input_shape, self.dynamic_shape.opt_input_shape, - self.dynamic_shape.disable_trt_plugin_fp16) + self.dynamic_shape.disable_trt_plugin_fp16, + ) return config else: config = paddle_infer.Config() @@ -59,15 +62,15 @@ class TrtConvertMulticlassNMS3Test(TrtLayerAutoScanTest): return config def sample_program_configs(self): - def generate_boxes(batch, num_boxes): - return np.arange(batch * num_boxes * 4, - dtype=np.float32).reshape([batch, num_boxes, 4]) + return np.arange(batch * num_boxes * 4, dtype=np.float32).reshape( + [batch, num_boxes, 4] + ) def generate_scores(batch, num_boxes, num_classes): - return np.arange(batch * num_classes * num_boxes, - dtype=np.float32).reshape( - [batch, num_classes, num_boxes]) + return np.arange( + batch * num_classes * num_boxes, dtype=np.float32 + ).reshape([batch, num_classes, num_boxes]) # return np.random.rand(batch, num_classes, num_boxes).astype(np.float32) for batch in [1, 2]: @@ -76,51 +79,61 @@ class TrtConvertMulticlassNMS3Test(TrtLayerAutoScanTest): for num_boxes, num_classes in [[80, 100], [40, 200], [20, 400]]: self.num_boxes, self.num_classes = num_boxes, num_classes for score_threshold in [ - 0.01, + 0.01, ]: - ops_config = [{ - "op_type": "multiclass_nms3", - "op_inputs": { - "BBoxes": ["input_bboxes"], - "Scores": ["input_scores"], - }, - "op_outputs": { - "Out": ["nms_output_boxes"], - "Index": ["nms_output_index"], - "NmsRoisNum": ["nms_output_num"] - }, - "op_attrs": { - "background_label": -1, - "score_threshold": score_threshold, - "nms_top_k": num_boxes, - "keep_top_k": num_boxes, - "nms_threshold": 0.3, - "normalized": False, - "nms_eta": nms_eta + ops_config = [ + { + "op_type": "multiclass_nms3", + "op_inputs": { + "BBoxes": ["input_bboxes"], + "Scores": ["input_scores"], + }, + "op_outputs": { + "Out": ["nms_output_boxes"], + "Index": ["nms_output_index"], + "NmsRoisNum": ["nms_output_num"], + }, + "op_attrs": { + "background_label": -1, + "score_threshold": score_threshold, + "nms_top_k": num_boxes, + "keep_top_k": num_boxes, + "nms_threshold": 0.3, + "normalized": False, + "nms_eta": nms_eta, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_bboxes": - TensorConfig(data_gen=partial( - generate_boxes, batch, num_boxes)), - "input_scores": - TensorConfig( - data_gen=partial(generate_scores, batch, - num_boxes, num_classes)) + "input_bboxes": TensorConfig( + data_gen=partial( + generate_boxes, batch, num_boxes + ) + ), + "input_scores": TensorConfig( + data_gen=partial( + generate_scores, + batch, + num_boxes, + num_classes, + ) + ), }, outputs=[ - "nms_output_boxes", "nms_output_num", - "nms_output_index" - ]) + "nms_output_boxes", + "nms_output_num", + "nms_output_index", + ], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The last dim of input_bboxes should be static. self.dynamic_shape.min_input_shape = { @@ -152,41 +165,55 @@ class TrtConvertMulticlassNMS3Test(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 # self.trt_param.precision = paddle_infer.PrecisionType.Half # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, True), (1e-2, 1e-2) - def assert_tensors_near(self, atol: float, rtol: float, - tensor: Dict[str, np.array], - baseline: Dict[str, np.array]): + def assert_tensors_near( + self, + atol: float, + rtol: float, + tensor: Dict[str, np.array], + baseline: Dict[str, np.array], + ): # the order of tensorrt outputs are not consistent with paddle for key, arr in tensor.items(): if key == "nms_output_index": continue if key == "nms_output_boxes": basline_arr = np.array( - sorted(baseline[key].reshape((-1, 6)), - key=lambda i: [i[0], i[1]])) + sorted( + baseline[key].reshape((-1, 6)), + key=lambda i: [i[0], i[1]], + ) + ) arr = np.array( - sorted(arr.reshape((-1, 6)), key=lambda i: [i[0], i[1]])) + sorted(arr.reshape((-1, 6)), key=lambda i: [i[0], i[1]]) + ) else: basline_arr = np.array(baseline[key].reshape((-1, 1))) arr = np.array(arr.reshape((-1, 1))) self.assertTrue( basline_arr.shape == arr.shape, - "The output shapes are not equal, the baseline shape is " + - str(basline_arr.shape) + ', but got ' + str(arr.shape)) + "The output shapes are not equal, the baseline shape is " + + str(basline_arr.shape) + + ', but got ' + + str(arr.shape), + ) diff = abs(basline_arr - arr) np.testing.assert_allclose( basline_arr, @@ -194,7 +221,9 @@ class TrtConvertMulticlassNMS3Test(TrtLayerAutoScanTest): rtol=rtol, atol=atol, err_msg='Output has diff, Maximum absolute error: {}'.format( - np.amax(diff))) + np.amax(diff) + ), + ) def assert_op_size(self, trt_engine_num, paddle_op_num): # tensorrt op num is not consistent with paddle diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multihead_matmul.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multihead_matmul.py index a580828943b..db02ce1e0f4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multihead_matmul.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multihead_matmul.py @@ -22,12 +22,10 @@ from typing import List class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch, dim1): return np.random.random((batch, dim1, 768)).astype(np.float32) @@ -44,103 +42,86 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): self.batch = batch for reshape_shape in [[0, 0, 12, 64]]: for dim1 in [128]: - input2_shapes = [[batch, reshape_shape[2], dim1, dim1], - [batch, 1, 1, dim1]] + input2_shapes = [ + [batch, reshape_shape[2], dim1, dim1], + [batch, 1, 1, dim1], + ] for input2_shape in input2_shapes: for axis in [0]: - dics = [{ - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }, { - "axis": 2 - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }, { - "axis": 2 - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }, { - "axis": 2 - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "scale": 0.125, - "bias": 0.0, - "bias_after_scale": True - }, { - "alpha": 1.0, - "transpose_X": False, - "transpose_Y": True, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }, { - "axis": axis - }, { - "axis": -1, - "is_test": True - }, { - "seed": 0, - "dropout_prob": 0.10000000149011612, - "dropout_implementation": "upscale_in_train", - "fix_seed": False, - "is_test": True - }, { - "alpha": 1.0, - "transpose_X": False, - "transpose_Y": False, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }, { - "axis": [0, 2, 1, 3] - }, { - "shape": [0, 0, 768] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }] + dics = [ + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + {"axis": 2}, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + {"axis": 2}, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + {"axis": 2}, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + { + "scale": 0.125, + "bias": 0.0, + "bias_after_scale": True, + }, + { + "alpha": 1.0, + "transpose_X": False, + "transpose_Y": True, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + {"axis": axis}, + {"axis": -1, "is_test": True}, + { + "seed": 0, + "dropout_prob": 0.10000000149011612, + "dropout_implementation": "upscale_in_train", + "fix_seed": False, + "is_test": True, + }, + { + "alpha": 1.0, + "transpose_X": False, + "transpose_Y": False, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + {"axis": [0, 2, 1, 3]}, + {"shape": [0, 0, 768]}, + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + ] ops_config = [ { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul1_weight"] + "Y": ["mul1_weight"], }, - "op_outputs": { - "Out": ["mul1_output"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["mul1_output"]}, + "op_attrs": dics[0], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul1_output"], - "Y": ["elementwise_add1_weight"] + "Y": ["elementwise_add1_weight"], }, "op_outputs": { "Out": ["elementwise_add1_output"] }, - "op_attrs": dics[1] + "op_attrs": dics[1], }, { "op_type": "reshape2", @@ -149,42 +130,38 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): }, "op_outputs": { "Out": ["reshape21_output"], - "XShape": ["reshape21_output_xshape"] + "XShape": ["reshape21_output_xshape"], }, - "op_attrs": dics[2] + "op_attrs": dics[2], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape21_output"] - }, + "op_inputs": {"X": ["reshape21_output"]}, "op_outputs": { "Out": ["transpose21_output"], - "XShape": ["transpose21_output_xshape"] + "XShape": ["transpose21_output_xshape"], }, - "op_attrs": dics[3] + "op_attrs": dics[3], }, { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul2_weight"] + "Y": ["mul2_weight"], }, - "op_outputs": { - "Out": ["mul2_output"] - }, - "op_attrs": dics[4] + "op_outputs": {"Out": ["mul2_output"]}, + "op_attrs": dics[4], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul2_output"], - "Y": ["elementwise_add2_weight"] + "Y": ["elementwise_add2_weight"], }, "op_outputs": { "Out": ["elementwise_add2_output"] }, - "op_attrs": dics[5] + "op_attrs": dics[5], }, { "op_type": "reshape2", @@ -193,42 +170,38 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): }, "op_outputs": { "Out": ["reshape22_output"], - "XShape": ["reshape22_output_xshape"] + "XShape": ["reshape22_output_xshape"], }, - "op_attrs": dics[6] + "op_attrs": dics[6], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape22_output"] - }, + "op_inputs": {"X": ["reshape22_output"]}, "op_outputs": { "Out": ["transpose22_output"], - "XShape": ["transpose22_output_xshape"] + "XShape": ["transpose22_output_xshape"], }, - "op_attrs": dics[7] + "op_attrs": dics[7], }, { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul3_weight"] + "Y": ["mul3_weight"], }, - "op_outputs": { - "Out": ["mul3_output"] - }, - "op_attrs": dics[8] + "op_outputs": {"Out": ["mul3_output"]}, + "op_attrs": dics[8], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul3_output"], - "Y": ["elementwise_add3_weight"] + "Y": ["elementwise_add3_weight"], }, "op_outputs": { "Out": ["elementwise_add3_output"] }, - "op_attrs": dics[9] + "op_attrs": dics[9], }, { "op_type": "reshape2", @@ -237,30 +210,26 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): }, "op_outputs": { "Out": ["reshape23_output"], - "XShape": ["reshape23_output_xshape"] + "XShape": ["reshape23_output_xshape"], }, - "op_attrs": dics[10] + "op_attrs": dics[10], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape23_output"] - }, + "op_inputs": {"X": ["reshape23_output"]}, "op_outputs": { "Out": ["transpose23_output"], - "XShape": ["transpose23_output_xshape"] + "XShape": ["transpose23_output_xshape"], }, - "op_attrs": dics[11] + "op_attrs": dics[11], }, { "op_type": "scale", "op_inputs": { "X": ["transpose23_output"], }, - "op_outputs": { - "Out": ["scale_output"] - }, - "op_attrs": dics[12] + "op_outputs": {"Out": ["scale_output"]}, + "op_attrs": dics[12], }, { "op_type": "matmul", @@ -268,41 +237,35 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): "X": ["scale_output"], "Y": ["transpose22_output"], }, - "op_outputs": { - "Out": ["matmul1_output"] - }, - "op_attrs": dics[13] + "op_outputs": {"Out": ["matmul1_output"]}, + "op_attrs": dics[13], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["matmul1_output"], - "Y": ["input_data2"] + "Y": ["input_data2"], }, "op_outputs": { "Out": ["elementwise_add4_output"] }, - "op_attrs": dics[14] + "op_attrs": dics[14], }, { "op_type": "softmax", "op_inputs": { "X": ["elementwise_add4_output"] }, - "op_outputs": { - "Out": ["softmax_output"] - }, - "op_attrs": dics[15] + "op_outputs": {"Out": ["softmax_output"]}, + "op_attrs": dics[15], }, { "op_type": "dropout", "op_inputs": { "X": ["softmax_output"], }, - "op_outputs": { - "Out": ["dropout3_output"] - }, - "op_attrs": dics[16] + "op_outputs": {"Out": ["dropout3_output"]}, + "op_attrs": dics[16], }, { "op_type": "matmul", @@ -310,32 +273,26 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): "X": ["dropout3_output"], "Y": ["transpose21_output"], }, - "op_outputs": { - "Out": ["matmul2_output"] - }, - "op_attrs": dics[17] + "op_outputs": {"Out": ["matmul2_output"]}, + "op_attrs": dics[17], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["matmul2_output"] - }, + "op_inputs": {"X": ["matmul2_output"]}, "op_outputs": { "Out": ["transpose24_output"], - "XShape": ["transpose24_output_xshape"] + "XShape": ["transpose24_output_xshape"], }, - "op_attrs": dics[18] + "op_attrs": dics[18], }, { "op_type": "reshape2", - "op_inputs": { - "X": ["transpose24_output"] - }, + "op_inputs": {"X": ["transpose24_output"]}, "op_outputs": { "Out": ["reshape24_output"], - "XShape": ["reshape24_output_xshape"] + "XShape": ["reshape24_output_xshape"], }, - "op_attrs": dics[19] + "op_attrs": dics[19], }, # In order to fuse ops with # multihead_matmul_fuse_pass_v2, the last op @@ -344,72 +301,75 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): "op_type": "mul", "op_inputs": { "X": ["reshape24_output"], - "Y": ["mul4_weight"] + "Y": ["mul4_weight"], }, - "op_outputs": { - "Out": ["mul4_output"] - }, - "op_attrs": dics[20] - } + "op_outputs": {"Out": ["mul4_output"]}, + "op_attrs": dics[20], + }, ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "mul1_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul2_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul3_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul4_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "elementwise_add1_weight": - TensorConfig( - data_gen=partial(generate_weight2)), - "elementwise_add2_weight": - TensorConfig( - data_gen=partial(generate_weight2)), - "elementwise_add3_weight": - TensorConfig( - data_gen=partial(generate_weight2)), + "mul1_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul2_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul3_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul4_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "elementwise_add1_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), + "elementwise_add2_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), + "elementwise_add3_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial( - generate_input1, batch, dim1)), - "input_data2": - TensorConfig(data_gen=partial( - generate_input2, input2_shape)), + "input_data1": TensorConfig( + data_gen=partial( + generate_input1, batch, dim1 + ) + ), + "input_data2": TensorConfig( + data_gen=partial( + generate_input2, input2_shape + ) + ), }, - outputs=["mul4_output"]) + outputs=["mul4_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The last dim of input1 and input2 should be static. self.dynamic_shape.min_input_shape = { "input_data1": [1, 8, 768], "input_data2": [1, 1, 1, 128], - "reshape24_output": [1, 128, 768] + "reshape24_output": [1, 128, 768], } self.dynamic_shape.max_input_shape = { "input_data1": [16, 512, 768], "input_data2": [16, 256, 512, 128], - "reshape24_output": [1, 128, 768] + "reshape24_output": [1, 128, 768], } self.dynamic_shape.opt_input_shape = { "input_data1": [8, 128, 768], "input_data2": [8, 32, 64, 128], - "reshape24_output": [1, 128, 768] + "reshape24_output": [1, 128, 768], } def clear_dynamic_shape(): @@ -438,25 +398,30 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): yield self.create_inference_config(), (1, 3), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.trt_param.precision == paddle_infer.PrecisionType.Half: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "The output has diff between gpu and trt in fp16 mode.") + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "The output has diff between gpu and trt in fp16 mode.", + ) def teller2(program_config, predictor_config): - if self.trt_param.precision == paddle_infer.PrecisionType.Float32 and len( - self.dynamic_shape.min_input_shape) != 0 and self.batch > 2: + if ( + self.trt_param.precision == paddle_infer.PrecisionType.Float32 + and len(self.dynamic_shape.min_input_shape) != 0 + and self.batch > 2 + ): return True return False self.add_skip_case( - teller2, SkipReasons.TRT_NOT_IMPLEMENTED, - "The output has diff between gpu and trt when dynamic fp32 mode and batch size > 2." + teller2, + SkipReasons.TRT_NOT_IMPLEMENTED, + "The output has diff between gpu and trt when dynamic fp32 mode and batch size > 2.", ) def teller3(program_config, predictor_config): @@ -465,8 +430,10 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): return False self.add_skip_case( - teller3, SkipReasons.TRT_NOT_IMPLEMENTED, - "The output has diff between gpu and trt in int8 mode.") + teller3, + SkipReasons.TRT_NOT_IMPLEMENTED, + "The output has diff between gpu and trt in int8 mode.", + ) def test(self): self.add_skip_trt_case() @@ -474,9 +441,7 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): - def sample_program_configs(self): - def generate_input1(batch, dim1): return np.random.random((batch, dim1, 768)).astype(np.float32) @@ -493,112 +458,110 @@ class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): self.batch = batch for reshape_shape in [[0, 0, 12, 64]]: for dim1 in [128]: - input2_shapes = [[batch, reshape_shape[2], dim1, dim1], - [batch, 1, 1, dim1]] + input2_shapes = [ + [batch, reshape_shape[2], dim1, dim1], + [batch, 1, 1, dim1], + ] for input2_shape in input2_shapes: for axis in [0]: - dics = [{ - "x_num_col_dims": 2, - "y_num_col_dims": 1, - "enable_int8": True, - "Input_scale": 1.0, - }, { - "axis": 2, - "out_threshold": 1.0, - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1, - "enable_int8": True, - "Input_scale": 1.0, - }, { - "axis": 2, - "out_threshold": 1.0, - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1, - "enable_int8": True, - "Input_scale": 1.0, - }, { - "axis": 2, - "out_threshold": 1.0, - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "scale": 0.125, - "bias": 0.0, - "bias_after_scale": True - }, { - "alpha": 1.0, - "transpose_X": False, - "transpose_Y": True, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }, { - "axis": axis - }, { - "axis": -1, - "is_test": True - }, { - "seed": 0, - "dropout_prob": 0.10000000149011612, - "dropout_implementation": "upscale_in_train", - "fix_seed": False, - "is_test": True - }, { - "alpha": 1.0, - "transpose_X": False, - "transpose_Y": False, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }, { - "axis": [0, 2, 1, 3] - }, { - "shape": [0, 0, 768] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }] + dics = [ + { + "x_num_col_dims": 2, + "y_num_col_dims": 1, + "enable_int8": True, + "Input_scale": 1.0, + }, + { + "axis": 2, + "out_threshold": 1.0, + }, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + { + "x_num_col_dims": 2, + "y_num_col_dims": 1, + "enable_int8": True, + "Input_scale": 1.0, + }, + { + "axis": 2, + "out_threshold": 1.0, + }, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + { + "x_num_col_dims": 2, + "y_num_col_dims": 1, + "enable_int8": True, + "Input_scale": 1.0, + }, + { + "axis": 2, + "out_threshold": 1.0, + }, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + { + "scale": 0.125, + "bias": 0.0, + "bias_after_scale": True, + }, + { + "alpha": 1.0, + "transpose_X": False, + "transpose_Y": True, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + {"axis": axis}, + {"axis": -1, "is_test": True}, + { + "seed": 0, + "dropout_prob": 0.10000000149011612, + "dropout_implementation": "upscale_in_train", + "fix_seed": False, + "is_test": True, + }, + { + "alpha": 1.0, + "transpose_X": False, + "transpose_Y": False, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + {"axis": [0, 2, 1, 3]}, + {"shape": [0, 0, 768]}, + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + ] ops_config = [ { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul1_weight"] - }, - "op_outputs": { - "Out": ["mul1_output"] + "Y": ["mul1_weight"], }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["mul1_output"]}, + "op_attrs": dics[0], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul1_output"], - "Y": ["elementwise_add1_weight"] + "Y": ["elementwise_add1_weight"], }, "op_outputs": { "Out": ["elementwise_add1_output"] }, - "op_attrs": dics[1] + "op_attrs": dics[1], }, { "op_type": "reshape2", @@ -607,42 +570,38 @@ class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): }, "op_outputs": { "Out": ["reshape21_output"], - "XShape": ["reshape21_output_xshape"] + "XShape": ["reshape21_output_xshape"], }, - "op_attrs": dics[2] + "op_attrs": dics[2], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape21_output"] - }, + "op_inputs": {"X": ["reshape21_output"]}, "op_outputs": { "Out": ["transpose21_output"], - "XShape": ["transpose21_output_xshape"] + "XShape": ["transpose21_output_xshape"], }, - "op_attrs": dics[3] + "op_attrs": dics[3], }, { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul2_weight"] - }, - "op_outputs": { - "Out": ["mul2_output"] + "Y": ["mul2_weight"], }, - "op_attrs": dics[4] + "op_outputs": {"Out": ["mul2_output"]}, + "op_attrs": dics[4], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul2_output"], - "Y": ["elementwise_add2_weight"] + "Y": ["elementwise_add2_weight"], }, "op_outputs": { "Out": ["elementwise_add2_output"] }, - "op_attrs": dics[5] + "op_attrs": dics[5], }, { "op_type": "reshape2", @@ -651,42 +610,38 @@ class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): }, "op_outputs": { "Out": ["reshape22_output"], - "XShape": ["reshape22_output_xshape"] + "XShape": ["reshape22_output_xshape"], }, - "op_attrs": dics[6] + "op_attrs": dics[6], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape22_output"] - }, + "op_inputs": {"X": ["reshape22_output"]}, "op_outputs": { "Out": ["transpose22_output"], - "XShape": ["transpose22_output_xshape"] + "XShape": ["transpose22_output_xshape"], }, - "op_attrs": dics[7] + "op_attrs": dics[7], }, { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul3_weight"] - }, - "op_outputs": { - "Out": ["mul3_output"] + "Y": ["mul3_weight"], }, - "op_attrs": dics[8] + "op_outputs": {"Out": ["mul3_output"]}, + "op_attrs": dics[8], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul3_output"], - "Y": ["elementwise_add3_weight"] + "Y": ["elementwise_add3_weight"], }, "op_outputs": { "Out": ["elementwise_add3_output"] }, - "op_attrs": dics[9] + "op_attrs": dics[9], }, { "op_type": "reshape2", @@ -695,30 +650,26 @@ class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): }, "op_outputs": { "Out": ["reshape23_output"], - "XShape": ["reshape23_output_xshape"] + "XShape": ["reshape23_output_xshape"], }, - "op_attrs": dics[10] + "op_attrs": dics[10], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape23_output"] - }, + "op_inputs": {"X": ["reshape23_output"]}, "op_outputs": { "Out": ["transpose23_output"], - "XShape": ["transpose23_output_xshape"] + "XShape": ["transpose23_output_xshape"], }, - "op_attrs": dics[11] + "op_attrs": dics[11], }, { "op_type": "scale", "op_inputs": { "X": ["transpose23_output"], }, - "op_outputs": { - "Out": ["scale_output"] - }, - "op_attrs": dics[12] + "op_outputs": {"Out": ["scale_output"]}, + "op_attrs": dics[12], }, { "op_type": "matmul", @@ -726,41 +677,35 @@ class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): "X": ["scale_output"], "Y": ["transpose22_output"], }, - "op_outputs": { - "Out": ["matmul1_output"] - }, - "op_attrs": dics[13] + "op_outputs": {"Out": ["matmul1_output"]}, + "op_attrs": dics[13], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["matmul1_output"], - "Y": ["input_data2"] + "Y": ["input_data2"], }, "op_outputs": { "Out": ["elementwise_add4_output"] }, - "op_attrs": dics[14] + "op_attrs": dics[14], }, { "op_type": "softmax", "op_inputs": { "X": ["elementwise_add4_output"] }, - "op_outputs": { - "Out": ["softmax_output"] - }, - "op_attrs": dics[15] + "op_outputs": {"Out": ["softmax_output"]}, + "op_attrs": dics[15], }, { "op_type": "dropout", "op_inputs": { "X": ["softmax_output"], }, - "op_outputs": { - "Out": ["dropout3_output"] - }, - "op_attrs": dics[16] + "op_outputs": {"Out": ["dropout3_output"]}, + "op_attrs": dics[16], }, { "op_type": "matmul", @@ -768,32 +713,26 @@ class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): "X": ["dropout3_output"], "Y": ["transpose21_output"], }, - "op_outputs": { - "Out": ["matmul2_output"] - }, - "op_attrs": dics[17] + "op_outputs": {"Out": ["matmul2_output"]}, + "op_attrs": dics[17], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["matmul2_output"] - }, + "op_inputs": {"X": ["matmul2_output"]}, "op_outputs": { "Out": ["transpose24_output"], - "XShape": ["transpose24_output_xshape"] + "XShape": ["transpose24_output_xshape"], }, - "op_attrs": dics[18] + "op_attrs": dics[18], }, { "op_type": "reshape2", - "op_inputs": { - "X": ["transpose24_output"] - }, + "op_inputs": {"X": ["transpose24_output"]}, "op_outputs": { "Out": ["reshape24_output"], - "XShape": ["reshape24_output_xshape"] + "XShape": ["reshape24_output_xshape"], }, - "op_attrs": dics[19] + "op_attrs": dics[19], }, # In order to fuse ops with # multihead_matmul_fuse_pass_v2, the last op @@ -802,61 +741,62 @@ class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): "op_type": "mul", "op_inputs": { "X": ["reshape24_output"], - "Y": ["mul4_weight"] - }, - "op_outputs": { - "Out": ["mul4_output"] + "Y": ["mul4_weight"], }, - "op_attrs": dics[20] - } + "op_outputs": {"Out": ["mul4_output"]}, + "op_attrs": dics[20], + }, ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "mul1_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul2_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul3_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul4_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "elementwise_add1_weight": - TensorConfig( - data_gen=partial(generate_weight2)), - "elementwise_add2_weight": - TensorConfig( - data_gen=partial(generate_weight2)), - "elementwise_add3_weight": - TensorConfig( - data_gen=partial(generate_weight2)), + "mul1_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul2_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul3_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul4_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "elementwise_add1_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), + "elementwise_add2_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), + "elementwise_add3_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial( - generate_input1, batch, dim1)), - "input_data2": - TensorConfig(data_gen=partial( - generate_input2, input2_shape)), + "input_data1": TensorConfig( + data_gen=partial( + generate_input1, batch, dim1 + ) + ), + "input_data2": TensorConfig( + data_gen=partial( + generate_input2, input2_shape + ) + ), }, - outputs=["mul4_output"]) + outputs=["mul4_output"], + ) yield program_config class TrtConvertVitToMultiHeadMatmulTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch, length): return np.zeros((batch, length, 768), dtype=np.float32) @@ -870,216 +810,190 @@ class TrtConvertVitToMultiHeadMatmulTest(TrtLayerAutoScanTest): self.batch = batch for length in [64, 384]: self.length = length - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input_data1"], - "Y": ["matmul1_weight"] - }, - "op_outputs": { - "Out": ["matmul1_output"] - }, - "op_attrs": { - "trans_x": False, - "trans_y": False - } - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["matmul1_output"], - "Y": ["elementwise_add1_weight"] - }, - "op_outputs": { - "Out": ["elementwise_add1_output"] + ops_config = [ + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input_data1"], + "Y": ["matmul1_weight"], + }, + "op_outputs": {"Out": ["matmul1_output"]}, + "op_attrs": {"trans_x": False, "trans_y": False}, }, - "op_attrs": { - "Scale_out": 1.0, - "Scale_x": 1.0, - "Scale_y": 1.0, - "axis": 2 - } - }, { - "op_type": "reshape2", - "op_inputs": { - "X": ["elementwise_add1_output"], + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["matmul1_output"], + "Y": ["elementwise_add1_weight"], + }, + "op_outputs": {"Out": ["elementwise_add1_output"]}, + "op_attrs": { + "Scale_out": 1.0, + "Scale_x": 1.0, + "Scale_y": 1.0, + "axis": 2, + }, }, - "op_outputs": { - "Out": ["reshape1_output"], - "XShape": ["reshape1_output_xshape"] + { + "op_type": "reshape2", + "op_inputs": { + "X": ["elementwise_add1_output"], + }, + "op_outputs": { + "Out": ["reshape1_output"], + "XShape": ["reshape1_output_xshape"], + }, + "op_attrs": {"shape": [-1, self.length, 3, 12, 64]}, }, - "op_attrs": { - "shape": [-1, self.length, 3, 12, 64] - } - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["reshape1_output"] + { + "op_type": "transpose2", + "op_inputs": {"X": ["reshape1_output"]}, + "op_outputs": { + "Out": ["transpose1_output"], + "XShape": ["transpose1_output_xshape"], + }, + "op_attrs": { + "axis": [2, 0, 3, 1, 4], + "data_format": "AnyLayout", + }, }, - "op_outputs": { - "Out": ["transpose1_output"], - "XShape": ["transpose1_output_xshape"] + { + "op_type": "slice", + "op_inputs": { + "Input": ["transpose1_output"], + }, + "op_outputs": {"Out": ["slice1_output"]}, + "op_attrs": { + "axes": [0], + "starts": [0], + "ends": [1], + "decrease_axis": [0], + "infer_flags": [1], + }, }, - "op_attrs": { - "axis": [2, 0, 3, 1, 4], - "data_format": "AnyLayout" - } - }, { - "op_type": "slice", - "op_inputs": { - "Input": ["transpose1_output"], + { + "op_type": "slice", + "op_inputs": { + "Input": ["transpose1_output"], + }, + "op_outputs": {"Out": ["slice2_output"]}, + "op_attrs": { + "axes": [0], + "starts": [1], + "ends": [2], + "decrease_axis": [0], + "infer_flags": [1], + }, }, - "op_outputs": { - "Out": ["slice1_output"] + { + "op_type": "slice", + "op_inputs": { + "Input": ["transpose1_output"], + }, + "op_outputs": {"Out": ["slice3_output"]}, + "op_attrs": { + "axes": [0], + "starts": [2], + "ends": [3], + "decrease_axis": [0], + "infer_flags": [1], + }, }, - "op_attrs": { - "axes": [0], - "starts": [0], - "ends": [1], - "decrease_axis": [0], - "infer_flags": [1] - } - }, { - "op_type": "slice", - "op_inputs": { - "Input": ["transpose1_output"], + { + "op_type": "transpose2", + "op_inputs": {"X": ["slice2_output"]}, + "op_outputs": { + "Out": ["transpose2_output"], + }, + "op_attrs": { + "axis": [0, 1, 3, 2], + "data_format": "AnyLayout", + }, }, - "op_outputs": { - "Out": ["slice2_output"] + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["slice1_output"], + "Y": ["transpose2_output"], + }, + "op_outputs": {"Out": ["matmul2_output"]}, + "op_attrs": {"trans_x": False, "trans_y": False}, }, - "op_attrs": { - "axes": [0], - "starts": [1], - "ends": [2], - "decrease_axis": [0], - "infer_flags": [1] - } - }, { - "op_type": "slice", - "op_inputs": { - "Input": ["transpose1_output"], + { + "op_type": "scale", + "op_inputs": { + "X": ["matmul2_output"], + }, + "op_outputs": {"Out": ["scale_output"]}, + "op_attrs": { + "scale": 0.125, + "bias": 0.0, + "bias_after_scale": True, + }, }, - "op_outputs": { - "Out": ["slice3_output"] + { + "op_type": "softmax", + "op_inputs": {"X": ["scale_output"]}, + "op_outputs": {"Out": ["softmax_output"]}, + "op_attrs": {"axis": -1, "data_format": "AnyLayout"}, }, - "op_attrs": { - "axes": [0], - "starts": [2], - "ends": [3], - "decrease_axis": [0], - "infer_flags": [1] - } - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["slice2_output"] + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["softmax_output"], + "Y": ["slice3_output"], + }, + "op_outputs": {"Out": ["matmul3_output"]}, + "op_attrs": {"trans_x": False, "trans_y": False}, }, - "op_outputs": { - "Out": ["transpose2_output"], + { + "op_type": "transpose2", + "op_inputs": {"X": ["matmul3_output"]}, + "op_outputs": { + "Out": ["transpose3_output"], + "XShape": ["transpose3_output_xshape"], + }, + "op_attrs": { + "axis": [0, 2, 1, 3], + "data_format": "AnyLayout", + }, }, - "op_attrs": { - "axis": [0, 1, 3, 2], - "data_format": "AnyLayout" - } - }, { - "op_type": "matmul_v2", - "op_inputs": { - "X": ["slice1_output"], - "Y": ["transpose2_output"] + { + "op_type": "reshape2", + "op_inputs": {"X": ["transpose3_output"]}, + "op_outputs": { + "Out": ["reshape2_output"], + "XShape": ["reshape2_output_xshape"], + }, + "op_attrs": {"shape": [-1, self.length, 768]}, }, - "op_outputs": { - "Out": ["matmul2_output"] - }, - "op_attrs": { - "trans_x": False, - "trans_y": False - } - }, { - "op_type": "scale", - "op_inputs": { - "X": ["matmul2_output"], - }, - "op_outputs": { - "Out": ["scale_output"] - }, - "op_attrs": { - "scale": 0.125, - "bias": 0.0, - "bias_after_scale": True - } - }, { - "op_type": "softmax", - "op_inputs": { - "X": ["scale_output"] - }, - "op_outputs": { - "Out": ["softmax_output"] - }, - "op_attrs": { - "axis": -1, - "data_format": "AnyLayout" - } - }, { - "op_type": "matmul_v2", - "op_inputs": { - "X": ["softmax_output"], - "Y": ["slice3_output"] - }, - "op_outputs": { - "Out": ["matmul3_output"] - }, - "op_attrs": { - "trans_x": False, - "trans_y": False - } - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["matmul3_output"] - }, - "op_outputs": { - "Out": ["transpose3_output"], - "XShape": ["transpose3_output_xshape"] - }, - "op_attrs": { - "axis": [0, 2, 1, 3], - "data_format": "AnyLayout" - } - }, { - "op_type": "reshape2", - "op_inputs": { - "X": ["transpose3_output"] - }, - "op_outputs": { - "Out": ["reshape2_output"], - "XShape": ["reshape2_output_xshape"] - }, - "op_attrs": { - "shape": [-1, self.length, 768] - } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "matmul1_weight": - TensorConfig(data_gen=partial(generate_weight1)), - "elementwise_add1_weight": - TensorConfig(data_gen=partial(generate_weight2)) + "matmul1_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "elementwise_add1_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ - "input_data1": - TensorConfig( - data_gen=partial(generate_input1, batch, length)) + "input_data1": TensorConfig( + data_gen=partial(generate_input1, batch, length) + ) }, - outputs=["reshape2_output"]) + outputs=["reshape2_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The last dim of input1 and input2 should be static. self.dynamic_shape.min_input_shape = { @@ -1111,11 +1025,15 @@ class TrtConvertVitToMultiHeadMatmulTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.workspace_size = 2013265920 self.trt_param.precision = paddle_infer.PrecisionType.Half - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) self.trt_param.precision = paddle_infer.PrecisionType.Float32 - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-5, - 1e-5) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-5, + 1e-5, + ) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp.py index 71893db5e5f..1df42024992 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp.py @@ -22,7 +22,6 @@ import unittest class TrtConvertNearestInterpTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -30,8 +29,9 @@ class TrtConvertNearestInterpTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if attrs[0]['scale'] <= 0 and (attrs[0]['out_h'] <= 0 - or attrs[0]['out_w'] <= 0): + if attrs[0]['scale'] <= 0 and ( + attrs[0]['out_h'] <= 0 or attrs[0]['out_w'] <= 0 + ): return False if (attrs[0]['out_h'] <= 0) ^ (attrs[0]['out_w'] <= 0): return False @@ -39,7 +39,6 @@ class TrtConvertNearestInterpTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 3, 64, 64]).astype(np.float32) @@ -49,42 +48,49 @@ class TrtConvertNearestInterpTest(TrtLayerAutoScanTest): for scale in [2.0, -1.0, 0.0]: for out_h in [32, 64, 128 - 32]: for out_w in [32, -32]: - dics = [{ - "data_layout": data_layout, - "interp_method": interp_method, - "align_corners": align_corners, - "scale": scale, - "out_h": out_h, - "out_w": out_w - }] - - ops_config = [{ - "op_type": "nearest_interp", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["nearest_interp_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_layout": data_layout, + "interp_method": interp_method, + "align_corners": align_corners, + "scale": scale, + "out_h": out_h, + "out_w": out_w, + } + ] + + ops_config = [ + { + "op_type": "nearest_interp", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": { + "Out": [ + "nearest_interp_output_data" + ] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dics + ) + ) }, - outputs=["nearest_interp_output_data"]) + outputs=["nearest_interp_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -106,33 +112,39 @@ class TrtConvertNearestInterpTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[0].attrs[ - 'scale'] <= 0 and self.dynamic_shape.min_input_shape: + if ( + program_config.ops[0].attrs['scale'] <= 0 + and self.dynamic_shape.min_input_shape + ): return True if program_config.ops[0].attrs['align_corners'] == True: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "NOT Implemented: we need to add support scale <= 0 in dynamic shape in the future" + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "NOT Implemented: we need to add support scale <= 0 in dynamic shape in the future", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp_v2.py index a8156e37f9b..775db8a3c59 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp_v2.py @@ -21,47 +21,44 @@ import unittest class TrtConvertNearestInterpV2Test(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(): return np.ones([1, 3, 32, 32]).astype(np.float32) - ops_config = [{ - "op_type": "nearest_interp_v2", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["interp_output_data"] - }, - "op_attrs": { - "data_layout": "NCHW", - "interp_method": "nearest", - "align_corners": False, - "align_mode": 1, - "scale": [2., 2.], - "out_d": 0, - "out_h": 0, - "out_w": 0 + ops_config = [ + { + "op_type": "nearest_interp_v2", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["interp_output_data"]}, + "op_attrs": { + "data_layout": "NCHW", + "interp_method": "nearest", + "align_corners": False, + "align_mode": 1, + "scale": [2.0, 2.0], + "out_d": 0, + "out_h": 0, + "out_w": 0, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={"input_data": TensorConfig(data_gen=generate_input)}, - outputs=["interp_output_data"]) + outputs=["interp_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -83,19 +80,23 @@ class TrtConvertNearestInterpV2Test(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad.py index dafb7896181..12262a83022 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad.py @@ -22,7 +22,6 @@ import unittest class TrtConvertPadTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -39,7 +38,6 @@ class TrtConvertPadTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 3, 64, 64]).astype(np.float32) @@ -47,37 +45,40 @@ class TrtConvertPadTest(TrtLayerAutoScanTest): return np.random.random([24, 3, 3, 3]).astype(np.float32) for pad_value in [0.0, 1.0, 2.0, -100, 100.0]: - for paddings in [[0, 0, 0, 0, 1, 1, 1, 1], [0, 0, 0, 0, 1, 2, 3, 4], - [0, 0, 1, 1, 1, 1, 1, 1], - [0, 0, 0, 0, -1, -1, 1, 1]]: + for paddings in [ + [0, 0, 0, 0, 1, 1, 1, 1], + [0, 0, 0, 0, 1, 2, 3, 4], + [0, 0, 1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, -1, -1, 1, 1], + ]: dics = [{"pad_value": pad_value, "paddings": paddings}, {}] - ops_config = [{ - "op_type": "pad", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["pad_output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "pad", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["pad_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["pad_output_data"]) + outputs=["pad_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -102,22 +103,25 @@ class TrtConvertPadTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): for x in range(len(program_config.ops[0].attrs['paddings']) - 4): if program_config.ops[0].attrs['paddings'][x] != 0: @@ -125,8 +129,9 @@ class TrtConvertPadTest(TrtLayerAutoScanTest): return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "NOT Implemented: we need to add support pad not only inplement on h or w, such as paddings = [0, 0, 1, 1, 1, 1, 1, 1]" + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "NOT Implemented: we need to add support pad not only inplement on h or w, such as paddings = [0, 0, 1, 1, 1, 1, 1, 1]", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad3d.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad3d.py index d44da5534e6..02429bed44c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad3d.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad3d.py @@ -22,30 +22,30 @@ import unittest class TrtConvertPad3d(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.ones([1, 1, 3, 64, 64]).astype(np.float32) for value in [True, False]: - for paddings in [[0, 0, 0, 0, 1, 1], [0, 0, 1, 2, 3, 4], - [1, 1, 1, 1, 1, 1], [0, 0, -1, -1, 1, 1]]: + for paddings in [ + [0, 0, 0, 0, 1, 1], + [0, 0, 1, 2, 3, 4], + [1, 1, 1, 1, 1, 1], + [0, 0, -1, -1, 1, 1], + ]: dics = [{"value": value, "paddings": paddings}, {}] - ops_config = [{ - "op_type": "pad3d", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "pad3d", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) for i in range(10): @@ -53,16 +53,18 @@ class TrtConvertPad3d(TrtLayerAutoScanTest): ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 1, 3, 64, 64] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pool2d.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pool2d.py index 49d24e863ca..b4eacbb136f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pool2d.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pool2d.py @@ -23,7 +23,6 @@ import itertools class TrtConvertPool2dTest(TrtLayerAutoScanTest): - def is_paddings_valid(self, program_config: ProgramConfig) -> bool: exclusive = program_config.ops[0].attrs['exclusive'] paddings = program_config.ops[0].attrs['paddings'] @@ -65,39 +64,54 @@ class TrtConvertPool2dTest(TrtLayerAutoScanTest): ceil_mode_options = [True, False] configurations = [ - strides_options, paddings_options, pooling_type_options, - padding_algorithm_options, ksize_options, data_format_options, - global_pooling_options, exclusive_options, adaptive_option, - ceil_mode_options + strides_options, + paddings_options, + pooling_type_options, + padding_algorithm_options, + ksize_options, + data_format_options, + global_pooling_options, + exclusive_options, + adaptive_option, + ceil_mode_options, ] - for (strides, paddings, pooling_type, padding_algorithm, ksize, - data_format, global_pooling, exclusive, adaptive, - ceil_mode) in itertools.product(*configurations): - - attrs = [{ - "strides": strides, - "paddings": paddings, - "pooling_type": pooling_type, - "padding_algorithm": padding_algorithm, - "ksize": ksize, - "data_format": data_format, - "global_pooling": global_pooling, - "exclusive": exclusive, - "adaptive": adaptive, - "ceil_mode": ceil_mode, - }] - - ops_config = [{ - "op_type": "pool2d", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": attrs[0] - }] + for ( + strides, + paddings, + pooling_type, + padding_algorithm, + ksize, + data_format, + global_pooling, + exclusive, + adaptive, + ceil_mode, + ) in itertools.product(*configurations): + + attrs = [ + { + "strides": strides, + "paddings": paddings, + "pooling_type": pooling_type, + "padding_algorithm": padding_algorithm, + "ksize": ksize, + "data_format": data_format, + "global_pooling": global_pooling, + "exclusive": exclusive, + "adaptive": adaptive, + "ceil_mode": ceil_mode, + } + ] + + ops_config = [ + { + "op_type": "pool2d", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": attrs[0], + } + ] ops = self.generate_op_config(ops_config) @@ -105,16 +119,18 @@ class TrtConvertPool2dTest(TrtLayerAutoScanTest): ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, attrs)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, attrs) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [1, 3, 64, 64]} @@ -136,34 +152,40 @@ class TrtConvertPool2dTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller(program_config, predictor_config): - if program_config.ops[0].attrs['pooling_type'] == 'avg' and \ - program_config.ops[0].attrs['global_pooling'] == False and \ - program_config.ops[0].attrs['exclusive'] == True and \ - program_config.ops[0].attrs['adaptive'] == False and \ - program_config.ops[0].attrs['ceil_mode'] == True: + if ( + program_config.ops[0].attrs['pooling_type'] == 'avg' + and program_config.ops[0].attrs['global_pooling'] == False + and program_config.ops[0].attrs['exclusive'] == True + and program_config.ops[0].attrs['adaptive'] == False + and program_config.ops[0].attrs['ceil_mode'] == True + ): return True return False self.add_skip_case( - teller, SkipReasons.TRT_NOT_IMPLEMENTED, - "The results of some cases are Nan, but the results of TensorRT and GPU are the same." + teller, + SkipReasons.TRT_NOT_IMPLEMENTED, + "The results of some cases are Nan, but the results of TensorRT and GPU are the same.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_bias.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_bias.py index 284fd9231d1..9390d12ca8b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_bias.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_bias.py @@ -22,7 +22,6 @@ import unittest class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,14 +31,13 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if 'begin_norm_axis' in attrs[0] and attrs[0]['begin_norm_axis'] >= 0: if len(inputs['inputX_data'].shape) <= attrs[0]['begin_norm_axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): return np.ones([batch, 128, 768]).astype(np.float32) @@ -56,96 +54,100 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): for epsilon in [1e-5]: for begin_norm_axis in [2]: for enable_int8 in [False, True]: - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis, - }, {}] - - ops_config = [{ - "op_type": "elementwise_add", - "op_inputs": { - "X": ["inputX_data"], - "Y": ["EleBias"] - }, - "op_outputs": { - "Out": ["bias_out"] + dics = [ + { + "epsilon": epsilon, + "begin_norm_axis": begin_norm_axis, }, - "op_attrs": { - "axis": -1 - } - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["bias_out"], - "Y": ["inputY_data"] + {}, + ] + + ops_config = [ + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["inputX_data"], + "Y": ["EleBias"], + }, + "op_outputs": {"Out": ["bias_out"]}, + "op_attrs": {"axis": -1}, }, - "op_outputs": { - "Out": ["ele_out"] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["bias_out"], + "Y": ["inputY_data"], + }, + "op_outputs": {"Out": ["ele_out"]}, + "op_attrs": {"axis": -1}, }, - "op_attrs": { - "axis": -1 - } - }, { - "op_type": "layer_norm", - "op_inputs": { - "X": ["ele_out"], - "Bias": ["Bias"], - "Scale": ["Scale"] + { + "op_type": "layer_norm", + "op_inputs": { + "X": ["ele_out"], + "Bias": ["Bias"], + "Scale": ["Scale"], + }, + "op_outputs": { + "Y": ["layernorm_out"], + "Mean": ["Mean"], + "Variance": ["Variance"], + }, + "op_attrs": dics[0], }, - "op_outputs": { - "Y": ["layernorm_out"], - "Mean": ["Mean"], - "Variance": ["Variance"] - }, - "op_attrs": dics[0] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "Bias": - TensorConfig( - data_gen=partial(generate_weight1, dics)), - "Scale": - TensorConfig( - data_gen=partial(generate_weight2, dics)), - "EleBias": - TensorConfig( - data_gen=partial(generate_weight2, dics)) + "Bias": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + "Scale": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), + "EleBias": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), }, inputs={ - "inputX_data": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "inputY_data": - TensorConfig(data_gen=partial( - generate_input2, dics, batch)) + "inputX_data": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "inputY_data": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), }, - outputs=["ele_out", "layernorm_out"]) + outputs=["ele_out", "layernorm_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.opt_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } def clear_dynamic_shape(): @@ -164,10 +166,12 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 # atol=1e-2 while rtol is 1e-8 + attrs, True + ), 1e-2 # atol=1e-2 while rtol is 1e-8 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 # atol=1e-2 while rtol is 1e-8 + attrs, True + ), 1e-2 # atol=1e-2 while rtol is 1e-8 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_no_bias.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_no_bias.py index 74582cffb71..320bdd1925c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_no_bias.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_no_bias.py @@ -22,7 +22,6 @@ import unittest class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,14 +31,13 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if 'begin_norm_axis' in attrs[0] and attrs[0]['begin_norm_axis'] >= 0: if len(inputs['inputX_data'].shape) <= attrs[0]['begin_norm_axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): return np.ones([batch, 128, 768]).astype(np.float32) @@ -56,81 +54,88 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): for epsilon in [1e-5]: for begin_norm_axis in [2]: for enable_int8 in [False, True]: - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis, - }, {}] - - ops_config = [{ - "op_type": "elementwise_add", - "op_inputs": { - "X": ["inputX_data"], - "Y": ["inputY_data"] + dics = [ + { + "epsilon": epsilon, + "begin_norm_axis": begin_norm_axis, }, - "op_outputs": { - "Out": ["ele_out"] + {}, + ] + + ops_config = [ + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["inputX_data"], + "Y": ["inputY_data"], + }, + "op_outputs": {"Out": ["ele_out"]}, + "op_attrs": {"axis": -1}, }, - "op_attrs": { - "axis": -1 - } - }, { - "op_type": "layer_norm", - "op_inputs": { - "X": ["ele_out"], - "Bias": ["Bias"], - "Scale": ["Scale"] + { + "op_type": "layer_norm", + "op_inputs": { + "X": ["ele_out"], + "Bias": ["Bias"], + "Scale": ["Scale"], + }, + "op_outputs": { + "Y": ["layernorm_out"], + "Mean": ["Mean"], + "Variance": ["Variance"], + }, + "op_attrs": dics[0], }, - "op_outputs": { - "Y": ["layernorm_out"], - "Mean": ["Mean"], - "Variance": ["Variance"] - }, - "op_attrs": dics[0] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "Bias": - TensorConfig( - data_gen=partial(generate_weight1, dics)), - "Scale": - TensorConfig( - data_gen=partial(generate_weight2, dics)) + "Bias": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + "Scale": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), }, inputs={ - "inputX_data": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "inputY_data": - TensorConfig(data_gen=partial( - generate_input2, dics, batch)) + "inputX_data": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "inputY_data": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), }, - outputs=["ele_out", "layernorm_out"]) + outputs=["ele_out", "layernorm_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.opt_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } def clear_dynamic_shape(): @@ -149,10 +154,12 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 # atol=1e-2 while rtol is 1e-8 + attrs, True + ), 1e-2 # atol=1e-2 while rtol is 1e-8 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 # atol=1e-2 while rtol is 1e-8 + attrs, True + ), 1e-2 # atol=1e-2 while rtol is 1e-8 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py index 0bc26954f0f..862bf1ff610 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py @@ -22,12 +22,10 @@ import unittest class TrtConvertPreluTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch, dim1, dim2, dim3): shape = [batch] if dim1 != 0: @@ -41,8 +39,10 @@ class TrtConvertPreluTest(TrtLayerAutoScanTest): def generate_alpha(attrs: List[Dict[str, Any]], dim1, dim2, dim3): if attrs[0]["mode"] == "all": return np.random.random(size=(1)).astype(np.float32) - elif attrs[0]["mode"] == "channel" and attrs[0][ - "data_format"] == "NCHW": + elif ( + attrs[0]["mode"] == "channel" + and attrs[0]["data_format"] == "NCHW" + ): shape = [1] if dim1 != 0: shape.append(dim1) @@ -51,8 +51,10 @@ class TrtConvertPreluTest(TrtLayerAutoScanTest): if dim3 != 0: shape.append(1) return np.random.random(size=shape).astype(np.float32) - elif attrs[0]["mode"] == "channel" and attrs[0][ - "data_format"] == "NHWC": + elif ( + attrs[0]["mode"] == "channel" + and attrs[0]["data_format"] == "NHWC" + ): shape = [1] if dim1 != 0: shape.append(1) @@ -86,48 +88,66 @@ class TrtConvertPreluTest(TrtLayerAutoScanTest): for mode in ["all", "channel", "element"]: for data_format in ['NCHW', 'NHWC']: - if mode == "channel" and dim1 == 0 and data_format == "NCHW": + if ( + mode == "channel" + and dim1 == 0 + and data_format == "NCHW" + ): continue - if mode == "channel" and dim3 == 0 and data_format == "NHWC": + if ( + mode == "channel" + and dim3 == 0 + and data_format == "NHWC" + ): continue - dics = [{ - "mode": mode, - "data_format": data_format - }] - ops_config = [{ - "op_type": "prelu", - "op_inputs": { - "X": ["input_data"], - "Alpha": ["alpha_weight"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + {"mode": mode, "data_format": data_format} + ] + ops_config = [ + { + "op_type": "prelu", + "op_inputs": { + "X": ["input_data"], + "Alpha": ["alpha_weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "alpha_weight": - TensorConfig(data_gen=partial( - generate_alpha, dics, dim1, dim2, - dim3)) + "alpha_weight": TensorConfig( + data_gen=partial( + generate_alpha, + dics, + dim1, + dim2, + dim3, + ) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input, batch, dim1, dim2, - dim3)), + "input_data": TensorConfig( + data_gen=partial( + generate_input, + batch, + dim1, + dim2, + dim3, + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dim1 == 0: self.dynamic_shape.min_input_shape = { @@ -181,7 +201,12 @@ class TrtConvertPreluTest(TrtLayerAutoScanTest): ] def generate_trt_nodes_num(attrs, dynamic_shape): - if not dynamic_shape and self.dim1 == 0 and self.dim2 == 0 and self.dim3 == 0: + if ( + not dynamic_shape + and self.dim1 == 0 + and self.dim2 == 0 + and self.dim3 == 0 + ): return 0, 3 return 1, 2 @@ -189,19 +214,23 @@ class TrtConvertPreluTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): ver = paddle_infer.get_trt_compile_version() @@ -213,8 +242,9 @@ class TrtConvertPreluTest(TrtLayerAutoScanTest): return False self.add_skip_case( - teller, SkipReasons.TRT_NOT_IMPLEMENTED, - "Need to repair the case: the output of GPU and tensorrt has diff in trt6, the prelu static plugin has bug." + teller, + SkipReasons.TRT_NOT_IMPLEMENTED, + "Need to repair the case: the output of GPU and tensorrt has diff in trt6, the prelu static plugin has bug.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py index 95c9e24dcad..78ad7978b06 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py @@ -22,7 +22,6 @@ import unittest class TrtConvertReduceMeanTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -45,7 +44,6 @@ class TrtConvertReduceMeanTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(dtype, attrs: List[Dict[str, Any]]): if dtype == -1 or dtype == 5: return np.random.random([1, 3, 64, 64]).astype(np.float32) @@ -53,39 +51,52 @@ class TrtConvertReduceMeanTest(TrtLayerAutoScanTest): return np.random.random([1, 3, 64, 64]).astype(np.int32) for keep_dim in [True, False]: - for dim in [[], [1], [0], [0, 1], [1, 2, 3], [-2, 0, 3], [-3], - [-4, 1], [3, 4, 5]]: + for dim in [ + [], + [1], + [0], + [0, 1], + [1, 2, 3], + [-2, 0, 3], + [-3], + [-4, 1], + [3, 4, 5], + ]: for reduce_all in [True, False]: for out_dtype in [-1, 2, 5]: - dics = [{ - "keep_dim": keep_dim, - "dim": dim, - "reduce_all": reduce_all, - "out_dtype": out_dtype, - "in_dtype": out_dtype, - }, {}] - - ops_config = [{ - "op_type": "reduce_mean", - "op_inputs": { - "X": ["input_data"] + dics = [ + { + "keep_dim": keep_dim, + "dim": dim, + "reduce_all": reduce_all, + "out_dtype": out_dtype, + "in_dtype": out_dtype, }, - "op_outputs": { - "Out": ["reduce_output_data"] - }, - "op_attrs": dics[0] - }] + {}, + ] + + ops_config = [ + { + "op_type": "reduce_mean", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["reduce_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, out_dtype, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, out_dtype, dics + ) + ) }, - outputs=["reduce_output_data"]) + outputs=["reduce_output_data"], + ) if not self.is_program_valid(program_config): continue @@ -93,8 +104,8 @@ class TrtConvertReduceMeanTest(TrtLayerAutoScanTest): yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -125,19 +136,23 @@ class TrtConvertReduceMeanTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (5e-4, 5e-4) + attrs, False + ), (5e-4, 5e-4) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (5e-4, 5e-4) + attrs, True + ), (5e-4, 5e-4) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py index 41d491d3255..67f3ab26f37 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py @@ -23,7 +23,6 @@ import unittest class TrtConvertReduceSumTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -41,7 +40,6 @@ class TrtConvertReduceSumTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(dtype, attrs: List[Dict[str, Any]]): if dtype == -1 or dtype == 5: return np.random.random([1, 3, 32, 32]).astype(np.float32) @@ -49,39 +47,52 @@ class TrtConvertReduceSumTest(TrtLayerAutoScanTest): return np.random.random([1, 3, 32, 32]).astype(np.int32) for keep_dim in [True, False]: - for dim in [[], [1], [0], [0, 1], [1, 2, 3], [-2, 0, 3], [-3], - [-4, 1], [3, 4, 5]]: + for dim in [ + [], + [1], + [0], + [0, 1], + [1, 2, 3], + [-2, 0, 3], + [-3], + [-4, 1], + [3, 4, 5], + ]: for reduce_all in [True, False]: for out_dtype in [-1, 2, 5]: - dics = [{ - "keep_dim": keep_dim, - "dim": dim, - "reduce_all": reduce_all, - "out_dtype": out_dtype, - "in_dtype": out_dtype, - }, {}] - - ops_config = [{ - "op_type": "reduce_sum", - "op_inputs": { - "X": ["input_data"] + dics = [ + { + "keep_dim": keep_dim, + "dim": dim, + "reduce_all": reduce_all, + "out_dtype": out_dtype, + "in_dtype": out_dtype, }, - "op_outputs": { - "Out": ["reduce_output_data"] - }, - "op_attrs": dics[0] - }] + {}, + ] + + ops_config = [ + { + "op_type": "reduce_sum", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["reduce_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, out_dtype, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, out_dtype, dics + ) + ) }, - outputs=["reduce_output_data"]) + outputs=["reduce_output_data"], + ) if not self.is_program_valid(program_config): continue @@ -89,7 +100,6 @@ class TrtConvertReduceSumTest(TrtLayerAutoScanTest): yield program_config def sample_predictor_configs(self, program_config): - def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -120,19 +130,23 @@ class TrtConvertReduceSumTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-5, 1e-5) + attrs, True + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reshape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reshape.py index abd4be3f888..90aaddfcb40 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reshape.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reshape.py @@ -22,7 +22,6 @@ import unittest class TrtConvertReshapeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -31,7 +30,7 @@ class TrtConvertReshapeTest(TrtLayerAutoScanTest): if len(attrs[0]['shape']) != 1: return False - #To test if the shape contains 0 + # To test if the shape contains 0 if len(attrs[0]['shape']) == 3: if attrs[0]['shape'][1] == 0: if self.dims != 3: @@ -45,7 +44,6 @@ class TrtConvertReshapeTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: self.input_shape = [1, 2, 4, 6] @@ -70,9 +68,18 @@ class TrtConvertReshapeTest(TrtLayerAutoScanTest): return np.array([24]).astype(np.int32) for dims in [4, 3, 2, 1]: - for shape in [[1, 6, 8], [1, 2, 4, 6], [1, 1, 0, 12], [1, 0, 6], - [1, -1, 12], [2, -1], [3, 16], [3, 4, 4], [48], - [-1, 48]]: + for shape in [ + [1, 6, 8], + [1, 2, 4, 6], + [1, 1, 0, 12], + [1, 0, 6], + [1, -1, 12], + [2, -1], + [3, 16], + [3, 4, 4], + [48], + [-1, 48], + ]: dics = [ { "shape": shape, @@ -81,29 +88,31 @@ class TrtConvertReshapeTest(TrtLayerAutoScanTest): self.dims = dims dics_intput = [{"X": ["reshape_input"]}] - ops_config = [{ - "op_type": "reshape", - "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["reshape_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "reshape", + "op_inputs": dics_intput[0], + "op_outputs": {"Out": ["reshape_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "reshape_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "reshape_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["reshape_out"]) + outputs=["reshape_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -141,13 +150,14 @@ class TrtConvertReshapeTest(TrtLayerAutoScanTest): def generate_trt_nodes_num(attrs, dynamic_shape): # in static shape mode, here is consistent with op_teller.cc - if (not dynamic_shape): - if (attrs[0]['shape'][0] == 0): + if not dynamic_shape: + if attrs[0]['shape'][0] == 0: return 1, 2 - elif (len(attrs[0]['shape']) == 1): + elif len(attrs[0]['shape']) == 1: return 0, 3 - elif (np.prod(attrs[0]['shape'][1:]) == np.prod( - self.input_shape[1:])): + elif np.prod(attrs[0]['shape'][1:]) == np.prod( + self.input_shape[1:] + ): return 1, 2 else: return 0, 3 @@ -161,19 +171,23 @@ class TrtConvertReshapeTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass @@ -185,12 +199,10 @@ class TrtConvertReshapeTest(TrtLayerAutoScanTest): # reshape having three inputs. class TrtConvertReshapeTest2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: return np.random.random([1, 2, 4, 6]).astype(np.float32) @@ -203,9 +215,12 @@ class TrtConvertReshapeTest2(TrtLayerAutoScanTest): for dims in [4, 3, 2, 1]: for shape in [[-1, 48]]: - dics = [{ - "shape": shape, - }, {}] + dics = [ + { + "shape": shape, + }, + {}, + ] self.dims = dims dics_intput = [ { @@ -217,9 +232,7 @@ class TrtConvertReshapeTest2(TrtLayerAutoScanTest): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT1_data"] - }, + "op_outputs": {"Out": ["shapeT1_data"]}, "op_attrs": { "dtype": 2, "str_value": "2", @@ -229,9 +242,7 @@ class TrtConvertReshapeTest2(TrtLayerAutoScanTest): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT2_data"] - }, + "op_outputs": {"Out": ["shapeT2_data"]}, "op_attrs": { "dtype": 2, "str_value": "24", @@ -241,10 +252,8 @@ class TrtConvertReshapeTest2(TrtLayerAutoScanTest): { "op_type": "reshape", "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["reshape_out"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["reshape_out"]}, + "op_attrs": dics[0], }, ] ops = self.generate_op_config(ops_config) @@ -252,16 +261,18 @@ class TrtConvertReshapeTest2(TrtLayerAutoScanTest): ops=ops, weights={}, inputs={ - "reshape_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "reshape_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["reshape_out"]) + outputs=["reshape_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -309,12 +320,10 @@ class TrtConvertReshapeTest2(TrtLayerAutoScanTest): # reshape having 2 inputs. class TrtConvertReshapeTest3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: return np.random.random([1, 2, 12, 6]).astype(np.float32) @@ -327,9 +336,12 @@ class TrtConvertReshapeTest3(TrtLayerAutoScanTest): for dims in [4, 3, 2, 1]: for shape in [[-1, 144]]: - dics = [{ - "shape": shape, - }, {}] + dics = [ + { + "shape": shape, + }, + {}, + ] self.dims = dims dics_intput = [ { @@ -341,9 +353,7 @@ class TrtConvertReshapeTest3(TrtLayerAutoScanTest): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shape_data"] - }, + "op_outputs": {"Out": ["shape_data"]}, "op_attrs": { "dtype": 2, "str_value": "12", @@ -353,10 +363,8 @@ class TrtConvertReshapeTest3(TrtLayerAutoScanTest): { "op_type": "reshape", "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["reshape_out"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["reshape_out"]}, + "op_attrs": dics[0], }, ] ops = self.generate_op_config(ops_config) @@ -364,16 +372,18 @@ class TrtConvertReshapeTest3(TrtLayerAutoScanTest): ops=ops, weights={}, inputs={ - "reshape_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "reshape_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["reshape_out"]) + outputs=["reshape_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_rnn.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_rnn.py index 2b76e273aa0..4f65ea6abf4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_rnn.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_rnn.py @@ -23,7 +23,6 @@ import os class TrtConvertSliceTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -36,148 +35,168 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): for num_layers in [1, 2]: for is_bidirec in [True, False]: dics = [] - dics.append({ - "hidden_size": hidden_size, - "input_size": input_size, - "num_layers": num_layers, - "mode": "LSTM", - "is_bidirec": is_bidirec, - "is_test": True, - "dropout_prob": 0.0, - # for my convience - "batch": batch, - "seq_len": seq_len, - }) + dics.append( + { + "hidden_size": hidden_size, + "input_size": input_size, + "num_layers": num_layers, + "mode": "LSTM", + "is_bidirec": is_bidirec, + "is_test": True, + "dropout_prob": 0.0, + # for my convience + "batch": batch, + "seq_len": seq_len, + } + ) K = 1 - if (dics[0]["is_bidirec"]): + if dics[0]["is_bidirec"]: K = 2 def generate_input1(): - return np.random.random([ - batch, seq_len, input_size - ]).astype(np.float32) * 2 - 1 + return ( + np.random.random( + [batch, seq_len, input_size] + ).astype(np.float32) + * 2 + - 1 + ) # initial input -> hidden def generate_w0(): - return np.random.random([ - 4 * hidden_size, input_size - ]).astype(np.float32) * 2 - 1 + return ( + np.random.random( + [4 * hidden_size, input_size] + ).astype(np.float32) + * 2 + - 1 + ) # prev layer's output -> hidden def generate_w1(): - return np.random.random([ - 4 * hidden_size, K * hidden_size - ]).astype(np.float32) * 2 - 1 + return ( + np.random.random( + [4 * hidden_size, K * hidden_size] + ).astype(np.float32) + * 2 + - 1 + ) # def generate_w2(): - return np.random.random([ - 4 * hidden_size, hidden_size - ]).astype(np.float32) * 2 - 1 + return ( + np.random.random( + [4 * hidden_size, hidden_size] + ).astype(np.float32) + * 2 + - 1 + ) def generate_b(): - return np.random.random([ - 4 * hidden_size - ]).astype(np.float32) * 2 - 1 - - dics.append({ - "dtype": - 5, - "input_dim_idx": - 0, - "str_value": - "", - "value": - 0.0, - "shape": [K * num_layers, -1, hidden_size], - "output_dim_idx": - 1, - }) + return ( + np.random.random( + [4 * hidden_size] + ).astype(np.float32) + * 2 + - 1 + ) + + dics.append( + { + "dtype": 5, + "input_dim_idx": 0, + "str_value": "", + "value": 0.0, + "shape": [ + K * num_layers, + -1, + hidden_size, + ], + "output_dim_idx": 1, + } + ) dics.append({"axis": [1, 0, 2]}) # set weights WeightList = [ "weight" + str(i) - for i in range(4 * K * - dics[0]["num_layers"]) + for i in range( + 4 * K * dics[0]["num_layers"] + ) ] weights = {} for i in range((int)(len(WeightList) / 2)): # mean this weight : input->hidden # input has 2 case: initial input input_size, K * hidden form the prev layer. - if (i % 2 == 0): - if (i <= K): + if i % 2 == 0: + if i <= K: weights[ - WeightList[i]] = TensorConfig( - data_gen=partial( - generate_w0)) + WeightList[i] + ] = TensorConfig( + data_gen=partial(generate_w0) + ) else: weights[ - WeightList[i]] = TensorConfig( - data_gen=partial( - generate_w1)) + WeightList[i] + ] = TensorConfig( + data_gen=partial(generate_w1) + ) # mean this weight : hidden->hidden - if (i % 2 == 1): + if i % 2 == 1: weights[WeightList[i]] = TensorConfig( - data_gen=partial(generate_w2)) - for i in range((int)(len(WeightList) / 2), - len(WeightList)): + data_gen=partial(generate_w2) + ) + for i in range( + (int)(len(WeightList) / 2), len(WeightList) + ): weights[WeightList[i]] = TensorConfig( - data_gen=partial(generate_b)) + data_gen=partial(generate_b) + ) ops_config = [ { - "op_type": - "fill_constant_batch_size_like", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["prestate1"] - }, - "op_attrs": dics[1] + "op_type": "fill_constant_batch_size_like", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": {"Out": ["prestate1"]}, + "op_attrs": dics[1], }, { - "op_type": - "fill_constant_batch_size_like", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["prestate2"] - }, - "op_attrs": dics[1] + "op_type": "fill_constant_batch_size_like", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": {"Out": ["prestate2"]}, + "op_attrs": dics[1], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["input_data"] - }, + "op_inputs": {"X": ["input_data"]}, "op_outputs": { "Out": ["rnn_input_data"] }, - "op_attrs": dics[2] + "op_attrs": dics[2], }, { "op_type": "rnn", "op_inputs": { "Input": ["rnn_input_data"], # prev_c, prev_h - "PreState": - ["prestate1", "prestate2"], + "PreState": [ + "prestate1", + "prestate2", + ], "WeightList": WeightList, }, "op_outputs": { "Out": ["rnn_output_data"], "State": [ "state_output_data0", - "state_output_data1" + "state_output_data1", ], "Reserve": ["reserve_data"], - "DropoutState": - ["DropoutState_data"] + "DropoutState": [ + "DropoutState_data" + ], }, - "op_attrs": dics[0] - } + "op_attrs": dics[0], + }, ] ops = self.generate_op_config(ops_config) @@ -185,16 +204,18 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): ops=ops, weights=weights, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1)) + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ) }, - outputs=["rnn_output_data"]) + outputs=["rnn_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): + self, program_config + ) -> (paddle_infer.Config, List[int], float): attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) ] @@ -234,7 +255,7 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): # The output has diff between gpu and trt in PR-CI-Windows-Inference tol_fp32 = 1e-5 tol_half = 1e-2 - if (os.name == 'nt'): + if os.name == 'nt': tol_fp32 = 1e-2 tol_half = 1e-1 @@ -242,10 +263,12 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), tol_fp32 + attrs, True + ), tol_fp32 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), tol_half + attrs, True + ), tol_half def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roi_align.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roi_align.py index ee84df151b5..f89527359d4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roi_align.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roi_align.py @@ -22,12 +22,10 @@ from typing import Any, Dict, List class TrtConvertRoiAlignTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): return np.ones([batch, 256, 32, 32]).astype(np.float32) @@ -47,92 +45,111 @@ class TrtConvertRoiAlignTest(TrtLayerAutoScanTest): self.num_input = num_input if num_input == 1: batch = 1 - dics = [{ - "spatial_scale": spatial_scale, - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "sampling_ratio": sampling_ratio, - "aligned": aligned - }, {}] - dics_input = [{ - "X": ["roi_align_input"], - "ROIs": ["ROIs"], - "RoisNum": ["RoisNum"] - }, { - "X": ["roi_align_input"], - "ROIs": ["ROIs"] - }] - program_input = [{ - "roi_align_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "ROIs": - TensorConfig(data_gen=partial( - generate_input2, dics, batch)), - "RoisNum": - TensorConfig(data_gen=partial( - generate_input3, dics, batch)) - }, { - "roi_align_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "ROIs": - TensorConfig(data_gen=partial( - generate_input2, dics, batch), - lod=[[32, 3]]) - }] - ops_config = [{ - "op_type": - "roi_align", - "op_inputs": - dics_input[num_input], - "op_outputs": { - "Out": ["roi_align_out"] + dics = [ + { + "spatial_scale": spatial_scale, + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "sampling_ratio": sampling_ratio, + "aligned": aligned, + }, + {}, + ] + dics_input = [ + { + "X": ["roi_align_input"], + "ROIs": ["ROIs"], + "RoisNum": ["RoisNum"], + }, + { + "X": ["roi_align_input"], + "ROIs": ["ROIs"], + }, + ] + program_input = [ + { + "roi_align_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "ROIs": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), + "RoisNum": TensorConfig( + data_gen=partial( + generate_input3, dics, batch + ) + ), + }, + { + "roi_align_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "ROIs": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ), + lod=[[32, 3]], + ), }, - "op_attrs": - dics[0] - }] + ] + ops_config = [ + { + "op_type": "roi_align", + "op_inputs": dics_input[num_input], + "op_outputs": { + "Out": ["roi_align_out"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs=program_input[num_input], - outputs=["roi_align_out"]) + outputs=["roi_align_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.num_input == 0: self.dynamic_shape.min_input_shape = { "roi_align_input": [1, 256, 32, 32], "ROIs": [3, 4], - "RoisNum": [1] + "RoisNum": [1], } self.dynamic_shape.max_input_shape = { "roi_align_input": [1, 256, 64, 64], "ROIs": [3, 4], - "RoisNum": [1] + "RoisNum": [1], } self.dynamic_shape.opt_input_shape = { "roi_align_input": [1, 256, 64, 64], "ROIs": [3, 4], - "RoisNum": [1] + "RoisNum": [1], } elif self.num_input == 1: self.dynamic_shape.min_input_shape = { "roi_align_input": [1, 256, 32, 32], - "ROIs": [3, 4] + "ROIs": [3, 4], } self.dynamic_shape.max_input_shape = { "roi_align_input": [1, 256, 64, 64], - "ROIs": [3, 4] + "ROIs": [3, 4], } self.dynamic_shape.opt_input_shape = { "roi_align_input": [1, 256, 64, 64], - "ROIs": [3, 4] + "ROIs": [3, 4], } def clear_dynamic_shape(): @@ -159,29 +176,33 @@ class TrtConvertRoiAlignTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(program_config.inputs) == 3: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT RoisNum NOT SUPPORT") + self.add_skip_case( + teller1, SkipReasons.TRT_NOT_SUPPORT, "INPUT RoisNum NOT SUPPORT" + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roll.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roll.py index 6b6e1e0c4bf..47d415b1a6b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roll.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roll.py @@ -22,7 +22,6 @@ import unittest class TrtConvertRollTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,43 +31,44 @@ class TrtConvertRollTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 56, 56, 192]).astype(np.float32) for axis in [[1, 2]]: for shifts in [[-1, -1], [-3, -3]]: - dics = [{ - "axis": axis, - "shifts": shifts, - }] - - ops_config = [{ - "op_type": "roll", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["roll_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "axis": axis, + "shifts": shifts, + } + ] + + ops_config = [ + { + "op_type": "roll", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["roll_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["roll_output_data"]) + outputs=["roll_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 56, 56, 192] @@ -103,19 +103,23 @@ class TrtConvertRollTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scale.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scale.py index 33a7c060ecc..27658d92863 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scale.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scale.py @@ -22,12 +22,10 @@ import unittest class TrtConvertScaleTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -49,51 +47,60 @@ class TrtConvertScaleTest(TrtLayerAutoScanTest): for bias_after_scale in [False, True]: self.num_input = num_input self.dims = dims - dics = [{ - "scale": scale, - "bias": bias, - "bias_after_scale": bias_after_scale - }, {}] - - dics_intput = [{ - "X": ["scale_input"], - "ScaleTensor": ["ScaleTensor"], - }, { - "X": ["scale_input"] - }] - dics_intputs = [{ - "ScaleTensor": - TensorConfig(data_gen=partial( - generate_weight1, dics)) - }, {}] - - ops_config = [{ - "op_type": - "scale", - "op_inputs": - dics_intput[num_input], - "op_outputs": { - "Out": ["scale_out"] + dics = [ + { + "scale": scale, + "bias": bias, + "bias_after_scale": bias_after_scale, + }, + {}, + ] + + dics_intput = [ + { + "X": ["scale_input"], + "ScaleTensor": ["ScaleTensor"], }, - "op_attrs": - dics[0] - }] + {"X": ["scale_input"]}, + ] + dics_intputs = [ + { + "ScaleTensor": TensorConfig( + data_gen=partial( + generate_weight1, dics + ) + ) + }, + {}, + ] + + ops_config = [ + { + "op_type": "scale", + "op_inputs": dics_intput[num_input], + "op_outputs": {"Out": ["scale_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights=dics_intputs[num_input], inputs={ - "scale_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)) + "scale_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ) }, - outputs=["scale_out"]) + outputs=["scale_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -134,37 +141,46 @@ class TrtConvertScaleTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.num_input == 0: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT ScaleTensor and Shape NOT SUPPORT") + self.add_skip_case( + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "INPUT ScaleTensor and Shape NOT SUPPORT", + ) def teller2(program_config, predictor_config): if self.dims == 1 and len(self.dynamic_shape.min_input_shape) == 0: return True return False - self.add_skip_case(teller2, SkipReasons.TRT_NOT_SUPPORT, - "INPUT DIM EQUAL TO 1 OF STATIC SHAPE NOT SUPPORT") + self.add_skip_case( + teller2, + SkipReasons.TRT_NOT_SUPPORT, + "INPUT DIM EQUAL TO 1 OF STATIC SHAPE NOT SUPPORT", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scatter_nd_add.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scatter_nd_add.py index d0e6cf867ce..9376ade22a2 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scatter_nd_add.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scatter_nd_add.py @@ -22,12 +22,10 @@ import unittest class TrtConvertScatterNd(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([6]).astype(np.float32) @@ -37,38 +35,42 @@ class TrtConvertScatterNd(TrtLayerAutoScanTest): def generate_input3(): return np.random.random([4]).astype(np.float32) - ops_config = [{ - "op_type": "scatter_nd_add", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"], - "Updates": ["update_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "scatter_nd_add", + "op_inputs": { + "X": ["input_data"], + "Index": ["index_data"], + "Updates": ["update_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) for i in range(10): program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), - "index_data": - TensorConfig(data_gen=partial(generate_input2)), - "update_data": - TensorConfig(data_gen=partial(generate_input3)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), + "index_data": TensorConfig( + data_gen=partial(generate_input2) + ), + "update_data": TensorConfig( + data_gen=partial(generate_input3) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shape.py index 778a097c3ab..03df0287836 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shape.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shape.py @@ -22,12 +22,10 @@ import unittest class TrtConvertSumTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -41,31 +39,31 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): for dims in [1, 2, 3, 4]: for batch in [1, 4]: self.dims = dims - ops_config = [{ - "op_type": "shape", - "op_inputs": { - "Input": ["input1"] - }, - "op_outputs": { - "Out": ["output"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "shape", + "op_inputs": {"Input": ["input1"]}, + "op_outputs": {"Out": ["output"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1": - TensorConfig(data_gen=partial(generate_input1, batch)) + "input1": TensorConfig( + data_gen=partial(generate_input1, batch) + ) }, - outputs=["output"]) + outputs=["output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = {"input1": [1, 3, 24, 24]} @@ -87,7 +85,7 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): } def generate_trt_nodes_num(dynamic_shape): - if (not dynamic_shape): + if not dynamic_shape: return 0, 3 return 1, 2 @@ -100,10 +98,12 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-5 + False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-3 + False + ), 1e-3 # for dynamic_shape generate_dynamic_shape() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shuffle_channel.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shuffle_channel.py index bf5584e0029..04c1e3259fc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shuffle_channel.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shuffle_channel.py @@ -22,44 +22,41 @@ import unittest class TrtConvertShuffleChannelTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): return np.ones([batch, 6, 24, 24]).astype(np.float32) for batch in [1, 2, 4]: for group in [1, 2, 3]: dics = [{"group": group}, {}] - ops_config = [{ - "op_type": "shuffle_channel", - "op_inputs": { - "X": ["shuffle_channel_input"] - }, - "op_outputs": { - "Out": ["shuffle_channel_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "shuffle_channel", + "op_inputs": {"X": ["shuffle_channel_input"]}, + "op_outputs": {"Out": ["shuffle_channel_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "shuffle_channel_input": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "shuffle_channel_input": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["shuffle_channel_out"]) + outputs=["shuffle_channel_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "shuffle_channel_input": [1, 6, 24, 24] @@ -78,8 +75,10 @@ class TrtConvertShuffleChannelTest(TrtLayerAutoScanTest): def generate_trt_nodes_num(attrs, dynamic_shape): ver = paddle_infer.get_trt_compile_version() - if ver[0] * 1000 + ver[1] * 100 + ver[ - 2] * 10 < 8000 and dynamic_shape == True: + if ( + ver[0] * 1000 + ver[1] * 100 + ver[2] * 10 < 8000 + and dynamic_shape == True + ): return 0, 3 else: return 1, 2 @@ -92,19 +91,23 @@ class TrtConvertShuffleChannelTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_silu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_silu.py index 3b9ff8a0527..73df326fb01 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_silu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_silu.py @@ -22,12 +22,10 @@ import unittest class TrtConvertSwishTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([3]).astype(np.float32) @@ -42,33 +40,34 @@ class TrtConvertSwishTest(TrtLayerAutoScanTest): for beta in [1.0, 2.0, 3.0]: self.dims = dims - ops_config = [{ - "op_type": "silu", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "silu", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dims, {})) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dims, {}) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -113,19 +112,23 @@ class TrtConvertSwishTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_skip_layernorm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_skip_layernorm.py index 3a3b0603ec8..18ea2abe6bc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_skip_layernorm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_skip_layernorm.py @@ -22,7 +22,6 @@ import unittest class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,19 +31,20 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if attrs[0]['begin_norm_axis'] >= 0: - if len(inputs['skip_layernorm_inputX_data'].shape - ) <= attrs[0]['begin_norm_axis']: + if ( + len(inputs['skip_layernorm_inputX_data'].shape) + <= attrs[0]['begin_norm_axis'] + ): return False - #2D input is not supported. + # 2D input is not supported. if self.dims == 2: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 6, 128, 768]).astype(np.float32) @@ -73,107 +73,117 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): for begin_norm_axis in [0, 1, 2, -1]: for enable_int8 in [False, True]: self.dims = dims - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis, - "enable_int8": enable_int8 - }, {}] - ops_config = [{ - "op_type": "skip_layernorm", - "op_inputs": { - "X": ["skip_layernorm_inputX_data"], - "Y": ["skip_layernorm_inputY_data"], - "Bias": ["Bias"], - "Scale": ["Scale"] + dics = [ + { + "epsilon": epsilon, + "begin_norm_axis": begin_norm_axis, + "enable_int8": enable_int8, }, - "op_outputs": { - "Out": ["skip_layernorm_out"] - }, - "op_attrs": dics[0] - }] + {}, + ] + ops_config = [ + { + "op_type": "skip_layernorm", + "op_inputs": { + "X": ["skip_layernorm_inputX_data"], + "Y": ["skip_layernorm_inputY_data"], + "Bias": ["Bias"], + "Scale": ["Scale"], + }, + "op_outputs": { + "Out": ["skip_layernorm_out"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "Bias": - TensorConfig(data_gen=partial( - generate_weight1, dics)), - "Scale": - TensorConfig(data_gen=partial( - generate_weight2, dics)) + "Bias": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + "Scale": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), }, inputs={ - "skip_layernorm_inputX_data": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "skip_layernorm_inputY_data": - TensorConfig(data_gen=partial( - generate_input2, dics, batch)) + "skip_layernorm_inputX_data": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "skip_layernorm_inputY_data": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), }, - outputs=["skip_layernorm_out"]) + outputs=["skip_layernorm_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { "skip_layernorm_inputX_data": [1, 6, 128, 768], "skip_layernorm_inputY_data": [1, 6, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "skip_layernorm_inputX_data": [4, 6, 768, 3072], "skip_layernorm_inputY_data": [4, 6, 768, 3072], "Bias": [3072], - "Scale": [3072] + "Scale": [3072], } self.dynamic_shape.opt_input_shape = { "skip_layernorm_inputX_data": [2, 6, 128, 768], "skip_layernorm_inputY_data": [2, 6, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "skip_layernorm_inputX_data": [1, 128, 768], "skip_layernorm_inputY_data": [1, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "skip_layernorm_inputX_data": [4, 768, 3072], "skip_layernorm_inputY_data": [4, 768, 3072], "Bias": [3072], - "Scale": [3072] + "Scale": [3072], } self.dynamic_shape.opt_input_shape = { "skip_layernorm_inputX_data": [2, 128, 768], "skip_layernorm_inputY_data": [2, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "skip_layernorm_inputX_data": [1, 768], "skip_layernorm_inputY_data": [1, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "skip_layernorm_inputX_data": [4, 3072], "skip_layernorm_inputY_data": [4, 3072], "Bias": [3072], - "Scale": [3072] + "Scale": [3072], } self.dynamic_shape.opt_input_shape = { "skip_layernorm_inputX_data": [2, 768], "skip_layernorm_inputY_data": [2, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } def clear_dynamic_shape(): @@ -204,10 +214,12 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_slice.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_slice.py index 22eb7cef872..81d833f39f0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_slice.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_slice.py @@ -22,7 +22,6 @@ import unittest class TrtConvertSliceTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -34,13 +33,17 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): start = 0 end = 0 if attrs[0]["starts"][x] < 0: - start = attrs[0]["starts"][x] + inputs['input_data'].shape[ - attrs[0]["axes"][x]] + start = ( + attrs[0]["starts"][x] + + inputs['input_data'].shape[attrs[0]["axes"][x]] + ) else: start = attrs[0]["starts"][x] if attrs[0]["ends"][x] < 0: - end = attrs[0]["ends"][x] + inputs['input_data'].shape[ - attrs[0]["axes"][x]] + end = ( + attrs[0]["ends"][x] + + inputs['input_data'].shape[attrs[0]["axes"][x]] + ) else: end = attrs[0]["ends"][x] start = max(0, start) @@ -51,12 +54,11 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): for x in attrs[0]["decrease_axis"]: if x < 0: return False - if (out_shape[x] != 1): + if out_shape[x] != 1: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.random.random([6, 6, 64, 64]).astype(np.float32) @@ -65,41 +67,44 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): for ends in [[2, 2], [5, 5], [1, -1]]: for decrease_axis in [[], [1], [2], [-1], [-100]]: for infer_flags in [[-1]]: - dics = [{ - "axes": axes, - "starts": starts, - "ends": ends, - "decrease_axis": decrease_axis, - "infer_flags": infer_flags - }] - - ops_config = [{ - "op_type": "slice", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["slice_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "axes": axes, + "starts": starts, + "ends": ends, + "decrease_axis": decrease_axis, + "infer_flags": infer_flags, + } + ] + + ops_config = [ + { + "op_type": "slice", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": { + "Out": ["slice_output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["slice_output_data"]) + outputs=["slice_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [8, 8, 64, 64]} @@ -125,19 +130,23 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): # TODO(inference): fix. diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py index 66621463310..7f1ed848cb1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertSoftmaxTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,14 +31,13 @@ class TrtConvertSoftmaxTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if len(inputs['softmax_input'].shape) <= attrs[0]['axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -53,32 +51,31 @@ class TrtConvertSoftmaxTest(TrtLayerAutoScanTest): for axis in [-1, 0, 1, 2, 3]: self.dims = dims dics = [{"axis": axis}, {}] - ops_config = [{ - "op_type": "softmax", - "op_inputs": { - "X": ["softmax_input"] - }, - "op_outputs": { - "Out": ["softmax_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "softmax", + "op_inputs": {"X": ["softmax_input"]}, + "op_outputs": {"Out": ["softmax_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "softmax_input": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "softmax_input": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["softmax_out"]) + outputs=["softmax_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -123,19 +120,23 @@ class TrtConvertSoftmaxTest(TrtLayerAutoScanTest): else: self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_split.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_split.py index b60b9227395..ce9c533c625 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_split.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_split.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertSplitTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -35,13 +34,13 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): if len(inputs['split_input'].shape) <= attrs[0]['axis']: return False - #Sections and num cannot both be equal to 0. + # Sections and num cannot both be equal to 0. if len(attrs[0]['sections']) == 0: if attrs[0]['num'] == 0: return False - #When sections and num are not both equal to 0, sections has higher priority. - #The sum of sections should be equal to the input size. + # When sections and num are not both equal to 0, sections has higher priority. + # The sum of sections should be equal to the input size. if len(attrs[0]['sections']) != 0: if attrs[0]['num'] != 0: return False @@ -53,16 +52,18 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): if sum != inputs['split_input'].shape[attrs[0]['axis']]: return False - #The size of num should be equal to the input dimension. + # The size of num should be equal to the input dimension. if attrs[0]['num'] != 0: if len(outputs) != attrs[0]['num']: return False - #Test AxisTensor and SectionsTensorList + # Test AxisTensor and SectionsTensorList if self.num_input == 0: - if self.dims == 2 and attrs[0]['sections'] == [ - 10, 14 - ] and len(outputs) == 2: + if ( + self.dims == 2 + and attrs[0]['sections'] == [10, 14] + and len(outputs) == 2 + ): return True else: return False @@ -70,7 +71,6 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.random.random([batch, 3, 3, 24]).astype(np.float32) @@ -93,72 +93,95 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): for num_input in [0, 1]: for dims in [1, 2, 3, 4]: for batch in [3, 6, 9]: - for Out in [["output_var0", "output_var1"], - ["output_var0", "output_var1", "output_var2"]]: - for sections in [[], [1, 2], [2, 1], [10, 14], - [1, 1, 1], [2, 2, 2], [3, 3, 3], - [3, 7, 14]]: + for Out in [ + ["output_var0", "output_var1"], + ["output_var0", "output_var1", "output_var2"], + ]: + for sections in [ + [], + [1, 2], + [2, 1], + [10, 14], + [1, 1, 1], + [2, 2, 2], + [3, 3, 3], + [3, 7, 14], + ]: for num in [0, 3]: for axis in [0, 1, 2, 3]: self.batch = batch self.num_input = num_input self.dims = dims - dics = [{ - "sections": sections, - "num": num, - "axis": axis - }, {}] - - dics_intput = [{ - "X": ["split_input"], - "AxisTensor": ["AxisTensor"], - "SectionsTensorList": [ - "SectionsTensorList1", - "SectionsTensorList2" - ] - }, { - "X": ["split_input"] - }] - dics_intputs = [{ - "AxisTensor": - TensorConfig(data_gen=partial( - generate_AxisTensor, dics)), - "SectionsTensorList1": - TensorConfig(data_gen=partial( - generate_SectionsTensorList1, - dics)), - "SectionsTensorList2": - TensorConfig(data_gen=partial( - generate_SectionsTensorList2, dics)) - }, {}] - - ops_config = [{ - "op_type": - "split", - "op_inputs": - dics_intput[num_input], - "op_outputs": { - "Out": Out + dics = [ + { + "sections": sections, + "num": num, + "axis": axis, + }, + {}, + ] + + dics_intput = [ + { + "X": ["split_input"], + "AxisTensor": ["AxisTensor"], + "SectionsTensorList": [ + "SectionsTensorList1", + "SectionsTensorList2", + ], }, - "op_attrs": - dics[0] - }] + {"X": ["split_input"]}, + ] + dics_intputs = [ + { + "AxisTensor": TensorConfig( + data_gen=partial( + generate_AxisTensor, dics + ) + ), + "SectionsTensorList1": TensorConfig( + data_gen=partial( + generate_SectionsTensorList1, + dics, + ) + ), + "SectionsTensorList2": TensorConfig( + data_gen=partial( + generate_SectionsTensorList2, + dics, + ) + ), + }, + {}, + ] + + ops_config = [ + { + "op_type": "split", + "op_inputs": dics_intput[num_input], + "op_outputs": {"Out": Out}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights=dics_intputs[num_input], inputs={ - "split_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)) + "split_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ) }, - outputs=Out) + outputs=Out, + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -216,30 +239,35 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(program_config.weights) == 3: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT AxisTensor AND SectionsTensorList NOT SUPPORT.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "INPUT AxisTensor AND SectionsTensorList NOT SUPPORT.", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_squeeze2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_squeeze2.py index 31bf484c599..27860452091 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_squeeze2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_squeeze2.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertSplitTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -40,25 +39,25 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): self.dims = dims self.axes = axes dics = [{"axes": axes}] - ops_config = [{ - "op_type": "squeeze2", - "op_inputs": { - "X": ["in_data"] - }, - "op_outputs": { - "Out": ["out_data"], - "XShape": ["XShape_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "squeeze2", + "op_inputs": {"X": ["in_data"]}, + "op_outputs": { + "Out": ["out_data"], + "XShape": ["XShape_data"], + }, + "op_attrs": dics[0], + } + ] # new_axes is the update of axes new_axes = list(axes) for i in range(len(new_axes)): - if (new_axes[i] < 0): + if new_axes[i] < 0: new_axes[i] += dims - if (max(new_axes) >= dims): + if max(new_axes) >= dims: continue - # generate input data + # generate input data self.input_shape = [1] * dims for i in range(dims): self.input_shape[i] = np.random.randint(1, 20) @@ -68,24 +67,26 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): for i in new_axes: self.input_shape[i] = 1 return np.random.random(self.input_shape).astype( - np.float32) + np.float32 + ) ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "in_data": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "in_data": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["out_data"]) + outputs=["out_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): max_shape = list(self.input_shape) min_shape = list(self.input_shape) @@ -112,19 +113,23 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_stack.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_stack.py index dc2c8bb4162..91e5e499b19 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_stack.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_stack.py @@ -22,7 +22,6 @@ import unittest class TrtConvertStackTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,14 +30,13 @@ class TrtConvertStackTest(TrtLayerAutoScanTest): attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than the set axis. + # The input dimension should be less than the set axis. if len(inputs['stack_input1'].shape) < attrs[0]['axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.random.random([batch, 3, 24, 24]).astype(np.float32) @@ -74,103 +72,107 @@ class TrtConvertStackTest(TrtLayerAutoScanTest): for axis in [-2, -1, 0, 1, 2, 3]: self.dims = dims dics = [{"axis": axis}, {}] - ops_config = [{ - "op_type": "stack", - "op_inputs": { - "X": - ["stack_input1", "stack_input2", "stack_input3"] - }, - "op_outputs": { - "Y": ["stack_output"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "stack", + "op_inputs": { + "X": [ + "stack_input1", + "stack_input2", + "stack_input3", + ] + }, + "op_outputs": {"Y": ["stack_output"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "stack_input1": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)), - "stack_input2": - TensorConfig( - data_gen=partial(generate_input2, dics, batch)), - "stack_input3": - TensorConfig( - data_gen=partial(generate_input3, dics, batch)) + "stack_input1": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ), + "stack_input2": TensorConfig( + data_gen=partial(generate_input2, dics, batch) + ), + "stack_input3": TensorConfig( + data_gen=partial(generate_input3, dics, batch) + ), }, - outputs=["stack_output"]) + outputs=["stack_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { "stack_input1": [1, 3, 24, 24], "stack_input2": [1, 3, 24, 24], - "stack_input3": [1, 3, 24, 24] + "stack_input3": [1, 3, 24, 24], } self.dynamic_shape.max_input_shape = { "stack_input1": [4, 3, 48, 48], "stack_input2": [4, 3, 48, 48], - "stack_input3": [4, 3, 48, 48] + "stack_input3": [4, 3, 48, 48], } self.dynamic_shape.opt_input_shape = { "stack_input1": [1, 3, 24, 24], "stack_input2": [1, 3, 24, 24], - "stack_input3": [1, 3, 24, 24] + "stack_input3": [1, 3, 24, 24], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "stack_input1": [1, 3, 24], "stack_input2": [1, 3, 24], - "stack_input3": [1, 3, 24] + "stack_input3": [1, 3, 24], } self.dynamic_shape.max_input_shape = { "stack_input1": [4, 3, 48], "stack_input2": [4, 3, 48], - "stack_input3": [4, 3, 48] + "stack_input3": [4, 3, 48], } self.dynamic_shape.opt_input_shape = { "stack_input1": [1, 3, 24], "stack_input2": [1, 3, 24], - "stack_input3": [1, 3, 24] + "stack_input3": [1, 3, 24], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "stack_input1": [1, 24], "stack_input2": [1, 24], - "stack_input3": [1, 24] + "stack_input3": [1, 24], } self.dynamic_shape.max_input_shape = { "stack_input1": [4, 48], "stack_input2": [4, 48], - "stack_input3": [4, 48] + "stack_input3": [4, 48], } self.dynamic_shape.opt_input_shape = { "stack_input1": [1, 24], "stack_input2": [1, 24], - "stack_input3": [1, 24] + "stack_input3": [1, 24], } elif self.dims == 1: self.dynamic_shape.min_input_shape = { "stack_input1": [24], "stack_input2": [24], - "stack_input3": [24] + "stack_input3": [24], } self.dynamic_shape.max_input_shape = { "stack_input1": [48], "stack_input2": [48], - "stack_input3": [48] + "stack_input3": [48], } self.dynamic_shape.opt_input_shape = { "stack_input1": [24], "stack_input2": [24], - "stack_input3": [24] + "stack_input3": [24], } def clear_dynamic_shape(): @@ -191,19 +193,23 @@ class TrtConvertStackTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_strided_slice.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_strided_slice.py index 4f517b447a1..b8549c91873 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_strided_slice.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_strided_slice.py @@ -22,7 +22,6 @@ import unittest class TrtConvertStridedSliceTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,7 +31,6 @@ class TrtConvertStridedSliceTest(TrtLayerAutoScanTest): return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.random.random([1, 56, 56, 192]).astype(np.float32) @@ -42,42 +40,47 @@ class TrtConvertStridedSliceTest(TrtLayerAutoScanTest): for decrease_axis in [[]]: for infer_flags in [[1, 1]]: for strides in [[2, 2]]: - dics = [{ - "axes": axes, - "starts": starts, - "ends": ends, - "decrease_axis": decrease_axis, - "infer_flags": infer_flags, - "strides": strides - }] - - ops_config = [{ - "op_type": "strided_slice", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["slice_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "axes": axes, + "starts": starts, + "ends": ends, + "decrease_axis": decrease_axis, + "infer_flags": infer_flags, + "strides": strides, + } + ] + + ops_config = [ + { + "op_type": "strided_slice", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": { + "Out": ["slice_output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dics + ) + ) }, - outputs=["slice_output_data"]) + outputs=["slice_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 56, 56, 192] @@ -118,71 +121,81 @@ class TrtConvertStridedSliceTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 def test(self): self.run_test() class TrtConvertStridedSliceTest2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.random.random([1, 56, 56, 192]).astype(np.float32) for axes in [[1, 2], [2, 3], [1, 3]]: - for starts in [[-10, 1], [-10, 20], [-10, 15], [-10, 16], [-10, - 20]]: + for starts in [ + [-10, 1], + [-10, 20], + [-10, 15], + [-10, 16], + [-10, 20], + ]: for ends in [[-9, 10000], [-9, -1], [-9, 40]]: for decrease_axis in [[]]: for infer_flags in [[1, 1]]: for strides in [[2, 2]]: - dics = [{ - "axes": axes, - "starts": starts, - "ends": ends, - "decrease_axis": [axes[0]], - "infer_flags": infer_flags, - "strides": strides - }] - - ops_config = [{ - "op_type": "strided_slice", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["slice_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "axes": axes, + "starts": starts, + "ends": ends, + "decrease_axis": [axes[0]], + "infer_flags": infer_flags, + "strides": strides, + } + ] + + ops_config = [ + { + "op_type": "strided_slice", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": { + "Out": ["slice_output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dics + ) + ) }, - outputs=["slice_output_data"]) + outputs=["slice_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): self.dynamic_shape.min_input_shape = { "input_data": [1, 56, 56, 192] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_sum.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_sum.py index 673d20313e1..f7534215c62 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_sum.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_sum.py @@ -22,12 +22,10 @@ import unittest class TrtConvertSumTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -61,99 +59,101 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): for dims in [1, 2, 3, 4]: for batch in [1, 4]: self.dims = dims - ops_config = [{ - "op_type": "sum", - "op_inputs": { - "X": ["input1", "input2", "input3"] - }, - "op_outputs": { - "Out": ["output"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "sum", + "op_inputs": {"X": ["input1", "input2", "input3"]}, + "op_outputs": {"Out": ["output"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1": - TensorConfig(data_gen=partial(generate_input1, batch)), - "input2": - TensorConfig(data_gen=partial(generate_input2, batch)), - "input3": - TensorConfig(data_gen=partial(generate_input3, batch)) + "input1": TensorConfig( + data_gen=partial(generate_input1, batch) + ), + "input2": TensorConfig( + data_gen=partial(generate_input2, batch) + ), + "input3": TensorConfig( + data_gen=partial(generate_input3, batch) + ), }, - outputs=["output"]) + outputs=["output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = { "input1": [1, 3, 24, 24], "input2": [1, 3, 24, 24], - "input3": [1, 3, 24, 24] + "input3": [1, 3, 24, 24], } self.dynamic_shape.max_input_shape = { "input1": [4, 3, 48, 48], "input2": [4, 3, 48, 48], - "input3": [4, 3, 48, 48] + "input3": [4, 3, 48, 48], } self.dynamic_shape.opt_input_shape = { "input1": [1, 3, 24, 24], "input2": [1, 3, 24, 24], - "input3": [1, 3, 24, 24] + "input3": [1, 3, 24, 24], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "input1": [1, 3, 24], "input2": [1, 3, 24], - "input3": [1, 3, 24] + "input3": [1, 3, 24], } self.dynamic_shape.max_input_shape = { "input1": [4, 3, 48], "input2": [4, 3, 48], - "input3": [4, 3, 48] + "input3": [4, 3, 48], } self.dynamic_shape.opt_input_shape = { "input1": [1, 3, 24], "input2": [1, 3, 24], - "input3": [1, 3, 24] + "input3": [1, 3, 24], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "input1": [1, 24], "input2": [1, 24], - "input3": [1, 24] + "input3": [1, 24], } self.dynamic_shape.max_input_shape = { "input1": [4, 48], "input2": [4, 48], - "input3": [4, 48] + "input3": [4, 48], } self.dynamic_shape.opt_input_shape = { "input1": [1, 24], "input2": [1, 24], - "input3": [1, 24] + "input3": [1, 24], } elif self.dims == 1: self.dynamic_shape.min_input_shape = { "input1": [24], "input2": [24], - "input3": [24] + "input3": [24], } self.dynamic_shape.max_input_shape = { "input1": [48], "input2": [48], - "input3": [48] + "input3": [48], } self.dynamic_shape.opt_input_shape = { "input1": [24], "input2": [24], - "input3": [24] + "input3": [24], } def clear_dynamic_shape(): @@ -162,7 +162,7 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): self.dynamic_shape.opt_input_shape = {} def generate_trt_nodes_num(dynamic_shape): - if (self.dims == 1 and not dynamic_shape): + if self.dims == 1 and not dynamic_shape: return 0, 5 return 1, 4 @@ -170,10 +170,12 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-5 + False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-3 + False + ), 1e-3 # for dynamic_shape generate_dynamic_shape() @@ -188,12 +190,10 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): # special case when sum having olny one input class TrtConvertSumTest1(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -207,31 +207,31 @@ class TrtConvertSumTest1(TrtLayerAutoScanTest): for dims in [1, 2, 3, 4]: for batch in [1, 4]: self.dims = dims - ops_config = [{ - "op_type": "sum", - "op_inputs": { - "X": ["input1"] - }, - "op_outputs": { - "Out": ["output"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "sum", + "op_inputs": {"X": ["input1"]}, + "op_outputs": {"Out": ["output"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1": - TensorConfig(data_gen=partial(generate_input1, batch)), + "input1": TensorConfig( + data_gen=partial(generate_input1, batch) + ), }, - outputs=["output"]) + outputs=["output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = {"input1": [1, 3, 24, 24]} @@ -268,7 +268,7 @@ class TrtConvertSumTest1(TrtLayerAutoScanTest): self.dynamic_shape.opt_input_shape = {} def generate_trt_nodes_num(dynamic_shape): - if (self.dims == 1 and not dynamic_shape): + if self.dims == 1 and not dynamic_shape: return 0, 3 return 1, 2 @@ -276,10 +276,12 @@ class TrtConvertSumTest1(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-5 + False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-3 + False + ), 1e-3 # for dynamic_shape generate_dynamic_shape() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_swish.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_swish.py index 1b5398bb483..b31e165c2e0 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_swish.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_swish.py @@ -22,12 +22,10 @@ import unittest class TrtConvertSwishTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([3]).astype(np.float32) @@ -44,33 +42,34 @@ class TrtConvertSwishTest(TrtLayerAutoScanTest): dics = [{"beta": beta}] - ops_config = [{ - "op_type": "swish", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "swish", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dims, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dims, dics) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -115,19 +114,23 @@ class TrtConvertSwishTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py index 87cb2c6630d..d65d7e3c29f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py @@ -25,7 +25,6 @@ import hypothesis.strategies as st class TrtConvertTileTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -38,38 +37,37 @@ class TrtConvertTileTest(TrtLayerAutoScanTest): return True def sample_program_configs(self, *args, **kwargs): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 2, 3, 4]).astype(np.float32) dics = [{"repeat_times": kwargs['repeat_times']}] - ops_config = [{ - "op_type": "tile", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["tile_output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "tile", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["tile_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["tile_output_data"]) + outputs=["tile_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -98,19 +96,23 @@ class TrtConvertTileTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 @given(repeat_times=st.sampled_from([[100], [1, 2], [0, 3], [1, 2, 100]])) def test(self, *args, **kwargs): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k.py index 7e8caf4a7e4..b8150238407 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertActivationTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -44,34 +43,37 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): for k in [1, 3]: self.dims = dims dics = [{"k": k}] - ops_config = [{ - "op_type": "top_k", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"], - "Indices": ["indices_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "top_k", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": { + "Out": ["output_data"], + "Indices": ["indices_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dims, batch, dics + ) + ) }, - outputs=["output_data", "indices_data"]) + outputs=["output_data", "indices_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -114,19 +116,23 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 ## for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k_v2.py index c4114e0d4c5..477ea649eff 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k_v2.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertActivationTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -53,40 +52,48 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): for sort in [True, False]: self.dims = dims self.sort = sort - dics = [{ - "k": k, - "axis": axis, - "largest": largest, - "sorted": sort - }] - ops_config = [{ - "op_type": "top_k_v2", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"], - "Indices": ["indices_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "k": k, + "axis": axis, + "largest": largest, + "sorted": sort, + } + ] + ops_config = [ + { + "op_type": "top_k_v2", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": { + "Out": ["output_data"], + "Indices": ["indices_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, + dims, + batch, + dics, + ) + ) }, - outputs=["output_data", "indices_data"]) + outputs=["output_data", "indices_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -131,19 +138,23 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_transpose.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_transpose.py index 684a47f4232..9ebcd873992 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_transpose.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_transpose.py @@ -22,7 +22,6 @@ import unittest class TrtConvertTransposeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,14 +31,13 @@ class TrtConvertTransposeTest(TrtLayerAutoScanTest): program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The shape of input and axis should be equal. + # The shape of input and axis should be equal. if len(inputs['transpose_input'].shape) != len(attrs[0]['axis']): return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -50,37 +48,43 @@ class TrtConvertTransposeTest(TrtLayerAutoScanTest): for dims in [2, 3, 4]: for batch in [1, 2, 4]: - for axis in [[0, 1, 3, 2], [0, 3, 2, 1], [3, 2, 0, 1], - [0, 1, 2, 3], [0, 1, 2], [2, 0, 1], [1, 0], [0, - 1]]: + for axis in [ + [0, 1, 3, 2], + [0, 3, 2, 1], + [3, 2, 0, 1], + [0, 1, 2, 3], + [0, 1, 2], + [2, 0, 1], + [1, 0], + [0, 1], + ]: self.dims = dims dics = [{"axis": axis}, {}] - ops_config = [{ - "op_type": "transpose", - "op_inputs": { - "X": ["transpose_input"] - }, - "op_outputs": { - "Out": ["transpose_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "transpose", + "op_inputs": {"X": ["transpose_input"]}, + "op_outputs": {"Out": ["transpose_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "transpose_input": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "transpose_input": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["transpose_out"]) + outputs=["transpose_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -134,19 +138,23 @@ class TrtConvertTransposeTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unary.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unary.py index 65656fd96d6..56b17df2dd0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unary.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unary.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertActivationTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -42,40 +41,54 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): for dims in [1, 2, 3, 4]: for batch in [1, 4]: for op_type in [ - "exp", "log", "sqrt", "abs", "sin", "cos", "tan", - "sinh", "cosh", "asin", "acos", "atan", "asinh", - "atanh", "ceil", "floor" + "exp", + "log", + "sqrt", + "abs", + "sin", + "cos", + "tan", + "sinh", + "cosh", + "asin", + "acos", + "atan", + "asinh", + "atanh", + "ceil", + "floor", ]: self.dims = dims dics = [{}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dims, batch, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -118,19 +131,23 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unfold.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unfold.py index 685658ca746..5ec187daef4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unfold.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unfold.py @@ -22,46 +22,46 @@ import unittest class TrtConvertUnfold(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([1, 3, 24, 24]).astype(np.float32) - ops_config = [{ - "op_type": "unfold", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Y": ["output_data"] - }, - "op_attrs": { - "dilations": [1, 1], - "kernel_sizes": [4, 4], - "paddings": [0, 0, 0, 0], - "strides": [1, 1], + ops_config = [ + { + "op_type": "unfold", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Y": ["output_data"]}, + "op_attrs": { + "dilations": [1, 1], + "kernel_sizes": [4, 4], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, } - }] + ] ops = self.generate_op_config(ops_config) for i in range(10): program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 3, 4, 4], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unsqueeze2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unsqueeze2.py index b18e98e9690..d79f4ddf7fc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unsqueeze2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unsqueeze2.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List class TrtConvertSplitTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -34,17 +33,17 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): self.dims = dims self.axes = axes dics = [{"axes": axes}] - ops_config = [{ - "op_type": "unsqueeze2", - "op_inputs": { - "X": ["in_data"] - }, - "op_outputs": { - "Out": ["out_data"], - "XShape": ["XShape_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "unsqueeze2", + "op_inputs": {"X": ["in_data"]}, + "op_outputs": { + "Out": ["out_data"], + "XShape": ["XShape_data"], + }, + "op_attrs": dics[0], + } + ] # generate input data self.input_shape = [1] * dims @@ -54,24 +53,26 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): def generate_input1(attrs: List[Dict[str, Any]], batch): self.input_shape[0] = batch return np.random.random(self.input_shape).astype( - np.float32) + np.float32 + ) ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "in_data": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "in_data": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["out_data"]) + outputs=["out_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): max_shape = list(self.input_shape) min_shape = list(self.input_shape) @@ -98,19 +99,23 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box.py index ed9bb722a8a..a0d089c69c9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box.py @@ -23,19 +23,19 @@ import os class TrtConvertYoloBoxTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch, channel): if attrs[0]['iou_aware'] == True: - return np.ones([batch, 3 * (channel + 6), 13, - 13]).astype(np.float32) + return np.ones([batch, 3 * (channel + 6), 13, 13]).astype( + np.float32 + ) else: - return np.ones([batch, 3 * (channel + 5), 13, - 13]).astype(np.float32) + return np.ones([batch, 3 * (channel + 5), 13, 13]).astype( + np.float32 + ) def generate_input2(attrs: List[Dict[str, Any]], batch): return np.random.random([batch, 2]).astype(np.int32) @@ -49,89 +49,92 @@ class TrtConvertYoloBoxTest(TrtLayerAutoScanTest): for scale_x_y in [1.0, 0.9]: for iou_aware in [False, True]: for iou_aware_factor in [0.5]: - dics = [{ - "class_num": - class_num, - "anchors": - anchors, - "downsample_ratio": - downsample_ratio, - "conf_thresh": - conf_thresh, - "clip_bbox": - clip_bbox, - "scale_x_y": - scale_x_y, - "iou_aware": - iou_aware, - "iou_aware_factor": - iou_aware_factor - }, {}] - ops_config = [{ - "op_type": "yolo_box", - "op_inputs": { - "X": ["yolo_box_input"], - "ImgSize": ["imgsize"] - }, - "op_outputs": { - "Boxes": ["boxes"], - "Scores": ["scores"] + dics = [ + { + "class_num": class_num, + "anchors": anchors, + "downsample_ratio": downsample_ratio, + "conf_thresh": conf_thresh, + "clip_bbox": clip_bbox, + "scale_x_y": scale_x_y, + "iou_aware": iou_aware, + "iou_aware_factor": iou_aware_factor, }, - "op_attrs": dics[0] - }] + {}, + ] + ops_config = [ + { + "op_type": "yolo_box", + "op_inputs": { + "X": ["yolo_box_input"], + "ImgSize": ["imgsize"], + }, + "op_outputs": { + "Boxes": ["boxes"], + "Scores": ["scores"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config( - ops_config) + ops_config + ) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "yolo_box_input": - TensorConfig( + "yolo_box_input": TensorConfig( data_gen=partial( generate_input1, - dics, batch, - class_num)), - "imgsize": - TensorConfig( + dics, + batch, + class_num, + ) + ), + "imgsize": TensorConfig( data_gen=partial( generate_input2, - dics, batch)) + dics, + batch, + ) + ), }, - outputs=["boxes", "scores"]) + outputs=["boxes", "scores"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if attrs[0]['iou_aware'] == True: channel = 3 * (attrs[0]['class_num'] + 6) self.dynamic_shape.min_input_shape = { "yolo_box_input": [1, channel, 12, 12], - "imgsize": [1, 2] + "imgsize": [1, 2], } self.dynamic_shape.max_input_shape = { "yolo_box_input": [4, channel, 24, 24], - "imgsize": [4, 2] + "imgsize": [4, 2], } self.dynamic_shape.opt_input_shape = { "yolo_box_input": [1, channel, 24, 24], - "imgsize": [1, 2] + "imgsize": [1, 2], } else: channel = 3 * (attrs[0]['class_num'] + 5) self.dynamic_shape.min_input_shape = { "yolo_box_input": [1, channel, 12, 12], - "imgsize": [1, 2] + "imgsize": [1, 2], } self.dynamic_shape.max_input_shape = { "yolo_box_input": [4, channel, 24, 24], - "imgsize": [4, 2] + "imgsize": [4, 2], } self.dynamic_shape.opt_input_shape = { "yolo_box_input": [1, channel, 24, 24], - "imgsize": [1, 2] + "imgsize": [1, 2], } def clear_dynamic_shape(): @@ -149,30 +152,35 @@ class TrtConvertYoloBoxTest(TrtLayerAutoScanTest): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): - def teller2(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller2, SkipReasons.TRT_NOT_SUPPORT, - "The output has diff between gpu and trt in Windows.") + teller2, + SkipReasons.TRT_NOT_SUPPORT, + "The output has diff between gpu and trt in Windows.", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box_head.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box_head.py index 08a09338bf2..e8b4006933f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box_head.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box_head.py @@ -22,17 +22,18 @@ import unittest class TrtConvertYoloBoxHeadTest(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(attrs: List[Dict[str, Any]], batch, shape): gen_shape = shape.copy() gen_shape.insert(0, batch) return np.random.uniform(0, 1, gen_shape).astype("float32") input_shape = [[255, 19, 19], [255, 38, 38], [255, 76, 76]] - anchors = [[116, 90, 156, 198, 373, 326], [30, 61, 62, 45, 59, 119], - [10, 13, 16, 30, 33, 23]] + anchors = [ + [116, 90, 156, 198, 373, 326], + [30, 61, 62, 45, 59, 119], + [10, 13, 16, 30, 33, 23], + ] class_num = 80 for batch in [1, 4]: for i in range(len(anchors)): @@ -40,31 +41,40 @@ class TrtConvertYoloBoxHeadTest(TrtLayerAutoScanTest): "anchors": anchors[i], "class_num": class_num, } - ops_config = [{ - "op_type": "yolo_box_head", - "op_inputs": { - "X": ["yolo_box_head_input"], - }, - "op_outputs": { - "Out": ["yolo_box_head_output"], - }, - "op_attrs": attrs_dict - }] + ops_config = [ + { + "op_type": "yolo_box_head", + "op_inputs": { + "X": ["yolo_box_head_input"], + }, + "op_outputs": { + "Out": ["yolo_box_head_output"], + }, + "op_attrs": attrs_dict, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "yolo_box_head_input": - TensorConfig(data_gen=partial( - generate_input, attrs_dict, batch, input_shape[i])) + "yolo_box_head_input": TensorConfig( + data_gen=partial( + generate_input, + attrs_dict, + batch, + input_shape[i], + ) + ) }, - outputs=["yolo_box_head_output"]) + outputs=["yolo_box_head_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): + self, program_config + ) -> (paddle_infer.Config, List[int], float): # for static_shape self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), [1, 2], 1e-5 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_deformable_conv.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_deformable_conv.py index 21949e3d780..eb79b4ccd1d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_deformable_conv.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_deformable_conv.py @@ -25,19 +25,18 @@ os.environ['NVIDIA_TF32_OVERRIDE'] = '0' class TRTDeformableConvTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - input = fluid.data(name='input', - shape=self.input_size, - dtype=self.dtype) - offset = fluid.data(name='offset', - shape=self.offset_size, - dtype=self.dtype) - mask = fluid.data(name='mask', - shape=self.mask_size, - dtype=self.dtype) + input = fluid.data( + name='input', shape=self.input_size, dtype=self.dtype + ) + offset = fluid.data( + name='offset', shape=self.offset_size, dtype=self.dtype + ) + mask = fluid.data( + name='mask', shape=self.mask_size, dtype=self.dtype + ) output = fluid.layers.deformable_conv( input, @@ -50,19 +49,21 @@ class TRTDeformableConvTest(InferencePassTest): dilation=self.dilations, groups=self.groups, deformable_groups=self.deformable_groups, - im2col_step=self.im2col_step) + im2col_step=self.im2col_step, + ) self.feeds = { 'input': np.random.random(self.input_size).astype(self.dtype), 'offset': np.random.random(self.offset_size).astype(self.dtype), - 'mask': np.random.random(self.mask_size).astype(self.dtype) + 'mask': np.random.random(self.mask_size).astype(self.dtype), } self.enable_trt = True dtype = AnalysisConfig.Precision.Float32 if self.dtype == 'float16': dtype = AnalysisConfig.Precision.Half self.trt_parameters = TRTDeformableConvTest.TensorRTParam( - 1 << 30, self.bs, 0, dtype, False, False) + 1 << 30, self.bs, 0, dtype, False, False + ) self.fetch_list = [output] def set_params(self): @@ -77,13 +78,21 @@ class TRTDeformableConvTest(InferencePassTest): self.input_size = [self.bs, 8, 4, 4] self.num_filters = 8 self.filter_size = 3 - offset_c = 2 * self.deformable_groups * self.filter_size * self.filter_size + offset_c = ( + 2 * self.deformable_groups * self.filter_size * self.filter_size + ) mask_c = self.deformable_groups * self.filter_size * self.filter_size self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] self.dtype = 'float32' @@ -93,7 +102,8 @@ class TRTDeformableConvTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py index ae5f9f94673..ee021b24a3b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py @@ -21,27 +21,32 @@ from paddle.fluid.core import AnalysisConfig class TRTDynamicShapeTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 16, 16], - dtype="float32") - out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - groups=1, - padding=[1, 1], - bias_attr=False, - act=None) + data = fluid.data( + name="data", shape=[-1, 3, 16, 16], dtype="float32" + ) + out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + groups=1, + padding=[1, 1], + bias_attr=False, + act=None, + ) self.feeds = self.set_feeds() self.enable_trt = True self.trt_parameters = TRTDynamicShapeTest.TensorRTParam( - 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False + ) self.dynamic_shape_params = TRTDynamicShapeTest.DynamicShapeParam( - {'data': [1, 3, 8, 8]}, {'data': [1, 3, 32, 32]}, - {'data': [1, 3, 16, 16]}, False) + {'data': [1, 3, 8, 8]}, + {'data': [1, 3, 32, 32]}, + {'data': [1, 3, 16, 16]}, + False, + ) self.fetch_list = [out] def set_feeds(self): @@ -56,7 +61,6 @@ class TRTDynamicShapeTest(InferencePassTest): class TRTDynamicShapeOutOfBound1Test(TRTDynamicShapeTest): - def set_feeds(self): return { "data": np.random.random([1, 3, 64, 16]).astype("float32"), @@ -84,7 +88,6 @@ class TRTDynamicShapeOutOfBound1Test(TRTDynamicShapeTest): class TRTDynamicShapeOutOfBound3Test(TRTDynamicShapeTest): - def set_feeds(self): return { "data": np.random.random([1, 3, 4, 16]).astype("float32"), diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_elementwise_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_elementwise_op.py index b24595e3797..7fb0baf11ff 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_elementwise_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_elementwise_op.py @@ -24,15 +24,14 @@ from paddle.fluid.core import AnalysisConfig class TensorRTSubgraphPassElementwiseBroadcastTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[-1, 3, 64, 64], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 3, 64, 1], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 3, 64, 64], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 3, 64, 1], dtype="float32" + ) eltwise_out = self.append_eltwise(data1, data2) out = fluid.layers.batch_norm(eltwise_out, is_test=True) self.feeds = { @@ -40,8 +39,11 @@ class TensorRTSubgraphPassElementwiseBroadcastTest(InferencePassTest): "data2": np.random.random([1, 3, 64, 1]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassElementwiseBroadcastTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + self.trt_parameters = ( + TensorRTSubgraphPassElementwiseBroadcastTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) + ) self.fetch_list = [out] def append_eltwise(self, data1, data2): @@ -54,26 +56,27 @@ class TensorRTSubgraphPassElementwiseBroadcastTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassElementwiseBroadcastTest1( - TensorRTSubgraphPassElementwiseBroadcastTest): - + TensorRTSubgraphPassElementwiseBroadcastTest +): def append_eltwise(self, data1, data2): return fluid.layers.elementwise_sub(x=data1, y=data2, axis=0) class TensorRTSubgraphPassElementwiseBroadcastTest2( - TensorRTSubgraphPassElementwiseBroadcastTest): - + TensorRTSubgraphPassElementwiseBroadcastTest +): def append_eltwise(self, data1, data2): return fluid.layers.elementwise_mul(x=data1, y=data2, axis=0) class TensorRTSubgraphPassElementwiseBroadcastTest3( - TensorRTSubgraphPassElementwiseBroadcastTest): - + TensorRTSubgraphPassElementwiseBroadcastTest +): def append_eltwise(self, data1, data2): return fluid.layers.elementwise_div(x=data1, y=data2, axis=0) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_pass.py index 78436f644b4..606836c8d0f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_pass.py @@ -21,16 +21,14 @@ from paddle.fluid.core import AnalysisConfig class FCFusePassTRTTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128, 2, 2], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=128, - num_flatten_dims=1, - act="relu") + data = fluid.data( + name="data", shape=[32, 128, 2, 2], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=128, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -53,16 +51,14 @@ class FCFusePassTRTTest(InferencePassTest): class FCFusePassTRTStaticDims4Cols1Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128, 32, 8], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=1, - act="relu") + data = fluid.data( + name="data", shape=[32, 128, 32, 8], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -70,7 +66,8 @@ class FCFusePassTRTStaticDims4Cols1Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = FCFusePassTRTStaticDims4Cols1Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -82,16 +79,14 @@ class FCFusePassTRTStaticDims4Cols1Test(InferencePassTest): class FCFusePassTRTStaticDims4Cols2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[3, 24, 16, 16], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=32, - num_flatten_dims=2, - act="relu") + data = fluid.data( + name="data", shape=[3, 24, 16, 16], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=32, num_flatten_dims=2, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -99,7 +94,8 @@ class FCFusePassTRTStaticDims4Cols2Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = FCFusePassTRTStaticDims4Cols2Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -111,22 +107,27 @@ class FCFusePassTRTStaticDims4Cols2Test(InferencePassTest): class FCFusePassTRTDynamicDims2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[32, 128], dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=1, - act="relu") + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = {"data": np.random.random((32, 128)).astype("float32")} self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims2Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims2Test.DynamicShapeParam( - {'data': [1, 128]}, {'data': [64, 128]}, {'data': [32, 128]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims2Test.DynamicShapeParam( + {'data': [1, 128]}, + {'data': [64, 128]}, + {'data': [32, 128]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -138,23 +139,27 @@ class FCFusePassTRTDynamicDims2Test(InferencePassTest): class FCFusePassTRTDynamicDims3Cols1Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[32, 128, 32], dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=1, - act="relu") + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = {"data": np.random.random((32, 128, 32)).astype("float32")} self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims3Cols1Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims3Cols1Test.DynamicShapeParam( - {'data': [1, 128, 32]}, {'data': [64, 128, 32]}, - {'data': [32, 128, 32]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims3Cols1Test.DynamicShapeParam( + {'data': [1, 128, 32]}, + {'data': [64, 128, 32]}, + {'data': [32, 128, 32]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -166,23 +171,27 @@ class FCFusePassTRTDynamicDims3Cols1Test(InferencePassTest): class FCFusePassTRTDynamicDims3Cols2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[32, 128, 32], dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=2, - act="relu") + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=2, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = {"data": np.random.random((32, 128, 32)).astype("float32")} self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims3Cols2Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims3Cols2Test.DynamicShapeParam( - {'data': [1, 32, 32]}, {'data': [64, 256, 32]}, - {'data': [32, 128, 32]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims3Cols2Test.DynamicShapeParam( + {'data': [1, 32, 32]}, + {'data': [64, 256, 32]}, + {'data': [32, 128, 32]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -194,16 +203,14 @@ class FCFusePassTRTDynamicDims3Cols2Test(InferencePassTest): class FCFusePassTRTDynamicDims4Cols1Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 12, 4, 6], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=1, - act="relu") + data = fluid.data( + name="data", shape=[32, 12, 4, 6], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -211,10 +218,16 @@ class FCFusePassTRTDynamicDims4Cols1Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims4Cols1Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims4Cols1Test.DynamicShapeParam( - {'data': [1, 12, 4, 6]}, {'data': [64, 12, 4, 6]}, - {'data': [32, 12, 4, 6]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims4Cols1Test.DynamicShapeParam( + {'data': [1, 12, 4, 6]}, + {'data': [64, 12, 4, 6]}, + {'data': [32, 12, 4, 6]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -226,16 +239,14 @@ class FCFusePassTRTDynamicDims4Cols1Test(InferencePassTest): class FCFusePassTRTDynamicDims4Cols2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128, 32, 32], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=2, - act="relu") + data = fluid.data( + name="data", shape=[32, 128, 32, 32], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=2, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -243,10 +254,16 @@ class FCFusePassTRTDynamicDims4Cols2Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims4Cols2Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims4Cols2Test.DynamicShapeParam( - {'data': [1, 64, 32, 32]}, {'data': [64, 256, 32, 32]}, - {'data': [32, 128, 32, 32]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims4Cols2Test.DynamicShapeParam( + {'data': [1, 64, 32, 32]}, + {'data': [64, 256, 32, 32]}, + {'data': [32, 128, 32, 32]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -258,16 +275,14 @@ class FCFusePassTRTDynamicDims4Cols2Test(InferencePassTest): class FCFusePassTRTDynamicDims4Cols3Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128, 32, 32], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=3, - act="relu") + data = fluid.data( + name="data", shape=[32, 128, 32, 32], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=3, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -275,10 +290,16 @@ class FCFusePassTRTDynamicDims4Cols3Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims4Cols3Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims4Cols3Test.DynamicShapeParam( - {'data': [1, 128, 32, 32]}, {'data': [64, 128, 32, 32]}, - {'data': [32, 128, 32, 32]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims4Cols3Test.DynamicShapeParam( + {'data': [1, 128, 32, 32]}, + {'data': [64, 128, 32, 32]}, + {'data': [32, 128, 32, 32]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_quant_dequant_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_quant_dequant_pass.py index 847c836972f..179dcd140d8 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_quant_dequant_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_quant_dequant_pass.py @@ -23,19 +23,19 @@ from paddle.fluid.core import PassVersionChecker class FCQuantDequantFusePassTRTDims3Cols1Test(QuantDequantTest): - def setUp(self): - def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') - fc_out = fluid.layers.fc(input=self.data, - size=10, - num_flatten_dims=1, - bias_attr=False, - act="relu") + fc_out = fluid.layers.fc( + input=self.data, + size=10, + num_flatten_dims=1, + bias_attr=False, + act="relu", + ) result = fluid.layers.relu(fc_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) avg_loss = paddle.mean(loss) @@ -44,62 +44,64 @@ class FCQuantDequantFusePassTRTDims3Cols1Test(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random((1, 28, 28)).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = FCQuantDequantFusePassTRTDims3Cols1Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) - self.dynamic_shape_params = FCQuantDequantFusePassTRTDims3Cols1Test.DynamicShapeParam( - { - 'data': [1, 28, 28], - 'reshape2_1.tmp_0': [1, 1, 10] - }, { - 'data': [2, 28, 28], - 'reshape2_1.tmp_0': [2, 1, 10] - }, { - 'data': [1, 28, 28], - 'reshape2_1.tmp_0': [1, 1, 10] - }, False) + self.trt_parameters = ( + FCQuantDequantFusePassTRTDims3Cols1Test.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) + self.dynamic_shape_params = ( + FCQuantDequantFusePassTRTDims3Cols1Test.DynamicShapeParam( + {'data': [1, 28, 28], 'reshape2_1.tmp_0': [1, 1, 10]}, + {'data': [2, 28, 28], 'reshape2_1.tmp_0': [2, 1, 10]}, + {'data': [1, 28, 28], 'reshape2_1.tmp_0': [1, 1, 10]}, + False, + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-2, - flatten=False, - rtol=1e-2) + self.check_output_with_option( + use_gpu, atol=1e-2, flatten=False, rtol=1e-2 + ) self.assertTrue( PassVersionChecker.IsCompatible( - 'quant_conv2d_dequant_fuse_pass')) + 'quant_conv2d_dequant_fuse_pass' + ) + ) class FCQuantDequantFusePassTRTDims3Cols2Test(QuantDequantTest): - def setUp(self): - def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') - fc_out = fluid.layers.fc(input=self.data, - size=28, - num_flatten_dims=2, - bias_attr=False, - act=None) + fc_out = fluid.layers.fc( + input=self.data, + size=28, + num_flatten_dims=2, + bias_attr=False, + act=None, + ) c_out = fluid.layers.reshape(fc_out, shape=[0, 784]) result = fluid.layers.relu(c_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) @@ -109,64 +111,66 @@ class FCQuantDequantFusePassTRTDims3Cols2Test(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random((1, 28, 28)).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = FCQuantDequantFusePassTRTDims3Cols2Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) - self.dynamic_shape_params = FCQuantDequantFusePassTRTDims3Cols2Test.DynamicShapeParam( - { - 'data': [1, 28, 28], - 'reshape2_0.tmp_0': [1, 784] - }, { - 'data': [4, 28, 28], - 'reshape2_0.tmp_0': [4, 784] - }, { - 'data': [1, 28, 28], - 'reshape2_0.tmp_0': [1, 784] - }, False) + self.trt_parameters = ( + FCQuantDequantFusePassTRTDims3Cols2Test.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) + self.dynamic_shape_params = ( + FCQuantDequantFusePassTRTDims3Cols2Test.DynamicShapeParam( + {'data': [1, 28, 28], 'reshape2_0.tmp_0': [1, 784]}, + {'data': [4, 28, 28], 'reshape2_0.tmp_0': [4, 784]}, + {'data': [1, 28, 28], 'reshape2_0.tmp_0': [1, 784]}, + False, + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1e-1, flatten=False, rtol=1e-1 + ) self.assertTrue( PassVersionChecker.IsCompatible( - 'quant_conv2d_dequant_fuse_pass')) + 'quant_conv2d_dequant_fuse_pass' + ) + ) class FCQuantDequantFusePassTRTDims3Cols3Test(QuantDequantTest): - def setUp(self): - def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') label_shape = fluid.layers.reshape(self.label, shape=[1, 1, 1]) reshape_out = fluid.layers.reshape(self.data, shape=[1, 14, 14, 4]) - fc_out = fluid.layers.fc(input=reshape_out, - size=14, - num_flatten_dims=3, - bias_attr=False, - act=None) + fc_out = fluid.layers.fc( + input=reshape_out, + size=14, + num_flatten_dims=3, + bias_attr=False, + act=None, + ) c_out = fluid.layers.reshape(fc_out, shape=[1, 1, 2744]) result = fluid.layers.relu(c_out) loss = fluid.layers.cross_entropy(input=result, label=label_shape) @@ -176,49 +180,60 @@ class FCQuantDequantFusePassTRTDims3Cols3Test(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random((1, 28, 28)).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = FCQuantDequantFusePassTRTDims3Cols3Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) - self.dynamic_shape_params = FCQuantDequantFusePassTRTDims3Cols3Test.DynamicShapeParam( - { - 'data': [1, 28, 28], - "reshape2_1.tmp_0": [1, 14, 14, 4], - "reshape2_2.tmp_0": [1, 1, 2744] - }, { - 'data': [4, 28, 28], - "reshape2_1.tmp_0": [4, 14, 14, 4], - "reshape2_2.tmp_0": [4, 1, 2744] - }, { - 'data': [1, 28, 28], - "reshape2_1.tmp_0": [1, 14, 14, 4], - "reshape2_2.tmp_0": [1, 1, 2744] - }, False) + self.trt_parameters = ( + FCQuantDequantFusePassTRTDims3Cols3Test.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) + self.dynamic_shape_params = ( + FCQuantDequantFusePassTRTDims3Cols3Test.DynamicShapeParam( + { + 'data': [1, 28, 28], + "reshape2_1.tmp_0": [1, 14, 14, 4], + "reshape2_2.tmp_0": [1, 1, 2744], + }, + { + 'data': [4, 28, 28], + "reshape2_1.tmp_0": [4, 14, 14, 4], + "reshape2_2.tmp_0": [4, 1, 2744], + }, + { + 'data': [1, 28, 28], + "reshape2_1.tmp_0": [1, 14, 14, 4], + "reshape2_2.tmp_0": [1, 1, 2744], + }, + False, + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e0, - flatten=False, - rtol=1e0) + self.check_output_with_option( + use_gpu, atol=1e0, flatten=False, rtol=1e0 + ) self.assertTrue( PassVersionChecker.IsCompatible( - 'quant_conv2d_dequant_fuse_pass')) + 'quant_conv2d_dequant_fuse_pass' + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten2_matmul_fuse_pass.py index 698eae07ccf..b163d429e47 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten2_matmul_fuse_pass.py @@ -42,7 +42,8 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['mul', 'elementwise_add'], (1e-4, 1e-1) def add_ignore_pass_case(self): @@ -65,9 +66,10 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape and attr of flatten2 x_shape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=10), min_size=4, max_size=4 + ) + ) # [a, b, c, d] => [a, b*c*d] flatten_axis = 1 flatten_shape = [x_shape[0], x_shape[1] * x_shape[2] * x_shape[3]] @@ -79,9 +81,10 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = flatten_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -98,17 +101,11 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): "X": ["flatten2_x"], }, axis=flatten_axis, - outputs={ - "Out": ["flatten2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["flatten2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["flatten2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["flatten2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -123,10 +120,7 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -148,9 +142,11 @@ class TestFlatten2MatmulFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=25, - passes=["trt_flatten2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=25, + passes=["trt_flatten2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten_op.py index 41c0c7621ac..6d230134a35 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten_op.py @@ -22,12 +22,11 @@ from paddle.fluid.core import AnalysisConfig class TRTFlattenTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) flatten_out = self.append_flatten(data) out = fluid.layers.batch_norm(flatten_out, is_test=True) self.feeds = { @@ -35,7 +34,8 @@ class TRTFlattenTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTFlattenTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_flatten(self, data): @@ -46,16 +46,16 @@ class TRTFlattenTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTFlattenDynamicTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) flatten_out = self.append_flatten(data) out = fluid.layers.batch_norm(flatten_out, is_test=True) self.feeds = { @@ -63,18 +63,14 @@ class TRTFlattenDynamicTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTFlattenDynamicTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.dynamic_shape_params = TRTFlattenDynamicTest.DynamicShapeParam( - { - 'data': [2, 6, 64, 64], - 'flatten_0.tmp_0': [2, 6 * 64 * 64] - }, { - 'data': [2, 6, 64, 64], - 'flatten_0.tmp_0': [2, 6 * 64 * 64] - }, { - 'data': [2, 6, 64, 64], - 'flatten_0.tmp_0': [2, 6 * 64 * 64] - }, False) + {'data': [2, 6, 64, 64], 'flatten_0.tmp_0': [2, 6 * 64 * 64]}, + {'data': [2, 6, 64, 64], 'flatten_0.tmp_0': [2, 6 * 64 * 64]}, + {'data': [2, 6, 64, 64], 'flatten_0.tmp_0': [2, 6 * 64 * 64]}, + False, + ) self.fetch_list = [out] def append_flatten(self, data): @@ -85,7 +81,8 @@ class TRTFlattenDynamicTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py index 2f8a711287c..75dbf5477b0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py @@ -22,7 +22,6 @@ from paddle.fluid.core import AnalysisConfig class TRTGatherNdTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[-1, 3, 4], dtype="float32") @@ -32,40 +31,37 @@ class TRTGatherNdTest(InferencePassTest): self.feeds = { "data": np.random.random([2, 3, 4]).astype("float32"), - "index": np.array([[[0, 1], [1, 0]], [[1, 2], - [0, 1]]]).astype("int32"), + "index": np.array([[[0, 1], [1, 0]], [[1, 2], [0, 1]]]).astype( + "int32" + ), } self.enable_trt = True self.trt_parameters = TRTGatherNdTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTGatherNdTest.DynamicShapeParam( - { - 'data': [1, 3, 4], - 'index': [1, 2, 2] - }, { - 'data': [3, 3, 4], - 'index': [3, 2, 2] - }, { - 'data': [3, 3, 4], - 'index': [3, 2, 2] - }, False) + {'data': [1, 3, 4], 'index': [1, 2, 2]}, + {'data': [3, 3, 4], 'index': [3, 2, 2]}, + {'data': [3, 3, 4], 'index': [3, 2, 2]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTGatherNdFp16Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 1280, 192], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 1280, 192], dtype="float32" + ) index = fluid.data(name="index", shape=[-1, 1028, 2], dtype="int32") gather_nd = fluid.layers.gather_nd(data, index) out = fluid.layers.batch_norm(gather_nd, is_test=True) @@ -77,26 +73,23 @@ class TRTGatherNdFp16Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTGatherNdFp16Test.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTGatherNdFp16Test.DynamicShapeParam( - { - 'data': [1, 1280, 192], - 'index': [1, 1028, 2] - }, { - 'data': [3, 1280, 192], - 'index': [3, 1028, 2] - }, { - 'data': [3, 1280, 192], - 'index': [3, 1028, 2] - }, False) + {'data': [1, 1280, 192], 'index': [1, 1028, 2]}, + {'data': [3, 1280, 192], 'index': [3, 1028, 2]}, + {'data': [3, 1280, 192], 'index': [3, 1028, 2]}, + False, + ) def test_check_output(self, atol=1e-3): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_op.py index c765d27f0bc..72c62ff4127 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_op.py @@ -22,7 +22,6 @@ from paddle.fluid.core import AnalysisConfig class TRTGatherTest1(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): @@ -33,23 +32,19 @@ class TRTGatherTest1(InferencePassTest): self.feeds = { "data": np.random.random([self.bs, 128]).astype("float32"), - "index": self.index + "index": self.index, } self.enable_trt = True self.trt_parameters = TRTGatherTest1.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.dynamic_shape_params = TRTGatherTest1.DynamicShapeParam( - { - 'data': [1, 1], - 'index': [1, 1] - }, { - 'data': [32, 128], - 'index': [3, 1] - }, { - 'data': [32, 128], - 'index': [3, 1] - }, False) + {'data': [1, 1], 'index': [1, 1]}, + {'data': [32, 128], 'index': [3, 1]}, + {'data': [32, 128], 'index': [3, 1]}, + False, + ) self.fetch_list = [out] def set_params(self): @@ -61,11 +56,11 @@ class TRTGatherTest1(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, flatten=False) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTGatherTest2(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): @@ -76,23 +71,19 @@ class TRTGatherTest2(InferencePassTest): self.feeds = { "data": np.random.random([self.bs, 64]).astype("float32"), - "index": self.index + "index": self.index, } self.enable_trt = True self.trt_parameters = TRTGatherTest2.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.dynamic_shape_params = TRTGatherTest2.DynamicShapeParam( - { - 'data': [2, 4], - 'index': [1] - }, { - 'data': [256, 256], - 'index': [4] - }, { - 'data': [64, 32], - 'index': [2] - }, False) + {'data': [2, 4], 'index': [1]}, + {'data': [256, 256], 'index': [4]}, + {'data': [64, 32], 'index': [2]}, + False, + ) self.fetch_list = [out] def set_params(self): @@ -104,7 +95,8 @@ class TRTGatherTest2(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, flatten=False) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_inspector.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_inspector.py index 2aba345ffc6..cab331aaa3c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_inspector.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_inspector.py @@ -24,16 +24,17 @@ import subprocess class TensorRTInspectorTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[1, 16, 16], dtype="float32") - matmul_out = fluid.layers.matmul(x=data, - y=data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=data, + y=data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) self.feeds = { @@ -41,7 +42,8 @@ class TensorRTInspectorTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, 1, 0, AnalysisConfig.Precision.Float32, False, False, True) + 1 << 30, 1, 0, AnalysisConfig.Precision.Float32, False, False, True + ) self.fetch_list = [out] def set_params(self): @@ -53,12 +55,16 @@ class TensorRTInspectorTest(InferencePassTest): if core.is_compiled_with_cuda(): build_engine = subprocess.run( [sys.executable, 'test_trt_inspector.py', '--build-engine'], - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + ) engine_info = build_engine.stderr.decode('ascii') trt_compile_version = paddle.inference.get_trt_compile_version() trt_runtime_version = paddle.inference.get_trt_runtime_version() valid_version = (8, 2, 0) - if trt_compile_version >= valid_version and trt_runtime_version >= valid_version: + if ( + trt_compile_version >= valid_version + and trt_runtime_version >= valid_version + ): self.assertTrue('====== engine info ======' in engine_info) self.assertTrue('====== engine info end ======' in engine_info) self.assertTrue('matmul' in engine_info) @@ -66,8 +72,9 @@ class TensorRTInspectorTest(InferencePassTest): self.assertTrue('batch_norm' in engine_info) else: self.assertTrue( - 'Inspector needs TensorRT version 8.2 and after.' in - engine_info) + 'Inspector needs TensorRT version 8.2 and after.' + in engine_info + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_instance_norm_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_instance_norm_op.py index 6939e1c1a00..c69394d7adf 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_instance_norm_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_instance_norm_op.py @@ -25,7 +25,6 @@ from paddle.fluid.core import AnalysisConfig class TRTInstanceNormTest(InferencePassTest): - def setUp(self): self.bs = 4 self.channel = 4 @@ -37,7 +36,8 @@ class TRTInstanceNormTest(InferencePassTest): def build(self): self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, 2, self.precision, self.serialize, False) + 1 << 30, self.bs, 2, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): shape = [-1, self.channel, self.height, self.width] @@ -61,7 +61,8 @@ class TRTInstanceNormTest(InferencePassTest): atol = 2e-2 self.check_output_with_option(use_gpu, atol, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self, remove_cache=False): self.build() @@ -69,12 +70,14 @@ class TRTInstanceNormTest(InferencePassTest): def run_all_tests(self): precision_opt = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_opt = [False, True] - for precision, serialize in itertools.product(precision_opt, - serialize_opt): + for precision, serialize in itertools.product( + precision_opt, serialize_opt + ): self.precision = precision self.serialize = serialize self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul.py index 14b0e9fa145..9e24c0f3233 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul.py @@ -22,16 +22,17 @@ from paddle.fluid.core import AnalysisConfig class TensorRTMatMulDims2Test(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[24, 24], dtype="float32") - matmul_out = fluid.layers.matmul(x=data, - y=data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=data, + y=data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) self.feeds = { @@ -39,7 +40,8 @@ class TensorRTMatMulDims2Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTMatMulDims2Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -52,22 +54,24 @@ class TensorRTMatMulDims2Test(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 24, 24], - dtype="float32") - matmul_out = fluid.layers.matmul(x=data, - y=data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + data = fluid.data( + name="data", shape=[-1, 6, 24, 24], dtype="float32" + ) + matmul_out = fluid.layers.matmul( + x=data, + y=data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) self.feeds = { @@ -75,7 +79,8 @@ class TensorRTMatMulTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTMatMulTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -88,11 +93,11 @@ class TensorRTMatMulTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulTransposeXTest(TensorRTMatMulTest): - def set_params(self): self.transpose_x = True self.transpose_y = False @@ -100,7 +105,6 @@ class TensorRTMatMulTransposeXTest(TensorRTMatMulTest): class TensorRTMatMulTransposeYTest(TensorRTMatMulTest): - def set_params(self): self.transpose_x = False self.transpose_y = True @@ -108,7 +112,6 @@ class TensorRTMatMulTransposeYTest(TensorRTMatMulTest): class TensorRTMatMulScaleTest(TensorRTMatMulTest): - def set_params(self): self.transpose_x = False self.transpose_y = False @@ -116,29 +119,31 @@ class TensorRTMatMulScaleTest(TensorRTMatMulTest): class TensorRTMatMulBroadcastTest(InferencePassTest): - def setUp(self): self.set_params() place = fluid.CPUPlace() with fluid.program_guard(self.main_program, self.startup_program): - data_x = fluid.data(name="data_x", - shape=[-1, 6, 24], - dtype="float32") + data_x = fluid.data( + name="data_x", shape=[-1, 6, 24], dtype="float32" + ) data_y = fluid.data(name="data_y", shape=[24, 16], dtype="float32") - matmul_out = fluid.layers.matmul(x=data_x, - y=data_y, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=data_x, + y=data_y, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) self.feeds = { "data_x": np.ones([2, 6, 24]).astype("float32"), - "data_y": np.ones([24, 16]).astype("float32") + "data_y": np.ones([24, 16]).astype("float32"), } self.enable_trt = True self.trt_parameters = TensorRTMatMulBroadcastTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -151,7 +156,8 @@ class TensorRTMatMulBroadcastTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul_quant_dequant.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul_quant_dequant.py index b075c65c821..75db2ebb221 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul_quant_dequant.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul_quant_dequant.py @@ -23,25 +23,28 @@ from paddle.fluid.core import AnalysisConfig class TensorRTMatMulQuantDequantDims3Test(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') - matmul_out = fluid.layers.matmul(x=self.data, - y=self.data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) - fc_out = fluid.layers.fc(input=matmul_out, - size=10, - num_flatten_dims=1, - bias_attr=False, - act=None) + matmul_out = fluid.layers.matmul( + x=self.data, + y=self.data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) + fc_out = fluid.layers.fc( + input=matmul_out, + size=10, + num_flatten_dims=1, + bias_attr=False, + act=None, + ) result = fluid.layers.relu(fc_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) avg_loss = paddle.mean(loss) @@ -50,21 +53,23 @@ class TensorRTMatMulQuantDequantDims3Test(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True self.trt_parameters = TensorRTMatMulQuantDequantDims3Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -74,20 +79,20 @@ class TensorRTMatMulQuantDequantDims3Test(QuantDequantTest): self.alpha = 1.0 def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulQuantDequantDims3TransposeXTest( - TensorRTMatMulQuantDequantDims3Test): - + TensorRTMatMulQuantDequantDims3Test +): def set_params(self): self.transpose_x = True self.transpose_y = False @@ -95,8 +100,8 @@ class TensorRTMatMulQuantDequantDims3TransposeXTest( class TensorRTMatMulQuantDequantDims3TransposeYTest( - TensorRTMatMulQuantDequantDims3Test): - + TensorRTMatMulQuantDequantDims3Test +): def set_params(self): self.transpose_x = False self.transpose_y = True @@ -104,8 +109,8 @@ class TensorRTMatMulQuantDequantDims3TransposeYTest( class TensorRTMatMulQuantDequantDims3TransposeXYTest( - TensorRTMatMulQuantDequantDims3Test): - + TensorRTMatMulQuantDequantDims3Test +): def set_params(self): self.transpose_x = True self.transpose_y = True @@ -113,27 +118,30 @@ class TensorRTMatMulQuantDequantDims3TransposeXYTest( class TensorRTMatMulQuantDequantDims4Test(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') reshape_out = fluid.layers.reshape(self.data, shape=[1, 4, 14, 14]) - matmul_out = fluid.layers.matmul(x=reshape_out, - y=reshape_out, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=reshape_out, + y=reshape_out, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) - fc_out = fluid.layers.fc(input=matmul_out, - size=10, - num_flatten_dims=1, - bias_attr=False, - act=None) + fc_out = fluid.layers.fc( + input=matmul_out, + size=10, + num_flatten_dims=1, + bias_attr=False, + act=None, + ) result = fluid.layers.relu(fc_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) avg_loss = paddle.mean(loss) @@ -142,21 +150,23 @@ class TensorRTMatMulQuantDequantDims4Test(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True self.trt_parameters = TensorRTMatMulQuantDequantDims4Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -166,20 +176,20 @@ class TensorRTMatMulQuantDequantDims4Test(QuantDequantTest): self.alpha = 1.0 def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulQuantDequantDims4TransposeXTest( - TensorRTMatMulQuantDequantDims4Test): - + TensorRTMatMulQuantDequantDims4Test +): def set_params(self): self.transpose_x = True self.transpose_y = False @@ -187,8 +197,8 @@ class TensorRTMatMulQuantDequantDims4TransposeXTest( class TensorRTMatMulQuantDequantDims4TransposeYTest( - TensorRTMatMulQuantDequantDims4Test): - + TensorRTMatMulQuantDequantDims4Test +): def set_params(self): self.transpose_x = False self.transpose_y = True @@ -196,8 +206,8 @@ class TensorRTMatMulQuantDequantDims4TransposeYTest( class TensorRTMatMulQuantDequantDims4TransposeXYTest( - TensorRTMatMulQuantDequantDims4Test): - + TensorRTMatMulQuantDequantDims4Test +): def set_params(self): self.transpose_x = True self.transpose_y = True @@ -205,26 +215,29 @@ class TensorRTMatMulQuantDequantDims4TransposeXYTest( class TensorRTMatMulQuantDequantDims3DynamicTest(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[-1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[-1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') - matmul_out = fluid.layers.matmul(x=self.data, - y=self.data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=self.data, + y=self.data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) - fc_out = fluid.layers.fc(input=matmul_out, - size=10, - num_flatten_dims=1, - bias_attr=False, - act=None) + fc_out = fluid.layers.fc( + input=matmul_out, + size=10, + num_flatten_dims=1, + bias_attr=False, + act=None, + ) result = fluid.layers.relu(fc_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) avg_loss = paddle.mean(loss) @@ -233,24 +246,33 @@ class TensorRTMatMulQuantDequantDims3DynamicTest(QuantDequantTest): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([3, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = TensorRTMatMulQuantDequantDims3DynamicTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) - self.dynamic_shape_params = TensorRTMatMulQuantDequantDims3DynamicTest.DynamicShapeParam( - {'data': [1, 28, 28]}, {'data': [4, 28, 28]}, {'data': [3, 28, 28]}, - False) + self.trt_parameters = ( + TensorRTMatMulQuantDequantDims3DynamicTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) + self.dynamic_shape_params = ( + TensorRTMatMulQuantDequantDims3DynamicTest.DynamicShapeParam( + {'data': [1, 28, 28]}, + {'data': [4, 28, 28]}, + {'data': [3, 28, 28]}, + False, + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -260,20 +282,20 @@ class TensorRTMatMulQuantDequantDims3DynamicTest(QuantDequantTest): self.alpha = 1.0 def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulQuantDequantDims4TransposeXDynamicTest( - TensorRTMatMulQuantDequantDims3DynamicTest): - + TensorRTMatMulQuantDequantDims3DynamicTest +): def set_params(self): self.transpose_x = True self.transpose_y = False @@ -281,8 +303,8 @@ class TensorRTMatMulQuantDequantDims4TransposeXDynamicTest( class TensorRTMatMulQuantDequantDims4TransposeYDynamicTest( - TensorRTMatMulQuantDequantDims3DynamicTest): - + TensorRTMatMulQuantDequantDims3DynamicTest +): def set_params(self): self.transpose_x = False self.transpose_y = True @@ -290,8 +312,8 @@ class TensorRTMatMulQuantDequantDims4TransposeYDynamicTest( class TensorRTMatMulQuantDequantDims4TransposeXYDynamicTest( - TensorRTMatMulQuantDequantDims3DynamicTest): - + TensorRTMatMulQuantDequantDims3DynamicTest +): def set_params(self): self.transpose_x = True self.transpose_y = True diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py index 91ca32ef899..05df3b65082 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py @@ -24,19 +24,21 @@ from paddle.fluid.core import PassVersionChecker from paddle.fluid.core import AnalysisConfig -def multiclass_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=-1, - return_index=False, - return_rois_num=True, - rois_num=None, - name=None): +def multiclass_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=-1, + return_index=False, + return_rois_num=True, + rois_num=None, + name=None, +): """ This operator is to do multi-class non maximum suppression (NMS) on boxes and scores. @@ -127,12 +129,25 @@ def multiclass_nms(bboxes, return_index=True) """ if in_dygraph_mode(): - attrs = ('background_label', background_label, 'score_threshold', - score_threshold, 'nms_top_k', nms_top_k, 'nms_threshold', - nms_threshold, 'keep_top_k', keep_top_k, 'nms_eta', nms_eta, - 'normalized', normalized) + attrs = ( + 'background_label', + background_label, + 'score_threshold', + score_threshold, + 'nms_top_k', + nms_top_k, + 'nms_threshold', + nms_threshold, + 'keep_top_k', + keep_top_k, + 'nms_eta', + nms_eta, + 'normalized', + normalized, + ) output, index, nms_rois_num = core.ops.multiclass_nms3( - bboxes, scores, rois_num, *attrs) + bboxes, scores, rois_num, *attrs + ) if not return_index: index = None return output, nms_rois_num, index @@ -150,21 +165,24 @@ def multiclass_nms(bboxes, if return_rois_num: nms_rois_num = helper.create_variable_for_type_inference( - dtype='int32') + dtype='int32' + ) outputs['NmsRoisNum'] = nms_rois_num - helper.append_op(type="multiclass_nms3", - inputs=inputs, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs=outputs) + helper.append_op( + type="multiclass_nms3", + inputs=inputs, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs=outputs, + ) output.stop_gradient = True index.stop_gradient = True if not return_index: @@ -176,7 +194,6 @@ def multiclass_nms(bboxes, class TensorRTMultiClassNMS3Test(InferencePassTest): - def setUp(self): self.enable_trt = True self.enable_tensorrt_varseqlen = True @@ -184,25 +201,28 @@ class TensorRTMultiClassNMS3Test(InferencePassTest): self.serialize = False self.bs = 1 self.background_label = -1 - self.score_threshold = .5 + self.score_threshold = 0.5 self.nms_top_k = 8 - self.nms_threshold = .3 + self.nms_threshold = 0.3 self.keep_top_k = 8 self.normalized = False self.num_classes = 8 self.num_boxes = 8 self.nms_eta = 1.1 self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, 2, self.precision, self.serialize, False) + 1 << 30, self.bs, 2, self.precision, self.serialize, False + ) def build(self): with fluid.program_guard(self.main_program, self.startup_program): - boxes = fluid.data(name='bboxes', - shape=[-1, self.num_boxes, 4], - dtype='float32') - scores = fluid.data(name='scores', - shape=[-1, self.num_classes, self.num_boxes], - dtype='float32') + boxes = fluid.data( + name='bboxes', shape=[-1, self.num_boxes, 4], dtype='float32' + ) + scores = fluid.data( + name='scores', + shape=[-1, self.num_classes, self.num_boxes], + dtype='float32', + ) multiclass_nms_out, _, _ = multiclass_nms( bboxes=boxes, scores=scores, @@ -212,17 +232,26 @@ class TensorRTMultiClassNMS3Test(InferencePassTest): nms_threshold=self.nms_threshold, keep_top_k=self.keep_top_k, normalized=self.normalized, - nms_eta=self.nms_eta) - mutliclass_nms_out = multiclass_nms_out + 1. + nms_eta=self.nms_eta, + ) + mutliclass_nms_out = multiclass_nms_out + 1.0 multiclass_nms_out = fluid.layers.reshape( - multiclass_nms_out, [self.bs, 1, self.keep_top_k, 6], - name='reshape') + multiclass_nms_out, + [self.bs, 1, self.keep_top_k, 6], + name='reshape', + ) out = fluid.layers.batch_norm(multiclass_nms_out, is_test=True) - boxes_data = np.arange(self.num_boxes * 4).reshape( - [self.bs, self.num_boxes, 4]).astype('float32') - scores_data = np.arange(1 * self.num_classes * self.num_boxes).reshape( - [self.bs, self.num_classes, self.num_boxes]).astype('float32') + boxes_data = ( + np.arange(self.num_boxes * 4) + .reshape([self.bs, self.num_boxes, 4]) + .astype('float32') + ) + scores_data = ( + np.arange(1 * self.num_classes * self.num_boxes) + .reshape([self.bs, self.num_classes, self.num_boxes]) + .astype('float32') + ) self.feeds = { 'bboxes': boxes_data, 'scores': scores_data, @@ -235,7 +264,8 @@ class TensorRTMultiClassNMS3Test(InferencePassTest): def run_test_all(self): precision_opt = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_opt = [False, True] max_shape = { @@ -246,13 +276,15 @@ class TensorRTMultiClassNMS3Test(InferencePassTest): dynamic_shape_opt = [ None, InferencePassTest.DynamicShapeParam( - { - 'bboxes': [1, 1, 4], - 'scores': [1, 1, 1] - }, max_shape, opt_shape, False) + {'bboxes': [1, 1, 4], 'scores': [1, 1, 1]}, + max_shape, + opt_shape, + False, + ), ] for precision, serialize, dynamic_shape in itertools.product( - precision_opt, serialize_opt, dynamic_shape_opt): + precision_opt, serialize_opt, dynamic_shape_opt + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -264,7 +296,8 @@ class TensorRTMultiClassNMS3Test(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def test_base(self): self.run_test() @@ -284,10 +317,11 @@ class TensorRTMultiClassNMS3Test(InferencePassTest): } opt_shape = max_shape self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( - { - 'bboxes': [1, 1, 4], - 'scores': [1, 1, 1] - }, max_shape, opt_shape, False) + {'bboxes': [1, 1, 4], 'scores': [1, 1, 1]}, + max_shape, + opt_shape, + False, + ) self.run_test() def test_background(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms_op.py index 5e04241f149..ead11ba7ae1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms_op.py @@ -23,7 +23,6 @@ from paddle.fluid.core import AnalysisConfig class TensorRTMultiClassNMSTest(InferencePassTest): - def setUp(self): self.enable_trt = True self.enable_tensorrt_varseqlen = True @@ -31,24 +30,27 @@ class TensorRTMultiClassNMSTest(InferencePassTest): self.serialize = False self.bs = 1 self.background_label = -1 - self.score_threshold = .5 + self.score_threshold = 0.5 self.nms_top_k = 8 - self.nms_threshold = .3 + self.nms_threshold = 0.3 self.keep_top_k = 8 self.normalized = False self.num_classes = 8 self.num_boxes = 8 self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, 2, self.precision, self.serialize, False) + 1 << 30, self.bs, 2, self.precision, self.serialize, False + ) def build(self): with fluid.program_guard(self.main_program, self.startup_program): - boxes = fluid.data(name='bboxes', - shape=[-1, self.num_boxes, 4], - dtype='float32') - scores = fluid.data(name='scores', - shape=[-1, self.num_classes, self.num_boxes], - dtype='float32') + boxes = fluid.data( + name='bboxes', shape=[-1, self.num_boxes, 4], dtype='float32' + ) + scores = fluid.data( + name='scores', + shape=[-1, self.num_classes, self.num_boxes], + dtype='float32', + ) multiclass_nms_out = fluid.layers.multiclass_nms( bboxes=boxes, scores=scores, @@ -57,17 +59,26 @@ class TensorRTMultiClassNMSTest(InferencePassTest): nms_top_k=self.nms_top_k, nms_threshold=self.nms_threshold, keep_top_k=self.keep_top_k, - normalized=self.normalized) - mutliclass_nms_out = multiclass_nms_out + 1. + normalized=self.normalized, + ) + mutliclass_nms_out = multiclass_nms_out + 1.0 multiclass_nms_out = fluid.layers.reshape( - multiclass_nms_out, [self.bs, 1, self.keep_top_k, 6], - name='reshape') + multiclass_nms_out, + [self.bs, 1, self.keep_top_k, 6], + name='reshape', + ) out = fluid.layers.batch_norm(multiclass_nms_out, is_test=True) - boxes_data = np.arange(self.num_boxes * 4).reshape( - [self.bs, self.num_boxes, 4]).astype('float32') - scores_data = np.arange(1 * self.num_classes * self.num_boxes).reshape( - [self.bs, self.num_classes, self.num_boxes]).astype('float32') + boxes_data = ( + np.arange(self.num_boxes * 4) + .reshape([self.bs, self.num_boxes, 4]) + .astype('float32') + ) + scores_data = ( + np.arange(1 * self.num_classes * self.num_boxes) + .reshape([self.bs, self.num_classes, self.num_boxes]) + .astype('float32') + ) self.feeds = { 'bboxes': boxes_data, 'scores': scores_data, @@ -80,7 +91,8 @@ class TensorRTMultiClassNMSTest(InferencePassTest): def run_test_all(self): precision_opt = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_opt = [False, True] max_shape = { @@ -91,13 +103,15 @@ class TensorRTMultiClassNMSTest(InferencePassTest): dynamic_shape_opt = [ None, InferencePassTest.DynamicShapeParam( - { - 'bboxes': [1, 1, 4], - 'scores': [1, 1, 1] - }, max_shape, opt_shape, False) + {'bboxes': [1, 1, 4], 'scores': [1, 1, 1]}, + max_shape, + opt_shape, + False, + ), ] for precision, serialize, dynamic_shape in itertools.product( - precision_opt, serialize_opt, dynamic_shape_opt): + precision_opt, serialize_opt, dynamic_shape_opt + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -109,7 +123,8 @@ class TensorRTMultiClassNMSTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def test_base(self): self.run_test() @@ -129,10 +144,11 @@ class TensorRTMultiClassNMSTest(InferencePassTest): } opt_shape = max_shape self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( - { - 'bboxes': [1, 1, 4], - 'scores': [1, 1, 1] - }, max_shape, opt_shape, False) + {'bboxes': [1, 1, 4], 'scores': [1, 1, 1]}, + max_shape, + opt_shape, + False, + ) self.run_test() def test_background(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_op.py index e5f5908fd03..3582d9848d5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_op.py @@ -22,20 +22,23 @@ from paddle.fluid.core import AnalysisConfig class TRTNearestInterpTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): if self.data_layout == 'NCHW': shape = [ - -1, self.channels, self.origin_shape[0], - self.origin_shape[1] + -1, + self.channels, + self.origin_shape[0], + self.origin_shape[1], ] else: shape = [ - -1, self.origin_shape[0], self.origin_shape[1], - self.channels + -1, + self.origin_shape[0], + self.origin_shape[1], + self.channels, ] data = fluid.data(name='data', shape=shape, dtype='float32') resize_out = self.append_nearest_interp(data) @@ -43,13 +46,17 @@ class TRTNearestInterpTest(InferencePassTest): if self.data_layout == 'NCHW': shape = [ - self.bs, self.channels, self.origin_shape[0], - self.origin_shape[1] + self.bs, + self.channels, + self.origin_shape[0], + self.origin_shape[1], ] else: shape = [ - self.bs, self.origin_shape[0], self.origin_shape[1], - self.channels + self.bs, + self.origin_shape[0], + self.origin_shape[1], + self.channels, ] self.feeds = { @@ -57,7 +64,8 @@ class TRTNearestInterpTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTNearestInterpTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -71,26 +79,30 @@ class TRTNearestInterpTest(InferencePassTest): self.data_layout = 'NCHW' def append_nearest_interp(self, data): - if self.scale > 0.: - return fluid.layers.resize_nearest(data, - scale=self.scale, - align_corners=self.align_corners, - data_format=self.data_layout) - return fluid.layers.resize_nearest(data, - out_shape=self.resize_shape, - align_corners=self.align_corners, - data_format=self.data_layout) + if self.scale > 0.0: + return fluid.layers.resize_nearest( + data, + scale=self.scale, + align_corners=self.align_corners, + data_format=self.data_layout, + ) + return fluid.layers.resize_nearest( + data, + out_shape=self.resize_shape, + align_corners=self.align_corners, + data_format=self.data_layout, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTNearestInterpTest1(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -102,10 +114,9 @@ class TRTNearestInterpTest1(TRTNearestInterpTest): class TRTNearestInterpTest2(TRTNearestInterpTest): - def set_params(self): self.bs = 4 - self.scale = 2. + self.scale = 2.0 self.channels = 3 self.origin_shape = (16, 16) # HW self.resize_shape = (32, 32) # HW @@ -114,7 +125,6 @@ class TRTNearestInterpTest2(TRTNearestInterpTest): class TRTNearestInterpTest3(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = 0 @@ -126,7 +136,6 @@ class TRTNearestInterpTest3(TRTNearestInterpTest): class TRTNearestInterpTest4(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -138,7 +147,6 @@ class TRTNearestInterpTest4(TRTNearestInterpTest): class TRTNearestInterpTest5(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -150,10 +158,9 @@ class TRTNearestInterpTest5(TRTNearestInterpTest): class TRTNearestInterpTest6(TRTNearestInterpTest): - def set_params(self): self.bs = 4 - self.scale = 2. + self.scale = 2.0 self.channels = 3 self.origin_shape = (16, 16) # HW self.resize_shape = (32, 32) # HW @@ -162,7 +169,6 @@ class TRTNearestInterpTest6(TRTNearestInterpTest): class TRTNearestInterpTest7(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -174,7 +180,6 @@ class TRTNearestInterpTest7(TRTNearestInterpTest): class TRTNearestInterpTest8(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -186,7 +191,6 @@ class TRTNearestInterpTest8(TRTNearestInterpTest): class TRTNearestInterpTest9(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_v2_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_v2_op.py index 8e6a28c483d..fc145fc410a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_v2_op.py @@ -23,20 +23,23 @@ from paddle.fluid.core import AnalysisConfig class TRTNearestInterpTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): if self.data_layout == 'NCHW': shape = [ - -1, self.channels, self.origin_shape[0], - self.origin_shape[1] + -1, + self.channels, + self.origin_shape[0], + self.origin_shape[1], ] else: shape = [ - -1, self.origin_shape[0], self.origin_shape[1], - self.channels + -1, + self.origin_shape[0], + self.origin_shape[1], + self.channels, ] data = fluid.data(name='data', shape=shape, dtype='float32') resize_out = self.append_nearest_interp(data) @@ -44,13 +47,17 @@ class TRTNearestInterpTest(InferencePassTest): if self.data_layout == 'NCHW': shape = [ - self.bs, self.channels, self.origin_shape[0], - self.origin_shape[1] + self.bs, + self.channels, + self.origin_shape[0], + self.origin_shape[1], ] else: shape = [ - self.bs, self.origin_shape[0], self.origin_shape[1], - self.channels + self.bs, + self.origin_shape[0], + self.origin_shape[1], + self.channels, ] self.feeds = { @@ -58,7 +65,8 @@ class TRTNearestInterpTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTNearestInterpTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -71,31 +79,35 @@ class TRTNearestInterpTest(InferencePassTest): self.data_layout = 'NCHW' def append_nearest_interp(self, data): - if self.scale > 0.: - return F.interpolate(data, - scale_factor=self.scale, - align_corners=self.align_corners, - mode='nearest', - data_format=self.data_layout) - return F.interpolate(data, - size=self.resize_shape, - align_corners=self.align_corners, - mode='nearest', - data_format=self.data_layout) + if self.scale > 0.0: + return F.interpolate( + data, + scale_factor=self.scale, + align_corners=self.align_corners, + mode='nearest', + data_format=self.data_layout, + ) + return F.interpolate( + data, + size=self.resize_shape, + align_corners=self.align_corners, + mode='nearest', + data_format=self.data_layout, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTNearestInterpTest1(TRTNearestInterpTest): - def set_params(self): self.bs = 4 - self.scale = 2. + self.scale = 2.0 self.channels = 3 self.origin_shape = (16, 16) # HW self.resize_shape = (32, 32) # HW @@ -104,7 +116,6 @@ class TRTNearestInterpTest1(TRTNearestInterpTest): class TRTNearestInterpTest2(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -116,7 +127,6 @@ class TRTNearestInterpTest2(TRTNearestInterpTest): class TRTNearestInterpTest3(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -128,10 +138,9 @@ class TRTNearestInterpTest3(TRTNearestInterpTest): class TRTNearestInterpTest4(TRTNearestInterpTest): - def set_params(self): self.bs = 4 - self.scale = 2. + self.scale = 2.0 self.channels = 3 self.origin_shape = (16, 16) # HW self.resize_shape = (32, 32) # HW @@ -140,7 +149,6 @@ class TRTNearestInterpTest4(TRTNearestInterpTest): class TRTNearestInterpTest5(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pad_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pad_op.py index ae6dbefa86d..338d7d8a481 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pad_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pad_op.py @@ -21,15 +21,14 @@ from paddle.fluid.core import AnalysisConfig class PadOpTRTTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[1, 3, 128, 128], - dtype="float32") - pad_out = fluid.layers.pad(x=data, - paddings=[0, 0, 0, 0, 0, 1, 1, 2], - pad_value=0.0) + data = fluid.data( + name="data", shape=[1, 3, 128, 128], dtype="float32" + ) + pad_out = fluid.layers.pad( + x=data, paddings=[0, 0, 0, 0, 0, 1, 1, 2], pad_value=0.0 + ) out = fluid.layers.batch_norm(pad_out, is_test=True) self.feeds = { @@ -37,7 +36,8 @@ class PadOpTRTTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = PadOpTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool3d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool3d_op.py index c4488a57f96..7e7499a19c7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool3d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool3d_op.py @@ -26,7 +26,6 @@ from paddle.fluid.core import AnalysisConfig class TensorRTPool3dTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 3 @@ -44,10 +43,9 @@ class TensorRTPool3dTest(InferencePassTest): self.serialize = False self.precision = AnalysisConfig.Precision.Float32 self.feeds = { - 'data': - np.random.random( - [self.bs, self.channel, self.depth, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.depth, self.height, self.width] + ).astype('float32'), } def set_extra_config(self): @@ -56,22 +54,26 @@ class TensorRTPool3dTest(InferencePassTest): def build_network(self): self.set_extra_config() self.trt_parameters = TensorRTPool3dTest.TensorRTParam( - 1 << 30, self.bs, 0, self.precision, self.serialize, False) + 1 << 30, self.bs, 0, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data( name='data', shape=[-1, self.channel, self.depth, self.height, self.width], - dtype='float32') - pool_out = fluid.layers.pool3d(input=data, - pool_size=self.pool_size, - pool_type=self.pool_type, - pool_stride=self.pool_stride, - pool_padding=self.pool_padding, - global_pooling=self.global_pooling, - ceil_mode=self.ceil_mode, - exclusive=self.exclusive) - #out = fluid.layers.batch_norm(pool_out, is_test=True) + dtype='float32', + ) + pool_out = fluid.layers.pool3d( + input=data, + pool_size=self.pool_size, + pool_type=self.pool_type, + pool_stride=self.pool_stride, + pool_padding=self.pool_padding, + global_pooling=self.global_pooling, + ceil_mode=self.ceil_mode, + exclusive=self.exclusive, + ) + # out = fluid.layers.batch_norm(pool_out, is_test=True) self.fetch_list = [pool_out] def check_output(self): @@ -84,11 +86,13 @@ class TensorRTPool3dTest(InferencePassTest): elif self.precision == AnalysisConfig.Precision.Half: atol, rtol = (1e-3, 1e-3) else: - raise ValueError("Unsupported precision {}".format( - self.precision)) + raise ValueError( + "Unsupported precision {}".format(self.precision) + ) self.check_output_with_option(use_gpu, atol=atol, rtol=rtol) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build_network() @@ -96,30 +100,51 @@ class TensorRTPool3dTest(InferencePassTest): def test(self): precision_options = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_options = [False, True] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( { 'data': [ - self.bs, self.channel, self.depth // 2, self.height // 2, - self.width // 2 + self.bs, + self.channel, + self.depth // 2, + self.height // 2, + self.width // 2, ] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, False) + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + False, + ) dynamic_shape_options = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_options, serialize_options, dynamic_shape_options): + precision_options, serialize_options, dynamic_shape_options + ): is_dynamic = True if dynamic_shape_options is not None else False with self.subTest( - 'Precision: {}, Serialize: {}, Dynamic: {}'.format( - precision, serialize, is_dynamic)): + 'Precision: {}, Serialize: {}, Dynamic: {}'.format( + precision, serialize, is_dynamic + ) + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -127,7 +152,6 @@ class TensorRTPool3dTest(InferencePassTest): class TensorRTAvgPool3dTest(TensorRTPool3dTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'avg' @@ -139,7 +163,6 @@ class TensorRTAvgPool3dTest(TensorRTPool3dTest): class TensorRTGlobalPool3dTest(TensorRTPool3dTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -151,7 +174,6 @@ class TensorRTGlobalPool3dTest(TensorRTPool3dTest): class TensorRTCeilPool3dTest(TensorRTPool3dTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -163,7 +185,6 @@ class TensorRTCeilPool3dTest(TensorRTPool3dTest): class TensorRTExclusivePool3dTest(TensorRTPool3dTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -175,7 +196,6 @@ class TensorRTExclusivePool3dTest(TensorRTPool3dTest): class TensorRTSamePaddingPool3dTest(InferencePassTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -187,7 +207,6 @@ class TensorRTSamePaddingPool3dTest(InferencePassTest): class TensorRTValidPaddingPool3dTest(InferencePassTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -199,7 +218,6 @@ class TensorRTValidPaddingPool3dTest(InferencePassTest): class TensorRTAdaptiveAvgPool3DTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 3 @@ -210,24 +228,26 @@ class TensorRTAdaptiveAvgPool3DTest(InferencePassTest): self.serialize = False self.precision = AnalysisConfig.Precision.Float32 self.feeds = { - 'data': - np.random.random( - [self.bs, self.channel, self.depth, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.depth, self.height, self.width] + ).astype('float32'), } def build_network(self): self.trt_parameters = TensorRTPool3dTest.TensorRTParam( - 1 << 30, self.bs, 0, self.precision, self.serialize, False) + 1 << 30, self.bs, 0, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data( name='data', shape=[-1, self.channel, self.depth, self.height, self.width], - dtype='float32') + dtype='float32', + ) pool_out = paddle.nn.functional.adaptive_avg_pool3d( - x=data, output_size=[3, 3, 3]) - #out = fluid.layers.batch_norm(pool_out, is_test=True) + x=data, output_size=[3, 3, 3] + ) + # out = fluid.layers.batch_norm(pool_out, is_test=True) self.fetch_list = [pool_out] def check_output(self): @@ -237,7 +257,8 @@ class TensorRTAdaptiveAvgPool3DTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build_network() @@ -245,30 +266,51 @@ class TensorRTAdaptiveAvgPool3DTest(InferencePassTest): def test(self): precision_options = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_options = [False, True] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( { 'data': [ - self.bs, self.channel, self.depth // 2, self.height // 2, - self.width // 2 + self.bs, + self.channel, + self.depth // 2, + self.height // 2, + self.width // 2, ] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, False) + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + False, + ) dynamic_shape_options = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_options, serialize_options, dynamic_shape_options): + precision_options, serialize_options, dynamic_shape_options + ): is_dynamic = True if dynamic_shape_options is not None else False with self.subTest( - 'Precision: {}, Serialize: {}, Dynamic: {}'.format( - precision, serialize, is_dynamic)): + 'Precision: {}, Serialize: {}, Dynamic: {}'.format( + precision, serialize, is_dynamic + ) + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -276,7 +318,6 @@ class TensorRTAdaptiveAvgPool3DTest(InferencePassTest): class TensorRTAdaptiveMaxPool3DTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 3 @@ -287,24 +328,26 @@ class TensorRTAdaptiveMaxPool3DTest(InferencePassTest): self.serialize = False self.precision = AnalysisConfig.Precision.Float32 self.feeds = { - 'data': - np.random.random( - [self.bs, self.channel, self.depth, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.depth, self.height, self.width] + ).astype('float32'), } def build_network(self): self.trt_parameters = TensorRTPool3dTest.TensorRTParam( - 1 << 30, self.bs, 0, self.precision, self.serialize, False) + 1 << 30, self.bs, 0, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data( name='data', shape=[-1, self.channel, self.depth, self.height, self.width], - dtype='float32') + dtype='float32', + ) pool_out = paddle.nn.functional.adaptive_max_pool3d( - x=data, output_size=[3, 3, 3]) - #out = fluid.layers.batch_norm(pool_out, is_test=True) + x=data, output_size=[3, 3, 3] + ) + # out = fluid.layers.batch_norm(pool_out, is_test=True) self.fetch_list = [pool_out] def check_output(self): @@ -314,7 +357,8 @@ class TensorRTAdaptiveMaxPool3DTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build_network() @@ -322,30 +366,51 @@ class TensorRTAdaptiveMaxPool3DTest(InferencePassTest): def test(self): precision_options = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_options = [False, True] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( { 'data': [ - self.bs, self.channel, self.depth // 2, self.height // 2, - self.width // 2 + self.bs, + self.channel, + self.depth // 2, + self.height // 2, + self.width // 2, ] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, False) + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + False, + ) dynamic_shape_options = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_options, serialize_options, dynamic_shape_options): + precision_options, serialize_options, dynamic_shape_options + ): is_dynamic = True if dynamic_shape_options is not None else False with self.subTest( - 'Precision: {}, Serialize: {}, Dynamic: {}'.format( - precision, serialize, is_dynamic)): + 'Precision: {}, Serialize: {}, Dynamic: {}'.format( + precision, serialize, is_dynamic + ) + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool_op.py index f6eaa2fb8c7..2ceb891024e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool_op.py @@ -25,7 +25,6 @@ from paddle.fluid.core import AnalysisConfig class TensorRTPoolTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 2 @@ -42,9 +41,9 @@ class TensorRTPoolTest(InferencePassTest): self.serialize = False self.precision = AnalysisConfig.Precision.Float32 self.feeds = { - 'data': - np.random.random([self.bs, self.channel, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.height, self.width] + ).astype('float32'), } def set_extra_config(self): @@ -53,20 +52,25 @@ class TensorRTPoolTest(InferencePassTest): def build_network(self): self.set_extra_config() self.trt_parameters = TensorRTPoolTest.TensorRTParam( - 1 << 30, self.bs, 0, self.precision, self.serialize, False) + 1 << 30, self.bs, 0, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=[-1, self.channel, self.height, self.width], - dtype='float32') - pool_out = fluid.layers.pool2d(input=data, - pool_size=self.pool_size, - pool_type=self.pool_type, - pool_stride=self.pool_stride, - pool_padding=self.pool_padding, - global_pooling=self.global_pooling, - ceil_mode=self.ceil_mode, - exclusive=self.exclusive) + data = fluid.data( + name='data', + shape=[-1, self.channel, self.height, self.width], + dtype='float32', + ) + pool_out = fluid.layers.pool2d( + input=data, + pool_size=self.pool_size, + pool_type=self.pool_type, + pool_stride=self.pool_stride, + pool_padding=self.pool_padding, + global_pooling=self.global_pooling, + ceil_mode=self.ceil_mode, + exclusive=self.exclusive, + ) out = fluid.layers.batch_norm(pool_out, is_test=True) self.fetch_list = [out] @@ -80,11 +84,13 @@ class TensorRTPoolTest(InferencePassTest): elif self.precision == AnalysisConfig.Precision.Half: atol, rtol = (1e-3, 1e-3) else: - raise ValueError("Unsupported precision {}".format( - self.precision)) + raise ValueError( + "Unsupported precision {}".format(self.precision) + ) self.check_output_with_option(use_gpu, atol=atol, rtol=rtol) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build_network() @@ -92,23 +98,34 @@ class TensorRTPoolTest(InferencePassTest): def test(self): precision_options = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_options = [False, True] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( { - 'data': - [self.bs, self.channel, self.height // 2, self.width // 2] - }, {'data': [self.bs, self.channel, self.height, self.width]}, - {'data': [self.bs, self.channel, self.height, self.width]}, False) + 'data': [ + self.bs, + self.channel, + self.height // 2, + self.width // 2, + ] + }, + {'data': [self.bs, self.channel, self.height, self.width]}, + {'data': [self.bs, self.channel, self.height, self.width]}, + False, + ) dynamic_shape_options = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_options, serialize_options, dynamic_shape_options): + precision_options, serialize_options, dynamic_shape_options + ): is_dynamic = True if dynamic_shape_options is not None else False with self.subTest( - 'Precision: {}, Serialize: {}, Dynamic: {}'.format( - precision, serialize, is_dynamic)): + 'Precision: {}, Serialize: {}, Dynamic: {}'.format( + precision, serialize, is_dynamic + ) + ): self.precision = precision self.serialize = serialize self.dynamic_shape = dynamic_shape @@ -116,7 +133,6 @@ class TensorRTPoolTest(InferencePassTest): class TensorRTAvgPoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'avg' @@ -128,7 +144,6 @@ class TensorRTAvgPoolTest(TensorRTPoolTest): class TensorRTAvgCeilPoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'avg' @@ -140,7 +155,6 @@ class TensorRTAvgCeilPoolTest(TensorRTPoolTest): class TensorRTGlobalPoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -152,7 +166,6 @@ class TensorRTGlobalPoolTest(TensorRTPoolTest): class TensorRTCeilPoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -164,7 +177,6 @@ class TensorRTCeilPoolTest(TensorRTPoolTest): class TensorRTExclusivePoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -176,7 +188,6 @@ class TensorRTExclusivePoolTest(TensorRTPoolTest): class TensorRTSamePaddingPoolTest(InferencePassTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -188,7 +199,6 @@ class TensorRTSamePaddingPoolTest(InferencePassTest): class TensorRTValidPaddingPoolTest(InferencePassTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_mean_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_mean_op.py index f699c5572fa..05621516d0b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_mean_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_mean_op.py @@ -22,15 +22,14 @@ from paddle.fluid.core import AnalysisConfig class TRTReduceMeanTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, -1, -1], - dtype="float32") - reduce_mean = fluid.layers.reduce_mean(data, - dim=[2, -1], - keep_dim=True) + data = fluid.data( + name="data", shape=[-1, 3, -1, -1], dtype="float32" + ) + reduce_mean = fluid.layers.reduce_mean( + data, dim=[2, -1], keep_dim=True + ) out = fluid.layers.batch_norm(reduce_mean, is_test=True) self.feeds = { @@ -38,27 +37,31 @@ class TRTReduceMeanTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceMeanTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceMeanTest.DynamicShapeParam( - {'data': [1, 3, 16, 16]}, {'data': [3, 3, 56, 56]}, - {'data': [3, 3, 56, 56]}, False) + {'data': [1, 3, 16, 16]}, + {'data': [3, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanAllNoBatchTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, -1, -1], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, -1, -1], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -67,30 +70,36 @@ class TRTReduceMeanAllNoBatchTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceMeanAllNoBatchTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] - self.dynamic_shape_params = TRTReduceMeanAllNoBatchTest.DynamicShapeParam( - {'data': [1, 3, 16, 16]}, {'data': [3, 3, 56, 56]}, - {'data': [3, 3, 56, 56]}, False) + self.dynamic_shape_params = ( + TRTReduceMeanAllNoBatchTest.DynamicShapeParam( + {'data': [1, 3, 16, 16]}, + {'data': [3, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + False, + ) + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanTestFP16(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, -1, -1], - dtype="float32") - reduce_mean = fluid.layers.reduce_mean(data, - dim=[2, -1], - keep_dim=True) + data = fluid.data( + name="data", shape=[-1, 3, -1, -1], dtype="float32" + ) + reduce_mean = fluid.layers.reduce_mean( + data, dim=[2, -1], keep_dim=True + ) out = fluid.layers.batch_norm(reduce_mean, is_test=True) self.feeds = { @@ -98,27 +107,31 @@ class TRTReduceMeanTestFP16(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceMeanTestFP16.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceMeanTestFP16.DynamicShapeParam( - {'data': [1, 3, 16, 16]}, {'data': [3, 3, 56, 56]}, - {'data': [3, 3, 56, 56]}, False) + {'data': [1, 3, 16, 16]}, + {'data': [3, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanAllTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 56, 56], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 56, 56], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -127,30 +140,34 @@ class TRTReduceMeanAllTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceMeanAllTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceMeanAllTest.DynamicShapeParam( - {'data': [1, 3, 56, 56]}, {'data': [3, 3, 56, 56]}, - {'data': [3, 3, 56, 56]}, False) + {'data': [1, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanTestStatic(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[3, 3, 56, 56], - dtype="float32") - reduce_mean = fluid.layers.reduce_mean(data, - dim=[2, -1], - keep_dim=True) + data = fluid.data( + name="data", shape=[3, 3, 56, 56], dtype="float32" + ) + reduce_mean = fluid.layers.reduce_mean( + data, dim=[2, -1], keep_dim=True + ) out = fluid.layers.batch_norm(reduce_mean, is_test=True) self.feeds = { @@ -158,7 +175,8 @@ class TRTReduceMeanTestStatic(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceMeanTestStatic.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -166,16 +184,16 @@ class TRTReduceMeanTestStatic(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanStaticAllTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[4, 3, 56, 56], - dtype="float32") + data = fluid.data( + name="data", shape=[4, 3, 56, 56], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -184,7 +202,8 @@ class TRTReduceMeanStaticAllTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceMeanStaticAllTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -192,16 +211,16 @@ class TRTReduceMeanStaticAllTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanStaticFP16(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[4, 3, 56, 56], - dtype="float32") + data = fluid.data( + name="data", shape=[4, 3, 56, 56], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -210,27 +229,27 @@ class TRTReduceMeanStaticFP16(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceMeanStaticFP16.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [out] def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - flatten=True, - atol=1e-3, - rtol=1e-3) + self.check_output_with_option( + use_gpu, flatten=True, atol=1e-3, rtol=1e-3 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanFP16Static(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[4, 3, 56, 56], - dtype="float32") + data = fluid.data( + name="data", shape=[4, 3, 56, 56], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -239,18 +258,19 @@ class TRTReduceMeanFP16Static(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceMeanFP16Static.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, True, False + ) self.fetch_list = [out] def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - flatten=True, - atol=1e-3, - rtol=1e-3) + self.check_output_with_option( + use_gpu, flatten=True, atol=1e-3, rtol=1e-3 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py index 13663676c6d..09e9a652ca8 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py @@ -22,15 +22,14 @@ from paddle.fluid.core import AnalysisConfig class TRTReduceSumTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 10, 192], - dtype="float32") - reduce_sum = fluid.layers.reduce_sum(data, - dim=[2, -1], - keep_dim=True) + data = fluid.data( + name="data", shape=[-1, 3, 10, 192], dtype="float32" + ) + reduce_sum = fluid.layers.reduce_sum( + data, dim=[2, -1], keep_dim=True + ) out = fluid.layers.batch_norm(reduce_sum, is_test=True) self.feeds = { @@ -38,27 +37,31 @@ class TRTReduceSumTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceSumTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceSumTest.DynamicShapeParam( - {'data': [1, 3, 8, 8]}, {'data': [3, 3, 10, 192]}, - {'data': [3, 3, 10, 192]}, False) + {'data': [1, 3, 8, 8]}, + {'data': [3, 3, 10, 192]}, + {'data': [3, 3, 10, 192]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceSumAllTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 10, 192], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 10, 192], dtype="float32" + ) reduce_sum = fluid.layers.reduce_sum(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_sum, is_test=True) @@ -67,18 +70,23 @@ class TRTReduceSumAllTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReduceSumAllTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceSumAllTest.DynamicShapeParam( - {'data': [1, 3, 8, 8]}, {'data': [3, 3, 10, 192]}, - {'data': [3, 3, 10, 192]}, False) + {'data': [1, 3, 8, 8]}, + {'data': [3, 3, 10, 192]}, + {'data': [3, 3, 10, 192]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py index ade320796eb..075919b7bf6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py @@ -22,19 +22,20 @@ from paddle.fluid.core import AnalysisConfig class TRTReshapeTest(InferencePassTest): - def setUp(self): self.bs = 1 self.input_shape = [16, 3, 8] self.reshape = [-1, 4, 4, 24] self.data_shape = [ - self.bs, self.input_shape[0], self.input_shape[1], - self.input_shape[2] + self.bs, + self.input_shape[0], + self.input_shape[1], + self.input_shape[2], ] with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=self.data_shape, - dtype='float32') + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32' + ) reshape_out = self.append_reshape(data, self.reshape) out = fluid.layers.batch_norm(reshape_out, is_test=True) self.feeds = { @@ -42,7 +43,8 @@ class TRTReshapeTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTReshapeTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_reshape(self, data, reshape): @@ -53,23 +55,25 @@ class TRTReshapeTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReshapeTest1(TRTReshapeTest): - def setUp(self): self.bs = 2 self.input_shape = [23, 13, 12] self.reshape = [2, 0, -1, 6] self.data_shape = [ - self.bs, self.input_shape[0], self.input_shape[1], - self.input_shape[2] + self.bs, + self.input_shape[0], + self.input_shape[1], + self.input_shape[2], ] with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=self.data_shape, - dtype='float32') + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32' + ) reshape_out = self.append_reshape(data, self.reshape) out = fluid.layers.batch_norm(reshape_out, is_test=True) self.feeds = { @@ -77,24 +81,26 @@ class TRTReshapeTest1(TRTReshapeTest): } self.enable_trt = True self.trt_parameters = TRTReshapeTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] class TRTReshapeTest2(TRTReshapeTest): - def setUp(self): self.bs = 2 self.input_shape = [23, 13, 12] self.reshape = [2, 0, -1, 6] self.data_shape = [ - self.bs, self.input_shape[0], self.input_shape[1], - self.input_shape[2] + self.bs, + self.input_shape[0], + self.input_shape[1], + self.input_shape[2], ] with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=self.data_shape, - dtype='float32') + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32' + ) reshape_out = fluid.layers.reshape(x=data, shape=self.reshape) out = fluid.layers.batch_norm(reshape_out, is_test=True) self.feeds = { @@ -102,24 +108,26 @@ class TRTReshapeTest2(TRTReshapeTest): } self.enable_trt = True self.trt_parameters = TRTReshapeTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] class TRTReshapeTest3(TRTReshapeTest): - def setUp(self): self.bs = 1 self.input_shape = [7, 16, 27] self.reshape = [1, 8, 14, 0] self.data_shape = [ - self.bs, self.input_shape[0], self.input_shape[1], - self.input_shape[2] + self.bs, + self.input_shape[0], + self.input_shape[1], + self.input_shape[2], ] with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=self.data_shape, - dtype='float32') + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32' + ) bn_out = fluid.layers.batch_norm(data, is_test=True) out = self.append_reshape(bn_out, self.reshape) self.feeds = { @@ -127,7 +135,8 @@ class TRTReshapeTest3(TRTReshapeTest): } self.enable_trt = True self.trt_parameters = TRTReshapeTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) ''' self.dynamic_shape_params = TRTReshapeTest.DynamicShapeParam({ 'data': [1, 3, 8, 8] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_roi_align_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_roi_align_op.py index a854e74110c..3b1d75d6954 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_roi_align_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_roi_align_op.py @@ -22,7 +22,6 @@ from paddle.fluid.core import AnalysisConfig class TRTRoiAlignTest(InferencePassTest): - def setUp(self): self.bs = 2 self.num_rois = 4 @@ -35,21 +34,27 @@ class TRTRoiAlignTest(InferencePassTest): def build(self): self.trt_parameters = TRTRoiAlignTest.TensorRTParam( - 1 << 30, self.bs * self.num_rois, 1, self.precision, self.serialize, - False) + 1 << 30, + self.bs * self.num_rois, + 1, + self.precision, + self.serialize, + False, + ) with fluid.program_guard(self.main_program, self.startup_program): data_shape = [-1, self.channel, self.height, self.width] data = fluid.data(name='data', shape=data_shape, dtype='float32') - rois = fluid.data(name='rois', - shape=[-1, 4], - dtype='float32', - lod_level=1) + rois = fluid.data( + name='rois', shape=[-1, 4], dtype='float32', lod_level=1 + ) roi_align_out = fluid.layers.roi_align(data, rois) out = fluid.layers.batch_norm(roi_align_out, is_test=True) rois_lod = fluid.create_lod_tensor( np.random.random([self.bs * self.num_rois, 4]).astype('float32'), - [[self.num_rois, self.num_rois]], fluid.CPUPlace()) + [[self.num_rois, self.num_rois]], + fluid.CPUPlace(), + ) data_shape[0] = self.bs self.feeds = { @@ -66,27 +71,38 @@ class TRTRoiAlignTest(InferencePassTest): atol = 1e-3 self.check_output_with_option(use_gpu, atol, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def set_dynamic(self): min_shape_spec = dict() max_shape_spec = dict() opt_shape_spec = dict() min_shape_spec['data'] = [ - self.bs, self.channel, self.height // 2, self.width // 2 + self.bs, + self.channel, + self.height // 2, + self.width // 2, ] min_shape_spec['rois'] = [1, 4] max_shape_spec['data'] = [ - self.bs, self.channel, self.height * 2, self.width * 2 + self.bs, + self.channel, + self.height * 2, + self.width * 2, ] max_shape_spec['rois'] = [self.bs * self.num_rois, 4] opt_shape_spec['data'] = [ - self.bs, self.channel, self.height, self.width + self.bs, + self.channel, + self.height, + self.width, ] opt_shape_spec['rois'] = [self.bs * self.num_rois, 4] self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( - min_shape_spec, max_shape_spec, opt_shape_spec, False) + min_shape_spec, max_shape_spec, opt_shape_spec, False + ) def run_test(self): self.build() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_scale_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_scale_op.py index 77f2170f3e2..966e1e77e12 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_scale_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_scale_op.py @@ -22,7 +22,6 @@ from paddle.fluid.core import AnalysisConfig class TRTScaleTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[-1, 512], dtype="float32") @@ -34,30 +33,30 @@ class TRTScaleTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTScaleTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_scale(self, data): - return fluid.layers.scale(x=data, - scale=2.0, - bias=-1.0, - bias_after_scale=False) + return fluid.layers.scale( + x=data, scale=2.0, bias=-1.0, bias_after_scale=False + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTScaleShape2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 512, 512], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 512, 512], dtype="float32" + ) scale_out = self.append_scale(data) out = fluid.layers.batch_norm(scale_out, is_test=True) @@ -66,21 +65,22 @@ class TRTScaleShape2Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTScaleShape2Test.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_scale(self, data): - return fluid.layers.scale(x=data, - scale=2.0, - bias=-1.0, - bias_after_scale=False) + return fluid.layers.scale( + x=data, scale=2.0, bias=-1.0, bias_after_scale=False + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_shuffle_channel_detect_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_shuffle_channel_detect_pass.py index 0752599e76b..d5bbbcde1e1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_shuffle_channel_detect_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_shuffle_channel_detect_pass.py @@ -21,12 +21,11 @@ from paddle.fluid.core import AnalysisConfig class ShuffleChannelFuseTRTPassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) reshape1 = fluid.layers.reshape(x=data, shape=[-1, 2, 3, 64, 64]) trans = fluid.layers.transpose(x=reshape1, perm=[0, 2, 1, 3, 4]) reshape2 = fluid.layers.reshape(x=trans, shape=[-1, 6, 64, 64]) @@ -37,7 +36,8 @@ class ShuffleChannelFuseTRTPassTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = ShuffleChannelFuseTRTPassTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -45,7 +45,8 @@ class ShuffleChannelFuseTRTPassTest(InferencePassTest): self.check_output() self.assertTrue( - PassVersionChecker.IsCompatible('shuffle_channel_detect_pass')) + PassVersionChecker.IsCompatible('shuffle_channel_detect_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_dynamic_plugin.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_dynamic_plugin.py index e4d3f5b418e..e9ea9948af7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_dynamic_plugin.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_dynamic_plugin.py @@ -20,9 +20,8 @@ import paddle.fluid.core as core from paddle.fluid.core import AnalysisConfig -#normal starts && ends +# normal starts && ends class SlicePluginTRTDynamicTest(InferencePassTest): - def setUpSliceParams(self): self.params_axes = [1, 3] self.params_starts = [0, 1] @@ -30,11 +29,15 @@ class SlicePluginTRTDynamicTest(InferencePassTest): def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTDynamicTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.enable_trt = True self.dynamic_shape_params = SlicePluginTRTDynamicTest.DynamicShapeParam( - {'data': [1, 1, 1, 1]}, {'data': [8, 8, 8, 8]}, - {'data': [8, 8, 8, 8]}, False) + {'data': [1, 1, 1, 1]}, + {'data': [8, 8, 8, 8]}, + {'data': [8, 8, 8, 8]}, + False, + ) def setUp(self): self.setUpSliceParams() @@ -44,10 +47,9 @@ class SlicePluginTRTDynamicTest(InferencePassTest): axes = self.params_axes starts = self.params_starts ends = self.params_ends - slice_out = fluid.layers.slice(data, - axes=axes, - starts=starts, - ends=ends) + slice_out = fluid.layers.slice( + data, axes=axes, starts=starts, ends=ends + ) self.feeds = { "data": np.random.random((3, 3, 3, 3)).astype("float32"), @@ -66,7 +68,6 @@ class SlicePluginTRTDynamicTest(InferencePassTest): class SlicePluginTRTDynamicBoundTest(SlicePluginTRTDynamicTest): - def setUpSliceParams(self): self.params_axes = [1, 3] self.params_starts = [0, 1] @@ -74,27 +75,40 @@ class SlicePluginTRTDynamicBoundTest(SlicePluginTRTDynamicTest): def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTDynamicBoundTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.enable_trt = True - self.dynamic_shape_params = SlicePluginTRTDynamicBoundTest.DynamicShapeParam( - {'data': [1, 1, 1, 1]}, {'data': [8, 8, 8, 8]}, - {'data': [8, 8, 8, 8]}, False) + self.dynamic_shape_params = ( + SlicePluginTRTDynamicBoundTest.DynamicShapeParam( + {'data': [1, 1, 1, 1]}, + {'data': [8, 8, 8, 8]}, + {'data': [8, 8, 8, 8]}, + False, + ) + ) class SlicePluginTRTDynamicNegativeBoundTest(SlicePluginTRTDynamicTest): - def setUpSliceParams(self): self.params_axes = [1, 3] self.params_starts = [-5, 1] self.params_ends = [2, 1000] def setUpTensorRTParams(self): - self.trt_parameters = SlicePluginTRTDynamicNegativeBoundTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + self.trt_parameters = ( + SlicePluginTRTDynamicNegativeBoundTest.TensorRTParam( + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) + ) self.enable_trt = True - self.dynamic_shape_params = SlicePluginTRTDynamicNegativeBoundTest.DynamicShapeParam( - {'data': [1, 1, 1, 1]}, {'data': [8, 8, 8, 8]}, - {'data': [8, 8, 8, 8]}, False) + self.dynamic_shape_params = ( + SlicePluginTRTDynamicNegativeBoundTest.DynamicShapeParam( + {'data': [1, 1, 1, 1]}, + {'data': [8, 8, 8, 8]}, + {'data': [8, 8, 8, 8]}, + False, + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_plugin.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_plugin.py index 70334d92cc4..ab93e3f6475 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_plugin.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_plugin.py @@ -20,9 +20,8 @@ import paddle.fluid.core as core from paddle.fluid.core import AnalysisConfig -#normal starts && ends +# normal starts && ends class SlicePluginTRTTest(InferencePassTest): - def setUpSliceParams(self): self.params_axes = [1, 3] self.params_starts = [0, 1] @@ -30,7 +29,8 @@ class SlicePluginTRTTest(InferencePassTest): def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.enable_trt = True def setUp(self): @@ -41,10 +41,9 @@ class SlicePluginTRTTest(InferencePassTest): axes = self.params_axes starts = self.params_starts ends = self.params_ends - slice_out = fluid.layers.slice(data, - axes=axes, - starts=starts, - ends=ends) + slice_out = fluid.layers.slice( + data, axes=axes, starts=starts, ends=ends + ) out = fluid.layers.batch_norm(slice_out, is_test=True) self.feeds = { @@ -63,51 +62,48 @@ class SlicePluginTRTTest(InferencePassTest): self.check_output_with_option(use_gpu[i], atol) -#negative starts && ends +# negative starts && ends class SlicePluginTRTTestNegativeStartsAndEnds(SlicePluginTRTTest): - def setUpSliceParams(self): self.params_axes = [2, 3] self.params_starts = [-3, -2] self.params_ends = [-1, 3] -#exceeded bound starts && ends +# exceeded bound starts && ends class SlicePluginTRTTestStartsAndEndsBoundCheck(SlicePluginTRTTest): - def setUpSliceParams(self): self.params_axes = [2, 3] self.params_starts = [-5, -2] self.params_ends = [-1, 8] -#fp16 +# fp16 class SlicePluginTRTTestFp16(SlicePluginTRTTest): - def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.enable_trt = True class StaticSlicePluginTRTTestFp16(SlicePluginTRTTest): - def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, True, False + ) self.enable_trt = True class StaticSlicePluginTRTTestFp32(SlicePluginTRTTest): - def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, True, False + ) self.enable_trt = True class SlicePluginTRTTestInt32(SlicePluginTRTTest): - def setUp(self): self.setUpSliceParams() self.setUpTensorRTParams() @@ -116,10 +112,9 @@ class SlicePluginTRTTestInt32(SlicePluginTRTTest): axes = self.params_axes starts = self.params_starts ends = self.params_ends - slice_out = fluid.layers.slice(data, - axes=axes, - starts=starts, - ends=ends) + slice_out = fluid.layers.slice( + data, axes=axes, starts=starts, ends=ends + ) cast_out = fluid.layers.cast(slice_out, 'float32') out = fluid.layers.batch_norm(cast_out, is_test=True) @@ -130,10 +125,10 @@ class SlicePluginTRTTestInt32(SlicePluginTRTTest): class StaticSlicePluginTRTTestInt32(SlicePluginTRTTest): - def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, True, False + ) self.enable_trt = True def setUp(self): @@ -144,10 +139,9 @@ class StaticSlicePluginTRTTestInt32(SlicePluginTRTTest): axes = self.params_axes starts = self.params_starts ends = self.params_ends - slice_out = fluid.layers.slice(data, - axes=axes, - starts=starts, - ends=ends) + slice_out = fluid.layers.slice( + data, axes=axes, starts=starts, ends=ends + ) cast_out = fluid.layers.cast(slice_out, 'float32') out = fluid.layers.batch_norm(cast_out, is_test=True) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_squeeze2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_squeeze2_matmul_fuse_pass.py index 54c1f722685..ce86d97afb0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_squeeze2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_squeeze2_matmul_fuse_pass.py @@ -42,7 +42,8 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['mul', 'elementwise_add'], (1e-4, 1e-1) def add_ignore_pass_case(self): @@ -65,9 +66,10 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape of input:X of squeeze2 x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) # axes of squeeze2 == [2, 3] x_shape += [1, 1] axes = [2, 3] @@ -79,9 +81,10 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -103,17 +106,11 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): "X": ["squeeze2_x"], }, axes=axes, - outputs={ - "Out": ["squeeze2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["squeeze2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["squeeze2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["squeeze2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -128,10 +125,7 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -152,9 +146,11 @@ class TestSqueeze2MatmulFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=25, - passes=["trt_squeeze2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=25, + passes=["trt_squeeze2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py index 2472ff027e3..7478f6a3734 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py @@ -24,12 +24,11 @@ from paddle.fluid.core import AnalysisConfig class TensorRTSubgraphPassFcTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) fc_out = fluid.layers.fc(input=[data], act=None, size=1000) reshape_out = fluid.layers.reshape(x=fc_out, shape=[1, 1000]) self.feeds = { @@ -37,7 +36,8 @@ class TensorRTSubgraphPassFcTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassFcTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [reshape_out] def test_check_output(self): @@ -46,19 +46,19 @@ class TensorRTSubgraphPassFcTest(InferencePassTest): # TRT output shape of fc is (1, 1000, 1, 1). To compare the output value only, flatten the results. self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassConcatTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[-1, 3, 64, 64], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 3, 64, 64], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 3, 64, 64], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 3, 64, 64], dtype="float32" + ) concat_out = fluid.layers.concat([data1, data2], axis=2) out = fluid.layers.batch_norm(concat_out, is_test=True) self.feeds = { @@ -67,7 +67,8 @@ class TensorRTSubgraphPassConcatTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassConcatTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -75,16 +76,16 @@ class TensorRTSubgraphPassConcatTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassSplitTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) split_out = fluid.layers.split(data, dim=-1, num_or_sections=2) out = fluid.layers.batch_norm(split_out[0], is_test=True) self.feeds = { @@ -92,7 +93,8 @@ class TensorRTSubgraphPassSplitTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassSplitTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -100,16 +102,16 @@ class TensorRTSubgraphPassSplitTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassSplitSerializeTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) split_out = fluid.layers.split(data, dim=-1, num_or_sections=2) out = fluid.layers.batch_norm(split_out[0], is_test=True) self.feeds = { @@ -117,7 +119,8 @@ class TensorRTSubgraphPassSplitSerializeTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassSplitTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) self.fetch_list = [out] def test_check_output(self): @@ -127,16 +130,16 @@ class TensorRTSubgraphPassSplitSerializeTest(InferencePassTest): shutil.rmtree(self.path + "_opt_cache") self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassDynamicSplitFp16SerializeTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) split_out = fluid.layers.split(data, dim=-1, num_or_sections=2) out = fluid.layers.batch_norm(split_out[0], is_test=True) self.feeds = { @@ -144,10 +147,16 @@ class TensorRTSubgraphPassDynamicSplitFp16SerializeTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassSplitTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) - self.dynamic_shape_params = TensorRTSubgraphPassDynamicSplitFp16SerializeTest.DynamicShapeParam( - {'data': [1, 3, 8, 64]}, {'data': [1, 3, 512, 64]}, - {'data': [1, 3, 256, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassDynamicSplitFp16SerializeTest.DynamicShapeParam( + {'data': [1, 3, 8, 64]}, + {'data': [1, 3, 512, 64]}, + {'data': [1, 3, 256, 64]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -157,31 +166,36 @@ class TensorRTSubgraphPassDynamicSplitFp16SerializeTest(InferencePassTest): shutil.rmtree(self.path + "_opt_cache") self.check_output_with_option(use_gpu, 1e-3) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassInstanceNormTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) param_attr = fluid.ParamAttr( name='instance_norm_w', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) bias_attr = fluid.ParamAttr( name='instance_norm_b', - initializer=fluid.initializer.Constant(value=0.0)) - out = fluid.layers.instance_norm(input=data, - param_attr=param_attr, - bias_attr=bias_attr) + initializer=fluid.initializer.Constant(value=0.0), + ) + out = fluid.layers.instance_norm( + input=data, param_attr=param_attr, bias_attr=bias_attr + ) self.feeds = { "data": np.random.random([1, 3, 64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassInstanceNormTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TensorRTSubgraphPassInstanceNormTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -189,16 +203,16 @@ class TensorRTSubgraphPassInstanceNormTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, atol=1e-4, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassTransposeTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) transpose_out = self.append_transpose(data) out = fluid.layers.batch_norm(transpose_out, is_test=True) self.feeds = { @@ -206,7 +220,8 @@ class TensorRTSubgraphPassTransposeTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassTransposeTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_transpose(self, data): @@ -217,25 +232,27 @@ class TensorRTSubgraphPassTransposeTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLayerNormTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") - out = fluid.layers.layer_norm(data, - begin_norm_axis=self.begin_norm_axis) + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) + out = fluid.layers.layer_norm( + data, begin_norm_axis=self.begin_norm_axis + ) self.feeds = { "data": np.random.random([1, 3, 64, 64]).astype("float32"), } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassLayerNormTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -246,19 +263,20 @@ class TensorRTSubgraphPassLayerNormTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLayerNormDynamicTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") - out = fluid.layers.layer_norm(data, - begin_norm_axis=self.begin_norm_axis) + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) + out = fluid.layers.layer_norm( + data, begin_norm_axis=self.begin_norm_axis + ) self.feeds = { "data": np.random.random([1, 3, 64, 64]).astype("float32"), } @@ -267,16 +285,25 @@ class TensorRTSubgraphPassLayerNormDynamicTest(InferencePassTest): def set_trt_params(self): self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassLayerNormDynamicTest.TensorRTParam( - 1 << 30, 32, 0, self.precision, self.serialize, False) - self.dynamic_shape_params = TensorRTSubgraphPassLayerNormDynamicTest.DynamicShapeParam( - { - 'data': [1, 3, 64, 64], - }, { - 'data': [8, 8, 64, 64], - }, { - 'data': [4, 4, 64, 64], - }, False) + self.trt_parameters = ( + TensorRTSubgraphPassLayerNormDynamicTest.TensorRTParam( + 1 << 30, 32, 0, self.precision, self.serialize, False + ) + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassLayerNormDynamicTest.DynamicShapeParam( + { + 'data': [1, 3, 64, 64], + }, + { + 'data': [8, 8, 64, 64], + }, + { + 'data': [4, 4, 64, 64], + }, + False, + ) + ) def set_params(self): self.begin_norm_axis = 2 @@ -290,12 +317,13 @@ class TensorRTSubgraphPassLayerNormDynamicTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLayerNormDynamicFP16Test( - TensorRTSubgraphPassLayerNormDynamicTest): - + TensorRTSubgraphPassLayerNormDynamicTest +): def set_params(self): self.begin_norm_axis = 2 self.precision = AnalysisConfig.Precision.Half @@ -308,33 +336,33 @@ class TensorRTSubgraphPassLayerNormDynamicFP16Test( use_gpu = True self.check_output_with_option(use_gpu, atol=0.01, rtol=0.01) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLayerNormBeginNormAxis2Test( - TensorRTSubgraphPassLayerNormTest): - + TensorRTSubgraphPassLayerNormTest +): def set_params(self): self.begin_norm_axis = 2 class TensorRTSubgraphPassLayerNormBeginNormAxis3Test( - TensorRTSubgraphPassLayerNormTest): - + TensorRTSubgraphPassLayerNormTest +): def set_params(self): self.begin_norm_axis = 3 class TensorRTSubgraphPassElementwiseTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[-1, 3, 64, 64], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 3, 64, 64], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 3, 64, 64], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 3, 64, 64], dtype="float32" + ) eltwise_out = self.append_eltwise(data1, data2) out = fluid.layers.batch_norm(eltwise_out, is_test=True) self.feeds = { @@ -343,7 +371,8 @@ class TensorRTSubgraphPassElementwiseTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassElementwiseTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_eltwise(self, data1, data2): @@ -354,38 +383,40 @@ class TensorRTSubgraphPassElementwiseTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) -class TensorRTSubgraphPassElementwiseMulTest(TensorRTSubgraphPassElementwiseTest - ): - +class TensorRTSubgraphPassElementwiseMulTest( + TensorRTSubgraphPassElementwiseTest +): def append_eltwise(self, data1, data2): return fluid.layers.elementwise_mul(x=data1, y=data2) class TensorRTSubgraphPassElementwiseSerializeTest( - TensorRTSubgraphPassElementwiseTest): - + TensorRTSubgraphPassElementwiseTest +): def setUp(self): super(TensorRTSubgraphPassElementwiseSerializeTest, self).setUp() self.trt_parameters = TensorRTSubgraphPassElementwiseTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) def test_check_output(self): if os.path.exists(self.path + "_opt_cache"): shutil.rmtree(self.path + "_opt_cache") - super(TensorRTSubgraphPassElementwiseSerializeTest, - self).test_check_output() + super( + TensorRTSubgraphPassElementwiseSerializeTest, self + ).test_check_output() class TensorRTSubgraphPassElementwiseBroadcastDynamicTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[-1, 3, 64, 64], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 3, 64, 64], dtype="float32" + ) data2 = fluid.data(name="data2", shape=[64, 64], dtype="float32") eltwise_out = self.append_eltwise(data1, data2) out = fluid.layers.batch_norm(eltwise_out, is_test=True) @@ -394,19 +425,17 @@ class TensorRTSubgraphPassElementwiseBroadcastDynamicTest(InferencePassTest): "data2": np.random.random([64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassElementwiseBroadcastDynamicTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + self.trt_parameters = ( + TensorRTSubgraphPassElementwiseBroadcastDynamicTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) + ) self.dynamic_shape_params = TensorRTSubgraphPassElementwiseBroadcastDynamicTest.DynamicShapeParam( - { - 'data1': [1, 3, 8, 64], - 'data2': [8, 64] - }, { - 'data1': [1, 3, 512, 64], - 'data2': [512, 64] - }, { - 'data1': [1, 3, 256, 64], - 'data2': [256, 64] - }, False) + {'data1': [1, 3, 8, 64], 'data2': [8, 64]}, + {'data1': [1, 3, 512, 64], 'data2': [512, 64]}, + {'data1': [1, 3, 256, 64], 'data2': [256, 64]}, + False, + ) self.fetch_list = [out] def append_eltwise(self, data1, data2): @@ -419,24 +448,27 @@ class TensorRTSubgraphPassElementwiseBroadcastDynamicTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassShuffleChannelTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) sc_out = fluid.layers.shuffle_channel(data, group=3) out = fluid.layers.batch_norm(sc_out, is_test=True) self.feeds = { "data": np.random.random([1, 6, 64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassShuffleChannelTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TensorRTSubgraphPassShuffleChannelTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -444,7 +476,8 @@ class TensorRTSubgraphPassShuffleChannelTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tile_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tile_op.py index bb3772ca5ae..52fa99fc81a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tile_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tile_op.py @@ -23,12 +23,11 @@ from paddle.fluid.core import AnalysisConfig class TRTTileTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[4, 3, 224, 256], - dtype="float32") + data = fluid.data( + name="data", shape=[4, 3, 224, 256], dtype="float32" + ) tile_out = paddle.tile(x=data, repeat_times=[1, 1, 1, 1]) out = fluid.layers.batch_norm(tile_out, is_test=True) @@ -37,7 +36,8 @@ class TRTTileTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTTileTest.TensorRTParam( - 1 << 30, 16, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 16, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -45,11 +45,11 @@ class TRTTileTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTTileExpandTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[1, 1, 1, 1], dtype="float32") @@ -61,7 +61,8 @@ class TRTTileExpandTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTTileExpandTest.TensorRTParam( - 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -69,11 +70,11 @@ class TRTTileExpandTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTTileExpandStaticTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[1, 1, 1, 1], dtype="float32") @@ -85,7 +86,8 @@ class TRTTileExpandStaticTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTTileExpandStaticTest.TensorRTParam( - 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, True, False + ) self.fetch_list = [out] def test_check_output(self): @@ -93,11 +95,11 @@ class TRTTileExpandStaticTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTTileExpandHalfTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[1, 1, 1, 1], dtype="float32") @@ -109,7 +111,8 @@ class TRTTileExpandHalfTest(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTTileExpandHalfTest.TensorRTParam( - 1 << 30, 1, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 1, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -117,7 +120,8 @@ class TRTTileExpandHalfTest(InferencePassTest): use_gpu = True self.check_output_with_option(use_gpu, 1e-4, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_transpose_flatten_concat_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_transpose_flatten_concat_fuse_pass.py index 3a15f0dcf34..409d36600d2 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_transpose_flatten_concat_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_transpose_flatten_concat_fuse_pass.py @@ -21,15 +21,14 @@ from paddle.fluid.core import AnalysisConfig class TransposeFlattenConcatFusePassTRTTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[8, 32, 128], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[8, 32, 128], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[8, 32, 128], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[8, 32, 128], dtype="float32" + ) trans1 = fluid.layers.transpose(data1, perm=[0, 2, 1]) trans2 = fluid.layers.transpose(data2, perm=[0, 2, 1]) flatt1 = fluid.layers.flatten(trans1) @@ -42,11 +41,14 @@ class TransposeFlattenConcatFusePassTRTTest(InferencePassTest): self.feeds = { "data1": np.random.random([8, 32, 128]).astype("float32"), - "data2": np.random.random([8, 32, 128]).astype("float32") + "data2": np.random.random([8, 32, 128]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TransposeFlattenConcatFusePassTRTTest.TensorRTParam( - 1 << 20, 8, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TransposeFlattenConcatFusePassTRTTest.TensorRTParam( + 1 << 20, 8, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [out] def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tuned_dynamic_shape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tuned_dynamic_shape.py index 3ef7ca22aca..44345b89c19 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tuned_dynamic_shape.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tuned_dynamic_shape.py @@ -22,7 +22,6 @@ from paddle.inference import Config, create_predictor class TRTTunedDynamicShapeTest(unittest.TestCase): - def get_model(self): place = fluid.CUDAPlace(0) exe = fluid.Executor(place) @@ -30,21 +29,25 @@ class TRTTunedDynamicShapeTest(unittest.TestCase): main_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - groups=1, - padding=0, - bias_attr=False, - act=None) + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + groups=1, + padding=0, + bias_attr=False, + act=None, + ) exe.run(startup_program) serialized_program = paddle.static.serialize_program( - data, conv_out, program=main_program) + data, conv_out, program=main_program + ) serialized_params = paddle.static.serialize_persistables( - data, conv_out, executor=exe, program=main_program) + data, conv_out, executor=exe, program=main_program + ) return serialized_program, serialized_params def get_config(self, model, params, tuned=False): @@ -61,9 +64,11 @@ class TRTTunedDynamicShapeTest(unittest.TestCase): min_subgraph_size=0, precision_mode=paddle.inference.PrecisionType.Float32, use_static=True, - use_calib_mode=False) - config.enable_tuned_tensorrt_dynamic_shape('shape_range.pbtxt', - True) + use_calib_mode=False, + ) + config.enable_tuned_tensorrt_dynamic_shape( + 'shape_range.pbtxt', True + ) return config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_yolo_box_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_yolo_box_op.py index 5eeafe0228a..d2ad64cf72e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_yolo_box_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_yolo_box_op.py @@ -22,26 +22,26 @@ from paddle.fluid.core import AnalysisConfig class TRTYoloBoxTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): image_shape = [self.bs, self.channel, self.height, self.width] image = fluid.data(name='image', shape=image_shape, dtype='float32') - image_size = fluid.data(name='image_size', - shape=[self.bs, 2], - dtype='int32') + image_size = fluid.data( + name='image_size', shape=[self.bs, 2], dtype='int32' + ) boxes, scores = self.append_yolobox(image, image_size) self.feeds = { - 'image': - np.random.random(image_shape).astype('float32'), - 'image_size': - np.random.randint(32, 64, size=(self.bs, 2)).astype('int32'), + 'image': np.random.random(image_shape).astype('float32'), + 'image_size': np.random.randint(32, 64, size=(self.bs, 2)).astype( + 'int32' + ), } self.enable_trt = True self.trt_parameters = TRTYoloBoxTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [scores, boxes] def set_params(self): @@ -51,35 +51,37 @@ class TRTYoloBoxTest(InferencePassTest): self.width = 64 self.class_num = 80 self.anchors = [10, 13, 16, 30, 33, 23] - self.conf_thresh = .1 + self.conf_thresh = 0.1 self.downsample_ratio = 32 def append_yolobox(self, image, image_size): - return fluid.layers.yolo_box(x=image, - img_size=image_size, - class_num=self.class_num, - anchors=self.anchors, - conf_thresh=self.conf_thresh, - downsample_ratio=self.downsample_ratio) + return fluid.layers.yolo_box( + x=image, + img_size=image_size, + class_num=self.class_num, + anchors=self.anchors, + conf_thresh=self.conf_thresh, + downsample_ratio=self.downsample_ratio, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTYoloBoxFP16Test(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): image_shape = [self.bs, self.channel, self.height, self.width] image = fluid.data(name='image', shape=image_shape, dtype='float32') - image_size = fluid.data(name='image_size', - shape=[self.bs, 2], - dtype='int32') + image_size = fluid.data( + name='image_size', shape=[self.bs, 2], dtype='int32' + ) boxes, scores = self.append_yolobox(image, image_size) self.feeds = { @@ -88,7 +90,8 @@ class TRTYoloBoxFP16Test(InferencePassTest): } self.enable_trt = True self.trt_parameters = TRTYoloBoxFP16Test.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [scores, boxes] def set_params(self): @@ -98,46 +101,49 @@ class TRTYoloBoxFP16Test(InferencePassTest): self.class_num = 1 self.anchors = [106, 148, 92, 300, 197, 334] self.channel = 18 - self.conf_thresh = .05 + self.conf_thresh = 0.05 self.downsample_ratio = 32 def append_yolobox(self, image, image_size): - return fluid.layers.yolo_box(x=image, - img_size=image_size, - class_num=self.class_num, - anchors=self.anchors, - conf_thresh=self.conf_thresh, - downsample_ratio=self.downsample_ratio) + return fluid.layers.yolo_box( + x=image, + img_size=image_size, + class_num=self.class_num, + anchors=self.anchors, + conf_thresh=self.conf_thresh, + downsample_ratio=self.downsample_ratio, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True, rtol=1e-1) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTYoloBoxIoUAwareTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): image_shape = [self.bs, self.channel, self.height, self.width] image = fluid.data(name='image', shape=image_shape, dtype='float32') - image_size = fluid.data(name='image_size', - shape=[self.bs, 2], - dtype='int32') + image_size = fluid.data( + name='image_size', shape=[self.bs, 2], dtype='int32' + ) boxes, scores = self.append_yolobox(image, image_size) self.feeds = { - 'image': - np.random.random(image_shape).astype('float32'), - 'image_size': - np.random.randint(32, 64, size=(self.bs, 2)).astype('int32'), + 'image': np.random.random(image_shape).astype('float32'), + 'image_size': np.random.randint(32, 64, size=(self.bs, 2)).astype( + 'int32' + ), } self.enable_trt = True self.trt_parameters = TRTYoloBoxTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [scores, boxes] def set_params(self): @@ -147,27 +153,30 @@ class TRTYoloBoxIoUAwareTest(InferencePassTest): self.width = 64 self.class_num = 80 self.anchors = [10, 13, 16, 30, 33, 23] - self.conf_thresh = .1 + self.conf_thresh = 0.1 self.downsample_ratio = 32 self.iou_aware = True self.iou_aware_factor = 0.5 def append_yolobox(self, image, image_size): - return fluid.layers.yolo_box(x=image, - img_size=image_size, - class_num=self.class_num, - anchors=self.anchors, - conf_thresh=self.conf_thresh, - downsample_ratio=self.downsample_ratio, - iou_aware=self.iou_aware, - iou_aware_factor=self.iou_aware_factor) + return fluid.layers.yolo_box( + x=image, + img_size=image_size, + class_num=self.class_num, + anchors=self.anchors, + conf_thresh=self.conf_thresh, + downsample_ratio=self.downsample_ratio, + iou_aware=self.iou_aware, + iou_aware_factor=self.iou_aware_factor, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py index 7ea2b494f3d..d9626ebd1de 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py @@ -40,7 +40,8 @@ class TestUnsqueezeEltwiseFusePass(PassAutoScanTest): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, [ 'elementwise_mul', ], (1e-5, 1e-5) @@ -48,9 +49,10 @@ class TestUnsqueezeEltwiseFusePass(PassAutoScanTest): def sample_program_config(self, draw): # 1. Generate shape and attr of mul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=10), min_size=4, max_size=4 + ) + ) axis = -1 # 2. Generate legal shape and attr of input:Y of unsqueeze2 @@ -62,20 +64,14 @@ class TestUnsqueezeEltwiseFusePass(PassAutoScanTest): inputs={ "X": ["unsqueeze2_x"], "AxesTensor": [], - "AxesTensorList": [] + "AxesTensorList": [], }, axes=unsqueeze2_axes, - outputs={ - "Out": ["unsqueeze2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["unsqueeze2_out"], "XShape": ["xshape"]}, ) mul_op = OpConfig( "elementwise_mul", - inputs={ - "Y": ["unsqueeze2_out"], - "X": ["mul_x"] - }, + inputs={"Y": ["unsqueeze2_out"], "X": ["mul_x"]}, axis=axis, outputs={"Out": ["mul_out"]}, ) @@ -97,9 +93,11 @@ class TestUnsqueezeEltwiseFusePass(PassAutoScanTest): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["unsqueeze2_eltwise_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["unsqueeze2_eltwise_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_yolo_box_post.py b/python/paddle/fluid/tests/unittests/ir/inference/test_yolo_box_post.py index 74f251ec62b..fd1d03a6a72 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_yolo_box_post.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_yolo_box_post.py @@ -20,22 +20,24 @@ from paddle.fluid.layer_helper import LayerHelper paddle.enable_static() -def yolo_box_post(box0, - box1, - box2, - im_shape, - im_scale, - anchors0=[116, 90, 156, 198, 373, 326], - anchors1=[30, 61, 62, 45, 59, 119], - anchors2=[10, 13, 16, 30, 33, 23], - class_num=80, - conf_thresh=0.005, - downsample_ratio0=32, - downsample_ratio1=16, - downsample_ratio2=8, - clip_bbox=True, - scale_x_y=1., - nms_threshold=0.45): +def yolo_box_post( + box0, + box1, + box2, + im_shape, + im_scale, + anchors0=[116, 90, 156, 198, 373, 326], + anchors1=[30, 61, 62, 45, 59, 119], + anchors2=[10, 13, 16, 30, 33, 23], + class_num=80, + conf_thresh=0.005, + downsample_ratio0=32, + downsample_ratio1=16, + downsample_ratio2=8, + clip_bbox=True, + scale_x_y=1.0, + nms_threshold=0.45, +): helper = LayerHelper('yolo_box_post', **locals()) output = helper.create_variable_for_type_inference(dtype=box0.dtype) nms_rois_num = helper.create_variable_for_type_inference(dtype='int32') @@ -44,35 +46,37 @@ def yolo_box_post(box0, 'Boxes1': box1, 'Boxes2': box2, "ImageShape": im_shape, - "ImageScale": im_scale + "ImageScale": im_scale, } outputs = {'Out': output, 'NmsRoisNum': nms_rois_num} - helper.append_op(type="yolo_box_post", - inputs=inputs, - attrs={ - 'anchors0': anchors0, - 'anchors1': anchors1, - 'anchors2': anchors2, - 'class_num': class_num, - 'conf_thresh': conf_thresh, - 'downsample_ratio0': downsample_ratio0, - 'downsample_ratio1': downsample_ratio1, - 'downsample_ratio2': downsample_ratio2, - 'clip_bbox': clip_bbox, - 'scale_x_y': scale_x_y, - 'nms_threshold': nms_threshold - }, - outputs=outputs) + helper.append_op( + type="yolo_box_post", + inputs=inputs, + attrs={ + 'anchors0': anchors0, + 'anchors1': anchors1, + 'anchors2': anchors2, + 'class_num': class_num, + 'conf_thresh': conf_thresh, + 'downsample_ratio0': downsample_ratio0, + 'downsample_ratio1': downsample_ratio1, + 'downsample_ratio2': downsample_ratio2, + 'clip_bbox': clip_bbox, + 'scale_x_y': scale_x_y, + 'nms_threshold': nms_threshold, + }, + outputs=outputs, + ) output.stop_gradient = True nms_rois_num.stop_gradient = True return output, nms_rois_num -@unittest.skipIf(not paddle.is_compiled_with_cuda(), - "only support cuda kernel.") +@unittest.skipIf( + not paddle.is_compiled_with_cuda(), "only support cuda kernel." +) class TestYoloBoxPost(unittest.TestCase): - def test_yolo_box_post(self): place = paddle.CUDAPlace(0) program = paddle.static.Program() @@ -90,8 +94,8 @@ class TestYoloBoxPost(unittest.TestCase): "box0": np.random.uniform(size=[1, 255, 19, 19]).astype("float32"), "box1": np.random.uniform(size=[1, 255, 38, 38]).astype("float32"), "box2": np.random.uniform(size=[1, 255, 76, 76]).astype("float32"), - "im_shape": np.array([[608., 608.]], "float32"), - "im_scale": np.array([[1., 1.]], "float32") + "im_shape": np.array([[608.0, 608.0]], "float32"), + "im_scale": np.array([[1.0, 1.0]], "float32"), } outs = exe.run(program, feed=feed, fetch_list=[out.name, rois_num.name]) diff --git a/python/paddle/fluid/tests/unittests/ir/pass_test.py b/python/paddle/fluid/tests/unittests/ir/pass_test.py index 0336dcb2da5..4cae2e7fec1 100644 --- a/python/paddle/fluid/tests/unittests/ir/pass_test.py +++ b/python/paddle/fluid/tests/unittests/ir/pass_test.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import Block class PassTest(unittest.TestCase): - @classmethod def setUpClass(self): self.main_program = fluid.Program() @@ -68,10 +67,12 @@ class PassTest(unittest.TestCase): self.check_output_with_place(place, startup_on_cpu, atol) def _run_program(self, executor, program): - outs = executor.run(program=program, - feed=self.feeds, - fetch_list=self.fetch_list, - return_numpy=False) + outs = executor.run( + program=program, + feed=self.feeds, + fetch_list=self.fetch_list, + return_numpy=False, + ) outs_np = [] outs_lod = [] for out in outs: @@ -128,8 +129,10 @@ class PassTest(unittest.TestCase): outs, lods = self._run_program(executor, self.main_program) self.assertTrue( len(self.fetch_list) == len(outs), - "Checking the number of fetchs failed. Expected: {}, Received: {}". - format(len(self.fetch_list), len(outs))) + "Checking the number of fetchs failed. Expected: {}, Received: {}".format( + len(self.fetch_list), len(outs) + ), + ) # Parameters may be changed in ir passes. opt_program = self._apply_ir_passes() @@ -138,13 +141,16 @@ class PassTest(unittest.TestCase): if startup_on_cpu and not isinstance(place, fluid.CPUPlace): warnings.warn( "Parameters are on CPU, and will be transferred to GPU " - "automatically by data transform.") + "automatically by data transform." + ) outs_opt, lods_opt = self._run_program(executor, opt_program) self.assertTrue( len(self.fetch_list) == len(outs_opt), - "Checking the number of fetchs failed. Expected: {}, Received: {}". - format(len(self.fetch_list), len(outs_opt))) + "Checking the number of fetchs failed. Expected: {}, Received: {}".format( + len(self.fetch_list), len(outs_opt) + ), + ) for i in range(len(self.fetch_list)): is_allclose = np.allclose(outs_opt[i], outs[i], atol=atol) if not is_allclose: @@ -156,9 +162,17 @@ class PassTest(unittest.TestCase): self.assertTrue( is_allclose, "Output (name: %s, shape: %s, dtype: %s) has diff at %s. The maximum diff is %e, first error element is %d, expected %e, but got %e" - % (self.fetch_list[i].name, str(self.fetch_list[i].shape), - self.fetch_list[i].dtype, str(place), max_diff, offset, - a.flatten()[offset], b.flatten()[offset])) + % ( + self.fetch_list[i].name, + str(self.fetch_list[i].shape), + self.fetch_list[i].dtype, + str(place), + max_diff, + offset, + a.flatten()[offset], + b.flatten()[offset], + ), + ) def _check_fused_ops(self, program): ''' @@ -179,9 +193,10 @@ class PassTest(unittest.TestCase): self.assertTrue( self.num_fused_ops == acctual_num_fused_ops, "Checking of the number of fused operator < {} > failed. " - "Expected: {}, Received: {}".format(self.fused_op_type, - self.num_fused_ops, - acctual_num_fused_ops)) + "Expected: {}, Received: {}".format( + self.fused_op_type, self.num_fused_ops, acctual_num_fused_ops + ), + ) def check_program(self, program=None): ''' @@ -196,13 +211,16 @@ class PassTest(unittest.TestCase): self.assertTrue( self.main_program.desc != program.desc, "The optimized program and the origin main_program hold the same " - "desc.") + "desc.", + ) self.assertTrue( self.main_program.num_blocks == program.num_blocks, "The number of blocks of the origin program and the optimized " "program are different ({} vs {}).".format( - self.main_program.num_blocks, program.num_blocks)) + self.main_program.num_blocks, program.num_blocks + ), + ) is_different = False for i in range(program.num_blocks): @@ -219,7 +237,8 @@ class PassTest(unittest.TestCase): break if len(self.main_program.block(i).vars) != len( - program.block(i).vars): + program.block(i).vars + ): # The number of vars in the block i of the origin program and # the optimized program is different. is_different = True @@ -235,7 +254,8 @@ class PassTest(unittest.TestCase): self.assertTrue( is_different, "The optimized program is logically the same with the origin " - "program.") + "program.", + ) def _find_op(self, specified_op, program, block_id): is_find = False diff --git a/python/paddle/fluid/tests/unittests/ir/test_convert_to_mixed_precision.py b/python/paddle/fluid/tests/unittests/ir/test_convert_to_mixed_precision.py index 20891b22fde..6169cb98b6b 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_convert_to_mixed_precision.py +++ b/python/paddle/fluid/tests/unittests/ir/test_convert_to_mixed_precision.py @@ -23,55 +23,75 @@ from paddle.inference import PrecisionType, PlaceType from paddle.inference import convert_to_mixed_precision -@unittest.skipIf(not paddle.is_compiled_with_cuda() - or paddle.get_cudnn_version() < 8000, - 'should compile with cuda.') +@unittest.skipIf( + not paddle.is_compiled_with_cuda() or paddle.get_cudnn_version() < 8000, + 'should compile with cuda.', +) class TestConvertToMixedPrecision(unittest.TestCase): - def test_convert_to_fp16(self): model = resnet50(True) net = to_static( - model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')]) + model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')] + ) paddle.jit.save(net, 'resnet50/inference') - convert_to_mixed_precision('resnet50/inference.pdmodel', - 'resnet50/inference.pdiparams', - 'mixed/inference.pdmodel', - 'mixed/inference.pdiparams', - PrecisionType.Half, PlaceType.GPU, True) + convert_to_mixed_precision( + 'resnet50/inference.pdmodel', + 'resnet50/inference.pdiparams', + 'mixed/inference.pdmodel', + 'mixed/inference.pdiparams', + PrecisionType.Half, + PlaceType.GPU, + True, + ) def test_convert_to_fp16_with_fp16_input(self): model = resnet50(True) net = to_static( - model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')]) + model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')] + ) paddle.jit.save(net, 'resnet50/inference') - convert_to_mixed_precision('resnet50/inference.pdmodel', - 'resnet50/inference.pdiparams', - 'mixed1/inference.pdmodel', - 'mixed1/inference.pdiparams', - PrecisionType.Half, PlaceType.GPU, False) + convert_to_mixed_precision( + 'resnet50/inference.pdmodel', + 'resnet50/inference.pdiparams', + 'mixed1/inference.pdmodel', + 'mixed1/inference.pdiparams', + PrecisionType.Half, + PlaceType.GPU, + False, + ) def test_convert_to_fp16_with_blacklist(self): model = resnet50(True) net = to_static( - model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')]) + model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')] + ) paddle.jit.save(net, 'resnet50/inference') - convert_to_mixed_precision('resnet50/inference.pdmodel', - 'resnet50/inference.pdiparams', - 'mixed2/inference.pdmodel', - 'mixed2/inference.pdiparams', - PrecisionType.Half, PlaceType.GPU, False, - set('conv2d')) + convert_to_mixed_precision( + 'resnet50/inference.pdmodel', + 'resnet50/inference.pdiparams', + 'mixed2/inference.pdmodel', + 'mixed2/inference.pdiparams', + PrecisionType.Half, + PlaceType.GPU, + False, + set('conv2d'), + ) def test_convert_to_bf16(self): model = resnet50(True) net = to_static( - model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')]) + model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')] + ) paddle.jit.save(net, 'resnet50/inference') - convert_to_mixed_precision('resnet50/inference.pdmodel', - 'resnet50/inference.pdiparams', - 'mixed3/inference.pdmodel', - 'mixed3/inference.pdiparams', - PrecisionType.Bfloat16, PlaceType.GPU, True) + convert_to_mixed_precision( + 'resnet50/inference.pdmodel', + 'resnet50/inference.pdiparams', + 'mixed3/inference.pdmodel', + 'mixed3/inference.pdiparams', + PrecisionType.Bfloat16, + PlaceType.GPU, + True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/test_fuse_resnet_unit.py b/python/paddle/fluid/tests/unittests/ir/test_fuse_resnet_unit.py index 58b227f4cb5..4dabcdbcf35 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_fuse_resnet_unit.py +++ b/python/paddle/fluid/tests/unittests/ir/test_fuse_resnet_unit.py @@ -22,13 +22,14 @@ paddle.enable_static() np.random.seed(0) -@unittest.skipIf(not paddle.is_compiled_with_cuda() - or paddle.get_cudnn_version() < 8000 - or paddle.device.cuda.get_device_capability()[0] < 7, - "only support with cuda and cudnn version is at least 8.0 " - "and device's compute capability is at least 7.0") +@unittest.skipIf( + not paddle.is_compiled_with_cuda() + or paddle.get_cudnn_version() < 8000 + or paddle.device.cuda.get_device_capability()[0] < 7, + "only support with cuda and cudnn version is at least 8.0 " + "and device's compute capability is at least 7.0", +) class TestFuseResNetUnit(unittest.TestCase): - def test_fuse_resenet_unit(self): place = paddle.CUDAPlace(0) program = paddle.static.Program() @@ -36,14 +37,12 @@ class TestFuseResNetUnit(unittest.TestCase): with paddle.static.amp.fp16_guard(): with paddle.static.program_guard(program, startup_program): x = paddle.static.data("x", [1, 64, 64, 8]) - conv2d = paddle.nn.Conv2D(8, - 32, - 1, - bias_attr=False, - data_format='NHWC') - batch_norm = paddle.nn.BatchNorm(32, - act='relu', - data_layout='NHWC') + conv2d = paddle.nn.Conv2D( + 8, 32, 1, bias_attr=False, data_format='NHWC' + ) + batch_norm = paddle.nn.BatchNorm( + 32, act='relu', data_layout='NHWC' + ) out = batch_norm(conv2d(x)) graph = core.Graph(program.desc) core.get_pass("fuse_resnet_unit").apply(graph) @@ -52,15 +51,15 @@ class TestFuseResNetUnit(unittest.TestCase): after_params = paddle.static.amp.cast_model_to_fp16(after_program) exe = paddle.static.Executor(place) exe.run(startup_program) - paddle.static.amp.cast_parameters_to_fp16(place, - program, - to_fp16_var_names=params) paddle.static.amp.cast_parameters_to_fp16( - place, after_program, to_fp16_var_names=after_params) + place, program, to_fp16_var_names=params + ) + paddle.static.amp.cast_parameters_to_fp16( + place, after_program, to_fp16_var_names=after_params + ) feed = {"x": np.random.randn(1, 64, 64, 8).astype("float16")} before_out = exe.run(program, feed=feed, fetch_list=[out.name]) after_out = exe.run(after_program, feed=feed, fetch_list=[out.name]) - np.testing.assert_allclose(before_out[0], - after_out[0], - rtol=1e-05, - atol=0.005) + np.testing.assert_allclose( + before_out[0], after_out[0], rtol=1e-05, atol=0.005 + ) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_embedding_eltwise_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_embedding_eltwise_layernorm_fuse_pass.py index 6a573e7beac..6b0a69568a3 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_embedding_eltwise_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_embedding_eltwise_layernorm_fuse_pass.py @@ -21,89 +21,102 @@ import paddle.fluid.core as core class EmbEltwiseLayerNormFusePassTest(PassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - word_id = fluid.layers.data(name="word_id", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - pos_id = fluid.layers.data(name="pos_id", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - sent_id = fluid.layers.data(name="sent_id", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - word_emb = fluid.layers.embedding(input=word_id, - size=(128, 768), - dtype='float32') - pos_emb = fluid.layers.embedding(input=pos_id, - size=(128, 768), - dtype='float32') - sent_emb = fluid.layers.embedding(input=sent_id, - size=(128, 768), - dtype='float32') + word_id = fluid.layers.data( + name="word_id", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + pos_id = fluid.layers.data( + name="pos_id", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + sent_id = fluid.layers.data( + name="sent_id", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + word_emb = fluid.layers.embedding( + input=word_id, size=(128, 768), dtype='float32' + ) + pos_emb = fluid.layers.embedding( + input=pos_id, size=(128, 768), dtype='float32' + ) + sent_emb = fluid.layers.embedding( + input=sent_id, size=(128, 768), dtype='float32' + ) add1 = fluid.layers.elementwise_add(word_emb, pos_emb) add2 = fluid.layers.elementwise_add(add1, sent_emb) hidden1 = fluid.layers.layer_norm(input=add2, begin_norm_axis=2) - id1 = fluid.layers.data(name="id1", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - id2 = fluid.layers.data(name="id2", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - id3 = fluid.layers.data(name="id3", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - id4 = fluid.layers.data(name="id4", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - emb1 = fluid.layers.embedding(input=id1, - size=(128, 768), - dtype='float32') - emb2 = fluid.layers.embedding(input=id2, - size=(128, 768), - dtype='float32') - emb3 = fluid.layers.embedding(input=id3, - size=(128, 768), - dtype='float32') - emb4 = fluid.layers.embedding(input=id4, - size=(128, 768), - dtype='float32') + id1 = fluid.layers.data( + name="id1", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + id2 = fluid.layers.data( + name="id2", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + id3 = fluid.layers.data( + name="id3", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + id4 = fluid.layers.data( + name="id4", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + emb1 = fluid.layers.embedding( + input=id1, size=(128, 768), dtype='float32' + ) + emb2 = fluid.layers.embedding( + input=id2, size=(128, 768), dtype='float32' + ) + emb3 = fluid.layers.embedding( + input=id3, size=(128, 768), dtype='float32' + ) + emb4 = fluid.layers.embedding( + input=id4, size=(128, 768), dtype='float32' + ) add_1 = fluid.layers.elementwise_add(emb1, emb2) add_2 = fluid.layers.elementwise_add(add_1, emb3) add_3 = fluid.layers.elementwise_add(add_2, emb4) hidden_1 = fluid.layers.layer_norm(input=add_3, begin_norm_axis=2) self.feeds = { - "word_id": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "pos_id": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "sent_id": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "id1": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "id2": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "id3": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "id4": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), + "word_id": np.random.randint( + low=0, high=128, size=(1, 128, 1) + ).astype("int64"), + "pos_id": np.random.randint( + low=0, high=128, size=(1, 128, 1) + ).astype("int64"), + "sent_id": np.random.randint( + low=0, high=128, size=(1, 128, 1) + ).astype("int64"), + "id1": np.random.randint(low=0, high=128, size=(1, 128, 1)).astype( + "int64" + ), + "id2": np.random.randint(low=0, high=128, size=(1, 128, 1)).astype( + "int64" + ), + "id3": np.random.randint(low=0, high=128, size=(1, 128, 1)).astype( + "int64" + ), + "id4": np.random.randint(low=0, high=128, size=(1, 128, 1)).astype( + "int64" + ), } self.fetch_list = [hidden1, hidden_1] self.pass_names = "embedding_eltwise_layernorm_fuse_pass" @@ -115,9 +128,7 @@ class EmbEltwiseLayerNormFusePassTest(PassTest): if not core.is_compiled_with_cuda(): return self.pass_attrs = { - "embedding_eltwise_layernorm_fuse_pass": { - "use_gpu": True - } + "embedding_eltwise_layernorm_fuse_pass": {"use_gpu": True} } place = fluid.CUDAPlace(0) self.check_output_with_place(place, startup_on_cpu=True) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_fc_fuse_pass.py index 060d63cc332..6022f98fa39 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_fc_fuse_pass.py @@ -21,17 +21,14 @@ import paddle.fluid.core as core class FCFusePassTest(PassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128], - dtype="float32", - lod_level=0) - tmp_0 = fluid.layers.fc(input=data, - size=128, - num_flatten_dims=1, - act="relu") + data = fluid.data( + name="data", shape=[32, 128], dtype="float32", lod_level=0 + ) + tmp_0 = fluid.layers.fc( + input=data, size=128, num_flatten_dims=1, act="relu" + ) tmp_1 = fluid.layers.fc(input=tmp_0, size=32, num_flatten_dims=1) tmp_2 = fluid.layers.softmax(input=tmp_1) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_fusion_group_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_fusion_group_pass.py index 10b861fad54..3775a4d08d1 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_fusion_group_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_fusion_group_pass.py @@ -22,12 +22,12 @@ import paddle.fluid.core as core class FusionGroupPassTest(PassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2) self.feed_vars.append( - fluid.data(name="data2", shape=[128, 128], dtype=dtype)) + fluid.data(name="data2", shape=[128, 128], dtype=dtype) + ) # subgraph with only 1 op node tmp_0 = self.feed_vars[0] * self.feed_vars[1] @@ -78,7 +78,6 @@ class FusionGroupPassTest(PassTest): class FusionGroupPassComplicatedTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 64], dtype, 5) @@ -87,7 +86,8 @@ class FusionGroupPassComplicatedTest(FusionGroupPassTest): tmp_0 = one * self.feed_vars[0] # subgraph with 9 op nodes tmp_1 = tmp_0 * layers.sigmoid(self.feed_vars[1]) + layers.sigmoid( - self.feed_vars[2]) * layers.tanh(self.feed_vars[3]) + self.feed_vars[2] + ) * layers.tanh(self.feed_vars[3]) tmp_2 = layers.tanh(tmp_1) + layers.sigmoid(self.feed_vars[4]) self.append_gradients(tmp_2) @@ -97,12 +97,12 @@ class FusionGroupPassComplicatedTest(FusionGroupPassTest): class FusionGroupPassInplaceTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3) self.feed_vars.append( - fluid.data(name="data3", shape=[128, 32], dtype=dtype)) + fluid.data(name="data3", shape=[128, 32], dtype=dtype) + ) # subgraph with 3 op node tmp_0 = self.feed_vars[0] - self.feed_vars[1] @@ -115,7 +115,6 @@ class FusionGroupPassInplaceTest(FusionGroupPassTest): class FusionGroupPassTestFP64(FusionGroupPassTest): - def setUp(self): self.build_program("float64") self.feeds = self._feed_random_data(self.feed_vars) @@ -124,12 +123,12 @@ class FusionGroupPassTestFP64(FusionGroupPassTest): class FusionGroupPassTestCastAndFP16(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2) self.feed_vars.append( - fluid.data(name="data2", shape=[128, 128], dtype=dtype)) + fluid.data(name="data2", shape=[128, 128], dtype=dtype) + ) # subgraph with 2 op nodes tmp_0 = self.feed_vars[0] * self.feed_vars[1] @@ -152,16 +151,17 @@ class FusionGroupPassTestCastAndFP16(FusionGroupPassTest): class FusionGroupPassSumTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3) self.feed_vars.append( - fluid.data(name="data3", shape=[128, 128], dtype=dtype)) + fluid.data(name="data3", shape=[128, 128], dtype=dtype) + ) # subgraph with 2 op nodes tmp_0 = layers.sum( - [self.feed_vars[0], self.feed_vars[1], self.feed_vars[2]]) + [self.feed_vars[0], self.feed_vars[1], self.feed_vars[2]] + ) tmp_1 = layers.sqrt(tmp_0) tmp_2 = layers.mul(tmp_0, self.feed_vars[3]) # subgraph with 2 op nodes @@ -174,7 +174,6 @@ class FusionGroupPassSumTest(FusionGroupPassTest): class FusionGroupPassCastTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([2, 2], dtype, 2) @@ -196,17 +195,15 @@ class FusionGroupPassCastTest(FusionGroupPassTest): class FusionGroupPassFillConstantTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([2, 2], dtype, 2) tmp_0 = layers.elementwise_add(self.feed_vars[0], self.feed_vars[1]) tmp_1 = layers.fill_constant(shape=[2, 2], dtype=dtype, value=2.0) - tmp_2 = layers.scale(tmp_1, - scale=3.0, - bias=1.0, - bias_after_scale=True) + tmp_2 = layers.scale( + tmp_1, scale=3.0, bias=1.0, bias_after_scale=True + ) tmp_3 = layers.elementwise_mul(tmp_2, tmp_0) self.append_gradients(tmp_3) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_generate_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_generate_pass.py index e4d8f1b32a7..76b58d3ce90 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_generate_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_generate_pass.py @@ -23,9 +23,7 @@ import numpy as np # 1: relu(X=ewadd(X=mul(X=x, Y=w), Y=b)) => fc(Input=x, W=w, Bias=b) @ir.RegisterPass def generate_fc_fuse(): - def create_pass_pair(with_relu): - def pattern(x, w, b): mul = ir.PassDesc.OP.mul(X=x, Y=w) ewadd = ir.PassDesc.OP.elementwise_add(X=mul, Y=b) @@ -36,8 +34,9 @@ def generate_fc_fuse(): def replace(x, w, b): fc = ir.PassDesc.OP.fc(Input=x, W=w, Bias=b) - fc.Attr("in_num_col_dims").MappedPattern(op="mul", - name="x_num_col_dims") + fc.Attr("in_num_col_dims").MappedPattern( + op="mul", name="x_num_col_dims" + ) if with_relu: fc.SetAttr("activation_type", "relu") return fc @@ -57,7 +56,6 @@ def multi_add_to_sum_v1(): @ir.RegisterPass def multi_add_to_sum_v2(): - def pattern(x, y, z): ewadd1 = ir.PassDesc.OP.elementwise_add(X=x, Y=y) ewadd2 = ir.PassDesc.OP.elementwise_add(X=ewadd1, Y=z) @@ -75,13 +73,14 @@ def multi_add_to_sum_v3(): # mul(x, y1), mul(x, y2) => slice(mul(x, concat(y1, y2))) -@ir.RegisterPass(input_specs={ - 'x': InputSpec([16, 32]), - 'y1': InputSpec([32, 12]), - 'y2': InputSpec([32, 48]) -}) +@ir.RegisterPass( + input_specs={ + 'x': InputSpec([16, 32]), + 'y1': InputSpec([32, 12]), + 'y2': InputSpec([32, 48]), + } +) def generate_combine_mul_v1(): - def pattern(x, y1, y2): mul1 = paddle.matmul(x, y1) mul2 = paddle.matmul(x, y2) @@ -99,7 +98,6 @@ def generate_combine_mul_v1(): @ir.RegisterPass def generate_combine_mul_v2(): - def pattern(x, y1, y2): mul1 = ir.PassDesc.OP.matmul_v2(X=x, Y=y1) mul2 = ir.PassDesc.OP.matmul_v2(X=x, Y=y2) @@ -118,7 +116,6 @@ def generate_combine_mul_v2(): # reshape(reshape(x)) => x @ir.RegisterPass(input_specs={'x': InputSpec([10, 16, 16])}) def generate_simplify_inference_v1(): - def pattern(x): transpose = paddle.transpose(x, [0, 2, 1]) return paddle.transpose(transpose, [0, 2, 1]) @@ -128,7 +125,6 @@ def generate_simplify_inference_v1(): @ir.RegisterPass def generate_simplify_inference_v2(): - def pattern(x): op1 = ir.PassDesc.OP.transpose2 op2 = ir.PassDesc.OP.transpose2 @@ -140,7 +136,6 @@ def generate_simplify_inference_v2(): @ir.RegisterPass def generate_layer_norm_fuse_pass(): - def pattern(x, gamma, beta): gamma.Attr("shape").Size().EQ(1) gamma.Attr("shape")[0].EQ(x.Attr("shape")[-1]) @@ -175,7 +170,6 @@ def generate_layer_norm_fuse_pass(): @ir.RegisterPass def unimplemented_operand_exception(): - def pattern(x, y): return ir.PassDesc.OP.elementwise_add(X=x, Y=y) @@ -189,7 +183,6 @@ def unimplemented_operand_exception(): @ir.RegisterPass def unimplemented_operation_exception(): - def pattern(x, y): return ir.PassDesc.OP.elementwise_add(X=x, Y=y) @@ -208,7 +201,6 @@ def get_multi_pass_desc_from_str(s): class TestGeneratePass(unittest.TestCase): - def convert_ops_to_op_dicts(self, ops): op_dicts = dict() for op in ops: @@ -237,13 +229,13 @@ class TestGeneratePass(unittest.TestCase): core.get_pass("unimplemented_operation_exception").apply(graph) def test_generate_fc_fuse(self): - def _check_fc_fuse_pass(pass_desc, with_relu): pattern_op_dicts = self.convert_ops_to_op_dicts(pass_desc.pattern) replace_op_dicts = self.convert_ops_to_op_dicts(pass_desc.replace) self.assertEqual(len(pattern_op_dicts.get("mul", [])), 1) - self.assertEqual(len(pattern_op_dicts.get("elementwise_add", [])), - 1) + self.assertEqual( + len(pattern_op_dicts.get("elementwise_add", [])), 1 + ) if with_relu: self.assertEqual(len(pattern_op_dicts.get("relu", [])), 1) pattern_op_num = 3 # relu, ewadd, mul @@ -285,12 +277,12 @@ class TestGeneratePass(unittest.TestCase): feed = { "x": np.random.random([10, 10, 10]).astype("float32"), "y": np.random.random([10, 10, 10]).astype("float32"), - "z": np.random.random([10, 10, 10]).astype("float32") + "z": np.random.random([10, 10, 10]).astype("float32"), } before_out = executor.run(program, feed=feed, fetch_list=[out.name]) - after_out = executor.run(after_program, - feed=feed, - fetch_list=[out.name]) + after_out = executor.run( + after_program, feed=feed, fetch_list=[out.name] + ) np.testing.assert_allclose(before_out, after_out, rtol=1e-05) def test_multi_add_to_sum(self): @@ -320,13 +312,14 @@ class TestGeneratePass(unittest.TestCase): feed = { "x": np.random.random([16, 32]).astype("float32"), "y": np.random.random([32, 12]).astype("float32"), - "z": np.random.random([32, 48]).astype("float32") + "z": np.random.random([32, 48]).astype("float32"), } before_out1, before_out2 = executor.run( - program, feed=feed, fetch_list=[out1.name, out2.name]) - after_out1, after_out2 = executor.run(after_program, - feed=feed, - fetch_list=[out1.name, out2.name]) + program, feed=feed, fetch_list=[out1.name, out2.name] + ) + after_out1, after_out2 = executor.run( + after_program, feed=feed, fetch_list=[out1.name, out2.name] + ) np.testing.assert_allclose(before_out1, after_out1, rtol=1e-05) np.testing.assert_allclose(before_out2, after_out2, rtol=1e-05) @@ -366,9 +359,9 @@ class TestGeneratePass(unittest.TestCase): executor.run(startup_program) feed = {"x": np.random.random([10, 16, 16]).astype("float32")} before_out = executor.run(program, feed=feed, fetch_list=[out.name]) - after_out = executor.run(after_program, - feed=feed, - fetch_list=[out.name]) + after_out = executor.run( + after_program, feed=feed, fetch_list=[out.name] + ) np.testing.assert_allclose(before_out, after_out, rtol=1e-05) def test_generate_simplify_inference(self): @@ -381,12 +374,12 @@ class TestGeneratePass(unittest.TestCase): startup_program = paddle.static.Program() with paddle.static.program_guard(program, startup_program): x = paddle.static.data("x", [3, 64, 120], "float32") - gamma = paddle.static.create_parameter(shape=[120], - dtype="float32", - is_bias=True) - beta = paddle.static.create_parameter(shape=[120], - dtype="float32", - is_bias=True) + gamma = paddle.static.create_parameter( + shape=[120], dtype="float32", is_bias=True + ) + beta = paddle.static.create_parameter( + shape=[120], dtype="float32", is_bias=True + ) x_sub_mean = x - paddle.mean(x, axis=-1, keepdim=True) std_dev = paddle.mean(x_sub_mean.pow(2), axis=-1, keepdim=True) @@ -402,7 +395,7 @@ class TestGeneratePass(unittest.TestCase): executor.run(startup_program) feed = {"x": np.random.random([3, 64, 120]).astype("float32")} before_out = executor.run(program, feed=feed, fetch_list=[out.name]) - after_out = executor.run(after_program, - feed=feed, - fetch_list=[out.name]) + after_out = executor.run( + after_program, feed=feed, fetch_list=[out.name] + ) np.testing.assert_allclose(before_out, after_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_graph_to_program_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_graph_to_program_pass.py index c4f9147e33a..1b68c64805c 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_graph_to_program_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_graph_to_program_pass.py @@ -31,7 +31,6 @@ def IRGraph_to_program(ir_graph): class GraphToProgramPassTest(unittest.TestCase): - def check_vars_equal(self, o_block, c_block): o_params = sorted(o_block.all_parameters(), key=lambda p: p.name) c_params = sorted(c_block.all_parameters(), key=lambda p: p.name) @@ -69,12 +68,12 @@ class GraphToProgramPassTest(unittest.TestCase): o_attr = o_attrs[attr_idx] c_attr = c_attrs[attr_idx] self.assertEqual(o_attr, c_attr) - self.assertEqual(o_op.desc.attr_type(o_attr), - c_op.desc.attr_type(c_attr)) + self.assertEqual( + o_op.desc.attr_type(o_attr), c_op.desc.attr_type(c_attr) + ) class SingleGraphToProgramPass(GraphToProgramPassTest): - def setUp(self): self.origin_program = self.build_program() ir_graph = program_to_IRGraph(self.origin_program) @@ -91,10 +90,12 @@ class SingleGraphToProgramPass(GraphToProgramPassTest): return program def test_check_parameter(self): - origin_parameter = sorted(self.origin_program.all_parameters(), - key=lambda p: p.name) - converted_parameter = sorted(self.converted_program.all_parameters(), - key=lambda p: p.name) + origin_parameter = sorted( + self.origin_program.all_parameters(), key=lambda p: p.name + ) + converted_parameter = sorted( + self.converted_program.all_parameters(), key=lambda p: p.name + ) self.assertEqual(len(origin_parameter), len(converted_parameter)) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_preln_residual_bias_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_preln_residual_bias_fuse_pass.py index a8d2cdc01c4..2f367b83ae6 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_preln_residual_bias_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_preln_residual_bias_fuse_pass.py @@ -19,20 +19,18 @@ import paddle class PrelnResidualBiasFusePassTest(PassTest): - def setUp(self): paddle.enable_static() - with paddle.static.program_guard(self.main_program, - self.startup_program): - x = paddle.static.data(name="x", - shape=[128, 768], - dtype="float32", - lod_level=0) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + x = paddle.static.data( + name="x", shape=[128, 768], dtype="float32", lod_level=0 + ) bias = paddle.static.create_parameter(shape=[768], dtype='float32') - y = paddle.static.data(name="y", - shape=[128, 768], - dtype="float32", - lod_level=0) + y = paddle.static.data( + name="y", shape=[128, 768], dtype="float32", lod_level=0 + ) x = x + bias elementwise_out = x + y out = paddle.static.nn.layer_norm(input=elementwise_out) @@ -57,19 +55,17 @@ class PrelnResidualBiasFusePassTest(PassTest): class PrelnResidualBiasFusePassNoBiasTest(PassTest): - def setUp(self): paddle.enable_static() - with paddle.static.program_guard(self.main_program, - self.startup_program): - x = paddle.static.data(name="x", - shape=[128, 768], - dtype="float32", - lod_level=0) - y = paddle.static.data(name="y", - shape=[128, 768], - dtype="float32", - lod_level=0) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + x = paddle.static.data( + name="x", shape=[128, 768], dtype="float32", lod_level=0 + ) + y = paddle.static.data( + name="y", shape=[128, 768], dtype="float32", lod_level=0 + ) elementwise_out = x + y out = paddle.static.nn.layer_norm(input=elementwise_out) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_skip_layernorm_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_skip_layernorm_pass.py index d53e7109d6a..82f53f95d9e 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_skip_layernorm_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_skip_layernorm_pass.py @@ -21,18 +21,15 @@ import paddle.fluid.core as core class SkipLayerNormFusePassTest(PassTest): - def setUp(self): paddle.enable_static() with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name="x", - shape=[128, 768], - dtype="float32", - lod_level=0) - y = fluid.data(name="y", - shape=[128, 768], - dtype="float32", - lod_level=0) + x = fluid.data( + name="x", shape=[128, 768], dtype="float32", lod_level=0 + ) + y = fluid.data( + name="y", shape=[128, 768], dtype="float32", lod_level=0 + ) elementwise_out = fluid.layers.elementwise_add(x=x, y=y) out = fluid.layers.layer_norm(input=elementwise_out) @@ -42,7 +39,7 @@ class SkipLayerNormFusePassTest(PassTest): self.num_fused_ops = 1 self.graph_attrs = { "embedding_eltwise_layernorm_fuse_pass_flag": True, - "multihead_matmul_fuse_pass_flag": True + "multihead_matmul_fuse_pass_flag": True, } def test_check_program(self): diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_subgraph_python_interface.py b/python/paddle/fluid/tests/unittests/ir/test_ir_subgraph_python_interface.py index 56f2e5e0a24..bc9e63826cd 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_subgraph_python_interface.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_subgraph_python_interface.py @@ -27,13 +27,11 @@ paddle.enable_static() class TestQuantizationSubGraph(unittest.TestCase): - def build_graph_with_sub_graph(self): - def linear_fc(num): - data = fluid.layers.data(name='image', - shape=[1, 32, 32], - dtype='float32') + data = fluid.layers.data( + name='image', shape=[1, 32, 32], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') hidden = data for _ in range(num): @@ -63,7 +61,8 @@ class TestQuantizationSubGraph(unittest.TestCase): graph = IrGraph(core_graph, for_test=True) sub_graph = graph.get_sub_graph(0) all_sub_graphs = graph.all_sub_graphs( - for_test=True) # same reason for subgraph + for_test=True + ) # same reason for subgraph # Should return graph and sub_graphs at the same time. If only return sub_graph, the graph will # be destructed and the sub_graphs will be empty. return graph, all_sub_graphs @@ -75,7 +74,8 @@ class TestQuantizationSubGraph(unittest.TestCase): scope=fluid.global_scope(), place=place, activation_quantize_type='abs_max', - weight_quantize_type='range_abs_max') + weight_quantize_type='range_abs_max', + ) Find_inserted_quant_op = False for sub_graph in sub_graphs: transform_pass.apply(sub_graph) @@ -87,8 +87,9 @@ class TestQuantizationSubGraph(unittest.TestCase): def test_quant_sub_graphs_cpu(self): self.test_quant_sub_graphs(use_cuda=False) - @OpTestTool.skip_if(not paddle.is_compiled_with_cuda(), - "Not GPU version paddle") + @OpTestTool.skip_if( + not paddle.is_compiled_with_cuda(), "Not GPU version paddle" + ) def test_quant_sub_graphs_gpu(self): self.test_quant_sub_graphs(use_cuda=True) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_yolo_box_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_yolo_box_pass.py index e86327f62f8..58bda7ab838 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_yolo_box_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_yolo_box_pass.py @@ -20,15 +20,17 @@ from paddle.fluid.layer_helper import LayerHelper paddle.enable_static() -def multiclass_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=-1): +def multiclass_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=-1, +): helper = LayerHelper('multiclass_nms3', **locals()) output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) index = helper.create_variable_for_type_inference(dtype='int32') @@ -36,18 +38,20 @@ def multiclass_nms(bboxes, inputs = {'BBoxes': bboxes, 'Scores': scores} outputs = {'Out': output, 'Index': index, 'NmsRoisNum': nms_rois_num} - helper.append_op(type="multiclass_nms3", - inputs=inputs, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs=outputs) + helper.append_op( + type="multiclass_nms3", + inputs=inputs, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs=outputs, + ) output.stop_gradient = True index.stop_gradient = True @@ -55,7 +59,6 @@ def multiclass_nms(bboxes, class TestYoloBoxPass(unittest.TestCase): - def test_yolo_box_pass(self): program = paddle.static.Program() with paddle.static.program_guard(program): @@ -67,18 +70,22 @@ class TestYoloBoxPass(unittest.TestCase): div = paddle.divide(im_shape, im_scale) cast = paddle.cast(div, "int32") boxes0, scores0 = paddle.vision.ops.yolo_box( - yolo_box0_x, cast, [116, 90, 156, 198, 373, 326], 80, 0.005, 32) + yolo_box0_x, cast, [116, 90, 156, 198, 373, 326], 80, 0.005, 32 + ) boxes1, scores1 = paddle.vision.ops.yolo_box( - yolo_box1_x, cast, [30, 61, 62, 45, 59, 119], 80, 0.005, 16) + yolo_box1_x, cast, [30, 61, 62, 45, 59, 119], 80, 0.005, 16 + ) boxes2, scores2 = paddle.vision.ops.yolo_box( - yolo_box2_x, cast, [10, 13, 16, 30, 33, 23], 80, 0.005, 8) + yolo_box2_x, cast, [10, 13, 16, 30, 33, 23], 80, 0.005, 8 + ) transpose0 = paddle.transpose(scores0, [0, 2, 1]) transpose1 = paddle.transpose(scores1, [0, 2, 1]) transpose2 = paddle.transpose(scores2, [0, 2, 1]) concat0 = paddle.concat([boxes0, boxes1, boxes2], 1) concat1 = paddle.concat([transpose0, transpose1, transpose2], 2) - out0, out1, out2 = multiclass_nms(concat0, concat1, 0.01, 1000, 100, - 0.45, True, 1., 80) + out0, out1, out2 = multiclass_nms( + concat0, concat1, 0.01, 1000, 100, 0.45, True, 1.0, 80 + ) graph = core.Graph(program.desc) core.get_pass("yolo_box_fuse_pass").apply(graph) graph = paddle.fluid.framework.IrGraph(graph) diff --git a/python/paddle/fluid/tests/unittests/ir_memory_optimize_net_base.py b/python/paddle/fluid/tests/unittests/ir_memory_optimize_net_base.py index ca518747821..88c2024cff1 100644 --- a/python/paddle/fluid/tests/unittests/ir_memory_optimize_net_base.py +++ b/python/paddle/fluid/tests/unittests/ir_memory_optimize_net_base.py @@ -31,20 +31,22 @@ os.environ['CPU_NUM'] = '2' class BuildIrMemOptBase(unittest.TestCase): - def setup_reader(self): self.batch_size = 32 self.word_dict = paddle.dataset.imdb.word_dict() - self.train_reader = paddle.batch(paddle.dataset.imdb.train( - self.word_dict), - batch_size=self.batch_size) - - def check_network_convergence(self, - network, - use_cuda=True, - use_ir_memory_optimize=True, - enable_inplace=True, - iter=5): + self.train_reader = paddle.batch( + paddle.dataset.imdb.train(self.word_dict), + batch_size=self.batch_size, + ) + + def check_network_convergence( + self, + network, + use_cuda=True, + use_ir_memory_optimize=True, + enable_inplace=True, + iter=5, + ): if use_cuda and not core.is_compiled_with_cuda(): print('Skip use_cuda=True because Paddle is not compiled with cuda') return @@ -57,10 +59,9 @@ class BuildIrMemOptBase(unittest.TestCase): fluid.default_startup_program().random_seed = 100 fluid.default_main_program().random_seed = 100 - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") @@ -79,8 +80,9 @@ class BuildIrMemOptBase(unittest.TestCase): exe.run(fluid.default_startup_program()) train_cp = compiler.CompiledProgram(fluid.default_main_program()) - train_cp = train_cp.with_data_parallel(loss_name=cost.name, - build_strategy=build_strategy) + train_cp = train_cp.with_data_parallel( + loss_name=cost.name, build_strategy=build_strategy + ) fetch_list = [cost.name] begin = time.time() @@ -100,21 +102,23 @@ class BuildIrMemOptBase(unittest.TestCase): break end = time.time() - print("%.4f Instance per second" % ((self.batch_size * iter) / - (end - begin))) + print( + "%.4f Instance per second" + % ((self.batch_size * iter) / (end - begin)) + ) print(first_loss, last_loss) avg_last_loss_val = np.array(last_loss).mean() avg_first_loss_val = np.array(first_loss).mean() if math.isnan(float(avg_last_loss_val)) or math.isnan( - float(avg_first_loss_val)): + float(avg_first_loss_val) + ): sys.exit("got NaN loss, training failed.") return first_loss, last_loss class TestIrMemOptBase(BuildIrMemOptBase): - def setUp(self): self.network = None @@ -126,15 +130,22 @@ class TestIrMemOptBase(BuildIrMemOptBase): with fluid.program_guard(fluid.Program(), fluid.Program()): with fluid.scope_guard(core.Scope()): - baseline_first_loss, baseline_last_loss = self.check_network_convergence( - self.network) + ( + baseline_first_loss, + baseline_last_loss, + ) = self.check_network_convergence(self.network) cur_first_loss, cur_last_loss = self.check_network_convergence( - self.network) - - self.assertAlmostEquals(np.mean(baseline_last_loss), - np.mean(cur_last_loss), - delta=1e-6) - self.assertAlmostEquals(np.mean(baseline_first_loss), - np.mean(cur_first_loss), - delta=1e-6) + self.network + ) + + self.assertAlmostEquals( + np.mean(baseline_last_loss), + np.mean(cur_last_loss), + delta=1e-6, + ) + self.assertAlmostEquals( + np.mean(baseline_first_loss), + np.mean(cur_first_loss), + delta=1e-6, + ) diff --git a/python/paddle/fluid/tests/unittests/jit_load_rename_var.py b/python/paddle/fluid/tests/unittests/jit_load_rename_var.py index caaca0716cd..2ba1e5522d1 100644 --- a/python/paddle/fluid/tests/unittests/jit_load_rename_var.py +++ b/python/paddle/fluid/tests/unittests/jit_load_rename_var.py @@ -32,8 +32,9 @@ def rename_var_with_generator(names_old): else: temp_name = "_".join(temp_name) name_new = _generate_unique_var_name_sync_with_main_program( - temp_name) - if name_new not in names_old[:var_idx] + names_old[var_idx + 1:]: + temp_name + ) + if name_new not in names_old[:var_idx] + names_old[var_idx + 1 :]: break dict_rename_var_old_new[name_old] = name_new return dict_rename_var_old_new diff --git a/python/paddle/fluid/tests/unittests/launch_function_helper.py b/python/paddle/fluid/tests/unittests/launch_function_helper.py index 992bc58b5e4..b8bb4cbc35a 100644 --- a/python/paddle/fluid/tests/unittests/launch_function_helper.py +++ b/python/paddle/fluid/tests/unittests/launch_function_helper.py @@ -59,7 +59,6 @@ def wait(procs, timeout=30): def _find_free_port(port_set): - def __free_port(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(('', 0)) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/check_flags_mkldnn_ops_on_off.py b/python/paddle/fluid/tests/unittests/mkldnn/check_flags_mkldnn_ops_on_off.py index 4042b4da633..133320c7f37 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/check_flags_mkldnn_ops_on_off.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/check_flags_mkldnn_ops_on_off.py @@ -23,15 +23,23 @@ _enable_legacy_dygraph() def check(): - print("check: _global_flags()['FLAGS_use_mkldnn']=", - _global_flags()["FLAGS_use_mkldnn"]) - print("check: fluid.get_flags('FLAGS_use_mkldnn')=", - fluid.get_flags(['FLAGS_use_mkldnn'])) + print( + "check: _global_flags()['FLAGS_use_mkldnn']=", + _global_flags()["FLAGS_use_mkldnn"], + ) + print( + "check: fluid.get_flags('FLAGS_use_mkldnn')=", + fluid.get_flags(['FLAGS_use_mkldnn']), + ) print("check: DNNL_VERBOSE=", os.environ['DNNL_VERBOSE']) - print("check: FLAGS_tracer_mkldnn_ops_on=", - _global_flags()['FLAGS_tracer_mkldnn_ops_on']) - print("check: FLAGS_tracer_mkldnn_ops_off=", - _global_flags()['FLAGS_tracer_mkldnn_ops_off']) + print( + "check: FLAGS_tracer_mkldnn_ops_on=", + _global_flags()['FLAGS_tracer_mkldnn_ops_on'], + ) + print( + "check: FLAGS_tracer_mkldnn_ops_off=", + _global_flags()['FLAGS_tracer_mkldnn_ops_off'], + ) a_np = np.random.uniform(-2, 2, (10, 20, 30)).astype(np.float32) b_np = np.random.uniform(-5, 5, (10, 20, 30)).astype(np.float32) helper = LayerHelper(fluid.unique_name.generate(str("test")), act="relu") diff --git a/python/paddle/fluid/tests/unittests/mkldnn/check_flags_use_mkldnn.py b/python/paddle/fluid/tests/unittests/mkldnn/check_flags_use_mkldnn.py index 03786e0aed7..1bdca9244cc 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/check_flags_use_mkldnn.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/check_flags_use_mkldnn.py @@ -20,10 +20,14 @@ from paddle.fluid.framework import _global_flags def check(): - print("check: _global_flags()['FLAGS_use_mkldnn']=", - _global_flags()["FLAGS_use_mkldnn"]) - print("check: fluid.get_flags('FLAGS_use_mkldnn')=", - fluid.get_flags(['FLAGS_use_mkldnn'])) + print( + "check: _global_flags()['FLAGS_use_mkldnn']=", + _global_flags()["FLAGS_use_mkldnn"], + ) + print( + "check: fluid.get_flags('FLAGS_use_mkldnn')=", + fluid.get_flags(['FLAGS_use_mkldnn']), + ) print("check: DNNL_VERBOSE=", os.environ['DNNL_VERBOSE']) a_np = np.random.uniform(-2, 2, (10, 20, 30)).astype(np.float32) helper = LayerHelper(fluid.unique_name.generate(str("test")), act="relu") @@ -32,7 +36,7 @@ def check(): a = fluid.dygraph.to_variable(a_np) res1 = func(a) res2 = np.maximum(a_np, 0) - assert (np.array_equal(res1.numpy(), res2)) + assert np.array_equal(res1.numpy(), res2) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/mkldnn_op_test.py b/python/paddle/fluid/tests/unittests/mkldnn/mkldnn_op_test.py index 3b6788e6ab1..9d79ee3bf69 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/mkldnn_op_test.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/mkldnn_op_test.py @@ -18,12 +18,14 @@ import paddle.fluid as fluid def __assert_close(test_case, tensor, np_array, msg, atol=1e-4): - test_case.assertTrue(np.allclose(np.array(tensor), np_array, atol=atol), - msg) + test_case.assertTrue( + np.allclose(np.array(tensor), np_array, atol=atol), msg + ) -def check_if_mkldnn_primitives_exist_in_bwd(test_case, op_type, x, out, - out_grad, x_grad): +def check_if_mkldnn_primitives_exist_in_bwd( + test_case, op_type, x, out, out_grad, x_grad +): place = core.CPUPlace() var_dict = {'x': x, 'out': out, 'out@GRAD': out_grad, 'x@GRAD': x_grad} @@ -34,20 +36,23 @@ def check_if_mkldnn_primitives_exist_in_bwd(test_case, op_type, x, out, with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype=np.float32, - shape=ground_truth[name].shape) + block.create_var( + name=name, dtype=np.float32, shape=ground_truth[name].shape + ) - op = block.append_op(type=op_type, - inputs={ - 'X': block.var('x'), - }, - outputs={'Out': block.var('out')}, - attrs={'use_mkldnn': True}) + op = block.append_op( + type=op_type, + inputs={ + 'X': block.var('x'), + }, + outputs={'Out': block.var('out')}, + attrs={'use_mkldnn': True}, + ) # Generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - op.desc, set(), []) + op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -65,29 +70,35 @@ def check_if_mkldnn_primitives_exist_in_bwd(test_case, op_type, x, out, for i in range(2): out = exe.run( program, - feed={name: var_dict[name] - for name in ['x', 'out@GRAD']}, - fetch_list=['x@GRAD', 'out']) + feed={name: var_dict[name] for name in ['x', 'out@GRAD']}, + fetch_list=['x@GRAD', 'out'], + ) __assert_close(test_case, x_grad, out[0], 'x@GRAD') -def check_if_mkldnn_batchnorm_primitives_exist_in_bwd(test_case, var_dict, - place, shape, - data_layout): +def check_if_mkldnn_batchnorm_primitives_exist_in_bwd( + test_case, var_dict, place, shape, data_layout +): var_names = [ - 'x', 'scale', 'bias', 'mean', 'variance', 'y', 'saved_mean', - 'saved_variance' + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y', + 'saved_mean', + 'saved_variance', ] ground_truth = {name: var_dict[name] for name in var_names} program = fluid.Program() with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, dtype='float32', shape=ground_truth[name].shape + ) bn_op = block.append_op( type="batch_norm", inputs={ @@ -95,14 +106,14 @@ def check_if_mkldnn_batchnorm_primitives_exist_in_bwd(test_case, var_dict, "Scale": block.var('scale'), "Bias": block.var('bias'), "Mean": block.var('mean'), - "Variance": block.var('variance') + "Variance": block.var('variance'), }, outputs={ "Y": block.var('y'), "MeanOut": block.var('mean'), # share memory "VarianceOut": block.var('variance'), # share memory "SavedMean": block.var('saved_mean'), - "SavedVariance": block.var('saved_variance') + "SavedVariance": block.var('saved_variance'), }, attrs={ "momentum": test_case.momentum, @@ -111,15 +122,17 @@ def check_if_mkldnn_batchnorm_primitives_exist_in_bwd(test_case, var_dict, "data_layout": data_layout, "use_mkldnn": test_case.use_mkldnn, "fuse_with_relu": test_case.fuse_with_relu, - "use_global_stats": test_case.use_global_stats - }) - block.create_var(name='y@GRAD', - dtype='float32', - shape=var_dict['y'].shape) + "use_global_stats": test_case.use_global_stats, + }, + ) + block.create_var( + name='y@GRAD', dtype='float32', shape=var_dict['y'].shape + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - bn_op.desc, test_case.no_grad_set, []) + bn_op.desc, test_case.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -139,10 +152,17 @@ def check_if_mkldnn_batchnorm_primitives_exist_in_bwd(test_case, var_dict, program, feed={ name: var_dict[name] - for name in - ['x', 'scale', 'bias', 'mean', 'variance', 'y@GRAD'] + for name in [ + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y@GRAD', + ] }, - fetch_list=test_case.fetch_list) + fetch_list=test_case.fetch_list, + ) for id, name in enumerate(test_case.fetch_list): __assert_close(test_case, var_dict[name], out[id], name) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py index f7d6763bddc..de9c8dcf495 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py @@ -17,14 +17,16 @@ import unittest import numpy as np from scipy.special import erf import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTestTool, + convert_float_to_uint16, +) from paddle.fluid.tests.unittests.test_activation_op import TestActivation from paddle.fluid.tests.unittests.test_gelu_op import gelu @OpTestTool.skip_if_not_cpu_bf16() class MKLDNNBF16ActivationOp(metaclass=abc.ABCMeta): - @abc.abstractmethod def config(self): pass @@ -62,14 +64,15 @@ class MKLDNNBF16ActivationOp(metaclass=abc.ABCMeta): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.out)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.out)], + ) class TestMKLDNNSigmoidBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "sigmoid" @@ -81,7 +84,6 @@ class TestMKLDNNSigmoidBF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNSqrtBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "sqrt" @@ -96,7 +98,6 @@ class TestMKLDNNSqrtBF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNGeluErfBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "gelu" @@ -104,19 +105,19 @@ class TestMKLDNNGeluErfBF16Op(MKLDNNBF16ActivationOp, TestActivation): return gelu(x, False) def op_grad(self, dout, x): - return (dout * - (0.5 + 0.5 * erf(x / np.sqrt(2)) + - (x / np.sqrt(2 * np.pi) * np.exp(-0.5 * np.power(x, 2))))) + return dout * ( + 0.5 + + 0.5 * erf(x / np.sqrt(2)) + + (x / np.sqrt(2 * np.pi) * np.exp(-0.5 * np.power(x, 2))) + ) class TestMKLDNNGeluErfDim2BF16Op(TestMKLDNNGeluErfBF16Op): - def init_data(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(np.float32) class TestMKLDNNGeluTanhBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "gelu" @@ -125,23 +126,30 @@ class TestMKLDNNGeluTanhBF16Op(MKLDNNBF16ActivationOp, TestActivation): def op_grad(self, dout, x): grad_part = np.tanh( - np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))) - return dout * 0.5 * (1 + grad_part) * (1 + np.sqrt(2 / np.pi) * - (x + 0.134145 * np.power(x, 3)) * - (1 - grad_part)) + np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)) + ) + return ( + dout + * 0.5 + * (1 + grad_part) + * ( + 1 + + np.sqrt(2 / np.pi) + * (x + 0.134145 * np.power(x, 3)) + * (1 - grad_part) + ) + ) def set_attrs(self): self.attrs = {"use_mkldnn": True, "approximate": True} class TestMKLDNNGeluTanhDim2BF16Op(TestMKLDNNGeluTanhBF16Op): - def init_data(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(np.float32) class TestMKLDNNReluBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "relu" @@ -153,7 +161,6 @@ class TestMKLDNNReluBF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNMishBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "mish" @@ -161,14 +168,17 @@ class TestMKLDNNMishBF16Op(MKLDNNBF16ActivationOp, TestActivation): return x * np.tanh(np.log(1 + np.exp(x))) def op_grad(self, dout, x): - omega = np.exp( - 3 * x) + 4 * np.exp(2 * x) + np.exp(x) * (4 * x + 6) + 4 * (x + 1) + omega = ( + np.exp(3 * x) + + 4 * np.exp(2 * x) + + np.exp(x) * (4 * x + 6) + + 4 * (x + 1) + ) delta = np.exp(2 * x) + 2 * np.exp(x) + 2 return dout * ((np.exp(x) * omega) / delta**2) class TestMKLDNNRelu6BF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "relu6" @@ -180,7 +190,6 @@ class TestMKLDNNRelu6BF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNLeakyReluBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "leaky_relu" @@ -196,7 +205,6 @@ class TestMKLDNNLeakyReluBF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNSwishBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "swish" @@ -215,7 +223,6 @@ class TestMKLDNNSwishBF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNHardSwishBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "hard_swish" @@ -229,7 +236,6 @@ class TestMKLDNNHardSwishBF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNTanhBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "tanh" @@ -237,11 +243,10 @@ class TestMKLDNNTanhBF16Op(MKLDNNBF16ActivationOp, TestActivation): return np.tanh(x) def op_grad(self, dout, x): - return dout * (1 - np.tanh(x)**2) + return dout * (1 - np.tanh(x) ** 2) class TestMKLDNNAbsBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "abs" @@ -253,7 +258,6 @@ class TestMKLDNNAbsBF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNEluBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "elu" @@ -269,7 +273,6 @@ class TestMKLDNNEluBF16Op(MKLDNNBF16ActivationOp, TestActivation): class TestMKLDNNExpBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "exp" diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_mkldnn_op.py index 0ca003b45f2..12ab822d3e1 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_mkldnn_op.py @@ -17,13 +17,23 @@ import numpy as np from scipy.special import expit import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -from paddle.fluid.tests.unittests.test_activation_op import TestActivation, TestRelu, TestTanh, TestSqrt, TestAbs, TestLeakyRelu, TestSwish, TestHardSwish, TestRelu6, TestSigmoid +from paddle.fluid.tests.unittests.test_activation_op import ( + TestActivation, + TestRelu, + TestTanh, + TestSqrt, + TestAbs, + TestLeakyRelu, + TestSwish, + TestHardSwish, + TestRelu6, + TestSigmoid, +) from paddle.fluid.tests.unittests.test_gelu_op import gelu from mkldnn_op_test import check_if_mkldnn_primitives_exist_in_bwd class TestMKLDNNReluDim2(TestRelu): - def setUp(self): super(TestMKLDNNReluDim2, self).setUp() @@ -34,7 +44,6 @@ class TestMKLDNNReluDim2(TestRelu): class TestMKLDNNRelu6Dim2(TestRelu6): - def setUp(self): super(TestMKLDNNRelu6Dim2, self).setUp() self.attrs.update({"use_mkldnn": True}) @@ -44,7 +53,6 @@ class TestMKLDNNRelu6Dim2(TestRelu6): class TestMKLDNNLeakyReluDim2(TestLeakyRelu): - def setUp(self): super(TestMKLDNNLeakyReluDim2, self).setUp() @@ -55,7 +63,6 @@ class TestMKLDNNLeakyReluDim2(TestLeakyRelu): class TestMKLDNNGeluDim2(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.float32 @@ -69,7 +76,6 @@ class TestMKLDNNGeluDim2(TestActivation): class TestMKLDNNGeluDim2Approx(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.float32 @@ -83,7 +89,6 @@ class TestMKLDNNGeluDim2Approx(TestActivation): class TestMKLDNNTanhDim2(TestTanh): - def setUp(self): super(TestMKLDNNTanhDim2, self).setUp() @@ -94,7 +99,6 @@ class TestMKLDNNTanhDim2(TestTanh): class TestMKLDNNSqrtDim2(TestSqrt): - def setUp(self): super(TestMKLDNNSqrtDim2, self).setUp() @@ -105,7 +109,6 @@ class TestMKLDNNSqrtDim2(TestSqrt): class TestMKLDNNAbsDim2(TestAbs): - def setUp(self): super(TestMKLDNNAbsDim2, self).setUp() self.attrs = {"use_mkldnn": True} @@ -115,7 +118,6 @@ class TestMKLDNNAbsDim2(TestAbs): class TestMKLDNNSwishDim2(TestSwish): - def setUp(self): super(TestMKLDNNSwishDim2, self).setUp() @@ -127,21 +129,18 @@ class TestMKLDNNSwishDim2(TestSwish): class TestMKLDNNHardSwishDim2(TestHardSwish): - def setUp(self): super(TestMKLDNNHardSwishDim2, self).setUp() self.attrs = {"use_mkldnn": True} class TestMKLDNNSigmoidDim2(TestSigmoid): - def setUp(self): super(TestMKLDNNSigmoidDim2, self).setUp() self.attrs = {"use_mkldnn": True} class TestMKLDNNReluDim4(TestRelu): - def setUp(self): super(TestMKLDNNReluDim4, self).setUp() @@ -159,7 +158,6 @@ class TestMKLDNNReluDim4(TestRelu): class TestMKLDNNLeakyReluDim4(TestLeakyRelu): - def setUp(self): super(TestMKLDNNLeakyReluDim4, self).setUp() @@ -177,7 +175,6 @@ class TestMKLDNNLeakyReluDim4(TestLeakyRelu): class TestMKLDNNGeluDim4(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.float32 @@ -191,7 +188,6 @@ class TestMKLDNNGeluDim4(TestActivation): class TestMKLDNNGeluDim4Approx(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.float32 @@ -204,10 +200,10 @@ class TestMKLDNNGeluDim4Approx(TestActivation): self.attrs = {"use_mkldnn": True, "approximate": True} -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestMKLDNNGeluBf16Dim4(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.uint16 @@ -226,10 +222,10 @@ class TestMKLDNNGeluBf16Dim4(TestActivation): pass -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestMKLDNNGeluBf16Dim4Approx(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.uint16 @@ -249,7 +245,6 @@ class TestMKLDNNGeluBf16Dim4Approx(TestActivation): class TestMKLDNNTanhDim4(TestTanh): - def setUp(self): super(TestMKLDNNTanhDim4, self).setUp() @@ -261,7 +256,6 @@ class TestMKLDNNTanhDim4(TestTanh): class TestMKLDNNSqrtDim4(TestSqrt): - def setUp(self): super(TestMKLDNNSqrtDim4, self).setUp() @@ -273,7 +267,6 @@ class TestMKLDNNSqrtDim4(TestSqrt): class TestMKLDNNAbsDim4(TestAbs): - def setUp(self): super(TestMKLDNNAbsDim4, self).setUp() @@ -289,7 +282,6 @@ class TestMKLDNNAbsDim4(TestAbs): class TestMKLDNNSwishDim4(TestSwish): - def setUp(self): super(TestMKLDNNSwishDim4, self).setUp() @@ -311,12 +303,12 @@ def ref_hardswish(x, threshold=6.0, scale=6.0, offset=3.0): if x_dtype == 'float16': x_dtype = 'float16' x = x.astype('float32') - return (x * np.minimum(np.maximum(x + offset, 0.), threshold) / - scale).astype(x_dtype) + return ( + x * np.minimum(np.maximum(x + offset, 0.0), threshold) / scale + ).astype(x_dtype) class TestMKLDNNHardSwishDim4(TestHardSwish): - def setUp(self): super(TestMKLDNNHardSwishDim4, self).setUp() @@ -338,7 +330,6 @@ class TestMKLDNNHardSwishDim4(TestHardSwish): class TestMKLDNNMish(TestActivation): - def setUp(self): self.op_type = "mish" self.dtype = np.float32 @@ -352,7 +343,6 @@ class TestMKLDNNMish(TestActivation): class TestMKLDNNRound(TestActivation): - def setUp(self): self.op_type = "round" @@ -365,7 +355,6 @@ class TestMKLDNNRound(TestActivation): class TestMKLDNNSigmoidDim4(TestSigmoid): - def setUp(self): super(TestMKLDNNSigmoidDim4, self).setUp() @@ -377,7 +366,6 @@ class TestMKLDNNSigmoidDim4(TestSigmoid): class TestMKLDNNEluDefaultAlpha(TestActivation): - def setUp(self): self.op_type = "elu" self.set_alpha() @@ -387,8 +375,8 @@ class TestMKLDNNEluDefaultAlpha(TestActivation): self.inputs = {'X': x} self.attrs = {'use_mkldnn': True, 'alpha': self.alpha} self.outputs = { - 'Out': - np.maximum(0, x) + np.minimum(0, self.alpha * (np.exp(x) - 1)) + 'Out': np.maximum(0, x) + + np.minimum(0, self.alpha * (np.exp(x) - 1)) } def set_alpha(self): @@ -396,13 +384,11 @@ class TestMKLDNNEluDefaultAlpha(TestActivation): class TestMKLDNNEluCustomAlpha(TestMKLDNNEluDefaultAlpha): - def set_alpha(self): self.alpha = 2.5 class TestMKLDNNExpOp(TestActivation): - def setUp(self): self.op_type = "exp" x = np.random.random((5, 5, 4)).astype("float32") @@ -414,7 +400,6 @@ class TestMKLDNNExpOp(TestActivation): # Check if primitives already exist in backward class TestMKLDNNAbsPrimitivesAlreadyExist(unittest.TestCase): - def setUp(self): super(TestMKLDNNAbsPrimitivesAlreadyExist, self).setUp() @@ -430,9 +415,9 @@ class TestMKLDNNAbsPrimitivesAlreadyExist(unittest.TestCase): return out_grad * np.sign(x) def test_check(self): - check_if_mkldnn_primitives_exist_in_bwd(self, self.op_type, self.x, - self.out, self.out_grad, - self.x_grad) + check_if_mkldnn_primitives_exist_in_bwd( + self, self.op_type, self.x, self.out, self.out_grad, self.x_grad + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_batch_norm_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_batch_norm_mkldnn_op.py index f8c5ca3d1be..f4190f06634 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_batch_norm_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_batch_norm_mkldnn_op.py @@ -16,47 +16,69 @@ import unittest import numpy as np import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import _set_use_system_allocator -from paddle.fluid.tests.unittests.test_batch_norm_op import TestBatchNormOpInference, TestBatchNormOpTraining, _reference_training, _reference_grad +from paddle.fluid.tests.unittests.test_batch_norm_op import ( + TestBatchNormOpInference, + TestBatchNormOpTraining, + _reference_training, + _reference_grad, +) from mkldnn_op_test import check_if_mkldnn_batchnorm_primitives_exist_in_bwd _set_use_system_allocator(True) class TestMKLDNNBatchNormOpTraining(TestBatchNormOpTraining): - def init_kernel_type(self): self.use_mkldnn = True self.data_formats = ["NCHW"] - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): if data_layout != "NCHW" and data_layout != "NHWC": raise ValueError("Unknown data order.") # run forward y, saved_mean, saved_variance = _reference_training( - x, scale, bias, epsilon, data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = saved_variance * (1. - momentum) + momentum * variance + x, scale, bias, epsilon, data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = saved_variance * (1.0 - momentum) + momentum * variance # run backward - x_grad, scale_grad, bias_grad = _reference_grad(x, y_grad, scale, - saved_mean, - saved_variance, epsilon, - data_layout) - - return y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad + x_grad, scale_grad, bias_grad = _reference_grad( + x, y_grad, scale, saved_mean, saved_variance, epsilon, data_layout + ) + + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) class TestMKLDNNBatchNormOpTraining_NHWC(TestMKLDNNBatchNormOpTraining): - def init_kernel_type(self): self.use_mkldnn = True self.data_formats = ["NHWC"] class TestMKLDNNBatchNormOpExistedPrimitives(TestMKLDNNBatchNormOpTraining): - def init_test_case(self): TestMKLDNNBatchNormOpTraining.init_test_case(self) self.fetch_list = ['y', 'x@GRAD'] @@ -74,20 +96,38 @@ class TestMKLDNNBatchNormOpExistedPrimitives(TestMKLDNNBatchNormOpTraining): mean, variance = self.set_mean_variance(scale_shape, x, data_layout) y_grad = np.random.random_sample(shape).astype(np.float32) - y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad = self.ref_forward_backward( - x, y_grad, scale, bias, mean, variance, self.epsilon, self.momentum, - shape, data_layout) + ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) = self.ref_forward_backward( + x, + y_grad, + scale, + bias, + mean, + variance, + self.epsilon, + self.momentum, + shape, + data_layout, + ) var_dict = locals() var_dict['y@GRAD'] = y_grad var_dict['x@GRAD'] = x_grad var_dict['scale@GRAD'] = scale_grad var_dict['bias@GRAD'] = bias_grad check_if_mkldnn_batchnorm_primitives_exist_in_bwd( - self, var_dict, place, shape, data_layout) + self, var_dict, place, shape, data_layout + ) class TestMKLDNNBatchNormOpInference(TestBatchNormOpInference): - def init_kernel_type(self): self.use_mkldnn = True @@ -98,7 +138,6 @@ class TestMKLDNNBatchNormOpInference(TestBatchNormOpInference): class TestMKLDNNBatchNormOpInference_NHWC(TestMKLDNNBatchNormOpInference): - def test_check_output(self): place = core.CPUPlace() data_format = "NHWC" @@ -106,7 +145,6 @@ class TestMKLDNNBatchNormOpInference_NHWC(TestMKLDNNBatchNormOpInference): class TestMKLDNNBatchNormOpWithReluInference(TestBatchNormOpInference): - def init_kernel_type(self): self.use_mkldnn = True self.fuse_with_relu = True @@ -119,5 +157,6 @@ class TestMKLDNNBatchNormOpWithReluInference(TestBatchNormOpInference): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_mkldnn_op.py index 2a91d625730..273b232ecf8 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_mkldnn_op.py @@ -19,12 +19,9 @@ from paddle.fluid.tests.unittests.op_test import OpTest from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -def bilinear_interp_mkldnn_np(input, - out_h, - out_w, - out_size=None, - actual_shape=None, - data_layout='NCHW'): +def bilinear_interp_mkldnn_np( + input, out_h, out_w, out_size=None, actual_shape=None, data_layout='NCHW' +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -54,10 +51,12 @@ def bilinear_interp_mkldnn_np(input, input_h1_w0 = input[:, :, h1, w0] input_h0_w1 = input[:, :, h0, w1] input_h1_w1 = input[:, :, h1, w1] - out[:, :, oh, - ow] = input_h0_w0 * (1 - Wh) * (1 - Ww) + input_h1_w0 * Wh * ( - 1 - Ww) + input_h0_w1 * (1 - - Wh) * Ww + input_h1_w1 * Wh * Ww + out[:, :, oh, ow] = ( + input_h0_w0 * (1 - Wh) * (1 - Ww) + + input_h1_w0 * Wh * (1 - Ww) + + input_h0_w1 * (1 - Wh) * Ww + + input_h1_w1 * Wh * Ww + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -67,7 +66,6 @@ def bilinear_interp_mkldnn_np(input, @skip_check_grad_ci(reason="Haven not implement interpolate grad kernel.") class TestBilinearInterpMKLDNNOp(OpTest): - def init_test_case(self): pass @@ -102,9 +100,14 @@ class TestBilinearInterpMKLDNNOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_mkldnn_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.data_layout) + output_np = bilinear_interp_mkldnn_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: @@ -117,7 +120,7 @@ class TestBilinearInterpMKLDNNOp(OpTest): 'out_w': self.out_w, 'scale': self.scale, 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': output_np} @@ -126,7 +129,6 @@ class TestBilinearInterpMKLDNNOp(OpTest): class TestBilinearInterpOpMKLDNNNHWC(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 27 @@ -136,73 +138,67 @@ class TestBilinearInterpOpMKLDNNNHWC(TestBilinearInterpMKLDNNOp): class TestBilinearNeighborInterpMKLDNNCase2(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 1. + self.scale = 1.0 class TestBilinearNeighborInterpDataLayout(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [2, 4, 4, 5] self.out_h = 6 self.out_w = 7 - self.scale = 0. + self.scale = 0.0 self.data_layout = "NHWC" class TestBilinearNeighborInterpCase3(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 128 - self.scale = 0. + self.scale = 0.0 class TestBilinearNeighborInterpCase4(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") class TestBilinearNeighborInterpCase5(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([13, 13]).astype("int32") class TestBilinearNeighborInterpCase6(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") class TestBilinearNeighborInterpSame(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_v2_mkldnn_op.py index 55f57764b12..e6cf8381fa2 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_v2_mkldnn_op.py @@ -19,12 +19,9 @@ from paddle.fluid.tests.unittests.op_test import OpTest from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -def bilinear_interp_mkldnn_np(input, - out_h, - out_w, - out_size=None, - actual_shape=None, - data_layout='NCHW'): +def bilinear_interp_mkldnn_np( + input, out_h, out_w, out_size=None, actual_shape=None, data_layout='NCHW' +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -54,10 +51,12 @@ def bilinear_interp_mkldnn_np(input, input_h1_w0 = input[:, :, h1, w0] input_h0_w1 = input[:, :, h0, w1] input_h1_w1 = input[:, :, h1, w1] - out[:, :, oh, - ow] = input_h0_w0 * (1 - Wh) * (1 - Ww) + input_h1_w0 * Wh * ( - 1 - Ww) + input_h0_w1 * (1 - - Wh) * Ww + input_h1_w1 * Wh * Ww + out[:, :, oh, ow] = ( + input_h0_w0 * (1 - Wh) * (1 - Ww) + + input_h1_w0 * Wh * (1 - Ww) + + input_h0_w1 * (1 - Wh) * Ww + + input_h1_w1 * Wh * Ww + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -67,7 +66,6 @@ def bilinear_interp_mkldnn_np(input, @skip_check_grad_ci(reason="Haven not implement interpolate grad kernel.") class TestBilinearInterpMKLDNNOp(OpTest): - def init_test_case(self): pass @@ -116,9 +114,14 @@ class TestBilinearInterpMKLDNNOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_mkldnn_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.data_layout) + output_np = bilinear_interp_mkldnn_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.data_layout, + ) if isinstance(self.scale, float): self.scale = [self.scale, self.scale] @@ -134,7 +137,7 @@ class TestBilinearInterpMKLDNNOp(OpTest): 'out_w': self.out_w, 'scale': self.scale, 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': output_np} @@ -143,7 +146,6 @@ class TestBilinearInterpMKLDNNOp(OpTest): class TestBilinearInterpOpMKLDNNNHWC(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 27 @@ -153,7 +155,6 @@ class TestBilinearInterpOpMKLDNNNHWC(TestBilinearInterpMKLDNNOp): class TestBilinearNeighborInterpMKLDNNCase2(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 @@ -161,7 +162,6 @@ class TestBilinearNeighborInterpMKLDNNCase2(TestBilinearInterpMKLDNNOp): class TestBilinearNeighborInterpCase3(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -170,7 +170,6 @@ class TestBilinearNeighborInterpCase3(TestBilinearInterpMKLDNNOp): class TestBilinearNeighborInterpCase4(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -180,7 +179,6 @@ class TestBilinearNeighborInterpCase4(TestBilinearInterpMKLDNNOp): class TestBilinearNeighborInterpCase5(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 9, 6] self.out_h = 12 @@ -189,7 +187,6 @@ class TestBilinearNeighborInterpCase5(TestBilinearInterpMKLDNNOp): class TestBilinearNeighborInterpCase6(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -199,7 +196,6 @@ class TestBilinearNeighborInterpCase6(TestBilinearInterpMKLDNNOp): class TestBilinearNeighborInterpSame(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 @@ -210,5 +206,6 @@ class TestBilinearNeighborInterpSame(TestBilinearInterpMKLDNNOp): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_cast_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_cast_mkldnn_op.py index 8f7267d212e..92b98d35d02 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_cast_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_cast_mkldnn_op.py @@ -20,10 +20,10 @@ import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestCastBF16ToFP32MKLDNNOp(OpTest): - def init_data(self): self.out = np.random.random(size=[10, 10]).astype("float32") self.x = convert_float_to_uint16(self.out) @@ -32,12 +32,15 @@ class TestCastBF16ToFP32MKLDNNOp(OpTest): self.init_data() self.inputs = {'X': self.x} self.outputs = {'Out': self.out} - prepare_dtype = lambda x: int(core.VarDesc.VarType.BF16 if x.dtype != np - .float32 else core.VarDesc.VarType.FP32) + prepare_dtype = lambda x: int( + core.VarDesc.VarType.BF16 + if x.dtype != np.float32 + else core.VarDesc.VarType.FP32 + ) self.attrs = { 'in_dtype': prepare_dtype(self.x), 'out_dtype': prepare_dtype(self.out), - 'use_mkldnn': True + 'use_mkldnn': True, } self.op_type = 'cast' @@ -46,29 +49,28 @@ class TestCastBF16ToFP32MKLDNNOp(OpTest): def test_check_grad(self): self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", check_dygraph=False, user_defined_grads=[self.inputs['X']], - user_defined_grad_outputs=[self.outputs['Out']]) + user_defined_grad_outputs=[self.outputs['Out']], + ) class TestCastFP32ToBF16MKLDNNOp(TestCastBF16ToFP32MKLDNNOp): - def init_data(self): self.x = np.random.random(size=[2, 6]).astype("float32") self.out = convert_float_to_uint16(self.x) class TestCastBF16ToBF16MKLDNNOp(TestCastBF16ToFP32MKLDNNOp): - def init_data(self): self.x = np.random.random(size=[6, 13]).astype("uint16") self.out = self.x class TestCastFP32ToFP32MKLDNNOp(TestCastBF16ToFP32MKLDNNOp): - def init_data(self): self.x = np.random.random(size=[7, 15]).astype("float32") self.out = self.x diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_clip_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_clip_mkldnn_op.py index b9a9119e5ad..012c7cb018b 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_clip_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_clip_mkldnn_op.py @@ -14,14 +14,17 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) import paddle import paddle.fluid.core as core @OpTestTool.skip_if_not_cpu_bf16() class TestClipOneDNNOp(OpTest): - def setUp(self): self.op_type = "clip" self.set_inputs() @@ -29,10 +32,16 @@ class TestClipOneDNNOp(OpTest): self.set_additional_inputs() self.adjust_op_settings() - self.min = self.attrs[ - 'min'] if 'Min' not in self.inputs else self.inputs['Min'] - self.max = self.attrs[ - 'max'] if 'Max' not in self.inputs else self.inputs['Max'] + self.min = ( + self.attrs['min'] + if 'Min' not in self.inputs + else self.inputs['Min'] + ) + self.max = ( + self.attrs['max'] + if 'Max' not in self.inputs + else self.inputs['Max'] + ) self.outputs = {'Out': np.clip(self.x_fp32, self.min, self.max)} @@ -57,19 +66,16 @@ class TestClipOneDNNOp(OpTest): class TestClipMinAsInputOneDNNOp(TestClipOneDNNOp): - def set_additional_inputs(self): self.inputs['Min'] = np.array([6.8]).astype('float32') class TestClipMaxAsInputOneDNNOp(TestClipOneDNNOp): - def set_additional_inputs(self): self.inputs['Max'] = np.array([9.1]).astype('float32') class TestClipMaxAndMinAsInputsOneDNNOp(TestClipOneDNNOp): - def set_additional_inputs(self): self.inputs['Max'] = np.array([8.5]).astype('float32') self.inputs['Min'] = np.array([7.1]).astype('float32') @@ -77,10 +83,8 @@ class TestClipMaxAndMinAsInputsOneDNNOp(TestClipOneDNNOp): # BF16 TESTS def create_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestClipBF16OneDNNOp(parent): - def set_inputs(self): self.x_fp32 = np.random.random((10, 10)).astype(np.float32) * 25 self.inputs = {'X': convert_float_to_uint16(self.x_fp32)} @@ -95,8 +99,10 @@ def create_bf16_test_class(parent): for i in range(self.dx.shape[0]): for j in range(self.dx.shape[1]): - if self.x_fp32[j][i] > self.min and self.x_fp32[j][ - i] < self.max: + if ( + self.x_fp32[j][i] > self.min + and self.x_fp32[j][i] < self.max + ): self.dx[j][i] = self.dout[j][i] def test_check_output(self): @@ -105,10 +111,12 @@ def create_bf16_test_class(parent): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "BF16") TestClipBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_bf16_mkldnn_op.py index f10b04bdb5b..eefb3c766ab 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_bf16_mkldnn_op.py @@ -20,10 +20,10 @@ from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestConcatBf16Op(OpTest): - def setUp(self): self.op_type = "concat" self.use_mkldnn = True @@ -35,14 +35,15 @@ class TestConcatBf16Op(OpTest): self.attrs = { 'axis': self.axis, 'use_mkldnn': True, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } self.sections = [self.x0.shape[self.axis]] * 2 self.sections[1] += self.x1.shape[self.axis] - self.output = np.concatenate((self.x0, self.x1, self.x2), - axis=self.axis).astype(np.uint16) + self.output = np.concatenate( + (self.x0, self.x1, self.x2), axis=self.axis + ).astype(np.uint16) self.outputs = {'Out': self.output} def calculate_grads(self): @@ -55,20 +56,25 @@ class TestConcatBf16Op(OpTest): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["x0", "x1", "x2"], + core.CPUPlace(), + ["x0", "x1", "x2"], "Out", user_defined_grads=[self.dxs[0], self.dxs[1], self.dxs[2]], - user_defined_grad_outputs=[self.dout]) + user_defined_grad_outputs=[self.dout], + ) -# --------------------test concat bf16 in with axis 0-------------------- + # --------------------test concat bf16 in with axis 0-------------------- def init_test_data(self): self.x0 = convert_float_to_uint16( - np.random.random(self.x0_shape).astype(np.float32)) + np.random.random(self.x0_shape).astype(np.float32) + ) self.x1 = convert_float_to_uint16( - np.random.random(self.x1_shape).astype(np.float32)) + np.random.random(self.x1_shape).astype(np.float32) + ) self.x2 = convert_float_to_uint16( - np.random.random(self.x2_shape).astype(np.float32)) + np.random.random(self.x2_shape).astype(np.float32) + ) def init_axis(self): self.axis = 0 @@ -83,7 +89,6 @@ class TestConcatBf16Op(OpTest): class TestAxis1Case(TestConcatBf16Op): - def init_axis(self): self.axis = 1 @@ -97,7 +102,6 @@ class TestAxis1Case(TestConcatBf16Op): class TestAxis2Case(TestConcatBf16Op): - def init_axis(self): self.axis = 2 @@ -111,7 +115,6 @@ class TestAxis2Case(TestConcatBf16Op): class TestAxis3Case(TestConcatBf16Op): - def init_axis(self): self.axis = 3 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_int8_mkldnn_op.py index da07bce9447..5d275c7fb43 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_int8_mkldnn_op.py @@ -18,7 +18,6 @@ from paddle.fluid.tests.unittests.op_test import OpTest class TestConcatOp(OpTest): - def setUp(self): self.op_type = "concat" self.use_mkldnn = True @@ -29,8 +28,9 @@ class TestConcatOp(OpTest): self.inputs = {'X': [('x0', self.x0), ('x1', self.x1), ('x2', self.x2)]} self.attrs = {'axis': self.axis, 'use_mkldnn': True} - self.output = np.concatenate((self.x0, self.x1, self.x2), - axis=self.axis).astype('int') + self.output = np.concatenate( + (self.x0, self.x1, self.x2), axis=self.axis + ).astype('int') self.outputs = {'Out': self.output} @@ -38,7 +38,7 @@ class TestConcatOp(OpTest): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.check_output(check_dygraph=False) -#--------------------test concat s8 in with axis 0-------------------- + # --------------------test concat s8 in with axis 0-------------------- def init_test_data(self): self.x0 = (np.random.randint(0, 100, self.x0_shape) - 50).astype('int8') @@ -54,11 +54,10 @@ class TestConcatOp(OpTest): self.x2_shape = [3, 2, 1, 2] -#--------------------test concat u8 in with axis 0-------------------- +# --------------------test concat u8 in with axis 0-------------------- class TestConcatOp2(TestConcatOp): - def init_test_data(self): self.x0 = (np.random.randint(0, 100, self.x0_shape)).astype('uint8') self.x1 = (np.random.randint(0, 50, self.x1_shape)).astype('uint8') @@ -75,10 +74,9 @@ class TestConcatOp2(TestConcatOp): def create_test_int8_class(parent): - #--------------------test concat s8/u8 in with axis 1-------------------- + # --------------------test concat s8/u8 in with axis 1-------------------- class TestAxis1Case(parent): - def init_axis(self): self.axis = 1 @@ -87,10 +85,9 @@ def create_test_int8_class(parent): self.x1_shape = [1, 2, 5, 5] self.x2_shape = [1, 3, 5, 5] -#--------------------test concat s8/u8 in with axis 2-------------------- + # --------------------test concat s8/u8 in with axis 2-------------------- class TestAxis2Case(parent): - def init_axis(self): self.axis = 2 @@ -99,11 +96,9 @@ def create_test_int8_class(parent): self.x1_shape = [2, 3, 5, 5] self.x2_shape = [2, 3, 6, 5] - -#--------------------test concat s8/u8 in with axis 3-------------------- + # --------------------test concat s8/u8 in with axis 3-------------------- class TestAxis3Case(parent): - def init_axis(self): self.axis = 3 @@ -122,10 +117,12 @@ def create_test_int8_class(parent): globals()[cls_name_2] = TestAxis2Case globals()[cls_name_3] = TestAxis3Case + create_test_int8_class(TestConcatOp) create_test_int8_class(TestConcatOp2) if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_mkldnn_op.py index 1788923e20b..549c9620be8 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_mkldnn_op.py @@ -21,7 +21,6 @@ from paddle import enable_static class TestConcatAxis0OneDNNOp(OpTest): - def setUp(self): self.op_type = "concat" self.mkldnn_data_type = "float32" @@ -33,11 +32,12 @@ class TestConcatAxis0OneDNNOp(OpTest): self.attrs = { 'axis': self.axis, 'use_mkldnn': True, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } - self.output = np.concatenate((self.x0, self.x1, self.x2), - axis=self.axis).astype(self.dtype) + self.output = np.concatenate( + (self.x0, self.x1, self.x2), axis=self.axis + ).astype(self.dtype) self.outputs = {'Out': self.output} @@ -68,7 +68,6 @@ class TestConcatAxis0OneDNNOp(OpTest): class TestConcatAxis1OneDNNOp(TestConcatAxis0OneDNNOp): - def init_axis(self): self.axis = 1 @@ -79,7 +78,6 @@ class TestConcatAxis1OneDNNOp(TestConcatAxis0OneDNNOp): class TestConcatAxis2OneDNNOp(TestConcatAxis0OneDNNOp): - def init_axis(self): self.axis = 2 @@ -90,7 +88,6 @@ class TestConcatAxis2OneDNNOp(TestConcatAxis0OneDNNOp): class TestConcatAxis3OneDNNOp(TestConcatAxis0OneDNNOp): - def init_axis(self): self.axis = 3 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_bf16_mkldnn_op.py index e4f23f195d2..80657bad631 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_bf16_mkldnn_op.py @@ -16,8 +16,15 @@ import unittest import numpy as np import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16, OpTestTool -from paddle.fluid.tests.unittests.test_conv2d_op import conv2d_forward_naive, TestConv2DOp +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + OpTestTool, +) +from paddle.fluid.tests.unittests.test_conv2d_op import ( + conv2d_forward_naive, + TestConv2DOp, +) def conv2d_residual_naive(out, residual): @@ -26,10 +33,10 @@ def conv2d_residual_naive(out, residual): return out -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestConv2DBF16Op(TestConv2DOp): - def setUp(self): self.op_type = "conv2d" self.use_cudnn = False @@ -53,7 +60,7 @@ class TestConv2DBF16Op(TestConv2DOp): self.conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } self.input = np.random.random(self.input_size).astype(np.float32) @@ -61,16 +68,18 @@ class TestConv2DBF16Op(TestConv2DOp): self.inputs_fp32 = {'Input': self.input, 'Filter': self.filter} - conv_out, _, _, _, _ = conv2d_forward_naive(self.input, self.filter, - self.groups, - self.conv2d_param) + conv_out, _, _, _, _ = conv2d_forward_naive( + self.input, self.filter, self.groups, self.conv2d_param + ) self.conv_output_float = conv_out if self.fuse_residual: self.input_residual = np.random.random( - self.input_residual_size).astype(np.float32) + self.input_residual_size + ).astype(np.float32) self.conv_output_float = conv2d_residual_naive( - self.conv_output_float, self.input_residual) + self.conv_output_float, self.input_residual + ) self.conv_output = convert_float_to_uint16(self.conv_output_float) self.outputs = {'Output': self.conv_output} elif self.force_fp32_output: @@ -87,15 +96,16 @@ class TestConv2DBF16Op(TestConv2DOp): self.filter = convert_float_to_uint16(self.filter) self.inputs = { - 'Input': - self.input, - 'Filter': - OpTest.np_dtype_to_fluid_dtype(self.filter.astype(self.weight_type)) + 'Input': self.input, + 'Filter': OpTest.np_dtype_to_fluid_dtype( + self.filter.astype(self.weight_type) + ), } if self.fuse_residual: self.inputs['ResidualData'] = OpTest.np_dtype_to_fluid_dtype( - convert_float_to_uint16(self.input_residual)) + convert_float_to_uint16(self.input_residual) + ) self.attrs = { 'strides': self.stride, @@ -106,7 +116,7 @@ class TestConv2DBF16Op(TestConv2DOp): 'use_mkldnn': self.use_mkldnn, 'mkldnn_data_type': self.mkldnn_data_type, 'force_fp32_output': self.force_fp32_output, - 'fuse_residual_connection': self.fuse_residual + 'fuse_residual_connection': self.fuse_residual, } self.init_additional_attrs() @@ -156,7 +166,6 @@ class TestConv2DBF16Op(TestConv2DOp): @OpTestTool.skip_if_not_cpu_bf16() class TestConv2DWithGradBF16Op(TestConv2DBF16Op): - def init_fuse_relu(self): self.fuse_activation = None @@ -177,10 +186,12 @@ class TestConv2DWithGradBF16Op(TestConv2DBF16Op): dx, dweights = conv_backward(dout, x, w, self.conv2d_param) self.check_grad_with_place( - core.CPUPlace(), ["Input", "Filter"], + core.CPUPlace(), + ["Input", "Filter"], "Output", user_defined_grads=[dx, dweights], - user_defined_grad_outputs=[convert_float_to_uint16(dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(dout)], + ) def test_check_grad_no_filter(self): dout = self.conv_output_float @@ -190,11 +201,13 @@ class TestConv2DWithGradBF16Op(TestConv2DBF16Op): dx, _ = conv_backward(dout, x, w, self.conv2d_param) self.check_grad_with_place( - core.CPUPlace(), ["Input"], + core.CPUPlace(), + ["Input"], "Output", set(['Filter']), user_defined_grads=[dx], - user_defined_grad_outputs=[convert_float_to_uint16(dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(dout)], + ) def test_check_grad_no_input(self): dout = self.conv_output_float @@ -204,11 +217,13 @@ class TestConv2DWithGradBF16Op(TestConv2DBF16Op): _, dweights = conv_backward(dout, x, w, self.conv2d_param) self.check_grad_with_place( - core.CPUPlace(), ["Filter"], + core.CPUPlace(), + ["Filter"], "Output", set(['Input']), user_defined_grads=[dweights], - user_defined_grad_outputs=[convert_float_to_uint16(dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(dout)], + ) def conv_backward(dout, x, w, params): @@ -224,7 +239,7 @@ def conv_backward(dout, x, w, params): H_out = int(1 + (H + 2 * padding - KH) / stride[0]) W_out = int(1 + (W + 2 * padding - KW) / stride[1]) - x_padded = np.pad(x, ((0, ), (0, ), (padding, ), (padding, )), 'constant') + x_padded = np.pad(x, ((0,), (0,), (padding,), (padding,)), 'constant') for n in range(N): for oc in range(OC): @@ -233,11 +248,17 @@ def conv_backward(dout, x, w, params): for k in range(H_out): for l in range(W_out): for ic in range(IC): - dweights[oc, ic, i, j] += x_padded[ - n, ic, i + k * stride[0], - j + l * stride[1]] * dout[n, oc, k, l] - - dx_padded = np.pad(dx, ((0, ), (0, ), (padding, ), (padding, )), 'constant') + dweights[oc, ic, i, j] += ( + x_padded[ + n, + ic, + i + k * stride[0], + j + l * stride[1], + ] + * dout[n, oc, k, l] + ) + + dx_padded = np.pad(dx, ((0,), (0,), (padding,), (padding,)), 'constant') w_ = np.zeros_like(w) for i in range(KH): @@ -251,10 +272,14 @@ def conv_backward(dout, x, w, params): for kh in range(KH): for kw in range(KW): for ic in range(IC): - dx_padded[n, ic, stride[0] * i + kh, - stride[1] * j + - kw] += dout[n, oc, i, j] * w[oc, ic, - kh, kw] + dx_padded[ + n, + ic, + stride[0] * i + kh, + stride[1] * j + kw, + ] += ( + dout[n, oc, i, j] * w[oc, ic, kh, kw] + ) if padding == 0: dx = dx_padded @@ -265,21 +290,18 @@ def conv_backward(dout, x, w, params): class TestConv2DBF16WithPadding1(TestConv2DWithGradBF16Op): - def init_test_case(self): TestConv2DWithGradBF16Op.init_test_case(self) self.pad = [1, 1] class TestConv2DBF16WithStride2(TestConv2DWithGradBF16Op): - def init_test_case(self): TestConv2DWithGradBF16Op.init_test_case(self) self.stride = [2, 3] class TestConv2D(TestConv2DBF16Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -294,7 +316,6 @@ class TestConv2D(TestConv2DBF16Op): class TestWithPad(TestConv2D): - def init_test_case(self): TestConv2D.init_test_case(self) self.pad = [1, 1] @@ -302,13 +323,11 @@ class TestWithPad(TestConv2D): class TestWithGroup(TestConv2D): - def init_group(self): self.groups = 3 class TestWithStride(TestConv2DBF16Op): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -323,7 +342,6 @@ class TestWithStride(TestConv2DBF16Op): class TestWithDilations(TestConv2DBF16Op): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -339,7 +357,6 @@ class TestWithDilations(TestConv2DBF16Op): class TestWith1x1ForceFP32Output(TestConv2DBF16Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -356,7 +373,6 @@ class TestWith1x1ForceFP32Output(TestConv2DBF16Op): class TestWithInput1x1Filter1x1(TestConv2DBF16Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -372,5 +388,6 @@ class TestWithInput1x1Filter1x1(TestConv2DBF16Op): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_int8_mkldnn_op.py index 4ec61b0bb0b..c4113c12c4c 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_int8_mkldnn_op.py @@ -17,7 +17,10 @@ import numpy as np import os import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest -from paddle.fluid.tests.unittests.test_conv2d_op import conv2d_forward_naive, TestConv2DOp +from paddle.fluid.tests.unittests.test_conv2d_op import ( + conv2d_forward_naive, + TestConv2DOp, +) def conv2d_forward_refer(input, filter, group, conv_param): @@ -25,10 +28,10 @@ def conv2d_forward_refer(input, filter, group, conv_param): return out -@unittest.skipIf(not core.supports_int8(), - "place does not support int8 computation") +@unittest.skipIf( + not core.supports_int8(), "place does not support int8 computation" +) class TestConv2DInt8Op(TestConv2DOp): - def setUp(self): self.op_type = "conv2d" self.use_cudnn = False @@ -49,85 +52,109 @@ class TestConv2DInt8Op(TestConv2DOp): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } # This implementation of convolution quantization is based on OneDNN documentation # https://oneapi-src.github.io/oneDNN/dev_guide_int8_computations.html#doxid-dev-guide-int8-computations-1dg-i8-comp-s11 - inner_scale = 1. if self.fuse_activation != "" else self.scale_out - activation_scale = self.scale_out if self.fuse_activation != "" else 1. - scale_output_shift = (inner_scale / - (self.scale_in * self.scale_weights[0])) + inner_scale = 1.0 if self.fuse_activation != "" else self.scale_out + activation_scale = self.scale_out if self.fuse_activation != "" else 1.0 + scale_output_shift = inner_scale / ( + self.scale_in * self.scale_weights[0] + ) filter = np.random.random(self.filter_size).astype(self.weighttype) # When the Intel AVX2 or Intel AVX512 Instruction Set is used # the reorder additionally scales the weights by 0.5 # to overcome the potential overflow issue. If the processor supports VNNI instructions, # modification of the weights is not necessary. - avx_scale = 0.5 if not core.supports_vnni( - ) and self.srctype == np.int8 else 1. - filter_int = np.round(filter * self.scale_weights[0] * - avx_scale).astype(np.int32) + avx_scale = ( + 0.5 if not core.supports_vnni() and self.srctype == np.int8 else 1.0 + ) + filter_int = np.round( + filter * self.scale_weights[0] * avx_scale + ).astype(np.int32) scale_output_shift = scale_output_shift / avx_scale def conv2d_forward_refer_helper(input_): - return conv2d_forward_refer(input_.astype(np.int32), filter_int, - self.groups, conv2d_param).astype( - np.float32) * scale_output_shift + return ( + conv2d_forward_refer( + input_.astype(np.int32), + filter_int, + self.groups, + conv2d_param, + ).astype(np.float32) + * scale_output_shift + ) def residual_helper(init_low, init_high, output_): input_residual_ = np.random.randint( - init_low, init_high, - self.input_residual_size).astype(self.srctype) - return (output_ + input_residual_ * - (inner_scale / self.scale_in_eltwise)), input_residual_ + init_low, init_high, self.input_residual_size + ).astype(self.srctype) + return ( + output_ + + input_residual_ * (inner_scale / self.scale_in_eltwise) + ), input_residual_ if self.srctype == np.int8: init_low, init_high = (-5, 5) - input = np.random.randint(init_low, init_high, - self.input_size).astype(self.srctype) + input = np.random.randint( + init_low, init_high, self.input_size + ).astype(self.srctype) input_shift = (np.ones(self.input_size) * 128).astype(np.uint8) output1 = conv2d_forward_refer_helper( - np.round(input + input_shift).astype(np.int32)) + np.round(input + input_shift).astype(np.int32) + ) output2 = conv2d_forward_refer_helper( - np.round(input_shift).astype(np.int32)) + np.round(input_shift).astype(np.int32) + ) output = output1 - output2 else: init_low, init_high = (0, 10) - input = np.random.randint(init_low, init_high, - self.input_size).astype(self.srctype) + input = np.random.randint( + init_low, init_high, self.input_size + ).astype(self.srctype) output = conv2d_forward_refer_helper(input) if self.fuse_residual: - output, input_residual = residual_helper(init_low, init_high, - output) + output, input_residual = residual_helper( + init_low, init_high, output + ) if self.fuse_activation == "": pass elif self.fuse_activation == "relu": output = activation_scale * np.maximum(output, 0) elif self.fuse_activation == "hard_swish": - output = activation_scale * output / 6. * np.minimum( - np.maximum(0, output + 3.), 6) + output = ( + activation_scale + * output + / 6.0 + * np.minimum(np.maximum(0, output + 3.0), 6) + ) elif self.fuse_activation == "relu6": output = activation_scale * np.maximum(0, np.minimum(6, output)) elif self.fuse_activation == "swish": - output = activation_scale * output / (1. + np.exp(-1. * output)) + output = activation_scale * output / (1.0 + np.exp(-1.0 * output)) elif self.fuse_activation == "leaky_relu": output = activation_scale * np.maximum(output, 0.02 * output) else: - raise NotImplementedError("test for " + self.fuse_activation + - " activation not implemented") + raise NotImplementedError( + "test for " + + self.fuse_activation + + " activation not implemented" + ) output = np.round(output).astype(self.dsttype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input.astype(self.srctype)), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } if self.fuse_residual: self.inputs['ResidualData'] = OpTest.np_dtype_to_fluid_dtype( - input_residual) + input_residual + ) self.attrs = { 'strides': self.stride, @@ -146,15 +173,15 @@ class TestConv2DInt8Op(TestConv2DOp): 'fuse_alpha': self.fuse_alpha, 'fuse_beta': self.fuse_beta, 'fuse_residual_connection': self.fuse_residual, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } self.outputs = {'Output': output} def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(core.CPUPlace(), - atol=0, - check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), atol=0, check_dygraph=False + ) def test_check_grad(self): pass @@ -193,7 +220,6 @@ class TestConv2DInt8Op(TestConv2DOp): class TestConv2D(TestConv2DInt8Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -209,7 +235,6 @@ class TestConv2D(TestConv2DInt8Op): class TestWithHardSwish(TestConv2D): - def init_fuse_activation(self): self.fuse_activation = "hard_swish" self.fuse_alpha = 0 @@ -217,7 +242,6 @@ class TestWithHardSwish(TestConv2D): class TestWithRelu6(TestConv2D): - def init_fuse_activation(self): self.fuse_activation = "relu6" self.fuse_alpha = 6 @@ -225,7 +249,6 @@ class TestWithRelu6(TestConv2D): class TestWithSwish(TestConv2D): - def init_fuse_activation(self): self.fuse_activation = "swish" self.fuse_alpha = 1 @@ -233,7 +256,6 @@ class TestWithSwish(TestConv2D): class TestWithLeakyRelu(TestConv2D): - def init_fuse_activation(self): self.fuse_activation = "leaky_relu" self.fuse_alpha = 0.02 @@ -241,7 +263,6 @@ class TestWithLeakyRelu(TestConv2D): class TestWithPad(TestConv2D): - def init_test_case(self): TestConv2D.init_test_case(self) self.pad = [1, 1] @@ -249,13 +270,11 @@ class TestWithPad(TestConv2D): class TestWithGroup(TestConv2D): - def init_group(self): self.groups = 3 class TestWithStride(TestConv2DInt8Op): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -271,7 +290,6 @@ class TestWithStride(TestConv2DInt8Op): class TestWithDilations(TestConv2DInt8Op): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -288,7 +306,6 @@ class TestWithDilations(TestConv2DInt8Op): class TestWith1x1(TestConv2DInt8Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -304,7 +321,6 @@ class TestWith1x1(TestConv2DInt8Op): class TestWithInput1x1Filter1x1(TestConv2DInt8Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -335,37 +351,31 @@ def create_test_int8_class(parent): # --------------------test conv2d s8 in and u8 out-------------------- class TestS8U8Case(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.int8, "relu", False) # --------------------test conv2d s8 in and s8 out-------------------- class TestS8S8Case(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.int8, "", False) # --------------------test conv2d u8 in and s8 out-------------------- class TestU8S8Case(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.uint8, "", False) # --------------------test conv2d u8 in and u8 out without residual fuse-------------------- class TestU8U8Case(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.uint8, "relu", False) # --------------------test conv2d s8 in and s8 out with residual fuse-------------------- class TestS8S8ResCase(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.int8, "", True) # --------------------test conv2d u8 in and s8 out with residual fuse-------------------- class TestU8S8ResCase(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.uint8, "", True) @@ -375,9 +385,11 @@ def create_test_int8_class(parent): cls_name_u8u8 = "{0}_relu_{1}_residual_0".format(parent.__name__, "1") cls_name_s8s8_re_1 = "{0}_relu_{1}_residual_{2}".format( - parent.__name__, "0", "1") + parent.__name__, "0", "1" + ) cls_name_u8s8_re_1 = "{0}_relu_{1}_residual_{2}".format( - parent.__name__, "0", "1") + parent.__name__, "0", "1" + ) TestS8U8Case.__name__ = cls_name_s8u8 TestS8S8Case.__name__ = cls_name_s8s8 TestU8S8Case.__name__ = cls_name_u8s8 @@ -395,12 +407,12 @@ def create_test_int8_class(parent): if os.name != 'nt': # --------------------test conv2d s8 in and u8 out with residual fuse-------------------- class TestS8U8ResCase(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.int8, "relu", True) cls_name_s8u8_re_1 = "{0}_relu_{1}_residual_{2}".format( - parent.__name__, "1", "1") + parent.__name__, "1", "1" + ) TestS8U8ResCase.__name__ = cls_name_s8u8_re_1 globals()[cls_name_s8u8_re_1] = TestS8U8ResCase @@ -415,7 +427,6 @@ create_test_int8_class(TestWithInput1x1Filter1x1) class TestConv2DOp_AsyPadding_INT_MKLDNN(TestConv2DInt8Op): - def init_kernel_type(self): self.use_mkldnn = True @@ -425,14 +436,12 @@ class TestConv2DOp_AsyPadding_INT_MKLDNN(TestConv2DInt8Op): class TestConv2DOp_Same_INT_MKLDNN(TestConv2DOp_AsyPadding_INT_MKLDNN): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" class TestConv2DOp_Valid_INT_MKLDNN(TestConv2DOp_AsyPadding_INT_MKLDNN): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -440,5 +449,6 @@ class TestConv2DOp_Valid_INT_MKLDNN(TestConv2DOp_AsyPadding_INT_MKLDNN): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_mkldnn_op.py index d3e9bdde1a6..65147ea6c2b 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_mkldnn_op.py @@ -16,7 +16,10 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci -from paddle.fluid.tests.unittests.test_conv2d_op import TestConv2DOp, TestConv2DOp_v2 +from paddle.fluid.tests.unittests.test_conv2d_op import ( + TestConv2DOp, + TestConv2DOp_v2, +) def conv2d_bias_naive(out, bias): @@ -34,7 +37,6 @@ def conv2d_residual_naive(out, residual): class TestConv2DMKLDNNOp(TestConv2DOp): - def init_group(self): self.groups = 1 @@ -65,7 +67,7 @@ class TestConv2DMKLDNNOp(TestConv2DOp): output = self.outputs['Output'] - #mkldnn only support either conv-sum-relu, or conv-relu. + # mkldnn only support either conv-sum-relu, or conv-relu. if self.fuse_bias and self.bias_size is not None: bias = np.random.random(self.bias_size).astype(self.dtype) output = conv2d_bias_naive(output, bias) @@ -73,22 +75,29 @@ class TestConv2DMKLDNNOp(TestConv2DOp): self.attrs['fuse_bias'] = self.fuse_bias self.inputs['Bias'] = OpTest.np_dtype_to_fluid_dtype(bias) - if self.fuse_residual_connection and self.input_residual_size is not None: + if ( + self.fuse_residual_connection + and self.input_residual_size is not None + ): input_residual = np.random.random(self.input_residual_size).astype( - self.dtype) + self.dtype + ) output = conv2d_residual_naive(output, input_residual) self.attrs[ - 'fuse_residual_connection'] = self.fuse_residual_connection + 'fuse_residual_connection' + ] = self.fuse_residual_connection self.inputs['ResidualData'] = OpTest.np_dtype_to_fluid_dtype( - input_residual) + input_residual + ) if self.fuse_activation == "relu": output = np.maximum(output, 0).astype(self.dsttype) if self.fuse_activation == "relu6": - output = np.minimum(np.maximum(output, 0), - self.fuse_alpha).astype(self.dsttype) + output = np.minimum(np.maximum(output, 0), self.fuse_alpha).astype( + self.dsttype + ) output = output.astype(self.dtype) self.attrs['fuse_bias'] = self.fuse_bias @@ -101,9 +110,9 @@ class TestConv2DMKLDNNOp(TestConv2DOp): @skip_check_grad_ci( - reason="Fusion is for inference only, check_grad is not required.") + reason="Fusion is for inference only, check_grad is not required." +) class TestWithbreluFusion(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.fuse_activation = "relu6" @@ -112,9 +121,9 @@ class TestWithbreluFusion(TestConv2DMKLDNNOp): @skip_check_grad_ci( - reason="Fusion is for inference only, check_grad is not required.") + reason="Fusion is for inference only, check_grad is not required." +) class TestWithFuse(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -125,7 +134,6 @@ class TestWithFuse(TestConv2DMKLDNNOp): class TestWithPadWithBias(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -133,7 +141,6 @@ class TestWithPadWithBias(TestConv2DMKLDNNOp): class TestWithStride(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -142,7 +149,6 @@ class TestWithStride(TestConv2DMKLDNNOp): class TestWithGroup(TestConv2DMKLDNNOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -156,14 +162,12 @@ class TestWithGroup(TestConv2DMKLDNNOp): class TestWith1x1(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.filter_size = [40, 3, 1, 1] class TestWithInput1x1Filter1x1(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.input_size = [2, 60, 1, 1] # NCHW @@ -176,7 +180,6 @@ class TestWithInput1x1Filter1x1(TestConv2DMKLDNNOp): class TestConv2DOp_AsyPadding_MKLDNN(TestConv2DOp_v2): - def init_kernel_type(self): self.use_mkldnn = True self.dtype = np.float32 @@ -187,21 +190,18 @@ class TestConv2DOp_AsyPadding_MKLDNN(TestConv2DOp_v2): class TestConv2DOp_Same_MKLDNN(TestConv2DOp_AsyPadding_MKLDNN): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" class TestConv2DOp_Valid_MKLDNN(TestConv2DOp_AsyPadding_MKLDNN): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" class TestConv2DOp_Valid_NHWC_MKLDNN(TestConv2DOp_Valid_MKLDNN): - def init_data_format(self): self.data_format = "NHWC" @@ -211,21 +211,18 @@ class TestConv2DOp_Valid_NHWC_MKLDNN(TestConv2DOp_Valid_MKLDNN): class TestConv2DOp_Same_NHWC_MKLDNN(TestConv2DOp_Valid_NHWC_MKLDNN): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" class TestConv2DOp_AsyPadding_NHWC_MKLDNN(TestConv2DOp_Valid_NHWC_MKLDNN): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestMKLDNNDilations(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.pad = [0, 0] @@ -244,5 +241,6 @@ class TestMKLDNNDilations(TestConv2DMKLDNNOp): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_bf16_mkldnn_op.py index e9f4f5a3251..e605dc6bf95 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_bf16_mkldnn_op.py @@ -17,7 +17,9 @@ import numpy as np import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -from paddle.fluid.tests.unittests.test_conv2d_transpose_op import conv2dtranspose_forward_naive +from paddle.fluid.tests.unittests.test_conv2d_transpose_op import ( + conv2dtranspose_forward_naive, +) from paddle import enable_static @@ -29,10 +31,10 @@ def conv2d_bias_naive(out, bias): return out -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestConv2DTransposeBF16MKLDNNOp(OpTest): - def test_check_output(self): self.check_output_with_place(core.CPUPlace()) @@ -95,7 +97,7 @@ class TestConv2DTransposeBF16MKLDNNOp(OpTest): 'data_format': self.data_format, 'fuse_activation': self.fuse_activation, 'fuse_alpha': self.fuse_alpha, - 'fuse_beta': self.fuse_beta + 'fuse_beta': self.fuse_beta, } if self.output_size is not None: self.attrs['output_size'] = self.output_size @@ -103,15 +105,16 @@ class TestConv2DTransposeBF16MKLDNNOp(OpTest): if len(self.output_padding) > 0: self.attrs['output_padding'] = self.output_padding - output = conv2dtranspose_forward_naive(input, filter, - self.attrs).astype(np.float32) + output = conv2dtranspose_forward_naive( + input, filter, self.attrs + ).astype(np.float32) if self.input_type is not np.float32: input = convert_float_to_uint16(input) self.inputs = { 'Input': input.view(self.input_type), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } if self.fuse_bias and self.bias_size is not None: @@ -132,7 +135,6 @@ class TestConv2DTransposeBF16MKLDNNOp(OpTest): class TestMKLDNNFuseBias(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNFuseBias, self).init_test_case() self.pad = [1, 1] @@ -141,7 +143,6 @@ class TestMKLDNNFuseBias(TestConv2DTransposeBF16MKLDNNOp): class TestMKLDNNWithPad(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithPad, self).init_test_case() self.pad = [1, 1] @@ -149,7 +150,6 @@ class TestMKLDNNWithPad(TestConv2DTransposeBF16MKLDNNOp): class TestMKLDNNWithStride(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithStride, self).init_test_case() self.pad = [1, 1] @@ -158,7 +158,6 @@ class TestMKLDNNWithStride(TestConv2DTransposeBF16MKLDNNOp): class TestMKLDNNWithAsymPad(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithAsymPad, self).init_test_case() self.pad = [0, 0, 1, 2] @@ -166,7 +165,6 @@ class TestMKLDNNWithAsymPad(TestConv2DTransposeBF16MKLDNNOp): class TestMKLDNNWithSamePad(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithSamePad, self).init_test_case() self.pad = [0, 0] @@ -174,7 +172,6 @@ class TestMKLDNNWithSamePad(TestConv2DTransposeBF16MKLDNNOp): class TestMKLDNNWithValidPad(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithValidPad, self).init_test_case() self.pad = [1, 1] @@ -182,7 +179,6 @@ class TestMKLDNNWithValidPad(TestConv2DTransposeBF16MKLDNNOp): class TestMKLDNNWithValidPad_NHWC(TestMKLDNNWithValidPad): - def init_test_case(self): super(TestMKLDNNWithValidPad_NHWC, self).init_test_case() self.data_format = 'NHWC' @@ -191,11 +187,12 @@ class TestMKLDNNWithValidPad_NHWC(TestMKLDNNWithValidPad): class TestConv2DTransposeMKLDNNWithDilationsExplicitPad( - TestConv2DTransposeBF16MKLDNNOp): - + TestConv2DTransposeBF16MKLDNNOp +): def init_test_case(self): - super(TestConv2DTransposeMKLDNNWithDilationsExplicitPad, - self).init_test_case() + super( + TestConv2DTransposeMKLDNNWithDilationsExplicitPad, self + ).init_test_case() self.stride = [2, 1] self.dilations = [1, 2] self.groups = 1 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_mkldnn_op.py index 081f70a02e3..14e42fc50ce 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_mkldnn_op.py @@ -16,7 +16,9 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest from paddle import enable_static -from paddle.fluid.tests.unittests.test_conv2d_transpose_op import TestConv2DTransposeOp +from paddle.fluid.tests.unittests.test_conv2d_transpose_op import ( + TestConv2DTransposeOp, +) def conv2d_bias_naive(out, bias): @@ -28,7 +30,6 @@ def conv2d_bias_naive(out, bias): class TestConv2DTransposeMKLDNNOp(TestConv2DTransposeOp): - def test_check_grad(self): return @@ -87,7 +88,6 @@ class TestConv2DTransposeMKLDNNOp(TestConv2DTransposeOp): class TestMKLDNNFuseBias(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -96,7 +96,6 @@ class TestMKLDNNFuseBias(TestConv2DTransposeMKLDNNOp): class TestMKLDNNWithPad(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -104,7 +103,6 @@ class TestMKLDNNWithPad(TestConv2DTransposeMKLDNNOp): class TestMKLDNNWithStride(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -113,7 +111,6 @@ class TestMKLDNNWithStride(TestConv2DTransposeMKLDNNOp): class TestMKLDNNWithAsymPad(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [0, 0, 1, 2] @@ -121,7 +118,6 @@ class TestMKLDNNWithAsymPad(TestConv2DTransposeMKLDNNOp): class TestMKLDNNWithSamePad(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [0, 0] @@ -129,7 +125,6 @@ class TestMKLDNNWithSamePad(TestConv2DTransposeMKLDNNOp): class TestMKLDNNWithValidPad(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -137,7 +132,6 @@ class TestMKLDNNWithValidPad(TestConv2DTransposeMKLDNNOp): class TestMKLDNNWithValidPad_NHWC(TestMKLDNNWithValidPad): - def init_test_case(self): super(TestMKLDNNWithValidPad_NHWC, self).init_test_case() self.data_format = "NHWC" @@ -146,8 +140,8 @@ class TestMKLDNNWithValidPad_NHWC(TestMKLDNNWithValidPad): class TestConv2DTransposeMKLDNNWithDilationsExplicitPad( - TestConv2DTransposeMKLDNNOp): - + TestConv2DTransposeMKLDNNOp +): def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.stride = [2, 1] @@ -161,7 +155,6 @@ class TestConv2DTransposeMKLDNNWithDilationsExplicitPad( class TestMKLDNNWithGroups(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -172,7 +165,6 @@ class TestMKLDNNWithGroups(TestConv2DTransposeMKLDNNOp): class TestMKLDNNWithGroups_NHWC(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv3d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv3d_mkldnn_op.py index 6e31912a909..30561ec8f0f 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv3d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv3d_mkldnn_op.py @@ -14,11 +14,17 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.test_conv3d_op import TestCase1, TestConv3DOp, TestWith1x1, TestWithGroup1, TestWithGroup2, TestWithInput1x1Filter1x1 +from paddle.fluid.tests.unittests.test_conv3d_op import ( + TestCase1, + TestConv3DOp, + TestWith1x1, + TestWithGroup1, + TestWithGroup2, + TestWithInput1x1Filter1x1, +) class TestMKLDNN(TestConv3DOp): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -26,7 +32,6 @@ class TestMKLDNN(TestConv3DOp): class TestMKLDNNCase1(TestCase1): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -34,7 +39,6 @@ class TestMKLDNNCase1(TestCase1): class TestMKLDNNGroup1(TestWithGroup1): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -42,7 +46,6 @@ class TestMKLDNNGroup1(TestWithGroup1): class TestMKLDNNGroup2(TestWithGroup2): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -50,7 +53,6 @@ class TestMKLDNNGroup2(TestWithGroup2): class TestMKLDNNWith1x1(TestWith1x1): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -58,7 +60,6 @@ class TestMKLDNNWith1x1(TestWith1x1): class TestMKLDNNWithInput1x1Filter1x1(TestWithInput1x1Filter1x1): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -66,7 +67,6 @@ class TestMKLDNNWithInput1x1Filter1x1(TestWithInput1x1Filter1x1): class TestConv3DOp_AsyPadding_MKLDNN(TestConv3DOp): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -78,7 +78,6 @@ class TestConv3DOp_AsyPadding_MKLDNN(TestConv3DOp): class TestConv3DOp_Same_MKLDNN(TestConv3DOp_AsyPadding_MKLDNN): - def init_paddings(self): self.pad = [0, 0, 0] self.padding_algorithm = "SAME" @@ -90,7 +89,6 @@ class TestConv3DOp_Same_MKLDNN(TestConv3DOp_AsyPadding_MKLDNN): class TestConv3DOp_Valid_MKLDNN(TestConv3DOp_AsyPadding_MKLDNN): - def init_paddings(self): self.pad = [1, 1, 1] self.padding_algorithm = "VALID" @@ -103,5 +101,6 @@ class TestConv3DOp_Valid_MKLDNN(TestConv3DOp_AsyPadding_MKLDNN): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_dequantize_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_dequantize_mkldnn_op.py index 70a1fd82394..2e3a4609599 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_dequantize_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_dequantize_mkldnn_op.py @@ -19,7 +19,6 @@ import paddle class TestDeQuantizeOp(OpTest): - def setUp(self): self.op_type = 'dequantize' self.scale = 127.0 @@ -45,19 +44,22 @@ class TestDeQuantizeOp(OpTest): def prepare_input_int8(self): if self.data_type == 'int8': # input data values are integers from interval [-128, 128) - self.input = (np.random.randint(0, 256, self.input_size) - - 128).astype(self.data_type) + self.input = ( + np.random.randint(0, 256, self.input_size) - 128 + ).astype(self.data_type) else: # input data values are integers from interval [0, 256) self.input = (np.random.randint(0, 256, self.input_size)).astype( - self.data_type) + self.data_type + ) self.inputs = {'Input': OpTest.np_dtype_to_fluid_dtype(self.input)} self.attrs = {'Scale': self.scale, 'Shift': self.shift} def prepare_output_int8(self): - output = (self.input / self.scale - - (self.shift / self.scale)).astype('float') + output = (self.input / self.scale - (self.shift / self.scale)).astype( + 'float' + ) self.outputs = {'Output': output} def test_check_output(self): @@ -87,7 +89,6 @@ class TestDeQuantizeOp(OpTest): class TestDeQuantizeOp1(TestDeQuantizeOp): - def set_scale(self): self.scale = 1.5 @@ -96,7 +97,6 @@ class TestDeQuantizeOp1(TestDeQuantizeOp): class TestDeQuantizeOp2(TestDeQuantizeOp): - def set_scale(self): self.scale = 0.8 @@ -105,7 +105,6 @@ class TestDeQuantizeOp2(TestDeQuantizeOp): class TestDeQuantizeOpBf16(TestDeQuantizeOp): - def set_scale(self): self.scale = 1.0 @@ -116,7 +115,6 @@ class TestDeQuantizeOpBf16(TestDeQuantizeOp): # 2-dim input # P - positive input, with shift class TestDeQuantizeOpShift_2_P(TestDeQuantizeOp): - def set_data_type(self): self.data_type = 'uint8' @@ -133,7 +131,6 @@ class TestDeQuantizeOpShift_2_P(TestDeQuantizeOp): # 2-dim input # N - negative input, with shift class TestDeQuantizeOpShift_2_N(TestDeQuantizeOpShift_2_P): - def set_data_type(self): self.data_type = 'int8' @@ -149,26 +146,22 @@ class TestDeQuantizeOpShift_2_N(TestDeQuantizeOpShift_2_P): # 3-dim input class TestDeQuantizeOpShift_3_P(TestDeQuantizeOpShift_2_P): - def set_input_size(self): self.input_size = [2, 3, 4] class TestDeQuantizeOpShift_3_N(TestDeQuantizeOpShift_2_N): - def set_input_size(self): self.input_size = [2, 3, 4] # 4-dim input class TestDeQuantizeOpShift_4_P(TestDeQuantizeOpShift_2_P): - def set_input_size(self): self.input_size = [2, 3, 4, 5] class TestDeQuantizeOpShift_4_N(TestDeQuantizeOpShift_2_N): - def set_input_size(self): self.input_size = [2, 3, 4, 5] diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_bf16_mkldnn_op.py index 6511b1a1538..8e2ae04bb7c 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_bf16_mkldnn_op.py @@ -19,10 +19,10 @@ from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestElementwiseAddBf16MklDNNOp(OpTest): - def setUp(self): self.op_type = "elementwise_add" self.use_mkldnn = True @@ -38,8 +38,12 @@ class TestElementwiseAddBf16MklDNNOp(OpTest): self.outputs = {'Out': convert_float_to_uint16(self.out)} def generate_data(self): - self.x = np.random.random(100, ).astype(np.float32) - self.y = np.random.random(100, ).astype(np.float32) + self.x = np.random.random( + 100, + ).astype(np.float32) + self.y = np.random.random( + 100, + ).astype(np.float32) self.out = np.add(self.x, self.y) def test_check_output(self): @@ -47,30 +51,39 @@ class TestElementwiseAddBf16MklDNNOp(OpTest): # elementwise_add grad (no braodcasting) is just passing upper gradients to either X or Y or both def test_check_grad_normal(self): - self.check_grad_with_place(core.CPUPlace(), ["X", "Y"], - "Out", - check_dygraph=False, - user_defined_grads=[self.x, self.x], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X", "Y"], + "Out", + check_dygraph=False, + user_defined_grads=[self.x, self.x], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ingore_x(self): - self.check_grad_with_place(core.CPUPlace(), ["Y"], - "Out", - check_dygraph=False, - user_defined_grads=[self.y], - user_defined_grad_outputs=[self.y_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["Y"], + "Out", + check_dygraph=False, + user_defined_grads=[self.y], + user_defined_grad_outputs=[self.y_bf16], + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - check_dygraph=False, - user_defined_grads=[self.x], - user_defined_grad_outputs=[self.x_bf16]) - + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + check_dygraph=False, + user_defined_grads=[self.x], + user_defined_grad_outputs=[self.x_bf16], + ) -class TestElementwiseAddBroadCastingBf16MklDNNOp(TestElementwiseAddBf16MklDNNOp - ): +class TestElementwiseAddBroadCastingBf16MklDNNOp( + TestElementwiseAddBf16MklDNNOp +): def generate_data(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(np.float32) self.y = np.random.uniform(1, 2, [100]).astype(np.float32) @@ -85,20 +98,23 @@ class TestElementwiseAddBroadCastingBf16MklDNNOp(TestElementwiseAddBf16MklDNNOp def test_check_grad_normal(self): self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], + core.CPUPlace(), + ["X", "Y"], "Out", check_dygraph=False, - user_defined_grads=[self.x, - self.compute_reduced_gradients(self.x)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grads=[self.x, self.compute_reduced_gradients(self.x)], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ingore_x(self): self.check_grad_with_place( - core.CPUPlace(), ["Y"], + core.CPUPlace(), + ["Y"], "Out", check_dygraph=False, user_defined_grads=[self.compute_reduced_gradients(self.x)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grad_outputs=[self.x_bf16], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py index 2b89d3c4b10..4001e2ba76b 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py @@ -15,12 +15,13 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -from paddle.fluid.tests.unittests.test_elementwise_add_op import TestElementwiseAddOp +from paddle.fluid.tests.unittests.test_elementwise_add_op import ( + TestElementwiseAddOp, +) from paddle import enable_static class TestMKLDNNElementwiseAddOp(TestElementwiseAddOp): - def init_kernel_type(self): self.use_mkldnn = True @@ -29,15 +30,13 @@ class TestMKLDNNElementwiseAddOp(TestElementwiseAddOp): class TestMKLDNNElementwiseAddOp2(TestMKLDNNElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestMKLDNNElementwiseAddOp3(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) @@ -45,7 +44,6 @@ class TestMKLDNNElementwiseAddOp3(TestMKLDNNElementwiseAddOp): class TestMKLDNNElementwiseAddOp4(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype) self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype) @@ -60,7 +58,6 @@ class TestMKLDNNElementwiseAddOp4(TestMKLDNNElementwiseAddOp): class TestMKLDNNElementwiseAddOp5(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -68,7 +65,6 @@ class TestMKLDNNElementwiseAddOp5(TestMKLDNNElementwiseAddOp): class TestMKLDNNElementwiseAddOpBroadcastXintoY(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 50, 1]).astype(self.dtype) self.y = np.random.uniform(1, 2, [2, 50, 160]).astype(self.dtype) @@ -76,7 +72,6 @@ class TestMKLDNNElementwiseAddOpBroadcastXintoY(TestMKLDNNElementwiseAddOp): class TestMKLDNNElementwiseAddOp_broadcast_3(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -87,7 +82,6 @@ class TestMKLDNNElementwiseAddOp_broadcast_3(TestMKLDNNElementwiseAddOp): class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -111,9 +105,9 @@ class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestMKLDNNElementwiseAddOp): @skip_check_grad_ci( - reason="oneDNN's int8 elementwise_ops don't implemend grad kernel.") + reason="oneDNN's int8 elementwise_ops don't implemend grad kernel." +) class TestInt8(TestElementwiseAddOp): - def init_kernel_type(self): self.use_mkldnn = True self._cpu_only = True @@ -147,7 +141,6 @@ class TestInt8(TestElementwiseAddOp): class TestInt8Scales(TestInt8): - def quantize(self, tensor, dt="int8"): max_int = 127.0 if dt == "int8" else 255.0 scale = max_int / np.abs(np.amax(tensor)) @@ -155,8 +148,8 @@ class TestInt8Scales(TestInt8): return scale, quantized def init_input_output(self): - self.x_f = np.random.random((100, )).astype("float") - self.y_f = np.random.random((100, )).astype("float") + self.x_f = np.random.random((100,)).astype("float") + self.y_f = np.random.random((100,)).astype("float") self.out_f = np.add(self.x_f, self.y_f) self.scale_x, self.x = self.quantize(self.x_f) @@ -172,15 +165,15 @@ class TestInt8Scales(TestInt8): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.init_scales() int_atol = 1 # different quantization techniques - self.check_output(check_dygraph=(self.use_mkldnn == False), - atol=int_atol) + self.check_output( + check_dygraph=(self.use_mkldnn == False), atol=int_atol + ) class TestUint8Scales(TestInt8Scales): - def init_input_output(self): - self.x_f = np.random.random((100, )).astype("float") - self.y_f = np.random.random((100, )).astype("float") + self.x_f = np.random.random((100,)).astype("float") + self.y_f = np.random.random((100,)).astype("float") self.out_f = np.add(self.x_f, self.y_f) self.scale_x, self.x = self.quantize(self.x_f, "uint8") diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_div_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_div_mkldnn_op.py index 9312e945903..8adeb32dc9f 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_div_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_div_mkldnn_op.py @@ -15,15 +15,20 @@ import unittest import numpy as np from paddle import enable_static -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) from paddle.fluid.framework import _current_expected_place import paddle.fluid.core as core -@OpTestTool.skip_if(not (isinstance(_current_expected_place(), core.CPUPlace)), - "GPU is not supported") +@OpTestTool.skip_if( + not (isinstance(_current_expected_place(), core.CPUPlace)), + "GPU is not supported", +) class TestMKLDNNElementwiseDivOp(OpTest): - def setUp(self): self.op_type = "elementwise_div" self.init_dtype() @@ -32,7 +37,7 @@ class TestMKLDNNElementwiseDivOp(OpTest): self.init_axis() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -65,7 +70,6 @@ class TestMKLDNNElementwiseDivOp(OpTest): class TestMKLDNNElementwiseDivOp2(TestMKLDNNElementwiseDivOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [100]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [100]).astype(self.dtype) @@ -73,7 +77,6 @@ class TestMKLDNNElementwiseDivOp2(TestMKLDNNElementwiseDivOp): class TestMKLDNNElementwiseDivOp3(TestMKLDNNElementwiseDivOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) @@ -81,7 +84,6 @@ class TestMKLDNNElementwiseDivOp3(TestMKLDNNElementwiseDivOp): class TestMKLDNNElementwiseDivOp4(TestMKLDNNElementwiseDivOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype) self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype) @@ -96,7 +98,6 @@ class TestMKLDNNElementwiseDivOp4(TestMKLDNNElementwiseDivOp): class TestMKLDNNElementwiseDivOp5(TestMKLDNNElementwiseDivOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -112,7 +113,6 @@ class TestMKLDNNElementwiseDivOp5(TestMKLDNNElementwiseDivOp): @OpTestTool.skip_if_not_cpu_bf16() class TestBf16(TestMKLDNNElementwiseDivOp): - def setUp(self): self.op_type = "elementwise_div" self.init_dtype() @@ -139,34 +139,43 @@ class TestBf16(TestMKLDNNElementwiseDivOp): self.check_output_with_place(core.CPUPlace()) def test_check_grad_normal(self): - self.check_grad_with_place(core.CPUPlace(), ["X", "Y"], - "Out", - user_defined_grads=[ - np.divide(self.x, self.y), - np.divide((np.multiply(-self.x, self.x)), - np.multiply(self.y, self.y)) - ], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X", "Y"], + "Out", + user_defined_grads=[ + np.divide(self.x, self.y), + np.divide( + (np.multiply(-self.x, self.x)), np.multiply(self.y, self.y) + ), + ], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ignore_x(self): - self.check_grad_with_place(core.CPUPlace(), ["Y"], - "Out", - user_defined_grads=[ - np.divide((np.multiply(-self.x, self.y)), - np.multiply(self.y, self.y)) - ], - user_defined_grad_outputs=[self.y_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["Y"], + "Out", + user_defined_grads=[ + np.divide( + (np.multiply(-self.x, self.y)), np.multiply(self.y, self.y) + ) + ], + user_defined_grad_outputs=[self.y_bf16], + ) def test_check_grad_ignore_y(self): self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[np.divide(self.x, self.y)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grad_outputs=[self.x_bf16], + ) class TestBf16Broadcasting(TestBf16): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py index 26531d826b5..d7e31b06155 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py @@ -19,10 +19,10 @@ from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestElementwiseMulBf16MklDNNOp(OpTest): - def setUp(self): self.op_type = "elementwise_mul" self.use_mkldnn = True @@ -37,43 +37,54 @@ class TestElementwiseMulBf16MklDNNOp(OpTest): self.outputs = {'Out': convert_float_to_uint16(self.out)} def generate_data(self): - self.x = np.random.random(100, ).astype(np.float32) - self.y = np.random.random(100, ).astype(np.float32) + self.x = np.random.random( + 100, + ).astype(np.float32) + self.y = np.random.random( + 100, + ).astype(np.float32) self.out = np.multiply(self.x, self.y) def test_check_output(self): self.check_output_with_place(core.CPUPlace()) def test_check_grad_normal(self): - self.check_grad_with_place(core.CPUPlace(), ["X", "Y"], - "Out", - check_dygraph=False, - user_defined_grads=[ - np.multiply(self.x, self.y), - np.multiply(self.x, self.x) - ], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X", "Y"], + "Out", + check_dygraph=False, + user_defined_grads=[ + np.multiply(self.x, self.y), + np.multiply(self.x, self.x), + ], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ingore_x(self): self.check_grad_with_place( - core.CPUPlace(), ["Y"], + core.CPUPlace(), + ["Y"], "Out", check_dygraph=False, user_defined_grads=[np.multiply(self.y, self.x)], - user_defined_grad_outputs=[self.y_bf16]) + user_defined_grad_outputs=[self.y_bf16], + ) def test_check_grad_ingore_y(self): self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", check_dygraph=False, user_defined_grads=[np.multiply(self.x, self.y)], - user_defined_grad_outputs=[self.x_bf16]) - + user_defined_grad_outputs=[self.x_bf16], + ) -class TestElementwiseMulBroadcastingBf16MklDNNOp(TestElementwiseMulBf16MklDNNOp - ): +class TestElementwiseMulBroadcastingBf16MklDNNOp( + TestElementwiseMulBf16MklDNNOp +): def generate_data(self): self.x = np.random.uniform(1, 2, [1, 2, 3, 100]).astype(np.float32) self.y = np.random.uniform(1, 2, [100]).astype(np.float32) @@ -90,7 +101,7 @@ class TestElementwiseMulBroadcastingBf16MklDNNOp(TestElementwiseMulBf16MklDNNOp # accuracy problems that need to be explained def test_check_grad_normal(self): pass - #self.check_grad_with_place( + # self.check_grad_with_place( # core.CPUPlace(), ["X", "Y"], # "Out", # check_dy_graph=False, @@ -102,7 +113,7 @@ class TestElementwiseMulBroadcastingBf16MklDNNOp(TestElementwiseMulBf16MklDNNOp def test_check_grad_ingore_x(self): pass - #self.check_grad_with_place( + # self.check_grad_with_place( # core.CPUPlace(), ["Y"], # "Out", # check_dy_graph=False, diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py index 7f3bd429eb3..3ca09093b81 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py @@ -15,12 +15,13 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -from paddle.fluid.tests.unittests.test_elementwise_mul_op import ElementwiseMulOp +from paddle.fluid.tests.unittests.test_elementwise_mul_op import ( + ElementwiseMulOp, +) from paddle import enable_static class TestMKLDNNElementwiseMulOp(ElementwiseMulOp): - def init_kernel_type(self): self.use_mkldnn = True @@ -29,15 +30,13 @@ class TestMKLDNNElementwiseMulOp(ElementwiseMulOp): class TestMKLDNNElementwiseMulOp2(TestMKLDNNElementwiseMulOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.multiply(self.x, self.y) class TestMKLDNNElementwiseMulOp3(TestMKLDNNElementwiseMulOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) @@ -45,7 +44,6 @@ class TestMKLDNNElementwiseMulOp3(TestMKLDNNElementwiseMulOp): class TestMKLDNNElementwiseMulOp4(TestMKLDNNElementwiseMulOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype) self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype) @@ -60,7 +58,6 @@ class TestMKLDNNElementwiseMulOp4(TestMKLDNNElementwiseMulOp): class TestMKLDNNElementwiseMulOp5(TestMKLDNNElementwiseMulOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -81,9 +78,9 @@ class TestMKLDNNElementwiseMulOp5(TestMKLDNNElementwiseMulOp): @skip_check_grad_ci( - reason="oneDNN's int8 elementwise_ops don't implemend grad kernel.") + reason="oneDNN's int8 elementwise_ops don't implemend grad kernel." +) class TestInt8(ElementwiseMulOp): - def init_kernel_type(self): self.use_mkldnn = True self._cpu_only = True @@ -117,7 +114,6 @@ class TestInt8(ElementwiseMulOp): class TestInt8Scales(TestInt8): - def quantize(self, tensor, dt="int8"): max_int = 127.0 if dt == "int8" else 255.0 scale = max_int / np.abs(np.amax(tensor)) @@ -125,8 +121,8 @@ class TestInt8Scales(TestInt8): return scale, quantized def init_input_output(self): - self.x_f = np.random.random((100, )).astype("float") - self.y_f = np.random.random((100, )).astype("float") + self.x_f = np.random.random((100,)).astype("float") + self.y_f = np.random.random((100,)).astype("float") self.out_f = np.multiply(self.x_f, self.y_f) self.scale_x, self.x = self.quantize(self.x_f) @@ -142,15 +138,15 @@ class TestInt8Scales(TestInt8): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.init_scales() int_atol = 1 # different quantization techniques - self.check_output(check_dygraph=(self.use_mkldnn == False), - atol=int_atol) + self.check_output( + check_dygraph=(self.use_mkldnn == False), atol=int_atol + ) class TestUint8Scales(TestInt8Scales): - def init_input_output(self): - self.x_f = np.random.random((100, )).astype("float") - self.y_f = np.random.random((100, )).astype("float") + self.x_f = np.random.random((100,)).astype("float") + self.y_f = np.random.random((100,)).astype("float") self.out_f = np.multiply(self.x_f, self.y_f) self.scale_x, self.x = self.quantize(self.x_f, "uint8") diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_sub_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_sub_mkldnn_op.py index b91e81550cd..6d1f12d1414 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_sub_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_sub_mkldnn_op.py @@ -15,15 +15,20 @@ import unittest import numpy as np from paddle import enable_static -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) from paddle.fluid.framework import _current_expected_place import paddle.fluid.core as core -@OpTestTool.skip_if(not (isinstance(_current_expected_place(), core.CPUPlace)), - "GPU is not supported") +@OpTestTool.skip_if( + not (isinstance(_current_expected_place(), core.CPUPlace)), + "GPU is not supported", +) class TestMKLDNNElementwiseSubOp(OpTest): - def setUp(self): self.op_type = "elementwise_sub" self.init_dtype() @@ -32,7 +37,7 @@ class TestMKLDNNElementwiseSubOp(OpTest): self.init_axis() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -65,15 +70,13 @@ class TestMKLDNNElementwiseSubOp(OpTest): class TestMKLDNNElementwiseSubOp2(TestMKLDNNElementwiseSubOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.subtract(self.x, self.y) class TestMKLDNNElementwiseSubOp3(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) @@ -81,7 +84,6 @@ class TestMKLDNNElementwiseSubOp3(TestMKLDNNElementwiseSubOp): class TestMKLDNNElementwiseSubOp4(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype) self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype) @@ -89,7 +91,6 @@ class TestMKLDNNElementwiseSubOp4(TestMKLDNNElementwiseSubOp): class TestMKLDNNElementwiseSubOp40(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 2, [180, 1]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [1, 256]).astype(self.dtype) @@ -106,7 +107,6 @@ class TestMKLDNNElementwiseSubOp40(TestMKLDNNElementwiseSubOp): class TestMKLDNNElementwiseSubOp5(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -114,7 +114,6 @@ class TestMKLDNNElementwiseSubOp5(TestMKLDNNElementwiseSubOp): class TestMKLDNNElementwiseSubOp_broadcast(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -125,7 +124,6 @@ class TestMKLDNNElementwiseSubOp_broadcast(TestMKLDNNElementwiseSubOp): class TestElementwiseSubOp_xsize_lessthan_ysize_sub(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -146,7 +144,6 @@ class TestElementwiseSubOp_xsize_lessthan_ysize_sub(TestMKLDNNElementwiseSubOp): @OpTestTool.skip_if_not_cpu_bf16() class TestBf16(TestMKLDNNElementwiseSubOp): - def setUp(self): self.op_type = "elementwise_sub" self.init_dtype() @@ -165,34 +162,46 @@ class TestBf16(TestMKLDNNElementwiseSubOp): self.mkldnn_data_type = "bfloat16" def init_input_output(self): - self.x = np.random.random(100, ).astype(self.dtype) - self.y = np.random.random(100, ).astype(self.dtype) + self.x = np.random.random( + 100, + ).astype(self.dtype) + self.y = np.random.random( + 100, + ).astype(self.dtype) self.out = np.subtract(self.x, self.y) def test_check_output(self): self.check_output_with_place(core.CPUPlace()) def test_check_grad_normal(self): - self.check_grad_with_place(core.CPUPlace(), ["X", "Y"], - "Out", - user_defined_grads=[self.x, -self.x], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X", "Y"], + "Out", + user_defined_grads=[self.x, -self.x], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ignore_x(self): - self.check_grad_with_place(core.CPUPlace(), ["Y"], - "Out", - user_defined_grads=[-self.y], - user_defined_grad_outputs=[self.y_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["Y"], + "Out", + user_defined_grads=[-self.y], + user_defined_grad_outputs=[self.y_bf16], + ) def test_check_grad_ignore_y(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - user_defined_grads=[self.x], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + user_defined_grads=[self.x], + user_defined_grad_outputs=[self.x_bf16], + ) class TestBf16Broadcasting(TestBf16): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -206,22 +215,24 @@ class TestBf16Broadcasting(TestBf16): def test_check_grad_normal(self): self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], + core.CPUPlace(), + ["X", "Y"], "Out", - user_defined_grads=[self.x, - self.compute_reduced_gradients(self.x)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grads=[self.x, self.compute_reduced_gradients(self.x)], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ignore_x(self): self.check_grad_with_place( - core.CPUPlace(), ["Y"], + core.CPUPlace(), + ["Y"], "Out", user_defined_grads=[self.compute_reduced_gradients(self.x)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grad_outputs=[self.x_bf16], + ) class TestInt8(TestMKLDNNElementwiseSubOp): - def init_kernel_type(self): self.use_mkldnn = True self._cpu_only = True diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py index cf98324809d..f156cceb823 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py @@ -16,13 +16,18 @@ import unittest import numpy as np import paddle from paddle.fluid import core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) -@OpTestTool.skip_if(core.is_compiled_with_cuda(), - "CUDA required dygraph so oneDNN UT must be skipped") +@OpTestTool.skip_if( + core.is_compiled_with_cuda(), + "CUDA required dygraph so oneDNN UT must be skipped", +) class TestExpandV2OneDNNOp(OpTest): - def setUp(self): self.op_type = "expand_v2" self.init_data() @@ -52,7 +57,6 @@ class TestExpandV2OneDNNOp(OpTest): class TestExpandV2ExpandDimOneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] @@ -60,7 +64,6 @@ class TestExpandV2ExpandDimOneDNNOp(TestExpandV2OneDNNOp): class TestExpandV2CopyScenarioOneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) @@ -68,7 +71,6 @@ class TestExpandV2CopyScenarioOneDNNOp(TestExpandV2OneDNNOp): class TestExpandV2CopyScenarioShapeNotGivenOneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) @@ -76,7 +78,6 @@ class TestExpandV2CopyScenarioShapeNotGivenOneDNNOp(TestExpandV2OneDNNOp): class TestExpandV2ExpandShapesTensor1OneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = [100, 1] self.expand_times = [1, 2] @@ -86,8 +87,9 @@ class TestExpandV2ExpandShapesTensor1OneDNNOp(TestExpandV2OneDNNOp): def calc_expand_shapes_tensor(self): self.expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - self.expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) def set_additional_inputs(self): self.calc_expand_shapes_tensor() @@ -95,8 +97,8 @@ class TestExpandV2ExpandShapesTensor1OneDNNOp(TestExpandV2OneDNNOp): class TestExpandV2ExpandShapesTensor2OneDNNOp( - TestExpandV2ExpandShapesTensor1OneDNNOp): - + TestExpandV2ExpandShapesTensor1OneDNNOp +): def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -105,7 +107,6 @@ class TestExpandV2ExpandShapesTensor2OneDNNOp( class TestExpandV2ShapesTensorOneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = [100] self.expand_times = [2, 1] @@ -118,10 +119,8 @@ class TestExpandV2ShapesTensorOneDNNOp(TestExpandV2OneDNNOp): # BF16 TESTS def create_expand_v2_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestExpandV2BF16OneDNNOp(parent): - def set_inputs(self): self.attrs['mkldnn_data_type'] = 'bfloat16' self.inputs = {"X": convert_float_to_uint16(self.x)} @@ -137,10 +136,12 @@ def create_expand_v2_bf16_test_class(parent): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[convert_float_to_uint16(self.dx)], - user_defined_grad_outputs=[self.dout]) + user_defined_grad_outputs=[self.dout], + ) cls_name = "{0}_{1}".format(parent.__name__, "Expand_v2_BF16") TestExpandV2BF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_bf16_mkldnn_op.py index b5b83bf47ea..b35b06915b5 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_bf16_mkldnn_op.py @@ -25,16 +25,15 @@ def fully_connected_naive(input, weights, bias_data): class MatrixGenerate: - def __init__(self, mb, ic, oc, h, w): self.input = np.random.random((mb, ic * h * w)).astype(np.float32) self.weights = np.random.random((ic * h * w, oc)).astype(np.float32) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestFcBf16MklDNNOp(OpTest): - def generate_data(self): self.matrix = MatrixGenerate(1, 10, 15, 3, 3) self.bias = np.random.random(15).astype("float32") @@ -46,20 +45,21 @@ class TestFcBf16MklDNNOp(OpTest): self.force_fp32_output = False self.generate_data() - self.output = fully_connected_naive(self.matrix.input, - self.matrix.weights, self.bias) + self.output = fully_connected_naive( + self.matrix.input, self.matrix.weights, self.bias + ) if not self.force_fp32_output: self.output = convert_float_to_uint16(self.output) self.inputs = { 'Input': convert_float_to_uint16(self.matrix.input), 'W': self.matrix.weights, - 'Bias': self.bias + 'Bias': self.bias, } self.attrs = { 'use_mkldnn': self.use_mkldnn, - 'force_fp32_output': self.force_fp32_output + 'force_fp32_output': self.force_fp32_output, } self.outputs = {'Out': self.output} @@ -75,7 +75,6 @@ class TestFcBf16MklDNNOp(OpTest): class TestFCMKLDNNOp1(TestFcBf16MklDNNOp): - def generate_data(self): self.matrix = MatrixGenerate(2, 15, 48, 2, 2) self.bias = np.random.random(48).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_int8_mkldnn_op.py index 5cecbf159cc..95493251735 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_int8_mkldnn_op.py @@ -19,7 +19,6 @@ from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool @OpTestTool.skip_if_not_cpu() class TestFCINT8OneDNNOp(OpTest): - def setUp(self): self.op_type = "fc" self._cpu_only = True @@ -32,7 +31,7 @@ class TestFCINT8OneDNNOp(OpTest): 'Scale_in': self.x_scale, 'Scale_weights': [self.y_scale], 'Scale_out': self.out_scale, - 'force_fp32_output': self.force_fp32_output + 'force_fp32_output': self.force_fp32_output, } if self.force_fp32_output: @@ -50,7 +49,7 @@ class TestFCINT8OneDNNOp(OpTest): self.inputs = {'Input': self.x, 'W': self.y_float, 'Bias': self.bias} def quantize(self, tensor): - scale = 63. / np.abs(np.amax(tensor)) + scale = 63.0 / np.abs(np.amax(tensor)) quantized = np.round(scale * tensor).astype("int8") return scale, quantized @@ -74,7 +73,6 @@ class TestFCINT8OneDNNOp(OpTest): class TestFCINT8NoBiasOneDNNOp(TestFCINT8OneDNNOp): - def configure(self): self.use_bias = False self.force_fp32_output = False @@ -87,7 +85,6 @@ class TestFCINT8NoBiasOneDNNOp(TestFCINT8OneDNNOp): class TestFCINT8ForceFP32OutputOneDNNOp(TestFCINT8NoBiasOneDNNOp): - def configure(self): self.use_bias = False self.force_fp32_output = True @@ -95,5 +92,6 @@ class TestFCINT8ForceFP32OutputOneDNNOp(TestFCINT8NoBiasOneDNNOp): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_mkldnn_op.py index 0c1d9bef032..d64e0b441ee 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_mkldnn_op.py @@ -23,14 +23,12 @@ def fully_connected_naive(input, weights, bias_data): class MatrixGenerate: - def __init__(self, mb, ic, oc, h, w): self.input = np.random.random((mb, ic * h * w)).astype("float32") self.weights = np.random.random((ic * h * w, oc)).astype("float32") class TestFCMKLDNNOp(OpTest): - def create_data(self): self.matrix = MatrixGenerate(1, 10, 15, 3, 3) self.bias = np.random.random(15).astype("float32") @@ -43,15 +41,15 @@ class TestFCMKLDNNOp(OpTest): self.inputs = { 'Input': self.matrix.input, 'W': self.matrix.weights, - 'Bias': self.bias + 'Bias': self.bias, } self.attrs = {'use_mkldnn': self.use_mkldnn} self.outputs = { - 'Out': - fully_connected_naive(self.matrix.input, self.matrix.weights, - self.bias) + 'Out': fully_connected_naive( + self.matrix.input, self.matrix.weights, self.bias + ) } def test_check_output(self): @@ -66,7 +64,6 @@ class TestFCMKLDNNOp(OpTest): class TestFCMKLDNNOp1(TestFCMKLDNNOp): - def create_data(self): self.matrix = MatrixGenerate(2, 15, 48, 2, 2) self.bias = np.random.random(48).astype("float32") @@ -74,5 +71,6 @@ class TestFCMKLDNNOp1(TestFCMKLDNNOp): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fill_constant_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fill_constant_mkldnn_op.py index f4dfcee10bc..92e274229b0 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fill_constant_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fill_constant_mkldnn_op.py @@ -20,7 +20,6 @@ import paddle @OpTestTool.skip_if_not_cpu_bf16() class TestFillConstant2DOneDNNOp(OpTest): - def setUp(self): self.op_type = "fill_constant" self.dtype = np.float32 @@ -62,59 +61,58 @@ class TestFillConstant2DOneDNNOp(OpTest): self.check_output() -class TestFillZerosLike4DShapeTensorPriorityOneDNNOp(TestFillConstant2DOneDNNOp - ): - +class TestFillZerosLike4DShapeTensorPriorityOneDNNOp( + TestFillConstant2DOneDNNOp +): def set_inputs(self): self.inputs = {'ShapeTensor': np.array([5, 6, 7, 8]).astype("int32")} class TestFillZerosLike4DShapeTensorListPriorityOneDNNOp( - TestFillConstant2DOneDNNOp): - + TestFillConstant2DOneDNNOp +): def set_inputs(self): shape = (4, 5, 6, 7) self.shape_tensor_list = [] for index, elem in enumerate(shape): - self.shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * elem)) + self.shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * elem) + ) self.inputs = {'ShapeTensorList': self.shape_tensor_list} class TestFillZerosLike2DStringValueInfOneDNNOp(TestFillConstant2DOneDNNOp): - def set_attrs(self): self.str_value = "inf" self.attrs = {'shape': (10, 13), 'use_mkldnn': True, 'str_value': "inf"} -class TestFillZerosLike2DStringValueMinusInfOneDNNOp(TestFillConstant2DOneDNNOp - ): - +class TestFillZerosLike2DStringValueMinusInfOneDNNOp( + TestFillConstant2DOneDNNOp +): def set_attrs(self): self.str_value = "-inf" self.attrs = { 'shape': (10, 13), 'use_mkldnn': True, - 'str_value': "-inf" + 'str_value': "-inf", } class TestFillZerosLike2DStringValueFloatOneDNNOp(TestFillConstant2DOneDNNOp): - def set_attrs(self): self.str_value = "0.123" self.attrs = { 'shape': (10, 13), 'use_mkldnn': True, - 'str_value': "0.123" + 'str_value': "0.123", } class TestFillZerosLike2DValueTensorPriorityOneDNNOp( - TestFillZerosLike2DStringValueFloatOneDNNOp): - + TestFillZerosLike2DStringValueFloatOneDNNOp +): def set_inputs(self): self.inputs = {'ValueTensor': np.atleast_1d(2.25).astype("float32")} diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_flags_mkldnn_ops_on_off.py b/python/paddle/fluid/tests/unittests/mkldnn/test_flags_mkldnn_ops_on_off.py index ec4e87918d7..b59ce2ee714 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_flags_mkldnn_ops_on_off.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_flags_mkldnn_ops_on_off.py @@ -20,7 +20,6 @@ import re class TestFlagsUseMkldnn(unittest.TestCase): - def setUp(self): self._python_interp = sys.executable self._python_interp += " check_flags_mkldnn_ops_on_off.py" @@ -30,16 +29,22 @@ class TestFlagsUseMkldnn(unittest.TestCase): self.env[str("FLAGS_use_mkldnn")] = str("1") self.relu_regex = b"^onednn_verbose,exec,cpu,eltwise,.+alg:eltwise_relu alpha:0 beta:0,10x20x20" - self.ew_add_regex = b"^onednn_verbose,exec,cpu,binary.+alg:binary_add,10x20x30:10x20x30" - self.matmul_regex = b"^onednn_verbose,exec,cpu,matmul,.*10x20x30:10x30x20:10x20x20" + self.ew_add_regex = ( + b"^onednn_verbose,exec,cpu,binary.+alg:binary_add,10x20x30:10x20x30" + ) + self.matmul_regex = ( + b"^onednn_verbose,exec,cpu,matmul,.*10x20x30:10x30x20:10x20x20" + ) def flags_use_mkl_dnn_common(self, e): cmd = self._python_interp env = dict(self.env, **e) - proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env) + proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) out, err = proc.communicate() returncode = proc.returncode @@ -98,7 +103,7 @@ class TestFlagsUseMkldnn(unittest.TestCase): def test_flags_use_mkl_dnn_on_off(self): env = { str("FLAGS_tracer_mkldnn_ops_on"): str("elementwise_add"), - str("FLAGS_tracer_mkldnn_ops_off"): str("matmul") + str("FLAGS_tracer_mkldnn_ops_off"): str("matmul"), } out, err = self.flags_use_mkl_dnn_common(env) assert self.not_found(self.relu_regex, out, err) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_flags_use_mkldnn.py b/python/paddle/fluid/tests/unittests/mkldnn/test_flags_use_mkldnn.py index 1e79d8e82c5..35ca3188f19 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_flags_use_mkldnn.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_flags_use_mkldnn.py @@ -20,7 +20,6 @@ import re class TestFlagsUseMkldnn(unittest.TestCase): - def setUp(self): self._python_interp = sys.executable self._python_interp += " check_flags_use_mkldnn.py" @@ -45,10 +44,12 @@ class TestFlagsUseMkldnn(unittest.TestCase): def test_flags_use_mkl_dnn(self): cmd = self._python_interp - proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.env) + proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.env, + ) out, err = proc.communicate() returncode = proc.returncode diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_flatten_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_flatten_mkldnn_op.py index 32e7cec2999..0786b8d7d7a 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_flatten_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_flatten_mkldnn_op.py @@ -17,12 +17,15 @@ import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) @OpTestTool.skip_if_not_cpu_bf16() class TestFlattenOneDNNOp(OpTest): - def setUp(self): self.set_op_type() self.init_test_case() @@ -50,7 +53,6 @@ class TestFlattenOneDNNOp(OpTest): class TestFlattenOneDNNOp1(TestFlattenOneDNNOp): - def init_test_case(self): self.in_shape = (3, 2, 2, 10) self.axis = 0 @@ -58,7 +60,6 @@ class TestFlattenOneDNNOp1(TestFlattenOneDNNOp): class TestFlattenOneDNNOpSixDims(TestFlattenOneDNNOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 @@ -66,28 +67,23 @@ class TestFlattenOneDNNOpSixDims(TestFlattenOneDNNOp): class TestFlatten2OneDNNOp(TestFlattenOneDNNOp): - def set_op_type(self): self.op_type = "flatten2" class TestFlatten2OneDNNOp1(TestFlattenOneDNNOp1): - def set_op_type(self): self.op_type = "flatten2" class TestFlatten2OneDNNOpSixDims(TestFlattenOneDNNOpSixDims): - def set_op_type(self): self.op_type = "flatten2" # BF16 TESTS def create_flatten_bf16_test_classes(parent): - class TestFlatten2BF16OneDNNOp(parent): - def set_inputs(self): self.dtype = np.uint16 self.inputs = { @@ -99,22 +95,25 @@ def create_flatten_bf16_test_classes(parent): self.dx = np.reshape(self.dout, self.ori_shape) def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), - no_check_set=["XShape"]) + self.check_output_with_place( + core.CPUPlace(), no_check_set=["XShape"] + ) def test_check_grad(self): self.calculate_grads() - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - user_defined_grads=[self.dx], - user_defined_grad_outputs=[self.dout]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + user_defined_grads=[self.dx], + user_defined_grad_outputs=[self.dout], + ) cls_name = "{0}_{1}".format(parent.__name__, "Flatten2_BF16") TestFlatten2BF16OneDNNOp.__name__ = cls_name globals()[cls_name] = TestFlatten2BF16OneDNNOp class TestFlattenBF16OneDNNOp(parent): - def set_op_type(self): self.dtype = np.uint16 self.op_type = "flatten" @@ -138,10 +137,12 @@ def create_flatten_bf16_test_classes(parent): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "Flatten_BF16") TestFlattenBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py index 62759913e84..da8d5149360 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py @@ -20,10 +20,10 @@ from paddle.fluid.tests.unittests.test_fusion_gru_op import fusion_gru from paddle.fluid.tests.unittests.test_fusion_lstm_op import ACTIVATION -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestFusionGRUBF16MKLDNNOp(OpTest): - def set_confs(self): pass @@ -66,19 +66,30 @@ class TestFusionGRUBF16MKLDNNOp(OpTest): wh_bf16 = convert_float_to_uint16(wh_fp32) # bias is fp32 despite other inputs being in bf16 - bias = np.random.rand( - 1, 3 * self.D).astype('float32') if self.with_bias else np.zeros( - (1, 3 * self.D), dtype='float32') - - h0_fp32 = np.random.rand( - N, self.D).astype('float32') if self.with_h0 else np.zeros( - (N, self.D), dtype='float32') - - _, _, _, hidden = fusion_gru(x_fp32, self.lod, h0_fp32, wx_fp32, - wh_fp32, bias, self.is_reverse, - self.origin_mode, - ACTIVATION[self.act_state], - ACTIVATION[self.act_gate]) + bias = ( + np.random.rand(1, 3 * self.D).astype('float32') + if self.with_bias + else np.zeros((1, 3 * self.D), dtype='float32') + ) + + h0_fp32 = ( + np.random.rand(N, self.D).astype('float32') + if self.with_h0 + else np.zeros((N, self.D), dtype='float32') + ) + + _, _, _, hidden = fusion_gru( + x_fp32, + self.lod, + h0_fp32, + wx_fp32, + wh_fp32, + bias, + self.is_reverse, + self.origin_mode, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate], + ) hidden_bf16 = convert_float_to_uint16(hidden) @@ -86,13 +97,13 @@ class TestFusionGRUBF16MKLDNNOp(OpTest): self.inputs = { 'X': (x_bf16, self.lod), 'WeightX': wx_bf16, - 'WeightH': wh_bf16 + 'WeightH': wh_bf16, } elif self.weights_dtype == 'fp32': self.inputs = { 'X': (x_bf16, self.lod), 'WeightX': wx_fp32, - 'WeightH': wh_fp32 + 'WeightH': wh_fp32, } if self.with_bias: @@ -120,24 +131,22 @@ class TestFusionGRUBF16MKLDNNOp(OpTest): class TestFusionGRUINT8MKLDNNOp2(TestFusionGRUBF16MKLDNNOp): - def set_confs(self): self.origin_mode = False class TestFusionGRUINT8MKLDNNOp3(TestFusionGRUBF16MKLDNNOp): - def set_confs(self): self.with_bias = False class TestFusionGRUINT8MKLDNNBF16WeightsOp(TestFusionGRUBF16MKLDNNOp): - def set_confs(self): self.weights_dtype = 'bf16' if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py index 9cbbddd6a42..fdd9ef2b4cc 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py @@ -20,7 +20,6 @@ from paddle.fluid.tests.unittests.test_fusion_lstm_op import ACTIVATION class TestFusionGRUINT8MKLDNNOp(OpTest): - def set_confs(self): pass @@ -62,35 +61,60 @@ class TestFusionGRUINT8MKLDNNOp(OpTest): # WeightH data shape in PP: [OC, 2 * OC] + [OC, OC] # Scales shape in oneDNN: [3, OC] s8_max = 127.0 - scale_ur = s8_max / np.max(np.abs( - np.concatenate([ - wx[:, :2 * self.OC], - wh.flatten()[:2 * self.OC * self.OC].reshape( - self.OC, 2 * self.OC) - ], - axis=0)), - axis=0) - scale_o = s8_max / np.max(np.abs( - np.concatenate([ - wx[:, 2 * self.OC:], - wh.flatten()[2 * self.OC * self.OC:].reshape(self.OC, self.OC) - ], - axis=0)), - axis=0) + scale_ur = s8_max / np.max( + np.abs( + np.concatenate( + [ + wx[:, : 2 * self.OC], + wh.flatten()[: 2 * self.OC * self.OC].reshape( + self.OC, 2 * self.OC + ), + ], + axis=0, + ) + ), + axis=0, + ) + scale_o = s8_max / np.max( + np.abs( + np.concatenate( + [ + wx[:, 2 * self.OC :], + wh.flatten()[2 * self.OC * self.OC :].reshape( + self.OC, self.OC + ), + ], + axis=0, + ) + ), + axis=0, + ) scale_weights = np.concatenate([scale_ur, scale_o]).astype('float') - bias = np.random.rand( - 1, 3 * self.OC).astype('float32') if self.with_bias else np.zeros( - (1, 3 * self.OC), dtype='float32') - h0 = np.random.rand( - N, self.OC).astype('float32') if self.with_h0 else np.zeros( - (N, self.OC), dtype='float32') - - _, _, _, hidden_f32 = fusion_gru(x_f32, self.lod, h0, wx, wh, bias, - self.is_reverse, self.origin_mode, - ACTIVATION[self.act_state], - ACTIVATION[self.act_gate]) + bias = ( + np.random.rand(1, 3 * self.OC).astype('float32') + if self.with_bias + else np.zeros((1, 3 * self.OC), dtype='float32') + ) + h0 = ( + np.random.rand(N, self.OC).astype('float32') + if self.with_h0 + else np.zeros((N, self.OC), dtype='float32') + ) + + _, _, _, hidden_f32 = fusion_gru( + x_f32, + self.lod, + h0, + wx, + wh, + bias, + self.is_reverse, + self.origin_mode, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate], + ) self.inputs = {'X': (x_u8, self.lod), 'WeightX': wx, 'WeightH': wh} @@ -106,7 +130,8 @@ class TestFusionGRUINT8MKLDNNOp(OpTest): else: self.error_margin = 1 hidden_u8 = np.rint(hidden_f32 * scale_data + shift_data).astype( - np.uint8) + np.uint8 + ) # hidden_u8 = (hidden_f32 * scale_data + shift_data).astype(np.uint8) self.outputs = {'Hidden': (hidden_u8, self.lod)} @@ -120,7 +145,7 @@ class TestFusionGRUINT8MKLDNNOp(OpTest): 'force_fp32_output': self.force_fp32_output, 'Scale_data': scale_data, 'Shift_data': shift_data, - 'Scale_weights': scale_weights + 'Scale_weights': scale_weights, } def test_check_output(self): @@ -128,30 +153,27 @@ class TestFusionGRUINT8MKLDNNOp(OpTest): class TestFusionGRUINT8MKLDNNOp2(TestFusionGRUINT8MKLDNNOp): - def set_confs(self): self.force_fp32_output = False class TestFusionGRUINT8MKLDNNOp3(TestFusionGRUINT8MKLDNNOp): - def set_confs(self): self.origin_mode = False class TestFusionGRUINT8MKLDNNOp4(TestFusionGRUINT8MKLDNNOp): - def set_confs(self): self.with_bias = False class TestFusionGRUINT8MKLDNNOp5(TestFusionGRUINT8MKLDNNOp): - def set_confs(self): self.with_h0 = False if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_mkldnn_op.py index e47105e733c..aa38c004759 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_mkldnn_op.py @@ -17,41 +17,35 @@ from paddle.fluid.tests.unittests.test_fusion_gru_op import TestFusionGRUOp class TestFusionGRUMKLDNNOp(TestFusionGRUOp): - def set_confs(self): self.use_mkldnn = True class TestFusionGRUMKLDNNOpNoInitial(TestFusionGRUOp): - def set_confs(self): self.with_h0 = False self.use_mkldnn = True class TestFusionGRUMKLDNNOpNoBias(TestFusionGRUOp): - def set_confs(self): self.with_bias = False self.use_mkldnn = True class TestFusionGRUMKLDNNOpReverse(TestFusionGRUOp): - def set_confs(self): self.is_reverse = True self.use_mkldnn = True class TestFusionGRUMKLDNNOpOriginMode(TestFusionGRUOp): - def set_confs(self): self.origin_mode = True self.use_mkldnn = True class TestFusionGRUMKLDNNOpMD1(TestFusionGRUOp): - def set_confs(self): self.M = 36 self.D = 8 @@ -59,7 +53,6 @@ class TestFusionGRUMKLDNNOpMD1(TestFusionGRUOp): class TestFusionGRUMKLDNNOpMD2(TestFusionGRUOp): - def set_confs(self): self.M = 8 self.D = 8 @@ -67,7 +60,6 @@ class TestFusionGRUMKLDNNOpMD2(TestFusionGRUOp): class TestFusionGRUMKLDNNOpMD3(TestFusionGRUOp): - def set_confs(self): self.M = 17 self.D = 15 @@ -75,7 +67,6 @@ class TestFusionGRUMKLDNNOpMD3(TestFusionGRUOp): class TestFusionGRUMKLDNNOpBS1(TestFusionGRUOp): - def set_confs(self): self.lod = [[3]] self.D = 16 @@ -84,5 +75,6 @@ class TestFusionGRUMKLDNNOpBS1(TestFusionGRUOp): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py index a2fdf54da9d..4d2842d26ed 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py @@ -16,22 +16,25 @@ import unittest import numpy as np import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -from paddle.fluid.tests.unittests.test_fusion_lstm_op import ACTIVATION, fusion_lstm +from paddle.fluid.tests.unittests.test_fusion_lstm_op import ( + ACTIVATION, + fusion_lstm, +) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestFusionLSTMBF16ONEDNNOp(OpTest): - def set_confs(self): pass def test_check_output(self): for use_seq in {True, False}: self.attrs['use_seq'] = use_seq - self.check_output(check_dygraph=False, - no_check_set=["Cell"], - atol=2e-2) + self.check_output( + check_dygraph=False, no_check_set=["Cell"], atol=2e-2 + ) def setUp(self): self.op_type = 'fusion_lstm' @@ -75,8 +78,8 @@ class TestFusionLSTMBF16ONEDNNOp(OpTest): b = np.random.normal(size=(1, 7 * self.D)).astype('float32') else: b = np.random.normal(size=(1, 4 * self.D)).astype('float32') - w_b = np.copy(b[:, 0:4 * self.D]) - w_c = b[:, 4 * self.D:] if self.use_peepholes else None + w_b = np.copy(b[:, 0 : 4 * self.D]) + w_c = b[:, 4 * self.D :] if self.use_peepholes else None wx = np.random.normal(size=(self.M, 4 * self.D)).astype('float32') @@ -84,12 +87,23 @@ class TestFusionLSTMBF16ONEDNNOp(OpTest): wh_bf16 = convert_float_to_uint16(wh) bx = np.random.normal(size=(1, 4 * self.D)).astype('float32') - b[0, 0:4 * self.D] += bx[0, :] - - hidden, c = fusion_lstm(x, self.lod, wx, bx, h0, c0, wh, w_b, w_c, - self.is_reverse, ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + b[0, 0 : 4 * self.D] += bx[0, :] + + hidden, c = fusion_lstm( + x, + self.lod, + wx, + bx, + h0, + c0, + wh, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) hidden = hidden.astype('float32') hidden_bf16 = convert_float_to_uint16(hidden) @@ -99,14 +113,14 @@ class TestFusionLSTMBF16ONEDNNOp(OpTest): 'X': (x_bf16, self.lod), 'WeightX': wx_bf16, 'WeightH': wh_bf16, - 'Bias': b + 'Bias': b, } elif self.weights_dtype == 'fp32': self.inputs = { 'X': (x_bf16, self.lod), 'WeightX': wx, 'WeightH': wh, - 'Bias': b + 'Bias': b, } if self.has_initial_state: @@ -135,30 +149,27 @@ class TestFusionLSTMBF16ONEDNNOp(OpTest): class TestFusionLSTMBF16ONEDNNPeepholesOp(TestFusionLSTMBF16ONEDNNOp): - def set_confs(self): self.use_peepholes = True class TestFusionLSTMBF16ONEDNNInitializedStateOp(TestFusionLSTMBF16ONEDNNOp): - def set_confs(self): self.has_initial_state = True class TestFusionLSTMBF16ONEDNNReverseOp(TestFusionLSTMBF16ONEDNNOp): - def set_confs(self): self.is_reverse = True class TestFusionLSTMBF16ONEDNNBF16WeightsOp(TestFusionLSTMBF16ONEDNNOp): - def set_confs(self): self.weights_dtype = 'bf16' if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_int8_mkldnn_op.py index 968c01faa17..0b252e7cbaf 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_int8_mkldnn_op.py @@ -15,11 +15,13 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest -from paddle.fluid.tests.unittests.test_fusion_lstm_op import ACTIVATION, fusion_lstm +from paddle.fluid.tests.unittests.test_fusion_lstm_op import ( + ACTIVATION, + fusion_lstm, +) class TestFusionLSTMINT8MKLDNNOp(OpTest): - def set_confs(self): pass @@ -59,7 +61,8 @@ class TestFusionLSTMINT8MKLDNNOp(OpTest): s8_max = 127.0 scale_weights = s8_max / np.max( - np.abs(np.concatenate([wx[:, :], wh[:, :]], axis=0)), axis=0) + np.abs(np.concatenate([wx[:, :], wh[:, :]], axis=0)), axis=0 + ) scale_weights = scale_weights.astype('float') @@ -67,11 +70,11 @@ class TestFusionLSTMINT8MKLDNNOp(OpTest): b = np.random.rand(1, 7 * self.OC).astype('float32') else: b = np.random.rand(1, 4 * self.OC).astype('float32') - w_b = np.copy(b[:, 0:4 * self.OC]) - w_c = b[:, 4 * self.OC:] if self.use_peepholes else None + w_b = np.copy(b[:, 0 : 4 * self.OC]) + w_c = b[:, 4 * self.OC :] if self.use_peepholes else None bx = np.random.normal(size=(1, 4 * self.OC)).astype('float32') - b[0, 0:4 * self.OC] += bx[0, :] + b[0, 0 : 4 * self.OC] += bx[0, :] if self.has_initial_state: h0 = np.random.rand(N, self.OC).astype('float32') @@ -80,17 +83,27 @@ class TestFusionLSTMINT8MKLDNNOp(OpTest): h0 = np.zeros((N, self.OC)).astype('float32') c0 = np.zeros((N, self.OC)).astype('float32') - hidden_f32, c = fusion_lstm(x_f32, self.lod, wx, bx, h0, c0, wh, w_b, - w_c, self.is_reverse, - ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + hidden_f32, c = fusion_lstm( + x_f32, + self.lod, + wx, + bx, + h0, + c0, + wh, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = { 'X': (x_u8, self.lod), 'WeightX': wx, 'WeightH': wh, - 'Bias': b + 'Bias': b, } if self.has_initial_state: @@ -101,15 +114,16 @@ class TestFusionLSTMINT8MKLDNNOp(OpTest): self.error_margin = 1e-1 self.outputs = { 'Hidden': (hidden_f32, self.lod), - 'Cell': (c, self.lod) + 'Cell': (c, self.lod), } else: self.error_margin = 2 hidden_u8 = np.rint(hidden_f32 * scale_data + shift_data).astype( - np.uint8) + np.uint8 + ) self.outputs = { 'Hidden': (hidden_u8, self.lod), - 'Cell': (c, self.lod) + 'Cell': (c, self.lod), } self.attrs = { @@ -123,36 +137,36 @@ class TestFusionLSTMINT8MKLDNNOp(OpTest): 'force_fp32_output': self.force_fp32_output, 'Scale_data': scale_data, 'Shift_data': shift_data, - 'Scale_weights': scale_weights + 'Scale_weights': scale_weights, } def test_check_output(self): for use_seq in {True, False}: self.attrs['use_seq'] = use_seq - self.check_output(check_dygraph=False, - no_check_set=["Cell"], - atol=self.error_margin) + self.check_output( + check_dygraph=False, + no_check_set=["Cell"], + atol=self.error_margin, + ) class TestFusionLSTMINT8MKLDNNOp2(TestFusionLSTMINT8MKLDNNOp): - def set_confs(self): self.force_fp32_output = True class TestFusionLSTMINT8MKLDNNOp4(TestFusionLSTMINT8MKLDNNOp): - def set_confs(self): self.is_reverse = True class TestFusionLSTMINT8MKLDNNOp5(TestFusionLSTMINT8MKLDNNOp): - def set_confs(self): self.has_initial_state = True if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_mkldnn_op.py index 5ce307fe35e..8df2aedb376 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_mkldnn_op.py @@ -17,7 +17,6 @@ from paddle.fluid.tests.unittests.test_fusion_lstm_op import TestFusionLSTMOp class TestFusionLSTMONEDNNOp(TestFusionLSTMOp): - def set_conf(self): self.use_mkldnn = True @@ -28,14 +27,12 @@ class TestFusionLSTMONEDNNOp(TestFusionLSTMOp): class TestFusionLSTMONEDNNOpReverse(TestFusionLSTMONEDNNOp): - def set_conf(self): self.is_reverse = True self.use_mkldnn = True class TestFusionLSTMONEDNNOpInitReverse(TestFusionLSTMONEDNNOp): - def set_conf(self): self.has_initial_state = True self.is_reverse = True @@ -43,7 +40,6 @@ class TestFusionLSTMONEDNNOpInitReverse(TestFusionLSTMONEDNNOp): class TestFusionLSTMONEDNNOpMD1(TestFusionLSTMONEDNNOp): - def set_conf(self): self.M = 36 self.D = 8 @@ -51,7 +47,6 @@ class TestFusionLSTMONEDNNOpMD1(TestFusionLSTMONEDNNOp): class TestFusionLSTMONEDNNOpMD2(TestFusionLSTMONEDNNOp): - def set_conf(self): self.M = 8 self.D = 8 @@ -59,7 +54,6 @@ class TestFusionLSTMONEDNNOpMD2(TestFusionLSTMONEDNNOp): class TestFusionLSTMONEDNNOpMD3(TestFusionLSTMONEDNNOp): - def set_conf(self): self.M = 15 self.D = 3 @@ -67,7 +61,6 @@ class TestFusionLSTMONEDNNOpMD3(TestFusionLSTMONEDNNOp): class TestFusionLSTMONEDNNOpBS1(TestFusionLSTMONEDNNOp): - def set_conf(self): self.lod = [[3]] self.D = 16 @@ -75,7 +68,6 @@ class TestFusionLSTMONEDNNOpBS1(TestFusionLSTMONEDNNOp): class TestFusionLSTMONEDNNOpPeepholesInit(TestFusionLSTMONEDNNOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True @@ -84,5 +76,6 @@ class TestFusionLSTMONEDNNOpPeepholesInit(TestFusionLSTMONEDNNOp): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_gaussian_random_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_gaussian_random_mkldnn_op.py index 545caa7b73f..8a7c7c23e40 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_gaussian_random_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_gaussian_random_mkldnn_op.py @@ -14,17 +14,17 @@ import unittest -from paddle.fluid.tests.unittests.test_gaussian_random_op import TestGaussianRandomOp +from paddle.fluid.tests.unittests.test_gaussian_random_op import ( + TestGaussianRandomOp, +) class TestMKLDNNGaussianRandomOpSeed10(TestGaussianRandomOp): - def init_kernel_type(self): self.use_mkldnn = True class TestMKLDNNGaussianRandomOpSeed0(TestGaussianRandomOp): - def setUp(self): TestGaussianRandomOp.setUp(self) self.use_mkldnn = True @@ -33,7 +33,7 @@ class TestMKLDNNGaussianRandomOpSeed0(TestGaussianRandomOp): "mean": 1.0, "std": 2.0, "seed": 10, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_bf16_mkldnn_op.py index ab4cc17a713..29dd1a8f360 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_bf16_mkldnn_op.py @@ -22,8 +22,12 @@ import paddle.fluid as fluid from paddle import enable_static from functools import reduce -from paddle.fluid.tests.unittests.mkldnn.test_layer_norm_mkldnn_op import TestLayerNormMKLDNNOp -from paddle.fluid.tests.unittests.mkldnn.test_layer_norm_mkldnn_op import _reference_layer_norm_naive +from paddle.fluid.tests.unittests.mkldnn.test_layer_norm_mkldnn_op import ( + TestLayerNormMKLDNNOp, +) +from paddle.fluid.tests.unittests.mkldnn.test_layer_norm_mkldnn_op import ( + _reference_layer_norm_naive, +) from paddle.fluid.tests.unittests.op_test import convert_float_to_uint16 from paddle.fluid.tests.unittests.op_test import _set_use_system_allocator @@ -32,26 +36,22 @@ np.random.random(123) _set_use_system_allocator(True) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLayerNormBF16MKLDNNOp(TestLayerNormMKLDNNOp): - def __assert_close(self, tensor, np_array, msg, rtol=2e-02, atol=2): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=rtol, - atol=atol, - err_msg=msg) - - def check_forward(self, - shape, - begin_norm_axis, - with_scale_bias=True, - with_is_test=False): + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=rtol, atol=atol, err_msg=msg + ) + + def check_forward( + self, shape, begin_norm_axis, with_scale_bias=True, with_is_test=False + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) @@ -66,8 +66,9 @@ class TestLayerNormBF16MKLDNNOp(TestLayerNormMKLDNNOp): bias = np.array([]) # reference forward & backward - y, mean, variance = _reference_layer_norm_naive(x, scale, bias, epsilon, - begin_norm_axis) + y, mean, variance = _reference_layer_norm_naive( + x, scale, bias, epsilon, begin_norm_axis + ) y_bf16 = convert_float_to_uint16(y) @@ -85,13 +86,17 @@ class TestLayerNormBF16MKLDNNOp(TestLayerNormMKLDNNOp): # scale and bias are fp32 and other vars are of bf16 for name in ground_truth: if name == 'x_bf16' or name == 'y_bf16': - block.create_var(name=name, - dtype='uint16', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='uint16', + shape=ground_truth[name].shape, + ) else: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = {"X": block.var('x_bf16')} if with_scale_bias: @@ -110,8 +115,9 @@ class TestLayerNormBF16MKLDNNOp(TestLayerNormMKLDNNOp): "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, "use_mkldnn": True, - "is_test": with_is_test - }) + "is_test": with_is_test, + }, + ) exe = fluid.Executor(core.CPUPlace()) @@ -120,19 +126,20 @@ class TestLayerNormBF16MKLDNNOp(TestLayerNormMKLDNNOp): input_list.append('scale') input_list.append('bias') - out = exe.run(program, - feed={name: var_dict[name] - for name in input_list}, - fetch_list=['y_bf16', 'mean', 'variance']) + out = exe.run( + program, + feed={name: var_dict[name] for name in input_list}, + fetch_list=['y_bf16', 'mean', 'variance'], + ) self.__assert_close(y_bf16, out[0], "y_bf16", 2) if not with_is_test: self.__assert_close(mean, out[1], "mean") self.__assert_close(variance, out[2], "variance", 1e-3) def test_check_forward_with_is_test(self): - self.check_forward(shape=[2, 3, 4, 5], - begin_norm_axis=3, - with_is_test=True) + self.check_forward( + shape=[2, 3, 4, 5], begin_norm_axis=3, with_is_test=True + ) # TODO (jczaja): Enable those to test when enabling training using bf16 def test_check_forward_with_scale_and_bias(self): diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_mkldnn_op.py index 6506ea41ccd..2233bdfda95 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_mkldnn_op.py @@ -22,7 +22,10 @@ import paddle.fluid as fluid from paddle import enable_static from functools import reduce -from paddle.fluid.tests.unittests.op_test import _set_use_system_allocator, OpTestTool +from paddle.fluid.tests.unittests.op_test import ( + _set_use_system_allocator, + OpTestTool, +) np.random.random(123) @@ -32,7 +35,7 @@ _set_use_system_allocator(True) def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): x_shape = x.shape N = reduce(mul, x_shape[0:begin_norm_axis], 1) - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) x.shape = [N, D] if scale.size == 0 and beta.size == 0: scale = np.ones([1, D]) @@ -43,34 +46,32 @@ def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): mean = np.mean(x, axis=1) var = np.var(x, axis=1) + epsilon - output = scale * np.divide((x - mean.reshape([N, 1])), - (np.sqrt(var)).reshape([N, 1])) + beta + output = ( + scale + * np.divide((x - mean.reshape([N, 1])), (np.sqrt(var)).reshape([N, 1])) + + beta + ) x.shape, output.shape = x_shape, x_shape return output, mean, var class TestLayerNormMKLDNNOp(unittest.TestCase): - def setUp(self): self.use_mkldnn = True def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) - - def check_forward(self, - shape, - begin_norm_axis, - with_scale_bias=True, - with_is_test=False): + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) + + def check_forward( + self, shape, begin_norm_axis, with_scale_bias=True, with_is_test=False + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) @@ -84,8 +85,9 @@ class TestLayerNormMKLDNNOp(unittest.TestCase): bias = np.array([]) # reference forward & backward - y, mean, variance = _reference_layer_norm_naive(x, scale, bias, epsilon, - begin_norm_axis) + y, mean, variance = _reference_layer_norm_naive( + x, scale, bias, epsilon, begin_norm_axis + ) var_dict = locals() var_names = ['x', 'mean', 'variance', 'y'] @@ -99,9 +101,9 @@ class TestLayerNormMKLDNNOp(unittest.TestCase): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, dtype='float32', shape=ground_truth[name].shape + ) inputs = {"X": block.var('x')} if with_scale_bias: @@ -120,8 +122,9 @@ class TestLayerNormMKLDNNOp(unittest.TestCase): "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, "use_mkldnn": True, - "is_test": with_is_test - }) + "is_test": with_is_test, + }, + ) exe = fluid.Executor(core.CPUPlace()) @@ -130,10 +133,11 @@ class TestLayerNormMKLDNNOp(unittest.TestCase): input_list.append('scale') input_list.append('bias') - out = exe.run(program, - feed={name: var_dict[name] - for name in input_list}, - fetch_list=['y', 'mean', 'variance']) + out = exe.run( + program, + feed={name: var_dict[name] for name in input_list}, + fetch_list=['y', 'mean', 'variance'], + ) self.__assert_close(y, out[0], "y") if not with_is_test: self.__assert_close(mean, out[1], "mean") @@ -147,14 +151,14 @@ class TestLayerNormMKLDNNOp(unittest.TestCase): self.check_forward(shape=[2, 3, 4, 5], begin_norm_axis=3) def test_check_forward_without_scale_and_bias(self): - self.check_forward(shape=[2, 3, 4, 5], - begin_norm_axis=3, - with_scale_bias=False) + self.check_forward( + shape=[2, 3, 4, 5], begin_norm_axis=3, with_scale_bias=False + ) def test_check_forward_with_is_test(self): - self.check_forward(shape=[2, 3, 4, 5], - begin_norm_axis=3, - with_is_test=True) + self.check_forward( + shape=[2, 3, 4, 5], begin_norm_axis=3, with_is_test=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_log_softmax_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_log_softmax_mkldnn_op.py index 89de5198101..4279ffc5915 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_log_softmax_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_log_softmax_mkldnn_op.py @@ -17,12 +17,15 @@ import numpy as np import paddle from paddle.fluid import core from paddle.fluid.tests.unittests.test_log_softmax import ref_log_softmax -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) @OpTestTool.skip_if_not_cpu_bf16() class TestLogSoftmaxOneDNNOp(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.set_dtype() @@ -53,45 +56,39 @@ class TestLogSoftmaxOneDNNOp(OpTest): class TestLogSoftmax1DOneDNNOp(TestLogSoftmaxOneDNNOp): - def set_shape(self): self.shape = [100] class TestLogSoftmax3DOneDNNOp(TestLogSoftmaxOneDNNOp): - def set_shape(self): self.shape = [12, 10, 3] class TestLogSoftmax5DOneDNNOp(TestLogSoftmaxOneDNNOp): - def set_shape(self): self.shape = [2, 3, 4, 5, 6] class TestLogSoftmaxPositiveAxisOneDNNOp(TestLogSoftmaxOneDNNOp): - def set_axis(self): self.axis = 2 # BF16 TESTS class TestLogSoftmax1DBF16OneDNNOp(TestLogSoftmax1DOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 -class TestLogSoftmaxPositiveAxisBF16OneDNNOp(TestLogSoftmaxPositiveAxisOneDNNOp - ): - +class TestLogSoftmaxPositiveAxisBF16OneDNNOp( + TestLogSoftmaxPositiveAxisOneDNNOp +): def set_dtype(self): self.dtype = np.uint16 class TestLogSoftmax5DBF16OneDNNOp(TestLogSoftmax5DOneDNNOp): - def set_shape(self): self.shape = [2, 3, 4, 5, 6] diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_lrn_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_lrn_mkldnn_op.py index 8f3913a86f4..f81a0d22aa7 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_lrn_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_lrn_mkldnn_op.py @@ -17,7 +17,6 @@ from paddle.fluid.tests.unittests.test_lrn_op import TestLRNOp class TestLRNMKLDNNOp(TestLRNOp): - def get_attrs(self): attrs = TestLRNOp.get_attrs(self) attrs['use_mkldnn'] = True @@ -26,36 +25,31 @@ class TestLRNMKLDNNOp(TestLRNOp): def test_check_output(self): # We cannot validate MidOut as LRN REF has diffrent meaning in it # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output(atol=0.002, - no_check_set=['MidOut'], - check_dygraph=False) + self.check_output( + atol=0.002, no_check_set=['MidOut'], check_dygraph=False + ) def test_check_grad_normal(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['X'], - 'Out', - max_relative_error=0.01, - check_dygraph=False) + self.check_grad( + ['X'], 'Out', max_relative_error=0.01, check_dygraph=False + ) class TestLRNMKLDNNOpWithIsTest(TestLRNMKLDNNOp): - def get_attrs(self): attrs = TestLRNMKLDNNOp.get_attrs(self) attrs['is_test'] = True return attrs def test_check_grad_normal(self): - def check_raise_is_test(): try: - self.check_grad(['X'], - 'Out', - max_relative_error=0.01, - check_dygraph=False) + self.check_grad( + ['X'], 'Out', max_relative_error=0.01, check_dygraph=False + ) except Exception as e: - t = \ - "is_test attribute should be set to False in training phase." + t = "is_test attribute should be set to False in training phase." if t in str(e): raise AttributeError @@ -63,12 +57,12 @@ class TestLRNMKLDNNOpWithIsTest(TestLRNMKLDNNOp): class TestLRNMKLDNNOpNHWC(TestLRNMKLDNNOp): - def init_test_case(self): self.data_format = 'NHWC' if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_bf16_mkldnn_op.py index 8940502b806..cdb9dfd8ee8 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_bf16_mkldnn_op.py @@ -19,10 +19,10 @@ from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestMatmulBf16MklDNNOp(OpTest): - def generate_data(self): self.x_fp32 = np.random.random((25, 2, 2)).astype(np.float32) self.y_fp32 = np.random.random((25, 2, 2)).astype(np.float32) @@ -35,7 +35,7 @@ class TestMatmulBf16MklDNNOp(OpTest): "mkldnn_data_type": self.mkldnn_data_type, "force_fp32_output": self.force_fp32_output, 'transpose_X': False, - 'transpose_Y': False + 'transpose_Y': False, } def setUp(self): @@ -62,11 +62,13 @@ class TestMatmulBf16MklDNNOp(OpTest): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], + core.CPUPlace(), + ["X", "Y"], "Out", check_dygraph=False, user_defined_grads=[self.dx, self.dy], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) def matmul_grad(self, x, transpose_x, y, transpose_y): x_transpose_axes = [1, 0] if x.ndim == 2 else [0, 2, 1] @@ -81,23 +83,35 @@ class TestMatmulBf16MklDNNOp(OpTest): x_transpose_axes = [1, 0] if self.x_fp32.ndim == 2 else [0, 2, 1] y_transpose_axes = [1, 0] if self.y_fp32.ndim == 2 else [0, 2, 1] - x = np.transpose(self.x_fp32, x_transpose_axes - ) if self.attrs['transpose_X'] is True else self.x_fp32 - y = np.transpose(self.y_fp32, y_transpose_axes - ) if self.attrs['transpose_Y'] is True else self.y_fp32 + x = ( + np.transpose(self.x_fp32, x_transpose_axes) + if self.attrs['transpose_X'] is True + else self.x_fp32 + ) + y = ( + np.transpose(self.y_fp32, y_transpose_axes) + if self.attrs['transpose_Y'] is True + else self.y_fp32 + ) dout = self.alpha * np.matmul(x, y) - if self.attrs['transpose_X'] is True and self.attrs[ - 'transpose_Y'] is True: + if ( + self.attrs['transpose_X'] is True + and self.attrs['transpose_Y'] is True + ): self.dx = self.matmul_grad(self.y_fp32, True, dout, True) self.dy = self.matmul_grad(dout, True, self.x_fp32, True) - elif self.attrs['transpose_X'] is True and self.attrs[ - 'transpose_Y'] is False: + elif ( + self.attrs['transpose_X'] is True + and self.attrs['transpose_Y'] is False + ): self.dx = self.matmul_grad(self.y_fp32, False, dout, True) self.dy = self.matmul_grad(self.x_fp32, False, dout, False) - elif self.attrs['transpose_X'] is False and self.attrs[ - 'transpose_Y'] is True: + elif ( + self.attrs['transpose_X'] is False + and self.attrs['transpose_Y'] is True + ): self.dx = self.matmul_grad(dout, False, self.y_fp32, False) self.dy = self.matmul_grad(dout, True, self.x_fp32, False) else: @@ -108,7 +122,6 @@ class TestMatmulBf16MklDNNOp(OpTest): class TestDnnlMatMulOpAlpha(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((17, 2, 3)).astype(np.float32) self.y_fp32 = np.random.random((17, 3, 2)).astype(np.float32) @@ -117,7 +130,6 @@ class TestDnnlMatMulOpAlpha(TestMatmulBf16MklDNNOp): class TestDnnlMatMulOp2D(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((12, 9)).astype(np.float32) self.y_fp32 = np.random.random((9, 12)).astype(np.float32) @@ -125,7 +137,6 @@ class TestDnnlMatMulOp2D(TestMatmulBf16MklDNNOp): class TestDnnlMatMulOpTransposeX(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((12, 9)).astype(np.float32) self.y_fp32 = np.random.random((12, 9)).astype(np.float32) @@ -136,12 +147,11 @@ class TestDnnlMatMulOpTransposeX(TestMatmulBf16MklDNNOp): "use_mkldnn": self.use_mkldnn, "mkldnn_data_type": self.mkldnn_data_type, 'transpose_X': True, - 'transpose_Y': False + 'transpose_Y': False, } class TestDnnlMatMulOpTransposeY(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((12, 9)).astype(np.float32) self.y_fp32 = np.random.random((12, 9)).astype(np.float32) @@ -152,12 +162,11 @@ class TestDnnlMatMulOpTransposeY(TestMatmulBf16MklDNNOp): "use_mkldnn": self.use_mkldnn, "mkldnn_data_type": self.mkldnn_data_type, 'transpose_Y': True, - 'transpose_X': False + 'transpose_X': False, } class TestMatmulBf16MklDNNForceFp32Output(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((12, 9)).astype(np.float32) self.y_fp32 = np.random.random((9, 12)).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_mkldnn_op.py index ffaa725614b..4d4b6a384a5 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_mkldnn_op.py @@ -18,7 +18,6 @@ from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci class TestDnnlMatMulOp(OpTest): - def generate_data(self): self.x = np.random.random((25, 2, 2)).astype("float32") self.y = np.random.random((25, 2, 2)).astype("float32") @@ -47,13 +46,11 @@ class TestDnnlMatMulOp(OpTest): class TestDnnlMatMulWithGradOp(TestDnnlMatMulOp): - def test_check_grad(self): self.check_grad(['X', 'Y'], 'Out', max_relative_error=1e-2) class TestDnnlMatMulOpMixedDims1(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((17, 2, 3)).astype("float32") self.y = np.random.random((3, 4)).astype("float32") @@ -61,7 +58,6 @@ class TestDnnlMatMulOpMixedDims1(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpMixedDimsYWiderTransposeY(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((8, 2, 3)).astype("float32") self.y = np.random.random((4, 3)).astype("float32") @@ -72,7 +68,6 @@ class TestDnnlMatMulOpMixedDimsYWiderTransposeY(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpMixedDimsYWiderTransposeX(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((8, 3, 2)).astype("float32") self.y = np.random.random((3, 4)).astype("float32") @@ -83,31 +78,30 @@ class TestDnnlMatMulOpMixedDimsYWiderTransposeX(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpMixedDimsXWiderTransposeXY(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((8, 3, 2)).astype("float32") self.y = np.random.random((4, 3)).astype("float32") - self.out = np.matmul(np.transpose(self.x, (0, 2, 1)), - np.transpose(self.y)) + self.out = np.matmul( + np.transpose(self.x, (0, 2, 1)), np.transpose(self.y) + ) def set_attributes(self): self.attrs = {'transpose_X': True, 'transpose_Y': True} class TestDnnlMatMulOpMixedDimsYWiderTransposeXY(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((3, 2)).astype("float32") self.y = np.random.random((8, 4, 3)).astype("float32") - self.out = np.matmul(np.transpose(self.x), - np.transpose(self.y, (0, 2, 1))) + self.out = np.matmul( + np.transpose(self.x), np.transpose(self.y, (0, 2, 1)) + ) def set_attributes(self): self.attrs = {'transpose_X': True, 'transpose_Y': True} class TestDnnlMatMulOpMixedDimsXWiderTransposeX(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((5, 4)).astype("float32") self.y = np.random.random((8, 5, 4)).astype("float32") @@ -118,7 +112,6 @@ class TestDnnlMatMulOpMixedDimsXWiderTransposeX(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpVectorMultiply(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((5)).astype("float32") self.y = np.random.random((5)).astype("float32") @@ -126,7 +119,6 @@ class TestDnnlMatMulOpVectorMultiply(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpVectorMultiplyTranspose(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((5)).astype("float32") x_resized = np.copy(self.x) @@ -141,7 +133,6 @@ class TestDnnlMatMulOpVectorMultiplyTranspose(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpMixedDims2(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((2, 3)).astype("float32") self.y = np.random.random((17, 3, 4)).astype("float32") @@ -149,7 +140,6 @@ class TestDnnlMatMulOpMixedDims2(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpAlpha(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((17, 2, 3)).astype("float32") self.y = np.random.random((17, 3, 2)).astype("float32") @@ -158,7 +148,6 @@ class TestDnnlMatMulOpAlpha(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOp2D(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((12, 9)).astype("float32") self.y = np.random.random((9, 12)).astype("float32") @@ -166,7 +155,6 @@ class TestDnnlMatMulOp2D(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpTransposeX(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((12, 9)).astype("float32") self.y = np.random.random((12, 9)).astype("float32") @@ -177,7 +165,6 @@ class TestDnnlMatMulOpTransposeX(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpTransposeY(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((12, 9)).astype("float32") self.y = np.random.random((12, 9)).astype("float32") @@ -188,7 +175,6 @@ class TestDnnlMatMulOpTransposeY(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpTransposeY3D(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((17, 3, 2)).astype("float32") self.y = np.random.random((17, 3, 2)).astype("float32") @@ -199,7 +185,6 @@ class TestDnnlMatMulOpTransposeY3D(TestDnnlMatMulWithGradOp): class TestDnnlMatMulOpInt8NoScales(TestDnnlMatMulOp): - def generate_data(self): self.x = np.random.random((12, 9)).astype("int8") self.y = np.random.random((9, 12)).astype("int8") @@ -211,7 +196,7 @@ class TestDnnlMatMulOpInt8(TestDnnlMatMulOp): # on older platforms (BDW, SKX) we needed to reduce # range from [-127, 127] to [-63, 63] def quantize(self, tensor): - scale = 63. / np.abs(np.amax(tensor)) + scale = 63.0 / np.abs(np.amax(tensor)) quantized = np.round(scale * tensor).astype("int8") return scale, quantized @@ -238,7 +223,6 @@ class TestDnnlMatMulOpInt8(TestDnnlMatMulOp): class TestDnnlMatMulOpInt8ForceFP32(TestDnnlMatMulOpInt8): - def generate_data(self): x_float = np.random.random((12, 9)).astype("float32") self.x_scale, self.x = self.quantize(x_float) @@ -253,12 +237,11 @@ class TestDnnlMatMulOpInt8ForceFP32(TestDnnlMatMulOpInt8): self.attrs = { 'Scale_x': self.x_scale, 'Scale_y': self.y_scale, - 'force_fp32_output': True + 'force_fp32_output': True, } class TestDnnlMatMulOpInt8ForceFP32BasicScales(TestDnnlMatMulOp): - def generate_data(self): self.x = np.random.randint(0, 3, (12, 9)).astype("int8") self.y = np.random.randint(0, 3, (9, 12)).astype("int8") @@ -270,15 +253,22 @@ class TestDnnlMatMulOpInt8ForceFP32BasicScales(TestDnnlMatMulOp): @skip_check_grad_ci(reason="DNNL's MatMul doesn't implement grad kernel.") class TestReshapeTransposeMatMulOp(OpTest): - def init_data_type(self): self.data_type_ = 'float32' def generate_data(self): - self.x = np.random.random([2, 128, 768]).astype("float32").reshape( - [2, 128, 12, 64]).transpose([0, 2, 1, 3]) - self.y = np.random.random([2, 128, 768]).astype("float32").reshape( - [2, 128, 12, 64]).transpose([0, 2, 1, 3]) + self.x = ( + np.random.random([2, 128, 768]) + .astype("float32") + .reshape([2, 128, 12, 64]) + .transpose([0, 2, 1, 3]) + ) + self.y = ( + np.random.random([2, 128, 768]) + .astype("float32") + .reshape([2, 128, 12, 64]) + .transpose([0, 2, 1, 3]) + ) self.out = np.matmul(self.x, self.y.transpose([0, 1, 3, 2])) self.fused_reshape_X = [] self.fused_transpose_X = [] @@ -300,7 +290,7 @@ class TestReshapeTransposeMatMulOp(OpTest): self.inputs = {'X': self.x, 'Y': self.y} self.attrs = { 'use_mkldnn': self.use_mkldnn, - self.transpose_y_name: self.transpose_y + self.transpose_y_name: self.transpose_y, } if len(self.fused_transpose_X) > 0: self.attrs['fused_transpose_X'] = self.fused_transpose_X @@ -318,49 +308,53 @@ class TestReshapeTransposeMatMulOp(OpTest): class TestReshapeTransposeMatMulOp4DXFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): self.x = np.random.random([2, 128, 768]).astype("float32") - self.y = np.random.random([2, 128, 768]).astype("float32").reshape( - [2, 128, 12, 64]).transpose([0, 2, 1, 3]) + self.y = ( + np.random.random([2, 128, 768]) + .astype("float32") + .reshape([2, 128, 12, 64]) + .transpose([0, 2, 1, 3]) + ) self.fused_transpose_X = [0, 2, 1, 3] self.fused_reshape_X = [0, 0, 12, 64] self.fused_transpose_Y = [] self.fused_reshape_Y = [] self.out = np.matmul( self.x.reshape([2, 128, 12, 64]).transpose([0, 2, 1, 3]), - self.y.transpose([0, 1, 3, 2])) + self.y.transpose([0, 1, 3, 2]), + ) class TestReshapeTransposeMatMulOp4DXInt8(TestReshapeTransposeMatMulOp4DXFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp4DYFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): - self.x = np.random.random([2, 128, 768]).astype("float32").reshape( - [2, 128, 12, 64]).transpose([0, 2, 1, 3]) + self.x = ( + np.random.random([2, 128, 768]) + .astype("float32") + .reshape([2, 128, 12, 64]) + .transpose([0, 2, 1, 3]) + ) self.y = np.random.random([2, 128, 768]).astype("float32") self.fused_transpose_X = [] self.fused_reshape_X = [] self.fused_transpose_Y = [0, 2, 1, 3] self.fused_reshape_Y = [0, 0, 12, 64] self.out = np.matmul( - self.x, - self.y.reshape([2, 128, 12, 64]).transpose([0, 2, 3, 1])) + self.x, self.y.reshape([2, 128, 12, 64]).transpose([0, 2, 3, 1]) + ) class TestReshapeTransposeMatMulOp4DYInt8(TestReshapeTransposeMatMulOp4DYFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp4DXYFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): self.x = np.random.random([2, 128, 768]).astype("float32") self.y = np.random.random([2, 128, 768]).astype("float32") @@ -370,42 +364,48 @@ class TestReshapeTransposeMatMulOp4DXYFloat(TestReshapeTransposeMatMulOp): self.fused_reshape_Y = [0, 0, 12, 64] self.out = np.matmul( self.x.reshape([2, 128, 12, 64]).transpose([0, 2, 1, 3]), - self.y.reshape([2, 128, 12, 64]).transpose([0, 2, 3, 1])) - + self.y.reshape([2, 128, 12, 64]).transpose([0, 2, 3, 1]), + ) -class TestReshapeTransposeMatMulOp4DXYInt8(TestReshapeTransposeMatMulOp4DXYFloat - ): +class TestReshapeTransposeMatMulOp4DXYInt8( + TestReshapeTransposeMatMulOp4DXYFloat +): def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp2DXFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): self.x = np.random.random([2, 5, 10]).astype("float32") - self.y = np.random.random([2, 5, 10]).astype("float32").reshape( - [10, 10]).transpose([1, 0]) + self.y = ( + np.random.random([2, 5, 10]) + .astype("float32") + .reshape([10, 10]) + .transpose([1, 0]) + ) self.fused_transpose_X = [1, 0] self.fused_reshape_X = [10, 10] self.fused_transpose_Y = [] self.fused_reshape_Y = [] self.out = np.matmul( - self.x.reshape([10, 10]).transpose([1, 0]), self.y.transpose([1, - 0])) + self.x.reshape([10, 10]).transpose([1, 0]), self.y.transpose([1, 0]) + ) class TestReshapeTransposeMatMulOp2DXInt8(TestReshapeTransposeMatMulOp2DXFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp2DYFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): - self.x = np.random.random([2, 5, 10]).astype("float32").reshape( - [10, 10]).transpose([1, 0]) + self.x = ( + np.random.random([2, 5, 10]) + .astype("float32") + .reshape([10, 10]) + .transpose([1, 0]) + ) self.y = np.random.random([2, 5, 10]).astype("float32") self.fused_transpose_X = [] self.fused_reshape_X = [] @@ -415,37 +415,42 @@ class TestReshapeTransposeMatMulOp2DYFloat(TestReshapeTransposeMatMulOp): class TestReshapeTransposeMatMulOp2DYInt8(TestReshapeTransposeMatMulOp2DYFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp3DXFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): self.x = np.random.random([2, 2, 5, 5]).astype("float32") - self.y = np.random.random([2, 2, 5, 5]).astype("float32").reshape( - [2, 10, 5]).transpose([0, 2, 1]) + self.y = ( + np.random.random([2, 2, 5, 5]) + .astype("float32") + .reshape([2, 10, 5]) + .transpose([0, 2, 1]) + ) self.fused_transpose_X = [0, 2, 1] self.fused_reshape_X = [2, 10, 5] self.fused_transpose_Y = [] self.fused_reshape_Y = [] self.out = np.matmul( self.x.reshape([2, 10, 5]).transpose(0, 2, 1), - self.y.transpose(0, 2, 1)) + self.y.transpose(0, 2, 1), + ) class TestReshapeTransposeMatMulOp3DXInt8(TestReshapeTransposeMatMulOp3DXFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp3DYFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): - self.x = np.random.random([2, 2, 5, 5]).astype(self.data_type_).reshape( - [2, 10, 5]).transpose([0, 2, 1]) + self.x = ( + np.random.random([2, 2, 5, 5]) + .astype(self.data_type_) + .reshape([2, 10, 5]) + .transpose([0, 2, 1]) + ) self.y = np.random.random([2, 2, 5, 5]).astype(self.data_type_) self.fused_transpose_X = [] self.fused_reshape_X = [] @@ -455,14 +460,12 @@ class TestReshapeTransposeMatMulOp3DYFloat(TestReshapeTransposeMatMulOp): class TestReshapeTransposeMatMulOp3DYInt8(TestReshapeTransposeMatMulOp3DYFloat): - def init_data_type(self): self.data_type_ = 'int8' @skip_check_grad_ci(reason="Tests inference only optimization.") class TestMatMulOpTransposeReshapeEmptyFloat(OpTest): - def init_data_type(self): self.data_type_ = np.float32 @@ -512,39 +515,44 @@ class TestMatMulOpTransposeReshapeEmptyFloat(OpTest): class TestMatMulOpTransposeReshapeIntEmptyInt( - TestMatMulOpTransposeReshapeEmptyFloat): - + TestMatMulOpTransposeReshapeEmptyFloat +): def init_data_type(self): self.data_type_ = np.int8 class TestMatMulOpTransposeReshapeBasicFloat( - TestMatMulOpTransposeReshapeEmptyFloat): - + TestMatMulOpTransposeReshapeEmptyFloat +): def generate_data(self): self.bs = 8 - self.x = np.random.random([self.bs, 12, 128, - 128]).astype(self.data_type_) - self.y = np.random.random([self.bs, 12, 128, - 64]).astype(self.data_type_) + self.x = np.random.random([self.bs, 12, 128, 128]).astype( + self.data_type_ + ) + self.y = np.random.random([self.bs, 12, 128, 64]).astype( + self.data_type_ + ) def init_params_and_out(self): self.transpose_out = [0, 2, 1, 3] self.reshape_out = [0, 0, self.x.shape[1] * self.y.shape[-1]] - self.out = np.matmul(self.x, self.y).transpose([0, 2, 1, 3]).reshape( - [self.bs, -1, self.x.shape[1] * self.y.shape[-1]]) + self.out = ( + np.matmul(self.x, self.y) + .transpose([0, 2, 1, 3]) + .reshape([self.bs, -1, self.x.shape[1] * self.y.shape[-1]]) + ) class TestMatMulOpTransposeReshapeBasicInt( - TestMatMulOpTransposeReshapeBasicFloat): - + TestMatMulOpTransposeReshapeBasicFloat +): def init_data_type(self): self.data_type_ = np.int8 class TestMatMulOpTransposeReshapeOtherDimFloat( - TestMatMulOpTransposeReshapeBasicFloat): - + TestMatMulOpTransposeReshapeBasicFloat +): def generate_data(self): self.bs = 11 self.x = np.random.random([self.bs, 12, 14, 18]).astype(self.data_type_) @@ -552,13 +560,14 @@ class TestMatMulOpTransposeReshapeOtherDimFloat( class TestMatMulOpTransposeReshapeOtherDimInt( - TestMatMulOpTransposeReshapeOtherDimFloat): - + TestMatMulOpTransposeReshapeOtherDimFloat +): def init_data_type(self): self.data_type_ = np.int8 if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_v2_mkldnn_op.py index 50381145821..4a4e0d3ac66 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_v2_mkldnn_op.py @@ -15,17 +15,26 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) import paddle.fluid.core as core import paddle from paddle.fluid.tests.unittests.mkldnn.test_matmul_mkldnn_op import ( TestMatMulOpTransposeReshapeEmptyFloat, TestMatMulOpTransposeReshapeBasicFloat, - TestMatMulOpTransposeReshapeOtherDimFloat, TestReshapeTransposeMatMulOp, - TestReshapeTransposeMatMulOp4DXFloat, TestReshapeTransposeMatMulOp4DYFloat, - TestReshapeTransposeMatMulOp4DXYFloat, TestReshapeTransposeMatMulOp2DXFloat, - TestReshapeTransposeMatMulOp2DYFloat, TestReshapeTransposeMatMulOp3DXFloat, - TestReshapeTransposeMatMulOp3DYFloat) + TestMatMulOpTransposeReshapeOtherDimFloat, + TestReshapeTransposeMatMulOp, + TestReshapeTransposeMatMulOp4DXFloat, + TestReshapeTransposeMatMulOp4DYFloat, + TestReshapeTransposeMatMulOp4DXYFloat, + TestReshapeTransposeMatMulOp2DXFloat, + TestReshapeTransposeMatMulOp2DYFloat, + TestReshapeTransposeMatMulOp3DXFloat, + TestReshapeTransposeMatMulOp3DYFloat, +) def reference_matmul(X, Y, transpose_x=False, transpose_y=False): @@ -34,7 +43,7 @@ def reference_matmul(X, Y, transpose_x=False, transpose_y=False): # transpose X and Y appropriately. if transpose_x: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -43,7 +52,7 @@ def reference_matmul(X, Y, transpose_x=False, transpose_y=False): X = np.transpose(X, tuple(dim)) if transpose_y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -54,10 +63,9 @@ def reference_matmul(X, Y, transpose_x=False, transpose_y=False): class TestMatMulV2VectorXVectorOneDNNOp(OpTest): - def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False self._cpu_only = True @@ -77,14 +85,15 @@ class TestMatMulV2VectorXVectorOneDNNOp(OpTest): # -0.1 ~ 0.1 x = -0.1 + 0.2 * x y = -0.1 + 0.2 * y - result = reference_matmul(x, y, self.trans_x, - self.trans_y).astype("float32") + result = reference_matmul(x, y, self.trans_x, self.trans_y).astype( + "float32" + ) self.set_inputs(x, y) self.attrs = { 'trans_x': self.trans_x, 'trans_y': self.trans_y, - 'use_mkldnn': True + 'use_mkldnn': True, } self.set_dtype_attr() self.outputs = {'Out': result} @@ -97,45 +106,42 @@ class TestMatMulV2VectorXVectorOneDNNOp(OpTest): class TestMatMulV2VectorXMatrixTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.trans_x = False self.trans_y = True class TestMatMulV2VectorXMatrixOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False class TestMatMulV2MatrixXVectorTransposeXOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False class TestMatMulV2MatrixXVectorOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (1, 2, 1, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False class TestMatMulV2MatrixXMatrixOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (1, 1, 2, 100) self.y_shape = (1, 1, 100, 1) @@ -144,8 +150,8 @@ class TestMatMulV2MatrixXMatrixOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): class TestMatMulV2MatrixXMatrixTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (1, 1, 1, 100) self.y_shape = (2, 1, 2, 100) @@ -154,7 +160,6 @@ class TestMatMulV2MatrixXMatrixTransposeYOneDNNOp( class TestMatMulV2MatrixXMatrix2OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (2, 1, 12, 9) self.y_shape = (1, 3, 9, 12) @@ -163,7 +168,6 @@ class TestMatMulV2MatrixXMatrix2OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): class TestMatMulV2MatrixXMatrix3OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (2, 1, 2, 100) self.y_shape = (1, 1, 100, 2) @@ -172,8 +176,8 @@ class TestMatMulV2MatrixXMatrix3OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): class TestMatMulV2MatrixXMatrixTranposeXOneDNNOp2( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (2, 1, 4, 25) self.y_shape = (1, 1, 4, 25) @@ -182,8 +186,8 @@ class TestMatMulV2MatrixXMatrixTranposeXOneDNNOp2( class TestMatMulV2MatrixXMatrixTranposeX2OneDNNOp3( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (2, 2, 7, 4) self.y_shape = (2, 2, 7, 5) @@ -192,8 +196,8 @@ class TestMatMulV2MatrixXMatrixTranposeX2OneDNNOp3( class TestMatMulV2MatrixXMatrixTransposeX3OneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (3, 1, 6, 7) self.y_shape = (1, 2, 6, 9) @@ -202,7 +206,6 @@ class TestMatMulV2MatrixXMatrixTransposeX3OneDNNOp( class TestMatMulV2MatrixXMatrix4OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (3, 1, 6, 6) self.y_shape = (1, 2, 6, 9) @@ -211,26 +214,24 @@ class TestMatMulV2MatrixXMatrix4OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): class TestMatMulV2VectorXMatrix5DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.trans_x = False self.trans_y = False class TestMatMulV2Matrix3DXVectorOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False class TestMatMulV2MatrixXMatrixTransposeXTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (3, 1, 10, 8) self.y_shape = (1, 2, 9, 10) @@ -239,8 +240,8 @@ class TestMatMulV2MatrixXMatrixTransposeXTransposeYOneDNNOp( class TestMatMulV2MatrixXMatrixTransposeY2OneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (3, 1, 10, 10) self.y_shape = (1, 2, 9, 10) @@ -249,8 +250,8 @@ class TestMatMulV2MatrixXMatrixTransposeY2OneDNNOp( class TestMatMulV2MatrixXMatrix5DTranposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (1, 3, 1, 10, 10) self.y_shape = (3, 1, 2, 9, 10) @@ -259,7 +260,6 @@ class TestMatMulV2MatrixXMatrix5DTranposeYOneDNNOp( class TestMatMulV2MatrixXMatrix6Dx2DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (1, 1, 2, 1, 8, 9) self.y_shape = (9, 12) @@ -268,7 +268,6 @@ class TestMatMulV2MatrixXMatrix6Dx2DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): class TestMatMulV2MatrixXMatrix2Dx5DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (20, 5) self.y_shape = (1, 2, 1, 5, 11) @@ -277,8 +276,8 @@ class TestMatMulV2MatrixXMatrix2Dx5DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): class TestMatMulV2MatrixXMatrix4Dx3DTransposeXOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (5, 4, 15, 10) self.y_shape = (1, 15, 20) @@ -287,8 +286,8 @@ class TestMatMulV2MatrixXMatrix4Dx3DTransposeXOneDNNOp( class TestMatMulV2MatrixXMatrix3Dx4DTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (2, 10, 15) self.y_shape = (4, 2, 20, 15) @@ -297,8 +296,8 @@ class TestMatMulV2MatrixXMatrix3Dx4DTransposeYOneDNNOp( class TestMatMulV2MatrixXMatrix5Dx3DTransposeXTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (4, 3, 2, 15, 10) self.y_shape = (1, 20, 15) @@ -307,7 +306,6 @@ class TestMatMulV2MatrixXMatrix5Dx3DTransposeXTransposeYOneDNNOp( class TestMatMulV2MatrixXMatrix3Dx4DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (1, 1, 32, 16) self.y_shape = (16, 16, 16) @@ -317,14 +315,12 @@ class TestMatMulV2MatrixXMatrix3Dx4DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): # BF16 TESTS def create_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestMatMulV2Bf16OneDNNOp(parent): - def set_inputs(self, x, y): self.inputs = { 'X': convert_float_to_uint16(x), - 'Y': convert_float_to_uint16(y) + 'Y': convert_float_to_uint16(y), } self.x_fp32 = x self.y_fp32 = y @@ -338,16 +334,24 @@ def create_bf16_test_class(parent): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], + core.CPUPlace(), + ["X", "Y"], "Out", user_defined_grads=[self.dx, self.dy], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) def matmul_grad(self, x, transpose_x, y, transpose_y): - x = np.transpose( - x, self.shape_transpose_axes[x.ndim]) if transpose_x else x - y = np.transpose( - y, self.shape_transpose_axes[y.ndim]) if transpose_y else y + x = ( + np.transpose(x, self.shape_transpose_axes[x.ndim]) + if transpose_x + else x + ) + y = ( + np.transpose(y, self.shape_transpose_axes[y.ndim]) + if transpose_y + else y + ) return np.matmul(x, y) @@ -357,7 +361,7 @@ def create_bf16_test_class(parent): 3: [0, 2, 1], 4: [0, 1, 3, 2], 5: [0, 1, 2, 4, 3], - 6: [0, 1, 2, 3, 5, 4] + 6: [0, 1, 2, 3, 5, 4], } # expand vector so it will be a valid matrix for multiplication @@ -369,10 +373,16 @@ def create_bf16_test_class(parent): x_transpose_axes = self.shape_transpose_axes[self.x_fp32.ndim] y_transpose_axes = self.shape_transpose_axes[self.y_fp32.ndim] - x = np.transpose(self.x_fp32, x_transpose_axes - ) if self.attrs['trans_x'] is True else self.x_fp32 - y = np.transpose(self.y_fp32, y_transpose_axes - ) if self.attrs['trans_y'] is True else self.y_fp32 + x = ( + np.transpose(self.x_fp32, x_transpose_axes) + if self.attrs['trans_x'] is True + else self.x_fp32 + ) + y = ( + np.transpose(self.y_fp32, y_transpose_axes) + if self.attrs['trans_y'] is True + else self.y_fp32 + ) dout = np.matmul(x, y) @@ -389,12 +399,14 @@ def create_bf16_test_class(parent): if self.attrs['trans_x'] is True and self.attrs['trans_y'] is True: self.dx = self.matmul_grad(self.y_fp32, True, dout, True) self.dy = self.matmul_grad(dout, True, self.x_fp32, True) - elif self.attrs['trans_x'] is True and self.attrs[ - 'trans_y'] is False: + elif ( + self.attrs['trans_x'] is True and self.attrs['trans_y'] is False + ): self.dx = self.matmul_grad(self.y_fp32, False, dout, True) self.dy = self.matmul_grad(self.x_fp32, False, dout, False) - elif self.attrs['trans_x'] is False and self.attrs[ - 'trans_y'] is True: + elif ( + self.attrs['trans_x'] is False and self.attrs['trans_y'] is True + ): self.dx = self.matmul_grad(dout, False, self.y_fp32, False) self.dy = self.matmul_grad(dout, True, self.x_fp32, False) else: @@ -405,21 +417,25 @@ def create_bf16_test_class(parent): x_reduce_axis = [] y_reduce_axis = [] for index, (first, second) in enumerate( - zip(x_shape[0:-2], self.dx.shape[0:-2])): + zip(x_shape[0:-2], self.dx.shape[0:-2]) + ): if first != second: x_reduce_axis.append(index) for index, (first, second) in enumerate( - zip(y_shape[0:-2], self.dy.shape[0:-2])): + zip(y_shape[0:-2], self.dy.shape[0:-2]) + ): if first != second: y_reduce_axis.append(index) if x_reduce_axis: - self.dx = self.dx.sum(axis=tuple(x_reduce_axis), - keepdims=True) + self.dx = self.dx.sum( + axis=tuple(x_reduce_axis), keepdims=True + ) if y_reduce_axis: - self.dy = self.dy.sum(axis=tuple(y_reduce_axis), - keepdims=True) + self.dy = self.dy.sum( + axis=tuple(y_reduce_axis), keepdims=True + ) # after multiplying with vector one dimension is deleted from tensor if len(x_shape) == 2 and x_shape[0] == 1: @@ -456,84 +472,83 @@ create_bf16_test_class(TestMatMulV2MatrixXMatrix2Dx5DOneDNNOp) class TestMatMulV2OpTransposeReshapeEmptyFloat( - TestMatMulOpTransposeReshapeEmptyFloat): - + TestMatMulOpTransposeReshapeEmptyFloat +): def set_op_type(self): self.op_type = "matmul_v2" class TestMatMulV2OpTransposeReshapeBasicFloat( - TestMatMulOpTransposeReshapeBasicFloat): - + TestMatMulOpTransposeReshapeBasicFloat +): def set_op_type(self): self.op_type = "matmul_v2" class TestMatMulV2OpTransposeReshapeOtherDimFloat( - TestMatMulOpTransposeReshapeOtherDimFloat): - + TestMatMulOpTransposeReshapeOtherDimFloat +): def set_op_type(self): self.op_type = "matmul_v2" class TestMatMulV2OpReshapeTranspose(TestReshapeTransposeMatMulOp): - def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose4DXFloat( - TestReshapeTransposeMatMulOp4DXFloat): - + TestReshapeTransposeMatMulOp4DXFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose4DYFloat( - TestReshapeTransposeMatMulOp4DYFloat): - + TestReshapeTransposeMatMulOp4DYFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose4DXYFloat( - TestReshapeTransposeMatMulOp4DXYFloat): - + TestReshapeTransposeMatMulOp4DXYFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose2DXFloat( - TestReshapeTransposeMatMulOp2DXFloat): - + TestReshapeTransposeMatMulOp2DXFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose2DYFloat( - TestReshapeTransposeMatMulOp2DYFloat): - + TestReshapeTransposeMatMulOp2DYFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose3DXFloat( - TestReshapeTransposeMatMulOp3DXFloat): - + TestReshapeTransposeMatMulOp3DXFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose3DYFloat( - TestReshapeTransposeMatMulOp3DYFloat): - + TestReshapeTransposeMatMulOp3DYFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_mul_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_mul_int8_mkldnn_op.py index 944cab82c90..9011b1229bd 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_mul_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_mul_int8_mkldnn_op.py @@ -17,17 +17,16 @@ import numpy as np import paddle import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci + ''' test case for s8 * s8 ''' @skip_check_grad_ci( - reason= - "mul_mkldnn_op does not implement grad operator, check_grad is not required." + reason="mul_mkldnn_op does not implement grad operator, check_grad is not required." ) class TestMKLDNNMulOpS8S8(OpTest): - def setUp(self): self.op_type = "mul" self.init_kernel_type() @@ -65,10 +64,9 @@ class TestMKLDNNMulOpS8S8(OpTest): quant_B = np.round(B_data * self.scale_y[0]).astype(np.int_) output = np.dot(A_data, quant_B) - scale_output_shift = (self.scale_out) / \ - (self.scale_x * self.scale_y[0]) + scale_output_shift = (self.scale_out) / (self.scale_x * self.scale_y[0]) - if (self.force_fp32): + if self.force_fp32: output = (output * scale_output_shift).astype(self.dsttype) else: output = np.round(output * scale_output_shift).astype(self.dsttype) @@ -78,9 +76,9 @@ class TestMKLDNNMulOpS8S8(OpTest): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(core.CPUPlace(), - atol=0, - check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), atol=0, check_dygraph=False + ) ''' @@ -89,7 +87,6 @@ class TestMKLDNNMulOpS8S8(OpTest): class TestMKLDNNMulOpS8U8(TestMKLDNNMulOpS8S8): - def init_data_type(self): self.srctype = np.uint8 self.dsttype = np.float32 if self.force_fp32 else np.int8 @@ -101,7 +98,6 @@ class TestMKLDNNMulOpS8U8(TestMKLDNNMulOpS8S8): class TestMKLDNNMulOpS8S8WithFlatten(TestMKLDNNMulOpS8S8): - def setUp(self): self.op_type = "mul" self.init_kernel_type() @@ -128,8 +124,9 @@ class TestMKLDNNMulOpS8S8WithFlatten(TestMKLDNNMulOpS8S8): else: A_data = np.random.randint(0, 127, (3, 4, 4, 3)).astype(np.uint8) - B_data = np.random.uniform(-127, 127, - (2, 6, 1, 2, 3)).astype(np.float32) + B_data = np.random.uniform(-127, 127, (2, 6, 1, 2, 3)).astype( + np.float32 + ) A_data_reshape = A_data.reshape(3 * 4, 4 * 3) B_data_reshape = B_data.reshape(2 * 6, 1 * 2 * 3) @@ -137,10 +134,9 @@ class TestMKLDNNMulOpS8S8WithFlatten(TestMKLDNNMulOpS8S8): quant_B = np.round(B_data_reshape * self.scale_y[0]).astype(np.int_) output = np.dot(A_data_reshape, quant_B) - scale_output_shift = (self.scale_out) / \ - (self.scale_x * self.scale_y[0]) + scale_output_shift = (self.scale_out) / (self.scale_x * self.scale_y[0]) - if (self.force_fp32): + if self.force_fp32: output = (output * scale_output_shift).astype(self.dsttype) else: output = np.round(output * scale_output_shift).astype(self.dsttype) @@ -157,7 +153,6 @@ class TestMKLDNNMulOpS8S8WithFlatten(TestMKLDNNMulOpS8S8): class TestMKLDNNMulOpS8U8WithFlatten(TestMKLDNNMulOpS8S8WithFlatten): - def init_data_type(self): self.srctype = np.uint8 self.dsttype = np.float32 if self.force_fp32 else np.int8 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_mul_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_mul_mkldnn_op.py index 85a17094fdc..0752d4a0104 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_mul_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_mul_mkldnn_op.py @@ -16,12 +16,15 @@ import unittest import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16, OpTestTool +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + OpTestTool, +) @OpTestTool.skip_if_not_cpu_bf16() class TestMulOneDNNOp(OpTest): - def setUp(self): self.op_type = "mul" self.attrs = {'use_mkldnn': True} @@ -37,8 +40,10 @@ class TestMulOneDNNOp(OpTest): self.inputs = {'X': self.x, 'Y': self.y} - output = np.dot(np.reshape(self.x_fp32, self.np_x_shape), - np.reshape(self.y_fp32, self.np_y_shape)) + output = np.dot( + np.reshape(self.x_fp32, self.np_x_shape), + np.reshape(self.y_fp32, self.np_y_shape), + ) self.outputs = {'Out': np.reshape(output, self.out_shape)} def init_shapes_and_attrs(self): @@ -67,7 +72,6 @@ class TestMulOneDNNOp(OpTest): class TestMulXNumColDims2OneDNNOp(TestMulOneDNNOp): - def init_shapes_and_attrs(self): self.x_shape = (6, 7, 5) self.y_shape = (5, 21) @@ -81,7 +85,6 @@ class TestMulXNumColDims2OneDNNOp(TestMulOneDNNOp): class TestMulYNumColDims2OneDNNOp(TestMulOneDNNOp): - def init_shapes_and_attrs(self): self.x_shape = (20, 6) self.y_shape = (2, 3, 21) @@ -95,7 +98,6 @@ class TestMulYNumColDims2OneDNNOp(TestMulOneDNNOp): class TestMulYAndXNumColDims2OneDNNOp(TestMulOneDNNOp): - def init_shapes_and_attrs(self): self.x_shape = (10, 5, 6) self.y_shape = (2, 3, 21) @@ -110,7 +112,6 @@ class TestMulYAndXNumColDims2OneDNNOp(TestMulOneDNNOp): class TestMulBF16OneDNNOp(TestMulOneDNNOp): - def init_inputs_dtype(self): self.x = convert_float_to_uint16(self.x) self.y = convert_float_to_uint16(self.y) @@ -131,28 +132,34 @@ class TestMulBF16OneDNNOp(TestMulOneDNNOp): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ['X', 'Y'], + core.CPUPlace(), + ['X', 'Y'], 'Out', user_defined_grads=[self.dx, self.dy], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) def test_check_grad_ingore_x(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ['Y'], + core.CPUPlace(), + ['Y'], 'Out', set('X'), user_defined_grads=[self.dy], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) def test_check_grad_ingore_y(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ['X'], + core.CPUPlace(), + ['X'], 'Out', set('Y'), user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_multi_gru_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_multi_gru_mkldnn_op.py index a5f4583c3e5..273138496fc 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_multi_gru_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_multi_gru_mkldnn_op.py @@ -15,33 +15,55 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest -from paddle.fluid.tests.unittests.test_fusion_gru_op import fusion_gru, ACTIVATION +from paddle.fluid.tests.unittests.test_fusion_gru_op import ( + fusion_gru, + ACTIVATION, +) def multi_gru( - x, # T x M - lod, # 1 x N - h0, # N x D - wx, # M x 3D - wh, # D x 3D - bias, # 1 x 3D - origin_mode, - layers): + x, # T x M + lod, # 1 x N + h0, # N x D + wx, # M x 3D + wh, # D x 3D + bias, # 1 x 3D + origin_mode, + layers, +): act_state = ACTIVATION['tanh'] act_gate = ACTIVATION['sigmoid'] input = x for i in range(0, layers * 2, 2): - _, _, _, gru1_out = fusion_gru(input, lod, h0[i], wx[i], wh[i], bias[i], - False, origin_mode, act_state, act_gate) - _, _, _, gru2_out = fusion_gru(input, lod, h0[i + 1], wx[i + 1], - wh[i + 1], bias[i + 1], True, - origin_mode, act_state, act_gate) + _, _, _, gru1_out = fusion_gru( + input, + lod, + h0[i], + wx[i], + wh[i], + bias[i], + False, + origin_mode, + act_state, + act_gate, + ) + _, _, _, gru2_out = fusion_gru( + input, + lod, + h0[i + 1], + wx[i + 1], + wh[i + 1], + bias[i + 1], + True, + origin_mode, + act_state, + act_gate, + ) input = np.concatenate((gru1_out, gru2_out), axis=1) return input class TestMultiGruMkldnnOp(OpTest): - def set_confs(self): pass @@ -96,17 +118,22 @@ class TestMultiGruMkldnnOp(OpTest): wx.append(np.random.rand(IC, 3 * OC).astype('float32')) wh.append(np.random.rand(OC, 3 * OC).astype('float32')) bias.append( - np.random.rand(1, 3 * OC).astype('float32') if self. - with_bias else np.zeros((1, 3 * OC), dtype='float32')) + np.random.rand(1, 3 * OC).astype('float32') + if self.with_bias + else np.zeros((1, 3 * OC), dtype='float32') + ) h0.append(np.zeros((N, OC), dtype='float32')) - self.inputs['WeightX'] = [('wx' + str(i), wx[i]) - for i in range(self.layers * 2)] - self.inputs['WeightH'] = [('wh' + str(i), wh[i]) - for i in range(self.layers * 2)] + self.inputs['WeightX'] = [ + ('wx' + str(i), wx[i]) for i in range(self.layers * 2) + ] + self.inputs['WeightH'] = [ + ('wh' + str(i), wh[i]) for i in range(self.layers * 2) + ] if self.with_bias: - self.inputs['Bias'] = [('b' + str(i), bias[i]) - for i in range(self.layers * 2)] + self.inputs['Bias'] = [ + ('b' + str(i), bias[i]) for i in range(self.layers * 2) + ] if is_int8: s8_max = 127.0 @@ -114,38 +141,54 @@ class TestMultiGruMkldnnOp(OpTest): for layer in range(self.layers): OC = self.OCs[layer] for j in range(2): - scale_ur = s8_max / np.max(np.abs( - np.concatenate([ - wx[2 * layer + j][:, :2 * OC], - wh[2 * layer + j].flatten()[:2 * OC * OC].reshape( - OC, 2 * OC) - ], - axis=0)), - axis=0) - scale_o = s8_max / np.max(np.abs( - np.concatenate([ - wx[2 * layer + j][:, 2 * OC:], - wh[2 * layer + j].flatten()[2 * OC * OC:].reshape( - OC, OC) - ], - axis=0)), - axis=0) + scale_ur = s8_max / np.max( + np.abs( + np.concatenate( + [ + wx[2 * layer + j][:, : 2 * OC], + wh[2 * layer + j] + .flatten()[: 2 * OC * OC] + .reshape(OC, 2 * OC), + ], + axis=0, + ) + ), + axis=0, + ) + scale_o = s8_max / np.max( + np.abs( + np.concatenate( + [ + wx[2 * layer + j][:, 2 * OC :], + wh[2 * layer + j] + .flatten()[2 * OC * OC :] + .reshape(OC, OC), + ], + axis=0, + ) + ), + axis=0, + ) scale_weights.append( - np.concatenate([scale_ur, scale_o]).astype('float32')) - self.inputs['Scale_weights'] = [('w_scale' + str(i), - scale_weights[i]) - for i in range(self.layers * 2)] + np.concatenate([scale_ur, scale_o]).astype('float32') + ) + self.inputs['Scale_weights'] = [ + ('w_scale' + str(i), scale_weights[i]) + for i in range(self.layers * 2) + ] self.error_margin = 1e-1 if self.force_fp32_output else 1 - hidden_f32 = multi_gru(x_f32, self.lod, h0, wx, wh, bias, - self.origin_mode, self.layers) + hidden_f32 = multi_gru( + x_f32, self.lod, h0, wx, wh, bias, self.origin_mode, self.layers + ) if self.dtype == 'float32' or self.force_fp32_output: self.outputs = {'Hidden': (hidden_f32, self.lod)} else: hidden_u8 = np.rint(hidden_f32 * scale_data + shift_data).astype( - np.uint8) + np.uint8 + ) self.outputs = {'Hidden': (hidden_u8, self.lod)} self.attrs = { @@ -166,13 +209,11 @@ class TestMultiGruMkldnnOp(OpTest): class TestMultiGruMkldnnOpNoBias(TestMultiGruMkldnnOp): - def set_confs(self): self.with_bias = False class TestMultiGruMkldnnOpLayers2(TestMultiGruMkldnnOp): - def set_confs(self): self.layers = 2 self.ICs = [2, 6] @@ -180,7 +221,6 @@ class TestMultiGruMkldnnOpLayers2(TestMultiGruMkldnnOp): class TestMultiGruMkldnnOpLayers3(TestMultiGruMkldnnOp): - def set_confs(self): self.layers = 3 self.ICs = [2, 6, 12] @@ -188,71 +228,64 @@ class TestMultiGruMkldnnOpLayers3(TestMultiGruMkldnnOp): class TestMultiGruMkldnnOpOriginMode(TestMultiGruMkldnnOp): - def set_confs(self): self.origin_mode = True class TestMultiGruMkldnnInt8Op(TestMultiGruMkldnnOp): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpForceFP32Output(TestMultiGruMkldnnInt8Op): - def set_force_fp32_output(self): self.force_fp32_output = True class TestMultiGruMkldnnInt8OpNoBias(TestMultiGruMkldnnOpNoBias): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpNoBiasForceFP32Output( - TestMultiGruMkldnnInt8OpNoBias): - + TestMultiGruMkldnnInt8OpNoBias +): def set_force_fp32_output(self): self.force_fp32_output = True class TestMultiGruMkldnnInt8OpLayers2(TestMultiGruMkldnnOpLayers2): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpLayers2ForceFP32Output( - TestMultiGruMkldnnInt8OpLayers2): - + TestMultiGruMkldnnInt8OpLayers2 +): def set_force_fp32_output(self): self.force_fp32_output = True class TestMultiGruMkldnnInt8OpLayers3(TestMultiGruMkldnnOpLayers3): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpLayers3ForceFP32Output( - TestMultiGruMkldnnInt8OpLayers3): - + TestMultiGruMkldnnInt8OpLayers3 +): def set_force_fp32_output(self): self.force_fp32_output = True class TestMultiGruMkldnnInt8OpOriginMode(TestMultiGruMkldnnOpOriginMode): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpOriginModeForceFP32Output( - TestMultiGruMkldnnInt8OpOriginMode): - + TestMultiGruMkldnnInt8OpOriginMode +): def set_force_fp32_output(self): self.force_fp32_output = True diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_mkldnn_op.py index f60050509e6..e6a53f122bb 100755 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_mkldnn_op.py @@ -18,12 +18,9 @@ from paddle.fluid.tests.unittests.op_test import OpTest from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -def nearest_neighbor_interp_mkldnn_np(X, - out_h, - out_w, - out_size=None, - actual_shape=None, - data_layout='NCHW'): +def nearest_neighbor_interp_mkldnn_np( + X, out_h, out_w, out_size=None, actual_shape=None, data_layout='NCHW' +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -37,9 +34,9 @@ def nearest_neighbor_interp_mkldnn_np(X, n, c, in_h, in_w = X.shape fh = fw = 0.0 - if (out_h > 1): + if out_h > 1: fh = out_h * 1.0 / in_h - if (out_w > 1): + if out_w > 1: fw = out_w * 1.0 / in_w out = np.zeros((n, c, out_h, out_w)) @@ -58,7 +55,6 @@ def nearest_neighbor_interp_mkldnn_np(X, @skip_check_grad_ci(reason="Haven not implement interpolate grad kernel.") class TestNearestInterpMKLDNNOp(OpTest): - def init_test_case(self): pass @@ -86,8 +82,9 @@ class TestNearestInterpMKLDNNOp(OpTest): input_np = np.random.random(self.input_shape).astype(self.dtype) else: init_low, init_high = (-5, 5) if self.dtype == np.int8 else (0, 10) - input_np = np.random.randint(init_low, init_high, - self.input_shape).astype(self.dtype) + input_np = np.random.randint( + init_low, init_high, self.input_shape + ).astype(self.dtype) if self.data_layout == "NCHW": in_h = self.input_shape[2] @@ -103,10 +100,14 @@ class TestNearestInterpMKLDNNOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_mkldnn_np(input_np, out_h, out_w, - self.out_size, - self.actual_shape, - self.data_layout) + output_np = nearest_neighbor_interp_mkldnn_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: @@ -119,7 +120,7 @@ class TestNearestInterpMKLDNNOp(OpTest): 'out_w': self.out_w, 'scale': self.scale, 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': output_np} @@ -128,7 +129,6 @@ class TestNearestInterpMKLDNNOp(OpTest): class TestNearestInterpOpMKLDNNNHWC(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 27 @@ -138,40 +138,36 @@ class TestNearestInterpOpMKLDNNNHWC(TestNearestInterpMKLDNNOp): class TestNearestNeighborInterpMKLDNNCase2(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 1. + self.scale = 1.0 class TestNearestNeighborInterpCase3(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 128 - self.scale = 0. + self.scale = 0.0 class TestNearestNeighborInterpCase4(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") class TestNearestNeighborInterpSame(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 def create_test_class(parent): @@ -180,12 +176,10 @@ def create_test_class(parent): ''' class TestInt8Case(parent): - def init_data_type(self): self.dtype = np.int8 class TestUint8Case(parent): - def init_data_type(self): self.dtype = np.uint8 @@ -204,5 +198,6 @@ create_test_class(TestNearestNeighborInterpSame) if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_v2_mkldnn_op.py index 6e5601e91b0..6d5480a03f5 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_v2_mkldnn_op.py @@ -14,16 +14,17 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -def nearest_neighbor_interp_mkldnn_np(X, - out_h, - out_w, - out_size=None, - actual_shape=None, - data_layout='NCHW'): +def nearest_neighbor_interp_mkldnn_np( + X, out_h, out_w, out_size=None, actual_shape=None, data_layout='NCHW' +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -37,9 +38,9 @@ def nearest_neighbor_interp_mkldnn_np(X, n, c, in_h, in_w = X.shape fh = fw = 0.0 - if (out_h > 1): + if out_h > 1: fh = out_h * 1.0 / in_h - if (out_w > 1): + if out_w > 1: fw = out_w * 1.0 / in_w out = np.zeros((n, c, out_h, out_w)) @@ -59,7 +60,6 @@ def nearest_neighbor_interp_mkldnn_np(X, @skip_check_grad_ci(reason="Haven not implement interpolate grad kernel.") @OpTestTool.skip_if_not_cpu_bf16() class TestNearestInterpV2MKLDNNOp(OpTest): - def init_test_case(self): pass @@ -87,8 +87,9 @@ class TestNearestInterpV2MKLDNNOp(OpTest): input_np = np.random.random(self.input_shape).astype(self.dtype) else: init_low, init_high = (-5, 5) if self.dtype == np.int8 else (0, 10) - input_np = np.random.randint(init_low, init_high, - self.input_shape).astype(self.dtype) + input_np = np.random.randint( + init_low, init_high, self.input_shape + ).astype(self.dtype) if self.data_layout == "NCHW": in_h = self.input_shape[2] @@ -118,10 +119,14 @@ class TestNearestInterpV2MKLDNNOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_mkldnn_np(input_np, out_h, out_w, - self.out_size, - self.actual_shape, - self.data_layout) + output_np = nearest_neighbor_interp_mkldnn_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.data_layout, + ) if isinstance(self.scale, float): self.scale = [self.scale] @@ -140,7 +145,7 @@ class TestNearestInterpV2MKLDNNOp(OpTest): 'out_w': self.out_w, 'scale': self.scale, 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': output_np} @@ -149,7 +154,6 @@ class TestNearestInterpV2MKLDNNOp(OpTest): class TestNearestInterpOpV2MKLDNNNHWC(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 27 @@ -159,7 +163,6 @@ class TestNearestInterpOpV2MKLDNNNHWC(TestNearestInterpV2MKLDNNOp): class TestNearestNeighborInterpV2MKLDNNCase2(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 @@ -167,7 +170,6 @@ class TestNearestNeighborInterpV2MKLDNNCase2(TestNearestInterpV2MKLDNNOp): class TestNearestNeighborInterpV2MKLDNNCase3(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -176,7 +178,6 @@ class TestNearestNeighborInterpV2MKLDNNCase3(TestNearestInterpV2MKLDNNOp): class TestNearestNeighborInterpV2MKLDNNCase4(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -186,7 +187,6 @@ class TestNearestNeighborInterpV2MKLDNNCase4(TestNearestInterpV2MKLDNNOp): class TestNearestNeighborInterpV2MKLDNNSame(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 @@ -200,17 +200,14 @@ def create_test_class(parent): ''' class TestBf16Case(parent): - def init_data_type(self): self.dtype = np.uint16 class TestInt8Case(parent): - def init_data_type(self): self.dtype = np.int8 class TestUint8Case(parent): - def init_data_type(self): self.dtype = np.uint8 @@ -231,5 +228,6 @@ create_test_class(TestNearestNeighborInterpV2MKLDNNSame) if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_onnx_format_quantization_mobilenetv1.py b/python/paddle/fluid/tests/unittests/mkldnn/test_onnx_format_quantization_mobilenetv1.py index ae4a8c16d67..1b27a39f2e9 100755 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_onnx_format_quantization_mobilenetv1.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_onnx_format_quantization_mobilenetv1.py @@ -74,13 +74,14 @@ def process_image(sample, mode, color_jitter, rotate): return img, sample[1] -def _reader_creator(file_list, - mode, - shuffle=False, - color_jitter=False, - rotate=False, - data_dir=DATA_DIR): - +def _reader_creator( + file_list, + mode, + shuffle=False, + color_jitter=False, + rotate=False, + data_dir=DATA_DIR, +): def reader(): with open(file_list) as flist: full_lines = [line.strip() for line in flist] @@ -95,10 +96,9 @@ def _reader_creator(file_list, continue yield img_path, int(label) - mapper = functools.partial(process_image, - mode=mode, - color_jitter=color_jitter, - rotate=rotate) + mapper = functools.partial( + process_image, mode=mode, color_jitter=color_jitter, rotate=rotate + ) return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE) @@ -109,11 +109,11 @@ def val(data_dir=DATA_DIR): class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.int8_download = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.int8_download) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.int8_download + ) self.data_cache_folder = '' data_urls = [] data_md5s = [] @@ -126,27 +126,32 @@ class TestPostTrainingQuantization(unittest.TestCase): 'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partab' ) data_md5s.append('1e9f15f64e015e58d6f9ec3210ed18b5') - self.data_cache_folder = self.download_data(data_urls, data_md5s, - "full_data", False) + self.data_cache_folder = self.download_data( + data_urls, data_md5s, "full_data", False + ) else: data_urls.append( 'http://paddle-inference-dist.bj.bcebos.com/int8/calibration_test_data.tar.gz' ) data_md5s.append('1b6c1c434172cca1bf9ba1e4d7a3157d') - self.data_cache_folder = self.download_data(data_urls, data_md5s, - "small_data", False) + self.data_cache_folder = self.download_data( + data_urls, data_md5s, "small_data", False + ) # reader/decorator.py requires the relative path to the data folder if not os.path.exists("./data/ILSVRC2012"): - cmd = 'rm -rf {0} && ln -s {1} {0}'.format("data", - self.data_cache_folder) + cmd = 'rm -rf {0} && ln -s {1} {0}'.format( + "data", self.data_cache_folder + ) os.system(cmd) self.batch_size = 1 if os.environ.get('DATASET') == 'full' else 50 - self.sample_iterations = 50 if os.environ.get( - 'DATASET') == 'full' else 2 - self.infer_iterations = 50000 if os.environ.get( - 'DATASET') == 'full' else 2 + self.sample_iterations = ( + 50 if os.environ.get('DATASET') == 'full' else 2 + ) + self.infer_iterations = ( + 50000 if os.environ.get('DATASET') == 'full' else 2 + ) self.int8_model = "post_training_quantization" print("self.int8_model: ", self.int8_model) @@ -159,7 +164,8 @@ class TestPostTrainingQuantization(unittest.TestCase): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_data(self, data_urls, data_md5s, folder_name, is_model=True): @@ -171,13 +177,15 @@ class TestPostTrainingQuantization(unittest.TestCase): download(data_urls[i], self.int8_download, data_md5s[i]) file_names.append(data_urls[i].split('/')[-1]) - zip_path = os.path.join(self.cache_folder, - 'full_imagenet_val.tar.gz') + zip_path = os.path.join( + self.cache_folder, 'full_imagenet_val.tar.gz' + ) if not os.path.exists(zip_path): cat_command = 'cat' for file_name in file_names: - cat_command += ' ' + os.path.join(self.cache_folder, - file_name) + cat_command += ' ' + os.path.join( + self.cache_folder, file_name + ) cat_command += ' > ' + zip_path os.system(cat_command) @@ -193,11 +201,9 @@ class TestPostTrainingQuantization(unittest.TestCase): def download_model(self): pass - def run_program(self, - model_path, - batch_size, - infer_iterations, - is_quantized_model=False): + def run_program( + self, model_path, batch_size, infer_iterations, is_quantized_model=False + ): image_shape = [3, 224, 224] config = paddle.inference.Config(model_path) config.disable_gpu() @@ -223,8 +229,9 @@ class TestPostTrainingQuantization(unittest.TestCase): cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(image_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(image_shape) for x in data]).astype( + "float32" + ) label = np.array([x[1] for x in data]).astype("int64") label = label.reshape([-1, 1]) @@ -252,80 +259,106 @@ class TestPostTrainingQuantization(unittest.TestCase): acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) - def generate_quantized_model(self, - model_path, - quantizable_op_type, - algo="KL", - round_type="round", - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - onnx_format=False): + def generate_quantized_model( + self, + model_path, + quantizable_op_type, + algo="KL", + round_type="round", + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + onnx_format=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) val_reader = val() - ptq = PostTrainingQuantization(executor=exe, - sample_generator=val_reader, - model_dir=model_path, - algo=algo, - quantizable_op_type=quantizable_op_type, - round_type=round_type, - is_full_quantize=is_full_quantize, - optimize_model=is_optimize_model, - onnx_format=onnx_format, - is_use_cache_file=is_use_cache_file) + ptq = PostTrainingQuantization( + executor=exe, + sample_generator=val_reader, + model_dir=model_path, + algo=algo, + quantizable_op_type=quantizable_op_type, + round_type=round_type, + is_full_quantize=is_full_quantize, + optimize_model=is_optimize_model, + onnx_format=onnx_format, + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() ptq.save_quantized_model(self.int8_model) - def run_test(self, - model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=True): + def run_test( + self, + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=True, + ): infer_iterations = self.infer_iterations batch_size = self.batch_size sample_iterations = self.sample_iterations model_cache_folder = self.download_data(data_urls, data_md5s, model) - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model, sample_iterations * batch_size)) - self.generate_quantized_model(os.path.join(model_cache_folder, "model"), - quantizable_op_type, algo, round_type, - is_full_quantize, is_use_cache_file, - is_optimize_model, onnx_format) + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model, sample_iterations * batch_size + ) + ) + self.generate_quantized_model( + os.path.join(model_cache_folder, "model"), + quantizable_op_type, + algo, + round_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + onnx_format, + ) - print("Start FP32 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) + print( + "Start FP32 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) (fp32_throughput, fp32_latency, fp32_acc1) = self.run_program( - os.path.join(model_cache_folder, "model"), batch_size, - infer_iterations) + os.path.join(model_cache_folder, "model"), + batch_size, + infer_iterations, + ) - print("Start INT8 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) - (int8_throughput, int8_latency, - int8_acc1) = self.run_program(self.int8_model, - batch_size, - infer_iterations, - is_quantized_model=True) + print( + "Start INT8 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1) = self.run_program( + self.int8_model, + batch_size, + infer_iterations, + is_quantized_model=True, + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}." - .format(model, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.".format( + model, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n" - .format(model, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n".format( + model, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = int8_latency - fp32_latency @@ -333,7 +366,6 @@ class TestPostTrainingQuantization(unittest.TestCase): class TestMKLDNNInt8ForResnet50AvgONNXFormat(TestPostTrainingQuantization): - def test_onnx_format_avg_resnet50(self): model = "resnet50" algo = "avg" @@ -351,17 +383,19 @@ class TestMKLDNNInt8ForResnet50AvgONNXFormat(TestPostTrainingQuantization): is_use_cache_file = False is_optimize_model = False diff_threshold = 0 - self.run_test(model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=True) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_bf16_mkldnn_op.py index d9202534476..81ca727b638 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_bf16_mkldnn_op.py @@ -15,15 +15,23 @@ import unittest import numpy as np import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 -from paddle.fluid.tests.unittests.test_pool2d_op import TestPool2D_Op_Mixin, max_pool2D_forward_naive -from paddle.fluid.tests.unittests.npu.test_pool2d_op_npu import pool2d_backward_navie as pool2d_backward_naive +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) +from paddle.fluid.tests.unittests.test_pool2d_op import ( + TestPool2D_Op_Mixin, + max_pool2D_forward_naive, +) +from paddle.fluid.tests.unittests.npu.test_pool2d_op_npu import ( + pool2d_backward_navie as pool2d_backward_naive, +) from paddle import enable_static @OpTestTool.skip_if_not_cpu_bf16() class TestPoolBf16MklDNNOpGrad(TestPool2D_Op_Mixin, OpTest): - def init_kernel_type(self): self.use_mkldnn = True @@ -35,11 +43,17 @@ class TestPoolBf16MklDNNOpGrad(TestPool2D_Op_Mixin, OpTest): self.attrs['mkldnn_data_type'] = "bfloat16" self.x_fp32 = np.random.random(self.shape).astype(np.float32) - output = self.pool2D_forward_naive(self.x_fp32, self.ksize, - self.strides, self.paddings, - self.global_pool, self.ceil_mode, - self.exclusive, self.adaptive, - "float32").astype(np.float32) + output = self.pool2D_forward_naive( + self.x_fp32, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + "float32", + ).astype(np.float32) self.inputs = {'X': convert_float_to_uint16(self.x_fp32)} self.outputs = {'Out': convert_float_to_uint16(output)} @@ -48,27 +62,27 @@ class TestPoolBf16MklDNNOpGrad(TestPool2D_Op_Mixin, OpTest): self.check_output_with_place(core.CPUPlace()) def test_check_grad(self): - x_grad = pool2d_backward_naive(self.x_fp32, - ksize=self.ksize, - strides=self.strides, - paddings=self.paddings, - global_pool=self.global_pool, - ceil_mode=False, - exclusive=self.exclusive, - adaptive=self.adaptive, - data_format=self.data_format, - pool_type=self.pool_type, - padding_algorithm=self.padding_algorithm) + x_grad = pool2d_backward_naive( + self.x_fp32, + ksize=self.ksize, + strides=self.strides, + paddings=self.paddings, + global_pool=self.global_pool, + ceil_mode=False, + exclusive=self.exclusive, + adaptive=self.adaptive, + data_format=self.data_format, + pool_type=self.pool_type, + padding_algorithm=self.padding_algorithm, + ) x_grad = x_grad / np.prod(self.outputs['Out'].shape) - self.check_grad_with_place(core.CPUPlace(), - set(['X']), - 'Out', - user_defined_grads=[x_grad]) + self.check_grad_with_place( + core.CPUPlace(), set(['X']), 'Out', user_defined_grads=[x_grad] + ) @OpTestTool.skip_if_not_cpu_bf16() class TestPoolBf16MklDNNOp(TestPool2D_Op_Mixin, OpTest): - def init_kernel_type(self): self.use_mkldnn = True @@ -77,11 +91,19 @@ class TestPoolBf16MklDNNOp(TestPool2D_Op_Mixin, OpTest): self.dtype = np.uint16 input = np.random.random(self.shape).astype(np.float32) - output = (self.pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, - "float32")).astype(np.float32) + output = ( + self.pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + "float32", + ) + ).astype(np.float32) self.inputs = {'X': convert_float_to_uint16(input)} self.outputs = {'Out': convert_float_to_uint16(output)} @@ -94,7 +116,6 @@ class TestPoolBf16MklDNNOp(TestPool2D_Op_Mixin, OpTest): class TestCase1Avg(TestPoolBf16MklDNNOp): - def init_test_case(self): self.shape = [2, 3, 7, 7] self.ksize = [3, 3] @@ -109,7 +130,6 @@ class TestCase1Avg(TestPoolBf16MklDNNOp): class TestCase2Avg(TestPoolBf16MklDNNOp): - def init_test_case(self): self.shape = [2, 3, 7, 7] self.ksize = [3, 3] @@ -124,28 +144,24 @@ class TestCase2Avg(TestPoolBf16MklDNNOp): class TestCase0Max(TestPoolBf16MklDNNOp): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase1Max(TestCase1Avg): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase2Max(TestCase2Avg): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase1PadZeroExclusiveAvgGrad(TestPoolBf16MklDNNOpGrad): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -164,27 +180,23 @@ class TestCase1PadZeroExclusiveAvgGrad(TestPoolBf16MklDNNOpGrad): class TestCase2PadOneNonExclusiveAvgGrad(TestCase1PadZeroExclusiveAvgGrad): - def init_exclusive(self): self.exclusive = False class TestCase0InitialMaxGrad(TestPoolBf16MklDNNOpGrad): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase1PadZeroExclusiveMaxGrad(TestCase1PadZeroExclusiveAvgGrad): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase2PadOneNonExclusiveMaxGrad(TestCase2PadOneNonExclusiveAvgGrad): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_int8_mkldnn_op.py index 27595afac1f..f21891f35e7 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_int8_mkldnn_op.py @@ -17,11 +17,13 @@ import numpy as np import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest -from paddle.fluid.tests.unittests.test_pool2d_op import TestPool2D_Op, max_pool2D_forward_naive +from paddle.fluid.tests.unittests.test_pool2d_op import ( + TestPool2D_Op, + max_pool2D_forward_naive, +) class TestPool2DMKLDNNInt8_Op(TestPool2D_Op): - def init_kernel_type(self): self.use_mkldnn = True @@ -30,29 +32,38 @@ class TestPool2DMKLDNNInt8_Op(TestPool2D_Op): def setUp(self): TestPool2D_Op.setUp(self) - assert self.dtype in [np.int8, - np.uint8], 'Dtype should be int8 or uint8' + assert self.dtype in [ + np.int8, + np.uint8, + ], 'Dtype should be int8 or uint8' input = np.random.randint(0, 100, self.shape).astype(self.dtype) - output = (self.pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, - self.dtype)).astype(self.dtype) + output = ( + self.pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.dtype, + ) + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} self.outputs = {'Out': output} def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(core.CPUPlace(), - atol=1e-5, - check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), atol=1e-5, check_dygraph=False + ) def test_check_grad(self): pass class TestCase1Avg(TestPool2DMKLDNNInt8_Op): - def init_test_case(self): self.shape = [2, 3, 7, 7] self.ksize = [3, 3] @@ -67,7 +78,6 @@ class TestCase1Avg(TestPool2DMKLDNNInt8_Op): class TestCase2Avg(TestPool2DMKLDNNInt8_Op): - def init_test_case(self): self.shape = [2, 3, 7, 7] self.ksize = [3, 3] @@ -82,35 +92,29 @@ class TestCase2Avg(TestPool2DMKLDNNInt8_Op): class TestCase0Max(TestPool2DMKLDNNInt8_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase1Max(TestCase1Avg): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase2Max(TestCase2Avg): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive def create_test_s8_u8_class(parent): - class TestS8Case(parent): - def init_data_type(self): self.dtype = np.int8 class TestU8Case(parent): - def init_data_type(self): self.dtype = np.uint8 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_mkldnn_op.py index 8a201a2e1e4..e4e8ecb5bef 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_mkldnn_op.py @@ -14,13 +14,19 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.test_pool2d_op import TestPool2D_Op, TestCase1, TestCase2, TestCase3, TestCase4, TestCase5, avg_pool2D_forward_naive +from paddle.fluid.tests.unittests.test_pool2d_op import ( + TestPool2D_Op, + TestCase1, + TestCase2, + TestCase3, + TestCase4, + TestCase5, + avg_pool2D_forward_naive, +) def create_test_mkldnn_use_ceil_class(parent): - class TestMKLDNNPool2DUseCeilCase(parent): - def init_kernel_type(self): self.use_mkldnn = True @@ -41,9 +47,7 @@ create_test_mkldnn_use_ceil_class(TestCase2) def create_test_mkldnn_class(parent): - class TestMKLDNNCase(parent): - def init_kernel_type(self): self.use_mkldnn = True @@ -64,7 +68,6 @@ create_test_mkldnn_class(TestCase5) class TestAvgPoolAdaptive(TestPool2D_Op): - def init_adaptive(self): self.adaptive = True @@ -87,7 +90,6 @@ class TestAvgPoolAdaptive(TestPool2D_Op): class TestAvgPoolAdaptive2(TestAvgPoolAdaptive): - def init_test_case(self): self.ksize = [2, 3] self.strides = [1, 1] @@ -97,7 +99,6 @@ class TestAvgPoolAdaptive2(TestAvgPoolAdaptive): class TestAvgPoolAdaptive3(TestAvgPoolAdaptive): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -107,7 +108,6 @@ class TestAvgPoolAdaptive3(TestAvgPoolAdaptive): class TestAsymPad(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -133,81 +133,68 @@ class TestAsymPad(TestPool2D_Op): class TestAsymPadCase1(TestAsymPad): - def init_paddings(self): self.paddings = [1, 1, 0, 0] class TestAsymPadCase2(TestAsymPad): - def init_paddings(self): self.paddings = [1, 0, 1, 2] class TestAsymPadCase3(TestAsymPad): - def init_paddings(self): self.paddings = [1, 2, 1, 2] class TestAsymPadCase4(TestAsymPad): - def init_paddings(self): self.paddings = [1, 0, 1, 2] class TestAsymPadCase5(TestAsymPad): - def init_paddings(self): self.paddings = [2, 2, 1, 2] class TestAsymPadMaxCase1(TestAsymPadCase1): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadMaxCase2(TestAsymPadCase2): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadMaxCase3(TestAsymPadCase3): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadMaxCase4(TestAsymPadCase4): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadMaxCase5(TestAsymPadCase5): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadSame(TestAsymPad): - def init_paddings(self): self.paddings = [0, 0] self.padding_algorithm = "SAME" class TestAsymPadValid(TestAsymPad): - def init_paddings(self): self.paddings = [0, 0, 0, 0] self.padding_algorithm = "VALID" class TestAsymPadValidNHWC(TestAsymPadValid): - def init_data_format(self): self.data_format = "NHWC" @@ -217,5 +204,6 @@ class TestAsymPadValidNHWC(TestAsymPadValid): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_prelu_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_prelu_mkldnn_op.py index 809d5efd1e4..1b1c79248ce 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_prelu_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_prelu_mkldnn_op.py @@ -16,7 +16,11 @@ import unittest import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) def ref_prelu(x, weight, mode): @@ -27,12 +31,14 @@ def ref_prelu(x, weight, mode): elif mode == "channel": if len(weight.shape) > 1: for i in range(x.shape[1]): - result[:, i] = np.where(x[:, i] > 0, x[:, i], - x[:, i] * weight[0, i]) + result[:, i] = np.where( + x[:, i] > 0, x[:, i], x[:, i] * weight[0, i] + ) else: for i in range(x.shape[1]): - result[:, i] = np.where(x[:, i] > 0, x[:, i], - x[:, i] * weight[i]) + result[:, i] = np.where( + x[:, i] > 0, x[:, i], x[:, i] * weight[i] + ) elif mode == "element": result = np.where(x[:] > 0, x[:], x[:] * weight) @@ -40,7 +46,6 @@ def ref_prelu(x, weight, mode): class TestPReluModeChannelOneDNNOp(OpTest): - def init_attrs(self): self.mode = "element" self.alpha = np.random.random((1, 4, 5, 5)).astype("float32") @@ -69,7 +74,6 @@ class TestPReluModeChannelOneDNNOp(OpTest): class TestPReluModeAllOneDNNOp(TestPReluModeChannelOneDNNOp): - def init_attrs(self): self.mode = "all" self.alpha = np.random.random((1, 1, 1, 1)).astype("float32") @@ -81,14 +85,12 @@ class TestPReluModeAllOneDNNOp(TestPReluModeChannelOneDNNOp): class TestPReluModeElementOneDNNOp(TestPReluModeChannelOneDNNOp): - def init_attrs(self): self.mode = "element" self.alpha = np.random.random((1, 4, 5, 5)).astype("float32") class TestPReluModeChannel3DOneDNNOp(TestPReluModeChannelOneDNNOp): - def init_attrs(self): self.mode = "channel" self.x = np.random.random((1, 100, 1)).astype("float32") @@ -96,7 +98,6 @@ class TestPReluModeChannel3DOneDNNOp(TestPReluModeChannelOneDNNOp): class TestPReluModeChannelAlpha1DOneDNNOp(TestPReluModeChannelOneDNNOp): - def init_attrs(self): self.mode = "channel" self.x = np.random.random((1, 100, 1)).astype("float32") @@ -104,7 +105,6 @@ class TestPReluModeChannelAlpha1DOneDNNOp(TestPReluModeChannelOneDNNOp): class TestPReluModeAllAlpha1DOneDNNOp(TestPReluModeAllOneDNNOp): - def init_attrs(self): self.mode = "channel" self.x = np.random.random((1, 1, 100)).astype("float32") @@ -113,14 +113,14 @@ class TestPReluModeAllAlpha1DOneDNNOp(TestPReluModeAllOneDNNOp): # BF16 TESTS def create_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestPReluBF16OneDNNOp(parent): - - def set_inputs(self, ): + def set_inputs( + self, + ): self.inputs = { 'X': convert_float_to_uint16(self.x), - 'Alpha': convert_float_to_uint16(self.alpha) + 'Alpha': convert_float_to_uint16(self.alpha), } def set_dtype_attr(self): @@ -136,12 +136,18 @@ def create_bf16_test_class(parent): elif self.mode == "channel": if len(self.alpha.shape) > 1: for i in range(self.x.shape[1]): - self.dx[:, i] = np.where(self.x[:, i] > 0, dout[:, i], - dout[:, i] * self.alpha[0, i]) + self.dx[:, i] = np.where( + self.x[:, i] > 0, + dout[:, i], + dout[:, i] * self.alpha[0, i], + ) else: for i in range(self.x.shape[1]): - self.dx[:, i] = np.where(self.x[:, i] > 0, dout[:, i], - dout[:, i] * self.alpha[i]) + self.dx[:, i] = np.where( + self.x[:, i] > 0, + dout[:, i], + dout[:, i] * self.alpha[i], + ) self.dx elif self.mode == "element": self.dx = np.where(self.x[:] > 0, dout[:], dout[:] * self.alpha) @@ -155,10 +161,12 @@ def create_bf16_test_class(parent): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X", "Alpha"], + core.CPUPlace(), + ["X", "Alpha"], "Out", user_defined_grads=[self.dx, self.dalpha], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "BF16") TestPReluBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_quantize_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_quantize_mkldnn_op.py index 52498793d58..4712cf43269 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_quantize_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_quantize_mkldnn_op.py @@ -19,7 +19,6 @@ import paddle class TestQuantizeOp(OpTest): - def setUp(self): self.op_type = 'quantize' self.scale = 255.0 @@ -38,25 +37,28 @@ class TestQuantizeOp(OpTest): def prepare_input(self): if self.is_negative: # input data values are from interval [-1.0, 1.0) - self.input = (2 * np.random.random_sample(self.input_size) - - 1).astype('float32') + self.input = ( + 2 * np.random.random_sample(self.input_size) - 1 + ).astype('float32') else: # input data values are from interval [0.0, 1.0) - self.input = (np.random.random_sample( - self.input_size)).astype('float32') + self.input = (np.random.random_sample(self.input_size)).astype( + 'float32' + ) self.inputs = {'Input': OpTest.np_dtype_to_fluid_dtype(self.input)} self.attrs = { 'Scale': self.scale, 'Shift': self.shift, 'is_negative_input': self.is_negative, - 'output_format': self.output_format + 'output_format': self.output_format, } def prepare_output(self): input_data_type = 'int8' if self.is_negative else 'uint8' - output = np.rint(self.input * self.scale + - self.shift).astype(input_data_type) + output = np.rint(self.input * self.scale + self.shift).astype( + input_data_type + ) self.outputs = {'Output': output} def test_check_output(self): @@ -89,7 +91,6 @@ class TestQuantizeOp(OpTest): class TestQuantizeOp1(TestQuantizeOp): - def set_scale(self): self.scale = 127.0 @@ -98,7 +99,6 @@ class TestQuantizeOp1(TestQuantizeOp): class TestQuantizeOp2(TestQuantizeOp): - def set_scale(self): self.scale = 255.0 @@ -109,7 +109,6 @@ class TestQuantizeOp2(TestQuantizeOp): # 2-dim input # P - positive input class TestQuantizeOpShift_NCHW_2_P(TestQuantizeOp): - def set_output_format(self): self.output_format = 'NCHW' @@ -129,7 +128,6 @@ class TestQuantizeOpShift_NCHW_2_P(TestQuantizeOp): # 2-dim input # N - negative input class TestQuantizeOpShift_NCHW_2_N(TestQuantizeOpShift_NCHW_2_P): - def set_is_negative(self): self.is_nagative = True @@ -141,63 +139,53 @@ class TestQuantizeOpShift_NCHW_2_N(TestQuantizeOpShift_NCHW_2_P): class TestQuantizeOpShift_NHWC_2_P(TestQuantizeOpShift_NCHW_2_P): - def set_output_format(self): self.output_format = 'NHWC' class TestQuantizeOpShift_NHWC_2_N(TestQuantizeOpShift_NCHW_2_N): - def set_output_format(self): self.output_format = 'NHWC' # 3-dim input class TestQuantizeOpShift_NCHW_3_P(TestQuantizeOpShift_NCHW_2_P): - def set_input_size(self): self.input_size = [2, 3, 4] class TestQuantizeOpShift_NCHW_3_N(TestQuantizeOpShift_NCHW_2_N): - def set_input_size(self): self.input_size = [2, 3, 4] class TestQuantizeOpShift_NHWC_3_P(TestQuantizeOpShift_NCHW_3_P): - def set_output_format(self): self.output_format = 'NHWC' class TestQuantizeOpShift_NHWC_3_N(TestQuantizeOpShift_NCHW_3_N): - def set_output_format(self): self.output_format = 'NHWC' # 4-dim input class TestQuantizeOpShift_NCHW_4_P(TestQuantizeOpShift_NCHW_2_P): - def set_input_size(self): self.input_size = [2, 3, 4, 5] class TestQuantizeOpShift_NCHW_4_N(TestQuantizeOpShift_NCHW_2_N): - def set_input_size(self): self.input_size = [2, 3, 4, 5] class TestQuantizeOpShift_NHWC_4_P(TestQuantizeOpShift_NCHW_4_P): - def set_output_format(self): self.output_format = 'NHWC' class TestQuantizeOpShift_NHWC_4_N(TestQuantizeOpShift_NCHW_4_N): - def set_output_format(self): self.output_format = 'NHWC' diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_bf16_mkldnn_op.py index 12b8a468b46..fdd24498f88 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_bf16_mkldnn_op.py @@ -14,7 +14,12 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTestTool, OpTest, skip_check_grad_ci, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTestTool, + OpTest, + skip_check_grad_ci, + convert_float_to_uint16, +) import paddle.fluid.core as core import paddle @@ -23,7 +28,6 @@ paddle.enable_static() @OpTestTool.skip_if_not_cpu_bf16() class TestReduceSumDefaultBF16OneDNNOp(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -69,20 +73,21 @@ class TestReduceSumDefaultBF16OneDNNOp(OpTest): class TestReduceDefaultWithGradBF16OneDNNOp(TestReduceSumDefaultBF16OneDNNOp): - def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", check_dygraph=False, user_defined_grads=[self.grad_X], - user_defined_grad_outputs=[convert_float_to_uint16(self.grad_Out)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.grad_Out)], + ) class TestReduceSum4DReduceAllDimAttributeBF16OneDNNOp( - TestReduceDefaultWithGradBF16OneDNNOp): - + TestReduceDefaultWithGradBF16OneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -94,8 +99,8 @@ class TestReduceSum4DReduceAllDimAttributeBF16OneDNNOp( class TestReduceSum4DReduceAllWithoutReduceAllAttributeNegativeDimsBF16OneDNNOp( - TestReduceDefaultWithGradBF16OneDNNOp): - + TestReduceDefaultWithGradBF16OneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -107,8 +112,8 @@ class TestReduceSum4DReduceAllWithoutReduceAllAttributeNegativeDimsBF16OneDNNOp( class TestReduceSum5DReduceAllKeepDimsBF16OneDNNOp( - TestReduceDefaultWithGradBF16OneDNNOp): - + TestReduceDefaultWithGradBF16OneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -119,9 +124,9 @@ class TestReduceSum5DReduceAllKeepDimsBF16OneDNNOp( self.outputs = {'Out': self.x_fp32.sum(keepdims=self.attrs['keep_dim'])} -class TestReduceSum4DReduceAllBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp - ): - +class TestReduceSum4DReduceAllBF16OneDNNOp( + TestReduceDefaultWithGradBF16OneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -134,7 +139,8 @@ class TestReduceSum4DReduceAllBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMax3DBF16OneDNNOp(TestReduceSumDefaultBF16OneDNNOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -150,9 +156,11 @@ class TestReduceMax3DBF16OneDNNOp(TestReduceSumDefaultBF16OneDNNOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMax4DNegativeAndPositiveDimsBF16OneDNNOp( - TestReduceSumDefaultBF16OneDNNOp): + TestReduceSumDefaultBF16OneDNNOp +): """Remove Max with subgradient from gradient check to confirm the success of CI.""" def setUp(self): @@ -167,7 +175,8 @@ class TestReduceMax4DNegativeAndPositiveDimsBF16OneDNNOp( @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMin3DBF16OneDNNOp(TestReduceSumDefaultBF16OneDNNOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -182,7 +191,6 @@ class TestReduceMin3DBF16OneDNNOp(TestReduceSumDefaultBF16OneDNNOp): class TestReduceMean3DBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp): - def setUp(self): self.op_type = "reduce_mean" self.use_mkldnn = True @@ -194,7 +202,6 @@ class TestReduceMean3DBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp): class TestReduceMean4DBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp): - def setUp(self): self.op_type = "reduce_mean" self.use_mkldnn = True @@ -203,9 +210,8 @@ class TestReduceMean4DBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp): self.inputs = {'X': self.x_bf16} self.attrs = {'use_mkldnn': self.use_mkldnn, 'dim': [0, 1]} self.outputs = { - 'Out': - self.x_fp32.sum(axis=tuple(self.attrs['dim'])) / - (self.x_fp32.shape[0] * self.x_fp32.shape[1]) + 'Out': self.x_fp32.sum(axis=tuple(self.attrs['dim'])) + / (self.x_fp32.shape[0] * self.x_fp32.shape[1]) } diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_mkldnn_op.py index 93e81070cb6..b6f2c30a682 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_mkldnn_op.py @@ -14,12 +14,15 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, skip_check_grad_ci +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + skip_check_grad_ci, +) import paddle class TestReduceSumDefaultOneDNNOp(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -32,13 +35,11 @@ class TestReduceSumDefaultOneDNNOp(OpTest): class TestReduceDefaultWithGradOneDNNOp(TestReduceSumDefaultOneDNNOp): - def test_check_grad(self): self.check_grad(['X'], 'Out') class TestReduceSum4DOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -50,8 +51,8 @@ class TestReduceSum4DOneDNNOp(TestReduceDefaultWithGradOneDNNOp): class TestReduceSum4DReduceAllDimAttributeBF16OneDNNOp( - TestReduceDefaultWithGradOneDNNOp): - + TestReduceDefaultWithGradOneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -63,22 +64,21 @@ class TestReduceSum4DReduceAllDimAttributeBF16OneDNNOp( class TestReduceSum5DKeepDimsOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True self.inputs = {'X': np.random.random((2, 5, 3, 2, 2)).astype("float32")} self.attrs = {'dim': (2, 3, 4), 'keep_dim': True, 'use_mkldnn': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } -class TestReduceSum5DReduceAllKeepDimsOneDNNOp(TestReduceDefaultWithGradOneDNNOp - ): - +class TestReduceSum5DReduceAllKeepDimsOneDNNOp( + TestReduceDefaultWithGradOneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -90,7 +90,6 @@ class TestReduceSum5DReduceAllKeepDimsOneDNNOp(TestReduceDefaultWithGradOneDNNOp class TestReduceSum4DReduceAllOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -101,8 +100,8 @@ class TestReduceSum4DReduceAllOneDNNOp(TestReduceDefaultWithGradOneDNNOp): @OpTestTool.skip_if_not_cpu() class TestReduceSum4DNoReduceSimpleCopyOneDNNOp( - TestReduceDefaultWithGradOneDNNOp): - + TestReduceDefaultWithGradOneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -113,7 +112,8 @@ class TestReduceSum4DNoReduceSimpleCopyOneDNNOp( @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMax3DOneDNNOp(TestReduceSumDefaultOneDNNOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -129,9 +129,11 @@ class TestReduceMax3DOneDNNOp(TestReduceSumDefaultOneDNNOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMax4DNegativeAndPositiveDimsOneDNNOp( - TestReduceSumDefaultOneDNNOp): + TestReduceSumDefaultOneDNNOp +): """Remove Max with subgradient from gradient check to confirm the success of CI.""" def setUp(self): @@ -146,7 +148,8 @@ class TestReduceMax4DNegativeAndPositiveDimsOneDNNOp( @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMin3DOneDNNOp(TestReduceSumDefaultOneDNNOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -161,7 +164,6 @@ class TestReduceMin3DOneDNNOp(TestReduceSumDefaultOneDNNOp): class TestReduceMean3DOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_mean" self.use_mkldnn = True @@ -173,15 +175,14 @@ class TestReduceMean3DOneDNNOp(TestReduceDefaultWithGradOneDNNOp): class TestReduceMean4DReduceAllOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_mean" self.use_mkldnn = True self.inputs = {'X': np.random.random((5, 6, 8, 10)).astype("float32")} self.attrs = {'reduce_all': True, 'use_mkldnn': self.use_mkldnn} self.outputs = { - 'Out': - self.inputs['X'].sum() / np.asarray(self.inputs['X'].shape).prod() + 'Out': self.inputs['X'].sum() + / np.asarray(self.inputs['X'].shape).prod() } diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_requantize_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_requantize_mkldnn_op.py index f49bb89e3e5..a908b5f408d 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_requantize_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_requantize_mkldnn_op.py @@ -21,7 +21,6 @@ from mkldnn_op_test import format_reorder class TestReQuantizeOp(OpTest): - def set_input_size(self): self.input_size = [1, 1, 10, 10] self.format_reorder = format_reorder @@ -43,32 +42,36 @@ class TestReQuantizeOp(OpTest): def prepare_input(self): if self.input_data_type == 'int8': # input data values are integers from interval [-128, 128) - self.input = (np.random.randint(0, 256, self.input_size) - - 128).astype(self.input_data_type) + self.input = ( + np.random.randint(0, 256, self.input_size) - 128 + ).astype(self.input_data_type) else: # input data values are integers from interval [0, 256) self.input = (np.random.randint(0, 256, self.input_size)).astype( - self.input_data_type) + self.input_data_type + ) self.inputs = {'Input': OpTest.np_dtype_to_fluid_dtype(self.input)} self.attrs = { 'Scale_in': self.scale_in, 'Scale_out': self.scale_out, 'Shift_in': self.shift_in, - 'Shift_out': self.shift_out + 'Shift_out': self.shift_out, } def prepare_output(self): scale_ratio = self.scale_out / self.scale_in - with_shift = (self.shift_in != 0.0 or self.shift_out != 0.0) + with_shift = self.shift_in != 0.0 or self.shift_out != 0.0 if with_shift or self.input_data_type == 'uint8': dst_type = 'uint8' type_min = 0 type_max = 255 new_shift = np.clip( - np.rint(self.shift_out - scale_ratio * self.shift_in), type_min, - type_max) + np.rint(self.shift_out - scale_ratio * self.shift_in), + type_min, + type_max, + ) else: dst_type = 'int8' type_min = -128 @@ -77,15 +80,19 @@ class TestReQuantizeOp(OpTest): output_tmp = np.clip( np.rint(self.input.astype('float32') * scale_ratio + new_shift), - type_min, type_max).astype(dst_type) + type_min, + type_max, + ).astype(dst_type) self.output = self.format_reorder(output_tmp, self.input_size) self.outputs = {'Output': self.output} def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.assertTrue(self.input_data_type == 'uint8' or self.shift_in == 0.0, - 'Input data must be unsigned if it has nonzero shift.') + self.assertTrue( + self.input_data_type == 'uint8' or self.shift_in == 0.0, + 'Input data must be unsigned if it has nonzero shift.', + ) self.check_output(check_dygraph=False) def check_raise_error(self, msg): @@ -111,28 +118,24 @@ class TestReQuantizeOp(OpTest): class TestReQuantizeOp_S8_SameScales(TestReQuantizeOp): - def set_scales(self): self.scale_in = 127.0 self.scale_out = 127.0 class TestReQuantizeOp_S8_DifferentScales_1(TestReQuantizeOp): - def set_scales(self): self.scale_in = 127.0 self.scale_out = 100.0 class TestReQuantizeOp_S8_DifferentScales_2(TestReQuantizeOp): - def set_scales(self): self.scale_in = 100.0 self.scale_out = 127.0 class TestReQuantizeOp_S8_ZeroInputScale(TestReQuantizeOp): - def set_scales(self): self.scale_in = 0.0 self.scale_out = 127.0 @@ -142,12 +145,14 @@ class TestReQuantizeOp_S8_ZeroInputScale(TestReQuantizeOp): self.outputs = {'Output': self.output} def test_check_output(self): - self.assertRaises(AttributeError, self.check_raise_error, - 'Scale of input cannot be 0.0') + self.assertRaises( + AttributeError, + self.check_raise_error, + 'Scale of input cannot be 0.0', + ) class TestReQuantizeOp_S8_ZeroOutputScale(TestReQuantizeOp): - def set_scales(self): self.scale_in = 127.0 self.scale_out = 0.0 @@ -157,29 +162,31 @@ class TestReQuantizeOp_S8_ZeroOutputScale(TestReQuantizeOp): self.outputs = {'Output': self.output} def test_check_output(self): - self.assertRaises(AttributeError, self.check_raise_error, - 'Scale of output cannot be 0.0') + self.assertRaises( + AttributeError, + self.check_raise_error, + 'Scale of output cannot be 0.0', + ) # ---------------test requantize with u8 input, no shift-------------------- class TestReQuantizeOp_U8_SameScales(TestReQuantizeOp_S8_SameScales): - def set_input_data_type(self): self.input_data_type = 'uint8' class TestReQuantizeOp_U8_DifferentScales_1( - TestReQuantizeOp_S8_DifferentScales_1): - + TestReQuantizeOp_S8_DifferentScales_1 +): def set_input_data_type(self): self.input_data_type = 'uint8' class TestReQuantizeOp_U8_DifferentScales_2( - TestReQuantizeOp_S8_DifferentScales_2): - + TestReQuantizeOp_S8_DifferentScales_2 +): def set_input_data_type(self): self.input_data_type = 'uint8' @@ -188,7 +195,6 @@ class TestReQuantizeOp_U8_DifferentScales_2( class TestReQuantizeOp_S8_WithShift(TestReQuantizeOp): - def set_scales(self): self.scale_in = 60.0 self.scale_out = 127.0 @@ -199,12 +205,13 @@ class TestReQuantizeOp_S8_WithShift(TestReQuantizeOp): def test_check_output(self): self.assertRaises( - AttributeError, self.check_raise_error, - 'Requantize does not support nonzero shift for signed input.') + AttributeError, + self.check_raise_error, + 'Requantize does not support nonzero shift for signed input.', + ) class TestReQuantizeOp_S8_WithOutputShift(TestReQuantizeOp): - def set_scales(self): self.scale_in = 127.0 self.scale_out = 60.0 @@ -218,71 +225,70 @@ class TestReQuantizeOp_S8_WithOutputShift(TestReQuantizeOp): class TestReQuantizeOp_U8_SameScales_SameShift(TestReQuantizeOp_U8_SameScales): - def set_shifts(self): self.shift_in = 128.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_SameScales_DifferentShift_1( - TestReQuantizeOp_U8_SameScales): - + TestReQuantizeOp_U8_SameScales +): def set_shifts(self): self.shift_in = 60.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_SameScales_DifferentShift_2( - TestReQuantizeOp_U8_SameScales): - + TestReQuantizeOp_U8_SameScales +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 60.0 class TestReQuantizeOp_U8_DifferentScales_1_SameShift( - TestReQuantizeOp_U8_DifferentScales_1): - + TestReQuantizeOp_U8_DifferentScales_1 +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_DifferentScales_2_SameShift( - TestReQuantizeOp_U8_DifferentScales_2): - + TestReQuantizeOp_U8_DifferentScales_2 +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_DifferentScales_1_DifferentShift_1( - TestReQuantizeOp_U8_DifferentScales_1): - + TestReQuantizeOp_U8_DifferentScales_1 +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 60.0 class TestReQuantizeOp_U8_DifferentScales_2_DifferentShift_1( - TestReQuantizeOp_U8_DifferentScales_2): - + TestReQuantizeOp_U8_DifferentScales_2 +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 60.0 class TestReQuantizeOp_U8_DifferentScales_1_DifferentShift_2( - TestReQuantizeOp_U8_DifferentScales_1): - + TestReQuantizeOp_U8_DifferentScales_1 +): def set_shifts(self): self.shift_in = 60.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_DifferentScales_2_DifferentShift_2( - TestReQuantizeOp_U8_DifferentScales_2): - + TestReQuantizeOp_U8_DifferentScales_2 +): def set_shifts(self): self.shift_in = 60.0 self.shift_out = 128.0 @@ -292,7 +298,6 @@ class TestReQuantizeOp_U8_DifferentScales_2_DifferentShift_2( class TestReQuantizeOp_2DimFormat(TestReQuantizeOp): - def format_reorder_2Dim(self, out, size): return out @@ -305,7 +310,6 @@ class TestReQuantizeOp_2DimFormat(TestReQuantizeOp): class TestReQuantizeOpReused(TestReQuantizeOp): - def setUp(self): # self.input_size = [1, 1, 10, 10] self.input_size = [1, 1, 2, 2] @@ -337,38 +341,40 @@ class TestReQuantizeOpReused(TestReQuantizeOp): with fluid.program_guard(program): block = program.global_block() for name in variables: - block.create_var(name=name, - dtype="int8", - shape=variables[name].shape) - block.append_op(type="requantize", - inputs={ - 'Input': block.var('input'), - }, - outputs={"Output": block.var('output')}, - attrs={ - 'Scale_in': self.scale_in, - 'Scale_out': self.scale_out, - 'Shift_in': self.shift_in, - 'Shift_out': self.shift_out - }) + block.create_var( + name=name, dtype="int8", shape=variables[name].shape + ) + block.append_op( + type="requantize", + inputs={ + 'Input': block.var('input'), + }, + outputs={"Output": block.var('output')}, + attrs={ + 'Scale_in': self.scale_in, + 'Scale_out': self.scale_out, + 'Shift_in': self.shift_in, + 'Shift_out': self.shift_out, + }, + ) place = core.CPUPlace() exe = fluid.Executor(place) for i in range(2): - out = exe.run(program, - feed={'input': variables['input']}, - fetch_list=['output']) + out = exe.run( + program, + feed={'input': variables['input']}, + fetch_list=['output'], + ) - np.testing.assert_allclose(variables['output'], - out[0], - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + variables['output'], out[0], rtol=1e-05, atol=1e-4 + ) # ---------------test reused requantize op, no shift------------------------ class TestReQuantizeOpReused_WithShift(TestReQuantizeOpReused): - def set_input_data_type(self): self.input_data_type = 'uint8' diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_bf16_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_bf16_op.py index 7fbdf66ee7d..efa1459b047 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_bf16_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_bf16_op.py @@ -20,10 +20,10 @@ from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestReshapeBf16Op(OpTest): - def setUp(self): self.op_type = "reshape2" self.use_mkldnn = False @@ -35,11 +35,11 @@ class TestReshapeBf16Op(OpTest): self.attrs = { 'shape': self.new_shape, 'use_mkldnn': self.use_mkldnn, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype(np.float32) + 'XShape': np.random.random(self.ori_shape).astype(np.float32), } def init_data(self): @@ -49,21 +49,24 @@ class TestReshapeBf16Op(OpTest): def init_input_data(self): self.input_data_fp32 = np.random.random(self.ori_shape).astype( - np.float32) + np.float32 + ) self.input_data = convert_float_to_uint16(self.input_data_fp32) def test_check_output(self): self.check_output_with_place(core.CPUPlace(), no_check_set=['XShape']) def test_check_grad(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - check_dygraph=False, - user_defined_grads=[self.input_data_fp32], - user_defined_grad_outputs=[ - self.inputs["X"].reshape( - self.infered_shape) - ]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + check_dygraph=False, + user_defined_grads=[self.input_data_fp32], + user_defined_grad_outputs=[ + self.inputs["X"].reshape(self.infered_shape) + ], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_mkldnn_op.py index ef2748155e7..46d232cfc08 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_mkldnn_op.py @@ -16,13 +16,18 @@ import unittest import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) -@OpTestTool.skip_if(core.is_compiled_with_cuda(), - "CUDA has to be skipped because it forces dygraph") +@OpTestTool.skip_if( + core.is_compiled_with_cuda(), + "CUDA has to be skipped because it forces dygraph", +) class TestReshape2OneDNNOp(OpTest): - def setUp(self): self.init_data() self.set_op_type() @@ -47,7 +52,7 @@ class TestReshape2OneDNNOp(OpTest): def set_outputs(self): self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -63,7 +68,6 @@ class TestReshape2OneDNNOp(OpTest): class TestReshape2OneDNNOpDimInfer1(TestReshape2OneDNNOp): - def init_data(self): self.ori_shape = (5, 25) self.new_shape = (5, -1, 5) @@ -71,7 +75,6 @@ class TestReshape2OneDNNOpDimInfer1(TestReshape2OneDNNOp): class TestReshape2OneDNNOpDimInfer2(TestReshape2OneDNNOp): - def init_data(self): self.ori_shape = (6, 20) self.new_shape = (0, -1, 20) @@ -83,12 +86,11 @@ class TestReshape2OneDNNOpDimInfer2(TestReshape2OneDNNOp): def set_outputs(self): self.outputs = { "Out": self.inputs["X"].reshape(self.actual_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } class TestReshape2OneDNNOp_attr_OnlyShape(TestReshape2OneDNNOp): - def set_additional_inputs(self): self.inputs["Shape"] = np.array(self.new_shape, dtype="int32") @@ -98,7 +100,7 @@ class TestReshape2OneDNNOp_attr_OnlyShape(TestReshape2OneDNNOp): def set_outputs(self): self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -108,8 +110,8 @@ class TestReshape2OneDNNOp_attr_OnlyShape(TestReshape2OneDNNOp): class TestReshape2OneDNNOpDimInfer1_attr_OnlyShape( - TestReshape2OneDNNOp_attr_OnlyShape): - + TestReshape2OneDNNOp_attr_OnlyShape +): def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 10) @@ -118,12 +120,12 @@ class TestReshape2OneDNNOpDimInfer1_attr_OnlyShape( class TestReshape2OneDNNOpDimInfer1_attr_ShapeTensor(TestReshape2OneDNNOp): - def set_additional_inputs(self): shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs["ShapeTensor"] = shape_tensor @@ -135,20 +137,20 @@ class TestReshape2OneDNNOpDimInfer1_attr_ShapeTensor(TestReshape2OneDNNOp): class TestReshape2OneDNNOpDimInfer1_attr_ShapeTensorAndShape( - TestReshape2OneDNNOpDimInfer1_attr_ShapeTensor): - + TestReshape2OneDNNOpDimInfer1_attr_ShapeTensor +): def set_additional_inputs(self): shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs["Shape"] = np.array((1, 2, 3, 4), dtype="int32") self.inputs["ShapeTensor"] = shape_tensor class TestReshapeOneDNNOp(TestReshape2OneDNNOp): - def set_op_type(self): self.op_type = "reshape" @@ -160,7 +162,6 @@ class TestReshapeOneDNNOp(TestReshape2OneDNNOp): class TestReshapeOneDNNOpDimInfer1(TestReshapeOneDNNOp): - def init_data(self): self.ori_shape = (5, 25) self.new_shape = (5, -1, 5) @@ -168,7 +169,6 @@ class TestReshapeOneDNNOpDimInfer1(TestReshapeOneDNNOp): class TestReshapeOneDNNOp_attr_OnlyShape(TestReshape2OneDNNOp_attr_OnlyShape): - def set_op_type(self): self.op_type = "reshape" @@ -180,8 +180,8 @@ class TestReshapeOneDNNOp_attr_OnlyShape(TestReshape2OneDNNOp_attr_OnlyShape): class TestReshapeOneDNNOpDimInfer1_attr_OnlyShape( - TestReshapeOneDNNOp_attr_OnlyShape): - + TestReshapeOneDNNOp_attr_OnlyShape +): def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 10) @@ -191,10 +191,8 @@ class TestReshapeOneDNNOpDimInfer1_attr_OnlyShape( # BF16 TESTS def create_reshape_bf16_test_classes(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestReshape2BF16OneDNNOp(parent): - def set_inputs(self): self.dtype = np.uint16 self.inputs = {"X": convert_float_to_uint16(self.x)} @@ -204,22 +202,25 @@ def create_reshape_bf16_test_classes(parent): self.dx = np.reshape(self.dout, self.ori_shape) def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), - no_check_set=["XShape"]) + self.check_output_with_place( + core.CPUPlace(), no_check_set=["XShape"] + ) def test_check_grad(self): self.calculate_grads() - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - user_defined_grads=[self.dx], - user_defined_grad_outputs=[self.dout]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + user_defined_grads=[self.dx], + user_defined_grad_outputs=[self.dout], + ) cls_name = "{0}_{1}".format(parent.__name__, "Reshape2_BF16") TestReshape2BF16OneDNNOp.__name__ = cls_name globals()[cls_name] = TestReshape2BF16OneDNNOp class TestReshapeBF16OneDNNOp(TestReshape2BF16OneDNNOp): - def set_op_type(self): self.dtype = np.uint16 self.op_type = "reshape" @@ -233,10 +234,12 @@ def create_reshape_bf16_test_classes(parent): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "Reshape_BF16") TestReshapeBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_scale_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_scale_bf16_mkldnn_op.py index 18224e07d42..e22161672fe 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_scale_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_scale_bf16_mkldnn_op.py @@ -19,12 +19,14 @@ import paddle import paddle.fluid.core as core -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") -@unittest.skipIf(core.is_compiled_with_cuda(), - "core is compiled with CUDA which has no BF implementation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) +@unittest.skipIf( + core.is_compiled_with_cuda(), + "core is compiled with CUDA which has no BF implementation", +) class TestScaleOpBF16(OpTest): - def setUp(self): self.op_type = "scale" self.x_fp32 = np.random.random((10, 10)).astype(np.float32) @@ -47,7 +49,7 @@ class TestScaleOpBF16(OpTest): scale = self.attrs['ScaleTensor'] self.out = (self.x_fp32 * scale) + bias - self.dx = (self.out * scale) + self.dx = self.out * scale def test_check_output(self): self.check_output(check_dygraph=False) @@ -55,15 +57,16 @@ class TestScaleOpBF16(OpTest): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", check_dygraph=False, user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.out)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.out)], + ) class TestScaleOpBF16BiasNotAfterScale(TestScaleOpBF16): - def setUp(self): self.op_type = "scale" self.x_fp32 = np.random.random((10, 10)).astype(np.float32) @@ -74,7 +77,7 @@ class TestScaleOpBF16BiasNotAfterScale(TestScaleOpBF16): 'scale': self.scale, 'use_mkldnn': True, 'bias': 0.0, - 'bias_after_scale': False + 'bias_after_scale': False, } self.use_mkldnn = True self.outputs = { @@ -83,7 +86,6 @@ class TestScaleOpBF16BiasNotAfterScale(TestScaleOpBF16): class TestScaleOpBF16ScaleTensor(TestScaleOpBF16): - def setUp(self): self.op_type = "scale" self.scale = -2.3 @@ -92,14 +94,13 @@ class TestScaleOpBF16ScaleTensor(TestScaleOpBF16): self.scale_tensor = np.array([self.scale]).astype(np.float32) self.inputs = { 'X': self.x_bf16, - 'ScaleTensor': convert_float_to_uint16(self.scale_tensor) + 'ScaleTensor': convert_float_to_uint16(self.scale_tensor), } self.attrs = {'use_mkldnn': True} self.outputs = {'Out': self.x_fp32 * self.scale} class TestScaleOpBF16ScaleTensorNotBiasAfterScale(TestScaleOpBF16): - def setUp(self): self.op_type = "scale" self.scale = 1.2 @@ -108,12 +109,12 @@ class TestScaleOpBF16ScaleTensorNotBiasAfterScale(TestScaleOpBF16): self.scale_tensor = np.array([self.scale]).astype(np.float32) self.inputs = { 'X': self.x_bf16, - 'ScaleTensor': convert_float_to_uint16(self.scale_tensor) + 'ScaleTensor': convert_float_to_uint16(self.scale_tensor), } self.attrs = { 'bias': -1.1, 'bias_after_scale': False, - 'use_mkldnn': True + 'use_mkldnn': True, } self.outputs = {'Out': (self.x_fp32 + self.attrs['bias']) * self.scale} diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_scale_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_scale_mkldnn_op.py index 9fcbb98dfa2..b812f8d876d 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_scale_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_scale_mkldnn_op.py @@ -19,7 +19,6 @@ import paddle class TestScaleOp(OpTest): - def setUp(self): self.op_type = "scale" self.inputs = {'X': np.random.random((10, 10)).astype(np.float32)} @@ -37,7 +36,6 @@ class TestScaleOp(OpTest): class TestScaleOpBiasNotAfterScale(OpTest): - def setUp(self): self.op_type = "scale" self.inputs = {'X': np.random.random((10, 10)).astype(np.float32)} @@ -45,7 +43,7 @@ class TestScaleOpBiasNotAfterScale(OpTest): 'scale': 1.5, 'use_mkldnn': True, 'bias': 2.3, - 'bias_after_scale': False + 'bias_after_scale': False, } self.use_mkldnn = True self.outputs = { @@ -60,13 +58,12 @@ class TestScaleOpBiasNotAfterScale(OpTest): class TestScaleOpScaleTensor(OpTest): - def setUp(self): self.op_type = "scale" self.scale = -2.3 self.inputs = { 'X': np.random.random((10, 10)).astype(np.float32), - 'ScaleTensor': np.array([self.scale]).astype(np.float32) + 'ScaleTensor': np.array([self.scale]).astype(np.float32), } self.attrs = {} self.outputs = {'Out': self.inputs['X'] * self.scale} @@ -79,18 +76,17 @@ class TestScaleOpScaleTensor(OpTest): class TestScaleOpScaleTensorNotBiasAfterScale(OpTest): - def setUp(self): self.op_type = "scale" self.scale = -1.2 self.inputs = { 'X': np.random.random((10, 10)).astype(np.float32), - 'ScaleTensor': np.array([self.scale]).astype(np.float32) + 'ScaleTensor': np.array([self.scale]).astype(np.float32), } self.attrs = {'bias': -6.8, 'bias_after_scale': False} self.outputs = { - 'Out': - (self.inputs['X'] + self.attrs['bias']) * self.inputs['ScaleTensor'] + 'Out': (self.inputs['X'] + self.attrs['bias']) + * self.inputs['ScaleTensor'] } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_shape_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_shape_mkldnn_op.py index 573d4a9b2cf..98401c3ca17 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_shape_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_shape_mkldnn_op.py @@ -21,7 +21,6 @@ from paddle.fluid import core @OpTestTool.skip_if_not_cpu_bf16() class TestShape3DFP32OneDNNOp(OpTest): - def setUp(self): self.op_type = "shape" self.config() @@ -38,21 +37,18 @@ class TestShape3DFP32OneDNNOp(OpTest): class TestShape6DBF16OneDNNOp(TestShape3DFP32OneDNNOp): - def config(self): self.shape = [10, 2, 3, 4, 5, 2] self.dtype = np.uint16 class TestShape9DINT8OneDNNOp(TestShape3DFP32OneDNNOp): - def config(self): self.shape = [1, 2, 3, 4, 5, 6, 7, 8, 9] self.dtype = np.int8 class TestShape2DUINT8OneDNNOp(TestShape3DFP32OneDNNOp): - def config(self): self.shape = [7, 11] self.dtype = np.uint8 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_shuffle_channel_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_shuffle_channel_mkldnn_op.py index 0ed91e95924..b5320c0653a 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_shuffle_channel_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_shuffle_channel_mkldnn_op.py @@ -21,7 +21,6 @@ import paddle.fluid.core as core @OpTestTool.skip_if_not_cpu_bf16() class TestShuffleChannelOneDNNOp(OpTest): - def setUp(self): self.op_type = "shuffle_channel" self.set_dtype() @@ -30,8 +29,9 @@ class TestShuffleChannelOneDNNOp(OpTest): self.attrs = {'use_mkldnn': True, 'group': self.group} _, c, h, w = self.inputs['X'].shape - input_reshaped = np.reshape(self.inputs['X'], - (-1, self.group, c // self.group, h, w)) + input_reshaped = np.reshape( + self.inputs['X'], (-1, self.group, c // self.group, h, w) + ) input_transposed = np.transpose(input_reshaped, (0, 2, 1, 3, 4)) self.outputs = {'Out': np.reshape(input_transposed, (-1, c, h, w))} @@ -46,13 +46,11 @@ class TestShuffleChannelOneDNNOp(OpTest): class TestShuffleChannelSingleGroupOneDNNOp(TestShuffleChannelOneDNNOp): - def set_group(self): self.group = 1 class TestShuffleChannelBF16OneDNNOp(TestShuffleChannelOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_slice_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_slice_mkldnn_op.py index 0812f4da8b2..8dca568bb1e 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_slice_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_slice_mkldnn_op.py @@ -15,14 +15,19 @@ import unittest import numpy as np import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) import paddle -@OpTestTool.skip_if(core.is_compiled_with_cuda(), - "CUDA required dygraph so oneDNN UT must be skipped") +@OpTestTool.skip_if( + core.is_compiled_with_cuda(), + "CUDA required dygraph so oneDNN UT must be skipped", +) class TestSliceOneDNNOp(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -33,7 +38,7 @@ class TestSliceOneDNNOp(OpTest): 'starts': self.starts, 'ends': self.ends, 'infer_flags': self.infer_flags, - 'use_mkldnn': True + 'use_mkldnn': True, } self.set_attrs() @@ -59,7 +64,6 @@ class TestSliceOneDNNOp(OpTest): class TestSliceOneDNNOp1(TestSliceOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-3, 0, 2] @@ -70,7 +74,6 @@ class TestSliceOneDNNOp1(TestSliceOneDNNOp): class TestSliceOneDNNOp2(TestSliceOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-3, 0, 2] @@ -81,7 +84,6 @@ class TestSliceOneDNNOp2(TestSliceOneDNNOp): class TestSliceDecrease1AxisOneDNNOp(TestSliceOneDNNOp): - def set_attrs(self): self.attrs['decrease_axis'] = self.decrease_axis @@ -96,7 +98,6 @@ class TestSliceDecrease1AxisOneDNNOp(TestSliceOneDNNOp): class TestSliceDecrease2AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [1, 0, 2] @@ -108,7 +109,6 @@ class TestSliceDecrease2AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): class TestSliceDecrease3AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-1, 0, 2] @@ -120,7 +120,6 @@ class TestSliceDecrease3AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): class TestSliceDecrease4AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype("float32") self.starts = [0, 1, 2, 3] @@ -132,7 +131,6 @@ class TestSliceDecrease4AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): class TestSlice5DOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6, 7]).astype("float32") self.starts = [-1] @@ -144,7 +142,6 @@ class TestSlice5DOneDNNOp(TestSliceDecrease1AxisOneDNNOp): class TestSlice3DOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([5, 4, 5]).astype("float32") self.starts = [-1] @@ -156,8 +153,8 @@ class TestSlice3DOneDNNOp(TestSliceDecrease1AxisOneDNNOp): class TestSliceOneDNNOp_decs_dim_starts_ListTensor( - TestSliceDecrease1AxisOneDNNOp): - + TestSliceDecrease1AxisOneDNNOp +): def set_inputs(self): starts_tensor = [] for index, ele in enumerate(self.starts): @@ -175,7 +172,6 @@ class TestSliceOneDNNOp_decs_dim_starts_ListTensor( class TestSlice4DInferDimsOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([1, 1, 10, 10]).astype("float32") self.starts = [1, 2] @@ -187,7 +183,6 @@ class TestSlice4DInferDimsOneDNNOp(TestSliceDecrease1AxisOneDNNOp): class TestSlice4DInferDimsOneDNNOp2(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([1, 1, 10, 10]).astype("float32") self.starts = [4, 2] @@ -200,10 +195,8 @@ class TestSlice4DInferDimsOneDNNOp2(TestSliceDecrease1AxisOneDNNOp): # BF16 TESTS def create_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestSliceBF16OneDNNOp(parent): - def set_inputs(self): self.dtype = np.uint16 self.inputs = {'Input': convert_float_to_uint16(self.input)} @@ -218,8 +211,12 @@ def create_bf16_test_class(parent): for i in range(len(self.axes)): begin[self.axes[i]] = self.starts[i] end[self.axes[i]] = self.ends[i] - self.dx[begin[0]:end[0], begin[1]:end[1], begin[2]:end[2], - begin[3]:end[3]] = self.dout + self.dx[ + begin[0] : end[0], + begin[1] : end[1], + begin[2] : end[2], + begin[3] : end[3], + ] = self.dout def test_check_output(self): self.check_output_with_place(core.CPUPlace()) @@ -227,10 +224,12 @@ def create_bf16_test_class(parent): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["Input"], + core.CPUPlace(), + ["Input"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "BF16") TestSliceBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_bf16_mkldnn_op.py index 23643261836..e87214b6b27 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_bf16_mkldnn_op.py @@ -16,21 +16,28 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import convert_float_to_uint16 import paddle.fluid.core as core -from paddle.fluid.tests.unittests.test_softmax_op import TestSoftmaxOp, TestSoftmaxOp2, TestSoftmaxOp3, TestSoftmaxOp4, TestSoftmaxOp5, TestSoftmaxOp6 +from paddle.fluid.tests.unittests.test_softmax_op import ( + TestSoftmaxOp, + TestSoftmaxOp2, + TestSoftmaxOp3, + TestSoftmaxOp4, + TestSoftmaxOp5, + TestSoftmaxOp6, +) from paddle import enable_static def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" - shiftx = x - np.max(x).clip(-64.) + shiftx = x - np.max(x).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestSoftmaxMKLDNNOp(TestSoftmaxOp): - def get_x_shape(self): return [10, 10] @@ -47,7 +54,8 @@ class TestSoftmaxMKLDNNOp(TestSoftmaxOp): x = np.random.uniform(0.1, 1, self.shape).astype(np.float64) out = convert_float_to_uint16( - np.apply_along_axis(stable_softmax, self.axis, x)) + np.apply_along_axis(stable_softmax, self.axis, x) + ) self.inputs = {'X': convert_float_to_uint16(x)} self.outputs = {'Out': out} @@ -64,31 +72,26 @@ class TestSoftmaxMKLDNNOp(TestSoftmaxOp): class TestSoftmaxMKLDNNOp2(TestSoftmaxOp2): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp3(TestSoftmaxOp3): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp4(TestSoftmaxOp4): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp5(TestSoftmaxOp5): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp6(TestSoftmaxOp6): - def init_kernel_type(self): self.use_mkldnn = True diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_mkldnn_op.py index 1422d1aeb34..c181c124886 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_mkldnn_op.py @@ -16,19 +16,25 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest import paddle.fluid.core as core -from paddle.fluid.tests.unittests.test_softmax_op import TestSoftmaxOp, TestSoftmaxOp2, TestSoftmaxOp3, TestSoftmaxOp4, TestSoftmaxOp5, TestSoftmaxOp6 +from paddle.fluid.tests.unittests.test_softmax_op import ( + TestSoftmaxOp, + TestSoftmaxOp2, + TestSoftmaxOp3, + TestSoftmaxOp4, + TestSoftmaxOp5, + TestSoftmaxOp6, +) from mkldnn_op_test import check_if_mkldnn_primitives_exist_in_bwd def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" - shiftx = x - np.max(x).clip(-64.) + shiftx = x - np.max(x).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) class TestSoftmaxMKLDNNOp(TestSoftmaxOp): - def get_x_shape(self): return [10, 10] @@ -52,7 +58,7 @@ class TestSoftmaxMKLDNNOp(TestSoftmaxOp): self.attrs = { 'axis': self.axis, 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def test_check_output(self): @@ -68,53 +74,49 @@ class TestSoftmaxMKLDNNOp(TestSoftmaxOp): if self.use_cudnn or self.dtype == np.float16: place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ["X"], - "Out", - max_relative_error=0.01, - check_dygraph=False) + self.check_grad_with_place( + place, + ["X"], + "Out", + max_relative_error=0.01, + check_dygraph=False, + ) else: - self.check_grad(["X"], - "Out", - max_relative_error=0.01, - check_dygraph=False) + self.check_grad( + ["X"], "Out", max_relative_error=0.01, check_dygraph=False + ) def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp2(TestSoftmaxOp2): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp3(TestSoftmaxOp3): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp4(TestSoftmaxOp4): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp5(TestSoftmaxOp5): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp6(TestSoftmaxOp6): - def init_kernel_type(self): self.use_mkldnn = True # Check if primitives already exist in backward class TestSoftmaxMKLDNNPrimitivesAlreadyExist(unittest.TestCase): - def setUp(self): super(TestSoftmaxMKLDNNPrimitivesAlreadyExist, self).setUp() @@ -130,12 +132,13 @@ class TestSoftmaxMKLDNNPrimitivesAlreadyExist(unittest.TestCase): return out * (out_grad - np.dot(out, out_grad)) def test_check(self): - check_if_mkldnn_primitives_exist_in_bwd(self, self.op_type, self.x, - self.out, self.out_grad, - self.x_grad) + check_if_mkldnn_primitives_exist_in_bwd( + self, self.op_type, self.x, self.out, self.out_grad, self.x_grad + ) if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_softplus_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_softplus_mkldnn_op.py index db7eeb88f1d..49846a2280c 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_softplus_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_softplus_mkldnn_op.py @@ -14,20 +14,25 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) import paddle def ref_softplus(x, beta, threshold): x_beta = beta * x - out = np.select([x_beta <= threshold, x_beta > threshold], - [np.log(1 + np.exp(x_beta)) / beta, x]) + out = np.select( + [x_beta <= threshold, x_beta > threshold], + [np.log(1 + np.exp(x_beta)) / beta, x], + ) return out @OpTestTool.skip_if_not_cpu_bf16() class TestSoftplusOneDNNOp(OpTest): - def setUp(self): self.op_type = "softplus" self.beta = 1 @@ -57,52 +62,45 @@ class TestSoftplusOneDNNOp(OpTest): class TestSoftplus4DOneDNNOp(TestSoftplusOneDNNOp): - def config(self): self.x_shape = (10, 5, 4, 2) class TestSoftplus6DOneDNNOp(TestSoftplusOneDNNOp): - def config(self): self.x_shape = (3, 2, 2, 5, 4, 2) class TestSoftplus6DExtendedFunctorOneDNNOp(TestSoftplusOneDNNOp): - def config(self): self.x_shape = (3, 5, 2, 5, 4, 2) self.beta = 2.5 class TestSoftplus3DExtendedFunctorOneDNNOp(TestSoftplusOneDNNOp): - def config(self): self.x_shape = (20, 4, 2) self.beta = 0.4 class TestSoftplusBF16OneDNNOp(TestSoftplusOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 class TestSoftplus4DBF16OneDNNOp(TestSoftplus4DOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 class TestSoftplus6DBF16OneDNNOp(TestSoftplus6DOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 class TestSoftplus3DExtendedFunctorBF16OneDNNOp( - TestSoftplus3DExtendedFunctorOneDNNOp): - + TestSoftplus3DExtendedFunctorOneDNNOp +): def set_dtype(self): self.dtype = np.uint16 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_split_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_split_bf16_mkldnn_op.py index 56416629d77..a3345c1922d 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_split_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_split_bf16_mkldnn_op.py @@ -19,12 +19,14 @@ from paddle.fluid import core from paddle.fluid.tests.unittests.op_test import OpTest -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") -@unittest.skipIf(core.is_compiled_with_cuda(), - "core is compiled with CUDA which has no BF implementation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) +@unittest.skipIf( + core.is_compiled_with_cuda(), + "core is compiled with CUDA which has no BF implementation", +) class TestSplitSectionsBF16OneDNNOp(OpTest): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("uint16") self.axis = 1 @@ -43,7 +45,7 @@ class TestSplitSectionsBF16OneDNNOp(OpTest): self.attrs = { 'use_mkldnn': True, 'num': self.num, - 'mkldnn_data_type': "bfloat16" + 'mkldnn_data_type': "bfloat16", } if self.axis is not None: @@ -55,8 +57,9 @@ class TestSplitSectionsBF16OneDNNOp(OpTest): if self.sections_tensor_list is not None: self.inputs['SectionsTensorList'] = self.sections_tensor_list - self.outputs = {'Out': [('out%d' % i, self.out[i]) \ - for i in range(len(self.out))]} + self.outputs = { + 'Out': [('out%d' % i, self.out[i]) for i in range(len(self.out))] + } def test_check_output(self): self.check_output_with_place(core.CPUPlace()) @@ -73,50 +76,47 @@ class TestSplitSectionsBF16OneDNNOp(OpTest): class TestSplitNumBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): - def init_data(self): self.x = np.random.random((4, 8, 5, 3)).astype("uint16") self.axis = 1 self.sections = [] self.num = 4 - indices_or_sections = 4 #indices + indices_or_sections = 4 # indices self.out = np.split(self.x, indices_or_sections, self.axis) class TestSplitNumAxisTensorBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("uint16") self.axis = None self.sections = [] self.num = 3 - indices_or_sections = 3 #indices + indices_or_sections = 3 # indices self.axis_tensor = np.array([2]).astype("int32") self.out = np.split(self.x, indices_or_sections, 2) class TestSplitSectionsTensorBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("uint16") self.axis = 1 self.sections = [2, 1, 2] self.sections_tensor_list = [] for index, ele in enumerate(self.sections): - self.sections_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.sections_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.sections = [-1, -1, -1] - indices_or_sections = [2, 3] #sections + indices_or_sections = [2, 3] # sections self.out = np.split(self.x, indices_or_sections, self.axis) class TestSplitOpUnknownSectionBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("uint16") self.axis = 2 self.sections = [2, 2, -1] - indices_or_sections = [2, 4] #sections + indices_or_sections = [2, 4] # sections self.out = np.split(self.x, indices_or_sections, self.axis) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_split_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_split_mkldnn_op.py index 2b1e1fc9f22..e148e0cdcab 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_split_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_split_mkldnn_op.py @@ -19,7 +19,6 @@ from paddle.fluid.tests.unittests.op_test import OpTest class TestSplitSectionsOneDNNOp(OpTest): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("float32") self.axis = 1 @@ -46,8 +45,9 @@ class TestSplitSectionsOneDNNOp(OpTest): if self.sections_tensor_list is not None: self.inputs['SectionsTensorList'] = self.sections_tensor_list - self.outputs = {'Out': [('out%d' % i, self.out[i]) \ - for i in range(len(self.out))]} + self.outputs = { + 'Out': [('out%d' % i, self.out[i]) for i in range(len(self.out))] + } def test_check_output(self): self.check_output() @@ -58,13 +58,12 @@ class TestSplitSectionsOneDNNOp(OpTest): # test with attr(num) class TestSplitNumOneDNNOp(TestSplitSectionsOneDNNOp): - def init_data(self): self.x = np.random.random((4, 8, 5, 3)).astype("float32") self.axis = 1 self.sections = [] self.num = 4 - indices_or_sections = 4 #indices + indices_or_sections = 4 # indices self.out = np.split(self.x, indices_or_sections, self.axis) def test_check_grad(self): @@ -72,40 +71,38 @@ class TestSplitNumOneDNNOp(TestSplitSectionsOneDNNOp): class TestSplitNumAxisTensorOneDNNOp(TestSplitSectionsOneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("float32") self.axis = None self.sections = [] self.num = 3 - indices_or_sections = 3 #indices + indices_or_sections = 3 # indices self.axis_tensor = np.array([2]).astype("int32") self.out = np.split(self.x, indices_or_sections, 2) # attr(sections) is list containing Tensor class TestSplitSectionsTensorOneDNNOp(TestSplitSectionsOneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("float32") self.axis = 1 self.sections = [2, 1, 2] self.sections_tensor_list = [] for index, ele in enumerate(self.sections): - self.sections_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.sections_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.sections = [-1, -1, -1] - indices_or_sections = [2, 3] #sections + indices_or_sections = [2, 3] # sections self.out = np.split(self.x, indices_or_sections, self.axis) class TestSplitOpUnknownSectionOneDNNOp(TestSplitSectionsOneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("float32") self.axis = 2 self.sections = [2, 2, -1] - indices_or_sections = [2, 4] #sections + indices_or_sections = [2, 4] # sections self.out = np.split(self.x, indices_or_sections, self.axis) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_squeeze2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_squeeze2_mkldnn_op.py index 9d8647b9086..0b5cca9d5ad 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_squeeze2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_squeeze2_mkldnn_op.py @@ -16,13 +16,18 @@ import unittest import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) -@OpTestTool.skip_if(core.is_compiled_with_cuda(), - "CUDA has to be skipped because it forces dygraph") +@OpTestTool.skip_if( + core.is_compiled_with_cuda(), + "CUDA has to be skipped because it forces dygraph", +) class TestSqueeze2OneDNNOp(OpTest): - def set_op_type(self): self.op_type = "squeeze2" @@ -40,7 +45,7 @@ class TestSqueeze2OneDNNOp(OpTest): def set_outputs(self): self.outputs = { "Out": self.x.reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def setUp(self): @@ -59,7 +64,6 @@ class TestSqueeze2OneDNNOp(OpTest): class TestSqueezeOneDNNOp(TestSqueeze2OneDNNOp): - def set_op_type(self): self.op_type = "squeeze" @@ -71,7 +75,6 @@ class TestSqueezeOneDNNOp(TestSqueeze2OneDNNOp): class TestSqueeze2OneDNNOp1(TestSqueeze2OneDNNOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -79,7 +82,6 @@ class TestSqueeze2OneDNNOp1(TestSqueeze2OneDNNOp): class TestSqueezeOneDNNOp1(TestSqueezeOneDNNOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -87,7 +89,6 @@ class TestSqueezeOneDNNOp1(TestSqueezeOneDNNOp): class TestSqueeze2OneDNNOp2(TestSqueeze2OneDNNOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -95,7 +96,6 @@ class TestSqueeze2OneDNNOp2(TestSqueeze2OneDNNOp): class TestSqueezeOneDNNOp2(TestSqueezeOneDNNOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -103,7 +103,6 @@ class TestSqueezeOneDNNOp2(TestSqueezeOneDNNOp): class TestSqueeze2OneDNNOp3(TestSqueeze2OneDNNOp): - def init_test_case(self): self.ori_shape = (25, 1, 1, 4, 1) self.axes = (1, -1) @@ -111,7 +110,6 @@ class TestSqueeze2OneDNNOp3(TestSqueeze2OneDNNOp): class TestSqueezeOneDNNOp3(TestSqueezeOneDNNOp): - def init_test_case(self): self.ori_shape = (25, 1, 1, 4, 1) self.axes = (1, -1) @@ -120,10 +118,8 @@ class TestSqueezeOneDNNOp3(TestSqueezeOneDNNOp): # BF16 TESTS def create_squeeze_bf16_test_classes(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestSqueeze2BF16OneDNNOp(parent): - def set_inputs(self): self.dtype = np.uint16 self.inputs = {"X": convert_float_to_uint16(self.x)} @@ -134,17 +130,19 @@ def create_squeeze_bf16_test_classes(parent): def test_check_grad(self): self.calculate_grads() - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - user_defined_grads=[self.dx], - user_defined_grad_outputs=[self.dout]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + user_defined_grads=[self.dx], + user_defined_grad_outputs=[self.dout], + ) cls_name = "{0}_{1}".format(parent.__name__, "Squeeze2_BF16") TestSqueeze2BF16OneDNNOp.__name__ = cls_name globals()[cls_name] = TestSqueeze2BF16OneDNNOp class TestSqueezeBF16OneDNNOp(TestSqueeze2BF16OneDNNOp): - def set_op_type(self): self.dtype = np.uint16 self.op_type = "squeeze" diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py index 5638c14b9e4..4052d50a474 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py @@ -21,7 +21,6 @@ import paddle.fluid.core as core @OpTestTool.skip_if_not_cpu() class TestStack2DOneDNNOp(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (2, 2) @@ -45,7 +44,8 @@ class TestStack2DOneDNNOp(OpTest): for i in range(self.num_inputs): self.op_inputs.append( - np.random.random(size=self.input_dim).astype(np.float32)) + np.random.random(size=self.input_dim).astype(np.float32) + ) input_list = [] input_names = self.getInputNames() @@ -65,21 +65,18 @@ class TestStack2DOneDNNOp(OpTest): class TestStack1DOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): - self.input_dim = (100) + self.input_dim = 100 self.axis = 0 class TestStack1DAxis1OneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): - self.input_dim = (100) + self.input_dim = 100 self.axis = 1 class TestStack2DAxisLastOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (13, 24) self.num_inputs = 5 @@ -87,14 +84,12 @@ class TestStack2DAxisLastOneDNNOp(TestStack2DOneDNNOp): class TestStack3DAxisNegativeOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (10, 128, 128) self.axis = -2 class TestStack3DOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (10, 128, 128) self.num_inputs = 3 @@ -102,7 +97,6 @@ class TestStack3DOneDNNOp(TestStack2DOneDNNOp): class TestStack4DOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (2, 2, 2, 2) self.num_inputs = 3 @@ -110,7 +104,6 @@ class TestStack4DOneDNNOp(TestStack2DOneDNNOp): class TestStack5DOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (2, 3, 4, 5, 6) self.num_inputs = 6 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_sum_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_sum_bf16_mkldnn_op.py index 6792eaf7e69..435651f38a7 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_sum_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_sum_bf16_mkldnn_op.py @@ -20,10 +20,10 @@ from paddle import enable_static import numpy as np -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestSumBF16MKLDNN(TestSumOp): - def setUp(self): self.op_type = "sum" self.use_mkldnn = True diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_sum_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_sum_mkldnn_op.py index a9c345e9dfa..ab26f101687 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_sum_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_sum_mkldnn_op.py @@ -20,7 +20,6 @@ import paddle.fluid.op as fluid_op class TestSumMKLDNN(TestSumOp): - def setUp(self): self.op_type = "sum" self.init_data_type() @@ -46,7 +45,6 @@ class TestSumMKLDNN(TestSumOp): class TestMKLDNNSumInplaceOp(unittest.TestCase): - def setUp(self): self.op_type = "sum" self.init_data_type() @@ -70,10 +68,9 @@ class TestMKLDNNSumInplaceOp(unittest.TestCase): tensor = var.get_tensor() tensor.set(var_value, place) - sum_op = fluid_op.Operator("sum", - X=["x0", "x1"], - Out=out_var_name, - use_mkldnn=True) + sum_op = fluid_op.Operator( + "sum", X=["x0", "x1"], Out=out_var_name, use_mkldnn=True + ) expected_out = np.array(self.x0 + self.x1) sum_op.run(scope, place) out = scope.find_var("x0").get_tensor() @@ -83,7 +80,7 @@ class TestMKLDNNSumInplaceOp(unittest.TestCase): out_array, rtol=1e-05, atol=1e-05, - err_msg='Inplace sum_mkldnn_op output has diff with expected output' + err_msg='Inplace sum_mkldnn_op output has diff with expected output', ) def test_check_grad(self): @@ -92,5 +89,6 @@ class TestMKLDNNSumInplaceOp(unittest.TestCase): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_bf16_mkldnn_op.py index 2edab93d778..352e35f66dd 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_bf16_mkldnn_op.py @@ -19,10 +19,10 @@ from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestTransposeOp(OpTest): - def setUp(self): self.op_type = "transpose2" self.use_mkldnn = True @@ -36,12 +36,12 @@ class TestTransposeOp(OpTest): self.attrs = { 'axis': list(self.axis), 'use_mkldnn': self.use_mkldnn, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } self.outputs = { 'XShape': np.random.random(self.shape).astype(np.uint16), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def test_check_output(self): @@ -52,11 +52,11 @@ class TestTransposeOp(OpTest): def init_test_data(self): self.input_data = convert_float_to_uint16( - np.random.random(self.shape).astype(np.float32)) + np.random.random(self.shape).astype(np.float32) + ) class TestBF16Case(TestTransposeOp): - def init_test_case(self): self.shape = (2, 4, 6, 8) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_int8_mkldnn_op.py index 5b3be06aa75..15c89a37e09 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_int8_mkldnn_op.py @@ -20,7 +20,6 @@ from mkldnn_op_test import format_reorder class TestTransposeOp(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -31,7 +30,7 @@ class TestTransposeOp(OpTest): self.inputs = { 'X': format_reorder(self.input_data, self.shape).astype(np.int8) - } #transform data format to 'NHWC' for INT8 transpose specially. + } # transform data format to 'NHWC' for INT8 transpose specially. self.attrs = { 'axis': list(self.axis), @@ -40,7 +39,7 @@ class TestTransposeOp(OpTest): self.outputs = { 'XShape': np.random.random(self.shape).astype(np.int8), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -48,37 +47,37 @@ class TestTransposeOp(OpTest): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(core.CPUPlace(), - 1e-5, - no_check_set=['XShape'], - check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), 1e-5, no_check_set=['XShape'], check_dygraph=False + ) def initTestCase(self): self.shape = (2, 3, 4, 5) def initInputData(self): self.input_data = (np.random.randint(0, 100, self.shape) - 50).astype( - np.int8) + np.int8 + ) class TestINT8Case(TestTransposeOp): - def initTestCase(self): self.shape = (2, 4, 6, 8) def initInputData(self): self.input_data = (np.random.randint(0, 100, self.shape) - 50).astype( - np.int8) + np.int8 + ) class TestUINT8Case(TestTransposeOp): - def initTestCase(self): self.shape = (1, 3, 5, 7) def initDataType(self): - self.input_data = (np.random.randint(0, 100, - self.shape)).astype(np.uint8) + self.input_data = (np.random.randint(0, 100, self.shape)).astype( + np.uint8 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_mkldnn_op.py index 5e81358c688..292c98b4dae 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_mkldnn_op.py @@ -19,7 +19,6 @@ import numpy as np class TestTransposeMKLDNN(TestTransposeOp): - def setUp(self): self.init_op_type() self.initTestCase() @@ -30,7 +29,7 @@ class TestTransposeMKLDNN(TestTransposeOp): } self.outputs = { 'XShape': np.random.random(self.shape).astype("float32"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -52,42 +51,36 @@ class TestTransposeMKLDNN(TestTransposeOp): class TestCase0MKLDNN(TestTransposeMKLDNN): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1a(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase1b(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (2, 1, 0) class TestCase2(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) diff --git a/python/paddle/fluid/tests/unittests/mlu/c_comm_init_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/c_comm_init_op_mlu.py index 9e6385bda4a..ca779db12c2 100644 --- a/python/paddle/fluid/tests/unittests/mlu/c_comm_init_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/c_comm_init_op_mlu.py @@ -16,14 +16,15 @@ import unittest import os import paddle.fluid.core as core import paddle.fluid as fluid -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) import paddle paddle.enable_static() class TestCCommInitOp(unittest.TestCase): - def setUp(self): self.endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS").split(',') self.current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT") @@ -43,24 +44,29 @@ class TestCCommInitOp(unittest.TestCase): cncl_id_var = block.create_var( name=fluid.unique_name.generate('cncl_id'), persistable=True, - type=fluid.core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_cncl_id', - inputs={}, - outputs={'Out': cncl_id_var}, - attrs={ - 'rank': self.rank, - 'endpoint': self.current_endpoint, - 'other_endpoints': self.other_endpoints - }) - block.append_op(type='c_comm_init', - inputs={'X': cncl_id_var}, - outputs={}, - attrs={ - 'nranks': self.nranks, - 'rank': self.rank, - 'ring_id': 0, - 'device_id': self.mlu_id - }) + type=fluid.core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_cncl_id', + inputs={}, + outputs={'Out': cncl_id_var}, + attrs={ + 'rank': self.rank, + 'endpoint': self.current_endpoint, + 'other_endpoints': self.other_endpoints, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': cncl_id_var}, + outputs={}, + attrs={ + 'nranks': self.nranks, + 'rank': self.rank, + 'ring_id': 0, + 'device_id': self.mlu_id, + }, + ) self.exe.run(program) diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_allgather_api.py b/python/paddle/fluid/tests/unittests/mlu/collective_allgather_api.py index c9c681ae38b..1d8d11ad112 100755 --- a/python/paddle/fluid/tests/unittests/mlu/collective_allgather_api.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_allgather_api.py @@ -30,22 +30,24 @@ import unittest from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_collective_api_base_mlu import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_api_base_mlu import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) paddle.enable_static() class TestCollectiveAllgatherAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): tensor_list = [] - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.all_gather(tensor_list, tindata) return tensor_list diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_allgather_op.py b/python/paddle/fluid/tests/unittests/mlu/collective_allgather_op.py index 727f3011ff1..8040e834d65 100755 --- a/python/paddle/fluid/tests/unittests/mlu/collective_allgather_op.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_allgather_op.py @@ -35,7 +35,6 @@ paddle.enable_static() class TestCollectiveAllgather(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -43,26 +42,28 @@ class TestCollectiveAllgather(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofallgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allgather", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allgather", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_api.py b/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_api.py index 7ea3a2f00c5..22ca990c55a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_api.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_api.py @@ -30,21 +30,23 @@ import unittest from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_collective_api_base_mlu import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_api_base_mlu import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) paddle.enable_static() class TestCollectiveAllreduceAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.all_reduce(tindata) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_op.py b/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_op.py index 19da310741d..609d9e7c416 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_op.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_op.py @@ -36,30 +36,34 @@ paddle.enable_static() class TestCollectiveAllreduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, col_type): ring_id = 0 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outof" + col_type, dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_" + col_type, - inputs={'X': tindata}, - attrs={'ring_id': ring_id}, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_" + col_type, + inputs={'X': tindata}, + attrs={'ring_id': ring_id}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_api.py b/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_api.py index c14456704df..0c1ae572251 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_api.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_api.py @@ -30,21 +30,23 @@ import unittest from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_collective_api_base_mlu import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_api_base_mlu import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) paddle.enable_static() class TestCollectiveBroadcastAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype="float32") + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype="float32" + ) paddle.distributed.broadcast(tindata, src=1) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_op.py b/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_op.py index 3dd50f09a8f..982d5d204fc 100755 --- a/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_op.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_op.py @@ -36,7 +36,6 @@ paddle.enable_static() class TestCollectiveBroadcast(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -44,26 +43,28 @@ class TestCollectiveBroadcast(TestCollectiveRunnerBase): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofbroadcast", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_broadcast", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_broadcast", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root': rootid}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_reduce_api.py b/python/paddle/fluid/tests/unittests/mlu/collective_reduce_api.py index 4f257d07fa3..33a0d9d0d51 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_reduce_api.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_reduce_api.py @@ -30,21 +30,23 @@ import unittest from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_collective_api_base_mlu import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_api_base_mlu import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) paddle.enable_static() class TestCollectiveReduceAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.reduce(tindata, dst=0) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_reduce_op.py b/python/paddle/fluid/tests/unittests/mlu/collective_reduce_op.py index 733440cb0de..d9271376b0f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_reduce_op.py @@ -36,7 +36,6 @@ paddle.enable_static() class TestCollectiveReduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -44,26 +43,28 @@ class TestCollectiveReduce(TestCollectiveRunnerBase): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outof" + col_type, dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_" + col_type, - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root_id': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_" + col_type, + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root_id': rootid}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/mlu/multi_process_mlu.py b/python/paddle/fluid/tests/unittests/mlu/multi_process_mlu.py index 782475ff8cb..96148a7bacc 100644 --- a/python/paddle/fluid/tests/unittests/mlu/multi_process_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/multi_process_mlu.py @@ -26,12 +26,18 @@ def train(prefix): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_mlus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_mlus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) @@ -48,23 +54,34 @@ def train_abort(prefix): # train abort exit(1) except SystemExit: - name = "abort>>> selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_mlus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "abort>>> selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_mlus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) with open( - "multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) raise else: # sleep 30s to make sure paddle.distributed.launch will terminate this process time.sleep(30) - name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_mlus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_mlus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) diff --git a/python/paddle/fluid/tests/unittests/mlu/nproc_process_mlu.py b/python/paddle/fluid/tests/unittests/mlu/nproc_process_mlu.py index 9b2713532e4..2a070bf6722 100644 --- a/python/paddle/fluid/tests/unittests/mlu/nproc_process_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/nproc_process_mlu.py @@ -25,8 +25,13 @@ def train(prefix): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_mlus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_mlus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) with open("{}.check_{}.log".format(prefix, trainer_id), "w") as f: diff --git a/python/paddle/fluid/tests/unittests/mlu/parallel_dygraph_sync_batch_norm.py b/python/paddle/fluid/tests/unittests/mlu/parallel_dygraph_sync_batch_norm.py index b562f9ce4ac..56abdfc26dd 100644 --- a/python/paddle/fluid/tests/unittests/mlu/parallel_dygraph_sync_batch_norm.py +++ b/python/paddle/fluid/tests/unittests/mlu/parallel_dygraph_sync_batch_norm.py @@ -32,37 +32,42 @@ from test_dist_base import runtime_main, TestParallelDyGraphRunnerBase class TestLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(TestLayer, self).__init__() - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) self._sync_batch_norm = SyncBatchNorm(num_filters) - self._conv2 = Conv2D(in_channels=num_filters, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv2 = Conv2D( + in_channels=num_filters, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) - self._sync_batch_norm2 = SyncBatchNorm(num_filters, - weight_attr=False, - bias_attr=False) + self._sync_batch_norm2 = SyncBatchNorm( + num_filters, weight_attr=False, bias_attr=False + ) def forward(self, inputs): y = self._conv(inputs) @@ -74,20 +79,23 @@ class TestLayer(fluid.dygraph.Layer): class TestSyncBatchNorm(TestParallelDyGraphRunnerBase): - def get_model(self): model = TestLayer(3, 64, 7) - train_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=32, - drop_last=True) - opt = fluid.optimizer.Adam(learning_rate=1e-3, - parameter_list=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), + batch_size=32, + drop_last=True, + ) + opt = fluid.optimizer.Adam( + learning_rate=1e-3, parameter_list=model.parameters() + ) return model, train_reader, opt def run_one_loop(self, model, opt, data): batch_size = len(data) - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') + dy_x_data = np.array([x[0].reshape(3, 224, 224) for x in data]).astype( + 'float32' + ) img = to_variable(dy_x_data) img.stop_gradient = False diff --git a/python/paddle/fluid/tests/unittests/mlu/sync_batch_norm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/sync_batch_norm_op_mlu.py index eb410f14f54..3ea79600954 100644 --- a/python/paddle/fluid/tests/unittests/mlu/sync_batch_norm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/sync_batch_norm_op_mlu.py @@ -31,7 +31,10 @@ import unittest from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_sync_batch_norm_base_mlu import TestSyncBatchNormRunnerBase, runtime_main +from test_sync_batch_norm_base_mlu import ( + TestSyncBatchNormRunnerBase, + runtime_main, +) from op_test import OpTest, _set_use_system_allocator from test_sync_batch_norm_op import create_or_get_tensor @@ -41,7 +44,6 @@ paddle.enable_static() class TestSyncBatchNormOpTraining(TestSyncBatchNormRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -54,29 +56,34 @@ class TestSyncBatchNormOpTraining(TestSyncBatchNormRunnerBase): self.dshape = [self.N, self.C, self.H, self.W] self.atol = 1e-3 - def get_model(self, - main, - startup, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def get_model( + self, + main, + startup, + place, + layout, + seed, + sync_bn=False, + only_forward=False, + ): """Build program.""" use_cudnn = False with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - data = fluid.layers.data(name='input', - shape=self.dshape, - dtype=self.dtype, - append_batch_size=False) + data = fluid.layers.data( + name='input', + shape=self.dshape, + dtype=self.dtype, + append_batch_size=False, + ) conv = fluid.layers.conv2d( input=data, num_filters=32, filter_size=1, param_attr=fluid.ParamAttr(name='conv2d_weight'), bias_attr=False, - use_cudnn=use_cudnn) + use_cudnn=use_cudnn, + ) if self.bn_dtype == np.float16: conv = fluid.layers.cast(conv, 'float16') bn = fluid.layers.batch_norm( @@ -86,7 +93,8 @@ class TestSyncBatchNormOpTraining(TestSyncBatchNormRunnerBase): moving_mean_name='bn_moving_mean', moving_variance_name='bn_moving_variance', data_layout=layout, - is_test=only_forward) + is_test=only_forward, + ) if self.bn_dtype == np.float16: bn = fluid.layers.cast(bn, 'float32') sigmoid = fluid.layers.sigmoid(bn) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_abs_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_abs_op_mlu.py index fe56236b8e2..c9e09fccf20 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_abs_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_abs_op_mlu.py @@ -29,7 +29,6 @@ np.random.seed(10) class TestAbs(OpTest): - def setUp(self): self.op_type = "abs" self.set_mlu() @@ -56,12 +55,12 @@ class TestAbs(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], ['Out'], - check_eager=False) + self.check_grad_with_place( + self.place, ['X'], ['Out'], check_eager=False + ) class TestAbsHalf(OpTest): - def setUp(self): self.op_type = "abs" self.set_mlu() @@ -88,8 +87,9 @@ class TestAbsHalf(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], ['Out'], - check_eager=False) + self.check_grad_with_place( + self.place, ['X'], ['Out'], check_eager=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/mlu/test_accuracy_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_accuracy_op_mlu.py index 0cdc2045ec7..e37be2f9f3d 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_accuracy_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_accuracy_op_mlu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestAccuracyOp(OpTest): - def setUp(self): self.op_type = "accuracy" self.place = paddle.device.MLUPlace(0) @@ -47,7 +46,7 @@ class TestAccuracyOp(OpTest): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } def init_dtype(self): @@ -58,7 +57,6 @@ class TestAccuracyOp(OpTest): class TestAccuracyOpFp16(TestAccuracyOp): - def init_dtype(self): self.dtype = np.float16 @@ -67,15 +65,15 @@ class TestAccuracyOpFp16(TestAccuracyOp): class TestAccuracyOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.MLUPlace(0)) - label = fluid.layers.data(name='label', - shape=[-1, 1], - dtype="int32") + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.MLUPlace(0) + ) + label = fluid.layers.data( + name='label', shape=[-1, 1], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.accuracy, x1, label) self.assertRaises(TypeError, paddle.metric.accuracy, x1, label) # The input dtype of accuracy_op must be float32 or float64. @@ -88,40 +86,42 @@ class TestAccuracyOpError(unittest.TestCase): class TestAccuracyAPI1(unittest.TestCase): - def setUp(self): - self.predictions = paddle.static.data(shape=[2, 5], - name="predictions", - dtype="float32") - self.label = paddle.static.data(shape=[2, 1], - name="labels", - dtype="int32") - self.result = paddle.static.accuracy(input=self.predictions, - label=self.label, - k=1) + self.predictions = paddle.static.data( + shape=[2, 5], name="predictions", dtype="float32" + ) + self.label = paddle.static.data( + shape=[2, 1], name="labels", dtype="int32" + ) + self.result = paddle.static.accuracy( + input=self.predictions, label=self.label, k=1 + ) self.input_predictions = np.array( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype="float32") + dtype="float32", + ) self.input_labels = np.array([[2], [0]], dtype="int32") self.expect_value = np.array([0.5], dtype='float32') def test_api(self): exe = paddle.static.Executor() - result, = exe.run(feed={ - "predictions": self.input_predictions, - 'labels': self.input_labels - }, - fetch_list=[self.result.name]) + (result,) = exe.run( + feed={ + "predictions": self.input_predictions, + 'labels': self.input_labels, + }, + fetch_list=[self.result.name], + ) self.assertEqual((result == self.expect_value).all(), True) class TestAccuracyAPI2(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): predictions = paddle.to_tensor( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype='float32') + dtype='float32', + ) label = paddle.to_tensor([[2], [0]], dtype="int32") result = paddle.static.accuracy(input=predictions, label=label, k=1) expect_value = np.array([0.5], dtype='float32') @@ -129,12 +129,12 @@ class TestAccuracyAPI2(unittest.TestCase): class TestAccuracyAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): predictions = paddle.to_tensor( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype='float32') + dtype='float32', + ) label = paddle.to_tensor([[2], [0]], dtype="int32") result = paddle.metric.accuracy(input=predictions, label=label, k=1) expect_value = np.array([0.5], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/mlu/test_adam_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_adam_op_mlu.py index 3ac6e23d21a..70a3a2671b6 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_adam_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_adam_op_mlu.py @@ -28,7 +28,6 @@ SEED = 2022 class TestAdam(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adam" @@ -52,20 +51,19 @@ class TestAdam(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_mlu(self): @@ -80,7 +78,6 @@ class TestAdam(OpTest): class TestAdamWithEpsilonTensor(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adam" @@ -112,15 +109,14 @@ class TestAdamWithEpsilonTensor(OpTest): self.attrs = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_mlu(self): @@ -135,7 +131,6 @@ class TestAdamWithEpsilonTensor(OpTest): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adam" @@ -188,7 +183,6 @@ class TestAdamOpWithSkipUpdate(OpTest): class TestAdamOpWithGlobalBetaPow(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adam" @@ -220,8 +214,7 @@ class TestAdamOpWithGlobalBetaPow(OpTest): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.attrs = {'use_global_beta_pow': True} @@ -231,7 +224,7 @@ class TestAdamOpWithGlobalBetaPow(OpTest): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([]), - 'Beta2PowOut': np.array([]) + 'Beta2PowOut': np.array([]), } def set_mlu(self): @@ -246,7 +239,6 @@ class TestAdamOpWithGlobalBetaPow(OpTest): class TestNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -261,9 +253,9 @@ class TestNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -287,16 +279,17 @@ class TestNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_adamw_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_adamw_op_mlu.py index e60664f27a0..c9b3b527e72 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_adamw_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_adamw_op_mlu.py @@ -28,7 +28,6 @@ SEED = 2022 class TestAdamW(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adamw" @@ -52,7 +51,7 @@ class TestAdamW(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = { @@ -60,18 +59,19 @@ class TestAdamW(OpTest): 'beta1': beta1, 'beta2': beta2, "coeff": 0.9, - "with_decay": True + "with_decay": True, } - param_out, moment1_out, \ - moment2_out = adamw_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adamw_step( + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_mlu(self): @@ -86,7 +86,6 @@ class TestAdamW(OpTest): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adamw" @@ -139,7 +138,6 @@ class TestAdamOpWithSkipUpdate(OpTest): class TestAdamOpWithoutDecay(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adamw" @@ -192,7 +190,6 @@ class TestAdamOpWithoutDecay(OpTest): class TestNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -207,9 +204,9 @@ class TestNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -233,16 +230,17 @@ class TestNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_amp_check_finite_and_scale_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_amp_check_finite_and_scale_op_mlu.py index 3e113847b7a..7a2e22a18ad 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_amp_check_finite_and_scale_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_amp_check_finite_and_scale_op_mlu.py @@ -25,7 +25,6 @@ SEED = 2022 class TestCheckFiniteAndUnscaleOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "check_finite_and_unscale" @@ -54,7 +53,6 @@ class TestCheckFiniteAndUnscaleOp(OpTest): class TestCheckFiniteAndUnscaleOpWithNan(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x = np.random.random((129, 129)).astype(self.dtype) x[128][128] = np.nan @@ -73,7 +71,6 @@ class TestCheckFiniteAndUnscaleOpWithNan(TestCheckFiniteAndUnscaleOp): class TestCheckFiniteAndUnscaleOpWithInf(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x = np.random.random((129, 129)).astype(self.dtype) x[128][128] = np.inf @@ -92,7 +89,6 @@ class TestCheckFiniteAndUnscaleOpWithInf(TestCheckFiniteAndUnscaleOp): class TestCheckFiniteAndUnscaleOpMultiInput(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x0 = np.random.random((129, 129)).astype(self.dtype) x1 = np.random.random((129, 129)).astype(self.dtype) @@ -106,7 +102,6 @@ class TestCheckFiniteAndUnscaleOpMultiInput(TestCheckFiniteAndUnscaleOp): class TestCheckFiniteAndUnscaleOpMultiInputWithNan(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x0 = np.random.random((129, 129)).astype(self.dtype) x0[128][128] = np.nan @@ -126,7 +121,6 @@ class TestCheckFiniteAndUnscaleOpMultiInputWithNan(TestCheckFiniteAndUnscaleOp): class TestCheckFiniteAndUnscaleOpMultiInputWithInf(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x0 = np.random.random((129, 129)).astype(self.dtype) x0[128][128] = np.nan diff --git a/python/paddle/fluid/tests/unittests/mlu/test_arg_max_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_arg_max_op_mlu.py index e8a41018743..81d854b4875 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_arg_max_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_arg_max_op_mlu.py @@ -27,7 +27,6 @@ paddle.enable_static() class BaseTestCase(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.MLUPlace(0) @@ -51,7 +50,6 @@ class BaseTestCase(OpTest): class TestArgMaxSameValue1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dtype = 'float32' @@ -67,7 +65,6 @@ class TestArgMaxSameValue1(BaseTestCase): class TestArgMaxSameValue2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dtype = 'float16' @@ -84,7 +81,6 @@ class TestArgMaxSameValue2(BaseTestCase): # test argmax, dtype: float16 class TestArgMaxFloat16Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -93,7 +89,6 @@ class TestArgMaxFloat16Case1(BaseTestCase): class TestArgMaxFloat16Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -102,7 +97,6 @@ class TestArgMaxFloat16Case2(BaseTestCase): class TestArgMaxFloat16Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -111,7 +105,6 @@ class TestArgMaxFloat16Case3(BaseTestCase): class TestArgMaxFloat16Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -120,7 +113,6 @@ class TestArgMaxFloat16Case4(BaseTestCase): class TestArgMaxFloat16Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -129,7 +121,6 @@ class TestArgMaxFloat16Case5(BaseTestCase): class TestArgMaxFloat16Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -138,7 +129,6 @@ class TestArgMaxFloat16Case6(BaseTestCase): class TestArgMaxFloat16Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -147,35 +137,31 @@ class TestArgMaxFloat16Case7(BaseTestCase): class TestArgMaxFloat16Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float16' self.axis = 0 class TestArgMaxFloat16Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float16' self.axis = 0 class TestArgMaxFloat16Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float16' self.axis = 0 # test argmax, dtype: float32 class TestArgMaxFloat32Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -184,7 +170,6 @@ class TestArgMaxFloat32Case1(BaseTestCase): class TestArgMaxFloat32Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -193,7 +178,6 @@ class TestArgMaxFloat32Case2(BaseTestCase): class TestArgMaxFloat32Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -202,7 +186,6 @@ class TestArgMaxFloat32Case3(BaseTestCase): class TestArgMaxFloat32Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -211,7 +194,6 @@ class TestArgMaxFloat32Case4(BaseTestCase): class TestArgMaxFloat32Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -220,7 +202,6 @@ class TestArgMaxFloat32Case5(BaseTestCase): class TestArgMaxFloat32Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -229,7 +210,6 @@ class TestArgMaxFloat32Case6(BaseTestCase): class TestArgMaxFloat32Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -238,34 +218,30 @@ class TestArgMaxFloat32Case7(BaseTestCase): class TestArgMaxFloat32Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float32' self.axis = 0 class TestArgMaxFloat32Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float32' self.axis = 0 class TestArgMaxFloat32Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float32' self.axis = 0 class BaseTestComplex1_1(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.MLUPlace(0) @@ -283,7 +259,7 @@ class BaseTestComplex1_1(OpTest): self.inputs = {'X': self.x} self.attrs = { 'axis': self.axis, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { 'Out': np.argmax(self.x, axis=self.axis).astype("int32") @@ -294,7 +270,6 @@ class BaseTestComplex1_1(OpTest): class BaseTestComplex1_2(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.MLUPlace(0) @@ -312,7 +287,7 @@ class BaseTestComplex1_2(OpTest): self.inputs = {'X': self.x} self.attrs = { 'axis': self.axis, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { 'Out': np.argmax(self.x, axis=self.axis).astype("int32") @@ -323,7 +298,6 @@ class BaseTestComplex1_2(OpTest): class TestArgMaxAPI(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -335,7 +309,6 @@ class TestArgMaxAPI(unittest.TestCase): self.place = [paddle.MLUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2022) @@ -343,9 +316,9 @@ class TestArgMaxAPI(unittest.TestCase): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input, axis=self.axis) paddle_output = paddle.argmax(tensor_input, axis=self.axis) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() for place in self.place: @@ -353,7 +326,6 @@ class TestArgMaxAPI(unittest.TestCase): class TestArgMaxAPI_2(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -366,20 +338,20 @@ class TestArgMaxAPI_2(unittest.TestCase): self.place = [paddle.MLUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2022) numpy_input = (np.random.random(self.dims)).astype(self.dtype) tensor_input = paddle.to_tensor(numpy_input) - numpy_output = np.argmax(numpy_input, - axis=self.axis).reshape(1, 4, 5) - paddle_output = paddle.argmax(tensor_input, - axis=self.axis, - keepdim=self.keep_dims) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + numpy_output = np.argmax(numpy_input, axis=self.axis).reshape( + 1, 4, 5 + ) + paddle_output = paddle.argmax( + tensor_input, axis=self.axis, keepdim=self.keep_dims + ) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -388,7 +360,6 @@ class TestArgMaxAPI_2(unittest.TestCase): class TestArgMaxAPI_3(unittest.TestCase): - def initTestCase(self): self.dims = (1, 9) self.dtype = 'float32' @@ -399,7 +370,6 @@ class TestArgMaxAPI_3(unittest.TestCase): self.place = [paddle.MLUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2022) @@ -407,9 +377,9 @@ class TestArgMaxAPI_3(unittest.TestCase): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input).reshape([1]) paddle_output = paddle.argmax(tensor_input) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_argsort_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_argsort_op_mlu.py index 6e474ca604e..4f96e3b5ccd 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_argsort_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_argsort_op_mlu.py @@ -26,9 +26,7 @@ SEED = 2022 def gen_test_class(dtype, axis, descending): - class TestArgsortOp(OpTest): - def setUp(self): np.random.seed(SEED) self.set_mlu() @@ -48,9 +46,11 @@ def gen_test_class(dtype, axis, descending): def get_output(self): if descending: self.indices = np.flip( - np.argsort(self.x, kind='heapsort', axis=axis), axis) + np.argsort(self.x, kind='heapsort', axis=axis), axis + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=axis), axis) + np.sort(self.x, kind='heapsort', axis=axis), axis + ) else: self.indices = np.argsort(self.x, kind='heapsort', axis=axis) self.sorted_x = np.sort(self.x, kind='heapsort', axis=axis) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_assign_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_assign_op_mlu.py index 91327f74845..7ee93fa0097 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_assign_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_assign_op_mlu.py @@ -25,7 +25,6 @@ SEED = 2022 class TestAssign(OpTest): - def setUp(self): self.set_mlu() self.op_type = "assign" diff --git a/python/paddle/fluid/tests/unittests/mlu/test_assign_value_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_assign_value_op_mlu.py index 6f120560f98..535ae9f8c86 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_assign_value_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_assign_value_op_mlu.py @@ -29,7 +29,6 @@ numpy.random.seed(2022) class TestAssignValueMLUOp(op_test.OpTest): - def setUp(self): self.set_mlu() self.op_type = "assign_value" @@ -39,7 +38,8 @@ class TestAssignValueMLUOp(op_test.OpTest): self.attrs["shape"] = self.value.shape self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( - self.value.dtype) + self.value.dtype + ) self.outputs = {"Out": self.value} def set_mlu(self): @@ -55,24 +55,22 @@ class TestAssignValueMLUOp(op_test.OpTest): class TestAssignValueMLUOp2(TestAssignValueMLUOp): - def init_data(self): self.value = numpy.random.random(size=(2, 5)).astype(numpy.int32) self.attrs["int32_values"] = [int(v) for v in self.value.flat] class TestAssignValueMLUOp3(TestAssignValueMLUOp): - def init_data(self): self.value = numpy.random.random(size=(2, 5)).astype(numpy.int64) self.attrs["int64_values"] = [int(v) for v in self.value.flat] class TestAssignValueMLUOp4(TestAssignValueMLUOp): - def init_data(self): - self.value = numpy.random.choice(a=[False, True], - size=(2, 5)).astype(numpy.bool) + self.value = numpy.random.choice(a=[False, True], size=(2, 5)).astype( + numpy.bool + ) self.attrs["bool_values"] = [int(v) for v in self.value.flat] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu.py index d97a1e074d6..53b78e18f88 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu.py @@ -158,12 +158,20 @@ def _reference_grad(x, y_grad, scale, mean, var, epsilon, data_format): x = np.transpose(x, (0, 2, 3, 1)) y_grad = np.transpose(y_grad, (0, 2, 3, 1)) - x_grad = scale * (y_grad - np.mean(y_grad, axis=(0, 1, 2)) - - (x - mean) * np.mean(y_grad * - (x - mean), axis=(0, 1, 2)) / - (var + epsilon)) / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + x_grad = ( + scale + * ( + y_grad + - np.mean(y_grad, axis=(0, 1, 2)) + - (x - mean) + * np.mean(y_grad * (x - mean), axis=(0, 1, 2)) + / (var + epsilon) + ) + / np.sqrt(var + epsilon) + ) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -188,7 +196,6 @@ def create_or_get_tensor(scope, var_name, var, place): def set_output_grad(scope, outputs, place, feed_dict=None): - def __set_tensor__(name, data=None): out_tensor = scope.find_var(name).get_tensor() grad_tensor = scope.var(grad_var_name(name)).get_tensor() @@ -210,17 +217,15 @@ def set_output_grad(scope, outputs, place, feed_dict=None): class TestBatchNormOpInference(unittest.TestCase): - def setUp(self): self.dtype = np.float32 self.fuse_with_relu = False self.init_kernel_type() def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, atol=atol, err_msg=msg + ) def check_with_place(self, place, data_layout, dtype, shape): epsilon = 0.00001 @@ -246,34 +251,39 @@ class TestBatchNormOpInference(unittest.TestCase): mean = np.zeros(scale_shape).astype(np.float32) variance = np.ones(scale_shape).astype(np.float32) - y_out = _reference_testing(x_val, scale_val, bias_val, mean, variance, - epsilon, data_layout).astype(dtype) + y_out = _reference_testing( + x_val, scale_val, bias_val, mean, variance, epsilon, data_layout + ).astype(dtype) if self.fuse_with_relu: y_out = np.maximum(y_out, 0) scope = core.Scope() # create input - x_tensor = create_or_get_tensor(scope, "x_val", - OpTest.np_dtype_to_fluid_dtype(x_val), - place) + x_tensor = create_or_get_tensor( + scope, "x_val", OpTest.np_dtype_to_fluid_dtype(x_val), place + ) scale_tensor = create_or_get_tensor( - scope, "scale_val", OpTest.np_dtype_to_fluid_dtype(scale_val), - place) + scope, "scale_val", OpTest.np_dtype_to_fluid_dtype(scale_val), place + ) bias_tensor = create_or_get_tensor( - scope, "bias_val", OpTest.np_dtype_to_fluid_dtype(bias_val), place) - mean_tensor = create_or_get_tensor(scope, "mean", - OpTest.np_dtype_to_fluid_dtype(mean), - place) + scope, "bias_val", OpTest.np_dtype_to_fluid_dtype(bias_val), place + ) + mean_tensor = create_or_get_tensor( + scope, "mean", OpTest.np_dtype_to_fluid_dtype(mean), place + ) variance_tensor = create_or_get_tensor( - scope, "variance", OpTest.np_dtype_to_fluid_dtype(variance), place) + scope, "variance", OpTest.np_dtype_to_fluid_dtype(variance), place + ) # create output y_tensor = create_or_get_tensor(scope, "y_out", None, place) - saved_mean_tensor = create_or_get_tensor(scope, "saved_mean", None, - place) - saved_variance_tensor = create_or_get_tensor(scope, "saved_variance", - None, place) + saved_mean_tensor = create_or_get_tensor( + scope, "saved_mean", None, place + ) + saved_variance_tensor = create_or_get_tensor( + scope, "saved_variance", None, place + ) mean_out_tensor = mean_tensor variance_out_tensor = variance_tensor @@ -296,17 +306,25 @@ class TestBatchNormOpInference(unittest.TestCase): data_layout=data_layout, use_mkldnn=False, fuse_with_relu=self.fuse_with_relu, - epsilon=epsilon) + epsilon=epsilon, + ) batch_norm_op.run(scope, place) # check inference result - self.__assert_close(y_tensor, - y_out, - "inference output are different at " + str(place) + - ", " + data_layout + ", " + str(np.dtype(dtype)) + - str(np.array(y_tensor)) + str(y_out), - atol=1e-3) + self.__assert_close( + y_tensor, + y_out, + "inference output are different at " + + str(place) + + ", " + + data_layout + + ", " + + str(np.dtype(dtype)) + + str(np.array(y_tensor)) + + str(y_out), + atol=1e-3, + ) def test_check_output(self): places = [core.CPUPlace()] @@ -315,8 +333,9 @@ class TestBatchNormOpInference(unittest.TestCase): for place in places: for data_format in ["NCHW", "NHWC"]: - self.check_with_place(place, data_format, self.dtype, - [2, 3, 4, 5]) + self.check_with_place( + place, data_format, self.dtype, [2, 3, 4, 5] + ) self.check_with_place(place, data_format, self.dtype, [2, 3]) def init_kernel_type(self): @@ -324,7 +343,6 @@ class TestBatchNormOpInference(unittest.TestCase): class TestFP16BatchNormOpInference(TestBatchNormOpInference): - def setUp(self): self.dtype = np.float16 self.fuse_with_relu = False @@ -337,13 +355,13 @@ class TestFP16BatchNormOpInference(TestBatchNormOpInference): for place in places: for data_format in ["NCHW", "NHWC"]: - self.check_with_place(place, data_format, self.dtype, - [2, 3, 4, 5]) + self.check_with_place( + place, data_format, self.dtype, [2, 3, 4, 5] + ) self.check_with_place(place, data_format, self.dtype, [2, 3]) class TestBatchNormOpTraining(unittest.TestCase): - def setUp(self): self.fuse_with_relu = False self.data_formats = ["NCHW", "NHWC"] @@ -357,27 +375,54 @@ class TestBatchNormOpTraining(unittest.TestCase): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def __assert_close(self, tensor, np_array, msg, atol=1e-4): np.allclose(np.array(tensor), np_array, atol=atol) - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): # run forward - y, saved_mean, var_ref = _reference_training(x, scale, bias, epsilon, - data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = var_ref * (1. - momentum) + momentum * variance - saved_variance = 1. / np.sqrt(var_ref + epsilon) + y, saved_mean, var_ref = _reference_training( + x, scale, bias, epsilon, data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = var_ref * (1.0 - momentum) + momentum * variance + saved_variance = 1.0 / np.sqrt(var_ref + epsilon) # run backward - x_grad, scale_grad, bias_grad = _reference_grad(x, y_grad, scale, - saved_mean, var_ref, - epsilon, data_layout) - - return y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad + x_grad, scale_grad, bias_grad = _reference_grad( + x, y_grad, scale, saved_mean, var_ref, epsilon, data_layout + ) + + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) def set_mean_variance(self, scale_shape, x, data_layout): mean, variance = _cal_mean_variance(x, self.epsilon, data_layout) @@ -386,12 +431,11 @@ class TestBatchNormOpTraining(unittest.TestCase): # computing global mean/variance for one step if self.use_global_stats: mom = self.momentum - mean = mean * (1. - mom) + mom * mean_pre - variance = variance * (1. - mom) + mom * variance_pre + mean = mean * (1.0 - mom) + mom * mean_pre + variance = variance * (1.0 - mom) + mom * variance_pre return mean, variance def test_forward_backward(self): - def test_with_place(place, data_layout, shape): # attr epsilon = self.epsilon @@ -410,9 +454,27 @@ class TestBatchNormOpTraining(unittest.TestCase): y_grad = np.random.random_sample(shape).astype(np.float32) momentum_var = np.array([momentum]).astype(np.float32) - y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad = self.ref_forward_backward( - x, y_grad, scale, bias, mean, variance, epsilon, momentum, - shape, data_layout) + ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) = self.ref_forward_backward( + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -421,8 +483,15 @@ class TestBatchNormOpTraining(unittest.TestCase): var_dict['bias@GRAD'] = bias_grad var_names = [ - 'x', 'scale', 'bias', 'mean', 'variance', 'y', 'saved_mean', - 'saved_variance', 'momentum_var' + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y', + 'saved_mean', + 'saved_variance', + 'momentum_var', ] ground_truth = {name: var_dict[name] for name in var_names} @@ -430,15 +499,17 @@ class TestBatchNormOpTraining(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = { "X": block.var('x'), "Scale": block.var('scale'), "Bias": block.var('bias'), "Mean": block.var('mean'), - "Variance": block.var('variance') + "Variance": block.var('variance'), } attrs = { "epsilon": epsilon, @@ -446,7 +517,7 @@ class TestBatchNormOpTraining(unittest.TestCase): "data_layout": data_layout, "use_mkldnn": False, "fuse_with_relu": self.fuse_with_relu, - "use_global_stats": self.use_global_stats + "use_global_stats": self.use_global_stats, } if self.use_momentum_variable: inputs['MomentumTensor'] = block.var('momentum_var') @@ -458,19 +529,22 @@ class TestBatchNormOpTraining(unittest.TestCase): "MeanOut": block.var('mean'), # share memory "VarianceOut": block.var('variance'), # share memory "SavedMean": block.var('saved_mean'), - "SavedVariance": block.var('saved_variance') + "SavedVariance": block.var('saved_variance'), } block.create_var(name="reserve_space", dtype='float32') outputs["ReserveSpace"] = block.var('reserve_space') - bn_op = block.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + bn_op = block.append_op( + type="batch_norm", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) block.create_var(name='y@GRAD', dtype='float32', shape=y.shape) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - bn_op.desc, self.no_grad_set, []) + bn_op.desc, self.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -485,22 +559,28 @@ class TestBatchNormOpTraining(unittest.TestCase): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in [ - 'x', 'scale', 'bias', 'mean', 'variance', - 'y@GRAD', 'momentum_var' - ] - }, - fetch_list=self.fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in [ + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y@GRAD', + 'momentum_var', + ] + }, + fetch_list=self.fetch_list, + ) for id, name in enumerate(self.fetch_list): if name == 'variance': - self.__assert_close(var_dict[name], - out[id], - name, - atol=1e-3) + self.__assert_close( + var_dict[name], out[id], name, atol=1e-3 + ) continue self.__assert_close(var_dict[name], out[id], name) print("op test forward passed: ", str(place), data_layout) @@ -519,7 +599,6 @@ class TestBatchNormOpTraining(unittest.TestCase): class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -527,19 +606,23 @@ class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): class TestBatchNormOpTrainingCase2(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] os.environ['FLAGS_cudnn_batchnorm_spatial_persistent'] = "1" class TestBatchNormOpTrainingCase3(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['x@GRAD']) @@ -547,24 +630,33 @@ class TestBatchNormOpTrainingCase3(TestBatchNormOpTraining): class TestBatchNormOpTrainingMomentumVariable(TestBatchNormOpTraining): - def init_test_case(self): self.use_momentum_variable = True self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = True self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'x@GRAD', 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): @@ -573,8 +665,9 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): y_grad = np.transpose(y_grad, (0, 2, 3, 1)) x_grad = scale * y_grad / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -585,8 +678,19 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): return x_grad, grad_scale, grad_offset - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): if data_layout != "NCHW" and data_layout != "NHWC": raise ValueError("Unknown data order.") @@ -604,17 +708,27 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): mean_out = mean variance_out = variance - saved_variance = 1. / np.sqrt(variance + epsilon) + saved_variance = 1.0 / np.sqrt(variance + epsilon) # run backward x_grad, scale_grad, bias_grad = self.reference_grad( - x, y_grad, scale, mean, variance, epsilon, data_layout) + x, y_grad, scale, mean, variance, epsilon, data_layout + ) - return y, mean_out, variance_out, mean, saved_variance, x_grad, scale_grad, bias_grad + return ( + y, + mean_out, + variance_out, + mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) class TestBatchNormOpFreezeStatsAndScaleBiasTraining( - TestBatchNormOpFreezeStatsTraining): - + TestBatchNormOpFreezeStatsTraining +): def init_test_case(self): self.use_global_stats = True self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -622,12 +736,12 @@ class TestBatchNormOpFreezeStatsAndScaleBiasTraining( class TestBatchNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of batch_norm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.batch_norm, x1) # the input dtype of batch_norm must be float16 or float32 or float64 @@ -637,13 +751,13 @@ class TestBatchNormOpError(unittest.TestCase): class TestDygraphBatchNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_norm = fluid.dygraph.BatchNorm(10) # the input of BatchNorm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, batch_norm, x1) # the input dtype of BatchNorm must be float16 or float32 or float64 @@ -653,7 +767,6 @@ class TestDygraphBatchNormAPIError(unittest.TestCase): class TestDygraphBatchNormTrainableStats(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_mlu(): @@ -666,7 +779,8 @@ class TestDygraphBatchNormTrainableStats(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -688,7 +802,8 @@ class TestDygraphBatchNormTrainableStats(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) @@ -702,7 +817,6 @@ class TestDygraphBatchNormTrainableStats(unittest.TestCase): class TestDygraphBatchNormOpenReserveSpace(unittest.TestCase): - def test_reservespace(self): with program_guard(Program(), Program()): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu_v2.py b/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu_v2.py index 27d25c6146e..72e7ac89caf 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu_v2.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu_v2.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestBatchNorm(unittest.TestCase): - def test_name(self): places = [fluid.CPUPlace()] if core.is_compiled_with_mlu(): @@ -45,7 +44,7 @@ class TestBatchNorm(unittest.TestCase): if core.is_compiled_with_mlu(): places.append(fluid.MLUPlace(0)) for p in places: - #paddle.disable_static() + # paddle.disable_static() x_data_4 = np.random.random(size=(2, 1, 3, 3)).astype('float32') x_data_3 = np.random.random(size=(2, 1, 3)).astype('float32') @@ -99,7 +98,8 @@ class TestBatchNorm(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -116,19 +116,22 @@ class TestBatchNorm(unittest.TestCase): is_test=is_test, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(1.0), - trainable=False), + trainable=False, + ), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(0.0), - trainable=False), - trainable_statistics=trainable_statistics) + trainable=False, + ), + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() def compute_v4(x): with fluid.dygraph.guard(p): - bn = paddle.nn.BatchNorm2D(shape[1], - weight_attr=False, - bias_attr=False) + bn = paddle.nn.BatchNorm2D( + shape[1], weight_attr=False, bias_attr=False + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -153,7 +156,8 @@ class TestBatchNorm(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) @@ -176,7 +180,6 @@ class TestBatchNorm(unittest.TestCase): class TestBatchNormChannelLast(unittest.TestCase): - def setUp(self): self.original_dtyep = paddle.get_default_dtype() paddle.set_default_dtype("float32") @@ -199,10 +202,9 @@ class TestBatchNormChannelLast(unittest.TestCase): channel_first_x = paddle.transpose(x, [0, 2, 1]) y2 = net2(channel_first_x) y2 = paddle.transpose(y2, [0, 2, 1]) - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) def test_2d(self): for p in self.places: @@ -216,10 +218,9 @@ class TestBatchNormChannelLast(unittest.TestCase): channel_first_x = paddle.transpose(x, [0, 3, 1, 2]) y2 = net2(channel_first_x) y2 = paddle.transpose(y2, [0, 2, 3, 1]) - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) def test_3d(self): for p in self.places: @@ -233,10 +234,9 @@ class TestBatchNormChannelLast(unittest.TestCase): channel_first_x = paddle.transpose(x, [0, 4, 1, 2, 3]) y2 = net2(channel_first_x) y2 = paddle.transpose(y2, [0, 2, 3, 4, 1]) - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) # res = np.allclose(y1.numpy(), y2.numpy()) # if res == False: # np.savetxt("./y1.txt", y1.numpy().flatten(), fmt='%.10f', delimiter='\n') @@ -245,7 +245,6 @@ class TestBatchNormChannelLast(unittest.TestCase): class TestBatchNormUseGlobalStats(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_mlu(): @@ -264,11 +263,14 @@ class TestBatchNormUseGlobalStats(unittest.TestCase): net1 = paddle.fluid.dygraph.BatchNorm( 6, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.0)), + initializer=fluid.initializer.Constant(1.0) + ), use_global_stats=self.use_global_stats, - trainable_statistics=self.trainable_statistics) + trainable_statistics=self.trainable_statistics, + ) net2 = paddle.nn.BatchNorm2D( - 6, use_global_stats=self.use_global_stats) + 6, use_global_stats=self.use_global_stats + ) net2.weight = net1.weight net2.bias = net1.bias if self.trainable_statistics == True: diff --git a/python/paddle/fluid/tests/unittests/mlu/test_bce_loss_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_bce_loss_mlu.py index 3805d27a14f..58ccf798136 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_bce_loss_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_bce_loss_mlu.py @@ -24,88 +24,76 @@ from op_test import OpTest paddle.enable_static() -def test_static_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') - bce_loss = paddle.nn.loss.BCELoss(weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) + bce_loss = paddle.nn.loss.BCELoss( + weight=weight, reduction=reduction + ) else: bce_loss = paddle.nn.loss.BCELoss(reduction=reduction) res = bce_loss(input, label) exe = paddle.static.Executor(place) - static_result = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + static_result = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result[0] -def test_static_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') - res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) + res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + static_result = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result[0] -def test_dygraph_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static() if weight_np is not None: weight = paddle.to_tensor(weight_np) @@ -118,25 +106,22 @@ def test_dygraph_layer(place, return dy_result -def test_dygraph_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static() input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) if weight_np is not None: weight = paddle.to_tensor(weight_np) - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -144,11 +129,19 @@ def test_dygraph_functional(place, def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): if weight_np is None: - expected = -1 * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = -1 * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) else: - expected = -1 * weight_np * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = ( + -1 + * weight_np + * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) + ) if reduction == 'mean': expected = np.mean(expected) @@ -161,7 +154,6 @@ def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): class TestBCELoss(unittest.TestCase): - def test_BCELoss(self): input_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float32) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float32) @@ -169,84 +161,82 @@ class TestBCELoss(unittest.TestCase): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static_layer(place, input_np, label_np, - reduction) - dy_result = test_dygraph_layer(place, input_np, label_np, - reduction) + static_result = test_static_layer( + place, input_np, label_np, reduction + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction + ) expected = calc_bceloss(input_np, label_np, reduction) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) static_functional = test_static_functional( - place, input_np, label_np, reduction) - dy_functional = test_dygraph_functional(place, input_np, - label_np, reduction) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-6) + place, input_np, label_np, reduction + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-6 + ) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCELoss_weight(self): - input_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float32) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float32) + input_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float32 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float32 + ) weight_np = np.random.random(size=(3, 4, 10)).astype(np.float32) place = fluid.MLUPlace(0) for reduction in ['sum', 'mean', 'none']: - static_result = test_static_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_result = test_dygraph_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - expected = calc_bceloss(input_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + expected = calc_bceloss( + input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) - static_functional = test_static_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_functional = test_dygraph_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) + static_functional = test_static_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-6) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCELoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.loss.BCELoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, paddle.nn.loss.BCELoss, reduction="unsupport reduction" + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def bce_loss(input, label): - return -1 * (label * np.log(input) + (1. - label) * np.log(1. - input)) + return -1 * (label * np.log(input) + (1.0 - label) * np.log(1.0 - input)) class TestBceLossOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "bce_loss" @@ -270,13 +260,11 @@ class TestBceLossOp(OpTest): class TestBceLossOpCase1(TestBceLossOp): - def init_test_case(self): self.shape = [2, 3, 4, 5] class TestBceLossOpCase2(TestBceLossOp): - def init_test_case(self): self.shape = [2, 3, 20] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_bce_with_logits_loss_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_bce_with_logits_loss_mlu.py index 6b0b91cce93..0aafe99276f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_bce_with_logits_loss_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_bce_with_logits_loss_mlu.py @@ -20,43 +20,51 @@ import sys sys.path.append('..') from op_test import OpTest -from test_bce_with_logits_loss import call_bce_layer, call_bce_functional, test_dygraph, calc_bce_with_logits_loss - - -def test_static(place, - logit_np, - label_np, - weight_np=None, - reduction='mean', - pos_weight_np=None, - functional=False): +from test_bce_with_logits_loss import ( + call_bce_layer, + call_bce_functional, + test_dygraph, + calc_bce_with_logits_loss, +) + + +def test_static( + place, + logit_np, + label_np, + weight_np=None, + reduction='mean', + pos_weight_np=None, + functional=False, +): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - logit = paddle.fluid.data(name='logit', - shape=logit_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + logit = paddle.fluid.data( + name='logit', shape=logit_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) feed_dict = {"logit": logit_np, "label": label_np} pos_weight = None weight = None if pos_weight_np is not None: - pos_weight = paddle.fluid.data(name='pos_weight', - shape=pos_weight_np.shape, - dtype='float32') + pos_weight = paddle.fluid.data( + name='pos_weight', shape=pos_weight_np.shape, dtype='float32' + ) feed_dict["pos_weight"] = pos_weight_np if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) feed_dict["weight"] = weight_np if functional: - res = call_bce_functional(logit, label, weight, reduction, - pos_weight) + res = call_bce_functional( + logit, label, weight, reduction, pos_weight + ) else: res = call_bce_layer(logit, label, weight, reduction, pos_weight) exe = paddle.static.Executor(place) @@ -68,7 +76,6 @@ paddle.enable_static() class TestBCEWithLogitsLoss(unittest.TestCase): - def test_BCEWithLogitsLoss(self): logit_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float32) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float32) @@ -76,125 +83,150 @@ class TestBCEWithLogitsLoss(unittest.TestCase): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static(place, - logit_np, - label_np, - reduction=reduction) - dy_result = test_dygraph(place, - logit_np, - label_np, - reduction=reduction) - expected = calc_bce_with_logits_loss(logit_np, label_np, - reduction) + static_result = test_static( + place, logit_np, label_np, reduction=reduction + ) + dy_result = test_dygraph( + place, logit_np, label_np, reduction=reduction + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction + ) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) - static_functional = test_static(place, - logit_np, - label_np, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - reduction=reduction, - functional=True) - - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-6) + static_functional = test_static( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) + + np.testing.assert_allclose( + static_functional, expected, rtol=1e-6 + ) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCEWithLogitsLoss_weight(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float32) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float32) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float32 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float32 + ) weight_np = np.random.random(size=(2, 3, 4, 10)).astype(np.float32) place = fluid.MLUPlace(0) for reduction in ['sum', 'mean', 'none']: - static_result = test_static(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction) - dy_result = test_dygraph(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction) - expected = calc_bce_with_logits_loss(logit_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + ) + dy_result = test_dygraph( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) - static_functional = test_static(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-6) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCEWithLogitsLoss_pos_weight(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float32) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float32) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float32 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float32 + ) pos_weight_np = np.random.random(size=(3, 4, 10)).astype(np.float32) weight_np = np.random.random(size=(2, 3, 4, 10)).astype(np.float32) place = fluid.MLUPlace(0) reduction = "mean" - static_result = test_static(place, logit_np, label_np, weight_np, - reduction, pos_weight_np) - dy_result = test_dygraph(place, logit_np, label_np, weight_np, - reduction, pos_weight_np) - expected = calc_bce_with_logits_loss(logit_np, label_np, reduction, - weight_np, pos_weight_np) + static_result = test_static( + place, logit_np, label_np, weight_np, reduction, pos_weight_np + ) + dy_result = test_dygraph( + place, logit_np, label_np, weight_np, reduction, pos_weight_np + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction, weight_np, pos_weight_np + ) np.testing.assert_allclose(static_result, expected) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected) - static_functional = test_static(place, - logit_np, - label_np, - weight_np, - reduction, - pos_weight_np, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - weight_np, - reduction, - pos_weight_np, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + weight_np, + reduction, + pos_weight_np, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + weight_np, + reduction, + pos_weight_np, + functional=True, + ) np.testing.assert_allclose(static_functional, expected) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected) def test_BCEWithLogitsLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.BCEWithLogitsLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.BCEWithLogitsLoss, + reduction="unsupport reduction", + ) logit = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy_with_logits, - logit=logit, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy_with_logits, + logit=logit, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_bilinear_interp_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_bilinear_interp_v2_op_mlu.py index b4ae7cf6da0..912578cda8c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_bilinear_interp_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_bilinear_interp_v2_op_mlu.py @@ -26,16 +26,18 @@ import paddle paddle.enable_static() -def bilinear_interp_np(input, - out_h, - out_w, - scale_w=0, - scale_h=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): +def bilinear_interp_np( + input, + out_h, + out_w, + scale_w=0, + scale_h=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -49,7 +51,7 @@ def bilinear_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -57,7 +59,7 @@ def bilinear_interp_np(input, else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -68,37 +70,40 @@ def bilinear_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for i in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (i + 0.5) - 0.5) else: h = int(ratio_h * i) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (i + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: h1lambda = ratio_h * i - h h2lambda = 1.0 - h1lambda for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, i, j] = h2lambda*(w2lambda*input[:, :, h, w] + - w1lambda*input[:, :, h, w+wid]) + \ - h1lambda*(w2lambda*input[:, :, h+hid, w] + - w1lambda*input[:, :, h+hid, w+wid]) + out[:, :, i, j] = h2lambda * ( + w2lambda * input[:, :, h, w] + + w1lambda * input[:, :, h, w + wid] + ) + h1lambda * ( + w2lambda * input[:, :, h + hid, w] + + w1lambda * input[:, :, h + hid, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -107,7 +112,6 @@ def bilinear_interp_np(input, class TestBilinearInterpOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -129,7 +133,7 @@ class TestBilinearInterpOp(OpTest): scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -142,10 +146,18 @@ class TestBilinearInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -158,12 +170,12 @@ class TestBilinearInterpOp(OpTest): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -181,44 +193,41 @@ class TestBilinearInterpOp(OpTest): self.input_shape = [2, 3, 5, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 @@ -227,46 +236,42 @@ class TestBilinearInterpCase3(TestBilinearInterpOp): class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 33]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase7(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -278,38 +283,35 @@ class TestBilinearInterpCase7(TestBilinearInterpOp): class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpDataLayout(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 5, 5, 3] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -317,52 +319,46 @@ class TestBilinearInterpDataLayout(TestBilinearInterpOp): class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -374,7 +370,6 @@ class TestBilinearInterpScale3(TestBilinearInterpOp): class TestBilinearInterpScale4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -386,7 +381,6 @@ class TestBilinearInterpScale4(TestBilinearInterpOp): class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -398,7 +392,6 @@ class TestBilinearInterpZero(TestBilinearInterpOp): class TestBilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -438,8 +431,9 @@ class TestBilinearInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -451,9 +445,16 @@ class TestBilinearInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -467,33 +468,31 @@ class TestBilinearInterpOp_attr_tensor(OpTest): self.input_shape = [2, 3, 5, 5] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True # out_size is a 1-D tensor class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True @@ -501,7 +500,6 @@ class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -514,23 +512,22 @@ class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): class TestBilinearInterpOpAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) out1 = fluid.layers.resize_bilinear(x, out_shape=[12, 12]) out2 = fluid.layers.resize_bilinear(x, out_shape=[12, dim]) out3 = fluid.layers.resize_bilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_bilinear(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_bilinear( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_bilinear(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -545,29 +542,30 @@ class TestBilinearInterpOpAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = bilinear_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = bilinear_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-6) class TestBilinearInterpOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_mlu(): place = paddle.device.MLUPlace(0) else: @@ -575,21 +573,19 @@ class TestBilinearInterpOpAPI_dy(unittest.TestCase): with fluid.dygraph.guard(place): input_data = np.random.random((2, 3, 6, 6)).astype("float32") input_x = paddle.to_tensor(input_data) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=[12, 12], - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, size=[12, 12], mode="bilinear", align_corners=False + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-6) class TestBilinearInterpOpAPI_dy2(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_mlu(): place = paddle.device.MLUPlace(0) else: @@ -599,21 +595,19 @@ class TestBilinearInterpOpAPI_dy2(unittest.TestCase): size_np = np.array([12, 12]).astype("int64") input_x = paddle.to_tensor(input_data) size = paddle.to_tensor(size_np) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=size, - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, size=size, mode="bilinear", align_corners=False + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-6) class TestBilinearInterpOpAPI_dy3(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_mlu(): place = paddle.device.MLUPlace(0) else: @@ -623,21 +617,22 @@ class TestBilinearInterpOpAPI_dy3(unittest.TestCase): size_1 = np.array([12]).astype("int64") input_x = paddle.to_tensor(input_data) size = paddle.to_tensor(size_1) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=[size, size], - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + size=[size, size], + mode="bilinear", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-6) class TestBilinearInterpOpAPI_dy4(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_mlu(): place = paddle.device.MLUPlace(0) else: @@ -647,14 +642,15 @@ class TestBilinearInterpOpAPI_dy4(unittest.TestCase): scale_np = np.array([2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="bilinear", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-6) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_cast_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_cast_op_mlu.py index 4b75767efe6..a5c5f7b8a2f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_cast_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_cast_op_mlu.py @@ -28,14 +28,13 @@ paddle.enable_static() class TestCastOpFp32ToFp16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float16')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -47,14 +46,13 @@ class TestCastOpFp32ToFp16(OpTest): class TestCastOpFp16ToFp32(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float16')} self.outputs = {'Out': ipt.astype('float32')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP16), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -66,14 +64,13 @@ class TestCastOpFp16ToFp32(OpTest): class TestCastOpFp32ToFp64(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float64')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP64) + 'out_dtype': int(core.VarDesc.VarType.FP64), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -85,14 +82,13 @@ class TestCastOpFp32ToFp64(OpTest): class TestCastOpInt32ToInt32(OpTest): - def setUp(self): ipt = np.random.randint(1000, size=(10, 10)) self.inputs = {'X': ipt.astype('int32')} self.outputs = {'Out': ipt.astype('int32')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.INT32), - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -103,14 +99,13 @@ class TestCastOpInt32ToInt32(OpTest): class TestCastOpInt32ToFp32(OpTest): - def setUp(self): ipt = np.random.randint(1000, size=[10, 10]) self.inputs = {'X': ipt.astype('int32')} self.outputs = {'Out': ipt.astype('float32')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.INT32), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -121,14 +116,13 @@ class TestCastOpInt32ToFp32(OpTest): class TestCastOpInt16ToFp64(OpTest): - def setUp(self): ipt = np.random.randint(1000, size=[10, 10]) self.inputs = {'X': ipt.astype('int16')} self.outputs = {'Out': ipt.astype('int64')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.INT16), - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -139,12 +133,12 @@ class TestCastOpInt16ToFp64(OpTest): class TestCastOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of cast_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.MLUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.MLUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.cast, x1, 'int32') diff --git a/python/paddle/fluid/tests/unittests/mlu/test_clip_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_clip_op_mlu.py index 12c5dbc2cd2..c306ab96b61 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_clip_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_clip_op_mlu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestClipOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -73,7 +72,6 @@ class TestClipOp(OpTest): class TestCase1(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (8, 16, 8) @@ -82,7 +80,6 @@ class TestCase1(TestClipOp): class TestCase2(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (8, 16) @@ -91,7 +88,6 @@ class TestCase2(TestClipOp): class TestCase3(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 16) @@ -100,7 +96,6 @@ class TestCase3(TestClipOp): class TestCase4(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 8) @@ -111,7 +106,6 @@ class TestCase4(TestClipOp): class TestCase5(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 16) @@ -120,7 +114,6 @@ class TestCase5(TestClipOp): class TestCase6(TestClipOp): - def initTestCase(self): self.dtype = np.float16 self.shape = (4, 8, 8) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_coalesce_tensor_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_coalesce_tensor_op_mlu.py index 8eacd633db8..97bc47971f4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_coalesce_tensor_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_coalesce_tensor_op_mlu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestAllocContinuousSpace(OpTest): - def setUp(self): self.op_type = "coalesce_tensor" self.dtype, self.fluid_dtype = self.init_dtype() @@ -36,7 +35,8 @@ class TestAllocContinuousSpace(OpTest): self.set_constant = attrs["set_constant"] self.Inputs = self.init_input() self.Outputs, self.FusedOutput = self.init_output( - self.Inputs, self.set_constant, self.constant) + self.Inputs, self.set_constant, self.constant + ) self.inputs = {'Input': self.Inputs} self.attrs = attrs self.outputs = {'Output': self.Outputs, 'FusedOutput': self.FusedOutput} @@ -59,7 +59,7 @@ class TestAllocContinuousSpace(OpTest): "copy_data": True, "set_constant": False, "constant": 0.0, - "dtype": self.fluid_dtype + "dtype": self.fluid_dtype, } def init_output(self, input_list, set_constant, constant): @@ -76,32 +76,36 @@ class TestAllocContinuousSpace(OpTest): coalesce_tensor_var = np.concatenate([input for input in inputs]) if set_constant: coalesce_tensor_var = np.ones((len(coalesce_tensor_var))) * constant - outputs = [(out[0], - np.ones(out[1].shape).astype(self.dtype) * constant) - for out in outputs] + outputs = [ + (out[0], np.ones(out[1].shape).astype(self.dtype) * constant) + for out in outputs + ] return outputs, coalesce_tensor_var def test_check_output(self): - self.check_output_with_place(place=paddle.device.MLUPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=paddle.device.MLUPlace(0), + no_check_set=["FusedOutput"], + atol=1e-5, + ) class TestAllocContinuousSpace2(TestAllocContinuousSpace): - def init_attr(self): return { "copy_data": False, "set_constant": True, "constant": 5, "dtype": self.fluid_dtype, - "user_defined_size_of_dtype": 2 + "user_defined_size_of_dtype": 2, } def test_check_output(self): - self.check_output_with_place(place=paddle.device.MLUPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=paddle.device.MLUPlace(0), + no_check_set=["FusedOutput"], + atol=1e-5, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather.py index 236eb8f1003..d4c7779807a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather.py @@ -23,32 +23,36 @@ paddle.enable_static() class TestCAllgatherOp(TestDistBase): - def _setup_config(self): pass def test_allgather_fp32(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "float32") + self.check_with_place( + "collective_allgather_op.py", "allgather", "float32" + ) def test_allgather_fp16(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "float16") + self.check_with_place( + "collective_allgather_op.py", "allgather", "float16" + ) def test_allgather_int32(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "int32") + self.check_with_place( + "collective_allgather_op.py", "allgather", "int32" + ) def test_allgather_int16(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "int16") + self.check_with_place( + "collective_allgather_op.py", "allgather", "int16" + ) def test_allgather_int8(self): self.check_with_place("collective_allgather_op.py", "allgather", "int8") def test_allgather_uint8(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "uint8") + self.check_with_place( + "collective_allgather_op.py", "allgather", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather_api_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather_api_mlu.py index 42b8f92850d..dfd29e09a23 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather_api_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather_api_mlu.py @@ -22,21 +22,23 @@ paddle.enable_static() class TestCollectiveAllgatherAPI(TestDistBase): - def _setup_config(self): pass def test_allgather_cncl_fp16(self): - self.check_with_place("collective_allgather_api.py", "allgather", - "float16") + self.check_with_place( + "collective_allgather_api.py", "allgather", "float16" + ) def test_allgather_cncl_fp32(self): - self.check_with_place("collective_allgather_api.py", "allgather", - "float32") + self.check_with_place( + "collective_allgather_api.py", "allgather", "float32" + ) def test_allgather_cncl_int32(self): - self.check_with_place("collective_allgather_api.py", "allgather", - "int32") + self.check_with_place( + "collective_allgather_api.py", "allgather", "int32" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_api_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_api_mlu.py index 74ab7c76045..f7d7d140579 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_api_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_api_mlu.py @@ -22,21 +22,23 @@ paddle.enable_static() class TestCollectiveAllreduceAPI(TestDistBase): - def _setup_config(self): pass def test_allreduce_cncl_fp16(self): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "float16") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "float16" + ) def test_allreduce_cncl_fp32(self): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "float32") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "float32" + ) def test_allreduce_cncl_int32(self): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "int32") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "int32" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_max.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_max.py index ed05fed8db0..fe6cd747106 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_max.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_max.py @@ -23,33 +23,38 @@ paddle.enable_static() class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_max_fp32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "float32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "float32" + ) def test_allreduce_max_fp16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "float16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "float16" + ) def test_allreduce_max_int32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "int32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "int32" + ) def test_allreduce_max_int16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "int16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "int16" + ) def test_allreduce_max_int8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "int8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "int8" + ) def test_allreduce_max_uint8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "uint8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_min.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_min.py index 2b79a7caafe..c3b52a61126 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_min.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_min.py @@ -23,33 +23,38 @@ paddle.enable_static() class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_min_fp32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "float32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "float32" + ) def test_allreduce_min_fp16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "float16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "float16" + ) def test_allreduce_min_int32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "int32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "int32" + ) def test_allreduce_min_int16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "int16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "int16" + ) def test_allreduce_min_int8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "int8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "int8" + ) def test_allreduce_min_uint8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "uint8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_prod.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_prod.py index 96bee38af50..a10275553c8 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_prod.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_prod.py @@ -23,33 +23,38 @@ paddle.enable_static() class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_prod_fp32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "float32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "float32" + ) def test_allreduce_prod_fp16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "float16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "float16" + ) def test_allreduce_prod_int32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "int32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "int32" + ) def test_allreduce_prod_int16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "int16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "int16" + ) def test_allreduce_prod_int8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "int8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "int8" + ) def test_allreduce_prod_uint8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "uint8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_sum.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_sum.py index 302c51beaaf..353e41b094a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_sum.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_sum.py @@ -23,33 +23,38 @@ paddle.enable_static() class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_sum_fp32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "float32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "float32" + ) def test_allreduce_sum_fp16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "float16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "float16" + ) def test_allreduce_sum_int32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "int32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "int32" + ) def test_allreduce_sum_int16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "int16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "int16" + ) def test_allreduce_sum_int8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "int8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "int8" + ) def test_allreduce_sum_uint8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "uint8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_api_base_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_api_base_mlu.py index b405bdda8d7..26e7c2972b3 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_api_base_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_api_base_mlu.py @@ -40,10 +40,10 @@ def DataTypeCast(date_type): class TestCollectiveAPIRunnerBase(object): - def get_model(self, train_prog, startup_prog, rank, indata=None): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -65,12 +65,12 @@ class TestCollectiveAPIRunnerBase(object): fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=fetch_list) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=fetch_list + ) else: out = self.get_model(train_prog, startup_prog, rank, indata) - #print(out, sys.stderr) + # print(out, sys.stderr) sys.stdout.buffer.write(pickle.dumps(out)) @@ -95,19 +95,20 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -120,13 +121,13 @@ class TestDistBase(unittest.TestCase): def _run_cluster(self, model_file, envs): worker_endpoints = self._ps_endpoints.split(",") w0_ep, w1_ep = worker_endpoints - #print("w0_ep:",w0_ep," w1_ep:",w1_ep) + # print("w0_ep:",w0_ep," w1_ep:",w1_ep) env0 = { "FLAGS_selected_mlus": "0", "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -134,9 +135,9 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) if os.getenv('WITH_COVERAGE', 'OFF') == 'ON': @@ -147,16 +148,20 @@ class TestDistBase(unittest.TestCase): tr1_cmd = tr_cmd % (self._python_interp, model_file) tr0_pipe = open("/tmp/tr0_err_%d.log" % os.getpid(), "w") tr1_pipe = open("/tmp/tr1_err_%d.log" % os.getpid(), "w") - #print(tr0_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + # print(tr0_cmd) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -169,17 +174,23 @@ class TestDistBase(unittest.TestCase): sys.stderr.write('trainer 0 stderr file: %s\n' % f.read()) with open("/tmp/tr1_err_%d.log" % os.getpid(), "r") as f: sys.stderr.write('trainer 1 stderr file: %s\n' % f.read()) - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - data_type, - path_id="0", - static_mode="1", - check_error_log=False, - need_envs={}): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, + model_file, + col_type, + data_type, + path_id="0", + static_mode="1", + check_error_log=False, + need_envs={}, + ): required_envs = { "FLAGS_fraction_of_gpu_memory_to_use": "0.15", "FLAGS_eager_delete_tensor_gb": "0.0", @@ -193,7 +204,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_WITH_GLOO": '0', "BACKEND": "cncl", "PATH_ID": path_id, - "DATA_TYPE": data_type + "DATA_TYPE": data_type, } required_envs.update(need_envs) if check_error_log: @@ -201,7 +212,8 @@ class TestDistBase(unittest.TestCase): required_envs["GLOG_logtostderr"] = "1" required_envs["GLOO_LOG_LEVEL"] = "TRACE" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) np_data_type = DataTypeCast(data_type) np.random.seed(pid0) input1 = np.random.random((10, 1000)).astype(np_data_type) @@ -213,14 +225,12 @@ class TestDistBase(unittest.TestCase): np.testing.assert_allclose(tr1_out[0], need_result) elif col_type == "allreduce": need_result = input1 + input2 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "reduce": need_result = input1 + input2 np.testing.assert_allclose(tr0_out[0], need_result) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_base_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_base_mlu.py index c2e6f63f4d5..ab089aa26b7 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_base_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_base_mlu.py @@ -52,10 +52,10 @@ def DataTypeCast(date_type): class TestCollectiveRunnerBase(object): - def get_model(self, train_prog, startup_prog, col_type): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): while True: @@ -63,13 +63,15 @@ class TestCollectiveRunnerBase(object): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -77,44 +79,51 @@ class TestCollectiveRunnerBase(object): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - cncl_id_var = block.create_var(name=nameGen.generate('cncl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - - block.append_op(type='c_gen_cncl_id', - inputs={}, - outputs={'Out': cncl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - - block.append_op(type='c_comm_init', - inputs={'X': cncl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': self.global_ring_id - }) + cncl_id_var = block.create_var( + name=nameGen.generate('cncl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + + block.append_op( + type='c_gen_cncl_id', + inputs={}, + outputs={'Out': cncl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + + block.append_op( + type='c_comm_init', + inputs={'X': cncl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': self.global_ring_id, + }, + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -123,8 +132,9 @@ class TestCollectiveRunnerBase(object): rank = args["trainerid"] current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) self.rank = rank result = self.get_model(train_prog, startup_prog, args["col_type"]) device_id = int(os.getenv("FLAGS_selected_mlus", "0")) @@ -134,9 +144,9 @@ class TestCollectiveRunnerBase(object): np.random.seed(os.getpid()) np_data_type = DataTypeCast(args["data_type"]) indata = np.random.random((10, 1000)).astype(np_data_type) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=[result.name]) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=[result.name] + ) sys.stdout.buffer.write(pickle.dumps(out)) @@ -159,19 +169,20 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -190,7 +201,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -198,9 +209,9 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) tr_cmd = "%s %s" @@ -209,15 +220,19 @@ class TestDistBase(unittest.TestCase): tr0_pipe = open("/tmp/tr0_err.log", "wb") tr1_pipe = open("/tmp/tr1_err.log", "wb") - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -226,15 +241,21 @@ class TestDistBase(unittest.TestCase): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - data_type, - check_error_log=False, - need_envs={}): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, + model_file, + col_type, + data_type, + check_error_log=False, + need_envs={}, + ): required_envs = { "FLAGS_eager_delete_tensor_gb": "0.0", "PATH": os.getenv("PATH"), @@ -250,7 +271,8 @@ class TestDistBase(unittest.TestCase): required_envs["GLOG_v"] = "3" required_envs["GLOG_logtostderr"] = "1" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) np_data_type = DataTypeCast(data_type) np.random.seed(pid0) input1 = np.random.random((10, 1000)).astype(np_data_type) @@ -262,44 +284,36 @@ class TestDistBase(unittest.TestCase): np.testing.assert_allclose(tr1_out[0], need_result) elif col_type == "allreduce_sum": need_result = input1 + input2 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "allreduce_prod": need_result = input1 * input2 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "allreduce_max": need_result = np.maximum(input1, input2) - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "allreduce_min": need_result = np.minimum(input1, input2) - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "reduce_sum": need_result = input1 + input2 np.testing.assert_allclose(tr1_out[0], need_result) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast.py index 723a14edb93..d06a599243e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast.py @@ -23,32 +23,36 @@ paddle.enable_static() class TestCBroadcastOp(TestDistBase): - def _setup_config(self): pass def test_broadcast_fp32(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "float32") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "float32" + ) def test_broadcast_fp16(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "float16") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "float16" + ) def test_broadcast_int32(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "int32") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "int32" + ) def test_broadcast_int16(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "int16") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "int16" + ) def test_broadcast_int8(self): self.check_with_place("collective_broadcast_op.py", "broadcast", "int8") def test_broadcast_uint8(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "uint8") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast_api_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast_api_mlu.py index 0619c550714..d40da0f1db6 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast_api_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast_api_mlu.py @@ -22,21 +22,23 @@ paddle.enable_static() class TestCollectiveBroadcastAPI(TestDistBase): - def _setup_config(self): pass def test_broadcast_cncl_fp16(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "float16") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "float16" + ) def test_broadcast_cncl_fp32(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "float32") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "float32" + ) def test_broadcast_cncl_int32(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "int32") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "int32" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_api_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_api_mlu.py index d76429196cc..69c1ebaf43c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_api_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_api_mlu.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveReduceAPI(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_max.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_max.py index fd84b1c1eab..ac07b352e0c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_max.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_max.py @@ -23,17 +23,18 @@ paddle.enable_static() class TestCReduceOp(TestDistBase): - def _setup_config(self): pass def test_reduce_max_fp32(self): - self.check_with_place("collective_reduce_op.py", "reduce_max", - "float32") + self.check_with_place( + "collective_reduce_op.py", "reduce_max", "float32" + ) def test_reduce_max_fp16(self): - self.check_with_place("collective_reduce_op.py", "reduce_max", - "float16") + self.check_with_place( + "collective_reduce_op.py", "reduce_max", "float16" + ) def test_reduce_max_int32(self): self.check_with_place("collective_reduce_op.py", "reduce_max", "int32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_min.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_min.py index 6976e047e91..abe970bafb5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_min.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_min.py @@ -23,17 +23,18 @@ paddle.enable_static() class TestCReduceOp(TestDistBase): - def _setup_config(self): pass def test_reduce_min_fp32(self): - self.check_with_place("collective_reduce_op.py", "reduce_min", - "float32") + self.check_with_place( + "collective_reduce_op.py", "reduce_min", "float32" + ) def test_reduce_min_fp16(self): - self.check_with_place("collective_reduce_op.py", "reduce_min", - "float16") + self.check_with_place( + "collective_reduce_op.py", "reduce_min", "float16" + ) def test_reduce_min_int32(self): self.check_with_place("collective_reduce_op.py", "reduce_min", "int32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_prod.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_prod.py index a5ed2ff8b1e..6fc9bd331ca 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_prod.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_prod.py @@ -23,17 +23,18 @@ paddle.enable_static() class TestCReduceOp(TestDistBase): - def _setup_config(self): pass def test_reduce_prod_fp32(self): - self.check_with_place("collective_reduce_op.py", "reduce_prod", - "float32") + self.check_with_place( + "collective_reduce_op.py", "reduce_prod", "float32" + ) def test_reduce_prod_fp16(self): - self.check_with_place("collective_reduce_op.py", "reduce_prod", - "float16") + self.check_with_place( + "collective_reduce_op.py", "reduce_prod", "float16" + ) def test_reduce_prod_int32(self): self.check_with_place("collective_reduce_op.py", "reduce_prod", "int32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_sum.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_sum.py index a86749de5fb..141a424a978 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_sum.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_sum.py @@ -23,17 +23,18 @@ paddle.enable_static() class TestCReduceOp(TestDistBase): - def _setup_config(self): pass def test_reduce_sum_fp32(self): - self.check_with_place("collective_reduce_op.py", "reduce_sum", - "float32") + self.check_with_place( + "collective_reduce_op.py", "reduce_sum", "float32" + ) def test_reduce_sum_fp16(self): - self.check_with_place("collective_reduce_op.py", "reduce_sum", - "float16") + self.check_with_place( + "collective_reduce_op.py", "reduce_sum", "float16" + ) def test_reduce_sum_int32(self): self.check_with_place("collective_reduce_op.py", "reduce_sum", "int32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_compare_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_compare_op_mlu.py index f314a6759d9..f994fb185f6 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_compare_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_compare_op_mlu.py @@ -24,9 +24,7 @@ from paddle.fluid import Program, program_guard def create_test_class(op_type, typename, callback): - class Cls(OpTest): - def setUp(self): self.set_mlu() self.place = paddle.MLUPlace(0) @@ -77,46 +75,42 @@ def create_test_class(op_type, typename, callback): def test_broadcast_api_1(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 1, 3], - dtype=typename) - y = paddle.static.data(name='y', - shape=[1, 2, 3], - dtype=typename) + x = paddle.static.data( + name='x', shape=[1, 2, 1, 3], dtype=typename + ) + y = paddle.static.data( + name='y', shape=[1, 2, 3], dtype=typename + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(typename) input_y = np.arange(0, 6).reshape((1, 2, 3)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") def test_broadcast_api_2(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 3], - dtype=typename) - y = paddle.static.data(name='y', - shape=[1, 2, 1, 3], - dtype=typename) + x = paddle.static.data( + name='x', shape=[1, 2, 3], dtype=typename + ) + y = paddle.static.data( + name='y', shape=[1, 2, 1, 3], dtype=typename + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(0, 6).reshape((1, 2, 3)).astype(typename) input_y = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") @@ -131,11 +125,9 @@ def create_test_class(op_type, typename, callback): input_x = np.arange(0, 5).reshape((5)).astype(typename) input_y = np.array([5, 3, 2]).reshape((3, 1)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_concat_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_concat_op_mlu.py index 91bed72ff28..d0caa993c33 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_concat_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_concat_op_mlu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestConcatOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "concat" @@ -43,8 +42,9 @@ class TestConcatOp(OpTest): self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } def set_mlu(self): @@ -69,7 +69,6 @@ class TestConcatOp(OpTest): class TestConcatOp2(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -78,9 +77,9 @@ class TestConcatOp2(TestConcatOp): @skip_check_grad_ci( - reason="The function 'check_grad' for large inputs is too slow.") + reason="The function 'check_grad' for large inputs is too slow." +) class TestConcatOp3(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((1, 256, 170, 256)).astype(self.dtype) self.x1 = np.random.random((1, 128, 170, 256)).astype(self.dtype) @@ -92,11 +91,9 @@ class TestConcatOp3(TestConcatOp): @skip_check_grad_ci( - reason= - "This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." + reason="This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." ) class TestConcatOp4(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -108,7 +105,6 @@ class TestConcatOp4(TestConcatOp): class TestConcatOp5(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((5, 1, 4, 5)).astype(self.dtype) self.x1 = np.random.random((5, 2, 4, 5)).astype(self.dtype) @@ -116,11 +112,9 @@ class TestConcatOp5(TestConcatOp): self.axis = -3 -#----------------Concat Fp16---------------- +# ----------------Concat Fp16---------------- def create_test_fp16(parent): - class TestConcatFp16(parent): - def init_dtype(self): self.dtype = np.float16 @@ -136,11 +130,9 @@ create_test_fp16(TestConcatOp4) create_test_fp16(TestConcatOp5) -#----------------Concat Int64---------------- +# ----------------Concat Int64---------------- def create_test_int64(parent): - class TestConcatInt64(parent): - def init_dtype(self): self.dtype = np.int64 @@ -159,11 +151,9 @@ create_test_int64(TestConcatOp4) create_test_int64(TestConcatOp5) -#----------------Concat Int32---------------- +# ----------------Concat Int32---------------- def create_test_int32(parent): - class TestConcatInt32(parent): - def init_dtype(self): self.dtype = np.int32 @@ -182,11 +172,9 @@ create_test_int32(TestConcatOp4) create_test_int32(TestConcatOp5) -#----------------Concat AxisTensor---------------- +# ----------------Concat AxisTensor---------------- def create_test_AxisTensor(parent): - class TestConcatAxisTensor(parent): - def setUp(self): self.op_type = "concat" self.init_dtype() @@ -194,20 +182,22 @@ def create_test_AxisTensor(parent): self.inputs = { 'X': [('x0', self.x0), ('x1', self.x1), ('x2', self.x2)], - 'AxisTensor': np.array([self.axis]).astype("int32") + 'AxisTensor': np.array([self.axis]).astype("int32"), } self.attrs = {} if self.axis < 0: self.actual_axis = self.axis + len(self.x0.shape) - self.actual_axis = self.actual_axis if self.actual_axis > 0 else 0 + self.actual_axis = ( + self.actual_axis if self.actual_axis > 0 else 0 + ) else: self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), - axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } self.place = paddle.device.MLUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_depthwise_conv_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_depthwise_conv_mlu.py index 8fac2941e43..593bbc7096e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_depthwise_conv_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_depthwise_conv_mlu.py @@ -25,13 +25,19 @@ import paddle.fluid.core as core import paddle.fluid as fluid from op_test import OpTest from paddle.fluid import Program, program_guard -from test_conv2d_op_mlu import TestConv2DOp, TestConv2DOp_v2, create_test_padding_SAME_class, create_test_padding_VALID_class, create_test_channel_last_class, create_test_fp16_class +from test_conv2d_op_mlu import ( + TestConv2DOp, + TestConv2DOp_v2, + create_test_padding_SAME_class, + create_test_padding_VALID_class, + create_test_channel_last_class, + create_test_fp16_class, +) -#----------------TestDepthwiseConv ----- +# ----------------TestDepthwiseConv ----- class TestDepthwiseConv(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -44,7 +50,6 @@ class TestDepthwiseConv(TestConv2DOp): class TestDepthwiseConv2(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -57,7 +62,6 @@ class TestDepthwiseConv2(TestConv2DOp): class TestDepthwiseConv3(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -70,7 +74,6 @@ class TestDepthwiseConv3(TestConv2DOp): class TestDepthwiseConvandFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -84,7 +87,6 @@ class TestDepthwiseConvandFuse(TestConv2DOp): class TestDepthwiseConv2andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -98,7 +100,6 @@ class TestDepthwiseConv2andFuse(TestConv2DOp): class TestDepthwiseConv3andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -112,7 +113,6 @@ class TestDepthwiseConv3andFuse(TestConv2DOp): class TestDepthwiseConv_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 5, 5] # NCHW @@ -128,7 +128,6 @@ class TestDepthwiseConv_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConv2_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -144,7 +143,6 @@ class TestDepthwiseConv2_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConv3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -160,7 +158,6 @@ class TestDepthwiseConv3_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConvandFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -178,7 +175,6 @@ class TestDepthwiseConvandFuse_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConv2andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -196,7 +192,6 @@ class TestDepthwiseConv2andFuse_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConv3andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_mlu.py index 7f79caa5499..7153ea09ef2 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_mlu.py @@ -28,9 +28,7 @@ paddle.enable_static() def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -44,9 +42,7 @@ def create_test_channel_last_class(parent): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -57,9 +53,7 @@ def create_test_padding_SAME_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -70,9 +64,7 @@ def create_test_padding_VALID_class(parent): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_dtype(self): self.dtype = np.float16 @@ -82,7 +74,6 @@ def create_test_fp16_class(parent): class TestConv2DOp(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -105,22 +96,24 @@ class TestConv2DOp(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, - filter, - self.groups, - conv2d_param, - data_format=self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + data_format=self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -137,28 +130,37 @@ class TestConv2DOp(OpTest): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -176,7 +178,6 @@ class TestConv2DOp(OpTest): class TestWithPad(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -187,7 +188,6 @@ class TestWithPad(TestConv2DOp): class TestWithStride(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -198,7 +198,6 @@ class TestWithStride(TestConv2DOp): class TestWithGroup(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -210,7 +209,6 @@ class TestWithGroup(TestConv2DOp): class TestWith1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -227,7 +225,6 @@ class TestWith1x1(TestConv2DOp): class TestWithDepthWise5x5(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -241,7 +238,6 @@ class TestWithDepthWise5x5(TestConv2DOp): class TestWithDepthWise7x7(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -255,7 +251,6 @@ class TestWithDepthWise7x7(TestConv2DOp): class TestWithDilation(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -274,7 +269,6 @@ class TestWithDilation(TestConv2DOp): class TestWithInput1x1Filter1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -288,7 +282,6 @@ class TestWithInput1x1Filter1x1(TestConv2DOp): class TestConv2DOp_v2(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -308,20 +301,24 @@ class TestConv2DOp_v2(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, filter, self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -339,27 +336,36 @@ class TestConv2DOp_v2(OpTest): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.02, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.02, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -390,14 +396,12 @@ class TestConv2DOp_v2(OpTest): class TestConv2DOp_AsyPadding(TestConv2DOp_v2): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -411,7 +415,6 @@ class TestWithPad_AsyPadding(TestConv2DOp_v2): class TestWithStride_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -425,7 +428,6 @@ class TestWithStride_AsyPadding(TestConv2DOp_v2): class TestWithGroup_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 2] @@ -437,7 +439,6 @@ class TestWithGroup_AsyPadding(TestConv2DOp_v2): class TestWith1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -454,7 +455,6 @@ class TestWith1x1_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [3, 4, 10, 10] # NCHW @@ -476,7 +476,6 @@ class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 4, 10, 10] # NCHW @@ -493,7 +492,6 @@ class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 8, 10, 10] # NCHW @@ -510,7 +508,6 @@ class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): class TestWithDilation_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -532,7 +529,6 @@ class TestWithDilation_AsyPadding(TestConv2DOp_v2): class TestWithInput1x1Filter1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [100, 1, 1, 1] # NCHW diff --git a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_transposed_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_transposed_op_mlu.py index 0d33bf85369..076c6e2ca3f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_transposed_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_transposed_op_mlu.py @@ -27,9 +27,10 @@ from paddle.fluid.tests.unittests.op_test import OpTest def conv2dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 3, 1, 2]) @@ -40,18 +41,22 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -85,37 +90,49 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): if 'output_padding' in attrs: out_pad_h = attrs['output_padding'][0] out_pad_w = attrs['output_padding'][1] - out = np.zeros((in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), - dtype=input_.dtype) + out = np.zeros( + (in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), dtype=input_.dtype + ) for n in range(in_n): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, g * sub_in_c:(g + 1) * sub_in_c, i, - j] # (c) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, i, j + ] # (c) input_masked = np.reshape(input_masked, (sub_in_c, 1, 1)) input_masked = np.tile(input_masked, (1, f_h, f_w)) for k in range(f_out_c): tmp_out = np.sum( - input_masked * - filter_[g * sub_in_c:(g + 1) * sub_in_c, k, :, :], - axis=0) + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, k, :, : + ], + axis=0, + ) i1, i2 = i * stride[0], i * stride[0] + d_bolck_h j1, j2 = j * stride[1], j * stride[1] + d_bolck_w - out[n, g * f_out_c + k, i1:i2:dilations[0], - j1:j2:dilations[1]] += tmp_out - - out = out[:, :, pad_h_0:out_h - pad_h_1 + out_pad_h, - pad_w_0:out_w - pad_w_1 + out_pad_w] + out[ + n, + g * f_out_c + k, + i1 : i2 : dilations[0], + j1 : j2 : dilations[1], + ] += tmp_out + + out = out[ + :, + :, + pad_h_0 : out_h - pad_h_1 + out_pad_h, + pad_w_0 : out_w - pad_w_1 + out_pad_w, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 1]) return out class TestConv2DTransposeOp(OpTest): - def setUp(self): # init as conv transpose self.dtype = np.float32 @@ -145,7 +162,7 @@ class TestConv2DTransposeOp(OpTest): 'use_cudnn': self.use_cudnn, 'is_test': self.is_test, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } if self.output_size is not None: self.attrs['output_size'] = self.output_size @@ -153,8 +170,9 @@ class TestConv2DTransposeOp(OpTest): if len(self.output_padding) > 0: self.attrs['output_padding'] = self.output_padding - output = conv2dtranspose_forward_naive(input_, filter_, - self.attrs).astype(self.dtype) + output = conv2dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype(self.dtype) self.outputs = {'Output': output} @@ -167,23 +185,28 @@ class TestConv2DTransposeOp(OpTest): def test_check_grad_no_input(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Input']), + ) def test_check_grad_no_filter(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad(self): if self.need_check_grad: - self.check_grad_with_place(self.place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02) + self.check_grad_with_place( + self.place, + set(['Input', 'Filter']), + 'Output', + max_relative_error=0.02, + ) def init_test_case(self): self.pad = [0, 0] @@ -199,7 +222,6 @@ class TestConv2DTransposeOp(OpTest): class TestWithSymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -211,7 +233,6 @@ class TestWithSymmetricPad(TestConv2DTransposeOp): class TestWithAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -223,7 +244,6 @@ class TestWithAsymmetricPad(TestConv2DTransposeOp): class TestWithSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [2, 1] self.dilations = [1, 2] @@ -235,7 +255,6 @@ class TestWithSAMEPad(TestConv2DTransposeOp): class TestWithVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -247,7 +266,6 @@ class TestWithVALIDPad(TestConv2DTransposeOp): class TestWithGroups(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -259,7 +277,6 @@ class TestWithGroups(TestConv2DTransposeOp): class TestWithStride(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -271,7 +288,6 @@ class TestWithStride(TestConv2DTransposeOp): class TestWithDilation(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -283,7 +299,6 @@ class TestWithDilation(TestConv2DTransposeOp): class TestWithEvenUpsample(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -296,7 +311,6 @@ class TestWithEvenUpsample(TestConv2DTransposeOp): class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -309,7 +323,6 @@ class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): class Test_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -322,7 +335,6 @@ class Test_NHWC(TestConv2DTransposeOp): class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -335,7 +347,6 @@ class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -348,7 +359,6 @@ class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): class TestWithGroups_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -361,7 +371,6 @@ class TestWithGroups_NHWC(TestConv2DTransposeOp): class TestWithStride_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -374,7 +383,6 @@ class TestWithStride_NHWC(TestConv2DTransposeOp): class TestWithDilation_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -387,7 +395,6 @@ class TestWithDilation_NHWC(TestConv2DTransposeOp): class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -401,7 +408,6 @@ class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -415,7 +421,6 @@ class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): class TestMLU_FP16(TestConv2DTransposeOp): - def init_test_case(self): self.dtype = np.float16 self.set_mlu() @@ -440,7 +445,6 @@ class TestMLU_FP16(TestConv2DTransposeOp): class TestMLU_NHWC_FP16(TestMLU_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [0, 0] @@ -454,7 +458,6 @@ class TestMLU_NHWC_FP16(TestMLU_FP16): class TestMLUWithGroups_NHWC_FP16(TestMLU_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -468,7 +471,6 @@ class TestMLUWithGroups_NHWC_FP16(TestMLU_FP16): class TestMLUWithEvenUpsample_NHWC_FP16(TestMLU_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [2, 2] @@ -483,7 +485,6 @@ class TestMLUWithEvenUpsample_NHWC_FP16(TestMLU_FP16): class TestConv2DTransposeAPI(unittest.TestCase): - def setUp(self): self.set_mlu() @@ -492,67 +493,78 @@ class TestConv2DTransposeAPI(unittest.TestCase): self.place = paddle.device.MLUPlace(0) def test_case1(self): - data1 = fluid.layers.data(name='data1', - shape=[3, 5, 5], - dtype='float32') - data2 = fluid.layers.data(name='data2', - shape=[5, 5, 3], - dtype='float32') - out1 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - data_format='NCHW') - out2 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - data_format='NHWC') - out3 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - data_format='NHWC') - out4 = fluid.layers.conv2d_transpose(input=data1, - groups=3, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [2, 1], - [0, 0]], - data_format='NCHW') - out5 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='SAME', - data_format='NCHW') - out6 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding='VALID', - data_format='NHWC') - out7 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - output_size=[7, 7], - padding=[0, 0], - data_format='NHWC') + data1 = fluid.layers.data( + name='data1', shape=[3, 5, 5], dtype='float32' + ) + data2 = fluid.layers.data( + name='data2', shape=[5, 5, 3], dtype='float32' + ) + out1 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + data_format='NCHW', + ) + out2 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) + out3 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + data_format='NHWC', + ) + out4 = fluid.layers.conv2d_transpose( + input=data1, + groups=3, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [2, 1], [0, 0]], + data_format='NCHW', + ) + out5 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='SAME', + data_format='NCHW', + ) + out6 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding='VALID', + data_format='NHWC', + ) + out7 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + output_size=[7, 7], + padding=[0, 0], + data_format='NHWC', + ) data1_np = np.random.random((2, 3, 5, 5)).astype("float32") data2_np = np.random.random((2, 5, 5, 3)).astype("float32") exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2, out3, out4, out5, out6, out7], + return_numpy=True, + ) self.assertIsNotNone(results[0]) self.assertIsNotNone(results[1]) self.assertIsNotNone(results[2]) @@ -563,7 +575,6 @@ class TestConv2DTransposeAPI(unittest.TestCase): class TestConv2DTransposeOpException(unittest.TestCase): - def setUp(self): self.set_mlu() @@ -575,68 +586,74 @@ class TestConv2DTransposeOpException(unittest.TestCase): data = fluid.layers.data(name='data', shape=[3, 5, 5], dtype="float32") def attr_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - data_format="NCDHW") + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + data_format="NCDHW", + ) self.assertRaises(ValueError, attr_data_format) def attr_padding_str(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding='Vald') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding='Vald', + ) self.assertRaises(ValueError, attr_padding_str) def attr_padding_list(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [1, 1], [0, 0], - [0, 0]]) + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [1, 1], [0, 0], [0, 0]], + ) self.assertRaises(ValueError, attr_padding_list) def attr_padding_with_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [0, 0], [0, 0], - [1, 1]], - data_format='NHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [0, 0], [0, 0], [1, 1]], + data_format='NHWC', + ) self.assertRaises(ValueError, attr_padding_with_data_format) - error_input = fluid.layers.data(name='error_data', - shape=[1], - dtype="float32") + error_input = fluid.layers.data( + name='error_data', shape=[1], dtype="float32" + ) def error_input_size(): - out = fluid.layers.conv2d_transpose(input=error_input, - groups=1, - num_filters=6, - filter_size=3) + out = fluid.layers.conv2d_transpose( + input=error_input, groups=1, num_filters=6, filter_size=3 + ) self.assertRaises(ValueError, error_input_size) def error_groups(): - out = fluid.layers.conv2d_transpose(input=data, - groups=0, - num_filters=6, - filter_size=3, - data_format='NHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=0, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) self.assertRaises(ValueError, error_groups) class TestConv2DTransposeRepr(unittest.TestCase): - def setUp(self): self.set_mlu() @@ -646,7 +663,7 @@ class TestConv2DTransposeRepr(unittest.TestCase): def test_case(self): paddle.disable_static() - x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = nn.Conv2DTranspose(4, 6, (3, 3), output_padding=1, stride=2) print(conv) y_var = conv(x_var) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_cumsum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_cumsum_op_mlu.py index 77847a8f64d..ca1b99ec79c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_cumsum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_cumsum_op_mlu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestMLUCumSumOp(OpTest): - def setUp(self): self.op_type = "cumsum" self.set_mlu() @@ -48,18 +47,17 @@ class TestMLUCumSumOp(OpTest): class TestMLUCumSumOp2(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': -1, 'reverse': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.outputs = { - 'Out': np.flip(np.flip(self.inputs['X'], axis=2).cumsum(axis=2), - axis=2) + 'Out': np.flip( + np.flip(self.inputs['X'], axis=2).cumsum(axis=2), axis=2 + ) } class TestMLUCumSumOp3(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 1} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} @@ -67,7 +65,6 @@ class TestMLUCumSumOp3(TestMLUCumSumOp): class TestMLUCumSumOp4(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 0} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} @@ -75,107 +72,115 @@ class TestMLUCumSumOp4(TestMLUCumSumOp): class TestMLUCumSumOp5(TestMLUCumSumOp): - def init_testcase(self): self.inputs = {'X': np.random.random((5, 20)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum(axis=1)} class TestMLUCumSumOp7(TestMLUCumSumOp): - def init_testcase(self): self.inputs = {'X': np.random.random((100)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum(axis=0)} class TestNPUCumSumExclusive1(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 65)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive2(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 888)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive3(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 888)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive4(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 3049)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive5(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 3096)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumReverseExclusive(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, 'reverse': True, "exclusive": True} a = np.random.random((4, 5, 6)).astype(self.dtype) self.inputs = {'X': a} a = np.flip(a, axis=2) self.outputs = { - 'Out': - np.concatenate( - (np.flip(a[:, :, :-1].cumsum(axis=2), - axis=2), np.zeros((4, 5, 1), dtype=self.dtype)), - axis=2) + 'Out': np.concatenate( + ( + np.flip(a[:, :, :-1].cumsum(axis=2), axis=2), + np.zeros((4, 5, 1), dtype=self.dtype), + ), + axis=2, + ) } class TestNPUCumSumWithFlatten1(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'flatten': True} self.inputs = {'X': np.random.random((5, 6)).astype(self.dtype)} @@ -183,7 +188,6 @@ class TestNPUCumSumWithFlatten1(TestMLUCumSumOp): class TestNPUCumSumWithFlatten2(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'flatten': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} diff --git a/python/paddle/fluid/tests/unittests/mlu/test_deformable_conv_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_deformable_conv_op_mlu.py index 352d9641827..5332299c44b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_deformable_conv_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_deformable_conv_op_mlu.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestModulatedDeformableConvOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -42,7 +41,7 @@ class TestModulatedDeformableConvOp(OpTest): conv_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -50,15 +49,16 @@ class TestModulatedDeformableConvOp(OpTest): mask = 10 * np.random.random(self.mask_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = dconv_im2col_gemm(input, offset, mask, filter, self.groups, - conv_param) + output = dconv_im2col_gemm( + input, offset, mask, filter, self.groups, conv_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Offset': OpTest.np_dtype_to_fluid_dtype(offset), 'Mask': OpTest.np_dtype_to_fluid_dtype(mask), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -74,11 +74,13 @@ class TestModulatedDeformableConvOp(OpTest): self.check_output_with_place(self.place, check_eager=False) def test_check_grad(self): - self.check_grad_with_place(self.place, - {'Input', 'Offset', 'Mask', 'Filter'}, - 'Output', - max_relative_error=0.05, - check_eager=False) + self.check_grad_with_place( + self.place, + {'Input', 'Offset', 'Mask', 'Filter'}, + 'Output', + max_relative_error=0.05, + check_eager=False, + ) def init_test_case(self): self.pad = [1, 1] @@ -90,15 +92,26 @@ class TestModulatedDeformableConvOp(OpTest): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -112,7 +125,6 @@ class TestModulatedDeformableConvOp(OpTest): class TestWithStride(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [3, 3] self.stride = [2, 2] @@ -122,20 +134,30 @@ class TestWithStride(TestModulatedDeformableConvOp): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestWithDilation(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [1, 1] @@ -145,15 +167,26 @@ class TestWithDilation(TestModulatedDeformableConvOp): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -161,7 +194,6 @@ class TestWithDilation(TestModulatedDeformableConvOp): class TestWith3x3(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -171,15 +203,26 @@ class TestWith3x3(TestModulatedDeformableConvOp): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py index 0bbec2b7ae0..9c115c3bd38 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py @@ -27,7 +27,6 @@ SEED = 2022 class TestDropoutOp(OpTest): - def setUp(self): self.set_mlu() self.init_dtype() @@ -39,7 +38,7 @@ class TestDropoutOp(OpTest): 'dropout_prob': self.dropout_prob, 'fix_seed': self.fix_seed, 'is_test': self.is_test, - 'dropout_implementation': self.dropout_implementation + 'dropout_implementation': self.dropout_implementation, } out = self.inputs['X'] * (1.0 - self.dropout_prob) @@ -74,21 +73,21 @@ class TestDropoutOp(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return self.check_grad_with_place(self.place, ['X'], 'Out') class TestDropoutOpInput1d(TestDropoutOp): - def init_inputs_shape(self): self.shape = [2000] class TestDropoutOp2(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64] @@ -100,13 +99,11 @@ class TestDropoutOp2(TestDropoutOp): class TestDropoutOp3(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64, 2] class TestDropoutOp4(TestDropoutOp): - def init_attrs(self): self.__class__.no_need_check_grad = True self.dropout_prob = 0.35 @@ -116,7 +113,6 @@ class TestDropoutOp4(TestDropoutOp): class TestDropoutOp5(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64, 3] @@ -129,7 +125,6 @@ class TestDropoutOp5(TestDropoutOp): class TestDropoutOp6(TestDropoutOp): - def init_attrs(self): self.__class__.no_need_check_grad = True self.dropout_prob = 0.0 @@ -146,16 +141,16 @@ class TestDropoutOpWithSeed(TestDropoutOp): self.dtype = np.float32 self.inputs = { "X": np.random.random((32, 64)).astype(self.dtype), - "Seed": np.asarray([125], dtype="int32") + "Seed": np.asarray([125], dtype="int32"), } self.attrs = { 'dropout_prob': 0.0, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } def set_mlu(self): @@ -192,7 +187,7 @@ class TestDropoutOpInference(OpTest): 'dropout_prob': 0.35, 'fix_seed': True, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -209,7 +204,6 @@ class TestDropoutOpInference(OpTest): @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOpInference2(TestDropoutOpInference): - def setUp(self): self.op_type = "dropout" self.set_mlu() @@ -218,13 +212,12 @@ class TestDropoutOpInference2(TestDropoutOpInference): self.attrs = { 'dropout_prob': 0.75, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} class TestDropoutAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace(), paddle.device.MLUPlace(0)] @@ -232,43 +225,44 @@ class TestDropoutAPI(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[40, 40], dtype="float32") - res1 = paddle.nn.functional.dropout(x=input, - p=0., - training=False, - mode='upscale_in_train') - res2 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='upscale_in_train') - res3 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='upscale_in_train') - res4 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='upscale_in_train') - res5 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='upscale_in_train') - res6 = paddle.nn.functional.dropout(x=input, - p=1., - training=True, - mode='upscale_in_train') + res1 = paddle.nn.functional.dropout( + x=input, p=0.0, training=False, mode='upscale_in_train' + ) + res2 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=True, mode='upscale_in_train' + ) + res3 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=False, mode='upscale_in_train' + ) + res4 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='upscale_in_train', + ) + res5 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='upscale_in_train', + ) + res6 = paddle.nn.functional.dropout( + x=input, p=1.0, training=True, mode='upscale_in_train' + ) res7 = paddle.fluid.layers.dropout( x=input, - dropout_prob=0., - dropout_implementation='upscale_in_train') - res8 = paddle.nn.functional.dropout(x=input, - p=0., - axis=(0, 1), - training=False, - mode='upscale_in_train') + dropout_prob=0.0, + dropout_implementation='upscale_in_train', + ) + res8 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=(0, 1), + training=False, + mode='upscale_in_train', + ) in_np = np.random.random([40, 40]).astype("float32") res_np = in_np @@ -277,13 +271,17 @@ class TestDropoutAPI(unittest.TestCase): exe = fluid.Executor(place) res_list = [res1, res2, res3, res4, res5, res7, res8] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np) - fetches2 = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res6]) + fetches2 = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res6], + ) np.testing.assert_allclose(fetches2[0], res_np2) def test_static(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_add_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_add_op_mlu.py index b76969332dd..f5e79bb64ec 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_add_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_add_op_mlu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestElementwiseAddOp(OpTest): - def set_mlu(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -41,7 +40,7 @@ class TestElementwiseAddOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -52,25 +51,31 @@ class TestElementwiseAddOp(OpTest): def test_check_grad_normal(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.01 + ) def test_check_grad_ingore_x(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - max_relative_error=0.01) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + max_relative_error=0.01, + ) def test_check_grad_ingore_y(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('Y'), - max_relative_error=0.01) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set('Y'), + max_relative_error=0.01, + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -85,7 +90,6 @@ class TestElementwiseAddOp(OpTest): class TestFP16ElementwiseAddOp(TestElementwiseAddOp): - def init_dtype(self): self.dtype = np.float16 @@ -94,9 +98,9 @@ class TestFP16ElementwiseAddOp(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -104,9 +108,9 @@ class TestElementwiseAddOp_scalar(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -114,9 +118,9 @@ class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -124,9 +128,9 @@ class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -134,23 +138,20 @@ class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestFP16ElementwiseAddOp_Vector(TestFP16ElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -161,7 +162,6 @@ class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -172,7 +172,6 @@ class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -183,7 +182,6 @@ class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -194,7 +192,6 @@ class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -202,7 +199,6 @@ class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -210,7 +206,6 @@ class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 1).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -221,7 +216,6 @@ class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -232,7 +226,6 @@ class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -243,7 +236,6 @@ class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -254,7 +246,6 @@ class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -262,7 +253,6 @@ class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -270,7 +260,6 @@ class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -278,7 +267,6 @@ class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) @@ -286,7 +274,6 @@ class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -294,7 +281,6 @@ class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -305,7 +291,6 @@ class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -316,9 +301,9 @@ class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -329,9 +314,9 @@ class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -342,7 +327,6 @@ class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -353,7 +337,6 @@ class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -364,7 +347,6 @@ class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -375,7 +357,6 @@ class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -386,7 +367,6 @@ class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -397,7 +377,6 @@ class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -408,7 +387,6 @@ class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 1, 12).astype(self.dtype) self.y = np.random.rand(10, 2, 12).astype(self.dtype) @@ -419,14 +397,15 @@ class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.MLUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.MLUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.MLUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.MLUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, y1) # the input dtype of elementwise_add must be float16 or float32 @@ -436,7 +415,6 @@ class TestElementwiseAddOpError(unittest.TestCase): class TestAddApi(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.add(x, y, name) @@ -454,7 +432,7 @@ class TestAddApi(unittest.TestCase): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -464,7 +442,7 @@ class TestAddApi(unittest.TestCase): place = fluid.MLUPlace(0) exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -475,18 +453,16 @@ class TestAddApi(unittest.TestCase): y = fluid.dygraph.to_variable(np_y) z = self._executed_api(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) class TestAddInplaceApi(TestAddApi): - def _executed_api(self, x, y, name=None): return x.add_(y, name) class TestAddInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float32') self.y_numpy = np.random.rand(3, 4).astype('float32') @@ -503,21 +479,18 @@ class TestAddInplaceBroadcastSuccess(unittest.TestCase): class TestAddInplaceBroadcastSuccess2(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float32') self.y_numpy = np.random.rand(3, 1).astype('float32') class TestAddInplaceBroadcastSuccess3(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float32') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float32') class TestAddInplaceBroadcastError(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(3, 4).astype('float32') self.y_numpy = np.random.rand(2, 3, 4).astype('float32') @@ -536,21 +509,18 @@ class TestAddInplaceBroadcastError(unittest.TestCase): class TestAddInplaceBroadcastError2(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(2, 1, 4).astype('float32') self.y_numpy = np.random.rand(2, 3, 4).astype('float32') class TestAddInplaceBroadcastError3(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float32') self.y_numpy = np.random.rand(2, 3, 4).astype('float32') class TestBoolAddFloatElementwiseAddop(unittest.TestCase): - def test_static_add(self): paddle.enable_static() a = 1.5 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py index 2171e5043f5..2ca8beed3b3 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py @@ -27,7 +27,6 @@ SEED = 2022 class TestElementwiseDiv(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" @@ -40,7 +39,7 @@ class TestElementwiseDiv(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.attrs = {} self.outputs = {'Out': out} @@ -56,25 +55,30 @@ class TestElementwiseDiv(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.05) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.05 + ) def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.05, - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.05, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.05, - no_grad_set=set("Y")) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.05, + no_grad_set=set("Y"), + ) class TestElementwiseDivFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" @@ -87,7 +91,7 @@ class TestElementwiseDivFp16(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.attrs = {} self.outputs = {'Out': out} @@ -105,124 +109,120 @@ class TestElementwiseDivFp16(OpTest): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestTestElementwiseDiv_scalar(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [20, 3, 4]).astype(np.float32), - 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float32) + 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} class TestTestElementwiseDiv_Vector(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [100]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32"), } self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} class TestTestElementwiseDiv_broadcast_0(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [100, 3, 4]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32"), } self.attrs = {'axis': 0} self.outputs = { - 'Out': np.divide(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestTestElementwiseDiv_broadcast_1(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 100, 4]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32"), } self.attrs = {'axis': 1} self.outputs = { - 'Out': np.divide(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestTestElementwiseDiv_broadcast_2(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32"), } self.outputs = { - 'Out': np.divide(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestTestElementwiseDiv_broadcast_3(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 10, 12, 5]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [10, 12]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [10, 12]).astype("float32"), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 10, 12, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 10, 12, 1) + ) } class TestTestElementwiseDiv_broadcast_4(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 50]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype("float32"), } self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} class TestTestElementwiseDiv_broadcast_5(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 4, 20]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype("float32"), } self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} class TestTestElementwiseDiv_commonuse_1(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" @@ -234,7 +234,6 @@ class TestTestElementwiseDiv_commonuse_1(TestElementwiseDiv): class TestTestElementwiseDiv_commonuse_2(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" @@ -246,7 +245,6 @@ class TestTestElementwiseDiv_commonuse_2(TestElementwiseDiv): class TestTestElementwiseDiv_xsize_lessthan_ysize(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_max_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_max_op_mlu.py index 4fc1533b1bc..cdc354acdcb 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_max_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_max_op_mlu.py @@ -41,7 +41,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_x)) or ( - shape_out[ax] > 1 and shape_x[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_x[ax - src_axis] == 1 + ): reduce_axes_x.append(ax) if shape_y != shape_out: @@ -52,7 +53,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_y)) or ( - shape_out[ax] > 1 and shape_y[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_y[ax - src_axis] == 1 + ): reduce_axes_y.append(ax) if len(reduce_axes_x) > 0: @@ -79,7 +81,6 @@ def ComputeGrad(x, y, out, axis): class TestElementwiseMaxOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_max" @@ -90,7 +91,7 @@ class TestElementwiseMaxOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -106,7 +107,8 @@ class TestElementwiseMaxOp(OpTest): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) sgn = np.random.choice([-1, 1], [13, 17]).astype(self.dtype) self.y = self.x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype( - self.dtype) + self.dtype + ) self.out = np.maximum(self.x, self.y) def init_axis(self): @@ -117,9 +119,9 @@ class TestElementwiseMaxOp(OpTest): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place( self.place, @@ -129,10 +131,13 @@ class TestElementwiseMaxOp(OpTest): def test_check_grad_ingore_x(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + max_relative_error=0.9, + ) else: self.check_grad_with_place( self.place, @@ -143,10 +148,13 @@ class TestElementwiseMaxOp(OpTest): def test_check_grad_ingore_y(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - max_relative_error=0.1) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + max_relative_error=0.1, + ) else: self.check_grad_with_place( self.place, @@ -157,7 +165,6 @@ class TestElementwiseMaxOp(OpTest): class TestElementwiseMaxOp_int32(TestElementwiseMaxOp): - def init_dtype(self): self.dtype = np.int32 @@ -173,15 +180,14 @@ class TestElementwiseMaxOp_int32(TestElementwiseMaxOp): class TestElementwiseMaxOp_FP16(TestElementwiseMaxOp): - def init_dtype(self): self.dtype = np.float16 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMaxOp_scalar(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.random_integers(-5, 5, [2, 3, 20]).astype(self.dtype) self.y = np.array([0.5]).astype(self.dtype) @@ -189,22 +195,22 @@ class TestElementwiseMaxOp_scalar(TestElementwiseMaxOp): class TestElementwiseMaxOp_vector(TestElementwiseMaxOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x + sgn * np.random.uniform(0.1, 1, - (100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(0.1, 1, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxOp_broadcast_0(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (100, 5, 2)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(100, 1, 1)) def init_axis(self): @@ -212,12 +218,12 @@ class TestElementwiseMaxOp_broadcast_0(TestElementwiseMaxOp): class TestElementwiseMaxOp_broadcast_1(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(1, 100, 1)) def init_axis(self): @@ -225,56 +231,68 @@ class TestElementwiseMaxOp_broadcast_1(TestElementwiseMaxOp): def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwiseMaxOp_broadcast_2(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(1, 1, 100)) def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwiseMaxOp_broadcast_3(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 50, 2, 1)).astype(self.dtype) sgn = np.random.choice([-1, 1], (50, 2)).astype(self.dtype) - self.y = self.x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (50, 2)).astype(self.dtype) + self.y = self.x[0, :, :, 0] + sgn * np.random.uniform( + 1, 2, (50, 2) + ).astype(self.dtype) self.out = np.maximum(self.x, self.y.reshape(1, 50, 2, 1)) def init_axis(self): @@ -282,27 +300,26 @@ class TestElementwiseMaxOp_broadcast_3(TestElementwiseMaxOp): class TestElementwiseMaxOp_broadcast_4(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 5)).astype(self.dtype) - self.y = self.x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 5)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(1, 2, (2, 3, 1, 5)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxOp_broadcast_5(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 1)).astype(self.dtype) - self.y = self.x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 1)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(1, 2, (2, 3, 1, 1)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -317,9 +334,9 @@ class TestElementwiseMaxNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.maximum(a, b) @@ -342,16 +359,17 @@ class TestElementwiseMaxNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_min_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_min_op_mlu.py index 31e22904bde..f1546b5ac63 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_min_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_min_op_mlu.py @@ -28,7 +28,6 @@ SEED = 2022 class TestElementwiseMinOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_min" @@ -36,7 +35,7 @@ class TestElementwiseMinOp(OpTest): self.init_input_output() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis} @@ -52,7 +51,8 @@ class TestElementwiseMinOp(OpTest): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) self.sgn = np.random.choice([-1, 1], [13, 17]).astype(self.dtype) self.y = self.x + self.sgn * np.random.uniform(0.1, 1, [13, 17]).astype( - self.dtype) + self.dtype + ) self.out = np.minimum(self.x, self.y) self.axis = -1 @@ -64,9 +64,9 @@ class TestElementwiseMinOp(OpTest): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place( self.place, @@ -76,10 +76,13 @@ class TestElementwiseMinOp(OpTest): def test_check_grad_ingore_x(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + max_relative_error=0.9, + ) else: self.check_grad_with_place( self.place, @@ -90,10 +93,13 @@ class TestElementwiseMinOp(OpTest): def test_check_grad_ingore_y(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - max_relative_error=0.1) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + max_relative_error=0.1, + ) else: self.check_grad_with_place( self.place, @@ -104,32 +110,30 @@ class TestElementwiseMinOp(OpTest): class TestElementwiseMinOpFp16(TestElementwiseMinOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOp_Vector(TestElementwiseMinOp): - def init_input_output(self): - self.x = np.random.uniform(1, 2, (100, )).astype(self.dtype) - self.sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x + self.sgn * np.random.uniform(0.1, 1, (100, )).astype( - self.dtype) + self.x = np.random.uniform(1, 2, (100,)).astype(self.dtype) + self.sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x + self.sgn * np.random.uniform(0.1, 1, (100,)).astype( + self.dtype + ) self.out = np.minimum(self.x, self.y) self.axis = -1 class TestElementwiseMinOpFp16_Vector(TestElementwiseMinOp_Vector): - def init_dtype(self): self.dtype = np.float16 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOp_scalar(TestElementwiseMinOp): - def init_input_output(self): self.x = np.random.random_integers(-5, 5, [10, 3, 4]).astype(self.dtype) self.y = np.array([0.5]).astype(self.dtype) @@ -138,32 +142,30 @@ class TestElementwiseMinOp_scalar(TestElementwiseMinOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOpFp16_scalar(TestElementwiseMinOp_scalar): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOp_broadcast(TestElementwiseMinOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - self.sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, 0, :] + self.sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + self.sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, 0, :] + self.sgn * np.random.uniform( + 1, 2, (100,) + ).astype(self.dtype) self.out = np.minimum(self.x, self.y.reshape(1, 1, 100)) self.axis = -1 class TestElementwiseMinOpFp16_broadcast(TestElementwiseMinOp_broadcast): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOpNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -178,9 +180,9 @@ class TestElementwiseMinOpNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.minimum(a, b) @@ -203,16 +205,17 @@ class TestElementwiseMinOpNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_mul_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_mul_op_mlu.py index fc5e8e89254..cead808d42c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_mul_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_mul_op_mlu.py @@ -30,7 +30,6 @@ paddle.enable_static() class ElementwiseMulOp(OpTest): - def init_kernel_type(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -46,7 +45,7 @@ class ElementwiseMulOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis} @@ -58,14 +57,14 @@ class ElementwiseMulOp(OpTest): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('Y')) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set('Y') + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -80,33 +79,31 @@ class ElementwiseMulOp(OpTest): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float32), - 'Y': np.random.rand(1).astype(np.float32) + 'Y': np.random.rand(1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.random((100, )).astype("float32"), - 'Y': np.random.random((100, )).astype("float32") + 'X': np.random.random((100,)).astype("float32"), + 'Y': np.random.random((100,)).astype("float32"), } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} self.init_kernel_type() class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -117,12 +114,11 @@ class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.attrs = {'axis': 1} @@ -133,12 +129,11 @@ class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.outputs = { @@ -148,12 +143,11 @@ class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float32), - 'Y': np.random.rand(10, 12).astype(np.float32) + 'Y': np.random.rand(10, 12).astype(np.float32), } self.attrs = {'axis': 1} @@ -164,66 +158,60 @@ class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 2, 11).astype(np.float32), - 'Y': np.random.rand(10, 1, 11).astype(np.float32) + 'Y': np.random.rand(10, 1, 11).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 4, 2, 3).astype(np.float32), - 'Y': np.random.rand(10, 4, 1, 3).astype(np.float32) + 'Y': np.random.rand(10, 4, 1, 3).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOpFp16(ElementwiseMulOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(1, 1, 100).astype(np.float32) + 'Y': np.random.rand(1, 1, 100).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(30, 3, 1, 5).astype(np.float32), - 'Y': np.random.rand(30, 1, 4, 1).astype(np.float32) + 'Y': np.random.rand(30, 1, 4, 1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 10).astype(np.float32), - 'Y': np.random.rand(2, 2, 10, 10).astype(np.float32) + 'Y': np.random.rand(2, 2, 10, 10).astype(np.float32), } self.attrs = {'axis': 2} @@ -235,14 +223,15 @@ class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): class TestElementwiseMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_mul must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.elementwise_mul, x1, y1) # the input dtype of elementwise_mul must be float16 or float32 or int32 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_pow_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_pow_op_mlu.py index 300323dba05..7215ce6f9ee 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_pow_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_pow_op_mlu.py @@ -42,7 +42,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_x)) or ( - shape_out[ax] > 1 and shape_x[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_x[ax - src_axis] == 1 + ): reduce_axes_x.append(ax) if shape_y != shape_out: @@ -53,7 +54,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_y)) or ( - shape_out[ax] > 1 and shape_y[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_y[ax - src_axis] == 1 + ): reduce_axes_y.append(ax) if len(reduce_axes_x) > 0: @@ -79,7 +81,6 @@ def ComputeGrad(x, y, out, axis): class TestElementwisePow(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_pow" @@ -90,7 +91,7 @@ class TestElementwisePow(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -116,27 +117,32 @@ class TestElementwisePow(OpTest): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowFp16(TestElementwisePow): - def init_input_output(self): np.random.seed(SEED) self.x = np.random.uniform(1, 2, [11, 17]).astype(self.dtype) @@ -156,7 +162,6 @@ class TestElementwisePowFp16(TestElementwisePow): class TestElementwisePowOp_broadcast_0(TestElementwisePow): - def init_axis(self): self.axis = 1 @@ -168,27 +173,32 @@ class TestElementwisePowOp_broadcast_0(TestElementwisePow): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowOp_broadcast_1(TestElementwisePow): - def init_axis(self): self.axis = 1 @@ -200,27 +210,32 @@ class TestElementwisePowOp_broadcast_1(TestElementwisePow): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowOp_broadcast_2(TestElementwisePow): - def init_axis(self): self.axis = 0 @@ -232,23 +247,29 @@ class TestElementwisePowOp_broadcast_2(TestElementwisePow): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_sub_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_sub_op_mlu.py index 4062cbeaf95..5b2d0a43588 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_sub_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_sub_op_mlu.py @@ -27,7 +27,6 @@ SEED = 2022 class TestElementwiseSubOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" @@ -37,7 +36,7 @@ class TestElementwiseSubOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -64,52 +63,56 @@ class TestElementwiseSubOp(OpTest): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseSubOp_scalar(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float32), - 'Y': np.random.rand(1).astype(np.float32) + 'Y': np.random.rand(1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_Vector(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { - 'X': np.random.random((100, )).astype("float32"), - 'Y': np.random.random((100, )).astype("float32") + 'X': np.random.random((100,)).astype("float32"), + 'Y': np.random.random((100,)).astype("float32"), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_broadcast_0(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(100, 3, 2).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.attrs = {'axis': 0} self.outputs = { @@ -118,13 +121,12 @@ class TestElementwiseSubOp_broadcast_0(TestElementwiseSubOp): class TestElementwiseSubOp_broadcast_1(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.attrs = {'axis': 1} self.outputs = { @@ -133,13 +135,12 @@ class TestElementwiseSubOp_broadcast_1(TestElementwiseSubOp): class TestElementwiseSubOp_broadcast_2(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.outputs = { 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 1, 100) @@ -147,13 +148,12 @@ class TestElementwiseSubOp_broadcast_2(TestElementwiseSubOp): class TestElementwiseSubOp_broadcast_3(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float32), - 'Y': np.random.rand(10, 12).astype(np.float32) + 'Y': np.random.rand(10, 12).astype(np.float32), } self.attrs = {'axis': 1} self.outputs = { @@ -162,49 +162,45 @@ class TestElementwiseSubOp_broadcast_3(TestElementwiseSubOp): class TestElementwiseSubOp_broadcast_4(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 5, 3, 12).astype(np.float32), - 'Y': np.random.rand(2, 5, 1, 12).astype(np.float32) + 'Y': np.random.rand(2, 5, 1, 12).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_1(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(1, 1, 100).astype(np.float32) + 'Y': np.random.rand(1, 1, 100).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_2(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 3, 1, 4).astype(np.float32), - 'Y': np.random.rand(10, 1, 12, 1).astype(np.float32) + 'Y': np.random.rand(10, 1, 12, 1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_xsize_lessthan_ysize(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 12).astype(np.float32), - 'Y': np.random.rand(2, 3, 10, 12).astype(np.float32) + 'Y': np.random.rand(2, 3, 10, 12).astype(np.float32), } self.attrs = {'axis': 2} self.outputs = { diff --git a/python/paddle/fluid/tests/unittests/mlu/test_exp_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_exp_op_mlu.py index b2d77a4c85f..e4c61921386 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_exp_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_exp_op_mlu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestExp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "exp" @@ -55,7 +54,6 @@ class TestExp(OpTest): class TestExpFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "exp" @@ -82,7 +80,6 @@ class TestExpFp16(OpTest): class TestExpNeg(OpTest): - def setUp(self): self.set_mlu() self.op_type = "exp" diff --git a/python/paddle/fluid/tests/unittests/mlu/test_expand_as_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_expand_as_v2_op_mlu.py index 5143a5f83a7..0a88f759831 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_expand_as_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_expand_as_v2_op_mlu.py @@ -25,9 +25,7 @@ paddle.enable_static() def test_class1(op_type, typename): - class TestExpandAsBasic(OpTest): - def setUp(self): self.set_mlu() self.op_type = "expand_as_v2" @@ -54,9 +52,7 @@ def test_class1(op_type, typename): def test_class2(op_type, typename): - class TestExpandAsOpRank2(OpTest): - def setUp(self): self.set_mlu() self.op_type = "expand_as_v2" @@ -83,9 +79,7 @@ def test_class2(op_type, typename): def test_class3(op_type, typename): - class TestExpandAsOpRank3(OpTest): - def setUp(self): self.set_mlu() self.op_type = "expand_as_v2" @@ -112,9 +106,7 @@ def test_class3(op_type, typename): def test_class4(op_type, typename): - class TestExpandAsOpRank4(OpTest): - def setUp(self): self.set_mlu() self.op_type = "expand_as_v2" @@ -142,34 +134,39 @@ def test_class4(op_type, typename): # Test python API class TestExpandAsV2API(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([2, 12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[2, 12, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[2, 12, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = paddle.expand_as(x, y=y) exe = fluid.Executor(place=fluid.MLUPlace(0)) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (2, 1, 1))) for _typename in { - 'float16', 'float32', 'int64', 'int32', 'int8', 'uint8', 'bool' + 'float16', + 'float32', + 'int64', + 'int32', + 'int8', + 'uint8', + 'bool', }: test_class1('expand_as_v2', _typename) test_class2('expand_as_v2', _typename) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_expand_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_expand_v2_op_mlu.py index 527a9bd2e55..f1e8b06cd9a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_expand_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_expand_v2_op_mlu.py @@ -28,7 +28,6 @@ paddle.enable_static() # Situation 1: shape is a list(without tensor) class TestExpandV2OpRank1(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -54,7 +53,6 @@ class TestExpandV2OpRank1(OpTest): class TestExpandV2OpRank2_DimExpanding(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] @@ -62,7 +60,6 @@ class TestExpandV2OpRank2_DimExpanding(TestExpandV2OpRank1): class TestExpandV2OpRank2(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = [1, 140] self.shape = [12, 140] @@ -70,7 +67,6 @@ class TestExpandV2OpRank2(TestExpandV2OpRank1): class TestExpandV2OpRank3_Corner(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) @@ -78,7 +74,6 @@ class TestExpandV2OpRank3_Corner(TestExpandV2OpRank1): class TestExpandV2OpRank4(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) @@ -86,7 +81,6 @@ class TestExpandV2OpRank4(TestExpandV2OpRank1): class TestExpandV2OpRank5(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 4, 1, 15) self.shape = (2, -1, 4, -1) @@ -94,7 +88,6 @@ class TestExpandV2OpRank5(TestExpandV2OpRank1): class TestExpandV2OpRank6(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (4, 1, 30) self.shape = (2, -1, 4, 30) @@ -103,7 +96,6 @@ class TestExpandV2OpRank6(TestExpandV2OpRank1): # Situation 2: shape is a list(with tensor) class TestExpandV2OpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -111,8 +103,9 @@ class TestExpandV2OpRank1_tensor_attr(OpTest): self.init_data() expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float32"), @@ -136,7 +129,6 @@ class TestExpandV2OpRank1_tensor_attr(OpTest): class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -146,7 +138,6 @@ class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpRank1_tensor_attr): # Situation 3: shape is a tensor class TestExpandV2OpRank1_tensor(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -175,7 +166,6 @@ class TestExpandV2OpRank1_tensor(OpTest): # Situation 4: input x is Integer class TestExpandV2OpInteger(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -193,7 +183,6 @@ class TestExpandV2OpInteger(OpTest): # Situation 5: input x is Bool class TestExpandV2OpBoolean(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -209,7 +198,6 @@ class TestExpandV2OpBoolean(OpTest): # Situation 56: input x is Integer class TestExpandV2OpInt64_t(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -226,11 +214,11 @@ class TestExpandV2OpInt64_t(OpTest): class TestExpandV2Error(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.device.MLUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.device.MLUPlace(0) + ) shape = [2, 2] self.assertRaises(TypeError, paddle.tensor.expand, x1, shape) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -242,19 +230,19 @@ class TestExpandV2Error(unittest.TestCase): # Test python API class TestExpandV2API(unittest.TestCase): - def test_api(self): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.expand(x, shape=[12, 14]) out_2 = paddle.expand(x, shape=[positive_2, 14]) @@ -263,28 +251,28 @@ class TestExpandV2API(unittest.TestCase): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=paddle.device.MLUPlace(0)) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) assert np.array_equal(res_3, np.tile(input, (1, 1))) class TestExpandInferShape(unittest.TestCase): - def test_shape_with_var(self): with program_guard(Program(), Program()): x = paddle.static.data(shape=[-1, 1, 3], name='x') fake_var = paddle.randn([2, 3]) target_shape = [ - -1, paddle.shape(fake_var)[0], - paddle.shape(fake_var)[1] + -1, + paddle.shape(fake_var)[0], + paddle.shape(fake_var)[1], ] out = paddle.expand(x, shape=target_shape) self.assertListEqual(list(out.shape), [-1, -1, -1]) @@ -292,7 +280,6 @@ class TestExpandInferShape(unittest.TestCase): # Test python Dygraph API class TestExpandV2DygraphAPI(unittest.TestCase): - def test_expand_times_is_tensor(self): with paddle.fluid.dygraph.guard(): paddle.seed(1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_fill_any_like_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_fill_any_like_op_mlu.py index 249aa48e3d8..021f45aee06 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_fill_any_like_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_fill_any_like_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestFillAnyLikeOp(OpTest): - def setUp(self): self.init_dtype() self.set_mlu() @@ -54,25 +53,21 @@ class TestFillAnyLikeOp(OpTest): class TestFillAnyLikeOp2(TestFillAnyLikeOp): - def set_value(self): self.value = -0.0 class TestFillAnyLikeOp3(TestFillAnyLikeOp): - def set_value(self): self.value = 1.0 class TestFillAnyLikeOp4(TestFillAnyLikeOp): - def set_value(self): self.value = 1e-9 class TestFillAnyLikeOp5(TestFillAnyLikeOp): - def set_value(self): if self.dtype == "float16": self.value = 0.05 @@ -81,7 +76,6 @@ class TestFillAnyLikeOp5(TestFillAnyLikeOp): class TestFillAnyLikeOpInt32(TestFillAnyLikeOp): - def init_dtype(self): self.dtype = np.int32 @@ -90,7 +84,6 @@ class TestFillAnyLikeOpInt32(TestFillAnyLikeOp): class TestFillAnyLikeOpInt64(TestFillAnyLikeOp): - def init_dtype(self): self.dtype = np.int64 @@ -99,7 +92,6 @@ class TestFillAnyLikeOpInt64(TestFillAnyLikeOp): class TestFillAnyLikeOpFloat32(TestFillAnyLikeOp): - def init_dtype(self): self.dtype = np.float32 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_batch_size_like_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_batch_size_like_op_mlu.py index ba5983fbb1e..79b5d7abe7d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_batch_size_like_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_batch_size_like_op_mlu.py @@ -27,20 +27,21 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ paddle.enable_static() -def fill_constant_batch_size_like(input, - shape, - value, - data_type, - input_dim_idx=0, - output_dim_idx=0, - force_cpu=False): +def fill_constant_batch_size_like( + input, + shape, + value, + data_type, + input_dim_idx=0, + output_dim_idx=0, + force_cpu=False, +): return paddle.fluid.layers.fill_constant_batch_size_like( - input, shape, data_type, value, input_dim_idx, output_dim_idx, - force_cpu) + input, shape, data_type, value, input_dim_idx, output_dim_idx, force_cpu + ) class TestFillConstantBatchSizeLike(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -61,11 +62,12 @@ class TestFillConstantBatchSizeLike(OpTest): 'dtype': self.dtype, 'force_cpu': self.force_cpu, 'input_dim_idx': self.input_dim_idx, - 'output_dim_idx': self.output_dim_idx + 'output_dim_idx': self.output_dim_idx, } self.outputs = { - 'Out': np.full(self.output_shape, self.output_value, - self.output_dtype) + 'Out': np.full( + self.output_shape, self.output_value, self.output_dtype + ) } def init_shape(self): @@ -94,7 +96,6 @@ class TestFillConstantBatchSizeLike(OpTest): class TestFillConstantBatchSizeLike2(TestFillConstantBatchSizeLike): - def init_shape(self): # test shape self.input_shape = [4, 5, 6, 7] @@ -103,7 +104,6 @@ class TestFillConstantBatchSizeLike2(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike3(TestFillConstantBatchSizeLike): - def init_value(self): # use 'str_value' rather than 'value' self.value = 3.8 @@ -112,7 +112,6 @@ class TestFillConstantBatchSizeLike3(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike4(TestFillConstantBatchSizeLike): - def init_value(self): # str_value = 'inf' self.value = 3.8 @@ -121,7 +120,6 @@ class TestFillConstantBatchSizeLike4(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike5(TestFillConstantBatchSizeLike): - def init_value(self): # str_value = '-inf' self.value = 3.8 @@ -130,7 +128,6 @@ class TestFillConstantBatchSizeLike5(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike6(TestFillConstantBatchSizeLike): - def init_dtype(self): self.dtype = core.VarDesc.VarType.FP16 self.output_dtype = np.float16 @@ -140,20 +137,17 @@ class TestFillConstantBatchSizeLike6(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike7(TestFillConstantBatchSizeLike): - def init_dtype(self): self.dtype = core.VarDesc.VarType.INT32 self.output_dtype = np.int32 class TestFillConstantBatchSizeLike8(TestFillConstantBatchSizeLike): - def init_force_cpu(self): self.force_cpu = True class TestFillConstantBatchSizeLike9(TestFillConstantBatchSizeLike): - def init_shape(self): self.input_shape = [4, 5] self.shape = [123, 92] @@ -187,11 +181,12 @@ class TestFillConstantBatchSizeLikeLodTensor(TestFillConstantBatchSizeLike): 'dtype': self.dtype, 'force_cpu': self.force_cpu, 'input_dim_idx': self.input_dim_idx, - 'output_dim_idx': self.output_dim_idx + 'output_dim_idx': self.output_dim_idx, } self.outputs = { - 'Out': np.full(self.output_shape, self.output_value, - self.output_dtype) + 'Out': np.full( + self.output_shape, self.output_value, self.output_dtype + ) } def init_shape(self): @@ -201,7 +196,8 @@ class TestFillConstantBatchSizeLikeLodTensor(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLikeLodTensor2( - TestFillConstantBatchSizeLikeLodTensor): + TestFillConstantBatchSizeLikeLodTensor +): # test LodTensor with 'input_dim_idx' != 0 def init_shape(self): self.input_shape = [10, 20] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_op_mlu.py index b05f720b5f1..7a19779b049 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_op_mlu.py @@ -31,10 +31,8 @@ paddle.enable_static() # Situation 1: Attr(shape) is a list(without tensor) class TestFillConstantOp1(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.inputs = {} @@ -49,10 +47,8 @@ class TestFillConstantOp1(OpTest): class TestFillConstantOp2(OpTest): - def setUp(self): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' self.op_type = "fill_constant" self.inputs = {} @@ -67,10 +63,8 @@ class TestFillConstantOp2(OpTest): class TestFillConstantOp3(OpTest): - def setUp(self): - '''Test fill_constant op with specified int64 value - ''' + '''Test fill_constant op with specified int64 value''' self.op_type = "fill_constant" self.inputs = {} @@ -85,10 +79,8 @@ class TestFillConstantOp3(OpTest): class TestFillConstantOp4(OpTest): - def setUp(self): - '''Test fill_constant op with specified int value - ''' + '''Test fill_constant op with specified int value''' self.op_type = "fill_constant" self.inputs = {} @@ -103,17 +95,15 @@ class TestFillConstantOp4(OpTest): class TestFillConstantOpWithSelectedRows(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() # create Out Variable out = scope.var('Out').get_selected_rows() # create and run fill_constant_op operator - fill_constant_op = Operator("fill_constant", - shape=[123, 92], - value=3.8, - Out='Out') + fill_constant_op = Operator( + "fill_constant", shape=[123, 92], value=3.8, Out='Out' + ) fill_constant_op.run(scope, place) # get result from Out @@ -133,16 +123,15 @@ class TestFillConstantOpWithSelectedRows(unittest.TestCase): # Situation 2: Attr(shape) is a list(with tensor) class TestFillConstantOp1_ShapeTensorList(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape, 'value': self.value} @@ -161,16 +150,15 @@ class TestFillConstantOp1_ShapeTensorList(OpTest): class TestFillConstantOp2_ShapeTensorList(OpTest): - def setUp(self): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' self.op_type = "fill_constant" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape} @@ -188,7 +176,6 @@ class TestFillConstantOp2_ShapeTensorList(OpTest): class TestFillConstantOp3_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): - def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -196,7 +183,6 @@ class TestFillConstantOp3_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): class TestFillConstantOp4_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): - def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -205,10 +191,8 @@ class TestFillConstantOp4_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): # Situation 3: shape is a tensor class TestFillConstantOp1_ShapeTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() @@ -229,16 +213,14 @@ class TestFillConstantOp1_ShapeTensor(OpTest): # Situation 4: value is a tensor class TestFillConstantOp1_ValueTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype("float32") + 'ValueTensor': np.array([self.value]).astype("float32"), } self.attrs = {'value': self.value + 1.0} self.outputs = {'Out': np.full(self.shape, self.value)} @@ -247,7 +229,7 @@ class TestFillConstantOp1_ValueTensor(OpTest): self.__class__.use_mlu = True def init_data(self): - #self.shape = [123, 92] + # self.shape = [123, 92] self.shape = [2, 2] self.value = 3.8 self.dtype = np.float32 @@ -258,16 +240,14 @@ class TestFillConstantOp1_ValueTensor(OpTest): # Situation 5: value is a tensor class TestFillConstantOp2_ValueTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype("int32") + 'ValueTensor': np.array([self.value]).astype("int32"), } self.attrs = {'value': self.value, 'dtype': 2} self.outputs = {'Out': np.full(self.shape, self.value)} @@ -286,56 +266,55 @@ class TestFillConstantOp2_ValueTensor(OpTest): # Test python API class TestFillConstantAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 2) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - - out_1 = fluid.layers.fill_constant(shape=[1, 2], - dtype="float32", - value=1.1) - - out_2 = fluid.layers.fill_constant(shape=[1, positive_2_int32], - dtype="float32", - value=1.1) - - out_3 = fluid.layers.fill_constant(shape=[1, positive_2_int64], - dtype="float32", - value=1.1) - - out_4 = fluid.layers.fill_constant(shape=shape_tensor_int32, - dtype="float32", - value=1.1) - - out_5 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype="float32", - value=1.1) - - out_6 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=1.1) - - val1 = fluid.layers.fill_constant(shape=[1], - dtype=np.float32, - value=1.1) - val2 = fluid.layers.fill_constant(shape=[1], - dtype=np.float64, - value=1.1) - out_7 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=val1) - - out_8 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=val2) + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + + out_1 = fluid.layers.fill_constant( + shape=[1, 2], dtype="float32", value=1.1 + ) + + out_2 = fluid.layers.fill_constant( + shape=[1, positive_2_int32], dtype="float32", value=1.1 + ) + + out_3 = fluid.layers.fill_constant( + shape=[1, positive_2_int64], dtype="float32", value=1.1 + ) + + out_4 = fluid.layers.fill_constant( + shape=shape_tensor_int32, dtype="float32", value=1.1 + ) + + out_5 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype="float32", value=1.1 + ) + + out_6 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=1.1 + ) + + val1 = fluid.layers.fill_constant( + shape=[1], dtype=np.float32, value=1.1 + ) + val2 = fluid.layers.fill_constant( + shape=[1], dtype=np.float64, value=1.1 + ) + out_7 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=val1 + ) + + out_8 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=val2 + ) exe = fluid.Executor(place=fluid.CPUPlace()) res_1, res_2, res_3, res_4, res_5, res_6, res_7, res_8 = exe.run( @@ -344,7 +323,8 @@ class TestFillConstantAPI(unittest.TestCase): "shape_tensor_int32": np.array([1, 2]).astype("int32"), "shape_tensor_int64": np.array([1, 2]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.array_equal(res_1, np.full([1, 2], 1.1, dtype="float32")) assert np.array_equal(res_2, np.full([1, 2], 1.1, dtype="float32")) @@ -357,7 +337,6 @@ class TestFillConstantAPI(unittest.TestCase): class TestFillConstantImperative(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): data1 = np.array([1, 2]).astype('int32') @@ -366,26 +345,30 @@ class TestFillConstantImperative(unittest.TestCase): shape = fluid.dygraph.to_variable(data1) val = fluid.dygraph.to_variable(data2) value = fluid.dygraph.to_variable(data3) - res1 = fluid.layers.fill_constant(shape=[1, 2], - dtype='float32', - value=1.1) - res2 = fluid.layers.fill_constant(shape=shape, - dtype='float32', - value=1.1) - res3 = fluid.layers.fill_constant(shape=shape, - dtype='float32', - value=val) - res4 = fluid.layers.fill_constant(shape=shape, - dtype='int32', - value=value) - assert np.array_equal(res1.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res2.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res3.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res4.numpy(), - np.full([1, 2], 88, dtype="int32")) + res1 = fluid.layers.fill_constant( + shape=[1, 2], dtype='float32', value=1.1 + ) + res2 = fluid.layers.fill_constant( + shape=shape, dtype='float32', value=1.1 + ) + res3 = fluid.layers.fill_constant( + shape=shape, dtype='float32', value=val + ) + res4 = fluid.layers.fill_constant( + shape=shape, dtype='int32', value=value + ) + assert np.array_equal( + res1.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res2.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res3.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res4.numpy(), np.full([1, 2], 88, dtype="int32") + ) def test_nan(self): with fluid.dygraph.guard(): @@ -405,42 +388,49 @@ class TestFillConstantImperative(unittest.TestCase): class TestFillConstantOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage + # for ci coverage x1 = fluid.layers.data(name='x1', shape=[1], dtype="int16") - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1], - value=5, - dtype='uint4') - - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1.1], - value=5, - dtype='float32', - out=x1) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1], + value=5, + dtype='uint4', + ) + + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1.1], + value=5, + dtype='float32', + out=x1, + ) # The argument dtype of fill_constant_op must be one of bool, float16, - #float32, float64, uint8, int16, int32 or int64 + # float32, float64, uint8, int16, int32 or int64 x2 = fluid.layers.data(name='x2', shape=[1], dtype="int32") - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1], - value=5, - dtype='float64', - out=x2) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1], + value=5, + dtype='float64', + out=x2, + ) x3 = np.random.randn(100, 100).astype('int32') - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[100, 100], - value=5, - dtype='float64', - out=x3) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[100, 100], + value=5, + dtype='float64', + out=x3, + ) # The argument shape's type of fill_constant_op must be list, tuple or Variable. def test_shape_type(): @@ -456,22 +446,22 @@ class TestFillConstantOpError(unittest.TestCase): # The shape dtype of fill_constant_op must be int32 or int64. def test_shape_tensor_dtype(): - shape = fluid.data(name="shape_tensor", - shape=[2], - dtype="float32") - fluid.layers.fill_constant(shape=shape, - dtype="float32", - value=1) + shape = fluid.data( + name="shape_tensor", shape=[2], dtype="float32" + ) + fluid.layers.fill_constant( + shape=shape, dtype="float32", value=1 + ) self.assertRaises(TypeError, test_shape_tensor_dtype) def test_shape_tensor_list_dtype(): - shape = fluid.data(name="shape_tensor_list", - shape=[1], - dtype="bool") - fluid.layers.fill_constant(shape=[shape, 2], - dtype="float32", - value=1) + shape = fluid.data( + name="shape_tensor_list", shape=[1], dtype="bool" + ) + fluid.layers.fill_constant( + shape=[shape, 2], dtype="float32", value=1 + ) self.assertRaises(TypeError, test_shape_tensor_list_dtype) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_flatten2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_flatten2_op_mlu.py index c799fd01143..ace458ff42c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_flatten2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_flatten2_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestFlattenOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -35,7 +34,7 @@ class TestFlattenOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def test_check_output(self): @@ -54,7 +53,6 @@ class TestFlattenOp(OpTest): class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.axis = 0 @@ -62,7 +60,6 @@ class TestFlattenOp1(TestFlattenOp): class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -72,7 +69,6 @@ class TestFlattenOpWithDefaultAxis(TestFlattenOp): class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 @@ -80,7 +76,6 @@ class TestFlattenOpSixDims(TestFlattenOp): class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): - def execute_api(self, x, axis=1): return fluid.layers.flatten(x, axis=axis) @@ -88,15 +83,14 @@ class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): paddle.enable_static() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[-1, 3, -1, -1], - dtype='float32') + x = paddle.static.data( + name="x", shape=[-1, 3, -1, -1], dtype='float32' + ) out = self.execute_api(x, axis=2) self.assertTrue((-1, -1) == out.shape) class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_data = np.random.random((3, 2, 4, 5)).astype("float64") @@ -109,9 +103,9 @@ class TestFlatten2OpError(unittest.TestCase): def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = fluid.layers.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2 = fluid.layers.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) fluid.layers.flatten(x2, axis=1) self.assertRaises(TypeError, test_type) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_flatten_contigous_range_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_flatten_contigous_range_op_mlu.py index 1a988f6104d..942d5f9c897 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_flatten_contigous_range_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_flatten_contigous_range_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestFlattenOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "flatten_contiguous_range" @@ -39,7 +38,7 @@ class TestFlattenOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def set_mlu(self): @@ -49,24 +48,23 @@ class TestFlattenOp(OpTest): self.check_output_with_place(self.place, no_check_set=["XShape"]) def test_check_grad(self): - #pass + # pass self.check_grad_with_place(self.place, ["X"], "Out") def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 self.stop_axis = -1 - self.new_shape = (120) + self.new_shape = 120 def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 1 @@ -76,12 +74,11 @@ class TestFlattenOp_1(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_2(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -91,12 +88,11 @@ class TestFlattenOp_2(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_3(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -106,12 +102,11 @@ class TestFlattenOp_3(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_4(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = -2 @@ -121,12 +116,11 @@ class TestFlattenOp_4(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_5(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 2 @@ -136,12 +130,11 @@ class TestFlattenOp_5(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.start_axis = 3 @@ -151,12 +144,11 @@ class TestFlattenOpSixDims(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_Float32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -167,12 +159,11 @@ class TestFlattenOp_Float32(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_int32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -183,7 +174,7 @@ class TestFlattenOp_int32(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -191,7 +182,6 @@ class TestFlattenOp_int32(TestFlattenOp): class TestFlattenOp_uint8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -202,7 +192,7 @@ class TestFlattenOp_uint8(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -210,7 +200,6 @@ class TestFlattenOp_uint8(TestFlattenOp): class TestFlattenOp_int8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -221,7 +210,7 @@ class TestFlattenOp_int8(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -229,7 +218,6 @@ class TestFlattenOp_int8(TestFlattenOp): class TestFlattenOp_int64(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -240,7 +228,7 @@ class TestFlattenOp_int64(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -248,45 +236,58 @@ class TestFlattenOp_int64(TestFlattenOp): class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_ValueError1(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) out = paddle.flatten(x_var, start_axis=2, stop_axis=1) self.assertRaises(ValueError, test_ValueError1) def test_ValueError2(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=10, stop_axis=1) self.assertRaises(ValueError, test_ValueError2) def test_ValueError3(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=2, stop_axis=10) self.assertRaises(ValueError, test_ValueError3) def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x2 = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x2 = x2.astype('float16') - x2_var = paddle.fluid.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2_var = paddle.fluid.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) paddle.flatten(x2_var) self.assertRaises(TypeError, test_type) @@ -298,7 +299,6 @@ class TestFlatten2OpError(unittest.TestCase): class TestStaticFlattenPythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -308,9 +308,9 @@ class TestStaticFlattenPythonAPI(unittest.TestCase): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[2, 3, 4, 4], - dtype='float32') + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32' + ) out = self.execute_api(x, start_axis=-2, stop_axis=-1) exe = paddle.static.Executor(place=paddle.MLUPlace(0)) @@ -319,17 +319,22 @@ class TestStaticFlattenPythonAPI(unittest.TestCase): class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): - def execute_api(self, x, start_axis=0, stop_axis=-1): return x.flatten_(start_axis, stop_axis) class TestFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_InputError(): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_flatten_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_flatten_op_mlu.py index d06c165e8d4..0b8da1f18f0 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_flatten_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_flatten_op_mlu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestFlattenOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -52,7 +51,6 @@ class TestFlattenOp(OpTest): class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 2, 10) self.axis = 0 @@ -60,7 +58,6 @@ class TestFlattenOp1(TestFlattenOp): class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -70,7 +67,6 @@ class TestFlattenOpWithDefaultAxis(TestFlattenOp): class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_floor_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_floor_op_mlu.py index bdee19091fb..2e661b217da 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_floor_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_floor_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestFloor(OpTest): - def setUp(self): self.op_type = "floor" self.place = paddle.device.MLUPlace(0) @@ -49,7 +48,6 @@ class TestFloor(OpTest): class TestFloorFP16(TestFloor): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_gather_nd_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_gather_nd_op_mlu.py index ad903da68bc..3fb5634df45 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_gather_nd_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_gather_nd_op_mlu.py @@ -26,11 +26,11 @@ paddle.enable_static() def gather_nd_grad(x, index): # for TestGatherNdOpWithLowIndex - dout_shape = index.shape[:-1] + x.shape[index.shape[-1]:] + dout_shape = index.shape[:-1] + x.shape[index.shape[-1] :] numel = 1 for i in dout_shape: numel = numel * i - dout = np.full(dout_shape, 1. / numel) + dout = np.full(dout_shape, 1.0 / numel) dx = np.full_like(x, 0) index = tuple(index.reshape(-1, index.shape[-1]).T) @@ -40,7 +40,6 @@ def gather_nd_grad(x, index): def test_class1(op_type, typename): - class TestGatherNdOpWithEmptyIndex(OpTest): # Index has empty element, which means copy entire tensor @@ -51,7 +50,7 @@ def test_class1(op_type, typename): xnp = np.random.random((5, 20)).astype(typename) self.inputs = { 'X': xnp, - 'Index': np.array([[], []]).astype("int32") + 'Index': np.array([[], []]).astype("int32"), } self.outputs = { 'Out': np.vstack((xnp[np.newaxis, :], xnp[np.newaxis, :])) @@ -76,9 +75,7 @@ def test_class1(op_type, typename): def test_class2(op_type, typename): - class TestGatherNdOpWithIndex1(OpTest): - def setUp(self): self.set_mlu() self.op_type = "gather_nd" @@ -106,9 +103,8 @@ def test_class2(op_type, typename): def test_class3(op_type, typename): - class TestGatherNdOpWithLowIndex(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.set_mlu() @@ -132,9 +128,9 @@ def test_class3(op_type, typename): if typename == "float16": self.__class__.no_need_check_grad = True else: - self.check_grad_with_place(self.place, ['X'], - 'Out', - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Out', user_defined_grads=[self.x_grad] + ) cls_name = "{0}_{1}_3".format(op_type, typename) TestGatherNdOpWithLowIndex.__name__ = cls_name @@ -142,9 +138,8 @@ def test_class3(op_type, typename): def test_class4(op_type, typename): - class TestGatherNdOpIndex1(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.set_mlu() @@ -176,9 +171,8 @@ def test_class4(op_type, typename): def test_class5(op_type, typename): - class TestGatherNdOpWithSameIndexAsX(OpTest): - #Index has same rank as X's rank + # Index has same rank as X's rank def setUp(self): self.set_mlu() @@ -188,7 +182,7 @@ def test_class5(op_type, typename): index = np.array([[1, 1], [2, 1]]).astype("int64") self.inputs = {'X': xnp, 'Index': index} - self.outputs = {'Out': xnp[tuple(index.T)]} #[25, 22] + self.outputs = {'Out': xnp[tuple(index.T)]} # [25, 22] def set_mlu(self): self.__class__.use_mlu = True @@ -209,9 +203,8 @@ def test_class5(op_type, typename): def test_class6(op_type, typename): - class TestGatherNdOpWithHighRankSame(OpTest): - #Both Index and X have high rank, and Rank(Index) = Rank(X) + # Both Index and X have high rank, and Rank(Index) = Rank(X) def setUp(self): self.set_mlu() @@ -219,8 +212,9 @@ def test_class6(op_type, typename): self.python_api = paddle.gather_nd shape = (5, 2, 3, 1, 10) xnp = np.random.rand(*shape).astype(typename) - index = np.vstack([np.random.randint(0, s, size=2) - for s in shape]).T + index = np.vstack( + [np.random.randint(0, s, size=2) for s in shape] + ).T self.inputs = {'X': xnp, 'Index': index.astype("int32")} self.outputs = {'Out': xnp[tuple(index.T)]} @@ -244,9 +238,8 @@ def test_class6(op_type, typename): def test_class7(op_type, typename): - class TestGatherNdOpWithHighRankDiff(OpTest): - #Both Index and X have high rank, and Rank(Index) < Rank(X) + # Both Index and X have high rank, and Rank(Index) < Rank(X) def setUp(self): self.set_mlu() @@ -255,7 +248,8 @@ def test_class7(op_type, typename): shape = (2, 3, 4, 1, 10) xnp = np.random.rand(*shape).astype(typename) index = np.vstack( - [np.random.randint(0, s, size=200) for s in shape]).T + [np.random.randint(0, s, size=200) for s in shape] + ).T index_re = index.reshape([20, 5, 2, 5]) self.inputs = {'X': xnp, 'Index': index_re.astype("int32")} @@ -279,9 +273,8 @@ def test_class7(op_type, typename): globals()[cls_name] = TestGatherNdOpWithHighRankDiff -#Test Python API +# Test Python API class TestGatherNdAPI2(unittest.TestCase): - def test_imperative(self): paddle.disable_static() input_1 = np.array([[1, 2], [3, 4], [5, 6]]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_gather_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_gather_op_mlu.py index 32881fe3427..2f50c10a62f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_gather_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_gather_op_mlu.py @@ -34,7 +34,6 @@ def gather_numpy(x, index, axis): class TestGatherOp(OpTest): - def setUp(self): self.op_type = "gather" self.place = paddle.MLUPlace(0) @@ -44,7 +43,7 @@ class TestGatherOp(OpTest): xnp = np.random.random(self.x_shape).astype(self.x_type) self.inputs = { 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) + 'Index': np.array(self.index).astype(self.index_type), } self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} @@ -65,31 +64,28 @@ class TestGatherOp(OpTest): class TestCase1(TestGatherOp): - def config(self): """ For one dimension input """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float32" self.index = [1, 3, 5] self.index_type = "int32" class TestCase2(TestGatherOp): - def config(self): """ For int64_t index type """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float32" self.index = [1, 3, 5] self.index_type = "int64" class API_TestDygraphGather(unittest.TestCase): - def test_out1(self): paddle.disable_static() input_1 = np.array([[1, 2], [3, 4], [5, 6]]).astype('int32') @@ -127,18 +123,18 @@ class API_TestDygraphGather(unittest.TestCase): class TestGathertError(unittest.TestCase): - def test_error1(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='int8', name='x') axis = paddle.fluid.data(shape=[1], dtype='float32', name='axis') index = paddle.fluid.data(shape=shape, dtype='int32', name='index') - index_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = paddle.fluid.data( + shape=shape, dtype='float32', name='index_float' + ) def test_x_type(): paddle.gather(x, index) @@ -166,9 +162,9 @@ class TestGathertError(unittest.TestCase): shape = [8, 9, 6] x = fluid.data(shape=shape, dtype='int8', name='x') index = fluid.data(shape=shape, dtype='int32', name='mask') - index_float = fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = fluid.data( + shape=shape, dtype='float32', name='index_float' + ) def test_x_type(): paddle.fluid.layers.gather(x, index) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_gaussian_random_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_gaussian_random_op_mlu.py index 83540bfadd2..4cac3a2e36b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_gaussian_random_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_gaussian_random_op_mlu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestGaussianRandomOp(OpTest): - def setUp(self): self.op_type = "gaussian_random" self.place = paddle.device.MLUPlace(0) @@ -48,7 +47,7 @@ class TestGaussianRandomOp(OpTest): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def test_check_output(self): self.check_output_with_place_customized(self.verify_output, self.place) @@ -67,7 +66,6 @@ class TestGaussianRandomOp(OpTest): class TestMeanStdAreInt(TestGaussianRandomOp): - def set_attrs(self): self.mean = 1 self.std = 2 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_gelu_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_gelu_op_mlu.py index b905eba2371..8a879aa2221 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_gelu_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_gelu_op_mlu.py @@ -32,7 +32,6 @@ def np_gelu(x): class TestGelu(OpTest): - def setUp(self): self.set_mlu() self.op_type = "gelu" @@ -57,13 +56,12 @@ class TestGelu(OpTest): self.check_output_with_place(self.place, atol=1e-3) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.007) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.007 + ) class TestGeluFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "gelu" @@ -90,7 +88,6 @@ class TestGeluFp16(OpTest): class TestGeluNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -105,9 +102,9 @@ class TestGeluNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) @@ -131,16 +128,17 @@ class TestGeluNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_grid_sampler_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_grid_sampler_op_mlu.py index 96dbaab9ee1..df173ebf18c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_grid_sampler_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_grid_sampler_op_mlu.py @@ -28,12 +28,15 @@ def AffineGrid(theta, grid_shape): n = grid_shape[0] h = grid_shape[1] w = grid_shape[2] - h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, - axis=0).T[:, :, np.newaxis] - w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, - axis=0)[:, :, np.newaxis] - grid = np.concatenate([w_idx, h_idx, np.ones([h, w, 1])], - axis=2) # h * w * 3 + h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, axis=0).T[ + :, :, np.newaxis + ] + w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, axis=0)[ + :, :, np.newaxis + ] + grid = np.concatenate( + [w_idx, h_idx, np.ones([h, w, 1])], axis=2 + ) # h * w * 3 grid = np.repeat(grid[np.newaxis, :], n, axis=0) # n * h * w *3 ret = np.zeros([n, h * w, 2]) @@ -53,13 +56,17 @@ def getGridPointValue(data, x, y): out_H = x.shape[1] out_W = x.shape[2] - #out = np.zeros(data_shape, dtype='float32') + # out = np.zeros(data_shape, dtype='float32') out = np.zeros([N, C, out_H, out_W], dtype='float32') for i in range(N): for j in range(out_H): for k in range(out_W): - if y[i, j, k] < 0 or y[i, j, k] > in_H - 1 or x[ - i, j, k] < 0 or x[i, j, k] > in_W - 1: + if ( + y[i, j, k] < 0 + or y[i, j, k] > in_H - 1 + or x[i, j, k] < 0 + or x[i, j, k] > in_W - 1 + ): out[i, :, j, k] = 0 else: out[i, :, j, k] = data[i, :, y[i, j, k], x[i, j, k]] @@ -75,27 +82,28 @@ def unnormalizeAndClip(grid_slice, max_val, align_corners, padding_mode): if align_corners: grid_slice = 0.5 * ((grid_slice.astype('float32') + 1.0) * max_val) else: - grid_slice = 0.5 * ((grid_slice.astype('float32') + 1.0) * - (max_val + 1)) - 0.5 + grid_slice = ( + 0.5 * ((grid_slice.astype('float32') + 1.0) * (max_val + 1)) - 0.5 + ) if padding_mode == "border": grid_slice = clip(grid_slice, 0, max_val) elif padding_mode == "reflection": double_range = 2 * max_val if align_corners else (max_val + 1) * 2 - grid_abs = np.abs(grid_slice) if align_corners else np.abs(grid_slice + - 0.5) + grid_abs = ( + np.abs(grid_slice) if align_corners else np.abs(grid_slice + 0.5) + ) extra = grid_abs - np.floor(grid_abs / double_range) * double_range grid_slice = np.minimum(extra, double_range - extra) - grid_slice = grid_slice if align_corners else clip( - grid_slice - 0.5, 0, max_val) + grid_slice = ( + grid_slice if align_corners else clip(grid_slice - 0.5, 0, max_val) + ) return grid_slice -def GridSampler(data, - grid, - align_corners=True, - mode="bilinear", - padding_mode="zeros"): +def GridSampler( + data, grid, align_corners=True, mode="bilinear", padding_mode="zeros" +): dims = data.shape N = dims[0] in_C = dims[1] @@ -119,14 +127,18 @@ def GridSampler(data, y0 = np.floor(y).astype('int32') y1 = y0 + 1 - wa = np.tile(((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wb = np.tile(((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wc = np.tile(((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wd = np.tile(((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) + wa = np.tile( + ((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wb = np.tile( + ((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wc = np.tile( + ((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wd = np.tile( + ((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) va = getGridPointValue(data, x0, y0) vb = getGridPointValue(data, x0, y1) @@ -142,7 +154,6 @@ def GridSampler(data, class TestGridSamplerOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -166,12 +177,12 @@ class TestGridSamplerOp(OpTest): 'use_cudnn': False, "align_corners": self.align_corners, "padding_mode": self.padding_mode, - "mode": self.mode + "mode": self.mode, } self.outputs = { - 'Output': - GridSampler(x, grid, self.align_corners, self.mode, - self.padding_mode) + 'Output': GridSampler( + x, grid, self.align_corners, self.mode, self.padding_mode + ) } def test_check_output(self): @@ -187,7 +198,6 @@ class TestGridSamplerOp(OpTest): class Case1(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -198,7 +208,6 @@ class Case1(TestGridSamplerOp): class LargeInputCase(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 128, 128) self.grid_shape = (2, 130, 130, 2) @@ -209,7 +218,6 @@ class LargeInputCase(TestGridSamplerOp): class Case2(LargeInputCase): - def initTestCase(self): self.x_shape = (2, 3, 128, 128) self.grid_shape = (2, 130, 130, 2) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_hard_sigmoid_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_hard_sigmoid_op_mlu.py index fca95416a86..695d37ec54d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_hard_sigmoid_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_hard_sigmoid_op_mlu.py @@ -28,11 +28,10 @@ np.random.seed(SEED) def ref_hardsigmoid(x, slope=0.166666666666667, offset=0.5): - return np.maximum(np.minimum(x * slope + offset, 1.), 0.).astype(x.dtype) + return np.maximum(np.minimum(x * slope + offset, 1.0), 0.0).astype(x.dtype) class TestMLUHardSigmoid(OpTest): - def setUp(self): paddle.enable_static() @@ -43,7 +42,7 @@ class TestMLUHardSigmoid(OpTest): x = np.random.uniform(-5, 5, [10, 12]).astype(self.dtype) lower_threshold = -self.offset / self.slope - upper_threshold = (1. - self.offset) / self.slope + upper_threshold = (1.0 - self.offset) / self.slope # Same reason as TestAbs delta = 0.005 @@ -75,21 +74,18 @@ class TestMLUHardSigmoid(OpTest): class TestMLUHardSigmoid2(TestMLUHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.5 class TestMLUHardSigmoid3(TestMLUHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.4 class TestMLUHardSigmoidFp16(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -114,17 +110,21 @@ class TestMLUHardSigmoidFp16(unittest.TestCase): cpu_diff_1 = np.divide( np.sum(np.abs(self.float32_y.numpy() - self.float16_y)), - np.sum(np.abs(self.float32_y.numpy()))) + np.sum(np.abs(self.float32_y.numpy())), + ) mlu_diff_1 = np.divide( np.sum(np.abs(self.float32_y.numpy() - mlu_float16_y.numpy())), - np.sum(np.abs(self.float32_y.numpy()))) + np.sum(np.abs(self.float32_y.numpy())), + ) cpu_diff_2 = np.divide( np.sum(np.square(self.float32_y.numpy() - self.float16_y)), - np.sum(np.square(self.float32_y.numpy()))) + np.sum(np.square(self.float32_y.numpy())), + ) mlu_diff_2 = np.divide( np.sum(np.square(self.float32_y.numpy() - mlu_float16_y.numpy())), - np.sum(np.square(self.float32_y.numpy()))) + np.sum(np.square(self.float32_y.numpy())), + ) assert mlu_diff_1 <= cpu_diff_1 assert mlu_diff_2 <= cpu_diff_2 @@ -179,14 +179,14 @@ class TestHardsigmoidAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.hardsigmoid, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardsigmoid, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardsigmoid(x_fp16) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_hard_swish_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_hard_swish_op_mlu.py index a88103154e9..2774162bc79 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_hard_swish_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_hard_swish_op_mlu.py @@ -38,15 +38,17 @@ def ref_hard_swish_grad(x, threshold, scale, offset, data_type): threshold = scalarToType(threshold, data_type) scale = scalarToType(scale, data_type) offset = scalarToType(offset, data_type) - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) tmp = ((x + offset) < threshold).astype(x.dtype) - dx = dout * (((x + offset) > 0).astype(x.dtype) * - (2 * x + offset) * tmp / scale + 1.0 - tmp) + dx = dout * ( + ((x + offset) > 0).astype(x.dtype) * (2 * x + offset) * tmp / scale + + 1.0 + - tmp + ) return dx class TestHardSwishMLU(OpTest): - def setUp(self): paddle.enable_static() @@ -63,11 +65,11 @@ class TestHardSwishMLU(OpTest): x[np.abs(x - threshold + offset) < 0.005] = threshold - offset + 0.02 out = ( - x * - (np.minimum(np.maximum(x + offset, 0.), threshold) / scale)).astype( - self.dtype) - self.x_grad = ref_hard_swish_grad(x, threshold, scale, offset, - self.dtype) + x * (np.minimum(np.maximum(x + offset, 0.0), threshold) / scale) + ).astype(self.dtype) + self.x_grad = ref_hard_swish_grad( + x, threshold, scale, offset, self.dtype + ) self.set_mlu() self.inputs = {'X': x} self.attrs = {'threshold': threshold, 'scale': scale, 'offset': offset} @@ -87,7 +89,6 @@ class TestHardSwishMLU(OpTest): class TestHardSwishMLUWithCPUFloat16(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -111,11 +112,21 @@ class TestHardSwishMLUWithCPUFloat16(unittest.TestCase): threshold = scalarToType(threshold, np.float16) scale = scalarToType(scale, np.float16) offset = scalarToType(offset, np.float16) - self.float16_y = (self.float16_x * (np.minimum( - np.maximum(self.float16_x + offset, scalarToType(0., np.float16)), - threshold) / scale)).astype(np.float16) - self.float16_grad = ref_hard_swish_grad(self.float16_x, threshold, - scale, offset, np.float16) + self.float16_y = ( + self.float16_x + * ( + np.minimum( + np.maximum( + self.float16_x + offset, scalarToType(0.0, np.float16) + ), + threshold, + ) + / scale + ) + ).astype(np.float16) + self.float16_grad = ref_hard_swish_grad( + self.float16_x, threshold, scale, offset, np.float16 + ) def test_check_output_and_grad_mlu(self): # mlu float16 @@ -127,36 +138,45 @@ class TestHardSwishMLUWithCPUFloat16(unittest.TestCase): cpu_diff_1 = np.divide( np.sum(np.abs(self.float32_y.numpy() - self.float16_y)), - np.sum(np.abs(self.float32_y.numpy()))) + np.sum(np.abs(self.float32_y.numpy())), + ) mlu_diff_1 = np.divide( np.sum(np.abs(self.float32_y.numpy() - mlu_float16_y.numpy())), - np.sum(np.abs(self.float32_y.numpy()))) + np.sum(np.abs(self.float32_y.numpy())), + ) cpu_diff_2 = np.divide( np.sum(np.square(self.float32_y.numpy() - self.float16_y)), - np.sum(np.square(self.float32_y.numpy()))) + np.sum(np.square(self.float32_y.numpy())), + ) mlu_diff_2 = np.divide( np.sum(np.square(self.float32_y.numpy() - mlu_float16_y.numpy())), - np.sum(np.square(self.float32_y.numpy()))) + np.sum(np.square(self.float32_y.numpy())), + ) assert mlu_diff_1 <= cpu_diff_1 assert mlu_diff_2 <= cpu_diff_2 cpu_diff_1 = np.divide( np.sum(np.abs(self.float32_grad.numpy() - self.float16_grad)), - np.sum(np.abs(self.float32_grad.numpy()))) + np.sum(np.abs(self.float32_grad.numpy())), + ) mlu_diff_1 = np.divide( - np.sum(np.abs(self.float32_grad.numpy() - - mlu_float16_grad.numpy())), - np.sum(np.abs(self.float32_grad.numpy()))) + np.sum( + np.abs(self.float32_grad.numpy() - mlu_float16_grad.numpy()) + ), + np.sum(np.abs(self.float32_grad.numpy())), + ) cpu_diff_2 = np.divide( np.sum(np.square(self.float32_grad.numpy() - self.float16_grad)), - np.sum(np.square(self.float32_grad.numpy()))) + np.sum(np.square(self.float32_grad.numpy())), + ) mlu_diff_2 = np.divide( np.sum( - np.square(self.float32_grad.numpy() - - mlu_float16_grad.numpy())), - np.sum(np.square(self.float32_grad.numpy()))) + np.square(self.float32_grad.numpy() - mlu_float16_grad.numpy()) + ), + np.sum(np.square(self.float32_grad.numpy())), + ) assert mlu_diff_1 <= cpu_diff_1 assert mlu_diff_2 <= cpu_diff_2 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_huber_loss_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_huber_loss_op_mlu.py index 40e0dab0803..35003418095 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_huber_loss_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_huber_loss_op_mlu.py @@ -34,7 +34,6 @@ def huber_loss_forward(val, delta): class TestHuberLossOp(OpTest): - def setUp(self): self.op_type = 'huber_loss' self.set_mlu() @@ -44,16 +43,17 @@ class TestHuberLossOp(OpTest): self.init_input() shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - self.delta).astype('float32') + loss = np.vectorize(huber_loss_forward)(residual, self.delta).astype( + 'float32' + ) self.attrs = {'delta': self.delta} self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def init_input(self): shape = self.set_shape() self.inputs = { - 'X': np.random.uniform(0, 1., shape).astype('float32'), - 'Y': np.random.uniform(0, 1., shape).astype('float32'), + 'X': np.random.uniform(0, 1.0, shape).astype('float32'), + 'Y': np.random.uniform(0, 1.0, shape).astype('float32'), } def set_mlu(self): @@ -70,38 +70,40 @@ class TestHuberLossOp(OpTest): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.008, - no_grad_set=set("residual")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.008, + no_grad_set=set("residual"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.008, - no_grad_set=set('residual')) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.008, + no_grad_set=set('residual'), + ) def TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (64) + return 64 def TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (6, 6) def TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (6, 6, 1) class TestHuberLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input and label must be Variable @@ -116,10 +118,12 @@ class TestHuberLossOpError(unittest.TestCase): # the dtype of input and label must be float32 or float64 xw2 = fluid.data(name='xw2', shape=[None, 6], dtype="int32") lw2 = fluid.data(name='lw2', shape=[None, 6], dtype="int32") - self.assertRaises(TypeError, fluid.layers.huber_loss, xw2, lr, - delta) - self.assertRaises(TypeError, fluid.layers.huber_loss, xr, lw2, - delta) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xw2, lr, delta + ) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xr, lw2, delta + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_iou_similarity_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_iou_similarity_op_mlu.py index ac206d6649a..a9035df0409 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_iou_similarity_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_iou_similarity_op_mlu.py @@ -28,7 +28,6 @@ np.random.seed(2022) class TestMluIouSimilarityOp(OpTest): - def setUp(self): self.op_type = "iou_similarity" self.set_mlu() @@ -58,7 +57,7 @@ class TestMluIouSimilarityOp(OpTest): self.boxes1_lod = [[1 for _ in range(self.N)]] self.inputs = { 'X': (self.boxes1, self.boxes1_lod), - 'Y': self.boxes2 + 'Y': self.boxes2, } else: self.inputs = {'X': self.boxes1, 'Y': self.boxes2} @@ -74,7 +73,9 @@ class TestMluIouSimilarityOp(OpTest): def test_check_output(self): self.check_output_with_place(self.place) - def _compute_iou(self, ): + def _compute_iou( + self, + ): for row in range(self.boxes1.shape[0]): for col in range(self.boxes2.shape[0]): xmin1, ymin1, xmax1, ymax1 = self.boxes1[row] @@ -104,7 +105,6 @@ class TestMluIouSimilarityOp(OpTest): class TestMluIouSimilarityOpWithLoD(TestMluIouSimilarityOp): - def set_init_config(self): super(TestMluIouSimilarityOpWithLoD, self).set_init_config() self.box_normalized = True @@ -112,7 +112,6 @@ class TestMluIouSimilarityOpWithLoD(TestMluIouSimilarityOp): class TestMluIouSimilarityOpWithBoxNormalized(TestMluIouSimilarityOp): - def set_init_config(self): super(TestMluIouSimilarityOpWithBoxNormalized, self).set_init_config() self.box_normalized = True @@ -120,7 +119,6 @@ class TestMluIouSimilarityOpWithBoxNormalized(TestMluIouSimilarityOp): def TestMluIouSimilarityOpFp16(TestMluIouSimilarityOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_label_smooth_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_label_smooth_op_mlu.py index dfc01e42202..0379c8c9cd8 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_label_smooth_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_label_smooth_op_mlu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestLabelSmoothOp(OpTest): - def init_dtype(self): self.dtype = np.float32 @@ -46,7 +45,8 @@ class TestLabelSmoothOp(OpTest): self.init_dtype() self.config() smoothed_label = ( - 1 - self.epsilon) * self.label + self.epsilon / self.label_dim + 1 - self.epsilon + ) * self.label + self.epsilon / self.label_dim smoothed_label = smoothed_label.astype(self.dtype) self.inputs = {'X': self.label} self.attrs = {'epsilon': self.epsilon} @@ -57,7 +57,6 @@ class TestLabelSmoothOp(OpTest): class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): - def setUp(self): self.init_dtype() self.config() @@ -70,45 +69,43 @@ class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): class TestLabelSmoothOp3D(TestLabelSmoothOp): - def setUp(self): super(TestLabelSmoothOp3D, self).setUp() self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) class TestLabelSmoothOpWithPriorDist3D(TestLabelSmoothOpWithPriorDist): - def setUp(self): super(TestLabelSmoothOpWithPriorDist3D, self).setUp() self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) class TestLabelSmoothOpFP16(TestLabelSmoothOp): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOpWithPriorDistFP16(TestLabelSmoothOpWithPriorDist): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOp3DFP16(TestLabelSmoothOp3D): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOpWithPriorDist3DFP16(TestLabelSmoothOpWithPriorDist3D): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_layer_norm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_layer_norm_op_mlu.py index 838ee571772..1a0eac49ecc 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_layer_norm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_layer_norm_op_mlu.py @@ -26,8 +26,13 @@ import sys sys.path.append('..') from op_test import _set_use_system_allocator from paddle.fluid import Program, program_guard -from paddle.fluid.contrib.mixed_precision.fp16_utils import _keep_layer_norm_scale_bias_to_fp32 -from test_layer_norm_op import _reference_layer_norm_naive, _reference_layer_norm_grad +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _keep_layer_norm_scale_bias_to_fp32, +) +from test_layer_norm_op import ( + _reference_layer_norm_naive, + _reference_layer_norm_grad, +) paddle.enable_static() @@ -37,51 +42,57 @@ _set_use_system_allocator(True) class TestLayerNormOp(unittest.TestCase): - def setUp(self): self.use_cudnn = True self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-5, - atol=atol, - err_msg=msg) - - def check_forward_backward(self, - shape, - begin_norm_axis, - has_scale=True, - has_bias=True, - y_grad_scale=1.0, - use_mkldnn=False): - - def test_with_place(place, - shape, - begin_norm_axis, - use_mkldnn=use_mkldnn): + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-5, atol=atol, err_msg=msg + ) + + def check_forward_backward( + self, + shape, + begin_norm_axis, + has_scale=True, + has_bias=True, + y_grad_scale=1.0, + use_mkldnn=False, + ): + def test_with_place( + place, shape, begin_norm_axis, use_mkldnn=use_mkldnn + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) x = np.random.random_sample(x_shape).astype(np.float32) - scale = np.random.random_sample(scale_shape).astype( - np.float32) if has_scale else None - bias = np.random.random_sample(scale_shape).astype( - np.float32) if has_bias else None + scale = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_scale + else None + ) + bias = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_bias + else None + ) y_grad = (np.random.random_sample(x_shape) * y_grad_scale).astype( - np.float32) + np.float32 + ) # reference forward & backward y, mean, variance = _reference_layer_norm_naive( - x, scale, bias, epsilon, begin_norm_axis) + x, scale, bias, epsilon, begin_norm_axis + ) x_grad, scale_grad, bias_grad = _reference_layer_norm_grad( - x, y_grad, scale, bias, mean, variance, begin_norm_axis) + x, y_grad, scale, bias, mean, variance, begin_norm_axis + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -96,9 +107,11 @@ class TestLayerNormOp(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = {"X": block.var('x')} fetch_list = [ 'y', @@ -118,17 +131,20 @@ class TestLayerNormOp(unittest.TestCase): outputs={ "Y": block.var('y'), "Mean": block.var('mean'), # share the same memory - "Variance": - block.var('variance'), # share the same memory + "Variance": block.var( + 'variance' + ), # share the same memory }, attrs={ "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, - "use_mkldnn": use_mkldnn - }) + "use_mkldnn": use_mkldnn, + }, + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - layer_norm_op.desc, set(), []) + layer_norm_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -142,105 +158,130 @@ class TestLayerNormOp(unittest.TestCase): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'scale', 'bias', 'y@GRAD'] - }, - fetch_list=fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in ['x', 'scale', 'bias', 'y@GRAD'] + }, + fetch_list=fetch_list, + ) self.__assert_close(y, out[0], "y") self.__assert_close(mean, out[1], "mean") - self.__assert_close(1 / np.sqrt(variance), out[2], "variance", - 1e-3) + self.__assert_close( + 1 / np.sqrt(variance), out[2], "variance", 1e-3 + ) self.__assert_close(x_grad, out[3], "x_grad") if has_scale: - self.__assert_close(scale_grad.reshape(-1), - out[fetch_list.index('scale@GRAD')], - "scale_grad", 1e-3) + self.__assert_close( + scale_grad.reshape(-1), + out[fetch_list.index('scale@GRAD')], + "scale_grad", + 1e-3, + ) if has_bias: - self.__assert_close(bias_grad.reshape(-1), - out[fetch_list.index('bias@GRAD')], - "bias_grad") + self.__assert_close( + bias_grad.reshape(-1), + out[fetch_list.index('bias@GRAD')], + "bias_grad", + ) test_with_place(self.place, shape, begin_norm_axis) def test_check_forward_backward_with_scale_and_bias(self): self.check_forward_backward(shape=[1, 3, 4, 5], begin_norm_axis=1) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=1) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=True) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=True, - has_bias=False) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=False) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=True, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=True, + has_bias=False, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=False, + ) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=3) - self.check_forward_backward(shape=[92, 513, 129], - begin_norm_axis=2, - y_grad_scale=0.1) + self.check_forward_backward( + shape=[92, 513, 129], begin_norm_axis=2, y_grad_scale=0.1 + ) self.check_forward_backward(shape=[3, 34, 1134], begin_norm_axis=2) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=False, - has_bias=True, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=True, - has_bias=False, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=False, - has_bias=False, - y_grad_scale=0.1) - self.check_forward_backward(shape=[512, 1024], - begin_norm_axis=1, - has_scale=True, - has_bias=True) + self.check_forward_backward( + shape=[92, 513, 1134], begin_norm_axis=2, y_grad_scale=0.1 + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=False, + has_bias=True, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=True, + has_bias=False, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=False, + has_bias=False, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[512, 1024], begin_norm_axis=1, has_scale=True, has_bias=True + ) class TestLayerNormAPI(unittest.TestCase): - def test_case(self): - x = fluid.layers.data(name='x', - shape=[64, 32, 256], - dtype='float32', - append_batch_size=False) - x = fluid.layers.layer_norm(x, - scale=True, - shift=True, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None) - x = fluid.layers.layer_norm(x, - scale=False, - shift=False, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None) - x = fluid.layers.layer_norm(x, - scale=False, - shift=False, - begin_norm_axis=1, - epsilon=1e-05, - param_attr="scale", - bias_attr="shift") + x = fluid.layers.data( + name='x', + shape=[64, 32, 256], + dtype='float32', + append_batch_size=False, + ) + x = fluid.layers.layer_norm( + x, + scale=True, + shift=True, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + ) + x = fluid.layers.layer_norm( + x, + scale=False, + shift=False, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + ) + x = fluid.layers.layer_norm( + x, + scale=False, + shift=False, + begin_norm_axis=1, + epsilon=1e-05, + param_attr="scale", + bias_attr="shift", + ) class TestDygraphLayerNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): paddle.enable_static() @@ -256,7 +297,6 @@ class TestDygraphLayerNormAPIError(unittest.TestCase): class TestFP16ScaleBiasLayerNorm(unittest.TestCase): - def check_main(self, x_np, weight_np, bias_np, dtype): paddle.disable_static() @@ -285,9 +325,11 @@ class TestFP16ScaleBiasLayerNorm(unittest.TestCase): bias_np = np.random.random([20]).astype('float16') y_np_1, x_g_np_1, w_g_np_1, b_g_np_1 = self.check_main( - x_np, weight_np, bias_np, 'float16') + x_np, weight_np, bias_np, 'float16' + ) y_np_2, x_g_np_2, w_g_np_2, b_g_np_2 = self.check_main( - x_np, weight_np, bias_np, 'float32') + x_np, weight_np, bias_np, 'float32' + ) def assert_equal(x, y): np.testing.assert_allclose(x, y) @@ -299,7 +341,6 @@ class TestFP16ScaleBiasLayerNorm(unittest.TestCase): class TestGetSetKeepLayerNormScaleBiasFP32Flag(unittest.TestCase): - def test_main(self): self.assertTrue(_keep_layer_norm_scale_bias_to_fp32()) _keep_layer_norm_scale_bias_to_fp32(False) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_leaky_relu_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_leaky_relu_op_mlu.py index 23fd10ffbd8..05d78e7f31a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_leaky_relu_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_leaky_relu_op_mlu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestLeadyRelu(OpTest): - def setUp(self): self.set_mlu() self.op_type = "leaky_relu" @@ -63,33 +62,29 @@ class TestLeadyRelu(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.006 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') class TestLeadyReluFP16(TestLeadyRelu): - def init_dtype(self): self.dtype = np.float16 class TestLeadyRelu2(TestLeadyRelu): - def set_attrs(self): self.attrs = {'alpha': 0.5} class TestLeadyRelu3(TestLeadyRelu): - def set_attrs(self): self.attrs = {'alpha': -0.5} class TestLeakyReluNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -102,9 +97,9 @@ class TestLeakyReluNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): x = paddle.static.data(name="x", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) y = paddle.nn.functional.leaky_relu(x) @@ -127,15 +122,17 @@ class TestLeakyReluNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "x": x_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"x": x_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_log_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_log_op_mlu.py index 1dca8d9a0bc..11df0949a6f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_log_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_log_op_mlu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestActivation(OpTest): - def setUp(self): self.set_mlu() self.op_type = "exp" @@ -57,7 +56,6 @@ class TestActivation(OpTest): class TestLog(TestActivation): - def setUp(self): self.set_mlu() self.op_type = "log" @@ -72,21 +70,18 @@ class TestLog(TestActivation): self.outputs = {'Out': out} def test_error(self): - in1 = fluid.layers.data(name="in1", - shape=[11, 17], - append_batch_size=False, - dtype="int32") - in2 = fluid.layers.data(name="in2", - shape=[11, 17], - append_batch_size=False, - dtype="int64") + in1 = fluid.layers.data( + name="in1", shape=[11, 17], append_batch_size=False, dtype="int32" + ) + in2 = fluid.layers.data( + name="in2", shape=[11, 17], append_batch_size=False, dtype="int64" + ) self.assertRaises(TypeError, fluid.layers.log, in1) self.assertRaises(TypeError, fluid.layers.log, in2) class TestLog2(TestActivation): - def setUp(self): self.set_mlu() self.op_type = "log2" @@ -107,19 +102,22 @@ class TestLog2(TestActivation): self.assertRaises(TypeError, paddle.log2, in2) def test_api(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float32") - data_x = paddle.static.data(name="data_x", - shape=[11, 17], - dtype="float32") + data_x = paddle.static.data( + name="data_x", shape=[11, 17], dtype="float32" + ) out1 = paddle.log2(data_x) exe = paddle.static.Executor(place=fluid.CPUPlace()) exe.run(paddle.static.default_startup_program()) - res1 = exe.run(paddle.static.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + res1 = exe.run( + paddle.static.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log2(input_x) np.testing.assert_allclose(res1[0], expected_res, rtol=1e-6) @@ -136,7 +134,6 @@ class TestLog2(TestActivation): class TestLog10(TestActivation): - def setUp(self): self.set_mlu() self.op_type = "log10" @@ -157,19 +154,22 @@ class TestLog10(TestActivation): self.assertRaises(TypeError, paddle.log10, in2) def test_api(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float32") - data_x = paddle.static.data(name="data_x", - shape=[11, 17], - dtype="float32") + data_x = paddle.static.data( + name="data_x", shape=[11, 17], dtype="float32" + ) out1 = paddle.log10(data_x) exe = paddle.static.Executor(place=paddle.CPUPlace()) exe.run(paddle.static.default_startup_program()) - res1 = exe.run(paddle.static.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + res1 = exe.run( + paddle.static.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log10(input_x) np.testing.assert_allclose(res1[0], expected_res, rtol=1e-6) @@ -184,7 +184,6 @@ class TestLog10(TestActivation): class TestLogHalf(TestLog): - def init_dtype(self): self.dtype = np.float16 @@ -193,7 +192,6 @@ class TestLogHalf(TestLog): class TestLog2Half(TestLog2): - def init_dtype(self): self.dtype = np.float16 @@ -202,7 +200,6 @@ class TestLog2Half(TestLog2): class TestLog10Half(TestLog10): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_log_softmax_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_log_softmax_op_mlu.py index 91e8a86ce68..82d22af8933 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_log_softmax_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_log_softmax_op_mlu.py @@ -24,7 +24,7 @@ paddle.enable_static() def ref_log_softmax(x): - shiftx = (x - np.max(x)) + shiftx = x - np.max(x) out = shiftx - np.log(np.exp(shiftx).sum()) return out @@ -34,14 +34,14 @@ def ref_log_softmax_grad(x, axis): axis += len(x.shape) out = np.apply_along_axis(ref_log_softmax, axis, x) axis_dim = x.shape[axis] - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) dx = dout - np.exp(out) * dout.copy().sum(axis=axis, keepdims=True).repeat( - axis_dim, axis=axis) + axis_dim, axis=axis + ) return dx class TestLogSoftmaxOp(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.set_mlu() @@ -51,7 +51,7 @@ class TestLogSoftmaxOp(OpTest): self.axis = -1 self.set_attrs() - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -70,24 +70,22 @@ class TestLogSoftmaxOp(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], ['Out'], - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], ['Out'], user_defined_grads=[self.x_grad] + ) class TestLogSoftmaxShape(TestLogSoftmaxOp): - def set_attrs(self): self.shape = [12, 10] class TestLogSoftmaxAxis(TestLogSoftmaxOp): - def set_attrs(self): self.axis = 1 class TestLogSoftmaxOpFp16(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.set_mlu() @@ -97,7 +95,7 @@ class TestLogSoftmaxOpFp16(OpTest): self.axis = -1 self.set_attrs() - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -116,17 +114,20 @@ class TestLogSoftmaxOpFp16(OpTest): self.check_output_with_place(self.place, atol=1e-2) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], ['Out'], - user_defined_grads=[self.x_grad], - max_relative_error=0.015) + self.check_grad_with_place( + self.place, + ['X'], + ['Out'], + user_defined_grads=[self.x_grad], + max_relative_error=0.015, + ) class TestNNLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.set_mlu() self.x_shape = [2, 3, 4, 5] - self.x = np.random.uniform(-1., 1., self.x_shape).astype(np.float32) + self.x = np.random.uniform(-1.0, 1.0, self.x_shape).astype(np.float32) def set_mlu(self): self.__class__.use_mlu = True @@ -158,7 +159,6 @@ class TestNNLogSoftmaxAPI(unittest.TestCase): class TestNNFunctionalLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.set_mlu() self.x_shape = [2, 3, 4, 5] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_logical_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_logical_op_mlu.py index c032748eb9a..6923f669678 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_logical_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_logical_op_mlu.py @@ -26,84 +26,32 @@ paddle.enable_static() SUPPORTED_DTYPES = [bool, np.int8, np.int16, np.int32, np.float32] -TEST_META_OP_DATA = [{ - 'op_str': 'logical_and', - 'binary_op': True -}, { - 'op_str': 'logical_or', - 'binary_op': True -}, { - 'op_str': 'logical_xor', - 'binary_op': True -}, { - 'op_str': 'logical_not', - 'binary_op': False -}] +TEST_META_OP_DATA = [ + {'op_str': 'logical_and', 'binary_op': True}, + {'op_str': 'logical_or', 'binary_op': True}, + {'op_str': 'logical_xor', 'binary_op': True}, + {'op_str': 'logical_not', 'binary_op': False}, +] TEST_META_SHAPE_DATA = { - 'XDimLargerThanYDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 5] - }, - 'XDimLargerThanYDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 1] - }, - 'XDimLargerThanYDim3': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [1, 4, 1] - }, - 'XDimLargerThanYDim4': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4, 1] - }, - 'XDimLargerThanYDim5': { - 'x_shape': [2, 3, 1, 5], - 'y_shape': [3, 1, 1] - }, - 'XDimLessThanYDim1': { - 'x_shape': [4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim2': { - 'x_shape': [1, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim3': { - 'x_shape': [3, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim4': { - 'x_shape': [3, 1, 1], - 'y_shape': [2, 3, 1, 5] - }, - 'XDimLessThanYDim5': { - 'x_shape': [4, 5], - 'y_shape': [2, 3, 4, 5] - }, - 'Axis1InLargerDim': { - 'x_shape': [1, 4, 5], - 'y_shape': [2, 3, 1, 5] - }, - 'EqualDim1': { - 'x_shape': [10, 7], - 'y_shape': [10, 7] - }, - 'EqualDim2': { - 'x_shape': [1, 1, 4, 5], - 'y_shape': [2, 3, 1, 5] - } + 'XDimLargerThanYDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 5]}, + 'XDimLargerThanYDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 1]}, + 'XDimLargerThanYDim3': {'x_shape': [2, 3, 4, 5], 'y_shape': [1, 4, 1]}, + 'XDimLargerThanYDim4': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4, 1]}, + 'XDimLargerThanYDim5': {'x_shape': [2, 3, 1, 5], 'y_shape': [3, 1, 1]}, + 'XDimLessThanYDim1': {'x_shape': [4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim2': {'x_shape': [1, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim3': {'x_shape': [3, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim4': {'x_shape': [3, 1, 1], 'y_shape': [2, 3, 1, 5]}, + 'XDimLessThanYDim5': {'x_shape': [4, 5], 'y_shape': [2, 3, 4, 5]}, + 'Axis1InLargerDim': {'x_shape': [1, 4, 5], 'y_shape': [2, 3, 1, 5]}, + 'EqualDim1': {'x_shape': [10, 7], 'y_shape': [10, 7]}, + 'EqualDim2': {'x_shape': [1, 1, 4, 5], 'y_shape': [2, 3, 1, 5]}, } TEST_META_WRONG_SHAPE_DATA = { - 'ErrorDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4] - }, - 'ErrorDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 3] - } + 'ErrorDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4]}, + 'ErrorDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 3]}, } @@ -162,16 +110,20 @@ def test(unit_test, use_mlu=False, test_error=False): META_DATA = dict(TEST_META_WRONG_SHAPE_DATA) for shape_data in META_DATA.values(): for data_type in SUPPORTED_DTYPES: - meta_data['x_np'] = np_data_generator(shape_data['x_shape'], - dtype=data_type) - meta_data['y_np'] = np_data_generator(shape_data['y_shape'], - dtype=data_type) + meta_data['x_np'] = np_data_generator( + shape_data['x_shape'], dtype=data_type + ) + meta_data['y_np'] = np_data_generator( + shape_data['y_shape'], dtype=data_type + ) if meta_data['binary_op'] and test_error: # catch C++ Exception - unit_test.assertRaises(BaseException, run_static, - **meta_data) - unit_test.assertRaises(BaseException, run_dygraph, - **meta_data) + unit_test.assertRaises( + BaseException, run_static, **meta_data + ) + unit_test.assertRaises( + BaseException, run_dygraph, **meta_data + ) continue static_result = run_static(**meta_data) dygraph_result = run_dygraph(**meta_data) @@ -181,11 +133,11 @@ def test(unit_test, use_mlu=False, test_error=False): np_result = np_op(meta_data['x_np']) unit_test.assertTrue((static_result == np_result).all()) unit_test.assertTrue( - (dygraph_result.numpy() == np_result).all()) + (dygraph_result.numpy() == np_result).all() + ) def test_type_error(unit_test, use_mlu, type_str_map): - def check_type(op_str, x, y, binary_op): op = getattr(paddle, op_str) error_type = ValueError @@ -220,24 +172,24 @@ def test_type_error(unit_test, use_mlu, type_str_map): startup_program = paddle.static.Program() main_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.static.data(name='x', - shape=[10], - dtype=type_str_map['x']) - y = paddle.static.data(name='y', - shape=[10], - dtype=type_str_map['y']) + x = paddle.static.data( + name='x', shape=[10], dtype=type_str_map['x'] + ) + y = paddle.static.data( + name='y', shape=[10], dtype=type_str_map['y'] + ) check_type(meta_data['op_str'], x, y, binary_op) def type_map_factory(): - return [{ - 'x': x_type, - 'y': y_type - } for x_type in SUPPORTED_DTYPES for y_type in SUPPORTED_DTYPES] + return [ + {'x': x_type, 'y': y_type} + for x_type in SUPPORTED_DTYPES + for y_type in SUPPORTED_DTYPES + ] class TestMLU(unittest.TestCase): - def test(self): test(self, True) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_lookup_table_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_lookup_table_v2_op_mlu.py index 29b58a4bbec..f8b962d44d1 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_lookup_table_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_lookup_table_v2_op_mlu.py @@ -26,7 +26,6 @@ SEED = 2022 class TestLookupTableV2(OpTest): - def setUp(self): self.set_mlu() self.op_type = "lookup_table_v2" @@ -36,22 +35,22 @@ class TestLookupTableV2(OpTest): self.init_padding_idx() np.random.seed(SEED) w = np.random.random([self.vocab, self.dim]).astype(self.dtype) - x = np.random.randint(0, self.vocab, - size=(self.bsz, - self.seqlen)).astype(self.ids_dtype) + x = np.random.randint( + 0, self.vocab, size=(self.bsz, self.seqlen) + ).astype(self.ids_dtype) out = w[x] if self.padding_idx != -1: out[np.squeeze(x == self.padding_idx)] = np.zeros(self.dim) self.inputs = { 'W': OpTest.np_dtype_to_fluid_dtype(w), - 'Ids': OpTest.np_dtype_to_fluid_dtype(x) + 'Ids': OpTest.np_dtype_to_fluid_dtype(x), } self.attrs = { 'is_sparse': False, 'is_distributed': False, 'remote_prefetch': False, - 'padding_idx': self.padding_idx + 'padding_idx': self.padding_idx, } self.outputs = {'Out': out} @@ -78,9 +77,9 @@ class TestLookupTableV2(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['W'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['W'], 'Out', max_relative_error=0.01 + ) else: self.check_grad_with_place(self.place, ['W'], 'Out') @@ -98,7 +97,6 @@ class TestLookupTableV2FP16(TestLookupTableV2): class TestLookupTableV2Dim32(TestLookupTableV2): - def init_dims(self): self.bsz = 6 self.seqlen = 8 @@ -126,13 +124,11 @@ class TestLookupTableV2Dim32FP16(TestLookupTableV2): class TestLookupTableV2WithPadding(TestLookupTableV2): - def init_padding_idx(self): self.padding_idx = np.random.randint(0, self.vocab) class TestLookupTableV2WithPadding1(TestLookupTableV2): - def init_padding_idx(self): self.padding_idx = np.random.randint(0, self.vocab) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_masked_select_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_masked_select_op_mlu.py index c60eea07874..11e9dc86cd4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_masked_select_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_masked_select_op_mlu.py @@ -39,7 +39,6 @@ def np_masked_select(shape, x, mask): class TestMaskedSelectOp(OpTest): - def setUp(self): self.init() self.__class__.use_mlu = True @@ -63,20 +62,17 @@ class TestMaskedSelectOp(OpTest): class TestMaskedSelectOp1(TestMaskedSelectOp): - def init(self): self.shape = (6, 8, 9, 18) class TestMaskedSelectOp2(TestMaskedSelectOp): - def init(self): - self.shape = (168, ) + self.shape = (168,) @skip_check_grad_ci(reason="get_numeric_gradient not support int32") class TestMaskedSelectOpInt32(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.int32 @@ -85,20 +81,18 @@ class TestMaskedSelectOpInt32(TestMaskedSelectOp): class TestMaskedSelectOpFp16(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.float16 def test_check_grad(self): x_grad = self.inputs['Mask'].astype(self.dtype) x_grad = x_grad * (1 / x_grad.size) - self.check_grad_with_place(self.place, ['X'], - 'Y', - user_defined_grads=[x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Y', user_defined_grads=[x_grad] + ) class TestMaskedSelectAPI(unittest.TestCase): - def test_imperative_mode(self): paddle.disable_static() shape = (88, 6, 8) @@ -123,27 +117,26 @@ class TestMaskedSelectAPI(unittest.TestCase): exe = paddle.static.Executor(place=paddle.device.MLUPlace(0)) - res = exe.run(paddle.static.default_main_program(), - feed={ - "x": np_x, - "mask": np_mask - }, - fetch_list=[out]) + res = exe.run( + paddle.static.default_main_program(), + feed={"x": np_x, "mask": np_mask}, + fetch_list=[out], + ) self.assertEqual(np.allclose(res, np_out), True) class TestMaskedSelectError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') mask = paddle.fluid.data(shape=shape, dtype='bool', name='mask') - mask_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='mask_float') + mask_float = paddle.fluid.data( + shape=shape, dtype='float32', name='mask_float' + ) np_x = np.random.random(shape).astype('float32') np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_matmul_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_matmul_op_mlu.py index 76daa775756..a33ef9d5412 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_matmul_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_matmul_op_mlu.py @@ -31,7 +31,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False, scale=1.0): # transpose X and Y appropriately. if transpose_X: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -40,7 +40,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False, scale=1.0): X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -76,14 +76,15 @@ class TestMatMulOp(OpTest): X = -0.1 + 0.2 * X Y = -0.1 + 0.2 * Y - Out = reference_matmul(X, Y, self.transpose_X, self.transpose_Y, - self.alpha) + Out = reference_matmul( + X, Y, self.transpose_X, self.transpose_Y, self.alpha + ) Out = Out.astype(self.dtype) self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, 'transpose_Y': self.transpose_Y, - 'alpha': self.alpha + 'alpha': self.alpha, } self.outputs = {'Out': Out} @@ -92,8 +93,8 @@ class TestMatMulOp(OpTest): self.place = paddle.device.MLUPlace(0) def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.transpose_X = False self.transpose_Y = False @@ -116,7 +117,7 @@ class TestMatMulOp1(TestMatMulOp): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.transpose_X = False self.transpose_Y = True @@ -129,7 +130,7 @@ class TestMatMulOp2(TestMatMulOp): def config(self): self.x_shape = (1, 2, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.transpose_X = True self.transpose_Y = False @@ -248,7 +249,7 @@ class TestMatMulOp12(TestMatMulOp): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.transpose_X = False self.transpose_Y = False @@ -261,13 +262,13 @@ class TestMatMulOp13(TestMatMulOp): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.transpose_X = False self.transpose_Y = False # TODO(mlu): alpha will be supported in next version -#--------------------test matmul alpha-------------------- +# --------------------test matmul alpha-------------------- # def create_test_alpha_class(parent): # class TestMatMulOpAlphaCase(parent): # def init_alpha(self): @@ -291,11 +292,9 @@ class TestMatMulOp13(TestMatMulOp): # create_test_alpha_class(TestMatMulOp13) -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): - class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -303,9 +302,12 @@ def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): self.check_output_with_place(self.place, atol=atol) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=max_relative_error) + self.check_grad_with_place( + self.place, + ['X', 'Y'], + 'Out', + max_relative_error=max_relative_error, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mlu/test_matmul_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_matmul_v2_op_mlu.py index 4c782980b3d..f1b142902b1 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_matmul_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_matmul_v2_op_mlu.py @@ -36,8 +36,8 @@ class TestMatMulV2Op(OpTest): self.place = paddle.device.MLUPlace(0) def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -76,7 +76,7 @@ class TestMatMuklOp2(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.trans_x = False self.trans_y = True @@ -88,7 +88,7 @@ class TestMatMuklOp3(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False @@ -100,7 +100,7 @@ class TestMatMuklOp4(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -113,7 +113,7 @@ class TestMatMuklOp5(TestMatMulV2Op): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False @@ -125,7 +125,7 @@ class TestMatMuklOp6(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 102, 1) - self.y_shape = (102, ) + self.y_shape = (102,) self.trans_x = True self.trans_y = False @@ -137,7 +137,7 @@ class TestMatMuklOp7(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 1, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -244,7 +244,7 @@ class TestMatMuklOp16(TestMatMulV2Op): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -257,7 +257,7 @@ class TestMatMuklOp17(TestMatMulV2Op): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False @@ -298,13 +298,11 @@ class TestMatMuklOpBroadcast2(TestMatMulV2Op): self.trans_y = True -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): - class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -312,9 +310,12 @@ def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): self.check_output_with_place(self.place, atol=atol) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=max_relative_error) + self.check_grad_with_place( + self.place, + ['X', 'Y'], + 'Out', + max_relative_error=max_relative_error, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name @@ -342,7 +343,6 @@ create_test_fp16_class(TestMatMuklOp18) class TestMatMulV2API(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if paddle.is_compiled_with_mlu(): @@ -359,12 +359,11 @@ class TestMatMulV2API(unittest.TestCase): y_np = np.random.random([3, 4]).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": x_np, - "input_y": y_np - }, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np, "input_y": y_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: diff --git a/python/paddle/fluid/tests/unittests/mlu/test_mean_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_mean_op_mlu.py index 5dcdf1db845..60f9d7b8e45 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_mean_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_mean_op_mlu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestMean(OpTest): - def setUp(self): self.set_mlu() self.place = paddle.device.MLUPlace(0) @@ -55,7 +54,6 @@ class TestMean(OpTest): class TestMeanFP16(OpTest): - def setUp(self): self.set_mlu() self.place = paddle.MLUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_merged_adam_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_merged_adam_op_mlu.py index 3aa61e9f982..242e1c8e663 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_merged_adam_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_merged_adam_op_mlu.py @@ -21,20 +21,22 @@ from paddle import _C_ops, _legacy_C_ops from paddle.fluid.framework import in_dygraph_mode -def run_adam_op(params, - grads, - lrs, - moment1s, - moment2s, - beta1_pows, - beta2_pows, - master_params, - epsilon, - beta1, - beta2, - place, - multi_precision=False, - use_merged=False): +def run_adam_op( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + epsilon, + beta1, + beta2, + place, + multi_precision=False, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(lrs) assert len(params) == len(moment1s) @@ -59,25 +61,71 @@ def run_adam_op(params, if not use_merged: for i in range(len(param_vars)): _, _, _, _, _, _ = _legacy_C_ops.adam( - param_vars[i], grad_vars[i], lr_vars[i], moment1_vars[i], - moment2_vars[i], beta1_pow_vars[i], beta2_pow_vars[i], - master_param_vars[i], param_vars[i], moment1_vars[i], - moment2_vars[i], beta1_pow_vars[i], beta2_pow_vars[i], - master_param_vars[i], 'epsilon', epsilon, 'beta1', beta1, - 'beta2', beta2, 'multi_precision', multi_precision) + param_vars[i], + grad_vars[i], + lr_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + param_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) else: if in_dygraph_mode(): _, _, _, _, _, _ = _C_ops.merged_adam_( - param_vars, grad_vars, lr_vars, moment1_vars, moment2_vars, - beta1_pow_vars, beta2_pow_vars, master_param_vars, beta1, beta2, - epsilon, multi_precision, False) + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + beta1, + beta2, + epsilon, + multi_precision, + False, + ) else: _, _, _, _, _, _ = _legacy_C_ops.merged_adam( - param_vars, grad_vars, lr_vars, moment1_vars, moment2_vars, - beta1_pow_vars, beta2_pow_vars, master_param_vars, param_vars, - moment1_vars, moment2_vars, beta1_pow_vars, beta2_pow_vars, - master_param_vars, 'epsilon', epsilon, 'beta1', beta1, 'beta2', - beta2, 'multi_precision', multi_precision) + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + param_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) outputs = { 'ParamOut': param_vars, @@ -85,14 +133,13 @@ def run_adam_op(params, 'Moment2Out': moment2_vars, 'Beta1PowOut': beta1_pow_vars, 'Beta2PowOut': beta2_pow_vars, - 'MasterParamOut': master_param_vars + 'MasterParamOut': master_param_vars, } return outputs class TestMergedAdam(unittest.TestCase): - def setUp(self): paddle.disable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -116,27 +163,46 @@ class TestMergedAdam(unittest.TestCase): beta1_pows = self.gen_rand_data([[1], [1], [1], [1]], mp_dtype) beta2_pows = self.gen_rand_data([[1], [1], [1], [1]], mp_dtype) master_params = [p.astype(mp_dtype) for p in params] - return params, grads, lrs, moment1s, moment2s, beta1_pows, beta2_pows, master_params + return ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) def check_with_place(self, place, multi_precision): - params, grads, lrs, moment1s, moment2s, beta1_pows, beta2_pows, master_params = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): - return run_adam_op(params=params, - grads=grads, - lrs=lrs, - moment1s=moment1s, - moment2s=moment2s, - beta1_pows=beta1_pows, - beta2_pows=beta2_pows, - master_params=master_params, - epsilon=0.9, - beta1=0.9, - beta2=0.99, - place=place, - multi_precision=multi_precision, - use_merged=use_merged) + return run_adam_op( + params=params, + grads=grads, + lrs=lrs, + moment1s=moment1s, + moment2s=moment2s, + beta1_pows=beta1_pows, + beta2_pows=beta2_pows, + master_params=master_params, + epsilon=0.9, + beta1=0.9, + beta2=0.99, + place=place, + multi_precision=multi_precision, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -149,10 +215,9 @@ class TestMergedAdam(unittest.TestCase): if place == 'mlu': np.testing.assert_array_equal(value1[i], value2[i]) else: - np.testing.assert_allclose(value1[i], - value2[i], - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + value1[i], value2[i], rtol=1e-05, atol=1e-07 + ) def test_main(self): for multi_precision in [False, True]: diff --git a/python/paddle/fluid/tests/unittests/mlu/test_merged_momentum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_merged_momentum_op_mlu.py index 7a1f590cf3d..e7c4c4bb4ca 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_merged_momentum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_merged_momentum_op_mlu.py @@ -22,16 +22,18 @@ from paddle.fluid.layer_helper import LayerHelper from collections import OrderedDict -def run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False): +def run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -48,48 +50,70 @@ def run_momentum_op(params, } param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -99,8 +123,9 @@ def run_momentum_op(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -111,10 +136,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars[i] outputs['MasterParamOut'] = master_param_vars[i] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -126,10 +150,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -137,17 +160,19 @@ def run_momentum_op(params, return exe.run(main, feed=feed_dict, fetch_list=fetch_list) -def run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False, - use_nesterov=True): +def run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, + use_nesterov=True, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -159,48 +184,70 @@ def run_momentum_op2(params, helper = LayerHelper(op_type, **locals()) param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -210,8 +257,9 @@ def run_momentum_op2(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -230,10 +278,9 @@ def run_momentum_op2(params, 'regularization_method': 'l2_decay', 'regularization_coeff': 2.0, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -246,22 +293,18 @@ def run_momentum_op2(params, inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars attrs = { - 'mu': - mu, - 'multi_precision': - multi_precision, - 'rescale_grad': - rescale_grad, - 'use_nesterov': - use_nesterov, - 'regularization_method': - ['l2_decay' for i in range(len(param_vars))], + 'mu': mu, + 'multi_precision': multi_precision, + 'rescale_grad': rescale_grad, + 'use_nesterov': use_nesterov, + 'regularization_method': [ + 'l2_decay' for i in range(len(param_vars)) + ], 'regularization_coeff': [2.0 for i in range(len(param_vars))], } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -270,7 +313,6 @@ def run_momentum_op2(params, class TestMergedMomentum(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -296,21 +338,28 @@ class TestMergedMomentum(unittest.TestCase): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): # MLU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged) + return run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -323,7 +372,6 @@ class TestMergedMomentum(unittest.TestCase): class TestMergedMomentum2(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -349,22 +397,29 @@ class TestMergedMomentum2(unittest.TestCase): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_nesterov, use_merged): # MLU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged, - use_nesterov=use_nesterov) + return run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + use_nesterov=use_nesterov, + ) outs1 = run_op(use_nesterov=True, use_merged=True) outs2 = run_op(use_nesterov=True, use_merged=False) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_meshgrid_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_meshgrid_op_mlu.py index d0c59692265..6353ca638a5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_meshgrid_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_meshgrid_op_mlu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestMeshgridOp(OpTest): - def setUp(self): self.op_type = "meshgrid" self.place = paddle.device.MLUPlace(0) @@ -50,7 +49,7 @@ class TestMeshgridOp(OpTest): ins = [] outs = [] for i in range(len(self.shape)): - ins.append(np.random.random((self.shape[i], )).astype(self.dtype)) + ins.append(np.random.random((self.shape[i],)).astype(self.dtype)) for i in range(len(self.shape)): out_reshape = [1] * len(self.shape) @@ -64,23 +63,29 @@ class TestMeshgridOp(OpTest): class TestMeshgridOp2(TestMeshgridOp): - def get_x_shape(self): return [100, 300] class TestMeshgridOp3(unittest.TestCase): - def test_api(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, + 100, + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -89,28 +94,34 @@ class TestMeshgridOp3(unittest.TestCase): exe = fluid.Executor(place=fluid.MLUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid(x, y) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp4(unittest.TestCase): - def test_list_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -119,29 +130,35 @@ class TestMeshgridOp4(unittest.TestCase): exe = fluid.Executor(place=fluid.MLUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid([x, y]) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp5(unittest.TestCase): - def test_tuple_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -150,26 +167,32 @@ class TestMeshgridOp5(unittest.TestCase): exe = fluid.Executor(place=fluid.MLUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid((x, y)) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp7(unittest.TestCase): - def test_api_with_dygraph_list_input(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, + 100, + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) @@ -181,14 +204,21 @@ class TestMeshgridOp7(unittest.TestCase): class TestMeshgridOp8(unittest.TestCase): - def test_api_with_dygraph_tuple_input(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_momentum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_momentum_op_mlu.py index 542675ada9f..8efc129b688 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_momentum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_momentum_op_mlu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestMomentumOp1(OpTest): - def setUp(self): self.op_type = "momentum" self.dtype = np.float32 @@ -47,7 +46,7 @@ class TestMomentumOp1(OpTest): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu} @@ -58,7 +57,8 @@ class TestMomentumOp1(OpTest): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -74,7 +74,6 @@ class TestMomentumOp1(OpTest): class TestMomentumOpFp16(TestMomentumOp1): - def init_dtype(self): self.dtype = np.float16 @@ -83,8 +82,7 @@ class TestMomentumOpFp16(TestMomentumOp1): class TestMomentumOp2(OpTest): - '''Test Momentum with default values for attributes - ''' + '''Test Momentum with default values for attributes''' def setUp(self): self.op_type = "momentum" @@ -102,7 +100,7 @@ class TestMomentumOp2(OpTest): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu, 'use_nesterov': use_nesterov} @@ -113,7 +111,8 @@ class TestMomentumOp2(OpTest): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -122,16 +121,15 @@ class TestMomentumOp2(OpTest): class TestMomentumV2(unittest.TestCase): - def test_momentum_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - momentum=0.9, - parameters=linear.parameters()) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, momentum=0.9, parameters=linear.parameters() + ) out = linear(a) out.backward() adam.step() @@ -148,13 +146,15 @@ class TestMomentumV2(unittest.TestCase): cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - rms_optimizer = paddle.optimizer.Momentum(learning_rate=0.1, - momentum=0.9) + rms_optimizer = paddle.optimizer.Momentum( + learning_rate=0.1, momentum=0.9 + ) rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -162,14 +162,13 @@ class TestMomentumV2(unittest.TestCase): exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list) def test_raise_error(self): - self.assertRaises(ValueError, - paddle.optimizer.Momentum, - learning_rate=None) + self.assertRaises( + ValueError, paddle.optimizer.Momentum, learning_rate=None + ) self.assertRaises(ValueError, paddle.optimizer.Momentum, momentum=None) class TestMomentumOpWithDecay(OpTest): - def setUp(self): self.op_type = "momentum" self.place = paddle.device.MLUPlace(0) @@ -193,14 +192,14 @@ class TestMomentumOpWithDecay(OpTest): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = { 'mu': mu, 'use_nesterov': use_nesterov, 'regularization_method': regularization_method, - 'regularization_coeff': regularization_coeff + 'regularization_coeff': regularization_coeff, } grad = grad + regularization_coeff * param @@ -211,7 +210,8 @@ class TestMomentumOpWithDecay(OpTest): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -224,7 +224,6 @@ class TestMomentumOpWithDecay(OpTest): class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): - def init_config(self): self.dtype = np.float16 @@ -233,13 +232,11 @@ class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): class TestMomentumOpWithDecay2(TestMomentumOpWithDecay): - def init_config(self): self.use_nesterov = False class TestMomentumOpWithDecayAPI(unittest.TestCase): - def _test_momentum_dygraph_common(self, regularization): paddle.disable_static() inp = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -252,13 +249,16 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): learning_rate=0.01, momentum=0.9, parameter_list=linear.parameters(), - regularization=regularization) + regularization=regularization, + ) momentum.minimize(loss) def test_momentum_dygraph_1(self): self._test_momentum_dygraph_common( regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ) + ) def test_momentum_static(self): paddle.enable_static() @@ -272,12 +272,14 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): avg_cost = paddle.mean(cost) momentum_optimizer = paddle.fluid.contrib.optimizer.Momentum( - learning_rate=0.1, momentum=0.9) + learning_rate=0.1, momentum=0.9 + ) momentum_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -286,23 +288,23 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): class TestFusedMomentumWithDecayAPI(unittest.TestCase): - def get_program(self, weight_attr, bias_attr=False): main_program = paddle.static.Program() startup_program = paddle.static.Program() - with paddle.static.program_guard(main_program=main_program, - startup_program=startup_program): + with paddle.static.program_guard( + main_program=main_program, startup_program=startup_program + ): x = paddle.static.data(name='x', shape=[10, 10]) - linear = paddle.nn.Linear(10, - 10, - weight_attr=weight_attr, - bias_attr=bias_attr) + linear = paddle.nn.Linear( + 10, 10, weight_attr=weight_attr, bias_attr=bias_attr + ) out = linear(x) loss = paddle.mean(out) optimizer = paddle.optimizer.Momentum( learning_rate=0.01, momentum=0.9, - weight_decay=paddle.regularizer.L2Decay(0.5)) + weight_decay=paddle.regularizer.L2Decay(0.5), + ) optimizer.minimize(loss) return main_program @@ -311,7 +313,8 @@ class TestFusedMomentumWithDecayAPI(unittest.TestCase): weight_attr = paddle.ParamAttr( name="weight", initializer=paddle.nn.initializer.Constant(value=0.5), - regularizer=paddle.regularizer.L2Decay(0.1)) + regularizer=paddle.regularizer.L2Decay(0.1), + ) program = self.get_program(weight_attr, bias_attr=False) ops = program.global_block().ops @@ -326,11 +329,13 @@ class TestFusedMomentumWithDecayAPI(unittest.TestCase): weight_attr = paddle.ParamAttr( name="weight", initializer=paddle.nn.initializer.Constant(value=0.5), - regularizer=paddle.regularizer.L1Decay(0.1)) + regularizer=paddle.regularizer.L1Decay(0.1), + ) bias_attr = paddle.ParamAttr( name="bias", - initializer=paddle.nn.initializer.Constant(value=0.), - regularizer=None) + initializer=paddle.nn.initializer.Constant(value=0.0), + regularizer=None, + ) program = self.get_program(weight_attr, bias_attr) ops = program.global_block().ops @@ -345,8 +350,9 @@ class TestFusedMomentumWithDecayAPI(unittest.TestCase): self.assertEqual(ops[-1].attr('regularization_coeff'), 0) if 'bias' in ops[-2].input('Param'): self.assertEqual(ops[-2].attr('regularization_method'), 'l2_decay') - self.assertEqual(ops[-2].attr('regularization_coeff'), - np.float32(0.5)) + self.assertEqual( + ops[-2].attr('regularization_coeff'), np.float32(0.5) + ) def test_param_has_no_regularizer(self): paddle.enable_static() @@ -360,11 +366,11 @@ class TestFusedMomentumWithDecayAPI(unittest.TestCase): class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): - def __update_params(self, momentum, linear): for i in range(10): - inp = paddle.full(shape=[2, 2], fill_value=i, - dtype='float32').astype("float32") + inp = paddle.full( + shape=[2, 2], fill_value=i, dtype='float32' + ).astype("float32") inp = paddle.to_tensor(inp) out = linear(inp) loss = paddle.mean(out) @@ -379,32 +385,39 @@ class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_old = paddle.fluid.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_old.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_old, linear=linear_old) linear_new = paddle.nn.Linear( 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_new = paddle.fluid.contrib.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_new.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_new, linear=linear_new) self.assertEqual( (linear_old.weight.numpy() == linear_new.weight.numpy()).all(), True, - 'the param weight updated by two Momentum optimizers should equal') + 'the param weight updated by two Momentum optimizers should equal', + ) def test_vs(self, place=fluid.MLUPlace(0)): places = [fluid.MLUPlace(0)] @@ -413,7 +426,6 @@ class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): class TestMomentumV2Group(TestMomentumV2): - def test_momentum_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -421,22 +433,20 @@ class TestMomentumV2Group(TestMomentumV2): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - 'learning_rate': - 0.1, - 'momentum': - 0.99 - }], - weight_decay=0.1, - momentum=0.9) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + }, + ], + weight_decay=0.1, + momentum=0.9, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -445,13 +455,14 @@ class TestMomentumV2Group(TestMomentumV2): class TestMultiTensorMomentumDygraph(unittest.TestCase): - - def _momentum_optimize_dygraph(self, - place, - use_param_attr=False, - use_param_group=False, - use_amp=False, - use_multi_tensor=False): + def _momentum_optimize_dygraph( + self, + place, + use_param_attr=False, + use_param_group=False, + use_amp=False, + use_multi_tensor=False, + ): paddle.disable_static() paddle.seed(10) paddle.set_device(place) @@ -459,7 +470,8 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): weight_attr = paddle.ParamAttr( learning_rate=0.5, regularizer=paddle.regularizer.L2Decay(1.0), - trainable=True) + trainable=True, + ) if use_param_attr: model = paddle.nn.Linear(5, 5, weight_attr) else: @@ -468,17 +480,21 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): optimizer = paddle.optimizer.Momentum( parameters=model.parameters(), use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + multi_precision=use_amp, + ) else: optimizer = paddle.optimizer.Momentum( - parameters=[{ - 'params': model.parameters(), - 'weight_decay': 0.001, - 'learning_rate': 0.1, - 'momentum': 0.99 - }], + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + } + ], use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + multi_precision=use_amp, + ) for idx in range(5): if place == 'mlu' and use_amp == True: model = paddle.amp.decorate(models=model, level='O2') @@ -506,9 +522,11 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): def _check_with_place_amp(self, place, use_amp): output1, params1 = self._momentum_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=True) + place=place, use_amp=use_amp, use_multi_tensor=True + ) output2, params2 = self._momentum_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=False) + place=place, use_amp=use_amp, use_multi_tensor=False + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -518,12 +536,14 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): place=place, use_amp=use_amp, use_param_attr=True, - use_multi_tensor=True) + use_multi_tensor=True, + ) output2, params2 = self._momentum_optimize_dygraph( place=place, use_amp=use_amp, use_param_attr=True, - use_multi_tensor=False) + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -533,12 +553,14 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): place=place, use_amp=use_amp, use_param_group=True, - use_multi_tensor=True) + use_multi_tensor=True, + ) output2, params2 = self._momentum_optimize_dygraph( place=place, use_amp=use_amp, use_param_group=True, - use_multi_tensor=False) + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -554,11 +576,9 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): class TestMultiTensorMomentumStatic(unittest.TestCase): - - def _momentum_optimize_static(self, - place, - use_amp=False, - use_multi_tensor=False): + def _momentum_optimize_static( + self, place, use_amp=False, use_multi_tensor=False + ): paddle.enable_static() paddle.seed(10) np.random.seed(10) @@ -567,24 +587,26 @@ class TestMultiTensorMomentumStatic(unittest.TestCase): exe = paddle.static.Executor(place=paddle.device.MLUPlace(0)) train_program = paddle.static.Program() startup_program = paddle.static.Program() - optimizer = paddle.optimizer.Momentum(multi_precision=use_amp, - use_multi_tensor=use_multi_tensor) + optimizer = paddle.optimizer.Momentum( + multi_precision=use_amp, use_multi_tensor=use_multi_tensor + ) if use_amp: optimizer = paddle.static.amp.decorate( optimizer, init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if use_amp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -596,9 +618,9 @@ class TestMultiTensorMomentumStatic(unittest.TestCase): x = numpy.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -607,12 +629,12 @@ class TestMultiTensorMomentumStatic(unittest.TestCase): return places def _check_with_place_amp(self, place, use_amp): - output1 = self._momentum_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=True) - output2 = self._momentum_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=False) + output1 = self._momentum_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=True + ) + output2 = self._momentum_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=False + ) for idx in range(len(output1)): np.testing.assert_allclose(output1[idx], output2[idx], rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_nearest_interp_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_nearest_interp_v2_op_mlu.py index 6cf320d56b1..1ab286740ef 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_nearest_interp_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_nearest_interp_v2_op_mlu.py @@ -27,15 +27,17 @@ from paddle.nn.functional import interpolate paddle.enable_static() -def nearest_neighbor_interp_np(X, - out_h, - out_w, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp_np( + X, + out_h, + out_w, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -48,16 +50,16 @@ def nearest_neighbor_interp_np(X, n, c, in_h, in_w = X.shape ratio_h = ratio_w = 0.0 - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -85,17 +87,19 @@ def nearest_neighbor_interp_np(X, return out.astype(X.dtype) -def nearest_neighbor_interp3d_np(X, - out_d, - out_h, - out_w, - scale_d=0, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp3d_np( + X, + out_d, + out_h, + out_w, + scale_d=0, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -110,24 +114,24 @@ def nearest_neighbor_interp3d_np(X, n, c, in_d, in_h, in_w = X.shape ratio_d = ratio_h = ratio_w = 0.0 - if (out_d > 1): - if (align_corners): + if out_d > 1: + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: if scale_d > 0: ratio_d = 1.0 / scale_d else: ratio_d = 1.0 * in_d / out_d - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -159,7 +163,6 @@ def nearest_neighbor_interp3d_np(X, class TestNearestInterpOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -216,15 +219,30 @@ class TestNearestInterpOp(OpTest): if len(self.input_shape) == 4: output_np = nearest_neighbor_interp_np( - input_np, out_h, out_w, scale_h, scale_w, self.out_size, - self.actual_shape, self.align_corners, self.data_layout) + input_np, + out_h, + out_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) elif len(self.input_shape) == 5: - output_np = nearest_neighbor_interp3d_np(input_np, out_d, out_h, - out_w, scale_d, scale_h, - scale_w, self.out_size, - self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp3d_np( + input_np, + out_d, + out_h, + out_w, + scale_d, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -237,7 +255,7 @@ class TestNearestInterpOp(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } else: self.attrs = { @@ -245,7 +263,7 @@ class TestNearestInterpOp(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -267,7 +285,7 @@ class TestNearestInterpOp(OpTest): self.input_shape = [2, 3, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True @@ -285,119 +303,108 @@ class TestNearestInterpOp(OpTest): class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") self.align_corners = True class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 4, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 8]).astype("int32") self.align_corners = True self.data_layout = "NHWC" class TestNearestInterpWithoutCorners(TestNearestInterpOp): - def set_align_corners(self): self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -409,7 +416,6 @@ class TestNearestNeighborInterpScale2(TestNearestInterpOp): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] @@ -421,7 +427,6 @@ class TestNearestNeighborInterpScale3(TestNearestInterpOp): class TestNearestInterpOp_attr_tensor(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -461,8 +466,9 @@ class TestNearestInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -474,9 +480,16 @@ class TestNearestInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -490,33 +503,31 @@ class TestNearestInterpOp_attr_tensor(OpTest): self.input_shape = [2, 5, 4, 4] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True @@ -524,7 +535,6 @@ class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -537,7 +547,6 @@ class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): class TestNearestAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 6, 3], dtype="float32") @@ -545,27 +554,25 @@ class TestNearestAPI(unittest.TestCase): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = fluid.layers.resize_nearest(y, - out_shape=[12, 12], - data_format='NHWC', - align_corners=False) - out2 = fluid.layers.resize_nearest(x, - out_shape=[12, dim], - align_corners=False) - out3 = fluid.layers.resize_nearest(x, - out_shape=shape_tensor, - align_corners=False) - out4 = fluid.layers.resize_nearest(x, - out_shape=[4, 4], - actual_shape=actual_size, - align_corners=False) - out5 = fluid.layers.resize_nearest(x, - scale=scale_tensor, - align_corners=False) + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = fluid.layers.resize_nearest( + y, out_shape=[12, 12], data_format='NHWC', align_corners=False + ) + out2 = fluid.layers.resize_nearest( + x, out_shape=[12, dim], align_corners=False + ) + out3 = fluid.layers.resize_nearest( + x, out_shape=shape_tensor, align_corners=False + ) + out4 = fluid.layers.resize_nearest( + x, out_shape=[4, 4], actual_shape=actual_size, align_corners=False + ) + out5 = fluid.layers.resize_nearest( + x, scale=scale_tensor, align_corners=False + ) x_data = np.random.random((2, 3, 6, 6)).astype("float32") dim_data = np.array([12]).astype("int32") @@ -576,39 +583,41 @@ class TestNearestAPI(unittest.TestCase): place = paddle.MLUPlace(0) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = nearest_neighbor_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 1))) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = nearest_neighbor_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 1)) + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res) class TestNearestInterpException(unittest.TestCase): - def test_exception(self): import paddle + input = fluid.data(name="input", shape=[1, 3, 6, 6], dtype="float32") def attr_data_format(): # for 4-D input, data_format can only be NCHW or NHWC - out = fluid.layers.resize_nearest(input, - out_shape=[4, 8], - data_format='NDHWC') + out = fluid.layers.resize_nearest( + input, out_shape=[4, 8], data_format='NDHWC' + ) def attr_scale_type(): out = fluid.layers.resize_nearest(input, scale='scale') @@ -622,9 +631,9 @@ class TestNearestInterpException(unittest.TestCase): def mode_error(): x = paddle.randn([1, 3]) - out = paddle.nn.functional.interpolate(x, - scale_factor='scale', - mode="BILINEAR") + out = paddle.nn.functional.interpolate( + x, scale_factor='scale', mode="BILINEAR" + ) self.assertRaises(ValueError, attr_data_format) self.assertRaises(TypeError, attr_scale_type) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_one_hot_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_one_hot_v2_op_mlu.py index cc6db215713..3605ab25416 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_one_hot_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_one_hot_v2_op_mlu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestOneHotOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -55,7 +54,6 @@ class TestOneHotOp(OpTest): class TestOneHotOp_attr(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -66,8 +64,9 @@ class TestOneHotOp_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -83,7 +82,6 @@ class TestOneHotOp_attr(OpTest): class TestOneHotOp_default_dtype(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -109,7 +107,6 @@ class TestOneHotOp_default_dtype(OpTest): class TestOneHotOp_default_dtype_attr(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -120,8 +117,9 @@ class TestOneHotOp_default_dtype_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -135,7 +133,6 @@ class TestOneHotOp_default_dtype_attr(OpTest): class TestOneHotOp_exception(unittest.TestCase): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -153,30 +150,34 @@ class TestOneHotOp_exception(unittest.TestCase): def test_check_output(self): program = Program() with program_guard(program): - x = fluid.layers.data(name='x', - shape=[self.dimension], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='x', shape=[self.dimension], dtype='float32', lod_level=1 + ) block = program.current_block() - one_hot_out = block.create_var(name="one_hot_out", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype='float32') - block.append_op(type='one_hot', - inputs={'X': x}, - attrs={'depth': self.depth}, - outputs={'Out': one_hot_out}) + one_hot_out = block.create_var( + name="one_hot_out", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype='float32', + ) + block.append_op( + type='one_hot', + inputs={'X': x}, + attrs={'depth': self.depth}, + outputs={'Out': one_hot_out}, + ) exe = fluid.Executor(self.place) def run(): - exe.run(feed={'x': self.x}, - fetch_list=[one_hot_out], - return_numpy=False) + exe.run( + feed={'x': self.x}, + fetch_list=[one_hot_out], + return_numpy=False, + ) self.assertRaises(ValueError, run) class TestOneHotOpApi(unittest.TestCase): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -191,14 +192,17 @@ class TestOneHotOpApi(unittest.TestCase): def test_api_with_dygraph(self): depth = 10 - label = np.array([np.random.randint(0, depth - 1) - for i in range(6)]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, depth - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(): one_hot_label = fluid.one_hot( - input=fluid.dygraph.to_variable(label), depth=depth) + input=fluid.dygraph.to_variable(label), depth=depth + ) one_hot_label = paddle.nn.functional.one_hot( - fluid.dygraph.to_variable(label), depth) + fluid.dygraph.to_variable(label), depth + ) # with _test_eager_guard(): # one_hot_label = paddle.nn.functional.one_hot( # paddle.to_tensor(label), depth) @@ -207,20 +211,22 @@ class TestOneHotOpApi(unittest.TestCase): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = fluid.one_hot(input=label, depth=depth) - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) class BadInputTestOnehotV2(unittest.TestCase): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -229,10 +235,12 @@ class BadInputTestOnehotV2(unittest.TestCase): with fluid.program_guard(fluid.Program()): def test_bad_x(): - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="float32") + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="float32", + ) one_hot_label = fluid.one_hot(input=label, depth=4) self.assertRaises(TypeError, test_bad_x) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_parallel_dygraph_sync_batch_norm_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_parallel_dygraph_sync_batch_norm_mlu.py index b42567c5d0c..4d39cd6cdd7 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_parallel_dygraph_sync_batch_norm_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_parallel_dygraph_sync_batch_norm_mlu.py @@ -31,7 +31,6 @@ print("file: {}".format(flag_name)) class TestParallelDygraphMnistMLU(TestDistBase): - def _setup_config(self): self._sync_mode = False self._cncl_mode = True @@ -49,7 +48,7 @@ class TestParallelDygraphMnistMLU(TestDistBase): "FLAGS_call_stack_level": "2", "GLOG_v": "2", "PADDLE_WITH_GLOO": '0', - "BACKEND": "cncl" + "BACKEND": "cncl", } if check_error_log: @@ -60,14 +59,16 @@ class TestParallelDygraphMnistMLU(TestDistBase): required_envs.update(need_envs) return required_envs - def _run_local(self, - model, - envs, - check_error_log=False, - batch_size=DEFAULT_BATCH_SIZE, - batch_merge_repeat=1, - log_name="", - devices="1"): + def _run_local( + self, + model, + envs, + check_error_log=False, + batch_size=DEFAULT_BATCH_SIZE, + batch_merge_repeat=1, + log_name="", + devices="1", + ): cmd = self._python_interp @@ -75,8 +76,10 @@ class TestParallelDygraphMnistMLU(TestDistBase): envs['COVERAGE_FILE'] = os.getenv('COVERAGE_FILE', '') cmd += " -m coverage run --branch -p" - cmd += " %s --role trainer --update_method local --lr %f" % (model, - self._lr) + cmd += " %s --role trainer --update_method local --lr %f" % ( + model, + self._lr, + ) if batch_size != DEFAULT_BATCH_SIZE: cmd += " --batch_size %d" % batch_size @@ -90,7 +93,7 @@ class TestParallelDygraphMnistMLU(TestDistBase): env_local = { "FLAGS_selected_mlus": devices, "PADDLE_TRAINERS_NUM": "1", - "PADDLE_TRAINER_ID": "0" + "PADDLE_TRAINER_ID": "0", } else: env_local = {'CPU_NUM': '1'} @@ -111,30 +114,36 @@ class TestParallelDygraphMnistMLU(TestDistBase): if check_error_log: path = "/tmp/local_err_%d.log" % os.getpid() err_log = open(path, "w") - local_proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=err_log, - env=env_local) + local_proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=err_log, + env=env_local, + ) else: - local_proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env_local) + local_proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env_local, + ) local_out, local_err = local_proc.communicate() if check_error_log: err_log.close() sys.stderr.write( - '\n--run_local-- trainer 0 stderr file saved in: %s\n' % (path)) + '\n--run_local-- trainer 0 stderr file saved in: %s\n' % (path) + ) sys.stderr.write('local_stderr: %s\n' % local_err) sys.stderr.write('local_stdout: %s\n' % pickle.loads(local_out)) return pickle.loads(local_out) - def _run_cluster_nccl2(self, model, envs, update_method, check_error_log, - log_name): + def _run_cluster_nccl2( + self, model, envs, update_method, check_error_log, log_name + ): # NOTE: we reuse ps_endpoints as nccl2 worker endpoints worker_endpoints = self._ps_endpoints.split(",") @@ -144,20 +153,28 @@ class TestParallelDygraphMnistMLU(TestDistBase): pipes = [] for i in range(0, trainer_num): tr_cmd, tr_env = self._get_nccl2_trainer_cmd( - model, worker_endpoints[i], update_method, i, trainer_num) + model, worker_endpoints[i], update_method, i, trainer_num + ) tr_env.update(envs) - print("use_hallreduce:{} \ntr{}_cmd:{}, env: {}".format( - self._use_hallreduce, i, tr_cmd, tr_env)) + print( + "use_hallreduce:{} \ntr{}_cmd:{}, env: {}".format( + self._use_hallreduce, i, tr_cmd, tr_env + ) + ) tr_pipe = open("/tmp/tr%d_err_%d.log" % (i, os.getpid()), "w") sys.stderr.write( "\n{} going to start process {} with nccl2\n".format( - type(self).__name__, i)) - tr_proc = subprocess.Popen(tr_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr_pipe, - env=tr_env) + type(self).__name__, i + ) + ) + tr_proc = subprocess.Popen( + tr_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr_pipe, + env=tr_env, + ) procs.append(tr_proc) pipes.append(tr_pipe) @@ -170,7 +187,9 @@ class TestParallelDygraphMnistMLU(TestDistBase): sys.stderr.write('trainer {} stderr: {}\n'.format(i, tr_err)) sys.stderr.write( 'trainer {} glog file saved in: /tmp/tr{}_err_{}.log \n'.format( - i, i, os.getpid())) + i, i, os.getpid() + ) + ) if check_error_log: print("outs[0]:", pickle.loads(outs[0])) @@ -184,7 +203,8 @@ class TestParallelDygraphMnistMLU(TestDistBase): os.path.abspath("parallel_dygraph_sync_batch_norm.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/mlu/test_pool2d_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_pool2d_op_mlu.py index e9add5e6a5c..57202e62a43 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_pool2d_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_pool2d_op_mlu.py @@ -23,30 +23,40 @@ import sys sys.path.append('..') from op_test import OpTest -from test_pool2d_op import pool2D_forward_naive, avg_pool2D_forward_naive, max_pool2D_forward_naive, adaptive_start_index, adaptive_end_index +from test_pool2d_op import ( + pool2D_forward_naive, + avg_pool2D_forward_naive, + max_pool2D_forward_naive, + adaptive_start_index, + adaptive_end_index, +) paddle.enable_static() -def pool2d_backward_navie(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCHW', - pool_type="max", - padding_algorithm="EXPLICIT"): +def pool2d_backward_navie( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCHW', + pool_type="max", + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -56,9 +66,10 @@ def pool2d_backward_navie(x, if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0] @@ -66,7 +77,8 @@ def pool2d_backward_navie(x, raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCHW": @@ -95,10 +107,20 @@ def pool2d_backward_navie(x, if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 - W_out = (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + H_out = ( + (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + ) x_grad = np.zeros_like(x) for i in range(H_out): @@ -126,20 +148,26 @@ def pool2d_backward_navie(x, in_w_end = np.min((in_w_end, W)) if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) - x_grad[:, :, in_h_start:in_h_end, - in_w_start:in_w_end] += 1 / field_size + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) + x_grad[:, :, in_h_start:in_h_end, in_w_start:in_w_end] += ( + 1 / field_size + ) elif pool_type == 'max': for n in range(N): for c in range(C): - idx = np.argmax(x[n, c, in_h_start:in_h_end, - in_w_start:in_w_end].flatten()) + idx = np.argmax( + x[ + n, c, in_h_start:in_h_end, in_w_start:in_w_end + ].flatten() + ) idx_h = idx // (in_w_end - in_w_start) idx_w = idx % (in_w_end - in_w_start) - x_grad[n, c, in_h_start + idx_h, - in_w_start + idx_w] += 1 + x_grad[ + n, c, in_h_start + idx_h, in_w_start + idx_w + ] += 1 if data_format == "NHWC": x_grad = x_grad.transpose([0, 2, 3, 1]) @@ -147,7 +175,6 @@ def pool2d_backward_navie(x, class TestPool2D_Op_Mixin(object): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -165,12 +192,19 @@ class TestPool2D_Op_Mixin(object): self.init_shape() input = np.random.random(self.shape).astype(self.dtype) - output = pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, self.data_format, - self.pool_type, - self.padding_algorithm).astype(self.dtype) + output = pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} self.attrs = { @@ -192,23 +226,27 @@ class TestPool2D_Op_Mixin(object): self.check_output_with_place(self.place) def test_check_grad(self): - x_grad = pool2d_backward_navie(self.inputs["X"], - ksize=self.ksize, - strides=self.strides, - paddings=self.paddings, - global_pool=self.global_pool, - ceil_mode=False, - exclusive=self.exclusive, - adaptive=self.adaptive, - data_format=self.data_format, - pool_type=self.pool_type, - padding_algorithm=self.padding_algorithm) + x_grad = pool2d_backward_navie( + self.inputs["X"], + ksize=self.ksize, + strides=self.strides, + paddings=self.paddings, + global_pool=self.global_pool, + ceil_mode=False, + exclusive=self.exclusive, + adaptive=self.adaptive, + data_format=self.data_format, + pool_type=self.pool_type, + padding_algorithm=self.padding_algorithm, + ) x_grad = x_grad / np.prod(self.outputs['Out'].shape) - self.check_grad_with_place(self.place, - set(['X']), - 'Out', - max_relative_error=0.06, - user_defined_grads=[x_grad]) + self.check_grad_with_place( + self.place, + set(['X']), + 'Out', + max_relative_error=0.06, + user_defined_grads=[x_grad], + ) def init_data_format(self): self.data_format = "NCHW" @@ -249,7 +287,6 @@ class TestPool2D_Op(TestPool2D_Op_Mixin, OpTest): class TestCase1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -269,7 +306,6 @@ class TestCase1(TestPool2D_Op): class TestCase2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -289,30 +325,25 @@ class TestCase2(TestPool2D_Op): class TestCase3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_data_type(self): self.dtype = np.float16 @@ -332,13 +363,11 @@ create_test_fp16_class(TestCase3) create_test_fp16_class(TestCase4) create_test_fp16_class(TestCase5) -#--------------------test pool2d use ceil mode-------------------- +# --------------------test pool2d use ceil mode-------------------- def create_test_use_ceil_class(parent): - class TestPool2DUseCeilCase(parent): - def init_ceil_mode(self): self.ceil_mode = True @@ -352,19 +381,16 @@ create_test_use_ceil_class(TestCase2) class TestAvgInclude(TestCase2): - def init_exclusive(self): self.exclusive = False class TestAvgPoolAdaptive(TestCase1): - def init_adaptive(self): self.adaptive = True class TestAvgPoolAdaptiveAsyOutSize(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -377,11 +403,10 @@ class TestAvgPoolAdaptiveAsyOutSize(TestCase1): self.paddings = [0, 0, 0, 0] -#-------test pool2d with asymmetric padding----- +# -------test pool2d with asymmetric padding----- class TestPool2D_AsyPadding(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -392,7 +417,6 @@ class TestPool2D_AsyPadding(TestPool2D_Op): class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -403,7 +427,6 @@ class TestCase1_AsyPadding(TestCase1): class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -414,7 +437,6 @@ class TestCase2_AsyPadding(TestCase2): class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -425,7 +447,6 @@ class TestCase3_AsyPadding(TestCase3): class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -436,7 +457,6 @@ class TestCase4_AsyPadding(TestCase4): class TestCase5_AsyPadding((TestCase5)): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -451,7 +471,6 @@ create_test_use_ceil_class(TestCase2_AsyPadding) class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -465,7 +484,6 @@ class TestAvgInclude_AsyPadding(TestCase2): class TestAvgPoolAdaptive_AsyPadding(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -478,9 +496,8 @@ class TestAvgPoolAdaptive_AsyPadding(TestCase1): self.shape = [2, 3, 7, 7] -#----------- test channel_last -------------- +# ----------- test channel_last -------------- class TestPool2D_channel_last(TestPool2D_Op): - def init_data_format(self): self.data_format = "NHWC" @@ -489,7 +506,6 @@ class TestPool2D_channel_last(TestPool2D_Op): class TestCase1_channel_last(TestCase1): - def init_data_format(self): self.data_format = "NHWC" @@ -498,7 +514,6 @@ class TestCase1_channel_last(TestCase1): class TestCase2_channel_last(TestCase2): - def init_data_format(self): self.data_format = "NHWC" @@ -507,7 +522,6 @@ class TestCase2_channel_last(TestCase2): class TestCase3_channel_last(TestCase3): - def init_data_format(self): self.data_format = "NHWC" @@ -516,7 +530,6 @@ class TestCase3_channel_last(TestCase3): class TestCase4_channel_last(TestCase4): - def init_data_format(self): self.data_format = "NHWC" @@ -525,7 +538,6 @@ class TestCase4_channel_last(TestCase4): class TestCase5_channel_last(TestCase5): - def init_data_format(self): self.data_format = "NHWC" @@ -538,13 +550,11 @@ create_test_use_ceil_class(TestCase2_channel_last) class TestCase5_Max(TestCase2): - def init_pool_type(self): self.pool_type = "max" class TestCase5_channel_last_Max(TestCase5_Max): - def init_data_format(self): self.data_format = "NHWC" @@ -553,19 +563,16 @@ class TestCase5_channel_last_Max(TestCase5_Max): class TestAvgInclude_channel_last(TestCase2_channel_last): - def init_exclusive(self): self.exclusive = False class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): - def init_adaptive(self): self.adaptive = True class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -574,7 +581,6 @@ class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -583,7 +589,6 @@ class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -592,7 +597,6 @@ class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -601,7 +605,6 @@ class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -610,7 +613,6 @@ class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -623,7 +625,6 @@ create_test_use_ceil_class(TestCase2_AsyPadding_channel_last) class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -635,9 +636,7 @@ class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.paddings = [0, 0] self.padding_algorithm = "SAME" @@ -663,9 +662,7 @@ create_test_padding_SAME_class(TestCase5_channel_last) def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.paddings = [1, 1] self.padding_algorithm = "VALID" @@ -691,7 +688,6 @@ create_test_padding_VALID_class(TestCase5_channel_last) class TestCase1_strides(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 2] @@ -705,96 +701,121 @@ create_test_padding_SAME_class(TestCase1_strides) # ----- test API class TestPool2DAPI(unittest.TestCase): - def test_api(self): x_NHWC = np.random.random([2, 5, 5, 3]).astype("float32") x_NCHW = np.random.random([2, 3, 5, 5]).astype("float32") - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") - - input_NHWC_negetive = fluid.layers.data(name="input_NHWC_negetive", - shape=[2, -1, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW_negetive = fluid.layers.data(name="input_NCHW_negetive", - shape=[2, 3, -1, -1], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) + + input_NHWC_negetive = fluid.layers.data( + name="input_NHWC_negetive", + shape=[2, -1, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW_negetive = fluid.layers.data( + name="input_NCHW_negetive", + shape=[2, 3, -1, -1], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3] - out_1 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - data_format="NHWC") - - out_2 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - data_format="NHWC") - - out_3 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [0, 0], [1, 1], - [1, 1]], - data_format="NCHW") - - out_4 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[1, 2, 1, 0], - data_format="NCHW") + out_1 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + data_format="NHWC", + ) + + out_2 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + data_format="NHWC", + ) + + out_3 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [0, 0], [1, 1], [1, 1]], + data_format="NCHW", + ) + + out_4 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[1, 2, 1, 0], + data_format="NCHW", + ) # test VALID - out_5 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding="VALID", - data_format="NCHW") - - out_6 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - data_format="NHWC") + out_5 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding="VALID", + data_format="NCHW", + ) + + out_6 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + data_format="NHWC", + ) # test SAME - out_7 = fluid.layers.pool2d(input=input_NCHW, - pool_size=[4, 4], - pool_type="avg", - pool_padding="SAME", - data_format="NCHW") - - out_8 = fluid.layers.pool2d(input=input_NHWC, - pool_size=[4, 4], - pool_type="max", - pool_padding="SAME", - data_format="NHWC") + out_7 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=[4, 4], + pool_type="avg", + pool_padding="SAME", + data_format="NCHW", + ) + + out_8 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=[4, 4], + pool_type="max", + pool_padding="SAME", + data_format="NHWC", + ) # test negetive - out_9 = fluid.layers.pool2d(input=input_NHWC_negetive, - pool_size=ksize, - pool_type="avg", - pool_padding=[0, 0], - data_format="NHWC") + out_9 = fluid.layers.pool2d( + input=input_NHWC_negetive, + pool_size=ksize, + pool_type="avg", + pool_padding=[0, 0], + data_format="NHWC", + ) assert out_9.shape == (2, -1, 3, 3) - out_10 = fluid.layers.pool2d(input=input_NCHW_negetive, - pool_size=ksize, - pool_type="avg", - pool_padding=[0, 0], - data_format="NCHW") + out_10 = fluid.layers.pool2d( + input=input_NCHW_negetive, + pool_size=ksize, + pool_type="avg", + pool_padding=[0, 0], + data_format="NCHW", + ) assert out_10.shape == (2, 3, -1, -1) exe = fluid.Executor(place=fluid.MLUPlace(0)) @@ -804,46 +825,61 @@ class TestPool2DAPI(unittest.TestCase): "input_NHWC": x_NHWC, "input_NCHW": x_NCHW, "input_NHWC_negetive": x_NHWC, - "input_NCHW_negetive": x_NCHW + "input_NCHW_negetive": x_NCHW, }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.allclose( res_1, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1], - paddings=[1, 1], - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1], + paddings=[1, 1], + data_format="NHWC", + ), + ) assert np.allclose( res_2, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 1, 1, 1], - data_format="NHWC")) - assert np.allclose(res_3, - pool2D_forward_naive(x=x_NCHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 1, 1, 1], - data_format="NCHW"), - rtol=0.07, - atol=1e-05) - - assert np.allclose(res_4, - pool2D_forward_naive(x=x_NCHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 2, 1, 0], - data_format="NCHW"), - rtol=0.07, - atol=1e-05) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 1, 1, 1], + data_format="NHWC", + ), + ) + assert np.allclose( + res_3, + pool2D_forward_naive( + x=x_NCHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 1, 1, 1], + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) + + assert np.allclose( + res_4, + pool2D_forward_naive( + x=x_NCHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 2, 1, 0], + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) # VALID assert np.allclose( @@ -855,175 +891,215 @@ class TestPool2DAPI(unittest.TestCase): strides=[1, 1], paddings=[10, 20], # any ele is ok padding_algorithm="VALID", - data_format="NCHW"), + data_format="NCHW", + ), rtol=0.07, - atol=1e-05) + atol=1e-05, + ) assert np.allclose( res_6, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="VALID", - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="VALID", + data_format="NHWC", + ), + ) # SAME - assert np.allclose(res_7, - pool2D_forward_naive(x=x_NCHW, - ksize=[4, 4], - pool_type="avg", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NCHW"), - rtol=0.07, - atol=1e-05) + assert np.allclose( + res_7, + pool2D_forward_naive( + x=x_NCHW, + ksize=[4, 4], + pool_type="avg", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) assert np.allclose( res_8, - pool2D_forward_naive(x=x_NHWC, - ksize=[4, 4], - pool_type="max", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=[4, 4], + pool_type="max", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NHWC", + ), + ) class TestPool2DAPI_Error(unittest.TestCase): - def test_api(self): - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3] # data_format value error def run_2(): - out_2 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - data_format="NHWCC") + out_2 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + data_format="NHWCC", + ) self.assertRaises(ValueError, run_2) # padding str value error def run_3(): - out_3 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALIDSAME", - data_format="NHWC") + out_3 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALIDSAME", + data_format="NHWC", + ) self.assertRaises(ValueError, run_3) # padding str valid and ceil_mode value error def run_4(): - out_4 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - ceil_mode=True, - data_format="NHWC") + out_4 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + ceil_mode=True, + data_format="NHWC", + ) self.assertRaises(ValueError, run_4) # padding with 8 ele. value error def run_5(): - out_5 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[[1, 1], [0, 0], [0, 0], - [1, 1]], - data_format="NHWC") + out_5 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[[1, 1], [0, 0], [0, 0], [1, 1]], + data_format="NHWC", + ) self.assertRaises(ValueError, run_5) class TestDygraphPool2DAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of Pool2D must be Variable. data1 = np.random.random((3, 32, 32, 5)).astype('float32') - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - global_pooling=False) + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False, + ) self.assertRaises(TypeError, pool2d, data1) # the input dtype of mlu Pool2D must be float16 or float32 - data2 = fluid.layers.data(name='x1', - shape=[3, 32, 32, 5], - dtype="int32") + data2 = fluid.layers.data( + name='x1', shape=[3, 32, 32, 5], dtype="int32" + ) self.assertRaises(TypeError, pool2d, data2) def test_data_format_error(self): with program_guard(Program(), Program()): # the data_format must be 'NCHW' or 'NHWC' data1 = np.random.random((3, 32, 32, 5)).astype('float32') - self.assertRaises(ValueError, - fluid.dygraph.Pool2D, - pool_size=2, - pool_type='max', - pool_stride=1, - global_pooling=False, - data_format='NWHC') + self.assertRaises( + ValueError, + fluid.dygraph.Pool2D, + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False, + data_format='NWHC', + ) class TestDygraphPool2DAPI(unittest.TestCase): - def test_nhwc(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='NHWC') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='NHWC', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2) def test_lower_case(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='nhwc') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='nhwc', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2) def test_upper_case(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='MAX', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='nhwc') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='MAX', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='nhwc', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_prior_box_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_prior_box_op_mlu.py index 2e2b96c9e77..8f8473cceab 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_prior_box_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_prior_box_op_mlu.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import unittest import sys @@ -28,7 +26,6 @@ paddle.enable_static() class TestMLUPriorBox(OpTest): - def setUp(self): self.op_type = "prior_box" self.set_mlu() @@ -60,7 +57,7 @@ class TestMLUPriorBox(OpTest): 'min_max_aspect_ratios_order': self.min_max_aspect_ratios_order, 'step_w': self.step_w, 'step_h': self.step_h, - 'offset': self.offset + 'offset': self.offset, } if len(self.max_sizes) > 0: self.attrs['max_sizes'] = self.max_sizes @@ -95,8 +92,9 @@ class TestMLUPriorBox(OpTest): self.flip = True self.set_min_max_aspect_ratios_order() self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] - self.aspect_ratios = np.array(self.aspect_ratios, - dtype=np.float64).flatten() + self.aspect_ratios = np.array( + self.aspect_ratios, dtype=np.float64 + ).flatten() self.variances = [0.1, 0.1, 0.2, 0.2] self.variances = np.array(self.variances, dtype=np.float64).flatten() @@ -108,12 +106,12 @@ class TestMLUPriorBox(OpTest): def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype('float32') + (self.batch_size, self.image_channels, self.image_w, self.image_h) + ).astype('float32') self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype('float32') + (self.batch_size, self.input_channels, self.layer_w, self.layer_h) + ).astype('float32') def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -134,73 +132,78 @@ class TestMLUPriorBox(OpTest): ar = self.real_aspect_ratios[r] c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 else: - c_w = c_h = min_size / 2. - out_boxes[h, w, idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + c_w = c_h = min_size / 2.0 + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # rest of priors for r in range(len(self.real_aspect_ratios)): ar = self.real_aspect_ratios[r] - if abs(ar - 1.) < 1e-6: + if abs(ar - 1.0) < 1e-6: continue c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) # set the variance. - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype('float32') self.out_var = out_var.astype('float32') class TestMLUPriorBoxWithoutMaxSize(TestMLUPriorBox): - def set_max_sizes(self): self.max_sizes = [] class TestMLUPriorBoxWithoutSpecifiedOutOrder(TestMLUPriorBox): - def set_min_max_aspect_ratios_order(self): self.min_max_aspect_ratios_order = False diff --git a/python/paddle/fluid/tests/unittests/mlu/test_randperm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_randperm_op_mlu.py index 5412e6c4a7b..3ee3144905e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_randperm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_randperm_op_mlu.py @@ -27,31 +27,38 @@ paddle.enable_static() def check_randperm_out(n, data_np): - assert isinstance(data_np, np.ndarray), \ - "The input data_np should be np.ndarray." + assert isinstance( + data_np, np.ndarray + ), "The input data_np should be np.ndarray." gt_sorted = np.arange(n) out_sorted = np.sort(data_np) return list(gt_sorted == out_sorted) def error_msg(data_np): - return "The sorted ground truth and sorted out should " + \ - "be equal, out = " + str(data_np) + return ( + "The sorted ground truth and sorted out should " + + "be equal, out = " + + str(data_np) + ) def convert_dtype(dtype_str): dtype_str_list = ["int32", "int64", "float32", "float64"] dtype_num_list = [ - core.VarDesc.VarType.INT32, core.VarDesc.VarType.INT64, - core.VarDesc.VarType.FP32, core.VarDesc.VarType.FP64 + core.VarDesc.VarType.INT32, + core.VarDesc.VarType.INT64, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP64, ] - assert dtype_str in dtype_str_list, dtype_str + \ - " should in " + str(dtype_str_list) + assert dtype_str in dtype_str_list, ( + dtype_str + " should in " + str(dtype_str_list) + ) return dtype_num_list[dtype_str_list.index(dtype_str)] class TestRandpermOp(OpTest): - """ Test randperm op.""" + """Test randperm op.""" def setUp(self): self.op_type = "randperm" @@ -77,36 +84,32 @@ class TestRandpermOp(OpTest): def verify_output(self, outs): out_np = np.array(outs[0]) - self.assertTrue(check_randperm_out(self.n, out_np), - msg=error_msg(out_np)) + self.assertTrue( + check_randperm_out(self.n, out_np), msg=error_msg(out_np) + ) class TestRandpermOpN(TestRandpermOp): - def init_attrs(self): self.n = 10000 class TestRandpermOpInt32(TestRandpermOp): - def init_attrs(self): self.dtype = "int32" class TestRandpermOpFloat32(TestRandpermOp): - def init_attrs(self): self.dtype = "float32" class TestRandpermOpFloat64(TestRandpermOp): - def init_attrs(self): self.dtype = "float64" class TestRandpermOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(ValueError, paddle.randperm, -3) @@ -114,7 +117,6 @@ class TestRandpermOpError(unittest.TestCase): class TestRandpermAPI(unittest.TestCase): - def test_out(self): n = 10 place = paddle.MLUPlace(0) @@ -132,20 +134,19 @@ class TestRandpermAPI(unittest.TestCase): class TestRandpermImperative(unittest.TestCase): - def test_out(self): paddle.disable_static() n = 10 for dtype in ['int32', np.int64, 'float32', 'float64']: data_p = paddle.randperm(n, dtype) data_np = data_p.numpy() - self.assertTrue(check_randperm_out(n, data_np), - msg=error_msg(data_np)) + self.assertTrue( + check_randperm_out(n, data_np), msg=error_msg(data_np) + ) paddle.enable_static() class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -158,25 +159,70 @@ class TestRandomValue(unittest.TestCase): x = paddle.randperm(30000, dtype='int32').numpy() expect = [ - 24562, 8409, 9379, 10328, 20503, 18059, 9681, 21883, 11783, 27413 + 24562, + 8409, + 9379, + 10328, + 20503, + 18059, + 9681, + 21883, + 11783, + 27413, ] np.testing.assert_allclose(x[0:10], expect) expect = [ - 29477, 27100, 9643, 16637, 8605, 16892, 27767, 2724, 1612, 13096 + 29477, + 27100, + 9643, + 16637, + 8605, + 16892, + 27767, + 2724, + 1612, + 13096, ] np.testing.assert_allclose(x[10000:10010], expect) expect = [ - 298, 4104, 16479, 22714, 28684, 7510, 14667, 9950, 15940, 28343 + 298, + 4104, + 16479, + 22714, + 28684, + 7510, + 14667, + 9950, + 15940, + 28343, ] np.testing.assert_allclose(x[20000:20010], expect) x = paddle.randperm(30000, dtype='int64').numpy() expect = [ - 6587, 1909, 5525, 23001, 6488, 14981, 14355, 3083, 29561, 8171 + 6587, + 1909, + 5525, + 23001, + 6488, + 14981, + 14355, + 3083, + 29561, + 8171, ] np.testing.assert_allclose(x[0:10], expect) expect = [ - 23460, 12394, 22501, 5427, 20185, 9100, 5127, 1651, 25806, 4818 + 23460, + 12394, + 22501, + 5427, + 20185, + 9100, + 5127, + 1651, + 25806, + 4818, ] np.testing.assert_allclose(x[10000:10010], expect) expect = [5829, 4508, 16193, 24836, 8526, 242, 9984, 9243, 1977, 11839] @@ -184,35 +230,83 @@ class TestRandomValue(unittest.TestCase): x = paddle.randperm(30000, dtype='float32').numpy() expect = [ - 5154., 10537., 14362., 29843., 27185., 28399., 27561., 4144., - 22906., 10705. + 5154.0, + 10537.0, + 14362.0, + 29843.0, + 27185.0, + 28399.0, + 27561.0, + 4144.0, + 22906.0, + 10705.0, ] np.testing.assert_allclose(x[0:10], expect) expect = [ - 1958., 18414., 20090., 21910., 22746., 27346., 22347., 3002., 4564., - 26991. + 1958.0, + 18414.0, + 20090.0, + 21910.0, + 22746.0, + 27346.0, + 22347.0, + 3002.0, + 4564.0, + 26991.0, ] np.testing.assert_allclose(x[10000:10010], expect) expect = [ - 25580., 12606., 553., 16387., 29536., 4241., 20946., 16899., 16339., - 4662. + 25580.0, + 12606.0, + 553.0, + 16387.0, + 29536.0, + 4241.0, + 20946.0, + 16899.0, + 16339.0, + 4662.0, ] np.testing.assert_allclose(x[20000:20010], expect) x = paddle.randperm(30000, dtype='float64').numpy() expect = [ - 19051., 2449., 21940., 11121., 282., 7330., 13747., 24321., 21147., - 9163. + 19051.0, + 2449.0, + 21940.0, + 11121.0, + 282.0, + 7330.0, + 13747.0, + 24321.0, + 21147.0, + 9163.0, ] np.testing.assert_allclose(x[0:10], expect) expect = [ - 15483., 1315., 5723., 20954., 13251., 25539., 5074., 1823., 14945., - 17624. + 15483.0, + 1315.0, + 5723.0, + 20954.0, + 13251.0, + 25539.0, + 5074.0, + 1823.0, + 14945.0, + 17624.0, ] np.testing.assert_allclose(x[10000:10010], expect) expect = [ - 10516., 2552., 29970., 5941., 986., 8007., 24805., 26753., 12202., - 21404. + 10516.0, + 2552.0, + 29970.0, + 5941.0, + 986.0, + 8007.0, + 24805.0, + 26753.0, + 12202.0, + 21404.0, ] np.testing.assert_allclose(x[20000:20010], expect) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_range_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_range_op_mlu.py index d65040ecb39..6ebbb089e58 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_range_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_range_op_mlu.py @@ -29,7 +29,6 @@ def arange_wrapper(start, end, step, dtype=None): class TestRangeOp(OpTest): - def setUp(self): self.op_type = "range" self.place = paddle.device.MLUPlace(0) @@ -38,13 +37,13 @@ class TestRangeOp(OpTest): self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange(self.case[0], self.case[1], self.case[2]).astype( + self.dtype + ) } def init_config(self): @@ -57,7 +56,6 @@ class TestRangeOp(OpTest): class TestFloatRangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.float32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -65,7 +63,6 @@ class TestFloatRangeOpCase0(TestRangeOp): class TestInt32RangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -73,7 +70,6 @@ class TestInt32RangeOpCase0(TestRangeOp): class TestInt32RangeOpCase1(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -81,7 +77,6 @@ class TestInt32RangeOpCase1(TestRangeOp): class TestInt32RangeOpCase2(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reciprocal_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reciprocal_op_mlu.py index e93afd7cc29..0b6d08bdfd3 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reciprocal_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reciprocal_op_mlu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestMLUReciprocal(OpTest): - def setUp(self): self.op_type = "reciprocal" self.set_mlu() @@ -41,9 +40,9 @@ class TestMLUReciprocal(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.01 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -54,7 +53,6 @@ class TestMLUReciprocal(OpTest): class TestMLUReciprocalFp16(TestMLUReciprocal): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.MLUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_max_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_max_op_mlu.py index dc90a521d37..b225e25b06b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_max_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_max_op_mlu.py @@ -26,7 +26,8 @@ paddle.enable_static() @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestMLUReduceMaxOp(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -54,7 +55,8 @@ class TestMLUReduceMaxOp(OpTest): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpMultiAxises(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -72,7 +74,8 @@ class TestReduceMaxOpMultiAxises(TestMLUReduceMaxOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceAll(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -88,7 +91,8 @@ class TestReduceAll(TestMLUReduceMaxOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_int32(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -100,11 +104,12 @@ class TestReduceMaxOpWithOutDtype_int32(TestMLUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.int32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } def init_dtype(self): @@ -113,7 +118,8 @@ class TestReduceMaxOpWithOutDtype_int32(TestMLUReduceMaxOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp16(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -125,12 +131,12 @@ class TestReduceMaxOpWithOutDtype_fp16(TestMLUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def init_dtype(self): @@ -142,7 +148,8 @@ class TestReduceMaxOpWithOutDtype_fp16(TestMLUReduceMaxOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp32(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -154,12 +161,12 @@ class TestReduceMaxOpWithOutDtype_fp32(TestMLUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_mean_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_mean_op_mlu.py index 588dc949aeb..bbb35ef40d3 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_mean_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_mean_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestMeanOp(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -44,7 +43,6 @@ class TestMeanOp(OpTest): class TestMeanOp5D(TestMeanOp): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -55,7 +53,6 @@ class TestMeanOp5D(TestMeanOp): class TestMeanOp6D(TestMeanOp): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -66,7 +63,6 @@ class TestMeanOp6D(TestMeanOp): class TestMeanOp8D(TestMeanOp): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -78,7 +74,6 @@ class TestMeanOp8D(TestMeanOp): class Test1DReduce(TestMeanOp): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -87,7 +82,6 @@ class Test1DReduce(TestMeanOp): class Test2DReduce0(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -97,7 +91,6 @@ class Test2DReduce0(Test1DReduce): class Test2DReduce1(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -109,7 +102,6 @@ class Test2DReduce1(Test1DReduce): class Test3DReduce0(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -121,7 +113,6 @@ class Test3DReduce0(Test1DReduce): class Test3DReduce1(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -133,7 +124,6 @@ class Test3DReduce1(Test1DReduce): class Test3DReduce2(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -145,7 +135,6 @@ class Test3DReduce2(Test1DReduce): class Test3DReduce3(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -157,21 +146,19 @@ class Test3DReduce3(Test1DReduce): class TestKeepDimReduce(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" self.inputs = {'X': np.random.random((5, 6, 10)).astype("float32")} self.attrs = {'dim': [1], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestKeepDim8DReduce(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -180,14 +167,13 @@ class TestKeepDim8DReduce(Test1DReduce): } self.attrs = {'dim': (3, 4, 5), 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestReduceAll(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_min_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_min_op_mlu.py index 9d69d6d8ccc..d6649f214c4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_min_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_min_op_mlu.py @@ -26,7 +26,8 @@ paddle.enable_static() @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestMLUReduceMinOp(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -54,7 +55,8 @@ class TestMLUReduceMinOp(OpTest): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpMultiAxises(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -72,7 +74,8 @@ class TestReduceMinOpMultiAxises(TestMLUReduceMinOp): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceAll(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -88,7 +91,8 @@ class TestReduceAll(TestMLUReduceMinOp): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_int32(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -100,11 +104,12 @@ class TestReduceMinOpWithOutDtype_int32(TestMLUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.int32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } def init_dtype(self): @@ -113,7 +118,8 @@ class TestReduceMinOpWithOutDtype_int32(TestMLUReduceMinOp): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp16(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -125,12 +131,12 @@ class TestReduceMinOpWithOutDtype_fp16(TestMLUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def init_dtype(self): @@ -142,7 +148,8 @@ class TestReduceMinOpWithOutDtype_fp16(TestMLUReduceMinOp): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp32(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -154,12 +161,12 @@ class TestReduceMinOpWithOutDtype_fp32(TestMLUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_prod_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_prod_op_mlu.py index 85179a72f03..fe690daabea 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_prod_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_prod_op_mlu.py @@ -29,7 +29,6 @@ def raw_reduce_prod(x, dim=[0], keep_dim=False): class TestProdOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.place = paddle.device.MLUPlace(0) @@ -47,7 +46,6 @@ class TestProdOp(OpTest): class TestProd6DOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.place = paddle.device.MLUPlace(0) @@ -70,7 +68,6 @@ class TestProd6DOp(OpTest): class TestProd8DOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.place = paddle.device.MLUPlace(0) @@ -78,8 +75,9 @@ class TestProd8DOp(OpTest): self.python_api = raw_reduce_prod self.init_data_type() self.inputs = { - 'X': np.random.random( - (2, 5, 3, 2, 2, 3, 4, 2)).astype(self.data_type) + 'X': np.random.random((2, 5, 3, 2, 2, 3, 4, 2)).astype( + self.data_type + ) } self.attrs = {'dim': [2, 3, 4]} self.outputs = { diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_sum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_sum_op_mlu.py index 823a7398f07..219b17a039e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_sum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_sum_op_mlu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestMLUReduceSumOp(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -32,16 +31,16 @@ class TestMLUReduceSumOp(OpTest): self.attrs = { 'dim': self.axis, 'keep_dim': self.keep_dim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.inputs = {'X': np.random.random(self.shape).astype("float32")} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].sum()} else: self.outputs = { - 'Out': - self.inputs['X'].sum(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def set_mlu(self): @@ -62,105 +61,92 @@ class TestMLUReduceSumOp(OpTest): def initTestCase(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestSumOp5D(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (1, 2, 5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestSumOp6D(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (1, 1, 2, 5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestSumOp8D(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (1, 3, 1, 2, 1, 4, 3, 10) self.axis = (0, 3) class Test1DReduce(TestMLUReduceSumOp): - def initTestCase(self): self.shape = 120 - self.axis = (0, ) + self.axis = (0,) class Test2DReduce0(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (20, 10) - self.axis = (0, ) + self.axis = (0,) class Test2DReduce1(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (20, 10) - self.axis = (1, ) + self.axis = (1,) class Test3DReduce0(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (1, ) + self.axis = (1,) class Test3DReduce1(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (2, ) + self.axis = (2,) class Test3DReduce2(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (-2, ) + self.axis = (-2,) class Test3DReduce3(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 7) self.axis = (1, 2) class TestKeepDimReduce(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 10) - self.axis = (1, ) + self.axis = (1,) self.keep_dim = True class TestKeepDim8DReduce(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (2, 5, 3, 2, 2, 3, 4, 2) self.axis = (3, 4, 5) self.keep_dim = True def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.03) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.03 + ) class TestReduceAll(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 2, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = True diff --git a/python/paddle/fluid/tests/unittests/mlu/test_relu6_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_relu6_op_mlu.py index 45a1a825c43..4e8ebacf0eb 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_relu6_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_relu6_op_mlu.py @@ -34,7 +34,6 @@ def ref_relu6(x, threshold=6.0): class TestRelu6(OpTest): - def setUp(self): self.set_mlu() self.op_type = "relu6" @@ -64,7 +63,6 @@ class TestRelu6(OpTest): class TestRelu6Float16(TestRelu6): - def set_mlu(self): self.__class__.use_mlu = True self.__class__.no_need_check_grad = True @@ -77,7 +75,6 @@ class TestRelu6Float16(TestRelu6): class TestReluNeg(TestRelu6): - def setUp(self): self.set_mlu() self.op_type = "relu6" @@ -104,7 +101,6 @@ class TestReluNeg(TestRelu6): class TestRelu6Net(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -119,9 +115,9 @@ class TestRelu6Net(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.nn.functional.relu6(sum) @@ -145,16 +141,17 @@ class TestRelu6Net(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_relu_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_relu_op_mlu.py index a6c4303a208..db62e592ac2 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_relu_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_relu_op_mlu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestRelu(OpTest): - def setUp(self): self.set_mlu() self.op_type = "relu" @@ -52,7 +51,6 @@ class TestRelu(OpTest): class TestReluFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "relu" @@ -79,7 +77,6 @@ class TestReluFp16(OpTest): class TestReluNeg(OpTest): - def setUp(self): self.set_mlu() self.op_type = "relu" @@ -105,7 +102,6 @@ class TestReluNeg(OpTest): class TestReluNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -120,9 +116,9 @@ class TestReluNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.nn.functional.relu(sum) @@ -146,16 +142,17 @@ class TestReluNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reshape2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reshape2_op_mlu.py index 3354acf6d0c..94090b9a177 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reshape2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reshape2_op_mlu.py @@ -26,7 +26,6 @@ SEED = 2022 class TestReshape2(OpTest): - def setUp(self): self.set_mlu() self.op_type = "reshape2" @@ -37,7 +36,7 @@ class TestReshape2(OpTest): self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -56,7 +55,6 @@ class TestReshape2(OpTest): class TestReshape2_case2(TestReshape2): - def init_data(self): self.ori_shape = (2, 100) self.new_shape = (-1, 10) @@ -64,7 +62,6 @@ class TestReshape2_case2(TestReshape2): class TestReshape2_case3(TestReshape2): - def init_data(self): self.ori_shape = (100, 5, 6) self.new_shape = (-1, 0, 3) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_rnn_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_rnn_op_mlu.py index e5f9d154bfd..22597342f7f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_rnn_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_rnn_op_mlu.py @@ -35,7 +35,6 @@ paddle.enable_static() class TestRNNOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(self.num_layers): @@ -64,41 +63,53 @@ class TestRNNOp(OpTest): self.direction_num = 2 if self.is_bidirec else 1 direction = "bidirectional" if self.is_bidirec else "forward" - input = np.random.uniform(low=-0.1, - high=0.1, - size=(self.seq_length, self.batch_size, - self.input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, + high=0.1, + size=(self.seq_length, self.batch_size, self.input_size), + ).astype(self.dtype) input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 - rnn1 = LSTM(self.input_size, - self.hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - dtype=self.dtype) + rnn1 = LSTM( + self.input_size, + self.hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) - output, (last_hidden, - last_cell) = rnn1(input, sequence_length=self.sequence_length) + output, (last_hidden, last_cell) = rnn1( + input, sequence_length=self.sequence_length + ) init_h = np.zeros( - (self.num_layers * self.direction_num, self.batch_size, - self.hidden_size)).astype(self.dtype) + ( + self.num_layers * self.direction_num, + self.batch_size, + self.hidden_size, + ) + ).astype(self.dtype) init_c = np.zeros( - (self.num_layers * self.direction_num, self.batch_size, - self.hidden_size)).astype(self.dtype) + ( + self.num_layers * self.direction_num, + self.batch_size, + self.hidden_size, + ) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") self.inputs = { 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h), ('init_c', init_c)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -113,13 +124,13 @@ class TestRNNOp(OpTest): 'hidden_size': self.hidden_size, 'num_layers': self.num_layers, 'mode': self.mode, - 'is_test': self.is_test + 'is_test': self.is_test, } self.outputs = { 'Out': output, "State": [('last_hidden', last_hidden), ('last_cell', last_cell)], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def init_dtype(self): @@ -135,7 +146,8 @@ class TestRNNOp(OpTest): self.check_output_with_place( self.place, atol=1e-4, - no_check_set=['Reserve', 'DropoutState', 'State']) + no_check_set=['Reserve', 'DropoutState', 'State'], + ) def set_attrs(self): pass @@ -146,39 +158,38 @@ class TestRNNOp(OpTest): var_name_list = self.get_weight_names() grad_check_list = ['Input', 'init_h', 'init_c'] grad_check_list.extend(var_name_list) - self.check_grad_with_place(self.place, set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad_with_place( + self.place, + set(grad_check_list), + ['Out', 'last_hidden', 'last_cell'], + ) class TestRNNOp1(TestRNNOp): - def set_attrs(self): self.sequence_length = None class TestRNNOp2(TestRNNOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True class TestRNNOp3(TestRNNOp): - def set_attrs(self): self.is_test = True self.sequence_length = None class TestRNNOp4(TestRNNOp): - def set_attrs(self): self.is_test = True self.sequence_length = None self.is_bidirec = True -#TODO(chenxiao): cnnl doesn't support num_layers > 1 case +# TODO(chenxiao): cnnl doesn't support num_layers > 1 case # class TestRNNOp5(TestRNNOp): # def set_attrs(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_roi_align_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_roi_align_op_mlu.py index 61ca7b38a80..0f7b347315f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_roi_align_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_roi_align_op_mlu.py @@ -26,7 +26,6 @@ np.random.seed(1243) class TestROIAlignMLUOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -37,7 +36,7 @@ class TestROIAlignMLUOp(OpTest): self.inputs = { 'X': self.x, 'ROIs': self.rois[:, 1:5], - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } # print("self.inputs: ",self.inputs) @@ -46,7 +45,7 @@ class TestROIAlignMLUOp(OpTest): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.aligned + 'aligned': self.aligned, } self.outputs = {'Out': self.out_data} @@ -68,25 +67,45 @@ class TestROIAlignMLUOp(OpTest): self.x = np.random.random(self.x_dim).astype('float32') - def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, - bin_size_h, bin_size_w): + def pre_calc( + self, + x_i, + roi_xmin, + roi_ymin, + roi_bin_grid_h, + roi_bin_grid_w, + bin_size_h, + bin_size_w, + ): count = roi_bin_grid_h * roi_bin_grid_w bilinear_pos = np.zeros( [self.channels, self.pooled_height, self.pooled_width, count, 4], - np.float32) - bilinear_w = np.zeros([self.pooled_height, self.pooled_width, count, 4], - np.float32) + np.float32, + ) + bilinear_w = np.zeros( + [self.pooled_height, self.pooled_width, count, 4], np.float32 + ) for ph in range(self.pooled_width): for pw in range(self.pooled_height): c = 0 for iy in range(roi_bin_grid_h): - y = roi_ymin + ph * bin_size_h + (iy + 0.5) * \ - bin_size_h / roi_bin_grid_h + y = ( + roi_ymin + + ph * bin_size_h + + (iy + 0.5) * bin_size_h / roi_bin_grid_h + ) for ix in range(roi_bin_grid_w): - x = roi_xmin + pw * bin_size_w + (ix + 0.5) * \ - bin_size_w / roi_bin_grid_w - if y < -1.0 or y > self.height or \ - x < -1.0 or x > self.width: + x = ( + roi_xmin + + pw * bin_size_w + + (ix + 0.5) * bin_size_w / roi_bin_grid_w + ) + if ( + y < -1.0 + or y > self.height + or x < -1.0 + or x > self.width + ): continue if y <= 0: y = 0 @@ -107,14 +126,18 @@ class TestROIAlignMLUOp(OpTest): hy = 1 - ly hx = 1 - lx for ch in range(self.channels): - bilinear_pos[ch, ph, pw, c, 0] = x_i[ch, y_low, - x_low] - bilinear_pos[ch, ph, pw, c, 1] = x_i[ch, y_low, - x_high] - bilinear_pos[ch, ph, pw, c, 2] = x_i[ch, y_high, - x_low] - bilinear_pos[ch, ph, pw, c, 3] = x_i[ch, y_high, - x_high] + bilinear_pos[ch, ph, pw, c, 0] = x_i[ + ch, y_low, x_low + ] + bilinear_pos[ch, ph, pw, c, 1] = x_i[ + ch, y_low, x_high + ] + bilinear_pos[ch, ph, pw, c, 2] = x_i[ + ch, y_high, x_low + ] + bilinear_pos[ch, ph, pw, c, 3] = x_i[ + ch, y_high, x_high + ] bilinear_w[ph, pw, c, 0] = hy * hx bilinear_w[ph, pw, c, 1] = hy * lx bilinear_w[ph, pw, c, 2] = ly * hx @@ -124,10 +147,15 @@ class TestROIAlignMLUOp(OpTest): def calc_roi_align(self): self.out_data = np.zeros( - (self.rois_num, self.channels, self.pooled_height, - self.pooled_width)).astype('float32') - - offset = 0.5 if self.aligned else 0. + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ).astype('float32') + + offset = 0.5 if self.aligned else 0.0 for i in range(self.rois_num): roi = self.rois[i] roi_batch_id = int(roi[0]) @@ -145,16 +173,27 @@ class TestROIAlignMLUOp(OpTest): bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) - roi_bin_grid_h = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_height / self.pooled_height) - roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_width / self.pooled_width) + roi_bin_grid_h = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_height / self.pooled_height) + ) + roi_bin_grid_w = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_width / self.pooled_width) + ) count = max(int(roi_bin_grid_h * roi_bin_grid_w), 1) pre_size = count * self.pooled_width * self.pooled_height - bilinear_pos, bilinear_w = self.pre_calc(x_i, roi_xmin, roi_ymin, - int(roi_bin_grid_h), - int(roi_bin_grid_w), - bin_size_h, bin_size_w) + bilinear_pos, bilinear_w = self.pre_calc( + x_i, + roi_xmin, + roi_ymin, + int(roi_bin_grid_h), + int(roi_bin_grid_w), + bin_size_h, + bin_size_w, + ) for ch in range(self.channels): align_per_bin = (bilinear_pos[ch] * bilinear_w).sum(axis=-1) output_val = align_per_bin.mean(axis=-1) @@ -168,14 +207,18 @@ class TestROIAlignMLUOp(OpTest): # self.rois_lod[0].append(bno) self.rois_lod[0].append(1) x1 = np.random.randint( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.randint( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.randint(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.randint(y1 + self.pooled_height, - self.height // self.spatial_scale) + x2 = np.random.randint( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.randint( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) @@ -197,7 +240,6 @@ class TestROIAlignMLUOp(OpTest): class TestROIAlignOpWithMinusSample(TestROIAlignMLUOp): - def init_test_case(self): self.batch_size = 3 self.channels = 3 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_scale_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_scale_op_mlu.py index 9b04042458d..c7858a48229 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_scale_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_scale_op_mlu.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestScaleOp(OpTest): - def setUp(self): self.op_type = "scale" self.place = paddle.device.MLUPlace(0) @@ -49,7 +48,6 @@ class TestScaleOp(OpTest): class TestScaleOpScaleVariable(OpTest): - def setUp(self): self.op_type = "scale" self.place = paddle.device.MLUPlace(0) @@ -59,7 +57,7 @@ class TestScaleOpScaleVariable(OpTest): self.scale = -2.3 self.inputs = { 'X': np.random.random((10, 10)).astype(self.dtype), - 'ScaleTensor': np.array([self.scale]).astype('float32') + 'ScaleTensor': np.array([self.scale]).astype('float32'), } self.attrs = {} self.outputs = {'Out': self.inputs['X'] * self.dtype(self.scale)} @@ -72,7 +70,6 @@ class TestScaleOpScaleVariable(OpTest): class TestScaleOpSelectedRows(unittest.TestCase): - def init_dtype_type(self): pass @@ -91,8 +88,9 @@ class TestScaleOpSelectedRows(unittest.TestCase): in_selected_rows = scope.var(in_name).get_selected_rows() in_selected_rows.set_height(in_height) in_selected_rows.set_rows(in_rows) - in_array = np.random.random( - (len(in_rows), in_row_numel)).astype(self.dtype) + in_array = np.random.random((len(in_rows), in_row_numel)).astype( + self.dtype + ) in_tensor = in_selected_rows.get_tensor() in_tensor.set(in_array, place) @@ -131,9 +129,7 @@ class TestScaleOpSelectedRows(unittest.TestCase): class TestScaleRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.scale([10]) @@ -141,10 +137,10 @@ class TestScaleRaiseError(unittest.TestCase): # Add FP16 test -@unittest.skipIf(not core.is_compiled_with_mlu(), - "core is not compiled with MLU") +@unittest.skipIf( + not core.is_compiled_with_mlu(), "core is not compiled with MLU" +) class TestScaleFp16Op(TestScaleOp): - def init_dtype_type(self): self.dtype = np.float16 @@ -152,10 +148,10 @@ class TestScaleFp16Op(TestScaleOp): self.check_output_with_place(self.place, atol=0.002) -@unittest.skipIf(not core.is_compiled_with_mlu(), - "core is not compiled with MLU") +@unittest.skipIf( + not core.is_compiled_with_mlu(), "core is not compiled with MLU" +) class TestScaleFp16OpSelectedRows(TestScaleOpSelectedRows): - def init_dtype_type(self): self.dtype = np.float16 @@ -169,7 +165,6 @@ class TestScaleFp16OpSelectedRows(TestScaleOpSelectedRows): class TestScaleApiStatic(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -187,13 +182,11 @@ class TestScaleApiStatic(unittest.TestCase): class TestScaleInplaceApiStatic(TestScaleApiStatic): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) class TestScaleApiDygraph(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -207,7 +200,6 @@ class TestScaleApiDygraph(unittest.TestCase): class TestScaleInplaceApiDygraph(TestScaleApiDygraph): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_scatter_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_scatter_op_mlu.py index 76042074b0e..cb13f305cae 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_scatter_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_scatter_op_mlu.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestScatterOp(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -50,7 +49,6 @@ class TestScatterOp(OpTest): class TestScatterOp0(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -73,7 +71,6 @@ class TestScatterOp0(OpTest): class TestScatterOp1(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -99,7 +96,6 @@ class TestScatterOp1(OpTest): class TestScatterOp2(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -121,7 +117,6 @@ class TestScatterOp2(OpTest): class TestScatterAPI(unittest.TestCase): - def setUp(self): self.places = [paddle.device.MLUPlace(0)] self.__class__.use_mlu = True @@ -139,19 +134,26 @@ class TestScatterAPI(unittest.TestCase): input_data = np.array([[1, 1], [2, 2], [3, 3]]).astype(np.float32) index_data = np.array([2, 1, 0, 1]).astype(np.int64) - updates_data = np.array([[1, 1], [2, 2], [3, 3], - [4, 4]]).astype(np.float32) + updates_data = np.array([[1, 1], [2, 2], [3, 3], [4, 4]]).astype( + np.float32 + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input": input_data, - "index": index_data, - "updates": updates_data - }, - fetch_list=[result]) - self.assertEqual((fetches[0] == \ - np.array([[3., 3.],[6., 6.],[1., 1.]])).all(), True) + fetches = exe.run( + fluid.default_main_program(), + feed={ + "input": input_data, + "index": index_data, + "updates": updates_data, + }, + fetch_list=[result], + ) + self.assertEqual( + ( + fetches[0] == np.array([[3.0, 3.0], [6.0, 6.0], [1.0, 1.0]]) + ).all(), + True, + ) def test_static(self): for place in self.places: @@ -162,16 +164,22 @@ class TestScatterAPI(unittest.TestCase): with fluid.dygraph.guard(place): x_data = np.array([[1, 1], [2, 2], [3, 3]]).astype(np.float32) index_data = np.array([2, 1, 0, 1]).astype(np.int64) - updates_data = np.array([[1, 1], [2, 2], [3, 3], - [4, 4]]).astype(np.float32) + updates_data = np.array( + [[1, 1], [2, 2], [3, 3], [4, 4]] + ).astype(np.float32) x = fluid.dygraph.to_variable(x_data) index = fluid.dygraph.to_variable(index_data) updates = fluid.dygraph.to_variable(updates_data) output1 = self.scatter(x, index, updates, overwrite=False) - self.assertEqual((output1.numpy() == \ - np.array([[3., 3.],[6., 6.],[1., 1.]])).all(), True) + self.assertEqual( + ( + output1.numpy() + == np.array([[3.0, 3.0], [6.0, 6.0], [1.0, 1.0]]) + ).all(), + True, + ) def test_large_data(self): if os.name == "nt": @@ -183,27 +191,30 @@ class TestScatterAPI(unittest.TestCase): def test_dygraph(): with fluid.dygraph.guard(): - mlu_out = paddle.scatter(paddle.to_tensor(x), - paddle.to_tensor(index), - paddle.to_tensor(updates)) + mlu_out = paddle.scatter( + paddle.to_tensor(x), + paddle.to_tensor(index), + paddle.to_tensor(updates), + ) return mlu_out.numpy() @switch_to_static_graph def test_static_graph(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_t = paddle.static.data(name="x", dtype=x.dtype, shape=x.shape) - index_t = paddle.static.data(name="index", - dtype=index.dtype, - shape=index.shape) - updates_t = paddle.static.data(name="updates", - dtype=updates.dtype, - shape=updates.shape) + index_t = paddle.static.data( + name="index", dtype=index.dtype, shape=index.shape + ) + updates_t = paddle.static.data( + name="updates", dtype=updates.dtype, shape=updates.shape + ) out_t = paddle.scatter(x_t, index_t, updates_t) feed = { x_t.name: x, index_t.name: index, - updates_t.name: updates + updates_t.name: updates, } fetch = [out_t] @@ -215,7 +226,6 @@ class TestScatterAPI(unittest.TestCase): class TestScatterOpFp16(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -238,7 +248,6 @@ class TestScatterOpFp16(OpTest): class TestScatterInplaceAPI(TestScatterAPI): - def executed_api(self): self.scatter = paddle.scatter_ diff --git a/python/paddle/fluid/tests/unittests/mlu/test_set_value_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_set_value_op_mlu.py index d621aab9507..d9db940b029 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_set_value_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_set_value_op_mlu.py @@ -24,7 +24,6 @@ from paddle.fluid import core class TestSetValueBase(unittest.TestCase): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -55,7 +54,6 @@ class TestSetValueBase(unittest.TestCase): class TestSetValueApi(TestSetValueBase): - def _run_static(self): paddle.enable_static() with paddle.static.program_guard(self.program): @@ -71,15 +69,18 @@ class TestSetValueApi(TestSetValueBase): static_out = self._run_static() self._get_answer() - error_msg = "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" - self.assertTrue((self.data == static_out).all(), - msg=error_msg.format("static", self.data, static_out)) + error_msg = ( + "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" + ) + self.assertTrue( + (self.data == static_out).all(), + msg=error_msg.format("static", self.data, static_out), + ) # 1. Test different type of item: int, Python slice, Paddle Tensor # 1.1 item is int class TestSetValueItemInt(TestSetValueApi): - def _call_setitem(self, x): x[0] = self.value @@ -90,7 +91,6 @@ class TestSetValueItemInt(TestSetValueApi): # 1.2 item is slice # 1.2.1 step is 1 class TestSetValueItemSlice(TestSetValueApi): - def _call_setitem(self, x): x[0:2] = self.value @@ -99,7 +99,6 @@ class TestSetValueItemSlice(TestSetValueApi): class TestSetValueItemSlice2(TestSetValueApi): - def _call_setitem(self, x): x[0:-1] = self.value @@ -108,7 +107,6 @@ class TestSetValueItemSlice2(TestSetValueApi): class TestSetValueItemSlice3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2] = self.value @@ -117,7 +115,6 @@ class TestSetValueItemSlice3(TestSetValueApi): class TestSetValueItemSlice4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2, :] = self.value @@ -126,7 +123,6 @@ class TestSetValueItemSlice4(TestSetValueApi): class TestSetValueItemSlice5(TestSetValueApi): - def set_shape(self): self.shape = [100, 426, 640] @@ -137,8 +133,8 @@ class TestSetValueItemSlice5(TestSetValueApi): self.data[0:-1] = self.value -#TODO: Fix this after MLU support while_loop -#class TestSetValueItemSliceInWhile(TestSetValueApi): +# TODO: Fix this after MLU support while_loop +# class TestSetValueItemSliceInWhile(TestSetValueApi): # def _call_setitem(self, x): # def cond(i, x): # return i < 1 @@ -157,7 +153,6 @@ class TestSetValueItemSlice5(TestSetValueApi): # 1.2.2 step > 1 class TestSetValueItemSliceStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 5, 5] @@ -169,7 +164,6 @@ class TestSetValueItemSliceStep(TestSetValueApi): class TestSetValueItemSliceStep2(TestSetValueApi): - def set_shape(self): self.shape = [7, 5, 5] @@ -181,7 +175,6 @@ class TestSetValueItemSliceStep2(TestSetValueApi): class TestSetValueItemSliceStep3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2, ::2] = self.value @@ -190,7 +183,6 @@ class TestSetValueItemSliceStep3(TestSetValueApi): class TestSetValueItemSliceStep4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2:2, :] = self.value @@ -200,7 +192,6 @@ class TestSetValueItemSliceStep4(TestSetValueApi): # 1.2.3 step < 0 class TestSetValueItemSliceNegetiveStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 2] @@ -215,7 +206,6 @@ class TestSetValueItemSliceNegetiveStep(TestSetValueApi): class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): - def set_shape(self): self.shape = [5] @@ -230,7 +220,6 @@ class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): - def set_shape(self): self.shape = [3] @@ -245,7 +234,6 @@ class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -260,7 +248,6 @@ class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): class TestSetValueItemEllipsis1(TestSetValueApi): - def _call_setitem(self, x): x[0:, ..., 1:] = self.value @@ -269,7 +256,6 @@ class TestSetValueItemEllipsis1(TestSetValueApi): class TestSetValueItemEllipsis2(TestSetValueApi): - def _call_setitem(self, x): x[0:, ...] = self.value @@ -278,7 +264,6 @@ class TestSetValueItemEllipsis2(TestSetValueApi): class TestSetValueItemEllipsis3(TestSetValueApi): - def _call_setitem(self, x): x[..., 1:] = self.value @@ -287,7 +272,6 @@ class TestSetValueItemEllipsis3(TestSetValueApi): class TestSetValueItemEllipsis4(TestSetValueApi): - def _call_setitem(self, x): x[...] = self.value @@ -297,7 +281,6 @@ class TestSetValueItemEllipsis4(TestSetValueApi): # 1.4 item is Paddle Tensor class TestSetValueItemTensor(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") x[zero] = self.value @@ -307,7 +290,6 @@ class TestSetValueItemTensor(TestSetValueApi): class TestSetValueItemTensor2(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -318,7 +300,6 @@ class TestSetValueItemTensor2(TestSetValueApi): class TestSetValueItemTensor3(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -329,7 +310,6 @@ class TestSetValueItemTensor3(TestSetValueApi): class TestSetValueItemTensor4(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -340,7 +320,6 @@ class TestSetValueItemTensor4(TestSetValueApi): class TestSetValueItemTensor5(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -351,7 +330,6 @@ class TestSetValueItemTensor5(TestSetValueApi): class TestSetValueItemTensor6(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -366,7 +344,6 @@ class TestSetValueItemTensor6(TestSetValueApi): # 1.5 item is None class TestSetValueItemNone1(TestSetValueApi): - def _call_setitem(self, x): x[None] = self.value @@ -375,7 +352,6 @@ class TestSetValueItemNone1(TestSetValueApi): class TestSetValueItemNone2(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 1] = self.value @@ -384,7 +360,6 @@ class TestSetValueItemNone2(TestSetValueApi): class TestSetValueItemNone3(TestSetValueApi): - def _call_setitem(self, x): x[:, None, None, 1] = self.value @@ -393,7 +368,6 @@ class TestSetValueItemNone3(TestSetValueApi): class TestSetValueItemNone4(TestSetValueApi): - def _call_setitem(self, x): x[0, 0, None, 1] = self.value @@ -402,7 +376,6 @@ class TestSetValueItemNone4(TestSetValueApi): class TestSetValueItemNone5(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 0, None, 1] = self.value @@ -411,7 +384,6 @@ class TestSetValueItemNone5(TestSetValueApi): class TestSetValueItemNone6(TestSetValueApi): - def _call_setitem(self, x): x[None, 0, 0, None, 0] = self.value @@ -420,7 +392,6 @@ class TestSetValueItemNone6(TestSetValueApi): class TestSetValueItemNone7(TestSetValueApi): - def _call_setitem(self, x): x[:, None, 1] = np.zeros(self.shape)[:, None, 0] @@ -429,7 +400,6 @@ class TestSetValueItemNone7(TestSetValueApi): class TestSetValueItemNone8(TestSetValueApi): - def _call_setitem(self, x): x[:, 1, None] = np.zeros(self.shape)[:, 0, None] @@ -438,7 +408,6 @@ class TestSetValueItemNone8(TestSetValueApi): class TestSetValueItemNone9(TestSetValueApi): - def _call_setitem(self, x): x[None, :, 1, ..., None] = np.zeros(self.shape)[0, 0, :, None] @@ -448,7 +417,6 @@ class TestSetValueItemNone9(TestSetValueApi): # 1.5 item is list or Tensor of bol class TestSetValueItemBool1(TestSetValueApi): - def _call_setitem(self, x): x[[True, False]] = self.value @@ -457,7 +425,6 @@ class TestSetValueItemBool1(TestSetValueApi): class TestSetValueItemBool2(TestSetValueApi): - def _call_setitem(self, x): x[[False, False]] = self.value @@ -466,7 +433,6 @@ class TestSetValueItemBool2(TestSetValueApi): class TestSetValueItemBool3(TestSetValueApi): - def _call_setitem(self, x): x[[False, True]] = np.zeros(self.shape[2]) @@ -475,7 +441,6 @@ class TestSetValueItemBool3(TestSetValueApi): class TestSetValueItemBool4(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign(np.array([False, True])) x[idx] = np.zeros(self.shape[2]) @@ -485,19 +450,19 @@ class TestSetValueItemBool4(TestSetValueApi): class TestSetValueItemBool5(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign( - np.array([[False, True, False], [True, True, False]])) + np.array([[False, True, False], [True, True, False]]) + ) x[idx] = self.value def _get_answer(self): - self.data[np.array([[False, True, False], [True, True, - False]])] = self.value + self.data[ + np.array([[False, True, False], [True, True, False]]) + ] = self.value class TestSetValueItemBool6(TestSetValueApi): - def _call_setitem(self, x): x[0, ...] = 0 x[x > 0] = self.value @@ -508,9 +473,7 @@ class TestSetValueItemBool6(TestSetValueApi): def create_test_value_int32(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -531,9 +494,7 @@ create_test_value_int32(TestSetValueItemSlice5) def create_test_value_tensor_fp32(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "float32" @@ -559,7 +520,6 @@ create_test_value_tensor_fp32(TestSetValueItemSlice5) # 3. Test different shape of value class TestSetValueValueShape1(TestSetValueApi): - def set_value(self): self.value = np.array([3, 4, 5, 6]) # shape is (4,) @@ -571,7 +531,6 @@ class TestSetValueValueShape1(TestSetValueApi): class TestSetValueValueShape2(TestSetValueApi): - def set_value(self): self.value = np.array([[3, 4, 5, 6]]) # shape is (1,4) @@ -583,10 +542,10 @@ class TestSetValueValueShape2(TestSetValueApi): class TestSetValueValueShape3(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, 3]]) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ) # shape is (3,4) def _call_setitem(self, x): x[0] = self.value @@ -596,11 +555,12 @@ class TestSetValueValueShape3(TestSetValueApi): class TestSetValueValueShape4(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, - 3]]).astype(self.dtype) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ).astype( + self.dtype + ) # shape is (3,4) def _call_setitem(self, x): x[0] = paddle.assign(self.value) # x is Paddle.Tensor @@ -610,7 +570,6 @@ class TestSetValueValueShape4(TestSetValueApi): class TestSetValueValueShape5(TestSetValueApi): - def set_value(self): self.value = np.array([3, 3, 3]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_shape_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_shape_op_mlu.py index 7e743dc13c4..f18ba5532b4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_shape_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_shape_op_mlu.py @@ -19,6 +19,7 @@ import sys sys.path.append("..") from op_test import OpTest import paddle + # from paddle.fluid import core # from paddle.fluid.op import Operator @@ -27,7 +28,6 @@ SEED = 2022 class TestShape(OpTest): - def setUp(self): self.set_mlu() self.op_type = "shape" @@ -53,43 +53,36 @@ class TestShape(OpTest): class TestShape_fp16(TestShape): - def init_dtype(self): self.dtype = np.float16 class TestShape_double(TestShape): - def init_dtype(self): self.dtype = np.float64 class TestShape_int32(TestShape): - def init_dtype(self): self.dtype = np.int32 class TestShape_int64(TestShape): - def init_dtype(self): self.dtype = np.int64 class TestShape_int8(TestShape): - def init_dtype(self): self.dtype = np.int8 class TestShape_uint8(TestShape): - def init_dtype(self): self.dtype = np.uint8 class TestShape_bool(TestShape): - def init_dtype(self): self.dtype = bool diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_cross_entropy_with_logits_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_cross_entropy_with_logits_op_mlu.py index ead15510e19..7e594cb3db5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_cross_entropy_with_logits_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_cross_entropy_with_logits_op_mlu.py @@ -29,8 +29,7 @@ paddle.enable_static() class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -40,13 +39,14 @@ class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint(0, 2, (batch_size, num_classes)).astype( + self.dtype + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -71,10 +71,10 @@ class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): self.dtype = np.float32 -class TestSigmoidCrossEntropyWithLogitsOp3(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ +class TestSigmoidCrossEntropyWithLogitsOp3( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -84,13 +84,14 @@ class TestSigmoidCrossEntropyWithLogitsOp3(TestSigmoidCrossEntropyWithLogitsOp1 batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.uniform(0, 1, (batch_size, num_classes)).astype( + self.dtype + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -102,10 +103,10 @@ class TestSigmoidCrossEntropyWithLogitsOp3(TestSigmoidCrossEntropyWithLogitsOp1 self.outputs = {'Out': -term1 - term2} -class TestSigmoidCrossEntropyWithLogitsOp5(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ +class TestSigmoidCrossEntropyWithLogitsOp5( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -115,14 +116,14 @@ class TestSigmoidCrossEntropyWithLogitsOp5(TestSigmoidCrossEntropyWithLogitsOp1 batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, tuple(batch_size + [num_classes])).astype( - self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -134,10 +135,10 @@ class TestSigmoidCrossEntropyWithLogitsOp5(TestSigmoidCrossEntropyWithLogitsOp1 self.outputs = {'Out': -term1 - term2} -class TestSigmoidCrossEntropyWithLogitsOp6(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ +class TestSigmoidCrossEntropyWithLogitsOp6( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -147,14 +148,14 @@ class TestSigmoidCrossEntropyWithLogitsOp6(TestSigmoidCrossEntropyWithLogitsOp1 batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, tuple(batch_size + [num_classes])).astype( - self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.randint( + 0, 2, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } # Fw Pass is implemented as elementwise sigmoid followed by diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_op_mlu.py index b45dba73c5d..0d7e5b08ee1 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_op_mlu.py @@ -24,7 +24,6 @@ SEED = 2021 class TestMLUSigmoid(OpTest): - def setUp(self): self.op_type = "sigmoid" self.set_mlu() @@ -41,9 +40,9 @@ class TestMLUSigmoid(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.01 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -54,7 +53,6 @@ class TestMLUSigmoid(OpTest): class TestMLUSigmoidFp16(TestMLUSigmoid): - def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_size_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_size_op_mlu.py index a56ca8fe5c9..977359f8a96 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_size_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_size_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestSizeOp(OpTest): - def setUp(self): self.op_type = "size" self.shape = [] @@ -42,31 +41,26 @@ class TestSizeOp(OpTest): class TestRank1Tensor(TestSizeOp): - def config(self): self.shape = [2] class TestRank2Tensor(TestSizeOp): - def config(self): self.shape = [2, 3] class TestRank3Tensor(TestSizeOp): - def config(self): self.shape = [2, 3, 100] class TestLargeTensor(TestSizeOp): - def config(self): self.shape = [2**10] class TestSizeAPI(unittest.TestCase): - def test_size_static(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -80,17 +74,19 @@ class TestSizeAPI(unittest.TestCase): out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) exe = paddle.static.Executor(place=paddle.MLUPlace(0)) - res_1, res_2 = exe.run(feed={ - "x_1": input_1, - "x_2": input_2, - }, - fetch_list=[out_1, out_2]) - assert (np.array_equal(res_1, - np.array([np.size(input_1) - ]).astype("int64"))) - assert (np.array_equal(res_2, - np.array([np.size(input_2) - ]).astype("int64"))) + res_1, res_2 = exe.run( + feed={ + "x_1": input_1, + "x_2": input_2, + }, + fetch_list=[out_1, out_2], + ) + assert np.array_equal( + res_1, np.array([np.size(input_1)]).astype("int64") + ) + assert np.array_equal( + res_2, np.array([np.size(input_2)]).astype("int64") + ) def test_size_imperative(self): paddle.disable_static(paddle.MLUPlace(0)) @@ -100,8 +96,8 @@ class TestSizeAPI(unittest.TestCase): x_2 = paddle.to_tensor(input_2) out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) - assert (np.array_equal(out_1.numpy().item(0), np.size(input_1))) - assert (np.array_equal(out_2.numpy().item(0), np.size(input_2))) + assert np.array_equal(out_1.numpy().item(0), np.size(input_1)) + assert np.array_equal(out_2.numpy().item(0), np.size(input_2)) paddle.enable_static() def test_error(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_slice_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_slice_op_mlu.py index a9f21a24e68..bc23ea270b5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_slice_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_slice_op_mlu.py @@ -29,7 +29,6 @@ paddle.enable_static() # Situation 1: starts(list, no tensor), ends(list, no tensor) # 1.1 without attr(decrease) class TestSliceOp(OpTest): - def setUp(self): self.op_type = "slice" self.set_mlu() @@ -40,7 +39,7 @@ class TestSliceOp(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -55,9 +54,9 @@ class TestSliceOp(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -65,7 +64,6 @@ class TestSliceOp(OpTest): class TestCase1(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-3, 0, 2] @@ -76,7 +74,6 @@ class TestCase1(TestSliceOp): class TestCase2(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-3, 0, 2] @@ -88,7 +85,6 @@ class TestCase2(TestSliceOp): # 1.2 with attr(decrease) class TestSliceOp_decs_dim(OpTest): - def setUp(self): self.op_type = "slice" self.set_mlu() @@ -116,9 +112,9 @@ class TestSliceOp_decs_dim(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -126,7 +122,6 @@ class TestSliceOp_decs_dim(OpTest): class TestSliceOp_decs_dim_2(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [1, 0, 2] @@ -138,7 +133,6 @@ class TestSliceOp_decs_dim_2(TestSliceOp_decs_dim): class TestSliceOp_decs_dim_3(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-1, 0, 2] @@ -150,7 +144,6 @@ class TestSliceOp_decs_dim_3(TestSliceOp_decs_dim): class TestSliceOp_decs_dim_4(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype("float32") self.starts = [0, 1, 2, 3] @@ -162,7 +155,6 @@ class TestSliceOp_decs_dim_4(TestSliceOp_decs_dim): class TestSliceOp_decs_dim_5(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-1] @@ -174,7 +166,6 @@ class TestSliceOp_decs_dim_5(TestSliceOp_decs_dim): class TestSliceOp_decs_dim_6(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [0, 1, 2, 3] @@ -188,7 +179,6 @@ class TestSliceOp_decs_dim_6(TestSliceOp_decs_dim): # Situation 2: starts(list, have tensor), ends(list, no tensor) # without attr(decrease) class TestSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.set_mlu() @@ -196,8 +186,9 @@ class TestSliceOp_starts_ListTensor(OpTest): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int64') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int64') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.out} @@ -205,7 +196,7 @@ class TestSliceOp_starts_ListTensor(OpTest): 'axes': self.axes, 'starts': self.starts_infer, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -222,9 +213,9 @@ class TestSliceOp_starts_ListTensor(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -234,7 +225,6 @@ class TestSliceOp_starts_ListTensor(OpTest): # Situation 2: starts(list, have tensor), ends(list, no tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.set_mlu() @@ -242,8 +232,9 @@ class TestSliceOp_decs_dim_starts_ListTensor(OpTest): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} @@ -271,9 +262,9 @@ class TestSliceOp_decs_dim_starts_ListTensor(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -281,8 +272,8 @@ class TestSliceOp_decs_dim_starts_ListTensor(OpTest): class TestSliceOp_decs_dim_5_starts_ListTensor( - TestSliceOp_decs_dim_starts_ListTensor): - + TestSliceOp_decs_dim_starts_ListTensor +): def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-1] @@ -298,7 +289,6 @@ class TestSliceOp_decs_dim_5_starts_ListTensor( # Situation 3: starts(tensor), ends(list, no tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -306,7 +296,7 @@ class TestSliceOp_decs_dim_starts_OneTensor(OpTest): self.config() self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { @@ -330,15 +320,14 @@ class TestSliceOp_decs_dim_starts_OneTensor(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Situation 4: starts(tensor), ends(tensor) # without attr(decrease) class TestSliceOp_starts_OneTensor_ends_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -348,14 +337,14 @@ class TestSliceOp_starts_OneTensor_ends_OneTensor(OpTest): self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int64"), - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, #'starts': self.starts, #'ends': self.ends_infer, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -370,15 +359,14 @@ class TestSliceOp_starts_OneTensor_ends_OneTensor(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Situation 5: starts(tensor), ends(tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_and_ends_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -387,7 +375,7 @@ class TestSliceOp_decs_dim_starts_and_ends_OneTensor(OpTest): self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { @@ -411,15 +399,14 @@ class TestSliceOp_decs_dim_starts_and_ends_OneTensor(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Situation 6: starts(tensor), ends(list, have tensor) # without attr(decrease) class TestSliceOp_starts_OneTensor_ends_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -428,20 +415,21 @@ class TestSliceOp_starts_OneTensor_ends_ListTensor(OpTest): ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("y" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("y" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - 'EndsTensorList': ends_tensor + 'EndsTensorList': ends_tensor, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, #'starts': self.starts, 'ends': self.ends_infer, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -458,14 +446,13 @@ class TestSliceOp_starts_OneTensor_ends_ListTensor(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Test float16 class TestFP16(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -477,7 +464,7 @@ class TestFP16(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -493,13 +480,12 @@ class TestFP16(OpTest): self.check_output_with_place(self.place, atol=1e-5) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestFP16_2(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -511,7 +497,7 @@ class TestFP16_2(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -527,24 +513,28 @@ class TestFP16_2(OpTest): self.check_output_with_place(self.place, atol=1e-5) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006, - numeric_grad_delta=0.5) + self.check_grad_with_place( + self.place, + ['Input'], + 'Out', + max_relative_error=0.006, + numeric_grad_delta=0.5, + ) class TestSliceApiWithTensor(unittest.TestCase): - def test_starts_ends_is_tensor(self): with paddle.fluid.dygraph.guard(): a = paddle.rand(shape=[4, 5, 6], dtype='float32') axes = [0, 1, 2] starts = [-3, 0, 2] ends = [3, 2, 4] - a_1 = paddle.slice(a, - axes=axes, - starts=paddle.to_tensor(starts, dtype='int32'), - ends=paddle.to_tensor(ends, dtype='int32')) + a_1 = paddle.slice( + a, + axes=axes, + starts=paddle.to_tensor(starts, dtype='int32'), + ends=paddle.to_tensor(ends, dtype='int32'), + ) a_2 = paddle.slice(a, axes=axes, starts=starts, ends=ends) np.testing.assert_allclose(a_1.numpy(), a_2.numpy()) @@ -567,24 +557,22 @@ class TestSliceApiWithTensor(unittest.TestCase): class TestImperativeVarBaseGetItem(unittest.TestCase): - def test_getitem_with_long(self): with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') var = fluid.dygraph.to_variable(data) - sliced = var[:, 10:, :var.shape[1]] # var.shape[1] is 80L here + sliced = var[:, 10:, : var.shape[1]] # var.shape[1] is 80L here self.assertEqual(sliced.shape, [2, 70, 80]) - sliced = var[:, var.shape[0]:, var.shape[0]:var.shape[1]] + sliced = var[:, var.shape[0] :, var.shape[0] : var.shape[1]] self.assertEqual(sliced.shape, [2, 78, 78]) def test_getitem_with_float(self): - def test_float_in_slice_item(): with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') var = fluid.dygraph.to_variable(data) - sliced = var[:, 1.1:, :var.shape[1]] + sliced = var[:, 1.1:, : var.shape[1]] self.assertRaises(Exception, test_float_in_slice_item) @@ -598,7 +586,6 @@ class TestImperativeVarBaseGetItem(unittest.TestCase): class TestInferShape(unittest.TestCase): - def test_axis_less_than_zero(self): # Using paddle.disable_static will make other unittests fail. @@ -606,13 +593,18 @@ class TestInferShape(unittest.TestCase): x_arr = np.arange(0, 24, dtype=np.float32).reshape([2, 3, 4]) x = paddle.to_tensor(x_arr) - pp_slice = paddle.slice(x, [ - 100, - ], [0], [1]) + pp_slice = paddle.slice( + x, + [ + 100, + ], + [0], + [1], + ) np_slice = x_arr[:, :, 0:1] np.testing.assert_allclose(pp_slice, np_slice) - pp_slice = paddle.slice(x, (-100, ), [0], [1]) + pp_slice = paddle.slice(x, (-100,), [0], [1]) np_slice = x_arr[0:1] np.testing.assert_allclose(pp_slice, np_slice) @@ -620,9 +612,11 @@ class TestInferShape(unittest.TestCase): x = paddle.to_tensor(np.reshape(x_arr, (0, 0, 0))) starts = paddle.to_tensor( - np.reshape(np.array([], dtype=np.int32), (0, ))) + np.reshape(np.array([], dtype=np.int32), (0,)) + ) ends = paddle.to_tensor( - np.reshape(np.array([], dtype=np.int32), (0, ))) + np.reshape(np.array([], dtype=np.int32), (0,)) + ) with self.assertRaises(ValueError): paddle.slice(x, [-1000000], starts, ends) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_softmax_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_softmax_op_mlu.py index b9362c9a27d..8c1ebbe01e5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_softmax_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_softmax_op_mlu.py @@ -32,7 +32,7 @@ def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -47,7 +47,6 @@ def ref_softmax(x, axis=None, dtype=None): class TestSoftmaxOp(OpTest): - def get_x_shape(self): return [10, 10] @@ -79,19 +78,17 @@ class TestSoftmaxOp(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ["X"], - "Out", - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ["X"], "Out", max_relative_error=0.01 + ) class TestSoftmaxOp2(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] class TestSoftmaxOp3(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -100,7 +97,6 @@ class TestSoftmaxOp3(TestSoftmaxOp): class TestSoftmaxOp4(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -109,7 +105,6 @@ class TestSoftmaxOp4(TestSoftmaxOp): class TestSoftmaxOp5(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -118,7 +113,6 @@ class TestSoftmaxOp5(TestSoftmaxOp): class TestSoftmaxOp6(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -127,10 +121,9 @@ class TestSoftmaxOp6(TestSoftmaxOp): class TestSoftmaxAPI(unittest.TestCase): - def setUp(self): self.place = paddle.MLUPlace(0) - self.x_np = np.random.uniform(-1., 1., [2, 3, 4, 5]).astype('float32') + self.x_np = np.random.uniform(-1.0, 1.0, [2, 3, 4, 5]).astype('float32') self.out_ref = np.apply_along_axis(stable_softmax, -1, self.x_np) self.executed_api() @@ -180,19 +173,18 @@ class TestSoftmaxAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, self.softmax, 1) # The input dtype must be float16, float32 - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) self.assertRaises(TypeError, self.softmax, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[2, 3], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[2, 3], dtype='float16' + ) self.softmax(x_fp16) class TestSoftmaxInplaceAPI(TestSoftmaxAPI): - def executed_api(self): self.softmax = F.softmax_ diff --git a/python/paddle/fluid/tests/unittests/mlu/test_softmax_with_cross_entropy_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_softmax_with_cross_entropy_op_mlu.py index e7ae6dd5111..bf77d525329 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_softmax_with_cross_entropy_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_softmax_with_cross_entropy_op_mlu.py @@ -28,7 +28,6 @@ SEED = 2021 class TestSoftmaxWithCrossEntropyOp(OpTest): - def set_mlu(self): self.__class__.use_mlu = True @@ -51,8 +50,10 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): self.initParams() logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, logits) if self.soft_label: @@ -63,8 +64,9 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): self.shape[self.axis] = 1 labels = np.random.randint(0, axis_dim, self.shape, dtype="int64") - loss = cross_entropy(softmax, labels, self.soft_label, self.axis, - self.ignore_index) + loss = cross_entropy( + softmax, labels, self.soft_label, self.axis, self.ignore_index + ) one_hot_label = np.eye(axis_dim)[labels.reshape(-1)] @@ -72,7 +74,7 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): self.outputs = { "Backprop": (softmax - one_hot_label).astype(self.dtype), "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -90,14 +92,16 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): if self.dtype == np.float16: return # fp32 has low precision, cpu and mlu both need to relax the max_relative_error if using fp32 - self.check_grad_with_place(self.place, ['Logits'], - 'Loss', - numeric_grad_delta=0.001, - max_relative_error=0.5) + self.check_grad_with_place( + self.place, + ['Logits'], + 'Loss', + numeric_grad_delta=0.001, + max_relative_error=0.5, + ) class TestPowNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -112,9 +116,9 @@ class TestPowNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -138,16 +142,17 @@ class TestPowNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_spawn_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_spawn_mlu.py index e97f4deead5..605c2693afc 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_spawn_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_spawn_mlu.py @@ -19,12 +19,15 @@ import paddle import paddle.nn as nn import paddle.optimizer as opt import paddle.distributed as dist -from paddle.distributed.spawn import _get_subprocess_env_list, _options_valid_check, _get_default_nprocs +from paddle.distributed.spawn import ( + _get_subprocess_env_list, + _options_valid_check, + _get_default_nprocs, +) from paddle.fluid import core class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear1 = nn.Linear(10, 10) @@ -62,7 +65,6 @@ def train(print_result=False): class TestSpawn(unittest.TestCase): - def test_nprocs_greater_than_device_num_error(self): with self.assertRaises(RuntimeError): _get_subprocess_env_list(nprocs=100, options=dict()) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_split_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_split_op_mlu.py index 28585629dca..072b985613e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_split_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_split_op_mlu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestCase1(OpTest): - def setUp(self): self.set_mlu() self.set_example() @@ -35,9 +34,9 @@ class TestCase1(OpTest): self.place = paddle.device.MLUPlace(0) ipt = self.x.astype(self.dtype) axis = self.axis if isinstance(self.axis, int) else int(self.axis[0]) - tmp_outs = np.split(ipt, - axis=axis, - indices_or_sections=self.num_or_sections) + tmp_outs = np.split( + ipt, axis=axis, indices_or_sections=self.num_or_sections + ) tmp_outs = [o.astype(self.dtype) for o in tmp_outs] self.outputs = {'Out': []} self.outs = [] @@ -64,7 +63,6 @@ class TestCase1(OpTest): class TestCase2(TestCase1): - def set_example(self): self.dtype = "float32" self.x = np.random.random((20, 4, 50)) @@ -73,7 +71,6 @@ class TestCase2(TestCase1): class TestCase4(TestCase1): - def set_example(self): self.dtype = "float16" self.x = np.random.random((4, 50, 20)) @@ -83,7 +80,6 @@ class TestCase4(TestCase1): # Test Sections class TestCase5(TestCase1): - def set_example(self): super().set_example() self.x = np.random.random((2, 10, 4)) @@ -96,7 +92,6 @@ class TestCase5(TestCase1): class API_TestSplit(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float32') @@ -105,13 +100,12 @@ class API_TestSplit(unittest.TestCase): exe = fluid.Executor(place) input1 = np.random.random([1, 10]).astype('float32') r0, r1 = exe.run(feed={"data": input1}, fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (3, ), axis=1) + ex_x0, ex_x1 = np.split(input1, (3,), axis=1) np.testing.assert_allclose(ex_x0, r0) np.testing.assert_allclose(ex_x1, r1) class API_TestSplit2(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float32') @@ -126,7 +120,6 @@ class API_TestSplit2(unittest.TestCase): class API_TestDygraphSplit(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(paddle.MLUPlace(0)): input_1 = np.random.random([4, 6, 6]).astype("int32") @@ -158,20 +151,18 @@ class API_TestDygraphSplit(unittest.TestCase): # attr(axis) is Tensor class TestSplitOp_AxisTensor(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() self.init_data() self.inputs = { 'X': self.x, - 'AxisTensor': np.array([self.axis]).astype("int32") + 'AxisTensor': np.array([self.axis]).astype("int32"), } self.attrs = {'sections': self.sections, 'num': self.num} out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -194,7 +185,6 @@ class TestSplitOp_AxisTensor(OpTest): class TestSplitOp_SectionsTensor(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -203,20 +193,20 @@ class TestSplitOp_SectionsTensor(OpTest): sections_tensor = [] for index, ele in enumerate(self.sections): - sections_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + sections_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SectionsTensorList'] = sections_tensor self.attrs = { 'axis': self.axis, 'sections': self.sections_infer, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sqrt_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sqrt_op_mlu.py index cc02f975abb..211c8e349eb 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sqrt_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sqrt_op_mlu.py @@ -29,7 +29,6 @@ np.random.seed(10) class TestSqrt(OpTest): - def setUp(self): self.op_type = "sqrt" self.dtype = 'float32' @@ -55,7 +54,6 @@ class TestSqrt(OpTest): class TestSqrtHalf(OpTest): - def setUp(self): self.op_type = "sqrt" self.dtype = 'float16' @@ -74,10 +72,9 @@ class TestSqrtHalf(OpTest): self.place = paddle.device.MLUPlace(0) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - check_eager=False, - max_relative_error=0.85) + self.check_grad_with_place( + self.place, ['X'], 'Out', check_eager=False, max_relative_error=0.85 + ) def test_check_output(self): self.check_output_with_place(self.place) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_squared_l2_norm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_squared_l2_norm_op_mlu.py index a08b2f263af..27642b92c3d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_squared_l2_norm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_squared_l2_norm_op_mlu.py @@ -26,8 +26,7 @@ paddle.enable_static() class TestL2LossOp(OpTest): - """Test squared_l2_norm - """ + """Test squared_l2_norm""" def setUp(self): self.place = paddle.device.MLUPlace(0) @@ -44,13 +43,12 @@ class TestL2LossOp(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=self.max_relative_error) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=self.max_relative_error + ) class TestL2LossDeterministic(unittest.TestCase): - def check_place(self, place): with paddle.fluid.dygraph.guard(place): x_np = np.random.rand(5, 11, 13).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/mlu/test_squeeze2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_squeeze2_op_mlu.py index bddd71ce2e1..62bd3d86b45 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_squeeze2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_squeeze2_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() # Correct: General. class TestSqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -34,7 +33,7 @@ class TestSqueezeOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -58,7 +57,6 @@ class TestSqueezeOp(OpTest): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -67,7 +65,6 @@ class TestSqueezeOp1(TestSqueezeOp): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -76,7 +73,6 @@ class TestSqueezeOp2(TestSqueezeOp): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_squeeze_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_squeeze_op_mlu.py index 24f02492c63..9df87453929 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_squeeze_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_squeeze_op_mlu.py @@ -30,7 +30,6 @@ paddle.enable_static() # Correct: General. class TestSqueezeOp(OpTest): - def setUp(self): self.op_type = "squeeze" self.init_test_case() @@ -61,7 +60,6 @@ class TestSqueezeOp(OpTest): class TestSqueezeBF16Op(OpTest): - def setUp(self): self.op_type = "squeeze" self.dtype = np.uint16 @@ -89,7 +87,6 @@ class TestSqueezeBF16Op(OpTest): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 3, 1, 40) self.axes = (0, -2) @@ -98,7 +95,6 @@ class TestSqueezeOp1(TestSqueezeOp): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -107,7 +103,6 @@ class TestSqueezeOp2(TestSqueezeOp): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -116,7 +111,6 @@ class TestSqueezeOp3(TestSqueezeOp): # Correct: The demension of axis is not of size 1 remains unchanged. class TestSqueezeOp4(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, 2) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_stack_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_stack_op_mlu.py index 1d4550a11a5..eefe1d7d691 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_stack_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_stack_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestStackOpBase(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -49,7 +48,8 @@ class TestStackOpBase(OpTest): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(self.dtype)) + np.random.random(size=self.input_dim).astype(self.dtype) + ) tmp = [] x_names = self.get_x_names() @@ -73,61 +73,51 @@ class TestStackOpBase(OpTest): class TestStackOp1(TestStackOpBase): - def initParameters(self): self.num_inputs = 16 class TestStackOp2(TestStackOpBase): - def initParameters(self): self.num_inputs = 20 class TestStackOp3(TestStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestStackOpBase): - def initParameters(self): self.axis = -4 class TestStackOp5(TestStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestStackOpBase): - def initParameters(self): self.axis = 3 class TestStackOpINT32(TestStackOpBase): - def init_dtype(self): self.dtype = np.int32 class TestStackOpINT64(TestStackOpBase): - def init_dtype(self): self.dtype = np.int64 class TestStackOpHalf(TestStackOpBase): - def init_dtype(self): self.dtype = np.float16 class API_test(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[1, 2], dtype='float32') @@ -139,12 +129,10 @@ class API_test(unittest.TestCase): input1 = np.random.random([1, 2]).astype('float32') input2 = np.random.random([1, 2]).astype('float32') input3 = np.random.random([1, 2]).astype('float32') - result, = exe.run(feed={ - "data1": input1, - "data2": input2, - "data3": input3 - }, - fetch_list=[result_stack]) + (result,) = exe.run( + feed={"data1": input1, "data2": input2, "data3": input3}, + fetch_list=[result_stack], + ) expected_result = np.stack([input1, input2, input3], axis=0) np.testing.assert_allclose(expected_result, result) @@ -155,7 +143,6 @@ class API_test(unittest.TestCase): class API_DygraphTest(unittest.TestCase): - def test_out(self): data1 = np.array([[1.0, 2.0]]).astype("float32") data2 = np.array([[3.0, 4.0]]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_strided_slice_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_strided_slice_op_mlu.py index 21388f1131f..35069485054 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_strided_slice_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_strided_slice_op_mlu.py @@ -26,16 +26,15 @@ paddle.enable_static() class TestStrideSliceOp(OpTest): - def setUp(self): self.initTestCase() self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True self.op_type = 'strided_slice' self.python_api = paddle.strided_slice - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.inputs = {'Input': self.input} self.outputs = {'Out': self.output} @@ -44,17 +43,16 @@ class TestStrideSliceOp(OpTest): 'starts': self.starts, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def test_check_output(self): self.check_output_with_place(self.place, check_eager=False) def test_check_grad(self): - self.check_grad_with_place(self.place, - set(['Input']), - 'Out', - check_eager=False) + self.check_grad_with_place( + self.place, set(['Input']), 'Out', check_eager=False + ) def initTestCase(self): self.input = np.random.rand(100).astype(np.float32) @@ -66,7 +64,6 @@ class TestStrideSliceOp(OpTest): class TestStrideSliceOp1(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100).astype(np.float32) self.axes = [0] @@ -77,7 +74,6 @@ class TestStrideSliceOp1(TestStrideSliceOp): class TestStrideSliceOp2(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100).astype(np.float32) self.axes = [0] @@ -88,7 +84,6 @@ class TestStrideSliceOp2(TestStrideSliceOp): class TestStrideSliceOp3(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100).astype(np.float32) self.axes = [0] @@ -99,7 +94,6 @@ class TestStrideSliceOp3(TestStrideSliceOp): class TestStrideSliceOp4(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 4, 10).astype(np.float32) self.axes = [0, 1, 2] @@ -110,7 +104,6 @@ class TestStrideSliceOp4(TestStrideSliceOp): class TestStrideSliceOp5(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5).astype(np.float32) self.axes = [0, 1, 2] @@ -121,7 +114,6 @@ class TestStrideSliceOp5(TestStrideSliceOp): class TestStrideSliceOp6(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5).astype(np.float32) self.axes = [0, 1, 2] @@ -132,7 +124,6 @@ class TestStrideSliceOp6(TestStrideSliceOp): class TestStrideSliceOp7(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5).astype(np.float32) self.axes = [0, 1, 2] @@ -143,7 +134,6 @@ class TestStrideSliceOp7(TestStrideSliceOp): class TestStrideSliceOp8(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1).astype(np.float32) self.axes = [1] @@ -154,7 +144,6 @@ class TestStrideSliceOp8(TestStrideSliceOp): class TestStrideSliceOp9(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1).astype(np.float32) self.axes = [1] @@ -165,7 +154,6 @@ class TestStrideSliceOp9(TestStrideSliceOp): class TestStrideSliceOp10(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(10, 10).astype(np.float32) self.axes = [0, 1] @@ -176,7 +164,6 @@ class TestStrideSliceOp10(TestStrideSliceOp): class TestStrideSliceOp11(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4).astype(np.float32) self.axes = [0, 1, 2, 3] @@ -187,7 +174,6 @@ class TestStrideSliceOp11(TestStrideSliceOp): class TestStrideSliceOp12(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5).astype(np.float32) self.axes = [0, 1, 2, 3, 4] @@ -198,7 +184,6 @@ class TestStrideSliceOp12(TestStrideSliceOp): class TestStrideSliceOp13(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype(np.float32) self.axes = [0, 1, 2, 3, 4, 5] @@ -209,7 +194,6 @@ class TestStrideSliceOp13(TestStrideSliceOp): class TestStrideSliceOp14(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(4, 4, 4, 4).astype(np.float32) self.axes = [1, 2, 3] @@ -220,13 +204,11 @@ class TestStrideSliceOp14(TestStrideSliceOp): class TestStrideSliceOpBool(TestStrideSliceOp): - def test_check_grad(self): pass class TestStrideSliceOpBool1D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(100).astype("bool") self.axes = [0] @@ -237,7 +219,6 @@ class TestStrideSliceOpBool1D(TestStrideSliceOpBool): class TestStrideSliceOpBool2D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(10, 10).astype("bool") self.axes = [0, 1] @@ -248,7 +229,6 @@ class TestStrideSliceOpBool2D(TestStrideSliceOpBool): class TestStrideSliceOpBool3D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 4, 10).astype("bool") self.axes = [0, 1, 2] @@ -259,7 +239,6 @@ class TestStrideSliceOpBool3D(TestStrideSliceOpBool): class TestStrideSliceOpBool4D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4).astype("bool") self.axes = [0, 1, 2, 3] @@ -270,7 +249,6 @@ class TestStrideSliceOpBool4D(TestStrideSliceOpBool): class TestStrideSliceOpBool5D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5).astype("bool") self.axes = [0, 1, 2, 3, 4] @@ -281,7 +259,6 @@ class TestStrideSliceOpBool5D(TestStrideSliceOpBool): class TestStrideSliceOpBool6D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype("bool") self.axes = [0, 1, 2, 3, 4, 5] @@ -292,7 +269,6 @@ class TestStrideSliceOpBool6D(TestStrideSliceOpBool): class TestStridedSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -301,8 +277,9 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.output} @@ -311,7 +288,7 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): 'starts': self.starts_infer, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -321,9 +298,9 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.starts_infer = [1, 10, 2] @@ -331,13 +308,12 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_ends_ListTensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -346,8 +322,9 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'EndsTensorList': ends_tensor} self.outputs = {'Out': self.output} @@ -356,7 +333,7 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): 'starts': self.starts, 'ends': self.ends_infer, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -366,9 +343,9 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 2] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.ends_infer = [3, 1, 4] @@ -376,13 +353,12 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_starts_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -390,7 +366,7 @@ class TestStridedSliceOp_starts_Tensor(OpTest): self.config() self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -408,21 +384,20 @@ class TestStridedSliceOp_starts_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_ends_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -430,7 +405,7 @@ class TestStridedSliceOp_ends_Tensor(OpTest): self.config() self.inputs = { 'Input': self.input, - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -448,35 +423,35 @@ class TestStridedSliceOp_ends_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_listTensor_Tensor(OpTest): - def setUp(self): self.config() self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.op_type = "strided_slice" self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensorList": ends_tensor + "EndsTensorList": ends_tensor, } self.outputs = {'Out': self.output} self.attrs = { @@ -494,21 +469,20 @@ class TestStridedSliceOp_listTensor_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_strides_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -516,7 +490,7 @@ class TestStridedSliceOp_strides_Tensor(OpTest): self.config() self.inputs = { 'Input': self.input, - "StridesTensor": np.array(self.strides, dtype="int32") + "StridesTensor": np.array(self.strides, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -534,63 +508,65 @@ class TestStridedSliceOp_strides_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, -1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Test python API class TestStridedSliceAPI(unittest.TestCase): - def test_1(self): input = np.random.random([3, 4, 5, 6]).astype("float32") minus_1 = fluid.layers.fill_constant([1], "int32", -1) minus_3 = fluid.layers.fill_constant([1], "int32", -3) - starts = fluid.layers.data(name='starts', - shape=[3], - dtype='int32', - append_batch_size=False) - ends = fluid.layers.data(name='ends', - shape=[3], - dtype='int32', - append_batch_size=False) - strides = fluid.layers.data(name='strides', - shape=[3], - dtype='int32', - append_batch_size=False) - - x = fluid.layers.data(name="x", - shape=[3, 4, 5, 6], - append_batch_size=False, - dtype="float32") - out_1 = paddle.strided_slice(x, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_2 = paddle.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_3 = paddle.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, minus_1], - strides=[1, 1, 1]) - out_4 = paddle.strided_slice(x, - axes=[0, 1, 2], - starts=starts, - ends=ends, - strides=strides) + starts = fluid.layers.data( + name='starts', shape=[3], dtype='int32', append_batch_size=False + ) + ends = fluid.layers.data( + name='ends', shape=[3], dtype='int32', append_batch_size=False + ) + strides = fluid.layers.data( + name='strides', shape=[3], dtype='int32', append_batch_size=False + ) + + x = fluid.layers.data( + name="x", + shape=[3, 4, 5, 6], + append_batch_size=False, + dtype="float32", + ) + out_1 = paddle.strided_slice( + x, + axes=[0, 1, 2], + starts=[-3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_2 = paddle.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_3 = paddle.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, minus_1], + strides=[1, 1, 1], + ) + out_4 = paddle.strided_slice( + x, axes=[0, 1, 2], starts=starts, ends=ends, strides=strides + ) out_5 = x[-3:3, 0:100:2, -1:2:-1] out_6 = x[minus_3:3:1, 0:100:2, :, minus_1:2:minus_1] @@ -603,9 +579,10 @@ class TestStridedSliceAPI(unittest.TestCase): "x": input, 'starts': np.array([-3, 0, 2]).astype("int32"), 'ends': np.array([3, 2147483648, -1]).astype("int64"), - 'strides': np.array([1, 1, 1]).astype("int32") + 'strides': np.array([1, 1, 1]).astype("int32"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, input[-3:3, 0:100, 2:-1, :]) assert np.array_equal(res_2, input[-3:3, 0:100, :, 2:-1]) assert np.array_equal(res_3, input[-3:3, 0:100, :, 2:-1]) @@ -620,11 +597,9 @@ class TestStridedSliceAPI(unittest.TestCase): starts = [-3, 0, 2] ends = [3, 2, 4] strides_1 = [1, 1, 1] - sliced_1 = paddle.strided_slice(x, - axes=axes, - starts=starts, - ends=ends, - strides=strides_1) + sliced_1 = paddle.strided_slice( + x, axes=axes, starts=starts, ends=ends, strides=strides_1 + ) assert sliced_1.shape == (3, 2, 2, 2) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sum_op_mlu.py index ebf00a9db08..0dae84dc79b 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_sum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sum_op_mlu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestSum1(OpTest): - def setUp(self): self.set_mlu() self.init_dtype() @@ -54,7 +53,6 @@ class TestSum1(OpTest): class TestSum2(OpTest): - def setUp(self): self.set_mlu() self.init_dtype() @@ -72,8 +70,12 @@ class TestSum2(OpTest): # For example, the results of `x0+x1+x2+x3` is different from that of # `x3+x2+x1+x0` if the dtype is fp16. # Therefore, converting the input to fp32 for calculation. - y = (x0.astype(np.float32) + x1.astype(np.float32) + - x2.astype(np.float32) + x3.astype(np.float32)).astype(self.dtype) + y = ( + x0.astype(np.float32) + + x1.astype(np.float32) + + x2.astype(np.float32) + + x3.astype(np.float32) + ).astype(self.dtype) self.outputs = {'Out': y} self.attrs = {'use_mkldnn': False} @@ -89,7 +91,6 @@ class TestSum2(OpTest): class TestSum3(OpTest): - def setUp(self): self.set_mlu() self.init_dtype() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_base_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_base_mlu.py index 0d73e5af5c5..dd842800198 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_base_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_base_mlu.py @@ -41,17 +41,19 @@ SEED = 10 class TestSyncBatchNormRunnerBase(object): - - def get_model(self, - main, - startup, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def get_model( + self, + main, + startup, + place, + layout, + seed, + sync_bn=False, + only_forward=False, + ): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): assert not isinstance(endpoints, str) @@ -60,13 +62,15 @@ class TestSyncBatchNormRunnerBase(object): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -74,39 +78,47 @@ class TestSyncBatchNormRunnerBase(object): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - cncl_id_var = block.create_var(name=nameGen.generate('cncl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_cncl_id', - inputs={}, - outputs={'Out': cncl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - block.append_op(type='c_comm_init', - inputs={'X': cncl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': self.global_ring_id - }) + cncl_id_var = block.create_var( + name=nameGen.generate('cncl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_cncl_id', + inputs={}, + outputs={'Out': cncl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': cncl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': self.global_ring_id, + }, + ) def run_trainer(self, args): device_id = int(os.getenv("FLAGS_selected_mlus", "0")) @@ -139,24 +151,30 @@ class TestSyncBatchNormRunnerBase(object): sys.stdout.buffer.write( pickle.dumps( - 'training, inference, fp32, fp16, NCHW, NHWC all passed')) + 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) + ) def _compare(self, args, place, layout, only_forward): scope = core.Scope() np.random.seed(SEED) - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 sys.stderr.write("data: " + str(data) + "\n") - data = create_or_get_tensor(scope, "input", - OpTest.np_dtype_to_fluid_dtype(data), place) + data = create_or_get_tensor( + scope, "input", OpTest.np_dtype_to_fluid_dtype(data), place + ) - bn_fetches = self._cal_single_card(args, data, place, layout, - only_forward) + bn_fetches = self._cal_single_card( + args, data, place, layout, only_forward + ) fetch_names, sync_bn_fetches = self._cal_multiple_cards( - args, data, place, layout, only_forward) + args, data, place, layout, only_forward + ) - sys.stderr.write("len(sync_bn_fetches): " + str(len(sync_bn_fetches)) + - "\n") + sys.stderr.write( + "len(sync_bn_fetches): " + str(len(sync_bn_fetches)) + "\n" + ) for i in range(0, len(sync_bn_fetches)): sys.stderr.write("i: " + str(i) + "\n") sys.stderr.write("fetch_names[i]): " + fetch_names[i] + "\n") @@ -164,13 +182,14 @@ class TestSyncBatchNormRunnerBase(object): bn_val = bn_fetches[i] sync_bn_val = sync_bn_fetches[i] if sync_bn_val.shape != bn_val.shape: - sync_bn_val = sync_bn_val[:bn_val.shape[0]] + sync_bn_val = sync_bn_val[: bn_val.shape[0]] # i = 0 if fetch_names[i] == 'reduce_sum_0.tmp_0': # sys.stderr.write("skip reduce_sum_0.tmp_0 (Out of reduce_sum op)" + "\n") - sys.stderr.write("reduce_sum_0.tmp_0 (Out of reduce_sum op)" + - "\n") + sys.stderr.write( + "reduce_sum_0.tmp_0 (Out of reduce_sum op)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -198,7 +217,8 @@ class TestSyncBatchNormRunnerBase(object): if fetch_names[i] == 'batch_norm_0.tmp_2': # sys.stderr.write("skip batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n") sys.stderr.write( - "batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n") + "batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -231,8 +251,9 @@ class TestSyncBatchNormRunnerBase(object): # i = 8 if fetch_names[i] == 'batch_norm_0.tmp_1': - sys.stderr.write("skip batch_norm_0.tmp_1 (SavedVariance)" + - "\n") + sys.stderr.write( + "skip batch_norm_0.tmp_1 (SavedVariance)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -278,10 +299,16 @@ class TestSyncBatchNormRunnerBase(object): if fetch_names[i] == 'conv2d_0.tmp_0@GRAD': atol = 1e-2 - assert np.allclose( - bn_val, sync_bn_val, atol=atol), "Output (" + fetch_names[ - i] + ") has diff. \n" + "\nBN " + str( - bn_val) + "\n" + "Sync BN " + str(sync_bn_val) + assert np.allclose(bn_val, sync_bn_val, atol=atol), ( + "Output (" + + fetch_names[i] + + ") has diff. \n" + + "\nBN " + + str(bn_val) + + "\n" + + "Sync BN " + + str(sync_bn_val) + ) def _cal_single_card(self, args, data, place, layout, only_forward): # Single-MLU, N = 32 per MLU @@ -291,23 +318,31 @@ class TestSyncBatchNormRunnerBase(object): startup_prog.global_seed(SEED) paddle.seed(SEED) - outs = self.get_model(train_prog, startup_prog, place, layout, SEED, - False, only_forward) + outs = self.get_model( + train_prog, startup_prog, place, layout, SEED, False, only_forward + ) exe = fluid.Executor(place) exe.run(startup_prog) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - bn_fetches = exe.run(program=train_prog, - feed={'input': data}, - fetch_list=fetch_names) + bn_fetches = exe.run( + program=train_prog, feed={'input': data}, fetch_list=fetch_names + ) return bn_fetches @@ -328,8 +363,9 @@ class TestSyncBatchNormRunnerBase(object): current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) # sys.stderr.write("after init, startup_prog: " + # startup_prog.to_string(True) + "\n") train_prog.global_seed(SEED) @@ -339,8 +375,9 @@ class TestSyncBatchNormRunnerBase(object): paddle.seed(SEED) self.rank = rank - outs = self.get_model(train_prog, startup_prog, place, layout, SEED, - True, only_forward) + outs = self.get_model( + train_prog, startup_prog, place, layout, SEED, True, only_forward + ) # sys.stderr.write("after get_model, train_prog: " + # train_prog.to_string(True) + "\n") # sys.stderr.write("after get_model, startup_prog: " + @@ -363,17 +400,24 @@ class TestSyncBatchNormRunnerBase(object): exe = fluid.Executor(place) exe.run(startup_prog) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - sync_bn_fetches = exe.run(program=train_prog, - feed={'input': data}, - fetch_list=fetch_names) + sync_bn_fetches = exe.run( + program=train_prog, feed={'input': data}, fetch_list=fetch_names + ) return fetch_names, sync_bn_fetches @@ -396,19 +440,20 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -437,7 +482,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) @@ -448,15 +493,19 @@ class TestDistBase(unittest.TestCase): tr1_pipe = open("/tmp/tr1_err_%d.log" % os.getpid(), "w") print("tr0_cmd: {}, env: {}\n".format(tr0_cmd, env0)) print("tr1_cmd: {}, env: {}\n".format(tr1_cmd, env1)) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -470,14 +519,16 @@ class TestDistBase(unittest.TestCase): sys.stderr.write('trainer 0 stderr file: %s\n' % f.read()) with open("/tmp/tr1_err_%d.log" % os.getpid(), "r") as f: sys.stderr.write('trainer 1 stderr file: %s\n' % f.read()) - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - check_error_log=False, - need_envs={}): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, model_file, col_type, check_error_log=False, need_envs={} + ): required_envs = { "FLAGS_fraction_of_gpu_memory_to_use": "0.15", "FLAGS_eager_delete_tensor_gb": "0.0", @@ -488,7 +539,7 @@ class TestDistBase(unittest.TestCase): "FLAGS_call_stack_level": "2", "GLOG_v": "3", "PADDLE_WITH_GLOO": '0', - "BACKEND": "cncl" + "BACKEND": "cncl", } required_envs.update(need_envs) if check_error_log: @@ -496,8 +547,11 @@ class TestDistBase(unittest.TestCase): required_envs["GLOG_logtostderr"] = "1" required_envs["GLOO_LOG_LEVEL"] = "TRACE" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) self.assertEqual( - tr0_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed') + tr0_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) self.assertEqual( - tr1_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed') + tr1_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_baseline.py b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_baseline.py index 44dc8cf2359..597c033ba78 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_baseline.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_baseline.py @@ -28,14 +28,13 @@ paddle.enable_static() class TestSyncBatchNormOp(TestDistBase): - def _setup_config(self): pass def test_identity(self, col_type="identity"): - self.check_with_place("sync_batch_norm_op_mlu.py", - col_type, - check_error_log=True) + self.check_with_place( + "sync_batch_norm_op_mlu.py", col_type, check_error_log=True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_extra.py b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_extra.py index 6488f757c50..fd19512528b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_extra.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_extra.py @@ -34,15 +34,15 @@ paddle.enable_static() class TestDygraphSyncBatchNormAPIError(unittest.TestCase): - def test_errors(self): if not core.is_compiled_with_mlu(): return with program_guard(Program(), Program()): my_sync_batch_norm = paddle.nn.SyncBatchNorm(10) - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.MLUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.MLUPlace(0) + ) self.assertRaises(TypeError, my_sync_batch_norm, x1) # the input dtype of SyncBatchNorm must be float16 or float32 @@ -51,36 +51,39 @@ class TestDygraphSyncBatchNormAPIError(unittest.TestCase): class TestConvertSyncBatchNorm(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_mlu(): return with program_guard(Program(), Program()): - compare_model = paddle.nn.Sequential(paddle.nn.Conv2D(3, 5, 3), - paddle.nn.BatchNorm2D(5), - paddle.nn.BatchNorm2D(5)) + compare_model = paddle.nn.Sequential( + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), + paddle.nn.BatchNorm2D(5), + ) model = paddle.nn.Sequential( - paddle.nn.Conv2D(3, 5, 3), paddle.nn.BatchNorm2D(5), + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), paddle.nn.BatchNorm2D( 5, weight_attr=fluid.ParamAttr(name='bn.scale'), - bias_attr=fluid.ParamAttr(name='bn.bias'))) + bias_attr=fluid.ParamAttr(name='bn.bias'), + ), + ) model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model) for idx, sublayer in enumerate(compare_model.sublayers()): if isinstance(sublayer, paddle.nn.BatchNorm2D): self.assertEqual( - isinstance(model[idx], paddle.nn.SyncBatchNorm), True) + isinstance(model[idx], paddle.nn.SyncBatchNorm), True + ) class TestConvertSyncBatchNormCast1(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_mlu(): return class Net(nn.Layer): - def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2D(3, 5, 3) @@ -105,7 +108,6 @@ class TestConvertSyncBatchNormCast1(unittest.TestCase): class TestConvertSyncBatchNormCase2(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_mlu(): return @@ -113,16 +115,19 @@ class TestConvertSyncBatchNormCase2(unittest.TestCase): with fluid.dygraph.guard(fluid.MLUPlace(0)): class SyBNNet(paddle.nn.Layer): - def __init__(self, in_ch=3, out_ch=3, dirate=1): super(SyBNNet, self).__init__() self.bn_s1 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( paddle.nn.BatchNorm3D( out_ch, weight_attr=paddle.ParamAttr( - regularizer=paddle.regularizer.L2Decay(0.)))) + regularizer=paddle.regularizer.L2Decay(0.0) + ), + ) + ) self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( - paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC')) + paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC') + ) def forward(self, x): x = self.bn_s1(x) @@ -130,15 +135,17 @@ class TestConvertSyncBatchNormCase2(unittest.TestCase): return out class BNNet(paddle.nn.Layer): - def __init__(self, in_ch=3, out_ch=3, dirate=1): super(BNNet, self).__init__() self.bn_s1 = paddle.nn.BatchNorm3D( out_ch, weight_attr=paddle.ParamAttr( - regularizer=paddle.regularizer.L2Decay(0.))) + regularizer=paddle.regularizer.L2Decay(0.0) + ), + ) self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( - paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC')) + paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC') + ) def forward(self, x): x = self.bn_s1(x) @@ -156,12 +163,16 @@ class TestConvertSyncBatchNormCase2(unittest.TestCase): bn_out.numpy(), sybn_out.numpy(), rtol=1e-05, - err_msg='Output has diff. \n' + '\nBN ' + - str(bn_out.numpy()) + '\n' + 'Sync BN ' + str(sybn_out.numpy())) + err_msg='Output has diff. \n' + + '\nBN ' + + str(bn_out.numpy()) + + '\n' + + 'Sync BN ' + + str(sybn_out.numpy()), + ) class TestDygraphSyncBatchNormDataFormatError(unittest.TestCase): - def test_errors(self): if not core.is_compiled_with_mlu(): return diff --git a/python/paddle/fluid/tests/unittests/mlu/test_tanh_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_tanh_op_mlu.py index a50ad86523a..c346dd0867a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_tanh_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_tanh_op_mlu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestTanh(OpTest): - def setUp(self): self.set_mlu() self.op_type = "tanh" @@ -58,7 +57,6 @@ class TestTanh(OpTest): class TestTanhFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "tanh" @@ -85,7 +83,6 @@ class TestTanhFp16(OpTest): class TestTanhNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -100,9 +97,9 @@ class TestTanhNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.tanh(c) @@ -126,16 +123,17 @@ class TestTanhNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_tile_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_tile_op_mlu.py index c42695f2b5d..2f1a0d990bf 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_tile_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_tile_op_mlu.py @@ -25,9 +25,8 @@ from paddle.fluid import compiler, Program, program_guard paddle.enable_static() -#Situation 1: repeat_times is a list (without tensor) +# Situation 1: repeat_times is a list (without tensor) class TestTileOpRank1(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -51,42 +50,36 @@ class TestTileOpRank1(OpTest): # with dimension expanding class TestTileOpRank2Expanding(TestTileOpRank1): - def init_data(self): self.ori_shape = [120] self.repeat_times = [2, 2] class TestTileOpRank2(TestTileOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] class TestTileOpRank3_Corner(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (1, 1, 1) class TestTileOpRank3_Corner2(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (2, 2) class TestTileOpRank3(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.repeat_times = (2, 1, 4) class TestTileOpRank4(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.repeat_times = (3, 2, 1, 2) @@ -94,7 +87,6 @@ class TestTileOpRank4(TestTileOpRank1): # Situation 2: repeat_times is a list (with tensor) class TestTileOpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -102,8 +94,9 @@ class TestTileOpRank1_tensor_attr(OpTest): self.init_data() repeat_times_tensor = [] for index, ele in enumerate(self.repeat_times): - repeat_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + repeat_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float32"), @@ -126,7 +119,6 @@ class TestTileOpRank1_tensor_attr(OpTest): class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [1, 1] @@ -134,7 +126,6 @@ class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -143,7 +134,6 @@ class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): # Situation 3: repeat_times is a tensor class TestTileOpRank1_tensor(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -169,7 +159,6 @@ class TestTileOpRank1_tensor(OpTest): class TestTileOpRank2_tensor(TestTileOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -177,7 +166,6 @@ class TestTileOpRank2_tensor(TestTileOpRank1_tensor): # Situation 4: input x is Integer class TestTileOpInteger(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -195,7 +183,6 @@ class TestTileOpInteger(OpTest): # Situation 5: input x is Bool class TestTileOpBoolean(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -211,7 +198,6 @@ class TestTileOpBoolean(OpTest): # Situation 56: input x is Integer class TestTileOpInt64_t(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -228,11 +214,11 @@ class TestTileOpInt64_t(OpTest): class TestTileError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) repeat_times = [2, 2] self.assertRaises(TypeError, paddle.tile, x1, repeat_times) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -243,7 +229,6 @@ class TestTileError(unittest.TestCase): class TestTileAPIStatic(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): repeat_times = [2, 2] @@ -255,7 +240,6 @@ class TestTileAPIStatic(unittest.TestCase): # Test python API class TestTileAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): np_x = np.random.random([12, 14]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_top_k_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_top_k_op_mlu.py index 8d7a5eb2c60..8e4a9d48389 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_top_k_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_top_k_op_mlu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestTopkOp(OpTest): - def setUp(self): self.variable_k = False self.set_args() @@ -66,7 +65,6 @@ class TestTopkOp(OpTest): class TestTopkFP16Op(TestTopkOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_top_k_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_top_k_v2_op_mlu.py index 8516fa224fa..4c9d426e82d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_top_k_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_top_k_v2_op_mlu.py @@ -39,7 +39,6 @@ def numpy_topk(x, k=1, axis=-1, largest=True): class TestTopkOp(OpTest): - def init_args(self): self.k = 3 self.axis = 1 @@ -55,10 +54,9 @@ class TestTopkOp(OpTest): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -67,7 +65,6 @@ class TestTopkOp(OpTest): class TestTopkOp1(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 0 @@ -75,7 +72,6 @@ class TestTopkOp1(TestTopkOp): class TestTopkOp2(TestTopkOp): - def init_args(self): self.k = 4 self.axis = 0 @@ -83,7 +79,6 @@ class TestTopkOp2(TestTopkOp): class TestTopkOp3(OpTest): - def init_args(self): self.k = 6 self.axis = 1 @@ -96,15 +91,13 @@ class TestTopkOp3(OpTest): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp4(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -119,15 +112,13 @@ class TestTopkOp4(TestTopkOp): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp5(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -142,15 +133,13 @@ class TestTopkOp5(TestTopkOp): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp6(OpTest): - def init_args(self): self.k = 100 self.axis = 1 @@ -165,15 +154,13 @@ class TestTopkOp6(OpTest): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopKAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.dtype = np.float32 @@ -219,22 +206,23 @@ class TestTopKAPI(unittest.TestCase): np.testing.assert_allclose(paddle_result[1].numpy(), numpy_result[1]) # test case for basic test case 7 for the unsorted paddle_result = paddle.topk(input_tensor, k=2, axis=1, sorted=False) - sort_paddle = numpy_topk(np.array(paddle_result[0].numpy()), - axis=1, - k=2) + sort_paddle = numpy_topk( + np.array(paddle_result[0].numpy()), axis=1, k=2 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=1) np.testing.assert_allclose(sort_paddle[0], numpy_result[0]) def run_static(self, place): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[6, 7, 8], - dtype="float32") - large_input_tensor = paddle.static.data(name="large_x", - shape=[2, 1030], - dtype="float32") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[6, 7, 8], dtype="float32" + ) + large_input_tensor = paddle.static.data( + name="large_x", shape=[2, 1030], dtype="float32" + ) k_tensor = paddle.static.data(name="k", shape=[1], dtype="int32") result1 = paddle.topk(input_tensor, k=2) result2 = paddle.topk(input_tensor, k=2, axis=-1) @@ -248,18 +236,29 @@ class TestTopKAPI(unittest.TestCase): exe = paddle.static.Executor(place) input_data = np.random.rand(10, 20).astype("float32") large_input_data = np.random.rand(2, 100).astype("float32") - paddle_result = exe.run(feed={ - "x": self.input_data, - "large_x": self.large_input_data, - "k": np.array([2]).astype("int32") - }, - fetch_list=[ - result1[0], result1[1], result2[0], - result2[1], result3[0], result3[1], - result4[0], result4[1], result5[0], - result5[1], result6[0], result6[1], - result7[0], result7[1] - ]) + paddle_result = exe.run( + feed={ + "x": self.input_data, + "large_x": self.large_input_data, + "k": np.array([2]).astype("int32"), + }, + fetch_list=[ + result1[0], + result1[1], + result2[0], + result2[1], + result3[0], + result3[1], + result4[0], + result4[1], + result5[0], + result5[1], + result6[0], + result6[1], + result7[0], + result7[1], + ], + ) numpy_result = numpy_topk(self.input_data, k=2) np.testing.assert_allclose(paddle_result[0], numpy_result[0]) np.testing.assert_allclose(paddle_result[1], numpy_result[1]) @@ -269,16 +268,14 @@ class TestTopKAPI(unittest.TestCase): numpy_result = numpy_topk(self.input_data, k=2, axis=1) np.testing.assert_allclose(paddle_result[4], numpy_result[0]) np.testing.assert_allclose(paddle_result[5], numpy_result[1]) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=1, - largest=False) + numpy_result = numpy_topk( + self.input_data, k=2, axis=1, largest=False + ) np.testing.assert_allclose(paddle_result[6], numpy_result[0]) np.testing.assert_allclose(paddle_result[7], numpy_result[1]) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=-1, - largest=False) + numpy_result = numpy_topk( + self.input_data, k=2, axis=-1, largest=False + ) np.testing.assert_allclose(paddle_result[8], numpy_result[0]) np.testing.assert_allclose(paddle_result[9], numpy_result[1]) numpy_result = numpy_topk(self.large_input_data, k=1, axis=-1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_transpose_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_transpose_op_mlu.py index 6aca81d0ba9..30e45198879 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_transpose_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_transpose_op_mlu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestTransposeOp(OpTest): - def setUp(self): self.init_op_type() self.initKernelType() @@ -57,92 +56,79 @@ class TestTransposeOp(OpTest): class TestCase0(TestTransposeOp): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1(TestTransposeOp): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase2(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) class TestCase5(TestTransposeOp): - def initTestCase(self): self.shape = (2, 16, 96) self.axis = (0, 2, 1) class TestCase6(TestTransposeOp): - def initTestCase(self): self.shape = (2, 10, 12, 16) self.axis = (3, 1, 2, 0) class TestCase7(TestTransposeOp): - def initTestCase(self): self.shape = (2, 10, 2, 16) self.axis = (0, 1, 3, 2) class TestCase8(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (0, 1, 3, 2, 4, 5, 6, 7) class TestCase9(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) class TestTransposeOpBool(TestTransposeOp): - def test_check_grad(self): pass class TestTransposeOpBool1D(TestTransposeOpBool): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = {'Out': self.inputs['X'].transpose(self.axis)} class TestTransposeOpBool2D(TestTransposeOpBool): - def initTestCase(self): self.shape = (3, 40) self.axis = (1, 0) @@ -151,7 +137,6 @@ class TestTransposeOpBool2D(TestTransposeOpBool): class TestTransposeOpBool3D(TestTransposeOpBool): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) @@ -160,7 +145,6 @@ class TestTransposeOpBool3D(TestTransposeOpBool): class TestTransposeOpBool4D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) @@ -169,7 +153,6 @@ class TestTransposeOpBool4D(TestTransposeOpBool): class TestTransposeOpBool5D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) @@ -178,7 +161,6 @@ class TestTransposeOpBool5D(TestTransposeOpBool): class TestTransposeOpBool6D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) @@ -187,7 +169,6 @@ class TestTransposeOpBool6D(TestTransposeOpBool): class TestTransposeOpBool7D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3) self.axis = (0, 1, 3, 2, 4, 5, 6) @@ -196,7 +177,6 @@ class TestTransposeOpBool7D(TestTransposeOpBool): class TestTransposeOpBool8D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) @@ -205,7 +185,6 @@ class TestTransposeOpBool8D(TestTransposeOpBool): class TestTransposeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -238,7 +217,6 @@ class TestTransposeOpError(unittest.TestCase): class TestTransposeApi(unittest.TestCase): - def test_static_out(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -248,8 +226,9 @@ class TestTransposeApi(unittest.TestCase): place = paddle.MLUPlace(0) exe = paddle.static.Executor(place) x_np = np.random.random([2, 3, 4]).astype("float32") - result1, result2 = exe.run(feed={"x": x_np}, - fetch_list=[x_trans1, x_trans2]) + result1, result2 = exe.run( + feed={"x": x_np}, fetch_list=[x_trans1, x_trans2] + ) expected_result1 = np.transpose(x_np, [1, 0, 2]) expected_result2 = np.transpose(x_np, (2, 1, 0)) @@ -275,7 +254,6 @@ class TestTransposeApi(unittest.TestCase): class TestTAPI(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program()): data = fluid.data(shape=[10], dtype="float32", name="data") @@ -283,7 +261,7 @@ class TestTAPI(unittest.TestCase): place = fluid.MLUPlace(0) exe = fluid.Executor(place) data_np = np.random.random([10]).astype("float32") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -293,7 +271,7 @@ class TestTAPI(unittest.TestCase): place = fluid.MLUPlace(0) exe = fluid.Executor(place) data_np = np.random.random([10, 5]).astype("float32") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -303,7 +281,7 @@ class TestTAPI(unittest.TestCase): place = fluid.MLUPlace(0) exe = fluid.Executor(place) data_np = np.random.random([1, 5]).astype("float32") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -342,7 +320,6 @@ class TestTAPI(unittest.TestCase): class TestMoveAxis(unittest.TestCase): - def test_moveaxis1(self): x_np = np.random.randn(2, 3, 4, 5, 7).astype('float32') expected = np.moveaxis(x_np, [0, 4, 3, 2], [1, 3, 2, 0]) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_tril_triu_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_tril_triu_op_mlu.py index fe74cbb05b0..2ed56de3c6d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_tril_triu_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_tril_triu_op_mlu.py @@ -27,14 +27,15 @@ paddle.enable_static() class TrilTriuOpDefaultTest(OpTest): - """ the base class of other op testcases - """ + """the base class of other op testcases""" def setUp(self): self.initTestCase() self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) - self.python_api = paddle.tril if self.real_op_type == 'tril' else paddle.triu + self.python_api = ( + paddle.tril if self.real_op_type == 'tril' else paddle.triu + ) self.real_np_op = getattr(np, self.real_op_type) self.op_type = "tril_triu" @@ -44,9 +45,9 @@ class TrilTriuOpDefaultTest(OpTest): 'lower': True if self.real_op_type == 'tril' else False, } self.outputs = { - 'Out': - self.real_np_op(self.X, self.diagonal) - if self.diagonal else self.real_np_op(self.X) + 'Out': self.real_np_op(self.X, self.diagonal) + if self.diagonal + else self.real_np_op(self.X) } def test_check_output(self): @@ -64,27 +65,29 @@ def case_generator(op_type, Xshape, diagonal, expected): If arg`expercted` is 'success', it will register an Optest case and expect to pass. Otherwise, it will register an API case and check the expect failure. """ - cls_name = "{0}_{1}_shape_{2}_diag_{3}".format(expected, op_type, Xshape, - diagonal) + cls_name = "{0}_{1}_shape_{2}_diag_{3}".format( + expected, op_type, Xshape, diagonal + ) errmsg = { - "diagonal: TypeError": - "diagonal in {} must be a python Int".format(op_type), - "input: ValueError": - "x shape in {} must be at least 2-D".format(op_type), + "diagonal: TypeError": "diagonal in {} must be a python Int".format( + op_type + ), + "input: ValueError": "x shape in {} must be at least 2-D".format( + op_type + ), } class FailureCase(unittest.TestCase): - def test_failure(self): paddle.enable_static() data = fluid.data(shape=Xshape, dtype='float64', name=cls_name) - with self.assertRaisesRegexp(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegexp( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal=diagonal) class SuccessCase(TrilTriuOpDefaultTest): - def initTestCase(self): paddle.enable_static() @@ -109,15 +112,13 @@ cases = { (20, 20): [ '2020', [20], - { - 20: 20 - }, + {20: 20}, (20, 20), 20.20, ], # str, list, dict, tuple, float }, 'input: ValueError': { - (2020, ): [None], + (2020,): [None], }, } for _op_type in ['tril', 'triu']: @@ -126,12 +127,15 @@ for _op_type in ['tril', 'triu']: list( map( lambda _diagonal: case_generator( - _op_type, _Xshape, _diagonal, _expected), _diaglist)) + _op_type, _Xshape, _diagonal, _expected + ), + _diaglist, + ) + ) class TestTrilTriuOpAPI(unittest.TestCase): - """ test case by using API and has -1 dimension - """ + """test case by using API and has -1 dimension""" def test_api(self): paddle.enable_static() @@ -163,8 +167,10 @@ class TestTrilTriuOpAPI(unittest.TestCase): with fluid.dygraph.guard(): data = np.random.random([1, 9, 9, 4]).astype(dtype) x = fluid.dygraph.to_variable(data) - tril_out, triu_out = tensor.tril(x).numpy(), tensor.triu( - x).numpy() + tril_out, triu_out = ( + tensor.tril(x).numpy(), + tensor.triu(x).numpy(), + ) np.testing.assert_allclose(tril_out, np.tril(data)) np.testing.assert_allclose(triu_out, np.triu(data)) @@ -182,9 +188,11 @@ class TestTrilTriuOpAPI(unittest.TestCase): place = fluid.MLUPlace(0) exe = fluid.Executor(place) - triu_out = exe.run(fluid.default_main_program(), - feed={"x": data}, - fetch_list=[triu_out]) + triu_out = exe.run( + fluid.default_main_program(), + feed={"x": data}, + fetch_list=[triu_out], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_truncated_gaussian_random_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_truncated_gaussian_random_op_mlu.py index 50e5537fbff..98bcd7109d0 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_truncated_gaussian_random_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_truncated_gaussian_random_op_mlu.py @@ -30,14 +30,13 @@ paddle.enable_static() class TestTrunctedGaussianRandomOp(unittest.TestCase): - def setUp(self): self.op_type = "truncated_gaussian_random" self.inputs = {} self.attrs = { "shape": [10000], - "mean": .0, - "std": 1., + "mean": 0.0, + "std": 1.0, "seed": 10, } self.outputs = ["Out"] @@ -56,9 +55,9 @@ class TestTrunctedGaussianRandomOp(unittest.TestCase): program = fluid.Program() block = program.global_block() vout = block.create_var(name="Out") - op = block.append_op(type=self.op_type, - outputs={"Out": vout}, - attrs=self.attrs) + op = block.append_op( + type=self.op_type, outputs={"Out": vout}, attrs=self.attrs + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) @@ -70,7 +69,7 @@ class TestTrunctedGaussianRandomOp(unittest.TestCase): exe = Executor(place) outs = exe.run(program, fetch_list=fetch_list) tensor = outs[0] - self.assertAlmostEqual(numpy.mean(tensor), .0, delta=0.1) + self.assertAlmostEqual(numpy.mean(tensor), 0.0, delta=0.1) self.assertAlmostEqual(numpy.var(tensor), 0.773, delta=0.1) # TruncatedNormal.__call__ has no return value, so here call _C_ops api @@ -79,9 +78,14 @@ class TestTrunctedGaussianRandomOp(unittest.TestCase): with fluid.dygraph.guard(place): with _test_eager_guard(): out = paddle._C_ops.truncated_gaussian_random( - self.attrs["shape"], self.attrs["mean"], self.attrs["std"], - self.attrs["seed"], core.VarDesc.VarType.FP32, place) - self.assertAlmostEqual(numpy.mean(out.numpy()), .0, delta=0.1) + self.attrs["shape"], + self.attrs["mean"], + self.attrs["std"], + self.attrs["seed"], + core.VarDesc.VarType.FP32, + place, + ) + self.assertAlmostEqual(numpy.mean(out.numpy()), 0.0, delta=0.1) self.assertAlmostEqual(numpy.var(out.numpy()), 0.773, delta=0.1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_uniform_random_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_uniform_random_op_mlu.py index 0dc702b5a22..20abed4245c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_uniform_random_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_uniform_random_op_mlu.py @@ -25,7 +25,10 @@ import paddle from paddle.fluid.op import Operator import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from test_uniform_random_op import TestUniformRandomOp, TestUniformRandomOpSelectedRows +from test_uniform_random_op import ( + TestUniformRandomOp, + TestUniformRandomOpSelectedRows, +) paddle.enable_static() @@ -39,7 +42,6 @@ def output_hist(out): class TestMLUUniformRandomOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "uniform_random" @@ -53,7 +55,7 @@ class TestMLUUniformRandomOp(OpTest): "shape": [1000, 784], "min": -5.0, "max": 10.0, - "seed": 10 + "seed": 10, } self.output_hist = output_hist @@ -73,7 +75,6 @@ class TestMLUUniformRandomOp(OpTest): class TestMLUUniformRandomOpSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_mlu(): @@ -88,12 +89,14 @@ class TestMLUUniformRandomOpSelectedRows(unittest.TestCase): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze2_op_mlu.py index 3a520193575..281a9a1ca2d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze2_op_mlu.py @@ -27,7 +27,6 @@ paddle.enable_static() # Correct: General. class TestUnsqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -36,7 +35,7 @@ class TestUnsqueezeOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -60,16 +59,14 @@ class TestUnsqueezeOp(OpTest): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -78,7 +75,6 @@ class TestUnsqueezeOp2(TestUnsqueezeOp): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -87,7 +83,6 @@ class TestUnsqueezeOp3(TestUnsqueezeOp): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -96,7 +91,6 @@ class TestUnsqueezeOp4(TestUnsqueezeOp): # axes is a list(with tensor) class TestUnsqueezeOp_AxesTensorList(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -104,17 +98,18 @@ class TestUnsqueezeOp_AxesTensorList(OpTest): axes_tensor_list = [] for index, ele in enumerate(self.axes): - axes_tensor_list.append(("axes" + str(index), np.ones( - (1)).astype('int32') * ele)) + axes_tensor_list.append( + ("axes" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - "AxesTensorList": axes_tensor_list + "AxesTensorList": axes_tensor_list, } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -137,15 +132,13 @@ class TestUnsqueezeOp_AxesTensorList(OpTest): class TestUnsqueezeOp1_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueezeOp2_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -153,7 +146,6 @@ class TestUnsqueezeOp2_AxesTensorList(TestUnsqueezeOp_AxesTensorList): class TestUnsqueezeOp3_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -161,7 +153,6 @@ class TestUnsqueezeOp3_AxesTensorList(TestUnsqueezeOp_AxesTensorList): class TestUnsqueezeOp4_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -170,7 +161,6 @@ class TestUnsqueezeOp4_AxesTensorList(TestUnsqueezeOp_AxesTensorList): # axes is a Tensor class TestUnsqueezeOp_AxesTensor(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -178,12 +168,12 @@ class TestUnsqueezeOp_AxesTensor(OpTest): self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - "AxesTensor": np.array(self.axes).astype("int32") + "AxesTensor": np.array(self.axes).astype("int32"), } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -206,15 +196,13 @@ class TestUnsqueezeOp_AxesTensor(OpTest): class TestUnsqueezeOp1_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueezeOp2_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -222,7 +210,6 @@ class TestUnsqueezeOp2_AxesTensor(TestUnsqueezeOp_AxesTensor): class TestUnsqueezeOp3_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -230,7 +217,6 @@ class TestUnsqueezeOp3_AxesTensor(TestUnsqueezeOp_AxesTensor): class TestUnsqueezeOp4_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze_op_mlu.py index 11317b61219..033f58d0e72 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze_op_mlu.py @@ -27,7 +27,6 @@ paddle.enable_static() # Correct: General. class TestUnsqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -57,16 +56,14 @@ class TestUnsqueezeOp(OpTest): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -75,7 +72,6 @@ class TestUnsqueezeOp2(TestUnsqueezeOp): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -84,7 +80,6 @@ class TestUnsqueezeOp3(TestUnsqueezeOp): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_unstack_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_unstack_op_mlu.py index 2ec9726bcf2..6e46a9ff0b2 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_unstack_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_unstack_op_mlu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestUnStackOpBase(OpTest): - def initDefaultParameters(self): self.input_dim = (5, 6, 7) self.axis = 0 @@ -74,25 +73,21 @@ class TestUnStackOpBase(OpTest): class TestStackOp3(TestUnStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestUnStackOpBase): - def initParameters(self): self.axis = -3 class TestStackOp5(TestUnStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestUnStackOpBase): - def initParameters(self): self.axis = 2 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_where_index_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_where_index_op_mlu.py index 2486bd8324a..163d47e33c2 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_where_index_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_where_index_op_mlu.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestWhereIndexOp(OpTest): - def setUp(self): self.op_type = "where_index" self.place = paddle.device.MLUPlace(0) @@ -47,7 +46,6 @@ class TestWhereIndexOp(OpTest): class TestAllFalse(unittest.TestCase): - def setUp(self): self.op_type = "where_index" self.place = paddle.device.MLUPlace(0) @@ -78,7 +76,6 @@ class TestAllFalse(unittest.TestCase): class TestRank2(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([[True, False], [False, True]]), @@ -88,24 +85,26 @@ class TestRank2(TestWhereIndexOp): class TestRank3(TestWhereIndexOp): - def init_config(self): self.inputs = { - 'Condition': - np.array([[[True, False], [False, True]], - [[False, True], [True, False]], - [[False, False], [False, True]]]), + 'Condition': np.array( + [ + [[True, False], [False, True]], + [[False, True], [True, False]], + [[False, False], [False, True]], + ] + ), } self.outputs = { - 'Out': - np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], - dtype='int64') + 'Out': np.array( + [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], + dtype='int64', + ) } class TestWhereOpError(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): cond = fluid.layers.data(name='cond', shape=[4], dtype='bool') @@ -118,9 +117,7 @@ class TestWhereOpError(unittest.TestCase): class TestWhereRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.where([10]) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_where_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_where_op_mlu.py index 5f757015656..2bb780e1f26 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_where_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_where_op_mlu.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestWhereOp(OpTest): - def setUp(self): self.op_type = 'where' self.place = paddle.device.MLUPlace(0) @@ -55,7 +54,6 @@ class TestWhereOp(OpTest): class TestWhereOp2(TestWhereOp): - def init_config(self): self.x = np.random.uniform((-5), 5, (60, 2)).astype('float32') self.y = np.random.uniform((-5), 5, (60, 2)).astype('float32') @@ -63,7 +61,6 @@ class TestWhereOp2(TestWhereOp): class TestWhereOp3(TestWhereOp): - def init_config(self): self.x = np.random.uniform((-3), 5, (20, 2, 4)).astype('float32') self.y = np.random.uniform((-3), 5, (20, 2, 4)).astype('float32') @@ -71,7 +68,6 @@ class TestWhereOp3(TestWhereOp): class TestWhereAPI(unittest.TestCase): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -95,45 +91,49 @@ class TestWhereAPI(unittest.TestCase): for x_stop_gradient in [False, True]: for y_stop_gradient in [False, True]: with fluid.program_guard(Program(), Program()): - cond = fluid.layers.data(name='cond', - shape=self.shape, - dtype='bool') - x = fluid.layers.data(name='x', - shape=self.shape, - dtype='float32') - y = fluid.layers.data(name='y', - shape=self.shape, - dtype='float32') + cond = fluid.layers.data( + name='cond', shape=self.shape, dtype='bool' + ) + x = fluid.layers.data( + name='x', shape=self.shape, dtype='float32' + ) + y = fluid.layers.data( + name='y', shape=self.shape, dtype='float32' + ) x.stop_gradient = x_stop_gradient y.stop_gradient = y_stop_gradient result = paddle.where(cond, x, y) append_backward(paddle.mean(result)) for use_mlu in [False, True]: - place = (paddle.device.MLUPlace(0) - if use_mlu else fluid.CPUPlace()) + place = ( + paddle.device.MLUPlace(0) + if use_mlu + else fluid.CPUPlace() + ) exe = fluid.Executor(place) fetch_list = [result, result.grad_name] - if (x_stop_gradient is False): + if x_stop_gradient is False: fetch_list.append(x.grad_name) - if (y_stop_gradient is False): + if y_stop_gradient is False: fetch_list.append(y.grad_name) - out = exe.run(fluid.default_main_program(), - feed={ - 'cond': self.cond, - 'x': self.x, - 'y': self.y - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={'cond': self.cond, 'x': self.x, 'y': self.y}, + fetch_list=fetch_list, + ) assert np.array_equal(out[0], self.out) - if (x_stop_gradient is False): - assert np.array_equal(out[2], - self.ref_x_backward(out[1])) - if (y.stop_gradient is False): + if x_stop_gradient is False: + assert np.array_equal( + out[2], self.ref_x_backward(out[1]) + ) + if y.stop_gradient is False: assert np.array_equal( - out[3], self.ref_y_backward(out[1])) - elif (y.stop_gradient is False): - assert np.array_equal(out[2], - self.ref_y_backward(out[1])) + out[3], self.ref_y_backward(out[1]) + ) + elif y.stop_gradient is False: + assert np.array_equal( + out[2], self.ref_y_backward(out[1]) + ) def test_api_broadcast(self, use_mlu=False): main_program = Program() @@ -141,19 +141,20 @@ class TestWhereAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32') y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32') x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype('float32') - y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, - 1.0]]).astype('float32') + y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]).astype( + 'float32' + ) result = paddle.where((x > 1), x=x, y=y) for use_mlu in [False, True]: - place = (paddle.device.MLUPlace(0) - if use_mlu else fluid.CPUPlace()) + place = ( + paddle.device.MLUPlace(0) if use_mlu else fluid.CPUPlace() + ) exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'x': x_i, 'y': y_i}, + fetch_list=[result], + ) assert np.array_equal(out[0], np.where((x_i > 1), x_i, y_i)) def test_scalar(self): @@ -161,20 +162,23 @@ class TestWhereAPI(unittest.TestCase): main_program = Program() with fluid.program_guard(main_program): cond_shape = [2, 4] - cond = fluid.layers.data(name='cond', - shape=cond_shape, - dtype='bool') + cond = fluid.layers.data( + name='cond', shape=cond_shape, dtype='bool' + ) x_data = 1.0 y_data = 2.0 cond_data = np.array([False, False, True, True]).astype('bool') result = paddle.where(condition=cond, x=x_data, y=y_data) for use_mlu in [False, True]: - place = (paddle.device.MLUPlace(0) - if use_mlu else fluid.CPUPlace()) + place = ( + paddle.device.MLUPlace(0) if use_mlu else fluid.CPUPlace() + ) exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={'cond': cond_data}, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'cond': cond_data}, + fetch_list=[result], + ) expect = np.where(cond_data, x_data, y_data) assert np.array_equal(out[0], expect) @@ -182,27 +186,26 @@ class TestWhereAPI(unittest.TestCase): paddle.enable_static() main_program = Program() with fluid.program_guard(main_program): - cond = fluid.layers.data(name='cond', - shape=cond_shape, - dtype='bool') + cond = fluid.layers.data( + name='cond', shape=cond_shape, dtype='bool' + ) x = fluid.layers.data(name='x', shape=x_shape, dtype='float32') y = fluid.layers.data(name='y', shape=y_shape, dtype='float32') cond_data_tmp = np.random.random(size=cond_shape).astype('float32') - cond_data = (cond_data_tmp < 0.3) + cond_data = cond_data_tmp < 0.3 x_data = np.random.random(size=x_shape).astype('float32') y_data = np.random.random(size=y_shape).astype('float32') result = paddle.where(condition=cond, x=x, y=y) for use_mlu in [False, True]: - place = (paddle.device.MLUPlace(0) - if use_mlu else fluid.CPUPlace()) + place = ( + paddle.device.MLUPlace(0) if use_mlu else fluid.CPUPlace() + ) exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'cond': cond_data, - 'x': x_data, - 'y': y_data - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'cond': cond_data, 'x': x_data, 'y': y_data}, + fetch_list=[result], + ) expect = np.where(cond_data, x_data, y_data) assert np.array_equal(out[0], expect) @@ -256,7 +259,6 @@ class TestWhereAPI(unittest.TestCase): class TestWhereDygraphAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype('float32') @@ -280,7 +282,7 @@ class TestWhereDygraphAPI(unittest.TestCase): def __test_where_with_broadcast_dygraph(self, cond_shape, a_shape, b_shape): with fluid.dygraph.guard(): cond_tmp = paddle.rand(cond_shape) - cond = (cond_tmp < 0.3) + cond = cond_tmp < 0.3 a = paddle.rand(a_shape) b = paddle.rand(b_shape) result = paddle.where(cond, a, b) @@ -345,9 +347,9 @@ class TestWhereDygraphAPI(unittest.TestCase): self.assertEqual(len(y), 2) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(paddle.device.MLUPlace(0)) - (res, ) = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0, 0], [1, 1]]) np.testing.assert_allclose(expect_out, np.array(res)) data = np.array([True, True, False]) @@ -358,15 +360,14 @@ class TestWhereDygraphAPI(unittest.TestCase): self.assertEqual(len(y), 1) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(paddle.device.MLUPlace(0)) - (res, ) = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0], [1]]) np.testing.assert_allclose(expect_out, np.array(res)) class TestWhereOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype('float32') @@ -390,7 +391,7 @@ class TestWhereOpError(unittest.TestCase): with fluid.dygraph.guard(): cond_shape = [2, 2, 4] cond_tmp = paddle.rand(cond_shape) - cond = (cond_tmp < 0.3) + cond = cond_tmp < 0.3 a = paddle.rand(cond_shape) self.assertRaises(ValueError, paddle.where, cond, a) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_yolo_box_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_yolo_box_op_mlu.py index 86bc314eafc..fd8855aeb57 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_yolo_box_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_yolo_box_op_mlu.py @@ -29,7 +29,7 @@ paddle.enable_static() def sigmoid(x): - return (1.0 / (1.0 + np.exp(((-1.0) * x)))) + return 1.0 / (1.0 + np.exp(((-1.0) * x))) def YoloBox(x, img_size, attrs): @@ -43,9 +43,9 @@ def YoloBox(x, img_size, attrs): scale_x_y = attrs['scale_x_y'] iou_aware = attrs['iou_aware'] iou_aware_factor = attrs['iou_aware_factor'] - bias_x_y = ((-0.5) * (scale_x_y - 1.0)) - input_h = (downsample * h) - input_w = (downsample * w) + bias_x_y = (-0.5) * (scale_x_y - 1.0) + input_h = downsample * h + input_w = downsample * w if iou_aware: ioup = x[:, :an_num, :, :] ioup = np.expand_dims(ioup, axis=(-1)) @@ -54,49 +54,54 @@ def YoloBox(x, img_size, attrs): pred_box = x[:, :, :, :, :4].copy() grid_x = np.tile(np.arange(w).reshape((1, w)), (h, 1)) grid_y = np.tile(np.arange(h).reshape((h, 1)), (1, w)) - pred_box[:, :, :, :, 0] = (( - (grid_x + (sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y)) + bias_x_y) / - w) - pred_box[:, :, :, :, 1] = (( - (grid_y + (sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y)) + bias_x_y) / - h) - anchors = [(anchors[i], anchors[(i + 1)]) - for i in range(0, len(anchors), 2)] - anchors_s = np.array([((an_w / input_w), (an_h / input_h)) - for (an_w, an_h) in anchors]) + pred_box[:, :, :, :, 0] = ( + (grid_x + (sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y)) + bias_x_y + ) / w + pred_box[:, :, :, :, 1] = ( + (grid_y + (sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y)) + bias_x_y + ) / h + anchors = [ + (anchors[i], anchors[(i + 1)]) for i in range(0, len(anchors), 2) + ] + anchors_s = np.array( + [((an_w / input_w), (an_h / input_h)) for (an_w, an_h) in anchors] + ) anchor_w = anchors_s[:, 0:1].reshape((1, an_num, 1, 1)) anchor_h = anchors_s[:, 1:2].reshape((1, an_num, 1, 1)) - pred_box[:, :, :, :, 2] = (np.exp(pred_box[:, :, :, :, 2]) * anchor_w) - pred_box[:, :, :, :, 3] = (np.exp(pred_box[:, :, :, :, 3]) * anchor_h) + pred_box[:, :, :, :, 2] = np.exp(pred_box[:, :, :, :, 2]) * anchor_w + pred_box[:, :, :, :, 3] = np.exp(pred_box[:, :, :, :, 3]) * anchor_h if iou_aware: - pred_conf = ((sigmoid(x[:, :, :, :, 4:5])**(1 - iou_aware_factor)) * - (sigmoid(ioup)**iou_aware_factor)) + pred_conf = (sigmoid(x[:, :, :, :, 4:5]) ** (1 - iou_aware_factor)) * ( + sigmoid(ioup) ** iou_aware_factor + ) else: pred_conf = sigmoid(x[:, :, :, :, 4:5]) pred_conf[(pred_conf < conf_thresh)] = 0.0 - pred_score = (sigmoid(x[:, :, :, :, 5:]) * pred_conf) - pred_box = (pred_box * (pred_conf > 0.0).astype('float32')) + pred_score = sigmoid(x[:, :, :, :, 5:]) * pred_conf + pred_box = pred_box * (pred_conf > 0.0).astype('float32') pred_box = pred_box.reshape((n, (-1), 4)) - (pred_box[:, :, :2], - pred_box[:, :, 2:4]) = ((pred_box[:, :, :2] - (pred_box[:, :, 2:4] / 2.0)), - (pred_box[:, :, :2] + (pred_box[:, :, 2:4] / 2.0))) - pred_box[:, :, 0] = (pred_box[:, :, 0] * img_size[:, 1][:, np.newaxis]) - pred_box[:, :, 1] = (pred_box[:, :, 1] * img_size[:, 0][:, np.newaxis]) - pred_box[:, :, 2] = (pred_box[:, :, 2] * img_size[:, 1][:, np.newaxis]) - pred_box[:, :, 3] = (pred_box[:, :, 3] * img_size[:, 0][:, np.newaxis]) + (pred_box[:, :, :2], pred_box[:, :, 2:4]) = ( + (pred_box[:, :, :2] - (pred_box[:, :, 2:4] / 2.0)), + (pred_box[:, :, :2] + (pred_box[:, :, 2:4] / 2.0)), + ) + pred_box[:, :, 0] = pred_box[:, :, 0] * img_size[:, 1][:, np.newaxis] + pred_box[:, :, 1] = pred_box[:, :, 1] * img_size[:, 0][:, np.newaxis] + pred_box[:, :, 2] = pred_box[:, :, 2] * img_size[:, 1][:, np.newaxis] + pred_box[:, :, 3] = pred_box[:, :, 3] * img_size[:, 0][:, np.newaxis] if clip_bbox: for i in range(len(pred_box)): pred_box[i, :, 0] = np.clip(pred_box[i, :, 0], 0, np.inf) pred_box[i, :, 1] = np.clip(pred_box[i, :, 1], 0, np.inf) - pred_box[i, :, 2] = np.clip(pred_box[i, :, 2], (-np.inf), - (img_size[(i, 1)] - 1)) - pred_box[i, :, 3] = np.clip(pred_box[i, :, 3], (-np.inf), - (img_size[(i, 0)] - 1)) + pred_box[i, :, 2] = np.clip( + pred_box[i, :, 2], (-np.inf), (img_size[(i, 1)] - 1) + ) + pred_box[i, :, 3] = np.clip( + pred_box[i, :, 3], (-np.inf), (img_size[(i, 0)] - 1) + ) return (pred_box, pred_score.reshape((n, (-1), class_num))) class TestYoloBoxOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'yolo_box' @@ -114,7 +119,7 @@ class TestYoloBoxOp(OpTest): 'clip_bbox': self.clip_bbox, 'scale_x_y': self.scale_x_y, 'iou_aware': self.iou_aware, - 'iou_aware_factor': self.iou_aware_factor + 'iou_aware_factor': self.iou_aware_factor, } self.inputs = {'X': x, 'ImgSize': img_size} (boxes, scores) = YoloBox(x, img_size, self.attrs) @@ -131,8 +136,12 @@ class TestYoloBoxOp(OpTest): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = False @@ -140,7 +149,6 @@ class TestYoloBoxOp(OpTest): class TestYoloBoxOpNoClipBbox(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -149,8 +157,12 @@ class TestYoloBoxOpNoClipBbox(TestYoloBoxOp): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = False - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = False @@ -158,7 +170,6 @@ class TestYoloBoxOpNoClipBbox(TestYoloBoxOp): class TestYoloBoxOpScaleXY(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -167,8 +178,12 @@ class TestYoloBoxOpScaleXY(TestYoloBoxOp): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.2 self.iou_aware = False @@ -176,7 +191,6 @@ class TestYoloBoxOpScaleXY(TestYoloBoxOp): class TestYoloBoxOpIoUAware(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -185,8 +199,12 @@ class TestYoloBoxOpIoUAware(TestYoloBoxOp): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (6 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (6 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = True @@ -194,67 +212,72 @@ class TestYoloBoxOpIoUAware(TestYoloBoxOp): class TestYoloBoxDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() img_size = np.ones((2, 2)).astype('int32') img_size = paddle.to_tensor(img_size) x1 = np.random.random([2, 14, 8, 8]).astype('float32') x1 = paddle.to_tensor(x1) - (boxes, scores) = paddle.vision.ops.yolo_box(x1, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x1, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + ) + assert (boxes is not None) and (scores is not None) x2 = np.random.random([2, 16, 8, 8]).astype('float32') x2 = paddle.to_tensor(x2) - (boxes, scores) = paddle.vision.ops.yolo_box(x2, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0, - iou_aware=True, - iou_aware_factor=0.5) + (boxes, scores) = paddle.vision.ops.yolo_box( + x2, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + iou_aware=True, + iou_aware_factor=0.5, + ) paddle.enable_static() class TestYoloBoxStatic(unittest.TestCase): - def test_static(self): x1 = paddle.static.data('x1', [2, 14, 8, 8], 'float32') img_size = paddle.static.data('img_size', [2, 2], 'int32') - (boxes, scores) = paddle.vision.ops.yolo_box(x1, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x1, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + ) + assert (boxes is not None) and (scores is not None) x2 = paddle.static.data('x2', [2, 16, 8, 8], 'float32') - (boxes, scores) = paddle.vision.ops.yolo_box(x2, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0, - iou_aware=True, - iou_aware_factor=0.5) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x2, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + iou_aware=True, + iou_aware_factor=0.5, + ) + assert (boxes is not None) and (scores is not None) class TestYoloBoxOpHW(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) diff --git a/python/paddle/fluid/tests/unittests/multi_process.py b/python/paddle/fluid/tests/unittests/multi_process.py index fa6b7200f32..74283be7b37 100644 --- a/python/paddle/fluid/tests/unittests/multi_process.py +++ b/python/paddle/fluid/tests/unittests/multi_process.py @@ -25,12 +25,18 @@ def train(prefix): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_gpus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_gpus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) @@ -47,23 +53,34 @@ def train_abort(prefix): # train abort exit(1) except SystemExit: - name = "abort>>> selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_gpus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "abort>>> selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_gpus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) with open( - "multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) raise else: # sleep 30s to make sure paddle.distributed.launch will terminate this process time.sleep(30) - name = "selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_gpus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_gpus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) diff --git a/python/paddle/fluid/tests/unittests/my_data_generator.py b/python/paddle/fluid/tests/unittests/my_data_generator.py index 5279973e01a..b08e5c53815 100644 --- a/python/paddle/fluid/tests/unittests/my_data_generator.py +++ b/python/paddle/fluid/tests/unittests/my_data_generator.py @@ -16,14 +16,14 @@ import paddle.distributed.fleet as fleet class MyDataset(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): elements = line.strip().split()[0:] - output = [("show", [int(elements[0])]), - ("click", [int(elements[1])]), - ("slot1", [int(elements[2])])] + output = [ + ("show", [int(elements[0])]), + ("click", [int(elements[1])]), + ("slot1", [int(elements[2])]), + ] yield output return data_iter diff --git a/python/paddle/fluid/tests/unittests/nproc_process.py b/python/paddle/fluid/tests/unittests/nproc_process.py index bbd0b6a0e5e..7a01005fe2e 100644 --- a/python/paddle/fluid/tests/unittests/nproc_process.py +++ b/python/paddle/fluid/tests/unittests/nproc_process.py @@ -28,8 +28,13 @@ def train(prefix): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "selected_devices:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_devices, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_devices:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_devices, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) with open("{}.check_{}.log".format(prefix, trainer_id), "w") as f: diff --git a/python/paddle/fluid/tests/unittests/npu/collective_identity_op_npu.py b/python/paddle/fluid/tests/unittests/npu/collective_identity_op_npu.py index e4ed24a73f5..f8f8612ca89 100644 --- a/python/paddle/fluid/tests/unittests/npu/collective_identity_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/collective_identity_op_npu.py @@ -35,7 +35,6 @@ paddle.enable_static() class TestCollectiveIdentity(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -43,22 +42,22 @@ class TestCollectiveIdentity(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_identity", - inputs={'X': tindata}, - outputs={'Out': toutdata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_identity", + inputs={'X': tindata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/npu/process_group_hccl.py b/python/paddle/fluid/tests/unittests/npu/process_group_hccl.py index 4b96002536e..474d4104b04 100644 --- a/python/paddle/fluid/tests/unittests/npu/process_group_hccl.py +++ b/python/paddle/fluid/tests/unittests/npu/process_group_hccl.py @@ -37,7 +37,6 @@ def init_process_group(strategy=None): class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -50,8 +49,9 @@ class TestProcessGroupFp32(unittest.TestCase): def test_create_process_group_nccl(self): with _test_eager_guard(): - paddle.set_device('npu:%d' % - paddle.distributed.ParallelEnv().dev_id) + paddle.set_device( + 'npu:%d' % paddle.distributed.ParallelEnv().dev_id + ) pg = init_process_group() @@ -147,8 +147,9 @@ class TestProcessGroupFp32(unittest.TestCase): task.wait() paddle.device.cuda.synchronize() out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) assert np.array_equal(tensor_x, out_1) assert np.array_equal(tensor_y, out_2) print("test allgather api ok\n") @@ -163,10 +164,12 @@ class TestProcessGroupFp32(unittest.TestCase): tensor_y = paddle.to_tensor(y) tensor_out1 = paddle.to_tensor(out1) tensor_out2 = paddle.to_tensor(out2) - raw_tensor_x_2 = paddle.slice(tensor_x, [0], [self.shape[0] // 2], - [self.shape[0]]) - raw_tensor_y_1 = paddle.slice(tensor_y, [0], [0], - [self.shape[0] // 2]) + raw_tensor_x_2 = paddle.slice( + tensor_x, [0], [self.shape[0] // 2], [self.shape[0]] + ) + raw_tensor_y_1 = paddle.slice( + tensor_y, [0], [0], [self.shape[0] // 2] + ) if pg.rank() == 0: task = pg.alltoall(tensor_x, tensor_out1) task.wait() @@ -176,8 +179,9 @@ class TestProcessGroupFp32(unittest.TestCase): task = pg.alltoall(tensor_y, tensor_out2) task.wait() paddle.device.cuda.synchronize() - out1_2 = paddle.slice(tensor_out1, [0], [self.shape[0] // 2], - [self.shape[0]]) + out1_2 = paddle.slice( + tensor_out1, [0], [self.shape[0] // 2], [self.shape[0]] + ) out2_1 = paddle.slice(tensor_out2, [0], [0], [self.shape[0] // 2]) if pg.rank() == 0: assert np.array_equal(out1_2.numpy(), raw_tensor_y_1.numpy()) @@ -223,8 +227,9 @@ class TestProcessGroupFp32(unittest.TestCase): task.wait() paddle.device.cuda.synchronize() out1 = paddle.slice(tensor_x, [0], [0], [self.shape[0]]) - out2 = paddle.slice(tensor_x, [0], [self.shape[0]], - [self.shape[0] * 2]) + out2 = paddle.slice( + tensor_x, [0], [self.shape[0]], [self.shape[0] * 2] + ) if pg.rank() == 0: assert np.array_equal(tensor_y, out1) else: @@ -233,7 +238,6 @@ class TestProcessGroupFp32(unittest.TestCase): class TestProcessGroupFp16(TestProcessGroupFp32): - def setUp(self): paddle.seed(2022) random.seed(2022) diff --git a/python/paddle/fluid/tests/unittests/npu/sync_batch_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/sync_batch_norm_op_npu.py index 84f93bc88a1..4960fee949b 100644 --- a/python/paddle/fluid/tests/unittests/npu/sync_batch_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/sync_batch_norm_op_npu.py @@ -31,17 +31,24 @@ import unittest from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_sync_batch_norm_base_npu import TestSyncBatchNormRunnerBase, runtime_main -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from test_sync_batch_norm_base_npu import ( + TestSyncBatchNormRunnerBase, + runtime_main, +) +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) -from paddle.fluid.tests.unittests.test_sync_batch_norm_op import create_or_get_tensor +from paddle.fluid.tests.unittests.test_sync_batch_norm_op import ( + create_or_get_tensor, +) _set_use_system_allocator(False) paddle.enable_static() class TestSyncBatchNormOpTraining(TestSyncBatchNormRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -53,29 +60,34 @@ class TestSyncBatchNormOpTraining(TestSyncBatchNormRunnerBase): self.dshape = [self.N, self.C, self.H, self.W] self.atol = 1e-3 - def get_model(self, - main, - startup, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def get_model( + self, + main, + startup, + place, + layout, + seed, + sync_bn=False, + only_forward=False, + ): """Build program.""" use_cudnn = False with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - data = fluid.layers.data(name='input', - shape=self.dshape, - dtype=self.dtype, - append_batch_size=False) + data = fluid.layers.data( + name='input', + shape=self.dshape, + dtype=self.dtype, + append_batch_size=False, + ) conv = fluid.layers.conv2d( input=data, num_filters=32, filter_size=1, param_attr=fluid.ParamAttr(name='conv2d_weight'), bias_attr=False, - use_cudnn=use_cudnn) + use_cudnn=use_cudnn, + ) bn = fluid.layers.batch_norm( conv, param_attr=fluid.ParamAttr(name='bn_scale'), @@ -83,7 +95,8 @@ class TestSyncBatchNormOpTraining(TestSyncBatchNormRunnerBase): moving_mean_name='bn_moving_mean', moving_variance_name='bn_moving_variance', data_layout=layout, - is_test=only_forward) + is_test=only_forward, + ) # if self.dtype == np.float16: # bn = fluid.layers.cast(bn, 'float32') sigmoid = fluid.layers.sigmoid(bn) diff --git a/python/paddle/fluid/tests/unittests/npu/test_abs_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_abs_op_npu.py index 89269375094..f28552f14e7 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_abs_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_abs_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestNPUAbs(OpTest): - def setUp(self): self.op_type = "abs" self.set_npu() diff --git a/python/paddle/fluid/tests/unittests/npu/test_accuracy_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_accuracy_op_npu.py index c46a51dfe3a..54d029384c6 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_accuracy_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_accuracy_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestAccuracy(OpTest): - def setUp(self): self.op_type = "accuracy" self.set_npu() @@ -47,7 +46,7 @@ class TestAccuracy(OpTest): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } def set_npu(self): @@ -62,7 +61,6 @@ class TestAccuracy(OpTest): class TestAccuracy2(TestAccuracy): - def setUp(self): self.op_type = "accuracy" self.set_npu() @@ -82,12 +80,11 @@ class TestAccuracy2(TestAccuracy): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } class TestAccuracyType(TestAccuracy): - def setUp(self): self.op_type = "accuracy" self.set_npu() @@ -107,12 +104,11 @@ class TestAccuracyType(TestAccuracy): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } class TestAccuracyType2(TestAccuracy): - def setUp(self): self.op_type = "accuracy" self.set_npu() @@ -132,7 +128,7 @@ class TestAccuracyType2(TestAccuracy): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } diff --git a/python/paddle/fluid/tests/unittests/npu/test_adam_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_adam_op_npu.py index 70ab75ef5f2..49589d5d9dc 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_adam_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_adam_op_npu.py @@ -28,7 +28,6 @@ SEED = 2021 class TestAdam(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -53,20 +52,19 @@ class TestAdam(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_npu(self): @@ -80,7 +78,6 @@ class TestAdam(OpTest): class TestAdamWithEpsilonTensor(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -113,15 +110,14 @@ class TestAdamWithEpsilonTensor(OpTest): self.attrs = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_npu(self): @@ -135,7 +131,6 @@ class TestAdamWithEpsilonTensor(OpTest): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -188,7 +183,6 @@ class TestAdamOpWithSkipUpdate(OpTest): class TestAdamOpWithGlobalBetaPow(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -221,8 +215,7 @@ class TestAdamOpWithGlobalBetaPow(OpTest): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.attrs = {'use_global_beta_pow': True} @@ -232,7 +225,7 @@ class TestAdamOpWithGlobalBetaPow(OpTest): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([]), - 'Beta2PowOut': np.array([]) + 'Beta2PowOut': np.array([]), } def set_npu(self): @@ -246,7 +239,6 @@ class TestAdamOpWithGlobalBetaPow(OpTest): class TestNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -261,9 +253,9 @@ class TestNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -287,16 +279,17 @@ class TestNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -309,13 +302,14 @@ class TestNet(unittest.TestCase): class TestNetWithEpsilonTensor(unittest.TestCase): - - def _test(self, - place, - use_tensor=True, - use_fluid_api=True, - use_global_beta_pow=False, - flatten_param_grads=False): + def _test( + self, + place, + use_tensor=True, + use_fluid_api=True, + use_global_beta_pow=False, + flatten_param_grads=False, + ): paddle.enable_static() main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -329,29 +323,30 @@ class TestNetWithEpsilonTensor(unittest.TestCase): weight_attr1 = paddle.ParamAttr( name="weight1", initializer=fluid.initializer.Constant(value=1.0), - trainable=True) + trainable=True, + ) weight_attr2 = paddle.ParamAttr( name="weight2", initializer=fluid.initializer.Constant(value=2.0), - trainable=True) + trainable=True, + ) clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) with paddle.static.program_guard(main_prog, startup_prog): with paddle.utils.unique_name.guard(): a = paddle.static.data(name="a", shape=[2, 2], dtype='float32') b = paddle.static.data(name="b", shape=[2, 2], dtype='float32') - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) fc_1 = fluid.layers.fc(input=z, size=2, param_attr=weight_attr1) - prediction = fluid.layers.fc(input=fc_1, - size=2, - param_attr=weight_attr2, - act='softmax') + prediction = fluid.layers.fc( + input=fc_1, size=2, param_attr=weight_attr2, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) loss = fluid.layers.reduce_mean(cost) @@ -364,19 +359,22 @@ class TestNetWithEpsilonTensor(unittest.TestCase): value=float(beta1_init), dtype='float32', persistable=True, - name="beta1") + name="beta1", + ) beta2 = fluid.layers.create_global_var( shape=[1], value=float(beta2_init), dtype='float32', persistable=True, - name="beta2") + name="beta2", + ) epsilon = fluid.layers.create_global_var( shape=[1], value=float(epsilon_init), dtype='float32', persistable=True, - name="epsilon") + name="epsilon", + ) if use_fluid_api: adam = fluid.optimizer.Adam( learning_rate=0.01, @@ -386,13 +384,16 @@ class TestNetWithEpsilonTensor(unittest.TestCase): use_global_beta_pow=use_global_beta_pow, flatten_param_grads=flatten_param_grads, align_size=256, - grad_clip=clip) + grad_clip=clip, + ) else: - adam = paddle.optimizer.Adam(learning_rate=0.01, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - grad_clip=clip) + adam = paddle.optimizer.Adam( + learning_rate=0.01, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + grad_clip=clip, + ) else: if use_fluid_api: adam = fluid.optimizer.Adam( @@ -403,13 +404,16 @@ class TestNetWithEpsilonTensor(unittest.TestCase): use_global_beta_pow=use_global_beta_pow, flatten_param_grads=flatten_param_grads, align_size=256, - grad_clip=clip) + grad_clip=clip, + ) else: - adam = fluid.optimizer.Adam(learning_rate=0.01, - beta1=beta1_init, - beta2=beta2_init, - epsilon=epsilon_init, - grad_clip=clip) + adam = fluid.optimizer.Adam( + learning_rate=0.01, + beta1=beta1_init, + beta2=beta2_init, + epsilon=epsilon_init, + grad_clip=clip, + ) adam.minimize(loss) @@ -420,15 +424,16 @@ class TestNetWithEpsilonTensor(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(10): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) paddle.disable_static() return pred_res, loss_res @@ -440,10 +445,13 @@ class TestNetWithEpsilonTensor(unittest.TestCase): for use_fluid_api in [True, False]: for use_global_beta_pow in [True, False]: for flatten_param_grads in [True, False]: - pred, loss = self._test(place, use_tensor, - use_fluid_api, - use_global_beta_pow, - flatten_param_grads) + pred, loss = self._test( + place, + use_tensor, + use_fluid_api, + use_global_beta_pow, + flatten_param_grads, + ) preds.append(pred) losses.append(loss) for pred in preds: diff --git a/python/paddle/fluid/tests/unittests/npu/test_adamw_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_adamw_op_npu.py index 579892dee3d..0211eb196d5 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_adamw_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_adamw_op_npu.py @@ -28,7 +28,6 @@ SEED = 2021 class TestAdamW(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -53,7 +52,7 @@ class TestAdamW(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = { @@ -61,18 +60,19 @@ class TestAdamW(OpTest): 'beta1': beta1, 'beta2': beta2, "coeff": 0.9, - "with_decay": True + "with_decay": True, } - param_out, moment1_out, \ - moment2_out = adamw_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adamw_step( + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_npu(self): @@ -86,7 +86,6 @@ class TestAdamW(OpTest): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -139,7 +138,6 @@ class TestAdamOpWithSkipUpdate(OpTest): class TestAdamOpWithoutDecay(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -192,7 +190,6 @@ class TestAdamOpWithoutDecay(OpTest): class TestNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -207,9 +204,9 @@ class TestNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -233,16 +230,17 @@ class TestNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_amp_check_finite_and_scale_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_amp_check_finite_and_scale_op_npu.py index ee43d18ae2f..ff11f1e68f0 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_amp_check_finite_and_scale_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_amp_check_finite_and_scale_op_npu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestCheckFiniteAndUnscale(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = paddle.static.Program() @@ -35,20 +34,21 @@ class TestCheckFiniteAndUnscale(unittest.TestCase): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') scale = paddle.static.data(name="scale", shape=[1], dtype='float32') - float_status = paddle.static.data(name="status", - shape=[8], - dtype='float32') + float_status = paddle.static.data( + name="status", shape=[8], dtype='float32' + ) main_program.global_block().append_op( - type="alloc_float_status", - outputs={"FloatStatus": float_status}) + type="alloc_float_status", outputs={"FloatStatus": float_status} + ) main_program.global_block().append_op( type="clear_float_status", inputs={"FloatStatus": float_status}, - outputs={"FloatStatusOut": float_status}) + outputs={"FloatStatusOut": float_status}, + ) c = paddle.fluid.layers.elementwise_div(a, b) - out, found_inf = check_finite_and_unscale([c], - scale, - float_status=float_status) + out, found_inf = check_finite_and_unscale( + [c], scale, float_status=float_status + ) return main_program, out, found_inf, float_status @@ -58,12 +58,9 @@ class TestCheckFiniteAndUnscale(unittest.TestCase): exe = fluid.Executor(place) out_, founf_inf_, float_status_ = exe.run( main_program, - feed={ - "a": a, - "b": b, - "scale": scale - }, - fetch_list=[out, found_inf, float_status]) + feed={"a": a, "b": b, "scale": scale}, + fetch_list=[out, found_inf, float_status], + ) print(float_status_) return out_, founf_inf_ @@ -100,7 +97,6 @@ class TestCheckFiniteAndUnscale(unittest.TestCase): class TestCheckFiniteAndUnscaleClearFloatStatus(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = paddle.static.Program() @@ -108,31 +104,33 @@ class TestCheckFiniteAndUnscaleClearFloatStatus(unittest.TestCase): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') scale = paddle.static.data(name="scale", shape=[1], dtype='float32') - float_status = paddle.static.data(name="status", - shape=[8], - dtype='float32') + float_status = paddle.static.data( + name="status", shape=[8], dtype='float32' + ) main_program.global_block().append_op( - type="alloc_float_status", - outputs={"FloatStatus": float_status}) + type="alloc_float_status", outputs={"FloatStatus": float_status} + ) main_program.global_block().append_op( type="clear_float_status", inputs={"FloatStatus": float_status}, - outputs={"FloatStatusOut": float_status}) + outputs={"FloatStatusOut": float_status}, + ) c = paddle.fluid.layers.elementwise_div(a, b) - out, found_inf = check_finite_and_unscale([c], - scale, - float_status=float_status) + out, found_inf = check_finite_and_unscale( + [c], scale, float_status=float_status + ) main_program.global_block().append_op( - type="alloc_float_status", - outputs={"FloatStatus": float_status}) + type="alloc_float_status", outputs={"FloatStatus": float_status} + ) main_program.global_block().append_op( type="clear_float_status", inputs={"FloatStatus": float_status}, - outputs={"FloatStatusOut": float_status}) + outputs={"FloatStatusOut": float_status}, + ) d = paddle.fluid.layers.elementwise_add(a, b) - out, found_inf = check_finite_and_unscale([d], - scale, - float_status=float_status) + out, found_inf = check_finite_and_unscale( + [d], scale, float_status=float_status + ) return main_program, out, found_inf, float_status @@ -142,12 +140,9 @@ class TestCheckFiniteAndUnscaleClearFloatStatus(unittest.TestCase): exe = fluid.Executor(place) out_, founf_inf_, float_status_ = exe.run( main_program, - feed={ - "a": a, - "b": b, - "scale": scale - }, - fetch_list=[out, found_inf, float_status]) + feed={"a": a, "b": b, "scale": scale}, + fetch_list=[out, found_inf, float_status], + ) print(float_status_) return out_, founf_inf_ diff --git a/python/paddle/fluid/tests/unittests/npu/test_arg_max_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_arg_max_op_npu.py index 46e21378b84..9b089462b01 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_arg_max_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_arg_max_op_npu.py @@ -27,7 +27,6 @@ paddle.enable_static() class BaseTestCase(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -52,7 +51,6 @@ class BaseTestCase(OpTest): # test argmax, dtype: float16 class TestArgMaxFloat16Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -61,7 +59,6 @@ class TestArgMaxFloat16Case1(BaseTestCase): class TestArgMaxFloat16Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -70,7 +67,6 @@ class TestArgMaxFloat16Case2(BaseTestCase): class TestArgMaxFloat16Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -79,7 +75,6 @@ class TestArgMaxFloat16Case3(BaseTestCase): class TestArgMaxFloat16Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -88,7 +83,6 @@ class TestArgMaxFloat16Case4(BaseTestCase): class TestArgMaxFloat16Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -97,7 +91,6 @@ class TestArgMaxFloat16Case5(BaseTestCase): class TestArgMaxFloat16Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -106,7 +99,6 @@ class TestArgMaxFloat16Case6(BaseTestCase): class TestArgMaxFloat16Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -115,35 +107,31 @@ class TestArgMaxFloat16Case7(BaseTestCase): class TestArgMaxFloat16Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float16' self.axis = 0 class TestArgMaxFloat16Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float16' self.axis = 0 class TestArgMaxFloat16Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float16' self.axis = 0 # test argmax, dtype: float32 class TestArgMaxFloat32Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -152,7 +140,6 @@ class TestArgMaxFloat32Case1(BaseTestCase): class TestArgMaxFloat32Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -161,7 +148,6 @@ class TestArgMaxFloat32Case2(BaseTestCase): class TestArgMaxFloat32Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -170,7 +156,6 @@ class TestArgMaxFloat32Case3(BaseTestCase): class TestArgMaxFloat32Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -179,7 +164,6 @@ class TestArgMaxFloat32Case4(BaseTestCase): class TestArgMaxFloat32Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -188,7 +172,6 @@ class TestArgMaxFloat32Case5(BaseTestCase): class TestArgMaxFloat32Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -197,7 +180,6 @@ class TestArgMaxFloat32Case6(BaseTestCase): class TestArgMaxFloat32Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -206,34 +188,30 @@ class TestArgMaxFloat32Case7(BaseTestCase): class TestArgMaxFloat32Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float32' self.axis = 0 class TestArgMaxFloat32Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float32' self.axis = 0 class TestArgMaxFloat32Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float32' self.axis = 0 class BaseTestComplex1_1(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -251,7 +229,7 @@ class BaseTestComplex1_1(OpTest): self.inputs = {'X': self.x} self.attrs = { 'axis': self.axis, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { 'Out': np.argmax(self.x, axis=self.axis).astype("int32") @@ -262,7 +240,6 @@ class BaseTestComplex1_1(OpTest): class BaseTestComplex1_2(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -280,7 +257,7 @@ class BaseTestComplex1_2(OpTest): self.inputs = {'X': self.x} self.attrs = { 'axis': self.axis, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { 'Out': np.argmax(self.x, axis=self.axis).astype("int32") @@ -291,7 +268,6 @@ class BaseTestComplex1_2(OpTest): class TestArgMaxAPI(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -303,7 +279,6 @@ class TestArgMaxAPI(unittest.TestCase): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -311,9 +286,9 @@ class TestArgMaxAPI(unittest.TestCase): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input, axis=self.axis) paddle_output = paddle.argmax(tensor_input, axis=self.axis) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() for place in self.place: @@ -321,7 +296,6 @@ class TestArgMaxAPI(unittest.TestCase): class TestArgMaxAPI_2(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -334,20 +308,20 @@ class TestArgMaxAPI_2(unittest.TestCase): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) numpy_input = (np.random.random(self.dims)).astype(self.dtype) tensor_input = paddle.to_tensor(numpy_input) - numpy_output = np.argmax(numpy_input, - axis=self.axis).reshape(1, 4, 5) - paddle_output = paddle.argmax(tensor_input, - axis=self.axis, - keepdim=self.keep_dims) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + numpy_output = np.argmax(numpy_input, axis=self.axis).reshape( + 1, 4, 5 + ) + paddle_output = paddle.argmax( + tensor_input, axis=self.axis, keepdim=self.keep_dims + ) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -356,7 +330,6 @@ class TestArgMaxAPI_2(unittest.TestCase): class TestArgMaxAPI_3(unittest.TestCase): - def initTestCase(self): self.dims = (1, 9) self.dtype = 'float32' @@ -367,7 +340,6 @@ class TestArgMaxAPI_3(unittest.TestCase): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -375,9 +347,9 @@ class TestArgMaxAPI_3(unittest.TestCase): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input).reshape([1]) paddle_output = paddle.argmax(tensor_input) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/npu/test_arg_min_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_arg_min_op_npu.py index 8aeb32e5e5d..af8c038ac76 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_arg_min_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_arg_min_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class BaseTestCase(OpTest): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -51,7 +50,6 @@ class BaseTestCase(OpTest): # test argmin, dtype: float16 class TestArgMinFloat16Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -60,7 +58,6 @@ class TestArgMinFloat16Case1(BaseTestCase): class TestArgMinFloat16Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -69,7 +66,6 @@ class TestArgMinFloat16Case2(BaseTestCase): class TestArgMinFloat16Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -78,7 +74,6 @@ class TestArgMinFloat16Case3(BaseTestCase): class TestArgMinFloat16Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -87,7 +82,6 @@ class TestArgMinFloat16Case4(BaseTestCase): class TestArgMinFloat16Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -96,7 +90,6 @@ class TestArgMinFloat16Case5(BaseTestCase): class TestArgMinFloat16Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -105,7 +98,6 @@ class TestArgMinFloat16Case6(BaseTestCase): class TestArgMinFloat16Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -114,35 +106,31 @@ class TestArgMinFloat16Case7(BaseTestCase): class TestArgMinFloat16Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float16' self.axis = 0 class TestArgMinFloat16Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float16' self.axis = 0 class TestArgMinFloat16Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float16' self.axis = 0 # test argmin, dtype: float32 class TestArgMinFloat32Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -151,7 +139,6 @@ class TestArgMinFloat32Case1(BaseTestCase): class TestArgMinFloat32Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -160,7 +147,6 @@ class TestArgMinFloat32Case2(BaseTestCase): class TestArgMinFloat32Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -169,7 +155,6 @@ class TestArgMinFloat32Case3(BaseTestCase): class TestArgMinFloat32Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -178,7 +163,6 @@ class TestArgMinFloat32Case4(BaseTestCase): class TestArgMinFloat32Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -187,7 +171,6 @@ class TestArgMinFloat32Case5(BaseTestCase): class TestArgMinFloat32Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -196,7 +179,6 @@ class TestArgMinFloat32Case6(BaseTestCase): class TestArgMinFloat32Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -205,34 +187,30 @@ class TestArgMinFloat32Case7(BaseTestCase): class TestArgMinFloat32Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float32' self.axis = 0 class TestArgMinFloat32Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float32' self.axis = 0 class TestArgMinFloat32Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float32' self.axis = 0 class TestArgMinAPI(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -244,7 +222,6 @@ class TestArgMinAPI(unittest.TestCase): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -252,9 +229,9 @@ class TestArgMinAPI(unittest.TestCase): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmin(numpy_input, axis=self.axis) paddle_output = paddle.argmin(tensor_input, axis=self.axis) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() for place in self.place: @@ -262,7 +239,6 @@ class TestArgMinAPI(unittest.TestCase): class TestArgMaxAPI_2(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -275,20 +251,20 @@ class TestArgMaxAPI_2(unittest.TestCase): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) numpy_input = (np.random.random(self.dims)).astype(self.dtype) tensor_input = paddle.to_tensor(numpy_input) - numpy_output = np.argmin(numpy_input, - axis=self.axis).reshape(1, 4, 5) - paddle_output = paddle.argmin(tensor_input, - axis=self.axis, - keepdim=self.keep_dims) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + numpy_output = np.argmin(numpy_input, axis=self.axis).reshape( + 1, 4, 5 + ) + paddle_output = paddle.argmin( + tensor_input, axis=self.axis, keepdim=self.keep_dims + ) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/npu/test_argsort_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_argsort_op_npu.py index 36f0e5e55f9..5aaf1de2451 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_argsort_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_argsort_op_npu.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestArgsortOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "argsort" @@ -50,9 +49,11 @@ class TestArgsortOp(OpTest): def get_output(self): if self.descending: self.indices = np.flip( - np.argsort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.argsort(self.x, kind='heapsort', axis=self.axis), self.axis + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.sort(self.x, kind='heapsort', axis=self.axis), self.axis + ) else: self.indices = np.argsort(self.x, kind='heapsort', axis=self.axis) self.sorted_x = np.sort(self.x, kind='heapsort', axis=self.axis) @@ -78,67 +79,56 @@ class TestArgsortOp(OpTest): class TestArgsortOpAxis0NPU(TestArgsortOp): - def init_axis(self): self.axis = 0 class TestArgsortOpAxis1NPU(TestArgsortOp): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2NPU(TestArgsortOp): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1NPU(TestArgsortOp): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2NPU(TestArgsortOp): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisNPU(TestArgsortOp): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0NPU(TestArgsortOpAxis0NPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1NPU(TestArgsortOpAxis1NPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2NPU(TestArgsortOpAxis2NPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1NPU(TestArgsortOpAxisNeg1NPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2NPU(TestArgsortOpAxisNeg2NPU): - def init_direction(self): self.descending = True @@ -151,7 +141,6 @@ class TestArgsortOpDescendingAxisNeg2NPU(TestArgsortOpAxisNeg2NPU): class TestArgsortOpAxis0NPUFP32(TestArgsortOp): - def init_axis(self): self.axis = 0 @@ -165,74 +154,63 @@ class TestArgsortOpAxis0NPUFP32(TestArgsortOp): self.__class__.use_npu = True def test_check_grad(self): - self.check_grad_with_place(self.place, ["X"], - "Out", - max_relative_error=0.03) + self.check_grad_with_place( + self.place, ["X"], "Out", max_relative_error=0.03 + ) class TestArgsortOpAxis1NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisNPUFP32(TestArgsortOpAxis0NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1NPUFP32(TestArgsortOpAxis1NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2NPUFP32(TestArgsortOpAxis2NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1NPUFP32(TestArgsortOpAxisNeg1NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2NPUFP32(TestArgsortOpAxisNeg2NPUFP32): - def init_direction(self): self.descending = True # test cases for int64 class TestArgsortOpAxis0NPUINT64(TestArgsortOp): - def setUp(self): self.set_npu() self.op_type = "argsort" @@ -242,10 +220,9 @@ class TestArgsortOpAxis0NPUINT64(TestArgsortOp): self.init_axis() self.init_direction() - self.x = np.random.randint(low=-100, - high=100, - size=self.input_shape, - dtype=self.dtype).astype(self.dtype) + self.x = np.random.randint( + low=-100, high=100, size=self.input_shape, dtype=self.dtype + ).astype(self.dtype) self.inputs = {"X": self.x} self.attrs = {"axis": self.axis, "descending": self.descending} self.get_output() @@ -265,61 +242,51 @@ class TestArgsortOpAxis0NPUINT64(TestArgsortOp): class TestArgsortOpAxis1NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisNPUINT64(TestArgsortOpAxis0NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1NPUINT64(TestArgsortOpAxis1NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2NPUINT64(TestArgsortOpAxis2NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1NPUINT64(TestArgsortOpAxisNeg1NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2NPUINT64(TestArgsortOpAxisNeg2NPUINT64): - def init_direction(self): self.descending = True diff --git a/python/paddle/fluid/tests/unittests/npu/test_assign_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_assign_op_npu.py index 82969805dc1..98da7c4141d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_assign_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_assign_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestAssign(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_assign_value_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_assign_value_op_npu.py index 7ffbcd1db13..402b90bc49b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_assign_value_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_assign_value_op_npu.py @@ -29,7 +29,6 @@ np.random.seed(2021) class TestAssignValueNPUOp(op_test.OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -41,7 +40,8 @@ class TestAssignValueNPUOp(op_test.OpTest): self.attrs["shape"] = self.value.shape self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( - self.value.dtype) + self.value.dtype + ) self.outputs = {"Out": self.value} def set_npu(self): @@ -56,35 +56,36 @@ class TestAssignValueNPUOp(op_test.OpTest): class TestAssignValueNPUOp2(TestAssignValueNPUOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int32) self.attrs["int32_values"] = [int(v) for v in self.value.flat] class TestAssignValueNPUOp3(TestAssignValueNPUOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int64) self.attrs["int64_values"] = [int(v) for v in self.value.flat] class TestAssignValueNPUOp4(TestAssignValueNPUOp): - def init_data(self): - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) self.attrs["bool_values"] = [int(v) for v in self.value.flat] class TestAssignApi(unittest.TestCase): - def setUp(self): self.init_dtype() self.value = (-100 + 200 * np.random.random(size=(2, 5))).astype( - self.dtype) - self.place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + self.dtype + ) + self.place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) def init_dtype(self): self.dtype = "float32" @@ -102,25 +103,26 @@ class TestAssignApi(unittest.TestCase): class TestAssignApi2(TestAssignApi): - def init_dtype(self): self.dtype = "int32" class TestAssignApi3(TestAssignApi): - def init_dtype(self): self.dtype = "int64" class TestAssignApi4(TestAssignApi): - def setUp(self): self.init_dtype() - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) - self.place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) + self.place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) def init_dtype(self): self.dtype = "bool" diff --git a/python/paddle/fluid/tests/unittests/npu/test_atan_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_atan_op_npu.py index bb13d823669..fe22453f4fa 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_atan_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_atan_op_npu.py @@ -26,7 +26,6 @@ SEED = 1024 class TestAtan(OpTest): - def setUp(self): self.set_npu() self.op_type = "atan" @@ -57,7 +56,7 @@ class TestAtan(OpTest): out = paddle.atan(data, name='Y') place = paddle.NPUPlace(0) exe = fluid.Executor(place) - result, = exe.run(feed={"X": np_x}, fetch_list=[out]) + (result,) = exe.run(feed={"X": np_x}, fetch_list=[out]) expected = np.arctan(np_x) self.assertEqual(result, expected) @@ -74,13 +73,11 @@ class TestAtan(OpTest): class TestAtanShape(TestAtan): - def set_attrs(self): self.shape = [12, 23, 10] class TestAtanFloat16(TestAtan): - def set_attrs(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_batch_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_batch_norm_op_npu.py index 37c6fcce1e1..e39506eed7a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_batch_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_batch_norm_op_npu.py @@ -24,24 +24,27 @@ from paddle.fluid.op import Operator from op_test import OpTest, _set_use_system_allocator from paddle.fluid import Program, program_guard -from test_batch_norm_op import _reference_testing, _cal_mean_variance, _reference_training, _reference_grad +from test_batch_norm_op import ( + _reference_testing, + _cal_mean_variance, + _reference_training, + _reference_grad, +) _set_use_system_allocator(False) paddle.enable_static() class TestBatchNormOpInference(unittest.TestCase): - def setUp(self): self.dtype = np.float32 self.init_kernel_type() self.data_formats = ["NCHW", "NHWC"] def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, atol=atol, err_msg=msg + ) def check_with_place(self, place, data_layout, dtype, shape): epsilon = epsilon = 0.00001 @@ -72,8 +75,9 @@ class TestBatchNormOpInference(unittest.TestCase): bias = np.random.random_sample(scale_shape).astype(np.float32) mean = np.zeros(scale_shape).astype(np.float32) variance = np.ones(scale_shape).astype(np.float32) - y = _reference_testing(x, scale, bias, mean, variance, epsilon, - data_layout).astype(dtype) + y = _reference_testing( + x, scale, bias, mean, variance, epsilon, data_layout + ).astype(dtype) var_dict = locals() var_names = ["x", "scale", "bias", "mean", "variance", "y"] ground_truth = {name: var_dict[name] for name in var_names} @@ -84,15 +88,15 @@ class TestBatchNormOpInference(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype="float32", - shape=ground_truth[name].shape) + block.create_var( + name=name, dtype="float32", shape=ground_truth[name].shape + ) inputs = { "X": block.var("x"), "Scale": block.var("scale"), "Bias": block.var("bias"), "Mean": block.var("mean"), - "Variance": block.var("variance") + "Variance": block.var("variance"), } attrs = { "epsilon": epsilon, @@ -106,14 +110,13 @@ class TestBatchNormOpInference(unittest.TestCase): "MeanOut": block.var("mean"), # share memory "VarianceOut": block.var("variance"), # share memory "SavedMean": block.var("saved_mean"), - "SavedVariance": block.var("saved_variance") + "SavedVariance": block.var("saved_variance"), } block.create_var(name="reserve_space", dtype='float32') outputs["ReserveSpace"] = block.var('reserve_space') - bn_op = block.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + bn_op = block.append_op( + type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) program._sync_with_cpp() @@ -124,7 +127,8 @@ class TestBatchNormOpInference(unittest.TestCase): name: ground_truth[name] for name in ["x", "scale", "bias", "mean", "variance"] }, - fetch_list=["y"]) + fetch_list=["y"], + ) self.__assert_close(var_dict["y"], out[0], "y", atol=1e-3) def test_check_output(self): @@ -138,7 +142,6 @@ class TestBatchNormOpInference(unittest.TestCase): class TestFP16BatchNormOpInference(TestBatchNormOpInference): - def setUp(self): self.dtype = np.float16 self.init_kernel_type() @@ -146,7 +149,6 @@ class TestFP16BatchNormOpInference(TestBatchNormOpInference): class TestBatchNormOpTraining(unittest.TestCase): - def set_npu(self): self.__class__.use_npu = True @@ -169,27 +171,54 @@ class TestBatchNormOpTraining(unittest.TestCase): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - "y", 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + "y", + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def __assert_close(self, tensor, np_array, msg, atol=1e-4): np.allclose(np.array(tensor), np_array, atol=atol) - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): # run forward - y, saved_mean, var_ref = _reference_training(x, scale, bias, epsilon, - data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = var_ref * (1. - momentum) + momentum * variance - saved_variance = 1. / np.sqrt(var_ref + epsilon) + y, saved_mean, var_ref = _reference_training( + x, scale, bias, epsilon, data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = var_ref * (1.0 - momentum) + momentum * variance + saved_variance = 1.0 / np.sqrt(var_ref + epsilon) # run backward - x_grad, scale_grad, bias_grad = _reference_grad(x, y_grad, scale, - saved_mean, var_ref, - epsilon, data_layout) - - return y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad + x_grad, scale_grad, bias_grad = _reference_grad( + x, y_grad, scale, saved_mean, var_ref, epsilon, data_layout + ) + + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) def set_mean_variance(self, scale_shape, x, data_layout): mean, variance = _cal_mean_variance(x, self.epsilon, data_layout) @@ -198,12 +227,11 @@ class TestBatchNormOpTraining(unittest.TestCase): # computing global mean/variance for one step if self.use_global_stats: mom = self.momentum - mean = mean * (1. - mom) + mom * mean_pre - variance = variance * (1. - mom) + mom * variance_pre + mean = mean * (1.0 - mom) + mom * mean_pre + variance = variance * (1.0 - mom) + mom * variance_pre return mean, variance def test_forward_backward(self): - def test_with_place(place, data_layout, shape): # attr epsilon = self.epsilon @@ -236,9 +264,27 @@ class TestBatchNormOpTraining(unittest.TestCase): y_grad = np.random.random_sample(shape).astype(self.dtype) momentum_var = np.array([momentum]).astype(np.float32) - y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad = self.ref_forward_backward( - x, y_grad, scale, bias, mean, variance, epsilon, momentum, - shape, data_layout) + ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) = self.ref_forward_backward( + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -247,8 +293,15 @@ class TestBatchNormOpTraining(unittest.TestCase): var_dict['bias@GRAD'] = bias_grad var_names = [ - 'x', 'scale', 'bias', 'mean', 'variance', "y", 'saved_mean', - 'saved_variance', 'momentum_var' + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + "y", + 'saved_mean', + 'saved_variance', + 'momentum_var', ] ground_truth = {name: var_dict[name] for name in var_names} @@ -256,15 +309,17 @@ class TestBatchNormOpTraining(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = { "X": block.var('x'), "Scale": block.var('scale'), "Bias": block.var('bias'), "Mean": block.var('mean'), - "Variance": block.var('variance') + "Variance": block.var('variance'), } attrs = { "epsilon": epsilon, @@ -272,7 +327,7 @@ class TestBatchNormOpTraining(unittest.TestCase): "data_layout": data_layout, "use_mkldnn": self.use_mkldnn, "fuse_with_relu": self.fuse_with_relu, - "use_global_stats": self.use_global_stats + "use_global_stats": self.use_global_stats, } if self.use_momentum_variable: inputs['MomentumTensor'] = block.var('momentum_var') @@ -284,19 +339,22 @@ class TestBatchNormOpTraining(unittest.TestCase): "MeanOut": block.var('mean'), # share memory "VarianceOut": block.var('variance'), # share memory "SavedMean": block.var('saved_mean'), - "SavedVariance": block.var('saved_variance') + "SavedVariance": block.var('saved_variance'), } block.create_var(name="reserve_space", dtype='float32') outputs["ReserveSpace"] = block.var('reserve_space') - bn_op = block.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + bn_op = block.append_op( + type="batch_norm", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) block.create_var(name='y@GRAD', dtype=self.dtype, shape=y.shape) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - bn_op.desc, self.no_grad_set, []) + bn_op.desc, self.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -311,22 +369,28 @@ class TestBatchNormOpTraining(unittest.TestCase): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in [ - 'x', 'scale', 'bias', 'mean', 'variance', - 'y@GRAD', 'momentum_var' - ] - }, - fetch_list=self.fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in [ + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y@GRAD', + 'momentum_var', + ] + }, + fetch_list=self.fetch_list, + ) for id, name in enumerate(self.fetch_list): if name == 'variance': - self.__assert_close(var_dict[name], - out[id], - name, - atol=1e-3) + self.__assert_close( + var_dict[name], out[id], name, atol=1e-3 + ) continue self.__assert_close(var_dict[name], out[id], name) print("op test forward passed: ", str(place), data_layout) @@ -340,13 +404,11 @@ class TestBatchNormOpTraining(unittest.TestCase): class TestFP16BatchNormOpTraining(TestBatchNormOpTraining): - def init_dtype(self): self.dtype = np.float16 class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -354,24 +416,33 @@ class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): class TestBatchNormOpTrainingMomentumVariable(TestBatchNormOpTraining): - def init_test_case(self): self.use_momentum_variable = True self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = True self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'x@GRAD', 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): @@ -379,20 +450,23 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): if len(x_shape) == 3: if data_format == "NCHW": # NCL -> NCL1 x = np.reshape(x, (x_shape[0], x_shape[1], x_shape[2], 1)) - y_grad = np.reshape(y_grad, - (x_shape[0], x_shape[1], x_shape[2], 1)) + y_grad = np.reshape( + y_grad, (x_shape[0], x_shape[1], x_shape[2], 1) + ) else: # NLC -> NL1C x = np.reshape(x, (x_shape[0], x_shape[1], 1, x_shape[2])) - y_grad = np.reshape(y_grad, - (x_shape[0], x_shape[1], 1, x_shape[2])) + y_grad = np.reshape( + y_grad, (x_shape[0], x_shape[1], 1, x_shape[2]) + ) if data_format == "NCHW": x = np.transpose(x, (0, 2, 3, 1)) y_grad = np.transpose(y_grad, (0, 2, 3, 1)) x_grad = scale * y_grad / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -406,8 +480,19 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): return x_grad, grad_scale, grad_offset - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): if data_layout != "NCHW" and data_layout != "NHWC": raise ValueError("Unknown data order.") @@ -415,12 +500,14 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): if len(x_shape) == 3: if data_layout == "NCHW": # NCL -> NCL1 x = np.reshape(x, (x_shape[0], x_shape[1], x_shape[2], 1)) - y_grad = np.reshape(y_grad, - (x_shape[0], x_shape[1], x_shape[2], 1)) + y_grad = np.reshape( + y_grad, (x_shape[0], x_shape[1], x_shape[2], 1) + ) else: # NLC -> NL1C x = np.reshape(x, (x_shape[0], x_shape[1], 1, x_shape[2])) - y_grad = np.reshape(y_grad, - (x_shape[0], x_shape[1], 1, x_shape[2])) + y_grad = np.reshape( + y_grad, (x_shape[0], x_shape[1], 1, x_shape[2]) + ) if data_layout == "NCHW": x = np.transpose(x, (0, 2, 3, 1)) @@ -436,21 +523,31 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): mean_out = mean variance_out = variance - saved_variance = 1. / np.sqrt(variance + epsilon) + saved_variance = 1.0 / np.sqrt(variance + epsilon) # run backward x_grad, scale_grad, bias_grad = self.reference_grad( - x, y_grad, scale, mean, variance, epsilon, data_layout) + x, y_grad, scale, mean, variance, epsilon, data_layout + ) if len(x_shape) == 3: y = np.reshape(y, x_shape) x_grad = np.reshape(x_grad, x_shape) - return y, mean_out, variance_out, mean, saved_variance, x_grad, scale_grad, bias_grad + return ( + y, + mean_out, + variance_out, + mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) class TestBatchNormOpFreezeStatsAndScaleBiasTraining( - TestBatchNormOpFreezeStatsTraining): - + TestBatchNormOpFreezeStatsTraining +): def init_test_case(self): self.use_global_stats = True self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -458,7 +555,6 @@ class TestBatchNormOpFreezeStatsAndScaleBiasTraining( class TestDygraphBatchNormTrainableStats(unittest.TestCase): - def test_dygraph(self): places = [fluid.NPUPlace(0)] for p in places: @@ -469,7 +565,8 @@ class TestDygraphBatchNormTrainableStats(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -489,7 +586,8 @@ class TestDygraphBatchNormTrainableStats(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) diff --git a/python/paddle/fluid/tests/unittests/npu/test_bce_loss_npu.py b/python/paddle/fluid/tests/unittests/npu/test_bce_loss_npu.py index 10ec6f3fae1..d9e968964af 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_bce_loss_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_bce_loss_npu.py @@ -24,88 +24,76 @@ from op_test import OpTest paddle.enable_static() -def test_static_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') - bce_loss = paddle.nn.loss.BCELoss(weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) + bce_loss = paddle.nn.loss.BCELoss( + weight=weight, reduction=reduction + ) else: bce_loss = paddle.nn.loss.BCELoss(reduction=reduction) res = bce_loss(input, label) exe = paddle.static.Executor(place) - static_result = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + static_result = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result[0] -def test_static_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') - res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) + res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + static_result = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result[0] -def test_dygraph_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static(place) if weight_np is not None: weight = paddle.to_tensor(weight_np) @@ -118,25 +106,22 @@ def test_dygraph_layer(place, return dy_result -def test_dygraph_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static(place) input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) if weight_np is not None: weight = paddle.to_tensor(weight_np) - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -144,11 +129,19 @@ def test_dygraph_functional(place, def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): if weight_np is None: - expected = -1 * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = -1 * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) else: - expected = -1 * weight_np * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = ( + -1 + * weight_np + * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) + ) if reduction == 'mean': expected = np.mean(expected) @@ -161,7 +154,6 @@ def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): class TestBCELoss(unittest.TestCase): - def test_BCELoss(self): input_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float32) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float32) @@ -171,87 +163,88 @@ class TestBCELoss(unittest.TestCase): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static_layer(place, input_np, label_np, - reduction) - dy_result = test_dygraph_layer(place, input_np, label_np, - reduction) + static_result = test_static_layer( + place, input_np, label_np, reduction + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction + ) expected = calc_bceloss(input_np, label_np, reduction) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) static_functional = test_static_functional( - place, input_np, label_np, reduction) - dy_functional = test_dygraph_functional(place, input_np, - label_np, reduction) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-6) + place, input_np, label_np, reduction + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-6 + ) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCELoss_weight(self): - input_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float32) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float32) + input_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float32 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float32 + ) weight_np = np.random.random(size=(3, 4, 10)).astype(np.float32) - place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) for reduction in ['sum', 'mean', 'none']: - static_result = test_static_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_result = test_dygraph_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - expected = calc_bceloss(input_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + expected = calc_bceloss( + input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result, rtol=1e-6) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) - static_functional = test_static_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_functional = test_dygraph_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) + static_functional = test_static_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-6) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-6) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-6 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCELoss_error(self): paddle.disable_static(paddle.NPUPlace(0)) - self.assertRaises(ValueError, - paddle.nn.loss.BCELoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, paddle.nn.loss.BCELoss, reduction="unsupport reduction" + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def bce_loss(input, label): - return -1 * (label * np.log(input) + (1. - label) * np.log(1. - input)) + return -1 * (label * np.log(input) + (1.0 - label) * np.log(1.0 - input)) class TestBceLossOp(OpTest): - def setUp(self): self.set_npu() self.init_test_case() @@ -278,13 +271,11 @@ class TestBceLossOp(OpTest): class TestBceLossOpCase1(OpTest): - def init_test_cast(self): self.shape = [2, 3, 4, 5] class TestBceLossOpCase2(OpTest): - def init_test_cast(self): self.shape = [2, 3, 20] diff --git a/python/paddle/fluid/tests/unittests/npu/test_beam_search_decode_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_beam_search_decode_op_npu.py index f00fc60fb42..7f5588f34ac 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_beam_search_decode_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_beam_search_decode_op_npu.py @@ -42,28 +42,41 @@ class TestBeamSearchDecodeNPUOp(unittest.TestCase): # beam_size = 2, end_id = 1 # start with start_id [ - self.append_lod_tensor(array, [[0, 1, 2], [0, 1, 2]], - np.array([0, 0], dtype=dtype)) + self.append_lod_tensor( + array, [[0, 1, 2], [0, 1, 2]], np.array([0, 0], dtype=dtype) + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 1, 2], [0, 2, 4]], - np.array([2, 3, 4, 5], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 1, 2], [0, 2, 4]], + np.array([2, 3, 4, 5], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 2, 2, 4, 4]], - np.array([3, 1, 5, 4], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 2, 2, 4, 4]], + np.array([3, 1, 5, 4], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 1, 2, 3, 4]], - np.array([1, 1, 3, 5], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 1, 2, 3, 4]], + np.array([1, 1, 3, 5], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 0, 0, 2, 2]], - np.array([5, 1], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 0, 0, 2, 2]], + np.array([5, 1], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] @@ -89,7 +102,8 @@ class TestBeamSearchDecodeNPUOp(unittest.TestCase): self.assertEqual(sentence_scores.lod(), expected_lod) expected_data = np.array( - [0, 2, 3, 1, 0, 2, 1, 0, 4, 5, 3, 5, 0, 4, 5, 3, 1], "int64") + [0, 2, 3, 1, 0, 2, 1, 0, 4, 5, 3, 5, 0, 4, 5, 3, 1], "int64" + ) np.testing.assert_array_equal(np.array(sentence_ids), expected_data) np.testing.assert_array_equal(np.array(sentence_scores), expected_data) diff --git a/python/paddle/fluid/tests/unittests/npu/test_beam_search_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_beam_search_op_npu.py index d185260a86a..df340b97cdb 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_beam_search_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_beam_search_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestBeamSearchNPUOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -35,19 +34,19 @@ class TestBeamSearchNPUOp(OpTest): 'pre_ids': (self.pre_ids, self.lod), 'pre_scores': (self.pre_score, self.lod), 'ids': (self.ids, self.lod), - 'scores': (self.score, self.lod) + 'scores': (self.score, self.lod), } # The `target_lod` attribute is still based on offset self.attrs = { 'level': 0, 'beam_size': self.beam_size, 'end_id': 0, - 'is_accumulated': self.is_accumulated + 'is_accumulated': self.is_accumulated, } self.outputs = { 'selected_ids': (self.selected_ids, self.out_lod), 'selected_scores': (self.selected_scores, self.out_lod), - 'parent_idx': self.parent_idx + 'parent_idx': self.parent_idx, } def set_npu(self): @@ -57,18 +56,21 @@ class TestBeamSearchNPUOp(OpTest): self.beam_size = 2 self.is_accumulated = True self.pre_ids = np.array([[1], [2], [3], [4]], dtype='int64') - self.ids = np.array([[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], - dtype='int64') + self.ids = np.array( + [[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], dtype='int64' + ) self.lod = [[2, 2], [1, 1, 1, 1]] self.out_lod = [[2, 2], [1, 1, 1, 1]] self.offset_lod = [[0, 2, 4], [0, 1, 2, 3, 4]] - self.score = np.array([ - [0.5, 0.3, 0.2], - [0.6, 0.3, 0.1], - [0.9, 0.5, 0.1], - [0.7, 0.5, 0.1], - ], - dtype='float32') + self.score = np.array( + [ + [0.5, 0.3, 0.2], + [0.6, 0.3, 0.1], + [0.9, 0.5, 0.1], + [0.7, 0.5, 0.1], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [0.2], [0.3], [0.4]], dtype='float32') self.selected_ids = np.array([4, 2, 3, 8])[:, np.newaxis] self.selected_scores = np.array([0.5, 0.6, 0.9, 0.7])[:, np.newaxis] @@ -79,7 +81,6 @@ class TestBeamSearchNPUOp(OpTest): class TestBeamSearchNPUOp2(TestBeamSearchNPUOp): - def init_data(self): self.beam_size = 2 self.is_accumulated = True @@ -88,13 +89,15 @@ class TestBeamSearchNPUOp2(TestBeamSearchNPUOp): self.lod = [[2, 2], [1, 1, 1, 1]] self.out_lod = [[2, 2], [2, 0, 1, 1]] self.offset_lod = [[0, 2, 4], [0, 2, 2, 3, 4]] - self.score = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + self.score = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [0.2], [0.3], [0.4]], dtype='float32') self.selected_ids = np.array([4, 2, 3, 1])[:, np.newaxis] self.selected_scores = np.array([0.6, 0.9, 0.9, 0.7])[:, np.newaxis] @@ -102,7 +105,6 @@ class TestBeamSearchNPUOp2(TestBeamSearchNPUOp): class TestBeamSearchNPUOp3(TestBeamSearchNPUOp): - def init_data(self): # end_id = 0 self.beam_size = 2 @@ -112,13 +114,15 @@ class TestBeamSearchNPUOp3(TestBeamSearchNPUOp): self.lod = [[2, 2], [1, 1, 1, 1]] self.out_lod = [[2, 2], [1, 1, 0, 2]] self.offset_lod = [[0, 2, 4], [0, 1, 2, 2, 4]] - self.score = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.6, 0.7], - ], - dtype='float32') + self.score = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.6, 0.7], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [1.2], [0.5], [0.4]], dtype='float32') self.selected_ids = np.array([2, 0, 8, 1])[:, np.newaxis] self.selected_scores = np.array([0.9, 1.2, 0.6, 0.7])[:, np.newaxis] @@ -126,7 +130,6 @@ class TestBeamSearchNPUOp3(TestBeamSearchNPUOp): class TestBeamSearchNPUOp4(TestBeamSearchNPUOp): - def init_data(self): # is_accumulated = False self.beam_size = 2 @@ -136,22 +139,24 @@ class TestBeamSearchNPUOp4(TestBeamSearchNPUOp): self.lod = [[2, 2], [1, 1, 1, 1]] self.out_lod = [[2, 2], [0, 2, 1, 1]] self.offset_lod = [[0, 2, 4], [0, 0, 2, 3, 4]] - self.score = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + self.score = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [2.2], [0.3], [0.4]], dtype='float32') self.selected_ids = np.array([7, 3, 3, 1])[:, np.newaxis] - self.selected_scores = np.array([1.50685, 0.996027, 0.194639, - 0.043325])[:, np.newaxis] + self.selected_scores = np.array( + [1.50685, 0.996027, 0.194639, 0.043325] + )[:, np.newaxis] self.parent_idx = np.array([1, 1, 2, 3]) class TestBeamSearchNPUOp5(TestBeamSearchNPUOp): - def init_data(self): # beam_size = 1 self.beam_size = 1 @@ -161,13 +166,15 @@ class TestBeamSearchNPUOp5(TestBeamSearchNPUOp): self.lod = [[1, 1, 1, 1], [1, 1, 1, 1]] self.out_lod = [[1, 1, 1, 1], [1, 1, 1, 1]] self.offset_lod = [[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]] - self.score = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + self.score = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [0.2], [0.3], [0.4]], dtype='float32') self.selected_ids = np.array([2, 7, 3, 1])[:, np.newaxis] self.selected_scores = np.array([0.9, 0.5, 0.9, 0.7])[:, np.newaxis] diff --git a/python/paddle/fluid/tests/unittests/npu/test_bilinear_interp_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_bilinear_interp_v2_op_npu.py index 5bcd736f540..5429125b4ef 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_bilinear_interp_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_bilinear_interp_v2_op_npu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestBilinearInterpOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -53,7 +52,7 @@ class TestBilinearInterpOp(OpTest): scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -66,10 +65,18 @@ class TestBilinearInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, scale_w, scale_h, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + scale_w, + scale_h, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: @@ -83,11 +90,11 @@ class TestBilinearInterpOp(OpTest): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -106,13 +113,19 @@ class TestBilinearInterpOp(OpTest): output_names = ['Out'] no_grad_set = set() cpu_place = fluid.CPUPlace() - cpu_grads = self._get_gradient(inputs_to_check, cpu_place, output_names, - no_grad_set) - npu_grads = self._get_gradient(inputs_to_check, self.place, - output_names, no_grad_set) - self._assert_is_close(cpu_grads, npu_grads, inputs_to_check, - self.max_relative_error, - "Gradient Check between places") + cpu_grads = self._get_gradient( + inputs_to_check, cpu_place, output_names, no_grad_set + ) + npu_grads = self._get_gradient( + inputs_to_check, self.place, output_names, no_grad_set + ) + self._assert_is_close( + cpu_grads, + npu_grads, + inputs_to_check, + self.max_relative_error, + "Gradient Check between places", + ) def init_test_case(self): self.interp_method = 'bilinear' @@ -127,7 +140,6 @@ class TestBilinearInterpOp(OpTest): class TestBilinearInterpCaseFP16(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCaseFP16, self).init_test_case() self.dtype = 'float16' @@ -135,70 +147,63 @@ class TestBilinearInterpCaseFP16(TestBilinearInterpOp): class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase1, self).init_test_case() self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase2, self).init_test_case() self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase3, self).init_test_case() self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase4, self).init_test_case() self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase5, self).init_test_case() self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase6, self).init_test_case() self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 33]).astype("int32") class TestBilinearInterpCase7(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase7, self).init_test_case() self.input_shape = [1, 1, 32, 64] @@ -208,81 +213,72 @@ class TestBilinearInterpCase7(TestBilinearInterpOp): class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpSame, self).init_test_case() self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpActualShape, self).init_test_case() self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") class TestBilinearInterpDataLayout(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpDataLayout, self).init_test_case() self.input_shape = [2, 5, 5, 3] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.data_layout = "NHWC" class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpScale1, self).init_test_case() self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpScale2, self).init_test_case() self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpZero, self).init_test_case() self.input_shape = [2, 3, 5, 7] diff --git a/python/paddle/fluid/tests/unittests/npu/test_box_coder_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_box_coder_op_npu.py index 6655c8f09f6..a8409474e06 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_box_coder_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_box_coder_op_npu.py @@ -40,9 +40,11 @@ def box_decoder(t_box, p_box, pb_v, output_box, norm, axis=0): pb_y = pb_y.reshape(shape) if pb_v.ndim == 2: - var_shape = (1, pb_v.shape[0], - pb_v.shape[1]) if axis == 0 else (pb_v.shape[0], 1, - pb_v.shape[1]) + var_shape = ( + (1, pb_v.shape[0], pb_v.shape[1]) + if axis == 0 + else (pb_v.shape[0], 1, pb_v.shape[1]) + ) pb_v = pb_v.reshape(var_shape) if pb_v.ndim == 1: tb_x = pb_v[0] * t_box[:, :, 0] * pb_w + pb_x @@ -99,20 +101,24 @@ def batch_box_coder(p_box, pb_v, t_box, lod, code_type, norm, axis=0): cur_offset = 0 for i in range(len(lod)): - if (code_type == "encode_center_size"): - box_encoder(t_box[cur_offset:(cur_offset + lod[i]), :], p_box, pb_v, - output_box[cur_offset:(cur_offset + lod[i]), :, :], - norm) - elif (code_type == "decode_center_size"): + if code_type == "encode_center_size": + box_encoder( + t_box[cur_offset : (cur_offset + lod[i]), :], + p_box, + pb_v, + output_box[cur_offset : (cur_offset + lod[i]), :, :], + norm, + ) + elif code_type == "decode_center_size": box_decoder(t_box, p_box, pb_v, output_box, norm, axis) cur_offset += lod[i] return output_box -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestBoxCoderOp(OpTest): - def setUp(self): self.op_type = "box_coder" self.set_npu() @@ -143,10 +149,10 @@ class TestBoxCoderOp(OpTest): def set_inputs(self): self.inputs = {} - assert (self.code_type in ['decode_center_size', 'encode_center_size']) - assert (self.axis in [0, 1]) + assert self.code_type in ['decode_center_size', 'encode_center_size'] + assert self.axis in [0, 1] if self.code_type == 'decode_center_size': - assert (not self.use_variance or not self.without_prior_box_var) + assert not self.use_variance or not self.without_prior_box_var self.prior_box = np.random.random((self.M, 4)).astype(self.dtype) @@ -156,50 +162,60 @@ class TestBoxCoderOp(OpTest): if self.without_prior_box_var: self.prior_box_var = np.ones((self.M, 4)).astype(self.dtype) else: - self.prior_box_var = np.random.random( - (self.M, 4)).astype(self.dtype) + self.prior_box_var = np.random.random((self.M, 4)).astype( + self.dtype + ) if self.axis == 0: - self.target_box = np.random.random( - (self.N, self.M, 4)).astype(self.dtype) + self.target_box = np.random.random((self.N, self.M, 4)).astype( + self.dtype + ) else: - self.target_box = np.random.random( - (self.M, self.N, 4)).astype(self.dtype) + self.target_box = np.random.random((self.M, self.N, 4)).astype( + self.dtype + ) self.inputs['PriorBox'] = self.prior_box self.inputs['TargetBox'] = self.target_box - if (not self.use_variance and not self.without_prior_box_var): + if not self.use_variance and not self.without_prior_box_var: self.inputs['PriorBoxVar'] = self.prior_box_var else: - #encode_center_size + # encode_center_size self.prior_box = np.random.random((self.M, 4)).astype(self.dtype) if self.use_variance: self.prior_box_var = np.random.random(4).astype(self.dtype) else: - self.prior_box_var = np.random.random( - (self.M, 4)).astype(self.dtype) + self.prior_box_var = np.random.random((self.M, 4)).astype( + self.dtype + ) self.target_box = np.random.random((self.N, 4)).astype(self.dtype) self.inputs['PriorBox'] = self.prior_box - #self.inputs['PriorBoxVar'] = self.prior_box_var + # self.inputs['PriorBoxVar'] = self.prior_box_var self.inputs['TargetBox'] = (self.target_box, self.lod) - if (not self.use_variance): + if not self.use_variance: self.inputs['PriorBoxVar'] = self.prior_box_var def set_attrs(self): self.attrs = { 'code_type': self.code_type, - 'box_normalized': self.box_normalized + 'box_normalized': self.box_normalized, } if self.use_variance: self.attrs['variance'] = self.prior_box_var.astype( - np.float64).flatten() + np.float64 + ).flatten() if self.axis != 0: self.attrs['axis'] = self.axis def set_outputs(self): - output_box = batch_box_coder(self.prior_box, self.prior_box_var, - self.target_box, self.lod[0], - self.code_type, self.box_normalized, - self.axis) + output_box = batch_box_coder( + self.prior_box, + self.prior_box_var, + self.target_box, + self.lod[0], + self.code_type, + self.box_normalized, + self.axis, + ) self.outputs = {'OutputBox': output_box.astype(self.dtype)} def test_check_output(self): @@ -207,7 +223,6 @@ class TestBoxCoderOp(OpTest): class TestBoxCoderOpWithoutBoxVar(TestBoxCoderOp): - def set_init_config(self): super(TestBoxCoderOpWithoutBoxVar, self).set_init_config() self.without_prior_box_var = True @@ -215,7 +230,6 @@ class TestBoxCoderOpWithoutBoxVar(TestBoxCoderOp): class TestBoxCoderOpWithLoD(TestBoxCoderOp): - def set_init_config(self): super(TestBoxCoderOpWithLoD, self).set_init_config() self.M = 20 @@ -226,28 +240,24 @@ class TestBoxCoderOpWithLoD(TestBoxCoderOp): class TestBoxCoderOpWithLoDWithVariance(TestBoxCoderOpWithLoD): - def set_init_config(self): super(TestBoxCoderOpWithLoDWithVariance, self).set_init_config() self.use_variance = True class TestBoxCoderOpWithAxis(TestBoxCoderOp): - def set_init_config(self): super(TestBoxCoderOpWithAxis, self).set_init_config() self.axis = 1 class TestBoxCoderOpWithVariance(TestBoxCoderOp): - def set_init_config(self): super(TestBoxCoderOpWithVariance, self).set_init_config() self.use_variance = True class TestBoxCoderOpFP16(TestBoxCoderOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_c_embedding_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_c_embedding_op_npu.py index fb9447cc2d4..eaceefe7e6d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_c_embedding_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_c_embedding_op_npu.py @@ -21,7 +21,11 @@ from op_test import OpTest import paddle import paddle.fluid as fluid import paddle.fluid.core as core -from paddle.fluid.tests.unittests.c_embedding_op_base import TestCEmbeddingCPU, TestCEmbeddingOpBase, TestCEmbeddingOpFP32 +from paddle.fluid.tests.unittests.c_embedding_op_base import ( + TestCEmbeddingCPU, + TestCEmbeddingOpBase, + TestCEmbeddingOpFP32, +) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/npu/test_c_identity_npu.py b/python/paddle/fluid/tests/unittests/npu/test_c_identity_npu.py index 5701739c883..381fd5baa46 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_c_identity_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_c_identity_npu.py @@ -23,15 +23,14 @@ paddle.enable_static() class TestIdentityOp(TestDistBase): - def _setup_config(self): pass def test_identity(self, col_type="identity"): dist_env = os.environ - self.check_with_place("collective_identity_op_npu.py", - col_type, - need_envs=dist_env) + self.check_with_place( + "collective_identity_op_npu.py", col_type, need_envs=dist_env + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_cast_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_cast_op_npu.py index 4b3e877fef1..dfc65107463 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_cast_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_cast_op_npu.py @@ -28,7 +28,6 @@ SEED = 2021 @skip_check_grad_ci(reason="[skip NPU cast grad check] not implemented yet.") class TestCast1(OpTest): - def setUp(self): self.set_npu() self.op_type = "cast" @@ -40,7 +39,7 @@ class TestCast1(OpTest): self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } def set_npu(self): @@ -52,7 +51,6 @@ class TestCast1(OpTest): @skip_check_grad_ci(reason="[skip NPU cast grad check] not implemented yet.") class TestCast2(OpTest): - def setUp(self): self.set_npu() self.op_type = "cast" @@ -64,7 +62,7 @@ class TestCast2(OpTest): self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP16), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } def set_npu(self): @@ -76,7 +74,6 @@ class TestCast2(OpTest): @skip_check_grad_ci(reason="[skip NPU cast grad check] not implemented yet.") class TestCast3(OpTest): - def setUp(self): self.set_npu() self.op_type = "cast" @@ -88,7 +85,7 @@ class TestCast3(OpTest): self.attrs = { 'in_dtype': int(core.VarDesc.VarType.INT32), - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } def set_npu(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_clip_by_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_clip_by_norm_op_npu.py index a1c397796cc..bff2e674e54 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_clip_by_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_clip_by_norm_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestClipByNormOp(OpTest): - def setUp(self): self.set_npu() self.max_relative_error = 0.006 @@ -55,7 +54,7 @@ class TestClipByNormOp(OpTest): self.check_output_with_place(self.place) def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1.0 def init_dtype(self): @@ -63,28 +62,24 @@ class TestClipByNormOp(OpTest): class TestCase1(TestClipByNormOp): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestCase2(TestClipByNormOp): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestCase3(TestClipByNormOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 class TestClipByNormOpFp16(TestClipByNormOp): - def init_dtype(self): self.dtype = np.float16 @@ -93,21 +88,18 @@ class TestClipByNormOpFp16(TestClipByNormOp): class TestClipByNormOpFp16Case1(TestClipByNormOpFp16): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestClipByNormOpFp16Case2(TestClipByNormOpFp16): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestClipByNormOpFp16Case3(TestClipByNormOpFp16): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 diff --git a/python/paddle/fluid/tests/unittests/npu/test_clip_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_clip_op_npu.py index 3c6b6cb8949..8b13546d9a2 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_clip_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_clip_op_npu.py @@ -24,7 +24,6 @@ from op_test import OpTest class TestClipOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -75,7 +74,6 @@ class TestClipOp(OpTest): class TestCase1(TestClipOp): - def initTestCase(self): self.shape = (8, 16, 8) self.max = 0.7 @@ -83,7 +81,6 @@ class TestCase1(TestClipOp): class TestCase2(TestClipOp): - def initTestCase(self): self.shape = (8, 16) self.max = 1.0 @@ -91,7 +88,6 @@ class TestCase2(TestClipOp): class TestCase3(TestClipOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max = 0.7 @@ -99,7 +95,6 @@ class TestCase3(TestClipOp): class TestCase4(TestClipOp): - def initTestCase(self): self.shape = (4, 8, 8) self.max = 0.7 @@ -109,7 +104,6 @@ class TestCase4(TestClipOp): class TestCase5(TestClipOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max = 0.5 @@ -117,7 +111,6 @@ class TestCase5(TestClipOp): class TestClipOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -137,7 +130,6 @@ class TestClipOpError(unittest.TestCase): class TestClipAPI(unittest.TestCase): - def _executed_api(self, x, min=None, max=None): return paddle.clip(x, min, max) @@ -149,8 +141,11 @@ class TestClipAPI(unittest.TestCase): min = fluid.data(name='min', shape=[1], dtype='float32') max = fluid.data(name='max', shape=[1], dtype='float32') - place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) out_1 = self._executed_api(images, min=min, max=max) @@ -159,7 +154,7 @@ class TestClipAPI(unittest.TestCase): out_4 = self._executed_api(images, max=0.7) out_5 = self._executed_api(images, min=min) out_6 = self._executed_api(images, max=max) - out_7 = self._executed_api(images, max=-1.) + out_7 = self._executed_api(images, max=-1.0) out_8 = self._executed_api(images) res1, res2, res3, res4, res5, res6, res7, res8 = exe.run( @@ -167,9 +162,10 @@ class TestClipAPI(unittest.TestCase): feed={ "image": data, "min": np.array([0.2]).astype('float32'), - "max": np.array([0.8]).astype('float32') + "max": np.array([0.8]).astype('float32'), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) np.testing.assert_allclose(res1, data.clip(0.2, 0.8)) np.testing.assert_allclose(res2, data.clip(0.2, 0.9)) @@ -183,8 +179,11 @@ class TestClipAPI(unittest.TestCase): def test_clip_dygraph(self): paddle.disable_static() - place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) paddle.disable_static(place) data_shape = [1, 9, 9, 4] data = np.random.random(data_shape).astype('float32') @@ -212,7 +211,6 @@ class TestClipAPI(unittest.TestCase): class TestInplaceClipAPI(TestClipAPI): - def _executed_api(self, x, min=None, max=None): return x.clip_(min, max) diff --git a/python/paddle/fluid/tests/unittests/npu/test_coalesce_tensor_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_coalesce_tensor_op_npu.py index ec257b84d43..8f6ef521070 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_coalesce_tensor_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_coalesce_tensor_op_npu.py @@ -28,7 +28,6 @@ alignment = 512 class TestAllocContinuousSpace(OpTest): - def setUp(self): self.__class__.use_npu = True self.op_type = "coalesce_tensor" @@ -39,7 +38,8 @@ class TestAllocContinuousSpace(OpTest): self.set_constant = attrs["set_constant"] self.Inputs = self.init_input() self.Outputs, self.FusedOutput = self.init_output( - self.Inputs, self.set_constant, self.constant) + self.Inputs, self.set_constant, self.constant + ) self.inputs = {'Input': self.Inputs} self.attrs = attrs self.outputs = {'Output': self.Outputs, 'FusedOutput': self.FusedOutput} @@ -59,7 +59,7 @@ class TestAllocContinuousSpace(OpTest): "set_constant": False, "constant": 0.0, "use_align": True, - "dtype": self.fluid_dtype + "dtype": self.fluid_dtype, } def init_output(self, input_list, set_constant, constant): @@ -85,7 +85,6 @@ class TestAllocContinuousSpace(OpTest): class TestAllocContinuousSpace2(TestAllocContinuousSpace): - def init_attr(self): return { "copy_data": True, @@ -93,7 +92,7 @@ class TestAllocContinuousSpace2(TestAllocContinuousSpace): "constant": 0.5, "use_align": True, "dtype": self.fluid_dtype, - "user_defined_size_of_dtype": 2 + "user_defined_size_of_dtype": 2, } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_collective_base_npu.py b/python/paddle/fluid/tests/unittests/npu/test_collective_base_npu.py index fbbb64aaf07..6b7e056a7f8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_collective_base_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_collective_base_npu.py @@ -29,10 +29,10 @@ from paddle.fluid import core class TestCollectiveRunnerBase(object): - def get_model(self, train_prog, startup_prog): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): assert not isinstance(endpoints, str) @@ -41,13 +41,15 @@ class TestCollectiveRunnerBase(object): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -55,43 +57,50 @@ class TestCollectiveRunnerBase(object): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - hccl_id_var = block.create_var(name=nameGen.generate('hccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': hccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': hccl_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': self.global_ring_id, - 'device_id': int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks - }) + hccl_id_var = block.create_var( + name=nameGen.generate('hccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': hccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': hccl_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': self.global_ring_id, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + }, + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -100,8 +109,9 @@ class TestCollectiveRunnerBase(object): rank = args["trainerid"] current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) self.rank = rank result = self.get_model(train_prog, startup_prog) device_id = int(os.getenv("FLAGS_selected_npus", "0")) @@ -110,9 +120,9 @@ class TestCollectiveRunnerBase(object): exe.run(startup_prog) np.random.seed(os.getpid()) indata = np.random.random((10, 1000)) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=[result.name]) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=[result.name] + ) sys.stdout.buffer.write(pickle.dumps(out)) @@ -134,19 +144,20 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -159,7 +170,7 @@ class TestDistBase(unittest.TestCase): def _run_cluster(self, model_file, envs): worker_endpoints = self._ps_endpoints.split(",") w0_ep, w1_ep = worker_endpoints - #print("w0_ep:",w0_ep," w1_ep:",w1_ep) + # print("w0_ep:",w0_ep," w1_ep:",w1_ep) env0 = { "FLAGS_selected_npus": "0", "PADDLE_TRAINER_ID": "0", @@ -175,7 +186,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) tr_cmd = "%s %s" @@ -183,16 +194,20 @@ class TestDistBase(unittest.TestCase): tr1_cmd = tr_cmd % (self._python_interp, model_file) tr0_pipe = open("/tmp/tr0_err.log", "wb") tr1_pipe = open("/tmp/tr1_err.log", "wb") - #print(tr0_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + # print(tr0_cmd) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -201,8 +216,12 @@ class TestDistBase(unittest.TestCase): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) def check_with_place(self, model_file, col_type, need_envs={}): diff --git a/python/paddle/fluid/tests/unittests/npu/test_collective_process_group_hccl.py b/python/paddle/fluid/tests/unittests/npu/test_collective_process_group_hccl.py index f26d33840ae..181ba75c11b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_collective_process_group_hccl.py +++ b/python/paddle/fluid/tests/unittests/npu/test_collective_process_group_hccl.py @@ -20,7 +20,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestProcessGroup(TestMultipleGpus): - def test_process_group_nccl(self): self.run_mnist_2gpu('process_group_hccl.py') diff --git a/python/paddle/fluid/tests/unittests/npu/test_compare_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_compare_op_npu.py index b31b805fb77..4fd0be14c12 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_compare_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_compare_op_npu.py @@ -24,9 +24,7 @@ from paddle.fluid import Program, program_guard def create_test_class(op_type, typename, callback): - class Cls(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -77,46 +75,42 @@ def create_test_class(op_type, typename, callback): def test_broadcast_api_1(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 1, 3], - dtype=typename) - y = paddle.static.data(name='y', - shape=[1, 2, 3], - dtype=typename) + x = paddle.static.data( + name='x', shape=[1, 2, 1, 3], dtype=typename + ) + y = paddle.static.data( + name='y', shape=[1, 2, 3], dtype=typename + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(typename) input_y = np.arange(0, 6).reshape((1, 2, 3)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") def test_broadcast_api_2(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 3], - dtype=typename) - y = paddle.static.data(name='y', - shape=[1, 2, 1, 3], - dtype=typename) + x = paddle.static.data( + name='x', shape=[1, 2, 3], dtype=typename + ) + y = paddle.static.data( + name='y', shape=[1, 2, 1, 3], dtype=typename + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(0, 6).reshape((1, 2, 3)).astype(typename) input_y = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") @@ -131,11 +125,9 @@ def create_test_class(op_type, typename, callback): input_x = np.arange(0, 5).reshape((5)).astype(typename) input_y = np.array([5, 3, 2]).reshape((3, 1)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") diff --git a/python/paddle/fluid/tests/unittests/npu/test_concat_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_concat_op_npu.py index a9c43a868f5..301ce9c4773 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_concat_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_concat_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestConcatOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "concat" @@ -43,8 +42,9 @@ class TestConcatOp(OpTest): self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } def set_npu(self): @@ -69,7 +69,6 @@ class TestConcatOp(OpTest): class TestConcatOp2(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -78,9 +77,9 @@ class TestConcatOp2(TestConcatOp): @skip_check_grad_ci( - reason="The function 'check_grad' for large inputs is too slow.") + reason="The function 'check_grad' for large inputs is too slow." +) class TestConcatOp3(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((1, 256, 170, 256)).astype(self.dtype) self.x1 = np.random.random((1, 128, 170, 256)).astype(self.dtype) @@ -92,11 +91,9 @@ class TestConcatOp3(TestConcatOp): @skip_check_grad_ci( - reason= - "This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." + reason="This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." ) class TestConcatOp4(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -108,7 +105,6 @@ class TestConcatOp4(TestConcatOp): class TestConcatOp5(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((5, 1, 4, 5)).astype(self.dtype) self.x1 = np.random.random((5, 2, 4, 5)).astype(self.dtype) @@ -116,11 +112,9 @@ class TestConcatOp5(TestConcatOp): self.axis = -3 -#----------------Concat Fp16---------------- +# ----------------Concat Fp16---------------- def create_test_fp16(parent): - class TestConcatFp16(parent): - def init_dtype(self): self.dtype = np.float16 @@ -136,11 +130,9 @@ create_test_fp16(TestConcatOp4) create_test_fp16(TestConcatOp5) -#----------------Concat Int64---------------- +# ----------------Concat Int64---------------- def create_test_int64(parent): - class TestConcatInt64(parent): - def init_dtype(self): self.dtype = np.int64 @@ -179,9 +171,9 @@ class TestConcatAPIWithLoDTensorArray(unittest.TestCase): with fluid.program_guard(self.program): input = fluid.layers.assign(self.x) tensor_array = fluid.layers.create_array(dtype='float32') - zero = fluid.layers.fill_constant(shape=[1], - value=0, - dtype="int64") + zero = fluid.layers.fill_constant( + shape=[1], value=0, dtype="int64" + ) for i in range(self.iter_num): fluid.layers.array_write(input, zero + i, tensor_array) @@ -217,7 +209,8 @@ class TestConcatAPIWithLoDTensorArray(unittest.TestCase): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_allclose( - res[0], np.concatenate([self.x] * self.iter_num, axis=self.axis)) + res[0], np.concatenate([self.x] * self.iter_num, axis=self.axis) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_depthwise_conv_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_depthwise_conv_npu.py index 0161f6bcdce..537912edb46 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_depthwise_conv_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_depthwise_conv_npu.py @@ -31,9 +31,7 @@ SEED = 2021 def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -47,9 +45,7 @@ def create_test_channel_last_class(parent): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -60,9 +56,7 @@ def create_test_padding_SAME_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -73,9 +67,7 @@ def create_test_padding_VALID_class(parent): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_data_type(self): self.dtype = np.float16 @@ -85,7 +77,6 @@ def create_test_fp16_class(parent): class TestDepthwiseConvNPU(OpTest): - def setUp(self): self.set_npu() self.op_type = "depthwise_conv2d" @@ -97,21 +88,26 @@ class TestDepthwiseConvNPU(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, filter, self.groups, - conv2d_param, "EXPLICIT", - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + "EXPLICIT", + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { @@ -143,41 +139,59 @@ class TestDepthwiseConvNPU(OpTest): def test_check_grad(self): if self.dilations[0] == 1 and self.dilations[1] == 1: if self.dtype == np.float16: - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.9, + ) else: - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter']), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + no_grad_set=set(['Filter']), + max_relative_error=0.9, + ) else: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter']), - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + no_grad_set=set(['Filter']), + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dilations[0] == 1 and self.dilations[1] == 1: if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + max_relative_error=0.9, + ) else: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def init_data_format(self): self.data_format = "NCHW" @@ -190,7 +204,6 @@ class TestDepthwiseConvNPU(OpTest): class TestDepthwiseConvNPU2(TestDepthwiseConvNPU): - def init_test_case(self): self.pad = [1, 1] self.dilations = [1, 1] @@ -203,7 +216,6 @@ class TestDepthwiseConvNPU2(TestDepthwiseConvNPU): class TestDepthwiseConvNPU3(TestDepthwiseConvNPU): - def init_test_case(self): self.pad = [1, 1] self.dilations = [1, 1] @@ -216,7 +228,6 @@ class TestDepthwiseConvNPU3(TestDepthwiseConvNPU): class TestDepthwiseConvNPU4(TestDepthwiseConvNPU): - def init_test_case(self): self.pad = [1, 1] self.dilations = [1, 1] @@ -229,7 +240,6 @@ class TestDepthwiseConvNPU4(TestDepthwiseConvNPU): class TestDepthwiseConvNPU_Padding(OpTest): - def setUp(self): self.op_type = "depthwise_conv2d" self.dtype = np.float32 @@ -243,21 +253,25 @@ class TestDepthwiseConvNPU_Padding(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, filter, self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { @@ -266,7 +280,7 @@ class TestDepthwiseConvNPU_Padding(OpTest): 'padding_algorithm': self.padding_algorithm, 'groups': self.groups, 'dilations': self.dilations, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -289,40 +303,58 @@ class TestDepthwiseConvNPU_Padding(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=1.2) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=1.2, + ) else: - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.7, - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.7, + no_grad_set=set(['Filter']), + ) else: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.8, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.8, + no_grad_set=set(['Input']), + ) else: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_data_format(self): self.data_format = "NCHW" @@ -339,7 +371,6 @@ class TestDepthwiseConvNPU_Padding(OpTest): class TestDepthwiseConvNPU2_Padding(TestDepthwiseConvNPU_Padding): - def init_test_case(self): self.pad = [1, 1, 0, 1] self.dilations = [1, 1] @@ -356,7 +387,6 @@ class TestDepthwiseConvNPU2_Padding(TestDepthwiseConvNPU_Padding): class TestDepthwiseConvNPU3_Padding(TestDepthwiseConvNPU_Padding): - def init_test_case(self): self.pad = [1, 1, 0, 1] self.dilations = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_npu.py index 974607bc252..d30a68ac916 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_npu.py @@ -28,9 +28,7 @@ paddle.enable_static() def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -44,9 +42,7 @@ def create_test_channel_last_class(parent): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -57,9 +53,7 @@ def create_test_padding_SAME_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -70,9 +64,7 @@ def create_test_padding_VALID_class(parent): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_dtype(self): self.dtype = np.float16 @@ -82,7 +74,6 @@ def create_test_fp16_class(parent): class TestConv2DOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -104,22 +95,24 @@ class TestConv2DOp(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, - filter, - self.groups, - conv2d_param, - data_format=self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + data_format=self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -134,24 +127,33 @@ class TestConv2DOp(OpTest): self.check_output_with_place(fluid.NPUPlace(0), atol=1e-2) def test_check_grad(self): - self.check_grad_with_place(fluid.NPUPlace(0), {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + fluid.NPUPlace(0), + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): - self.check_grad_with_place(fluid.NPUPlace(0), ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + fluid.NPUPlace(0), + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): - self.check_grad_with_place(fluid.NPUPlace(0), ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + fluid.NPUPlace(0), + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -169,7 +171,6 @@ class TestConv2DOp(OpTest): class TestWithPad(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -180,7 +181,6 @@ class TestWithPad(TestConv2DOp): class TestWithStride(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -191,7 +191,6 @@ class TestWithStride(TestConv2DOp): class TestWithGroup(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -203,7 +202,6 @@ class TestWithGroup(TestConv2DOp): class TestWith1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -220,7 +218,6 @@ class TestWith1x1(TestConv2DOp): class TestWithDepthWise5x5(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -234,7 +231,6 @@ class TestWithDepthWise5x5(TestConv2DOp): class TestWithDepthWise7x7(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -248,7 +244,6 @@ class TestWithDepthWise7x7(TestConv2DOp): class TestWithDilation(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -265,7 +260,6 @@ class TestWithDilation(TestConv2DOp): class TestWithInput1x1Filter1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -279,7 +273,6 @@ class TestWithInput1x1Filter1x1(TestConv2DOp): class TestConv2DOp_v2(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -301,20 +294,24 @@ class TestConv2DOp_v2(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, filter, self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -331,39 +328,57 @@ class TestConv2DOp_v2(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(paddle.NPUPlace(0), {'Input', 'Filter'}, - 'Output', - max_relative_error=1.1) + self.check_grad_with_place( + paddle.NPUPlace(0), + {'Input', 'Filter'}, + 'Output', + max_relative_error=1.1, + ) else: - self.check_grad_with_place(paddle.NPUPlace(0), {'Input', 'Filter'}, - 'Output', - max_relative_error=0.02, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + paddle.NPUPlace(0), + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.02, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: - self.check_grad_with_place(paddle.NPUPlace(0), ['Input'], - 'Output', - max_relative_error=0.99, - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + paddle.NPUPlace(0), + ['Input'], + 'Output', + max_relative_error=0.99, + no_grad_set=set(['Filter']), + ) else: - self.check_grad_with_place(paddle.NPUPlace(0), ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + paddle.NPUPlace(0), + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: - self.check_grad_with_place(paddle.NPUPlace(0), ['Filter'], - 'Output', - max_relative_error=0.99, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + paddle.NPUPlace(0), + ['Filter'], + 'Output', + max_relative_error=0.99, + no_grad_set=set(['Input']), + ) else: - self.check_grad_with_place(paddle.NPUPlace(0), ['Filter'], - 'Output', - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + paddle.NPUPlace(0), + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -394,14 +409,12 @@ class TestConv2DOp_v2(OpTest): class TestConv2DOp_AsyPadding(TestConv2DOp_v2): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -415,7 +428,6 @@ class TestWithPad_AsyPadding(TestConv2DOp_v2): class TestWithStride_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -429,7 +441,6 @@ class TestWithStride_AsyPadding(TestConv2DOp_v2): class TestWithGroup_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 2] @@ -441,7 +452,6 @@ class TestWithGroup_AsyPadding(TestConv2DOp_v2): class TestWith1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -458,7 +468,6 @@ class TestWith1x1_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [3, 4, 10, 10] # NCHW @@ -478,7 +487,6 @@ class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 4, 10, 10] # NCHW @@ -495,7 +503,6 @@ class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 8, 10, 10] # NCHW @@ -512,7 +519,6 @@ class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): class TestWithDilation_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -532,7 +538,6 @@ class TestWithDilation_AsyPadding(TestConv2DOp_v2): class TestWithInput1x1Filter1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [100, 1, 1, 1] # NCHW diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv2d_transpose_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv2d_transpose_op_npu.py index ff66e35b0c6..d27c98b270b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_conv2d_transpose_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv2d_transpose_op_npu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestConv2DTransposeOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -62,15 +61,16 @@ class TestConv2DTransposeOp(OpTest): 'use_cudnn': False, 'is_test': False, 'use_mkldnn': False, - 'data_format': self.data_format + 'data_format': self.data_format, } if self.output_size is not None: self.attrs['output_size'] = self.output_size if len(self.output_padding) > 0: self.attrs['output_padding'] = self.output_padding - output = conv2dtranspose_forward_naive(input_, filter_, - self.attrs).astype(self.dtype) + output = conv2dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype(self.dtype) self.outputs = {'Output': output} @@ -79,26 +79,34 @@ class TestConv2DTransposeOp(OpTest): def test_check_grad_no_input(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter']), - max_relative_error=0.006, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + no_grad_set=set(['Filter']), + max_relative_error=0.006, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad(self): if self.need_check_grad: - self.check_grad_with_place(self.place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + set(['Input', 'Filter']), + 'Output', + max_relative_error=0.02, + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -117,7 +125,6 @@ class TestConv2DTransposeOp(OpTest): class TestWithSymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -129,14 +136,12 @@ class TestWithSymmetricPad(TestConv2DTransposeOp): class TestWithSymmetricPad_FP16(TestWithSymmetricPad): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -148,14 +153,12 @@ class TestWithAsymmetricPad(TestConv2DTransposeOp): class TestWithAsymmetricPad_FP16(TestWithAsymmetricPad): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [2, 1] self.dilations = [1, 2] @@ -167,14 +170,12 @@ class TestWithSAMEPad(TestConv2DTransposeOp): class TestWithSAMEPad_FP16(TestWithSAMEPad): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -186,14 +187,12 @@ class TestWithVALIDPad(TestConv2DTransposeOp): class TestWithVALIDPad_FP16(TestWithVALIDPad): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithGroups(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -205,14 +204,12 @@ class TestWithGroups(TestConv2DTransposeOp): class TestWithGroups_FP16(TestWithGroups): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithStride(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -224,14 +221,12 @@ class TestWithStride(TestConv2DTransposeOp): class TestWithStride_FP16(TestWithStride): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithDilation(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -243,14 +238,12 @@ class TestWithDilation(TestConv2DTransposeOp): class TestWithDilation_FP16(TestWithDilation): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithEvenUpsample(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -263,14 +256,12 @@ class TestWithEvenUpsample(TestConv2DTransposeOp): class TestWithEvenUpsample_FP16(TestWithEvenUpsample): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -283,14 +274,12 @@ class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): class TestWithEvenUpsampleOutputPadding_FP16(TestWithEvenUpsampleOutputPadding): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class Test_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -303,14 +292,12 @@ class Test_NHWC(TestConv2DTransposeOp): class Test_NHWC_FP16(Test_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -323,14 +310,12 @@ class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): class TestWithSymmetricPad_NHWC_FP16(TestWithSymmetricPad_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -343,14 +328,12 @@ class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): class TestWithAsymmetricPad_NHWC_FP16(TestWithAsymmetricPad_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithGroups_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -363,14 +346,12 @@ class TestWithGroups_NHWC(TestConv2DTransposeOp): class TestWithGroups_NHWC_FP16(TestWithGroups_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithStride_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -383,14 +364,12 @@ class TestWithStride_NHWC(TestConv2DTransposeOp): class TestWithStride_NHWC_FP16(TestWithStride_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithDilation_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -403,14 +382,12 @@ class TestWithDilation_NHWC(TestConv2DTransposeOp): class TestWithDilation_NHWC_FP16(TestWithDilation_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -424,14 +401,12 @@ class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): class TestWithEvenUpsample_NHWC_FP16(TestWithEvenUpsample_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -445,64 +420,75 @@ class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): class TestWithEvenUpsample_NHWC_output_padding_FP16( - TestWithEvenUpsample_NHWC_output_padding): - + TestWithEvenUpsample_NHWC_output_padding +): def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestConv2DTransposeAPI(unittest.TestCase): - def test_case1(self): - data1 = fluid.layers.data(name='data1', - shape=[3, 5, 5], - dtype='float32') - data2 = fluid.layers.data(name='data2', - shape=[5, 5, 3], - dtype='float32') - out1 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - data_format='NCHW') - out2 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - data_format='NHWC') - out3 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - data_format='NHWC') - out4 = fluid.layers.conv2d_transpose(input=data1, - groups=3, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [2, 1], - [0, 0]], - data_format='NCHW') - out5 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='SAME', - data_format='NCHW') - out6 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding='VALID', - data_format='NHWC') - out7 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - output_size=[7, 7], - padding=[0, 0], - data_format='NHWC') + data1 = fluid.layers.data( + name='data1', shape=[3, 5, 5], dtype='float32' + ) + data2 = fluid.layers.data( + name='data2', shape=[5, 5, 3], dtype='float32' + ) + out1 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + data_format='NCHW', + ) + out2 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) + out3 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + data_format='NHWC', + ) + out4 = fluid.layers.conv2d_transpose( + input=data1, + groups=3, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [2, 1], [0, 0]], + data_format='NCHW', + ) + out5 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='SAME', + data_format='NCHW', + ) + out6 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding='VALID', + data_format='NHWC', + ) + out7 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + output_size=[7, 7], + padding=[0, 0], + data_format='NHWC', + ) data1_np = np.random.random((2, 3, 5, 5)).astype("float32") data2_np = np.random.random((2, 5, 5, 3)).astype("float32") @@ -510,13 +496,12 @@ class TestConv2DTransposeAPI(unittest.TestCase): place = core.NPUPlace(0) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2, out3, out4, out5, out6, out7], + return_numpy=True, + ) self.assertIsNotNone(results[0]) self.assertIsNotNone(results[1]) self.assertIsNotNone(results[2]) @@ -527,10 +512,9 @@ class TestConv2DTransposeAPI(unittest.TestCase): class TestConv2DTransposeRepr(unittest.TestCase): - def test_case(self): paddle.disable_static(paddle.NPUPlace(0)) - x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = nn.Conv2DTranspose(4, 6, (3, 3), output_padding=1, stride=2) print(conv) y_var = conv(x_var) diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv3d_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv3d_op_npu.py index fcbfde79e3f..81af05b156c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_conv3d_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv3d_op_npu.py @@ -29,9 +29,7 @@ paddle.enable_static() def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0, 0] self.padding_algorithm = "SAME" @@ -42,9 +40,7 @@ def create_test_padding_SAME_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1, 1] self.padding_algorithm = "VALID" @@ -55,9 +51,7 @@ def create_test_padding_VALID_class(parent): def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NDHWC" @@ -71,9 +65,7 @@ def create_test_channel_last_class(parent): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_dtype(self): self.dtype = np.float16 @@ -83,7 +75,6 @@ def create_test_fp16_class(parent): class TestConv3DOp(OpTest): - def setUp(self): self.op_type = "conv3d" self.set_npu() @@ -96,7 +87,7 @@ class TestConv3DOp(OpTest): conv3d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilations': self.dilations + 'dilations': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -110,14 +101,14 @@ class TestConv3DOp(OpTest): self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, 'paddings': self.pad, 'groups': self.groups, 'dilations': self.dilations, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -128,30 +119,39 @@ class TestConv3DOp(OpTest): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def set_npu(self): self.__class__.use_npu = True @@ -179,7 +179,6 @@ class TestConv3DOp(OpTest): class TestCase1(TestConv3DOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -193,7 +192,6 @@ class TestCase1(TestConv3DOp): class TestConv3DOp_2(OpTest): - def setUp(self): self.op_type = "conv3d" self.set_npu() @@ -209,18 +207,23 @@ class TestConv3DOp_2(OpTest): conv3d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilations': self.dilations + 'dilations': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = conv3d_forward_naive(input, filter, self.groups, conv3d_param, - self.padding_algorithm, - self.data_format).astype(self.dtype) + output = conv3d_forward_naive( + input, + filter, + self.groups, + conv3d_param, + self.padding_algorithm, + self.data_format, + ).astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -228,7 +231,7 @@ class TestConv3DOp_2(OpTest): 'padding_algorithm': self.padding_algorithm, 'groups': self.groups, 'dilations': self.dilations, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -239,30 +242,39 @@ class TestConv3DOp_2(OpTest): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def set_npu(self): self.__class__.use_npu = True @@ -296,7 +308,6 @@ class TestConv3DOp_2(OpTest): class TestConv3DOp_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 2] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -310,7 +321,6 @@ class TestConv3DOp_AsyPadding(TestConv3DOp_2): class TestConv3DOp_DiffDataInDiffDim(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 2] self.input_size = [2, 3, 4, 5, 5] # NCDHW @@ -324,7 +334,6 @@ class TestConv3DOp_DiffDataInDiffDim(TestConv3DOp_2): class TestCase1_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -339,196 +348,228 @@ class TestCase1_AsyPadding(TestConv3DOp_2): # --------- test python API --------------- class TestConv3DAPI(unittest.TestCase): - def test_api(self): - input_NDHWC = fluid.layers.data(name="input_NDHWC", - shape=[2, 5, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCDHW = fluid.layers.data(name="input_NCDHW", - shape=[2, 3, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - fluid.layers.conv3d(input=input_NDHWC, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=0, - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[1, 2, 1, 0, 1, 0], - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NDHWC, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], - dilation=[1, 1, 1], - groups=1, - data_format="NDHWC") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding="SAME", - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding="VALID", - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") + input_NDHWC = fluid.layers.data( + name="input_NDHWC", + shape=[2, 5, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCDHW = fluid.layers.data( + name="input_NCDHW", + shape=[2, 3, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + fluid.layers.conv3d( + input=input_NDHWC, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=0, + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[1, 2, 1, 0, 1, 0], + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NDHWC, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], + dilation=[1, 1, 1], + groups=1, + data_format="NDHWC", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding="SAME", + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding="VALID", + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) class TestConv3DAPI_Error(unittest.TestCase): - def test_api(self): - input = fluid.layers.data(name="input", - shape=[2, 5, 5, 5, 4], - append_batch_size=False, - dtype="float32") + input = fluid.layers.data( + name="input", + shape=[2, 5, 5, 5, 4], + append_batch_size=False, + dtype="float32", + ) # ValueError: cudnn def run_1(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=1, - use_cudnn=[0], - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + use_cudnn=[0], + data_format="NCDHW", + ) self.assertRaises(ValueError, run_1) # ValueError: data_format def run_2(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=0, - dilation=[1, 1, 1], - groups=1, - use_cudnn=False, - data_format="NCHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=0, + dilation=[1, 1, 1], + groups=1, + use_cudnn=False, + data_format="NCHWC", + ) self.assertRaises(ValueError, run_2) # ValueError: padding def run_3(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding="SAMEE", - dilation=1, - groups=1, - use_cudnn=False, - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding="SAMEE", + dilation=1, + groups=1, + use_cudnn=False, + data_format="NCDHW", + ) self.assertRaises(ValueError, run_3) def run_4(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=[[0, 1], [0, 0], [0, 1], [0, 1], [0, - 1]], - dilation=1, - groups=1, - use_cudnn=False, - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=[[0, 1], [0, 0], [0, 1], [0, 1], [0, 1]], + dilation=1, + groups=1, + use_cudnn=False, + data_format="NCDHW", + ) self.assertRaises(ValueError, run_4) def run_5(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=0, - stride=0, - padding=[[0, 1], [0, 1], [0, 1], [0, 1], [0, - 1]], - dilation=1, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=0, + stride=0, + padding=[[0, 1], [0, 1], [0, 1], [0, 1], [0, 1]], + dilation=1, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_5) # ValueError: channel dimmention - x = fluid.layers.data(name="x", - shape=[2, 5, 5, 5, -1], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name="x", + shape=[2, 5, 5, 5, -1], + append_batch_size=False, + dtype="float32", + ) def run_6(): - fluid.layers.conv3d(input=x, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=x, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_6) # ValueError: groups def run_7(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=3, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=3, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_7) # ValueError: filter num def run_8(): - fluid.layers.conv3d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_8) diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv3d_transpose_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv3d_transpose_op_npu.py index da6966cf50a..29113544c3b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_conv3d_transpose_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv3d_transpose_op_npu.py @@ -29,9 +29,10 @@ paddle.enable_static() def conv3dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 4, 1, 2, 3]) @@ -42,17 +43,21 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -88,34 +93,50 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, - g * sub_in_c:(g + 1) * sub_in_c, - d, i, j] # (c) - input_masked = np.reshape(input_masked, - (sub_in_c, 1, 1, 1)) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, d, i, j + ] # (c) + input_masked = np.reshape( + input_masked, (sub_in_c, 1, 1, 1) + ) input_masked = np.tile(input_masked, (1, f_d, f_h, f_w)) for k in range(f_out_c): - tmp_out = np.sum(input_masked * - filter_[g * sub_in_c:(g + 1) * - sub_in_c, k, :, :, :], - axis=0) + tmp_out = np.sum( + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, + k, + :, + :, + :, + ], + axis=0, + ) d1, d2 = d * stride[0], d * stride[0] + d_bolck_d i1, i2 = i * stride[1], i * stride[1] + d_bolck_h j1, j2 = j * stride[2], j * stride[2] + d_bolck_w - out[n, g * f_out_c + k, d1:d2:dilations[0], - i1:i2:dilations[1], - j1:j2:dilations[2]] += tmp_out - - out = out[:, :, pad_d_0:out_d - pad_d_1, pad_h_0:out_h - pad_h_1, - pad_w_0:out_w - pad_w_1] + out[ + n, + g * f_out_c + k, + d1 : d2 : dilations[0], + i1 : i2 : dilations[1], + j1 : j2 : dilations[2], + ] += tmp_out + + out = out[ + :, + :, + pad_d_0 : out_d - pad_d_1, + pad_h_0 : out_h - pad_h_1, + pad_w_0 : out_w - pad_w_1, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 4, 1]) return out class TestConv3DTransposeOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -141,11 +162,12 @@ class TestConv3DTransposeOp(OpTest): 'padding_algorithm': self.padding_algorithm, 'dilations': self.dilations, 'groups': self.groups, - 'data_format': self.data_format + 'data_format': self.data_format, } - output = conv3dtranspose_forward_naive(input_, filter_, - self.attrs).astype("float32") + output = conv3dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype("float32") self.outputs = {'Output': output} @@ -166,7 +188,6 @@ class TestConv3DTransposeOp(OpTest): class TestWithSymmetricPad(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_input = True self.pad = [1, 1, 1] @@ -179,7 +200,6 @@ class TestWithSymmetricPad(TestConv3DTransposeOp): class TestWithAsymmetricPad(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 0, 1, 2] self.stride = [1, 1, 1] @@ -191,7 +211,6 @@ class TestWithAsymmetricPad(TestConv3DTransposeOp): class TestWithSAMEPad(TestConv3DTransposeOp): - def init_test_case(self): self.stride = [1, 1, 2] self.dilations = [1, 2, 1] @@ -203,7 +222,6 @@ class TestWithSAMEPad(TestConv3DTransposeOp): class TestWithVALIDPad(TestConv3DTransposeOp): - def init_test_case(self): self.stride = [2, 1, 1] self.dilations = [1, 1, 1] @@ -215,7 +233,6 @@ class TestWithVALIDPad(TestConv3DTransposeOp): class TestWithStride(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_filter = True self.pad = [1, 1, 1] @@ -228,7 +245,6 @@ class TestWithStride(TestConv3DTransposeOp): class TestWithDilation(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -240,7 +256,6 @@ class TestWithDilation(TestConv3DTransposeOp): class Test_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] diff --git a/python/paddle/fluid/tests/unittests/npu/test_cos_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_cos_op_npu.py index f3e73b0c65e..1ab4edef710 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_cos_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_cos_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestCos(OpTest): - def setUp(self): self.set_npu() self.op_type = "cos" @@ -55,7 +54,6 @@ class TestCos(OpTest): class TestCosFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "cos" @@ -82,7 +80,6 @@ class TestCosFp16(OpTest): class TestCosNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -97,9 +94,9 @@ class TestCosNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.cos(c) @@ -123,16 +120,17 @@ class TestCosNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_crop_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_crop_op_npu.py index 446ba19c2a4..1ab9f21fdce 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_crop_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_crop_op_npu.py @@ -28,7 +28,6 @@ np.random.seed(10) class TestCropOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -41,7 +40,7 @@ class TestCropOp(OpTest): if self.crop_by_input: self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.crop_shape).astype(self.dtype) + 'Y': np.random.random(self.crop_shape).astype(self.dtype), } else: self.attrs['shape'] = self.crop_shape @@ -74,7 +73,6 @@ class TestCropOp(OpTest): class TestCase1(TestCropOp): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [2, 2, 3] @@ -82,7 +80,6 @@ class TestCase1(TestCropOp): class TestCase2(TestCropOp): - def initTestCase(self): self.x_shape = (15, 8) self.crop_shape = [15, 8] @@ -90,7 +87,6 @@ class TestCase2(TestCropOp): class TestCase3(TestCropOp): - def initTestCase(self): self.x_shape = (4, 10) self.crop_shape = [2, 3] @@ -99,7 +95,6 @@ class TestCase3(TestCropOp): class TestCase4(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -107,7 +102,6 @@ class TestCase4(TestCropOp): class TestCase5(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -116,7 +110,6 @@ class TestCase5(TestCropOp): class TestCase6(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -127,7 +120,6 @@ class TestCase6(TestCropOp): class TestCase7(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -137,7 +129,6 @@ class TestCase7(TestCropOp): class TestCase8(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -146,7 +137,6 @@ class TestCase8(TestCropOp): class TestCase9(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -155,7 +145,6 @@ class TestCase9(TestCropOp): class TestCase10(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] diff --git a/python/paddle/fluid/tests/unittests/npu/test_cumsum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_cumsum_op_npu.py index 64cfef9ced6..4f9badc230a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_cumsum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_cumsum_op_npu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestCumsumOp(unittest.TestCase): - def run_cases(self): data_np = np.arange(12).reshape(3, 4) data = paddle.to_tensor(data_np) @@ -65,11 +64,17 @@ class TestCumsumOp(unittest.TestCase): place = fluid.NPUPlace(0) if use_npu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - out = exe.run(feed={'X': data_np}, - fetch_list=[ - y.name, y2.name, y3.name, y4.name, y5.name, - y6.name - ]) + out = exe.run( + feed={'X': data_np}, + fetch_list=[ + y.name, + y2.name, + y3.name, + y4.name, + y5.name, + y6.name, + ], + ) z = np.cumsum(data_np) np.testing.assert_allclose(z, out[0]) @@ -95,7 +100,6 @@ class TestCumsumOp(unittest.TestCase): class TestNPUCumSumOp1(OpTest): - def setUp(self): self.op_type = "cumsum" self.set_npu() @@ -119,18 +123,17 @@ class TestNPUCumSumOp1(OpTest): class TestNPUCumSumOp2(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': -1, 'reverse': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.outputs = { - 'Out': np.flip(np.flip(self.inputs['X'], axis=2).cumsum(axis=2), - axis=2) + 'Out': np.flip( + np.flip(self.inputs['X'], axis=2).cumsum(axis=2), axis=2 + ) } class TestNPUCumSumOp3(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 1} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} @@ -138,7 +141,6 @@ class TestNPUCumSumOp3(TestNPUCumSumOp1): class TestNPUCumSumOp4(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 0} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} @@ -146,107 +148,115 @@ class TestNPUCumSumOp4(TestNPUCumSumOp1): class TestNPUCumSumOp5(TestNPUCumSumOp1): - def init_testcase(self): self.inputs = {'X': np.random.random((5, 20)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum(axis=1)} class TestNPUCumSumOp7(TestNPUCumSumOp1): - def init_testcase(self): self.inputs = {'X': np.random.random((100)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum(axis=0)} class TestNPUCumSumExclusive1(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 65)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive2(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 888)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive3(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 888)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive4(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 3049)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive5(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 3096)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumReverseExclusive(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, 'reverse': True, "exclusive": True} a = np.random.random((4, 5, 6)).astype(self.dtype) self.inputs = {'X': a} a = np.flip(a, axis=2) self.outputs = { - 'Out': - np.concatenate( - (np.flip(a[:, :, :-1].cumsum(axis=2), - axis=2), np.zeros((4, 5, 1), dtype=self.dtype)), - axis=2) + 'Out': np.concatenate( + ( + np.flip(a[:, :, :-1].cumsum(axis=2), axis=2), + np.zeros((4, 5, 1), dtype=self.dtype), + ), + axis=2, + ) } class TestNPUCumSumWithFlatten1(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'flatten': True} self.inputs = {'X': np.random.random((5, 6)).astype(self.dtype)} @@ -254,31 +264,28 @@ class TestNPUCumSumWithFlatten1(TestNPUCumSumOp1): class TestNPUCumSumWithFlatten2(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'flatten': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum()} -#----------------Cumsum Int64---------------- +# ----------------Cumsum Int64---------------- class TestNPUCumSumOpInt64(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': -1, 'reverse': True} self.inputs = { 'X': np.random.randint(1, 10000, size=(5, 6, 10)).astype(self.dtype) } self.outputs = { - 'Out': np.flip(np.flip(self.inputs['X'], axis=2).cumsum(axis=2), - axis=2) + 'Out': np.flip( + np.flip(self.inputs['X'], axis=2).cumsum(axis=2), axis=2 + ) } def create_test_int64(parent): - class TestCumSumInt64(parent): - def init_dtype(self): self.dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_density_prior_box_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_density_prior_box_op_npu.py index c3bb0880299..8c7ea643edc 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_density_prior_box_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_density_prior_box_op_npu.py @@ -27,12 +27,11 @@ np.random.seed(2021) class TestNpuDensityPriorBoxOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() self.init_test_output() - #self.init_test_output2() + # self.init_test_output2() self.inputs = {'Input': self.input, 'Image': self.image} self.attrs = { @@ -44,7 +43,7 @@ class TestNpuDensityPriorBoxOp(OpTest): 'densities': self.densities, 'fixed_sizes': self.fixed_sizes, 'fixed_ratios': self.fixed_ratios, - 'flatten_to_2d': self.flatten_to_2d + 'flatten_to_2d': self.flatten_to_2d, } self.outputs = {'Boxes': self.out_boxes, 'Variances': self.out_var} @@ -89,19 +88,20 @@ class TestNpuDensityPriorBoxOp(OpTest): if len(self.fixed_sizes) > 0 and len(self.densities) > 0: for density in self.densities: if len(self.fixed_ratios) > 0: - self.num_priors += len(self.fixed_ratios) * (pow( - density, 2)) + self.num_priors += len(self.fixed_ratios) * ( + pow(density, 2) + ) self.offset = 0.5 self.atol = 1e-5 def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_h, - self.image_w)).astype(self.dtype) + (self.batch_size, self.image_channels, self.image_h, self.image_w) + ).astype(self.dtype) self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_h, - self.layer_w)).astype(self.dtype) + (self.batch_size, self.input_channels, self.layer_h, self.layer_w) + ).astype(self.dtype) def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -115,32 +115,56 @@ class TestNpuDensityPriorBoxOp(OpTest): c_x = (w + self.offset) * self.step_w c_y = (h + self.offset) * self.step_h # Generate density prior boxes with fixed size - for density, fixed_size in zip(self.densities, - self.fixed_sizes): - if (len(self.fixed_ratios) > 0): + for density, fixed_size in zip( + self.densities, self.fixed_sizes + ): + if len(self.fixed_ratios) > 0: for ar in self.fixed_ratios: shift = int(step_average / density) box_width_ratio = fixed_size * math.sqrt(ar) box_height_ratio = fixed_size / math.sqrt(ar) for di in range(density): for dj in range(density): - c_x_temp = c_x - step_average / 2.0 + shift / 2.0 + dj * shift - c_y_temp = c_y - step_average / 2.0 + shift / 2.0 + di * shift + c_x_temp = ( + c_x + - step_average / 2.0 + + shift / 2.0 + + dj * shift + ) + c_y_temp = ( + c_y + - step_average / 2.0 + + shift / 2.0 + + di * shift + ) out_boxes[h, w, idx, :] = [ - max((c_x_temp - box_width_ratio / 2.0) / - self.image_w, 0), - max((c_y_temp - box_height_ratio / 2.0) - / self.image_h, 0), - min((c_x_temp + box_width_ratio / 2.0) / - self.image_w, 1), - min((c_y_temp + box_height_ratio / 2.0) - / self.image_h, 1) + max( + (c_x_temp - box_width_ratio / 2.0) + / self.image_w, + 0, + ), + max( + (c_y_temp - box_height_ratio / 2.0) + / self.image_h, + 0, + ), + min( + (c_x_temp + box_width_ratio / 2.0) + / self.image_w, + 1, + ), + min( + (c_y_temp + box_height_ratio / 2.0) + / self.image_h, + 1, + ), ] idx += 1 if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype(self.dtype) self.out_var = out_var.astype(self.dtype) if self.flatten_to_2d: @@ -149,7 +173,6 @@ class TestNpuDensityPriorBoxOp(OpTest): class TestNpuDensityPriorBoxFlatten(TestNpuDensityPriorBoxOp): - def set_density(self): self.densities = [3, 4] self.fixed_sizes = [1.0, 2.0] @@ -162,7 +185,6 @@ class TestNpuDensityPriorBoxFlatten(TestNpuDensityPriorBoxOp): class TestNpuDensityPriorBoxOp1(TestNpuDensityPriorBoxOp): - def set_density(self): super(TestNpuDensityPriorBoxOp1, self).set_density() self.layer_w = 1 @@ -170,7 +192,6 @@ class TestNpuDensityPriorBoxOp1(TestNpuDensityPriorBoxOp): class TestNpuDensityPriorBoxOp2(TestNpuDensityPriorBoxOp): - def set_density(self): super(TestNpuDensityPriorBoxOp2, self).set_density() self.layer_w = 15 @@ -180,14 +201,12 @@ class TestNpuDensityPriorBoxOp2(TestNpuDensityPriorBoxOp): class TestNpuDensityPriorBoxOp3(TestNpuDensityPriorBoxOp): - def set_density(self): super(TestNpuDensityPriorBoxOp3, self).set_density() self.fixed_ratios = [1.0, 4.0] class TestNpuDensityPriorBoxOpFP16(TestNpuDensityPriorBoxOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_dropout_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_dropout_op_npu.py index 3c55d7537b0..b9233f3fd68 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_dropout_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_dropout_op_npu.py @@ -28,7 +28,6 @@ EPOCH = 100 class TestDropoutOp(OpTest): - def setUp(self): self.op_type = "dropout" self.set_npu() @@ -38,11 +37,11 @@ class TestDropoutOp(OpTest): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } def init_dtype(self): @@ -70,11 +69,11 @@ class TestDropoutOpInput1d(TestDropoutOp): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((3, 62)).astype('uint8') + 'Mask': np.ones((3, 62)).astype('uint8'), } @@ -89,11 +88,11 @@ class TestDropoutOpInput1d_1(TestDropoutOp): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((2000)).astype('uint8') + 'Mask': np.ones((2000)).astype('uint8'), } @@ -108,11 +107,11 @@ class TestDropoutOp2(TestDropoutOp): 'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': np.zeros((32, 64)).astype('float32'), - 'Mask': np.zeros((32, 64)).astype('uint8') + 'Mask': np.zeros((32, 64)).astype('uint8'), } @@ -127,11 +126,11 @@ class TestDropoutOp3(TestDropoutOp): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64, 2)).astype('uint8') + 'Mask': np.ones((32, 64, 2)).astype('uint8'), } @@ -147,7 +146,7 @@ class TestDropoutOpInference(OpTest): 'dropout_prob': 0.35, 'fix_seed': True, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -164,7 +163,6 @@ class TestDropoutOpInference(OpTest): @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOpInference2(TestDropoutOpInference): - def setUp(self): self.op_type = "dropout" self.set_npu() @@ -173,7 +171,7 @@ class TestDropoutOpInference2(TestDropoutOpInference): self.attrs = { 'dropout_prob': 0.75, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -186,16 +184,16 @@ class TestDropoutOpWithSeed(TestDropoutOp): self.init_dtype() self.inputs = { "X": np.random.random((32, 64)).astype(self.dtype), - "Seed": np.asarray([125], dtype="int32") + "Seed": np.asarray([125], dtype="int32"), } self.attrs = { 'dropout_prob': 0.0, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } @@ -211,7 +209,6 @@ class TestDropoutOpFp16(TestDropoutOp): class TestDropoutAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace(), paddle.NPUPlace(0)] @@ -219,43 +216,44 @@ class TestDropoutAPI(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[40, 40], dtype="float32") - res1 = paddle.nn.functional.dropout(x=input, - p=0., - training=False, - mode='upscale_in_train') - res2 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='upscale_in_train') - res3 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='upscale_in_train') - res4 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='upscale_in_train') - res5 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='upscale_in_train') - res6 = paddle.nn.functional.dropout(x=input, - p=1., - training=True, - mode='upscale_in_train') + res1 = paddle.nn.functional.dropout( + x=input, p=0.0, training=False, mode='upscale_in_train' + ) + res2 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=True, mode='upscale_in_train' + ) + res3 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=False, mode='upscale_in_train' + ) + res4 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='upscale_in_train', + ) + res5 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='upscale_in_train', + ) + res6 = paddle.nn.functional.dropout( + x=input, p=1.0, training=True, mode='upscale_in_train' + ) res7 = paddle.fluid.layers.dropout( x=input, - dropout_prob=0., - dropout_implementation='upscale_in_train') - res8 = paddle.nn.functional.dropout(x=input, - p=0., - axis=(0, 1), - training=False, - mode='upscale_in_train') + dropout_prob=0.0, + dropout_implementation='upscale_in_train', + ) + res8 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=(0, 1), + training=False, + mode='upscale_in_train', + ) in_np = np.random.random([40, 40]).astype("float32") res_np = in_np @@ -264,13 +262,17 @@ class TestDropoutAPI(unittest.TestCase): exe = fluid.Executor(place) res_list = [res1, res2, res3, res4, res5, res7, res8] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np) - fetches2 = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res6]) + fetches2 = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res6], + ) np.testing.assert_allclose(fetches2[0], res_np2) def test_static(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_add_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_add_op_npu.py index b4da50a47ac..735b1d0ac06 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_add_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_add_op_npu.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestElementwiseAddOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_add" @@ -40,7 +39,7 @@ class TestElementwiseAddOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -128,21 +127,19 @@ class TestElementwiseAddOp(OpTest): class TestFP16ElementwiseAddOp(TestElementwiseAddOp): - def init_dtype(self): self.dtype = np.float16 class TestINT64ElementwiseAddOp(TestElementwiseAddOp): - def init_dtype(self): self.dtype = np.int64 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -150,9 +147,9 @@ class TestElementwiseAddOp_scalar(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -160,9 +157,9 @@ class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -170,9 +167,9 @@ class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -180,7 +177,6 @@ class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): class TestAddAPI(unittest.TestCase): - def test_name(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[2, 3], dtype="float32") @@ -205,66 +201,65 @@ class TestAddAPI(unittest.TestCase): place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - x_value, y_value, z_value = exe.run(feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[x, y, z]) + x_value, y_value, z_value = exe.run( + feed={"x": x_np, "y": y_np}, fetch_list=[x, y, z] + ) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual( (x_value == x_np).all(), True, - msg="x_value = {}, but expected {}".format(x_value, x_np)) + msg="x_value = {}, but expected {}".format(x_value, x_np), + ) self.assertEqual( (y_value == y_np).all(), True, - msg="y_value = {}, but expected {}".format(y_value, y_np)) + msg="y_value = {}, but expected {}".format(y_value, y_np), + ) self.assertEqual( (z_value == z_expected).all(), True, - msg="z_value = {}, but expected {}".format(z_value, z_expected)) + msg="z_value = {}, but expected {}".format(z_value, z_expected), + ) class TestAddError(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, paddle.add, x1, y1) # the input dtype must be float16 or float32 or float64 or int32 or int64 - x2 = paddle.static.data(name='x2', - shape=[3, 4, 5, 6], - dtype="uint8") - y2 = paddle.static.data(name='y2', - shape=[3, 4, 5, 6], - dtype="uint8") + x2 = paddle.static.data( + name='x2', shape=[3, 4, 5, 6], dtype="uint8" + ) + y2 = paddle.static.data( + name='y2', shape=[3, 4, 5, 6], dtype="uint8" + ) self.assertRaises(TypeError, paddle.add, x2, y2) class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestFP16ElementwiseAddOp_Vector(TestFP16ElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -275,7 +270,6 @@ class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -286,7 +280,6 @@ class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -297,7 +290,6 @@ class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -308,7 +300,6 @@ class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -316,7 +307,6 @@ class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -324,7 +314,6 @@ class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 1).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -335,7 +324,6 @@ class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -346,7 +334,6 @@ class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -357,7 +344,6 @@ class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -368,7 +354,6 @@ class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -376,7 +361,6 @@ class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -384,7 +368,6 @@ class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -392,7 +375,6 @@ class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) @@ -400,7 +382,6 @@ class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -408,7 +389,6 @@ class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -419,7 +399,6 @@ class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -430,9 +409,9 @@ class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -443,9 +422,9 @@ class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -456,7 +435,6 @@ class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -467,7 +445,6 @@ class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -478,7 +455,6 @@ class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -489,7 +465,6 @@ class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -500,7 +475,6 @@ class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -511,7 +485,6 @@ class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -522,7 +495,6 @@ class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 1, 12).astype(self.dtype) self.y = np.random.rand(10, 2, 12).astype(self.dtype) @@ -533,14 +505,15 @@ class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, y1) # the input dtype of elementwise_add must be float16 or float32 or float64 or int32 or int64 @@ -551,7 +524,6 @@ class TestElementwiseAddOpError(unittest.TestCase): class TestAddApi(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.add(x, y, name) @@ -569,7 +541,7 @@ class TestAddApi(unittest.TestCase): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -579,7 +551,7 @@ class TestAddApi(unittest.TestCase): place = fluid.NPUPlace(0) exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -590,18 +562,16 @@ class TestAddApi(unittest.TestCase): y = fluid.dygraph.to_variable(np_y) z = self._executed_api(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) class TestAddInplaceApi(TestAddApi): - def _executed_api(self, x, y, name=None): return x.add_(y, name) class TestAddInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float') self.y_numpy = np.random.rand(3, 4).astype('float') @@ -618,21 +588,18 @@ class TestAddInplaceBroadcastSuccess(unittest.TestCase): class TestAddInplaceBroadcastSuccess2(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') self.y_numpy = np.random.rand(3, 1).astype('float') class TestAddInplaceBroadcastSuccess3(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') class TestAddInplaceBroadcastError(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(3, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') @@ -651,14 +618,12 @@ class TestAddInplaceBroadcastError(unittest.TestCase): class TestAddInplaceBroadcastError2(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestAddInplaceBroadcastError3(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_div_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_div_op_npu.py index ab9c74fcf70..acdb8c75db5 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_div_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_div_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestElementwiseDiv(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_div" @@ -41,7 +40,7 @@ class TestElementwiseDiv(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.attrs = {} self.outputs = {'Out': out} @@ -73,13 +72,12 @@ class TestElementwiseDiv(OpTest): ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y")) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set("Y") + ) class TestElementwiseDivFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_div" @@ -93,7 +91,7 @@ class TestElementwiseDivFp16(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.attrs = {} self.outputs = {'Out': out} @@ -110,7 +108,6 @@ class TestElementwiseDivFp16(OpTest): class TestElementwiseDivNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -129,9 +126,9 @@ class TestElementwiseDivNet(unittest.TestCase): b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') c = paddle.static.data(name="c", shape=[32, 32], dtype='float32') d = paddle.static.data(name="d", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) e = paddle.multiply(a, b) f = paddle.multiply(c, d) @@ -157,18 +154,23 @@ class TestElementwiseDivNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -181,7 +183,6 @@ class TestElementwiseDivNet(unittest.TestCase): class TestFloatStatus(unittest.TestCase): - def test_overflow(self): paddle.disable_static() paddle.set_device('npu') @@ -191,9 +192,9 @@ class TestFloatStatus(unittest.TestCase): self.assertEqual(flag.numpy().sum(), 0.0) x = paddle.to_tensor([12.564], stop_gradient=False) - y = paddle.to_tensor([2.], stop_gradient=False) + y = paddle.to_tensor([2.0], stop_gradient=False) z = x / y - out = 32768. * z + out = 32768.0 * z ops.get_float_status(flag, flag) self.assertEqual(flag.numpy().sum(), 0.0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_floordiv_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_floordiv_op_npu.py index 7301f582fca..6762c51ca38 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_floordiv_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_floordiv_op_npu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestElementwiseFloorDiv(OpTest): - def setUp(self): self.op_type = "elementwise_floordiv" self.set_npu() @@ -33,7 +32,7 @@ class TestElementwiseFloorDiv(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {} self.outputs = {'Out': self.out} @@ -55,7 +54,6 @@ class TestElementwiseFloorDiv(OpTest): class TestElementwiseFloorDiv2(TestElementwiseFloorDiv): - def init_dtype(self): self.dtype = "int32" diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_max_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_max_op_npu.py index 92a91b384fc..fe3d5847929 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_max_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_max_op_npu.py @@ -41,7 +41,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_x)) or ( - shape_out[ax] > 1 and shape_x[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_x[ax - src_axis] == 1 + ): reduce_axes_x.append(ax) if shape_y != shape_out: @@ -52,7 +53,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_y)) or ( - shape_out[ax] > 1 and shape_y[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_y[ax - src_axis] == 1 + ): reduce_axes_y.append(ax) if len(reduce_axes_x) > 0: @@ -79,7 +81,6 @@ def ComputeGrad(x, y, out, axis): class TestElementwiseMaxOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_max" @@ -91,7 +92,7 @@ class TestElementwiseMaxOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -106,7 +107,8 @@ class TestElementwiseMaxOp(OpTest): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) sgn = np.random.choice([-1, 1], [13, 17]).astype(self.dtype) self.y = self.x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype( - self.dtype) + self.dtype + ) self.out = np.maximum(self.x, self.y) def init_axis(self): @@ -119,18 +121,17 @@ class TestElementwiseMaxOp(OpTest): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y")) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set("Y") + ) class TestElementwiseMaxOp_int32(TestElementwiseMaxOp): - def init_dtype(self): self.dtype = np.int32 @@ -146,7 +147,6 @@ class TestElementwiseMaxOp_int32(TestElementwiseMaxOp): class TestElementwiseMaxOp_scalar(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.random_integers(-5, 5, [2, 3, 20]).astype(self.dtype) self.y = np.array([0.5]).astype(self.dtype) @@ -154,22 +154,22 @@ class TestElementwiseMaxOp_scalar(TestElementwiseMaxOp): class TestElementwiseMaxOp_vector(TestElementwiseMaxOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x + sgn * np.random.uniform(0.1, 1, - (100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(0.1, 1, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxOp_broadcast_0(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (100, 5, 2)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(100, 1, 1)) def init_axis(self): @@ -177,12 +177,12 @@ class TestElementwiseMaxOp_broadcast_0(TestElementwiseMaxOp): class TestElementwiseMaxOp_broadcast_1(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(1, 100, 1)) def init_axis(self): @@ -190,56 +190,68 @@ class TestElementwiseMaxOp_broadcast_1(TestElementwiseMaxOp): def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwiseMaxOp_broadcast_2(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(1, 1, 100)) def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwiseMaxOp_broadcast_3(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 50, 2, 1)).astype(self.dtype) sgn = np.random.choice([-1, 1], (50, 2)).astype(self.dtype) - self.y = self.x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (50, 2)).astype(self.dtype) + self.y = self.x[0, :, :, 0] + sgn * np.random.uniform( + 1, 2, (50, 2) + ).astype(self.dtype) self.out = np.maximum(self.x, self.y.reshape(1, 50, 2, 1)) def init_axis(self): @@ -247,27 +259,26 @@ class TestElementwiseMaxOp_broadcast_3(TestElementwiseMaxOp): class TestElementwiseMaxOp_broadcast_4(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 5)).astype(self.dtype) - self.y = self.x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 5)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(1, 2, (2, 3, 1, 5)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxOp_broadcast_5(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 1)).astype(self.dtype) - self.y = self.x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 1)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(1, 2, (2, 3, 1, 1)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -282,9 +293,9 @@ class TestElementwiseMaxNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.maximum(a, b) @@ -307,16 +318,17 @@ class TestElementwiseMaxNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_min_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_min_op_npu.py index 5956faba0e1..8cd51765bd8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_min_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_min_op_npu.py @@ -28,7 +28,6 @@ SEED = 2021 class TestElementwiseMinOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_min" @@ -37,7 +36,7 @@ class TestElementwiseMinOp(OpTest): self.init_input_output() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis} @@ -52,7 +51,8 @@ class TestElementwiseMinOp(OpTest): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) self.sgn = np.random.choice([-1, 1], [13, 17]).astype(self.dtype) self.y = self.x + self.sgn * np.random.uniform(0.1, 1, [13, 17]).astype( - self.dtype) + self.dtype + ) self.out = np.minimum(self.x, self.y) self.axis = -1 @@ -64,9 +64,9 @@ class TestElementwiseMinOp(OpTest): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place( self.place, @@ -76,10 +76,13 @@ class TestElementwiseMinOp(OpTest): def test_check_grad_ingore_x(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + max_relative_error=0.9, + ) else: self.check_grad_with_place( self.place, @@ -90,10 +93,13 @@ class TestElementwiseMinOp(OpTest): def test_check_grad_ingore_y(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - max_relative_error=0.1) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + max_relative_error=0.1, + ) else: self.check_grad_with_place( self.place, @@ -104,32 +110,30 @@ class TestElementwiseMinOp(OpTest): class TestElementwiseMinOpFp16(TestElementwiseMinOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOp_Vector(TestElementwiseMinOp): - def init_input_output(self): - self.x = np.random.uniform(1, 2, (100, )).astype(self.dtype) - self.sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x + self.sgn * np.random.uniform(0.1, 1, (100, )).astype( - self.dtype) + self.x = np.random.uniform(1, 2, (100,)).astype(self.dtype) + self.sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x + self.sgn * np.random.uniform(0.1, 1, (100,)).astype( + self.dtype + ) self.out = np.minimum(self.x, self.y) self.axis = -1 class TestElementwiseMinOpFp16_Vector(TestElementwiseMinOp_Vector): - def init_dtype(self): self.dtype = np.float16 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOp_scalar(TestElementwiseMinOp): - def init_input_output(self): self.x = np.random.random_integers(-5, 5, [10, 3, 4]).astype(self.dtype) self.y = np.array([0.5]).astype(self.dtype) @@ -138,32 +142,30 @@ class TestElementwiseMinOp_scalar(TestElementwiseMinOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOpFp16_scalar(TestElementwiseMinOp_scalar): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOp_broadcast(TestElementwiseMinOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - self.sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, 0, :] + self.sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + self.sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, 0, :] + self.sgn * np.random.uniform( + 1, 2, (100,) + ).astype(self.dtype) self.out = np.minimum(self.x, self.y.reshape(1, 1, 100)) self.axis = -1 class TestElementwiseMinOpFp16_broadcast(TestElementwiseMinOp_broadcast): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOpNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -178,9 +180,9 @@ class TestElementwiseMinOpNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.minimum(a, b) @@ -203,16 +205,17 @@ class TestElementwiseMinOpNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_mod_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_mod_op_npu.py index fb5c773c825..efb81fbad6e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_mod_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_mod_op_npu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestElementwiseModOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -42,7 +41,7 @@ class TestElementwiseModOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -69,13 +68,11 @@ class TestElementwiseModOp(OpTest): class TestElementwiseModOpInt64(TestElementwiseModOp): - def init_dtype(self): self.dtype = np.int64 class TestElementwiseModOp_scalar(TestElementwiseModOp): - def init_input_output(self): scale_x = random.randint(0, 100000000) scale_y = random.randint(1, 100000000) @@ -85,7 +82,6 @@ class TestElementwiseModOp_scalar(TestElementwiseModOp): class TestElementwiseModOpFloat(TestElementwiseModOp): - def init_dtype(self): self.dtype = np.float32 @@ -99,7 +95,6 @@ class TestElementwiseModOpFloat(TestElementwiseModOp): class TestElementwiseModOpDouble(TestElementwiseModOpFloat): - def init_dtype(self): self.dtype = np.float64 @@ -108,7 +103,6 @@ class TestElementwiseModOpDouble(TestElementwiseModOpFloat): class TestElementwiseModOpFP16(TestElementwiseModOpFloat): - def init_dtype(self): self.dtype = np.float16 @@ -117,7 +111,6 @@ class TestElementwiseModOpFP16(TestElementwiseModOpFloat): class TestElementwiseModOp_broadcast_0(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -128,7 +121,6 @@ class TestElementwiseModOp_broadcast_0(TestElementwiseModOp): class TestElementwiseModOp_broadcast_1(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -139,7 +131,6 @@ class TestElementwiseModOp_broadcast_1(TestElementwiseModOp): class TestElementwiseModOp_broadcast_2(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -150,7 +141,6 @@ class TestElementwiseModOp_broadcast_2(TestElementwiseModOp): class TestRemainderOp(unittest.TestCase): - def test_name(self): paddle.set_device('npu:0') with fluid.program_guard(fluid.Program()): @@ -172,7 +162,7 @@ class TestRemainderOp(unittest.TestCase): self.assertEqual((np_z == z_expected).all(), True) np_x = np.array([-3.3, 11.5, -2, 3.5]) - np_y = np.array([-1.2, 2., 3.3, -2.3]) + np_y = np.array([-1.2, 2.0, 3.3, -2.3]) x = paddle.to_tensor(np_x) y = paddle.to_tensor(np_y) z = x % y diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_mul_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_mul_op_npu.py index a3fe2dcf453..fa0b80d6338 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_mul_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_mul_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class ElementwiseMulOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -41,7 +40,7 @@ class ElementwiseMulOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis} @@ -53,14 +52,14 @@ class ElementwiseMulOp(OpTest): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('Y')) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set('Y') + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -75,33 +74,31 @@ class ElementwiseMulOp(OpTest): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float32), - 'Y': np.random.rand(1).astype(np.float32) + 'Y': np.random.rand(1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.random((100, )).astype("float32"), - 'Y': np.random.random((100, )).astype("float32") + 'X': np.random.random((100,)).astype("float32"), + 'Y': np.random.random((100,)).astype("float32"), } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -112,13 +109,12 @@ class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.attrs = {'axis': 1} @@ -128,13 +124,12 @@ class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.outputs = { @@ -143,13 +138,12 @@ class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float32), - 'Y': np.random.rand(10, 12).astype(np.float32) + 'Y': np.random.rand(10, 12).astype(np.float32), } self.attrs = {'axis': 1} @@ -159,69 +153,64 @@ class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 2, 11).astype(np.float32), - 'Y': np.random.rand(10, 1, 11).astype(np.float32) + 'Y': np.random.rand(10, 1, 11).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 4, 2, 3).astype(np.float32), - 'Y': np.random.rand(10, 4, 1, 3).astype(np.float32) + 'Y': np.random.rand(10, 4, 1, 3).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "paddle is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "paddle is not compiled with NPU" +) class TestElementwiseMulOpFp16(ElementwiseMulOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(1, 1, 100).astype(np.float32) + 'Y': np.random.rand(1, 1, 100).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(30, 3, 1, 5).astype(np.float32), - 'Y': np.random.rand(30, 1, 4, 1).astype(np.float32) + 'Y': np.random.rand(30, 1, 4, 1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 10).astype(np.float32), - 'Y': np.random.rand(2, 2, 10, 10).astype(np.float32) + 'Y': np.random.rand(2, 2, 10, 10).astype(np.float32), } self.attrs = {'axis': 2} diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_pow_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_pow_op_npu.py index 71528994238..b872c5bf83e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_pow_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_pow_op_npu.py @@ -42,7 +42,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_x)) or ( - shape_out[ax] > 1 and shape_x[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_x[ax - src_axis] == 1 + ): reduce_axes_x.append(ax) if shape_y != shape_out: @@ -53,7 +54,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_y)) or ( - shape_out[ax] > 1 and shape_y[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_y[ax - src_axis] == 1 + ): reduce_axes_y.append(ax) if len(reduce_axes_x) > 0: @@ -79,7 +81,6 @@ def ComputeGrad(x, y, out, axis): class TestElementwisePow(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_pow" @@ -91,7 +92,7 @@ class TestElementwisePow(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -116,27 +117,32 @@ class TestElementwisePow(OpTest): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowFp16(TestElementwisePow): - def init_input_output(self): np.random.seed(SEED) self.x = np.random.uniform(1, 2, [11, 17]).astype(self.dtype) @@ -155,7 +161,6 @@ class TestElementwisePowFp16(TestElementwisePow): class TestElementwisePowDouble(TestElementwisePow): - def init_input_output(self): np.random.seed(SEED) self.x = np.random.uniform(1, 2, [11, 17]).astype(self.dtype) @@ -174,7 +179,6 @@ class TestElementwisePowDouble(TestElementwisePow): class TestElementwisePowOp_broadcast_0(TestElementwisePow): - def init_axis(self): self.axis = 1 @@ -186,27 +190,32 @@ class TestElementwisePowOp_broadcast_0(TestElementwisePow): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowOp_broadcast_1(TestElementwisePow): - def init_axis(self): self.axis = 1 @@ -218,27 +227,32 @@ class TestElementwisePowOp_broadcast_1(TestElementwisePow): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowOp_broadcast_2(TestElementwisePow): - def init_axis(self): self.axis = 0 @@ -250,27 +264,32 @@ class TestElementwisePowOp_broadcast_2(TestElementwisePow): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -285,9 +304,9 @@ class TestElementwisePowNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.pow(a, b) @@ -310,16 +329,17 @@ class TestElementwisePowNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_sub_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_sub_op_npu.py index ac3658f0c34..8542ed6bdc3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_sub_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_sub_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestElementwiseSubOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_sub" @@ -39,7 +38,7 @@ class TestElementwiseSubOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -91,19 +90,16 @@ class TestElementwiseSubOp(OpTest): class TestElementwiseSubOpInt32(TestElementwiseSubOp): - def init_dtype(self): self.dtype = np.int32 class TestElementwiseSubOpInt64(TestElementwiseSubOp): - def init_dtype(self): self.dtype = np.int64 class TestSubtractAPI(unittest.TestCase): - def test_name(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[2, 3], dtype="float32") @@ -128,50 +124,51 @@ class TestSubtractAPI(unittest.TestCase): place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - x_value, y_value, z_value = exe.run(feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[x, y, z]) + x_value, y_value, z_value = exe.run( + feed={"x": x_np, "y": y_np}, fetch_list=[x, y, z] + ) - z_expected = np.array([1., -2., 2.]) + z_expected = np.array([1.0, -2.0, 2.0]) self.assertEqual( (x_value == x_np).all(), True, - msg="x_value = {}, but expected {}".format(x_value, x_np)) + msg="x_value = {}, but expected {}".format(x_value, x_np), + ) self.assertEqual( (y_value == y_np).all(), True, - msg="y_value = {}, but expected {}".format(y_value, y_np)) + msg="y_value = {}, but expected {}".format(y_value, y_np), + ) self.assertEqual( (z_value == z_expected).all(), True, - msg="z_value = {}, but expected {}".format(z_value, z_expected)) + msg="z_value = {}, but expected {}".format(z_value, z_expected), + ) class TestSubtractError(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, paddle.subtract, x1, y1) # the input dtype must be float16 or float32 or float64 or int32 or int64 - x2 = paddle.static.data(name='x2', - shape=[3, 4, 5, 6], - dtype="uint8") - y2 = paddle.static.data(name='y2', - shape=[3, 4, 5, 6], - dtype="uint8") + x2 = paddle.static.data( + name='x2', shape=[3, 4, 5, 6], dtype="uint8" + ) + y2 = paddle.static.data( + name='y2', shape=[3, 4, 5, 6], dtype="uint8" + ) self.assertRaises(TypeError, paddle.subtract, x2, y2) class TestSubtractNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -186,9 +183,9 @@ class TestSubtractNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) c = paddle.assign(b) @@ -212,16 +209,17 @@ class TestSubtractNet(unittest.TestCase): for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_exp_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_exp_op_npu.py index 44ba219d40f..ca43877d453 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_exp_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_exp_op_npu.py @@ -30,7 +30,6 @@ SEED = 2049 class TestExpNPUOP(OpTest): - def setUp(self): self.set_npu() @@ -63,7 +62,6 @@ class TestExpNPUOP(OpTest): class TestExpNPUOPFloat64(TestExpNPUOP): - def init_dtype(self): self.dtype = np.float64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_expand_as_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_expand_as_v2_op_npu.py index 76ab1d2959a..ca2b0195b42 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_expand_as_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_expand_as_v2_op_npu.py @@ -26,7 +26,6 @@ np.random.seed(10) class TestExpandAsOpRank1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -50,7 +49,6 @@ class TestExpandAsOpRank1(OpTest): class TestExpandAsOpRank2(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -74,7 +72,6 @@ class TestExpandAsOpRank2(OpTest): class TestExpandAsOpRank3(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -98,7 +95,6 @@ class TestExpandAsOpRank3(OpTest): class TestExpandAsOpRank4(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -123,29 +119,28 @@ class TestExpandAsOpRank4(OpTest): # Test python API class TestExpandAsV2API(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([2, 12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[2, 12, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[2, 12, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = paddle.expand_as(x, y=y) exe = fluid.Executor(place=fluid.NPUPlace(0)) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (2, 1, 1))) diff --git a/python/paddle/fluid/tests/unittests/npu/test_expand_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_expand_op_npu.py index 3096369b196..0ac8cfc5d3c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_expand_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_expand_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestExpand(OpTest): - def setUp(self): self.set_npu() self.op_type = "expand" @@ -55,7 +54,6 @@ class TestExpand(OpTest): class TestExpandV2(TestExpand): - def setUp(self): self.set_npu() self.op_type = "expand" @@ -69,7 +67,7 @@ class TestExpandV2(TestExpand): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'ExpandTimes': OpTest.np_dtype_to_fluid_dtype(expand_times) + 'ExpandTimes': OpTest.np_dtype_to_fluid_dtype(expand_times), } self.attrs = {} self.outputs = {'Out': out} @@ -83,7 +81,6 @@ class TestExpandFp16(TestExpand): class TestExpandNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -96,9 +93,9 @@ class TestExpandNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 1], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) res = paddle.fluid.layers.expand(a, [1, 32]) loss = res.sum() @@ -115,12 +112,11 @@ class TestExpandNet(unittest.TestCase): for epoch in range(100): - loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "label": label_np - }, - fetch_list=[loss]) + loss_res = exe.run( + main_prog, + feed={"a": a_np, "label": label_np}, + fetch_list=[loss], + ) if epoch % 10 == 0: print("Epoch {} | Loss: {}".format(epoch, loss)) @@ -139,7 +135,6 @@ class TestExpandNet(unittest.TestCase): class TestExpand_expand_times_all_one(TestExpand): - def setUp(self): self.set_npu() self.op_type = "expand" diff --git a/python/paddle/fluid/tests/unittests/npu/test_expand_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_expand_v2_op_npu.py index d922cef46c8..0ea52f04d9d 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_expand_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_expand_v2_op_npu.py @@ -29,7 +29,6 @@ np.random.seed(10) # CANN Op Support X: float16, float32, int32, int8 ,uint8 # Situation 1: shape is a list(without tensor) class TestExpandV2NPUOpRank1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -58,7 +57,6 @@ class TestExpandV2NPUOpRank1(OpTest): class TestExpandV2OpRank2_DimExpanding(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] @@ -66,7 +64,6 @@ class TestExpandV2OpRank2_DimExpanding(TestExpandV2NPUOpRank1): class TestExpandV2OpRank2(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = [1, 140] self.shape = [12, 140] @@ -74,7 +71,6 @@ class TestExpandV2OpRank2(TestExpandV2NPUOpRank1): class TestExpandV2OpRank3_Corner(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) @@ -82,7 +78,6 @@ class TestExpandV2OpRank3_Corner(TestExpandV2NPUOpRank1): class TestExpandV2OpRank4(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) @@ -90,7 +85,6 @@ class TestExpandV2OpRank4(TestExpandV2NPUOpRank1): class TestExpandV2OpRank5(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = (2, 4, 1, 15) self.shape = (2, -1, 4, -1) @@ -98,7 +92,6 @@ class TestExpandV2OpRank5(TestExpandV2NPUOpRank1): class TestExpandV2OpRank6(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = (4, 1, 30) self.shape = (2, -1, 4, 30) @@ -107,7 +100,6 @@ class TestExpandV2OpRank6(TestExpandV2NPUOpRank1): # Situation 2: shape is a list(with tensor) class TestExpandV2OpNPURank1_tensor_attr(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -116,8 +108,9 @@ class TestExpandV2OpNPURank1_tensor_attr(OpTest): self.dtype = np.float32 expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -143,9 +136,9 @@ class TestExpandV2OpNPURank1_tensor_attr(OpTest): self.check_grad_with_place(self.place, ['X'], 'Out') -class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpNPURank1_tensor_attr - ): - +class TestExpandV2OpRank2_Corner_tensor_attr( + TestExpandV2OpNPURank1_tensor_attr +): def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -155,7 +148,6 @@ class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpNPURank1_tensor_attr # Situation 3: shape is a tensor class TestExpandV2NPUOpRank1_tensor(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -189,7 +181,6 @@ class TestExpandV2NPUOpRank1_tensor(OpTest): # Situation 4: input x is float16 # skip grad check for float16 class TestExpandV2OpFloat(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -212,7 +203,6 @@ class TestExpandV2OpFloat(OpTest): # Situation 5: input x is int32 # skip grad check for int32 class TestExpandV2OpInteger(OpTest): - def init_dtype(self): self.dtype = 'int32' @@ -236,13 +226,11 @@ class TestExpandV2OpInteger(OpTest): class TesstExpandV2OpInt64(TestExpandV2OpInteger): - def init_dtype(self): self.dtype = 'int64' class TesstExpandV2OpBool(TestExpandV2OpInteger): - def init_dtype(self): self.dtype = 'bool' @@ -257,11 +245,11 @@ class TesstExpandV2OpBool(TestExpandV2OpInteger): class TestExpandV2Error(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.NPUPlace(0) + ) shape = [2, 2] self.assertRaises(TypeError, paddle.tensor.expand, x1, shape) x2 = fluid.layers.data(name='x2', shape=[2], dtype="uint8") @@ -273,20 +261,23 @@ class TestExpandV2Error(unittest.TestCase): # Test python API class TestExpandV2API(unittest.TestCase): - def test_static(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', + shape=[12, 14], + append_batch_size=False, + dtype="float32", + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.expand(x, shape=[12, 14]) out_2 = paddle.expand(x, shape=[positive_2, 14]) @@ -295,14 +286,14 @@ class TestExpandV2API(unittest.TestCase): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=paddle.NPUPlace(0)) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) diff --git a/python/paddle/fluid/tests/unittests/npu/test_eye_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_eye_op_npu.py index fe0ba17d201..e80be96f2cd 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_eye_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_eye_op_npu.py @@ -28,7 +28,6 @@ np.random.seed(10) class TestEyeOp(OpTest): - def setUp(self): ''' Test eye op with specified shape @@ -47,14 +46,14 @@ class TestEyeOp(OpTest): if self.num_columns == 0: self.attrs = { 'num_rows': self.num_rows, - 'dtype': framework.convert_np_dtype_to_dtype_(self.dtype) + 'dtype': framework.convert_np_dtype_to_dtype_(self.dtype), } self.outputs = {'Out': np.eye(self.num_rows, dtype=self.dtype)} else: self.attrs = { 'num_rows': self.num_rows, 'num_columns': self.num_columns, - 'dtype': framework.convert_np_dtype_to_dtype_(self.dtype) + 'dtype': framework.convert_np_dtype_to_dtype_(self.dtype), } self.outputs = { 'Out': np.eye(self.num_rows, self.num_columns, dtype=self.dtype) @@ -73,41 +72,35 @@ class TestEyeOp(OpTest): class TestEyeOp1(TestEyeOp): - def initTestCase(self): self.num_rows = 50 class TestEyeOp2(TestEyeOp): - def initTestCase(self): self.num_rows = 50 self.dtype = np.int32 class TestEyeOp3(TestEyeOp): - def initTestCase(self): self.num_rows = 50 self.dtype = np.float16 class TestEyeOp4(TestEyeOp): - def initTestCase(self): self.num_rows = 1 self.num_columns = 99 class TestEyeOp5(TestEyeOp): - def initTestCase(self): self.num_rows = 100 self.num_columns = 100 class TestEyeOp6(TestEyeOp): - def initTestCase(self): self.num_rows = 100 self.num_columns = 100 @@ -115,13 +108,12 @@ class TestEyeOp6(TestEyeOp): class API_TestTensorEye(unittest.TestCase): - def test_out(self): with paddle.static.program_guard(paddle.static.Program()): data = paddle.eye(10) place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, dtype="float32") self.assertEqual((result == expected_result).all(), True) @@ -129,7 +121,7 @@ class API_TestTensorEye(unittest.TestCase): data = paddle.eye(10, num_columns=7, dtype="float16") place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, 7, dtype="float16") self.assertEqual((result == expected_result).all(), True) @@ -137,7 +129,7 @@ class API_TestTensorEye(unittest.TestCase): data = paddle.eye(10, dtype="int32") place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, dtype="int32") self.assertEqual((result == expected_result).all(), True) @@ -159,8 +151,9 @@ class API_TestTensorEye(unittest.TestCase): result = tmp_result expected_result = np.stack(result, axis=0) paddle.enable_static() - self.assertEqual(out.numpy().shape == np.array(expected_result).shape, - True) + self.assertEqual( + out.numpy().shape == np.array(expected_result).shape, True + ) self.assertEqual((out.numpy() == expected_result).all(), True) paddle.disable_static(paddle.NPUPlace(0)) @@ -175,8 +168,9 @@ class API_TestTensorEye(unittest.TestCase): result = tmp_result expected_result = np.stack(result, axis=0) paddle.enable_static() - self.assertEqual(out.numpy().shape == np.array(expected_result).shape, - True) + self.assertEqual( + out.numpy().shape == np.array(expected_result).shape, True + ) self.assertEqual((out.numpy() == expected_result).all(), True) def test_errors(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_fill_any_like_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_fill_any_like_op_npu.py index 23f07d299e2..09e38fe0665 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_fill_any_like_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_fill_any_like_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestFillAnyLikeNPUOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -52,47 +51,40 @@ class TestFillAnyLikeNPUOp(OpTest): class TestFillAnyLikeNPUOpInt32(TestFillAnyLikeNPUOp): - def init(self): self.dtype = np.int32 self.value = -1 class TestFillAnyLikeNPUOpInt64(TestFillAnyLikeNPUOp): - def init(self): self.dtype = np.int64 self.value = -1 class TestFillAnyLikeNPUOpFloat32(TestFillAnyLikeNPUOp): - def init(self): self.dtype = np.float32 self.value = 0.09 class TestFillAnyLikeNPUOpFloat16(TestFillAnyLikeNPUOp): - def init(self): self.dtype = np.float16 self.value = 0.05 class TestFillAnyLikeNPUOpValue1(TestFillAnyLikeNPUOp): - def init(self): self.value = 1.0 class TestFillAnyLikeNPUOpValue2(TestFillAnyLikeNPUOp): - def init(self): self.value = 1e-9 class TestFillAnyLikeNPUOpShape(TestFillAnyLikeNPUOp): - def init(self): self.shape = [12, 10] diff --git a/python/paddle/fluid/tests/unittests/npu/test_fill_constant_batch_size_like_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_fill_constant_batch_size_like_op_npu.py index f157845d24b..0ceaa0faf34 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_fill_constant_batch_size_like_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_fill_constant_batch_size_like_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestFillConstantBatchSizeLike(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -48,11 +47,12 @@ class TestFillConstantBatchSizeLike(OpTest): 'dtype': self.dtype, 'force_cpu': self.force_cpu, 'input_dim_idx': self.input_dim_idx, - 'output_dim_idx': self.output_dim_idx + 'output_dim_idx': self.output_dim_idx, } self.outputs = { - 'Out': np.full(self.output_shape, self.output_value, - self.output_dtype) + 'Out': np.full( + self.output_shape, self.output_value, self.output_dtype + ) } def set_npu(self): @@ -84,7 +84,6 @@ class TestFillConstantBatchSizeLike(OpTest): class TestFillConstantBatchSizeLike2(TestFillConstantBatchSizeLike): - def init_shape(self): # test shape self.input_shape = [4, 5, 6, 7] @@ -93,7 +92,6 @@ class TestFillConstantBatchSizeLike2(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike3(TestFillConstantBatchSizeLike): - def init_value(self): # use 'str_value' rather than 'value' self.value = 3.8 @@ -102,7 +100,6 @@ class TestFillConstantBatchSizeLike3(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike4(TestFillConstantBatchSizeLike): - def init_value(self): # str_value = 'inf' self.value = 3.8 @@ -111,7 +108,6 @@ class TestFillConstantBatchSizeLike4(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike5(TestFillConstantBatchSizeLike): - def init_value(self): # str_value = '-inf' self.value = 3.8 @@ -120,7 +116,6 @@ class TestFillConstantBatchSizeLike5(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike6(TestFillConstantBatchSizeLike): - def init_dtype(self): self.dtype = core.VarDesc.VarType.FP16 self.output_dtype = np.float16 @@ -130,20 +125,17 @@ class TestFillConstantBatchSizeLike6(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLike7(TestFillConstantBatchSizeLike): - def init_dtype(self): self.dtype = core.VarDesc.VarType.INT32 self.output_dtype = np.int32 class TestFillConstantBatchSizeLike8(TestFillConstantBatchSizeLike): - def init_force_cpu(self): self.force_cpu = True class TestFillConstantBatchSizeLike9(TestFillConstantBatchSizeLike): - def init_shape(self): self.input_shape = [4, 5] self.shape = [123, 92] @@ -177,11 +169,12 @@ class TestFillConstantBatchSizeLikeLodTensor(TestFillConstantBatchSizeLike): 'dtype': self.dtype, 'force_cpu': self.force_cpu, 'input_dim_idx': self.input_dim_idx, - 'output_dim_idx': self.output_dim_idx + 'output_dim_idx': self.output_dim_idx, } self.outputs = { - 'Out': np.full(self.output_shape, self.output_value, - self.output_dtype) + 'Out': np.full( + self.output_shape, self.output_value, self.output_dtype + ) } def init_shape(self): @@ -191,7 +184,8 @@ class TestFillConstantBatchSizeLikeLodTensor(TestFillConstantBatchSizeLike): class TestFillConstantBatchSizeLikeLodTensor2( - TestFillConstantBatchSizeLikeLodTensor): + TestFillConstantBatchSizeLikeLodTensor +): # test LodTensor with 'input_dim_idx' != 0 def init_shape(self): self.input_shape = [10, 20] diff --git a/python/paddle/fluid/tests/unittests/npu/test_fill_constant_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_fill_constant_op_npu.py index 3d0f770b4eb..d3087514c63 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_fill_constant_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_fill_constant_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestFillConstant(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -49,7 +48,6 @@ class TestFillConstant(OpTest): class TestFillConstantInt(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -59,7 +57,7 @@ class TestFillConstantInt(OpTest): self.attrs = { 'shape': [123, 92], 'value': 1, - 'dtype': core.VarDesc.VarType.INT32 + 'dtype': core.VarDesc.VarType.INT32, } self.outputs = {'Out': np.full((123, 92), 1).astype(self.dtype)} @@ -74,7 +72,6 @@ class TestFillConstantInt(OpTest): class TestFillConstantInt64(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -84,7 +81,7 @@ class TestFillConstantInt64(OpTest): self.attrs = { 'shape': [123, 92], 'value': 1, - 'dtype': core.VarDesc.VarType.INT64 + 'dtype': core.VarDesc.VarType.INT64, } self.outputs = {'Out': np.full((123, 92), 1).astype(self.dtype)} @@ -99,7 +96,6 @@ class TestFillConstantInt64(OpTest): class TestFillConstantFP16(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -109,7 +105,7 @@ class TestFillConstantFP16(OpTest): self.attrs = { 'shape': [123, 92], 'value': 1.0, - 'dtype': core.VarDesc.VarType.FP16 + 'dtype': core.VarDesc.VarType.FP16, } self.outputs = {'Out': np.full((123, 92), 1.0).astype(self.dtype)} @@ -124,7 +120,6 @@ class TestFillConstantFP16(OpTest): class TestFillConstantBool(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -134,7 +129,7 @@ class TestFillConstantBool(OpTest): self.attrs = { 'shape': [123, 92], 'value': True, - 'dtype': core.VarDesc.VarType.BOOL + 'dtype': core.VarDesc.VarType.BOOL, } self.outputs = {'Out': np.full((123, 92), True).astype(self.dtype)} @@ -149,7 +144,6 @@ class TestFillConstantBool(OpTest): class TestFillConstantWithPlaceType(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_fill_zeros_like_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_fill_zeros_like_op_npu.py index 3095c181792..59b0c43c157 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_fill_zeros_like_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_fill_zeros_like_op_npu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestFillZerosLikeOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -44,31 +43,26 @@ class TestFillZerosLikeOp(OpTest): class TestFillZerosLikeOpBool(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.bool_ class TestFillZerosLikeOpFp16(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.float16 class TestFillZerosLikeOpFp64(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.float64 class TestFillZerosLikeOpInt32(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.int32 class TestFillZerosLikeOpInt64(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_flags_check_nan_inf_npu.py b/python/paddle/fluid/tests/unittests/npu/test_flags_check_nan_inf_npu.py index 69c586fb2d8..88949de237c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_flags_check_nan_inf_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_flags_check_nan_inf_npu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestCheckFiniteAndUnscale(unittest.TestCase): - def setUp(self): fluid.set_flags({'FLAGS_check_nan_inf': True}) diff --git a/python/paddle/fluid/tests/unittests/npu/test_flatten2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_flatten2_op_npu.py index f384620f2a2..caf370ed098 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_flatten2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_flatten2_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestFlatten2Op(OpTest): - def setUp(self): self.set_npu() self.op_type = "flatten2" @@ -35,7 +34,7 @@ class TestFlatten2Op(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def set_npu(self): @@ -57,7 +56,6 @@ class TestFlatten2Op(OpTest): class TestFlatten2OpWithCornerAxis(TestFlatten2Op): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.axis = 0 @@ -65,7 +63,6 @@ class TestFlatten2OpWithCornerAxis(TestFlatten2Op): class TestFlatten2OpWithDefaultAxis(TestFlatten2Op): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -75,7 +72,6 @@ class TestFlatten2OpWithDefaultAxis(TestFlatten2Op): class TestFlatten2OpSixDims(TestFlatten2Op): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/npu/test_flatten_contiguous_range_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_flatten_contiguous_range_op_npu.py index 7c1b6f1776a..1ee72cbac8c 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_flatten_contiguous_range_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_flatten_contiguous_range_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestFlattenOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "flatten_contiguous_range" @@ -39,7 +38,7 @@ class TestFlattenOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def set_npu(self): @@ -55,17 +54,16 @@ class TestFlattenOp(OpTest): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 self.stop_axis = -1 - self.new_shape = (120) + self.new_shape = 120 def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 1 @@ -75,12 +73,11 @@ class TestFlattenOp_1(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_2(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -90,12 +87,11 @@ class TestFlattenOp_2(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_3(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -105,12 +101,11 @@ class TestFlattenOp_3(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_4(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = -2 @@ -120,12 +115,11 @@ class TestFlattenOp_4(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_5(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 2 @@ -135,12 +129,11 @@ class TestFlattenOp_5(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.start_axis = 3 @@ -150,12 +143,11 @@ class TestFlattenOpSixDims(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_Float32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -166,12 +158,11 @@ class TestFlattenOp_Float32(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_int32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -182,7 +173,7 @@ class TestFlattenOp_int32(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -190,7 +181,6 @@ class TestFlattenOp_int32(TestFlattenOp): class TestFlattenOp_uint8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -201,7 +191,7 @@ class TestFlattenOp_uint8(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -209,7 +199,6 @@ class TestFlattenOp_uint8(TestFlattenOp): class TestFlattenOp_int8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -220,7 +209,7 @@ class TestFlattenOp_int8(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -228,7 +217,6 @@ class TestFlattenOp_int8(TestFlattenOp): class TestFlattenOp_int64(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -239,7 +227,7 @@ class TestFlattenOp_int64(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -247,45 +235,58 @@ class TestFlattenOp_int64(TestFlattenOp): class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_ValueError1(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) out = paddle.flatten(x_var, start_axis=2, stop_axis=1) self.assertRaises(ValueError, test_ValueError1) def test_ValueError2(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=10, stop_axis=1) self.assertRaises(ValueError, test_ValueError2) def test_ValueError3(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=2, stop_axis=10) self.assertRaises(ValueError, test_ValueError3) def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x2 = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x2 = x2.astype('float16') - x2_var = paddle.fluid.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2_var = paddle.fluid.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) paddle.flatten(x2_var) self.assertRaises(TypeError, test_type) @@ -297,7 +298,6 @@ class TestFlatten2OpError(unittest.TestCase): class TestStaticFlattenPythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -307,9 +307,9 @@ class TestStaticFlattenPythonAPI(unittest.TestCase): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[2, 3, 4, 4], - dtype='float32') + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32' + ) out = self.execute_api(x, start_axis=-2, stop_axis=-1) exe = paddle.static.Executor(place=paddle.NPUPlace(0)) @@ -318,17 +318,22 @@ class TestStaticFlattenPythonAPI(unittest.TestCase): class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): - def execute_api(self, x, start_axis=0, stop_axis=-1): return x.flatten_(start_axis, stop_axis) class TestFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_InputError(): diff --git a/python/paddle/fluid/tests/unittests/npu/test_float_status_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_float_status_op_npu.py index 3db98ea8f38..fdc9b91dd4d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_float_status_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_float_status_op_npu.py @@ -23,7 +23,6 @@ import paddle._legacy_C_ops as ops class TestGetFloatStatusOp(unittest.TestCase): - def setUp(self): device = paddle.set_device('npu') @@ -64,7 +63,6 @@ class TestGetFloatStatusOp(unittest.TestCase): class TestClearFloatStatusOp(unittest.TestCase): - def setUp(self): device = paddle.set_device('npu') diff --git a/python/paddle/fluid/tests/unittests/npu/test_gather_nd_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_gather_nd_op_npu.py index fa6da7817de..4d66c28d6fa 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_gather_nd_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_gather_nd_op_npu.py @@ -23,11 +23,11 @@ import paddle def gather_nd_grad(x, index): - dout_shape = index.shape[:-1] + x.shape[index.shape[-1]:] + dout_shape = index.shape[:-1] + x.shape[index.shape[-1] :] numel = 1 for i in dout_shape: numel = numel * i - dout = np.full(dout_shape, 1. / numel) + dout = np.full(dout_shape, 1.0 / numel) dx = np.full_like(x, 0) index = tuple(index.reshape(-1, index.shape[-1]).T) @@ -37,9 +37,8 @@ def gather_nd_grad(x, index): def test_class1(op_type, typename): - class TestGatherNdOpWithEmptyIndex(OpTest): - #Index has empty element, which means copy entire tensor + # Index has empty element, which means copy entire tensor def setUp(self): self.set_npu() @@ -48,7 +47,7 @@ def test_class1(op_type, typename): xnp = np.random.random((5, 20)).astype(typename) self.inputs = { 'X': xnp, - 'Index': np.array([[], []]).astype("int32") + 'Index': np.array([[], []]).astype("int32"), } self.outputs = { 'Out': np.vstack((xnp[np.newaxis, :], xnp[np.newaxis, :])) @@ -72,9 +71,7 @@ def test_class1(op_type, typename): def test_class2(op_type, typename): - class TestGatherNdOpWithIndex1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -101,9 +98,8 @@ def test_class2(op_type, typename): def test_class3(op_type, typename): - class TestGatherNdOpWithLowIndex(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.set_npu() @@ -126,9 +122,9 @@ def test_class3(op_type, typename): if typename == "float16" or typename == "int64": self.__class__.no_need_check_grad = True else: - self.check_grad_with_place(self.place, ['X'], - 'Out', - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Out', user_defined_grads=[self.x_grad] + ) cls_name = "{0}_{1}_3".format(op_type, typename) TestGatherNdOpWithLowIndex.__name__ = cls_name @@ -136,9 +132,8 @@ def test_class3(op_type, typename): def test_class4(op_type, typename): - class TestGatherNdOpIndex1(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.set_npu() @@ -169,9 +164,8 @@ def test_class4(op_type, typename): def test_class5(op_type, typename): - class TestGatherNdOpWithSameIndexAsX(OpTest): - #Index has same rank as X's rank + # Index has same rank as X's rank def setUp(self): self.set_npu() @@ -181,7 +175,7 @@ def test_class5(op_type, typename): index = np.array([[1, 1], [2, 1]]).astype("int64") self.inputs = {'X': xnp, 'Index': index} - self.outputs = {'Out': xnp[tuple(index.T)]} #[25, 22] + self.outputs = {'Out': xnp[tuple(index.T)]} # [25, 22] def set_npu(self): self.__class__.use_npu = True @@ -201,9 +195,8 @@ def test_class5(op_type, typename): def test_class6(op_type, typename): - class TestGatherNdOpWithHighRankSame(OpTest): - #Both Index and X have high rank, and Rank(Index) = Rank(X) + # Both Index and X have high rank, and Rank(Index) = Rank(X) def setUp(self): self.set_npu() @@ -211,8 +204,9 @@ def test_class6(op_type, typename): self.op_type = "gather_nd" shape = (5, 2, 3, 1, 10) xnp = np.random.rand(*shape).astype(typename) - index = np.vstack([np.random.randint(0, s, size=2) - for s in shape]).T + index = np.vstack( + [np.random.randint(0, s, size=2) for s in shape] + ).T self.inputs = {'X': xnp, 'Index': index.astype("int32")} self.outputs = {'Out': xnp[tuple(index.T)]} @@ -235,9 +229,8 @@ def test_class6(op_type, typename): def test_class7(op_type, typename): - class TestGatherNdOpWithHighRankDiff(OpTest): - #Both Index and X have high rank, Rank(Index) < Rank(X) + # Both Index and X have high rank, Rank(Index) < Rank(X) def setUp(self): self.set_npu() @@ -246,7 +239,8 @@ def test_class7(op_type, typename): shape = (2, 3, 4, 1, 10) xnp = np.random.rand(*shape).astype(typename) index = np.vstack( - [np.random.randint(0, s, size=200) for s in shape]).T + [np.random.randint(0, s, size=200) for s in shape] + ).T index_re = index.reshape([20, 5, 2, 5]) self.inputs = {'X': xnp, 'Index': index_re.astype("int32")} @@ -270,7 +264,6 @@ def test_class7(op_type, typename): class TestGatherNdAPI(unittest.TestCase): - def test_imperative(self): paddle.disable_static() input_1 = np.array([[1, 2], [3, 4], [5, 6]]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_gather_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_gather_op_npu.py index bedc6f1efc7..5e783b952af 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_gather_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_gather_op_npu.py @@ -34,7 +34,6 @@ def gather_numpy(x, index, axis): class TestGatherOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -43,7 +42,7 @@ class TestGatherOp(OpTest): xnp = np.random.random(self.x_shape).astype(self.x_type) self.inputs = { 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) + 'Index': np.array(self.index).astype(self.index_type), } self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} @@ -72,19 +71,17 @@ class TestGatherOp(OpTest): class TestCase1(TestGatherOp): - def config(self): """ For one dimension input """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float32" self.index = [1, 3, 5] self.index_type = "int32" class API_TestGather(unittest.TestCase): - def test_out1(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[-1, 2], dtype='float32') @@ -94,17 +91,16 @@ class API_TestGather(unittest.TestCase): exe = fluid.Executor(place) input = np.array([[1, 2], [3, 4], [5, 6]]) index_1 = np.array([1, 2]) - result, = exe.run(feed={ - "data1": input, - "index": index_1 - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"data1": input, "index": index_1}, fetch_list=[out] + ) expected_output = np.array([[3, 4], [5, 6]]) np.testing.assert_allclose(result, expected_output, rtol=1e-5) def test_out2(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data('x', shape=[-1, 2], dtype='float32') index = paddle.fluid.data('index', shape=[-1, 1], dtype='int32') out = paddle.gather(x, index) @@ -112,17 +108,14 @@ class API_TestGather(unittest.TestCase): exe = paddle.static.Executor(place) x_np = np.array([[1, 2], [3, 4], [5, 6]]).astype('float32') index_np = np.array([1, 1]).astype('int32') - result, = exe.run(feed={ - "x": x_np, - "index": index_np - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"x": x_np, "index": index_np}, fetch_list=[out] + ) expected_output = gather_numpy(x_np, index_np, axis=0) np.testing.assert_allclose(result, expected_output, rtol=1e-5) class TestGatherGrad(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -135,9 +128,9 @@ class TestGatherGrad(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[8192, 768], dtype='float32') - index = paddle.static.data(name="index", - shape=[1232, 1], - dtype='int32') + index = paddle.static.data( + name="index", shape=[1232, 1], dtype='int32' + ) a.stop_gradient = False b = paddle.gather(a, index) @@ -156,15 +149,17 @@ class TestGatherGrad(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "index": index_np - }, - fetch_list=[b, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "index": index_np}, + fetch_list=[b, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res[0])) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res[0] + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_gaussian_random_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_gaussian_random_op_npu.py index 5f6b756f175..326945bb478 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_gaussian_random_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_gaussian_random_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestNPUGaussianRandomOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "gaussian_random" @@ -39,7 +38,7 @@ class TestNPUGaussianRandomOp(OpTest): "mean": self.mean, "std": self.std, "seed": 10, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } paddle.seed(10) @@ -47,7 +46,7 @@ class TestNPUGaussianRandomOp(OpTest): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def set_npu(self): self.__class__.use_npu = True @@ -68,12 +67,13 @@ class TestNPUGaussianRandomOp(OpTest): hist2, _ = np.histogram(data, range=(-3, 5)) hist2 = hist2.astype("float32") hist2 /= float(outs[0].size) - np.testing.assert_allclose(hist, - hist2, - rtol=0, - atol=0.01, - err_msg="hist: " + str(hist) + " hist2: " + - str(hist2)) + np.testing.assert_allclose( + hist, + hist2, + rtol=0, + atol=0.01, + err_msg="hist: " + str(hist) + " hist2: " + str(hist2), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/npu/test_gelu_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_gelu_op_npu.py index 2a03c36622d..0c2d163beca 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_gelu_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_gelu_op_npu.py @@ -32,7 +32,6 @@ def np_gelu(x): class TestGelu(OpTest): - def setUp(self): self.set_npu() self.op_type = "gelu" @@ -57,13 +56,12 @@ class TestGelu(OpTest): self.check_output_with_place(self.place, atol=1e-3) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.007) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.007 + ) class TestGeluFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "gelu" @@ -90,7 +88,6 @@ class TestGeluFp16(OpTest): class TestGeluNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -105,9 +102,9 @@ class TestGeluNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) @@ -131,16 +128,17 @@ class TestGeluNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_group_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_group_norm_op_npu.py index e995e7f9475..06d0f5dd1d0 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_group_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_group_norm_op_npu.py @@ -46,7 +46,6 @@ def group_norm_naive(x, scale, bias, epsilon, groups, data_layout): class TestGroupNormOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -58,9 +57,9 @@ class TestGroupNormOpError(unittest.TestCase): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[2, 100, 3, 5], - dtype='int32') + x2 = fluid.layers.data( + name='x2', shape=[2, 100, 3, 5], dtype='int32' + ) groups = 2 fluid.layers.group_norm(x2, groups) @@ -68,7 +67,6 @@ class TestGroupNormOpError(unittest.TestCase): class TestGroupNormOp(OpTest): - def setUp(self): self.set_npu() self.op_type = 'group_norm' @@ -89,15 +87,19 @@ class TestGroupNormOp(OpTest): input = np.transpose(input, (0, 2, 3, 1)) scale = np.random.random([self.shape[1]]).astype(self.dtype) bias = np.random.random([self.shape[1]]).astype(self.dtype) - output, mean, var = group_norm_naive(input, scale, bias, - self.attrs['epsilon'], - self.attrs['groups'], - self.data_format) + output, mean, var = group_norm_naive( + input, + scale, + bias, + self.attrs['epsilon'], + self.attrs['groups'], + self.data_format, + ) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(input), 'Scale': OpTest.np_dtype_to_fluid_dtype(scale), - 'Bias': OpTest.np_dtype_to_fluid_dtype(bias) + 'Bias': OpTest.np_dtype_to_fluid_dtype(bias), } self.outputs = {'Y': output, 'Mean': mean, 'Variance': var} self.attrs['data_layout'] = self.data_format @@ -120,67 +122,65 @@ class TestGroupNormOp(OpTest): output_names = 'Y' no_grad_set = set() cpu_place = fluid.CPUPlace() - cpu_grads = self._get_gradient(inputs_to_check, cpu_place, output_names, - no_grad_set) - npu_grads = self._get_gradient(inputs_to_check, self.place, - output_names, no_grad_set) - - self._assert_is_close(cpu_grads, npu_grads, inputs_to_check, - self.max_relative_error, - "Gradient Check between places") + cpu_grads = self._get_gradient( + inputs_to_check, cpu_place, output_names, no_grad_set + ) + npu_grads = self._get_gradient( + inputs_to_check, self.place, output_names, no_grad_set + ) + + self._assert_is_close( + cpu_grads, + npu_grads, + inputs_to_check, + self.max_relative_error, + "Gradient Check between places", + ) def init_test_case(self): pass class TestGroupNormOp1(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 class TestGroupNormOp2(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 class TestGroupNormOpBigEps1(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.attrs['epsilon'] = 0.5 class TestGroupNormOpBigEps2(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.attrs['epsilon'] = 0.5 class TestGroupNormOpBigEps3(TestGroupNormOp): - def init_test_case(self): self.attrs['epsilon'] = 0.5 class TestGroupNormOp1_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.data_format = "NHWC" class TestGroupNormOp2_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.data_format = "NHWC" class TestGroupNormOpBigEps1_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.attrs['epsilon'] = 0.5 @@ -188,7 +188,6 @@ class TestGroupNormOpBigEps1_With_NHWC(TestGroupNormOp): class TestGroupNormOpBigEps2_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.attrs['epsilon'] = 0.5 @@ -196,20 +195,17 @@ class TestGroupNormOpBigEps2_With_NHWC(TestGroupNormOp): class TestGroupNormOpBigEps3_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['epsilon'] = 0.5 self.data_format = "NHWC" class TestGroupNormOpFP16(TestGroupNormOp): - def init_dtype(self): self.dtype = np.float16 class TestGroupNormOpFP16_With_NHWC(TestGroupNormOp): - def init_dtype(self): self.dtype = np.float16 @@ -223,9 +219,9 @@ class TestGroupNormException(unittest.TestCase): data = fluid.data(name='data', shape=[None, 3, 3, 4], dtype="float64") def attr_data_format(): - out = fluid.layers.group_norm(input=data, - groups=2, - data_layout="NDHW") + out = fluid.layers.group_norm( + input=data, groups=2, data_layout="NDHW" + ) self.assertRaises(ValueError, attr_data_format) diff --git a/python/paddle/fluid/tests/unittests/npu/test_hard_sigmoid_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_hard_sigmoid_op_npu.py index c7c69eef5e5..44155e43880 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_hard_sigmoid_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_hard_sigmoid_op_npu.py @@ -26,11 +26,10 @@ SEED = 2021 def ref_hardsigmoid(x, slope=0.166666666666667, offset=0.5): - return np.maximum(np.minimum(x * slope + offset, 1.), 0.).astype(x.dtype) + return np.maximum(np.minimum(x * slope + offset, 1.0), 0.0).astype(x.dtype) class TestNPUHardSigmoid(OpTest): - def setUp(self): paddle.enable_static() @@ -41,7 +40,7 @@ class TestNPUHardSigmoid(OpTest): x = np.random.uniform(-5, 5, [10, 12]).astype(self.dtype) lower_threshold = -self.offset / self.slope - upper_threshold = (1. - self.offset) / self.slope + upper_threshold = (1.0 - self.offset) / self.slope # Same reason as TestAbs delta = 0.005 @@ -73,21 +72,18 @@ class TestNPUHardSigmoid(OpTest): class TestNPUHardSigmoid2(TestNPUHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.5 class TestNPUHardSigmoid3(TestNPUHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.4 class TestNPUHardSigmoidFp16(TestNPUHardSigmoid): - def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) @@ -144,14 +140,14 @@ class TestHardsigmoidAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.hardsigmoid, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardsigmoid, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardsigmoid(x_fp16) diff --git a/python/paddle/fluid/tests/unittests/npu/test_hard_swish_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_hard_swish_op_npu.py index 3b0b5a712c2..86c4bf5402d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_hard_swish_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_hard_swish_op_npu.py @@ -24,15 +24,17 @@ import paddle.nn.functional as F def ref_hard_swish_grad(x, threshold=6.0, scale=6.0, offset=3.0): - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) tmp = ((x + offset) < threshold).astype(x.dtype) - dx = dout * (((x + offset) > 0).astype(x.dtype) * - (2 * x + offset) * tmp / scale + 1.0 - tmp) + dx = dout * ( + ((x + offset) > 0).astype(x.dtype) * (2 * x + offset) * tmp / scale + + 1.0 + - tmp + ) return dx class TestHardSwishNPU(OpTest): - def setUp(self): paddle.enable_static() @@ -45,13 +47,12 @@ class TestHardSwishNPU(OpTest): threshold = 6.0 scale = 6.0 offset = 3.0 - #the same with TestAbs + # the same with TestAbs x[np.abs(x + offset) < 0.005] = 0.02 x[np.abs(x - threshold + offset) < 0.005] = threshold - offset + 0.02 out = ( - x * - (np.minimum(np.maximum(x + offset, 0.), threshold) / scale)).astype( - self.dtype) + x * (np.minimum(np.maximum(x + offset, 0.0), threshold) / scale) + ).astype(self.dtype) self.x_grad = ref_hard_swish_grad(x, threshold, scale, offset) self.inputs = {'X': x} @@ -72,13 +73,12 @@ class TestHardSwishNPU(OpTest): # can't satisfy the default precision requirement # when compared with numeric_grads, but the results on # NPU and CPU are same (verified in TestHardSwishNPUWithCPU) - self.check_grad_with_place(self.place, ['X'], - 'Out', - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Out', user_defined_grads=[self.x_grad] + ) class TestHardSwishNPUFp16(TestHardSwishNPU): - def test_check_output(self): self.check_output_with_place(self.place) @@ -88,7 +88,6 @@ class TestHardSwishNPUFp16(TestHardSwishNPU): # test the result of hard_swish and hard_swish_grad on CPU and NPU class TestHardSwishNPUWithCPU(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -116,17 +115,33 @@ class TestHardSwishNPUWithCPU(unittest.TestCase): np.testing.assert_allclose( self.out_y.numpy(), y.numpy(), - err_msg="Output of NPU HardSwish forward has diff at " + - str(self.place) + "\nExpect " + str(self.out_y) + "\n" + "But Got" + - str(y) + " in class " + self.__class__.__name__ + ".", - rtol=1e-5) + err_msg="Output of NPU HardSwish forward has diff at " + + str(self.place) + + "\nExpect " + + str(self.out_y) + + "\n" + + "But Got" + + str(y) + + " in class " + + self.__class__.__name__ + + ".", + rtol=1e-5, + ) np.testing.assert_allclose( self.out_g.numpy(), data.grad.numpy(), - err_msg="Output of NPU HardSwish backward has diff at " + - str(self.place) + "\nExpect " + str(self.out_g) + "\n" + "But Got" + - str(data.grad) + " in class " + self.__class__.__name__ + ".", - rtol=1e-5) + err_msg="Output of NPU HardSwish backward has diff at " + + str(self.place) + + "\nExpect " + + str(self.out_g) + + "\n" + + "But Got" + + str(data.grad) + + " in class " + + self.__class__.__name__ + + ".", + rtol=1e-5, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_huber_loss_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_huber_loss_op_npu.py index d9a83d74e1f..0e81f00de62 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_huber_loss_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_huber_loss_op_npu.py @@ -33,10 +33,10 @@ def huber_loss_forward(val, delta): return delta * (abs_val - 0.5 * delta) -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestHuberLossOp(OpTest): - def setUp(self): self.set_npu() self.op_type = 'huber_loss' @@ -50,11 +50,11 @@ class TestHuberLossOp(OpTest): def set_inputs(self): shape = self.set_shape() - x = np.random.uniform(0, 1., shape).astype(self.dtype) - y = np.random.uniform(0, 1., shape).astype(self.dtype) + x = np.random.uniform(0, 1.0, shape).astype(self.dtype) + y = np.random.uniform(0, 1.0, shape).astype(self.dtype) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } def set_attrs(self): @@ -64,8 +64,9 @@ class TestHuberLossOp(OpTest): delta = self.attrs['delta'] shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - delta).astype(self.dtype) + loss = np.vectorize(huber_loss_forward)(residual, delta).astype( + self.dtype + ) self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def set_shape(self): @@ -84,46 +85,48 @@ class TestHuberLossOp(OpTest): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.008, - no_grad_set=set("residual")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.008, + no_grad_set=set("residual"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.008, - no_grad_set=set('residual')) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.008, + no_grad_set=set('residual'), + ) def TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (64) + return 64 def TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (6, 6) def TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (6, 6, 1) def TestHuberLossOpFP16(TestHuberLossOp): - def init_dtype(self): self.dtype = np.float16 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestHuberLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input and label must be Variable @@ -138,10 +141,12 @@ class TestHuberLossOpError(unittest.TestCase): # the dtype of input and label must be float32 or float64 xw2 = fluid.data(name='xw2', shape=[None, 6], dtype="int32") lw2 = fluid.data(name='lw2', shape=[None, 6], dtype="int32") - self.assertRaises(TypeError, fluid.layers.huber_loss, xw2, lr, - delta) - self.assertRaises(TypeError, fluid.layers.huber_loss, xr, lw2, - delta) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xw2, lr, delta + ) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xr, lw2, delta + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_increment_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_increment_op_npu.py index c84abc5d479..39f11737bce 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_increment_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_increment_op_npu.py @@ -29,7 +29,6 @@ NPUPlace = 0 class TestIncrement(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(NPUPlace) @@ -37,8 +36,9 @@ class TestIncrement(OpTest): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(np.array([1]).astype(self.dtype)), + 'X': OpTest.np_dtype_to_fluid_dtype( + np.array([1]).astype(self.dtype) + ), } self.attrs = {"Step": 1} @@ -56,7 +56,6 @@ class TestIncrement(OpTest): class TestIncrementFP16(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(NPUPlace) @@ -64,8 +63,9 @@ class TestIncrementFP16(OpTest): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(np.array([1]).astype(self.dtype)), + 'X': OpTest.np_dtype_to_fluid_dtype( + np.array([1]).astype(self.dtype) + ), } self.pre_input_id = id(self.inputs['X']) @@ -83,7 +83,6 @@ class TestIncrementFP16(OpTest): class TestIncrementINT64(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(NPUPlace) @@ -91,8 +90,9 @@ class TestIncrementINT64(OpTest): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(np.array([1]).astype(self.dtype)), + 'X': OpTest.np_dtype_to_fluid_dtype( + np.array([1]).astype(self.dtype) + ), } self.pre_input_id = id(self.inputs['X']) @@ -110,7 +110,6 @@ class TestIncrementINT64(OpTest): class TestIncrementInplace(unittest.TestCase): - def test_npu(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -129,9 +128,13 @@ class TestIncrementInplace(unittest.TestCase): exe = paddle.static.Executor(place) exe.run(startup_prog) - b_value = exe.run(main_prog, feed={ - "a": a_np, - }, fetch_list=[b]) + b_value = exe.run( + main_prog, + feed={ + "a": a_np, + }, + fetch_list=[b], + ) print('input a id is : {}'.format(id(a))) print('input b id is : {}'.format(id(b))) diff --git a/python/paddle/fluid/tests/unittests/npu/test_index_sample_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_index_sample_op_npu.py index e64d556ebdc..5883ef7b567 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_index_sample_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_index_sample_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestIndexSampleOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -35,10 +34,9 @@ class TestIndexSampleOp(OpTest): self.op_type = "index_sample" self.config() xnp = np.random.random(self.x_shape).astype(self.dtype) - indexnp = np.random.randint(low=0, - high=self.x_shape[1], - size=self.index_shape).astype( - self.index_type) + indexnp = np.random.randint( + low=0, high=self.x_shape[1], size=self.index_shape + ).astype(self.index_type) self.inputs = {'X': xnp, 'Index': indexnp} index_array = [] for i in range(self.index_shape[0]): @@ -65,7 +63,6 @@ class TestIndexSampleOp(OpTest): class TestCase1(TestIndexSampleOp): - def config(self): """ For one dimension input @@ -77,7 +74,6 @@ class TestCase1(TestIndexSampleOp): class TestCase2(TestIndexSampleOp): - def config(self): """ For int64_t index type @@ -89,7 +85,6 @@ class TestCase2(TestIndexSampleOp): class TestCase3(TestIndexSampleOp): - def config(self): """ For int index type @@ -101,7 +96,6 @@ class TestCase3(TestIndexSampleOp): class TestCase4(TestIndexSampleOp): - def config(self): """ For int64 index type @@ -113,7 +107,6 @@ class TestCase4(TestIndexSampleOp): class TestCase5(TestIndexSampleOp): - def config(self): """ For float16 x type @@ -129,7 +122,6 @@ class TestCase5(TestIndexSampleOp): class TestCase6(TestCase5): - def config(self): """ For int32 x type @@ -142,7 +134,6 @@ class TestCase6(TestCase5): class TestCase7(TestCase5): - def config(self): """ For int64 x type @@ -155,7 +146,6 @@ class TestCase7(TestCase5): class TestIndexSampleShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() # create x value @@ -166,8 +156,9 @@ class TestIndexSampleShape(unittest.TestCase): # create index value index_shape = (2, 3) index_type = "int32" - index_np = np.random.randint(low=0, high=x_shape[1], - size=index_shape).astype(index_type) + index_np = np.random.randint( + low=0, high=x_shape[1], size=index_shape + ).astype(index_type) x = fluid.data(name='x', shape=[-1, 5], dtype='float32') index = fluid.data(name='index', shape=[-1, 3], dtype='int32') @@ -182,18 +173,24 @@ class TestIndexSampleShape(unittest.TestCase): class TestIndexSampleDynamic(unittest.TestCase): - def test_result(self): with fluid.dygraph.guard(paddle.NPUPlace(0)): - x = paddle.to_tensor([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]], - dtype='float32') - index = paddle.to_tensor([[0, 1, 2], [1, 2, 3], [0, 0, 0]], - dtype='int32') + x = paddle.to_tensor( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + dtype='float32', + ) + index = paddle.to_tensor( + [[0, 1, 2], [1, 2, 3], [0, 0, 0]], dtype='int32' + ) out_z1 = paddle.index_sample(x, index) - except_output = np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0], - [9.0, 9.0, 9.0]]) + except_output = np.array( + [[1.0, 2.0, 3.0], [6.0, 7.0, 8.0], [9.0, 9.0, 9.0]] + ) assert out_z1.numpy().all() == except_output.all() diff --git a/python/paddle/fluid/tests/unittests/npu/test_index_select_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_index_select_op_npu.py index 7050afa4411..39b0a011893 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_index_select_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_index_select_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestNPUIndexSelect(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -34,15 +33,17 @@ class TestNPUIndexSelect(OpTest): self.config() x_np = np.random.random(self.x_shape).astype(self.x_type) - index_np = np.random.randint(low=0, - high=self.x_shape[self.dim], - size=self.index_size, - dtype=self.index_type) + index_np = np.random.randint( + low=0, + high=self.x_shape[self.dim], + size=self.index_size, + dtype=self.index_type, + ) # compute real output as baseline. - outer_loop = np.prod(self.x_shape[:self.dim]) + outer_loop = np.prod(self.x_shape[: self.dim]) outer_loop = outer_loop.astype(self.index_type) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] @@ -76,7 +77,6 @@ class TestNPUIndexSelect(OpTest): class TestNPUIndexSelectCase2(TestNPUIndexSelect): - def config(self): self.dim = -2 self.x_type = np.float32 @@ -86,7 +86,6 @@ class TestNPUIndexSelectCase2(TestNPUIndexSelect): class TestNPUIndexSelectCase3(TestNPUIndexSelect): - def config(self): self.dim = 0 self.x_type = np.float32 @@ -96,7 +95,6 @@ class TestNPUIndexSelectCase3(TestNPUIndexSelect): class TestNPUIndexSelectCase4(TestNPUIndexSelect): - def config(self): self.dim = -1 self.x_type = np.float32 @@ -106,10 +104,14 @@ class TestNPUIndexSelectCase4(TestNPUIndexSelect): class TestNPUIndexSelectAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]]).astype('float32') + self.data_x = np.array( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] + ).astype('float32') self.data_index = np.array([0, 1, 1]).astype('int32') def test_index_select_api(self): @@ -123,14 +125,14 @@ class TestNPUIndexSelectAPI(unittest.TestCase): index = paddle.static.data(name='index', shape=[3], dtype='int32') z = paddle.index_select(x, index, axis=1) exe = paddle.static.Executor(paddle.NPUPlace(0)) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]).astype('float32') + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ).astype('float32') np.testing.assert_allclose(expect_out, np.array(res)) # case 2: @@ -139,14 +141,14 @@ class TestNPUIndexSelectAPI(unittest.TestCase): index = paddle.static.data(name='index', shape=[3], dtype='int32') z = paddle.index_select(x, index) exe = paddle.static.Executor(paddle.NPUPlace(0)) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]).astype('float32') + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ).astype('float32') np.testing.assert_allclose(expect_out, np.array(res)) def test_dygraph_index_select_api(self): @@ -159,8 +161,9 @@ class TestNPUIndexSelectAPI(unittest.TestCase): index = paddle.to_tensor(self.data_index) z = paddle.index_select(x, index) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]).astype('float32') + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ).astype('float32') np.testing.assert_allclose(expect_out, np_z) # case 2: @@ -168,8 +171,9 @@ class TestNPUIndexSelectAPI(unittest.TestCase): index = paddle.to_tensor(self.data_index) z = paddle.index_select(x, index, axis=1) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]).astype('float32') + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ).astype('float32') np.testing.assert_allclose(expect_out, np_z) diff --git a/python/paddle/fluid/tests/unittests/npu/test_instance_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_instance_norm_op_npu.py index 761c38a8576..1abc4af24cd 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_instance_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_instance_norm_op_npu.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestInstanceNorm(unittest.TestCase): - def test_dygraph(self): places = [fluid.NPUPlace(0)] for p in places: diff --git a/python/paddle/fluid/tests/unittests/npu/test_iou_similarity_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_iou_similarity_op_npu.py index 2485c8308c1..9b7b005dd3a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_iou_similarity_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_iou_similarity_op_npu.py @@ -28,7 +28,6 @@ np.random.seed(2021) class TestNpuIouSimilarityOp(OpTest): - def setUp(self): self.op_type = "iou_similarity" self.set_npu() @@ -58,7 +57,7 @@ class TestNpuIouSimilarityOp(OpTest): self.boxes1_lod = [[1 for _ in range(self.N)]] self.inputs = { 'X': (self.boxes1, self.boxes1_lod), - 'Y': self.boxes2 + 'Y': self.boxes2, } else: self.inputs = {'X': self.boxes1, 'Y': self.boxes2} @@ -74,7 +73,9 @@ class TestNpuIouSimilarityOp(OpTest): def test_check_output(self): self.check_output_with_place(self.place) - def _compute_iou(self, ): + def _compute_iou( + self, + ): for row in range(self.boxes1.shape[0]): for col in range(self.boxes2.shape[0]): xmin1, ymin1, xmax1, ymax1 = self.boxes1[row] @@ -104,7 +105,6 @@ class TestNpuIouSimilarityOp(OpTest): class TestNpuIouSimilarityOpWithLoD(TestNpuIouSimilarityOp): - def set_init_config(self): super(TestNpuIouSimilarityOpWithLoD, self).set_init_config() self.box_normalized = True @@ -112,7 +112,6 @@ class TestNpuIouSimilarityOpWithLoD(TestNpuIouSimilarityOp): class TestNpuIouSimilarityOpWithBoxNormalized(TestNpuIouSimilarityOp): - def set_init_config(self): super(TestNpuIouSimilarityOpWithBoxNormalized, self).set_init_config() self.box_normalized = True @@ -120,7 +119,6 @@ class TestNpuIouSimilarityOpWithBoxNormalized(TestNpuIouSimilarityOp): def TestNpuIouSimilarityOpFp16(TestNpuIouSimilarityOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_is_empty_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_is_empty_op_npu.py index f939716f37f..9834ea0d5f2 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_is_empty_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_is_empty_op_npu.py @@ -23,10 +23,10 @@ import paddle paddle.enable_static() -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestEmpty(OpTest): - def setUp(self): self.set_npu() self.init_dtype() @@ -48,23 +48,24 @@ class TestEmpty(OpTest): self.check_output_with_place(self.place) -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestNotEmpty(TestEmpty): - def set_data(self): self.inputs = {'X': np.array([])} self.outputs = {'Out': np.array([True])} -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestIsEmptyOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_data = np.random.random((3, 2)).astype("float32") def test_Variable(): @@ -75,27 +76,27 @@ class TestIsEmptyOpError(unittest.TestCase): def test_type(): # dtype must be float32, float16 in NPU - x3 = paddle.static.data(name="x3", - shape=[4, 32, 32], - dtype="bool") + x3 = paddle.static.data( + name="x3", shape=[4, 32, 32], dtype="bool" + ) res = paddle.is_empty(x=x3) self.assertRaises(TypeError, test_type) def test_name_type(): # name type must be string. - x4 = paddle.static.data(name="x4", - shape=[3, 2], - dtype="float32") + x4 = paddle.static.data( + name="x4", shape=[3, 2], dtype="float32" + ) res = paddle.is_empty(x=x4, name=1) self.assertRaises(TypeError, test_name_type) -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestIsEmptyOpDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static(paddle.NPUPlace(0)) input = paddle.rand(shape=[4, 32, 32], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/npu/test_kldiv_loss_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_kldiv_loss_op_npu.py index bf5042c01d2..3a55d9973af 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_kldiv_loss_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_kldiv_loss_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestKLDivLossOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -55,10 +54,13 @@ class TestKLDivLossOp(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Loss', - no_grad_set=set(["Target"]), - max_relative_error=0.15) + self.check_grad_with_place( + self.place, + ['X'], + 'Loss', + no_grad_set=set(["Target"]), + max_relative_error=0.15, + ) def initTestCase(self): self.x_shape = (4, 5, 5) @@ -66,28 +68,24 @@ class TestKLDivLossOp(OpTest): class TestKLDivLossOp2(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (3, 2, 7, 7) self.reduction = 'none' class TestKLDivLossOp3(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 7, 9) self.reduction = 'mean' class TestKLDivLossOp4(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (5, 20) self.reduction = 'sum' class TestKLDivLossOp_fp16(TestKLDivLossOp): - def init_dtype(self): self.dtype = 'float16' @@ -95,17 +93,22 @@ class TestKLDivLossOp_fp16(TestKLDivLossOp): self.check_output_with_place(self.place, atol=3e-1) def test_check_grad(self): - input_grad = -self.inputs['Target'] * ( - self.inputs['Target'] > 0) / self.inputs['Target'].shape[0] - self.check_grad_with_place(self.place, ['X'], - 'Loss', - no_grad_set=set(["Target"]), - max_relative_error=0.2, - user_defined_grads=[input_grad]) + input_grad = ( + -self.inputs['Target'] + * (self.inputs['Target'] > 0) + / self.inputs['Target'].shape[0] + ) + self.check_grad_with_place( + self.place, + ['X'], + 'Loss', + no_grad_set=set(["Target"]), + max_relative_error=0.2, + user_defined_grads=[input_grad], + ) class TestKLDivLossDygraph(unittest.TestCase): - def run_kl_loss(self, reduction, shape=(5, 20)): x = np.random.uniform(-10, 10, shape).astype('float32') target = np.random.uniform(-10, 10, shape).astype('float32') @@ -113,8 +116,9 @@ class TestKLDivLossDygraph(unittest.TestCase): with paddle.fluid.dygraph.guard(paddle.NPUPlace(0)): kldiv_criterion = paddle.nn.KLDivLoss(reduction) - pred_loss = kldiv_criterion(paddle.to_tensor(x), - paddle.to_tensor(target)) + pred_loss = kldiv_criterion( + paddle.to_tensor(x), paddle.to_tensor(target) + ) np.testing.assert_allclose(pred_loss.numpy(), gt_loss, rtol=1e-6) def test_kl_loss_batchmean(self): @@ -140,7 +144,6 @@ class TestKLDivLossDygraph(unittest.TestCase): class TestKLDivLossTypePromotion(unittest.TestCase): - def test_kl_div_promotion(self): with paddle.fluid.dygraph.guard(paddle.NPUPlace(0)): x1 = paddle.rand([5, 20], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/npu/test_label_smooth_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_label_smooth_op_npu.py index 26bcf0f6f86..8bc5c9f5439 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_label_smooth_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_label_smooth_op_npu.py @@ -25,10 +25,10 @@ paddle.enable_static() SEED = 2021 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestLabelSmoothOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "label_smooth" @@ -62,8 +62,9 @@ class TestLabelSmoothOp(OpTest): self.attrs = {"epsilon": epsilon} def set_outputs(self): - dist = None if 'PriorDist' not in self.inputs else self.inputs[ - 'PriorDist'] + dist = ( + None if 'PriorDist' not in self.inputs else self.inputs['PriorDist'] + ) out = self.calc_out(self.inputs['X'], self.attrs['epsilon'], dist) self.outputs = {'Out': out} @@ -78,15 +79,14 @@ class TestLabelSmoothOp(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): - def set_inputs(self): super(TestLabelSmoothOpWithPriorDist, self).set_inputs() label_dim = self.inputs['X'].shape[-1] @@ -95,39 +95,33 @@ class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): class TestLabelSmoothOp3D(TestLabelSmoothOp): - def set_inputs(self): super(TestLabelSmoothOp3D, self).set_inputs() self.inputs['X'].reshape([2, -1, self.inputs['X'].shape[-1]]) class TestLabelSmoothOpWithPriorDist3D(TestLabelSmoothOpWithPriorDist): - def set_inputs(self): super(TestLabelSmoothOpWithPriorDist3D, self).set_inputs() self.inputs['X'].reshape([2, -1, self.inputs['X'].shape[-1]]) class TestLabelSmoothOpFP16(TestLabelSmoothOp): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOpWithPriorDistFP16(TestLabelSmoothOpWithPriorDist): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOp3DFP16(TestLabelSmoothOp3D): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOpWithPriorDist3DFP16(TestLabelSmoothOpWithPriorDist3D): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_layer_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_layer_norm_op_npu.py index f87cf8c8053..6ca621b647d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_layer_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_layer_norm_op_npu.py @@ -23,7 +23,10 @@ from operator import mul import paddle import paddle.fluid as fluid import paddle.fluid.core as core -from test_layer_norm_op import _reference_layer_norm_naive, _reference_layer_norm_grad +from test_layer_norm_op import ( + _reference_layer_norm_naive, + _reference_layer_norm_grad, +) paddle.enable_static() @@ -36,7 +39,6 @@ _set_use_system_allocator(False) class TestLayerNormOp(unittest.TestCase): - def setUp(self): self.use_cudnn = True self.set_npu() @@ -51,44 +53,54 @@ class TestLayerNormOp(unittest.TestCase): self.atol = 1e-4 def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor).astype( - np_array.dtype).reshape(np_array.shape), - np_array, - atol=atol, - err_msg=msg) - - def check_forward_backward(self, - shape, - begin_norm_axis, - has_scale=True, - has_bias=True, - y_grad_scale=1.0, - use_mkldnn=False): - - def test_with_place(place, - shape, - begin_norm_axis, - use_mkldnn=use_mkldnn): + np.testing.assert_allclose( + np.array(tensor).astype(np_array.dtype).reshape(np_array.shape), + np_array, + atol=atol, + err_msg=msg, + ) + + def check_forward_backward( + self, + shape, + begin_norm_axis, + has_scale=True, + has_bias=True, + y_grad_scale=1.0, + use_mkldnn=False, + ): + def test_with_place( + place, shape, begin_norm_axis, use_mkldnn=use_mkldnn + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) x = np.random.random_sample(x_shape).astype(self.dtype) - scale = np.random.random_sample(scale_shape).astype( - np.float32) if has_scale else None - bias = np.random.random_sample(scale_shape).astype( - np.float32) if has_bias else None + scale = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_scale + else None + ) + bias = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_bias + else None + ) y_grad = (np.random.random_sample(x_shape) * y_grad_scale).astype( - self.dtype) + self.dtype + ) # reference forward & backward y, mean, variance = _reference_layer_norm_naive( - x, scale, bias, epsilon, begin_norm_axis) + x, scale, bias, epsilon, begin_norm_axis + ) x_grad, scale_grad, bias_grad = _reference_layer_norm_grad( - x, y_grad, scale, bias, mean, variance, begin_norm_axis) + x, y_grad, scale, bias, mean, variance, begin_norm_axis + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -103,9 +115,11 @@ class TestLayerNormOp(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype=self.dtype, - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype=self.dtype, + shape=ground_truth[name].shape, + ) inputs = {"X": block.var('x')} fetch_list = [ 'y', @@ -125,17 +139,20 @@ class TestLayerNormOp(unittest.TestCase): outputs={ "Y": block.var('y'), "Mean": block.var('mean'), # share the same memory - "Variance": - block.var('variance'), # share the same memory + "Variance": block.var( + 'variance' + ), # share the same memory }, attrs={ "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, - "use_mkldnn": use_mkldnn - }) + "use_mkldnn": use_mkldnn, + }, + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - layer_norm_op.desc, set(), []) + layer_norm_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -149,46 +166,59 @@ class TestLayerNormOp(unittest.TestCase): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'scale', 'bias', 'y@GRAD'] - }, - fetch_list=fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in ['x', 'scale', 'bias', 'y@GRAD'] + }, + fetch_list=fetch_list, + ) self.__assert_close(y, out[0], "y", self.atol) self.__assert_close(mean, out[1], "mean") self.__assert_close(variance, out[2], "variance", 1e-3) self.__assert_close(x_grad, out[3], "x_grad", 1e-2) if has_scale: - self.__assert_close(scale_grad, - out[fetch_list.index('scale@GRAD')], - "scale_grad", 1e-2) + self.__assert_close( + scale_grad, + out[fetch_list.index('scale@GRAD')], + "scale_grad", + 1e-2, + ) if has_bias: - self.__assert_close(bias_grad, - out[fetch_list.index('bias@GRAD')], - "bias_grad", self.atol) + self.__assert_close( + bias_grad, + out[fetch_list.index('bias@GRAD')], + "bias_grad", + self.atol, + ) test_with_place(self.place, shape, begin_norm_axis) def test_check_forward_backward_with_scale_and_bias(self): self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=1) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=True) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=True, - has_bias=False) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=False) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=True, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=True, + has_bias=False, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=False, + ) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=3) class TestLayerNormOpFP16(TestLayerNormOp): - def init_dtype(self): self.dtype = np.float16 self.atol = 1e-2 diff --git a/python/paddle/fluid/tests/unittests/npu/test_leaky_relu_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_leaky_relu_op_npu.py index 07067ee2519..e91a65faeec 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_leaky_relu_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_leaky_relu_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestLeadyRelu(OpTest): - def setUp(self): self.set_npu() self.op_type = "leaky_relu" @@ -63,33 +62,29 @@ class TestLeadyRelu(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.006 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') class TestLeadyReluFP16(TestLeadyRelu): - def init_dtype(self): self.dtype = np.float16 class TestLeadyRelu2(TestLeadyRelu): - def set_attrs(self): self.attrs = {'alpha': 0.5} class TestLeadyRelu3(TestLeadyRelu): - def set_attrs(self): self.attrs = {'alpha': -0.5} class TestLeakyReluNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -102,9 +97,9 @@ class TestLeakyReluNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): x = paddle.static.data(name="x", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) y = paddle.nn.functional.leaky_relu(x) @@ -127,15 +122,17 @@ class TestLeakyReluNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "x": x_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"x": x_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_log_loss_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_log_loss_op_npu.py index febc99093f1..87cd872e8cc 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_log_loss_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_log_loss_op_npu.py @@ -28,10 +28,10 @@ def sigmoid_array(x): return 1 / (1 + np.exp(-x)) -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestLogLossOp(OpTest): - def setUp(self): self.set_npu() self.op_type = 'log_loss' @@ -58,8 +58,9 @@ class TestLogLossOp(OpTest): epsilon = self.attrs['epsilon'] labels = self.inputs['Labels'] predicted = self.inputs['Predicted'] - loss = -labels * np.log(predicted + epsilon) - ( - 1 - labels) * np.log(1 - predicted + epsilon) + loss = -labels * np.log(predicted + epsilon) - (1 - labels) * np.log( + 1 - predicted + epsilon + ) self.outputs = {'Loss': loss} def set_npu(self): @@ -75,10 +76,10 @@ class TestLogLossOp(OpTest): self.check_grad_with_place(self.place, ['Predicted'], 'Loss') -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestLogLossOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/npu/test_log_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_log_op_npu.py index ea7dd3abde9..8745a66b45a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_log_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_log_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestLog(OpTest): - def setUp(self): self.set_npu() self.op_type = "log" @@ -55,7 +54,6 @@ class TestLog(OpTest): class TestLogFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "log" @@ -82,7 +80,6 @@ class TestLogFp16(OpTest): class TestLogNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -97,9 +94,9 @@ class TestLogNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.log(c) @@ -123,16 +120,17 @@ class TestLogNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_log_softmax_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_log_softmax_op_npu.py index d5dabdeb201..fc0c428983f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_log_softmax_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_log_softmax_op_npu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestLogSoftmaxNPUOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -39,7 +38,7 @@ class TestLogSoftmaxNPUOp(OpTest): self.axis = -1 self.set_attrs() self.set_dtype() - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) self.inputs = {'X': x} @@ -64,18 +63,21 @@ class TestLogSoftmaxNPUOp(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], ['Out'], - user_defined_grads=[self.x_grad], - max_relative_error=0.02) + self.check_grad_with_place( + self.place, + ['X'], + ['Out'], + user_defined_grads=[self.x_grad], + max_relative_error=0.02, + ) else: - self.check_grad_with_place(self.place, ['X'], ['Out'], - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], ['Out'], user_defined_grads=[self.x_grad] + ) def test_class(op_type, typename): - class TestLogSoftmaxShape(TestLogSoftmaxNPUOp): - def set_attrs(self): self.shape = [12, 10] @@ -88,9 +90,7 @@ def test_class(op_type, typename): def test_class2(op_type, typename): - class TestLogSoftmaxAxis(TestLogSoftmaxNPUOp): - def set_attrs(self): self.axis = 0 @@ -109,13 +109,14 @@ for _typename in {np.float32, np.float16}: class TestNNLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] - self.x = np.random.uniform(-1., 1., self.x_shape).astype(np.float32) - self.place = paddle.NPUPlace(0) \ - if paddle.fluid.core.is_compiled_with_npu() \ + self.x = np.random.uniform(-1.0, 1.0, self.x_shape).astype(np.float32) + self.place = ( + paddle.NPUPlace(0) + if paddle.fluid.core.is_compiled_with_npu() else paddle.CPUPlace() + ) def check_api(self, axis=-1): ref_out = np.apply_along_axis(ref_log_softmax, axis, self.x) @@ -142,13 +143,14 @@ class TestNNLogSoftmaxAPI(unittest.TestCase): class TestNNFunctionalLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) - self.place = paddle.NPUPlace(0) \ - if paddle.fluid.core.is_compiled_with_npu() \ + self.place = ( + paddle.NPUPlace(0) + if paddle.fluid.core.is_compiled_with_npu() else paddle.CPUPlace() + ) def check_api(self, axis=-1, dtype=None): x = self.x.copy() diff --git a/python/paddle/fluid/tests/unittests/npu/test_logical_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_logical_op_npu.py index 14422e8565a..a783e04c676 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_logical_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_logical_op_npu.py @@ -24,81 +24,31 @@ from paddle.static import Program, program_guard SUPPORTED_DTYPES = [bool] -TEST_META_OP_DATA = [{ - 'op_str': 'logical_and', - 'binary_op': True -}, { - 'op_str': 'logical_or', - 'binary_op': True -}, { - 'op_str': 'logical_not', - 'binary_op': False -}] +TEST_META_OP_DATA = [ + {'op_str': 'logical_and', 'binary_op': True}, + {'op_str': 'logical_or', 'binary_op': True}, + {'op_str': 'logical_not', 'binary_op': False}, +] TEST_META_SHAPE_DATA = { - 'XDimLargerThanYDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 5] - }, - 'XDimLargerThanYDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 1] - }, - 'XDimLargerThanYDim3': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [1, 4, 1] - }, - 'XDimLargerThanYDim4': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4, 1] - }, - 'XDimLargerThanYDim5': { - 'x_shape': [2, 3, 1, 5], - 'y_shape': [3, 1, 1] - }, - 'XDimLessThanYDim1': { - 'x_shape': [4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim2': { - 'x_shape': [1, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim3': { - 'x_shape': [3, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim4': { - 'x_shape': [3, 1, 1], - 'y_shape': [2, 3, 1, 5] - }, - 'XDimLessThanYDim5': { - 'x_shape': [4, 5], - 'y_shape': [2, 3, 4, 5] - }, - 'Axis1InLargerDim': { - 'x_shape': [1, 4, 5], - 'y_shape': [2, 3, 1, 5] - }, - 'EqualDim1': { - 'x_shape': [10, 7], - 'y_shape': [10, 7] - }, - 'EqualDim2': { - 'x_shape': [1, 1, 4, 5], - 'y_shape': [2, 3, 1, 5] - } + 'XDimLargerThanYDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 5]}, + 'XDimLargerThanYDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 1]}, + 'XDimLargerThanYDim3': {'x_shape': [2, 3, 4, 5], 'y_shape': [1, 4, 1]}, + 'XDimLargerThanYDim4': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4, 1]}, + 'XDimLargerThanYDim5': {'x_shape': [2, 3, 1, 5], 'y_shape': [3, 1, 1]}, + 'XDimLessThanYDim1': {'x_shape': [4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim2': {'x_shape': [1, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim3': {'x_shape': [3, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim4': {'x_shape': [3, 1, 1], 'y_shape': [2, 3, 1, 5]}, + 'XDimLessThanYDim5': {'x_shape': [4, 5], 'y_shape': [2, 3, 4, 5]}, + 'Axis1InLargerDim': {'x_shape': [1, 4, 5], 'y_shape': [2, 3, 1, 5]}, + 'EqualDim1': {'x_shape': [10, 7], 'y_shape': [10, 7]}, + 'EqualDim2': {'x_shape': [1, 1, 4, 5], 'y_shape': [2, 3, 1, 5]}, } TEST_META_WRONG_SHAPE_DATA = { - 'ErrorDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4] - }, - 'ErrorDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 3] - } + 'ErrorDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4]}, + 'ErrorDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 3]}, } @@ -157,16 +107,20 @@ def test(unit_test, use_npu=False, test_error=False): META_DATA = dict(TEST_META_WRONG_SHAPE_DATA) for shape_data in META_DATA.values(): for data_type in SUPPORTED_DTYPES: - meta_data['x_np'] = np_data_generator(shape_data['x_shape'], - dtype=data_type) - meta_data['y_np'] = np_data_generator(shape_data['y_shape'], - dtype=data_type) + meta_data['x_np'] = np_data_generator( + shape_data['x_shape'], dtype=data_type + ) + meta_data['y_np'] = np_data_generator( + shape_data['y_shape'], dtype=data_type + ) if meta_data['binary_op'] and test_error: # catch C++ Exception - unit_test.assertRaises(BaseException, run_static, - **meta_data) - unit_test.assertRaises(BaseException, run_dygraph, - **meta_data) + unit_test.assertRaises( + BaseException, run_static, **meta_data + ) + unit_test.assertRaises( + BaseException, run_dygraph, **meta_data + ) continue static_result = run_static(**meta_data) dygraph_result = run_dygraph(**meta_data) @@ -176,11 +130,11 @@ def test(unit_test, use_npu=False, test_error=False): np_result = np_op(meta_data['x_np']) unit_test.assertTrue((static_result == np_result).all()) unit_test.assertTrue( - (dygraph_result.numpy() == np_result).all()) + (dygraph_result.numpy() == np_result).all() + ) def test_type_error(unit_test, use_npu, type_str_map): - def check_type(op_str, x, y, binary_op): op = getattr(paddle, op_str) error_type = ValueError @@ -215,24 +169,24 @@ def test_type_error(unit_test, use_npu, type_str_map): startup_program = paddle.static.Program() main_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.static.data(name='x', - shape=[10], - dtype=type_str_map['x']) - y = paddle.static.data(name='y', - shape=[10], - dtype=type_str_map['y']) + x = paddle.static.data( + name='x', shape=[10], dtype=type_str_map['x'] + ) + y = paddle.static.data( + name='y', shape=[10], dtype=type_str_map['y'] + ) check_type(meta_data['op_str'], x, y, binary_op) def type_map_factory(): - return [{ - 'x': x_type, - 'y': y_type - } for x_type in SUPPORTED_DTYPES for y_type in SUPPORTED_DTYPES] + return [ + {'x': x_type, 'y': y_type} + for x_type in SUPPORTED_DTYPES + for y_type in SUPPORTED_DTYPES + ] class TestCPU(unittest.TestCase): - def test(self): test(self) @@ -246,7 +200,6 @@ class TestCPU(unittest.TestCase): class TestNPU(unittest.TestCase): - def test(self): test(self, True) diff --git a/python/paddle/fluid/tests/unittests/npu/test_lookup_table_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_lookup_table_v2_op_npu.py index 9efec50ddc5..0670b69851a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_lookup_table_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_lookup_table_v2_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestLookupTableV2(OpTest): - def setUp(self): self.set_npu() self.op_type = "lookup_table_v2" @@ -37,22 +36,22 @@ class TestLookupTableV2(OpTest): self.init_padding_idx() np.random.seed(SEED) w = np.random.random([self.vocab, self.dim]).astype(self.dtype) - x = np.random.randint(0, self.vocab, - size=(self.bsz, - self.seqlen)).astype(self.ids_dtype) + x = np.random.randint( + 0, self.vocab, size=(self.bsz, self.seqlen) + ).astype(self.ids_dtype) out = w[x] if self.padding_idx != -1: out[np.squeeze(x == self.padding_idx)] = np.zeros(self.dim) self.inputs = { 'W': OpTest.np_dtype_to_fluid_dtype(w), - 'Ids': OpTest.np_dtype_to_fluid_dtype(x) + 'Ids': OpTest.np_dtype_to_fluid_dtype(x), } self.attrs = { 'is_sparse': False, 'is_distributed': False, 'remote_prefetch': False, - 'padding_idx': self.padding_idx + 'padding_idx': self.padding_idx, } self.outputs = {'Out': out} @@ -78,9 +77,9 @@ class TestLookupTableV2(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['W'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['W'], 'Out', max_relative_error=0.01 + ) else: self.check_grad_with_place(self.place, ['W'], 'Out') @@ -98,7 +97,6 @@ class TestLookupTableV2FP16(TestLookupTableV2): class TestLookupTableV2Dim32(TestLookupTableV2): - def init_dims(self): self.bsz = 6 self.seqlen = 8 @@ -126,13 +124,11 @@ class TestLookupTableV2Dim32FP16(TestLookupTableV2): class TestLookupTableV2WithPadding(TestLookupTableV2): - def init_padding_idx(self): self.padding_idx = np.random.randint(0, self.vocab) class TestLookupTableV2WithPadding1(TestLookupTableV2): - def init_padding_idx(self): self.padding_idx = np.random.randint(0, self.vocab) diff --git a/python/paddle/fluid/tests/unittests/npu/test_masked_select_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_masked_select_op_npu.py index 50c9ca56f66..8943b5ba95f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_masked_select_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_masked_select_op_npu.py @@ -33,7 +33,6 @@ def np_masked_select(x, mask): class TestMaskedSelectOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -63,33 +62,29 @@ class TestMaskedSelectOp(OpTest): class TestMaskedSelectOp1(TestMaskedSelectOp): - def init(self): self.shape = (6, 8, 9, 18) class TestMaskedSelectOp2(TestMaskedSelectOp): - def init(self): - self.shape = (168, ) + self.shape = (168,) class TestMaskedSelectOpFp16(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.float16 def test_check_grad(self): x_grad = self.inputs['Mask'].astype(self.dtype) x_grad = x_grad * (1 / x_grad.sum()) - self.check_grad_with_place(self.place, ['X'], - 'Y', - user_defined_grads=[x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Y', user_defined_grads=[x_grad] + ) @skip_check_grad_ci(reason="get_numeric_gradient not support int32") class TestMaskedSelectOpInt32(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.int32 @@ -99,7 +94,6 @@ class TestMaskedSelectOpInt32(TestMaskedSelectOp): @skip_check_grad_ci(reason="get_numeric_gradient not support int64") class TestMaskedSelectOpInt64(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.int64 @@ -108,7 +102,6 @@ class TestMaskedSelectOpInt64(TestMaskedSelectOp): class TestMaskedSelectAPI(unittest.TestCase): - def test_imperative_mode(self): paddle.disable_static(paddle.NPUPlace(0)) shape = (88, 6, 8) @@ -133,27 +126,26 @@ class TestMaskedSelectAPI(unittest.TestCase): exe = paddle.static.Executor(place=paddle.NPUPlace(0)) - res = exe.run(paddle.static.default_main_program(), - feed={ - "x": np_x, - "mask": np_mask - }, - fetch_list=[out]) + res = exe.run( + paddle.static.default_main_program(), + feed={"x": np_x, "mask": np_mask}, + fetch_list=[out], + ) self.assertEqual(np.allclose(res, np_out), True) class TestMaskedSelectError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') mask = paddle.fluid.data(shape=shape, dtype='bool', name='mask') - mask_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='mask_float') + mask_float = paddle.fluid.data( + shape=shape, dtype='float32', name='mask_float' + ) np_x = np.random.random(shape).astype('float32') np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) diff --git a/python/paddle/fluid/tests/unittests/npu/test_matmul_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_matmul_op_npu.py index d00c8ab8674..9a2563ad306 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_matmul_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_matmul_op_npu.py @@ -31,7 +31,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False, scale=1.0): # transpose X and Y appropriately. if transpose_X: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -40,7 +40,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False, scale=1.0): X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -76,14 +76,15 @@ class TestMatMulOp(OpTest): X = -0.1 + 0.2 * X Y = -0.1 + 0.2 * Y - Out = reference_matmul(X, Y, self.transpose_X, self.transpose_Y, - self.alpha) + Out = reference_matmul( + X, Y, self.transpose_X, self.transpose_Y, self.alpha + ) Out = Out.astype(self.dtype) self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, 'transpose_Y': self.transpose_Y, - 'alpha': self.alpha + 'alpha': self.alpha, } self.outputs = {'Out': Out} @@ -92,8 +93,8 @@ class TestMatMulOp(OpTest): self.place = paddle.NPUPlace(0) def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.transpose_X = False self.transpose_Y = False @@ -116,7 +117,7 @@ class TestMatMulOp1(TestMatMulOp): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.transpose_X = False self.transpose_Y = True @@ -129,7 +130,7 @@ class TestMatMulOp2(TestMatMulOp): def config(self): self.x_shape = (1, 2, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.transpose_X = True self.transpose_Y = False @@ -248,7 +249,7 @@ class TestMatMulOp12(TestMatMulOp): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.transpose_X = False self.transpose_Y = False @@ -261,16 +262,14 @@ class TestMatMulOp13(TestMatMulOp): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.transpose_X = False self.transpose_Y = False -#--------------------test matmul alpha-------------------- +# --------------------test matmul alpha-------------------- def create_test_alpha_class(parent): - class TestMatMulOpAlphaCase(parent): - def init_alpha(self): self.alpha = 0.125 @@ -293,11 +292,9 @@ create_test_alpha_class(TestMatMulOp12) create_test_alpha_class(TestMatMulOp13) -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): - class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -305,9 +302,12 @@ def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): self.check_output_with_place(self.place, atol=atol) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=max_relative_error) + self.check_grad_with_place( + self.place, + ['X', 'Y'], + 'Out', + max_relative_error=max_relative_error, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/npu/test_matmulv2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_matmulv2_op_npu.py index 1d3cdaf5915..91883824cf5 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_matmulv2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_matmulv2_op_npu.py @@ -36,8 +36,8 @@ class TestMatMulV2Op(OpTest): self.place = paddle.NPUPlace(0) def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -76,7 +76,7 @@ class TestMatMulOp2(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.trans_x = False self.trans_y = True @@ -88,7 +88,7 @@ class TestMatMulOp3(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False @@ -100,7 +100,7 @@ class TestMatMulOp4(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -113,7 +113,7 @@ class TestMatMulOp5(TestMatMulV2Op): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False @@ -125,7 +125,7 @@ class TestMatMulOp6(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 102, 1) - self.y_shape = (102, ) + self.y_shape = (102,) self.trans_x = True self.trans_y = False @@ -137,7 +137,7 @@ class TestMatMulOp7(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 1, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -244,7 +244,7 @@ class TestMatMulOp16(TestMatMulV2Op): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -257,7 +257,7 @@ class TestMatMulOp17(TestMatMulV2Op): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False @@ -286,13 +286,11 @@ class TestMatMulOpBroadcast2(TestMatMulV2Op): self.trans_y = True -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): - class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -300,9 +298,12 @@ def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): self.check_output_with_place(self.place, atol=atol) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=max_relative_error) + self.check_grad_with_place( + self.place, + ['X', 'Y'], + 'Out', + max_relative_error=max_relative_error, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name @@ -329,7 +330,6 @@ create_test_fp16_class(TestMatMulOp17) class TestMatMulV2API(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if paddle.is_compiled_with_npu(): @@ -346,12 +346,11 @@ class TestMatMulV2API(unittest.TestCase): y_np = np.random.random([3, 4]).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": x_np, - "input_y": y_np - }, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np, "input_y": y_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: diff --git a/python/paddle/fluid/tests/unittests/npu/test_mean_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_mean_op_npu.py index 37e2afbb892..4041ee9a2c1 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_mean_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_mean_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestMean(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -55,7 +54,6 @@ class TestMean(OpTest): class TestMeanFP16(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_memcpy_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_memcpy_op_npu.py index 5367ea8a92b..6ed36b710ab 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_memcpy_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_memcpy_op_npu.py @@ -28,66 +28,77 @@ SEED = 2021 class TestMemcpy_FillConstant(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = Program() with program_guard(main_program): cpu_var_name = "tensor@Cpu" npu_var_name = "tensor@Npu" - cpu_var = main_program.global_block().create_var(name=cpu_var_name, - shape=[10, 10], - dtype='float32', - persistable=False, - stop_gradient=True) - npu_var = main_program.global_block().create_var(name=npu_var_name, - shape=[10, 10], - dtype='float32', - persistable=False, - stop_gradient=True) - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": npu_var_name}, - attrs={ - "shape": [10, 10], - "dtype": npu_var.dtype, - "value": 1.0, - "place_type": 4 - }) - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": cpu_var_name}, - attrs={ - "shape": [10, 10], - "dtype": cpu_var.dtype, - "value": 0.0, - "place_type": 0 - }) + cpu_var = main_program.global_block().create_var( + name=cpu_var_name, + shape=[10, 10], + dtype='float32', + persistable=False, + stop_gradient=True, + ) + npu_var = main_program.global_block().create_var( + name=npu_var_name, + shape=[10, 10], + dtype='float32', + persistable=False, + stop_gradient=True, + ) + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": npu_var_name}, + attrs={ + "shape": [10, 10], + "dtype": npu_var.dtype, + "value": 1.0, + "place_type": 4, + }, + ) + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": cpu_var_name}, + attrs={ + "shape": [10, 10], + "dtype": cpu_var.dtype, + "value": 0.0, + "place_type": 0, + }, + ) return main_program, npu_var, cpu_var def test_npu_cpoy_to_cpu(self): main_program, npu_var, cpu_var = self.get_prog() - main_program.global_block().append_op(type='memcpy', - inputs={'X': npu_var}, - outputs={'Out': cpu_var}, - attrs={'dst_place_type': 0}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': npu_var}, + outputs={'Out': cpu_var}, + attrs={'dst_place_type': 0}, + ) place = fluid.NPUPlace(0) exe = fluid.Executor(place) - npu_, cpu_ = exe.run(main_program, - feed={}, - fetch_list=[npu_var.name, cpu_var.name]) + npu_, cpu_ = exe.run( + main_program, feed={}, fetch_list=[npu_var.name, cpu_var.name] + ) np.testing.assert_allclose(npu_, cpu_) np.testing.assert_allclose(cpu_, np.ones((10, 10))) def test_cpu_cpoy_npu(self): main_program, npu_var, cpu_var = self.get_prog() - main_program.global_block().append_op(type='memcpy', - inputs={'X': cpu_var}, - outputs={'Out': npu_var}, - attrs={'dst_place_type': 4}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': cpu_var}, + outputs={'Out': npu_var}, + attrs={'dst_place_type': 4}, + ) place = fluid.NPUPlace(0) exe = fluid.Executor(place) - npu_, cpu_ = exe.run(main_program, - feed={}, - fetch_list=[npu_var.name, cpu_var.name]) + npu_, cpu_ = exe.run( + main_program, feed={}, fetch_list=[npu_var.name, cpu_var.name] + ) np.testing.assert_allclose(npu_, cpu_) np.testing.assert_allclose(npu_, np.zeros((10, 10))) diff --git a/python/paddle/fluid/tests/unittests/npu/test_merged_momentum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_merged_momentum_op_npu.py index 086911a56db..d530438bd58 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_merged_momentum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_merged_momentum_op_npu.py @@ -22,16 +22,18 @@ from paddle.fluid.layer_helper import LayerHelper from collections import OrderedDict -def run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False): +def run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -48,48 +50,70 @@ def run_momentum_op(params, } param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -99,8 +123,9 @@ def run_momentum_op(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -111,10 +136,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars[i] outputs['MasterParamOut'] = master_param_vars[i] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -126,10 +150,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -137,17 +160,19 @@ def run_momentum_op(params, return exe.run(main, feed=feed_dict, fetch_list=fetch_list) -def run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False, - use_nesterov=True): +def run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, + use_nesterov=True, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -159,48 +184,70 @@ def run_momentum_op2(params, helper = LayerHelper(op_type, **locals()) param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -210,8 +257,9 @@ def run_momentum_op2(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -230,10 +278,9 @@ def run_momentum_op2(params, 'regularization_method': 'l2_decay', 'regularization_coeff': 2.0, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -246,22 +293,18 @@ def run_momentum_op2(params, inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars attrs = { - 'mu': - mu, - 'multi_precision': - multi_precision, - 'rescale_grad': - rescale_grad, - 'use_nesterov': - use_nesterov, - 'regularization_method': - ['l2_decay' for i in range(len(param_vars))], + 'mu': mu, + 'multi_precision': multi_precision, + 'rescale_grad': rescale_grad, + 'use_nesterov': use_nesterov, + 'regularization_method': [ + 'l2_decay' for i in range(len(param_vars)) + ], 'regularization_coeff': [2.0 for i in range(len(param_vars))], } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -270,7 +313,6 @@ def run_momentum_op2(params, class TestMergedMomentum(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -296,21 +338,28 @@ class TestMergedMomentum(unittest.TestCase): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): # NPU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged) + return run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -323,7 +372,6 @@ class TestMergedMomentum(unittest.TestCase): class TestMergedMomentum2(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -349,22 +397,29 @@ class TestMergedMomentum2(unittest.TestCase): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_nesterov, use_merged): # NPU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged, - use_nesterov=use_nesterov) + return run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + use_nesterov=use_nesterov, + ) outs1 = run_op(use_nesterov=True, use_merged=True) outs2 = run_op(use_nesterov=True, use_merged=False) diff --git a/python/paddle/fluid/tests/unittests/npu/test_meshgrid_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_meshgrid_op_npu.py index 8ff2fb65aa7..8af8f899244 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_meshgrid_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_meshgrid_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestMeshgridOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "meshgrid" @@ -55,7 +54,7 @@ class TestMeshgridOp(OpTest): ins = [] outs = [] for i in range(len(self.shape)): - ins.append(np.random.random((self.shape[i], )).astype(self.dtype)) + ins.append(np.random.random((self.shape[i],)).astype(self.dtype)) for i in range(len(self.shape)): out_reshape = [1] * len(self.shape) @@ -69,43 +68,47 @@ class TestMeshgridOp(OpTest): @skip_check_grad_ci( - reason="The backward test is not supported for float16 type on NPU.") + reason="The backward test is not supported for float16 type on NPU." +) class TestMeshgridOpFP16(TestMeshgridOp): - def get_dtype(self): return "float16" class TestMeshgridOpINT32(TestMeshgridOp): - def get_dtype(self): return "int32" class TestMeshgridOpINT64(TestMeshgridOp): - def get_dtype(self): return "int64" class TestMeshgridOp2(TestMeshgridOp): - def get_x_shape(self): return [100, 300] class TestMeshgridOp3(unittest.TestCase): - def test_api(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -114,29 +117,35 @@ class TestMeshgridOp3(unittest.TestCase): exe = fluid.Executor(place=fluid.NPUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid(x, y) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) np.testing.assert_allclose(res_1, out_1) np.testing.assert_allclose(res_2, out_2) class TestMeshgridOp4(unittest.TestCase): - def test_list_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -145,29 +154,35 @@ class TestMeshgridOp4(unittest.TestCase): exe = fluid.Executor(place=fluid.NPUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid([x, y]) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) np.testing.assert_allclose(res_1, out_1) np.testing.assert_allclose(res_2, out_2) class TestMeshgridOp5(unittest.TestCase): - def test_tuple_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -176,27 +191,33 @@ class TestMeshgridOp5(unittest.TestCase): exe = fluid.Executor(place=fluid.NPUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid((x, y)) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) np.testing.assert_allclose(res_1, out_1) np.testing.assert_allclose(res_2, out_2) class TestMeshgridOp6(unittest.TestCase): - def test_api_with_dygraph(self): paddle.disable_static(paddle.NPUPlace(0)) - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_3 = np.reshape(input_3, [100, 1]) out_3 = np.broadcast_to(out_3, [100, 200]) @@ -213,15 +234,22 @@ class TestMeshgridOp6(unittest.TestCase): class TestMeshgridOp7(unittest.TestCase): - def test_api_with_dygraph_list_input(self): paddle.disable_static(paddle.NPUPlace(0)) - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_3 = np.reshape(input_3, [100, 1]) out_3 = np.broadcast_to(out_3, [100, 200]) @@ -238,15 +266,22 @@ class TestMeshgridOp7(unittest.TestCase): class TestMeshgridOp8(unittest.TestCase): - def test_api_with_dygraph_tuple_input(self): paddle.disable_static(paddle.NPUPlace(0)) - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_3 = np.reshape(input_3, [100, 1]) out_3 = np.broadcast_to(out_3, [100, 200]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_mixed_precision_npu.py b/python/paddle/fluid/tests/unittests/npu/test_mixed_precision_npu.py index 9927316fddc..3685ab9e1e1 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_mixed_precision_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_mixed_precision_npu.py @@ -28,7 +28,6 @@ paddle.enable_static() class SimpleNet(nn.Layer): - def __init__(self, input_size, output_size): super(SimpleNet, self).__init__() self.linear1 = nn.Linear(input_size, output_size) @@ -42,16 +41,15 @@ class SimpleNet(nn.Layer): x = self.linear1(x) # currently, paddle's relu may hide nan/inf, relu(nan) = 0, relu(inf)= inf # so, do not use it here. - #x = self.relu1(x) + # x = self.relu1(x) x = self.linear2(x) - #x = self.relu2(x) + # x = self.relu2(x) x = self.linear3(x) return x class AMPTestNpu(unittest.TestCase): - def setUp(self): self.place = paddle.NPUPlace(0) @@ -67,10 +65,11 @@ class AMPTestNpu(unittest.TestCase): loss = mse(out, label) opt = paddle.fluid.optimizer.Adam( - learning_rate=0.0001, parameter_list=model.parameters()) # 定义优化器 - opt = paddle.static.amp.decorate(opt, - init_loss_scaling=128.0, - use_dynamic_loss_scaling=True) + learning_rate=0.0001, parameter_list=model.parameters() + ) # 定义优化器 + opt = paddle.static.amp.decorate( + opt, init_loss_scaling=128.0, use_dynamic_loss_scaling=True + ) opt.minimize(loss) return model, loss, opt @@ -85,11 +84,17 @@ class AMPTestNpu(unittest.TestCase): model, loss, opt = self.net() weight = model.linear1.weight moment1 = opt._optimizer._get_accumulator( - opt._optimizer._moment1_acc_str, weight) + opt._optimizer._moment1_acc_str, weight + ) beta_pow1 = opt._optimizer._get_accumulator( - opt._optimizer._beta1_pow_acc_str, weight) + opt._optimizer._beta1_pow_acc_str, weight + ) fetch_list = [ - loss, weight, moment1, beta_pow1, 'find_infinite_scale.tmp_0' + loss, + weight, + moment1, + beta_pow1, + 'find_infinite_scale.tmp_0', ] exe = paddle.static.Executor(self.place) @@ -104,20 +109,24 @@ class AMPTestNpu(unittest.TestCase): ] weight_, moment1_, beta_pow1_ = exe.run( - startup_prog, fetch_list=[weight, moment1, beta_pow1]) - pre_weight_, pre_moment1_, pre_beta_pow1_ = weight_, moment1_, beta_pow1_ + startup_prog, fetch_list=[weight, moment1, beta_pow1] + ) + pre_weight_, pre_moment1_, pre_beta_pow1_ = ( + weight_, + moment1_, + beta_pow1_, + ) for i in range(nums_batch): if i % 2: train_data[i][10] = np.inf loss_, weight_, moment1_, beta_pow1_, found_inf = exe.run( main_prog, - feed={ - "X": train_data[i], - "Y": labels[i] - }, - fetch_list=fetch_list) - print(loss_, weight_[0][0], moment1_[0][0], beta_pow1_, - found_inf) + feed={"X": train_data[i], "Y": labels[i]}, + fetch_list=fetch_list, + ) + print( + loss_, weight_[0][0], moment1_[0][0], beta_pow1_, found_inf + ) if i % 2: self.assertTrue(found_inf) np.testing.assert_array_equal(weight_, pre_weight_) @@ -128,7 +137,11 @@ class AMPTestNpu(unittest.TestCase): self.assertFalse(np.array_equal(weight_, pre_weight_)) self.assertFalse(np.array_equal(moment1_, pre_moment1_)) self.assertFalse(np.array_equal(beta_pow1_, pre_beta_pow1_)) - pre_weight_, pre_moment1_, pre_beta_pow1_ = weight_, moment1_, beta_pow1_ + pre_weight_, pre_moment1_, pre_beta_pow1_ = ( + weight_, + moment1_, + beta_pow1_, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_momentum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_momentum_op_npu.py index 852f4fd3dc2..9719c5582bc 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_momentum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_momentum_op_npu.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestMomentumOp1(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -48,7 +47,7 @@ class TestMomentumOp1(OpTest): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu, 'use_nesterov': self.use_nesterov} @@ -59,7 +58,8 @@ class TestMomentumOp1(OpTest): mu=mu, velocity=velocity, use_nesterov=self.use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -75,7 +75,6 @@ class TestMomentumOp1(OpTest): class TestMomentumOpFp16(TestMomentumOp1): - def init_dtype(self): self.dtype = np.float16 @@ -84,23 +83,21 @@ class TestMomentumOpFp16(TestMomentumOp1): class TestMomentumOp2(TestMomentumOp1): - def init_case(self): self.shape = (123, 321) self.use_nesterov = True class TestMomentumV2(unittest.TestCase): - def test_momentum_dygraph(self): paddle.disable_static(place=fluid.NPUPlace(0)) value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - momentum=0.9, - parameters=linear.parameters()) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, momentum=0.9, parameters=linear.parameters() + ) out = linear(a) out.backward() adam.step() @@ -117,13 +114,15 @@ class TestMomentumV2(unittest.TestCase): cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - rms_optimizer = paddle.optimizer.Momentum(learning_rate=0.1, - momentum=0.9) + rms_optimizer = paddle.optimizer.Momentum( + learning_rate=0.1, momentum=0.9 + ) rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -131,14 +130,13 @@ class TestMomentumV2(unittest.TestCase): exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list) def test_raise_error(self): - self.assertRaises(ValueError, - paddle.optimizer.Momentum, - learning_rate=None) + self.assertRaises( + ValueError, paddle.optimizer.Momentum, learning_rate=None + ) self.assertRaises(ValueError, paddle.optimizer.Momentum, momentum=None) class TestMomentumOpWithDecay(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -164,14 +162,14 @@ class TestMomentumOpWithDecay(OpTest): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = { 'mu': mu, 'use_nesterov': use_nesterov, 'regularization_method': regularization_method, - 'regularization_coeff': regularization_coeff + 'regularization_coeff': regularization_coeff, } grad = grad + regularization_coeff * param @@ -182,7 +180,8 @@ class TestMomentumOpWithDecay(OpTest): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -195,7 +194,6 @@ class TestMomentumOpWithDecay(OpTest): class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): - def init_config(self): self.dtype = np.float16 @@ -205,13 +203,11 @@ class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): class TestMomentumOpWithDecay2(TestMomentumOpWithDecay): - def init_config(self): self.use_nesterov = False class TestMomentumOpWithDecayAPI(unittest.TestCase): - def _test_momentum_dygraph_common(self, regularization): paddle.disable_static(fluid.NPUPlace(0)) inp = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -224,13 +220,16 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): learning_rate=0.01, momentum=0.9, parameter_list=linear.parameters(), - regularization=regularization) + regularization=regularization, + ) momentum.minimize(loss) def test_momentum_dygraph_1(self): self._test_momentum_dygraph_common( regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ) + ) def test_momentum_static(self): paddle.enable_static() @@ -244,12 +243,14 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): avg_cost = paddle.mean(cost) momentum_optimizer = paddle.fluid.contrib.optimizer.Momentum( - learning_rate=0.1, momentum=0.9) + learning_rate=0.1, momentum=0.9 + ) momentum_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -258,11 +259,11 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): - def __update_params(self, momentum, linear): for i in range(10): - inp = paddle.full(shape=[2, 2], fill_value=i, - dtype='float32').astype("float32") + inp = paddle.full( + shape=[2, 2], fill_value=i, dtype='float32' + ).astype("float32") inp = paddle.to_tensor(inp) out = linear(inp) loss = paddle.mean(out) @@ -276,39 +277,45 @@ class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_old = paddle.fluid.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_old.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_old, linear=linear_old) linear_new = paddle.nn.Linear( 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_new = paddle.fluid.contrib.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_new.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_new, linear=linear_new) self.assertEqual( (linear_old.weight.numpy() == linear_new.weight.numpy()).all(), True, - 'the param weight updated by two Momentum optimizers should equal') + 'the param weight updated by two Momentum optimizers should equal', + ) def test_vs(self, place=fluid.NPUPlace(0)): self.__test_vs(place=place) class TestMomentumV2Group(TestMomentumV2): - def test_momentum_dygraph(self): paddle.disable_static(place=fluid.NPUPlace(0)) value = np.arange(26).reshape(2, 13).astype("float32") @@ -316,22 +323,20 @@ class TestMomentumV2Group(TestMomentumV2): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - 'learning_rate': - 0.1, - 'momentum': - 0.99 - }], - weight_decay=0.1, - momentum=0.9) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + }, + ], + weight_decay=0.1, + momentum=0.9, + ) out = linear_1(a) out = linear_2(out) out.backward() diff --git a/python/paddle/fluid/tests/unittests/npu/test_mul_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_mul_op_npu.py index 0b3a0c322cd..8f8abea5396 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_mul_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_mul_op_npu.py @@ -40,7 +40,7 @@ class TestMul(OpTest): np.random.seed(SEED) self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} @@ -81,9 +81,9 @@ class TestMul(OpTest): @skip_check_grad_ci( - reason="Don't support grad checking for NPU OP with FP16 data type.") + reason="Don't support grad checking for NPU OP with FP16 data type." +) class TestMulFP16(TestMul): - def init_dtype(self): self.dtype = np.float16 @@ -112,7 +112,7 @@ class TestMul2(TestMul): np.random.seed(SEED) self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.outputs = { 'Out': np.dot(self.inputs['X'].reshape(20, 10), self.inputs['Y']) @@ -120,9 +120,9 @@ class TestMul2(TestMul): @skip_check_grad_ci( - reason="Don't support grad checking for NPU OP with FP16 data type.") + reason="Don't support grad checking for NPU OP with FP16 data type." +) class TestMul2FP16(TestMul2): - def init_dtype(self): self.dtype = np.float16 @@ -152,16 +152,16 @@ class TestMul3(TestMul): np.random.seed(SEED) self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.attrs = {"x_num_col_dims": 2} self.outputs = {'Out': np.matmul(self.inputs['X'], self.inputs['Y'])} @skip_check_grad_ci( - reason="Don't support grad checking for NPU OP with FP16 data type.") + reason="Don't support grad checking for NPU OP with FP16 data type." +) class TestMul3FP16(TestMul3): - def init_dtype(self): self.dtype = np.float16 @@ -190,7 +190,7 @@ class TestMul4(TestMul): np.random.seed(SEED) self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.outputs = { 'Out': np.dot(self.inputs['X'].reshape(20, 12), self.inputs['Y']) @@ -198,9 +198,9 @@ class TestMul4(TestMul): @skip_check_grad_ci( - reason="Don't support grad checking for NPU OP with FP16 data type.") + reason="Don't support grad checking for NPU OP with FP16 data type." +) class TestMul4FP16(TestMul4): - def init_dtype(self): self.dtype = np.float16 @@ -215,7 +215,6 @@ class TestMul4FP16(TestMul4): class TestMulNet(unittest.TestCase): - def init_dtype(self): self.dtype = np.float32 @@ -237,9 +236,9 @@ class TestMulNet(unittest.TestCase): b = paddle.static.data(name="b", shape=[2, 3], dtype=self.dtype) c = paddle.static.data(name="c", shape=[3, 2], dtype=self.dtype) d = paddle.static.data(name="d", shape=[3, 2], dtype=self.dtype) - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum_1 = paddle.add(a, b) sum_2 = paddle.add(c, d) @@ -263,18 +262,23 @@ class TestMulNet(unittest.TestCase): print("TestMulNet Start run on {} . ".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -288,7 +292,6 @@ class TestMulNet(unittest.TestCase): class TestMulNet3_2(unittest.TestCase): - def init_dtype(self): self.dtype = np.float32 @@ -310,9 +313,9 @@ class TestMulNet3_2(unittest.TestCase): b = paddle.static.data(name="b", shape=[2, 3, 4], dtype=self.dtype) c = paddle.static.data(name="c", shape=[12, 5], dtype=self.dtype) d = paddle.static.data(name="d", shape=[12, 5], dtype=self.dtype) - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum_1 = paddle.add(a, b) sum_2 = paddle.add(c, d) @@ -336,18 +339,23 @@ class TestMulNet3_2(unittest.TestCase): print("testMulNet3_2 tart run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -356,13 +364,13 @@ class TestMulNet3_2(unittest.TestCase): cpu_pred, cpu_loss = self._test(False) npu_pred, npu_loss = self._test(True) - np.testing.assert_allclose(npu_pred, cpu_pred, - atol=1e-5) # atol needed on cann 20.3 + np.testing.assert_allclose( + npu_pred, cpu_pred, atol=1e-5 + ) # atol needed on cann 20.3 np.testing.assert_allclose(npu_loss, cpu_loss, atol=1e-5) class TestMulNet3_2_xc2(unittest.TestCase): - def init_dtype(self): self.dtype = np.float32 @@ -384,9 +392,9 @@ class TestMulNet3_2_xc2(unittest.TestCase): b = paddle.static.data(name="b", shape=[2, 3, 4], dtype=self.dtype) c = paddle.static.data(name="c", shape=[4, 5], dtype=self.dtype) d = paddle.static.data(name="d", shape=[4, 5], dtype=self.dtype) - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum_1 = paddle.add(a, b) sum_2 = paddle.add(c, d) @@ -411,18 +419,23 @@ class TestMulNet3_2_xc2(unittest.TestCase): print("TestMulNet3_2_xc2. Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -436,7 +449,6 @@ class TestMulNet3_2_xc2(unittest.TestCase): class TestMulNet4_2(unittest.TestCase): - def init_dtype(self): self.dtype = np.float32 @@ -458,17 +470,18 @@ class TestMulNet4_2(unittest.TestCase): b = paddle.static.data(name="b", shape=[12, 5], dtype=self.dtype) c = paddle.static.data(name="c", shape=[12, 5], dtype=self.dtype) d = paddle.static.data(name="d", shape=[12, 5], dtype=self.dtype) - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum_1 = paddle.add(a, b) # [12, 5] sum_2 = paddle.add(c, d) # [12, 5] fc_1 = fluid.layers.fc(input=sum_1, size=2) # [12, 2] fc_1_re_shape = paddle.reshape(fc_1, shape=[2, 3, 2, 2]) fc_2 = fluid.layers.fc(input=sum_2, size=2) # [12, 2] - result = paddle.fluid.layers.mul(fc_1_re_shape, - fc_2) # [2, 3, 2, 2] * [12, 2] + result = paddle.fluid.layers.mul( + fc_1_re_shape, fc_2 + ) # [2, 3, 2, 2] * [12, 2] prediction = fluid.layers.fc(input=result, size=2, act='softmax') @@ -487,18 +500,23 @@ class TestMulNet4_2(unittest.TestCase): print("testMulNet4_2 tart run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -507,8 +525,9 @@ class TestMulNet4_2(unittest.TestCase): cpu_pred, cpu_loss = self._test(False) npu_pred, npu_loss = self._test(True) - np.testing.assert_allclose(npu_pred, cpu_pred, - atol=1e-5) # atol needed on cann 20.3 + np.testing.assert_allclose( + npu_pred, cpu_pred, atol=1e-5 + ) # atol needed on cann 20.3 np.testing.assert_allclose(npu_loss, cpu_loss, atol=1e-5) diff --git a/python/paddle/fluid/tests/unittests/npu/test_multinomial_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_multinomial_op_npu.py index d1599766756..ccff4ffd0cf 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_multinomial_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_multinomial_op_npu.py @@ -47,7 +47,6 @@ def sample_output_two_dimension(out, shape): class TestMultinomialOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "multinomial" @@ -65,8 +64,9 @@ class TestMultinomialOp(OpTest): self.attrs = {"num_samples": 100000, "replacement": True} def test_check_output(self): - self.check_output_customized(self.verify_output, - custom_place=self.place) + self.check_output_customized( + self.verify_output, custom_place=self.place + ) def sample_output(self, out): return sample_output_one_dimension(out, 4) @@ -79,7 +79,6 @@ class TestMultinomialOp(OpTest): class TestMultinomialOp2(TestMultinomialOp): - def init_data(self): # input probability is a matrix self.input_np = np.random.rand(3, 4) @@ -91,7 +90,6 @@ class TestMultinomialOp2(TestMultinomialOp): class TestMultinomialOp3(TestMultinomialOp): - def init_data(self): # replacement is False. number of samples must be less than number of categories. self.input_np = np.random.rand(1000) @@ -102,12 +100,13 @@ class TestMultinomialOp3(TestMultinomialOp): out = np.array(outs[0]) unique_out = np.unique(out) self.assertEqual( - len(unique_out), 100, - "replacement is False. categories can't be sampled repeatedly") + len(unique_out), + 100, + "replacement is False. categories can't be sampled repeatedly", + ) class TestMultinomialApi(unittest.TestCase): - def test_dygraph(self): # input probability is a vector, and replacement is True paddle.set_device('npu:0') @@ -144,8 +143,10 @@ class TestMultinomialApi(unittest.TestCase): unique_out = np.unique(out.numpy()) self.assertEqual( - len(unique_out), 100, - "replacement is False. categories can't be sampled repeatedly") + len(unique_out), + 100, + "replacement is False. categories can't be sampled repeatedly", + ) paddle.enable_static() def test_dygraph4(self): @@ -178,19 +179,17 @@ class TestMultinomialApi(unittest.TestCase): class TestMultinomialAlias(unittest.TestCase): - def test_alias(self): paddle.set_device('npu:0') x = paddle.rand([4]) out1 = paddle.multinomial(x, num_samples=10, replacement=True) out2 = paddle.tensor.multinomial(x, num_samples=10, replacement=True) - out3 = paddle.tensor.random.multinomial(x, - num_samples=10, - replacement=True) + out3 = paddle.tensor.random.multinomial( + x, num_samples=10, replacement=True + ) class TestMultinomialError(unittest.TestCase): - def setUp(self): paddle.set_device('npu:0') paddle.disable_static() @@ -199,7 +198,6 @@ class TestMultinomialError(unittest.TestCase): paddle.enable_static() def test_num_sample(self): - def test_num_sample_less_than_0(): x = paddle.rand([4]) out = paddle.multinomial(x, num_samples=-2) @@ -207,7 +205,6 @@ class TestMultinomialError(unittest.TestCase): self.assertRaises(ValueError, test_num_sample_less_than_0) def test_input_probs_dim(self): - def test_dim_larger_than_2(): x = paddle.rand([2, 3, 3]) out = paddle.multinomial(x) diff --git a/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_op_npu.py index 9c9d50e1dda..82a2ff48251 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_op_npu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestNearestInterpOp(OpTest): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -52,10 +51,15 @@ class TestNearestInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -67,7 +71,7 @@ class TestNearestInterpOp(OpTest): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -83,124 +87,113 @@ class TestNearestInterpOp(OpTest): self.input_shape = [2, 3, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = False def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - in_place=True, - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', in_place=True, max_relative_error=0.006 + ) class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") self.align_corners = False class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 4, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 8]).astype("int32") self.align_corners = False self.data_layout = "NHWC" class TestNearestInterpOpUint8(OpTest): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -209,8 +202,9 @@ class TestNearestInterpOpUint8(OpTest): self.actual_shape = None self.init_test_case() self.op_type = "nearest_interp" - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale > 0: out_h = int(self.input_shape[2] * self.scale) @@ -219,9 +213,14 @@ class TestNearestInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -230,7 +229,7 @@ class TestNearestInterpOpUint8(OpTest): 'out_w': self.out_w, 'scale': self.scale, 'interp_method': self.interp_method, - 'align_corners': self.align_corners + 'align_corners': self.align_corners, } self.outputs = {'Out': output_np} @@ -242,47 +241,43 @@ class TestNearestInterpOpUint8(OpTest): self.input_shape = [1, 3, 9, 6] self.out_h = 10 self.out_w = 9 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase1Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 80 self.out_w = 40 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase2Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 5 self.out_w = 13 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 15]).astype("int32") self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -294,19 +289,17 @@ class TestNearestNeighborInterpScale2(TestNearestInterpOp): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 1. + self.scale = 1.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False class TestNearestInterpOp_attr_tensor(OpTest): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -340,15 +333,21 @@ class TestNearestInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h self.attrs['out_w'] = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -362,33 +361,31 @@ class TestNearestInterpOp_attr_tensor(OpTest): self.input_shape = [2, 5, 4, 4] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = False # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = False # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False self.shape_by_1Dtensor = True @@ -396,7 +393,6 @@ class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -409,7 +405,6 @@ class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): class TestNearestAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 6, 3], dtype="float32") @@ -417,27 +412,25 @@ class TestNearestAPI(unittest.TestCase): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = fluid.layers.resize_nearest(y, - out_shape=[12, 12], - data_format='NHWC', - align_corners=False) - out2 = fluid.layers.resize_nearest(x, - out_shape=[12, dim], - align_corners=False) - out3 = fluid.layers.resize_nearest(x, - out_shape=shape_tensor, - align_corners=False) - out4 = fluid.layers.resize_nearest(x, - out_shape=[4, 4], - actual_shape=actual_size, - align_corners=False) - out5 = fluid.layers.resize_nearest(x, - scale=scale_tensor, - align_corners=False) + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = fluid.layers.resize_nearest( + y, out_shape=[12, 12], data_format='NHWC', align_corners=False + ) + out2 = fluid.layers.resize_nearest( + x, out_shape=[12, dim], align_corners=False + ) + out3 = fluid.layers.resize_nearest( + x, out_shape=shape_tensor, align_corners=False + ) + out4 = fluid.layers.resize_nearest( + x, out_shape=[4, 4], actual_shape=actual_size, align_corners=False + ) + out5 = fluid.layers.resize_nearest( + x, scale=scale_tensor, align_corners=False + ) x_data = np.random.random((2, 3, 6, 6)).astype("float32") dim_data = np.array([12]).astype("int32") @@ -448,38 +441,39 @@ class TestNearestAPI(unittest.TestCase): place = paddle.NPUPlace(0) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = nearest_neighbor_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 1))) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = nearest_neighbor_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 1)) + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res) class TestNearestInterpException(unittest.TestCase): - def test_exception(self): input = fluid.data(name="input", shape=[1, 3, 6, 6], dtype="float32") def attr_data_format(): # for 4-D input, data_format can only be NCHW or NHWC - out = fluid.layers.resize_nearest(input, - out_shape=[4, 8], - data_format='NDHWC') + out = fluid.layers.resize_nearest( + input, out_shape=[4, 8], data_format='NDHWC' + ) def attr_scale_type(): out = fluid.layers.resize_nearest(input, scale='scale') diff --git a/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_v2_op_npu.py index 6c4b6f97b03..4e0b6089835 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_v2_op_npu.py @@ -30,7 +30,6 @@ paddle.enable_static() class TestNearestInterpOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -68,11 +67,17 @@ class TestNearestInterpOp(OpTest): output_h = self.out_h output_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, output_h, output_w, - scale_h, scale_w, self.out_size, - self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp_np( + input_np, + output_h, + output_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -83,7 +88,7 @@ class TestNearestInterpOp(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -99,15 +104,17 @@ class TestNearestInterpOp(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - in_place=True, - max_relative_error=0.02) + self.check_grad_with_place( + self.place, ['X'], 'Out', in_place=True, max_relative_error=0.02 + ) else: - self.check_grad_with_place(self.place, ['X'], - 'Out', - in_place=True, - max_relative_error=0.006) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + in_place=True, + max_relative_error=0.006, + ) def init_dtype(self): self.dtype = np.float32 @@ -117,123 +124,112 @@ class TestNearestInterpOp(OpTest): self.input_shape = [2, 3, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = False class TestNearestNeighborInterpFP16(TestNearestInterpOp): - def init_dtype(self): self.dtype = np.float16 class TestNearestNeighborInterpCase1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") self.align_corners = False class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = None self.align_corners = False class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -245,7 +241,6 @@ class TestNearestNeighborInterpScale2(TestNearestInterpOp): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] @@ -257,7 +252,6 @@ class TestNearestNeighborInterpScale3(TestNearestInterpOp): class TestNearestInterpOp_attr_tensor(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -300,8 +294,9 @@ class TestNearestInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -313,9 +308,16 @@ class TestNearestInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -329,33 +331,31 @@ class TestNearestInterpOp_attr_tensor(OpTest): self.input_shape = [2, 5, 4, 4] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = False # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = False # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False self.shape_by_1Dtensor = True @@ -363,7 +363,6 @@ class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -376,9 +375,9 @@ class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): class TestNearestInterpOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_npu(): place = core.NPUPlace(0) else: @@ -388,14 +387,15 @@ class TestNearestInterpOpAPI_dy(unittest.TestCase): scale_np = np.array([2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = nearest_neighbor_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="nearest", - align_corners=False) + expect_res = nearest_neighbor_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="nearest", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res) diff --git a/python/paddle/fluid/tests/unittests/npu/test_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_norm_op_npu.py index e0a62d5641a..c59daf979fb 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_norm_op_npu.py @@ -24,7 +24,6 @@ from paddle.fluid.tests.unittests.test_norm_op import l2_norm class TestNPUNormOp(OpTest): - def setUp(self): paddle.enable_static() self.set_npu() @@ -54,13 +53,12 @@ class TestNPUNormOp(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.006 + ) class TestNPUNormOp2(TestNPUNormOp): - def init_test_case(self): self.shape = [5, 3, 9, 7] self.axis = 0 @@ -68,17 +66,17 @@ class TestNPUNormOp2(TestNPUNormOp): class TestNPUNormOp3(TestNPUNormOp): - def init_test_case(self): self.shape = [5, 3, 2, 7] self.axis = -1 self.epsilon = 1e-8 -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestNPUNormOp4(TestNPUNormOp): - def init_test_case(self): self.shape = [128, 1024, 14, 14] self.axis = 2 @@ -88,10 +86,11 @@ class TestNPUNormOp4(TestNPUNormOp): pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestNPUNormOp5(TestNPUNormOp): - def init_test_case(self): self.shape = [2048, 2048] self.axis = 1 @@ -102,7 +101,6 @@ class TestNPUNormOp5(TestNPUNormOp): class API_NormTest(unittest.TestCase): - def test_errors(self): paddle.enable_static() with fluid.program_guard(fluid.Program()): @@ -115,7 +113,6 @@ class API_NormTest(unittest.TestCase): class TestNPUNormOpFP16(TestNPUNormOp): - def set_npu(self): self.__class__.use_npu = True self.__class__.no_need_check_grad = True diff --git a/python/paddle/fluid/tests/unittests/npu/test_npu_place.py b/python/paddle/fluid/tests/unittests/npu/test_npu_place.py index 0075fa49e04..8f4a9ee03e3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_npu_place.py +++ b/python/paddle/fluid/tests/unittests/npu/test_npu_place.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestNpuPlace(unittest.TestCase): - def test(self): p = core.Place() p.set_place(paddle.NPUPlace(0)) @@ -31,7 +30,6 @@ class TestNpuPlace(unittest.TestCase): class TestNpuPlaceError(unittest.TestCase): - def test_static(self): # NPU is not supported in ParallelExecutor prog = paddle.static.Program() @@ -48,8 +46,9 @@ class TestNpuPlaceError(unittest.TestCase): place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - with self.assertRaisesRegex(RuntimeError, - "NPU is not supported in ParallelExecutor"): + with self.assertRaisesRegex( + RuntimeError, "NPU is not supported in ParallelExecutor" + ): exe.run(compiled_prog, feed={"x": x_np, "y": y_np}) diff --git a/python/paddle/fluid/tests/unittests/npu/test_one_hot_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_one_hot_op_npu.py index 5379e9505df..008645deca3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_one_hot_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_one_hot_op_npu.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestOneHotOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -42,8 +41,9 @@ class TestOneHotOp(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -57,7 +57,6 @@ class TestOneHotOp(OpTest): class TestOneHotOp_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -70,8 +69,9 @@ class TestOneHotOp_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -85,7 +85,6 @@ class TestOneHotOp_attr(OpTest): class TestOneHotOp_default_dtype(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -99,8 +98,9 @@ class TestOneHotOp_default_dtype(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -114,7 +114,6 @@ class TestOneHotOp_default_dtype(OpTest): class TestOneHotOp_default_dtype_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -127,8 +126,9 @@ class TestOneHotOp_default_dtype_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -142,7 +142,6 @@ class TestOneHotOp_default_dtype_attr(OpTest): class TestOneHotOp_out_of_range(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -154,8 +153,9 @@ class TestOneHotOp_out_of_range(OpTest): x = [np.random.choice([-1, depth]) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) self.inputs = {'X': (x, x_lod)} self.attrs = {'depth': depth, 'allow_out_of_range': True} @@ -166,7 +166,6 @@ class TestOneHotOp_out_of_range(OpTest): class TestOneHotOp_dtype_int64(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -179,8 +178,9 @@ class TestOneHotOp_dtype_int64(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int64').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 diff --git a/python/paddle/fluid/tests/unittests/npu/test_one_hot_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_one_hot_v2_op_npu.py index b3fd2e69221..4ccd33134be 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_one_hot_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_one_hot_v2_op_npu.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestOneHotOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -56,7 +55,6 @@ class TestOneHotOp(OpTest): class TestOneHotOp_non_lod(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -80,7 +78,6 @@ class TestOneHotOp_non_lod(OpTest): class TestOneHotOp_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -93,8 +90,9 @@ class TestOneHotOp_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -108,7 +106,6 @@ class TestOneHotOp_attr(OpTest): class TestOneHotOp_default_dtype(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -136,7 +133,6 @@ class TestOneHotOp_default_dtype(OpTest): class TestOneHotOp_default_dtype_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -149,8 +145,9 @@ class TestOneHotOp_default_dtype_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -164,7 +161,6 @@ class TestOneHotOp_default_dtype_attr(OpTest): class TestOneHotOp_out_of_range(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -187,7 +183,6 @@ class TestOneHotOp_out_of_range(OpTest): class TestOneHotOp_dtype_int64(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -210,7 +205,6 @@ class TestOneHotOp_dtype_int64(OpTest): class TestOneHotOpApi(unittest.TestCase): - def test_api(self): depth = 10 self._run(depth) @@ -221,27 +215,32 @@ class TestOneHotOpApi(unittest.TestCase): def test_api_with_dygraph(self): depth = 10 - label = np.array([np.random.randint(0, depth - 1) - for i in range(6)]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, depth - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(paddle.NPUPlace(0)): one_hot_label = fluid.one_hot( - input=fluid.dygraph.to_variable(label), depth=depth) + input=fluid.dygraph.to_variable(label), depth=depth + ) def _run(self, depth): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = fluid.one_hot(input=label, depth=depth) place = fluid.NPUPlace(0) - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_p_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_p_norm_op_npu.py index 5560b8bbd14..8cb1d0b11a8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_p_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_p_norm_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestPnormOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -41,7 +40,7 @@ class TestPnormOp(OpTest): 'epsilon': self.epsilon, 'axis': self.axis, 'keepdim': self.keepdim, - 'porder': float(self.porder) + 'porder': float(self.porder), } self.outputs = {'Out': norm} self.gradient = self.calc_gradient() @@ -53,9 +52,9 @@ class TestPnormOp(OpTest): self.check_output_with_place(paddle.NPUPlace(0)) def test_check_grad(self): - self.check_grad_with_place(paddle.NPUPlace(0), ['X'], - 'Out', - user_defined_grads=self.gradient) + self.check_grad_with_place( + paddle.NPUPlace(0), ['X'], 'Out', user_defined_grads=self.gradient + ) def init_test_case(self): self.shape = [2, 3, 4, 5] @@ -73,7 +72,7 @@ class TestPnormOp(OpTest): 'epsilon': self.epsilon, 'axis': self.axis, 'keepdim': self.keepdim, - 'porder': float(self.porder) + 'porder': float(self.porder), } x = self.inputs["X"] porder = self.attrs["porder"] @@ -87,8 +86,11 @@ class TestPnormOp(OpTest): grad[x_abs != norm] = 0.0 else: norm = p_norm(x, axis=axis, porder=porder, keepdims=True) - grad = np.power(norm, 1 - porder) * np.power( - np.abs(x), porder - 1) * np.sign(x) + grad = ( + np.power(norm, 1 - porder) + * np.power(np.abs(x), porder - 1) + * np.sign(x) + ) numel = 1 for s in x.shape: @@ -98,7 +100,6 @@ class TestPnormOp(OpTest): class TestPnormOp2(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -109,7 +110,6 @@ class TestPnormOp2(TestPnormOp): class TestPnormOp3(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -120,7 +120,6 @@ class TestPnormOp3(TestPnormOp): class TestPnormOp4(TestPnormOp3): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -131,7 +130,6 @@ class TestPnormOp4(TestPnormOp3): class TestPnormOp5(TestPnormOp3): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -142,7 +140,6 @@ class TestPnormOp5(TestPnormOp3): class TestPnormOp6(TestPnormOp3): - def init_test_case(self): self.shape = [2, 3, 4, 5] self.axis = 1 @@ -153,31 +150,26 @@ class TestPnormOp6(TestPnormOp3): class TestPnormOpfp16(TestPnormOp): - def init_dtype(self): self.dtype = "float16" class TestPnormOp2fp16(TestPnormOp2): - def init_dtype(self): self.dtype = "float16" class TestPnormOp3fp16(TestPnormOp3): - def init_dtype(self): self.dtype = "float16" class TestPnormOp4fp16(TestPnormOp4): - def init_dtype(self): self.dtype = "float16" class TestPnormOp5fp16(TestPnormOp5): - def init_dtype(self): self.dtype = "float16" diff --git a/python/paddle/fluid/tests/unittests/npu/test_pad3d_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_pad3d_op_npu.py index f388e66881c..5d0d25e1f20 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_pad3d_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_pad3d_op_npu.py @@ -27,7 +27,6 @@ import paddle.fluid as fluid class TestPad3dNPUOp(op_test.OpTest): - def setUp(self): paddle.enable_static() self.__class__.use_npu = True @@ -39,16 +38,18 @@ class TestPad3dNPUOp(op_test.OpTest): self.variable_paddings = False self.initTestCase() - self.value = 0 #Asend npu only support constant_values = 0 right now. + self.value = 0 # Asend npu only support constant_values = 0 right now. self.inputs = {'X': np.random.random(self.shape).astype(self.x_type)} self.attrs = {} if self.variable_paddings: self.attrs['paddings'] = [] - self.inputs['Paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.inputs['Paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) else: - self.attrs['paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.attrs['paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) self.attrs['value'] = self.value self.attrs['mode'] = self.mode self.attrs['data_format'] = self.data_format @@ -69,10 +70,12 @@ class TestPad3dNPUOp(op_test.OpTest): (0, 0), ] - out = np.pad(self.inputs['X'], - paddings, - mode=self.mode, - constant_values=self.value) + out = np.pad( + self.inputs['X'], + paddings, + mode=self.mode, + constant_values=self.value, + ) self.outputs = {'Out': out} @@ -89,7 +92,6 @@ class TestPad3dNPUOp(op_test.OpTest): class TestCase1(TestPad3dNPUOp): - def initTestCase(self): self.shape = (3, 4, 5, 6, 7) self.paddings = [0, 1, 2, 3, 4, 5] @@ -102,7 +104,6 @@ class TestCase1(TestPad3dNPUOp): class TestCase2(TestPad3dNPUOp): - def initTestCase(self): self.shape = (4, 5, 6, 7, 8) self.paddings = [1, 1, 1, 1, 1, 1] @@ -111,13 +112,9 @@ class TestCase2(TestPad3dNPUOp): class TestPadAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0, - data_format="NCDHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0, data_format="NCDHW" + ): if mode == "constant" and len(pad) == len(input_data.shape) * 2: pad = np.reshape(pad, (-1, 2)).tolist() elif data_format == "NCDHW": @@ -168,8 +165,11 @@ class TestPadAPI(unittest.TestCase): def test_static(self): paddle.enable_static() - self.place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + self.place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) with program_guard(Program(), Program()): input_shape = (1, 2, 3, 4, 5) pad = [1, 2, 1, 1, 3, 4] @@ -177,31 +177,25 @@ class TestPadAPI(unittest.TestCase): value = 0 input_data = np.random.rand(*input_shape).astype(np.float32) x = paddle.fluid.data(name="x", shape=input_shape) - result1 = F.pad(x=x, - pad=pad, - value=value, - mode=mode, - data_format="NCDHW") - result2 = F.pad(x=x, - pad=pad, - value=value, - mode=mode, - data_format="NDHWC") + result1 = F.pad( + x=x, pad=pad, value=value, mode=mode, data_format="NCDHW" + ) + result2 = F.pad( + x=x, pad=pad, value=value, mode=mode, data_format="NDHWC" + ) exe = Executor(self.place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result1, result2]) - - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NDHWC") + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result1, result2], + ) + + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NDHWC" + ) np.testing.assert_allclose(fetches[0], np_out1) np.testing.assert_allclose(fetches[1], np_out2) @@ -216,27 +210,19 @@ class TestPadAPI(unittest.TestCase): input_data = np.random.rand(*input_shape).astype(np.float32) tensor_data = paddle.to_tensor(input_data) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NDHWC") - - y1 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NCDHW") - y2 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NDHWC") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NDHWC" + ) + + y1 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NCDHW" + ) + y2 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NDHWC" + ) np.testing.assert_allclose(y1.numpy(), np_out1) np.testing.assert_allclose(y2.numpy(), np_out2) @@ -252,27 +238,19 @@ class TestPadAPI(unittest.TestCase): input_data = np.random.rand(*input_shape).astype(np.float32) tensor_data = paddle.to_tensor(input_data) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NHWC") - - y1 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NCHW") - y2 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NHWC") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NHWC" + ) + + y1 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NCHW" + ) + y2 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NHWC" + ) np.testing.assert_allclose(y1.numpy(), np_out1) np.testing.assert_allclose(y2.numpy(), np_out2) @@ -288,40 +266,28 @@ class TestPadAPI(unittest.TestCase): input_data = np.random.rand(*input_shape).astype(np.float32) tensor_data = paddle.to_tensor(input_data) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCL") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NLC") - - y1 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NCL") - y2 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NLC") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCL" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NLC" + ) + + y1 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NCL" + ) + y2 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NLC" + ) np.testing.assert_allclose(y1.numpy(), np_out1) np.testing.assert_allclose(y2.numpy(), np_out2) class TestPad1dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCL"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCL" + ): if data_format == "NCL": pad = [ (0, 0), @@ -348,36 +314,33 @@ class TestPad1dAPI(unittest.TestCase): input_data = np.random.rand(*input_shape).astype(np.float32) pad_constant = nn.Pad1D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad1D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad1D( + padding=pad_int, mode="constant", value=value + ) data = paddle.to_tensor(input_data) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 2, - "constant", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 2, + "constant", + value=value, + data_format="NCL", + ) np.testing.assert_allclose(output.numpy(), np_out) class TestPad2dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCHW" + ): if data_format == "NCHW": pad = [ (0, 0), @@ -406,36 +369,33 @@ class TestPad2dAPI(unittest.TestCase): input_data = np.random.rand(*input_shape).astype(np.float32) pad_constant = nn.Pad2D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad2D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad2D( + padding=pad_int, mode="constant", value=value + ) data = paddle.to_tensor(input_data) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 4, - "constant", - value=value, - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 4, + "constant", + value=value, + data_format="NCHW", + ) np.testing.assert_allclose(output.numpy(), np_out) class TestPad3dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCDHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCDHW" + ): if data_format == "NCDHW": pad = [ (0, 0), @@ -466,32 +426,31 @@ class TestPad3dAPI(unittest.TestCase): input_data = np.random.rand(*input_shape).astype(np.float32) pad_constant = nn.Pad3D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad3D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad3D( + padding=pad_int, mode="constant", value=value + ) data = paddle.to_tensor(input_data) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 6, - "constant", - value=value, - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 6, + "constant", + value=value, + data_format="NCDHW", + ) np.testing.assert_allclose(output.numpy(), np_out) class TestPad3dOpNpuError(unittest.TestCase): - def test_errors(self): - def test_value(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) @@ -538,14 +497,16 @@ class TestPad3dOpNpuError(unittest.TestCase): class TestPadDataformatError(unittest.TestCase): - def test_errors(self): - def test_ncl(): input_shape = (1, 2, 3, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) my_pad = nn.Pad1D(padding=pad, mode="replicate", data_format="NCL") data = paddle.to_tensor(data) result = my_pad(data) @@ -553,8 +514,12 @@ class TestPadDataformatError(unittest.TestCase): def test_nchw(): input_shape = (1, 2, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) my_pad = nn.Pad1D(padding=pad, mode="replicate", data_format="NCHW") data = paddle.to_tensor(data) result = my_pad(data) @@ -562,11 +527,15 @@ class TestPadDataformatError(unittest.TestCase): def test_ncdhw(): input_shape = (1, 2, 3, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 - my_pad = nn.Pad1D(padding=pad, - mode="replicate", - data_format="NCDHW") + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) + my_pad = nn.Pad1D( + padding=pad, mode="replicate", data_format="NCDHW" + ) data = paddle.to_tensor(data) result = my_pad(data) diff --git a/python/paddle/fluid/tests/unittests/npu/test_pad_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_pad_op_npu.py index 7b16b19f214..97d2a832601 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_pad_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_pad_op_npu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestPadOp(OpTest): - def setUp(self): self.op_type = "pad" self.set_npu() @@ -41,11 +40,12 @@ class TestPadOp(OpTest): self.attrs['paddings'] = np.array(self.paddings).flatten() self.attrs['pad_value'] = self.pad_value self.outputs = { - 'Out': - np.pad(self.inputs['X'], - self.paddings, - mode='constant', - constant_values=self.pad_value) + 'Out': np.pad( + self.inputs['X'], + self.paddings, + mode='constant', + constant_values=self.pad_value, + ) } def test_check_output(self): @@ -53,9 +53,9 @@ class TestPadOp(OpTest): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.6) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.6 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') @@ -73,7 +73,6 @@ class TestPadOp(OpTest): class TestCase1(TestPadOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [(0, 1), (2, 3), (2, 1), (1, 1)] @@ -81,7 +80,6 @@ class TestCase1(TestPadOp): class TestCase2(TestPadOp): - def initTestCase(self): self.shape = (5, 5, 5) self.paddings = [(0, 0), (0, 0), (1, 2)] @@ -89,20 +87,17 @@ class TestCase2(TestPadOp): class TestCase3(TestPadOp): - def initTestCase(self): - self.shape = (100) + self.shape = 100 self.paddings = [(0, 1)] self.pad_value = 0.0 -#----------------Pad Fp16---------------- +# ----------------Pad Fp16---------------- def create_test_fp16(parent): - class TestPadFp16(parent): - def init_dtype(self): self.dtype = np.float16 @@ -118,7 +113,6 @@ create_test_fp16(TestCase3) class TestPadOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((2, 2)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_parallel_dygraph_mnist_npu.py b/python/paddle/fluid/tests/unittests/npu/test_parallel_dygraph_mnist_npu.py index a2be1ccfa67..0eb98ff7a68 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_parallel_dygraph_mnist_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_parallel_dygraph_mnist_npu.py @@ -46,23 +46,20 @@ rank_table_file = b"""{ }""" need_envs = { - "ASCEND_AICPU_PATH": - os.getenv("ASCEND_AICPU_PATH", "/usr/local/Ascend/nnae/latest"), - "ASCEND_OPP_PATH": - os.getenv("ASCEND_OPP_PATH", "/usr/local/Ascend/nnae/latest/opp"), - "HCCL_CONNECT_TIMEOUT": - "7200", - "HCCL_WHITELIST_DISABLE": - "1", - "HCCL_SECURITY_MODE": - "1", - "RANK_TABLE_FILE": - "rank_table_file.json", + "ASCEND_AICPU_PATH": os.getenv( + "ASCEND_AICPU_PATH", "/usr/local/Ascend/nnae/latest" + ), + "ASCEND_OPP_PATH": os.getenv( + "ASCEND_OPP_PATH", "/usr/local/Ascend/nnae/latest/opp" + ), + "HCCL_CONNECT_TIMEOUT": "7200", + "HCCL_WHITELIST_DISABLE": "1", + "HCCL_SECURITY_MODE": "1", + "RANK_TABLE_FILE": "rank_table_file.json", } class TestParallelDygraphMnistNPU(TestDistBase): - def _setup_config(self): self._sync_mode = False self._hccl_mode = True @@ -78,11 +75,11 @@ class TestParallelDygraphMnistNPU(TestDistBase): delta=1e-3, check_error_log=True, need_envs=need_envs, - log_name=flag_name) + log_name=flag_name, + ) class TestFleetDygraphMnistNPU(TestParallelDygraphMnistNPU): - def _setup_config(self): self._sync_mode = False self._hccl_mode = True diff --git a/python/paddle/fluid/tests/unittests/npu/test_pool2d_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_pool2d_op_npu.py index cb554d3c1d4..3abe452b4e5 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_pool2d_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_pool2d_op_npu.py @@ -22,16 +22,20 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core from op_test import OpTest -from test_pool2d_op import pool2D_forward_naive, avg_pool2D_forward_naive, max_pool2D_forward_naive, adaptive_start_index, adaptive_end_index +from test_pool2d_op import ( + pool2D_forward_naive, + avg_pool2D_forward_naive, + max_pool2D_forward_naive, + adaptive_start_index, + adaptive_end_index, +) from paddle.nn.functional import avg_pool2d, max_pool2d paddle.enable_static() def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.paddings = [0, 0] self.padding_algorithm = "SAME" @@ -42,9 +46,7 @@ def create_test_padding_SAME_class(parent): def create_test_use_ceil_class(parent): - class TestPool2DUseCeilCase(parent): - def init_ceil_mode(self): self.ceil_mode = True @@ -54,9 +56,7 @@ def create_test_use_ceil_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.paddings = [1, 1] self.padding_algorithm = "VALID" @@ -67,9 +67,7 @@ def create_test_padding_VALID_class(parent): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = False self.dtype = np.float16 @@ -79,25 +77,29 @@ def create_test_fp16_class(parent): globals()[cls_name] = TestFp16Case -def pool2d_backward_navie(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCHW', - pool_type="max", - padding_algorithm="EXPLICIT"): +def pool2d_backward_navie( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCHW', + pool_type="max", + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -107,9 +109,10 @@ def pool2d_backward_navie(x, if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0] @@ -117,7 +120,8 @@ def pool2d_backward_navie(x, raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCHW": @@ -146,10 +150,20 @@ def pool2d_backward_navie(x, if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 - W_out = (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + H_out = ( + (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + ) x_grad = np.zeros_like(x) for i in range(H_out): @@ -177,20 +191,26 @@ def pool2d_backward_navie(x, in_w_end = np.min((in_w_end, W)) if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) - x_grad[:, :, in_h_start:in_h_end, - in_w_start:in_w_end] += 1 / field_size + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) + x_grad[:, :, in_h_start:in_h_end, in_w_start:in_w_end] += ( + 1 / field_size + ) elif pool_type == 'max': for n in range(N): for c in range(C): - idx = np.argmax(x[n, c, in_h_start:in_h_end, - in_w_start:in_w_end].flatten()) + idx = np.argmax( + x[ + n, c, in_h_start:in_h_end, in_w_start:in_w_end + ].flatten() + ) idx_h = idx // (in_w_end - in_w_start) idx_w = idx % (in_w_end - in_w_start) - x_grad[n, c, in_h_start + idx_h, - in_w_start + idx_w] += 1 + x_grad[ + n, c, in_h_start + idx_h, in_w_start + idx_w + ] += 1 if data_format == "NHWC": x_grad = x_grad.transpose([0, 2, 3, 1]) @@ -198,7 +218,6 @@ def pool2d_backward_navie(x, class TestPool2D_Op(OpTest): - def setUp(self): self.set_npu() self.op_type = "pool2d" @@ -218,14 +237,24 @@ class TestPool2D_Op(OpTest): input = np.random.random(self.shape).astype(self.dtype) if self.pool_type == "max": - input = np.array([x for x in range(np.prod(self.shape)) - ]).reshape(self.shape).astype(self.dtype) - output = pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, self.data_format, - self.pool_type, - self.padding_algorithm).astype(self.dtype) + input = ( + np.array([x for x in range(np.prod(self.shape))]) + .reshape(self.shape) + .astype(self.dtype) + ) + output = pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} self.attrs = { @@ -288,27 +317,30 @@ class TestPool2D_Op(OpTest): self.check_output_with_place(fluid.NPUPlace(0), atol=1e-3) def test_check_grad(self): - x_grad = pool2d_backward_navie(self.inputs["X"], - ksize=self.ksize, - strides=self.strides, - paddings=self.paddings, - global_pool=self.global_pool, - ceil_mode=False, - exclusive=self.exclusive, - adaptive=self.adaptive, - data_format=self.data_format, - pool_type=self.pool_type, - padding_algorithm=self.padding_algorithm) + x_grad = pool2d_backward_navie( + self.inputs["X"], + ksize=self.ksize, + strides=self.strides, + paddings=self.paddings, + global_pool=self.global_pool, + ceil_mode=False, + exclusive=self.exclusive, + adaptive=self.adaptive, + data_format=self.data_format, + pool_type=self.pool_type, + padding_algorithm=self.padding_algorithm, + ) x_grad = x_grad / np.prod(self.outputs['Out'].shape) - self.check_grad_with_place(fluid.NPUPlace(0), - set(['X']), - 'Out', - max_relative_error=0.06, - user_defined_grads=[x_grad]) + self.check_grad_with_place( + fluid.NPUPlace(0), + set(['X']), + 'Out', + max_relative_error=0.06, + user_defined_grads=[x_grad], + ) class TestCase1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -328,7 +360,6 @@ class TestCase1(TestPool2D_Op): class TestCase2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -348,34 +379,29 @@ class TestCase2(TestPool2D_Op): class TestCase3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestAvgInclude(TestCase2): - def init_exclusive(self): self.exclusive = False class TestAvgPoolAdaptive(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -389,7 +415,6 @@ class TestAvgPoolAdaptive(TestCase1): class TestAvgPoolAdaptiveAsyOutSize(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -404,9 +429,8 @@ class TestAvgPoolAdaptiveAsyOutSize(TestCase1): self.paddings = [0, 0, 0, 0] -#-------test pool2d with asymmetric padding----- +# -------test pool2d with asymmetric padding----- class TestPool2D_AsyPadding(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -417,7 +441,6 @@ class TestPool2D_AsyPadding(TestPool2D_Op): class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -428,7 +451,6 @@ class TestCase1_AsyPadding(TestCase1): class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -439,7 +461,6 @@ class TestCase2_AsyPadding(TestCase2): class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -450,7 +471,6 @@ class TestCase3_AsyPadding(TestCase3): class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -461,7 +481,6 @@ class TestCase4_AsyPadding(TestCase4): class TestCase5_AsyPadding((TestCase5)): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -472,7 +491,6 @@ class TestCase5_AsyPadding((TestCase5)): class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -486,7 +504,6 @@ class TestAvgInclude_AsyPadding(TestCase2): class TestAvgPoolAdaptive_AsyPadding(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -499,9 +516,8 @@ class TestAvgPoolAdaptive_AsyPadding(TestCase1): self.shape = [2, 3, 8, 8] -#----------- test channel_last -------------- +# ----------- test channel_last -------------- class TestPool2D_channel_last(TestPool2D_Op): - def init_data_format(self): self.data_format = "NHWC" @@ -510,7 +526,6 @@ class TestPool2D_channel_last(TestPool2D_Op): class TestCase1_channel_last(TestCase1): - def init_data_format(self): self.data_format = "NHWC" @@ -519,7 +534,6 @@ class TestCase1_channel_last(TestCase1): class TestCase2_channel_last(TestCase2): - def init_data_format(self): self.data_format = "NHWC" @@ -528,7 +542,6 @@ class TestCase2_channel_last(TestCase2): class TestCase3_channel_last(TestCase3): - def init_data_format(self): self.data_format = "NHWC" @@ -537,7 +550,6 @@ class TestCase3_channel_last(TestCase3): class TestCase4_channel_last(TestCase4): - def init_data_format(self): self.data_format = "NHWC" @@ -546,7 +558,6 @@ class TestCase4_channel_last(TestCase4): class TestCase5_channel_last(TestCase5): - def init_data_format(self): self.data_format = "NHWC" @@ -555,13 +566,11 @@ class TestCase5_channel_last(TestCase5): class TestCase5_Max(TestCase2): - def init_pool_type(self): self.pool_type = "max" class TestCase5_channel_last_Max(TestCase5_Max): - def init_data_format(self): self.data_format = "NHWC" @@ -570,13 +579,11 @@ class TestCase5_channel_last_Max(TestCase5_Max): class TestAvgInclude_channel_last(TestCase2_channel_last): - def init_exclusive(self): self.exclusive = False class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): - def init_adaptive(self): self.adaptive = True @@ -589,7 +596,6 @@ class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -598,7 +604,6 @@ class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -607,7 +612,6 @@ class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -616,7 +620,6 @@ class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -625,7 +628,6 @@ class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -634,7 +636,6 @@ class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -643,7 +644,6 @@ class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -651,9 +651,9 @@ class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): self.shape = [2, 7, 7, 3] -class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding - ): - +class TestAvgPoolAdaptive_AsyPadding_channel_last( + TestAvgPoolAdaptive_AsyPadding +): def init_data_format(self): self.data_format = "NHWC" @@ -662,7 +662,6 @@ class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding class TestCase1_strides(TestCase1): - def init_test_case(self): self.ksize = [3, 3] # fixme: CANN AvgPoolGradV3 dose not support asymmetric strides diff --git a/python/paddle/fluid/tests/unittests/npu/test_pow_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_pow_op_npu.py index 67bb2f9c40f..09f2d0fc055 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_pow_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_pow_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestPow(OpTest): - def setUp(self): self.set_npu() self.op_type = "pow" @@ -55,7 +54,6 @@ class TestPow(OpTest): class TestPowFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "pow" @@ -82,7 +80,6 @@ class TestPowFp16(OpTest): class TestPowNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -97,9 +94,9 @@ class TestPowNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -123,16 +120,17 @@ class TestPowNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_prior_box_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_prior_box_op_npu.py index 4af37416af5..08e9226a37c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_prior_box_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_prior_box_op_npu.py @@ -18,13 +18,15 @@ import paddle import sys import math -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) paddle.enable_static() class TestNPUPriorBox(OpTest): - def setUp(self): self.op_type = "prior_box" self.set_npu() @@ -56,7 +58,7 @@ class TestNPUPriorBox(OpTest): 'min_max_aspect_ratios_order': self.min_max_aspect_ratios_order, 'step_w': self.step_w, 'step_h': self.step_h, - 'offset': self.offset + 'offset': self.offset, } if len(self.max_sizes) > 0: self.attrs['max_sizes'] = self.max_sizes @@ -92,8 +94,9 @@ class TestNPUPriorBox(OpTest): self.flip = True self.set_min_max_aspect_ratios_order() self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] - self.aspect_ratios = np.array(self.aspect_ratios, - dtype=np.float64).flatten() + self.aspect_ratios = np.array( + self.aspect_ratios, dtype=np.float64 + ).flatten() self.variances = [0.1, 0.1, 0.2, 0.2] self.variances = np.array(self.variances, dtype=np.float64).flatten() @@ -105,12 +108,12 @@ class TestNPUPriorBox(OpTest): def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype('float32') + (self.batch_size, self.image_channels, self.image_w, self.image_h) + ).astype('float32') self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype('float32') + (self.batch_size, self.input_channels, self.layer_w, self.layer_h) + ).astype('float32') def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -131,73 +134,78 @@ class TestNPUPriorBox(OpTest): ar = self.real_aspect_ratios[r] c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 else: - c_w = c_h = min_size / 2. - out_boxes[h, w, idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + c_w = c_h = min_size / 2.0 + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # rest of priors for r in range(len(self.real_aspect_ratios)): ar = self.real_aspect_ratios[r] - if abs(ar - 1.) < 1e-6: + if abs(ar - 1.0) < 1e-6: continue c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) # set the variance. - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype('float32') self.out_var = out_var.astype('float32') class TestNPUPriorBoxWithoutMaxSize(TestNPUPriorBox): - def set_max_sizes(self): self.max_sizes = [] class TestNPUPriorBoxWithoutSpecifiedOutOrder(TestNPUPriorBox): - def set_min_max_aspect_ratios_order(self): self.min_max_aspect_ratios_order = False self.atol = 1e-1 diff --git a/python/paddle/fluid/tests/unittests/npu/test_randperm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_randperm_op_npu.py index ca2d41e7633..71d955f1960 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_randperm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_randperm_op_npu.py @@ -27,7 +27,7 @@ paddle.enable_static() class TestRandpermOp(OpTest): - """ Test randperm op.""" + """Test randperm op.""" def setUp(self): self.set_npu() @@ -57,36 +57,32 @@ class TestRandpermOp(OpTest): def verify_output(self, outs): out_np = np.array(outs[0]) - self.assertTrue(check_randperm_out(self.n, out_np), - msg=error_msg(out_np)) + self.assertTrue( + check_randperm_out(self.n, out_np), msg=error_msg(out_np) + ) class TestRandpermOpN(TestRandpermOp): - def init_attrs(self): self.n = 10000 class TestRandpermOpInt32(TestRandpermOp): - def init_attrs(self): self.dtype = "int32" class TestRandpermOpFloat32(TestRandpermOp): - def init_attrs(self): self.dtype = "float32" class TestRandpermOpFloat64(TestRandpermOp): - def init_attrs(self): self.dtype = "float64" class TestRandpermOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(ValueError, paddle.randperm, -3) @@ -94,7 +90,6 @@ class TestRandpermOpError(unittest.TestCase): class TestRandpermAPI(unittest.TestCase): - def test_out(self): n = 10 place = paddle.NPUPlace(0) @@ -112,15 +107,15 @@ class TestRandpermAPI(unittest.TestCase): class TestRandpermImperative(unittest.TestCase): - def test_out(self): paddle.disable_static(paddle.NPUPlace(0)) n = 10 for dtype in ['int32', np.int64, 'float32', 'float64']: data_p = paddle.randperm(n, dtype) data_np = data_p.numpy() - self.assertTrue(check_randperm_out(n, data_np), - msg=error_msg(data_np)) + self.assertTrue( + check_randperm_out(n, data_np), msg=error_msg(data_np) + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/npu/test_range_npu.py b/python/paddle/fluid/tests/unittests/npu/test_range_npu.py index 8a89b57e38a..b0f4ad242f7 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_range_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_range_npu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestRangeOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -36,13 +35,13 @@ class TestRangeOp(OpTest): self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange(self.case[0], self.case[1], self.case[2]).astype( + self.dtype + ) } def init_config(self): @@ -54,49 +53,42 @@ class TestRangeOp(OpTest): class TestFloatRangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.float32 self.case = (0, 5, 1) class TestInt32RangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.case = (0, 5, 2) class TestInt32RangeOpCase1(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.case = (10, 1, -2) class TestInt32RangeOpCase2(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.case = (-1, -10, -2) class TestInt64RangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.int64 self.case = (0, 5, 2) class TestInt64RangeOpCase1(TestRangeOp): - def init_config(self): self.dtype = np.int64 self.case = (10, 1, -2) class TestInt64RangeOpCase2(TestRangeOp): - def init_config(self): self.dtype = np.int64 self.case = (-1, -10, -2) diff --git a/python/paddle/fluid/tests/unittests/npu/test_reciprocal_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reciprocal_op_npu.py index b67cc43efcf..2344b22cdea 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reciprocal_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reciprocal_op_npu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestNPUReciprocal(OpTest): - def setUp(self): self.op_type = "reciprocal" self.set_npu() @@ -41,9 +40,9 @@ class TestNPUReciprocal(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.01 + ) def set_npu(self): self.__class__.use_npu = True @@ -54,7 +53,6 @@ class TestNPUReciprocal(OpTest): class TestNPUReciprocalFp64(TestNPUReciprocal): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -64,9 +62,9 @@ class TestNPUReciprocalFp64(TestNPUReciprocal): @skip_check_grad_ci( - reason="The backward test is not supported for float16 type on NPU.") + reason="The backward test is not supported for float16 type on NPU." +) class TestNPUReciprocalFp16(TestNPUReciprocal): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_any_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_any_op_npu.py index 9ab2e58228c..ece002e4799 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_any_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_any_op_npu.py @@ -28,14 +28,14 @@ paddle.enable_static() class TestAny8DOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" self.place = paddle.NPUPlace(0) self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (3, 5, 4)} self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -48,7 +48,6 @@ class TestAny8DOp(OpTest): class TestAnyOpWithDim(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" @@ -65,14 +64,14 @@ class TestAnyOpWithDim(OpTest): class TestAny8DOpWithDim(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" self.place = paddle.NPUPlace(0) self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (3, 6)} self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -85,7 +84,6 @@ class TestAny8DOpWithDim(OpTest): class TestAnyOpWithKeepDim(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" @@ -93,8 +91,9 @@ class TestAnyOpWithKeepDim(OpTest): self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} self.attrs = {'dim': (1), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].any(axis=self.attrs['dim']), axis=1) + 'Out': np.expand_dims( + self.inputs['X'].any(axis=self.attrs['dim']), axis=1 + ) } def set_npu(self): @@ -105,19 +104,20 @@ class TestAnyOpWithKeepDim(OpTest): class TestAny8DOpWithKeepDim(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" self.place = paddle.NPUPlace(0) self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (1), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].any(axis=self.attrs['dim']), axis=1) + 'Out': np.expand_dims( + self.inputs['X'].any(axis=self.attrs['dim']), axis=1 + ) } def set_npu(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_max_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_max_op_npu.py index 22ababf99b0..8e0cdd711b4 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_max_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_max_op_npu.py @@ -26,7 +26,8 @@ paddle.enable_static() @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestNPUReduceMaxOp(OpTest): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -54,7 +55,8 @@ class TestNPUReduceMaxOp(OpTest): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpMultiAxises(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -72,7 +74,8 @@ class TestReduceMaxOpMultiAxises(TestNPUReduceMaxOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceAll(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -88,7 +91,8 @@ class TestReduceAll(TestNPUReduceMaxOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_bool(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -100,17 +104,19 @@ class TestReduceMaxOpWithOutDtype_bool(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.BOOL) + 'out_dtype': int(core.VarDesc.VarType.BOOL), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.bool_) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.bool_) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_int16(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -122,18 +128,20 @@ class TestReduceMaxOpWithOutDtype_int16(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT16) + 'out_dtype': int(core.VarDesc.VarType.INT16), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.int16) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.int16) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_int32(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -145,17 +153,19 @@ class TestReduceMaxOpWithOutDtype_int32(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.int32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_int64(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -167,17 +177,19 @@ class TestReduceMaxOpWithOutDtype_int64(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.int64) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.int64) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp16(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -189,12 +201,12 @@ class TestReduceMaxOpWithOutDtype_fp16(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def test_check_output(self): @@ -203,7 +215,8 @@ class TestReduceMaxOpWithOutDtype_fp16(TestNPUReduceMaxOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp32(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -215,18 +228,19 @@ class TestReduceMaxOpWithOutDtype_fp32(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp64(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -238,18 +252,19 @@ class TestReduceMaxOpWithOutDtype_fp64(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP64) + 'out_dtype': int(core.VarDesc.VarType.FP64), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float64) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float64) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp32_2(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -261,12 +276,12 @@ class TestReduceMaxOpWithOutDtype_fp32_2(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): @@ -275,7 +290,8 @@ class TestReduceMaxOpWithOutDtype_fp32_2(TestNPUReduceMaxOp): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpInt64(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -287,12 +303,12 @@ class TestReduceMaxOpInt64(TestNPUReduceMaxOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_mean_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_mean_op_npu.py index 232d53d4939..fe56cb152c6 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_mean_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_mean_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestMeanOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -43,7 +42,6 @@ class TestMeanOp(OpTest): class TestMeanOp5D(TestMeanOp): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -54,7 +52,6 @@ class TestMeanOp5D(TestMeanOp): class TestMeanOp6D(TestMeanOp): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -65,7 +62,6 @@ class TestMeanOp6D(TestMeanOp): class TestMeanOp8D(TestMeanOp): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -77,7 +73,6 @@ class TestMeanOp8D(TestMeanOp): class Test1DReduce(TestMeanOp): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -86,7 +81,6 @@ class Test1DReduce(TestMeanOp): class Test2DReduce0(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -96,7 +90,6 @@ class Test2DReduce0(Test1DReduce): class Test2DReduce1(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -108,7 +101,6 @@ class Test2DReduce1(Test1DReduce): class Test3DReduce0(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -120,7 +112,6 @@ class Test3DReduce0(Test1DReduce): class Test3DReduce1(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -132,7 +123,6 @@ class Test3DReduce1(Test1DReduce): class Test3DReduce2(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -144,7 +134,6 @@ class Test3DReduce2(Test1DReduce): class Test3DReduce3(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -156,21 +145,19 @@ class Test3DReduce3(Test1DReduce): class TestKeepDimReduce(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" self.inputs = {'X': np.random.random((5, 6, 10)).astype("float32")} self.attrs = {'dim': [1], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestKeepDim8DReduce(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -179,14 +166,13 @@ class TestKeepDim8DReduce(Test1DReduce): } self.attrs = {'dim': (3, 4, 5), 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestReduceAll(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_min_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_min_op_npu.py index da9d8a1893b..2ced99b6020 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_min_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_min_op_npu.py @@ -26,7 +26,8 @@ paddle.enable_static() @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestNPUReduceMinOp(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -54,7 +55,8 @@ class TestNPUReduceMinOp(OpTest): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpMultiAxises(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -72,7 +74,8 @@ class TestReduceMinOpMultiAxises(TestNPUReduceMinOp): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceAll(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -88,7 +91,8 @@ class TestReduceAll(TestNPUReduceMinOp): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_bool(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -100,17 +104,19 @@ class TestReduceMinOpWithOutDtype_bool(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.BOOL) + 'out_dtype': int(core.VarDesc.VarType.BOOL), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.bool_) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.bool_) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_int16(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -122,18 +128,20 @@ class TestReduceMinOpWithOutDtype_int16(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT16) + 'out_dtype': int(core.VarDesc.VarType.INT16), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.int16) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.int16) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_int32(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -145,17 +153,19 @@ class TestReduceMinOpWithOutDtype_int32(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.int32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_int64(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -167,17 +177,19 @@ class TestReduceMinOpWithOutDtype_int64(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.int64) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.int64) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp16(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -189,12 +201,12 @@ class TestReduceMinOpWithOutDtype_fp16(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def test_check_output(self): @@ -203,7 +215,8 @@ class TestReduceMinOpWithOutDtype_fp16(TestNPUReduceMinOp): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp32(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -215,18 +228,19 @@ class TestReduceMinOpWithOutDtype_fp32(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp64(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -238,18 +252,19 @@ class TestReduceMinOpWithOutDtype_fp64(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP64) + 'out_dtype': int(core.VarDesc.VarType.FP64), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float64) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float64) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp32_2(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -261,12 +276,12 @@ class TestReduceMinOpWithOutDtype_fp32_2(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): @@ -275,7 +290,8 @@ class TestReduceMinOpWithOutDtype_fp32_2(TestNPUReduceMinOp): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpInt64(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -287,12 +303,12 @@ class TestReduceMinOpInt64(TestNPUReduceMinOp): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_prod_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_prod_op_npu.py index e8f4430d655..956a2258224 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_prod_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_prod_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestNPUReduceProd(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -49,7 +48,6 @@ class TestNPUReduceProd(OpTest): class TestNPUReduceProd2(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -61,7 +59,6 @@ class TestNPUReduceProd2(TestNPUReduceProd): class TestNPUReduceProd3(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -73,7 +70,6 @@ class TestNPUReduceProd3(TestNPUReduceProd): class TestNPUReduceProd6D(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -89,7 +85,6 @@ class TestNPUReduceProd6D(TestNPUReduceProd): class TestNPUReduceProd8D(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -105,7 +100,6 @@ class TestNPUReduceProd8D(TestNPUReduceProd): class TestReduceAll(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -117,7 +111,6 @@ class TestReduceAll(TestNPUReduceProd): class TestNPUReduceProdWithOutDtype_bool(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -126,14 +119,13 @@ class TestNPUReduceProdWithOutDtype_bool(TestNPUReduceProd): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.BOOL)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.bool_) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.bool_) } class TestNPUReduceProdWithOutDtype_int16(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -142,14 +134,13 @@ class TestNPUReduceProdWithOutDtype_int16(TestNPUReduceProd): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.INT16)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.int16) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.int16) } class TestNPUReduceProdWithOutDtype_int32(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -158,14 +149,13 @@ class TestNPUReduceProdWithOutDtype_int32(TestNPUReduceProd): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.INT32)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.int32) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } class TestNPUReduceProdWithOutDtype_int64(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -174,14 +164,13 @@ class TestNPUReduceProdWithOutDtype_int64(TestNPUReduceProd): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.INT64)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.int64) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.int64) } class TestNPUReduceProdWithOutDtype_fp16(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -190,9 +179,9 @@ class TestNPUReduceProdWithOutDtype_fp16(TestNPUReduceProd): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.FP16)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def test_check_output(self): @@ -200,7 +189,6 @@ class TestNPUReduceProdWithOutDtype_fp16(TestNPUReduceProd): class TestNPUReduceProdWithOutDtype_fp32(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -209,14 +197,13 @@ class TestNPUReduceProdWithOutDtype_fp32(TestNPUReduceProd): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.FP32)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } class TestNPUReduceProdWithOutDtype_fp64(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -225,15 +212,14 @@ class TestNPUReduceProdWithOutDtype_fp64(TestNPUReduceProd): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.FP64)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.float64) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.float64) } @skip_check_grad_ci(reason="right now not implement grad op") class TestNPUReduceProdWithOutDtype_fp32_2(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -242,9 +228,9 @@ class TestNPUReduceProdWithOutDtype_fp32_2(TestNPUReduceProd): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.FP32)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_sum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_sum_op_npu.py index 498cc2d97ac..d5cd5febfb9 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_sum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_sum_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestReduceSum(OpTest): - def setUp(self): np.random.seed(SEED) self.set_npu() @@ -39,16 +38,16 @@ class TestReduceSum(OpTest): self.attrs = { 'dim': self.axis, 'keep_dim': self.keep_dim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.inputs = {'X': np.random.random(self.shape).astype(self.dtype)} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].sum()} else: self.outputs = { - 'Out': - self.inputs['X'].sum(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def set_npu(self): @@ -65,7 +64,7 @@ class TestReduceSum(OpTest): def initTestCase(self): self.shape = (5, 6) - self.axis = (0) + self.axis = 0 def test_check_output(self): self.check_output_with_place(self.place) @@ -79,13 +78,11 @@ class TestReduceSum(OpTest): class TestReduceSum2(OpTest): - def init_dtype(self): self.dtype = np.int32 class TestReduceSumNet(unittest.TestCase): - def set_reduce_sum_function(self, x): # keep_dim = False return paddle.fluid.layers.reduce_sum(x, dim=-1) @@ -104,9 +101,9 @@ class TestReduceSumNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[2, 3, 4], dtype='float32') b = paddle.static.data(name="b", shape=[2, 3, 4], dtype='float32') - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) a_1 = fluid.layers.fc(input=a, size=4, num_flatten_dims=2, act=None) b_1 = fluid.layers.fc(input=b, size=4, num_flatten_dims=2, act=None) @@ -131,16 +128,17 @@ class TestReduceSumNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -153,14 +151,12 @@ class TestReduceSumNet(unittest.TestCase): class TestReduceSumNet2(TestReduceSumNet): - def set_reduce_sum_function(self, x): # keep_dim = True return paddle.fluid.layers.reduce_sum(x, dim=-1, keep_dim=True) class TestReduceSumNet3(TestReduceSumNet): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -191,12 +187,9 @@ class TestReduceSumNet3(TestReduceSumNet): print("Start run on {}".format(place)) for epoch in range(100): - loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np - }, - fetch_list=[loss]) + loss_res = exe.run( + main_prog, feed={"a": a_np, "b": b_np}, fetch_list=[loss] + ) if epoch % 10 == 0: print("Epoch {} | Loss: {}".format(epoch, loss_res)) diff --git a/python/paddle/fluid/tests/unittests/npu/test_relu6_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_relu6_op_npu.py index 0fe796a0b0d..3988b4a6a93 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_relu6_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_relu6_op_npu.py @@ -34,7 +34,6 @@ def ref_relu6(x, threshold=6.0): class TestRelu6(OpTest): - def setUp(self): self.set_npu() self.op_type = "relu6" @@ -64,7 +63,6 @@ class TestRelu6(OpTest): class TestRelu6Float16(TestRelu6): - def set_npu(self): self.__class__.use_npu = True self.__class__.no_need_check_grad = True @@ -77,7 +75,6 @@ class TestRelu6Float16(TestRelu6): class TestReluNeg(TestRelu6): - def setUp(self): self.set_npu() self.op_type = "relu6" @@ -104,7 +101,6 @@ class TestReluNeg(TestRelu6): class TestRelu6Net(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -119,9 +115,9 @@ class TestRelu6Net(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.nn.functional.relu6(sum) @@ -145,16 +141,17 @@ class TestRelu6Net(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_relu_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_relu_op_npu.py index 98c46329350..a52e4d39cfd 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_relu_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_relu_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestRelu(OpTest): - def setUp(self): self.set_npu() self.op_type = "relu" @@ -53,21 +52,19 @@ class TestRelu(OpTest): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.006 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') class TestReluFp16(TestRelu): - def init_dtype(self): self.dtype = np.float16 class TestReluNeg(OpTest): - def setUp(self): self.set_npu() self.op_type = "relu" @@ -97,7 +94,6 @@ class TestReluNeg(OpTest): class TestReluNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -112,9 +108,9 @@ class TestReluNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.nn.functional.relu(sum) @@ -138,16 +134,17 @@ class TestReluNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_reshape2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reshape2_op_npu.py index 10ce920fb90..74175ca6bbc 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reshape2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reshape2_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestReshape2(OpTest): - def setUp(self): self.set_npu() self.op_type = "reshape2" @@ -37,7 +36,7 @@ class TestReshape2(OpTest): self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def set_npu(self): @@ -56,7 +55,6 @@ class TestReshape2(OpTest): class TestReshape2_case2(TestReshape2): - def init_data(self): self.ori_shape = (2, 100) self.new_shape = (-1, 10) @@ -64,7 +62,6 @@ class TestReshape2_case2(TestReshape2): class TestReshape2_case3(TestReshape2): - def init_data(self): self.ori_shape = (100, 5, 6) self.new_shape = (-1, 0, 3) diff --git a/python/paddle/fluid/tests/unittests/npu/test_rmsprop_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_rmsprop_op_npu.py index ed872aa0dea..9274d8daa1d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_rmsprop_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_rmsprop_op_npu.py @@ -28,7 +28,6 @@ SEED = 2021 class TestNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -43,9 +42,9 @@ class TestNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -69,16 +68,17 @@ class TestNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -91,7 +91,6 @@ class TestNet(unittest.TestCase): class TestCenteredNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -106,9 +105,9 @@ class TestCenteredNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -132,16 +131,17 @@ class TestCenteredNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_roi_align_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_roi_align_op_npu.py index 8e1529e43f5..fde9824d95d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_roi_align_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_roi_align_op_npu.py @@ -26,7 +26,6 @@ np.random.seed(1243) class TestROIAlignNPUOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -37,7 +36,7 @@ class TestROIAlignNPUOp(OpTest): self.inputs = { 'X': self.x, 'ROIs': self.rois[:, 1:5], - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } self.attrs = { @@ -45,7 +44,7 @@ class TestROIAlignNPUOp(OpTest): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.aligned + 'aligned': self.aligned, } self.outputs = {'Out': self.out_data} @@ -67,25 +66,45 @@ class TestROIAlignNPUOp(OpTest): self.x = np.random.random(self.x_dim).astype('float32') - def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, - bin_size_h, bin_size_w): + def pre_calc( + self, + x_i, + roi_xmin, + roi_ymin, + roi_bin_grid_h, + roi_bin_grid_w, + bin_size_h, + bin_size_w, + ): count = roi_bin_grid_h * roi_bin_grid_w bilinear_pos = np.zeros( [self.channels, self.pooled_height, self.pooled_width, count, 4], - np.float32) - bilinear_w = np.zeros([self.pooled_height, self.pooled_width, count, 4], - np.float32) + np.float32, + ) + bilinear_w = np.zeros( + [self.pooled_height, self.pooled_width, count, 4], np.float32 + ) for ph in range(self.pooled_width): for pw in range(self.pooled_height): c = 0 for iy in range(roi_bin_grid_h): - y = roi_ymin + ph * bin_size_h + (iy + 0.5) * \ - bin_size_h / roi_bin_grid_h + y = ( + roi_ymin + + ph * bin_size_h + + (iy + 0.5) * bin_size_h / roi_bin_grid_h + ) for ix in range(roi_bin_grid_w): - x = roi_xmin + pw * bin_size_w + (ix + 0.5) * \ - bin_size_w / roi_bin_grid_w - if y < -1.0 or y > self.height or \ - x < -1.0 or x > self.width: + x = ( + roi_xmin + + pw * bin_size_w + + (ix + 0.5) * bin_size_w / roi_bin_grid_w + ) + if ( + y < -1.0 + or y > self.height + or x < -1.0 + or x > self.width + ): continue if y <= 0: y = 0 @@ -106,14 +125,18 @@ class TestROIAlignNPUOp(OpTest): hy = 1 - ly hx = 1 - lx for ch in range(self.channels): - bilinear_pos[ch, ph, pw, c, 0] = x_i[ch, y_low, - x_low] - bilinear_pos[ch, ph, pw, c, 1] = x_i[ch, y_low, - x_high] - bilinear_pos[ch, ph, pw, c, 2] = x_i[ch, y_high, - x_low] - bilinear_pos[ch, ph, pw, c, 3] = x_i[ch, y_high, - x_high] + bilinear_pos[ch, ph, pw, c, 0] = x_i[ + ch, y_low, x_low + ] + bilinear_pos[ch, ph, pw, c, 1] = x_i[ + ch, y_low, x_high + ] + bilinear_pos[ch, ph, pw, c, 2] = x_i[ + ch, y_high, x_low + ] + bilinear_pos[ch, ph, pw, c, 3] = x_i[ + ch, y_high, x_high + ] bilinear_w[ph, pw, c, 0] = hy * hx bilinear_w[ph, pw, c, 1] = hy * lx bilinear_w[ph, pw, c, 2] = ly * hx @@ -123,10 +146,15 @@ class TestROIAlignNPUOp(OpTest): def calc_roi_align(self): self.out_data = np.zeros( - (self.rois_num, self.channels, self.pooled_height, - self.pooled_width)).astype('float32') - - offset = 0.5 if self.aligned else 0. + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ).astype('float32') + + offset = 0.5 if self.aligned else 0.0 for i in range(self.rois_num): roi = self.rois[i] roi_batch_id = int(roi[0]) @@ -144,16 +172,27 @@ class TestROIAlignNPUOp(OpTest): bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) - roi_bin_grid_h = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_height / self.pooled_height) - roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_width / self.pooled_width) + roi_bin_grid_h = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_height / self.pooled_height) + ) + roi_bin_grid_w = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_width / self.pooled_width) + ) count = max(int(roi_bin_grid_h * roi_bin_grid_w), 1) pre_size = count * self.pooled_width * self.pooled_height - bilinear_pos, bilinear_w = self.pre_calc(x_i, roi_xmin, roi_ymin, - int(roi_bin_grid_h), - int(roi_bin_grid_w), - bin_size_h, bin_size_w) + bilinear_pos, bilinear_w = self.pre_calc( + x_i, + roi_xmin, + roi_ymin, + int(roi_bin_grid_h), + int(roi_bin_grid_w), + bin_size_h, + bin_size_w, + ) for ch in range(self.channels): align_per_bin = (bilinear_pos[ch] * bilinear_w).sum(axis=-1) output_val = align_per_bin.mean(axis=-1) @@ -166,14 +205,18 @@ class TestROIAlignNPUOp(OpTest): # for i in range(bno + 1): self.rois_lod[0].append(bno) x1 = np.random.randint( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.randint( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.randint(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.randint(y1 + self.pooled_height, - self.height // self.spatial_scale) + x2 = np.random.randint( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.randint( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) @@ -195,7 +238,6 @@ class TestROIAlignNPUOp(OpTest): class TestROIAlignOpWithMinusSample(TestROIAlignNPUOp): - def init_test_case(self): self.batch_size = 3 self.channels = 3 diff --git a/python/paddle/fluid/tests/unittests/npu/test_run_program_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_run_program_op_npu.py index 52b95c7d9dc..899a3fdf03c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_run_program_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_run_program_op_npu.py @@ -50,10 +50,10 @@ def program_scope_guard(): # when create Operator, so here compare gradients with static graph # NOTE: Here rewrite a simple unittest framework for RunProgramOp class RunProgramNPUOpTest(unittest.TestCase): - def build_model(self): raise NotImplementedError( - "RunProgramOp test should implement build_model") + "RunProgramOp test should implement build_model" + ) def check_output(self): places = [fluid.NPUPlace(0)] @@ -84,9 +84,9 @@ class RunProgramNPUOpTest(unittest.TestCase): else: fetch_list = self.get_param_grad_names() - outs = exe.run(main_program, - feed=self.inputs['X'], - fetch_list=fetch_list) + outs = exe.run( + main_program, feed=self.inputs['X'], fetch_list=fetch_list + ) return outs def get_program_desc(self): @@ -95,9 +95,16 @@ class RunProgramNPUOpTest(unittest.TestCase): return fluid.default_main_program().desc, fwd_op_num def prepare_attrs(self): - return ('global_block', self.program_desc.block(0), 'start_op_index', 0, - 'end_op_index', self.fwd_op_num, 'program_id', - _hash_with_id(self.program_desc, self)) + return ( + 'global_block', + self.program_desc.block(0), + 'start_op_index', + 0, + 'end_op_index', + self.fwd_op_num, + 'program_id', + _hash_with_id(self.program_desc, self), + ) def get_param_grad_names(self): grad_names = [] @@ -111,10 +118,9 @@ class RunProgramNPUOpTest(unittest.TestCase): # Step 2. compare output for expect_v, actual_v in zip(self.expect_outs, actual_outs): - np.testing.assert_allclose(expect_v, - actual_v.numpy(), - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + expect_v, actual_v.numpy(), rtol=1e-05, atol=1e-05 + ) def check_grad_with_place(self, place): # Step 1. calc grads @@ -123,24 +129,20 @@ class RunProgramNPUOpTest(unittest.TestCase): # Step 2. compare grads for expect_v, actual_v in zip(self.expect_grads, actual_grads): np.testing.assert_array_almost_equal(expect_v, actual_v) - np.testing.assert_allclose(expect_v, - actual_v, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + expect_v, actual_v, rtol=1e-05, atol=1e-05 + ) def prepare_dygraph_input(self, place, return_param_list=False): - def create_var_base(is_input, name, np_value, stop_gradient): if _in_eager_mode_: - var = core.eager.Tensor(value=np_value, - name=name, - place=place, - zero_copy=True) + var = core.eager.Tensor( + value=np_value, name=name, place=place, zero_copy=True + ) else: - var = core.VarBase(value=np_value, - name=name, - place=place, - zero_copy=True) + var = core.VarBase( + value=np_value, name=name, place=place, zero_copy=True + ) var.stop_gradient = stop_gradient return var @@ -163,7 +165,6 @@ class RunProgramNPUOpTest(unittest.TestCase): return inputs def prepare_dygraph_output(self): - def create_var_base(is_input, name): var = framework._varbase_creator(dtype=None, shape=None, name=name) var.stop_gradient = False @@ -181,7 +182,8 @@ class RunProgramNPUOpTest(unittest.TestCase): outputs['OutScope'] = framework._varbase_creator( type=core.VarDesc.VarType.STEP_SCOPES, name="program_out_scope", - persistable=True) + persistable=True, + ) inner_scope = core.Scope() outputs['OutScope'].value().set_scope(inner_scope) @@ -196,9 +198,15 @@ class RunProgramNPUOpTest(unittest.TestCase): inputs = self.prepare_dygraph_input(place) outputs = self.prepare_dygraph_output() - _legacy_C_ops.run_program(inputs['X'], inputs['Params'], - outputs['Out'], outputs['OutScope'], - outputs['DOut'], None, *self.attrs) + _legacy_C_ops.run_program( + inputs['X'], + inputs['Params'], + outputs['Out'], + outputs['OutScope'], + outputs['DOut'], + None, + *self.attrs + ) return outputs['Out'] def calc_dygraph_grad(self, place): @@ -210,9 +218,15 @@ class RunProgramNPUOpTest(unittest.TestCase): inputs, input_param_list = self.prepare_dygraph_input(place, True) outputs = self.prepare_dygraph_output() - _legacy_C_ops.run_program(inputs['X'], inputs['Params'], - outputs['Out'], outputs['OutScope'], - outputs['DOut'], None, *self.attrs) + _legacy_C_ops.run_program( + inputs['X'], + inputs['Params'], + outputs['Out'], + outputs['OutScope'], + outputs['DOut'], + None, + *self.attrs + ) for param in input_param_list: var_type = self._get_grad_vartype(param.name) @@ -243,27 +257,29 @@ class RunProgramNPUOpTest(unittest.TestCase): class TestRunProgramOpWithFC(RunProgramNPUOpTest): - def setUp(self): self.op_type = "run_program" self.dtype = np.float32 self.input_names = { 'X': ['img'], - 'Params': ['weight_param', 'bias_param'] + 'Params': ['weight_param', 'bias_param'], } self.output_names = {'Out': ['fc_0.tmp_2']} self.inputs = { 'X': { - self.input_names['X'][0]: - np.random.random((32, 1, 28, 28)).astype(self.dtype) + self.input_names['X'][0]: np.random.random( + (32, 1, 28, 28) + ).astype(self.dtype) }, 'Params': { - self.input_names['Params'][0]: - np.random.random((784, 10)).astype(self.dtype), - self.input_names['Params'][1]: - np.random.random((32, 10)).astype(self.dtype) - } + self.input_names['Params'][0]: np.random.random( + (784, 10) + ).astype(self.dtype), + self.input_names['Params'][1]: np.random.random( + (32, 10) + ).astype(self.dtype), + }, } def test_check_output(self): @@ -274,26 +290,34 @@ class TestRunProgramOpWithFC(RunProgramNPUOpTest): def build_model(self): # 1. simple model - img = fluid.data(name=self.input_names['X'][0], - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name=self.input_names['X'][0], + shape=[None, 1, 28, 28], + dtype='float32', + ) weight_attr = fluid.ParamAttr( name=self.input_names['Params'][0], learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][0]]), - trainable=True) + self.inputs['Params'][self.input_names['Params'][0]] + ), + trainable=True, + ) bias_attr = fluid.ParamAttr( name=self.input_names['Params'][1], learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][1]]), - trainable=True) - pred = fluid.layers.fc(input=img, - size=10, - param_attr=weight_attr, - bias_attr=bias_attr, - act='relu') + self.inputs['Params'][self.input_names['Params'][1]] + ), + trainable=True, + ) + pred = fluid.layers.fc( + input=img, + size=10, + param_attr=weight_attr, + bias_attr=bias_attr, + act='relu', + ) # 2. get forward op num fwd_op_num = fluid.default_main_program().global_block().desc.op_size() # 3. append backward diff --git a/python/paddle/fluid/tests/unittests/npu/test_sampling_id_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sampling_id_op_npu.py index 399ac0d2522..d354e39bcf0 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sampling_id_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sampling_id_op_npu.py @@ -28,7 +28,6 @@ _set_use_system_allocator(False) class TestSamplingIdShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() x = fluid.layers.data(name='x', shape=[3], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/npu/test_save_load_npu.py b/python/paddle/fluid/tests/unittests/npu/test_save_load_npu.py index b6a6a0464a0..09f25273696 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_save_load_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_save_load_npu.py @@ -36,62 +36,78 @@ paddle.enable_static() class TestNPUSaveLoadBase(TestSaveLoadBase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUSaveLoadPartial(TestSaveLoadPartial): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUSaveLoadSetStateDict(TestSaveLoadSetStateDict): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUProgramStatePartial(TestProgramStatePartial): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPULoadFromOldInterface(TestLoadFromOldInterface): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPULoadFromOldInterfaceSingleFile(TestLoadFromOldInterfaceSingleFile): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUProgramStateOldSave(TestProgramStateOldSave): - def setUp(self): self.test_dygraph = False def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUProgramStateOldSaveSingleModel(TestProgramStateOldSaveSingleModel): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_scale_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_scale_op_npu.py index c7166159f9a..94477432e57 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_scale_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_scale_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestScale(OpTest): - def setUp(self): self.set_npu() self.op_type = "scale" @@ -34,14 +33,15 @@ class TestScale(OpTest): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype( - np.random.random((10, 10)).astype(self.dtype)) + 'X': OpTest.np_dtype_to_fluid_dtype( + np.random.random((10, 10)).astype(self.dtype) + ) } self.attrs = {'scale': -2.3, 'bias': 0, 'bias_after_scale': True} self.outputs = { 'Out': (self.inputs['X'] * self.dtype(self.attrs['scale'])).astype( - self.dtype) + self.dtype + ) } def set_npu(self): @@ -55,25 +55,21 @@ class TestScale(OpTest): class TestFP16Scale(TestScale): - def init_dtype(self): self.dtype = np.float16 class TestScaleInt(TestScale): - def init_dtype(self): self.dtype = np.int32 class TestScaleInt64(TestScale): - def init_dtype(self): self.dtype = np.int64 class TestBiasAfterScale(OpTest): - def setUp(self): self.set_npu() self.op_type = "scale" @@ -81,9 +77,9 @@ class TestBiasAfterScale(OpTest): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype( - np.random.random((10, 10)).astype(self.dtype)) + 'X': OpTest.np_dtype_to_fluid_dtype( + np.random.random((10, 10)).astype(self.dtype) + ) } self.attrs = {'scale': -2.3, 'bias': 0, 'bias_after_scale': False} self.outputs = { diff --git a/python/paddle/fluid/tests/unittests/npu/test_scatter_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_scatter_op_npu.py index 8bb28b10997..bfa591e3f10 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_scatter_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_scatter_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestCast1_FP32(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" @@ -51,7 +50,6 @@ class TestCast1_FP32(OpTest): class TestCast_INT32(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" @@ -75,7 +73,6 @@ class TestCast_INT32(OpTest): class TestCast2_FP32(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" @@ -99,7 +96,6 @@ class TestCast2_FP32(OpTest): class TestCast3_FP32(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" @@ -124,7 +120,6 @@ class TestCast3_FP32(OpTest): class TestCast_INT64(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" diff --git a/python/paddle/fluid/tests/unittests/npu/test_seed_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_seed_op_npu.py index 72402bd2b86..dd688a29c44 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_seed_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_seed_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestSeedOpFixSeed(OpTest): - def setUp(self): self.set_npu() self.op_type = "seed" @@ -42,7 +41,6 @@ class TestSeedOpFixSeed(OpTest): class TestSeedOpDiffSeed(OpTest): - def setUp(self): self.set_npu() self.op_type = "seed" diff --git a/python/paddle/fluid/tests/unittests/npu/test_sequence_mask_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sequence_mask_npu.py index df0cb03d58b..66642cd4bc6 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sequence_mask_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sequence_mask_npu.py @@ -21,13 +21,16 @@ from op_test import OpTest import paddle import paddle.fluid as fluid import paddle.fluid.core as core -from paddle.fluid.framework import convert_np_dtype_to_dtype_, Program, program_guard +from paddle.fluid.framework import ( + convert_np_dtype_to_dtype_, + Program, + program_guard, +) paddle.enable_static() class SequenceMaskTestBase(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -51,19 +54,19 @@ class SequenceMaskTestBase(OpTest): self.outputs = {'Y': self.calc_ground_truth_mask()} self.attrs = { 'maxlen': self.maxlen, - 'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype) + 'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype), } def calc_ground_truth_mask(self): maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen - shape = self.x.shape + (maxlen, ) - index_broadcast = np.broadcast_to(np.reshape( - range(maxlen), newshape=[1] * self.x.ndim + [-1]), - shape=shape) - x_broadcast = np.broadcast_to(np.reshape(self.x, - newshape=self.x.shape + - (-1, )), - shape=shape) + shape = self.x.shape + (maxlen,) + index_broadcast = np.broadcast_to( + np.reshape(range(maxlen), newshape=[1] * self.x.ndim + [-1]), + shape=shape, + ) + x_broadcast = np.broadcast_to( + np.reshape(self.x, newshape=self.x.shape + (-1,)), shape=shape + ) return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): @@ -71,43 +74,36 @@ class SequenceMaskTestBase(OpTest): class SequenceMaskTest1(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'bool' class SequenceMaskTest2(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'uint8' class SequenceMaskTest3(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'int32' class SequenceMaskTest4(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'float32' class SequenceMaskTest5(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'float64' class SequenceMaskTest6(SequenceMaskTestBase): - def initParameters(self): self.maxlen = -1 class SequenceMaskTestBase_tensor_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -134,14 +130,14 @@ class SequenceMaskTestBase_tensor_attr(OpTest): def calc_ground_truth_mask(self): maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen - shape = self.x.shape + (maxlen, ) - index_broadcast = np.broadcast_to(np.reshape( - range(maxlen), newshape=[1] * self.x.ndim + [-1]), - shape=shape) - x_broadcast = np.broadcast_to(np.reshape(self.x, - newshape=self.x.shape + - (-1, )), - shape=shape) + shape = self.x.shape + (maxlen,) + index_broadcast = np.broadcast_to( + np.reshape(range(maxlen), newshape=[1] * self.x.ndim + [-1]), + shape=shape, + ) + x_broadcast = np.broadcast_to( + np.reshape(self.x, newshape=self.x.shape + (-1,)), shape=shape + ) return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): @@ -149,37 +145,31 @@ class SequenceMaskTestBase_tensor_attr(OpTest): class SequenceMaskTest1_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'bool' class SequenceMaskTest2_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'uint8' class SequenceMaskTest3_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'int32' class SequenceMaskTest4_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'float32' class SequenceMaskTest5_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'float64' class TestSequenceMaskOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.uniform(1, 5, [4]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_set_value_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_set_value_op_npu.py index e1b866f31f5..96af4148833 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_set_value_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_set_value_op_npu.py @@ -24,7 +24,6 @@ from paddle.fluid import core class TestSetValueBase(unittest.TestCase): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -55,7 +54,6 @@ class TestSetValueBase(unittest.TestCase): class TestSetValueApi(TestSetValueBase): - def _run_static(self): paddle.enable_static() with paddle.static.program_guard(self.program): @@ -80,17 +78,22 @@ class TestSetValueApi(TestSetValueBase): dynamic_out = self._run_dynamic() self._get_answer() - error_msg = "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" - self.assertTrue((self.data == static_out).all(), - msg=error_msg.format("static", self.data, static_out)) - self.assertTrue((self.data == dynamic_out).all(), - msg=error_msg.format("dynamic", self.data, dynamic_out)) + error_msg = ( + "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" + ) + self.assertTrue( + (self.data == static_out).all(), + msg=error_msg.format("static", self.data, static_out), + ) + self.assertTrue( + (self.data == dynamic_out).all(), + msg=error_msg.format("dynamic", self.data, dynamic_out), + ) # 1. Test different type of item: int, Python slice, Paddle Tensor # 1.1 item is int class TestSetValueItemInt(TestSetValueApi): - def _call_setitem(self, x): x[0] = self.value @@ -101,7 +104,6 @@ class TestSetValueItemInt(TestSetValueApi): # 1.2 item is slice # 1.2.1 step is 1 class TestSetValueItemSlice(TestSetValueApi): - def _call_setitem(self, x): x[0:2] = self.value @@ -110,7 +112,6 @@ class TestSetValueItemSlice(TestSetValueApi): class TestSetValueItemSlice2(TestSetValueApi): - def _call_setitem(self, x): x[0:-1] = self.value @@ -119,7 +120,6 @@ class TestSetValueItemSlice2(TestSetValueApi): class TestSetValueItemSlice3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2] = self.value @@ -128,7 +128,6 @@ class TestSetValueItemSlice3(TestSetValueApi): class TestSetValueItemSlice4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2, :] = self.value @@ -156,7 +155,6 @@ class TestSetValueItemSlice4(TestSetValueApi): # 1.2.2 step > 1 class TestSetValueItemSliceStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 5, 5] @@ -168,7 +166,6 @@ class TestSetValueItemSliceStep(TestSetValueApi): class TestSetValueItemSliceStep2(TestSetValueApi): - def set_shape(self): self.shape = [7, 5, 5] @@ -180,7 +177,6 @@ class TestSetValueItemSliceStep2(TestSetValueApi): class TestSetValueItemSliceStep3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2, ::2] = self.value @@ -189,7 +185,6 @@ class TestSetValueItemSliceStep3(TestSetValueApi): class TestSetValueItemSliceStep4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2:2, :] = self.value @@ -199,7 +194,6 @@ class TestSetValueItemSliceStep4(TestSetValueApi): # 1.2.3 step < 0 class TestSetValueItemSliceNegetiveStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 2] @@ -214,7 +208,6 @@ class TestSetValueItemSliceNegetiveStep(TestSetValueApi): class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): - def set_shape(self): self.shape = [5] @@ -229,7 +222,6 @@ class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): - def set_shape(self): self.shape = [3] @@ -244,7 +236,6 @@ class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -259,7 +250,6 @@ class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): class TestSetValueItemEllipsis1(TestSetValueApi): - def _call_setitem(self, x): x[0:, ..., 1:] = self.value @@ -268,7 +258,6 @@ class TestSetValueItemEllipsis1(TestSetValueApi): class TestSetValueItemEllipsis2(TestSetValueApi): - def _call_setitem(self, x): x[0:, ...] = self.value @@ -277,7 +266,6 @@ class TestSetValueItemEllipsis2(TestSetValueApi): class TestSetValueItemEllipsis3(TestSetValueApi): - def _call_setitem(self, x): x[..., 1:] = self.value @@ -286,7 +274,6 @@ class TestSetValueItemEllipsis3(TestSetValueApi): class TestSetValueItemEllipsis4(TestSetValueApi): - def _call_setitem(self, x): x[...] = self.value @@ -296,7 +283,6 @@ class TestSetValueItemEllipsis4(TestSetValueApi): # 1.4 item is Paddle Tensor class TestSetValueItemTensor(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") x[zero] = self.value @@ -306,7 +292,6 @@ class TestSetValueItemTensor(TestSetValueApi): class TestSetValueItemTensor2(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -317,7 +302,6 @@ class TestSetValueItemTensor2(TestSetValueApi): class TestSetValueItemTensor3(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -328,7 +312,6 @@ class TestSetValueItemTensor3(TestSetValueApi): class TestSetValueItemTensor4(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -339,7 +322,6 @@ class TestSetValueItemTensor4(TestSetValueApi): class TestSetValueItemTensor5(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -350,7 +332,6 @@ class TestSetValueItemTensor5(TestSetValueApi): class TestSetValueItemTensor6(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -365,7 +346,6 @@ class TestSetValueItemTensor6(TestSetValueApi): # 1.5 item is None class TestSetValueItemNone1(TestSetValueApi): - def _call_setitem(self, x): x[None] = self.value @@ -374,7 +354,6 @@ class TestSetValueItemNone1(TestSetValueApi): class TestSetValueItemNone2(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 1] = self.value @@ -383,7 +362,6 @@ class TestSetValueItemNone2(TestSetValueApi): class TestSetValueItemNone3(TestSetValueApi): - def _call_setitem(self, x): x[:, None, None, 1] = self.value @@ -392,7 +370,6 @@ class TestSetValueItemNone3(TestSetValueApi): class TestSetValueItemNone4(TestSetValueApi): - def _call_setitem(self, x): x[0, 0, None, 1] = self.value @@ -401,7 +378,6 @@ class TestSetValueItemNone4(TestSetValueApi): class TestSetValueItemNone5(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 0, None, 1] = self.value @@ -410,7 +386,6 @@ class TestSetValueItemNone5(TestSetValueApi): class TestSetValueItemNone6(TestSetValueApi): - def _call_setitem(self, x): x[None, 0, 0, None, 0] = self.value @@ -419,7 +394,6 @@ class TestSetValueItemNone6(TestSetValueApi): class TestSetValueItemNone7(TestSetValueApi): - def _call_setitem(self, x): x[:, None, 1] = np.zeros(self.shape)[:, None, 0] @@ -428,7 +402,6 @@ class TestSetValueItemNone7(TestSetValueApi): class TestSetValueItemNone8(TestSetValueApi): - def _call_setitem(self, x): x[:, 1, None] = np.zeros(self.shape)[:, 0, None] @@ -437,7 +410,6 @@ class TestSetValueItemNone8(TestSetValueApi): class TestSetValueItemNone9(TestSetValueApi): - def _call_setitem(self, x): x[None, :, 1, ..., None] = np.zeros(self.shape)[0, 0, :, None] @@ -447,7 +419,6 @@ class TestSetValueItemNone9(TestSetValueApi): # 1.5 item is list or Tensor of bol class TestSetValueItemBool1(TestSetValueApi): - def _call_setitem(self, x): x[[True, False]] = self.value @@ -456,7 +427,6 @@ class TestSetValueItemBool1(TestSetValueApi): class TestSetValueItemBool2(TestSetValueApi): - def _call_setitem(self, x): x[[False, False]] = self.value @@ -465,7 +435,6 @@ class TestSetValueItemBool2(TestSetValueApi): class TestSetValueItemBool3(TestSetValueApi): - def _call_setitem(self, x): x[[False, True]] = np.zeros(self.shape[2]) @@ -474,7 +443,6 @@ class TestSetValueItemBool3(TestSetValueApi): class TestSetValueItemBool4(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign(np.array([False, True])) x[idx] = np.zeros(self.shape[2]) @@ -484,19 +452,19 @@ class TestSetValueItemBool4(TestSetValueApi): class TestSetValueItemBool5(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign( - np.array([[False, True, False], [True, True, False]])) + np.array([[False, True, False], [True, True, False]]) + ) x[idx] = self.value def _get_answer(self): - self.data[np.array([[False, True, False], [True, True, - False]])] = self.value + self.data[ + np.array([[False, True, False], [True, True, False]]) + ] = self.value class TestSetValueItemBool6(TestSetValueApi): - def _call_setitem(self, x): x[0, ...] = 0 x[x > 0] = self.value @@ -507,9 +475,7 @@ class TestSetValueItemBool6(TestSetValueApi): def create_test_value_int32(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -529,9 +495,7 @@ create_test_value_int32(TestSetValueItemSlice4) def create_test_value_int64(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -551,9 +515,7 @@ create_test_value_int64(TestSetValueItemSlice4) def create_test_value_tensor_fp32(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "float32" @@ -578,7 +540,6 @@ create_test_value_tensor_fp32(TestSetValueItemSlice4) # 3. Test different shape of value class TestSetValueValueShape1(TestSetValueApi): - def set_value(self): self.value = np.array([3, 4, 5, 6]) # shape is (4,) @@ -590,7 +551,6 @@ class TestSetValueValueShape1(TestSetValueApi): class TestSetValueValueShape2(TestSetValueApi): - def set_value(self): self.value = np.array([[3, 4, 5, 6]]) # shape is (1,4) @@ -602,10 +562,10 @@ class TestSetValueValueShape2(TestSetValueApi): class TestSetValueValueShape3(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, 3]]) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ) # shape is (3,4) def _call_setitem(self, x): x[0] = self.value @@ -615,11 +575,12 @@ class TestSetValueValueShape3(TestSetValueApi): class TestSetValueValueShape4(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, - 3]]).astype(self.dtype) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ).astype( + self.dtype + ) # shape is (3,4) def _call_setitem(self, x): x[0] = paddle.assign(self.value) # x is Paddle.Tensor @@ -629,7 +590,6 @@ class TestSetValueValueShape4(TestSetValueApi): class TestSetValueValueShape5(TestSetValueApi): - def set_value(self): self.value = np.array([3, 3, 3]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sgd_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sgd_op_npu.py index 03c48b415dd..97479538625 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sgd_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sgd_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestSGD(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -54,7 +53,6 @@ class TestSGD(OpTest): class TestNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -69,9 +67,9 @@ class TestNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -95,16 +93,17 @@ class TestNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_shape_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_shape_op_npu.py index 136bedea822..fdee5eced0c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_shape_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_shape_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestShape(OpTest): - def setUp(self): self.set_npu() self.op_type = "shape" @@ -52,25 +51,21 @@ class TestShape(OpTest): class TestShape_fp16(TestShape): - def init_dtype(self): self.dtype = np.float16 class TestShape_double(TestShape): - def init_dtype(self): self.dtype = np.float64 class TestShape_int32(TestShape): - def init_dtype(self): self.dtype = np.int32 class TestShape_int64(TestShape): - def init_dtype(self): self.dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_shard_index_op.py b/python/paddle/fluid/tests/unittests/npu/test_shard_index_op.py index 3ed6051b46f..2fc3d9beeb8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_shard_index_op.py +++ b/python/paddle/fluid/tests/unittests/npu/test_shard_index_op.py @@ -52,13 +52,12 @@ def common_setup(self, index_num, nshards, shard_id, ignore_value): 'index_num': index_num, 'nshards': nshards, 'shard_id': shard_id, - 'ignore_value': ignore_value + 'ignore_value': ignore_value, } self.outputs = {'Out': (out, x_lod)} class TestShardIndexShardId0Op(OpTest): - def setUp(self): common_setup(self, 20, 2, 0, -1) @@ -67,19 +66,16 @@ class TestShardIndexShardId0Op(OpTest): class TestShardIndexShardId1Op(TestShardIndexShardId0Op): - def setUp(self): common_setup(self, 20, 2, 1, -1) class TestShardIndexIgnoreValueOp(TestShardIndexShardId0Op): - def setUp(self): common_setup(self, 20, 2, 0, -2) class TestShardIndexNotEvenlyDividedOp(TestShardIndexShardId0Op): - def setUp(self): common_setup(self, 15, 2, 1, -1) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sigmoid_cross_entropy_with_logits_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sigmoid_cross_entropy_with_logits_op_npu.py index 1454edd7a75..e06c132df25 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sigmoid_cross_entropy_with_logits_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sigmoid_cross_entropy_with_logits_op_npu.py @@ -25,11 +25,11 @@ import paddle paddle.enable_static() -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -39,13 +39,14 @@ class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint(0, 2, (batch_size, num_classes)).astype( + self.dtype + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -70,12 +71,13 @@ class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): self.dtype = np.float32 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") -class TestSigmoidCrossEntropyWithLogitsOp3(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) +class TestSigmoidCrossEntropyWithLogitsOp3( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -85,13 +87,14 @@ class TestSigmoidCrossEntropyWithLogitsOp3(TestSigmoidCrossEntropyWithLogitsOp1 batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.uniform(0, 1, (batch_size, num_classes)).astype( + self.dtype + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -103,12 +106,13 @@ class TestSigmoidCrossEntropyWithLogitsOp3(TestSigmoidCrossEntropyWithLogitsOp1 self.outputs = {'Out': -term1 - term2} -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") -class TestSigmoidCrossEntropyWithLogitsOp5(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) +class TestSigmoidCrossEntropyWithLogitsOp5( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -118,14 +122,14 @@ class TestSigmoidCrossEntropyWithLogitsOp5(TestSigmoidCrossEntropyWithLogitsOp1 batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, tuple(batch_size + [num_classes])).astype( - self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -137,12 +141,13 @@ class TestSigmoidCrossEntropyWithLogitsOp5(TestSigmoidCrossEntropyWithLogitsOp1 self.outputs = {'Out': -term1 - term2} -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") -class TestSigmoidCrossEntropyWithLogitsOp6(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) +class TestSigmoidCrossEntropyWithLogitsOp6( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -152,14 +157,14 @@ class TestSigmoidCrossEntropyWithLogitsOp6(TestSigmoidCrossEntropyWithLogitsOp1 batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, tuple(batch_size + [num_classes])).astype( - self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.randint( + 0, 2, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } # Fw Pass is implemented as elementwise sigmoid followed by diff --git a/python/paddle/fluid/tests/unittests/npu/test_sigmoid_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sigmoid_op_npu.py index 47d881ee013..caac4ed6ba4 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sigmoid_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sigmoid_op_npu.py @@ -23,10 +23,10 @@ paddle.enable_static() SEED = 2021 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestNPUSigmoid(OpTest): - def setUp(self): self.op_type = "sigmoid" self.set_npu() @@ -43,9 +43,9 @@ class TestNPUSigmoid(OpTest): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.01 + ) def set_npu(self): self.__class__.use_npu = True @@ -55,10 +55,10 @@ class TestNPUSigmoid(OpTest): self.dtype = np.float32 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestNPUSigmoidFp16(TestNPUSigmoid): - def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sin_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sin_op_npu.py index 26e6ae9255d..971184956e6 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sin_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sin_op_npu.py @@ -17,7 +17,11 @@ import unittest import numpy as np from scipy.special import expit, erf -from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16, skip_check_grad_ci +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + skip_check_grad_ci, +) import paddle import paddle.nn as nn import paddle.nn.functional as F @@ -29,9 +33,7 @@ paddle.enable_static() def test_class(op_type, typename): - class TestSin(OpTest): - def setUp(self): self.op_type = "sin" self.__class__.use_npu = True @@ -57,7 +59,7 @@ def test_class(op_type, typename): out = eval("paddle.%s(data, name='Y')" % self.op_type) place = fluid.NPUPlace(0) exe = fluid.Executor(place) - result, = exe.run(feed={"X": np_x}, fetch_list=[out]) + (result,) = exe.run(feed={"X": np_x}, fetch_list=[out]) expected = eval("np.%s(np_x)" % self.op_type) self.assertEqual(result, expected) diff --git a/python/paddle/fluid/tests/unittests/npu/test_size_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_size_op_npu.py index e5fd0426742..1e768a5dd18 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_size_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_size_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestSizeOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -48,49 +47,42 @@ class TestSizeOp(OpTest): class TestSizeOp1(TestSizeOp): - def config(self): self.shape = [2] self.dtype = np.float64 class TestSizeOp2(TestSizeOp): - def config(self): self.shape = [2, 3] self.dtype = np.float32 class TestSizeOp3(TestSizeOp): - def config(self): self.shape = [2, 3, 100] self.dtype = np.float16 class TestSizeOp4(TestSizeOp): - def config(self): self.shape = [2**10] self.dtype = np.bool_ class TestSizeOp5(TestSizeOp): - def config(self): self.shape = [7, 8, 9, 10] self.dtype = np.int64 class TestSizeOp6(TestSizeOp): - def config(self): self.shape = [] self.dtype = np.int64 class TestSizeAPI(unittest.TestCase): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -111,17 +103,19 @@ class TestSizeAPI(unittest.TestCase): out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) exe = paddle.static.Executor(place=self.place) - res_1, res_2 = exe.run(feed={ - "x_1": input_1, - "x_2": input_2, - }, - fetch_list=[out_1, out_2]) - assert (np.array_equal(res_1, - np.array([np.size(input_1) - ]).astype("int64"))) - assert (np.array_equal(res_2, - np.array([np.size(input_2) - ]).astype("int64"))) + res_1, res_2 = exe.run( + feed={ + "x_1": input_1, + "x_2": input_2, + }, + fetch_list=[out_1, out_2], + ) + assert np.array_equal( + res_1, np.array([np.size(input_1)]).astype("int64") + ) + assert np.array_equal( + res_2, np.array([np.size(input_2)]).astype("int64") + ) def test_size_imperative(self): paddle.disable_static(self.place) @@ -131,8 +125,8 @@ class TestSizeAPI(unittest.TestCase): x_2 = paddle.to_tensor(input_2) out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) - assert (np.array_equal(out_1.numpy().item(0), np.size(input_1))) - assert (np.array_equal(out_2.numpy().item(0), np.size(input_2))) + assert np.array_equal(out_1.numpy().item(0), np.size(input_1)) + assert np.array_equal(out_2.numpy().item(0), np.size(input_2)) paddle.enable_static() def test_error(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_slice_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_slice_op_npu.py index ba2b45ab9f4..5bc1700cc16 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_slice_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_slice_op_npu.py @@ -27,7 +27,6 @@ EPOCH = 100 class TestSliceOp(OpTest): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -39,7 +38,7 @@ class TestSliceOp(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -65,15 +64,14 @@ class TestSliceOp(OpTest): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.02) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.02 + ) else: self.check_grad_with_place(self.place, ['Input'], 'Out') class TestSliceOp2(TestSliceOp): - def config(self): self.input = np.random.random([10, 5, 6]).astype(self.dtype) self.starts = [0] @@ -84,7 +82,6 @@ class TestSliceOp2(TestSliceOp): class TestSliceOpFp16(TestSliceOp): - def init_dtype(self): self.dtype = np.float16 @@ -95,7 +92,6 @@ class TestSliceOpFp16(TestSliceOp): class TestSliceOpTensor(TestSliceOp): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -104,14 +100,14 @@ class TestSliceOpTensor(TestSliceOp): self.inputs = { 'Input': self.input, 'StartsTensor': self.starts, - 'EndsTensor': self.ends + 'EndsTensor': self.ends, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1, -1, -1], 'ends': [-1, -1, -1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -124,7 +120,6 @@ class TestSliceOpTensor(TestSliceOp): class TestSliceOpTensor2(TestSliceOpTensor): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -133,14 +128,14 @@ class TestSliceOpTensor2(TestSliceOpTensor): self.inputs = { 'Input': self.input, 'StartsTensor': self.starts, - 'EndsTensor': self.ends + 'EndsTensor': self.ends, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1], 'ends': [-1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -153,7 +148,6 @@ class TestSliceOpTensor2(TestSliceOpTensor): class TestSliceOpFp16Tensor(TestSliceOpTensor): - def init_dtype(self): self.dtype = np.float16 @@ -164,7 +158,6 @@ class TestSliceOpFp16Tensor(TestSliceOpTensor): class TestSliceOpTensorList(TestSliceOp): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -173,25 +166,27 @@ class TestSliceOpTensorList(TestSliceOp): self.starts_tensor_list = [] for index, ele in enumerate(self.starts): - self.starts_tensor_list.append(("start" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.starts_tensor_list.append( + ("start" + str(index), np.ones((1)).astype('int32') * ele) + ) self.ends_tensor_list = [] for index, ele in enumerate(self.ends): - self.ends_tensor_list.append(("end" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.ends_tensor_list.append( + ("end" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, 'StartsTensorList': self.starts_tensor_list, - 'EndsTensorList': self.ends_tensor_list + 'EndsTensorList': self.ends_tensor_list, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1, -1, -1], 'ends': [-1, -1, -1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -204,7 +199,6 @@ class TestSliceOpTensorList(TestSliceOp): class TestSliceOpTensorList2(TestSliceOpTensorList): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -213,25 +207,27 @@ class TestSliceOpTensorList2(TestSliceOpTensorList): self.starts_tensor_list = [] for index, ele in enumerate(self.starts): - self.starts_tensor_list.append(("start" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.starts_tensor_list.append( + ("start" + str(index), np.ones((1)).astype('int32') * ele) + ) self.ends_tensor_list = [] for index, ele in enumerate(self.ends): - self.ends_tensor_list.append(("end" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.ends_tensor_list.append( + ("end" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, 'StartsTensorList': self.starts_tensor_list, - 'EndsTensorList': self.ends_tensor_list + 'EndsTensorList': self.ends_tensor_list, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1], 'ends': [-1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -244,7 +240,6 @@ class TestSliceOpTensorList2(TestSliceOpTensorList): class TestSliceOpFp16TensorList(TestSliceOpTensorList): - def init_dtype(self): self.dtype = np.float16 @@ -255,7 +250,6 @@ class TestSliceOpFp16TensorList(TestSliceOpTensorList): class TestSliceNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -272,9 +266,9 @@ class TestSliceNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=data_shape, dtype='float32') b = paddle.static.data(name="b", shape=data_shape, dtype='float32') - label = paddle.static.data(name="label", - shape=[batch_size, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[batch_size, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.slice(sum, axes=[0, 1], starts=[0, 0], ends=[33, 2]) @@ -282,7 +276,8 @@ class TestSliceNet(unittest.TestCase): prediction = paddle.static.nn.fc(z, size=2, activation='softmax') cost = paddle.fluid.layers.softmax_with_cross_entropy( - logits=prediction, label=label) + logits=prediction, label=label + ) loss = paddle.mean(cost) sgd = paddle.optimizer.SGD(learning_rate=0.01) sgd.minimize(loss) @@ -297,16 +292,17 @@ class TestSliceNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(EPOCH): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -319,7 +315,6 @@ class TestSliceNet(unittest.TestCase): class TestSliceOpDecsDim(OpTest): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -365,21 +360,19 @@ class TestSliceOpDecsDim(OpTest): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place(self.place, ['Input'], 'Out') class TestSliceOpDecsDimFp16(TestSliceOpDecsDim): - def init_dtype(self): self.dtype = np.float16 class TestSliceOpDecsDim2(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [1, 0, 2] @@ -391,7 +384,6 @@ class TestSliceOpDecsDim2(TestSliceOpDecsDim): class TestSliceOpDecsDim3(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1, 0, 2] @@ -403,7 +395,6 @@ class TestSliceOpDecsDim3(TestSliceOpDecsDim): class TestSliceOpDecsDim4(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype(self.dtype) self.starts = [0, 1, 2, 3] @@ -415,7 +406,6 @@ class TestSliceOpDecsDim4(TestSliceOpDecsDim): class TestSliceOpDecsDim5(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1] @@ -427,7 +417,6 @@ class TestSliceOpDecsDim5(TestSliceOpDecsDim): class TestSliceOpDecsDim6(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [0, 1, 2, 3] @@ -439,11 +428,10 @@ class TestSliceOpDecsDim6(TestSliceOpDecsDim): class TestSliceOpDecsDimStartsTensor(TestSliceOpDecsDim): - def set_inputs(self): self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype='int32') + "StartsTensor": np.array(self.starts, dtype='int32'), } def set_attrs(self): @@ -466,18 +454,16 @@ class TestSliceOpDecsDimStartsTensor(TestSliceOpDecsDim): class TestSliceOpDecsDimStartsTensorFP16(TestSliceOpDecsDimStartsTensor): - def init_dtype(self): self.dtype = np.float16 class TestSliceOpDecsDimStartsTensorStartsAndEndsTensor(TestSliceOpDecsDim): - def set_inputs(self): self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype='int64'), - "EndsTensor": np.array(self.ends, dtype='int32') + "EndsTensor": np.array(self.ends, dtype='int32'), } def set_attrs(self): @@ -500,19 +486,19 @@ class TestSliceOpDecsDimStartsTensorStartsAndEndsTensor(TestSliceOpDecsDim): class TestSliceOpDecsDimStartsTensorStartsAndEndsTensorFP16( - TestSliceOpDecsDimStartsTensorStartsAndEndsTensor): - + TestSliceOpDecsDimStartsTensorStartsAndEndsTensor +): def init_dtype(self): self.dtype = np.float16 class TestSliceOpDecsDimStartsListTensor(TestSliceOpDecsDim): - def set_inputs(self): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} @@ -538,7 +524,6 @@ class TestSliceOpDecsDimStartsListTensor(TestSliceOpDecsDim): class TestSliceOpDecsDimStartsListTensor2(TestSliceOpDecsDimStartsListTensor): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1] @@ -551,15 +536,14 @@ class TestSliceOpDecsDimStartsListTensor2(TestSliceOpDecsDimStartsListTensor): self.starts_infer = [-1] -class TestSliceOpDecsDimStartsListTensorFP16(TestSliceOpDecsDimStartsListTensor - ): - +class TestSliceOpDecsDimStartsListTensorFP16( + TestSliceOpDecsDimStartsListTensor +): def init_dtype(self): self.dtype = np.float16 class TestSliceOpInt64(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -575,12 +559,13 @@ class TestSliceOpInt64(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): - self.input = np.random.randint(100, - size=(3, 4, 5, 6)).astype(self.dtype) + self.input = np.random.randint(100, size=(3, 4, 5, 6)).astype( + self.dtype + ) self.starts = [1, 0, 2] self.ends = [3, 3, 4] self.axes = [0, 1, 2] @@ -595,7 +580,6 @@ class TestSliceOpInt64(OpTest): class TestSliceOpTensorInt64(TestSliceOpInt64): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -604,19 +588,20 @@ class TestSliceOpTensorInt64(TestSliceOpInt64): self.inputs = { 'Input': self.input, 'StartsTensor': self.starts, - 'EndsTensor': self.ends + 'EndsTensor': self.ends, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1, -1, -1], 'ends': [-1, -1, -1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): - self.input = np.random.randint(100, - size=(3, 4, 5, 6)).astype(self.dtype) + self.input = np.random.randint(100, size=(3, 4, 5, 6)).astype( + self.dtype + ) self.starts = np.array([1, 0, 2]).astype('int32') self.ends = np.array([3, 3, 4]).astype('int32') self.axes = [0, 1, 2] diff --git a/python/paddle/fluid/tests/unittests/npu/test_smooth_l1_loss_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_smooth_l1_loss_op_npu.py index f133e7865cc..54e625a9f4a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_smooth_l1_loss_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_smooth_l1_loss_op_npu.py @@ -33,7 +33,6 @@ def smooth_l1_loss_forward(val, sigma2): class TestSmoothL1LossOp1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -41,7 +40,7 @@ class TestSmoothL1LossOp1(OpTest): dims = (5, 20) self.inputs = { 'X': np.random.random(dims).astype("float32"), - 'Y': np.random.random(dims).astype("float32") + 'Y': np.random.random(dims).astype("float32"), } sigma = 3.0 self.attrs = {'sigma': sigma} @@ -51,7 +50,7 @@ class TestSmoothL1LossOp1(OpTest): loss = loss.reshape((dims[0], 1)) self.outputs = { 'Diff': diff.astype('float32'), - 'Out': loss.astype('float32') + 'Out': loss.astype('float32'), } def set_npu(self): @@ -61,25 +60,30 @@ class TestSmoothL1LossOp1(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.02) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.02 + ) def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.03, - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.03, - no_grad_set=set('Y')) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set('Y'), + ) class TestSmoothL1LossOp2(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -89,7 +93,7 @@ class TestSmoothL1LossOp2(OpTest): 'X': np.random.random(dims).astype("float32"), 'Y': np.random.random(dims).astype("float32"), 'InsideWeight': np.random.random(dims).astype("float32"), - 'OutsideWeight': np.random.random(dims).astype("float32") + 'OutsideWeight': np.random.random(dims).astype("float32"), } sigma = 3.0 self.attrs = {'sigma': sigma} @@ -101,7 +105,7 @@ class TestSmoothL1LossOp2(OpTest): loss = loss.sum(1).reshape((dims[0], 1)) self.outputs = { 'Diff': diff.astype('float32'), - 'Out': loss.astype('float32') + 'Out': loss.astype('float32'), } def set_npu(self): @@ -111,34 +115,39 @@ class TestSmoothL1LossOp2(OpTest): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.03) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.03 + ) def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.03, - no_grad_set=set( - ['X', 'InsideWeight', 'OutsideWeight'])) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight']), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.03, - no_grad_set=set( - ['Y', 'InsideWeight', 'OutsideWeight'])) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight']), + ) class TestSmoothL1LossOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.NPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.NPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.smooth_l1, x1, y1) # The input dtype of accuracy_op must be float32 or float64. x2 = fluid.layers.data(name='x2', shape=[4], dtype="int32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_softmax_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_softmax_op_npu.py index 1c145c49721..41ccda3dba7 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_softmax_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_softmax_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestSoftmax(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -53,7 +52,6 @@ class TestSoftmax(OpTest): class TestSoftmaxNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -68,9 +66,9 @@ class TestSoftmaxNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[4, 32], dtype='float32') b = paddle.static.data(name="b", shape=[4, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[4, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[4, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.sqrt(c) @@ -99,16 +97,17 @@ class TestSoftmaxNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_softmax_with_cross_entropy_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_softmax_with_cross_entropy_op_npu.py index 79654bf170e..f19e892f9a3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_softmax_with_cross_entropy_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_softmax_with_cross_entropy_op_npu.py @@ -28,7 +28,6 @@ SEED = 2021 class TestSoftmaxWithCrossEntropyOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -51,8 +50,10 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): self.initParams() logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, logits) if self.soft_label: @@ -63,8 +64,9 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): self.shape[self.axis] = 1 labels = np.random.randint(0, axis_dim, self.shape, dtype="int64") - loss = cross_entropy(softmax, labels, self.soft_label, self.axis, - self.ignore_index) + loss = cross_entropy( + softmax, labels, self.soft_label, self.axis, self.ignore_index + ) one_hot_label = np.eye(axis_dim)[labels.reshape(-1)] @@ -72,7 +74,7 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): self.outputs = { "Backprop": (softmax - one_hot_label).astype(self.dtype), "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -88,14 +90,16 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): def test_check_grad(self): # fp32 has low precision, cpu and npu both need to relax the max_relative_error if using fp32 - self.check_grad_with_place(self.place, ['Logits'], - 'Loss', - numeric_grad_delta=0.001, - max_relative_error=0.5) + self.check_grad_with_place( + self.place, + ['Logits'], + 'Loss', + numeric_grad_delta=0.001, + max_relative_error=0.5, + ) class TestPowNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -110,9 +114,9 @@ class TestPowNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -136,16 +140,17 @@ class TestPowNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_split_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_split_op_npu.py index 7550735c496..d6a6cf9d573 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_split_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_split_op_npu.py @@ -26,10 +26,10 @@ paddle.enable_static() SEED = 2021 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestCase1(OpTest): - def setUp(self): self.set_npu() self.set_example() @@ -37,9 +37,9 @@ class TestCase1(OpTest): self.place = paddle.NPUPlace(0) ipt = self.x.astype(self.dtype) axis = self.axis if isinstance(self.axis, int) else int(self.axis[0]) - tmp_outs = np.split(ipt, - axis=axis, - indices_or_sections=self.num_or_sections) + tmp_outs = np.split( + ipt, axis=axis, indices_or_sections=self.num_or_sections + ) tmp_outs = [o.astype(self.dtype) for o in tmp_outs] self.outputs = {'Out': []} self.outs = [] @@ -69,7 +69,6 @@ class TestCase1(OpTest): class TestCase2(TestCase1): - def set_example(self): self.dtype = "float32" self.x = np.random.random((20, 4, 50)) @@ -78,7 +77,6 @@ class TestCase2(TestCase1): class TestCase4(TestCase1): - def set_example(self): self.dtype = "float16" self.x = np.random.random((4, 50, 20)) @@ -88,7 +86,6 @@ class TestCase4(TestCase1): # Test Sections class TestCase5(TestCase1): - def set_example(self): super().set_example() self.x = np.random.random((2, 10, 4)) @@ -101,7 +98,6 @@ class TestCase5(TestCase1): class API_TestSplit(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float32') @@ -110,13 +106,12 @@ class API_TestSplit(unittest.TestCase): exe = fluid.Executor(place) input1 = np.random.random([1, 10]).astype('float32') r0, r1 = exe.run(feed={"data": input1}, fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (3, ), axis=1) + ex_x0, ex_x1 = np.split(input1, (3,), axis=1) np.testing.assert_allclose(ex_x0, r0) np.testing.assert_allclose(ex_x1, r1) class API_TestSplit2(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float32') @@ -131,7 +126,6 @@ class API_TestSplit2(unittest.TestCase): class API_TestDygraphSplit(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(paddle.NPUPlace(0)): input_1 = np.random.random([4, 6, 6]).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_sqrt_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sqrt_op_npu.py index 5aca464aeba..d28f67a51e3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sqrt_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sqrt_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestSqrt(OpTest): - def setUp(self): self.set_npu() self.op_type = "sqrt" @@ -58,7 +57,6 @@ class TestSqrt(OpTest): class TestSqrtFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "sqrt" @@ -85,7 +83,6 @@ class TestSqrtFp16(OpTest): class TestSqrtNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -100,9 +97,9 @@ class TestSqrtNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.sqrt(c) @@ -126,16 +123,17 @@ class TestSqrtNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_square_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_square_op_npu.py index 86849cdb9c2..f6dbefee32a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_square_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_square_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestSquare(OpTest): - def setUp(self): self.set_npu() self.op_type = "square" @@ -55,7 +54,6 @@ class TestSquare(OpTest): class TestSquareFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "square" @@ -82,7 +80,6 @@ class TestSquareFp16(OpTest): class TestSquareNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -97,9 +94,9 @@ class TestSquareNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.square(c) @@ -123,16 +120,17 @@ class TestSquareNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_squared_l2_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_squared_l2_norm_op_npu.py index 6c4f66958ac..ac1e57f2959 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_squared_l2_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_squared_l2_norm_op_npu.py @@ -25,8 +25,7 @@ paddle.enable_static() class TestL2LossOp(OpTest): - """Test npu squared_l2_norm - """ + """Test npu squared_l2_norm""" def setUp(self): self.set_npu() @@ -46,9 +45,9 @@ class TestL2LossOp(OpTest): self.check_output_with_place(place=self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=self.max_relative_error) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=self.max_relative_error + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/npu/test_squeeze_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_squeeze_op_npu.py index 372c13fb900..35940f62b83 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_squeeze_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_squeeze_op_npu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestSqueezeOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "squeeze" @@ -62,7 +61,6 @@ class TestSqueezeOp(OpTest): class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 3, 1, 40) self.axes = (0, -2) @@ -73,7 +71,6 @@ class TestSqueezeOp1(TestSqueezeOp): class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -84,7 +81,6 @@ class TestSqueezeOp2(TestSqueezeOp): class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -95,7 +91,6 @@ class TestSqueezeOp3(TestSqueezeOp): class TestSqueezeOp4(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, 2) @@ -103,13 +98,13 @@ class TestSqueezeOp4(TestSqueezeOp): class TestSqueezeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input type of softmax_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.NPUPlace(0) + ) self.assertRaises(TypeError, paddle.squeeze, x1) # The input axes of squeeze must be list. x2 = paddle.static.data(name='x2', shape=[4], dtype="int32") @@ -120,7 +115,6 @@ class TestSqueezeOpError(unittest.TestCase): class API_TestSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -129,29 +123,29 @@ class API_TestSqueeze(unittest.TestCase): def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - data1 = paddle.static.data('data1', - shape=[-1, 1, 10], - dtype='float64') + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + data1 = paddle.static.data( + 'data1', shape=[-1, 1, 10], dtype='float64' + ) result_squeeze = self.squeeze(data1, axis=[1]) place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) input1 = np.random.random([5, 1, 10]).astype('float64') - result, = exe.run(feed={"data1": input1}, - fetch_list=[result_squeeze]) + (result,) = exe.run( + feed={"data1": input1}, fetch_list=[result_squeeze] + ) expected_result = np.squeeze(input1, axis=1) np.testing.assert_allclose(expected_result, result) class API_TestStaticSqueeze_(API_TestSqueeze): - def executed_api(self): self.squeeze = paddle.squeeze_ class API_TestDygraphSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -210,14 +204,12 @@ class API_TestDygraphSqueeze(unittest.TestCase): class API_TestDygraphSqueezeInplace(API_TestDygraphSqueeze): - def executed_api(self): self.squeeze = paddle.squeeze_ # Correct: General. class TestSqueeze2Op(OpTest): - def setUp(self): self.set_npu() self.op_type = "squeeze2" @@ -226,15 +218,16 @@ class TestSqueeze2Op(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_npu(self): self.__class__.use_npu = True def test_check_output(self): - self.check_output_with_place(paddle.NPUPlace(0), - no_check_set=['XShape']) + self.check_output_with_place( + paddle.NPUPlace(0), no_check_set=['XShape'] + ) def test_check_grad(self): self.check_grad_with_place(paddle.NPUPlace(0), ["X"], "Out") @@ -250,7 +243,6 @@ class TestSqueeze2Op(OpTest): # Correct: There is mins axis. class TestSqueeze2Op1(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -259,7 +251,6 @@ class TestSqueeze2Op1(TestSqueeze2Op): # Correct: No axes input. class TestSqueeze2Op2(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -268,7 +259,6 @@ class TestSqueeze2Op2(TestSqueeze2Op): # Correct: Just part of axes be squeezed. class TestSqueeze2Op3(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) diff --git a/python/paddle/fluid/tests/unittests/npu/test_stack_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_stack_op_npu.py index 5f768c78af7..8a9e8879b82 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_stack_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_stack_op_npu.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestStackOpBase(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -50,7 +49,8 @@ class TestStackOpBase(OpTest): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(self.dtype)) + np.random.random(size=self.input_dim).astype(self.dtype) + ) tmp = [] x_names = self.get_x_names() @@ -78,49 +78,41 @@ class TestStackOpBase(OpTest): class TestStackOp1(TestStackOpBase): - def initParameters(self): self.num_inputs = 16 class TestStackOp2(TestStackOpBase): - def initParameters(self): self.num_inputs = 20 class TestStackOp3(TestStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestStackOpBase): - def initParameters(self): self.axis = -4 class TestStackOp5(TestStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestStackOpBase): - def initParameters(self): self.axis = 3 class TestStackOpINT32(TestStackOpBase): - def init_dtype(self): self.dtype = np.int32 class TestStackOpINT64(TestStackOpBase): - def init_dtype(self): self.dtype = np.int64 @@ -135,8 +127,11 @@ class TestStackAPIWithLoDTensorArray(unittest.TestCase): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = paddle.NPUPlace(0) \ - if paddle.is_compiled_with_npu() else paddle.CPUPlace() + self.place = ( + paddle.NPUPlace(0) + if paddle.is_compiled_with_npu() + else paddle.CPUPlace() + ) self.set_program() def set_program(self): @@ -156,7 +151,8 @@ class TestStackAPIWithLoDTensorArray(unittest.TestCase): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_allclose( - res[0], np.stack([self.x] * self.iter_num, axis=self.axis)) + res[0], np.stack([self.x] * self.iter_num, axis=self.axis) + ) class TestTensorStackAPIWithLoDTensorArray(unittest.TestCase): @@ -169,8 +165,11 @@ class TestTensorStackAPIWithLoDTensorArray(unittest.TestCase): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = paddle.NPUPlace(0) \ - if paddle.is_compiled_with_npu() else paddle.CPUPlace() + self.place = ( + paddle.NPUPlace(0) + if paddle.is_compiled_with_npu() + else paddle.CPUPlace() + ) self.set_program() def set_program(self): @@ -190,11 +189,11 @@ class TestTensorStackAPIWithLoDTensorArray(unittest.TestCase): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_allclose( - res[0], np.stack([self.x] * self.iter_num, axis=self.axis)) + res[0], np.stack([self.x] * self.iter_num, axis=self.axis) + ) class API_test(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[1, 2], dtype='float32') @@ -206,12 +205,10 @@ class API_test(unittest.TestCase): input1 = np.random.random([1, 2]).astype('float32') input2 = np.random.random([1, 2]).astype('float32') input3 = np.random.random([1, 2]).astype('float32') - result, = exe.run(feed={ - "data1": input1, - "data2": input2, - "data3": input3 - }, - fetch_list=[result_stack]) + (result,) = exe.run( + feed={"data1": input1, "data2": input2, "data3": input3}, + fetch_list=[result_stack], + ) expected_result = np.stack([input1, input2, input3], axis=0) np.testing.assert_allclose(expected_result, result) @@ -222,7 +219,6 @@ class API_test(unittest.TestCase): class API_DygraphTest(unittest.TestCase): - def test_out(self): data1 = np.array([[1.0, 2.0]]) data2 = np.array([[3.0, 4.0]]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_strided_slice_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_strided_slice_op_npu.py index bf32653455c..6dcd8fcdae2 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_strided_slice_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_strided_slice_op_npu.py @@ -40,33 +40,52 @@ def strided_slice_native_forward(input, axes, starts, ends, strides): stride[axes[i]] = strides[i] result = { - 1: lambda input, start, end, stride: input[start[0]:end[0]:stride[0]], - 2: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1]], - 3: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2]], - 4: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3]], - 5: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3], start[4]:end[4]:stride[4]], - 6: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3], \ - start[4]:end[4]:stride[4], start[5]:end[5]:stride[5]] + 1: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0] + ], + 2: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], start[1] : end[1] : stride[1] + ], + 3: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + ], + 4: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + ], + 5: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + start[4] : end[4] : stride[4], + ], + 6: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + start[4] : end[4] : stride[4], + start[5] : end[5] : stride[5], + ], }[dim](input, start, end, stride) return result class TestStridedSliceOp(OpTest): - def setUp(self): self.initTestCase() self.set_npu() self.place = paddle.NPUPlace(0) self.op_type = 'strided_slice' - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.inputs = {'Input': self.input} self.outputs = {'Out': self.output} @@ -75,7 +94,7 @@ class TestStridedSliceOp(OpTest): 'starts': self.starts, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def set_npu(self): @@ -97,7 +116,6 @@ class TestStridedSliceOp(OpTest): class TestStridedSliceOp1(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -108,7 +126,6 @@ class TestStridedSliceOp1(TestStridedSliceOp): class TestStridedSliceOp2(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -119,7 +136,6 @@ class TestStridedSliceOp2(TestStridedSliceOp): class TestStridedSliceOp3(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -130,7 +146,6 @@ class TestStridedSliceOp3(TestStridedSliceOp): class TestStridedSliceOp4(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 4, 10) self.axes = [0, 1, 2] @@ -141,7 +156,6 @@ class TestStridedSliceOp4(TestStridedSliceOp): class TestStridedSliceOp5(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -152,7 +166,6 @@ class TestStridedSliceOp5(TestStridedSliceOp): class TestStridedSliceOp6(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -163,7 +176,6 @@ class TestStridedSliceOp6(TestStridedSliceOp): class TestStridedSliceOp7(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -174,7 +186,6 @@ class TestStridedSliceOp7(TestStridedSliceOp): class TestStridedSliceOp8(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1) self.axes = [1] @@ -185,7 +196,6 @@ class TestStridedSliceOp8(TestStridedSliceOp): class TestStridedSliceOp9(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1) self.axes = [1] @@ -196,7 +206,6 @@ class TestStridedSliceOp9(TestStridedSliceOp): class TestStridedSliceOp10(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(10, 10) self.axes = [0, 1] @@ -207,7 +216,6 @@ class TestStridedSliceOp10(TestStridedSliceOp): class TestStridedSliceOp11(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4) self.axes = [0, 1, 2, 3] @@ -218,7 +226,6 @@ class TestStridedSliceOp11(TestStridedSliceOp): class TestStridedSliceOp12(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5) self.axes = [0, 1, 2, 3, 4] @@ -229,7 +236,6 @@ class TestStridedSliceOp12(TestStridedSliceOp): class TestStridedSliceOp13(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8) self.axes = [0, 1, 2, 3, 4, 5] @@ -240,13 +246,11 @@ class TestStridedSliceOp13(TestStridedSliceOp): class TestStridedSliceOpBool(TestStridedSliceOp): - def test_check_grad(self): pass class TestStridedSliceOpBool1D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(100).astype("bool") self.axes = [0] @@ -257,7 +261,6 @@ class TestStridedSliceOpBool1D(TestStridedSliceOpBool): class TestStridedSliceOpBool2D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(10, 10).astype("bool") self.axes = [0, 1] @@ -268,7 +271,6 @@ class TestStridedSliceOpBool2D(TestStridedSliceOpBool): class TestStridedSliceOpBool3D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 4, 10).astype("bool") self.axes = [0, 1, 2] @@ -279,7 +281,6 @@ class TestStridedSliceOpBool3D(TestStridedSliceOpBool): class TestStridedSliceOpBool4D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4).astype("bool") self.axes = [0, 1, 2, 3] @@ -290,7 +291,6 @@ class TestStridedSliceOpBool4D(TestStridedSliceOpBool): class TestStridedSliceOpBool5D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5).astype("bool") self.axes = [0, 1, 2, 3, 4] @@ -301,7 +301,6 @@ class TestStridedSliceOpBool5D(TestStridedSliceOpBool): class TestStridedSliceOpBool6D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype("bool") self.axes = [0, 1, 2, 3, 4, 5] @@ -312,7 +311,6 @@ class TestStridedSliceOpBool6D(TestStridedSliceOpBool): class TestStridedSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -321,8 +319,9 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.output} @@ -331,7 +330,7 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): 'starts': self.starts_infer, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def set_npu(self): @@ -344,9 +343,9 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.starts_infer = [1, 10, 2] @@ -358,7 +357,6 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): class TestStridedSliceOp_ends_ListTensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -367,8 +365,9 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'EndsTensorList': ends_tensor} self.outputs = {'Out': self.output} @@ -377,7 +376,7 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): 'starts': self.starts, 'ends': self.ends_infer, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def set_npu(self): @@ -390,9 +389,9 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 2] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.ends_infer = [3, 1, 4] @@ -404,7 +403,6 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): class TestStridedSliceOp_starts_Tensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -413,7 +411,7 @@ class TestStridedSliceOp_starts_Tensor(OpTest): self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -434,9 +432,9 @@ class TestStridedSliceOp_starts_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) @@ -446,7 +444,6 @@ class TestStridedSliceOp_starts_Tensor(OpTest): class TestStridedSliceOp_ends_Tensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -455,7 +452,7 @@ class TestStridedSliceOp_ends_Tensor(OpTest): self.inputs = { 'Input': self.input, - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -476,9 +473,9 @@ class TestStridedSliceOp_ends_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) @@ -488,7 +485,6 @@ class TestStridedSliceOp_ends_Tensor(OpTest): class TestStridedSliceOp_listTensor_Tensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -497,13 +493,14 @@ class TestStridedSliceOp_listTensor_Tensor(OpTest): ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensorList": ends_tensor + "EndsTensorList": ends_tensor, } self.outputs = {'Out': self.output} self.attrs = { @@ -524,9 +521,9 @@ class TestStridedSliceOp_listTensor_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) @@ -536,7 +533,6 @@ class TestStridedSliceOp_listTensor_Tensor(OpTest): class TestStridedSliceOp_strides_Tensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -545,7 +541,7 @@ class TestStridedSliceOp_strides_Tensor(OpTest): self.inputs = { 'Input': self.input, - "StridesTensor": np.array(self.strides, dtype="int32") + "StridesTensor": np.array(self.strides, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -566,9 +562,9 @@ class TestStridedSliceOp_strides_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, -1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) @@ -576,51 +572,54 @@ class TestStridedSliceOp_strides_Tensor(OpTest): def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['Input'], 'Out') - # Test python API -class TestStridedSliceAPI(unittest.TestCase): + +class TestStridedSliceAPI(unittest.TestCase): def test_1(self): input = np.random.random([3, 4, 5, 6]).astype("float64") minus_1 = fluid.layers.fill_constant([1], "int32", -1) minus_3 = fluid.layers.fill_constant([1], "int32", -3) - starts = fluid.layers.data(name='starts', - shape=[3], - dtype='int32', - append_batch_size=False) - ends = fluid.layers.data(name='ends', - shape=[3], - dtype='int32', - append_batch_size=False) - strides = fluid.layers.data(name='strides', - shape=[3], - dtype='int32', - append_batch_size=False) - - x = fluid.layers.data(name="x", - shape=[3, 4, 5, 6], - append_batch_size=False, - dtype="float64") - out_1 = fluid.layers.strided_slice(x, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_2 = fluid.layers.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_3 = fluid.layers.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, minus_1], - strides=[1, 1, 1]) - out_4 = fluid.layers.strided_slice(x, - axes=[0, 1, 2], - starts=starts, - ends=ends, - strides=strides) + starts = fluid.layers.data( + name='starts', shape=[3], dtype='int32', append_batch_size=False + ) + ends = fluid.layers.data( + name='ends', shape=[3], dtype='int32', append_batch_size=False + ) + strides = fluid.layers.data( + name='strides', shape=[3], dtype='int32', append_batch_size=False + ) + + x = fluid.layers.data( + name="x", + shape=[3, 4, 5, 6], + append_batch_size=False, + dtype="float64", + ) + out_1 = fluid.layers.strided_slice( + x, + axes=[0, 1, 2], + starts=[-3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_2 = fluid.layers.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_3 = fluid.layers.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, minus_1], + strides=[1, 1, 1], + ) + out_4 = fluid.layers.strided_slice( + x, axes=[0, 1, 2], starts=starts, ends=ends, strides=strides + ) out_5 = x[-3:3, 0:100:2, -1:2:-1] out_6 = x[minus_3:3:1, 0:100:2, :, minus_1:2:minus_1] @@ -633,9 +632,10 @@ class TestStridedSliceAPI(unittest.TestCase): "x": input, 'starts': np.array([-3, 0, 2]).astype("int32"), 'ends': np.array([3, 2147483648, -1]).astype("int64"), - 'strides': np.array([1, 1, 1]).astype("int32") + 'strides': np.array([1, 1, 1]).astype("int32"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, input[-3:3, 0:100, 2:-1, :]) assert np.array_equal(res_2, input[-3:3, 0:100, :, 2:-1]) assert np.array_equal(res_3, input[-3:3, 0:100, :, 2:-1]) @@ -650,11 +650,9 @@ class TestStridedSliceAPI(unittest.TestCase): starts = [-3, 0, 2] ends = [3, 2, 4] strides_1 = [1, 1, 1] - sliced_1 = paddle.strided_slice(x, - axes=axes, - starts=starts, - ends=ends, - strides=strides_1) + sliced_1 = paddle.strided_slice( + x, axes=axes, starts=starts, ends=ends, strides=strides_1 + ) assert sliced_1.shape == (3, 2, 2, 2) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sum_op_npu.py index bda9ebe9d11..8471fa09e20 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_sum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sum_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestSum1(OpTest): - def setUp(self): self.set_npu() self.init_dtype() @@ -54,7 +53,6 @@ class TestSum1(OpTest): class TestSum2(OpTest): - def setUp(self): self.set_npu() self.init_dtype() @@ -72,8 +70,12 @@ class TestSum2(OpTest): # For example, the results of `x0+x1+x2+x3` is different from that of # `x3+x2+x1+x0` if the dtype is fp16. # Therefore, converting the input to fp32 for calculation. - y = (x0.astype(np.float32) + x1.astype(np.float32) + - x2.astype(np.float32) + x3.astype(np.float32)).astype(self.dtype) + y = ( + x0.astype(np.float32) + + x1.astype(np.float32) + + x2.astype(np.float32) + + x3.astype(np.float32) + ).astype(self.dtype) self.outputs = {'Out': y} self.attrs = {'use_mkldnn': False} @@ -89,7 +91,6 @@ class TestSum2(OpTest): class TestSum3(OpTest): - def setUp(self): self.set_npu() self.init_dtype() diff --git a/python/paddle/fluid/tests/unittests/npu/test_swish_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_swish_op_npu.py index 9c45c84c43b..d967e1fc06d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_swish_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_swish_op_npu.py @@ -27,7 +27,6 @@ SEED = 1024 class TestSwishOp(OpTest): - def setUp(self): self.op_type = "swish" self.set_npu() @@ -49,10 +48,13 @@ class TestSwishOp(OpTest): dx = beta * out + expit(x) * (1 - beta * out) dx = dx / x.size - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01, - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.01, + user_defined_grads=[dx], + ) def set_npu(self): self.__class__.use_npu = True @@ -63,7 +65,6 @@ class TestSwishOp(OpTest): class TestSwishOpFp16(TestSwishOp): - def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_base_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_base_npu.py index 30aa5941143..1629c8e8fb2 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_base_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_base_npu.py @@ -30,9 +30,14 @@ import paddle.fluid.unique_name as nameGen from paddle.fluid import core import paddle -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) -from paddle.fluid.tests.unittests.test_sync_batch_norm_op import create_or_get_tensor +from paddle.fluid.tests.unittests.test_sync_batch_norm_op import ( + create_or_get_tensor, +) _set_use_system_allocator(False) paddle.enable_static() @@ -41,17 +46,19 @@ SEED = 10 class TestSyncBatchNormRunnerBase(object): - - def get_model(self, - main, - startup, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def get_model( + self, + main, + startup, + place, + layout, + seed, + sync_bn=False, + only_forward=False, + ): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): assert not isinstance(endpoints, str) @@ -60,13 +67,15 @@ class TestSyncBatchNormRunnerBase(object): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -74,43 +83,50 @@ class TestSyncBatchNormRunnerBase(object): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - hccl_id_var = block.create_var(name=nameGen.generate('hccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': hccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': hccl_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': self.global_ring_id, - 'device_id': int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks - }) + hccl_id_var = block.create_var( + name=nameGen.generate('hccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': hccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': hccl_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': self.global_ring_id, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + }, + ) def run_trainer(self, args): device_id = int(os.getenv("FLAGS_selected_npus", "0")) @@ -143,24 +159,30 @@ class TestSyncBatchNormRunnerBase(object): sys.stdout.buffer.write( pickle.dumps( - 'training, inference, fp32, fp16, NCHW, NHWC all passed')) + 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) + ) def _compare(self, args, place, layout, only_forward): scope = core.Scope() np.random.seed(SEED) - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 sys.stderr.write("data: " + str(data) + "\n") - data = create_or_get_tensor(scope, "input", - OpTest.np_dtype_to_fluid_dtype(data), place) + data = create_or_get_tensor( + scope, "input", OpTest.np_dtype_to_fluid_dtype(data), place + ) - bn_fetches = self._cal_single_card(args, data, place, layout, - only_forward) + bn_fetches = self._cal_single_card( + args, data, place, layout, only_forward + ) fetch_names, sync_bn_fetches = self._cal_multiple_cards( - args, data, place, layout, only_forward) + args, data, place, layout, only_forward + ) - sys.stderr.write("len(sync_bn_fetches): " + str(len(sync_bn_fetches)) + - "\n") + sys.stderr.write( + "len(sync_bn_fetches): " + str(len(sync_bn_fetches)) + "\n" + ) for i in range(0, len(sync_bn_fetches)): sys.stderr.write("i: " + str(i) + "\n") sys.stderr.write("fetch_names[i]): " + fetch_names[i] + "\n") @@ -168,13 +190,14 @@ class TestSyncBatchNormRunnerBase(object): bn_val = bn_fetches[i] sync_bn_val = sync_bn_fetches[i] if sync_bn_val.shape != bn_val.shape: - sync_bn_val = sync_bn_val[:bn_val.shape[0]] + sync_bn_val = sync_bn_val[: bn_val.shape[0]] # i = 0 if fetch_names[i] == 'reduce_sum_0.tmp_0': # sys.stderr.write("skip reduce_sum_0.tmp_0 (Out of reduce_sum op)" + "\n") - sys.stderr.write("reduce_sum_0.tmp_0 (Out of reduce_sum op)" + - "\n") + sys.stderr.write( + "reduce_sum_0.tmp_0 (Out of reduce_sum op)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -202,7 +225,8 @@ class TestSyncBatchNormRunnerBase(object): if fetch_names[i] == 'batch_norm_0.tmp_2': # sys.stderr.write("skip batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n") sys.stderr.write( - "batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n") + "batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -235,8 +259,9 @@ class TestSyncBatchNormRunnerBase(object): # i = 8 if fetch_names[i] == 'batch_norm_0.tmp_1': - sys.stderr.write("skip batch_norm_0.tmp_1 (SavedVariance)" + - "\n") + sys.stderr.write( + "skip batch_norm_0.tmp_1 (SavedVariance)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -282,10 +307,16 @@ class TestSyncBatchNormRunnerBase(object): if fetch_names[i] == 'conv2d_0.tmp_0@GRAD': atol = 1e-2 - assert np.allclose( - bn_val, sync_bn_val, atol=atol), "Output (" + fetch_names[ - i] + ") has diff. \n" + "\nBN " + str( - bn_val) + "\n" + "Sync BN " + str(sync_bn_val) + assert np.allclose(bn_val, sync_bn_val, atol=atol), ( + "Output (" + + fetch_names[i] + + ") has diff. \n" + + "\nBN " + + str(bn_val) + + "\n" + + "Sync BN " + + str(sync_bn_val) + ) def _cal_single_card(self, args, data, place, layout, only_forward): # Single-NPU, N = 32 per NPU @@ -295,23 +326,31 @@ class TestSyncBatchNormRunnerBase(object): startup_prog.global_seed(SEED) paddle.seed(SEED) - outs = self.get_model(train_prog, startup_prog, place, layout, SEED, - False, only_forward) + outs = self.get_model( + train_prog, startup_prog, place, layout, SEED, False, only_forward + ) exe = fluid.Executor(place) exe.run(startup_prog) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - bn_fetches = exe.run(program=train_prog, - feed={'input': data}, - fetch_list=fetch_names) + bn_fetches = exe.run( + program=train_prog, feed={'input': data}, fetch_list=fetch_names + ) return bn_fetches @@ -333,10 +372,12 @@ class TestSyncBatchNormRunnerBase(object): current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) - sys.stderr.write("after init, startup_prog: " + - startup_prog.to_string(True) + "\n") + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) + sys.stderr.write( + "after init, startup_prog: " + startup_prog.to_string(True) + "\n" + ) train_prog.global_seed(SEED) train_prog._sync_with_cpp() startup_prog.global_seed(SEED) @@ -344,12 +385,17 @@ class TestSyncBatchNormRunnerBase(object): paddle.seed(SEED) self.rank = rank - outs = self.get_model(train_prog, startup_prog, place, layout, SEED, - True, only_forward) - sys.stderr.write("after get_model, train_prog: " + - train_prog.to_string(True) + "\n") - sys.stderr.write("after get_model, startup_prog: " + - startup_prog.to_string(True) + "\n") + outs = self.get_model( + train_prog, startup_prog, place, layout, SEED, True, only_forward + ) + sys.stderr.write( + "after get_model, train_prog: " + train_prog.to_string(True) + "\n" + ) + sys.stderr.write( + "after get_model, startup_prog: " + + startup_prog.to_string(True) + + "\n" + ) ops = train_prog.blocks[0].ops for i, op in enumerate(ops): @@ -362,23 +408,33 @@ class TestSyncBatchNormRunnerBase(object): sys.stderr.write("op type: " + op.type + "\n") op.desc.set_type('sync_batch_norm_grad') - sys.stderr.write("after update sync_batch_norm, train_prog: " + - train_prog.to_string(True) + "\n") + sys.stderr.write( + "after update sync_batch_norm, train_prog: " + + train_prog.to_string(True) + + "\n" + ) exe = fluid.Executor(place) exe.run(startup_prog) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - sync_bn_fetches = exe.run(program=train_prog, - feed={'input': data}, - fetch_list=fetch_names) + sync_bn_fetches = exe.run( + program=train_prog, feed={'input': data}, fetch_list=fetch_names + ) return fetch_names, sync_bn_fetches @@ -401,19 +457,20 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -442,7 +499,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) @@ -453,15 +510,19 @@ class TestDistBase(unittest.TestCase): tr1_pipe = open("/tmp/tr1_err.log", "wb") # print(tr0_cmd) # print(tr1_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -471,12 +532,18 @@ class TestDistBase(unittest.TestCase): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) def check_with_place(self, model_file, col_type, need_envs={}): tr0_out, tr1_out, pid0, pid1 = self._run_cluster(model_file, need_envs) self.assertEqual( - tr0_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed') + tr0_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) self.assertEqual( - tr1_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed') + tr1_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_baseline.py b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_baseline.py index d66cb04e009..2efe1f91cb8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_baseline.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_baseline.py @@ -20,7 +20,10 @@ import sys sys.path.append("..") -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) from test_sync_batch_norm_base_npu import TestDistBase @@ -29,15 +32,14 @@ paddle.enable_static() class TestSyncBatchNormOp(TestDistBase): - def _setup_config(self): pass def test_identity(self, col_type="identity"): dist_env = os.environ - self.check_with_place("sync_batch_norm_op_npu.py", - col_type, - need_envs=dist_env) + self.check_with_place( + "sync_batch_norm_op_npu.py", col_type, need_envs=dist_env + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_extra.py b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_extra.py index 44644ef245e..15c49170429 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_extra.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_extra.py @@ -26,19 +26,22 @@ import paddle.fluid as fluid import paddle.nn as nn from paddle.fluid import Program, program_guard -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) # _set_use_system_allocator(False) paddle.enable_static() class TestDygraphSyncBatchNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): my_sync_batch_norm = paddle.nn.SyncBatchNorm(10) - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, my_sync_batch_norm, x1) # the input dtype of SyncBatchNorm must be float16 or float32 @@ -48,31 +51,33 @@ class TestDygraphSyncBatchNormAPIError(unittest.TestCase): class TestConvertSyncBatchNorm(unittest.TestCase): - def test_convert(self): with program_guard(Program(), Program()): - compare_model = paddle.nn.Sequential(paddle.nn.Conv2D(3, 5, 3), - paddle.nn.BatchNorm2D(5), - paddle.nn.BatchNorm2D(5)) + compare_model = paddle.nn.Sequential( + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), + paddle.nn.BatchNorm2D(5), + ) model = paddle.nn.Sequential( - paddle.nn.Conv2D(3, 5, 3), paddle.nn.BatchNorm2D(5), + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), paddle.nn.BatchNorm2D( 5, weight_attr=fluid.ParamAttr(name='bn.scale'), - bias_attr=fluid.ParamAttr(name='bn.bias'))) + bias_attr=fluid.ParamAttr(name='bn.bias'), + ), + ) model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model) for idx, sublayer in enumerate(compare_model.sublayers()): if isinstance(sublayer, paddle.nn.BatchNorm2D): self.assertEqual( - isinstance(model[idx], paddle.nn.SyncBatchNorm), True) + isinstance(model[idx], paddle.nn.SyncBatchNorm), True + ) class TestConvertSyncBatchNormCast1(unittest.TestCase): - def test_convert(self): - class Net(nn.Layer): - def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2D(3, 5, 3) @@ -97,7 +102,6 @@ class TestConvertSyncBatchNormCast1(unittest.TestCase): class TestDygraphSyncBatchNormDataFormatError(unittest.TestCase): - def test_errors(self): with fluid.dygraph.guard(fluid.NPUPlace(0)): my_sync_batch_norm = paddle.nn.SyncBatchNorm(10, data_format='CN') diff --git a/python/paddle/fluid/tests/unittests/npu/test_take_along_axis_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_take_along_axis_op_npu.py index d0cc8e3abde..d3e34c39366 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_take_along_axis_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_take_along_axis_op_npu.py @@ -27,7 +27,6 @@ paddle.enable_static() @unittest.skip(reason="Skip unsupported ut, need paddle surpport cann 5.0.4+") class TestTakeAlongAxisOp(OpTest): - def setUp(self): self.set_npu() self.init_data() @@ -59,14 +58,14 @@ class TestTakeAlongAxisOp(OpTest): self.x_type = "float64" self.x_shape = (5, 5, 5) self.index_type = "int32" - self.index = np.array([[[1]], [[1]], [[2]], [[4]], - [[3]]]).astype(self.index_type) + self.index = np.array([[[1]], [[1]], [[2]], [[4]], [[3]]]).astype( + self.index_type + ) self.axis = 2 self.axis_type = "int64" class TestCase1(TestTakeAlongAxisOp): - def init_data(self): self.x_type = "float64" self.x_shape = (5, 5, 5) @@ -78,7 +77,6 @@ class TestCase1(TestTakeAlongAxisOp): @unittest.skip(reason="Skip unsupported ut, need paddle surpport cann 5.0.4+") class TestTakeAlongAxisAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [3, 3] @@ -95,13 +93,12 @@ class TestTakeAlongAxisAPI(unittest.TestCase): index = paddle.fluid.data('Index', self.index_shape, "int64") out = paddle.take_along_axis(x, index, self.axis) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'X': self.x_np, - 'Index': self.index_np - }, - fetch_list=[out]) + res = exe.run( + feed={'X': self.x_np, 'Index': self.index_np}, fetch_list=[out] + ) out_ref = np.array( - np.take_along_axis(self.x_np, self.index_np, self.axis)) + np.take_along_axis(self.x_np, self.index_np, self.axis) + ) for out in res: np.testing.assert_allclose(out, out_ref, rtol=0.001) @@ -111,20 +108,21 @@ class TestTakeAlongAxisAPI(unittest.TestCase): self.index = paddle.to_tensor(self.index_np) out = paddle.take_along_axis(x_tensor, self.index, self.axis) out_ref = np.array( - np.take_along_axis(self.x_np, self.index_np, self.axis)) + np.take_along_axis(self.x_np, self.index_np, self.axis) + ) np.testing.assert_allclose(out.numpy(), out_ref, rtol=0.001) paddle.enable_static() @unittest.skip(reason="Skip unsupported ut, need paddle surpport cann 5.0.4+") class TestTakeAlongAxisAPICase1(TestTakeAlongAxisAPI): - def setUp(self): np.random.seed(0) self.shape = [2, 2] self.index_shape = [4, 2] - self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, - 0]]).astype('int64') + self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, 0]]).astype( + 'int64' + ) self.x_np = np.random.random(self.shape).astype(np.float32) self.place = paddle.NPUPlace(0) self.axis = 0 diff --git a/python/paddle/fluid/tests/unittests/npu/test_tanh_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_tanh_op_npu.py index 0b1563282c6..8cbb0d217eb 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_tanh_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_tanh_op_npu.py @@ -26,7 +26,6 @@ SEED = 2021 class TestTanh(OpTest): - def setUp(self): self.set_npu() self.op_type = "tanh" @@ -58,7 +57,6 @@ class TestTanh(OpTest): class TestTanhFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "tanh" @@ -85,7 +83,6 @@ class TestTanhFp16(OpTest): class TestTanhNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -100,9 +97,9 @@ class TestTanhNet(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.tanh(c) @@ -126,16 +123,17 @@ class TestTanhNet(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_tile_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_tile_op_npu.py index 723beb31c9a..e8003f82aa9 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_tile_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_tile_op_npu.py @@ -27,9 +27,8 @@ paddle.enable_static() np.random.seed(10) -#Situation 1: repeat_times is a list (without tensor) +# Situation 1: repeat_times is a list (without tensor) class TestTileOpRank1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -55,44 +54,38 @@ class TestTileOpRank1(OpTest): pass -#with dimension expanding +# with dimension expanding class TestTileOpRank2Expanding(TestTileOpRank1): - def init_data(self): self.ori_shape = [120] self.repeat_times = [2, 2] class TestTileOpRank2(TestTileOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] class TestTileOpRank3_Corner(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (1, 1, 1) class TestTileOpRank3_Corner2(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (2, 2) class TestTileOpRank3(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.repeat_times = (2, 1, 4) class TestTileOpRank4(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.repeat_times = (3, 2, 1, 2) @@ -100,7 +93,6 @@ class TestTileOpRank4(TestTileOpRank1): # Situation 2: repeat_times is a list (with tensor) class TestTileOpRank1_tensor_attr(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -108,8 +100,9 @@ class TestTileOpRank1_tensor_attr(OpTest): self.init_data() repeat_times_tensor = [] for index, ele in enumerate(self.repeat_times): - repeat_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + repeat_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float32"), @@ -135,7 +128,6 @@ class TestTileOpRank1_tensor_attr(OpTest): class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [1, 1] @@ -143,7 +135,6 @@ class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -152,7 +143,6 @@ class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): # Situation 3: repeat_times is a tensor class TestTileOpRank1_tensor(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -182,7 +172,6 @@ class TestTileOpRank1_tensor(OpTest): class TestTileOpRank2_tensor(TestTileOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -190,7 +179,6 @@ class TestTileOpRank2_tensor(TestTileOpRank1_tensor): # Situation 4: input x is Integer class TestTileOpInteger(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -211,7 +199,6 @@ class TestTileOpInteger(OpTest): # Situation 5: input x is Integer class TestTileOpInt64_t(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -232,7 +219,6 @@ class TestTileOpInt64_t(OpTest): # Situation 6: input x is Bool class TestTileOpBool(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -251,7 +237,6 @@ class TestTileOpBool(OpTest): # Test python API class TestTileAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(paddle.NPUPlace(0)): np_x = np.random.random([12, 14]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_top_k_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_top_k_op_npu.py index 6e73f282c8c..f4c7bab1105 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_top_k_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_top_k_op_npu.py @@ -28,20 +28,24 @@ SEED = 2021 class TestTopk(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) self.op_type = "top_k" self.init_dtype() - x = np.array([[0.78104149, 0.88745828, 0.32362268], - [0.82196718, 0.48763277, 0.42826136], - [0.96527182, 0.34851612, 0.12959783]]).astype(self.dtype) + x = np.array( + [ + [0.78104149, 0.88745828, 0.32362268], + [0.82196718, 0.48763277, 0.42826136], + [0.96527182, 0.34851612, 0.12959783], + ] + ).astype(self.dtype) self.inputs = {'X': x} - np_out = np.array([[0.88745828], [0.82196718], - [0.96527182]]).astype(self.dtype) + np_out = np.array([[0.88745828], [0.82196718], [0.96527182]]).astype( + self.dtype + ) np_indices = np.array([[1], [0], [0]]) self.attrs = {'k': 1, "axis": -1} @@ -59,20 +63,28 @@ class TestTopk(OpTest): class TestTopkV2(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) self.op_type = "top_k" self.init_dtype() - x = np.array([[0.78104149, 0.88745828, 0.32362268], - [0.82196718, 0.48763277, 0.42826136], - [0.96527182, 0.34851612, 0.12959783]]).astype(self.dtype) + x = np.array( + [ + [0.78104149, 0.88745828, 0.32362268], + [0.82196718, 0.48763277, 0.42826136], + [0.96527182, 0.34851612, 0.12959783], + ] + ).astype(self.dtype) self.inputs = {'X': x} - np_out = np.array([[0.88745828, 0.78104149], [0.82196718, 0.48763277], - [0.96527182, 0.34851612]]).astype(self.dtype) + np_out = np.array( + [ + [0.88745828, 0.78104149], + [0.82196718, 0.48763277], + [0.96527182, 0.34851612], + ] + ).astype(self.dtype) np_indices = np.array([[1, 0], [0, 1], [0, 1]]) self.attrs = {'k': 2, "axis": -1} @@ -90,7 +102,6 @@ class TestTopkV2(OpTest): class TestTopkV3(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -99,10 +110,9 @@ class TestTopkV3(OpTest): self.init_dtype() self.set_input_data() self.set_attrs() - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=True) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=True + ) self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis} @@ -123,8 +133,9 @@ class TestTopkV3(OpTest): self.axis = 1 def set_input_data(self): - self.input_data = np.random.choice(10000, size=(10, 20), - replace=False).astype(self.dtype) + self.input_data = np.random.choice( + 10000, size=(10, 20), replace=False + ).astype(self.dtype) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_top_k_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_top_k_v2_op_npu.py index c72ce4831f7..6b46a41fe1d 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_top_k_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_top_k_v2_op_npu.py @@ -39,7 +39,6 @@ def numpy_topk(x, k=1, axis=-1, largest=True): class TestTopkV2NPUOp(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "top_k_v2" @@ -48,10 +47,9 @@ class TestTopkV2NPUOp(OpTest): self.set_dtype() self.set_input_data() self.set_attrs() - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} @@ -66,8 +64,9 @@ class TestTopkV2NPUOp(OpTest): self.largest = True def set_input_data(self): - self.input_data = np.random.choice(10000, size=(10, 20), - replace=False).astype(self.dtype) + self.input_data = np.random.choice( + 10000, size=(10, 20), replace=False + ).astype(self.dtype) def test_check_output(self): self.__class__.no_need_check_grad = True @@ -82,7 +81,6 @@ class TestTopkV2NPUOp(OpTest): class TestTopkV2OpFloat16(TestTopkV2NPUOp): - def set_attrs(self): self.k = 3 self.axis = 1 @@ -96,7 +94,6 @@ class TestTopkV2OpFloat16(TestTopkV2NPUOp): class TestTopkV2OP1Int32(TestTopkV2NPUOp): - def set_attrs(self): self.k = 3 self.axis = 0 @@ -104,7 +101,6 @@ class TestTopkV2OP1Int32(TestTopkV2NPUOp): class TestTopkV2OP2Int32(TestTopkV2NPUOp): - def set_attrs(self): self.k = 4 self.axis = 0 @@ -112,7 +108,6 @@ class TestTopkV2OP2Int32(TestTopkV2NPUOp): class TestTopkV2OP3Int32(TestTopkV2NPUOp): - def set_attrs(self): self.k = 6 self.axis = 1 @@ -120,7 +115,6 @@ class TestTopkV2OP3Int32(TestTopkV2NPUOp): class TestTopkV2OP4Int32(TestTopkV2NPUOp): - def set_attrs(self): self.k = 3 self.axis = 1 @@ -128,31 +122,26 @@ class TestTopkV2OP4Int32(TestTopkV2NPUOp): class TestTopkV2Op1Int64(TestTopkV2OP1Int32): - def set_dtype(self): self.dtype = np.int64 class TestTopkV2Op2Int64(TestTopkV2OP2Int32): - def set_dtype(self): self.dtype = np.int64 class TestTopkV2Op3Int64(TestTopkV2OP3Int32): - def set_dtype(self): self.dtype = np.int64 class TestTopkV2Op4Int64(TestTopkV2OP4Int32): - def set_dtype(self): self.dtype = np.int64 class TestTopkV2Op1Float32(TestTopkV2OP1Int32): - def set_dtype(self): self.dtype = np.float32 @@ -161,7 +150,6 @@ class TestTopkV2Op1Float32(TestTopkV2OP1Int32): class TestTopkV2Op2Float32(TestTopkV2OP2Int32): - def set_dtype(self): self.dtype = np.float32 @@ -170,7 +158,6 @@ class TestTopkV2Op2Float32(TestTopkV2OP2Int32): class TestTopkV2Op3Float32(TestTopkV2OP3Int32): - def set_dtype(self): self.dtype = np.float32 @@ -179,7 +166,6 @@ class TestTopkV2Op3Float32(TestTopkV2OP3Int32): class TestTopkV2Op4Float32(TestTopkV2OP4Int32): - def set_dtype(self): self.dtype = np.float32 @@ -188,7 +174,6 @@ class TestTopkV2Op4Float32(TestTopkV2OP4Int32): class TestTopkV2Op1Float64(TestTopkV2OP1Int32): - def set_dtype(self): self.dtype = np.float64 @@ -197,7 +182,6 @@ class TestTopkV2Op1Float64(TestTopkV2OP1Int32): class TestTopkV2Op2Float64(TestTopkV2OP2Int32): - def set_dtype(self): self.dtype = np.float64 @@ -206,7 +190,6 @@ class TestTopkV2Op2Float64(TestTopkV2OP2Int32): class TestTopkV2Op3Float64(TestTopkV2OP3Int32): - def set_dtype(self): self.dtype = np.float64 @@ -215,7 +198,6 @@ class TestTopkV2Op3Float64(TestTopkV2OP3Int32): class TestTopkV2Op4Float64(TestTopkV2OP4Int32): - def set_dtype(self): self.dtype = np.float64 @@ -224,7 +206,6 @@ class TestTopkV2Op4Float64(TestTopkV2OP4Int32): class TestTopKAPI(unittest.TestCase): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -275,22 +256,23 @@ class TestTopKAPI(unittest.TestCase): np.testing.assert_allclose(paddle_result[1].numpy(), numpy_result[1]) # test case for basic test case 7 for the unsorted paddle_result = paddle.topk(input_tensor, k=2, axis=1, sorted=False) - sort_paddle = numpy_topk(np.array(paddle_result[0].numpy()), - axis=1, - k=2) + sort_paddle = numpy_topk( + np.array(paddle_result[0].numpy()), axis=1, k=2 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=1) np.testing.assert_allclose(sort_paddle[0], numpy_result[0]) def run_static(self, place): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[6, 7, 8], - dtype="float64") - large_input_tensor = paddle.static.data(name="large_x", - shape=[2, 1030], - dtype="float64") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[6, 7, 8], dtype="float64" + ) + large_input_tensor = paddle.static.data( + name="large_x", shape=[2, 1030], dtype="float64" + ) k_tensor = paddle.static.data(name="k", shape=[1], dtype="int32") result1 = paddle.topk(input_tensor, k=2) result2 = paddle.topk(input_tensor, k=2, axis=-1) @@ -304,18 +286,29 @@ class TestTopKAPI(unittest.TestCase): exe = paddle.static.Executor(place) input_data = np.random.rand(10, 20).astype("float64") large_input_data = np.random.rand(2, 100).astype("float64") - paddle_result = exe.run(feed={ - "x": self.input_data, - "large_x": self.large_input_data, - "k": np.array([2]).astype("int32") - }, - fetch_list=[ - result1[0], result1[1], result2[0], - result2[1], result3[0], result3[1], - result4[0], result4[1], result5[0], - result5[1], result6[0], result6[1], - result7[0], result7[1] - ]) + paddle_result = exe.run( + feed={ + "x": self.input_data, + "large_x": self.large_input_data, + "k": np.array([2]).astype("int32"), + }, + fetch_list=[ + result1[0], + result1[1], + result2[0], + result2[1], + result3[0], + result3[1], + result4[0], + result4[1], + result5[0], + result5[1], + result6[0], + result6[1], + result7[0], + result7[1], + ], + ) numpy_result = numpy_topk(self.input_data, k=2) np.testing.assert_allclose(paddle_result[0], numpy_result[0]) np.testing.assert_allclose(paddle_result[1], numpy_result[1]) @@ -328,17 +321,15 @@ class TestTopKAPI(unittest.TestCase): np.testing.assert_allclose(paddle_result[4], numpy_result[0]) np.testing.assert_allclose(paddle_result[5], numpy_result[1]) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=1, - largest=False) + numpy_result = numpy_topk( + self.input_data, k=2, axis=1, largest=False + ) np.testing.assert_allclose(paddle_result[6], numpy_result[0]) np.testing.assert_allclose(paddle_result[7], numpy_result[1]) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=-1, - largest=False) + numpy_result = numpy_topk( + self.input_data, k=2, axis=-1, largest=False + ) np.testing.assert_allclose(paddle_result[8], numpy_result[0]) np.testing.assert_allclose(paddle_result[9], numpy_result[1]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_transpose_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_transpose_op_npu.py index e6a457ca3e6..9503e28208e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_transpose_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_transpose_op_npu.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestTransposeOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "transpose2" @@ -55,83 +54,72 @@ class TestTransposeOp(OpTest): class TestCase_ZeroDim(TestTransposeOp): - def init_shape_axis(self): self.shape = () self.axis = () -class TestCase0(TestTransposeOp): +class TestCase0(TestTransposeOp): def init_shape_axis(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1(TestTransposeOp): - def init_shape_axis(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase2(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) class TestCase5(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 16, 96) self.axis = (0, 2, 1) class TestCase6(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 10, 12, 16) self.axis = (3, 1, 2, 0) class TestCase7(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 10, 2, 16) self.axis = (0, 1, 3, 2) class TestCase8(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (0, 1, 3, 2, 4, 5, 6, 7) class TestCase9(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) class TestTransposeOpFP16(TestTransposeOp): - def init_dtype(self): self.dtype = np.float16 @@ -140,7 +128,6 @@ class TestTransposeOpFP16(TestTransposeOp): class TestTransposeOpInt64(TestTransposeOp): - def init_dtype(self): self.dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_tril_triu_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_tril_triu_op_npu.py index 2aecede417a..aca61989b7b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_tril_triu_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_tril_triu_op_npu.py @@ -23,8 +23,7 @@ paddle.enable_static() class TestNPUTrilTriu(OpTest): - """ the base class of other op testcases - """ + """the base class of other op testcases""" def setUp(self): self.op_type = "tril_triu" @@ -40,9 +39,9 @@ class TestNPUTrilTriu(OpTest): 'lower': True if self.real_op_type == 'tril' else False, } self.outputs = { - 'Out': - self.real_np_op(self.X, self.diagonal) - if self.diagonal else self.real_np_op(self.X) + 'Out': self.real_np_op(self.X, self.diagonal) + if self.diagonal + else self.real_np_op(self.X) } def test_check_output(self): @@ -67,27 +66,29 @@ def case_generator(op_type, Xshape, diagonal, expected): If arg `expercted` is 'success', it will register an Optest case and expect to pass. Otherwise, it will register an API case and check the expect failure. """ - cls_name = "{0}_{1}_shape_{2}_diag_{3}".format(expected, op_type, Xshape, - diagonal) + cls_name = "{0}_{1}_shape_{2}_diag_{3}".format( + expected, op_type, Xshape, diagonal + ) errmsg = { - "diagonal: TypeError": - "diagonal in {} must be a python Int".format(op_type), - "input: ValueError": - "x shape in {} must be at least 2-D".format(op_type), + "diagonal: TypeError": "diagonal in {} must be a python Int".format( + op_type + ), + "input: ValueError": "x shape in {} must be at least 2-D".format( + op_type + ), } class FailureCase(unittest.TestCase): - def test_failure(self): paddle.enable_static() data = fluid.data(shape=Xshape, dtype='float32', name=cls_name) - with self.assertRaisesRegexp(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegexp( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal=diagonal) class SuccessCase(TestNPUTrilTriu): - def initTestCase(self): paddle.enable_static() @@ -112,15 +113,13 @@ cases = { (20, 20): [ '2020', [20], - { - 20: 20 - }, + {20: 20}, (20, 20), 20.20, ], # str, list, dict, tuple, float }, 'input: ValueError': { - (2020, ): [None], + (2020,): [None], }, } for _op_type in ['tril', 'triu']: @@ -129,12 +128,15 @@ for _op_type in ['tril', 'triu']: list( map( lambda _diagonal: case_generator( - _op_type, _Xshape, _diagonal, _expected), _diaglist)) + _op_type, _Xshape, _diagonal, _expected + ), + _diaglist, + ) + ) class TestTrilTriuOpAPI(unittest.TestCase): - """ test case by using API and has -1 dimension - """ + """test case by using API and has -1 dimension""" def test_api(self): paddle.enable_static() @@ -166,8 +168,10 @@ class TestTrilTriuOpAPI(unittest.TestCase): with fluid.dygraph.guard(): data = np.random.random([1, 9, 9, 4]).astype(dtype) x = fluid.dygraph.to_variable(data) - tril_out, triu_out = tensor.tril(x).numpy(), tensor.triu( - x).numpy() + tril_out, triu_out = ( + tensor.tril(x).numpy(), + tensor.triu(x).numpy(), + ) np.testing.assert_allclose(tril_out, np.tril(data)) np.testing.assert_allclose(triu_out, np.triu(data)) @@ -185,14 +189,15 @@ class TestTrilTriuOpAPI(unittest.TestCase): place = fluid.NPUPlace(0) exe = fluid.Executor(place) - triu_out = exe.run(fluid.default_main_program(), - feed={"x": data}, - fetch_list=[triu_out]) + triu_out = exe.run( + fluid.default_main_program(), + feed={"x": data}, + fetch_list=[triu_out], + ) # @skip_check_grad_ci(reason="[NPU does not support grad right now.") class TestNPUTrilTriu_bool(TestNPUTrilTriu): - def test_check_output(self): self.check_output_with_place(self.place) diff --git a/python/paddle/fluid/tests/unittests/npu/test_truncated_gaussian_random_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_truncated_gaussian_random_op_npu.py index 98bc517b908..dfa70ec0b65 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_truncated_gaussian_random_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_truncated_gaussian_random_op_npu.py @@ -29,7 +29,6 @@ SEED = 2021 class TestTruncatedNormal(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -44,12 +43,13 @@ class TestTruncatedNormal(unittest.TestCase): with paddle.static.program_guard(main_prog, startup_prog): weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, - std=2.0)) - linear = paddle.nn.Linear(2, - 2, - weight_attr=weight_attr, - bias_attr=False) + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0 + ), + ) + linear = paddle.nn.Linear( + 2, 2, weight_attr=weight_attr, bias_attr=False + ) if run_npu: place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_uniform_random_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_uniform_random_op_npu.py index 0c3e37a54a8..2fdf53b23a7 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_uniform_random_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_uniform_random_op_npu.py @@ -25,7 +25,10 @@ import paddle from paddle.fluid.op import Operator import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from test_uniform_random_op import TestUniformRandomOp, TestUniformRandomOpSelectedRows +from test_uniform_random_op import ( + TestUniformRandomOp, + TestUniformRandomOpSelectedRows, +) paddle.enable_static() @@ -39,7 +42,6 @@ def output_hist(out): class TestNPUUniformRandomOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "uniform_random" @@ -53,7 +55,7 @@ class TestNPUUniformRandomOp(OpTest): "shape": [1000, 784], "min": -5.0, "max": 10.0, - "seed": 10 + "seed": 10, } self.output_hist = output_hist @@ -73,7 +75,6 @@ class TestNPUUniformRandomOp(OpTest): class TestNPUUniformRandomOpSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_npu(): @@ -88,12 +89,14 @@ class TestNPUUniformRandomOpSelectedRows(unittest.TestCase): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) diff --git a/python/paddle/fluid/tests/unittests/npu/test_unsqueeze_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_unsqueeze_op_npu.py index 4861c267922..0905e928e04 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_unsqueeze_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_unsqueeze_op_npu.py @@ -28,7 +28,6 @@ paddle.enable_static() # unsqueeze class TestUnsqueezeOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "unsqueeze" @@ -60,7 +59,6 @@ class TestUnsqueezeOp(OpTest): class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (3, 40) self.axes = (0, -2) @@ -69,7 +67,6 @@ class TestUnsqueezeOp1(TestUnsqueezeOp): # No axes input. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = () @@ -78,7 +75,6 @@ class TestUnsqueezeOp2(TestUnsqueezeOp): # Just part of axes be squeezed. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (6, 5, 1, 4) self.axes = (1, -1) @@ -87,7 +83,6 @@ class TestUnsqueezeOp3(TestUnsqueezeOp): # unsqueeze 2 class TestUnsqueeze2Op(OpTest): - def setUp(self): self.set_npu() self.op_type = "unsqueeze2" @@ -98,7 +93,7 @@ class TestUnsqueeze2Op(OpTest): self.init_attrs() self.outputs = { "Out": self.x.reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_npu(self): @@ -121,7 +116,6 @@ class TestUnsqueeze2Op(OpTest): # Correct: There is mins axis. class TestUnsqueeze2Op1(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -2) @@ -130,7 +124,6 @@ class TestUnsqueeze2Op1(TestUnsqueeze2Op): # Correct: No axes input. class TestUnsqueeze2Op2(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = () @@ -139,7 +132,6 @@ class TestUnsqueeze2Op2(TestUnsqueeze2Op): # Correct: Just part of axes be squeezed. class TestUnsqueeze2Op3(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (6, 5, 1, 4) self.axes = (1, -1) diff --git a/python/paddle/fluid/tests/unittests/npu/test_unstack_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_unstack_op_npu.py index 91f2fb047fe..5c20b083bfc 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_unstack_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_unstack_op_npu.py @@ -24,7 +24,6 @@ paddle.enable_static() class TestUnStackOpBase(OpTest): - def initDefaultParameters(self): self.input_dim = (5, 6, 7) self.axis = 0 @@ -74,25 +73,21 @@ class TestUnStackOpBase(OpTest): class TestStackOp3(TestUnStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestUnStackOpBase): - def initParameters(self): self.axis = -3 class TestStackOp5(TestUnStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestUnStackOpBase): - def initParameters(self): self.axis = 2 diff --git a/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_min_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_min_op_npu.py index 48df4ad454a..48edac9ab33 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_min_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_min_op_npu.py @@ -29,7 +29,6 @@ SEED = 2021 class TestUpdateLossScalingOpMinLossScalingBad(TestUpdateLossScalingOpBad): - def setUp(self): self.set_npu() self.op_type = "update_loss_scaling" @@ -48,14 +47,14 @@ class TestUpdateLossScalingOpMinLossScalingBad(TestUpdateLossScalingOpBad): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', np.zeros_like(x))], 'LossScaling': np.array([1639.0]).astype(self.dtype), 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def init(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_op_npu.py index 678e50247af..8d364af89e3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_op_npu.py @@ -27,7 +27,6 @@ SEED = 2021 class TestUpdateLossScalingOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "update_loss_scaling" @@ -42,14 +41,14 @@ class TestUpdateLossScalingOp(OpTest): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', x)], 'LossScaling': self.prev_loss_scaling * self.incr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def set_npu(self): @@ -75,7 +74,6 @@ class TestUpdateLossScalingOp(OpTest): class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): - def setUp(self): self.set_npu() self.op_type = "update_loss_scaling" @@ -93,33 +91,32 @@ class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', np.zeros_like(x))], 'LossScaling': self.prev_loss_scaling * self.decr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } class TestUpdateLossScalingLayer(unittest.TestCase): - def loss_scaling_check(self, use_npu=True, scope=fluid.Scope()): a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -133,33 +130,41 @@ class TestUpdateLossScalingLayer(unittest.TestCase): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = paddle.NPUPlace(0) if use_npu else fluid.CPUPlace() exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], a_v) assert np.array_equal(result_v[1], b_v) assert np.array_equal(result_v[0], result_v[2]) @@ -174,15 +179,15 @@ class TestUpdateLossScalingLayer(unittest.TestCase): b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -199,33 +204,41 @@ class TestUpdateLossScalingLayer(unittest.TestCase): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = paddle.NPUPlace(0) if use_npu else fluid.CPUPlace() exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], np.zeros_like(a_v)) assert np.array_equal(result_v[1], np.zeros_like(b_v)) assert np.array_equal(result_v[2], np.zeros_like(a_v)) diff --git a/python/paddle/fluid/tests/unittests/npu/test_where_index_npu.py b/python/paddle/fluid/tests/unittests/npu/test_where_index_npu.py index ff159c1473f..370a1934bff 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_where_index_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_where_index_npu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestWhereIndexOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "where_index" @@ -49,7 +48,6 @@ class TestWhereIndexOp(OpTest): class TestNotBool(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([1, 0, 8]), @@ -59,7 +57,6 @@ class TestNotBool(TestWhereIndexOp): class TestAllFalse(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([False, False, False]), @@ -69,7 +66,6 @@ class TestAllFalse(TestWhereIndexOp): class TestRank2(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([[True, False], [False, True]]), @@ -79,24 +75,26 @@ class TestRank2(TestWhereIndexOp): class TestRank3(TestWhereIndexOp): - def init_config(self): self.inputs = { - 'Condition': - np.array([[[True, False], [False, True]], - [[False, True], [True, False]], - [[False, False], [False, True]]]), + 'Condition': np.array( + [ + [[True, False], [False, True]], + [[False, True], [True, False]], + [[False, False], [False, True]], + ] + ), } self.outputs = { - 'Out': - np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], - dtype='int64') + 'Out': np.array( + [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], + dtype='int64', + ) } class TestWhereOpError(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): cond = fluid.layers.data(name='cond', shape=[4], dtype='bool') @@ -109,9 +107,7 @@ class TestWhereOpError(unittest.TestCase): class TestWhereRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.where([10]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_where_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_where_op_npu.py index ab26d506c2f..9e1126d0aa0 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_where_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_where_op_npu.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestNPUWhereOp(OpTest): - def setUp(self): self.op_type = "where" self.set_npu() @@ -52,7 +51,6 @@ class TestNPUWhereOp(OpTest): class TestNPUWhereOp2(TestNPUWhereOp): - def init_config(self): self.x = np.random.uniform(-5, 5, (60, 2)).astype("float64") self.y = np.random.uniform(-5, 5, (60, 2)).astype("float64") @@ -60,7 +58,6 @@ class TestNPUWhereOp2(TestNPUWhereOp): class TestNPUWhereOp3(TestNPUWhereOp): - def init_config(self): self.x = np.random.uniform(-3, 5, (20, 2, 4)).astype("float64") self.y = np.random.uniform(-3, 5, (20, 2, 4)).astype("float64") @@ -68,7 +65,6 @@ class TestNPUWhereOp3(TestNPUWhereOp): class TestNPUWhereAPI(unittest.TestCase): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -93,9 +89,9 @@ class TestNPUWhereAPI(unittest.TestCase): train_prog = fluid.Program() startup = fluid.Program() with fluid.program_guard(train_prog, startup): - cond = fluid.data(name='cond', - shape=self.shape, - dtype='bool') + cond = fluid.data( + name='cond', shape=self.shape, dtype='bool' + ) x = fluid.data(name='x', shape=self.shape, dtype='float32') y = fluid.data(name='y', shape=self.shape, dtype='float32') @@ -113,24 +109,25 @@ class TestNPUWhereAPI(unittest.TestCase): fetch_list.append(x.grad_name) if y_stop_gradient is False: fetch_list.append(y.grad_name) - out = exe.run(train_prog, - feed={ - 'cond': self.cond, - 'x': self.x, - 'y': self.y - }, - fetch_list=fetch_list) + out = exe.run( + train_prog, + feed={'cond': self.cond, 'x': self.x, 'y': self.y}, + fetch_list=fetch_list, + ) assert np.array_equal(out[0], self.out) if x_stop_gradient is False: - assert np.array_equal(out[2], - self.ref_x_backward(out[1])) + assert np.array_equal( + out[2], self.ref_x_backward(out[1]) + ) if y.stop_gradient is False: - assert np.array_equal(out[3], - self.ref_y_backward(out[1])) + assert np.array_equal( + out[3], self.ref_y_backward(out[1]) + ) elif y.stop_gradient is False: - assert np.array_equal(out[2], - self.ref_y_backward(out[1])) + assert np.array_equal( + out[2], self.ref_y_backward(out[1]) + ) def test_api_broadcast(self, use_cuda=False): train_prog = fluid.Program() @@ -139,24 +136,21 @@ class TestNPUWhereAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32') y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32') x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype("float32") - y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, - 1.0]]).astype("float32") + y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]).astype( + "float32" + ) result = paddle.where(x > 1, x=x, y=y) exe = fluid.Executor(self.place) exe.run(startup) - out = exe.run(train_prog, - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + train_prog, feed={'x': x_i, 'y': y_i}, fetch_list=[result] + ) assert np.array_equal(out[0], np.where(x_i > 1, x_i, y_i)) class TestWhereDygraphAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(paddle.NPUPlace(0)): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/npu/test_while_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_while_op_npu.py index 54e11413fe7..45aba89a84e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_while_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_while_op_npu.py @@ -26,20 +26,16 @@ paddle.enable_static() class TestWhileOp(unittest.TestCase): - def simple_net(self): - d0 = layers.data("d0", - shape=[10], - append_batch_size=False, - dtype='float32') - d1 = layers.data("d1", - shape=[10], - append_batch_size=False, - dtype='float32') - d2 = layers.data("d2", - shape=[10], - append_batch_size=False, - dtype='float32') + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32' + ) + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32' + ) + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32' + ) # fill_constant npu op doesn't support int64 i = layers.zeros(shape=[1], dtype='int32') i = layers.cast(i, 'int64') @@ -107,12 +103,10 @@ class TestWhileOp(unittest.TestCase): for i in range(3): d.append(numpy.random.random(size=[10]).astype('float32')) - outs = exe.run(feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2] - }, - fetch_list=[sum_result]) + outs = exe.run( + feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, + fetch_list=[sum_result], + ) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) def test_simple_net_forward(self): diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index 01a4d788901..6147c88dc56 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -27,12 +27,21 @@ import paddle import paddle.fluid as fluid from paddle.fluid.framework import _dygraph_tracer import paddle.fluid.core as core -from paddle.fluid.framework import _in_legacy_dygraph, _enable_legacy_dygraph, _in_eager_without_dygraph_check, _disable_legacy_dygraph +from paddle.fluid.framework import ( + _in_legacy_dygraph, + _enable_legacy_dygraph, + _in_eager_without_dygraph_check, + _disable_legacy_dygraph, +) from paddle.fluid.framework import _test_eager_guard from paddle.fluid.backward import append_backward from paddle.fluid.op import Operator from paddle.fluid.executor import Executor -from paddle.fluid.framework import OpProtoHolder, Program, _current_expected_place +from paddle.fluid.framework import ( + OpProtoHolder, + Program, + _current_expected_place, +) from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import parse_arg_and_kwargs @@ -54,8 +63,12 @@ from white_list import ( # For switch new eager mode globally g_is_in_eager = _in_eager_without_dygraph_check() -g_enable_legacy_dygraph = _enable_legacy_dygraph if g_is_in_eager else lambda: None -g_disable_legacy_dygraph = _disable_legacy_dygraph if g_is_in_eager else lambda: None +g_enable_legacy_dygraph = ( + _enable_legacy_dygraph if g_is_in_eager else lambda: None +) +g_disable_legacy_dygraph = ( + _disable_legacy_dygraph if g_is_in_eager else lambda: None +) def check_out_dtype(api_fn, in_specs, expect_dtypes, target_index=0, **configs): @@ -85,12 +98,15 @@ def check_out_dtype(api_fn, in_specs, expect_dtypes, target_index=0, **configs): shape, dtype = spec else: raise ValueError( - "Value of in_specs[{}] should contains two elements: [shape, dtype]" - .format(index)) + "Value of in_specs[{}] should contains two elements: [shape, dtype]".format( + index + ) + ) input_t.append( - paddle.static.data(name='data_%s' % index, - shape=shape, - dtype=dtype)) + paddle.static.data( + name='data_%s' % index, shape=shape, dtype=dtype + ) + ) out = api_fn(*input_t, **configs) out_dtype = fluid.data_feeder.convert_dtype(out.dtype) @@ -98,7 +114,9 @@ def check_out_dtype(api_fn, in_specs, expect_dtypes, target_index=0, **configs): if out_dtype != expect_dtype: raise ValueError( "Expected out.dtype is {}, but got {} from {}.".format( - expect_dtype, out_dtype, api_fn.__name__)) + expect_dtype, out_dtype, api_fn.__name__ + ) + ) def _set_use_system_allocator(value=None): @@ -110,22 +128,25 @@ def _set_use_system_allocator(value=None): def randomize_probability(batch_size, class_num, dtype='float32'): - prob = np.random.uniform(0.1, 1.0, - size=(batch_size, class_num)).astype(dtype) + prob = np.random.uniform(0.1, 1.0, size=(batch_size, class_num)).astype( + dtype + ) prob_sum = prob.sum(axis=1) for i in range(len(prob)): prob[i] /= prob_sum[i] return prob -def get_numeric_gradient(place, - scope, - op, - inputs, - input_to_check, - output_names, - delta=0.005, - in_place=False): +def get_numeric_gradient( + place, + scope, + op, + inputs, + input_to_check, + output_names, + delta=0.005, + in_place=False, +): # FIXME: change this method by compile time concepts set_input(scope, op, inputs, place) @@ -150,9 +171,12 @@ def get_numeric_gradient(place, elif tensor_to_check_dtype == core.VarDesc.VarType.COMPLEX128: tensor_to_check_dtype = np.complex128 else: - raise ValueError("Not supported data type " + - str(tensor_to_check_dtype) + ", tensor name : " + - str(input_to_check)) + raise ValueError( + "Not supported data type " + + str(tensor_to_check_dtype) + + ", tensor name : " + + str(input_to_check) + ) def get_output(): sum = [] @@ -167,7 +191,7 @@ def get_numeric_gradient(place, sum.append(output_numpy.astype(tensor_to_check_dtype).mean()) return tensor_to_check_dtype(np.array(sum).sum() / len(output_names)) - gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype) + gradient_flat = np.zeros(shape=(tensor_size,), dtype=tensor_to_check_dtype) def __get_elem__(tensor, i): if tensor_to_check_dtype == np.float16: @@ -179,15 +203,16 @@ def get_numeric_gradient(place, numpy_tensor = numpy_tensor.flatten() return struct.unpack( ' 1 and is_np_data( - sub_val_value[1]): # case 3 + sub_val_value[1] + ): # case 3 dtype_set.add(sub_val_value[1].dtype) - elif len(sub_val_value) > 1 and isinstance(sub_val_value[1], (list, tuple)) \ - and is_np_data(sub_val_value[1][0]): # case 4 + elif ( + len(sub_val_value) > 1 + and isinstance(sub_val_value[1], (list, tuple)) + and is_np_data(sub_val_value[1][0]) + ): # case 4 dtype_set.add(sub_val_value[1][0].dtype) # infer dtype from inputs, and dtype means the precision of the test @@ -466,7 +525,7 @@ class OpTest(unittest.TestCase): np.dtype(np.int16), np.dtype(np.int8), np.dtype(np.uint8), - np.dtype(np.bool_) + np.dtype(np.bool_), ] # check the dtype in dtype_list in order, select the first dtype that in dtype_set for dtype in dtype_list: @@ -501,7 +560,8 @@ class OpTest(unittest.TestCase): if isinstance(self.inputs[var_name], tuple): tensor.set(self.inputs[var_name][0], place) tensor.set_recursive_sequence_lengths( - self.inputs[var_name][1]) + self.inputs[var_name][1] + ) else: tensor.set(self.inputs[var_name], place) feed_map[var_name] = tensor @@ -509,7 +569,9 @@ class OpTest(unittest.TestCase): return feed_map def _append_ops(self, block): - self.__class__.op_type = self.op_type # for ci check, please not delete it for now + self.__class__.op_type = ( + self.op_type + ) # for ci check, please not delete it for now if self.is_mkldnn_op(): self.__class__.use_mkldnn = True @@ -524,23 +586,28 @@ class OpTest(unittest.TestCase): self.output_dtype = np.uint16 else: self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) - inputs = append_input_output(block, op_proto, self.inputs, True, - self.dtype) - outputs = append_input_output(block, op_proto, self.outputs, False, - self.dtype) + inputs = append_input_output( + block, op_proto, self.inputs, True, self.dtype + ) + outputs = append_input_output( + block, op_proto, self.outputs, False, self.dtype + ) if hasattr(self, "cache_name_list"): for name in self.cache_name_list: - inputs[name] = block.create_var(name=name, - persistable=True, - type=core.VarDesc.VarType.RAW, - stop_gradient=True) + inputs[name] = block.create_var( + name=name, + persistable=True, + type=core.VarDesc.VarType.RAW, + stop_gradient=True, + ) op = block.append_op( type=self.op_type, inputs=inputs, outputs=outputs, - attrs=copy(self.attrs) if hasattr(self, "attrs") else dict()) + attrs=copy(self.attrs) if hasattr(self, "attrs") else dict(), + ) # infer variable type and infer shape in compile-time op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) @@ -605,9 +672,12 @@ class OpTest(unittest.TestCase): def lod_has_continuous_zero(self, lod): for i in range(len(lod) - 3): - if lod[i] != 0 and lod[i + - 1] == 0 and lod[i + - 2] == 0 and lod[i + 3] != 0: + if ( + lod[i] != 0 + and lod[i + 1] == 0 + and lod[i + 2] == 0 + and lod[i + 3] != 0 + ): return True return False @@ -625,8 +695,12 @@ class OpTest(unittest.TestCase): if shape is None: shape = [12, 10] assert len(lod[0]) >= 8 - assert lod[0][0] == 0 and lod[0][1] == 0 and lod[0][-1] == 0 and lod[0][ - -2] == 0 + assert ( + lod[0][0] == 0 + and lod[0][1] == 0 + and lod[0][-1] == 0 + and lod[0][-2] == 0 + ) assert self.lod_has_single_zero(lod[0]) is True assert self.lod_has_continuous_zero(lod[0]) is True assert sum(lod[0]) == shape[0] @@ -634,9 +708,9 @@ class OpTest(unittest.TestCase): x = np.random.uniform(0.1, 1, shape).astype('float32') return (x, lod) - def append_input_output_for_dygraph(self, op_proto, np_list, is_input, - if_return_inputs_grad_dict, block): - + def append_input_output_for_dygraph( + self, op_proto, np_list, is_input, if_return_inputs_grad_dict, block + ): def create_var(np_value, name, is_input, if_return_inputs_grad_dict): np_value_temp = np_value has_lod = False @@ -656,13 +730,16 @@ class OpTest(unittest.TestCase): if has_lod: v.value().get_tensor().set_recursive_sequence_lengths( - lod_temp) + lod_temp + ) else: - v = block.create_var(name=name, - dtype=np_value_temp.dtype, - type=core.VarDesc.VarType.LOD_TENSOR, - persistable=False, - stop_gradient=False) + v = block.create_var( + name=name, + dtype=np_value_temp.dtype, + type=core.VarDesc.VarType.LOD_TENSOR, + persistable=False, + stop_gradient=False, + ) return v # prepare variable for input or output @@ -676,21 +753,23 @@ class OpTest(unittest.TestCase): continue if name not in np_list: assert var_proto.intermediate, "{} not found".format(name) - v = block.create_var(dtype='float32', - type=core.VarDesc.VarType.LOD_TENSOR) + v = block.create_var( + dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR + ) var_dict[name].append(v) if if_return_inputs_grad_dict: inputs_grad_dict[name] = v continue if var_proto.duplicable: assert isinstance( - np_list[name], - list), "Duplicable {} should be set as list".format(name) + np_list[name], list + ), "Duplicable {} should be set as list".format(name) var_list = [] slot_name = name for (name, np_value) in np_list[name]: - v = create_var(np_value, name, is_input, - if_return_inputs_grad_dict) + v = create_var( + np_value, name, is_input, if_return_inputs_grad_dict + ) var_list.append(v) if if_return_inputs_grad_dict: inputs_grad_dict[name] = v @@ -704,8 +783,12 @@ class OpTest(unittest.TestCase): else: nplist_value_temp = np_list[name] name_temp = unique_name.generate("%s_out" % (name)) - v = create_var(nplist_value_temp, name_temp, is_input, - if_return_inputs_grad_dict) + v = create_var( + nplist_value_temp, + name_temp, + is_input, + if_return_inputs_grad_dict, + ) var_dict[name].append(v) if if_return_inputs_grad_dict: inputs_grad_dict[name] = v @@ -716,10 +799,10 @@ class OpTest(unittest.TestCase): return var_dict def _check_api_outs_by_dygraph_outs(self, api_outs, dygraph_outs, place): - """ for quick verify, here we take a simplest strategy: - 1. we only check variable in api_outs. - 2. we simply check the numpy (tensor) . - 3. we set atol and rtol as 1e-5, because they are unrelated to dtype. + """for quick verify, here we take a simplest strategy: + 1. we only check variable in api_outs. + 2. we simply check the numpy (tensor) . + 3. we set atol and rtol as 1e-5, because they are unrelated to dtype. """ for name in api_outs: np_api = np.array(api_outs[name]) @@ -729,19 +812,28 @@ class OpTest(unittest.TestCase): np_dyg, rtol=1e-05, equal_nan=False, - err_msg='Output (' + name + ') has diff at ' + str(place) + - '\nExpect ' + str(np_dyg) + '\n' + 'But Got' + str(np_api) + - ' in class ' + self.__class__.__name__) + err_msg='Output (' + + name + + ') has diff at ' + + str(place) + + '\nExpect ' + + str(np_dyg) + + '\n' + + 'But Got' + + str(np_api) + + ' in class ' + + self.__class__.__name__, + ) def _calc_python_api_output(self, place, egr_inps=None, egr_oups=None): - """ set egr_inps and egr_oups = None if you want to create it by yourself. - """ + """set egr_inps and egr_oups = None if you want to create it by yourself.""" - def prepare_python_api_arguments(api, op_proto_ins, op_proto_attrs, - kernel_sig): - """ map from `op proto inputs and attrs` to `api input list and api attrs dict` + def prepare_python_api_arguments( + api, op_proto_ins, op_proto_attrs, kernel_sig + ): + """map from `op proto inputs and attrs` to `api input list and api attrs dict` - NOTE: the op_proto_attrs and op_proto_ins is a default dict. default value is [] + NOTE: the op_proto_attrs and op_proto_ins is a default dict. default value is [] """ class Empty: @@ -751,19 +843,19 @@ class OpTest(unittest.TestCase): return isinstance(a, Empty) def get_default(idx, defaults): - assert not isinstance( - defaults[idx], Empty - ), "%d-th params of python api don't have default value." % idx + assert not isinstance(defaults[idx], Empty), ( + "%d-th params of python api don't have default value." % idx + ) return defaults[idx] def to_defaults_list(params, defaults): return [defaults[p] for p in params if p in defaults] def parse_attri_value(name, op_inputs, op_attrs): - """ parse true value from inputs and attrs, if there is no name passed by OpTest, return Empty - 1. if the name in op_attrs, use the op_attrs[name] - 2. if the name in op_inputs, convert the op_inputs to [type of default value] - 3. if the name not in op_attrs ans op_inputs, return Empty. (this will use the default value from python api) + """parse true value from inputs and attrs, if there is no name passed by OpTest, return Empty + 1. if the name in op_attrs, use the op_attrs[name] + 2. if the name in op_inputs, convert the op_inputs to [type of default value] + 3. if the name not in op_attrs ans op_inputs, return Empty. (this will use the default value from python api) """ if name in op_proto_attrs: return op_proto_attrs[name] @@ -771,8 +863,9 @@ class OpTest(unittest.TestCase): if len(op_inputs[name]) == 1: # why don't use numpy().item() : if the Tensor is float64, we will change it to python.float32, where we loss accuracy: [allclose_op] # why we reconstruct a tensor: because we want the tensor in cpu. - return paddle.to_tensor(op_inputs[name][0].numpy(), - place='cpu') + return paddle.to_tensor( + op_inputs[name][0].numpy(), place='cpu' + ) else: # if this is a list (test_unsqueeze2_op): we just pass it into the python api. return op_inputs[name] @@ -799,7 +892,8 @@ class OpTest(unittest.TestCase): Empty() for i in range(len(api_params) - len(api_defaults)) ] + api_defaults assert len(api_defaults) == len( - api_params), "Error happens. contack xiongkun03 to solve." + api_params + ), "Error happens. contack xiongkun03 to solve." inputs_sig, attrs_sig, outputs_sig = kernel_sig inputs_and_attrs = inputs_sig + attrs_sig input_arguments = [ @@ -815,7 +909,7 @@ class OpTest(unittest.TestCase): if arg_name in api_ignore_param_list: results.append(get_default(idx, api_defaults)) else: - if (idx_of_op_proto_arguments < len(input_arguments)): + if idx_of_op_proto_arguments < len(input_arguments): tmp = input_arguments[idx_of_op_proto_arguments] idx_of_op_proto_arguments += 1 else: @@ -838,9 +932,9 @@ class OpTest(unittest.TestCase): return {a: [b] for a, b in zip(output_sig, ret_tuple)} else: # [assumption]: return multi-Tensor in a single output. such as paddle.split() - assert len( - output_sig - ) == 1, "Don't support multi-output with multi-tensor output. (May be you can use set `python_out_sig`, see `test_squeeze2_op` as a example.)" + assert ( + len(output_sig) == 1 + ), "Don't support multi-output with multi-tensor output. (May be you can use set `python_out_sig`, see `test_squeeze2_op` as a example.)" return {output_sig[0]: ret_tuple} def assumption_assert_and_transform(args, inp_num): @@ -853,29 +947,35 @@ class OpTest(unittest.TestCase): only support "X" is list of Tensor, currently don't support other structure like dict. """ - inp_args = [[inp] if inp is None else inp - for inp in args[:inp_num]] # convert None -> [None] + inp_args = [ + [inp] if inp is None else inp for inp in args[:inp_num] + ] # convert None -> [None] for inp in inp_args: assert isinstance( inp, list ), "currently only support `X` is [Tensor], don't support other structure." - args = [inp[0] if len(inp) == 1 else inp - for inp in inp_args] + args[inp_num:] + args = [ + inp[0] if len(inp) == 1 else inp for inp in inp_args + ] + args[inp_num:] return args - def _get_kernel_signature(eager_tensor_inputs, eager_tensor_outputs, - attrs_outputs): + def _get_kernel_signature( + eager_tensor_inputs, eager_tensor_outputs, attrs_outputs + ): try: kernel_sig = _dygraph_tracer()._get_kernel_signature( - self.op_type, eager_tensor_inputs, eager_tensor_outputs, - attrs_outputs) + self.op_type, + eager_tensor_inputs, + eager_tensor_outputs, + attrs_outputs, + ) except RuntimeError as re: - """ we think the kernel_sig is missing. - """ + """we think the kernel_sig is missing.""" kernel_sig = None print( "[Warning: op_test.py] Kernel Signature is not found for %s, fall back to intermediate state." - % self.op_type) + % self.op_type + ) return kernel_sig def cal_python_api(python_api, args, kernel_sig): @@ -888,11 +988,21 @@ class OpTest(unittest.TestCase): block = fluid.default_main_program().global_block() op_proto = OpProtoHolder.instance().get_op_proto(self.op_type) # prepare input variable - eager_tensor_inputs = egr_inps if egr_inps else self.append_input_output_for_dygraph( - op_proto, self.inputs, True, False, block) + eager_tensor_inputs = ( + egr_inps + if egr_inps + else self.append_input_output_for_dygraph( + op_proto, self.inputs, True, False, block + ) + ) # prepare output variable - eager_tensor_outputs = egr_oups if egr_oups else self.append_input_output_for_dygraph( - op_proto, self.outputs, False, False, block) + eager_tensor_outputs = ( + egr_oups + if egr_oups + else self.append_input_output_for_dygraph( + op_proto, self.outputs, False, False, block + ) + ) # prepare attributes attrs_outputs = {} @@ -901,23 +1011,26 @@ class OpTest(unittest.TestCase): if self.attrs[attrs_name] is not None: attrs_outputs[attrs_name] = self.attrs[attrs_name] - kernel_sig = _get_kernel_signature(eager_tensor_inputs, - eager_tensor_outputs, - attrs_outputs) + kernel_sig = _get_kernel_signature( + eager_tensor_inputs, eager_tensor_outputs, attrs_outputs + ) if not kernel_sig: return None - assert hasattr( - self, "python_api" - ), "Detect there is KernelSignature for `%s` op, please set the `self.python_api` if you set check_eager = True" % self.op_type - args = prepare_python_api_arguments(self.python_api, - eager_tensor_inputs, - attrs_outputs, kernel_sig) + assert hasattr(self, "python_api"), ( + "Detect there is KernelSignature for `%s` op, please set the `self.python_api` if you set check_eager = True" + % self.op_type + ) + args = prepare_python_api_arguments( + self.python_api, eager_tensor_inputs, attrs_outputs, kernel_sig + ) """ we directly return the cal_python_api value because the value is already tensor. """ return cal_python_api(self.python_api, args, kernel_sig) def _calc_dygraph_output(self, place, parallel=False, no_check_set=None): - self.__class__.op_type = self.op_type # for ci check, please not delete it for now + self.__class__.op_type = ( + self.op_type + ) # for ci check, please not delete it for now with fluid.dygraph.base.guard(place=place): block = fluid.default_main_program().global_block() @@ -925,10 +1038,12 @@ class OpTest(unittest.TestCase): # prepare input variable inputs = self.append_input_output_for_dygraph( - op_proto, self.inputs, True, False, block) + op_proto, self.inputs, True, False, block + ) # prepare output variable outputs = self.append_input_output_for_dygraph( - op_proto, self.outputs, False, False, block) + op_proto, self.outputs, False, False, block + ) # prepare attributes attrs_outputs = {} @@ -941,16 +1056,19 @@ class OpTest(unittest.TestCase): type=self.op_type, inputs=inputs, outputs=outputs, - attrs=attrs_outputs if hasattr(self, "attrs") else None) + attrs=attrs_outputs if hasattr(self, "attrs") else None, + ) return outputs - def _calc_output(self, - place, - parallel=False, - no_check_set=None, - loss=None, - enable_inplace=None, - for_inplace_test=None): + def _calc_output( + self, + place, + parallel=False, + no_check_set=None, + loss=None, + enable_inplace=None, + for_inplace_test=None, + ): program = Program() block = program.global_block() op = self._append_ops(block) @@ -974,7 +1092,8 @@ class OpTest(unittest.TestCase): if isinstance(place, fluid.CUDAPlace): use_cuda = True compiled_prog = fluid.CompiledProgram(program).with_data_parallel( - loss_name=loss.name if loss else None, places=place) + loss_name=loss.name if loss else None, places=place + ) program = compiled_prog fetch_list = getattr(self, "fetch_list", []) # if the fetch_list is customized by user, we use it directly. @@ -998,14 +1117,14 @@ class OpTest(unittest.TestCase): build_strategy.enable_inplace = enable_inplace compiled_prog = fluid.CompiledProgram(program).with_data_parallel( - build_strategy=build_strategy, places=place) + build_strategy=build_strategy, places=place + ) program = compiled_prog executor = Executor(place) - outs = executor.run(program, - feed=feed_map, - fetch_list=fetch_list, - return_numpy=False) + outs = executor.run( + program, feed=feed_map, fetch_list=fetch_list, return_numpy=False + ) self.op = op self.program = original_program if for_inplace_test: @@ -1013,12 +1132,9 @@ class OpTest(unittest.TestCase): else: return outs, fetch_list - def _compare_expect_and_actual_outputs(self, - place, - fetch_list, - expect_outs, - actual_outs, - inplace_atol=None): + def _compare_expect_and_actual_outputs( + self, place, fetch_list, expect_outs, actual_outs, inplace_atol=None + ): """Compare expect outs and actual outs of an tested op. Args: @@ -1045,21 +1161,41 @@ class OpTest(unittest.TestCase): actual_out, rtol=1e-05, atol=inplace_atol, - err_msg='Output (' + name + ') has diff at ' + str(place) + - ' when using and not using inplace' + '\nExpect ' + - str(expect_out) + '\n' + 'But Got' + str(actual_out) + - ' in class ' + self.__class__.__name__) + err_msg='Output (' + + name + + ') has diff at ' + + str(place) + + ' when using and not using inplace' + + '\nExpect ' + + str(expect_out) + + '\n' + + 'But Got' + + str(actual_out) + + ' in class ' + + self.__class__.__name__, + ) else: np.testing.assert_array_equal( expect_out, actual_out, - err_msg='Output (' + name + ') has diff at ' + str(place) + - ' when using and not using inplace' + '\nExpect ' + - str(expect_out) + '\n' + 'But Got' + str(actual_out) + - ' in class ' + self.__class__.__name__ + '\n') - - def _construct_grad_program_from_forward(self, fwd_program, grad_op_desc, - op_grad_to_var): + err_msg='Output (' + + name + + ') has diff at ' + + str(place) + + ' when using and not using inplace' + + '\nExpect ' + + str(expect_out) + + '\n' + + 'But Got' + + str(actual_out) + + ' in class ' + + self.__class__.__name__ + + '\n', + ) + + def _construct_grad_program_from_forward( + self, fwd_program, grad_op_desc, op_grad_to_var + ): """Generate grad_program which contains the grad_op. Args: @@ -1077,19 +1213,23 @@ class OpTest(unittest.TestCase): grad_program._sync_with_cpp() # Create grad vars based on fwd vars (shape and dtype) - for arg in grad_op_desc.input_arg_names( - ) + grad_op_desc.output_arg_names(): + for arg in ( + grad_op_desc.input_arg_names() + grad_op_desc.output_arg_names() + ): fwd_var_name = op_grad_to_var.get(arg, None) if fwd_var_name is None: fwd_var_name = arg fwd_var = fwd_program.global_block().vars.get(fwd_var_name) assert fwd_var is not None, "{} cannot be found".format( - fwd_var_name) - grad_var = grad_block.create_var(name=arg, - dtype=fwd_var.dtype, - shape=fwd_var.shape, - type=fwd_var.type, - persistable=False) + fwd_var_name + ) + grad_var = grad_block.create_var( + name=arg, + dtype=fwd_var.dtype, + shape=fwd_var.shape, + type=fwd_var.type, + persistable=False, + ) # Some variables' tensors hold no buffer (tensor's _holder is NULL), like XShape in reshape2 op, # and the shapes of those variables contain 0 (eg. Xshape.shape = [0, 2, 5]). @@ -1100,8 +1240,9 @@ class OpTest(unittest.TestCase): grad_program._sync_with_cpp() return grad_program - def _construct_grad_feed_map_from_forward(self, place, fwd_res, - grad_op_desc, op_grad_to_var): + def _construct_grad_feed_map_from_forward( + self, place, fwd_res, grad_op_desc, op_grad_to_var + ): """Generate grad_feed_map for grad_program. since we don`t really check gradient accuracy, but check the consistency when using and not using inplace, @@ -1117,7 +1258,13 @@ class OpTest(unittest.TestCase): Returns: grad_feed_map (dict): The feed_map of grad_op. """ - fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc = fwd_res + ( + fwd_outs, + fwd_fetch_list, + fwd_feed_map, + fwd_program, + fwd_op_desc, + ) = fwd_res p = core.Place() p.set_place(place) grad_feed_map = {} @@ -1167,14 +1314,16 @@ class OpTest(unittest.TestCase): else: # get grad_op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - op_desc, set(), []) + op_desc, set(), [] + ) if not grad_op_desc_list: has_infer_inplace_in_grad_descendants = False else: for i, grad_op_desc in enumerate(grad_op_desc_list): - if grad_op_desc.type( - ) not in visited_ops and _dfs_grad_op( - grad_op_desc, fwd_op_desc=op_desc): + if ( + grad_op_desc.type() not in visited_ops + and _dfs_grad_op(grad_op_desc, fwd_op_desc=op_desc) + ): has_infer_inplace_in_grad_descendants = True if has_infer_inplace or has_infer_inplace_in_grad_descendants: need_run_ops.append((op_desc, fwd_op_desc)) @@ -1185,10 +1334,9 @@ class OpTest(unittest.TestCase): _dfs_grad_op(op_desc, fwd_op_desc=fwd_op_desc) return need_run_ops - def _check_forward_inplace(self, - place, - no_check_set=None, - inplace_atol=None): + def _check_forward_inplace( + self, place, no_check_set=None, inplace_atol=None + ): """Check the inplace correctness of given op (self.op_type). Run the op twice with same inputs, one enable inplace and another disable, compare their outputs. @@ -1202,27 +1350,31 @@ class OpTest(unittest.TestCase): We return this to construct grad_program and grad_feed_map for grad inplace check. """ # _calc_output() returns in the form tuple(outs, fetch_list, feed_map, program, op_desc) when for_inplace_test=True. - expect_res = self._calc_output(place, - no_check_set=no_check_set, - enable_inplace=False, - for_inplace_test=True) - actual_res = self._calc_output(place, - no_check_set=no_check_set, - enable_inplace=True, - for_inplace_test=True) + expect_res = self._calc_output( + place, + no_check_set=no_check_set, + enable_inplace=False, + for_inplace_test=True, + ) + actual_res = self._calc_output( + place, + no_check_set=no_check_set, + enable_inplace=True, + for_inplace_test=True, + ) # compare expect_outs and actual_outs - self._compare_expect_and_actual_outputs(place, - expect_res[1], - expect_res[0], - actual_res[0], - inplace_atol=inplace_atol) + self._compare_expect_and_actual_outputs( + place, + expect_res[1], + expect_res[0], + actual_res[0], + inplace_atol=inplace_atol, + ) return expect_res - def _calc_grad_output(self, - place, - fwd_res, - grad_op_desc, - enable_inplace=None): + def _calc_grad_output( + self, place, fwd_res, grad_op_desc, enable_inplace=None + ): """Calculate grad_output for given grad_op_desc. since we don`t really check gradient accuracy, but check the consistency when using and not using inplace, @@ -1238,13 +1390,22 @@ class OpTest(unittest.TestCase): Returns: res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given grad_op_desc. """ - fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc = fwd_res + ( + fwd_outs, + fwd_fetch_list, + fwd_feed_map, + fwd_program, + fwd_op_desc, + ) = fwd_res grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - fwd_op_desc, set(), []) + fwd_op_desc, set(), [] + ) grad_program = self._construct_grad_program_from_forward( - fwd_program, grad_op_desc, op_grad_to_var) + fwd_program, grad_op_desc, op_grad_to_var + ) grad_feed_map = self._construct_grad_feed_map_from_forward( - place, fwd_res, grad_op_desc, op_grad_to_var) + place, fwd_res, grad_op_desc, op_grad_to_var + ) grad_fetch_list = grad_op_desc.output_arg_names() exe = Executor(place) program = grad_program @@ -1252,22 +1413,23 @@ class OpTest(unittest.TestCase): build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = enable_inplace compiled_program = fluid.CompiledProgram( - grad_program).with_data_parallel(loss_name="", - build_strategy=build_strategy, - places=place) + grad_program + ).with_data_parallel( + loss_name="", build_strategy=build_strategy, places=place + ) program = compiled_program - outs = exe.run(program, - feed=grad_feed_map, - fetch_list=grad_fetch_list, - return_numpy=False) + outs = exe.run( + program, + feed=grad_feed_map, + fetch_list=grad_fetch_list, + return_numpy=False, + ) return outs, grad_fetch_list, grad_feed_map, grad_program, grad_op_desc - def _check_grad_inplace(self, - place, - fwd_res, - grad_op_desc, - inplace_atol=None): + def _check_grad_inplace( + self, place, fwd_res, grad_op_desc, inplace_atol=None + ): """Check the inplace correctness of given grad_op_desc. Run the grad op twice with same inputs, one enable inplace and another disable, compare their outputs. @@ -1285,26 +1447,25 @@ class OpTest(unittest.TestCase): expect_res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given op. We return this to construct grad_program and grad_feed_map for grad inplace check. """ - expect_res = self._calc_grad_output(place, - fwd_res, - grad_op_desc, - enable_inplace=False) - actual_res = self._calc_grad_output(place, - fwd_res, - grad_op_desc, - enable_inplace=True) - - self._compare_expect_and_actual_outputs(place, - expect_res[1], - expect_res[0], - actual_res[0], - inplace_atol=inplace_atol) + expect_res = self._calc_grad_output( + place, fwd_res, grad_op_desc, enable_inplace=False + ) + actual_res = self._calc_grad_output( + place, fwd_res, grad_op_desc, enable_inplace=True + ) + + self._compare_expect_and_actual_outputs( + place, + expect_res[1], + expect_res[0], + actual_res[0], + inplace_atol=inplace_atol, + ) return expect_res - def check_inplace_output_with_place(self, - place, - no_check_set=None, - inplace_atol=None): + def check_inplace_output_with_place( + self, place, no_check_set=None, inplace_atol=None + ): """Chech the inplace correctness of given op, its grad op, its grad_grad op, etc. (1) Get all ops need to run. (see conditions in _get_need_run_ops()) @@ -1324,9 +1485,9 @@ class OpTest(unittest.TestCase): has_infer_inplace = fluid.core.has_infer_inplace(self.op_type) has_grad_op_maker = fluid.core.has_grad_op_maker(self.op_type) - fwd_res = self._calc_output(place, - no_check_set=no_check_set, - for_inplace_test=True) + fwd_res = self._calc_output( + place, no_check_set=no_check_set, for_inplace_test=True + ) op_desc = fwd_res[4] need_run_ops = self._get_need_run_ops(op_desc) @@ -1341,17 +1502,19 @@ class OpTest(unittest.TestCase): res[op_desc] = self._check_forward_inplace( place, no_check_set=no_check_set, - inplace_atol=inplace_atol) + inplace_atol=inplace_atol, + ) else: - res[op_desc] = self._calc_output(place, - no_check_set=no_check_set, - for_inplace_test=True) + res[op_desc] = self._calc_output( + place, no_check_set=no_check_set, for_inplace_test=True + ) else: # TODO(zhiqiu): enhance inplace_grad test for ops (sum and activation) using mkldnn # skip op that use_mkldnn currently flags_use_mkldnn = fluid.core.globals()["FLAGS_use_mkldnn"] attrs_use_mkldnn = hasattr(self, 'attrs') and bool( - self.attrs.get('use_mkldnn', False)) + self.attrs.get('use_mkldnn', False) + ) if flags_use_mkldnn or attrs_use_mkldnn: warnings.warn( "check inplace_grad for ops using mkldnn is not supported" @@ -1360,19 +1523,23 @@ class OpTest(unittest.TestCase): if has_infer_inplace: fwd_res = res[father_op_desc] res[op_desc] = self._check_grad_inplace( - place, fwd_res, op_desc, inplace_atol=inplace_atol) + place, fwd_res, op_desc, inplace_atol=inplace_atol + ) else: res[op_desc] = self._calc_grad_output( - place, fwd_res, op_desc) + place, fwd_res, op_desc + ) - def check_output_with_place(self, - place, - atol=0, - no_check_set=None, - equal_nan=False, - check_dygraph=True, - inplace_atol=None, - check_eager=False): + def check_output_with_place( + self, + place, + atol=0, + no_check_set=None, + equal_nan=False, + check_dygraph=True, + inplace_atol=None, + check_eager=False, + ): # disable legacy dygraph check when check_eager is True if check_eager == True: @@ -1387,26 +1554,29 @@ class OpTest(unittest.TestCase): if var.name == target_name: return dygraph_outs[name][i] self.assertTrue( - False, "Found failed {} {}".format(dygraph_outs.keys(), - target_name)) + False, + "Found failed {} {}".format(dygraph_outs.keys(), target_name), + ) def find_actual(target_name, fetch_list): found = [ - i for i, var_name in enumerate(fetch_list) + i + for i, var_name in enumerate(fetch_list) if var_name == target_name ] self.assertTrue( - len(found) == 1, "Found {} {}".format(len(found), target_name)) + len(found) == 1, "Found {} {}".format(len(found), target_name) + ) return found[0] class Checker(object): - """ base class for check with self.outputs. - currently don't support check between checkers. + """base class for check with self.outputs. + currently don't support check between checkers. """ def __init__(self, op_test, expect_dict): - """ expect_dict is the self.outputs - support : {str: [numpy]} and {str: [(str, numpy), (str, numpy)]} + """expect_dict is the self.outputs + support : {str: [numpy]} and {str: [(str, numpy), (str, numpy)]} """ self.expects = expect_dict self.checker_name = "checker" @@ -1433,8 +1603,7 @@ class OpTest(unittest.TestCase): return False def find_actual_value(self, name): - """ return: (actual_tensor(var_base), actual_numpy) - """ + """return: (actual_tensor(var_base), actual_numpy)""" raise NotImplementedError("base class, not implement!") def _compare_numpy(self, name, actual_np, expect_np): @@ -1444,20 +1613,26 @@ class OpTest(unittest.TestCase): expect_np, atol=atol, rtol=self.rtol if hasattr(self, 'rtol') else 1e-5, - equal_nan=equal_nan), "Output (" + name + - ") has diff at " + str(place) + " in " + self.checker_name) + equal_nan=equal_nan, + ), + "Output (" + + name + + ") has diff at " + + str(place) + + " in " + + self.checker_name, + ) def _compare_list(self, name, actual, expect): - """ if expect is a tuple, we need to compare list. - """ + """if expect is a tuple, we need to compare list.""" raise NotImplementedError("base class, not implement!") def compare_single_output_with_expect(self, name, expect): actual, actual_np = self.find_actual_value(name) - expect_np = expect[0] \ - if isinstance(expect, tuple) else expect + expect_np = expect[0] if isinstance(expect, tuple) else expect actual_np, expect_np = self.convert_uint16_to_float_ifneed( - actual_np, expect_np) + actual_np, expect_np + ) # NOTE(zhiqiu): np.allclose([], [1.]) returns True # see details: https://stackoverflow.com/questions/38331703/why-does-numpys-broadcasting-sometimes-allow-comparing-arrays-of-different-leng if expect_np.size == 0: @@ -1468,17 +1643,20 @@ class OpTest(unittest.TestCase): def compare_outputs_with_expects(self): for out_name, out_dup in Operator.get_op_outputs(self.op_type): - if self._is_skip_name(out_name): continue + if self._is_skip_name(out_name): + continue if out_dup: # if self.output = {'name': [(subname, Tensor), (subname, Tensor)]} sub_out = self.expects[out_name] if not isinstance(sub_out, list): - raise AssertionError("sub_out type %s is not list", - type(sub_out)) + raise AssertionError( + "sub_out type %s is not list", type(sub_out) + ) for item in sub_out: sub_out_name, expect = item[0], item[1] self.compare_single_output_with_expect( - sub_out_name, expect) + sub_out_name, expect + ) else: expect = self.expects[out_name] self.compare_single_output_with_expect(out_name, expect) @@ -1494,13 +1672,13 @@ class OpTest(unittest.TestCase): self.compare_outputs_with_expects() class StaticChecker(Checker): - def init(self): self.checker_name = "static checker" def calculate_output(self): outs, fetch_list = self.op_test._calc_output( - place, no_check_set=no_check_set) + place, no_check_set=no_check_set + ) self.outputs = outs self.fetch_list = fetch_list @@ -1516,13 +1694,17 @@ class OpTest(unittest.TestCase): return True | False """ if actual_np.dtype == np.uint16 and expect_np.dtype in [ - np.float32, np.float64 + np.float32, + np.float64, ]: actual_np = convert_uint16_to_float(actual_np) - self.rtol = 1.e-2 + self.rtol = 1.0e-2 else: - self.rtol = 1.e-5 - if expect_np.dtype == np.uint16 and actual_np.dtype == np.uint16: + self.rtol = 1.0e-5 + if ( + expect_np.dtype == np.uint16 + and actual_np.dtype == np.uint16 + ): nonlocal atol expect_np = convert_uint16_to_float(expect_np) actual_np = convert_uint16_to_float(actual_np) @@ -1530,36 +1712,40 @@ class OpTest(unittest.TestCase): return actual_np, expect_np def _compare_list(self, name, actual, expect): - """ if expect is a tuple, we need to compare list. - """ + """if expect is a tuple, we need to compare list.""" self.op_test.assertListEqual( - actual.recursive_sequence_lengths(), expect[1], - "Output (" + name + ") has different lod at " + str(place)) + actual.recursive_sequence_lengths(), + expect[1], + "Output (" + name + ") has different lod at " + str(place), + ) class DygraphChecker(Checker): - def init(self): self.checker_name = "dygraph checker" def calculate_output(self): self.outputs = self.op_test._calc_dygraph_output( - place, no_check_set=no_check_set) + place, no_check_set=no_check_set + ) def find_actual_value(self, name): with fluid.dygraph.base.guard(place=place): imperative_actual = find_imperative_actual( - name, self.outputs, place) + name, self.outputs, place + ) imperative_actual_t = np.array( - imperative_actual.value().get_tensor()) + imperative_actual.value().get_tensor() + ) return imperative_actual, imperative_actual_t def convert_uint16_to_float_ifneed(self, actual_np, expect_np): if actual_np.dtype == np.uint16 and expect_np.dtype in [ - np.float32, np.float64 + np.float32, + np.float64, ]: - self.rtol = 1.e-2 + self.rtol = 1.0e-2 else: - self.rtol = 1.e-5 + self.rtol = 1.0e-5 if self.op_test.is_bfloat16_op(): if actual_np.dtype == np.uint16: actual_np = convert_uint16_to_float(actual_np) @@ -1568,20 +1754,27 @@ class OpTest(unittest.TestCase): return actual_np, expect_np def _compare_list(self, name, actual, expect): - """ if expect is a tuple, we need to compare list. - """ + """if expect is a tuple, we need to compare list.""" with fluid.dygraph.base.guard(place=place): self.op_test.assertListEqual( - actual.value().get_tensor().recursive_sequence_lengths( - ), expect[1], - "Output (" + name + ") has different lod at " + - str(place) + " in dygraph mode") + actual.value() + .get_tensor() + .recursive_sequence_lengths(), + expect[1], + "Output (" + + name + + ") has different lod at " + + str(place) + + " in dygraph mode", + ) def _compare_numpy(self, name, actual_np, expect_np): - if functools.reduce(lambda x, y: x * y, actual_np.shape, - 1) == 0 and functools.reduce( - lambda x, y: x * y, expect_np.shape, - 1) == 0: + if ( + functools.reduce(lambda x, y: x * y, actual_np.shape, 1) + == 0 + and functools.reduce(lambda x, y: x * y, expect_np.shape, 1) + == 0 + ): pass else: self.op_test.assertTrue( @@ -1590,12 +1783,17 @@ class OpTest(unittest.TestCase): expect_np, atol=atol, rtol=self.rtol if hasattr(self, 'rtol') else 1e-5, - equal_nan=equal_nan), - "Output (" + name + ") has diff at " + str(place) + - " in " + self.checker_name) + equal_nan=equal_nan, + ), + "Output (" + + name + + ") has diff at " + + str(place) + + " in " + + self.checker_name, + ) class EagerChecker(DygraphChecker): - def init(self): self.checker_name = "eager checker" @@ -1604,12 +1802,14 @@ class OpTest(unittest.TestCase): with _test_eager_guard(): self.is_python_api_test = True eager_dygraph_outs = self.op_test._calc_python_api_output( - place) + place + ) if eager_dygraph_outs is None: self.is_python_api_test = False # missing KernelSignature, fall back to eager middle output. eager_dygraph_outs = self.op_test._calc_dygraph_output( - place, no_check_set=no_check_set) + place, no_check_set=no_check_set + ) self.outputs = eager_dygraph_outs def _compare_numpy(self, name, actual_np, expect_np): @@ -1619,30 +1819,35 @@ class OpTest(unittest.TestCase): def convert_uint16_to_float_ifneed(self, actual_np, expect_np): with _test_eager_guard(): return super().convert_uint16_to_float_ifneed( - actual_np, expect_np) + actual_np, expect_np + ) def find_actual_value(self, name): with _test_eager_guard(): return super().find_actual_value(name) def _compare_list(self, name, actual, expect): - """ if expect is a tuple, we need to compare list. - """ + """if expect is a tuple, we need to compare list.""" with _test_eager_guard(): super()._compare_list(name, actual, expect) def _is_skip_name(self, name): # if in final state and kernel signature don't have name, then skip it. - if self.is_python_api_test and hasattr( - self.op_test, "python_out_sig" - ) and name not in self.op_test.python_out_sig: + if ( + self.is_python_api_test + and hasattr(self.op_test, "python_out_sig") + and name not in self.op_test.python_out_sig + ): return True return super()._is_skip_name(name) # set some flags by the combination of arguments. self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST + ): atol = 0 if self.is_bfloat16_op(): @@ -1650,7 +1855,8 @@ class OpTest(unittest.TestCase): check_dygraph = False check_eager = False if hasattr(self, 'force_fp32_output') and getattr( - self, 'force_fp32_output'): + self, 'force_fp32_output' + ): atol = 1e-2 else: atol = 2 @@ -1658,9 +1864,13 @@ class OpTest(unittest.TestCase): atol = 1e-1 if no_check_set is not None: - if self.op_type not in no_check_set_white_list.no_check_set_white_list: + if ( + self.op_type + not in no_check_set_white_list.no_check_set_white_list + ): raise AssertionError( - "no_check_set of op %s must be set to None." % self.op_type) + "no_check_set of op %s must be set to None." % self.op_type + ) static_checker = StaticChecker(self, self.outputs) static_checker.check() outs, fetch_list = static_checker.outputs, static_checker.fetch_list @@ -1691,13 +1901,15 @@ class OpTest(unittest.TestCase): # Check inplace for given op, its grad op, its grad_grad op, etc. # No effect on original OpTest # Currently not support ParallelExecutor on XPUPlace. - if not paddle.is_compiled_with_xpu( - ) and not paddle.is_compiled_with_npu( - ) and not paddle.is_compiled_with_mlu() and not isinstance( - place, core.CustomPlace): - self.check_inplace_output_with_place(place, - no_check_set=no_check_set, - inplace_atol=inplace_atol) + if ( + not paddle.is_compiled_with_xpu() + and not paddle.is_compiled_with_npu() + and not paddle.is_compiled_with_mlu() + and not isinstance(place, core.CustomPlace) + ): + self.check_inplace_output_with_place( + place, no_check_set=no_check_set, inplace_atol=inplace_atol + ) if check_eager: assert check_dygraph == False @@ -1708,10 +1920,10 @@ class OpTest(unittest.TestCase): return outs, fetch_list def check_compile_vs_runtime(self, fetch_list, fetch_outs): - def find_fetch_index(target_name, fetch_list): found = [ - i for i, var_name in enumerate(fetch_list) + i + for i, var_name in enumerate(fetch_list) if var_name == target_name ] if len(found) == 0: @@ -1719,7 +1931,8 @@ class OpTest(unittest.TestCase): else: self.assertTrue( len(found) == 1, - "Found {} {}".format(len(found), target_name)) + "Found {} {}".format(len(found), target_name), + ) return found[0] for name in self.op.desc.output_names(): @@ -1745,16 +1958,22 @@ class OpTest(unittest.TestCase): else: lod_level_compile = 0 self.assertEqual( - lod_level_compile, lod_level_runtime, - "The lod_level of Output (" + name + - ") is different between compile-time and runtime (" + - str(lod_level_compile) + " vs " + str(lod_level_runtime) + - ")") + lod_level_compile, + lod_level_runtime, + "The lod_level of Output (" + + name + + ") is different between compile-time and runtime (" + + str(lod_level_compile) + + " vs " + + str(lod_level_runtime) + + ")", + ) def _get_places(self): if self.dtype == np.float16: if core.is_compiled_with_cuda() and core.op_support_gpu( - self.op_type): + self.op_type + ): place = core.CUDAPlace(0) if core.is_float16_supported(place): return [place] @@ -1764,18 +1983,23 @@ class OpTest(unittest.TestCase): return [] places = [fluid.CPUPlace()] cpu_only = self._cpu_only if hasattr(self, '_cpu_only') else False - if core.is_compiled_with_cuda() and core.op_support_gpu(self.op_type)\ - and not cpu_only: + if ( + core.is_compiled_with_cuda() + and core.op_support_gpu(self.op_type) + and not cpu_only + ): places.append(core.CUDAPlace(0)) return places - def check_output(self, - atol=1e-5, - no_check_set=None, - equal_nan=False, - check_dygraph=True, - inplace_atol=None, - check_eager=False): + def check_output( + self, + atol=1e-5, + no_check_set=None, + equal_nan=False, + check_dygraph=True, + inplace_atol=None, + check_eager=False, + ): # disable legacy dygraph check when check_eager is True if check_eager == True: @@ -1790,13 +2014,15 @@ class OpTest(unittest.TestCase): places = self._get_places() for place in places: - res = self.check_output_with_place(place, - atol, - no_check_set, - equal_nan, - check_dygraph, - inplace_atol, - check_eager=check_eager) + res = self.check_output_with_place( + place, + atol, + no_check_set, + equal_nan, + check_dygraph, + inplace_atol, + check_eager=check_eager, + ) if check_eager: assert check_dygraph == False outs, eager_dygraph_outs, fetch_list = res @@ -1804,7 +2030,10 @@ class OpTest(unittest.TestCase): outs, dygraph_outs, fetch_list = res else: outs, fetch_list = res - if self.op_type not in compile_vs_runtime_white_list.COMPILE_RUN_OP_WHITE_LIST: + if ( + self.op_type + not in compile_vs_runtime_white_list.COMPILE_RUN_OP_WHITE_LIST + ): self.check_compile_vs_runtime(fetch_list, outs) def check_output_customized(self, checker, custom_place=None): @@ -1823,8 +2052,14 @@ class OpTest(unittest.TestCase): outs.sort(key=len) checker(outs) - def _assert_is_close(self, numeric_grads, analytic_grads, names, - max_relative_error, msg_prefix): + def _assert_is_close( + self, + numeric_grads, + analytic_grads, + names, + max_relative_error, + msg_prefix, + ): for a, b, name in zip(numeric_grads, analytic_grads, names): # It asserts np.abs(a - b) / np.abs(a) < max_relative_error, in which # max_relative_error is 1e-7. According to the value of np.abs(a), we @@ -1834,8 +2069,11 @@ class OpTest(unittest.TestCase): # which is the same as np.abs(a - b) / np.abs(a) < max_relative_error*1e4. abs_a = np.abs(a) if abs_a.ndim > 0: - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST + ): abs_a[abs_a < 1e-10] = 1e-3 abs_a[np.logical_and(abs_a > 1e-10, abs_a <= 1e-8)] *= 1e4 abs_a[np.logical_and(abs_a > 1e-8, abs_a <= 1e-6)] *= 1e2 @@ -1844,8 +2082,11 @@ class OpTest(unittest.TestCase): else: abs_a[abs_a < 1e-3] = 1 elif abs_a.ndim == 0: - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST + ): if abs_a < 1e-10: abs_a = 1e-3 elif abs_a > 1e-10 and abs_a <= 1e-8: @@ -1862,10 +2103,21 @@ class OpTest(unittest.TestCase): def err_msg(): offset = np.argmax(diff_mat > max_relative_error) - return ("Operator %s error, %s variable %s (shape: %s, dtype: %s) max gradient diff %e over limit %e, " - "the first error element is %d, expected %e, but got %e.") \ - % (self.op_type, msg_prefix, name, str(a.shape), self.dtype, max_diff, max_relative_error, - offset, a.flatten()[offset], b.flatten()[offset]) + return ( + "Operator %s error, %s variable %s (shape: %s, dtype: %s) max gradient diff %e over limit %e, " + "the first error element is %d, expected %e, but got %e." + ) % ( + self.op_type, + msg_prefix, + name, + str(a.shape), + self.dtype, + max_diff, + max_relative_error, + offset, + a.flatten()[offset], + b.flatten()[offset], + ) self.assertLessEqual(max_diff, max_relative_error, err_msg()) @@ -1876,17 +2128,19 @@ class OpTest(unittest.TestCase): if self.dtype == np.float64: self.__class__.exist_fp64_check_grad = True - def check_grad(self, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grads=None, - user_defined_grad_outputs=None, - check_dygraph=True, - check_eager=False): + def check_grad( + self, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grads=None, + user_defined_grad_outputs=None, + check_dygraph=True, + check_eager=False, + ): # disable legacy dygraph check when check_eager is True if check_eager == True: @@ -1895,31 +2149,35 @@ class OpTest(unittest.TestCase): self._check_grad_helper() places = self._get_places() for place in places: - self.check_grad_with_place(place, - inputs_to_check, - output_names, - no_grad_set, - numeric_grad_delta, - in_place, - max_relative_error, - user_defined_grads, - user_defined_grad_outputs, - check_dygraph, - check_eager=check_eager) - - def check_grad_with_place(self, - place, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grads=None, - user_defined_grad_outputs=None, - check_dygraph=True, - numeric_place=None, - check_eager=False): + self.check_grad_with_place( + place, + inputs_to_check, + output_names, + no_grad_set, + numeric_grad_delta, + in_place, + max_relative_error, + user_defined_grads, + user_defined_grad_outputs, + check_dygraph, + check_eager=check_eager, + ) + + def check_grad_with_place( + self, + place, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grads=None, + user_defined_grad_outputs=None, + check_dygraph=True, + numeric_place=None, + check_eager=False, + ): # disable legacy dygraph check when check_eager is True if check_eager == True: @@ -1935,8 +2193,11 @@ class OpTest(unittest.TestCase): check_dygraph = False check_eager = False - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST + ): numeric_grad_delta = 1e-5 max_relative_error = 1e-7 @@ -1950,12 +2211,14 @@ class OpTest(unittest.TestCase): op_attrs["use_mkldnn"] = False use_onednn = True - self.op = create_op(self.scope, - self.op_type, - op_inputs, - op_outputs, - op_attrs, - cache_list=cache_list) + self.op = create_op( + self.scope, + self.op_type, + op_inputs, + op_outputs, + op_attrs, + cache_list=cache_list, + ) if use_onednn: op_attrs["use_mkldnn"] = True @@ -1963,18 +2226,25 @@ class OpTest(unittest.TestCase): if no_grad_set is None: no_grad_set = set() else: - if (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST - ) and (self.op_type - not in no_grad_set_white_list.NOT_CHECK_OP_LIST) and ( - not self.is_bfloat16_op()): - raise AssertionError("no_grad_set must be None, op_type is " + - self.op_type + " Op.") + if ( + (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST) + and ( + self.op_type not in no_grad_set_white_list.NOT_CHECK_OP_LIST + ) + and (not self.is_bfloat16_op()) + ): + raise AssertionError( + "no_grad_set must be None, op_type is " + + self.op_type + + " Op." + ) for input_to_check in inputs_to_check: set_input(self.scope, self.op, self.inputs, place) tensor_to_check = self.scope.find_var(input_to_check).get_tensor() - tensor_size = functools.reduce(lambda a, b: a * b, - tensor_to_check.shape(), 1) + tensor_size = functools.reduce( + lambda a, b: a * b, tensor_to_check.shape(), 1 + ) tensor_ndim = len(tensor_to_check.shape()) # for 0D Tensor, it's additional case for OP, so not raise error if tensor_ndim > 0 and tensor_size < 100: @@ -1987,26 +2257,34 @@ class OpTest(unittest.TestCase): numeric_place = place numeric_grads = user_defined_grads or [ - get_numeric_gradient(numeric_place, - self.scope, - self.op, - self.inputs, - input_to_check, - output_names, - delta=numeric_grad_delta, - in_place=in_place) + get_numeric_gradient( + numeric_place, + self.scope, + self.op, + self.inputs, + input_to_check, + output_names, + delta=numeric_grad_delta, + in_place=in_place, + ) for input_to_check in inputs_to_check ] - analytic_grads = self._get_gradient(inputs_to_check, place, - output_names, no_grad_set, - user_defined_grad_outputs) + analytic_grads = self._get_gradient( + inputs_to_check, + place, + output_names, + no_grad_set, + user_defined_grad_outputs, + ) # comparison of bf16 results will happen as fp32 # loop over list of grads and convert bf16 to fp32 fp32_analytic_grads = [] for grad in analytic_grads: if grad.dtype == np.uint16: grad = convert_uint16_to_float(grad) - max_relative_error = 0.04 if max_relative_error < 0.04 else max_relative_error + max_relative_error = ( + 0.04 if max_relative_error < 0.04 else max_relative_error + ) fp32_analytic_grads.append(grad) analytic_grads = fp32_analytic_grads @@ -2014,32 +2292,50 @@ class OpTest(unittest.TestCase): for grad in numeric_grads: if grad.dtype == np.uint16: grad = convert_uint16_to_float(grad) - max_relative_error = 0.04 if max_relative_error < 0.04 else max_relative_error + max_relative_error = ( + 0.04 if max_relative_error < 0.04 else max_relative_error + ) fp32_numeric_grads.append(grad) numeric_grads = fp32_numeric_grads - self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check, - max_relative_error, - "Gradient Check On %s" % str(place)) + self._assert_is_close( + numeric_grads, + analytic_grads, + inputs_to_check, + max_relative_error, + "Gradient Check On %s" % str(place), + ) if check_dygraph: # ensure switch into legacy dygraph g_enable_legacy_dygraph() - dygraph_grad = self._get_dygraph_grad(inputs_to_check, place, - output_names, - user_defined_grad_outputs, - no_grad_set, False) + dygraph_grad = self._get_dygraph_grad( + inputs_to_check, + place, + output_names, + user_defined_grad_outputs, + no_grad_set, + False, + ) fp32_grads = [] for grad in dygraph_grad: if grad.dtype == np.uint16: grad = convert_uint16_to_float(grad) - max_relative_error = 0.03 if max_relative_error < 0.03 else max_relative_error + max_relative_error = ( + 0.03 + if max_relative_error < 0.03 + else max_relative_error + ) fp32_grads.append(grad) dygraph_grad = fp32_grads - self._assert_is_close(numeric_grads, dygraph_grad, inputs_to_check, - max_relative_error, - "Gradient Check On %s" % str(place)) + self._assert_is_close( + numeric_grads, + dygraph_grad, + inputs_to_check, + max_relative_error, + "Gradient Check On %s" % str(place), + ) # ensure switch back eager dygraph g_disable_legacy_dygraph() @@ -2047,18 +2343,31 @@ class OpTest(unittest.TestCase): with fluid.dygraph.base.guard(place): with _test_eager_guard(): eager_dygraph_grad = self._get_dygraph_grad( - inputs_to_check, place, output_names, - user_defined_grad_outputs, no_grad_set, check_eager) + inputs_to_check, + place, + output_names, + user_defined_grad_outputs, + no_grad_set, + check_eager, + ) fp32_grads = [] for grad in eager_dygraph_grad: if grad.dtype == np.uint16: grad = convert_uint16_to_float(grad) - max_relative_error = 0.03 if max_relative_error < 0.03 else max_relative_error + max_relative_error = ( + 0.03 + if max_relative_error < 0.03 + else max_relative_error + ) fp32_grads.append(grad) eager_dygraph_grad = fp32_grads - self._assert_is_close(numeric_grads, eager_dygraph_grad, - inputs_to_check, max_relative_error, - "Gradient Check On %s" % str(place)) + self._assert_is_close( + numeric_grads, + eager_dygraph_grad, + inputs_to_check, + max_relative_error, + "Gradient Check On %s" % str(place), + ) def _find_var_in_dygraph(self, output_vars, name): if name in output_vars: @@ -2069,13 +2378,15 @@ class OpTest(unittest.TestCase): if output_vars_selected.name == name: return output_vars_selected - def _get_dygraph_grad(self, - inputs_to_check, - place, - output_names, - user_defined_grad_outputs=None, - no_grad_set=None, - check_eager=False): + def _get_dygraph_grad( + self, + inputs_to_check, + place, + output_names, + user_defined_grad_outputs=None, + no_grad_set=None, + check_eager=False, + ): with fluid.dygraph.base.guard(place=place): block = fluid.default_main_program().global_block() @@ -2083,11 +2394,13 @@ class OpTest(unittest.TestCase): # prepare input variable inputs, inputs_grad_dict = self.append_input_output_for_dygraph( - op_proto, self.inputs, True, True, block) + op_proto, self.inputs, True, True, block + ) # prepare output variable outputs = self.append_input_output_for_dygraph( - op_proto, self.outputs, False, False, block) + op_proto, self.outputs, False, False, block + ) # prepare attributes attrs_outputs = {} @@ -2098,37 +2411,42 @@ class OpTest(unittest.TestCase): if check_eager: eager_outputs = self._calc_python_api_output( - place, inputs, outputs) + place, inputs, outputs + ) # if outputs is None, kernel sig is empty or other error is happens. if not check_eager or eager_outputs is None: block.append_op( type=self.op_type, inputs=inputs, outputs=outputs, - attrs=attrs_outputs if hasattr(self, "attrs") else None) + attrs=attrs_outputs if hasattr(self, "attrs") else None, + ) else: outputs = eager_outputs if self.dtype == np.uint16: - cast_inputs = self._find_var_in_dygraph(outputs, - output_names[0]) - cast_outputs = block.create_var(dtype="float32", - shape=cast_inputs[0].shape) - cast_op = block.append_op(inputs={"X": cast_inputs}, - outputs={"Out": cast_outputs}, - type="cast", - attrs={ - "in_dtype": - core.VarDesc.VarType.BF16, - "out_dtype": - core.VarDesc.VarType.FP32 - }) + cast_inputs = self._find_var_in_dygraph( + outputs, output_names[0] + ) + cast_outputs = block.create_var( + dtype="float32", shape=cast_inputs[0].shape + ) + cast_op = block.append_op( + inputs={"X": cast_inputs}, + outputs={"Out": cast_outputs}, + type="cast", + attrs={ + "in_dtype": core.VarDesc.VarType.BF16, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) outputs = {output_names[0]: cast_outputs} outputs_valid = {} for output_name in output_names: outputs_valid[output_name] = self._find_var_in_dygraph( - outputs, output_name) + outputs, output_name + ) if user_defined_grad_outputs is None: if len(outputs_valid) == 1: @@ -2137,13 +2455,15 @@ class OpTest(unittest.TestCase): type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, stop_gradient=False, - shape=[1]) + shape=[1], + ) for outputs_valid_key in outputs_valid: block.append_op( type="mean", inputs={"X": outputs_valid[outputs_valid_key]}, outputs={"Out": [loss]}, - attrs=None) + attrs=None, + ) else: avg_sum = [] for cur_loss in outputs_valid: @@ -2151,32 +2471,41 @@ class OpTest(unittest.TestCase): dtype=self.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - block.append_op(type="mean", - inputs={"X": outputs_valid[cur_loss]}, - outputs={"Out": [cur_avg_loss]}, - attrs=None) + stop_gradient=False, + ) + block.append_op( + type="mean", + inputs={"X": outputs_valid[cur_loss]}, + outputs={"Out": [cur_avg_loss]}, + attrs=None, + ) avg_sum.append(cur_avg_loss) loss_sum = block.create_var( dtype=self.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, stop_gradient=False, - shape=[1]) - block.append_op(type='sum', - inputs={"X": avg_sum}, - outputs={"Out": loss_sum}, - attrs=None) + shape=[1], + ) + block.append_op( + type='sum', + inputs={"X": avg_sum}, + outputs={"Out": loss_sum}, + attrs=None, + ) loss = block.create_var( dtype=self.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, stop_gradient=False, - shape=[1]) - block.append_op(type='scale', - inputs={"X": loss_sum}, - outputs={"Out": loss}, - attrs={'scale': 1.0 / float(len(avg_sum))}) + shape=[1], + ) + block.append_op( + type='scale', + inputs={"X": loss_sum}, + outputs={"Out": loss}, + attrs={'scale': 1.0 / float(len(avg_sum))}, + ) loss.backward() fetch_list_grad = [] @@ -2193,11 +2522,12 @@ class OpTest(unittest.TestCase): grad_outputs.append(paddle.to_tensor(grad_out_value)) # delete the inputs which no need to calculate grad for no_grad_val in no_grad_set: - del (inputs[no_grad_val]) + del inputs[no_grad_val] if not _in_legacy_dygraph(): - core.eager.run_backward(fluid.layers.utils.flatten(outputs), - grad_outputs, False) + core.eager.run_backward( + fluid.layers.utils.flatten(outputs), grad_outputs, False + ) grad_inputs = [] for inputs_list in inputs.values(): for inp in inputs_list: @@ -2207,7 +2537,8 @@ class OpTest(unittest.TestCase): grad_inputs = paddle.grad( outputs=fluid.layers.utils.flatten(outputs), inputs=fluid.layers.utils.flatten(inputs), - grad_outputs=grad_outputs) + grad_outputs=grad_outputs, + ) return [grad.numpy() for grad in grad_inputs] @staticmethod @@ -2230,13 +2561,15 @@ class OpTest(unittest.TestCase): def np_value_to_fluid_value(input): return input - def _get_gradient(self, - input_to_check, - place, - output_names, - no_grad_set, - user_defined_grad_outputs=None, - parallel=False): + def _get_gradient( + self, + input_to_check, + place, + output_names, + no_grad_set, + user_defined_grad_outputs=None, + parallel=False, + ): prog = Program() scope = core.Scope() block = prog.global_block() @@ -2249,36 +2582,43 @@ class OpTest(unittest.TestCase): if user_defined_grad_outputs is None: if self.dtype == np.uint16: cast_inputs = list(map(block.var, output_names)) - cast_outputs = block.create_var(dtype="float32", - shape=cast_inputs[0].shape) - cast_op = block.append_op(inputs={"X": cast_inputs}, - outputs={"Out": cast_outputs}, - type="cast", - attrs={ - "in_dtype": - core.VarDesc.VarType.BF16, - "out_dtype": - core.VarDesc.VarType.FP32 - }) + cast_outputs = block.create_var( + dtype="float32", shape=cast_inputs[0].shape + ) + cast_op = block.append_op( + inputs={"X": cast_inputs}, + outputs={"Out": cast_outputs}, + type="cast", + attrs={ + "in_dtype": core.VarDesc.VarType.BF16, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) cast_op.desc.infer_var_type(block.desc) cast_op.desc.infer_shape(block.desc) output_names = [cast_outputs.name] loss = append_loss_ops(block, output_names) - param_grad_list = append_backward(loss=loss, - parameter_list=input_to_check, - no_grad_set=no_grad_set) + param_grad_list = append_backward( + loss=loss, + parameter_list=input_to_check, + no_grad_set=no_grad_set, + ) fetch_list = [g for p, g in param_grad_list] else: - assert parallel is False, "unsupported parallel mode when giving custom grad outputs." + assert ( + parallel is False + ), "unsupported parallel mode when giving custom grad outputs." # user_defined_grad_outputs here are numpy arrays if not isinstance(user_defined_grad_outputs, list): user_defined_grad_outputs = [user_defined_grad_outputs] grad_outputs = [] for grad_out_value in user_defined_grad_outputs: # `presistable` is used to avoid executor create new var in local scope - var = block.create_var(shape=grad_out_value.shape, - dtype=grad_out_value.dtype, - persistable=True) + var = block.create_var( + shape=grad_out_value.shape, + dtype=grad_out_value.dtype, + persistable=True, + ) true_var = scope.var(var.name) tensor = true_var.get_tensor() tensor.set(grad_out_value, place) @@ -2287,8 +2627,9 @@ class OpTest(unittest.TestCase): outputs[name] for name in outputs if name in output_names ] inputs = [inputs[name] for name in input_to_check if name in inputs] - grad_inputs = paddle.static.gradients(targets, inputs, grad_outputs, - no_grad_set) + grad_inputs = paddle.static.gradients( + targets, inputs, grad_outputs, no_grad_set + ) fetch_list = grad_inputs if parallel: @@ -2296,21 +2637,21 @@ class OpTest(unittest.TestCase): if isinstance(place, fluid.CUDAPlace): use_cuda = True compiled_prog = fluid.CompiledProgram(prog).with_data_parallel( - loss_name=loss.name, places=place) + loss_name=loss.name, places=place + ) prog = compiled_prog executor = fluid.Executor(place) return list( map( np.array, - executor.run(prog, - feed_dict, - fetch_list, - scope=scope, - return_numpy=False))) + executor.run( + prog, feed_dict, fetch_list, scope=scope, return_numpy=False + ), + ) + ) class OpTestTool: - @classmethod def skip_if(cls, condition: object, reason: str): return unittest.skipIf(condition, reason) @@ -2318,12 +2659,16 @@ class OpTestTool: @classmethod def skip_if_not_cpu_bf16(cls): return OpTestTool.skip_if( - not (isinstance(_current_expected_place(), core.CPUPlace) - and core.supports_bfloat16()), - "Place does not support BF16 evaluation") + not ( + isinstance(_current_expected_place(), core.CPUPlace) + and core.supports_bfloat16() + ), + "Place does not support BF16 evaluation", + ) @classmethod def skip_if_not_cpu(cls): return OpTestTool.skip_if( not isinstance(_current_expected_place(), core.CPUPlace), - "OneDNN supports only CPU for now") + "OneDNN supports only CPU for now", + ) diff --git a/python/paddle/fluid/tests/unittests/op_test_xpu.py b/python/paddle/fluid/tests/unittests/op_test_xpu.py index 82c8eb73969..220bd09f2ca 100644 --- a/python/paddle/fluid/tests/unittests/op_test_xpu.py +++ b/python/paddle/fluid/tests/unittests/op_test_xpu.py @@ -22,11 +22,14 @@ from paddle.fluid.framework import Program, convert_np_dtype_to_dtype_ from testsuite import append_loss_ops, create_op, set_input from white_list import op_threshold_white_list, no_grad_set_white_list from op_test import OpTest -from xpu.get_test_cover_info import is_empty_grad_op_type, get_xpu_op_support_types, type_dict_str_to_numpy +from xpu.get_test_cover_info import ( + is_empty_grad_op_type, + get_xpu_op_support_types, + type_dict_str_to_numpy, +) class XPUOpTest(OpTest): - @classmethod def setUpClass(cls): '''Fix random seeds to remove randomness from tests''' @@ -60,25 +63,36 @@ class XPUOpTest(OpTest): places = [paddle.XPUPlace(0)] return places - def check_output(self, - atol=0.001, - no_check_set=None, - equal_nan=False, - check_dygraph=True, - inplace_atol=None, - check_eager=False): + def check_output( + self, + atol=0.001, + no_check_set=None, + equal_nan=False, + check_dygraph=True, + inplace_atol=None, + check_eager=False, + ): place = paddle.XPUPlace(0) - self.check_output_with_place(place, atol, no_check_set, equal_nan, - check_dygraph, inplace_atol, check_eager) - - def check_output_with_place(self, - place, - atol=0.001, - no_check_set=None, - equal_nan=False, - check_dygraph=True, - inplace_atol=None, - check_eager=False): + self.check_output_with_place( + place, + atol, + no_check_set, + equal_nan, + check_dygraph, + inplace_atol, + check_eager, + ) + + def check_output_with_place( + self, + place, + atol=0.001, + no_check_set=None, + equal_nan=False, + check_dygraph=True, + inplace_atol=None, + check_eager=False, + ): self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) if self.dtype == np.float64: return @@ -89,46 +103,60 @@ class XPUOpTest(OpTest): if self.dtype == np.float16: atol = 0.1 - return super().check_output_with_place(place, atol, no_check_set, - equal_nan, check_dygraph, - inplace_atol) - - def check_grad(self, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grads=None, - user_defined_grad_outputs=None, - check_dygraph=True, - numeric_place=None, - check_eager=False): + return super().check_output_with_place( + place, atol, no_check_set, equal_nan, check_dygraph, inplace_atol + ) + + def check_grad( + self, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grads=None, + user_defined_grad_outputs=None, + check_dygraph=True, + numeric_place=None, + check_eager=False, + ): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, inputs_to_check, output_names, - no_grad_set, numeric_grad_delta, in_place, - max_relative_error, user_defined_grads, - user_defined_grad_outputs, check_dygraph, - numeric_place, check_eager) - - def check_grad_with_place(self, - place, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grads=None, - user_defined_grad_outputs=None, - check_dygraph=True, - numeric_place=None, - check_eager=False): + self.check_grad_with_place( + place, + inputs_to_check, + output_names, + no_grad_set, + numeric_grad_delta, + in_place, + max_relative_error, + user_defined_grads, + user_defined_grad_outputs, + check_dygraph, + numeric_place, + check_eager, + ) + + def check_grad_with_place( + self, + place, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grads=None, + user_defined_grad_outputs=None, + check_dygraph=True, + numeric_place=None, + check_eager=False, + ): if hasattr(self, 'op_type_need_check_grad'): xpu_version = core.get_xpu_device_version(0) - if is_empty_grad_op_type(xpu_version, self.op_type, - self.in_type_str): + if is_empty_grad_op_type( + xpu_version, self.op_type, self.in_type_str + ): self._check_grad_helper() return @@ -137,7 +165,7 @@ class XPUOpTest(OpTest): for ctype in cast_grad_op_types: cast_grad_op_types_np.append(type_dict_str_to_numpy[ctype]) - if (self.dtype not in cast_grad_op_types_np): + if self.dtype not in cast_grad_op_types_np: return if self.dtype == np.float64: @@ -150,51 +178,73 @@ class XPUOpTest(OpTest): if self.dtype == np.float16: max_relative_error = 1.0 return super().check_grad_with_place( - place, inputs_to_check, output_names, no_grad_set, - numeric_grad_delta, in_place, max_relative_error, - user_defined_grads, user_defined_grad_outputs, check_dygraph) + place, + inputs_to_check, + output_names, + no_grad_set, + numeric_grad_delta, + in_place, + max_relative_error, + user_defined_grads, + user_defined_grad_outputs, + check_dygraph, + ) a1 = self.get_grad_with_place( place, inputs_to_check, output_names, no_grad_set=no_grad_set, - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) a2 = self.get_grad_with_place( place, inputs_to_check, output_names, no_grad_set=no_grad_set, - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) a3 = self.get_grad_with_place( paddle.CPUPlace(), inputs_to_check, output_names, no_grad_set=no_grad_set, - user_defined_grad_outputs=user_defined_grad_outputs) - self._assert_is_close(a1, a2, inputs_to_check, 0.00000001, - "Gradient Check On two xpu") - self._assert_is_close(a1, a3, inputs_to_check, max_relative_error, - "Gradient Check On cpu & xpu") - - def get_grad_with_place(self, - place, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grad_outputs=None, - check_dygraph=True): + user_defined_grad_outputs=user_defined_grad_outputs, + ) + self._assert_is_close( + a1, a2, inputs_to_check, 0.00000001, "Gradient Check On two xpu" + ) + self._assert_is_close( + a1, + a3, + inputs_to_check, + max_relative_error, + "Gradient Check On cpu & xpu", + ) + + def get_grad_with_place( + self, + place, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grad_outputs=None, + check_dygraph=True, + ): self.scope = core.Scope() op_inputs = self.inputs if hasattr(self, "inputs") else dict() op_outputs = self.outputs if hasattr(self, "outputs") else dict() op_attrs = self.attrs if hasattr(self, "attrs") else dict() self._check_grad_helper() - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST + ): numeric_grad_delta = 1e-5 max_relative_error = 1e-7 @@ -213,12 +263,14 @@ class XPUOpTest(OpTest): for mtype in mean_grad_op_types: mean_grad_op_types_np.append(type_dict_str_to_numpy[mtype]) - self.op = create_op(self.scope, - self.op_type, - op_inputs, - op_outputs, - op_attrs, - cache_list=cache_list) + self.op = create_op( + self.scope, + self.op_type, + op_inputs, + op_outputs, + op_attrs, + cache_list=cache_list, + ) if use_onednn: op_attrs["use_mkldnn"] = True @@ -226,12 +278,18 @@ class XPUOpTest(OpTest): if no_grad_set is None: no_grad_set = set() else: - if (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST - ) and (self.op_type - not in no_grad_set_white_list.NOT_CHECK_OP_LIST) and ( - not self.is_bfloat16_op()): - raise AssertionError("no_grad_set must be None, op_type is " + - self.op_type + " Op.") + if ( + (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST) + and ( + self.op_type not in no_grad_set_white_list.NOT_CHECK_OP_LIST + ) + and (not self.is_bfloat16_op()) + ): + raise AssertionError( + "no_grad_set must be None, op_type is " + + self.op_type + + " Op." + ) for input_to_check in inputs_to_check: set_input(self.scope, self.op, self.inputs, place) @@ -239,7 +297,7 @@ class XPUOpTest(OpTest): if not type(output_names) is list: output_names = [output_names] - if (self.dtype not in mean_grad_op_types_np): + if self.dtype not in mean_grad_op_types_np: prog = Program() block = prog.global_block() @@ -250,18 +308,18 @@ class XPUOpTest(OpTest): outputs = self._get_outputs(block) feed_dict = self.feed_var(inputs, place) cast_inputs = list(map(block.var, output_names)) - cast_outputs = block.create_var(dtype="float32", - shape=cast_inputs[0].shape) - cast_op = block.append_op(type="cast", - inputs={"X": cast_inputs}, - outputs={"Out": cast_outputs}, - attrs={ - "in_dtype": - convert_np_dtype_to_dtype_( - self.dtype), - "out_dtype": - core.VarDesc.VarType.FP32 - }) + cast_outputs = block.create_var( + dtype="float32", shape=cast_inputs[0].shape + ) + cast_op = block.append_op( + type="cast", + inputs={"X": cast_inputs}, + outputs={"Out": cast_outputs}, + attrs={ + "in_dtype": convert_np_dtype_to_dtype_(self.dtype), + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) cast_op.desc.infer_var_type(block.desc) cast_op.desc.infer_shape(block.desc) @@ -270,41 +328,48 @@ class XPUOpTest(OpTest): loss = append_loss_ops(block, output_names) loss_names = [loss.name] recast_inputs = list(map(block.var, loss_names)) - recast_loss = block.create_var(dtype=self.dtype, - shape=recast_inputs[0].shape) - - recast_op = block.append_op(type="cast", - inputs={"X": recast_inputs}, - outputs={"Out": recast_loss}, - attrs={ - "in_dtype": - core.VarDesc.VarType.FP32, - "out_dtype": - convert_np_dtype_to_dtype_( - self.dtype) - }) + recast_loss = block.create_var( + dtype=self.dtype, shape=recast_inputs[0].shape + ) + + recast_op = block.append_op( + type="cast", + inputs={"X": recast_inputs}, + outputs={"Out": recast_loss}, + attrs={ + "in_dtype": core.VarDesc.VarType.FP32, + "out_dtype": convert_np_dtype_to_dtype_(self.dtype), + }, + ) recast_op.desc.infer_var_type(block.desc) recast_op.desc.infer_shape(block.desc) - param_grad_list = append_backward(loss=recast_loss, - parameter_list=[input_to_check], - no_grad_set=no_grad_set) + param_grad_list = append_backward( + loss=recast_loss, + parameter_list=[input_to_check], + no_grad_set=no_grad_set, + ) fetch_list = [g for p, g in param_grad_list] executor = fluid.Executor(place) return list( map( np.array, - executor.run(prog, - feed_dict, - fetch_list, - scope=scope, - return_numpy=False))) + executor.run( + prog, + feed_dict, + fetch_list, + scope=scope, + return_numpy=False, + ), + ) + ) analytic_grads = self._get_gradient( inputs_to_check, place, output_names, no_grad_set, - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) return analytic_grads diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_dataparallel_with_pylayer.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_dataparallel_with_pylayer.py index b85caf2c966..d49202def5b 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_dataparallel_with_pylayer.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_dataparallel_with_pylayer.py @@ -18,7 +18,9 @@ import paddle import numpy as np import paddle.distributed as dist from paddle.autograd import PyLayer -from paddle.distributed.fleet.utils.hybrid_parallel_util import fused_allreduce_gradients +from paddle.distributed.fleet.utils.hybrid_parallel_util import ( + fused_allreduce_gradients, +) batch = 5 in_dim = 20 @@ -26,7 +28,6 @@ out_dim = 10 class cus_tanh(PyLayer): - @staticmethod def forward(ctx, x): y = paddle.tanh(x) @@ -35,13 +36,12 @@ class cus_tanh(PyLayer): @staticmethod def backward(ctx, dy): - y, = ctx.saved_tensor() + (y,) = ctx.saved_tensor() grad = dy * (1 - paddle.square(y)) return grad class SimpleNet(paddle.nn.Layer): - def __init__(self, train_id, model_id): super(SimpleNet, self).__init__() self.w = self.create_parameter(shape=[in_dim, batch], dtype="float32") @@ -62,7 +62,6 @@ class SimpleNet(paddle.nn.Layer): class TestDistTraning(unittest.TestCase): - def test_multiple_gpus(self): self.trainer_id = dist.get_rank() dist.init_parallel_env() diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py index bb66e117f33..d33bde8e406 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py @@ -29,17 +29,19 @@ out_dim = 20 class SimpleNet(fluid.Layer): - def __init__(self, train_id): super(SimpleNet, self).__init__() - self.w1 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") - self.w2 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") + self.w1 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) + self.w2 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) self.share_net = Linear(out_dim, 10) - self.unused_param = self.create_parameter(shape=[out_dim, in_dim], - dtype="float64") + self.unused_param = self.create_parameter( + shape=[out_dim, in_dim], dtype="float64" + ) # just for test sync_params_buffers self.register_buffer("queue", paddle.randn([10, 5])) @@ -49,9 +51,10 @@ class SimpleNet(fluid.Layer): self.trainer_id = train_id def forward(self, x): - is_use = (paddle.equal_all( - x, paddle.ones(shape=(batch, in_dim))).numpy()[0] - and self.trainer_id == 1) + is_use = ( + paddle.equal_all(x, paddle.ones(shape=(batch, in_dim))).numpy()[0] + and self.trainer_id == 1 + ) if is_use: tmp = paddle.matmul(x, self.w1) @@ -62,7 +65,6 @@ class SimpleNet(fluid.Layer): class TestDistTraning(unittest.TestCase): - def test_multiple_gpus(self): dist.init_parallel_env() self.trainer_id = dist.get_rank() @@ -100,10 +102,12 @@ class TestDistTraning(unittest.TestCase): self.check_gradient(model_b.parameters()) # test acc gradient - w1_grad_sum = self.check_acc(model_a._layers.w1.grad, w1_grad_sum, - model_b._layers.w1.grad) - w2_grad_sum = self.check_acc(model_a._layers.w2.grad, w2_grad_sum, - model_b._layers.w2.grad) + w1_grad_sum = self.check_acc( + model_a._layers.w1.grad, w1_grad_sum, model_b._layers.w1.grad + ) + w2_grad_sum = self.check_acc( + model_a._layers.w2.grad, w2_grad_sum, model_b._layers.w2.grad + ) model_a.clear_gradients() diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check_in_eager_mode.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check_in_eager_mode.py index 506e025225d..595aa09db26 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check_in_eager_mode.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check_in_eager_mode.py @@ -30,17 +30,19 @@ out_dim = 20 class SimpleNet(fluid.Layer): - def __init__(self, train_id): super(SimpleNet, self).__init__() - self.w1 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") - self.w2 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") + self.w1 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) + self.w2 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) self.share_net = Linear(out_dim, 10) - self.unused_param = self.create_parameter(shape=[out_dim, in_dim], - dtype="float64") + self.unused_param = self.create_parameter( + shape=[out_dim, in_dim], dtype="float64" + ) # just for test sync_params_buffers # self.register_buffer("queue", paddle.randn([10, 5])) @@ -50,9 +52,10 @@ class SimpleNet(fluid.Layer): self.trainer_id = train_id def forward(self, x): - is_use = (paddle.equal_all( - x, paddle.ones(shape=(batch, in_dim))).numpy()[0] - and self.trainer_id == 1) + is_use = ( + paddle.equal_all(x, paddle.ones(shape=(batch, in_dim))).numpy()[0] + and self.trainer_id == 1 + ) if is_use: tmp = paddle.matmul(x, self.w1) @@ -63,7 +66,6 @@ class SimpleNet(fluid.Layer): class TestDistTraning(unittest.TestCase): - def test_multiple_gpus(self): self.trainer_id = dist.get_rank() with _test_eager_guard(): @@ -75,12 +77,12 @@ class TestDistTraning(unittest.TestCase): state_dict = model_a.state_dict() model_b.set_state_dict(state_dict) - model_a = paddle.DataParallel(model_a, - find_unused_parameters=True, - group=self.pg) - model_b = paddle.DataParallel(model_b, - find_unused_parameters=True, - group=self.pg) + model_a = paddle.DataParallel( + model_a, find_unused_parameters=True, group=self.pg + ) + model_b = paddle.DataParallel( + model_b, find_unused_parameters=True, group=self.pg + ) ones_input = paddle.ones(shape=(batch, in_dim)) ones_input.stop_gradient = True @@ -106,12 +108,16 @@ class TestDistTraning(unittest.TestCase): self.check_gradient(model_b.parameters()) # test acc gradient - w1_grad_sum = self.check_acc(model_a._layers.w1.grad, - w1_grad_sum, - model_b._layers.w1.grad) - w2_grad_sum = self.check_acc(model_a._layers.w2.grad, - w2_grad_sum, - model_b._layers.w2.grad) + w1_grad_sum = self.check_acc( + model_a._layers.w1.grad, + w1_grad_sum, + model_b._layers.w1.grad, + ) + w2_grad_sum = self.check_acc( + model_a._layers.w2.grad, + w2_grad_sum, + model_b._layers.w2.grad, + ) model_a.clear_gradients() diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_mnist.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_mnist.py index 8b07142ac2e..694e1cf0e62 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_mnist.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_mnist.py @@ -23,43 +23,48 @@ from test_dist_base import runtime_main, TestParallelDyGraphRunnerBase class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=False, - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=False, + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=None, - bias_attr=None, - use_cudnn=use_cudnn) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=None, + bias_attr=None, + use_cudnn=use_cudnn, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -68,33 +73,30 @@ class SimpleImgConvPool(fluid.dygraph.Layer): class MNIST(fluid.dygraph.Layer): - def __init__(self): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(1, - 20, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_1 = SimpleImgConvPool( + 1, 20, 5, 2, 2, act="relu" + ) - self._simple_img_conv_pool_2 = SimpleImgConvPool(20, - 50, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_2 = SimpleImgConvPool( + 20, 50, 5, 2, 2, act="relu" + ) self.pool_2_shape = 50 * 4 * 4 SIZE = 10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 - self._fc = Linear(self.pool_2_shape, - 10, - param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=scale)), - act="softmax") + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 + self._fc = Linear( + self.pool_2_shape, + 10, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + act="softmax", + ) def forward(self, inputs, label): x = self._simple_img_conv_pool_1(inputs) @@ -107,22 +109,26 @@ class MNIST(fluid.dygraph.Layer): class TestMnist(TestParallelDyGraphRunnerBase): - def get_model(self): model = MNIST() - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=2, - drop_last=True) - opt = paddle.optimizer.Adam(learning_rate=1e-3, - parameters=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=2, drop_last=True + ) + opt = paddle.optimizer.Adam( + learning_rate=1e-3, parameters=model.parameters() + ) return model, train_reader, opt def run_one_loop(self, model, opt, data): batch_size = len(data) - dy_x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + dy_x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) label.stop_gradient = True diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py index 5713c93920f..a46b3344333 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py @@ -27,15 +27,18 @@ batch_num = 1000 class SimpleNet(fluid.Layer): - def __init__(self): super(SimpleNet, self).__init__() - self.net_a = paddle.nn.Sequential(paddle.nn.Linear(10, 20), - paddle.nn.Linear(20, 20), - paddle.nn.Linear(20, 5)) - self.net_b = paddle.nn.Sequential(paddle.nn.Linear(10, 20), - paddle.nn.Linear(20, 20), - paddle.nn.Linear(20, 5)) + self.net_a = paddle.nn.Sequential( + paddle.nn.Linear(10, 20), + paddle.nn.Linear(20, 20), + paddle.nn.Linear(20, 5), + ) + self.net_b = paddle.nn.Sequential( + paddle.nn.Linear(10, 20), + paddle.nn.Linear(20, 20), + paddle.nn.Linear(20, 5), + ) self.step = 0 def forward(self, x): @@ -43,24 +46,23 @@ class SimpleNet(fluid.Layer): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ class TestSimpleNet(TestParallelDyGraphRunnerBase): - def get_model(self): model = SimpleNet() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_shared_unused_var.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_shared_unused_var.py index 851bf3b2fdc..60b6601b842 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_shared_unused_var.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_shared_unused_var.py @@ -24,7 +24,6 @@ paddle.seed(1024) class SimpleNet(fluid.Layer): - def __init__(self): # bias is unused parameters, and it share with net_a super(SimpleNet, self).__init__() @@ -41,24 +40,23 @@ batch_num = 1000 def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ class TestSimpleNet(TestParallelDyGraphRunnerBase): - def get_model(self): model = SimpleNet() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding.py index c8e9bacad62..f2e5ff7751c 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding.py @@ -23,14 +23,15 @@ from test_dist_base import runtime_main, TestParallelDyGraphRunnerBase class SimpleNet(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype="float32"): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype="float32", + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -42,28 +43,35 @@ class SimpleNet(fluid.Layer): is_sparse=is_sparse, param_attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ) + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) fc = fluid.layers.matmul(x_emb, self.softmax_weight) fc = fluid.layers.elementwise_add(fc, self.softmax_bias) projection = fluid.layers.reshape(fc, shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -81,7 +89,6 @@ init_scale = 0.1 def fake_sample_reader(): - def __reader__(): for i in range(batch_num): x_data = np.arange(num_steps).astype('int64') @@ -92,20 +99,22 @@ def fake_sample_reader(): class TestSparseEmbedding(TestParallelDyGraphRunnerBase): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=True) - - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + model = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=True, + ) + + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_fp64.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_fp64.py index 599c0754e7f..d2a07868e88 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_fp64.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_fp64.py @@ -22,14 +22,15 @@ paddle.set_default_dtype("float64") class SimpleNet(Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype="float64"): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype="float64", + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -40,26 +41,35 @@ class SimpleNet(Layer): self.hidden_size, sparse=True, weight_attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Uniform(low=-init_scale, - high=init_scale))) + initializer=paddle.nn.initializer.Uniform( + low=-init_scale, high=init_scale + ) + ), + ) self.softmax_weight = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.tmp = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) @@ -67,7 +77,8 @@ class SimpleNet(Layer): fc = paddle.add(fc, self.softmax_bias) projection = paddle.reshape(fc, shape=[-1, self.vocab_size]) loss = paddle.nn.functional.softmax_with_cross_entropy( - logits=projection, label=label, soft_label=False) + logits=projection, label=label, soft_label=False + ) loss = paddle.reshape(loss, shape=[-1, self.num_steps]) loss = paddle.mean(loss, axis=[0]) loss = paddle.sum(loss) @@ -85,7 +96,6 @@ init_scale = 0.1 def fake_sample_reader(): - def __reader__(): for i in range(batch_num): x_data = np.arange(num_steps).astype('int64') @@ -96,20 +106,22 @@ def fake_sample_reader(): class TestSparseEmbeddingFP64(TestParallelDyGraphRunnerBase): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=True) - - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + model = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=True, + ) + + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_over_height.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_over_height.py index 7bd30deea64..9d70390b135 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_over_height.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_over_height.py @@ -14,7 +14,11 @@ import paddle import paddle.fluid as fluid -from parallel_dygraph_sparse_embedding import SimpleNet, fake_sample_reader, TestSparseEmbedding +from parallel_dygraph_sparse_embedding import ( + SimpleNet, + fake_sample_reader, + TestSparseEmbedding, +) from test_dist_base import runtime_main @@ -29,20 +33,22 @@ init_scale = 0.1 class TestSparseEmbeddingOverHeight(TestSparseEmbedding): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=True) - - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - - optimizer = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=model.parameters()) + model = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=True, + ) + + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + + optimizer = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_unused_variables.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_unused_variables.py index bcd8d765a27..1c8c30b856d 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_unused_variables.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_unused_variables.py @@ -20,14 +20,15 @@ from paddle.nn import Layer, Embedding class SimpleNet(Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype="float32"): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype="float32", + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -38,27 +39,36 @@ class SimpleNet(Layer): self.hidden_size, sparse=is_sparse, weight_attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Uniform(low=-init_scale, - high=init_scale))) + initializer=paddle.nn.initializer.Uniform( + low=-init_scale, high=init_scale + ) + ), + ) self.softmax_weight = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) # add tmp var self.tmp = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) @@ -69,7 +79,8 @@ class SimpleNet(Layer): fc = paddle.add(fc, self.softmax_bias) projection = paddle.reshape(fc, shape=[-1, self.vocab_size]) loss = paddle.nn.functional.softmax_with_cross_entropy( - logits=projection, label=label, soft_label=False) + logits=projection, label=label, soft_label=False + ) loss = paddle.reshape(loss, shape=[-1, self.num_steps]) loss = paddle.mean(loss, axis=[0]) loss = paddle.sum(loss) @@ -87,7 +98,6 @@ init_scale = 0.1 def fake_sample_reader(): - def __reader__(): for i in range(batch_num): x_data = np.arange(num_steps).astype('int64') @@ -98,20 +108,22 @@ def fake_sample_reader(): class TestSparseEmbeddingUnusedVars(TestParallelDyGraphRunnerBase): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=False) - - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + model = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=False, + ) + + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py index 2d467ef50ef..88caad052ad 100644 --- a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py +++ b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py @@ -30,35 +30,37 @@ DeviceType = core.DeviceType class TestParallelExecutorBase(unittest.TestCase): - @classmethod - def check_network_convergence(cls, - method, - use_device=DeviceType.CUDA, - iter=5, - batch_size=None, - feed_dict=None, - feed_data_reader=None, - get_data_from_feeder=None, - use_parallel_executor=True, - use_reduce=False, - use_ir_memory_optimize=False, - enable_inplace=True, - fuse_elewise_add_act_ops=False, - fuse_all_optimizer_ops=False, - fuse_all_reduce_ops=False, - fuse_relu_depthwise_conv=False, - optimizer=fluid.optimizer.Adam, - use_fast_executor=False, - enable_sequential_execution=False): - + def check_network_convergence( + cls, + method, + use_device=DeviceType.CUDA, + iter=5, + batch_size=None, + feed_dict=None, + feed_data_reader=None, + get_data_from_feeder=None, + use_parallel_executor=True, + use_reduce=False, + use_ir_memory_optimize=False, + enable_inplace=True, + fuse_elewise_add_act_ops=False, + fuse_all_optimizer_ops=False, + fuse_all_reduce_ops=False, + fuse_relu_depthwise_conv=False, + optimizer=fluid.optimizer.Adam, + use_fast_executor=False, + enable_sequential_execution=False, + ): def run_executor(exe, binary, feed, fetch_list): if feed_data_reader is None: res = exe.run(binary, feed=feed, fetch_list=fetch_list) else: - res = exe.run(binary, - feed=feed_data_reader.get_next(exe, binary), - fetch_list=fetch_list) + res = exe.run( + binary, + feed=feed_data_reader.get_next(exe, binary), + fetch_list=fetch_list, + ) return res if feed_data_reader is not None: @@ -72,63 +74,79 @@ class TestParallelExecutorBase(unittest.TestCase): startup = fluid.Program() with fluid.program_guard(main, startup): - feed_dict, loss = cls.build_model(feed_dict, get_data_from_feeder, - main, method, optimizer) + feed_dict, loss = cls.build_model( + feed_dict, get_data_from_feeder, main, method, optimizer + ) - place = fluid.CUDAPlace( - 0) if use_device == DeviceType.CUDA else fluid.XPUPlace( - 0) if use_device == DeviceType.XPU else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if use_device == DeviceType.CUDA + else fluid.XPUPlace(0) + if use_device == DeviceType.XPU + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup) build_strategy, exec_strategy = cls.set_strategy( - enable_inplace, enable_sequential_execution, fuse_all_optimizer_ops, - fuse_all_reduce_ops, fuse_elewise_add_act_ops, - fuse_relu_depthwise_conv, use_fast_executor, use_ir_memory_optimize, - use_reduce, use_device) + enable_inplace, + enable_sequential_execution, + fuse_all_optimizer_ops, + fuse_all_reduce_ops, + fuse_elewise_add_act_ops, + fuse_relu_depthwise_conv, + use_fast_executor, + use_ir_memory_optimize, + use_reduce, + use_device, + ) if use_parallel_executor: binary = compiler.CompiledProgram(main).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - exec_strategy=exec_strategy) + exec_strategy=exec_strategy, + ) else: binary = main if batch_size is not None: - batch_size *= fluid.core.get_cuda_device_count( - ) if use_device == DeviceType.CUDA else fluid.core.get_xpu_device_count( - ) if use_device == DeviceType.XPU else int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + batch_size *= ( + fluid.core.get_cuda_device_count() + if use_device == DeviceType.CUDA + else fluid.core.get_xpu_device_count() + if use_device == DeviceType.XPU + else int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + ) area_below_loss = 0 begin = time.time() - first_loss, = run_executor(exe=exe, - binary=binary, - feed=feed_dict, - fetch_list=[loss.name]) + (first_loss,) = run_executor( + exe=exe, binary=binary, feed=feed_dict, fetch_list=[loss.name] + ) area_below_loss += 0.5 * first_loss.mean() for _ in range(iter): - mid_loss = run_executor(exe=exe, - binary=binary, - feed=feed_dict, - fetch_list=[loss.name]) + mid_loss = run_executor( + exe=exe, binary=binary, feed=feed_dict, fetch_list=[loss.name] + ) area_below_loss += mid_loss[0].mean() - last_loss, = run_executor(exe=exe, - binary=binary, - feed=feed_dict, - fetch_list=[loss.name]) + (last_loss,) = run_executor( + exe=exe, binary=binary, feed=feed_dict, fetch_list=[loss.name] + ) area_below_loss += 0.5 * last_loss.mean() end = time.time() if batch_size is not None: - print("%.4f Instance per second" % ((batch_size * iter + 2) / - (end - begin))) + print( + "%.4f Instance per second" + % ((batch_size * iter + 2) / (end - begin)) + ) avg_last_loss_val = np.array(last_loss).mean() avg_first_loss_val = np.array(first_loss).mean() if math.isnan(float(avg_last_loss_val)) or math.isnan( - float(avg_first_loss_val)): + float(avg_first_loss_val) + ): sys.exit("got NaN loss, training failed.") print(first_loss, last_loss, area_below_loss) @@ -136,59 +154,85 @@ class TestParallelExecutorBase(unittest.TestCase): return first_loss, last_loss, area_below_loss @classmethod - def check_pass_conflict(cls, - method, - use_device=DeviceType.CUDA, - feed_dict=None, - get_data_from_feeder=None, - use_reduce=False, - use_ir_memory_optimize=True, - enable_inplace=True, - fuse_elewise_add_act_ops=False, - fuse_all_optimizer_ops=False, - fuse_all_reduce_ops=False, - fuse_relu_depthwise_conv=False, - optimizer=fluid.optimizer.Adam, - use_fast_executor=True, - enable_sequential_execution=False): + def check_pass_conflict( + cls, + method, + use_device=DeviceType.CUDA, + feed_dict=None, + get_data_from_feeder=None, + use_reduce=False, + use_ir_memory_optimize=True, + enable_inplace=True, + fuse_elewise_add_act_ops=False, + fuse_all_optimizer_ops=False, + fuse_all_reduce_ops=False, + fuse_relu_depthwise_conv=False, + optimizer=fluid.optimizer.Adam, + use_fast_executor=True, + enable_sequential_execution=False, + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): - feed_dict, loss = cls.build_model(feed_dict, get_data_from_feeder, - main, method, optimizer) + feed_dict, loss = cls.build_model( + feed_dict, get_data_from_feeder, main, method, optimizer + ) - place = fluid.CUDAPlace( - 0) if use_device == DeviceType.CUDA else fluid.XPUPlace( - 0) if use_device == DeviceType.XPU else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if use_device == DeviceType.CUDA + else fluid.XPUPlace(0) + if use_device == DeviceType.XPU + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup) build_strategy, exec_strategy = cls.set_strategy( - enable_inplace, enable_sequential_execution, fuse_all_optimizer_ops, - fuse_all_reduce_ops, fuse_elewise_add_act_ops, - fuse_relu_depthwise_conv, use_fast_executor, use_ir_memory_optimize, - use_reduce, use_device) + enable_inplace, + enable_sequential_execution, + fuse_all_optimizer_ops, + fuse_all_reduce_ops, + fuse_elewise_add_act_ops, + fuse_relu_depthwise_conv, + use_fast_executor, + use_ir_memory_optimize, + use_reduce, + use_device, + ) binary = compiler.CompiledProgram(main).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - exec_strategy=exec_strategy) + exec_strategy=exec_strategy, + ) exe.run(binary, feed=feed_dict, fetch_list=[loss.name]) @classmethod - def set_strategy(cls, enable_inplace, enable_sequential_execution, - fuse_all_optimizer_ops, fuse_all_reduce_ops, - fuse_elewise_add_act_ops, fuse_relu_depthwise_conv, - use_fast_executor, use_ir_memory_optimize, use_reduce, - use_device): + def set_strategy( + cls, + enable_inplace, + enable_sequential_execution, + fuse_all_optimizer_ops, + fuse_all_reduce_ops, + fuse_elewise_add_act_ops, + fuse_relu_depthwise_conv, + use_fast_executor, + use_ir_memory_optimize, + use_reduce, + use_device, + ): exec_strategy = fluid.ExecutionStrategy() if use_fast_executor: exec_strategy.use_experimental_executor = True build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce \ - if use_reduce else fluid.BuildStrategy.ReduceStrategy.AllReduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + if use_reduce + else fluid.BuildStrategy.ReduceStrategy.AllReduce + ) build_strategy.fuse_elewise_add_act_ops = fuse_elewise_add_act_ops build_strategy.fuse_relu_depthwise_conv = fuse_relu_depthwise_conv build_strategy.fuse_all_optimizer_ops = fuse_all_optimizer_ops @@ -210,8 +254,9 @@ class TestParallelExecutorBase(unittest.TestCase): return build_strategy, exec_strategy @classmethod - def build_model(cls, feed_dict, get_data_from_feeder, main, method, - optimizer): + def build_model( + cls, feed_dict, get_data_from_feeder, main, method, optimizer + ): loss = method(use_feed=feed_dict is not None) # NOTE(zjl): memory_optimize/inplace pass would not require # that loss.persistable = True. diff --git a/python/paddle/fluid/tests/unittests/ps/dataset_generator_A.py b/python/paddle/fluid/tests/unittests/ps/dataset_generator_A.py index 1ab4b3580d6..a53da275939 100755 --- a/python/paddle/fluid/tests/unittests/ps/dataset_generator_A.py +++ b/python/paddle/fluid/tests/unittests/ps/dataset_generator_A.py @@ -23,7 +23,6 @@ categorical_range_ = range(14, 40) class CriteoDataset(dg.MultiSlotDataGenerator): - def generate_sample(self, line): """ Read the data line by line and process it as a dictionary @@ -38,7 +37,8 @@ class CriteoDataset(dg.MultiSlotDataGenerator): sparse_feature = [] for idx in categorical_range_: sparse_feature.append( - [hash(str(idx) + features[idx]) % hash_dim_]) + [hash(str(idx) + features[idx]) % hash_dim_] + ) for idx in categorical_range_: feature_name.append("C" + str(idx - 13)) yield list(zip(feature_name, sparse_feature)) diff --git a/python/paddle/fluid/tests/unittests/ps/dataset_generator_B.py b/python/paddle/fluid/tests/unittests/ps/dataset_generator_B.py index 76b2468592d..77178ee4c33 100755 --- a/python/paddle/fluid/tests/unittests/ps/dataset_generator_B.py +++ b/python/paddle/fluid/tests/unittests/ps/dataset_generator_B.py @@ -23,7 +23,6 @@ categorical_range_ = range(14, 40) class CriteoDataset(dg.MultiSlotDataGenerator): - def generate_sample(self, line): """ Read the data line by line and process it as a dictionary @@ -40,8 +39,9 @@ class CriteoDataset(dg.MultiSlotDataGenerator): dense_feature.append(0.0) else: dense_feature.append( - (float(features[idx]) - cont_min_[idx - 1]) / - cont_diff_[idx - 1]) + (float(features[idx]) - cont_min_[idx - 1]) + / cont_diff_[idx - 1] + ) label = [int(features[0])] feature_name = ["dense_feature"] feature_name.append("label") diff --git a/python/paddle/fluid/tests/unittests/ps/fl_ps_trainer.py b/python/paddle/fluid/tests/unittests/ps/fl_ps_trainer.py index aebf38a4585..66e6b1cb62e 100755 --- a/python/paddle/fluid/tests/unittests/ps/fl_ps_trainer.py +++ b/python/paddle/fluid/tests/unittests/ps/fl_ps_trainer.py @@ -44,16 +44,22 @@ def get_dataset(inputs, config, pipe_cmd, role="worker"): def fl_ps_train(): # 0. get role import paddle.distributed.fleet.base.role_maker as role_maker + role_maker = role_maker.PaddleCloudRoleMaker() role_maker._generate_role() fleet.util._set_role_maker(role_maker) # 1. load yaml-config to dict-config - from ps_dnn_trainer import YamlHelper, StaticModel, get_user_defined_strategy + from ps_dnn_trainer import ( + YamlHelper, + StaticModel, + get_user_defined_strategy, + ) + yaml_helper = YamlHelper() config_yaml_path = '../ps/fl_async_ps_config.yaml' config = yaml_helper.load_yaml(config_yaml_path) - #yaml_helper.print_yaml(config) + # yaml_helper.print_yaml(config) # 2. get static model paddle.enable_static() @@ -67,18 +73,25 @@ def fl_ps_train(): a_sync_configs = user_defined_strategy.a_sync_configs a_sync_configs["launch_barrier"] = True user_defined_strategy.a_sync_configs = a_sync_configs - print("launch_barrier: ", - user_defined_strategy.a_sync_configs["launch_barrier"]) + print( + "launch_barrier: ", + user_defined_strategy.a_sync_configs["launch_barrier"], + ) learning_rate = config.get("hyper_parameters.optimizer.learning_rate") inner_optimizer = paddle.optimizer.Adam(learning_rate, lazy_mode=True) - from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ParameterServerOptimizer + from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ( + ParameterServerOptimizer, + ) + ps_optimizer = ParameterServerOptimizer(inner_optimizer) - ps_optimizer._set_basic_info(loss, role_maker, inner_optimizer, - user_defined_strategy) + ps_optimizer._set_basic_info( + loss, role_maker, inner_optimizer, user_defined_strategy + ) ps_optimizer.minimize_impl(loss) # 4. runtime from paddle.distributed.ps.the_one_ps import TheOnePSRuntime + _runtime_handle = TheOnePSRuntime() # ps 目录下重构版的 TheOnePSRuntime _runtime_handle._set_basic_info(ps_optimizer.pass_ctx._attrs) epoch_num = int(config.get('runner.epoch_num')) @@ -94,21 +107,29 @@ def fl_ps_train(): _runtime_handle._init_worker() print('trainer get dataset') inputs = feeds_list[1:-1] - dataset, file_list = get_dataset(inputs, config, - "python dataset_generator_A.py") - print("fluid.default_main_program: {}".format( - fluid.default_main_program()._heter_pipeline_opt)) + dataset, file_list = get_dataset( + inputs, config, "python dataset_generator_A.py" + ) + print( + "fluid.default_main_program: {}".format( + fluid.default_main_program()._heter_pipeline_opt + ) + ) for epoch in range(epoch_num): # A 方和 B 方如果要以文件粒度 shuffle 时，则需要固定同一个种子 dataset.set_filelist(file_list) start_time = time.time() - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - print_period=2, - debug=False) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + print_period=2, + debug=False, + ) end_time = time.time() - print("trainer epoch %d finished, use time=%d\n" % - ((epoch), end_time - start_time)) + print( + "trainer epoch %d finished, use time=%d\n" + % ((epoch), end_time - start_time) + ) exe.close() _runtime_handle._stop_worker() print("Fl partyA Trainer Success!") @@ -116,19 +137,26 @@ def fl_ps_train(): exe = fluid.Executor() exe.run(fluid.default_startup_program()) _runtime_handle._init_worker() - inputs = [feeds_list[0], - feeds_list[-1]] # 顺序务必要和 dataset_generator_B.py 中保持一致 - dataset, file_list = get_dataset(inputs, config, - "python dataset_generator_B.py", - "heter_worker") - print("fluid.default_main_program: {}".format( - fluid.default_main_program()._heter_pipeline_opt)) + inputs = [ + feeds_list[0], + feeds_list[-1], + ] # 顺序务必要和 dataset_generator_B.py 中保持一致 + dataset, file_list = get_dataset( + inputs, config, "python dataset_generator_B.py", "heter_worker" + ) + print( + "fluid.default_main_program: {}".format( + fluid.default_main_program()._heter_pipeline_opt + ) + ) for epoch in range(epoch_num): dataset.set_filelist(file_list) - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - print_period=2, - debug=False) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + print_period=2, + debug=False, + ) exe.close() _runtime_handle._stop_worker() print("Fl partB Trainer Success!") diff --git a/python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py b/python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py index 8d63c9aa0ac..08055ae1703 100755 --- a/python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py +++ b/python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py @@ -13,7 +13,12 @@ # limitations under the License. import paddle.distributed.fleet.base.role_maker as role_maker -from paddle.distributed.ps.utils.ps_program_builder import debug_program, logger, new_pass, ps_log_root_dir +from paddle.distributed.ps.utils.ps_program_builder import ( + debug_program, + logger, + new_pass, + ps_log_root_dir, +) import paddle.distributed.fleet as fleet import argparse import sys @@ -42,7 +47,6 @@ def is_distributed_env(): class YamlHelper(object): - def load_yaml(self, yaml_file, other_part=None): part_list = ["runner", "hyper_parameters"] if other_part: @@ -114,7 +118,8 @@ class YamlHelper(object): max_k = max(max_k, len(k)) h_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format( - max_k, " " * spacing, max_v) + max_k, " " * spacing, max_v + ) l_format = " " + "|{{:>{}s}}{{}}{{:^{}s}}|\n".format(max_k, max_v) length = max_k + max_v + spacing @@ -150,7 +155,7 @@ def get_user_defined_strategy(config): logger.warn( "Not Find Distributed env, Change To local train mode. If you want train with fleet, please use [fleetrun] command." ) - #return None + # return None sync_mode = config.get("runner.sync_mode") assert sync_mode in ["async", "sync", "geo", "heter", "gpubox"] if sync_mode == "sync": @@ -159,8 +164,9 @@ def get_user_defined_strategy(config): elif sync_mode == "async": strategy = paddle.distributed.fleet.DistributedStrategy() strategy.a_sync = True - strategy.is_fl_ps_mode = True if config.get( - "runner.is_fl_ps_mode") == 1 else False + strategy.is_fl_ps_mode = ( + True if config.get("runner.is_fl_ps_mode") == 1 else False + ) if strategy.is_fl_ps_mode == True: strategy.pipeline = False micro_num = 1 @@ -191,7 +197,7 @@ def get_user_defined_strategy(config): "dump_param": config.get("runner.dump_param", []), "stat_var_names": config.get("stat_var_names", []), "local_sparse": config.get("runner.local_sparse", []), - "remote_sparse": config.get("runner.remote_sparse", []) + "remote_sparse": config.get("runner.remote_sparse", []), } print("strategy:", strategy.trainer_desc_configs) @@ -200,7 +206,7 @@ def get_user_defined_strategy(config): "uri": config.get("runner.fs_client.uri", ""), "user": config.get("runner.fs_client.user", ""), "passwd": config.get("runner.fs_client.passwd", ""), - "hadoop_bin": config.get("runner.fs_client.hadoop_bin", "hadoop") + "hadoop_bin": config.get("runner.fs_client.hadoop_bin", "hadoop"), } print("strategy:", strategy.fs_client_param) @@ -225,7 +231,9 @@ def get_user_defined_strategy(config): def get_distributed_strategy(user_defined_strategy): # pslib - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, + ) k_steps = user_defined_strategy.a_sync_configs["k_steps"] strategy = None @@ -254,45 +262,38 @@ def get_model(config): def parse_args(): parser = argparse.ArgumentParser("PsTest train script") - parser.add_argument('-m', - '--config_yaml', - type=str, - required=True, - help='config file path') - parser.add_argument('-bf16', - '--pure_bf16', - type=ast.literal_eval, - default=False, - help="whether use bf16") - - parser.add_argument('--run_minimize', - type=int, - default=0, - help="test single pass") - parser.add_argument('--run_single_pass', - type=int, - default=0, - help="test single pass") - parser.add_argument('--run_the_one_ps', - type=int, - default=0, - help="test the_one_ps") - parser.add_argument('--debug_new_minimize', - type=int, - default=0, - help="test single pass") - parser.add_argument('--debug_new_pass', - type=int, - default=0, - help="test single pass") - parser.add_argument('--applied_pass_name', - type=str, - default="", - help="test single pass") - parser.add_argument('--debug_the_one_ps', - type=int, - default=0, - help="test the_one_ps") + parser.add_argument( + '-m', '--config_yaml', type=str, required=True, help='config file path' + ) + parser.add_argument( + '-bf16', + '--pure_bf16', + type=ast.literal_eval, + default=False, + help="whether use bf16", + ) + + parser.add_argument( + '--run_minimize', type=int, default=0, help="test single pass" + ) + parser.add_argument( + '--run_single_pass', type=int, default=0, help="test single pass" + ) + parser.add_argument( + '--run_the_one_ps', type=int, default=0, help="test the_one_ps" + ) + parser.add_argument( + '--debug_new_minimize', type=int, default=0, help="test single pass" + ) + parser.add_argument( + '--debug_new_pass', type=int, default=0, help="test single pass" + ) + parser.add_argument( + '--applied_pass_name', type=str, default="", help="test single pass" + ) + parser.add_argument( + '--debug_the_one_ps', type=int, default=0, help="test the_one_ps" + ) args = parser.parse_args() args.abs_dir = os.path.dirname(os.path.abspath(args.config_yaml)) @@ -317,7 +318,6 @@ def bf16_to_fp32(val): class DnnTrainer(object): - def __init__(self, config): self.metrics = {} self.config = config @@ -351,36 +351,59 @@ class DnnTrainer(object): loss = self.model._cost user_defined_strategy = get_user_defined_strategy(self.config) learning_rate = self.config.get( - "hyper_parameters.optimizer.learning_rate") + "hyper_parameters.optimizer.learning_rate" + ) sync_mode = self.config.get("runner.sync_mode") inner_optimizer = paddle.optimizer.Adam(learning_rate, lazy_mode=True) self.role_maker._generate_role() # 必要 if self.config['debug_new_minimize'] == 1: print("entering run_minimize -- new") - from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ParameterServerOptimizer + from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ( + ParameterServerOptimizer, + ) + ps_optimizer = ParameterServerOptimizer(inner_optimizer) - ps_optimizer._set_basic_info(loss, self.role_maker, inner_optimizer, - user_defined_strategy) + ps_optimizer._set_basic_info( + loss, self.role_maker, inner_optimizer, user_defined_strategy + ) ps_optimizer.minimize_impl(loss) else: print("entering run_minimize -- old") fleet_obj = fleet.distributed_optimizer( - inner_optimizer, user_defined_strategy) ## Fleet 对象 + inner_optimizer, user_defined_strategy + ) ## Fleet 对象 fleet_obj.minimize(loss) if fleet.is_server(): - _main_file = ps_log_root_dir + sync_mode + '_run_minimize' + '_debug:_' + str( - self.config['debug_new_minimize']) + '_server_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_minimize' + + '_debug:_' + + str(self.config['debug_new_minimize']) + + '_server_main.prototxt' + ) debug_program(_main_file, loss.block.program) elif fleet.is_worker(): - _main_file = ps_log_root_dir + sync_mode + '_run_minimize' + '_debug:_' + str( - self.config['debug_new_minimize']) + '_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_minimize' + + '_debug:_' + + str(self.config['debug_new_minimize']) + + '_worker_main.prototxt' + ) debug_program(_main_file, loss.block.program) elif self.role_maker._is_heter_worker(): - _main_file = ps_log_root_dir + sync_mode + '_run_minimize' + '_debug:_' + str( - self.config['debug_new_minimize'] - ) + '_heter_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_minimize' + + '_debug:_' + + str(self.config['debug_new_minimize']) + + '_heter_worker_main.prototxt' + ) debug_program(_main_file, loss.block.program) def run_single_pass(self): @@ -396,42 +419,70 @@ class DnnTrainer(object): startup_program = paddle.static.default_startup_program() inner_optimizer.minimize(loss, startup_program) if self.config['debug_new_pass'] == 1: - print("entering run {} - new".format( - str(config["applied_pass_name"]))) - from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ParameterServerOptimizer + print( + "entering run {} - new".format(str(config["applied_pass_name"])) + ) + from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ( + ParameterServerOptimizer, + ) + ps_optimizer = ParameterServerOptimizer(inner_optimizer) - ps_optimizer._set_basic_info(loss, self.role_maker, inner_optimizer, - user_defined_strategy) + ps_optimizer._set_basic_info( + loss, self.role_maker, inner_optimizer, user_defined_strategy + ) ps_optimizer._set_origin_programs([loss]) ps_optimizer._init_ps_pass_context(loss, startup_program) _main = ps_optimizer.pass_ctx._attrs['cloned_main'] - append_send_ops_pass = new_pass(config["applied_pass_name"], - ps_optimizer.pass_ctx._attrs) + append_send_ops_pass = new_pass( + config["applied_pass_name"], ps_optimizer.pass_ctx._attrs + ) append_send_ops_pass.apply([_main], [None], ps_optimizer.pass_ctx) else: - print("entering run {} - old".format( - str(config["applied_pass_name"]))) - from paddle.fluid.incubate.fleet.parameter_server.ir import public as public + print( + "entering run {} - old".format(str(config["applied_pass_name"])) + ) + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + public as public, + ) + dist_strategy = get_distributed_strategy(user_defined_strategy) compiled_config = public.CompileTimeStrategy( - loss.block.program, startup_program, dist_strategy, - self.role_maker) + loss.block.program, + startup_program, + dist_strategy, + self.role_maker, + ) _main = compiled_config.origin_main_program.clone() _startup = compiled_config.origin_startup_program.clone() - from paddle.fluid.incubate.fleet.parameter_server.ir import trainer_pass as worker + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + trainer_pass as worker, + ) + _main = worker.append_send_ops_pass(_main, compiled_config) if fleet.is_server(): - _main_file = ps_log_root_dir + sync_mode + "_" + str( - config["applied_pass_name"]) + '_debug:_' + str( - self.config['debug_new_pass']) + '_server_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + "_" + + str(config["applied_pass_name"]) + + '_debug:_' + + str(self.config['debug_new_pass']) + + '_server_main.prototxt' + ) debug_program(_main_file, _main) elif fleet.is_worker(): - _main_file = ps_log_root_dir + sync_mode + "_" + str( - config["applied_pass_name"]) + '_debug:_' + str( - self.config['debug_new_pass']) + '_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + "_" + + str(config["applied_pass_name"]) + + '_debug:_' + + str(self.config['debug_new_pass']) + + '_worker_main.prototxt' + ) debug_program(_main_file, _main) def run_the_one_ps(self): @@ -442,7 +493,8 @@ class DnnTrainer(object): loss = self.model._cost user_defined_strategy = get_user_defined_strategy(self.config) learning_rate = self.config.get( - "hyper_parameters.optimizer.learning_rate") + "hyper_parameters.optimizer.learning_rate" + ) sync_mode = self.config.get("runner.sync_mode") inner_optimizer = paddle.optimizer.Adam(learning_rate, lazy_mode=True) @@ -450,28 +502,37 @@ class DnnTrainer(object): if self.config['debug_the_one_ps'] == 1: print("entering run_the_one_ps -- new") - from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ParameterServerOptimizer + from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ( + ParameterServerOptimizer, + ) + ps_optimizer = ParameterServerOptimizer(inner_optimizer) - ps_optimizer._set_basic_info(loss, self.role_maker, inner_optimizer, - user_defined_strategy) + ps_optimizer._set_basic_info( + loss, self.role_maker, inner_optimizer, user_defined_strategy + ) ps_optimizer.minimize_impl(loss) from paddle.distributed.ps.the_one_ps import TheOnePSRuntime + _runtime_handle = TheOnePSRuntime() # ps 目录下重构版的 TheOnePSRuntime _runtime_handle._set_basic_info(ps_optimizer.pass_ctx._attrs) if fleet.is_worker(): - worker_desc = _runtime_handle.ps_desc_builder.build_worker_desc( + worker_desc = ( + _runtime_handle.ps_desc_builder.build_worker_desc() ) with open( - ps_log_root_dir + sync_mode + '_' + - 'new_worker_ps_desc', 'w') as f: + ps_log_root_dir + sync_mode + '_' + 'new_worker_ps_desc', + 'w', + ) as f: f.write(worker_desc) if fleet.is_server(): - server_desc = _runtime_handle.ps_desc_builder.build_server_desc( + server_desc = ( + _runtime_handle.ps_desc_builder.build_server_desc() ) with open( - ps_log_root_dir + sync_mode + '_' + - 'new_server_ps_desc', 'w') as f: + ps_log_root_dir + sync_mode + '_' + 'new_server_ps_desc', + 'w', + ) as f: f.write(server_desc) else: @@ -492,16 +553,34 @@ class DnnTrainer(object): f.write(str(server_desc) + str(fleet_obj._runtime_handle._get_fs_client_desc().to_string())) ''' if fleet.is_server(): - _main_file = ps_log_root_dir + sync_mode + '_run_the_one_ps' + '_debug:_' + str( - self.config['debug_the_one_ps']) + '_server_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_the_one_ps' + + '_debug:_' + + str(self.config['debug_the_one_ps']) + + '_server_main.prototxt' + ) debug_program(_main_file, loss.block.program) elif fleet.is_worker(): - _main_file = ps_log_root_dir + sync_mode + '_run_the_one_ps' + '_debug:_' + str( - self.config['debug_the_one_ps']) + '_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_the_one_ps' + + '_debug:_' + + str(self.config['debug_the_one_ps']) + + '_worker_main.prototxt' + ) debug_program(_main_file, loss.block.program) elif self.role_maker._is_heter_worker(): - _main_file = ps_log_root_dir + sync_mode + '_run_the_one_ps' + '_debug:_' + str( - self.config['debug_the_one_ps']) + '_heter_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_the_one_ps' + + '_debug:_' + + str(self.config['debug_the_one_ps']) + + '_heter_worker_main.prototxt' + ) debug_program(_main_file, loss.block.program) diff --git a/python/paddle/fluid/tests/unittests/ps/test_fl_ps.py b/python/paddle/fluid/tests/unittests/ps/test_fl_ps.py index 2726d7c10cd..dc315b7df29 100755 --- a/python/paddle/fluid/tests/unittests/ps/test_fl_ps.py +++ b/python/paddle/fluid/tests/unittests/ps/test_fl_ps.py @@ -16,12 +16,13 @@ import unittest import shlex # noqa: F401 -from paddle.fluid.tests.unittests.distributed_passes.dist_pass_test_base import remove_path_if_exists # noqa: F401 +from paddle.fluid.tests.unittests.distributed_passes.dist_pass_test_base import ( + remove_path_if_exists, +) # noqa: F401 import os class FlPsTest(unittest.TestCase): - def test_launch_fl_ps(self): ''' cmd = [ diff --git a/python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py b/python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py index b561d82fc5f..c9d0c9fc943 100755 --- a/python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py +++ b/python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py @@ -14,14 +14,16 @@ import unittest -from paddle.fluid.tests.unittests.distributed_passes.ps_pass_test_base import PsPassTestBase, remove_path_if_exists +from paddle.fluid.tests.unittests.distributed_passes.ps_pass_test_base import ( + PsPassTestBase, + remove_path_if_exists, +) from paddle.distributed.ps.utils.public import logger, ps_log_root_dir import paddle.distributed.fleet.proto.the_one_ps_pb2 as ps_pb2 # noqa: F401 from google.protobuf import text_format # noqa: F401 class TestTheOnePs(PsPassTestBase): - def setUp(self): pass @@ -56,14 +58,16 @@ class TestTheOnePs(PsPassTestBase): self.config['run_the_one_ps'] = '1' self.config['debug_the_one_ps'] = '0' - self.config[ - 'log_dir'] = ps_log_root_dir + "async_cpu_log_old_the_one_ps" + self.config['log_dir'] = ( + ps_log_root_dir + "async_cpu_log_old_the_one_ps" + ) remove_path_if_exists(self.config['log_dir']) self.ps_launch() self.config['debug_the_one_ps'] = '1' - self.config[ - 'log_dir'] = ps_log_root_dir + "async_cpu_log_new_the_one_ps" + self.config['log_dir'] = ( + ps_log_root_dir + "async_cpu_log_new_the_one_ps" + ) remove_path_if_exists(self.config['log_dir']) self.ps_launch() diff --git a/python/paddle/fluid/tests/unittests/ps_dnn_model.py b/python/paddle/fluid/tests/unittests/ps_dnn_model.py index 89472c73bd3..c2775fbca31 100755 --- a/python/paddle/fluid/tests/unittests/ps_dnn_model.py +++ b/python/paddle/fluid/tests/unittests/ps_dnn_model.py @@ -18,14 +18,15 @@ import math class DNNLayer(nn.Layer): - - def __init__(self, - sparse_feature_number, - sparse_feature_dim, - dense_feature_dim, - num_field, - layer_sizes, - sync_mode=None): + def __init__( + self, + sparse_feature_number, + sparse_feature_dim, + dense_feature_dim, + num_field, + layer_sizes, + sync_mode=None, + ): super(DNNLayer, self).__init__() self.sync_mode = sync_mode self.sparse_feature_number = sparse_feature_number @@ -40,10 +41,15 @@ class DNNLayer(nn.Layer): sparse=True, weight_attr=paddle.ParamAttr( name="SparseFeatFactors", - initializer=paddle.nn.initializer.Uniform())) - - sizes = [sparse_feature_dim * num_field + dense_feature_dim - ] + self.layer_sizes + [2] + initializer=paddle.nn.initializer.Uniform(), + ), + ) + + sizes = ( + [sparse_feature_dim * num_field + dense_feature_dim] + + self.layer_sizes + + [2] + ) acts = ["relu" for _ in range(len(self.layer_sizes))] + [None] self._mlp_layers = [] for i in range(len(layer_sizes) + 1): @@ -52,7 +58,10 @@ class DNNLayer(nn.Layer): out_features=sizes[i + 1], weight_attr=paddle.ParamAttr( initializer=paddle.nn.initializer.Normal( - std=1.0 / math.sqrt(sizes[i])))) + std=1.0 / math.sqrt(sizes[i]) + ) + ), + ) self.add_sublayer('linear_%d' % i, linear) self._mlp_layers.append(linear) if acts[i] == 'relu': @@ -68,7 +77,8 @@ class DNNLayer(nn.Layer): emb = paddle.fluid.contrib.sparse_embedding( input=s_input, size=[self.sparse_feature_number, self.sparse_feature_dim], - param_attr=paddle.ParamAttr(name="embedding")) + param_attr=paddle.ParamAttr(name="embedding"), + ) else: emb = self.embedding(s_input) emb = paddle.reshape(emb, shape=[-1, self.sparse_feature_dim]) @@ -89,13 +99,14 @@ class DNNLayer(nn.Layer): class FlDNNLayer(nn.Layer): - - def __init__(self, - sparse_feature_number, - sparse_feature_dim, - dense_feature_dim, - sparse_number, - sync_mode=None): + def __init__( + self, + sparse_feature_number, + sparse_feature_dim, + dense_feature_dim, + sparse_number, + sync_mode=None, + ): super(FlDNNLayer, self).__init__() self.PART_A_DEVICE_FlAG = 'gpu:0' @@ -109,8 +120,11 @@ class FlDNNLayer(nn.Layer): self.slot_num = sparse_number self.dense_feature_dim = dense_feature_dim - layer_sizes_a = [self.slot_num * self.sparse_feature_dim, 5, - 7] # for test + layer_sizes_a = [ + self.slot_num * self.sparse_feature_dim, + 5, + 7, + ] # for test layer_sizes_b = [self.dense_feature_dim, 6, 7] layer_sizes_top = [7, 2] @@ -120,7 +134,9 @@ class FlDNNLayer(nn.Layer): sparse=True, weight_attr=paddle.ParamAttr( name="SparseFeatFactors", - initializer=paddle.nn.initializer.Uniform())) + initializer=paddle.nn.initializer.Uniform(), + ), + ) # part_a fc acts = ["relu" for _ in range(len(layer_sizes_a))] @@ -131,7 +147,10 @@ class FlDNNLayer(nn.Layer): out_features=layer_sizes_a[i + 1], weight_attr=paddle.ParamAttr( initializer=paddle.nn.initializer.Normal( - std=1.0 / math.sqrt(layer_sizes_a[i])))) + std=1.0 / math.sqrt(layer_sizes_a[i]) + ) + ), + ) self.add_sublayer('linear_%d' % i, linear) self._mlp_layers_a.append(linear) act = paddle.nn.ReLU() @@ -147,7 +166,10 @@ class FlDNNLayer(nn.Layer): out_features=layer_sizes_b[i + 1], weight_attr=paddle.ParamAttr( initializer=paddle.nn.initializer.Normal( - std=1.0 / math.sqrt(layer_sizes_b[i])))) + std=1.0 / math.sqrt(layer_sizes_b[i]) + ) + ), + ) self.add_sublayer('linear_%d' % i, linear) self._mlp_layers_b.append(linear) act = paddle.nn.ReLU() @@ -163,7 +185,10 @@ class FlDNNLayer(nn.Layer): out_features=layer_sizes_top[i + 1], weight_attr=paddle.ParamAttr( initializer=paddle.nn.initializer.Normal( - std=1.0 / math.sqrt(layer_sizes_top[i])))) + std=1.0 / math.sqrt(layer_sizes_top[i]) + ) + ), + ) self.add_sublayer('linear_%d' % i, linear) self._mlp_layers_top.append(linear) act = paddle.nn.ReLU() @@ -184,7 +209,8 @@ class FlDNNLayer(nn.Layer): y = self._mlp_layers_a[2](y) with paddle.fluid.device_guard( - self.PART_A_JOINT_OP_DEVICE_FlAG): # joint point + self.PART_A_JOINT_OP_DEVICE_FlAG + ): # joint point bottom_a = self._mlp_layers_a[3](y) return bottom_a @@ -201,9 +227,11 @@ class FlDNNLayer(nn.Layer): def interactive_layer(self, bottom_a, bottom_b): with paddle.fluid.device_guard( - self.PART_B_JOINT_OP_DEVICE_FlAG): # joint point + self.PART_B_JOINT_OP_DEVICE_FlAG + ): # joint point interactive = paddle.fluid.layers.elementwise_add( - bottom_a, bottom_b) + bottom_a, bottom_b + ) return interactive def top_layer(self, interactive, label_input): @@ -211,16 +239,25 @@ class FlDNNLayer(nn.Layer): y = self._mlp_layers_top[0](interactive) y_top = self._mlp_layers_top[1](y) predict_2d = paddle.nn.functional.softmax(y_top) - auc, batch_auc, [ - self.batch_stat_pos, self.batch_stat_neg, self.stat_pos, - self.stat_neg - ] = paddle.static.auc(input=predict_2d, - label=label_input, - num_thresholds=2**12, - slide_steps=20) - - cost = paddle.nn.functional.cross_entropy(input=y_top, - label=label_input) + ( + auc, + batch_auc, + [ + self.batch_stat_pos, + self.batch_stat_neg, + self.stat_pos, + self.stat_neg, + ], + ) = paddle.static.auc( + input=predict_2d, + label=label_input, + num_thresholds=2**12, + slide_steps=20, + ) + + cost = paddle.nn.functional.cross_entropy( + input=y_top, label=label_input + ) avg_cost = paddle.mean(x=cost) return auc, avg_cost @@ -237,8 +274,7 @@ class FlDNNLayer(nn.Layer): return auc, avg_cost -class StaticModel(): - +class StaticModel: def __init__(self, config): self.cost = None self.infer_target_var = None @@ -254,21 +290,28 @@ class StaticModel(): self.distributed_embedding = True self.sparse_feature_number = self.config.get( - "hyper_parameters.sparse_feature_number") + "hyper_parameters.sparse_feature_number" + ) self.sparse_feature_dim = self.config.get( - "hyper_parameters.sparse_feature_dim") + "hyper_parameters.sparse_feature_dim" + ) self.sparse_inputs_slots = self.config.get( - "hyper_parameters.sparse_inputs_slots") + "hyper_parameters.sparse_inputs_slots" + ) self.dense_input_dim = self.config.get( - "hyper_parameters.dense_input_dim") + "hyper_parameters.dense_input_dim" + ) self.learning_rate = self.config.get( - "hyper_parameters.optimizer.learning_rate") + "hyper_parameters.optimizer.learning_rate" + ) self.fc_sizes = self.config.get("hyper_parameters.fc_sizes") def create_feeds(self, is_infer=False): - dense_input = paddle.static.data(name="dense_input", - shape=[None, self.dense_input_dim], - dtype="float32") + dense_input = paddle.static.data( + name="dense_input", + shape=[None, self.dense_input_dim], + dtype="float32", + ) sparse_input_ids = [ paddle.static.data(name=str(i), shape=[None, 1], dtype="int64") @@ -282,33 +325,44 @@ class StaticModel(): def net(self, input, is_infer=False): self.label_input = input[0] - self.sparse_inputs = input[1:self.sparse_inputs_slots] + self.sparse_inputs = input[1 : self.sparse_inputs_slots] self.dense_input = input[-1] sparse_number = self.sparse_inputs_slots - 1 - dnn_model = DNNLayer(self.sparse_feature_number, - self.sparse_feature_dim, - self.dense_input_dim, - sparse_number, - self.fc_sizes, - sync_mode=self.sync_mode) + dnn_model = DNNLayer( + self.sparse_feature_number, + self.sparse_feature_dim, + self.dense_input_dim, + sparse_number, + self.fc_sizes, + sync_mode=self.sync_mode, + ) raw_predict_2d = dnn_model.forward(self.sparse_inputs, self.dense_input) predict_2d = paddle.nn.functional.softmax(raw_predict_2d) self.predict = predict_2d - auc, batch_auc, [ - self.batch_stat_pos, self.batch_stat_neg, self.stat_pos, - self.stat_neg - ] = paddle.static.auc(input=self.predict, - label=self.label_input, - num_thresholds=2**12, - slide_steps=20) + ( + auc, + batch_auc, + [ + self.batch_stat_pos, + self.batch_stat_neg, + self.stat_pos, + self.stat_neg, + ], + ) = paddle.static.auc( + input=self.predict, + label=self.label_input, + num_thresholds=2**12, + slide_steps=20, + ) self.inference_target_var = auc if is_infer: fetch_dict = {'auc': auc} return fetch_dict - cost = paddle.nn.functional.cross_entropy(input=raw_predict_2d, - label=self.label_input) + cost = paddle.nn.functional.cross_entropy( + input=raw_predict_2d, label=self.label_input + ) avg_cost = paddle.mean(x=cost) self._cost = avg_cost @@ -317,18 +371,21 @@ class StaticModel(): def fl_net(self, input, is_infer=False): self.label_input = input[0] - self.sparse_inputs = input[1:self.sparse_inputs_slots] + self.sparse_inputs = input[1 : self.sparse_inputs_slots] self.dense_input = input[-1] self.sparse_number = self.sparse_inputs_slots - 1 - fl_dnn_model = FlDNNLayer(self.sparse_feature_number, - self.sparse_feature_dim, - self.dense_input_dim, - self.sparse_number, - sync_mode=self.sync_mode) - - auc, avg_cost = fl_dnn_model.forward(self.sparse_inputs, - self.dense_input, self.label_input) + fl_dnn_model = FlDNNLayer( + self.sparse_feature_number, + self.sparse_feature_dim, + self.dense_input_dim, + self.sparse_number, + sync_mode=self.sync_mode, + ) + + auc, avg_cost = fl_dnn_model.forward( + self.sparse_inputs, self.dense_input, self.label_input + ) fetch_dict = {'cost': avg_cost, 'auc': auc} self._cost = avg_cost return fetch_dict diff --git a/python/paddle/fluid/tests/unittests/py_precise_roi_pool.py b/python/paddle/fluid/tests/unittests/py_precise_roi_pool.py index 29721c86200..e18c302fd7b 100644 --- a/python/paddle/fluid/tests/unittests/py_precise_roi_pool.py +++ b/python/paddle/fluid/tests/unittests/py_precise_roi_pool.py @@ -17,7 +17,6 @@ import numpy as np class PyPrRoIPool(object): - def __init__(self): pass @@ -28,50 +27,65 @@ class PyPrRoIPool(object): else: return data[h][w] - def _PrRoIPoolingMatCalculation(self, this_data, s_h, s_w, e_h, e_w, y0, x0, - y1, x1, h0, w0): + def _PrRoIPoolingMatCalculation( + self, this_data, s_h, s_w, e_h, e_w, y0, x0, y1, x1, h0, w0 + ): sum_out = 0.0 alpha = x0 - float(s_w) beta = y0 - float(s_h) lim_alpha = x1 - float(s_w) lim_beta = y1 - float(s_h) - tmp = (lim_alpha - 0.5 * lim_alpha * lim_alpha - alpha + - 0.5 * alpha * alpha) * (lim_beta - 0.5 * lim_beta * lim_beta - - beta + 0.5 * beta * beta) + tmp = ( + lim_alpha + - 0.5 * lim_alpha * lim_alpha + - alpha + + 0.5 * alpha * alpha + ) * (lim_beta - 0.5 * lim_beta * lim_beta - beta + 0.5 * beta * beta) sum_out += self._PrRoIPoolingGetData(this_data, s_h, s_w, h0, w0) * tmp alpha = float(e_w) - x1 lim_alpha = float(e_w) - x0 - tmp = (lim_alpha - 0.5 * lim_alpha * lim_alpha - alpha + - 0.5 * alpha * alpha) * (lim_beta - 0.5 * lim_beta * lim_beta - - beta + 0.5 * beta * beta) + tmp = ( + lim_alpha + - 0.5 * lim_alpha * lim_alpha + - alpha + + 0.5 * alpha * alpha + ) * (lim_beta - 0.5 * lim_beta * lim_beta - beta + 0.5 * beta * beta) sum_out += self._PrRoIPoolingGetData(this_data, s_h, e_w, h0, w0) * tmp alpha = x0 - float(s_w) beta = float(e_h) - y1 lim_alpha = x1 - float(s_w) lim_beta = float(e_h) - y0 - tmp = (lim_alpha - 0.5 * lim_alpha * lim_alpha - alpha + - 0.5 * alpha * alpha) * (lim_beta - 0.5 * lim_beta * lim_beta - - beta + 0.5 * beta * beta) + tmp = ( + lim_alpha + - 0.5 * lim_alpha * lim_alpha + - alpha + + 0.5 * alpha * alpha + ) * (lim_beta - 0.5 * lim_beta * lim_beta - beta + 0.5 * beta * beta) sum_out += self._PrRoIPoolingGetData(this_data, e_h, s_w, h0, w0) * tmp alpha = float(e_w) - x1 lim_alpha = float(e_w) - x0 - tmp = (lim_alpha - 0.5 * lim_alpha * lim_alpha - alpha + - 0.5 * alpha * alpha) * (lim_beta - 0.5 * lim_beta * lim_beta - - beta + 0.5 * beta * beta) + tmp = ( + lim_alpha + - 0.5 * lim_alpha * lim_alpha + - alpha + + 0.5 * alpha * alpha + ) * (lim_beta - 0.5 * lim_beta * lim_beta - beta + 0.5 * beta * beta) sum_out += self._PrRoIPoolingGetData(this_data, e_h, e_w, h0, w0) * tmp return sum_out - def compute(self, - x, - rois, - output_channels, - spatial_scale=0.1, - pooled_height=1, - pooled_width=1): + def compute( + self, + x, + rois, + output_channels, + spatial_scale=0.1, + pooled_height=1, + pooled_width=1, + ): ''' calculate the precise roi pooling values Note: This function is implements as pure python without any paddle concept involved @@ -134,14 +148,18 @@ class PyPrRoIPool(object): for w_iter in range(int(s_w), int(e_w)): for h_iter in range(int(s_h), int(e_h)): sum_out += self._PrRoIPoolingMatCalculation( - x_i[c_in], h_iter, w_iter, h_iter + 1, + x_i[c_in], + h_iter, + w_iter, + h_iter + 1, w_iter + 1, max(win_start_h, float(h_iter)), max(win_start_w, float(w_iter)), - min(win_end_h, - float(h_iter) + 1.0), - min(win_end_w, - float(w_iter + 1.0)), height, width) + min(win_end_h, float(h_iter) + 1.0), + min(win_end_w, float(w_iter + 1.0)), + height, + width, + ) out_data[i, c, ph, pw] = sum_out / win_size diff --git a/python/paddle/fluid/tests/unittests/rnn/convert.py b/python/paddle/fluid/tests/unittests/rnn/convert.py index b26db11e1d3..c41f80c7eab 100644 --- a/python/paddle/fluid/tests/unittests/rnn/convert.py +++ b/python/paddle/fluid/tests/unittests/rnn/convert.py @@ -41,22 +41,28 @@ def convert_params_for_net(np_net, paddle_net): def convert_params_for_net_static(np_net, paddle_net, place): for np_layer, paddle_layer in zip(np_net, paddle_net): if hasattr(np_layer, "cell"): - convert_params_for_cell_static(np_layer.cell, paddle_layer.cell, - place) + convert_params_for_cell_static( + np_layer.cell, paddle_layer.cell, place + ) else: - convert_params_for_cell_static(np_layer.cell_fw, - paddle_layer.cell_fw, place) - convert_params_for_cell_static(np_layer.cell_bw, - paddle_layer.cell_bw, place) + convert_params_for_cell_static( + np_layer.cell_fw, paddle_layer.cell_fw, place + ) + convert_params_for_cell_static( + np_layer.cell_bw, paddle_layer.cell_bw, place + ) def get_params_for_cell(np_cell, num_layers, idx): state = np_cell.parameters - weight_list = [('{}.weight_{}'.format(num_layers, idx), state['weight_ih']), - ('{}.weight_{}'.format(num_layers, - idx + 1), state['weight_hh'])] - bias_list = [('{}.bias_{}'.format(num_layers, idx), state['bias_ih']), - ('{}.bias_{}'.format(num_layers, idx + 1), state['bias_hh'])] + weight_list = [ + ('{}.weight_{}'.format(num_layers, idx), state['weight_ih']), + ('{}.weight_{}'.format(num_layers, idx + 1), state['weight_hh']), + ] + bias_list = [ + ('{}.bias_{}'.format(num_layers, idx), state['bias_ih']), + ('{}.bias_{}'.format(num_layers, idx + 1), state['bias_hh']), + ] return weight_list, bias_list diff --git a/python/paddle/fluid/tests/unittests/rnn/rnn_numpy.py b/python/paddle/fluid/tests/unittests/rnn/rnn_numpy.py index fbdc3ec8a48..20859f896d1 100644 --- a/python/paddle/fluid/tests/unittests/rnn/rnn_numpy.py +++ b/python/paddle/fluid/tests/unittests/rnn/rnn_numpy.py @@ -17,13 +17,11 @@ import math class LayerMixin(object): - def __call__(self, *args, **kwargs): return self.forward(*args, **kwargs) class LayerListMixin(LayerMixin): - def __init__(self, layers=None): self._layers = list(layers) if layers else [] @@ -35,34 +33,39 @@ class LayerListMixin(LayerMixin): class SimpleRNNCell(LayerMixin): - - def __init__(self, - input_size, - hidden_size, - bias=True, - nonlinearity="RNN_TANH", - dtype="float64"): + def __init__( + self, + input_size, + hidden_size, + bias=True, + nonlinearity="RNN_TANH", + dtype="float64", + ): self.input_size = input_size self.hidden_size = hidden_size self.bias = bias if nonlinearity == 'RNN_TANH': self.nonlinearity = np.tanh else: - self.nonlinearity = lambda x: np.maximum(x, 0.) + self.nonlinearity = lambda x: np.maximum(x, 0.0) self.parameters = dict() std = 1.0 / math.sqrt(hidden_size) self.weight_ih = np.random.uniform( - -std, std, (hidden_size, input_size)).astype(dtype) + -std, std, (hidden_size, input_size) + ).astype(dtype) self.weight_hh = np.random.uniform( - -std, std, (hidden_size, hidden_size)).astype(dtype) + -std, std, (hidden_size, hidden_size) + ).astype(dtype) self.parameters['weight_ih'] = self.weight_ih self.parameters['weight_hh'] = self.weight_hh if bias: - self.bias_ih = np.random.uniform(-std, std, - (hidden_size, )).astype(dtype) - self.bias_hh = np.random.uniform(-std, std, - (hidden_size, )).astype(dtype) + self.bias_ih = np.random.uniform(-std, std, (hidden_size,)).astype( + dtype + ) + self.bias_hh = np.random.uniform(-std, std, (hidden_size,)).astype( + dtype + ) self.parameters['bias_ih'] = self.bias_ih self.parameters['bias_hh'] = self.bias_hh else: @@ -88,7 +91,6 @@ class SimpleRNNCell(LayerMixin): class GRUCell(LayerMixin): - def __init__(self, input_size, hidden_size, bias=True, dtype="float64"): self.input_size = input_size self.hidden_size = hidden_size @@ -96,16 +98,20 @@ class GRUCell(LayerMixin): self.parameters = dict() std = 1.0 / math.sqrt(hidden_size) self.weight_ih = np.random.uniform( - -std, std, (3 * hidden_size, input_size)).astype(dtype) + -std, std, (3 * hidden_size, input_size) + ).astype(dtype) self.weight_hh = np.random.uniform( - -std, std, (3 * hidden_size, hidden_size)).astype(dtype) + -std, std, (3 * hidden_size, hidden_size) + ).astype(dtype) self.parameters['weight_ih'] = self.weight_ih self.parameters['weight_hh'] = self.weight_hh if bias: - self.bias_ih = np.random.uniform(-std, std, - (3 * hidden_size)).astype(dtype) - self.bias_hh = np.random.uniform(-std, std, - (3 * hidden_size)).astype(dtype) + self.bias_ih = np.random.uniform( + -std, std, (3 * hidden_size) + ).astype(dtype) + self.bias_hh = np.random.uniform( + -std, std, (3 * hidden_size) + ).astype(dtype) self.parameters['bias_ih'] = self.bias_ih self.parameters['bias_hh'] = self.bias_hh else: @@ -137,7 +143,6 @@ class GRUCell(LayerMixin): class LSTMCell(LayerMixin): - def __init__(self, input_size, hidden_size, bias=True, dtype="float64"): self.input_size = input_size self.hidden_size = hidden_size @@ -145,16 +150,20 @@ class LSTMCell(LayerMixin): self.parameters = dict() std = 1.0 / math.sqrt(hidden_size) self.weight_ih = np.random.uniform( - -std, std, (4 * hidden_size, input_size)).astype(dtype) + -std, std, (4 * hidden_size, input_size) + ).astype(dtype) self.weight_hh = np.random.uniform( - -std, std, (4 * hidden_size, hidden_size)).astype(dtype) + -std, std, (4 * hidden_size, hidden_size) + ).astype(dtype) self.parameters['weight_ih'] = self.weight_ih self.parameters['weight_hh'] = self.weight_hh if bias: - self.bias_ih = np.random.uniform(-std, std, - (4 * hidden_size)).astype(dtype) - self.bias_hh = np.random.uniform(-std, std, - (4 * hidden_size)).astype(dtype) + self.bias_ih = np.random.uniform( + -std, std, (4 * hidden_size) + ).astype(dtype) + self.bias_hh = np.random.uniform( + -std, std, (4 * hidden_size) + ).astype(dtype) self.parameters['bias_ih'] = self.bias_ih self.parameters['bias_hh'] = self.bias_hh else: @@ -204,12 +213,14 @@ def update_state(mask, new, old): return tuple(map(lambda x, y: np.where(mask, x, y), new, old)) -def rnn(cell, - inputs, - initial_states, - sequence_length=None, - time_major=False, - is_reverse=False): +def rnn( + cell, + inputs, + initial_states, + sequence_length=None, + time_major=False, + is_reverse=False, +): if not time_major: inputs = np.transpose(inputs, [1, 0, 2]) if is_reverse: @@ -234,7 +245,7 @@ def rnn(cell, if mask is not None: m_t = mask[t] y, new_state = cell(x_t, state) - y = np.where(m_t, y, 0.) + y = np.where(m_t, y, 0.0) outputs.append(y) state = update_state(m_t, new_state, state) else: @@ -252,25 +263,27 @@ def rnn(cell, return outputs, final_state -def birnn(cell_fw, - cell_bw, - inputs, - initial_states, - sequence_length=None, - time_major=False): +def birnn( + cell_fw, + cell_bw, + inputs, + initial_states, + sequence_length=None, + time_major=False, +): states_fw, states_bw = initial_states - outputs_fw, states_fw = rnn(cell_fw, - inputs, - states_fw, - sequence_length, - time_major=time_major) - - outputs_bw, states_bw = rnn(cell_bw, - inputs, - states_bw, - sequence_length, - time_major=time_major, - is_reverse=True) + outputs_fw, states_fw = rnn( + cell_fw, inputs, states_fw, sequence_length, time_major=time_major + ) + + outputs_bw, states_bw = rnn( + cell_bw, + inputs, + states_bw, + sequence_length, + time_major=time_major, + is_reverse=True, + ) outputs = np.concatenate((outputs_fw, outputs_bw), -1) final_states = (states_fw, states_bw) @@ -332,7 +345,6 @@ def concat_states(states, bidirectional=False, state_components=1): class RNN(LayerMixin): - def __init__(self, cell, is_reverse=False, time_major=False): super(RNN, self).__init__() self.cell = cell @@ -343,124 +355,135 @@ class RNN(LayerMixin): self.time_major = time_major def forward(self, inputs, initial_states=None, sequence_length=None): - final_outputs, final_states = rnn(self.cell, - inputs, - initial_states=initial_states, - sequence_length=sequence_length, - time_major=self.time_major, - is_reverse=self.is_reverse) + final_outputs, final_states = rnn( + self.cell, + inputs, + initial_states=initial_states, + sequence_length=sequence_length, + time_major=self.time_major, + is_reverse=self.is_reverse, + ) return final_outputs, final_states class BiRNN(LayerMixin): - def __init__(self, cell_fw, cell_bw, time_major=False): super(BiRNN, self).__init__() self.cell_fw = cell_fw self.cell_bw = cell_bw self.time_major = time_major - def forward(self, - inputs, - initial_states=None, - sequence_length=None, - **kwargs): + def forward( + self, inputs, initial_states=None, sequence_length=None, **kwargs + ): if isinstance(initial_states, (list, tuple)): - assert len(initial_states) == 2, \ - "length of initial_states should be 2 when it is a list/tuple" + assert ( + len(initial_states) == 2 + ), "length of initial_states should be 2 when it is a list/tuple" else: initial_states = [initial_states, initial_states] - outputs, final_states = birnn(self.cell_fw, self.cell_bw, inputs, - initial_states, sequence_length, - self.time_major) + outputs, final_states = birnn( + self.cell_fw, + self.cell_bw, + inputs, + initial_states, + sequence_length, + self.time_major, + ) return outputs, final_states class RNNMixin(LayerListMixin): - def forward(self, inputs, initial_states=None, sequence_length=None): batch_index = 1 if self.time_major else 0 batch_size = inputs.shape[batch_index] dtype = inputs.dtype if initial_states is None: - state_shape = (self.num_layers * self.num_directions, batch_size, - self.hidden_size) + state_shape = ( + self.num_layers * self.num_directions, + batch_size, + self.hidden_size, + ) if self.state_components == 1: initial_states = np.zeros(state_shape, dtype) else: - initial_states = tuple([ - np.zeros(state_shape, dtype) - for _ in range(self.state_components) - ]) - - states = split_states(initial_states, self.num_directions == 2, - self.state_components) + initial_states = tuple( + [ + np.zeros(state_shape, dtype) + for _ in range(self.state_components) + ] + ) + + states = split_states( + initial_states, self.num_directions == 2, self.state_components + ) final_states = [] input_temp = inputs for i, rnn_layer in enumerate(self): if i > 0: input_temp = dropout(inputs, self.dropout) - outputs, final_state = rnn_layer(input_temp, states[i], - sequence_length) + outputs, final_state = rnn_layer( + input_temp, states[i], sequence_length + ) final_states.append(final_state) inputs = outputs - final_states = concat_states(final_states, self.num_directions == 2, - self.state_components) + final_states = concat_states( + final_states, self.num_directions == 2, self.state_components + ) return outputs, final_states class SimpleRNN(RNNMixin): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - nonlinearity="RNN_TANH", - direction="forward", - dropout=0., - time_major=False, - dtype="float64"): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + nonlinearity="RNN_TANH", + direction="forward", + dropout=0.0, + time_major=False, + dtype="float64", + ): super(SimpleRNN, self).__init__() bidirectional_list = ["bidirectional", "bidirect"] if direction in ["forward"]: is_reverse = False - cell = SimpleRNNCell(input_size, - hidden_size, - nonlinearity=nonlinearity, - dtype=dtype) + cell = SimpleRNNCell( + input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype + ) self.append(RNN(cell, is_reverse, time_major)) for i in range(1, num_layers): - cell = SimpleRNNCell(hidden_size, - hidden_size, - nonlinearity=nonlinearity, - dtype=dtype) + cell = SimpleRNNCell( + hidden_size, + hidden_size, + nonlinearity=nonlinearity, + dtype=dtype, + ) self.append(RNN(cell, is_reverse, time_major)) elif direction in bidirectional_list: - cell_fw = SimpleRNNCell(input_size, - hidden_size, - nonlinearity=nonlinearity, - dtype=dtype) - cell_bw = SimpleRNNCell(input_size, - hidden_size, - nonlinearity=nonlinearity, - dtype=dtype) + cell_fw = SimpleRNNCell( + input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype + ) + cell_bw = SimpleRNNCell( + input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype + ) self.append(BiRNN(cell_fw, cell_bw, time_major)) for i in range(1, num_layers): - cell_fw = SimpleRNNCell(2 * hidden_size, - hidden_size, - nonlinearity, - dtype=dtype) - cell_bw = SimpleRNNCell(2 * hidden_size, - hidden_size, - nonlinearity, - dtype=dtype) + cell_fw = SimpleRNNCell( + 2 * hidden_size, hidden_size, nonlinearity, dtype=dtype + ) + cell_bw = SimpleRNNCell( + 2 * hidden_size, hidden_size, nonlinearity, dtype=dtype + ) self.append(BiRNN(cell_fw, cell_bw, time_major)) else: raise ValueError( "direction should be forward, backward or bidirectional, " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.input_size = input_size self.hidden_size = hidden_size @@ -472,15 +495,16 @@ class SimpleRNN(RNNMixin): class LSTM(RNNMixin): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - dropout=0., - time_major=False, - dtype="float64"): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + dropout=0.0, + time_major=False, + dtype="float64", + ): super(LSTM, self).__init__() bidirectional_list = ["bidirectional", "bidirect"] @@ -502,7 +526,8 @@ class LSTM(RNNMixin): else: raise ValueError( "direction should be forward, backward or bidirectional, " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.input_size = input_size self.hidden_size = hidden_size @@ -514,15 +539,16 @@ class LSTM(RNNMixin): class GRU(RNNMixin): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - dropout=0., - time_major=False, - dtype="float64"): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + dropout=0.0, + time_major=False, + dtype="float64", + ): super(GRU, self).__init__() bidirectional_list = ["bidirectional", "bidirect"] @@ -544,7 +570,8 @@ class GRU(RNNMixin): else: raise ValueError( "direction should be forward, backward or bidirectional, " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.input_size = input_size self.hidden_size = hidden_size diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells.py index 33dca32b76c..ee2e7cfaa0f 100644 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells.py @@ -24,20 +24,19 @@ from convert import convert_params_for_cell class TestSimpleRNNCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestSimpleRNNCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): paddle.disable_static(self.place) rnn1 = SimpleRNNCell(16, 32, bias=self.bias) - rnn2 = paddle.nn.SimpleRNNCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.SimpleRNNCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) convert_params_for_cell(rnn1, rnn2) self.rnn1 = rnn1 @@ -65,7 +64,6 @@ class TestSimpleRNNCell(unittest.TestCase): np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5) def test_errors(self): - def test_zero_hidden_size(): cell = paddle.nn.SimpleRNNCell(-1, 0) @@ -78,20 +76,19 @@ class TestSimpleRNNCell(unittest.TestCase): class TestGRUCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestGRUCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): paddle.disable_static(self.place) rnn1 = GRUCell(16, 32, bias=self.bias) - rnn2 = paddle.nn.GRUCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.GRUCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) convert_params_for_cell(rnn1, rnn2) self.rnn1 = rnn1 @@ -119,7 +116,6 @@ class TestGRUCell(unittest.TestCase): np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5) def test_errors(self): - def test_zero_hidden_size(): cell = paddle.nn.GRUCell(-1, 0) @@ -132,19 +128,18 @@ class TestGRUCell(unittest.TestCase): class TestLSTMCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestLSTMCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): rnn1 = LSTMCell(16, 32, bias=self.bias) - rnn2 = paddle.nn.LSTMCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.LSTMCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) convert_params_for_cell(rnn1, rnn2) self.rnn1 = rnn1 @@ -159,9 +154,10 @@ class TestLSTMCell(unittest.TestCase): prev_c = np.random.randn(4, 32) y1, (h1, c1) = rnn1(x, (prev_h, prev_c)) - y2, (h2, - c2) = rnn2(paddle.to_tensor(x), - (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c))) + y2, (h2, c2) = rnn2( + paddle.to_tensor(x), + (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c)), + ) np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5) @@ -177,7 +173,6 @@ class TestLSTMCell(unittest.TestCase): np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5) def test_errors(self): - def test_zero_hidden_size(): cell = paddle.nn.LSTMCell(-1, 0) @@ -191,8 +186,9 @@ class TestLSTMCell(unittest.TestCase): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for bias in [True, False]: for device in devices: for test_class in [TestSimpleRNNCell, TestGRUCell, TestLSTMCell]: diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells_static.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells_static.py index b4a5887c593..a9e2a013a25 100644 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells_static.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells_static.py @@ -25,12 +25,12 @@ from rnn_numpy import SimpleRNNCell, LSTMCell, GRUCell class TestSimpleRNNCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestSimpleRNNCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): rnn1 = SimpleRNNCell(16, 32, bias=self.bias) @@ -39,10 +39,9 @@ class TestSimpleRNNCell(unittest.TestCase): sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.SimpleRNNCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.SimpleRNNCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) place = self.place exe = paddle.static.Executor(place) @@ -75,11 +74,15 @@ class TestSimpleRNNCell(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [-1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [-1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data, init_h) feed_dict = {x_data.name: x, init_h.name: prev_h} @@ -103,17 +106,18 @@ class TestSimpleRNNCell(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data) feed_dict = {x_data.name: x} with paddle.static.scope_guard(scope): - y2, h2 = exe.run(mp, - feed=feed_dict, - fetch_list=[y, h], - use_prune=True) + y2, h2 = exe.run( + mp, feed=feed_dict, fetch_list=[y, h], use_prune=True + ) np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5) @@ -123,12 +127,12 @@ class TestSimpleRNNCell(unittest.TestCase): class TestGRUCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestGRUCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): rnn1 = GRUCell(16, 32, bias=self.bias) @@ -137,10 +141,9 @@ class TestGRUCell(unittest.TestCase): sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.GRUCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.GRUCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) place = self.place exe = paddle.static.Executor(place) @@ -174,11 +177,15 @@ class TestGRUCell(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [-1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [-1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data, init_h) feed_dict = {x_data.name: x, init_h.name: prev_h} @@ -202,17 +209,18 @@ class TestGRUCell(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data) feed_dict = {x_data.name: x} with paddle.static.scope_guard(scope): - y2, h2 = exe.run(mp, - feed=feed_dict, - fetch_list=[y, h], - use_prune=True) + y2, h2 = exe.run( + mp, feed=feed_dict, fetch_list=[y, h], use_prune=True + ) np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5) @@ -222,12 +230,12 @@ class TestGRUCell(unittest.TestCase): class TestLSTMCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestLSTMCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): rnn1 = LSTMCell(16, 32, bias=self.bias) @@ -236,10 +244,9 @@ class TestLSTMCell(unittest.TestCase): sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.LSTMCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.LSTMCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) place = self.place exe = paddle.static.Executor(place) @@ -274,14 +281,20 @@ class TestLSTMCell(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [-1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [-1, 32], + dtype=paddle.framework.get_default_dtype(), + ) init_c = paddle.fluid.data( - "init_c", [-1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_c", + [-1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, (h, c) = rnn2(x_data, (init_h, init_c)) feed_dict = {x_data.name: x, init_h.name: prev_h, init_c.name: prev_c} @@ -306,17 +319,18 @@ class TestLSTMCell(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, (h, c) = rnn2(x_data) feed_dict = {x_data.name: x} with paddle.static.scope_guard(scope): - y2, h2, c2 = exe.run(mp, - feed=feed_dict, - fetch_list=[y, h, c], - use_prune=True) + y2, h2, c2 = exe.run( + mp, feed=feed_dict, fetch_list=[y, h, c], use_prune=True + ) np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5) np.testing.assert_allclose(c1, c2, atol=1e-8, rtol=1e-5) @@ -328,8 +342,9 @@ class TestLSTMCell(unittest.TestCase): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for bias in [True, False]: for device in devices: for test_class in [TestSimpleRNNCell, TestGRUCell, TestLSTMCell]: diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cudnn_params_packing.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cudnn_params_packing.py index f4dbc3bbbc7..5949addbea1 100644 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cudnn_params_packing.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cudnn_params_packing.py @@ -18,15 +18,13 @@ from unittest import TestCase def create_model(): hidden_size = 32 - bilstm = paddle.nn.LSTM(hidden_size, - hidden_size, - num_layers=1, - direction='bidirectional') + bilstm = paddle.nn.LSTM( + hidden_size, hidden_size, num_layers=1, direction='bidirectional' + ) return bilstm class TestRNNProgramClone(TestCase): - def setUp(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py index bf33d553201..0e3407631dc 100755 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py @@ -29,7 +29,6 @@ bidirectional_list = ["bidirectional", "bidirect"] class TestSimpleRNN(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestSimpleRNN, self).__init__("runTest") self.time_major = time_major @@ -42,16 +41,12 @@ class TestSimpleRNN(unittest.TestCase): # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) paddle.disable_static(place) - rnn1 = SimpleRNN(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) - rnn2 = paddle.nn.SimpleRNN(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = SimpleRNN( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) + rnn2 = paddle.nn.SimpleRNN( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) convert_params_for_net(rnn1, rnn2) self.rnn1 = rnn1 @@ -117,7 +112,6 @@ class TestSimpleRNN(unittest.TestCase): class TestGRU(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestGRU, self).__init__("runTest") self.time_major = time_major @@ -130,16 +124,12 @@ class TestGRU(unittest.TestCase): # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) paddle.disable_static(place) - rnn1 = GRU(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) - rnn2 = paddle.nn.GRU(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = GRU( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) + rnn2 = paddle.nn.GRU( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) convert_params_for_net(rnn1, rnn2) self.rnn1 = rnn1 @@ -205,7 +195,6 @@ class TestGRU(unittest.TestCase): class TestLSTM(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestLSTM, self).__init__("runTest") self.time_major = time_major @@ -218,16 +207,12 @@ class TestLSTM(unittest.TestCase): # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) paddle.disable_static(place) - rnn1 = LSTM(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) - rnn2 = paddle.nn.LSTM(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = LSTM( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) + rnn2 = paddle.nn.LSTM( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) convert_params_for_net(rnn1, rnn2) self.rnn1 = rnn1 @@ -244,9 +229,10 @@ class TestLSTM(unittest.TestCase): prev_c = np.random.randn(2 * self.num_directions, 4, 32) y1, (h1, c1) = rnn1(x, (prev_h, prev_c)) - y2, (h2, - c2) = rnn2(paddle.to_tensor(x), - (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c))) + y2, (h2, c2) = rnn2( + paddle.to_tensor(x), + (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c)), + ) np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5) @@ -305,14 +291,11 @@ def predict_test_util(place, mode, stop_gradient=True): np.random.seed(123) class Net(paddle.nn.Layer): - def __init__(self): super(Net, self).__init__() - self.rnn = getattr(paddle.nn, mode)(16, - 32, - 2, - direction="bidirectional", - dropout=0.1) + self.rnn = getattr(paddle.nn, mode)( + 16, 32, 2, direction="bidirectional", dropout=0.1 + ) def forward(self, input): return self.rnn(input) @@ -327,8 +310,9 @@ def predict_test_util(place, mode, stop_gradient=True): y = y * mask loss = paddle.mean(y) loss.backward() - optimizer = paddle.optimizer.Adam(learning_rate=0.1, - parameters=rnn.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.1, parameters=rnn.parameters() + ) optimizer.step() rnn.eval() y, _ = rnn(x) @@ -337,7 +321,8 @@ def predict_test_util(place, mode, stop_gradient=True): rnn.train() rnn = paddle.jit.to_static( - rnn, [paddle.static.InputSpec(shape=[None, None, 16], dtype=x.dtype)]) + rnn, [paddle.static.InputSpec(shape=[None, None, 16], dtype=x.dtype)] + ) temp_dir = tempfile.TemporaryDirectory() save_dirname = os.path.join(temp_dir.name, "./inference/%s_infer" % mode) @@ -348,13 +333,19 @@ def predict_test_util(place, mode, stop_gradient=True): new_scope = paddle.static.Scope() with paddle.static.scope_guard(new_scope): exe = paddle.static.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model(save_dirname, exe) - results = exe.run(inference_program, - feed={feed_target_names[0]: x.numpy()}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(save_dirname, exe) + results = exe.run( + inference_program, + feed={feed_target_names[0]: x.numpy()}, + fetch_list=fetch_targets, + ) np.testing.assert_equal( - y.numpy(), results[0]) # eval results equal predict results + y.numpy(), results[0] + ) # eval results equal predict results paddle.disable_static() temp_dir.cleanup() @@ -362,8 +353,9 @@ def predict_test_util(place, mode, stop_gradient=True): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for direction in ["forward", "bidirectional", "bidirect"]: for time_major in [True, False]: for device in devices: diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets_static.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets_static.py index 436bf0b6ea0..e186f28c35c 100755 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets_static.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets_static.py @@ -29,7 +29,6 @@ bidirectional_list = ["bidirectional", "bidirect"] class TestSimpleRNN(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestSimpleRNN, self).__init__("runTest") self.time_major = time_major @@ -41,21 +40,21 @@ class TestSimpleRNN(unittest.TestCase): # Since `set_device` is global, set `set_device` in `setUp` rather than # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) - rnn1 = SimpleRNN(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = SimpleRNN( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) mp = paddle.static.Program() sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.SimpleRNN(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn2 = paddle.nn.SimpleRNN( + 16, + 32, + 2, + time_major=self.time_major, + direction=self.direction, + ) exe = paddle.static.Executor(place) scope = paddle.fluid.Scope() @@ -90,11 +89,15 @@ class TestSimpleRNN(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [2 * self.num_directions, -1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [2 * self.num_directions, -1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data, init_h) feed_dict = {x_data.name: x, init_h.name: prev_h} @@ -121,8 +124,10 @@ class TestSimpleRNN(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data) feed_dict = {x_data.name: x} @@ -151,8 +156,10 @@ class TestSimpleRNN(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) seq_len = paddle.fluid.data("seq_len", [-1], dtype="int64") mask = sequence_mask(seq_len, dtype=paddle.get_default_dtype()) if self.time_major: @@ -176,7 +183,6 @@ class TestSimpleRNN(unittest.TestCase): class TestGRU(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestGRU, self).__init__("runTest") self.time_major = time_major @@ -188,21 +194,21 @@ class TestGRU(unittest.TestCase): # Since `set_device` is global, set `set_device` in `setUp` rather than # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) - rnn1 = GRU(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = GRU( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) mp = paddle.static.Program() sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.GRU(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn2 = paddle.nn.GRU( + 16, + 32, + 2, + time_major=self.time_major, + direction=self.direction, + ) exe = paddle.static.Executor(place) scope = paddle.fluid.Scope() @@ -238,11 +244,15 @@ class TestGRU(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [2 * self.num_directions, -1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [2 * self.num_directions, -1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data, init_h) feed_dict = {x_data.name: x, init_h.name: prev_h} @@ -269,8 +279,10 @@ class TestGRU(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data) feed_dict = {x_data.name: x} @@ -299,8 +311,10 @@ class TestGRU(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) seq_len = paddle.fluid.data("seq_len", [-1], dtype="int64") mask = sequence_mask(seq_len, dtype=paddle.get_default_dtype()) if self.time_major: @@ -323,7 +337,6 @@ class TestGRU(unittest.TestCase): class TestLSTM(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestLSTM, self).__init__("runTest") self.time_major = time_major @@ -335,21 +348,21 @@ class TestLSTM(unittest.TestCase): # Since `set_device` is global, set `set_device` in `setUp` rather than # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) - rnn1 = LSTM(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = LSTM( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) mp = paddle.static.Program() sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.LSTM(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn2 = paddle.nn.LSTM( + 16, + 32, + 2, + time_major=self.time_major, + direction=self.direction, + ) exe = paddle.static.Executor(place) scope = paddle.fluid.Scope() @@ -385,14 +398,20 @@ class TestLSTM(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [2 * self.num_directions, -1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [2 * self.num_directions, -1, 32], + dtype=paddle.framework.get_default_dtype(), + ) init_c = paddle.fluid.data( - "init_c", [2 * self.num_directions, -1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_c", + [2 * self.num_directions, -1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, (h, c) = rnn2(x_data, (init_h, init_c)) feed_dict = {x_data.name: x, init_h.name: prev_h, init_c.name: prev_c} @@ -420,8 +439,10 @@ class TestLSTM(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, (h, c) = rnn2(x_data) feed_dict = {x_data.name: x} @@ -451,8 +472,10 @@ class TestLSTM(unittest.TestCase): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) seq_len = paddle.fluid.data("seq_len", [-1], dtype="int64") mask = sequence_mask(seq_len, dtype=paddle.get_default_dtype()) if self.time_major: @@ -478,8 +501,9 @@ class TestLSTM(unittest.TestCase): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for direction in ["forward", "bidirectional", "bidirect"]: for time_major in [True, False]: for device in devices: diff --git a/python/paddle/fluid/tests/unittests/rnn/test_wrappers.py b/python/paddle/fluid/tests/unittests/rnn/test_wrappers.py index 2442e6b7a3b..27e1293cb68 100755 --- a/python/paddle/fluid/tests/unittests/rnn/test_wrappers.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_wrappers.py @@ -25,25 +25,29 @@ from rnn_numpy import GRUCell, RNN, BiRNN class TestRNNWrapper(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestRNNWrapper, self).__init__("runTest") self.time_major = time_major self.direction = direction - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): paddle.disable_static(self.place) cell1 = GRUCell(16, 32) cell2 = paddle.nn.GRUCell(16, 32) convert_params_for_cell(cell1, cell2) - rnn1 = RNN(cell1, - is_reverse=self.direction == "backward", - time_major=self.time_major) - rnn2 = paddle.nn.RNN(cell2, - is_reverse=self.direction == "backward", - time_major=self.time_major) + rnn1 = RNN( + cell1, + is_reverse=self.direction == "backward", + time_major=self.time_major, + ) + rnn2 = paddle.nn.RNN( + cell2, + is_reverse=self.direction == "backward", + time_major=self.time_major, + ) self.rnn1 = rnn1 self.rnn2 = rnn2 @@ -104,12 +108,12 @@ class TestRNNWrapper(unittest.TestCase): class TestBiRNNWrapper(unittest.TestCase): - def __init__(self, time_major=True, place="cpu"): super(TestBiRNNWrapper, self).__init__("runTest") self.time_major = time_major - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): paddle.disable_static(self.place) @@ -138,7 +142,8 @@ class TestBiRNNWrapper(unittest.TestCase): y1, (fw_h1, bw_h1) = rnn1(x, (fw_prev_h, bw_prev_h)) y2, (fw_h2, bw_h2) = rnn2( paddle.to_tensor(x), - (paddle.to_tensor(fw_prev_h), paddle.to_tensor(bw_prev_h))) + (paddle.to_tensor(fw_prev_h), paddle.to_tensor(bw_prev_h)), + ) np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(fw_h1, fw_h2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(bw_h1, bw_h2.numpy(), atol=1e-8, rtol=1e-5) @@ -188,8 +193,9 @@ class TestBiRNNWrapper(unittest.TestCase): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for direction in ["forward", "backward"]: for device in devices: for time_major in [False]: diff --git a/python/paddle/fluid/tests/unittests/rpc/rpc_launch_sync_add.py b/python/paddle/fluid/tests/unittests/rpc/rpc_launch_sync_add.py index 3294f159b77..794efb22fc7 100644 --- a/python/paddle/fluid/tests/unittests/rpc/rpc_launch_sync_add.py +++ b/python/paddle/fluid/tests/unittests/rpc/rpc_launch_sync_add.py @@ -59,11 +59,11 @@ def main(): shape=(10 * world_size, 100), ) for i in range(world_size): - a = mmap_data1[i * 10:(i + 1) * 10, :] - b = mmap_data2[i * 10:(i + 1) * 10, :] + a = mmap_data1[i * 10 : (i + 1) * 10, :] + b = mmap_data2[i * 10 : (i + 1) * 10, :] args = (a, b) out = rpc_add(worker_name(i), args) - mmap_out[i * 10:(i + 1) * 10, :] = out[:] + mmap_out[i * 10 : (i + 1) * 10, :] = out[:] dist.rpc.shutdown() diff --git a/python/paddle/fluid/tests/unittests/rpc/test_rpc.py b/python/paddle/fluid/tests/unittests/rpc/test_rpc.py index c6c2eb43be2..67e8b65423c 100644 --- a/python/paddle/fluid/tests/unittests/rpc/test_rpc.py +++ b/python/paddle/fluid/tests/unittests/rpc/test_rpc.py @@ -35,7 +35,6 @@ def paddle_add(a, b): class TestMultiProcessRpc(RpcTestBase): - def test_one_server_sync_paddle_add(self): a = np.random.random((10, 100)) b = np.random.random((10, 100)) @@ -78,7 +77,6 @@ class TestMultiProcessRpc(RpcTestBase): class TestSingleProcessRpc(RpcTestBase): - def setUp(self): self._port_set = set() master_endpoint = "127.0.0.1:{}".format(self._find_free_port()) @@ -123,7 +121,6 @@ class TestSingleProcessRpc(RpcTestBase): class RpcLaunchTest(RpcLaunchTestBase): - def test_sync_rpc_paddle_add1(self): nnodes = 2 nproc_per_node = 1 diff --git a/python/paddle/fluid/tests/unittests/rpc/test_rpc_base.py b/python/paddle/fluid/tests/unittests/rpc/test_rpc_base.py index 95a59267d53..daf50b88845 100644 --- a/python/paddle/fluid/tests/unittests/rpc/test_rpc_base.py +++ b/python/paddle/fluid/tests/unittests/rpc/test_rpc_base.py @@ -64,10 +64,9 @@ def run_rpc_sync_master_working( ) if dist.get_rank() == 0: for i in range(1, dist.get_rank()): - res = dist.rpc.rpc_sync(worker_name(i), - fn, - args=args, - kwargs=kwargs) + res = dist.rpc.rpc_sync( + worker_name(i), fn, args=args, kwargs=kwargs + ) queue.put(res) dist.rpc.shutdown() @@ -109,16 +108,14 @@ def run_rpc_async_master_working( ) if dist.get_rank() == 0: for i in range(1, dist.get_rank()): - res = dist.rpc.rpc_async(worker_name(i), - fn, - args=args, - kwargs=kwargs) + res = dist.rpc.rpc_async( + worker_name(i), fn, args=args, kwargs=kwargs + ) queue.put(res.wait()) dist.rpc.shutdown() class RpcTestBase(unittest.TestCase): - def setUp(self): self._port_set = set() print("RPC setUp...") @@ -129,10 +126,10 @@ class RpcTestBase(unittest.TestCase): print("RPC tearDown...") def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(("", 0)) return s.getsockname()[1] @@ -162,7 +159,8 @@ class RpcTestBase(unittest.TestCase): fn_args, fn_kwargs, ), - )) + ) + ) else: self.processes.append( Process( @@ -176,13 +174,13 @@ class RpcTestBase(unittest.TestCase): fn_args, fn_kwargs, ), - )) + ) + ) [p.start() for p in self.processes] return queues class RpcLaunchTestBase(unittest.TestCase): - def setUp(self): self._port_set = set() print("Launch RPC setUp...") @@ -192,10 +190,10 @@ class RpcLaunchTestBase(unittest.TestCase): print("Launch RPC tearDown...") def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(("", 0)) return s.getsockname()[1] @@ -221,8 +219,8 @@ class RpcLaunchTestBase(unittest.TestCase): for i in range(nnodes * nproc_per_node): a = np.random.random((10, 100)).astype(np.float32) b = np.random.random((10, 100)).astype(np.float32) - mmap_data1[i * 10:(i + 1) * 10, :] = a - mmap_data2[i * 10:(i + 1) * 10, :] = b + mmap_data1[i * 10 : (i + 1) * 10, :] = a + mmap_data2[i * 10 : (i + 1) * 10, :] = b return mmap_data1, mmap_data2 def remove_data(self): @@ -234,8 +232,15 @@ class RpcLaunchTestBase(unittest.TestCase): log_dir = "log" tr_cmd = "python -m paddle.distributed.launch --master {} --rank {} --nnodes {} --nproc_per_node {} --run_mode rpc {} --log_dir {}" cmds = [ - tr_cmd.format(master_endpoint, rank, nnodes, nproc_per_node, - model_file, log_dir) for rank in range(nnodes) + tr_cmd.format( + master_endpoint, + rank, + nnodes, + nproc_per_node, + model_file, + log_dir, + ) + for rank in range(nnodes) ] processes = [subprocess.Popen(cmd.strip().split()) for cmd in cmds] [proc.communicate() for proc in processes] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_concat.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_concat.py index 4b60a0cbcfc..80c975221ad 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_concat.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_concat.py @@ -24,7 +24,6 @@ from paddle import fluid class TestSequenceConcat(OpTest): - def setLoD(self): self.lod1 = [7, 3] self.lod2 = [12, 8] @@ -35,8 +34,14 @@ class TestSequenceConcat(OpTest): x2 = np.random.random(size=(20, 80)).astype('float64') self.setLoD() - out = np.concatenate((x1[0:self.lod1[0]], x2[0:self.lod2[0]], - x1[self.lod1[0]:], x2[self.lod2[0]:])) + out = np.concatenate( + ( + x1[0 : self.lod1[0]], + x2[0 : self.lod2[0]], + x1[self.lod1[0] :], + x2[self.lod2[0] :], + ) + ) self.op_type = "sequence_concat" self.inputs = { @@ -52,7 +57,6 @@ class TestSequenceConcat(OpTest): class TestSequenceConcatCase2(TestSequenceConcat): - def setLoD(self): self.lod1 = [10, 0] self.lod2 = [12, 8] @@ -60,7 +64,6 @@ class TestSequenceConcatCase2(TestSequenceConcat): class TestSequenceConcatCase3(TestSequenceConcat): - def setLoD(self): self.lod1 = [10, 0] self.lod2 = [20, 0] @@ -68,7 +71,6 @@ class TestSequenceConcatCase3(TestSequenceConcat): class TestSequenceConcatCase4(TestSequenceConcat): - def setLoD(self): self.lod1 = [0, 10] self.lod2 = [0, 20] @@ -76,7 +78,6 @@ class TestSequenceConcatCase4(TestSequenceConcat): class TestSequenceConcatCase5(TestSequenceConcat): - def setLoD(self): self.lod1 = [0, 10] self.lod2 = [20, 0] @@ -84,9 +85,7 @@ class TestSequenceConcatCase5(TestSequenceConcat): class TestSequenceConcatOpError(unittest.TestCase): - def test_errors(self): - def test_input_list(): # the input type must be list x_data = fluid.layers.data(name='x', shape=[4], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_conv.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_conv.py index 047ec6e2943..78fc6089cfe 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_conv.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_conv.py @@ -21,13 +21,15 @@ sys.path.append("../") from op_test import OpTest -def seqconv(x, - lod, - filter, - context_length, - context_start, - padding_trainable=False, - padding_data=None): +def seqconv( + x, + lod, + filter, + context_length, + context_start, + padding_trainable=False, + padding_data=None, +): [T, M] = x.shape col = np.zeros((T, context_length * M)).astype('float32') offset = [0] @@ -42,58 +44,79 @@ def seqconv(x, out_end = offset[i + 1] if in_begin < offset[i]: pad_size = np.min( - [offset[i] - in_begin, offset[i + 1] - offset[i]]) + [offset[i] - in_begin, offset[i + 1] - offset[i]] + ) if padding_trainable: - sub_w = padding_data[j:j + pad_size, :] - col[offset[i]:offset[i] + pad_size, - j * M:(j + 1) * M] = sub_w + sub_w = padding_data[j : j + pad_size, :] + col[ + offset[i] : offset[i] + pad_size, j * M : (j + 1) * M + ] = sub_w out_begin = offset[i] + pad_size in_begin = offset[i] if in_end > offset[i + 1]: pad_size = np.min( - [in_end - offset[i + 1], offset[i + 1] - offset[i]]) + [in_end - offset[i + 1], offset[i + 1] - offset[i]] + ) if padding_trainable: - sub_w = padding_data[begin_pad + context_start + j - - pad_size:begin_pad + context_start + - j, :] - col[offset[i + 1] - pad_size:offset[i + 1], - j * M:(j + 1) * M] = sub_w + sub_w = padding_data[ + begin_pad + + context_start + + j + - pad_size : begin_pad + + context_start + + j, + :, + ] + col[ + offset[i + 1] - pad_size : offset[i + 1], + j * M : (j + 1) * M, + ] = sub_w in_end = offset[i + 1] out_end = offset[i + 1] - pad_size if in_end <= in_begin: continue in_sub = x[in_begin:in_end, :] - col[out_begin:out_end, j * M:(j + 1) * M] += in_sub + col[out_begin:out_end, j * M : (j + 1) * M] += in_sub return np.dot(col, filter) class TestSeqProject(OpTest): - def setUp(self): self.init_test_case() self.op_type = 'sequence_conv' - if self.context_length == 1 \ - and self.context_start == 0 \ - and self.padding_trainable: - print("If context_start is 0 " \ - "and context_length is 1," \ - " padding_trainable should be false.") + if ( + self.context_length == 1 + and self.context_start == 0 + and self.padding_trainable + ): + print( + "If context_start is 0 " + "and context_length is 1," + " padding_trainable should be false." + ) return # one level, batch size x = np.random.uniform( - 0.1, 1, [self.input_size[0], self.input_size[1]]).astype('float32') - w = np.random.uniform(0.1, 1, [ - self.context_length * self.input_size[1], self.output_represention - ]).astype('float32') + 0.1, 1, [self.input_size[0], self.input_size[1]] + ).astype('float32') + w = np.random.uniform( + 0.1, + 1, + [ + self.context_length * self.input_size[1], + self.output_represention, + ], + ).astype('float32') begin_pad = np.max([0, -self.context_start]) end_pad = np.max([0, self.context_start + self.context_length - 1]) total_pad = begin_pad + end_pad padding_data = np.random.uniform( - 0.1, 1, [total_pad, self.input_size[1]]).astype('float32') + 0.1, 1, [total_pad, self.input_size[1]] + ).astype('float32') self.pad_data = padding_data self.inputs = { 'X': (x, self.lod), @@ -113,10 +136,17 @@ class TestSeqProject(OpTest): 'contextStart': self.context_start, 'contextLength': self.context_length, 'paddingTrainable': self.padding_trainable, - 'contextStride': self.context_stride + 'contextStride': self.context_stride, } - out = seqconv(x, self.lod, w, self.context_length, self.context_start, - self.padding_trainable, self.pad_data) + out = seqconv( + x, + self.lod, + w, + self.context_length, + self.context_start, + self.padding_trainable, + self.pad_data, + ) self.outputs = {'Out': out} def test_check_output(self): @@ -124,48 +154,58 @@ class TestSeqProject(OpTest): def test_check_grad(self): if self.padding_trainable: - self.check_grad(set(self.inputs_val), - 'Out', - max_relative_error=0.05) + self.check_grad( + set(self.inputs_val), 'Out', max_relative_error=0.05 + ) def test_check_grad_input(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.05, - no_grad_set=set(self.inputs_val_no_x)) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.05, + no_grad_set=set(self.inputs_val_no_x), + ) def test_check_grad_padding_data(self): if self.padding_trainable: - self.check_grad(['PaddingData'], - 'Out', - no_grad_set=set(['X', 'Filter'])) + self.check_grad( + ['PaddingData'], 'Out', no_grad_set=set(['X', 'Filter']) + ) def test_check_grad_Filter(self): - self.check_grad(['Filter'], - 'Out', - max_relative_error=0.05, - no_grad_set=set(self.inputs_val_no_f)) + self.check_grad( + ['Filter'], + 'Out', + max_relative_error=0.05, + no_grad_set=set(self.inputs_val_no_f), + ) def test_check_grad_input_filter(self): if self.padding_trainable: - self.check_grad(['X', 'Filter'], - 'Out', - max_relative_error=0.05, - no_grad_set=set(['PaddingData'])) + self.check_grad( + ['X', 'Filter'], + 'Out', + max_relative_error=0.05, + no_grad_set=set(['PaddingData']), + ) def test_check_grad_padding_input(self): if self.padding_trainable: - self.check_grad(self.inputs_val_no_f, - 'Out', - max_relative_error=0.05, - no_grad_set=set(['Filter'])) + self.check_grad( + self.inputs_val_no_f, + 'Out', + max_relative_error=0.05, + no_grad_set=set(['Filter']), + ) def test_check_grad_padding_filter(self): if self.padding_trainable: - self.check_grad(self.inputs_val_no_x, - 'Out', - max_relative_error=0.05, - no_grad_set=set(['X'])) + self.check_grad( + self.inputs_val_no_x, + 'Out', + max_relative_error=0.05, + no_grad_set=set(['X']), + ) def init_test_case(self): self.input_row = 11 @@ -184,7 +224,6 @@ class TestSeqProject(OpTest): class TestSeqProjectCase1(TestSeqProject): - def init_test_case(self): self.input_row = 11 self.context_start = -1 @@ -202,7 +241,6 @@ class TestSeqProjectCase1(TestSeqProject): class TestSeqProjectCase2Len0(TestSeqProject): - def init_test_case(self): self.input_row = 11 self.context_start = -1 @@ -220,7 +258,6 @@ class TestSeqProjectCase2Len0(TestSeqProject): class TestSeqProjectCase3(TestSeqProject): - def init_test_case(self): self.input_row = 25 self.context_start = 2 @@ -231,8 +268,9 @@ class TestSeqProjectCase3(TestSeqProject): self.input_size = [self.input_row, 25] idx = list(range(self.input_size[0])) del idx[0] - offset_lod = [[0] + np.sort(random.sample(idx, 8)).tolist() + - [self.input_size[0]]] + offset_lod = [ + [0] + np.sort(random.sample(idx, 8)).tolist() + [self.input_size[0]] + ] self.lod = [[]] # convert from offset-based lod to length-based lod for i in range(len(offset_lod[0]) - 1): @@ -241,19 +279,18 @@ class TestSeqProjectCase3(TestSeqProject): class TestSeqConvApi(unittest.TestCase): - def test_api(self): import paddle.fluid as fluid x = fluid.layers.data('x', shape=[32], lod_level=1) - y = fluid.layers.sequence_conv(input=x, - num_filters=2, - filter_size=3, - padding_start=None) + y = fluid.layers.sequence_conv( + input=x, num_filters=2, filter_size=3, padding_start=None + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(10, 32).astype("float32"), [[2, 3, 1, 4]], place) + np.random.rand(10, 32).astype("float32"), [[2, 3, 1, 4]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) ret = exe.run(feed={'x': x_tensor}, fetch_list=[y], return_numpy=False) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_enumerate_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_enumerate_op.py index 68c6d7d7535..4f75fc08bef 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_enumerate_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_enumerate_op.py @@ -30,15 +30,15 @@ def sequence_enumerate(input_seq, in_lod, win_size, pad_value): single_seq = [] for word_idx in range(win_size): word_pos = idx + word_idx - dat = input_seq[word_pos] if word_pos < lod0[i+1] \ - else pad_value + dat = ( + input_seq[word_pos] if word_pos < lod0[i + 1] else pad_value + ) single_seq.append(dat) out_seq.append(single_seq) return out_seq class TestSequenceEnumerateOp(OpTest): - def setUp(self): self.op_type = "sequence_enumerate" self.init_test_case() @@ -54,68 +54,69 @@ class TestSequenceEnumerateOp(OpTest): self.lod = [[9, 4, 11, 6]] self.win_size = 2 self.pad_value = 0 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") class TesSequenceEnumerateOpInt64(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int64") self.lod = [[9, 4, 11, 6]] self.win_size = 2 self.pad_value = 0 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int64") class TestSequenceEnumerateOpLargeWinSize(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") self.lod = [[9, 4, 11, 6]] self.win_size = 5 self.pad_value = 0 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") class TestSequenceEnumerateOpMaxWinSize(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") self.lod = [[9, 4, 11, 6]] self.win_size = 30 self.pad_value = 0 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") class TestSequenceEnumerateOpLargePadValue(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") self.lod = [[9, 4, 11, 6]] self.win_size = 5 self.pad_value = 5 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") class TestSequenceEnumerateOpLargePadValueSeqLen0(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") self.lod = [[0, 14, 0, 16, 0]] self.win_size = 5 self.pad_value = 5 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_erase_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_erase_op.py index 99d72e11ff1..4c4ea068117 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_erase_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_erase_op.py @@ -26,7 +26,7 @@ def sequence_erase(in_seq, lod0, tokens): offset = 0 for i in range(0, len(lod0)): num_out = 0 - for dat in in_seq[offset:(offset + lod0[i])]: + for dat in in_seq[offset : (offset + lod0[i])]: if dat not in tokens: out_seq.append(dat) num_out += 1 @@ -36,7 +36,6 @@ def sequence_erase(in_seq, lod0, tokens): class TestSequenceEraseOpInt32(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") @@ -52,7 +51,6 @@ class TestSequenceEraseOpInt32(OpTest): class TestSequenceEraseOpInt32LoD2(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") @@ -68,7 +66,6 @@ class TestSequenceEraseOpInt32LoD2(OpTest): class TestSequenceEraseOpInt64(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int64") @@ -84,7 +81,6 @@ class TestSequenceEraseOpInt64(OpTest): class TestSequenceEraseOpInt64SeqLen0(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int64") @@ -100,7 +96,6 @@ class TestSequenceEraseOpInt64SeqLen0(OpTest): class TestSequenceEraseOpEmpty(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand.py index 3d6f66f2a55..4144812fb68 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand.py @@ -21,7 +21,6 @@ from op_test import OpTest class TestSequenceExpand(OpTest): - def set_data(self): x_data = np.random.uniform(0.1, 1, [3, 40]).astype('float64') y_data = np.random.uniform(0.1, 1, [8, 1]).astype('float64') @@ -38,7 +37,7 @@ class TestSequenceExpand(OpTest): else: ref_level = len(y_lod) - 1 - out = np.zeros(shape=((0, ) + x_data.shape[1:]), dtype=x_data.dtype) + out = np.zeros(shape=((0,) + x_data.shape[1:]), dtype=x_data.dtype) if x_lod is None: # x_idx = [i for i in xrange(x_data.shape[0] + 1)] @@ -53,7 +52,7 @@ class TestSequenceExpand(OpTest): x_len = x_idx[i] if repeat_num > 0: - x_sub = x_data[offset:(offset + x_len), :] + x_sub = x_data[offset : (offset + x_len), :] stacked_x_sub = x_sub for r in range(repeat_num - 1): stacked_x_sub = np.vstack((stacked_x_sub, x_sub)) @@ -81,7 +80,6 @@ class TestSequenceExpand(OpTest): class TestSequenceExpandCase1(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [5, 20]).astype('float64') y_data = np.random.uniform(0.1, 1, [13, 1]).astype('float64') @@ -91,7 +89,6 @@ class TestSequenceExpandCase1(TestSequenceExpand): class TestSequenceExpandCase2(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [1, 2, 50]).astype('float64') x_lod = [[1]] @@ -102,7 +99,6 @@ class TestSequenceExpandCase2(TestSequenceExpand): class TestSequenceExpandCase3(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [4, 25]).astype('float64') x_lod = [[1, 1, 1, 1]] @@ -112,7 +108,6 @@ class TestSequenceExpandCase3(TestSequenceExpand): class TestSequenceExpandCase4(TestSequenceExpand): - def set_data(self): data = np.random.uniform(0.1, 1, [5 * 20, 1]) x_data = np.array(data).reshape([5, 20]).astype('float64') @@ -123,7 +118,6 @@ class TestSequenceExpandCase4(TestSequenceExpand): class TestSequenceExpandCase5(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [6, 20]).astype('float64') y_data = np.random.uniform(0.1, 1, [13, 1]).astype('float64') @@ -133,7 +127,6 @@ class TestSequenceExpandCase5(TestSequenceExpand): class TestSequenceExpandCase6(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [4, 25]).astype('float64') x_lod = [[1, 1, 0, 1, 1]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand_as.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand_as.py index 00b6ca518e8..b1d6bc915ec 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand_as.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand_as.py @@ -23,7 +23,6 @@ from paddle.fluid import Program, program_guard class TestSequenceExpandAs(OpTest): - def setUp(self): self.op_type = 'sequence_expand_as' self.set_data() @@ -60,7 +59,6 @@ class TestSequenceExpandAs(OpTest): class TestSequenceExpandAsCase1(TestSequenceExpandAs): - def set_data(self): x_data = np.random.uniform(0.1, 1, [5, 20]).astype('float64') x_lod = [[2, 3]] @@ -70,7 +68,6 @@ class TestSequenceExpandAsCase1(TestSequenceExpandAs): class TestSequenceExpandAsCase2(TestSequenceExpandAs): - def set_data(self): x_data = np.random.uniform(0.1, 1, [5, 20]).astype('float64') x_lod = [[2, 3]] @@ -80,7 +77,6 @@ class TestSequenceExpandAsCase2(TestSequenceExpandAs): class TestSequenceExpandAsCase3(TestSequenceExpandAs): - def set_data(self): x_data = np.random.uniform(0.1, 1, [1, 2, 50]).astype('float64') x_lod = [[1]] @@ -90,7 +86,6 @@ class TestSequenceExpandAsCase3(TestSequenceExpandAs): class TestSequenceExpandAsOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input x must be Variable diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_first_step.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_first_step.py index 25589f2ad9b..a7771634980 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_first_step.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_first_step.py @@ -21,7 +21,6 @@ sys.path.append("../") class TestSequenceFirstStepOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -34,11 +33,13 @@ class TestSequenceFirstStepOpError(unittest.TestCase): def test_input_dtype(): # the dtype of input must be int64 - type_data = fluid.layers.data(name='type_data', - shape=[7, 1], - append_batch_size=False, - dtype='int64', - lod_level=1) + type_data = fluid.layers.data( + name='type_data', + shape=[7, 1], + append_batch_size=False, + dtype='int64', + lod_level=1, + ) fluid.layers.sequence_last_step(type_data) self.assertRaises(TypeError, test_input_dtype) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_last_step.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_last_step.py index 0ddc348044d..ca5ff76a8e4 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_last_step.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_last_step.py @@ -21,7 +21,6 @@ sys.path.append("../") class TestSequenceLastStepOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -34,11 +33,13 @@ class TestSequenceLastStepOpError(unittest.TestCase): def test_input_dtype(): # the dtype of input must be int64 - type_data = fluid.layers.data(name='type_data', - shape=[7, 1], - append_batch_size=False, - dtype='int64', - lod_level=1) + type_data = fluid.layers.data( + name='type_data', + shape=[7, 1], + append_batch_size=False, + dtype='int64', + lod_level=1, + ) fluid.layers.sequence_last_step(type_data) self.assertRaises(TypeError, test_input_dtype) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_mask.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_mask.py index f240bc99004..24210c318c2 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_mask.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_mask.py @@ -13,7 +13,11 @@ # limitations under the License. import paddle.fluid as fluid -from paddle.fluid.framework import convert_np_dtype_to_dtype_, Program, program_guard +from paddle.fluid.framework import ( + convert_np_dtype_to_dtype_, + Program, + program_guard, +) import numpy as np import unittest import sys @@ -23,7 +27,6 @@ from op_test import OpTest class SequenceMaskTestBase(OpTest): - def initDefaultParameters(self): self.op_type = 'sequence_mask' self.maxlen = 10 @@ -43,19 +46,19 @@ class SequenceMaskTestBase(OpTest): self.outputs = {'Y': self.calc_ground_truth_mask()} self.attrs = { 'maxlen': self.maxlen, - 'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype) + 'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype), } def calc_ground_truth_mask(self): maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen - shape = self.x.shape + (maxlen, ) - index_broadcast = np.broadcast_to(np.reshape( - range(maxlen), newshape=[1] * self.x.ndim + [-1]), - shape=shape) - x_broadcast = np.broadcast_to(np.reshape(self.x, - newshape=self.x.shape + - (-1, )), - shape=shape) + shape = self.x.shape + (maxlen,) + index_broadcast = np.broadcast_to( + np.reshape(range(maxlen), newshape=[1] * self.x.ndim + [-1]), + shape=shape, + ) + x_broadcast = np.broadcast_to( + np.reshape(self.x, newshape=self.x.shape + (-1,)), shape=shape + ) return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): @@ -63,43 +66,36 @@ class SequenceMaskTestBase(OpTest): class SequenceMaskTest1(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'bool' class SequenceMaskTest2(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'uint8' class SequenceMaskTest3(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'int32' class SequenceMaskTest4(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'float32' class SequenceMaskTest5(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'float64' class SequenceMaskTest6(SequenceMaskTestBase): - def initParameters(self): self.maxlen = -1 class SequenceMaskTestBase_tensor_attr(OpTest): - def initDefaultParameters(self): self.op_type = 'sequence_mask' self.maxlen = 10 @@ -122,14 +118,14 @@ class SequenceMaskTestBase_tensor_attr(OpTest): def calc_ground_truth_mask(self): maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen - shape = self.x.shape + (maxlen, ) - index_broadcast = np.broadcast_to(np.reshape( - range(maxlen), newshape=[1] * self.x.ndim + [-1]), - shape=shape) - x_broadcast = np.broadcast_to(np.reshape(self.x, - newshape=self.x.shape + - (-1, )), - shape=shape) + shape = self.x.shape + (maxlen,) + index_broadcast = np.broadcast_to( + np.reshape(range(maxlen), newshape=[1] * self.x.ndim + [-1]), + shape=shape, + ) + x_broadcast = np.broadcast_to( + np.reshape(self.x, newshape=self.x.shape + (-1,)), shape=shape + ) return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): @@ -137,37 +133,31 @@ class SequenceMaskTestBase_tensor_attr(OpTest): class SequenceMaskTest1_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'bool' class SequenceMaskTest2_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'uint8' class SequenceMaskTest3_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'int32' class SequenceMaskTest4_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'float32' class SequenceMaskTest5_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'float64' class TestSequenceMaskOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.uniform(1, 5, [4]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_pad_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_pad_op.py index 934e0ebe8fd..d04091d9332 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_pad_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_pad_op.py @@ -24,7 +24,6 @@ import paddle.fluid.core as core class TestSequencePadOp(OpTest): - def set_attr(self): self.x_shape = [12, 10] self.x_len_lod = [[2, 3, 4, 3]] @@ -37,7 +36,7 @@ class TestSequencePadOp(OpTest): pad_value_data = np.array(self.pad_value).astype(self.dtype) self.inputs = { 'X': (x_data, self.x_len_lod), - 'PadValue': pad_value_data + 'PadValue': pad_value_data, } self.attrs = {'padded_length': self.padded_length} @@ -54,9 +53,10 @@ class TestSequencePadOp(OpTest): # do padding x_data = self.inputs['X'][0] pad_value_data = self.inputs['PadValue'] - if pad_value_data.shape == (1, ): - pad_value_data = np.broadcast_to(pad_value_data, - shape=x_data.shape[1:]) + if pad_value_data.shape == (1,): + pad_value_data = np.broadcast_to( + pad_value_data, shape=x_data.shape[1:] + ) padded_sequences = [] start_idx = 0 for l in x_len_lod_0: @@ -86,7 +86,6 @@ class TestSequencePadOp(OpTest): class TestSequencePadOp2(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 10] self.x_len_lod = [[2, 3, 4, 3]] @@ -96,7 +95,6 @@ class TestSequencePadOp2(TestSequencePadOp): class TestSequencePadOp3(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 10] self.x_len_lod = [[2, 3, 4, 3]] @@ -106,7 +104,6 @@ class TestSequencePadOp3(TestSequencePadOp): class TestSequencePadOp4(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 10] self.x_len_lod = [[2, 3, 4, 3]] @@ -116,7 +113,6 @@ class TestSequencePadOp4(TestSequencePadOp): class TestSequencePadOp5(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 2, 5] self.x_len_lod = [[2, 3, 4, 3]] @@ -126,7 +122,6 @@ class TestSequencePadOp5(TestSequencePadOp): class TestSequencePadOp6(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 2, 5] self.x_len_lod = [[2, 3, 4, 3]] @@ -136,7 +131,6 @@ class TestSequencePadOp6(TestSequencePadOp): class TestSequencePadOp7(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 2, 5] self.x_len_lod = [[2, 3, 4, 3]] @@ -146,7 +140,6 @@ class TestSequencePadOp7(TestSequencePadOp): class TestSequencePadOp8(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 2, 5] self.x_len_lod = [[0, 8, 0, 4, 0]] @@ -156,35 +149,33 @@ class TestSequencePadOp8(TestSequencePadOp): class TestSequencePadOpError(unittest.TestCase): - def test_error(self): - def test_x_variable(): # the input x type must be Variable x = np.random.random((2, 4)).astype("float32") pad_value = fluid.layers.assign( - input=np.array([0.0], dtype=np.float32)) + input=np.array([0.0], dtype=np.float32) + ) fluid.layers.sequence_pad(x=x, pad_value=pad_value) self.assertRaises(TypeError, test_x_variable) def test_pad_value_variable(): - x1 = fluid.layers.data(name='x1', - shape=[10, 5], - dtype='float32', - lod_level=1) + x1 = fluid.layers.data( + name='x1', shape=[10, 5], dtype='float32', lod_level=1 + ) pad_value1 = np.array([0.0], dtype=np.float32) fluid.layers.sequence_pad(x=x1, pad_value=pad_value1) self.assertRaises(TypeError, test_pad_value_variable) def test_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[10, 5], - dtype='int16', - lod_level=1) + x2 = fluid.layers.data( + name='x2', shape=[10, 5], dtype='int16', lod_level=1 + ) pad_value2 = fluid.layers.assign( - input=np.array([0.0], dtype=np.int32)) + input=np.array([0.0], dtype=np.int32) + ) fluid.layers.sequence_pad(x=x2, pad_value=pad_value2) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_pool.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_pool.py index ee4f4fa0860..b4af6f239df 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_pool.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_pool.py @@ -27,7 +27,7 @@ def compute_seqpool_sum(x, offset, out, pad_value=0.0): if offset[level][i] == offset[level][i + 1]: out[i] = pad_value else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = sub_x.sum(axis=0) @@ -37,7 +37,7 @@ def compute_seqpool_avg(x, offset, out, pad_value=0.0): if offset[level][i] == offset[level][i + 1]: out[i] = pad_value else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = sub_x.mean(axis=0) @@ -47,13 +47,12 @@ def compute_seqpool_sqrt(x, offset, out, pad_value=0.0): if offset[level][i] == offset[level][i + 1]: out[i] = pad_value else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] seq_len = offset[level][i + 1] - offset[level][i] out[i] = sub_x.sum(axis=0) / np.sqrt(seq_len) class TestSeqAvgPool(OpTest): - def set_lod(self): return [[11]] @@ -88,69 +87,63 @@ class TestSeqAvgPool(OpTest): def test_check_grad(self): # Remove MaxIndex after check_grad is refined. out = self.outputs['Out'] - if isinstance(out, tuple): out = out[0] - self.outputs['MaxIndex'] = \ - np.zeros(out.shape).astype('int32') + if isinstance(out, tuple): + out = out[0] + self.outputs['MaxIndex'] = np.zeros(out.shape).astype('int32') self.check_grad(["X"], "Out", check_dygraph=False) class TestSeqAvgPoolBatch1(TestSeqAvgPool): - def set_lod(self): return [[11]] def set_lod_data(self): lod = self.set_lod() - x, _ = self.get_sequence_batch_size_1_input(lod=lod, - shape=[lod[0][0], 23]) + x, _ = self.get_sequence_batch_size_1_input( + lod=lod, shape=[lod[0][0], 23] + ) return x class TestSeqAvgPoolInstance0(TestSeqAvgPool): - def set_lod(self): return [[0, 0, 4, 0, 3, 0, 0, 5, 0, 0]] def set_lod_data(self): lod = self.set_lod() - x, _ = self.get_sequence_instance_size_0_input(lod=lod, - shape=[sum(lod[0]), 10]) + x, _ = self.get_sequence_instance_size_0_input( + lod=lod, shape=[sum(lod[0]), 10] + ) return x class TestSeqAvgPoolLen0(TestSeqAvgPool): - def set_lod(self): return [[0, 4, 0, 7, 0]] class TestSeqAvgPoolLen0LoDLevel2(TestSeqAvgPool): - def set_lod(self): return [[2, 0, 1, 2], [0, 4, 0, 7, 0]] class TestSeqSumPool(TestSeqAvgPool): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.1, 'pooltype': "SUM"} compute_seqpool_sum(x, offset, out, self.attrs["pad_value"]) class TestSeqSumPoolLen0(TestSeqSumPool): - def set_lod(self): return [[0, 4, 0, 7, 0]] class TestSeqSumPoolLen0LoDLevel2(TestSeqSumPool): - def set_lod(self): return [[2, 0, 1, 2], [0, 4, 0, 7, 0]] class TestSeqMaxPool(TestSeqAvgPool): - def set_lod(self): return [[13]] @@ -178,43 +171,37 @@ class TestSeqMaxPool(TestSeqAvgPool): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = np.amax(sub_x, axis=0) class TestSeqMaxPoolLen0(TestSeqMaxPool): - def set_lod(self): return [[0, 1, 1, 5, 6, 0]] class TestSeqMaxPoolLen0LoDLevel2(TestSeqMaxPool): - def set_lod(self): return [[2, 0, 3, 1], [0, 1, 1, 5, 6, 0]] class TestSeqSqrtPool(TestSeqAvgPool): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "SQRT"} compute_seqpool_sqrt(x, offset, out, self.attrs["pad_value"]) class TestSeqSqrtPoolLen0(TestSeqSqrtPool): - def set_lod(self): return [[0, 7, 0, 2, 2, 0]] class TestSeqSqrtPoolLen0LoDLevel2(TestSeqSqrtPool): - def set_lod(self): return [[1, 2, 0, 3], [0, 7, 0, 2, 2, 0]] class TestSeqLastPool(TestSeqAvgPool): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "LAST"} level = len(offset) - 1 @@ -222,24 +209,21 @@ class TestSeqLastPool(TestSeqAvgPool): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = sub_x[-1, :] class TestSeqLastPoolLen0(TestSeqLastPool): - def set_lod(self): return [[0, 3, 4, 0, 4, 0]] class TestSeqLastPoolLen0LoDLevel2(TestSeqLastPool): - def set_lod(self): return [[1, 0, 2, 3], [0, 3, 4, 0, 4, 0]] class TestSeqFirstPool(TestSeqAvgPool): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.3, 'pooltype': "FIRST"} level = len(offset) - 1 @@ -247,24 +231,21 @@ class TestSeqFirstPool(TestSeqAvgPool): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = sub_x[0, :] class TestSeqFirstPoolLen0(TestSeqFirstPool): - def set_lod(self): return [[0, 2, 0, 3, 6, 0]] class TestSeqFirstPoolLen0LoDLevel2(TestSeqFirstPool): - def set_lod(self): return [[1, 0, 2, 3], [0, 2, 0, 3, 6, 0]] class TestSeqAvgPool2D(TestSeqAvgPool): - def set_lod(self): return [[4, 1, 3, 5]] @@ -287,25 +268,23 @@ class TestSeqAvgPool2D(TestSeqAvgPool): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) out[i] = np.reshape(sub_x.mean(axis=0), (3, 17)) class TestSeqAvgPool2DLen0(TestSeqAvgPool2D): - def set_lod(self): return [[0, 5, 0, 8, 0]] class TestSeqAvgPool2DLen0LoDLevel2(TestSeqAvgPool2D): - def set_lod(self): return [[1, 0, 4], [0, 5, 0, 8, 0]] class TestSeqSumPool2D(TestSeqAvgPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.2, 'pooltype': "SUM"} level = len(offset) - 1 @@ -313,25 +292,23 @@ class TestSeqSumPool2D(TestSeqAvgPool2D): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) out[i] = np.reshape(sub_x.sum(axis=0), (3, 17)) class TestSeqSumPool2DLen0(TestSeqSumPool2D): - def set_lod(self): return [[0, 8, 0, 5, 0]] class TestSeqSumPool2DLen0LoDLevel2(TestSeqSumPool2D): - def set_lod(self): return [[1, 0, 4], [0, 8, 0, 5, 0]] class TestSeqSqrtPool2D(TestSeqAvgPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "SQRT"} level = len(offset) - 1 @@ -339,39 +316,36 @@ class TestSeqSqrtPool2D(TestSeqAvgPool2D): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) seq_len = offset[level][i + 1] - offset[level][i] out[i] = np.reshape( - sub_x.sum(axis=0) / np.sqrt(seq_len), (3, 17)) + sub_x.sum(axis=0) / np.sqrt(seq_len), (3, 17) + ) def test_check_grad(self): # Remove MaxIndex after check_grad is refined. out = self.outputs['Out'] if isinstance(out, tuple): out = out[0] - self.outputs['MaxIndex'] = \ - np.zeros(out.shape).astype('int32') - self.check_grad(["X"], - "Out", - max_relative_error=0.06, - check_dygraph=False) + self.outputs['MaxIndex'] = np.zeros(out.shape).astype('int32') + self.check_grad( + ["X"], "Out", max_relative_error=0.06, check_dygraph=False + ) class TestSeqSqrtPool2DLen0(TestSeqSqrtPool2D): - def set_lod(self): return [[0, 8, 0, 5, 0]] class TestSeqSqrtPool2DLen0LoDLevel2(TestSeqSqrtPool2D): - def set_lod(self): return [[1, 0, 2, 2], [0, 8, 0, 5, 0]] class TestSeqMaxPool2D(TestSeqAvgPool2D): - def set_lod(self): return [[4, 1, 3, 5]] @@ -399,27 +373,27 @@ class TestSeqMaxPool2D(TestSeqAvgPool2D): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 11)) continue - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 11)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 11) + ) out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 11)) class TestSeqMaxPool2DLen0(TestSeqMaxPool2D): - def set_lod(self): return [[0, 3, 0, 10, 0]] class TestSeqMaxPool2DLen0LoDLevel2(TestSeqMaxPool2D): - def set_lod(self): return [[1, 0, 2, 2], [0, 3, 0, 10, 0]] -@skip_check_grad_ci(reason="Grad computation does not apply to Sequence MAX " - "Pool executed when is_test is true.") +@skip_check_grad_ci( + reason="Grad computation does not apply to Sequence MAX " + "Pool executed when is_test is true." +) class TestSeqMaxPool2DInference(TestSeqMaxPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 1.0, 'pooltype': "MAX", 'is_test': True} level = len(offset) - 1 @@ -427,30 +401,28 @@ class TestSeqMaxPool2DInference(TestSeqMaxPool2D): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 11)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 11)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 11) + ) out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 11)) def test_check_grad(self): """Grad computation does not apply to Sequence MAX - Pool executed when is_test is true """ + Pool executed when is_test is true""" return class TestSeqMaxPool2DInferenceLen0(TestSeqMaxPool2DInference): - def set_lod(self): return [[0, 3, 0, 10, 0]] class TestSeqMaxPool2DInferenceLen0LoDLevel2(TestSeqMaxPool2DInference): - def set_lod(self): return [[1, 0, 2, 2], [0, 3, 0, 10, 0]] class TestSeqLastPool2D(TestSeqAvgPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "LAST"} level = len(offset) - 1 @@ -458,25 +430,23 @@ class TestSeqLastPool2D(TestSeqAvgPool2D): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) out[i] = np.reshape(sub_x[-1, :], (3, 17)) class TestSeqLastPool2DLen0(TestSeqLastPool2D): - def set_lod(self): return [[0, 3, 0, 1, 9, 0]] class TestSeqLastPool2DLen0LoDLevel2(TestSeqLastPool2D): - def set_lod(self): return [[1, 0, 2, 3], [0, 3, 0, 1, 9, 0]] class TestSeqFirstPool2D(TestSeqAvgPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "FIRST"} level = len(offset) - 1 @@ -484,19 +454,18 @@ class TestSeqFirstPool2D(TestSeqAvgPool2D): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) out[i] = np.reshape(sub_x[0, :], (3, 17)) class TestSeqFirstPool2DLen0(TestSeqFirstPool2D): - def set_lod(self): return [[0, 3, 0, 3, 7, 0]] class TestSeqFirstPool2DLen0LoDLevel2(TestSeqFirstPool2D): - def set_lod(self): return [[1, 0, 2, 3], [0, 3, 0, 3, 7, 0]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_reshape.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_reshape.py index 5577ad052c0..ea1b5bbf916 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_reshape.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_reshape.py @@ -23,7 +23,6 @@ import paddle.fluid as fluid class TestSequenceReshape(OpTest): - def init_data(self): self.dimension = 12 self.x_lod = [[4, 1, 3, 3]] @@ -57,7 +56,6 @@ class TestSequenceReshape(OpTest): class TestSequenceReshape_reduce(TestSequenceReshape): - def init_data(self): self.dimension = 24 self.x_lod = [[4, 2, 2, 4]] @@ -65,7 +63,6 @@ class TestSequenceReshape_reduce(TestSequenceReshape): class TestSequenceReshape_same(TestSequenceReshape): - def init_data(self): self.dimension = 12 self.x_lod = [[4, 2, 2, 4]] @@ -73,7 +70,6 @@ class TestSequenceReshape_same(TestSequenceReshape): class TestSequenceReshape_reduce_seq_len0(TestSequenceReshape): - def init_data(self): self.dimension = 24 self.x_lod = [[0, 6, 0, 2, 4]] @@ -81,7 +77,6 @@ class TestSequenceReshape_reduce_seq_len0(TestSequenceReshape): class TestSequenceReshape_reduce_seq_len0_case1(TestSequenceReshape): - def init_data(self): self.dimension = 24 self.x_lod = [[0, 2, 8, 2, 0]] @@ -89,9 +84,7 @@ class TestSequenceReshape_reduce_seq_len0_case1(TestSequenceReshape): class TestSequenceReshapeOpError(unittest.TestCase): - def test_error(self): - def test_variable(): x = np.random.random((2, 4)).astype("float32") fluid.layers.sequence_reshape(x=x, new_dim=4) @@ -99,11 +92,13 @@ class TestSequenceReshapeOpError(unittest.TestCase): self.assertRaises(TypeError, test_variable) def test_dtype(): - x1 = fluid.layers.data(name='x1', - shape=[2, 6], - append_batch_size=False, - dtype='float16', - lod_level=1) + x1 = fluid.layers.data( + name='x1', + shape=[2, 6], + append_batch_size=False, + dtype='float16', + lod_level=1, + ) fluid.layers.sequence_reshape(x=x1, new_dim=4) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_reverse.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_reverse.py index e4612b064eb..80b03dd01b4 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_reverse.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_reverse.py @@ -22,7 +22,6 @@ from op_test import OpTest class TestSequenceReverseBase(OpTest): - def initParameters(self): pass @@ -36,14 +35,20 @@ class TestSequenceReverseBase(OpTest): self.y = self.get_output() self.inputs = { - 'X': (self.x, [ - self.lod, - ]), + 'X': ( + self.x, + [ + self.lod, + ], + ), } self.outputs = { - 'Y': (self.y, [ - self.lod, - ]), + 'Y': ( + self.y, + [ + self.lod, + ], + ), } def get_output(self): @@ -65,37 +70,31 @@ class TestSequenceReverseBase(OpTest): class TestSequenceReserve1(TestSequenceReverseBase): - def initParameters(self): self.size = (12, 10) self.lod = [4, 5, 3] class TestSequenceReverse2(TestSequenceReverseBase): - def initParameters(self): self.size = (12, 10) self.lod = [12] class TestSequenceReverse3(TestSequenceReverseBase): - def initParameters(self): self.size = (12, 10) self.lod = [3, 0, 6, 3] class TestSequenceReverse4(TestSequenceReverseBase): - def initParameters(self): self.size = (12, 10) self.lod = [0, 2, 10, 0] class TestSequenceReverseOpError(unittest.TestCase): - def test_error(self): - def test_variable(): # the input type must be Variable x_data = np.random.random((2, 4)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_scatter_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_scatter_op.py index 2b8e8261829..ad4025a349d 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_scatter_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_scatter_op.py @@ -21,7 +21,6 @@ from op_test import OpTest class TestSequenceScatterOp(OpTest): - def init_lod(self): return [[30, 50, 40]] @@ -45,7 +44,7 @@ class TestSequenceScatterOp(OpTest): self.inputs = { 'X': X_data, 'Ids': (Ids_data, Ids_lod), - 'Updates': (Updates_data, Updates_lod) + 'Updates': (Updates_data, Updates_lod), } self.outputs = {'Out': Out_data} @@ -57,37 +56,31 @@ class TestSequenceScatterOp(OpTest): class TestSequenceScatterOpSeqLen0(TestSequenceScatterOp): - def init_lod(self): return [[60, 60, 00]] class TestSequenceScatterOpSeqLen0Case1(TestSequenceScatterOp): - def init_lod(self): return [[0, 60, 60]] class TestSequenceScatterOpSeqLen0Case2(TestSequenceScatterOp): - def init_lod(self): return [[60, 0, 60]] class TestSequenceScatterOpSeqLen0Case3(TestSequenceScatterOp): - def init_lod(self): return [[120, 0, 0]] class TestSequenceScatterOpSeqLen0Case4(TestSequenceScatterOp): - def init_lod(self): return [[0, 120, 0]] class TestSequenceScatterOpSeqLen0Case5(TestSequenceScatterOp): - def init_lod(self): return [[0, 0, 120]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_slice_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_slice_op.py index 052e0fad062..c3b1e291de8 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_slice_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_slice_op.py @@ -21,7 +21,6 @@ from op_test import OpTest class TestSequenceSliceOp(OpTest): - def set_data(self): self.init_test_case() # only supprot one level LoD @@ -31,12 +30,17 @@ class TestSequenceSliceOp(OpTest): length = np.array(self.length).astype("int64") self.inputs = {'X': (x, lod), 'Offset': offset, 'Length': length} - outs = [] #np.zeros((100, 3, 2)).astype('float32') + outs = [] # np.zeros((100, 3, 2)).astype('float32') out_lod = [[]] lod_offset = 0 for i in range(len(offset)): - sub_x = x[lod_offset + offset[i, 0]:lod_offset + offset[i, 0] + - length[i, 0], :] + sub_x = x[ + lod_offset + + offset[i, 0] : lod_offset + + offset[i, 0] + + length[i, 0], + :, + ] outs.append(sub_x) out_lod[0].append(len(sub_x)) lod_offset += lod[0][i] @@ -61,7 +65,6 @@ class TestSequenceSliceOp(OpTest): class TestSequenceSliceOpSeqlen0Case0(TestSequenceSliceOp): - def init_test_case(self): self.x_dim = (100, 3, 2) self.x_lod = [[20, 30, 0, 30, 20]] @@ -70,7 +73,6 @@ class TestSequenceSliceOpSeqlen0Case0(TestSequenceSliceOp): class TestSequenceSliceOpSeqlen0Case1(TestSequenceSliceOp): - def init_test_case(self): self.x_dim = (100, 3, 2) self.x_lod = [[0, 70, 0, 30, 0]] @@ -79,7 +81,6 @@ class TestSequenceSliceOpSeqlen0Case1(TestSequenceSliceOp): class TestSequenceSliceOpSeqlen0Case2(TestSequenceSliceOp): - def init_test_case(self): self.x_dim = (100, 3, 2) self.x_lod = [[0, 100, 0, 0, 0]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_softmax_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_softmax_op.py index 1918dc37a0b..a08cde6af4d 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_softmax_op.py @@ -23,7 +23,6 @@ from test_softmax_op import stable_softmax class TestSequenceSoftmaxOp(OpTest): - def setUp(self): self.op_type = "sequence_softmax" self.use_cudnn = False @@ -34,13 +33,14 @@ class TestSequenceSoftmaxOp(OpTest): out = np.zeros((110, 1)).astype(self.dtype) offset = 0 for i in range(len(self.lod[0])): - if (self.lod[0][i] == 0): + if self.lod[0][i] == 0: continue - sub_x = x[offset:offset + self.lod[0][i], :] + sub_x = x[offset : offset + self.lod[0][i], :] sub_x = sub_x.reshape(1, self.lod[0][i]) sub_out = stable_softmax(sub_x) - out[offset:offset + self.lod[0][i], :] = sub_out.reshape( - self.lod[0][i], 1) + out[offset : offset + self.lod[0][i], :] = sub_out.reshape( + self.lod[0][i], 1 + ) offset += self.lod[0][i] self.inputs = {"X": (x, self.lod)} @@ -71,28 +71,25 @@ class TestSequenceSoftmaxOp(OpTest): # ----------------cudnn Sequencesoftmax---------------- -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSequenceSoftmaxCUDNNOp(TestSequenceSoftmaxOp): - def init_op_type(self): self.use_cudnn = True class TestSequenceSoftmaxOpSeqLen0Case0(TestSequenceSoftmaxOp): - def init_lod(self): self.lod = [[40, 0, 40, 30]] class TestSequenceSoftmaxOpSeqLen0Case1(TestSequenceSoftmaxOp): - def init_lod(self): self.lod = [[0, 40, 70, 0]] class TestSequenceSoftmaxOpSeqLen0Case2(TestSequenceSoftmaxOp): - def init_lod(self): self.lod = [[0, 0, 0, 110]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_topk_avg_pooling.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_topk_avg_pooling.py index 415e9688cfb..651ce9213da 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_topk_avg_pooling.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_topk_avg_pooling.py @@ -22,7 +22,6 @@ from op_test import OpTest class TestSequenceTopkAvgPoolingOp(OpTest): - def setUp(self): self.init_op_type() self.set_data() @@ -51,7 +50,7 @@ class TestSequenceTopkAvgPoolingOp(OpTest): self.inputs = { 'X': (x_data, x_lod), 'ROW': (row_data, [row]), - 'COLUMN': (col_data, [col]) + 'COLUMN': (col_data, [col]), } def calc_gradient(self, pos_data, topks, channel_num, row, col): @@ -60,7 +59,7 @@ class TestSequenceTopkAvgPoolingOp(OpTest): in_numel = sum([row[i] * col[i] for i in range(len(row))]) * channel_num out_numel = sum(row) * len(topks) * channel_num gradient = np.zeros(shape=(in_numel), dtype="float32") - dout_val = 1. / out_numel + dout_val = 1.0 / out_numel pos_offset, in_offset = 0, 0 for bs_idx in range(len(row)): # batch row_size = row[bs_idx] @@ -72,9 +71,9 @@ class TestSequenceTopkAvgPoolingOp(OpTest): for k_idx in range(len(topks)): for k in range(topks[k_idx]): if pos_data[pos_idx + k] != -1: - gradient[in_idx + - pos_data[pos_idx + - k]] += dout_val / topks[k_idx] + gradient[in_idx + pos_data[pos_idx + k]] += ( + dout_val / topks[k_idx] + ) in_offset += row_size * col_size pos_offset += row_size * max_k return gradient @@ -87,7 +86,7 @@ class TestSequenceTopkAvgPoolingOp(OpTest): col_data, col_lod = self.inputs['COLUMN'] channel_num = self.attrs['channel_num'] out = np.zeros((0, len(topks) * channel_num), dtype=x_data.dtype) - pos = np.zeros((0, ), dtype='int32') + pos = np.zeros((0,), dtype='int32') out_lod = deepcopy(row_lod) offset = 0 @@ -95,45 +94,51 @@ class TestSequenceTopkAvgPoolingOp(OpTest): x_len = x_lod[0][idx] self.assertTrue( x_len == channel_num * row_lod[0][idx] * col_lod[0][idx], - "x_len: %s can't mod channel_num: %s" % (x_len, channel_num)) - out_tmp = np.zeros((0, ), dtype=x_data.dtype) - pos_tmp = np.zeros((0, ), dtype='int32') + "x_len: %s can't mod channel_num: %s" % (x_len, channel_num), + ) + out_tmp = np.zeros((0,), dtype=x_data.dtype) + pos_tmp = np.zeros((0,), dtype='int32') for ch in range(channel_num): for r_id in range(row_lod[0][idx]): - x_sub = x_data[offset:(offset + col_lod[0][idx])] + x_sub = x_data[offset : (offset + col_lod[0][idx])] topk_val, topk_pos = self.get_topk(x_sub, max_k) sum_data = self.topk_sum(topk_val, topk_pos, max_k) new_feature = np.array( - [sum_data[topk] / topk for topk in topks]) + [sum_data[topk] / topk for topk in topks] + ) out_tmp = np.hstack((out_tmp, new_feature)) pos_tmp = np.hstack((pos_tmp, topk_pos)) offset += col_lod[0][idx] - out_tmp = out_tmp.reshape([channel_num, -1, - len(topks)]).transpose(1, 0, 2) - pos_tmp = pos_tmp.reshape([channel_num, -1, - max_k]).transpose(1, 0, 2) + out_tmp = out_tmp.reshape([channel_num, -1, len(topks)]).transpose( + 1, 0, 2 + ) + pos_tmp = pos_tmp.reshape([channel_num, -1, max_k]).transpose( + 1, 0, 2 + ) out = np.vstack( - (out, out_tmp.reshape([-1, len(topks) * channel_num]))) + (out, out_tmp.reshape([-1, len(topks) * channel_num])) + ) pos = np.hstack((pos, pos_tmp.flatten())) self.outputs = {'Out': (out.astype('float32'), out_lod), 'pos': pos} - self.gradient = self.calc_gradient(pos, topks, channel_num, row_lod[0], - col_lod[0]) + self.gradient = self.calc_gradient( + pos, topks, channel_num, row_lod[0], col_lod[0] + ) def get_topk(self, x, topk): real_topk = topk if topk < len(x) else len(x) topk_pos = np.array(x).argsort()[-topk:][::-1] topk_val = np.array(x)[topk_pos] if real_topk < topk: - topk_pos = np.hstack((topk_pos, np.full((topk - real_topk, ), -1))) - topk_val = np.hstack((topk_val, np.full((topk - real_topk, ), 0.0))) + topk_pos = np.hstack((topk_pos, np.full((topk - real_topk,), -1))) + topk_val = np.hstack((topk_val, np.full((topk - real_topk,), 0.0))) return topk_val, topk_pos def topk_sum(self, x, pos, max_k): - sum_data = [0.] * (max_k + 1) + sum_data = [0.0] * (max_k + 1) for i in range(1, max_k + 1): if pos[i - 1] == -1: sum_data[i] = sum_data[i - 1] @@ -149,7 +154,6 @@ class TestSequenceTopkAvgPoolingOp(OpTest): class TestSequenceTopkAvgPoolingOpCase1(TestSequenceTopkAvgPoolingOp): - def set_data(self): topks = [2, 3] channel_num = 5 @@ -160,32 +164,32 @@ class TestSequenceTopkAvgPoolingOpCase1(TestSequenceTopkAvgPoolingOp): def test_api(self): import paddle.fluid as fluid + x = fluid.layers.data(name='x', shape=[1], lod_level=1) row = fluid.layers.data(name='row', shape=[10], lod_level=1) col = fluid.layers.data(name='col', shape=[10], lod_level=1) - topk_avg = fluid.contrib.sequence_topk_avg_pooling(input=x, - row=row, - col=col, - topks=[1, 3, 5], - channel_num=5) + topk_avg = fluid.contrib.sequence_topk_avg_pooling( + input=x, row=row, col=col, topks=[1, 3, 5], channel_num=5 + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(45, 1).astype('float32'), [[30, 15]], place) + np.random.rand(45, 1).astype('float32'), [[30, 15]], place + ) row_tensor = fluid.create_lod_tensor( - np.random.rand(5, 10).astype('float32'), [[2, 3]], place) + np.random.rand(5, 10).astype('float32'), [[2, 3]], place + ) col_tensor = fluid.create_lod_tensor( - np.random.rand(4, 10).astype('float32'), [[3, 1]], place) + np.random.rand(4, 10).astype('float32'), [[3, 1]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_tensor, - 'row': row_tensor, - 'col': col_tensor - }, - fetch_list=[topk_avg], - return_numpy=False) + ret = exe.run( + feed={'x': x_tensor, 'row': row_tensor, 'col': col_tensor}, + fetch_list=[topk_avg], + return_numpy=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_unpad_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_unpad_op.py index 7345274caeb..07dfd70ca2b 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_unpad_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_unpad_op.py @@ -23,7 +23,6 @@ import paddle.fluid as fluid class TestSequenceUnpadOp(OpTest): - def init(self): self.length = [2, 3, 4] self.x_shape = (3, 40) @@ -34,13 +33,13 @@ class TestSequenceUnpadOp(OpTest): x = np.random.random(self.x_shape).astype(self.dtype) out_lod = [self.length] - out = x[0, 0:self.length[0]] + out = x[0, 0 : self.length[0]] for i in range(1, x.shape[0]): - out = np.append(out, x[i, 0:self.length[i]], axis=0) + out = np.append(out, x[i, 0 : self.length[i]], axis=0) - out_shape = (sum(self.length), ) + out_shape = (sum(self.length),) if len(self.x_shape) == 2: - out_shape = out_shape + (1, ) + out_shape = out_shape + (1,) else: out_shape = out_shape + self.x_shape[2:] @@ -60,7 +59,6 @@ class TestSequenceUnpadOp(OpTest): class TestSequenceUnpadOp2(TestSequenceUnpadOp): - def init(self): self.length = [2, 3, 4] self.x_shape = (3, 5, 4, 3) @@ -68,7 +66,6 @@ class TestSequenceUnpadOp2(TestSequenceUnpadOp): class TestSequenceUnpadOp3(TestSequenceUnpadOp): - def init(self): self.length = [5, 2, 3, 4] self.x_shape = (4, 5, 3, 3, 6) @@ -76,7 +73,6 @@ class TestSequenceUnpadOp3(TestSequenceUnpadOp): class TestSequenceUnpadOp4(TestSequenceUnpadOp): - def init(self): self.length = [5, 0, 0, 4] self.x_shape = (4, 5, 3, 3, 6) @@ -84,7 +80,6 @@ class TestSequenceUnpadOp4(TestSequenceUnpadOp): class TestSequenceUnpadOp5(TestSequenceUnpadOp): - def init(self): self.length = [0, 4, 3, 0] self.x_shape = (4, 5, 3, 3, 6) @@ -92,9 +87,7 @@ class TestSequenceUnpadOp5(TestSequenceUnpadOp): class TestSequenceUnpadOpError(unittest.TestCase): - def test_error(self): - def test_x_variable(): x = np.random.random((10, 5)).astype("float64") len = fluid.data(name='length2', shape=[10], dtype='int64') diff --git a/python/paddle/fluid/tests/unittests/seresnext_net.py b/python/paddle/fluid/tests/unittests/seresnext_net.py index d1676380afe..b2bc25e35ae 100644 --- a/python/paddle/fluid/tests/unittests/seresnext_net.py +++ b/python/paddle/fluid/tests/unittests/seresnext_net.py @@ -48,37 +48,40 @@ def squeeze_excitation(input, num_channels, reduction_ratio): # input=input, pool_size=0, pool_type='avg', global_pooling=True) conv = input shape = conv.shape - reshape = fluid.layers.reshape(x=conv, - shape=[-1, shape[1], shape[2] * shape[3]]) + reshape = fluid.layers.reshape( + x=conv, shape=[-1, shape[1], shape[2] * shape[3]] + ) pool = fluid.layers.reduce_mean(input=reshape, dim=2) - squeeze = fluid.layers.fc(input=pool, - size=num_channels // reduction_ratio, - act='relu') - excitation = fluid.layers.fc(input=squeeze, - size=num_channels, - act='sigmoid') + squeeze = fluid.layers.fc( + input=pool, size=num_channels // reduction_ratio, act='relu' + ) + excitation = fluid.layers.fc( + input=squeeze, size=num_channels, act='sigmoid' + ) scale = fluid.layers.elementwise_mul(x=input, y=excitation, axis=0) return scale -def conv_bn_layer(input, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): - conv = fluid.layers.conv2d(input=input, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - use_cudnn=(not remove_cudnn_conv), - bias_attr=False) - return conv if remove_bn else fluid.layers.batch_norm( - input=conv, act=act, momentum=0.1) +def conv_bn_layer( + input, num_filters, filter_size, stride=1, groups=1, act=None +): + conv = fluid.layers.conv2d( + input=input, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + use_cudnn=(not remove_cudnn_conv), + bias_attr=False, + ) + return ( + conv + if remove_bn + else fluid.layers.batch_norm(input=conv, act=act, momentum=0.1) + ) def shortcut(input, ch_out, stride): @@ -96,23 +99,25 @@ def shortcut(input, ch_out, stride): def bottleneck_block(input, num_filters, stride, cardinality, reduction_ratio): # The number of first 1x1 convolutional channels for each bottleneck build block # was halved to reduce the compution cost. - conv0 = conv_bn_layer(input=input, - num_filters=num_filters, - filter_size=1, - act='relu') - conv1 = conv_bn_layer(input=conv0, - num_filters=num_filters * 2, - filter_size=3, - stride=stride, - groups=cardinality, - act='relu') - conv2 = conv_bn_layer(input=conv1, - num_filters=num_filters * 2, - filter_size=1, - act=None) - scale = squeeze_excitation(input=conv2, - num_channels=num_filters * 2, - reduction_ratio=reduction_ratio) + conv0 = conv_bn_layer( + input=input, num_filters=num_filters, filter_size=1, act='relu' + ) + conv1 = conv_bn_layer( + input=conv0, + num_filters=num_filters * 2, + filter_size=3, + stride=stride, + groups=cardinality, + act='relu', + ) + conv2 = conv_bn_layer( + input=conv1, num_filters=num_filters * 2, filter_size=1, act=None + ) + scale = squeeze_excitation( + input=conv2, + num_channels=num_filters * 2, + reduction_ratio=reduction_ratio, + ) short = shortcut(input, num_filters * 2, stride) @@ -127,26 +132,18 @@ def SE_ResNeXt50Small(use_feed): img = fluid.layers.data(name='image', shape=img_shape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') - conv = conv_bn_layer(input=img, - num_filters=16, - filter_size=3, - stride=2, - act='relu') - conv = conv_bn_layer(input=conv, - num_filters=16, - filter_size=3, - stride=1, - act='relu') - conv = conv_bn_layer(input=conv, - num_filters=16, - filter_size=3, - stride=1, - act='relu') - conv = fluid.layers.pool2d(input=conv, - pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + conv = conv_bn_layer( + input=img, num_filters=16, filter_size=3, stride=2, act='relu' + ) + conv = conv_bn_layer( + input=conv, num_filters=16, filter_size=3, stride=1, act='relu' + ) + conv = conv_bn_layer( + input=conv, num_filters=16, filter_size=3, stride=1, act='relu' + ) + conv = fluid.layers.pool2d( + input=conv, pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) cardinality = 32 reduction_ratio = 16 @@ -155,18 +152,24 @@ def SE_ResNeXt50Small(use_feed): for block in range(len(depth)): for i in range(depth[block]): - conv = bottleneck_block(input=conv, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=cardinality, - reduction_ratio=reduction_ratio) + conv = bottleneck_block( + input=conv, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio, + ) shape = conv.shape - reshape = fluid.layers.reshape(x=conv, - shape=[-1, shape[1], shape[2] * shape[3]]) + reshape = fluid.layers.reshape( + x=conv, shape=[-1, shape[1], shape[2] * shape[3]] + ) pool = fluid.layers.reduce_mean(input=reshape, dim=2) - dropout = pool if remove_dropout else fluid.layers.dropout( - x=pool, dropout_prob=0.2, seed=1) + dropout = ( + pool + if remove_dropout + else fluid.layers.dropout(x=pool, dropout_prob=0.2, seed=1) + ) # Classifier layer: prediction = fluid.layers.fc(input=dropout, size=1000, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -176,11 +179,12 @@ def SE_ResNeXt50Small(use_feed): def optimizer(learning_rate=0.01): optimizer = fluid.optimizer.Momentum( - learning_rate=cosine_decay(learning_rate=learning_rate, - step_each_epoch=2, - epochs=1), + learning_rate=cosine_decay( + learning_rate=learning_rate, step_each_epoch=2, epochs=1 + ), momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer @@ -203,10 +207,13 @@ def iter(use_device): gpu_img, gpu_label = init_data( batch_size=batch_size(use_device=DeviceType.CUDA), img_shape=img_shape, - label_range=999) -cpu_img, cpu_label = init_data(batch_size=batch_size(use_device=DeviceType.CPU), - img_shape=img_shape, - label_range=999) + label_range=999, +) +cpu_img, cpu_label = init_data( + batch_size=batch_size(use_device=DeviceType.CPU), + img_shape=img_shape, + label_range=999, +) feed_dict_gpu = {"image": gpu_img, "label": gpu_label} feed_dict_cpu = {"image": cpu_img, "label": cpu_label} diff --git a/python/paddle/fluid/tests/unittests/seresnext_test_base.py b/python/paddle/fluid/tests/unittests/seresnext_test_base.py index 739de92db60..2199d1ed20a 100644 --- a/python/paddle/fluid/tests/unittests/seresnext_test_base.py +++ b/python/paddle/fluid/tests/unittests/seresnext_test_base.py @@ -20,30 +20,33 @@ import numpy as np class TestResnetBase(TestParallelExecutorBase): - - def _compare_result_with_origin_model(self, - check_func, - use_device, - delta2=1e-5, - compare_separately=True): + def _compare_result_with_origin_model( + self, check_func, use_device, delta2=1e-5, compare_separately=True + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return - func_1_first_loss, func_1_last_loss, func_1_loss_area = self.check_network_convergence( + ( + func_1_first_loss, + func_1_last_loss, + func_1_loss_area, + ) = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), batch_size=seresnext_net.batch_size(use_device), use_device=use_device, use_reduce=False, - optimizer=seresnext_net.optimizer) + optimizer=seresnext_net.optimizer, + ) func_2_first_loss, func_2_last_loss, func_2_loss_area = check_func( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), batch_size=seresnext_net.batch_size(use_device), - use_device=use_device) + use_device=use_device, + ) if compare_separately: for loss in zip(func_1_first_loss, func_2_first_loss): @@ -51,12 +54,12 @@ class TestResnetBase(TestParallelExecutorBase): for loss in zip(func_1_last_loss, func_2_last_loss): self.assertAlmostEquals(loss[0], loss[1], delta=delta2) else: - np.testing.assert_allclose(func_1_loss_area, - func_2_loss_area, - rtol=delta2) - self.assertAlmostEquals(np.mean(func_1_first_loss), - func_2_first_loss[0], - delta=1e-5) - self.assertAlmostEquals(np.mean(func_1_last_loss), - func_2_last_loss[0], - delta=delta2) + np.testing.assert_allclose( + func_1_loss_area, func_2_loss_area, rtol=delta2 + ) + self.assertAlmostEquals( + np.mean(func_1_first_loss), func_2_first_loss[0], delta=1e-5 + ) + self.assertAlmostEquals( + np.mean(func_1_last_loss), func_2_last_loss[0], delta=delta2 + ) diff --git a/python/paddle/fluid/tests/unittests/simnet_dataset_reader.py b/python/paddle/fluid/tests/unittests/simnet_dataset_reader.py index 5eae0ff22e1..5b4a49e693a 100644 --- a/python/paddle/fluid/tests/unittests/simnet_dataset_reader.py +++ b/python/paddle/fluid/tests/unittests/simnet_dataset_reader.py @@ -22,6 +22,5 @@ logger.setLevel(logging.INFO) class DatasetSimnetReader(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): pass diff --git a/python/paddle/fluid/tests/unittests/simple_nets.py b/python/paddle/fluid/tests/unittests/simple_nets.py index 9326d515915..f85a0f9e135 100644 --- a/python/paddle/fluid/tests/unittests/simple_nets.py +++ b/python/paddle/fluid/tests/unittests/simple_nets.py @@ -24,8 +24,10 @@ def simple_fc_net_with_inputs(img, label, class_num=10): hidden, size=100, act='relu', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) prediction = fluid.layers.fc(hidden, size=class_num, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) loss = paddle.mean(loss) @@ -45,8 +47,10 @@ def batchnorm_fc_with_inputs(img, label, class_num=10): hidden, size=200, act='relu', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) hidden = fluid.layers.batch_norm(input=hidden) @@ -62,26 +66,27 @@ def fc_with_batchnorm(use_feed=None): return batchnorm_fc_with_inputs(img, label, class_num=10) -def bow_net(use_feed, - dict_dim, - is_sparse=False, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): +def bow_net( + use_feed, + dict_dim, + is_sparse=False, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") - emb = fluid.layers.embedding(input=data, - is_sparse=is_sparse, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -98,7 +103,9 @@ def init_data(batch_size=32, img_shape=[784], label_range=9): assert isinstance(img_shape, list) input_shape = [batch_size] + img_shape img = np.random.random(size=input_shape).astype(np.float32) - label = np.array( - [np.random.randint(0, label_range) for _ in range(batch_size)]).reshape( - (-1, 1)).astype("int64") + label = ( + np.array([np.random.randint(0, label_range) for _ in range(batch_size)]) + .reshape((-1, 1)) + .astype("int64") + ) return img, label diff --git a/python/paddle/fluid/tests/unittests/spawn_runner_base.py b/python/paddle/fluid/tests/unittests/spawn_runner_base.py index d832781ffbc..2b35fe70cf9 100644 --- a/python/paddle/fluid/tests/unittests/spawn_runner_base.py +++ b/python/paddle/fluid/tests/unittests/spawn_runner_base.py @@ -30,7 +30,6 @@ class SpawnAssistTestArgs(object): class TestDistSpawnRunner(unittest.TestCase): - def setUp(self): # NOTE(chenweihang): keep consistent with # TestDistBase.check_with_place @@ -42,10 +41,12 @@ class TestDistSpawnRunner(unittest.TestCase): def _run_parallel(self, model, args): args.update_method = "nccl2" - context = paddle.distributed.spawn(func=model.run_trainer_with_spawn, - args=(args, ), - nprocs=self.nprocs, - join=True) + context = paddle.distributed.spawn( + func=model.run_trainer_with_spawn, + args=(args,), + nprocs=self.nprocs, + join=True, + ) result_list = [] for res_queue in context.return_queues: result_list.append(res_queue.get()) @@ -53,10 +54,12 @@ class TestDistSpawnRunner(unittest.TestCase): def check_dist_result_with_spawn(self, test_class, delta=1e-3): with _test_eager_guard(): - self.check_dist_result_with_spawn_func(test_class=test_class, - delta=delta) - self.check_dist_result_with_spawn_func(test_class=test_class, - delta=delta) + self.check_dist_result_with_spawn_func( + test_class=test_class, delta=delta + ) + self.check_dist_result_with_spawn_func( + test_class=test_class, delta=delta + ) def check_dist_result_with_spawn_func(self, test_class, delta=1e-3): # 0. prepare model and args @@ -83,7 +86,8 @@ class TestDistSpawnRunner(unittest.TestCase): loss, dist_loss, delta=delta, - msg= - "The results of single-card execution and multi-card execution are inconsistent." - "signal-card loss is:\n{}\nmulti-card average loss is:\n{}\n". - format(loss, dist_loss)) + msg="The results of single-card execution and multi-card execution are inconsistent." + "signal-card loss is:\n{}\nmulti-card average loss is:\n{}\n".format( + loss, dist_loss + ), + ) diff --git a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_attention.py b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_attention.py index c6c31853616..2d825af4b33 100644 --- a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_attention.py +++ b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_attention.py @@ -25,9 +25,11 @@ paddle.enable_static() def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr @@ -40,14 +42,16 @@ hidden = n_head * d_key def create_model(data, rank): np.random.seed(2021) - pre_ln_w = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - pre_ln_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - qkv_w = np.random.uniform(-1, 1, - size=(3, n_head, d_key, hidden)).astype(DTYPE) + pre_ln_w = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + pre_ln_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + qkv_w = np.random.uniform(-1, 1, size=(3, n_head, d_key, hidden)).astype( + DTYPE + ) qkv_b = np.random.uniform(-1, 1, size=(3, n_head, d_key)).astype(DTYPE) - linear_w = np.random.uniform(-1, 1, - size=(n_head * d_key, hidden)).astype(DTYPE) - linear_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + linear_w = np.random.uniform(-1, 1, size=(n_head * d_key, hidden)).astype( + DTYPE + ) + linear_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) data.stop_gradient = False if rank is not None: @@ -55,46 +59,50 @@ def create_model(data, rank): end = start + n_head // MODEL_PARALLEL_SIZE col_qkv_w = qkv_w[:, start:end, :, :] col_qkv_b = qkv_b[:, start:end, :] - row_linear_w = linear_w[(start * d_key):(end * d_key), :] + row_linear_w = linear_w[(start * d_key) : (end * d_key), :] pre_ln_w_attr, pre_ln_b_attr = get_param_attr(pre_ln_w, pre_ln_b) qkv_w_attr, qkv_b_attr = get_param_attr(col_qkv_w, col_qkv_b) linear_w_attr, linear_b_attr = get_param_attr(row_linear_w, linear_b) - attn = FusedMultiHeadAttention(hidden, - n_head, - dropout_rate=0.0, - attn_dropout_rate=0.0, - normalize_before=False, - qkv_weight_attr=qkv_w_attr, - qkv_bias_attr=qkv_b_attr, - linear_weight_attr=linear_w_attr, - linear_bias_attr=linear_b_attr, - pre_ln_scale_attr=pre_ln_w_attr, - pre_ln_bias_attr=pre_ln_b_attr, - ln_scale_attr=pre_ln_w_attr, - ln_bias_attr=pre_ln_b_attr, - nranks=MODEL_PARALLEL_SIZE, - ring_id=0) + attn = FusedMultiHeadAttention( + hidden, + n_head, + dropout_rate=0.0, + attn_dropout_rate=0.0, + normalize_before=False, + qkv_weight_attr=qkv_w_attr, + qkv_bias_attr=qkv_b_attr, + linear_weight_attr=linear_w_attr, + linear_bias_attr=linear_b_attr, + pre_ln_scale_attr=pre_ln_w_attr, + pre_ln_bias_attr=pre_ln_b_attr, + ln_scale_attr=pre_ln_w_attr, + ln_bias_attr=pre_ln_b_attr, + nranks=MODEL_PARALLEL_SIZE, + ring_id=0, + ) result = attn(data) else: pre_ln_w_attr, pre_ln_b_attr = get_param_attr(pre_ln_w, pre_ln_b) qkv_w_attr, qkv_b_attr = get_param_attr(qkv_w, qkv_b) linear_w_attr, linear_b_attr = get_param_attr(linear_w, linear_b) - attn = FusedMultiHeadAttention(hidden, - n_head, - dropout_rate=0.0, - attn_dropout_rate=0.0, - normalize_before=False, - qkv_weight_attr=qkv_w_attr, - qkv_bias_attr=qkv_b_attr, - linear_weight_attr=linear_w_attr, - linear_bias_attr=linear_b_attr, - pre_ln_scale_attr=pre_ln_w_attr, - pre_ln_bias_attr=pre_ln_b_attr, - ln_scale_attr=pre_ln_w_attr, - ln_bias_attr=pre_ln_b_attr) + attn = FusedMultiHeadAttention( + hidden, + n_head, + dropout_rate=0.0, + attn_dropout_rate=0.0, + normalize_before=False, + qkv_weight_attr=qkv_w_attr, + qkv_bias_attr=qkv_b_attr, + linear_weight_attr=linear_w_attr, + linear_bias_attr=linear_b_attr, + pre_ln_scale_attr=pre_ln_w_attr, + pre_ln_bias_attr=pre_ln_b_attr, + ln_scale_attr=pre_ln_w_attr, + ln_bias_attr=pre_ln_b_attr, + ) result = attn(data) predict = paddle.sum(result) @@ -102,20 +110,20 @@ def create_model(data, rank): class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data seq_len = 2 - data_in = fluid.data(name='data_in', - shape=[batch_size, seq_len, hidden], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, seq_len, hidden], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -128,8 +136,9 @@ class TestModelParallel(TestDistRunnerBase): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_feedforward.py b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_feedforward.py index 409c84eaab5..f15dd57123f 100644 --- a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_feedforward.py +++ b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_feedforward.py @@ -30,20 +30,22 @@ OUT_SIZE = 2 * MODEL_PARALLEL_SIZE def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr def create_model(data, rank): np.random.seed(2021) - ln_w = np.random.uniform(-1, 1, size=(IN_SIZE, )).astype(DTYPE) - ln_b = np.random.uniform(-1, 1, size=(IN_SIZE, )).astype(DTYPE) + ln_w = np.random.uniform(-1, 1, size=(IN_SIZE,)).astype(DTYPE) + ln_b = np.random.uniform(-1, 1, size=(IN_SIZE,)).astype(DTYPE) w0 = np.random.uniform(-1, 1, size=(IN_SIZE, OUT_SIZE)).astype(DTYPE) - b0 = np.random.uniform(-1, 1, size=(OUT_SIZE, )).astype(DTYPE) + b0 = np.random.uniform(-1, 1, size=(OUT_SIZE,)).astype(DTYPE) w1 = np.random.uniform(-1, 1, size=(OUT_SIZE, IN_SIZE)).astype(DTYPE) - b1 = np.random.uniform(-1, 1, size=(IN_SIZE, )).astype(DTYPE) + b1 = np.random.uniform(-1, 1, size=(IN_SIZE,)).astype(DTYPE) data.stop_gradient = False if rank is not None: start = 0 if rank == 0 else OUT_SIZE // MODEL_PARALLEL_SIZE @@ -56,38 +58,42 @@ def create_model(data, rank): w0_attr, b0_attr = get_param_attr(col_w0, col_b0) w1_attr, b1_attr = get_param_attr(row_w1, b1) - ffn = FusedFeedForward(IN_SIZE, - OUT_SIZE, - dropout_rate=0.0, - activation='gelu', - normalize_before=True, - linear1_weight_attr=w0_attr, - linear1_bias_attr=b0_attr, - linear2_weight_attr=w1_attr, - linear2_bias_attr=b1_attr, - ln1_scale_attr=ln_w_attr, - ln1_bias_attr=ln_b_attr, - nranks=MODEL_PARALLEL_SIZE, - ring_id=0) - #ffn.eval() + ffn = FusedFeedForward( + IN_SIZE, + OUT_SIZE, + dropout_rate=0.0, + activation='gelu', + normalize_before=True, + linear1_weight_attr=w0_attr, + linear1_bias_attr=b0_attr, + linear2_weight_attr=w1_attr, + linear2_bias_attr=b1_attr, + ln1_scale_attr=ln_w_attr, + ln1_bias_attr=ln_b_attr, + nranks=MODEL_PARALLEL_SIZE, + ring_id=0, + ) + # ffn.eval() result = ffn(data) else: ln_w_attr, ln_b_attr = get_param_attr(ln_w, ln_b) w0_attr, b0_attr = get_param_attr(w0, b0) w1_attr, b1_attr = get_param_attr(w1, b1) - ffn = FusedFeedForward(IN_SIZE, - OUT_SIZE, - dropout_rate=0.0, - activation='gelu', - normalize_before=True, - linear1_weight_attr=w0_attr, - linear1_bias_attr=b0_attr, - linear2_weight_attr=w1_attr, - linear2_bias_attr=b1_attr, - ln1_scale_attr=ln_w_attr, - ln1_bias_attr=ln_b_attr) - #ffn.eval() + ffn = FusedFeedForward( + IN_SIZE, + OUT_SIZE, + dropout_rate=0.0, + activation='gelu', + normalize_before=True, + linear1_weight_attr=w0_attr, + linear1_bias_attr=b0_attr, + linear2_weight_attr=w1_attr, + linear2_bias_attr=b1_attr, + ln1_scale_attr=ln_w_attr, + ln1_bias_attr=ln_b_attr, + ) + # ffn.eval() result = ffn(data) predict = paddle.sum(result) @@ -95,20 +101,20 @@ def create_model(data, rank): class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data seq_len = 2 - data_in = fluid.data(name='data_in', - shape=[batch_size, seq_len, IN_SIZE], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, seq_len, IN_SIZE], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -121,8 +127,9 @@ class TestModelParallel(TestDistRunnerBase): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py index 0ab441d2963..f9125dae345 100644 --- a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py +++ b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py @@ -25,9 +25,11 @@ paddle.enable_static() def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr @@ -41,28 +43,30 @@ dim_ffn = 4 * hidden def create_model(data, rank): np.random.seed(2021) - ln_w = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - ln_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - qkv_w = np.random.uniform(-1, 1, size=(3, num_head, dim_head, - hidden)).astype(DTYPE) + ln_w = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + ln_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + qkv_w = np.random.uniform( + -1, 1, size=(3, num_head, dim_head, hidden) + ).astype(DTYPE) qkv_b = np.random.uniform(-1, 1, size=(3, num_head, dim_head)).astype(DTYPE) - linear_w = np.random.uniform(-1, 1, size=(num_head * dim_head, - hidden)).astype(DTYPE) - linear_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + linear_w = np.random.uniform( + -1, 1, size=(num_head * dim_head, hidden) + ).astype(DTYPE) + linear_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) - ffn_ln_w = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - ffn_ln_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + ffn_ln_w = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + ffn_ln_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) ffn1_w = np.random.uniform(-1, 1, size=(hidden, dim_ffn)).astype(DTYPE) - ffn1_b = np.random.uniform(-1, 1, size=(dim_ffn, )).astype(DTYPE) + ffn1_b = np.random.uniform(-1, 1, size=(dim_ffn,)).astype(DTYPE) ffn2_w = np.random.uniform(-1, 1, size=(dim_ffn, hidden)).astype(DTYPE) - ffn2_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + ffn2_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) if rank is not None: start = 0 if rank == 0 else (num_head // MODEL_PARALLEL_SIZE) end = start + (num_head // MODEL_PARALLEL_SIZE) col_qkv_w = qkv_w[:, start:end, :, :] col_qkv_b = qkv_b[:, start:end, :] - row_linear_w = linear_w[(start * dim_head):(end * dim_head), :] + row_linear_w = linear_w[(start * dim_head) : (end * dim_head), :] ln_w_attr, ln_b_attr = get_param_attr(ln_w, ln_b) qkv_w_attr, qkv_b_attr = get_param_attr(col_qkv_w, col_qkv_b) @@ -98,7 +102,8 @@ def create_model(data, rank): ffn2_weight_attrs=[ffn2_w_attr], ffn2_bias_attrs=[ffn2_b_attr], nranks=MODEL_PARALLEL_SIZE, - ring_id=0) + ring_id=0, + ) result = multi_transformer(data) else: ln_w_attr, ln_b_attr = get_param_attr(ln_w, ln_b) @@ -127,7 +132,8 @@ def create_model(data, rank): ffn1_weight_attrs=[ffn1_w_attr], ffn1_bias_attrs=[ffn1_b_attr], ffn2_weight_attrs=[ffn2_w_attr], - ffn2_bias_attrs=[ffn2_b_attr]) + ffn2_bias_attrs=[ffn2_b_attr], + ) result = multi_transformer(data) # fused_multi_transformer have no backward @@ -137,20 +143,20 @@ def create_model(data, rank): class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data seq_len = 2 - data_in = fluid.data(name='data_in', - shape=[batch_size, seq_len, hidden], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, seq_len, hidden], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -163,8 +169,9 @@ class TestModelParallel(TestDistRunnerBase): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/test_Tensor_type.py b/python/paddle/fluid/tests/unittests/test_Tensor_type.py index 2211aed945e..f18818b8ef5 100644 --- a/python/paddle/fluid/tests/unittests/test_Tensor_type.py +++ b/python/paddle/fluid/tests/unittests/test_Tensor_type.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class TensorTypeTest(unittest.TestCase): - def func_type_totensor(self): paddle.disable_static() inx = np.array([1, 2]) diff --git a/python/paddle/fluid/tests/unittests/test_accuracy_op.py b/python/paddle/fluid/tests/unittests/test_accuracy_op.py index 834ab8f77d8..0de81034d33 100755 --- a/python/paddle/fluid/tests/unittests/test_accuracy_op.py +++ b/python/paddle/fluid/tests/unittests/test_accuracy_op.py @@ -21,7 +21,6 @@ from paddle.fluid import Program, program_guard class TestAccuracyOp(OpTest): - def setUp(self): self.op_type = "accuracy" self.dtype = np.float32 @@ -40,7 +39,7 @@ class TestAccuracyOp(OpTest): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } def init_dtype(self): @@ -51,7 +50,6 @@ class TestAccuracyOp(OpTest): class TestAccuracyOpFp16(TestAccuracyOp): - def init_dtype(self): self.dtype = np.float16 @@ -60,15 +58,15 @@ class TestAccuracyOpFp16(TestAccuracyOp): class TestAccuracyOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - label = fluid.layers.data(name='label', - shape=[-1, 1], - dtype="int32") + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + label = fluid.layers.data( + name='label', shape=[-1, 1], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.accuracy, x1, label) self.assertRaises(TypeError, paddle.metric.accuracy, x1, label) # The input dtype of accuracy_op must be float32 or float64. @@ -81,40 +79,42 @@ class TestAccuracyOpError(unittest.TestCase): class TestAccuracyAPI1(unittest.TestCase): - def setUp(self): - self.predictions = paddle.static.data(shape=[2, 5], - name="predictions", - dtype="float32") - self.label = paddle.static.data(shape=[2, 1], - name="labels", - dtype="int64") - self.result = paddle.static.accuracy(input=self.predictions, - label=self.label, - k=1) + self.predictions = paddle.static.data( + shape=[2, 5], name="predictions", dtype="float32" + ) + self.label = paddle.static.data( + shape=[2, 1], name="labels", dtype="int64" + ) + self.result = paddle.static.accuracy( + input=self.predictions, label=self.label, k=1 + ) self.input_predictions = np.array( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype="float32") + dtype="float32", + ) self.input_labels = np.array([[2], [0]], dtype="int64") self.expect_value = np.array([0.5], dtype='float32') def test_api(self): exe = paddle.static.Executor() - result, = exe.run(feed={ - "predictions": self.input_predictions, - 'labels': self.input_labels - }, - fetch_list=[self.result.name]) + (result,) = exe.run( + feed={ + "predictions": self.input_predictions, + 'labels': self.input_labels, + }, + fetch_list=[self.result.name], + ) self.assertEqual((result == self.expect_value).all(), True) class TestAccuracyAPI2(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): predictions = paddle.to_tensor( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype='float32') + dtype='float32', + ) label = paddle.to_tensor([[2], [0]], dtype="int64") result = paddle.static.accuracy(input=predictions, label=label, k=1) expect_value = np.array([0.5], dtype='float32') @@ -122,12 +122,12 @@ class TestAccuracyAPI2(unittest.TestCase): class TestAccuracyAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): predictions = paddle.to_tensor( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype='float32') + dtype='float32', + ) label = paddle.to_tensor([[2], [0]], dtype="int64") result = paddle.metric.accuracy(input=predictions, label=label, k=1) expect_value = np.array([0.5], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py index cee8310cd18..bc5a14070e5 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py @@ -26,7 +26,6 @@ from decorator_helper import prog_scope class TestSigmoidTripleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -37,11 +36,9 @@ class TestSigmoidTripleGradCheck(unittest.TestCase): y = layers.sigmoid(x) x_arr = np.random.random(shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.triple_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -53,7 +50,6 @@ class TestSigmoidTripleGradCheck(unittest.TestCase): class TestSigmoidDoubleGradCheck(unittest.TestCase): - def sigmoid_wrapper(self, x): return fluid.layers.sigmoid(x[0]) @@ -67,17 +63,13 @@ class TestSigmoidDoubleGradCheck(unittest.TestCase): y = layers.sigmoid(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.sigmoid_wrapper, - [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.sigmoid_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -90,7 +82,6 @@ class TestSigmoidDoubleGradCheck(unittest.TestCase): class TestTanhTripleGradCheck(unittest.TestCase): - def tanh_wrapper(self, x): return paddle.tanh(x[0]) @@ -104,16 +95,13 @@ class TestTanhTripleGradCheck(unittest.TestCase): y = layers.tanh(x) x_arr = np.random.random(shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.triple_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.tanh_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.tanh_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -126,7 +114,6 @@ class TestTanhTripleGradCheck(unittest.TestCase): class TestTanhDoubleGradCheck(unittest.TestCase): - def tanh_wrapper(self, x): return paddle.tanh(x[0]) @@ -140,16 +127,13 @@ class TestTanhDoubleGradCheck(unittest.TestCase): y = paddle.tanh(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.tanh_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.tanh_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -162,7 +146,6 @@ class TestTanhDoubleGradCheck(unittest.TestCase): class TestAbsDoubleGradCheck(unittest.TestCase): - def abs_wrapper(self, x): return paddle.abs(x[0]) @@ -176,16 +159,13 @@ class TestAbsDoubleGradCheck(unittest.TestCase): y = paddle.abs(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.abs_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.abs_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -198,7 +178,6 @@ class TestAbsDoubleGradCheck(unittest.TestCase): class TestReluDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -211,11 +190,9 @@ class TestReluDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -227,7 +204,6 @@ class TestReluDoubleGradCheck(unittest.TestCase): class TestLeakyReluDoubleGradCheck(unittest.TestCase): - def leaky_relu_wrapper(self, x): return paddle.nn.functional.leaky_relu(x[0], negative_slope=0.2) @@ -245,16 +221,12 @@ class TestLeakyReluDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.leaky_relu_wrapper, - [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.leaky_relu_wrapper, [x], y, x_init=x_arr, place=place + ) def test_grad(self): paddle.enable_static() @@ -266,7 +238,6 @@ class TestLeakyReluDoubleGradCheck(unittest.TestCase): class TestELUDoubleGradCheck(unittest.TestCase): - def elu_wrapper(self, x): return paddle.nn.functional.elu(x[0], alpha=0.2) @@ -284,16 +255,13 @@ class TestELUDoubleGradCheck(unittest.TestCase): y = layers.elu(x, alpha=alpha) np.random.RandomState(SEED) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.elu_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.elu_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -306,7 +274,6 @@ class TestELUDoubleGradCheck(unittest.TestCase): class TestCELUDoubleGradCheck(unittest.TestCase): - def celu_wrapper(self, x): return paddle.nn.functional.celu(x[0], alpha=0.2) @@ -324,16 +291,13 @@ class TestCELUDoubleGradCheck(unittest.TestCase): y = F.celu(x, alpha=alpha) np.random.RandomState(SEED) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.celu_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.celu_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -346,7 +310,6 @@ class TestCELUDoubleGradCheck(unittest.TestCase): class TestSqrtDoubleGradCheck(unittest.TestCase): - def sqrt_wrapper(self, x): return paddle.sqrt(x[0]) @@ -362,15 +325,12 @@ class TestSqrtDoubleGradCheck(unittest.TestCase): y = layers.sqrt(x) x_arr = np.random.uniform(0.1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.sqrt_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.sqrt_wrapper, [x], y, x_init=x_arr, place=place + ) def test_grad(self): paddle.enable_static() @@ -382,7 +342,6 @@ class TestSqrtDoubleGradCheck(unittest.TestCase): class TestRsqrtDoubleGradCheck(unittest.TestCase): - def rsqrt_wrapper(self, x): return paddle.rsqrt(x[0]) @@ -398,15 +357,12 @@ class TestRsqrtDoubleGradCheck(unittest.TestCase): y = layers.rsqrt(x) x_arr = np.random.uniform(0.1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.rsqrt_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.rsqrt_wrapper, [x], y, x_init=x_arr, place=place + ) def test_grad(self): paddle.enable_static() @@ -418,7 +374,6 @@ class TestRsqrtDoubleGradCheck(unittest.TestCase): class TestSquareDoubleGradCheck(unittest.TestCase): - def square_wrapper(self, x): return paddle.square(x[0]) @@ -434,16 +389,13 @@ class TestSquareDoubleGradCheck(unittest.TestCase): y = layers.square(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.square_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.square_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -456,7 +408,6 @@ class TestSquareDoubleGradCheck(unittest.TestCase): class TestAbsDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -473,11 +424,9 @@ class TestAbsDoubleGradCheck(unittest.TestCase): # we should avoid this x_arr[np.abs(x_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -489,7 +438,6 @@ class TestAbsDoubleGradCheck(unittest.TestCase): class TestLogDoubleGradCheck(unittest.TestCase): - def log_wrapper(self, x): return paddle.log(x[0]) @@ -505,16 +453,13 @@ class TestLogDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(0.1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.log_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.log_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index 49c19f1cbff..a933d591a87 100755 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -29,26 +29,24 @@ paddle.enable_static() class TestSqrtOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of sqrt op must be Variable or numpy.ndarray. in1 = 1 self.assertRaises(TypeError, fluid.layers.sqrt, in1) # The input dtype of sqrt op must be float16, float32, float64. - in2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + in2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.sqrt, in2) - in3 = fluid.layers.data(name='input3', - shape=[12, 10], - dtype="float16") + in3 = fluid.layers.data( + name='input3', shape=[12, 10], dtype="float16" + ) fluid.layers.sqrt(x=in3) class TestActivation(OpTest): - def setUp(self): self.op_type = "exp" self.init_dtype() @@ -89,13 +87,11 @@ class TestActivation(OpTest): class TestActivation_ZeroDim(TestActivation): - def init_shape(self): self.shape = [] class TestExpm1(TestActivation): - def setUp(self): self.op_type = "expm1" self.python_api = paddle.expm1 @@ -117,13 +113,11 @@ class TestExpm1(TestActivation): class TestExpm1_ZeroDim(TestExpm1): - def init_shape(self): self.shape = [] class TestExpm1API(unittest.TestCase): - def init_dtype(self): self.dtype = 'float64' self.shape = [11, 17] @@ -153,7 +147,6 @@ class TestExpm1API(unittest.TestCase): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) X = paddle.to_tensor(self.x) @@ -173,7 +166,6 @@ class TestExpm1API(unittest.TestCase): class TestParameter(object): - def test_out_name(self): with fluid.program_guard(fluid.Program()): np_x = np.array([0.1]) @@ -181,7 +173,7 @@ class TestParameter(object): out = eval("paddle.%s(data, name='Y')" % self.op_type) place = fluid.CPUPlace() exe = fluid.Executor(place) - result, = exe.run(feed={"X": np_x}, fetch_list=[out]) + (result,) = exe.run(feed={"X": np_x}, fetch_list=[out]) expected = eval("np.%s(np_x)" % self.op_type) np.testing.assert_allclose(result, expected, rtol=1e-05) @@ -195,7 +187,6 @@ class TestParameter(object): class TestSigmoid(TestActivation): - def setUp(self): self.op_type = "sigmoid" self.init_dtype() @@ -218,15 +209,14 @@ class TestSigmoid(TestActivation): class TestSigmoid_ZeroDim(TestSigmoid): - def init_shape(self): self.shape = [] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSigmoidBF16(OpTest): - def setUp(self): self.op_type = "sigmoid" self.init_dtype() @@ -265,7 +255,6 @@ class TestSigmoidBF16_ZeroDim(TestSigmoidBF16): class TestSilu(TestActivation): - def setUp(self): self.op_type = "silu" self.init_dtype() @@ -288,7 +277,6 @@ class TestSilu(TestActivation): class TestSilu_ZeroDim(TestSilu): - def init_shape(self): self.shape = [] @@ -297,8 +285,11 @@ class TestSiluAPI(unittest.TestCase): # test paddle.nn.Silu, paddle.nn.functional.silu def setUp(self): self.x_np = np.random.uniform(-1, 1, [11, 17]).astype('float32') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -329,19 +320,18 @@ class TestSiluAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.silu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[11, 17], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[11, 17], dtype='int32' + ) self.assertRaises(TypeError, F.silu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[11, 17], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[11, 17], dtype='float16' + ) F.silu(x_fp16) class TestLogSigmoid(TestActivation): - def setUp(self): self.op_type = "logsigmoid" self.init_dtype() @@ -361,7 +351,6 @@ class TestLogSigmoid(TestActivation): class TestLogSigmoid_ZeroDim(TestLogSigmoid): - def init_shape(self): self.shape = [] @@ -371,8 +360,11 @@ class TestLogSigmoidAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [11, 17]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -414,19 +406,18 @@ class TestLogSigmoidAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.log_sigmoid, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[11, 17], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[11, 17], dtype='int32' + ) self.assertRaises(TypeError, F.log_sigmoid, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[11, 17], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[11, 17], dtype='float16' + ) F.log_sigmoid(x_fp16) class TestTanh(TestActivation, TestParameter): - def setUp(self): self.op_type = "tanh" self.init_dtype() @@ -445,14 +436,13 @@ class TestTanh(TestActivation, TestParameter): self.check_grad(['X'], 'Out') def init_dtype(self): - #TODO If dtype is float64, the output (Out) has diff at CPUPlace + # TODO If dtype is float64, the output (Out) has diff at CPUPlace # when using and not using inplace. Therefore, set dtype as float32 # for now. self.dtype = np.float32 class TestTanh_ZeroDim(TestTanh): - def init_shape(self): self.shape = [] @@ -463,8 +453,11 @@ class TestTanhAPI(unittest.TestCase): self.dtype = 'float32' np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(self.dtype) - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.executed_api() def executed_api(self): @@ -511,14 +504,14 @@ class TestTanhAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, self.tanh, 1) # The input dtype must be float16, float32. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, self.tanh, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) self.tanh(x_fp16) @@ -529,7 +522,6 @@ class TestTanhInplaceAPI(TestTanhAPI): class TestAtan(TestActivation, TestParameter): - def setUp(self): self.op_type = "atan" self.init_dtype() @@ -554,7 +546,7 @@ class TestAtan(TestActivation, TestParameter): out = paddle.atan(data, name='Y') place = fluid.CPUPlace() exe = fluid.Executor(place) - result, = exe.run(feed={"X": np_x}, fetch_list=[out]) + (result,) = exe.run(feed={"X": np_x}, fetch_list=[out]) expected = np.arctan(np_x) self.assertEqual(result, expected) @@ -568,13 +560,11 @@ class TestAtan(TestActivation, TestParameter): class TestAtan_ZeroDim(TestTanh): - def init_shape(self): self.shape = [] class TestSinh(TestActivation): - def setUp(self): self.op_type = "sinh" self.init_dtype() @@ -594,13 +584,11 @@ class TestSinh(TestActivation): class TestSinh_ZeroDim(TestSinh): - def init_shape(self): self.shape = [] class TestSinhAPI(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): np_x = np.array([0.1]) @@ -612,19 +600,24 @@ class TestSinhAPI(unittest.TestCase): def test_api(self): test_data_shape = [11, 17] with fluid.program_guard(fluid.Program(), fluid.Program()): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") - data_x = fluid.layers.data(name="data_x", - shape=test_data_shape, - append_batch_size=False, - dtype="float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) + data_x = fluid.layers.data( + name="data_x", + shape=test_data_shape, + append_batch_size=False, + dtype="float32", + ) pd_sinh_out = fluid.layers.sinh(data_x) exe = fluid.Executor(place=fluid.CPUPlace()) exe.run(fluid.default_startup_program()) - np_sinh_res, = exe.run(fluid.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[pd_sinh_out]) + (np_sinh_res,) = exe.run( + fluid.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[pd_sinh_out], + ) expected_res = np.sinh(input_x) np.testing.assert_allclose(np_sinh_res, expected_res, rtol=1e-05) @@ -632,8 +625,9 @@ class TestSinhAPI(unittest.TestCase): def test_backward(self): test_data_shape = [11, 17] with fluid.dygraph.guard(): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) var = fluid.dygraph.to_variable(input_x) var.stop_gradient = False loss = fluid.layers.sinh(var) @@ -643,7 +637,6 @@ class TestSinhAPI(unittest.TestCase): class TestSinhOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program()): # The input type must be Variable. @@ -657,7 +650,6 @@ class TestSinhOpError(unittest.TestCase): class TestCosh(TestActivation): - def setUp(self): self.op_type = "cosh" self.init_dtype() @@ -677,13 +669,11 @@ class TestCosh(TestActivation): class TestCosh_ZeroDim(TestCosh): - def init_shape(self): self.shape = [] class TestCoshAPI(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): np_x = np.array([0.1]) @@ -695,19 +685,24 @@ class TestCoshAPI(unittest.TestCase): def test_api(self): test_data_shape = [11, 17] with fluid.program_guard(fluid.Program(), fluid.Program()): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") - data_x = fluid.layers.data(name="data_x", - shape=test_data_shape, - append_batch_size=False, - dtype="float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) + data_x = fluid.layers.data( + name="data_x", + shape=test_data_shape, + append_batch_size=False, + dtype="float32", + ) pd_cosh_out = paddle.cosh(data_x) exe = fluid.Executor(place=fluid.CPUPlace()) exe.run(fluid.default_startup_program()) - np_cosh_res, = exe.run(fluid.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[pd_cosh_out]) + (np_cosh_res,) = exe.run( + fluid.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[pd_cosh_out], + ) expected_res = np.cosh(input_x) np.testing.assert_allclose(np_cosh_res, expected_res, rtol=1e-05) @@ -715,8 +710,9 @@ class TestCoshAPI(unittest.TestCase): def test_backward(self): test_data_shape = [11, 17] with fluid.dygraph.guard(): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) var = fluid.dygraph.to_variable(input_x) var.stop_gradient = False loss = fluid.layers.cosh(var) @@ -726,7 +722,6 @@ class TestCoshAPI(unittest.TestCase): class TestCoshOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program()): # The input type must be Variable. @@ -745,7 +740,6 @@ def ref_tanhshrink(x): class TestTanhshrink(TestActivation): - def setUp(self): self.op_type = "tanh_shrink" self.init_dtype() @@ -765,7 +759,6 @@ class TestTanhshrink(TestActivation): class TestTanhshrink_ZeroDim(TestTanhshrink): - def init_shape(self): self.shape = [] @@ -775,8 +768,11 @@ class TestTanhshrinkAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(10, 20, [10, 17]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -818,14 +814,14 @@ class TestTanhshrinkAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.tanhshrink, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.tanhshrink, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.tanhshrink(x_fp16) @@ -836,7 +832,6 @@ def ref_hardshrink(x, threshold): class TestHardShrink(TestActivation): - def setUp(self): self.op_type = "hard_shrink" self.init_dtype() @@ -865,7 +860,6 @@ class TestHardShrink(TestActivation): class TestHardShrink_threshold_negative(TestHardShrink): - def set_attrs(self): self.threshold = -0.1 @@ -883,8 +877,11 @@ class TestHardShrinkAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -933,14 +930,14 @@ class TestHardShrinkAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.hardshrink, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardshrink, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardshrink(x_fp16) @@ -957,8 +954,11 @@ class TestHardtanhAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-3, 3, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -997,26 +997,26 @@ class TestHardtanhAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.hardtanh, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardtanh, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardtanh(x_fp16) def ref_softshrink(x, threshold=0.5): out = np.copy(x) out = (out < -threshold) * (out + threshold) + (out > threshold) * ( - out - threshold) + out - threshold + ) return out class TestSoftshrink(TestActivation): - def setUp(self): self.op_type = "softshrink" self.check_eager = True @@ -1040,7 +1040,6 @@ class TestSoftshrink(TestActivation): class TestSoftshrink_ZeroDim(TestSoftshrink): - def init_shape(self): self.shape = [] @@ -1051,8 +1050,11 @@ class TestSoftshrinkAPI(unittest.TestCase): self.threshold = 0.8 np.random.seed(1024) self.x_np = np.random.uniform(0.25, 10, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -1094,24 +1096,23 @@ class TestSoftshrinkAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.softshrink, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.softshrink, x_int32) # The threshold must be no less than zero - x_fp32 = paddle.fluid.data(name='x_fp32', - shape=[12, 10], - dtype='float32') + x_fp32 = paddle.fluid.data( + name='x_fp32', shape=[12, 10], dtype='float32' + ) self.assertRaises(ValueError, F.softshrink, x_fp32, -1.0) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.softshrink(x_fp16) class TestSqrt(TestActivation, TestParameter): - def setUp(self): self.op_type = "sqrt" self.python_api = paddle.sqrt @@ -1135,15 +1136,14 @@ class TestSqrt(TestActivation, TestParameter): class TestSqrt_ZeroDim(TestSqrt): - def init_shape(self): self.shape = [] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSqrtBF16(OpTest): - def setUp(self): self.op_type = "sqrt" self.python_api = paddle.sqrt @@ -1175,7 +1175,6 @@ class TestSqrtBF16(OpTest): class TestRsqrt(TestActivation): - def setUp(self): self.op_type = "rsqrt" self.python_api = paddle.rsqrt @@ -1195,10 +1194,9 @@ class TestRsqrt(TestActivation): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad(['X'], - 'Out', - max_relative_error=0.0005, - check_eager=True) + self.check_grad( + ['X'], 'Out', max_relative_error=0.0005, check_eager=True + ) ''' @@ -1210,7 +1208,6 @@ class TestRsqrt_ZeroDim(TestRsqrt): class TestAbs(TestActivation): - def setUp(self): self.op_type = "abs" self.init_dtype() @@ -1238,13 +1235,11 @@ class TestAbs(TestActivation): class TestAbs_ZeroDim(TestAbs): - def init_shape(self): self.shape = [] class TestCeil(TestActivation): - def setUp(self): self.op_type = "ceil" self.check_eager = True @@ -1268,13 +1263,11 @@ class TestCeil(TestActivation): class TestCeil_ZeroDim(TestCeil): - def init_shape(self): self.shape = [] class TestFloor(TestActivation): - def setUp(self): self.op_type = "floor" self.check_eager = True @@ -1300,13 +1293,11 @@ class TestFloor(TestActivation): class TestFloor_ZeroDim(TestFloor): - def init_shape(self): self.shape = [] class TestCos(TestActivation): - def setUp(self): self.op_type = "cos" self.init_dtype() @@ -1329,13 +1320,11 @@ class TestCos(TestActivation): class TestCos_ZeroDim(TestCos): - def init_shape(self): self.shape = [] class TestTan(TestActivation): - def setUp(self): np.random.seed(1024) self.op_type = "tan" @@ -1344,8 +1333,11 @@ class TestTan(TestActivation): self.dtype = 'float32' self.x_np = np.random.uniform(-1, 1, self.shape).astype(self.dtype) - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) out = np.tan(self.x_np) @@ -1362,19 +1354,20 @@ class TestTan(TestActivation): class TestTan_ZeroDim(TestTan): - def init_shape(self): self.shape = [] class TestTanAPI(unittest.TestCase): - def setUp(self): np.random.seed(1024) self.dtype = 'float32' self.x_np = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_dygraph_api(self): paddle.disable_static(self.place) @@ -1397,8 +1390,9 @@ class TestTanAPI(unittest.TestCase): def test_backward(self): test_data_shape = [11, 17] with fluid.dygraph.guard(): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) var = paddle.to_tensor(input_x) var.stop_gradient = False loss = paddle.tan(var) @@ -1408,7 +1402,6 @@ class TestTanAPI(unittest.TestCase): class TestAcos(TestActivation): - def setUp(self): self.op_type = "acos" self.init_dtype() @@ -1431,13 +1424,11 @@ class TestAcos(TestActivation): class TestAcos_ZeroDim(TestAcos): - def init_shape(self): self.shape = [] class TestSin(TestActivation, TestParameter): - def setUp(self): self.op_type = "sin" self.init_dtype() @@ -1460,13 +1451,11 @@ class TestSin(TestActivation, TestParameter): class TestSin_ZeroDim(TestSin): - def init_shape(self): self.shape = [] class TestAsin(TestActivation): - def setUp(self): self.op_type = "asin" self.init_dtype() @@ -1489,13 +1478,11 @@ class TestAsin(TestActivation): class TestAsin_ZeroDim(TestAsin): - def init_shape(self): self.shape = [] class TestAcosh(TestActivation): - def setUp(self): self.op_type = "acosh" self.init_dtype() @@ -1518,13 +1505,11 @@ class TestAcosh(TestActivation): class TestAcosh_ZeroDim(TestAcosh): - def init_shape(self): self.shape = [] class TestAsinh(TestActivation): - def setUp(self): self.op_type = "asinh" self.init_dtype() @@ -1547,13 +1532,11 @@ class TestAsinh(TestActivation): class TestAsinh_ZeroDim(TestAsinh): - def init_shape(self): self.shape = [] class TestAtanh(TestActivation): - def setUp(self): self.op_type = "atanh" self.init_dtype() @@ -1576,13 +1559,11 @@ class TestAtanh(TestActivation): class TestAtanh_ZeroDim(TestAtanh): - def init_shape(self): self.shape = [] class TestRound(TestActivation): - def setUp(self): self.op_type = "round" self.check_eager = True @@ -1605,13 +1586,11 @@ class TestRound(TestActivation): class TestRound_ZeroDim(TestRound): - def init_shape(self): self.shape = [] class TestRelu(TestActivation): - def setUp(self): self.op_type = "relu" self.init_dtype() @@ -1640,7 +1619,6 @@ class TestRelu(TestActivation): class TestRelu_ZeroDim(TestRelu): - def init_shape(self): self.shape = [] @@ -1650,8 +1628,11 @@ class TestReluAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.executed_api() def executed_api(self): @@ -1687,14 +1668,14 @@ class TestReluAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, self.relu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[10, 12], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[10, 12], dtype='int32' + ) self.assertRaises(TypeError, self.relu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[10, 12], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[10, 12], dtype='float16' + ) self.relu(x_fp16) @@ -1711,7 +1692,6 @@ def ref_leaky_relu(x, alpha=0.01): class TestLeakyRelu(TestActivation): - def get_alpha(self): return 0.02 @@ -1738,25 +1718,21 @@ class TestLeakyRelu(TestActivation): class TestLeakyReluAlpha1(TestLeakyRelu): - def get_alpha(self): return 2 class TestLeakyReluAlpha2(TestLeakyRelu): - def get_alpha(self): return -0.01 class TestLeakyReluAlpha3(TestLeakyRelu): - def get_alpha(self): return -2.0 class TestLeakyRelu_ZeroDim(TestLeakyRelu): - def init_shape(self): self.shape = [] @@ -1767,8 +1743,11 @@ class TestLeakyReluAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -1817,28 +1796,33 @@ class TestLeakyReluAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.leaky_relu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.leaky_relu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.leaky_relu(x_fp16) def gelu(x, approximate): if approximate: - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * ( + 1.0 + + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))) + ) + ) else: y_ref = 0.5 * x * (1 + erf(x / np.sqrt(2))) return y_ref.astype(x.dtype) class TestGeluApproximate(TestActivation): - def setUp(self): self.op_type = "gelu" self.init_dtype() @@ -1859,7 +1843,6 @@ class TestGeluApproximate(TestActivation): class TestGelu(TestActivation): - def setUp(self): self.op_type = "gelu" self.init_dtype() @@ -1880,7 +1863,6 @@ class TestGelu(TestActivation): class TestGelu_ZeroDim(TestGelu): - def init_shape(self): self.shape = [] @@ -1890,8 +1872,11 @@ class TestGELUAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [11, 17]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -1930,19 +1915,18 @@ class TestGELUAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.gelu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[11, 17], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[11, 17], dtype='int32' + ) self.assertRaises(TypeError, F.gelu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[11, 17], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[11, 17], dtype='float16' + ) F.gelu(x_fp16) class TestBRelu(TestActivation): - def setUp(self): self.op_type = "brelu" self.init_dtype() @@ -1972,15 +1956,18 @@ class TestBreluAPI(unittest.TestCase): # test paddle.fluid.layers.brelu def setUp(self): np.random.seed(1024) - self.t_min = 0. - self.t_max = 24. + self.t_min = 0.0 + self.t_max = 24.0 self.x_np = np.random.uniform(-1, 30, [10, 12]).astype('float32') self.out_ref = np.copy(self.x_np) self.out_ref[self.out_ref < self.t_min] = self.t_min self.out_ref[self.out_ref > self.t_max] = self.t_max self.out_ref = self.out_ref.astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_fluid_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -2004,9 +1991,9 @@ class TestBreluAPI(unittest.TestCase): x_int32 = fluid.data(name='x_int32', shape=[12, 10], dtype='int32') self.assertRaises(TypeError, fluid.layers.brelu, x_int32) # support the input dtype is float16 - x_fp16 = fluid.layers.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = fluid.layers.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) fluid.layers.brelu(x_fp16) @@ -2018,7 +2005,6 @@ def ref_relu6(x, threshold=6.0): class TestRelu6(TestActivation): - def setUp(self): self.op_type = "relu6" self.init_dtype() @@ -2044,7 +2030,6 @@ class TestRelu6(TestActivation): class TestRelu6_ZeroDim(TestRelu6): - def init_shape(self): self.shape = [] @@ -2055,8 +2040,11 @@ class TestRelu6API(unittest.TestCase): np.random.seed(1024) self.x_np = np.random.uniform(-1, 10, [10, 12]).astype(np.float64) self.x_np[np.abs(self.x_np) < 0.005] = 0.02 - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -2098,14 +2086,14 @@ class TestRelu6API(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.relu6, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.relu6, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.relu6(x_fp16) @@ -2114,12 +2102,12 @@ def ref_hardswish(x, threshold=6.0, scale=6.0, offset=3.0): if x_dtype == 'float16': x_dtype = 'float16' x = x.astype('float32') - return (x * np.minimum(np.maximum(x + offset, 0.), threshold) / - scale).astype(x_dtype) + return ( + x * np.minimum(np.maximum(x + offset, 0.0), threshold) / scale + ).astype(x_dtype) class TestHardSwish(TestActivation): - def setUp(self): self.op_type = 'hard_swish' self.init_dtype() @@ -2131,7 +2119,7 @@ class TestHardSwish(TestActivation): threshold = 6.0 scale = 6.0 offset = 3.0 - #the same with TestAbs + # the same with TestAbs x[np.abs(x + offset) < 0.005] = 0.02 x[np.abs(x - threshold + offset) < 0.005] = threshold - offset + 0.02 out = ref_hardswish(x, threshold, scale, offset) @@ -2151,7 +2139,6 @@ class TestHardSwish(TestActivation): class TestHardSwish_ZeroDim(TestHardSwish): - def init_shape(self): self.shape = [] @@ -2160,8 +2147,11 @@ class TestHardswishAPI(unittest.TestCase): # test paddle.nn.Hardswish, paddle.nn.functional.hardswish def setUp(self): self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -2177,11 +2167,11 @@ class TestHardswishAPI(unittest.TestCase): def test_dygraph_api(self): paddle.disable_static(self.place) - x = paddle.to_tensor([11648., 11448.]) + x = paddle.to_tensor([11648.0, 11448.0]) out1 = F.hardswish(x) m = paddle.nn.Hardswish() out2 = m(x) - out_ref = [11648., 11448.] + out_ref = [11648.0, 11448.0] for r in [out1, out2]: np.testing.assert_allclose(out_ref, r.numpy(), rtol=1e-05) paddle.enable_static() @@ -2206,14 +2196,14 @@ class TestHardswishAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.hardswish, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardswish, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardswish(x_fp16) def test_api_eager_dygraph(self): @@ -2223,7 +2213,6 @@ class TestHardswishAPI(unittest.TestCase): class TestSoftRelu(TestActivation): - def setUp(self): self.op_type = "soft_relu" self.init_dtype() @@ -2250,7 +2239,6 @@ class TestSoftRelu(TestActivation): class TestSoftReluOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program()): # The input type must be Variable. @@ -2269,7 +2257,6 @@ def elu(x, alpha): class TestELU(TestActivation): - def setUp(self): self.op_type = "elu" self.init_dtype() @@ -2294,17 +2281,15 @@ class TestELU(TestActivation): self.check_grad(['X'], 'Out') def get_alpha(self): - return 1. + return 1.0 class TestELUAlpha(TestELU): - def get_alpha(self): return -0.2 class TestELU_ZeroDim(TestELU): - def init_shape(self): self.shape = [] @@ -2314,8 +2299,11 @@ class TestELUAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-3, 3, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.executed_api() def executed_api(self): @@ -2360,14 +2348,14 @@ class TestELUAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, self.elu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[10, 12], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[10, 12], dtype='int32' + ) self.assertRaises(TypeError, self.elu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[10, 12], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[10, 12], dtype='float16' + ) self.elu(x_fp16) @@ -2389,7 +2377,6 @@ def celu(x, alpha): class TestCELU(TestActivation): - def setUp(self): self.op_type = "celu" self.init_dtype() @@ -2414,7 +2401,6 @@ class TestCELU(TestActivation): class TestCELU_ZeroDim(TestCELU): - def init_shape(self): self.shape = [] @@ -2424,8 +2410,11 @@ class TestCELUAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-3, 3, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.executed_api() def executed_api(self): @@ -2470,19 +2459,19 @@ class TestCELUAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, self.celu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[10, 12], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[10, 12], dtype='int32' + ) self.assertRaises(TypeError, self.celu, x_int32) # The alpha must be not equal 0 - x_fp32 = paddle.fluid.data(name='x_fp32', - shape=[10, 12], - dtype='float32') + x_fp32 = paddle.fluid.data( + name='x_fp32', shape=[10, 12], dtype='float32' + ) self.assertRaises(ZeroDivisionError, F.celu, x_fp32, 0) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[10, 12], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[10, 12], dtype='float16' + ) self.celu(x_fp16) def test_api_eager_dygraph(self): @@ -2492,7 +2481,6 @@ class TestCELUAPI(unittest.TestCase): class TestReciprocal(TestActivation): - def setUp(self): self.op_type = "reciprocal" self.python_api = paddle.reciprocal @@ -2516,13 +2504,11 @@ class TestReciprocal(TestActivation): class TestReciprocal_ZeroDim(TestReciprocal): - def init_shape(self): self.shape = [] class TestLog(TestActivation): - def setUp(self): self.op_type = "log" self.check_eager = True @@ -2543,27 +2529,23 @@ class TestLog(TestActivation): self.check_grad(['X'], 'Out', check_eager=True) def test_error(self): - in1 = fluid.layers.data(name="in1", - shape=[11, 17], - append_batch_size=False, - dtype="int32") - in2 = fluid.layers.data(name="in2", - shape=[11, 17], - append_batch_size=False, - dtype="int64") + in1 = fluid.layers.data( + name="in1", shape=[11, 17], append_batch_size=False, dtype="int32" + ) + in2 = fluid.layers.data( + name="in2", shape=[11, 17], append_batch_size=False, dtype="int64" + ) self.assertRaises(TypeError, fluid.layers.log, in1) self.assertRaises(TypeError, fluid.layers.log, in2) class TestLog_ZeroDim(TestLog): - def init_shape(self): self.shape = [] class TestLog2(TestActivation): - def setUp(self): self.op_type = "log2" self.check_eager = True @@ -2590,19 +2572,22 @@ class TestLog2(TestActivation): self.assertRaises(TypeError, paddle.log2, in2) def test_api(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float64") - data_x = paddle.static.data(name="data_x", - shape=[11, 17], - dtype="float64") + data_x = paddle.static.data( + name="data_x", shape=[11, 17], dtype="float64" + ) out1 = paddle.log2(data_x) exe = paddle.static.Executor(place=fluid.CPUPlace()) exe.run(paddle.static.default_startup_program()) - res1, = exe.run(paddle.static.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + (res1,) = exe.run( + paddle.static.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log2(input_x) np.testing.assert_allclose(res1, expected_res, rtol=1e-05) @@ -2617,13 +2602,11 @@ class TestLog2(TestActivation): class TestLog2_ZeroDim(TestLog2): - def init_shape(self): self.shape = [] class TestLog10(TestActivation): - def setUp(self): self.op_type = "log10" self.check_eager = True @@ -2644,13 +2627,11 @@ class TestLog10(TestActivation): class TestLog10_ZeroDim(TestLog10): - def init_shape(self): self.shape = [] class TestLog10API(unittest.TestCase): - def test_error(self): in1 = paddle.static.data(name="in1", shape=[11, 17], dtype="int32") in2 = paddle.static.data(name="in2", shape=[11, 17], dtype="int64") @@ -2659,19 +2640,22 @@ class TestLog10API(unittest.TestCase): self.assertRaises(TypeError, paddle.log10, in2) def test_api(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float64") - data_x = paddle.static.data(name="data_x", - shape=[11, 17], - dtype="float64") + data_x = paddle.static.data( + name="data_x", shape=[11, 17], dtype="float64" + ) out1 = paddle.log10(data_x) exe = paddle.static.Executor(place=paddle.CPUPlace()) exe.run(paddle.static.default_startup_program()) - res1, = exe.run(paddle.static.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + (res1,) = exe.run( + paddle.static.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log10(input_x) np.testing.assert_allclose(res1, expected_res, rtol=1e-05) @@ -2686,7 +2670,6 @@ class TestLog10API(unittest.TestCase): class TestLog1p(TestActivation): - def setUp(self): self.op_type = "log1p" self.check_eager = True @@ -2708,27 +2691,29 @@ class TestLog1p(TestActivation): class TestLog1p_ZeroDim(TestLog1p): - def init_shape(self): self.shape = [] class TestLog1pAPI(unittest.TestCase): - def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float64") - data_x = fluid.layers.data(name="data_x", - shape=[11, 17], - append_batch_size=False, - dtype="float64") + data_x = fluid.layers.data( + name="data_x", + shape=[11, 17], + append_batch_size=False, + dtype="float64", + ) out1 = paddle.log1p(data_x) exe = fluid.Executor(place=fluid.CPUPlace()) exe.run(fluid.default_startup_program()) - res1, = exe.run(fluid.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + (res1,) = exe.run( + fluid.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log1p(input_x) np.testing.assert_allclose(res1, expected_res, rtol=1e-05) @@ -2743,7 +2728,6 @@ class TestLog1pAPI(unittest.TestCase): class TestSquare(TestActivation): - def setUp(self): self.op_type = "square" self.python_api = paddle.square @@ -2760,25 +2744,23 @@ class TestSquare(TestActivation): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad(['X'], - 'Out', - max_relative_error=0.007, - check_eager=True) + self.check_grad( + ['X'], 'Out', max_relative_error=0.007, check_eager=True + ) def test_check_output(self): self.check_output(check_eager=True) class TestSquare_ZeroDim(TestSquare): - def init_shape(self): self.shape = [] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSquareBF16(OpTest): - def setUp(self): self.op_type = "square" self.python_api = paddle.square @@ -2802,14 +2784,12 @@ class TestSquareBF16(OpTest): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - numeric_grad_delta=0.5, - check_eager=True) + self.check_grad_with_place( + place, ['X'], 'Out', numeric_grad_delta=0.5, check_eager=True + ) class TestPow(TestActivation): - def setUp(self): self.op_type = "pow" self.python_api = paddle.pow @@ -2835,13 +2815,11 @@ class TestPow(TestActivation): class TestPow_ZeroDim(TestPow): - def init_shape(self): self.shape = [] class TestPow_factor_tensor(TestActivation): - def setUp(self): self.op_type = "pow" self.check_eager = False @@ -2854,7 +2832,7 @@ class TestPow_factor_tensor(TestActivation): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'FactorTensor': np.array([3.0]).astype("float32") + 'FactorTensor': np.array([3.0]).astype("float32"), } self.attrs = {} @@ -2870,14 +2848,12 @@ class TestPow_factor_tensor(TestActivation): def test_api(self): input = np.random.uniform(1, 2, [11, 17]).astype("float32") - x = fluid.layers.data(name="x", - shape=[11, 17], - append_batch_size=False, - dtype="float32") - res = fluid.layers.data(name="res", - shape=[11, 17], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name="x", shape=[11, 17], append_batch_size=False, dtype="float32" + ) + res = fluid.layers.data( + name="res", shape=[11, 17], append_batch_size=False, dtype="float32" + ) factor_1 = 2.0 factor_2 = fluid.layers.fill_constant([1], "float32", 3.0) @@ -2891,29 +2867,26 @@ class TestPow_factor_tensor(TestActivation): res_1, res_2, res, res_6 = exe.run( fluid.default_main_program(), feed={"x": input}, - fetch_list=[out_1, out_2, res, out_6]) + fetch_list=[out_1, out_2, res, out_6], + ) assert np.allclose(res_1, np.power(input, 2)) assert np.allclose(res_2, np.power(input, 3)) assert np.allclose(res_6, np.power(input, 3)) def test_error(self): - in1 = fluid.layers.data(name="in1", - shape=[11, 17], - append_batch_size=False, - dtype="int32") - in2 = fluid.layers.data(name="in2", - shape=[11, 17], - append_batch_size=False, - dtype="int64") - in3 = fluid.layers.data(name="in3", - shape=[11, 17], - append_batch_size=False, - dtype="float32") - in4 = fluid.layers.data(name="in4", - shape=[11, 17], - append_batch_size=False, - dtype="float64") + in1 = fluid.layers.data( + name="in1", shape=[11, 17], append_batch_size=False, dtype="int32" + ) + in2 = fluid.layers.data( + name="in2", shape=[11, 17], append_batch_size=False, dtype="int64" + ) + in3 = fluid.layers.data( + name="in3", shape=[11, 17], append_batch_size=False, dtype="float32" + ) + in4 = fluid.layers.data( + name="in4", shape=[11, 17], append_batch_size=False, dtype="float64" + ) factor_1 = fluid.layers.fill_constant([1], "float64", 3.0) @@ -2929,7 +2902,6 @@ def ref_stanh(x, scale_a=0.67, scale_b=1.7159): class TestSTanh(TestActivation): - def get_scale_a(self): return 0.67 @@ -2960,19 +2932,16 @@ class TestSTanh(TestActivation): class TestSTanhScaleA(TestSTanh): - def get_scale_a(self): return 2.0 class TestSTanhScaleB(TestSTanh): - def get_scale_b(self): return 0.5 class TestSTanh_ZeroDim(TestSTanh): - def init_shape(self): self.shape = [] @@ -2990,8 +2959,11 @@ class TestSTanhAPI(unittest.TestCase): self.x_np = np.random.uniform(-1, 1, [10, 12]).astype('float32') self.scale_a = self.get_scale_a() self.scale_b = self.get_scale_b() - self.place=paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3029,38 +3001,37 @@ class TestSTanhAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, paddle.stanh, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, paddle.stanh, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) paddle.stanh(x_fp16) class TestSTanhAPIScaleA(TestSTanhAPI): - def get_scale_a(self): return 2.0 class TestSTanhAPIScaleB(TestSTanhAPI): - def get_scale_b(self): return 0.5 def ref_softplus(x, beta=1, threshold=20): x_beta = beta * x - out = np.select([x_beta <= threshold, x_beta > threshold], - [np.log(1 + np.exp(x_beta)) / beta, x]) + out = np.select( + [x_beta <= threshold, x_beta > threshold], + [np.log(1 + np.exp(x_beta)) / beta, x], + ) return out class TestSoftplus(TestActivation): - def setUp(self): self.op_type = "softplus" self.python_api = paddle.nn.functional.softplus @@ -3091,15 +3062,14 @@ class TestSoftplus(TestActivation): class TestSoftplus_ZeroDim(TestSoftplus): - def init_shape(self): self.shape = [] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftplusBF16(OpTest): - def setUp(self): self.op_type = "softplus" self.init_dtype() @@ -3133,8 +3103,11 @@ class TestSoftplusAPI(unittest.TestCase): self.threshold = 15 np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3176,14 +3149,14 @@ class TestSoftplusAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.softplus, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.softplus, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.softplus(x_fp16) @@ -3193,7 +3166,6 @@ def ref_softsign(x): class TestSoftsign(TestActivation): - def setUp(self): self.op_type = "softsign" self.init_dtype() @@ -3217,7 +3189,6 @@ class TestSoftsign(TestActivation): class TestSoftsign_ZeroDim(TestSoftsign): - def init_shape(self): self.shape = [] @@ -3227,8 +3198,11 @@ class TestSoftsignAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3270,14 +3244,14 @@ class TestSoftsignAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.softsign, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.softsign, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.softsign(x_fp16) @@ -3287,7 +3261,6 @@ def ref_thresholded_relu(x, threshold=1.0): class TestThresholdedRelu(TestActivation): - def setUp(self): self.op_type = "thresholded_relu" self.init_dtype() @@ -3313,7 +3286,6 @@ class TestThresholdedRelu(TestActivation): class TestThresholdedRelu_ZeroDim(TestThresholdedRelu): - def init_shape(self): self.shape = [] @@ -3325,8 +3297,11 @@ class TestThresholdedReluAPI(unittest.TestCase): np.random.seed(1024) self.x_np = np.random.uniform(-20, 20, [10, 12]).astype(np.float64) self.x_np[np.abs(self.x_np) < 0.005] = 0.02 - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3368,23 +3343,22 @@ class TestThresholdedReluAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.thresholded_relu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.thresholded_relu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.thresholded_relu(x_fp16) def ref_hardsigmoid(x, slope=0.166666666666667, offset=0.5): - return np.maximum(np.minimum(x * slope + offset, 1.), 0.).astype(x.dtype) + return np.maximum(np.minimum(x * slope + offset, 1.0), 0.0).astype(x.dtype) class TestHardSigmoid(TestActivation): - def setUp(self): self.op_type = "hard_sigmoid" self.dtype = 'float64' @@ -3395,7 +3369,7 @@ class TestHardSigmoid(TestActivation): x = np.random.uniform(-5, 5, self.shape).astype(self.dtype) lower_threshold = -self.offset / self.slope - upper_threshold = (1. - self.offset) / self.slope + upper_threshold = (1.0 - self.offset) / self.slope # Same reason as TestAbs delta = 0.005 @@ -3416,20 +3390,17 @@ class TestHardSigmoid(TestActivation): class TestHardSigmoidFP32(TestHardSigmoid): - def set_attrs(self): self.dtype = 'float32' class TestHardSigmoidSlopeOffset(TestHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.4 class TestHardSigmoid_ZeroDim(TestHardSigmoid): - def init_shape(self): self.shape = [] @@ -3438,8 +3409,11 @@ class TestHardsigmoidAPI(unittest.TestCase): # test paddle.nn.Hardsigmoid, paddle.nn.functional.hardsigmoid def setUp(self): self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -3484,14 +3458,14 @@ class TestHardsigmoidAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.hardsigmoid, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardsigmoid, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardsigmoid(x_fp16) @@ -3501,7 +3475,6 @@ def ref_swish(x): class TestSwish(TestActivation): - def setUp(self): self.op_type = "swish" self.python_api = paddle.nn.functional.swish @@ -3530,7 +3503,6 @@ class TestSwish(TestActivation): class TestSwish_ZeroDim(TestSwish): - def init_shape(self): self.shape = [] @@ -3540,8 +3512,11 @@ class TestSwishAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3588,25 +3563,25 @@ class TestSwishAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.swish, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.swish, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.swish(x_fp16) -def ref_mish(x, threshold=20.): - softplus = np.select([x <= threshold, x > threshold], - [np.log(1 + np.exp(x)), x]) +def ref_mish(x, threshold=20.0): + softplus = np.select( + [x <= threshold, x > threshold], [np.log(1 + np.exp(x)), x] + ) return x * np.tanh(softplus) class TestMish(TestActivation): - def setUp(self): self.op_type = "mish" self.python_api = paddle.fluid.layers.nn.mish @@ -3632,7 +3607,6 @@ class TestMish(TestActivation): class TestMish_ZeroDim(TestMish): - def init_shape(self): self.shape = [] @@ -3642,8 +3616,11 @@ class TestMishAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3685,32 +3662,30 @@ class TestMishAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.mish, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.mish, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.mish(x_fp16) -#------------------ Test Error Activation---------------------- +# ------------------ Test Error Activation---------------------- def create_test_error_class(op_type): - class TestOpErrors(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): op = getattr(fluid.layers, op_type) # The input dtype of op_type must be float32, float64. - in1 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") - in2 = fluid.layers.data(name='input3', - shape=[12, 10], - dtype="int64") + in1 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) + in2 = fluid.layers.data( + name='input3', shape=[12, 10], dtype="int64" + ) self.assertRaises(TypeError, op, in1) self.assertRaises(TypeError, op, in2) @@ -3737,13 +3712,12 @@ create_test_error_class('asinh') create_test_error_class('atanh') -#------------------ Test Cudnn Activation---------------------- +# ------------------ Test Cudnn Activation---------------------- def create_test_act_cudnn_class(parent, atol=1e-3, grad_atol=1e-3): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestActCudnn(parent): - def init_kernel_type(self): self.attrs = {"use_cudnn": True} @@ -3758,16 +3732,14 @@ create_test_act_cudnn_class(TestSigmoid) create_test_act_cudnn_class(TestTanh) -#------------------ Test Fp16 ---------------------- -def create_test_act_fp16_class(parent, - atol=1e-3, - grad_check=True, - grad_atol=0.80): - - @unittest.skipIf(not paddle.is_compiled_with_cuda(), - "core is not compiled with CUDA") +# ------------------ Test Fp16 ---------------------- +def create_test_act_fp16_class( + parent, atol=1e-3, grad_check=True, grad_atol=0.80 +): + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestActFp16(parent): - def init_dtype(self): self.dtype = np.float16 @@ -3781,9 +3753,9 @@ def create_test_act_fp16_class(parent, place = core.CUDAPlace(0) support_fp16 = core.is_float16_supported(place) if support_fp16 and grad_check: - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=grad_atol) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=grad_atol + ) cls_name = "{0}_{1}".format(parent.__name__, "fp16") TestActFp16.__name__ = cls_name @@ -3843,15 +3815,13 @@ create_test_act_fp16_class(TestHardSwish) create_test_act_fp16_class(TestMish, grad_atol=0.9) -def create_test_act_bf16_class(parent, - atol=1e-2, - grad_check=True, - grad_atol=0.80): - - @unittest.skipIf(not paddle.is_compiled_with_cuda(), - "core is not compiled with CUDA") +def create_test_act_bf16_class( + parent, atol=1e-2, grad_check=True, grad_atol=0.80 +): + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestActBF16(parent): - def init_dtype(self): self.dtype = np.uint16 @@ -3861,9 +3831,9 @@ def create_test_act_bf16_class(parent, def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=grad_atol) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=grad_atol + ) cls_name = "{0}_{1}".format(parent.__name__, "bf16") TestActBF16.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/test_activation_sparse_op.py b/python/paddle/fluid/tests/unittests/test_activation_sparse_op.py index ceab2d48ecf..472dc334d66 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_sparse_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_sparse_op.py @@ -20,7 +20,6 @@ import paddle class TestSparseSquareOp(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() @@ -58,7 +57,6 @@ class TestSparseSquareOp(unittest.TestCase): class TestSparseSqrtOp(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() diff --git a/python/paddle/fluid/tests/unittests/test_adadelta_op.py b/python/paddle/fluid/tests/unittests/test_adadelta_op.py index 3e246517ccb..12122c8e05f 100644 --- a/python/paddle/fluid/tests/unittests/test_adadelta_op.py +++ b/python/paddle/fluid/tests/unittests/test_adadelta_op.py @@ -20,7 +20,6 @@ import paddle.fluid as fluid class TestAdadeltaOp1(OpTest): - def setUp(self): self.op_type = "adadelta" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -37,27 +36,33 @@ class TestAdadeltaOp1(OpTest): 'Param': param, 'Grad': grad, 'AvgSquaredGrad': avg_squared_grad, - 'AvgSquaredUpdate': avg_squared_update + 'AvgSquaredUpdate': avg_squared_update, } self.attrs = {'rho': rho, 'epsilon': epsilon} - avg_squared_grad_out = rho * avg_squared_grad + \ - (1 - rho) * np.square(grad) + avg_squared_grad_out = rho * avg_squared_grad + (1 - rho) * np.square( + grad + ) update = -np.multiply( np.sqrt( - np.divide(avg_squared_update + epsilon, - avg_squared_grad_out + epsilon)), grad) + np.divide( + avg_squared_update + epsilon, avg_squared_grad_out + epsilon + ) + ), + grad, + ) - avg_squared_update_out = rho * avg_squared_update + \ - (1 - rho) * np.square(update) + avg_squared_update_out = rho * avg_squared_update + ( + 1 - rho + ) * np.square(update) param_out = param + update self.outputs = { 'ParamOut': param_out, 'AvgSquaredGradOut': avg_squared_grad_out, - 'AvgSquaredUpdateOut': avg_squared_update_out + 'AvgSquaredUpdateOut': avg_squared_update_out, } def test_check_output(self): @@ -65,8 +70,7 @@ class TestAdadeltaOp1(OpTest): class TestAdadeltaOp2(OpTest): - '''Test Adadelta op with default attribute values - ''' + '''Test Adadelta op with default attribute values''' def setUp(self): self.op_type = "adadelta" @@ -84,25 +88,31 @@ class TestAdadeltaOp2(OpTest): 'Param': param, 'Grad': grad, 'AvgSquaredGrad': avg_squared_grad, - 'AvgSquaredUpdate': avg_squared_update + 'AvgSquaredUpdate': avg_squared_update, } - avg_squared_grad_out = rho * avg_squared_grad + \ - (1 - rho) * np.square(grad) + avg_squared_grad_out = rho * avg_squared_grad + (1 - rho) * np.square( + grad + ) update = -np.multiply( np.sqrt( - np.divide(avg_squared_update + epsilon, - avg_squared_grad_out + epsilon)), grad) + np.divide( + avg_squared_update + epsilon, avg_squared_grad_out + epsilon + ) + ), + grad, + ) - avg_squared_update_out = rho * avg_squared_update + \ - (1 - rho) * np.square(update) + avg_squared_update_out = rho * avg_squared_update + ( + 1 - rho + ) * np.square(update) param_out = param + update self.outputs = { 'ParamOut': param_out, 'AvgSquaredGradOut': avg_squared_grad_out, - 'AvgSquaredUpdateOut': avg_squared_update_out + 'AvgSquaredUpdateOut': avg_squared_update_out, } def test_check_output(self): @@ -110,16 +120,17 @@ class TestAdadeltaOp2(OpTest): class TestAdadeltaV2(unittest.TestCase): - def test_adadelta_dygraph(self): paddle.disable_static(paddle.CPUPlace()) value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Adadelta(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.Adadelta( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.01, + ) out = linear(a) out.backward() adam.step() @@ -140,8 +151,9 @@ class TestAdadeltaV2(unittest.TestCase): rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -150,18 +162,18 @@ class TestAdadeltaV2(unittest.TestCase): def test_raise_error(self): self.assertRaises(ValueError, paddle.optimizer.Adadelta, None) - self.assertRaises(ValueError, - paddle.optimizer.Adadelta, - learning_rate=0.1, - rho=None) - self.assertRaises(ValueError, - paddle.optimizer.Adadelta, - learning_rate=0.1, - epsilon=None) + self.assertRaises( + ValueError, paddle.optimizer.Adadelta, learning_rate=0.1, rho=None + ) + self.assertRaises( + ValueError, + paddle.optimizer.Adadelta, + learning_rate=0.1, + epsilon=None, + ) class TestAdadeltaV2Group(TestAdadeltaV2): - def test_adadelta_dygraph(self): paddle.disable_static(paddle.CPUPlace()) value = np.arange(26).reshape(2, 13).astype("float32") @@ -169,17 +181,17 @@ class TestAdadeltaV2Group(TestAdadeltaV2): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Adadelta(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - }], - weight_decay=0.1) + adam = paddle.optimizer.Adadelta( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + }, + ], + weight_decay=0.1, + ) out = linear_1(a) out = linear_2(out) out.backward() diff --git a/python/paddle/fluid/tests/unittests/test_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_adagrad_op.py index fcac8599d0c..036322ee62d 100644 --- a/python/paddle/fluid/tests/unittests/test_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_adagrad_op.py @@ -22,8 +22,7 @@ import paddle class TestAdagradOp1(OpTest): - ''' Test Adagrad operator with explicit attributes - ''' + '''Test Adagrad operator with explicit attributes''' def setUp(self): self.op_type = "adagrad" @@ -37,7 +36,7 @@ class TestAdagradOp1(OpTest): 'Param': param, 'Grad': grad, 'Moment': moment, - 'LearningRate': np.array([lr]).astype("float32") + 'LearningRate': np.array([lr]).astype("float32"), } self.attrs = {'epsilon': epsilon} @@ -52,8 +51,7 @@ class TestAdagradOp1(OpTest): class TestAdagradOp2(OpTest): - ''' Test Adagrad operator with default attributes - ''' + '''Test Adagrad operator with default attributes''' def setUp(self): self.op_type = "adagrad" @@ -68,7 +66,7 @@ class TestAdagradOp2(OpTest): 'Param': param, 'Grad': grad, 'Moment': moment, - 'LearningRate': np.array([lr]).astype("float32") + 'LearningRate': np.array([lr]).astype("float32"), } self.attrs = {'epsilon': epsilon} @@ -83,7 +81,6 @@ class TestAdagradOp2(OpTest): class TestSparseAdagradOp(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() @@ -118,14 +115,16 @@ class TestSparseAdagradOp(unittest.TestCase): moment.set(moment_np_array, place) # create and run sgd operator - adagrad_op = Operator("adagrad", - Param='Param', - Grad='Grad', - ParamOut='Param', - Moment='Moment', - MomentOut='Moment', - LearningRate='LearningRate', - epsilon=2.0) + adagrad_op = Operator( + "adagrad", + Param='Param', + Grad='Grad', + ParamOut='Param', + Moment='Moment', + MomentOut='Moment', + LearningRate='LearningRate', + epsilon=2.0, + ) adagrad_op.run(scope, place) @@ -149,31 +148,35 @@ class TestSparseAdagradOp(unittest.TestCase): def get_out(param, lr, grad, m, epsilon): return param - lr * grad / (math.sqrt(m) + epsilon) - self.assertAlmostEqual(get_out(5.0, 2.0, 2.0, 6.0, 2.0), - result_array[rows[0], 0], - places=5) - self.assertAlmostEqual(get_out(5.0, 2.0, 1.0, 3.0, 2.0), - result_array[rows[0], 2], - places=5) - self.assertAlmostEqual(get_out(5.0, 2.0, 0.0, 2.0, 2.0), - result_array[1, 0], - places=5) + self.assertAlmostEqual( + get_out(5.0, 2.0, 2.0, 6.0, 2.0), result_array[rows[0], 0], places=5 + ) + self.assertAlmostEqual( + get_out(5.0, 2.0, 1.0, 3.0, 2.0), result_array[rows[0], 2], places=5 + ) + self.assertAlmostEqual( + get_out(5.0, 2.0, 0.0, 2.0, 2.0), result_array[1, 0], places=5 + ) # grad_merge = 1.0 + 1.0 # m = 6.0 - self.assertAlmostEqual(get_out(5.0, 2.0, 2.0, 6.0, 2.0), - result_array[rows[1], 10], - places=5) - - self.assertAlmostEqual(get_out(5.0, 2.0, 0.0, 2.0, 2.0), - result_array[5, 8], - places=5) - self.assertAlmostEqual(get_out(5.0, 2.0, 1.0, 3.0, 2.0), - result_array[rows[2], 1], - places=5) - self.assertAlmostEqual(get_out(5.0, 2.0, 4.0, 18.0, 2.0), - result_array[rows[2], 8], - places=5) + self.assertAlmostEqual( + get_out(5.0, 2.0, 2.0, 6.0, 2.0), + result_array[rows[1], 10], + places=5, + ) + + self.assertAlmostEqual( + get_out(5.0, 2.0, 0.0, 2.0, 2.0), result_array[5, 8], places=5 + ) + self.assertAlmostEqual( + get_out(5.0, 2.0, 1.0, 3.0, 2.0), result_array[rows[2], 1], places=5 + ) + self.assertAlmostEqual( + get_out(5.0, 2.0, 4.0, 18.0, 2.0), + result_array[rows[2], 8], + places=5, + ) def test_sparse_adagrad(self): places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_adagrad_op_v2.py b/python/paddle/fluid/tests/unittests/test_adagrad_op_v2.py index e483dc1fa89..c3eee954b61 100644 --- a/python/paddle/fluid/tests/unittests/test_adagrad_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_adagrad_op_v2.py @@ -17,22 +17,21 @@ import paddle class TestAdagradOpV2(unittest.TestCase): - def test_v20_coverage(self): paddle.disable_static() inp = paddle.rand(shape=[10, 10]) linear = paddle.nn.Linear(10, 10) out = linear(inp) loss = paddle.mean(out) - adagrad = paddle.optimizer.Adagrad(learning_rate=0.1, - parameters=linear.parameters()) + adagrad = paddle.optimizer.Adagrad( + learning_rate=0.1, parameters=linear.parameters() + ) out.backward() adagrad.step() adagrad.clear_grad() class TestAdagradOpV2Group(TestAdagradOpV2): - def test_v20_coverage(self): paddle.disable_static() inp = paddle.rand(shape=[10, 10]) @@ -41,17 +40,17 @@ class TestAdagradOpV2Group(TestAdagradOpV2): out = linear_1(inp) out = linear_2(out) loss = paddle.mean(out) - adagrad = paddle.optimizer.Adagrad(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - }], - weight_decay=0.1) + adagrad = paddle.optimizer.Adagrad( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + }, + ], + weight_decay=0.1, + ) out.backward() adagrad.step() adagrad.clear_grad() diff --git a/python/paddle/fluid/tests/unittests/test_adam_op.py b/python/paddle/fluid/tests/unittests/test_adam_op.py index 76309ab15eb..bfe1429342f 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_op.py +++ b/python/paddle/fluid/tests/unittests/test_adam_op.py @@ -23,10 +23,8 @@ from paddle.fluid.framework import _test_eager_guard class TestAdamOp1(OpTest): - def setUp(self): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -48,20 +46,19 @@ class TestAdamOp1(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -69,13 +66,11 @@ class TestAdamOp1(OpTest): class TestAdamOp2(OpTest): - def set_shape(self): self.shape = (102, 105) def setUp(self): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' self.op_type = "adam" self.set_shape() param = np.random.uniform(-1, 1, self.shape).astype("float32") @@ -98,20 +93,19 @@ class TestAdamOp2(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } attributes = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -119,16 +113,13 @@ class TestAdamOp2(OpTest): class TestAdamOnlyTailOp(TestAdamOp2): - def set_shape(self): - self.shape = (3) + self.shape = 3 class TestAdamOpMultipleSteps(OpTest): - def setUp(self): - '''Test Adam Operator with supplied attributes - ''' + '''Test Adam Operator with supplied attributes''' self.op_type = "adam" self.num_steps = 10 @@ -152,19 +143,20 @@ class TestAdamOpMultipleSteps(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([self.beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([self.beta2_pow]).astype("float32") + 'Beta2Pow': np.array([self.beta2_pow]).astype("float32"), } self.attrs = { 'epsilon': epsilon, 'beta1': self.beta1, - 'beta2': self.beta2 + 'beta2': self.beta2, } def test_check_output(self): for _ in range(self.num_steps): - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step( + self.inputs, self.attrs + ) beta1_pow_out = self.inputs['Beta1Pow'] * self.beta1 beta2_pow_out = self.inputs['Beta2Pow'] * self.beta2 @@ -173,7 +165,7 @@ class TestAdamOpMultipleSteps(OpTest): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } # Verify output for this step @@ -189,8 +181,9 @@ class TestAdamOpMultipleSteps(OpTest): self.inputs['Beta2Pow'] = beta2_pow_out # Randomize gradient for next step - self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + self.inputs['Grad'] = np.random.uniform(-1, 1, (102, 105)).astype( + "float32" + ) def test_api_eager_dygraph(self): with _test_eager_guard(): @@ -270,8 +263,9 @@ def adamw_step(inputs, attributes): return param_out, moment1_out, moment2_out -def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, - lazy_mode): +def adam_step_sparse( + inputs, attributes, height, rows, row_numel, np_grad, lazy_mode +): ''' Simulate one step of the adam optimizer :param inputs: dict of inputs @@ -296,13 +290,16 @@ def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, param_out = np.zeros(shape=[height, row_numel]) def update_row(row_id, update_value): - moment1_out[row_id] = beta1 * moment1[row_id] + (1 - - beta1) * update_value - moment2_out[row_id] = beta2 * moment2[row_id] + ( - 1 - beta2) * np.square(update_value) + moment1_out[row_id] = ( + beta1 * moment1[row_id] + (1 - beta1) * update_value + ) + moment2_out[row_id] = beta2 * moment2[row_id] + (1 - beta2) * np.square( + update_value + ) lr_t = lr * np.sqrt(1 - beta2_pow) / (1 - beta1_pow) param_out[row_id] = param[row_id] - lr_t * ( - moment1_out[row_id] / (np.sqrt(moment2_out[row_id]) + epsilon)) + moment1_out[row_id] / (np.sqrt(moment2_out[row_id]) + epsilon) + ) if lazy_mode: for idx, row_id in enumerate(rows): @@ -318,7 +315,6 @@ def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, class TestSparseAdamOp(unittest.TestCase): - def setup(self, scope, place, lazy_mode): beta1 = 0.78 beta2 = 0.836 @@ -337,14 +333,14 @@ class TestSparseAdamOp(unittest.TestCase): "Moment2": np.full((height, row_numel), 5.0).astype("float32"), 'Beta1Pow': beta1_pow, 'Beta2Pow': beta2_pow, - "LearningRate": np.full((1), 2.0).astype("float32") + "LearningRate": np.full((1), 2.0).astype("float32"), } self.init_output = np.full((height, row_numel), 0.0).astype("float32") self.attrs = { 'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2, - 'min_row_size_to_use_multithread': 2 + 'min_row_size_to_use_multithread': 2, } grad_selected_rows = scope.var('Grad').get_selected_rows() @@ -359,15 +355,21 @@ class TestSparseAdamOp(unittest.TestCase): self.sparse_inputs = ["Grad"] - param_out, mom1, mom2 = adam_step_sparse(self.dense_inputs, self.attrs, - height, rows, row_numel, - np_array, lazy_mode) + param_out, mom1, mom2 = adam_step_sparse( + self.dense_inputs, + self.attrs, + height, + rows, + row_numel, + np_array, + lazy_mode, + ) self.outputs = { "ParamOut": param_out, "Moment1Out": mom1, "Moment2Out": mom2, 'Beta1PowOut': beta1_pow * beta1, - 'Beta2PowOut': beta2_pow * beta2 + 'Beta2PowOut': beta2_pow * beta2, } def check_with_place(self, place, lazy_mode): @@ -412,10 +414,8 @@ class TestSparseAdamOp(unittest.TestCase): class TestAdamOpBetaVariable(OpTest): - def setUp(self): - '''Test Adam Op with beta as Variable - ''' + '''Test Adam Op with beta as Variable''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -444,15 +444,14 @@ class TestAdamOpBetaVariable(OpTest): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -460,10 +459,8 @@ class TestAdamOpBetaVariable(OpTest): class TestAdamOpBetaEpsilonVariable(OpTest): - def setUp(self): - '''Test Adam Op with beta/epsilon as Variable - ''' + '''Test Adam Op with beta/epsilon as Variable''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -493,15 +490,14 @@ class TestAdamOpBetaEpsilonVariable(OpTest): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -509,10 +505,8 @@ class TestAdamOpBetaEpsilonVariable(OpTest): class TestAdamOpWithGlobalBetaPow(OpTest): - def setUp(self): - '''Test Adam Op with global_beta_pow - ''' + '''Test Adam Op with global_beta_pow''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -542,8 +536,7 @@ class TestAdamOpWithGlobalBetaPow(OpTest): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.attrs = {'use_global_beta_pow': True} @@ -553,7 +546,7 @@ class TestAdamOpWithGlobalBetaPow(OpTest): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([]), - 'Beta2PowOut': np.array([]) + 'Beta2PowOut': np.array([]), } def test_check_output(self): @@ -561,10 +554,8 @@ class TestAdamOpWithGlobalBetaPow(OpTest): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): - '''Test Adam Op with global_beta_pow - ''' + '''Test Adam Op with global_beta_pow''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -611,7 +602,6 @@ class TestAdamOpWithSkipUpdate(OpTest): class TestAdamOpV2(unittest.TestCase): - def test_adam_op(self): place = fluid.CPUPlace() shape = [2, 3, 8, 8] @@ -624,20 +614,20 @@ class TestAdamOpV2(unittest.TestCase): conv = fluid.layers.conv2d(data, 8, 3) loss = fluid.layers.reduce_mean(conv) - beta1 = fluid.layers.create_global_var(shape=[1], - value=0.85, - dtype='float32', - persistable=True) - beta2 = fluid.layers.create_global_var(shape=[1], - value=0.95, - dtype='float32', - persistable=True) + beta1 = fluid.layers.create_global_var( + shape=[1], value=0.85, dtype='float32', persistable=True + ) + beta2 = fluid.layers.create_global_var( + shape=[1], value=0.95, dtype='float32', persistable=True + ) betas = [beta1, beta2] - opt = paddle.optimizer.Adam(learning_rate=1e-5, - beta1=beta1, - beta2=beta2, - weight_decay=0.01, - epsilon=1e-8) + opt = paddle.optimizer.Adam( + learning_rate=1e-5, + beta1=beta1, + beta2=beta2, + weight_decay=0.01, + epsilon=1e-8, + ) opt.minimize(loss) exe.run(startup) @@ -651,8 +641,9 @@ class TestAdamOpV2(unittest.TestCase): a = fluid.dygraph.to_variable(value) linear = fluid.Linear(13, 5, dtype="float32") - adam = paddle.optimizer.Adam(learning_rate=0.01, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=0.01, parameters=linear.parameters() + ) out = linear(a) out.backward() adam.step() @@ -668,26 +659,29 @@ class TestAdamOpV2(unittest.TestCase): state_dict = adam.state_dict() adam.set_state_dict(state_dict) - #learning_rate is LRScheduler + # learning_rate is LRScheduler learning_rate = paddle.optimizer.lr.CosineAnnealingDecay( - learning_rate=0.1, T_max=10) + learning_rate=0.1, T_max=10 + ) adam = paddle.optimizer.Adam( learning_rate=learning_rate, weight_decay=fluid.regularizer.L2Decay(0.001), - parameters=emb.parameters()) + parameters=emb.parameters(), + ) lr = adam.get_lr() state_dict = adam.state_dict() adam.set_state_dict(state_dict) - #leanrning_rate is Tensor + # leanrning_rate is Tensor with self.assertRaises(TypeError): learning_rate = np.array([0.01]).astype("float32") learning_rate = paddle.to_tensor(learning_rate) - adam = paddle.optimizer.Adam(learning_rate=learning_rate, - parameters=emb.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=learning_rate, parameters=emb.parameters() + ) params = adam.get_opti_var_name_list() - assert (params is not None) + assert params is not None paddle.enable_static() def test_adam_with_grad_clip(self): @@ -696,9 +690,9 @@ class TestAdamOpV2(unittest.TestCase): a = fluid.dygraph.to_variable(value) linear = fluid.Linear(13, 5, dtype="float32") clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0) - adam = paddle.optimizer.Adam(0.1, - parameters=linear.parameters(), - grad_clip=clip) + adam = paddle.optimizer.Adam( + 0.1, parameters=linear.parameters(), grad_clip=clip + ) out = linear(a) out.backward() adam.step() @@ -713,11 +707,11 @@ class TestAdamOpV2(unittest.TestCase): lr = 0.01 adam.set_lr(lr) cur_lr = adam.get_lr() - assert (lr == cur_lr) + assert lr == cur_lr with self.assertRaises(TypeError): - lr_var = paddle.fluid.layers.create_global_var(shape=[1], - value=lr, - dtype='float32') + lr_var = paddle.fluid.layers.create_global_var( + shape=[1], value=lr, dtype='float32' + ) adam.set_lr(lr_var) paddle.enable_static() @@ -725,17 +719,17 @@ class TestAdamOpV2(unittest.TestCase): paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adam(0.1, - beta1=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + 0.1, beta1=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adam(0.1, - beta2=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + 0.1, beta2=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adam(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + 0.1, epsilon=-1, parameters=linear.parameters() + ) paddle.enable_static() def test_adam_op_with_sparse_input_and_weight_decay(self): @@ -744,9 +738,9 @@ class TestAdamOpV2(unittest.TestCase): x_data = np.arange(0, 10).reshape((10, 1)).astype(np.int64) x = paddle.to_tensor(x_data, stop_gradient=False) emb = paddle.nn.Embedding(10, 10, sparse=True) - adam = paddle.optimizer.Adam(0.001, - parameters=emb.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.Adam( + 0.001, parameters=emb.parameters(), weight_decay=0.01 + ) with self.assertRaises(RuntimeError): out = emb(x) @@ -764,13 +758,14 @@ class TestAdamOpV2(unittest.TestCase): class TestAdamOptimizer(unittest.TestCase): - - def _test(self, - place, - use_tensor=True, - use_fluid_api=True, - use_global_beta_pow=False, - flatten_param_grads=False): + def _test( + self, + place, + use_tensor=True, + use_fluid_api=True, + use_global_beta_pow=False, + flatten_param_grads=False, + ): paddle.enable_static() main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -784,29 +779,30 @@ class TestAdamOptimizer(unittest.TestCase): weight_attr1 = paddle.ParamAttr( name="weight1", initializer=fluid.initializer.Constant(value=1.0), - trainable=True) + trainable=True, + ) weight_attr2 = paddle.ParamAttr( name="weight2", initializer=fluid.initializer.Constant(value=2.0), - trainable=True) + trainable=True, + ) clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) with paddle.static.program_guard(main_prog, startup_prog): with paddle.utils.unique_name.guard(): a = paddle.static.data(name="a", shape=[2, 2], dtype='float32') b = paddle.static.data(name="b", shape=[2, 2], dtype='float32') - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) fc_1 = fluid.layers.fc(input=z, size=2, param_attr=weight_attr1) - prediction = fluid.layers.fc(input=fc_1, - size=2, - param_attr=weight_attr2, - act='softmax') + prediction = fluid.layers.fc( + input=fc_1, size=2, param_attr=weight_attr2, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) loss = fluid.layers.reduce_mean(cost) @@ -819,19 +815,22 @@ class TestAdamOptimizer(unittest.TestCase): value=float(beta1_init), dtype='float32', persistable=True, - name="beta1") + name="beta1", + ) beta2 = fluid.layers.create_global_var( shape=[1], value=float(beta2_init), dtype='float32', persistable=True, - name="beta2") + name="beta2", + ) epsilon = fluid.layers.create_global_var( shape=[1], value=float(epsilon_init), dtype='float32', persistable=True, - name="epsilon") + name="epsilon", + ) if use_fluid_api: adam = fluid.optimizer.Adam( learning_rate=0.01, @@ -841,13 +840,16 @@ class TestAdamOptimizer(unittest.TestCase): use_global_beta_pow=use_global_beta_pow, flatten_param_grads=flatten_param_grads, align_size=256, - grad_clip=clip) + grad_clip=clip, + ) else: - adam = paddle.optimizer.Adam(learning_rate=0.01, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - grad_clip=clip) + adam = paddle.optimizer.Adam( + learning_rate=0.01, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + grad_clip=clip, + ) else: if use_fluid_api: adam = fluid.optimizer.Adam( @@ -858,13 +860,16 @@ class TestAdamOptimizer(unittest.TestCase): use_global_beta_pow=use_global_beta_pow, flatten_param_grads=flatten_param_grads, align_size=256, - grad_clip=clip) + grad_clip=clip, + ) else: - adam = fluid.optimizer.Adam(learning_rate=0.01, - beta1=beta1_init, - beta2=beta2_init, - epsilon=epsilon_init, - grad_clip=clip) + adam = fluid.optimizer.Adam( + learning_rate=0.01, + beta1=beta1_init, + beta2=beta2_init, + epsilon=epsilon_init, + grad_clip=clip, + ) adam.minimize(loss) @@ -875,15 +880,16 @@ class TestAdamOptimizer(unittest.TestCase): print("Start run on {}".format(place)) for epoch in range(10): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) paddle.disable_static() return pred_res, loss_res @@ -895,10 +901,13 @@ class TestAdamOptimizer(unittest.TestCase): for use_fluid_api in [True, False]: for use_global_beta_pow in [True, False]: for flatten_param_grads in [True, False]: - pred, loss = self._test(place, use_tensor, - use_fluid_api, - use_global_beta_pow, - flatten_param_grads) + pred, loss = self._test( + place, + use_tensor, + use_fluid_api, + use_global_beta_pow, + flatten_param_grads, + ) preds.append(pred) losses.append(loss) for pred in preds: @@ -920,21 +929,22 @@ class TestAdamOptimizer(unittest.TestCase): name="weight1", initializer=fluid.initializer.Constant(value=1.0), regularizer=fluid.regularizer.L1DecayRegularizer( - regularization_coeff=0.1), - trainable=True) + regularization_coeff=0.1 + ), + trainable=True, + ) with fluid.program_guard(main): x = fluid.data(name='x', shape=[None, 13], dtype='float32') y = fluid.data(name='y', shape=[None, 1], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1, - act=None, - param_attr=weight_attr) + y_predict = fluid.layers.fc( + input=x, size=1, act=None, param_attr=weight_attr + ) cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - adam = fluid.optimizer.AdamOptimizer(0.01, - flatten_param_grads=True, - align_size=256) + adam = fluid.optimizer.AdamOptimizer( + 0.01, flatten_param_grads=True, align_size=256 + ) adam.minimize(avg_cost) paddle.disable_static() @@ -957,13 +967,16 @@ class TestAdamOptimizer(unittest.TestCase): adam = fluid.optimizer.Adam(use_global_beta_pow=True) adam.minimize(loss) self.assertRaises(Exception, adam._get_global_accumulator, 'tmp') - adam._add_global_accumulator('tmp', - type=core.VarDesc.VarType.LOD_TENSOR) + adam._add_global_accumulator( + 'tmp', type=core.VarDesc.VarType.LOD_TENSOR + ) adam._get_global_accumulator('tmp') - self.assertRaises(Exception, - adam._add_global_accumulator, - adam._beta1_pow_acc_str, - type=core.VarDesc.VarType.LOD_TENSOR) + self.assertRaises( + Exception, + adam._add_global_accumulator, + adam._beta1_pow_acc_str, + type=core.VarDesc.VarType.LOD_TENSOR, + ) paddle.disable_static() def test_adam_save_load(self): @@ -974,12 +987,14 @@ class TestAdamOptimizer(unittest.TestCase): state_dict = linear.state_dict() fluid.save_dygraph(state_dict, "paddle_dy") - scheduler = paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True) - adam = paddle.fluid.optimizer.Adam(learning_rate=scheduler, - parameter_list=linear.parameters(), - use_global_beta_pow=True) + scheduler = paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ) + adam = paddle.fluid.optimizer.Adam( + learning_rate=scheduler, + parameter_list=linear.parameters(), + use_global_beta_pow=True, + ) adam.minimize(b) state_dict = adam.state_dict() fluid.save_dygraph(state_dict, "paddle_dy") @@ -1000,13 +1015,14 @@ class TestAdamOptimizer(unittest.TestCase): state_dict = linear.state_dict() fluid.save_dygraph(state_dict, "paddle_dy") - scheduler = paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True) + scheduler = paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ) adam = paddle.fluid.optimizer.Adam( learning_rate=scheduler, parameter_list=linear.parameters(), - use_global_beta_pow=True) + use_global_beta_pow=True, + ) adam.minimize(b) return adam @@ -1021,14 +1037,14 @@ class TestAdamOptimizer(unittest.TestCase): self.assertRaises(AssertionError, adam2.set_state_dict, opt_state_dict) adam3 = get_opt('float32', [10, 10]) # shape not match - opt_state_dict['beta1_pow_acc_0'] = np.array([0.9, 0.9], - dtype='float32') + opt_state_dict['beta1_pow_acc_0'] = np.array( + [0.9, 0.9], dtype='float32' + ) self.assertRaises(AssertionError, adam3.set_state_dict, opt_state_dict) paddle.enable_static() class TestAdamOpV2Group(TestAdamOpV2): - def test_adam_op(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -1036,16 +1052,19 @@ class TestAdamOpV2Group(TestAdamOpV2): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Adam(learning_rate=0.01, - parameters=[{ - 'params': linear_1.parameters() - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001, - 'beta1': 0.1, - 'beta2': 0.99 - }], - weight_decay=0.1) + adam = paddle.optimizer.Adam( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + }, + ], + weight_decay=0.1, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -1054,13 +1073,14 @@ class TestAdamOpV2Group(TestAdamOpV2): class TestMultiTensorAdam(unittest.TestCase): - - def _adam_optimize_dygraph(self, - place, - use_param_attr=False, - use_param_group=False, - use_amp=False, - use_multi_tensor=False): + def _adam_optimize_dygraph( + self, + place, + use_param_attr=False, + use_param_group=False, + use_amp=False, + use_multi_tensor=False, + ): paddle.disable_static() paddle.seed(10) paddle.set_device(place) @@ -1070,29 +1090,32 @@ class TestMultiTensorAdam(unittest.TestCase): weight_attr = paddle.ParamAttr( learning_rate=0.5, regularizer=paddle.regularizer.L2Decay(1.0), - trainable=True) + trainable=True, + ) if use_param_attr: model = paddle.nn.Linear(5, 5, weight_attr) else: model = paddle.nn.Linear(5, 5) if not use_param_group: - optimizer = paddle.optimizer.Adam(parameters=model.parameters(), - use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + optimizer = paddle.optimizer.Adam( + parameters=model.parameters(), + use_multi_tensor=use_multi_tensor, + multi_precision=use_amp, + ) else: - optimizer = paddle.optimizer.Adam(parameters=[{ - 'params': - model.parameters(), - 'weight_decay': - 0.001, - 'beta1': - 0.1, - 'beta2': - 0.99 - }], - use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + optimizer = paddle.optimizer.Adam( + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + } + ], + use_multi_tensor=use_multi_tensor, + multi_precision=use_amp, + ) for idx in range(2): if place == 'gpu' and use_amp == True: @@ -1116,10 +1139,9 @@ class TestMultiTensorAdam(unittest.TestCase): return output, model.parameters() - def _adam_optimize_static(self, - place, - use_amp=False, - use_multi_tensor=False): + def _adam_optimize_static( + self, place, use_amp=False, use_multi_tensor=False + ): paddle.enable_static() paddle.seed(10) np.random.seed(10) @@ -1128,24 +1150,26 @@ class TestMultiTensorAdam(unittest.TestCase): exe = paddle.static.Executor(place=place) train_program = paddle.static.Program() startup_program = paddle.static.Program() - optimizer = paddle.optimizer.Adam(multi_precision=use_amp, - use_multi_tensor=use_multi_tensor) + optimizer = paddle.optimizer.Adam( + multi_precision=use_amp, use_multi_tensor=use_multi_tensor + ) if use_amp: optimizer = paddle.static.amp.decorate( optimizer, init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if use_amp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -1157,9 +1181,9 @@ class TestMultiTensorAdam(unittest.TestCase): x = np.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -1172,49 +1196,59 @@ class TestMultiTensorAdam(unittest.TestCase): def _check_with_place_amp(self, place, use_amp): # test dygraph mode output_dygraph1, params_dygraph1 = self._adam_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=True) + place=place, use_amp=use_amp, use_multi_tensor=True + ) output_dygraph2, params_dygraph2 = self._adam_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=False) + place=place, use_amp=use_amp, use_multi_tensor=False + ) np.testing.assert_allclose(output_dygraph1, output_dygraph2, rtol=1e-05) for idx in range(len(params_dygraph1)): - np.testing.assert_allclose(params_dygraph1[idx], - params_dygraph2[idx], - rtol=1e-05) + np.testing.assert_allclose( + params_dygraph1[idx], params_dygraph2[idx], rtol=1e-05 + ) # test static mode - output_static1 = self._adam_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=True) - output_static2 = self._adam_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=False) + output_static1 = self._adam_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=True + ) + output_static2 = self._adam_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=False + ) for idx in range(len(output_static1)): - np.testing.assert_allclose(output_static1[idx], - output_static2[idx], - rtol=1e-05) + np.testing.assert_allclose( + output_static1[idx], output_static2[idx], rtol=1e-05 + ) def _check_with_param_arrt(self, place, use_amp): - output1, params1 = self._adam_optimize_dygraph(place=place, - use_amp=use_amp, - use_param_attr=True, - use_multi_tensor=True) - output2, params2 = self._adam_optimize_dygraph(place=place, - use_amp=use_amp, - use_param_attr=True, - use_multi_tensor=False) + output1, params1 = self._adam_optimize_dygraph( + place=place, + use_amp=use_amp, + use_param_attr=True, + use_multi_tensor=True, + ) + output2, params2 = self._adam_optimize_dygraph( + place=place, + use_amp=use_amp, + use_param_attr=True, + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) def _check_with_param_group(self, place, use_amp): - output1, params1 = self._adam_optimize_dygraph(place=place, - use_amp=use_amp, - use_param_group=True, - use_multi_tensor=True) - output2, params2 = self._adam_optimize_dygraph(place=place, - use_amp=use_amp, - use_param_group=True, - use_multi_tensor=False) + output1, params1 = self._adam_optimize_dygraph( + place=place, + use_amp=use_amp, + use_param_group=True, + use_multi_tensor=True, + ) + output2, params2 = self._adam_optimize_dygraph( + place=place, + use_amp=use_amp, + use_param_group=True, + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): diff --git a/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py b/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py index 1f08eb085a3..904b02e778c 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py +++ b/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py @@ -39,8 +39,9 @@ def main_test_func(place, dtype): adam_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = fluid.io.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = fluid.io.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -49,7 +50,6 @@ def main_test_func(place, dtype): class AdamFp32Test(unittest.TestCase): - def setUp(self): self.dtype = 'float32' @@ -59,7 +59,6 @@ class AdamFp32Test(unittest.TestCase): class AdamFp64Test(AdamFp32Test): - def setUp(self): self.dtype = 'float64' diff --git a/python/paddle/fluid/tests/unittests/test_adamax_api.py b/python/paddle/fluid/tests/unittests/test_adamax_api.py index 87aea099ac5..78e9553a806 100644 --- a/python/paddle/fluid/tests/unittests/test_adamax_api.py +++ b/python/paddle/fluid/tests/unittests/test_adamax_api.py @@ -20,15 +20,16 @@ from paddle.fluid.framework import _test_eager_guard class TestAdamaxAPI(unittest.TestCase): - def func_adamax_api_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.Adamax(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.Adamax( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.01, + ) out = linear(a) out.backward() adam.step() @@ -53,11 +54,13 @@ class TestAdamaxAPI(unittest.TestCase): loss = paddle.mean(conv) beta1 = 0.85 beta2 = 0.95 - opt = paddle.optimizer.Adamax(learning_rate=1e-5, - beta1=beta1, - beta2=beta2, - weight_decay=0.01, - epsilon=1e-8) + opt = paddle.optimizer.Adamax( + learning_rate=1e-5, + beta1=beta1, + beta2=beta2, + weight_decay=0.01, + epsilon=1e-8, + ) opt.minimize(loss) exe.run(startup) @@ -72,7 +75,6 @@ class TestAdamaxAPI(unittest.TestCase): class TestAdamaxAPIGroup(TestAdamaxAPI): - def func_adamax_api_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -80,21 +82,19 @@ class TestAdamaxAPIGroup(TestAdamaxAPI): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Adamax(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - 'beta1': - 0.1, - 'beta2': - 0.99 - }], - weight_decay=0.1) + adam = paddle.optimizer.Adamax( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + }, + ], + weight_decay=0.1, + ) out = linear_1(a) out = linear_2(out) out.backward() diff --git a/python/paddle/fluid/tests/unittests/test_adamax_op.py b/python/paddle/fluid/tests/unittests/test_adamax_op.py index 6ea12544d87..b6432714434 100644 --- a/python/paddle/fluid/tests/unittests/test_adamax_op.py +++ b/python/paddle/fluid/tests/unittests/test_adamax_op.py @@ -18,10 +18,8 @@ from op_test import OpTest class TestAdamaxOp1(OpTest): - def setUp(self): - '''Test Adamax Operator with supplied attributes - ''' + '''Test Adamax Operator with supplied attributes''' self.op_type = "adamax" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -41,18 +39,19 @@ class TestAdamaxOp1(OpTest): 'Moment': moment, 'InfNorm': inf_norm, 'LearningRate': np.array([learning_rate]).astype("float32"), - 'Beta1Pow': np.array([beta1_pow]).astype("float32") + 'Beta1Pow': np.array([beta1_pow]).astype("float32"), } self.attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} param_out, moment_out, inf_norm_out = adamax_step( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out + 'InfNormOut': inf_norm_out, } def test_check_output(self): @@ -60,8 +59,7 @@ class TestAdamaxOp1(OpTest): class TestAdamaxOp2(OpTest): - '''Test Adamax Operator with default attributes - ''' + '''Test Adamax Operator with default attributes''' def setUp(self): self.op_type = "adamax" @@ -83,7 +81,7 @@ class TestAdamaxOp2(OpTest): 'Moment': moment, 'InfNorm': inf_norm, 'LearningRate': np.array([learning_rate]).astype("float32"), - 'Beta1Pow': np.array([beta1_pow]).astype("float32") + 'Beta1Pow': np.array([beta1_pow]).astype("float32"), } attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} @@ -92,7 +90,7 @@ class TestAdamaxOp2(OpTest): self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out + 'InfNormOut': inf_norm_out, } def test_check_output(self): @@ -100,10 +98,8 @@ class TestAdamaxOp2(OpTest): class TestAdamaxOpMultipleSteps(OpTest): - def setUp(self): - '''Test Adamax Operator with supplied attributes - ''' + '''Test Adamax Operator with supplied attributes''' self.op_type = "adamax" self.num_steps = 10 @@ -125,7 +121,7 @@ class TestAdamaxOpMultipleSteps(OpTest): 'Moment': moment, 'InfNorm': inf_norm, 'LearningRate': np.array([learning_rate]).astype("float32"), - 'Beta1Pow': np.array([beta1_pow]).astype("float32") + 'Beta1Pow': np.array([beta1_pow]).astype("float32"), } self.attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} @@ -133,12 +129,13 @@ class TestAdamaxOpMultipleSteps(OpTest): def test_check_output(self): for _ in range(self.num_steps): param_out, moment_out, inf_norm_out = adamax_step( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out + 'InfNormOut': inf_norm_out, } # Verify output for this step @@ -153,8 +150,9 @@ class TestAdamaxOpMultipleSteps(OpTest): self.inputs['Beta1Pow'] *= self.attrs['beta1'] # Randomize gradient for next step - self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + self.inputs['Grad'] = np.random.uniform(-1, 1, (102, 105)).astype( + "float32" + ) def adamax_step(inputs, attributes): @@ -178,30 +176,30 @@ def adamax_step(inputs, attributes): moment_out = beta1 * moment + (1 - beta1) * grad inf_norm_out = np.maximum(beta2 * inf_norm + epsilon, np.abs(grad)) - lr_t = (lr / (1 - beta1_pow)) + lr_t = lr / (1 - beta1_pow) param_out = param - lr_t * np.divide(moment_out, inf_norm_out) return param_out, moment_out, inf_norm_out class TestAdamaxOpV2(unittest.TestCase): - def test_adamax_op_invalid_input(self): import paddle + paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adamax(0.1, - beta1=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adamax( + 0.1, beta1=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adamax(0.1, - beta2=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adamax( + 0.1, beta2=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adamax(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adamax( + 0.1, epsilon=-1, parameters=linear.parameters() + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_adamw_op.py b/python/paddle/fluid/tests/unittests/test_adamw_op.py index e39638d8655..6e4a7b43f20 100644 --- a/python/paddle/fluid/tests/unittests/test_adamw_op.py +++ b/python/paddle/fluid/tests/unittests/test_adamw_op.py @@ -60,10 +60,8 @@ def adamw_step(inputs, attributes): class TestAdamW(OpTest): - def setUp(self): - '''Test AdamW Op with supplied attributes - ''' + '''Test AdamW Op with supplied attributes''' self.op_type = "adamw" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -85,7 +83,7 @@ class TestAdamW(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = { @@ -93,31 +91,31 @@ class TestAdamW(OpTest): 'beta1': beta1, 'beta2': beta2, "coeff": 0.5, - "with_decay": True + "with_decay": True, } - param_out, moment1_out, \ - moment2_out = adamw_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adamw_step( + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAdamW2(OpTest): - def setUp(self): - '''Test AdamW Op with supplied attributes - ''' + '''Test AdamW Op with supplied attributes''' self.op_type = "adamw" param = np.random.uniform(-1, 1, (2, 2)).astype("float32") grad = np.random.uniform(-1, 1, (2, 2)).astype("float32") @@ -139,7 +137,7 @@ class TestAdamW2(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = { @@ -148,18 +146,19 @@ class TestAdamW2(OpTest): 'beta2': beta2, "lr_ratio": 0.1, "coeff": 0.5, - "with_decay": True + "with_decay": True, } param_out, moment1_out, moment2_out = adamw_step( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -167,16 +166,17 @@ class TestAdamW2(OpTest): class TestAdamWOp(unittest.TestCase): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=linear.parameters(), - apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=linear.parameters(), + apply_decay_param_fun=lambda name: True, + weight_decay=0.01, + ) for _ in range(2): out = linear(a) @@ -189,11 +189,13 @@ class TestAdamWOp(unittest.TestCase): value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=0.0, - parameters=linear.parameters(), - apply_decay_param_fun=lambda name: True, - weight_decay=0.01) - assert (adam.__str__() is not None) + adam = paddle.optimizer.AdamW( + learning_rate=0.0, + parameters=linear.parameters(), + apply_decay_param_fun=lambda name: True, + weight_decay=0.01, + ) + assert adam.__str__() is not None def test_adamw_op(self): paddle.enable_static() @@ -208,20 +210,20 @@ class TestAdamWOp(unittest.TestCase): conv = fluid.layers.conv2d(data, 8, 3) loss = paddle.mean(conv) - beta1 = fluid.layers.create_global_var(shape=[1], - value=0.85, - dtype='float32', - persistable=True) - beta2 = fluid.layers.create_global_var(shape=[1], - value=0.95, - dtype='float32', - persistable=True) + beta1 = fluid.layers.create_global_var( + shape=[1], value=0.85, dtype='float32', persistable=True + ) + beta2 = fluid.layers.create_global_var( + shape=[1], value=0.95, dtype='float32', persistable=True + ) betas = [beta1, beta2] - opt = paddle.optimizer.AdamW(learning_rate=1e-5, - beta1=beta1, - beta2=beta2, - weight_decay=0.01, - epsilon=1e-8) + opt = paddle.optimizer.AdamW( + learning_rate=1e-5, + beta1=beta1, + beta2=beta2, + weight_decay=0.01, + epsilon=1e-8, + ) opt.minimize(loss) exe.run(startup) @@ -234,17 +236,17 @@ class TestAdamWOp(unittest.TestCase): paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - beta1=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, beta1=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - beta2=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, beta2=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, epsilon=-1, parameters=linear.parameters() + ) def test_api_eager_dygraph(self): with _test_eager_guard(): @@ -253,25 +255,21 @@ class TestAdamWOp(unittest.TestCase): class TestAdamWOpGroup(TestAdamWOp): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001 - }], - apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + {'params': linear_2.parameters(), 'weight_decay': 0.001}, + ], + apply_decay_param_fun=lambda name: True, + weight_decay=0.01, + ) for _ in range(2): out = linear_1(a) @@ -282,7 +280,6 @@ class TestAdamWOpGroup(TestAdamWOp): class TestAdamWOpMultiPrecison(unittest.TestCase): - def _test_adamw_op_dygraph_place_amp(self, place, use_amp=False): paddle.disable_static() paddle.seed(10) @@ -292,17 +289,17 @@ class TestAdamWOpMultiPrecison(unittest.TestCase): model = paddle.nn.Linear(5, 5) - optimizer = paddle.optimizer.AdamW(parameters=[{ - 'params': - model.parameters(), - 'weight_decay': - 0.001, - 'beta1': - 0.1, - 'beta2': - 0.99 - }], - multi_precision=use_amp) + optimizer = paddle.optimizer.AdamW( + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + } + ], + multi_precision=use_amp, + ) for idx in range(2): if place == 'gpu' and use_amp == True: @@ -338,51 +335,59 @@ class TestAdamWOpMultiPrecison(unittest.TestCase): class TestAdamWOpError(unittest.TestCase): - def test_api_errors(self): - def test_weight_decay_dtype(): linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=1) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=1, + ) def test_parameters_dtype1(): - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=paddle.randn((5, 5)), - weight_decay=0.1) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=paddle.randn((5, 5)), + weight_decay=0.1, + ) def test_parameters_dtype2(): linear = paddle.nn.Linear(13, 5) adam = paddle.optimizer.AdamW( learning_rate=0.01, parameters={'params': linear.parameters()}, - weight_decay=0.1) + weight_decay=0.1, + ) def test_parameters_dtype3(): - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=None, - weight_decay=0.1) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, parameters=None, weight_decay=0.1 + ) def test_parameters_dtype4(): linear = paddle.nn.Linear(13, 5) adam = paddle.optimizer.AdamW( learning_rate=0.01, parameters={'params': set(linear.parameters())}, - weight_decay=0.1) + weight_decay=0.1, + ) def test_learning_rate_dtype(): linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=1, - parameters=linear.parameters(), - weight_decay=0.1) + adam = paddle.optimizer.AdamW( + learning_rate=1, + parameters=linear.parameters(), + weight_decay=0.1, + ) def test_grad_clip_dtype(): linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.1, - grad_clip=0.1) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.1, + grad_clip=0.1, + ) self.assertRaises(TypeError, test_weight_decay_dtype) self.assertRaises(TypeError, test_parameters_dtype1) @@ -394,7 +399,6 @@ class TestAdamWOpError(unittest.TestCase): class TestAdamWOpGroupWithLR(TestAdamWOp): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -403,16 +407,21 @@ class TestAdamWOpGroupWithLR(TestAdamWOp): linear_2 = paddle.nn.Linear(5, 3) adam = paddle.optimizer.AdamW( learning_rate=paddle.optimizer.lr.PiecewiseDecay( - boundaries=[3, 6], values=[0.1, 0.2, 0.3]), - parameters=[{ - 'params': linear_1.parameters(), - 'learning_rate': 0.1, - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001, - }], + boundaries=[3, 6], values=[0.1, 0.2, 0.3] + ), + parameters=[ + { + 'params': linear_1.parameters(), + 'learning_rate': 0.1, + }, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + }, + ], apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + weight_decay=0.01, + ) for _ in range(2): out = linear_1(a) @@ -430,13 +439,13 @@ def simple_lr_setting(param, decay_rate, n_layers): else: depth = 0 - return decay_rate**(n_layers + 2 - depth) + return decay_rate ** (n_layers + 2 - depth) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAdamWOpLayerwiseLR(TestAdamWOp): - def setUp(self): random.seed(2022) np.random.seed(2022) @@ -445,9 +454,11 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): def test_adamw_op_dygraph(self): paddle.disable_static() linear1 = paddle.nn.Linear( - 13, 8, bias_attr=paddle.nn.initializer.Constant(value=1.0)) + 13, 8, bias_attr=paddle.nn.initializer.Constant(value=1.0) + ) linear2 = paddle.nn.Linear( - 8, 5, bias_attr=paddle.nn.initializer.Constant(value=1.0)) + 8, 5, bias_attr=paddle.nn.initializer.Constant(value=1.0) + ) # fix the linear name, simple_lr_setting function will use the name linear1.weight.name = "linear_1.w_0" @@ -475,15 +486,18 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): beta1 = 0.9 beta2 = 0.999 - opt = paddle.optimizer.AdamW(learning_rate=learning_rate, - parameters=[{ - 'params': linear1.parameters() - }, { - 'params': linear2.parameters(), - }], - apply_decay_param_fun=lambda name: True, - weight_decay=weight_decay, - lr_ratio=simple_lr_fun) + opt = paddle.optimizer.AdamW( + learning_rate=learning_rate, + parameters=[ + {'params': linear1.parameters()}, + { + 'params': linear2.parameters(), + }, + ], + apply_decay_param_fun=lambda name: True, + weight_decay=weight_decay, + lr_ratio=simple_lr_fun, + ) def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): np_inputs = { @@ -493,7 +507,7 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1**t]).astype("float32"), - 'Beta2Pow': np.array([beta2**t]).astype("float32") + 'Beta2Pow': np.array([beta2**t]).astype("float32"), } np_attrs = { @@ -502,32 +516,54 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): 'beta2': beta2, "lr_ratio": lr_ratio, "coeff": weight_decay, - "with_decay": True + "with_decay": True, } param_out, moment1_out, moment2_out = adamw_step( - np_inputs, np_attrs) + np_inputs, np_attrs + ) return param_out, moment1_out, moment2_out for i in range(5): a = paddle.to_tensor( - np.random.uniform(-1, 1, (2, 13)).astype("float32")) + np.random.uniform(-1, 1, (2, 13)).astype("float32") + ) a1 = linear1(a) out = linear2(a1) out = paddle.mean(out) out.backward() fc1_w, fc1_w_mon1, fc1_w_mon2 = get_numpy_output( - fc1_w, np.array(linear1.weight.grad), fc1_w_mon1, fc1_w_mon2, - simple_lr_fun(linear1.weight), i + 1) + fc1_w, + np.array(linear1.weight.grad), + fc1_w_mon1, + fc1_w_mon2, + simple_lr_fun(linear1.weight), + i + 1, + ) fc1_b, fc1_b_mon1, fc1_b_mon2 = get_numpy_output( - fc1_b, np.array(linear1.bias.grad), fc1_b_mon1, fc1_b_mon2, - simple_lr_fun(linear1.bias), i + 1) + fc1_b, + np.array(linear1.bias.grad), + fc1_b_mon1, + fc1_b_mon2, + simple_lr_fun(linear1.bias), + i + 1, + ) fc2_w, fc2_w_mon1, fc2_w_mon2 = get_numpy_output( - fc2_w, np.array(linear2.weight.grad), fc2_w_mon1, fc2_w_mon2, - simple_lr_fun(linear2.weight), i + 1) + fc2_w, + np.array(linear2.weight.grad), + fc2_w_mon1, + fc2_w_mon2, + simple_lr_fun(linear2.weight), + i + 1, + ) fc2_b, fc2_b_mon1, fc2_b_mon2 = get_numpy_output( - fc2_b, np.array(linear2.bias.grad), fc2_b_mon1, fc2_b_mon2, - simple_lr_fun(linear2.bias), i + 1) + fc2_b, + np.array(linear2.bias.grad), + fc2_b_mon1, + fc2_b_mon2, + simple_lr_fun(linear2.bias), + i + 1, + ) opt.step() opt.clear_gradients() @@ -557,19 +593,19 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): weight_attr1 = paddle.framework.ParamAttr(name="linear_0.w_0") bias_attr1 = paddle.framework.ParamAttr( name="linear_0.b_0", - initializer=paddle.nn.initializer.Constant(value=1.0)) + initializer=paddle.nn.initializer.Constant(value=1.0), + ) weight_attr2 = paddle.framework.ParamAttr(name="linear_1.w_0") bias_attr2 = paddle.framework.ParamAttr( name="linear_1.b_0", - initializer=paddle.nn.initializer.Constant(value=1.0)) - linear1 = paddle.nn.Linear(10, - 32, - weight_attr=weight_attr1, - bias_attr=bias_attr1) - linear2 = paddle.nn.Linear(32, - 1, - weight_attr=weight_attr2, - bias_attr=bias_attr2) + initializer=paddle.nn.initializer.Constant(value=1.0), + ) + linear1 = paddle.nn.Linear( + 10, 32, weight_attr=weight_attr1, bias_attr=bias_attr1 + ) + linear2 = paddle.nn.Linear( + 32, 1, weight_attr=weight_attr2, bias_attr=bias_attr2 + ) out = linear1(x) out = linear2(out) @@ -586,16 +622,18 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): cost = fluid.layers.square_error_cost(input=out, label=y) avg_cost = paddle.mean(cost) - simple_lr_fun = partial(simple_lr_setting, - decay_rate=0.8, - n_layers=2) - - opt = paddle.optimizer.AdamW(learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - weight_decay=weight_decay, - epsilon=epsilon, - lr_ratio=simple_lr_fun) + simple_lr_fun = partial( + simple_lr_setting, decay_rate=0.8, n_layers=2 + ) + + opt = paddle.optimizer.AdamW( + learning_rate=learning_rate, + beta1=beta1, + beta2=beta2, + weight_decay=weight_decay, + epsilon=epsilon, + lr_ratio=simple_lr_fun, + ) opt.minimize(avg_cost) def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): @@ -606,7 +644,7 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1**t]).astype("float32"), - 'Beta2Pow': np.array([beta2**t]).astype("float32") + 'Beta2Pow': np.array([beta2**t]).astype("float32"), } np_attrs = { @@ -615,19 +653,28 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): 'beta2': beta2, "lr_ratio": lr_ratio, "coeff": weight_decay, - "with_decay": True + "with_decay": True, } param_out, moment1_out, moment2_out = adamw_step( - np_inputs, np_attrs) + np_inputs, np_attrs + ) return param_out, moment1_out, moment2_out fetch_list1 = [ - "linear_0.w_0", "linear_0.b_0", "linear_1.w_0", "linear_1.b_0" + "linear_0.w_0", + "linear_0.b_0", + "linear_1.w_0", + "linear_1.b_0", ] fetch_list2 = [ - "linear_0.w_0", "linear_0.w_0@GRAD", "linear_0.b_0", - "linear_0.b_0@GRAD", "linear_1.w_0", "linear_1.w_0@GRAD", - "linear_1.b_0", "linear_1.b_0@GRAD" + "linear_0.w_0", + "linear_0.w_0@GRAD", + "linear_0.b_0", + "linear_0.b_0@GRAD", + "linear_1.w_0", + "linear_1.w_0@GRAD", + "linear_1.b_0", + "linear_1.b_0@GRAD", ] exe = fluid.Executor(place) @@ -638,18 +685,16 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): inputs = np.random.random(size=[8, 10]).astype('float32') outputs = np.random.random(size=[8, 1]).astype('float32') - param = exe.run(test_prog, - feed={ - "x": inputs, - "y": outputs - }, - fetch_list=fetch_list1) - params_and_gras = exe.run(train_prog, - feed={ - "x": inputs, - "y": outputs - }, - fetch_list=fetch_list2) + param = exe.run( + test_prog, + feed={"x": inputs, "y": outputs}, + fetch_list=fetch_list1, + ) + params_and_gras = exe.run( + train_prog, + feed={"x": inputs, "y": outputs}, + fetch_list=fetch_list2, + ) fc1_w = param[0] fc1_w_grad = params_and_gras[1] @@ -661,17 +706,37 @@ class TestAdamWOpLayerwiseLR(TestAdamWOp): fc2_b_grad = params_and_gras[7] fc1_w, fc1_w_mon1, fc1_w_mon2 = get_numpy_output( - fc1_w, fc1_w_grad, fc1_w_mon1, fc1_w_mon2, - simple_lr_fun(linear1.weight), i + 1) + fc1_w, + fc1_w_grad, + fc1_w_mon1, + fc1_w_mon2, + simple_lr_fun(linear1.weight), + i + 1, + ) fc1_b, fc1_b_mon1, fc1_b_mon2 = get_numpy_output( - fc1_b, fc1_b_grad, fc1_b_mon1, fc1_b_mon2, - simple_lr_fun(linear1.bias), i + 1) + fc1_b, + fc1_b_grad, + fc1_b_mon1, + fc1_b_mon2, + simple_lr_fun(linear1.bias), + i + 1, + ) fc2_w, fc2_w_mon1, fc2_w_mon2 = get_numpy_output( - fc2_w, fc2_w_grad, fc2_w_mon1, fc2_w_mon2, - simple_lr_fun(linear2.weight), i + 1) + fc2_w, + fc2_w_grad, + fc2_w_mon1, + fc2_w_mon2, + simple_lr_fun(linear2.weight), + i + 1, + ) fc2_b, fc2_b_mon1, fc2_b_mon2 = get_numpy_output( - fc2_b, fc2_b_grad, fc2_b_mon1, fc2_b_mon2, - simple_lr_fun(linear2.bias), i + 1) + fc2_b, + fc2_b_grad, + fc2_b_mon1, + fc2_b_mon2, + simple_lr_fun(linear2.bias), + i + 1, + ) np.testing.assert_allclose(params_and_gras[0], fc1_w, rtol=1e-6) np.testing.assert_allclose(params_and_gras[2], fc1_b, rtol=1e-6) diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py index 36c102128cb..532c2e8a45b 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py @@ -30,24 +30,28 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def avg_pool1D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=False, - adaptive=False, - data_type=np.float64): +def avg_pool1D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=False, + adaptive=False, + data_type=np.float64, +): N, C, L = x.shape if global_pool == 1: ksize = [L] if adaptive: L_out = ksize[0] else: - L_out = (L - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + L_out = ( + (L - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) out = np.zeros((N, C, L_out)) for i in range(L_out): @@ -59,19 +63,21 @@ def avg_pool1D_forward_naive(x, r_end = np.min((i * strides[0] + ksize[0] - paddings[0], L)) x_masked = x[:, :, r_start:r_end] - field_size = (r_end - r_start) \ - if (exclusive or adaptive) else (ksize[0]) + field_size = ( + (r_end - r_start) if (exclusive or adaptive) else (ksize[0]) + ) if data_type == np.int8 or data_type == np.uint8: - out[:, :, i] = (np.rint(np.sum(x_masked, axis=(2, 3)) / - field_size)).astype(data_type) + out[:, :, i] = ( + np.rint(np.sum(x_masked, axis=(2, 3)) / field_size) + ).astype(data_type) else: - out[:, :, - i] = (np.sum(x_masked, axis=(2)) / field_size).astype(data_type) + out[:, :, i] = (np.sum(x_masked, axis=(2)) / field_size).astype( + data_type + ) return out class TestPool1D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -83,22 +89,21 @@ class TestPool1D_API(unittest.TestCase): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) result = F.adaptive_avg_pool1d(input, output_size=16) - result_np = avg_pool1D_forward_naive(input_np, - ksize=[16], - strides=[0], - paddings=[0], - adaptive=True) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[16], strides=[0], paddings=[0], adaptive=True + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) ada_max_pool1d_dg = paddle.nn.layer.AdaptiveAvgPool1D( - output_size=16) + output_size=16 + ) result = ada_max_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - result = paddle.nn.functional.common.interpolate(input, - mode="area", - size=16) + result = paddle.nn.functional.common.interpolate( + input, mode="area", size=16 + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) def check_adaptive_avg_static_results(self, place): @@ -107,16 +112,16 @@ class TestPool1D_API(unittest.TestCase): result = F.adaptive_avg_pool1d(input, output_size=16) input_np = np.random.random([2, 3, 32]).astype("float32") - result_np = avg_pool1D_forward_naive(input_np, - ksize=[16], - strides=[2], - paddings=[0], - adaptive=True) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[16], strides=[2], paddings=[0], adaptive=True + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def test_adaptive_avg_pool1d(self): diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py index 9ca75c6b8b1..9c12d5977d4 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py @@ -32,16 +32,18 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def adaptive_pool2d_forward(x, - output_size, - data_format='NCHW', - pool_type="avg"): +def adaptive_pool2d_forward( + x, output_size, data_format='NCHW', pool_type="avg" +): N = x.shape[0] - C, H, W = [x.shape[1], x.shape[2], x.shape[3]] if data_format == 'NCHW' \ + C, H, W = ( + [x.shape[1], x.shape[2], x.shape[3]] + if data_format == 'NCHW' else [x.shape[3], x.shape[1], x.shape[2]] + ) - if (isinstance(output_size, int) or output_size == None): + if isinstance(output_size, int) or output_size == None: H_out = output_size W_out = output_size output_size = [H_out, W_out] @@ -55,8 +57,11 @@ def adaptive_pool2d_forward(x, output_size[1] = W W_out = W - out = np.zeros((N, C, H_out, W_out)) if data_format=='NCHW' \ + out = ( + np.zeros((N, C, H_out, W_out)) + if data_format == 'NCHW' else np.zeros((N, H_out, W_out, C)) + ) for i in range(H_out): in_h_start = adaptive_start_index(i, H, output_size[0]) @@ -69,16 +74,18 @@ def adaptive_pool2d_forward(x, if data_format == 'NCHW': x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end] if pool_type == 'avg': - field_size = ((in_h_end - in_h_start) * - (in_w_end - in_w_start)) + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size elif pool_type == 'max': out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) elif data_format == 'NHWC': x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :] if pool_type == 'avg': - field_size = ((in_h_end - in_h_start) * - (in_w_end - in_w_start)) + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size elif pool_type == 'max': out[:, i, j, :] = np.max(x_masked, axis=(1, 2)) @@ -86,57 +93,60 @@ def adaptive_pool2d_forward(x, class TestAdaptiveAvgPool2DAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="avg") + self.res_1_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="avg" + ) - self.res_2_np = adaptive_pool2d_forward(x=self.x_np, - output_size=5, - pool_type="avg") + self.res_2_np = adaptive_pool2d_forward( + x=self.x_np, output_size=5, pool_type="avg" + ) - self.res_3_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[2, 5], - pool_type="avg") + self.res_3_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[2, 5], pool_type="avg" + ) - self.res_4_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="avg", - data_format="NHWC") + self.res_4_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="avg", data_format="NHWC" + ) - self.res_5_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[None, 3], - pool_type="avg") + self.res_5_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[None, 3], pool_type="avg" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() x = paddle.fluid.data(name="x", shape=[2, 3, 7, 7], dtype="float32") - out_1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[3, 3]) + out_1 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[3, 3] + ) out_2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=5) - out_3 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[2, 5]) + out_3 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[2, 5] + ) - out_4 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[3, 3], - data_format="NHWC") + out_4 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[3, 3], data_format="NHWC" + ) out_5 = paddle.nn.functional.adaptive_avg_pool2d( - x=x, output_size=[None, 3]) + x=x, output_size=[None, 3] + ) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + [res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -149,30 +159,34 @@ class TestAdaptiveAvgPool2DAPI(unittest.TestCase): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) - out_1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[3, 3]) + out_1 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[3, 3] + ) out_2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=5) - out_3 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[2, 5]) + out_3 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[2, 5] + ) - out_4 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[3, 3], - data_format="NHWC") + out_4 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[3, 3], data_format="NHWC" + ) out_5 = paddle.nn.functional.adaptive_avg_pool2d( - x=x, output_size=[None, 3]) + x=x, output_size=[None, 3] + ) - out_6 = paddle.nn.functional.interpolate(x=x, - mode="area", - size=[2, 5]) + out_6 = paddle.nn.functional.interpolate( + x=x, mode="area", size=[2, 5] + ) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -188,33 +202,32 @@ class TestAdaptiveAvgPool2DAPI(unittest.TestCase): class TestAdaptiveAvgPool2DClassAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="avg") + self.res_1_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="avg" + ) - self.res_2_np = adaptive_pool2d_forward(x=self.x_np, - output_size=5, - pool_type="avg") + self.res_2_np = adaptive_pool2d_forward( + x=self.x_np, output_size=5, pool_type="avg" + ) - self.res_3_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[2, 5], - pool_type="avg") + self.res_3_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[2, 5], pool_type="avg" + ) - self.res_4_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="avg", - data_format="NHWC") + self.res_4_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="avg", data_format="NHWC" + ) - self.res_5_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[None, 3], - pool_type="avg") + self.res_5_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[None, 3], pool_type="avg" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() x = paddle.fluid.data(name="x", shape=[2, 3, 7, 7], dtype="float32") @@ -228,19 +241,22 @@ class TestAdaptiveAvgPool2DClassAPI(unittest.TestCase): adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D(output_size=[2, 5]) out_3 = adaptive_avg_pool(x=x) - adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D(output_size=[3, 3], - data_format="NHWC") + adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D( + output_size=[3, 3], data_format="NHWC" + ) out_4 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D( - output_size=[None, 3]) + output_size=[None, 3] + ) out_5 = adaptive_avg_pool(x=x) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + [res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -253,8 +269,9 @@ class TestAdaptiveAvgPool2DClassAPI(unittest.TestCase): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) @@ -268,12 +285,14 @@ class TestAdaptiveAvgPool2DClassAPI(unittest.TestCase): adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D(output_size=[2, 5]) out_3 = adaptive_avg_pool(x=x) - adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D(output_size=[3, 3], - data_format="NHWC") + adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D( + output_size=[3, 3], data_format="NHWC" + ) out_4 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D( - output_size=[None, 3]) + output_size=[None, 3] + ) out_5 = adaptive_avg_pool(x=x) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -288,7 +307,6 @@ class TestAdaptiveAvgPool2DClassAPI(unittest.TestCase): class TestOutputSizeTensor(UnittestBase): - def init_info(self): self.shapes = [[1, 3, 6, 6]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -313,8 +331,9 @@ class TestOutputSizeTensor(UnittestBase): exe.run(starup_prog) res = exe.run(fetch_list=[out1, out2]) np.testing.assert_allclose(res[0], res[1]) - paddle.static.save_inference_model(self.save_path, [x], - [out1, out2], exe) + paddle.static.save_inference_model( + self.save_path, [x], [out1, out2], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() np.testing.assert_array_equal(infer_outs[0].shape, (1, 3, 3, 3)) @@ -330,13 +349,13 @@ class TestOutputSizeTensor(UnittestBase): # list[Tensor] output_size = [paddle.assign([3]), paddle.assign([3])] out1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=[3, 3]) - out2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=output_size) + out2 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=output_size + ) return out1, out2 class TestOutputSizeListTensor(TestOutputSizeTensor): - def path_prefix(self): return 'pool2d_tensors' @@ -344,13 +363,13 @@ class TestOutputSizeListTensor(TestOutputSizeTensor): # list[int, Tensor] output_size = [paddle.assign([3]), 3] out1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=[3, 3]) - out2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=output_size) + out2 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=output_size + ) return out1, out2 class TestOutputSizeListTensor2(TestOutputSizeTensor): - def path_prefix(self): return 'pool2d_tensor2' @@ -358,8 +377,9 @@ class TestOutputSizeListTensor2(TestOutputSizeTensor): # A Tensor output_size = paddle.assign([3, 3]) out1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=[3, 3]) - out2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=output_size) + out2 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=output_size + ) return out1, out2 diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py index 5f3e2d38eb1..e7a8685a8f1 100755 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py @@ -28,17 +28,18 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def adaptive_pool3d_forward(x, - output_size, - adaptive=True, - data_format='NCDHW', - pool_type='avg'): +def adaptive_pool3d_forward( + x, output_size, adaptive=True, data_format='NCDHW', pool_type='avg' +): N = x.shape[0] - C, D, H, W = [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] \ - if data_format == 'NCDHW' else [x.shape[4], x.shape[1], x.shape[2],x.shape[3]] + C, D, H, W = ( + [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] + if data_format == 'NCDHW' + else [x.shape[4], x.shape[1], x.shape[2], x.shape[3]] + ) - if (isinstance(output_size, int) or output_size == None): + if isinstance(output_size, int) or output_size == None: H_out = output_size W_out = output_size D_out = output_size @@ -56,8 +57,11 @@ def adaptive_pool3d_forward(x, output_size[2] = W W_out = W - out = np.zeros((N, C, D_out, H_out, W_out)) if data_format=='NCDHW' \ + out = ( + np.zeros((N, C, D_out, H_out, W_out)) + if data_format == 'NCDHW' else np.zeros((N, D_out, H_out, W_out, C)) + ) for k in range(D_out): d_start = adaptive_start_index(k, D, output_size[0]) d_end = adaptive_end_index(k, D, output_size[0]) @@ -71,82 +75,99 @@ def adaptive_pool3d_forward(x, w_end = adaptive_end_index(j, W, output_size[2]) if data_format == 'NCDHW': - x_masked = x[:, :, d_start:d_end, h_start:h_end, - w_start:w_end] + x_masked = x[ + :, :, d_start:d_end, h_start:h_end, w_start:w_end + ] if pool_type == 'avg': - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) - out[:, :, k, i, - j] = np.sum(x_masked, axis=(2, 3, 4)) / field_size + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + out[:, :, k, i, j] = ( + np.sum(x_masked, axis=(2, 3, 4)) / field_size + ) elif pool_type == 'max': out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) elif data_format == 'NDHWC': - x_masked = x[:, d_start:d_end, h_start:h_end, - w_start:w_end, :] + x_masked = x[ + :, d_start:d_end, h_start:h_end, w_start:w_end, : + ] if pool_type == 'avg': - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) - out[:, k, i, j, :] = np.sum(x_masked, - axis=(1, 2, 3)) / field_size + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + out[:, k, i, j, :] = ( + np.sum(x_masked, axis=(1, 2, 3)) / field_size + ) elif pool_type == 'max': out[:, k, i, j, :] = np.max(x_masked, axis=(1, 2, 3)) return out class TestAdaptiveAvgPool3DAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 5, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="avg") + self.res_1_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[3, 3, 3], pool_type="avg" + ) - self.res_2_np = adaptive_pool3d_forward(x=self.x_np, - output_size=5, - pool_type="avg") + self.res_2_np = adaptive_pool3d_forward( + x=self.x_np, output_size=5, pool_type="avg" + ) - self.res_3_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[2, 3, 5], - pool_type="avg") + self.res_3_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[2, 3, 5], pool_type="avg" + ) - self.res_4_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="avg", - data_format="NDHWC") + self.res_4_np = adaptive_pool3d_forward( + x=self.x_np, + output_size=[3, 3, 3], + pool_type="avg", + data_format="NDHWC", + ) - self.res_5_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[None, 3, None], - pool_type="avg") + self.res_5_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[None, 3, None], pool_type="avg" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x = paddle.fluid.data(name="x", - shape=[2, 3, 5, 7, 7], - dtype="float32") + x = paddle.fluid.data( + name="x", shape=[2, 3, 5, 7, 7], dtype="float32" + ) out_1 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[3, 3, 3]) + x=x, output_size=[3, 3, 3] + ) out_2 = paddle.nn.functional.adaptive_avg_pool3d(x=x, output_size=5) out_3 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[2, 3, 5]) + x=x, output_size=[2, 3, 5] + ) out_4 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[3, 3, 3], data_format="NDHWC") + x=x, output_size=[3, 3, 3], data_format="NDHWC" + ) out_5 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[None, 3, None]) + x=x, output_size=[None, 3, None] + ) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + [res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -159,29 +180,34 @@ class TestAdaptiveAvgPool3DAPI(unittest.TestCase): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) out_1 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[3, 3, 3]) + x=x, output_size=[3, 3, 3] + ) out_2 = paddle.nn.functional.adaptive_avg_pool3d(x=x, output_size=5) out_3 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[2, 3, 5]) + x=x, output_size=[2, 3, 5] + ) out_4 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[3, 3, 3], data_format="NDHWC") + x=x, output_size=[3, 3, 3], data_format="NDHWC" + ) out_5 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[None, 3, None]) + x=x, output_size=[None, 3, None] + ) - out_6 = paddle.nn.functional.interpolate(x=x, - mode="area", - size=[2, 3, 5]) + out_6 = paddle.nn.functional.interpolate( + x=x, mode="area", size=[2, 3, 5] + ) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -197,63 +223,70 @@ class TestAdaptiveAvgPool3DAPI(unittest.TestCase): class TestAdaptiveAvgPool3DClassAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 5, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="avg") + self.res_1_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[3, 3, 3], pool_type="avg" + ) - self.res_2_np = adaptive_pool3d_forward(x=self.x_np, - output_size=5, - pool_type="avg") + self.res_2_np = adaptive_pool3d_forward( + x=self.x_np, output_size=5, pool_type="avg" + ) - self.res_3_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[2, 3, 5], - pool_type="avg") + self.res_3_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[2, 3, 5], pool_type="avg" + ) - self.res_4_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="avg", - data_format="NDHWC") + self.res_4_np = adaptive_pool3d_forward( + x=self.x_np, + output_size=[3, 3, 3], + pool_type="avg", + data_format="NDHWC", + ) - self.res_5_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[None, 3, None], - pool_type="avg") + self.res_5_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[None, 3, None], pool_type="avg" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x = paddle.fluid.data(name="x", - shape=[2, 3, 5, 7, 7], - dtype="float32") + x = paddle.fluid.data( + name="x", shape=[2, 3, 5, 7, 7], dtype="float32" + ) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[3, 3, 3]) + output_size=[3, 3, 3] + ) out_1 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D(output_size=5) out_2 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[2, 3, 5]) + output_size=[2, 3, 5] + ) out_3 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[3, 3, 3], data_format="NDHWC") + output_size=[3, 3, 3], data_format="NDHWC" + ) out_4 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[None, 3, None]) + output_size=[None, 3, None] + ) out_5 = adaptive_avg_pool(x=x) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + [res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -266,29 +299,34 @@ class TestAdaptiveAvgPool3DClassAPI(unittest.TestCase): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[3, 3, 3]) + output_size=[3, 3, 3] + ) out_1 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D(output_size=5) out_2 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[2, 3, 5]) + output_size=[2, 3, 5] + ) out_3 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[3, 3, 3], data_format="NDHWC") + output_size=[3, 3, 3], data_format="NDHWC" + ) out_4 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[None, 3, None]) + output_size=[None, 3, None] + ) out_5 = adaptive_avg_pool(x=x) assert np.allclose(out_1.numpy(), self.res_1_np) diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool1d.py b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool1d.py index d64ff0a3c76..a83ef2bfd5b 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool1d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool1d.py @@ -29,24 +29,28 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def max_pool1D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=False, - adaptive=False, - data_type=np.float64): +def max_pool1D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=False, + adaptive=False, + data_type=np.float64, +): N, C, L = x.shape if global_pool == 1: ksize = [L] if adaptive: L_out = ksize[0] else: - L_out = (L - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + L_out = ( + (L - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) out = np.zeros((N, C, L_out)) for i in range(L_out): @@ -63,7 +67,6 @@ def max_pool1D_forward_naive(x, class TestPool1D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -76,15 +79,14 @@ class TestPool1D_API(unittest.TestCase): input = fluid.dygraph.to_variable(input_np) result = F.adaptive_max_pool1d(input, output_size=16) - result_np = max_pool1D_forward_naive(input_np, - ksize=[16], - strides=[0], - paddings=[0], - adaptive=True) + result_np = max_pool1D_forward_naive( + input_np, ksize=[16], strides=[0], paddings=[0], adaptive=True + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) ada_max_pool1d_dg = paddle.nn.layer.AdaptiveMaxPool1D( - output_size=16) + output_size=16 + ) result = ada_max_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -94,16 +96,16 @@ class TestPool1D_API(unittest.TestCase): result = F.adaptive_max_pool1d(input, output_size=16) input_np = np.random.random([2, 3, 32]).astype("float32") - result_np = max_pool1D_forward_naive(input_np, - ksize=[16], - strides=[2], - paddings=[0], - adaptive=True) + result_np = max_pool1D_forward_naive( + input_np, ksize=[16], strides=[2], paddings=[0], adaptive=True + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def test_adaptive_max_pool1d(self): @@ -113,14 +115,15 @@ class TestPool1D_API(unittest.TestCase): class TestOutDtype(unittest.TestCase): - def test_max_pool(self): api_fn = F.adaptive_max_pool1d shape = [1, 3, 32] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64'], - output_size=16) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64'], + output_size=16, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool2d.py b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool2d.py index 5ee4918a753..9884cb6c90d 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool2d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool2d.py @@ -30,16 +30,18 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def adaptive_pool2d_forward(x, - output_size, - data_format='NCHW', - pool_type="max"): +def adaptive_pool2d_forward( + x, output_size, data_format='NCHW', pool_type="max" +): N = x.shape[0] - C, H, W = [x.shape[1], x.shape[2], x.shape[3]] if data_format == 'NCHW' \ + C, H, W = ( + [x.shape[1], x.shape[2], x.shape[3]] + if data_format == 'NCHW' else [x.shape[3], x.shape[1], x.shape[2]] + ) - if (isinstance(output_size, int) or output_size == None): + if isinstance(output_size, int) or output_size == None: H_out = output_size W_out = output_size output_size = [H_out, W_out] @@ -53,8 +55,11 @@ def adaptive_pool2d_forward(x, output_size[1] = W W_out = W - out = np.zeros((N, C, H_out, W_out)) if data_format=='NCHW' \ + out = ( + np.zeros((N, C, H_out, W_out)) + if data_format == 'NCHW' else np.zeros((N, H_out, W_out, C)) + ) for i in range(H_out): in_h_start = adaptive_start_index(i, H, output_size[0]) @@ -67,16 +72,18 @@ def adaptive_pool2d_forward(x, if data_format == 'NCHW': x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end] if pool_type == 'avg': - field_size = ((in_h_end - in_h_start) * - (in_w_end - in_w_start)) + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size elif pool_type == 'max': out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) elif data_format == 'NHWC': x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :] if pool_type == 'avg': - field_size = ((in_h_end - in_h_start) * - (in_w_end - in_w_start)) + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size elif pool_type == 'max': out[:, i, j, :] = np.max(x_masked, axis=(1, 2)) @@ -84,20 +91,19 @@ def adaptive_pool2d_forward(x, class TestAdaptiveMaxPool2DAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="max") + self.res_1_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="max" + ) - self.res_2_np = adaptive_pool2d_forward(x=self.x_np, - output_size=5, - pool_type="max") + self.res_2_np = adaptive_pool2d_forward( + x=self.x_np, output_size=5, pool_type="max" + ) - self.res_3_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[2, 5], - pool_type="max") + self.res_3_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[2, 5], pool_type="max" + ) """ self.res_4_np = adaptive_pool2d_forward( x=self.x_np, @@ -105,36 +111,41 @@ class TestAdaptiveMaxPool2DAPI(unittest.TestCase): pool_type="max", data_format="NHWC") """ - self.res_5_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[None, 3], - pool_type="max") + self.res_5_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[None, 3], pool_type="max" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() x = paddle.fluid.data(name="x", shape=[2, 3, 7, 7], dtype="float32") - out_1 = paddle.nn.functional.adaptive_max_pool2d(x=x, - output_size=[3, 3]) + out_1 = paddle.nn.functional.adaptive_max_pool2d( + x=x, output_size=[3, 3] + ) out_2 = paddle.nn.functional.adaptive_max_pool2d(x=x, output_size=5) - out_3 = paddle.nn.functional.adaptive_max_pool2d(x=x, - output_size=[2, 5]) + out_3 = paddle.nn.functional.adaptive_max_pool2d( + x=x, output_size=[2, 5] + ) - #out_4 = paddle.nn.functional.adaptive_max_pool2d( + # out_4 = paddle.nn.functional.adaptive_max_pool2d( # x=x, output_size=[3, 3], data_format="NHWC") out_5 = paddle.nn.functional.adaptive_max_pool2d( - x=x, output_size=[None, 3]) + x=x, output_size=[None, 3] + ) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_5]) + [res_1, res_2, res_3, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -142,31 +153,34 @@ class TestAdaptiveMaxPool2DAPI(unittest.TestCase): assert np.allclose(res_3, self.res_3_np) - #assert np.allclose(res_4, self.res_4_np) + # assert np.allclose(res_4, self.res_4_np) assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) - out_1 = paddle.nn.functional.adaptive_max_pool2d(x=x, - return_mask=False, - output_size=[3, 3]) + out_1 = paddle.nn.functional.adaptive_max_pool2d( + x=x, return_mask=False, output_size=[3, 3] + ) out_2 = paddle.nn.functional.adaptive_max_pool2d(x=x, output_size=5) - out_3 = paddle.nn.functional.adaptive_max_pool2d(x=x, - output_size=[2, 5]) + out_3 = paddle.nn.functional.adaptive_max_pool2d( + x=x, output_size=[2, 5] + ) - #out_4 = paddle.nn.functional.adaptive_max_pool2d( + # out_4 = paddle.nn.functional.adaptive_max_pool2d( # x=x, output_size=[3, 3], data_format="NHWC") out_5 = paddle.nn.functional.adaptive_max_pool2d( - x=x, output_size=[None, 3]) + x=x, output_size=[None, 3] + ) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -174,40 +188,40 @@ class TestAdaptiveMaxPool2DAPI(unittest.TestCase): assert np.allclose(out_3.numpy(), self.res_3_np) - #assert np.allclose(out_4.numpy(), self.res_4_np) + # assert np.allclose(out_4.numpy(), self.res_4_np) assert np.allclose(out_5.numpy(), self.res_5_np) class TestAdaptiveMaxPool2DClassAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="max") + self.res_1_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="max" + ) - self.res_2_np = adaptive_pool2d_forward(x=self.x_np, - output_size=5, - pool_type="max") + self.res_2_np = adaptive_pool2d_forward( + x=self.x_np, output_size=5, pool_type="max" + ) - self.res_3_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[2, 5], - pool_type="max") + self.res_3_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[2, 5], pool_type="max" + ) - #self.res_4_np = adaptive_pool2d_forward( + # self.res_4_np = adaptive_pool2d_forward( # x=self.x_np, # output_size=[3, 3], # pool_type="max", # data_format="NHWC") - self.res_5_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[None, 3], - pool_type="max") + self.res_5_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[None, 3], pool_type="max" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() x = paddle.fluid.data(name="x", shape=[2, 3, 7, 7], dtype="float32") @@ -226,14 +240,16 @@ class TestAdaptiveMaxPool2DClassAPI(unittest.TestCase): # out_4 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D( - output_size=[None, 3]) + output_size=[None, 3] + ) out_5 = adaptive_max_pool(x=x) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_5]) + [res_1, res_2, res_3, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -241,13 +257,14 @@ class TestAdaptiveMaxPool2DClassAPI(unittest.TestCase): assert np.allclose(res_3, self.res_3_np) - #assert np.allclose(res_4, self.res_4_np) + # assert np.allclose(res_4, self.res_4_np) assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) @@ -261,12 +278,13 @@ class TestAdaptiveMaxPool2DClassAPI(unittest.TestCase): adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D(output_size=[2, 5]) out_3 = adaptive_max_pool(x=x) - #adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D( + # adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D( # output_size=[3, 3], data_format="NHWC") - #out_4 = adaptive_max_pool(x=x) + # out_4 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D( - output_size=[None, 3]) + output_size=[None, 3] + ) out_5 = adaptive_max_pool(x=x) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -275,20 +293,21 @@ class TestAdaptiveMaxPool2DClassAPI(unittest.TestCase): assert np.allclose(out_3.numpy(), self.res_3_np) - #assert np.allclose(out_4.numpy(), self.res_4_np) + # assert np.allclose(out_4.numpy(), self.res_4_np) assert np.allclose(out_5.numpy(), self.res_5_np) class TestOutDtype(unittest.TestCase): - def test_max_pool(self): api_fn = F.adaptive_max_pool2d shape = [1, 3, 32, 32] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64'], - output_size=16) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64'], + output_size=16, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool3d.py b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool3d.py index 214b6d90b61..e90c4061980 100755 --- a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool3d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool3d.py @@ -30,17 +30,18 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def adaptive_pool3d_forward(x, - output_size, - adaptive=True, - data_format='NCDHW', - pool_type='max'): +def adaptive_pool3d_forward( + x, output_size, adaptive=True, data_format='NCDHW', pool_type='max' +): N = x.shape[0] - C, D, H, W = [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] \ - if data_format == 'NCDHW' else [x.shape[4], x.shape[1], x.shape[2],x.shape[3]] + C, D, H, W = ( + [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] + if data_format == 'NCDHW' + else [x.shape[4], x.shape[1], x.shape[2], x.shape[3]] + ) - if (isinstance(output_size, int) or output_size == None): + if isinstance(output_size, int) or output_size == None: H_out = output_size W_out = output_size D_out = output_size @@ -58,8 +59,11 @@ def adaptive_pool3d_forward(x, output_size[2] = W W_out = W - out = np.zeros((N, C, D_out, H_out, W_out)) if data_format=='NCDHW' \ + out = ( + np.zeros((N, C, D_out, H_out, W_out)) + if data_format == 'NCDHW' else np.zeros((N, D_out, H_out, W_out, C)) + ) for k in range(D_out): d_start = adaptive_start_index(k, D, output_size[0]) d_end = adaptive_end_index(k, D, output_size[0]) @@ -73,82 +77,98 @@ def adaptive_pool3d_forward(x, w_end = adaptive_end_index(j, W, output_size[2]) if data_format == 'NCDHW': - x_masked = x[:, :, d_start:d_end, h_start:h_end, - w_start:w_end] + x_masked = x[ + :, :, d_start:d_end, h_start:h_end, w_start:w_end + ] if pool_type == 'avg': - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) - out[:, :, k, i, - j] = np.sum(x_masked, axis=(2, 3, 4)) / field_size + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + out[:, :, k, i, j] = ( + np.sum(x_masked, axis=(2, 3, 4)) / field_size + ) elif pool_type == 'max': out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) elif data_format == 'NDHWC': - x_masked = x[:, d_start:d_end, h_start:h_end, - w_start:w_end, :] + x_masked = x[ + :, d_start:d_end, h_start:h_end, w_start:w_end, : + ] if pool_type == 'avg': - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) - out[:, k, i, j, :] = np.sum(x_masked, - axis=(1, 2, 3)) / field_size + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + out[:, k, i, j, :] = ( + np.sum(x_masked, axis=(1, 2, 3)) / field_size + ) elif pool_type == 'max': out[:, k, i, j, :] = np.max(x_masked, axis=(1, 2, 3)) return out class TestAdaptiveMaxPool3DAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 5, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="max") + self.res_1_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[3, 3, 3], pool_type="max" + ) - self.res_2_np = adaptive_pool3d_forward(x=self.x_np, - output_size=5, - pool_type="max") + self.res_2_np = adaptive_pool3d_forward( + x=self.x_np, output_size=5, pool_type="max" + ) - self.res_3_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[2, 3, 5], - pool_type="max") + self.res_3_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[2, 3, 5], pool_type="max" + ) - self.res_4_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="max", - data_format="NDHWC") + self.res_4_np = adaptive_pool3d_forward( + x=self.x_np, + output_size=[3, 3, 3], + pool_type="max", + data_format="NDHWC", + ) - self.res_5_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[None, 3, None], - pool_type="max") + self.res_5_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[None, 3, None], pool_type="max" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x = paddle.fluid.data(name="x", - shape=[2, 3, 5, 7, 7], - dtype="float32") + x = paddle.fluid.data( + name="x", shape=[2, 3, 5, 7, 7], dtype="float32" + ) out_1 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[3, 3, 3]) + x=x, output_size=[3, 3, 3] + ) out_2 = paddle.nn.functional.adaptive_max_pool3d(x=x, output_size=5) out_3 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[2, 3, 5]) + x=x, output_size=[2, 3, 5] + ) - #out_4 = paddle.nn.functional.adaptive_max_pool3d( + # out_4 = paddle.nn.functional.adaptive_max_pool3d( # x=x, output_size=[3, 3, 3], data_format="NDHWC") out_5 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[None, 3, None]) + x=x, output_size=[None, 3, None] + ) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_5]) + [res_1, res_2, res_3, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -156,30 +176,34 @@ class TestAdaptiveMaxPool3DAPI(unittest.TestCase): assert np.allclose(res_3, self.res_3_np) - #assert np.allclose(res_4, self.res_4_np) + # assert np.allclose(res_4, self.res_4_np) assert np.allclose(res_5, self.res_5_np) def func_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) out_1 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[3, 3, 3]) + x=x, output_size=[3, 3, 3] + ) out_2 = paddle.nn.functional.adaptive_max_pool3d(x=x, output_size=5) out_3 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[2, 3, 5]) + x=x, output_size=[2, 3, 5] + ) - #out_4 = paddle.nn.functional.adaptive_max_pool3d( + # out_4 = paddle.nn.functional.adaptive_max_pool3d( # x=x, output_size=[3, 3, 3], data_format="NDHWC") out_5 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[None, 3, None]) + x=x, output_size=[None, 3, None] + ) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -187,7 +211,7 @@ class TestAdaptiveMaxPool3DAPI(unittest.TestCase): assert np.allclose(out_3.numpy(), self.res_3_np) - #assert np.allclose(out_4.numpy(), self.res_4_np) + # assert np.allclose(out_4.numpy(), self.res_4_np) assert np.allclose(out_5.numpy(), self.res_5_np) @@ -198,20 +222,19 @@ class TestAdaptiveMaxPool3DAPI(unittest.TestCase): class TestAdaptiveMaxPool3DClassAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 5, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="max") + self.res_1_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[3, 3, 3], pool_type="max" + ) - self.res_2_np = adaptive_pool3d_forward(x=self.x_np, - output_size=5, - pool_type="max") + self.res_2_np = adaptive_pool3d_forward( + x=self.x_np, output_size=5, pool_type="max" + ) - self.res_3_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[2, 3, 5], - pool_type="max") + self.res_3_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[2, 3, 5], pool_type="max" + ) # self.res_4_np = adaptive_pool3d_forward( # x=self.x_np, @@ -219,28 +242,31 @@ class TestAdaptiveMaxPool3DClassAPI(unittest.TestCase): # pool_type="max", # data_format="NDHWC") - self.res_5_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[None, 3, None], - pool_type="max") + self.res_5_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[None, 3, None], pool_type="max" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x = paddle.fluid.data(name="x", - shape=[2, 3, 5, 7, 7], - dtype="float32") + x = paddle.fluid.data( + name="x", shape=[2, 3, 5, 7, 7], dtype="float32" + ) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[3, 3, 3]) + output_size=[3, 3, 3] + ) out_1 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D(output_size=5) out_2 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[2, 3, 5]) + output_size=[2, 3, 5] + ) out_3 = adaptive_max_pool(x=x) # adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( @@ -248,14 +274,16 @@ class TestAdaptiveMaxPool3DClassAPI(unittest.TestCase): # out_4 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[None, 3, None]) + output_size=[None, 3, None] + ) out_5 = adaptive_max_pool(x=x) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_5]) + [res_1, res_2, res_3, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -268,21 +296,24 @@ class TestAdaptiveMaxPool3DClassAPI(unittest.TestCase): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[3, 3, 3]) + output_size=[3, 3, 3] + ) out_1 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D(output_size=5) out_2 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[2, 3, 5]) + output_size=[2, 3, 5] + ) out_3 = adaptive_max_pool(x=x) # adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( @@ -290,7 +321,8 @@ class TestAdaptiveMaxPool3DClassAPI(unittest.TestCase): # out_4 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[None, 3, None]) + output_size=[None, 3, None] + ) out_5 = adaptive_max_pool(x=x) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -305,14 +337,15 @@ class TestAdaptiveMaxPool3DClassAPI(unittest.TestCase): class TestOutDtype(unittest.TestCase): - def test_max_pool(self): api_fn = F.adaptive_max_pool3d shape = [1, 3, 32, 32, 32] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64'], - output_size=16) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64'], + output_size=16, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_add_position_encoding_op.py b/python/paddle/fluid/tests/unittests/test_add_position_encoding_op.py index aa510efc59d..791c2351d73 100644 --- a/python/paddle/fluid/tests/unittests/test_add_position_encoding_op.py +++ b/python/paddle/fluid/tests/unittests/test_add_position_encoding_op.py @@ -31,13 +31,15 @@ def add_position_encoding(input, alpha=1.0, beta=1.0): for i in range(batch_size): for j in range(max_length): for k in range(half_shape): - val = j / pow( - 10000.0, k * 1.0 / - (half_shape - 1)) if half_shape > 1 else j / 10000.0 - out[i, j, k] = \ - input[i, j, k] * alpha + math.sin(val) * beta - out[i, j, half_shape + k] = \ + val = ( + j / pow(10000.0, k * 1.0 / (half_shape - 1)) + if half_shape > 1 + else j / 10000.0 + ) + out[i, j, k] = input[i, j, k] * alpha + math.sin(val) * beta + out[i, j, half_shape + k] = ( input[i, j, half_shape + k] * alpha + math.cos(val) * beta + ) return out @@ -133,45 +135,49 @@ class TestAddPositionEncodingLoDTensorOp(OpTest): max_length = self.lod[0][i] for j in range(max_length): for k in range(half_shape): - val = j / pow( - 10000.0, k * 1.0 / - (half_shape - 1)) if half_shape > 1 else j / 10000.0 + val = ( + j / pow(10000.0, k * 1.0 / (half_shape - 1)) + if half_shape > 1 + else j / 10000.0 + ) pos = start + j - self.out[pos, k] = \ + self.out[pos, k] = ( self.x[pos, k] * self.alpha + math.sin(val) * self.beta - self.out[pos, half_shape + k] = \ - self.x[pos, half_shape + k] * self.alpha + math.cos(val) * self.beta + ) + self.out[pos, half_shape + k] = ( + self.x[pos, half_shape + k] * self.alpha + + math.cos(val) * self.beta + ) start += max_length class TestAddPositionEncodingOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((4, 16, 8)).astype("float32") def test_Variable(): # the input type must be Variable - fluid.layers.add_position_encoding(input=input_data, - alpha=1.0, - beta=1.0) + fluid.layers.add_position_encoding( + input=input_data, alpha=1.0, beta=1.0 + ) self.assertRaises(TypeError, test_Variable) class TestAddPositionEncodingOpDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() tensor = np.random.randn(16, 32, 64) position_tensor = paddle.fluid.layers.add_position_encoding( - input=paddle.to_tensor(tensor), alpha=1.0, beta=1.0).numpy() + input=paddle.to_tensor(tensor), alpha=1.0, beta=1.0 + ).numpy() paddle.enable_static() position_tensor_np = add_position_encoding(tensor, 1.0, 1.0) - np.testing.assert_allclose(position_tensor, - position_tensor_np, - rtol=1e-05) + np.testing.assert_allclose( + position_tensor, position_tensor_np, rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_add_reader_dependency.py b/python/paddle/fluid/tests/unittests/test_add_reader_dependency.py index c7479e059b4..1604445a873 100644 --- a/python/paddle/fluid/tests/unittests/test_add_reader_dependency.py +++ b/python/paddle/fluid/tests/unittests/test_add_reader_dependency.py @@ -21,15 +21,16 @@ import time def inplace_add(x, bias): helper = LayerHelper('scale', **locals()) - helper.append_op(type='scale', - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={'bias': bias}) + helper.append_op( + type='scale', + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={'bias': bias}, + ) return x class TestAddReaderDependency(unittest.TestCase): - def setUp(self): self.batch_num = 3 self.sleep_time = 2 @@ -49,17 +50,19 @@ class TestAddReaderDependency(unittest.TestCase): feed_list=[tmp_in], capacity=16, iterable=False, - use_double_buffer=self.use_double_buffer) + use_double_buffer=self.use_double_buffer, + ) def data_source(): for _ in range(self.batch_num): time.sleep(self.sleep_time) # sleep some times - yield np.random.uniform(low=-1, high=1, - size=[1]).astype('float32'), + yield np.random.uniform( + low=-1, high=1, size=[1] + ).astype('float32'), - persistable_in = fluid.data(name='persistable_in', - dtype='float32', - shape=[1]) + persistable_in = fluid.data( + name='persistable_in', dtype='float32', shape=[1] + ) persistable_in.persistable = True persistable_in = inplace_add(persistable_in, bias=1) @@ -74,31 +77,34 @@ class TestAddReaderDependency(unittest.TestCase): while True: if batch_id == 0: feed = { - persistable_in.name: - np.array([-1]).astype('float32') + persistable_in.name: np.array([-1]).astype( + 'float32' + ) } else: feed = None - ret, = exe.run(prog, - feed=feed, - fetch_list=[persistable_in]) - self.assertEqual(ret.shape, (1, )) + (ret,) = exe.run( + prog, feed=feed, fetch_list=[persistable_in] + ) + self.assertEqual(ret.shape, (1,)) self.assertEqual(ret[0], batch_id) batch_id += 1 except fluid.core.EOFException: loader.reset() self.assertEqual(batch_id, self.batch_num) - t = fluid.global_scope().find_var( - persistable_in.name).get_tensor() + t = ( + fluid.global_scope() + .find_var(persistable_in.name) + .get_tensor() + ) t_val = np.array(t) - self.assertEqual(t_val.shape, (1, )) + self.assertEqual(t_val.shape, (1,)) self.assertEqual(t_val[0] + 1, batch_id) class TestAddReaderDependencyWithoutDoubleBuffer(TestAddReaderDependency): - def setUp(self): self.batch_num = 3 self.sleep_time = 2 diff --git a/python/paddle/fluid/tests/unittests/test_addmm_op.py b/python/paddle/fluid/tests/unittests/test_addmm_op.py index de87233cdb5..f101f55dc73 100644 --- a/python/paddle/fluid/tests/unittests/test_addmm_op.py +++ b/python/paddle/fluid/tests/unittests/test_addmm_op.py @@ -33,8 +33,8 @@ class TestAddMMOp(OpTest): 'Y': np.random.random((10, 20)).astype(self.dtype), } self.outputs = { - 'Out': - self.inputs['Input'] + np.dot(self.inputs['X'], self.inputs['Y']) + 'Out': self.inputs['Input'] + + np.dot(self.inputs['X'], self.inputs['Y']) } def init_dtype_type(self): @@ -62,91 +62,122 @@ class TestAddMMOpError(unittest.TestCase): with program_guard(Program(), Program()): # The input type of addmm_op must be Variable. - input = fluid.create_lod_tensor(np.array([[-1, -1], [-1, -1]]), - [[2]], fluid.CPUPlace()) - x1 = fluid.create_lod_tensor(np.array([[-1, -1], [-1, -1]]), [[2]], - fluid.CPUPlace()) - x2 = fluid.create_lod_tensor(np.array([[-1, -1], [-1, -1]]), [[2]], - fluid.CPUPlace()) + input = fluid.create_lod_tensor( + np.array([[-1, -1], [-1, -1]]), [[2]], fluid.CPUPlace() + ) + x1 = fluid.create_lod_tensor( + np.array([[-1, -1], [-1, -1]]), [[2]], fluid.CPUPlace() + ) + x2 = fluid.create_lod_tensor( + np.array([[-1, -1], [-1, -1]]), [[2]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.addmm, input, x1, x2) # The input dtype of mul_op must be float32 or float64. - input = fluid.layers.data(name='input', - shape=[4, 4], - dtype="int32", - append_batch_size=False) - x3 = fluid.layers.data(name='x3', - shape=[4, 4], - dtype="int32", - append_batch_size=False) - x4 = fluid.layers.data(name='x4', - shape=[4, 4], - dtype="int32", - append_batch_size=False) + input = fluid.layers.data( + name='input', + shape=[4, 4], + dtype="int32", + append_batch_size=False, + ) + x3 = fluid.layers.data( + name='x3', shape=[4, 4], dtype="int32", append_batch_size=False + ) + x4 = fluid.layers.data( + name='x4', shape=[4, 4], dtype="int32", append_batch_size=False + ) self.assertRaises(TypeError, paddle.addmm, input, x3, x4) # x and y dimension mismatch - x5 = fluid.layers.data(name='x5', - shape=[4, 5], - dtype="float32", - append_batch_size=False) - x6 = fluid.layers.data(name='x6', - shape=[4, 4], - dtype="float32", - append_batch_size=False) + x5 = fluid.layers.data( + name='x5', + shape=[4, 5], + dtype="float32", + append_batch_size=False, + ) + x6 = fluid.layers.data( + name='x6', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input, x5, x6) # input and x are not broadcastable - x7 = fluid.layers.data(name='x7', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - x8 = fluid.layers.data(name='x8', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - input1 = fluid.layers.data(name='input1', - shape=[2, 4], - dtype="float32", - append_batch_size=False) + x7 = fluid.layers.data( + name='x7', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + x8 = fluid.layers.data( + name='x8', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + input1 = fluid.layers.data( + name='input1', + shape=[2, 4], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input1, x7, x8) # input and x are not broadcastable - x9 = fluid.layers.data(name='x9', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - x10 = fluid.layers.data(name='x10', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - input2 = fluid.layers.data(name='input2', - shape=[1, 2], - dtype="float32", - append_batch_size=False) + x9 = fluid.layers.data( + name='x9', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + x10 = fluid.layers.data( + name='x10', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + input2 = fluid.layers.data( + name='input2', + shape=[1, 2], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input2, x9, x10) - x11 = fluid.layers.data(name='x11', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - x12 = fluid.layers.data(name='x12', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - input3 = fluid.layers.data(name='input3', - shape=[4, 2], - dtype="float32", - append_batch_size=False) + x11 = fluid.layers.data( + name='x11', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + x12 = fluid.layers.data( + name='x12', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + input3 = fluid.layers.data( + name='input3', + shape=[4, 2], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input3, x11, x12) - x13 = fluid.layers.data(name='x13', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - x14 = fluid.layers.data(name='x14', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - input4 = fluid.layers.data(name='input4', - shape=[3, 1], - dtype="float32", - append_batch_size=False) + x13 = fluid.layers.data( + name='x13', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + x14 = fluid.layers.data( + name='x14', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + input4 = fluid.layers.data( + name='input4', + shape=[3, 1], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input4, x13, x14) @@ -166,8 +197,10 @@ class TestAddMMOp2(TestAddMMOp): 'Alpha': 0.1, 'Beta': 1.0, } - self.outputs = {'Out': self.attrs['Beta'] * self.inputs['Input'] + \ - self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y'])} + self.outputs = { + 'Out': self.attrs['Beta'] * self.inputs['Input'] + + self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y']) + } class TestAddMMOp3(OpTest): @@ -185,8 +218,10 @@ class TestAddMMOp3(OpTest): 'Alpha': 0.5, 'Beta': 2.0, } - self.outputs = {'Out': self.attrs['Beta'] * self.inputs['Input'] + \ - self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y'])} + self.outputs = { + 'Out': self.attrs['Beta'] * self.inputs['Input'] + + self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y']) + } def init_dtype_type(self): pass @@ -222,8 +257,10 @@ class TestAddMMOp4(OpTest): 'Alpha': 0.5, 'Beta': 2.0, } - self.outputs = {'Out': self.attrs['Beta'] * self.inputs['Input'] + \ - self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y'])} + self.outputs = { + 'Out': self.attrs['Beta'] * self.inputs['Input'] + + self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y']) + } def init_dtype_type(self): pass @@ -245,7 +282,6 @@ class TestAddMMOp4(OpTest): class TestAddMMOp5(unittest.TestCase): - def test_api_with_dygraph(self): np_input = np.random.random((20, 30)).astype(np.float32) np_x = np.random.random((20, 6)).astype(np.float32) @@ -260,7 +296,6 @@ class TestAddMMOp5(unittest.TestCase): class TestAddMMAPI(unittest.TestCase): - def test_api_error(self): data_x = np.ones((2, 2)).astype(np.float32) data_y = np.ones((2, 2)).astype(np.float32) @@ -273,11 +308,9 @@ class TestAddMMAPI(unittest.TestCase): x = paddle.to_tensor(data_x_wrong) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - out = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=0.5, - alpha=5.0) + out = paddle.tensor.addmm( + input=input, x=x, y=y, beta=0.5, alpha=5.0 + ) self.assertRaises(ValueError, test_error1) @@ -286,11 +319,9 @@ class TestAddMMAPI(unittest.TestCase): x = paddle.to_tensor(data_x_wrong) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - out = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=0.5, - alpha=5.0) + out = paddle.tensor.addmm( + input=input, x=x, y=y, beta=0.5, alpha=5.0 + ) self.assertRaises(ValueError, test_error2) @@ -299,11 +330,9 @@ class TestAddMMAPI(unittest.TestCase): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input_wrong) - out = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=0.5, - alpha=5.0) + out = paddle.tensor.addmm( + input=input, x=x, y=y, beta=0.5, alpha=5.0 + ) self.assertRaises(ValueError, test_error3) @@ -312,11 +341,9 @@ class TestAddMMAPI(unittest.TestCase): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input_wrong) - out = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=0.5, - alpha=5.0) + out = paddle.tensor.addmm( + input=input, x=x, y=y, beta=0.5, alpha=5.0 + ) self.assertRaises(ValueError, test_error4) @@ -334,17 +361,16 @@ class TestAddMMAPI(unittest.TestCase): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - paddle_output = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=data_beta, - alpha=data_alpha) + paddle_output = paddle.tensor.addmm( + input=input, x=x, y=y, beta=data_beta, alpha=data_alpha + ) numpy_output = data_beta * data_input + data_alpha * np.dot( - data_x, data_y) + data_x, data_y + ) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() @@ -360,17 +386,16 @@ class TestAddMMAPI(unittest.TestCase): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - paddle_output = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=data_beta, - alpha=data_alpha) + paddle_output = paddle.tensor.addmm( + input=input, x=x, y=y, beta=data_beta, alpha=data_alpha + ) numpy_output = data_beta * data_input + data_alpha * np.dot( - data_x, data_y) + data_x, data_y + ) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() @@ -386,17 +411,16 @@ class TestAddMMAPI(unittest.TestCase): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - paddle_output = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=data_beta, - alpha=data_alpha) + paddle_output = paddle.tensor.addmm( + input=input, x=x, y=y, beta=data_beta, alpha=data_alpha + ) numpy_output = data_beta * data_input + data_alpha * np.dot( - data_x, data_y) + data_x, data_y + ) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_affine_channel_op.py b/python/paddle/fluid/tests/unittests/test_affine_channel_op.py index 4bd91002256..f09e5be5775 100644 --- a/python/paddle/fluid/tests/unittests/test_affine_channel_op.py +++ b/python/paddle/fluid/tests/unittests/test_affine_channel_op.py @@ -33,7 +33,6 @@ def affine_channel(x, scale, bias, layout): class TestAffineChannelOp(OpTest): - def setUp(self): self.op_type = "affine_channel" self.init_test_case() @@ -67,7 +66,6 @@ class TestAffineChannelOp(OpTest): class TestAffineChannelOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): @@ -78,32 +76,31 @@ class TestAffineChannelOpError(unittest.TestCase): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[None, 1, 2, 2], - dtype='int32') + x2 = fluid.layers.data( + name='x2', shape=[None, 1, 2, 2], dtype='int32' + ) fluid.layers.affine_channel(x2) self.assertRaises(TypeError, test_x_dtype) def test_scale_type(): - x3 = fluid.layers.data(name='x3', - shape=[None, 1, 2, 2], - dtype='float32') + x3 = fluid.layers.data( + name='x3', shape=[None, 1, 2, 2], dtype='float32' + ) fluid.layers.affine_channel(x3, scale=1) self.assertRaises(TypeError, test_scale_type) def test_bias_type(): - x4 = fluid.layers.data(name='x4', - shape=[None, 1, 2, 2], - dtype='float32') + x4 = fluid.layers.data( + name='x4', shape=[None, 1, 2, 2], dtype='float32' + ) fluid.layers.affine_channel(x4, bias=1) self.assertRaises(TypeError, test_bias_type) class TestAffineChannelNHWC(TestAffineChannelOp): - def init_test_case(self): self.shape = [2, 3, 3, 100] self.C = 100 @@ -117,7 +114,6 @@ class TestAffineChannelNHWC(TestAffineChannelOp): class TestAffineChannel2D(TestAffineChannelOp): - def init_test_case(self): self.shape = [2, 100] self.C = 100 @@ -131,7 +127,7 @@ class TestAffineChannel2D(TestAffineChannelOp): # TODO(qingqing): disable unit testing for large shape -#class TestAffineChannelNCHWLargeShape(TestAffineChannelOp): +# class TestAffineChannelNCHWLargeShape(TestAffineChannelOp): # def init_test_case(self): # self.shape = [4, 128, 112, 112] # self.C = 128 @@ -147,7 +143,7 @@ class TestAffineChannel2D(TestAffineChannelOp): # def test_check_grad_stopgrad_dscale_dbias(self): # pass -#class TestAffineChannelNHWCLargeShape(TestAffineChannelNCHWLargeShape): +# class TestAffineChannelNHWCLargeShape(TestAffineChannelNCHWLargeShape): # def init_test_case(self): # self.shape = [64, 32, 32, 128] # self.C = 128 diff --git a/python/paddle/fluid/tests/unittests/test_affine_grid_function.py b/python/paddle/fluid/tests/unittests/test_affine_grid_function.py index 9091a1c9b6d..cd1cecfb618 100644 --- a/python/paddle/fluid/tests/unittests/test_affine_grid_function.py +++ b/python/paddle/fluid/tests/unittests/test_affine_grid_function.py @@ -21,15 +21,16 @@ import unittest class AffineGridTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - theta_shape=(20, 2, 3), - output_shape=[20, 2, 5, 7], - align_corners=True, - dtype="float32", - invalid_theta=False, - variable_output_shape=False): + def __init__( + self, + methodName='runTest', + theta_shape=(20, 2, 3), + output_shape=[20, 2, 5, 7], + align_corners=True, + dtype="float32", + invalid_theta=False, + variable_output_shape=False, + ): super(AffineGridTestCase, self).__init__(methodName) self.theta_shape = theta_shape @@ -48,14 +49,14 @@ class AffineGridTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - theta_var = fluid.data("input", - self.theta_shape, - dtype=self.dtype) + theta_var = fluid.data( + "input", self.theta_shape, dtype=self.dtype + ) y_var = fluid.layers.affine_grid(theta_var, self.output_shape) feed_dict = {"input": self.theta} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -64,28 +65,33 @@ class AffineGridTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - theta_var = fluid.data("input", - self.theta_shape, - dtype=self.dtype) - y_var = F.affine_grid(theta_var, - self.output_shape, - align_corners=self.align_corners) + theta_var = fluid.data( + "input", self.theta_shape, dtype=self.dtype + ) + y_var = F.affine_grid( + theta_var, + self.output_shape, + align_corners=self.align_corners, + ) feed_dict = {"input": self.theta} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_dygraph_layer(self): paddle.disable_static() - theta_var = dg.to_variable( - self.theta) if not self.invalid_theta else "invalid" - output_shape = dg.to_variable( - self.output_shape - ) if self.variable_output_shape else self.output_shape - y_var = F.affine_grid(theta_var, - output_shape, - align_corners=self.align_corners) + theta_var = ( + dg.to_variable(self.theta) if not self.invalid_theta else "invalid" + ) + output_shape = ( + dg.to_variable(self.output_shape) + if self.variable_output_shape + else self.output_shape + ) + y_var = F.affine_grid( + theta_var, output_shape, align_corners=self.align_corners + ) y_np = y_var.numpy() return y_np @@ -108,7 +114,6 @@ class AffineGridTestCase(unittest.TestCase): class AffineGridErrorTestCase(AffineGridTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -122,22 +127,26 @@ def add_cases(suite): suite.addTest(AffineGridTestCase(methodName='runTest', align_corners=False)) suite.addTest( - AffineGridTestCase(methodName='runTest', variable_output_shape=True)) + AffineGridTestCase(methodName='runTest', variable_output_shape=True) + ) suite.addTest( - AffineGridTestCase(methodName='runTest', - theta_shape=(20, 2, 3), - output_shape=[20, 1, 7, 7], - align_corners=True)) + AffineGridTestCase( + methodName='runTest', + theta_shape=(20, 2, 3), + output_shape=[20, 1, 7, 7], + align_corners=True, + ) + ) def add_error_cases(suite): suite.addTest( - AffineGridErrorTestCase(methodName='runTest', output_shape="not_valid")) + AffineGridErrorTestCase(methodName='runTest', output_shape="not_valid") + ) suite.addTest( - AffineGridErrorTestCase( - methodName='runTest', - invalid_theta=True)) # to test theta not variable error checking + AffineGridErrorTestCase(methodName='runTest', invalid_theta=True) + ) # to test theta not variable error checking def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_affine_grid_op.py b/python/paddle/fluid/tests/unittests/test_affine_grid_op.py index f8c0dedc184..096b4fe1277 100644 --- a/python/paddle/fluid/tests/unittests/test_affine_grid_op.py +++ b/python/paddle/fluid/tests/unittests/test_affine_grid_op.py @@ -26,12 +26,21 @@ def AffineGrid4D(theta, size, align_corners): if not align_corners: h_factor = (h - 1) / float(h) w_factor = (w - 1) / float(w) - h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, - axis=0).T[:, :, np.newaxis] * h_factor - w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, - axis=0)[:, :, np.newaxis] * w_factor - grid = np.concatenate([w_idx, h_idx, np.ones([h, w, 1])], - axis=2) # h * w * 3 + h_idx = ( + np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, axis=0).T[ + :, :, np.newaxis + ] + * h_factor + ) + w_idx = ( + np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, axis=0)[ + :, :, np.newaxis + ] + * w_factor + ) + grid = np.concatenate( + [w_idx, h_idx, np.ones([h, w, 1])], axis=2 + ) # h * w * 3 grid = np.repeat(grid[np.newaxis, :], size[0], axis=0) # n * h * w *3 ret = np.zeros([n, h * w, 2]) @@ -51,20 +60,39 @@ def AffineGrid5D(theta, size, align_corners): d_factor = (d - 1) / float(d) h_factor = (h - 1) / float(h) w_factor = (w - 1) / float(w) - d_idx = np.repeat(np.repeat( - np.linspace(-1, 1, d)[:, np.newaxis, np.newaxis], h, axis=1), - w, - axis=2)[:, :, :, np.newaxis] * d_factor - h_idx = np.repeat(np.repeat( - np.linspace(-1, 1, h)[np.newaxis, :, np.newaxis], w, axis=2), - d, - axis=0)[:, :, :, np.newaxis] * h_factor - w_idx = np.repeat(np.repeat( - np.linspace(-1, 1, w)[np.newaxis, np.newaxis, :], h, axis=1), - d, - axis=0)[:, :, :, np.newaxis] * w_factor + d_idx = ( + np.repeat( + np.repeat( + np.linspace(-1, 1, d)[:, np.newaxis, np.newaxis], h, axis=1 + ), + w, + axis=2, + )[:, :, :, np.newaxis] + * d_factor + ) + h_idx = ( + np.repeat( + np.repeat( + np.linspace(-1, 1, h)[np.newaxis, :, np.newaxis], w, axis=2 + ), + d, + axis=0, + )[:, :, :, np.newaxis] + * h_factor + ) + w_idx = ( + np.repeat( + np.repeat( + np.linspace(-1, 1, w)[np.newaxis, np.newaxis, :], h, axis=1 + ), + d, + axis=0, + )[:, :, :, np.newaxis] + * w_factor + ) grid = np.concatenate( - [w_idx, h_idx, d_idx, np.ones([d, h, w, 1])], axis=3) # d * h * w * 4 + [w_idx, h_idx, d_idx, np.ones([d, h, w, 1])], axis=3 + ) # d * h * w * 4 grid = np.repeat(grid[np.newaxis, :], size[0], axis=0) # n * d * h * w * 4 ret = np.zeros([n, d * h * w, 3]) @@ -75,7 +103,6 @@ def AffineGrid5D(theta, size, align_corners): class TestAffineGridOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = "affine_grid" @@ -84,31 +111,32 @@ class TestAffineGridOp(OpTest): self.inputs = {'Theta': theta} self.attrs = { "use_cudnn": self.use_cudnn, - "align_corners": self.align_corners + "align_corners": self.align_corners, } if self.dynamic_shape: self.inputs['OutputShape'] = self.output_shape else: self.attrs['output_shape'] = self.output_shape - if (self.theta_shape[1] == 2 and self.theta_shape[2] == 3): + if self.theta_shape[1] == 2 and self.theta_shape[2] == 3: self.outputs = { - 'Output': AffineGrid4D(theta, self.output_shape, - self.align_corners) + 'Output': AffineGrid4D( + theta, self.output_shape, self.align_corners + ) } else: self.outputs = { - 'Output': AffineGrid5D(theta, self.output_shape, - self.align_corners) + 'Output': AffineGrid5D( + theta, self.output_shape, self.align_corners + ) } def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['Theta'], - 'Output', - no_grad_set=['OutputShape'], - check_eager=True) + self.check_grad( + ['Theta'], 'Output', no_grad_set=['OutputShape'], check_eager=True + ) def initTestCase(self): self.theta_shape = (17, 2, 3) @@ -119,19 +147,19 @@ class TestAffineGridOp(OpTest): class TestAffineGridOpCase1(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 2, 3) self.output_shape = np.array([20, 2, 5, 7]).astype("int32") self.dynamic_shape = True self.use_cudnn = True if paddle.fluid.core.is_compiled_with_rocm(): - self.use_cudnn = False # ROCM platform do not have MIOPEN kernel for affine_grid + self.use_cudnn = ( + False # ROCM platform do not have MIOPEN kernel for affine_grid + ) self.align_corners = True class TestAffineGridOpCase2(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 2, 3) self.output_shape = np.array([20, 2, 5, 7]).astype("int32") @@ -141,7 +169,6 @@ class TestAffineGridOpCase2(TestAffineGridOp): class TestAffineGridOpCase3(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 2, 3) self.output_shape = np.array([20, 2, 5, 7]).astype("int32") @@ -151,7 +178,6 @@ class TestAffineGridOpCase3(TestAffineGridOp): class TestAffineGridOpCase4(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (25, 2, 3) self.output_shape = np.array([25, 2, 5, 6]).astype("int32") @@ -161,7 +187,6 @@ class TestAffineGridOpCase4(TestAffineGridOp): class TestAffineGridOp5DCase1(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 3, 4) self.output_shape = np.array([20, 1, 2, 5, 7]).astype("int32") @@ -171,7 +196,6 @@ class TestAffineGridOp5DCase1(TestAffineGridOp): class TestAffineGridOp5DCase2(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 3, 4) self.output_shape = np.array([20, 1, 2, 5, 7]).astype("int32") @@ -181,7 +205,6 @@ class TestAffineGridOp5DCase2(TestAffineGridOp): class TestAffineGridOp5DCase3(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 3, 4) self.output_shape = np.array([20, 1, 2, 5, 7]).astype("int32") @@ -191,7 +214,6 @@ class TestAffineGridOp5DCase3(TestAffineGridOp): class TestAffineGridOp5DCase4(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (25, 3, 4) self.output_shape = np.array([25, 1, 2, 5, 6]).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/test_allclose_layer.py b/python/paddle/fluid/tests/unittests/test_allclose_layer.py index 66afbcfe209..c406241c65b 100644 --- a/python/paddle/fluid/tests/unittests/test_allclose_layer.py +++ b/python/paddle/fluid/tests/unittests/test_allclose_layer.py @@ -20,68 +20,53 @@ from paddle.fluid.framework import _test_eager_guard class TestAllcloseLayer(unittest.TestCase): - def allclose_check(self, use_cuda, dtype='float32'): a = fluid.data(name="a", shape=[2], dtype=dtype) b = fluid.data(name="b", shape=[2], dtype=dtype) - result = paddle.allclose(a, - b, - rtol=1e-05, - atol=1e-08, - equal_nan=False, - name="ignore_nan") - result_nan = paddle.allclose(a, - b, - rtol=1e-05, - atol=1e-08, - equal_nan=True, - name="equal_nan") - result_corner = paddle.allclose(a, - b, - rtol=0.01, - atol=0.0, - name="corner_case") + result = paddle.allclose( + a, b, rtol=1e-05, atol=1e-08, equal_nan=False, name="ignore_nan" + ) + result_nan = paddle.allclose( + a, b, rtol=1e-05, atol=1e-08, equal_nan=True, name="equal_nan" + ) + result_corner = paddle.allclose( + a, b, rtol=0.01, atol=0.0, name="corner_case" + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - x = np.array([10000., 1e-07]).astype(dtype) + x = np.array([10000.0, 1e-07]).astype(dtype) y = np.array([10000.1, 1e-08]).astype(dtype) - result_v, result_nan_v = exe.run(feed={ - 'a': x, - 'b': y - }, - fetch_list=[result, result_nan]) + result_v, result_nan_v = exe.run( + feed={'a': x, 'b': y}, fetch_list=[result, result_nan] + ) self.assertEqual(result_v[0], False) self.assertEqual(result_nan_v[0], False) - x = np.array([10000., 1e-08]).astype(dtype) + x = np.array([10000.0, 1e-08]).astype(dtype) y = np.array([10000.1, 1e-09]).astype(dtype) - result_v, result_nan_v = exe.run(feed={ - 'a': x, - 'b': y - }, - fetch_list=[result, result_nan]) + result_v, result_nan_v = exe.run( + feed={'a': x, 'b': y}, fetch_list=[result, result_nan] + ) self.assertEqual(result_v[0], True) self.assertEqual(result_nan_v[0], True) x = np.array([1.0, float('nan')]).astype(dtype) y = np.array([1.0, float('nan')]).astype(dtype) - result_v, result_nan_v = exe.run(feed={ - 'a': x, - 'b': y - }, - fetch_list=[result, result_nan]) + result_v, result_nan_v = exe.run( + feed={'a': x, 'b': y}, fetch_list=[result, result_nan] + ) self.assertEqual(result_v[0], False) self.assertEqual(result_nan_v[0], True) # for corner case x = np.array([10.1, 10.1]).astype(dtype) y = np.array([10, 10]).astype(dtype) - result_c, = exe.run(feed={'a': x, 'b': y}, fetch_list=[result_corner]) - corner_res = (dtype == 'float64') + (result_c,) = exe.run(feed={'a': x, 'b': y}, fetch_list=[result_corner]) + corner_res = dtype == 'float64' self.assertEqual(result_c[0], corner_res) def test_allclose_cpu_fp32(self): @@ -115,9 +100,9 @@ class TestAllcloseLayer(unittest.TestCase): self.allclose_check(use_cuda=True, dtype='float64') def func_dygraph_mode(self): - x_1 = np.array([10000., 1e-07]).astype("float32") + x_1 = np.array([10000.0, 1e-07]).astype("float32") y_1 = np.array([10000.1, 1e-08]).astype("float32") - x_2 = np.array([10000., 1e-08]).astype("float32") + x_2 = np.array([10000.0, 1e-08]).astype("float32") y_2 = np.array([10000.1, 1e-09]).astype("float32") x_3 = np.array([1.0, float('nan')]).astype("float32") y_3 = np.array([1.0, float('nan')]).astype("float32") @@ -129,68 +114,76 @@ class TestAllcloseLayer(unittest.TestCase): with fluid.dygraph.guard(): x_v_1 = paddle.to_tensor(x_1) y_v_1 = paddle.to_tensor(y_1) - ret_1 = paddle.allclose(x_v_1, - y_v_1, - rtol=1e-05, - atol=1e-08, - equal_nan=False, - name='test_1') + ret_1 = paddle.allclose( + x_v_1, + y_v_1, + rtol=1e-05, + atol=1e-08, + equal_nan=False, + name='test_1', + ) self.assertEqual(ret_1.numpy()[0], False) - ret_1 = paddle.allclose(x_v_1, - y_v_1, - rtol=1e-05, - atol=1e-08, - equal_nan=True, - name='test_2') + ret_1 = paddle.allclose( + x_v_1, + y_v_1, + rtol=1e-05, + atol=1e-08, + equal_nan=True, + name='test_2', + ) self.assertEqual(ret_1.numpy()[0], False) x_v_2 = paddle.to_tensor(x_2) y_v_2 = paddle.to_tensor(y_2) - ret_2 = paddle.allclose(x_v_2, - y_v_2, - rtol=1e-05, - atol=1e-08, - equal_nan=False, - name='test_3') + ret_2 = paddle.allclose( + x_v_2, + y_v_2, + rtol=1e-05, + atol=1e-08, + equal_nan=False, + name='test_3', + ) self.assertEqual(ret_2.numpy()[0], True) - ret_2 = paddle.allclose(x_v_2, - y_v_2, - rtol=1e-05, - atol=1e-08, - equal_nan=True, - name='test_4') + ret_2 = paddle.allclose( + x_v_2, + y_v_2, + rtol=1e-05, + atol=1e-08, + equal_nan=True, + name='test_4', + ) self.assertEqual(ret_2.numpy()[0], True) x_v_3 = paddle.to_tensor(x_3) y_v_3 = paddle.to_tensor(y_3) - ret_3 = paddle.allclose(x_v_3, - y_v_3, - rtol=1e-05, - atol=1e-08, - equal_nan=False, - name='test_5') + ret_3 = paddle.allclose( + x_v_3, + y_v_3, + rtol=1e-05, + atol=1e-08, + equal_nan=False, + name='test_5', + ) self.assertEqual(ret_3.numpy()[0], False) - ret_3 = paddle.allclose(x_v_3, - y_v_3, - rtol=1e-05, - atol=1e-08, - equal_nan=True, - name='test_6') + ret_3 = paddle.allclose( + x_v_3, + y_v_3, + rtol=1e-05, + atol=1e-08, + equal_nan=True, + name='test_6', + ) self.assertEqual(ret_3.numpy()[0], True) # for corner case x_v_4 = paddle.to_tensor(x_4) y_v_4 = paddle.to_tensor(y_4) - ret_4 = paddle.allclose(x_v_4, - y_v_4, - rtol=0.01, - atol=0.0, - name='test_7') + ret_4 = paddle.allclose( + x_v_4, y_v_4, rtol=0.01, atol=0.0, name='test_7' + ) self.assertEqual(ret_4.numpy()[0], False) x_v_5 = paddle.to_tensor(x_5) y_v_5 = paddle.to_tensor(y_5) - ret_5 = paddle.allclose(x_v_5, - y_v_5, - rtol=0.015, - atol=0.0, - name='test_8') + ret_5 = paddle.allclose( + x_v_5, y_v_5, rtol=0.015, atol=0.0, name='test_8' + ) self.assertEqual(ret_5.numpy()[0], True) def test_dygraph_mode(self): diff --git a/python/paddle/fluid/tests/unittests/test_allclose_op.py b/python/paddle/fluid/tests/unittests/test_allclose_op.py index 26351abe802..0eb47aa4441 100644 --- a/python/paddle/fluid/tests/unittests/test_allclose_op.py +++ b/python/paddle/fluid/tests/unittests/test_allclose_op.py @@ -19,9 +19,8 @@ import paddle class TestAllcloseOp(OpTest): - def set_args(self): - self.input = np.array([10000., 1e-07]).astype("float32") + self.input = np.array([10000.0, 1e-07]).astype("float32") self.other = np.array([10000.1, 1e-08]).astype("float32") self.rtol = np.array([1e-05]).astype("float64") self.atol = np.array([1e-08]).astype("float64") @@ -35,18 +34,21 @@ class TestAllcloseOp(OpTest): 'Input': self.input, 'Other': self.other, "Rtol": self.rtol, - "Atol": self.atol + "Atol": self.atol, } self.attrs = {'equal_nan': self.equal_nan} self.outputs = { - 'Out': - np.array([ - np.allclose(self.inputs['Input'], - self.inputs['Other'], - rtol=self.rtol, - atol=self.atol, - equal_nan=self.equal_nan) - ]) + 'Out': np.array( + [ + np.allclose( + self.inputs['Input'], + self.inputs['Other'], + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, + ) + ] + ) } def test_check_output(self): @@ -54,9 +56,7 @@ class TestAllcloseOp(OpTest): class TestAllcloseOpException(TestAllcloseOp): - def test_check_output(self): - def test_rtol_num(): self.inputs['Rtol'] = np.array([1e-05, 1e-05]).astype("float64") self.inputs['Atol'] = np.array([1e-08]).astype("float64") @@ -87,9 +87,8 @@ class TestAllcloseOpException(TestAllcloseOp): class TestAllcloseOpSmallNum(TestAllcloseOp): - def set_args(self): - self.input = np.array([10000., 1e-08]).astype("float32") + self.input = np.array([10000.0, 1e-08]).astype("float32") self.other = np.array([10000.1, 1e-09]).astype("float32") self.rtol = np.array([1e-05]).astype("float64") self.atol = np.array([1e-08]).astype("float64") @@ -97,7 +96,6 @@ class TestAllcloseOpSmallNum(TestAllcloseOp): class TestAllcloseOpNanFalse(TestAllcloseOp): - def set_args(self): self.input = np.array([1.0, float('nan')]).astype("float32") self.other = np.array([1.0, float('nan')]).astype("float32") @@ -107,7 +105,6 @@ class TestAllcloseOpNanFalse(TestAllcloseOp): class TestAllcloseOpNanTrue(TestAllcloseOp): - def set_args(self): self.input = np.array([1.0, float('nan')]).astype("float32") self.other = np.array([1.0, float('nan')]).astype("float32") @@ -117,7 +114,6 @@ class TestAllcloseOpNanTrue(TestAllcloseOp): class TestAllcloseDygraph(unittest.TestCase): - def test_api_case(self): paddle.disable_static() x_data = np.random.rand(10, 10) @@ -131,12 +127,11 @@ class TestAllcloseDygraph(unittest.TestCase): class TestAllcloseError(unittest.TestCase): - def test_input_dtype(self): - def test_x_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float16') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='float64') result = paddle.allclose(x, y) @@ -144,8 +139,9 @@ class TestAllcloseError(unittest.TestCase): self.assertRaises(TypeError, test_x_dtype) def test_y_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='int32') result = paddle.allclose(x, y) @@ -173,7 +169,6 @@ class TestAllcloseError(unittest.TestCase): class TestAllcloseOpFloat32(TestAllcloseOp): - def set_args(self): self.input = np.array([10.1]).astype("float32") self.other = np.array([10]).astype("float32") @@ -183,7 +178,6 @@ class TestAllcloseOpFloat32(TestAllcloseOp): class TestAllcloseOpFloat64(TestAllcloseOp): - def set_args(self): self.input = np.array([10.1]).astype("float64") self.other = np.array([10]).astype("float64") @@ -193,7 +187,6 @@ class TestAllcloseOpFloat64(TestAllcloseOp): class TestAllcloseOpLargeDimInput(TestAllcloseOp): - def set_args(self): self.input = np.array(np.zeros([2048, 1024])).astype("float64") self.other = np.array(np.zeros([2048, 1024])).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/test_allgather.py b/python/paddle/fluid/tests/unittests/test_allgather.py index 90da7128dcf..89441f394cf 100644 --- a/python/paddle/fluid/tests/unittests/test_allgather.py +++ b/python/paddle/fluid/tests/unittests/test_allgather.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestAllGatherOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py b/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py index b44c294f3ac..7101d3439f6 100644 --- a/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py @@ -24,7 +24,6 @@ def check_finite_and_unscale_wrapper(x, scale): class TestCheckFiniteAndUnscaleOp(OpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.python_api = check_finite_and_unscale_wrapper @@ -47,7 +46,6 @@ class TestCheckFiniteAndUnscaleOp(OpTest): class TestCheckFiniteAndUnscaleOpWithNan(OpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() @@ -73,7 +71,6 @@ class TestCheckFiniteAndUnscaleOpWithNan(OpTest): class TestCheckFiniteAndUnscaleOpWithInf(OpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() diff --git a/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py b/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py index b5ce42ee758..3766dc95ab9 100644 --- a/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py +++ b/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py @@ -17,8 +17,9 @@ import numpy as np from op_test import OpTest -def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, - variances, stride, offset): +def anchor_generator_in_python( + input_feat, anchor_sizes, aspect_ratios, variances, stride, offset +): num_anchors = len(aspect_ratios) * len(anchor_sizes) layer_h = input_feat.shape[2] layer_w = input_feat.shape[3] @@ -42,11 +43,12 @@ def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, scale_h = anchor_size / stride[1] w = scale_w * base_w h = scale_h * base_h - out_anchors[h_idx, w_idx, - idx, :] = [(x_ctr - 0.5 * (w - 1)), - (y_ctr - 0.5 * (h - 1)), - (x_ctr + 0.5 * (w - 1)), - (y_ctr + 0.5 * (h - 1))] + out_anchors[h_idx, w_idx, idx, :] = [ + (x_ctr - 0.5 * (w - 1)), + (y_ctr - 0.5 * (h - 1)), + (x_ctr + 0.5 * (w - 1)), + (y_ctr + 0.5 * (h - 1)), + ] idx += 1 # set the variance. @@ -57,7 +59,6 @@ def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, class TestAnchorGeneratorOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -87,9 +88,9 @@ class TestAnchorGeneratorOp(OpTest): self.layer_h = 2 self.layer_w = 2 - self.anchor_sizes = [64., 128., 256., 512.] - self.aspect_ratios = [0.5, 1., 2.] - self.stride = [16., 16.] + self.anchor_sizes = [64.0, 128.0, 256.0, 512.0] + self.aspect_ratios = [0.5, 1.0, 2.0] + self.stride = [16.0, 16.0] self.offset = 0.5 @@ -97,13 +98,18 @@ class TestAnchorGeneratorOp(OpTest): def init_test_input(self): self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_h, - self.layer_w)).astype('float32') + (self.batch_size, self.input_channels, self.layer_h, self.layer_w) + ).astype('float32') def init_test_output(self): self.out_anchors, self.out_var = anchor_generator_in_python( - self.input, self.anchor_sizes, self.aspect_ratios, self.variances, - self.stride, self.offset) + self.input, + self.anchor_sizes, + self.aspect_ratios, + self.variances, + self.stride, + self.offset, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_angle_op.py b/python/paddle/fluid/tests/unittests/test_angle_op.py index 9d428215010..767a274ad32 100644 --- a/python/paddle/fluid/tests/unittests/test_angle_op.py +++ b/python/paddle/fluid/tests/unittests/test_angle_op.py @@ -29,7 +29,7 @@ def angle_grad(x, dout): def angle_grad_element(xi, douti): if xi == 0: return 0 - rsquare = np.abs(xi)**2 + rsquare = np.abs(xi) ** 2 return -douti * xi.imag / rsquare + 1j * douti * xi.real / rsquare return np.vectorize(angle_grad_element)(x, dout) @@ -38,7 +38,6 @@ def angle_grad(x, dout): class TestAngleOpFloat(OpTest): - def setUp(self): self.op_type = "angle" self.python_api = paddle.angle @@ -52,17 +51,17 @@ class TestAngleOpFloat(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ - angle_grad(self.x, - np.ones_like(self.x) / self.x.size) - ], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ + angle_grad(self.x, np.ones_like(self.x) / self.x.size) + ], + check_eager=True, + ) class TestAngleOpComplex(OpTest): - def setUp(self): self.op_type = "angle" self.python_api = paddle.angle @@ -78,17 +77,17 @@ class TestAngleOpComplex(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ - angle_grad(self.x, - np.ones_like(self.x) / self.x.size) - ], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ + angle_grad(self.x, np.ones_like(self.x) / self.x.size) + ], + check_eager=True, + ) class TestAngleAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randn(2, 3) + 1j * np.random.randn(2, 3) self.out = np.angle(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_apply_pass_to_program.py b/python/paddle/fluid/tests/unittests/test_apply_pass_to_program.py index 54a83d2a5ec..fff492e9176 100644 --- a/python/paddle/fluid/tests/unittests/test_apply_pass_to_program.py +++ b/python/paddle/fluid/tests/unittests/test_apply_pass_to_program.py @@ -26,9 +26,9 @@ def get_resnet50_model(): main = paddle.static.Program() startup = paddle.static.Program() with paddle.static.program_guard(main, startup): - image = paddle.static.data(name="image", - shape=[None, 3, 224, 224], - dtype="float32") + image = paddle.static.data( + name="image", shape=[None, 3, 224, 224], dtype="float32" + ) label = paddle.static.data(name="label", shape=[None, 1], dtype="int64") model = resnet50() loss_fn = CrossEntropyLoss() @@ -48,7 +48,6 @@ def global_block_contains_op(program, op_type): class TestApplyPassToProgram(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -67,21 +66,23 @@ class TestApplyPassToProgram(unittest.TestCase): "size_t_attr": "size_t", "float32_attr": "float32", } - ret_attrs = _apply_pass(main, startup, "fuse_elewise_add_act_pass", - attrs, attr_types) + ret_attrs = _apply_pass( + main, startup, "fuse_elewise_add_act_pass", attrs, attr_types + ) self.assertEqual(attrs, ret_attrs) self.assertTrue(global_block_contains_op(main, fused_op)) class TestIRPassBase(unittest.TestCase): - def setUp(self): paddle.enable_static() if paddle.is_compiled_with_cuda(): - fluid.set_flags({ - 'FLAGS_cudnn_deterministic': 1, - 'FLAGS_max_inplace_grad_add': 6, - }) + fluid.set_flags( + { + 'FLAGS_cudnn_deterministic': 1, + 'FLAGS_max_inplace_grad_add': 6, + } + ) self.place = paddle.CUDAPlace(0) else: self.place = paddle.CPUPlace() @@ -104,7 +105,8 @@ class TestIRPassBase(unittest.TestCase): self.assertFalse(global_block_contains_op(main, "share_buffer")) self.assertFalse(global_block_contains_op(main, "coalesce_tensor")) self.assertFalse( - global_block_contains_op(main, "fused_elemwise_add_activation")) + global_block_contains_op(main, "fused_elemwise_add_activation") + ) adam_cnt = 0 for op in main.global_block().ops: @@ -119,7 +121,8 @@ class TestIRPassBase(unittest.TestCase): self.assertTrue(global_block_contains_op(main, "coalesce_tensor")) self.assertTrue(global_block_contains_op(main, "depend")) self.assertTrue( - global_block_contains_op(main, "fused_elemwise_add_activation")) + global_block_contains_op(main, "fused_elemwise_add_activation") + ) share_dims_cnt = 0 non_share_dims_cnt = 0 @@ -167,8 +170,9 @@ class TestIRPassBase(unittest.TestCase): setattr(build_strategy, k, v) self.check_before_applied(main2, startup2) - apply_build_strategy(main2, startup2, build_strategy, - {"use_cuda": self.use_cuda}) + apply_build_strategy( + main2, startup2, build_strategy, {"use_cuda": self.use_cuda} + ) self.check_after_applied(main2, startup2) image_shape = [batch_size] + list(image.shape)[1:] @@ -186,22 +190,22 @@ class TestIRPassBase(unittest.TestCase): for idx in range(batch_num): feed = { - image.name: - np.random.rand(*image_shape).astype('float32'), - label.name: - np.random.randint(low=0, - high=self.num_classes, - size=label_shape, - dtype='int64'), + image.name: np.random.rand(*image_shape).astype('float32'), + label.name: np.random.randint( + low=0, + high=self.num_classes, + size=label_shape, + dtype='int64', + ), } with paddle.static.scope_guard(scope1): - loss_value1 = self.executor.run(main1, - feed=feed, - fetch_list=[loss1])[0] + loss_value1 = self.executor.run( + main1, feed=feed, fetch_list=[loss1] + )[0] with paddle.static.scope_guard(scope2): - loss_value2 = self.executor.run(main2, - feed=feed, - fetch_list=[loss2])[0] + loss_value2 = self.executor.run( + main2, feed=feed, fetch_list=[loss2] + )[0] self.assertEqual(loss_value1, loss_value2, "batch {}".format(idx)) diff --git a/python/paddle/fluid/tests/unittests/test_arange.py b/python/paddle/fluid/tests/unittests/test_arange.py index 277b6d67d94..a7c5c4231d4 100644 --- a/python/paddle/fluid/tests/unittests/test_arange.py +++ b/python/paddle/fluid/tests/unittests/test_arange.py @@ -21,20 +21,19 @@ from op_test import OpTest class TestArangeOp(OpTest): - def setUp(self): self.op_type = "range" self.init_config() self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange(self.case[0], self.case[1], self.case[2]).astype( + self.dtype + ) } def init_config(self): @@ -46,48 +45,45 @@ class TestArangeOp(OpTest): class TestFloatArangeOp(TestArangeOp): - def init_config(self): self.dtype = np.float32 self.case = (0, 5, 1) class TestInt32ArangeOp(TestArangeOp): - def init_config(self): self.dtype = np.int32 self.case = (0, 5, 2) class TestFloat64ArangeOp(TestArangeOp): - def init_config(self): self.dtype = np.float64 self.case = (10, 1, -2) class TestInt64ArangeOp(TestArangeOp): - def init_config(self): self.dtype = np.int64 self.case = (-1, -10, -2) class TestArangeOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(TypeError, paddle.arange, 10, dtype='int8') class TestArangeAPI(unittest.TestCase): - def test_out(self): with program_guard(Program(), Program()): x1 = paddle.arange(0, 5, 1, 'float32') - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) out = exe.run(fetch_list=[x1]) @@ -96,10 +92,12 @@ class TestArangeAPI(unittest.TestCase): class TestArangeImperative(unittest.TestCase): - def test_out(self): - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) paddle.disable_static(place) x1 = paddle.arange(0, 5, 1) x2 = paddle.tensor.arange(5) diff --git a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py index 05e38f4706e..6c9d09bbab9 100644 --- a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py @@ -23,7 +23,6 @@ from test_attribute_var import UnittestBase class BaseTestCase(OpTest): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -45,7 +44,6 @@ class BaseTestCase(OpTest): class TestCase0(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -54,7 +52,6 @@ class TestCase0(BaseTestCase): class TestCase1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -63,7 +60,6 @@ class TestCase1(BaseTestCase): class TestCase2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -71,10 +67,10 @@ class TestCase2(BaseTestCase): self.axis = 0 -@unittest.skipIf(not paddle.is_compiled_with_cuda(), - "FP16 test runs only on GPU") +@unittest.skipIf( + not paddle.is_compiled_with_cuda(), "FP16 test runs only on GPU" +) class TestCase0FP16(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -82,10 +78,10 @@ class TestCase0FP16(BaseTestCase): self.axis = 0 -@unittest.skipIf(not paddle.is_compiled_with_cuda(), - "FP16 test runs only on GPU") +@unittest.skipIf( + not paddle.is_compiled_with_cuda(), "FP16 test runs only on GPU" +) class TestCase1FP16(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -94,7 +90,6 @@ class TestCase1FP16(BaseTestCase): class TestCase2_1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -103,33 +98,29 @@ class TestCase2_1(BaseTestCase): class TestCase3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'int64' self.axis = 0 class TestCase4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'int32' self.axis = 0 class TestCase3_(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.axis = 0 class BaseTestComplex1_1(OpTest): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (4, 5, 6) @@ -153,7 +144,6 @@ class BaseTestComplex1_1(OpTest): class BaseTestComplex1_2(OpTest): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (4, 5, 6) @@ -177,7 +167,6 @@ class BaseTestComplex1_2(OpTest): class BaseTestComplex2_1(OpTest): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (4, 5, 6) @@ -193,20 +182,19 @@ class BaseTestComplex2_1(OpTest): self.attrs = {'keep_dims': True} if self.op_type == "arg_min": self.outputs = { - 'Out': - np.argmin(self.x, - axis=self.axis).asdtype("int32").reshape(4, 5, 1) + 'Out': np.argmin(self.x, axis=self.axis) + .asdtype("int32") + .reshape(4, 5, 1) } else: self.outputs = { - 'Out': - np.argmax(self.x, - axis=self.axis).asdtype("int32").reshape(4, 5, 1) + 'Out': np.argmax(self.x, axis=self.axis) + .asdtype("int32") + .reshape(4, 5, 1) } class BaseTestComplex2_2(OpTest): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (4, 5, 6) @@ -222,20 +210,19 @@ class BaseTestComplex2_2(OpTest): self.attrs = {'keep_dims': True} if self.op_type == "arg_min": self.outputs = { - 'Out': - np.argmin(self.x, - axis=self.axis).asdtype("int32").reshape(4, 5, 1) + 'Out': np.argmin(self.x, axis=self.axis) + .asdtype("int32") + .reshape(4, 5, 1) } else: self.outputs = { - 'Out': - np.argmax(self.x, - axis=self.axis).asdtype("int32").reshape(4, 5, 1) + 'Out': np.argmax(self.x, axis=self.axis) + .asdtype("int32") + .reshape(4, 5, 1) } class TestArgMaxTensorAxis(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.x = [np.random.randn(*shape) for shape in self.shapes] @@ -259,8 +246,9 @@ class TestArgMaxTensorAxis(UnittestBase): exe = paddle.static.Executor() exe.run(starup_prog) res = exe.run(fetch_list=[feat, out]) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) gt = np.argmax(res[0], 0) np.testing.assert_allclose(res[1], gt) @@ -282,7 +270,6 @@ class TestArgMaxTensorAxis(UnittestBase): class TestArgMinTensorAxis(TestArgMaxTensorAxis): - def test_static(self): main_prog = Program() starup_prog = Program() @@ -301,8 +288,9 @@ class TestArgMinTensorAxis(TestArgMaxTensorAxis): exe = paddle.static.Executor() exe.run(starup_prog) res = exe.run(fetch_list=[feat, out]) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) gt = np.argmin(res[0], 1) np.testing.assert_allclose(np.squeeze(res[1]), gt) diff --git a/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py b/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py index b59a4d16998..a6f3fee2109 100644 --- a/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py @@ -22,9 +22,7 @@ from paddle.fluid import Program, program_guard def create_kernel_case(op_type, numpy_op_type): - class ArgMinMaxKernelBaseCase(OpTest): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type @@ -35,7 +33,7 @@ def create_kernel_case(op_type, numpy_op_type): self.initTestCase() self.dims = (4, 5, 6) self.dtype = "float64" - self.x = (1000 * np.random.random(self.dims).astype(self.dtype)) + self.x = 1000 * np.random.random(self.dims).astype(self.dtype) self.inputs = {'X': self.x} self.attrs = {"axis": self.axis} self.numpy_op = eval("np.%s" % (numpy_op_type)) @@ -46,40 +44,35 @@ def create_kernel_case(op_type, numpy_op_type): self.check_output() class ArgMinMaxKernelCase0(ArgMinMaxKernelBaseCase): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type self.axis = 1 class ArgMinMaxKernelCase1(ArgMinMaxKernelBaseCase): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type self.axis = 2 class ArgMinMaxKernelCase2(ArgMinMaxKernelBaseCase): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type self.axis = -1 class ArgMinMaxKernelCase3(ArgMinMaxKernelBaseCase): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type self.axis = -2 class ArgMinMaxKernelCase4(ArgMinMaxKernelBaseCase): - def setUp(self): self.initTestCase() self.dims = (4, 5, 6) self.dtype = "float64" - self.x = (1000 * np.random.random(self.dims).astype(self.dtype)) + self.x = 1000 * np.random.random(self.dims).astype(self.dtype) self.inputs = {'X': self.x} self.attrs = {"axis": self.axis, "keepdims": True} self.numpy_op = eval("np.%s" % (numpy_op_type)) @@ -88,12 +81,11 @@ def create_kernel_case(op_type, numpy_op_type): } class ArgMinMaxKernelCase5(ArgMinMaxKernelBaseCase): - def setUp(self): self.initTestCase() - self.dims = (4) + self.dims = 4 self.dtype = "float64" - self.x = (1000 * np.random.random(self.dims).astype(self.dtype)) + self.x = 1000 * np.random.random(self.dims).astype(self.dtype) self.inputs = {'X': self.x} self.attrs = {"axis": self.axis, "flatten": True} self.numpy_op = eval("np.%s" % (numpy_op_type)) @@ -102,12 +94,11 @@ def create_kernel_case(op_type, numpy_op_type): } class ArgMinMaxKernelCase6(ArgMinMaxKernelBaseCase): - def setUp(self): self.initTestCase() - self.dims = (4) + self.dims = 4 self.dtype = "float64" - self.x = (1000 * np.random.random(self.dims).astype(self.dtype)) + self.x = 1000 * np.random.random(self.dims).astype(self.dtype) self.inputs = {'X': self.x} self.attrs = {"axis": self.axis, "flatten": True, "keepdims": True} self.numpy_op = eval("np.%s" % (numpy_op_type)) @@ -153,9 +144,7 @@ for op_type, numpy_op_type in zip(['arg_max', 'arg_min'], ['argmax', 'argmin']): def create_test_case(op_type): - class ArgMaxMinTestCase(unittest.TestCase): - def setUp(self): np.random.seed(123) self.input_data = np.random.rand(10, 10).astype("float32") @@ -169,61 +158,67 @@ def create_test_case(op_type): def run_static(self, place): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) op = eval("paddle.%s" % (op_type)) result = op(data_var) exe = paddle.static.Executor(place) - result_data = exe.run(feed={"data": self.input_data}, - fetch_list=[result]) + result_data = exe.run( + feed={"data": self.input_data}, fetch_list=[result] + ) expected_data = self.numpy_op(self.input_data) - self.assertTrue((result_data == np.array(expected_data)).all(), - True) + self.assertTrue( + (result_data == np.array(expected_data)).all(), True + ) with paddle.static.program_guard(paddle.static.Program()): - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) op = eval("paddle.%s" % (op_type)) result = op(data_var, axis=1) exe = paddle.static.Executor(place) - result_data = exe.run(feed={"data": self.input_data}, - fetch_list=[result]) + result_data = exe.run( + feed={"data": self.input_data}, fetch_list=[result] + ) expected_data = self.numpy_op(self.input_data, axis=1) self.assertTrue((result_data == expected_data).all(), True) with paddle.static.program_guard(paddle.static.Program()): - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) op = eval("paddle.%s" % (op_type)) result = op(data_var, axis=-1) exe = paddle.static.Executor(place) - result_data = exe.run(feed={"data": self.input_data}, - fetch_list=[result]) + result_data = exe.run( + feed={"data": self.input_data}, fetch_list=[result] + ) expected_data = self.numpy_op(self.input_data, axis=-1) self.assertTrue((result_data == expected_data).all(), True) with paddle.static.program_guard(paddle.static.Program()): - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) op = eval("paddle.%s" % (op_type)) result = op(data_var, axis=-1, keepdim=True) exe = paddle.static.Executor(place) - result_data = exe.run(feed={"data": self.input_data}, - fetch_list=[result]) + result_data = exe.run( + feed={"data": self.input_data}, fetch_list=[result] + ) expected_data = self.numpy_op(self.input_data, axis=-1).reshape( - (10, 1)) + (10, 1) + ) self.assertTrue((result_data == expected_data).all(), True) with paddle.static.program_guard(paddle.static.Program()): op = eval("paddle.%s" % (op_type)) - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) result = op(data_var, axis=-1, name="test_arg_api") self.assertTrue("test_arg_api" in result.name) @@ -232,28 +227,28 @@ def create_test_case(op_type): op = eval("paddle.%s" % (op_type)) data_tensor = paddle.to_tensor(self.input_data) - #case 1 + # case 1 result_data = op(data_tensor) excepted_data = self.numpy_op(self.input_data) self.assertTrue((result_data.numpy() == excepted_data).all(), True) - #case 2 + # case 2 result_data = op(data_tensor, axis=1) excepted_data = self.numpy_op(self.input_data, axis=1) self.assertTrue((result_data.numpy() == excepted_data).all(), True) - #case 3 + # case 3 result_data = op(data_tensor, axis=-1) excepted_data = self.numpy_op(self.input_data, axis=-1) self.assertTrue((result_data.numpy() == excepted_data).all(), True) - #case 4 + # case 4 result_data = op(data_tensor, axis=-1, keepdim=True) excepted_data = self.numpy_op(self.input_data, axis=-1) excepted_data = excepted_data.reshape((10, 1)) self.assertTrue((result_data.numpy() == excepted_data).all(), True) - #case 5 + # case 5 result_data = op(data_tensor, axis=-1, keepdim=True, dtype="int32") self.assertTrue(result_data.numpy().dtype == np.int32) @@ -288,7 +283,6 @@ for op_type in ['argmin', 'argmax']: class TestArgMinMaxOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -306,49 +300,49 @@ class TestArgMinMaxOpError(unittest.TestCase): self.assertRaises(TypeError, test_argmin_x_type) def test_argmax_attr_type(): - data = paddle.static.data(name="test_argmax", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmax", shape=[10], dtype="float32" + ) output = paddle.argmax(x=data, dtype="float32") self.assertRaises(TypeError, test_argmax_attr_type) def test_argmin_attr_type(): - data = paddle.static.data(name="test_argmax", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmax", shape=[10], dtype="float32" + ) output = paddle.argmin(x=data, dtype="float32") self.assertRaises(TypeError, test_argmin_attr_type) def test_argmax_axis_type(): - data = paddle.static.data(name="test_argmax", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmax", shape=[10], dtype="float32" + ) output = paddle.argmax(x=data, axis=1.2) self.assertRaises(TypeError, test_argmax_axis_type) def test_argmin_axis_type(): - data = paddle.static.data(name="test_argmin", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmin", shape=[10], dtype="float32" + ) output = paddle.argmin(x=data, axis=1.2) self.assertRaises(TypeError, test_argmin_axis_type) def test_argmax_dtype_type(): - data = paddle.static.data(name="test_argmax", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmax", shape=[10], dtype="float32" + ) output = paddle.argmax(x=data, dtype=None) self.assertRaises(ValueError, test_argmax_dtype_type) def test_argmin_dtype_type(): - data = paddle.static.data(name="test_argmin", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmin", shape=[10], dtype="float32" + ) output = paddle.argmin(x=data, dtype=None) self.assertRaises(ValueError, test_argmin_dtype_type) diff --git a/python/paddle/fluid/tests/unittests/test_argsort_op.py b/python/paddle/fluid/tests/unittests/test_argsort_op.py index d9106e26627..d8df9fefbbd 100644 --- a/python/paddle/fluid/tests/unittests/test_argsort_op.py +++ b/python/paddle/fluid/tests/unittests/test_argsort_op.py @@ -26,7 +26,6 @@ np.random.seed(123) class PyArgsort(object): - def __init__(self, input_shape, axis, descending, dtype): self.x = np.random.random(input_shape).astype(dtype) self.label = np.random.random(input_shape).astype(dtype) @@ -39,17 +38,21 @@ class PyArgsort(object): def forward(self): if self.descending: self.indices = np.flip( - np.argsort(self.x, kind='quicksort', axis=self.axis), self.axis) + np.argsort(self.x, kind='quicksort', axis=self.axis), self.axis + ) self.sorted_x = np.flip( - np.sort(self.x, kind='quicksort', axis=self.axis), self.axis) + np.sort(self.x, kind='quicksort', axis=self.axis), self.axis + ) else: self.indices = np.argsort(self.x, kind='quicksort', axis=self.axis) self.sorted_x = np.sort(self.x, kind='quicksort', axis=self.axis) self.loss = self.sorted_x * self.label self.loss = np.sum(self.loss) - out = (np.array(self.indices, dtype=self.indices.dtype), - np.array(self.sorted_x, dtype=self.sorted_x.dtype), - np.array([self.loss], dtype=self.loss.dtype)) + out = ( + np.array(self.indices, dtype=self.indices.dtype), + np.array(self.sorted_x, dtype=self.sorted_x.dtype), + np.array([self.loss], dtype=self.loss.dtype), + ) return out @@ -60,7 +63,6 @@ def create_tensor(np_data, place): class TestArgsortOpCPU(unittest.TestCase): - def setup_program(self): self.main_program = Program() self.startup_program = Program() @@ -76,19 +78,21 @@ class TestArgsortOpCPU(unittest.TestCase): self.feed_data_field = {"x", "label"} self.grad_data_field = {"x"} - self.py_argsort = PyArgsort(self.input_shape, self.axis, - self.descending, self.dtype) + self.py_argsort = PyArgsort( + self.input_shape, self.axis, self.descending, self.dtype + ) with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.layers.data(name="x", - shape=self.input_shape, - dtype=self.dtype) + x = fluid.layers.data( + name="x", shape=self.input_shape, dtype=self.dtype + ) x.stop_gradient = False - label = fluid.layers.data(name="label", - shape=self.input_shape, - dtype=self.dtype) + label = fluid.layers.data( + name="label", shape=self.input_shape, dtype=self.dtype + ) self.sorted_x, self.index = fluid.layers.argsort( - input=x, axis=self.axis, descending=self.descending) + input=x, axis=self.axis, descending=self.descending + ) self.sorted_x.stop_gradient = False loss = fluid.layers.elementwise_mul(self.sorted_x, label) self.loss = fluid.layers.reduce_sum(loss) @@ -99,9 +103,11 @@ class TestArgsortOpCPU(unittest.TestCase): for x in self.feed_data_field } exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=[self.index, self.sorted_x, self.loss]) + out = exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=[self.index, self.sorted_x, self.loss], + ) return out def backward(self): @@ -114,10 +120,12 @@ class TestArgsortOpCPU(unittest.TestCase): for x in self.grad_data_field ] exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=fetch_list, - return_numpy=False) + out = exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=fetch_list, + return_numpy=False, + ) return out def test_backward(self, numeric_grad_delta=1e-5, max_relative_error=1e-7): @@ -129,23 +137,22 @@ class TestArgsortOpCPU(unittest.TestCase): ana_grad = [np.array(x) for x in self.backward()] num_grad = self.get_numerical_gradient(delta=numeric_grad_delta) - self.assert_is_close(num_grad, - ana_grad, - 'x', - max_relative_error=max_relative_error, - msg_prefix="Gradient Check On %s" % - str(self.place)) + self.assert_is_close( + num_grad, + ana_grad, + 'x', + max_relative_error=max_relative_error, + msg_prefix="Gradient Check On %s" % str(self.place), + ) def check_forward(self): pd_outputs = self.forward() py_outputs = self.py_argsort.forward() for pd_output, py_output in zip(pd_outputs, py_outputs): self.assertEqual(pd_output.shape, py_output.shape) - np.testing.assert_allclose(pd_output, - py_output, - rtol=1e-05, - atol=0, - equal_nan=False) + np.testing.assert_allclose( + pd_output, py_output, rtol=1e-05, atol=0, equal_nan=False + ) def get_numerical_gradient(self, delta=1e-7): if self.dtype == 'float16': @@ -167,8 +174,14 @@ class TestArgsortOpCPU(unittest.TestCase): return grad_list - def assert_is_close(self, numeric_grads, analytic_grads, names, - max_relative_error, msg_prefix): + def assert_is_close( + self, + numeric_grads, + analytic_grads, + names, + max_relative_error, + msg_prefix, + ): for a, b, name in zip(numeric_grads, analytic_grads, names): abs_a = np.abs(a) abs_a[abs_a < 1e-3] = 1 @@ -178,10 +191,19 @@ class TestArgsortOpCPU(unittest.TestCase): def err_msg(): offset = np.argmax(diff_mat > max_relative_error) - return ("%s error, %s variable %s max gradient diff %f over limit %f, " - "the first error element is %d, expected %f, but got %f.") \ - % ('argsort', msg_prefix, name, max_diff, max_relative_error, - offset, a.flatten()[offset], b.flatten()[offset]) + return ( + "%s error, %s variable %s max gradient diff %f over limit %f, " + "the first error element is %d, expected %f, but got %f." + ) % ( + 'argsort', + msg_prefix, + name, + max_diff, + max_relative_error, + offset, + a.flatten()[offset], + b.flatten()[offset], + ) self.assertLessEqual(max_diff, max_relative_error, err_msg()) @@ -202,7 +224,6 @@ class TestArgsortOpCPU(unittest.TestCase): class TestArgsortOpGPU(TestArgsortOpCPU): - def init_place(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -211,144 +232,120 @@ class TestArgsortOpGPU(TestArgsortOpCPU): class TestArgsortOpAxis0CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = 0 class TestArgsortOpAxis0GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = 0 class TestArgsortOpAxis1CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis1GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = 2 class TestArgsortOpAxis2GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg1GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = -2 class TestArgsortOpAxisNeg2GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisCPU(TestArgsortOpCPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisGPU(TestArgsortOpGPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0CPU(TestArgsortOpAxis0CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0GPU(TestArgsortOpAxis0GPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1CPU(TestArgsortOpAxis1CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1GPU(TestArgsortOpAxis1GPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2CPU(TestArgsortOpAxis2CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2GPU(TestArgsortOpAxis2GPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1CPU(TestArgsortOpAxisNeg1CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1GPU(TestArgsortOpAxisNeg1GPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2CPU(TestArgsortOpAxisNeg2CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2GPU(TestArgsortOpAxisNeg2GPU): - def init_direction(self): self.descending = True class TestArgsortErrorOnCPU(unittest.TestCase): - def setUp(self): self.place = core.CPUPlace() def test_error(self): - def test_fluid_var_type(): with fluid.program_guard(fluid.Program()): x = [1] @@ -363,7 +360,6 @@ class TestArgsortErrorOnCPU(unittest.TestCase): class TestArgsortErrorOnGPU(TestArgsortErrorOnCPU): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -372,7 +368,6 @@ class TestArgsortErrorOnGPU(TestArgsortErrorOnCPU): class TestArgsort(unittest.TestCase): - def init(self): self.input_shape = [ 10000, @@ -389,16 +384,17 @@ class TestArgsort(unittest.TestCase): def test_api(self): with fluid.program_guard(fluid.Program()): - input = fluid.data(name="input", - shape=self.input_shape, - dtype="float64") + input = fluid.data( + name="input", shape=self.input_shape, dtype="float64" + ) output = paddle.argsort(input, axis=self.axis) output2 = paddle.argsort(input, axis=self.axis, descending=True) exe = fluid.Executor(self.place) - result, result2 = exe.run(feed={'input': self.data}, - fetch_list=[output, output2]) + result, result2 = exe.run( + feed={'input': self.data}, fetch_list=[output, output2] + ) np_result = np.argsort(self.data, axis=self.axis) self.assertEqual((result == np_result).all(), True) @@ -408,28 +404,24 @@ class TestArgsort(unittest.TestCase): class TestArgsort2(TestArgsort): - def init(self): self.input_shape = [10000, 1] self.axis = 0 class TestArgsort3(TestArgsort): - def init(self): self.input_shape = [1, 10000] self.axis = 1 class TestArgsort4(TestArgsort): - def init(self): self.input_shape = [2, 3, 4] self.axis = 1 class TestArgsortImperative(unittest.TestCase): - def init(self): self.input_shape = [ 10000, @@ -459,28 +451,24 @@ class TestArgsortImperative(unittest.TestCase): class TestArgsortImperative2(TestArgsortImperative): - def init(self): self.input_shape = [10000, 1] self.axis = 0 class TestArgsortImperative3(TestArgsortImperative): - def init(self): self.input_shape = [1, 10000] self.axis = 1 class TestArgsortImperative4(TestArgsortImperative): - def init(self): self.input_shape = [2, 3, 4] self.axis = 1 class TestArgsortWithInputNaN(unittest.TestCase): - def init(self): self.axis = 0 diff --git a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py index 4c1c7563d8b..f5f54017d99 100644 --- a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py +++ b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py @@ -58,12 +58,11 @@ def _test_read_write(x): class TestArrayReadWrite(unittest.TestCase): - def test_read_write(self): x = [ layers.data(name='x0', shape=[100]), layers.data(name='x1', shape=[100]), - layers.data(name='x2', shape=[100]) + layers.data(name='x2', shape=[100]), ] for each_x in x: each_x.stop_gradient = False @@ -73,13 +72,11 @@ class TestArrayReadWrite(unittest.TestCase): place = core.CPUPlace() exe = Executor(place) - outs = exe.run(feed={ - 'x0': tensor, - 'x1': tensor, - 'x2': tensor - }, - fetch_list=[a_sum, x_sum], - scope=core.Scope()) + outs = exe.run( + feed={'x0': tensor, 'x1': tensor, 'x2': tensor}, + fetch_list=[a_sum, x_sum], + scope=core.Scope(), + ) self.assertEqual(outs[0], outs[1]) total_sum = layers.sums(input=[a_sum, x_sum]) @@ -88,15 +85,17 @@ class TestArrayReadWrite(unittest.TestCase): append_backward(total_sum_scaled) g_vars = list( - map(default_main_program().global_block().var, - [each_x.name + "@GRAD" for each_x in x])) + map( + default_main_program().global_block().var, + [each_x.name + "@GRAD" for each_x in x], + ) + ) g_out = [ - item.sum() for item in exe.run(feed={ - 'x0': tensor, - 'x1': tensor, - 'x2': tensor - }, - fetch_list=g_vars) + item.sum() + for item in exe.run( + feed={'x0': tensor, 'x1': tensor, 'x2': tensor}, + fetch_list=g_vars, + ) ] g_out_sum = np.array(g_out).sum() @@ -116,9 +115,11 @@ class TestArrayReadWrite(unittest.TestCase): self.assertEqual(a_sum_dygraph, x_sum_dygraph) total_sum_dygraph = layers.sums( - input=[a_sum_dygraph, x_sum_dygraph]) - total_sum_scaled_dygraph = layers.scale(x=total_sum_dygraph, - scale=1 / 6.0) + input=[a_sum_dygraph, x_sum_dygraph] + ) + total_sum_scaled_dygraph = layers.scale( + x=total_sum_dygraph, scale=1 / 6.0 + ) total_sum_scaled_dygraph.backward() g_out_dygraph = [ item._grad_ivar().numpy().sum() for item in x_dygraph @@ -129,40 +130,33 @@ class TestArrayReadWrite(unittest.TestCase): class TestArrayReadWriteOpError(unittest.TestCase): - def _test_errors(self, use_fluid_api=True): if use_fluid_api: with program_guard(Program(), Program()): x1 = np.random.randn(2, 4).astype('int32') - x2 = fluid.layers.fill_constant(shape=[1], - dtype='int32', - value=1) + x2 = fluid.layers.fill_constant( + shape=[1], dtype='int32', value=1 + ) x3 = np.random.randn(2, 4).astype('int32') - self.assertRaises(TypeError, - fluid.layers.array_read, - array=x1, - i=x2) - self.assertRaises(TypeError, - fluid.layers.array_write, - array=x1, - i=x2, - out=x3) + self.assertRaises( + TypeError, fluid.layers.array_read, array=x1, i=x2 + ) + self.assertRaises( + TypeError, fluid.layers.array_write, array=x1, i=x2, out=x3 + ) else: with program_guard(Program(), Program()): x1 = np.random.randn(2, 4).astype('int32') x2 = paddle.ones(shape=[1], dtype='int32') x3 = np.random.randn(2, 4).astype('int32') - self.assertRaises(TypeError, - paddle.tensor.array_read, - array=x1, - i=x2) - self.assertRaises(TypeError, - paddle.tensor.array_write, - array=x1, - i=x2, - out=x3) + self.assertRaises( + TypeError, paddle.tensor.array_read, array=x1, i=x2 + ) + self.assertRaises( + TypeError, paddle.tensor.array_write, array=x1, i=x2, out=x3 + ) def test_fluid_api(self): self._test_errors(use_fluid_api=True) @@ -172,7 +166,6 @@ class TestArrayReadWriteOpError(unittest.TestCase): class TestArrayReadWriteApi(unittest.TestCase): - def test_api(self): paddle.disable_static() arr = paddle.tensor.create_array(dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_ascend_trigger.py b/python/paddle/fluid/tests/unittests/test_ascend_trigger.py index 8d8a896a424..16b38865593 100644 --- a/python/paddle/fluid/tests/unittests/test_ascend_trigger.py +++ b/python/paddle/fluid/tests/unittests/test_ascend_trigger.py @@ -18,7 +18,7 @@ import unittest class TestAscendTriggerOP(unittest.TestCase): - """ TestCases for ascend_trigger op""" + """TestCases for ascend_trigger op""" def test_ascend_trigger_op(self): paddle.enable_static() @@ -27,10 +27,12 @@ class TestAscendTriggerOP(unittest.TestCase): with fluid.program_guard(program): x = fluid.data(name='x', shape=[1], dtype='int64', lod_level=0) y = fluid.data(name='y', shape=[1], dtype='int64', lod_level=0) - block.append_op(type="ascend_trigger", - inputs={"FeedList": [x]}, - outputs={"FetchList": [y]}, - attrs={'graph_idx': 0}) + block.append_op( + type="ascend_trigger", + inputs={"FeedList": [x]}, + outputs={"FetchList": [y]}, + attrs={'graph_idx': 0}, + ) exe = paddle.static.Executor(paddle.CPUPlace()) try: diff --git a/python/paddle/fluid/tests/unittests/test_assert_op.py b/python/paddle/fluid/tests/unittests/test_assert_op.py index 481a58b441e..9a91ebca5d8 100644 --- a/python/paddle/fluid/tests/unittests/test_assert_op.py +++ b/python/paddle/fluid/tests/unittests/test_assert_op.py @@ -19,7 +19,6 @@ import unittest class TestAssertOp(unittest.TestCase): - def run_network(self, net_func): main_program = fluid.Program() startup_program = fluid.Program() @@ -29,39 +28,35 @@ class TestAssertOp(unittest.TestCase): exe.run(main_program) def test_assert_true(self): - def net_func(): - condition = layers.fill_constant(shape=[1], - dtype='bool', - value=True) + condition = layers.fill_constant( + shape=[1], dtype='bool', value=True + ) layers.Assert(condition, []) self.run_network(net_func) def test_assert_false(self): - def net_func(): - condition = layers.fill_constant(shape=[1], - dtype='bool', - value=False) + condition = layers.fill_constant( + shape=[1], dtype='bool', value=False + ) layers.Assert(condition) with self.assertRaises(ValueError): self.run_network(net_func) def test_assert_cond_numel_error(self): - def net_func(): - condition = layers.fill_constant(shape=[1, 2], - dtype='bool', - value=True) + condition = layers.fill_constant( + shape=[1, 2], dtype='bool', value=True + ) layers.Assert(condition, []) with self.assertRaises(ValueError): self.run_network(net_func) def test_assert_print_data(self): - def net_func(): zero = layers.fill_constant(shape=[1], dtype='int64', value=0) one = layers.fill_constant(shape=[1], dtype='int64', value=1) @@ -73,18 +68,16 @@ class TestAssertOp(unittest.TestCase): self.run_network(net_func) def test_assert_summary(self): - def net_func(): x = layers.fill_constant(shape=[10], dtype='float32', value=2.0) condition = layers.reduce_max(x) < 1.0 - layers.Assert(condition, (x, ), 5) + layers.Assert(condition, (x,), 5) print("test_assert_summary") with self.assertRaises(ValueError): self.run_network(net_func) def test_assert_summary_greater_than_size(self): - def net_func(): x = layers.fill_constant(shape=[2, 3], dtype='float32', value=2.0) condition = layers.reduce_max(x) < 1.0 diff --git a/python/paddle/fluid/tests/unittests/test_assign_op.py b/python/paddle/fluid/tests/unittests/test_assign_op.py index f1ae5362e08..a59d9fb6688 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_op.py @@ -27,7 +27,6 @@ import paddle.fluid.layers as layers class TestAssignOp(op_test.OpTest): - def setUp(self): self.python_api = paddle.assign self.op_type = "assign" @@ -53,7 +52,6 @@ class TestAssignOp(op_test.OpTest): class TestAssignFP16Op(op_test.OpTest): - def setUp(self): self.python_api = paddle.assign self.op_type = "assign" @@ -79,7 +77,6 @@ class TestAssignFP16Op(op_test.OpTest): class TestAssignOpWithLoDTensorArray(unittest.TestCase): - def test_assign_LoDTensorArray(self): paddle.enable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -88,9 +85,9 @@ class TestAssignOpWithLoDTensorArray(unittest.TestCase): with program_guard(main_program): x = fluid.data(name='x', shape=[100, 10], dtype='float32') x.stop_gradient = False - y = fluid.layers.fill_constant(shape=[100, 10], - dtype='float32', - value=1) + y = fluid.layers.fill_constant( + shape=[100, 10], dtype='float32', value=1 + ) z = fluid.layers.elementwise_add(x=x, y=y) i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0) init_array = fluid.layers.array_write(x=z, i=i) @@ -100,28 +97,33 @@ class TestAssignOpWithLoDTensorArray(unittest.TestCase): append_backward(mean) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feed_x = np.random.random(size=(100, 10)).astype('float32') ones = np.ones((100, 10)).astype('float32') feed_add = feed_x + ones - res = exe.run(main_program, - feed={'x': feed_x}, - fetch_list=[sums.name, x.grad_name]) + res = exe.run( + main_program, + feed={'x': feed_x}, + fetch_list=[sums.name, x.grad_name], + ) np.testing.assert_allclose(res[0], feed_add, rtol=1e-05) np.testing.assert_allclose(res[1], ones / 1000.0, rtol=1e-05) paddle.disable_static() class TestAssignOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The type of input must be Variable or numpy.ndarray. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.assign, x1) # When the type of input is numpy.ndarray, the dtype of input must be float32, int32. x2 = np.array([[2.5, 2.5]], dtype='uint8') @@ -130,7 +132,6 @@ class TestAssignOpError(unittest.TestCase): class TestAssignOApi(unittest.TestCase): - def test_assign_LoDTensorArray(self): paddle.enable_static() main_program = Program() @@ -138,9 +139,9 @@ class TestAssignOApi(unittest.TestCase): with program_guard(main_program): x = fluid.data(name='x', shape=[100, 10], dtype='float32') x.stop_gradient = False - y = fluid.layers.fill_constant(shape=[100, 10], - dtype='float32', - value=1) + y = fluid.layers.fill_constant( + shape=[100, 10], dtype='float32', value=1 + ) z = fluid.layers.elementwise_add(x=x, y=y) i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0) init_array = fluid.layers.array_write(x=z, i=i) @@ -149,15 +150,20 @@ class TestAssignOApi(unittest.TestCase): mean = paddle.mean(sums) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feed_x = np.random.random(size=(100, 10)).astype('float32') ones = np.ones((100, 10)).astype('float32') feed_add = feed_x + ones - res = exe.run(main_program, - feed={'x': feed_x}, - fetch_list=[sums.name, x.grad_name]) + res = exe.run( + main_program, + feed={'x': feed_x}, + fetch_list=[sums.name, x.grad_name], + ) np.testing.assert_allclose(res[0], feed_add, rtol=1e-05) np.testing.assert_allclose(res[1], ones / 1000.0, rtol=1e-05) paddle.disable_static() @@ -225,23 +231,25 @@ class TestAssignOApi(unittest.TestCase): x = paddle.static.data("X", shape=[2, 3]) clone_x = paddle.clone(x) exe = paddle.static.Executor() - y_np = exe.run(paddle.static.default_main_program(), - feed={'X': x_np}, - fetch_list=[clone_x])[0] + y_np = exe.run( + paddle.static.default_main_program(), + feed={'X': x_np}, + fetch_list=[clone_x], + )[0] np.testing.assert_array_equal(y_np, x_np) paddle.disable_static() class TestAssignOpErrorApi(unittest.TestCase): - def test_errors(self): paddle.enable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) with program_guard(Program(), Program()): # The type of input must be Variable or numpy.ndarray. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.assign, x1) # When the type of input is numpy.ndarray, the dtype of input must be float32, int32. x2 = np.array([[2.5, 2.5]], dtype='uint8') @@ -259,7 +267,6 @@ class TestAssignOpErrorApi(unittest.TestCase): class TestAssignDoubleGradCheck(unittest.TestCase): - def assign_wrapper(self, x): return paddle.fluid.layers.assign(x[0]) @@ -274,17 +281,13 @@ class TestAssignDoubleGradCheck(unittest.TestCase): out = paddle.fluid.layers.assign(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.assign_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.assign_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -296,7 +299,6 @@ class TestAssignDoubleGradCheck(unittest.TestCase): class TestAssignTripleGradCheck(unittest.TestCase): - def assign_wrapper(self, x): return paddle.fluid.layers.assign(x[0]) @@ -311,17 +313,13 @@ class TestAssignTripleGradCheck(unittest.TestCase): out = paddle.fluid.layers.assign(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.assign_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.assign_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_assign_pos_op.py b/python/paddle/fluid/tests/unittests/test_assign_pos_op.py index 2440d349e0a..d04fecd7b05 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_pos_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_pos_op.py @@ -24,7 +24,7 @@ from paddle.fluid.framework import _test_eager_guard def assign_pos(x, _cum_count): cum_count = np.copy(_cum_count) x = x.reshape(-1) - res = np.zeros((cum_count[-1], ), dtype=np.int64) + res = np.zeros((cum_count[-1],), dtype=np.int64) for i, idx in enumerate(x): p = cum_count[idx] cum_count[idx] -= 1 @@ -34,7 +34,7 @@ def assign_pos(x, _cum_count): def count(x, upper_num): - res = np.zeros((upper_num, )).astype(int) + res = np.zeros((upper_num,)).astype(int) for i in x.reshape(-1): if i >= 0 and i < len(res): res[i] += 1 @@ -62,17 +62,16 @@ def assert_allclose(res, out, cum_count): def get_redefined_allclose(cum_count): - def redefined_allclose(x, y, *args, **kwargs): return assert_allclose(x, y, cum_count) return redefined_allclose -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAssignPosOpInt64(op_test.OpTest): - def setUp(self): x = np.random.randint(0, 16, size=(100, 2)).astype("int64") y = count(x, 16) @@ -81,7 +80,7 @@ class TestAssignPosOpInt64(op_test.OpTest): self.inputs = { 'X': x, "cum_count": cum_count, - "eff_num_len": np.array([cum_count[-1]]) + "eff_num_len": np.array([cum_count[-1]]), } self.outputs = {'Out': assign_pos(x, cum_count)} self.cum_count = cum_count @@ -91,10 +90,10 @@ class TestAssignPosOpInt64(op_test.OpTest): self.check_output_with_place(paddle.CUDAPlace(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAssignPosAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randint(0, 16, size=(100, 2)).astype("int64") y = count(self.x, 16) @@ -106,16 +105,15 @@ class TestAssignPosAPI(unittest.TestCase): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('x', self.x.shape, dtype="int64") - cum_count = paddle.fluid.data('cum_count', - self.cum_count.shape, - dtype="int64") + cum_count = paddle.fluid.data( + 'cum_count', self.cum_count.shape, dtype="int64" + ) out = utils._assign_pos(x, cum_count) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'x': self.x, - "cum_count": self.cum_count - }, - fetch_list=[out]) + res = exe.run( + feed={'x': self.x, "cum_count": self.cum_count}, + fetch_list=[out], + ) assert_allclose(res[0], self.out, self.cum_count) def func_api_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_assign_value_op.py b/python/paddle/fluid/tests/unittests/test_assign_value_op.py index df3becbbd9c..cac321c2a27 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_value_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_value_op.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestAssignValueOp(op_test.OpTest): - def setUp(self): self.op_type = "assign_value" self.inputs = {} @@ -33,7 +32,8 @@ class TestAssignValueOp(op_test.OpTest): self.init_data() self.attrs["shape"] = self.value.shape self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( - self.value.dtype) + self.value.dtype + ) self.outputs = {"Out": self.value} def init_data(self): @@ -45,35 +45,36 @@ class TestAssignValueOp(op_test.OpTest): class TestAssignValueOp2(TestAssignValueOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int32) self.attrs["int32_values"] = [int(v) for v in self.value.flat] class TestAssignValueOp3(TestAssignValueOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int64) self.attrs["int64_values"] = [int(v) for v in self.value.flat] class TestAssignValueOp4(TestAssignValueOp): - def init_data(self): - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) self.attrs["bool_values"] = [int(v) for v in self.value.flat] class TestAssignApi(unittest.TestCase): - def setUp(self): self.init_dtype() self.value = (-100 + 200 * np.random.random(size=(2, 5))).astype( - self.dtype) - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.dtype + ) + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def init_dtype(self): self.dtype = "float32" @@ -91,25 +92,26 @@ class TestAssignApi(unittest.TestCase): class TestAssignApi2(TestAssignApi): - def init_dtype(self): self.dtype = "int32" class TestAssignApi3(TestAssignApi): - def init_dtype(self): self.dtype = "int64" class TestAssignApi4(TestAssignApi): - def setUp(self): self.init_dtype() - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def init_dtype(self): self.dtype = "bool" diff --git a/python/paddle/fluid/tests/unittests/test_async_ssa_graph_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_async_ssa_graph_executor_mnist.py index f46d5303f52..c56235cd0d1 100644 --- a/python/paddle/fluid/tests/unittests/test_async_ssa_graph_executor_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_async_ssa_graph_executor_mnist.py @@ -34,21 +34,26 @@ def convolutional_neural_network(use_py_reader): capacity=64, feed_list=[img, label], iterable=False, - use_double_buffer=False) - - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + use_double_buffer=False, + ) + + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -65,20 +70,30 @@ def test(): place = fluid.CPUPlace() exe = fluid.Executor(place) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=BATCH_SIZE) - - array, img, label, prediction, avg_loss, acc, py_reader = convolutional_neural_network( - use_py_reader=False) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=BATCH_SIZE + ) + + ( + array, + img, + label, + prediction, + avg_loss, + acc, + py_reader, + ) = convolutional_neural_network(use_py_reader=False) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) def train_test(train_test_program, train_test_feed, train_test_reader): acc_set = [] avg_loss_set = [] for test_data in train_test_reader(): - acc_np, avg_loss_np = exe.run(program=train_test_program, - feed=train_test_feed.feed(test_data), - fetch_list=[acc, avg_loss]) + acc_np, avg_loss_np = exe.run( + program=train_test_program, + feed=train_test_feed.feed(test_data), + fetch_list=[acc, avg_loss], + ) acc_set.append(float(acc_np)) avg_loss_set.append(float(avg_loss_np)) # get test acc and loss @@ -90,7 +105,8 @@ def test(): avg_loss_val, acc_val = train_test( train_test_program=fluid.default_main_program(), train_test_reader=test_reader, - train_test_feed=feeder) + train_test_feed=feeder, + ) print("Test: avg_cost: %s, acc: %s" % (avg_loss_val, acc_val)) assert acc_val > 0.96 @@ -101,17 +117,25 @@ def train(use_cuda, thread_num, cpu_num): print("paddle is not compiled with cuda, exit!") return - array, img, label, prediction, avg_loss, acc, py_reader = convolutional_neural_network( - use_py_reader=True) + ( + array, + img, + label, + prediction, + avg_loss, + acc, + py_reader, + ) = convolutional_neural_network(use_py_reader=True) print("build convolutional neural network done.") optimizer = fluid.optimizer.Adam(learning_rate=0.001) optimizer.minimize(avg_loss) print("Adam optimizer minimize done.") - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=BATCH_SIZE, + ) print("declared train reader done.") place = fluid.CPUPlace() @@ -133,11 +157,13 @@ def train(use_cuda, thread_num, cpu_num): exec_strategy.num_iteration_per_run = 10 main_program = fluid.default_main_program() - pe = fluid.ParallelExecutor(use_cuda=False, - loss_name=avg_loss.name, - main_program=main_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) + pe = fluid.ParallelExecutor( + use_cuda=False, + loss_name=avg_loss.name, + main_program=main_program, + build_strategy=build_strategy, + exec_strategy=exec_strategy, + ) print("declare parallel executor done.") py_reader.set_sample_list_generator(train_reader) @@ -148,15 +174,18 @@ def train(use_cuda, thread_num, cpu_num): try: while True: array_v, acc_v, prediction_v, loss_val = pe.run( - fetch_list=[array, acc, prediction, avg_loss.name]) + fetch_list=[array, acc, prediction, avg_loss.name] + ) assert numpy.allclose(array_v[0], prediction_v) == True assert numpy.allclose(array_v[1], acc_v) == True loss_val = numpy.mean(loss_val) if step % 10 == 0: - print("Pass %d, Batch %d, Cost %f, queue size %d" % - (pass_id, step, loss_val, py_reader.queue.size())) + print( + "Pass %d, Batch %d, Cost %f, queue size %d" + % (pass_id, step, loss_val, py_reader.queue.size()) + ) step += 1 except fluid.core.EOFException: print("train end pass = " + str(pass_id)) @@ -166,24 +195,33 @@ def train(use_cuda, thread_num, cpu_num): class TestAsyncSSAGraphExecutor(unittest.TestCase): - def test_check_async_ssa_exe_train(self): step_list = [] for cpu_num in [1, 2, 4]: print("run cpu_num -> " + str(cpu_num)) with fluid.scope_guard(fluid.core.Scope()): - with fluid.program_guard(main_program=fluid.Program(), - startup_program=fluid.Program()): + with fluid.program_guard( + main_program=fluid.Program(), + startup_program=fluid.Program(), + ): start_time = time.time() - step = train(use_cuda=False, - thread_num=cpu_num, - cpu_num=cpu_num) + step = train( + use_cuda=False, thread_num=cpu_num, cpu_num=cpu_num + ) end_time = time.time() step_list.append(step) - print("cpu_num -> " + str(cpu_num) + " step -> " + str(step) + - " time -> " + str(end_time - start_time)) - with fluid.program_guard(main_program=fluid.Program(), - startup_program=fluid.Program()): + print( + "cpu_num -> " + + str(cpu_num) + + " step -> " + + str(step) + + " time -> " + + str(end_time - start_time) + ) + with fluid.program_guard( + main_program=fluid.Program(), + startup_program=fluid.Program(), + ): test() assert abs(int(step_list[0] / 2) - int(step_list[1])) < 5 assert abs(int(step_list[1] / 2) - int(step_list[2])) < 5 diff --git a/python/paddle/fluid/tests/unittests/test_atan2_op.py b/python/paddle/fluid/tests/unittests/test_atan2_op.py index c0200d3c4a1..b74aeb90570 100644 --- a/python/paddle/fluid/tests/unittests/test_atan2_op.py +++ b/python/paddle/fluid/tests/unittests/test_atan2_op.py @@ -30,7 +30,6 @@ def atan2_grad(x1, x2, dout): class TestAtan2(OpTest): - def setUp(self): self.op_type = "atan2" self.python_api = paddle.atan2 @@ -54,40 +53,39 @@ class TestAtan2(OpTest): class TestAtan2_float(TestAtan2): - def init_dtype(self): self.dtype = np.float32 def test_check_grad(self): if self.dtype not in [np.int32, np.int64]: - self.check_grad(['X1', 'X2'], - 'Out', - user_defined_grads=atan2_grad( - self.inputs['X1'], self.inputs['X2'], - 1 / self.inputs['X1'].size), - check_eager=True) + self.check_grad( + ['X1', 'X2'], + 'Out', + user_defined_grads=atan2_grad( + self.inputs['X1'], + self.inputs['X2'], + 1 / self.inputs['X1'].size, + ), + check_eager=True, + ) class TestAtan2_float16(TestAtan2_float): - def init_dtype(self): self.dtype = np.float16 class TestAtan2_int32(TestAtan2_float): - def init_dtype(self): self.dtype = np.int32 class TestAtan2_int64(TestAtan2_float): - def init_dtype(self): self.dtype = np.int64 class TestAtan2API(unittest.TestCase): - def init_dtype(self): self.dtype = 'float64' self.shape = [11, 17] @@ -118,7 +116,6 @@ class TestAtan2API(unittest.TestCase): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) X1 = paddle.to_tensor(self.x1) diff --git a/python/paddle/fluid/tests/unittests/test_attention_lstm_op.py b/python/paddle/fluid/tests/unittests/test_attention_lstm_op.py index b67ab48494a..053912005cc 100644 --- a/python/paddle/fluid/tests/unittests/test_attention_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_attention_lstm_op.py @@ -20,17 +20,18 @@ from test_softmax_op import stable_softmax def attention_lstm( - x, # T x M - lod, # 1 x N - h0, # N x D - c0, # N x D - fcws, # (M+D) x 1, 1x1 - fcbs, # 1 x 1, 1x1 - w, # (M+D) x 4D - b, # 1 x 4D - act_gate, - act_cell, - act_cand): + x, # T x M + lod, # 1 x N + h0, # N x D + c0, # N x D + fcws, # (M+D) x 1, 1x1 + fcbs, # 1 x 1, 1x1 + w, # (M+D) x 4D + b, # 1 x 4D + act_gate, + act_cell, + act_cand, +): T = sum(lod[0]) N = len(lod[0]) @@ -44,8 +45,9 @@ def attention_lstm( start_offset = 0 for bid in range(N): seq_len = lod[0][bid] - xi = np.copy(x[start_offset:start_offset + seq_len, :]).reshape( - seq_len, M) + xi = np.copy(x[start_offset : start_offset + seq_len, :]).reshape( + seq_len, M + ) prev_cell = np.copy(c0[bid]).reshape([1, D]) prev_hidden = np.copy(h0[bid]).reshape([1, D]) for step in range(seq_len): @@ -86,7 +88,6 @@ def attention_lstm( class TestAttentionLSTMOp(OpTest): - def set_conf(self): pass @@ -117,14 +118,24 @@ class TestAttentionLSTMOp(OpTest): fcb2 = np.random.normal(size=(1, 1)).astype('float32') # lstm weight and bias - w = np.random.normal(size=(self.M + self.D, - self.D * 4)).astype('float32') + w = np.random.normal(size=(self.M + self.D, self.D * 4)).astype( + 'float32' + ) b = np.random.normal(size=(1, self.D * 4)).astype('float32') - h, c = attention_lstm(x, self.lod, h0, c0, [fcw1, fcw2], [fcb1, fcb2], - w, b, ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + h, c = attention_lstm( + x, + self.lod, + h0, + c0, + [fcw1, fcw2], + [fcb1, fcb2], + w, + b, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = { 'X': (x, self.lod), @@ -134,7 +145,7 @@ class TestAttentionLSTMOp(OpTest): 'AttentionScalar': fcw2, 'AttentionScalarBias': fcb2, 'LSTMWeight': w, - 'LSTMBias': b + 'LSTMBias': b, } if self.has_initial_hidden: @@ -147,7 +158,7 @@ class TestAttentionLSTMOp(OpTest): self.attrs = { 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, - 'candidate_activation': self.act_cand + 'candidate_activation': self.act_cand, } def test_check_output(self): @@ -155,13 +166,11 @@ class TestAttentionLSTMOp(OpTest): class TestAttentionOpNonInit(TestAttentionLSTMOp): - def set_conf(self): self.has_initial_hidden = False class TestAttentionOpAct(TestAttentionLSTMOp): - def set_conf(self): self.M = 3 self.D = 2 @@ -171,28 +180,24 @@ class TestAttentionOpAct(TestAttentionLSTMOp): class TestAttentionOpMD1(TestAttentionLSTMOp): - def set_conf(self): self.M = 36 self.D = 8 class TestAttentionOpMD2(TestAttentionLSTMOp): - def set_conf(self): self.M = 8 self.D = 8 class TestAttentionOpMD3(TestAttentionLSTMOp): - def set_conf(self): self.M = 15 self.D = 30 class TestAttentionOpBS1(TestAttentionLSTMOp): - def set_conf(self): self.lod = [[5]] self.M = 16 @@ -200,13 +205,11 @@ class TestAttentionOpBS1(TestAttentionLSTMOp): class TestAttentionOpBS2(TestAttentionLSTMOp): - def set_conf(self): self.lod = [[3, 6]] class TestAttentionOpBS5(TestAttentionLSTMOp): - def set_conf(self): self.lod = [[3, 2, 4, 7, 5]] diff --git a/python/paddle/fluid/tests/unittests/test_attribute_var.py b/python/paddle/fluid/tests/unittests/test_attribute_var.py index 6e8e3c66750..60d202eb1b5 100644 --- a/python/paddle/fluid/tests/unittests/test_attribute_var.py +++ b/python/paddle/fluid/tests/unittests/test_attribute_var.py @@ -25,7 +25,6 @@ paddle.enable_static() class UnittestBase(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.init_info() @@ -41,8 +40,9 @@ class UnittestBase(unittest.TestCase): return type(self).__name__ def infer_prog(self): - config = paddle_infer.Config(self.save_path + '.pdmodel', - self.save_path + '.pdiparams') + config = paddle_infer.Config( + self.save_path + '.pdmodel', self.save_path + '.pdiparams' + ) predictor = paddle_infer.create_predictor(config) input_names = predictor.get_input_names() for i, shape in enumerate(self.shapes): @@ -65,7 +65,6 @@ class UnittestBase(unittest.TestCase): class TestDropout(UnittestBase): - def init_info(self): self.shapes = [[10, 10]] self.save_path = os.path.join(self.temp_dir.name, 'dropout') @@ -98,11 +97,11 @@ class TestDropout(UnittestBase): self.assertEqual( main_prog.block(0).ops[4].all_attrs()['dropout_prob'].name, - p.name) + p.name, + ) class TestTileTensorList(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, 'tile_tensors') @@ -136,7 +135,6 @@ class TestTileTensorList(UnittestBase): class TestTileTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, 'tile_tensor') @@ -169,13 +167,12 @@ class TestTileTensor(UnittestBase): class TestRegiterSupportTensorInOpMaker(unittest.TestCase): - def setUp(self): self.all_protos = OpProtoHolder.instance() self.support_tensor_attrs = { 'dropout': ['dropout_prob'], 'tile': ['repeat_times'], - 'concat': ['axis'] + 'concat': ['axis'], } # Just add a op example to test not support tensor self.not_support_tensor_attrs = {'svd': ['full_matrices']} @@ -189,8 +186,9 @@ class TestRegiterSupportTensorInOpMaker(unittest.TestCase): # All Attribute not tagged with .SupportTensor() in OpMaker will return False for op_type, attr_names in self.not_support_tensor_attrs.items(): for attr_name in attr_names: - self.assertFalse(self.is_support_tensor_attr( - op_type, attr_name)) + self.assertFalse( + self.is_support_tensor_attr(op_type, attr_name) + ) def is_support_tensor_attr(self, op_type, attr_name): proto = self.all_protos.get_op_proto(op_type) diff --git a/python/paddle/fluid/tests/unittests/test_auc_op.py b/python/paddle/fluid/tests/unittests/test_auc_op.py index 56c0b1c6977..0ea67b19b13 100644 --- a/python/paddle/fluid/tests/unittests/test_auc_op.py +++ b/python/paddle/fluid/tests/unittests/test_auc_op.py @@ -21,7 +21,6 @@ import paddle class TestAucOp(OpTest): - def setUp(self): self.op_type = "auc" pred = np.random.random((128, 2)).astype("float32") @@ -30,25 +29,27 @@ class TestAucOp(OpTest): slide_steps = 1 stat_pos = np.zeros( - (1 + slide_steps) * (num_thresholds + 1) + 1, ).astype("int64") + (1 + slide_steps) * (num_thresholds + 1) + 1, + ).astype("int64") stat_neg = np.zeros( - (1 + slide_steps) * (num_thresholds + 1) + 1, ).astype("int64") + (1 + slide_steps) * (num_thresholds + 1) + 1, + ).astype("int64") self.inputs = { 'Predict': pred, 'Label': labels, "StatPos": stat_pos, - "StatNeg": stat_neg + "StatNeg": stat_neg, } self.attrs = { 'curve': 'ROC', 'num_thresholds': num_thresholds, - "slide_steps": slide_steps + "slide_steps": slide_steps, } - python_auc = metrics.Auc(name="auc", - curve='ROC', - num_thresholds=num_thresholds) + python_auc = metrics.Auc( + name="auc", curve='ROC', num_thresholds=num_thresholds + ) python_auc.update(pred, labels) pos = python_auc._stat_pos * 2 @@ -58,7 +59,7 @@ class TestAucOp(OpTest): self.outputs = { 'AUC': np.array(python_auc.eval()), 'StatPosOut': np.array(pos), - 'StatNegOut': np.array(neg) + 'StatNegOut': np.array(neg), } def test_check_output(self): @@ -66,7 +67,6 @@ class TestAucOp(OpTest): class TestGlobalAucOp(OpTest): - def setUp(self): self.op_type = "auc" pred = np.random.random((128, 2)).astype("float32") @@ -81,17 +81,17 @@ class TestGlobalAucOp(OpTest): 'Predict': pred, 'Label': labels, "StatPos": stat_pos, - "StatNeg": stat_neg + "StatNeg": stat_neg, } self.attrs = { 'curve': 'ROC', 'num_thresholds': num_thresholds, - "slide_steps": slide_steps + "slide_steps": slide_steps, } - python_auc = metrics.Auc(name="auc", - curve='ROC', - num_thresholds=num_thresholds) + python_auc = metrics.Auc( + name="auc", curve='ROC', num_thresholds=num_thresholds + ) python_auc.update(pred, labels) pos = python_auc._stat_pos @@ -99,7 +99,7 @@ class TestGlobalAucOp(OpTest): self.outputs = { 'AUC': np.array(python_auc.eval()), 'StatPosOut': np.array(pos), - 'StatNegOut': np.array(neg) + 'StatNegOut': np.array(neg), } def test_check_output(self): @@ -107,17 +107,16 @@ class TestGlobalAucOp(OpTest): class TestAucAPI(unittest.TestCase): - def test_static(self): paddle.enable_static() data = paddle.static.data(name="input", shape=[-1, 1], dtype="float32") label = paddle.static.data(name="label", shape=[4], dtype="int64") - ins_tag_weight = paddle.static.data(name="ins_tag_weight", - shape=[4], - dtype="float32") - result = paddle.static.auc(input=data, - label=label, - ins_tag_weight=ins_tag_weight) + ins_tag_weight = paddle.static.data( + name="ins_tag_weight", shape=[4], dtype="float32" + ) + result = paddle.static.auc( + input=data, label=label, ins_tag_weight=ins_tag_weight + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -128,37 +127,34 @@ class TestAucAPI(unittest.TestCase): y = np.array([0, 0, 1, 0]).astype('int64') z = np.array([1, 1, 1, 1]).astype('float32') - output, = exe.run(feed={ - "input": x, - "label": y, - "ins_tag_weight": z - }, - fetch_list=[result[0]]) + (output,) = exe.run( + feed={"input": x, "label": y, "ins_tag_weight": z}, + fetch_list=[result[0]], + ) auc_np = np.array([0.66666667]).astype("float32") np.testing.assert_allclose(output, auc_np, rtol=1e-05) class TestAucOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): def test_type1(): data1 = fluid.data(name="input1", shape=[-1, 2], dtype="int") label1 = fluid.data(name="label1", shape=[-1], dtype="int") - ins_tag_w1 = paddle.static.data(name="label1", - shape=[-1], - dtype="int") - result1 = paddle.static.auc(input=data1, - label=label1, - ins_tag_weight=ins_tag_w1) + ins_tag_w1 = paddle.static.data( + name="label1", shape=[-1], dtype="int" + ) + result1 = paddle.static.auc( + input=data1, label=label1, ins_tag_weight=ins_tag_w1 + ) self.assertRaises(TypeError, test_type1) def test_type2(): - data2 = fluid.data(name="input2", - shape=[-1, 2], - dtype="float32") + data2 = fluid.data( + name="input2", shape=[-1, 2], dtype="float32" + ) label2 = fluid.data(name="label2", shape=[-1], dtype="float32") result2 = fluid.layers.auc(input=data2, label=label2) diff --git a/python/paddle/fluid/tests/unittests/test_auc_single_pred_op.py b/python/paddle/fluid/tests/unittests/test_auc_single_pred_op.py index 28e2808be35..13482d0af3a 100644 --- a/python/paddle/fluid/tests/unittests/test_auc_single_pred_op.py +++ b/python/paddle/fluid/tests/unittests/test_auc_single_pred_op.py @@ -19,7 +19,6 @@ from paddle.fluid import metrics class TestAucSinglePredOp(OpTest): - def setUp(self): self.op_type = "auc" pred = np.random.random((128, 2)).astype("float32") @@ -29,25 +28,27 @@ class TestAucSinglePredOp(OpTest): slide_steps = 1 stat_pos = np.zeros( - (1 + slide_steps) * (num_thresholds + 1) + 1, ).astype("int64") + (1 + slide_steps) * (num_thresholds + 1) + 1, + ).astype("int64") stat_neg = np.zeros( - (1 + slide_steps) * (num_thresholds + 1) + 1, ).astype("int64") + (1 + slide_steps) * (num_thresholds + 1) + 1, + ).astype("int64") self.inputs = { 'Predict': pred0, 'Label': labels, "StatPos": stat_pos, - "StatNeg": stat_neg + "StatNeg": stat_neg, } self.attrs = { 'curve': 'ROC', 'num_thresholds': num_thresholds, - "slide_steps": slide_steps + "slide_steps": slide_steps, } - python_auc = metrics.Auc(name="auc", - curve='ROC', - num_thresholds=num_thresholds) + python_auc = metrics.Auc( + name="auc", curve='ROC', num_thresholds=num_thresholds + ) for i in range(128): pred[i][1] = pred[i][0] python_auc.update(pred, labels) @@ -59,7 +60,7 @@ class TestAucSinglePredOp(OpTest): self.outputs = { 'AUC': np.array(python_auc.eval()), 'StatPosOut': np.array(pos), - 'StatNegOut': np.array(neg) + 'StatNegOut': np.array(neg), } def test_check_output(self): @@ -67,7 +68,6 @@ class TestAucSinglePredOp(OpTest): class TestAucGlobalSinglePredOp(OpTest): - def setUp(self): self.op_type = "auc" pred = np.random.random((128, 2)).astype("float32") @@ -83,17 +83,17 @@ class TestAucGlobalSinglePredOp(OpTest): 'Predict': pred0, 'Label': labels, "StatPos": stat_pos, - "StatNeg": stat_neg + "StatNeg": stat_neg, } self.attrs = { 'curve': 'ROC', 'num_thresholds': num_thresholds, - "slide_steps": slide_steps + "slide_steps": slide_steps, } - python_auc = metrics.Auc(name="auc", - curve='ROC', - num_thresholds=num_thresholds) + python_auc = metrics.Auc( + name="auc", curve='ROC', num_thresholds=num_thresholds + ) for i in range(128): pred[i][1] = pred[i][0] python_auc.update(pred, labels) @@ -103,7 +103,7 @@ class TestAucGlobalSinglePredOp(OpTest): self.outputs = { 'AUC': np.array(python_auc.eval()), 'StatPosOut': np.array(pos), - 'StatNegOut': np.array(neg) + 'StatNegOut': np.array(neg), } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_auto_growth_gpu_memory_limit.py b/python/paddle/fluid/tests/unittests/test_auto_growth_gpu_memory_limit.py index 948aa58990d..7f4f6c78c19 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_growth_gpu_memory_limit.py +++ b/python/paddle/fluid/tests/unittests/test_auto_growth_gpu_memory_limit.py @@ -23,7 +23,6 @@ if fluid.is_compiled_with_cuda(): class TestBase(unittest.TestCase): - def setUp(self): if fluid.is_compiled_with_cuda(): self._limit = fluid.core.globals()['FLAGS_gpu_memory_limit_mb'] @@ -36,8 +35,10 @@ class TestBase(unittest.TestCase): place = fluid.CUDAPlace(0) t = fluid.LoDTensor() - t.set(np.ndarray([int(self._limit / 2), other_dim], dtype='float32'), - place) + t.set( + np.ndarray([int(self._limit / 2), other_dim], dtype='float32'), + place, + ) del t t = fluid.LoDTensor() diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_autoconvert.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_autoconvert.py index 0777f5217b5..82969299efd 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_autoconvert.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_autoconvert.py @@ -18,7 +18,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestAutoParallelAutoConvert(TestMultipleGpus): - def test_auto_parallel_autoconvert(self): self.run_mnist_2gpu('auto_parallel_autoconvert.py') diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_cluster.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_cluster.py index df11100755e..a632d29f691 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_cluster.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_cluster.py @@ -199,7 +199,6 @@ cluster_json = """ class TestAutoParallelCluster(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -207,8 +206,9 @@ class TestAutoParallelCluster(unittest.TestCase): self.temp_dir.cleanup() def test_cluster(self): - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -295,8 +295,9 @@ class TestAutoParallelCluster(unittest.TestCase): self.assertEqual(device2_machine0.global_id, 2) self.assertEqual(device2_machine0.local_id, 0) self.assertEqual(device2_machine0.type, DeviceType.CPU) - self.assertEqual(device2_machine0.model, - "Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GH") + self.assertEqual( + device2_machine0.model, "Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GH" + ) self.assertAlmostEqual(device2_machine0.sp_gflops, 150) self.assertAlmostEqual(device2_machine0.dp_gflops, 75) self.assertAlmostEqual(device2_machine0.memory, 1510) @@ -401,8 +402,9 @@ class TestAutoParallelCluster(unittest.TestCase): self.assertEqual(device6_machine1.global_id, 6) self.assertEqual(device6_machine1.local_id, 0) self.assertEqual(device6_machine1.type, DeviceType.CPU) - self.assertEqual(device6_machine1.model, - "Intel(R) Xeon(R) Gold 6271C CPU @ 2.60G") + self.assertEqual( + device6_machine1.model, "Intel(R) Xeon(R) Gold 6271C CPU @ 2.60G" + ) self.assertAlmostEqual(device6_machine1.sp_gflops, 150) self.assertAlmostEqual(device6_machine1.dp_gflops, 75) self.assertAlmostEqual(device6_machine1.memory, 503) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_completion.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_completion.py index c757e3a45af..5e1659b49c1 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_completion.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_completion.py @@ -33,38 +33,42 @@ _global_process_mesh2 = None class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") def forward(self, input): if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) out = self.norm(input) out = self.linear0(out) @@ -76,79 +80,93 @@ class MLPLayer(nn.Layer): def mlp_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input, - process_mesh=_global_process_mesh, - shard_spec=["dp", None, None]) - - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + auto.shard_tensor( + input, + process_mesh=_global_process_mesh, + shard_spec=["dp", None, None], + ) + + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = mlp(input) return train_program, start_program class TestMLPAutoCompletion(unittest.TestCase): - def test_mlp_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_mlp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_mlp_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) # def test_mlp_misc(self): @@ -203,14 +221,15 @@ class TestMLPAutoCompletion(unittest.TestCase): class AttentionLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - sequence_len=512, - intermediate_size=4 * 1024, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + sequence_len=512, + intermediate_size=4 * 1024, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(AttentionLayer, self).__init__() self.hidden_size = hidden_size self.sequence_len = sequence_len @@ -219,38 +238,38 @@ class AttentionLayer(nn.Layer): self.vdim = self.embed_dim self.num_heads = num_heads self.head_dim = self.embed_dim // self.num_heads - assert self.head_dim * self.num_heads == self.embed_dim, \ - "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" self.dropout_ratio = dropout_ratio self.initializer_range = initializer_range self.training = True self.attn_mask = None weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.q_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) def forward(self, input): if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input, - process_mesh=_global_process_mesh, - shard_spec=["dp", None, None]) + auto.shard_tensor( + input, + process_mesh=_global_process_mesh, + shard_spec=["dp", None, None], + ) q = self.q_proj(input) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) @@ -260,15 +279,21 @@ class AttentionLayer(nn.Layer): v = self.v_proj(input) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -276,10 +301,9 @@ class AttentionLayer(nn.Layer): v = tensor.transpose(x=v, perm=[0, 2, 1, 3]) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.attn_mask is not None: product = product + self.attn_mask @@ -287,10 +311,12 @@ class AttentionLayer(nn.Layer): weights = F.softmax(product) if self.dropout_ratio: - weights = F.dropout(weights, - self.dropout_ratio, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout_ratio, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -301,98 +327,113 @@ class AttentionLayer(nn.Layer): # project to output out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) return out def attn_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="query", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - attn = AttentionLayer(hidden_size=hidden_size, - sequence_len=sequence_len, - intermediate_size=4 * hidden_size, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02) + input = static.data( + name="query", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + attn = AttentionLayer( + hidden_size=hidden_size, + sequence_len=sequence_len, + intermediate_size=4 * hidden_size, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = attn(input) return train_program, start_program class TestAttentionAutoCompletion(unittest.TestCase): - def test_attn_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = attn_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_attn_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = attn_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_attn_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = attn_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) class DecoderLayer(nn.Layer): - - def __init__(self, - vocab_size=32768, - hidden_size=1024, - sequence_len=512, - max_position_embeddings=512, - intermediate_size=4 * 1024, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + vocab_size=32768, + hidden_size=1024, + sequence_len=512, + max_position_embeddings=512, + intermediate_size=4 * 1024, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(DecoderLayer, self).__init__() self.vocab_size = vocab_size self.hidden_size = hidden_size @@ -408,57 +449,64 @@ class DecoderLayer(nn.Layer): self.attn_mask = None self.head_dim = self.embed_dim // self.num_heads - assert self.head_dim * self.num_heads == self.embed_dim, \ - "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" self.word_embeddings = nn.Embedding( self.vocab_size, self.hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=self.initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( self.max_position_embeddings, self.hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=self.initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ), + ), + ) - weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)) + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ) bias_attr = None - self.q_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) intermediate_size = 4 * self.hidden_size d_model = self.hidden_size dim_feedforward = intermediate_size - weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)) + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout1 = nn.Dropout(self.dropout_ratio) @@ -467,17 +515,21 @@ class DecoderLayer(nn.Layer): def forward(self, input_ids, position_ids): if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input_ids, - process_mesh=_global_process_mesh, - shard_spec=["dp", None]) + auto.shard_tensor( + input_ids, + process_mesh=_global_process_mesh, + shard_spec=["dp", None], + ) input_embeddings = self.word_embeddings(input_ids) position_embeddings = self.position_embeddings(position_ids) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.word_embeddings.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.word_embeddings.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) embeddings = input_embeddings + position_embeddings embeddings = self.dropout1(embeddings) @@ -494,15 +546,21 @@ class DecoderLayer(nn.Layer): v = self.v_proj(target) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -510,10 +568,9 @@ class DecoderLayer(nn.Layer): v = tensor.transpose(x=v, perm=[0, 2, 1, 3]) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.attn_mask is not None: product = product + self.attn_mask @@ -521,10 +578,12 @@ class DecoderLayer(nn.Layer): weights = F.softmax(product) if self.dropout_ratio: - weights = F.dropout(weights, - self.dropout_ratio, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout_ratio, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -536,9 +595,11 @@ class DecoderLayer(nn.Layer): out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # Add residual residual = embeddings + self.dropout2(out) @@ -552,12 +613,16 @@ class DecoderLayer(nn.Layer): out3 = self.linear1(out2) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # Add residual final = residual + self.dropout3(out3) @@ -565,81 +630,91 @@ class DecoderLayer(nn.Layer): def decoder_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input_ids = static.data(name="input_ids", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') - decoder = DecoderLayer(vocab_size=32768, - hidden_size=hidden_size, - sequence_len=sequence_len, - max_position_embeddings=512, - intermediate_size=4 * hidden_size, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02) + input_ids = static.data( + name="input_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + decoder = DecoderLayer( + vocab_size=32768, + hidden_size=hidden_size, + sequence_len=sequence_len, + max_position_embeddings=512, + intermediate_size=4 * hidden_size, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = decoder(input_ids, position_ids) return train_program, start_program class TestDecoderLayerAutoCompletion(unittest.TestCase): - def test_decoder_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = decoder_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_decoder_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = decoder_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_decoder_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = decoder_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_completion_gpt.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_completion_gpt.py index 239033e7c8b..0511ee97e55 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_completion_gpt.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_completion_gpt.py @@ -42,17 +42,19 @@ class MultiHeadAttention(nn.Layer): Cache = collections.namedtuple("Cache", ["k", "v"]) StaticCache = collections.namedtuple("StaticCache", ["k", "v"]) - def __init__(self, - embed_dim, - num_heads, - dropout=0., - kdim=None, - vdim=None, - need_weights=False, - weight_attr=None, - bias_attr=None, - topo=None, - fuse=False): + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + kdim=None, + vdim=None, + need_weights=False, + weight_attr=None, + bias_attr=None, + topo=None, + fuse=False, + ): super(MultiHeadAttention, self).__init__() self.embed_dim = embed_dim self.kdim = kdim if kdim is not None else embed_dim @@ -63,38 +65,36 @@ class MultiHeadAttention(nn.Layer): self.fuse = fuse self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" if topo is None or topo.mp_info.size == 1: if self.fuse: assert self.kdim == embed_dim assert self.vdim == embed_dim - self.qkv_proj = nn.Linear(embed_dim, - 3 * embed_dim, - weight_attr, - bias_attr=bias_attr) + self.qkv_proj = nn.Linear( + embed_dim, 3 * embed_dim, weight_attr, bias_attr=bias_attr + ) else: - self.q_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) def _fuse_prepare_qkv(self, query): mix_layer = self.qkv_proj(query) - mix_layer = paddle.reshape_(mix_layer, - [0, 0, self.num_heads, 3 * self.head_dim]) + mix_layer = paddle.reshape_( + mix_layer, [0, 0, self.num_heads, 3 * self.head_dim] + ) mix_layer = paddle.transpose(mix_layer, [0, 2, 1, 3]) q, k, v = paddle.split(mix_layer, num_or_sections=3, axis=-1) return q, k, v @@ -108,9 +108,11 @@ class MultiHeadAttention(nn.Layer): q = self.q_proj(query) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) q = tensor.transpose(x=q, perm=[0, 2, 1, 3]) @@ -144,12 +146,16 @@ class MultiHeadAttention(nn.Layer): v = self.v_proj(value) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -171,24 +177,22 @@ class MultiHeadAttention(nn.Layer): input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) v = layers.fill_constant_batch_size_like( input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) return self.Cache(k, v) else: # incremental_state with initial value, mainly for usage like UniLM return self.Cache(key, value) - def forward(self, - query, - key, - value, - attn_mask=None, - use_cache=False, - cache=None): + def forward( + self, query, key, value, attn_mask=None, use_cache=False, cache=None + ): r""" Applies multi-head attention to map queries and a set of key-value pairs to outputs. @@ -202,23 +206,25 @@ class MultiHeadAttention(nn.Layer): else: q, k, v = self._prepare_qkv(query, key, value, use_cache, cache) else: - q, k, v, cache = self._prepare_qkv(query, key, value, use_cache, - cache) + q, k, v, cache = self._prepare_qkv( + query, key, value, use_cache, cache + ) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if attn_mask is not None: product = product + attn_mask weights = F.softmax(product) if self.dropout: - weights = F.dropout(weights, - self.dropout, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -230,9 +236,11 @@ class MultiHeadAttention(nn.Layer): out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) outs = [out] if self.need_weights: @@ -247,12 +255,9 @@ class TransformerDecoder(nn.Layer): TransformerDecoder is a stack of N decoder layers. """ - def __init__(self, - decoder_layers, - num_layers, - norm=None, - hidden_size=None, - topo=None): + def __init__( + self, decoder_layers, num_layers, norm=None, hidden_size=None, topo=None + ): super(TransformerDecoder, self).__init__() self.topo = topo @@ -265,13 +270,15 @@ class TransformerDecoder(nn.Layer): raise ValueError("Only support LayerNorm") self.checkpoints = [] - def forward(self, - tgt, - memory, - tgt_mask=None, - memory_mask=None, - use_cache=False, - cache=None): + def forward( + self, + tgt, + memory, + tgt_mask=None, + memory_mask=None, + use_cache=False, + cache=None, + ): r""" Applies a stack of N Transformer decoder layers on inputs. If `norm` is provided, also applies layer normalization on the output of last decoder @@ -284,25 +291,31 @@ class TransformerDecoder(nn.Layer): for i, mod in enumerate(self.layers): if cache is None: if use_cache: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) new_caches.append(new_cache) else: - output = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache[i]) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache[i], + ) new_caches.append(new_cache) self.checkpoints.append(output.name) @@ -317,7 +330,7 @@ class TransformerDecoder(nn.Layer): produced by `TransformerDecoderLayer.gen_cache`. See `TransformerDecoderLayer.gen_cache` for more details. If `do_zip` is True, apply `zip` on these tuples to get a list with two elements. - """ + """ cache = [layer.gen_cache(memory) for layer in self.layers] if do_zip: cache = list(zip(*cache)) @@ -330,18 +343,20 @@ class TransformerDecoderLayer(nn.Layer): It contains multiheadattention and some linear layers. """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="gelu", - attn_dropout=None, - act_dropout=None, - normalize_before=True, - weight_attr=None, - bias_attr=None, - topo=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="gelu", + attn_dropout=None, + act_dropout=None, + normalize_before=True, + weight_attr=None, + bias_attr=None, + topo=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 @@ -354,21 +369,27 @@ class TransformerDecoderLayer(nn.Layer): weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0], - topo=topo) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + topo=topo, + ) if topo is None or topo.mp_info.size == 1: - self.linear1 = nn.Linear(d_model, - dim_feedforward, - weight_attrs[2], - bias_attr=bias_attrs[2]) - self.linear2 = nn.Linear(dim_feedforward, - d_model, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.linear1 = nn.Linear( + d_model, + dim_feedforward, + weight_attrs[2], + bias_attr=bias_attrs[2], + ) + self.linear2 = nn.Linear( + dim_feedforward, + d_model, + weight_attrs[2], + bias_attr=bias_attrs[2], + ) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) @@ -385,8 +406,9 @@ class TransformerDecoderLayer(nn.Layer): if use_cache is False: tgt = self.self_attn(tgt, tgt, tgt, tgt_mask, use_cache, cache) else: - tgt, incremental_cache = self.self_attn(tgt, tgt, tgt, tgt_mask, - use_cache, cache) + tgt, incremental_cache = self.self_attn( + tgt, tgt, tgt, tgt_mask, use_cache, cache + ) tgt = residual + self.dropout1(tgt) if not self.normalize_before: tgt = self.norm1(tgt) @@ -396,12 +418,16 @@ class TransformerDecoderLayer(nn.Layer): tgt = self.norm2(tgt) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear2.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear2.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # tgt = self.dropout2( # self.linear2(F.gelu( @@ -417,8 +443,9 @@ class TransformerDecoderLayer(nn.Layer): return tgt if use_cache is False else (tgt, incremental_cache) def gen_cache(self, memory): - incremental_cache = self.self_attn.gen_cache(memory, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + memory, type=self.self_attn.Cache + ) return incremental_cache @@ -427,29 +454,38 @@ class GPTEmbeddings(nn.Layer): Include embeddings from word, position and token_type embeddings """ - def __init__(self, - vocab_size, - hidden_size=768, - hidden_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - topo=None): + def __init__( + self, + vocab_size, + hidden_size=768, + hidden_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + topo=None, + ): super(GPTEmbeddings, self).__init__() if topo is None or topo.mp_info.size == 1: self.word_embeddings = nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( max_position_embeddings, hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.dropout = nn.Dropout(hidden_dropout_prob) @@ -462,9 +498,11 @@ class GPTEmbeddings(nn.Layer): input_embedings = self.word_embeddings(input_ids) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.word_embeddings.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.word_embeddings.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) position_embeddings = self.position_embeddings(position_ids) embeddings = input_embedings + position_embeddings @@ -477,20 +515,22 @@ class GPTModel(nn.Layer): The base model of gpt. """ - def __init__(self, - vocab_size, - hidden_size=768, - num_hidden_layers=12, - num_attention_heads=12, - intermediate_size=3072, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - topo=None): + def __init__( + self, + vocab_size, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + topo=None, + ): super(GPTModel, self).__init__() self.pad_token_id = pad_token_id @@ -503,71 +543,94 @@ class GPTModel(nn.Layer): if self.pipline_mode: self.layer_per_stage = num_hidden_layers // self.topo.pp_info.size - self.embeddings = GPTEmbeddings(vocab_size, hidden_size, - hidden_dropout_prob, - max_position_embeddings, - type_vocab_size, self.initializer_range, - topo) + self.embeddings = GPTEmbeddings( + vocab_size, + hidden_size, + hidden_dropout_prob, + max_position_embeddings, + type_vocab_size, + self.initializer_range, + topo, + ) decoder_layers = nn.LayerList() for i in range(num_hidden_layers): DecoderLayer = TransformerDecoderLayer decoder_layers.append( - DecoderLayer(d_model=hidden_size, - nhead=num_attention_heads, - dim_feedforward=intermediate_size, - dropout=hidden_dropout_prob, - activation=hidden_act, - attn_dropout=attention_probs_dropout_prob, - act_dropout=hidden_dropout_prob, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)), - bias_attr=None, - topo=topo)) + DecoderLayer( + d_model=hidden_size, + nhead=num_attention_heads, + dim_feedforward=intermediate_size, + dropout=hidden_dropout_prob, + activation=hidden_act, + attn_dropout=attention_probs_dropout_prob, + act_dropout=hidden_dropout_prob, + weight_attr=paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ), + bias_attr=None, + topo=topo, + ) + ) Decoder = TransformerDecoder - self.decoder = Decoder(decoder_layers, - num_hidden_layers, - norm="LayerNorm", - hidden_size=hidden_size, - topo=topo) + self.decoder = Decoder( + decoder_layers, + num_hidden_layers, + norm="LayerNorm", + hidden_size=hidden_size, + topo=topo, + ) self.checkpoints = [] - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - use_cache=False, - cache=None): + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + use_cache=False, + cache=None, + ): self.checkpoints = [] if attention_mask is None: length = paddle.shape(input_ids)[1] # Use bool mask attention_mask = paddle.tensor.tril( - paddle.ones((length, length), - dtype=self.embeddings.word_embeddings.weight.dtype)) + paddle.ones( + (length, length), + dtype=self.embeddings.word_embeddings.weight.dtype, + ) + ) if position_ids is None: past_length = 0 if cache is not None: past_length = paddle.shape(cache[0].k)[-2] - position_ids = paddle.arange(past_length, - paddle.shape(input_ids)[-1] + - past_length, - dtype='int64') + position_ids = paddle.arange( + past_length, + paddle.shape(input_ids)[-1] + past_length, + dtype='int64', + ) position_ids = position_ids.unsqueeze(0) # .expand_as(input_ids) position_ids = paddle.fluid.layers.expand_as( - position_ids, input_ids) - embedding_output = self.embeddings(input_ids=input_ids, - position_ids=position_ids) + position_ids, input_ids + ) + embedding_output = self.embeddings( + input_ids=input_ids, position_ids=position_ids + ) # TODO, use registered buffer - causal_mask = paddle.tensor.triu(paddle.ones( - (paddle.shape(input_ids)[-1], paddle.shape(input_ids)[-1])) * -1e9, - diagonal=1) + causal_mask = paddle.tensor.triu( + paddle.ones( + (paddle.shape(input_ids)[-1], paddle.shape(input_ids)[-1]) + ) + * -1e9, + diagonal=1, + ) if attention_mask is not None: attention_mask = attention_mask + causal_mask @@ -577,11 +640,13 @@ class GPTModel(nn.Layer): # The tensor returned by triu not in static graph. attention_mask.stop_gradient = True - encoder_outputs = self.decoder(embedding_output, - memory=None, - tgt_mask=attention_mask, - use_cache=use_cache, - cache=cache) + encoder_outputs = self.decoder( + embedding_output, + memory=None, + tgt_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) self.checkpoints.extend(self.decoder.checkpoints) return encoder_outputs @@ -603,11 +668,12 @@ class GPTForPretraining(nn.Layer): def parallel_matmul(self, lm_output, logit_weights, parallel_output, topo): if topo is not None and topo.mp_info.size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=None) + lm_output, group=None + ) - logits = paddle.matmul(input_parallel, - logit_weights, - transpose_y=True) + logits = paddle.matmul( + input_parallel, logit_weights, transpose_y=True + ) if parallel_output: return logits @@ -617,24 +683,29 @@ class GPTForPretraining(nn.Layer): logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - masked_positions=None, - use_cache=False, - cache=None): - outputs = self.gpt(input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - use_cache=use_cache, - cache=cache) + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + masked_positions=None, + use_cache=False, + cache=None, + ): + outputs = self.gpt( + input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) if use_cache: encoder_outputs, cached_kvs = outputs[:2] else: encoder_outputs = outputs - logits = self.parallel_matmul(encoder_outputs, self.weight, True, - self.gpt.topo) + logits = self.parallel_matmul( + encoder_outputs, self.weight, True, self.gpt.topo + ) if use_cache: return logits, cached_kvs @@ -653,11 +724,14 @@ class GPTPretrainingCriterion(nn.Layer): if topo is None or topo.mp_info.size == 1: self.loss_func = paddle.nn.CrossEntropyLoss(reduction="none") else: - self.loss_func = paddle.distributed.collective._c_softmax_with_cross_entropy + self.loss_func = ( + paddle.distributed.collective._c_softmax_with_cross_entropy + ) def forward(self, prediction_scores, masked_lm_labels, loss_mask): - masked_lm_loss = self.loss_func(prediction_scores, - masked_lm_labels.unsqueeze(2)) + masked_lm_loss = self.loss_func( + prediction_scores, masked_lm_labels.unsqueeze(2) + ) loss_mask = loss_mask.reshape([-1]) masked_lm_loss = paddle.sum(masked_lm_loss.reshape([-1]) * loss_mask) @@ -666,45 +740,51 @@ class GPTPretrainingCriterion(nn.Layer): def gpt_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 16 sequence_len = 512 - input_ids = static.data(name="input_ids", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + input_ids = static.data( + name="input_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float64') - labels = static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float64') + dtype='float64', + ) + labels = static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float64' + ) if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input_ids, - process_mesh=_global_process_mesh, - shard_spec=["dp", None]) - - gpt = GPTModel(vocab_size=32768, - hidden_size=1024, - num_hidden_layers=2, - num_attention_heads=16, - intermediate_size=4096, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=1024, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - topo=None) + auto.shard_tensor( + input_ids, + process_mesh=_global_process_mesh, + shard_spec=["dp", None], + ) + + gpt = GPTModel( + vocab_size=32768, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=1024, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + topo=None, + ) model = GPTForPretraining(gpt) @@ -718,57 +798,64 @@ def gpt_pretrain_forward(train_program, start_program): class TestGPTAutoCompletion(unittest.TestCase): - def test_gpt_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = gpt_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_gpt_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = gpt_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_gpt_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = gpt_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_cost_model.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_cost_model.py index 821fee4df16..bef9619ead3 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_cost_model.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_cost_model.py @@ -42,27 +42,27 @@ device = "gpu" if core.is_compiled_with_cuda() else "cpu" class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=256, - intermediate_size=4 * 256, - initializer_range=0.02, - is_distributed=True): + def __init__( + self, + hidden_size=256, + intermediate_size=4 * 256, + initializer_range=0.02, + is_distributed=True, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.is_distributed = is_distributed @@ -84,13 +84,14 @@ def get_single_node_data(): train_program = paddle.static.Program() startup_program = paddle.static.Program() - loss, train_program, startup_program = mlp_forward(train_program, - startup_program, - is_distributed=False) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program, is_distributed=False + ) cost_model = core.CostModel() - cost_data = cost_model.profile_measure(train_program, startup_program, - device, ["time"]) + cost_data = cost_model.profile_measure( + train_program, startup_program, device, ["time"] + ) op_name2cost = [{}, {}] for idx, op in enumerate(train_program.blocks[0].ops): @@ -102,34 +103,37 @@ def get_single_node_data(): def mlp_forward(train_program, start_program, is_distributed=True): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 256 sequence_len = 128 if is_distributed: - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) else: - input = paddle.ones(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = paddle.ones(name="label", - shape=[batch_size, 1], - dtype='float32') + input = paddle.ones( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = paddle.ones( + name="label", shape=[batch_size, 1], dtype='float32' + ) if is_distributed: auto.shard_tensor(input, PP_MESH_0, ["x", None]) auto.shard_tensor(label, PP_MESH_1, ["x", None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02, - is_distributed=is_distributed) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + is_distributed=is_distributed, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -139,8 +143,9 @@ def mlp_forward(train_program, start_program, is_distributed=True): def get_dist_prog(train_program, startup_program, dist_context, rank_id): - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -150,25 +155,38 @@ def get_dist_prog(train_program, startup_program, dist_context, rank_id): # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) - return auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) def check_runtime_estimation(cost): @@ -199,18 +217,19 @@ def check_empty_program_memory(cost): class TestCostModel(unittest.TestCase): - def test_empty_program_cost_model(self): empty_program = paddle.static.Program() startup_program = paddle.static.Program() standalone_cost_data = [{}] empty_pp_cfg = None cluster = None - cost = estimate_cost([empty_program], - cluster=cluster, - pipeline_config=empty_pp_cfg, - standalone_cost_data=standalone_cost_data, - batch_size=1) + cost = estimate_cost( + [empty_program], + cluster=cluster, + pipeline_config=empty_pp_cfg, + standalone_cost_data=standalone_cost_data, + batch_size=1, + ) self.assertTrue(check_empty_program_runtime(cost)) self.assertTrue(check_empty_program_memory(cost)) @@ -222,18 +241,30 @@ class TestCostModel(unittest.TestCase): train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() - distributed_program, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) - resharder = Resharder(distributed_program, dist_startup_prog, - rank_id, dist_context, dist_params_grads) + ( + distributed_program, + dist_startup_prog, + dist_params_grads, + ) = get_dist_prog( + train_program, startup_program, dist_context, rank_id + ) + resharder = Resharder( + distributed_program, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() dist_program.append(distributed_program) cluster = None - cost = estimate_cost(dist_program, - cluster=cluster, - pipeline_config=pp_cfg, - standalone_cost_data=standalone_cost_data, - batch_size=4) + cost = estimate_cost( + dist_program, + cluster=cluster, + pipeline_config=pp_cfg, + standalone_cost_data=standalone_cost_data, + batch_size=4, + ) self.assertTrue(check_runtime_estimation(cost)) self.assertTrue(check_memory_estimation(cost)) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_data_unshard.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_data_unshard.py index 34b1f5dd6d4..f641961b5af 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_data_unshard.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_data_unshard.py @@ -18,7 +18,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestAutoParallelDataUnshard(TestMultipleGpus): - def test_auto_parallel_data_unshard(self): self.run_mnist_2gpu('auto_parallel_data_unshard.py') diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_dist_tensor.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_dist_tensor.py index a3b842bed8f..e1f50460239 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_dist_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_dist_tensor.py @@ -23,18 +23,23 @@ from paddle.distributed.auto_parallel.parallelizer import AutoParallelizer from paddle.distributed.auto_parallel.partitioner import Partitioner from paddle.distributed.auto_parallel.dist_context import DistributedContext from paddle.distributed.auto_parallel.dist_tensor import DistributedTensor -from paddle.distributed.auto_parallel.dist_attribute import TensorDistributedAttribute +from paddle.distributed.auto_parallel.dist_attribute import ( + TensorDistributedAttribute, +) import test_auto_parallel_reshard from test_auto_parallel_reshard import mlp_forward -def get_dist_prog(train_program, - startup_program, - dist_context, - rank_id, - complete_train_program=None): - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) +def get_dist_prog( + train_program, + startup_program, + dist_context, + rank_id, + complete_train_program=None, +): + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -43,49 +48,68 @@ def get_dist_prog(train_program, # serial forward & backward completion completer = Completer(dist_context) - complete_train_program = completer.complete_forward_annotation( - train_program - ) if complete_train_program is None else complete_train_program + complete_train_program = ( + completer.complete_forward_annotation(train_program) + if complete_train_program is None + else complete_train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) - return auto_parallel_main_prog, auto_parallel_startup_prog, complete_train_program + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + complete_train_program, + ) class TestDistributedTensor(unittest.TestCase): - def test_new_local_tensor(self): test_auto_parallel_reshard._global_process_mesh = auto.ProcessMesh( - mesh=[0, 1], dim_names=["x"]) + mesh=[0, 1], dim_names=["x"] + ) test_auto_parallel_reshard._global_parallel_strategy = "dp" train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() rank_id = 0 - dist_main_prog, dist_startup_prog, complete_train_program = get_dist_prog( - train_program, startup_program, dist_context, rank_id) + ( + dist_main_prog, + dist_startup_prog, + complete_train_program, + ) = get_dist_prog(train_program, startup_program, dist_context, rank_id) dist_context.dist_main_programs[rank_id] = dist_main_prog dist_context.dist_startup_programs[rank_id] = dist_startup_prog name = "layer_norm_1.tmp_2" dist_tensor = dist_context.get_dist_tensor_for_program( - complete_train_program.global_block().vars[name]) + complete_train_program.global_block().vars[name] + ) dist_tensor._dist_context = dist_context intermediate_var_0 = dist_tensor.new_local_tensor( - name="intermediate_var_0") + name="intermediate_var_0" + ) self.assertEqual(intermediate_var_0.shape, (2, 1024)) self.assertEqual(intermediate_var_0.name, "intermediate_var_0") @@ -93,25 +117,34 @@ class TestDistributedTensor(unittest.TestCase): train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() - dist_main_prog, dist_startup_prog, complete_train_program = get_dist_prog( - train_program, startup_program, dist_context, rank_id, None) + ( + dist_main_prog, + dist_startup_prog, + complete_train_program, + ) = get_dist_prog( + train_program, startup_program, dist_context, rank_id, None + ) dist_context.dist_main_programs[rank_id] = dist_main_prog dist_context.dist_startup_programs[rank_id] = dist_startup_prog name = "layer_norm_1.tmp_2" dist_tensor = dist_context.get_dist_tensor_for_program( - complete_train_program.global_block().vars[name]) + complete_train_program.global_block().vars[name] + ) dist_tensor._dist_context = dist_context intermediate_var_1 = dist_tensor.new_local_tensor( - rank=rank_id, name="intermediate_var_1") + rank=rank_id, name="intermediate_var_1" + ) self.assertEqual(intermediate_var_0.shape, (2, 1024)) self.assertEqual(intermediate_var_1.name, "intermediate_var_1") name = "linear_0.w_0" dist_tensor = dist_context.get_dist_tensor_for_program( - complete_train_program.global_block().vars[name]) + complete_train_program.global_block().vars[name] + ) dist_tensor._dist_context = dist_context intermediate_var_1 = dist_tensor.new_local_tensor( - rank=rank_id, name="linear_0.w_0_intermediate") + rank=rank_id, name="linear_0.w_0_intermediate" + ) self.assertEqual(intermediate_var_1.shape, (1024, 4096)) self.assertEqual(intermediate_var_1.name, "linear_0.w_0_intermediate") @@ -121,7 +154,10 @@ class TestDistributedTensor(unittest.TestCase): id(copied_dist_context), id( copied_dist_context.get_dist_tensor_for_program( - dist_tensor.serial_tensor).dist_context)) + dist_tensor.serial_tensor + ).dist_context + ), + ) def test_static_method(self): dims_mapping = [1, 0] @@ -133,58 +169,63 @@ class TestDistributedTensor(unittest.TestCase): # rank 1 [(2, 4), (0, 3)] # rank 4 [(2, 4), (3, 6)] rank = 0 - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes + ) self.assertEqual(local_sizes, [2, 3]) local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank) + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_offsets, [0, 0]) - local_shard = DistributedTensor.get_local_shard(global_sizes, - dims_mapping, topology, - processes, rank) + local_shard = DistributedTensor.get_local_shard( + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_shard, [(0, 2), (0, 3)]) rank = 1 - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes + ) self.assertEqual(local_sizes, [2, 3]) local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank) + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_offsets, [2, 0]) - local_shard = DistributedTensor.get_local_shard(global_sizes, - dims_mapping, topology, - processes, rank) + local_shard = DistributedTensor.get_local_shard( + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_shard, [(2, 4), (0, 3)]) rank = 4 - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes + ) self.assertEqual(local_sizes, [2, 3]) local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank) + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_offsets, [2, 3]) - local_shard = DistributedTensor.get_local_shard(global_sizes, - dims_mapping, topology, - processes, rank) + local_shard = DistributedTensor.get_local_shard( + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_shard, [(2, 4), (3, 6)]) # global sizes local_sizes = [2, 3] global_sizes = DistributedTensor.get_global_sizes( - local_sizes, dims_mapping, topology, processes) + local_sizes, dims_mapping, topology, processes + ) self.assertEqual(global_sizes, [6, 6]) def test_instance_method(self): tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.dims_mapping = [1, 0] tensor_dist_attr.process_mesh = auto.ProcessMesh( - mesh=[[0, 1, 2], [3, 4, 5]]) - serial_tensor = paddle.static.data(name="data", - shape=[6, 6], - dtype='float32') + mesh=[[0, 1, 2], [3, 4, 5]] + ) + serial_tensor = paddle.static.data( + name="data", shape=[6, 6], dtype='float32' + ) dist_tensor = DistributedTensor(serial_tensor, tensor_dist_attr) # rank 0 [(0, 2), (0, 3)] diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_graph.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_graph.py index d8abad85cca..2e43bf6f928 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_graph.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_graph.py @@ -17,7 +17,6 @@ from paddle.distributed.auto_parallel.graph import Graph class TestAutoParallelGraph(unittest.TestCase): - def test_graph(self): graph = Graph(name="foo") self.assertEqual(graph.attrs["name"], "foo") diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_mapper.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_mapper.py index 5d473fe5507..e18a585d33f 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_mapper.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_mapper.py @@ -361,11 +361,9 @@ cluster_json = """ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=64, - intermediate_size=4 * 64, - initializer_range=0.02): + def __init__( + self, hidden_size=64, intermediate_size=4 * 64, initializer_range=0.02 + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size @@ -379,37 +377,37 @@ class MLPLayer(nn.Layer): weight_attr2 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr2)) weight_attr3 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr3)) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) - self.linear2 = nn.Linear(d_model, - dim_feedforward, - weight_attr2, - bias_attr=bias_attr) - self.linear3 = nn.Linear(dim_feedforward, - d_model, - weight_attr3, - bias_attr=bias_attr) + self.linear2 = nn.Linear( + d_model, dim_feedforward, weight_attr2, bias_attr=bias_attr + ) + self.linear3 = nn.Linear( + dim_feedforward, d_model, weight_attr3, bias_attr=bias_attr + ) def forward(self, input): if _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh[0], - [None, "y"]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh[0], [None, "y"] + ) - auto.shard_tensor(self.linear1.weight, _global_process_mesh[0], - ["y", None]) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh[0], ["y", None] + ) - auto.shard_tensor(self.linear2.weight, _global_process_mesh[1], - [None, "y"]) + auto.shard_tensor( + self.linear2.weight, _global_process_mesh[1], [None, "y"] + ) - auto.shard_tensor(self.linear3.weight, _global_process_mesh[1], - ["y", None]) + auto.shard_tensor( + self.linear3.weight, _global_process_mesh[1], ["y", None] + ) out = self.norm(input) out = self.linear0(out) @@ -425,22 +423,25 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program,start_program), \ - utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 64 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "dp_mp_pp": auto.shard_tensor(input, _global_process_mesh[0], ["x", None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) loss = paddle.mean(error_cost) @@ -448,8 +449,9 @@ def mlp_forward(train_program, start_program): def get_dist_prog(train_program, startup_program, dist_context, rank_id): - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -459,24 +461,38 @@ def get_dist_prog(train_program, startup_program, dist_context, rank_id): # auto completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) partitioner = Partitioner(dist_context, rank_id) - dist_train_program, dist_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + dist_train_program, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - dist_train_program, dist_startup_prog, dist_params_grads) - - resharder = Resharder(dist_train_program, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + dist_train_program, dist_startup_prog, dist_params_grads + ) + + resharder = Resharder( + dist_train_program, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() return dist_train_program, dist_startup_prog @@ -496,7 +512,6 @@ def get_device_local_ids(machine): class TestAutoParallelMapper(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -504,8 +519,9 @@ class TestAutoParallelMapper(unittest.TestCase): self.temp_dir.cleanup() def test_mapper_dp_mp_pp(self): - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -519,7 +535,7 @@ class TestAutoParallelMapper(unittest.TestCase): global _global_process_mesh _global_process_mesh = [ auto.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), - auto.ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]) + auto.ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]), ] processes = [0, 1, 2, 3, 4, 5, 6, 7] @@ -529,7 +545,8 @@ class TestAutoParallelMapper(unittest.TestCase): startup_program = static.Program() dist_context = DistributedContext() dist_train_program, dist_startup_prog = get_dist_prog( - train_program, startup_program, dist_context, rank_id) + train_program, startup_program, dist_context, rank_id + ) # if rank_id == 0: # print_program_with_dist_attr(dist_train_program, dist_context) dist_programs[rank_id] = [dist_train_program, None] @@ -547,8 +564,9 @@ class TestAutoParallelMapper(unittest.TestCase): self.assertTrue(is_in_machine(device_ids[0], machine)) machine_mapped_ranks.add(rank) machine_mapped_device_local_ids.add(device_ids[0]) - self.assertEqual(len(machine_mapped_ranks), - len(machine_mapped_device_local_ids)) + self.assertEqual( + len(machine_mapped_ranks), len(machine_mapped_device_local_ids) + ) all_mapped_ranks.update(machine_mapped_ranks) self.assertEqual(set(processes), all_mapped_ranks) @@ -575,35 +593,30 @@ class TestAutoParallelMapper(unittest.TestCase): dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) broadcast_op = train_program.global_block().append_op( type="c_broadcast", inputs={'X': input}, - attrs={ - 'ring_id': ring_id, - 'root': root_id - }, - outputs={'Out': output}) + attrs={'ring_id': ring_id, 'root': root_id}, + outputs={'Out': output}, + ) self.assertEqual(get_comm_volume(broadcast_op, 0, 1), 400) self.assertEqual(get_comm_volume(broadcast_op, 1, 0), None) allgather_op = train_program.global_block().append_op( type="c_allgather", inputs={'X': input}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': output}) + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': output}, + ) self.assertEqual(get_comm_volume(allgather_op, 0, 1), 400) self.assertEqual(get_comm_volume(allgather_op, 0, 0), None) reduce_op = train_program.global_block().append_op( type="c_reduce_sum", inputs={'X': input}, - attrs={ - 'ring_id': ring_id, - 'root_id': root_id - }, - outputs={'Out': output}) + attrs={'ring_id': ring_id, 'root_id': root_id}, + outputs={'Out': output}, + ) self.assertEqual(get_comm_volume(reduce_op, 0, 1), None) self.assertEqual(get_comm_volume(reduce_op, 1, 0), 400) cast_op = train_program.global_block().append_op( @@ -612,8 +625,9 @@ class TestAutoParallelMapper(unittest.TestCase): outputs={"Out": output}, attrs={ "in_dtype": fluid.core.VarDesc.VarType.FP32, - "out_dtype": fluid.core.VarDesc.VarType.FP32 - }) + "out_dtype": fluid.core.VarDesc.VarType.FP32, + }, + ) self.assertRaises(ValueError, get_comm_volume, cast_op, 0, 1) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner.py index b5af410a2d0..366e42cc686 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner.py @@ -44,16 +44,26 @@ def get_programs(annotated_func): train_program, start_program = annotated_func(train_program, start_program) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) rank_id = 3 dist_strategy = fleet.DistributedStrategy() partitioner = Partitioner(dist_context, rank_id) - test_auto_parallel_dist_main_prog, test_auto_parallel_dist_startup_prog, _ = partitioner.partition( - complete_train_program, start_program, []) + ( + test_auto_parallel_dist_main_prog, + test_auto_parallel_dist_startup_prog, + _, + ) = partitioner.partition(complete_train_program, start_program, []) - return complete_train_program, start_program, test_auto_parallel_dist_main_prog, test_auto_parallel_dist_startup_prog, dist_context + return ( + complete_train_program, + start_program, + test_auto_parallel_dist_main_prog, + test_auto_parallel_dist_startup_prog, + dist_context, + ) def is_all_parameters_shape_equal(prog1, prog2): @@ -84,41 +94,62 @@ def check_tensor_split(prog1, varnames1, prog2, varnames2, axis, nsplit): return True -def initialization_check(mode, dist_context, dist_startup_prog, - serial_startup_prog, var_need_broadcast, process_mesh, - mp_parallel_axis, dp_parallel_axis): +def initialization_check( + mode, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + process_mesh, + mp_parallel_axis, + dp_parallel_axis, +): if 'mp' in mode: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, mp_parallel_axis, - 3) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, mp_parallel_axis, 3 + ) mp_ring_id = new_process_group(group_ranks).id broadcast_ops = [ - op for op in dist_startup_prog.global_block().ops if - (op.type == "c_broadcast" and op.desc.attr("ring_id") == mp_ring_id) + op + for op in dist_startup_prog.global_block().ops + if ( + op.type == "c_broadcast" + and op.desc.attr("ring_id") == mp_ring_id + ) ] broadcast_varnames = sorted( - [op.desc.output_arg_names()[0] for op in broadcast_ops]) + [op.desc.output_arg_names()[0] for op in broadcast_ops] + ) if broadcast_varnames != var_need_broadcast: return False if 'dp' in mode: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, dp_parallel_axis, - 3) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, dp_parallel_axis, 3 + ) dp_ring_id = new_process_group(group_ranks).id nparam = len(serial_startup_prog.all_parameters()) - nbroadcast_dp = len([ - op for op in dist_startup_prog.global_block().ops if - (op.type == "c_broadcast" and op.desc.attr("ring_id") == dp_ring_id) - ]) + nbroadcast_dp = len( + [ + op + for op in dist_startup_prog.global_block().ops + if ( + op.type == "c_broadcast" + and op.desc.attr("ring_id") == dp_ring_id + ) + ] + ) if nparam != nbroadcast_dp: return False if "dp" in mode and 'mp' in mode: - nbroadcast = len([ - op for op in dist_startup_prog.global_block().ops - if op.type == "c_broadcast" - ]) + nbroadcast = len( + [ + op + for op in dist_startup_prog.global_block().ops + if op.type == "c_broadcast" + ] + ) if len(var_need_broadcast) + nbroadcast_dp != nbroadcast: return False @@ -141,14 +172,17 @@ def get_output_var_dist_attr(op, main_program, dist_context): def check_equal_var_dist_attr(serial_dist_attr, dist_attr): equal = True - if serial_dist_attr.process_mesh != dist_attr.process_mesh or \ - serial_dist_attr.dims_mapping != dist_attr.dims_mapping: + if ( + serial_dist_attr.process_mesh != dist_attr.process_mesh + or serial_dist_attr.dims_mapping != dist_attr.dims_mapping + ): equal = False return equal -def check_equal_dist_op_attr(dist_context, dist_main_prog, serial_op, dist_ops, - dist_op_idx): +def check_equal_dist_op_attr( + dist_context, dist_main_prog, serial_op, dist_ops, dist_op_idx +): equal = True # get serial op's process_mesh and impl_idx serial_op_dist_attr = dist_context.get_op_dist_attr_for_program(serial_op) @@ -161,34 +195,40 @@ def check_equal_dist_op_attr(dist_context, dist_main_prog, serial_op, dist_ops, for in_varname in dist_ops[i].desc.input_arg_names(): in_var = dist_main_prog.global_block().var(in_varname) tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - in_var) + in_var + ) tensor_dims_mapping = tensor_dist_attr.dims_mapping in_var_dims_mapping = op_dist_attr.get_input_dims_mapping( - in_varname) + in_varname + ) if tensor_dims_mapping != in_var_dims_mapping: equal = False for out_varname in dist_ops[i].desc.output_arg_names(): out_var = dist_main_prog.global_block().var(out_varname) tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - out_var) + out_var + ) tensor_dims_mapping = tensor_dist_attr.dims_mapping out_var_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_varname) + out_varname + ) if tensor_dims_mapping != out_var_dims_mapping: equal = False dist_op_process_mesh = op_dist_attr.process_mesh dist_op_impl_idx = op_dist_attr.impl_idx - if serial_op.desc.id() == dist_ops[i].desc.id() or \ - serial_process_mesh != dist_op_process_mesh or \ - serial_impl_idx != dist_op_impl_idx: + if ( + serial_op.desc.id() == dist_ops[i].desc.id() + or serial_process_mesh != dist_op_process_mesh + or serial_impl_idx != dist_op_impl_idx + ): equal = False return equal -def distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx): +def distributed_attr_check_for_dist_op( + serial_main_prog, dist_main_prog, dist_context, serial_op_idx, dist_op_idx +): equal = True serial_ops = serial_main_prog.global_block().ops @@ -200,27 +240,34 @@ def distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, if dist_op_0.type == "c_identity": # serial op input's dist_attr serial_in_dist_attr = get_input_var_dist_attr( - serial_op, serial_main_prog, dist_context) + serial_op, serial_main_prog, dist_context + ) # c_identity output's(new var) dist_attr identity_out_dist_attr = get_output_var_dist_attr( - dist_op_0, dist_main_prog, dist_context) + dist_op_0, dist_main_prog, dist_context + ) # check var dist_attr - equal = check_equal_var_dist_attr(serial_in_dist_attr, - identity_out_dist_attr) + equal = check_equal_var_dist_attr( + serial_in_dist_attr, identity_out_dist_attr + ) else: # serial op output's dist_attr serial_out_dist_attr = get_output_var_dist_attr( - serial_op, serial_main_prog, dist_context) + serial_op, serial_main_prog, dist_context + ) # dist op output's(new var) dist_attr - out_dist_attr = get_output_var_dist_attr(dist_op_0, dist_main_prog, - dist_context) + out_dist_attr = get_output_var_dist_attr( + dist_op_0, dist_main_prog, dist_context + ) # check var dist_attr - equal = check_equal_var_dist_attr(serial_out_dist_attr, - out_dist_attr) + equal = check_equal_var_dist_attr( + serial_out_dist_attr, out_dist_attr + ) # check op's dist_attr - equal = check_equal_dist_op_attr(dist_context, dist_main_prog, - serial_op, dist_ops, dist_op_idx[i]) + equal = check_equal_dist_op_attr( + dist_context, dist_main_prog, serial_op, dist_ops, dist_op_idx[i] + ) return equal @@ -242,45 +289,53 @@ def distributed_attr_check_for_program(dist_main_prog, dist_context): class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") def forward(self, input): if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) else: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, None]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, None], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, None], + ) out = self.norm(input) out = self.linear0(out) @@ -292,46 +347,61 @@ class MLPLayer(nn.Layer): def mlp_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input, - process_mesh=_global_process_mesh, - shard_spec=["dp", None, None]) - - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + auto.shard_tensor( + input, + process_mesh=_global_process_mesh, + shard_spec=["dp", None, None], + ) + + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = mlp(input) return train_program, start_program class TestMLPAutoPartitioner(unittest.TestCase): - def test_mlp_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) - - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - mlp_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) + + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(mlp_pretrain_forward) # parameter should not be partitioned self.assertTrue( - is_all_parameters_shape_equal(serial_main_prog, dist_main_prog)) + is_all_parameters_shape_equal(serial_main_prog, dist_main_prog) + ) self.assertTrue( - is_all_parameters_shape_equal(serial_startup_prog, - dist_startup_prog)) + is_all_parameters_shape_equal( + serial_startup_prog, dist_startup_prog + ) + ) # op in main prog should be the same serial_ops = serial_main_prog.global_block().ops @@ -343,152 +413,217 @@ class TestMLPAutoPartitioner(unittest.TestCase): # parameter initialization var_need_broadcast = [] self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=None, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=None, + dp_parallel_axis=0, + ) + ) def test_mlp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - mlp_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(mlp_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = ['linear_0.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = ['linear_0.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['linear_1.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = ['linear_1.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'layer_norm', 'c_identity', 'matmul_v2', 'elementwise_add', 'gelu', - 'matmul_v2', 'c_allreduce_sum', 'elementwise_add', 'dropout' + 'layer_norm', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'gelu', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', + 'dropout', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization var_need_broadcast = sorted( - ['layer_norm_0.b_0', 'layer_norm_0.w_0', 'linear_1.b_0']) + ['layer_norm_0.b_0', 'layer_norm_0.w_0', 'linear_1.b_0'] + ) self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=0, - dp_parallel_axis=None)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=0, + dp_parallel_axis=None, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr for dist op serial_op_idx = [1, 4] dist_op_idx = [[1, 2], [5, 6]] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) def test_mlp_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - mlp_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(mlp_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = ['linear_0.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = ['linear_0.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['linear_1.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = ['linear_1.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'layer_norm', 'c_identity', 'matmul_v2', 'elementwise_add', 'gelu', - 'matmul_v2', 'c_allreduce_sum', 'elementwise_add', 'dropout' + 'layer_norm', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'gelu', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', + 'dropout', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization var_need_broadcast = sorted( - ['layer_norm_0.b_0', 'layer_norm_0.w_0', 'linear_1.b_0']) + ['layer_norm_0.b_0', 'layer_norm_0.w_0', 'linear_1.b_0'] + ) self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=1, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=1, + dp_parallel_axis=0, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr for dist op serial_op_idx = [1, 4] dist_op_idx = [[1, 2], [5, 6]] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) class AttentionLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - sequence_len=512, - intermediate_size=4 * 1024, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + sequence_len=512, + intermediate_size=4 * 1024, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(AttentionLayer, self).__init__() self.hidden_size = hidden_size self.sequence_len = sequence_len @@ -497,38 +632,38 @@ class AttentionLayer(nn.Layer): self.vdim = self.embed_dim self.num_heads = num_heads self.head_dim = self.embed_dim // self.num_heads - assert self.head_dim * self.num_heads == self.embed_dim, \ - "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" self.dropout_ratio = dropout_ratio self.initializer_range = initializer_range self.training = True self.attn_mask = None weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.q_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) def forward(self, input): if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input, - process_mesh=_global_process_mesh, - shard_spec=["dp", None, None]) + auto.shard_tensor( + input, + process_mesh=_global_process_mesh, + shard_spec=["dp", None, None], + ) q = self.q_proj(input) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) @@ -538,15 +673,21 @@ class AttentionLayer(nn.Layer): v = self.v_proj(input) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -554,10 +695,9 @@ class AttentionLayer(nn.Layer): v = tensor.transpose(x=v, perm=[0, 2, 1, 3]) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.attn_mask is not None: product = product + self.attn_mask @@ -565,10 +705,12 @@ class AttentionLayer(nn.Layer): weights = F.softmax(product) if self.dropout_ratio: - weights = F.dropout(weights, - self.dropout_ratio, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout_ratio, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -580,50 +722,65 @@ class AttentionLayer(nn.Layer): out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) return out def attn_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="query", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - attn = AttentionLayer(hidden_size=hidden_size, - sequence_len=sequence_len, - intermediate_size=4 * hidden_size, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02) + input = static.data( + name="query", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + attn = AttentionLayer( + hidden_size=hidden_size, + sequence_len=sequence_len, + intermediate_size=4 * hidden_size, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = attn(input) return train_program, start_program class TestAttentionAutoPartitioner(unittest.TestCase): - def test_attn_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) - - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - attn_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) + + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(attn_pretrain_forward) # parameter should not be partitioned self.assertTrue( - is_all_parameters_shape_equal(serial_main_prog, dist_main_prog)) + is_all_parameters_shape_equal(serial_main_prog, dist_main_prog) + ) self.assertTrue( - is_all_parameters_shape_equal(serial_startup_prog, - dist_startup_prog)) + is_all_parameters_shape_equal( + serial_startup_prog, dist_startup_prog + ) + ) # op in main prog should be the same serial_ops = serial_main_prog.global_block().ops @@ -635,162 +792,247 @@ class TestAttentionAutoPartitioner(unittest.TestCase): # parameter initialization var_need_broadcast = [] self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=None, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=None, + dp_parallel_axis=0, + ) + ) def test_attn_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) - - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - attn_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) + + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(attn_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = ['linear_0.w_0', 'linear_1.w_0', 'linear_2.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = ['linear_0.b_0', 'linear_1.b_0', 'linear_2.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['linear_3.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = ['linear_3.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'c_identity', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'c_identity', 'matmul_v2', 'elementwise_add', - 'c_identity', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'reshape2', 'transpose2', 'matmul', 'softmax', - 'dropout', 'matmul_v2', 'transpose2', 'reshape2', 'matmul_v2', - 'c_allreduce_sum', 'elementwise_add' + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'reshape2', + 'transpose2', + 'matmul', + 'softmax', + 'dropout', + 'matmul_v2', + 'transpose2', + 'reshape2', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization var_need_broadcast = ['linear_3.b_0'] self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=0, - dp_parallel_axis=None)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=0, + dp_parallel_axis=None, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr for dist op serial_op_idx = [0, 4, 6, 18] dist_op_idx = [[0, 1], [5, 6], [8, 9], [21, 22]] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) def test_attn_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) - - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - attn_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) + + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(attn_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = ['linear_0.w_0', 'linear_1.w_0', 'linear_2.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = ['linear_0.b_0', 'linear_1.b_0', 'linear_2.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['linear_3.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = ['linear_3.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'c_identity', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'c_identity', 'matmul_v2', 'elementwise_add', - 'c_identity', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'reshape2', 'transpose2', 'matmul', 'softmax', - 'dropout', 'matmul_v2', 'transpose2', 'reshape2', 'matmul_v2', - 'c_allreduce_sum', 'elementwise_add' + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'reshape2', + 'transpose2', + 'matmul', + 'softmax', + 'dropout', + 'matmul_v2', + 'transpose2', + 'reshape2', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization var_need_broadcast = ['linear_3.b_0'] self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=1, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=1, + dp_parallel_axis=0, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr for dist op serial_op_idx = [0, 4, 6, 18] dist_op_idx = [[0, 1], [5, 6], [8, 9], [21, 22]] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) class DecoderLayer(nn.Layer): - - def __init__(self, - vocab_size=32768, - hidden_size=1024, - sequence_len=512, - max_position_embeddings=512, - intermediate_size=4 * 1024, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + vocab_size=32768, + hidden_size=1024, + sequence_len=512, + max_position_embeddings=512, + intermediate_size=4 * 1024, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(DecoderLayer, self).__init__() self.vocab_size = vocab_size self.hidden_size = hidden_size @@ -806,57 +1048,64 @@ class DecoderLayer(nn.Layer): self.attn_mask = None self.head_dim = self.embed_dim // self.num_heads - assert self.head_dim * self.num_heads == self.embed_dim, \ - "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" self.word_embeddings = nn.Embedding( self.vocab_size, self.hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=self.initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( self.max_position_embeddings, self.hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=self.initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ), + ), + ) - weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)) + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ) bias_attr = None - self.q_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) intermediate_size = 4 * self.hidden_size d_model = self.hidden_size dim_feedforward = intermediate_size - weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)) + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout1 = nn.Dropout(self.dropout_ratio) self.dropout2 = nn.Dropout(self.dropout_ratio, mode="upscale_in_train") @@ -864,17 +1113,21 @@ class DecoderLayer(nn.Layer): def forward(self, input_ids, position_ids): if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input_ids, - process_mesh=_global_process_mesh, - shard_spec=["dp", None]) + auto.shard_tensor( + input_ids, + process_mesh=_global_process_mesh, + shard_spec=["dp", None], + ) input_embeddings = self.word_embeddings(input_ids) position_embeddings = self.position_embeddings(position_ids) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.word_embeddings.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.word_embeddings.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) embeddings = input_embeddings + position_embeddings embeddings = self.dropout1(embeddings) @@ -891,15 +1144,21 @@ class DecoderLayer(nn.Layer): v = self.v_proj(target) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -907,10 +1166,9 @@ class DecoderLayer(nn.Layer): v = tensor.transpose(x=v, perm=[0, 2, 1, 3]) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.attn_mask is not None: product = product + self.attn_mask @@ -918,10 +1176,12 @@ class DecoderLayer(nn.Layer): weights = F.softmax(product) if self.dropout_ratio: - weights = F.dropout(weights, - self.dropout_ratio, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout_ratio, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -933,13 +1193,17 @@ class DecoderLayer(nn.Layer): out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) else: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, None], + ) # Add residual residual = embeddings + self.dropout2(out) @@ -953,12 +1217,16 @@ class DecoderLayer(nn.Layer): out3 = self.linear1(out2) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # Add residual final = residual + self.dropout3(out3) @@ -966,184 +1234,343 @@ class DecoderLayer(nn.Layer): def decoder_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input_ids = static.data(name="input_ids", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') - decoder = DecoderLayer(vocab_size=32768, - hidden_size=hidden_size, - sequence_len=sequence_len, - max_position_embeddings=512, - intermediate_size=4 * hidden_size, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02) + input_ids = static.data( + name="input_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + decoder = DecoderLayer( + vocab_size=32768, + hidden_size=hidden_size, + sequence_len=sequence_len, + max_position_embeddings=512, + intermediate_size=4 * hidden_size, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = decoder(input_ids, position_ids) return train_program, start_program class TestDecoderLayerPartitioner(unittest.TestCase): - def test_decoder_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - decoder_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(decoder_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = [ - 'linear_0.w_0', 'linear_1.w_0', 'linear_2.w_0', 'linear_4.w_0' + 'linear_0.w_0', + 'linear_1.w_0', + 'linear_2.w_0', + 'linear_4.w_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = [ - 'linear_0.b_0', 'linear_1.b_0', 'linear_2.b_0', 'linear_4.b_0' + 'linear_0.b_0', + 'linear_1.b_0', + 'linear_2.b_0', + 'linear_4.b_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['word_embeddings', 'linear_3.w_0', 'linear_5.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = [ - 'linear_3.b_0', 'pos_embeddings', 'layer_norm_0.b_0', - 'layer_norm_0.w_0', 'linear_5.b_0' + 'linear_3.b_0', + 'pos_embeddings', + 'layer_norm_0.b_0', + 'layer_norm_0.w_0', + 'linear_5.b_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'c_embedding', 'c_allreduce_sum', 'lookup_table_v2', - 'elementwise_add', 'dropout', 'layer_norm', 'c_identity', - 'matmul_v2', 'elementwise_add', 'reshape2', 'transpose2', - 'c_identity', 'matmul_v2', 'elementwise_add', 'c_identity', - 'matmul_v2', 'elementwise_add', 'reshape2', 'transpose2', - 'reshape2', 'transpose2', 'matmul', 'softmax', 'dropout', - 'matmul_v2', 'transpose2', 'reshape2', 'matmul_v2', - 'c_allreduce_sum', 'elementwise_add', 'dropout', 'elementwise_add', - 'layer_norm', 'c_identity', 'matmul_v2', 'elementwise_add', 'gelu', - 'matmul_v2', 'c_allreduce_sum', 'elementwise_add', 'dropout', - 'elementwise_add' + 'c_embedding', + 'c_allreduce_sum', + 'lookup_table_v2', + 'elementwise_add', + 'dropout', + 'layer_norm', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'reshape2', + 'transpose2', + 'matmul', + 'softmax', + 'dropout', + 'matmul_v2', + 'transpose2', + 'reshape2', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', + 'dropout', + 'elementwise_add', + 'layer_norm', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'gelu', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', + 'dropout', + 'elementwise_add', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization - var_need_broadcast = sorted([ - 'linear_3.b_0', 'pos_embeddings', 'layer_norm_0.b_0', - 'layer_norm_0.w_0', 'linear_5.b_0' - ]) + var_need_broadcast = sorted( + [ + 'linear_3.b_0', + 'pos_embeddings', + 'layer_norm_0.b_0', + 'layer_norm_0.w_0', + 'linear_5.b_0', + ] + ) self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=1, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=1, + dp_parallel_axis=0, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr serial_op_idx = [0, 5, 9, 11, 23, 28, 31] - dist_op_idx = [[0, 1], [6, 7], [11, 12], [14, 15], [27, 28], [33, 34], - [37, 38]] + dist_op_idx = [ + [0, 1], + [6, 7], + [11, 12], + [14, 15], + [27, 28], + [33, 34], + [37, 38], + ] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) def test_decoder_noparallel(self): global _global_parallel_strategy _global_parallel_strategy = "None" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["x", "y"]) - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - decoder_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["x", "y"] + ) + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(decoder_pretrain_forward) # param should be partition nrank = 1 # col parallel weights = [ - 'linear_0.w_0', 'linear_1.w_0', 'linear_2.w_0', 'linear_4.w_0' + 'linear_0.w_0', + 'linear_1.w_0', + 'linear_2.w_0', + 'linear_4.w_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = [ - 'linear_0.b_0', 'linear_1.b_0', 'linear_2.b_0', 'linear_4.b_0' + 'linear_0.b_0', + 'linear_1.b_0', + 'linear_2.b_0', + 'linear_4.b_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['word_embeddings', 'linear_3.w_0', 'linear_5.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = [ - 'linear_3.b_0', 'pos_embeddings', 'layer_norm_0.b_0', - 'layer_norm_0.w_0', 'linear_5.b_0' + 'linear_3.b_0', + 'pos_embeddings', + 'layer_norm_0.b_0', + 'layer_norm_0.w_0', + 'linear_5.b_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'lookup_table_v2', 'lookup_table_v2', 'elementwise_add', 'dropout', - 'layer_norm', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'matmul_v2', 'elementwise_add', 'matmul_v2', - 'elementwise_add', 'reshape2', 'transpose2', 'reshape2', - 'transpose2', 'matmul', 'softmax', 'dropout', 'matmul_v2', - 'transpose2', 'reshape2', 'matmul_v2', 'elementwise_add', 'dropout', - 'elementwise_add', 'layer_norm', 'matmul_v2', 'elementwise_add', - 'gelu', 'matmul_v2', 'elementwise_add', 'dropout', 'elementwise_add' + 'lookup_table_v2', + 'lookup_table_v2', + 'elementwise_add', + 'dropout', + 'layer_norm', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'matmul_v2', + 'elementwise_add', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'reshape2', + 'transpose2', + 'matmul', + 'softmax', + 'dropout', + 'matmul_v2', + 'transpose2', + 'reshape2', + 'matmul_v2', + 'elementwise_add', + 'dropout', + 'elementwise_add', + 'layer_norm', + 'matmul_v2', + 'elementwise_add', + 'gelu', + 'matmul_v2', + 'elementwise_add', + 'dropout', + 'elementwise_add', ] self.assertTrue(dist_ops == ref_ops) dist_ops = dist_startup_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'gaussian_random', 'gaussian_random', 'gaussian_random', - 'fill_constant', 'gaussian_random', 'fill_constant', - 'gaussian_random', 'fill_constant', 'gaussian_random', - 'fill_constant', 'gaussian_random', 'fill_constant', - 'gaussian_random', 'fill_constant', 'fill_constant', - 'fill_constant', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast' + 'gaussian_random', + 'gaussian_random', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', ] self.assertTrue(dist_ops == ref_ops) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner_gpt.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner_gpt.py index 990b42947e6..852c6ab74b1 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner_gpt.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner_gpt.py @@ -88,17 +88,19 @@ class MultiHeadAttention(nn.Layer): Cache = collections.namedtuple("Cache", ["k", "v"]) StaticCache = collections.namedtuple("StaticCache", ["k", "v"]) - def __init__(self, - embed_dim, - num_heads, - dropout=0., - kdim=None, - vdim=None, - need_weights=False, - weight_attr=None, - bias_attr=None, - topo=None, - fuse=False): + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + kdim=None, + vdim=None, + need_weights=False, + weight_attr=None, + bias_attr=None, + topo=None, + fuse=False, + ): super(MultiHeadAttention, self).__init__() self.embed_dim = embed_dim self.kdim = kdim if kdim is not None else embed_dim @@ -109,38 +111,36 @@ class MultiHeadAttention(nn.Layer): self.fuse = fuse self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" if topo is None or topo.mp_info.size == 1: if self.fuse: assert self.kdim == embed_dim assert self.vdim == embed_dim - self.qkv_proj = nn.Linear(embed_dim, - 3 * embed_dim, - weight_attr, - bias_attr=bias_attr) + self.qkv_proj = nn.Linear( + embed_dim, 3 * embed_dim, weight_attr, bias_attr=bias_attr + ) else: - self.q_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) def _fuse_prepare_qkv(self, query): mix_layer = self.qkv_proj(query) - mix_layer = paddle.reshape_(mix_layer, - [0, 0, self.num_heads, 3 * self.head_dim]) + mix_layer = paddle.reshape_( + mix_layer, [0, 0, self.num_heads, 3 * self.head_dim] + ) mix_layer = paddle.transpose(mix_layer, [0, 2, 1, 3]) q, k, v = paddle.split(mix_layer, num_or_sections=3, axis=-1) return q, k, v @@ -154,9 +154,11 @@ class MultiHeadAttention(nn.Layer): q = self.q_proj(query) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) q = tensor.transpose(x=q, perm=[0, 2, 1, 3]) @@ -190,12 +192,16 @@ class MultiHeadAttention(nn.Layer): v = self.v_proj(value) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -217,24 +223,22 @@ class MultiHeadAttention(nn.Layer): input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) v = layers.fill_constant_batch_size_like( input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) return self.Cache(k, v) else: # incremental_state with initial value, mainly for usage like UniLM return self.Cache(key, value) - def forward(self, - query, - key, - value, - attn_mask=None, - use_cache=False, - cache=None): + def forward( + self, query, key, value, attn_mask=None, use_cache=False, cache=None + ): r""" Applies multi-head attention to map queries and a set of key-value pairs to outputs. @@ -248,23 +252,25 @@ class MultiHeadAttention(nn.Layer): else: q, k, v = self._prepare_qkv(query, key, value, use_cache, cache) else: - q, k, v, cache = self._prepare_qkv(query, key, value, use_cache, - cache) + q, k, v, cache = self._prepare_qkv( + query, key, value, use_cache, cache + ) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if attn_mask is not None: product = product + attn_mask weights = F.softmax(product) if self.dropout: - weights = F.dropout(weights, - self.dropout, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -276,9 +282,11 @@ class MultiHeadAttention(nn.Layer): out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) outs = [out] if self.need_weights: @@ -293,12 +301,9 @@ class TransformerDecoder(nn.Layer): TransformerDecoder is a stack of N decoder layers. """ - def __init__(self, - decoder_layers, - num_layers, - norm=None, - hidden_size=None, - topo=None): + def __init__( + self, decoder_layers, num_layers, norm=None, hidden_size=None, topo=None + ): super(TransformerDecoder, self).__init__() self.topo = topo @@ -311,13 +316,15 @@ class TransformerDecoder(nn.Layer): raise ValueError("Only support LayerNorm") self.checkpoints = [] - def forward(self, - tgt, - memory, - tgt_mask=None, - memory_mask=None, - use_cache=False, - cache=None): + def forward( + self, + tgt, + memory, + tgt_mask=None, + memory_mask=None, + use_cache=False, + cache=None, + ): r""" Applies a stack of N Transformer decoder layers on inputs. If `norm` is provided, also applies layer normalization on the output of last decoder @@ -330,25 +337,31 @@ class TransformerDecoder(nn.Layer): for i, mod in enumerate(self.layers): if cache is None: if use_cache: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) new_caches.append(new_cache) else: - output = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache[i]) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache[i], + ) new_caches.append(new_cache) self.checkpoints.append(output.name) @@ -363,7 +376,7 @@ class TransformerDecoder(nn.Layer): produced by `TransformerDecoderLayer.gen_cache`. See `TransformerDecoderLayer.gen_cache` for more details. If `do_zip` is True, apply `zip` on these tuples to get a list with two elements. - """ + """ cache = [layer.gen_cache(memory) for layer in self.layers] if do_zip: cache = list(zip(*cache)) @@ -376,18 +389,20 @@ class TransformerDecoderLayer(nn.Layer): It contains multiheadattention and some linear layers. """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="gelu", - attn_dropout=None, - act_dropout=None, - normalize_before=True, - weight_attr=None, - bias_attr=None, - topo=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="gelu", + attn_dropout=None, + act_dropout=None, + normalize_before=True, + weight_attr=None, + bias_attr=None, + topo=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 @@ -400,21 +415,27 @@ class TransformerDecoderLayer(nn.Layer): weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0], - topo=topo) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + topo=topo, + ) if topo is None or topo.mp_info.size == 1: - self.linear1 = nn.Linear(d_model, - dim_feedforward, - weight_attrs[2], - bias_attr=bias_attrs[2]) - self.linear2 = nn.Linear(dim_feedforward, - d_model, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.linear1 = nn.Linear( + d_model, + dim_feedforward, + weight_attrs[2], + bias_attr=bias_attrs[2], + ) + self.linear2 = nn.Linear( + dim_feedforward, + d_model, + weight_attrs[2], + bias_attr=bias_attrs[2], + ) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) @@ -431,8 +452,9 @@ class TransformerDecoderLayer(nn.Layer): if use_cache is False: tgt = self.self_attn(tgt, tgt, tgt, tgt_mask, use_cache, cache) else: - tgt, incremental_cache = self.self_attn(tgt, tgt, tgt, tgt_mask, - use_cache, cache) + tgt, incremental_cache = self.self_attn( + tgt, tgt, tgt, tgt_mask, use_cache, cache + ) tgt = residual + self.dropout1(tgt) if not self.normalize_before: tgt = self.norm1(tgt) @@ -442,12 +464,16 @@ class TransformerDecoderLayer(nn.Layer): tgt = self.norm2(tgt) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear2.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear2.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # tgt = self.dropout2( # self.linear2(F.gelu( @@ -463,8 +489,9 @@ class TransformerDecoderLayer(nn.Layer): return tgt if use_cache is False else (tgt, incremental_cache) def gen_cache(self, memory): - incremental_cache = self.self_attn.gen_cache(memory, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + memory, type=self.self_attn.Cache + ) return incremental_cache @@ -473,29 +500,38 @@ class GPTEmbeddings(nn.Layer): Include embeddings from word, position and token_type embeddings """ - def __init__(self, - vocab_size, - hidden_size=768, - hidden_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - topo=None): + def __init__( + self, + vocab_size, + hidden_size=768, + hidden_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + topo=None, + ): super(GPTEmbeddings, self).__init__() if topo is None or topo.mp_info.size == 1: self.word_embeddings = nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( max_position_embeddings, hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.dropout = nn.Dropout(hidden_dropout_prob) @@ -508,9 +544,11 @@ class GPTEmbeddings(nn.Layer): input_embedings = self.word_embeddings(input_ids) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.word_embeddings.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.word_embeddings.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) position_embeddings = self.position_embeddings(position_ids) embeddings = input_embedings + position_embeddings @@ -523,20 +561,22 @@ class GPTModel(nn.Layer): The base model of gpt. """ - def __init__(self, - vocab_size, - hidden_size=768, - num_hidden_layers=4, - num_attention_heads=12, - intermediate_size=3072, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - topo=None): + def __init__( + self, + vocab_size, + hidden_size=768, + num_hidden_layers=4, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + topo=None, + ): super(GPTModel, self).__init__() self.pad_token_id = pad_token_id @@ -549,71 +589,94 @@ class GPTModel(nn.Layer): if self.pipline_mode: self.layer_per_stage = num_hidden_layers // self.topo.pp_info.size - self.embeddings = GPTEmbeddings(vocab_size, hidden_size, - hidden_dropout_prob, - max_position_embeddings, - type_vocab_size, self.initializer_range, - topo) + self.embeddings = GPTEmbeddings( + vocab_size, + hidden_size, + hidden_dropout_prob, + max_position_embeddings, + type_vocab_size, + self.initializer_range, + topo, + ) decoder_layers = nn.LayerList() for i in range(num_hidden_layers): DecoderLayer = TransformerDecoderLayer decoder_layers.append( - DecoderLayer(d_model=hidden_size, - nhead=num_attention_heads, - dim_feedforward=intermediate_size, - dropout=hidden_dropout_prob, - activation=hidden_act, - attn_dropout=attention_probs_dropout_prob, - act_dropout=hidden_dropout_prob, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)), - bias_attr=None, - topo=topo)) + DecoderLayer( + d_model=hidden_size, + nhead=num_attention_heads, + dim_feedforward=intermediate_size, + dropout=hidden_dropout_prob, + activation=hidden_act, + attn_dropout=attention_probs_dropout_prob, + act_dropout=hidden_dropout_prob, + weight_attr=paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ), + bias_attr=None, + topo=topo, + ) + ) Decoder = TransformerDecoder - self.decoder = Decoder(decoder_layers, - num_hidden_layers, - norm="LayerNorm", - hidden_size=hidden_size, - topo=topo) + self.decoder = Decoder( + decoder_layers, + num_hidden_layers, + norm="LayerNorm", + hidden_size=hidden_size, + topo=topo, + ) self.checkpoints = [] - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - use_cache=False, - cache=None): + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + use_cache=False, + cache=None, + ): self.checkpoints = [] if attention_mask is None: length = paddle.shape(input_ids)[1] # Use bool mask attention_mask = paddle.tensor.tril( - paddle.ones((length, length), - dtype=self.embeddings.word_embeddings.weight.dtype)) + paddle.ones( + (length, length), + dtype=self.embeddings.word_embeddings.weight.dtype, + ) + ) if position_ids is None: past_length = 0 if cache is not None: past_length = paddle.shape(cache[0].k)[-2] - position_ids = paddle.arange(past_length, - paddle.shape(input_ids)[-1] + - past_length, - dtype='int64') + position_ids = paddle.arange( + past_length, + paddle.shape(input_ids)[-1] + past_length, + dtype='int64', + ) position_ids = position_ids.unsqueeze(0) # .expand_as(input_ids) position_ids = paddle.fluid.layers.expand_as( - position_ids, input_ids) - embedding_output = self.embeddings(input_ids=input_ids, - position_ids=position_ids) + position_ids, input_ids + ) + embedding_output = self.embeddings( + input_ids=input_ids, position_ids=position_ids + ) # TODO, use registered buffer - causal_mask = paddle.tensor.triu(paddle.ones( - (paddle.shape(input_ids)[-1], paddle.shape(input_ids)[-1])) * -1e9, - diagonal=1) + causal_mask = paddle.tensor.triu( + paddle.ones( + (paddle.shape(input_ids)[-1], paddle.shape(input_ids)[-1]) + ) + * -1e9, + diagonal=1, + ) if attention_mask is not None: attention_mask = attention_mask + causal_mask @@ -623,11 +686,13 @@ class GPTModel(nn.Layer): # The tensor returned by triu not in static graph. attention_mask.stop_gradient = True - encoder_outputs = self.decoder(embedding_output, - memory=None, - tgt_mask=attention_mask, - use_cache=use_cache, - cache=cache) + encoder_outputs = self.decoder( + embedding_output, + memory=None, + tgt_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) self.checkpoints.extend(self.decoder.checkpoints) return encoder_outputs @@ -649,11 +714,12 @@ class GPTForPretraining(nn.Layer): def parallel_matmul(self, lm_output, logit_weights, parallel_output, topo): if topo is not None and topo.mp_info.size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=None) + lm_output, group=None + ) - logits = paddle.matmul(input_parallel, - logit_weights, - transpose_y=True) + logits = paddle.matmul( + input_parallel, logit_weights, transpose_y=True + ) if parallel_output: return logits @@ -663,24 +729,29 @@ class GPTForPretraining(nn.Layer): logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - masked_positions=None, - use_cache=False, - cache=None): - outputs = self.gpt(input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - use_cache=use_cache, - cache=cache) + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + masked_positions=None, + use_cache=False, + cache=None, + ): + outputs = self.gpt( + input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) if use_cache: encoder_outputs, cached_kvs = outputs[:2] else: encoder_outputs = outputs - logits = self.parallel_matmul(encoder_outputs, self.weight, True, - self.gpt.topo) + logits = self.parallel_matmul( + encoder_outputs, self.weight, True, self.gpt.topo + ) if use_cache: return logits, cached_kvs @@ -699,11 +770,14 @@ class GPTPretrainingCriterion(nn.Layer): if topo is None or topo.mp_info.size == 1: self.loss_func = paddle.nn.CrossEntropyLoss(reduction="none") else: - self.loss_func = paddle.distributed.collective._c_softmax_with_cross_entropy + self.loss_func = ( + paddle.distributed.collective._c_softmax_with_cross_entropy + ) def forward(self, prediction_scores, masked_lm_labels, loss_mask): - masked_lm_loss = self.loss_func(prediction_scores, - masked_lm_labels.unsqueeze(2)) + masked_lm_loss = self.loss_func( + prediction_scores, masked_lm_labels.unsqueeze(2) + ) loss_mask = loss_mask.reshape([-1]) masked_lm_loss = paddle.sum(masked_lm_loss.reshape([-1]) * loss_mask) @@ -712,45 +786,51 @@ class GPTPretrainingCriterion(nn.Layer): def gpt_pretrain_forward(train_program, startup_program): - with static.program_guard(train_program, - startup_program), utils.unique_name.guard(): + with static.program_guard( + train_program, startup_program + ), utils.unique_name.guard(): batch_size = 16 sequence_len = 512 - input_ids = static.data(name="input_ids", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + input_ids = static.data( + name="input_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float64') - labels = static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float64') + dtype='float64', + ) + labels = static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float64' + ) if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input_ids, - process_mesh=_global_process_mesh, - shard_spec=["dp", None]) - - gpt = GPTModel(vocab_size=32768, - hidden_size=768, - num_hidden_layers=2, - num_attention_heads=12, - intermediate_size=4096, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=1024, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - topo=None) + auto.shard_tensor( + input_ids, + process_mesh=_global_process_mesh, + shard_spec=["dp", None], + ) + + gpt = GPTModel( + vocab_size=32768, + hidden_size=768, + num_hidden_layers=2, + num_attention_heads=12, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=1024, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + topo=None, + ) model = GPTForPretraining(gpt) @@ -764,29 +844,26 @@ def gpt_pretrain_forward(train_program, startup_program): class FakeStrategy(object): - def __init__(self): self.amp = False self.recompute = False class FakeFleet(object): - def __init__(self): self.user_defined_optimizer = None self._user_defined_strategy = FakeStrategy() class TestGPTPartitioner(unittest.TestCase): - def test_gpt_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() startup_program = static.Program() @@ -795,24 +872,33 @@ class TestGPTPartitioner(unittest.TestCase): dist_context.process_mesh = _global_process_mesh train_program, startup_program, loss = gpt_pretrain_forward( - train_program, startup_program) + train_program, startup_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) # serial backward pass - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) rank_id = 3 partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) nrank = 4 # col parallel @@ -822,59 +908,96 @@ class TestGPTPartitioner(unittest.TestCase): 'linear_10.w_0', ] self.assertTrue( - check_tensor_split(auto_parallel_main_prog, weights, - complete_train_program, weights, 1, nrank)) + check_tensor_split( + auto_parallel_main_prog, + weights, + complete_train_program, + weights, + 1, + nrank, + ) + ) # row parallel weights = ['word_embeddings', 'linear_9.w_0', 'linear_11.w_0'] self.assertTrue( - check_tensor_split(auto_parallel_main_prog, weights, - complete_train_program, weights, 0, nrank)) + check_tensor_split( + auto_parallel_main_prog, + weights, + complete_train_program, + weights, + 0, + nrank, + ) + ) weights = ['pos_embeddings', 'layer_norm_0.b_0', 'layer_norm_4.w_0'] self.assertTrue( - check_tensor_split(auto_parallel_main_prog, weights, - complete_train_program, weights, 0, 1)) + check_tensor_split( + auto_parallel_main_prog, + weights, + complete_train_program, + weights, + 0, + 1, + ) + ) all_params = sorted( - [param.name for param in startup_program.all_parameters()]) + [param.name for param in startup_program.all_parameters()] + ) allreduce_grads = [ - 'layer_norm_5.tmp_2', 'layer_norm_5.tmp_2', 'layer_norm_5.tmp_2', - 'layer_norm_6.tmp_2', 'layer_norm_7.tmp_2', 'layer_norm_7.tmp_2', - 'layer_norm_7.tmp_2', 'layer_norm_8.tmp_2' + 'layer_norm_5.tmp_2', + 'layer_norm_5.tmp_2', + 'layer_norm_5.tmp_2', + 'layer_norm_6.tmp_2', + 'layer_norm_7.tmp_2', + 'layer_norm_7.tmp_2', + 'layer_norm_7.tmp_2', + 'layer_norm_8.tmp_2', ] process_mesh = _global_process_mesh mp_parallel_axis = 1 dp_parallel_axis = 0 - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, mp_parallel_axis, - 3) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, mp_parallel_axis, 3 + ) mp_ring_id = new_process_group(group_ranks).id - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, dp_parallel_axis, - 3) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, dp_parallel_axis, 3 + ) dp_ring_id = new_process_group(group_ranks).id - tensor_parallel_allreduce_vars = sorted([ - op.desc.output_arg_names()[0].split("@")[0] - for op in auto_parallel_main_prog.global_block().ops - if (op.type == "c_allreduce_sum" and op.attr('op_role') == 1 - and op.desc.attr("ring_id") == mp_ring_id) - ]) - data_parallel_allreduce_vars = sorted([ - op.desc.output_arg_names()[0].split("@")[0] - for op in auto_parallel_main_prog.global_block().ops - if (op.type == "c_allreduce_sum" - and op.desc.attr("ring_id") == dp_ring_id) - ]) + tensor_parallel_allreduce_vars = sorted( + [ + op.desc.output_arg_names()[0].split("@")[0] + for op in auto_parallel_main_prog.global_block().ops + if ( + op.type == "c_allreduce_sum" + and op.attr('op_role') == 1 + and op.desc.attr("ring_id") == mp_ring_id + ) + ] + ) + data_parallel_allreduce_vars = sorted( + [ + op.desc.output_arg_names()[0].split("@")[0] + for op in auto_parallel_main_prog.global_block().ops + if ( + op.type == "c_allreduce_sum" + and op.desc.attr("ring_id") == dp_ring_id + ) + ] + ) self.assertTrue(all_params == data_parallel_allreduce_vars) self.assertTrue(allreduce_grads == tensor_parallel_allreduce_vars) self.assertTrue( - is_valid_completed_program(dist_context, auto_parallel_main_prog)) + is_valid_completed_program(dist_context, auto_parallel_main_prog) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard.py index c604bf299af..e47cc35d6af 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard.py @@ -26,7 +26,10 @@ from paddle.distributed import fleet from paddle.distributed.auto_parallel.parallelizer import AutoParallelizer from paddle.distributed.auto_parallel.partitioner import Partitioner from paddle.distributed.auto_parallel.reshard import Resharder -from paddle.distributed.auto_parallel.process_group import _g_process_group_map, ProcessGroup +from paddle.distributed.auto_parallel.process_group import ( + _g_process_group_map, + ProcessGroup, +) paddle.enable_static() _global_parallel_strategy = None @@ -36,26 +39,26 @@ PP_MESH_1 = None class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -63,10 +66,12 @@ class MLPLayer(nn.Layer): auto.shard_tensor(self.linear0.weight, PP_MESH_0, [None, None]) auto.shard_tensor(self.linear1.weight, PP_MESH_1, [None, None]) else: - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, None]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, None] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, None] + ) out = self.norm(input) out = self.linear0(out) @@ -77,17 +82,18 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "pp": auto.shard_tensor(input, PP_MESH_0, [None, None]) @@ -97,9 +103,11 @@ def mlp_forward(train_program, start_program): else: auto.shard_tensor(input, _global_process_mesh, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -108,13 +116,16 @@ def mlp_forward(train_program, start_program): return loss, train_program, start_program -def get_dist_prog(train_program, - startup_program, - dist_context, - rank_id, - change_process_mesh=False): - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) +def get_dist_prog( + train_program, + startup_program, + dist_context, + rank_id, + change_process_mesh=False, +): + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -124,30 +135,43 @@ def get_dist_prog(train_program, # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) if change_process_mesh: global PP_MESH_1 dist_context.get_tensor_dist_attr_for_program( - train_program.global_block( - ).vars["gelu_0.tmp_0"]).process_mesh = PP_MESH_1 - - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + train_program.global_block().vars["gelu_0.tmp_0"] + ).process_mesh = PP_MESH_1 + + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) - return auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) def check_backward_dist_attr(dist_context, dist_main_prog, op_need_check): @@ -159,16 +183,28 @@ def check_backward_dist_attr(dist_context, dist_main_prog, op_need_check): has_dist_attr = False for var_name in op_need_check.input_arg_names: - if not op_dist_attr.get_input_dims_mapping(var_name) or \ - not dist_context.get_tensor_dist_attr_for_program(vars[var_name]).dims_mapping or \ - not dist_context.get_tensor_dist_attr_for_program(vars[var_name]).process_mesh: + if ( + not op_dist_attr.get_input_dims_mapping(var_name) + or not dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).dims_mapping + or not dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).process_mesh + ): has_dist_attr = False break if has_dist_attr: for var_name in op_need_check.output_arg_names: - if not dist_context.get_tensor_dist_attr_for_program(vars[var_name]).dims_mapping or \ - not dist_context.get_tensor_dist_attr_for_program(vars[var_name]).process_mesh: + if ( + not dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).dims_mapping + or not dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).process_mesh + ): has_dist_attr = False break @@ -184,14 +220,22 @@ def check_send_recv_result(dist_main_prog, rank_id): for idx, op in enumerate(ops): if op.type == "send_v2" and "gelu_0.tmp_0" in op.input_arg_names: send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0@GRAD" in op.output_arg_names[ - 0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0@GRAD" in op.output_arg_names[0] + ): recv_result = True else: for idx, op in enumerate(ops): - if op.type == "send_v2" and "gelu_0.tmp_0@GRAD" in op.input_arg_names: + if ( + op.type == "send_v2" + and "gelu_0.tmp_0@GRAD" in op.input_arg_names + ): send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0" in op.output_arg_names[0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0" in op.output_arg_names[0] + ): recv_result = True return send_result and recv_result @@ -200,8 +244,10 @@ def check_send_recv_result(dist_main_prog, rank_id): def check_initialization(dist_startup_prog, rank_id): if rank_id == 0: need_check_params = [ - "layer_norm_0.b_0", "layer_norm_0.w_0", "linear_0.w_0", - "linear_0.b_0" + "layer_norm_0.b_0", + "layer_norm_0.w_0", + "linear_0.w_0", + "linear_0.b_0", ] else: need_check_params = ['linear_1.w_0', 'linear_1.b_0'] @@ -216,7 +262,10 @@ def check_initialization(dist_startup_prog, rank_id): def check_initialization_for_dp(dist_startup_prog): need_check_params = [ - "layer_norm_0.b_0", "layer_norm_0.w_0", "linear_0.w_0", "linear_0.b_0" + "layer_norm_0.b_0", + "layer_norm_0.w_0", + "linear_0.w_0", + "linear_0.b_0", ] + ['linear_1.w_0', 'linear_1.b_0'] params = [] for var_name, var in dist_startup_prog.global_block().vars.items(): @@ -227,12 +276,14 @@ def check_initialization_for_dp(dist_startup_prog): if op.type == "c_broadcast": broadcast_varnames.append(op.output_arg_names[0]) - return sorted(params) == sorted(need_check_params) == sorted( - broadcast_varnames) + return ( + sorted(params) + == sorted(need_check_params) + == sorted(broadcast_varnames) + ) class TestMLPReshard(unittest.TestCase): - def test_complete_backward_annotation(self): global _global_process_mesh _global_process_mesh = auto.ProcessMesh(mesh=[0, 1]) @@ -242,7 +293,8 @@ class TestMLPReshard(unittest.TestCase): dist_context = DistributedContext() rank_id = 0 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, 0) + train_program, startup_program, dist_context, 0 + ) op_need_check = None for op in dist_main_prog.global_block().ops: @@ -252,8 +304,10 @@ class TestMLPReshard(unittest.TestCase): # grad op should have dist attr self.assertTrue( - check_backward_dist_attr(dist_context, dist_main_prog, - op_need_check)) + check_backward_dist_attr( + dist_context, dist_main_prog, op_need_check + ) + ) # clear _g_process_group_map _g_process_group_map.clear() @@ -274,9 +328,15 @@ class TestMLPReshard(unittest.TestCase): dist_context = DistributedContext() rank_id = 1 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + train_program, startup_program, dist_context, rank_id + ) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # check send and recv result @@ -303,9 +363,15 @@ class TestMLPReshard(unittest.TestCase): dist_context = DistributedContext() rank_id = 1 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id, True) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + train_program, startup_program, dist_context, rank_id, True + ) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # check send and recv result self.assertTrue(check_send_recv_result(dist_main_prog, rank_id)) @@ -326,9 +392,15 @@ class TestMLPReshard(unittest.TestCase): dist_context = DistributedContext() rank_id = 0 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + train_program, startup_program, dist_context, rank_id + ) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # send and recv should not exist in dp scene. diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_dpmppp.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_dpmppp.py index 4e4991579e9..502d5dae172 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_dpmppp.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_dpmppp.py @@ -31,33 +31,34 @@ from paddle.distributed.auto_parallel.cluster import Cluster paddle.enable_static() _global_parallel_strategy = "dp_mp_pp" -_global_process_mesh = auto.ProcessMesh([[[0, 1], [4, 5]], [[2, 3], [6, 7]]], - dim_names=["x", "y", "z"]) +_global_process_mesh = auto.ProcessMesh( + [[[0, 1], [4, 5]], [[2, 3], [6, 7]]], dim_names=["x", "y", "z"] +) PP_MESH_0 = auto.ProcessMesh([[0, 1], [4, 5]], dim_names=["x", "y"]) PP_MESH_1 = auto.ProcessMesh([[2, 3], [6, 7]], dim_names=["x", "y"]) class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -68,8 +69,9 @@ class MLPLayer(nn.Layer): out = self.linear0(out) out = F.gelu(out, approximate=True) out = self.linear1(out) - param = paddle.fluid.layers.create_parameter([1024, 4096], - paddle.float32) + param = paddle.fluid.layers.create_parameter( + [1024, 4096], paddle.float32 + ) auto.shard_tensor(param, PP_MESH_1, [None, "y"]) out = paddle.fluid.layers.mul(out, param) @@ -77,24 +79,27 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) auto.shard_tensor(input, PP_MESH_0, ["x", None]) auto.shard_tensor(label, PP_MESH_1, ["x", None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -106,8 +111,9 @@ def mlp_forward(train_program, start_program): def get_dist_prog(train_program, startup_program, dist_context, rank_id): global _global_process_mesh dist_context.process_mesh = _global_process_mesh - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -117,24 +123,37 @@ def get_dist_prog(train_program, startup_program, dist_context, rank_id): # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) - return auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) def check_send_recv_result(dist_main_prog, rank_id): @@ -145,14 +164,22 @@ def check_send_recv_result(dist_main_prog, rank_id): for idx, op in enumerate(ops): if op.type == "send_v2" and "gelu_0.tmp_0" in op.input_arg_names: send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0@GRAD" in op.output_arg_names[ - 0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0@GRAD" in op.output_arg_names[0] + ): recv_result = True else: for idx, op in enumerate(ops): - if op.type == "send_v2" and "gelu_0.tmp_0@GRAD" in op.input_arg_names: + if ( + op.type == "send_v2" + and "gelu_0.tmp_0@GRAD" in op.input_arg_names + ): send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0" in op.output_arg_names[0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0" in op.output_arg_names[0] + ): recv_result = True return send_result and recv_result @@ -168,14 +195,14 @@ def check_initialization_for_dpmppp(dist_startup_prog): class TestMLPReshard(unittest.TestCase): - def test_mlp_dpmppp(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() rank_id = 2 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) + train_program, startup_program, dist_context, rank_id + ) # test estimator cluster = Cluster() @@ -183,16 +210,23 @@ class TestMLPReshard(unittest.TestCase): cost_estimator = CostEstimator(train_program, cluster) global_cost = cost_estimator.estimate(dist_context) max_memory = cost_estimator._estimate_max_memory_by_dist_op( - dist_context) + dist_context + ) # test cache global_cost = cost_estimator.estimate(dist_context) max_memory = cost_estimator._estimate_max_memory_by_dist_op( - dist_context) + dist_context + ) assert global_cost.time > 0 assert max_memory > 0 - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # print_program_with_dist_attr(dist_main_prog, dist_context) # check send and recv result diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_mppp.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_mppp.py index 21efd284b5d..62ebc76c6fa 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_mppp.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_mppp.py @@ -37,37 +37,40 @@ PP_MESH_1 = auto.ProcessMesh([2, 3], dim_names=["x"]) class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None self.word_embeddings = nn.Embedding( hidden_size, hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) - - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) - self.linear2 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) + + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) + self.linear2 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) def forward(self, input): auto.shard_tensor(self.word_embeddings.weight, PP_MESH_0, ["x", None]) @@ -76,8 +79,9 @@ class MLPLayer(nn.Layer): auto.shard_tensor(self.linear2.weight, PP_MESH_1, ["x", None]) w_out = self.word_embeddings(input) out = self.linear0(w_out) - param = paddle.fluid.layers.create_parameter([4096, 4096], - paddle.float32) + param = paddle.fluid.layers.create_parameter( + [4096, 4096], paddle.float32 + ) auto.shard_tensor(param, PP_MESH_0, ["x", None]) out = paddle.fluid.layers.mul(out, param) gelu_out = F.gelu(out, approximate=True) @@ -89,22 +93,25 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 input = static.data(name="input", shape=[batch_size], dtype='int32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) auto.shard_tensor(input, PP_MESH_0, [None]) auto.shard_tensor(label, PP_MESH_1, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -116,8 +123,9 @@ def mlp_forward(train_program, start_program): def get_dist_prog(train_program, startup_program, dist_context, rank_id): global _global_process_mesh dist_context.process_mesh = _global_process_mesh - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -127,23 +135,36 @@ def get_dist_prog(train_program, startup_program, dist_context, rank_id): # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) - return auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) def check_send_recv_result(dist_main_prog, rank_id): @@ -154,15 +175,22 @@ def check_send_recv_result(dist_main_prog, rank_id): for idx, op in enumerate(ops): if op.type == "send_v2" and "gelu_0.tmp_0" in op.input_arg_names: send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0@GRAD" in op.output_arg_names[ - 0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0@GRAD" in op.output_arg_names[0] + ): recv_result = True else: for idx, op in enumerate(ops): - if op.type == "send_v2" and "gelu_0.tmp_0@GRAD" in op.input_arg_names[ - 0]: + if ( + op.type == "send_v2" + and "gelu_0.tmp_0@GRAD" in op.input_arg_names[0] + ): send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0" in op.output_arg_names[0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0" in op.output_arg_names[0] + ): recv_result = True return send_result and recv_result @@ -196,23 +224,29 @@ def check_allgather(dist_main_program): class TestMLPReshard(unittest.TestCase): - def test_mlp_mppp(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() rank_id = 2 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + train_program, startup_program, dist_context, rank_id + ) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # check send and recv result self.assertTrue(check_send_recv_result(dist_main_prog, rank_id)) # parameter which not been sliced should be the same in the mp scene self.assertTrue( - check_initialization_for_mppp(dist_startup_prog, rank_id)) + check_initialization_for_mppp(dist_startup_prog, rank_id) + ) def test_allgather(self): train_program = paddle.static.Program() @@ -224,8 +258,9 @@ class TestMLPReshard(unittest.TestCase): w = paddle.static.data(name="w", shape=[4, 4], dtype='float32') w = auto.shard_tensor(w, process_mesh, [None, None]) - y = paddle.distributed.shard_op(paddle.matmul, process_mesh, - [[None, None], [None, None]])(x, w) + y = paddle.distributed.shard_op( + paddle.matmul, process_mesh, [[None, None], [None, None]] + )(x, w) rank_id = 0 dist_context = DistributedContext() @@ -233,10 +268,14 @@ class TestMLPReshard(unittest.TestCase): partitioner = Partitioner(dist_context, rank_id) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - partitioned_main_prog, partitioned_startup_prog, partitioned_params_grads = partitioner.partition( - complete_train_program, startup_program, []) + ( + partitioned_main_prog, + partitioned_startup_prog, + partitioned_params_grads, + ) = partitioner.partition(complete_train_program, startup_program, []) # test estimator cluster = Cluster() @@ -244,16 +283,23 @@ class TestMLPReshard(unittest.TestCase): cost_estimator = CostEstimator(train_program, cluster) global_cost = cost_estimator.estimate(dist_context) max_memory = cost_estimator._estimate_max_memory_by_dist_op( - dist_context) + dist_context + ) # test cache global_cost = cost_estimator.estimate(dist_context) max_memory = cost_estimator._estimate_max_memory_by_dist_op( - dist_context) + dist_context + ) assert global_cost.time > 0 assert max_memory > 0 - resharder = Resharder(partitioned_main_prog, partitioned_startup_prog, - rank_id, dist_context, partitioned_params_grads) + resharder = Resharder( + partitioned_main_prog, + partitioned_startup_prog, + rank_id, + dist_context, + partitioned_params_grads, + ) resharder.reshard() # the x should not be slice self.assertTrue(check_allgather(partitioned_main_prog)) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_serial.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_serial.py index ff91bfe19f5..697998b9de0 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_serial.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_serial.py @@ -15,6 +15,7 @@ import unittest import os + if os.getenv("CUDA_VISIBLE_DEVICES", None) is None: os.environ["CUDA_VISIBLE_DEVICES"] = '0' @@ -24,7 +25,9 @@ import paddle.static as static import paddle.nn.functional as F import paddle.utils as utils from paddle.distributed.fleet import auto -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) from paddle.distributed import fleet paddle.enable_static() @@ -33,26 +36,26 @@ _global_process_mesh = None class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -60,10 +63,12 @@ class MLPLayer(nn.Layer): auto.shard_tensor(self.linear0.weight, PP_MESH_0, [None, None]) auto.shard_tensor(self.linear1.weight, PP_MESH_1, [None, None]) else: - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, None]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, None] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, None] + ) out = self.norm(input) out = self.linear0(out) @@ -74,17 +79,18 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "pp": auto.shard_tensor(input, PP_MESH_0, [None, None]) @@ -94,9 +100,11 @@ def mlp_forward(train_program, start_program): else: auto.shard_tensor(input, _global_process_mesh, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -105,8 +113,9 @@ def mlp_forward(train_program, start_program): return loss, train_program, start_program -def get_dist_prog_with_parallelizer(train_program, startup_program, - dist_context): +def get_dist_prog_with_parallelizer( + train_program, startup_program, dist_context +): global _global_process_mesh dist_strategy = fleet.DistributedStrategy() @@ -118,18 +127,25 @@ def get_dist_prog_with_parallelizer(train_program, startup_program, dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) - - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) + + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, startup_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, startup_program) return distributed_main_program, distributed_startup_program @@ -142,21 +158,28 @@ def check_send_recv_result(dist_main_prog, rank_id): for idx, op in enumerate(ops): if op.type == "send_v2" and "gelu_0.tmp_0" in op.input_arg_names: send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0@GRAD" in op.output_arg_names[ - 0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0@GRAD" in op.output_arg_names[0] + ): recv_result = True else: for idx, op in enumerate(ops): - if op.type == "send_v2" and "gelu_0.tmp_0@GRAD" in op.input_arg_names: + if ( + op.type == "send_v2" + and "gelu_0.tmp_0@GRAD" in op.input_arg_names + ): send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0" in op.output_arg_names[0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0" in op.output_arg_names[0] + ): recv_result = True return send_result and recv_result class TestMLPReshard(unittest.TestCase): - def test_mlp_serial(self): global _global_parallel_strategy _global_parallel_strategy = None @@ -168,7 +191,8 @@ class TestMLPReshard(unittest.TestCase): dist_context = get_default_distributed_context() rank_id = 0 dist_main_prog, dist_startup_prog = get_dist_prog_with_parallelizer( - train_program, startup_program, dist_context) + train_program, startup_program, dist_context + ) # send and recv should not exist in serial scene. self.assertFalse(check_send_recv_result(dist_main_prog, rank_id)) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_save_load.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_save_load.py index b2a478b365b..7b9cf4c0004 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_save_load.py @@ -18,7 +18,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestAutoParallelSaveLoad(TestMultipleGpus): - def test_auto_parallel_save_load(self): self.run_mnist_2gpu('auto_parallel_save_load.py') diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_searcher.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_searcher.py index cb5e4e00e9a..98880667e34 100755 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_searcher.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_searcher.py @@ -22,35 +22,43 @@ import paddle.utils as utils from paddle.distributed.fleet import auto from paddle.distributed.auto_parallel.planner import PlanSpace from paddle.distributed.auto_parallel.dist_context import DistributedContext -from paddle.distributed.auto_parallel.dist_attribute import TensorDistributedAttribute -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute -from paddle.distributed.auto_parallel.utils import update_op_dims_mapping_by_default_dist_impl -from paddle.distributed.auto_parallel.utils import update_op_dims_mapping_by_elementwise_like_dist_impl +from paddle.distributed.auto_parallel.dist_attribute import ( + TensorDistributedAttribute, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) +from paddle.distributed.auto_parallel.utils import ( + update_op_dims_mapping_by_default_dist_impl, +) +from paddle.distributed.auto_parallel.utils import ( + update_op_dims_mapping_by_elementwise_like_dist_impl, +) paddle.enable_static() class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -64,21 +72,24 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) loss_func = paddle.nn.CrossEntropyLoss(reduction="none") - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = loss_func(predict, label) @@ -98,18 +109,22 @@ def set_default_dist_attr(program, dist_context, process_mesh): tensor_dist_attr.process_mesh = process_mesh tensor_dist_attr.dims_mapping = [-1 for i in vars[var_name].shape] dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) - op_dist_attr.set_input_dims_mapping(var_name, - tensor_dist_attr.dims_mapping) + vars[var_name], tensor_dist_attr + ) + op_dist_attr.set_input_dims_mapping( + var_name, tensor_dist_attr.dims_mapping + ) for var_name in op.output_arg_names: tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = process_mesh tensor_dist_attr.dims_mapping = [-1 for i in vars[var_name].shape] dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) - op_dist_attr.set_output_dims_mapping(var_name, - tensor_dist_attr.dims_mapping) + vars[var_name], tensor_dist_attr + ) + op_dist_attr.set_output_dims_mapping( + var_name, tensor_dist_attr.dims_mapping + ) dist_context.set_op_dist_attr_for_program(op, op_dist_attr) dist_context.add_process_mesh(process_mesh) @@ -123,73 +138,111 @@ def check_process_meshes(processes): def check_pipeline_enumerater(program, process_mesh_topology): - valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh = PlanSpace.enum_valid_dist_attr_for_program( - program, process_mesh_topology, True) - if valid_dist_attr_dict and len( - pipeline_process_meshes) > 1 and not global_process_mesh: + ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) = PlanSpace.enum_valid_dist_attr_for_program( + program, process_mesh_topology, True + ) + if ( + valid_dist_attr_dict + and len(pipeline_process_meshes) > 1 + and not global_process_mesh + ): return True return False def check_nonpipeline_enumerater(program, process_mesh_topology): - valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh = PlanSpace.enum_valid_dist_attr_for_program( - program, process_mesh_topology, False) - if valid_dist_attr_dict and not pipeline_process_meshes and global_process_mesh: + ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) = PlanSpace.enum_valid_dist_attr_for_program( + program, process_mesh_topology, False + ) + if ( + valid_dist_attr_dict + and not pipeline_process_meshes + and global_process_mesh + ): return True return False class TestMLPSearcher(unittest.TestCase): - def test_update(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() - _, train_program, startup_program = mlp_forward(train_program, - startup_program) + _, train_program, startup_program = mlp_forward( + train_program, startup_program + ) global_process_mesh = auto.ProcessMesh(mesh=[0, 1]) dist_context = DistributedContext() set_default_dist_attr(train_program, dist_context, global_process_mesh) ops = train_program.global_block().ops vars = train_program.global_block().vars - from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container - from paddle.distributed.auto_parallel.operators.common import is_elementwise_op + from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, + ) + from paddle.distributed.auto_parallel.operators.common import ( + is_elementwise_op, + ) from paddle.distributed.auto_parallel.dist_op import DistributedOperator for op in ops: dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + op.type + ) if dist_op_impl_container is None: op_dist_attr = dist_context.get_op_dist_attr_for_program(op) dist_op = DistributedOperator(op, op_dist_attr) if is_elementwise_op(op.type): - changed = update_op_dims_mapping_by_elementwise_like_dist_impl( - dist_op) + changed = ( + update_op_dims_mapping_by_elementwise_like_dist_impl( + dist_op + ) + ) self.assertFalse(changed) dist_op.dist_attr.set_output_dims_mapping( - op.output_arg_names[0], [0] + [ - -1 for i in range( - 1, len(vars[op.output_arg_names[0]].shape)) - ]) + op.output_arg_names[0], + [0] + + [ + -1 + for i in range( + 1, len(vars[op.output_arg_names[0]].shape) + ) + ], + ) try: changed = update_op_dims_mapping_by_elementwise_like_dist_impl( - dist_op) + dist_op + ) except: continue self.assertTrue(changed) else: changed = update_op_dims_mapping_by_default_dist_impl( - dist_op) + dist_op + ) self.assertFalse(changed) dist_op.dist_attr.set_output_dims_mapping( - op.output_arg_names[0], [0] + [ - -1 for i in range( - 1, len(vars[op.output_arg_names[0]].shape)) - ]) + op.output_arg_names[0], + [0] + + [ + -1 + for i in range( + 1, len(vars[op.output_arg_names[0]].shape) + ) + ], + ) try: changed = update_op_dims_mapping_by_default_dist_impl( - dist_op) + dist_op + ) except: continue self.assertTrue(changed) @@ -200,13 +253,16 @@ class TestMLPSearcher(unittest.TestCase): train_program = paddle.static.Program() startup_program = paddle.static.Program() - _, train_program, startup_program = mlp_forward(train_program, - startup_program) + _, train_program, startup_program = mlp_forward( + train_program, startup_program + ) process_mesh_topology = [4] self.assertTrue( - check_pipeline_enumerater(train_program, process_mesh_topology)) + check_pipeline_enumerater(train_program, process_mesh_topology) + ) self.assertTrue( - check_nonpipeline_enumerater(train_program, process_mesh_topology)) + check_nonpipeline_enumerater(train_program, process_mesh_topology) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_auto_search_dist_matmul_op.py b/python/paddle/fluid/tests/unittests/test_auto_search_dist_matmul_op.py index fd84c9e6f0c..0962b2e8d5b 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_search_dist_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_auto_search_dist_matmul_op.py @@ -20,8 +20,12 @@ import paddle.nn.functional as F import paddle.utils as utils import paddle.fluid.core as core from paddle.fluid import layers -from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute +from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) from paddle.distributed.auto_parallel.dist_op import DistributedOperator paddle.enable_static() @@ -29,26 +33,26 @@ device = "gpu" if core.is_compiled_with_cuda() else "cpu" class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -61,8 +65,9 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sqrt_hidden_size = 32 @@ -75,16 +80,20 @@ def mlp_forward(train_program, start_program): input = embedding(input) input = paddle.reshape(input, [hidden_size, batch_size]) input = paddle.transpose(input, perm=[1, 0]) - matmulinput = static.data(name="matmulinput", - shape=[hidden_size, hidden_size], - dtype='float32') + matmulinput = static.data( + name="matmulinput", + shape=[hidden_size, hidden_size], + dtype='float32', + ) input = layers.matmul(x=input, y=matmulinput) - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -95,37 +104,38 @@ def mlp_forward(train_program, start_program): class TestCompatible(unittest.TestCase): - def test_matmulv2_matmul_2_compatible(self): valid_op_dist_attr_list = [] program = paddle.static.Program() startup_program = paddle.static.Program() loss, program, start_program = mlp_forward(program, startup_program) - with static.program_guard(program, - start_program), utils.unique_name.guard(): - matmulx3 = static.data(name="matmulx3", - shape=[6, 2, 6], - dtype='float32') - matmuly3 = static.data(name="matmuly3", - shape=[6, 6], - dtype='float32') + with static.program_guard( + program, start_program + ), utils.unique_name.guard(): + matmulx3 = static.data( + name="matmulx3", shape=[6, 2, 6], dtype='float32' + ) + matmuly3 = static.data( + name="matmuly3", shape=[6, 6], dtype='float32' + ) output1 = paddle.matmul(x=matmulx3, y=matmuly3) output_1 = layers.matmul(x=matmulx3, y=matmuly3) - matmulx4 = static.data(name="matmulx4", - shape=[6, 6, 2, 6], - dtype='float32') - matmuly4 = static.data(name="matmuly4", - shape=[6, 6, 6, 6], - dtype='float32') + matmulx4 = static.data( + name="matmulx4", shape=[6, 6, 2, 6], dtype='float32' + ) + matmuly4 = static.data( + name="matmuly4", shape=[6, 6, 6, 6], dtype='float32' + ) output2 = paddle.matmul(x=matmulx4, y=matmuly4) output_2 = layers.matmul(x=matmulx4, y=matmuly4) ops = program.global_block().ops vars = program.global_block().vars for idx, op in enumerate(ops): if op.type == 'matmul_v2' or op.type == 'matmul': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() X = op.input_arg_names[0] @@ -135,98 +145,157 @@ class TestCompatible(unittest.TestCase): op_dist_attr.set_input_dims_mapping(X, [-1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1]) - self.assertTrue(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 3 and len(vars[Y].shape) == 2: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1]) - self.assertTrue(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [1, -1, -1]) op_dist_attr.set_input_dims_mapping(X, [-1, -1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 4 and len(vars[Y].shape) == 4: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1, -1]) - self.assertTrue(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [0, -1, -1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [0, -1, -1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, 0, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 0, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 0, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) def test_matmulv2_matmul_1_compatible(self): valid_op_dist_attr_list = [] program = paddle.static.Program() startup_program = paddle.static.Program() loss, program, start_program = mlp_forward(program, startup_program) - with static.program_guard(program, - start_program), utils.unique_name.guard(): - matmulx3 = static.data(name="matmulx3", - shape=[6, 2, 6], - dtype='float32') - matmuly3 = static.data(name="matmuly3", - shape=[6, 6], - dtype='float32') + with static.program_guard( + program, start_program + ), utils.unique_name.guard(): + matmulx3 = static.data( + name="matmulx3", shape=[6, 2, 6], dtype='float32' + ) + matmuly3 = static.data( + name="matmuly3", shape=[6, 6], dtype='float32' + ) output1 = paddle.matmul(x=matmulx3, y=matmuly3) output_1 = layers.matmul(x=matmulx3, y=matmuly3) - matmulx4 = static.data(name="matmulx4", - shape=[6, 6, 6, 6], - dtype='float32') - matmuly4 = static.data(name="matmuly4", - shape=[6, 6, 6, 6], - dtype='float32') + matmulx4 = static.data( + name="matmulx4", shape=[6, 6, 6, 6], dtype='float32' + ) + matmuly4 = static.data( + name="matmuly4", shape=[6, 6, 6, 6], dtype='float32' + ) output2 = paddle.matmul(x=matmulx4, y=matmuly4) output_2 = layers.matmul(x=matmulx4, y=matmuly4) ops = program.global_block().ops vars = program.global_block().vars for idx, op in enumerate(ops): if op.type == 'matmul_v2' or op.type == 'matmul': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() X = op.input_arg_names[0] @@ -238,86 +307,133 @@ class TestCompatible(unittest.TestCase): op_dist_attr.set_output_dims_mapping(out, [-1, -1]) dist_op = DistributedOperator(op, op_dist_attr) op_dist_attr.set_output_dims_mapping(out, [1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 3 and len(vars[Y].shape) == 2: op_dist_attr.set_input_dims_mapping(X, [-1, -1, 1]) op_dist_attr.set_input_dims_mapping(Y, [1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1]) - self.assertTrue(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [1, -1, 1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(out, [-1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 0, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 4 and len(vars[Y].shape) == 4: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1, 1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, 1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1, -1]) - self.assertTrue(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [0, -1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [0, -1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, 0, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 0, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, 1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 0, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) def test_matmulv2_matmul_0_compatible(self): valid_op_dist_attr_list = [] program = paddle.static.Program() startup_program = paddle.static.Program() loss, program, start_program = mlp_forward(program, startup_program) - with static.program_guard(program, - start_program), utils.unique_name.guard(): - matmulx3 = static.data(name="matmulx3", - shape=[6, 2, 6], - dtype='float32') - matmuly3 = static.data(name="matmuly3", - shape=[6, 6], - dtype='float32') + with static.program_guard( + program, start_program + ), utils.unique_name.guard(): + matmulx3 = static.data( + name="matmulx3", shape=[6, 2, 6], dtype='float32' + ) + matmuly3 = static.data( + name="matmuly3", shape=[6, 6], dtype='float32' + ) output1 = paddle.matmul(x=matmulx3, y=matmuly3) output_1 = layers.matmul(x=matmulx3, y=matmuly3) - matmulx4 = static.data(name="matmulx4", - shape=[6, 6, 2, 6], - dtype='float32') - matmuly4 = static.data(name="matmuly4", - shape=[6, 6, 6, 6], - dtype='float32') + matmulx4 = static.data( + name="matmulx4", shape=[6, 6, 2, 6], dtype='float32' + ) + matmuly4 = static.data( + name="matmuly4", shape=[6, 6, 6, 6], dtype='float32' + ) output2 = paddle.matmul(x=matmulx4, y=matmuly4) output_2 = layers.matmul(x=matmulx4, y=matmuly4) ops = program.global_block().ops vars = program.global_block().vars for idx, op in enumerate(ops): if op.type == 'matmul_v2' or op.type == 'matmul': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() X = op.input_arg_names[0] @@ -327,81 +443,150 @@ class TestCompatible(unittest.TestCase): op_dist_attr.set_input_dims_mapping(X, [-1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, 1]) op_dist_attr.set_output_dims_mapping(out, [-1, 1]) - self.assertTrue(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [0, 0]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [0, -1]) op_dist_attr.set_output_dims_mapping(out, [1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 3 and len(vars[Y].shape) == 2: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, 1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 1]) - self.assertTrue(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 0, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [1, -1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 4 and len(vars[Y].shape) == 4: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, 1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1, 1]) - self.assertTrue(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [0, -1, -1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1, -1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, -1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [0, -1, -1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 1, 1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_auto_search_dist_op.py b/python/paddle/fluid/tests/unittests/test_auto_search_dist_op.py index 8c80a7ae808..16992603896 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_search_dist_op.py +++ b/python/paddle/fluid/tests/unittests/test_auto_search_dist_op.py @@ -20,8 +20,12 @@ import paddle.nn.functional as F import paddle.utils as utils import paddle.fluid.core as core from paddle.fluid import layers -from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute +from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) from paddle.distributed.auto_parallel.dist_op import DistributedOperator paddle.enable_static() @@ -29,26 +33,26 @@ device = "gpu" if core.is_compiled_with_cuda() else "cpu" class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -61,8 +65,9 @@ class MLPLayer(nn.Layer): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sqrt_hidden_size = 32 @@ -75,16 +80,20 @@ def mlp_forward(train_program, start_program): input = embedding(input) input = paddle.reshape(input, [hidden_size, batch_size]) input = paddle.transpose(input, perm=[1, 0]) - matmulinput = static.data(name="matmulinput", - shape=[hidden_size, hidden_size], - dtype='float32') + matmulinput = static.data( + name="matmulinput", + shape=[hidden_size, hidden_size], + dtype='float32', + ) input = layers.matmul(x=input, y=matmulinput) - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -95,7 +104,6 @@ def mlp_forward(train_program, start_program): class TestCompatible(unittest.TestCase): - def test_reshape_remove_compatible(self): valid_op_dist_attr_list = [] program = paddle.static.Program() @@ -104,55 +112,95 @@ class TestCompatible(unittest.TestCase): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'reshape2': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1, -1, -1]) - self.assertTrue(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1, -1, 1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, -1, -1, 1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1, 1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [0, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, -1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, 0, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) - - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1, -1, -1] + ) + self.assertTrue( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1, -1, 1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, -1, -1, 1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 1, -1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1, 1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [1, -1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [0, -1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, -1, -1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, 0, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) + + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) def test_reshape_add_compatible(self): valid_op_dist_attr_list = [] @@ -162,55 +210,91 @@ class TestCompatible(unittest.TestCase): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'reshape2': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], [-1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1]) - self.assertTrue(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 0]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1] + ) + self.assertTrue( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 0] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], [-1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [0, -1]) - - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [0, -1] + ) + + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [1, -1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [1, 1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1, 1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], [-1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [0, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [0, -1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) def test_transpose_compatible(self): valid_op_dist_attr_list = [] @@ -220,47 +304,58 @@ class TestCompatible(unittest.TestCase): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'transpose2': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertTrue(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 0, 0]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 0, 0] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, 0, 0]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, 0, 0] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 0, 0]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 0, 0] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, -1, -1]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, -1, -1] + ) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, 1, 1]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, 1, 1] + ) self.assertFalse(impls[0].is_auto_compatible(dist_op)) def test_softmax_compatible(self): @@ -271,23 +366,28 @@ class TestCompatible(unittest.TestCase): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'softmax': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertTrue(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [1]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, 1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, 1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) op.all_attrs()['axis'] = 2 @@ -301,52 +401,66 @@ class TestCompatible(unittest.TestCase): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'c_embedding' or op.type == 'lookup_table_v2': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[1], - [1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[1], [1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertTrue(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, 0, 0]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, 0, 0] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, 1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, 1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[1], - [-1, 1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[1], [-1, 1] + ) dist_op = DistributedOperator(op, op_dist_attr) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[1], - [1, 1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[1], [1, 1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, 1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, 1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[1], - [1, 1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[1], [1, 1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [1, 1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [1, 1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) diff --git a/python/paddle/fluid/tests/unittests/test_avoid_twice_initialization.py b/python/paddle/fluid/tests/unittests/test_avoid_twice_initialization.py index 87a074c2b50..43a2d08731d 100644 --- a/python/paddle/fluid/tests/unittests/test_avoid_twice_initialization.py +++ b/python/paddle/fluid/tests/unittests/test_avoid_twice_initialization.py @@ -17,7 +17,6 @@ import paddle.fluid as fluid class TestAvoidTwiceInitialization(unittest.TestCase): - def test_avoid_twice_initialization(self): cur_program = fluid.Program() cur_block = cur_program.current_block() @@ -25,24 +24,26 @@ class TestAvoidTwiceInitialization(unittest.TestCase): initializer=fluid.initializer.Constant(value=0.01), shape=[2, 2], dtype='float32', - name='var_a') - cur_block.append_op(type="c_broadcast", - inputs={"X": [var]}, - outputs={"Out": [var]}, - attrs={ - 'root': 0, - 'ring_id': 0, - 'use_calc_stream': False - }) - cur_block.append_op(type="c_sync_comm_stream", - inputs={'X': [var]}, - outputs={'Out': [var]}, - attrs={'ring_id': 0}) + name='var_a', + ) + cur_block.append_op( + type="c_broadcast", + inputs={"X": [var]}, + outputs={"Out": [var]}, + attrs={'root': 0, 'ring_id': 0, 'use_calc_stream': False}, + ) + cur_block.append_op( + type="c_sync_comm_stream", + inputs={'X': [var]}, + outputs={'Out': [var]}, + attrs={'ring_id': 0}, + ) var2 = cur_block.create_parameter( initializer=fluid.initializer.Constant(value=0.01), shape=[2, 2], dtype='float32', - name='var_a') + name='var_a', + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_backward.py b/python/paddle/fluid/tests/unittests/test_backward.py index 28c42b49d33..a186d57a1fe 100644 --- a/python/paddle/fluid/tests/unittests/test_backward.py +++ b/python/paddle/fluid/tests/unittests/test_backward.py @@ -56,8 +56,11 @@ class TestBackward(unittest.TestCase): """ def _check_all(self, net): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) main = fluid.Program() @@ -79,21 +82,28 @@ class TestBackward(unittest.TestCase): no_grad_dict = self._check_stop_gradient(main_program) # 1.2 find_op_path op_path, block_no_grad_set = self._check_op_path( - main_program.block(global_block_idx), [loss], [], no_grad_dict) + main_program.block(global_block_idx), [loss], [], no_grad_dict + ) # 1.3 _find_no_grad_vars no_grad_vars = self._check_find_no_grad_vars( - main_program.block(global_block_idx), op_path, [loss], - block_no_grad_set) + main_program.block(global_block_idx), + op_path, + [loss], + block_no_grad_set, + ) # update no_grad_dict block_no_grad_set.update(no_grad_vars) no_grad_dict[global_block_idx].update( - list(map(fluid.backward._append_grad_suffix_, block_no_grad_set))) + list(map(fluid.backward._append_grad_suffix_, block_no_grad_set)) + ) def _check_params_grad(self, loss, parameter_list=None, no_grad_set=None): - params_grads = fluid.backward.append_backward(loss, parameter_list, - no_grad_set) + params_grads = fluid.backward.append_backward( + loss, parameter_list, no_grad_set + ) params_names = set( - [param_var.name for (param_var, grad_var) in params_grads]) + [param_var.name for (param_var, grad_var) in params_grads] + ) self.assertSetEqual(params_names, self.net.params_names) return params_grads @@ -101,8 +111,10 @@ class TestBackward(unittest.TestCase): def _check_stop_gradient(self, program): no_grad_dict = fluid.backward._get_stop_gradients_(program) if no_grad_dict is not None and isinstance(no_grad_dict, dict): - self.assertSetEqual(no_grad_dict[self.global_block_idx], - self.net.stop_gradient_grad_vars) + self.assertSetEqual( + no_grad_dict[self.global_block_idx], + self.net.stop_gradient_grad_vars, + ) return no_grad_dict @@ -111,26 +123,35 @@ class TestBackward(unittest.TestCase): block_no_grad_set = None else: block_no_grad_set = set( - map(fluid.backward._strip_grad_suffix_, - no_grad_dict[self.global_block_idx])) - op_path = fluid.backward._find_op_path_(root_block, outputs, inputs, - block_no_grad_set) + map( + fluid.backward._strip_grad_suffix_, + no_grad_dict[self.global_block_idx], + ) + ) + op_path = fluid.backward._find_op_path_( + root_block, outputs, inputs, block_no_grad_set + ) op_types = [op.type for op in op_path] self.assertListEqual(op_types, self.net.op_path) return op_path, block_no_grad_set - def _check_find_no_grad_vars(self, root_block, op_path, targets, - block_no_grad_set): + def _check_find_no_grad_vars( + self, root_block, op_path, targets, block_no_grad_set + ): no_grad_vars = fluid.backward._find_no_grad_vars( - root_block, op_path, targets, block_no_grad_set) + root_block, op_path, targets, block_no_grad_set + ) self.assertSetEqual(no_grad_vars, self.net.no_grad_vars) return no_grad_vars def _check_error_param_list(self, net, parameter_list): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) main = fluid.Program() @@ -144,8 +165,11 @@ class TestBackward(unittest.TestCase): exe.run(feed=net.init_data()) def _check_error_no_grad_set(self, net, no_grad_set): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) main = fluid.Program() @@ -160,13 +184,16 @@ class TestBackward(unittest.TestCase): class SimpleNet(BackwardNet): - def __init__(self): super(SimpleNet, self).__init__() - self.stop_gradient_grad_vars = set([ - u'x_no_grad@GRAD', u'x2_no_grad@GRAD', u'x3_no_grad@GRAD', - u'label_no_grad@GRAD' - ]) + self.stop_gradient_grad_vars = set( + [ + u'x_no_grad@GRAD', + u'x2_no_grad@GRAD', + u'x3_no_grad@GRAD', + u'label_no_grad@GRAD', + ] + ) self.no_grad_vars = set() self.params_names = set([u'w2v', u'fc_predict.b_0', u'fc_w']) self.op_path = [ @@ -178,7 +205,7 @@ class SimpleNet(BackwardNet): u'softmax', # fc u'elementwise_sub', u'square', - u'reduce_mean' + u'reduce_mean', ] # loss self.shape = [16, 50] @@ -192,7 +219,7 @@ class SimpleNet(BackwardNet): 'x_no_grad': x, 'x2_no_grad': x2, 'x3_no_grad': x3, - 'label_no_grad': label + 'label_no_grad': label, } def build_model(self): @@ -200,37 +227,41 @@ class SimpleNet(BackwardNet): x = fluid.data(name='x_no_grad', shape=self.shape, dtype='int64') x2 = fluid.data(name='x2_no_grad', shape=self.shape, dtype='int64') x3 = fluid.data(name='x3_no_grad', shape=self.shape, dtype='int64') - label = fluid.data(name='label_no_grad', - shape=[self.shape[0], 1], - dtype='float32') + label = fluid.data( + name='label_no_grad', shape=[self.shape[0], 1], dtype='float32' + ) # shared layer, the grad of 'w2v' will be summed and renamed. # To test _addup_repetitive_outputs_ - x_emb = fluid.embedding(x, - size=[100, 64], - param_attr=fluid.ParamAttr(name='w2v')) - x2_emb = fluid.embedding(x2, - size=[100, 64], - param_attr=fluid.ParamAttr(name='w2v')) - x3_emb = fluid.embedding(x3, - size=[100, 64], - param_attr=fluid.ParamAttr(name='w2v')) + x_emb = fluid.embedding( + x, size=[100, 64], param_attr=fluid.ParamAttr(name='w2v') + ) + x2_emb = fluid.embedding( + x2, size=[100, 64], param_attr=fluid.ParamAttr(name='w2v') + ) + x3_emb = fluid.embedding( + x3, size=[100, 64], param_attr=fluid.ParamAttr(name='w2v') + ) # merge layers x_merge = fluid.layers.elementwise_add(x_emb, x2_emb, name='x_add_x2') - x2_merge = fluid.layers.elementwise_add(x2_emb, - x3_emb, - name='x2_add_x3') + x2_merge = fluid.layers.elementwise_add( + x2_emb, x3_emb, name='x2_add_x3' + ) # shared fc_w - predict = fluid.layers.fc(input=x_merge, - size=1, - act='softmax', - param_attr=fluid.ParamAttr(name='fc_w'), - name='fc_predict') + predict = fluid.layers.fc( + input=x_merge, + size=1, + act='softmax', + param_attr=fluid.ParamAttr(name='fc_w'), + name='fc_predict', + ) # useless layer for calculating loss - fc_no_use = fluid.layers.fc(input=x2_merge, - size=1, - act='sigmoid', - param_attr=fluid.ParamAttr(name='fc_w'), - name='fc_no_use') + fc_no_use = fluid.layers.fc( + input=x2_merge, + size=1, + act='sigmoid', + param_attr=fluid.ParamAttr(name='fc_w'), + name='fc_no_use', + ) # loss cost = fluid.layers.square_error_cost(input=predict, label=label) loss = paddle.mean(cost, name='mean_loss') @@ -239,7 +270,6 @@ class SimpleNet(BackwardNet): class TestSimpleNet(TestBackward): - def test_backward(self): """ Instantiate each NetClass to test backward. @@ -250,7 +280,6 @@ class TestSimpleNet(TestBackward): class TestGradientsError(unittest.TestCase): - def test_error(self): x = fluid.data(name='x', shape=[None, 2, 8, 8], dtype='float32') x.stop_gradient = False @@ -271,7 +300,6 @@ class TestGradientsError(unittest.TestCase): class TestSimpleNetWithErrorParamList(TestBackward): - def test_parameter_list_type_error(self): self.global_block_idx = 0 self.net = SimpleNet() @@ -285,7 +313,6 @@ class TestSimpleNetWithErrorParamList(TestBackward): class TestSimpleNetWithErrorNoGradSet(TestBackward): - def test_no_grad_set_type_error(self): self.global_block_idx = 0 self.net = SimpleNet() @@ -299,7 +326,6 @@ class TestSimpleNetWithErrorNoGradSet(TestBackward): class TestAppendBackwardWithError(unittest.TestCase): - def build_net(self): x = fluid.data(name='x', shape=[None, 13], dtype='int64') y = fluid.data(name='y', shape=[None, 1], dtype='float32') @@ -326,8 +352,9 @@ class TestAppendBackwardWithError(unittest.TestCase): def test_parameter_list_type_error(self): with self.assertRaises(TypeError): self.param_names[0] = np.random.random([10]) - fluid.backward.append_backward(loss=self.avg_loss, - parameter_list=self.param_names) + fluid.backward.append_backward( + loss=self.avg_loss, parameter_list=self.param_names + ) def test_callback_type_error(self): with self.assertRaises(TypeError): @@ -335,18 +362,19 @@ class TestAppendBackwardWithError(unittest.TestCase): def callback(block, context): return - fluid.backward.append_backward(loss=self.avg_loss, - callbacks=callback) + fluid.backward.append_backward( + loss=self.avg_loss, callbacks=callback + ) class TestGradientsWithOptimizer(unittest.TestCase): - def _check_grad_op_name(self, forward_list, optimiezed_list): backward_list = [op + "_grad" for op in reversed(forward_list)] idx = optimiezed_list.index(backward_list[0], len(backward_list)) - self.assertListEqual(backward_list, - optimiezed_list[idx:idx + len(backward_list)]) + self.assertListEqual( + backward_list, optimiezed_list[idx : idx + len(backward_list)] + ) def test_gradient_with_optimizer(self): main = fluid.Program() @@ -359,8 +387,12 @@ class TestGradientsWithOptimizer(unittest.TestCase): opt = paddle.optimizer.Momentum(learning_rate=0.01, momentum=0.9) forward_list = [o.type for o in main.current_block().ops] - optimize_ops, pram_grads = paddle.autograd.backward_mode.gradients_with_optimizer( - main, opt) + ( + optimize_ops, + pram_grads, + ) = paddle.autograd.backward_mode.gradients_with_optimizer( + main, opt + ) optimized_list = [o.type for o in main.current_block().ops] @@ -371,7 +403,6 @@ class TestGradientsWithOptimizer(unittest.TestCase): # TODO(Aurelius84): add conditional network test class ConditionalNet(BackwardNet): - def __init__(self): super(ConditionalNet, self).__init__() diff --git a/python/paddle/fluid/tests/unittests/test_backward_infer_var_data_type_shape.py b/python/paddle/fluid/tests/unittests/test_backward_infer_var_data_type_shape.py index 9f94124fca6..2a16d6d57e3 100644 --- a/python/paddle/fluid/tests/unittests/test_backward_infer_var_data_type_shape.py +++ b/python/paddle/fluid/tests/unittests/test_backward_infer_var_data_type_shape.py @@ -20,17 +20,16 @@ import warnings class TestBackwardInferVarDataTypeShape(unittest.TestCase): - def test_backward_infer_var_data_type_shape(self): paddle.enable_static() program = fluid.default_main_program() - dy = program.global_block().create_var(name="Tmp@GRAD", - shape=[1, 1], - dtype=np.float32, - persistable=True) + dy = program.global_block().create_var( + name="Tmp@GRAD", shape=[1, 1], dtype=np.float32, persistable=True + ) # invoke warning - fluid.backward._infer_var_data_type_shape_("Tmp@GRAD", - program.global_block()) + fluid.backward._infer_var_data_type_shape_( + "Tmp@GRAD", program.global_block() + ) res = False with warnings.catch_warnings(): res = True diff --git a/python/paddle/fluid/tests/unittests/test_base_layer.py b/python/paddle/fluid/tests/unittests/test_base_layer.py index 993bad83630..66cf1b489f6 100644 --- a/python/paddle/fluid/tests/unittests/test_base_layer.py +++ b/python/paddle/fluid/tests/unittests/test_base_layer.py @@ -24,26 +24,23 @@ from paddle.fluid.framework import _test_eager_guard, in_dygraph_mode class L1(fluid.Layer): - def __init__(self): super(L1, self).__init__() self._param_attr = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.1)) - self.w1 = self.create_parameter(attr=self._param_attr, - shape=[2, 2], - dtype='float32', - is_bias=False) - self.w2 = self.create_parameter(attr=self._param_attr, - shape=[2, 2], - dtype='float32', - is_bias=False) + initializer=fluid.initializer.Constant(value=0.1) + ) + self.w1 = self.create_parameter( + attr=self._param_attr, shape=[2, 2], dtype='float32', is_bias=False + ) + self.w2 = self.create_parameter( + attr=self._param_attr, shape=[2, 2], dtype='float32', is_bias=False + ) def forward(self): return self.w1 + self.w2 class L2(fluid.Layer): - def __init__(self): super(L2, self).__init__() self.layer1 = L1() @@ -54,7 +51,6 @@ class L2(fluid.Layer): class L3(fluid.Layer): - def __init__(self): super(L3, self).__init__() self.layer1 = L2() @@ -65,7 +61,6 @@ class L3(fluid.Layer): class TestBaseLayer(unittest.TestCase): - def func_test_one_level(self): with fluid.dygraph.guard(): l = L1() @@ -75,9 +70,9 @@ class TestBaseLayer(unittest.TestCase): for name, _ in l.named_parameters(prefix='l1'): self.assertEqual(name, expected_names[idx]) idx += 1 - np.testing.assert_allclose(ret.numpy(), - 0.2 * np.ones([2, 2]), - rtol=1e-05) + np.testing.assert_allclose( + ret.numpy(), 0.2 * np.ones([2, 2]), rtol=1e-05 + ) def test_one_level(self): with _test_eager_guard(): @@ -102,9 +97,9 @@ class TestBaseLayer(unittest.TestCase): self.assertEqual(name, expected_names[idx]) idx += 1 ret = l() - np.testing.assert_allclose(ret.numpy(), - 0.8 * np.ones([2, 2]), - rtol=1e-05) + np.testing.assert_allclose( + ret.numpy(), 0.8 * np.ones([2, 2]), rtol=1e-05 + ) def test_three_level(self): with _test_eager_guard(): @@ -143,7 +138,6 @@ class TestBaseLayer(unittest.TestCase): class BufferLayer(fluid.Layer): - def __init__(self): super(BufferLayer, self).__init__() buffer_var = to_variable(np.zeros([2, 4]).astype('int32')) @@ -154,13 +148,12 @@ class BufferLayer(fluid.Layer): class BufferNet(fluid.Layer): - def __init__(self): super(BufferNet, self).__init__() self.buffer_layer = BufferLayer() - self.w1 = self.create_parameter(shape=[2, 2], - dtype='float32', - is_bias=False) + self.w1 = self.create_parameter( + shape=[2, 2], dtype='float32', is_bias=False + ) buffer_var = to_variable(np.ones([2, 4]).astype('int32')) self.register_buffer("net_buffer", buffer_var) @@ -171,9 +164,7 @@ class BufferNet(fluid.Layer): class TestBuffer(unittest.TestCase): - def func_test_buffers_and_named_buffers(self): - def names(named_buffers): return [name for name, _ in named_buffers] @@ -187,11 +178,14 @@ class TestBuffer(unittest.TestCase): self.assertEqual(len(net.buffers()), 3) self.assertEqual( names(net.named_buffers()), - ['net_buffer', 'new_buffer', 'buffer_layer.layer_buffer']) + ['net_buffer', 'new_buffer', 'buffer_layer.layer_buffer'], + ) self.assertEqual(len(net.buffers(include_sublayers=False)), 2) - self.assertEqual(names(net.named_buffers(include_sublayers=False)), - ['net_buffer', 'new_buffer']) + self.assertEqual( + names(net.named_buffers(include_sublayers=False)), + ['net_buffer', 'new_buffer'], + ) def test_buffers_and_named_buffers(self): with _test_eager_guard(): @@ -203,25 +197,31 @@ class TestBuffer(unittest.TestCase): net = fluid.Layer() var = to_variable(np.zeros([1])) - with self.assertRaisesRegexp(TypeError, - "name of buffer should be a string"): + with self.assertRaisesRegexp( + TypeError, "name of buffer should be a string" + ): net.register_buffer(12, var) - with self.assertRaisesRegexp(TypeError, - "buffer should be a Paddle.Tensor"): + with self.assertRaisesRegexp( + TypeError, "buffer should be a Paddle.Tensor" + ): if in_dygraph_mode(): - net.register_buffer("buffer_name", - EagerParamBase([2, 2], 'float32')) + net.register_buffer( + "buffer_name", EagerParamBase([2, 2], 'float32') + ) else: - net.register_buffer("buffer_name", - ParamBase([2, 2], 'float32')) + net.register_buffer( + "buffer_name", ParamBase([2, 2], 'float32') + ) - with self.assertRaisesRegexp(KeyError, - "name of buffer can not contain"): + with self.assertRaisesRegexp( + KeyError, "name of buffer can not contain" + ): net.register_buffer("buffer.name", var) - with self.assertRaisesRegexp(KeyError, - "name of buffer can not be empty"): + with self.assertRaisesRegexp( + KeyError, "name of buffer can not be empty" + ): net.register_buffer("", var) net.attr_name = 10 @@ -358,8 +358,9 @@ class TestBuffer(unittest.TestCase): net.register_buffer("buffer_var2", var2, persistable=False) self.assertEqual(len(net.state_dict()), 1) - self.assertEqual([name for name, _ in net.state_dict().items()], - ["buffer_var1"]) + self.assertEqual( + [name for name, _ in net.state_dict().items()], ["buffer_var1"] + ) # load state_dict net_load = fluid.Layer() @@ -379,7 +380,6 @@ class TestBuffer(unittest.TestCase): class BufferNetWithModification(paddle.nn.Layer): - def __init__(self, shape): super(BufferNetWithModification, self).__init__() @@ -397,7 +397,6 @@ class BufferNetWithModification(paddle.nn.Layer): class TestModifiedBuffer(unittest.TestCase): - def funcsetUp(self): paddle.disable_static() self.prog_trans = ProgramTranslator() @@ -418,8 +417,9 @@ class TestModifiedBuffer(unittest.TestCase): st_outs = self._run(True) for i in range(len(dy_outs)): - np.testing.assert_array_equal(dy_outs[i].numpy(), - st_outs[i].numpy()) + np.testing.assert_array_equal( + dy_outs[i].numpy(), st_outs[i].numpy() + ) def test_modified(self): with _test_eager_guard(): @@ -428,7 +428,6 @@ class TestModifiedBuffer(unittest.TestCase): class TestLayerTo(unittest.TestCase): - def funcsetUp(self): paddle.disable_static() self.linear = paddle.nn.Linear(2, 2) @@ -442,30 +441,39 @@ class TestLayerTo(unittest.TestCase): def func_test_to_api(self): self.linear.to(dtype='double') - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) self.linear.to() - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) for p in self.linear.parameters(): if in_dygraph_mode(): self.assertTrue( - isinstance(p, paddle.fluid.framework.EagerParamBase)) + isinstance(p, paddle.fluid.framework.EagerParamBase) + ) else: self.assertTrue(isinstance(p, paddle.fluid.framework.ParamBase)) @@ -476,9 +484,11 @@ class TestLayerTo(unittest.TestCase): self.assertTrue(self.linear.buf_name.place.is_gpu_place()) self.assertEqual(self.linear.buf_name.place.gpu_device_id(), 0) self.assertTrue( - self.linear.weight._grad_ivar().place.is_gpu_place()) + self.linear.weight._grad_ivar().place.is_gpu_place() + ) self.assertEqual( - self.linear.weight._grad_ivar().place.gpu_device_id(), 0) + self.linear.weight._grad_ivar().place.gpu_device_id(), 0 + ) self.linear.to(device='gpu:0') self.assertTrue(self.linear.weight.place.is_gpu_place()) @@ -486,16 +496,20 @@ class TestLayerTo(unittest.TestCase): self.assertTrue(self.linear.buf_name.place.is_gpu_place()) self.assertEqual(self.linear.buf_name.place.gpu_device_id(), 0) self.assertTrue( - self.linear.weight._grad_ivar().place.is_gpu_place()) + self.linear.weight._grad_ivar().place.is_gpu_place() + ) self.assertEqual( - self.linear.weight._grad_ivar().place.gpu_device_id(), 0) + self.linear.weight._grad_ivar().place.gpu_device_id(), 0 + ) for p in self.linear.parameters(): if in_dygraph_mode(): self.assertTrue( - isinstance(p, paddle.fluid.framework.EagerParamBase)) + isinstance(p, paddle.fluid.framework.EagerParamBase) + ) else: self.assertTrue( - isinstance(p, paddle.fluid.framework.ParamBase)) + isinstance(p, paddle.fluid.framework.ParamBase) + ) self.linear.to(device=paddle.CPUPlace()) self.assertTrue(self.linear.weight.place.is_cpu_place()) @@ -513,59 +527,77 @@ class TestLayerTo(unittest.TestCase): def func_test_to_api_paddle_dtype(self): self.linear.to(dtype=paddle.float64) - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) self.linear.to() - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) for p in self.linear.parameters(): if in_dygraph_mode(): self.assertTrue( - isinstance(p, paddle.fluid.framework.EagerParamBase)) + isinstance(p, paddle.fluid.framework.EagerParamBase) + ) else: self.assertTrue(isinstance(p, paddle.fluid.framework.ParamBase)) def func_test_to_api_numpy_dtype(self): self.linear.to(dtype=np.float64) - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) self.linear.to() - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) for p in self.linear.parameters(): if in_dygraph_mode(): self.assertTrue( - isinstance(p, paddle.fluid.framework.EagerParamBase)) + isinstance(p, paddle.fluid.framework.EagerParamBase) + ) else: self.assertTrue(isinstance(p, paddle.fluid.framework.ParamBase)) diff --git a/python/paddle/fluid/tests/unittests/test_basic_gru_api.py b/python/paddle/fluid/tests/unittests/test_basic_gru_api.py index c0719d7643c..339323110a1 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_gru_api.py +++ b/python/paddle/fluid/tests/unittests/test_basic_gru_api.py @@ -34,27 +34,28 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. - - -def gru_np(input, - init_h, - hidden_size, - gate_weight, - gate_bias, - candidate_weight, - candidate_bias, - num_layers=1, - batch_first=False, - is_bidirect=False, - sequence_length=None): - + return (2.0 / (1.0 + np.exp(y))) - 1.0 + + +def gru_np( + input, + init_h, + hidden_size, + gate_weight, + gate_bias, + candidate_weight, + candidate_bias, + num_layers=1, + batch_first=False, + is_bidirect=False, + sequence_length=None, +): def step(step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b): concat_1 = np.concatenate([step_in, pre_hidden], 1) @@ -65,8 +66,9 @@ def gru_np(input, r_hidden = r * pre_hidden - candidate = np.matmul(np.concatenate([step_in, r_hidden], 1), - candidate_w) + candidate = np.matmul( + np.concatenate([step_in, r_hidden], 1), candidate_w + ) candidate += candidate_b c = tanh(candidate) @@ -94,8 +96,9 @@ def gru_np(input, if is_bidirect: direc_num = 2 if init_h: - init_h = np.reshape(init_h, - shape=[num_layers, direc_num, -1, hidden_size]) + init_h = np.reshape( + init_h, shape=[num_layers, direc_num, -1, hidden_size] + ) else: init_h = np.zeros([num_layers, direc_num, batch_size, hidden_size]) @@ -117,15 +120,19 @@ def gru_np(input, for i in range(num_layers): new_hidden = step( - step_input, pre_hidden_array[i], + step_input, + pre_hidden_array[i], gate_weight[direc_index * num_layers + i], gate_bias[direc_index * num_layers + i], candidate_weight[direc_index * num_layers + i], - candidate_bias[direc_index * num_layers + i]) + candidate_bias[direc_index * num_layers + i], + ) if mask is not None: - new_hidden = new_hidden * step_mask + ( - 1 - step_mask) * pre_hidden_array[i] + new_hidden = ( + new_hidden * step_mask + + (1 - step_mask) * pre_hidden_array[i] + ) pre_hidden_array[i] = new_hidden @@ -135,14 +142,15 @@ def gru_np(input, rnn_out = np.reshape(rnn_out, [seq_len, -1, hidden_size]) last_hidden_out = np.concatenate(pre_hidden_array, 0) - last_hidden_out = np.reshape(last_hidden_out, - [num_layers, -1, hidden_size]) + last_hidden_out = np.reshape( + last_hidden_out, [num_layers, -1, hidden_size] + ) return rnn_out, last_hidden_out - fw_rnn_out, fw_last_hidden = get_single_direction_output(input, - mask, - direc_index=0) + fw_rnn_out, fw_last_hidden = get_single_direction_output( + input, mask, direc_index=0 + ) if is_bidirect: bw_input = input[::-1] @@ -150,16 +158,17 @@ def gru_np(input, if mask is not None: bw_mask = mask[::-1] - bw_rnn_out, bw_last_hidden = get_single_direction_output(bw_input, - bw_mask, - direc_index=1) + bw_rnn_out, bw_last_hidden = get_single_direction_output( + bw_input, bw_mask, direc_index=1 + ) bw_rnn_out = bw_rnn_out[::-1] rnn_out = np.concatenate([fw_rnn_out, bw_rnn_out], 2) last_hidden = np.concatenate([fw_last_hidden, bw_last_hidden], 1) - last_hidden = np.reshape(last_hidden, - [num_layers * direc_num, -1, hidden_size]) + last_hidden = np.reshape( + last_hidden, [num_layers * direc_num, -1, hidden_size] + ) if batch_first: rnn_out = np.transpose(rnn_out, [1, 0, 2]) @@ -176,7 +185,6 @@ def gru_np(input, class TestBasicGRUApi(unittest.TestCase): - def setUp(self): self.hidden_size = 10 self.batch_size = 5 @@ -186,15 +194,24 @@ class TestBasicGRUApi(unittest.TestCase): self.batch_first = False def test_run(self): - x = layers.data(name='x', - shape=[-1, self.batch_size, self.hidden_size], - dtype='float32') - sequence_length = layers.data(name="sequence_length", - shape=[-1], - dtype='float32') - - rnn_out, last_hidden = basic_gru( x, None, self.hidden_size, num_layers=self.num_layers, \ - batch_first = self.batch_first, bidirectional=self.is_bidirect, sequence_length=sequence_length ) + x = layers.data( + name='x', + shape=[-1, self.batch_size, self.hidden_size], + dtype='float32', + ) + sequence_length = layers.data( + name="sequence_length", shape=[-1], dtype='float32' + ) + + rnn_out, last_hidden = basic_gru( + x, + None, + self.hidden_size, + num_layers=self.num_layers, + batch_first=self.batch_first, + bidirectional=self.is_bidirect, + sequence_length=sequence_length, + ) last_hidden.persisbale = True rnn_out.persisbale = True @@ -218,38 +235,52 @@ class TestBasicGRUApi(unittest.TestCase): for i in range(self.num_layers): gate_w_name = "basic_gru_layers_" + str(i) + "/BasicGRUUnit_0.w_0" gate_b_name = "basic_gru_layers_" + str(i) + "/BasicGRUUnit_0.b_0" - candidate_w_name = "basic_gru_layers_" + str( - i) + "/BasicGRUUnit_0.w_1" - candidate_b_name = "basic_gru_layers_" + str( - i) + "/BasicGRUUnit_0.b_1" + candidate_w_name = ( + "basic_gru_layers_" + str(i) + "/BasicGRUUnit_0.w_1" + ) + candidate_b_name = ( + "basic_gru_layers_" + str(i) + "/BasicGRUUnit_0.b_1" + ) gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) candidate_w = np.array( - fluid.global_scope().find_var(candidate_w_name).get_tensor()) + fluid.global_scope().find_var(candidate_w_name).get_tensor() + ) candidate_w = np.random.uniform( - -0.1, 0.1, size=candidate_w.shape).astype('float32') + -0.1, 0.1, size=candidate_w.shape + ).astype('float32') fluid.global_scope().find_var(candidate_w_name).get_tensor().set( - candidate_w, place) + candidate_w, place + ) candidate_b = np.array( - fluid.global_scope().find_var(candidate_b_name).get_tensor()) + fluid.global_scope().find_var(candidate_b_name).get_tensor() + ) candidate_b = np.random.uniform( - -0.1, 0.1, size=candidate_b.shape).astype('float32') + -0.1, 0.1, size=candidate_b.shape + ).astype('float32') fluid.global_scope().find_var(candidate_b_name).get_tensor().set( - candidate_b, place) + candidate_b, place + ) gate_weight.append(gate_w) gate_bias.append(gate_b) @@ -258,42 +289,58 @@ class TestBasicGRUApi(unittest.TestCase): if self.is_bidirect: for i in range(self.num_layers): - gate_w_name = "basic_gru_reverse_layers_" + str( - i) + "/BasicGRUUnit_0.w_0" - gate_b_name = "basic_gru_reverse_layers_" + str( - i) + "/BasicGRUUnit_0.b_0" - candidate_w_name = "basic_gru_reverse_layers_" + str( - i) + "/BasicGRUUnit_0.w_1" - candidate_b_name = "basic_gru_reverse_layers_" + str( - i) + "/BasicGRUUnit_0.b_1" + gate_w_name = ( + "basic_gru_reverse_layers_" + str(i) + "/BasicGRUUnit_0.w_0" + ) + gate_b_name = ( + "basic_gru_reverse_layers_" + str(i) + "/BasicGRUUnit_0.b_0" + ) + candidate_w_name = ( + "basic_gru_reverse_layers_" + str(i) + "/BasicGRUUnit_0.w_1" + ) + candidate_b_name = ( + "basic_gru_reverse_layers_" + str(i) + "/BasicGRUUnit_0.b_1" + ) gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) - candidate_w = np.array(fluid.global_scope().find_var( - candidate_w_name).get_tensor()) + candidate_w = np.array( + fluid.global_scope().find_var(candidate_w_name).get_tensor() + ) candidate_w = np.random.uniform( - -0.1, 0.1, size=candidate_w.shape).astype('float32') + -0.1, 0.1, size=candidate_w.shape + ).astype('float32') fluid.global_scope().find_var( - candidate_w_name).get_tensor().set(candidate_w, place) + candidate_w_name + ).get_tensor().set(candidate_w, place) - candidate_b = np.array(fluid.global_scope().find_var( - candidate_b_name).get_tensor()) + candidate_b = np.array( + fluid.global_scope().find_var(candidate_b_name).get_tensor() + ) candidate_b = np.random.uniform( - -0.1, 0.1, size=candidate_b.shape).astype('float32') + -0.1, 0.1, size=candidate_b.shape + ).astype('float32') fluid.global_scope().find_var( - candidate_b_name).get_tensor().set(candidate_b, place) + candidate_b_name + ).get_tensor().set(candidate_b, place) gate_weight.append(gate_w) gate_bias.append(gate_b) @@ -301,39 +348,39 @@ class TestBasicGRUApi(unittest.TestCase): candidate_bias.append(candidate_b) step_input_np = np.random.uniform( - -0.1, 0.1, - (self.seq_len, self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.seq_len, self.batch_size, self.hidden_size) + ).astype('float32') sequence_length_np = np.random.randint( - self.seq_len // 2, self.seq_len, - size=(self.batch_size)).astype('int64') + self.seq_len // 2, self.seq_len, size=(self.batch_size) + ).astype('int64') - out = exe.run(feed={ - 'x': step_input_np, - 'sequence_length': sequence_length_np - }, - fetch_list=[rnn_out, last_hidden]) + out = exe.run( + feed={'x': step_input_np, 'sequence_length': sequence_length_np}, + fetch_list=[rnn_out, last_hidden], + ) api_rnn_out = out[0] api_last_hidden = out[1] - np_out = gru_np(step_input_np, - None, - self.hidden_size, - gate_weight, - gate_bias, - candidate_weight, - candidate_bias, - num_layers=self.num_layers, - batch_first=self.batch_first, - is_bidirect=self.is_bidirect, - sequence_length=sequence_length_np) + np_out = gru_np( + step_input_np, + None, + self.hidden_size, + gate_weight, + gate_bias, + candidate_weight, + candidate_bias, + num_layers=self.num_layers, + batch_first=self.batch_first, + is_bidirect=self.is_bidirect, + sequence_length=sequence_length_np, + ) np.testing.assert_allclose(api_rnn_out, np_out[0], rtol=0.0001, atol=0) - np.testing.assert_allclose(api_last_hidden, - np_out[1], - rtol=0.0001, - atol=0) + np.testing.assert_allclose( + api_last_hidden, np_out[1], rtol=0.0001, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_basic_gru_unit_op.py b/python/paddle/fluid/tests/unittests/test_basic_gru_unit_op.py index ae3e6dc4f77..660aa184f57 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_gru_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_basic_gru_unit_op.py @@ -34,13 +34,13 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. + return (2.0 / (1.0 + np.exp(y))) - 1.0 def step(step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b): @@ -64,16 +64,15 @@ def step(step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b): class TestBasicGRUUnit(unittest.TestCase): - def setUp(self): self.hidden_size = 5 self.batch_size = 5 def test_run(self): x = layers.data(name='x', shape=[-1, self.hidden_size], dtype='float32') - pre_hidden = layers.data(name="pre_hidden", - shape=[-1, self.hidden_size], - dtype='float32') + pre_hidden = layers.data( + name="pre_hidden", shape=[-1, self.hidden_size], dtype='float32' + ) gru_unit = BasicGRUUnit("gru_unit", self.hidden_size) new_hidden = gru_unit(x, pre_hidden) @@ -98,48 +97,67 @@ class TestBasicGRUUnit(unittest.TestCase): candidate_b_name = "gru_unit/BasicGRUUnit_0.b_1" gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) candidate_w = np.array( - fluid.global_scope().find_var(candidate_w_name).get_tensor()) + fluid.global_scope().find_var(candidate_w_name).get_tensor() + ) candidate_w = np.random.uniform( - -0.1, 0.1, size=candidate_w.shape).astype('float32') + -0.1, 0.1, size=candidate_w.shape + ).astype('float32') fluid.global_scope().find_var(candidate_w_name).get_tensor().set( - candidate_w, place) + candidate_w, place + ) candidate_b = np.array( - fluid.global_scope().find_var(candidate_b_name).get_tensor()) + fluid.global_scope().find_var(candidate_b_name).get_tensor() + ) candidate_b = np.random.uniform( - -0.1, 0.1, size=candidate_b.shape).astype('float32') + -0.1, 0.1, size=candidate_b.shape + ).astype('float32') fluid.global_scope().find_var(candidate_b_name).get_tensor().set( - candidate_b, place) + candidate_b, place + ) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') - out = exe.run(feed={ - 'x': step_input_np, - 'pre_hidden': pre_hidden_np - }, - fetch_list=[new_hidden]) + out = exe.run( + feed={'x': step_input_np, 'pre_hidden': pre_hidden_np}, + fetch_list=[new_hidden], + ) api_out = out[0] - np_out = step(step_input_np, pre_hidden_np, gate_w, gate_b, candidate_w, - candidate_b) + np_out = step( + step_input_np, + pre_hidden_np, + gate_w, + gate_b, + candidate_w, + candidate_b, + ) np.testing.assert_allclose(api_out, np_out, rtol=0.0001, atol=0) diff --git a/python/paddle/fluid/tests/unittests/test_basic_lstm_api.py b/python/paddle/fluid/tests/unittests/test_basic_lstm_api.py index 9574a1ea973..befc7e2c5ff 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_lstm_api.py +++ b/python/paddle/fluid/tests/unittests/test_basic_lstm_api.py @@ -34,27 +34,28 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. - - -def lstm_np(input, - init_h, - init_c, - hidden_size, - gate_weight, - gate_bias, - num_layers=1, - batch_first=False, - is_bidirect=False, - sequence_length=None, - forget_bias=1.0): - + return (2.0 / (1.0 + np.exp(y))) - 1.0 + + +def lstm_np( + input, + init_h, + init_c, + hidden_size, + gate_weight, + gate_bias, + num_layers=1, + batch_first=False, + is_bidirect=False, + sequence_length=None, + forget_bias=1.0, +): def step(step_in, pre_hidden, pre_cell, gate_w, gate_b): concat_1 = np.concatenate([step_in, pre_hidden], 1) @@ -112,23 +113,27 @@ def lstm_np(input, if mask is not None: step_mask = mask[i] step_mask = np.reshape(step_mask, [-1, 1]) - #print("np mask", step_mask.shape ) + # print("np mask", step_mask.shape ) for i in range(num_layers): new_hidden, new_cell = step( - step_input, pre_hidden_array[i], pre_cell_array[i], + step_input, + pre_hidden_array[i], + pre_cell_array[i], gate_weight[direc_index * num_layers + i], - gate_bias[direc_index * num_layers + i]) + gate_bias[direc_index * num_layers + i], + ) if mask is not None: new_hidden = np.multiply( - new_hidden, step_mask) - np.multiply( - pre_hidden_array[i], (step_mask - 1.0)) - #new_hidden = new_hidden * step_mask - pre_hidden_array[i] * ( step_mask -1 ) - #new_cell = new_cell * step_mask - pre_cell_array[i] * (step_mask -1) + new_hidden, step_mask + ) - np.multiply(pre_hidden_array[i], (step_mask - 1.0)) + # new_hidden = new_hidden * step_mask - pre_hidden_array[i] * ( step_mask -1 ) + # new_cell = new_cell * step_mask - pre_cell_array[i] * (step_mask -1) new_cell = np.multiply(new_cell, step_mask) - np.multiply( - pre_cell_array[i], (step_mask - 1.0)) + pre_cell_array[i], (step_mask - 1.0) + ) pre_hidden_array[i] = new_hidden pre_cell_array[i] = new_cell @@ -139,8 +144,9 @@ def lstm_np(input, rnn_out = np.reshape(rnn_out, [seq_len, -1, hidden_size]) last_hidden_out = np.concatenate(pre_hidden_array, 0) - last_hidden_out = np.reshape(last_hidden_out, - [num_layers, -1, hidden_size]) + last_hidden_out = np.reshape( + last_hidden_out, [num_layers, -1, hidden_size] + ) last_cell_out = np.concatenate(pre_cell_array, 0) last_cell_out = np.reshape(last_cell_out, [num_layers, -1, hidden_size]) @@ -148,7 +154,8 @@ def lstm_np(input, return rnn_out, last_hidden_out, last_cell_out fw_rnn_out, fw_last_hidden, fw_last_cell = get_single_direction_output( - input, mask, direc_index=0) + input, mask, direc_index=0 + ) if is_bidirect: bw_input = input[::-1] @@ -157,18 +164,21 @@ def lstm_np(input, bw_mask = mask[::-1] bw_rnn_out, bw_last_hidden, bw_last_cell = get_single_direction_output( - bw_input, bw_mask, direc_index=1) + bw_input, bw_mask, direc_index=1 + ) bw_rnn_out = bw_rnn_out[::-1] rnn_out = np.concatenate([fw_rnn_out, bw_rnn_out], 2) last_hidden = np.concatenate([fw_last_hidden, bw_last_hidden], 1) - last_hidden = np.reshape(last_hidden, - [num_layers * direc_num, -1, hidden_size]) + last_hidden = np.reshape( + last_hidden, [num_layers * direc_num, -1, hidden_size] + ) last_cell = np.concatenate([fw_last_cell, bw_last_cell], 1) - last_cell = np.reshape(last_cell, - [num_layers * direc_num, -1, hidden_size]) + last_cell = np.reshape( + last_cell, [num_layers * direc_num, -1, hidden_size] + ) if batch_first: rnn_out = np.transpose(rnn_out, [1, 0, 2]) @@ -186,7 +196,6 @@ def lstm_np(input, class TestBasicLSTMApi(unittest.TestCase): - def setUp(self): self.hidden_size = 10 self.batch_size = 5 @@ -197,15 +206,26 @@ class TestBasicLSTMApi(unittest.TestCase): self.forget_bias = 1.0 def test_run(self): - x = layers.data(name='x', - shape=[-1, self.batch_size, self.hidden_size], - dtype='float32') - sequence_length = layers.data(name="sequence_length", - shape=[-1], - dtype='float32') - - rnn_out, last_hidden, last_cell = basic_lstm( x, None, None, self.hidden_size, num_layers=self.num_layers, \ - batch_first = self.batch_first, bidirectional=self.is_bidirect, sequence_length=sequence_length, forget_bias = self.forget_bias ) + x = layers.data( + name='x', + shape=[-1, self.batch_size, self.hidden_size], + dtype='float32', + ) + sequence_length = layers.data( + name="sequence_length", shape=[-1], dtype='float32' + ) + + rnn_out, last_hidden, last_cell = basic_lstm( + x, + None, + None, + self.hidden_size, + num_layers=self.num_layers, + batch_first=self.batch_first, + bidirectional=self.is_bidirect, + sequence_length=sequence_length, + forget_bias=self.forget_bias, + ) last_hidden.persisbale = True rnn_out.persisbale = True @@ -228,83 +248,100 @@ class TestBasicLSTMApi(unittest.TestCase): gate_b_name = "basic_lstm_layers_" + str(i) + "/BasicLSTMUnit_0.b_0" gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) gate_weight.append(gate_w) gate_bias.append(gate_b) if self.is_bidirect: for i in range(self.num_layers): - gate_w_name = "basic_lstm_reverse_layers_" + str( - i) + "/BasicLSTMUnit_0.w_0" - gate_b_name = "basic_lstm_reverse_layers_" + str( - i) + "/BasicLSTMUnit_0.b_0" + gate_w_name = ( + "basic_lstm_reverse_layers_" + + str(i) + + "/BasicLSTMUnit_0.w_0" + ) + gate_b_name = ( + "basic_lstm_reverse_layers_" + + str(i) + + "/BasicLSTMUnit_0.b_0" + ) gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) gate_weight.append(gate_w) gate_bias.append(gate_b) step_input_np = np.random.uniform( - -0.1, 0.1, - (self.seq_len, self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.seq_len, self.batch_size, self.hidden_size) + ).astype('float32') sequence_length_np = np.random.randint( - self.seq_len // 2, self.seq_len, - size=(self.batch_size)).astype('int64') + self.seq_len // 2, self.seq_len, size=(self.batch_size) + ).astype('int64') - out = exe.run(feed={ - 'x': step_input_np, - 'sequence_length': sequence_length_np - }, - fetch_list=[rnn_out, last_hidden, last_cell]) + out = exe.run( + feed={'x': step_input_np, 'sequence_length': sequence_length_np}, + fetch_list=[rnn_out, last_hidden, last_cell], + ) api_rnn_out = out[0] api_last_hidden = out[1] api_last_cell = out[2] - np_out = lstm_np(step_input_np, - None, - None, - self.hidden_size, - gate_weight, - gate_bias, - num_layers=self.num_layers, - batch_first=self.batch_first, - is_bidirect=self.is_bidirect, - sequence_length=sequence_length_np) + np_out = lstm_np( + step_input_np, + None, + None, + self.hidden_size, + gate_weight, + gate_bias, + num_layers=self.num_layers, + batch_first=self.batch_first, + is_bidirect=self.is_bidirect, + sequence_length=sequence_length_np, + ) np.testing.assert_allclose(api_rnn_out, np_out[0], rtol=0.0001, atol=0) - np.testing.assert_allclose(api_last_hidden, - np_out[1], - rtol=0.0001, - atol=0) - np.testing.assert_allclose(api_last_cell, - np_out[2], - rtol=0.0001, - atol=0) + np.testing.assert_allclose( + api_last_hidden, np_out[1], rtol=0.0001, atol=0 + ) + np.testing.assert_allclose( + api_last_cell, np_out[2], rtol=0.0001, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_basic_lstm_unit_op.py b/python/paddle/fluid/tests/unittests/test_basic_lstm_unit_op.py index 50ef22467df..cf32b95d76f 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_lstm_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_basic_lstm_unit_op.py @@ -34,13 +34,13 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. + return (2.0 / (1.0 + np.exp(y))) - 1.0 def step(step_in, pre_hidden, pre_cell, gate_w, gate_b, forget_bias=1.0): @@ -57,19 +57,18 @@ def step(step_in, pre_hidden, pre_cell, gate_w, gate_b, forget_bias=1.0): class TestBasicGRUUnit(unittest.TestCase): - def setUp(self): self.hidden_size = 5 self.batch_size = 5 def test_run(self): x = layers.data(name='x', shape=[-1, self.hidden_size], dtype='float32') - pre_hidden = layers.data(name="pre_hidden", - shape=[-1, self.hidden_size], - dtype='float32') - pre_cell = layers.data(name="pre_cell", - shape=[-1, self.hidden_size], - dtype='float32') + pre_hidden = layers.data( + name="pre_hidden", shape=[-1, self.hidden_size], dtype='float32' + ) + pre_cell = layers.data( + name="pre_cell", shape=[-1, self.hidden_size], dtype='float32' + ) lstm_unit = BasicLSTMUnit("lstm_unit", self.hidden_size) @@ -94,44 +93,57 @@ class TestBasicGRUUnit(unittest.TestCase): gate_b_name = "lstm_unit/BasicLSTMUnit_0.b_0" gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_cell_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') - - out = exe.run( feed={ 'x' : step_input_np, 'pre_hidden' : pre_hidden_np, \ - 'pre_cell' : pre_cell_np }, - fetch_list=[ new_hidden, new_cell]) + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') + + out = exe.run( + feed={ + 'x': step_input_np, + 'pre_hidden': pre_hidden_np, + 'pre_cell': pre_cell_np, + }, + fetch_list=[new_hidden, new_cell], + ) api_hidden_out = out[0] api_cell_out = out[1] - np_hidden_out, np_cell_out = step(step_input_np, pre_hidden_np, - pre_cell_np, gate_w, gate_b) - - np.testing.assert_allclose(api_hidden_out, - np_hidden_out, - rtol=0.0001, - atol=0) - np.testing.assert_allclose(api_cell_out, - np_cell_out, - rtol=0.0001, - atol=0) + np_hidden_out, np_cell_out = step( + step_input_np, pre_hidden_np, pre_cell_np, gate_w, gate_b + ) + + np.testing.assert_allclose( + api_hidden_out, np_hidden_out, rtol=0.0001, atol=0 + ) + np.testing.assert_allclose( + api_cell_out, np_cell_out, rtol=0.0001, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_basic_rnn_name.py b/python/paddle/fluid/tests/unittests/test_basic_rnn_name.py index ca56cd62a46..17bec037e84 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_rnn_name.py +++ b/python/paddle/fluid/tests/unittests/test_basic_rnn_name.py @@ -20,14 +20,19 @@ from test_imperative_base import new_program_scope class TestBasicGRUApiName(unittest.TestCase): - def setUp(self): - self.name_set = set([ - "test1_fw_w_0_gate", "test1_fw_w_0_candidate", "test1_fw_b_0_gate", - "test1_fw_b_0_candidate", "test1_bw_w_0_gate", - "test1_bw_w_0_candidate", "test1_bw_b_0_gate", - "test1_bw_b_0_candidate" - ]) + self.name_set = set( + [ + "test1_fw_w_0_gate", + "test1_fw_w_0_candidate", + "test1_fw_b_0_gate", + "test1_fw_b_0_candidate", + "test1_bw_w_0_gate", + "test1_bw_w_0_candidate", + "test1_bw_b_0_gate", + "test1_bw_b_0_candidate", + ] + ) def test_name(self): batch_size = 20 @@ -39,35 +44,54 @@ class TestBasicGRUApiName(unittest.TestCase): batch_first = False with new_program_scope(): - input = layers.data(name="input", - shape=[-1, batch_size, input_size], - dtype='float32') - pre_hidden = layers.data(name="pre_hidden", - shape=[-1, hidden_size], - dtype='float32') - sequence_length = layers.data(name="sequence_length", - shape=[-1], - dtype='int32') - - - rnn_out, last_hidden = basic_gru( input, pre_hidden, hidden_size, num_layers = num_layers, \ - sequence_length = sequence_length, dropout_prob=dropout, bidirectional = bidirectional, \ - batch_first = batch_first, param_attr=fluid.ParamAttr( name ="test1"), bias_attr=fluid.ParamAttr( name="test1"), name="basic_gru") + input = layers.data( + name="input", + shape=[-1, batch_size, input_size], + dtype='float32', + ) + pre_hidden = layers.data( + name="pre_hidden", shape=[-1, hidden_size], dtype='float32' + ) + sequence_length = layers.data( + name="sequence_length", shape=[-1], dtype='int32' + ) + + rnn_out, last_hidden = basic_gru( + input, + pre_hidden, + hidden_size, + num_layers=num_layers, + sequence_length=sequence_length, + dropout_prob=dropout, + bidirectional=bidirectional, + batch_first=batch_first, + param_attr=fluid.ParamAttr(name="test1"), + bias_attr=fluid.ParamAttr(name="test1"), + name="basic_gru", + ) var_list = fluid.io.get_program_parameter( - fluid.default_main_program()) + fluid.default_main_program() + ) for var in var_list: self.assertTrue(var.name in self.name_set) class TestBasicLSTMApiName(unittest.TestCase): - def setUp(self): - self.name_set = set([ - "test1_fw_w_0", "test1_fw_b_0", "test1_fw_w_1", "test1_fw_b_1", - "test1_bw_w_0", "test1_bw_b_0", "test1_bw_w_1", "test1_bw_b_1" - ]) + self.name_set = set( + [ + "test1_fw_w_0", + "test1_fw_b_0", + "test1_fw_w_1", + "test1_fw_b_1", + "test1_bw_w_0", + "test1_bw_b_0", + "test1_bw_w_1", + "test1_bw_b_1", + ] + ) def test_name(self): batch_size = 20 @@ -79,27 +103,38 @@ class TestBasicLSTMApiName(unittest.TestCase): batch_first = False with new_program_scope(): - input = layers.data(name="input", - shape=[-1, batch_size, input_size], - dtype='float32') - pre_hidden = layers.data(name="pre_hidden", - shape=[-1, hidden_size], - dtype='float32') - pre_cell = layers.data(name="pre_cell", - shape=[-1, hidden_size], - dtype='float32') - sequence_length = layers.data(name="sequence_length", - shape=[-1], - dtype='int32') - - rnn_out, last_hidden, last_cell = basic_lstm( input, pre_hidden, pre_cell, \ - hidden_size, num_layers = num_layers, \ - sequence_length = sequence_length, dropout_prob=dropout, bidirectional = bidirectional, \ - param_attr=fluid.ParamAttr( name ="test1"), bias_attr=fluid.ParamAttr( name = "test1"), \ - batch_first = batch_first) + input = layers.data( + name="input", + shape=[-1, batch_size, input_size], + dtype='float32', + ) + pre_hidden = layers.data( + name="pre_hidden", shape=[-1, hidden_size], dtype='float32' + ) + pre_cell = layers.data( + name="pre_cell", shape=[-1, hidden_size], dtype='float32' + ) + sequence_length = layers.data( + name="sequence_length", shape=[-1], dtype='int32' + ) + + rnn_out, last_hidden, last_cell = basic_lstm( + input, + pre_hidden, + pre_cell, + hidden_size, + num_layers=num_layers, + sequence_length=sequence_length, + dropout_prob=dropout, + bidirectional=bidirectional, + param_attr=fluid.ParamAttr(name="test1"), + bias_attr=fluid.ParamAttr(name="test1"), + batch_first=batch_first, + ) var_list = fluid.io.get_program_parameter( - fluid.default_main_program()) + fluid.default_main_program() + ) for var in var_list: self.assertTrue(var.name in self.name_set) diff --git a/python/paddle/fluid/tests/unittests/test_batch_fc_op.py b/python/paddle/fluid/tests/unittests/test_batch_fc_op.py index 6214859931f..2414514b2bd 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_fc_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_fc_op.py @@ -32,7 +32,6 @@ def np_cal_batchfc(input, w, bias): class TestBatchFCOp(OpTest): - def config(self): self.slot_pairs_num = 10 self.batch_size = 5 @@ -42,12 +41,15 @@ class TestBatchFCOp(OpTest): def setUp(self): self.config() - self.input = np.random.random((self.slot_pairs_num, self.batch_size, - self.in_dim)).astype(self.dtype) + self.input = np.random.random( + (self.slot_pairs_num, self.batch_size, self.in_dim) + ).astype(self.dtype) self.w = np.random.random( - (self.slot_pairs_num, self.in_dim, self.out_dim)).astype(self.dtype) + (self.slot_pairs_num, self.in_dim, self.out_dim) + ).astype(self.dtype) self.bias = np.random.random( - (self.slot_pairs_num, self.out_dim)).astype(self.dtype) + (self.slot_pairs_num, self.out_dim) + ).astype(self.dtype) self.op_type = "batch_fc" np_out = np_cal_batchfc(self.input, self.w, self.bias) np_out = np_out.astype(self.dtype) @@ -60,12 +62,12 @@ class TestBatchFCOp(OpTest): def test_check_grad_gpu(self): if core.is_compiled_with_cuda(): - self.check_grad_with_place(core.CUDAPlace(0), - ["Bias", "W", "Input"], "Out") + self.check_grad_with_place( + core.CUDAPlace(0), ["Bias", "W", "Input"], "Out" + ) class TestBatchFCOp1(OpTest): - def config(self): self.slot_pairs_num = 10 self.batch_size = 5 @@ -75,12 +77,15 @@ class TestBatchFCOp1(OpTest): def setUp(self): self.config() - self.input = np.random.random((self.slot_pairs_num, self.batch_size, - self.in_dim)).astype(self.dtype) + self.input = np.random.random( + (self.slot_pairs_num, self.batch_size, self.in_dim) + ).astype(self.dtype) self.w = np.random.random( - (self.slot_pairs_num, self.in_dim, self.out_dim)).astype(self.dtype) + (self.slot_pairs_num, self.in_dim, self.out_dim) + ).astype(self.dtype) self.bias = np.random.random( - (self.slot_pairs_num, self.out_dim)).astype(self.dtype) + (self.slot_pairs_num, self.out_dim) + ).astype(self.dtype) self.op_type = "batch_fc" np_out = np_cal_batchfc(self.input, self.w, self.bias) np_out = np_out.astype(self.dtype) @@ -95,8 +100,9 @@ class TestBatchFCOp1(OpTest): def test_check_grad_cpu(self): try: - self.check_grad_with_place(core.CPUPlace(), ["Bias", "W", "Input"], - "Out") + self.check_grad_with_place( + core.CPUPlace(), ["Bias", "W", "Input"], "Out" + ) except: print("do not support cpu test, skip") diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py index 33265b33bd1..381640621cb 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py @@ -154,12 +154,20 @@ def _reference_grad(x, y_grad, scale, mean, var, epsilon, data_format): x = np.transpose(x, (0, 2, 3, 1)) y_grad = np.transpose(y_grad, (0, 2, 3, 1)) - x_grad = scale * (y_grad - np.mean(y_grad, axis=(0, 1, 2)) - - (x - mean) * np.mean(y_grad * - (x - mean), axis=(0, 1, 2)) / - (var + epsilon)) / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + x_grad = ( + scale + * ( + y_grad + - np.mean(y_grad, axis=(0, 1, 2)) + - (x - mean) + * np.mean(y_grad * (x - mean), axis=(0, 1, 2)) + / (var + epsilon) + ) + / np.sqrt(var + epsilon) + ) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -184,7 +192,6 @@ def create_or_get_tensor(scope, var_name, var, place): def set_output_grad(scope, outputs, place, feed_dict=None): - def __set_tensor__(name, data=None): out_tensor = scope.find_var(name).get_tensor() grad_tensor = scope.var(grad_var_name(name)).get_tensor() @@ -206,7 +213,6 @@ def set_output_grad(scope, outputs, place, feed_dict=None): class TestBatchNormOpInference(unittest.TestCase): - def setUp(self): self.dtype = np.float32 self.use_mkldnn = False @@ -214,11 +220,9 @@ class TestBatchNormOpInference(unittest.TestCase): self.init_kernel_type() def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) def check_with_place(self, place, data_layout, dtype, shape): epsilon = 0.00001 @@ -244,34 +248,39 @@ class TestBatchNormOpInference(unittest.TestCase): mean = np.zeros(scale_shape).astype(np.float32) variance = np.ones(scale_shape).astype(np.float32) - y_out = _reference_testing(x_val, scale_val, bias_val, mean, variance, - epsilon, data_layout).astype(dtype) + y_out = _reference_testing( + x_val, scale_val, bias_val, mean, variance, epsilon, data_layout + ).astype(dtype) if self.fuse_with_relu: y_out = np.maximum(y_out, 0) scope = core.Scope() # create input - x_tensor = create_or_get_tensor(scope, "x_val", - OpTest.np_dtype_to_fluid_dtype(x_val), - place) + x_tensor = create_or_get_tensor( + scope, "x_val", OpTest.np_dtype_to_fluid_dtype(x_val), place + ) scale_tensor = create_or_get_tensor( - scope, "scale_val", OpTest.np_dtype_to_fluid_dtype(scale_val), - place) + scope, "scale_val", OpTest.np_dtype_to_fluid_dtype(scale_val), place + ) bias_tensor = create_or_get_tensor( - scope, "bias_val", OpTest.np_dtype_to_fluid_dtype(bias_val), place) - mean_tensor = create_or_get_tensor(scope, "mean", - OpTest.np_dtype_to_fluid_dtype(mean), - place) + scope, "bias_val", OpTest.np_dtype_to_fluid_dtype(bias_val), place + ) + mean_tensor = create_or_get_tensor( + scope, "mean", OpTest.np_dtype_to_fluid_dtype(mean), place + ) variance_tensor = create_or_get_tensor( - scope, "variance", OpTest.np_dtype_to_fluid_dtype(variance), place) + scope, "variance", OpTest.np_dtype_to_fluid_dtype(variance), place + ) # create output y_tensor = create_or_get_tensor(scope, "y_out", None, place) - saved_mean_tensor = create_or_get_tensor(scope, "saved_mean", None, - place) - saved_variance_tensor = create_or_get_tensor(scope, "saved_variance", - None, place) + saved_mean_tensor = create_or_get_tensor( + scope, "saved_mean", None, place + ) + saved_variance_tensor = create_or_get_tensor( + scope, "saved_variance", None, place + ) mean_out_tensor = mean_tensor variance_out_tensor = variance_tensor @@ -294,7 +303,8 @@ class TestBatchNormOpInference(unittest.TestCase): data_layout=data_layout, use_mkldnn=self.use_mkldnn, fuse_with_relu=self.fuse_with_relu, - epsilon=epsilon) + epsilon=epsilon, + ) batch_norm_op.run(scope, place) @@ -314,12 +324,19 @@ class TestBatchNormOpInference(unittest.TestCase): y_tensor._set_dims(dims) # check inference result - self.__assert_close(y_tensor, - y_out, - "inference output are different at " + str(place) + - ", " + data_layout + ", " + str(np.dtype(dtype)) + - str(np.array(y_tensor)) + str(y_out), - atol=1e-3) + self.__assert_close( + y_tensor, + y_out, + "inference output are different at " + + str(place) + + ", " + + data_layout + + ", " + + str(np.dtype(dtype)) + + str(np.array(y_tensor)) + + str(y_out), + atol=1e-3, + ) def test_check_output(self): places = [core.CPUPlace()] @@ -328,8 +345,9 @@ class TestBatchNormOpInference(unittest.TestCase): for place in places: for data_format in ["NCHW", "NHWC"]: - self.check_with_place(place, data_format, self.dtype, - [2, 3, 4, 5]) + self.check_with_place( + place, data_format, self.dtype, [2, 3, 4, 5] + ) self.check_with_place(place, data_format, self.dtype, [2, 3]) def init_kernel_type(self): @@ -337,7 +355,6 @@ class TestBatchNormOpInference(unittest.TestCase): class TestFP16BatchNormOpInference(TestBatchNormOpInference): - def setUp(self): self.dtype = np.float16 self.use_mkldnn = False @@ -351,15 +368,15 @@ class TestFP16BatchNormOpInference(TestBatchNormOpInference): if core.is_float16_supported(place): places.append(place) for place in places: - #for data_format in ["NCHW", "NHWC"]: + # for data_format in ["NCHW", "NHWC"]: for data_format in ["NCHW"]: - self.check_with_place(place, data_format, self.dtype, - [2, 3, 4, 5]) + self.check_with_place( + place, data_format, self.dtype, [2, 3, 4, 5] + ) self.check_with_place(place, data_format, self.dtype, [2, 3]) class TestBatchNormOpTraining(unittest.TestCase): - def setUp(self): self.use_mkldnn = False self.fuse_with_relu = False @@ -374,27 +391,54 @@ class TestBatchNormOpTraining(unittest.TestCase): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def __assert_close(self, tensor, np_array, msg, atol=1e-4): np.allclose(np.array(tensor), np_array, atol=atol) - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): # run forward - y, saved_mean, var_ref = _reference_training(x, scale, bias, epsilon, - data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = var_ref * (1. - momentum) + momentum * variance - saved_variance = 1. / np.sqrt(var_ref + epsilon) + y, saved_mean, var_ref = _reference_training( + x, scale, bias, epsilon, data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = var_ref * (1.0 - momentum) + momentum * variance + saved_variance = 1.0 / np.sqrt(var_ref + epsilon) # run backward - x_grad, scale_grad, bias_grad = _reference_grad(x, y_grad, scale, - saved_mean, var_ref, - epsilon, data_layout) - - return y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad + x_grad, scale_grad, bias_grad = _reference_grad( + x, y_grad, scale, saved_mean, var_ref, epsilon, data_layout + ) + + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) def set_mean_variance(self, scale_shape, x, data_layout): mean, variance = _cal_mean_variance(x, self.epsilon, data_layout) @@ -403,12 +447,11 @@ class TestBatchNormOpTraining(unittest.TestCase): # computing global mean/variance for one step if self.use_global_stats: mom = self.momentum - mean = mean * (1. - mom) + mom * mean_pre - variance = variance * (1. - mom) + mom * variance_pre + mean = mean * (1.0 - mom) + mom * mean_pre + variance = variance * (1.0 - mom) + mom * variance_pre return mean, variance def test_forward_backward(self): - def test_with_place(place, data_layout, shape): # attr epsilon = self.epsilon @@ -427,9 +470,27 @@ class TestBatchNormOpTraining(unittest.TestCase): y_grad = np.random.random_sample(shape).astype(np.float32) momentum_var = np.array([momentum]).astype(np.float32) - y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad = self.ref_forward_backward( - x, y_grad, scale, bias, mean, variance, epsilon, momentum, - shape, data_layout) + ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) = self.ref_forward_backward( + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -438,8 +499,15 @@ class TestBatchNormOpTraining(unittest.TestCase): var_dict['bias@GRAD'] = bias_grad var_names = [ - 'x', 'scale', 'bias', 'mean', 'variance', 'y', 'saved_mean', - 'saved_variance', 'momentum_var' + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y', + 'saved_mean', + 'saved_variance', + 'momentum_var', ] ground_truth = {name: var_dict[name] for name in var_names} @@ -447,15 +515,17 @@ class TestBatchNormOpTraining(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = { "X": block.var('x'), "Scale": block.var('scale'), "Bias": block.var('bias'), "Mean": block.var('mean'), - "Variance": block.var('variance') + "Variance": block.var('variance'), } attrs = { "epsilon": epsilon, @@ -463,7 +533,7 @@ class TestBatchNormOpTraining(unittest.TestCase): "data_layout": data_layout, "use_mkldnn": self.use_mkldnn, "fuse_with_relu": self.fuse_with_relu, - "use_global_stats": self.use_global_stats + "use_global_stats": self.use_global_stats, } if self.use_momentum_variable: inputs['MomentumTensor'] = block.var('momentum_var') @@ -475,19 +545,22 @@ class TestBatchNormOpTraining(unittest.TestCase): "MeanOut": block.var('mean'), # share memory "VarianceOut": block.var('variance'), # share memory "SavedMean": block.var('saved_mean'), - "SavedVariance": block.var('saved_variance') + "SavedVariance": block.var('saved_variance'), } block.create_var(name="reserve_space", dtype='float32') outputs["ReserveSpace"] = block.var('reserve_space') - bn_op = block.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + bn_op = block.append_op( + type="batch_norm", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) block.create_var(name='y@GRAD', dtype='float32', shape=y.shape) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - bn_op.desc, self.no_grad_set, []) + bn_op.desc, self.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -502,22 +575,28 @@ class TestBatchNormOpTraining(unittest.TestCase): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in [ - 'x', 'scale', 'bias', 'mean', 'variance', - 'y@GRAD', 'momentum_var' - ] - }, - fetch_list=self.fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in [ + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y@GRAD', + 'momentum_var', + ] + }, + fetch_list=self.fetch_list, + ) for id, name in enumerate(self.fetch_list): if name == 'variance': - self.__assert_close(var_dict[name], - out[id], - name, - atol=1e-3) + self.__assert_close( + var_dict[name], out[id], name, atol=1e-3 + ) continue self.__assert_close(var_dict[name], out[id], name) print("op test forward passed: ", str(place), data_layout) @@ -536,7 +615,6 @@ class TestBatchNormOpTraining(unittest.TestCase): class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -544,19 +622,23 @@ class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): class TestBatchNormOpTrainingCase2(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] os.environ['FLAGS_cudnn_batchnorm_spatial_persistent'] = "1" class TestBatchNormOpTrainingCase3(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['x@GRAD']) @@ -564,24 +646,33 @@ class TestBatchNormOpTrainingCase3(TestBatchNormOpTraining): class TestBatchNormOpTrainingMomentumVariable(TestBatchNormOpTraining): - def init_test_case(self): self.use_momentum_variable = True self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = True self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'x@GRAD', 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): @@ -590,8 +681,9 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): y_grad = np.transpose(y_grad, (0, 2, 3, 1)) x_grad = scale * y_grad / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -602,8 +694,19 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): return x_grad, grad_scale, grad_offset - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): if data_layout != "NCHW" and data_layout != "NHWC": raise ValueError("Unknown data order.") @@ -621,17 +724,27 @@ class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): mean_out = mean variance_out = variance - saved_variance = 1. / np.sqrt(variance + epsilon) + saved_variance = 1.0 / np.sqrt(variance + epsilon) # run backward x_grad, scale_grad, bias_grad = self.reference_grad( - x, y_grad, scale, mean, variance, epsilon, data_layout) + x, y_grad, scale, mean, variance, epsilon, data_layout + ) - return y, mean_out, variance_out, mean, saved_variance, x_grad, scale_grad, bias_grad + return ( + y, + mean_out, + variance_out, + mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) class TestBatchNormOpFreezeStatsAndScaleBiasTraining( - TestBatchNormOpFreezeStatsTraining): - + TestBatchNormOpFreezeStatsTraining +): def init_test_case(self): self.use_global_stats = True self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -639,12 +752,12 @@ class TestBatchNormOpFreezeStatsAndScaleBiasTraining( class TestBatchNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of batch_norm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.batch_norm, x1) # the input dtype of batch_norm must be float16 or float32 or float64 @@ -654,13 +767,13 @@ class TestBatchNormOpError(unittest.TestCase): class TestDygraphBatchNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_norm = fluid.dygraph.BatchNorm(10) # the input of BatchNorm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, batch_norm, x1) # the input dtype of BatchNorm must be float16 or float32 or float64 @@ -670,7 +783,6 @@ class TestDygraphBatchNormAPIError(unittest.TestCase): class TestDygraphBatchNormTrainableStats(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -683,7 +795,8 @@ class TestDygraphBatchNormTrainableStats(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -705,7 +818,8 @@ class TestDygraphBatchNormTrainableStats(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) @@ -719,7 +833,6 @@ class TestDygraphBatchNormTrainableStats(unittest.TestCase): class TestDygraphBatchNormOpenReserveSpace(unittest.TestCase): - def test_reservespace(self): with program_guard(Program(), Program()): paddle.enable_static() @@ -734,5 +847,6 @@ class TestDygraphBatchNormOpenReserveSpace(unittest.TestCase): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_batch_norm_op_v2.py index ed3b0560b52..778489eb668 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_op_v2.py @@ -23,7 +23,6 @@ import paddle class TestBatchNorm(unittest.TestCase): - def test_name(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -37,7 +36,7 @@ class TestBatchNorm(unittest.TestCase): if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) for p in places: - #paddle.disable_static() + # paddle.disable_static() x_data_4 = np.random.random(size=(2, 1, 3, 3)).astype('float32') x_data_3 = np.random.random(size=(2, 1, 3)).astype('float32') @@ -80,7 +79,6 @@ class TestBatchNorm(unittest.TestCase): self.assertRaises(ValueError, error3d_dataformat) def test_large_batch(self): - def compute_baseline(x): with fluid.dygraph.guard(p): bn = fluid.dygraph.BatchNorm(shape[1]) @@ -130,7 +128,7 @@ class TestBatchNorm(unittest.TestCase): def compute_v1(x): with fluid.dygraph.guard(p): bn = fluid.dygraph.BatchNorm(shape[1]) - #bn = paddle.nn.BatchNorm2D(shape[1]) + # bn = paddle.nn.BatchNorm2D(shape[1]) x1 = paddle.to_tensor(x) x1.stop_gradient = False y = bn(x1) @@ -166,7 +164,8 @@ class TestBatchNorm(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(paddle.to_tensor(x)) return y.numpy() @@ -188,19 +187,22 @@ class TestBatchNorm(unittest.TestCase): is_test=is_test, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(1.0), - trainable=False), + trainable=False, + ), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(0.0), - trainable=False), - trainable_statistics=trainable_statistics) + trainable=False, + ), + trainable_statistics=trainable_statistics, + ) y = bn(paddle.to_tensor(x)) return y.numpy() def compute_v4(x): with fluid.dygraph.guard(p): - bn = paddle.nn.BatchNorm2D(shape[1], - weight_attr=False, - bias_attr=False) + bn = paddle.nn.BatchNorm2D( + shape[1], weight_attr=False, bias_attr=False + ) y = bn(paddle.to_tensor(x)) return y.numpy() @@ -225,7 +227,8 @@ class TestBatchNorm(unittest.TestCase): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) @@ -248,7 +251,6 @@ class TestBatchNorm(unittest.TestCase): class TestBatchNormChannelLast(unittest.TestCase): - def setUp(self): self.original_dtyep = paddle.get_default_dtype() # MIOPEN not support data type of double @@ -277,14 +279,13 @@ class TestBatchNormChannelLast(unittest.TestCase): y2 = paddle.transpose(y2, [0, 2, 1]) if core.is_compiled_with_rocm(): # HIP will fail if no atol - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) else: - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05 + ) def test_2d(self): for p in self.places: @@ -300,14 +301,13 @@ class TestBatchNormChannelLast(unittest.TestCase): y2 = paddle.transpose(y2, [0, 2, 3, 1]) if core.is_compiled_with_rocm(): # HIP will fail if no atol - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) else: - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05 + ) def test_3d(self): for p in self.places: @@ -323,14 +323,13 @@ class TestBatchNormChannelLast(unittest.TestCase): y2 = paddle.transpose(y2, [0, 2, 3, 4, 1]) if core.is_compiled_with_rocm(): # HIP will fail if no atol - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) else: - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05 + ) def test_1d_opt(self): with fluid.dygraph.guard(): @@ -352,18 +351,18 @@ class TestBatchNormChannelLast(unittest.TestCase): y.backward() y2.backward() - assert np.allclose(y.numpy().flatten(), - y2.numpy().flatten(), - atol=1e-5, - rtol=1e-5) - assert np.allclose(bn1d.weight.grad.numpy().flatten(), - bn2d.weight.grad.numpy().flatten(), - atol=1e-5, - rtol=1e-5) + assert np.allclose( + y.numpy().flatten(), y2.numpy().flatten(), atol=1e-5, rtol=1e-5 + ) + assert np.allclose( + bn1d.weight.grad.numpy().flatten(), + bn2d.weight.grad.numpy().flatten(), + atol=1e-5, + rtol=1e-5, + ) class TestBatchNormUseGlobalStats(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -382,11 +381,14 @@ class TestBatchNormUseGlobalStats(unittest.TestCase): net1 = paddle.fluid.dygraph.BatchNorm( 6, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.0)), + initializer=fluid.initializer.Constant(1.0) + ), use_global_stats=self.use_global_stats, - trainable_statistics=self.trainable_statistics) + trainable_statistics=self.trainable_statistics, + ) net2 = paddle.nn.BatchNorm2D( - 6, use_global_stats=self.use_global_stats) + 6, use_global_stats=self.use_global_stats + ) net2.weight = net1.weight net2.bias = net1.bias if self.trainable_statistics == True: @@ -420,5 +422,6 @@ class TestBatchNormUseGlobalStatsCase3(TestBatchNormUseGlobalStats): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_batch_sampler.py b/python/paddle/fluid/tests/unittests/test_batch_sampler.py index 833d96e8e3c..a848898c53f 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_sampler.py +++ b/python/paddle/fluid/tests/unittests/test_batch_sampler.py @@ -15,14 +15,19 @@ import unittest import numpy as np -from paddle.io import BatchSampler, Dataset, Sampler, SequenceSampler, \ - RandomSampler, WeightedRandomSampler +from paddle.io import ( + BatchSampler, + Dataset, + Sampler, + SequenceSampler, + RandomSampler, + WeightedRandomSampler, +) IMAGE_SIZE = 32 class RandomDataset(Dataset): - def __init__(self, sample_num, class_num): self.sample_num = sample_num self.class_num = class_num @@ -30,7 +35,7 @@ class RandomDataset(Dataset): def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -38,7 +43,6 @@ class RandomDataset(Dataset): class TestSampler(unittest.TestCase): - def test_main(self): dataset = RandomDataset(100, 10) sampler = Sampler(dataset) @@ -50,7 +54,6 @@ class TestSampler(unittest.TestCase): class TestSequenceSampler(unittest.TestCase): - def test_main(self): dataset = RandomDataset(100, 10) sampler = SequenceSampler(dataset) @@ -61,7 +64,6 @@ class TestSequenceSampler(unittest.TestCase): class TestRandomSampler(unittest.TestCase): - def test_main(self): dataset = RandomDataset(100, 10) sampler = RandomSampler(dataset) @@ -96,10 +98,9 @@ class TestRandomSampler(unittest.TestCase): def test_with_generator_num_samples(self): dataset = RandomDataset(100, 10) generator = iter(range(0, 60)) - sampler = RandomSampler(dataset, - generator=generator, - num_samples=50, - replacement=True) + sampler = RandomSampler( + dataset, generator=generator, num_samples=50, replacement=True + ) assert len(sampler) == 50 rets = [] @@ -109,7 +110,6 @@ class TestRandomSampler(unittest.TestCase): class TestBatchSampler(unittest.TestCase): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -119,17 +119,20 @@ class TestBatchSampler(unittest.TestCase): def init_batch_sampler(self): dataset = RandomDataset(self.num_samples, self.num_classes) - bs = BatchSampler(dataset=dataset, - batch_size=self.batch_size, - shuffle=self.shuffle, - drop_last=self.drop_last) + bs = BatchSampler( + dataset=dataset, + batch_size=self.batch_size, + shuffle=self.shuffle, + drop_last=self.drop_last, + ) return bs def test_main(self): bs = self.init_batch_sampler() # length check - bs_len = (self.num_samples + int(not self.drop_last) \ - * (self.batch_size - 1)) // self.batch_size + bs_len = ( + self.num_samples + int(not self.drop_last) * (self.batch_size - 1) + ) // self.batch_size self.assertTrue(bs_len == len(bs)) # output indices check @@ -142,7 +145,6 @@ class TestBatchSampler(unittest.TestCase): class TestBatchSamplerDropLast(TestBatchSampler): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -152,7 +154,6 @@ class TestBatchSamplerDropLast(TestBatchSampler): class TestBatchSamplerShuffle(TestBatchSampler): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -162,18 +163,18 @@ class TestBatchSamplerShuffle(TestBatchSampler): class TestBatchSamplerWithSampler(TestBatchSampler): - def init_batch_sampler(self): dataset = RandomDataset(1000, 10) sampler = SequenceSampler(dataset) - bs = BatchSampler(sampler=sampler, - batch_size=self.batch_size, - drop_last=self.drop_last) + bs = BatchSampler( + sampler=sampler, + batch_size=self.batch_size, + drop_last=self.drop_last, + ) return bs class TestBatchSamplerWithSamplerDropLast(unittest.TestCase): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -183,7 +184,6 @@ class TestBatchSamplerWithSamplerDropLast(unittest.TestCase): class TestBatchSamplerWithSamplerShuffle(unittest.TestCase): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -195,20 +195,21 @@ class TestBatchSamplerWithSamplerShuffle(unittest.TestCase): try: dataset = RandomDataset(self.num_samples, self.num_classes) sampler = RandomSampler(dataset) - bs = BatchSampler(sampler=sampler, - shuffle=self.shuffle, - batch_size=self.batch_size, - drop_last=self.drop_last) + bs = BatchSampler( + sampler=sampler, + shuffle=self.shuffle, + batch_size=self.batch_size, + drop_last=self.drop_last, + ) self.assertTrue(False) except AssertionError: pass class TestWeightedRandomSampler(unittest.TestCase): - def init_probs(self, total, pos): - pos_probs = np.random.random((pos, )).astype('float32') - probs = np.zeros((total, )).astype('float32') + pos_probs = np.random.random((pos,)).astype('float32') + probs = np.zeros((total,)).astype('float32') probs[:pos] = pos_probs np.random.shuffle(probs) return probs @@ -218,7 +219,7 @@ class TestWeightedRandomSampler(unittest.TestCase): sampler = WeightedRandomSampler(probs, 30, True) assert len(sampler) == 30 for idx in iter(sampler): - assert probs[idx] > 0. + assert probs[idx] > 0.0 def test_no_replacement(self): probs = self.init_probs(20, 10) @@ -226,13 +227,13 @@ class TestWeightedRandomSampler(unittest.TestCase): assert len(sampler) == 10 idxs = [] for idx in iter(sampler): - assert probs[idx] > 0. + assert probs[idx] > 0.0 idxs.append(idx) assert len(set(idxs)) == len(idxs) def test_assert(self): # all zeros - probs = np.zeros((10, )).astype('float32') + probs = np.zeros((10,)).astype('float32') sampler = WeightedRandomSampler(probs, 10, True) try: for idx in iter(sampler): @@ -252,7 +253,7 @@ class TestWeightedRandomSampler(unittest.TestCase): self.assertTrue(True) # neg probs - probs = -1.0 * np.ones((10, )).astype('float32') + probs = -1.0 * np.ones((10,)).astype('float32') sampler = WeightedRandomSampler(probs, 10, True) try: for idx in iter(sampler): diff --git a/python/paddle/fluid/tests/unittests/test_bce_loss.py b/python/paddle/fluid/tests/unittests/test_bce_loss.py index 417f4795c4e..c77196c1d0d 100644 --- a/python/paddle/fluid/tests/unittests/test_bce_loss.py +++ b/python/paddle/fluid/tests/unittests/test_bce_loss.py @@ -19,88 +19,76 @@ import unittest from op_test import OpTest -def test_static_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float64') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float64') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float64' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float64' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float64') - bce_loss = paddle.nn.loss.BCELoss(weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float64' + ) + bce_loss = paddle.nn.loss.BCELoss( + weight=weight, reduction=reduction + ) else: bce_loss = paddle.nn.loss.BCELoss(reduction=reduction) res = bce_loss(input, label) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result -def test_static_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float64') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float64') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float64' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float64' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float64') - res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float64' + ) + res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result -def test_dygraph_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static() if weight_np is not None: weight = paddle.to_tensor(weight_np) @@ -113,25 +101,22 @@ def test_dygraph_layer(place, return dy_result -def test_dygraph_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static() input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) if weight_np is not None: weight = paddle.to_tensor(weight_np) - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -139,11 +124,19 @@ def test_dygraph_functional(place, def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): if weight_np is None: - expected = -1 * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = -1 * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) else: - expected = -1 * weight_np * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = ( + -1 + * weight_np + * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) + ) if reduction == 'mean': expected = np.mean(expected) @@ -156,7 +149,6 @@ def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): class TestBCELoss(unittest.TestCase): - def test_BCELoss(self): input_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float64) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float64) @@ -166,89 +158,90 @@ class TestBCELoss(unittest.TestCase): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static_layer(place, input_np, label_np, - reduction) - dy_result = test_dygraph_layer(place, input_np, label_np, - reduction) + static_result = test_static_layer( + place, input_np, label_np, reduction + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction + ) expected = calc_bceloss(input_np, label_np, reduction) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) static_functional = test_static_functional( - place, input_np, label_np, reduction) - dy_functional = test_dygraph_functional(place, input_np, - label_np, reduction) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + place, input_np, label_np, reduction + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_BCELoss_weight(self): - input_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float64) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float64) + input_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float64 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float64 + ) weight_np = np.random.random(size=(3, 4, 10)).astype(np.float64) - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) for reduction in ['sum', 'mean', 'none']: - static_result = test_static_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_result = test_dygraph_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - expected = calc_bceloss(input_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + expected = calc_bceloss( + input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_functional = test_dygraph_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) + static_functional = test_static_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_BCELoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.loss.BCELoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, paddle.nn.loss.BCELoss, reduction="unsupport reduction" + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def bce_loss(input, label): - return -1 * (label * np.log(input) + (1. - label) * np.log(1. - input)) + return -1 * (label * np.log(input) + (1.0 - label) * np.log(1.0 - input)) class TestBceLossOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "bce_loss" @@ -270,13 +263,11 @@ class TestBceLossOp(OpTest): class TestBceLossOpCase1(OpTest): - def init_test_cast(self): self.shape = [2, 3, 4, 5] class TestBceLossOpCase2(OpTest): - def init_test_cast(self): self.shape = [2, 3, 20] diff --git a/python/paddle/fluid/tests/unittests/test_bce_with_logits_loss.py b/python/paddle/fluid/tests/unittests/test_bce_with_logits_loss.py index 55da8aea9b5..8d921215bfa 100644 --- a/python/paddle/fluid/tests/unittests/test_bce_with_logits_loss.py +++ b/python/paddle/fluid/tests/unittests/test_bce_with_logits_loss.py @@ -19,76 +19,78 @@ import unittest from paddle.fluid.framework import _test_eager_guard -def call_bce_layer(logit, - label, - weight=None, - reduction='mean', - pos_weight=None): - bce_logit_loss = paddle.nn.loss.BCEWithLogitsLoss(weight=weight, - reduction=reduction, - pos_weight=pos_weight) +def call_bce_layer( + logit, label, weight=None, reduction='mean', pos_weight=None +): + bce_logit_loss = paddle.nn.loss.BCEWithLogitsLoss( + weight=weight, reduction=reduction, pos_weight=pos_weight + ) res = bce_logit_loss(logit, label) return res -def call_bce_functional(logit, - label, - weight=None, - reduction='mean', - pos_weight=None): +def call_bce_functional( + logit, label, weight=None, reduction='mean', pos_weight=None +): res = paddle.nn.functional.binary_cross_entropy_with_logits( - logit, label, weight=weight, reduction=reduction, pos_weight=pos_weight) + logit, label, weight=weight, reduction=reduction, pos_weight=pos_weight + ) return res -def test_static(place, - logit_np, - label_np, - weight_np=None, - reduction='mean', - pos_weight_np=None, - functional=False): +def test_static( + place, + logit_np, + label_np, + weight_np=None, + reduction='mean', + pos_weight_np=None, + functional=False, +): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - logit = paddle.fluid.data(name='logit', - shape=logit_np.shape, - dtype='float64') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float64') + logit = paddle.fluid.data( + name='logit', shape=logit_np.shape, dtype='float64' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float64' + ) feed_dict = {"logit": logit_np, "label": label_np} pos_weight = None weight = None if pos_weight_np is not None: - pos_weight = paddle.fluid.data(name='pos_weight', - shape=pos_weight_np.shape, - dtype='float64') + pos_weight = paddle.fluid.data( + name='pos_weight', shape=pos_weight_np.shape, dtype='float64' + ) feed_dict["pos_weight"] = pos_weight_np if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float64') + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float64' + ) feed_dict["weight"] = weight_np if functional: - res = call_bce_functional(logit, label, weight, reduction, - pos_weight) + res = call_bce_functional( + logit, label, weight, reduction, pos_weight + ) else: res = call_bce_layer(logit, label, weight, reduction, pos_weight) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, feed=feed_dict, fetch_list=[res]) + (static_result,) = exe.run(prog, feed=feed_dict, fetch_list=[res]) return static_result -def test_dygraph(place, - logit_np, - label_np, - weight_np=None, - reduction='mean', - pos_weight_np=None, - functional=False): +def test_dygraph( + place, + logit_np, + label_np, + weight_np=None, + reduction='mean', + pos_weight_np=None, + functional=False, +): with paddle.fluid.dygraph.base.guard(): logit = paddle.to_tensor(logit_np) label = paddle.to_tensor(label_np) @@ -99,22 +101,23 @@ def test_dygraph(place, if pos_weight_np is not None: pos_weight = paddle.to_tensor(pos_weight_np) if functional: - dy_res = call_bce_functional(logit, label, weight, reduction, - pos_weight) + dy_res = call_bce_functional( + logit, label, weight, reduction, pos_weight + ) else: dy_res = call_bce_layer(logit, label, weight, reduction, pos_weight) dy_result = dy_res.numpy() return dy_result -def calc_bce_with_logits_loss(logit_np, - label_np, - reduction='mean', - weight_np=None, - pos_weight=None): - expected = np.maximum( - logit_np, - 0) - logit_np * label_np + np.log(1 + np.exp(-np.abs(logit_np))) +def calc_bce_with_logits_loss( + logit_np, label_np, reduction='mean', weight_np=None, pos_weight=None +): + expected = ( + np.maximum(logit_np, 0) + - logit_np * label_np + + np.log(1 + np.exp(-np.abs(logit_np))) + ) if pos_weight is not None: expected = expected * ((pos_weight - 1) * label_np + 1) if weight_np is not None: @@ -131,7 +134,6 @@ def calc_bce_with_logits_loss(logit_np, class TestBCEWithLogitsLoss(unittest.TestCase): - def test_BCEWithLogitsLoss(self): logit_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float64) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float64) @@ -141,141 +143,174 @@ class TestBCEWithLogitsLoss(unittest.TestCase): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static(place, - logit_np, - label_np, - reduction=reduction) - dy_result = test_dygraph(place, - logit_np, - label_np, - reduction=reduction) - expected = calc_bce_with_logits_loss(logit_np, label_np, - reduction) + static_result = test_static( + place, logit_np, label_np, reduction=reduction + ) + dy_result = test_dygraph( + place, logit_np, label_np, reduction=reduction + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place, - logit_np, - label_np, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - reduction=reduction, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) with _test_eager_guard(): - eager_functional = test_dygraph(place, - logit_np, - label_np, - reduction=reduction, - functional=True) + eager_functional = test_dygraph( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) - np.testing.assert_allclose(eager_functional, - expected, - rtol=1e-05) + np.testing.assert_allclose( + eager_functional, expected, rtol=1e-05 + ) def test_BCEWithLogitsLoss_weight(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float64) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float64) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float64 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float64 + ) weight_np = np.random.random(size=(2, 3, 4, 10)).astype(np.float64) - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) for reduction in ['sum', 'mean', 'none']: - static_result = test_static(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction) - dy_result = test_dygraph(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction) - expected = calc_bce_with_logits_loss(logit_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + ) + dy_result = test_dygraph( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_BCEWithLogitsLoss_pos_weight(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float64) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float64) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float64 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float64 + ) pos_weight_np = np.random.random(size=(3, 4, 10)).astype(np.float64) weight_np = np.random.random(size=(2, 3, 4, 10)).astype(np.float64) - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) reduction = "mean" - static_result = test_static(place, logit_np, label_np, weight_np, - reduction, pos_weight_np) - dy_result = test_dygraph(place, logit_np, label_np, weight_np, - reduction, pos_weight_np) - expected = calc_bce_with_logits_loss(logit_np, label_np, reduction, - weight_np, pos_weight_np) + static_result = test_static( + place, logit_np, label_np, weight_np, reduction, pos_weight_np + ) + dy_result = test_dygraph( + place, logit_np, label_np, weight_np, reduction, pos_weight_np + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction, weight_np, pos_weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place, - logit_np, - label_np, - weight_np, - reduction, - pos_weight_np, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - weight_np, - reduction, - pos_weight_np, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + weight_np, + reduction, + pos_weight_np, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + weight_np, + reduction, + pos_weight_np, + functional=True, + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-05) np.testing.assert_allclose(static_functional, dy_functional, rtol=1e-05) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_BCEWithLogitsLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.BCEWithLogitsLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.BCEWithLogitsLoss, + reduction="unsupport reduction", + ) logit = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy_with_logits, - logit=logit, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy_with_logits, + logit=logit, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py index c2a379f9982..ee1e2fd6458 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py @@ -41,28 +41,41 @@ class TestBeamSearchDecodeOp(unittest.TestCase): # beam_size = 2, end_id = 1 # start with start_id [ - self.append_lod_tensor(array, [[0, 1, 2], [0, 1, 2]], - np.array([0, 0], dtype=dtype)) + self.append_lod_tensor( + array, [[0, 1, 2], [0, 1, 2]], np.array([0, 0], dtype=dtype) + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 1, 2], [0, 2, 4]], - np.array([2, 3, 4, 5], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 1, 2], [0, 2, 4]], + np.array([2, 3, 4, 5], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 2, 2, 4, 4]], - np.array([3, 1, 5, 4], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 2, 2, 4, 4]], + np.array([3, 1, 5, 4], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 1, 2, 3, 4]], - np.array([1, 1, 3, 5], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 1, 2, 3, 4]], + np.array([1, 1, 3, 5], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 0, 0, 2, 2]], - np.array([5, 1], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 0, 0, 2, 2]], + np.array([5, 1], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] @@ -88,22 +101,22 @@ class TestBeamSearchDecodeOp(unittest.TestCase): self.assertEqual(sentence_scores.lod(), expected_lod) expected_data = np.array( - [0, 2, 3, 1, 0, 2, 1, 0, 4, 5, 3, 5, 0, 4, 5, 3, 1], "int64") + [0, 2, 3, 1, 0, 2, 1, 0, 4, 5, 3, 5, 0, 4, 5, 3, 1], "int64" + ) np.testing.assert_array_equal(np.array(sentence_ids), expected_data) np.testing.assert_array_equal(np.array(sentence_scores), expected_data) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestBeamSearchDecodeOpGPU(TestBeamSearchDecodeOp): - def setUp(self): self.scope = core.Scope() self.place = core.CUDAPlace(0) class TestBeamSearchDecodeOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -111,10 +124,9 @@ class TestBeamSearchDecodeOpError(unittest.TestCase): # the input pre_ids must be Variable test_ids = np.random.randint(1, 5, [5, 1]).astype("int64") scores = fluid.layers.create_array(dtype='float32') - fluid.layers.beam_search_decode(test_ids, - scores, - beam_size=5, - end_id=0) + fluid.layers.beam_search_decode( + test_ids, scores, beam_size=5, end_id=0 + ) self.assertRaises(TypeError, test_id_Variable) @@ -122,10 +134,9 @@ class TestBeamSearchDecodeOpError(unittest.TestCase): # the input pre_scores must be Variable ids = fluid.layers.create_array(dtype='int64') test_scores = np.random.uniform(1, 5, [5, 1]).astype("float32") - fluid.layers.beam_search_decode(ids, - test_scores, - beam_size=5, - end_id=0) + fluid.layers.beam_search_decode( + ids, test_scores, beam_size=5, end_id=0 + ) self.assertRaises(TypeError, test_score_Variable) @@ -133,10 +144,9 @@ class TestBeamSearchDecodeOpError(unittest.TestCase): # the dtype of input pre_ids must be int64 type_ids = fluid.layers.create_array(dtype='float32') scores = fluid.layers.create_array(dtype='float32') - fluid.layers.beam_search_decode(type_ids, - scores, - beam_size=5, - end_id=0) + fluid.layers.beam_search_decode( + type_ids, scores, beam_size=5, end_id=0 + ) self.assertRaises(TypeError, test_id_dtype) @@ -144,10 +154,9 @@ class TestBeamSearchDecodeOpError(unittest.TestCase): # the dtype of input pre_scores must be float32 ids = fluid.layers.create_array(dtype='int64') type_scores = fluid.layers.create_array(dtype='int64') - fluid.layers.beam_search_decode(ids, - type_scores, - beam_size=5, - end_id=0) + fluid.layers.beam_search_decode( + ids, type_scores, beam_size=5, end_id=0 + ) self.assertRaises(TypeError, test_score_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_op.py index a274ae92d3b..7d99fad6422 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_op.py @@ -41,32 +41,34 @@ class BeamSearchOpTester(unittest.TestCase): self.scope.var('parent_idx').get_tensor() def test_run(self): - op = Operator('beam_search', - pre_ids='pre_ids', - pre_scores='pre_scores', - ids='ids', - scores='scores', - selected_ids='selected_ids', - selected_scores='selected_scores', - parent_idx='parent_idx', - level=0, - beam_size=self.beam_size, - end_id=0, - is_accumulated=self.is_accumulated) + op = Operator( + 'beam_search', + pre_ids='pre_ids', + pre_scores='pre_scores', + ids='ids', + scores='scores', + selected_ids='selected_ids', + selected_scores='selected_scores', + parent_idx='parent_idx', + level=0, + beam_size=self.beam_size, + end_id=0, + is_accumulated=self.is_accumulated, + ) op.run(self.scope, core.CPUPlace()) selected_ids = self.scope.find_var("selected_ids").get_tensor() selected_scores = self.scope.find_var("selected_scores").get_tensor() parent_idx = self.scope.find_var("parent_idx").get_tensor() - np.testing.assert_allclose(np.array(selected_ids), - self.output_ids, - rtol=1e-05) - np.testing.assert_allclose(np.array(selected_scores), - self.output_scores, - rtol=1e-05) + np.testing.assert_allclose( + np.array(selected_ids), self.output_ids, rtol=1e-05 + ) + np.testing.assert_allclose( + np.array(selected_scores), self.output_scores, rtol=1e-05 + ) self.assertEqual(selected_ids.lod(), self.output_lod) - np.testing.assert_allclose(np.array(parent_idx), - self.output_parent_idx, - rtol=1e-05) + np.testing.assert_allclose( + np.array(parent_idx), self.output_parent_idx, rtol=1e-05 + ) def _create_pre_ids(self): np_data = np.array([[1, 2, 3, 4]], dtype='int64') @@ -78,19 +80,22 @@ class BeamSearchOpTester(unittest.TestCase): def _create_ids(self): self.lod = [[0, 2, 4], [0, 1, 2, 3, 4]] - np_data = np.array([[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], - dtype='int64') + np_data = np.array( + [[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], dtype='int64' + ) tensor = create_tensor(self.scope, "ids", np_data) tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.5, 0.3, 0.2], - [0.6, 0.3, 0.1], - [0.9, 0.5, 0.1], - [0.7, 0.5, 0.1], - ], - dtype='float32') + np_data = np.array( + [ + [0.5, 0.3, 0.2], + [0.6, 0.3, 0.1], + [0.9, 0.5, 0.1], + [0.7, 0.5, 0.1], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -104,7 +109,6 @@ class BeamSearchOpTester(unittest.TestCase): class BeamSearchOpTester2(BeamSearchOpTester): - def _create_pre_ids(self): np_data = np.array([[1], [2], [3], [4]], dtype='int64') tensor = create_tensor(self.scope, 'pre_ids', np_data) @@ -120,13 +124,15 @@ class BeamSearchOpTester2(BeamSearchOpTester): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -156,13 +162,15 @@ class BeamSearchOpTester3(BeamSearchOpTester): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.6, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.6, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -192,13 +200,15 @@ class BeamSearchOpTester4(BeamSearchOpTester): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.6, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.6, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -228,13 +238,15 @@ class BeamSearchOpTester5(BeamSearchOpTester): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -242,8 +254,9 @@ class BeamSearchOpTester5(BeamSearchOpTester): self.beam_size = 2 self.is_accumulated = False self.output_ids = np.array([7, 3, 3, 1])[:, np.newaxis] - self.output_scores = np.array([1.50685, 0.996027, 0.194639, - 0.043325])[:, np.newaxis] + self.output_scores = np.array([1.50685, 0.996027, 0.194639, 0.043325])[ + :, np.newaxis + ] self.output_lod = [[0, 2, 4], [0, 0, 2, 3, 4]] self.output_parent_idx = np.array([1, 1, 2, 3]) @@ -265,13 +278,15 @@ class BeamSearchOpTester6(BeamSearchOpTester): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -285,100 +300,115 @@ class BeamSearchOpTester6(BeamSearchOpTester): class TestBeamSearchOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - pre_ids = fluid.data(name='pre_id', - shape=[1], - lod_level=2, - dtype='int64') - pre_scores = fluid.data(name='pre_scores', - shape=[1], - lod_level=2, - dtype='float32') + pre_ids = fluid.data( + name='pre_id', shape=[1], lod_level=2, dtype='int64' + ) + pre_scores = fluid.data( + name='pre_scores', shape=[1], lod_level=2, dtype='float32' + ) probs = fluid.data(name='probs', shape=[10000], dtype='float32') topk_scores, topk_indices = fluid.layers.topk(probs, k=4) accu_scores = fluid.layers.elementwise_add( x=fluid.layers.log(x=topk_scores), y=fluid.layers.reshape(pre_scores, shape=[-1]), - axis=0) + axis=0, + ) def test_preids_Variable(): # the input pre_ids must be Variable preids_data = np.random.randint(1, 5, [5, 1]).astype("int64") - fluid.layers.beam_search(pre_ids=preids_data, - pre_scores=pre_scores, - ids=topk_indices, - scores=accu_scores, - beam_size=4, - end_id=1) + fluid.layers.beam_search( + pre_ids=preids_data, + pre_scores=pre_scores, + ids=topk_indices, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_preids_Variable) def test_prescores_Variable(): # the input pre_scores must be Variable - prescores_data = np.random.uniform(1, 5, - [5, 1]).astype("float32") - fluid.layers.beam_search(pre_ids=pre_ids, - pre_scores=prescores_data, - ids=topk_indices, - scores=accu_scores, - beam_size=4, - end_id=1) + prescores_data = np.random.uniform(1, 5, [5, 1]).astype( + "float32" + ) + fluid.layers.beam_search( + pre_ids=pre_ids, + pre_scores=prescores_data, + ids=topk_indices, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_prescores_Variable) def test_ids_Variable(): # the input ids must be Variable or None ids_data = np.random.randint(1, 5, [5, 1]).astype("int64") - fluid.layers.beam_search(pre_ids=pre_ids, - pre_scores=pre_scores, - ids=ids_data, - scores=accu_scores, - beam_size=4, - end_id=1) + fluid.layers.beam_search( + pre_ids=pre_ids, + pre_scores=pre_scores, + ids=ids_data, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_ids_Variable) def test_scores_Variable(): # the input scores must be Variable scores_data = np.random.uniform(1, 5, [5, 1]).astype("float32") - fluid.layers.beam_search(pre_ids=pre_ids, - pre_scores=pre_scores, - ids=topk_indices, - scores=scores_data, - beam_size=4, - end_id=1) + fluid.layers.beam_search( + pre_ids=pre_ids, + pre_scores=pre_scores, + ids=topk_indices, + scores=scores_data, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_scores_Variable) def test_preids_dtype(): # the dtype of input pre_ids must be int64 - preids_type_data = fluid.data(name='preids_type_data', - shape=[1], - lod_level=2, - dtype='float32') - fluid.layers.beam_search(pre_ids=preids_type_data, - pre_scores=pre_scores, - ids=topk_indices, - scores=accu_scores, - beam_size=4, - end_id=1) + preids_type_data = fluid.data( + name='preids_type_data', + shape=[1], + lod_level=2, + dtype='float32', + ) + fluid.layers.beam_search( + pre_ids=preids_type_data, + pre_scores=pre_scores, + ids=topk_indices, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_preids_dtype) def test_prescores_dtype(): # the dtype of input pre_scores must be float32 - prescores_type_data = fluid.data(name='prescores_type_data', - shape=[1], - lod_level=2, - dtype='int64') - fluid.layers.beam_search(pre_ids=pre_ids, - pre_scores=prescores_type_data, - ids=topk_indices, - scores=accu_scores, - beam_size=4, - end_id=1) + prescores_type_data = fluid.data( + name='prescores_type_data', + shape=[1], + lod_level=2, + dtype='int64', + ) + fluid.layers.beam_search( + pre_ids=pre_ids, + pre_scores=prescores_type_data, + ids=topk_indices, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_prescores_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_bernoulli_op.py b/python/paddle/fluid/tests/unittests/test_bernoulli_op.py index 662fbfffe88..027afdb177a 100644 --- a/python/paddle/fluid/tests/unittests/test_bernoulli_op.py +++ b/python/paddle/fluid/tests/unittests/test_bernoulli_op.py @@ -27,7 +27,6 @@ def output_hist(out): class TestBernoulliOp(OpTest): - def setUp(self): self.op_type = "bernoulli" self.inputs = {"X": np.random.uniform(size=(1000, 784))} @@ -43,7 +42,6 @@ class TestBernoulliOp(OpTest): class TestBernoulliApi(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() x = paddle.rand([1024, 1024]) @@ -56,14 +54,14 @@ class TestBernoulliApi(unittest.TestCase): x = paddle.rand([1024, 1024]) out = paddle.bernoulli(x) exe = paddle.static.Executor(paddle.CPUPlace()) - out = exe.run(paddle.static.default_main_program(), - fetch_list=[out.name]) + out = exe.run( + paddle.static.default_main_program(), fetch_list=[out.name] + ) hist, prob = output_hist(out[0]) np.testing.assert_allclose(hist, prob, rtol=0, atol=0.01) class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -83,7 +81,7 @@ class TestRandomValue(unittest.TestCase): self.assertEqual(np.sum(index0), 260028995) self.assertEqual(np.sum(index1), 8582429431) self.assertEqual(np.sum(index2), 8581445798) - expect = [0., 0., 0., 0., 0., 0., 0., 1., 1., 1.] + expect = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0] np.testing.assert_array_equal(y[16, 500, 500:510], expect) x = paddle.to_tensor(x_np, dtype='float32') @@ -92,7 +90,7 @@ class TestRandomValue(unittest.TestCase): self.assertEqual(np.sum(index0), 260092343) self.assertEqual(np.sum(index1), 8583509076) self.assertEqual(np.sum(index2), 8582778540) - expect = [0., 0., 1., 1., 1., 1., 0., 1., 1., 1.] + expect = [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0] np.testing.assert_array_equal(y[16, 500, 500:510], expect) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_bfgs.py b/python/paddle/fluid/tests/unittests/test_bfgs.py index f7023fba86b..07b15d3aefc 100644 --- a/python/paddle/fluid/tests/unittests/test_bfgs.py +++ b/python/paddle/fluid/tests/unittests/test_bfgs.py @@ -45,37 +45,35 @@ def test_static_graph_H0(func, x0, H0, dtype='float32'): startup = paddle.static.Program() with paddle.static.program_guard(main, startup): X = paddle.static.data(name='x', shape=[x0.shape[0]], dtype=dtype) - H = paddle.static.data(name='h', - shape=[H0.shape[0], H0.shape[1]], - dtype=dtype) - Y = minimize_bfgs(func, - X, - initial_inverse_hessian_estimate=H, - dtype=dtype) + H = paddle.static.data( + name='h', shape=[H0.shape[0], H0.shape[1]], dtype=dtype + ) + Y = minimize_bfgs( + func, X, initial_inverse_hessian_estimate=H, dtype=dtype + ) exe = paddle.static.Executor() exe.run(startup) return exe.run(main, feed={'x': x0, 'h': H0}, fetch_list=[Y]) -def test_dynamic_graph(func, - x0, - H0=None, - line_search_fn='strong_wolfe', - dtype='float32'): +def test_dynamic_graph( + func, x0, H0=None, line_search_fn='strong_wolfe', dtype='float32' +): paddle.disable_static() x0 = paddle.to_tensor(x0) if H0 is not None: H0 = paddle.to_tensor(H0) - return minimize_bfgs(func, - x0, - initial_inverse_hessian_estimate=H0, - line_search_fn=line_search_fn, - dtype=dtype) + return minimize_bfgs( + func, + x0, + initial_inverse_hessian_estimate=H0, + line_search_fn=line_search_fn, + dtype=dtype, + ) class TestBfgs(unittest.TestCase): - def test_quadratic_nd(self): for dimension in [1, 10]: minimum = np.random.random(size=[dimension]).astype('float32') @@ -85,20 +83,19 @@ class TestBfgs(unittest.TestCase): minimum_ = paddle.assign(minimum) scale_ = paddle.assign(scale) return paddle.sum( - paddle.multiply(scale_, (F.square_error_cost(x, minimum_)))) + paddle.multiply(scale_, (F.square_error_cost(x, minimum_))) + ) x0 = np.random.random(size=[dimension]).astype('float32') results = test_static_graph(func=func, x0=x0) - np.testing.assert_allclose(minimum, - results[2], - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + minimum, results[2], rtol=1e-05, atol=1e-8 + ) results = test_dynamic_graph(func=func, x0=x0) - np.testing.assert_allclose(minimum, - results[2].numpy(), - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + minimum, results[2].numpy(), rtol=1e-05, atol=1e-8 + ) def test_inf_minima(self): extream_point = np.array([-1, 2]).astype('float32') @@ -106,16 +103,17 @@ class TestBfgs(unittest.TestCase): def func(x): # df = 3(x - 1.01)(x - 0.99) # f = x^3 - 3x^2 + 3*1.01*0.99x - return x * x * x / 3.0 - ( - extream_point[0] + extream_point[1] - ) * x * x / 2 + extream_point[0] * extream_point[1] * x + return ( + x * x * x / 3.0 + - (extream_point[0] + extream_point[1]) * x * x / 2 + + extream_point[0] * extream_point[1] * x + ) x0 = np.array([-1.7]).astype('float32') results = test_static_graph(func, x0) self.assertFalse(results[0][0]) def test_multi_minima(self): - def func(x): # df = 12(x + 1.1)(x - 0.2)(x - 0.8) # f = 3*x^4+0.4*x^3-5.46*x^2+2.112*x @@ -138,7 +136,7 @@ class TestBfgs(unittest.TestCase): # f(x, y) = (a - x)^2 + b (y - x^2)^2 # minimum = (a, a^2) x, y = position[0], position[1] - c = (a - x)**2 + b * (y - x**2)**2 + c = (a - x) ** 2 + b * (y - x**2) ** 2 # the return cant be np array[1], or in jacobin will cause flat error return c[0] @@ -153,7 +151,6 @@ class TestBfgs(unittest.TestCase): self.func_rosenbrock() def test_exception(self): - def func(x): return paddle.dot(x, x) @@ -162,10 +159,9 @@ class TestBfgs(unittest.TestCase): # test initial_inverse_hessian_estimate is good results = test_static_graph_H0(func, x0, H0, dtype='float32') - np.testing.assert_allclose([0.0, 0.0], - results[2], - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + [0.0, 0.0], results[2], rtol=1e-05, atol=1e-8 + ) self.assertTrue(results[0][0]) # test initial_inverse_hessian_estimate is bad @@ -173,11 +169,13 @@ class TestBfgs(unittest.TestCase): self.assertRaises(ValueError, test_dynamic_graph, func, x0, H0=H1) # test line_search_fn is bad - self.assertRaises(NotImplementedError, - test_static_graph, - func, - x0, - line_search_fn='other') + self.assertRaises( + NotImplementedError, + test_static_graph, + func, + x0, + line_search_fn='other', + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_bicubic_interp_op.py b/python/paddle/fluid/tests/unittests/test_bicubic_interp_op.py index ed8bc911378..886441429c2 100644 --- a/python/paddle/fluid/tests/unittests/test_bicubic_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_bicubic_interp_op.py @@ -47,13 +47,15 @@ def value_bound(input, w, h, x, y): return input[:, :, access_y, access_x] -def bicubic_interp_np(input, - out_h, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='kNCHW'): +def bicubic_interp_np( + input, + out_h, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='kNCHW', +): """trilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -67,13 +69,13 @@ def bicubic_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -81,14 +83,14 @@ def bicubic_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for k in range(out_h): - if (align_corners): + if align_corners: h = ratio_h * k else: h = ratio_h * (k + 0.5) - 0.5 input_y = np.floor(h) y_t = h - input_y for l in range(out_w): - if (align_corners): + if align_corners: w = ratio_w * l else: w = ratio_w * (l + 0.5) - 0.5 @@ -107,20 +109,23 @@ def bicubic_interp_np(input, coefficients[ii] = cubic_interp1d( input[i, j, access_y, access_x_0], input[i, j, access_y, access_x_1], - input[i, j, access_y, - access_x_2], input[i, j, access_y, - access_x_3], x_t) - out[i, j, k, - l] = cubic_interp1d(coefficients[0], coefficients[1], - coefficients[2], coefficients[3], - y_t) + input[i, j, access_y, access_x_2], + input[i, j, access_y, access_x_3], + x_t, + ) + out[i, j, k, l] = cubic_interp1d( + coefficients[0], + coefficients[1], + coefficients[2], + coefficients[3], + y_t, + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC return out.astype(input.dtype) class TestBicubicInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -146,9 +151,15 @@ class TestBicubicInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bicubic_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.data_layout) + output_np = bicubic_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -163,7 +174,7 @@ class TestBicubicInterpOp(OpTest): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -171,80 +182,73 @@ class TestBicubicInterpOp(OpTest): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 5, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True class TestBicubicInterpCase1(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestBicubicInterpCase2(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [3, 3, 9, 6] self.out_h = 10 self.out_w = 8 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestBicubicInterpCase3(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestBicubicInterpCase4(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True class TestBicubicInterpCase5(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [3, 3, 9, 6] self.out_h = 11 self.out_w = 11 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 4]).astype("int32") self.align_corners = False class TestBicubicInterpCase6(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [1, 1, 32, 64] @@ -256,31 +260,28 @@ class TestBicubicInterpCase6(TestBicubicInterpOp): class TestBicubicInterpSame(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestBicubicInterpDataLayout(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 5, 5, 3] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.data_layout = "NHWC" class TestBicubicInterpOpAPI(unittest.TestCase): - def test_case(self): np.random.seed(200) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -291,103 +292,101 @@ class TestBicubicInterpOpAPI(unittest.TestCase): prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") - shape_tensor = fluid.data(name="shape_tensor", - shape=[2], - dtype="int32") - actual_size = fluid.data(name="actual_size", - shape=[2], - dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) - out2 = interpolate(x, - size=[12, dim], - mode='bicubic', - align_corners=False) - out3 = interpolate(x, - size=shape_tensor, - mode='bicubic', - align_corners=False) - out4 = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) - out5 = interpolate(x, - scale_factor=scale_tensor, - mode='bicubic', - align_corners=False) + shape_tensor = fluid.data( + name="shape_tensor", shape=[2], dtype="int32" + ) + actual_size = fluid.data( + name="actual_size", shape=[2], dtype="int32" + ) + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) + out2 = interpolate( + x, size=[12, dim], mode='bicubic', align_corners=False + ) + out3 = interpolate( + x, size=shape_tensor, mode='bicubic', align_corners=False + ) + out4 = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) + out5 = interpolate( + x, + scale_factor=scale_tensor, + mode='bicubic', + align_corners=False, + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = bicubic_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = bicubic_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) - interp = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) + interp = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) dy_result = interp.numpy() - expect = bicubic_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) + expect = bicubic_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) np.testing.assert_allclose(dy_result, expect, rtol=1e-05) class TestBicubicOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of interpoalte must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, interpolate, x1) def test_mode_type(): # mode must be "BILINEAR" "TRILINEAR" "NEAREST" "BICUBIC" x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=[12, 12], - mode='UNKONWN', - align_corners=False) + out = interpolate( + x, size=[12, 12], mode='UNKONWN', align_corners=False + ) def test_input_shape(): x = fluid.data(name="x", shape=[2], dtype="float32") - out = interpolate(x, - size=[12, 12], - mode='BICUBIC', - align_corners=False) + out = interpolate( + x, size=[12, 12], mode='BICUBIC', align_corners=False + ) def test_align_corcers(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") @@ -395,77 +394,88 @@ class TestBicubicOpError(unittest.TestCase): def test_out_shape(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=[12], - mode='bicubic', - align_corners=False) + out = interpolate( + x, size=[12], mode='bicubic', align_corners=False + ) def test_attr_data_format(): # for 5-D input, data_format only can be NCDHW or NDHWC - input = fluid.data(name="input", - shape=[2, 3, 6, 9, 4], - dtype="float32") - out = interpolate(input, - size=[4, 8, 4, 5], - mode='trilinear', - data_format='NHWC') + input = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32" + ) + out = interpolate( + input, + size=[4, 8, 4, 5], + mode='trilinear', + data_format='NHWC', + ) def test_actual_shape(): # the actual_shape must be Variable. - x = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - out = interpolate(x, - size=[12, 12], - mode='BICUBIC', - align_corners=False) + x = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + out = interpolate( + x, size=[12, 12], mode='BICUBIC', align_corners=False + ) def test_scale_value(): # the scale must be greater than zero. x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='BICUBIC', - align_corners=False, - scale_factor=-2.0) + out = interpolate( + x, + size=None, + mode='BICUBIC', + align_corners=False, + scale_factor=-2.0, + ) def test_attr_5D_input(): # for 5-D input, data_format only can be NCDHW or NDHWC - input = fluid.data(name="input", - shape=[2, 3, 6, 9, 4], - dtype="float32") - out = interpolate(input, - size=[4, 8, 4, 5], - mode='trilinear', - data_format='NDHWC') + input = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32" + ) + out = interpolate( + input, + size=[4, 8, 4, 5], + mode='trilinear', + data_format='NDHWC', + ) def test_scale_type(): # the scale must be greater than zero. x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - scale = fluid.create_lod_tensor(np.array([-1, 3, 5, - 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=scale) + scale = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=scale, + ) def test_align_mode(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='nearest', - align_corners=False, - align_mode=2, - scale_factor=1.0) + out = interpolate( + x, + size=None, + mode='nearest', + align_corners=False, + align_mode=2, + scale_factor=1.0, + ) def test_outshape_and_scale(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=None) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=None, + ) self.assertRaises(ValueError, test_mode_type) self.assertRaises(ValueError, test_input_shape) diff --git a/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py index 8a58f8ec369..2bfe40abe53 100644 --- a/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py @@ -22,18 +22,20 @@ from paddle.fluid.framework import _test_eager_guard from paddle.nn.functional import interpolate -def bicubic_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='kNCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='bicubic', - align_corners=True, - align_mode=0): +def bicubic_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='kNCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='bicubic', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -43,12 +45,23 @@ def bicubic_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.bicubic_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, scale, - interp_method, align_corners, - align_mode) + return paddle._C_ops.bicubic_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) def cubic_1(x, a): @@ -77,15 +90,17 @@ def value_bound(input, w, h, x, y): return input[:, :, access_y, access_x] -def bicubic_interp_np(input, - out_h, - out_w, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='kNCHW'): +def bicubic_interp_np( + input, + out_h, + out_w, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='kNCHW', +): """trilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -99,7 +114,7 @@ def bicubic_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -108,7 +123,7 @@ def bicubic_interp_np(input, ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -119,14 +134,14 @@ def bicubic_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for k in range(out_h): - if (align_corners): + if align_corners: h = ratio_h * k else: h = ratio_h * (k + 0.5) - 0.5 input_y = np.floor(h) y_t = h - input_y for l in range(out_w): - if (align_corners): + if align_corners: w = ratio_w * l else: w = ratio_w * (l + 0.5) - 0.5 @@ -145,20 +160,23 @@ def bicubic_interp_np(input, coefficients[ii] = cubic_interp1d( input[i, j, access_y, access_x_0], input[i, j, access_y, access_x_1], - input[i, j, access_y, - access_x_2], input[i, j, access_y, - access_x_3], x_t) - out[i, j, k, - l] = cubic_interp1d(coefficients[0], coefficients[1], - coefficients[2], coefficients[3], - y_t) + input[i, j, access_y, access_x_2], + input[i, j, access_y, access_x_3], + x_t, + ) + out[i, j, k, l] = cubic_interp1d( + coefficients[0], + coefficients[1], + coefficients[2], + coefficients[3], + y_t, + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC return out.astype(input.dtype) class TestBicubicInterpOp(OpTest): - def setUp(self): self.python_api = bicubic_interp_test self.out_size = None @@ -181,7 +199,7 @@ class TestBicubicInterpOp(OpTest): if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -194,9 +212,17 @@ class TestBicubicInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bicubic_interp_np(input_np, out_h, out_w, scale_h, scale_w, - self.out_size, self.actual_shape, - self.align_corners, self.data_layout) + output_np = bicubic_interp_np( + input_np, + out_h, + out_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -210,11 +236,11 @@ class TestBicubicInterpOp(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -225,10 +251,9 @@ class TestBicubicInterpOp(OpTest): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bicubic' @@ -241,7 +266,6 @@ class TestBicubicInterpOp(OpTest): class TestBicubicInterpCase1(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [4, 1, 7, 8] @@ -252,7 +276,6 @@ class TestBicubicInterpCase1(TestBicubicInterpOp): class TestBicubicInterpCase2(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [3, 3, 9, 6] @@ -263,7 +286,6 @@ class TestBicubicInterpCase2(TestBicubicInterpOp): class TestBicubicInterpCase3(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [1, 1, 32, 64] @@ -274,7 +296,6 @@ class TestBicubicInterpCase3(TestBicubicInterpOp): class TestBicubicInterpCase4(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [4, 1, 7, 8] @@ -286,7 +307,6 @@ class TestBicubicInterpCase4(TestBicubicInterpOp): class TestBicubicInterpCase5(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [3, 3, 9, 6] @@ -298,7 +318,6 @@ class TestBicubicInterpCase5(TestBicubicInterpOp): class TestBicubicInterpCase6(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [1, 1, 32, 64] @@ -310,7 +329,6 @@ class TestBicubicInterpCase6(TestBicubicInterpOp): class TestBicubicInterpSame(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 32, 64] @@ -321,18 +339,16 @@ class TestBicubicInterpSame(TestBicubicInterpOp): class TestBicubicInterpScale(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = [1., 1.] + self.scale = [1.0, 1.0] self.align_corners = True class TestBicubicInterpDataLayout(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 5, 5, 3] @@ -345,7 +361,6 @@ class TestBicubicInterpDataLayout(TestBicubicInterpOp): class TestBicubicInterpOpAPI(unittest.TestCase): - def test_imperative_case(self): with _test_eager_guard(): self.func_case() @@ -361,52 +376,51 @@ class TestBicubicInterpOpAPI(unittest.TestCase): prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") - shape_tensor = fluid.data(name="shape_tensor", - shape=[2], - dtype="int32") - actual_size = fluid.data(name="actual_size", - shape=[2], - dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) - out2 = interpolate(x, - size=[12, dim], - mode='bicubic', - align_corners=False) - out3 = interpolate(x, - size=shape_tensor, - mode='bicubic', - align_corners=False) - out4 = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) - out5 = interpolate(x, - scale_factor=scale_tensor, - mode='bicubic', - align_corners=False) - out6 = interpolate(x, - scale_factor=2.0, - mode='bicubic', - align_corners=False) - out7 = interpolate(x, - scale_factor=[2.0, 2.0], - mode='bicubic', - align_corners=False) + shape_tensor = fluid.data( + name="shape_tensor", shape=[2], dtype="int32" + ) + actual_size = fluid.data( + name="actual_size", shape=[2], dtype="int32" + ) + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) + out2 = interpolate( + x, size=[12, dim], mode='bicubic', align_corners=False + ) + out3 = interpolate( + x, size=shape_tensor, mode='bicubic', align_corners=False + ) + out4 = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) + out5 = interpolate( + x, + scale_factor=scale_tensor, + mode='bicubic', + align_corners=False, + ) + out6 = interpolate( + x, scale_factor=2.0, mode='bicubic', align_corners=False + ) + out7 = interpolate( + x, scale_factor=[2.0, 2.0], mode='bicubic', align_corners=False + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -417,55 +431,51 @@ class TestBicubicInterpOpAPI(unittest.TestCase): "dim": dim_data, "shape_tensor": shape_data, "actual_size": actual_size_data, - "scale_tensor": scale_data + "scale_tensor": scale_data, }, fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + return_numpy=True, + ) - expect_res = bicubic_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) + expect_res = bicubic_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) - interp = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) + interp = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) dy_result = interp.numpy() - expect = bicubic_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) + expect = bicubic_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) np.testing.assert_allclose(dy_result, expect, rtol=1e-05) class TestBicubicOpError(unittest.TestCase): - def test_imperative_errors(self): # the input of interpoalte must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, interpolate, x1) def test_mode_type(): # mode must be "BILINEAR" "TRILINEAR" "NEAREST" "BICUBIC" x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=[12, 12], - mode='UNKONWN', - align_corners=False) + out = interpolate( + x, size=[12, 12], mode='UNKONWN', align_corners=False + ) def test_input_shape(): x = fluid.data(name="x", shape=[2], dtype="float32") - out = interpolate(x, - size=[12, 12], - mode='BICUBIC', - align_corners=False) + out = interpolate( + x, size=[12, 12], mode='BICUBIC', align_corners=False + ) def test_align_corcers(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") @@ -477,123 +487,138 @@ class TestBicubicOpError(unittest.TestCase): def test_attr_data_format(): # for 5-D input, data_format only can be NCDHW or NDHWC - input = fluid.data(name="input", - shape=[2, 3, 6, 9, 4], - dtype="float32") - out = interpolate(input, - size=[4, 8, 4, 5], - mode='trilinear', - data_format='NHWC') + input = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32" + ) + out = interpolate( + input, size=[4, 8, 4, 5], mode='trilinear', data_format='NHWC' + ) def test_actual_shape(): # the actual_shape must be Variable. - x = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) - out = interpolate(x, - size=[12, 12], - mode='BICUBIC', - align_corners=False) + x = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + out = interpolate( + x, size=[12, 12], mode='BICUBIC', align_corners=False + ) def test_scale_value(): # the scale must be greater than zero. x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='BICUBIC', - align_corners=False, - scale_factor=-2.0) + out = interpolate( + x, + size=None, + mode='BICUBIC', + align_corners=False, + scale_factor=-2.0, + ) def test_attr_5D_input(): # for 5-D input, data_format only can be NCDHW or NDHWC - input = fluid.data(name="input", - shape=[2, 3, 6, 9, 4], - dtype="float32") - out = interpolate(input, - size=[4, 8, 4, 5], - mode='trilinear', - data_format='NDHWC') + input = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32" + ) + out = interpolate( + input, size=[4, 8, 4, 5], mode='trilinear', data_format='NDHWC' + ) def test_scale_type(): # the scale must be greater than zero. x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - scale = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=scale) + scale = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=scale, + ) def test_align_mode(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='nearest', - align_corners=False, - align_mode=2, - scale_factor=1.0) + out = interpolate( + x, + size=None, + mode='nearest', + align_corners=False, + align_mode=2, + scale_factor=1.0, + ) def test_outshape_and_scale(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=None) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=None, + ) def test_align_corners_and_nearest(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='nearest', - align_corners=True, - scale_factor=None) + out = interpolate( + x, + size=None, + mode='nearest', + align_corners=True, + scale_factor=None, + ) def test_scale_shape(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='nearest', - align_corners=False, - scale_factor=[1, 2, 2]) + out = interpolate( + x, + size=None, + mode='nearest', + align_corners=False, + scale_factor=[1, 2, 2], + ) def test_scale_value_1(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=[1, 2, 2]) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=[1, 2, 2], + ) def test_size_and_scale(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=None) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=None, + ) def test_size_and_scale2(): x = fluid.data(name="input", shape=[2, 3, 6, 9, 4], dtype="float32") - out = interpolate(x, - size=[2, 2, 2], - mode='trilinear', - align_corners=False, - scale_factor=2.0) + out = interpolate( + x, + size=[2, 2, 2], + mode='trilinear', + align_corners=False, + scale_factor=2.0, + ) def test_size_type(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size={2, 2}, - mode='bicubic', - align_corners=False) + out = interpolate( + x, size={2, 2}, mode='bicubic', align_corners=False + ) def test_input_shape_1(): x = fluid.data(name="x", shape=[2, 1, 0, 0], dtype="float32") - out = interpolate(x, - size=[3, 3], - mode="bicubic", - align_corners=False) + out = interpolate( + x, size=[3, 3], mode="bicubic", align_corners=False + ) self.assertRaises(ValueError, test_mode_type) self.assertRaises(ValueError, test_input_shape) @@ -619,10 +644,10 @@ class TestBicubicOpError(unittest.TestCase): self.test_imperative_errors() -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestBicubicInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 5, 5] @@ -635,11 +660,13 @@ class TestBicubicInterpOpForFloat16(unittest.TestCase): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - size=self.out_size.tolist(), - mode=self.interp_method, - align_corners=self.align_corners, - data_format=self.data_layout) + y = interpolate( + x, + size=self.out_size.tolist(), + mode=self.interp_method, + align_corners=self.align_corners, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_bilateral_slice_op.py b/python/paddle/fluid/tests/unittests/test_bilateral_slice_op.py index 976e7df60b8..db0b8c182de 100644 --- a/python/paddle/fluid/tests/unittests/test_bilateral_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilateral_slice_op.py @@ -20,7 +20,6 @@ import math class Gsz: - def __init__(self, h, w, gd, gh, gw, input_chans): self.h = h self.w = w @@ -53,8 +52,9 @@ def d_weight_z(x): return d_diff_abs(x) -def naive_bilateral_slice_forward(output, grid, guide, input, gsz, has_offset, - total_count, output_chans): +def naive_bilateral_slice_forward( + output, grid, guide, input, gsz, has_offset, total_count, output_chans +): h = gsz.h w = gsz.w gd = gsz.gd @@ -72,7 +72,7 @@ def naive_bilateral_slice_forward(output, grid, guide, input, gsz, has_offset, x = idx % w y = idx // w % h out_c = (idx // (h * w)) % output_chans - b = (idx // (output_chans * w * h)) + b = idx // (output_chans * w * h) gx = (x + 0.5) * gw / (1.0 * w) gy = (y + 0.5) * gh / (1.0 * h) @@ -99,11 +99,12 @@ def naive_bilateral_slice_forward(output, grid, guide, input, gsz, has_offset, wz = weight_z(zz + 0.5 - gz) c_ = coeff_stride * out_c + in_c - coeff_sample += grid[int(b), - int(c_), - int(z_), - int(y_), - int(x_)] * wx * wy * wz + coeff_sample += ( + grid[int(b), int(c_), int(z_), int(y_), int(x_)] + * wx + * wy + * wz + ) if in_c < input_chans: value += coeff_sample * input[int(b), int(in_c), int(y), int(x)] @@ -132,15 +133,16 @@ def naive_bilateral_slice(x, guide, grid, has_offset): gsz = Gsz(h, w, gd, gh, gw, input_chans) total_count = bs * h * w * output.shape[1] - naive_bilateral_slice_forward(output, grid, guide, x, gsz, has_offset, - total_count, output.shape[1]) + naive_bilateral_slice_forward( + output, grid, guide, x, gsz, has_offset, total_count, output.shape[1] + ) return output -@unittest.skipIf(not paddle.fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not paddle.fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestBilateralSliceOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'bilateral_slice' @@ -177,27 +179,26 @@ class TestBilateralSliceOp(OpTest): self.data_type = 'float64' -@unittest.skipIf(not paddle.fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not paddle.fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestBilateralSliceOp1(TestBilateralSliceOp): - def initTestCase(self): self.has_offset = True self.data_type = 'float32' class TestBilateralSliceApi(unittest.TestCase): - def test_api(self): - x = paddle.fluid.data(name='x', - shape=[None, 3, 25, 15], - dtype='float32') - guide = paddle.fluid.data(name='guide', - shape=[None, 25, 15], - dtype='float32') - grid = paddle.fluid.data(name='grid', - shape=[None, None, 8, 5, 3], - dtype='float32') + x = paddle.fluid.data( + name='x', shape=[None, 3, 25, 15], dtype='float32' + ) + guide = paddle.fluid.data( + name='guide', shape=[None, 25, 15], dtype='float32' + ) + grid = paddle.fluid.data( + name='grid', shape=[None, None, 8, 5, 3], dtype='float32' + ) paddle.fluid.contrib.layers.bilateral_slice(x, guide, grid, False) if not paddle.fluid.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_api.py b/python/paddle/fluid/tests/unittests/test_bilinear_api.py index 031163cdc43..dcdb03d05ba 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_api.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_api.py @@ -21,10 +21,10 @@ import numpy as np class TestBilinearAPI(unittest.TestCase): - def test_api(self): - with fluid.program_guard(fluid.default_startup_program(), - fluid.default_main_program()): + with fluid.program_guard( + fluid.default_startup_program(), fluid.default_main_program() + ): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -37,29 +37,26 @@ class TestBilinearAPI(unittest.TestCase): layer1 = np.random.random((5, 5)).astype('float32') layer2 = np.random.random((5, 4)).astype('float32') - bilinear = paddle.nn.Bilinear(in1_features=5, - in2_features=4, - out_features=1000) + bilinear = paddle.nn.Bilinear( + in1_features=5, in2_features=4, out_features=1000 + ) ret = bilinear(data1, data2) exe.run(fluid.default_startup_program()) - ret_fetch = exe.run(feed={ - 'X1': layer1, - 'X2': layer2 - }, - fetch_list=[ret.name]) + ret_fetch = exe.run( + feed={'X1': layer1, 'X2': layer2}, fetch_list=[ret.name] + ) self.assertEqual(ret_fetch[0].shape, (5, 1000)) class TestBilinearAPIDygraph(unittest.TestCase): - def test_api(self): paddle.disable_static() layer1 = np.random.random((5, 5)).astype('float32') layer2 = np.random.random((5, 4)).astype('float32') - bilinear = paddle.nn.Bilinear(in1_features=5, - in2_features=4, - out_features=1000) + bilinear = paddle.nn.Bilinear( + in1_features=5, in2_features=4, out_features=1000 + ) ret = bilinear(paddle.to_tensor(layer1), paddle.to_tensor(layer2)) self.assertEqual(ret.shape, [5, 1000]) diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py index bb46558c093..b281d30a222 100755 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py @@ -22,14 +22,16 @@ import paddle paddle.enable_static() -def bilinear_interp_np(input, - out_h, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): +def bilinear_interp_np( + input, + out_h, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -43,12 +45,12 @@ def bilinear_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -56,37 +58,40 @@ def bilinear_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for i in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (i + 0.5) - 0.5) else: h = int(ratio_h * i) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (i + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: h1lambda = ratio_h * i - h h2lambda = 1.0 - h1lambda for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, i, j] = h2lambda*(w2lambda*input[:, :, h, w] + - w1lambda*input[:, :, h, w+wid]) + \ - h1lambda*(w2lambda*input[:, :, h+hid, w] + - w1lambda*input[:, :, h+hid, w+wid]) + out[:, :, i, j] = h2lambda * ( + w2lambda * input[:, :, h, w] + + w1lambda * input[:, :, h, w + wid] + ) + h1lambda * ( + w2lambda * input[:, :, h + hid, w] + + w1lambda * input[:, :, h + hid, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -95,7 +100,6 @@ def bilinear_interp_np(input, class TestBilinearInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -121,9 +125,16 @@ class TestBilinearInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -139,7 +150,7 @@ class TestBilinearInterpOp(OpTest): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -147,130 +158,120 @@ class TestBilinearInterpOp(OpTest): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 33]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpDataLayout(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 5, 5, 3] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -278,15 +279,15 @@ class TestBilinearInterpDataLayout(TestBilinearInterpOp): class TestBilinearInterpOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "bilinear_interp" self.check_eager = True - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale > 0: out_h = int(self.input_shape[2] * self.scale) @@ -295,9 +296,15 @@ class TestBilinearInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -309,97 +316,89 @@ class TestBilinearInterpOpUint8(OpTest): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=self.check_eager) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 3, 9, 6] self.out_h = 10 self.out_w = 9 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase1Uint8(TestBilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase2Uint8(TestBilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 5 self.out_w = 13 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 15]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -411,7 +410,6 @@ class TestBilinearInterpScale3(TestBilinearInterpOp): class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -423,7 +421,6 @@ class TestBilinearInterpZero(TestBilinearInterpOp): class TestBilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -456,58 +453,62 @@ class TestBilinearInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.check_eager = False self.attrs['out_h'] = self.out_h self.attrs['out_w'] = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 5] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True # out_size is a 1-D tensor class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True @@ -515,7 +516,6 @@ class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -528,23 +528,22 @@ class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): class TestBilinearInterpOpAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) out1 = fluid.layers.resize_bilinear(x, out_shape=[12, 12]) out2 = fluid.layers.resize_bilinear(x, out_shape=[12, dim]) out3 = fluid.layers.resize_bilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_bilinear(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_bilinear( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_bilinear(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -559,21 +558,22 @@ class TestBilinearInterpOpAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = bilinear_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = bilinear_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py index e25ffeaee53..1a3786dbdd5 100755 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py @@ -21,18 +21,20 @@ from paddle.nn.functional import interpolate import paddle -def bilinear_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='NCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='bilinear', - align_corners=True, - align_mode=0): +def bilinear_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='NCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='bilinear', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -42,24 +44,37 @@ def bilinear_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.bilinear_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, - scale, interp_method, align_corners, - align_mode) - - -def bilinear_interp_np(input, - out_h, - out_w, - scale_w=0, - scale_h=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): + return paddle._C_ops.bilinear_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) + + +def bilinear_interp_np( + input, + out_h, + out_w, + scale_w=0, + scale_h=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -73,7 +88,7 @@ def bilinear_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -81,7 +96,7 @@ def bilinear_interp_np(input, else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -92,37 +107,40 @@ def bilinear_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for i in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (i + 0.5) - 0.5) else: h = int(ratio_h * i) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (i + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: h1lambda = ratio_h * i - h h2lambda = 1.0 - h1lambda for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, i, j] = h2lambda*(w2lambda*input[:, :, h, w] + - w1lambda*input[:, :, h, w+wid]) + \ - h1lambda*(w2lambda*input[:, :, h+hid, w] + - w1lambda*input[:, :, h+hid, w+wid]) + out[:, :, i, j] = h2lambda * ( + w2lambda * input[:, :, h, w] + + w1lambda * input[:, :, h, w + wid] + ) + h1lambda * ( + w2lambda * input[:, :, h + hid, w] + + w1lambda * input[:, :, h + hid, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -131,7 +149,6 @@ def bilinear_interp_np(input, class TestBilinearInterpOp(OpTest): - def setUp(self): self.python_api = bilinear_interp_test self.out_size = None @@ -151,7 +168,7 @@ class TestBilinearInterpOp(OpTest): scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -164,10 +181,18 @@ class TestBilinearInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -180,11 +205,11 @@ class TestBilinearInterpOp(OpTest): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -209,7 +234,6 @@ class TestBilinearInterpOp(OpTest): class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] @@ -221,7 +245,6 @@ class TestBilinearInterpCase1(TestBilinearInterpOp): class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] @@ -233,7 +256,6 @@ class TestBilinearInterpCase2(TestBilinearInterpOp): class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -245,7 +267,6 @@ class TestBilinearInterpCase3(TestBilinearInterpOp): class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] @@ -258,7 +279,6 @@ class TestBilinearInterpCase4(TestBilinearInterpOp): class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] @@ -271,7 +291,6 @@ class TestBilinearInterpCase5(TestBilinearInterpOp): class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -284,7 +303,6 @@ class TestBilinearInterpCase6(TestBilinearInterpOp): class TestBilinearInterpCase7(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -296,7 +314,6 @@ class TestBilinearInterpCase7(TestBilinearInterpOp): class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] @@ -308,7 +325,6 @@ class TestBilinearInterpSame(TestBilinearInterpOp): class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -321,7 +337,6 @@ class TestBilinearInterpActualShape(TestBilinearInterpOp): class TestBilinearInterpDataLayout(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 5, 5, 3] @@ -335,15 +350,15 @@ class TestBilinearInterpDataLayout(TestBilinearInterpOp): class TestBilinearInterpOpUint8(OpTest): - def setUp(self): self.python_api = bilinear_interp_test self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "bilinear_interp_v2" - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -360,9 +375,17 @@ class TestBilinearInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -372,7 +395,7 @@ class TestBilinearInterpOpUint8(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -384,9 +407,9 @@ class TestBilinearInterpOpUint8(OpTest): self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=True) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=True + ) def init_test_case(self): self.interp_method = 'bilinear' @@ -399,7 +422,6 @@ class TestBilinearInterpOpUint8(OpTest): class TestBilinearInterpCase1Uint8(TestBilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] @@ -411,7 +433,6 @@ class TestBilinearInterpCase1Uint8(TestBilinearInterpOpUint8): class TestBilinearInterpCase2Uint8(TestBilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] @@ -424,52 +445,46 @@ class TestBilinearInterpCase2Uint8(TestBilinearInterpOpUint8): class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -481,7 +496,6 @@ class TestBilinearInterpScale3(TestBilinearInterpOp): class TestBilinearInterpScale4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -493,7 +507,6 @@ class TestBilinearInterpScale4(TestBilinearInterpOp): class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -505,7 +518,6 @@ class TestBilinearInterpZero(TestBilinearInterpOp): class TestBilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.python_api = bilinear_interp_test self.out_size = None @@ -544,8 +556,9 @@ class TestBilinearInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -557,9 +570,16 @@ class TestBilinearInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -580,7 +600,6 @@ class TestBilinearInterpOp_attr_tensor(OpTest): # out_size is a 1-D tensor class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] @@ -593,7 +612,6 @@ class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor): # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -607,7 +625,6 @@ class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -620,23 +637,22 @@ class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): class TestBilinearInterpOpAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) out1 = fluid.layers.resize_bilinear(x, out_shape=[12, 12]) out2 = fluid.layers.resize_bilinear(x, out_shape=[12, dim]) out3 = fluid.layers.resize_bilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_bilinear(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_bilinear( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_bilinear(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -651,29 +667,30 @@ class TestBilinearInterpOpAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = bilinear_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = bilinear_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) class TestBilinearInterpOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -681,21 +698,19 @@ class TestBilinearInterpOpAPI_dy(unittest.TestCase): with fluid.dygraph.guard(place): input_data = np.random.random((2, 3, 6, 6)).astype("float32") input_x = paddle.to_tensor(input_data) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=[12, 12], - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, size=[12, 12], mode="bilinear", align_corners=False + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestBilinearInterpOpAPI_dy2(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -705,21 +720,19 @@ class TestBilinearInterpOpAPI_dy2(unittest.TestCase): size_np = np.array([12, 12]).astype("int64") input_x = paddle.to_tensor(input_data) size = paddle.to_tensor(size_np) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=size, - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, size=size, mode="bilinear", align_corners=False + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestBilinearInterpOpAPI_dy3(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -729,21 +742,22 @@ class TestBilinearInterpOpAPI_dy3(unittest.TestCase): size_1 = np.array([12]).astype("int64") input_x = paddle.to_tensor(input_data) size = paddle.to_tensor(size_1) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=[size, size], - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + size=[size, size], + mode="bilinear", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestBilinearInterpOpAPI_dy4(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -753,21 +767,22 @@ class TestBilinearInterpOpAPI_dy4(unittest.TestCase): scale_np = np.array([2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="bilinear", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestBilinearInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 5] @@ -781,12 +796,14 @@ class TestBilinearInterpOpForFloat16(unittest.TestCase): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - size=self.out_size.tolist(), - mode=self.interp_method, - align_mode=self.align_mode, - align_corners=self.align_corners, - data_format=self.data_layout) + y = interpolate( + x, + size=self.out_size.tolist(), + mode=self.interp_method, + align_mode=self.align_mode, + align_corners=self.align_corners, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py index 6ffbbfdb999..79ef2ffabbc 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py @@ -20,15 +20,15 @@ import paddle class TestDygraphBilinearTensorProductAPIError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - layer = fluid.dygraph.nn.BilinearTensorProduct(input1_dim=5, - input2_dim=4, - output_dim=1000) + layer = fluid.dygraph.nn.BilinearTensorProduct( + input1_dim=5, input2_dim=4, output_dim=1000 + ) # the input must be Variable. - x0 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x0 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, layer, x0) # the input dtype must be float32 or float64 x1 = fluid.data(name='x1', shape=[-1, 5], dtype="float16") @@ -37,7 +37,6 @@ class TestDygraphBilinearTensorProductAPIError(unittest.TestCase): class TestBilinearTensorProductOp(OpTest): - def setUp(self): self.op_type = "bilinear_tensor_product" self.python_api = paddle.nn.functional.bilinear diff --git a/python/paddle/fluid/tests/unittests/test_bincount_op.py b/python/paddle/fluid/tests/unittests/test_bincount_op.py index e191b09fe42..c5e39f1b13f 100644 --- a/python/paddle/fluid/tests/unittests/test_bincount_op.py +++ b/python/paddle/fluid/tests/unittests/test_bincount_op.py @@ -42,16 +42,17 @@ class TestBincountOpAPI(unittest.TestCase): exe.run(startup_program) img = np.array([0, 1, 1, 3, 2, 1, 7]).astype(np.int64) w = np.array([0, 1, 1, 2, 2, 1, 0]).astype(np.int64) - res = exe.run(train_program, - feed={ - 'input': img, - 'weights': w - }, - fetch_list=[output]) + res = exe.run( + train_program, + feed={'input': img, 'weights': w}, + fetch_list=[output], + ) actual = np.array(res[0]) expected = np.bincount(img, weights=w) - self.assertTrue((actual == expected).all(), - msg='bincount output is wrong, out =' + str(actual)) + self.assertTrue( + (actual == expected).all(), + msg='bincount output is wrong, out =' + str(actual), + ) def test_dygraph(self): with fluid.dygraph.guard(): @@ -61,7 +62,8 @@ class TestBincountOpAPI(unittest.TestCase): expected = np.bincount(inputs) self.assertTrue( (actual.numpy() == expected).all(), - msg='bincount output is wrong, out =' + str(actual.numpy())) + msg='bincount output is wrong, out =' + str(actual.numpy()), + ) class TestBincountOpError(unittest.TestCase): @@ -112,7 +114,7 @@ class TestBincountOpError(unittest.TestCase): """Test input tensor should only contain non-negative ints.""" def net_func(): - input_value = paddle.to_tensor([1., 2., 3., 4., 5.]) + input_value = paddle.to_tensor([1.0, 2.0, 3.0, 4.0, 5.0]) paddle.bincount(input_value) with self.assertRaises(TypeError): @@ -161,12 +163,13 @@ class TestCase1(TestBincountOp): def init_test_case(self): self.minlength = 0 - self.np_weights = np.random.randint(low=0, high=20, - size=10).astype(np.float32) + self.np_weights = np.random.randint(low=0, high=20, size=10).astype( + np.float32 + ) self.np_input = np.random.randint(low=0, high=20, size=10) - self.Out = np.bincount(self.np_input, - weights=self.np_weights, - minlength=self.minlength).astype(np.float32) + self.Out = np.bincount( + self.np_input, weights=self.np_weights, minlength=self.minlength + ).astype(np.float32) class TestCase2(TestBincountOp): @@ -183,9 +186,9 @@ class TestCase2(TestBincountOp): self.minlength = 0 self.np_weights = np.random.randint(low=0, high=20, size=10) self.np_input = np.random.randint(low=0, high=20, size=10) - self.Out = np.bincount(self.np_input, - weights=self.np_weights, - minlength=self.minlength) + self.Out = np.bincount( + self.np_input, weights=self.np_weights, minlength=self.minlength + ) class TestCase3(TestBincountOp): @@ -200,8 +203,9 @@ class TestCase4(TestBincountOp): # with input(INT32) def init_test_case(self): self.minlength = 0 - self.np_input = np.random.randint(low=0, high=20, - size=10).astype(np.int32) + self.np_input = np.random.randint(low=0, high=20, size=10).astype( + np.int32 + ) self.Out = np.bincount(self.np_input, minlength=self.minlength) @@ -214,23 +218,27 @@ class TestCase5(TestBincountOp): class TestTensorMinlength(unittest.TestCase): - def setUp(self): paddle.disable_static() paddle.seed(2022) self.temp_dir = tempfile.TemporaryDirectory() - self.save_path = os.path.join(self.temp_dir.name, - 'tensor_minlength_bincount') - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.save_path = os.path.join( + self.temp_dir.name, 'tensor_minlength_bincount' + ) + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def test_dygraph(self): paddle.disable_static() x = np.random.randint(0, 10, [20]) minlength = 2 np_out = np.bincount(x, minlength=minlength) - pd_out = paddle.bincount(paddle.to_tensor(x), - minlength=paddle.to_tensor([2], dtype='int32')) + pd_out = paddle.bincount( + paddle.to_tensor(x), minlength=paddle.to_tensor([2], dtype='int32') + ) np.testing.assert_allclose(np_out, pd_out.numpy()) def test_static_and_infer(self): @@ -245,8 +253,9 @@ class TestTensorMinlength(unittest.TestCase): linear_out = linear(x) relu_out = paddle.nn.functional.relu(linear_out) minlength = paddle.full([1], 3, dtype='int32') - out = paddle.bincount(paddle.cast(relu_out, 'int32'), - minlength=minlength) + out = paddle.bincount( + paddle.cast(relu_out, 'int32'), minlength=minlength + ) exe = paddle.static.Executor(self.place) exe.run(starup_prog) @@ -254,8 +263,9 @@ class TestTensorMinlength(unittest.TestCase): # run infer paddle.static.save_inference_model(self.save_path, [x], [out], exe) - config = paddle_infer.Config(self.save_path + '.pdmodel', - self.save_path + '.pdiparams') + config = paddle_infer.Config( + self.save_path + '.pdmodel', self.save_path + '.pdiparams' + ) if paddle.is_compiled_with_cuda(): config.enable_use_gpu(100, 0) else: diff --git a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py index 39bed077eba..0412c04d8ca 100644 --- a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py +++ b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -34,7 +34,7 @@ def bipartite_match(distance, match_indices, match_dist): match_sorted = sorted(match_pair, key=lambda tup: tup[2], reverse=True) - row_indices = -1 * np.ones((row, ), dtype=np.int_) + row_indices = -1 * np.ones((row,), dtype=np.int_) idx = 0 for i, j, dist in match_sorted: @@ -72,18 +72,25 @@ def batch_bipartite_match(distance, lod, match_type=None, dist_threshold=None): match_dist = np.zeros((n, m), dtype=np.float32) cur_offset = 0 for i in range(n): - if lod[i] == 0: continue - bipartite_match(distance[cur_offset:(cur_offset + lod[i]), :], - match_indices[i, :], match_dist[i, :]) + if lod[i] == 0: + continue + bipartite_match( + distance[cur_offset : (cur_offset + lod[i]), :], + match_indices[i, :], + match_dist[i, :], + ) if match_type == 'per_prediction': - argmax_match(distance[cur_offset:(cur_offset + lod[i]), :], - match_indices[i, :], match_dist[i, :], dist_threshold) + argmax_match( + distance[cur_offset : (cur_offset + lod[i]), :], + match_indices[i, :], + match_dist[i, :], + dist_threshold, + ) cur_offset += lod[i] return match_indices, match_dist class TestBipartiteMatchOpWithLoD(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[5, 6, 12]] @@ -101,7 +108,6 @@ class TestBipartiteMatchOpWithLoD(OpTest): class TestBipartiteMatchOpWithoutLoD(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[8]] @@ -119,7 +125,6 @@ class TestBipartiteMatchOpWithoutLoD(OpTest): class TestBipartiteMatchOpWithoutLoDLargeScaleInput(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[300]] @@ -137,13 +142,13 @@ class TestBipartiteMatchOpWithoutLoDLargeScaleInput(OpTest): class TestBipartiteMatchOpWithPerPredictionType(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[5, 6, 12]] dist = np.random.random((23, 237)).astype('float32') match_indices, match_dist = batch_bipartite_match( - dist, lod[0], 'per_prediction', 0.5) + dist, lod[0], 'per_prediction', 0.5 + ) self.inputs = {'DistMat': (dist, lod)} self.outputs = { @@ -160,7 +165,6 @@ class TestBipartiteMatchOpWithPerPredictionType(OpTest): class TestBipartiteMatchOpWithEmptyLoD(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[5, 6, 0, 12]] diff --git a/python/paddle/fluid/tests/unittests/test_bitwise_op.py b/python/paddle/fluid/tests/unittests/test_bitwise_op.py index c387555ccda..6a7b039380b 100644 --- a/python/paddle/fluid/tests/unittests/test_bitwise_op.py +++ b/python/paddle/fluid/tests/unittests/test_bitwise_op.py @@ -22,21 +22,18 @@ paddle.enable_static() ################## TEST OP: BitwiseAnd ################## class TestBitwiseAnd(OpTest): - def setUp(self): self.op_type = "bitwise_and" self.init_dtype() self.init_shape() self.init_bound() - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_and(x, y) self.inputs = {'X': x, 'Y': y} @@ -61,7 +58,6 @@ class TestBitwiseAnd(OpTest): class TestBitwiseAndUInt8(TestBitwiseAnd): - def init_dtype(self): self.dtype = np.uint8 @@ -71,7 +67,6 @@ class TestBitwiseAndUInt8(TestBitwiseAnd): class TestBitwiseAndInt8(TestBitwiseAnd): - def init_dtype(self): self.dtype = np.int8 @@ -81,7 +76,6 @@ class TestBitwiseAndInt8(TestBitwiseAnd): class TestBitwiseAndInt16(TestBitwiseAnd): - def init_dtype(self): self.dtype = np.int16 @@ -91,7 +85,6 @@ class TestBitwiseAndInt16(TestBitwiseAnd): class TestBitwiseAndInt64(TestBitwiseAnd): - def init_dtype(self): self.dtype = np.int64 @@ -101,7 +94,6 @@ class TestBitwiseAndInt64(TestBitwiseAnd): class TestBitwiseAndBool(TestBitwiseAnd): - def setUp(self): self.op_type = "bitwise_and" self.init_shape() @@ -116,21 +108,18 @@ class TestBitwiseAndBool(TestBitwiseAnd): ################## TEST OP: BitwiseOr ################## class TestBitwiseOr(OpTest): - def setUp(self): self.op_type = "bitwise_or" self.init_dtype() self.init_shape() self.init_bound() - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_or(x, y) self.inputs = {'X': x, 'Y': y} @@ -155,7 +144,6 @@ class TestBitwiseOr(OpTest): class TestBitwiseOrUInt8(TestBitwiseOr): - def init_dtype(self): self.dtype = np.uint8 @@ -165,7 +153,6 @@ class TestBitwiseOrUInt8(TestBitwiseOr): class TestBitwiseOrInt8(TestBitwiseOr): - def init_dtype(self): self.dtype = np.int8 @@ -175,7 +162,6 @@ class TestBitwiseOrInt8(TestBitwiseOr): class TestBitwiseOrInt16(TestBitwiseOr): - def init_dtype(self): self.dtype = np.int16 @@ -185,7 +171,6 @@ class TestBitwiseOrInt16(TestBitwiseOr): class TestBitwiseOrInt64(TestBitwiseOr): - def init_dtype(self): self.dtype = np.int64 @@ -195,7 +180,6 @@ class TestBitwiseOrInt64(TestBitwiseOr): class TestBitwiseOrBool(TestBitwiseOr): - def setUp(self): self.op_type = "bitwise_or" self.init_shape() @@ -210,21 +194,18 @@ class TestBitwiseOrBool(TestBitwiseOr): ################## TEST OP: BitwiseXor ################## class TestBitwiseXor(OpTest): - def setUp(self): self.op_type = "bitwise_xor" self.init_dtype() self.init_shape() self.init_bound() - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_xor(x, y) self.inputs = {'X': x, 'Y': y} @@ -249,7 +230,6 @@ class TestBitwiseXor(OpTest): class TestBitwiseXorUInt8(TestBitwiseXor): - def init_dtype(self): self.dtype = np.uint8 @@ -259,7 +239,6 @@ class TestBitwiseXorUInt8(TestBitwiseXor): class TestBitwiseXorInt8(TestBitwiseXor): - def init_dtype(self): self.dtype = np.int8 @@ -269,7 +248,6 @@ class TestBitwiseXorInt8(TestBitwiseXor): class TestBitwiseXorInt16(TestBitwiseXor): - def init_dtype(self): self.dtype = np.int16 @@ -279,7 +257,6 @@ class TestBitwiseXorInt16(TestBitwiseXor): class TestBitwiseXorInt64(TestBitwiseXor): - def init_dtype(self): self.dtype = np.int64 @@ -289,7 +266,6 @@ class TestBitwiseXorInt64(TestBitwiseXor): class TestBitwiseXorBool(TestBitwiseXor): - def setUp(self): self.op_type = "bitwise_xor" self.init_shape() @@ -304,17 +280,15 @@ class TestBitwiseXorBool(TestBitwiseXor): ################## TEST OP: BitwiseNot ################## class TestBitwiseNot(OpTest): - def setUp(self): self.op_type = "bitwise_not" self.init_dtype() self.init_shape() self.init_bound() - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) out = np.bitwise_not(x) self.inputs = {'X': x} @@ -338,7 +312,6 @@ class TestBitwiseNot(OpTest): class TestBitwiseNotUInt8(TestBitwiseNot): - def init_dtype(self): self.dtype = np.uint8 @@ -348,7 +321,6 @@ class TestBitwiseNotUInt8(TestBitwiseNot): class TestBitwiseNotInt8(TestBitwiseNot): - def init_dtype(self): self.dtype = np.int8 @@ -357,7 +329,6 @@ class TestBitwiseNotInt8(TestBitwiseNot): class TestBitwiseNotInt16(TestBitwiseNot): - def init_dtype(self): self.dtype = np.int16 @@ -367,7 +338,6 @@ class TestBitwiseNotInt16(TestBitwiseNot): class TestBitwiseNotInt64(TestBitwiseNot): - def init_dtype(self): self.dtype = np.int64 @@ -376,7 +346,6 @@ class TestBitwiseNotInt64(TestBitwiseNot): class TestBitwiseNotBool(TestBitwiseNot): - def setUp(self): self.op_type = "bitwise_not" self.init_shape() diff --git a/python/paddle/fluid/tests/unittests/test_block_rename_var.py b/python/paddle/fluid/tests/unittests/test_block_rename_var.py index 322cb8bc447..02908052278 100644 --- a/python/paddle/fluid/tests/unittests/test_block_rename_var.py +++ b/python/paddle/fluid/tests/unittests/test_block_rename_var.py @@ -18,16 +18,15 @@ import paddle class TestBlockRenameVar(unittest.TestCase): - def setUp(self): self.program = paddle.static.Program() self.block = self.program.current_block() - self.var = self.block.create_var(name="X", - shape=[-1, 23, 48], - dtype='float32') - self.op = self.block.append_op(type="abs", - inputs={"X": [self.var]}, - outputs={"Out": [self.var]}) + self.var = self.block.create_var( + name="X", shape=[-1, 23, 48], dtype='float32' + ) + self.op = self.block.append_op( + type="abs", inputs={"X": [self.var]}, outputs={"Out": [self.var]} + ) self.new_var_name = self.get_new_var_name() def get_new_var_name(self): @@ -35,19 +34,20 @@ class TestBlockRenameVar(unittest.TestCase): def test_rename_var(self): self.block._rename_var(self.var.name, self.new_var_name) - new_var_name_str = self.new_var_name if isinstance( - self.new_var_name, str) else self.new_var_name.decode() + new_var_name_str = ( + self.new_var_name + if isinstance(self.new_var_name, str) + else self.new_var_name.decode() + ) self.assertTrue(new_var_name_str in self.block.vars) class TestBlockRenameVarStrCase2(TestBlockRenameVar): - def get_new_var_name(self): return "ABC" class TestBlockRenameVarBytes(TestBlockRenameVar): - def get_new_var_name(self): return b"Y" diff --git a/python/paddle/fluid/tests/unittests/test_bmm_op.py b/python/paddle/fluid/tests/unittests/test_bmm_op.py index 44a6be83b32..5c99a2e62cc 100644 --- a/python/paddle/fluid/tests/unittests/test_bmm_op.py +++ b/python/paddle/fluid/tests/unittests/test_bmm_op.py @@ -20,7 +20,6 @@ import paddle.fluid as fluid class TestBmmOp(OpTest): - def setUp(self): self.op_type = "bmm" self.python_api = paddle.tensor.bmm @@ -38,36 +37,40 @@ class TestBmmOp(OpTest): class API_TestBmm(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.layers.data('data1', - shape=[-1, 3, 4], - dtype='float64') - data2 = fluid.layers.data('data2', - shape=[-1, 4, 5], - dtype='float64') + data1 = fluid.layers.data( + 'data1', shape=[-1, 3, 4], dtype='float64' + ) + data2 = fluid.layers.data( + 'data2', shape=[-1, 4, 5], dtype='float64' + ) result_bmm = paddle.bmm(data1, data2) place = fluid.CPUPlace() exe = fluid.Executor(place) input1 = np.random.random([10, 3, 4]).astype('float64') input2 = np.random.random([10, 4, 5]).astype('float64') - result, = exe.run(feed={ - "data1": input1, - "data2": input2 - }, - fetch_list=[result_bmm]) + (result,) = exe.run( + feed={"data1": input1, "data2": input2}, fetch_list=[result_bmm] + ) expected_result = np.matmul(input1, input2) np.testing.assert_allclose(expected_result, result, rtol=1e-05) class API_TestDygraphBmm(unittest.TestCase): - def test_out(self): - input1 = np.array([[[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]], - [[3.0, 3.0, 3.0], [4.0, 4.0, 4.0]]]) - input2 = np.array([[[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], - [[4.0, 4.0], [5.0, 5.0], [6.0, 6.0]]]) + input1 = np.array( + [ + [[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]], + [[3.0, 3.0, 3.0], [4.0, 4.0, 4.0]], + ] + ) + input2 = np.array( + [ + [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], + [[4.0, 4.0], [5.0, 5.0], [6.0, 6.0]], + ] + ) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(input1) y = fluid.dygraph.to_variable(input2) @@ -78,7 +81,6 @@ class API_TestDygraphBmm(unittest.TestCase): class TestBmmAPIError(unittest.TestCase): - def test_api_error(self): x_data = np.arange(24, dtype='float32').reshape((2, 3, 4)) y_data = np.arange(16, dtype='float32').reshape((2, 4, 2)) diff --git a/python/paddle/fluid/tests/unittests/test_box_clip_op.py b/python/paddle/fluid/tests/unittests/test_box_clip_op.py index e3c9c8e6eb4..2ee356976f6 100644 --- a/python/paddle/fluid/tests/unittests/test_box_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_clip_op.py @@ -20,14 +20,18 @@ from op_test import OpTest def box_clip(input_box, im_info, output_box): im_w = round(im_info[1] / im_info[2]) im_h = round(im_info[0] / im_info[2]) - output_box[:, :, 0] = np.maximum(np.minimum(input_box[:, :, 0], im_w - 1), - 0) - output_box[:, :, 1] = np.maximum(np.minimum(input_box[:, :, 1], im_h - 1), - 0) - output_box[:, :, 2] = np.maximum(np.minimum(input_box[:, :, 2], im_w - 1), - 0) - output_box[:, :, 3] = np.maximum(np.minimum(input_box[:, :, 3], im_h - 1), - 0) + output_box[:, :, 0] = np.maximum( + np.minimum(input_box[:, :, 0], im_w - 1), 0 + ) + output_box[:, :, 1] = np.maximum( + np.minimum(input_box[:, :, 1], im_h - 1), 0 + ) + output_box[:, :, 2] = np.maximum( + np.minimum(input_box[:, :, 2], im_w - 1), 0 + ) + output_box[:, :, 3] = np.maximum( + np.minimum(input_box[:, :, 3], im_h - 1), 0 + ) def batch_box_clip(input_boxes, im_info, lod): @@ -36,15 +40,16 @@ def batch_box_clip(input_boxes, im_info, lod): output_boxes = np.zeros((n, m, 4), dtype=np.float32) cur_offset = 0 for i in range(len(lod)): - box_clip(input_boxes[cur_offset:(cur_offset + lod[i]), :, :], - im_info[i, :], - output_boxes[cur_offset:(cur_offset + lod[i]), :, :]) + box_clip( + input_boxes[cur_offset : (cur_offset + lod[i]), :, :], + im_info[i, :], + output_boxes[cur_offset : (cur_offset + lod[i]), :, :], + ) cur_offset += lod[i] return output_boxes class TestBoxClipOp(OpTest): - def test_check_output(self): self.check_output() @@ -52,7 +57,7 @@ class TestBoxClipOp(OpTest): self.op_type = "box_clip" lod = [[1, 2, 3]] input_boxes = np.random.random((6, 10, 4)) * 5 - im_info = np.array([[5, 8, 1.], [6, 6, 1.], [7, 5, 1.]]) + im_info = np.array([[5, 8, 1.0], [6, 6, 1.0], [7, 5, 1.0]]) output_boxes = batch_box_clip(input_boxes, im_info, lod[0]) self.inputs = { diff --git a/python/paddle/fluid/tests/unittests/test_box_coder_op.py b/python/paddle/fluid/tests/unittests/test_box_coder_op.py index 0e70f9904b5..6813a88070d 100644 --- a/python/paddle/fluid/tests/unittests/test_box_coder_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_coder_op.py @@ -32,9 +32,11 @@ def box_decoder(t_box, p_box, pb_v, output_box, norm, axis=0): pb_y = pb_y.reshape(shape) if pb_v.ndim == 2: - var_shape = (1, pb_v.shape[0], - pb_v.shape[1]) if axis == 0 else (pb_v.shape[0], 1, - pb_v.shape[1]) + var_shape = ( + (1, pb_v.shape[0], pb_v.shape[1]) + if axis == 0 + else (pb_v.shape[0], 1, pb_v.shape[1]) + ) pb_v = pb_v.reshape(var_shape) if pb_v.ndim == 1: tb_x = pb_v[0] * t_box[:, :, 0] * pb_w + pb_x @@ -90,18 +92,21 @@ def batch_box_coder(p_box, pb_v, t_box, lod, code_type, norm, axis=0): output_box = np.zeros((n, m, 4), dtype=np.float32) cur_offset = 0 for i in range(len(lod)): - if (code_type == "EncodeCenterSize"): - box_encoder(t_box[cur_offset:(cur_offset + lod[i]), :], p_box, pb_v, - output_box[cur_offset:(cur_offset + lod[i]), :, :], - norm) - elif (code_type == "DecodeCenterSize"): + if code_type == "EncodeCenterSize": + box_encoder( + t_box[cur_offset : (cur_offset + lod[i]), :], + p_box, + pb_v, + output_box[cur_offset : (cur_offset + lod[i]), :, :], + norm, + ) + elif code_type == "DecodeCenterSize": box_decoder(t_box, p_box, pb_v, output_box, norm, axis) cur_offset += lod[i] return output_box class TestBoxCoderOp(OpTest): - def test_check_output(self): self.check_output(check_eager=True) @@ -114,8 +119,14 @@ class TestBoxCoderOp(OpTest): target_box = np.random.random((20, 81, 4)).astype('float32') code_type = "DecodeCenterSize" box_normalized = False - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + ) self.inputs = { 'PriorBox': prior_box, 'PriorBoxVar': prior_box_var, @@ -123,13 +134,12 @@ class TestBoxCoderOp(OpTest): } self.attrs = { 'code_type': 'decode_center_size', - 'box_normalized': False + 'box_normalized': False, } self.outputs = {'OutputBox': output_box} class TestBoxCoderOpWithoutBoxVar(OpTest): - def test_check_output(self): self.check_output(check_eager=True) @@ -142,8 +152,14 @@ class TestBoxCoderOpWithoutBoxVar(OpTest): target_box = np.random.random((20, 81, 4)).astype('float32') code_type = "DecodeCenterSize" box_normalized = False - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + ) self.inputs = { 'PriorBox': prior_box, @@ -152,13 +168,12 @@ class TestBoxCoderOpWithoutBoxVar(OpTest): } self.attrs = { 'code_type': 'decode_center_size', - 'box_normalized': False + 'box_normalized': False, } self.outputs = {'OutputBox': output_box} class TestBoxCoderOpWithLoD(OpTest): - def test_check_output(self): self.check_output(check_eager=True) @@ -171,8 +186,14 @@ class TestBoxCoderOpWithLoD(OpTest): target_box = np.random.random((50, 4)).astype('float32') code_type = "EncodeCenterSize" box_normalized = True - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + ) self.inputs = { 'PriorBox': prior_box, @@ -184,7 +205,6 @@ class TestBoxCoderOpWithLoD(OpTest): class TestBoxCoderOpWithAxis(OpTest): - def test_check_output(self): self.check_output(check_eager=True) @@ -198,8 +218,15 @@ class TestBoxCoderOpWithAxis(OpTest): code_type = "DecodeCenterSize" box_normalized = False axis = 1 - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized, axis) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + axis, + ) self.inputs = { 'PriorBox': prior_box, @@ -209,13 +236,12 @@ class TestBoxCoderOpWithAxis(OpTest): self.attrs = { 'code_type': 'decode_center_size', 'box_normalized': False, - 'axis': axis + 'axis': axis, } self.outputs = {'OutputBox': output_box} class TestBoxCoderOpWithVariance(OpTest): - def test_check_output(self): self.check_output() @@ -228,8 +254,15 @@ class TestBoxCoderOpWithVariance(OpTest): code_type = "DecodeCenterSize" box_normalized = False axis = 1 - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized, axis) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + axis, + ) self.inputs = { 'PriorBox': prior_box, @@ -239,13 +272,12 @@ class TestBoxCoderOpWithVariance(OpTest): 'code_type': 'decode_center_size', 'box_normalized': False, 'variance': prior_box_var.astype(np.float64).flatten(), - 'axis': axis + 'axis': axis, } self.outputs = {'OutputBox': output_box} class TestBoxCoderOpWithVarianceDygraphAPI(unittest.TestCase): - def setUp(self): self.lod = [[1, 1, 1, 1, 1]] self.prior_box = np.random.random((30, 4)).astype('float32') @@ -254,16 +286,20 @@ class TestBoxCoderOpWithVarianceDygraphAPI(unittest.TestCase): self.code_type = "DecodeCenterSize" self.box_normalized = False self.axis = 1 - self.output_ref = batch_box_coder(self.prior_box, self.prior_box_var, - self.target_box, self.lod[0], - self.code_type, self.box_normalized, - self.axis) + self.output_ref = batch_box_coder( + self.prior_box, + self.prior_box_var, + self.target_box, + self.lod[0], + self.code_type, + self.box_normalized, + self.axis, + ) self.place = [paddle.CPUPlace()] if core.is_compiled_with_cuda(): self.place.append(paddle.CUDAPlace(0)) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) output_box = paddle.fluid.layers.box_coder( @@ -272,10 +308,11 @@ class TestBoxCoderOpWithVarianceDygraphAPI(unittest.TestCase): paddle.to_tensor(self.target_box), "decode_center_size", self.box_normalized, - axis=self.axis) - np.testing.assert_allclose(np.sum(self.output_ref), - np.sum(output_box.numpy()), - rtol=1e-05) + axis=self.axis, + ) + np.testing.assert_allclose( + np.sum(self.output_ref), np.sum(output_box.numpy()), rtol=1e-05 + ) paddle.enable_static() for place in self.place: diff --git a/python/paddle/fluid/tests/unittests/test_box_decoder_and_assign_op.py b/python/paddle/fluid/tests/unittests/test_box_decoder_and_assign_op.py index 41158533a62..0b826e362e7 100644 --- a/python/paddle/fluid/tests/unittests/test_box_decoder_and_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_decoder_and_assign_op.py @@ -60,7 +60,6 @@ def box_decoder_and_assign(deltas, weights, boxes, box_score, box_clip): class TestBoxDecoderAndAssignOpWithLoD(OpTest): - def test_check_output(self): self.check_output() @@ -74,7 +73,8 @@ class TestBoxDecoderAndAssignOpWithLoD(OpTest): box_score = np.random.random((20, num_classes)).astype('float32') box_clip = 4.135 output_box, output_assign_box = box_decoder_and_assign( - target_box, prior_box_var, prior_box, box_score, box_clip) + target_box, prior_box_var, prior_box, box_score, box_clip + ) self.inputs = { 'PriorBox': (prior_box, lod), @@ -85,7 +85,7 @@ class TestBoxDecoderAndAssignOpWithLoD(OpTest): self.attrs = {'box_clip': box_clip} self.outputs = { 'DecodeBox': output_box, - 'OutputAssignBox': output_assign_box + 'OutputAssignBox': output_assign_box, } diff --git a/python/paddle/fluid/tests/unittests/test_boxps.py b/python/paddle/fluid/tests/unittests/test_boxps.py index a3d3607fd8e..a51f86a7b2b 100644 --- a/python/paddle/fluid/tests/unittests/test_boxps.py +++ b/python/paddle/fluid/tests/unittests/test_boxps.py @@ -21,7 +21,7 @@ from paddle.fluid.transpiler import collective class TestTranspile(unittest.TestCase): - """ TestCases for BoxPS Preload """ + """TestCases for BoxPS Preload""" def get_transpile(self, mode, trainers="127.0.0.1:6174"): config = fluid.DistributeTranspilerConfig() @@ -34,44 +34,52 @@ class TestTranspile(unittest.TestCase): main_program = fluid.Program() startup_program = fluid.Program() t = self.get_transpile("single_process_multi_thread") - t.transpile(trainer_id=0, - startup_program=startup_program, - trainers="127.0.0.1:6174", - program=main_program) + t.transpile( + trainer_id=0, + startup_program=startup_program, + trainers="127.0.0.1:6174", + program=main_program, + ) t = self.get_transpile("grad_allreduce") try: - t.transpile(trainer_id=0, - startup_program=startup_program, - trainers="127.0.0.1:6174", - program=main_program) + t.transpile( + trainer_id=0, + startup_program=startup_program, + trainers="127.0.0.1:6174", + program=main_program, + ) except ValueError as e: print(e) def test_single_trainers(self): transpiler = collective.GradAllReduce(0) try: - transpiler.transpile(startup_program=fluid.Program(), - main_program=fluid.Program(), - rank=1, - endpoints="127.0.0.1:6174", - current_endpoint="127.0.0.1:6174", - wait_port="6174") + transpiler.transpile( + startup_program=fluid.Program(), + main_program=fluid.Program(), + rank=1, + endpoints="127.0.0.1:6174", + current_endpoint="127.0.0.1:6174", + wait_port="6174", + ) except ValueError as e: print(e) transpiler = collective.LocalSGD(0) try: - transpiler.transpile(startup_program=fluid.Program(), - main_program=fluid.Program(), - rank=1, - endpoints="127.0.0.1:6174", - current_endpoint="127.0.0.1:6174", - wait_port="6174") + transpiler.transpile( + startup_program=fluid.Program(), + main_program=fluid.Program(), + rank=1, + endpoints="127.0.0.1:6174", + current_endpoint="127.0.0.1:6174", + wait_port="6174", + ) except ValueError as e: print(e) class TestRunCmd(unittest.TestCase): - """ TestCases for run_cmd""" + """TestCases for run_cmd""" def test_run_cmd(self): ret1 = int(core.run_cmd("ls; echo $?").strip().split('\n')[-1]) @@ -81,20 +89,18 @@ class TestRunCmd(unittest.TestCase): class TestPullBoxSparseOP(unittest.TestCase): - """ TestCases for _pull_box_sparse op""" + """TestCases for _pull_box_sparse op""" def test_pull_box_sparse_op(self): paddle.enable_static() program = fluid.Program() with fluid.program_guard(program): - x = fluid.layers.data(name='x', - shape=[1], - dtype='int64', - lod_level=0) - y = fluid.layers.data(name='y', - shape=[1], - dtype='int64', - lod_level=0) + x = fluid.layers.data( + name='x', shape=[1], dtype='int64', lod_level=0 + ) + y = fluid.layers.data( + name='y', shape=[1], dtype='int64', lod_level=0 + ) emb_x, emb_y = _pull_box_sparse([x, y], size=1) diff --git a/python/paddle/fluid/tests/unittests/test_bpr_loss_op.py b/python/paddle/fluid/tests/unittests/test_bpr_loss_op.py index d527f4579ed..5f8953aaea3 100644 --- a/python/paddle/fluid/tests/unittests/test_bpr_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_bpr_loss_op.py @@ -18,8 +18,7 @@ from op_test import OpTest, randomize_probability class TestBprLossOp1(OpTest): - """Test BprLoss with discrete one-hot labels. - """ + """Test BprLoss with discrete one-hot labels.""" def setUp(self): self.op_type = "bpr_loss" @@ -33,7 +32,7 @@ class TestBprLossOp1(OpTest): for j in range(class_num): if j == label[i][0]: continue - sum += (-np.log(1.0 + np.exp(X[i][j] - X[i][label[i][0]]))) + sum += -np.log(1.0 + np.exp(X[i][j] - X[i][label[i][0]])) bpr_loss_result.append(-sum / (class_num - 1)) bpr_loss = np.asmatrix([[x] for x in bpr_loss_result], dtype="float64") self.inputs = {"X": X, "Label": label} diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_error.py b/python/paddle/fluid/tests/unittests/test_broadcast_error.py index bc1d0268158..517de67fd6d 100644 --- a/python/paddle/fluid/tests/unittests/test_broadcast_error.py +++ b/python/paddle/fluid/tests/unittests/test_broadcast_error.py @@ -19,7 +19,6 @@ import paddle.fluid.core as core class TestBroadcastOpCpu(OpTest): - def setUp(self): self.op_type = "broadcast" input = np.random.random((100, 2)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_shape.py b/python/paddle/fluid/tests/unittests/test_broadcast_shape.py index fedcb9bb465..1afb046b68b 100644 --- a/python/paddle/fluid/tests/unittests/test_broadcast_shape.py +++ b/python/paddle/fluid/tests/unittests/test_broadcast_shape.py @@ -17,18 +17,19 @@ import paddle class TestBroadcastShape(unittest.TestCase): - def test_result(self): shape = paddle.broadcast_shape([2, 1, 3], [1, 3, 1]) self.assertEqual(shape, [2, 3, 3]) shape = paddle.broadcast_shape( - [-1, 1, 3], [1, 3, 1]) #support compile time infershape + [-1, 1, 3], [1, 3, 1] + ) # support compile time infershape self.assertEqual(shape, [-1, 3, 3]) def test_error(self): - self.assertRaises(ValueError, paddle.broadcast_shape, [2, 1, 3], - [3, 3, 1]) + self.assertRaises( + ValueError, paddle.broadcast_shape, [2, 1, 3], [3, 3, 1] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py b/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py index 3c262048561..3872dadebd8 100644 --- a/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py +++ b/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py @@ -80,7 +80,6 @@ def gen_mixed_tensors_test(dtype): class TestCPUBroadcastTensorsOp(OpTest): - def set_place(self): self.place = core.CPUPlace() @@ -92,7 +91,9 @@ class TestCPUBroadcastTensorsOp(OpTest): self.use_mkldnn = False self.attrs = {'use_mkldnn': self.use_mkldnn} self.test_gen_func_list = [ - gen_rank_diff_test, gen_no_broadcast_test, gen_mixed_tensors_test + gen_rank_diff_test, + gen_no_broadcast_test, + gen_mixed_tensors_test, ] self.set_place() self.set_dtypes() @@ -117,35 +118,38 @@ class TestCPUBroadcastTensorsOp(OpTest): test_func(**args) def test_check_output(self): - self.run_dual_test(self.check_output_with_place, { - "place": self.place, - "atol": 1e-1, - "check_eager": True - }) + self.run_dual_test( + self.check_output_with_place, + {"place": self.place, "atol": 1e-1, "check_eager": True}, + ) def test_check_grad_normal(self): self.run_dual_test( - self.check_grad_with_place, { + self.check_grad_with_place, + { "place": self.place, "inputs_to_check": ['x0', 'x1'], "output_names": ['out0', 'out1'], "max_relative_error": 0.05, - "check_eager": True - }) + "check_eager": True, + }, + ) self.run_triple_in_test( - self.check_grad_with_place, { + self.check_grad_with_place, + { "place": self.place, "inputs_to_check": ['x0', 'x1', 'x2'], "output_names": ['out0', 'out1', "out2"], "max_relative_error": 0.05, - "check_eager": True - }) + "check_eager": True, + }, + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDABroadcastTensorsOp(TestCPUBroadcastTensorsOp): - def set_place(self): self.place = core.CUDAPlace(0) @@ -156,17 +160,15 @@ class TestCUDABroadcastTensorsOp(TestCPUBroadcastTensorsOp): class TestBroadcastTensorsAPI(unittest.TestCase): - def test_api(self): - def test_static(): inputs = [ - paddle.fluid.layers.data(shape=[4, 1, 4, 1], - dtype='float32', - name="x0"), - paddle.fluid.layers.data(shape=[1, 4, 1, 4], - dtype='float32', - name="x1") + paddle.fluid.layers.data( + shape=[4, 1, 4, 1], dtype='float32', name="x0" + ), + paddle.fluid.layers.data( + shape=[1, 4, 1, 4], dtype='float32', name="x1" + ), ] paddle.broadcast_tensors(inputs) @@ -175,9 +177,11 @@ class TestBroadcastTensorsAPI(unittest.TestCase): try: inputs = [ paddle.to_tensor( - np.random.random([4, 1, 4, 1]).astype("float32")), + np.random.random([4, 1, 4, 1]).astype("float32") + ), paddle.to_tensor( - np.random.random([1, 4, 1, 4]).astype("float32")) + np.random.random([1, 4, 1, 4]).astype("float32") + ), ] paddle.broadcast_tensors(inputs) finally: @@ -188,39 +192,37 @@ class TestBroadcastTensorsAPI(unittest.TestCase): class TestRaiseBroadcastTensorsError(unittest.TestCase): - def test_errors(self): - def test_type(): inputs = [ - paddle.fluid.layers.data(shape=[1, 1, 1, 1], - dtype='float32', - name="x4"), - paddle.fluid.layers.data(shape=[1, 4, 1, 1], - dtype='float64', - name="x5") + paddle.fluid.layers.data( + shape=[1, 1, 1, 1], dtype='float32', name="x4" + ), + paddle.fluid.layers.data( + shape=[1, 4, 1, 1], dtype='float64', name="x5" + ), ] paddle.broadcast_tensors(inputs) def test_dtype(): inputs = [ - paddle.fluid.layers.data(shape=[1, 1, 1, 1], - dtype='int8', - name="x6"), - paddle.fluid.layers.data(shape=[1, 4, 1, 1], - dtype='int8', - name="x7") + paddle.fluid.layers.data( + shape=[1, 1, 1, 1], dtype='int8', name="x6" + ), + paddle.fluid.layers.data( + shape=[1, 4, 1, 1], dtype='int8', name="x7" + ), ] paddle.broadcast_tensors(inputs) def test_bcast_semantics(): inputs = [ - paddle.fluid.layers.data(shape=[1, 3, 1, 1], - dtype='float32', - name="x9"), - paddle.fluid.layers.data(shape=[1, 8, 1, 1], - dtype='float32', - name="x10") + paddle.fluid.layers.data( + shape=[1, 3, 1, 1], dtype='float32', name="x9" + ), + paddle.fluid.layers.data( + shape=[1, 8, 1, 1], dtype='float32', name="x10" + ), ] paddle.broadcast_tensors(inputs) @@ -230,33 +232,37 @@ class TestRaiseBroadcastTensorsError(unittest.TestCase): class TestRaiseBroadcastTensorsErrorDyGraph(unittest.TestCase): - def test_errors(self): - def test_type(): inputs = [ paddle.to_tensor( - np.ones(shape=[1, 1, 1, 1], dtype='float32', name="x4")), + np.ones(shape=[1, 1, 1, 1], dtype='float32', name="x4") + ), paddle.to_tensor( - np.ones(shape=[1, 4, 1, 1], dtype='float64', name="x5")) + np.ones(shape=[1, 4, 1, 1], dtype='float64', name="x5") + ), ] paddle.broadcast_tensors(inputs) def test_dtype(): inputs = [ paddle.to_tensor( - np.ones(shape=[1, 1, 1, 1], dtype='int8', name="x6")), + np.ones(shape=[1, 1, 1, 1], dtype='int8', name="x6") + ), paddle.to_tensor( - np.ones(shape=[1, 4, 1, 1], dtype='int8', name="x7")) + np.ones(shape=[1, 4, 1, 1], dtype='int8', name="x7") + ), ] paddle.broadcast_tensors(inputs) def test_bcast_semantics(): inputs = [ paddle.to_tensor( - np.ones(shape=[1, 3, 1, 1], dtype='float32', name="x9")), + np.ones(shape=[1, 3, 1, 1], dtype='float32', name="x9") + ), paddle.to_tensor( - np.ones(shape=[1, 8, 1, 1], dtype='float32', name="x10")) + np.ones(shape=[1, 8, 1, 1], dtype='float32', name="x10") + ), ] paddle.broadcast_tensors(inputs) diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_to_op.py b/python/paddle/fluid/tests/unittests/test_broadcast_to_op.py index aa97e316a3d..e8e876766c3 100644 --- a/python/paddle/fluid/tests/unittests/test_broadcast_to_op.py +++ b/python/paddle/fluid/tests/unittests/test_broadcast_to_op.py @@ -22,11 +22,11 @@ paddle.enable_static() class TestBroadcastToError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) shape = [2, 2] self.assertRaises(TypeError, paddle.tensor.broadcast_to, x1, shape) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -38,19 +38,19 @@ class TestBroadcastToError(unittest.TestCase): # Test python API class TestBroadcastToAPI(unittest.TestCase): - def test_api(self): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.broadcast_to(x, shape=[12, 14]) out_2 = paddle.broadcast_to(x, shape=[positive_2, 14]) @@ -59,14 +59,14 @@ class TestBroadcastToAPI(unittest.TestCase): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) assert np.array_equal(res_3, np.tile(input, (1, 1))) diff --git a/python/paddle/fluid/tests/unittests/test_bucketize_api.py b/python/paddle/fluid/tests/unittests/test_bucketize_api.py index e62c1a6028f..e82d6e9d15c 100644 --- a/python/paddle/fluid/tests/unittests/test_bucketize_api.py +++ b/python/paddle/fluid/tests/unittests/test_bucketize_api.py @@ -38,20 +38,20 @@ class TestBucketizeAPI(unittest.TestCase): sorted_sequence = paddle.static.data( 'SortedSequence', shape=self.sorted_sequence.shape, - dtype="float64") + dtype="float64", + ) x = paddle.static.data('x', shape=self.x.shape, dtype="float64") out1 = paddle.bucketize(x, sorted_sequence) out2 = paddle.bucketize(x, sorted_sequence, right=True) exe = paddle.static.Executor(place) - res = exe.run(feed={ - 'SortedSequence': self.sorted_sequence, - 'x': self.x - }, - fetch_list=[out1, out2]) + res = exe.run( + feed={'SortedSequence': self.sorted_sequence, 'x': self.x}, + fetch_list=[out1, out2], + ) out_ref = np.searchsorted(self.sorted_sequence, self.x) - out_ref1 = np.searchsorted(self.sorted_sequence, - self.x, - side='right') + out_ref1 = np.searchsorted( + self.sorted_sequence, self.x, side='right' + ) np.testing.assert_allclose(out_ref, res[0], rtol=1e-05) np.testing.assert_allclose(out_ref1, res[1], rtol=1e-05) @@ -59,7 +59,6 @@ class TestBucketizeAPI(unittest.TestCase): run(place) def test_api_dygraph(self): - def run(place): paddle.disable_static(place) sorted_sequence = paddle.to_tensor(self.sorted_sequence) @@ -67,9 +66,9 @@ class TestBucketizeAPI(unittest.TestCase): out1 = paddle.bucketize(x, sorted_sequence) out2 = paddle.bucketize(x, sorted_sequence, right=True) out_ref1 = np.searchsorted(self.sorted_sequence, self.x) - out_ref2 = np.searchsorted(self.sorted_sequence, - self.x, - side='right') + out_ref2 = np.searchsorted( + self.sorted_sequence, self.x, side='right' + ) np.testing.assert_allclose(out_ref1, out1.numpy(), rtol=1e-05) np.testing.assert_allclose(out_ref2, out2.numpy(), rtol=1e-05) paddle.enable_static() @@ -86,9 +85,9 @@ class TestBucketizeAPI(unittest.TestCase): def test_bucketize_dims_error(self): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 2], - dtype="float64") + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 2], dtype="float64" + ) x = paddle.static.data('x', shape=[2, 5], dtype="float64") self.assertRaises(ValueError, paddle.bucketize, x, sorted_sequence) @@ -96,16 +95,18 @@ class TestBucketizeAPI(unittest.TestCase): for place in self.place: paddle.disable_static(place) sorted_sequence = paddle.to_tensor(self.sorted_sequence) - self.assertRaises(ValueError, paddle.bucketize, self.x, - sorted_sequence) + self.assertRaises( + ValueError, paddle.bucketize, self.x, sorted_sequence + ) def test_empty_input_error(self): for place in self.place: paddle.disable_static(place) sorted_sequence = paddle.to_tensor(self.sorted_sequence) x = paddle.to_tensor(self.x) - self.assertRaises(ValueError, paddle.bucketize, None, - sorted_sequence) + self.assertRaises( + ValueError, paddle.bucketize, None, sorted_sequence + ) self.assertRaises(AttributeError, paddle.bucketize, x, None) diff --git a/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass.py b/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass.py index e7d9b8e485d..5534ff67915 100644 --- a/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass.py +++ b/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass.py @@ -22,16 +22,14 @@ import unittest batch_size = 32 feed_dict = { - 'image': - np.random.random([batch_size, 784]).astype('float32'), - 'label': - np.random.random_integers(low=0, high=9, size=[batch_size, - 1]).astype('int64') + 'image': np.random.random([batch_size, 784]).astype('float32'), + 'label': np.random.random_integers( + low=0, high=9, size=[batch_size, 1] + ).astype('int64'), } class InplaceTestBase(unittest.TestCase): - def initParameter(self): self.use_cuda = True self.fuse_all_optimizer_ops = False @@ -61,8 +59,10 @@ class InplaceTestBase(unittest.TestCase): with fluid.scope_guard(scope): exe = fluid.Executor( - fluid.CUDAPlace(0) if self.use_cuda else fluid.CPUPlace( - )) + fluid.CUDAPlace(0) + if self.use_cuda + else fluid.CPUPlace() + ) exe.run(startup_program) return main_program, scope, exe, loss @@ -93,11 +93,14 @@ class InplaceTestBase(unittest.TestCase): build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = memory_optimize build_strategy.enable_inplace = enable_inplace - build_strategy.fuse_all_optimizer_ops = self.fuse_all_optimizer_ops + build_strategy.fuse_all_optimizer_ops = ( + self.fuse_all_optimizer_ops + ) compiled_prog = fluid.CompiledProgram(prog).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - places=self.place) + places=self.place, + ) compiled_programs.append(compiled_prog) all_vars_name = self.get_all_vars(prog1) @@ -107,24 +110,26 @@ class InplaceTestBase(unittest.TestCase): for fetch_var in repeated_var_names[:4]: for _ in range(2): with fluid.scope_guard(scope1): - fetch_val1, = exe.run(prog1, - feed=feed_dict, - fetch_list=[fetch_var]) + (fetch_val1,) = exe.run( + prog1, feed=feed_dict, fetch_list=[fetch_var] + ) for scope, compiled_prog in zip(scopes, compiled_programs): with fluid.scope_guard(scope): - fetch_val2, = exe.run(compiled_prog, - feed=feed_dict, - fetch_list=[fetch_var]) + (fetch_val2,) = exe.run( + compiled_prog, + feed=feed_dict, + fetch_list=[fetch_var], + ) np.testing.assert_array_equal( fetch_val1, fetch_val2, - err_msg= - 'error var name: {}, fetch_val1: {}, fetch_val2: {}' - .format( + err_msg='error var name: {}, fetch_val1: {}, fetch_val2: {}'.format( fetch_var, fetch_val1[~np.equal(fetch_val1, fetch_val2)], - fetch_val2[~np.equal(fetch_val1, fetch_val2)])) + fetch_val2[~np.equal(fetch_val1, fetch_val2)], + ), + ) def check_multi_card_fetch_var(self): if self.is_invalid_test(): @@ -146,11 +151,16 @@ class InplaceTestBase(unittest.TestCase): build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = memory_optimize build_strategy.enable_inplace = enable_inplace - build_strategy.fuse_all_optimizer_ops = self.fuse_all_optimizer_ops + build_strategy.fuse_all_optimizer_ops = ( + self.fuse_all_optimizer_ops + ) compiled_program = fluid.CompiledProgram( - prog).with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - places=places) + prog + ).with_data_parallel( + loss_name=loss.name, + build_strategy=build_strategy, + places=places, + ) compiled_programs.append(compiled_program) repeated_var_names = self.get_all_vars(prog1) @@ -161,9 +171,11 @@ class InplaceTestBase(unittest.TestCase): fetch_vals = [] for scope, compiled_prog in zip(scopes, compiled_programs): with fluid.scope_guard(scope): - fetch_val, = exe.run(compiled_prog, - feed=feed_dict, - fetch_list=[fetch_var]) + (fetch_val,) = exe.run( + compiled_prog, + feed=feed_dict, + fetch_list=[fetch_var], + ) fetch_vals.append(fetch_val) for item in fetch_vals: @@ -171,14 +183,15 @@ class InplaceTestBase(unittest.TestCase): np.testing.assert_array_equal( fetch_vals[0], item, - err_msg='error var name: {}, fetch_vals[0]: {}, item: {}' - .format(fetch_var, - fetch_vals[0][~np.equal(fetch_vals[0], item)], - item[~np.equal(fetch_vals[0], item)])) + err_msg='error var name: {}, fetch_vals[0]: {}, item: {}'.format( + fetch_var, + fetch_vals[0][~np.equal(fetch_vals[0], item)], + item[~np.equal(fetch_vals[0], item)], + ), + ) class CUDAInplaceTest(InplaceTestBase): - def initParameter(self): self.use_cuda = True self.fuse_all_optimizer_ops = False @@ -191,7 +204,6 @@ class CUDAInplaceTest(InplaceTestBase): class CPUInplaceTest(InplaceTestBase): - def initParameter(self): self.use_cuda = False self.fuse_all_optimizer_ops = False diff --git a/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass.py b/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass.py index 6ce5e64b0ee..e9e62bee006 100644 --- a/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass.py +++ b/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass.py @@ -17,7 +17,6 @@ import unittest class CUDAInplaceTestWithFuseOptimizationOps(InplaceTestBase): - def initParameter(self): self.use_cuda = True self.fuse_all_optimizer_ops = True @@ -31,7 +30,6 @@ class CUDAInplaceTestWithFuseOptimizationOps(InplaceTestBase): class CPUInplaceTestWithFuseOptimizationOps(InplaceTestBase): - def initParameter(self): self.use_cuda = False self.fuse_all_optimizer_ops = True diff --git a/python/paddle/fluid/tests/unittests/test_c_comm_init_all_op.py b/python/paddle/fluid/tests/unittests/test_c_comm_init_all_op.py index eb03c13602f..7f4eac0d3fb 100644 --- a/python/paddle/fluid/tests/unittests/test_c_comm_init_all_op.py +++ b/python/paddle/fluid/tests/unittests/test_c_comm_init_all_op.py @@ -18,7 +18,6 @@ import paddle.fluid as fluid class TestCCommInitAllOp(unittest.TestCase): - def setUp(self): self.place = fluid.CUDAPlace(0) self.exe = fluid.Executor(self.place) @@ -39,11 +38,9 @@ class TestCCommInitAllOp(unittest.TestCase): def test_specifying_devices(self): program = fluid.Program() block = program.global_block() - block.append_op(type='c_comm_init_all', - attrs={ - 'devices': [0], - 'ring_id': 1 - }) + block.append_op( + type='c_comm_init_all', attrs={'devices': [0], 'ring_id': 1} + ) self.exe.run(program) diff --git a/python/paddle/fluid/tests/unittests/test_c_embedding_op.py b/python/paddle/fluid/tests/unittests/test_c_embedding_op.py index 8b32a1857b2..66c68097127 100644 --- a/python/paddle/fluid/tests/unittests/test_c_embedding_op.py +++ b/python/paddle/fluid/tests/unittests/test_c_embedding_op.py @@ -13,7 +13,11 @@ # limitations under the License. import unittest -from paddle.fluid.tests.unittests.c_embedding_op_base import TestCEmbeddingCPU, TestCEmbeddingOpBase, TestCEmbeddingOpFP32 +from paddle.fluid.tests.unittests.c_embedding_op_base import ( + TestCEmbeddingCPU, + TestCEmbeddingOpBase, + TestCEmbeddingOpFP32, +) TestCEmbeddingCPU() diff --git a/python/paddle/fluid/tests/unittests/test_calc_gradient.py b/python/paddle/fluid/tests/unittests/test_calc_gradient.py index 350f30aaf1a..59bc1f574e4 100644 --- a/python/paddle/fluid/tests/unittests/test_calc_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_calc_gradient.py @@ -21,7 +21,6 @@ from paddle.fluid.backward import calc_gradient class TestCalcGradient(unittest.TestCase): - def test_calc_gradient(self): main = fluid.Program() startup = fluid.Program() @@ -39,7 +38,6 @@ class TestCalcGradient(unittest.TestCase): class TestDoubleGrad(unittest.TestCase): - def test1(self): main = fluid.Program() startup = fluid.Program() @@ -49,10 +47,11 @@ class TestDoubleGrad(unittest.TestCase): name='x', shape=[1], dtype='float32', - default_initializer=fluid.initializer.Constant(3)) - grad1, = fluid.gradients(net(x), x) # 2x = 6 + default_initializer=fluid.initializer.Constant(3), + ) + (grad1,) = fluid.gradients(net(x), x) # 2x = 6 z = net(x - grad1) - grad2, = fluid.gradients(z, x) # gradients( (x - 2x)^2) = 2x = 6 + (grad2,) = fluid.gradients(z, x) # gradients( (x - 2x)^2) = 2x = 6 place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -69,21 +68,21 @@ class TestDoubleGrad(unittest.TestCase): name='x', shape=[1], dtype='float32', - default_initializer=fluid.initializer.Constant(1)) + default_initializer=fluid.initializer.Constant(1), + ) y = x * x - dx1, = fluid.gradients(y, x) + (dx1,) = fluid.gradients(y, x) z = dx1 * dx1 + y * y - dx2, = fluid.gradients(z, x) + (dx2,) = fluid.gradients(z, x) place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) - out, = exe.run(main, fetch_list=[dx2]) + (out,) = exe.run(main, fetch_list=[dx2]) self.assertEqual(12, out[0]) class TestGradientWithPrune(unittest.TestCase): - def test_prune(self): with paddle.fluid.scope_guard(paddle.static.Scope()): x = fluid.data(name='x', shape=[3], dtype='float32') @@ -95,14 +94,15 @@ class TestGradientWithPrune(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) main = fluid.default_main_program() exe.run(fluid.default_startup_program()) - out = exe.run(main, - feed={'x': np.ones([3]).astype('float32')}, - fetch_list=[x1_grad]) + out = exe.run( + main, + feed={'x': np.ones([3]).astype('float32')}, + fetch_list=[x1_grad], + ) np.testing.assert_array_equal(out[0], [2.0, 0.0, 0.0]) class TestDoubleGradient(unittest.TestCase): - def build_program(self): start_prog = paddle.static.Program() main_prog = paddle.static.Program() @@ -128,16 +128,17 @@ class TestDoubleGradient(unittest.TestCase): start_prog, main_prog, fetch_list = self.build_program() exe = paddle.static.Executor() exe.run(start_prog) - ans = exe.run(main_prog, - feed={'x': np.ones([2, 2]).astype(np.float32)}, - fetch_list=fetch_list) + ans = exe.run( + main_prog, + feed={'x': np.ones([2, 2]).astype(np.float32)}, + fetch_list=fetch_list, + ) self.assertEqual(len(ans), 2) - self.assertListEqual(ans[0].tolist(), [[0., 0.], [0., 0.]]) - self.assertListEqual(ans[1].tolist(), [[2., 2.], [2., 2.]]) + self.assertListEqual(ans[0].tolist(), [[0.0, 0.0], [0.0, 0.0]]) + self.assertListEqual(ans[1].tolist(), [[2.0, 2.0], [2.0, 2.0]]) class TestDoubleGradient2(unittest.TestCase): - def build_program(self): start_prog = paddle.static.Program() main_prog = paddle.static.Program() @@ -156,8 +157,9 @@ class TestDoubleGradient2(unittest.TestCase): grad_x = paddle.static.gradients(y, x, grad_y) grad_x2 = paddle.static.gradients(y2, x, grad_y) # test with multi targets - jvp = paddle.static.gradients([grad_x[0], grad_x2[0]], grad_y, - [v, v]) + jvp = paddle.static.gradients( + [grad_x[0], grad_x2[0]], grad_y, [v, v] + ) return start_prog, main_prog, [grad_x, jvp] @@ -166,12 +168,14 @@ class TestDoubleGradient2(unittest.TestCase): start_prog, main_prog, fetch_list = self.build_program() exe = paddle.static.Executor() exe.run(start_prog) - ans = exe.run(main_prog, - feed={'x': np.ones([2, 2]).astype(np.float32)}, - fetch_list=fetch_list) + ans = exe.run( + main_prog, + feed={'x': np.ones([2, 2]).astype(np.float32)}, + fetch_list=fetch_list, + ) self.assertEqual(len(ans), 2) - self.assertListEqual(ans[0].tolist(), [[0., 0.], [0., 0.]]) - self.assertListEqual(ans[1].tolist(), [[5., 5.], [5., 5.]]) + self.assertListEqual(ans[0].tolist(), [[0.0, 0.0], [0.0, 0.0]]) + self.assertListEqual(ans[1].tolist(), [[5.0, 5.0], [5.0, 5.0]]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_case.py b/python/paddle/fluid/tests/unittests/test_case.py index e550adc6a9a..62a3898fb97 100644 --- a/python/paddle/fluid/tests/unittests/test_case.py +++ b/python/paddle/fluid/tests/unittests/test_case.py @@ -25,9 +25,7 @@ import paddle.fluid.optimizer as optimizer class TestAPICase(unittest.TestCase): - def test_return_single_var(self): - def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) @@ -47,16 +45,19 @@ class TestAPICase(unittest.TestCase): pred_1 = layers.less_than(z, x) # true: 0.2 < 0.3 # call fn_1 - out_0 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_1, fn_2)], - default=fn_3) + out_0 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_1, fn_2)], default=fn_3 + ) # call fn_2 - out_1 = layers.case(pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], - default=fn_3) + out_1 = layers.case( + pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], default=fn_3 + ) # call default fn_3 - out_2 = layers.case(pred_fn_pairs=((pred_2, fn_1), (pred_2, fn_2)), - default=fn_3) + out_2 = layers.case( + pred_fn_pairs=((pred_2, fn_1), (pred_2, fn_2)), default=fn_3 + ) # no default, call fn_2 out_3 = layers.case(pred_fn_pairs=[(pred_1, fn_2)]) @@ -64,12 +65,16 @@ class TestAPICase(unittest.TestCase): # no default, call fn_2. but pred_2 is false out_4 = layers.case(pred_fn_pairs=[(pred_2, fn_2)]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - fetch_list=[out_0, out_1, out_2, out_3, out_4]) + res = exe.run( + main_program, fetch_list=[out_0, out_1, out_2, out_3, out_4] + ) np.testing.assert_allclose(res[0], 1, rtol=1e-05) np.testing.assert_allclose(res[1], 2, rtol=1e-05) @@ -78,27 +83,20 @@ class TestAPICase(unittest.TestCase): np.testing.assert_allclose(res[4], 2, rtol=1e-05) def test_return_var_tuple(self): - def fn_1(): - return layers.fill_constant(shape=[1, 2], dtype='int32', - value=1), layers.fill_constant( - shape=[2, 3], - dtype='float32', - value=2) + return layers.fill_constant( + shape=[1, 2], dtype='int32', value=1 + ), layers.fill_constant(shape=[2, 3], dtype='float32', value=2) def fn_2(): - return layers.fill_constant(shape=[3, 4], dtype='int32', - value=3), layers.fill_constant( - shape=[4, 5], - dtype='float32', - value=4) + return layers.fill_constant( + shape=[3, 4], dtype='int32', value=3 + ), layers.fill_constant(shape=[4, 5], dtype='float32', value=4) def fn_3(): - return layers.fill_constant(shape=[5], dtype='int32', - value=5), layers.fill_constant( - shape=[5, 6], - dtype='float32', - value=6) + return layers.fill_constant( + shape=[5], dtype='int32', value=5 + ), layers.fill_constant(shape=[5, 6], dtype='float32', value=6) main_program = Program() startup_program = Program() @@ -112,56 +110,87 @@ class TestAPICase(unittest.TestCase): out = layers.case(((pred_1, fn_1), (pred_2, fn_2)), fn_3) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=out) - np.testing.assert_allclose(np.asarray(ret[0]), - np.full((1, 2), 1, np.int32), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(ret[1]), - np.full((2, 3), 2, np.float32), - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(ret[0]), np.full((1, 2), 1, np.int32), rtol=1e-05 + ) + np.testing.assert_allclose( + np.asarray(ret[1]), np.full((2, 3), 2, np.float32), rtol=1e-05 + ) class TestAPICase_Nested(unittest.TestCase): - def test_nested_case(self): - def fn_1(x=1): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) - out = layers.case(pred_fn_pairs=[ - (var_5 < var_6, - partial( - layers.fill_constant, shape=[1], dtype='int32', value=x)), - (var_5 == var_6, - partial( - layers.fill_constant, shape=[2], dtype='int32', value=x)) - ]) + out = layers.case( + pred_fn_pairs=[ + ( + var_5 < var_6, + partial( + layers.fill_constant, + shape=[1], + dtype='int32', + value=x, + ), + ), + ( + var_5 == var_6, + partial( + layers.fill_constant, + shape=[2], + dtype='int32', + value=x, + ), + ), + ] + ) return out def fn_2(x=2): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) - out = layers.case(pred_fn_pairs=[ - (var_5 < var_6, partial(fn_1, x=x)), - (var_5 == var_6, - partial( - layers.fill_constant, shape=[2], dtype='int32', value=x)) - ]) + out = layers.case( + pred_fn_pairs=[ + (var_5 < var_6, partial(fn_1, x=x)), + ( + var_5 == var_6, + partial( + layers.fill_constant, + shape=[2], + dtype='int32', + value=x, + ), + ), + ] + ) return out def fn_3(): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) - out = layers.case(pred_fn_pairs=[ - (var_5 < var_6, partial(fn_2, x=3)), - (var_5 == var_6, - partial( - layers.fill_constant, shape=[2], dtype='int32', value=7)) - ]) + out = layers.case( + pred_fn_pairs=[ + (var_5 < var_6, partial(fn_2, x=3)), + ( + var_5 == var_6, + partial( + layers.fill_constant, + shape=[2], + dtype='int32', + value=7, + ), + ), + ] + ) return out main_program = Program() @@ -173,17 +202,23 @@ class TestAPICase_Nested(unittest.TestCase): pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_1 = layers.less_than(z, x) # true: 0.2 < 0.3 - out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], - default=fn_3) + out_1 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3 + ) - out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], - default=fn_3) + out_2 = layers.case( + pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], default=fn_3 + ) - out_3 = layers.case(pred_fn_pairs=[(x == y, fn_1), (x == z, fn_2)], - default=fn_3) + out_3 = layers.case( + pred_fn_pairs=[(x == y, fn_1), (x == z, fn_2)], default=fn_3 + ) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) res = exe.run(main_program, fetch_list=[out_1, out_2, out_3]) @@ -194,9 +229,7 @@ class TestAPICase_Nested(unittest.TestCase): class TestAPICase_Error(unittest.TestCase): - def test_error(self): - def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) @@ -246,18 +279,17 @@ class TestAPICase_Error(unittest.TestCase): # when optimizer in case class TestMutiTask(unittest.TestCase): - def test_optimizer_in_case(self): BATCH_SIZE = 1 INPUT_SIZE = 784 EPOCH_NUM = 2 - x = fluid.data(name='x', - shape=[BATCH_SIZE, INPUT_SIZE], - dtype='float32') - y = fluid.data(name='y', - shape=[BATCH_SIZE, INPUT_SIZE], - dtype='float32') + x = fluid.data( + name='x', shape=[BATCH_SIZE, INPUT_SIZE], dtype='float32' + ) + y = fluid.data( + name='y', shape=[BATCH_SIZE, INPUT_SIZE], dtype='float32' + ) switch_id = fluid.data(name='switch_id', shape=[1], dtype='int32') @@ -282,16 +314,19 @@ class TestMutiTask(unittest.TestCase): for epoch in range(EPOCH_NUM): np.random.seed(epoch) - feed_image = np.random.random( - size=[BATCH_SIZE, INPUT_SIZE]).astype('float32') + feed_image = np.random.random(size=[BATCH_SIZE, INPUT_SIZE]).astype( + 'float32' + ) main_program = fluid.default_main_program() - out = exe.run(main_program, - feed={ - 'x': feed_image, - 'y': feed_image, - 'switch_id': np.array([epoch]).astype('int32') - }, - fetch_list=[]) + out = exe.run( + main_program, + feed={ + 'x': feed_image, + 'y': feed_image, + 'switch_id': np.array([epoch]).astype('int32'), + }, + fetch_list=[], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cast_op.py b/python/paddle/fluid/tests/unittests/test_cast_op.py index f4cabd6519c..8665e285fa5 100644 --- a/python/paddle/fluid/tests/unittests/test_cast_op.py +++ b/python/paddle/fluid/tests/unittests/test_cast_op.py @@ -27,14 +27,13 @@ import paddle.fluid.layers as layers class TestCastOpFp32ToFp64(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float64')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP64) + 'out_dtype': int(core.VarDesc.VarType.FP64), } self.op_type = 'cast' @@ -46,14 +45,13 @@ class TestCastOpFp32ToFp64(OpTest): class TestCastOpFp16ToFp32(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float16')} self.outputs = {'Out': ipt.astype('float32')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP16), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'cast' self.__class__.no_need_check_grad = True @@ -63,14 +61,13 @@ class TestCastOpFp16ToFp32(OpTest): class TestCastOpFp32ToFp16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float16')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.op_type = 'cast' self.__class__.no_need_check_grad = True @@ -80,14 +77,13 @@ class TestCastOpFp32ToFp16(OpTest): class TestCastOpBf16ToFp32(OpTest): - def setUp(self): ipt = np.array(np.random.randint(10, size=[10, 10])).astype('uint16') self.inputs = {'X': ipt} self.outputs = {'Out': convert_uint16_to_float(ipt)} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.BF16), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'cast' self.__class__.no_need_check_grad = True @@ -97,14 +93,13 @@ class TestCastOpBf16ToFp32(OpTest): class TestCastOpFp32ToBf16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]).astype('float32') self.inputs = {'X': ipt} self.outputs = {'Out': convert_float_to_uint16(ipt)} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.BF16) + 'out_dtype': int(core.VarDesc.VarType.BF16), } self.op_type = 'cast' self.__class__.no_need_check_grad = True @@ -114,32 +109,31 @@ class TestCastOpFp32ToBf16(OpTest): class TestCastOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of cast_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.cast, x1, 'int32') class TestCastOpEager(unittest.TestCase): - def test_eager(self): with paddle.fluid.dygraph.base.guard(): with _test_eager_guard(): x = paddle.ones([2, 2], dtype="float16") x.stop_gradient = False out = paddle.cast(x, "float32") - np.testing.assert_array_equal(out, - np.ones([2, 2]).astype('float32')) + np.testing.assert_array_equal( + out, np.ones([2, 2]).astype('float32') + ) out.backward() np.testing.assert_array_equal(x.gradient(), x.numpy()) self.assertTrue(x.gradient().dtype == np.float16) class TestCastDoubleGradCheck(unittest.TestCase): - def cast_wrapper(self, x): return paddle.cast(x[0], 'float64') @@ -154,17 +148,13 @@ class TestCastDoubleGradCheck(unittest.TestCase): out = paddle.cast(data, 'float64') data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.cast_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.cast_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -176,7 +166,6 @@ class TestCastDoubleGradCheck(unittest.TestCase): class TestCastTripleGradCheck(unittest.TestCase): - def cast_wrapper(self, x): return paddle.cast(x[0], 'float64') @@ -191,17 +180,13 @@ class TestCastTripleGradCheck(unittest.TestCase): out = paddle.cast(data, 'float64') data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.cast_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.cast_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_center_loss.py b/python/paddle/fluid/tests/unittests/test_center_loss.py index 59c8b417d10..b7eda71c021 100644 --- a/python/paddle/fluid/tests/unittests/test_center_loss.py +++ b/python/paddle/fluid/tests/unittests/test_center_loss.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class TestCenterLossOp(OpTest): - def setUp(self): self.op_type = "center_loss" self.dtype = np.float64 @@ -47,7 +46,7 @@ class TestCenterLossOp(OpTest): cout[labels[i]] += 1 var_sum[labels[i]] += output[i] for i in range(cluster_num): - var_sum[i] /= (1 + cout[i]) + var_sum[i] /= 1 + cout[i] var_sum *= 0.1 result = centers + var_sum rate = np.array([0.1]).astype(np.float64) @@ -56,20 +55,20 @@ class TestCenterLossOp(OpTest): 'X': feat, 'Label': labels, 'Centers': centers, - 'CenterUpdateRate': rate + 'CenterUpdateRate': rate, } if self.need_update == True: self.outputs = { 'SampleCenterDiff': output, 'Loss': loss, - 'CentersOut': result + 'CentersOut': result, } else: self.outputs = { 'SampleCenterDiff': output, 'Loss': loss, - 'CentersOut': centers + 'CentersOut': centers, } def config(self): @@ -86,35 +85,34 @@ class TestCenterLossOp(OpTest): class TestCenterLossOpNoUpdate(TestCenterLossOp): - def config(self): self.need_update = False class BadInputTestCenterLoss(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): data = [[1, 2, 3, 4], [5, 6, 7, 8]] - label = fluid.layers.data(name='label', - shape=[2, 1], - dtype='int32') + label = fluid.layers.data( + name='label', shape=[2, 1], dtype='int32' + ) res = fluid.layers.center_loss( data, label, num_classes=1000, alpha=0.2, param_attr=fluid.initializer.Xavier(uniform=False), - update_center=True) + update_center=True, + ) self.assertRaises(TypeError, test_bad_x) def test_bad_y(): - data = fluid.layers.data(name='data', - shape=[2, 32], - dtype='float32') + data = fluid.layers.data( + name='data', shape=[2, 32], dtype='float32' + ) label = [[2], [3]] res = fluid.layers.center_loss( data, @@ -122,30 +120,38 @@ class BadInputTestCenterLoss(unittest.TestCase): num_classes=1000, alpha=0.2, param_attr=fluid.initializer.Xavier(uniform=False), - update_center=True) + update_center=True, + ) self.assertRaises(TypeError, test_bad_y) def test_bad_alpha(): - data = fluid.layers.data(name='data2', - shape=[2, 32], - dtype='float32', - append_batch_size=False) - label = fluid.layers.data(name='label2', - shape=[2, 1], - dtype='int32', - append_batch_size=False) - alpha = fluid.layers.data(name='alpha', - shape=[1], - dtype='int64', - append_batch_size=False) + data = fluid.layers.data( + name='data2', + shape=[2, 32], + dtype='float32', + append_batch_size=False, + ) + label = fluid.layers.data( + name='label2', + shape=[2, 1], + dtype='int32', + append_batch_size=False, + ) + alpha = fluid.layers.data( + name='alpha', + shape=[1], + dtype='int64', + append_batch_size=False, + ) res = fluid.layers.center_loss( data, label, num_classes=1000, alpha=alpha, param_attr=fluid.initializer.Xavier(uniform=False), - update_center=True) + update_center=True, + ) self.assertRaises(TypeError, test_bad_alpha) diff --git a/python/paddle/fluid/tests/unittests/test_channel_shuffle.py b/python/paddle/fluid/tests/unittests/test_channel_shuffle.py index 50df6bbc024..0da2a651477 100644 --- a/python/paddle/fluid/tests/unittests/test_channel_shuffle.py +++ b/python/paddle/fluid/tests/unittests/test_channel_shuffle.py @@ -42,7 +42,6 @@ def channel_shuffle_np(x, groups, data_format="NCHW"): class TestChannelShuffleOp(OpTest): - def setUp(self): self.op_type = "channel_shuffle" self.init_data_format() @@ -73,13 +72,11 @@ class TestChannelShuffleOp(OpTest): class TestChannelLast(TestChannelShuffleOp): - def init_data_format(self): self.format = "NHWC" class TestChannelShuffleAPI(unittest.TestCase): - def setUp(self): self.x_1_np = np.random.random([2, 9, 4, 4]).astype("float64") self.x_2_np = np.random.random([2, 4, 4, 9]).astype("float64") @@ -87,47 +84,53 @@ class TestChannelShuffleAPI(unittest.TestCase): self.out_2_np = channel_shuffle_np(self.x_2_np, 3, "NHWC") def test_static_graph_functional(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 9, 4, 4], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 4, 4, 9], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64" + ) out_1 = F.channel_shuffle(x_1, 3) out_2 = F.channel_shuffle(x_2, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, self.out_1_np) assert np.allclose(res_2, self.out_2_np) # same test between layer and functional in this op. def test_static_graph_layer(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 9, 4, 4], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 4, 4, 9], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64" + ) # init instance ps_1 = paddle.nn.ChannelShuffle(3) ps_2 = paddle.nn.ChannelShuffle(3, "NHWC") @@ -137,15 +140,19 @@ class TestChannelShuffleAPI(unittest.TestCase): out_2_np = channel_shuffle_np(self.x_2_np, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, out_1_np) assert np.allclose(res_2, out_2_np) @@ -163,23 +170,26 @@ class TestChannelShuffleAPI(unittest.TestCase): npresult = channel_shuffle_np(x, groups, data_format) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) - channel_shuffle = paddle.nn.ChannelShuffle(groups, - data_format=data_format) + channel_shuffle = paddle.nn.ChannelShuffle( + groups, data_format=data_format + ) result = channel_shuffle(paddle.to_tensor(x)) np.testing.assert_allclose(result.numpy(), npresult, rtol=1e-05) - result_functional = F.channel_shuffle(paddle.to_tensor(x), 3, - data_format) - np.testing.assert_allclose(result_functional.numpy(), - npresult, - rtol=1e-05) + result_functional = F.channel_shuffle( + paddle.to_tensor(x), 3, data_format + ) + np.testing.assert_allclose( + result_functional.numpy(), npresult, rtol=1e-05 + ) channel_shuffle_str = 'groups={}'.format(groups) if data_format != 'NCHW': @@ -194,9 +204,7 @@ class TestChannelShuffleAPI(unittest.TestCase): class TestChannelShuffleError(unittest.TestCase): - def test_error_functional(self): - def error_input(): with paddle.fluid.dygraph.guard(): x = np.random.random([9, 4, 4]).astype("float64") @@ -221,13 +229,13 @@ class TestChannelShuffleError(unittest.TestCase): def error_data_format(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 9, 4, 4]).astype("float64") - channel_shuffle = F.channel_shuffle(paddle.to_tensor(x), 3, - "WOW") + channel_shuffle = F.channel_shuffle( + paddle.to_tensor(x), 3, "WOW" + ) self.assertRaises(ValueError, error_data_format) def test_error_layer(self): - def error_input_layer(): with paddle.fluid.dygraph.guard(): x = np.random.random([9, 4, 4]).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/test_check_import_scipy.py b/python/paddle/fluid/tests/unittests/test_check_import_scipy.py index 1707ff232a1..f06b5c97e3f 100644 --- a/python/paddle/fluid/tests/unittests/test_check_import_scipy.py +++ b/python/paddle/fluid/tests/unittests/test_check_import_scipy.py @@ -21,7 +21,6 @@ def my_import(name, globals=None, locals=None, fromlist=(), level=0): class importTest(unittest.TestCase): - def test_import(self): testOsName = 'nt' old_import = builtins.__import__ diff --git a/python/paddle/fluid/tests/unittests/test_checkpoint_saver.py b/python/paddle/fluid/tests/unittests/test_checkpoint_saver.py index b9851a283fe..f3b539d8057 100644 --- a/python/paddle/fluid/tests/unittests/test_checkpoint_saver.py +++ b/python/paddle/fluid/tests/unittests/test_checkpoint_saver.py @@ -20,7 +20,6 @@ from paddle.fluid.incubate.checkpoint.checkpoint_saver import CheckpointSaver class CheckpointerSaverTest(unittest.TestCase): - def test(self): fs = HDFSClient("/usr/local/hadoop-2.7.7", None) dir_path = "./checkpointsaver_test" diff --git a/python/paddle/fluid/tests/unittests/test_cholesky_op.py b/python/paddle/fluid/tests/unittests/test_cholesky_op.py index b903cd9159d..1f5d6084313 100644 --- a/python/paddle/fluid/tests/unittests/test_cholesky_op.py +++ b/python/paddle/fluid/tests/unittests/test_cholesky_op.py @@ -24,16 +24,15 @@ from decorator_helper import prog_scope @skip_check_grad_ci( - reason= - "The input of cholesky_op should always be symmetric positive-definite. " + reason="The input of cholesky_op should always be symmetric positive-definite. " "However, OpTest calculates the numeric gradient of each element in input " "via small finite difference, which makes the input no longer symmetric " "positive-definite thus can not compute the Cholesky decomposition. " "While we can use the gradient_checker.grad_check to perform gradient " "check of cholesky_op, since it supports check gradient with a program " - "and we can construct symmetric positive-definite matrices in the program") + "and we can construct symmetric positive-definite matrices in the program" +) class TestCholeskyOp(OpTest): - def setUp(self): self.op_type = "cholesky" self._input_shape = (2, 32, 32) @@ -41,12 +40,14 @@ class TestCholeskyOp(OpTest): self.init_config() self.trans_dims = list(range(len(self._input_shape) - 2)) + [ len(self._input_shape) - 1, - len(self._input_shape) - 2 + len(self._input_shape) - 2, ] self.root_data = np.random.random(self._input_shape).astype("float64") # construct symmetric positive-definite matrice - input_data = np.matmul( - self.root_data, self.root_data.transpose(self.trans_dims)) + 1e-05 + input_data = ( + np.matmul(self.root_data, self.root_data.transpose(self.trans_dims)) + + 1e-05 + ) output_data = np.linalg.cholesky(input_data).astype("float64") if self._upper: output_data = output_data.transpose(self.trans_dims) @@ -70,8 +71,9 @@ class TestCholeskyOp(OpTest): root_data = self.root_data[..., :3, :3] prog = fluid.Program() with fluid.program_guard(prog): - root = layers.create_parameter(dtype=root_data.dtype, - shape=root_data.shape) + root = layers.create_parameter( + dtype=root_data.dtype, shape=root_data.shape + ) root_t = layers.transpose(root, self.trans_dims) x = layers.matmul(x=root, y=root_t) + 1e-05 out = paddle.cholesky(x, upper=self.attrs["upper"]) @@ -82,19 +84,16 @@ class TestCholeskyOp(OpTest): class TestCholeskyOpLower(TestCholeskyOp): - def init_config(self): self._upper = False class TestCholeskyOp2D(TestCholeskyOp): - def init_config(self): self._input_shape = (64, 64) class TestDygraph(unittest.TestCase): - def test_dygraph(self): if core.is_compiled_with_rocm(): paddle.disable_static(place=fluid.CPUPlace()) @@ -108,7 +107,6 @@ class TestDygraph(unittest.TestCase): class TestCholeskySingularAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and (not core.is_compiled_with_rocm()): @@ -123,9 +121,11 @@ class TestCholeskySingularAPI(unittest.TestCase): exe = fluid.Executor(place) try: - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) except RuntimeError as ex: print("The mat is singular") except ValueError as ex: @@ -138,9 +138,12 @@ class TestCholeskySingularAPI(unittest.TestCase): def test_dygraph(self): for place in self.places: with fluid.dygraph.guard(place): - input_np = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], - [16, 17, 18]]]).astype("float64") + input_np = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + ] + ).astype("float64") input = fluid.dygraph.to_variable(input_np) try: result = paddle.cholesky(input) diff --git a/python/paddle/fluid/tests/unittests/test_cholesky_solve_op.py b/python/paddle/fluid/tests/unittests/test_cholesky_solve_op.py index fda558f13bc..5ed62fe2e00 100644 --- a/python/paddle/fluid/tests/unittests/test_cholesky_solve_op.py +++ b/python/paddle/fluid/tests/unittests/test_cholesky_solve_op.py @@ -28,7 +28,7 @@ from paddle.fluid import Program, program_guard, core paddle.enable_static() -#cholesky_solve implement 1 +# cholesky_solve implement 1 def cholesky_solution(X, B, upper=True): if upper: A = np.triu(X) @@ -39,10 +39,11 @@ def cholesky_solution(X, B, upper=True): L = A U = A.T return scipy.linalg.solve_triangular( - U, scipy.linalg.solve_triangular(L, B, lower=True)) + U, scipy.linalg.solve_triangular(L, B, lower=True) + ) -#cholesky_solve implement 2 +# cholesky_solve implement 2 def scipy_cholesky_solution(X, B, upper=True): if upper: umat = np.triu(X) @@ -54,7 +55,7 @@ def scipy_cholesky_solution(X, B, upper=True): return scipy.linalg.cho_solve(K, B) -#broadcast function used by cholesky_solve +# broadcast function used by cholesky_solve def broadcast_shape(matA, matB): shapeA = matA.shape shapeB = matB.shape @@ -67,14 +68,16 @@ def broadcast_shape(matA, matB): Broadshape.append(max(shapeA[idx], shapeB[idx])) else: raise Exception( - 'shapeA and shapeB should be broadcasted, but got {} and {}'. - format(shapeA, shapeB)) + 'shapeA and shapeB should be broadcasted, but got {} and {}'.format( + shapeA, shapeB + ) + ) bsA = Broadshape + list(shapeA[-2:]) bsB = Broadshape + list(shapeB[-2:]) return np.broadcast_to(matA, bsA), np.broadcast_to(matB, bsB) -#cholesky_solve implement in batch +# cholesky_solve implement in batch def scipy_cholesky_solution_batch(bumat, bB, upper=True): bumat, bB = broadcast_shape(bumat, bB) ushape = bumat.shape @@ -99,19 +102,21 @@ class TestCholeskySolveOp(OpTest): case 1 """ - #test condition set + # test condition set def config(self): self.y_shape = [15, 15] self.x_shape = [15, 5] self.upper = False - self.dtype = np.float64 #Here cholesky_solve Op only supports float64/float32 type, please check others if Op supports more types. + self.dtype = ( + np.float64 + ) # Here cholesky_solve Op only supports float64/float32 type, please check others if Op supports more types. - #get scipy result + # get scipy result def set_output(self): umat = self.inputs['Y'] - self.output = scipy_cholesky_solution_batch(umat, - self.inputs['X'], - upper=self.upper) + self.output = scipy_cholesky_solution_batch( + umat, self.inputs['X'], upper=self.upper + ) def setUp(self): self.op_type = "cholesky_solve" @@ -124,17 +129,17 @@ class TestCholeskySolveOp(OpTest): self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': umat + 'Y': umat, } self.attrs = {'upper': self.upper} self.set_output() self.outputs = {'Out': self.output} - #check Op forward result + # check Op forward result def test_check_output(self): self.check_output() - #check Op grad + # check Op grad def test_check_grad_normal(self): self.check_grad(['Y'], 'Out', max_relative_error=0.01) @@ -152,9 +157,8 @@ class TestCholeskySolveOp3(TestCholeskySolveOp): self.dtype = np.float64 -#API function test +# API function test class TestCholeskySolveAPI(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -180,22 +184,20 @@ class TestCholeskySolveAPI(unittest.TestCase): z2_np = scipy_cholesky_solution(umat, x_np, upper=self.upper) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": umat - }, - fetch_list=[z]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": umat}, + fetch_list=[z], + ) np.testing.assert_allclose(fetches[0], z_np, rtol=1e-05) - #test in static mode + # test in static mode def test_static(self): for place in self.place: self.check_static_result(place=place) - #test in dynamic mode + # test in dynamic mode def test_dygraph(self): - def run(place): paddle.disable_static(place) x_np = np.random.random([20, 2]).astype(self.dtype) @@ -213,9 +215,8 @@ class TestCholeskySolveAPI(unittest.TestCase): for idx, place in enumerate(self.place): run(place) - #test input with broadcast + # test input with broadcast def test_broadcast(self): - def run(place): paddle.disable_static() x_np = np.random.random([1, 30, 2]).astype(self.dtype) @@ -234,17 +235,18 @@ class TestCholeskySolveAPI(unittest.TestCase): run(place) -#test condition out of bounds +# test condition out of bounds class TestCholeskySolveOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input type of solve_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.linalg.cholesky_solve, x1, y1) # The data type of input must be float32 or float64. diff --git a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py index b6110a296f0..b82404e01a4 100644 --- a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py +++ b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py @@ -21,15 +21,17 @@ from paddle import fluid class Segment(object): - def __init__(self, chunk_type, start_idx, end_idx): self.chunk_type = chunk_type self.start_idx = start_idx self.end_idx = end_idx def __str__(self): - return '(Segment: %s, %s, %s)' % (self.chunk_type, self.start_idx, - self.end_idx) + return '(Segment: %s, %s, %s)' % ( + self.chunk_type, + self.start_idx, + self.end_idx, + ) __repr__ = __str__ @@ -48,17 +50,24 @@ class TestChunkEvalOp(OpTest): for chunk in chunks: if self.scheme == 'IOB': data[chunk.start_idx] = chunk.chunk_type * self.num_tag_types - data[chunk.start_idx + 1:chunk. - end_idx] = chunk.chunk_type * self.num_tag_types + ( - self.num_tag_types - 1) - data[chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( + data[ + chunk.start_idx + 1 : chunk.end_idx + ] = chunk.chunk_type * self.num_tag_types + ( self.num_tag_types - 1 - ) if chunk.start_idx < chunk.end_idx else data[chunk.start_idx] + ) + data[chunk.end_idx] = ( + chunk.chunk_type * self.num_tag_types + + (self.num_tag_types - 1) + if chunk.start_idx < chunk.end_idx + else data[chunk.start_idx] + ) elif self.scheme == 'IOE': - data[chunk.start_idx:chunk. - end_idx] = chunk.chunk_type * self.num_tag_types + data[chunk.start_idx : chunk.end_idx] = ( + chunk.chunk_type * self.num_tag_types + ) data[chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( - self.num_tag_types - 1) + self.num_tag_types - 1 + ) def rand_chunks(self, starts, num_chunks): if num_chunks < 0: @@ -66,15 +75,17 @@ class TestChunkEvalOp(OpTest): chunks = [] # generate chunk beginnings chunk_begins = sorted( - np.random.choice(list(range(starts[-1])), num_chunks, - replace=False)) + np.random.choice(list(range(starts[-1])), num_chunks, replace=False) + ) seq_chunk_begins = [] begin_idx = 0 # divide chunks into sequences for i in range(len(starts) - 1): tmp_chunk_begins = [] - while begin_idx < len( - chunk_begins) and chunk_begins[begin_idx] < starts[i + 1]: + while ( + begin_idx < len(chunk_begins) + and chunk_begins[begin_idx] < starts[i + 1] + ): tmp_chunk_begins.append(chunk_begins[begin_idx]) begin_idx += 1 seq_chunk_begins.append(tmp_chunk_begins) @@ -83,9 +94,11 @@ class TestChunkEvalOp(OpTest): for i in range(len(seq_chunk_begins)): for j in range(len(seq_chunk_begins[i])): low = seq_chunk_begins[i][j] - high = seq_chunk_begins[i][ - j + 1] if j < len(seq_chunk_begins[i]) - 1 else starts[i + - 1] + high = ( + seq_chunk_begins[i][j + 1] + if j < len(seq_chunk_begins[i]) - 1 + else starts[i + 1] + ) chunk_ends.append(np.random.randint(low, high)) # generate chunks for chunk_pos in zip(chunk_begins, chunk_ends): @@ -95,20 +108,25 @@ class TestChunkEvalOp(OpTest): def gen_chunks(self, infer, label, starts): chunks = self.rand_chunks( - starts, self.num_infer_chunks + self.num_label_chunks - - self.num_correct_chunks) - correct_chunks = np.random.choice(list(range(len(chunks))), - self.num_correct_chunks, - replace=False) + starts, + self.num_infer_chunks + + self.num_label_chunks + - self.num_correct_chunks, + ) + correct_chunks = np.random.choice( + list(range(len(chunks))), self.num_correct_chunks, replace=False + ) infer_chunks = np.random.choice( [x for x in range(len(chunks)) if x not in correct_chunks], self.num_infer_chunks - self.num_correct_chunks, - replace=False) + replace=False, + ) infer_chunks = sorted(correct_chunks.tolist() + infer_chunks.tolist()) label_chunks = np.random.choice( [x for x in range(len(chunks)) if x not in infer_chunks], self.num_label_chunks - self.num_correct_chunks, - replace=False) + replace=False, + ) label_chunks = sorted(correct_chunks.tolist() + label_chunks.tolist()) self.fill_with_chunks(infer, [chunks[idx] for idx in infer_chunks]) self.fill_with_chunks(label, [chunks[idx] for idx in label_chunks]) @@ -123,7 +141,11 @@ class TestChunkEvalOp(OpTest): for idx in label_chunks: if chunks[idx].chunk_type in self.excluded_chunk_types: self.num_label_chunks -= 1 - return self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks + return ( + self.num_correct_chunks, + self.num_infer_chunks, + self.num_label_chunks, + ) def set_confs(self): # Use the IOB scheme and labels with 2 chunk types @@ -134,43 +156,63 @@ class TestChunkEvalOp(OpTest): self.attrs = { 'num_chunk_types': self.num_chunk_types, 'chunk_scheme': self.scheme, - 'excluded_chunk_types': self.excluded_chunk_types + 'excluded_chunk_types': self.excluded_chunk_types, } self.parse_scheme() - self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 4, 5, 9 + ( + self.num_correct_chunks, + self.num_infer_chunks, + self.num_label_chunks, + ) = (4, 5, 9) def set_data(self): - infer = np.zeros((self.batch_size, )).astype('int64') + infer = np.zeros((self.batch_size,)).astype('int64') infer.fill(self.num_chunk_types * self.num_tag_types) label = np.copy(infer) - starts = np.random.choice(list(range(1, self.batch_size)), - self.num_sequences - 1, - replace=False).tolist() + starts = np.random.choice( + list(range(1, self.batch_size)), + self.num_sequences - 1, + replace=False, + ).tolist() starts.extend([0, self.batch_size]) starts = sorted(starts) - self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = self.gen_chunks( - infer, label, starts) + ( + self.num_correct_chunks, + self.num_infer_chunks, + self.num_label_chunks, + ) = self.gen_chunks(infer, label, starts) lod = [] for i in range(len(starts) - 1): lod.append(starts[i + 1] - starts[i]) self.set_input(infer, label, lod) - precision = float( - self.num_correct_chunks - ) / self.num_infer_chunks if self.num_infer_chunks else 0 - recall = float(self.num_correct_chunks - ) / self.num_label_chunks if self.num_label_chunks else 0 - f1 = float(2 * precision * recall) / ( - precision + recall) if self.num_correct_chunks else 0 + precision = ( + float(self.num_correct_chunks) / self.num_infer_chunks + if self.num_infer_chunks + else 0 + ) + recall = ( + float(self.num_correct_chunks) / self.num_label_chunks + if self.num_label_chunks + else 0 + ) + f1 = ( + float(2 * precision * recall) / (precision + recall) + if self.num_correct_chunks + else 0 + ) self.outputs = { 'Precision': np.asarray([precision], dtype='float32'), 'Recall': np.asarray([recall], dtype='float32'), 'F1-Score': np.asarray([f1], dtype='float32'), - 'NumInferChunks': np.asarray([self.num_infer_chunks], - dtype='int64'), - 'NumLabelChunks': np.asarray([self.num_label_chunks], - dtype='int64'), - 'NumCorrectChunks': np.asarray([self.num_correct_chunks], - dtype='int64') + 'NumInferChunks': np.asarray( + [self.num_infer_chunks], dtype='int64' + ), + 'NumLabelChunks': np.asarray( + [self.num_label_chunks], dtype='int64' + ), + 'NumCorrectChunks': np.asarray( + [self.num_correct_chunks], dtype='int64' + ), } def set_input(self, infer, label, lod): @@ -186,7 +228,6 @@ class TestChunkEvalOp(OpTest): class TestChunkEvalOpWithExclude(TestChunkEvalOp): - def set_confs(self): # Use the IOE scheme and labels with 3 chunk types self.scheme = 'IOE' @@ -196,14 +237,17 @@ class TestChunkEvalOpWithExclude(TestChunkEvalOp): self.attrs = { 'num_chunk_types': self.num_chunk_types, 'chunk_scheme': self.scheme, - 'excluded_chunk_types': self.excluded_chunk_types + 'excluded_chunk_types': self.excluded_chunk_types, } self.parse_scheme() - self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 15, 18, 20 + ( + self.num_correct_chunks, + self.num_infer_chunks, + self.num_label_chunks, + ) = (15, 18, 20) class TestChunkEvalOpWithTensorInput(TestChunkEvalOp): - def set_input(self, infer, label, lod): max_len = np.max(lod) pad_infer = [] @@ -212,13 +256,21 @@ class TestChunkEvalOpWithTensorInput(TestChunkEvalOp): for i in range(len(lod)): end = lod[i] + start pad_infer.append( - np.pad(infer[start:end], (0, max_len - lod[i]), - 'constant', - constant_values=(-1, ))) + np.pad( + infer[start:end], + (0, max_len - lod[i]), + 'constant', + constant_values=(-1,), + ) + ) pad_label.append( - np.pad(label[start:end], (0, max_len - lod[i]), - 'constant', - constant_values=(-1, ))) + np.pad( + label[start:end], + (0, max_len - lod[i]), + 'constant', + constant_values=(-1,), + ) + ) start = end pad_infer = np.expand_dims(np.array(pad_infer, dtype='int64'), 2) @@ -227,46 +279,51 @@ class TestChunkEvalOpWithTensorInput(TestChunkEvalOp): self.inputs = { 'Inference': pad_infer, 'Label': pad_label, - 'SeqLength': lod + 'SeqLength': lod, } class TestChunkEvalOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_input(): input_data = np.random.random(1, 1).astype("int64") label_data = np.random.random(1).astype("int64") - fluid.layers.chunk_eval(input=input_data, - label=label_data, - chunk_scheme="IOB", - num_chunk_types=3) + fluid.layers.chunk_eval( + input=input_data, + label=label_data, + chunk_scheme="IOB", + num_chunk_types=3, + ) self.assertRaises(TypeError, test_input) def test_label(): - input_ = fluid.data(name="input", - shape=[None, 1], - dtype="int64") + input_ = fluid.data( + name="input", shape=[None, 1], dtype="int64" + ) label_data = np.random.random(1).astype("int64") - fluid.layers.chunk_eval(input=input_, - label=label_data, - chunk_scheme="IOB", - num_chunk_types=3) + fluid.layers.chunk_eval( + input=input_, + label=label_data, + chunk_scheme="IOB", + num_chunk_types=3, + ) self.assertRaises(TypeError, test_label) def test_type(): - in_data = fluid.data(name="input_", - shape=[None, 1], - dtype="int32") + in_data = fluid.data( + name="input_", shape=[None, 1], dtype="int32" + ) label = fluid.data(name="label_", shape=[1], dtype="int64") - fluid.layers.chunk_eval(input=in_data, - label=label, - chunk_scheme="IOB", - num_chunk_types=3) + fluid.layers.chunk_eval( + input=in_data, + label=label, + chunk_scheme="IOB", + num_chunk_types=3, + ) self.assertRaises(TypeError, test_type) diff --git a/python/paddle/fluid/tests/unittests/test_chunk_op.py b/python/paddle/fluid/tests/unittests/test_chunk_op.py index 896da7d4c1d..f29d90ceb73 100644 --- a/python/paddle/fluid/tests/unittests/test_chunk_op.py +++ b/python/paddle/fluid/tests/unittests/test_chunk_op.py @@ -21,7 +21,6 @@ import paddle class TestChunkOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The type of axis in chunk_op should be int or Variable. @@ -54,7 +53,6 @@ class TestChunkOpError(unittest.TestCase): class API_TestChunk(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = paddle.fluid.data('data1', shape=[4, 6, 6], dtype='float64') @@ -64,11 +62,9 @@ class API_TestChunk(unittest.TestCase): exe = paddle.static.Executor(place) input1 = np.random.random([4, 6, 6]).astype('float64') input2 = np.array([2]).astype('int32') - r0, r1, r2, = exe.run(feed={ - "data1": input1, - "data2": input2 - }, - fetch_list=[x0, x1, x2]) + r0, r1, r2, = exe.run( + feed={"data1": input1, "data2": input2}, fetch_list=[x0, x1, x2] + ) ex_x0, ex_x1, ex_x2 = np.array_split(input1, 3, axis=2) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) @@ -76,7 +72,6 @@ class API_TestChunk(unittest.TestCase): class API_TestChunk1(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = paddle.fluid.data('data1', shape=[4, 6, 6], dtype='float64') @@ -84,8 +79,11 @@ class API_TestChunk1(unittest.TestCase): place = paddle.CPUPlace() exe = paddle.static.Executor(place) input1 = np.random.random([4, 6, 6]).astype('float64') - r0, r1, r2, = exe.run(feed={"data1": input1}, - fetch_list=[x0, x1, x2]) + ( + r0, + r1, + r2, + ) = exe.run(feed={"data1": input1}, fetch_list=[x0, x1, x2]) ex_x0, ex_x1, ex_x2 = np.array_split(input1, 3, axis=2) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) @@ -93,7 +91,6 @@ class API_TestChunk1(unittest.TestCase): class API_TestDygraphChunk(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(): input_1 = np.random.random([4, 6, 6]).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/test_class_center_sample_op.py b/python/paddle/fluid/tests/unittests/test_class_center_sample_op.py index 28b9ffd88b4..53b9cfad345 100644 --- a/python/paddle/fluid/tests/unittests/test_class_center_sample_op.py +++ b/python/paddle/fluid/tests/unittests/test_class_center_sample_op.py @@ -28,10 +28,13 @@ def class_center_sample_numpy(label, classes_list, num_samples): unique_label_per_device = [] for i in range(nranks): - index = np.logical_and(unique_label >= class_interval[i], - unique_label < class_interval[i + 1]) - pos_class_center_per_device.append(unique_label[index] - - class_interval[i]) + index = np.logical_and( + unique_label >= class_interval[i], + unique_label < class_interval[i + 1], + ) + pos_class_center_per_device.append( + unique_label[index] - class_interval[i] + ) unique_label_per_device.append(unique_label[index]) num_samples_per_device = [] @@ -41,8 +44,9 @@ def class_center_sample_numpy(label, classes_list, num_samples): remapped_dict = {} for i in range(nranks): - for idx, v in enumerate(unique_label_per_device[i], - sampled_class_interval[i]): + for idx, v in enumerate( + unique_label_per_device[i], sampled_class_interval[i] + ): remapped_dict[v] = idx remapped_label = [] @@ -62,14 +66,12 @@ def python_api( fix_seed=False, seed=0, ): - return paddle.nn.functional.class_center_sample(label, - num_classes=num_classes, - num_samples=num_samples, - group=None) + return paddle.nn.functional.class_center_sample( + label, num_classes=num_classes, num_samples=num_samples, group=None + ) class TestClassCenterSampleOp(OpTest): - def initParams(self): self.op_type = "class_center_sample" self.python_api = python_api @@ -88,17 +90,18 @@ class TestClassCenterSampleOp(OpTest): self.initParams() self.init_dtype() self.init_fix_seed() - label = np.random.randint(0, - self.num_classes, (self.batch_size, ), - dtype=self.dtype) + label = np.random.randint( + 0, self.num_classes, (self.batch_size,), dtype=self.dtype + ) remapped_label, sampled_class_center = class_center_sample_numpy( - label, [self.num_classes], self.num_samples) + label, [self.num_classes], self.num_samples + ) self.inputs = {'Label': label} self.outputs = { 'RemappedLabel': remapped_label.astype(self.dtype), - 'SampledLocalClassCenter': sampled_class_center.astype(self.dtype) + 'SampledLocalClassCenter': sampled_class_center.astype(self.dtype), } self.attrs = { @@ -109,24 +112,22 @@ class TestClassCenterSampleOp(OpTest): } def test_check_output(self): - self.check_output(no_check_set=['SampledLocalClassCenter'], - check_eager=True) + self.check_output( + no_check_set=['SampledLocalClassCenter'], check_eager=True + ) class TestClassCenterSampleOpINT32(TestClassCenterSampleOp): - def init_dtype(self): self.dtype = np.int32 class TestClassCenterSampleOpFixSeed(TestClassCenterSampleOp): - def init_fix_seed(self): self.fix_seed = True class TestClassCenterSampleV2(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -151,27 +152,37 @@ class TestClassCenterSampleV2(unittest.TestCase): def check_static_result(self, place): with program_guard(Program(), Program()): - label_np = np.random.randint(0, - self.num_classes, (self.batch_size, ), - dtype=self.dtype) - - label = paddle.static.data(name='label', - shape=[self.batch_size], - dtype=self.dtype) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples) - - remapped_label_np, sampled_class_center_np = class_center_sample_numpy( - label_np, [self.num_classes], self.num_samples) + label_np = np.random.randint( + 0, self.num_classes, (self.batch_size,), dtype=self.dtype + ) + + label = paddle.static.data( + name='label', shape=[self.batch_size], dtype=self.dtype + ) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples + ) + + ( + remapped_label_np, + sampled_class_center_np, + ) = class_center_sample_numpy( + label_np, [self.num_classes], self.num_samples + ) exe = paddle.fluid.Executor(place) - [remapped_label_res, sampled_class_index_res - ] = exe.run(paddle.fluid.default_main_program(), - feed={'label': label_np}, - fetch_list=[remapped_label, sampled_class_index]) + [remapped_label_res, sampled_class_index_res] = exe.run( + paddle.fluid.default_main_program(), + feed={'label': label_np}, + fetch_list=[remapped_label, sampled_class_index], + ) np.testing.assert_allclose(remapped_label_res, remapped_label_np) np.testing.assert_allclose( - sampled_class_index_res[:len(sampled_class_center_np[0])], - sampled_class_center_np[0]) + sampled_class_index_res[: len(sampled_class_center_np[0])], + sampled_class_center_np[0], + ) def test_dynamic(self): for place in self.places: @@ -179,33 +190,40 @@ class TestClassCenterSampleV2(unittest.TestCase): def check_dynamic_result(self, place): with paddle.fluid.dygraph.guard(place): - label_np = np.random.randint(0, - self.num_classes, (self.batch_size, ), - dtype=self.dtype) + label_np = np.random.randint( + 0, self.num_classes, (self.batch_size,), dtype=self.dtype + ) label = paddle.to_tensor(label_np, dtype=self.dtype) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples + ) - remapped_label_np, sampled_class_center_np = class_center_sample_numpy( - label_np, [self.num_classes], self.num_samples) + ( + remapped_label_np, + sampled_class_center_np, + ) = class_center_sample_numpy( + label_np, [self.num_classes], self.num_samples + ) remapped_label_res = remapped_label.numpy() sampled_class_index_res = sampled_class_index.numpy() np.testing.assert_allclose(remapped_label_res, remapped_label_np) np.testing.assert_allclose( - sampled_class_index_res[:len(sampled_class_center_np[0])], - sampled_class_center_np[0]) + sampled_class_index_res[: len(sampled_class_center_np[0])], + sampled_class_center_np[0], + ) class TestClassCenterSampleV2INT32(TestClassCenterSampleV2): - def init_dtype(self): self.dtype = np.int32 class TestClassCenterSampleAPIError(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -224,24 +242,28 @@ class TestClassCenterSampleAPIError(unittest.TestCase): self.dtype = np.int64 def test_dynamic_errors(self): - def test_num_samples(): for place in self.places: with paddle.fluid.dygraph.guard(place): - label_np = np.random.randint(0, - self.num_classes, - (self.batch_size, ), - dtype=self.dtype) + label_np = np.random.randint( + 0, + self.num_classes, + (self.batch_size,), + dtype=self.dtype, + ) label = paddle.to_tensor(label_np) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples + ) self.assertRaises(ValueError, test_num_samples) class TestClassCenterSampleAPIError1(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -260,26 +282,35 @@ class TestClassCenterSampleAPIError1(unittest.TestCase): self.dtype = np.int64 def test_dynamic_errors(self): - def test_empty_label(): for place in self.places: with paddle.fluid.dygraph.guard(place): label = paddle.to_tensor(np.array([], dtype=self.dtype)) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples + ) def test_group_value(): for place in self.places: with paddle.fluid.dygraph.guard(place): - label_np = np.random.randint(0, - self.num_classes, - (self.batch_size, ), - dtype=self.dtype) + label_np = np.random.randint( + 0, + self.num_classes, + (self.batch_size,), + dtype=self.dtype, + ) label = paddle.to_tensor(label_np) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples, group=True) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples, group=True + ) self.assertRaises(ValueError, test_empty_label) self.assertRaises(ValueError, test_group_value) diff --git a/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py b/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py index ec95442a37c..d7b9c6cd528 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py @@ -22,7 +22,6 @@ import paddle.fluid.core as core class TestClipByNormOp(OpTest): - def setUp(self): self.max_relative_error = 0.006 self.python_api = fluid.layers.clip_by_norm @@ -47,7 +46,7 @@ class TestClipByNormOp(OpTest): self.check_output(check_eager=True) def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1.0 def init_dtype(self): @@ -55,28 +54,24 @@ class TestClipByNormOp(OpTest): class TestCase1(TestClipByNormOp): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestCase2(TestClipByNormOp): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestCase3(TestClipByNormOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 class TestClipByNormOpFp16(TestClipByNormOp): - def init_dtype(self): self.dtype = np.float16 @@ -84,34 +79,30 @@ class TestClipByNormOpFp16(TestClipByNormOp): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_output_with_place(place, - atol=0.001, - check_eager=True) + self.check_output_with_place( + place, atol=0.001, check_eager=True + ) class TestClipByNormOpFp16Case1(TestClipByNormOpFp16): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestClipByNormOpFp16Case2(TestClipByNormOpFp16): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestClipByNormOpFp16Case3(TestClipByNormOpFp16): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 class TestClipByNormOpWithSelectedRows(unittest.TestCase): - def check_with_place(self, place): self.config_test_case() scope = core.Scope() @@ -128,10 +119,9 @@ class TestClipByNormOpWithSelectedRows(unittest.TestCase): out_selected_rows = scope.var('Out').get_selected_rows() # run clip_by_norm_op - clip_by_norm_op = fluid.op.Operator("clip_by_norm", - max_norm=self.max_norm, - X='X', - Out='Out') + clip_by_norm_op = fluid.op.Operator( + "clip_by_norm", max_norm=self.max_norm, X='X', Out='Out' + ) clip_by_norm_op.run(scope, place) # check output @@ -146,11 +136,13 @@ class TestClipByNormOpWithSelectedRows(unittest.TestCase): output = self.max_norm * y_np / norm else: output = y_np - np.testing.assert_allclose(np.array(out_tensor), - output, - rtol=1e-05, - atol=1e-05, - equal_nan=False) + np.testing.assert_allclose( + np.array(out_tensor), + output, + rtol=1e-05, + atol=1e-05, + equal_nan=False, + ) def test_clip_by_norm_with_selected_ros(self): places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_clip_op.py b/python/paddle/fluid/tests/unittests/test_clip_op.py index a8e3b90af90..e955bd71a64 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_op.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class TestClipOp(OpTest): - def setUp(self): self.max_relative_error = 0.006 self.python_api = paddle.clip @@ -70,7 +69,6 @@ class TestClipOp(OpTest): class TestCase1(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (8, 16, 8) @@ -79,7 +77,6 @@ class TestCase1(TestClipOp): class TestCase2(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (8, 16) @@ -88,7 +85,6 @@ class TestCase2(TestClipOp): class TestCase3(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 16) @@ -97,7 +93,6 @@ class TestCase3(TestClipOp): class TestCase4(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 8) @@ -108,7 +103,6 @@ class TestCase4(TestClipOp): class TestCase5(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 16) @@ -117,7 +111,6 @@ class TestCase5(TestClipOp): class TestCase6(TestClipOp): - def initTestCase(self): self.dtype == np.float16 self.shape = (4, 8, 8) @@ -128,7 +121,6 @@ class TestCase6(TestClipOp): class TestClipOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -148,7 +140,6 @@ class TestClipOpError(unittest.TestCase): class TestClipAPI(unittest.TestCase): - def _executed_api(self, x, min=None, max=None): return paddle.clip(x, min, max) @@ -160,8 +151,11 @@ class TestClipAPI(unittest.TestCase): min = fluid.data(name='min', shape=[1], dtype='float32') max = fluid.data(name='max', shape=[1], dtype='float32') - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) out_1 = self._executed_api(images, min=min, max=max) @@ -170,29 +164,51 @@ class TestClipAPI(unittest.TestCase): out_4 = self._executed_api(images, max=0.7) out_5 = self._executed_api(images, min=min) out_6 = self._executed_api(images, max=max) - out_7 = self._executed_api(images, max=-1.) + out_7 = self._executed_api(images, max=-1.0) out_8 = self._executed_api(images) - out_9 = self._executed_api(paddle.cast(images, 'float64'), - min=0.2, - max=0.9) - out_10 = self._executed_api(paddle.cast(images * 10, 'int32'), - min=2, - max=8) - out_11 = self._executed_api(paddle.cast(images * 10, 'int64'), - min=2, - max=8) - - res1, res2, res3, res4, res5, res6, res7, res8, res9, res10, res11 = exe.run( + out_9 = self._executed_api( + paddle.cast(images, 'float64'), min=0.2, max=0.9 + ) + out_10 = self._executed_api( + paddle.cast(images * 10, 'int32'), min=2, max=8 + ) + out_11 = self._executed_api( + paddle.cast(images * 10, 'int64'), min=2, max=8 + ) + + ( + res1, + res2, + res3, + res4, + res5, + res6, + res7, + res8, + res9, + res10, + res11, + ) = exe.run( fluid.default_main_program(), feed={ "image": data, "min": np.array([0.2]).astype('float32'), - "max": np.array([0.8]).astype('float32') + "max": np.array([0.8]).astype('float32'), }, fetch_list=[ - out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8, out_9, - out_10, out_11 - ]) + out_1, + out_2, + out_3, + out_4, + out_5, + out_6, + out_7, + out_8, + out_9, + out_10, + out_11, + ], + ) np.testing.assert_allclose(res1, data.clip(0.2, 0.8), rtol=1e-05) np.testing.assert_allclose(res2, data.clip(0.2, 0.9), rtol=1e-05) @@ -202,21 +218,24 @@ class TestClipAPI(unittest.TestCase): np.testing.assert_allclose(res6, data.clip(max=0.8), rtol=1e-05) np.testing.assert_allclose(res7, data.clip(max=-1), rtol=1e-05) np.testing.assert_allclose(res8, data, rtol=1e-05) - np.testing.assert_allclose(res9, - data.astype(np.float64).clip(0.2, 0.9), - rtol=1e-05) - np.testing.assert_allclose(res10, - (data * 10).astype(np.int32).clip(2, 8), - rtol=1e-05) - np.testing.assert_allclose(res11, - (data * 10).astype(np.int64).clip(2, 8), - rtol=1e-05) + np.testing.assert_allclose( + res9, data.astype(np.float64).clip(0.2, 0.9), rtol=1e-05 + ) + np.testing.assert_allclose( + res10, (data * 10).astype(np.int32).clip(2, 8), rtol=1e-05 + ) + np.testing.assert_allclose( + res11, (data * 10).astype(np.int64).clip(2, 8), rtol=1e-05 + ) paddle.disable_static() def func_clip_dygraph(self): paddle.disable_static() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) data_shape = [1, 9, 9, 4] data = np.random.random(data_shape).astype('float32') @@ -230,33 +249,33 @@ class TestClipAPI(unittest.TestCase): images = paddle.to_tensor(data, dtype='float32') out_3 = self._executed_api(images, min=v_min, max=v_max) - out_4 = self._executed_api(paddle.cast(images * 10, 'int32'), - min=2, - max=8) - out_5 = self._executed_api(paddle.cast(images * 10, 'int64'), - min=2, - max=8) + out_4 = self._executed_api( + paddle.cast(images * 10, 'int32'), min=2, max=8 + ) + out_5 = self._executed_api( + paddle.cast(images * 10, 'int64'), min=2, max=8 + ) # test with numpy.generic out_6 = self._executed_api(images, min=np.abs(0.2), max=np.abs(0.8)) - np.testing.assert_allclose(out_1.numpy(), - data.clip(0.2, 0.8), - rtol=1e-05) - np.testing.assert_allclose(out_2.numpy(), - data.clip(0.2, 0.9), - rtol=1e-05) - np.testing.assert_allclose(out_3.numpy(), - data.clip(0.2, 0.8), - rtol=1e-05) - np.testing.assert_allclose(out_4.numpy(), - (data * 10).astype(np.int32).clip(2, 8), - rtol=1e-05) - np.testing.assert_allclose(out_5.numpy(), - (data * 10).astype(np.int64).clip(2, 8), - rtol=1e-05) - np.testing.assert_allclose(out_6.numpy(), - data.clip(0.2, 0.8), - rtol=1e-05) + np.testing.assert_allclose( + out_1.numpy(), data.clip(0.2, 0.8), rtol=1e-05 + ) + np.testing.assert_allclose( + out_2.numpy(), data.clip(0.2, 0.9), rtol=1e-05 + ) + np.testing.assert_allclose( + out_3.numpy(), data.clip(0.2, 0.8), rtol=1e-05 + ) + np.testing.assert_allclose( + out_4.numpy(), (data * 10).astype(np.int32).clip(2, 8), rtol=1e-05 + ) + np.testing.assert_allclose( + out_5.numpy(), (data * 10).astype(np.int64).clip(2, 8), rtol=1e-05 + ) + np.testing.assert_allclose( + out_6.numpy(), data.clip(0.2, 0.8), rtol=1e-05 + ) def test_clip_dygraph(self): with _test_eager_guard(): @@ -292,7 +311,6 @@ class TestClipAPI(unittest.TestCase): class TestInplaceClipAPI(TestClipAPI): - def _executed_api(self, x, min=None, max=None): return x.clip_(min, max) diff --git a/python/paddle/fluid/tests/unittests/test_coalesce_tensor_op.py b/python/paddle/fluid/tests/unittests/test_coalesce_tensor_op.py index 2bbac7a09e9..bcd9eb412bc 100644 --- a/python/paddle/fluid/tests/unittests/test_coalesce_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_coalesce_tensor_op.py @@ -20,30 +20,40 @@ import paddle.fluid as fluid import paddle -def coalesce_tensor_eager_api(Input, - datatype=core.VarDesc.VarType.FP32, - copy_data=False, - set_constant=False, - persist_output=False, - constant=0.0, - use_align=True, - align_size=-1, - user_defined_size_of_dtype=-1, - concated_shapes=[], - concated_ranks=[]): +def coalesce_tensor_eager_api( + Input, + datatype=core.VarDesc.VarType.FP32, + copy_data=False, + set_constant=False, + persist_output=False, + constant=0.0, + use_align=True, + align_size=-1, + user_defined_size_of_dtype=-1, + concated_shapes=[], + concated_ranks=[], +): if datatype == int(core.VarDesc.VarType.FP32): datatype = core.VarDesc.VarType.FP32 - return paddle._C_ops.coalesce_tensor(Input, datatype, copy_data, - set_constant, persist_output, constant, - use_align, align_size, - user_defined_size_of_dtype, - concated_shapes, concated_ranks) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + return paddle._C_ops.coalesce_tensor( + Input, + datatype, + copy_data, + set_constant, + persist_output, + constant, + use_align, + align_size, + user_defined_size_of_dtype, + concated_shapes, + concated_ranks, + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAllocContinuousSpace(OpTest): - def setUp(self): self.python_api = coalesce_tensor_eager_api self.op_type = "coalesce_tensor" @@ -51,7 +61,8 @@ class TestAllocContinuousSpace(OpTest): self.attrs = self.init_attr() self.Inputs = self.init_input() self.Outputs, self.FusedOutput = self.init_output( - self.Inputs, self.attrs["set_constant"], self.attrs["constant"]) + self.Inputs, self.attrs["set_constant"], self.attrs["constant"] + ) self.inputs = {'Input': self.Inputs} self.outputs = {'Output': self.Outputs, 'FusedOutput': self.FusedOutput} @@ -73,7 +84,7 @@ class TestAllocContinuousSpace(OpTest): "copy_data": True, "set_constant": False, "constant": 0.0, - "dtype": self.fluid_dtype + "dtype": self.fluid_dtype, } def init_output(self, input_list, set_constant, constant): @@ -94,9 +105,10 @@ class TestAllocContinuousSpace(OpTest): coalesce_tensor_var = np.concatenate([input for input in inputs]) if set_constant: coalesce_tensor_var = np.ones((len(coalesce_tensor_var))) * constant - outputs = [(out[0], - np.ones(out[1].shape).astype(self.dtype) * constant) - for out in outputs] + outputs = [ + (out[0], np.ones(out[1].shape).astype(self.dtype) * constant) + for out in outputs + ] return outputs, coalesce_tensor_var def verify_output(self, place): @@ -109,54 +121,65 @@ class TestAllocContinuousSpace(OpTest): tensor_input, datatype=self.attrs["dtype"], copy_data=self.attrs["copy_data"] - if "copy_data" in self.attrs else False, + if "copy_data" in self.attrs + else False, set_constant=self.attrs["set_constant"] - if "set_constant" in self.attrs else False, + if "set_constant" in self.attrs + else False, persist_output=False, constant=self.attrs["constant"] - if "constant" in self.attrs else 0.0, + if "constant" in self.attrs + else 0.0, use_align=True, align_size=-1, - user_defined_size_of_dtype=self. - attrs["user_defined_size_of_dtype"] - if "user_defined_size_of_dtype" in self.attrs else -1, + user_defined_size_of_dtype=self.attrs[ + "user_defined_size_of_dtype" + ] + if "user_defined_size_of_dtype" in self.attrs + else -1, concated_shapes=[], - concated_ranks=[]) + concated_ranks=[], + ) for idx, (expected, eager_output) in enumerate( - zip(self.outputs['Output'], eager_outputs)): - np.testing.assert_allclose(expected[1], - eager_output, - atol=1e-5, - err_msg=f'not equal {idx}') - np.testing.assert_allclose(self.outputs['FusedOutput'], - eager_fused_output, - atol=1e-5, - err_msg='not equal fusedoutput') + zip(self.outputs['Output'], eager_outputs) + ): + np.testing.assert_allclose( + expected[1], + eager_output, + atol=1e-5, + err_msg=f'not equal {idx}', + ) + np.testing.assert_allclose( + self.outputs['FusedOutput'], + eager_fused_output, + atol=1e-5, + err_msg='not equal fusedoutput', + ) def test_check_output(self): - self.check_output_with_place(place=core.CUDAPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=core.CUDAPlace(0), no_check_set=["FusedOutput"], atol=1e-5 + ) self.verify_output(core.CUDAPlace(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAllocContinuousSpace2(TestAllocContinuousSpace): - def init_attr(self): return { "copy_data": False, "set_constant": True, "constant": 0.5, "dtype": self.fluid_dtype, - "user_defined_size_of_dtype": 2 + "user_defined_size_of_dtype": 2, } def test_check_output(self): - self.check_output_with_place(place=core.CUDAPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=core.CUDAPlace(0), no_check_set=["FusedOutput"], atol=1e-5 + ) self.verify_output(core.CUDAPlace(0)) diff --git a/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py b/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py index f06102fb2f2..06011ee62d8 100644 --- a/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py @@ -18,27 +18,29 @@ from op_test import OpTest class TestCollectFPNProposalstOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() - self.scores_input = [('y%d' % i, (self.scores[i].reshape(-1, 1), - self.rois_lod[i])) - for i in range(self.num_level)] + self.scores_input = [ + ('y%d' % i, (self.scores[i].reshape(-1, 1), self.rois_lod[i])) + for i in range(self.num_level) + ] self.rois, self.lod = self.calc_rois_collect() - inputs_x = [('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) - for i in range(self.num_level)] + inputs_x = [ + ('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) + for i in range(self.num_level) + ] self.inputs = { 'MultiLevelRois': inputs_x, "MultiLevelScores": self.scores_input, - 'MultiLevelRoIsNum': [] + 'MultiLevelRoIsNum': [], } self.attrs = { 'post_nms_topN': self.post_nms_top_n, } self.outputs = { 'FpnRois': (self.rois, [self.lod]), - 'RoisNum': np.array(self.lod).astype('int32') + 'RoisNum': np.array(self.lod).astype('int32'), } def init_test_case(self): @@ -61,7 +63,7 @@ class TestCollectFPNProposalstOp(OpTest): def calc_rois_collect(self): roi_inputs = np.concatenate(self.roi_inputs) scores = np.concatenate(self.scores) - inds = np.argsort(-scores)[:self.post_nms_top_n] + inds = np.argsort(-scores)[: self.post_nms_top_n] rois = roi_inputs[inds, :] new_rois, new_lod = self.resort_roi_by_batch_id(rois) return new_rois, new_lod @@ -100,31 +102,34 @@ class TestCollectFPNProposalstOp(OpTest): class TestCollectFPNProposalstOpWithRoisNum(TestCollectFPNProposalstOp): - def set_data(self): self.init_test_case() self.make_rois() - self.scores_input = [('y%d' % i, (self.scores[i].reshape(-1, 1), - self.rois_lod[i])) - for i in range(self.num_level)] + self.scores_input = [ + ('y%d' % i, (self.scores[i].reshape(-1, 1), self.rois_lod[i])) + for i in range(self.num_level) + ] self.rois, self.lod = self.calc_rois_collect() - inputs_x = [('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) - for i in range(self.num_level)] - rois_num_per_level = [('rois%d' % i, - np.array(self.rois_lod[i][0]).astype('int32')) - for i in range(self.num_level)] + inputs_x = [ + ('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) + for i in range(self.num_level) + ] + rois_num_per_level = [ + ('rois%d' % i, np.array(self.rois_lod[i][0]).astype('int32')) + for i in range(self.num_level) + ] self.inputs = { 'MultiLevelRois': inputs_x, "MultiLevelScores": self.scores_input, - 'MultiLevelRoIsNum': rois_num_per_level + 'MultiLevelRoIsNum': rois_num_per_level, } self.attrs = { 'post_nms_topN': self.post_nms_top_n, } self.outputs = { 'FpnRois': (self.rois, [self.lod]), - 'RoisNum': np.array(self.lod).astype('int32') + 'RoisNum': np.array(self.lod).astype('int32'), } diff --git a/python/paddle/fluid/tests/unittests/test_collective_api_base.py b/python/paddle/fluid/tests/unittests/test_collective_api_base.py index 3a3e5d99ca7..4f4be4e2dcf 100644 --- a/python/paddle/fluid/tests/unittests/test_collective_api_base.py +++ b/python/paddle/fluid/tests/unittests/test_collective_api_base.py @@ -82,7 +82,12 @@ def create_test_data(shape=None, dtype=None, seed=None): return create_float_test_data(shape=shape, dtype=bfloat16, seed=seed) elif dtype == "bool": return create_bool_test_data(shape=shape, seed=seed) - elif dtype == "int32" or dtype == "int64" or dtype == "int8" or dtype == "uint8": + elif ( + dtype == "int32" + or dtype == "int64" + or dtype == "int8" + or dtype == "uint8" + ): return create_int_test_data(shape=shape, dtype=dtype, seed=seed) elif dtype == "complex64" or dtype == "complex128": return create_complex_test_data(shape=shape, dtype=dtype, seed=seed) @@ -95,15 +100,12 @@ def create_test_data(shape=None, dtype=None, seed=None): class TestCollectiveAPIRunnerBase(object): - - def get_model(self, - train_prog, - startup_prog, - rank, - indata=None, - dtype=None): + def get_model( + self, train_prog, startup_prog, rank, indata=None, dtype=None + ): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -116,15 +118,16 @@ class TestCollectiveAPIRunnerBase(object): if args['backend'] == 'nccl': device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() elif args['backend'] == 'bkcl': device_id = int(os.getenv("FLAGS_selected_xpus", "0")) place = fluid.XPUPlace(device_id) else: place = fluid.CPUPlace() - indata = create_test_data(shape=(10, 1000), - dtype=args["dtype"], - seed=os.getpid()) + indata = create_test_data( + shape=(10, 1000), dtype=args["dtype"], seed=os.getpid() + ) if args['static_mode']: result = self.get_model(train_prog, startup_prog, rank) exe = fluid.Executor(place) @@ -132,12 +135,12 @@ class TestCollectiveAPIRunnerBase(object): fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=fetch_list) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=fetch_list + ) else: out = self.get_model(train_prog, startup_prog, rank, indata) - #print(out, sys.stderr) + # print(out, sys.stderr) sys.stdout.buffer.write(pickle.dumps(out)) @@ -161,12 +164,13 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self.temp_dir = tempfile.TemporaryDirectory() @@ -176,18 +180,20 @@ class TestDistBase(unittest.TestCase): nccl_version_str = subprocess.check_output( r"ldconfig -v | grep 'libnccl.so' | tail -n1 | sed -r 's/^.*\.so\.//'", stderr=subprocess.DEVNULL, - shell=True).decode('utf-8') - self._nccl_version = int("".join( - nccl_version_str.split("."))) if nccl_version_str else 0 + shell=True, + ).decode('utf-8') + self._nccl_version = ( + int("".join(nccl_version_str.split("."))) if nccl_version_str else 0 + ) def tearDown(self): self.temp_dir.cleanup() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -200,14 +206,14 @@ class TestDistBase(unittest.TestCase): def _run_cluster(self, model_file, envs): worker_endpoints = self._ps_endpoints.split(",") w0_ep, w1_ep = worker_endpoints - #print("w0_ep:",w0_ep," w1_ep:",w1_ep) + # print("w0_ep:",w0_ep," w1_ep:",w1_ep) if core.is_compiled_with_cuda(): env0 = { "FLAGS_selected_gpus": "0", "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -215,7 +221,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } elif core.is_compiled_with_xpu(): env0 = { @@ -223,7 +229,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -231,9 +237,9 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) if os.getenv('WITH_COVERAGE', 'OFF') == 'ON': @@ -242,22 +248,28 @@ class TestDistBase(unittest.TestCase): tr_cmd = "%s %s" tr0_cmd = tr_cmd % (self._python_interp, model_file) tr1_cmd = tr_cmd % (self._python_interp, model_file) - path0 = os.path.join(self.temp_dir.name, - "/tmp/tr0_err_%d.log" % os.getpid()) - path1 = os.path.join(self.temp_dir.name, - "/tmp/tr1_err_%d.log" % os.getpid()) + path0 = os.path.join( + self.temp_dir.name, "/tmp/tr0_err_%d.log" % os.getpid() + ) + path1 = os.path.join( + self.temp_dir.name, "/tmp/tr1_err_%d.log" % os.getpid() + ) tr0_pipe = open(path0, "w") tr1_pipe = open(path1, "w") - #print(tr0_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + # print(tr0_cmd) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -270,19 +282,25 @@ class TestDistBase(unittest.TestCase): sys.stderr.write('trainer 0 stderr file: %s\n' % f.read()) with open(path1, "r") as f: sys.stderr.write('trainer 1 stderr file: %s\n' % f.read()) - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - backend="nccl", - path_id="0", - static_mode="1", - check_error_log=False, - need_envs={}, - eager_mode=True, - dtype=None): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, + model_file, + col_type, + backend="nccl", + path_id="0", + static_mode="1", + check_error_log=False, + need_envs={}, + eager_mode=True, + dtype=None, + ): if backend == "nccl" or backend == "bkcl": with_gloo = '0' else: @@ -296,7 +314,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_DISTRI_BACKEND": backend, "BACKEND": backend, "PATH_ID": path_id, - "DTYPE": dtype + "DTYPE": dtype, } required_envs.update(additional_envs) required_envs.update(need_envs) @@ -307,7 +325,8 @@ class TestDistBase(unittest.TestCase): if os.getenv('NVIDIA_TF32_OVERRIDE', '') is not None: required_envs['NVIDIA_TF32_OVERRIDE'] = os.getenv( - 'NVIDIA_TF32_OVERRIDE', '') + 'NVIDIA_TF32_OVERRIDE', '' + ) if eager_mode: required_envs["FLAGS_enable_eager_mode"] = "%d" % 1 @@ -315,7 +334,8 @@ class TestDistBase(unittest.TestCase): required_envs["FLAGS_enable_eager_mode"] = "%d" % 0 tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) input1 = create_test_data(shape=(10, 1000), dtype=dtype, seed=pid0) input2 = create_test_data(shape=(10, 1000), dtype=dtype, seed=pid1) # cast bfloat16 to float32 for numeric comparison @@ -347,14 +367,14 @@ class TestDistBase(unittest.TestCase): np.testing.assert_allclose(tr0_out[0], need_result, rtol=rtol) elif col_type == "scatter": need_result = input2 - need_result1 = need_result[0:need_result.shape[0] // 2] - need_result2 = need_result[need_result.shape[0] // 2:] + need_result1 = need_result[0 : need_result.shape[0] // 2] + need_result2 = need_result[need_result.shape[0] // 2 :] np.testing.assert_allclose(tr0_out[0], need_result1, rtol=1e-05) np.testing.assert_allclose(tr1_out[0], need_result2, rtol=1e-05) elif col_type == "reduce_scatter": need_result = input1 + input2 - need_result1 = need_result[0:need_result.shape[0] // 2] - need_result2 = need_result[need_result.shape[0] // 2:] + need_result1 = need_result[0 : need_result.shape[0] // 2] + need_result2 = need_result[need_result.shape[0] // 2 :] if dtype == "bfloat16": rtol = 8e-03 else: @@ -369,14 +389,12 @@ class TestDistBase(unittest.TestCase): else: rtol = 1e-05 atol = 1e-05 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=rtol, - atol=atol) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=rtol, - atol=atol) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=rtol, atol=atol + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=rtol, atol=atol + ) elif col_type == "parallel_embedding": result_data = tr0_out[0] np.random.seed(2020) @@ -384,48 +402,51 @@ class TestDistBase(unittest.TestCase): for i in range(result_data.shape[0]): for j in range(result_data.shape[1]): data = result_data[i][j] - assert np.allclose(tr0_out[1][i][j], - need_result[data], - atol=1e-08) + assert np.allclose( + tr0_out[1][i][j], need_result[data], atol=1e-08 + ) elif col_type == "row_parallel_linear": result_data = tr0_out[0] np.random.seed(2020) weight = np.random.rand(1000, 16) need_result = np.matmul(input1, weight) - np.testing.assert_allclose(result_data, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + result_data, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "column_parallel_linear": result_data = tr0_out[0] np.random.seed(2020) weight = np.random.rand(1000, 16) need_result = np.matmul(input1, weight) - np.testing.assert_allclose(result_data, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + result_data, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "alltoall": - need_result1 = np.vstack((input1[0:input1.shape[0] // 2, :], - input2[0:input2.shape[0] // 2, :])) - need_result2 = np.vstack((input1[input1.shape[0] // 2:, :], - input2[input2.shape[0] // 2:, :])) + need_result1 = np.vstack( + ( + input1[0 : input1.shape[0] // 2, :], + input2[0 : input2.shape[0] // 2, :], + ) + ) + need_result2 = np.vstack( + ( + input1[input1.shape[0] // 2 :, :], + input2[input2.shape[0] // 2 :, :], + ) + ) tr0_out = np.vstack(tr0_out) tr1_out = np.vstack(tr1_out) - np.testing.assert_allclose(tr0_out, - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out, need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv": result_data = tr1_out[0] - np.testing.assert_allclose(input1, - result_data, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + input1, result_data, rtol=1e-05, atol=1e-05 + ) elif col_type == "global_gather": in_feat = 2 n_expert = 2 @@ -434,7 +455,8 @@ class TestDistBase(unittest.TestCase): np.random.seed(pid0) local_expert_count1 = np.random.randint( - 1, 4, size=tot_expert).astype("int") + 1, 4, size=tot_expert + ).astype("int") expert_ptr1 = np.ones(tot_expert, dtype=np.int32) expert_ptr1[0] = 0 for i in range(1, tot_expert): @@ -442,7 +464,8 @@ class TestDistBase(unittest.TestCase): np.random.seed(pid1) local_expert_count2 = np.random.randint( - 1, 4, size=tot_expert).astype("int") + 1, 4, size=tot_expert + ).astype("int") expert_ptr2 = np.ones(tot_expert, dtype=np.int32) expert_ptr2[0] = 0 for i in range(1, tot_expert): @@ -457,12 +480,14 @@ class TestDistBase(unittest.TestCase): np.random.seed(pid0) fwd_expert_count = sum(global_expert_count1).astype("int") - local_input_buf1 = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf1 = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) np.random.seed(pid1) fwd_expert_count = sum(global_expert_count2).astype("int") - local_input_buf2 = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf2 = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) output1 = [[], [], [], []] output2 = [[], [], [], []] send_ptr1 = 0 @@ -472,17 +497,21 @@ class TestDistBase(unittest.TestCase): for j in range(world_size): idx = j * n_expert + i if j == 0: - output1_part1 = local_input_buf1[send_ptr1: \ - send_ptr1 + global_expert_count1[idx], :] - output1_part2 = local_input_buf2[send_ptr2: \ - send_ptr2 + global_expert_count2[idx], :] + output1_part1 = local_input_buf1[ + send_ptr1 : send_ptr1 + global_expert_count1[idx], : + ] + output1_part2 = local_input_buf2[ + send_ptr2 : send_ptr2 + global_expert_count2[idx], : + ] output1[i].extend(output1_part1) output1[i + n_expert].extend(output1_part2) else: - output2_part1 = local_input_buf1[send_ptr1: \ - send_ptr1 + global_expert_count1[idx]] - output2_part2 = local_input_buf2[send_ptr2: \ - send_ptr2 + global_expert_count2[idx]] + output2_part1 = local_input_buf1[ + send_ptr1 : send_ptr1 + global_expert_count1[idx] + ] + output2_part2 = local_input_buf2[ + send_ptr2 : send_ptr2 + global_expert_count2[idx] + ] output2[i].extend(output2_part1) output2[i + n_expert].extend(output2_part2) send_ptr1 = send_ptr1 + global_expert_count1[idx] @@ -503,12 +532,14 @@ class TestDistBase(unittest.TestCase): output1 = np.array([]) else: output1 = np.concatenate(result1, axis=0).reshape( - sum(local_expert_count1), in_feat) + sum(local_expert_count1), in_feat + ) if result2 == []: output2 = np.array([]) else: output2 = np.concatenate(result2, axis=0).reshape( - sum(local_expert_count2), in_feat) + sum(local_expert_count2), in_feat + ) if tr0_out[0] is None or tr0_out[0].shape[0] == 0: tr0_out[0] = np.array([]) @@ -516,30 +547,27 @@ class TestDistBase(unittest.TestCase): if tr1_out[0] is None or tr1_out[0].shape[0] == 0: tr1_out[0] = np.array([]) - np.testing.assert_allclose(tr0_out[0], - output1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - output2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], output1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], output2, rtol=1e-05, atol=1e-05 + ) if static_mode == 0: - np.testing.assert_allclose(tr0_out[1], - 2 * local_input_buf1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[1], - 2 * local_input_buf2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[1], 2 * local_input_buf1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[1], 2 * local_input_buf2, rtol=1e-05, atol=1e-05 + ) elif col_type == "global_scatter": np.random.seed(pid0) local_expert_count1 = np.random.randint(1, 4, size=4).astype("int") fwd_expert_count = sum(local_expert_count1) - local_input_buf1 = np.random.rand(fwd_expert_count, - 2).astype("float32") + local_input_buf1 = np.random.rand(fwd_expert_count, 2).astype( + "float32" + ) expert_ptr1 = np.ones(4, dtype=np.int32) expert_ptr1[0] = 0 for i in range(1, 4): @@ -547,8 +575,9 @@ class TestDistBase(unittest.TestCase): np.random.seed(pid1) local_expert_count2 = np.random.randint(1, 4, size=4).astype("int") fwd_expert_count = sum(local_expert_count2) - local_input_buf2 = np.random.rand(fwd_expert_count, - 2).astype("float32") + local_input_buf2 = np.random.rand(fwd_expert_count, 2).astype( + "float32" + ) expert_ptr2 = np.ones(4, dtype=np.int32) expert_ptr2[0] = 0 for i in range(1, 4): @@ -561,15 +590,31 @@ class TestDistBase(unittest.TestCase): idx = j * 2 + i if j == 0: # send data to 0 card - output1.append(local_input_buf1[expert_ptr1[idx]: \ - expert_ptr1[idx]+local_expert_count1[idx]]) - output1.append(local_input_buf2[expert_ptr2[idx]:\ - expert_ptr2[idx]+local_expert_count2[idx]]) + output1.append( + local_input_buf1[ + expert_ptr1[idx] : expert_ptr1[idx] + + local_expert_count1[idx] + ] + ) + output1.append( + local_input_buf2[ + expert_ptr2[idx] : expert_ptr2[idx] + + local_expert_count2[idx] + ] + ) else: - output2.append(local_input_buf1[expert_ptr1[idx]: \ - expert_ptr1[idx]+local_expert_count1[idx]]) - output2.append(local_input_buf2[expert_ptr2[idx]:\ - expert_ptr2[idx]+local_expert_count2[idx]]) + output2.append( + local_input_buf1[ + expert_ptr1[idx] : expert_ptr1[idx] + + local_expert_count1[idx] + ] + ) + output2.append( + local_input_buf2[ + expert_ptr2[idx] : expert_ptr2[idx] + + local_expert_count2[idx] + ] + ) if output1 == []: output1 = np.array([]) else: @@ -585,22 +630,18 @@ class TestDistBase(unittest.TestCase): if tr1_out[0] is None or tr1_out[0].shape[0] == 0: tr1_out[0] = np.array([]) - np.testing.assert_allclose(tr0_out[0], - output1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - output2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], output1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], output2, rtol=1e-05, atol=1e-05 + ) if static_mode == 0: - np.testing.assert_allclose(tr0_out[1], - 2 * local_input_buf1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[1], - 2 * local_input_buf2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[1], 2 * local_input_buf1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[1], 2 * local_input_buf2, rtol=1e-05, atol=1e-05 + ) else: pass diff --git a/python/paddle/fluid/tests/unittests/test_collective_base.py b/python/paddle/fluid/tests/unittests/test_collective_base.py index 7300c0d13dc..2104f1865f4 100644 --- a/python/paddle/fluid/tests/unittests/test_collective_base.py +++ b/python/paddle/fluid/tests/unittests/test_collective_base.py @@ -27,10 +27,10 @@ from paddle.fluid import core class TestCollectiveRunnerBase(object): - def get_model(self, train_prog, startup_prog): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): while True: @@ -38,13 +38,15 @@ class TestCollectiveRunnerBase(object): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -52,44 +54,51 @@ class TestCollectiveRunnerBase(object): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - nccl_id_var = block.create_var(name=nameGen.generate('nccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) + nccl_id_var = block.create_var( + name=nameGen.generate('nccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': self.global_ring_id - }) + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': self.global_ring_id, + }, + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -98,20 +107,22 @@ class TestCollectiveRunnerBase(object): rank = args["trainerid"] current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) self.rank = rank result = self.get_model(train_prog, startup_prog) device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_prog) np.random.seed(os.getpid()) indata = np.random.random((10, 1000)) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=[result.name]) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=[result.name] + ) sys.stdout.buffer.write(pickle.dumps(out)) @@ -132,12 +143,13 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self.temp_dir = tempfile.TemporaryDirectory() @@ -146,10 +158,10 @@ class TestDistBase(unittest.TestCase): self.temp_dir.cleanup() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -162,13 +174,13 @@ class TestDistBase(unittest.TestCase): def _run_cluster(self, model_file, envs): worker_endpoints = self._ps_endpoints.split(",") w0_ep, w1_ep = worker_endpoints - #print("w0_ep:",w0_ep," w1_ep:",w1_ep) + # print("w0_ep:",w0_ep," w1_ep:",w1_ep) env0 = { "FLAGS_selected_gpus": "0", "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -176,9 +188,9 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) tr_cmd = "%s %s" @@ -188,16 +200,20 @@ class TestDistBase(unittest.TestCase): path1 = os.path.join(self.temp_dir.name, "/tmp/tr1_err.log") tr0_pipe = open(path0, "wb") tr1_pipe = open(path1, "wb") - #print(tr0_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) + # print(tr0_cmd) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -206,14 +222,16 @@ class TestDistBase(unittest.TestCase): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) - def check_with_place(self, - model_file, - col_type, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, col_type, check_error_log=False, need_envs={} + ): required_envs = { "FLAGS_fraction_of_gpu_memory_to_use": "0.15", "FLAGS_eager_delete_tensor_gb": "0.0", @@ -222,14 +240,15 @@ class TestDistBase(unittest.TestCase): "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "LD_PRELOAD": os.getenv("LD_PRELOAD", ""), "GLOG_v": "3", - "NCCL_P2P_DISABLE": "1" + "NCCL_P2P_DISABLE": "1", } required_envs.update(need_envs) if check_error_log: required_envs["GLOG_v"] = "3" required_envs["GLOG_logtostderr"] = "1" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) np.random.seed(pid0) input1 = np.random.random((10, 1000)) np.random.seed(pid1) @@ -247,38 +266,33 @@ class TestDistBase(unittest.TestCase): np.testing.assert_allclose(tr1_out[0], need_result, rtol=1e-05) elif col_type == "scatter": need_result = input2 - need_result1 = need_result[0:need_result.shape[0] // 2] - need_result2 = need_result[need_result.shape[0] // 2:] + need_result1 = need_result[0 : need_result.shape[0] // 2] + need_result2 = need_result[need_result.shape[0] // 2 :] np.testing.assert_allclose(tr0_out[0], need_result1, rtol=1e-05) np.testing.assert_allclose(tr1_out[0], need_result2, rtol=1e-05) elif col_type == "allreduce": need_result = input1 + input2 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "reduce_scatter": tmp = input1 + input2 - need_result1 = tmp[0:tmp.shape[0] // 2] - need_result2 = tmp[tmp.shape[0] // 2:] - np.testing.assert_allclose(tr0_out[0], - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result2, - rtol=1e-05, - atol=1e-05) + need_result1 = tmp[0 : tmp.shape[0] // 2] + need_result2 = tmp[tmp.shape[0] // 2 :] + np.testing.assert_allclose( + tr0_out[0], need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv": need_result = input1 - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "identity": need_result1 = input1 need_result2 = input2 @@ -296,35 +310,29 @@ class TestDistBase(unittest.TestCase): np.testing.assert_allclose(tr1_out, need_result2, rtol=1e-05) elif col_type == "concat": need_result = np.concatenate((input1, input2), axis=1) - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "split": need_result1 = np.split(input1, 2, axis=1)[0] need_result2 = np.split(input2, 2, axis=1)[1] - np.testing.assert_allclose(tr0_out[0], - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv_array": need_result1 = np.array([[0, 1, 2]]) need_result2 = np.array([[3, 4, 5]]) - np.testing.assert_allclose(tr1_out[0][0], - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0][1], - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr1_out[0][0], need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0][1], need_result2, rtol=1e-05, atol=1e-05 + ) else: pass diff --git a/python/paddle/fluid/tests/unittests/test_communicator_async.py b/python/paddle/fluid/tests/unittests/test_communicator_async.py index 0908dbc8df1..33a98a5c894 100644 --- a/python/paddle/fluid/tests/unittests/test_communicator_async.py +++ b/python/paddle/fluid/tests/unittests/test_communicator_async.py @@ -26,7 +26,6 @@ import paddle.distributed.fleet as fleet class TestCommunicator(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[1], dtype='float32') y = fluid.layers.data(name='y', shape=[1], dtype='float32') @@ -40,7 +39,8 @@ class TestCommunicator(unittest.TestCase): current_id=0, role=role_maker.Role.WORKER, worker_num=2, - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) avg_cost = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_communicator_geo.py b/python/paddle/fluid/tests/unittests/test_communicator_geo.py index 6c99eaac737..0efea743f33 100644 --- a/python/paddle/fluid/tests/unittests/test_communicator_geo.py +++ b/python/paddle/fluid/tests/unittests/test_communicator_geo.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestCommunicatorGeoEnd2End(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[13], dtype='float32') x1 = fluid.layers.data(name='x1', shape=[1], dtype='int64', lod_level=1) @@ -41,8 +40,10 @@ class TestCommunicatorGeoEnd2End(unittest.TestCase): size=[10000, 10], param_attr=fluid.ParamAttr( name="embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) pool = fluid.layers.sequence_pool(input=emb, pool_type="sum") z = fluid.layers.concat(input=[x, pool], axis=1) @@ -54,7 +55,6 @@ class TestCommunicatorGeoEnd2End(unittest.TestCase): return avg_cost, x, x1, y def fake_reader(self): - def reader(): for i in range(10000): x = numpy.random.random((1, 13)).astype('float32') @@ -91,9 +91,11 @@ class TestCommunicatorGeoEnd2End(unittest.TestCase): feeder = fluid.DataFeeder(place=place, feed_list=[x, z, y]) for batch_id, data in enumerate(train_reader()): - exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[]) + exe.run( + fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[], + ) fleet.stop_worker() @@ -166,9 +168,11 @@ half_run_server.run_ut() ps_cmd = "{} {}".format(_python, server_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) time.sleep(5) diff --git a/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py b/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py index 23af083af56..328874b3f55 100644 --- a/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py @@ -27,7 +27,6 @@ import paddle.distributed.fleet as fleet class TestCommunicator(unittest.TestCase): - def test_communicator_ps_gpu(self): temp_dir = tempfile.TemporaryDirectory() path = os.path.join(temp_dir.name, "test_communicator_ps_gpu.txt") @@ -42,9 +41,11 @@ class TestCommunicator(unittest.TestCase): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002,127.0.0.2:36002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36002,127.0.0.2:36002" os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["FLAGS_selected_gpus"] = "0" role = role_maker.PaddleCloudRoleMaker() @@ -71,10 +72,9 @@ class TestCommunicator(unittest.TestCase): optimizer.minimize(avg_cost) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=1, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=1, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist(["test_communicator_ps_gpu.txt"]) dataset.set_date("20211111") dataset.load_into_memory(is_shuffle=True) diff --git a/python/paddle/fluid/tests/unittests/test_compare_op.py b/python/paddle/fluid/tests/unittests/test_compare_op.py index a06ac3a9b02..d9636972a13 100755 --- a/python/paddle/fluid/tests/unittests/test_compare_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_op.py @@ -23,9 +23,7 @@ from paddle.fluid import Program, program_guard def create_test_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): a = numpy.random.random(size=(10, 7)).astype(typename) b = numpy.random.random(size=(10, 7)).astype(typename) @@ -45,11 +43,9 @@ def create_test_class(op_type, typename, callback): y = fluid.layers.data(name='y', shape=[2], dtype='int32') a = fluid.layers.data(name='a', shape=[2], dtype='int16') if self.op_type == "less_than": - self.assertRaises(TypeError, - fluid.layers.less_than, - x=x, - y=y, - force_cpu=1) + self.assertRaises( + TypeError, fluid.layers.less_than, x=x, y=y, force_cpu=1 + ) op = eval("fluid.layers.%s" % self.op_type) self.assertRaises(TypeError, op, x=x, y=y, cond=1) self.assertRaises(TypeError, op, x=x, y=a) @@ -75,9 +71,7 @@ for _type_name in {'float32', 'float64', 'int32', 'int64', 'float16'}: def create_paddle_case(op_type, callback): - class PaddleCls(unittest.TestCase): - def setUp(self): self.op_type = op_type self.input_x = np.array([1, 2, 3, 4]).astype(np.int64) @@ -95,11 +89,10 @@ def create_paddle_case(op_type, callback): op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = fluid.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[out], + ) self.assertEqual((res == self.real_result).all(), True) def test_api_float(self): @@ -111,11 +104,9 @@ def create_paddle_case(op_type, callback): op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = fluid.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": 1.0 - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": self.input_x, "y": 1.0}, fetch_list=[out] + ) self.real_result = np.array([1, 0, 0, 0]).astype(np.int64) self.assertEqual((res == self.real_result).all(), True) @@ -160,31 +151,36 @@ def create_paddle_case(op_type, callback): self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_inf_2(self): if self.op_type == "equal": paddle.disable_static() - x1 = np.array([1, float('inf'), - float('inf')]).astype(np.float32) + x1 = np.array([1, float('inf'), float('inf')]).astype( + np.float32 + ) x = paddle.to_tensor(x1) - y1 = np.array([1, float('-inf'), - float('inf')]).astype(np.float32) + y1 = np.array([1, float('-inf'), float('inf')]).astype( + np.float32 + ) y = paddle.to_tensor(y1) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_inf_3(self): if self.op_type == "equal": paddle.disable_static() - x1 = np.array([1, float('inf'), - float('-inf')]).astype(np.float32) + x1 = np.array([1, float('inf'), float('-inf')]).astype( + np.float32 + ) x = paddle.to_tensor(x1) y1 = np.array([1, 2, 3]).astype(np.float32) y = paddle.to_tensor(y1) @@ -193,7 +189,8 @@ def create_paddle_case(op_type, callback): self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_nan_1(self): @@ -208,31 +205,36 @@ def create_paddle_case(op_type, callback): self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_nan_2(self): if self.op_type == "equal": paddle.disable_static() - x1 = np.array([1, float('nan'), - float('nan')]).astype(np.float32) + x1 = np.array([1, float('nan'), float('nan')]).astype( + np.float32 + ) x = paddle.to_tensor(x1) - y1 = np.array([1, float('-nan'), - float('nan')]).astype(np.float32) + y1 = np.array([1, float('-nan'), float('nan')]).astype( + np.float32 + ) y = paddle.to_tensor(y1) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_nan_3(self): if self.op_type == "equal": paddle.disable_static() - x1 = np.array([1, float('-nan'), - float('nan')]).astype(np.float32) + x1 = np.array([1, float('-nan'), float('nan')]).astype( + np.float32 + ) x = paddle.to_tensor(x1) y1 = np.array([1, 2, 1]).astype(np.float32) y = paddle.to_tensor(y1) @@ -241,16 +243,19 @@ def create_paddle_case(op_type, callback): self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_not_equal(self): if self.op_type == "not_equal": paddle.disable_static() - x = paddle.to_tensor(np.array([1.2e-8, 2, 2, 1]), - dtype="float32") - y = paddle.to_tensor(np.array([1.1e-8, 2, 2, 1]), - dtype="float32") + x = paddle.to_tensor( + np.array([1.2e-8, 2, 2, 1]), dtype="float32" + ) + y = paddle.to_tensor( + np.array([1.1e-8, 2, 2, 1]), dtype="float32" + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) self.real_result = np.array([0, 0, 0, 0]).astype(np.int64) @@ -258,7 +263,6 @@ def create_paddle_case(op_type, callback): paddle.enable_static() def test_assert(self): - def test_dynamic_api_string(self): if self.op_type == "equal": paddle.disable_static() @@ -282,9 +286,9 @@ def create_paddle_case(op_type, callback): def test_broadcast_api_1(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 1, 3], - dtype='int32') + x = paddle.static.data( + name='x', shape=[1, 2, 1, 3], dtype='int32' + ) y = paddle.static.data(name='y', shape=[1, 2, 3], dtype='int32') op = eval("paddle.%s" % (self.op_type)) out = op(x, y) @@ -292,31 +296,27 @@ def create_paddle_case(op_type, callback): input_x = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(np.int32) input_y = np.arange(0, 6).reshape((1, 2, 3)).astype(np.int32) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_broadcast_api_2(self): paddle.enable_static() with program_guard(Program(), Program()): x = paddle.static.data(name='x', shape=[1, 2, 3], dtype='int32') - y = paddle.static.data(name='y', - shape=[1, 2, 1, 3], - dtype='int32') + y = paddle.static.data( + name='y', shape=[1, 2, 1, 3], dtype='int32' + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(0, 6).reshape((1, 2, 3)).astype(np.int32) input_y = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(np.int32) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_broadcast_api_3(self): @@ -330,11 +330,9 @@ def create_paddle_case(op_type, callback): input_x = np.arange(0, 5).reshape((5)).astype(np.int32) input_y = np.array([5, 3, 2]).reshape((3, 1)).astype(np.int32) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_bool_api_4(self): @@ -348,11 +346,9 @@ def create_paddle_case(op_type, callback): input_x = np.array([True, False, True]).astype(np.bool_) input_y = np.array([True, True, False]).astype(np.bool_) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_bool_broadcast_api_4(self): @@ -366,11 +362,9 @@ def create_paddle_case(op_type, callback): input_x = np.array([True, False, True]).astype(np.bool_) input_y = np.array([True]).astype(np.bool_) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_attr_name(self): @@ -396,19 +390,18 @@ create_paddle_case('not_equal', lambda _a, _b: _a != _b) class TestCompareOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input x and y of compare_op must be Variable. x = fluid.layers.data(name='x', shape=[1], dtype="float32") - y = fluid.create_lod_tensor(numpy.array([[-1]]), [[1]], - fluid.CPUPlace()) + y = fluid.create_lod_tensor( + numpy.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.greater_equal, x, y) class API_TestElementwise_Equal(unittest.TestCase): - def test_api(self): paddle.enable_static() with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -417,7 +410,7 @@ class API_TestElementwise_Equal(unittest.TestCase): out = paddle.equal(x=label, y=limit) place = fluid.CPUPlace() exe = fluid.Executor(place) - res, = exe.run(fetch_list=[out]) + (res,) = exe.run(fetch_list=[out]) self.assertEqual((res == np.array([True, False])).all(), True) with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -426,12 +419,11 @@ class API_TestElementwise_Equal(unittest.TestCase): out = paddle.equal(x=label, y=limit) place = fluid.CPUPlace() exe = fluid.Executor(place) - res, = exe.run(fetch_list=[out]) + (res,) = exe.run(fetch_list=[out]) self.assertEqual((res == np.array([True, True])).all(), True) class TestCompareOpPlace(unittest.TestCase): - def test_place_1(self): paddle.enable_static() place = paddle.CPUPlace() @@ -441,7 +433,7 @@ class TestCompareOpPlace(unittest.TestCase): limit = fluid.layers.assign(np.array([3, 2], dtype="int32")) out = fluid.layers.less_than(label, limit, force_cpu=True) exe = fluid.Executor(place) - res, = exe.run(fetch_list=[out]) + (res,) = exe.run(fetch_list=[out]) self.assertEqual((res == np.array([False, False])).all(), True) def test_place_2(self): diff --git a/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py b/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py index 5508f17a3bc..bc88cba96eb 100644 --- a/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py @@ -20,9 +20,7 @@ import paddle.fluid as fluid def create_test_not_equal_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): x = np.random.random(size=(10, 7)).astype(typename) y = np.random.random(size=(10, 7)).astype(typename) @@ -41,9 +39,7 @@ def create_test_not_equal_class(op_type, typename, callback): def create_test_not_shape_equal_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): x = np.random.random(size=(10, 7)).astype(typename) y = np.random.random(size=(10)).astype(typename) @@ -62,9 +58,7 @@ def create_test_not_shape_equal_class(op_type, typename, callback): def create_test_equal_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): x = y = np.random.random(size=(10, 7)).astype(typename) z = callback(x, y) @@ -82,9 +76,7 @@ def create_test_equal_class(op_type, typename, callback): def create_test_dim1_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): x = y = np.random.random(size=(1)).astype(typename) x = np.array([True, False, True]).astype(typename) @@ -112,7 +104,6 @@ for _type_name in {'float32', 'float64', 'int32', 'int64', 'bool'}: class TestEqualReduceAPI(unittest.TestCase): - def test_name(self): x = fluid.layers.assign(np.array([3, 4], dtype="int32")) y = fluid.layers.assign(np.array([3, 4], dtype="int32")) diff --git a/python/paddle/fluid/tests/unittests/test_compat.py b/python/paddle/fluid/tests/unittests/test_compat.py index e4fedede47c..d4ad8a8274e 100644 --- a/python/paddle/fluid/tests/unittests/test_compat.py +++ b/python/paddle/fluid/tests/unittests/test_compat.py @@ -17,7 +17,6 @@ import paddle.compat as cpt class TestCompatible(unittest.TestCase): - def test_to_text(self): self.assertIsNone(cpt.to_text(None)) diff --git a/python/paddle/fluid/tests/unittests/test_compiled_program.py b/python/paddle/fluid/tests/unittests/test_compiled_program.py index 9406d4d898e..8418de88fd8 100644 --- a/python/paddle/fluid/tests/unittests/test_compiled_program.py +++ b/python/paddle/fluid/tests/unittests/test_compiled_program.py @@ -22,84 +22,87 @@ from simple_nets import simple_fc_net class TestCompiledProgram(unittest.TestCase): - def setUp(self): self.seed = 100 self.img = np.random.random(size=(16, 784)).astype('float32') - self.label = np.random.randint(low=0, - high=10, - size=[16, 1], - dtype=np.int64) + self.label = np.random.randint( + low=0, high=10, size=[16, 1], dtype=np.int64 + ) with new_program_scope(): paddle.seed(self.seed) paddle.framework.random._manual_program_seed(self.seed) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) loss = simple_fc_net() exe.run(fluid.default_startup_program()) - loss_data, = exe.run(fluid.default_main_program(), - feed={ - "image": self.img, - "label": self.label - }, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + fluid.default_main_program(), + feed={"image": self.img, "label": self.label}, + fetch_list=[loss.name], + ) self.loss = loss_data[0] def test_compiled_program_base(self): with new_program_scope(): paddle.seed(self.seed) paddle.framework.random._manual_program_seed(self.seed) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) loss = simple_fc_net() exe.run(fluid.default_startup_program()) compiled_prog = fluid.CompiledProgram(fluid.default_main_program()) - loss_data, = exe.run(compiled_prog, - feed={ - "image": self.img, - "label": self.label - }, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + compiled_prog, + feed={"image": self.img, "label": self.label}, + fetch_list=[loss.name], + ) np.testing.assert_array_equal(loss_data[0], self.loss) def test_compiled_program_with_data_parallel(self): with new_program_scope(): paddle.seed(self.seed) paddle.framework.random._manual_program_seed(self.seed) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) loss = simple_fc_net() exe.run(fluid.default_startup_program()) compiled_prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name, places=[place]) - - loss_data, = exe.run(compiled_prog, - feed={ - "image": self.img, - "label": self.label - }, - fetch_list=[loss.name]) + fluid.default_main_program() + ).with_data_parallel(loss_name=loss.name, places=[place]) + + (loss_data,) = exe.run( + compiled_prog, + feed={"image": self.img, "label": self.label}, + fetch_list=[loss.name], + ) np.testing.assert_array_equal(loss_data[0], self.loss) class TestCompiledProgramError(unittest.TestCase): - def test_program_or_graph_error(self): self.assertRaises(TypeError, fluid.CompiledProgram, "program") def build_simple_model(self): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') prediction = fluid.layers.fc(input=img, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -112,7 +115,8 @@ class TestCompiledProgramError(unittest.TestCase): # compile program program = fluid.default_main_program() compiled_program = fluid.CompiledProgram( - program).with_data_parallel() + program + ).with_data_parallel() return compiled_program def compile_program(self): @@ -148,7 +152,8 @@ class TestCompiledProgramError(unittest.TestCase): # compile program program = fluid.default_main_program() compiled_program = fluid.CompiledProgram( - program).with_data_parallel(share_vars_from=source_program) + program + ).with_data_parallel(share_vars_from=source_program) scope = fluid.global_scope() place = fluid.CPUPlace() with self.assertRaises(ValueError): @@ -161,7 +166,8 @@ class TestCompiledProgramError(unittest.TestCase): # compile program program = fluid.default_main_program() compiled_program = fluid.CompiledProgram( - program).with_data_parallel(share_vars_from=source_program) + program + ).with_data_parallel(share_vars_from=source_program) scope = fluid.global_scope() place = fluid.CPUPlace() with self.assertRaises(ValueError): diff --git a/python/paddle/fluid/tests/unittests/test_complex_abs.py b/python/paddle/fluid/tests/unittests/test_complex_abs.py index e61732eba57..968bcb79613 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_abs.py +++ b/python/paddle/fluid/tests/unittests/test_complex_abs.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexAbsOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = paddle.abs @@ -37,7 +36,8 @@ class TestComplexAbsOp(OpTest): def init_input_output(self): self.x = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = np.abs(self.x) def init_grad_input_output(self): @@ -48,15 +48,16 @@ class TestComplexAbsOp(OpTest): self.check_output(check_eager=False) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) class TestComplexAbsOpZeroValues(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "abs" @@ -70,8 +71,9 @@ class TestComplexAbsOpZeroValues(OpTest): self.outputs = {'Out': self.out} def init_input_output(self): - self.x = np.zeros(self.shape).astype( - self.dtype) + 1J * np.zeros(self.shape).astype(self.dtype) + self.x = np.zeros(self.shape).astype(self.dtype) + 1j * np.zeros( + self.shape + ).astype(self.dtype) self.out = np.abs(self.x) def init_grad_input_output(self): @@ -82,15 +84,16 @@ class TestComplexAbsOpZeroValues(OpTest): self.check_output(check_eager=False) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) class TestAbs(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -111,7 +114,6 @@ class TestAbs(unittest.TestCase): class TestRealAbsOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = paddle.abs @@ -136,11 +138,13 @@ class TestRealAbsOp(OpTest): self.check_output(check_eager=False) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_complex_cast.py b/python/paddle/fluid/tests/unittests/test_complex_cast.py index 015c56965df..9065b2d008d 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_cast.py +++ b/python/paddle/fluid/tests/unittests/test_complex_cast.py @@ -20,12 +20,11 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexCastOp(unittest.TestCase): - def test_complex_to_real(self): r = np.random.random(size=[10, 10]) * 10 i = np.random.random(size=[10, 10]) - c_t = paddle.to_tensor(r + i * 1J, dtype='complex64') + c_t = paddle.to_tensor(r + i * 1j, dtype='complex64') self.assertEqual(c_t.cast('int64').dtype, paddle.int64) self.assertEqual(c_t.cast('int32').dtype, paddle.int32) @@ -33,21 +32,21 @@ class TestComplexCastOp(unittest.TestCase): self.assertEqual(c_t.cast('float64').dtype, paddle.float64) self.assertEqual(c_t.cast('bool').dtype, paddle.bool) - np.testing.assert_allclose(c_t.cast('int64').numpy(), - r.astype('int64'), - rtol=1e-05) - np.testing.assert_allclose(c_t.cast('int32').numpy(), - r.astype('int32'), - rtol=1e-05) - np.testing.assert_allclose(c_t.cast('float32').numpy(), - r.astype('float32'), - rtol=1e-05) - np.testing.assert_allclose(c_t.cast('float64').numpy(), - r.astype('float64'), - rtol=1e-05) - np.testing.assert_allclose(c_t.cast('bool').numpy(), - r.astype('bool'), - rtol=1e-05) + np.testing.assert_allclose( + c_t.cast('int64').numpy(), r.astype('int64'), rtol=1e-05 + ) + np.testing.assert_allclose( + c_t.cast('int32').numpy(), r.astype('int32'), rtol=1e-05 + ) + np.testing.assert_allclose( + c_t.cast('float32').numpy(), r.astype('float32'), rtol=1e-05 + ) + np.testing.assert_allclose( + c_t.cast('float64').numpy(), r.astype('float64'), rtol=1e-05 + ) + np.testing.assert_allclose( + c_t.cast('bool').numpy(), r.astype('bool'), rtol=1e-05 + ) def test_real_to_complex(self): r = np.random.random(size=[10, 10]) * 10 @@ -56,29 +55,29 @@ class TestComplexCastOp(unittest.TestCase): self.assertEqual(r_t.cast('complex64').dtype, paddle.complex64) self.assertEqual(r_t.cast('complex128').dtype, paddle.complex128) - np.testing.assert_allclose(r_t.cast('complex64').real().numpy(), - r, - rtol=1e-05) - np.testing.assert_allclose(r_t.cast('complex128').real().numpy(), - r, - rtol=1e-05) + np.testing.assert_allclose( + r_t.cast('complex64').real().numpy(), r, rtol=1e-05 + ) + np.testing.assert_allclose( + r_t.cast('complex128').real().numpy(), r, rtol=1e-05 + ) def test_complex64_complex128(self): r = np.random.random(size=[10, 10]) i = np.random.random(size=[10, 10]) - c = r + i * 1J + c = r + i * 1j c_64 = paddle.to_tensor(c, dtype='complex64') c_128 = paddle.to_tensor(c, dtype='complex128') self.assertTrue(c_64.cast('complex128').dtype, paddle.complex128) self.assertTrue(c_128.cast('complex128').dtype, paddle.complex64) - np.testing.assert_allclose(c_64.cast('complex128').numpy(), - c_128.numpy(), - rtol=1e-05) - np.testing.assert_allclose(c_128.cast('complex128').numpy(), - c_64.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + c_64.cast('complex128').numpy(), c_128.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + c_128.cast('complex128').numpy(), c_64.numpy(), rtol=1e-05 + ) def test_eager(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_complex_elementwise_layers.py b/python/paddle/fluid/tests/unittests/test_complex_elementwise_layers.py index cd66246bfd5..14bdec6101a 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_elementwise_layers.py +++ b/python/paddle/fluid/tests/unittests/test_complex_elementwise_layers.py @@ -30,7 +30,6 @@ paddle_apis = { class TestComplexElementwiseLayers(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -48,21 +47,27 @@ class TestComplexElementwiseLayers(unittest.TestCase): pd_result, np_result, rtol=1e-05, - err_msg= - '\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n'. - format(place, pd_result[~np.isclose(pd_result, np_result)], - np_result[~np.isclose(pd_result, np_result)])) + err_msg='\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n'.format( + place, + pd_result[~np.isclose(pd_result, np_result)], + np_result[~np.isclose(pd_result, np_result)], + ), + ) def compare_by_basic_api(self, x, y): for place in self._places: - self.assert_check(self.paddle_calc(x, y, "add", place), x + y, - place) - self.assert_check(self.paddle_calc(x, y, "sub", place), x - y, - place) - self.assert_check(self.paddle_calc(x, y, "mul", place), x * y, - place) - self.assert_check(self.paddle_calc(x, y, "div", place), x / y, - place) + self.assert_check( + self.paddle_calc(x, y, "add", place), x + y, place + ) + self.assert_check( + self.paddle_calc(x, y, "sub", place), x - y, place + ) + self.assert_check( + self.paddle_calc(x, y, "mul", place), x * y, place + ) + self.assert_check( + self.paddle_calc(x, y, "div", place), x / y, place + ) def compare_op_by_basic_api(self, x, y): for place in self._places: @@ -76,18 +81,21 @@ class TestComplexElementwiseLayers(unittest.TestCase): def test_complex_xy(self): for dtype in self._dtypes: - x = rand([2, 3, 4, 5 - ]).astype(dtype) + 1j * rand([2, 3, 4, 5]).astype(dtype) - y = rand([2, 3, 4, 5 - ]).astype(dtype) + 1j * rand([2, 3, 4, 5]).astype(dtype) + x = rand([2, 3, 4, 5]).astype(dtype) + 1j * rand( + [2, 3, 4, 5] + ).astype(dtype) + y = rand([2, 3, 4, 5]).astype(dtype) + 1j * rand( + [2, 3, 4, 5] + ).astype(dtype) self.compare_by_basic_api(x, y) self.compare_op_by_basic_api(x, y) def test_complex_x_real_y(self): for dtype in self._dtypes: - x = rand([2, 3, 4, 5 - ]).astype(dtype) + 1j * rand([2, 3, 4, 5]).astype(dtype) + x = rand([2, 3, 4, 5]).astype(dtype) + 1j * rand( + [2, 3, 4, 5] + ).astype(dtype) y = rand([4, 5]).astype(dtype) # promote types cases diff --git a/python/paddle/fluid/tests/unittests/test_complex_getitem.py b/python/paddle/fluid/tests/unittests/test_complex_getitem.py index 2d5428ce933..7446e8f6dfe 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_getitem.py +++ b/python/paddle/fluid/tests/unittests/test_complex_getitem.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexGetitemLayer(unittest.TestCase): - def setUp(self): self._places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_complex_grad_accumulated.py b/python/paddle/fluid/tests/unittests/test_complex_grad_accumulated.py index 10c4eed62fd..9505ea3011a 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_grad_accumulated.py +++ b/python/paddle/fluid/tests/unittests/test_complex_grad_accumulated.py @@ -22,52 +22,56 @@ from paddle.fluid.framework import _test_eager_guard class Optimization_ex1(paddle.nn.Layer): - - def __init__(self, - shape, - dtype, - param_attr=paddle.nn.initializer.Uniform(low=-5., high=5.)): + def __init__( + self, + shape, + dtype, + param_attr=paddle.nn.initializer.Uniform(low=-5.0, high=5.0), + ): super(Optimization_ex1, self).__init__() - self.theta0 = self.create_parameter(shape=shape, - attr=param_attr, - dtype=dtype, - is_bias=False) - self.theta1 = self.create_parameter(shape=shape, - attr=param_attr, - dtype=dtype, - is_bias=False) + self.theta0 = self.create_parameter( + shape=shape, attr=param_attr, dtype=dtype, is_bias=False + ) + self.theta1 = self.create_parameter( + shape=shape, attr=param_attr, dtype=dtype, is_bias=False + ) self.A = paddle.to_tensor( - np.random.random((4, 4)).astype(dtype) + - np.random.random((4, 4)).astype(dtype) * 1j) - self.B = paddle.to_tensor(np.random.random( - (4, 4)).astype(dtype) + np.random.random((4, 4)).astype(dtype) * 1j, - stop_gradient=False) + np.random.random((4, 4)).astype(dtype) + + np.random.random((4, 4)).astype(dtype) * 1j + ) + self.B = paddle.to_tensor( + np.random.random((4, 4)).astype(dtype) + + np.random.random((4, 4)).astype(dtype) * 1j, + stop_gradient=False, + ) def forward(self, mode=1): jj = paddle.to_tensor(np.array([1j]).astype(np.complex64)) if mode == 1: # run all calc in one step loss = paddle.sum(self.A + (self.theta0 + self.theta1 * jj)) * ( - paddle.sum(self.A + (self.theta0 + self.theta1 * jj)).conj()) + paddle.sum(self.A + (self.theta0 + self.theta1 * jj)).conj() + ) return loss.real() elif mode == 2: # run in two step self.theta = self.theta0 + self.theta1 * jj loss = paddle.sum(self.A + self.theta) * ( - paddle.sum(self.A + self.theta).conj()) + paddle.sum(self.A + self.theta).conj() + ) return loss.real() elif mode == 3: # run without param - loss = paddle.sum(self.A + self.B) * (paddle.sum(self.A + - self.B).conj()) + loss = paddle.sum(self.A + self.B) * ( + paddle.sum(self.A + self.B).conj() + ) return loss.real() else: raise NotImplementedError class TestComplexGradAccumulated(unittest.TestCase): - def setUp(self): self.devices = ['cpu'] if core.is_compiled_with_cuda(): @@ -81,8 +85,9 @@ class TestComplexGradAccumulated(unittest.TestCase): paddle.set_device(device) myLayer = Optimization_ex1(self.theta_size, dtype) - optimizer = paddle.optimizer.SGD(learning_rate=self.learning_rate, - parameters=myLayer.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=self.learning_rate, parameters=myLayer.parameters() + ) for iter in range(self.iter): loss = myLayer(mode) @@ -95,8 +100,9 @@ class TestComplexGradAccumulated(unittest.TestCase): paddle.set_device(device) myLayer = Optimization_ex1(self.theta_size, dtype) - optimizer = paddle.optimizer.SGD(learning_rate=self.learning_rate, - parameters=myLayer.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=self.learning_rate, parameters=myLayer.parameters() + ) for iter in range(self.iter): loss = myLayer(mode) diff --git a/python/paddle/fluid/tests/unittests/test_complex_kron.py b/python/paddle/fluid/tests/unittests/test_complex_kron.py index f216e6d037b..7be6a3737ed 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_kron.py +++ b/python/paddle/fluid/tests/unittests/test_complex_kron.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class ComplexKronTestCase(unittest.TestCase): - def __init__(self, methodName='runTest', x=None, y=None): super(ComplexKronTestCase, self).__init__(methodName) self.x = x @@ -43,9 +42,9 @@ class ComplexKronTestCase(unittest.TestCase): x_var = dg.to_variable(self.x) y_var = dg.to_variable(self.y) out_var = paddle.kron(x_var, y_var) - np.testing.assert_allclose(out_var.numpy(), - self.ref_result, - rtol=1e-05) + np.testing.assert_allclose( + out_var.numpy(), self.ref_result, rtol=1e-05 + ) def test_eager(self, place): with _test_eager_guard(): @@ -56,23 +55,35 @@ def load_tests(loader, standard_tests, pattern): suite = unittest.TestSuite() for dtype in ["float32", "float64"]: suite.addTest( - ComplexKronTestCase(x=np.random.randn(2, 2).astype(dtype) + - 1j * np.random.randn(2, 2).astype(dtype), - y=np.random.randn(3, 3).astype(dtype) + - 1j * np.random.randn(3, 3).astype(dtype))) + ComplexKronTestCase( + x=np.random.randn(2, 2).astype(dtype) + + 1j * np.random.randn(2, 2).astype(dtype), + y=np.random.randn(3, 3).astype(dtype) + + 1j * np.random.randn(3, 3).astype(dtype), + ) + ) suite.addTest( - ComplexKronTestCase(x=np.random.randn(2, 2).astype(dtype), - y=np.random.randn(3, 3).astype(dtype) + - 1j * np.random.randn(3, 3).astype(dtype))) + ComplexKronTestCase( + x=np.random.randn(2, 2).astype(dtype), + y=np.random.randn(3, 3).astype(dtype) + + 1j * np.random.randn(3, 3).astype(dtype), + ) + ) suite.addTest( - ComplexKronTestCase(x=np.random.randn(2, 2).astype(dtype) + - 1j * np.random.randn(2, 2).astype(dtype), - y=np.random.randn(3, 3).astype(dtype))) + ComplexKronTestCase( + x=np.random.randn(2, 2).astype(dtype) + + 1j * np.random.randn(2, 2).astype(dtype), + y=np.random.randn(3, 3).astype(dtype), + ) + ) suite.addTest( - ComplexKronTestCase(x=np.random.randn(2, 2).astype(dtype) + - 1j * np.random.randn(2, 2).astype(dtype), - y=np.random.randn(2, 2, 3).astype(dtype))) + ComplexKronTestCase( + x=np.random.randn(2, 2).astype(dtype) + + 1j * np.random.randn(2, 2).astype(dtype), + y=np.random.randn(2, 2, 3).astype(dtype), + ) + ) return suite diff --git a/python/paddle/fluid/tests/unittests/test_complex_matmul.py b/python/paddle/fluid/tests/unittests/test_complex_matmul.py index 55cf8516078..f803930d2ad 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_matmul.py +++ b/python/paddle/fluid/tests/unittests/test_complex_matmul.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexMatMulLayer(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [fluid.CPUPlace()] @@ -39,10 +38,12 @@ class TestComplexMatMulLayer(unittest.TestCase): pd_result, np_result, rtol=1e-05, - err_msg= - '\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n' - .format(place, pd_result[~np.isclose(pd_result, np_result)], - np_result[~np.isclose(pd_result, np_result)])) + err_msg='\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n'.format( + place, + pd_result[~np.isclose(pd_result, np_result)], + np_result[~np.isclose(pd_result, np_result)], + ), + ) def compare_op_by_basic_api(self, x, y, np_result): for place in self._places: @@ -55,19 +56,21 @@ class TestComplexMatMulLayer(unittest.TestCase): pd_result, np_result, rtol=1e-05, - err_msg= - '\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n' - .format(place, pd_result[~np.isclose(pd_result, np_result)], - np_result[~np.isclose(pd_result, np_result)])) + err_msg='\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n'.format( + place, + pd_result[~np.isclose(pd_result, np_result)], + np_result[~np.isclose(pd_result, np_result)], + ), + ) def test_complex_xy(self): for dtype in self._dtypes: - x = np.random.random( - (2, 3, 4, 5)).astype(dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(dtype) - y = np.random.random( - (2, 3, 5, 4)).astype(dtype) + 1J * np.random.random( - (2, 3, 5, 4)).astype(dtype) + x = np.random.random((2, 3, 4, 5)).astype( + dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(dtype) + y = np.random.random((2, 3, 5, 4)).astype( + dtype + ) + 1j * np.random.random((2, 3, 5, 4)).astype(dtype) np_result = np.matmul(x, y) @@ -76,9 +79,9 @@ class TestComplexMatMulLayer(unittest.TestCase): def test_complex_x_real_y(self): for dtype in self._dtypes: - x = np.random.random( - (2, 3, 4, 5)).astype(dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(dtype) + x = np.random.random((2, 3, 4, 5)).astype( + dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(dtype) y = np.random.random((2, 3, 5, 4)).astype(dtype) np_result = np.matmul(x, y) @@ -90,9 +93,9 @@ class TestComplexMatMulLayer(unittest.TestCase): def test_real_x_complex_y(self): for dtype in self._dtypes: x = np.random.random((2, 3, 4, 5)).astype(dtype) - y = np.random.random( - (2, 3, 5, 4)).astype(dtype) + 1J * np.random.random( - (2, 3, 5, 4)).astype(dtype) + y = np.random.random((2, 3, 5, 4)).astype( + dtype + ) + 1j * np.random.random((2, 3, 5, 4)).astype(dtype) np_result = np.matmul(x, y) @@ -103,11 +106,12 @@ class TestComplexMatMulLayer(unittest.TestCase): # for coverage def test_complex_xy_gemv(self): for dtype in self._dtypes: - x = np.random.random( - (2, 1, 100)).astype(dtype) + 1J * np.random.random( - (2, 1, 100)).astype(dtype) - y = np.random.random((100)).astype(dtype) + 1J * np.random.random( - (100)).astype(dtype) + x = np.random.random((2, 1, 100)).astype( + dtype + ) + 1j * np.random.random((2, 1, 100)).astype(dtype) + y = np.random.random((100)).astype(dtype) + 1j * np.random.random( + (100) + ).astype(dtype) np_result = np.matmul(x, y) @@ -117,12 +121,12 @@ class TestComplexMatMulLayer(unittest.TestCase): # for coverage def test_complex_xy_gemm(self): for dtype in self._dtypes: - x = np.random.random( - (1, 2, 50)).astype(dtype) + 1J * np.random.random( - (1, 2, 50)).astype(dtype) - y = np.random.random( - (1, 50, 2)).astype(dtype) + 1J * np.random.random( - (1, 50, 2)).astype(dtype) + x = np.random.random((1, 2, 50)).astype( + dtype + ) + 1j * np.random.random((1, 2, 50)).astype(dtype) + y = np.random.random((1, 50, 2)).astype( + dtype + ) + 1j * np.random.random((1, 50, 2)).astype(dtype) np_result = np.matmul(x, y) diff --git a/python/paddle/fluid/tests/unittests/test_complex_op.py b/python/paddle/fluid/tests/unittests/test_complex_op.py index 498e8b43a9e..01c6b55921b 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_op.py +++ b/python/paddle/fluid/tests/unittests/test_complex_op.py @@ -48,7 +48,6 @@ def ref_complex_grad(x, y, dout): class TestComplexOp(OpTest): - def init_spec(self): self.x_shape = [10, 10] self.y_shape = [10, 10] @@ -61,8 +60,9 @@ class TestComplexOp(OpTest): x = np.random.randn(*self.x_shape).astype(self.dtype) y = np.random.randn(*self.y_shape).astype(self.dtype) out_ref = ref_complex(x, y) - self.out_grad = np.random.randn(*self.x_shape).astype(self.dtype) \ - + 1j * np.random.randn(*self.y_shape).astype(self.dtype) + self.out_grad = np.random.randn(*self.x_shape).astype( + self.dtype + ) + 1j * np.random.randn(*self.y_shape).astype(self.dtype) self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': out_ref} @@ -71,41 +71,49 @@ class TestComplexOp(OpTest): def test_check_grad(self): dout = self.out_grad - dx, dy = ref_complex_grad(self.inputs['X'], self.inputs['Y'], - self.out_grad) - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy], - user_defined_grad_outputs=[dout], - check_eager=True) + dx, dy = ref_complex_grad( + self.inputs['X'], self.inputs['Y'], self.out_grad + ) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[dx, dy], + user_defined_grad_outputs=[dout], + check_eager=True, + ) def test_check_grad_ignore_x(self): dout = self.out_grad - dx, dy = ref_complex_grad(self.inputs['X'], self.inputs['Y'], - self.out_grad) + dx, dy = ref_complex_grad( + self.inputs['X'], self.inputs['Y'], self.out_grad + ) self.assertTupleEqual(dx.shape, tuple(self.x_shape)) self.assertTupleEqual(dy.shape, tuple(self.y_shape)) - self.check_grad(['Y'], - 'Out', - no_grad_set=set('X'), - user_defined_grads=[dy], - user_defined_grad_outputs=[dout], - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set('X'), + user_defined_grads=[dy], + user_defined_grad_outputs=[dout], + check_eager=True, + ) def test_check_grad_ignore_y(self): dout = self.out_grad - dx, dy = ref_complex_grad(self.inputs['X'], self.inputs['Y'], - self.out_grad) - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[dx], - user_defined_grad_outputs=[dout], - check_eager=True) + dx, dy = ref_complex_grad( + self.inputs['X'], self.inputs['Y'], self.out_grad + ) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[dx], + user_defined_grad_outputs=[dout], + check_eager=True, + ) class TestComplexOpBroadcast1(TestComplexOp): - def init_spec(self): self.x_shape = [10, 3, 1, 4] self.y_shape = [100, 1] @@ -113,7 +121,6 @@ class TestComplexOpBroadcast1(TestComplexOp): class TestComplexOpBroadcast2(TestComplexOp): - def init_spec(self): self.x_shape = [100, 1] self.y_shape = [10, 3, 1, 4] @@ -121,7 +128,6 @@ class TestComplexOpBroadcast2(TestComplexOp): class TestComplexOpBroadcast3(TestComplexOp): - def init_spec(self): self.x_shape = [1, 100] self.y_shape = [100] @@ -129,7 +135,6 @@ class TestComplexOpBroadcast3(TestComplexOp): class TestComplexAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randn(10, 10) self.y = np.random.randn(10, 10) @@ -151,12 +156,9 @@ class TestComplexAPI(unittest.TestCase): exe = static.Executor() exe.run(sp) - [out_np] = exe.run(mp, - feed={ - "x": self.x, - "y": self.y - }, - fetch_list=[out]) + [out_np] = exe.run( + mp, feed={"x": self.x, "y": self.y}, fetch_list=[out] + ) np.testing.assert_allclose(self.out, out_np, rtol=1e-05) def test_eager(self): diff --git a/python/paddle/fluid/tests/unittests/test_complex_reshape.py b/python/paddle/fluid/tests/unittests/test_complex_reshape.py index 29bdb5d7c89..0e8398cf816 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_reshape.py +++ b/python/paddle/fluid/tests/unittests/test_complex_reshape.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexReshape(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -30,24 +29,24 @@ class TestComplexReshape(unittest.TestCase): def test_shape_norm_dims(self): for dtype in self._dtypes: - x_np = np.random.randn( - 2, 3, - 4).astype(dtype) + 1j * np.random.randn(2, 3, 4).astype(dtype) + x_np = np.random.randn(2, 3, 4).astype( + dtype + ) + 1j * np.random.randn(2, 3, 4).astype(dtype) shape = (2, -1) for place in self._places: with dg.guard(place): x_var = dg.to_variable(x_np) y_var = paddle.reshape(x_var, shape) y_np = y_var.numpy() - np.testing.assert_allclose(np.reshape(x_np, shape), - y_np, - rtol=1e-05) + np.testing.assert_allclose( + np.reshape(x_np, shape), y_np, rtol=1e-05 + ) def test_shape_omit_dims(self): for dtype in self._dtypes: - x_np = np.random.randn( - 2, 3, - 4).astype(dtype) + 1j * np.random.randn(2, 3, 4).astype(dtype) + x_np = np.random.randn(2, 3, 4).astype( + dtype + ) + 1j * np.random.randn(2, 3, 4).astype(dtype) shape = (0, -1) shape_ = (2, 12) for place in self._places: @@ -55,9 +54,9 @@ class TestComplexReshape(unittest.TestCase): x_var = dg.to_variable(x_np) y_var = paddle.reshape(x_var, shape) y_np = y_var.numpy() - np.testing.assert_allclose(np.reshape(x_np, shape_), - y_np, - rtol=1e-05) + np.testing.assert_allclose( + np.reshape(x_np, shape_), y_np, rtol=1e-05 + ) def test_eager(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_complex_simplenet.py b/python/paddle/fluid/tests/unittests/test_complex_simplenet.py index ead80c15890..125b2188898 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_simplenet.py +++ b/python/paddle/fluid/tests/unittests/test_complex_simplenet.py @@ -22,19 +22,20 @@ from paddle.fluid.framework import _test_eager_guard class Optimization_ex1(paddle.nn.Layer): - - def __init__(self, - shape, - param_attr=paddle.nn.initializer.Uniform(low=-5., high=5.), - dtype='float32'): + def __init__( + self, + shape, + param_attr=paddle.nn.initializer.Uniform(low=-5.0, high=5.0), + dtype='float32', + ): super(Optimization_ex1, self).__init__() - self.theta = self.create_parameter(shape=shape, - attr=param_attr, - dtype=dtype, - is_bias=False) + self.theta = self.create_parameter( + shape=shape, attr=param_attr, dtype=dtype, is_bias=False + ) self.A = paddle.to_tensor( - np.random.randn(4, 4) + np.random.randn(4, 4) * 1j) + np.random.randn(4, 4) + np.random.randn(4, 4) * 1j + ) def forward(self): loss = paddle.add(self.theta, self.A) @@ -42,7 +43,6 @@ class Optimization_ex1(paddle.nn.Layer): class TestComplexSimpleNet(unittest.TestCase): - def setUp(self): self.devices = ['cpu'] if core.is_compiled_with_cuda(): @@ -55,8 +55,9 @@ class TestComplexSimpleNet(unittest.TestCase): paddle.set_device(device) myLayer = Optimization_ex1(self.theta_size) - optimizer = paddle.optimizer.Adam(learning_rate=self.learning_rate, - parameters=myLayer.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=self.learning_rate, parameters=myLayer.parameters() + ) for itr in range(self.iter): loss = myLayer() diff --git a/python/paddle/fluid/tests/unittests/test_complex_sum_layer.py b/python/paddle/fluid/tests/unittests/test_complex_sum_layer.py index 8dc3b433cd5..7de44b33b0f 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_sum_layer.py +++ b/python/paddle/fluid/tests/unittests/test_complex_sum_layer.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexSumLayer(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -32,9 +31,9 @@ class TestComplexSumLayer(unittest.TestCase): def test_complex_basic_api(self): for dtype in self._dtypes: - input = rand([ - 2, 10, 10 - ]).astype(dtype) + 1j * rand([2, 10, 10]).astype(dtype) + input = rand([2, 10, 10]).astype(dtype) + 1j * rand( + [2, 10, 10] + ).astype(dtype) for place in self._places: with dg.guard(place): var_x = dg.to_variable(input) diff --git a/python/paddle/fluid/tests/unittests/test_complex_trace_layer.py b/python/paddle/fluid/tests/unittests/test_complex_trace_layer.py index c0400d6fc5d..bb8bc7f445c 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_trace_layer.py +++ b/python/paddle/fluid/tests/unittests/test_complex_trace_layer.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexTraceLayer(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [fluid.CPUPlace()] @@ -31,14 +30,15 @@ class TestComplexTraceLayer(unittest.TestCase): def test_basic_api(self): for dtype in self._dtypes: - input = rand([ - 2, 20, 2, 3 - ]).astype(dtype) + 1j * rand([2, 20, 2, 3]).astype(dtype) + input = rand([2, 20, 2, 3]).astype(dtype) + 1j * rand( + [2, 20, 2, 3] + ).astype(dtype) for place in self._places: with dg.guard(place): var_x = dg.to_variable(input) - result = tensor.trace(var_x, offset=1, axis1=0, - axis2=2).numpy() + result = tensor.trace( + var_x, offset=1, axis1=0, axis2=2 + ).numpy() target = np.trace(input, offset=1, axis1=0, axis2=2) np.testing.assert_allclose(result, target, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_complex_transpose.py b/python/paddle/fluid/tests/unittests/test_complex_transpose.py index cfb8040674f..b08f7e1be07 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_transpose.py +++ b/python/paddle/fluid/tests/unittests/test_complex_transpose.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexTransposeLayer(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -30,9 +29,9 @@ class TestComplexTransposeLayer(unittest.TestCase): def test_transpose_by_complex_api(self): for dtype in self._dtypes: - data = np.random.random( - (2, 3, 4, 5)).astype(dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(dtype) + data = np.random.random((2, 3, 4, 5)).astype( + dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(dtype) perm = [3, 2, 0, 1] np_trans = np.transpose(data, perm) for place in self._places: diff --git a/python/paddle/fluid/tests/unittests/test_complex_variable.py b/python/paddle/fluid/tests/unittests/test_complex_variable.py index c7173a7b814..b3afc1d9df9 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_variable.py +++ b/python/paddle/fluid/tests/unittests/test_complex_variable.py @@ -23,10 +23,10 @@ from paddle.fluid.framework import _test_eager_guard class TestComplexVariable(unittest.TestCase): - def compare(self): - a = np.array([[1.0 + 1.0j, 2.0 + 1.0j], - [3.0 + 1.0j, 4.0 + 1.0j]]).astype(self._dtype) + a = np.array( + [[1.0 + 1.0j, 2.0 + 1.0j], [3.0 + 1.0j, 4.0 + 1.0j]] + ).astype(self._dtype) b = np.array([[1.0 + 1.0j, 1.0 + 1.0j]]).astype(self._dtype) with dg.guard(): @@ -46,16 +46,22 @@ class TestComplexVariable(unittest.TestCase): self.compare() def test_convert_np_dtype_to_dtype(self): - self.assertEqual(convert_np_dtype_to_dtype_(np.complex64), - core.VarDesc.VarType.COMPLEX64) - self.assertEqual(convert_np_dtype_to_dtype_(np.complex64), - core.VarDesc.VarType.COMPLEX64) + self.assertEqual( + convert_np_dtype_to_dtype_(np.complex64), + core.VarDesc.VarType.COMPLEX64, + ) + self.assertEqual( + convert_np_dtype_to_dtype_(np.complex64), + core.VarDesc.VarType.COMPLEX64, + ) def test_convert_dtype(self): - self.assertEqual(convert_dtype(core.VarDesc.VarType.COMPLEX64), - "complex64") - self.assertEqual(convert_dtype(core.VarDesc.VarType.COMPLEX128), - "complex128") + self.assertEqual( + convert_dtype(core.VarDesc.VarType.COMPLEX64), "complex64" + ) + self.assertEqual( + convert_dtype(core.VarDesc.VarType.COMPLEX128), "complex128" + ) def test_eager(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_complex_view_op.py b/python/paddle/fluid/tests/unittests/test_complex_view_op.py index ad283a05839..451469dfa21 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_view_op.py +++ b/python/paddle/fluid/tests/unittests/test_complex_view_op.py @@ -34,14 +34,14 @@ def ref_view_as_real(x): class TestViewAsComplexOp(OpTest): - def setUp(self): self.op_type = "as_complex" self.python_api = paddle.as_complex x = np.random.randn(10, 10, 2).astype("float64") out_ref = ref_view_as_complex(x) - self.out_grad = np.ones( - [10, 10], dtype="float64") + 1j * np.ones([10, 10], dtype="float64") + self.out_grad = np.ones([10, 10], dtype="float64") + 1j * np.ones( + [10, 10], dtype="float64" + ) self.inputs = {'X': x} self.outputs = {'Out': out_ref} @@ -49,15 +49,16 @@ class TestViewAsComplexOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ref_view_as_real(self.out_grad)], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ref_view_as_real(self.out_grad)], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) class TestViewAsRealOp(OpTest): - def setUp(self): self.op_type = "as_real" real = np.random.randn(10, 10).astype("float64") @@ -73,15 +74,16 @@ class TestViewAsRealOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ref_view_as_complex(self.out_grad)], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ref_view_as_complex(self.out_grad)], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) class TestViewAsComplexAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randn(10, 10, 2) self.out = ref_view_as_complex(self.x) @@ -109,7 +111,6 @@ class TestViewAsComplexAPI(unittest.TestCase): class TestViewAsRealAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randn(10, 10) + 1j * np.random.randn(10, 10) self.out = ref_view_as_real(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_concat_op.py b/python/paddle/fluid/tests/unittests/test_concat_op.py index 76397a2083b..624f2be4a9f 100644 --- a/python/paddle/fluid/tests/unittests/test_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_concat_op.py @@ -14,7 +14,11 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + skip_check_grad_ci, + convert_float_to_uint16, +) import paddle.fluid as fluid from paddle.fluid import Program, core, program_guard from paddle.fluid.framework import _test_eager_guard @@ -25,7 +29,6 @@ import paddle.fluid.layers as layers class TestConcatOp(OpTest): - def setUp(self): self.op_type = "concat" self.python_api = paddle.concat @@ -40,8 +43,9 @@ class TestConcatOp(OpTest): self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } def get_dtype(self): @@ -81,7 +85,6 @@ class TestConcatOp(OpTest): class TestConcatOp2(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -90,9 +93,9 @@ class TestConcatOp2(TestConcatOp): @skip_check_grad_ci( - reason="The function 'check_grad' for large inputs is too slow.") + reason="The function 'check_grad' for large inputs is too slow." +) class TestConcatOp3(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((1, 256, 170, 256)).astype(self.dtype) self.x1 = np.random.random((1, 128, 170, 256)).astype(self.dtype) @@ -104,11 +107,9 @@ class TestConcatOp3(TestConcatOp): @skip_check_grad_ci( - reason= - "This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." + reason="This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." ) class TestConcatOp4(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -120,7 +121,6 @@ class TestConcatOp4(TestConcatOp): class TestConcatOp5(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((5, 1, 4, 5)).astype(self.dtype) self.x1 = np.random.random((5, 2, 4, 5)).astype(self.dtype) @@ -129,7 +129,6 @@ class TestConcatOp5(TestConcatOp): class TestConcatOp6(TestConcatOp): - def setUp(self): self.op_type = "concat" self.dtype = self.get_dtype() @@ -138,8 +137,11 @@ class TestConcatOp6(TestConcatOp): self.lod = [[20, 80]] self.out_lod = [[20, 80, 20, 80, 20, 80]] self.inputs = { - 'X': [('x0', (self.x0, self.lod)), ('x1', (self.x1, self.lod)), - ('x2', (self.x2, self.lod))] + 'X': [ + ('x0', (self.x0, self.lod)), + ('x1', (self.x1, self.lod)), + ('x2', (self.x2, self.lod)), + ] } self.attrs = {'axis': self.axis} if self.axis < 0: @@ -166,9 +168,7 @@ class TestConcatOp6(TestConcatOp): def create_test_AxisTensor(parent): - class TestConcatAxisTensor(parent): - def setUp(self): self.op_type = "concat" self.python_api = paddle.concat @@ -177,20 +177,22 @@ def create_test_AxisTensor(parent): self.inputs = { 'X': [('x0', self.x0), ('x1', self.x1), ('x2', self.x2)], - 'AxisTensor': np.array([self.axis]).astype("int32") + 'AxisTensor': np.array([self.axis]).astype("int32"), } self.attrs = {} if self.axis < 0: self.actual_axis = self.axis + len(self.x0.shape) - self.actual_axis = self.actual_axis if self.actual_axis > 0 else 0 + self.actual_axis = ( + self.actual_axis if self.actual_axis > 0 else 0 + ) else: self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), - axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } cls_name = "{0}_{1}".format(parent.__name__, "AxisTensor") @@ -205,13 +207,11 @@ create_test_AxisTensor(TestConcatOp4) create_test_AxisTensor(TestConcatOp5) create_test_AxisTensor(TestConcatOp6) -#----------------Concat Fp16---------------- +# ----------------Concat Fp16---------------- def create_test_fp16(parent): - class TestConcatFp16(parent): - def get_dtype(self): return np.float16 @@ -228,13 +228,12 @@ create_test_fp16(TestConcatOp5) create_test_fp16(TestConcatOp6) -#----------------Concat Bf16---------------- +# ----------------Concat Bf16---------------- def create_test_bf16(parent): - - @unittest.skipIf(not paddle.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestConcatBf16(parent): - def get_dtype(self): return np.uint16 @@ -247,17 +246,18 @@ create_test_bf16(TestConcatOp) class TestConcatOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of concat_op should be list. x1 = fluid.layers.data(shape=[4], dtype='int32', name='x1') fluid.layers.concat(x1) # The item in input must be Variable. - x2 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - x3 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x2 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + x3 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.concat, [x2]) # The input dtype of concat_op must be float16, float32, float64, int32, int64. x4 = fluid.layers.data(shape=[4], dtype='uint8', name='x4') @@ -281,7 +281,6 @@ class TestConcatOpError(unittest.TestCase): class TestConcatAPI(unittest.TestCase): - def test_fluid_api(self): paddle.enable_static() x_1 = fluid.data(shape=[None, 1, 4, 5], dtype='int32', name='x_1') @@ -298,22 +297,20 @@ class TestConcatAPI(unittest.TestCase): out_3 = fluid.layers.concat(input=[x_2, x_3], axis=positive_1_int64) exe = fluid.Executor(place=fluid.CPUPlace()) - [res_1, res_2, res_3] = exe.run(fluid.default_main_program(), - feed={ - "x_1": input_2, - "x_2": input_2, - "x_3": input_3 - }, - fetch_list=[out_1, out_2, out_3]) + [res_1, res_2, res_3] = exe.run( + fluid.default_main_program(), + feed={"x_1": input_2, "x_2": input_2, "x_3": input_3}, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_2, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_3, np.concatenate((input_2, input_3), axis=1)) def test_api(self): paddle.enable_static() - x_1 = paddle.fluid.data(shape=[None, 1, 4, 5], - dtype='int32', - name='x_1') + x_1 = paddle.fluid.data( + shape=[None, 1, 4, 5], dtype='int32', name='x_1' + ) paddle.concat([x_1, x_1], 0) input_2 = np.random.random([2, 1, 4, 5]).astype("int32") @@ -329,14 +326,11 @@ class TestConcatAPI(unittest.TestCase): out_4 = paddle.concat(x=[x_2, x_3], axis=negative_int64) exe = paddle.static.Executor(place=paddle.CPUPlace()) - [res_1, res_2, res_3, - res_4] = exe.run(paddle.static.default_main_program(), - feed={ - "x_1": input_2, - "x_2": input_2, - "x_3": input_3 - }, - fetch_list=[out_1, out_2, out_3, out_4]) + [res_1, res_2, res_3, res_4] = exe.run( + paddle.static.default_main_program(), + feed={"x_1": input_2, "x_2": input_2, "x_3": input_3}, + fetch_list=[out_1, out_2, out_3, out_4], + ) assert np.array_equal(res_1, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_2, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_3, np.concatenate((input_2, input_3), axis=1)) @@ -367,10 +361,12 @@ class TestConcatAPI(unittest.TestCase): def test_errors(self): with program_guard(Program(), Program()): # The item in input must be Variable. - x2 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - x3 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x2 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + x3 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.concat, [x2]) # The input dtype of concat_op must be float16, float32, float64, int32, int64. x4 = paddle.fluid.data(shape=[4], dtype='uint8', name='x4') @@ -404,8 +400,11 @@ class TestConcatAPIWithLoDTensorArray(unittest.TestCase): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = fluid.CUDAPlace(0) \ - if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def set_program(self, use_fluid_api): paddle.enable_static() @@ -414,9 +413,9 @@ class TestConcatAPIWithLoDTensorArray(unittest.TestCase): with fluid.program_guard(self.program): input = fluid.layers.assign(self.x) tensor_array = fluid.layers.create_array(dtype='float32') - zero = fluid.layers.fill_constant(shape=[1], - value=0, - dtype="int64") + zero = fluid.layers.fill_constant( + shape=[1], value=0, dtype="int64" + ) for i in range(self.iter_num): fluid.layers.array_write(input, zero + i, tensor_array) @@ -449,11 +448,11 @@ class TestConcatAPIWithLoDTensorArray(unittest.TestCase): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_array_equal( - res[0], np.concatenate([self.x] * self.iter_num, axis=self.axis)) + res[0], np.concatenate([self.x] * self.iter_num, axis=self.axis) + ) class TestConcatDoubleGradCheck(unittest.TestCase): - def concat_wrapper(self, x): return paddle.concat(x) @@ -470,17 +469,21 @@ class TestConcatDoubleGradCheck(unittest.TestCase): out = paddle.concat([data1, data2]) data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) data2_arr = np.random.uniform(-1, 1, data2.shape).astype(dtype) - gradient_checker.double_grad_check([data1, data2], - out, - x_init=[data1_arr, data2_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) gradient_checker.double_grad_check_for_dygraph( - self.concat_wrapper, [data1, data2], + self.concat_wrapper, + [data1, data2], out, x_init=[data1_arr, data2_arr], - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -492,7 +495,6 @@ class TestConcatDoubleGradCheck(unittest.TestCase): class TestConcatTripleGradCheck(unittest.TestCase): - def concat_wrapper(self, x): return paddle.concat(x, 1) @@ -509,17 +511,21 @@ class TestConcatTripleGradCheck(unittest.TestCase): out = paddle.concat([data1, data2], 1) data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) data2_arr = np.random.uniform(-1, 1, data2.shape).astype(dtype) - gradient_checker.double_grad_check([data1, data2], - out, - x_init=[data1_arr, data2_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) gradient_checker.double_grad_check_for_dygraph( - self.concat_wrapper, [data1, data2], + self.concat_wrapper, + [data1, data2], out, x_init=[data1_arr, data2_arr], - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_cond.py b/python/paddle/fluid/tests/unittests/test_cond.py index f0c5516eef4..3d05d7694fa 100644 --- a/python/paddle/fluid/tests/unittests/test_cond.py +++ b/python/paddle/fluid/tests/unittests/test_cond.py @@ -29,7 +29,6 @@ np.random.seed(123) class TestCondInputOutput(unittest.TestCase): - def test_return_single_var(self): """ pseudocode: @@ -57,13 +56,16 @@ class TestCondInputOutput(unittest.TestCase): out = layers.cond(pred, true_func, false_func) # out is one tensor - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - ret, = exe.run(main_program, fetch_list=[out.name]) - np.testing.assert_allclose(np.asarray(ret), - np.full((3, 2), -1, np.int32), - rtol=1e-05) + (ret,) = exe.run(main_program, fetch_list=[out.name]) + np.testing.assert_allclose( + np.asarray(ret), np.full((3, 2), -1, np.int32), rtol=1e-05 + ) def test_return_var_tuple(self): """ @@ -78,18 +80,14 @@ class TestCondInputOutput(unittest.TestCase): paddle.enable_static() def true_func(): - return layers.fill_constant(shape=[1, 2], dtype='int32', - value=1), layers.fill_constant( - shape=[2, 3], - dtype='bool', - value=True) + return layers.fill_constant( + shape=[1, 2], dtype='int32', value=1 + ), layers.fill_constant(shape=[2, 3], dtype='bool', value=True) def false_func(): - return layers.fill_constant(shape=[3, 4], dtype='float32', - value=3), layers.fill_constant( - shape=[4, 5], - dtype='int64', - value=2) + return layers.fill_constant( + shape=[3, 4], dtype='float32', value=3 + ), layers.fill_constant(shape=[4, 5], dtype='int64', value=2) main_program = Program() startup_program = Program() @@ -98,16 +96,19 @@ class TestCondInputOutput(unittest.TestCase): out = layers.cond(pred, true_func, false_func) # out is a tuple containing 2 tensors - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=out) - np.testing.assert_allclose(np.asarray(ret[0]), - np.full((1, 2), 1, np.int32), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(ret[1]), - np.full((2, 3), True, bool), - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(ret[0]), np.full((1, 2), 1, np.int32), rtol=1e-05 + ) + np.testing.assert_allclose( + np.asarray(ret[1]), np.full((2, 3), True, bool), rtol=1e-05 + ) def test_pass_and_modify_var(self): """ @@ -135,20 +136,28 @@ class TestCondInputOutput(unittest.TestCase): with program_guard(main_program, startup_program): a = layers.fill_constant(shape=[3, 2, 1], dtype='int32', value=7) i = fluid.data(name="i", shape=[1], dtype='int32') - pred = ((i % 2) == 0) - a = layers.cond(pred, lambda: true_func(a, i), - lambda: false_func(a, i)) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + pred = (i % 2) == 0 + a = layers.cond( + pred, lambda: true_func(a, i), lambda: false_func(a, i) + ) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) for feed_i in range(5): expected_a = 7 * (feed_i + 1) if feed_i % 2 == 0 else 8 - feed_i - ret, = exe.run(main_program, - feed={'i': np.full((1), feed_i, np.int32)}, - fetch_list=[a]) - np.testing.assert_allclose(np.asarray(ret), - np.full((3, 2, 1), expected_a, np.int32), - rtol=1e-05) + (ret,) = exe.run( + main_program, + feed={'i': np.full((1), feed_i, np.int32)}, + fetch_list=[a], + ) + np.testing.assert_allclose( + np.asarray(ret), + np.full((3, 2, 1), expected_a, np.int32), + rtol=1e-05, + ) def test_return_none(self): """ @@ -172,12 +181,15 @@ class TestCondInputOutput(unittest.TestCase): startup_program = Program() with program_guard(main_program, startup_program): i = fluid.data(name="i", shape=[1], dtype='int32') - pred = ((i % 2) == 0) + pred = (i % 2) == 0 out1 = layers.cond(pred, true_func, false_func) out2 = layers.cond(pred, None, false_func) out3 = layers.cond(pred, true_func, None) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) for feed_i in range(5): # Test that output is None is runnable @@ -200,17 +212,15 @@ class TestCondInputOutput(unittest.TestCase): return layers.fill_constant(shape=[2, 7], dtype='int32', value=3) def func_return_two_tensors(): - return layers.fill_constant(shape=[3, 1], dtype='int32', - value=7), layers.fill_constant( - shape=[3, 1], - dtype='int32', - value=8) + return layers.fill_constant( + shape=[3, 1], dtype='int32', value=7 + ), layers.fill_constant(shape=[3, 1], dtype='int32', value=8) main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): i = fluid.data(name="i", shape=[1], dtype='int32') - pred = ((i % 2) == 0) + pred = (i % 2) == 0 with self.assertRaises(TypeError): out = layers.cond(pred, i, func_return_one_tensor) @@ -218,47 +228,57 @@ class TestCondInputOutput(unittest.TestCase): out = layers.cond(pred, func_return_one_tensor, np.asarray([3])) with self.assertRaises(Exception) as e: - out = layers.cond(pred, func_return_none, - func_return_one_tensor) + out = layers.cond( + pred, func_return_none, func_return_one_tensor + ) self.assertTrue( - "Incompatible return values of true_fn and false_fn in cond" in - str(e.exception)) + "Incompatible return values of true_fn and false_fn in cond" + in str(e.exception) + ) with self.assertRaises(Exception) as e: - out = layers.cond(pred, func_return_two_tensors, - func_return_none) + out = layers.cond( + pred, func_return_two_tensors, func_return_none + ) self.assertTrue( - "Incompatible return values of true_fn and false_fn in cond" in - str(e.exception)) + "Incompatible return values of true_fn and false_fn in cond" + in str(e.exception) + ) with self.assertRaises(Exception) as e: - out = layers.cond(pred, func_return_one_tensor, - func_return_two_tensors) + out = layers.cond( + pred, func_return_one_tensor, func_return_two_tensors + ) self.assertTrue( "true fn returns 1 vars, but false fn returns 2 vars, which is not equals" - in str(e.exception)) + in str(e.exception) + ) def test_extremely_simple_net_with_op_in_condition(self): paddle.enable_static() main_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - a = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=1.23) + a = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=1.23 + ) a.stop_gradient = False - b = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=1.25) + b = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=1.25 + ) b.stop_gradient = False out = layers.cond(a - b < -1.0, lambda: a, lambda: b) append_backward(out) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - ret = exe.run(main_program, - fetch_list=[out, b, a.grad_name, b.grad_name]) + ret = exe.run( + main_program, fetch_list=[out, b, a.grad_name, b.grad_name] + ) # Note: fill_constant has loss of precision, you have to assertEqual # with values doens't lose precision in float-point number. self.assertEqual(ret[0][0], ret[1][0]) @@ -267,7 +287,6 @@ class TestCondInputOutput(unittest.TestCase): class TestCondNestedControlFlow(unittest.TestCase): - def test_cond_inside_cond(self): """ pseudocode: @@ -288,25 +307,37 @@ class TestCondNestedControlFlow(unittest.TestCase): paddle.enable_static() def less_than_branch(i, a): - return layers.cond(i >= 3.0, lambda: layers.elementwise_add(a, a), - lambda: layers.elementwise_sub(a, a)) + return layers.cond( + i >= 3.0, + lambda: layers.elementwise_add(a, a), + lambda: layers.elementwise_sub(a, a), + ) def greater_equal_branch(i, a): - return layers.cond(i < 8.0, lambda: layers.elementwise_mul(a, a), - lambda: layers.elementwise_div(a, a)) + return layers.cond( + i < 8.0, + lambda: layers.elementwise_mul(a, a), + lambda: layers.elementwise_div(a, a), + ) main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): i = fluid.data(name="i", shape=[1], dtype='float32') a = 2.0 * i - out = layers.cond(i < 5.0, lambda: less_than_branch(i, a), - lambda: greater_equal_branch(i, a)) + out = layers.cond( + i < 5.0, + lambda: less_than_branch(i, a), + lambda: greater_equal_branch(i, a), + ) mean = paddle.mean(out) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) for feed_i in range(0, 10): expected_a = 2.0 * feed_i @@ -316,9 +347,11 @@ class TestCondNestedControlFlow(unittest.TestCase): else: expected_ret = expected_a * expected_a if feed_i < 8 else 1.0 expected_a_grad = 2.0 * expected_a if feed_i < 8 else 0.0 - ret = exe.run(main_program, - feed={'i': np.full((1), feed_i, np.float32)}, - fetch_list=[out.name, a.grad_name]) + ret = exe.run( + main_program, + feed={'i': np.full((1), feed_i, np.float32)}, + fetch_list=[out.name, a.grad_name], + ) self.assertEqual(ret[0][0], expected_ret) self.assertEqual(ret[1][0], expected_a_grad) @@ -328,24 +361,34 @@ class TestCondNestedControlFlow(unittest.TestCase): startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - a = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=1.23) + a = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=1.23 + ) a.stop_gradient = False - b = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=1.24) + b = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=1.24 + ) b.stop_gradient = False out = fluid.layers.cond( - a < b, lambda: fluid.layers.cond( - a - b < -1.0, lambda: fluid.layers.elementwise_add(a, b), - lambda: fluid.layers.elementwise_mul(a, b)), lambda: - fluid.layers.cond(a == b, lambda: fluid.layers.elementwise_sub( - a, b), lambda: fluid.layers.elementwise_pow(a, b))) + a < b, + lambda: fluid.layers.cond( + a - b < -1.0, + lambda: fluid.layers.elementwise_add(a, b), + lambda: fluid.layers.elementwise_mul(a, b), + ), + lambda: fluid.layers.cond( + a == b, + lambda: fluid.layers.elementwise_sub(a, b), + lambda: fluid.layers.elementwise_pow(a, b), + ), + ) append_backward(out) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=[out, a.grad_name, b.grad_name]) # Note: fill_constant has loss of precision, so we assertAlmostEqual. @@ -355,7 +398,6 @@ class TestCondNestedControlFlow(unittest.TestCase): class TestCondBackward(unittest.TestCase): - def backward_value_helper(self, cond_func, use_cuda, use_parallel_exe): """ Helper function that compares calculated backward value is close to dy/dx @@ -379,70 +421,76 @@ class TestCondBackward(unittest.TestCase): num_devices = 1 if use_parallel_exe: os.environ['CPU_NUM'] = str(2) - exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=main_program, - loss_name=loss.name) + exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=main_program, + loss_name=loss.name, + ) num_devices = exe.device_count delta = 0.005 for feed_i in range(0, 10): feed_img = np.random.random(size=[1, 9]).astype(np.float32) - feed_label = np.random.randint(low=0, - high=10, - size=[1, 1], - dtype=np.int64) + feed_label = np.random.randint( + low=0, high=10, size=[1, 1], dtype=np.int64 + ) if use_parallel_exe: img_grad, loss_value = exe.run( feed={ 'i': np.full((num_devices), feed_i, np.int32), 'image': np.repeat(feed_img, num_devices, axis=0), - 'label': np.repeat(feed_label, num_devices, axis=0) + 'label': np.repeat(feed_label, num_devices, axis=0), }, - fetch_list=[img.grad_name, loss.name]) + fetch_list=[img.grad_name, loss.name], + ) else: img_grad, loss_value = exe.run( main_program, feed={ 'i': np.full((1), feed_i, np.int32), 'image': feed_img, - 'label': feed_label + 'label': feed_label, }, - fetch_list=[img.grad_name, loss.name]) + fetch_list=[img.grad_name, loss.name], + ) numerical_grad = np.zeros(shape=[num_devices, 9], dtype=np.float32) feed_img_delta = np.copy(feed_img) for j in range(9): feed_img_delta[0][j] = feed_img[0][j] + delta if use_parallel_exe: - loss_delta = exe.run(feed={ - 'i': - np.full((num_devices), feed_i, np.int32), - 'image': - np.repeat(feed_img_delta, num_devices, axis=0), - 'label': - np.repeat(feed_label, num_devices, axis=0) - }, - fetch_list=[loss.name]) - multi_device_grad = (loss_delta[0] - - loss_value[0]) / delta / num_devices + loss_delta = exe.run( + feed={ + 'i': np.full((num_devices), feed_i, np.int32), + 'image': np.repeat( + feed_img_delta, num_devices, axis=0 + ), + 'label': np.repeat(feed_label, num_devices, axis=0), + }, + fetch_list=[loss.name], + ) + multi_device_grad = ( + (loss_delta[0] - loss_value[0]) / delta / num_devices + ) for d in range(num_devices): numerical_grad[d][j] = multi_device_grad[d] else: - loss_delta = exe.run(main_program, - feed={ - 'i': np.full((1), feed_i, - np.int32), - 'image': feed_img_delta, - 'label': feed_label - }, - fetch_list=[loss.name]) - numerical_grad[0][j] = (loss_delta[0] - - loss_value[0]) / delta + loss_delta = exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img_delta, + 'label': feed_label, + }, + fetch_list=[loss.name], + ) + numerical_grad[0][j] = ( + loss_delta[0] - loss_value[0] + ) / delta feed_img_delta[0][j] = feed_img[0][j] - np.testing.assert_allclose(img_grad, - numerical_grad, - rtol=0.05, - atol=0.05) + np.testing.assert_allclose( + img_grad, numerical_grad, rtol=0.05, atol=0.05 + ) def add_optimizer_helper(self, cond_func, use_cuda, use_parallel_exe): """ @@ -463,43 +511,49 @@ class TestCondBackward(unittest.TestCase): exe.run(startup_program) if use_parallel_exe: os.environ['CPU_NUM'] = str(2) - exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=main_program, - loss_name=loss.name) + exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=main_program, + loss_name=loss.name, + ) num_devices = exe.device_count for feed_i in range(0, 10): feed_img = np.random.random(size=[16, 784]).astype(np.float32) - feed_label = np.random.randint(low=0, - high=10, - size=[16, 1], - dtype=np.int64) + feed_label = np.random.randint( + low=0, high=10, size=[16, 1], dtype=np.int64 + ) if use_parallel_exe: - exe.run(feed={ - 'i': np.full((num_devices), feed_i, np.int32), - 'image': np.repeat(feed_img, num_devices, axis=0), - 'label': np.repeat(feed_label, num_devices, axis=0) - }, - fetch_list=[loss.name]) + exe.run( + feed={ + 'i': np.full((num_devices), feed_i, np.int32), + 'image': np.repeat(feed_img, num_devices, axis=0), + 'label': np.repeat(feed_label, num_devices, axis=0), + }, + fetch_list=[loss.name], + ) else: - exe.run(main_program, - feed={ - 'i': np.full((1), feed_i, np.int32), - 'image': feed_img, - 'label': feed_label - }, - fetch_list=[loss]) + exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img, + 'label': feed_label, + }, + fetch_list=[loss], + ) def test_cond_backward(self): paddle.enable_static() def cond_func(i, img, label): - predicate = ((i % 2) == 0) + predicate = (i % 2) == 0 return layers.cond( predicate, lambda: simple_fc_net_with_inputs(img, label, class_num=10), - lambda: batchnorm_fc_with_inputs(img, label, class_num=10)) + lambda: batchnorm_fc_with_inputs(img, label, class_num=10), + ) for use_parallel_exe in [False, True]: if use_parallel_exe and os.name == "nt": @@ -508,10 +562,12 @@ class TestCondBackward(unittest.TestCase): ) continue - self.backward_value_helper(cond_func, core.is_compiled_with_cuda(), - use_parallel_exe) - self.add_optimizer_helper(cond_func, core.is_compiled_with_cuda(), - use_parallel_exe) + self.backward_value_helper( + cond_func, core.is_compiled_with_cuda(), use_parallel_exe + ) + self.add_optimizer_helper( + cond_func, core.is_compiled_with_cuda(), use_parallel_exe + ) def test_half_nested_cond_backward(self): paddle.enable_static() @@ -520,15 +576,18 @@ class TestCondBackward(unittest.TestCase): return layers.cond( (i % 2) == 0, lambda: simple_fc_net_with_inputs(img, label, class_num=10), - lambda: batchnorm_fc_with_inputs(img, label, class_num=10)) + lambda: batchnorm_fc_with_inputs(img, label, class_num=10), + ) def cond_func_simple_net_at_true(i, img, label): - return layers.cond(i < 5, lambda: branch(i, img, label), - lambda: paddle.mean(img)) + return layers.cond( + i < 5, lambda: branch(i, img, label), lambda: paddle.mean(img) + ) def cond_func_simple_net_at_false(i, img, label): - return layers.cond(i < 5, lambda: paddle.mean(img), - lambda: branch(i, img, label)) + return layers.cond( + i < 5, lambda: paddle.mean(img), lambda: branch(i, img, label) + ) for use_parallel_exe in [False, True]: if use_parallel_exe and os.name == "nt": @@ -537,35 +596,47 @@ class TestCondBackward(unittest.TestCase): ) continue - self.backward_value_helper(cond_func_simple_net_at_true, - core.is_compiled_with_cuda(), - use_parallel_exe) - self.add_optimizer_helper(cond_func_simple_net_at_true, - core.is_compiled_with_cuda(), - use_parallel_exe) - self.backward_value_helper(cond_func_simple_net_at_false, - core.is_compiled_with_cuda(), - use_parallel_exe) - self.add_optimizer_helper(cond_func_simple_net_at_false, - core.is_compiled_with_cuda(), - use_parallel_exe) + self.backward_value_helper( + cond_func_simple_net_at_true, + core.is_compiled_with_cuda(), + use_parallel_exe, + ) + self.add_optimizer_helper( + cond_func_simple_net_at_true, + core.is_compiled_with_cuda(), + use_parallel_exe, + ) + self.backward_value_helper( + cond_func_simple_net_at_false, + core.is_compiled_with_cuda(), + use_parallel_exe, + ) + self.add_optimizer_helper( + cond_func_simple_net_at_false, + core.is_compiled_with_cuda(), + use_parallel_exe, + ) def test_nested_cond_backward(self): paddle.enable_static() def branch(i, img, label, mod_two): if mod_two: - predicate = ((i % 2) == 0) + predicate = (i % 2) == 0 else: - predicate = ((i % 2) != 0) + predicate = (i % 2) != 0 return layers.cond( predicate, lambda: simple_fc_net_with_inputs(img, label, class_num=10), - lambda: batchnorm_fc_with_inputs(img, label, class_num=10)) + lambda: batchnorm_fc_with_inputs(img, label, class_num=10), + ) def cond_func(i, img, label): - return layers.cond(i < 5, lambda: branch(i, img, label, True), - lambda: branch(i, img, label, False)) + return layers.cond( + i < 5, + lambda: branch(i, img, label, True), + lambda: branch(i, img, label, False), + ) for use_parallel_exe in [False, True]: if use_parallel_exe and os.name == "nt": @@ -573,14 +644,15 @@ class TestCondBackward(unittest.TestCase): "Skip use_parallel_exe=True in Windows because of flaky test when using PE under old Windows machine" ) continue - self.backward_value_helper(cond_func, core.is_compiled_with_cuda(), - use_parallel_exe) - self.add_optimizer_helper(cond_func, core.is_compiled_with_cuda(), - use_parallel_exe) + self.backward_value_helper( + cond_func, core.is_compiled_with_cuda(), use_parallel_exe + ) + self.add_optimizer_helper( + cond_func, core.is_compiled_with_cuda(), use_parallel_exe + ) class TestCondWithError(unittest.TestCase): - def test_input_type_error(self): paddle.enable_static() main_program = framework.Program() diff --git a/python/paddle/fluid/tests/unittests/test_conditional_block.py b/python/paddle/fluid/tests/unittests/test_conditional_block.py index 6c0a290794c..fc8d926070e 100644 --- a/python/paddle/fluid/tests/unittests/test_conditional_block.py +++ b/python/paddle/fluid/tests/unittests/test_conditional_block.py @@ -24,7 +24,6 @@ from paddle.fluid.layers.control_flow import ConditionalBlock class ConditionalBlockTest(unittest.TestCase): - def test_forward(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -50,12 +49,12 @@ class ConditionalBlockTest(unittest.TestCase): outs = exe.run( main_program, feed={'X': x}, - fetch_list=[main_program.block(0).var(data.name + "@GRAD")])[0] + fetch_list=[main_program.block(0).var(data.name + "@GRAD")], + )[0] print(outs) class TestConditionalBlockOpInferShape(unittest.TestCase): - def test_infer_shape(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -64,24 +63,21 @@ class TestConditionalBlockOpInferShape(unittest.TestCase): sub_block = main_program._create_block() main_program._rollback() step_scope = global_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) - cond_var = layers.fill_constant(shape=[1], - dtype='bool', - value=False) + type=core.VarDesc.VarType.STEP_SCOPES + ) + cond_var = layers.fill_constant( + shape=[1], dtype='bool', value=False + ) - op = global_block.append_op(type='conditional_block', - inputs={ - 'Cond': [cond_var], - 'Input': [], - }, - outputs={ - 'Out': [], - 'Scope': [step_scope] - }, - attrs={ - 'sub_block': sub_block, - 'is_scalar_condition': True - }) + op = global_block.append_op( + type='conditional_block', + inputs={ + 'Cond': [cond_var], + 'Input': [], + }, + outputs={'Out': [], 'Scope': [step_scope]}, + attrs={'sub_block': sub_block, 'is_scalar_condition': True}, + ) op.desc.infer_shape(global_block.desc) diff --git a/python/paddle/fluid/tests/unittests/test_conj_op.py b/python/paddle/fluid/tests/unittests/test_conj_op.py index 577d8afbef0..67647458f5d 100644 --- a/python/paddle/fluid/tests/unittests/test_conj_op.py +++ b/python/paddle/fluid/tests/unittests/test_conj_op.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestConjOp(OpTest): - def setUp(self): self.op_type = "conj" self.python_api = paddle.tensor.conj @@ -39,31 +38,34 @@ class TestConjOp(OpTest): self.dtype = np.complex64 def init_input_output(self): - x = (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(self.dtype) + x = ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(self.dtype) out = np.conj(x) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(x)} self.outputs = {'Out': out} def init_grad_input_output(self): - self.grad_out = (np.ones((12, 14)) + 1j * np.ones( - (12, 14))).astype(self.dtype) + self.grad_out = (np.ones((12, 14)) + 1j * np.ones((12, 14))).astype( + self.dtype + ) self.grad_in = np.conj(self.grad_out) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_in], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_in], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestComplexConjOp(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -72,9 +74,9 @@ class TestComplexConjOp(unittest.TestCase): def test_conj_api(self): for dtype in self._dtypes: - input = rand([ - 2, 20, 2, 3 - ]).astype(dtype) + 1j * rand([2, 20, 2, 3]).astype(dtype) + input = rand([2, 20, 2, 3]).astype(dtype) + 1j * rand( + [2, 20, 2, 3] + ).astype(dtype) for place in self._places: with dg.guard(place): var_x = paddle.to_tensor(input) @@ -84,9 +86,9 @@ class TestComplexConjOp(unittest.TestCase): def test_conj_operator(self): for dtype in self._dtypes: - input = rand([ - 2, 20, 2, 3 - ]).astype(dtype) + 1j * rand([2, 20, 2, 3]).astype(dtype) + input = rand([2, 20, 2, 3]).astype(dtype) + 1j * rand( + [2, 20, 2, 3] + ).astype(dtype) for place in self._places: with dg.guard(place): var_x = paddle.to_tensor(input) @@ -95,21 +97,22 @@ class TestComplexConjOp(unittest.TestCase): np.testing.assert_array_equal(result, target) def test_conj_static_mode(self): - def init_input_output(dtype): - input = rand([ - 2, 20, 2, 3 - ]).astype(dtype) + 1j * rand([2, 20, 2, 3]).astype(dtype) + input = rand([2, 20, 2, 3]).astype(dtype) + 1j * rand( + [2, 20, 2, 3] + ).astype(dtype) return {'x': input}, np.conj(input) for dtype in self._dtypes: input_dict, np_res = init_input_output(dtype) for place in self._places: with static.program_guard(static.Program()): - x_dtype = np.complex64 if dtype == "float32" else np.complex128 - x = static.data(name="x", - shape=[2, 20, 2, 3], - dtype=x_dtype) + x_dtype = ( + np.complex64 if dtype == "float32" else np.complex128 + ) + x = static.data( + name="x", shape=[2, 20, 2, 3], dtype=x_dtype + ) out = paddle.conj(x) exe = static.Executor(place) diff --git a/python/paddle/fluid/tests/unittests/test_const_value.py b/python/paddle/fluid/tests/unittests/test_const_value.py index e346f24ba68..58ac6fa0a9a 100644 --- a/python/paddle/fluid/tests/unittests/test_const_value.py +++ b/python/paddle/fluid/tests/unittests/test_const_value.py @@ -17,7 +17,6 @@ import paddle.fluid.framework as framework class ConstantTest(unittest.TestCase): - def test_const_value(self): self.assertEqual(framework.GRAD_VAR_SUFFIX, "@GRAD") self.assertEqual(framework.TEMP_VAR_NAME, "@TEMP@") diff --git a/python/paddle/fluid/tests/unittests/test_context_manager.py b/python/paddle/fluid/tests/unittests/test_context_manager.py index 93c1d7fa1a4..9ae5630b0af 100644 --- a/python/paddle/fluid/tests/unittests/test_context_manager.py +++ b/python/paddle/fluid/tests/unittests/test_context_manager.py @@ -19,7 +19,6 @@ import unittest class TestContextManagerRaiseException(unittest.TestCase): # When exception raised in 'with' context, we should safely exit the context def test_func1(self): - def foo(): with fluid.dygraph.guard(): print("raise error in context manager") diff --git a/python/paddle/fluid/tests/unittests/test_conv1d_layer.py b/python/paddle/fluid/tests/unittests/test_conv1d_layer.py index 0c1ffc6ddbe..bc035321844 100644 --- a/python/paddle/fluid/tests/unittests/test_conv1d_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv1d_layer.py @@ -21,22 +21,23 @@ import unittest class Conv1DTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=(16, ), - num_channels=6, - num_filters=8, - filter_size=3, - padding=0, - padding_mode="zeros", - stride=1, - dilation=1, - groups=1, - no_bias=False, - dtype="float32", - data_format="NCL"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=(16,), + num_channels=6, + num_filters=8, + filter_size=3, + padding=0, + padding_mode="zeros", + stride=1, + dilation=1, + groups=1, + no_bias=False, + dtype="float32", + data_format="NCL", + ): super(Conv1DTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -44,7 +45,7 @@ class Conv1DTestCase(unittest.TestCase): self.spartial_shape = spartial_shape self.filter_size = filter_size self.data_format = data_format - self.channel_last = (self.data_format == "NLC") + self.channel_last = self.data_format == "NLC" self.padding = padding self.padding_mode = padding_mode @@ -55,23 +56,28 @@ class Conv1DTestCase(unittest.TestCase): self.dtype = dtype def setUp(self): - input_shape = (self.batch_size, self.num_channels - ) + self.spartial_shape if not self.channel_last else ( - self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size, self.num_channels) + self.spartial_shape + if not self.channel_last + else (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_filters, self.num_channels - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_filters, + self.num_channels // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) else: self.bias = None @@ -80,42 +86,49 @@ class Conv1DTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, self.num_channels, - -1) if not self.channel_last else ( - -1, -1, self.num_channels) + input_shape = ( + (-1, self.num_channels, -1) + if not self.channel_last + else (-1, -1, self.num_channels) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) - y_var = F.conv1d(x_var, - w_var, - b_var if not self.no_bias else None, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) + y_var = F.conv1d( + x_var, + w_var, + b_var if not self.no_bias else None, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = paddle.to_tensor(self.input) - conv = nn.Conv1D(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - padding_mode=self.padding_mode, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv1D( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + padding_mode=self.padding_mode, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -139,7 +152,6 @@ class Conv1DTestCase(unittest.TestCase): class Conv1DErrorTestCase(Conv1DTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -148,7 +160,6 @@ class Conv1DErrorTestCase(Conv1DTestCase): class Conv1DTypeErrorTestCase(Conv1DTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -161,58 +172,75 @@ def add_cases(suite): suite.addTest(Conv1DTestCase(methodName='runTest', stride=[1], dilation=2)) suite.addTest(Conv1DTestCase(methodName='runTest', stride=2, dilation=(1))) suite.addTest( - Conv1DTestCase(methodName='runTest', padding="same", no_bias=True)) + Conv1DTestCase(methodName='runTest', padding="same", no_bias=True) + ) suite.addTest( - Conv1DTestCase(methodName='runTest', filter_size=3, padding='valid')) + Conv1DTestCase(methodName='runTest', filter_size=3, padding='valid') + ) suite.addTest( - Conv1DTestCase(methodName='runTest', num_filters=512, padding='valid')) + Conv1DTestCase(methodName='runTest', num_filters=512, padding='valid') + ) suite.addTest( - Conv1DTestCase(methodName='runTest', num_filters=512, padding=[1, 2])) + Conv1DTestCase(methodName='runTest', num_filters=512, padding=[1, 2]) + ) suite.addTest( - Conv1DTestCase(methodName='runTest', padding=2, data_format='NLC')) + Conv1DTestCase(methodName='runTest', padding=2, data_format='NLC') + ) suite.addTest(Conv1DTestCase(methodName='runTest', padding=[1])) suite.addTest(Conv1DTestCase(methodName='runTest', padding=[1, 2])) suite.addTest( - Conv1DTestCase(methodName='runTest', padding=[1, 2], data_format='NLC')) + Conv1DTestCase(methodName='runTest', padding=[1, 2], data_format='NLC') + ) suite.addTest(Conv1DTestCase(methodName='runTest', padding=2)) suite.addTest(Conv1DTestCase(methodName='runTest')) suite.addTest( - Conv1DTestCase(methodName='runTest', groups=2, padding="valid")) + Conv1DTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv1DTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid", - data_format='NLC')) + Conv1DTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + data_format='NLC', + ) + ) def add_error_cases(suite): suite.addTest( - Conv1DTypeErrorTestCase(methodName='runTest', - padding_mode="reflect", - padding="valid")) - suite.addTest(Conv1DErrorTestCase(methodName='runTest', - data_format="VALID")) + Conv1DTypeErrorTestCase( + methodName='runTest', padding_mode="reflect", padding="valid" + ) + ) + suite.addTest( + Conv1DErrorTestCase(methodName='runTest', data_format="VALID") + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', padding_mode="VALID")) + Conv1DErrorTestCase(methodName='runTest', padding_mode="VALID") + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', num_channels=5, groups=2)) + Conv1DErrorTestCase(methodName='runTest', num_channels=5, groups=2) + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', - num_filters=8, - num_channels=15, - groups=3)) + Conv1DErrorTestCase( + methodName='runTest', num_filters=8, num_channels=15, groups=3 + ) + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', padding=[1, 2, 3, 4, 5])) + Conv1DErrorTestCase(methodName='runTest', padding=[1, 2, 3, 4, 5]) + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', - padding=[1, 2, 3, 4, 5], - data_format='NLC')) + Conv1DErrorTestCase( + methodName='runTest', padding=[1, 2, 3, 4, 5], data_format='NLC' + ) + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', - num_filters=512, - padding=[1, 2, 3, 4, 5])) + Conv1DErrorTestCase( + methodName='runTest', num_filters=512, padding=[1, 2, 3, 4, 5] + ) + ) suite.addTest(Conv1DErrorTestCase(methodName='runTest', dilation=-10)) diff --git a/python/paddle/fluid/tests/unittests/test_conv1d_transpose_layer.py b/python/paddle/fluid/tests/unittests/test_conv1d_transpose_layer.py index fbd65ebbf7c..ec8b7ee6ef7 100644 --- a/python/paddle/fluid/tests/unittests/test_conv1d_transpose_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv1d_transpose_layer.py @@ -21,23 +21,24 @@ import unittest class Conv1DTransposeTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=16, - in_channels=6, - out_channels=8, - filter_size=3, - output_size=None, - padding=0, - output_padding=0, - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCL", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=16, + in_channels=6, + out_channels=8, + filter_size=3, + output_size=None, + padding=0, + output_padding=0, + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCL", + dtype="float32", + ): super(Conv1DTransposeTestCase, self).__init__(methodName) self.batch_size = batch_size self.in_channels = in_channels @@ -58,25 +59,32 @@ class Conv1DTransposeTestCase(unittest.TestCase): def setUp(self): self.channel_last = False if self.data_format == "NCL" else True - input_shape = (self.batch_size, self.in_channels, - self.spartial_shape) if not self.channel_last else ( - self.batch_size, - self.spartial_shape, - self.in_channels, - ) + input_shape = ( + (self.batch_size, self.in_channels, self.spartial_shape) + if not self.channel_last + else ( + self.batch_size, + self.spartial_shape, + self.in_channels, + ) + ) self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.in_channels, self.out_channels - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.in_channels, + self.out_channels // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.out_channels, )).astype(self.dtype) + -1, 1, size=(self.out_channels,) + ).astype(self.dtype) else: self.bias = None @@ -85,44 +93,51 @@ class Conv1DTransposeTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, self.in_channels, - -1) if not self.channel_last else ( - -1, -1, self.in_channels) + input_shape = ( + (-1, self.in_channels, -1) + if not self.channel_last + else (-1, -1, self.in_channels) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.out_channels, ), - dtype=self.dtype) - y_var = F.conv1d_transpose(x_var, - w_var, - None if self.no_bias else b_var, - output_size=self.output_size, - padding=self.padding, - output_padding=self.output_padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.out_channels,), dtype=self.dtype + ) + y_var = F.conv1d_transpose( + x_var, + w_var, + None if self.no_bias else b_var, + output_size=self.output_size, + padding=self.padding, + output_padding=self.output_padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = paddle.to_tensor(self.input) - conv = nn.Conv1DTranspose(self.in_channels, - self.out_channels, - self.filter_size, - padding=self.padding, - output_padding=self.output_padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv1DTranspose( + self.in_channels, + self.out_channels, + self.filter_size, + padding=self.padding, + output_padding=self.output_padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -146,7 +161,6 @@ class Conv1DTransposeTestCase(unittest.TestCase): class Conv1DTransposeErrorTestCase(Conv1DTransposeTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -157,64 +171,84 @@ class Conv1DTransposeErrorTestCase(Conv1DTransposeTestCase): def add_cases(suite): suite.addTest(Conv1DTransposeTestCase(methodName='runTest')) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - stride=[2], - no_bias=True, - dilation=2)) + Conv1DTransposeTestCase( + methodName='runTest', stride=[2], no_bias=True, dilation=2 + ) + ) + suite.addTest( + Conv1DTransposeTestCase( + methodName='runTest', + filter_size=(3), + output_size=[36], + stride=[2], + dilation=2, + ) + ) + suite.addTest( + Conv1DTransposeTestCase(methodName='runTest', stride=2, dilation=(2)) + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - filter_size=(3), - output_size=[36], - stride=[2], - dilation=2)) + Conv1DTransposeTestCase(methodName='runTest', padding="valid") + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', stride=2, dilation=(2))) - suite.addTest(Conv1DTransposeTestCase(methodName='runTest', - padding="valid")) - suite.addTest(Conv1DTransposeTestCase(methodName='runTest', - padding='valid')) + Conv1DTransposeTestCase(methodName='runTest', padding='valid') + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', filter_size=1, padding=3)) + Conv1DTransposeTestCase(methodName='runTest', filter_size=1, padding=3) + ) suite.addTest(Conv1DTransposeTestCase(methodName='runTest', padding=[2])) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', data_format="NLC")) + Conv1DTransposeTestCase(methodName='runTest', data_format="NLC") + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', groups=2, - padding="valid")) + Conv1DTransposeTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - out_channels=6, - in_channels=3, - groups=3, - padding="valid")) + Conv1DTransposeTestCase( + methodName='runTest', + out_channels=6, + in_channels=3, + groups=3, + padding="valid", + ) + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - data_format="NLC", - spartial_shape=16, - output_size=18)) + Conv1DTransposeTestCase( + methodName='runTest', + data_format="NLC", + spartial_shape=16, + output_size=18, + ) + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - data_format="NLC", - stride=3, - output_padding=2)) + Conv1DTransposeTestCase( + methodName='runTest', data_format="NLC", stride=3, output_padding=2 + ) + ) suite.addTest(Conv1DTransposeTestCase(methodName='runTest', padding=[1, 2])) def add_error_cases(suite): suite.addTest( - Conv1DTransposeErrorTestCase(methodName='runTest', - data_format="not_valid")) + Conv1DTransposeErrorTestCase( + methodName='runTest', data_format="not_valid" + ) + ) suite.addTest( - Conv1DTransposeErrorTestCase(methodName='runTest', - in_channels=5, - groups=2)) + Conv1DTransposeErrorTestCase( + methodName='runTest', in_channels=5, groups=2 + ) + ) suite.addTest( - Conv1DTransposeErrorTestCase(methodName='runTest', - stride=2, - output_padding=3)) + Conv1DTransposeErrorTestCase( + methodName='runTest', stride=2, output_padding=3 + ) + ) suite.addTest( - Conv1DTransposeErrorTestCase(methodName='runTest', - output_size="not_valid")) + Conv1DTransposeErrorTestCase( + methodName='runTest', output_size="not_valid" + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_api.py b/python/paddle/fluid/tests/unittests/test_conv2d_api.py index d7b9339c39c..b95bf27cdc8 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_api.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_api.py @@ -23,251 +23,295 @@ import paddle.fluid as fluid class TestConv2DAPI(unittest.TestCase): - def test_api(self): - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") - - fluid.layers.conv2d(input=input_NHWC, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - data_format="NCHW") - - fluid.layers.conv2d(input=input_NCHW, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[1, 2, 1, 0], - dilation=[1, 1], - groups=1, - data_format="NCHW") - - fluid.layers.conv2d(input=input_NCHW, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[[0, 0], [0, 0], [1, 1], [1, 1]], - dilation=[1, 1], - groups=1, - data_format="NCHW") - - fluid.layers.conv2d(input=input_NHWC, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[[0, 0], [1, 1], [1, 1], [0, 0]], - dilation=[1, 1], - groups=1, - data_format="NHWC") - - fluid.layers.conv2d(input=input_NCHW, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding="SAME", - dilation=[1, 1], - groups=1, - data_format="NCHW") - - fluid.layers.conv2d(input=input_NCHW, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding="VALID", - dilation=[1, 1], - groups=1, - data_format="NCHW") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) + + fluid.layers.conv2d( + input=input_NHWC, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) + + fluid.layers.conv2d( + input=input_NCHW, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[1, 2, 1, 0], + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) + + fluid.layers.conv2d( + input=input_NCHW, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[[0, 0], [0, 0], [1, 1], [1, 1]], + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) + + fluid.layers.conv2d( + input=input_NHWC, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + dilation=[1, 1], + groups=1, + data_format="NHWC", + ) + + fluid.layers.conv2d( + input=input_NCHW, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding="SAME", + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) + + fluid.layers.conv2d( + input=input_NCHW, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding="VALID", + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) def test_depthwise_conv2d(self): - x_var = paddle.uniform((2, 8, 8, 4), dtype='float32', min=-1., max=1.) - conv = paddle.nn.Conv2D(in_channels=4, - out_channels=4, - kernel_size=(3, 3), - groups=4, - data_format='NHWC') + x_var = paddle.uniform((2, 8, 8, 4), dtype='float32', min=-1.0, max=1.0) + conv = paddle.nn.Conv2D( + in_channels=4, + out_channels=4, + kernel_size=(3, 3), + groups=4, + data_format='NHWC', + ) y_var = conv(x_var) class TestConv2DAPI_Error(unittest.TestCase): - def test_api(self): - input = fluid.layers.data(name="input", - shape=[2, 5, 5, 5], - append_batch_size=False, - dtype="float32") + input = fluid.layers.data( + name="input", + shape=[2, 5, 5, 5], + append_batch_size=False, + dtype="float32", + ) # ValueError: cudnn def run_1(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - use_cudnn=[0], - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + use_cudnn=[0], + data_format="NCHW", + ) self.assertRaises(ValueError, run_1) # ValueError: data_format def run_2(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NCHWC") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NCHWC", + ) self.assertRaises(ValueError, run_2) # ValueError: padding def run_3(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding="SAMEE", - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding="SAMEE", + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_3) def run_4(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[[0, 1], [0, 1], [0, 1], [0, 1]], - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[[0, 1], [0, 1], [0, 1], [0, 1]], + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_4) def run_5(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[[0, 1], [0, 1], [0, 1], [0, 1]], - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NHWC") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[[0, 1], [0, 1], [0, 1], [0, 1]], + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_5) # ValueError: channel dimmention - x = fluid.layers.data(name="x", - shape=[2, 5, 5, -1], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name="x", + shape=[2, 5, 5, -1], + append_batch_size=False, + dtype="float32", + ) def run_6(): - fluid.layers.conv2d(input=x, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NHWC") + fluid.layers.conv2d( + input=x, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_6) # ValueError: groups def run_7(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=3, - use_cudnn=False, - data_format="NHWC") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=3, + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_7) # ValueError: filter num def run_8(): - fluid.layers.conv2d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=1, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=1, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_8) # ValueError: groups def run_9(): - fluid.layers.conv2d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=0, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=0, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_9) # ValueError: stride def run_10(): - fluid.layers.conv2d(input=input, - num_filters=1, - filter_size=1, - stride=0, - padding=0, - dilation=0, - groups=1, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=1, + filter_size=1, + stride=0, + padding=0, + dilation=0, + groups=1, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_10) def test_api_with_error_input(self): - input = fluid.layers.data(name="error_input", - shape=[1], - append_batch_size=False, - dtype="float32") + input = fluid.layers.data( + name="error_input", + shape=[1], + append_batch_size=False, + dtype="float32", + ) # ValueError: cudnn def run_1(): - fluid.layers.conv2d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=0, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=0, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_1) @@ -275,36 +319,44 @@ class TestConv2DAPI_Error(unittest.TestCase): # --------- test environment variable ------ @unittest.skipIf( not (core.is_compiled_with_cuda() or core.is_compiled_with_rocm()), - "core is not compiled with CUDA or ROCM") + "core is not compiled with CUDA or ROCM", +) class TestConv2DEnviron(unittest.TestCase): - def run1(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - inputs = fluid.layers.data(shape=[2, 3, 5, 5], - append_batch_size=False, - name="inputs", - dtype="float32") - result = fluid.layers.conv2d(input=inputs, - num_filters=4, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - data_format="NCHW") + inputs = fluid.layers.data( + shape=[2, 3, 5, 5], + append_batch_size=False, + name="inputs", + dtype="float32", + ) + result = fluid.layers.conv2d( + input=inputs, + num_filters=4, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fetches = exe.run(fluid.default_main_program(), - feed={"inputs": self.input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"inputs": self.input_np}, + fetch_list=[result], + ) def run2(self, place): with fluid.dygraph.guard(place): inputs = fluid.dygraph.to_variable(self.input_np) - conv = paddle.nn.Conv2D(in_channels=3, - out_channels=4, - kernel_size=(3, 3), - data_format="NCHW") + conv = paddle.nn.Conv2D( + in_channels=3, + out_channels=4, + kernel_size=(3, 3), + data_format="NCHW", + ) result = conv(inputs) def run3(self, place): diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_fusion_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_fusion_op.py index f92c0c7090e..7e4a0133157 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_fusion_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_fusion_op.py @@ -22,9 +22,7 @@ from test_conv2d_op import conv2d_forward_naive def create_test_padding_SAME_class(parent): - class TestPaddingSAMECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -35,9 +33,7 @@ def create_test_padding_SAME_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -48,7 +44,6 @@ def create_test_padding_VALID_class(parent): class TestConv2DFusionOp(OpTest): - def setUp(self): self.op_type = "conv2d_fusion" self.exhaustive_search = False @@ -71,32 +66,37 @@ class TestConv2DFusionOp(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) bias = np.random.random(self.filter_size[0]).astype(self.dtype) - self.output, _, _, _, _ = conv2d_forward_naive(input, filter, - self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + self.output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) self.output = self.output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), - 'Bias': OpTest.np_dtype_to_fluid_dtype(bias) + 'Bias': OpTest.np_dtype_to_fluid_dtype(bias), } if self.add_residual_data: residual_data = np.random.random(self.output.shape).astype( - self.dtype) + self.dtype + ) self.inputs['ResidualData'] = OpTest.np_dtype_to_fluid_dtype( - residual_data) + residual_data + ) self.output += residual_data # Add bias @@ -114,7 +114,7 @@ class TestConv2DFusionOp(OpTest): 'data_format': self.data_format, 'exhaustive_search': self.exhaustive_search, 'activation': self.activation, - 'padding_algorithm': self.padding_algorithm + 'padding_algorithm': self.padding_algorithm, } if self.split_channels is not None: self.attrs['split_channels'] = self.split_channels @@ -163,32 +163,27 @@ class TestConv2DFusionOp(OpTest): class TestWithoutResidual(TestConv2DFusionOp): - def init_residual(self): self.add_residual_data = False class TestIdentityActivation(TestConv2DFusionOp): - def init_activation(self): self.activation = 'identity' class TestIdentityActivation1(TestConv2DFusionOp): - def init_activation(self): self.activation = 'identity' self.add_residual_data = False class TestWithGroup(TestConv2DFusionOp): - def init_group(self): self.groups = 3 class TestWithDilation(TestConv2DFusionOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -205,13 +200,11 @@ class TestWithDilation(TestConv2DFusionOp): class TestCUDNNExhaustiveSearch(TestConv2DFusionOp): - def set_search_method(self): self.exhaustive_search = True class TestMultipleOutputs(TestConv2DFusionOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -228,14 +221,12 @@ class TestMultipleOutputs(TestConv2DFusionOp): class TestAsyPadding(TestConv2DFusionOp): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -249,7 +240,6 @@ class TestWithPad_AsyPadding(TestConv2DFusionOp): class TestWithStride_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -263,7 +253,6 @@ class TestWithStride_AsyPadding(TestConv2DFusionOp): class TestWith1x1_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -280,13 +269,11 @@ class TestWith1x1_AsyPadding(TestConv2DFusionOp): class TestWithGroup_AsyPadding(TestConv2DFusionOp): - def init_group(self): self.groups = 3 class TestWithDepthWise3x3_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [3, 4, 10, 10] # NCHW @@ -306,7 +293,6 @@ class TestWithDepthWise3x3_AsyPadding(TestConv2DFusionOp): class TestWithDepthWise5x5_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 4, 10, 10] # NCHW @@ -323,7 +309,6 @@ class TestWithDepthWise5x5_AsyPadding(TestConv2DFusionOp): class TestWithDepthWise7x7_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 8, 10, 10] # NCHW @@ -340,7 +325,6 @@ class TestWithDepthWise7x7_AsyPadding(TestConv2DFusionOp): class TestWithDilation_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -360,7 +344,6 @@ class TestWithDilation_AsyPadding(TestConv2DFusionOp): class TestWithInput1x1Filter1x1_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 1, 1] # NCHW diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_layer.py b/python/paddle/fluid/tests/unittests/test_conv2d_layer.py index 8782ee84a26..22e6f0a5634 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_layer.py @@ -27,22 +27,23 @@ def _reverse_repeat_list(t, n): class Conv2DTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=(16, 16), - num_channels=6, - num_filters=8, - filter_size=3, - padding=0, - padding_mode='zeros', - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCHW", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=(16, 16), + num_channels=6, + num_filters=8, + filter_size=3, + padding=0, + padding_mode='zeros', + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCHW", + dtype="float32", + ): super(Conv2DTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -53,9 +54,11 @@ class Conv2DTestCase(unittest.TestCase): self.padding = padding if padding_mode in {'reflect', 'replicate', 'circular'}: _paired_padding = fluid.layers.utils.convert_to_list( - padding, 2, 'padding') + padding, 2, 'padding' + ) self._reversed_padding_repeated_twice = _reverse_repeat_list( - _paired_padding, 2) + _paired_padding, 2 + ) self.padding_mode = padding_mode self.stride = stride self.dilation = dilation @@ -67,24 +70,31 @@ class Conv2DTestCase(unittest.TestCase): def setUp(self): self.channel_last = self.data_format == "NHWC" if self.channel_last: - input_shape = (self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) else: - input_shape = (self.batch_size, - self.num_channels) + self.spartial_shape + input_shape = ( + self.batch_size, + self.num_channels, + ) + self.spartial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] * 2 else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_filters, self.num_channels - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_filters, + self.num_channels // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) else: self.bias = None @@ -93,8 +103,11 @@ class Conv2DTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1,self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1) + input_shape = ( + (-1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) weight_attr = I.NumpyArrayInitializer(self.weight) if self.bias is None: @@ -102,29 +115,33 @@ class Conv2DTestCase(unittest.TestCase): else: bias_attr = I.NumpyArrayInitializer(self.bias) if self.padding_mode != 'zeros': - x_var = F.pad(x_var, - self._reversed_padding_repeated_twice, - mode=self.padding_mode, - data_format=self.data_format) + x_var = F.pad( + x_var, + self._reversed_padding_repeated_twice, + mode=self.padding_mode, + data_format=self.data_format, + ) padding = 0 else: padding = self.padding - y_var = fluid.layers.conv2d(x_var, - self.num_filters, - self.filter_size, - padding=padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - param_attr=weight_attr, - bias_attr=bias_attr, - data_format=self.data_format) + y_var = fluid.layers.conv2d( + x_var, + self.num_filters, + self.filter_size, + padding=padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + param_attr=weight_attr, + bias_attr=bias_attr, + data_format=self.data_format, + ) feed_dict = {"input": self.input} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -132,52 +149,62 @@ class Conv2DTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1,self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1) + input_shape = ( + (-1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) if self.padding_mode != 'zeros': - x_var = F.pad(x_var, - self._reversed_padding_repeated_twice, - mode=self.padding_mode, - data_format=self.data_format) + x_var = F.pad( + x_var, + self._reversed_padding_repeated_twice, + mode=self.padding_mode, + data_format=self.data_format, + ) padding = 0 else: padding = self.padding - y_var = F.conv2d(x_var, - w_var, - b_var if not self.no_bias else None, - padding=padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y_var = F.conv2d( + x_var, + w_var, + b_var if not self.no_bias else None, + padding=padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = paddle.to_tensor(self.input) x_var.stop_gradient = False - conv = nn.Conv2D(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - padding_mode=self.padding_mode, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv2D( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + padding_mode=self.padding_mode, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -210,7 +237,6 @@ class Conv2DTestCase(unittest.TestCase): class Conv2DErrorTestCase(Conv2DTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -221,63 +247,86 @@ class Conv2DErrorTestCase(Conv2DTestCase): def add_cases(suite): suite.addTest(Conv2DTestCase(methodName='runTest')) suite.addTest( - Conv2DTestCase(methodName='runTest', stride=[1, 2], dilation=2)) + Conv2DTestCase(methodName='runTest', stride=[1, 2], dilation=2) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', stride=2, dilation=(2, 1))) + Conv2DTestCase(methodName='runTest', stride=2, dilation=(2, 1)) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', padding="same", no_bias=True)) + Conv2DTestCase(methodName='runTest', padding="same", no_bias=True) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - filter_size=(3, 3), - padding='valid')) + Conv2DTestCase( + methodName='runTest', filter_size=(3, 3), padding='valid' + ) + ) suite.addTest(Conv2DTestCase(methodName='runTest', padding=(2, 3))) suite.addTest(Conv2DTestCase(methodName='runTest', padding=[1, 2, 2, 1])) suite.addTest( - Conv2DTestCase(methodName='runTest', - padding=[[0, 0], [0, 0], [1, 2], [2, 1]])) + Conv2DTestCase( + methodName='runTest', padding=[[0, 0], [0, 0], [1, 2], [2, 1]] + ) + ) suite.addTest(Conv2DTestCase(methodName='runTest', data_format="NHWC")) suite.addTest( - Conv2DTestCase(methodName='runTest', - data_format="NHWC", - padding=[[0, 0], [1, 1], [2, 2], [0, 0]])) + Conv2DTestCase( + methodName='runTest', + data_format="NHWC", + padding=[[0, 0], [1, 1], [2, 2], [0, 0]], + ) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', groups=2, padding="valid")) + Conv2DTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid")) + Conv2DTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + ) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - filter_size=(3, 3), - padding=1, - padding_mode='reflect')) + Conv2DTestCase( + methodName='runTest', + filter_size=(3, 3), + padding=1, + padding_mode='reflect', + ) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - filter_size=(3, 3), - padding=1, - padding_mode='replicate')) + Conv2DTestCase( + methodName='runTest', + filter_size=(3, 3), + padding=1, + padding_mode='replicate', + ) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - filter_size=(3, 3), - padding=1, - padding_mode='circular')) + Conv2DTestCase( + methodName='runTest', + filter_size=(3, 3), + padding=1, + padding_mode='circular', + ) + ) def add_error_cases(suite): suite.addTest( - Conv2DErrorTestCase(methodName='runTest', num_channels=5, groups=2)) + Conv2DErrorTestCase(methodName='runTest', num_channels=5, groups=2) + ) suite.addTest( - Conv2DErrorTestCase(methodName='runTest', - num_channels=5, - groups=2, - stride=0)) + Conv2DErrorTestCase( + methodName='runTest', num_channels=5, groups=2, stride=0 + ) + ) suite.addTest( - Conv2DErrorTestCase(methodName='runTest', - num_channels=5, - groups=2, - padding=[-1, -1])) + Conv2DErrorTestCase( + methodName='runTest', num_channels=5, groups=2, padding=[-1, -1] + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_op.py index 056a7c52efb..34a34f062ef 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_op.py @@ -19,29 +19,36 @@ import paddle import paddle import paddle.fluid.core as core import paddle.fluid as fluid -from paddle.fluid.tests.unittests.op_test import (OpTest, - convert_float_to_uint16, - get_numeric_gradient) +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + get_numeric_gradient, +) from paddle.fluid.tests.unittests.testsuite import create_op from paddle.fluid import Program, program_guard -def conv2d_forward_naive(input, - filter, - group, - conv_param, - padding_algorithm='EXPLICIT', - data_format='NCHW'): +def conv2d_forward_naive( + input, + filter, + group, + conv_param, + padding_algorithm='EXPLICIT', + data_format='NCHW', +): if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Unknown Attr(data_format): '%s' ." - "It can only be 'NCHW' or 'NHWC'." % str(data_format)) + raise ValueError( + "Unknown Attr(data_format): '%s' ." + "It can only be 'NCHW' or 'NHWC'." % str(data_format) + ) - channel_last = (data_format == "NHWC") + channel_last = data_format == "NHWC" if channel_last: input = np.transpose(input, [0, 3, 1, 2]) @@ -54,17 +61,22 @@ def conv2d_forward_naive(input, sub_out_c = out_c // group sub_f_n = f_n // group - stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ - 'dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -84,39 +96,52 @@ def conv2d_forward_naive(input, if len(pad) == 4: pad_h_0, pad_h_1 = pad[0], pad[1] pad_w_0, pad_w_1 = pad[2], pad[3] - out_h = 1 + (in_h + pad_h_0 + pad_h_1 - (dilation[0] * - (f_h - 1) + 1)) // stride[0] - out_w = 1 + (in_w + pad_w_0 + pad_w_1 - (dilation[1] * - (f_w - 1) + 1)) // stride[1] + out_h = ( + 1 + + (in_h + pad_h_0 + pad_h_1 - (dilation[0] * (f_h - 1) + 1)) + // stride[0] + ) + out_w = ( + 1 + + (in_w + pad_w_0 + pad_w_1 - (dilation[1] * (f_w - 1) + 1)) + // stride[1] + ) out = np.zeros((out_n, out_c, out_h, out_w)) - d_bolck_h = (dilation[0] * (f_h - 1) + 1) - d_bolck_w = (dilation[1] * (f_w - 1) + 1) + d_bolck_h = dilation[0] * (f_h - 1) + 1 + d_bolck_w = dilation[1] * (f_w - 1) + 1 - input_pad = np.pad(input, - ((0, 0), (0, 0), (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), - mode='constant', - constant_values=0) + input_pad = np.pad( + input, + ((0, 0), (0, 0), (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), + mode='constant', + constant_values=0, + ) filter_dilation = np.zeros((f_n, f_c, d_bolck_h, d_bolck_w)) - filter_dilation[:, :, 0:d_bolck_h:dilation[0], - 0:d_bolck_w:dilation[1]] = filter + filter_dilation[ + :, :, 0 : d_bolck_h : dilation[0], 0 : d_bolck_w : dilation[1] + ] = filter for i in range(out_h): for j in range(out_w): for g in range(group): - input_pad_masked = \ - input_pad[:, g * f_c:(g + 1) * f_c, - i * stride[0]:i * stride[0] + d_bolck_h, - j * stride[1]:j * stride[1] + d_bolck_w] - - f_sub = filter_dilation[g * sub_f_n:(g + 1) * sub_f_n, :, :, :] + input_pad_masked = input_pad[ + :, + g * f_c : (g + 1) * f_c, + i * stride[0] : i * stride[0] + d_bolck_h, + j * stride[1] : j * stride[1] + d_bolck_w, + ] + + f_sub = filter_dilation[ + g * sub_f_n : (g + 1) * sub_f_n, :, :, : + ] # sub_f_n == sub_out_c for k in range(sub_out_c): # Multiplication of Corresponding Elements, then sum all - out[:, g * sub_out_c + k, i, j] = \ - np.sum(input_pad_masked * f_sub[k, :, :, :], - axis=(1, 2, 3)) + out[:, g * sub_out_c + k, i, j] = np.sum( + input_pad_masked * f_sub[k, :, :, :], axis=(1, 2, 3) + ) if channel_last: out = np.transpose(out, [0, 2, 3, 1]) @@ -125,15 +150,15 @@ def conv2d_forward_naive(input, def create_test_cudnn_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNN") TestCUDNNCase.__name__ = cls_name @@ -141,11 +166,10 @@ def create_test_cudnn_class(parent): def create_test_cudnn_fp16_class(parent, grad_check=True): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestConv2DCUDNNFp16(parent): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -159,16 +183,16 @@ def create_test_cudnn_fp16_class(parent, grad_check=True): def test_check_grad_no_filter(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and grad_check: - self.check_grad_with_place(place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad_no_input(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and grad_check: - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNNFp16") TestConv2DCUDNNFp16.__name__ = cls_name @@ -176,20 +200,21 @@ def create_test_cudnn_fp16_class(parent, grad_check=True): def create_test_cudnn_bf16_class(parent): - @unittest.skipIf( not core.is_compiled_with_cuda() or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA and do not support bfloat16") + "core is not compiled with CUDA and do not support bfloat16", + ) class TestConv2DCUDNNBF16(parent): - def get_numeric_grad(self, place, check_name): scope = core.Scope() self._check_grad_helper() - op = create_op(scope, self.op_type, self.inputs, self.outputs, - self.attrs) - return get_numeric_gradient(place, scope, op, self.inputs_fp32, - check_name, ['Output']) + op = create_op( + scope, self.op_type, self.inputs, self.outputs, self.attrs + ) + return get_numeric_gradient( + place, scope, op, self.inputs_fp32, check_name, ['Output'] + ) def init_kernel_type(self): self.use_cudnn = True @@ -203,18 +228,24 @@ def create_test_cudnn_bf16_class(parent): def test_check_grad_no_filter(self): place = core.CUDAPlace(0) numeric_grads = self.get_numeric_grad(place, 'Input') - self.check_grad_with_place(place, ['Input'], - 'Output', - no_grad_set=set(['Filter']), - user_defined_grads=[numeric_grads]) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + no_grad_set=set(['Filter']), + user_defined_grads=[numeric_grads], + ) def test_check_grad_no_input(self): place = core.CUDAPlace(0) numeric_grads = self.get_numeric_grad(place, 'Filter') - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - user_defined_grads=[numeric_grads]) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + user_defined_grads=[numeric_grads], + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNNBF16") TestConv2DCUDNNBF16.__name__ = cls_name @@ -222,9 +253,7 @@ def create_test_cudnn_bf16_class(parent): def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -238,15 +267,15 @@ def create_test_channel_last_class(parent): def create_test_cudnn_channel_last_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCudnnChannelLastCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_data_format(self): self.data_format = "NHWC" @@ -261,11 +290,10 @@ def create_test_cudnn_channel_last_class(parent): def create_test_cudnn_channel_last_fp16_class(parent, grad_check=True): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCudnnChannelLastFp16(parent): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -279,16 +307,16 @@ def create_test_cudnn_channel_last_fp16_class(parent, grad_check=True): def test_check_grad_no_filter(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and grad_check: - self.check_grad_with_place(place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad_no_input(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and grad_check: - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def init_data_format(self): self.data_format = "NHWC" @@ -303,9 +331,7 @@ def create_test_cudnn_channel_last_fp16_class(parent, grad_check=True): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -316,9 +342,7 @@ def create_test_padding_SAME_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -329,15 +353,15 @@ def create_test_padding_VALID_class(parent): def create_test_cudnn_padding_SAME_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingSMAECase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_paddings(self): self.pad = [1, 1] @@ -349,15 +373,15 @@ def create_test_cudnn_padding_SAME_class(parent): def create_test_cudnn_padding_VALID_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingVALIDCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_paddings(self): self.pad = [1, 1] @@ -369,7 +393,6 @@ def create_test_cudnn_padding_VALID_class(parent): class TestConv2DOp(OpTest): - def setUp(self): self.op_type = "conv2d" self.use_cudnn = False @@ -387,17 +410,19 @@ class TestConv2DOp(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } if self.is_bfloat16_op(): input = np.random.random(self.input_size).astype(np.float32) - filter = np.random.uniform(-1, 1, - self.filter_size).astype(np.float32) + filter = np.random.uniform(-1, 1, self.filter_size).astype( + np.float32 + ) else: input = np.random.random(self.input_size).astype(self.dtype) - filter = np.random.uniform(-1, 1, - self.filter_size).astype(self.dtype) + filter = np.random.uniform(-1, 1, self.filter_size).astype( + self.dtype + ) if not self.has_cuda(): self.fuse_relu_before_depthwise_conv = False @@ -409,24 +434,25 @@ class TestConv2DOp(OpTest): else: input2 = input - output, _, _, _, _ = conv2d_forward_naive(input2, filter, self.groups, - conv2d_param) + output, _, _, _, _ = conv2d_forward_naive( + input2, filter, self.groups, conv2d_param + ) if self.is_bfloat16_op(): output = output.astype(np.float32) self.inputs = { 'Input': convert_float_to_uint16(input), - 'Filter': convert_float_to_uint16(filter) + 'Filter': convert_float_to_uint16(filter), } self.inputs_fp32 = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } else: output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { @@ -437,56 +463,71 @@ class TestConv2DOp(OpTest): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu_before_depthwise_conv': - self.fuse_relu_before_depthwise_conv, - 'exhaustive_search': self.exhaustive_search + 'fuse_relu_before_depthwise_conv': self.fuse_relu_before_depthwise_conv, + 'exhaustive_search': self.exhaustive_search, } self.outputs = {'Output': output} def has_cuda(self): - return core.is_compiled_with_cuda() and (self.use_cudnn - or self.use_cuda) + return core.is_compiled_with_cuda() and ( + self.use_cudnn or self.use_cuda + ) def test_check_output(self): place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(place, - atol=1e-5, - check_dygraph=(self.use_mkldnn == False)) + self.check_output_with_place( + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad(self): - if self.dtype == np.float16 or (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if self.dtype == np.float16 or ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.02, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.02, + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_filter(self): - if self.dtype == np.float16 or (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if self.dtype == np.float16 or ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter']), + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_input(self): - if self.dtype == np.float16 or (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if self.dtype == np.float16 or ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + check_dygraph=(self.use_mkldnn == False), + ) def init_test_case(self): self.pad = [0, 0] @@ -510,7 +551,6 @@ class TestConv2DOp(OpTest): class TestWithPad(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -521,7 +561,6 @@ class TestWithPad(TestConv2DOp): class TestWithStride(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -532,7 +571,6 @@ class TestWithStride(TestConv2DOp): class TestWithGroup(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -544,7 +582,6 @@ class TestWithGroup(TestConv2DOp): class TestWith1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -558,7 +595,6 @@ class TestWith1x1(TestConv2DOp): class TestWithDepthWise3x3(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -575,7 +611,6 @@ class TestWithDepthWise3x3(TestConv2DOp): class TestWithDepthWise5x5(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -589,7 +624,6 @@ class TestWithDepthWise5x5(TestConv2DOp): class TestWithDepthWise7x7(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -603,7 +637,6 @@ class TestWithDepthWise7x7(TestConv2DOp): class TestWithDilation(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -620,7 +653,6 @@ class TestWithDilation(TestConv2DOp): class TestWithInput1x1Filter1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -642,7 +674,7 @@ create_test_cudnn_class(TestWithGroup) create_test_cudnn_class(TestWith1x1) create_test_cudnn_class(TestWithInput1x1Filter1x1) -#----------------Conv2DCUDNN fp16---------------- +# ----------------Conv2DCUDNN fp16---------------- create_test_cudnn_fp16_class(TestConv2DOp, grad_check=False) create_test_cudnn_fp16_class(TestWithPad, grad_check=False) @@ -651,7 +683,7 @@ create_test_cudnn_fp16_class(TestWithGroup, grad_check=False) create_test_cudnn_fp16_class(TestWith1x1, grad_check=False) create_test_cudnn_fp16_class(TestWithInput1x1Filter1x1, grad_check=False) -#----------------Conv2DCUDNN bf16---------------- +# ----------------Conv2DCUDNN bf16---------------- create_test_cudnn_bf16_class(TestConv2DOp) create_test_cudnn_bf16_class(TestWithPad) @@ -662,7 +694,6 @@ create_test_cudnn_bf16_class(TestWithInput1x1Filter1x1) class TestCUDNNExhaustiveSearch(TestConv2DOp): - def init_kernel_type(self): self.use_cudnn = True self.exhaustive_search = True @@ -670,14 +701,14 @@ class TestCUDNNExhaustiveSearch(TestConv2DOp): class TestConv2DOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of conv2d must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) fluid.layers.conv2d(x1, 1, 1) self.assertRaises(TypeError, test_Variable) @@ -685,9 +716,9 @@ class TestConv2DOpError(unittest.TestCase): def test_dtype(): # the input dtype of conv2d must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.conv2d(x2, 1, 1) self.assertRaises(TypeError, test_dtype) @@ -703,7 +734,6 @@ class TestConv2DOpError(unittest.TestCase): class TestConv2DOp_v2(OpTest): - def setUp(self): self.op_type = "conv2d" self.use_cudnn = False @@ -723,7 +753,7 @@ class TestConv2DOp_v2(OpTest): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -737,15 +767,19 @@ class TestConv2DOp_v2(OpTest): else: input2 = input filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input2, filter, self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input2, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -756,53 +790,62 @@ class TestConv2DOp_v2(OpTest): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu_before_depthwise_conv': - self.fuse_relu_before_depthwise_conv, - 'exhaustive_search': self.exhaustive_search + 'fuse_relu_before_depthwise_conv': self.fuse_relu_before_depthwise_conv, + 'exhaustive_search': self.exhaustive_search, } self.outputs = {'Output': output} def has_cuda(self): - return core.is_compiled_with_cuda() and (self.use_cudnn - or self.use_cuda) + return core.is_compiled_with_cuda() and ( + self.use_cudnn or self.use_cuda + ) def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() - self.check_output_with_place(place, - atol=1e-5, - check_dygraph=(self.use_mkldnn == False)) + self.check_output_with_place( + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() - self.check_grad_with_place(place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.02, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.02, + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_filter(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter']), + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_input(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + check_dygraph=(self.use_mkldnn == False), + ) def init_test_case(self): self.pad = [0, 0] @@ -833,14 +876,12 @@ class TestConv2DOp_v2(OpTest): class TestConv2DOp_AsyPadding(TestConv2DOp_v2): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -854,7 +895,6 @@ class TestWithPad_AsyPadding(TestConv2DOp_v2): class TestWithStride_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -868,7 +908,6 @@ class TestWithStride_AsyPadding(TestConv2DOp_v2): class TestWithGroup_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 2] @@ -880,7 +919,6 @@ class TestWithGroup_AsyPadding(TestConv2DOp_v2): class TestWith1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -897,7 +935,6 @@ class TestWith1x1_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [3, 4, 10, 10] # NCHW @@ -917,7 +954,6 @@ class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 4, 10, 10] # NCHW @@ -934,7 +970,6 @@ class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 8, 10, 10] # NCHW @@ -951,7 +986,6 @@ class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): class TestWithDilation_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -971,7 +1005,6 @@ class TestWithDilation_AsyPadding(TestConv2DOp_v2): class TestWithInput1x1Filter1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [40, 3, 1, 1] # NCHW @@ -994,7 +1027,7 @@ create_test_cudnn_class(TestWithGroup_AsyPadding) create_test_cudnn_class(TestWith1x1_AsyPadding) create_test_cudnn_class(TestWithInput1x1Filter1x1_AsyPadding) -#---------- test SAME VALID ----------- +# ---------- test SAME VALID ----------- create_test_padding_SAME_class(TestConv2DOp_AsyPadding) create_test_padding_SAME_class(TestWithPad_AsyPadding) create_test_padding_SAME_class(TestWithStride_AsyPadding) @@ -1032,16 +1065,21 @@ create_test_cudnn_channel_last_class(TestWithStride_AsyPadding) create_test_cudnn_channel_last_class(TestWithGroup_AsyPadding) create_test_cudnn_channel_last_class(TestWithDilation_AsyPadding) -create_test_cudnn_channel_last_fp16_class(TestConv2DOp_AsyPadding, - grad_check=False) -create_test_cudnn_channel_last_fp16_class(TestWithPad_AsyPadding, - grad_check=False) -create_test_cudnn_channel_last_fp16_class(TestWithStride_AsyPadding, - grad_check=False) -create_test_cudnn_channel_last_fp16_class(TestWithGroup_AsyPadding, - grad_check=False) -create_test_cudnn_channel_last_fp16_class(TestWithDilation_AsyPadding, - grad_check=False) +create_test_cudnn_channel_last_fp16_class( + TestConv2DOp_AsyPadding, grad_check=False +) +create_test_cudnn_channel_last_fp16_class( + TestWithPad_AsyPadding, grad_check=False +) +create_test_cudnn_channel_last_fp16_class( + TestWithStride_AsyPadding, grad_check=False +) +create_test_cudnn_channel_last_fp16_class( + TestWithGroup_AsyPadding, grad_check=False +) +create_test_cudnn_channel_last_fp16_class( + TestWithDilation_AsyPadding, grad_check=False +) if __name__ == '__main__': paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_op_depthwise_conv.py b/python/paddle/fluid/tests/unittests/test_conv2d_op_depthwise_conv.py index 73f9b9e6ced..d6abdc0bcf4 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_op_depthwise_conv.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_op_depthwise_conv.py @@ -19,13 +19,20 @@ import paddle paddle.enable_static() import paddle.fluid.core as core -from test_conv2d_op import TestConv2DOp, TestConv2DOp_v2, create_test_padding_SAME_class, create_test_padding_VALID_class, create_test_channel_last_class, create_test_cudnn_padding_SAME_class, create_test_cudnn_channel_last_class +from test_conv2d_op import ( + TestConv2DOp, + TestConv2DOp_v2, + create_test_padding_SAME_class, + create_test_padding_VALID_class, + create_test_channel_last_class, + create_test_cudnn_padding_SAME_class, + create_test_cudnn_channel_last_class, +) -#----------------TestDepthwiseConv ----- +# ----------------TestDepthwiseConv ----- class TestDepthwiseConv(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -39,7 +46,6 @@ class TestDepthwiseConv(TestConv2DOp): class TestDepthwiseConv2(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -53,7 +59,6 @@ class TestDepthwiseConv2(TestConv2DOp): class TestDepthwiseConv3(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -67,7 +72,6 @@ class TestDepthwiseConv3(TestConv2DOp): class TestDepthwiseConvWithDilation(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -82,7 +86,6 @@ class TestDepthwiseConvWithDilation(TestConv2DOp): class TestDepthwiseConvWithDilation2(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -97,7 +100,6 @@ class TestDepthwiseConvWithDilation2(TestConv2DOp): class TestDepthwiseConvandFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -112,7 +114,6 @@ class TestDepthwiseConvandFuse(TestConv2DOp): class TestDepthwiseConv2andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -127,7 +128,6 @@ class TestDepthwiseConv2andFuse(TestConv2DOp): class TestDepthwiseConv3andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -142,7 +142,6 @@ class TestDepthwiseConv3andFuse(TestConv2DOp): class TestDepthwiseConvWithDilationandFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -158,7 +157,6 @@ class TestDepthwiseConvWithDilationandFuse(TestConv2DOp): class TestDepthwiseConvWithDilation2andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -174,7 +172,6 @@ class TestDepthwiseConvWithDilation2andFuse(TestConv2DOp): class TestDepthwiseConv_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.stride = [2, 2] @@ -191,7 +188,6 @@ class TestDepthwiseConv_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConv2_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.stride = [1, 1] @@ -208,7 +204,6 @@ class TestDepthwiseConv2_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConv3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.stride = [1, 1] @@ -225,7 +220,6 @@ class TestDepthwiseConv3_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConvWithDilation_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -244,7 +238,6 @@ class TestDepthwiseConvWithDilation_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConvWithDilation2_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -263,7 +256,6 @@ class TestDepthwiseConvWithDilation2_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConvandFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -282,7 +274,6 @@ class TestDepthwiseConvandFuse_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConv2andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -301,7 +292,6 @@ class TestDepthwiseConv2andFuse_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConv3andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -320,7 +310,6 @@ class TestDepthwiseConv3andFuse_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConvWithDilationandFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -340,7 +329,6 @@ class TestDepthwiseConvWithDilationandFuse_AsyPadding(TestConv2DOp_v2): class TestDepthwiseConvWithDilation2andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -382,12 +370,14 @@ create_test_channel_last_class(TestDepthwiseConvWithDilationandFuse_AsyPadding) if core.is_compiled_with_rocm(): create_test_cudnn_padding_SAME_class(TestDepthwiseConv_AsyPadding) create_test_cudnn_padding_SAME_class( - TestDepthwiseConvWithDilation_AsyPadding) + TestDepthwiseConvWithDilation_AsyPadding + ) create_test_padding_VALID_class(TestDepthwiseConv_AsyPadding) create_test_padding_VALID_class(TestDepthwiseConvWithDilation_AsyPadding) create_test_cudnn_channel_last_class(TestDepthwiseConv_AsyPadding) create_test_cudnn_channel_last_class( - TestDepthwiseConvWithDilation2_AsyPadding) + TestDepthwiseConvWithDilation2_AsyPadding + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_layer.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_layer.py index 74d50c545c6..8e6bdefc5ad 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_layer.py @@ -21,23 +21,24 @@ import unittest class Conv2DTransposeTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=(16, 16), - num_channels=6, - num_filters=8, - filter_size=3, - output_size=None, - output_padding=0, - padding=0, - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCHW", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=(16, 16), + num_channels=6, + num_filters=8, + filter_size=3, + output_size=None, + output_padding=0, + padding=0, + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCHW", + dtype="float32", + ): super(Conv2DTransposeTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -58,24 +59,31 @@ class Conv2DTransposeTestCase(unittest.TestCase): def setUp(self): self.channel_last = self.data_format == "NHWC" if self.channel_last: - input_shape = (self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) else: - input_shape = (self.batch_size, - self.num_channels) + self.spartial_shape + input_shape = ( + self.batch_size, + self.num_channels, + ) + self.spartial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] * 2 else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_channels, self.num_filters - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_channels, + self.num_filters // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) else: self.bias = None @@ -84,8 +92,11 @@ class Conv2DTransposeTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1,self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1) + input_shape = ( + (-1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) weight_attr = I.NumpyArrayInitializer(self.weight) if self.bias is None: @@ -104,11 +115,12 @@ class Conv2DTransposeTestCase(unittest.TestCase): groups=self.groups, param_attr=weight_attr, bias_attr=bias_attr, - data_format=self.data_format) + data_format=self.data_format, + ) feed_dict = {"input": self.input} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -116,36 +128,42 @@ class Conv2DTransposeTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1,self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1) + input_shape = ( + (-1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) if self.output_padding != 0: output_size = None else: output_size = self.output_size - y_var = F.conv2d_transpose(x_var, - w_var, - None if self.no_bias else b_var, - output_size=output_size, - padding=self.padding, - output_padding=self.output_padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y_var = F.conv2d_transpose( + x_var, + w_var, + None if self.no_bias else b_var, + output_size=output_size, + padding=self.padding, + output_padding=self.output_padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): @@ -156,15 +174,17 @@ class Conv2DTransposeTestCase(unittest.TestCase): else: output_size = self.output_size - conv = nn.Conv2DTranspose(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - output_padding=self.output_padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv2DTranspose( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + output_padding=self.output_padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -194,7 +214,6 @@ class Conv2DTransposeTestCase(unittest.TestCase): class Conv2DTransposeErrorTestCase(Conv2DTransposeTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -205,46 +224,61 @@ class Conv2DTransposeErrorTestCase(Conv2DTransposeTestCase): def add_cases(suite): suite.addTest(Conv2DTransposeTestCase(methodName='runTest')) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - stride=[1, 2], - no_bias=True, - dilation=2)) + Conv2DTransposeTestCase( + methodName='runTest', stride=[1, 2], no_bias=True, dilation=2 + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - filter_size=(3, 3), - output_size=[20, 36], - stride=[1, 2], - dilation=2)) + Conv2DTransposeTestCase( + methodName='runTest', + filter_size=(3, 3), + output_size=[20, 36], + stride=[1, 2], + dilation=2, + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', stride=2, - dilation=(2, 1))) - suite.addTest(Conv2DTransposeTestCase(methodName='runTest', - padding="valid")) + Conv2DTransposeTestCase(methodName='runTest', stride=2, dilation=(2, 1)) + ) + suite.addTest( + Conv2DTransposeTestCase(methodName='runTest', padding="valid") + ) suite.addTest(Conv2DTransposeTestCase(methodName='runTest', padding="same")) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - filter_size=1, - padding=(2, 3))) + Conv2DTransposeTestCase( + methodName='runTest', filter_size=1, padding=(2, 3) + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', padding=[1, 2, 2, 1])) + Conv2DTransposeTestCase(methodName='runTest', padding=[1, 2, 2, 1]) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - padding=[[0, 0], [0, 0], [1, 2], [2, 1]])) + Conv2DTransposeTestCase( + methodName='runTest', padding=[[0, 0], [0, 0], [1, 2], [2, 1]] + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', data_format="NHWC")) + Conv2DTransposeTestCase(methodName='runTest', data_format="NHWC") + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - data_format="NHWC", - padding=[[0, 0], [1, 1], [2, 2], [0, 0]])) + Conv2DTransposeTestCase( + methodName='runTest', + data_format="NHWC", + padding=[[0, 0], [1, 1], [2, 2], [0, 0]], + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', groups=2, - padding="valid")) + Conv2DTransposeTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid")) + Conv2DTransposeTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + ) + ) suite.addTest( Conv2DTransposeTestCase( methodName='runTest', @@ -257,17 +291,21 @@ def add_cases(suite): stride=2, output_size=[14, 14], output_padding=[1, 1], - )) + ) + ) def add_error_cases(suite): suite.addTest( - Conv2DTransposeErrorTestCase(methodName='runTest', - num_channels=5, - groups=2)) + Conv2DTransposeErrorTestCase( + methodName='runTest', num_channels=5, groups=2 + ) + ) suite.addTest( - Conv2DTransposeErrorTestCase(methodName='runTest', - output_size="not_valid")) + Conv2DTransposeErrorTestCase( + methodName='runTest', output_size="not_valid" + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py index da822c3b124..29ffbd80d36 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py @@ -30,9 +30,10 @@ from op_test import OpTest def conv2dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 3, 1, 2]) @@ -43,18 +44,22 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -88,37 +93,49 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): if 'output_padding' in attrs: out_pad_h = attrs['output_padding'][0] out_pad_w = attrs['output_padding'][1] - out = np.zeros((in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), - dtype=input_.dtype) + out = np.zeros( + (in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), dtype=input_.dtype + ) for n in range(in_n): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, g * sub_in_c:(g + 1) * sub_in_c, i, - j] # (c) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, i, j + ] # (c) input_masked = np.reshape(input_masked, (sub_in_c, 1, 1)) input_masked = np.tile(input_masked, (1, f_h, f_w)) for k in range(f_out_c): tmp_out = np.sum( - input_masked * - filter_[g * sub_in_c:(g + 1) * sub_in_c, k, :, :], - axis=0) + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, k, :, : + ], + axis=0, + ) i1, i2 = i * stride[0], i * stride[0] + d_bolck_h j1, j2 = j * stride[1], j * stride[1] + d_bolck_w - out[n, g * f_out_c + k, i1:i2:dilations[0], - j1:j2:dilations[1]] += tmp_out - - out = out[:, :, pad_h_0:out_h - pad_h_1 + out_pad_h, - pad_w_0:out_w - pad_w_1 + out_pad_w] + out[ + n, + g * f_out_c + k, + i1 : i2 : dilations[0], + j1 : j2 : dilations[1], + ] += tmp_out + + out = out[ + :, + :, + pad_h_0 : out_h - pad_h_1 + out_pad_h, + pad_w_0 : out_w - pad_w_1 + out_pad_w, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 1]) return out class TestConv2DTransposeOp(OpTest): - def setUp(self): # init as conv transpose self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 @@ -147,7 +164,7 @@ class TestConv2DTransposeOp(OpTest): 'use_cudnn': self.use_cudnn, 'is_test': self.is_test, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } if self.output_size is not None: self.attrs['output_size'] = self.output_size @@ -155,8 +172,9 @@ class TestConv2DTransposeOp(OpTest): if len(self.output_padding) > 0: self.attrs['output_padding'] = self.output_padding - output = conv2dtranspose_forward_naive(input_, filter_, - self.attrs).astype(self.dtype) + output = conv2dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype(self.dtype) self.outputs = {'Output': output} @@ -165,7 +183,8 @@ class TestConv2DTransposeOp(OpTest): if self.use_cudnn: place = core.CUDAPlace(0) self.check_output_with_place( - place, atol=1e-5, check_dygraph=(self.use_mkldnn == False)) + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) else: self.check_output(check_dygraph=(self.use_mkldnn == False)) @@ -173,39 +192,44 @@ class TestConv2DTransposeOp(OpTest): if self.need_check_grad: if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Filter'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Input']), + ) else: - self.check_grad(['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad( + ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def test_check_grad_no_filter(self): if self.need_check_grad: if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) else: - self.check_grad(['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad( + ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad(self): if self.need_check_grad: if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02) + self.check_grad_with_place( + place, + set(['Input', 'Filter']), + 'Output', + max_relative_error=0.02, + ) else: - self.check_grad(set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02) + self.check_grad( + set(['Input', 'Filter']), 'Output', max_relative_error=0.02 + ) def init_test_case(self): self.pad = [0, 0] @@ -221,7 +245,6 @@ class TestConv2DTransposeOp(OpTest): class TestWithSymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -233,7 +256,6 @@ class TestWithSymmetricPad(TestConv2DTransposeOp): class TestWithAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -245,7 +267,6 @@ class TestWithAsymmetricPad(TestConv2DTransposeOp): class TestWithSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [2, 1] self.dilations = [1, 2] @@ -257,7 +278,6 @@ class TestWithSAMEPad(TestConv2DTransposeOp): class TestWithVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -269,7 +289,6 @@ class TestWithVALIDPad(TestConv2DTransposeOp): class TestWithGroups(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -281,7 +300,6 @@ class TestWithGroups(TestConv2DTransposeOp): class TestWithStride(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -293,7 +311,6 @@ class TestWithStride(TestConv2DTransposeOp): class TestWithDilation(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -305,7 +322,6 @@ class TestWithDilation(TestConv2DTransposeOp): class TestWithEvenUpsample(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -318,7 +334,6 @@ class TestWithEvenUpsample(TestConv2DTransposeOp): class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -331,7 +346,6 @@ class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): class Test_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -344,7 +358,6 @@ class Test_NHWC(TestConv2DTransposeOp): class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -357,7 +370,6 @@ class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -370,7 +382,6 @@ class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): class TestWithGroups_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -383,7 +394,6 @@ class TestWithGroups_NHWC(TestConv2DTransposeOp): class TestWithStride_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -396,7 +406,6 @@ class TestWithStride_NHWC(TestConv2DTransposeOp): class TestWithDilation_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -409,7 +418,6 @@ class TestWithDilation_NHWC(TestConv2DTransposeOp): class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -423,7 +431,6 @@ class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -437,19 +444,19 @@ class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): # ------------ test_cudnn ------------ -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN(TestConv2DTransposeOp): - def init_op_type(self): self.use_cudnn = True self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -464,10 +471,10 @@ class TestCUDNNWithSymmetricPad(TestWithSymmetricPad): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad(TestWithAsymmetricPad): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -482,10 +489,10 @@ class TestCUDNNWithAsymmetricPad(TestWithAsymmetricPad): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSAMEPad(TestWithSAMEPad): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 2] @@ -500,10 +507,10 @@ class TestCUDNNWithSAMEPad(TestWithSAMEPad): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithVALIDPad(TestWithVALIDPad): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -518,10 +525,10 @@ class TestCUDNNWithVALIDPad(TestWithVALIDPad): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride(TestWithStride): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -536,10 +543,10 @@ class TestCUDNNWithStride(TestWithStride): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups(TestWithGroups): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -555,10 +562,10 @@ class TestCUDNNWithGroups(TestWithGroups): # ------------ test_cudnn ------------ -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithEvenUpsample(TestWithEvenUpsample): - def init_op_type(self): self.use_cudnn = True self.op_type = "conv2d_transpose" @@ -579,10 +586,10 @@ class TestCUDNNWithEvenUpsample(TestWithEvenUpsample): # self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -598,10 +605,10 @@ class TestCUDNN_NHWC(TestConv2DTransposeOp): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad_NHWC(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -617,10 +624,10 @@ class TestCUDNNWithSymmetricPad_NHWC(TestWithSymmetricPad): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad_NHWC(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 0, 2, 3] self.stride = [2, 2] @@ -636,10 +643,10 @@ class TestCUDNNWithAsymmetricPad_NHWC(TestWithSymmetricPad): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride_NHWC(TestWithStride): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -655,10 +662,10 @@ class TestCUDNNWithStride_NHWC(TestWithStride): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups_NHWC(TestWithGroups): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -674,10 +681,10 @@ class TestCUDNNWithGroups_NHWC(TestWithGroups): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithEvenUpsample_NHWC(TestWithEvenUpsample): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -694,10 +701,10 @@ class TestCUDNNWithEvenUpsample_NHWC(TestWithEvenUpsample): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN_FP16(TestConv2DTransposeOp): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -717,15 +724,16 @@ class TestCUDNN_FP16(TestConv2DTransposeOp): if self.use_cudnn: place = core.CUDAPlace(0) self.check_output_with_place( - place, atol=0.02, check_dygraph=(self.use_mkldnn == False)) + place, atol=0.02, check_dygraph=(self.use_mkldnn == False) + ) else: self.check_output(check_dygraph=(self.use_mkldnn == False)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [0, 0] @@ -738,10 +746,10 @@ class TestCUDNN_NHWC_FP16(TestCUDNN_FP16): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -754,10 +762,10 @@ class TestCUDNNWithSymmetricPad_NHWC_FP16(TestCUDNN_FP16): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 0, 2, 3] @@ -770,10 +778,10 @@ class TestCUDNNWithAsymmetricPad_NHWC_FP16(TestCUDNN_FP16): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -786,10 +794,10 @@ class TestCUDNNWithStride_NHWC_FP16(TestCUDNN_FP16): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -802,10 +810,10 @@ class TestCUDNNWithGroups_NHWC_FP16(TestCUDNN_FP16): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithEvenUpsample_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [2, 2] @@ -820,56 +828,67 @@ class TestCUDNNWithEvenUpsample_NHWC_FP16(TestCUDNN_FP16): class TestConv2DTransposeAPI(unittest.TestCase): - def test_case1(self): - data1 = fluid.layers.data(name='data1', - shape=[3, 5, 5], - dtype='float32') - data2 = fluid.layers.data(name='data2', - shape=[5, 5, 3], - dtype='float32') - out1 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - data_format='NCHW') - out2 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - data_format='NHWC') - out3 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - data_format='NHWC') - out4 = fluid.layers.conv2d_transpose(input=data1, - groups=3, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [2, 1], - [0, 0]], - data_format='NCHW') - out5 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='SAME', - data_format='NCHW') - out6 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding='VALID', - data_format='NHWC') - out7 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - output_size=[7, 7], - padding=[0, 0], - data_format='NHWC') + data1 = fluid.layers.data( + name='data1', shape=[3, 5, 5], dtype='float32' + ) + data2 = fluid.layers.data( + name='data2', shape=[5, 5, 3], dtype='float32' + ) + out1 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + data_format='NCHW', + ) + out2 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) + out3 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + data_format='NHWC', + ) + out4 = fluid.layers.conv2d_transpose( + input=data1, + groups=3, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [2, 1], [0, 0]], + data_format='NCHW', + ) + out5 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='SAME', + data_format='NCHW', + ) + out6 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding='VALID', + data_format='NHWC', + ) + out7 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + output_size=[7, 7], + padding=[0, 0], + data_format='NHWC', + ) data1_np = np.random.random((2, 3, 5, 5)).astype("float32") data2_np = np.random.random((2, 5, 5, 3)).astype("float32") @@ -880,13 +899,12 @@ class TestConv2DTransposeAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2, out3, out4, out5, out6, out7], + return_numpy=True, + ) self.assertIsNotNone(results[0]) self.assertIsNotNone(results[1]) self.assertIsNotNone(results[2]) @@ -897,76 +915,81 @@ class TestConv2DTransposeAPI(unittest.TestCase): class TestConv2DTransposeOpException(unittest.TestCase): - def test_exception(self): data = fluid.layers.data(name='data', shape=[3, 5, 5], dtype="float32") def attr_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - data_format="NCDHW") + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + data_format="NCDHW", + ) self.assertRaises(ValueError, attr_data_format) def attr_padding_str(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding='Vald') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding='Vald', + ) self.assertRaises(ValueError, attr_padding_str) def attr_padding_list(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [1, 1], [0, 0], - [0, 0]]) + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [1, 1], [0, 0], [0, 0]], + ) self.assertRaises(ValueError, attr_padding_list) def attr_padding_with_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [0, 0], [0, 0], - [1, 1]], - data_format='NHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [0, 0], [0, 0], [1, 1]], + data_format='NHWC', + ) self.assertRaises(ValueError, attr_padding_with_data_format) - error_input = fluid.layers.data(name='error_data', - shape=[1], - dtype="float32") + error_input = fluid.layers.data( + name='error_data', shape=[1], dtype="float32" + ) def error_input_size(): - out = fluid.layers.conv2d_transpose(input=error_input, - groups=1, - num_filters=6, - filter_size=3) + out = fluid.layers.conv2d_transpose( + input=error_input, groups=1, num_filters=6, filter_size=3 + ) self.assertRaises(ValueError, error_input_size) def error_groups(): - out = fluid.layers.conv2d_transpose(input=data, - groups=0, - num_filters=6, - filter_size=3, - data_format='NHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=0, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) self.assertRaises(ValueError, error_groups) class TestConv2DTransposeRepr(unittest.TestCase): - def test_case(self): paddle.disable_static() - x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = nn.Conv2DTranspose(4, 6, (3, 3), output_padding=1, stride=2) print(conv) y_var = conv(x_var) @@ -976,7 +999,6 @@ class TestConv2DTransposeRepr(unittest.TestCase): class TestTensorOutputSize1(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 8, 8]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -991,7 +1013,8 @@ class TestTensorOutputSize1(UnittestBase): w_var = paddle.randn((3, 6, 3, 3), dtype='float32') output_size = paddle.assign([17]) out = paddle.paddle.nn.functional.conv2d_transpose( - x, w_var, stride=2, output_size=output_size) + x, w_var, stride=2, output_size=output_size + ) return out def test_static(self): @@ -1013,15 +1036,15 @@ class TestTensorOutputSize1(UnittestBase): res = exe.run(fetch_list=[feat, out]) np.testing.assert_allclose(res[1].shape, (2, 6, 17, 17)) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() np.testing.assert_allclose(infer_outs[1].shape, (2, 6, 17, 17)) class TestTensorOutputSize2(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size2' @@ -1029,43 +1052,37 @@ class TestTensorOutputSize2(TestTensorOutputSize1): w_var = paddle.randn((3, 6, 3, 3), dtype='float32') output_size = [17, paddle.assign([17])] out = paddle.paddle.nn.functional.conv2d_transpose( - x, w_var, stride=2, output_size=output_size) + x, w_var, stride=2, output_size=output_size + ) return out class TestTensorOutputSize3(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size3' def call_func(self, x): w_var = paddle.randn((3, 6, 3, 3), dtype='float32') output_size = paddle.assign([17]) - out = paddle.fluid.layers.conv2d_transpose(x, - num_filters=6, - output_size=output_size, - filter_size=3, - stride=2) + out = paddle.fluid.layers.conv2d_transpose( + x, num_filters=6, output_size=output_size, filter_size=3, stride=2 + ) return out class TestTensorOutputSize4(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size4' def call_func(self, x): output_size = [17, paddle.assign([17])] - out = paddle.fluid.layers.conv2d_transpose(x, - num_filters=6, - output_size=output_size, - filter_size=3, - stride=2) + out = paddle.fluid.layers.conv2d_transpose( + x, num_filters=6, output_size=output_size, filter_size=3, stride=2 + ) return out class TestTensorOutputSize5(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size5' @@ -1077,13 +1094,13 @@ class TestTensorOutputSize5(TestTensorOutputSize1): num_filters=6, filter_size=3, output_size=output_size, - stride=2) + stride=2, + ) out = conv2d_trans(x) return out class TestTensorOutputSize6(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size6' @@ -1098,13 +1115,13 @@ class TestTensorOutputSize6(TestTensorOutputSize1): num_filters=6, filter_size=3, output_size=output_size, - stride=2) + stride=2, + ) out = conv2d_trans(x) return out class TestTensorOutputSize7(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size7' @@ -1119,13 +1136,13 @@ class TestTensorOutputSize7(TestTensorOutputSize1): num_filters=6, filter_size=3, output_size=output_size, - stride=2) + stride=2, + ) out = conv2d_trans(x) return out class TestTensorOutputSize8(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size8' @@ -1140,7 +1157,8 @@ class TestTensorOutputSize8(TestTensorOutputSize1): num_filters=6, filter_size=3, output_size=output_size, - stride=2) + stride=2, + ) out = conv2d_trans(x) return out diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op_depthwise_conv.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op_depthwise_conv.py index b837efaa783..a50b0e2cb96 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op_depthwise_conv.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op_depthwise_conv.py @@ -22,7 +22,6 @@ from test_conv2d_transpose_op import TestConv2DTransposeOp class TestDepthwiseConvTranspose(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -36,7 +35,6 @@ class TestDepthwiseConvTranspose(TestConv2DTransposeOp): class TestDepthwiseConvTransposeAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1, 2] self.stride = [1, 1] @@ -51,7 +49,6 @@ class TestDepthwiseConvTransposeAsymmetricPad(TestConv2DTransposeOp): class TestDepthwiseConvTransposeSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -65,7 +62,6 @@ class TestDepthwiseConvTransposeSAMEPad(TestConv2DTransposeOp): class TestDepthwiseConvTransposeVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -79,7 +75,6 @@ class TestDepthwiseConvTransposeVALIDPad(TestConv2DTransposeOp): class TestDepthwiseConvTranspose_NHWC_3x3kernel(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_layer.py b/python/paddle/fluid/tests/unittests/test_conv3d_layer.py index 5672fe0dd88..053bbb64166 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_layer.py @@ -23,21 +23,22 @@ import unittest class Conv3DTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=(8, 8, 8), - num_channels=6, - num_filters=8, - filter_size=3, - padding=0, - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCDHW", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=(8, 8, 8), + num_channels=6, + num_filters=8, + filter_size=3, + padding=0, + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCDHW", + dtype="float32", + ): super(Conv3DTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -56,24 +57,31 @@ class Conv3DTestCase(unittest.TestCase): def setUp(self): self.channel_last = self.data_format == "NDHWC" if self.channel_last: - input_shape = (self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) else: - input_shape = (self.batch_size, - self.num_channels) + self.spartial_shape + input_shape = ( + self.batch_size, + self.num_channels, + ) + self.spartial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] * 3 else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_filters, self.num_channels - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_filters, + self.num_channels // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) else: self.bias = None @@ -82,28 +90,33 @@ class Conv3DTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1, -1, self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1, -1) + input_shape = ( + (-1, -1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) weight_attr = I.NumpyArrayInitializer(self.weight) if self.bias is None: bias_attr = False else: bias_attr = I.NumpyArrayInitializer(self.bias) - y_var = fluid.layers.conv3d(x_var, - self.num_filters, - self.filter_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - param_attr=weight_attr, - bias_attr=bias_attr, - data_format=self.data_format) + y_var = fluid.layers.conv3d( + x_var, + self.num_filters, + self.filter_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + param_attr=weight_attr, + bias_attr=bias_attr, + data_format=self.data_format, + ) feed_dict = {"input": self.input} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -111,41 +124,49 @@ class Conv3DTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1, -1, self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1, -1) + input_shape = ( + (-1, -1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) - y_var = F.conv3d(x_var, - w_var, - None if self.no_bias else b_var, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) + y_var = F.conv3d( + x_var, + w_var, + None if self.no_bias else b_var, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = paddle.to_tensor(self.input) x_var.stop_gradient = False - conv = nn.Conv3D(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv3D( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -178,7 +199,6 @@ class Conv3DTestCase(unittest.TestCase): class Conv3DErrorTestCase(Conv3DTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -189,44 +209,60 @@ class Conv3DErrorTestCase(Conv3DTestCase): def add_cases(suite): suite.addTest(Conv3DTestCase(methodName='runTest')) suite.addTest( - Conv3DTestCase(methodName='runTest', stride=[1, 2, 1], dilation=2)) + Conv3DTestCase(methodName='runTest', stride=[1, 2, 1], dilation=2) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', stride=2, dilation=(2, 1, 2))) + Conv3DTestCase(methodName='runTest', stride=2, dilation=(2, 1, 2)) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', padding="same", no_bias=True)) + Conv3DTestCase(methodName='runTest', padding="same", no_bias=True) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', - filter_size=(3, 2, 3), - padding='valid')) + Conv3DTestCase( + methodName='runTest', filter_size=(3, 2, 3), padding='valid' + ) + ) suite.addTest(Conv3DTestCase(methodName='runTest', padding=(2, 3, 1))) suite.addTest( - Conv3DTestCase(methodName='runTest', padding=[1, 2, 2, 1, 2, 3])) + Conv3DTestCase(methodName='runTest', padding=[1, 2, 2, 1, 2, 3]) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', - padding=[[0, 0], [0, 0], [1, 2], [2, 1], [2, 2]])) + Conv3DTestCase( + methodName='runTest', + padding=[[0, 0], [0, 0], [1, 2], [2, 1], [2, 2]], + ) + ) suite.addTest(Conv3DTestCase(methodName='runTest', data_format="NDHWC")) suite.addTest( - Conv3DTestCase(methodName='runTest', - data_format="NDHWC", - padding=[[0, 0], [1, 1], [3, 3], [2, 2], [0, 0]])) + Conv3DTestCase( + methodName='runTest', + data_format="NDHWC", + padding=[[0, 0], [1, 1], [3, 3], [2, 2], [0, 0]], + ) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', groups=2, padding="valid")) + Conv3DTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv3DTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid")) + Conv3DTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + ) + ) def add_error_cases(suite): suite.addTest( - Conv3DErrorTestCase(methodName='runTest', num_channels=5, groups=2)) + Conv3DErrorTestCase(methodName='runTest', num_channels=5, groups=2) + ) suite.addTest( - Conv3DErrorTestCase(methodName='runTest', - num_channels=5, - groups=2, - padding=[-1, 1, 3])) + Conv3DErrorTestCase( + methodName='runTest', num_channels=5, groups=2, padding=[-1, 1, 3] + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_op.py index a65d1c73104..eaa6ba04c64 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_op.py @@ -21,24 +21,28 @@ import paddle.fluid as fluid import paddle -def conv3d_forward_naive(input, - filter, - group, - conv_param, - padding_algorithm='EXPLICIT', - data_format="NCDHW"): +def conv3d_forward_naive( + input, + filter, + group, + conv_param, + padding_algorithm='EXPLICIT', + data_format="NCDHW", +): if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if data_format not in ["NCDHW", "NDHWC"]: - raise ValueError("Unknown Attr(data_format): '%s' ." - "It can only be 'NCDHW' or 'NDHWC'." % - str(data_format)) + raise ValueError( + "Unknown Attr(data_format): '%s' ." + "It can only be 'NCDHW' or 'NDHWC'." % str(data_format) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" if channel_last: input = np.transpose(input, [0, 4, 1, 2, 3]) @@ -52,17 +56,22 @@ def conv3d_forward_naive(input, sub_out_c = out_c // group sub_f_n = f_n // group - stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ - 'dilations'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilations'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -85,59 +94,85 @@ def conv3d_forward_naive(input, pad_h_0, pad_h_1 = pad[2], pad[3] pad_w_0, pad_w_1 = pad[4], pad[5] - out_d = 1 + (in_d + pad_d_0 + pad_d_1 - (dilation[0] * - (f_d - 1) + 1)) // stride[0] - out_h = 1 + (in_h + pad_h_0 + pad_h_1 - (dilation[1] * - (f_h - 1) + 1)) // stride[1] - out_w = 1 + (in_w + pad_w_0 + pad_w_1 - (dilation[2] * - (f_w - 1) + 1)) // stride[2] + out_d = ( + 1 + + (in_d + pad_d_0 + pad_d_1 - (dilation[0] * (f_d - 1) + 1)) + // stride[0] + ) + out_h = ( + 1 + + (in_h + pad_h_0 + pad_h_1 - (dilation[1] * (f_h - 1) + 1)) + // stride[1] + ) + out_w = ( + 1 + + (in_w + pad_w_0 + pad_w_1 - (dilation[2] * (f_w - 1) + 1)) + // stride[2] + ) out = np.zeros((in_n, out_c, out_d, out_h, out_w)) - d_bolck_d = (dilation[0] * (f_d - 1) + 1) - d_bolck_h = (dilation[1] * (f_h - 1) + 1) - d_bolck_w = (dilation[2] * (f_w - 1) + 1) - - input_pad = np.pad(input, ((0, 0), (0, 0), (pad_d_0, pad_d_1), - (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), - mode='constant', - constant_values=0) + d_bolck_d = dilation[0] * (f_d - 1) + 1 + d_bolck_h = dilation[1] * (f_h - 1) + 1 + d_bolck_w = dilation[2] * (f_w - 1) + 1 + + input_pad = np.pad( + input, + ( + (0, 0), + (0, 0), + (pad_d_0, pad_d_1), + (pad_h_0, pad_h_1), + (pad_w_0, pad_w_1), + ), + mode='constant', + constant_values=0, + ) filter_dilation = np.zeros((f_n, f_c, d_bolck_d, d_bolck_h, d_bolck_w)) - filter_dilation[:, :, 0:d_bolck_d:dilation[0], 0:d_bolck_h:dilation[1], - 0:d_bolck_w:dilation[2]] = filter + filter_dilation[ + :, + :, + 0 : d_bolck_d : dilation[0], + 0 : d_bolck_h : dilation[1], + 0 : d_bolck_w : dilation[2], + ] = filter for d in range(out_d): for i in range(out_h): for j in range(out_w): for g in range(group): - input_pad_masked = \ - input_pad[:, g * f_c:(g + 1) * f_c, - d * stride[0]:d * stride[0] + d_bolck_d, - i * stride[1]:i * stride[1] + d_bolck_h, - j * stride[2]:j * stride[2] + d_bolck_w] - - f_sub = filter_dilation[g * sub_f_n:(g + 1) * - sub_f_n, :, :, :, :] + input_pad_masked = input_pad[ + :, + g * f_c : (g + 1) * f_c, + d * stride[0] : d * stride[0] + d_bolck_d, + i * stride[1] : i * stride[1] + d_bolck_h, + j * stride[2] : j * stride[2] + d_bolck_w, + ] + + f_sub = filter_dilation[ + g * sub_f_n : (g + 1) * sub_f_n, :, :, :, : + ] for k in range(sub_out_c): - out[:, g * sub_out_c + k, d, i, j] = \ - np.sum(input_pad_masked * f_sub[k, :, :, :, :], - axis=(1, 2, 3, 4)) + out[:, g * sub_out_c + k, d, i, j] = np.sum( + input_pad_masked * f_sub[k, :, :, :, :], + axis=(1, 2, 3, 4), + ) if channel_last: out = np.transpose(out, [0, 2, 3, 4, 1]) return out def create_test_cudnn_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNN") TestCUDNNCase.__name__ = cls_name @@ -145,9 +180,7 @@ def create_test_cudnn_class(parent): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0, 0] self.padding_algorithm = "SAME" @@ -158,9 +191,7 @@ def create_test_padding_SAME_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1, 1] self.padding_algorithm = "VALID" @@ -171,15 +202,15 @@ def create_test_padding_VALID_class(parent): def create_test_cudnn_padding_SAME_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingSMAECase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_paddings(self): self.pad = [1, 1, 1] @@ -191,15 +222,15 @@ def create_test_cudnn_padding_SAME_class(parent): def create_test_cudnn_padding_VALID_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingVALIDCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_paddings(self): self.pad = [1, 1, 1] @@ -211,9 +242,7 @@ def create_test_cudnn_padding_VALID_class(parent): def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NDHWC" @@ -227,15 +256,15 @@ def create_test_channel_last_class(parent): def create_test_cudnn_channel_last_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCudnnChannelLastCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_data_format(self): self.data_format = "NDHWC" @@ -250,7 +279,6 @@ def create_test_cudnn_channel_last_class(parent): class TestConv3DOp(OpTest): - def setUp(self): self.op_type = "conv3d" self.use_cudnn = False @@ -265,7 +293,7 @@ class TestConv3DOp(OpTest): conv3d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilations': self.dilations + 'dilations': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -279,7 +307,7 @@ class TestConv3DOp(OpTest): self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -288,7 +316,7 @@ class TestConv3DOp(OpTest): 'dilations': self.dilations, 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -298,41 +326,50 @@ class TestConv3DOp(OpTest): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() - self.check_output_with_place(place, - atol=1e-5, - check_dygraph=(self.use_mkldnn == False)) + self.check_output_with_place( + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + check_dygraph=(self.use_mkldnn == False), + ) def init_test_case(self): self.pad = [0, 0, 0] @@ -356,7 +393,6 @@ class TestConv3DOp(OpTest): class TestCase1(TestConv3DOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -367,19 +403,16 @@ class TestCase1(TestConv3DOp): class TestWithGroup1(TestConv3DOp): - def init_group(self): self.groups = 3 class TestWithGroup2(TestCase1): - def init_group(self): self.groups = 3 class TestWith1x1(TestConv3DOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -396,7 +429,6 @@ class TestWith1x1(TestConv3DOp): class TestWithInput1x1Filter1x1(TestConv3DOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -413,7 +445,6 @@ class TestWithInput1x1Filter1x1(TestConv3DOp): class TestWithDilation(TestConv3DOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -429,22 +460,22 @@ class TestWithDilation(TestConv3DOp): self.groups = 3 -#---------------- Conv3DCUDNN ---------------- +# ---------------- Conv3DCUDNN ---------------- -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN(TestConv3DOp): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16CUDNN(TestConv3DOp): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -456,19 +487,19 @@ class TestFP16CUDNN(TestConv3DOp): self.check_output_with_place(place, atol=2e-2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestWithGroup1CUDNN(TestWithGroup1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16WithGroup1CUDNN(TestWithGroup1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -480,19 +511,19 @@ class TestFP16WithGroup1CUDNN(TestWithGroup1): self.check_output_with_place(place, atol=2e-2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestWithGroup2CUDNN(TestWithGroup2): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16WithGroup2CUDNN(TestWithGroup2): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -504,19 +535,19 @@ class TestFP16WithGroup2CUDNN(TestWithGroup2): self.check_output_with_place(place, atol=2e-2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestWith1x1CUDNN(TestWith1x1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16With1x1CUDNN(TestWith1x1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -528,19 +559,19 @@ class TestFP16With1x1CUDNN(TestWith1x1): self.check_output_with_place(place, atol=2e-2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestWithInput1x1Filter1x1CUDNN(TestWithInput1x1Filter1x1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16WithInput1x1Filter1x1CUDNN(TestWithInput1x1Filter1x1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -553,7 +584,6 @@ class TestFP16WithInput1x1Filter1x1CUDNN(TestWithInput1x1Filter1x1): class TestCUDNNExhaustiveSearch(TestCUDNN): - def init_kernel_type(self): self.use_cudnn = True self.exhaustive_search = True @@ -564,7 +594,6 @@ class TestCUDNNExhaustiveSearch(TestCUDNN): class TestConv3DOp_2(OpTest): - def setUp(self): self.op_type = "conv3d" self.use_cudnn = False @@ -583,18 +612,23 @@ class TestConv3DOp_2(OpTest): conv3d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilations': self.dilations + 'dilations': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = conv3d_forward_naive(input, filter, self.groups, conv3d_param, - self.padding_algorithm, - self.data_format).astype(self.dtype) + output = conv3d_forward_naive( + input, + filter, + self.groups, + conv3d_param, + self.padding_algorithm, + self.data_format, + ).astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -604,7 +638,7 @@ class TestConv3DOp_2(OpTest): 'dilations': self.dilations, 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -619,27 +653,33 @@ class TestConv3DOp_2(OpTest): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() - self.check_grad_with_place(place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03) + self.check_grad_with_place( + place, {'Input', 'Filter'}, 'Output', max_relative_error=0.03 + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() - self.check_grad_with_place(place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + ) def init_test_case(self): self.stride = [1, 1, 1] @@ -669,7 +709,6 @@ class TestConv3DOp_2(OpTest): class TestConv3DOp_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 2] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -683,7 +722,6 @@ class TestConv3DOp_AsyPadding(TestConv3DOp_2): class TestConv3DOp_DiffDataInDiffDim(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 2] self.input_size = [2, 3, 4, 5, 5] # NCDHW @@ -702,7 +740,6 @@ create_test_channel_last_class(TestConv3DOp_DiffDataInDiffDim) class TestCase1_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -716,7 +753,6 @@ class TestCase1_AsyPadding(TestConv3DOp_2): class TestWithGroup1_AsyPadding(TestConv3DOp_2): - def init_group(self): self.groups = 3 @@ -726,7 +762,6 @@ class TestWithGroup1_AsyPadding(TestConv3DOp_2): class TestWithGroup2_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -743,7 +778,6 @@ class TestWithGroup2_AsyPadding(TestConv3DOp_2): class TestWith1x1_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] @@ -763,7 +797,6 @@ class TestWith1x1_AsyPadding(TestConv3DOp_2): class TestWithDilation_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 6, 6, 6] @@ -829,196 +862,228 @@ create_test_cudnn_channel_last_class(TestWith1x1_AsyPadding) # --------- test python API --------------- class TestConv3DAPI(unittest.TestCase): - def test_api(self): - input_NDHWC = fluid.layers.data(name="input_NDHWC", - shape=[2, 5, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCDHW = fluid.layers.data(name="input_NCDHW", - shape=[2, 3, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - fluid.layers.conv3d(input=input_NDHWC, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=0, - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[1, 2, 1, 0, 1, 0], - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NDHWC, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], - dilation=[1, 1, 1], - groups=1, - data_format="NDHWC") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding="SAME", - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding="VALID", - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") + input_NDHWC = fluid.layers.data( + name="input_NDHWC", + shape=[2, 5, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCDHW = fluid.layers.data( + name="input_NCDHW", + shape=[2, 3, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + fluid.layers.conv3d( + input=input_NDHWC, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=0, + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[1, 2, 1, 0, 1, 0], + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NDHWC, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], + dilation=[1, 1, 1], + groups=1, + data_format="NDHWC", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding="SAME", + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding="VALID", + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) class TestConv3DAPI_Error(unittest.TestCase): - def test_api(self): - input = fluid.layers.data(name="input", - shape=[2, 5, 5, 5, 4], - append_batch_size=False, - dtype="float32") + input = fluid.layers.data( + name="input", + shape=[2, 5, 5, 5, 4], + append_batch_size=False, + dtype="float32", + ) # ValueError: cudnn def run_1(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=1, - use_cudnn=[0], - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + use_cudnn=[0], + data_format="NCDHW", + ) self.assertRaises(ValueError, run_1) # ValueError: data_format def run_2(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=0, - dilation=[1, 1, 1], - groups=1, - use_cudnn=False, - data_format="NCHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=0, + dilation=[1, 1, 1], + groups=1, + use_cudnn=False, + data_format="NCHWC", + ) self.assertRaises(ValueError, run_2) # ValueError: padding def run_3(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding="SAMEE", - dilation=1, - groups=1, - use_cudnn=False, - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding="SAMEE", + dilation=1, + groups=1, + use_cudnn=False, + data_format="NCDHW", + ) self.assertRaises(ValueError, run_3) def run_4(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=[[0, 1], [0, 0], [0, 1], [0, 1], [0, - 1]], - dilation=1, - groups=1, - use_cudnn=False, - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=[[0, 1], [0, 0], [0, 1], [0, 1], [0, 1]], + dilation=1, + groups=1, + use_cudnn=False, + data_format="NCDHW", + ) self.assertRaises(ValueError, run_4) def run_5(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=0, - stride=0, - padding=[[0, 1], [0, 1], [0, 1], [0, 1], [0, - 1]], - dilation=1, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=0, + stride=0, + padding=[[0, 1], [0, 1], [0, 1], [0, 1], [0, 1]], + dilation=1, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_5) # ValueError: channel dimmention - x = fluid.layers.data(name="x", - shape=[2, 5, 5, 5, -1], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name="x", + shape=[2, 5, 5, 5, -1], + append_batch_size=False, + dtype="float32", + ) def run_6(): - fluid.layers.conv3d(input=x, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=x, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_6) # ValueError: groups def run_7(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=3, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=3, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_7) # ValueError: filter num def run_8(): - fluid.layers.conv3d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_8) diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_layer.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_layer.py index 9ad3eaaccfc..fa2138956b9 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_layer.py @@ -21,22 +21,23 @@ import unittest class Conv3DTransposeTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=2, - spartial_shape=(8, 8, 8), - num_channels=6, - num_filters=8, - filter_size=3, - output_size=None, - padding=0, - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCDHW", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=2, + spartial_shape=(8, 8, 8), + num_channels=6, + num_filters=8, + filter_size=3, + output_size=None, + padding=0, + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCDHW", + dtype="float32", + ): super(Conv3DTransposeTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -56,34 +57,44 @@ class Conv3DTransposeTestCase(unittest.TestCase): def setUp(self): self.channel_last = self.data_format == "NDHWC" if self.channel_last: - input_shape = (self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) else: - input_shape = (self.batch_size, - self.num_channels) + self.spartial_shape + input_shape = ( + self.batch_size, + self.num_channels, + ) + self.spartial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] * 3 else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_channels, self.num_filters - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_channels, + self.num_filters // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if self.no_bias: self.bias = None else: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) def fluid_layer(self, place): main = fluid.Program() start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1, -1, self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1, -1) + input_shape = ( + (-1, -1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) weight_attr = I.NumpyArrayInitializer(self.weight) if self.bias is None: @@ -101,11 +112,12 @@ class Conv3DTransposeTestCase(unittest.TestCase): groups=self.groups, param_attr=weight_attr, bias_attr=bias_attr, - data_format=self.data_format) + data_format=self.data_format, + ) feed_dict = {"input": self.input} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -113,41 +125,49 @@ class Conv3DTransposeTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1, -1, self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1, -1) + input_shape = ( + (-1, -1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) - y_var = F.conv3d_transpose(x_var, - w_var, - None if self.no_bias else b_var, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) + y_var = F.conv3d_transpose( + x_var, + w_var, + None if self.no_bias else b_var, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = dg.to_variable(self.input) - conv = nn.Conv3DTranspose(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv3DTranspose( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -174,7 +194,6 @@ class Conv3DTransposeTestCase(unittest.TestCase): class Conv3DTransposeErrorTestCase(Conv3DTransposeTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -185,65 +204,85 @@ class Conv3DTransposeErrorTestCase(Conv3DTransposeTestCase): def add_cases(suite): suite.addTest(Conv3DTransposeTestCase(methodName='runTest')) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - stride=[1, 2, 1], - dilation=2, - no_bias=True)) + Conv3DTransposeTestCase( + methodName='runTest', stride=[1, 2, 1], dilation=2, no_bias=True + ) + ) + suite.addTest( + Conv3DTransposeTestCase( + methodName='runTest', + output_size=[12, 19, 12], + stride=[1, 2, 1], + dilation=2, + ) + ) + suite.addTest( + Conv3DTransposeTestCase( + methodName='runTest', stride=2, dilation=(2, 1, 2) + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - output_size=[12, 19, 12], - stride=[1, 2, 1], - dilation=2)) + Conv3DTransposeTestCase(methodName='runTest', padding="valid") + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - stride=2, - dilation=(2, 1, 2))) - suite.addTest(Conv3DTransposeTestCase(methodName='runTest', - padding="valid")) - suite.addTest(Conv3DTransposeTestCase(methodName='runTest', - padding='valid')) + Conv3DTransposeTestCase(methodName='runTest', padding='valid') + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - filter_size=1, - padding=(2, 3, 1))) + Conv3DTransposeTestCase( + methodName='runTest', filter_size=1, padding=(2, 3, 1) + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - padding=[1, 2, 2, 3, 2, 1])) + Conv3DTransposeTestCase( + methodName='runTest', padding=[1, 2, 2, 3, 2, 1] + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - padding=[[0, 0], [0, 0], [2, 3], [1, 2], [2, - 1]])) + Conv3DTransposeTestCase( + methodName='runTest', + padding=[[0, 0], [0, 0], [2, 3], [1, 2], [2, 1]], + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', data_format="NDHWC")) + Conv3DTransposeTestCase(methodName='runTest', data_format="NDHWC") + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - data_format="NDHWC", - padding=[[0, 0], [1, 1], [2, 2], [3, 3], [0, - 0]])) + Conv3DTransposeTestCase( + methodName='runTest', + data_format="NDHWC", + padding=[[0, 0], [1, 1], [2, 2], [3, 3], [0, 0]], + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', groups=2, - padding="valid")) + Conv3DTransposeTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid")) + Conv3DTransposeTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + ) + ) def add_error_cases(suite): suite.addTest( - Conv3DTransposeErrorTestCase(methodName='runTest', - num_channels=5, - groups=2)) + Conv3DTransposeErrorTestCase( + methodName='runTest', num_channels=5, groups=2 + ) + ) suite.addTest( - Conv3DTransposeErrorTestCase(methodName='runTest', - output_size="not_valid")) + Conv3DTransposeErrorTestCase( + methodName='runTest', output_size="not_valid" + ) + ) suite.addTest( - Conv3DTransposeErrorTestCase(methodName='runTest', - num_channels=5, - groups=2, - padding=[-1, 1, 3])) + Conv3DTransposeErrorTestCase( + methodName='runTest', num_channels=5, groups=2, padding=[-1, 1, 3] + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py index ccda74b498a..869d1d5f29a 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py @@ -25,9 +25,10 @@ from op_test import OpTest def conv3dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 4, 1, 2, 3]) @@ -38,17 +39,21 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -84,34 +89,50 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, - g * sub_in_c:(g + 1) * sub_in_c, - d, i, j] # (c) - input_masked = np.reshape(input_masked, - (sub_in_c, 1, 1, 1)) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, d, i, j + ] # (c) + input_masked = np.reshape( + input_masked, (sub_in_c, 1, 1, 1) + ) input_masked = np.tile(input_masked, (1, f_d, f_h, f_w)) for k in range(f_out_c): - tmp_out = np.sum(input_masked * - filter_[g * sub_in_c:(g + 1) * - sub_in_c, k, :, :, :], - axis=0) + tmp_out = np.sum( + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, + k, + :, + :, + :, + ], + axis=0, + ) d1, d2 = d * stride[0], d * stride[0] + d_bolck_d i1, i2 = i * stride[1], i * stride[1] + d_bolck_h j1, j2 = j * stride[2], j * stride[2] + d_bolck_w - out[n, g * f_out_c + k, d1:d2:dilations[0], - i1:i2:dilations[1], - j1:j2:dilations[2]] += tmp_out - - out = out[:, :, pad_d_0:out_d - pad_d_1, pad_h_0:out_h - pad_h_1, - pad_w_0:out_w - pad_w_1] + out[ + n, + g * f_out_c + k, + d1 : d2 : dilations[0], + i1 : i2 : dilations[1], + j1 : j2 : dilations[2], + ] += tmp_out + + out = out[ + :, + :, + pad_d_0 : out_d - pad_d_1, + pad_h_0 : out_h - pad_h_1, + pad_w_0 : out_w - pad_w_1, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 4, 1]) return out class TestConv3DTransposeOp(OpTest): - def setUp(self): # init as conv transpose self.use_cudnn = False @@ -134,11 +155,12 @@ class TestConv3DTransposeOp(OpTest): 'dilations': self.dilations, 'groups': self.groups, 'use_cudnn': self.use_cudnn, - 'data_format': self.data_format + 'data_format': self.data_format, } - output = conv3dtranspose_forward_naive(input_, filter_, - self.attrs).astype("float32") + output = conv3dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype("float32") self.outputs = {'Output': output} @@ -152,40 +174,52 @@ class TestConv3DTransposeOp(OpTest): def test_check_grad(self): if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.03) + self.check_grad_with_place( + place, + set(['Input', 'Filter']), + 'Output', + max_relative_error=0.03, + ) else: - self.check_grad(set(['Input', 'Filter']), - 'Output', - max_relative_error=0.03) + self.check_grad( + set(['Input', 'Filter']), 'Output', max_relative_error=0.03 + ) def test_check_grad_no_filter(self): if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + ) elif self.check_no_filter: - self.check_grad(['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter'])) + self.check_grad( + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + ) def test_check_grad_no_input(self): if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + ) elif self.check_no_input: - self.check_grad(['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input'])) + self.check_grad( + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + ) def init_test_case(self): self.pad = [0, 0, 0] @@ -201,7 +235,6 @@ class TestConv3DTransposeOp(OpTest): class TestWithSymmetricPad(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_input = True self.pad = [1, 1, 1] @@ -214,7 +247,6 @@ class TestWithSymmetricPad(TestConv3DTransposeOp): class TestWithAsymmetricPad(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 0, 1, 2] self.stride = [1, 1, 1] @@ -226,7 +258,6 @@ class TestWithAsymmetricPad(TestConv3DTransposeOp): class TestWithSAMEPad(TestConv3DTransposeOp): - def init_test_case(self): self.stride = [1, 1, 2] self.dilations = [1, 2, 1] @@ -238,7 +269,6 @@ class TestWithSAMEPad(TestConv3DTransposeOp): class TestWithVALIDPad(TestConv3DTransposeOp): - def init_test_case(self): self.stride = [2, 1, 1] self.dilations = [1, 1, 1] @@ -250,7 +280,6 @@ class TestWithVALIDPad(TestConv3DTransposeOp): class TestWithStride(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_filter = True self.pad = [1, 1, 1] @@ -263,7 +292,6 @@ class TestWithStride(TestConv3DTransposeOp): class TestWithGroups(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -275,7 +303,6 @@ class TestWithGroups(TestConv3DTransposeOp): class TestWithDilation(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -287,7 +314,6 @@ class TestWithDilation(TestConv3DTransposeOp): class Test_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -300,19 +326,19 @@ class Test_NHWC(TestConv3DTransposeOp): # ------------ test_cudnn ------------ -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN(TestConv3DTransposeOp): - def init_op_type(self): self.use_cudnn = True self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -327,10 +353,10 @@ class TestCUDNNWithSymmetricPad(TestWithSymmetricPad): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad(TestWithAsymmetricPad): - def init_test_case(self): self.pad = [1, 1, 1, 0, 0, 2] self.stride = [1, 1, 1] @@ -345,10 +371,10 @@ class TestCUDNNWithAsymmetricPad(TestWithAsymmetricPad): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSAMEPad(TestWithSAMEPad): - def init_test_case(self): self.stride = [1, 1, 2] self.dilations = [1, 2, 1] @@ -363,10 +389,10 @@ class TestCUDNNWithSAMEPad(TestWithSAMEPad): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithVALIDPad(TestWithVALIDPad): - def init_test_case(self): self.stride = [1, 1, 1] self.dilations = [1, 1, 1] @@ -381,10 +407,10 @@ class TestCUDNNWithVALIDPad(TestWithVALIDPad): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride(TestWithStride): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [2, 2, 2] @@ -399,10 +425,10 @@ class TestCUDNNWithStride(TestWithStride): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups(TestWithGroups): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -432,10 +458,10 @@ class TestCUDNNWithGroups(TestWithGroups): # self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -451,10 +477,10 @@ class TestCUDNN_NHWC(TestConv3DTransposeOp): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad_NHWC(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -470,10 +496,10 @@ class TestCUDNNWithSymmetricPad_NHWC(TestWithSymmetricPad): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad_NHWC(TestWithAsymmetricPad): - def init_test_case(self): self.pad = [1, 0, 1, 0, 0, 2] self.stride = [1, 1, 1] @@ -489,10 +515,10 @@ class TestCUDNNWithAsymmetricPad_NHWC(TestWithAsymmetricPad): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride_NHWC(TestWithStride): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [2, 2, 2] @@ -508,10 +534,10 @@ class TestCUDNNWithStride_NHWC(TestWithStride): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups_NHWC(TestWithGroups): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py index 12d21378215..0aed6c1c515 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py @@ -21,7 +21,6 @@ from test_conv3d_transpose_op import TestConv3DTransposeOp class TestWithSymmetricPad_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -34,7 +33,6 @@ class TestWithSymmetricPad_NHWC(TestConv3DTransposeOp): class TestWithAsymmetricPad_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 0, 1, 2] self.stride = [1, 1, 1] @@ -47,7 +45,6 @@ class TestWithAsymmetricPad_NHWC(TestConv3DTransposeOp): class TestWithGroups_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_filter = True self.pad = [1, 1, 1] @@ -61,7 +58,6 @@ class TestWithGroups_NHWC(TestConv3DTransposeOp): class TestWithStride_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [2, 2, 2] @@ -74,7 +70,6 @@ class TestWithStride_NHWC(TestConv3DTransposeOp): class TestWithDilation_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_input = True self.pad = [1, 1, 1] @@ -88,57 +83,68 @@ class TestWithDilation_NHWC(TestConv3DTransposeOp): class TestConv3DTransposeAPI(unittest.TestCase): - def test_case1(self): - data1 = fluid.layers.data(name='data1', - shape=[3, 5, 5, 5], - dtype='float32') - data2 = fluid.layers.data(name='data2', - shape=[5, 5, 5, 3], - dtype='float32') - - out1 = fluid.layers.conv3d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - data_format='NCDHW') - out2 = fluid.layers.conv3d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - data_format='NDHWC') - out3 = fluid.layers.conv3d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [1, 1], - [0, 0], [1, 1]], - data_format='NCDHW') - out4 = fluid.layers.conv3d_transpose(input=data2, - groups=3, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [1, 1], - [1, 2], [0, 0]], - data_format='NDHWC') - out5 = fluid.layers.conv3d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='SAME', - data_format='NCDHW') - out6 = fluid.layers.conv3d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='VALID', - data_format='NDHWC') - out7 = fluid.layers.conv3d_transpose(input=data2, - groups=1, - num_filters=6, - output_size=[7, 7, 7], - padding=[0, 0, 0], - data_format='NDHWC') + data1 = fluid.layers.data( + name='data1', shape=[3, 5, 5, 5], dtype='float32' + ) + data2 = fluid.layers.data( + name='data2', shape=[5, 5, 5, 3], dtype='float32' + ) + + out1 = fluid.layers.conv3d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + data_format='NCDHW', + ) + out2 = fluid.layers.conv3d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + data_format='NDHWC', + ) + out3 = fluid.layers.conv3d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [1, 1], [0, 0], [1, 1]], + data_format='NCDHW', + ) + out4 = fluid.layers.conv3d_transpose( + input=data2, + groups=3, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [1, 1], [1, 2], [0, 0]], + data_format='NDHWC', + ) + out5 = fluid.layers.conv3d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='SAME', + data_format='NCDHW', + ) + out6 = fluid.layers.conv3d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='VALID', + data_format='NDHWC', + ) + out7 = fluid.layers.conv3d_transpose( + input=data2, + groups=1, + num_filters=6, + output_size=[7, 7, 7], + padding=[0, 0, 0], + data_format='NDHWC', + ) data1_np = np.random.random((2, 3, 5, 5, 5)).astype("float32") data2_np = np.random.random((2, 5, 5, 5, 3)).astype("float32") @@ -149,13 +155,12 @@ class TestConv3DTransposeAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2, out3, out4, out5, out6, out7], + return_numpy=True, + ) self.assertIsNotNone(results[0]) self.assertIsNotNone(results[1]) self.assertIsNotNone(results[2]) @@ -166,48 +171,53 @@ class TestConv3DTransposeAPI(unittest.TestCase): class TestConv3DTransposeOpException(unittest.TestCase): - def test_exception(self): - data = fluid.layers.data(name='data', - shape=[3, 5, 5, 5], - dtype="float32") + data = fluid.layers.data( + name='data', shape=[3, 5, 5, 5], dtype="float32" + ) def attr_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - data_format="NCDW") + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + data_format="NCDW", + ) self.assertRaises(ValueError, attr_data_format) def attr_padding_str(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding='Vald') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding='Vald', + ) self.assertRaises(ValueError, attr_padding_str) def attr_padding_list(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [1, 1], [0, 0], - [0, 0], [1, 1]]) + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [1, 1], [0, 0], [0, 0], [1, 1]], + ) self.assertRaises(ValueError, attr_padding_list) def attr_padding_with_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [0, 0], [0, 0], - [1, 0], [1, 1]], - data_format='NDHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [0, 0], [0, 0], [1, 0], [1, 1]], + data_format='NDHWC', + ) self.assertRaises(ValueError, attr_padding_with_data_format) diff --git a/python/paddle/fluid/tests/unittests/test_conv_nn_grad.py b/python/paddle/fluid/tests/unittests/test_conv_nn_grad.py index 99d457c073e..b09a86f5bfa 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_conv_nn_grad.py @@ -25,7 +25,6 @@ from decorator_helper import prog_scope class TestConvDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 3, 3] @@ -39,11 +38,9 @@ class TestConvDoubleGradCheck(unittest.TestCase): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -55,7 +52,6 @@ class TestConvDoubleGradCheck(unittest.TestCase): class TestConvDoubleGradCheckTest0(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 3, 3] @@ -69,11 +65,9 @@ class TestConvDoubleGradCheckTest0(unittest.TestCase): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -84,7 +78,6 @@ class TestConvDoubleGradCheckTest0(unittest.TestCase): class TestConvDoubleGradCheckTest1(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 3, 3, 3] @@ -98,11 +91,9 @@ class TestConvDoubleGradCheckTest1(unittest.TestCase): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -113,7 +104,6 @@ class TestConvDoubleGradCheckTest1(unittest.TestCase): class TestConv3DDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 3, 4, 2] @@ -127,14 +117,12 @@ class TestConv3DDoubleGradCheck(unittest.TestCase): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): - #places = [fluid.CPUPlace()] + # places = [fluid.CPUPlace()] places = [] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -143,7 +131,6 @@ class TestConv3DDoubleGradCheck(unittest.TestCase): class TestConv3DDoubleGradCheckTest1(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 5, 3, 2] @@ -157,11 +144,9 @@ class TestConv3DDoubleGradCheckTest1(unittest.TestCase): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -172,30 +157,29 @@ class TestConv3DDoubleGradCheckTest1(unittest.TestCase): class TestConv2DoubleGradCheck_AsyPadding(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 0, 1], - bias_attr=False, - use_cudnn=True) + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 0, 1], + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -206,30 +190,29 @@ class TestConv2DoubleGradCheck_AsyPadding(unittest.TestCase): class TestConv2DoubleGradCheck_PaddingSAME(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding="SAME", - bias_attr=False, - use_cudnn=True) + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding="SAME", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -240,30 +223,29 @@ class TestConv2DoubleGradCheck_PaddingSAME(unittest.TestCase): class TestConv2DoubleGradCheck_PaddingVALID(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding="VALID", - bias_attr=False, - use_cudnn=True) + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding="VALID", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -274,32 +256,31 @@ class TestConv2DoubleGradCheck_PaddingVALID(unittest.TestCase): class TestConv2DoubleGradCheck_ChannelLast(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 1], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NHWC") + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 1], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -310,32 +291,31 @@ class TestConv2DoubleGradCheck_ChannelLast(unittest.TestCase): class TestConv2DoubleGradCheck_ChannelLast_AsyPadding(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 1, 0], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NHWC") + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 1, 0], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -346,30 +326,29 @@ class TestConv2DoubleGradCheck_ChannelLast_AsyPadding(unittest.TestCase): class TestConv3DDoubleGradCheck_AsyPadding(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 2, 2, 2] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 0, 1, 1, 2], - bias_attr=False, - use_cudnn=True) + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 0, 1, 1, 2], + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -380,31 +359,30 @@ class TestConv3DDoubleGradCheck_AsyPadding(unittest.TestCase): class TestConv3DoubleGradCheck_PaddingSAME(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 2, 2, 2] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding="SAME", - groups=1, - bias_attr=False, - use_cudnn=True) + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding="SAME", + groups=1, + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -415,30 +393,29 @@ class TestConv3DoubleGradCheck_PaddingSAME(unittest.TestCase): class TestConv3DoubleGradCheck_PaddingVALID(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3, 2] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding="VALID", - bias_attr=False, - use_cudnn=True) + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding="VALID", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -449,32 +426,31 @@ class TestConv3DoubleGradCheck_PaddingVALID(unittest.TestCase): class TestConv3DDoubleGradCheck_ChannelLast(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 2, 2, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 1, 1], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NDHWC") + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 1, 1], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NDHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -485,32 +461,31 @@ class TestConv3DDoubleGradCheck_ChannelLast(unittest.TestCase): class TestConv3DDoubleGradCheck_ChannelLast_AsyPadding(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 2, 2, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 1, 0, 1, 0], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NDHWC") + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 1, 0, 1, 0], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NDHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -521,7 +496,6 @@ class TestConv3DDoubleGradCheck_ChannelLast_AsyPadding(unittest.TestCase): class TestDepthWiseConvDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 3, 3] @@ -533,23 +507,18 @@ class TestDepthWiseConvDoubleGradCheck(unittest.TestCase): # use_cudnn == False # groups == filters # num_filters % num_channels == 0 - y = layers.conv2d(x, - shape[1], - 1, - groups=shape[1], - bias_attr=False, - use_cudnn=False) + y = layers.conv2d( + x, shape[1], 1, groups=shape[1], bias_attr=False, use_cudnn=False + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [] @@ -560,7 +529,6 @@ class TestDepthWiseConvDoubleGradCheck(unittest.TestCase): class TestDepthWiseConvDoubleGradCheckCase1(unittest.TestCase): - def depthwise_conv2d_wrapper(self, x): return paddle.nn.functional.conv2d(x[0], x[1], groups=4) @@ -582,16 +550,16 @@ class TestDepthWiseConvDoubleGradCheckCase1(unittest.TestCase): x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) w_arr = np.random.uniform(-1, 1, w_shape).astype(dtype) - gradient_checker.double_grad_check([x, w], - y, - x_init=[x_arr, w_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, w], y, x_init=[x_arr, w_arr], place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.depthwise_conv2d_wrapper, [x, w], + self.depthwise_conv2d_wrapper, + [x, w], y, x_init=[x_arr, w_arr], - place=place) + place=place, + ) def test_grad(self): places = [] @@ -602,7 +570,6 @@ class TestDepthWiseConvDoubleGradCheckCase1(unittest.TestCase): class TestConv3DDoubleGradCheck_NN(unittest.TestCase): - def conv3d_wrapper(self, x): return paddle.nn.functional.conv3d(x[0], x[1]) @@ -620,16 +587,12 @@ class TestConv3DDoubleGradCheck_NN(unittest.TestCase): x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) w_arr = np.random.uniform(-1, 1, w_shape).astype(dtype) - gradient_checker.double_grad_check([x, w], - y, - x_init=[x_arr, w_arr], - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.conv3d_wrapper, - [x, w], - y, - x_init=[x_arr, w_arr], - place=place) + gradient_checker.double_grad_check( + [x, w], y, x_init=[x_arr, w_arr], place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.conv3d_wrapper, [x, w], y, x_init=[x_arr, w_arr], place=place + ) def test_grad(self): places = [] diff --git a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py index aa9d612c5db..57a2038d3f5 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py @@ -29,7 +29,6 @@ def conv_shift_forward(x, y): class TestConvShiftOp(OpTest): - def setUp(self): self.op_type = "conv_shift" diff --git a/python/paddle/fluid/tests/unittests/test_conv_transpose_nn_grad.py b/python/paddle/fluid/tests/unittests/test_conv_transpose_nn_grad.py index 235f4a64f85..f4c139a5463 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_transpose_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_conv_transpose_nn_grad.py @@ -25,7 +25,6 @@ from decorator_helper import prog_scope class TestConvTransposeDoubleGradCheck(unittest.TestCase): - def conv_transpose_wrapper(self, x): return paddle.nn.functional.conv2d_transpose(x[0], x[1], groups=1) @@ -37,11 +36,9 @@ class TestConvTransposeDoubleGradCheck(unittest.TestCase): if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(x, - 2, - filter_size=1, - groups=1, - bias_attr=False) + y = layers.conv2d_transpose( + x, 2, filter_size=1, groups=1, bias_attr=False + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -50,23 +47,25 @@ class TestConvTransposeDoubleGradCheck(unittest.TestCase): w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) def test_grad(self): places = [] @@ -78,13 +77,12 @@ class TestConvTransposeDoubleGradCheck(unittest.TestCase): class TestConvTranspose2DoubleGradCheck_AsyPadding( - TestConvTransposeDoubleGradCheck): - + TestConvTransposeDoubleGradCheck +): def conv_transpose_wrapper(self, x): - return paddle.nn.functional.conv2d_transpose(x[0], - x[1], - groups=1, - padding=[1, 0, 0, 1]) + return paddle.nn.functional.conv2d_transpose( + x[0], x[1], groups=1, padding=[1, 0, 0, 1] + ) @prog_scope() def func(self, place): @@ -94,12 +92,14 @@ class TestConvTranspose2DoubleGradCheck_AsyPadding( if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 0, 1], - bias_attr=False, - use_cudnn=True) + y = layers.conv2d_transpose( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 0, 1], + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -108,33 +108,34 @@ class TestConvTranspose2DoubleGradCheck_AsyPadding( w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) class TestConvTranspose2DoubleGradCheck_PaddingSAME( - TestConvTransposeDoubleGradCheck): - + TestConvTransposeDoubleGradCheck +): def conv_transpose_wrapper(self, x): - return paddle.nn.functional.conv2d_transpose(x[0], - x[1], - groups=1, - padding="SAME") + return paddle.nn.functional.conv2d_transpose( + x[0], x[1], groups=1, padding="SAME" + ) @prog_scope() def func(self, place): @@ -144,12 +145,14 @@ class TestConvTranspose2DoubleGradCheck_PaddingSAME( if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(input=x, - num_filters=2, - filter_size=1, - padding="SAME", - bias_attr=False, - use_cudnn=True) + y = layers.conv2d_transpose( + input=x, + num_filters=2, + filter_size=1, + padding="SAME", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -158,33 +161,34 @@ class TestConvTranspose2DoubleGradCheck_PaddingSAME( w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) class TestConvTranspose2DoubleGradCheck_PaddingVALID( - TestConvTransposeDoubleGradCheck): - + TestConvTransposeDoubleGradCheck +): def conv_transpose_wrapper(self, x): - return paddle.nn.functional.conv2d_transpose(x[0], - x[1], - groups=1, - padding="VALID") + return paddle.nn.functional.conv2d_transpose( + x[0], x[1], groups=1, padding="VALID" + ) @prog_scope() def func(self, place): @@ -194,12 +198,14 @@ class TestConvTranspose2DoubleGradCheck_PaddingVALID( if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(input=x, - num_filters=2, - filter_size=1, - padding="VALID", - bias_attr=False, - use_cudnn=True) + y = layers.conv2d_transpose( + input=x, + num_filters=2, + filter_size=1, + padding="VALID", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -208,34 +214,34 @@ class TestConvTranspose2DoubleGradCheck_PaddingVALID( w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) class TestConvTranspose2DoubleGradCheck_ChannelLast( - TestConvTransposeDoubleGradCheck): - + TestConvTransposeDoubleGradCheck +): def conv_transpose_wrapper(self, x): - return paddle.nn.functional.conv2d_transpose(x[0], - x[1], - groups=1, - padding=[1, 1], - data_format="NHWC") + return paddle.nn.functional.conv2d_transpose( + x[0], x[1], groups=1, padding=[1, 1], data_format="NHWC" + ) @prog_scope() def func(self, place): @@ -245,14 +251,16 @@ class TestConvTranspose2DoubleGradCheck_ChannelLast( if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(input=x, - num_filters=2, - filter_size=1, - padding=[1, 1], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NHWC") + y = layers.conv2d_transpose( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 1], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -261,23 +269,25 @@ class TestConvTranspose2DoubleGradCheck_ChannelLast( w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_corr.py b/python/paddle/fluid/tests/unittests/test_corr.py index 252948d8eed..82f28d3cd00 100644 --- a/python/paddle/fluid/tests/unittests/test_corr.py +++ b/python/paddle/fluid/tests/unittests/test_corr.py @@ -28,7 +28,6 @@ def numpy_corr(np_arr, rowvar=True, dtype='float64'): class Corr_Test(unittest.TestCase): - def setUp(self): self.shape = [4, 5] @@ -49,14 +48,13 @@ class Corr_Test(unittest.TestCase): corr = paddle.linalg.corrcoef(tensor) np_corr = numpy_corr(np_arr, rowvar=True, dtype=dtype) if dtype == 'float32': - np.testing.assert_allclose(np_corr, - corr.numpy(), - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + np_corr, corr.numpy(), rtol=1e-05, atol=1e-05 + ) else: - np.testing.assert_allclose(np_corr, - corr.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np_corr, corr.numpy(), rtol=1e-05 + ) def test_tensor_corr_rowvar(self): typelist = ['float64', 'float32'] @@ -76,37 +74,32 @@ class Corr_Test(unittest.TestCase): corr = paddle.linalg.corrcoef(tensor, rowvar=False) np_corr = numpy_corr(np_arr, rowvar=False, dtype=dtype) if dtype == 'float32': - np.testing.assert_allclose(np_corr, - corr.numpy(), - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + np_corr, corr.numpy(), rtol=1e-05, atol=1e-05 + ) else: - np.testing.assert_allclose(np_corr, - corr.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np_corr, corr.numpy(), rtol=1e-05 + ) # Input(x) only support N-D (1<=N<=2) tensor class Corr_Test2(Corr_Test): - def setUp(self): self.shape = [10] class Corr_Test3(Corr_Test): - def setUp(self): self.shape = [4, 5] # Input(x) only support N-D (1<=N<=2) tensor class Corr_Test4(unittest.TestCase): - def setUp(self): self.shape = [2, 5, 2] def test_errors(self): - def test_err(): np_arr = np.random.rand(*self.shape).astype('float64') tensor = paddle.to_tensor(np_arr) @@ -117,7 +110,6 @@ class Corr_Test4(unittest.TestCase): # test unsupported complex input class Corr_Comeplex_Test(unittest.TestCase): - def setUp(self): self.dtype = 'complex128' @@ -129,7 +121,6 @@ class Corr_Comeplex_Test(unittest.TestCase): class Corr_Test5(Corr_Comeplex_Test): - def setUp(self): self.dtype = 'complex64' diff --git a/python/paddle/fluid/tests/unittests/test_cos_sim_op.py b/python/paddle/fluid/tests/unittests/test_cos_sim_op.py index 5943214e9a3..86d1e0e1a8c 100644 --- a/python/paddle/fluid/tests/unittests/test_cos_sim_op.py +++ b/python/paddle/fluid/tests/unittests/test_cos_sim_op.py @@ -20,21 +20,23 @@ from paddle.fluid import Program, program_guard class TestCosSimOp(OpTest): - def setUp(self): self.op_type = "cos_sim" self.inputs = { 'X': np.random.random((6, 20)).astype("float32"), - 'Y': np.random.random((6, 20)).astype("float32") + 'Y': np.random.random((6, 20)).astype("float32"), } expect_x_norm = np.linalg.norm(self.inputs['X'], axis=1) expect_y_norm = np.linalg.norm(self.inputs['Y'], axis=1) - expect_out = (self.inputs['X'] * self.inputs['Y']).sum(axis=1) / \ - expect_x_norm / expect_y_norm + expect_out = ( + (self.inputs['X'] * self.inputs['Y']).sum(axis=1) + / expect_x_norm + / expect_y_norm + ) self.outputs = { 'XNorm': np.expand_dims(expect_x_norm, 1), 'YNorm': np.expand_dims(expect_y_norm, 1), - 'Out': np.expand_dims(expect_out, 1) + 'Out': np.expand_dims(expect_out, 1), } def test_check_output(self): @@ -44,84 +46,89 @@ class TestCosSimOp(OpTest): self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.06) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.06, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.06, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.06, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.06, no_grad_set=set('Y') + ) class TestCosSimOp2(TestCosSimOp): - def setUp(self): self.op_type = "cos_sim" self.inputs = { 'X': np.random.random((6, 100)).astype("float32"), - 'Y': np.random.random((1, 100)).astype("float32") + 'Y': np.random.random((1, 100)).astype("float32"), } expect_x_norm = np.linalg.norm(self.inputs['X'], axis=1) expect_y_norm = np.linalg.norm(self.inputs['Y'], axis=1) - expect_out = (self.inputs['X'] * self.inputs['Y']).sum(axis=1) / \ - expect_x_norm / expect_y_norm + expect_out = ( + (self.inputs['X'] * self.inputs['Y']).sum(axis=1) + / expect_x_norm + / expect_y_norm + ) self.outputs = { 'XNorm': np.expand_dims(expect_x_norm, 1), 'YNorm': np.expand_dims(expect_y_norm, 1), - 'Out': np.expand_dims(expect_out, 1) + 'Out': np.expand_dims(expect_out, 1), } class TestCosSimOp3(TestCosSimOp): - def setUp(self): self.op_type = "cos_sim" self.inputs = { 'X': np.random.random((6, 5, 4)).astype("float32"), - 'Y': np.random.random((6, 5, 4)).astype("float32") + 'Y': np.random.random((6, 5, 4)).astype("float32"), } expect_x_norm = np.linalg.norm(self.inputs['X'], axis=(1, 2)) expect_y_norm = np.linalg.norm(self.inputs['Y'], axis=(1, 2)) - expect_out = (self.inputs['X'] * self.inputs['Y']).sum(axis=(1, 2)) / \ - expect_x_norm / expect_y_norm + expect_out = ( + (self.inputs['X'] * self.inputs['Y']).sum(axis=(1, 2)) + / expect_x_norm + / expect_y_norm + ) self.outputs = { 'XNorm': np.expand_dims(expect_x_norm, 1), 'YNorm': np.expand_dims(expect_y_norm, 1), - 'Out': np.expand_dims(expect_out, 1) + 'Out': np.expand_dims(expect_out, 1), } class TestCosSimOp4(TestCosSimOp): - def setUp(self): self.op_type = "cos_sim" self.inputs = { 'X': np.random.random((6, 5, 20)).astype("float32"), - 'Y': np.random.random((1, 5, 20)).astype("float32") + 'Y': np.random.random((1, 5, 20)).astype("float32"), } expect_x_norm = np.linalg.norm(self.inputs['X'], axis=(1, 2)) expect_y_norm = np.linalg.norm(self.inputs['Y'], axis=(1, 2)) - expect_out = (self.inputs['X'] * self.inputs['Y']).sum(axis=(1, 2)) / \ - expect_x_norm / expect_y_norm + expect_out = ( + (self.inputs['X'] * self.inputs['Y']).sum(axis=(1, 2)) + / expect_x_norm + / expect_y_norm + ) self.outputs = { 'XNorm': np.expand_dims(expect_x_norm, 1), 'YNorm': np.expand_dims(expect_y_norm, 1), - 'Out': np.expand_dims(expect_out, 1) + 'Out': np.expand_dims(expect_out, 1), } class TestCosSimOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of batch_norm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - x2 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + x2 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.cos_sim, x1, x2) # the input dtype of batch_norm must be float32 diff --git a/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py b/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py index 765ed8a67ac..784e583b4f5 100644 --- a/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py +++ b/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py @@ -39,7 +39,6 @@ def cosine_embedding_loss(input1, input2, label, margin=0.5, reduction='mean'): class TestFunctionCosineEmbeddingLoss(unittest.TestCase): - def setUp(self): self.input1_np = np.random.random(size=(5, 3)).astype(np.float64) self.input2_np = np.random.random(size=(5, 3)).astype(np.float64) @@ -52,43 +51,43 @@ class TestFunctionCosineEmbeddingLoss(unittest.TestCase): input1 = paddle.to_tensor(self.input1_np) input2 = paddle.to_tensor(self.input2_np) label = paddle.to_tensor(self.label_np) - dy_result = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') - expected1 = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='mean') + dy_result = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) + expected1 = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(dy_result.numpy(), expected1, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) - dy_result = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='sum') - expected2 = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='sum') + dy_result = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='sum' + ) + expected2 = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='sum', + ) np.testing.assert_allclose(dy_result.numpy(), expected2, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) - dy_result = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='none') - expected3 = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='none') + dy_result = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='none' + ) + expected3 = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='none', + ) np.testing.assert_allclose(dy_result.numpy(), expected3, rtol=1e-05) self.assertTrue(dy_result.shape, [5]) @@ -97,50 +96,52 @@ class TestFunctionCosineEmbeddingLoss(unittest.TestCase): input1 = static.data(name='input1', shape=[5, 3], dtype='float64') input2 = static.data(name='input2', shape=[5, 3], dtype='float64') label = static.data(name='label', shape=[5], dtype='int32') - result0 = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='none') - result1 = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='sum') - result2 = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + result0 = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='none' + ) + result1 = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='sum' + ) + result2 = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) place = paddle.CUDAPlace(0) if use_gpu else paddle.CPUPlace() exe = static.Executor(place) exe.run(static.default_startup_program()) - static_result = exe.run(feed={ - "input1": self.input1_np, - "input2": self.input2_np, - "label": self.label_np - }, - fetch_list=[result0, result1, result2]) - expected = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='none') + static_result = exe.run( + feed={ + "input1": self.input1_np, + "input2": self.input2_np, + "label": self.label_np, + }, + fetch_list=[result0, result1, result2], + ) + expected = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='none', + ) np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) - expected = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='sum') + expected = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='sum', + ) np.testing.assert_allclose(static_result[1], expected, rtol=1e-05) - expected = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='mean') + expected = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(static_result[2], expected, rtol=1e-05) @@ -171,72 +172,62 @@ class TestFunctionCosineEmbeddingLoss(unittest.TestCase): def test_label_shape_error(): label = paddle.to_tensor( - np.random.randint(low=0, high=2, size=(2, 3))) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + np.random.randint(low=0, high=2, size=(2, 3)) + ) + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_label_shape_error) def test_input_different_shape_error(): input1 = paddle.to_tensor(self.input1_np[0]) label = paddle.to_tensor(np.ndarray([1])) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_input_different_shape_error) def test_input_shape2D_error(): input1 = paddle.to_tensor( - np.random.random(size=(2, 3, 4)).astype(np.float64)) + np.random.random(size=(2, 3, 4)).astype(np.float64) + ) input2 = paddle.to_tensor( - np.random.random(size=(2, 3, 4)).astype(np.float64)) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + np.random.random(size=(2, 3, 4)).astype(np.float64) + ) + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_input_shape2D_error) def test_label_value_error(): label = paddle.to_tensor(np.ndarray([-1, -2])) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_label_value_error) def test_input_type_error(): input1 = paddle.to_tensor(self.input1_np.astype(np.int64)) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_input_type_error) def test_label_type_error(): label = paddle.to_tensor(self.label_np.astype(np.int16)) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_label_type_error) class TestClassCosineEmbeddingLoss(unittest.TestCase): - def setUp(self): self.input1_np = np.random.random(size=(10, 3)).astype(np.float32) self.input2_np = np.random.random(size=(10, 3)).astype(np.float32) @@ -252,14 +243,17 @@ class TestClassCosineEmbeddingLoss(unittest.TestCase): input1 = paddle.to_tensor(self.input1_np) input2 = paddle.to_tensor(self.input2_np) label = paddle.to_tensor(self.label_np) - CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss(margin=0.5, - reduction='mean') + CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss( + margin=0.5, reduction='mean' + ) dy_result = CosineEmbeddingLoss(input1, input2, label) - expected1 = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='mean') + expected1 = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(dy_result.numpy(), expected1, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) @@ -267,35 +261,42 @@ class TestClassCosineEmbeddingLoss(unittest.TestCase): input2_1D = paddle.to_tensor(self.input2_np_1D) label_1D = paddle.to_tensor(self.label_np_1D) dy_result = CosineEmbeddingLoss(input1_1D, input2_1D, label_1D) - expected2 = cosine_embedding_loss(self.input1_np_1D, - self.input2_np_1D, - self.label_np_1D, - margin=0.5, - reduction='mean') + expected2 = cosine_embedding_loss( + self.input1_np_1D, + self.input2_np_1D, + self.label_np_1D, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(dy_result.numpy(), expected2, rtol=1e-05) def run_static(self): input1 = static.data(name='input1', shape=[10, 3], dtype='float32') input2 = static.data(name='input2', shape=[10, 3], dtype='float32') label = static.data(name='label', shape=[10], dtype='int64') - CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss(margin=0.5, - reduction='mean') + CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss( + margin=0.5, reduction='mean' + ) result = CosineEmbeddingLoss(input1, input2, label) place = paddle.CPUPlace() exe = static.Executor(place) exe.run(static.default_startup_program()) - static_result = exe.run(feed={ - "input1": self.input1_np, - "input2": self.input2_np, - "label": self.label_np - }, - fetch_list=[result]) - expected = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='mean') + static_result = exe.run( + feed={ + "input1": self.input1_np, + "input2": self.input2_np, + "label": self.label_np, + }, + fetch_list=[result], + ) + expected = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) @@ -308,16 +309,17 @@ class TestClassCosineEmbeddingLoss(unittest.TestCase): self.run_static() def test_errors(self): - def test_margin_error(): CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss( - margin=2, reduction='mean') + margin=2, reduction='mean' + ) self.assertRaises(ValueError, test_margin_error) def test_reduction_error(): CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss( - margin=2, reduction='reduce_mean') + margin=2, reduction='reduce_mean' + ) self.assertRaises(ValueError, test_reduction_error) diff --git a/python/paddle/fluid/tests/unittests/test_cosine_similarity_api.py b/python/paddle/fluid/tests/unittests/test_cosine_similarity_api.py index db0bb0147cd..1c658d0ecc4 100644 --- a/python/paddle/fluid/tests/unittests/test_cosine_similarity_api.py +++ b/python/paddle/fluid/tests/unittests/test_cosine_similarity_api.py @@ -23,7 +23,6 @@ from paddle.fluid import Program, program_guard, Executor, default_main_program class TestCosineSimilarityAPI(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if core.is_compiled_with_cuda(): @@ -52,12 +51,11 @@ class TestCosineSimilarityAPI(unittest.TestCase): x2 = paddle.fluid.data(name="x2", shape=shape) result = F.cosine_similarity(x1, x2, axis=axis, eps=eps) exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={ - "x1": np_x1, - "x2": np_x2 - }, - fetch_list=[result]) + fetches = exe.run( + default_main_program(), + feed={"x1": np_x1, "x2": np_x2}, + fetch_list=[result], + ) np_out = self._get_numpy_out(np_x1, np_x2, axis=axis, eps=eps) np.testing.assert_allclose(fetches[0], np_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_cost_model.py b/python/paddle/fluid/tests/unittests/test_cost_model.py index ba9b4c35721..92fa062ad7a 100644 --- a/python/paddle/fluid/tests/unittests/test_cost_model.py +++ b/python/paddle/fluid/tests/unittests/test_cost_model.py @@ -24,13 +24,13 @@ device = "gpu" if core.is_compiled_with_cuda() else "cpu" class TestCostModel(unittest.TestCase): - def test_profiler_measure_empty_program(self): cost_model = core.CostModel() empty_program = paddle.static.Program() startup_program = paddle.static.Program() - cost_data = cost_model.profile_measure(empty_program, startup_program, - device, ["time"]) + cost_data = cost_model.profile_measure( + empty_program, startup_program, device, ["time"] + ) self.assertEqual(cost_data.get_whole_time_ms(), 0) def test_profiler_measure_program(self): @@ -42,14 +42,16 @@ class TestCostModel(unittest.TestCase): hidden = paddle.static.nn.fc(data, 10) loss = paddle.mean(hidden) cost_model = core.CostModel() - cost_data = cost_model.profile_measure(main_program, startup_program, - device, ["time"]) + cost_data = cost_model.profile_measure( + main_program, startup_program, device, ["time"] + ) fc_op_time = cost_data.get_op_time_ms(0) mean_op_time = cost_data.get_op_time_ms(1) self.assertGreater(fc_op_time, 0) self.assertGreater(mean_op_time, 0) - self.assertGreaterEqual(cost_data.get_whole_time_ms(), - fc_op_time + mean_op_time) + self.assertGreaterEqual( + cost_data.get_whole_time_ms(), fc_op_time + mean_op_time + ) def test_static_op_benchmark_cost_model(self): op_name = "abs" @@ -70,16 +72,18 @@ class TestCostModel(unittest.TestCase): print("conv2d_op_time:", conv2d_op_time) print("conv2d_op_config:", conv2d_op_config) - conv2d_backward_op_cost = cost_model.get_static_op_time("conv2d", - forward=False) + conv2d_backward_op_cost = cost_model.get_static_op_time( + "conv2d", forward=False + ) conv2d_backward_op_time = conv2d_backward_op_cost["op_time"] conv2d_backward_op_config = conv2d_backward_op_cost["config"] self.assertGreater(float(conv2d_backward_op_time), 0) print("conv2d_backward_op_time:", conv2d_backward_op_time) print("conv2d_backward_op_config:", conv2d_backward_op_config) - conv2d_fp16_op_cost = cost_model.get_static_op_time("conv2d", - dtype="float16") + conv2d_fp16_op_cost = cost_model.get_static_op_time( + "conv2d", dtype="float16" + ) conv2d_fp16_op_time = conv2d_fp16_op_cost["op_time"] conv2d_fp16_op_config = conv2d_fp16_op_cost["config"] self.assertGreater(float(conv2d_fp16_op_time), 0) diff --git a/python/paddle/fluid/tests/unittests/test_count_nonzero_api.py b/python/paddle/fluid/tests/unittests/test_count_nonzero_api.py index 48a0f08fd4a..55f34dd2612 100644 --- a/python/paddle/fluid/tests/unittests/test_count_nonzero_api.py +++ b/python/paddle/fluid/tests/unittests/test_count_nonzero_api.py @@ -26,8 +26,11 @@ class TestCountNonzeroAPI(unittest.TestCase): def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_api_static(self): paddle.enable_static() @@ -40,8 +43,9 @@ class TestCountNonzeroAPI(unittest.TestCase): out4 = paddle.count_nonzero(x, axis) out5 = paddle.count_nonzero(x, tuple(axis)) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': self.x}, - fetch_list=[out1, out2, out3, out4, out5]) + res = exe.run( + feed={'X': self.x}, fetch_list=[out1, out2, out3, out4, out5] + ) out_ref = np.count_nonzero(self.x) for out in res: np.testing.assert_allclose(out, out_ref, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_cov.py b/python/paddle/fluid/tests/unittests/test_cov.py index d7fec9028ab..b00fc5b3fe2 100644 --- a/python/paddle/fluid/tests/unittests/test_cov.py +++ b/python/paddle/fluid/tests/unittests/test_cov.py @@ -20,15 +20,16 @@ from paddle.fluid.framework import _test_eager_guard def numpy_cov(np_arr, rowvar=True, ddof=1, fweights=None, aweights=None): - return np.cov(np_arr, - rowvar=rowvar, - ddof=int(ddof), - fweights=fweights, - aweights=aweights) + return np.cov( + np_arr, + rowvar=rowvar, + ddof=int(ddof), + fweights=fweights, + aweights=aweights, + ) class Cov_Test(unittest.TestCase): - def setUp(self): self.shape = [20, 10] self.weightshape = [10] @@ -48,16 +49,12 @@ class Cov_Test(unittest.TestCase): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) tensor = paddle.to_tensor(np_arr, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=None, - aweights=None) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=1, - fweights=None, - aweights=None) + cov = paddle.linalg.cov( + tensor, rowvar=True, ddof=True, fweights=None, aweights=None + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=1, fweights=None, aweights=None + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_default(self): @@ -80,16 +77,16 @@ class Cov_Test(unittest.TestCase): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) tensor = paddle.to_tensor(np_arr, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=False, - ddof=True, - fweights=None, - aweights=None) - np_cov = numpy_cov(np_arr, - rowvar=False, - ddof=1, - fweights=None, - aweights=None) + cov = paddle.linalg.cov( + tensor, + rowvar=False, + ddof=True, + fweights=None, + aweights=None, + ) + np_cov = numpy_cov( + np_arr, rowvar=False, ddof=1, fweights=None, aweights=None + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_rowvar(self): @@ -112,16 +109,16 @@ class Cov_Test(unittest.TestCase): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) tensor = paddle.to_tensor(np_arr, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=False, - fweights=None, - aweights=None) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=0, - fweights=None, - aweights=None) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=False, + fweights=None, + aweights=None, + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=0, fweights=None, aweights=None + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_ddof(self): @@ -143,20 +140,21 @@ class Cov_Test(unittest.TestCase): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) - np_fw = np.random.randint(10, - size=self.weightshape).astype('int32') + np_fw = np.random.randint(10, size=self.weightshape).astype( + 'int32' + ) tensor = paddle.to_tensor(np_arr, place=p) fweights = paddle.to_tensor(np_fw, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=fweights, - aweights=None) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=1, - fweights=np_fw, - aweights=None) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=True, + fweights=fweights, + aweights=None, + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=1, fweights=np_fw, aweights=None + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_fweights(self): @@ -178,20 +176,21 @@ class Cov_Test(unittest.TestCase): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) - np_aw = np.random.randint(10, - size=self.weightshape).astype('int32') + np_aw = np.random.randint(10, size=self.weightshape).astype( + 'int32' + ) tensor = paddle.to_tensor(np_arr, place=p) aweights = paddle.to_tensor(np_aw, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=None, - aweights=aweights) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=1, - fweights=None, - aweights=np_aw) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=True, + fweights=None, + aweights=aweights, + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=1, fweights=None, aweights=np_aw + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_aweights(self): @@ -213,22 +212,23 @@ class Cov_Test(unittest.TestCase): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) - np_fw = np.random.randint(10, - size=self.weightshape).astype('int64') + np_fw = np.random.randint(10, size=self.weightshape).astype( + 'int64' + ) np_aw = np.random.rand(*self.weightshape).astype('float64') tensor = paddle.to_tensor(np_arr, place=p) fweights = paddle.to_tensor(np_fw, place=p) aweights = paddle.to_tensor(np_aw, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=fweights, - aweights=aweights) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=1, - fweights=np_fw, - aweights=np_aw) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=True, + fweights=fweights, + aweights=aweights, + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=1, fweights=np_fw, aweights=np_aw + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_weights(self): @@ -238,7 +238,6 @@ class Cov_Test(unittest.TestCase): class Cov_Test2(Cov_Test): - def setUp(self): self.shape = [10] self.weightshape = [10] @@ -246,30 +245,32 @@ class Cov_Test2(Cov_Test): # Input(x) only support N-D (1<=N<=2) tensor class Cov_Test3(unittest.TestCase): - def setUp(self): self.shape = [2, 5, 10] self.fweightshape = [10] self.aweightshape = [10] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 def func_test_errors(self): - def test_err(): np_arr = np.random.rand(*self.shape).astype('float64') - np_fw = self.fw_s * np.random.rand( - *self.fweightshape).astype('int32') - np_aw = self.aw_s * np.random.rand( - *self.aweightshape).astype('float64') + np_fw = self.fw_s * np.random.rand(*self.fweightshape).astype( + 'int32' + ) + np_aw = self.aw_s * np.random.rand(*self.aweightshape).astype( + 'float64' + ) tensor = paddle.to_tensor(np_arr) fweights = paddle.to_tensor(np_fw) aweights = paddle.to_tensor(np_aw) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=fweights, - aweights=aweights) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=True, + fweights=fweights, + aweights=aweights, + ) self.assertRaises(ValueError, test_err) @@ -279,70 +280,64 @@ class Cov_Test3(unittest.TestCase): self.func_test_errors() -#Input(fweights) only support N-D (N<=1) tensor +# Input(fweights) only support N-D (N<=1) tensor class Cov_Test4(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [2, 10] self.aweightshape = [10] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 -#The number of Input(fweights) should equal to x's dim[1] +# The number of Input(fweights) should equal to x's dim[1] class Cov_Test5(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [5] self.aweightshape = [10] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 -#The value of Input(fweights) cannot be negtive +# The value of Input(fweights) cannot be negtive class Cov_Test6(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [10] self.aweightshape = [10] - self.fw_s = -1. - self.aw_s = 1. + self.fw_s = -1.0 + self.aw_s = 1.0 -#Input(aweights) only support N-D (N<=1) tensor +# Input(aweights) only support N-D (N<=1) tensor class Cov_Test7(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [10] self.aweightshape = [2, 10] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 -#The number of Input(aweights) should equal to x's dim[1] +# The number of Input(aweights) should equal to x's dim[1] class Cov_Test8(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [10] self.aweightshape = [5] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 -#The value of Input(aweights) cannot be negtive +# The value of Input(aweights) cannot be negtive class Cov_Test9(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [10] self.aweightshape = [10] - self.fw_s = 1. - self.aw_s = -1. + self.fw_s = 1.0 + self.aw_s = -1.0 if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cpuonly_spawn.py b/python/paddle/fluid/tests/unittests/test_cpuonly_spawn.py index 3f5fadff3af..c0e6d173f57 100644 --- a/python/paddle/fluid/tests/unittests/test_cpuonly_spawn.py +++ b/python/paddle/fluid/tests/unittests/test_cpuonly_spawn.py @@ -21,7 +21,6 @@ import paddle.distributed as dist class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear1 = nn.Linear(10, 10) @@ -58,7 +57,6 @@ def train(print_result=False): class TestSpawn(unittest.TestCase): - def test_spawn(self): dist.spawn(train, backend='gloo', nprocs=4) diff --git a/python/paddle/fluid/tests/unittests/test_create_global_var.py b/python/paddle/fluid/tests/unittests/test_create_global_var.py index 6a9f188b363..45dc1e52022 100644 --- a/python/paddle/fluid/tests/unittests/test_create_global_var.py +++ b/python/paddle/fluid/tests/unittests/test_create_global_var.py @@ -19,7 +19,6 @@ from paddle.fluid import Program, program_guard class TestCreateGlobalVarError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): diff --git a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py index af9b162e909..07c89eefc32 100644 --- a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py +++ b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py @@ -17,7 +17,6 @@ import paddle.fluid.layers as layers class TestDocString(unittest.TestCase): - def test_layer_doc_string(self): print(layers.dropout.__doc__) diff --git a/python/paddle/fluid/tests/unittests/test_create_parameter.py b/python/paddle/fluid/tests/unittests/test_create_parameter.py index 53260374c5d..d70e06f1843 100644 --- a/python/paddle/fluid/tests/unittests/test_create_parameter.py +++ b/python/paddle/fluid/tests/unittests/test_create_parameter.py @@ -20,7 +20,6 @@ import paddle class TestCreateParameterError(unittest.TestCase): - def func_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -36,18 +35,18 @@ class TestCreateParameterError(unittest.TestCase): self.assertRaises(TypeError, test_shape_item) def test_attr(): - fluid.layers.create_parameter([1, 2, 3], - np.float32, - attr=np.array( - [i for i in range(6)])) + fluid.layers.create_parameter( + [1, 2, 3], np.float32, attr=np.array([i for i in range(6)]) + ) self.assertRaises(TypeError, test_attr) def test_default_initializer(): - fluid.layers.create_parameter([1, 2, 3], - np.float32, - default_initializer=np.array( - [i for i in range(6)])) + fluid.layers.create_parameter( + [1, 2, 3], + np.float32, + default_initializer=np.array([i for i in range(6)]), + ) self.assertRaises(TypeError, test_default_initializer) diff --git a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py index 2dc0853f719..8336bad6de1 100644 --- a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py +++ b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py @@ -20,10 +20,10 @@ from op_test import OpTest class CRFDecoding(object): - - def __init__(self, emission_weights, transition_weights, - seq_start_positions): - assert (emission_weights.shape[0] == sum(seq_start_positions)) + def __init__( + self, emission_weights, transition_weights, seq_start_positions + ): + assert emission_weights.shape[0] == sum(seq_start_positions) self.tag_num = emission_weights.shape[1] self.seq_num = len(seq_start_positions) @@ -34,10 +34,12 @@ class CRFDecoding(object): self.b = transition_weights[1, :] self.w = transition_weights[2:, :] - self.track = np.zeros((sum(seq_start_positions), self.tag_num), - dtype="int64") - self.decoded_path = np.zeros((sum(seq_start_positions), 1), - dtype="int64") + self.track = np.zeros( + (sum(seq_start_positions), self.tag_num), dtype="int64" + ) + self.decoded_path = np.zeros( + (sum(seq_start_positions), 1), dtype="int64" + ) def _decode_one_sequence(self, decoded_path, x): seq_len, tag_num = x.shape @@ -77,8 +79,9 @@ class CRFDecoding(object): start = cur_pos cur_pos += self.seq_start_positions[i] end = cur_pos - self._decode_one_sequence(self.decoded_path[start:end, :], - self.x[start:end, :]) + self._decode_one_sequence( + self.decoded_path[start:end, :], self.x[start:end, :] + ) return self.decoded_path @@ -98,10 +101,12 @@ class TestCRFDecodingOp1(OpTest): for i in range(SEQ_NUM): lod[-1].append(random.randint(1, MAX_SEQ_LEN)) total_len += lod[-1][-1] - emission = np.random.uniform(-1, 1, - [total_len, TAG_NUM]).astype("float64") - transition = np.random.uniform(-0.5, 0.5, - [TAG_NUM + 2, TAG_NUM]).astype("float64") + emission = np.random.uniform(-1, 1, [total_len, TAG_NUM]).astype( + "float64" + ) + transition = np.random.uniform( + -0.5, 0.5, [TAG_NUM + 2, TAG_NUM] + ).astype("float64") self.inputs = { "Emission": (emission, lod), @@ -136,27 +141,29 @@ class TestCRFDecodingOp2(OpTest): self.init_lod() total_len = sum(self.lod[-1]) - transition = np.repeat(np.arange(TAG_NUM, - dtype="float64").reshape(1, TAG_NUM), - TAG_NUM + 2, - axis=0) - emission = np.repeat(np.arange(TAG_NUM, - dtype="float64").reshape(1, TAG_NUM), - total_len, - axis=0) - - labels = np.random.randint(low=0, - high=TAG_NUM, - size=(total_len, 1), - dtype="int64") - predicted_labels = np.ones( - (total_len, 1), dtype="int64") * (TAG_NUM - 1) + transition = np.repeat( + np.arange(TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + TAG_NUM + 2, + axis=0, + ) + emission = np.repeat( + np.arange(TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + total_len, + axis=0, + ) + + labels = np.random.randint( + low=0, high=TAG_NUM, size=(total_len, 1), dtype="int64" + ) + predicted_labels = np.ones((total_len, 1), dtype="int64") * ( + TAG_NUM - 1 + ) expected_output = (labels == predicted_labels).astype("int64") self.inputs = { "Emission": (emission, self.lod), "Transition": transition, - "Label": (labels, self.lod) + "Label": (labels, self.lod), } self.outputs = {"ViterbiPath": expected_output} @@ -166,13 +173,11 @@ class TestCRFDecodingOp2(OpTest): class TestCRFDecodingOp3(TestCRFDecodingOp2): - def init_lod(self): self.lod = [[1, 0, 0, 4]] class TestCRFDecodingOp4(TestCRFDecodingOp2): - def init_lod(self): self.lod = [[0, 2, 3, 0]] @@ -183,7 +188,7 @@ def seq_pad(data, length): padded = np.zeros(shape).astype(data.dtype) offset = 0 for i, l in enumerate(length): - padded[i, 0:l] = data[offset:offset + l] + padded[i, 0:l] = data[offset : offset + l] offset += l return np.squeeze(padded) @@ -204,10 +209,12 @@ class TestCRFDecodingOp5(OpTest): for i in range(SEQ_NUM): lod[-1].append(random.randint(1, MAX_SEQ_LEN)) total_len += lod[-1][-1] - emission = np.random.uniform(-1, 1, - [total_len, TAG_NUM]).astype("float64") - transition = np.random.uniform(-0.5, 0.5, - [TAG_NUM + 2, TAG_NUM]).astype("float64") + emission = np.random.uniform(-1, 1, [total_len, TAG_NUM]).astype( + "float64" + ) + transition = np.random.uniform( + -0.5, 0.5, [TAG_NUM + 2, TAG_NUM] + ).astype("float64") self.inputs = { "Emission": seq_pad(emission, lod[0]), @@ -229,7 +236,6 @@ class TestCRFDecodingOp5(OpTest): class TestCRFDecodingOp6(OpTest): - def init_lod(self): self.lod = [[1, 2, 3, 4]] @@ -239,21 +245,23 @@ class TestCRFDecodingOp6(OpTest): self.init_lod() total_len = sum(self.lod[-1]) - transition = np.repeat(np.arange(TAG_NUM, - dtype="float64").reshape(1, TAG_NUM), - TAG_NUM + 2, - axis=0) - emission = np.repeat(np.arange(TAG_NUM, - dtype="float64").reshape(1, TAG_NUM), - total_len, - axis=0) - - labels = np.random.randint(low=0, - high=TAG_NUM, - size=(total_len, 1), - dtype="int64") - predicted_labels = np.ones( - (total_len, 1), dtype="int64") * (TAG_NUM - 1) + transition = np.repeat( + np.arange(TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + TAG_NUM + 2, + axis=0, + ) + emission = np.repeat( + np.arange(TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + total_len, + axis=0, + ) + + labels = np.random.randint( + low=0, high=TAG_NUM, size=(total_len, 1), dtype="int64" + ) + predicted_labels = np.ones((total_len, 1), dtype="int64") * ( + TAG_NUM - 1 + ) expected_output = (labels == predicted_labels).astype("int64") self.inputs = { diff --git a/python/paddle/fluid/tests/unittests/test_crop_op.py b/python/paddle/fluid/tests/unittests/test_crop_op.py index 32e31e5366f..f495be8cbef 100644 --- a/python/paddle/fluid/tests/unittests/test_crop_op.py +++ b/python/paddle/fluid/tests/unittests/test_crop_op.py @@ -20,7 +20,6 @@ import paddle.fluid as fluid def crop(data, offsets, crop_shape): - def indexOf(shape, index): result = [] for dim in reversed(shape): @@ -34,15 +33,17 @@ def crop(data, offsets, crop_shape): selected = True if len(index) == len(offsets): for j, offset in enumerate(offsets): - selected = selected and index[j] >= offset and index[ - j] < crop_shape[j] + offset + selected = ( + selected + and index[j] >= offset + and index[j] < crop_shape[j] + offset + ) if selected: result.append(value) return np.array(result).reshape(crop_shape) class TestCropOp(OpTest): - def setUp(self): self.op_type = "crop" self.crop_by_input = False @@ -52,7 +53,7 @@ class TestCropOp(OpTest): if self.crop_by_input: self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'Y': np.random.random(self.crop_shape).astype("float64") + 'Y': np.random.random(self.crop_shape).astype("float64"), } else: self.attrs['shape'] = self.crop_shape @@ -85,7 +86,6 @@ class TestCropOp(OpTest): class TestCase1(TestCropOp): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [2, 2, 3] @@ -93,7 +93,6 @@ class TestCase1(TestCropOp): class TestCase2(TestCropOp): - def initTestCase(self): self.x_shape = (15, 8) self.crop_shape = [15, 8] @@ -101,7 +100,6 @@ class TestCase2(TestCropOp): class TestCase3(TestCropOp): - def initTestCase(self): self.x_shape = (4, 8, 16) self.crop_shape = [2, 2, 3] @@ -110,7 +108,6 @@ class TestCase3(TestCropOp): class TestCase4(TestCropOp): - def initTestCase(self): self.x_shape = (10, 10) self.crop_shape = [10, 10] @@ -119,7 +116,6 @@ class TestCase4(TestCropOp): class TestCase5(TestCropOp): - def initTestCase(self): self.x_shape = (3, 4, 10) self.crop_shape = [2, 2, 3] @@ -128,7 +124,6 @@ class TestCase5(TestCropOp): class TestCase6(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -138,7 +133,6 @@ class TestCase6(TestCropOp): class TestCropNoneOffset(unittest.TestCase): - def test_crop_none_offset(self): x = fluid.data(name="input1", shape=[3, 6, 6], dtype="float32") crop_shape = [2, 2, 2] @@ -147,7 +141,6 @@ class TestCropNoneOffset(unittest.TestCase): class TestCropNoneShape(unittest.TestCase): - def test_crop_none_shape(self): x = fluid.data(name="input1", shape=[3, 6, 6], dtype="float32") crop = paddle.crop(x) @@ -156,5 +149,6 @@ class TestCropNoneShape(unittest.TestCase): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_crop_tensor_op.py b/python/paddle/fluid/tests/unittests/test_crop_tensor_op.py index 8af23c6be5e..aeb0cfb6414 100644 --- a/python/paddle/fluid/tests/unittests/test_crop_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_crop_tensor_op.py @@ -20,7 +20,6 @@ import paddle.fluid as fluid def crop(data, offsets, crop_shape): - def indexOf(shape, index): result = [] for dim in reversed(shape): @@ -34,15 +33,17 @@ def crop(data, offsets, crop_shape): selected = True if len(index) == len(offsets): for j, offset in enumerate(offsets): - selected = selected and index[j] >= offset and index[ - j] < crop_shape[j] + offset + selected = ( + selected + and index[j] >= offset + and index[j] < crop_shape[j] + offset + ) if selected: result.append(value) return np.array(result).reshape(crop_shape) class TestCropTensorOp(OpTest): - def setUp(self): self.op_type = "crop_tensor" self.shape_by_input = False @@ -55,7 +56,7 @@ class TestCropTensorOp(OpTest): if self.shape_by_input: self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'Shape': np.array(self.crop_shape).astype("int32") + 'Shape': np.array(self.crop_shape).astype("int32"), } else: self.attrs['shape'] = self.crop_shape @@ -86,15 +87,13 @@ class TestCropTensorOp(OpTest): class TestCase1(TestCropTensorOp): - def initTestCase(self): - self.x_shape = (100) + self.x_shape = 100 self.crop_shape = [64] self.offsets = [13] class TestCase2(TestCropTensorOp): - def initTestCase(self): self.x_shape = (12, 24) self.crop_shape = [-1, 8] @@ -102,7 +101,6 @@ class TestCase2(TestCropTensorOp): class TestCase3(TestCropTensorOp): - def initTestCase(self): self.x_shape = (4, 8, 16) self.crop_shape = [2, 2, 3] @@ -111,7 +109,6 @@ class TestCase3(TestCropTensorOp): class TestCase4(TestCropTensorOp): - def initTestCase(self): self.x_shape = (8, 3, 6, 6) self.crop_shape = [-1, 3, -1, 4] @@ -120,7 +117,6 @@ class TestCase4(TestCropTensorOp): class TestCase5(TestCropTensorOp): - def initTestCase(self): self.x_shape = (2, 4, 5, 8, 8) self.crop_shape = [1, 1, 2, 4, 4] @@ -129,7 +125,6 @@ class TestCase5(TestCropTensorOp): class TestCase6(TestCropTensorOp): - def initTestCase(self): self.x_shape = (2, 2, 4, 4, 4, 2) self.crop_shape = [1, 1, 4, 2, 2, 2] @@ -139,7 +134,6 @@ class TestCase6(TestCropTensorOp): class TestCropTensorOpTensorAttr(OpTest): - def setUp(self): self.op_type = "crop_tensor" self.OffsetsTensor = False @@ -151,22 +145,24 @@ class TestCropTensorOpTensorAttr(OpTest): if self.ShapeTensor: shape_tensor = [] for index, ele in enumerate(self.crop_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'ShapeTensor': shape_tensor + 'ShapeTensor': shape_tensor, } self.attrs['shape'] = self.shape_attr if self.OffsetsTensor: offsets_tensor = [] for index, ele in enumerate(self.offsets): - offsets_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + offsets_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'OffsetsTensor': offsets_tensor + 'OffsetsTensor': offsets_tensor, } self.attrs['offsets'] = self.offsets_attr @@ -192,7 +188,6 @@ class TestCropTensorOpTensorAttr(OpTest): class TestCropTensorOpTensorAttrCase1(TestCropTensorOpTensorAttr): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [-1, -1, 3] @@ -201,7 +196,6 @@ class TestCropTensorOpTensorAttrCase1(TestCropTensorOpTensorAttr): class TestCropTensorOpTensorAttrCase2(TestCropTensorOpTensorAttr): - def initTestCase(self): self.x_shape = (4, 8, 16, 8) self.crop_shape = [2, 2, 3, 4] @@ -210,7 +204,6 @@ class TestCropTensorOpTensorAttrCase2(TestCropTensorOpTensorAttr): class TestCropTensorOpTensorAttrCase3(TestCropTensorOpTensorAttr): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [2, 2, 3] @@ -221,7 +214,6 @@ class TestCropTensorOpTensorAttrCase3(TestCropTensorOpTensorAttr): class TestCropTensorOpTensorAttrCase4(TestCropTensorOpTensorAttr): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [2, 2, 3] @@ -232,7 +224,6 @@ class TestCropTensorOpTensorAttrCase4(TestCropTensorOpTensorAttr): class TestCropTensorException(unittest.TestCase): - def test_exception(self): input1 = fluid.data(name="input1", shape=[2, 3, 6, 6], dtype="float32") input2 = fluid.data(name="input2", shape=[2, 3, 6, 6], dtype="float16") @@ -255,14 +246,14 @@ class TestCropTensorException(unittest.TestCase): out = paddle.crop(input1, shape=[2, 2, 3, 3], offsets=0) def attr_offsets_dtype(): - out = paddle.crop(input1, - shape=[2, 2, 3, 3], - offsets=[0, 1.0, 0, 0]) + out = paddle.crop( + input1, shape=[2, 2, 3, 3], offsets=[0, 1.0, 0, 0] + ) def attr_offsets_value(): - out = paddle.crop(input1, - shape=[2, 2, 3, 3], - offsets=[0, -1, offset, 0]) + out = paddle.crop( + input1, shape=[2, 2, 3, 3], offsets=[0, -1, offset, 0] + ) def input_dtype(): out = paddle.crop(input2, shape=[2, 2, 3, 3]) @@ -279,5 +270,6 @@ class TestCropTensorException(unittest.TestCase): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy2_op.py b/python/paddle/fluid/tests/unittests/test_cross_entropy2_op.py index 13fbd933899..9b8acb884ab 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy2_op.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy2_op.py @@ -18,7 +18,6 @@ import numpy as np class CrossEntropy2OpTestBase(OpTest): - def initParameters(self): return [32, 64], 'float64', -100, False @@ -33,26 +32,33 @@ class CrossEntropy2OpTestBase(OpTest): return ret, match_x def setUp(self): - self.shape, self.dtype, self.ignore_index, self.drop_last_dim = self.initParameters( - ) + ( + self.shape, + self.dtype, + self.ignore_index, + self.drop_last_dim, + ) = self.initParameters() self.op_type = 'cross_entropy2' feature_size = int(self.shape[-1]) batch_size = int(np.prod(self.shape) / feature_size) logits = (np.random.random(size=self.shape) + 1).astype(self.dtype) - label_shape = self.shape[ - 0:-1] if self.drop_last_dim else self.shape[0:-1] + [1] - label = np.random.random_integers(low=0, - high=feature_size - 1, - size=label_shape).astype('int64') + label_shape = ( + self.shape[0:-1] if self.drop_last_dim else self.shape[0:-1] + [1] + ) + label = np.random.random_integers( + low=0, high=feature_size - 1, size=label_shape + ).astype('int64') outputs, match_x = self.calc_output( np.reshape(logits, [batch_size, feature_size]), - np.reshape(label, [batch_size, 1]), self.ignore_index) + np.reshape(label, [batch_size, 1]), + self.ignore_index, + ) self.inputs = {'X': logits, 'Label': label} out_shape = label_shape self.outputs = { 'Y': np.reshape(outputs, out_shape), 'MatchX': np.reshape(match_x, self.shape[:-1] + [1]), - 'XShape': np.zeros(shape=logits.shape, dtype=logits.dtype) + 'XShape': np.zeros(shape=logits.shape, dtype=logits.dtype), } self.attrs = {'ignore_index': self.ignore_index} @@ -60,37 +66,34 @@ class CrossEntropy2OpTestBase(OpTest): self.check_output(no_check_set=['XShape']) def test_check_grad(self): - self.check_grad(inputs_to_check=['X'], - output_names=['Y'], - no_grad_set=['XShape', 'MatchX', 'Label']) + self.check_grad( + inputs_to_check=['X'], + output_names=['Y'], + no_grad_set=['XShape', 'MatchX', 'Label'], + ) class CrossEntropy2OpTest2(CrossEntropy2OpTestBase): - def initParameters(self): return [32, 64], 'float64', 3, False class CrossEntropy2OpTest2RemoveLastDim(CrossEntropy2OpTestBase): - def initParameters(self): return [32, 64], 'float64', 3, True class CrossEntropy2OpTest3(CrossEntropy2OpTestBase): - def initParameters(self): return [4, 8, 16, 32], 'float64', -100, False class CrossEntropy2OpTest3RemoveLastDim(CrossEntropy2OpTestBase): - def initParameters(self): return [4, 8, 16, 32], 'float64', -100, True class CrossEntropy2OpTest4(CrossEntropy2OpTestBase): - def initParameters(self): return [4, 8, 16, 32], 'float64', 3, False diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py b/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py index 4ee9b8037fe..f3e66f9bb8d 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py @@ -27,11 +27,9 @@ def log_softmax(x, axis=-1): return np.log(softmax_out) -def cross_entropy_loss_1d(input, - label, - weight=None, - reduction='mean', - ignore_index=-100): +def cross_entropy_loss_1d( + input, label, weight=None, reduction='mean', ignore_index=-100 +): log_softmax_out = log_softmax(input) input_shape = log_softmax_out.shape N = input_shape[0] @@ -59,11 +57,9 @@ def cross_entropy_loss_1d(input, return out -def cross_entropy_loss_2d(input, - label, - weight=None, - reduction='mean', - ignore_index=-100): +def cross_entropy_loss_2d( + input, label, weight=None, reduction='mean', ignore_index=-100 +): log_softmax_out = log_softmax(input) input_shape = log_softmax_out.shape N = input_shape[0] @@ -81,8 +77,9 @@ def cross_entropy_loss_2d(input, continue cur_weight = weight[cur_target] if weight is not None else 1 total_weight += cur_weight - out[i][h][ - w] = -log_softmax_out[i][h][w][cur_target] * cur_weight + out[i][h][w] = ( + -log_softmax_out[i][h][w][cur_target] * cur_weight + ) if reduction == 'sum': return np.sum(out), np.array([total_weight]).astype('float64') elif reduction == 'mean': @@ -92,27 +89,20 @@ def cross_entropy_loss_2d(input, return out -def cross_entropy_soft(softmax, - label, - axis, - N, - weight=None, - reduction='mean', - ignore_index=-100): - #1.loss +def cross_entropy_soft( + softmax, label, axis, N, weight=None, reduction='mean', ignore_index=-100 +): + # 1.loss loss = cross_entropy( - softmax, - label, - True, #soft_label, - axis, - ignore_index) + softmax, label, True, axis, ignore_index # soft_label, + ) if weight is None and reduction == 'none': return loss - #2.weight + # 2.weight weighted_loss = loss - total_weight = N #for weight is None + total_weight = N # for weight is None if weight is not None: weighted_loss = np.zeros_like(loss).astype(np.float64) total_weight = 0 @@ -122,7 +112,7 @@ def cross_entropy_soft(softmax, total_weight += cur_weight weighted_loss[i] = loss[i] * cur_weight - #3.reduce + # 3.reduce if reduction == 'none': return weighted_loss @@ -136,29 +126,28 @@ def cross_entropy_soft(softmax, return weighted_loss_sum -def cross_entropy_soft_2d(softmax, - label, - axis, - N, - H, - W, - weight=None, - reduction='mean', - ignore_index=-100): - #1.loss +def cross_entropy_soft_2d( + softmax, + label, + axis, + N, + H, + W, + weight=None, + reduction='mean', + ignore_index=-100, +): + # 1.loss loss = cross_entropy( - softmax, - label, - True, #soft_label, - axis, - ignore_index) + softmax, label, True, axis, ignore_index # soft_label, + ) if weight is None and reduction == 'none': return loss - #2.weight + # 2.weight weighted_loss = loss - total_weight = N #for weight is None + total_weight = N # for weight is None if weight is not None: weighted_loss = np.zeros_like(loss).astype(np.float64) total_weight = 0 @@ -170,7 +159,7 @@ def cross_entropy_soft_2d(softmax, total_weight += cur_weight weighted_loss[i][h][w] = loss[i][h][w] * cur_weight - #3.reduce + # 3.reduce if reduction == 'none': return weighted_loss @@ -185,19 +174,20 @@ def cross_entropy_soft_2d(softmax, class CrossEntropyLoss(unittest.TestCase): - def setUp(self): - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) ###test for deprecated softmax_with_cross_entropy def test_softmax_with_cross_entropy(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -205,20 +195,24 @@ class CrossEntropyLoss(unittest.TestCase): self.reduction = 'none' self.weight = None self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") @@ -227,7 +221,8 @@ class CrossEntropyLoss(unittest.TestCase): fluid.dygraph.to_variable(self.logits), fluid.dygraph.to_variable(self.labels), soft_label=True, - axis=self.axis) + axis=self.axis, + ) paddle_loss_ce = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), @@ -235,12 +230,14 @@ class CrossEntropyLoss(unittest.TestCase): soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight) - if self.weight is not None else None, - reduction=self.reduction) - - np.testing.assert_allclose(paddle_loss_swce.numpy(), - expected, - rtol=1e-05) + if self.weight is not None + else None, + reduction=self.reduction, + ) + + np.testing.assert_allclose( + paddle_loss_swce.numpy(), expected, rtol=1e-05 + ) np.testing.assert_allclose(paddle_loss_ce.numpy(), expected, rtol=1e-05) ###soft_label test start @@ -248,10 +245,11 @@ class CrossEntropyLoss(unittest.TestCase): def test_cross_entropy_loss_soft_1d(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -259,24 +257,28 @@ class CrossEntropyLoss(unittest.TestCase): self.reduction = 'none' self.weight = None self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") - #2. dygraph + # 2. dygraph paddle.disable_static() paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), @@ -284,35 +286,43 @@ class CrossEntropyLoss(unittest.TestCase): soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight) - if self.weight is not None else None, - reduction=self.reduction) + if self.weight is not None + else None, + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[self.N, self.C], dtype=self.dtype + ) + label = fluid.data( + name='label', shape=[self.N, self.C], dtype=self.dtype + ) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction=self.reduction, soft_label=True) + reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -323,10 +333,11 @@ class CrossEntropyLoss(unittest.TestCase): def test_cross_entropy_loss_soft_1d_weight(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -334,34 +345,38 @@ class CrossEntropyLoss(unittest.TestCase): self.reduction = 'none' self.weight = np.random.uniform(0.1, 1.0, self.C).astype(self.dtype) self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) if self.soft_label: - self.labels = np.random.uniform(0.1, 1.0, - self.shape).astype(self.dtype) + self.labels = np.random.uniform(0.1, 1.0, self.shape).astype( + self.dtype + ) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) else: axis_dim = self.shape[self.axis] self.shape[self.axis] = 1 - self.labels = np.random.randint(0, - axis_dim, - self.shape, - dtype="int64") - - #1. numpy - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + self.labels = np.random.randint( + 0, axis_dim, self.shape, dtype="int64" + ) + + # 1. numpy + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") - #2. dygraph + # 2. dygraph paddle.disable_static() paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), @@ -369,36 +384,43 @@ class CrossEntropyLoss(unittest.TestCase): soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight), - reduction=self.reduction) + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() # 3.static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[self.N, self.C], dtype=self.dtype + ) + label = fluid.data( + name='label', shape=[self.N, self.C], dtype=self.dtype + ) weight = fluid.data(name='weight', shape=[self.C], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction=self.reduction, soft_label=True) + weight=weight, reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - "weight": self.weight - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + "weight": self.weight, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -409,10 +431,11 @@ class CrossEntropyLoss(unittest.TestCase): def test_cross_entropy_loss_soft_1d_mean(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -420,25 +443,29 @@ class CrossEntropyLoss(unittest.TestCase): self.reduction = 'mean' self.weight = None self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - #1. numpy - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + # 1. numpy + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") - #2 dygraph + # 2 dygraph paddle.disable_static() paddle_loss_mean = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), @@ -446,34 +473,38 @@ class CrossEntropyLoss(unittest.TestCase): soft_label=True, axis=self.axis, weight=self.weight, - reduction=self.reduction) + reduction=self.reduction, + ) dy_ret_value = paddle_loss_mean.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[self.N, self.C], dtype=self.dtype + ) + label = fluid.data( + name='label', shape=[self.N, self.C], dtype=self.dtype + ) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction=self.reduction, soft_label=True) + reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={'input': self.logits, 'label': self.labels}, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -484,10 +515,11 @@ class CrossEntropyLoss(unittest.TestCase): def test_cross_entropy_loss_soft_1d_weight_mean(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -495,61 +527,72 @@ class CrossEntropyLoss(unittest.TestCase): self.reduction = 'mean' self.weight = np.random.uniform(0.1, 1.0, self.C).astype(self.dtype) self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - #1. numpy - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + # 1. numpy + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") paddle.disable_static() - #2. dygraph + # 2. dygraph paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), fluid.dygraph.to_variable(self.labels), soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight), - reduction=self.reduction) + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[self.N, self.C], dtype=self.dtype + ) + label = fluid.data( + name='label', shape=[self.N, self.C], dtype=self.dtype + ) weight = fluid.data(name='weight', shape=[self.C], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction=self.reduction, soft_label=True) + weight=weight, reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - "weight": self.weight - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + "weight": self.weight, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -558,14 +601,14 @@ class CrossEntropyLoss(unittest.TestCase): ###soft_label test 5 def test_cross_entropy_loss_soft_2d(self): - def inner_cross_entropy_loss_soft_2d(soft_label): self.numeric_stable_mode = False self.soft_label = soft_label - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 3 self.H = 2 self.W = 2 @@ -575,64 +618,82 @@ class CrossEntropyLoss(unittest.TestCase): self.reduction = 'none' self.weight = None self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) - softmax = np.apply_along_axis(stable_softmax, self.axis, - self.logits) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) + softmax = np.apply_along_axis( + stable_softmax, self.axis, self.logits + ) - self.labels = np.random.uniform(0.1, 1.0, - self.shape).astype(self.dtype) + self.labels = np.random.uniform(0.1, 1.0, self.shape).astype( + self.dtype + ) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - #1. numpy - expected = cross_entropy_soft_2d(softmax, - self.labels, - self.axis, - self.N, - self.H, - self.W, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + # 1. numpy + expected = cross_entropy_soft_2d( + softmax, + self.labels, + self.axis, + self.N, + self.H, + self.W, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") paddle.disable_static() - #2. dygraph + # 2. dygraph paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), fluid.dygraph.to_variable(self.labels), soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight) - if self.weight is not None else None, - reduction=self.reduction) + if self.weight is not None + else None, + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.H, self.W, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.H, self.W, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', + shape=[self.N, self.H, self.W, self.C], + dtype=self.dtype, + ) + label = fluid.data( + name='label', + shape=[self.N, self.H, self.W, self.C], + dtype=self.dtype, + ) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction=self.reduction, soft_label=True) + reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -647,10 +708,11 @@ class CrossEntropyLoss(unittest.TestCase): def test_cross_entropy_loss_soft_2d_weight_mean(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 3 self.H = 2 self.W = 2 @@ -660,63 +722,78 @@ class CrossEntropyLoss(unittest.TestCase): self.reduction = 'mean' self.weight = np.random.uniform(0.1, 1.0, self.C).astype(self.dtype) self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - #1. numpy - expected = cross_entropy_soft_2d(softmax, - self.labels, - self.axis, - self.N, - self.H, - self.W, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + # 1. numpy + expected = cross_entropy_soft_2d( + softmax, + self.labels, + self.axis, + self.N, + self.H, + self.W, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") paddle.disable_static() - #2. dygraph + # 2. dygraph paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), fluid.dygraph.to_variable(self.labels), soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight), - reduction=self.reduction) + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.H, self.W, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.H, self.W, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', + shape=[self.N, self.H, self.W, self.C], + dtype=self.dtype, + ) + label = fluid.data( + name='label', + shape=[self.N, self.H, self.W, self.C], + dtype=self.dtype, + ) weight = fluid.data(name='weight', shape=[self.C], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction=self.reduction, soft_label=True) + weight=weight, reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - "weight": self.weight - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + "weight": self.weight, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -732,8 +809,11 @@ class CrossEntropyLoss(unittest.TestCase): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[2, 4], dtype=self.dtype) label = fluid.data(name='label', shape=[2], dtype='int64') @@ -741,20 +821,25 @@ class CrossEntropyLoss(unittest.TestCase): ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) expected = cross_entropy_loss_1d(input_np, label_np)[0] with fluid.dygraph.guard(): - cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss(axis=1, - ignore_index=0) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( + axis=1, ignore_index=0 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_1d(input_np, label_np, ignore_index=0)[0] @@ -770,28 +855,37 @@ class CrossEntropyLoss(unittest.TestCase): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[N, C], dtype=self.dtype) label = fluid.data(name='label', shape=[N], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - ignore_index=-1) + ignore_index=-1 + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - axis=1, ignore_index=-1) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + axis=1, ignore_index=-1 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_1d(input_np, label_np, ignore_index=-1)[0] @@ -809,40 +903,49 @@ class CrossEntropyLoss(unittest.TestCase): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[N, C], dtype=self.dtype) label = fluid.data(name='label', shape=[N], dtype='int64') - weight = fluid.data(name='weight', shape=[C], - dtype=self.dtype) #weight for each class - cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss(weight=weight, - ignore_index=0) + weight = fluid.data( + name='weight', shape=[C], dtype=self.dtype + ) # weight for each class + cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( + weight=weight, ignore_index=0 + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( weight=fluid.dygraph.to_variable(weight_np), axis=1, - ignore_index=0) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + ignore_index=0, + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - ignore_index=0)[0] + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, ignore_index=0 + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) @@ -858,172 +961,205 @@ class CrossEntropyLoss(unittest.TestCase): with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), ignore_index=255) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), ignore_index=255 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - ignore_index=255)[0] + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, ignore_index=255 + )[0] np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_with_weight_mean(self): input_np = np.random.random([2, 4]).astype(self.dtype) label_np = np.random.randint(0, 4, size=(2)).astype(np.int64) - weight_np = np.random.random([4]).astype(self.dtype) #shape:C + weight_np = np.random.random([4]).astype(self.dtype) # shape:C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[2, 4], dtype=self.dtype) label = fluid.data(name='label', shape=[2], dtype='int64') - weight = fluid.data(name='weight', shape=[4], - dtype=self.dtype) #weight for each class + weight = fluid.data( + name='weight', shape=[4], dtype=self.dtype + ) # weight for each class cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss(weight=weight) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) - expected = cross_entropy_loss_1d(input_np, label_np, - weight=weight_np)[0] + expected = cross_entropy_loss_1d(input_np, label_np, weight=weight_np)[ + 0 + ] with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), axis=1) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), axis=1 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, label_np, - weight=weight_np)[0] + expected = cross_entropy_loss_1d(input_np, label_np, weight=weight_np)[ + 0 + ] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_with_weight_sum(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 - weight_np = np.random.random([200]).astype(self.dtype) #C + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 + weight_np = np.random.random([200]).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') weight = fluid.data(name='weight', shape=[200], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='sum') + weight=weight, reduction='sum' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='sum') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='sum' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='sum')[0] + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, reduction='sum' + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_with_weight_none(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 - weight_np = np.random.random([200]).astype(self.dtype) #C + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 + weight_np = np.random.random([200]).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') weight = fluid.data(name='weight', shape=[200], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='none') + weight=weight, reduction='none' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='none') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='none' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='none') + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, reduction='none' + ) np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret, expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_with_weight_none_func(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N - weight_np = np.random.random([200]).astype(self.dtype) #C + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N + weight_np = np.random.random([200]).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') weight = fluid.data(name='weight', shape=[200], dtype=self.dtype) - ret = paddle.nn.functional.cross_entropy(input, - label, - weight=weight, - reduction='none') + ret = paddle.nn.functional.cross_entropy( + input, label, weight=weight, reduction='none' + ) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): @@ -1031,43 +1167,47 @@ class CrossEntropyLoss(unittest.TestCase): fluid.dygraph.to_variable(input_np), fluid.dygraph.to_variable(label_np), weight=fluid.dygraph.to_variable(weight_np), - reduction='none') + reduction='none', + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='none') + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, reduction='none' + ) np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret, expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_mean(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss() ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={'input': input_np, 'label': label_np}, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss() - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_1d(input_np, label_np)[0] @@ -1076,32 +1216,38 @@ class CrossEntropyLoss(unittest.TestCase): np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_sum(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='sum') + reduction='sum' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={'input': input_np, 'label': label_np}, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='sum') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='sum' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_1d(input_np, label_np, reduction='sum')[0] @@ -1110,33 +1256,39 @@ class CrossEntropyLoss(unittest.TestCase): np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_none(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='none') + reduction='none' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={'input': input_np, 'label': label_np}, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='none') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='none' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) @@ -1146,97 +1298,122 @@ class CrossEntropyLoss(unittest.TestCase): np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_with_weight_none(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW1 - weight_np = np.random.random(size=(3, )).astype(self.dtype) #C + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW1 + weight_np = np.random.random(size=(3,)).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='none') + weight=weight, reduction='none' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='none') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='none' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='none') + expected = cross_entropy_loss_2d( + input_np, label_np, weight=weight_np, reduction='none' + ) np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret, expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_with_weight_axis_change_mean(self): - input_np = np.random.random(size=(2, 3, 2, 2)).astype(self.dtype) #NCHW - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW - weight_np = np.random.random(size=(3, )).astype(self.dtype) #C + input_np = np.random.random(size=(2, 3, 2, 2)).astype( + self.dtype + ) # NCHW + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW + weight_np = np.random.random(size=(3,)).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 3, 2, 2], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 3, 2, 2], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='mean', axis=1) + weight=weight, reduction='mean', axis=1 + ) # specify the class channels to axis 1 ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( weight=fluid.dygraph.to_variable(weight_np), reduction='mean', - axis=1) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + axis=1, + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(np.transpose(input_np, [0, 2, 3, 1]), - label_np, - weight=weight_np, - reduction='mean')[0] + expected = cross_entropy_loss_2d( + np.transpose(input_np, [0, 2, 3, 1]), + label_np, + weight=weight_np, + reduction='mean', + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) @@ -1252,137 +1429,173 @@ class CrossEntropyLoss(unittest.TestCase): weight_np = np.random.random([C]).astype(self.dtype) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), ignore_index=255) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), ignore_index=255 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, - label_np, - weight=weight_np, - ignore_index=255)[0] + expected = cross_entropy_loss_2d( + input_np, label_np, weight=weight_np, ignore_index=255 + )[0] np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_with_weight_mean(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW - weight_np = np.random.random(size=(3, )).astype(self.dtype) #C + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW + weight_np = np.random.random(size=(3,)).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='mean') + weight=weight, reduction='mean' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='mean') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='mean' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='mean')[0] + expected = cross_entropy_loss_2d( + input_np, label_np, weight=weight_np, reduction='mean' + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_with_weight_sum(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW - weight_np = np.random.random(size=(3, )).astype(self.dtype) #C + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW + weight_np = np.random.random(size=(3,)).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='sum') + weight=weight, reduction='sum' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='sum') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='sum' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='sum')[0] + expected = cross_entropy_loss_2d( + input_np, label_np, weight=weight_np, reduction='sum' + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_none(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='none') + reduction='none' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='none') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='none' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) @@ -1392,75 +1605,100 @@ class CrossEntropyLoss(unittest.TestCase): np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_mean(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='mean') + reduction='mean' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='mean') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='mean' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, label_np, - reduction='mean')[0] + expected = cross_entropy_loss_2d(input_np, label_np, reduction='mean')[ + 0 + ] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_sum(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='sum') + reduction='sum' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='sum') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='sum' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_2d(input_np, label_np, reduction='sum')[0] @@ -1486,8 +1724,7 @@ class CrossEntropyLoss(unittest.TestCase): self.test_cross_entropy_loss_1d_with_mean_ignore() self.test_cross_entropy_loss_1d_with_mean_ignore_negative() self.test_cross_entropy_loss_1d_with_weight_mean_ignore() - self.test_cross_entropy_loss_1d_with_weight_mean_ignore_exceedlabel( - ) + self.test_cross_entropy_loss_1d_with_weight_mean_ignore_exceedlabel() self.test_cross_entropy_loss_1d_with_weight_mean() self.test_cross_entropy_loss_1d_with_weight_sum() self.test_cross_entropy_loss_1d_with_weight_none() @@ -1497,8 +1734,7 @@ class CrossEntropyLoss(unittest.TestCase): self.test_cross_entropy_loss_1d_none() self.test_cross_entropy_loss_2d_with_weight_none() self.test_cross_entropy_loss_2d_with_weight_axis_change_mean() - self.test_cross_entropy_loss_2d_with_weight_mean_ignore_exceedlabel( - ) + self.test_cross_entropy_loss_2d_with_weight_mean_ignore_exceedlabel() self.test_cross_entropy_loss_2d_with_weight_mean() self.test_cross_entropy_loss_2d_with_weight_sum() self.test_cross_entropy_loss_2d_none() @@ -1507,51 +1743,53 @@ class CrossEntropyLoss(unittest.TestCase): class TestCrossEntropyFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_WeightLength_NotEqual(): input_data = paddle.rand(shape=[20, 100]) - label_data = paddle.randint(0, - 100, - shape=[20, 1], - dtype="int64") + label_data = paddle.randint( + 0, 100, shape=[20, 1], dtype="int64" + ) weight_data = paddle.rand([100 + 1]) - paddle.nn.functional.cross_entropy(input=input_data, - label=label_data, - weight=weight_data, - ignore_index=-100) + paddle.nn.functional.cross_entropy( + input=input_data, + label=label_data, + weight=weight_data, + ignore_index=-100, + ) self.assertRaises(ValueError, test_WeightLength_NotEqual) def test_LabelValue_ExceedMax(): input_data = paddle.rand(shape=[20, 100]) - label_data = paddle.randint(0, - 100, - shape=[20, 1], - dtype="int64") + label_data = paddle.randint( + 0, 100, shape=[20, 1], dtype="int64" + ) label_data[0] = 100 weight_data = paddle.rand([100]) - paddle.nn.functional.cross_entropy(input=input_data, - label=label_data, - weight=weight_data, - ignore_index=-100) + paddle.nn.functional.cross_entropy( + input=input_data, + label=label_data, + weight=weight_data, + ignore_index=-100, + ) self.assertRaises(ValueError, test_LabelValue_ExceedMax) def test_LabelValue_ExceedMin(): input_data = paddle.rand(shape=[20, 100]) - label_data = paddle.randint(0, - 100, - shape=[20, 1], - dtype="int64") + label_data = paddle.randint( + 0, 100, shape=[20, 1], dtype="int64" + ) label_data[0] = -1 weight_data = paddle.rand([100]) - paddle.nn.functional.cross_entropy(input=input_data, - label=label_data, - weight=weight_data, - ignore_index=-100) + paddle.nn.functional.cross_entropy( + input=input_data, + label=label_data, + weight=weight_data, + ignore_index=-100, + ) self.assertRaises(ValueError, test_LabelValue_ExceedMin) @@ -1562,28 +1800,34 @@ class TestCrossEntropyFAPIError(unittest.TestCase): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( - ) else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 4], - dtype='float32') + input = fluid.data( + name='input', shape=[2, 4], dtype='float32' + ) label = fluid.data(name='label', shape=[2], dtype='int64') - weight = fluid.data(name='weight', - shape=[3], - dtype='float32') #weight for each class + weight = fluid.data( + name='weight', shape=[3], dtype='float32' + ) # weight for each class cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight) + weight=weight + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) self.assertRaises(ValueError, static_test_WeightLength_NotEqual) diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py index a0f74302628..c9fe88f318e 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py @@ -21,8 +21,7 @@ from paddle.fluid import Program, program_guard class TestCrossEntropyOp(OpTest): - """Test cross-entropy with discrete one-hot labels. - """ + """Test cross-entropy with discrete one-hot labels.""" def setUp(self): self.op_type = "cross_entropy" @@ -43,24 +42,27 @@ class TestCrossEntropyOp(OpTest): self.outputs = {"Y": self.cross_entropy} self.attrs = { "soft_label": self.soft_label, - "ignore_index": self.ignore_index + "ignore_index": self.ignore_index, } def init_x(self): - self.x = randomize_probability(self.batch_size, - self.class_num, - dtype=self.dtype) + self.x = randomize_probability( + self.batch_size, self.class_num, dtype=self.dtype + ) def init_label(self): - self.label = np.random.randint(0, - self.class_num, (self.batch_size, 1), - dtype="int64") + self.label = np.random.randint( + 0, self.class_num, (self.batch_size, 1), dtype="int64" + ) def get_cross_entropy(self): self.cross_entropy = np.asmatrix( - [[-np.log(self.x[i][self.label[i][0]])] - for i in range(self.x.shape[0])], - dtype="float64") + [ + [-np.log(self.x[i][self.label[i][0]])] + for i in range(self.x.shape[0]) + ], + dtype="float64", + ) def init_attr_type(self): pass @@ -79,32 +81,35 @@ class TestCrossEntropyOp(OpTest): class TestCrossEntropyOpRemoveLastDim(TestCrossEntropyOp): - """Test cross-entropy with discrete one-hot labels with shape [batch_size] - """ + """Test cross-entropy with discrete one-hot labels with shape [batch_size]""" def init_label(self): - self.label = np.random.randint(0, - self.class_num, (self.batch_size), - dtype="int64") + self.label = np.random.randint( + 0, self.class_num, (self.batch_size), dtype="int64" + ) def get_cross_entropy(self): self.cross_entropy = np.asmatrix( [-np.log(self.x[i][self.label[i]]) for i in range(self.x.shape[0])], - dtype="float64") + dtype="float64", + ) class TestCrossEntropyOp2(TestCrossEntropyOp): - """Test cross-entropy with vectorized soft labels. - """ + """Test cross-entropy with vectorized soft labels.""" def init_label(self): self.label = np.random.uniform( - 0.1, 1.0, [self.batch_size, self.class_num]).astype(self.dtype) + 0.1, 1.0, [self.batch_size, self.class_num] + ).astype(self.dtype) self.label /= self.label.sum(axis=1, keepdims=True) def get_cross_entropy(self): - self.cross_entropy = (-self.label * np.log(self.x)).sum( - axis=1, keepdims=True).astype(self.dtype) + self.cross_entropy = ( + (-self.label * np.log(self.x)) + .sum(axis=1, keepdims=True) + .astype(self.dtype) + ) def init_attr_type(self): self.soft_label = True @@ -117,26 +122,28 @@ class TestCrossEntropyOp2(TestCrossEntropyOp): self.class_num = 37 def test_check_grad(self): - self.check_grad(["X"], - "Y", - max_relative_error=0.05, - numeric_grad_delta=0.001) + self.check_grad( + ["X"], "Y", max_relative_error=0.05, numeric_grad_delta=0.001 + ) class TestCrossEntropyOp3(TestCrossEntropyOp): - """Test cross-entropy with vectorized one-hot representation of labels. - """ + """Test cross-entropy with vectorized one-hot representation of labels.""" def init_label(self): - self.label_index = np.random.randint(0, self.class_num, - (self.batch_size)) + self.label_index = np.random.randint( + 0, self.class_num, (self.batch_size) + ) self.label = np.zeros(self.x.shape).astype(self.dtype) self.label[np.arange(self.batch_size), self.label_index] = 1 def get_cross_entropy(self): self.cross_entropy = np.asmatrix( - [[-np.log(self.x[i][self.label_index[i]])] - for i in range(self.x.shape[0])]).astype(self.dtype) + [ + [-np.log(self.x[i][self.label_index[i]])] + for i in range(self.x.shape[0]) + ] + ).astype(self.dtype) def init_attr_type(self): self.soft_label = True @@ -149,35 +156,38 @@ class TestCrossEntropyOp3(TestCrossEntropyOp): self.class_num = 27 def test_check_grad(self): - self.check_grad(["X"], - "Y", - max_relative_error=0.05, - numeric_grad_delta=0.001) + self.check_grad( + ["X"], "Y", max_relative_error=0.05, numeric_grad_delta=0.001 + ) class TestCrossEntropyOp4(TestCrossEntropyOp): - """Test high rank tensor cross-entropy with discrete one-hot labels. - """ + """Test high rank tensor cross-entropy with discrete one-hot labels.""" def init_x(self): self.shape = [10, 2, 4] self.ins_num = np.prod(np.array(self.shape)) - self.X_2d = randomize_probability(self.ins_num, - self.class_num).astype(self.dtype) + self.X_2d = randomize_probability(self.ins_num, self.class_num).astype( + self.dtype + ) self.x = self.X_2d.reshape(self.shape + [self.class_num]) def init_label(self): - self.label_2d = np.random.randint(0, - self.class_num, (self.ins_num, 1), - dtype="int64") + self.label_2d = np.random.randint( + 0, self.class_num, (self.ins_num, 1), dtype="int64" + ) self.label = self.label_2d.reshape(self.shape + [1]) def get_cross_entropy(self): cross_entropy_2d = np.asmatrix( - [[-np.log(self.X_2d[i][self.label_2d[i][0]])] - for i in range(self.X_2d.shape[0])]).astype(self.dtype) - self.cross_entropy = np.array(cross_entropy_2d).reshape(self.shape + - [1]) + [ + [-np.log(self.X_2d[i][self.label_2d[i][0]])] + for i in range(self.X_2d.shape[0]) + ] + ).astype(self.dtype) + self.cross_entropy = np.array(cross_entropy_2d).reshape( + self.shape + [1] + ) def init_attr_type(self): self.soft_label = False @@ -190,44 +200,51 @@ class TestCrossEntropyOp4(TestCrossEntropyOp): class TestCrossEntropyOp4RemoveLastDim(TestCrossEntropyOp4): - """Test high rank tensor cross-entropy with discrete one-hot labels with shape [batch_size] - """ + """Test high rank tensor cross-entropy with discrete one-hot labels with shape [batch_size]""" def init_label(self): - self.label_2d = np.random.randint(0, - self.class_num, (self.ins_num, 1), - dtype="int64") + self.label_2d = np.random.randint( + 0, self.class_num, (self.ins_num, 1), dtype="int64" + ) self.label = self.label_2d.reshape(self.shape) def get_cross_entropy(self): cross_entropy_2d = np.asmatrix( - [[-np.log(self.X_2d[i][self.label_2d[i][0]])] - for i in range(self.X_2d.shape[0])]).astype(self.dtype) + [ + [-np.log(self.X_2d[i][self.label_2d[i][0]])] + for i in range(self.X_2d.shape[0]) + ] + ).astype(self.dtype) self.cross_entropy = np.array(cross_entropy_2d).reshape(self.shape) class TestCrossEntropyOp5(TestCrossEntropyOp): - """Test high rank tensor cross-entropy with vectorized soft labels. - """ + """Test high rank tensor cross-entropy with vectorized soft labels.""" def init_x(self): self.shape = [4, 3] self.ins_num = np.prod(np.array(self.shape)) - self.X_2d = randomize_probability(self.ins_num, - self.class_num).astype(self.dtype) + self.X_2d = randomize_probability(self.ins_num, self.class_num).astype( + self.dtype + ) self.x = self.X_2d.reshape(self.shape + [self.class_num]) def init_label(self): self.label_2d = np.random.uniform( - 0.1, 1.0, [self.ins_num, self.class_num]).astype(self.dtype) + 0.1, 1.0, [self.ins_num, self.class_num] + ).astype(self.dtype) self.label_2d /= self.label_2d.sum(axis=1, keepdims=True) self.label = self.label_2d.reshape(self.shape + [self.class_num]) def get_cross_entropy(self): - cross_entropy_2d = (-self.label_2d * np.log(self.X_2d)).sum( - axis=1, keepdims=True).astype(self.dtype) - self.cross_entropy = np.array(cross_entropy_2d).reshape(self.shape + - [1]) + cross_entropy_2d = ( + (-self.label_2d * np.log(self.X_2d)) + .sum(axis=1, keepdims=True) + .astype(self.dtype) + ) + self.cross_entropy = np.array(cross_entropy_2d).reshape( + self.shape + [1] + ) def init_attr_type(self): self.soft_label = True @@ -239,39 +256,44 @@ class TestCrossEntropyOp5(TestCrossEntropyOp): self.class_num = 37 def test_check_grad(self): - self.check_grad(["X"], - "Y", - max_relative_error=0.05, - numeric_grad_delta=0.001) + self.check_grad( + ["X"], "Y", max_relative_error=0.05, numeric_grad_delta=0.001 + ) class TestCrossEntropyOp6(TestCrossEntropyOp): - """Test high rank tensor cross-entropy with vectorized one-hot representation of labels. - """ + """Test high rank tensor cross-entropy with vectorized one-hot representation of labels.""" def init_x(self): self.shape = [4, 3, 2] self.ins_num = np.prod(np.array(self.shape)) - self.X_2d = randomize_probability(self.ins_num, - self.class_num).astype(self.dtype) + self.X_2d = randomize_probability(self.ins_num, self.class_num).astype( + self.dtype + ) self.x = self.X_2d.reshape(self.shape + [self.class_num]) def init_label(self): - self.label_index_2d = np.random.randint(0, - self.class_num, (self.ins_num), - dtype="int64") + self.label_index_2d = np.random.randint( + 0, self.class_num, (self.ins_num), dtype="int64" + ) label_2d = np.zeros(self.X_2d.shape) label_2d[np.arange(self.ins_num), self.label_index_2d] = 1 self.label = label_2d.reshape(self.shape + [self.class_num]).astype( - self.dtype) + self.dtype + ) def get_cross_entropy(self): cross_entropy_2d = np.asmatrix( - [[-np.log(self.X_2d[i][self.label_index_2d[i]])] - for i in range(self.X_2d.shape[0])]) - self.cross_entropy = np.array(cross_entropy_2d).reshape(self.shape + - [1]).astype( - self.dtype) + [ + [-np.log(self.X_2d[i][self.label_index_2d[i]])] + for i in range(self.X_2d.shape[0]) + ] + ) + self.cross_entropy = ( + np.array(cross_entropy_2d) + .reshape(self.shape + [1]) + .astype(self.dtype) + ) def init_attr_type(self): self.soft_label = True @@ -283,26 +305,28 @@ class TestCrossEntropyOp6(TestCrossEntropyOp): self.class_num = 17 def test_check_grad(self): - self.check_grad(["X"], - "Y", - max_relative_error=0.05, - numeric_grad_delta=0.001) + self.check_grad( + ["X"], "Y", max_relative_error=0.05, numeric_grad_delta=0.001 + ) class TestCrossEntropyOp7(TestCrossEntropyOp): - """Test cross-entropy with ignore index. - """ + """Test cross-entropy with ignore index.""" def init_label(self): - self.label = np.random.randint(0, - self.class_num, (self.batch_size, 1), - dtype="int64") + self.label = np.random.randint( + 0, self.class_num, (self.batch_size, 1), dtype="int64" + ) def get_cross_entropy(self): self.cross_entropy = np.asmatrix( - [[-np.log(self.x[i][self.label[i][0]])] - if self.label[i][0] != self.ignore_index else [0] - for i in range(self.x.shape[0])]).astype(self.dtype) + [ + [-np.log(self.x[i][self.label[i][0]])] + if self.label[i][0] != self.ignore_index + else [0] + for i in range(self.x.shape[0]) + ] + ).astype(self.dtype) def init_attr_type(self): self.soft_label = False @@ -317,30 +341,35 @@ class TestCrossEntropyOp7(TestCrossEntropyOp): class TestCrossEntropyOp7RemoveLastDim(TestCrossEntropyOp7): - """Test cross-entropy with ignore index with shape [batch_size]. - """ + """Test cross-entropy with ignore index with shape [batch_size].""" def init_label(self): - self.label = np.random.randint(0, - self.class_num, (self.batch_size), - dtype="int64") + self.label = np.random.randint( + 0, self.class_num, (self.batch_size), dtype="int64" + ) def get_cross_entropy(self): self.cross_entropy = np.asmatrix( - [[-np.log(self.x[i][self.label[i]])] - if self.label[i] != self.ignore_index else [0] - for i in range(self.x.shape[0])]).astype(self.dtype) - self.cross_entropy = np.array(self.cross_entropy).reshape( - [self.batch_size]).astype(self.dtype) + [ + [-np.log(self.x[i][self.label[i]])] + if self.label[i] != self.ignore_index + else [0] + for i in range(self.x.shape[0]) + ] + ).astype(self.dtype) + self.cross_entropy = ( + np.array(self.cross_entropy) + .reshape([self.batch_size]) + .astype(self.dtype) + ) # Add Fp16 test def create_test_class(parent, cls_name): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCrossEntropyFP16Op(parent): - def init_dtype_type(self): return np.float16 @@ -352,9 +381,9 @@ def create_test_class(parent, cls_name): def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X'], - 'Y', - max_relative_error=0.9) + self.check_grad_with_place( + place, ['X'], 'Y', max_relative_error=0.9 + ) cls_name = "{0}".format(cls_name) TestCrossEntropyFP16Op.__name__ = cls_name @@ -362,29 +391,32 @@ def create_test_class(parent, cls_name): create_test_class(TestCrossEntropyOp, "TestCrossEntropyF16Op") -#create_test_class(TestCrossEntropyOp2, "TestCrossEntropyF16Op2") +# create_test_class(TestCrossEntropyOp2, "TestCrossEntropyF16Op2") create_test_class(TestCrossEntropyOp3, "TestCrossEntropyF16Op3") create_test_class(TestCrossEntropyOp4, "TestCrossEntropyF16Op4") -create_test_class(TestCrossEntropyOp4RemoveLastDim, - "TestCrossEntropyF16Op4RemoveLastDim") -#create_test_class(TestCrossEntropyOp5, "TestCrossEntropyF16Op5") +create_test_class( + TestCrossEntropyOp4RemoveLastDim, "TestCrossEntropyF16Op4RemoveLastDim" +) +# create_test_class(TestCrossEntropyOp5, "TestCrossEntropyF16Op5") create_test_class(TestCrossEntropyOp6, "TestCrossEntropyF16Op6") create_test_class(TestCrossEntropyOp7, "TestCrossEntropyF16Op7") -create_test_class(TestCrossEntropyOp7RemoveLastDim, - "TestCrossEntropyF16Op7RemoveLastDim") +create_test_class( + TestCrossEntropyOp7RemoveLastDim, "TestCrossEntropyF16Op7RemoveLastDim" +) class TestCrossEntropyOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of cross_entropy must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - lab1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + lab1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) fluid.layers.cross_entropy(x1, lab1) self.assertRaises(TypeError, test_Variable) @@ -392,12 +424,12 @@ class TestCrossEntropyOpError(unittest.TestCase): def test_dtype(): # the input dtype of cross_entropy must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") - lab2 = fluid.layers.data(name='lab2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) + lab2 = fluid.layers.data( + name='lab2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.cross_entropy(x2, lab2) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_cross_op.py b/python/paddle/fluid/tests/unittests/test_cross_op.py index 7ff724d2b5a..73580d6fc8f 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_op.py +++ b/python/paddle/fluid/tests/unittests/test_cross_op.py @@ -21,14 +21,13 @@ from paddle.fluid import Program, program_guard class TestCrossOp(OpTest): - def setUp(self): self.op_type = "cross" self.python_api = paddle.cross self.initTestCase() self.inputs = { 'X': np.random.random(self.shape).astype(self.dtype), - 'Y': np.random.random(self.shape).astype(self.dtype) + 'Y': np.random.random(self.shape).astype(self.dtype), } self.init_output() @@ -53,7 +52,6 @@ class TestCrossOp(OpTest): class TestCrossOpCase1(TestCrossOp): - def initTestCase(self): self.shape = (2048, 3) self.dtype = np.float32 @@ -66,12 +64,13 @@ class TestCrossOpCase1(TestCrossOp): class TestCrossAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 1.0, 1.0], [2.0, 2.0, 2.0], - [3.0, 3.0, 3.0]]) - self.data_y = np.array([[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], - [1.0, 1.0, 1.0]]) + self.data_x = np.array( + [[1.0, 1.0, 1.0], [2.0, 2.0, 2.0], [3.0, 3.0, 3.0]] + ) + self.data_y = np.array( + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]] + ) def test_cross_api(self): self.input_data() @@ -82,14 +81,14 @@ class TestCrossAPI(unittest.TestCase): y = fluid.layers.data(name='y', shape=[-1, 3]) z = paddle.cross(x, y, axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'y': self.data_y - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'y': self.data_y}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 2: @@ -98,14 +97,14 @@ class TestCrossAPI(unittest.TestCase): y = fluid.layers.data(name='y', shape=[-1, 3]) z = paddle.cross(x, y) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'y': self.data_y - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[-1.0, -1.0, -1.0], [2.0, 2.0, 2.0], - [-1.0, -1.0, -1.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'y': self.data_y}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[-1.0, -1.0, -1.0], [2.0, 2.0, 2.0], [-1.0, -1.0, -1.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 3: @@ -134,8 +133,9 @@ class TestCrossAPI(unittest.TestCase): y = fluid.dygraph.to_variable(self.data_y) z = paddle.cross(x, y, axis=1) np_z = z.numpy() - expect_out = np.array([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0]]) + expect_out = np.array( + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_crypto.py b/python/paddle/fluid/tests/unittests/test_crypto.py index 120a4588773..147615905fd 100644 --- a/python/paddle/fluid/tests/unittests/test_crypto.py +++ b/python/paddle/fluid/tests/unittests/test_crypto.py @@ -19,7 +19,6 @@ import unittest class CipherUtilsTestCase(unittest.TestCase): - def test_gen_key(self): key1 = CipherUtils.gen_key(256) key2 = CipherUtils.gen_key_to_file(256, "paddle_aes_test.keyfile") @@ -31,7 +30,6 @@ class CipherUtilsTestCase(unittest.TestCase): class CipherTestCase(unittest.TestCase): - def test_aes_cipher(self): plaintext = "hello world" key = CipherUtils.gen_key(256) diff --git a/python/paddle/fluid/tests/unittests/test_ctc_align.py b/python/paddle/fluid/tests/unittests/test_ctc_align.py index 38e4a0721c6..b11dfe0254a 100644 --- a/python/paddle/fluid/tests/unittests/test_ctc_align.py +++ b/python/paddle/fluid/tests/unittests/test_ctc_align.py @@ -28,8 +28,9 @@ def CTCAlign(input, lod, blank, merge_repeated, padding=0, input_length=None): prev_token = -1 for j in range(cur_offset, cur_offset + lod0[i]): token = input[j][0] - if (token != blank) and not (merge_repeated - and token == prev_token): + if (token != blank) and not ( + merge_repeated and token == prev_token + ): result.append(token) prev_token = token cur_offset += lod0[i] @@ -44,37 +45,44 @@ def CTCAlign(input, lod, blank, merge_repeated, padding=0, input_length=None): prev_token = -1 for j in range(input_length[i][0]): token = input[i][j] - if (token != blank) and not (merge_repeated - and token == prev_token): + if (token != blank) and not ( + merge_repeated and token == prev_token + ): result[i].append(token) prev_token = token start = len(result[i]) output_length.append([start]) for j in range(start, len(input[i])): result[i].append(padding) - result = np.array(result).reshape([len(input), - len(input[0])]).astype("int32") - output_length = np.array(output_length).reshape([len(input), - 1]).astype("int32") + result = ( + np.array(result) + .reshape([len(input), len(input[0])]) + .astype("int32") + ) + output_length = ( + np.array(output_length).reshape([len(input), 1]).astype("int32") + ) return result, output_length class TestCTCAlignOp(OpTest): - def config(self): self.op_type = "ctc_align" self.input_lod = [[11, 7]] self.blank = 0 self.merge_repeated = False - self.input = np.array( - [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, - 0]).reshape([18, 1]).astype("int32") + self.input = ( + np.array([0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0]) + .reshape([18, 1]) + .astype("int32") + ) def setUp(self): self.config() - output = CTCAlign(self.input, self.input_lod, self.blank, - self.merge_repeated) + output = CTCAlign( + self.input, self.input_lod, self.blank, self.merge_repeated + ) self.inputs = { "Input": (self.input, self.input_lod), @@ -82,7 +90,7 @@ class TestCTCAlignOp(OpTest): self.outputs = {"Output": output} self.attrs = { "blank": self.blank, - "merge_repeated": self.merge_repeated + "merge_repeated": self.merge_repeated, } def test_check_output(self): @@ -90,19 +98,19 @@ class TestCTCAlignOp(OpTest): class TestCTCAlignOpCase1(TestCTCAlignOp): - def config(self): self.op_type = "ctc_align" self.input_lod = [[11, 8]] self.blank = 0 self.merge_repeated = True - self.input = np.array( - [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0, - 0]).reshape([19, 1]).astype("int32") + self.input = ( + np.array([0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0, 0]) + .reshape([19, 1]) + .astype("int32") + ) class TestCTCAlignOpCase2(TestCTCAlignOp): - def config(self): self.op_type = "ctc_align" self.input_lod = [[4]] @@ -112,32 +120,43 @@ class TestCTCAlignOpCase2(TestCTCAlignOp): class TestCTCAlignPaddingOp(OpTest): - def config(self): self.op_type = "ctc_align" self.input_lod = [] self.blank = 0 self.padding_value = 0 self.merge_repeated = True - self.input = np.array([[0, 2, 4, 4, 0, 6, 3, 6, 6, 0, 0], - [1, 1, 3, 0, 0, 4, 5, 6, 0, 0, - 0]]).reshape([2, 11]).astype("int32") + self.input = ( + np.array( + [ + [0, 2, 4, 4, 0, 6, 3, 6, 6, 0, 0], + [1, 1, 3, 0, 0, 4, 5, 6, 0, 0, 0], + ] + ) + .reshape([2, 11]) + .astype("int32") + ) self.input_length = np.array([[9], [8]]).reshape([2, 1]).astype("int32") def setUp(self): self.config() - output, output_length = CTCAlign(self.input, self.input_lod, self.blank, - self.merge_repeated, - self.padding_value, self.input_length) + output, output_length = CTCAlign( + self.input, + self.input_lod, + self.blank, + self.merge_repeated, + self.padding_value, + self.input_length, + ) self.inputs = { "Input": (self.input, self.input_lod), - "InputLength": self.input_length + "InputLength": self.input_length, } self.outputs = {"Output": output, "OutputLength": output_length} self.attrs = { "blank": self.blank, "merge_repeated": self.merge_repeated, - "padding_value": self.padding_value + "padding_value": self.padding_value, } def test_check_output(self): @@ -145,18 +164,22 @@ class TestCTCAlignPaddingOp(OpTest): class TestCTCAlignOpCase3(TestCTCAlignPaddingOp): - def config(self): self.op_type = "ctc_align" self.blank = 0 self.input_lod = [] self.merge_repeated = True self.padding_value = 0 - self.input = np.array([[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], - [0, 7, 7, 7, 0, 0]]).reshape([3, - 6]).astype("int32") - self.input_length = np.array([[6], [5], - [4]]).reshape([3, 1]).astype("int32") + self.input = ( + np.array( + [[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], [0, 7, 7, 7, 0, 0]] + ) + .reshape([3, 6]) + .astype("int32") + ) + self.input_length = ( + np.array([[6], [5], [4]]).reshape([3, 1]).astype("int32") + ) class TestCTCAlignOpCase4(TestCTCAlignPaddingOp): @@ -170,30 +193,38 @@ class TestCTCAlignOpCase4(TestCTCAlignPaddingOp): self.input_lod = [] self.merge_repeated = False self.padding_value = 0 - self.input = np.array([[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], - [0, 7, 7, 7, 0, 0]]).reshape([3, - 6]).astype("int32") - self.input_length = np.array([[6], [5], - [4]]).reshape([3, 1]).astype("int32") + self.input = ( + np.array( + [[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], [0, 7, 7, 7, 0, 0]] + ) + .reshape([3, 6]) + .astype("int32") + ) + self.input_length = ( + np.array([[6], [5], [4]]).reshape([3, 1]).astype("int32") + ) class TestCTCAlignOpCase5(TestCTCAlignPaddingOp): - def config(self): self.op_type = "ctc_align" self.blank = 0 self.input_lod = [] self.merge_repeated = False self.padding_value = 1 - self.input = np.array([[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], - [0, 7, 1, 7, 0, 0]]).reshape([3, - 6]).astype("int32") - self.input_length = np.array([[6], [5], - [4]]).reshape([3, 1]).astype("int32") + self.input = ( + np.array( + [[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], [0, 7, 1, 7, 0, 0]] + ) + .reshape([3, 6]) + .astype("int32") + ) + self.input_length = ( + np.array([[6], [5], [4]]).reshape([3, 1]).astype("int32") + ) class TestCTCAlignOpApi(unittest.TestCase): - def test_api(self): x = fluid.layers.data('x', shape=[4], dtype='float32') y = fluid.layers.ctc_greedy_decoder(x, blank=0) @@ -201,11 +232,13 @@ class TestCTCAlignOpApi(unittest.TestCase): x_pad = fluid.layers.data('x_pad', shape=[4, 4], dtype='float32') x_pad_len = fluid.layers.data('x_pad_len', shape=[1], dtype='int64') y_pad, y_pad_len = fluid.layers.ctc_greedy_decoder( - x_pad, blank=0, input_length=x_pad_len) + x_pad, blank=0, input_length=x_pad_len + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(8, 4).astype("float32"), [[4, 4]], place) + np.random.rand(8, 4).astype("float32"), [[4, 4]], place + ) x_pad_tensor = np.random.rand(2, 4, 4).astype("float32") x_pad_len_tensor = np.array([[4], [4]]).reshape([2, 1]).astype("int64") @@ -213,17 +246,18 @@ class TestCTCAlignOpApi(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_tensor, - 'x_pad': x_pad_tensor, - 'x_pad_len': x_pad_len_tensor - }, - fetch_list=[y, y_pad, y_pad_len], - return_numpy=False) + ret = exe.run( + feed={ + 'x': x_tensor, + 'x_pad': x_pad_tensor, + 'x_pad_len': x_pad_len_tensor, + }, + fetch_list=[y, y_pad, y_pad_len], + return_numpy=False, + ) class BadInputTestCTCAlignr(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_cudnn_version.py b/python/paddle/fluid/tests/unittests/test_cuda_cudnn_version.py index 36637971f9e..d8229247a81 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_cudnn_version.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_cudnn_version.py @@ -17,7 +17,6 @@ import paddle class TestCPUVersion(unittest.TestCase): - def test_cuda_cudnn_version_in_cpu_package(self): if not paddle.is_compiled_with_cuda(): self.assertEqual(paddle.version.cuda(), 'False') diff --git a/python/paddle/fluid/tests/unittests/test_cuda_device_count.py b/python/paddle/fluid/tests/unittests/test_cuda_device_count.py index 482a3413caf..7789bcd944d 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_device_count.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_device_count.py @@ -17,7 +17,6 @@ import unittest class TestDeviceCount(unittest.TestCase): - def test_device_count(self): s = paddle.device.cuda.device_count() self.assertIsNotNone(s) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_device_name_capability.py b/python/paddle/fluid/tests/unittests/test_cuda_device_name_capability.py index 0d749c5d177..07ef914f0a3 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_device_name_capability.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_device_name_capability.py @@ -17,7 +17,6 @@ import unittest class TestDeviceName(unittest.TestCase): - def test_device_name_default(self): if paddle.is_compiled_with_cuda(): name = paddle.device.cuda.get_device_name() @@ -35,7 +34,6 @@ class TestDeviceName(unittest.TestCase): class TestDeviceCapability(unittest.TestCase): - def test_device_capability_default(self): if paddle.is_compiled_with_cuda(): capability = paddle.device.cuda.get_device_capability() @@ -49,7 +47,8 @@ class TestDeviceCapability(unittest.TestCase): def test_device_capability_CUDAPlace(self): if paddle.is_compiled_with_cuda(): capability = paddle.device.cuda.get_device_capability( - paddle.CUDAPlace(0)) + paddle.CUDAPlace(0) + ) self.assertIsNotNone(capability) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_empty_cache.py b/python/paddle/fluid/tests/unittests/test_cuda_empty_cache.py index 0ec066eb7cd..4aefb234bbf 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_empty_cache.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_empty_cache.py @@ -17,7 +17,6 @@ import unittest class TestEmptyCache(unittest.TestCase): - def test_empty_cache(self): x = paddle.randn((2, 10, 12)).astype('float32') del x diff --git a/python/paddle/fluid/tests/unittests/test_cuda_graph.py b/python/paddle/fluid/tests/unittests/test_cuda_graph.py index 035c71b1dad..ce446264f39 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_graph.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_graph.py @@ -28,19 +28,21 @@ def can_use_cuda_graph(): class TestCUDAGraph(unittest.TestCase): - def setUp(self): if can_use_cuda_graph(): - paddle.set_flags({ - 'FLAGS_allocator_strategy': 'auto_growth', - 'FLAGS_sync_nccl_allreduce': False, - 'FLAGS_cudnn_deterministic': True, - 'FLAGS_use_stream_safe_cuda_allocator': False, - }) + paddle.set_flags( + { + 'FLAGS_allocator_strategy': 'auto_growth', + 'FLAGS_sync_nccl_allreduce': False, + 'FLAGS_cudnn_deterministic': True, + 'FLAGS_use_stream_safe_cuda_allocator': False, + } + ) def random_tensor(self, shape): return paddle.to_tensor( - np.random.randint(low=0, high=10, size=shape).astype("float32")) + np.random.randint(low=0, high=10, size=shape).astype("float32") + ) @switch_to_static_graph def test_cuda_graph_static_graph(self): @@ -48,10 +50,12 @@ class TestCUDAGraph(unittest.TestCase): return seed = 100 - loss_cuda_graph = self.cuda_graph_static_graph_main(seed, - use_cuda_graph=True) + loss_cuda_graph = self.cuda_graph_static_graph_main( + seed, use_cuda_graph=True + ) loss_no_cuda_graph = self.cuda_graph_static_graph_main( - seed, use_cuda_graph=False) + seed, use_cuda_graph=False + ) self.assertEqual(loss_cuda_graph, loss_no_cuda_graph) def cuda_graph_static_graph_main(self, seed, use_cuda_graph): @@ -65,18 +69,19 @@ class TestCUDAGraph(unittest.TestCase): startup = paddle.static.Program() main = paddle.static.Program() with paddle.static.program_guard(main, startup): - image = paddle.static.data(name="image", - shape=image_shape, - dtype='float32') - label = paddle.static.data(name="label", - shape=label_shape, - dtype='int64') + image = paddle.static.data( + name="image", shape=image_shape, dtype='float32' + ) + label = paddle.static.data( + name="label", shape=label_shape, dtype='int64' + ) image.persistable = True label.persistable = True loss = simple_fc_net_with_inputs(image, label, class_num) loss.persistable = True lr = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04]) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04] + ) optimizer = paddle.optimizer.SGD(learning_rate=lr) optimizer.minimize(loss) place = paddle.CUDAPlace(0) @@ -89,9 +94,10 @@ class TestCUDAGraph(unittest.TestCase): build_strategy.fix_op_run_order = True build_strategy.fuse_all_optimizer_ops = True compiled_program = paddle.static.CompiledProgram( - main).with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - places=place) + main + ).with_data_parallel( + loss_name=loss.name, build_strategy=build_strategy, places=place + ) image_t = scope.var(image.name).get_tensor() label_t = scope.var(label.name).get_tensor() loss_t = scope.var(loss.name).get_tensor() @@ -101,12 +107,14 @@ class TestCUDAGraph(unittest.TestCase): cuda_graph = None for batch_id in range(20): image_t.set( - np.random.rand(*image_shape).astype('float32'), place) + np.random.rand(*image_shape).astype('float32'), place + ) label_t.set( - np.random.randint(low=0, - high=class_num, - size=label_shape, - dtype='int64'), place) + np.random.randint( + low=0, high=class_num, size=label_shape, dtype='int64' + ), + place, + ) if batch_id == 1 and use_cuda_graph: cuda_graph = CUDAGraph(place, mode="global") @@ -195,7 +203,6 @@ class TestCUDAGraph(unittest.TestCase): return class AutoIncDataset(paddle.io.Dataset): - def __init__(self, n, dtype): self.n = n self.dtype = dtype @@ -209,10 +216,9 @@ class TestCUDAGraph(unittest.TestCase): n = 100 dtype = 'int64' dataset = AutoIncDataset(n, dtype) - data_loader = paddle.io.DataLoader(dataset, - batch_size=1, - num_workers=2, - use_buffer_reader=True) + data_loader = paddle.io.DataLoader( + dataset, batch_size=1, num_workers=2, use_buffer_reader=True + ) x = None y = None diff --git a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph.py b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph.py index ecd3f406e08..7ae6a9f8e19 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph.py @@ -20,7 +20,6 @@ from paddle.device.cuda.graphs import wrap_cuda_graph, is_cuda_graph_supported class SimpleModel(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleModel, self).__init__() self.linear = nn.Linear(in_size, out_size) @@ -39,7 +38,6 @@ class SimpleModel(nn.Layer): class TestSimpleModel(unittest.TestCase): - def setUp(self): paddle.set_flags({'FLAGS_eager_delete_tensor_gb': 0.0}) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static.py b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static.py index b45a862ffbf..077caebc03f 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static.py @@ -21,7 +21,6 @@ paddle.enable_static() class SimpleModel(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleModel, self).__init__() self.linear = nn.Linear(in_size, out_size) @@ -40,7 +39,6 @@ class SimpleModel(nn.Layer): class TestCudaGraphAttrAll(unittest.TestCase): - def test_all_program(self): if not is_cuda_graph_supported(): return @@ -59,8 +57,10 @@ class TestCudaGraphAttrAll(unittest.TestCase): if op._cuda_graph_attr is None: # the loss and opt are not wrapped assert op.type in [ - 'sgd', 'reduce_mean', 'fill_constant', - 'reduce_mean_grad' + 'sgd', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', ] else: assert op._cuda_graph_attr == 'thread_local;0;0' diff --git a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static_run.py b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static_run.py index 445211d35a1..0d2b737816b 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static_run.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static_run.py @@ -16,13 +16,16 @@ import paddle import paddle.nn as nn import unittest import numpy as np -from paddle.device.cuda.graphs import wrap_cuda_graph, is_cuda_graph_supported, cuda_graph_transform +from paddle.device.cuda.graphs import ( + wrap_cuda_graph, + is_cuda_graph_supported, + cuda_graph_transform, +) paddle.enable_static() class SimpleModel(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleModel, self).__init__() self.linear = nn.Linear(in_size, out_size) @@ -41,7 +44,6 @@ class SimpleModel(nn.Layer): class TestCudaGraphAttrAll(unittest.TestCase): - def setUp(self): paddle.set_flags({'FLAGS_eager_delete_tensor_gb': 0.0}) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_max_memory_allocated.py b/python/paddle/fluid/tests/unittests/test_cuda_max_memory_allocated.py index 5f047977ed3..909b9a44b11 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_max_memory_allocated.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_max_memory_allocated.py @@ -15,12 +15,15 @@ import paddle import unittest from paddle.fluid import core -from paddle.device.cuda import device_count, memory_allocated, max_memory_allocated +from paddle.device.cuda import ( + device_count, + memory_allocated, + max_memory_allocated, +) from paddle.fluid.framework import _test_eager_guard class TestMaxMemoryAllocated(unittest.TestCase): - def func_test_max_memory_allocated(self, device=None): if core.is_compiled_with_cuda(): alloc_time = 100 @@ -29,13 +32,15 @@ class TestMaxMemoryAllocated(unittest.TestCase): for i in range(alloc_time): shape = paddle.randint(max_alloc_size) tensor = paddle.zeros(shape) - peak_memory_allocated_size = max(peak_memory_allocated_size, - memory_allocated(device)) + peak_memory_allocated_size = max( + peak_memory_allocated_size, memory_allocated(device) + ) del shape del tensor - self.assertEqual(peak_memory_allocated_size, - max_memory_allocated(device)) + self.assertEqual( + peak_memory_allocated_size, max_memory_allocated(device) + ) def test_max_memory_allocated(self): with _test_eager_guard(): @@ -60,7 +65,11 @@ class TestMaxMemoryAllocated(unittest.TestCase): if core.is_compiled_with_cuda(): wrong_device = [ core.CPUPlace(), - device_count() + 1, -2, 0.5, "gpu1", "npu" + device_count() + 1, + -2, + 0.5, + "gpu1", + "npu", ] for device in wrong_device: with self.assertRaises(BaseException): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_max_memory_reserved.py b/python/paddle/fluid/tests/unittests/test_cuda_max_memory_reserved.py index 936a084abb7..e067d293074 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_max_memory_reserved.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_max_memory_reserved.py @@ -15,11 +15,14 @@ import paddle import unittest from paddle.fluid import core -from paddle.device.cuda import device_count, memory_reserved, max_memory_reserved +from paddle.device.cuda import ( + device_count, + memory_reserved, + max_memory_reserved, +) class TestMaxMemoryreserved(unittest.TestCase): - def test_max_memory_reserved(self, device=None): if core.is_compiled_with_cuda(): alloc_time = 100 @@ -28,13 +31,15 @@ class TestMaxMemoryreserved(unittest.TestCase): for i in range(alloc_time): shape = paddle.randint(max_alloc_size) tensor = paddle.zeros(shape) - peak_memory_reserved_size = max(peak_memory_reserved_size, - memory_reserved(device)) + peak_memory_reserved_size = max( + peak_memory_reserved_size, memory_reserved(device) + ) del shape del tensor - self.assertEqual(peak_memory_reserved_size, - max_memory_reserved(device)) + self.assertEqual( + peak_memory_reserved_size, max_memory_reserved(device) + ) def test_max_memory_reserved_for_all_places(self): if core.is_compiled_with_cuda(): @@ -49,7 +54,11 @@ class TestMaxMemoryreserved(unittest.TestCase): if core.is_compiled_with_cuda(): wrong_device = [ core.CPUPlace(), - device_count() + 1, -2, 0.5, "gpu1", "npu" + device_count() + 1, + -2, + 0.5, + "gpu1", + "npu", ] for device in wrong_device: with self.assertRaises(BaseException): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_memory_allocated.py b/python/paddle/fluid/tests/unittests/test_cuda_memory_allocated.py index 4e82ab7326c..5b21a9b07e8 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_memory_allocated.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_memory_allocated.py @@ -19,7 +19,6 @@ from paddle.device.cuda import device_count, memory_allocated class TestMemoryAllocated(unittest.TestCase): - def test_memory_allocated(self, device=None): if core.is_compiled_with_cuda(): tensor = paddle.zeros(shape=[256]) @@ -40,7 +39,11 @@ class TestMemoryAllocated(unittest.TestCase): if core.is_compiled_with_cuda(): wrong_device = [ core.CPUPlace(), - device_count() + 1, -2, 0.5, "gpu1", "npu" + device_count() + 1, + -2, + 0.5, + "gpu1", + "npu", ] for device in wrong_device: with self.assertRaises(BaseException): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_memory_reserved.py b/python/paddle/fluid/tests/unittests/test_cuda_memory_reserved.py index f5db61a1e65..6b2cdc45f19 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_memory_reserved.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_memory_reserved.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class TestMemoryreserved(unittest.TestCase): - def func_test_memory_reserved(self, device=None): if core.is_compiled_with_cuda(): tensor = paddle.zeros(shape=[256]) @@ -51,7 +50,11 @@ class TestMemoryreserved(unittest.TestCase): if core.is_compiled_with_cuda(): wrong_device = [ core.CPUPlace(), - device_count() + 1, -2, 0.5, "gpu1", "npu" + device_count() + 1, + -2, + 0.5, + "gpu1", + "npu", ] for device in wrong_device: with self.assertRaises(BaseException): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_random_seed.py b/python/paddle/fluid/tests/unittests/test_cuda_random_seed.py index f12d3a5a4d6..d9bef511c9c 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_random_seed.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_random_seed.py @@ -24,8 +24,9 @@ import shutil import tempfile -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Only test cuda Random Generator") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Only test cuda Random Generator" +) class TestGeneratorSeed(unittest.TestCase): """ Test cases for cpu generator seed. @@ -39,18 +40,15 @@ class TestGeneratorSeed(unittest.TestCase): gen.manual_seed(111111111) st = paddle.get_cuda_rng_state() - x = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) - x_again = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) - x_third = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) + x = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) + x_again = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) + x_third = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) print("x: {}".format(x.numpy())) print("x_again: {}".format(x_again.numpy())) x = x + x_again + x_third @@ -58,18 +56,15 @@ class TestGeneratorSeed(unittest.TestCase): paddle.set_cuda_rng_state(st) - x1 = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) - x1_again = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) - x1_third = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) + x1 = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) + x1_again = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) + x1_third = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) x1 = x1 + x1_again + x1_third y1 = fluid.layers.dropout(x1, 0.5) y_np = y.numpy() @@ -136,26 +131,30 @@ class TestGeneratorSeed(unittest.TestCase): result_1 = fluid.layers.fc( input=x, size=10, - param_attr=fluid.initializer.TruncatedNormal(loc=0.0, - scale=2.0)) + param_attr=fluid.initializer.TruncatedNormal( + loc=0.0, scale=2.0 + ), + ) result_2 = fluid.layers.fc( input=x, size=10, - param_attr=fluid.initializer.TruncatedNormal(loc=0.0, - scale=2.0)) + param_attr=fluid.initializer.TruncatedNormal( + loc=0.0, scale=2.0 + ), + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) paddle.seed(123123143) with fluid.program_guard(train_program, startup_program): exe.run(startup_program) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_stream_event.py b/python/paddle/fluid/tests/unittests/test_cuda_stream_event.py index 099f19bd03e..600cd04d197 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_stream_event.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_stream_event.py @@ -21,7 +21,6 @@ import numpy as np class TestCurrentStream(unittest.TestCase): - def test_current_stream(self): if paddle.is_compiled_with_cuda(): s = cuda.current_stream() @@ -39,7 +38,6 @@ class TestCurrentStream(unittest.TestCase): class TestSynchronize(unittest.TestCase): - def test_synchronize(self): if paddle.is_compiled_with_cuda(): self.assertIsNone(cuda.synchronize()) @@ -50,7 +48,6 @@ class TestSynchronize(unittest.TestCase): class TestCUDAStream(unittest.TestCase): - def test_cuda_stream(self): if paddle.is_compiled_with_cuda(): s = paddle.device.cuda.Stream() @@ -88,7 +85,6 @@ class TestCUDAStream(unittest.TestCase): class TestCUDAEvent(unittest.TestCase): - def test_cuda_event(self): if paddle.is_compiled_with_cuda(): e = paddle.device.cuda.Event(True, False, False) @@ -158,14 +154,15 @@ class TestStreamGuard(unittest.TestCase): def test_set_current_stream_raise_error(self): if paddle.is_compiled_with_cuda(): - self.assertRaises(TypeError, paddle.device.cuda._set_current_stream, - np.zeros(5)) - self.assertRaises(TypeError, paddle.device.cuda._set_current_stream, - None) + self.assertRaises( + TypeError, paddle.device.cuda._set_current_stream, np.zeros(5) + ) + self.assertRaises( + TypeError, paddle.device.cuda._set_current_stream, None + ) class TestRawStream(unittest.TestCase): - def test_cuda_stream(self): if paddle.is_compiled_with_cuda(): cuda_stream = paddle.device.cuda.current_stream().cuda_stream diff --git a/python/paddle/fluid/tests/unittests/test_cudnn_grucell.py b/python/paddle/fluid/tests/unittests/test_cudnn_grucell.py index d82da93ba04..39c25c909fd 100644 --- a/python/paddle/fluid/tests/unittests/test_cudnn_grucell.py +++ b/python/paddle/fluid/tests/unittests/test_cudnn_grucell.py @@ -23,15 +23,16 @@ np.random.seed = 123 def sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) def tanh(x): - return 2. * sigmoid(2. * x) - 1. + return 2.0 * sigmoid(2.0 * x) - 1.0 -def cudnn_step(step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh, - bias_hh): +def cudnn_step( + step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh, bias_hh +): igates = np.matmul(step_input_np, weight_ih.transpose(1, 0)) igates += bias_ih hgates = np.matmul(pre_hidden_np, weight_hh.transpose(1, 0)) @@ -55,8 +56,9 @@ def cudnn_step(step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh, return new_hidden -def non_cudnn_step(step_in, pre_hidden, gate_w, gate_b, candidate_w, - candidate_b): +def non_cudnn_step( + step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b +): concat_1 = np.concatenate([step_in, pre_hidden], 1) gate_input = np.matmul(concat_1, gate_w) @@ -77,7 +79,6 @@ def non_cudnn_step(step_in, pre_hidden, gate_w, gate_b, candidate_w, class TestCudnnGRU(unittest.TestCase): - def setUp(self): self.input_size = 100 self.hidden_size = 200 @@ -93,8 +94,9 @@ class TestCudnnGRU(unittest.TestCase): with fluid.dygraph.guard(place): param_attr = fluid.ParamAttr(name="param_attr") bias_attr = fluid.ParamAttr(name="bias_attr") - named_cudnn_gru = GRUCell(self.hidden_size, self.input_size, - param_attr, bias_attr) + named_cudnn_gru = GRUCell( + self.hidden_size, self.input_size, param_attr, bias_attr + ) cudnn_gru = GRUCell(self.hidden_size, self.input_size) param_list = cudnn_gru.state_dict() @@ -109,51 +111,55 @@ class TestCudnnGRU(unittest.TestCase): weight_ih = param_list[weight_ih_name].numpy() weight_ih = np.random.uniform( - -0.1, 0.1, size=weight_ih.shape).astype('float64') + -0.1, 0.1, size=weight_ih.shape + ).astype('float64') param_list[weight_ih_name].set_value(weight_ih) named_param_list[weight_ih_name].set_value(weight_ih) bias_ih = param_list[bias_ih_name].numpy() - bias_ih = np.random.uniform(-0.1, 0.1, - size=bias_ih.shape).astype('float64') + bias_ih = np.random.uniform(-0.1, 0.1, size=bias_ih.shape).astype( + 'float64' + ) param_list[bias_ih_name].set_value(bias_ih) named_param_list[bias_ih_name].set_value(bias_ih) weight_hh = param_list[weight_hh_name].numpy() weight_hh = np.random.uniform( - -0.1, 0.1, size=weight_hh.shape).astype('float64') + -0.1, 0.1, size=weight_hh.shape + ).astype('float64') param_list[weight_hh_name].set_value(weight_hh) named_param_list[weight_hh_name].set_value(weight_hh) bias_hh = param_list[bias_hh_name].numpy() - bias_hh = np.random.uniform(-0.1, 0.1, - size=bias_hh.shape).astype('float64') + bias_hh = np.random.uniform(-0.1, 0.1, size=bias_hh.shape).astype( + 'float64' + ) param_list[bias_hh_name].set_value(bias_hh) named_param_list[bias_hh_name].set_value(bias_hh) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float64') pre_hidden_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') step_input_var = fluid.dygraph.to_variable(step_input_np) pre_hidden_var = fluid.dygraph.to_variable(pre_hidden_np) api_out = cudnn_gru(step_input_var, pre_hidden_var) named_api_out = named_cudnn_gru(step_input_var, pre_hidden_var) - np_out = cudnn_step(step_input_np, pre_hidden_np, weight_ih, bias_ih, - weight_hh, bias_hh) + np_out = cudnn_step( + step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh, bias_hh + ) np.testing.assert_allclose(api_out.numpy(), np_out, rtol=1e-05, atol=0) - np.testing.assert_allclose(named_api_out.numpy(), - np_out, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + named_api_out.numpy(), np_out, rtol=1e-05, atol=0 + ) class TestNonCudnnGRU(unittest.TestCase): - def setUp(self): self.input_size = 100 self.hidden_size = 200 @@ -169,14 +175,16 @@ class TestNonCudnnGRU(unittest.TestCase): with fluid.dygraph.guard(place): param_attr = fluid.ParamAttr(name="param_attr") bias_attr = fluid.ParamAttr(name="bias_attr") - named_non_cudnn_gru = GRUCell(self.hidden_size, - self.input_size, - param_attr, - bias_attr, - use_cudnn_impl=False) - non_cudnn_gru = GRUCell(self.hidden_size, - self.input_size, - use_cudnn_impl=False) + named_non_cudnn_gru = GRUCell( + self.hidden_size, + self.input_size, + param_attr, + bias_attr, + use_cudnn_impl=False, + ) + non_cudnn_gru = GRUCell( + self.hidden_size, self.input_size, use_cudnn_impl=False + ) param_list = non_cudnn_gru.state_dict() named_param_list = named_non_cudnn_gru.state_dict() @@ -189,48 +197,58 @@ class TestNonCudnnGRU(unittest.TestCase): candidate_b_name = "_candidate_bias" gate_w = param_list[gate_w_name].numpy() - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float64') + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float64' + ) param_list[gate_w_name].set_value(gate_w) named_param_list[gate_w_name].set_value(gate_w) gate_b = param_list[gate_b_name].numpy() - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float64') + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float64' + ) param_list[gate_b_name].set_value(gate_b) named_param_list[gate_b_name].set_value(gate_b) candidate_w = param_list[candidate_w_name].numpy() candidate_w = np.random.uniform( - -0.1, 0.1, size=candidate_w.shape).astype('float64') + -0.1, 0.1, size=candidate_w.shape + ).astype('float64') param_list[candidate_w_name].set_value(candidate_w) named_param_list[candidate_w_name].set_value(candidate_w) candidate_b = param_list[candidate_b_name].numpy() candidate_b = np.random.uniform( - -0.1, 0.1, size=candidate_b.shape).astype('float64') + -0.1, 0.1, size=candidate_b.shape + ).astype('float64') param_list[candidate_b_name].set_value(candidate_b) named_param_list[candidate_b_name].set_value(candidate_b) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float64') pre_hidden_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') step_input_var = fluid.dygraph.to_variable(step_input_np) pre_hidden_var = fluid.dygraph.to_variable(pre_hidden_np) api_out = non_cudnn_gru(step_input_var, pre_hidden_var) named_api_out = named_non_cudnn_gru(step_input_var, pre_hidden_var) - np_out = non_cudnn_step(step_input_np, pre_hidden_np, gate_w, gate_b, - candidate_w, candidate_b) + np_out = non_cudnn_step( + step_input_np, + pre_hidden_np, + gate_w, + gate_b, + candidate_w, + candidate_b, + ) np.testing.assert_allclose(api_out.numpy(), np_out, rtol=1e-05, atol=0) - np.testing.assert_allclose(named_api_out.numpy(), - np_out, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + named_api_out.numpy(), np_out, rtol=1e-05, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cudnn_lstmcell.py b/python/paddle/fluid/tests/unittests/test_cudnn_lstmcell.py index d21ec3033ed..4eacff27d14 100644 --- a/python/paddle/fluid/tests/unittests/test_cudnn_lstmcell.py +++ b/python/paddle/fluid/tests/unittests/test_cudnn_lstmcell.py @@ -23,19 +23,16 @@ np.random.seed = 123 def sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) def tanh(x): - return 2. * sigmoid(2. * x) - 1. + return 2.0 * sigmoid(2.0 * x) - 1.0 -def non_cudnn_step(step_in, - pre_hidden, - pre_cell, - gate_w, - gate_b, - forget_bias=1.0): +def non_cudnn_step( + step_in, pre_hidden, pre_cell, gate_w, gate_b, forget_bias=1.0 +): concat_1 = np.concatenate([step_in, pre_hidden], 1) gate_input = np.matmul(concat_1, gate_w) @@ -48,8 +45,15 @@ def non_cudnn_step(step_in, return new_hidden, new_cell -def cudnn_step(step_input_np, pre_hidden_np, pre_cell_np, weight_ih, bias_ih, - weight_hh, bias_hh): +def cudnn_step( + step_input_np, + pre_hidden_np, + pre_cell_np, + weight_ih, + bias_ih, + weight_hh, + bias_hh, +): igates = np.matmul(step_input_np, weight_ih.transpose(1, 0)) igates = igates + bias_ih @@ -78,7 +82,6 @@ def cudnn_step(step_input_np, pre_hidden_np, pre_cell_np, weight_ih, bias_ih, class TestCudnnLSTM(unittest.TestCase): - def setUp(self): self.input_size = 100 self.hidden_size = 200 @@ -93,8 +96,9 @@ class TestCudnnLSTM(unittest.TestCase): with fluid.dygraph.guard(place): param_attr = fluid.ParamAttr(name="param_attr") bias_attr = fluid.ParamAttr(name="bias_attr") - named_cudnn_lstm = LSTMCell(self.hidden_size, self.input_size, - param_attr, bias_attr) + named_cudnn_lstm = LSTMCell( + self.hidden_size, self.input_size, param_attr, bias_attr + ) cudnn_lstm = LSTMCell(self.hidden_size, self.input_size) param_list = cudnn_lstm.state_dict() @@ -108,73 +112,79 @@ class TestCudnnLSTM(unittest.TestCase): bias_hh_name = "_bias_hh" weight_ih = param_list[weight_ih_name].numpy() weight_ih = np.random.uniform( - -0.1, 0.1, size=weight_ih.shape).astype('float64') + -0.1, 0.1, size=weight_ih.shape + ).astype('float64') param_list[weight_ih_name].set_value(weight_ih) named_param_list[weight_ih_name].set_value(weight_ih) bias_ih = param_list[bias_ih_name].numpy() - bias_ih = np.random.uniform(-0.1, 0.1, - size=bias_ih.shape).astype('float64') + bias_ih = np.random.uniform(-0.1, 0.1, size=bias_ih.shape).astype( + 'float64' + ) param_list[bias_ih_name].set_value(bias_ih) named_param_list[bias_ih_name].set_value(bias_ih) weight_hh = param_list[weight_hh_name].numpy() weight_hh = np.random.uniform( - -0.1, 0.1, size=weight_hh.shape).astype('float64') + -0.1, 0.1, size=weight_hh.shape + ).astype('float64') param_list[weight_hh_name].set_value(weight_hh) named_param_list[weight_hh_name].set_value(weight_hh) bias_hh = param_list[bias_hh_name].numpy() - bias_hh = np.random.uniform(-0.1, 0.1, - size=bias_hh.shape).astype('float64') + bias_hh = np.random.uniform(-0.1, 0.1, size=bias_hh.shape).astype( + 'float64' + ) param_list[bias_hh_name].set_value(bias_hh) named_param_list[bias_hh_name].set_value(bias_hh) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float64') pre_hidden_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') pre_cell_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') step_input_var = fluid.dygraph.to_variable(step_input_np) pre_hidden_var = fluid.dygraph.to_variable(pre_hidden_np) pre_cell_var = fluid.dygraph.to_variable(pre_cell_np) api_out = cudnn_lstm(step_input_var, pre_hidden_var, pre_cell_var) - named_api_out = named_cudnn_lstm(step_input_var, pre_hidden_var, - pre_cell_var) + named_api_out = named_cudnn_lstm( + step_input_var, pre_hidden_var, pre_cell_var + ) api_hidden_out = api_out[0] api_cell_out = api_out[1] named_api_hidden_out = named_api_out[0] named_api_cell_out = named_api_out[1] - np_hidden_out, np_cell_out = cudnn_step(step_input_np, - pre_hidden_np, pre_cell_np, - weight_ih, bias_ih, - weight_hh, bias_hh) - np.testing.assert_allclose(api_hidden_out.numpy(), - np_hidden_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(api_cell_out.numpy(), - np_cell_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(named_api_hidden_out.numpy(), - np_hidden_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(named_api_cell_out.numpy(), - np_cell_out, - rtol=1e-05, - atol=0) + np_hidden_out, np_cell_out = cudnn_step( + step_input_np, + pre_hidden_np, + pre_cell_np, + weight_ih, + bias_ih, + weight_hh, + bias_hh, + ) + np.testing.assert_allclose( + api_hidden_out.numpy(), np_hidden_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + api_cell_out.numpy(), np_cell_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + named_api_hidden_out.numpy(), np_hidden_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + named_api_cell_out.numpy(), np_cell_out, rtol=1e-05, atol=0 + ) class TestNonCudnnLSTM(unittest.TestCase): - def setUp(self): self.input_size = 100 self.hidden_size = 200 @@ -189,14 +199,16 @@ class TestNonCudnnLSTM(unittest.TestCase): with fluid.dygraph.guard(place): param_attr = fluid.ParamAttr(name="param_attr") bias_attr = fluid.ParamAttr(name="bias_attr") - named_cudnn_lstm = LSTMCell(self.hidden_size, - self.input_size, - param_attr, - bias_attr, - use_cudnn_impl=False) - cudnn_lstm = LSTMCell(self.hidden_size, - self.input_size, - use_cudnn_impl=False) + named_cudnn_lstm = LSTMCell( + self.hidden_size, + self.input_size, + param_attr, + bias_attr, + use_cudnn_impl=False, + ) + cudnn_lstm = LSTMCell( + self.hidden_size, self.input_size, use_cudnn_impl=False + ) param_list = cudnn_lstm.state_dict() named_param_list = named_cudnn_lstm.state_dict() @@ -207,59 +219,58 @@ class TestNonCudnnLSTM(unittest.TestCase): gate_b_name = "_bias" gate_w = param_list[gate_w_name].numpy() - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float64') + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float64' + ) param_list[gate_w_name].set_value(gate_w) named_param_list[gate_w_name].set_value(gate_w) gate_b = param_list[gate_b_name].numpy() - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float64') + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float64' + ) param_list[gate_b_name].set_value(gate_b) named_param_list[gate_b_name].set_value(gate_b) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float64') pre_hidden_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') pre_cell_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') step_input_var = fluid.dygraph.to_variable(step_input_np) pre_hidden_var = fluid.dygraph.to_variable(pre_hidden_np) pre_cell_var = fluid.dygraph.to_variable(pre_cell_np) api_out = cudnn_lstm(step_input_var, pre_hidden_var, pre_cell_var) - named_api_out = named_cudnn_lstm(step_input_var, pre_hidden_var, - pre_cell_var) + named_api_out = named_cudnn_lstm( + step_input_var, pre_hidden_var, pre_cell_var + ) api_hidden_out = api_out[0] api_cell_out = api_out[1] named_api_hidden_out = named_api_out[0] named_api_cell_out = named_api_out[1] - np_hidden_out, np_cell_out = non_cudnn_step(step_input_np, - pre_hidden_np, - pre_cell_np, gate_w, - gate_b) - - np.testing.assert_allclose(api_hidden_out.numpy(), - np_hidden_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(api_cell_out.numpy(), - np_cell_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(named_api_hidden_out.numpy(), - np_hidden_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(named_api_cell_out.numpy(), - np_cell_out, - rtol=1e-05, - atol=0) + np_hidden_out, np_cell_out = non_cudnn_step( + step_input_np, pre_hidden_np, pre_cell_np, gate_w, gate_b + ) + + np.testing.assert_allclose( + api_hidden_out.numpy(), np_hidden_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + api_cell_out.numpy(), np_cell_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + named_api_hidden_out.numpy(), np_hidden_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + named_api_cell_out.numpy(), np_cell_out, rtol=1e-05, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cumprod_op.py b/python/paddle/fluid/tests/unittests/test_cumprod_op.py index c4c6724468f..a6aea5e33fd 100644 --- a/python/paddle/fluid/tests/unittests/test_cumprod_op.py +++ b/python/paddle/fluid/tests/unittests/test_cumprod_op.py @@ -47,8 +47,9 @@ def cumprod_grad(x, y, dy, dx, shape, dim): else: elem = dy[pos] * y[index - inner_dim] if pos > index: - for m in range(index + inner_dim, pos + inner_dim, - inner_dim): + for m in range( + index + inner_dim, pos + inner_dim, inner_dim + ): elem *= x[m] elif pos < index: elem = 0 @@ -57,7 +58,6 @@ def cumprod_grad(x, y, dy, dx, shape, dim): # test function. class TestCumprod(OpTest): - def init_params(self): self.shape = (2, 3, 4, 5) self.zero_nums = [0, 10, 20, 30, int(np.prod(self.shape))] @@ -98,8 +98,9 @@ class TestCumprod(OpTest): if self.dtype == np.complex128 or self.dtype == np.complex64: reshape_x = np.conj(reshape_x) out_data = np.conj(out_data) - cumprod_grad(reshape_x, out_data, self.grad_out, self.grad_x, - self.shape, dim) + cumprod_grad( + reshape_x, out_data, self.grad_out, self.grad_x, self.shape, dim + ) self.grad_x = self.grad_x.reshape(self.shape) self.grad_out = self.grad_out.reshape(self.shape) @@ -119,37 +120,35 @@ class TestCumprod(OpTest): if self.dtype == np.float64: self.check_grad(['X'], 'Out', check_eager=True) else: - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) # test float32 case. class TestCumprod_float32(TestCumprod): - def init_dtype(self): self.dtype = np.float32 # test complex64 case. class TestCumprod_complex64(TestCumprod): - def init_dtype(self): self.dtype = np.complex64 # test complex128 case. class TestCumprod_complex128(TestCumprod): - def init_dtype(self): self.dtype = np.complex128 # test api. class TestCumprodAPI(unittest.TestCase): - def init_dtype(self): self.dtype = 'float64' self.shape = [2, 3, 10, 10] @@ -182,7 +181,6 @@ class TestCumprodAPI(unittest.TestCase): # test dynamic graph api. def test_dygraph_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_cumsum_op.py b/python/paddle/fluid/tests/unittests/test_cumsum_op.py index e63252c2c08..b28c2863d18 100644 --- a/python/paddle/fluid/tests/unittests/test_cumsum_op.py +++ b/python/paddle/fluid/tests/unittests/test_cumsum_op.py @@ -27,7 +27,6 @@ import paddle.fluid.layers as layers class TestCumsumOp(unittest.TestCase): - def run_cases(self): data_np = np.arange(12).reshape(3, 4) data = paddle.to_tensor(data_np) @@ -68,11 +67,17 @@ class TestCumsumOp(unittest.TestCase): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - out = exe.run(feed={'X': data_np}, - fetch_list=[ - y.name, y2.name, y3.name, y4.name, y5.name, - y6.name - ]) + out = exe.run( + feed={'X': data_np}, + fetch_list=[ + y.name, + y2.name, + y3.name, + y4.name, + y5.name, + y6.name, + ], + ) z = np.cumsum(data_np) np.testing.assert_allclose(z, out[0], rtol=1e-05) @@ -109,7 +114,6 @@ class TestCumsumOp(unittest.TestCase): class TestSumOp1(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2} @@ -124,14 +128,14 @@ class TestSumOp1(OpTest): class TestSumOp2(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': -1, 'reverse': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype("float64")} self.outputs = { - 'Out': np.flip(np.flip(self.inputs['X'], axis=2).cumsum(axis=2), - axis=2) + 'Out': np.flip( + np.flip(self.inputs['X'], axis=2).cumsum(axis=2), axis=2 + ) } def test_check_output(self): @@ -142,7 +146,6 @@ class TestSumOp2(OpTest): class TestSumOp3(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 1} @@ -157,7 +160,6 @@ class TestSumOp3(OpTest): class TestSumOp4(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 0} @@ -172,7 +174,6 @@ class TestSumOp4(OpTest): class TestSumOp5(OpTest): - def setUp(self): self.op_type = "cumsum" self.inputs = {'X': np.random.random((5, 20)).astype("float64")} @@ -186,7 +187,6 @@ class TestSumOp5(OpTest): class TestSumOp7(OpTest): - def setUp(self): self.op_type = "cumsum" self.inputs = {'X': np.random.random((100)).astype("float64")} @@ -200,7 +200,6 @@ class TestSumOp7(OpTest): class TestCumsumFP16(unittest.TestCase): - def check_main(self, x_np, dtype): paddle.disable_static() x = paddle.to_tensor(x_np.astype(dtype)) @@ -226,17 +225,19 @@ class TestCumsumFP16(unittest.TestCase): class TestSumOpExclusive1(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 65)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -244,17 +245,19 @@ class TestSumOpExclusive1(OpTest): class TestSumOpExclusive2(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 888)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -262,17 +265,19 @@ class TestSumOpExclusive2(OpTest): class TestSumOpExclusive3(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 888)).astype("float32") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -280,17 +285,19 @@ class TestSumOpExclusive3(OpTest): class TestSumOpExclusive4(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 3049)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -298,17 +305,19 @@ class TestSumOpExclusive4(OpTest): class TestSumOpExclusive5(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 3096)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -316,17 +325,19 @@ class TestSumOpExclusive5(OpTest): class TestSumOpExclusiveFP16(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True, "dtype": "float16"} a = np.random.random((4, 5, 3096)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -334,7 +345,6 @@ class TestSumOpExclusiveFP16(OpTest): class TestSumOpReverseExclusive(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, 'reverse': True, "exclusive": True} @@ -342,11 +352,13 @@ class TestSumOpReverseExclusive(OpTest): self.inputs = {'X': a} a = np.flip(a, axis=2) self.outputs = { - 'Out': - np.concatenate( - (np.flip(a[:, :, :-1].cumsum(axis=2), - axis=2), np.zeros((4, 5, 1), dtype=np.float64)), - axis=2) + 'Out': np.concatenate( + ( + np.flip(a[:, :, :-1].cumsum(axis=2), axis=2), + np.zeros((4, 5, 1), dtype=np.float64), + ), + axis=2, + ) } def test_check_output(self): @@ -354,7 +366,6 @@ class TestSumOpReverseExclusive(OpTest): class BadInputTest(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): @@ -366,21 +377,24 @@ class BadInputTest(unittest.TestCase): class TestTensorAxis(unittest.TestCase): - def setUp(self): paddle.seed(2022) self.temp_dir = tempfile.TemporaryDirectory() self.save_path = os.path.join(self.temp_dir.name, 'tensor_axis_cumsum') - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def test_dygraph(self): paddle.disable_static() x = np.random.randn(5, 6) axis = 1 np_out = np.cumsum(x, axis) - pd_out = paddle.cumsum(paddle.to_tensor(x), - axis=paddle.to_tensor([axis], dtype='int32')) + pd_out = paddle.cumsum( + paddle.to_tensor(x), axis=paddle.to_tensor([axis], dtype='int32') + ) np.testing.assert_allclose(np_out, pd_out.numpy()) def test_static_and_infer(self): @@ -398,7 +412,7 @@ class TestTensorAxis(unittest.TestCase): axis = paddle.full([1], 2, dtype='int64') out = paddle.cumsum(relu_out, axis=axis) loss = paddle.mean(out) - sgd = paddle.optimizer.SGD(learning_rate=0.) + sgd = paddle.optimizer.SGD(learning_rate=0.0) sgd.minimize(paddle.mean(out)) exe = paddle.static.Executor(self.place) @@ -407,8 +421,9 @@ class TestTensorAxis(unittest.TestCase): # run infer paddle.static.save_inference_model(self.save_path, [x], [out], exe) - config = paddle_infer.Config(self.save_path + '.pdmodel', - self.save_path + '.pdiparams') + config = paddle_infer.Config( + self.save_path + '.pdmodel', self.save_path + '.pdiparams' + ) if paddle.is_compiled_with_cuda(): config.enable_use_gpu(100, 0) else: @@ -428,7 +443,6 @@ class TestTensorAxis(unittest.TestCase): class TestCumsumDoubleGradCheck(unittest.TestCase): - def cumsum_wrapper(self, x): return paddle.cumsum(x[0], 0) @@ -443,17 +457,13 @@ class TestCumsumDoubleGradCheck(unittest.TestCase): out = paddle.cumsum(data, 0) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.cumsum_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.cumsum_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -465,7 +475,6 @@ class TestCumsumDoubleGradCheck(unittest.TestCase): class TestCumsumTripleGradCheck(unittest.TestCase): - def cumsum_wrapper(self, x): return paddle.cumsum(x[0], 0) @@ -480,17 +489,13 @@ class TestCumsumTripleGradCheck(unittest.TestCase): out = paddle.cumsum(data, 0) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.cumsum_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.cumsum_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_custom_grad_input.py b/python/paddle/fluid/tests/unittests/test_custom_grad_input.py index 562290c41cf..74f63f03f45 100644 --- a/python/paddle/fluid/tests/unittests/test_custom_grad_input.py +++ b/python/paddle/fluid/tests/unittests/test_custom_grad_input.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestTensorBackward(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -45,9 +44,9 @@ class TestTensorBackward(unittest.TestCase): x_grad = np.matmul(grad, y.T) - np.testing.assert_allclose(x_grad, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_tensor_backward(self): with _test_eager_guard(): @@ -56,7 +55,6 @@ class TestTensorBackward(unittest.TestCase): class TestBackwardAPI(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -77,14 +75,15 @@ class TestBackwardAPI(unittest.TestCase): z_tensor2 = paddle.matmul(x_tensor, y_tensor) grad_tensor = paddle.to_tensor(grad) - paddle.autograd.backward([z_tensor1, z_tensor2], - [grad_tensor, grad_tensor], True) + paddle.autograd.backward( + [z_tensor1, z_tensor2], [grad_tensor, grad_tensor], True + ) x_grad = np.matmul(grad, y.T) - np.testing.assert_allclose(x_grad * 2, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad * 2, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_backward_api(self): with _test_eager_guard(): @@ -108,9 +107,9 @@ class TestBackwardAPI(unittest.TestCase): x_grad = np.matmul(grad, y.T) - np.testing.assert_allclose(x_grad, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_backward_single_tensor(self): with _test_eager_guard(): @@ -133,9 +132,9 @@ class TestBackwardAPI(unittest.TestCase): x_grad = np.matmul(grad, y.T) - np.testing.assert_allclose(x_grad, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_backward_none_grad_tensor(self): with _test_eager_guard(): @@ -144,15 +143,21 @@ class TestBackwardAPI(unittest.TestCase): def func_backward_accumulator_with_init_grad(self): for dtype in self._dtypes: - x = np.random.random([ - 10, - ]).astype(dtype) - y_grad = np.random.random([ - 10, - ]).astype(dtype) - z_grad = np.random.random([ - 10, - ]).astype(dtype) + x = np.random.random( + [ + 10, + ] + ).astype(dtype) + y_grad = np.random.random( + [ + 10, + ] + ).astype(dtype) + z_grad = np.random.random( + [ + 10, + ] + ).astype(dtype) self._places = [paddle.CPUPlace()] for place in self._places: with dg.guard(place): @@ -162,16 +167,17 @@ class TestBackwardAPI(unittest.TestCase): y_grad_tensor = paddle.to_tensor(y_grad) z_grad_tensor = paddle.to_tensor(z_grad) - paddle.autograd.backward([y_tensor, z_tensor], - [y_grad_tensor, z_grad_tensor]) + paddle.autograd.backward( + [y_tensor, z_tensor], [y_grad_tensor, z_grad_tensor] + ) y = x**2 z = x**3 x_grad = 2 * x * (y_grad + 3 * y * y * z_grad) - np.testing.assert_allclose(x_grad, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_backward_accumulator_with_init_grad(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_cvm_op.py b/python/paddle/fluid/tests/unittests/test_cvm_op.py index b4f8af82b3c..a6197e7d726 100644 --- a/python/paddle/fluid/tests/unittests/test_cvm_op.py +++ b/python/paddle/fluid/tests/unittests/test_cvm_op.py @@ -52,7 +52,7 @@ def cvm_grad_compute(DY, CVM, item_width, use_cvm): class TestCVMOpWithLodTensor(OpTest): """ - Test cvm op with discrete one-hot labels. + Test cvm op with discrete one-hot labels. """ def setUp(self): @@ -64,11 +64,13 @@ class TestCVMOpWithLodTensor(OpTest): lod = [[1]] self.inputs = { - 'X': (np.random.uniform( - 0, 1, - [self.batch_size, self.item_width]).astype("float32"), lod), - 'CVM': - np.array([[0.6, 0.4]]).astype("float32"), + 'X': ( + np.random.uniform( + 0, 1, [self.batch_size, self.item_width] + ).astype("float32"), + lod, + ), + 'CVM': np.array([[0.6, 0.4]]).astype("float32"), } self.attrs = {'use_cvm': False} out = [] @@ -80,15 +82,16 @@ class TestCVMOpWithLodTensor(OpTest): self.check_output(check_dygraph=False) def test_check_grad(self): - user_grads = np.array( - [1.0 / (self.item_width - 2)] * self.item_width).reshape( - (self.batch_size, self.item_width)).astype("float32") + user_grads = ( + np.array([1.0 / (self.item_width - 2)] * self.item_width) + .reshape((self.batch_size, self.item_width)) + .astype("float32") + ) user_grads[:, :2] = self.inputs['CVM'].reshape(self.batch_size, 2) user_grads = [user_grads] - self.check_grad(['X'], - 'Y', - user_defined_grads=user_grads, - check_dygraph=False) + self.check_grad( + ['X'], 'Y', user_defined_grads=user_grads, check_dygraph=False + ) class TestCVMOpWithOutLodTensor1(OpTest): @@ -104,10 +107,14 @@ class TestCVMOpWithOutLodTensor1(OpTest): self.item_width = 11 input = np.random.uniform( - 0, 1, (self.batch_size, self.item_width)).astype('float32') + 0, 1, (self.batch_size, self.item_width) + ).astype('float32') output = cvm_compute(input, self.item_width, self.use_cvm) - cvm = np.array([[0.6, 0.4] * self.batch_size]).reshape( - (self.batch_size, 2)).astype("float32") + cvm = ( + np.array([[0.6, 0.4] * self.batch_size]) + .reshape((self.batch_size, 2)) + .astype("float32") + ) self.inputs = {'X': input, 'CVM': cvm} self.attrs = {'use_cvm': self.use_cvm} @@ -118,14 +125,16 @@ class TestCVMOpWithOutLodTensor1(OpTest): def test_check_grad(self): numel = self.batch_size * self.item_width - user_grads = np.array([1.0 / numel] * numel).reshape( - (self.batch_size, self.item_width)).astype("float32") + user_grads = ( + np.array([1.0 / numel] * numel) + .reshape((self.batch_size, self.item_width)) + .astype("float32") + ) user_grads[:, :2] = self.inputs['CVM'].reshape(self.batch_size, 2) user_grads = [user_grads] - self.check_grad(['X'], - 'Y', - user_defined_grads=user_grads, - check_dygraph=False) + self.check_grad( + ['X'], 'Y', user_defined_grads=user_grads, check_dygraph=False + ) class TestCVMOpWithOutLodTensor2(OpTest): @@ -141,10 +150,14 @@ class TestCVMOpWithOutLodTensor2(OpTest): self.item_width = 11 input = np.random.uniform( - 0, 1, (self.batch_size, self.item_width)).astype('float32') + 0, 1, (self.batch_size, self.item_width) + ).astype('float32') output = cvm_compute(input, self.item_width, self.use_cvm) - cvm = np.array([[0.6, 0.4] * self.batch_size]).reshape( - (self.batch_size, 2)).astype("float32") + cvm = ( + np.array([[0.6, 0.4] * self.batch_size]) + .reshape((self.batch_size, 2)) + .astype("float32") + ) self.inputs = {'X': input, 'CVM': cvm} self.attrs = {'use_cvm': self.use_cvm} @@ -155,15 +168,16 @@ class TestCVMOpWithOutLodTensor2(OpTest): def test_check_grad(self): numel = self.batch_size * self.item_width - user_grads = np.array( - [1.0 / (self.batch_size * (self.item_width - 2))] * numel).reshape( - (self.batch_size, self.item_width)).astype("float32") + user_grads = ( + np.array([1.0 / (self.batch_size * (self.item_width - 2))] * numel) + .reshape((self.batch_size, self.item_width)) + .astype("float32") + ) user_grads[:, :2] = self.inputs['CVM'].reshape(self.batch_size, 2) user_grads = [user_grads] - self.check_grad(['X'], - 'Y', - user_defined_grads=user_grads, - check_dygraph=False) + self.check_grad( + ['X'], 'Y', user_defined_grads=user_grads, check_dygraph=False + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cyclic_cifar_dataset.py b/python/paddle/fluid/tests/unittests/test_cyclic_cifar_dataset.py index e014a25ab79..01a588c4058 100644 --- a/python/paddle/fluid/tests/unittests/test_cyclic_cifar_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_cyclic_cifar_dataset.py @@ -17,7 +17,6 @@ import unittest class TestCifar10(unittest.TestCase): - def test_main(self): reader = paddle.dataset.cifar.train10(cycle=False) sample_num = 0 diff --git a/python/paddle/fluid/tests/unittests/test_data.py b/python/paddle/fluid/tests/unittests/test_data.py index 9beb89205f5..bcb956e25c3 100644 --- a/python/paddle/fluid/tests/unittests/test_data.py +++ b/python/paddle/fluid/tests/unittests/test_data.py @@ -22,7 +22,6 @@ import paddle.fluid.core as core class TestApiDataError(unittest.TestCase): - def test_fluid_data(self): with program_guard(Program(), Program()): @@ -55,7 +54,6 @@ class TestApiDataError(unittest.TestCase): class TestApiStaticDataError(unittest.TestCase): - def test_fluid_dtype(self): with program_guard(Program(), Program()): x1 = paddle.static.data(name="x1", shape=[2, 25]) @@ -100,19 +98,16 @@ class TestApiStaticDataError(unittest.TestCase): class TestApiErrorWithDynamicMode(unittest.TestCase): - def test_error(self): with program_guard(Program(), Program()): paddle.disable_static() self.assertRaises(AssertionError, fluid.data, 'a', [2, 25]) - self.assertRaises(AssertionError, - fluid.layers.data, - 'b', - shape=[2, 25]) - self.assertRaises(AssertionError, - paddle.static.data, - 'c', - shape=[2, 25]) + self.assertRaises( + AssertionError, fluid.layers.data, 'b', shape=[2, 25] + ) + self.assertRaises( + AssertionError, paddle.static.data, 'c', shape=[2, 25] + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_data_generator.py b/python/paddle/fluid/tests/unittests/test_data_generator.py index 6134a1dd86c..b804e6a9602 100644 --- a/python/paddle/fluid/tests/unittests/test_data_generator.py +++ b/python/paddle/fluid/tests/unittests/test_data_generator.py @@ -15,9 +15,7 @@ import paddle.distributed.fleet as fleet class MyMultiSlotDataGenerator(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -28,9 +26,7 @@ class MyMultiSlotDataGenerator(fleet.MultiSlotDataGenerator): class MyMultiSlotStringDataGenerator(fleet.MultiSlotStringDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -41,9 +37,7 @@ class MyMultiSlotStringDataGenerator(fleet.MultiSlotStringDataGenerator): class MyMultiSlotDataGenerator_error(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -54,9 +48,7 @@ class MyMultiSlotDataGenerator_error(fleet.MultiSlotDataGenerator): class MyMultiSlotDataGenerator_error_2(fleet.MultiSlotStringDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -67,9 +59,7 @@ class MyMultiSlotDataGenerator_error_2(fleet.MultiSlotStringDataGenerator): class MyMultiSlotDataGenerator_error_3(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -80,9 +70,7 @@ class MyMultiSlotDataGenerator_error_3(fleet.MultiSlotDataGenerator): class MyMultiSlotDataGenerator_error_4(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -93,9 +81,7 @@ class MyMultiSlotDataGenerator_error_4(fleet.MultiSlotDataGenerator): class MyMultiSlotDataGenerator_error_5(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -106,9 +92,7 @@ class MyMultiSlotDataGenerator_error_5(fleet.MultiSlotDataGenerator): class MyMultiSlotStringDataGenerator_zip(fleet.MultiSlotStringDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -121,9 +105,7 @@ class MyMultiSlotStringDataGenerator_zip(fleet.MultiSlotStringDataGenerator): class MyMultiSlotDataGenerator_zip(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -136,7 +118,6 @@ class MyMultiSlotDataGenerator_zip(fleet.MultiSlotDataGenerator): class TestMultiSlotDataGenerator(unittest.TestCase): - def test_MultiSlotDataGenerator_basic(self): my_ms_dg = MyMultiSlotDataGenerator() my_ms_dg.set_batch(1) @@ -144,7 +125,6 @@ class TestMultiSlotDataGenerator(unittest.TestCase): class TestMultiSlotStringDataGenerator(unittest.TestCase): - def test_MyMultiSlotStringDataGenerator_basic(self): my_ms_dg = MyMultiSlotStringDataGenerator() my_ms_dg.set_batch(1) @@ -152,7 +132,6 @@ class TestMultiSlotStringDataGenerator(unittest.TestCase): class TestMultiSlotDataGenerator_error(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error() @@ -161,7 +140,6 @@ class TestMultiSlotDataGenerator_error(unittest.TestCase): class TestMultiSlotDataGenerator_error_2(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error_2() @@ -170,7 +148,6 @@ class TestMultiSlotDataGenerator_error_2(unittest.TestCase): class TestMultiSlotDataGenerator_error_3(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error_3() @@ -179,7 +156,6 @@ class TestMultiSlotDataGenerator_error_3(unittest.TestCase): class TestMultiSlotDataGenerator_error_4(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error_4() @@ -188,7 +164,6 @@ class TestMultiSlotDataGenerator_error_4(unittest.TestCase): class TestMultiSlotDataGenerator_error_5(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error_5() @@ -197,7 +172,6 @@ class TestMultiSlotDataGenerator_error_5(unittest.TestCase): class TestMultiSlotStringDataGeneratorZip(unittest.TestCase): - def test_MultiSlotStringDataGenerator_zip(self): my_ms_dg = MyMultiSlotStringDataGenerator_zip() my_ms_dg.set_batch(1) @@ -205,7 +179,6 @@ class TestMultiSlotStringDataGeneratorZip(unittest.TestCase): class TestMultiSlotDataGeneratorZip(unittest.TestCase): - def test_MultiSlotDataGenerator_zip(self): my_ms_dg = MyMultiSlotDataGenerator_zip() my_ms_dg.set_batch(1) diff --git a/python/paddle/fluid/tests/unittests/test_data_norm_op.py b/python/paddle/fluid/tests/unittests/test_data_norm_op.py index cf56d571734..1f32feb3527 100644 --- a/python/paddle/fluid/tests/unittests/test_data_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_data_norm_op.py @@ -38,8 +38,9 @@ def _reference_testing(x, batch_size, batch_sum, batch_square_sum, slot_dim=-1): for j in range(0, x_shape[1], slot_dim): if x[i][j] <= -min_precision or x[i][j] >= min_precision: for k in range(0, slot_dim): - y[i][j + k] = (x[i][j + k] - - means_arr[j + k]) * scales_arr[j + k] + y[i][j + k] = ( + x[i][j + k] - means_arr[j + k] + ) * scales_arr[j + k] return y @@ -66,19 +67,19 @@ class TestDataNormOpInference(unittest.TestCase): self.use_mkldnn = False def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) - - def check_with_place(self, - place, - data_layout, - dtype, - shape, - slot_dim=-1, - enable_scale_and_shift=False): + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) + + def check_with_place( + self, + place, + data_layout, + dtype, + shape, + slot_dim=-1, + enable_scale_and_shift=False, + ): """ do forward and check @@ -109,24 +110,31 @@ class TestDataNormOpInference(unittest.TestCase): batch_square_sum = np.ones(scale_shape).astype(np.float32) batch_square_sum *= 1e4 - y_out = _reference_testing(x_val, batch_size, batch_sum, - batch_square_sum, slot_dim).astype(dtype) + y_out = _reference_testing( + x_val, batch_size, batch_sum, batch_square_sum, slot_dim + ).astype(dtype) scope = core.Scope() # create input - x_tensor = create_or_get_tensor(scope, "x_val", - OpTest.np_dtype_to_fluid_dtype(x_val), - place) + x_tensor = create_or_get_tensor( + scope, "x_val", OpTest.np_dtype_to_fluid_dtype(x_val), place + ) batch_size_tensor = create_or_get_tensor( - scope, "batch_size", OpTest.np_dtype_to_fluid_dtype(batch_size), - place) + scope, + "batch_size", + OpTest.np_dtype_to_fluid_dtype(batch_size), + place, + ) batch_sum_tensor = create_or_get_tensor( - scope, "batch_sum", OpTest.np_dtype_to_fluid_dtype(batch_sum), - place) + scope, "batch_sum", OpTest.np_dtype_to_fluid_dtype(batch_sum), place + ) batch_square_sum_tensor = create_or_get_tensor( - scope, "batch_square_sum", - OpTest.np_dtype_to_fluid_dtype(batch_square_sum), place) + scope, + "batch_square_sum", + OpTest.np_dtype_to_fluid_dtype(batch_square_sum), + place, + ) # create output y_tensor = create_or_get_tensor(scope, "y_out", None, place) @@ -149,15 +157,17 @@ class TestDataNormOpInference(unittest.TestCase): epsilon=epsilon, use_mkldnn=self.use_mkldnn, slot_dim=slot_dim, - enable_scale_and_shift=False) + enable_scale_and_shift=False, + ) else: scale_w = np.ones(scale_shape).astype(np.float32) bias = np.zeros(scale_shape).astype(np.float32) scale_w_tensor = create_or_get_tensor( - scope, "scale_w", OpTest.np_dtype_to_fluid_dtype(scale_w), - place) + scope, "scale_w", OpTest.np_dtype_to_fluid_dtype(scale_w), place + ) bias_tensor = create_or_get_tensor( - scope, "bias", OpTest.np_dtype_to_fluid_dtype(bias), place) + scope, "bias", OpTest.np_dtype_to_fluid_dtype(bias), place + ) data_norm_op = Operator( "data_norm", # inputs @@ -175,17 +185,25 @@ class TestDataNormOpInference(unittest.TestCase): epsilon=epsilon, use_mkldnn=self.use_mkldnn, slot_dim=slot_dim, - enable_scale_and_shift=True) + enable_scale_and_shift=True, + ) data_norm_op.run(scope, place) # check inference result - self.__assert_close(y_tensor, - y_out, - "inference output are different at " + str(place) + - ", " + data_layout + ", " + str(np.dtype(dtype)) + - str(np.array(y_tensor)) + str(y_out), - atol=1e-3) + self.__assert_close( + y_tensor, + y_out, + "inference output are different at " + + str(place) + + ", " + + data_layout + + ", " + + str(np.dtype(dtype)) + + str(np.array(y_tensor)) + + str(y_out), + atol=1e-3, + ) def test_check_output(self): """ @@ -199,9 +217,11 @@ class TestDataNormOpInference(unittest.TestCase): self.check_with_place( place, data_format, - self.dtype, [2, 3], + self.dtype, + [2, 3], slot_dim=slot_dim, - enable_scale_and_shift=enable_scale_and_shift) + enable_scale_and_shift=enable_scale_and_shift, + ) class TestDataNormOp(OpTest): @@ -237,7 +257,7 @@ class TestDataNormOp(OpTest): "X": x_val, "BatchSize": batch_size, "BatchSum": batch_sum, - "BatchSquareSum": batch_square_sum + "BatchSquareSum": batch_square_sum, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = {"epsilon": epsilon, "use_mkldnn": self.use_mkldnn} @@ -294,14 +314,14 @@ class TestDataNormOpWithEnableScaleAndShift(OpTest): "BatchSum": batch_sum, "BatchSquareSum": batch_square_sum, "scale_w": scale_w, - "bias": bias + "bias": bias, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = { "epsilon": epsilon, "use_mkldnn": self.use_mkldnn, "slot_dim": slot_dim, - "enable_scale_and_shift": True + "enable_scale_and_shift": True, } def test_check_output(self): @@ -356,7 +376,7 @@ class TestDataNormOpWithoutEnableScaleAndShift(OpTest): "BatchSum": batch_sum, "BatchSquareSum": batch_square_sum, "scale_w": scale_w, - "bias": bias + "bias": bias, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = {"epsilon": epsilon, "use_mkldnn": self.use_mkldnn} @@ -413,14 +433,14 @@ class TestDataNormOpWithEnableScaleAndShift_1(OpTest): "BatchSum": batch_sum, "BatchSquareSum": batch_square_sum, "scale_w": scale_w, - "bias": bias + "bias": bias, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = { "epsilon": epsilon, "use_mkldnn": self.use_mkldnn, "slot_dim": slot_dim, - "enable_scale_and_shift": True + "enable_scale_and_shift": True, } def test_check_output(self): @@ -470,13 +490,13 @@ class TestDataNormOpWithSlotDim(OpTest): "X": x_val, "BatchSize": batch_size, "BatchSum": batch_sum, - "BatchSquareSum": batch_square_sum + "BatchSquareSum": batch_square_sum, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = { "epsilon": epsilon, "use_mkldnn": self.use_mkldnn, - "slot_dim": slot_dim + "slot_dim": slot_dim, } def test_check_output(self): @@ -493,14 +513,13 @@ class TestDataNormOpWithSlotDim(OpTest): class TestDataNormOpErrorr(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x2 = fluid.layers.data(name='x2', shape=[3, 4], dtype="int32") - #self.assertRaises(TypeError, fluid.data_norm, x2) - fluid.layers.data_norm(input=x2, - param_attr={}, - enable_scale_and_shift=True) + # self.assertRaises(TypeError, fluid.data_norm, x2) + fluid.layers.data_norm( + input=x2, param_attr={}, enable_scale_and_shift=True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py b/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py index 29169ca6d76..00630a39822 100755 --- a/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py @@ -25,13 +25,12 @@ import os class RandomDataset(Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([10]).astype('float32') - label = np.random.randint(0, 10 - 1, (1, )).astype('int64') + label = np.random.randint(0, 10 - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -39,7 +38,6 @@ class RandomDataset(Dataset): class SimpleNet(nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.fc = nn.Linear(10, 10) @@ -49,20 +47,22 @@ class SimpleNet(nn.Layer): class TestAutoTune(unittest.TestCase): - def setUp(self): self.batch_size = 1 self.dataset = RandomDataset(10) def test_dataloader_use_autotune(self): paddle.incubate.autotune.set_config( - config={"dataloader": { - "enable": True, - "tuning_steps": 1, - }}) - loader = DataLoader(self.dataset, - batch_size=self.batch_size, - num_workers=0) + config={ + "dataloader": { + "enable": True, + "tuning_steps": 1, + } + } + ) + loader = DataLoader( + self.dataset, batch_size=self.batch_size, num_workers=0 + ) def test_dataloader_disable_autotune(self): config = {"dataloader": {"enable": False, "tuning_steps": 1}} @@ -71,29 +71,32 @@ class TestAutoTune(unittest.TestCase): tfile.close() paddle.incubate.autotune.set_config(tfile.name) os.remove(tfile.name) - loader = DataLoader(self.dataset, - batch_size=self.batch_size, - num_workers=2) - if (sys.platform == 'darwin' or sys.platform == 'win32'): + loader = DataLoader( + self.dataset, batch_size=self.batch_size, num_workers=2 + ) + if sys.platform == 'darwin' or sys.platform == 'win32': self.assertEqual(loader.num_workers, 0) else: self.assertEqual(loader.num_workers, 2) def test_distributer_batch_sampler_autotune(self): paddle.incubate.autotune.set_config( - config={"dataloader": { - "enable": True, - "tuning_steps": 1, - }}) + config={ + "dataloader": { + "enable": True, + "tuning_steps": 1, + } + } + ) batch_sampler = paddle.io.DistributedBatchSampler( - self.dataset, batch_size=self.batch_size) - loader = DataLoader(self.dataset, - batch_sampler=batch_sampler, - num_workers=2) + self.dataset, batch_size=self.batch_size + ) + loader = DataLoader( + self.dataset, batch_sampler=batch_sampler, num_workers=2 + ) class TestAutoTuneAPI(unittest.TestCase): - def test_set_config_warnings(self): with warnings.catch_warnings(record=True) as w: config = {"kernel": {"enable": 1, "tuning_range": True}} diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py b/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py index 6b3ab060c3d..2c4bcfa444b 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class TestDatasetAbstract(unittest.TestCase): - def func_test_main(self): dataset = Dataset() try: @@ -44,20 +43,21 @@ class TestDatasetAbstract(unittest.TestCase): class TestDatasetWithDiffOutputPlace(unittest.TestCase): - def get_dataloader(self, num_workers): dataset = paddle.vision.datasets.MNIST( mode='test', - transform=transforms.Compose([ - transforms.CenterCrop(20), - transforms.RandomResizedCrop(14), - transforms.Normalize(), - transforms.ToTensor() - ])) - loader = paddle.io.DataLoader(dataset, - batch_size=32, - num_workers=num_workers, - shuffle=True) + transform=transforms.Compose( + [ + transforms.CenterCrop(20), + transforms.RandomResizedCrop(14), + transforms.Normalize(), + transforms.ToTensor(), + ] + ), + ) + loader = paddle.io.DataLoader( + dataset, batch_size=32, num_workers=num_workers, shuffle=True + ) return loader def run_check_on_cpu(self): diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_early_reset.py b/python/paddle/fluid/tests/unittests/test_dataloader_early_reset.py index 3c03a289728..fa0bac6c5bd 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_early_reset.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_early_reset.py @@ -25,7 +25,6 @@ def infinite_reader(): class TestDataLoaderEarlyReset(unittest.TestCase): - def setUp(self): self.stop_batch = 10 self.iterable = True @@ -45,9 +44,9 @@ class TestDataLoaderEarlyReset(unittest.TestCase): def create_data_loader(self): self.x = fluid.data(name='x', shape=[None, 32], dtype='float32') - return fluid.io.DataLoader.from_generator(feed_list=[self.x], - capacity=10, - iterable=self.iterable) + return fluid.io.DataLoader.from_generator( + feed_list=[self.x], capacity=10, iterable=self.iterable + ) def test_main(self): with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -68,7 +67,7 @@ class TestDataLoaderEarlyReset(unittest.TestCase): batch_id = 0 if loader.iterable: for data in loader(): - x_val, = exe.run(prog, feed=data, fetch_list=[self.x]) + (x_val,) = exe.run(prog, feed=data, fetch_list=[self.x]) self.assertTrue(np.all(x_val == batch_id)) batch_id += 1 if batch_id >= self.stop_batch: @@ -89,7 +88,6 @@ class TestDataLoaderEarlyReset(unittest.TestCase): class TestDataLoaderEarlyReset2(TestDataLoaderEarlyReset): - def setUp(self): self.stop_batch = 20 self.iterable = False diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_keep_order.py b/python/paddle/fluid/tests/unittests/test_dataloader_keep_order.py index 5d07586c7e6..f2dd18c0080 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_keep_order.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_keep_order.py @@ -19,7 +19,6 @@ import os def create_reader(shape, batch_number): - def __impl__(): idx = 0 for _ in range(batch_number): @@ -30,7 +29,6 @@ def create_reader(shape, batch_number): class DataLoaderKeepOrderTestBase(unittest.TestCase): - def initParameters(self): self.iterable = False self.break_num = 100 @@ -43,15 +41,17 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): def build_network(self, places): input_data = fluid.data(shape=self.shape, dtype='float32', name="input") - loader = fluid.io.DataLoader.from_generator(capacity=16, - feed_list=[input_data], - iterable=self.iterable) + loader = fluid.io.DataLoader.from_generator( + capacity=16, feed_list=[input_data], iterable=self.iterable + ) fc = fluid.layers.fc(input_data, size=10) loss = fluid.layers.reduce_mean(fc) - loader.set_batch_generator(create_reader(self.shape, self.batch_num), - places=places if loader.iterable else None) + loader.set_batch_generator( + create_reader(self.shape, self.batch_num), + places=places if loader.iterable else None, + ) return input_data, loss, loader @@ -65,12 +65,15 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): self.assertTrue((input_tensor == start_val).all()) start_val += 1 else: - self.assertEqual(list(input_data.shape), - [self.shape[0] * dev_cnt] + self.shape[1:]) + self.assertEqual( + list(input_data.shape), + [self.shape[0] * dev_cnt] + self.shape[1:], + ) start_val = dev_cnt * batch_id for idx in range(dev_cnt): - data_part = input_data[idx * self.shape[0]:(idx + 1) * - self.shape[0], :] + data_part = input_data[ + idx * self.shape[0] : (idx + 1) * self.shape[0], : + ] self.assertTrue((data_part == start_val).all()) start_val += 1 @@ -81,8 +84,8 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): place_list.extend([fluid.cuda_places(0)]) else: place_list.extend( - [fluid.cuda_places(0), - fluid.cuda_places([0, 1])]) + [fluid.cuda_places(0), fluid.cuda_places([0, 1])] + ) return place_list def test_main(self): @@ -107,11 +110,12 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): main_program = fluid.default_main_program() if use_compiled_program: main_program = fluid.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name, - places=places) + main_program + ).with_data_parallel(loss_name=loss.name, places=places) - max_batch_num = min(self.break_num, - int(self.batch_num / dev_cnt)) + max_batch_num = min( + self.break_num, int(self.batch_num / dev_cnt) + ) if loader.iterable: early_break = False @@ -123,9 +127,11 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): early_break = True break self.assertInputData(batch_id, data, dev_cnt) - fetch_val, = exe.run(program=main_program, - feed=data, - fetch_list=fetch_list) + (fetch_val,) = exe.run( + program=main_program, + feed=data, + fetch_list=fetch_list, + ) self.assertInputData(batch_id, fetch_val, dev_cnt) batch_id += 1 @@ -142,10 +148,12 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): if batch_id >= self.break_num: loader.reset() break - fetch_val, = exe.run(program=main_program, - fetch_list=fetch_list) - self.assertInputData(batch_id, fetch_val, - dev_cnt) + (fetch_val,) = exe.run( + program=main_program, fetch_list=fetch_list + ) + self.assertInputData( + batch_id, fetch_val, dev_cnt + ) batch_id += 1 except fluid.core.EOFException: loader.reset() @@ -154,35 +162,30 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): class IterableDataLoaderKeepOrderTest2(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 100 class IterableDataLoaderKeepOrderTest3(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = False self.break_num = 2 class IterableDataLoaderKeepOrderTest4(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 2 class IterableDataLoaderKeepOrderTest5(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = False self.break_num = 0 class IterableDataLoaderKeepOrderTest6(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 0 diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_unkeep_order.py b/python/paddle/fluid/tests/unittests/test_dataloader_unkeep_order.py index 236ffe19b6e..8e121ba401e 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_unkeep_order.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_unkeep_order.py @@ -22,7 +22,6 @@ keep_data_loader_order(False) def create_reader(shape, batch_number): - def __impl__(): idx = 0 for _ in range(batch_number): @@ -33,7 +32,6 @@ def create_reader(shape, batch_number): class DataLoaderKeepOrderTestBase(unittest.TestCase): - def initParameters(self): self.iterable = False self.break_num = 10000 @@ -49,23 +47,23 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): def build_network(self, places): input_data = fluid.data(shape=self.shape, dtype='float32', name="input") - loader = fluid.io.DataLoader.from_generator(capacity=16, - feed_list=[input_data], - iterable=self.iterable) + loader = fluid.io.DataLoader.from_generator( + capacity=16, feed_list=[input_data], iterable=self.iterable + ) fc = fluid.layers.fc(input_data, size=10) loss = fluid.layers.reduce_mean(fc) - loader.set_batch_generator(create_reader(self.shape, self.batch_num), - places=places if loader.iterable else None) + loader.set_batch_generator( + create_reader(self.shape, self.batch_num), + places=places if loader.iterable else None, + ) return input_data, loss, loader - def assertInputData(self, - batch_id, - input_data, - dev_cnt, - check_visited=True): + def assertInputData( + self, batch_id, input_data, dev_cnt, check_visited=True + ): if isinstance(input_data, list): self.assertTrue(len(input_data), dev_cnt) start_val = dev_cnt * batch_id @@ -82,12 +80,15 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): start_val += 1 else: - self.assertEqual(list(input_data.shape), - [self.shape[0] * dev_cnt] + self.shape[1:]) + self.assertEqual( + list(input_data.shape), + [self.shape[0] * dev_cnt] + self.shape[1:], + ) start_val = dev_cnt * batch_id for idx in range(dev_cnt): - data_part = input_data[idx * self.shape[0]:(idx + 1) * - self.shape[0], :] + data_part = input_data[ + idx * self.shape[0] : (idx + 1) * self.shape[0], : + ] num = data_part.flatten()[0] self.assertTrue((data_part == num).all()) if check_visited: @@ -103,8 +104,8 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): place_list.extend([fluid.cuda_places(0)]) else: place_list.extend( - [fluid.cuda_places(0), - fluid.cuda_places([0, 1])]) + [fluid.cuda_places(0), fluid.cuda_places([0, 1])] + ) return place_list def test_main(self): @@ -129,11 +130,12 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): main_program = fluid.default_main_program() if use_compiled_program: main_program = fluid.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name, - places=places) + main_program + ).with_data_parallel(loss_name=loss.name, places=places) - max_batch_num = min(self.break_num, - int(self.batch_num / dev_cnt)) + max_batch_num = min( + self.break_num, int(self.batch_num / dev_cnt) + ) if loader.iterable: early_break = False @@ -145,13 +147,14 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): if batch_id >= self.break_num: early_break = True break - self.assertInputData(batch_id, - data, - dev_cnt, - check_visited=False) - fetch_val, = exe.run(program=main_program, - feed=data, - fetch_list=fetch_list) + self.assertInputData( + batch_id, data, dev_cnt, check_visited=False + ) + (fetch_val,) = exe.run( + program=main_program, + feed=data, + fetch_list=fetch_list, + ) self.assertInputData(batch_id, fetch_val, dev_cnt) batch_id += 1 @@ -172,10 +175,12 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): if batch_id >= self.break_num: loader.reset() break - fetch_val, = exe.run(program=main_program, - fetch_list=fetch_list) - self.assertInputData(batch_id, fetch_val, - dev_cnt) + (fetch_val,) = exe.run( + program=main_program, fetch_list=fetch_list + ) + self.assertInputData( + batch_id, fetch_val, dev_cnt + ) batch_id += 1 except fluid.core.EOFException: loader.reset() @@ -187,35 +192,30 @@ class DataLoaderKeepOrderTestBase(unittest.TestCase): class IterableDataLoaderKeepOrderTest2(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 10000 class IterableDataLoaderKeepOrderTest3(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = False self.break_num = 2 class IterableDataLoaderKeepOrderTest4(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 2 class IterableDataLoaderKeepOrderTest5(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = False self.break_num = 0 class IterableDataLoaderKeepOrderTest6(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 0 diff --git a/python/paddle/fluid/tests/unittests/test_dataset.py b/python/paddle/fluid/tests/unittests/test_dataset.py index 8e09e8f75c9..5c0dc9db6a2 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_dataset.py @@ -25,7 +25,7 @@ import unittest class TestDataset(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def setUp(self): self.use_data_loader = False @@ -33,7 +33,7 @@ class TestDataset(unittest.TestCase): self.drop_last = False def test_dataset_create(self): - """ Testcase for dataset create. """ + """Testcase for dataset create.""" try: dataset = paddle.distributed.InMemoryDataset() except: @@ -99,22 +99,22 @@ class TestDataset(unittest.TestCase): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.update_settings(pipe_command="cat1") - dataset._init_distributed_settings(parse_ins_id=True, - parse_content=True, - fea_eval=True, - candidate_size=10000) + dataset._init_distributed_settings( + parse_ins_id=True, + parse_content=True, + fea_eval=True, + candidate_size=10000, + ) dataset.set_filelist([dump_a_path, dump_b_path]) dataset.load_into_memory() dataset.local_shuffle() @@ -136,7 +136,7 @@ class TestDataset(unittest.TestCase): temp_dir.cleanup() def test_dataset_config(self): - """ Testcase for dataset configuration. """ + """Testcase for dataset configuration.""" dataset = fluid.core.Dataset("MultiSlotDataset") dataset.set_thread_num(12) dataset.set_filelist(["a.txt", "b.txt", "c.txt"]) @@ -169,10 +169,12 @@ class TestDataset(unittest.TestCase): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "afs:test_in_memory_dataset_run_a.txt") - filename2 = os.path.join(temp_dir.name, - "afs:test_in_memory_dataset_run_b.txt") + filename1 = os.path.join( + temp_dir.name, "afs:test_in_memory_dataset_run_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "afs:test_in_memory_dataset_run_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -189,18 +191,19 @@ class TestDataset(unittest.TestCase): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - download_cmd="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, + thread_num=3, + pipe_command="cat", + download_cmd="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() paddle.enable_static() @@ -212,7 +215,8 @@ class TestDataset(unittest.TestCase): exe.run(startup_program) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(main_program, feed=data) @@ -230,10 +234,12 @@ class TestDataset(unittest.TestCase): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset_run_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset_run_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset_run_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset_run_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -250,17 +256,15 @@ class TestDataset(unittest.TestCase): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset._init_distributed_settings(fea_eval=True, candidate_size=1) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() @@ -272,15 +276,17 @@ class TestDataset(unittest.TestCase): exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) @@ -291,10 +297,12 @@ class TestDataset(unittest.TestCase): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset_masterpatch_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset_masterpatch_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset_masterpatch_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset_masterpatch_b.txt" + ) with open(filename1, "w") as f: data = "1 id1 1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -320,28 +328,27 @@ class TestDataset(unittest.TestCase): startup_program = fluid.Program() with fluid.program_guard(train_program, startup_program): for slot in slots[:2]: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) for slot in slots[2:]: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="float32", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=1, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=1, pipe_command="cat", use_var=slots_vars + ) dataset._init_distributed_settings(parse_ins_id=True) - dataset.set_filelist([ - "test_in_memory_dataset_masterpatch_a.txt", - "test_in_memory_dataset_masterpatch_b.txt" - ]) + dataset.set_filelist( + [ + "test_in_memory_dataset_masterpatch_a.txt", + "test_in_memory_dataset_masterpatch_b.txt", + ] + ) dataset.load_into_memory() dataset.local_shuffle() @@ -356,7 +363,7 @@ class TestDataset(unittest.TestCase): except Exception as e: self.assertTrue(False) - #dataset._set_merge_by_lineid(2) + # dataset._set_merge_by_lineid(2) dataset.update_settings(merge_size=2) dataset.dataset.merge_by_lineid() @@ -367,10 +374,12 @@ class TestDataset(unittest.TestCase): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset_masterpatch1_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset_masterpatch1_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset_masterpatch1_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset_masterpatch1_b.txt" + ) with open(filename1, "w") as f: data = "1 id1 1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -394,34 +403,31 @@ class TestDataset(unittest.TestCase): train_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(train_program, startup_program): - var1 = fluid.layers.data(name="slot1", - shape=[1], - dtype="int64", - lod_level=0) - var2 = fluid.layers.data(name="slot2", - shape=[1], - dtype="int64", - lod_level=0) - var3 = fluid.layers.data(name="slot3", - shape=[1], - dtype="float32", - lod_level=0) - var4 = fluid.layers.data(name="slot4", - shape=[1], - dtype="float32", - lod_level=0) + var1 = fluid.layers.data( + name="slot1", shape=[1], dtype="int64", lod_level=0 + ) + var2 = fluid.layers.data( + name="slot2", shape=[1], dtype="int64", lod_level=0 + ) + var3 = fluid.layers.data( + name="slot3", shape=[1], dtype="float32", lod_level=0 + ) + var4 = fluid.layers.data( + name="slot4", shape=[1], dtype="float32", lod_level=0 + ) slots_vars = [var1, var2, var3, var4] dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=1, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=1, pipe_command="cat", use_var=slots_vars + ) dataset._init_distributed_settings(parse_ins_id=True) - dataset.set_filelist([ - "test_in_memory_dataset_masterpatch1_a.txt", - "test_in_memory_dataset_masterpatch1_b.txt" - ]) + dataset.set_filelist( + [ + "test_in_memory_dataset_masterpatch1_a.txt", + "test_in_memory_dataset_masterpatch1_b.txt", + ] + ) dataset.load_into_memory() dataset.local_shuffle() @@ -448,10 +454,12 @@ class TestDataset(unittest.TestCase): Use float type id """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset_run_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset_run_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset_run_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset_run_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -468,43 +476,44 @@ class TestDataset(unittest.TestCase): slots = ["slot1_f", "slot2_f", "slot3_f", "slot4_f"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="float32", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() dataset.local_shuffle() - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(fluid.default_startup_program()) for i in range(2): try: exe.train_from_dataset(fluid.default_main_program(), dataset) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=1) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=2) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=2) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=3) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=4) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=1 + ) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=2 + ) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=2 + ) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=3 + ) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=4 + ) except ImportError as e: pass except Exception as e: @@ -512,15 +521,17 @@ class TestDataset(unittest.TestCase): if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) @@ -536,20 +547,22 @@ class TestDataset(unittest.TestCase): dataset._set_parse_ins_id(False) dataset.load_into_memory() dataset.dataset.merge_by_lineid() - dataset.update_settings(batch_size=1, - thread_num=2, - input_type=1, - pipe_command="cat", - use_var=[], - fs_name="", - fs_ugi="", - download_cmd="cat", - merge_size=-1, - parse_ins_id=False, - parse_content=False, - fleet_send_batch_size=2, - fleet_send_sleep_seconds=2, - fea_eval=True) + dataset.update_settings( + batch_size=1, + thread_num=2, + input_type=1, + pipe_command="cat", + use_var=[], + fs_name="", + fs_ugi="", + download_cmd="cat", + merge_size=-1, + parse_ins_id=False, + parse_content=False, + fleet_send_batch_size=2, + fleet_send_sleep_seconds=2, + fea_eval=True, + ) fleet_ptr = fluid.core.Fleet() fleet_ptr.set_client2client_config(1, 1, 1) fleet_ptr.get_cache_threshold(0) @@ -579,40 +592,39 @@ class TestDataset(unittest.TestCase): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.QueueDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist([filename1, filename2]) exe = fluid.Executor(fluid.CPUPlace()) exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) dataset2 = paddle.distributed.QueueDataset() - dataset2.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset2.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist([]) try: exe.train_from_dataset(fluid.default_main_program(), dataset2) @@ -648,33 +660,36 @@ class TestDataset(unittest.TestCase): slots = ["slot1_f", "slot2_f", "slot3_f", "slot4_f"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="float32", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.QueueDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist([filename1, filename2]) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) @@ -706,35 +721,41 @@ class TestDataset(unittest.TestCase): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.data(name=slot, - shape=[None, 1], - dtype="int64", - lod_level=1) + var = fluid.data( + name=slot, shape=[None, 1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=1, - thread_num=2, - input_type=1, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=1, + thread_num=2, + input_type=1, + pipe_command="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) @@ -764,22 +785,25 @@ class TestDataset(unittest.TestCase): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=1, - pipe_command="cat", - data_feed_type="SlotRecordInMemoryDataFeed", - use_var=slots_vars) - dataset._init_distributed_settings(parse_ins_id=True, - parse_content=True, - fea_eval=True, - candidate_size=10000) + dataset.init( + batch_size=32, + thread_num=1, + pipe_command="cat", + data_feed_type="SlotRecordInMemoryDataFeed", + use_var=slots_vars, + ) + dataset._init_distributed_settings( + parse_ins_id=True, + parse_content=True, + fea_eval=True, + candidate_size=10000, + ) dataset.set_filelist([dump_a_path, dump_b_path]) dataset.load_into_memory() @@ -827,10 +851,9 @@ class TestDatasetWithFetchHandler(unittest.TestCase): slots_vars = [] poolings = [] for slot in slots: - data = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) var = fluid.layers.cast(x=data, dtype='float32') pool = fluid.layers.sequence_pool(input=var, pool_type='AVERAGE') @@ -850,10 +873,9 @@ class TestDatasetWithFetchHandler(unittest.TestCase): files(list): files of get_dataset """ dataset = paddle.distributed.QueueDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=inputs) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=inputs + ) dataset.set_filelist(files) return dataset @@ -862,10 +884,12 @@ class TestDatasetWithFetchHandler(unittest.TestCase): Test Dataset With Fetch Handler. TestCases. """ self.temp_dir = tempfile.TemporaryDirectory() - self.filename1 = os.path.join(self.temp_dir.name, - "test_queue_dataset_run_a.txt") - self.filename2 = os.path.join(self.temp_dir.name, - "test_queue_dataset_run_b.txt") + self.filename1 = os.path.join( + self.temp_dir.name, "test_queue_dataset_run_a.txt" + ) + self.filename2 = os.path.join( + self.temp_dir.name, "test_queue_dataset_run_b.txt" + ) with open(self.filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -940,9 +964,11 @@ class TestDatasetWithFetchHandler(unittest.TestCase): fh.help() try: - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_handler=fh) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + fetch_handler=fh, + ) except ImportError as e: print("warning: we skip trainer_desc_pb2 import problem in windows") except RuntimeError as e: @@ -953,10 +979,10 @@ class TestDatasetWithFetchHandler(unittest.TestCase): class TestDataset2(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def setUp(self): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" self.use_data_loader = False self.epoch_num = 10 self.drop_last = False @@ -966,10 +992,12 @@ class TestDataset2(unittest.TestCase): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run_b.txt" + ) self.skipTest("parameter server will add pslib UT later") @@ -988,16 +1016,21 @@ class TestDataset2(unittest.TestCase): train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, + ) + with fluid.program_guard(train_program, startup_program): slots = ["slot1_ff", "slot2_ff", "slot3_ff", "slot4_ff"] slots_vars = [] for slot in slots: - var = fluid.layers.data(\ - name=slot, shape=[1], dtype="float32", lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) - fake_cost = \ - fluid.layers.elementwise_sub(slots_vars[0], slots_vars[-1]) + fake_cost = fluid.layers.elementwise_sub( + slots_vars[0], slots_vars[-1] + ) fake_cost = paddle.mean(fake_cost) with fluid.scope_guard(scope): place = fluid.CPUPlace() @@ -1017,10 +1050,12 @@ class TestDataset2(unittest.TestCase): exe.run(startup_program) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, + thread_num=3, + pipe_command="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() fleet._opt_info = None @@ -1033,10 +1068,12 @@ class TestDataset2(unittest.TestCase): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run2_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run2_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run2_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run2_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -1054,15 +1091,18 @@ class TestDataset2(unittest.TestCase): startup_program = fluid.Program() scope = fluid.Scope() from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet + with fluid.program_guard(train_program, startup_program): slots = ["slot1_ff", "slot2_ff", "slot3_ff", "slot4_ff"] slots_vars = [] for slot in slots: - var = fluid.layers.data(\ - name=slot, shape=[1], dtype="float32", lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) - fake_cost = \ - fluid.layers.elementwise_sub(slots_vars[0], slots_vars[-1]) + fake_cost = fluid.layers.elementwise_sub( + slots_vars[0], slots_vars[-1] + ) fake_cost = paddle.mean(fake_cost) with fluid.scope_guard(scope): place = fluid.CPUPlace() @@ -1073,17 +1113,15 @@ class TestDataset2(unittest.TestCase): print("warning: no mpi4py") adam = fluid.optimizer.Adam(learning_rate=0.000005) try: - adam = fleet.distributed_optimizer(adam, - strategy={ - "fs_uri": - "fs_uri_xxx", - "fs_user": - "fs_user_xxx", - "fs_passwd": - "fs_passwd_xxx", - "fs_hadoop_bin": - "fs_hadoop_bin_xxx" - }) + adam = fleet.distributed_optimizer( + adam, + strategy={ + "fs_uri": "fs_uri_xxx", + "fs_user": "fs_user_xxx", + "fs_passwd": "fs_passwd_xxx", + "fs_hadoop_bin": "fs_hadoop_bin_xxx", + }, + ) adam.minimize([fake_cost], [scope]) except AttributeError as e: print("warning: no mpi") @@ -1091,10 +1129,12 @@ class TestDataset2(unittest.TestCase): print("warning: no mpi4py") exe.run(startup_program) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, + thread_num=3, + pipe_command="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() try: @@ -1132,7 +1172,7 @@ class TestDataset2(unittest.TestCase): dataset.global_shuffle() except: print("warning: catch expected error") - #dataset.get_pv_data_size() + # dataset.get_pv_data_size() dataset.get_memory_data_size() dataset.get_shuffle_data_size() dataset = paddle.distributed.QueueDataset() @@ -1161,10 +1201,12 @@ class TestDataset2(unittest.TestCase): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run2_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run2_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run2_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run2_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -1182,15 +1224,18 @@ class TestDataset2(unittest.TestCase): startup_program = fluid.Program() scope = fluid.Scope() from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet + with fluid.program_guard(train_program, startup_program): slots = ["slot1_ff", "slot2_ff", "slot3_ff", "slot4_ff"] slots_vars = [] for slot in slots: - var = fluid.layers.data(\ - name=slot, shape=[1], dtype="float32", lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) - fake_cost = \ - fluid.layers.elementwise_sub(slots_vars[0], slots_vars[-1]) + fake_cost = fluid.layers.elementwise_sub( + slots_vars[0], slots_vars[-1] + ) fake_cost = paddle.mean(fake_cost) with fluid.scope_guard(scope): place = fluid.CPUPlace() @@ -1201,17 +1246,15 @@ class TestDataset2(unittest.TestCase): print("warning: no mpi4py") adam = fluid.optimizer.Adam(learning_rate=0.000005) try: - adam = fleet.distributed_optimizer(adam, - strategy={ - "fs_uri": - "fs_uri_xxx", - "fs_user": - "fs_user_xxx", - "fs_passwd": - "fs_passwd_xxx", - "fs_hadoop_bin": - "fs_hadoop_bin_xxx" - }) + adam = fleet.distributed_optimizer( + adam, + strategy={ + "fs_uri": "fs_uri_xxx", + "fs_user": "fs_user_xxx", + "fs_passwd": "fs_passwd_xxx", + "fs_hadoop_bin": "fs_hadoop_bin_xxx", + }, + ) adam.minimize([fake_cost], [scope]) except AttributeError as e: print("warning: no mpi") @@ -1219,10 +1262,12 @@ class TestDataset2(unittest.TestCase): print("warning: no mpi4py") exe.run(startup_program) dataset = paddle.distributed.fleet.BoxPSDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, + thread_num=3, + pipe_command="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() try: @@ -1232,14 +1277,16 @@ class TestDataset2(unittest.TestCase): fleet._opt_info = None fleet._fleet_ptr = None dataset = paddle.distributed.fleet.BoxPSDataset() - dataset.init(rank_offset="", - pv_batch_size=1, - fs_name="", - fs_ugi="", - data_feed_type="MultiSlotInMemoryDataFeed", - parse_logkey=True, - merge_by_sid=True, - enable_pv_merge=True) + dataset.init( + rank_offset="", + pv_batch_size=1, + fs_name="", + fs_ugi="", + data_feed_type="MultiSlotInMemoryDataFeed", + parse_logkey=True, + merge_by_sid=True, + enable_pv_merge=True, + ) d = paddle.distributed.fleet.DatasetBase() try: dataset._set_feed_type("MultiSlotInMemoryDataFeed") @@ -1270,7 +1317,7 @@ class TestDataset2(unittest.TestCase): dataset.global_shuffle() except: print("warning: catch expected error") - #dataset.get_pv_data_size() + # dataset.get_pv_data_size() dataset.get_memory_data_size() dataset.get_shuffle_data_size() temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py b/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py index 46ba155f017..51463e7ff62 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py +++ b/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py @@ -23,46 +23,57 @@ import tempfile import unittest import paddle.fluid.incubate.data_generator as dg -#paddle.enable_static() +# paddle.enable_static() # fluid.disable_dygraph() fluid.disable_dygraph() url_schema_len = 5 query_schema = [ - 'Q_query_basic', 'Q_query_phrase', 'Q_quq', 'Q_timelevel', - 'Q_context_title_basic1', 'Q_context_title_basic2', - 'Q_context_title_basic3', 'Q_context_title_basic4', - 'Q_context_title_basic5', 'Q_context_title_phrase1', - 'Q_context_title_phrase2', 'Q_context_title_phrase3', - 'Q_context_title_phrase4', 'Q_context_title_phrase5', 'Q_context_site1', - 'Q_context_site2', 'Q_context_site3', 'Q_context_site4', 'Q_context_site5' + 'Q_query_basic', + 'Q_query_phrase', + 'Q_quq', + 'Q_timelevel', + 'Q_context_title_basic1', + 'Q_context_title_basic2', + 'Q_context_title_basic3', + 'Q_context_title_basic4', + 'Q_context_title_basic5', + 'Q_context_title_phrase1', + 'Q_context_title_phrase2', + 'Q_context_title_phrase3', + 'Q_context_title_phrase4', + 'Q_context_title_phrase5', + 'Q_context_site1', + 'Q_context_site2', + 'Q_context_site3', + 'Q_context_site4', + 'Q_context_site5', ] class CTRDataset(dg.MultiSlotDataGenerator): - def __init__(self, mode): self.test = mode def generate_sample(self, line): - def reader(): ins = line.strip().split(';') label_pos_num = int(ins[1].split(' ')[0]) label_neg_num = int(ins[1].split(' ')[1]) - #query fea parse + # query fea parse bias = 2 query_len = 0 sparse_query_feature = [] for index in range(len(query_schema)): pos = index + bias sparse_query_feature.append( - [int(x) for x in ins[pos].split(' ')]) + [int(x) for x in ins[pos].split(' ')] + ) if index == 0: query_len = len(ins[pos].split(' ')) query_len = 1.0 / (1 + pow(2.7182818, 3 - 1.0 * query_len)) - #positive url fea parse + # positive url fea parse bias = 2 + len(query_schema) pos_url_feas = [] pos_click_feas = [] @@ -73,14 +84,24 @@ class CTRDataset(dg.MultiSlotDataGenerator): for index in range(url_schema_len - 1): pos = bias + k * (url_schema_len) + index pos_url_fea.append([int(x) for x in ins[pos].split(' ')]) - #click info - if (ins[pos + 1] == ''): + # click info + if ins[pos + 1] == '': continue item = ins[pos + 1].split(' ') if len(item) != 17: continue - stat_fea = [[max(float(item[i]), 0.0)] for i in range(len(item)) \ - if not (i == 5 or i == 9 or i == 13 or i == 14 or i ==15 or i ==16)] + stat_fea = [ + [max(float(item[i]), 0.0)] + for i in range(len(item)) + if not ( + i == 5 + or i == 9 + or i == 13 + or i == 14 + or i == 15 + or i == 16 + ) + ] pos_url_feas.append(pos_url_fea) pos_click_feas.append(stat_fea) @@ -90,7 +111,7 @@ class CTRDataset(dg.MultiSlotDataGenerator): pos_context_fea = [[query_serach], [query_len]] pos_context_feas.append(pos_context_fea) - #negative url fea parse + # negative url fea parse bias = 2 + len(query_schema) + label_pos_num * (url_schema_len) neg_url_feas = [] neg_click_feas = [] @@ -101,15 +122,25 @@ class CTRDataset(dg.MultiSlotDataGenerator): for index in range(url_schema_len - 1): pos = bias + k * (url_schema_len) + index neg_url_fea.append([int(x) for x in ins[pos].split(' ')]) - if (ins[pos + 1] == ''): + if ins[pos + 1] == '': continue item = ins[pos + 1].split(' ') - #zdf_tmp + # zdf_tmp if len(item) != 17: continue - #print ins[pos + 1] - stat_fea = [[max(float(item[i]), 0.0)] for i in range(len(item)) \ - if not (i == 5 or i == 9 or i == 13 or i == 14 or i == 15 or i == 16)] + # print ins[pos + 1] + stat_fea = [ + [max(float(item[i]), 0.0)] + for i in range(len(item)) + if not ( + i == 5 + or i == 9 + or i == 13 + or i == 14 + or i == 15 + or i == 16 + ) + ] neg_click_feas.append(stat_fea) neg_url_feas.append(neg_url_fea) @@ -119,7 +150,7 @@ class CTRDataset(dg.MultiSlotDataGenerator): neg_context_fea = [[query_serach], [query_len]] neg_context_feas.append(neg_context_fea) - #make train data + # make train data if self.test == 1: for p in range(len(pos_url_feas)): # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] @@ -129,9 +160,17 @@ class CTRDataset(dg.MultiSlotDataGenerator): pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] - yield zip(feature_name, [[1]] + sparse_query_feature + - pos_url_fea + pos_click_fea + pos_context_fea + - pos_url_fea + pos_click_fea + pos_context_fea) + yield zip( + feature_name, + [[1]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + pos_url_fea + + pos_click_fea + + pos_context_fea, + ) for n in range(len(neg_url_feas)): feature_name = ["click"] for i in range(1, 54): @@ -139,18 +178,26 @@ class CTRDataset(dg.MultiSlotDataGenerator): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - yield zip(feature_name, [[0]] + sparse_query_feature + - neg_url_fea + neg_click_fea + neg_context_fea + - neg_url_fea + neg_click_fea + neg_context_fea) + yield zip( + feature_name, + [[0]] + + sparse_query_feature + + neg_url_fea + + neg_click_fea + + neg_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) elif self.test == 0: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -160,21 +207,32 @@ class CTRDataset(dg.MultiSlotDataGenerator): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, [[1]] + sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + [[1]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) elif self.test == 2: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -184,21 +242,32 @@ class CTRDataset(dg.MultiSlotDataGenerator): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, [[1], [2]] + sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + [[1], [2]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) elif self.test == 3: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -208,21 +277,32 @@ class CTRDataset(dg.MultiSlotDataGenerator): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, [[1], [2.0]] + sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + [[1], [2.0]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) elif self.test == 4: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -232,21 +312,32 @@ class CTRDataset(dg.MultiSlotDataGenerator): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, [[], [2.0]] + sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + [[], [2.0]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) elif self.test == 5: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -256,18 +347,28 @@ class CTRDataset(dg.MultiSlotDataGenerator): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) return reader class TestDataset(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def setUp(self): pass @@ -292,74 +393,80 @@ class TestDataset(unittest.TestCase): f.write(data) slot_data = [] - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) slot_data.append(label) # sprase_query_feat_names len_sparse_query = 19 for feat_name in range(1, len_sparse_query + 1): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='int64', - lod_level=1)) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='int64', lod_level=1 + ) + ) # sparse_url_feat_names for feat_name in range(len_sparse_query + 1, len_sparse_query + 5): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='int64', - lod_level=1)) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='int64', lod_level=1 + ) + ) # dense_feat_names for feat_name in range(len_sparse_query + 5, len_sparse_query + 16): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='float32')) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='float32' + ) + ) # context_feat_namess for feat_name in range(len_sparse_query + 16, len_sparse_query + 18): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='float32')) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='float32' + ) + ) # neg sparse_url_feat_names for feat_name in range(len_sparse_query + 18, len_sparse_query + 22): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='int64', - lod_level=1)) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='int64', lod_level=1 + ) + ) # neg dense_feat_names for feat_name in range(len_sparse_query + 22, len_sparse_query + 33): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='float32')) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='float32' + ) + ) # neg context_feat_namess for feat_name in range(len_sparse_query + 33, len_sparse_query + 35): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='float32')) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='float32' + ) + ) dataset = paddle.distributed.InMemoryDataset() print("========================================") generator_class = CTRDataset(mode=0) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) print("case 1: check passed!") except Exception as e: print("warning: catch expected error") @@ -370,9 +477,9 @@ class TestDataset(unittest.TestCase): print("========================================") generator_class = CTRDataset(mode=2) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) except Exception as e: print("warning: case 2 catch expected error") print(e) @@ -382,9 +489,9 @@ class TestDataset(unittest.TestCase): print("========================================") generator_class = CTRDataset(mode=3) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) except Exception as e: print("warning: case 3 catch expected error") print(e) @@ -394,9 +501,9 @@ class TestDataset(unittest.TestCase): print("========================================") generator_class = CTRDataset(mode=4) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) except Exception as e: print("warning: case 4 catch expected error") print(e) @@ -406,9 +513,9 @@ class TestDataset(unittest.TestCase): print("========================================") generator_class = CTRDataset(mode=5) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) except Exception as e: print("warning: case 5 catch expected error") print(e) diff --git a/python/paddle/fluid/tests/unittests/test_dataset_dataloader.py b/python/paddle/fluid/tests/unittests/test_dataset_dataloader.py index ec61ee3a838..5c7c97991d6 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset_dataloader.py +++ b/python/paddle/fluid/tests/unittests/test_dataset_dataloader.py @@ -44,9 +44,12 @@ def write_reader_data_to_file(filename, reader): with open(filename, 'w') as fid: for instance_list in reader(): for i, instance in enumerate(instance_list): - instance = np.reshape(instance, [ - instance.size, - ]) + instance = np.reshape( + instance, + [ + instance.size, + ], + ) fid.write(str(instance.size) + ' ') fid.write(' '.join(map(str, instance))) fid.write(' ') @@ -55,21 +58,20 @@ def write_reader_data_to_file(filename, reader): def fake_reader(batch_size=BATCH_SIZE, batch_num=BATCH_NUM): - def __reader__(): iteration = BATCH_SIZE * BATCH_NUM iteration = int(iteration + BATCH_SIZE / 2) for _ in range(iteration): image = np.random.random(size=IMAGE_SHAPE).astype('float32') - label = np.random.random_integers(size=LABEL_SHAPE, low=0, - high=9).astype('int64') + label = np.random.random_integers( + size=LABEL_SHAPE, low=0, high=9 + ).astype('int64') yield image, label return __reader__ class DatasetLoaderTestBase(unittest.TestCase): - def setUp(self): self.dataset_name = "QueueDataset" self.drop_last = False @@ -82,12 +84,12 @@ class DatasetLoaderTestBase(unittest.TestCase): main_prog = fluid.Program() startup_prog = fluid.Program() with fluid.program_guard(main_prog, startup_prog): - image = fluid.layers.data(name='image', - shape=IMAGE_SHAPE, - dtype='float32') - label = fluid.layers.data(name='label', - shape=LABEL_SHAPE, - dtype='int64') + image = fluid.layers.data( + name='image', shape=IMAGE_SHAPE, dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=LABEL_SHAPE, dtype='int64' + ) simple_fc_net_with_inputs(image, label) @@ -114,19 +116,23 @@ class DatasetLoaderTestBase(unittest.TestCase): filelist = [] if file_num > 1 and randomize_batch_num: random_delta_batch_size = np.random.random_integers( - low=-BATCH_NUM / 2, high=BATCH_NUM / 2, size=[file_num]) + low=-BATCH_NUM / 2, high=BATCH_NUM / 2, size=[file_num] + ) random_delta_batch_size[-1] = -int( - np.sum(random_delta_batch_size[0:-1])) + np.sum(random_delta_batch_size[0:-1]) + ) else: random_delta_batch_size = np.zeros(shape=[file_num]) for i in range(file_num): - filename = os.path.join(self.temp_dir.name, - 'dataset_test_{}.txt'.format(i)) + filename = os.path.join( + self.temp_dir.name, 'dataset_test_{}.txt'.format(i) + ) filelist.append(filename) write_reader_data_to_file( filename, - fake_reader(batch_num=BATCH_NUM + random_delta_batch_size[i])) + fake_reader(batch_num=BATCH_NUM + random_delta_batch_size[i]), + ) dataset.set_filelist(filelist) dataset._set_use_var(feeds) @@ -134,9 +140,9 @@ class DatasetLoaderTestBase(unittest.TestCase): if self.dataset_name == 'InMemoryDataset': dataset.load_into_memory() - dataloader = fluid.io.DataLoader.from_dataset(dataset=dataset, - places=places, - drop_last=self.drop_last) + dataloader = fluid.io.DataLoader.from_dataset( + dataset=dataset, places=places, drop_last=self.drop_last + ) prog = fluid.CompiledProgram(main_prog).with_data_parallel() exe = fluid.Executor(place) @@ -159,22 +165,29 @@ class DatasetLoaderTestBase(unittest.TestCase): batch_size = BATCH_SIZE self.assertEquals(image.shape()[1:], IMAGE_SHAPE) - self.assertTrue(image._place()._equals(places[idx]), - msg=get_place_string(image._place()) + - ' vs ' + get_place_string(places[idx])) + self.assertTrue( + image._place()._equals(places[idx]), + msg=get_place_string(image._place()) + + ' vs ' + + get_place_string(places[idx]), + ) if self.drop_last: self.assertEquals(image.shape()[0], BATCH_SIZE) else: - self.assertTrue(image.shape()[0] == BATCH_SIZE - or image.shape()[0] == BATCH_SIZE / 2) + self.assertTrue( + image.shape()[0] == BATCH_SIZE + or image.shape()[0] == BATCH_SIZE / 2 + ) self.assertEquals(label.shape()[1:], LABEL_SHAPE) self.assertTrue(label._place()._equals(places[idx])) if self.drop_last: self.assertEquals(label.shape()[0], BATCH_SIZE) else: - self.assertTrue(label.shape()[0] == BATCH_SIZE - or label.shape()[0] == BATCH_SIZE / 2) + self.assertTrue( + label.shape()[0] == BATCH_SIZE + or label.shape()[0] == BATCH_SIZE / 2 + ) self.assertEquals(image.shape()[0], label.shape()[0]) @@ -203,7 +216,6 @@ class DatasetLoaderTestBase(unittest.TestCase): class QueueDatasetTestWithoutDropLast(DatasetLoaderTestBase): - def setUp(self): self.dataset_name = "QueueDataset" self.drop_last = True @@ -211,7 +223,6 @@ class QueueDatasetTestWithoutDropLast(DatasetLoaderTestBase): class InMemoryDatasetTestWithoutDropLast(DatasetLoaderTestBase): - def setUp(self): self.dataset_name = "InMemoryDataset" self.drop_last = False @@ -219,7 +230,6 @@ class InMemoryDatasetTestWithoutDropLast(DatasetLoaderTestBase): class InMemoryDatasetTestWithDropLast(DatasetLoaderTestBase): - def setUp(self): self.dataset_name = "InMemoryDataset" self.drop_last = True diff --git a/python/paddle/fluid/tests/unittests/test_dataset_download.py b/python/paddle/fluid/tests/unittests/test_dataset_download.py index 06f015edf95..f1fba215b93 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset_download.py +++ b/python/paddle/fluid/tests/unittests/test_dataset_download.py @@ -18,7 +18,6 @@ from paddle.dataset.common import download, DATA_HOME, md5file class TestDataSetDownload(unittest.TestCase): - def setUp(self): flower_path = DATA_HOME + "/flowers/imagelabels.mat" diff --git a/python/paddle/fluid/tests/unittests/test_debugger.py b/python/paddle/fluid/tests/unittests/test_debugger.py index 7e2e40e4ed0..23beb7f1a63 100644 --- a/python/paddle/fluid/tests/unittests/test_debugger.py +++ b/python/paddle/fluid/tests/unittests/test_debugger.py @@ -19,42 +19,41 @@ from paddle.fluid.framework import Program class TestDebugger(unittest.TestCase): - def test_debug_str(self): p = Program() b = p.current_block() - #selected_rows - b.create_var(name='selected_rows', - dtype="float32", - shape=[5, 10], - type=core.VarDesc.VarType.SELECTED_ROWS) + # selected_rows + b.create_var( + name='selected_rows', + dtype="float32", + shape=[5, 10], + type=core.VarDesc.VarType.SELECTED_ROWS, + ) - #tensor array - b.create_var(name='tensor_array', - shape=[5, 10], - type=core.VarDesc.VarType.LOD_TENSOR_ARRAY) + # tensor array + b.create_var( + name='tensor_array', + shape=[5, 10], + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ) - #operator - mul_x = b.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = b.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = b.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - b.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) + # operator + mul_x = b.create_parameter( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = b.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = b.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + b.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) print(debugger.pprint_program_codes(p)) diff --git a/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py index 5025186349b..dae5fd7b7b7 100644 --- a/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py @@ -18,8 +18,7 @@ from op_test import OpTest class TestDecayedAdagradOp1(OpTest): - ''' Test DecayedAdagrad operator with explicit attributes - ''' + '''Test DecayedAdagrad operator with explicit attributes''' def setUp(self): self.op_type = "decayed_adagrad" @@ -35,7 +34,7 @@ class TestDecayedAdagradOp1(OpTest): 'Param': param, 'Grad': grad, 'Moment': moment, - 'LearningRate': np.array([lr]).astype("float32") + 'LearningRate': np.array([lr]).astype("float32"), } self.attrs = {'decay': decay, 'epsilon': epsilon} @@ -50,8 +49,7 @@ class TestDecayedAdagradOp1(OpTest): class TestDecayedAdagradOp2(OpTest): - ''' Test DecayedAdagrad operator with default attributes - ''' + '''Test DecayedAdagrad operator with default attributes''' def setUp(self): self.op_type = "decayed_adagrad" @@ -67,7 +65,7 @@ class TestDecayedAdagradOp2(OpTest): 'Param': param, 'Grad': grad, 'Moment': moment, - 'LearningRate': np.array([lr]).astype("float32") + 'LearningRate': np.array([lr]).astype("float32"), } self.attrs = {'decay': decay, 'epsilon': epsilon} @@ -83,5 +81,6 @@ class TestDecayedAdagradOp2(OpTest): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py index 9b5ed11d9b9..d7a5d371431 100644 --- a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py +++ b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py @@ -40,14 +40,16 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): with fluid.unique_name.guard(): with fluid.program_guard(main_prog, startup_prog): - image = fluid.layers.data(name='image', - shape=[784], - dtype='float32') + image = fluid.layers.data( + name='image', shape=[784], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') - py_reader = fluid.io.PyReader(feed_list=[image, label], - capacity=4, - iterable=not use_legacy_py_reader, - use_double_buffer=use_double_buffer) + py_reader = fluid.io.PyReader( + feed_list=[image, label], + capacity=4, + iterable=not use_legacy_py_reader, + use_double_buffer=use_double_buffer, + ) hidden = image for hidden_size in [10, 20, 30]: hidden = fluid.layers.fc( @@ -55,13 +57,16 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): size=hidden_size, act='tanh', bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) + initializer=fluid.initializer.Constant(value=1.0) + ), + ) - predict_label = fluid.layers.fc(hidden, - size=CLASS_NUM, - act='softmax') + predict_label = fluid.layers.fc( + hidden, size=CLASS_NUM, act='softmax' + ) loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -69,28 +74,35 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): class TestBase(unittest.TestCase): - - def run_main(self, use_legacy_py_reader, with_data_parallel, places, - use_double_buffer): + def run_main( + self, + use_legacy_py_reader, + with_data_parallel, + places, + use_double_buffer, + ): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, py_reader, loss = simple_fc_net( - places, use_legacy_py_reader, use_double_buffer) + places, use_legacy_py_reader, use_double_buffer + ) reader = paddle.batch(random_reader, batch_size=BATCH_SIZE) ps = places if use_double_buffer else fluid.cpu_places(len(places)) py_reader.decorate_sample_list_generator( - reader, places=ps if py_reader.iterable else None) + reader, places=ps if py_reader.iterable else None + ) exe = fluid.Executor(place=places[0]) exe.run(startup_prog) prog = fluid.CompiledProgram(main_prog) if with_data_parallel: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step = 0 step_list = [] @@ -102,9 +114,11 @@ class TestBase(unittest.TestCase): py_reader.start() while True: try: - L, = exe.run(program=prog, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 except fluid.core.EOFException: @@ -123,10 +137,12 @@ class TestBase(unittest.TestCase): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(ps[i]) assert label._place()._equals(ps[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -134,7 +150,7 @@ class TestBase(unittest.TestCase): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } return ret @@ -163,11 +179,13 @@ class TestBase(unittest.TestCase): use_legacy_py_reader=use_legacy_py_reader, with_data_parallel=with_data_parallel, places=p, - use_double_buffer=use_double_buffer) + use_double_buffer=use_double_buffer, + ) results.append(ret) if not use_double_buffer: diff = np.max( - np.abs(results[0]['loss'] - results[1]['loss'])) + np.abs(results[0]['loss'] - results[1]['loss']) + ) self.assertLess(diff, 1e-3) diff --git a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py index 0d271981276..107792a892d 100644 --- a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py +++ b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py @@ -19,7 +19,6 @@ import unittest class TestClass(unittest.TestCase): - def setUp(self): self.use_double_buffer = True self.use_py_reader = True @@ -34,7 +33,8 @@ class TestClass(unittest.TestCase): for _ in range(batch_size * batch_num): img = np.random.random(size=img_shape).astype('float32') label = np.random.random_integers( - low=0, high=9, size=label_shape).astype('int64') + low=0, high=9, size=label_shape + ).astype('int64') yield img, label reader = paddle.reader.cache(fake_reader) @@ -48,18 +48,20 @@ class TestClass(unittest.TestCase): main_prog = fluid.Program() startup_prog = fluid.Program() with fluid.program_guard(main_prog, startup_prog): - img = fluid.layers.data(shape=img_shape, - dtype='float32', - name='image') - label = fluid.layers.data(shape=label_shape, - dtype='int64', - name='label') + img = fluid.layers.data( + shape=img_shape, dtype='float32', name='image' + ) + label = fluid.layers.data( + shape=label_shape, dtype='int64', name='label' + ) feeder = fluid.DataFeeder(feed_list=[img, label], place=p) use_double_buffer = self.use_double_buffer - if p._type() != fluid.CPUPlace()._type( - ) and not use_double_buffer: + if ( + p._type() != fluid.CPUPlace()._type() + and not use_double_buffer + ): use_double_buffer = True if self.use_py_reader: @@ -67,15 +69,17 @@ class TestClass(unittest.TestCase): feed_list=[img, label], capacity=4, iterable=True, - use_double_buffer=use_double_buffer) - py_reader.decorate_sample_list_generator(batch_reader, - places=p) + use_double_buffer=use_double_buffer, + ) + py_reader.decorate_sample_list_generator( + batch_reader, places=p + ) else: py_reader = fluid.io.DataLoader.from_generator( feed_list=[img, label], capacity=4, iterable=True, - use_double_buffer=use_double_buffer + use_double_buffer=use_double_buffer, ).set_sample_list_generator(batch_reader, places=p) for break_beforehand in [True, False]: @@ -97,7 +101,8 @@ class TestClass(unittest.TestCase): batch_id += 1 if break_beforehand and batch_id >= int( - batch_num / 2): + batch_num / 2 + ): break if break_beforehand: @@ -107,21 +112,18 @@ class TestClass(unittest.TestCase): class TestClass2(TestClass): - def setUp(self): self.use_double_buffer = False self.use_py_reader = True class TestClass3(TestClass): - def setUp(self): self.use_double_buffer = True self.use_py_reader = False class TestClass4(TestClass): - def setUp(self): self.use_double_buffer = False self.use_py_reader = False diff --git a/python/paddle/fluid/tests/unittests/test_default_dtype.py b/python/paddle/fluid/tests/unittests/test_default_dtype.py index 70c23d62073..ca95c820a71 100644 --- a/python/paddle/fluid/tests/unittests/test_default_dtype.py +++ b/python/paddle/fluid/tests/unittests/test_default_dtype.py @@ -18,7 +18,6 @@ from paddle.framework import set_default_dtype, get_default_dtype class TestDefaultType(unittest.TestCase): - def check_default(self): self.assertEqual("float32", get_default_dtype()) @@ -45,7 +44,6 @@ class TestDefaultType(unittest.TestCase): class TestRaiseError(unittest.TestCase): - def test_error(self): self.assertRaises(TypeError, set_default_dtype, "int32") self.assertRaises(TypeError, set_default_dtype, np.int32) diff --git a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py index beae3124ced..88686585853 100644 --- a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py +++ b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py @@ -12,12 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.fluid.default_scope_funcs import enter_local_scope, find_var, get_cur_scope, leave_local_scope, scoped_function, var +from paddle.fluid.default_scope_funcs import ( + enter_local_scope, + find_var, + get_cur_scope, + leave_local_scope, + scoped_function, + var, +) import unittest class TestDefaultScopeFuncs(unittest.TestCase): - def test_cur_scope(self): self.assertIsNotNone(get_cur_scope()) @@ -33,7 +39,6 @@ class TestDefaultScopeFuncs(unittest.TestCase): leave_local_scope() def test_var_get_int(self): - def __new_scope__(): i = var("var_i") self.assertFalse(i.is_int()) diff --git a/python/paddle/fluid/tests/unittests/test_deform_conv2d.py b/python/paddle/fluid/tests/unittests/test_deform_conv2d.py index 53eea9ac4ab..80bf06fa6db 100644 --- a/python/paddle/fluid/tests/unittests/test_deform_conv2d.py +++ b/python/paddle/fluid/tests/unittests/test_deform_conv2d.py @@ -40,45 +40,70 @@ class TestDeformConv2D(TestCase): np.random.seed(1) paddle.seed(1) if isinstance(self.kernel_size, int): - filter_shape = (self.kernel_size, ) * 2 + filter_shape = (self.kernel_size,) * 2 else: filter_shape = tuple(self.kernel_size) self.filter_shape = filter_shape self.weight = np.random.uniform( - -1, 1, (self.out_channels, self.in_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.out_channels, self.in_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - def out_size(in_size, pad_size, dilation_size, kernel_size, - stride_size): - return (in_size + 2 * pad_size - - (dilation_size * (kernel_size - 1) + 1)) / stride_size + 1 + def out_size( + in_size, pad_size, dilation_size, kernel_size, stride_size + ): + return ( + in_size + 2 * pad_size - (dilation_size * (kernel_size - 1) + 1) + ) / stride_size + 1 out_h = int( - out_size(self.spatial_shape[0], self.padding[0], self.dilation[0], - self.kernel_size[0], self.stride[0])) + out_size( + self.spatial_shape[0], + self.padding[0], + self.dilation[0], + self.kernel_size[0], + self.stride[0], + ) + ) out_w = int( - out_size(self.spatial_shape[1], self.padding[1], self.dilation[1], - self.kernel_size[1], self.stride[1])) + out_size( + self.spatial_shape[1], + self.padding[1], + self.dilation[1], + self.kernel_size[1], + self.stride[1], + ) + ) out_shape = (out_h, out_w) - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.offset_shape = (self.batch_size, self.deformable_groups * 2 * - filter_shape[0] * filter_shape[1]) + out_shape + self.offset_shape = ( + self.batch_size, + self.deformable_groups * 2 * filter_shape[0] * filter_shape[1], + ) + out_shape - self.mask_shape = (self.batch_size, self.deformable_groups * - filter_shape[0] * filter_shape[1]) + out_shape + self.mask_shape = ( + self.batch_size, + self.deformable_groups * filter_shape[0] * filter_shape[1], + ) + out_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) - self.offset = np.random.uniform(-1, 1, - self.offset_shape).astype(self.dtype) + self.offset = np.random.uniform(-1, 1, self.offset_shape).astype( + self.dtype + ) self.mask = np.random.uniform(-1, 1, self.mask_shape).astype(self.dtype) @@ -87,16 +112,34 @@ class TestDeformConv2D(TestCase): start = paddle.static.Program() paddle.enable_static() with paddle.static.program_guard(main, start): - x = paddle.static.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = paddle.static.data( + "input", (-1, self.in_channels, -1, -1), dtype=self.dtype + ) offset = paddle.static.data( - "offset", (-1, self.deformable_groups * 2 * - self.filter_shape[0] * self.filter_shape[1], -1, -1), - dtype=self.dtype) + "offset", + ( + -1, + self.deformable_groups + * 2 + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) mask = paddle.static.data( - "mask", (-1, self.deformable_groups * self.filter_shape[0] * - self.filter_shape[1], -1, -1), - dtype=self.dtype) + "mask", + ( + -1, + self.deformable_groups + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) y_v1 = paddle.fluid.layers.deformable_conv( input=x, @@ -112,7 +155,8 @@ class TestDeformConv2D(TestCase): im2col_step=1, param_attr=I.Assign(self.weight), bias_attr=False if self.no_bias else I.Assign(self.bias), - modulated=False) + modulated=False, + ) y_v2 = paddle.fluid.layers.deformable_conv( input=x, @@ -127,17 +171,20 @@ class TestDeformConv2D(TestCase): deformable_groups=self.deformable_groups, im2col_step=1, param_attr=I.Assign(self.weight), - bias_attr=False if self.no_bias else I.Assign(self.bias)) + bias_attr=False if self.no_bias else I.Assign(self.bias), + ) exe = paddle.static.Executor(self.place) exe.run(start) - out_v1, out_v2 = exe.run(main, - feed={ - "input": self.input, - "offset": self.offset, - "mask": self.mask - }, - fetch_list=[y_v1, y_v2]) + out_v1, out_v2 = exe.run( + main, + feed={ + "input": self.input, + "offset": self.offset, + "mask": self.mask, + }, + fetch_list=[y_v1, y_v2], + ) return out_v1, out_v2 def dygraph_case_dcn(self): @@ -158,7 +205,8 @@ class TestDeformConv2D(TestCase): deformable_groups=self.deformable_groups, groups=self.groups, weight_attr=I.Assign(self.weight), - bias_attr=False if self.no_bias else I.Assign(self.bias)) + bias_attr=False if self.no_bias else I.Assign(self.bias), + ) y_v1 = deform_conv2d(x, offset) y_v2 = deform_conv2d(x, offset, mask) @@ -208,45 +256,70 @@ class TestDeformConv2DFunctional(TestCase): np.random.seed(1) paddle.seed(1) if isinstance(self.kernel_size, int): - filter_shape = (self.kernel_size, ) * 2 + filter_shape = (self.kernel_size,) * 2 else: filter_shape = tuple(self.kernel_size) self.filter_shape = filter_shape self.weight = np.random.uniform( - -1, 1, (self.out_channels, self.in_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.out_channels, self.in_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - def out_size(in_size, pad_size, dilation_size, kernel_size, - stride_size): - return (in_size + 2 * pad_size - - (dilation_size * (kernel_size - 1) + 1)) / stride_size + 1 + def out_size( + in_size, pad_size, dilation_size, kernel_size, stride_size + ): + return ( + in_size + 2 * pad_size - (dilation_size * (kernel_size - 1) + 1) + ) / stride_size + 1 out_h = int( - out_size(self.spatial_shape[0], self.padding[0], self.dilation[0], - self.kernel_size[0], self.stride[0])) + out_size( + self.spatial_shape[0], + self.padding[0], + self.dilation[0], + self.kernel_size[0], + self.stride[0], + ) + ) out_w = int( - out_size(self.spatial_shape[1], self.padding[1], self.dilation[1], - self.kernel_size[1], self.stride[1])) + out_size( + self.spatial_shape[1], + self.padding[1], + self.dilation[1], + self.kernel_size[1], + self.stride[1], + ) + ) out_shape = (out_h, out_w) - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.offset_shape = (self.batch_size, self.deformable_groups * 2 * - filter_shape[0] * filter_shape[1]) + out_shape + self.offset_shape = ( + self.batch_size, + self.deformable_groups * 2 * filter_shape[0] * filter_shape[1], + ) + out_shape - self.mask_shape = (self.batch_size, self.deformable_groups * - filter_shape[0] * filter_shape[1]) + out_shape + self.mask_shape = ( + self.batch_size, + self.deformable_groups * filter_shape[0] * filter_shape[1], + ) + out_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) - self.offset = np.random.uniform(-1, 1, - self.offset_shape).astype(self.dtype) + self.offset = np.random.uniform(-1, 1, self.offset_shape).astype( + self.dtype + ) self.mask = np.random.uniform(-1, 1, self.mask_shape).astype(self.dtype) @@ -255,16 +328,34 @@ class TestDeformConv2DFunctional(TestCase): start = paddle.static.Program() paddle.enable_static() with paddle.static.program_guard(main, start): - x = paddle.static.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = paddle.static.data( + "input", (-1, self.in_channels, -1, -1), dtype=self.dtype + ) offset = paddle.static.data( - "offset", (-1, self.deformable_groups * 2 * - self.filter_shape[0] * self.filter_shape[1], -1, -1), - dtype=self.dtype) + "offset", + ( + -1, + self.deformable_groups + * 2 + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) mask = paddle.static.data( - "mask", (-1, self.deformable_groups * self.filter_shape[0] * - self.filter_shape[1], -1, -1), - dtype=self.dtype) + "mask", + ( + -1, + self.deformable_groups + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) y_v1 = paddle.fluid.layers.deformable_conv( input=x, @@ -280,7 +371,8 @@ class TestDeformConv2DFunctional(TestCase): im2col_step=1, param_attr=I.Assign(self.weight), bias_attr=False if self.no_bias else I.Assign(self.bias), - modulated=False) + modulated=False, + ) y_v2 = paddle.fluid.layers.deformable_conv( input=x, @@ -295,17 +387,20 @@ class TestDeformConv2DFunctional(TestCase): deformable_groups=self.deformable_groups, im2col_step=1, param_attr=I.Assign(self.weight), - bias_attr=False if self.no_bias else I.Assign(self.bias)) + bias_attr=False if self.no_bias else I.Assign(self.bias), + ) exe = paddle.static.Executor(self.place) exe.run(start) - out_v1, out_v2 = exe.run(main, - feed={ - "input": self.input, - "offset": self.offset, - "mask": self.mask - }, - fetch_list=[y_v1, y_v2]) + out_v1, out_v2 = exe.run( + main, + feed={ + "input": self.input, + "offset": self.offset, + "mask": self.mask, + }, + fetch_list=[y_v1, y_v2], + ) return out_v1, out_v2 def dygraph_case_dcn(self): @@ -351,20 +446,38 @@ class TestDeformConv2DFunctional(TestCase): start = paddle.static.Program() paddle.enable_static() with paddle.static.program_guard(main, start): - x = paddle.static.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = paddle.static.data( + "input", (-1, self.in_channels, -1, -1), dtype=self.dtype + ) offset = paddle.static.data( - "offset", (-1, self.deformable_groups * 2 * - self.filter_shape[0] * self.filter_shape[1], -1, -1), - dtype=self.dtype) + "offset", + ( + -1, + self.deformable_groups + * 2 + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) mask = paddle.static.data( - "mask", (-1, self.deformable_groups * self.filter_shape[0] * - self.filter_shape[1], -1, -1), - dtype=self.dtype) + "mask", + ( + -1, + self.deformable_groups + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) - weight = paddle.static.data("weight", - list(self.weight.shape), - dtype=self.dtype) + weight = paddle.static.data( + "weight", list(self.weight.shape), dtype=self.dtype + ) if not self.no_bias: bias = paddle.static.data("bias", [-1], dtype=self.dtype) @@ -400,7 +513,7 @@ class TestDeformConv2DFunctional(TestCase): "input": self.input, "offset": self.offset, "mask": self.mask, - "weight": self.weight + "weight": self.weight, } if not self.no_bias: feed_dict["bias"] = self.bias @@ -412,8 +525,10 @@ class TestDeformConv2DFunctional(TestCase): self.prepare() static_dcn_v1, static_dcn_v2 = self.static_graph_case_dcn() dy_dcn_v1, dy_dcn_v2 = self.dygraph_case_dcn() - new_static_dcn_v1, new_static_dcn_v2 = self.new_api_static_graph_case_dcn( - ) + ( + new_static_dcn_v1, + new_static_dcn_v2, + ) = self.new_api_static_graph_case_dcn() np.testing.assert_array_almost_equal(static_dcn_v1, dy_dcn_v1) np.testing.assert_array_almost_equal(static_dcn_v2, dy_dcn_v2) np.testing.assert_array_almost_equal(static_dcn_v1, new_static_dcn_v1) @@ -434,7 +549,6 @@ class TestDeformConv2DFunctional(TestCase): # testcases for DeformConv2D class TestDeformConv2DWithPadding(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -448,7 +562,6 @@ class TestDeformConv2DWithPadding(TestDeformConv2D): class TestDeformConv2DWithBias(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -462,7 +575,6 @@ class TestDeformConv2DWithBias(TestDeformConv2D): class TestDeformConv2DWithAsynPadding(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -476,7 +588,6 @@ class TestDeformConv2DWithAsynPadding(TestDeformConv2D): class TestDeformConv2DWithDilation(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -490,7 +601,6 @@ class TestDeformConv2DWithDilation(TestDeformConv2D): class TestDeformConv2DWithStride(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -504,7 +614,6 @@ class TestDeformConv2DWithStride(TestDeformConv2D): class TestDeformConv2DWithDeformable_Groups(TestDeformConv2D): - def setUp(self): self.in_channels = 5 self.out_channels = 5 @@ -518,7 +627,6 @@ class TestDeformConv2DWithDeformable_Groups(TestDeformConv2D): class TestDeformConv2DWithGroups(TestDeformConv2D): - def setUp(self): self.in_channels = 5 self.out_channels = 5 @@ -533,7 +641,6 @@ class TestDeformConv2DWithGroups(TestDeformConv2D): # testcases for deform_conv2d class TestDeformConv2DFunctionalWithPadding(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -547,7 +654,6 @@ class TestDeformConv2DFunctionalWithPadding(TestDeformConv2DFunctional): class TestDeformConv2DFunctionalWithBias(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -561,7 +667,6 @@ class TestDeformConv2DFunctionalWithBias(TestDeformConv2DFunctional): class TestDeformConv2DFunctionalWithAsynPadding(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -575,7 +680,6 @@ class TestDeformConv2DFunctionalWithAsynPadding(TestDeformConv2DFunctional): class TestDeformConv2DFunctionalWithDilation(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -589,7 +693,6 @@ class TestDeformConv2DFunctionalWithDilation(TestDeformConv2DFunctional): class TestDeformConv2DFunctionalWithStride(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -602,9 +705,9 @@ class TestDeformConv2DFunctionalWithStride(TestDeformConv2DFunctional): self.no_bias = False -class TestDeformConv2DFunctionalWithDeformable_Groups(TestDeformConv2DFunctional - ): - +class TestDeformConv2DFunctionalWithDeformable_Groups( + TestDeformConv2DFunctional +): def setUp(self): self.in_channels = 5 self.out_channels = 5 @@ -618,7 +721,6 @@ class TestDeformConv2DFunctionalWithDeformable_Groups(TestDeformConv2DFunctional class TestDeformConv2DFunctionalWithGroups(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 5 self.out_channels = 5 diff --git a/python/paddle/fluid/tests/unittests/test_deformable_conv_op.py b/python/paddle/fluid/tests/unittests/test_deformable_conv_op.py index 6c0d7360e58..f89ef98e950 100644 --- a/python/paddle/fluid/tests/unittests/test_deformable_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_deformable_conv_op.py @@ -61,8 +61,11 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): assert f_c * group == in_c assert np.mod(out_c, group) == 0 - stride, pad, dilation = conv_param['stride'], conv_param['pad'],\ - conv_param['dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) // stride[0] out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) // stride[1] assert out_h == in_h @@ -75,31 +78,47 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): for w in range(out_w): for kh in range(f_h): for kw in range(f_w): - offset_h_table = \ - offset[n, ::2, h, w].reshape(f_h, f_w) - offset_w_table = \ - offset[n, 1::2, h, w].reshape(f_h, f_w) - mask_table = \ - mask[n, :, h, w].reshape(f_h, f_w) + offset_h_table = offset[n, ::2, h, w].reshape( + f_h, f_w + ) + offset_w_table = offset[n, 1::2, h, w].reshape( + f_h, f_w + ) + mask_table = mask[n, :, h, w].reshape(f_h, f_w) offset_h = offset_h_table[kh, kw] offset_w = offset_w_table[kh, kw] val = 0 - im_h = h * stride[0] + kh * dilation[0] \ - + offset_h - pad[0] - im_w = w * stride[0] + kw * dilation[0] \ - + offset_w - pad[1] - if im_h > -1 and im_w > -1 and \ - im_h < in_h and im_w < in_h: - val = dmc_bilinear(input[n, c], in_h, in_w, - im_h, im_w) + im_h = ( + h * stride[0] + + kh * dilation[0] + + offset_h + - pad[0] + ) + im_w = ( + w * stride[0] + + kw * dilation[0] + + offset_w + - pad[1] + ) + if ( + im_h > -1 + and im_w > -1 + and im_h < in_h + and im_w < in_h + ): + val = dmc_bilinear( + input[n, c], in_h, in_w, im_h, im_w + ) val_out = val * mask_table[kh, kw] - col_buffer[n, c * f_h * f_w + kh * f_w + kw, - h * in_w + w] = val_out + col_buffer[ + n, c * f_h * f_w + kh * f_w + kw, h * in_w + w + ] = val_out out = np.zeros((in_n, group, int(out_c // group), out_h * out_w)) weight = filter.reshape(group, int(out_c // group), f_c * f_h * f_w) col_buffer = col_buffer.reshape( - (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w)) + (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w) + ) for n in range(in_n): for g in range(group): out[n, g] = np.matmul(weight[g], col_buffer[n, g]) @@ -107,23 +126,33 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): return out -def deform_conv2d_wrapper(x, - offset, - weight, - mask=None, - stride=1, - padding=0, - dilation=1, - deformable_groups=1, - groups=1, - im2col_step=1): - return paddle.vision.ops.deform_conv2d(x, offset, weight, None, stride, - padding, dilation, deformable_groups, - groups, mask) +def deform_conv2d_wrapper( + x, + offset, + weight, + mask=None, + stride=1, + padding=0, + dilation=1, + deformable_groups=1, + groups=1, + im2col_step=1, +): + return paddle.vision.ops.deform_conv2d( + x, + offset, + weight, + None, + stride, + padding, + dilation, + deformable_groups, + groups, + mask, + ) class TestModulatedDeformableConvOp(OpTest): - def setUp(self): self.python_api = deform_conv2d_wrapper self.op_type = "deformable_conv" @@ -135,7 +164,7 @@ class TestModulatedDeformableConvOp(OpTest): conv_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -143,15 +172,16 @@ class TestModulatedDeformableConvOp(OpTest): mask = 10 * np.random.random(self.mask_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = dconv_im2col_gemm(input, offset, mask, filter, self.groups, - conv_param) + output = dconv_im2col_gemm( + input, offset, mask, filter, self.groups, conv_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Offset': OpTest.np_dtype_to_fluid_dtype(offset), 'Mask': OpTest.np_dtype_to_fluid_dtype(mask), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -167,10 +197,12 @@ class TestModulatedDeformableConvOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad({'Input', 'Offset', 'Mask', 'Filter'}, - 'Output', - max_relative_error=0.05, - check_eager=True) + self.check_grad( + {'Input', 'Offset', 'Mask', 'Filter'}, + 'Output', + max_relative_error=0.05, + check_eager=True, + ) def init_test_case(self): self.pad = [1, 1] @@ -182,15 +214,26 @@ class TestModulatedDeformableConvOp(OpTest): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -204,7 +247,6 @@ class TestModulatedDeformableConvOp(OpTest): class TestWithStride(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [3, 3] self.stride = [2, 2] @@ -214,20 +256,30 @@ class TestWithStride(TestModulatedDeformableConvOp): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestWithDilation(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [1, 1] @@ -237,15 +289,26 @@ class TestWithDilation(TestModulatedDeformableConvOp): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -253,7 +316,6 @@ class TestWithDilation(TestModulatedDeformableConvOp): class TestWith3x3(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -263,26 +325,35 @@ class TestWith3x3(TestModulatedDeformableConvOp): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestWithGroup(TestModulatedDeformableConvOp): - def init_group(self): self.groups = 2 class TestWithDouble(TestModulatedDeformableConvOp): - def init_type(self): self.dtype = np.float64 @@ -296,74 +367,77 @@ class TestWithDouble(TestModulatedDeformableConvOp): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestModulatedDeformableConvInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): paddle.enable_static() input = [1, 3, 32, 32] - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=1) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=1 + ) self.assertRaises(TypeError, test_invalid_input) def test_invalid_offset(): paddle.enable_static() - input = fluid.data(name='input', - shape=[None, 3, 32, 32], - dtype='int32') - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=1) + input = fluid.data( + name='input', shape=[None, 3, 32, 32], dtype='int32' + ) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=1 + ) self.assertRaises(TypeError, test_invalid_offset) def test_invalid_filter(): paddle.enable_static() - input = fluid.data(name='input_filter', - shape=[None, 3, 32, 32], - dtype='float32') - offset = fluid.data(name='offset_filter', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask_filter', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=0) + input = fluid.data( + name='input_filter', shape=[None, 3, 32, 32], dtype='float32' + ) + offset = fluid.data( + name='offset_filter', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask_filter', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=0 + ) self.assertRaises(ValueError, test_invalid_filter) @@ -373,45 +447,39 @@ class TestModulatedDeformableConvInvalidInput(unittest.TestCase): class TestDeformConv2DAPI(unittest.TestCase): - def test_api(self): - def test_deform_conv2d_v1(): paddle.enable_static() - input = paddle.static.data(name='input_v1', - shape=[None, 3, 32, 32], - dtype='float32') - offset = paddle.static.data(name='offset_v1', - shape=[None, 4, 32, 32], - dtype='float32') - out = paddle.static.nn.deform_conv2d(input, - offset, - None, - num_filters=4, - filter_size=1) - - assert (out.shape == (-1, 4, 32, 32)) + input = paddle.static.data( + name='input_v1', shape=[None, 3, 32, 32], dtype='float32' + ) + offset = paddle.static.data( + name='offset_v1', shape=[None, 4, 32, 32], dtype='float32' + ) + out = paddle.static.nn.deform_conv2d( + input, offset, None, num_filters=4, filter_size=1 + ) + + assert out.shape == (-1, 4, 32, 32) test_deform_conv2d_v1() def test_deform_conv2d_v2(): paddle.enable_static() - input = paddle.static.data(name='input_v2', - shape=[None, 3, 32, 32], - dtype='float32') - offset = paddle.static.data(name='offset_v2', - shape=[None, 4, 32, 32], - dtype='float32') - mask = paddle.static.data(name='mask_v2', - shape=[None, 2, 32, 32], - dtype='float32') - out = paddle.static.nn.deform_conv2d(input, - offset, - mask, - num_filters=4, - filter_size=1) - - assert (out.shape == (-1, 4, 32, 32)) + input = paddle.static.data( + name='input_v2', shape=[None, 3, 32, 32], dtype='float32' + ) + offset = paddle.static.data( + name='offset_v2', shape=[None, 4, 32, 32], dtype='float32' + ) + mask = paddle.static.data( + name='mask_v2', shape=[None, 2, 32, 32], dtype='float32' + ) + out = paddle.static.nn.deform_conv2d( + input, offset, mask, num_filters=4, filter_size=1 + ) + + assert out.shape == (-1, 4, 32, 32) test_deform_conv2d_v2() diff --git a/python/paddle/fluid/tests/unittests/test_deformable_conv_v1_op.py b/python/paddle/fluid/tests/unittests/test_deformable_conv_v1_op.py index eacf6dba273..19a81e21a6a 100644 --- a/python/paddle/fluid/tests/unittests/test_deformable_conv_v1_op.py +++ b/python/paddle/fluid/tests/unittests/test_deformable_conv_v1_op.py @@ -58,8 +58,11 @@ def dconv_im2col_gemm(input, offset, filter, group, conv_param): assert f_c * group == in_c assert np.mod(out_c, group) == 0 - stride, pad, dilation = conv_param['stride'], conv_param['pad'],\ - conv_param['dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) // stride[0] out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) // stride[1] assert out_h == in_h @@ -72,30 +75,47 @@ def dconv_im2col_gemm(input, offset, filter, group, conv_param): for w in range(out_w): for kh in range(f_h): for kw in range(f_w): - offset_h_table = \ - offset[n, ::2, h, w].reshape(f_h, f_w) - offset_w_table = \ - offset[n, 1::2, h, w].reshape(f_h, f_w) + offset_h_table = offset[n, ::2, h, w].reshape( + f_h, f_w + ) + offset_w_table = offset[n, 1::2, h, w].reshape( + f_h, f_w + ) offset_h = offset_h_table[kh, kw] offset_w = offset_w_table[kh, kw] val = 0 - im_h = h * stride[0] + kh * dilation[0] \ - + offset_h - pad[0] - im_w = w * stride[0] + kw * dilation[0] \ - + offset_w - pad[1] - if im_h > -1 and im_w > -1 and \ - im_h < in_h and im_w < in_h: - val = dmc_bilinear(input[n, c], in_h, in_w, - im_h, im_w) + im_h = ( + h * stride[0] + + kh * dilation[0] + + offset_h + - pad[0] + ) + im_w = ( + w * stride[0] + + kw * dilation[0] + + offset_w + - pad[1] + ) + if ( + im_h > -1 + and im_w > -1 + and im_h < in_h + and im_w < in_h + ): + val = dmc_bilinear( + input[n, c], in_h, in_w, im_h, im_w + ) val_out = val - col_buffer[n, c * f_h * f_w + kh * f_w + kw, - h * in_w + w] = val_out + col_buffer[ + n, c * f_h * f_w + kh * f_w + kw, h * in_w + w + ] = val_out out = np.zeros((in_n, group, int(out_c // group), out_h * out_w)) weight = filter.reshape(group, int(out_c // group), f_c * f_h * f_w) col_buffer = col_buffer.reshape( - (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w)) + (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w) + ) for n in range(in_n): for g in range(group): out[n, g] = np.matmul(weight[g], col_buffer[n, g]) @@ -103,23 +123,33 @@ def dconv_im2col_gemm(input, offset, filter, group, conv_param): return out -def deform_conv2d_wrapper(x, - offset, - weight, - mask=None, - stride=1, - padding=0, - dilation=1, - deformable_groups=1, - groups=1, - im2col_step=1): - return paddle.vision.ops.deform_conv2d(x, offset, weight, None, stride, - padding, dilation, deformable_groups, - groups, mask) +def deform_conv2d_wrapper( + x, + offset, + weight, + mask=None, + stride=1, + padding=0, + dilation=1, + deformable_groups=1, + groups=1, + im2col_step=1, +): + return paddle.vision.ops.deform_conv2d( + x, + offset, + weight, + None, + stride, + padding, + dilation, + deformable_groups, + groups, + mask, + ) class TestModulatedDeformableConvOp(OpTest): - def setUp(self): self.python_api = deform_conv2d_wrapper self.op_type = "deformable_conv_v1" @@ -131,20 +161,21 @@ class TestModulatedDeformableConvOp(OpTest): conv_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) offset = 10 * np.random.random(self.offset_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = dconv_im2col_gemm(input, offset, filter, self.groups, - conv_param) + output = dconv_im2col_gemm( + input, offset, filter, self.groups, conv_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Offset': OpTest.np_dtype_to_fluid_dtype(offset), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -160,17 +191,21 @@ class TestModulatedDeformableConvOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['Input', 'Offset', 'Filter'], - 'Output', - max_relative_error=0.05, - check_eager=True) + self.check_grad( + ['Input', 'Offset', 'Filter'], + 'Output', + max_relative_error=0.05, + check_eager=True, + ) def test_check_grad_no_filter(self): - self.check_grad(['Input', 'Offset'], - 'Output', - max_relative_error=0.1, - no_grad_set=set(['Filter']), - check_eager=True) + self.check_grad( + ['Input', 'Offset'], + 'Output', + max_relative_error=0.1, + no_grad_set=set(['Filter']), + check_eager=True, + ) def init_test_case(self): self.pad = [1, 1] @@ -182,10 +217,17 @@ class TestModulatedDeformableConvOp(OpTest): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -199,7 +241,6 @@ class TestModulatedDeformableConvOp(OpTest): class TestWithStride(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [3, 3] self.stride = [2, 2] @@ -209,15 +250,21 @@ class TestWithStride(TestModulatedDeformableConvOp): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] class TestWithDilation(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [1, 1] @@ -227,10 +274,17 @@ class TestWithDilation(TestModulatedDeformableConvOp): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -238,7 +292,6 @@ class TestWithDilation(TestModulatedDeformableConvOp): class TestWith1x1(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -248,15 +301,21 @@ class TestWith1x1(TestModulatedDeformableConvOp): self.filter_size = [40, f_c, 1, 1] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] class TestWithGroup(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -267,10 +326,17 @@ class TestWithGroup(TestModulatedDeformableConvOp): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] def init_group(self): @@ -278,42 +344,43 @@ class TestWithGroup(TestModulatedDeformableConvOp): class TestWithDouble(TestModulatedDeformableConvOp): - def init_type(self): self.dtype = np.float64 class TestModulatedDeformableConvV1InvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): input = [1, 3, 32, 32] - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask=None, - num_filters=4, - filter_size=1, - modulated=False) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, + offset, + mask=None, + num_filters=4, + filter_size=1, + modulated=False, + ) self.assertRaises(TypeError, test_invalid_input) def test_invalid_offset(): - input = fluid.data(name='input', - shape=[None, 3, 32, 32], - dtype='int32') - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask=None, - num_filters=4, - filter_size=1, - modulated=False) + input = fluid.data( + name='input', shape=[None, 3, 32, 32], dtype='int32' + ) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, + offset, + mask=None, + num_filters=4, + filter_size=1, + modulated=False, + ) self.assertRaises(TypeError, test_invalid_offset) diff --git a/python/paddle/fluid/tests/unittests/test_deformable_psroi_pooling.py b/python/paddle/fluid/tests/unittests/test_deformable_psroi_pooling.py index 4bf8a99137b..8dbe2dfa322 100644 --- a/python/paddle/fluid/tests/unittests/test_deformable_psroi_pooling.py +++ b/python/paddle/fluid/tests/unittests/test_deformable_psroi_pooling.py @@ -24,9 +24,17 @@ def set_input(input, rois, trans): return inputs -def set_attrs(no_trans, spatial_scale, output_channels, group_size, - pooled_height, pooled_width, part_size, sample_per_part, - trans_std): +def set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, +): attrs = { 'no_trans': no_trans, 'spatial_scale': spatial_scale, @@ -36,7 +44,7 @@ def set_attrs(no_trans, spatial_scale, output_channels, group_size, 'pooled_width': pooled_width, 'part_size': part_size, 'sample_per_part': sample_per_part, - 'trans_std': trans_std + 'trans_std': trans_std, } return attrs @@ -44,13 +52,12 @@ def set_attrs(no_trans, spatial_scale, output_channels, group_size, def set_outputs(output, top_count): outputs = { 'Output': output.astype('float32'), - 'TopCount': top_count.astype('float32') + 'TopCount': top_count.astype('float32'), } return outputs class TestDeformablePSROIPoolOp(OpTest): - def set_data(self): self.start_test1() self.start_test2() @@ -77,9 +84,17 @@ class TestDeformablePSROIPoolOp(OpTest): sample_per_part = self.sample_per_part trans_std = self.trans_std - self.attrs = set_attrs(no_trans, spatial_scale, output_channels, - group_size, pooled_height, pooled_width, - part_size, sample_per_part, trans_std) + self.attrs = set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, + ) output = self.out.astype('float32') top_count = self.top_count.astype('float32') @@ -105,9 +120,17 @@ class TestDeformablePSROIPoolOp(OpTest): sample_per_part = self.sample_per_part trans_std = self.trans_std - self.attrs = set_attrs(no_trans, spatial_scale, output_channels, - group_size, pooled_height, pooled_width, - part_size, sample_per_part, trans_std) + self.attrs = set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, + ) output = self.out.astype('float32') top_count = self.top_count.astype('float32') @@ -133,9 +156,17 @@ class TestDeformablePSROIPoolOp(OpTest): sample_per_part = self.sample_per_part trans_std = self.trans_std - self.attrs = set_attrs(no_trans, spatial_scale, output_channels, - group_size, pooled_height, pooled_width, - part_size, sample_per_part, trans_std) + self.attrs = set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, + ) output = self.out.astype('float32') top_count = self.top_count.astype('float32') @@ -161,9 +192,17 @@ class TestDeformablePSROIPoolOp(OpTest): sample_per_part = self.sample_per_part trans_std = self.trans_std - self.attrs = set_attrs(no_trans, spatial_scale, output_channels, - group_size, pooled_height, pooled_width, - part_size, sample_per_part, trans_std) + self.attrs = set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, + ) output = self.out.astype('float32') top_count = self.top_count.astype('float32') @@ -175,7 +214,10 @@ class TestDeformablePSROIPoolOp(OpTest): self.height = 12 self.width = 12 self.input_dim = [ - self.batch_size, self.channels, self.height, self.width + self.batch_size, + self.channels, + self.height, + self.width, ] self.no_trans = False self.spatial_scale = 1.0 / 4.0 @@ -194,7 +236,10 @@ class TestDeformablePSROIPoolOp(OpTest): self.height = 12 self.width = 12 self.input_dim = [ - self.batch_size, self.channels, self.height, self.width + self.batch_size, + self.channels, + self.height, + self.width, ] self.no_trans = True self.spatial_scale = 1.0 / 2.0 @@ -213,7 +258,10 @@ class TestDeformablePSROIPoolOp(OpTest): self.height = 12 self.width = 12 self.input_dim = [ - self.batch_size, self.channels, self.height, self.width + self.batch_size, + self.channels, + self.height, + self.width, ] self.no_trans = False self.spatial_scale = 1.0 / 4.0 @@ -232,7 +280,10 @@ class TestDeformablePSROIPoolOp(OpTest): self.height = 12 self.width = 12 self.input_dim = [ - self.batch_size, self.channels, self.height, self.width + self.batch_size, + self.channels, + self.height, + self.width, ] self.no_trans = True self.spatial_scale = 1.0 / 2.0 @@ -252,13 +303,17 @@ class TestDeformablePSROIPoolOp(OpTest): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x_1 = np.random.randint( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y_1 = np.random.randint( - 0, self.height // self.spatial_scale - self.pooled_height) - x_2 = np.random.randint(x_1 + self.pooled_width, - self.width // self.spatial_scale) - y_2 = np.random.randint(y_1 + self.pooled_height, - self.height // self.spatial_scale) + 0, self.height // self.spatial_scale - self.pooled_height + ) + x_2 = np.random.randint( + x_1 + self.pooled_width, self.width // self.spatial_scale + ) + y_2 = np.random.randint( + y_1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x_1, y_1, x_2, y_2] rois.append(roi) self.rois_num = len(rois) @@ -290,20 +345,38 @@ class TestDeformablePSROIPoolOp(OpTest): return val def calc_deformable_psroi_pooling(self): - output_shape = (self.rois_num, self.output_channels, self.pooled_height, - self.pooled_width) + output_shape = ( + self.rois_num, + self.output_channels, + self.pooled_height, + self.pooled_width, + ) self.out = np.zeros(output_shape) - self.trans = np.random.rand(self.rois_num, 2, self.part_size[0], - self.part_size[1]).astype('float32') + self.trans = np.random.rand( + self.rois_num, 2, self.part_size[0], self.part_size[1] + ).astype('float32') self.top_count = np.random.random((output_shape)).astype('float32') - count = self.rois_num * self.output_channels * self.pooled_height * self.pooled_width + count = ( + self.rois_num + * self.output_channels + * self.pooled_height + * self.pooled_width + ) for index in range(count): p_w = int(index % self.pooled_width) p_h = int(index / self.pooled_width % self.pooled_height) - ctop = int(index / self.pooled_width / self.pooled_height % - self.output_channels) - n_out = int(index / self.pooled_width / self.pooled_height / - self.output_channels) + ctop = int( + index + / self.pooled_width + / self.pooled_height + % self.output_channels + ) + n_out = int( + index + / self.pooled_width + / self.pooled_height + / self.output_channels + ) roi = self.rois[n_out] roi_batch_id = int(roi[0]) roi_start_w = int(np.round(roi[1])) * self.spatial_scale - 0.5 @@ -339,15 +412,22 @@ class TestDeformablePSROIPoolOp(OpTest): for i_h in range(self.sample_per_part): w_sample = wstart + i_w * sub_bin_size_w h_sample = hstart + i_h * sub_bin_size_h - if w_sample < -0.5 or w_sample > self.width - 0.5 or \ - h_sample < -0.5 or h_sample > self.height - 0.5: + if ( + w_sample < -0.5 + or w_sample > self.width - 0.5 + or h_sample < -0.5 + or h_sample > self.height - 0.5 + ): continue - w_sample = min(max(w_sample, 0.), self.width - 1.) - h_sample = min(max(h_sample, 0.), self.height - 1.) - c_sample = int((ctop * self.group_size[0] + g_h) * - self.group_size[1] + g_w) - val = self.dmc_bilinear(input_i[c_sample], h_sample, - w_sample) + w_sample = min(max(w_sample, 0.0), self.width - 1.0) + h_sample = min(max(h_sample, 0.0), self.height - 1.0) + c_sample = int( + (ctop * self.group_size[0] + g_h) * self.group_size[1] + + g_w + ) + val = self.dmc_bilinear( + input_i[c_sample], h_sample, w_sample + ) sum = sum + val num_sample = num_sample + 1 if num_sample == 0: @@ -368,131 +448,144 @@ class TestDeformablePSROIPoolOp(OpTest): class TestDeformablePSROIPoolOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - input1 = fluid.data(name="input1", - shape=[2, 192, 64, 64], - dtype='float32') - rois1 = fluid.data(name="rois1", - shape=[-1, 4], - dtype='float32', - lod_level=1) - trans1 = fluid.data(name="trans1", - shape=[2, 384, 64, 64], - dtype='float32') + input1 = fluid.data( + name="input1", shape=[2, 192, 64, 64], dtype='float32' + ) + rois1 = fluid.data( + name="rois1", shape=[-1, 4], dtype='float32', lod_level=1 + ) + trans1 = fluid.data( + name="trans1", shape=[2, 384, 64, 64], dtype='float32' + ) # The `input` must be Variable and the data type of `input` Tensor must be one of float32 and float64. def test_input_type(): - fluid.layers.deformable_roi_pooling(input=[3, 4], - rois=rois1, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=[3, 4], + rois=rois1, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_input_type) def test_input_tensor_dtype(): - input2 = fluid.data(name="input2", - shape=[2, 192, 64, 64], - dtype='int32') - fluid.layers.deformable_roi_pooling(input=input2, - rois=rois1, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + input2 = fluid.data( + name="input2", shape=[2, 192, 64, 64], dtype='int32' + ) + fluid.layers.deformable_roi_pooling( + input=input2, + rois=rois1, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_input_tensor_dtype) # The `rois` must be Variable and the data type of `rois` Tensor must be one of float32 and float64. def test_rois_type(): - fluid.layers.deformable_roi_pooling(input=input1, - rois=2, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=2, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_rois_type) def test_rois_tensor_dtype(): - rois2 = fluid.data(name="rois2", - shape=[-1, 4], - dtype='int32', - lod_level=1) - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois2, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + rois2 = fluid.data( + name="rois2", shape=[-1, 4], dtype='int32', lod_level=1 + ) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois2, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_rois_tensor_dtype) # The `trans` must be Variable and the data type of `trans` Tensor must be one of float32 and float64. def test_trans_type(): - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois1, - trans=[2], - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois1, + trans=[2], + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_trans_type) def test_trans_tensor_dtype(): - trans2 = fluid.data(name="trans2", - shape=[2, 384, 64, 64], - dtype='int32') - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois1, - trans=trans2, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + trans2 = fluid.data( + name="trans2", shape=[2, 384, 64, 64], dtype='int32' + ) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois1, + trans=trans2, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_trans_tensor_dtype) # The `group_size` must be one of list and tuple. # Each element must be int. def test_group_size_type(): - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois1, - trans=trans1, - group_size=1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois1, + trans=trans1, + group_size=1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_group_size_type) # The `part_size` must be one of list, tuple and None. # Each element must be int. def test_part_size_type(): - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois1, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=8, - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois1, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=8, + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_part_size_type) diff --git a/python/paddle/fluid/tests/unittests/test_deg2rad.py b/python/paddle/fluid/tests/unittests/test_deg2rad.py index 4c9ec1becc4..79e4541b635 100644 --- a/python/paddle/fluid/tests/unittests/test_deg2rad.py +++ b/python/paddle/fluid/tests/unittests/test_deg2rad.py @@ -22,11 +22,11 @@ paddle.enable_static() class TestDeg2radAPI(unittest.TestCase): - def setUp(self): self.x_dtype = 'float64' - self.x_np = np.array([180.0, -180.0, 360.0, -360.0, 90.0, - -90.0]).astype(np.float64) + self.x_np = np.array( + [180.0, -180.0, 360.0, -360.0, 90.0, -90.0] + ).astype(np.float64) self.x_shape = [6] self.out_np = np.deg2rad(self.x_np) @@ -37,12 +37,17 @@ class TestDeg2radAPI(unittest.TestCase): x = fluid.data(name='input', dtype=self.x_dtype, shape=self.x_shape) out = paddle.deg2rad(x) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(fluid.default_main_program(), - feed={'input': self.x_np}, - fetch_list=[out]) + res = exe.run( + fluid.default_main_program(), + feed={'input': self.x_np}, + fetch_list=[out], + ) self.assertTrue((np.array(out[0]) == self.out_np).all()) def test_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_density_prior_box_op.py b/python/paddle/fluid/tests/unittests/test_density_prior_box_op.py index ec747d5aef2..7254f9cc2f8 100644 --- a/python/paddle/fluid/tests/unittests/test_density_prior_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_density_prior_box_op.py @@ -19,7 +19,6 @@ from op_test import OpTest class TestDensityPriorBoxOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -35,7 +34,7 @@ class TestDensityPriorBoxOp(OpTest): 'densities': self.densities, 'fixed_sizes': self.fixed_sizes, 'fixed_ratios': self.fixed_ratios, - 'flatten_to_2d': self.flatten_to_2d + 'flatten_to_2d': self.flatten_to_2d, } self.outputs = {'Boxes': self.out_boxes, 'Variances': self.out_var} @@ -74,18 +73,19 @@ class TestDensityPriorBoxOp(OpTest): if len(self.fixed_sizes) > 0 and len(self.densities) > 0: for density in self.densities: if len(self.fixed_ratios) > 0: - self.num_priors += len(self.fixed_ratios) * (pow( - density, 2)) + self.num_priors += len(self.fixed_ratios) * ( + pow(density, 2) + ) self.offset = 0.5 def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype('float32') + (self.batch_size, self.image_channels, self.image_w, self.image_h) + ).astype('float32') self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype('float32') + (self.batch_size, self.input_channels, self.layer_w, self.layer_h) + ).astype('float32') def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -99,32 +99,56 @@ class TestDensityPriorBoxOp(OpTest): c_x = (w + self.offset) * self.step_w c_y = (h + self.offset) * self.step_h # Generate density prior boxes with fixed size - for density, fixed_size in zip(self.densities, - self.fixed_sizes): - if (len(self.fixed_ratios) > 0): + for density, fixed_size in zip( + self.densities, self.fixed_sizes + ): + if len(self.fixed_ratios) > 0: for ar in self.fixed_ratios: shift = int(step_average / density) box_width_ratio = fixed_size * math.sqrt(ar) box_height_ratio = fixed_size / math.sqrt(ar) for di in range(density): for dj in range(density): - c_x_temp = c_x - step_average / 2.0 + shift / 2.0 + dj * shift - c_y_temp = c_y - step_average / 2.0 + shift / 2.0 + di * shift + c_x_temp = ( + c_x + - step_average / 2.0 + + shift / 2.0 + + dj * shift + ) + c_y_temp = ( + c_y + - step_average / 2.0 + + shift / 2.0 + + di * shift + ) out_boxes[h, w, idx, :] = [ - max((c_x_temp - box_width_ratio / 2.0) / - self.image_w, 0), - max((c_y_temp - box_height_ratio / 2.0) - / self.image_h, 0), - min((c_x_temp + box_width_ratio / 2.0) / - self.image_w, 1), - min((c_y_temp + box_height_ratio / 2.0) - / self.image_h, 1) + max( + (c_x_temp - box_width_ratio / 2.0) + / self.image_w, + 0, + ), + max( + (c_y_temp - box_height_ratio / 2.0) + / self.image_h, + 0, + ), + min( + (c_x_temp + box_width_ratio / 2.0) + / self.image_w, + 1, + ), + min( + (c_y_temp + box_height_ratio / 2.0) + / self.image_h, + 1, + ), ] idx += 1 if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype('float32') self.out_var = out_var.astype('float32') if self.flatten_to_2d: @@ -133,7 +157,6 @@ class TestDensityPriorBoxOp(OpTest): class TestDensityPriorBox(TestDensityPriorBoxOp): - def set_density(self): self.densities = [3, 4] self.fixed_sizes = [1.0, 2.0] diff --git a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py index 9933a2243fb..ae24334fea5 100755 --- a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py +++ b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py @@ -51,9 +51,11 @@ def get_warning_index(api): doc_lst = api.__doc__.splitlines() for idx, val in enumerate(doc_lst): - if val.startswith("Warning: ") and val.endswith( - " instead." - ) and "and will be removed in future versions." in val: + if ( + val.startswith("Warning: ") + and val.endswith(" instead.") + and "and will be removed in future versions." in val + ): return idx return ERROR_WARNING_POSTION @@ -152,7 +154,7 @@ class TestDeprecatedDocorator(unittest.TestCase): def test_tensor_gradient(self): paddle.__version__ = '2.1.0' - x = paddle.to_tensor(5., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) y = paddle.pow(x, 4.0) y.backward() @@ -160,7 +162,8 @@ class TestDeprecatedDocorator(unittest.TestCase): grad = x.gradient() assert ( 'API "paddle.fluid.dygraph.varbase_patch_methods.gradient" is ' - 'deprecated since 2.1.0') in str(w[-1].message) + 'deprecated since 2.1.0' + ) in str(w[-1].message) def test_softmax_with_cross_entropy(self): paddle.__version__ = '2.0.0' @@ -173,11 +176,13 @@ class TestDeprecatedDocorator(unittest.TestCase): x = linear(data) with warnings.catch_warnings(record=True) as w: - out = paddle.nn.functional.softmax_with_cross_entropy(logits=x, - label=label) + out = paddle.nn.functional.softmax_with_cross_entropy( + logits=x, label=label + ) assert ( 'API "paddle.nn.functional.loss.softmax_with_cross_entropy" is ' - 'deprecated since 2.0.0') in str(w[-1].message) + 'deprecated since 2.0.0' + ) in str(w[-1].message) def test_deprecated_error(self): paddle.__version__ = '2.1.0' diff --git a/python/paddle/fluid/tests/unittests/test_deprecated_memory_optimize_interfaces.py b/python/paddle/fluid/tests/unittests/test_deprecated_memory_optimize_interfaces.py index bd91e14e34d..c3a21ba0bcb 100644 --- a/python/paddle/fluid/tests/unittests/test_deprecated_memory_optimize_interfaces.py +++ b/python/paddle/fluid/tests/unittests/test_deprecated_memory_optimize_interfaces.py @@ -18,7 +18,6 @@ from simple_nets import simple_fc_net class DeprecatedMemoryOptimizationInterfaceTest(unittest.TestCase): - def setUp(self): self.method = fluid.memory_optimize @@ -61,7 +60,6 @@ class DeprecatedMemoryOptimizationInterfaceTest(unittest.TestCase): class ReleaseMemoryTest(DeprecatedMemoryOptimizationInterfaceTest): - def setUp(self): self.method = fluid.release_memory diff --git a/python/paddle/fluid/tests/unittests/test_dequantize_abs_max_op.py b/python/paddle/fluid/tests/unittests/test_dequantize_abs_max_op.py index 039686777be..c4806866eff 100644 --- a/python/paddle/fluid/tests/unittests/test_dequantize_abs_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_dequantize_abs_max_op.py @@ -30,7 +30,6 @@ def dequantize_max_abs(x, scale, max_range): class TestDequantizeMaxAbsOp(OpTest): - def set_args(self): self.num_bits = 8 self.max_range = math.pow(2, self.num_bits - 1) - 1 @@ -45,7 +44,7 @@ class TestDequantizeMaxAbsOp(OpTest): self.inputs = { 'X': np.array(yq).astype(self.data_type), - 'Scale': np.array(scale).astype('float32') + 'Scale': np.array(scale).astype('float32'), } self.attrs = {'max_range': self.max_range} self.outputs = {'Out': ydq} @@ -55,7 +54,6 @@ class TestDequantizeMaxAbsOp(OpTest): class TestDequantizeMaxAbsOp5Bits(TestDequantizeMaxAbsOp): - def set_args(self): self.num_bits = 5 self.max_range = math.pow(2, self.num_bits - 1) - 1 @@ -63,7 +61,6 @@ class TestDequantizeMaxAbsOp5Bits(TestDequantizeMaxAbsOp): class TestDequantizeMaxAbsOpInt16(TestDequantizeMaxAbsOp): - def set_args(self): self.num_bits = 16 self.max_range = math.pow(2, self.num_bits - 1) - 1 diff --git a/python/paddle/fluid/tests/unittests/test_dequantize_log_op.py b/python/paddle/fluid/tests/unittests/test_dequantize_log_op.py index 939d5d6afcf..5fb9d07d77a 100644 --- a/python/paddle/fluid/tests/unittests/test_dequantize_log_op.py +++ b/python/paddle/fluid/tests/unittests/test_dequantize_log_op.py @@ -30,7 +30,6 @@ def dequantize_log(x, dict_data): class TestDequantizeLogOp(OpTest): - def setUp(self): self.op_type = "dequantize_log" x = np.random.randint(low=-128, high=127, size=(20, 10)).astype('int8') @@ -39,7 +38,7 @@ class TestDequantizeLogOp(OpTest): self.inputs = { 'X': np.array(x).astype('int8'), - 'Dict': np.array(dict_data).astype('float32') + 'Dict': np.array(dict_data).astype('float32'), } self.outputs = {'Out': xdq} diff --git a/python/paddle/fluid/tests/unittests/test_desc_clone.py b/python/paddle/fluid/tests/unittests/test_desc_clone.py index 2744487b588..ad0e95d4bbc 100644 --- a/python/paddle/fluid/tests/unittests/test_desc_clone.py +++ b/python/paddle/fluid/tests/unittests/test_desc_clone.py @@ -27,33 +27,41 @@ paddle.dataset.mnist.fetch() # random seed must set before configuring the network. # fluid.default_startup_program().random_seed = SEED def cnn_model(data): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=data, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=data, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) # TODO(dzhwinter) : refine the initializer and random seed settting SIZE = 10 input_shape = conv_pool_2.shape - param_shape = [functools.reduce(lambda a, b: a * b, input_shape[1:], 1) - ] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + param_shape = [functools.reduce(lambda a, b: a * b, input_shape[1:], 1)] + [ + SIZE + ] + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer(loc=0.0, - scale=scale))) + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + ) return predict @@ -69,23 +77,32 @@ def get_model(batch_size): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization - opt = fluid.optimizer.AdamOptimizer(learning_rate=0.001, - beta1=0.9, - beta2=0.999) + opt = fluid.optimizer.AdamOptimizer( + learning_rate=0.001, beta1=0.9, beta2=0.999 + ) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) def operator_equal(a, b): @@ -93,8 +110,9 @@ def operator_equal(a, b): raise ValueError("In operator_equal not equal\n") for k, v in a.__dict__.items(): - if isinstance(v, fluid.framework.Program) or \ - isinstance(v, fluid.framework.Block): + if isinstance(v, fluid.framework.Program) or isinstance( + v, fluid.framework.Block + ): continue elif isinstance(v, core.OpDesc): @@ -107,7 +125,7 @@ def operator_equal(a, b): if v0 != v1: raise ValueError("In operator_equal not equal:{0}\n".format(k)) - elif (v != b.__dict__[k]): + elif v != b.__dict__[k]: raise ValueError("In operator_equal not equal:{0}\n".format(k)) return True @@ -115,12 +133,15 @@ def operator_equal(a, b): def block_equal(a, b): for k, v in a.__dict__.items(): - if isinstance(v, core.ProgramDesc) or isinstance( - v, fluid.framework.Program) or isinstance(v, core.BlockDesc): + if ( + isinstance(v, core.ProgramDesc) + or isinstance(v, fluid.framework.Program) + or isinstance(v, core.BlockDesc) + ): continue elif k == "ops": - assert (len(a.ops) == len(b.ops)) + assert len(a.ops) == len(b.ops) for i in range(0, len(a.ops)): if not operator_equal(a.ops[i], b.ops[i]): raise ValueError("In block_equal not equal:{0}\n".format(k)) @@ -130,7 +151,7 @@ def block_equal(a, b): if str(value) != str(b.__dict__[k][key]): raise ValueError("In block_equal not equal:{0}\n".format(k)) - elif (v != b.__dict__[k]): + elif v != b.__dict__[k]: raise ValueError("In block_equal not equal:{0}\n".format(k)) return True @@ -145,19 +166,19 @@ def program_equal(a, b): for i in range(0, len(a.blocks)): if not block_equal(a.blocks[i], b.blocks[i]): raise ValueError( - "In operator_equal not equal:{0}\n".format(k)) + "In operator_equal not equal:{0}\n".format(k) + ) return False - assert (len(a.blocks) == len(b.blocks)) + assert len(a.blocks) == len(b.blocks) elif k == '_auto_checkpoint_name': continue - elif (v != b.__dict__[k]): + elif v != b.__dict__[k]: raise ValueError("In program_equal not equal:{0}\n".format(k)) return True class TestCloneWithStopGradient(unittest.TestCase): - def test_clone_with_stop_gradient(self): train_program = fluid.Program() startup_program = fluid.Program() @@ -168,18 +189,20 @@ class TestCloneWithStopGradient(unittest.TestCase): hidden2 = fluid.layers.dropout(hidden1, dropout_prob=0.5) loss = fluid.layers.cross_entropy( input=fluid.layers.fc(hidden2, size=10, act='softmax'), - label=fluid.layers.data(name='label', shape=[1], dtype='int64')) + label=fluid.layers.data(name='label', shape=[1], dtype='int64'), + ) avg_loss = paddle.mean(loss) test_program = train_program.clone(for_test=False) self.assertEqual( - test_program.block(0).var(hidden1.name).stop_gradient, True) + test_program.block(0).var(hidden1.name).stop_gradient, True + ) self.assertEqual( - test_program.block(0).var(hidden2.name).stop_gradient, False) + test_program.block(0).var(hidden2.name).stop_gradient, False + ) class TestCloneWithStopGradientInSubBlock(unittest.TestCase): - def test_clone_with_stop_gradient(self): train_program = fluid.Program() startup_program = fluid.Program() @@ -204,12 +227,14 @@ class TestCloneWithStopGradientInSubBlock(unittest.TestCase): loss = fluid.layers.cross_entropy( input=fluid.layers.fc(hidden2, size=10, act='softmax'), - label=fluid.layers.data(name='label', shape=[1], dtype='int64')) + label=fluid.layers.data(name='label', shape=[1], dtype='int64'), + ) avg_loss = paddle.mean(loss) test_program = train_program.clone(for_test=False) self.assertEqual( - test_program.block(0).var(hidden1.name).stop_gradient, True) + test_program.block(0).var(hidden1.name).stop_gradient, True + ) for var in test_program.block(1).vars.values(): var2 = train_program.block(1).var(var.name) self.assertEqual(var.stop_gradient, var2.stop_gradient) @@ -219,7 +244,6 @@ class TestCloneWithStopGradientInSubBlock(unittest.TestCase): class TestCloneWithRaise(unittest.TestCase): - def test_clone_with_stop_gradient(self): train_program = fluid.Program() startup_program = fluid.Program() @@ -243,14 +267,19 @@ class TestCloneWithRaise(unittest.TestCase): hidden2 = fluid.layers.cond(cond, true_fn, false_fn) loss = fluid.layers.cross_entropy( input=fluid.layers.fc(hidden2, size=10, act='softmax'), - label=fluid.layers.data(name='label', shape=[1], dtype='int64')) + label=fluid.layers.data(name='label', shape=[1], dtype='int64'), + ) avg_loss = paddle.mean(loss) test_program = train_program.clone(for_test=False) - self.assertRaises(ValueError, train_program._copy_data_info_from, - startup_program) - self.assertRaises(TypeError, train_program._copy_data_info_from, - startup_program.block(0)) + self.assertRaises( + ValueError, train_program._copy_data_info_from, startup_program + ) + self.assertRaises( + TypeError, + train_program._copy_data_info_from, + startup_program.block(0), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_detach.py b/python/paddle/fluid/tests/unittests/test_detach.py index 901f35a756b..48abc41e3c1 100644 --- a/python/paddle/fluid/tests/unittests/test_detach.py +++ b/python/paddle/fluid/tests/unittests/test_detach.py @@ -23,39 +23,51 @@ import unittest class Test_Detach(unittest.TestCase): - def generate_Data(self): - data = np.array([[1, 8, 3, 9], [7, 20, 9, 6], [4, 6, 8, - 10]]).astype('float32') + data = np.array([[1, 8, 3, 9], [7, 20, 9, 6], [4, 6, 8, 10]]).astype( + 'float32' + ) return data def no_detach_multi(self): data = self.generate_Data() with fluid.dygraph.guard(): linear_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(5.0)) + initializer=fluid.initializer.Constant(5.0) + ) linear_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(6.0)) - linear = Linear(4, - 10, - param_attr=linear_w_param_attrs, - bias_attr=linear_b_param_attrs) + initializer=fluid.initializer.Constant(6.0) + ) + linear = Linear( + 4, + 10, + param_attr=linear_w_param_attrs, + bias_attr=linear_b_param_attrs, + ) linear1_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(7.0)) + initializer=fluid.initializer.Constant(7.0) + ) linear1_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(8.0)) - linear1 = Linear(10, - 1, - param_attr=linear1_w_param_attrs, - bias_attr=linear1_b_param_attrs) + initializer=fluid.initializer.Constant(8.0) + ) + linear1 = Linear( + 10, + 1, + param_attr=linear1_w_param_attrs, + bias_attr=linear1_b_param_attrs, + ) linear2_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(9.0)) + initializer=fluid.initializer.Constant(9.0) + ) linear2_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(10.0)) - linear2 = Linear(10, - 1, - param_attr=linear2_w_param_attrs, - bias_attr=linear2_b_param_attrs) + initializer=fluid.initializer.Constant(10.0) + ) + linear2 = Linear( + 10, + 1, + param_attr=linear2_w_param_attrs, + bias_attr=linear2_b_param_attrs, + ) data = to_variable(data) x = linear(data) x1 = linear1(x) @@ -69,21 +81,29 @@ class Test_Detach(unittest.TestCase): data = self.generate_Data() with fluid.dygraph.guard(): linear_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(5.0)) + initializer=fluid.initializer.Constant(5.0) + ) linear_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(6.0)) - linear = Linear(4, - 10, - param_attr=linear_w_param_attrs, - bias_attr=linear_b_param_attrs) + initializer=fluid.initializer.Constant(6.0) + ) + linear = Linear( + 4, + 10, + param_attr=linear_w_param_attrs, + bias_attr=linear_b_param_attrs, + ) linear1_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(7.0)) + initializer=fluid.initializer.Constant(7.0) + ) linear1_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(8.0)) - linear1 = Linear(10, - 1, - param_attr=linear1_w_param_attrs, - bias_attr=linear1_b_param_attrs) + initializer=fluid.initializer.Constant(8.0) + ) + linear1 = Linear( + 10, + 1, + param_attr=linear1_w_param_attrs, + bias_attr=linear1_b_param_attrs, + ) data = to_variable(data) x = linear(data) x1 = linear1(x) @@ -96,29 +116,41 @@ class Test_Detach(unittest.TestCase): data = self.generate_Data() with fluid.dygraph.guard(): linear_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(5.0)) + initializer=fluid.initializer.Constant(5.0) + ) linear_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(6.0)) - linear = Linear(4, - 10, - param_attr=linear_w_param_attrs, - bias_attr=linear_b_param_attrs) + initializer=fluid.initializer.Constant(6.0) + ) + linear = Linear( + 4, + 10, + param_attr=linear_w_param_attrs, + bias_attr=linear_b_param_attrs, + ) linear1_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(7.0)) + initializer=fluid.initializer.Constant(7.0) + ) linear1_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(8.0)) - linear1 = Linear(10, - 1, - param_attr=linear1_w_param_attrs, - bias_attr=linear1_b_param_attrs) + initializer=fluid.initializer.Constant(8.0) + ) + linear1 = Linear( + 10, + 1, + param_attr=linear1_w_param_attrs, + bias_attr=linear1_b_param_attrs, + ) linear2_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(9.0)) + initializer=fluid.initializer.Constant(9.0) + ) linear2_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(10.0)) - linear2 = Linear(10, - 1, - param_attr=linear2_w_param_attrs, - bias_attr=linear2_b_param_attrs) + initializer=fluid.initializer.Constant(10.0) + ) + linear2 = Linear( + 10, + 1, + param_attr=linear2_w_param_attrs, + bias_attr=linear2_b_param_attrs, + ) data = to_variable(data) x = linear(data) x_detach = x.detach() @@ -144,7 +176,6 @@ class Test_Detach(unittest.TestCase): class TestInplace(unittest.TestCase): - def test_forward_version(self): with paddle.fluid.dygraph.guard(): var = paddle.to_tensor(np.ones((4, 2, 3)).astype(np.float32)) @@ -180,9 +211,11 @@ class TestInplace(unittest.TestCase): loss = paddle.nn.functional.relu(var_c + var_d) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): loss.backward() diff --git a/python/paddle/fluid/tests/unittests/test_detection_map_op.py b/python/paddle/fluid/tests/unittests/test_detection_map_op.py index 68afa3feed9..00b2c0700ca 100644 --- a/python/paddle/fluid/tests/unittests/test_detection_map_op.py +++ b/python/paddle/fluid/tests/unittests/test_detection_map_op.py @@ -20,7 +20,6 @@ from op_test import OpTest class TestDetectionMAPOp(OpTest): - def set_data(self): self.class_num = 4 self.init_test_case() @@ -30,8 +29,9 @@ class TestDetectionMAPOp(OpTest): self.mAP = np.array(self.mAP).astype('float32') if len(self.class_pos_count) > 0: - self.class_pos_count = np.array( - self.class_pos_count).astype('int32') + self.class_pos_count = np.array(self.class_pos_count).astype( + 'int32' + ) self.true_pos = np.array(self.true_pos).astype('float32') self.false_pos = np.array(self.false_pos).astype('float32') self.has_state = np.array([1]).astype('int32') @@ -42,7 +42,7 @@ class TestDetectionMAPOp(OpTest): 'HasState': self.has_state, 'PosCount': self.class_pos_count, 'TruePos': (self.true_pos, self.true_pos_lod), - 'FalsePos': (self.false_pos, self.false_pos_lod) + 'FalsePos': (self.false_pos, self.false_pos_lod), } else: self.inputs = { @@ -54,11 +54,12 @@ class TestDetectionMAPOp(OpTest): 'overlap_threshold': self.overlap_threshold, 'evaluate_difficult': self.evaluate_difficult, 'ap_type': self.ap_type, - 'class_num': self.class_num + 'class_num': self.class_num, } - self.out_class_pos_count = np.array( - self.out_class_pos_count).astype('int') + self.out_class_pos_count = np.array(self.out_class_pos_count).astype( + 'int' + ) self.out_true_pos = np.array(self.out_true_pos).astype('float32') self.out_false_pos = np.array(self.out_false_pos).astype('float32') @@ -66,7 +67,7 @@ class TestDetectionMAPOp(OpTest): 'MAP': self.mAP, 'AccumPosCount': self.out_class_pos_count, 'AccumTruePos': (self.out_true_pos, self.out_true_pos_lod), - 'AccumFalsePos': (self.out_false_pos, self.out_false_pos_lod) + 'AccumFalsePos': (self.out_false_pos, self.out_false_pos_lod), } def init_test_case(self): @@ -76,24 +77,36 @@ class TestDetectionMAPOp(OpTest): self.label_lod = [[2, 2]] # label difficult xmin ymin xmax ymax - self.label = [[1, 0, 0.1, 0.1, 0.3, 0.3], [1, 1, 0.6, 0.6, 0.8, 0.8], - [2, 0, 0.3, 0.3, 0.6, 0.5], [1, 0, 0.7, 0.1, 0.9, 0.3]] + self.label = [ + [1, 0, 0.1, 0.1, 0.3, 0.3], + [1, 1, 0.6, 0.6, 0.8, 0.8], + [2, 0, 0.3, 0.3, 0.6, 0.5], + [1, 0, 0.7, 0.1, 0.9, 0.3], + ] # label score xmin ymin xmax ymax difficult self.detect_lod = [[3, 4]] - self.detect = [[1, 0.3, 0.1, 0.0, 0.4, - 0.3], [1, 0.7, 0.0, 0.1, 0.2, 0.3], - [1, 0.9, 0.7, 0.6, 0.8, - 0.8], [2, 0.8, 0.2, 0.1, 0.4, 0.4], - [2, 0.1, 0.4, 0.3, 0.7, - 0.5], [1, 0.2, 0.8, 0.1, 1.0, 0.3], - [3, 0.2, 0.8, 0.1, 1.0, 0.3]] + self.detect = [ + [1, 0.3, 0.1, 0.0, 0.4, 0.3], + [1, 0.7, 0.0, 0.1, 0.2, 0.3], + [1, 0.9, 0.7, 0.6, 0.8, 0.8], + [2, 0.8, 0.2, 0.1, 0.4, 0.4], + [2, 0.1, 0.4, 0.3, 0.7, 0.5], + [1, 0.2, 0.8, 0.1, 1.0, 0.3], + [3, 0.2, 0.8, 0.1, 1.0, 0.3], + ] # label score true_pos false_pos self.tf_pos_lod = [[3, 4]] - self.tf_pos = [[1, 0.9, 1, 0], [1, 0.7, 1, 0], [1, 0.3, 0, 1], - [1, 0.2, 1, 0], [2, 0.8, 0, 1], [2, 0.1, 1, 0], - [3, 0.2, 0, 1]] + self.tf_pos = [ + [1, 0.9, 1, 0], + [1, 0.7, 1, 0], + [1, 0.3, 0, 1], + [1, 0.2, 1, 0], + [2, 0.8, 0, 1], + [2, 0.1, 1, 0], + [3, 0.2, 0, 1], + ] self.class_pos_count = [] self.true_pos_lod = [[]] @@ -105,8 +118,9 @@ class TestDetectionMAPOp(OpTest): mAP = 0.0 count = 0 - def get_input_pos(class_pos_count, true_pos, true_pos_lod, false_pos, - false_pos_lod): + def get_input_pos( + class_pos_count, true_pos, true_pos_lod, false_pos, false_pos_lod + ): class_pos_count_dict = collections.Counter() true_pos_dict = collections.defaultdict(list) false_pos_dict = collections.defaultdict(list) @@ -149,9 +163,13 @@ class TestDetectionMAPOp(OpTest): out_false_pos += false_pos_list out_false_pos_lod.append(len(false_pos_list)) - return out_class_pos_count, out_true_pos, [ - out_true_pos_lod - ], out_false_pos, [out_false_pos_lod] + return ( + out_class_pos_count, + out_true_pos, + [out_true_pos_lod], + out_false_pos, + [out_false_pos_lod], + ) def get_accumulation(pos_list): sorted_list = sorted(pos_list, key=lambda pos: pos[0], reverse=True) @@ -163,8 +181,12 @@ class TestDetectionMAPOp(OpTest): return accu_list label_count, true_pos, false_pos = get_input_pos( - self.class_pos_count, self.true_pos, self.true_pos_lod, - self.false_pos, self.false_pos_lod) + self.class_pos_count, + self.true_pos, + self.true_pos_lod, + self.false_pos, + self.false_pos_lod, + ) for v in self.label: label = v[0] difficult = False if len(v) == 5 else v[1] @@ -178,7 +200,8 @@ class TestDetectionMAPOp(OpTest): false_pos[label].append([score, fp]) for (label, label_pos_num) in label_count.items(): - if label_pos_num == 0: continue + if label_pos_num == 0: + continue if label not in true_pos: count += 1 continue @@ -193,8 +216,9 @@ class TestDetectionMAPOp(OpTest): for i in range(len(accu_tp_sum)): precision.append( - float(accu_tp_sum[i]) / - float(accu_tp_sum[i] + accu_fp_sum[i])) + float(accu_tp_sum[i]) + / float(accu_tp_sum[i] + accu_fp_sum[i]) + ) recall.append(float(accu_tp_sum[i]) / label_pos_num) if self.ap_type == "11point": @@ -218,14 +242,16 @@ class TestDetectionMAPOp(OpTest): prev_recall = 0.0 for i in range(len(accu_tp_sum)): if math.fabs(recall[i] - prev_recall) > 1e-6: - average_precisions += precision[i] * \ - math.fabs(recall[i] - prev_recall) + average_precisions += precision[i] * math.fabs( + recall[i] - prev_recall + ) prev_recall = recall[i] mAP += average_precisions count += 1 - pcnt, tp, tp_lod, fp, fp_lod = get_output_pos(label_count, true_pos, - false_pos) + pcnt, tp, tp_lod, fp, fp_lod = get_output_pos( + label_count, true_pos, false_pos + ) self.out_class_pos_count = pcnt self.out_true_pos = tp self.out_true_pos_lod = tp_lod @@ -244,7 +270,6 @@ class TestDetectionMAPOp(OpTest): class TestDetectionMAPOpSkipDiff(TestDetectionMAPOp): - def init_test_case(self): super(TestDetectionMAPOpSkipDiff, self).init_test_case() @@ -252,22 +277,30 @@ class TestDetectionMAPOpSkipDiff(TestDetectionMAPOp): self.tf_pos_lod = [[2, 4]] # label score true_pos false_pos - self.tf_pos = [[1, 0.7, 1, 0], [1, 0.3, 0, 1], [1, 0.2, 1, 0], - [2, 0.8, 0, 1], [2, 0.1, 1, 0], [3, 0.2, 0, 1]] + self.tf_pos = [ + [1, 0.7, 1, 0], + [1, 0.3, 0, 1], + [1, 0.2, 1, 0], + [2, 0.8, 0, 1], + [2, 0.1, 1, 0], + [3, 0.2, 0, 1], + ] class TestDetectionMAPOpWithoutDiff(TestDetectionMAPOp): - def init_test_case(self): super(TestDetectionMAPOpWithoutDiff, self).init_test_case() # label xmin ymin xmax ymax - self.label = [[1, 0.1, 0.1, 0.3, 0.3], [1, 0.6, 0.6, 0.8, 0.8], - [2, 0.3, 0.3, 0.6, 0.5], [1, 0.7, 0.1, 0.9, 0.3]] + self.label = [ + [1, 0.1, 0.1, 0.3, 0.3], + [1, 0.6, 0.6, 0.8, 0.8], + [2, 0.3, 0.3, 0.6, 0.5], + [1, 0.7, 0.1, 0.9, 0.3], + ] class TestDetectionMAPOp11Point(TestDetectionMAPOp): - def init_test_case(self): super(TestDetectionMAPOp11Point, self).init_test_case() @@ -275,18 +308,28 @@ class TestDetectionMAPOp11Point(TestDetectionMAPOp): class TestDetectionMAPOpMultiBatch(TestDetectionMAPOp): - def init_test_case(self): super(TestDetectionMAPOpMultiBatch, self).init_test_case() self.class_pos_count = [0, 2, 1, 0] self.true_pos_lod = [[0, 3, 2]] - self.true_pos = [[0.7, 1.], [0.3, 0.], [0.2, 1.], [0.8, 0.], [0.1, 1.]] + self.true_pos = [ + [0.7, 1.0], + [0.3, 0.0], + [0.2, 1.0], + [0.8, 0.0], + [0.1, 1.0], + ] self.false_pos_lod = [[0, 3, 2]] - self.false_pos = [[0.7, 0.], [0.3, 1.], [0.2, 0.], [0.8, 1.], [0.1, 0.]] + self.false_pos = [ + [0.7, 0.0], + [0.3, 1.0], + [0.2, 0.0], + [0.8, 1.0], + [0.1, 0.0], + ] class TestDetectionMAPOp11PointWithClassNoTP(TestDetectionMAPOp): - def init_test_case(self): self.overlap_threshold = 0.3 self.evaluate_difficult = True diff --git a/python/paddle/fluid/tests/unittests/test_determinant_op.py b/python/paddle/fluid/tests/unittests/test_determinant_op.py index abd44669b25..730864eb230 100644 --- a/python/paddle/fluid/tests/unittests/test_determinant_op.py +++ b/python/paddle/fluid/tests/unittests/test_determinant_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestDeterminantOp(OpTest): - def setUp(self): self.python_api = paddle.linalg.det self.init_data() @@ -43,7 +42,6 @@ class TestDeterminantOp(OpTest): class TestDeterminantOpCase1(TestDeterminantOp): - def init_data(self): np.random.seed(0) self.case = np.random.rand(10, 10).astype('float32') @@ -52,7 +50,6 @@ class TestDeterminantOpCase1(TestDeterminantOp): class TestDeterminantOpCase2(TestDeterminantOp): - def init_data(self): np.random.seed(0) # not invertible matrix @@ -62,7 +59,6 @@ class TestDeterminantOpCase2(TestDeterminantOp): class TestDeterminantAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [3, 3, 5, 5] @@ -95,7 +91,6 @@ class TestDeterminantAPI(unittest.TestCase): class TestSlogDeterminantOp(OpTest): - def setUp(self): self.op_type = "slogdeterminant" self.python_api = paddle.linalg.slogdet @@ -107,9 +102,9 @@ class TestSlogDeterminantOp(OpTest): def test_check_grad(self): # the slog det's grad value is always huge - self.check_grad(['Input'], ['Out'], - max_relative_error=0.1, - check_eager=True) + self.check_grad( + ['Input'], ['Out'], max_relative_error=0.1, check_eager=True + ) def init_data(self): np.random.seed(0) @@ -119,7 +114,6 @@ class TestSlogDeterminantOp(OpTest): class TestSlogDeterminantOpCase1(TestSlogDeterminantOp): - def init_data(self): np.random.seed(0) self.case = np.random.rand(2, 2, 5, 5).astype(np.float32) @@ -128,7 +122,6 @@ class TestSlogDeterminantOpCase1(TestSlogDeterminantOp): class TestSlogDeterminantAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [3, 3, 5, 5] diff --git a/python/paddle/fluid/tests/unittests/test_device.py b/python/paddle/fluid/tests/unittests/test_device.py index 3d1653f123f..cf95ea96f7b 100644 --- a/python/paddle/fluid/tests/unittests/test_device.py +++ b/python/paddle/fluid/tests/unittests/test_device.py @@ -21,7 +21,6 @@ import paddle.fluid.framework as framework class TestStaticDeviceManage(unittest.TestCase): - def _test_device(self, device_name, device_class): paddle.set_device(device_name) @@ -54,7 +53,6 @@ class TestStaticDeviceManage(unittest.TestCase): class TestImperativeDeviceManage(unittest.TestCase): - def test_cpu(self): with fluid.dygraph.guard(): paddle.set_device('cpu') @@ -64,7 +62,8 @@ class TestImperativeDeviceManage(unittest.TestCase): device = paddle.get_device() self.assertEqual( isinstance(framework._current_expected_place(), core.CPUPlace), - True) + True, + ) self.assertEqual(device, "cpu") def test_gpu(self): @@ -76,8 +75,11 @@ class TestImperativeDeviceManage(unittest.TestCase): out3 = paddle.concat(x=[out1, out2], axis=0) device = paddle.get_device() self.assertEqual( - isinstance(framework._current_expected_place(), - core.CUDAPlace), True) + isinstance( + framework._current_expected_place(), core.CUDAPlace + ), + True, + ) self.assertEqual(device, "gpu:0") def test_xpu(self): @@ -86,8 +88,11 @@ class TestImperativeDeviceManage(unittest.TestCase): out = paddle.to_tensor([1, 2]) device = paddle.get_device() self.assertEqual( - isinstance(framework._current_expected_place(), - core.XPUPlace), True) + isinstance( + framework._current_expected_place(), core.XPUPlace + ), + True, + ) self.assertTrue(out.place.is_xpu_place()) self.assertEqual(device, "xpu:0") @@ -100,8 +105,11 @@ class TestImperativeDeviceManage(unittest.TestCase): out3 = paddle.concat(x=[out1, out2], axis=0) device = paddle.get_device() self.assertEqual( - isinstance(framework._current_expected_place(), - core.NPUPlace), True) + isinstance( + framework._current_expected_place(), core.NPUPlace + ), + True, + ) self.assertTrue(out1.place.is_npu_place()) self.assertTrue(out2.place.is_npu_place()) self.assertTrue(out3.place.is_npu_place()) diff --git a/python/paddle/fluid/tests/unittests/test_device_guard.py b/python/paddle/fluid/tests/unittests/test_device_guard.py index f3ac33a4f85..b43fbb6e87f 100644 --- a/python/paddle/fluid/tests/unittests/test_device_guard.py +++ b/python/paddle/fluid/tests/unittests/test_device_guard.py @@ -39,17 +39,16 @@ def get_vaild_warning_num(warning, w): class TestDeviceGuard(unittest.TestCase): - def test_device_guard(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.full(shape=[1, 3, 8, 8], - fill_value=0.5, - dtype='float32') - data2 = paddle.full(shape=[1, 3, 5, 5], - fill_value=0.5, - dtype='float32') + data1 = paddle.full( + shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32' + ) + data2 = paddle.full( + shape=[1, 3, 5, 5], fill_value=0.5, dtype='float32' + ) shape = paddle.shape(data2) with paddle.static.device_guard("cpu"): shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4]) @@ -70,12 +69,12 @@ class TestDeviceGuard(unittest.TestCase): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.full(shape=[1, 3, 8, 8], - fill_value=0.5, - dtype='float32') - data2 = paddle.full(shape=[1, 3, 5, 5], - fill_value=0.5, - dtype='float32') + data1 = paddle.full( + shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32' + ) + data2 = paddle.full( + shape=[1, 3, 5, 5], fill_value=0.5, dtype='float32' + ) shape = paddle.shape(data2) with paddle.static.device_guard("cpu"): shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4]) @@ -96,32 +95,50 @@ class TestDeviceGuard(unittest.TestCase): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.full(shape=[2, 255, 13, 13], - fill_value=0.3, - dtype='float32') - gt_box = paddle.full(shape=[2, 6, 4], - fill_value=0.5, - dtype='float32') + x = paddle.full( + shape=[2, 255, 13, 13], fill_value=0.3, dtype='float32' + ) + gt_box = paddle.full( + shape=[2, 6, 4], fill_value=0.5, dtype='float32' + ) gt_label = paddle.full(shape=[2, 6], fill_value=1.0, dtype='int32') - gt_score = paddle.full(shape=[2, 6], - fill_value=0.5, - dtype='float32') + gt_score = paddle.full( + shape=[2, 6], fill_value=0.5, dtype='float32' + ) anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, - 198, 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] anchor_mask = [0, 1, 2] with paddle.static.device_guard("gpu"): # yolov3_loss only has cpu kernel, so its cpu kernel will be executed - loss = fluid.layers.yolov3_loss(x=x, - gt_box=gt_box, - gt_label=gt_label, - gt_score=gt_score, - anchors=anchors, - anchor_mask=anchor_mask, - class_num=80, - ignore_thresh=0.7, - downsample_ratio=32) + loss = fluid.layers.yolov3_loss( + x=x, + gt_box=gt_box, + gt_label=gt_label, + gt_score=gt_score, + anchors=anchors, + anchor_mask=anchor_mask, + class_num=80, + ignore_thresh=0.7, + downsample_ratio=32, + ) execute(main_program, startup_program) @@ -154,7 +171,6 @@ class TestDeviceGuard(unittest.TestCase): execute(main_program, startup_program) def test_error(self): - def device_attr(): with paddle.static.device_guard("cpu1"): out = paddle.full(shape=[1], fill_value=0.2, dtype='float32') @@ -171,17 +187,18 @@ class TestDeviceGuard(unittest.TestCase): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.static.data(name="data_1", - shape=[4, 2], - dtype="float32") - label = paddle.static.data(name="label", - shape=[4, 1], - dtype="int64") + data1 = paddle.static.data( + name="data_1", shape=[4, 2], dtype="float32" + ) + label = paddle.static.data( + name="label", shape=[4, 1], dtype="int64" + ) fc1 = paddle.static.nn.fc(x=data1, size=10) fc2 = paddle.static.nn.fc(x=fc1, size=10) with paddle.static.device_guard("gpu"): out = paddle.nn.functional.softmax_with_cross_entropy( - logits=fc1 + fc2, label=label) + logits=fc1 + fc2, label=label + ) loss = paddle.mean(out) opt = paddle.optimizer.SGD(0.1) opt.minimize(loss) diff --git a/python/paddle/fluid/tests/unittests/test_diag.py b/python/paddle/fluid/tests/unittests/test_diag.py index 5148efe1b08..4135db34218 100644 --- a/python/paddle/fluid/tests/unittests/test_diag.py +++ b/python/paddle/fluid/tests/unittests/test_diag.py @@ -21,7 +21,6 @@ from paddle.fluid import Program, program_guard class TestDiagOp(OpTest): - def setUp(self): self.op_type = "diag" self.init_config() @@ -38,13 +37,11 @@ class TestDiagOp(OpTest): class TestDiagOpCase1(TestDiagOp): - def init_config(self): self.case = np.array([3], dtype='int32') class TestDiagError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): diff --git a/python/paddle/fluid/tests/unittests/test_diag_embed.py b/python/paddle/fluid/tests/unittests/test_diag_embed.py index 8eeb7849da2..c91f5d9b86c 100644 --- a/python/paddle/fluid/tests/unittests/test_diag_embed.py +++ b/python/paddle/fluid/tests/unittests/test_diag_embed.py @@ -21,7 +21,6 @@ import paddle.fluid.core as core class TestDiagEmbedOp(OpTest): - def setUp(self): self.op_type = "diag_embed" self.python_api = F.diag_embed @@ -39,17 +38,16 @@ class TestDiagEmbedOp(OpTest): class TestDiagEmbedOpCase1(TestDiagEmbedOp): - def init_config(self): self.case = np.random.randn(2, 3).astype('float32') self.inputs = {'Input': self.case} self.attrs = {'offset': -1, 'dim1': 0, 'dim2': 2} - self.target = np.stack([np.diag(r, -1) for r in self.inputs['Input']], - 1) + self.target = np.stack( + [np.diag(r, -1) for r in self.inputs['Input']], 1 + ) class TestDiagEmbedAPICase(unittest.TestCase): - def test_case1(self): diag_embed = np.random.randn(2, 3, 4).astype('float32') data1 = fluid.data(name='data1', shape=[2, 3, 4], dtype='float32') @@ -58,14 +56,18 @@ class TestDiagEmbedAPICase(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) - results = exe.run(fluid.default_main_program(), - feed={"data1": diag_embed}, - fetch_list=[out1, out2], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": diag_embed}, + fetch_list=[out1, out2], + return_numpy=True, + ) target1 = np.stack( - [np.stack([np.diag(s, 0) for s in r], 0) for r in diag_embed], 0) + [np.stack([np.diag(s, 0) for s in r], 0) for r in diag_embed], 0 + ) target2 = np.stack( - [np.stack([np.diag(s, 1) for s in r], 0) for r in diag_embed], 0) + [np.stack([np.diag(s, 1) for s in r], 0) for r in diag_embed], 0 + ) np.testing.assert_allclose(results[0], target1, rtol=1e-05) np.testing.assert_allclose(results[1], target2, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_diag_v2.py b/python/paddle/fluid/tests/unittests/test_diag_v2.py index 5fb93f3b19a..f1ff2b77d40 100644 --- a/python/paddle/fluid/tests/unittests/test_diag_v2.py +++ b/python/paddle/fluid/tests/unittests/test_diag_v2.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class TestDiagV2Op(OpTest): - def setUp(self): self.op_type = "diag_v2" self.python_api = paddle.diag @@ -35,7 +34,7 @@ class TestDiagV2Op(OpTest): self.inputs = {'X': self.x} self.attrs = { 'offset': self.offset, - 'padding_value': self.padding_value + 'padding_value': self.padding_value, } self.outputs = {'Out': self.out} @@ -52,38 +51,36 @@ class TestDiagV2Op(OpTest): class TestDiagV2OpCase1(TestDiagV2Op): - def init_config(self): self.offset = 1 self.out = np.diag(self.x, self.offset) class TestDiagV2OpCase2(TestDiagV2Op): - def init_config(self): self.offset = -1 self.out = np.diag(self.x, self.offset) class TestDiagV2OpCase3(TestDiagV2Op): - def init_config(self): self.x = np.random.randint(-10, 10, size=(10, 10)).astype("float64") self.out = np.diag(self.x, self.offset) class TestDiagV2OpCase4(TestDiagV2Op): - def init_config(self): self.x = np.random.rand(100) self.padding_value = 2 n = self.x.size - self.out = self.padding_value * np.ones((n, n)) + np.diag( - self.x, self.offset) - np.diag(self.padding_value * np.ones(n)) + self.out = ( + self.padding_value * np.ones((n, n)) + + np.diag(self.x, self.offset) + - np.diag(self.padding_value * np.ones(n)) + ) class TestDiagV2Error(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -104,7 +101,6 @@ class TestDiagV2Error(unittest.TestCase): class TestDiagV2API(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10)).astype(np.float32) self.expected0 = np.diag(self.input_np) @@ -115,21 +111,27 @@ class TestDiagV2API(unittest.TestCase): self.offset = 0 self.padding_value = 8 n = self.input_np2.size - self.expected3 = self.padding_value * np.ones( - (n, n)) + np.diag(self.input_np2, self.offset) - np.diag( - self.padding_value * np.ones(n)) + self.expected3 = ( + self.padding_value * np.ones((n, n)) + + np.diag(self.input_np2, self.offset) + - np.diag(self.padding_value * np.ones(n)) + ) self.input_np3 = np.random.randint(-10, 10, size=(100)).astype(np.int64) self.padding_value = 8.0 n = self.input_np3.size - self.expected4 = self.padding_value * np.ones( - (n, n)) + np.diag(self.input_np3, self.offset) - np.diag( - self.padding_value * np.ones(n)) + self.expected4 = ( + self.padding_value * np.ones((n, n)) + + np.diag(self.input_np3, self.offset) + - np.diag(self.padding_value * np.ones(n)) + ) self.padding_value = -8 - self.expected5 = self.padding_value * np.ones( - (n, n)) + np.diag(self.input_np3, self.offset) - np.diag( - self.padding_value * np.ones(n)) + self.expected5 = ( + self.padding_value * np.ones((n, n)) + + np.diag(self.input_np3, self.offset) + - np.diag(self.padding_value * np.ones(n)) + ) self.input_np4 = np.random.random(size=(2000, 2000)).astype(np.float32) self.expected6 = np.diag(self.input_np4) @@ -194,13 +196,13 @@ class TestDiagV2API(unittest.TestCase): x = paddle.static.data(name='input', shape=[10, 10], dtype='float32') x2 = paddle.static.data(name='input2', shape=[100], dtype='float64') x3 = paddle.static.data(name='input3', shape=[100], dtype='int64') - x4 = paddle.static.data(name='input4', - shape=[2000, 2000], - dtype='float32') + x4 = paddle.static.data( + name='input4', shape=[2000, 2000], dtype='float32' + ) x5 = paddle.static.data(name='input5', shape=[2000], dtype='float32') - x6 = paddle.static.data(name='input6', - shape=[2000, 1500], - dtype='float32') + x6 = paddle.static.data( + name='input6', shape=[2000, 1500], dtype='float32' + ) result0 = paddle.diag(x) result1 = paddle.diag(x, offset=1) result2 = paddle.diag(x, offset=-1) @@ -219,19 +221,45 @@ class TestDiagV2API(unittest.TestCase): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - res0, res1, res2, res4, res5, res6, res7, res8, res9, res10, res11, res12, res13 = exe.run( + ( + res0, + res1, + res2, + res4, + res5, + res6, + res7, + res8, + res9, + res10, + res11, + res12, + res13, + ) = exe.run( feed={ "input": self.input_np, "input2": self.input_np2, 'input3': self.input_np3, 'input4': self.input_np4, 'input5': self.input_np5, - 'input6': self.input_np6 + 'input6': self.input_np6, }, fetch_list=[ - result0, result1, result2, result4, result5, result6, result7, - result8, result9, result10, result11, result12, result13 - ]) + result0, + result1, + result2, + result4, + result5, + result6, + result7, + result8, + result9, + result10, + result11, + result12, + result13, + ], + ) np.testing.assert_allclose(res0, self.expected0, rtol=1e-05) np.testing.assert_allclose(res1, self.expected1, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_diagflat.py b/python/paddle/fluid/tests/unittests/test_diagflat.py index ec6bf982991..f23c5051ddb 100644 --- a/python/paddle/fluid/tests/unittests/test_diagflat.py +++ b/python/paddle/fluid/tests/unittests/test_diagflat.py @@ -19,7 +19,6 @@ from paddle.static import Program, program_guard class TestDiagFlatError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -35,7 +34,6 @@ class TestDiagFlatError(unittest.TestCase): class TestDiagFlatAPI(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10)).astype(np.float64) self.expected0 = np.diagflat(self.input_np) @@ -77,11 +75,10 @@ class TestDiagFlatAPI(unittest.TestCase): place = paddle.CUDAPlace(0) if use_gpu else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) - res0, res3 = exe.run(feed={ - "input": self.input_np, - 'input2': self.input_np2 - }, - fetch_list=[result0, result3]) + res0, res3 = exe.run( + feed={"input": self.input_np, 'input2': self.input_np2}, + fetch_list=[result0, result3], + ) np.testing.assert_allclose(res0, self.expected0, rtol=1e-05) np.testing.assert_allclose(res3, self.expected3, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_diagonal_op.py b/python/paddle/fluid/tests/unittests/test_diagonal_op.py index c7976856bd6..bdc64827fd2 100644 --- a/python/paddle/fluid/tests/unittests/test_diagonal_op.py +++ b/python/paddle/fluid/tests/unittests/test_diagonal_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestDiagonalOp(OpTest): - def setUp(self): self.op_type = "diagonal" self.python_api = paddle.diagonal @@ -39,62 +38,68 @@ class TestDiagonalOp(OpTest): self.case = np.random.randn(10, 5, 2).astype('float64') self.inputs = {'Input': self.case} self.attrs = {'offset': 0, 'axis1': 0, 'axis2': 1} - self.target = np.diagonal(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.diagonal( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) class TestDiagonalOpCase1(TestDiagonalOp): - def init_config(self): self.case = np.random.randn(4, 2, 4, 4).astype('float32') self.inputs = {'Input': self.case} self.attrs = {'offset': -2, 'axis1': 3, 'axis2': 0} - self.target = np.diagonal(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.diagonal( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) class TestDiagonalOpCase2(TestDiagonalOp): - def init_config(self): self.case = np.random.randn(100, 100).astype('int64') self.inputs = {'Input': self.case} self.attrs = {'offset': 0, 'axis1': 0, 'axis2': 1} - self.target = np.diagonal(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.diagonal( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) self.grad_x = np.eye(100).astype('int64') self.grad_out = np.ones(100).astype('int64') def test_check_grad(self): - self.check_grad(['Input'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['Input'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestDiagonalOpCase3(TestDiagonalOp): - def init_config(self): self.case = np.random.randint(0, 2, (4, 2, 4, 4)).astype('bool') self.inputs = {'Input': self.case} self.attrs = {'offset': -2, 'axis1': 3, 'axis2': 0} - self.target = np.diagonal(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.diagonal( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) def test_check_grad(self): pass class TestDiagonalAPI(unittest.TestCase): - def setUp(self): self.shape = [10, 3, 4] self.x = np.random.random((10, 3, 4)).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_diff_op.py b/python/paddle/fluid/tests/unittests/test_diff_op.py index 8111d1b57b3..4b3981075a1 100644 --- a/python/paddle/fluid/tests/unittests/test_diff_op.py +++ b/python/paddle/fluid/tests/unittests/test_diff_op.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestDiffOp(unittest.TestCase): - def set_args(self): self.input = np.array([1, 4, 5, 2]).astype('float32') self.n = 1 @@ -31,21 +30,21 @@ class TestDiffOp(unittest.TestCase): def get_output(self): if self.prepend is not None and self.append is not None: - self.output = np.diff(self.input, - n=self.n, - axis=self.axis, - prepend=self.prepend, - append=self.append) + self.output = np.diff( + self.input, + n=self.n, + axis=self.axis, + prepend=self.prepend, + append=self.append, + ) elif self.prepend is not None: - self.output = np.diff(self.input, - n=self.n, - axis=self.axis, - prepend=self.prepend) + self.output = np.diff( + self.input, n=self.n, axis=self.axis, prepend=self.prepend + ) elif self.append is not None: - self.output = np.diff(self.input, - n=self.n, - axis=self.axis, - append=self.append) + self.output = np.diff( + self.input, n=self.n, axis=self.axis, append=self.append + ) else: self.output = np.diff(self.input, n=self.n, axis=self.axis) @@ -64,11 +63,13 @@ class TestDiffOp(unittest.TestCase): self.prepend = paddle.to_tensor(self.prepend, place=place) if self.append is not None: self.append = paddle.to_tensor(self.append, place=place) - out = paddle.diff(x, - n=self.n, - axis=self.axis, - prepend=self.prepend, - append=self.append) + out = paddle.diff( + x, + n=self.n, + axis=self.axis, + prepend=self.prepend, + append=self.append, + ) self.assertTrue((out.numpy() == self.output).all(), True) def test_dygraph(self): @@ -85,36 +86,40 @@ class TestDiffOp(unittest.TestCase): places.append(fluid.CUDAPlace(0)) for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): - x = paddle.fluid.data(name="input", - shape=self.input.shape, - dtype=self.input.dtype) + x = paddle.fluid.data( + name="input", shape=self.input.shape, dtype=self.input.dtype + ) has_pend = False prepend = None append = None if self.prepend is not None: has_pend = True - prepend = paddle.fluid.data(name="prepend", - shape=self.prepend.shape, - dtype=self.prepend.dtype) + prepend = paddle.fluid.data( + name="prepend", + shape=self.prepend.shape, + dtype=self.prepend.dtype, + ) if self.append is not None: has_pend = True - append = paddle.fluid.data(name="append", - shape=self.append.shape, - dtype=self.append.dtype) + append = paddle.fluid.data( + name="append", + shape=self.append.shape, + dtype=self.append.dtype, + ) exe = fluid.Executor(place) - out = paddle.diff(x, - n=self.n, - axis=self.axis, - prepend=prepend, - append=append) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input": self.input, - "prepend": self.prepend, - "append": self.append - }, - fetch_list=[out]) + out = paddle.diff( + x, n=self.n, axis=self.axis, prepend=prepend, append=append + ) + fetches = exe.run( + fluid.default_main_program(), + feed={ + "input": self.input, + "prepend": self.prepend, + "append": self.append, + }, + fetch_list=[out], + ) self.assertTrue((fetches[0] == self.output).all(), True) def func_grad(self): @@ -124,11 +129,13 @@ class TestDiffOp(unittest.TestCase): self.prepend = paddle.to_tensor(self.prepend, place=place) if self.append is not None: self.append = paddle.to_tensor(self.append, place=place) - out = paddle.diff(x, - n=self.n, - axis=self.axis, - prepend=self.prepend, - append=self.append) + out = paddle.diff( + x, + n=self.n, + axis=self.axis, + prepend=self.prepend, + append=self.append, + ) try: out.backward() x_grad = x.grad @@ -144,7 +151,6 @@ class TestDiffOp(unittest.TestCase): class TestDiffOpAxis(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -154,7 +160,6 @@ class TestDiffOpAxis(TestDiffOp): class TestDiffOpNDim(TestDiffOp): - def set_args(self): self.input = np.random.rand(10, 10).astype('float32') self.n = 1 @@ -164,7 +169,6 @@ class TestDiffOpNDim(TestDiffOp): class TestDiffOpBool(TestDiffOp): - def set_args(self): self.input = np.array([0, 1, 1, 0, 1, 0]).astype('bool') self.n = 1 @@ -174,7 +178,6 @@ class TestDiffOpBool(TestDiffOp): class TestDiffOpPrepend(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -184,18 +187,17 @@ class TestDiffOpPrepend(TestDiffOp): class TestDiffOpPrependAxis(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 self.axis = 0 - self.prepend = np.array([[0, 2, 3, 4], [1, 3, 5, 7], - [2, 5, 8, 0]]).astype('float32') + self.prepend = np.array( + [[0, 2, 3, 4], [1, 3, 5, 7], [2, 5, 8, 0]] + ).astype('float32') self.append = None class TestDiffOpAppend(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -205,7 +207,6 @@ class TestDiffOpAppend(TestDiffOp): class TestDiffOpAppendAxis(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -215,7 +216,6 @@ class TestDiffOpAppendAxis(TestDiffOp): class TestDiffOpPreAppend(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -225,7 +225,6 @@ class TestDiffOpPreAppend(TestDiffOp): class TestDiffOpPreAppendAxis(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 diff --git a/python/paddle/fluid/tests/unittests/test_digamma_op.py b/python/paddle/fluid/tests/unittests/test_digamma_op.py index 546744485b0..ff9e2b18275 100644 --- a/python/paddle/fluid/tests/unittests/test_digamma_op.py +++ b/python/paddle/fluid/tests/unittests/test_digamma_op.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import _test_eager_guard class TestDigammaOp(OpTest): - def setUp(self): # switch to static paddle.enable_static() @@ -49,7 +48,6 @@ class TestDigammaOp(OpTest): class TestDigammaOpFp32(TestDigammaOp): - def init_dtype_type(self): self.dtype = np.float32 @@ -58,7 +56,6 @@ class TestDigammaOpFp32(TestDigammaOp): class TestDigammaAPI(unittest.TestCase): - def setUp(self): # switch to static paddle.enable_static() @@ -70,7 +67,6 @@ class TestDigammaAPI(unittest.TestCase): self._shape = [8, 3, 32, 32] def test_in_static_mode(self): - def init_input_output(dtype): input = np.random.random(self._shape).astype(dtype) return {'x': input}, psi(input) diff --git a/python/paddle/fluid/tests/unittests/test_directory_migration.py b/python/paddle/fluid/tests/unittests/test_directory_migration.py index 46a2c53a731..408ad42379e 100644 --- a/python/paddle/fluid/tests/unittests/test_directory_migration.py +++ b/python/paddle/fluid/tests/unittests/test_directory_migration.py @@ -20,7 +20,6 @@ import unittest class TestDirectory(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -38,14 +37,25 @@ class TestDirectory(unittest.TestCase): def test_new_directory(self): new_directory = [ - 'paddle.enable_static', 'paddle.disable_static', - 'paddle.in_dynamic_mode', 'paddle.to_tensor', 'paddle.grad', - 'paddle.no_grad', 'paddle.static.save', 'paddle.static.load', - 'paddle.distributed.ParallelEnv', 'paddle.DataParallel', - 'paddle.jit', 'paddle.jit.TracedLayer', 'paddle.jit.to_static', - 'paddle.jit.ProgramTranslator', 'paddle.jit.TranslatedLayer', - 'paddle.jit.save', 'paddle.jit.load', - 'paddle.optimizer.lr.LRScheduler', 'paddle.optimizer.lr.NoamDecay', + 'paddle.enable_static', + 'paddle.disable_static', + 'paddle.in_dynamic_mode', + 'paddle.to_tensor', + 'paddle.grad', + 'paddle.no_grad', + 'paddle.static.save', + 'paddle.static.load', + 'paddle.distributed.ParallelEnv', + 'paddle.DataParallel', + 'paddle.jit', + 'paddle.jit.TracedLayer', + 'paddle.jit.to_static', + 'paddle.jit.ProgramTranslator', + 'paddle.jit.TranslatedLayer', + 'paddle.jit.save', + 'paddle.jit.load', + 'paddle.optimizer.lr.LRScheduler', + 'paddle.optimizer.lr.NoamDecay', 'paddle.optimizer.lr.PiecewiseDecay', 'paddle.optimizer.lr.NaturalExpDecay', 'paddle.optimizer.lr.ExponentialDecay', @@ -53,30 +63,47 @@ class TestDirectory(unittest.TestCase): 'paddle.optimizer.lr.PolynomialDecay', 'paddle.optimizer.lr.CosineAnnealingDecay', 'paddle.optimizer.lr.MultiStepDecay', - 'paddle.optimizer.lr.StepDecay', 'paddle.optimizer.lr.LambdaDecay', + 'paddle.optimizer.lr.StepDecay', + 'paddle.optimizer.lr.LambdaDecay', 'paddle.optimizer.lr.ReduceOnPlateau', - 'paddle.optimizer.lr.LinearWarmup', 'paddle.static.Executor', - 'paddle.static.global_scope', 'paddle.static.scope_guard', - 'paddle.static.append_backward', 'paddle.static.gradients', - 'paddle.static.BuildStrategy', 'paddle.static.CompiledProgram', + 'paddle.optimizer.lr.LinearWarmup', + 'paddle.static.Executor', + 'paddle.static.global_scope', + 'paddle.static.scope_guard', + 'paddle.static.append_backward', + 'paddle.static.gradients', + 'paddle.static.BuildStrategy', + 'paddle.static.CompiledProgram', 'paddle.static.ExecutionStrategy', 'paddle.static.default_main_program', - 'paddle.static.default_startup_program', 'paddle.static.Program', - 'paddle.static.name_scope', 'paddle.static.program_guard', - 'paddle.static.Print', 'paddle.static.py_func', + 'paddle.static.default_startup_program', + 'paddle.static.Program', + 'paddle.static.name_scope', + 'paddle.static.program_guard', + 'paddle.static.Print', + 'paddle.static.py_func', 'paddle.static.ParallelExecutor', - 'paddle.static.WeightNormParamAttr', 'paddle.static.nn.fc', + 'paddle.static.WeightNormParamAttr', + 'paddle.static.nn.fc', 'paddle.static.nn.batch_norm', 'paddle.static.nn.bilinear_tensor_product', - 'paddle.static.nn.conv2d', 'paddle.static.nn.conv2d_transpose', - 'paddle.static.nn.conv3d', 'paddle.static.nn.conv3d_transpose', + 'paddle.static.nn.conv2d', + 'paddle.static.nn.conv2d_transpose', + 'paddle.static.nn.conv3d', + 'paddle.static.nn.conv3d_transpose', 'paddle.static.nn.create_parameter', - 'paddle.static.nn.crf_decoding', 'paddle.static.nn.data_norm', - 'paddle.static.nn.deform_conv2d', 'paddle.static.nn.group_norm', - 'paddle.static.nn.instance_norm', 'paddle.static.nn.layer_norm', - 'paddle.static.nn.multi_box_head', 'paddle.static.nn.nce', - 'paddle.static.nn.prelu', 'paddle.static.nn.row_conv', - 'paddle.static.nn.spectral_norm', 'paddle.static.nn.embedding' + 'paddle.static.nn.crf_decoding', + 'paddle.static.nn.data_norm', + 'paddle.static.nn.deform_conv2d', + 'paddle.static.nn.group_norm', + 'paddle.static.nn.instance_norm', + 'paddle.static.nn.layer_norm', + 'paddle.static.nn.multi_box_head', + 'paddle.static.nn.nce', + 'paddle.static.nn.prelu', + 'paddle.static.nn.row_conv', + 'paddle.static.nn.spectral_norm', + 'paddle.static.nn.embedding', ] import_file = os.path.join(self.temp_dir.name, 'run_import_modules.py') @@ -89,58 +116,89 @@ class TestDirectory(unittest.TestCase): _python = sys.executable ps_cmd = "{} {}".format(_python, import_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) stdout, stderr = ps_proc.communicate() - self.assertFalse("Error" in str(stderr), - "ErrorMessage:\n{}".format(bytes.decode(stderr))) + self.assertFalse( + "Error" in str(stderr), + "ErrorMessage:\n{}".format(bytes.decode(stderr)), + ) def test_old_directory(self): old_directory = [ - 'paddle.enable_imperative', 'paddle.disable_imperative', - 'paddle.in_imperative_mode', 'paddle.imperative.to_variable', - 'paddle.imperative.enable', 'paddle.imperative.guard', - 'paddle.imperative.grad', 'paddle.imperative.no_grad', - 'paddle.imperative.save', 'paddle.imperative.load', + 'paddle.enable_imperative', + 'paddle.disable_imperative', + 'paddle.in_imperative_mode', + 'paddle.imperative.to_variable', + 'paddle.imperative.enable', + 'paddle.imperative.guard', + 'paddle.imperative.grad', + 'paddle.imperative.no_grad', + 'paddle.imperative.save', + 'paddle.imperative.load', 'paddle.imperative.ParallelEnv', 'paddle.imperative.prepare_context', - 'paddle.imperative.DataParalell', 'paddle.imperative.jit', - 'paddle.imperative.TracedLayer', 'paddle.imperative.declarative', + 'paddle.imperative.DataParalell', + 'paddle.imperative.jit', + 'paddle.imperative.TracedLayer', + 'paddle.imperative.declarative', 'paddle.imperative.ProgramTranslator', - 'paddle.imperative.TranslatedLayer', 'paddle.imperative.jit.save', - 'paddle.imperative.jit.load', 'paddle.imperative.NoamDecay' - 'paddle.imperative.PiecewiseDecay', + 'paddle.imperative.TranslatedLayer', + 'paddle.imperative.jit.save', + 'paddle.imperative.jit.load', + 'paddle.imperative.NoamDecay' 'paddle.imperative.PiecewiseDecay', 'paddle.imperative.NaturalExpDecay', 'paddle.imperative.ExponentialDecay', 'paddle.imperative.InverseTimeDecay', 'paddle.imperative.PolynomialDecay', - 'paddle.imperative.CosineDecay', 'paddle.Executor', - 'paddle.global_scope', 'paddle.scope_guard', - 'paddle.append_backward', 'paddle.gradients', - 'paddle.BuildStrategy', 'paddle.CompiledProgram', - 'paddle.ExecutionStrategy', 'paddle.name_scope', - 'paddle.program_guard', 'paddle.Print', 'paddle.py_func', - 'paddle.ParallelExecutor', 'paddle.default_main_program', - 'paddle.default_startup_program', 'paddle.Program', - 'paddle.WeightNormParamAttr', 'paddle.declarative.fc', + 'paddle.imperative.CosineDecay', + 'paddle.Executor', + 'paddle.global_scope', + 'paddle.scope_guard', + 'paddle.append_backward', + 'paddle.gradients', + 'paddle.BuildStrategy', + 'paddle.CompiledProgram', + 'paddle.ExecutionStrategy', + 'paddle.name_scope', + 'paddle.program_guard', + 'paddle.Print', + 'paddle.py_func', + 'paddle.ParallelExecutor', + 'paddle.default_main_program', + 'paddle.default_startup_program', + 'paddle.Program', + 'paddle.WeightNormParamAttr', + 'paddle.declarative.fc', 'paddle.declarative.batch_norm', 'paddle.declarative.bilinear_tensor_product', - 'paddle.declarative.conv2d', 'paddle.declarative.conv2d_transpose', - 'paddle.declarative.conv3d', 'paddle.declarative.conv3d_transpose', + 'paddle.declarative.conv2d', + 'paddle.declarative.conv2d_transpose', + 'paddle.declarative.conv3d', + 'paddle.declarative.conv3d_transpose', 'paddle.declarative.create_parameter', - 'paddle.declarative.crf_decoding', 'paddle.declarative.data_norm', + 'paddle.declarative.crf_decoding', + 'paddle.declarative.data_norm', 'paddle.declarative.deformable_conv', - 'paddle.declarative.group_norm', 'paddle.declarative.hsigmoid', - 'paddle.declarative.instance_norm', 'paddle.declarative.layer_norm', - 'paddle.declarative.multi_box_head', 'paddle.declarative.nce', - 'paddle.declarative.prelu', 'paddle.declarative.row_conv', - 'paddle.declarative.spectral_norm', 'paddle.declarative.embedding' + 'paddle.declarative.group_norm', + 'paddle.declarative.hsigmoid', + 'paddle.declarative.instance_norm', + 'paddle.declarative.layer_norm', + 'paddle.declarative.multi_box_head', + 'paddle.declarative.nce', + 'paddle.declarative.prelu', + 'paddle.declarative.row_conv', + 'paddle.declarative.spectral_norm', + 'paddle.declarative.embedding', ] - import_file = os.path.join(self.temp_dir.name, - 'run_old_import_modules.py') + import_file = os.path.join( + self.temp_dir.name, 'run_old_import_modules.py' + ) with open(import_file, "w") as wb: cmd_context_count = """ @@ -159,22 +217,26 @@ else: err_module = "{module}" """ cmd_context_loop = cmd_context_loop_template.format( - run_cmd=run_cmd, module=module) + run_cmd=run_cmd, module=module + ) wb.write(cmd_context_loop) cmd_context_print_template = """ if count != {len_old_directory}: print("Error: Module " + err_module + " should not be imported") """ cmd_context_print = cmd_context_print_template.format( - len_old_directory=str(len(old_directory))) + len_old_directory=str(len(old_directory)) + ) wb.write(cmd_context_print) _python = sys.executable ps_cmd = "{} {}".format(_python, import_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) stdout, stderr = ps_proc.communicate() self.assertFalse("Error" in str(stdout), bytes.decode(stdout)) diff --git a/python/paddle/fluid/tests/unittests/test_disable_signal_handler.py b/python/paddle/fluid/tests/unittests/test_disable_signal_handler.py index 50867ad8b0e..f0de021eaad 100644 --- a/python/paddle/fluid/tests/unittests/test_disable_signal_handler.py +++ b/python/paddle/fluid/tests/unittests/test_disable_signal_handler.py @@ -17,21 +17,27 @@ import signal import subprocess SignalsToTest = { - signal.SIGTERM, signal.SIGBUS, signal.SIGABRT, signal.SIGSEGV, - signal.SIGILL, signal.SIGFPE + signal.SIGTERM, + signal.SIGBUS, + signal.SIGABRT, + signal.SIGSEGV, + signal.SIGILL, + signal.SIGFPE, } class TestSignOpError(unittest.TestCase): - def test_errors(self): try: for sig in SignalsToTest: - output = subprocess.check_output([ - "python", "-c", - f"import paddle; import signal,os; paddle.disable_signal_handler(); os.kill(os.getpid(), {sig})" - ], - stderr=subprocess.STDOUT) + output = subprocess.check_output( + [ + "python", + "-c", + f"import paddle; import signal,os; paddle.disable_signal_handler(); os.kill(os.getpid(), {sig})", + ], + stderr=subprocess.STDOUT, + ) except Exception as e: # If paddle signal handler is enabled # One would expect "paddle::framework::SignalHandle" in STDERR diff --git a/python/paddle/fluid/tests/unittests/test_dist_allreduce_op.py b/python/paddle/fluid/tests/unittests/test_dist_allreduce_op.py index af2c8f1aee7..c2469ab92b7 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_allreduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_dist_allreduce_op.py @@ -20,7 +20,6 @@ paddle.enable_static() class TestDistMnistNCCL2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -30,6 +29,7 @@ class TestDistMnistNCCL2(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_allreduce_op.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_base.py b/python/paddle/fluid/tests/unittests/test_dist_base.py index d5287270769..4c109feaef2 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_base.py @@ -53,26 +53,30 @@ def eprint(*args, **kwargs): class TestDistRunnerBase(object): - - def get_model(self, - batch_size=DEFAULT_BATCH_SIZE, - lr=0.1, - single_device=False, - use_dgc=False, - dist_strategy=None): + def get_model( + self, + batch_size=DEFAULT_BATCH_SIZE, + lr=0.1, + single_device=False, + use_dgc=False, + dist_strategy=None, + ): raise NotImplementedError( - "get_model should be implemented by child classes.") + "get_model should be implemented by child classes." + ) @staticmethod - def get_transpiler(trainer_id, - main_program, - pserver_endpoints, - trainers, - sync_mode, - dc_asgd=False, - current_endpoint=None, - nccl_comm_num=1, - hogwild_mode=False): + def get_transpiler( + trainer_id, + main_program, + pserver_endpoints, + trainers, + sync_mode, + dc_asgd=False, + current_endpoint=None, + nccl_comm_num=1, + hogwild_mode=False, + ): # NOTE: import fluid until runtime, or else forking processes will cause error. config = fluid.DistributeTranspilerConfig() config.enable_dc_asgd = dc_asgd @@ -83,12 +87,14 @@ class TestDistRunnerBase(object): config.nccl_comm_num = nccl_comm_num # config.runtime_split_send_recv = True t = fluid.DistributeTranspiler(config=config) - t.transpile(trainer_id=trainer_id, - program=main_program, - pservers=pserver_endpoints, - trainers=trainers, - sync_mode=sync_mode, - current_endpoint=current_endpoint) + t.transpile( + trainer_id=trainer_id, + program=main_program, + pservers=pserver_endpoints, + trainers=trainers, + sync_mode=sync_mode, + current_endpoint=current_endpoint, + ) return t @staticmethod @@ -96,6 +102,7 @@ class TestDistRunnerBase(object): lr_sheduler = None if hasattr(program, 'lr_sheduler'): from paddle.optimizer.lr import LRScheduler + lr_sheduler = program.lr_sheduler assert isinstance(lr_sheduler, LRScheduler), "must be LRScheduler" return lr_sheduler @@ -105,16 +112,19 @@ class TestDistRunnerBase(object): self.get_model(batch_size=args.batch_size) # NOTE: pserver should not call memory optimize - t = self.get_transpiler(trainer_id=args.trainer_id, - main_program=fluid.default_main_program(), - pserver_endpoints=args.endpoints, - trainers=args.trainers, - sync_mode=args.sync_mode, - dc_asgd=args.dc_asgd, - hogwild_mode=args.hogwild) + t = self.get_transpiler( + trainer_id=args.trainer_id, + main_program=fluid.default_main_program(), + pserver_endpoints=args.endpoints, + trainers=args.trainers, + sync_mode=args.sync_mode, + dc_asgd=args.dc_asgd, + hogwild_mode=args.hogwild, + ) pserver_prog = t.get_pserver_program(args.current_endpoint) - startup_prog = t.get_startup_program(args.current_endpoint, - pserver_prog) + startup_prog = t.get_startup_program( + args.current_endpoint, pserver_prog + ) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -127,8 +137,17 @@ class TestDistRunnerBase(object): self.lr = args.lr dist_strategy = DistributedStrategy() - test_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader = \ - self.get_model(batch_size=args.batch_size, dist_strategy=dist_strategy) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + data_loader, + ) = self.get_model( + batch_size=args.batch_size, dist_strategy=dist_strategy + ) device_id = int(os.getenv("FLAGS_selected_gpus", "0")) eprint(type(self).__name__, "device_id: %d." % device_id) @@ -169,8 +188,14 @@ class TestDistRunnerBase(object): self.lr = args.lr print_to_err("use_fleet 2.0", "fleet.node_num:") - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=args.batch_size) if fluid.core.is_compiled_with_cuda(): device_id = int(os.getenv("FLAGS_selected_gpus", "0")) @@ -188,8 +213,8 @@ class TestDistRunnerBase(object): eprint(type(self).__name__, "run worker startup program done.") feed_var_list = [ - var for var in - fluid.default_main_program().global_block().vars.values() + var + for var in fluid.default_main_program().global_block().vars.values() if var.is_data ] @@ -203,8 +228,10 @@ class TestDistRunnerBase(object): def get_data(): origin_batch = next(reader_generator) - if paddle.distributed.get_world_size( - ) == 1 and args.update_method == 'gloo': # Gloo single mode + if ( + paddle.distributed.get_world_size() == 1 + and args.update_method == 'gloo' + ): # Gloo single mode return origin_batch elif args.update_method != "local" and args.use_reader_alloc: @@ -219,9 +246,11 @@ class TestDistRunnerBase(object): print_to_err(type(self).__name__, "begin to train on trainer") out_losses = [] for i in range(RUN_STEP): - loss, = exe.run(fluid.default_main_program(), - fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) + (loss,) = exe.run( + fluid.default_main_program(), + fetch_list=[avg_cost.name], + feed=feeder.feed(get_data()), + ) out_losses.append(loss[0]) print_to_err(type(self).__name__, "run step %d finished" % i) print_to_err(type(self).__name__, "trainer run finished") @@ -254,8 +283,16 @@ class TestDistRunnerBase(object): # "fleet.node_id:", fleet.node_id(), # "fleet.trainer_num:", fleet.worker_num()) - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size, dist_strategy=dist_strategy) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model( + batch_size=args.batch_size, dist_strategy=dist_strategy + ) trainer_prog = fleet._origin_program dist_prog = fleet.main_program @@ -276,7 +313,8 @@ class TestDistRunnerBase(object): eprint(type(self).__name__, "run worker startup program done.") feed_var_list = [ - var for var in trainer_prog.global_block().vars.values() + var + for var in trainer_prog.global_block().vars.values() if var.is_data ] @@ -304,9 +342,11 @@ class TestDistRunnerBase(object): print_to_err(type(self).__name__, "begin to train on trainer") out_losses = [] for i in range(RUN_STEP): - loss, = exe.run(dist_prog, - fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) + (loss,) = exe.run( + dist_prog, + fetch_list=[avg_cost.name], + feed=feeder.feed(get_data()), + ) out_losses.append(loss[0]) print_to_err(type(self).__name__, "run step %d finished" % i) print_to_err(type(self).__name__, "trainer run finished") @@ -316,86 +356,131 @@ class TestDistRunnerBase(object): if args.save_model: model_save_dir = "/tmp" if fleet.worker_index() == 0: - model_save_dir_fluid = os.path.join(model_save_dir, - "fluid_persistables") - model_save_dir_fleet = os.path.join(model_save_dir, - "fleet_persistables") - infer_save_dir_fluid = os.path.join(model_save_dir, - "fluid_infer") - infer_save_dir_fleet = os.path.join(model_save_dir, - "fleet_infer") + model_save_dir_fluid = os.path.join( + model_save_dir, "fluid_persistables" + ) + model_save_dir_fleet = os.path.join( + model_save_dir, "fleet_persistables" + ) + infer_save_dir_fluid = os.path.join( + model_save_dir, "fluid_infer" + ) + infer_save_dir_fleet = os.path.join( + model_save_dir, "fleet_infer" + ) else: - model_save_dir_fluid = os.path.join(model_save_dir, - "fluid_persistables_2") - model_save_dir_fleet = os.path.join(model_save_dir, - "fleet_persistables_2") - infer_save_dir_fluid = os.path.join(model_save_dir, - "fluid_infer_2") - infer_save_dir_fleet = os.path.join(model_save_dir, - "fleet_infer_2") - fluid.io.save_persistables(exe, model_save_dir_fluid, - fleet._origin_program) + model_save_dir_fluid = os.path.join( + model_save_dir, "fluid_persistables_2" + ) + model_save_dir_fleet = os.path.join( + model_save_dir, "fleet_persistables_2" + ) + infer_save_dir_fluid = os.path.join( + model_save_dir, "fluid_infer_2" + ) + infer_save_dir_fleet = os.path.join( + model_save_dir, "fleet_infer_2" + ) + fluid.io.save_persistables( + exe, model_save_dir_fluid, fleet._origin_program + ) fleet.save_persistables(executor=exe, dirname=model_save_dir_fleet) feeded_var_names = [var.name for var in feed_var_list] - fluid.io.save_inference_model(infer_save_dir_fluid, - feeded_var_names, [avg_cost], exe, - fleet._origin_program) - fleet.save_inference_model(exe, infer_save_dir_fleet, - feeded_var_names, [avg_cost]) + fluid.io.save_inference_model( + infer_save_dir_fluid, + feeded_var_names, + [avg_cost], + exe, + fleet._origin_program, + ) + fleet.save_inference_model( + exe, infer_save_dir_fleet, feeded_var_names, [avg_cost] + ) def run_trainer(self, args): self.lr = args.lr if args.nccl2_reduce_layer_local_run: - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size, single_device=True) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=args.batch_size, single_device=True) elif args.use_dgc: - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size, use_dgc=args.use_dgc) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=args.batch_size, use_dgc=args.use_dgc) else: - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=args.batch_size) if args.update_method == "pserver": print_to_err( type(self).__name__, - "begin to run transpile on trainer with pserver mode") - t = self.get_transpiler(trainer_id=args.trainer_id, - main_program=fluid.default_main_program(), - pserver_endpoints=args.endpoints, - trainers=args.trainers, - sync_mode=args.sync_mode, - dc_asgd=args.dc_asgd, - hogwild_mode=args.hogwild) + "begin to run transpile on trainer with pserver mode", + ) + t = self.get_transpiler( + trainer_id=args.trainer_id, + main_program=fluid.default_main_program(), + pserver_endpoints=args.endpoints, + trainers=args.trainers, + sync_mode=args.sync_mode, + dc_asgd=args.dc_asgd, + hogwild_mode=args.hogwild, + ) trainer_prog = t.get_trainer_program() print_to_err( type(self).__name__, - "get trainer program done with pserver mode.") - elif args.update_method == "nccl2" or args.update_method == "nccl2_reduce_layer": + "get trainer program done with pserver mode.", + ) + elif ( + args.update_method == "nccl2" + or args.update_method == "nccl2_reduce_layer" + ): # transpile for nccl2 config = fluid.DistributeTranspilerConfig() config.mode = "nccl2" config.nccl_comm_num = args.nccl_comm_num if args.use_hallreduce: config.use_hierarchical_allreduce = True - config.hierarchical_allreduce_inter_nranks = args.hallreduce_inter_nranks + config.hierarchical_allreduce_inter_nranks = ( + args.hallreduce_inter_nranks + ) print_to_err( type(self).__name__, - "begin to run transpile on trainer with nccl2 mode") + "begin to run transpile on trainer with nccl2 mode", + ) nccl2_t = fluid.DistributeTranspiler(config=config) - nccl2_t.transpile(args.trainer_id, - program=fluid.default_main_program(), - startup_program=fluid.default_startup_program(), - trainers=args.endpoints, - current_endpoint=args.current_endpoint) + nccl2_t.transpile( + args.trainer_id, + program=fluid.default_main_program(), + startup_program=fluid.default_startup_program(), + trainers=args.endpoints, + current_endpoint=args.current_endpoint, + ) print_to_err( - type(self).__name__, - "get trainer program done. with nccl2 mode") + type(self).__name__, "get trainer program done. with nccl2 mode" + ) trainer_prog = fluid.default_main_program() else: print_to_err( type(self).__name__, - "do nothing about main program, just use it") + "do nothing about main program, just use it", + ) trainer_prog = fluid.default_main_program() print_to_err(type(self).__name__, "use main program done.") @@ -431,9 +516,13 @@ class TestDistRunnerBase(object): build_stra.enable_backward_optimizer_op_deps = True if args.use_reduce: - build_stra.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + build_stra.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) else: - build_stra.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce + build_stra.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.AllReduce + ) pass_builder = None if args.batch_merge_repeat > 1: @@ -441,7 +530,10 @@ class TestDistRunnerBase(object): mypass = pass_builder.insert_pass(0, "multi_batch_merge_pass") mypass.set("num_repeats", args.batch_merge_repeat) - if args.update_method == "nccl2" or args.update_method == "nccl2_reduce_layer": + if ( + args.update_method == "nccl2" + or args.update_method == "nccl2_reduce_layer" + ): build_stra.num_trainers = len(args.endpoints.split(",")) build_stra.trainer_id = args.trainer_id else: @@ -453,11 +545,13 @@ class TestDistRunnerBase(object): binary = compiler.CompiledProgram(trainer_prog).with_data_parallel( loss_name=avg_cost.name, build_strategy=build_stra, - exec_strategy=exec_strategy) + exec_strategy=exec_strategy, + ) print_to_err(type(self).__name__, "program compiled with data parallel") feed_var_list = [ - var for var in trainer_prog.global_block().vars.values() + var + for var in trainer_prog.global_block().vars.values() if var.is_data ] @@ -479,9 +573,9 @@ class TestDistRunnerBase(object): print_to_err(type(self).__name__, "begin to train on trainer") out_losses = [] for i in range(RUN_STEP): - loss, = exe.run(binary, - fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) + (loss,) = exe.run( + binary, fetch_list=[avg_cost.name], feed=feeder.feed(get_data()) + ) out_losses.append(loss[0]) print_to_err(type(self).__name__, "run step %d finished" % i) if lr_scheduler is not None: @@ -493,18 +587,21 @@ class TestDistRunnerBase(object): class TestParallelDyGraphRunnerBase(object): - def get_model(self): raise NotImplementedError( - "get_model should be implemented by child classes.") + "get_model should be implemented by child classes." + ) def run_one_loop(self, model, opt, data): raise NotImplementedError( - "train_one_loop should be implemented by the child classes.") + "train_one_loop should be implemented by the child classes." + ) def _get_data(self, batch, args): - if paddle.distributed.get_world_size( - ) == 1 and args.update_method == 'gloo': # Gloo single mode + if ( + paddle.distributed.get_world_size() == 1 + and args.update_method == 'gloo' + ): # Gloo single mode return batch elif args.update_method != "local": new_batch = [] @@ -514,8 +611,9 @@ class TestParallelDyGraphRunnerBase(object): # the second rank will get [3,4,5]. # this function is for test sparse_embedding_differ_length if hasattr(args, "diff_batch") and args.diff_batch: - assert len( - batch) > 2, "in differ_batch mode, len(batch) must > 2." + assert ( + len(batch) > 2 + ), "in differ_batch mode, len(batch) must > 2." if paddle.distributed.get_rank() == 0: new_batch.append(batch[0]) elif paddle.distributed.get_rank() == 1: @@ -550,19 +648,25 @@ class TestParallelDyGraphRunnerBase(object): device_id = int(os.getenv("FLAGS_selected_mlus", "0")) place = fluid.MLUPlace(device_id) else: - assert ("Only support CUDAPlace or XPUPlace or CPU(Gloo) for now.") + assert "Only support CUDAPlace or XPUPlace or CPU(Gloo) for now." with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed np.random.seed(seed) import random + random.seed(seed) model, train_reader, opt = self.get_model() nranks = len(args.endpoints.split(",")) if args.endpoints else 1 - #if args.update_method == "nccl2": - if args.update_method == "nccl2" or args.update_method == "bkcl" or args.update_method == "hccl" or args.update_method == "cncl": + # if args.update_method == "nccl2": + if ( + args.update_method == "nccl2" + or args.update_method == "bkcl" + or args.update_method == "hccl" + or args.update_method == "cncl" + ): strategy = dygraph.parallel.ParallelStrategy() strategy.nranks = nranks strategy.local_rank = args.trainer_id @@ -571,24 +675,29 @@ class TestParallelDyGraphRunnerBase(object): paddle.distributed.init_parallel_env() print_to_err( type(self).__name__, - "begin to prepare context in dygraph with nccl2") + "begin to prepare context in dygraph with nccl2", + ) dygraph.parallel.prepare_context(strategy) if not args.find_unused_parameters: model = dygraph.parallel.DataParallel( - model, strategy, find_unused_parameters=False) + model, strategy, find_unused_parameters=False + ) else: model = dygraph.parallel.DataParallel( - model, strategy, find_unused_parameters=True) + model, strategy, find_unused_parameters=True + ) print_to_err(type(self).__name__, "model built in dygraph") elif args.update_method == "gloo": paddle.distributed.init_parallel_env() if not args.find_unused_parameters: model = dygraph.parallel.DataParallel( - model, find_unused_parameters=False) + model, find_unused_parameters=False + ) else: model = dygraph.parallel.DataParallel( - model, find_unused_parameters=True) + model, find_unused_parameters=True + ) out_losses = [] print_to_err(type(self).__name__, "begin to run dygraph training") @@ -600,7 +709,8 @@ class TestParallelDyGraphRunnerBase(object): if step_id % 10 == 0: print_to_err( type(self).__name__, - "loss at step %d: %f" % (step_id, loss.numpy())) + "loss at step %d: %f" % (step_id, loss.numpy()), + ) out_losses.append(loss.numpy()) loss.backward() @@ -632,7 +742,8 @@ class TestParallelDyGraphRunnerBase(object): model, train_reader, opt = self.get_model() if args.update_method in ["nccl2", "gloo"]: model = paddle.DataParallel( - model, find_unused_parameters=args.find_unused_parameters) + model, find_unused_parameters=args.find_unused_parameters + ) out_losses = [] for step_id, data in enumerate(train_reader()): @@ -650,6 +761,7 @@ class TestParallelDyGraphRunnerBase(object): def run_use_fleet_api_trainer(self, args): import paddle.distributed.fleet as fleet + # 1. enable dygraph paddle.disable_static() @@ -696,18 +808,25 @@ class TestParallelDyGraphRunnerBase(object): def runtime_main(test_class): parser = argparse.ArgumentParser(description='Run dist test.') - parser.add_argument('--role', - type=str, - required=True, - choices=['pserver', 'trainer']) + parser.add_argument( + '--role', type=str, required=True, choices=['pserver', 'trainer'] + ) parser.add_argument('--endpoints', type=str, required=False, default="") - parser.add_argument('--update_method', - type=str, - default="local", - choices=[ - "pserver", "nccl2", "bkcl", "local", - "nccl2_reduce_layer", "gloo", "hccl", "cncl" - ]) + parser.add_argument( + '--update_method', + type=str, + default="local", + choices=[ + "pserver", + "nccl2", + "bkcl", + "local", + "nccl2_reduce_layer", + "gloo", + "hccl", + "cncl", + ], + ) parser.add_argument('--trainer_id', type=int, required=False, default=0) parser.add_argument('--trainers', type=int, required=False, default=1) parser.add_argument('--nccl_comm_num', type=int, required=False, default=1) @@ -719,14 +838,12 @@ def runtime_main(test_class): parser.add_argument('--use_local_sgd', action='store_true') parser.add_argument('--diff_batch', action='store_true') parser.add_argument('--ut4grad_allreduce', action='store_true') - parser.add_argument('--hallreduce_inter_nranks', - type=int, - required=False, - default=2) - parser.add_argument('--current_endpoint', - type=str, - required=False, - default="") + parser.add_argument( + '--hallreduce_inter_nranks', type=int, required=False, default=2 + ) + parser.add_argument( + '--current_endpoint', type=str, required=False, default="" + ) parser.add_argument('--sync_mode', action='store_true') parser.add_argument('--use_cuda', action='store_true') parser.add_argument('--use_cpu', action='store_true') @@ -740,24 +857,24 @@ def runtime_main(test_class): parser.add_argument('--dc_asgd', action='store_true') parser.add_argument('--hogwild', action='store_true') parser.add_argument('--save_model', action='store_true') - parser.add_argument('--use_reader_alloc', - action='store_true', - required=False) + parser.add_argument( + '--use_reader_alloc', action='store_true', required=False + ) parser.add_argument('--batch_size', required=False, type=int, default=2) parser.add_argument('--lr', required=False, type=float, default=0.001) - parser.add_argument('--batch_merge_repeat', - required=False, - type=int, - default=1) - parser.add_argument('--nccl2_reduce_layer_local_run', - required=False, - type=bool, - default=False) + parser.add_argument( + '--batch_merge_repeat', required=False, type=int, default=1 + ) + parser.add_argument( + '--nccl2_reduce_layer_local_run', + required=False, + type=bool, + default=False, + ) parser.add_argument('--sync_batch_norm', action='store_true') - parser.add_argument('--fuse_all_reduce', - required=False, - type=ast.literal_eval, - default=None) + parser.add_argument( + '--fuse_all_reduce', required=False, type=ast.literal_eval, default=None + ) args = parser.parse_args() @@ -782,7 +899,6 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def _setup_config(self): raise NotImplementedError("tests should have _setup_config implemented") @@ -872,10 +988,14 @@ class TestDistBase(unittest.TestCase): if DIST_UT_PORT == 0: self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) else: self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT, DIST_UT_PORT + 1) + DIST_UT_PORT, + DIST_UT_PORT + 1, + ) DIST_UT_PORT += 2 self._dist_port = DIST_UT_PORT @@ -887,13 +1007,14 @@ class TestDistBase(unittest.TestCase): self.temp_dir.cleanup() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) print_to_err( - type(self).__name__, "socket name: %s" % s.getsockname()[1]) + type(self).__name__, "socket name: %s" % s.getsockname()[1] + ) return s.getsockname()[1] while True: @@ -902,11 +1023,9 @@ class TestDistBase(unittest.TestCase): self._port_set.add(port) return port - def start_pserver(self, - model_file, - check_error_log, - required_envs, - log_name=""): + def start_pserver( + self, model_file, check_error_log, required_envs, log_name="" + ): ps0_ep, ps1_ep = self._ps_endpoints.split(",") ps_cmd = "%s" @@ -916,12 +1035,20 @@ class TestDistBase(unittest.TestCase): ps_cmd += " %s --role pserver --endpoints %s --trainer_id 0 --current_endpoint %s --trainers %d --update_method pserver" - ps0_cmd = ps_cmd % \ - (self._python_interp, model_file, self._ps_endpoints, ps0_ep, - self._trainers) - ps1_cmd = ps_cmd % \ - (self._python_interp, model_file, self._ps_endpoints, ps1_ep, - self._trainers) + ps0_cmd = ps_cmd % ( + self._python_interp, + model_file, + self._ps_endpoints, + ps0_ep, + self._trainers, + ) + ps1_cmd = ps_cmd % ( + self._python_interp, + model_file, + self._ps_endpoints, + ps1_ep, + self._trainers, + ) if self._sync_mode: ps0_cmd += " --sync_mode" @@ -935,26 +1062,32 @@ class TestDistBase(unittest.TestCase): ps1_pipe = open(path1, "wb") print_to_err(type(self).__name__, "going to start pserver process 0") - ps0_proc = subprocess.Popen(ps0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps0_pipe, - env=required_envs) + ps0_proc = subprocess.Popen( + ps0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps0_pipe, + env=required_envs, + ) print_to_err(type(self).__name__, "going to start pserver process 1") - ps1_proc = subprocess.Popen(ps1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps1_pipe, - env=required_envs) + ps1_proc = subprocess.Popen( + ps1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps1_pipe, + env=required_envs, + ) return ps0_proc, ps1_proc, ps0_pipe, ps1_pipe - def _run_local(self, - model, - envs, - check_error_log=False, - batch_size=DEFAULT_BATCH_SIZE, - batch_merge_repeat=1, - log_name="", - devices="1"): + def _run_local( + self, + model, + envs, + check_error_log=False, + batch_size=DEFAULT_BATCH_SIZE, + batch_merge_repeat=1, + log_name="", + devices="1", + ): cmd = self._python_interp @@ -962,8 +1095,10 @@ class TestDistBase(unittest.TestCase): envs['COVERAGE_FILE'] = os.getenv('COVERAGE_FILE', '') cmd += " -m coverage run --branch -p" - cmd += " %s --role trainer --update_method local --lr %f" % (model, - self._lr) + cmd += " %s --role trainer --update_method local --lr %f" % ( + model, + self._lr, + ) if batch_size != DEFAULT_BATCH_SIZE: cmd += " --batch_size %d" % batch_size @@ -977,21 +1112,21 @@ class TestDistBase(unittest.TestCase): env_local = { "CUDA_VISIBLE_DEVICES": devices, "PADDLE_TRAINERS_NUM": "1", - "PADDLE_TRAINER_ID": "0" + "PADDLE_TRAINER_ID": "0", } elif self.__use_xpu: cmd += " --use_xpu" env_local = { "FLAGS_selected_xpus": devices, "PADDLE_TRAINERS_NUM": "1", - "PADDLE_TRAINER_ID": "0" + "PADDLE_TRAINER_ID": "0", } elif self.__use_npu: cmd += " --use_npu" env_local = { "FLAGS_selected_npus": devices, "PADDLE_TRAINERS_NUM": "1", - "PADDLE_TRAINER_ID": "0" + "PADDLE_TRAINER_ID": "0", } else: env_local = {'CPU_NUM': '1'} @@ -1012,15 +1147,19 @@ class TestDistBase(unittest.TestCase): if check_error_log: path = os.path.join(self.temp_dir.name, log_name + "_local.log") err_log = open(path, "wb") - local_proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=err_log, - env=env_local) + local_proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=err_log, + env=env_local, + ) else: - local_proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env_local) + local_proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env_local, + ) local_out, local_err = local_proc.communicate() @@ -1032,27 +1171,29 @@ class TestDistBase(unittest.TestCase): return pickle.loads(local_out) - def _run_local_gloo(self, - model, - envs, - check_error_log=False, - batch_size=DEFAULT_BATCH_SIZE, - batch_merge_repeat=1, - log_name="", - devices="0"): + def _run_local_gloo( + self, + model, + envs, + check_error_log=False, + batch_size=DEFAULT_BATCH_SIZE, + batch_merge_repeat=1, + log_name="", + devices="0", + ): saved_endpoints = self._ps_endpoints self._ps_endpoints = self._ps_endpoints.split(',')[0] - result = self._run_cluster_gloo(model, envs, 'gloo', check_error_log, - log_name) + result = self._run_cluster_gloo( + model, envs, 'gloo', check_error_log, log_name + ) self._ps_endpoints = saved_endpoints return result def _run_cluster(self, model, envs, check_error_log, log_name): # Run dist train to compare with local results - ps0, ps1, ps0_pipe, ps1_pipe = self.start_pserver(model, - check_error_log, - envs, - log_name=log_name) + ps0, ps1, ps0_pipe, ps1_pipe = self.start_pserver( + model, check_error_log, envs, log_name=log_name + ) ps0_ep, ps1_ep = self._ps_endpoints.split(",") @@ -1064,12 +1205,24 @@ class TestDistBase(unittest.TestCase): tr_cmd += " %s --role trainer --endpoints %s --trainer_id %d --current_endpoint %s --trainers %d --update_method pserver --lr %f" - tr0_cmd = tr_cmd % \ - (self._python_interp, model, self._ps_endpoints, - 0, ps0_ep, self._trainers, self._lr) - tr1_cmd = tr_cmd % \ - (self._python_interp, model, self._ps_endpoints, - 1, ps1_ep, self._trainers, self._lr) + tr0_cmd = tr_cmd % ( + self._python_interp, + model, + self._ps_endpoints, + 0, + ps0_ep, + self._trainers, + self._lr, + ) + tr1_cmd = tr_cmd % ( + self._python_interp, + model, + self._ps_endpoints, + 1, + ps1_ep, + self._trainers, + self._lr, + ) if self._sync_mode: tr0_cmd += " --sync_mode" @@ -1104,15 +1257,19 @@ class TestDistBase(unittest.TestCase): tr1_pipe = open(path1, "wb") print_to_err(type(self).__name__, "going to start trainer process 0") - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) print_to_err(type(self).__name__, "going to start trainer process 1") - tr1_proc = subprocess.Popen(tr1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr1_proc = subprocess.Popen( + tr1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) # Wait until trainer process terminate while True: @@ -1140,8 +1297,9 @@ class TestDistBase(unittest.TestCase): return pickle.loads(tr0_out), pickle.loads(tr1_out) - def _get_gloo_trainer_cmd(self, model, ep, update_method, trainer_id, - trainer_num): + def _get_gloo_trainer_cmd( + self, model, ep, update_method, trainer_id, trainer_num + ): env = {} tr_cmd = "%s -u" @@ -1150,16 +1308,22 @@ class TestDistBase(unittest.TestCase): tr_cmd += " %s --role trainer --endpoints %s --trainer_id %d --current_endpoint %s --update_method %s --lr %f" - tr_cmd = tr_cmd % \ - (self._python_interp, model, self._ps_endpoints, - trainer_id, ep, update_method, self._lr) + tr_cmd = tr_cmd % ( + self._python_interp, + model, + self._ps_endpoints, + trainer_id, + ep, + update_method, + self._lr, + ) if self._use_reduce: tr_cmd += " --use_reduce" if self._use_reader_alloc: tr_cmd += " --use_reader_alloc" - #assert self._use_reduce == False, "gloo not support _use_reduce" - #assert self._use_reader_alloc == False, "gloo not support _use_reduce" + # assert self._use_reduce == False, "gloo not support _use_reduce" + # assert self._use_reader_alloc == False, "gloo not support _use_reduce" if self._save_model: tr_cmd += " --save_model" if self._diff_batch: @@ -1169,15 +1333,17 @@ class TestDistBase(unittest.TestCase): assert self.__use_cuda == False, "gloo not support use cuda" assert self.__use_xpu == False, "gloo not support use xpu" tr_cmd += " --use_cpu" - env.update({ - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - "PADDLE_CURRENT_ENDPOINT": ep, - "PADDLE_DISTRI_BACKEND": "gloo", - "GLOG_v": "2", - }) + env.update( + { + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + "PADDLE_CURRENT_ENDPOINT": ep, + "PADDLE_DISTRI_BACKEND": "gloo", + "GLOG_v": "2", + } + ) assert self._use_dgc == False, "gloo not support use dgc" @@ -1199,8 +1365,9 @@ class TestDistBase(unittest.TestCase): assert self._use_fleet_api_20 == False, "gloo not support use fleet api" return tr_cmd, env - def _get_nccl2_trainer_cmd(self, model, ep, update_method, trainer_id, - trainer_num): + def _get_nccl2_trainer_cmd( + self, model, ep, update_method, trainer_id, trainer_num + ): env = {} tr_cmd = "%s -u" @@ -1209,9 +1376,15 @@ class TestDistBase(unittest.TestCase): tr_cmd += " %s --role trainer --endpoints %s --trainer_id %d --current_endpoint %s --update_method %s --lr %f" - tr_cmd = tr_cmd % \ - (self._python_interp, model, self._ps_endpoints, - trainer_id, ep, update_method, self._lr) + tr_cmd = tr_cmd % ( + self._python_interp, + model, + self._ps_endpoints, + trainer_id, + ep, + update_method, + self._lr, + ) if self._use_reduce: tr_cmd += " --use_reduce" @@ -1221,47 +1394,55 @@ class TestDistBase(unittest.TestCase): tr_cmd += " --save_model" if self.__use_cuda: tr_cmd += " --use_cuda" - env.update({ - "FLAGS_selected_gpus": "{}".format(0), - "CUDA_VISIBLE_DEVICES": "{}".format(trainer_id), - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - }) + env.update( + { + "FLAGS_selected_gpus": "{}".format(0), + "CUDA_VISIBLE_DEVICES": "{}".format(trainer_id), + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + } + ) # TODO(liuyuhui):XPU_VISIBLE_DEVICES is not working right now, # will update it after Badiu Kunlun partners' support. elif self.__use_xpu: tr_cmd += " --use_xpu" - env.update({ - "FLAGS_selected_xpus": "{}".format(trainer_id), - #"XPU_VISIBLE_DEVICES": "{}".format(trainer_id + 1), - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - "GLOG_v": "2", - }) + env.update( + { + "FLAGS_selected_xpus": "{}".format(trainer_id), + # "XPU_VISIBLE_DEVICES": "{}".format(trainer_id + 1), + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + "GLOG_v": "2", + } + ) elif self.__use_npu: tr_cmd += " --use_npu" - env.update({ - "FLAGS_selected_npus": "{}".format(trainer_id), - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - "GLOG_v": "2", - }) + env.update( + { + "FLAGS_selected_npus": "{}".format(trainer_id), + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + "GLOG_v": "2", + } + ) elif self._use_mlu: tr_cmd += " --use_mlu" - env.update({ - "FLAGS_selected_mlus": "{}".format(trainer_id), - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - "GLOG_v": "4", - }) + env.update( + { + "FLAGS_selected_mlus": "{}".format(trainer_id), + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + "GLOG_v": "4", + } + ) else: env.update({'CPU_NUM': '1'}) @@ -1292,7 +1473,11 @@ class TestDistBase(unittest.TestCase): tr_cmd += " --fuse_all_reduce {}".format(self._fuse_all_reduce) if self._use_fleet_api: - tr_cmd += " --use_fleet_api_20" if self._use_fleet_api_20 else " --use_fleet_api" + tr_cmd += ( + " --use_fleet_api_20" + if self._use_fleet_api_20 + else " --use_fleet_api" + ) if self._use_local_sgd: tr_cmd += " --use_local_sgd" if self._ut4grad_allreduce: @@ -1305,10 +1490,16 @@ class TestDistBase(unittest.TestCase): return tr_cmd, env - def _run_cluster_gloo(self, model, envs, update_method, check_error_log, - log_name): - assert update_method == "gloo", "_run_cluster_gloo must have update_method: gloo, but get %s" % update_method - assert not self._use_hallreduce, "_run_cluster_gloo must have _use_hallreduce = false" + def _run_cluster_gloo( + self, model, envs, update_method, check_error_log, log_name + ): + assert update_method == "gloo", ( + "_run_cluster_gloo must have update_method: gloo, but get %s" + % update_method + ) + assert ( + not self._use_hallreduce + ), "_run_cluster_gloo must have _use_hallreduce = false" worker_endpoints = self._ps_endpoints.split(",") @@ -1317,27 +1508,33 @@ class TestDistBase(unittest.TestCase): procs = [] pipes = [] for i in range(0, trainer_num): - tr_cmd, tr_env = self._get_gloo_trainer_cmd(model, - worker_endpoints[i], - update_method, i, - trainer_num) + tr_cmd, tr_env = self._get_gloo_trainer_cmd( + model, worker_endpoints[i], update_method, i, trainer_num + ) tr_env.update(envs) tr_env["GLOG_vmodule"] = 'gloo_context=4' tr_env["GLOG_v"] = '3' - print("use_hallreduce:{} tr_cmd:{}, env: {}".format( - self._use_hallreduce, tr_cmd, tr_env)) + print( + "use_hallreduce:{} tr_cmd:{}, env: {}".format( + self._use_hallreduce, tr_cmd, tr_env + ) + ) - path = os.path.join(self.temp_dir.name, - log_name + "_tr{}_err.log".format(i)) + path = os.path.join( + self.temp_dir.name, log_name + "_tr{}_err.log".format(i) + ) tr_pipe = open(path, "wb") print_to_err( type(self).__name__, - "going to start process {} with nccl2".format(i)) - tr_proc = subprocess.Popen(tr_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr_pipe, - env=tr_env) + "going to start process {} with nccl2".format(i), + ) + tr_proc = subprocess.Popen( + tr_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr_pipe, + env=tr_env, + ) procs.append(tr_proc) pipes.append(tr_pipe) @@ -1350,7 +1547,8 @@ class TestDistBase(unittest.TestCase): sys.stderr.write('trainer {} stderr: {}\n'.format(i, tr_err)) if trainer_num == 1: - if check_error_log: print("outs[0]:", outs[0]) + if check_error_log: + print("outs[0]:", outs[0]) return pickle.loads(outs[0]) else: @@ -1359,8 +1557,9 @@ class TestDistBase(unittest.TestCase): print("outs[1]:", outs[1]) return pickle.loads(outs[0]), pickle.loads(outs[1]) - def _run_cluster_nccl2(self, model, envs, update_method, check_error_log, - log_name): + def _run_cluster_nccl2( + self, model, envs, update_method, check_error_log, log_name + ): if self._use_hallreduce: self._ps_endpoints = "" @@ -1369,7 +1568,8 @@ class TestDistBase(unittest.TestCase): # NOTE(wangxi). hallreduce test must use 4cards after nccl>=2.7 for i in range(0, 4): self._ps_endpoints += "127.0.0.1:%s," % ( - self._find_free_port()) + self._find_free_port() + ) else: for i in range(0, 4): self._ps_endpoints += "127.0.0.1:%s," % (DIST_UT_PORT + i) @@ -1385,22 +1585,30 @@ class TestDistBase(unittest.TestCase): pipes = [] for i in range(0, trainer_num): tr_cmd, tr_env = self._get_nccl2_trainer_cmd( - model, worker_endpoints[i], update_method, i, trainer_num) + model, worker_endpoints[i], update_method, i, trainer_num + ) tr_env.update(envs) - print("use_hallreduce:{} tr_cmd:{}, env: {}".format( - self._use_hallreduce, tr_cmd, tr_env)) + print( + "use_hallreduce:{} tr_cmd:{}, env: {}".format( + self._use_hallreduce, tr_cmd, tr_env + ) + ) - path = os.path.join(self.temp_dir.name, - log_name + "_tr{}_err.log".format(i)) + path = os.path.join( + self.temp_dir.name, log_name + "_tr{}_err.log".format(i) + ) tr_pipe = open(path, "wb") print_to_err( type(self).__name__, - "going to start process {} with nccl2".format(i)) - tr_proc = subprocess.Popen(tr_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr_pipe, - env=tr_env) + "going to start process {} with nccl2".format(i), + ) + tr_proc = subprocess.Popen( + tr_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr_pipe, + env=tr_env, + ) procs.append(tr_proc) pipes.append(tr_pipe) @@ -1429,7 +1637,8 @@ class TestDistBase(unittest.TestCase): pipes = [] for i in range(0, trainer_num): tr_cmd, tr_env = self._get_nccl2_trainer_cmd( - model, worker_endpoints[i], update_method, i, trainer_num) + model, worker_endpoints[i], update_method, i, trainer_num + ) tr_env.update(envs) tr_env['CUDA_VISIBLE_DEVICES'] = "0,1" tr_env['NCCL_SHM_DISABLE'] = '1' @@ -1442,11 +1651,14 @@ class TestDistBase(unittest.TestCase): print_to_err( type(self).__name__, - "going to start process {} with nccl2".format(i)) - tr_proc = subprocess.Popen(tr_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr_pipe, - env=tr_env) + "going to start process {} with nccl2".format(i), + ) + tr_proc = subprocess.Popen( + tr_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr_pipe, + env=tr_env, + ) procs.append(tr_proc) pipes.append(tr_pipe) @@ -1477,67 +1689,79 @@ class TestDistBase(unittest.TestCase): "http_proxy": "", "NCCL_P2P_DISABLE": "1", "NCCL_SHM_DISABLE": "1", - "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1" + "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1", } if check_error_log: - required_envs["GLOG_vmodule"] = \ - "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10," \ - "alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10,executor=10,operator=10," \ - "sparse_all_reduce_op_handle=10,gen_nccl_id_op=10,gen_nccl_id_op_help=10,nccl_helper=10,grpc_client=10," \ + required_envs["GLOG_vmodule"] = ( + "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10," + "alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10,executor=10,operator=10," + "sparse_all_reduce_op_handle=10,gen_nccl_id_op=10,gen_nccl_id_op_help=10,nccl_helper=10,grpc_client=10," "grpc_server=10,request_handler_impl=10,section_worker=10" + ) required_envs["GLOG_logtostderr"] = "1" if os.getenv('NVIDIA_TF32_OVERRIDE', '') is not None: required_envs['NVIDIA_TF32_OVERRIDE'] = os.getenv( - 'NVIDIA_TF32_OVERRIDE', '') + 'NVIDIA_TF32_OVERRIDE', '' + ) required_envs.update(need_envs) return required_envs - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): if self._dygraph and (self._gloo_mode or self._nccl2_mode): need_envs.update({"FLAGS_enable_eager_mode": "1"}) with _test_eager_guard(): - self.check_with_place_func(model_file=model_file, - delta=delta, - check_error_log=check_error_log, - need_envs=need_envs, - log_name=log_name) + self.check_with_place_func( + model_file=model_file, + delta=delta, + check_error_log=check_error_log, + need_envs=need_envs, + log_name=log_name, + ) need_envs.update({"FLAGS_enable_eager_mode": "0"}) - self.check_with_place_func(model_file=model_file, - delta=delta, - check_error_log=check_error_log, - need_envs=need_envs, - log_name=log_name) + self.check_with_place_func( + model_file=model_file, + delta=delta, + check_error_log=check_error_log, + need_envs=need_envs, + log_name=log_name, + ) else: - self.check_with_place_func(model_file=model_file, - delta=delta, - check_error_log=check_error_log, - need_envs=need_envs, - log_name=log_name) - - def check_with_place_func(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + self.check_with_place_func( + model_file=model_file, + delta=delta, + check_error_log=check_error_log, + need_envs=need_envs, + log_name=log_name, + ) + + def check_with_place_func( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): required_envs = self._get_required_envs(check_error_log, need_envs) if self._gloo_mode: - local_losses \ - = self._run_local_gloo(model_file, required_envs, - check_error_log, log_name=log_name) + local_losses = self._run_local_gloo( + model_file, required_envs, check_error_log, log_name=log_name + ) else: - local_losses \ - = self._run_local(model_file, required_envs, - check_error_log, log_name=log_name) + local_losses = self._run_local( + model_file, required_envs, check_error_log, log_name=log_name + ) if self._nccl2_mode: if self._nccl2_reduce_layer: @@ -1546,21 +1770,24 @@ class TestDistBase(unittest.TestCase): required_envs, update_method="nccl2_reduce_layer", check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) else: tr0_losses, tr1_losses = self._run_cluster_nccl2( model_file, required_envs, update_method='nccl2', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._bkcl_mode: tr0_losses, tr1_losses = self._run_cluster_nccl2( model_file, required_envs, update_method='bkcl', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._gloo_mode: # gloo mode, cpu only parallel train @xiongkun03 tr0_losses, tr1_losses = self._run_cluster_gloo( @@ -1568,31 +1795,32 @@ class TestDistBase(unittest.TestCase): required_envs, update_method='gloo', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._hccl_mode: tr0_losses, tr1_losses = self._run_cluster_nccl2( model_file, required_envs, update_method='hccl', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._cncl_mode: tr0_losses, tr1_losses = self._run_cluster_nccl2( model_file, required_envs, update_method='cncl', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._pipeline_mode: - tr0_losses, tr1_losses = self._run_pipeline(model_file, - required_envs, - check_error_log, - log_name=log_name) + tr0_losses, tr1_losses = self._run_pipeline( + model_file, required_envs, check_error_log, log_name=log_name + ) else: - tr0_losses, tr1_losses = self._run_cluster(model_file, - required_envs, - check_error_log, - log_name=log_name) + tr0_losses, tr1_losses = self._run_cluster( + model_file, required_envs, check_error_log, log_name=log_name + ) for step_id in range(RUN_STEP): local_loss = local_losses[step_id] @@ -1605,12 +1833,14 @@ class TestDistBase(unittest.TestCase): print("=======", local_loss, ":", dist_loss[0], "=======") self.assertAlmostEqual(local_loss, dist_loss[0], delta=delta) - def check_with_place_multi_cards(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place_multi_cards( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): # need open p2p or shm otherwise multi cards mode will hang need_envs.update({"NCCL_P2P_DISABLE": "0", "NCCL_SHM_DISABLE": "0"}) @@ -1618,19 +1848,22 @@ class TestDistBase(unittest.TestCase): required_envs = self._get_required_envs(check_error_log, need_envs) if self._use_dgc: - multi_cards_losses = self._run_local(model_file, - required_envs, - check_error_log, - log_name=log_name + - "_dgc_2cards", - devices="0,1") + multi_cards_losses = self._run_local( + model_file, + required_envs, + check_error_log, + log_name=log_name + "_dgc_2cards", + devices="0,1", + ) self._use_dgc = False - base_losses = self._run_local(model_file, - required_envs, - check_error_log, - log_name=log_name + "_base_2cards", - devices="0,1") + base_losses = self._run_local( + model_file, + required_envs, + check_error_log, + log_name=log_name + "_base_2cards", + devices="0,1", + ) self._use_dgc = True diff --git a/python/paddle/fluid/tests/unittests/test_dist_dygraph_apis.py b/python/paddle/fluid/tests/unittests/test_dist_dygraph_apis.py index a99d8221fe1..aa5e1e93064 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_dygraph_apis.py +++ b/python/paddle/fluid/tests/unittests/test_dist_dygraph_apis.py @@ -17,7 +17,6 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestDygraphFleetApi(TestMultipleGpus): - def test_dygraph_fleet_api(self): self.run_mnist_2gpu('dygraph_fleet_api.py') diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_async.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_async.py index b269e61ec78..4aaf596d570 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_async.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_async.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -30,8 +29,9 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer_trainer(self): os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py index f0cf60296ca..523170367fb 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -29,8 +28,9 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer1(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -43,16 +43,17 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): paddle.fluid.framework.switch_startup_program(startup_program) fleet.init(role_maker.PaddleCloudRoleMaker()) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) os.environ["FLAGS_LAUNCH_BARRIER"] = "0" diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py index dded4e7c7ba..a51b01209b1 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -31,8 +30,9 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer3(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -45,26 +45,31 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): paddle.fluid.framework.switch_startup_program(startup_program) fleet.init(role_maker.PaddleCloudRoleMaker()) - input_x = paddle.fluid.layers.data(name="x", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) + input_x = paddle.fluid.layers.data( + name="x", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) x_embedding = paddle.fluid.layers.embedding( is_distributed=False, input=input_x, size=[1000000000, 100000], param_attr=paddle.fluid.ParamAttr( name="embedding", - initializer=paddle.fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=paddle.fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=x_embedding, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) os.environ["FLAGS_LAUNCH_BARRIER"] = "0" diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py index 035506607e3..e6f618b63a6 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -30,8 +29,9 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer2(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -48,15 +48,16 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): input_x = paddle.fluid.layers.data(name="x", shape=[1], dtype='int64') input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') - emb = paddle.fluid.layers.embedding(input=input_x, - size=[100, 10], - is_sparse=True) + emb = paddle.fluid.layers.embedding( + input=input_x, size=[100, 10], is_sparse=True + ) fc_1 = paddle.fluid.layers.fc(input=emb, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) os.environ["FLAGS_LAUNCH_BARRIER"] = "0" strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_geo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_geo.py index 45d3b315bcd..05abc40a1d0 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_geo.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_geo.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -29,8 +28,9 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer_trainer(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -43,16 +43,17 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): paddle.fluid.framework.switch_startup_program(startup_program) fleet.init(role_maker.PaddleCloudRoleMaker()) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -75,16 +76,17 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): paddle.fluid.framework.switch_startup_program(startup_program) fleet.init(role_maker.PaddleCloudRoleMaker()) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_sync.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_sync.py index 753635e6904..837ea65d54f 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_sync.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_sync.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -31,8 +30,9 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_gradient_merge_optimizer(self): fleet.init(role_maker.PaddleCloudRoleMaker()) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py index c8ae35d3d55..6686fb03a01 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py @@ -17,6 +17,7 @@ import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker import paddle.fluid as fluid import paddle + """ high level unit test for distribute fleet. """ @@ -46,9 +47,9 @@ DIST_UT_PORT = 0 class FleetDistRunnerBase(object): """ - run_pserver,run_trainer : after init role, using transpiler split program - net : implment by child class, the network of model - do training : exe run program + run_pserver,run_trainer : after init role, using transpiler split program + net : implment by child class, the network of model + do training : exe run program """ def __init__(self): @@ -64,7 +65,8 @@ class FleetDistRunnerBase(object): current_id=args.current_id, role=role_maker.Role.SERVER, worker_endpoints=args.trainer_endpoints.split(","), - server_endpoints=args.endpoints.split(",")) + server_endpoints=args.endpoints.split(","), + ) else: role = role_maker.UserDefinedRoleMaker( is_collective=False, @@ -73,7 +75,8 @@ class FleetDistRunnerBase(object): current_id=args.current_id, role=role_maker.Role.WORKER, worker_endpoints=args.trainer_endpoints.split(","), - server_endpoints=args.endpoints.split(",")) + server_endpoints=args.endpoints.split(","), + ) self.role = role return role @@ -100,14 +103,13 @@ class FleetDistRunnerBase(object): debug = int(os.getenv("Debug", "0")) # TODO(update strategy to support dump params) if False: # debug: - self.strategy.set_debug_opt({ - "dump_param": - self.dump_param, - "dump_fields": - self.dump_fields, - "dump_fields_path": - self.dump_fields_path - }) + self.strategy.set_debug_opt( + { + "dump_param": self.dump_param, + "dump_fields": self.dump_fields, + "dump_fields_path": self.dump_fields_path, + } + ) return self.strategy @@ -126,7 +128,8 @@ class FleetDistRunnerBase(object): use_decay = int(os.getenv("USE_DECAY", "0")) if use_decay: scheduler = paddle.optimizer.lr.ExponentialDecay( - learning_rate=LEARNING_RATE, gamma=0.999, verbose=True) + learning_rate=LEARNING_RATE, gamma=0.999, verbose=True + ) optimizer = fluid.optimizer.SGD(scheduler, grad_clip=grad_clip) """ # learning rate decay method before 2.0 @@ -154,7 +157,8 @@ class FleetDistRunnerBase(object): def net(self, args, batch_size=4, lr=0.01): raise NotImplementedError( - "get_model should be implemented by child classes.") + "get_model should be implemented by child classes." + ) def get_executor(self): if self._exe is None: @@ -168,21 +172,24 @@ class FleetDistRunnerBase(object): def do_dataset_training(self, fleet): raise NotImplementedError( - "do_dataset_training should be implemented by child classes.") + "do_dataset_training should be implemented by child classes." + ) def do_pyreader_training(self, fleet): raise NotImplementedError( - "do_pyreader_training should be implemented by child classes.") + "do_pyreader_training should be implemented by child classes." + ) def do_distributed_testing(self, fleet): raise NotImplementedError( - "do_distributed_testing should be implemented by child classes.") + "do_distributed_testing should be implemented by child classes." + ) class TestFleetBase(unittest.TestCase): """ - start_pserver,start_trainer : add start cmd to test - run_cluster : using multi process to test distribute program + start_pserver,start_trainer : add start cmd to test + run_cluster : using multi process to test distribute program """ def _setup_config(self): @@ -210,15 +217,23 @@ class TestFleetBase(unittest.TestCase): if DIST_UT_PORT: print("set begin_port:", DIST_UT_PORT) self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT, DIST_UT_PORT + 1) + DIST_UT_PORT, + DIST_UT_PORT + 1, + ) self._tr_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT + 2, DIST_UT_PORT + 3) + DIST_UT_PORT + 2, + DIST_UT_PORT + 3, + ) DIST_UT_PORT += 4 else: self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._tr_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self._geo_sgd_need_push_nums = 5 @@ -226,10 +241,10 @@ class TestFleetBase(unittest.TestCase): self._setup_config() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -259,18 +274,24 @@ class TestFleetBase(unittest.TestCase): ps0_out = open(ps0_out_log, "wb+") ps1_out = open(ps1_out_log, "wb+") - ps0_proc = subprocess.Popen(ps0_cmd.strip().split(" "), - stdout=ps0_out, - stderr=ps0_err, - env=required_envs) - - ps1_proc = subprocess.Popen(ps1_cmd.strip().split(" "), - stdout=ps1_out, - stderr=ps1_err, - env=required_envs) - - return ((ps0_proc, ps0_out, ps0_err, ps0_out_log, ps0_err_log), - (ps1_proc, ps1_out, ps1_err, ps1_out_log, ps1_err_log)) + ps0_proc = subprocess.Popen( + ps0_cmd.strip().split(" "), + stdout=ps0_out, + stderr=ps0_err, + env=required_envs, + ) + + ps1_proc = subprocess.Popen( + ps1_cmd.strip().split(" "), + stdout=ps1_out, + stderr=ps1_err, + env=required_envs, + ) + + return ( + (ps0_proc, ps0_out, ps0_err, ps0_out_log, ps0_err_log), + (ps1_proc, ps1_out, ps1_err, ps1_out_log, ps1_err_log), + ) def _start_trainer(self, cmd, required_envs): tr0_cmd, tr1_cmd = cmd.format(0), cmd.format(1) @@ -292,18 +313,24 @@ class TestFleetBase(unittest.TestCase): tr0_out = open(tr0_out_log, "wb+") tr1_out = open(tr1_out_log, "wb+") - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(" "), - stdout=tr0_out, - stderr=tr0_err, - env=required_envs) - - tr1_proc = subprocess.Popen(tr1_cmd.strip().split(" "), - stdout=tr1_out, - stderr=tr1_err, - env=required_envs) - - return ((tr0_proc, tr0_out, tr0_err, tr0_out_log, tr0_err_log), - (tr1_proc, tr1_out, tr1_err, tr1_out_log, tr1_err_log)) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(" "), + stdout=tr0_out, + stderr=tr0_err, + env=required_envs, + ) + + tr1_proc = subprocess.Popen( + tr1_cmd.strip().split(" "), + stdout=tr1_out, + stderr=tr1_err, + env=required_envs, + ) + + return ( + (tr0_proc, tr0_out, tr0_err, tr0_out_log, tr0_err_log), + (tr1_proc, tr1_out, tr1_err, tr1_out_log, tr1_err_log), + ) def _run_cluster(self, model, envs): env = {'GRAD_CLIP': str(self._grad_clip_mode), 'WITH_DISTRIBUTE': 'ON'} @@ -316,14 +343,30 @@ class TestFleetBase(unittest.TestCase): env.update(envs) tr_cmd = "{0} {1} --role trainer --endpoints {2} --trainer_endpoints {3} --current_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --test {9}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._need_test) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._need_test, + ) ps_cmd = "{0} {1} --role pserver --endpoints {2} --trainer_endpoints {3} --current_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --test {9}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._need_test) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._need_test, + ) if self._model_dir: tr_cmd += " --model_dir {}".format(self._model_dir) @@ -340,7 +383,7 @@ class TestFleetBase(unittest.TestCase): tr1_proc, tr1_out, tr1_err, tr1_out_log, tr1_err_log = tr1 # Wait until trainer process terminate - #time_out = 120 + # time_out = 120 time_out = 60 cur_time = 0 @@ -383,27 +426,41 @@ class TestFleetBase(unittest.TestCase): def catlog(logx): basename = os.path.basename(logx) - print("\n================== Error {} begin =====================". - format(basename)) + print( + "\n================== Error {} begin =====================".format( + basename + ) + ) os.system("cat {}".format(logx)) - print("================== Error {} end =====================\n". - format(basename)) + print( + "================== Error {} end =====================\n".format( + basename + ) + ) if tr0_ret != 0 or tr1_ret != 0: if is_listen_failed(ps0_err) or is_listen_failed(ps1_err): print("find parameter server port bind failed, skip the error") tr0_ret, tr1_ret = 0, 0 else: - for out, err in [(ps0_out_log, ps0_err_log), - (ps1_out_log, ps1_err_log), - (tr0_out_log, tr0_err_log), - (tr1_out_log, tr1_err_log)]: + for out, err in [ + (ps0_out_log, ps0_err_log), + (ps1_out_log, ps1_err_log), + (tr0_out_log, tr0_err_log), + (tr1_out_log, tr1_err_log), + ]: catlog(out) catlog(err) for pipe in [ - tr0_err, tr0_out, tr1_err, tr1_out, ps0_err, ps0_out, ps1_err, - ps1_out + tr0_err, + tr0_out, + tr1_err, + tr1_out, + ps0_err, + ps0_out, + ps1_err, + ps1_out, ]: pipe.close() @@ -414,17 +471,15 @@ class TestFleetBase(unittest.TestCase): return 0, 0 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "" + "http_proxy": "", } required_envs.update(need_envs) @@ -438,23 +493,20 @@ class TestFleetBase(unittest.TestCase): def runtime_main(test_class): parser = argparse.ArgumentParser(description='Run Fleet test.') - parser.add_argument('--role', - type=str, - required=True, - choices=['pserver', 'trainer']) + parser.add_argument( + '--role', type=str, required=True, choices=['pserver', 'trainer'] + ) parser.add_argument('--endpoints', type=str, required=False, default="") - parser.add_argument('--trainer_endpoints', - type=str, - required=False, - default="") + parser.add_argument( + '--trainer_endpoints', type=str, required=False, default="" + ) parser.add_argument('--gloo_path', type=str, required=False, default="") parser.add_argument('--current_id', type=int, required=False, default=0) parser.add_argument('--trainers', type=int, required=False, default=1) parser.add_argument('--mode', type=str, required=False, default='geo') - parser.add_argument('--geo_sgd_need_push_nums', - type=int, - required=False, - default=2) + parser.add_argument( + '--geo_sgd_need_push_nums', type=int, required=False, default=2 + ) parser.add_argument('--reader', type=str, required=False, default='dataset') parser.add_argument('--test', type=int, required=False, default=0) parser.add_argument('--model_dir', type=str, required=False, default="") @@ -467,14 +519,17 @@ def runtime_main(test_class): if args.test and args.model_dir != "": avg_cost = model.net(args, is_train=False) dist_infer = DistributedInfer() - dist_infer.init_distributed_infer_env(exe=model.get_executor(), - loss=model.avg_cost, - role_maker=role, - dirname=args.model_dir) + dist_infer.init_distributed_infer_env( + exe=model.get_executor(), + loss=model.avg_cost, + role_maker=role, + dirname=args.model_dir, + ) if fleet.is_worker(): with paddle.static.program_guard( - main_program=dist_infer.get_dist_infer_program()): + main_program=dist_infer.get_dist_infer_program() + ): model.do_distributed_testing(fleet) fleet.stop_worker() return @@ -499,13 +554,17 @@ def runtime_main(test_class): test_origin_program = paddle.static.Program() test_startup_program = paddle.static.Program() with paddle.static.program_guard( - main_program=test_origin_program, - startup_program=test_startup_program): + main_program=test_origin_program, + startup_program=test_startup_program, + ): with paddle.utils.unique_name.guard(): avg_cost = model.net(args, is_train=False) - dist_infer = DistributedInfer(main_program=test_origin_program, - startup_program=test_startup_program) + dist_infer = DistributedInfer( + main_program=test_origin_program, + startup_program=test_startup_program, + ) with paddle.static.program_guard( - main_program=dist_infer.get_dist_infer_program()): + main_program=dist_infer.get_dist_infer_program() + ): model.do_distributed_testing(fleet) fleet.stop_worker() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py index 06d57c86bd7..33006b9d9e8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py @@ -18,17 +18,14 @@ from test_dist_fleet_base import TestFleetBase class TestDistMnistAsyncInMemoryDataset2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" - #self._reader = "pyreader" + # self._reader = "pyreader" self._reader = "dataset" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -38,14 +35,10 @@ class TestDistMnistAsyncInMemoryDataset2x2(TestFleetBase): "CPU_NUM": "2", "LOG_DIRNAME": "/tmp", "SAVE_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/model", - "SAVE_CACHE_DIRNAME": - "/tmp/TestDistMnistAsyncInMemoryDataset2x2/cache_model", - "SAVE_DENSE_PARAM_DIRNAME": - "/tmp/TestDistMnistAsyncInMemoryDataset2x2/dense_param", - "SAVE_ONE_TABLE_DIRNAME": - "/tmp/TestDistMnistAsyncInMemoryDataset2x2/table_0", - "SAVE_PATCH_DIRNAME": - "/tmp/TestDistMnistAsyncInMemoryDataset2x2/patch_model", + "SAVE_CACHE_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/cache_model", + "SAVE_DENSE_PARAM_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/dense_param", + "SAVE_ONE_TABLE_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/table_0", + "SAVE_PATCH_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/patch_model", "LOG_PREFIX": self.__class__.__name__, } @@ -58,22 +51,19 @@ class TestDistMnistAsyncInMemoryDataset2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) class TestDistMnistAsync2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -94,22 +84,19 @@ class TestDistMnistAsync2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) class TestDistCtrHalfAsync2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -133,9 +120,9 @@ class TestDistCtrHalfAsync2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr2.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr2.py index b86fabde620..d9c725c3a89 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr2.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr2.py @@ -20,17 +20,14 @@ from test_dist_fleet_base import TestFleetBase @unittest.skip(reason="Skip unstable ut, need paddle sync mode fix") class TestDistMnistSync2x2(TestFleetBase): - def _setup_config(self): self._mode = "sync" self._reader = "pyreader" self._need_test = 1 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -51,23 +48,20 @@ class TestDistMnistSync2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) # @unittest.skip(reason="Skip unstable ut, reader need to be rewrite") class TestDistMnistAsyncDataset2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "dataset" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -92,9 +86,9 @@ class TestDistMnistAsyncDataset2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_decay.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_decay.py index a5e2e7073d0..9d5ac645b61 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_decay.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_decay.py @@ -32,18 +32,17 @@ batch_size = 4 class TestNoamDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -56,20 +55,24 @@ class TestNoamDecay(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.WORKER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.WORKER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True) + scheduler = paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_geo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_geo.py index 37ed07534d9..2e63a28cd24 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_geo.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_geo.py @@ -28,17 +28,14 @@ paddle.enable_static() class TestDistGeoCtr_2x2(TestFleetBase): - def _setup_config(self): self._mode = "geo" self._reader = "pyreader" self._geo_sgd_need_push_nums = 5 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -58,19 +55,19 @@ class TestDistGeoCtr_2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) class TestGeoSgdTranspiler(unittest.TestCase): - def test_pserver(self): role = role_maker.UserDefinedRoleMaker( current_id=0, role=role_maker.Role.SERVER, worker_num=2, - server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"]) + server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"], + ) fleet.init(role) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_gloo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_gloo.py index 26590f5a75e..dd31dc8ef0d 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_gloo.py @@ -18,25 +18,25 @@ import tempfile import unittest import subprocess import time -#import paddle.fluid.incubate.fleet.base.role_maker as role_maker + +# import paddle.fluid.incubate.fleet.base.role_maker as role_maker from test_dist_fleet_base import TestFleetBase -#from dist_simnet_bow import train_network +# from dist_simnet_bow import train_network class TestDistGloo_2x2(TestFleetBase): - def _setup_config(self): self._mode = "sync" self._reader = "pyreader" self._path = "./tmp4" - if (os.path.exists(self._path)): + if os.path.exists(self._path): shutil.rmtree(self._path) # if not os.path.exists(self._path): # os.mkdir(self._path) def _start_pserver(self, cmd, required_envs): - #env.update(required_envs) + # env.update(required_envs) ps0_cmd = cmd ps1_cmd = cmd @@ -46,24 +46,28 @@ class TestDistGloo_2x2(TestFleetBase): required_envs["POD_IP"] = "127.0.0.1" required_envs["PADDLE_PSERVER_ID"] = "0" required_envs["PADDLE_PORT"] = "36011" - ps0_proc = subprocess.Popen(ps0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps0_pipe, - env=required_envs) + ps0_proc = subprocess.Popen( + ps0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps0_pipe, + env=required_envs, + ) print("PADDLE_PSERVER_ID=0:") print(required_envs) required_envs["PADDLE_PSERVER_ID"] = "1" required_envs["PADDLE_PORT"] = "36012" - ps1_proc = subprocess.Popen(ps1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps1_pipe, - env=required_envs) + ps1_proc = subprocess.Popen( + ps1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps1_pipe, + env=required_envs, + ) print("PADDLE_PSERVER_ID=1:") print(required_envs) return ps0_proc, ps1_proc, ps0_pipe, ps1_pipe def _start_trainer(self, cmd, required_envs): - #env.update(required_envs) + # env.update(required_envs) tr0_cmd = cmd tr1_cmd = cmd @@ -71,17 +75,21 @@ class TestDistGloo_2x2(TestFleetBase): tr0_pipe = open(tempfile.gettempdir() + "/tr0_err.log", "wb+") tr1_pipe = open(tempfile.gettempdir() + "/tr1_err.log", "wb+") required_envs["PADDLE_TRAINER_ID"] = "0" - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=required_envs) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=required_envs, + ) print("PADDLE_TRAINER_ID=0:") print(required_envs) required_envs["PADDLE_TRAINER_ID"] = "1" - tr1_proc = subprocess.Popen(tr1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=required_envs) + tr1_proc = subprocess.Popen( + tr1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=required_envs, + ) print("PADDLE_TRAINER_ID=1:") print(required_envs) return tr0_proc, tr1_proc, tr0_pipe, tr1_pipe @@ -138,11 +146,9 @@ class TestDistGloo_2x2(TestFleetBase): return 0, 0 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -150,16 +156,16 @@ class TestDistGloo_2x2(TestFleetBase): "FLAGS_rpc_deadline": "5000", # 5sec to fail fast "http_proxy": "", "CPU_NUM": "2", - #PSERVER + # PSERVER "PADDLE_PSERVERS_IP_PORT_LIST": "127.0.0.1:36011,127.0.0.1:36012", - #"PADDLE_PSERVER_PORT_ARRAY":"(36011 36012)", + # "PADDLE_PSERVER_PORT_ARRAY":"(36011 36012)", "PADDLE_PSERVER_NUMS": "2", "PADDLE_TRAINER_ID": "0", - #TRAINER + # TRAINER "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:36013,127.0.0.1:36014", "PADDLE_TRAINERS_NUM": "2", "PADDLE_PSERVER_ID": "0", - #GLOO FLAG + # GLOO FLAG "PADDLE_WITH_GLOO": "1", } @@ -173,9 +179,9 @@ class TestDistGloo_2x2(TestFleetBase): def test_dist_train(self): print("path is not delete", os.path.exists("./tmp4")) - self.check_with_place("dist_fleet_debug_gloo.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_debug_gloo.py", delta=1e-5, check_error_log=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_base.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_base.py index 13178ad233e..18c20ef6754 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_base.py @@ -41,9 +41,9 @@ DIST_UT_PORT = 0 class FleetDistHeterRunnerBase(object): """ - run_pserver,run_trainer : after init role, using transpiler split program - net : implment by child class, the network of model - do training : exe run program + run_pserver,run_trainer : after init role, using transpiler split program + net : implment by child class, the network of model + do training : exe run program """ def build_role(self, args): @@ -54,33 +54,45 @@ class FleetDistHeterRunnerBase(object): environs["PADDLE_PSERVERS_IP_PORT_LIST"] = args.endpoints environs["PADDLE_TRAINER_ENDPOINTS"] = args.trainer_endpoints environs[ - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST"] = all_heter_trainer_endpoints - environs["POD_IP"] = args.endpoints.split(",")[int( - args.current_id)].split(":")[0] - environs["PADDLE_PORT"] = args.endpoints.split(",")[int( - args.current_id)].split(":")[1] + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST" + ] = all_heter_trainer_endpoints + environs["POD_IP"] = args.endpoints.split(",")[ + int(args.current_id) + ].split(":")[0] + environs["PADDLE_PORT"] = args.endpoints.split(",")[ + int(args.current_id) + ].split(":")[1] environs["TRAINING_ROLE"] = args.role.upper() environs["PADDLE_TRAINERS_NUM"] = args.trainers elif args.role.upper() == "HETER_TRAINER": - previous_endpoints = args.trainer_endpoints if args.stage_id == 2 else heter_trainer_endpoints[ - 0] - next_endpoints = heter_trainer_endpoints[ - 1] if args.stage_id == 2 else "" - heter_device = args.heter_trainer_device.split(";")[args.stage_id - - 2] + previous_endpoints = ( + args.trainer_endpoints + if args.stage_id == 2 + else heter_trainer_endpoints[0] + ) + next_endpoints = ( + heter_trainer_endpoints[1] if args.stage_id == 2 else "" + ) + heter_device = args.heter_trainer_device.split(";")[ + args.stage_id - 2 + ] environs["PADDLE_PSERVERS_IP_PORT_LIST"] = args.endpoints environs["PADDLE_TRAINER_ENDPOINTS"] = args.trainer_endpoints environs["PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST"] = next_endpoints environs[ - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST"] = previous_endpoints + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST" + ] = previous_endpoints environs[ - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST"] = all_heter_trainer_endpoints + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST" + ] = all_heter_trainer_endpoints environs["HETER_DEVICE_TYPE"] = heter_device environs["TRAINING_ROLE"] = args.role.upper() - environs["POD_IP"] = all_heter_trainer_endpoints.split(",")[int( - args.current_id)].split(":")[0] + environs["POD_IP"] = all_heter_trainer_endpoints.split(",")[ + int(args.current_id) + ].split(":")[0] environs["PADDLE_PORT"] = all_heter_trainer_endpoints.split(",")[ - int(args.current_id)].split(":")[1] + int(args.current_id) + ].split(":")[1] environs["PADDLE_TRAINERS_NUM"] = args.trainers environs["PADDLE_STAGE_TRAINERS_NUM"] = [2, 2, 2] environs["FLAGS_selected_gpus"] = 0 @@ -93,18 +105,21 @@ class FleetDistHeterRunnerBase(object): environs["PADDLE_PSERVERS_IP_PORT_LIST"] = args.endpoints environs["PADDLE_TRAINER_ENDPOINTS"] = args.trainer_endpoints environs[ - "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST"] = heter_trainer_endpoints[ - 0] + "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST" + ] = heter_trainer_endpoints[0] environs["PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST"] = "" environs[ - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST"] = all_heter_trainer_endpoints + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST" + ] = all_heter_trainer_endpoints environs["HETER_DEVICE_TYPE"] = "cpu" environs["TRAINING_ROLE"] = args.role.upper() environs["PADDLE_TRAINER_ID"] = args.current_id - environs["POD_IP"] = args.trainer_endpoints.split(",")[int( - args.current_id)].split(":")[0] - environs["PADDLE_PORT"] = args.trainer_endpoints.split(",")[int( - args.current_id)].split(":")[1] + environs["POD_IP"] = args.trainer_endpoints.split(",")[ + int(args.current_id) + ].split(":")[0] + environs["PADDLE_PORT"] = args.trainer_endpoints.split(",")[ + int(args.current_id) + ].split(":")[1] environs["PADDLE_TRAINERS_NUM"] = args.trainers environs["PADDLE_STAGE_TRAINERS_NUM"] = [2, 2, 2] environs["FLAGS_selected_gpus"] = 0 @@ -126,12 +141,12 @@ class FleetDistHeterRunnerBase(object): self.strategy.a_sync = True self.strategy.a_sync_configs = { "launch_barrier": True, - "heter_worker_device_guard": 'gpu' + "heter_worker_device_guard": 'gpu', } self.strategy.pipeline = True self.strategy.pipeline_configs = { "accumulate_steps": 1, - "micro_batch_size": 2048 + "micro_batch_size": 2048, } return self.strategy @@ -152,21 +167,24 @@ class FleetDistHeterRunnerBase(object): def net(self, args, batch_size=4, lr=0.01): raise NotImplementedError( - "get_model should be implemented by child classes.") + "get_model should be implemented by child classes." + ) def do_dataset_training(self, fleet): raise NotImplementedError( - "do_dataset_training should be implemented by child classes.") + "do_dataset_training should be implemented by child classes." + ) def do_dataset_heter_training(self, fleet): raise NotImplementedError( - "do_dataset_heter_training should be implemented by child classes.") + "do_dataset_heter_training should be implemented by child classes." + ) class TestFleetHeterBase(unittest.TestCase): """ - start_pserver,start_trainer : add start cmd to test - run_cluster : using multi process to test distribute program + start_pserver,start_trainer : add start cmd to test + run_cluster : using multi process to test distribute program """ def _setup_config(self): @@ -194,23 +212,39 @@ class TestFleetHeterBase(unittest.TestCase): if DIST_UT_PORT: print("set begin_port:", DIST_UT_PORT) self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT, DIST_UT_PORT + 1) + DIST_UT_PORT, + DIST_UT_PORT + 1, + ) self._tr_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT + 2, DIST_UT_PORT + 3) + DIST_UT_PORT + 2, + DIST_UT_PORT + 3, + ) self._heter_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT + 4, DIST_UT_PORT + 5) + DIST_UT_PORT + 4, + DIST_UT_PORT + 5, + ) self._heter_endpoints_2 = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT + 6, DIST_UT_PORT + 7) + DIST_UT_PORT + 6, + DIST_UT_PORT + 7, + ) DIST_UT_PORT += 8 else: self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._tr_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._heter_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._heter_endpoints_2 = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self._geo_sgd_need_push_nums = 5 @@ -218,10 +252,10 @@ class TestFleetHeterBase(unittest.TestCase): self._setup_config() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -237,14 +271,18 @@ class TestFleetHeterBase(unittest.TestCase): ps0_pipe = open(tempfile.gettempdir() + "/ps0_err.log", "wb+") ps1_pipe = open(tempfile.gettempdir() + "/ps1_err.log", "wb+") - ps0_proc = subprocess.Popen(ps0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps0_pipe, - env=required_envs) - ps1_proc = subprocess.Popen(ps1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps1_pipe, - env=required_envs) + ps0_proc = subprocess.Popen( + ps0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps0_pipe, + env=required_envs, + ) + ps1_proc = subprocess.Popen( + ps1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps1_pipe, + env=required_envs, + ) return ps0_proc, ps1_proc, ps0_pipe, ps1_pipe def _start_trainer(self, cmd, required_envs): @@ -256,20 +294,28 @@ class TestFleetHeterBase(unittest.TestCase): tr0_out = open(tempfile.gettempdir() + "/tr0_out.log", "wb+") tr1_out = open(tempfile.gettempdir() + "/tr1_out.log", "wb+") - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(" "), - stdout=tr0_out, - stderr=tr0_pipe, - env=required_envs) - tr1_proc = subprocess.Popen(tr1_cmd.strip().split(" "), - stdout=tr1_out, - stderr=tr1_pipe, - env=required_envs) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(" "), + stdout=tr0_out, + stderr=tr0_pipe, + env=required_envs, + ) + tr1_proc = subprocess.Popen( + tr1_cmd.strip().split(" "), + stdout=tr1_out, + stderr=tr1_pipe, + env=required_envs, + ) return tr0_proc, tr1_proc, tr0_pipe, tr1_pipe def _start_heter_trainer(self, cmd, required_envs): - heter0_cmd, heter1_cmd, heter2_cmd, heter3_cmd = cmd.format( - 0, 2), cmd.format(1, 2), cmd.format(2, 3), cmd.format(3, 3) + heter0_cmd, heter1_cmd, heter2_cmd, heter3_cmd = ( + cmd.format(0, 2), + cmd.format(1, 2), + cmd.format(2, 3), + cmd.format(3, 3), + ) heter0_pipe = open(tempfile.gettempdir() + "/heter0_err.log", "wb+") heter1_pipe = open(tempfile.gettempdir() + "/heter1_err.log", "wb+") @@ -280,29 +326,46 @@ class TestFleetHeterBase(unittest.TestCase): heter2_out = open(tempfile.gettempdir() + "/heter2_out.log", "wb+") heter3_out = open(tempfile.gettempdir() + "/heter3_out.log", "wb+") - heter0_proc = subprocess.Popen(heter0_cmd.strip().split(" "), - stdout=heter0_out, - stderr=heter0_pipe, - env=required_envs) - heter1_proc = subprocess.Popen(heter1_cmd.strip().split(" "), - stdout=heter1_out, - stderr=heter1_pipe, - env=required_envs) - heter2_proc = subprocess.Popen(heter2_cmd.strip().split(" "), - stdout=heter2_out, - stderr=heter2_pipe, - env=required_envs) - heter3_proc = subprocess.Popen(heter3_cmd.strip().split(" "), - stdout=heter3_out, - stderr=heter3_pipe, - env=required_envs) - - return heter0_proc, heter1_proc, heter2_proc, heter3_proc, heter0_pipe, heter1_pipe, heter2_pipe, heter3_pipe + heter0_proc = subprocess.Popen( + heter0_cmd.strip().split(" "), + stdout=heter0_out, + stderr=heter0_pipe, + env=required_envs, + ) + heter1_proc = subprocess.Popen( + heter1_cmd.strip().split(" "), + stdout=heter1_out, + stderr=heter1_pipe, + env=required_envs, + ) + heter2_proc = subprocess.Popen( + heter2_cmd.strip().split(" "), + stdout=heter2_out, + stderr=heter2_pipe, + env=required_envs, + ) + heter3_proc = subprocess.Popen( + heter3_cmd.strip().split(" "), + stdout=heter3_out, + stderr=heter3_pipe, + env=required_envs, + ) + + return ( + heter0_proc, + heter1_proc, + heter2_proc, + heter3_proc, + heter0_pipe, + heter1_pipe, + heter2_pipe, + heter3_pipe, + ) def _run_cluster(self, model, envs): env = { 'GRAD_CLIP': str(self._grad_clip_mode), - 'FLAGS_eager_delete_tensor_gb': str(-1) + 'FLAGS_eager_delete_tensor_gb': str(-1), } python_path = self._python_interp gloo_path = tempfile.mkdtemp() @@ -312,31 +375,64 @@ class TestFleetHeterBase(unittest.TestCase): python_path += " -m coverage run --branch -p" env.update(envs) self._all_heter_endpoints = ";".join( - (self._heter_endpoints, self._heter_endpoints_2)) + (self._heter_endpoints, self._heter_endpoints_2) + ) tr_cmd = "{0} {1} --role trainer --endpoints {2} --trainer_endpoints {3} --current_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --heter_trainer_endpoints {9} --heter_trainer_device {10}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._all_heter_endpoints, - self._heter_device) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._all_heter_endpoints, + self._heter_device, + ) ps_cmd = "{0} {1} --role pserver --endpoints {2} --trainer_endpoints {3} --current_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --heter_trainer_endpoints {9} --heter_trainer_device {10}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._all_heter_endpoints, - self._heter_device) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._all_heter_endpoints, + self._heter_device, + ) heter_cmd = "{0} {1} --role heter_trainer --endpoints {2} --trainer_endpoints {3} --current_id {{}} --stage_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --heter_trainer_endpoints {9} --heter_trainer_device {10}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._all_heter_endpoints, - self._heter_device) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._all_heter_endpoints, + self._heter_device, + ) # Run dist train to compare with local results ps0, ps1, ps0_pipe, ps1_pipe = self._start_pserver(ps_cmd, env) tr0, tr1, tr0_pipe, tr1_pipe = self._start_trainer(tr_cmd, env) - heter0, heter1, heter2, heter3, heter0_pipe, heter1_pipe, heter2_pipe, heter3_pipe = self._start_heter_trainer( - heter_cmd, env) + ( + heter0, + heter1, + heter2, + heter3, + heter0_pipe, + heter1_pipe, + heter2_pipe, + heter3_pipe, + ) = self._start_heter_trainer(heter_cmd, env) # Wait until trainer process terminate while True: @@ -379,17 +475,15 @@ class TestFleetHeterBase(unittest.TestCase): shutil.rmtree(gloo_path) return 0, 0 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "" + "http_proxy": "", } required_envs.update(need_envs) @@ -403,32 +497,30 @@ class TestFleetHeterBase(unittest.TestCase): def runtime_main(test_class): parser = argparse.ArgumentParser(description='Run Fleet test.') - parser.add_argument('--role', - type=str, - required=True, - choices=['pserver', 'trainer', 'heter_trainer']) + parser.add_argument( + '--role', + type=str, + required=True, + choices=['pserver', 'trainer', 'heter_trainer'], + ) parser.add_argument('--endpoints', type=str, required=False, default="") - parser.add_argument('--trainer_endpoints', - type=str, - required=False, - default="") - parser.add_argument('--heter_trainer_endpoints', - type=str, - required=False, - default="") - parser.add_argument('--heter_trainer_device', - type=str, - required=False, - default="gpu") + parser.add_argument( + '--trainer_endpoints', type=str, required=False, default="" + ) + parser.add_argument( + '--heter_trainer_endpoints', type=str, required=False, default="" + ) + parser.add_argument( + '--heter_trainer_device', type=str, required=False, default="gpu" + ) parser.add_argument('--gloo_path', type=str, required=False, default="") parser.add_argument('--current_id', type=int, required=False, default=0) parser.add_argument('--trainers', type=int, required=False, default=1) parser.add_argument('--stage_id', type=int, required=False, default=1) parser.add_argument('--mode', type=str, required=False, default='async') - parser.add_argument('--geo_sgd_need_push_nums', - type=int, - required=False, - default=2) + parser.add_argument( + '--geo_sgd_need_push_nums', type=int, required=False, default=2 + ) parser.add_argument('--reader', type=str, required=False, default='dataset') args = parser.parse_args() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_program.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_program.py index bf7ced9ce60..251a2bf25d9 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_program.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_program.py @@ -25,16 +25,18 @@ paddle.enable_static() class TestDistFleetHeterProgram(unittest.TestCase): - def build_role(self): environs = {} environs[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36012,127.0.0.1:36013" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36012,127.0.0.1:36013" environs["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36014,127.0.0.1:36015" environs[ - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST"] = "127.0.0.1:36016,127.0.0.1:36017" + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST" + ] = "127.0.0.1:36016,127.0.0.1:36017" environs[ - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST"] = "127.0.0.1:36014,127.0.0.1:36015" + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST" + ] = "127.0.0.1:36014,127.0.0.1:36015" environs["PADDLE_HETER_TRAINER_DEVICE"] = "gpu" environs["TRAINING_ROLE"] = "HETER_TRAINER" environs["STAGE_ID"] = 2 @@ -58,20 +60,20 @@ class TestDistFleetHeterProgram(unittest.TestCase): self.strategy.a_sync = True self.strategy.a_sync_configs = { "launch_barrier": False, - "heter_worker_device_guard": "gpu" + "heter_worker_device_guard": "gpu", } return self.strategy def build_input(self): - dense_input = fluid.layers.data(name="dense_input", - shape=[10], - dtype="float32") + dense_input = fluid.layers.data( + name="dense_input", shape=[10], dtype="float32" + ) sparse_input_ids = [ - fluid.layers.data(name="C" + str(i), - shape=[1], - lod_level=1, - dtype="int64") for i in range(1, 27) + fluid.layers.data( + name="C" + str(i), shape=[1], lod_level=1, dtype="int64" + ) + for i in range(1, 27) ] label = fluid.layers.data(name="label", shape=[1], dtype="float32") @@ -80,7 +82,6 @@ class TestDistFleetHeterProgram(unittest.TestCase): return inputs def build_net(self, inputs): - def embedding_layer(input): return fluid.layers.embedding( input=input, @@ -88,7 +89,8 @@ class TestDistFleetHeterProgram(unittest.TestCase): size=[100001, 10], param_attr=fluid.ParamAttr( name="SparseFeatFactors", - initializer=fluid.initializer.Uniform()), + initializer=fluid.initializer.Uniform(), + ), ) sparse_embed_seq = list(map(embedding_layer, inputs[1:-1])) @@ -100,35 +102,50 @@ class TestDistFleetHeterProgram(unittest.TestCase): input=concated, size=400, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(concated.shape[1]))), - name="fc1") + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(concated.shape[1]) + ) + ), + name="fc1", + ) with fluid.device_guard("cpu"): fc2 = fluid.layers.fc( input=fc1, size=400, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(fc1.shape[1]))), - name="fc2") + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(fc1.shape[1]) + ) + ), + name="fc2", + ) with fluid.device_guard("gpu"): fc3 = fluid.layers.fc( input=fc2, size=400, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(fc2.shape[1]))), - name="fc3") + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(fc2.shape[1]) + ) + ), + name="fc3", + ) with fluid.device_guard("cpu"): predict = fluid.layers.fc( input=fc3, size=2, act="softmax", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(fc3.shape[1]))), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(fc3.shape[1]) + ) + ), ) with fluid.device_guard("gpu"): diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_infer.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_infer.py index 362474e2767..c0c7070cf80 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_infer.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_infer.py @@ -20,16 +20,13 @@ from test_dist_fleet_base import TestFleetBase class TestDistCtrInfer(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -54,35 +51,34 @@ class TestDistCtrInfer(TestFleetBase): def test_dist_infer(self): model_dirname = tempfile.mkdtemp() - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False, - need_envs={ - "SAVE_DIRNAME": model_dirname, - }) + self.check_with_place( + "dist_fleet_ctr.py", + delta=1e-5, + check_error_log=False, + need_envs={ + "SAVE_DIRNAME": model_dirname, + }, + ) self._need_test = 1 self._model_dir = model_dirname - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) shutil.rmtree(model_dirname) class TestDistCtrTrainInfer(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" self._need_test = 1 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), @@ -106,9 +102,9 @@ class TestDistCtrTrainInfer(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train_infer(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps.py index 137beef5f2d..8d5ac58d62a 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps.py @@ -34,35 +34,34 @@ batch_size = 4 class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ class TestPSPassWithBow(unittest.TestCase): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.layers.embedding( input=q, @@ -82,8 +80,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -95,14 +95,15 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.layers.embedding( input=pt, @@ -111,8 +112,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.layers.embedding( input=nt, @@ -139,8 +143,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -152,8 +158,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -164,14 +172,18 @@ class TestPSPassWithBow(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = fleet.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = fleet.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps10.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps10.py index 332f314106b..db553b99034 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps10.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps10.py @@ -35,18 +35,17 @@ batch_size = 4 class TestExponentialDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -59,20 +58,24 @@ class TestExponentialDecay(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.InverseTimeDecay(learning_rate=base_lr, - gamma=0.999, - verbose=True) + scheduler = paddle.optimizer.lr.InverseTimeDecay( + learning_rate=base_lr, gamma=0.999, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps11.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps11.py index 38fccfa5d88..b0d8df316a8 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps11.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps11.py @@ -34,35 +34,34 @@ batch_size = 4 class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -78,7 +77,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -90,7 +91,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt @@ -102,7 +105,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -114,8 +119,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt nt = fluid.layers.data(name="3", shape=[1], dtype="int64", lod_level=1) # embedding @@ -125,7 +132,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -137,8 +146,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -155,9 +166,11 @@ class TestPSPassWithBow(unittest.TestCase): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002,127.0.0.2:36002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36002,127.0.0.2:36002" os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["FLAGS_selected_gpus"] = "0" role = role_maker.PaddleCloudRoleMaker() @@ -191,22 +204,22 @@ class TestPSPassWithBow(unittest.TestCase): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() dataset._set_use_ps_gpu(True) - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) - dataset.set_filelist([ - "test_in_memory_dataset_run_a.txt", - "test_in_memory_dataset_run_b.txt" - ]) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) + dataset.set_filelist( + [ + "test_in_memory_dataset_run_a.txt", + "test_in_memory_dataset_run_b.txt", + ] + ) os.remove("./test_in_memory_dataset_run_a.txt") os.remove("./test_in_memory_dataset_run_b.txt") diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps12.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps12.py index 814e49247dc..b4c10116a55 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps12.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps12.py @@ -37,35 +37,34 @@ batch_size = 4 class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -81,7 +80,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -93,7 +94,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt @@ -105,7 +108,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -117,8 +122,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt nt = fluid.layers.data(name="3", shape=[1], dtype="int64", lod_level=1) # embedding @@ -128,7 +135,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -140,8 +149,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -158,7 +169,8 @@ class TestPSPassWithBow(unittest.TestCase): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ["TRAINING_ROLE"] = "PSERVER" role = role_maker.PaddleCloudRoleMaker() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps13.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps13.py index 99dd076e829..47cbaefd68d 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps13.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps13.py @@ -38,35 +38,34 @@ batch_size = 4 # this unittest is tested for SparseSharedAdamSGDRule class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = fluid.layers.mean(loss_op3) return avg_cost @@ -74,10 +73,9 @@ class TestPSPassWithBow(unittest.TestCase): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.contrib.layers.sparse_embedding( input=q, @@ -85,7 +83,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -97,14 +97,15 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.contrib.layers.sparse_embedding( input=pt, @@ -112,7 +113,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.contrib.layers.sparse_embedding( input=nt, @@ -138,7 +142,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -150,8 +156,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -168,7 +176,8 @@ class TestPSPassWithBow(unittest.TestCase): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ["TRAINING_ROLE"] = "PSERVER" role = role_maker.PaddleCloudRoleMaker() @@ -180,10 +189,8 @@ class TestPSPassWithBow(unittest.TestCase): configs = {} configs['__emb__'] = { - "table_parameters.__emb__.accessor.embed_sgd_param.name": - "SparseSharedAdamSGDRule", - "table_parameters.__emb__.accessor.embedx_sgd_param.name": - "SparseSharedAdamSGDRule", + "table_parameters.__emb__.accessor.embed_sgd_param.name": "SparseSharedAdamSGDRule", + "table_parameters.__emb__.accessor.embedx_sgd_param.name": "SparseSharedAdamSGDRule", } strategy.sparse_table_configs = configs optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps2.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps2.py index 0ab59f7ca88..a4cdcb32bd4 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps2.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps2.py @@ -37,35 +37,34 @@ batch_size = 4 class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -73,10 +72,9 @@ class TestPSPassWithBow(unittest.TestCase): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.contrib.layers.sparse_embedding( input=q, @@ -84,7 +82,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -97,14 +97,15 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.contrib.layers.sparse_embedding( input=pt, @@ -112,7 +113,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.contrib.layers.sparse_embedding( input=nt, @@ -138,7 +142,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -150,8 +156,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -168,7 +176,8 @@ class TestPSPassWithBow(unittest.TestCase): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ["TRAINING_ROLE"] = "PSERVER" role = role_maker.PaddleCloudRoleMaker() @@ -180,14 +189,10 @@ class TestPSPassWithBow(unittest.TestCase): configs = {} configs['__emb__'] = { - "table_parameters.__emb__.enable_sparse_table_cache": - True, - "table_parameters.__emb__.shard_merge_rate": - 1, - "table_parameters.__emb__.accessor.embed_sgd_param.name": - "SparseNaiveSGDRule", - "table_parameters.__emb__.accessor.embedx_sgd_param.name": - "SparseAdamSGDRule", + "table_parameters.__emb__.enable_sparse_table_cache": True, + "table_parameters.__emb__.shard_merge_rate": 1, + "table_parameters.__emb__.accessor.embed_sgd_param.name": "SparseNaiveSGDRule", + "table_parameters.__emb__.accessor.embedx_sgd_param.name": "SparseAdamSGDRule", } strategy.sparse_table_configs = configs optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps3.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps3.py index 8a7556a2685..2eb62770184 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps3.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps3.py @@ -34,35 +34,34 @@ batch_size = 4 class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ class TestPSPassWithBow(unittest.TestCase): is_sparse = False # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.layers.embedding( input=q, @@ -82,8 +80,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -95,14 +95,15 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.layers.embedding( input=pt, @@ -111,8 +112,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.layers.embedding( input=nt, @@ -139,8 +143,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -152,8 +158,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -164,14 +172,18 @@ class TestPSPassWithBow(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = fleet.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.WORKER, - worker_num=2, - server_endpoints=endpoints) + role = fleet.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.WORKER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps4.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps4.py index fae0cd92618..0cb4ee6e3af 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps4.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps4.py @@ -34,35 +34,34 @@ batch_size = 4 class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ class TestPSPassWithBow(unittest.TestCase): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.contrib.layers.sparse_embedding( input=q, @@ -81,7 +79,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -93,14 +93,15 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.contrib.layers.sparse_embedding( input=pt, @@ -108,7 +109,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -120,13 +123,14 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.contrib.layers.sparse_embedding( input=nt, @@ -134,7 +138,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -146,8 +152,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -158,14 +166,18 @@ class TestPSPassWithBow(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps5.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps5.py index c91d86d89ca..fbb640fc8bb 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps5.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps5.py @@ -34,35 +34,34 @@ batch_size = 4 class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ class TestPSPassWithBow(unittest.TestCase): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.layers.embedding( input=q, @@ -82,8 +80,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -95,14 +95,15 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.layers.embedding( input=pt, @@ -111,8 +112,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.layers.embedding( input=nt, @@ -139,8 +143,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__tmp_", - learning_rate=emb_lr), - is_sparse=False) + learning_rate=emb_lr, + ), + is_sparse=False, + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -152,8 +158,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -164,23 +172,30 @@ class TestPSPassWithBow(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() optimizer = fluid.optimizer.Adam( - learning_rate=fluid.layers.exponential_decay(learning_rate=base_lr, - decay_steps=500, - decay_rate=0.969, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=base_lr, + decay_steps=500, + decay_rate=0.969, + staircase=True, + ) + ) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.a_sync = True diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps6.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps6.py index 21dfcee7e8e..e0b73b4344c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps6.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps6.py @@ -34,35 +34,34 @@ batch_size = 4 class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ class TestPSPassWithBow(unittest.TestCase): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.contrib.layers.sparse_embedding( input=q, @@ -81,7 +79,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -93,14 +93,15 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.contrib.layers.sparse_embedding( input=pt, @@ -108,7 +109,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -120,13 +123,14 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.contrib.layers.sparse_embedding( input=nt, @@ -134,7 +138,9 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -146,8 +152,10 @@ class TestPSPassWithBow(unittest.TestCase): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -158,14 +166,18 @@ class TestPSPassWithBow(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps7.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps7.py index bdba0d90bb0..9351791260b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps7.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps7.py @@ -36,18 +36,17 @@ batch_size = 4 class TestNaturalExpDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -60,20 +59,24 @@ class TestNaturalExpDecay(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.NaturalExpDecay(learning_rate=base_lr, - gamma=0.999, - verbose=True) + scheduler = paddle.optimizer.lr.NaturalExpDecay( + learning_rate=base_lr, gamma=0.999, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps8.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps8.py index 3dcb16f5444..cbf12f19714 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps8.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps8.py @@ -35,18 +35,17 @@ batch_size = 4 class TestNoamDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -59,20 +58,24 @@ class TestNoamDecay(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True) + scheduler = paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps9.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps9.py index 958ad4c6052..bebc6bbb965 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps9.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps9.py @@ -35,18 +35,17 @@ batch_size = 4 class TestExponentialDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -59,20 +58,24 @@ class TestExponentialDecay(unittest.TestCase): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.ExponentialDecay(learning_rate=base_lr, - gamma=0.999, - verbose=True) + scheduler = paddle.optimizer.lr.ExponentialDecay( + learning_rate=base_lr, gamma=0.999, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py index b49a2599b76..ca3c5badc09 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py @@ -22,7 +22,6 @@ flag_name = os.path.splitext(__file__)[0] class TestFleetMetaOptimizerPrecision(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -34,11 +33,14 @@ class TestFleetMetaOptimizerPrecision(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_fleet_raw_program_optimizer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_fleet_raw_program_optimizer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer_fuse_allreduce.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer_fuse_allreduce.py index be85ea71040..a6b994dde6d 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer_fuse_allreduce.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer_fuse_allreduce.py @@ -22,7 +22,6 @@ flag_name = os.path.splitext(__file__)[0] class TestFleetMetaOptimizerAllReduceFusePrecision(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -34,12 +33,14 @@ class TestFleetMetaOptimizerAllReduceFusePrecision(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place( "dist_fleet_raw_program_optimizer_fuse_allreduce.py", delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_simnet.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_simnet.py index b2bf73511ce..27c76bee6ed 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_simnet.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_simnet.py @@ -21,23 +21,20 @@ paddle.enable_static() class TestDistSimnetASync2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "FLAGS_rpc_deadline": "5000", # 5sec to fail fast "http_proxy": "", - "CPU_NUM": "2" + "CPU_NUM": "2", } required_envs.update(need_envs) @@ -49,9 +46,9 @@ class TestDistSimnetASync2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_simnet_bow.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_simnet_bow.py", delta=1e-5, check_error_log=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py index 7fc0cc60aec..13eaa703ddc 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py @@ -28,16 +28,13 @@ from dist_fleet_sparse_embedding_ctr import fake_ctr_reader @unittest.skip(reason="Skip unstable ut, need paddle sync mode fix") class TestDistMnistSync2x2(TestFleetBase): - def _setup_config(self): self._mode = "sync" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -58,22 +55,21 @@ class TestDistMnistSync2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) class TestDistMnistAsync2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -94,22 +90,21 @@ class TestDistMnistAsync2x2(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) class TestDistMnistAsync2x2WithDecay(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -131,22 +126,21 @@ class TestDistMnistAsync2x2WithDecay(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) class TestDistMnistAsync2x2WithUnifrom(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -168,20 +162,20 @@ class TestDistMnistAsync2x2WithUnifrom(TestFleetBase): tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) @unittest.skip(reason="Skip unstable ut, need tensor table to enhance") class TestDistMnistAsync2x2WithGauss(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" def _run_local_infer(self, model_file): - def net(): """ network definition @@ -194,21 +188,27 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase): """ dnn_input_dim, lr_input_dim = 10, 10 - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] @@ -220,10 +220,13 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase): input=dnn_data, size=[dnn_input_dim, dnn_layer_dims[0]], is_test=inference, - param_attr=fluid.ParamAttr(name="deep_embedding", - initializer=init)) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + param_attr=fluid.ParamAttr( + name="deep_embedding", initializer=init + ), + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): fc = fluid.layers.fc( @@ -231,8 +234,10 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -242,10 +247,13 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase): is_test=inference, param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01), + ), + ) - lr_pool = fluid.layers.sequence_pool(input=lr_embbding, - pool_type="sum") + lr_pool = fluid.layers.sequence_pool( + input=lr_embbding, pool_type="sum" + ) merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) predict = fluid.layers.fc(input=merge_layer, size=2, act='softmax') return datas, predict @@ -259,15 +267,15 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase): fluid.io.load_persistables(exe, model_file) for batch_id, data in enumerate(reader()): - score = exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[predict]) - - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + score = exe.run( + fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[predict], + ) + + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): model_dir = tempfile.mkdtemp() required_envs = { @@ -293,9 +301,11 @@ class TestDistMnistAsync2x2WithGauss(TestFleetBase): shutil.rmtree(model_dir) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_trainer_desc_config.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_trainer_desc_config.py index 0090d6bef15..e64b7d8010e 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_trainer_desc_config.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_trainer_desc_config.py @@ -23,15 +23,15 @@ paddle.enable_static() class TestDistStrategyTrainerDescConfig(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_trainer_desc_config(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -50,7 +50,7 @@ class TestDistStrategyTrainerDescConfig(unittest.TestCase): config = { "dump_fields_path": "dump_data", "dump_fields": ["xxx", "yyy"], - "dump_param": ['zzz'] + "dump_param": ['zzz'], } strategy.trainer_desc_configs = config @@ -62,8 +62,10 @@ class TestDistStrategyTrainerDescConfig(unittest.TestCase): self.assertEqual(program._fleet_opt["dump_fields_path"], "dump_data") self.assertEqual(len(program._fleet_opt["dump_fields"]), 2) self.assertEqual(len(program._fleet_opt["dump_param"]), 1) - self.assertEqual(program._fleet_opt["mpi_size"], - int(os.environ["PADDLE_TRAINERS_NUM"])) + self.assertEqual( + program._fleet_opt["mpi_size"], + int(os.environ["PADDLE_TRAINERS_NUM"]), + ) optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) @@ -73,8 +75,10 @@ class TestDistStrategyTrainerDescConfig(unittest.TestCase): self.assertEqual(program._fleet_opt["dump_fields_path"], "dump_data") self.assertEqual(len(program._fleet_opt["dump_fields"]), 2) self.assertEqual(len(program._fleet_opt["dump_param"]), 1) - self.assertEqual(program._fleet_opt["mpi_size"], - int(os.environ["PADDLE_TRAINERS_NUM"])) + self.assertEqual( + program._fleet_opt["mpi_size"], + int(os.environ["PADDLE_TRAINERS_NUM"]), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_lookup_sparse_table_fuse_ops.py b/python/paddle/fluid/tests/unittests/test_dist_lookup_sparse_table_fuse_ops.py index 8a753e52de7..f64a7e6882e 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_lookup_sparse_table_fuse_ops.py +++ b/python/paddle/fluid/tests/unittests/test_dist_lookup_sparse_table_fuse_ops.py @@ -25,7 +25,6 @@ paddle.enable_static() @unittest.skip("do not need currently") class TestLookupTableFuseOp(unittest.TestCase): - def test_fuse(self): places = [core.CPUPlace()] # currently only support CPU @@ -44,20 +43,23 @@ class TestLookupTableFuseOp(unittest.TestCase): persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[1], - dtype="float32") + dtype="float32", + ) ids = init_program.global_block().create_var( name="Ids", persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[100], - dtype="int64") + dtype="int64", + ) output = init_program.global_block().create_var( name="output", type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[100, 8], - dtype="float32") + dtype="float32", + ) metas = [] metas.append( @@ -71,27 +73,30 @@ class TestLookupTableFuseOp(unittest.TestCase): type="lookup_sparse_table_init", inputs=None, outputs=None, - attrs={"large_scale_metas": metas}) - - init_program.global_block().append_op(type="lookup_sparse_table_read", - inputs={"Ids": ids}, - outputs={"Out": output}, - attrs={ - "tablename": - "embedding_1.block0", - "init": True, - "value_names": ["Param"], - }) - - init_program.global_block().append_op(type="lookup_sparse_table_read", - inputs={"Ids": ids}, - outputs={"Out": output}, - attrs={ - "tablename": - "embedding_2.block0", - "init": True, - "value_names": ["Param"], - }) + attrs={"large_scale_metas": metas}, + ) + + init_program.global_block().append_op( + type="lookup_sparse_table_read", + inputs={"Ids": ids}, + outputs={"Out": output}, + attrs={ + "tablename": "embedding_1.block0", + "init": True, + "value_names": ["Param"], + }, + ) + + init_program.global_block().append_op( + type="lookup_sparse_table_read", + inputs={"Ids": ids}, + outputs={"Out": output}, + attrs={ + "tablename": "embedding_2.block0", + "init": True, + "value_names": ["Param"], + }, + ) executor = fluid.Executor(place) executor.run(init_program) @@ -99,9 +104,11 @@ class TestLookupTableFuseOp(unittest.TestCase): training_program = fluid.Program() scope.var('Beta1Pow').get_tensor().set( - np.array([0]).astype("float32"), place) + np.array([0]).astype("float32"), place + ) scope.var('Beta2Pow').get_tensor().set( - np.array([0]).astype("float32"), place) + np.array([0]).astype("float32"), place + ) rows = [0, 1, 2, 3, 4, 5, 6] row_numel = 8 @@ -119,28 +126,32 @@ class TestLookupTableFuseOp(unittest.TestCase): persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[1], - dtype="float32") + dtype="float32", + ) grads = training_program.global_block().create_var( name="Grad", persistable=True, type=fluid.core.VarDesc.VarType.SELECTED_ROWS, shape=[100, 8], - dtype="float32") + dtype="float32", + ) beta1 = training_program.global_block().create_var( name="Beta1Pow", persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[1], - dtype="float32") + dtype="float32", + ) beta2 = training_program.global_block().create_var( name="Beta2Pow", persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[1], - dtype="float32") + dtype="float32", + ) training_program.global_block().append_op( type="lookup_sparse_table_fuse_adam", @@ -150,27 +161,23 @@ class TestLookupTableFuseOp(unittest.TestCase): "Beta1Pow": beta1, "Beta2Pow": beta2, }, - outputs={ - "Beta1PowOut": beta1, - "Beta2PowOut": beta2 - }, + outputs={"Beta1PowOut": beta1, "Beta2PowOut": beta2}, attrs={ "is_entry": False, "tablename": "embedding_1.block0", "value_names": ["Param", "Moment1", "Moment2"], - }) + }, + ) training_program.global_block().append_op( type="lookup_sparse_table_fuse_sgd", - inputs={ - "Grad": grads, - "LearningRate": lr - }, + inputs={"Grad": grads, "LearningRate": lr}, attrs={ "is_entry": False, "tablename": "embedding_2.block0", "value_names": ["Param"], - }) + }, + ) executor.run(training_program) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_backward_deps.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_backward_deps.py index 425e8c21aae..ba1037b0573 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_backward_deps.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_backward_deps.py @@ -20,7 +20,6 @@ paddle.enable_static() class TestDistMnistNCCL2BackWardDeps(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -30,6 +29,7 @@ class TestDistMnistNCCL2BackWardDeps(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_batch_merge.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_batch_merge.py index b21e7b208b2..a480b987317 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_batch_merge.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_batch_merge.py @@ -23,7 +23,6 @@ flag_name = os.path.splitext(__file__)[0] class TestDistMnist2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -31,11 +30,9 @@ class TestDistMnist2x2(TestDistBase): def test_dist_train(self): self.check_with_place("dist_mnist_batch_merge.py", delta=1e-5) - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): # TODO(typhoonzero): should auto adapt GPU count on the machine. required_envs = { "PATH": os.getenv("PATH", ""), @@ -48,22 +45,27 @@ class TestDistMnist2x2(TestDistBase): required_envs.update(need_envs) if check_error_log: - required_envs["GLOG_vmodule"] = \ - "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" + required_envs[ + "GLOG_vmodule" + ] = "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" required_envs["GLOG_logtostderr"] = "1" - no_merge_losses = self._run_local(model_file, - required_envs, - check_error_log=check_error_log, - batch_size=4, - log_name=flag_name) + no_merge_losses = self._run_local( + model_file, + required_envs, + check_error_log=check_error_log, + batch_size=4, + log_name=flag_name, + ) - batch_merge_losses = self._run_local(model_file, - required_envs, - check_error_log=check_error_log, - batch_size=2, - batch_merge_repeat=2, - log_name=flag_name) + batch_merge_losses = self._run_local( + model_file, + required_envs, + check_error_log=check_error_log, + batch_size=2, + batch_merge_repeat=2, + log_name=flag_name, + ) # Ensure both result have values. self.assertGreater(len(no_merge_losses), 1) self.assertEqual(len(no_merge_losses), len(batch_merge_losses)) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_fleet_save.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_fleet_save.py index a83a3e83eca..9a1afd250d8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_fleet_save.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_fleet_save.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestDistMnistFleetSave(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -78,12 +77,14 @@ class TestDistMnistFleetSave(TestDistBase): self._rm_temp_files(dirname) return True - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): required_envs = self._get_required_envs(check_error_log, need_envs) tr0_losses, tr1_losses = self._run_cluster_nccl2( @@ -91,13 +92,15 @@ class TestDistMnistFleetSave(TestDistBase): required_envs, update_method='nccl2', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) dirname = '/tmp' self._test_saved_files(dirname) def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_fleetapi.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_fleetapi.py index 6766b1c1bae..fbfdccfeff4 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_fleetapi.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_fleetapi.py @@ -20,7 +20,6 @@ paddle.enable_static() class TestDistMnistNCCL2FleetApi(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -31,20 +30,24 @@ class TestDistMnistNCCL2FleetApi(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place( "dist_mnist.py", delta=1e-5, check_error_log=True, - need_envs={'FLAGS_allreduce_record_one_event': '1'}) + need_envs={'FLAGS_allreduce_record_one_event': '1'}, + ) class FleetCollectiveTest(unittest.TestCase): - def test_open_sync_batch_norm(self): import paddle.fluid as fluid import paddle.fluid.incubate.fleet.base.role_maker as role_maker - from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy + from paddle.fluid.incubate.fleet.collective import ( + fleet, + DistributedStrategy, + ) if not fluid.core.is_compiled_with_cuda(): # Operator "gen_nccl_id" has not been registered @@ -62,8 +65,9 @@ class FleetCollectiveTest(unittest.TestCase): dist_strategy = DistributedStrategy() dist_strategy.sync_batch_norm = True - dist_optimizer = fleet.distributed_optimizer(optimizer, - strategy=dist_strategy) + dist_optimizer = fleet.distributed_optimizer( + optimizer, strategy=dist_strategy + ) dist_optimizer.minimize(loss) self.assertEqual(dist_strategy.exec_strategy.num_threads, 1) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_fp16_allreduce.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_fp16_allreduce.py index cff295d5802..ce74d92a7b8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_fp16_allreduce.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_fp16_allreduce.py @@ -17,7 +17,6 @@ from test_dist_base import TestDistBase class TestDistMnist2x2FP16AllReduce(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -25,6 +24,7 @@ class TestDistMnist2x2FP16AllReduce(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist_fp16_allreduce.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_hallreduce.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_hallreduce.py index 93640b06ee6..ca1b7299b8c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_hallreduce.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_hallreduce.py @@ -23,7 +23,6 @@ flag_name = os.path.splitext(__file__)[0] class TestDistMnistNCCL2HAllreduce(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -34,11 +33,14 @@ class TestDistMnistNCCL2HAllreduce(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_lars.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_lars.py index 03a4da1f45e..81b96e7c899 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_lars.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_lars.py @@ -17,7 +17,6 @@ from test_dist_base import TestDistBase class TestDistMnist2x2Lars(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_multi_comm.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_multi_comm.py index 57d89b2d23d..82f09bcfae1 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_multi_comm.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_multi_comm.py @@ -23,7 +23,6 @@ flag_name = os.path.splitext(__file__)[0] class TestDistMnistNCCL2MultiNCCLComm(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,11 +32,14 @@ class TestDistMnistNCCL2MultiNCCLComm(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_pg.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_pg.py index b813e96971e..e517e38978e 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_pg.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_pg.py @@ -20,7 +20,6 @@ paddle.enable_static() class TestDistMnistNCCL2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -29,13 +28,16 @@ class TestDistMnistNCCL2(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist.py", - delta=1, - need_envs={ - "FLAGS_enable_parallel_graph": "1", - "FLAGS_sync_nccl_allreduce": "1" - }) + self.check_with_place( + "dist_mnist.py", + delta=1, + need_envs={ + "FLAGS_enable_parallel_graph": "1", + "FLAGS_sync_nccl_allreduce": "1", + }, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_ring_allreduce.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_ring_allreduce.py index c455dbe270a..40ae6686ed9 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_ring_allreduce.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_ring_allreduce.py @@ -20,7 +20,6 @@ paddle.enable_static() class TestDistMnistNCCL2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -29,6 +28,7 @@ class TestDistMnistNCCL2(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_train.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_train.py index af2392b6852..f6dfa739846 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_train.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_train.py @@ -21,55 +21,53 @@ flag_name = os.path.splitext(__file__)[0] class TestDistMnist2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False def test_dist_train(self): - self.check_with_place("dist_mnist.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestDistMnist2x2WithMemopt(TestDistBase): - def _setup_config(self): self._sync_mode = True self._mem_opt = True def test_dist_train(self): - self.check_with_place("dist_mnist.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestDistMnistAsync(TestDistBase): - def _setup_config(self): self._sync_mode = False self._use_reduce = False def test_dist_train(self): - self.check_with_place("dist_mnist.py", - delta=200, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", delta=200, check_error_log=True, log_name=flag_name + ) class TestDistMnistDcAsgd(TestDistBase): - def _setup_config(self): self._sync_mode = False self._dc_asgd = True def test_se_resnext(self): - self.check_with_place("dist_mnist.py", - delta=200, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", delta=200, check_error_log=True, log_name=flag_name + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_with_program.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_with_program.py index 4475b52ee79..83c9a340e32 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_with_program.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_with_program.py @@ -20,7 +20,6 @@ paddle.enable_static() class TestDistMnistLocalSGDFleetApi(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -31,12 +30,12 @@ class TestDistMnistLocalSGDFleetApi(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) class TestDistMnistGradAllReduceFleetApi(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -47,6 +46,7 @@ class TestDistMnistGradAllReduceFleetApi(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_oneps.py b/python/paddle/fluid/tests/unittests/test_dist_oneps.py index 7704a4c715e..c6b25263936 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_oneps.py +++ b/python/paddle/fluid/tests/unittests/test_dist_oneps.py @@ -22,7 +22,6 @@ from paddle.distributed.fleet.runtime.the_one_ps import Table class TestTable(unittest.TestCase): - def test_table_tensor(self): table = Table() table.id = 1001 diff --git a/python/paddle/fluid/tests/unittests/test_dist_op.py b/python/paddle/fluid/tests/unittests/test_dist_op.py index 37835d54176..e627567b1af 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_op.py +++ b/python/paddle/fluid/tests/unittests/test_dist_op.py @@ -23,7 +23,7 @@ paddle.enable_static() def dist(x, y, p): - if p == 0.: + if p == 0.0: out = np.count_nonzero(x - y) elif p == float("inf"): out = np.max(np.abs(x - y)) @@ -35,7 +35,6 @@ def dist(x, y, p): class TestDistOp(OpTest): - def setUp(self): self.op_type = 'dist' self.python_api = paddle.dist @@ -44,7 +43,7 @@ class TestDistOp(OpTest): self.init_data_type() self.inputs = { "X": np.random.random(self.x_shape).astype(self.data_type), - "Y": np.random.random(self.y_shape).astype(self.data_type) + "Y": np.random.random(self.y_shape).astype(self.data_type), } self.attrs["p"] = self.p @@ -54,13 +53,14 @@ class TestDistOp(OpTest): self.gradient = self.calc_gradient() def init_case(self): - self.x_shape = (120) - self.y_shape = (120) - self.p = 0. + self.x_shape = 120 + self.y_shape = 120 + self.p = 0.0 def init_data_type(self): - self.data_type = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.data_type = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def calc_gradient(self): x = self.inputs["X"] @@ -75,8 +75,11 @@ class TestDistOp(OpTest): grad[x_minux_y_abs != norm] = 0 else: norm = dist(x, y, p) - grad = np.power(norm, 1 - p) * np.power(np.abs(x - y), - p - 1) * np.sign(x - y) + grad = ( + np.power(norm, 1 - p) + * np.power(np.abs(x - y), p - 1) + * np.sign(x - y) + ) def get_reduce_dims(x, y): x_reduce_dims = [] @@ -111,30 +114,29 @@ class TestDistOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(["X", "Y"], - "Out", - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad( + ["X", "Y"], + "Out", + user_defined_grads=self.gradient, + check_eager=True, + ) class TestDistOpCase1(TestDistOp): - def init_case(self): self.x_shape = (3, 5, 5, 6) self.y_shape = (5, 5, 6) - self.p = 1. + self.p = 1.0 class TestDistOpCase2(TestDistOp): - def init_case(self): self.x_shape = (10, 10) self.y_shape = (4, 10, 10) - self.p = 2. + self.p = 2.0 class TestDistOpCase3(TestDistOp): - def init_case(self): self.x_shape = (15, 10) self.y_shape = (15, 10) @@ -142,7 +144,6 @@ class TestDistOpCase3(TestDistOp): class TestDistOpCase4(TestDistOp): - def init_case(self): self.x_shape = (2, 3, 4, 5, 8) self.y_shape = (3, 1, 5, 8) @@ -150,7 +151,6 @@ class TestDistOpCase4(TestDistOp): class TestDistOpCase5(TestDistOp): - def init_case(self): self.x_shape = (4, 1, 4, 8) self.y_shape = (2, 2, 1, 4, 4, 8) @@ -158,10 +158,10 @@ class TestDistOpCase5(TestDistOp): class TestDistAPI(unittest.TestCase): - def init_data_type(self): - self.data_type = 'float32' if core.is_compiled_with_rocm( - ) else 'float64' + self.data_type = ( + 'float32' if core.is_compiled_with_rocm() else 'float64' + ) def test_api(self): self.init_data_type() @@ -174,15 +174,17 @@ class TestDistAPI(unittest.TestCase): x_i = np.random.random((2, 3, 4, 5)).astype(self.data_type) y_i = np.random.random((3, 1, 5)).astype(self.data_type) result = paddle.dist(x, y, p) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'x': x_i, 'y': y_i}, + fetch_list=[result], + ) np.testing.assert_allclose(dist(x_i, y_i, p), out[0], rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_dist_save_load.py b/python/paddle/fluid/tests/unittests/test_dist_save_load.py index bee267b941a..d4de8786622 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_dist_save_load.py @@ -27,29 +27,31 @@ flag_name = os.path.splitext(__file__)[0] class TestDistSaveLoadDense2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._enforce_place = "CPU" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "http_proxy": "" + "http_proxy": "", } required_envs.update(need_envs) if check_error_log: - required_envs["GLOG_vmodule"] = \ - "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" + required_envs[ + "GLOG_vmodule" + ] = "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" required_envs["GLOG_logtostderr"] = "1" model_dir = tempfile.mkdtemp() @@ -65,10 +67,9 @@ class TestDistSaveLoadDense2x2(TestDistBase): cluster_env.update(required_envs) local_var = self._run_local(model_file, local_env, check_error_log) - tr0_var, tr1_var = self._run_cluster(model_file, - cluster_env, - check_error_log, - log_name=flag_name) + tr0_var, tr1_var = self._run_cluster( + model_file, cluster_env, check_error_log, log_name=flag_name + ) shutil.rmtree(model_dir) @@ -87,36 +88,40 @@ class TestDistSaveLoadDense2x2(TestDistBase): 'IS_SELF_CONTAINED_LR': '1', 'SAVE_MODE': 'LOCAL', } - self.check_with_place("dist_save_load.py", - delta=0, - check_error_log=False, - need_envs=need_envs) + self.check_with_place( + "dist_save_load.py", + delta=0, + check_error_log=False, + need_envs=need_envs, + ) class TestDistSaveLoadWithPServerStateDense2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._enforce_place = "CPU" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "http_proxy": "" + "http_proxy": "", } required_envs.update(need_envs) if check_error_log: - required_envs["GLOG_vmodule"] = \ - "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" + required_envs[ + "GLOG_vmodule" + ] = "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" required_envs["GLOG_logtostderr"] = "1" model_dir = tempfile.mkdtemp() @@ -127,19 +132,17 @@ class TestDistSaveLoadWithPServerStateDense2x2(TestDistBase): save_env["MODEL_DIR"] = model_dir save_env.update(required_envs) - tr0_var_1, tr1_var_1 = self._run_cluster(model_file, - save_env, - check_error_log, - log_name=flag_name) + tr0_var_1, tr1_var_1 = self._run_cluster( + model_file, save_env, check_error_log, log_name=flag_name + ) load_env = {} load_env["LOAD"] = "1" load_env["MODEL_DIR"] = model_dir load_env.update(required_envs) - tr0_var_2, tr1_var_2 = self._run_cluster(model_file, - load_env, - check_error_log, - log_name=flag_name) + tr0_var_2, tr1_var_2 = self._run_cluster( + model_file, load_env, check_error_log, log_name=flag_name + ) shutil.rmtree(model_dir) @@ -158,13 +161,15 @@ class TestDistSaveLoadWithPServerStateDense2x2(TestDistBase): 'IS_SELF_CONTAINED_LR': '1', 'SAVE_MODE': 'DIST', 'OPTIMIZER': 'ADAM', - 'SKIP_STEPS': str(np.random.randint(2, 6)) + 'SKIP_STEPS': str(np.random.randint(2, 6)), } - self.check_with_place("dist_save_load.py", - delta=0, - check_error_log=True, - need_envs=need_envs, - log_name=flag_name) + self.check_with_place( + "dist_save_load.py", + delta=0, + check_error_log=True, + need_envs=need_envs, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_se_resnext_nccl.py b/python/paddle/fluid/tests/unittests/test_dist_se_resnext_nccl.py index aeacf2c00a6..35ffbbb80ca 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_se_resnext_nccl.py +++ b/python/paddle/fluid/tests/unittests/test_dist_se_resnext_nccl.py @@ -24,7 +24,6 @@ flag_name = os.path.splitext(__file__)[0] class TestDistSeResneXtNCCL(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reader_alloc = False @@ -32,15 +31,17 @@ class TestDistSeResneXtNCCL(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_se_resnext.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_se_resnext.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestDistSeResneXtNCCLMP(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reader_alloc = False @@ -49,12 +50,15 @@ class TestDistSeResneXtNCCLMP(TestDistBase): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_se_resnext.py", - delta=1e-5, - check_error_log=True, - need_envs={"NCCL_P2P_DISABLE": "1"}, - log_name=flag_name) + self.check_with_place( + "dist_se_resnext.py", + delta=1e-5, + check_error_log=True, + need_envs={"NCCL_P2P_DISABLE": "1"}, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_se_resnext_sync.py b/python/paddle/fluid/tests/unittests/test_dist_se_resnext_sync.py index 2d288df2ae4..cc5905506e6 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_se_resnext_sync.py +++ b/python/paddle/fluid/tests/unittests/test_dist_se_resnext_sync.py @@ -22,17 +22,18 @@ flag_name = os.path.splitext(__file__)[0] class TestDistSeResneXt2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reader_alloc = False @unittest.skip(reason="Skip unstable ci") def test_dist_train(self): - self.check_with_place("dist_se_resnext.py", - delta=1e-7, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_se_resnext.py", + delta=1e-7, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_sharding_save.py b/python/paddle/fluid/tests/unittests/test_dist_sharding_save.py index 43c849d8a8c..29442a9fb68 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_sharding_save.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sharding_save.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestDistMnistFleetSave(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -40,11 +39,19 @@ class TestDistMnistFleetSave(TestDistBase): sharding_save_files = sorted(os.listdir(dirname)) check_files = [ - 'fc_0.b_0', 'fc_0.b_0_velocity_0', 'fc_0.w_0', - 'fc_0.w_0_velocity_0', 'fc_1.b_0', 'fc_1.b_0_velocity_0', - 'fc_1.w_0', 'fc_1.w_0_velocity_0', 'fc_2.b_0', - 'fc_2.b_0_velocity_0', 'fc_2.w_0', 'fc_2.w_0_velocity_0', - 'learning_rate_0' + 'fc_0.b_0', + 'fc_0.b_0_velocity_0', + 'fc_0.w_0', + 'fc_0.w_0_velocity_0', + 'fc_1.b_0', + 'fc_1.b_0_velocity_0', + 'fc_1.w_0', + 'fc_1.w_0_velocity_0', + 'fc_2.b_0', + 'fc_2.b_0_velocity_0', + 'fc_2.w_0', + 'fc_2.w_0_velocity_0', + 'learning_rate_0', ] if sharding_save_files != check_files: @@ -54,12 +61,14 @@ class TestDistMnistFleetSave(TestDistBase): return True - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=True, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=True, + need_envs={}, + log_name="", + ): required_envs = self._get_required_envs(check_error_log, need_envs) tr0_losses, tr1_losses = self._run_cluster_nccl2( @@ -67,13 +76,15 @@ class TestDistMnistFleetSave(TestDistBase): required_envs, update_method='nccl2', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) dirname = './ut_sharding_save_model' self._test_saved_files(dirname) def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_sharding_save.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps0.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps0.py index b4aa24bc9e3..bff5754df1f 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps0.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps0.py @@ -24,8 +24,7 @@ from paddle.distributed.fleet import fleet class SparseLoadOp(unittest.TestCase): - """ Test load operator. - """ + """Test load operator.""" def net(self, emb_array, fc_array): with fluid.unique_name.guard(): @@ -38,7 +37,9 @@ class SparseLoadOp(unittest.TestCase): param_attr=fluid.ParamAttr( name="embedding", initializer=fluid.initializer.NumpyArrayInitializer( - emb_array)), + emb_array + ), + ), ) fc1 = fluid.layers.fc( @@ -48,7 +49,10 @@ class SparseLoadOp(unittest.TestCase): param_attr=fluid.ParamAttr( name='fc', initializer=fluid.initializer.NumpyArrayInitializer( - fc_array))) + fc_array + ), + ), + ) loss = fluid.layers.reduce_mean(fc1) return loss @@ -70,7 +74,6 @@ class SparseLoadOp(unittest.TestCase): @unittest.skip(reason="Skip unstable ut, need rewrite with new implement") class TestSparseLoadOpCase1(SparseLoadOp): - def test_2ps_0_load(self): # init No.0 server env env = {} @@ -110,7 +113,8 @@ class TestSparseLoadOpCase1(SparseLoadOp): fc_w = np.array(fluid.global_scope().find_var("fc").get_tensor()) emb = np.array( - fluid.global_scope().find_var("embedding.block0").get_tensor()) + fluid.global_scope().find_var("embedding.block0").get_tensor() + ) assert fc_w.all() == fc_array.all() assert emb.all() == emb_array[::2].all() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps1.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps1.py index bbbddd5d7c1..1152ff3e82b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps1.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps1.py @@ -25,7 +25,6 @@ from test_dist_sparse_load_ps0 import SparseLoadOp @unittest.skip(reason="Skip unstable ut, need rewrite with new implement") class TestSparseLoadOpCase2(SparseLoadOp): - def test_2ps_0_load(self): # init No.1 server env env = {} @@ -64,7 +63,8 @@ class TestSparseLoadOpCase2(SparseLoadOp): optimizer.minimize(loss) fleet.init_server(model_path) emb = np.array( - fluid.global_scope().find_var("embedding.block1").get_tensor()) + fluid.global_scope().find_var("embedding.block1").get_tensor() + ) assert emb.all() == emb_array[1::2].all() shutil.rmtree(model_path) diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adagrad.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adagrad.py index 32e5a0ac1f3..5fca7c3afa7 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adagrad.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adagrad.py @@ -29,8 +29,9 @@ class TestSparseLoadProgramAdagrad(TestSparseLoadProgram): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.Adam(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adam.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adam.py index 292e6e64fb5..8305ade7b3e 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adam.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adam.py @@ -29,8 +29,9 @@ class TestSparseLoadProgramAdam(TestSparseLoadProgram): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.Adam(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_ftrl.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_ftrl.py index 5bee7b73ef5..8193e20037a 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_ftrl.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_ftrl.py @@ -29,8 +29,9 @@ class TestSparseLoadProgramFtrl(TestSparseLoadProgram): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.SGD(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_momentum.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_momentum.py index e3b01ee0b50..83fc785bddd 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_momentum.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_momentum.py @@ -29,8 +29,9 @@ class TestSparseLoadProgramMomentum(TestSparseLoadProgram): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.SGD(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_rmsprop.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_rmsprop.py index b3d9cde8b0a..049a9043cf1 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_rmsprop.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_rmsprop.py @@ -29,8 +29,9 @@ class TestSparseLoadProgramRmsprop(TestSparseLoadProgram): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.SGD(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_sgd.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_sgd.py index b3b20df47da..42737d3d69b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_sgd.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_sgd.py @@ -27,7 +27,8 @@ class TestSparseLoadProgram(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = "PSERVER" os.environ["PADDLE_PORT"] = "4001" @@ -45,9 +46,9 @@ class TestSparseLoadProgram(unittest.TestCase): with fluid.program_guard(train_program, startup_program): with fluid.unique_name.guard(): inputs = fluid.data('input', shape=[None, 1], dtype="int64") - emb = fluid.layers.embedding(inputs, - is_sparse=True, - size=[10000, 128]) + emb = fluid.layers.embedding( + inputs, is_sparse=True, size=[10000, 128] + ) fc1 = fluid.layers.fc(input=emb, size=128, act="relu") fc2 = fluid.layers.fc(input=fc1, size=64, act="relu") loss = fluid.layers.reduce_mean(fc2) @@ -55,14 +56,14 @@ class TestSparseLoadProgram(unittest.TestCase): class TestSparseLoadProgramSGD(TestSparseLoadProgram): - def test_server_init(self): scope, train_program, startup_program, loss = self.net() with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.SGD(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_text_classification.py b/python/paddle/fluid/tests/unittests/test_dist_text_classification.py index 7b16aa3f200..36215f4e26c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_text_classification.py +++ b/python/paddle/fluid/tests/unittests/test_dist_text_classification.py @@ -22,29 +22,31 @@ flag_name = os.path.splitext(__file__)[0] class TestDistTextClassification2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._enforce_place = "CPU" def test_text_classification(self): - self.check_with_place("dist_text_classification.py", - delta=1e-6, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_text_classification.py", + delta=1e-6, + check_error_log=True, + log_name=flag_name, + ) class TestDistTextClassification2x2Async(TestDistBase): - def _setup_config(self): self._sync_mode = False self._enforce_place = "CPU" def test_se_resnext(self): - self.check_with_place("dist_text_classification.py", - delta=100, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_text_classification.py", + delta=100, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_train.py b/python/paddle/fluid/tests/unittests/test_dist_train.py index 72998542419..65753803282 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_train.py +++ b/python/paddle/fluid/tests/unittests/test_dist_train.py @@ -30,19 +30,19 @@ from dist_test_utils import remove_ps_flag from paddle.fluid import core -RPC_OP_ROLE_ATTR_NAME = op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( -) +RPC_OP_ROLE_ATTR_NAME = ( + op_role_attr_name +) = core.op_proto_and_checker_maker.kOpRoleAttrName() RPC_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.RPC class TestSendOp(unittest.TestCase): - def test_send(self): remove_ps_flag(os.getpid()) # Run init_serv in a thread place = fluid.CPUPlace() # NOTE: python thread will not work here due to GIL. - p = Process(target=self.init_serv, args=(place, )) + p = Process(target=self.init_serv, args=(place,)) p.daemon = True p.start() @@ -79,14 +79,18 @@ class TestSendOp(unittest.TestCase): with fluid.program_guard(main): serv = ListenAndServ("127.0.0.1:0", ["X"], optimizer_mode=False) with serv.do(): - out_var = main.global_block().create_var(name="scale_0.tmp_0", - psersistable=True, - dtype="float32", - shape=[32, 32]) - x = layers.data(shape=[32, 32], - dtype='float32', - name="X", - append_batch_size=False) + out_var = main.global_block().create_var( + name="scale_0.tmp_0", + psersistable=True, + dtype="float32", + shape=[32, 32], + ) + x = layers.data( + shape=[32, 32], + dtype='float32', + name="X", + append_batch_size=False, + ) fluid.initializer.Constant(value=1.0)(x, main.global_block()) ops._scale(x=x, scale=10.0, out=out_var) @@ -96,20 +100,22 @@ class TestSendOp(unittest.TestCase): def init_client(self, place, port): main = fluid.Program() with fluid.program_guard(main): - main.global_block().append_op(type="fetch_barrier", - inputs={}, - outputs={"Out": []}, - attrs={ - "endpoints": - ["127.0.0.1:{0}".format(port)], - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) - - x = layers.data(shape=[32, 32], - dtype='float32', - name='X', - append_batch_size=False) + main.global_block().append_op( + type="fetch_barrier", + inputs={}, + outputs={"Out": []}, + attrs={ + "endpoints": ["127.0.0.1:{0}".format(port)], + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) + + x = layers.data( + shape=[32, 32], + dtype='float32', + name='X', + append_batch_size=False, + ) x.persistable = True fluid.initializer.Constant(value=2.3)(x, main.global_block()) @@ -117,7 +123,8 @@ class TestSendOp(unittest.TestCase): name="scale_0.tmp_0", # server side var dtype="float32", persistable=False, - shape=[32, 32]) + shape=[32, 32], + ) fluid.initializer.Constant(value=2.3)(get_var, main.global_block()) # NOTE(zjl): `Send` is async send, which means that the sent @@ -136,10 +143,12 @@ class TestSendOp(unittest.TestCase): def run_local(self, place): main = fluid.Program() with fluid.program_guard(main): - x = layers.data(shape=[32, 32], - dtype='float32', - name='X', - append_batch_size=False) + x = layers.data( + shape=[32, 32], + dtype='float32', + name='X', + append_batch_size=False, + ) fluid.initializer.Constant(value=2.3)(x, main.global_block()) o = layers.scale(x=x, scale=10.0) exe = fluid.Executor(place) diff --git a/python/paddle/fluid/tests/unittests/test_dist_transformer.py b/python/paddle/fluid/tests/unittests/test_dist_transformer.py index a4d5a5a2cbf..470468f7fe3 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transformer.py @@ -22,59 +22,65 @@ def download_files(): url_prefix = 'http://paddle-unittest-data.bj.bcebos.com/dist_transformer/' vocab_url = url_prefix + 'vocab.bpe.32000' vocab_md5 = 'a86d345ca6e27f6591d0dccb1b9be853' - paddle.dataset.common.download(vocab_url, 'test_dist_transformer', - vocab_md5) + paddle.dataset.common.download( + vocab_url, 'test_dist_transformer', vocab_md5 + ) local_train_url = url_prefix + 'train.tok.clean.bpe.32000.en-de' local_train_md5 = '033eb02b9449e6dd823f050782ac8914' - paddle.dataset.common.download(local_train_url, 'test_dist_transformer', - local_train_md5) + paddle.dataset.common.download( + local_train_url, 'test_dist_transformer', local_train_md5 + ) train0_url = url_prefix + 'train.tok.clean.bpe.32000.en-de.train_0' train0_md5 = 'ddce7f602f352a0405267285379a38b1' - paddle.dataset.common.download(train0_url, 'test_dist_transformer', - train0_md5) + paddle.dataset.common.download( + train0_url, 'test_dist_transformer', train0_md5 + ) train1_url = url_prefix + 'train.tok.clean.bpe.32000.en-de.train_1' train1_md5 = '8757798200180285b1a619cd7f408747' - paddle.dataset.common.download(train1_url, 'test_dist_transformer', - train1_md5) + paddle.dataset.common.download( + train1_url, 'test_dist_transformer', train1_md5 + ) test_url = url_prefix + 'newstest2013.tok.bpe.32000.en-de' test_md5 = '9dd74a266dbdb25314183899f269b4a2' paddle.dataset.common.download(test_url, 'test_dist_transformer', test_md5) # cut test data for faster CI - orig_path = os.path.join(paddle.dataset.common.DATA_HOME, - "test_dist_transformer", - "newstest2013.tok.bpe.32000.en-de") - head_path = os.path.join(paddle.dataset.common.DATA_HOME, - "test_dist_transformer", - "newstest2013.tok.bpe.32000.en-de.cut") + orig_path = os.path.join( + paddle.dataset.common.DATA_HOME, + "test_dist_transformer", + "newstest2013.tok.bpe.32000.en-de", + ) + head_path = os.path.join( + paddle.dataset.common.DATA_HOME, + "test_dist_transformer", + "newstest2013.tok.bpe.32000.en-de.cut", + ) os.system("head -n10 %s > %s" % (orig_path, head_path)) class TestDistTransformer2x2Sync(TestDistBase): - def _setup_config(self): self._sync_mode = True def test_dist_train(self): download_files() - self.check_with_place("dist_transformer.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_transformer.py", delta=1e-5, check_error_log=False + ) class TestDistTransformer2x2Async(TestDistBase): - def _setup_config(self): self._sync_mode = False def test_dist_train(self): download_files() - self.check_with_place("dist_transformer.py", - delta=1.0, - check_error_log=False) + self.check_with_place( + "dist_transformer.py", delta=1.0, check_error_log=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py index 8fc636af2de..b5419eca108 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py @@ -27,7 +27,6 @@ import paddle.fluid as fluid class TranspilerTest(unittest.TestCase): - def setUp(self): self.trainer_id = 0 self.trainers = 2 @@ -41,11 +40,13 @@ class TranspilerTest(unittest.TestCase): def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -68,8 +69,8 @@ class TranspilerTest(unittest.TestCase): trainer_main = t.get_trainer_program(wait_port=False) trainer_startup = fluid.default_startup_program() - assert (src.num_blocks == 1) - assert (trainer_startup.num_blocks == src.num_blocks) + assert src.num_blocks == 1 + assert trainer_startup.num_blocks == src.num_blocks return trainer_main, trainer_startup @@ -83,11 +84,13 @@ class TranspilerTest(unittest.TestCase): if not self.transpiler: main = self.get_main_program() self.transpiler = fluid.DistributeTranspiler(config=config) - self.transpiler.transpile(self.trainer_id, - program=main, - pservers=self.pserver_eps, - trainers=self.trainers, - sync_mode=sync_mode) + self.transpiler.transpile( + self.trainer_id, + program=main, + pservers=self.pserver_eps, + trainers=self.trainers, + sync_mode=sync_mode, + ) return self.transpiler @@ -109,7 +112,6 @@ class TranspilerTest(unittest.TestCase): class TestBasicModel(TranspilerTest): - def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) pserver2, startup2 = self.get_pserver(self.pserver2_ep) @@ -125,28 +127,57 @@ class TestBasicModel(TranspilerTest): self.assertTrue("fc_b@GRAD" not in trainer_startup.global_block().vars) src = [op.type for op in trainer_startup.global_block().ops] - dst = ['fill_constant', 'fill_constant', 'uniform_random', 'recv', 'recv', \ - 'fetch_barrier', 'concat'] + dst = [ + 'fill_constant', + 'fill_constant', + 'uniform_random', + 'recv', + 'recv', + 'fetch_barrier', + 'concat', + ] self.assertEqual(src, dst) - self.assertEqual([op.type for op in trainer.global_block().ops], [ - 'mul', 'elementwise_add', 'elementwise_sub', 'square', 'mean', - 'fill_constant', 'mean_grad', 'square_grad', 'elementwise_sub_grad', - 'elementwise_add_grad', 'send', 'mul_grad', 'split_byref', 'send', - 'send_barrier', 'recv', 'recv', 'fetch_barrier', 'concat' - ]) + self.assertEqual( + [op.type for op in trainer.global_block().ops], + [ + 'mul', + 'elementwise_add', + 'elementwise_sub', + 'square', + 'mean', + 'fill_constant', + 'mean_grad', + 'square_grad', + 'elementwise_sub_grad', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'split_byref', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', + 'concat', + ], + ) self.assertEqual(len(pserver.blocks), 3) # block0: listen_and_serv - self.assertEqual([op.type for op in pserver.blocks[0].ops], - ["listen_and_serv"]) + self.assertEqual( + [op.type for op in pserver.blocks[0].ops], ["listen_and_serv"] + ) # block1~2: optimize pass - self.assertEqual([op.type for op in pserver.blocks[1].ops], - ["sum", "scale", "sgd"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], ["sum", "scale", "sgd"] + ) # confirm startup program - self.assertEqual([op.type for op in startup.global_block().ops], - ["fill_constant", "fill_constant", "uniform_random"]) + self.assertEqual( + [op.type for op in startup.global_block().ops], + ["fill_constant", "fill_constant", "uniform_random"], + ) # the variable #fc_w will be split into two blocks fc_w_var = startup.global_block().var("fc_w.block1") self.assertEqual(fc_w_var.shape, (500, 1000)) @@ -172,7 +203,6 @@ class TestBasicModel(TranspilerTest): class TestBasicModelWithLargeBlockSize(TranspilerTest): - def transpiler_test_impl(self): config = fluid.DistributeTranspilerConfig() config.min_block_size = 1048576 @@ -182,23 +212,43 @@ class TestBasicModelWithLargeBlockSize(TranspilerTest): trainer, _ = self.get_trainer(config) - self.assertEqual([op.type for op in trainer.global_block().ops], [ - 'mul', 'elementwise_add', 'elementwise_sub', 'square', 'mean', - 'fill_constant', 'mean_grad', 'square_grad', 'elementwise_sub_grad', - 'elementwise_add_grad', 'send', 'mul_grad', 'send', 'send_barrier', - 'recv', 'recv', 'fetch_barrier' - ]) + self.assertEqual( + [op.type for op in trainer.global_block().ops], + [ + 'mul', + 'elementwise_add', + 'elementwise_sub', + 'square', + 'mean', + 'fill_constant', + 'mean_grad', + 'square_grad', + 'elementwise_sub_grad', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', + ], + ) self.assertEqual(len(pserver.blocks), 2) # block0: listen_and_serv - self.assertEqual([op.type for op in pserver.blocks[0].ops], - ["listen_and_serv"]) + self.assertEqual( + [op.type for op in pserver.blocks[0].ops], ["listen_and_serv"] + ) # block1~2: optimize pass - self.assertEqual([op.type for op in pserver.blocks[1].ops], - ["sum", "scale", "sgd"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], ["sum", "scale", "sgd"] + ) # confirm startup program - self.assertEqual([op.type for op in startup.global_block().ops], - ["fill_constant", "fill_constant"]) + self.assertEqual( + [op.type for op in startup.global_block().ops], + ["fill_constant", "fill_constant"], + ) # the variable #fc_w will be split into two blocks fc_w_var = startup2.global_block().var("fc_w") self.assertEqual(fc_w_var.shape, (1000, 1000)) @@ -224,7 +274,6 @@ class TestBasicModelWithLargeBlockSize(TranspilerTest): class TestNoSliceVar(TranspilerTest): - def setUp(self): super(TestNoSliceVar, self).setUp() @@ -244,22 +293,26 @@ class TestNoSliceVar(TranspilerTest): class TestLRDecay(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay(learning_rate=1.0, - decay_steps=2100, - decay_rate=0.1, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=1.0, + decay_steps=2100, + decay_rate=0.1, + staircase=True, + ) + ) sgd_optimizer.minimize(avg_cost) def transpiler_test_impl(self): @@ -268,30 +321,35 @@ class TestLRDecay(TranspilerTest): self.assertEqual(len(pserver.blocks), 4) lr_decay_ops = [op.type for op in pserver.blocks[1].ops] - self.assertEqual(lr_decay_ops, [ - "increment", "cast", "fill_constant", "elementwise_div", "floor", - "fill_constant", "elementwise_pow", "fill_constant", - "elementwise_mul" - ]) + self.assertEqual( + lr_decay_ops, + [ + "increment", + "cast", + "fill_constant", + "elementwise_div", + "floor", + "fill_constant", + "elementwise_pow", + "fill_constant", + "elementwise_mul", + ], + ) class TestFakeInit(TranspilerTest): - def net_conf(self): dict_size, embedding_size, neg_num = 10000, 8, 5 - input_word = fluid.layers.data(name="input_word", - shape=[1], - dtype='int64', - lod_level=1) - true_word = fluid.layers.data(name='true_label', - shape=[1], - dtype='int64', - lod_level=1) - neg_word = fluid.layers.data(name="neg_label", - shape=[1], - dtype='int64', - lod_level=1) + input_word = fluid.layers.data( + name="input_word", shape=[1], dtype='int64', lod_level=1 + ) + true_word = fluid.layers.data( + name='true_label', shape=[1], dtype='int64', lod_level=1 + ) + neg_word = fluid.layers.data( + name="neg_label", shape=[1], dtype='int64', lod_level=1 + ) inputs = [input_word, true_word, neg_word] init_width = 0.5 / embedding_size @@ -299,82 +357,99 @@ class TestFakeInit(TranspilerTest): input=inputs[0], is_sparse=True, size=[dict_size, embedding_size], - param_attr=fluid.ParamAttr(name='emb', - initializer=fluid.initializer.Uniform( - -init_width, init_width))) + param_attr=fluid.ParamAttr( + name='emb', + initializer=fluid.initializer.Uniform(-init_width, init_width), + ), + ) true_emb_w = fluid.layers.embedding( input=inputs[1], is_sparse=True, size=[dict_size, embedding_size], param_attr=fluid.ParamAttr( - name='emb_w', - initializer=fluid.initializer.Constant(value=0.0))) + name='emb_w', initializer=fluid.initializer.Constant(value=0.0) + ), + ) true_emb_b = fluid.layers.embedding( input=inputs[1], is_sparse=True, size=[dict_size, 1], param_attr=fluid.ParamAttr( - name='emb_b', - initializer=fluid.initializer.Constant(value=0.0))) + name='emb_b', initializer=fluid.initializer.Constant(value=0.0) + ), + ) neg_word_reshape = fluid.layers.reshape(inputs[2], shape=[-1, 1]) neg_word_reshape.stop_gradient = True - neg_emb_w = fluid.layers.embedding(input=neg_word_reshape, - is_sparse=True, - size=[dict_size, embedding_size], - param_attr=fluid.ParamAttr( - name='emb_w', learning_rate=1.0)) + neg_emb_w = fluid.layers.embedding( + input=neg_word_reshape, + is_sparse=True, + size=[dict_size, embedding_size], + param_attr=fluid.ParamAttr(name='emb_w', learning_rate=1.0), + ) - neg_emb_w_re = fluid.layers.reshape(neg_emb_w, - shape=[-1, neg_num, embedding_size]) + neg_emb_w_re = fluid.layers.reshape( + neg_emb_w, shape=[-1, neg_num, embedding_size] + ) - neg_emb_b = fluid.layers.embedding(input=neg_word_reshape, - is_sparse=True, - size=[dict_size, 1], - param_attr=fluid.ParamAttr( - name='emb_b', learning_rate=1.0)) + neg_emb_b = fluid.layers.embedding( + input=neg_word_reshape, + is_sparse=True, + size=[dict_size, 1], + param_attr=fluid.ParamAttr(name='emb_b', learning_rate=1.0), + ) neg_emb_b_vec = fluid.layers.reshape(neg_emb_b, shape=[-1, neg_num]) true_logits = fluid.layers.elementwise_add( - fluid.layers.reduce_sum(fluid.layers.elementwise_mul( - input_emb, true_emb_w), - dim=1, - keep_dim=True), true_emb_b) - - input_emb_re = fluid.layers.reshape(input_emb, - shape=[-1, 1, embedding_size]) - - neg_matmul = fluid.layers.matmul(input_emb_re, - neg_emb_w_re, - transpose_y=True) + fluid.layers.reduce_sum( + fluid.layers.elementwise_mul(input_emb, true_emb_w), + dim=1, + keep_dim=True, + ), + true_emb_b, + ) + + input_emb_re = fluid.layers.reshape( + input_emb, shape=[-1, 1, embedding_size] + ) + + neg_matmul = fluid.layers.matmul( + input_emb_re, neg_emb_w_re, transpose_y=True + ) neg_matmul_re = fluid.layers.reshape(neg_matmul, shape=[-1, neg_num]) neg_logits = fluid.layers.elementwise_add(neg_matmul_re, neg_emb_b_vec) # nce loss - label_ones = fluid.layers.fill_constant_batch_size_like(true_logits, - shape=[-1, 1], - value=1.0, - dtype='float32') + label_ones = fluid.layers.fill_constant_batch_size_like( + true_logits, shape=[-1, 1], value=1.0, dtype='float32' + ) label_zeros = fluid.layers.fill_constant_batch_size_like( - true_logits, shape=[-1, neg_num], value=0.0, dtype='float32') + true_logits, shape=[-1, neg_num], value=0.0, dtype='float32' + ) true_xent = fluid.layers.sigmoid_cross_entropy_with_logits( - true_logits, label_ones) + true_logits, label_ones + ) neg_xent = fluid.layers.sigmoid_cross_entropy_with_logits( - neg_logits, label_zeros) + neg_logits, label_zeros + ) cost = fluid.layers.elementwise_add( fluid.layers.reduce_sum(true_xent, dim=1), - fluid.layers.reduce_sum(neg_xent, dim=1)) + fluid.layers.reduce_sum(neg_xent, dim=1), + ) avg_cost = fluid.layers.reduce_mean(cost) sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay(learning_rate=1.0, - decay_steps=2100, - decay_rate=0.1, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=1.0, + decay_steps=2100, + decay_rate=0.1, + staircase=True, + ) + ) sgd_optimizer.minimize(avg_cost) def transpiler_test_impl(self): @@ -389,14 +464,15 @@ class TestFakeInit(TranspilerTest): class TestDecayedAdagrad(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -409,14 +485,15 @@ class TestDecayedAdagrad(TranspilerTest): class TestFtrl(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -429,20 +506,23 @@ class TestFtrl(TranspilerTest): class TestLRDecayConditional(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.piecewise_decay([10000, 20000], - [1.0, 0.5, 1.0])) + learning_rate=fluid.layers.piecewise_decay( + [10000, 20000], [1.0, 0.5, 1.0] + ) + ) sgd_optimizer.minimize(avg_cost) def transpiler_test_impl(self): @@ -460,13 +540,27 @@ class TestLRDecayConditional(TranspilerTest): self.assertEqual(len(pserver.blocks), 7) lr_decay_ops = [op.type for op in pserver.blocks[1].ops] - self.assertEqual(lr_decay_ops, [ - "increment", "cast", "fill_constant", "fill_constant", "less_than", - "logical_not", "conditional_block", "fill_constant", - "fill_constant", "less_than", "logical_not", "logical_and", - "logical_and", "conditional_block", "fill_constant", - "conditional_block" - ]) + self.assertEqual( + lr_decay_ops, + [ + "increment", + "cast", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "conditional_block", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "logical_and", + "logical_and", + "conditional_block", + "fill_constant", + "conditional_block", + ], + ) # test the condition blocks for b in sub_blocks: if b == 0: @@ -476,16 +570,17 @@ class TestLRDecayConditional(TranspilerTest): class TestL2Decay(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') y_predict = fluid.layers.fc( input=x, size=1000, act=None, - param_attr=fluid.ParamAttr(name='fc_w', - regularizer=fluid.regularizer.L2Decay()), - bias_attr=fluid.ParamAttr(name='fc_b')) + param_attr=fluid.ParamAttr( + name='fc_w', regularizer=fluid.regularizer.L2Decay() + ), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -502,22 +597,27 @@ class TestL2Decay(TranspilerTest): trainer, _ = self.get_trainer() self.assertEqual(len(pserver.blocks), 3) - self.assertEqual([op.type for op in pserver.blocks[1].ops], - ["sum", "scale", "clip", "sgd"]) - self.assertEqual([op.type for op in pserver.blocks[2].ops], - ["sum", "scale", "clip", "scale", "sum", "sgd"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], + ["sum", "scale", "clip", "sgd"], + ) + self.assertEqual( + [op.type for op in pserver.blocks[2].ops], + ["sum", "scale", "clip", "scale", "sum", "sgd"], + ) # TODO(typhoonzero): test clipping and L2Decay ops are removed from trainer class TestL2DecayWithPiecewise(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -525,10 +625,12 @@ class TestL2DecayWithPiecewise(TranspilerTest): bd = [1, 10, 20, 30] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] sgd_optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.piecewise_decay(boundaries=bd, - values=lr), + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr + ), momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) sgd_optimizer.minimize(avg_cost) def transpiler_test_impl(self): @@ -536,33 +638,62 @@ class TestL2DecayWithPiecewise(TranspilerTest): trainer, _ = self.get_trainer() self.assertEqual(len(pserver.blocks), 9) - self.assertEqual([op.type for op in pserver.blocks[1].ops], [ - "increment", "cast", "fill_constant", "fill_constant", "less_than", - "logical_not", "conditional_block", "fill_constant", - "fill_constant", "less_than", "logical_not", "logical_and", - "logical_and", "conditional_block", "fill_constant", - "fill_constant", "less_than", "logical_not", "logical_and", - "logical_and", "conditional_block", "fill_constant", - "fill_constant", "less_than", "logical_not", "logical_and", - "logical_and", "conditional_block", "fill_constant", - "conditional_block" - ]) - self.assertEqual([op.type for op in pserver.blocks[7].ops], - ["sum", "scale", "scale", "sum", "momentum"]) - self.assertEqual([op.type for op in pserver.blocks[8].ops], - ["sum", "scale", "scale", "sum", "momentum"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], + [ + "increment", + "cast", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "conditional_block", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "logical_and", + "logical_and", + "conditional_block", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "logical_and", + "logical_and", + "conditional_block", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "logical_and", + "logical_and", + "conditional_block", + "fill_constant", + "conditional_block", + ], + ) + self.assertEqual( + [op.type for op in pserver.blocks[7].ops], + ["sum", "scale", "scale", "sum", "momentum"], + ) + self.assertEqual( + [op.type for op in pserver.blocks[8].ops], + ["sum", "scale", "scale", "sum", "momentum"], + ) class TestEmptyPserverOptimizeBlocks(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') # only one parameter - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=False) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=False, + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -580,44 +711,45 @@ class TestEmptyPserverOptimizeBlocks(TranspilerTest): class TestDistLookupTableBase(TranspilerTest): - def network_with_table(self, is_sparse, is_distributed): self.table_size = 1000 self.emb_size = 64 self.lookup_table_name = 'shared_w' def emb_pool(ids, table_name, is_distributed): - emb = fluid.layers.embedding(input=ids, - size=[self.table_size, self.emb_size], - dtype='float32', - param_attr=table_name, - is_sparse=is_sparse, - is_distributed=is_distributed) + emb = fluid.layers.embedding( + input=ids, + size=[self.table_size, self.emb_size], + dtype='float32', + param_attr=table_name, + is_sparse=is_sparse, + is_distributed=is_distributed, + ) pool = fluid.layers.sequence_pool(input=emb, pool_type='average') return pool - title_ids = fluid.layers.data(name='title_ids', - shape=[1], - dtype='int64', - lod_level=1) - brand_ids = fluid.layers.data(name='brand_ids', - shape=[1], - dtype='int64', - lod_level=1) - profile_ids = fluid.layers.data(name='brand_ids', - shape=[1], - dtype='int64', - lod_level=1) + title_ids = fluid.layers.data( + name='title_ids', shape=[1], dtype='int64', lod_level=1 + ) + brand_ids = fluid.layers.data( + name='brand_ids', shape=[1], dtype='int64', lod_level=1 + ) + profile_ids = fluid.layers.data( + name='brand_ids', shape=[1], dtype='int64', lod_level=1 + ) title_emb = emb_pool(title_ids, self.lookup_table_name, is_distributed) brand_emb = emb_pool(brand_ids, self.lookup_table_name, is_distributed) profile_emb = emb_pool(profile_ids, "profile_emb", False) - fc0 = fluid.layers.concat(input=[title_emb, brand_emb, profile_emb], - axis=1) - predict = fluid.layers.fc(input=fc0, - size=2, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + fc0 = fluid.layers.concat( + input=[title_emb, brand_emb, profile_emb], axis=1 + ) + predict = fluid.layers.fc( + input=fc0, + size=2, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = fluid.layers.cross_entropy(input=predict, label=label) @@ -627,7 +759,6 @@ class TestDistLookupTableBase(TranspilerTest): class TestLocalLookupTable(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=False) @@ -637,36 +768,66 @@ class TestLocalLookupTable(TestDistLookupTableBase): self.assertEqual(len(pserver1.blocks), 4) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 2 optimize for table adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 3 optimize for table 2 adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[3].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[3].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) trainer, _ = self.get_trainer() self.assertEqual(len(trainer.blocks), 1) ops = [ - 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', - 'lookup_table', 'sequence_pool', 'concat', 'mul', 'elementwise_add', - 'cross_entropy2', 'mean', 'fill_constant', 'mean_grad', - 'cross_entropy_grad2', 'elementwise_add_grad', 'send', 'mul_grad', - 'send', 'concat_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'split_selected_rows', 'send', 'sequence_pool_grad', - 'lookup_table_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'sum', 'split_selected_rows', 'send', 'send_barrier', 'recv', - 'recv', 'fetch_barrier' + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_selected_rows', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) class TestDistLookupTable(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) @@ -676,48 +837,97 @@ class TestDistLookupTable(TestDistLookupTableBase): self.assertEqual(len(pserver1.blocks), 6) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 4 prefetch -> lookup_sparse_table_read for data0 - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 2 optimize for table sgd - self.assertEqual([op.type for op in pserver1.blocks[3].ops], - ["sum", "sgd"]) + self.assertEqual( + [op.type for op in pserver1.blocks[3].ops], ["sum", "sgd"] + ) # 3 prefetch -> lookup_sparse_table_read for data0 - self.assertEqual([op.type for op in pserver1.blocks[4].ops], - ["lookup_sparse_table_read"]) + self.assertEqual( + [op.type for op in pserver1.blocks[4].ops], + ["lookup_sparse_table_read"], + ) # 5 save table self.assertEqual([op.type for op in pserver1.blocks[5].ops], ["save"]) trainer, trainer_startup = self.get_trainer() self.assertEqual(len(trainer.blocks), 1) ops = [ - 'split_ids', 'prefetch', 'merge_ids', 'sequence_pool', - 'sequence_pool', 'lookup_table', 'sequence_pool', 'concat', 'mul', - 'elementwise_add', 'cross_entropy2', 'mean', 'fill_constant', - 'mean_grad', 'cross_entropy_grad2', 'elementwise_add_grad', 'send', - 'mul_grad', 'send', 'concat_grad', 'sequence_pool_grad', - 'lookup_table_grad', 'split_selected_rows', 'send', - 'sequence_pool_grad', 'lookup_table_grad', 'sequence_pool_grad', - 'lookup_table_grad', 'sum', 'split_ids', 'send', 'send_barrier', - 'recv', 'recv', 'fetch_barrier' + 'split_ids', + 'prefetch', + 'merge_ids', + 'sequence_pool', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_ids', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) startup_ops = [ - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'uniform_random', - 'uniform_random', 'recv', 'recv', 'recv', 'fetch_barrier', 'concat', - 'fake_init' + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'uniform_random', + 'uniform_random', + 'recv', + 'recv', + 'recv', + 'fetch_barrier', + 'concat', + 'fake_init', ] - self.assertEqual([op.type for op in trainer_startup.blocks[0].ops], - startup_ops) + self.assertEqual( + [op.type for op in trainer_startup.blocks[0].ops], startup_ops + ) class TestAsyncLocalLookupTable(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=False) @@ -728,34 +938,63 @@ class TestAsyncLocalLookupTable(TestDistLookupTableBase): self.assertEqual(len(pserver1.blocks), 4) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["adam", "scale", "scale"], + ) # 2 optimize for table adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["adam", "scale", "scale"], + ) # 3 optimize for table adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[3].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[3].ops], + ["adam", "scale", "scale"], + ) trainer, _ = self.get_trainer(config) self.assertEqual(len(trainer.blocks), 1) ops = [ - 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', - 'lookup_table', 'sequence_pool', 'concat', 'mul', 'elementwise_add', - 'cross_entropy2', 'mean', 'fill_constant', 'mean_grad', - 'cross_entropy_grad2', 'elementwise_add_grad', 'send', 'mul_grad', - 'send', 'concat_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'split_selected_rows', 'send', 'sequence_pool_grad', - 'lookup_table_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'sum', 'split_selected_rows', 'send', 'recv', 'recv' + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_selected_rows', + 'send', + 'recv', + 'recv', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) class TestAsyncDistLookupTable(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) @@ -767,46 +1006,93 @@ class TestAsyncDistLookupTable(TestDistLookupTableBase): self.assertEqual(len(pserver1.blocks), 6) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["adam", "scale", "scale"], + ) # 2 optimize for table adam - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["adam", "scale", "scale"], + ) # 3 optimize for table sgd self.assertEqual([op.type for op in pserver1.blocks[3].ops], ["sgd"]) # 4 prefetch -> lookup_sparse_table_read for data0 - self.assertEqual([op.type for op in pserver1.blocks[4].ops], - ["lookup_sparse_table_read"]) + self.assertEqual( + [op.type for op in pserver1.blocks[4].ops], + ["lookup_sparse_table_read"], + ) # 5 save table self.assertEqual([op.type for op in pserver1.blocks[5].ops], ["save"]) trainer, trainer_startup = self.get_trainer(config) self.assertEqual(len(trainer.blocks), 1) ops = [ - 'split_ids', 'prefetch', 'merge_ids', 'sequence_pool', - 'sequence_pool', 'lookup_table', 'sequence_pool', 'concat', 'mul', - 'elementwise_add', 'cross_entropy2', 'mean', 'fill_constant', - 'mean_grad', 'cross_entropy_grad2', 'elementwise_add_grad', 'send', - 'mul_grad', 'send', 'concat_grad', 'sequence_pool_grad', - 'lookup_table_grad', 'split_selected_rows', 'send', - 'sequence_pool_grad', 'lookup_table_grad', 'sequence_pool_grad', - 'lookup_table_grad', 'sum', 'split_ids', 'send', 'recv', 'recv' + 'split_ids', + 'prefetch', + 'merge_ids', + 'sequence_pool', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_ids', + 'send', + 'recv', + 'recv', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) startup_ops = [ - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'uniform_random', - 'uniform_random', 'recv', 'recv', 'recv', 'fetch_barrier', 'concat', - 'fake_init' + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'uniform_random', + 'uniform_random', + 'recv', + 'recv', + 'recv', + 'fetch_barrier', + 'concat', + 'fake_init', ] - self.assertEqual([op.type for op in trainer_startup.blocks[0].ops], - startup_ops) + self.assertEqual( + [op.type for op in trainer_startup.blocks[0].ops], startup_ops + ) class TestDistLookupTableSliceSize(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) @@ -816,14 +1102,14 @@ class TestDistLookupTableSliceSize(TestDistLookupTableBase): self.assertTrue(self.transpiler.has_distributed_lookup_table) lookup_table_var = pserver1.global_block().vars[ - self.transpiler.table_name] + self.transpiler.table_name + ] row_size = lookup_table_var.shape[0] calc_row_size = int(math.ceil(self.table_size / self.pservers)) self.assertEqual(row_size, calc_row_size) class TestDistArgsInProgram(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) @@ -832,21 +1118,24 @@ class TestDistArgsInProgram(TestDistLookupTableBase): self.assertTrue(trainer._is_distributed) self.assertTrue(trainer._is_chief) - self.assertEqual(trainer._distributed_lookup_table, - self.lookup_table_name) - self.assertEqual(trainer._endpoints, - [self.pserver1_ep, self.pserver2_ep]) + self.assertEqual( + trainer._distributed_lookup_table, self.lookup_table_name + ) + self.assertEqual( + trainer._endpoints, [self.pserver1_ep, self.pserver2_ep] + ) class TestRMSPropOptimizer(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -859,8 +1148,10 @@ class TestRMSPropOptimizer(TranspilerTest): self.assertEqual(len(pserver.blocks), 3) # block1~2: optimize pass - self.assertEqual([op.type for op in pserver.blocks[1].ops], - ["sum", "scale", "rmsprop"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], + ["sum", "scale", "rmsprop"], + ) # the variable #fc_w will be split into two blocks fc_w_var = startup.global_block().var("fc_w.block1") self.assertEqual(fc_w_var.shape, (500, 1000)) @@ -869,14 +1160,15 @@ class TestRMSPropOptimizer(TranspilerTest): class TestLoadSliceVar(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -888,9 +1180,11 @@ class TestLoadSliceVar(TranspilerTest): pserver2, _ = self.get_pserver(self.pserver2_ep) vars_ps1 = pserver._parameters_on_pservers.get_distributed_vars_by_ep( - self.pserver1_ep) + self.pserver1_ep + ) vars_ps2 = pserver._parameters_on_pservers.get_distributed_vars_by_ep( - self.pserver2_ep) + self.pserver2_ep + ) self.assertTrue(vars_ps1) self.assertTrue(vars_ps2) @@ -902,10 +1196,12 @@ class TestLoadSliceVar(TranspilerTest): ps1_var = vars_ps1[idx] if not ps1_var.is_slice: - total_numel = functools.reduce(lambda x, y: x * y, - vars_ps1[idx].origin.shape) - ps1_numel = functools.reduce(lambda x, y: x * y, - vars_ps1[idx].slice.shape) + total_numel = functools.reduce( + lambda x, y: x * y, vars_ps1[idx].origin.shape + ) + ps1_numel = functools.reduce( + lambda x, y: x * y, vars_ps1[idx].slice.shape + ) else: ps2_var = None for var in vars_ps2: @@ -913,18 +1209,20 @@ class TestLoadSliceVar(TranspilerTest): ps2_var = var break - total_numel = functools.reduce(lambda x, y: x * y, - ps1_var.origin.shape) - ps1_numel = functools.reduce(lambda x, y: x * y, - ps1_var.slice.shape) - ps2_numel = functools.reduce(lambda x, y: x * y, - ps2_var.slice.shape) + total_numel = functools.reduce( + lambda x, y: x * y, ps1_var.origin.shape + ) + ps1_numel = functools.reduce( + lambda x, y: x * y, ps1_var.slice.shape + ) + ps2_numel = functools.reduce( + lambda x, y: x * y, ps2_var.slice.shape + ) self.assertEqual(total_numel, ps1_numel + ps2_numel) class TestNCCL2Transpile(TranspilerTest): - def test_nccl2_transpile(self): if fluid.core.is_compiled_with_cuda(): # test nccl2 only with cuda main = fluid.Program() @@ -936,10 +1234,12 @@ class TestNCCL2Transpile(TranspilerTest): config.mode = "nccl2" config.wait_port = False t = fluid.DistributeTranspiler(config=config) - t.transpile(0, - trainers="127.0.0.1:6174,127.0.0.1:6175", - current_endpoint="127.0.0.1:6174", - startup_program=startup) + t.transpile( + 0, + trainers="127.0.0.1:6174,127.0.0.1:6175", + current_endpoint="127.0.0.1:6174", + startup_program=startup, + ) print([op.type for op in startup.global_block().ops]) self.assertEqual(startup.global_block().ops[-1].type, "gen_nccl_id") self.assertIsNotNone(startup.global_block().vars.get("NCCLID")) @@ -950,9 +1250,9 @@ class TestNCCL2Transpile(TranspilerTest): # test for remote prefetch class TestRemoteLookupTable(TestDistLookupTableBase): - def net_conf(self): import os + os.environ['PADDLE_ENABLE_REMOTE_PREFETCH'] = "1" self.network_with_table(is_sparse=True, is_distributed=False) @@ -962,37 +1262,67 @@ class TestRemoteLookupTable(TestDistLookupTableBase): self.assertEqual(len(pserver1.blocks), 4) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 2 optimize for table adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 3 optimize for table 2 adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[3].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[3].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) trainer, _ = self.get_trainer() self.assertEqual(len(trainer.blocks), 1) ops = [ - 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', - 'lookup_table', 'sequence_pool', 'concat', 'mul', 'elementwise_add', - 'cross_entropy2', 'mean', 'fill_constant', 'mean_grad', - 'cross_entropy_grad2', 'elementwise_add_grad', 'send', 'mul_grad', - 'send', 'concat_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'split_selected_rows', 'send', 'sequence_pool_grad', - 'lookup_table_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'sum', 'split_selected_rows', 'send', 'send_barrier', 'recv', - 'recv', 'fetch_barrier' + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_selected_rows', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) # test for remote prefetch class TestRemoteNce(TestDistLookupTableBase): - def network_with_table(self, is_sparse, is_distributed): num_total_classes = 20 @@ -1002,28 +1332,40 @@ class TestRemoteNce(TestDistLookupTableBase): input = fluid.layers.data(name="input", shape=[10], dtype="float32") label = fluid.layers.data(name="label", shape=[1], dtype="int64") - w_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 10], - dtype='float32', - name='nce_w', - initializer=fluid.initializer.ConstantInitializer()) - b_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 1], - dtype='float32', - name='nce_b', - initializer=fluid.initializer.ConstantInitializer()) - - cost = fluid.layers.nce(input=input, - label=label, - num_total_classes=num_total_classes, - sampler=sampler, - custom_dist=nid_freq_arr.tolist(), - sample_weight=None, - param_attr='nce_w', - bias_attr='nce_b', - seed=1, - num_neg_samples=5, - is_sparse=is_sparse) + w_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 10], + dtype='float32', + name='nce_w', + initializer=fluid.initializer.ConstantInitializer(), + ) + ) + b_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 1], + dtype='float32', + name='nce_b', + initializer=fluid.initializer.ConstantInitializer(), + ) + ) + + cost = fluid.layers.nce( + input=input, + label=label, + num_total_classes=num_total_classes, + sampler=sampler, + custom_dist=nid_freq_arr.tolist(), + sample_weight=None, + param_attr='nce_w', + bias_attr='nce_b', + seed=1, + num_neg_samples=5, + is_sparse=is_sparse, + ) avg_cost = paddle.mean(cost) # optimizer optimizer = fluid.optimizer.Adam(learning_rate=0.003) @@ -1031,6 +1373,7 @@ class TestRemoteNce(TestDistLookupTableBase): def net_conf(self): import os + os.environ['PADDLE_ENABLE_REMOTE_PREFETCH'] = "1" self.network_with_table(is_sparse=True, is_distributed=False) @@ -1055,44 +1398,59 @@ class TestRemoteNce(TestDistLookupTableBase): # test for remote prefetch class TestRemoteHsigmoid(TestDistLookupTableBase): - def network_with_table(self, is_sparse, is_distributed): num_total_classes = 3 input = fluid.layers.data(name="input", shape=[1], dtype="float32") label = fluid.layers.data(name="label", shape=[1], dtype="int64") - path_table = fluid.layers.data(name='path_table', - shape=[3], - dtype='int64') - path_code = fluid.layers.data(name='path_code', - shape=[3], - dtype='int64') - w_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 10], - dtype='float32', - name='hs_w', - initializer=fluid.initializer.ConstantInitializer()) - b_param = fluid.default_main_program().global_block().create_parameter( - shape=[3, 1], - dtype='float32', - name='hs_b', - initializer=fluid.initializer.ConstantInitializer()) + path_table = fluid.layers.data( + name='path_table', shape=[3], dtype='int64' + ) + path_code = fluid.layers.data( + name='path_code', shape=[3], dtype='int64' + ) + w_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 10], + dtype='float32', + name='hs_w', + initializer=fluid.initializer.ConstantInitializer(), + ) + ) + b_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[3, 1], + dtype='float32', + name='hs_b', + initializer=fluid.initializer.ConstantInitializer(), + ) + ) emb = fluid.layers.embedding( input=input, is_sparse=is_sparse, size=[3, 3], - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(num_total_classes)))) - - cost = fluid.layers.hsigmoid(input=emb, - label=label, - num_classes=num_total_classes, - path_table=path_table, - path_code=path_code, - is_custom=True, - is_sparse=is_sparse) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(num_total_classes) + ) + ), + ) + + cost = fluid.layers.hsigmoid( + input=emb, + label=label, + num_classes=num_total_classes, + path_table=path_table, + path_code=path_code, + is_custom=True, + is_sparse=is_sparse, + ) avg_cost = paddle.mean(cost) # optimizer optimizer = fluid.optimizer.SGD(learning_rate=0.003) @@ -1100,6 +1458,7 @@ class TestRemoteHsigmoid(TestDistLookupTableBase): def net_conf(self): import os + os.environ['PADDLE_ENABLE_REMOTE_PREFETCH'] = "1" self.network_with_table(is_sparse=True, is_distributed=False) diff --git a/python/paddle/fluid/tests/unittests/test_dist_tree_index.py b/python/paddle/fluid/tests/unittests/test_dist_tree_index.py index 0fae4fe0977..d3d3d249c41 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_tree_index.py +++ b/python/paddle/fluid/tests/unittests/test_dist_tree_index.py @@ -24,35 +24,32 @@ paddle.enable_static() def create_feeds(): - user_input = fluid.layers.data(name="item_id", - shape=[1], - dtype="int64", - lod_level=1) - - item = fluid.layers.data(name="unit_id", - shape=[1], - dtype="int64", - lod_level=1) - - label = fluid.layers.data(name="label", - shape=[1], - dtype="int64", - lod_level=1) - labels = fluid.layers.data(name="labels", - shape=[1], - dtype="int64", - lod_level=1) + user_input = fluid.layers.data( + name="item_id", shape=[1], dtype="int64", lod_level=1 + ) + + item = fluid.layers.data( + name="unit_id", shape=[1], dtype="int64", lod_level=1 + ) + + label = fluid.layers.data( + name="label", shape=[1], dtype="int64", lod_level=1 + ) + labels = fluid.layers.data( + name="labels", shape=[1], dtype="int64", lod_level=1 + ) feed_list = [user_input, item, label, labels] return feed_list class TestTreeIndex(unittest.TestCase): - def test_tree_index(self): path = download( "https://paddlerec.bj.bcebos.com/tree-based/data/mini_tree.pb", - "tree_index_unittest", "e2ba4561c2e9432b532df40546390efa") + "tree_index_unittest", + "e2ba4561c2e9432b532df40546390efa", + ) ''' path = download( "https://paddlerec.bj.bcebos.com/tree-based/data/mini_tree.pb", @@ -72,7 +69,8 @@ class TestTreeIndex(unittest.TestCase): for i in range(tree.height()): layer_node_codes.append(tree.get_layer_codes(i)) layer_node_ids.append( - [node.id() for node in tree.get_nodes(layer_node_codes[-1])]) + [node.id() for node in tree.get_nodes(layer_node_codes[-1])] + ) all_leaf_ids = [node.id() for node in tree.get_all_leafs()] self.assertEqual(sum(all_leaf_ids), sum(layer_node_ids[-1])) @@ -97,8 +95,9 @@ class TestTreeIndex(unittest.TestCase): self.assertEqual(pi_relation[all_leaf_ids[0]], ancestor_codes[0]) # get_travel_path - travel_path_codes = tree.get_travel_path(travel_codes[0], - travel_codes[-1]) + travel_path_codes = tree.get_travel_path( + travel_codes[0], travel_codes[-1] + ) travel_path_ids = [ node.id() for node in tree.get_nodes(travel_path_codes) ] @@ -113,7 +112,6 @@ class TestTreeIndex(unittest.TestCase): class TestIndexSampler(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -123,14 +121,17 @@ class TestIndexSampler(unittest.TestCase): def test_layerwise_sampler(self): path = download( "https://paddlerec.bj.bcebos.com/tree-based/data/mini_tree.pb", - "tree_index_unittest", "e2ba4561c2e9432b532df40546390efa") + "tree_index_unittest", + "e2ba4561c2e9432b532df40546390efa", + ) tdm_layer_counts = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - #tree = TreeIndex("demo", path) - file_name = os.path.join(self.temp_dir.name, - "test_in_memory_dataset_tdm_sample_run.txt") + # tree = TreeIndex("demo", path) + file_name = os.path.join( + self.temp_dir.name, "test_in_memory_dataset_tdm_sample_run.txt" + ) with open(file_name, "w") as f: - #data = "29 d 29 d 29 29 29 29 29 29 29 29 29 29 29 29\n" + # data = "29 d 29 d 29 29 29 29 29 29 29 29 29 29 29 29\n" data = "1 1 1 15 15 15\n" data += "1 1 1 15 15 15\n" f.write(data) @@ -142,26 +143,30 @@ class TestIndexSampler(unittest.TestCase): slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=1, - pipe_command="cat", - download_cmd="cat", - use_var=slots_vars) + dataset.init( + batch_size=1, + pipe_command="cat", + download_cmd="cat", + use_var=slots_vars, + ) dataset.set_filelist([file_name]) - #dataset.update_settings(pipe_command="cat") - #dataset._init_distributed_settings( + # dataset.update_settings(pipe_command="cat") + # dataset._init_distributed_settings( # parse_ins_id=True, # parse_content=True, # fea_eval=True, # candidate_size=10000) dataset.load_into_memory() - dataset.tdm_sample('demo', - tree_path=path, - tdm_layer_counts=tdm_layer_counts, - start_sample_layer=1, - with_hierachy=False, - seed=0, - id_slot=2) + dataset.tdm_sample( + 'demo', + tree_path=path, + tdm_layer_counts=tdm_layer_counts, + start_sample_layer=1, + with_hierachy=False, + seed=0, + id_slot=2, + ) self.assertTrue(dataset.get_shuffle_data_size() == 8) diff --git a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py index 6d9727f6f3a..be100d7699f 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py @@ -21,43 +21,46 @@ flag_name = os.path.splitext(__file__)[0] class TestDistW2V2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._enforce_place = "CPU" def test_dist_train(self): - self.check_with_place("dist_word2vec.py", - delta=1e-4, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_word2vec.py", + delta=1e-4, + check_error_log=True, + log_name=flag_name, + ) class TestDistW2V2x2WithMemOpt(TestDistBase): - def _setup_config(self): self._sync_mode = True self._mem_opt = True self._enforce_place = "CPU" def test_dist_train(self): - self.check_with_place("dist_word2vec.py", - delta=1e-4, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_word2vec.py", + delta=1e-4, + check_error_log=True, + log_name=flag_name, + ) class TestDistW2V2x2Async(TestDistBase): - def _setup_config(self): self._sync_mode = False self._enforce_place = "CPU" def test_dist_train(self): - self.check_with_place("dist_word2vec.py", - delta=100, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_word2vec.py", + delta=100, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py b/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py index 29bd02a578e..354a630bc38 100644 --- a/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py @@ -19,18 +19,26 @@ import paddle from op_test import OpTest -def distribute_fpn_proposals_wrapper(fpn_rois, rois_num, min_level, max_level, - refer_level, refer_scale, pixel_offset): - return paddle.vision.ops.distribute_fpn_proposals(fpn_rois=fpn_rois, - min_level=min_level, - max_level=max_level, - refer_level=refer_level, - refer_scale=refer_scale, - rois_num=rois_num) +def distribute_fpn_proposals_wrapper( + fpn_rois, + rois_num, + min_level, + max_level, + refer_level, + refer_scale, + pixel_offset, +): + return paddle.vision.ops.distribute_fpn_proposals( + fpn_rois=fpn_rois, + min_level=min_level, + max_level=max_level, + refer_level=refer_level, + refer_scale=refer_scale, + rois_num=rois_num, + ) class TestDistributeFPNProposalsOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -43,8 +51,9 @@ class TestDistributeFPNProposalsOp(OpTest): 'refer_level': self.canonical_level, 'pixel_offset': self.pixel_offset, } - output = [('out%d' % i, self.rois_fpn[i]) - for i in range(len(self.rois_fpn))] + output = [ + ('out%d' % i, self.rois_fpn[i]) for i in range(len(self.rois_fpn)) + ] self.outputs = { 'MultiFpnRois': output, @@ -63,8 +72,8 @@ class TestDistributeFPNProposalsOp(OpTest): def boxes_area(self, boxes): offset = 1 if self.pixel_offset else 0 - w = (boxes[:, 2] - boxes[:, 0] + offset) - h = (boxes[:, 3] - boxes[:, 1] + offset) + w = boxes[:, 2] - boxes[:, 0] + offset + h = boxes[:, 3] - boxes[:, 1] + offset areas = w * h assert np.all(areas >= 0), 'Negative areas founds' return areas @@ -85,7 +94,7 @@ class TestDistributeFPNProposalsOp(OpTest): return sub_lod def add_multilevel_roi(self, rois, target_lvls, lvl_min, lvl_max): - rois_idx_order = np.empty((0, )) + rois_idx_order = np.empty((0,)) rois_fpn = [] for lvl in range(lvl_min, lvl_max + 1): idx_lvl = np.where(target_lvls == lvl)[0] @@ -95,17 +104,20 @@ class TestDistributeFPNProposalsOp(OpTest): sub_lod = self.get_sub_lod(rois[idx_lvl, 0]) rois_fpn.append((rois[idx_lvl, 1:], [sub_lod])) rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) - rois_idx_restore = np.argsort(rois_idx_order).astype(np.int32, - copy=False) + rois_idx_restore = np.argsort(rois_idx_order).astype( + np.int32, copy=False + ) return rois_fpn, rois_idx_restore def calc_rois_distribute(self): lvl_min = self.roi_min_level lvl_max = self.roi_max_level - target_lvls = self.map_rois_to_fpn_levels(self.rois[:, 1:5], lvl_min, - lvl_max) + target_lvls = self.map_rois_to_fpn_levels( + self.rois[:, 1:5], lvl_min, lvl_max + ) rois_fpn, rois_idx_restore = self.add_multilevel_roi( - self.rois, target_lvls, lvl_min, lvl_max) + self.rois, target_lvls, lvl_min, lvl_max + ) return rois_fpn, rois_idx_restore def make_rois(self): @@ -133,14 +145,13 @@ class TestDistributeFPNProposalsOp(OpTest): class TestDistributeFPNProposalsOpWithRoisNum(TestDistributeFPNProposalsOp): - def set_data(self): self.init_test_case() self.make_rois() self.rois_fpn, self.rois_idx_restore = self.calc_rois_distribute() self.inputs = { 'FpnRois': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': np.array(self.rois_lod[0]).astype('int32') + 'RoisNum': np.array(self.rois_lod[0]).astype('int32'), } self.attrs = { 'max_level': self.roi_max_level, @@ -149,26 +160,30 @@ class TestDistributeFPNProposalsOpWithRoisNum(TestDistributeFPNProposalsOp): 'refer_level': self.canonical_level, 'pixel_offset': self.pixel_offset, } - output = [('out%d' % i, self.rois_fpn[i]) - for i in range(len(self.rois_fpn))] - rois_num_per_level = [('rois_num%d' % i, - np.array(self.rois_fpn[i][1][0]).astype('int32')) - for i in range(len(self.rois_fpn))] + output = [ + ('out%d' % i, self.rois_fpn[i]) for i in range(len(self.rois_fpn)) + ] + rois_num_per_level = [ + ('rois_num%d' % i, np.array(self.rois_fpn[i][1][0]).astype('int32')) + for i in range(len(self.rois_fpn)) + ] self.outputs = { 'MultiFpnRois': output, 'RestoreIndex': self.rois_idx_restore.reshape(-1, 1), - 'MultiLevelRoIsNum': rois_num_per_level + 'MultiLevelRoIsNum': rois_num_per_level, } self.python_api = distribute_fpn_proposals_wrapper self.python_out_sig = [ - 'MultiFpnRois', 'MultiLevelRoIsNum', 'RestoreIndex' + 'MultiFpnRois', + 'MultiLevelRoIsNum', + 'RestoreIndex', ] class TestDistributeFPNProposalsOpNoOffset( - TestDistributeFPNProposalsOpWithRoisNum): - + TestDistributeFPNProposalsOpWithRoisNum +): def init_test_case(self): self.roi_max_level = 5 self.roi_min_level = 2 @@ -179,7 +194,6 @@ class TestDistributeFPNProposalsOpNoOffset( class TestDistributeFpnProposalsAPI(unittest.TestCase): - def setUp(self): np.random.seed(678) self.rois_np = np.random.rand(10, 4).astype('float32') @@ -188,26 +202,30 @@ class TestDistributeFpnProposalsAPI(unittest.TestCase): def test_dygraph_with_static(self): paddle.enable_static() rois = paddle.static.data(name='rois', shape=[10, 4], dtype='float32') - rois_num = paddle.static.data(name='rois_num', - shape=[None], - dtype='int32') - multi_rois, restore_ind, rois_num_per_level = paddle.vision.ops.distribute_fpn_proposals( + rois_num = paddle.static.data( + name='rois_num', shape=[None], dtype='int32' + ) + ( + multi_rois, + restore_ind, + rois_num_per_level, + ) = paddle.vision.ops.distribute_fpn_proposals( fpn_rois=rois, min_level=2, max_level=5, refer_level=4, refer_scale=224, - rois_num=rois_num) + rois_num=rois_num, + ) fetch_list = multi_rois + [restore_ind] + rois_num_per_level exe = paddle.static.Executor() - output_stat = exe.run(paddle.static.default_main_program(), - feed={ - 'rois': self.rois_np, - 'rois_num': self.rois_num_np - }, - fetch_list=fetch_list, - return_numpy=False) + output_stat = exe.run( + paddle.static.default_main_program(), + feed={'rois': self.rois_np, 'rois_num': self.rois_num_np}, + fetch_list=fetch_list, + return_numpy=False, + ) output_stat_np = [] for output in output_stat: output_np = np.array(output) @@ -217,13 +235,18 @@ class TestDistributeFpnProposalsAPI(unittest.TestCase): paddle.disable_static() rois_dy = paddle.to_tensor(self.rois_np) rois_num_dy = paddle.to_tensor(self.rois_num_np) - multi_rois_dy, restore_ind_dy, rois_num_per_level_dy = paddle.vision.ops.distribute_fpn_proposals( + ( + multi_rois_dy, + restore_ind_dy, + rois_num_per_level_dy, + ) = paddle.vision.ops.distribute_fpn_proposals( fpn_rois=rois_dy, min_level=2, max_level=5, refer_level=4, refer_scale=224, - rois_num=rois_num_dy) + rois_num=rois_num_dy, + ) output_dy = multi_rois_dy + [restore_ind_dy] + rois_num_per_level_dy output_dy_np = [] for output in output_dy: diff --git a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py index fac4e79ce13..7638f7cf243 100644 --- a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py +++ b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py @@ -35,10 +35,12 @@ def remove_file_if_exists(file_name): shutil.rmtree(file_name) -def run_test(clip_after_allreduce=True, - max_global_norm=-1.0, - gradient_merge_steps=1, - use_master_acc_grad=True): +def run_test( + clip_after_allreduce=True, + max_global_norm=-1.0, + gradient_merge_steps=1, + use_master_acc_grad=True, +): temp_dir = tempfile.TemporaryDirectory() if not paddle.is_compiled_with_cuda(): return @@ -66,17 +68,18 @@ def run_test(clip_after_allreduce=True, touch_file_env = 'SUCCESS_TOUCH_FILE' touch_file_name = os.path.join( temp_dir.name, - 'distributed_fused_lamb_touch_file_{}'.format(os.getpid())) + 'distributed_fused_lamb_touch_file_{}'.format(os.getpid()), + ) os.environ[touch_file_env] = touch_file_name try: assert os.system(cmd) == 0 and os.path.exists( - touch_file_name), 'Test failed when {}'.format(args) + touch_file_name + ), 'Test failed when {}'.format(args) finally: temp_dir.cleanup() class TestDistributedFusedLambWithClip(unittest.TestCase): - def test_1(self): run_test(clip_after_allreduce=True, max_global_norm=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py index 01ca09916a1..37b315e61a8 100644 --- a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py +++ b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py @@ -17,17 +17,20 @@ import unittest class TestDistributedFusedLambGradientMerge(unittest.TestCase): - def test_gm(self): - run_test(clip_after_allreduce=True, - max_global_norm=-1.0, - gradient_merge_steps=2) + run_test( + clip_after_allreduce=True, + max_global_norm=-1.0, + gradient_merge_steps=2, + ) def test_gm_with_fp16_acc_grad(self): - run_test(clip_after_allreduce=True, - max_global_norm=-1.0, - gradient_merge_steps=2, - use_master_acc_grad=False) + run_test( + clip_after_allreduce=True, + max_global_norm=-1.0, + gradient_merge_steps=2, + use_master_acc_grad=False, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_without_clip.py b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_without_clip.py index 8d4dfa84d2f..6e9d8ddea17 100644 --- a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_without_clip.py +++ b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_without_clip.py @@ -17,7 +17,6 @@ import unittest class TestDistributedFusedLambWithoutClip(unittest.TestCase): - def test_1(self): run_test(clip_after_allreduce=True, max_global_norm=-1.0) diff --git a/python/paddle/fluid/tests/unittests/test_distributions.py b/python/paddle/fluid/tests/unittests/test_distributions.py index d79e8784e2b..a032b74f796 100644 --- a/python/paddle/fluid/tests/unittests/test_distributions.py +++ b/python/paddle/fluid/tests/unittests/test_distributions.py @@ -16,13 +16,18 @@ import numpy as np import unittest from paddle import fluid from paddle.fluid import layers -from paddle.fluid.layers.distributions import Categorical, MultivariateNormalDiag, Normal, Uniform +from paddle.fluid.layers.distributions import ( + Categorical, + MultivariateNormalDiag, + Normal, + Uniform, +) import math -class DistributionNumpy(): +class DistributionNumpy: """ - Distribution is the abstract base class for probability distributions. + Distribution is the abstract base class for probability distributions. """ def sample(self): @@ -43,15 +48,15 @@ class DistributionNumpy(): class UniformNumpy(DistributionNumpy): - def __init__(self, low, high): self.low = np.array(low).astype('float32') self.high = np.array(high).astype('float32') def sample(self, shape): shape = tuple(shape) + (self.low + self.high).shape - return self.low + (np.random.uniform(size=shape) * - (self.high - self.low)) + return self.low + ( + np.random.uniform(size=shape) * (self.high - self.low) + ) def log_prob(self, value): lb = np.less(self.low, value).astype('float32') @@ -63,7 +68,6 @@ class UniformNumpy(DistributionNumpy): class NormalNumpy(DistributionNumpy): - def __init__(self, loc, scale): self.loc = np.array(loc).astype('float32') self.scale = np.array(scale).astype('float32') @@ -75,24 +79,28 @@ class NormalNumpy(DistributionNumpy): def log_prob(self, value): var = self.scale * self.scale log_scale = np.log(self.scale) - return -((value - self.loc) * - (value - self.loc)) / (2. * var) - log_scale - math.log( - math.sqrt(2. * math.pi)) + return ( + -((value - self.loc) * (value - self.loc)) / (2.0 * var) + - log_scale + - math.log(math.sqrt(2.0 * math.pi)) + ) def entropy(self): - return 0.5 + 0.5 * np.log(np.array( - 2. * math.pi).astype('float32')) + np.log(self.scale) + return ( + 0.5 + + 0.5 * np.log(np.array(2.0 * math.pi).astype('float32')) + + np.log(self.scale) + ) def kl_divergence(self, other): - var_ratio = (self.scale / other.scale) + var_ratio = self.scale / other.scale var_ratio = var_ratio * var_ratio - t1 = ((self.loc - other.loc) / other.scale) - t1 = (t1 * t1) + t1 = (self.loc - other.loc) / other.scale + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) class CategoricalNumpy(DistributionNumpy): - def __init__(self, logits): self.logits = np.array(logits).astype('float32') @@ -101,23 +109,28 @@ class CategoricalNumpy(DistributionNumpy): e_logits = np.exp(logits) z = np.sum(e_logits, axis=-1, keepdims=True) prob = e_logits / z - return -1. * np.sum(prob * (logits - np.log(z)), axis=-1, keepdims=True) + return -1.0 * np.sum( + prob * (logits - np.log(z)), axis=-1, keepdims=True + ) def kl_divergence(self, other): logits = self.logits - np.max(self.logits, axis=-1, keepdims=True) other_logits = other.logits - np.max( - other.logits, axis=-1, keepdims=True) + other.logits, axis=-1, keepdims=True + ) e_logits = np.exp(logits) other_e_logits = np.exp(other_logits) z = np.sum(e_logits, axis=-1, keepdims=True) other_z = np.sum(other_e_logits, axis=-1, keepdims=True) prob = e_logits / z - return np.sum(prob * (logits - np.log(z) - other_logits \ - + np.log(other_z)), axis=-1, keepdims=True) + return np.sum( + prob * (logits - np.log(z) - other_logits + np.log(other_z)), + axis=-1, + keepdims=True, + ) class MultivariateNormalDiagNumpy(DistributionNumpy): - def __init__(self, loc, scale): self.loc = np.array(loc).astype('float32') self.scale = np.array(scale).astype('float32') @@ -139,14 +152,17 @@ class MultivariateNormalDiagNumpy(DistributionNumpy): return inv_diag def entropy(self): - return 0.5 * (self.scale.shape[0] * - (1.0 + np.log(np.array(2 * math.pi).astype('float32'))) + - np.log(self._det(self.scale))) + return 0.5 * ( + self.scale.shape[0] + * (1.0 + np.log(np.array(2 * math.pi).astype('float32'))) + + np.log(self._det(self.scale)) + ) def kl_divergence(self, other): tr_cov_matmul = np.sum(self._inv(other.scale) * self.scale) - loc_matmul_cov = np.matmul((other.loc - self.loc), - self._inv(other.scale)) + loc_matmul_cov = np.matmul( + (other.loc - self.loc), self._inv(other.scale) + ) tri_matmul = np.matmul(loc_matmul_cov, (other.loc - self.loc)) k = list(self.scale.shape)[0] ln_cov = np.log(self._det(other.scale)) - np.log(self._det(self.scale)) @@ -156,7 +172,6 @@ class MultivariateNormalDiagNumpy(DistributionNumpy): class DistributionTest(unittest.TestCase): - def setUp(self, use_gpu=False): self.use_gpu = use_gpu if not use_gpu: @@ -167,20 +182,31 @@ class DistributionTest(unittest.TestCase): self.gpu_id = 0 self.executor = fluid.Executor(place) - def build_normal_program(self, test_program, batch_size, dims, loc_float, - scale_float, other_loc_float, other_scale_float, - scale_np, other_scale_np, loc_np, other_loc_np, - values_np): + def build_normal_program( + self, + test_program, + batch_size, + dims, + loc_float, + scale_float, + other_loc_float, + other_scale_float, + scale_np, + other_scale_np, + loc_np, + other_loc_np, + values_np, + ): with fluid.program_guard(test_program): loc = layers.data(name='loc', shape=[dims], dtype='float32') scale = layers.data(name='scale', shape=[dims], dtype='float32') - other_loc = layers.data(name='other_loc', - shape=[dims], - dtype='float32') - other_scale = layers.data(name='other_scale', - shape=[dims], - dtype='float32') + other_loc = layers.data( + name='other_loc', shape=[dims], dtype='float32' + ) + other_scale = layers.data( + name='other_scale', shape=[dims], dtype='float32' + ) values = layers.data(name='values', shape=[dims], dtype='float32') @@ -188,8 +214,9 @@ class DistributionTest(unittest.TestCase): other_normal_float = Normal(other_loc_float, other_scale_float) normal_float_np_broadcast = Normal(loc_float, scale_np) - other_normal_float_np_broadcast = Normal(other_loc_float, - other_scale_np) + other_normal_float_np_broadcast = Normal( + other_loc_float, other_scale_np + ) normal_np = Normal(loc_np, scale_np) other_normal_np = Normal(other_loc_np, other_scale_np) @@ -199,7 +226,8 @@ class DistributionTest(unittest.TestCase): sample_float = normal_float.sample([batch_size, dims]) sample_float_np_broadcast = normal_float_np_broadcast.sample( - [batch_size, dims]) + [batch_size, dims] + ) sample_np = normal_np.sample([batch_size, dims]) sample_variable = normal_variable.sample([batch_size, dims]) @@ -214,22 +242,34 @@ class DistributionTest(unittest.TestCase): kl_float = normal_float.kl_divergence(other_normal_float) kl_float_np_broadcast = normal_float_np_broadcast.kl_divergence( - other_normal_float_np_broadcast) + other_normal_float_np_broadcast + ) kl_np = normal_np.kl_divergence(other_normal_np) kl_variable = normal_variable.kl_divergence(other_normal_variable) fetch_list = [ - sample_float, sample_float_np_broadcast, sample_np, sample_variable, - entropy_float, entropy_float_np_broadcast, entropy_np, - entropy_variable, lp_float_np_broadcast, lp_np, lp_variable, - kl_float, kl_float_np_broadcast, kl_np, kl_variable + sample_float, + sample_float_np_broadcast, + sample_np, + sample_variable, + entropy_float, + entropy_float_np_broadcast, + entropy_np, + entropy_variable, + lp_float_np_broadcast, + lp_np, + lp_variable, + kl_float, + kl_float_np_broadcast, + kl_np, + kl_variable, ] feed_vars = { 'loc': loc_np, 'scale': scale_np, 'other_loc': other_loc_np, 'other_scale': other_scale_np, - 'values': values_np + 'values': values_np, } return feed_vars, fetch_list @@ -255,117 +295,176 @@ class DistributionTest(unittest.TestCase): scale_np = np.random.randn(batch_size, dims).astype('float32') while not np.all(other_scale_np > 0): other_scale_np = np.random.randn(batch_size, dims).astype('float32') - return loc_np, other_loc_np, loc_float, scale_float, other_loc_float, \ - other_scale_float, scale_np, other_scale_np, values_np + return ( + loc_np, + other_loc_np, + loc_float, + scale_float, + other_loc_float, + other_scale_float, + scale_np, + other_scale_np, + values_np, + ) def test_normal_distribution(self, batch_size=2, dims=3, tolerance=1e-6): test_program = fluid.Program() - loc_np, other_loc_np, loc_float, scale_float, other_loc_float, other_scale_float, scale_np, other_scale_np, values_np = self.get_normal_random_input( - batch_size, dims) + ( + loc_np, + other_loc_np, + loc_float, + scale_float, + other_loc_float, + other_scale_float, + scale_np, + other_scale_np, + values_np, + ) = self.get_normal_random_input(batch_size, dims) feed_vars, fetch_list = self.build_normal_program( - test_program, batch_size, dims, loc_float, scale_float, - other_loc_float, other_scale_float, scale_np, other_scale_np, - loc_np, other_loc_np, values_np) + test_program, + batch_size, + dims, + loc_float, + scale_float, + other_loc_float, + other_scale_float, + scale_np, + other_scale_np, + loc_np, + other_loc_np, + values_np, + ) self.executor.run(fluid.default_startup_program()) np_normal_float = NormalNumpy(loc_float, scale_float) np_other_normal_float = NormalNumpy(other_loc_float, other_scale_float) np_normal_float_np_broadcast = NormalNumpy(loc_float, scale_np) np_other_normal_float_np_broadcast = NormalNumpy( - other_loc_float, other_scale_np) + other_loc_float, other_scale_np + ) np_normal = NormalNumpy(loc_np, scale_np) np_other_normal = NormalNumpy(other_loc_np, other_scale_np) gt_sample_float = np_normal_float.sample([batch_size, dims]) gt_sample_float_np_broadcast = np_normal_float_np_broadcast.sample( - [batch_size, dims]) + [batch_size, dims] + ) gt_sample_np = np_normal.sample([batch_size, dims]) gt_entropy_float = np_normal_float.entropy() gt_entropy_float_np_broadcast = np_normal_float_np_broadcast.entropy() gt_entropy = np_normal.entropy() gt_lp_float_np_broadcast = np_normal_float_np_broadcast.log_prob( - values_np) + values_np + ) gt_lp = np_normal.log_prob(values_np) gt_kl_float = np_normal_float.kl_divergence(np_other_normal_float) gt_kl_float_np_broadcast = np_normal_float_np_broadcast.kl_divergence( - np_other_normal_float_np_broadcast) + np_other_normal_float_np_broadcast + ) gt_kl = np_normal.kl_divergence(np_other_normal) [ - output_sample_float, output_sample_float_np_broadcast, - output_sample_np, output_sample_variable, output_entropy_float, - output_entropy_float_np_broadcast, output_entropy_np, - output_entropy_variable, output_lp_float_np_broadcast, output_lp_np, - output_lp_variable, output_kl_float, output_kl_float_np_broadcast, - output_kl_np, output_kl_variable - ] = self.executor.run(program=test_program, - feed=feed_vars, - fetch_list=fetch_list) - - np.testing.assert_allclose(output_sample_float.shape, - gt_sample_float.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_float_np_broadcast.shape, - gt_sample_float_np_broadcast.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_np.shape, - gt_sample_np.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_variable.shape, - gt_sample_np.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_float, - gt_entropy_float, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_float_np_broadcast, - gt_entropy_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_np, - gt_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_variable, - gt_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_float_np_broadcast, - gt_lp_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_np, - gt_lp, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_variable, - gt_lp, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_float, - gt_kl_float, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_float_np_broadcast, - gt_kl_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_np, - gt_kl, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_variable, - gt_kl, - rtol=tolerance, - atol=tolerance) - - def build_uniform_program(self, test_program, batch_size, dims, low_float, - high_float, high_np, low_np, values_np): + output_sample_float, + output_sample_float_np_broadcast, + output_sample_np, + output_sample_variable, + output_entropy_float, + output_entropy_float_np_broadcast, + output_entropy_np, + output_entropy_variable, + output_lp_float_np_broadcast, + output_lp_np, + output_lp_variable, + output_kl_float, + output_kl_float_np_broadcast, + output_kl_np, + output_kl_variable, + ] = self.executor.run( + program=test_program, feed=feed_vars, fetch_list=fetch_list + ) + + np.testing.assert_allclose( + output_sample_float.shape, + gt_sample_float.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_float_np_broadcast.shape, + gt_sample_float_np_broadcast.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_np.shape, + gt_sample_np.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_variable.shape, + gt_sample_np.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_float, + gt_entropy_float, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_float_np_broadcast, + gt_entropy_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_np, gt_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_entropy_variable, gt_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_lp_float_np_broadcast, + gt_lp_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_lp_np, gt_lp, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_lp_variable, gt_lp, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_float, gt_kl_float, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_float_np_broadcast, + gt_kl_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_kl_np, gt_kl, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_variable, gt_kl, rtol=tolerance, atol=tolerance + ) + + def build_uniform_program( + self, + test_program, + batch_size, + dims, + low_float, + high_float, + high_np, + low_np, + values_np, + ): with fluid.program_guard(test_program): low = layers.data(name='low', shape=[dims], dtype='float32') high = layers.data(name='high', shape=[dims], dtype='float32') @@ -379,7 +478,8 @@ class DistributionTest(unittest.TestCase): sample_float = uniform_float.sample([batch_size, dims]) sample_float_np_broadcast = uniform_float_np_broadcast.sample( - [batch_size, dims]) + [batch_size, dims] + ) sample_np = uniform_np.sample([batch_size, dims]) sample_variable = uniform_variable.sample([batch_size, dims]) @@ -393,9 +493,17 @@ class DistributionTest(unittest.TestCase): lp_variable = uniform_variable.log_prob(values) fetch_list = [ - sample_float, sample_float_np_broadcast, sample_np, sample_variable, - entropy_float, entropy_float_np_broadcast, entropy_np, - entropy_variable, lp_float_np_broadcast, lp_np, lp_variable + sample_float, + sample_float_np_broadcast, + sample_np, + sample_variable, + entropy_float, + entropy_float_np_broadcast, + entropy_np, + entropy_variable, + lp_float_np_broadcast, + lp_np, + lp_variable, ] feed_vars = {'low': low_np, 'high': high_np, 'values': values_np} return feed_vars, fetch_list @@ -406,13 +514,21 @@ class DistributionTest(unittest.TestCase): low_np = np.random.randn(batch_size, dims).astype('float32') low_float = np.random.uniform(-2, 1) high_float = np.random.uniform(1, 3) - high_np = np.random.uniform(-5.0, 5.0, - (batch_size, dims)).astype('float32') + high_np = np.random.uniform(-5.0, 5.0, (batch_size, dims)).astype( + 'float32' + ) values_np = np.random.randn(batch_size, dims).astype('float32') feed_vars, fetch_list = self.build_uniform_program( - test_program, batch_size, dims, low_float, high_float, high_np, - low_np, values_np) + test_program, + batch_size, + dims, + low_float, + high_float, + high_np, + low_np, + values_np, + ) self.executor.run(fluid.default_startup_program()) @@ -422,75 +538,92 @@ class DistributionTest(unittest.TestCase): gt_sample_float = np_uniform_float.sample([batch_size, dims]) gt_sample_float_np_broadcast = np_uniform_float_np_broadcast.sample( - [batch_size, dims]) + [batch_size, dims] + ) gt_sample_np = np_uniform.sample([batch_size, dims]) gt_entropy_float = np_uniform_float.entropy() gt_entropy_float_np_broadcast = np_uniform_float_np_broadcast.entropy() gt_entropy = np_uniform.entropy() gt_lp_float_np_broadcast = np_uniform_float_np_broadcast.log_prob( - values_np) + values_np + ) gt_lp = np_uniform.log_prob(values_np) # result calculated by paddle [ - output_sample_float, output_sample_float_np_broadcast, - output_sample_np, output_sample_variable, output_entropy_float, - output_entropy_float_np_broadcast, output_entropy_np, - output_entropy_variable, output_lp_float_np_broadcast, output_lp_np, - output_lp_variable - ] = self.executor.run(program=test_program, - feed=feed_vars, - fetch_list=fetch_list) - - np.testing.assert_allclose(output_sample_float.shape, - gt_sample_float.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_float_np_broadcast.shape, - gt_sample_float_np_broadcast.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_np.shape, - gt_sample_np.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_variable.shape, - gt_sample_np.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_float, - gt_entropy_float, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_float_np_broadcast, - gt_entropy_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_np, - gt_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_variable, - gt_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_float_np_broadcast, - gt_lp_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_np, - gt_lp, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_variable, - gt_lp, - rtol=tolerance, - atol=tolerance) - - def test_categorical_distribution(self, - batch_size=2, - dims=3, - tolerance=1e-6): + output_sample_float, + output_sample_float_np_broadcast, + output_sample_np, + output_sample_variable, + output_entropy_float, + output_entropy_float_np_broadcast, + output_entropy_np, + output_entropy_variable, + output_lp_float_np_broadcast, + output_lp_np, + output_lp_variable, + ] = self.executor.run( + program=test_program, feed=feed_vars, fetch_list=fetch_list + ) + + np.testing.assert_allclose( + output_sample_float.shape, + gt_sample_float.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_float_np_broadcast.shape, + gt_sample_float_np_broadcast.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_np.shape, + gt_sample_np.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_variable.shape, + gt_sample_np.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_float, + gt_entropy_float, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_float_np_broadcast, + gt_entropy_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_np, gt_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_entropy_variable, gt_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_lp_float_np_broadcast, + gt_lp_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_lp_np, gt_lp, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_lp_variable, gt_lp, rtol=tolerance, atol=tolerance + ) + + def test_categorical_distribution( + self, batch_size=2, dims=3, tolerance=1e-6 + ): test_program = fluid.Program() logits_np = np.random.randn(batch_size, dims).astype('float32') @@ -498,9 +631,9 @@ class DistributionTest(unittest.TestCase): with fluid.program_guard(test_program): logits = layers.data(name='logits', shape=[dims], dtype='float32') - other_logits = layers.data(name='other_logits', - shape=[dims], - dtype='float32') + other_logits = layers.data( + name='other_logits', shape=[dims], dtype='float32' + ) categorical_np = Categorical(logits_np) other_categorical_np = Categorical(other_logits_np) @@ -516,55 +649,74 @@ class DistributionTest(unittest.TestCase): gt_kl_np = np_categorical.kl_divergence(np_other_categorical) # result calculated by paddle - [output_entropy_np, - output_kl_np] = self.executor.run(program=test_program, - feed={'logits': logits_np}, - fetch_list=[entropy_np, kl_np]) - np.testing.assert_allclose(output_entropy_np, - gt_entropy_np, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_np, - gt_kl_np, - rtol=tolerance, - atol=tolerance) - - def test_multivariateNormalDiag_distribution(self, - batch_size=2, - tolerance=1e-6): + [output_entropy_np, output_kl_np] = self.executor.run( + program=test_program, + feed={'logits': logits_np}, + fetch_list=[entropy_np, kl_np], + ) + np.testing.assert_allclose( + output_entropy_np, gt_entropy_np, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_np, gt_kl_np, rtol=tolerance, atol=tolerance + ) + + def test_multivariateNormalDiag_distribution( + self, batch_size=2, tolerance=1e-6 + ): test_program = fluid.Program() - loc_np = np.random.random(batch_size, ).astype('float32') - scale_np = np.diag(np.random.random(batch_size, )).astype('float32') - other_loc_np = np.random.random(batch_size, ).astype('float32') - other_scale_np = np.diag(np.random.random( - batch_size, )).astype('float32') + loc_np = np.random.random( + batch_size, + ).astype('float32') + scale_np = np.diag( + np.random.random( + batch_size, + ) + ).astype('float32') + other_loc_np = np.random.random( + batch_size, + ).astype('float32') + other_scale_np = np.diag( + np.random.random( + batch_size, + ) + ).astype('float32') with fluid.program_guard(test_program): - loc = layers.data(name='loc', - shape=[ - batch_size, - ], - dtype='float32', - append_batch_size=False) - scale = layers.data(name='scale', - shape=[batch_size, batch_size], - dtype='float32', - append_batch_size=False) - other_loc = layers.data(name='other_loc', - shape=[ - batch_size, - ], - dtype='float32', - append_batch_size=False) - other_scale = layers.data(name='other_scale', - shape=[batch_size, batch_size], - dtype='float32', - append_batch_size=False) + loc = layers.data( + name='loc', + shape=[ + batch_size, + ], + dtype='float32', + append_batch_size=False, + ) + scale = layers.data( + name='scale', + shape=[batch_size, batch_size], + dtype='float32', + append_batch_size=False, + ) + other_loc = layers.data( + name='other_loc', + shape=[ + batch_size, + ], + dtype='float32', + append_batch_size=False, + ) + other_scale = layers.data( + name='other_scale', + shape=[batch_size, batch_size], + dtype='float32', + append_batch_size=False, + ) multivariate_np = MultivariateNormalDiag(loc, scale) other_multivariate_np = MultivariateNormalDiag( - other_loc, other_scale) + other_loc, other_scale + ) entropy_np = multivariate_np.entropy() other_entropy_np = other_multivariate_np.entropy() @@ -574,32 +726,31 @@ class DistributionTest(unittest.TestCase): np_multivariate = MultivariateNormalDiagNumpy(loc_np, scale_np) np_other_multivariate = MultivariateNormalDiagNumpy( - other_loc_np, other_scale_np) + other_loc_np, other_scale_np + ) gt_entropy_np = np_multivariate.entropy() gt_kl_np = np_multivariate.kl_divergence(np_other_multivariate) # result calculated by paddle - [output_entropy_np, - output_kl_np] = self.executor.run(program=test_program, - feed={ - 'loc': loc_np, - 'scale': scale_np, - 'other_loc': other_loc_np, - 'other_scale': other_scale_np - }, - fetch_list=[entropy_np, kl_np]) - np.testing.assert_allclose(output_entropy_np, - gt_entropy_np, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_np, - gt_kl_np, - rtol=tolerance, - atol=tolerance) + [output_entropy_np, output_kl_np] = self.executor.run( + program=test_program, + feed={ + 'loc': loc_np, + 'scale': scale_np, + 'other_loc': other_loc_np, + 'other_scale': other_scale_np, + }, + fetch_list=[entropy_np, kl_np], + ) + np.testing.assert_allclose( + output_entropy_np, gt_entropy_np, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_np, gt_kl_np, rtol=tolerance, atol=tolerance + ) class DistributionTestError(unittest.TestCase): - def test_normal_error(self): loc = int(1) scale = int(1) @@ -658,8 +809,9 @@ class DistributionTestError(unittest.TestCase): categorical_other = Normal(1.0, 2.0) # type of other must be an instance of Normal - self.assertRaises(TypeError, categorical.kl_divergence, - categorical_other) + self.assertRaises( + TypeError, categorical.kl_divergence, categorical_other + ) def test_multivariate_normal_diag_error(self): loc = 1.0 diff --git a/python/paddle/fluid/tests/unittests/test_dot_op.py b/python/paddle/fluid/tests/unittests/test_dot_op.py index d9850a88d0d..006c91d60e6 100644 --- a/python/paddle/fluid/tests/unittests/test_dot_op.py +++ b/python/paddle/fluid/tests/unittests/test_dot_op.py @@ -22,7 +22,6 @@ from paddle.fluid import Program, program_guard class DotOp(OpTest): - def setUp(self): self.op_type = "dot" self.python_api = paddle.dot @@ -31,7 +30,7 @@ class DotOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {} @@ -45,35 +44,38 @@ class DotOp(OpTest): ['X', 'Y'], 'Out', user_defined_grads=[self.inputs['Y'], self.inputs['X']], - check_eager=True) + check_eager=True, + ) else: self.check_grad(['X', 'Y'], 'Out', check_eager=True) def test_check_grad_ingore_x(self): if core.is_compiled_with_rocm(): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.inputs['X']], - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.inputs['X']], + check_eager=True, + ) else: - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], 'Out', no_grad_set=set("X"), check_eager=True + ) def test_check_grad_ingore_y(self): if core.is_compiled_with_rocm(): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.inputs['Y']], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.inputs['Y']], + check_eager=True, + ) else: - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], 'Out', no_grad_set=set('Y'), check_eager=True + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [121]).astype(self.dtype) @@ -85,12 +87,15 @@ class DotOp(OpTest): class DotOpBatch(DotOp): - def init_input_output(self): - self.x = np.random.uniform(0.1, 1, - [132]).astype(self.dtype).reshape([11, 12]) - self.y = np.random.uniform(1, 3, - [132]).astype(self.dtype).reshape([11, 12]) + self.x = ( + np.random.uniform(0.1, 1, [132]) + .astype(self.dtype) + .reshape([11, 12]) + ) + self.y = ( + np.random.uniform(1, 3, [132]).astype(self.dtype).reshape([11, 12]) + ) self.out = np.sum(self.x * self.y, axis=1).reshape([11, 1]) def test_check_grad_normal(self): @@ -104,7 +109,6 @@ class DotOpBatch(DotOp): class TestDotOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -124,25 +128,26 @@ class TestDotOpError(unittest.TestCase): class TestDygraph(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): x1 = fluid.dygraph.to_variable(np.array([1, 3]).astype(np.float32)) y1 = fluid.dygraph.to_variable(np.array([2, 5]).astype(np.float32)) - np.testing.assert_allclose(paddle.dot(x1, y1).numpy(), - np.array([17]), - rtol=1e-05) + np.testing.assert_allclose( + paddle.dot(x1, y1).numpy(), np.array([17]), rtol=1e-05 + ) x1 = fluid.dygraph.to_variable( - np.array([[1, 3], [3, 5]]).astype(np.float32)) + np.array([[1, 3], [3, 5]]).astype(np.float32) + ) y1 = fluid.dygraph.to_variable( - np.array([[2, 5], [6, 8]]).astype(np.float32)) + np.array([[2, 5], [6, 8]]).astype(np.float32) + ) np.testing.assert_array_equal( - paddle.dot(x1, y1).numpy(), np.array([[17], [58]])) + paddle.dot(x1, y1).numpy(), np.array([[17], [58]]) + ) class TestComplexDotOp(OpTest): - def setUp(self): self.op_type = "dot" self.python_api = paddle.dot @@ -152,7 +157,7 @@ class TestComplexDotOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} @@ -161,13 +166,15 @@ class TestComplexDotOp(OpTest): def init_input_output(self): self.x = np.random.random(100).astype( - self.dtype) + 1J * np.random.random(100).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(100).astype(self.dtype) self.y = np.random.random(100).astype( - self.dtype) + 1J * np.random.random(100).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(100).astype(self.dtype) self.out = np.dot(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones(1, self.dtype) + 1J * np.ones(1, self.dtype) + self.grad_out = np.ones(1, self.dtype) + 1j * np.ones(1, self.dtype) self.grad_x = self.grad_out * np.conj(self.y) self.grad_y = self.grad_out * np.conj(self.x) @@ -175,31 +182,36 @@ class TestComplexDotOp(OpTest): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestComplexDotOp2D(OpTest): - def setUp(self): self.op_type = "dot" self.init_base_dtype() @@ -208,7 +220,7 @@ class TestComplexDotOp2D(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} @@ -216,17 +228,18 @@ class TestComplexDotOp2D(OpTest): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (2, 100)).astype(self.dtype) + 1J * np.random.random( - (2, 100)).astype(self.dtype) - self.y = np.random.random( - (2, 100)).astype(self.dtype) + 1J * np.random.random( - (2, 100)).astype(self.dtype) + self.x = np.random.random((2, 100)).astype( + self.dtype + ) + 1j * np.random.random((2, 100)).astype(self.dtype) + self.y = np.random.random((2, 100)).astype( + self.dtype + ) + 1j * np.random.random((2, 100)).astype(self.dtype) self.out = np.diag(np.dot(self.x, self.y.T)).reshape(-1, 1) def init_grad_input_output(self): - self.grad_out = np.ones((2, 1), self.dtype) + 1J * np.ones( - (2, 1), self.dtype) + self.grad_out = np.ones((2, 1), self.dtype) + 1j * np.ones( + (2, 1), self.dtype + ) self.grad_x = self._get_grad(self.grad_out, self.y) self.grad_y = self._get_grad(self.grad_out, self.x) @@ -240,24 +253,30 @@ class TestComplexDotOp2D(OpTest): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_downpoursgd.py b/python/paddle/fluid/tests/unittests/test_downpoursgd.py index f241a854c89..6626f0ebfa0 100644 --- a/python/paddle/fluid/tests/unittests/test_downpoursgd.py +++ b/python/paddle/fluid/tests/unittests/test_downpoursgd.py @@ -18,7 +18,10 @@ import paddle.fluid as fluid import os import unittest import sys -from paddle.fluid.incubate.fleet.parameter_server.pslib.node import DownpourWorker, DownpourServer +from paddle.fluid.incubate.fleet.parameter_server.pslib.node import ( + DownpourWorker, + DownpourServer, +) from google.protobuf import text_format import paddle.fluid.incubate.fleet.parameter_server.pslib.ps_pb2 as pslib from paddle.fluid.trainer_factory import TrainerFactory @@ -40,15 +43,17 @@ class TestListenAndServOp(unittest.TestCase): pass else: print(sys.platform) - if not os.path.exists('{}/{}'.format(cache_path, - 'fleet_desc.prototxt')): + if not os.path.exists( + '{}/{}'.format(cache_path, 'fleet_desc.prototxt') + ): cmd = "wget --no-check-certificate https://pslib.bj.bcebos.com/fleet_desc.prototxt -P {}/".format( - cache_path) + cache_path + ) os.system(cmd) x = fluid.layers.data(name='x', shape=[1], dtype='int64') - x_emb = fluid.layers.embedding(input=x, - size=[1, 2], - is_distributed=True) + x_emb = fluid.layers.embedding( + input=x, size=[1, 2], is_distributed=True + ) y_predict = fluid.layers.fc(input=x_emb, size=1, act=None) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) @@ -67,7 +72,7 @@ class TestListenAndServOp(unittest.TestCase): program_configs = {} program_configs[program_id] = { "pull_sparse": [0], - "push_sparse": [0] + "push_sparse": [0], } program_configs[program_id]["pull_dense"] = [1] program_configs[program_id]["push_dense"] = [1] @@ -100,15 +105,17 @@ class TestListenAndServOp(unittest.TestCase): pass else: print(sys.platform) - if not os.path.exists('{}/{}'.format(cache_path, - 'fleet_desc.prototxt')): + if not os.path.exists( + '{}/{}'.format(cache_path, 'fleet_desc.prototxt') + ): cmd = "wget --no-check-certificate https://pslib.bj.bcebos.com/fleet_desc.prototxt -P {}/".format( - cache_path) + cache_path + ) os.system(cmd) x = fluid.layers.data(name='x', shape=[1], dtype='int64') - x_emb = fluid.layers.embedding(input=x, - size=[1, 2], - is_distributed=True) + x_emb = fluid.layers.embedding( + input=x, size=[1, 2], is_distributed=True + ) y_predict = fluid.layers.fc(input=x_emb, size=1, act=None) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) @@ -127,7 +134,7 @@ class TestListenAndServOp(unittest.TestCase): program_configs = {} program_configs[program_id] = { "pull_sparse": [0], - "push_sparse": [0] + "push_sparse": [0], } program_configs[program_id]["pull_dense"] = [1] program_configs[program_id]["push_dense"] = [1] @@ -158,15 +165,17 @@ class TestListenAndServOp(unittest.TestCase): pass else: print(sys.platform) - if not os.path.exists('{}/{}'.format(cache_path, - 'fleet_desc.prototxt')): + if not os.path.exists( + '{}/{}'.format(cache_path, 'fleet_desc.prototxt') + ): cmd = "wget --no-check-certificate https://pslib.bj.bcebos.com/fleet_desc.prototxt -P {}/".format( - cache_path) + cache_path + ) os.system(cmd) x = fluid.layers.data(name='x', shape=[1], dtype='int64') - x_emb = fluid.layers.embedding(input=x, - size=[1, 2], - is_distributed=True) + x_emb = fluid.layers.embedding( + input=x, size=[1, 2], is_distributed=True + ) y_predict = fluid.layers.fc(input=x_emb, size=1, act=None) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) @@ -185,7 +194,7 @@ class TestListenAndServOp(unittest.TestCase): program_configs = {} program_configs[program_id] = { "pull_sparse": [0], - "push_sparse": [0] + "push_sparse": [0], } program_configs[program_id]["pull_dense"] = [1] program_configs[program_id]["push_dense"] = [1] diff --git a/python/paddle/fluid/tests/unittests/test_dpsgd_op.py b/python/paddle/fluid/tests/unittests/test_dpsgd_op.py index 1b529d17601..2e505c05a89 100644 --- a/python/paddle/fluid/tests/unittests/test_dpsgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_dpsgd_op.py @@ -18,10 +18,8 @@ from op_test import OpTest class TestDpsgdOp(OpTest): - def setUp(self): - '''Test Dpsgd Operator with supplied attributes - ''' + '''Test Dpsgd Operator with supplied attributes''' self.op_type = "dpsgd" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -34,7 +32,7 @@ class TestDpsgdOp(OpTest): self.inputs = { 'Param': param, 'Grad': grad, - 'LearningRate': np.array([learning_rate]).astype("float32") + 'LearningRate': np.array([learning_rate]).astype("float32"), } self.attrs = {'clip': clip, 'batch_size': batch_size, 'sigma': sigma} @@ -70,5 +68,6 @@ def dpsgd_step(inputs, attributes): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py b/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py index d558032dd37..a21ec9625f9 100644 --- a/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py +++ b/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py @@ -24,34 +24,44 @@ from paddle import _legacy_C_ops from paddle.static import default_main_program -def dropout_nd(x, - p=0.5, - axis=None, - training=True, - mode="upscale_in_train", - name=None): +def dropout_nd( + x, p=0.5, axis=None, training=True, mode="upscale_in_train", name=None +): drop_axes = [axis] if isinstance(axis, int) else list(axis) seed = None - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed - out, mask = _legacy_C_ops.dropout_nd(x, 'dropout_prob', p, 'is_test', - not training, 'fix_seed', seed - is not None, 'seed', - seed if seed is not None else 0, - 'dropout_implementation', mode, - 'axis', drop_axes) + out, mask = _legacy_C_ops.dropout_nd( + x, + 'dropout_prob', + p, + 'is_test', + not training, + 'fix_seed', + seed is not None, + 'seed', + seed if seed is not None else 0, + 'dropout_implementation', + mode, + 'axis', + drop_axes, + ) return out helper = LayerHelper('dropout_nd', **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'dropout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'dropout' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) mask = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) def get_attrs(prog, dropout_prob, is_test, seed): if (seed is None or seed == 0) and prog.random_seed != 0: @@ -62,19 +72,18 @@ def dropout_nd(x, 'fix_seed': seed is not None, 'seed': seed if seed is not None else 0, 'dropout_implementation': mode, - 'axis': drop_axes + 'axis': drop_axes, } return attrs attrs = get_attrs(helper.main_program, p, not training, seed) - helper.append_op(type='dropout_nd', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs=attrs) + helper.append_op( + type='dropout_nd', + inputs={'X': [x]}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs=attrs, + ) return out @@ -82,7 +91,6 @@ paddle.enable_static() class TestDropoutNdOp(OpTest): - def setUp(self): self.op_type = "dropout_nd" self.inputs = {'X': np.random.random((4, 32, 16)).astype("float64")} @@ -90,11 +98,11 @@ class TestDropoutNdOp(OpTest): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'axis': [1] + 'axis': [1], } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((1, 32, 1)).astype('uint8') + 'Mask': np.ones((1, 32, 1)).astype('uint8'), } def test_check_output(self): @@ -105,7 +113,6 @@ class TestDropoutNdOp(OpTest): class TestDropoutNdAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -118,7 +125,7 @@ class TestDropoutNdAPI(unittest.TestCase): with fluid.dygraph.guard(place): in_np = np.random.random([4, 32, 16]).astype("float32") input = paddle.to_tensor(in_np) - res1 = dropout_nd(x=input, p=0., axis=[0, 1]) + res1 = dropout_nd(x=input, p=0.0, axis=[0, 1]) res2 = dropout_nd(x=input, p=0.5, axis=[0, 1]) np.testing.assert_allclose(res1.numpy(), in_np, rtol=1e-05) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_dropout_op.py b/python/paddle/fluid/tests/unittests/test_dropout_op.py index b7da098182b..862ca0c1f91 100644 --- a/python/paddle/fluid/tests/unittests/test_dropout_op.py +++ b/python/paddle/fluid/tests/unittests/test_dropout_op.py @@ -26,14 +26,13 @@ from paddle import _C_ops class TestDropoutOp(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False} self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } def test_check_output(self): @@ -44,14 +43,13 @@ class TestDropoutOp(OpTest): class TestDropoutOpInput1d(OpTest): - def setUp(self): self.op_type = "dropout" - self.inputs = {'X': np.random.random((2000, )).astype("float32")} + self.inputs = {'X': np.random.random((2000,)).astype("float32")} self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False} self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((2000)).astype('uint8') + 'Mask': np.ones((2000)).astype('uint8'), } def test_check_output(self): @@ -62,32 +60,29 @@ class TestDropoutOpInput1d(OpTest): class TestDropoutOp2(TestDropoutOp): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} self.attrs = {'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False} self.outputs = { 'Out': np.zeros((32, 64)).astype('float32'), - 'Mask': np.zeros((32, 64)).astype('uint8') + 'Mask': np.zeros((32, 64)).astype('uint8'), } class TestDropoutOp3(TestDropoutOp): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 2)).astype("float32")} self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False} self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64, 2)).astype('uint8') + 'Mask': np.ones((32, 64, 2)).astype('uint8'), } @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOp4(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} @@ -102,7 +97,6 @@ class TestDropoutOp4(OpTest): @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOp5(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")} @@ -116,7 +110,6 @@ class TestDropoutOp5(OpTest): class TestDropoutOp6(TestDropoutOp): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} @@ -124,16 +117,15 @@ class TestDropoutOp6(TestDropoutOp): 'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': np.zeros((32, 64)).astype('float32'), - 'Mask': np.zeros((32, 64)).astype('uint8') + 'Mask': np.zeros((32, 64)).astype('uint8'), } class TestDropoutOp7(TestDropoutOp): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 2)).astype("float32")} @@ -141,17 +133,16 @@ class TestDropoutOp7(TestDropoutOp): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64, 2)).astype('uint8') + 'Mask': np.ones((32, 64, 2)).astype('uint8'), } @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOp8(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} @@ -159,7 +150,7 @@ class TestDropoutOp8(OpTest): 'dropout_prob': 0.35, 'fix_seed': True, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -169,14 +160,13 @@ class TestDropoutOp8(OpTest): @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOp9(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")} self.attrs = { 'dropout_prob': 0.75, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -185,19 +175,18 @@ class TestDropoutOp9(OpTest): class TestDropoutOpWithSeed(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = { "X": np.random.random((32, 64)).astype("float32"), - "Seed": np.asarray([125], dtype="int32") + "Seed": np.asarray([125], dtype="int32"), } self.attrs = { 'dropout_prob': 0.0, } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } def test_check_output(self): @@ -207,13 +196,12 @@ class TestDropoutOpWithSeed(OpTest): self.check_grad(['X'], 'Out', max_relative_error=0.05) -@unittest.skipIf(not core.is_compiled_with_cuda() - or not core.op_support_gpu("dropout"), - "core is not compiled with CUDA or core is not support dropout" - ) +@unittest.skipIf( + not core.is_compiled_with_cuda() or not core.op_support_gpu("dropout"), + "core is not compiled with CUDA or core is not support dropout", +) @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestFP16DropoutOp(OpTest): - def setUp(self): self.op_type = "dropout" self.init_test_case() @@ -224,7 +212,7 @@ class TestFP16DropoutOp(OpTest): self.attrs = { 'dropout_prob': self.prob, 'fix_seed': self.fix_seed, - 'is_test': True + 'is_test': True, } self.outputs = {'Out': out} @@ -237,13 +225,12 @@ class TestFP16DropoutOp(OpTest): self.check_output_with_place(core.CUDAPlace(0), atol=1e-3) -@unittest.skipIf(not core.is_compiled_with_cuda() - or not core.op_support_gpu("dropout"), - "core is not compiled with CUDA or core is not support dropout" - ) +@unittest.skipIf( + not core.is_compiled_with_cuda() or not core.op_support_gpu("dropout"), + "core is not compiled with CUDA or core is not support dropout", +) @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestFP16DropoutOp2(TestFP16DropoutOp): - def init_test_case(self): self.input_size = [32, 64, 3] self.prob = 0.75 @@ -251,7 +238,6 @@ class TestFP16DropoutOp2(TestFP16DropoutOp): class TestBF16DropoutOp(OpTest): - def setUp(self): self.op_type = "dropout" self.dtype = np.uint16 @@ -260,9 +246,10 @@ class TestBF16DropoutOp(OpTest): self.inputs = {'X': convert_float_to_uint16(x)} self.attrs = {'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False} self.outputs = { - 'Out': - convert_float_to_uint16(np.zeros((32, 64)).astype('float32')), - 'Mask': np.zeros((32, 64)).astype('uint8') + 'Out': convert_float_to_uint16( + np.zeros((32, 64)).astype('float32') + ), + 'Mask': np.zeros((32, 64)).astype('uint8'), } def test_check_output(self): @@ -273,7 +260,6 @@ class TestBF16DropoutOp(OpTest): class TestDropoutOpWithSeedOnCPUPlace(unittest.TestCase): - def test_seed_cpu_place(self): paddle.enable_static() main_program = Program() @@ -288,51 +274,52 @@ class TestDropoutOpWithSeedOnCPUPlace(unittest.TestCase): shape=[1], dtype='int32', persistable=False, - stop_gradient=True) + stop_gradient=True, + ) x_out_var = main_program.global_block().create_var( name=x_out_var, shape=[40, 40], dtype='float32', persistable=False, - stop_gradient=True) - x_var = main_program.global_block().create_var(name=x_var_name, - shape=[40, 40], - dtype='float32', - persistable=False, - stop_gradient=True) + stop_gradient=True, + ) + x_var = main_program.global_block().create_var( + name=x_var_name, + shape=[40, 40], + dtype='float32', + persistable=False, + stop_gradient=True, + ) mask_var = main_program.global_block().create_var( name=mask_var_name, shape=[1], dtype='int', persistable=False, - stop_gradient=True) - - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": x_var_name}, - attrs={ - "shape": [40, 40], - "dtype": x_var.dtype, - "value": 1.0, - "place_type": 0 - }) + stop_gradient=True, + ) + + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": x_var_name}, + attrs={ + "shape": [40, 40], + "dtype": x_var.dtype, + "value": 1.0, + "place_type": 0, + }, + ) main_program.global_block().append_op( type='seed', inputs={}, outputs={'Out': seed_input_var}, - attrs={ - 'seed': 1, - 'force_cpu': True - }) - main_program.global_block().append_op(type='dropout', - inputs={ - 'X': x_var, - 'Seed': seed_input_var - }, - attrs={'dropout_prob': 0.}, - outputs={ - 'Out': x_out_var, - 'Mask': mask_var - }) + attrs={'seed': 1, 'force_cpu': True}, + ) + main_program.global_block().append_op( + type='dropout', + inputs={'X': x_var, 'Seed': seed_input_var}, + attrs={'dropout_prob': 0.0}, + outputs={'Out': x_out_var, 'Mask': mask_var}, + ) place = fluid.CPUPlace() if core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) @@ -340,20 +327,21 @@ class TestDropoutOpWithSeedOnCPUPlace(unittest.TestCase): x_out, mask_out = exe.run( main_program, feed={}, - fetch_list=[x_out_var.name, mask_var.name]) + fetch_list=[x_out_var.name, mask_var.name], + ) x_in_np = np.ones([40, 40]).astype("float32") np.testing.assert_allclose(x_out, x_in_np, rtol=1e-05) class TestDropoutOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) fluid.layers.dropout(x1, dropout_prob=0.5) self.assertRaises(TypeError, test_Variable) @@ -361,16 +349,15 @@ class TestDropoutOpError(unittest.TestCase): def test_dtype(): # the input dtype of dropout must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.dropout(x2, dropout_prob=0.5) self.assertRaises(TypeError, test_dtype) class TestDropoutFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -380,60 +367,64 @@ class TestDropoutFAPI(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[-1, -1], dtype="float32") - res1 = paddle.nn.functional.dropout(x=input, p=0., training=False) - res2 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='upscale_in_train') - res3 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='downscale_in_infer') - res4 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='upscale_in_train') - res5 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='downscale_in_infer') - res6 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='upscale_in_train') - res7 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='downscale_in_infer') - res8 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='upscale_in_train') - res9 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='downscale_in_infer') - res10 = paddle.nn.functional.dropout(x=input, p=1., training=True) - res11 = paddle.fluid.layers.dropout(x=input, dropout_prob=0.) - res12 = paddle.nn.functional.dropout(x=input, - p=0., - axis=(0, 1), - training=False, - mode='upscale_in_train') - - res13 = paddle.nn.functional.dropout(x=input, - p=0.7, - axis=1, - training=True, - mode='upscale_in_train') + res1 = paddle.nn.functional.dropout(x=input, p=0.0, training=False) + res2 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=True, mode='upscale_in_train' + ) + res3 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=True, mode='downscale_in_infer' + ) + res4 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=False, mode='upscale_in_train' + ) + res5 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=False, + mode='downscale_in_infer', + ) + res6 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='upscale_in_train', + ) + res7 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='downscale_in_infer', + ) + res8 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='upscale_in_train', + ) + res9 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='downscale_in_infer', + ) + res10 = paddle.nn.functional.dropout(x=input, p=1.0, training=True) + res11 = paddle.fluid.layers.dropout(x=input, dropout_prob=0.0) + res12 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=(0, 1), + training=False, + mode='upscale_in_train', + ) + + res13 = paddle.nn.functional.dropout( + x=input, p=0.7, axis=1, training=True, mode='upscale_in_train' + ) in_np = np.ones([40, 40]).astype("float32") res_np = in_np @@ -441,21 +432,36 @@ class TestDropoutFAPI(unittest.TestCase): exe = fluid.Executor(place) res_list = [ - res1, res2, res3, res4, res5, res6, res7, res8, res9, res11, - res12 + res1, + res2, + res3, + res4, + res5, + res6, + res7, + res8, + res9, + res11, + res12, ] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np, rtol=1e-05) - fetches2 = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res10]) + fetches2 = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res10], + ) np.testing.assert_allclose(fetches2[0], res_np2, rtol=1e-05) - fetches3 = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res13]) + fetches3 = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res13], + ) def test_static(self): for place in self.places: @@ -469,68 +475,99 @@ class TestDropoutFAPI(unittest.TestCase): res_np2 = np.zeros_like(in_np) input = fluid.dygraph.to_variable(in_np) - res1 = paddle.nn.functional.dropout(x=input, - p=0., - training=False) - res2 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='upscale_in_train') - res3 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='downscale_in_infer') - res4 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='upscale_in_train') - res5 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='downscale_in_infer') - res6 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='upscale_in_train') - res7 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='downscale_in_infer') - res8 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='upscale_in_train') - res9 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='downscale_in_infer') - res10 = paddle.nn.functional.dropout(x=input, - p=1., - training=True) - dropout = paddle.fluid.dygraph.Dropout(p=0, ) + res1 = paddle.nn.functional.dropout( + x=input, p=0.0, training=False + ) + res2 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=True, + mode='upscale_in_train', + ) + res3 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=True, + mode='downscale_in_infer', + ) + res4 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=False, + mode='upscale_in_train', + ) + res5 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=False, + mode='downscale_in_infer', + ) + res6 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='upscale_in_train', + ) + res7 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='downscale_in_infer', + ) + res8 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='upscale_in_train', + ) + res9 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='downscale_in_infer', + ) + res10 = paddle.nn.functional.dropout( + x=input, p=1.0, training=True + ) + dropout = paddle.fluid.dygraph.Dropout( + p=0, + ) res11 = dropout(input) - res12 = paddle.nn.functional.dropout(x=input, - p=0., - axis=(0, 1), - training=False, - mode='upscale_in_train') - res13 = paddle.nn.functional.dropout(x=input, - p=0.5, - axis=1, - training=True, - mode='upscale_in_train') + res12 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=(0, 1), + training=False, + mode='upscale_in_train', + ) + res13 = paddle.nn.functional.dropout( + x=input, + p=0.5, + axis=1, + training=True, + mode='upscale_in_train', + ) res_list = [ - res1, res2, res3, res4, res5, res6, res7, res8, res9, res11, - res12 + res1, + res2, + res3, + res4, + res5, + res6, + res7, + res8, + res9, + res11, + res12, ] for res in res_list: np.testing.assert_allclose(res.numpy(), res_np, rtol=1e-05) @@ -538,22 +575,23 @@ class TestDropoutFAPI(unittest.TestCase): class TestDropoutFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) paddle.nn.functional.dropout(x1, p=0.5) self.assertRaises(TypeError, test_Variable) def test_Variable2(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) paddle.nn.functional.dropout(x1, p=0.5, axis=0) self.assertRaises(TypeError, test_Variable2) @@ -617,7 +655,6 @@ class TestDropoutFAPIError(unittest.TestCase): class TestDropoutCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -630,16 +667,15 @@ class TestDropoutCAPI(unittest.TestCase): input_np = np.random.random([40, 40]).astype("float32") result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.Dropout(p=0.) + m = paddle.nn.Dropout(p=0.0) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - result_np, - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), result_np, rtol=1e-05 + ) class TestDropout2DFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -648,17 +684,15 @@ class TestDropout2DFAPI(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 4, 5], - dtype="float32") - res1 = paddle.nn.functional.dropout2d(x=input, - p=0., - training=False, - data_format='NCHW') - res2 = paddle.nn.functional.dropout2d(x=input, - p=0., - training=False, - data_format='NHWC') + input = fluid.data( + name="input", shape=[2, 3, 4, 5], dtype="float32" + ) + res1 = paddle.nn.functional.dropout2d( + x=input, p=0.0, training=False, data_format='NCHW' + ) + res2 = paddle.nn.functional.dropout2d( + x=input, p=0.0, training=False, data_format='NHWC' + ) in_np = np.random.random([2, 3, 4, 5]).astype("float32") res_np = in_np @@ -666,9 +700,11 @@ class TestDropout2DFAPI(unittest.TestCase): exe = fluid.Executor(place) res_list = [res1, res2] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np, rtol=1e-05) def test_static(self): @@ -682,14 +718,12 @@ class TestDropout2DFAPI(unittest.TestCase): res_np = in_np input = fluid.dygraph.to_variable(in_np) - res1 = paddle.nn.functional.dropout2d(x=input, - p=0., - training=False, - data_format='NCHW') - res2 = paddle.nn.functional.dropout2d(x=input, - p=0., - training=False, - data_format='NHWC') + res1 = paddle.nn.functional.dropout2d( + x=input, p=0.0, training=False, data_format='NCHW' + ) + res2 = paddle.nn.functional.dropout2d( + x=input, p=0.0, training=False, data_format='NHWC' + ) res_list = [res1, res2] for res in res_list: @@ -697,7 +731,6 @@ class TestDropout2DFAPI(unittest.TestCase): class TestDropout2DFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -717,7 +750,6 @@ class TestDropout2DFAPIError(unittest.TestCase): class TestDropout2DCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -730,16 +762,15 @@ class TestDropout2DCAPI(unittest.TestCase): input_np = np.random.random([2, 3, 4, 5]).astype("float32") result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.Dropout2D(p=0.) + m = paddle.nn.Dropout2D(p=0.0) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - result_np, - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), result_np, rtol=1e-05 + ) class TestDropout3DFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -748,17 +779,15 @@ class TestDropout3DFAPI(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 4, 5, 6], - dtype="float32") - res1 = paddle.nn.functional.dropout3d(x=input, - p=0., - training=False, - data_format='NCDHW') - res2 = paddle.nn.functional.dropout3d(x=input, - p=0., - training=False, - data_format='NDHWC') + input = fluid.data( + name="input", shape=[2, 3, 4, 5, 6], dtype="float32" + ) + res1 = paddle.nn.functional.dropout3d( + x=input, p=0.0, training=False, data_format='NCDHW' + ) + res2 = paddle.nn.functional.dropout3d( + x=input, p=0.0, training=False, data_format='NDHWC' + ) in_np = np.random.random([2, 3, 4, 5, 6]).astype("float32") res_np = in_np @@ -766,9 +795,11 @@ class TestDropout3DFAPI(unittest.TestCase): exe = fluid.Executor(place) res_list = [res1, res2] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np, rtol=1e-05) def test_static(self): @@ -782,14 +813,12 @@ class TestDropout3DFAPI(unittest.TestCase): res_np = in_np input = fluid.dygraph.to_variable(in_np) - res1 = paddle.nn.functional.dropout3d(x=input, - p=0., - training=False, - data_format='NCDHW') - res2 = paddle.nn.functional.dropout3d(x=input, - p=0., - training=False, - data_format='NDHWC') + res1 = paddle.nn.functional.dropout3d( + x=input, p=0.0, training=False, data_format='NCDHW' + ) + res2 = paddle.nn.functional.dropout3d( + x=input, p=0.0, training=False, data_format='NDHWC' + ) res_list = [res1, res2] for res in res_list: @@ -797,7 +826,6 @@ class TestDropout3DFAPI(unittest.TestCase): class TestDropout3DFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -817,7 +845,6 @@ class TestDropout3DFAPIError(unittest.TestCase): class TestDropout3DCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -830,16 +857,15 @@ class TestDropout3DCAPI(unittest.TestCase): input_np = np.random.random([2, 3, 4, 5, 6]).astype("float32") result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.Dropout3D(p=0.) + m = paddle.nn.Dropout3D(p=0.0) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - result_np, - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), result_np, rtol=1e-05 + ) class TestAlphaDropoutFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -849,11 +875,11 @@ class TestAlphaDropoutFAPI(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[40, 40], dtype="float32") - res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.) - res2 = paddle.nn.functional.alpha_dropout(x=input, - p=0., - training=False) - res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.) + res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.0) + res2 = paddle.nn.functional.alpha_dropout( + x=input, p=0.0, training=False + ) + res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.0) in_np = np.random.random([40, 40]).astype("float32") res_np = in_np @@ -862,13 +888,17 @@ class TestAlphaDropoutFAPI(unittest.TestCase): exe = fluid.Executor(place) res_list = [res1, res2] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np, rtol=1e-05) - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res3]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res3], + ) np.testing.assert_allclose(fetches[0], res_np3, rtol=1e-05) def test_static(self): @@ -883,11 +913,11 @@ class TestAlphaDropoutFAPI(unittest.TestCase): res_np3 = np.zeros_like(in_np) input = fluid.dygraph.to_variable(in_np) - res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.) - res2 = paddle.nn.functional.alpha_dropout(x=input, - p=0., - training=False) - res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.) + res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.0) + res2 = paddle.nn.functional.alpha_dropout( + x=input, p=0.0, training=False + ) + res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.0) res_list = [res1, res2] for res in res_list: @@ -896,14 +926,14 @@ class TestAlphaDropoutFAPI(unittest.TestCase): class TestAlphaDropoutFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) paddle.nn.functional.alpha_dropout(x1, p=0.5) self.assertRaises(TypeError, test_Variable) @@ -931,7 +961,6 @@ class TestAlphaDropoutFAPIError(unittest.TestCase): class TestAlphaDropoutCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -944,16 +973,15 @@ class TestAlphaDropoutCAPI(unittest.TestCase): input_np = np.random.random([40, 40]).astype("float32") result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.AlphaDropout(p=0.) + m = paddle.nn.AlphaDropout(p=0.0) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - result_np, - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), result_np, rtol=1e-05 + ) class TestDropoutWithDeterminateSeedGenerator(unittest.TestCase): - def setUp(self): paddle.framework.random.set_random_seed_generator('seed0', 123) paddle.framework.random.set_random_seed_generator('seed1', 123) @@ -964,19 +992,26 @@ class TestDropoutWithDeterminateSeedGenerator(unittest.TestCase): self.places.append(paddle.CUDAPlace(0)) def check_static_result(self, place): - from paddle.distributed.fleet.meta_parallel.parallel_layers.random import dropout + from paddle.distributed.fleet.meta_parallel.parallel_layers.random import ( + dropout, + ) + with static.program_guard(static.Program(), static.Program()): input = static.data(name="input", shape=[40, 40], dtype="float32") - res1 = dropout(input, - p=0.3, - training=True, - mode='upscale_in_train', - rng_name='seed0') - res2 = dropout(input, - p=0.3, - training=True, - mode='upscale_in_train', - rng_name='seed1') + res1 = dropout( + input, + p=0.3, + training=True, + mode='upscale_in_train', + rng_name='seed0', + ) + res2 = dropout( + input, + p=0.3, + training=True, + mode='upscale_in_train', + rng_name='seed1', + ) res3 = dropout(input, p=0.3) in_np = np.random.random([40, 40]).astype("float32") @@ -984,9 +1019,11 @@ class TestDropoutWithDeterminateSeedGenerator(unittest.TestCase): exe = static.Executor(place) res_list = [res1, res2] for i in range(2): - out1, out2 = exe.run(static.default_main_program(), - feed={"input": in_np}, - fetch_list=res_list) + out1, out2 = exe.run( + static.default_main_program(), + feed={"input": in_np}, + fetch_list=res_list, + ) np.testing.assert_allclose(out1, out2, rtol=1e-05) def test_static(self): @@ -995,7 +1032,6 @@ class TestDropoutWithDeterminateSeedGenerator(unittest.TestCase): class TestDropoutBackward(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -1020,7 +1056,8 @@ class TestDropoutBackward(unittest.TestCase): np.testing.assert_array_equal( input.gradient(), - self.cal_grad_downscale_in_infer(mask.numpy())) + self.cal_grad_downscale_in_infer(mask.numpy()), + ) def test_backward_downscale_in_infer_eager(self): for place in self.places: @@ -1028,12 +1065,14 @@ class TestDropoutBackward(unittest.TestCase): with _test_eager_guard(): input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = _C_ops.dropout(input, None, 0.5, False, - "downgrade_in_infer", 0, False) + out, mask = _C_ops.dropout( + input, None, 0.5, False, "downgrade_in_infer", 0, False + ) out.backward() np.testing.assert_array_equal( input.gradient(), - self.cal_grad_downscale_in_infer(mask.numpy())) + self.cal_grad_downscale_in_infer(mask.numpy()), + ) def test_backward_upscale_train(self): _enable_legacy_dygraph() @@ -1043,15 +1082,20 @@ class TestDropoutBackward(unittest.TestCase): prob = 0.5 input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = core.ops.dropout(input, 'dropout_prob', prob, - "dropout_implementation", - "upscale_in_train") + out, mask = core.ops.dropout( + input, + 'dropout_prob', + prob, + "dropout_implementation", + "upscale_in_train", + ) out.backward() - np.testing.assert_allclose(input.gradient(), - self.cal_grad_upscale_train( - mask.numpy(), prob), - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), + self.cal_grad_upscale_train(mask.numpy(), prob), + rtol=1e-05, + ) def test_backward_upscale_train_eager(self): for place in self.places: @@ -1060,14 +1104,16 @@ class TestDropoutBackward(unittest.TestCase): prob = 0.5 input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = _C_ops.dropout(input, None, 0.5, False, - "upscale_in_train", 0, False) + out, mask = _C_ops.dropout( + input, None, 0.5, False, "upscale_in_train", 0, False + ) out.backward() - np.testing.assert_allclose(input.gradient(), - self.cal_grad_upscale_train( - mask.numpy(), prob), - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), + self.cal_grad_upscale_train(mask.numpy(), prob), + rtol=1e-05, + ) def test_backward_upscale_train_2(self): _enable_legacy_dygraph() @@ -1077,15 +1123,20 @@ class TestDropoutBackward(unittest.TestCase): prob = 0.3 input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = core.ops.dropout(input, 'dropout_prob', prob, - "dropout_implementation", - "upscale_in_train") + out, mask = core.ops.dropout( + input, + 'dropout_prob', + prob, + "dropout_implementation", + "upscale_in_train", + ) out.backward() - np.testing.assert_allclose(input.gradient(), - self.cal_grad_upscale_train( - mask.numpy(), prob), - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), + self.cal_grad_upscale_train(mask.numpy(), prob), + rtol=1e-05, + ) def test_backward_upscale_train_2_eager(self): for place in self.places: @@ -1095,24 +1146,28 @@ class TestDropoutBackward(unittest.TestCase): prob = 0.3 input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = _C_ops.dropout(input, None, 0.3, False, - "upscale_in_train", 0, False) + out, mask = _C_ops.dropout( + input, None, 0.3, False, "upscale_in_train", 0, False + ) out.backward() - np.testing.assert_allclose(input.gradient(), - self.cal_grad_upscale_train( - mask.numpy(), prob), - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), + self.cal_grad_upscale_train(mask.numpy(), prob), + rtol=1e-05, + ) class TestDropOutWithProbTensor(unittest.TestCase): - def setUp(self): self.init_info() self.input = np.random.random(self.shape).astype("float32") - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def init_info(self): self.shape = [10, 10] @@ -1151,21 +1206,18 @@ class TestDropOutWithProbTensor(unittest.TestCase): class TestDropOut2DWithProbTensor(TestDropOutWithProbTensor): - def init_info(self): self.shape = [2, 3, 10, 10] self.api = paddle.nn.functional.dropout2d class TestDropOut3DWithProbTensor(TestDropOutWithProbTensor): - def init_info(self): self.shape = [2, 3, 8, 8, 8] self.api = paddle.nn.functional.dropout3d class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -1188,8 +1240,16 @@ class TestRandomValue(unittest.TestCase): self.assertEqual(np.sum(index2), 12872777397) self.assertEqual(np.sum(out), 16778744.0) expect = [ - 0.6914956, 0.5294584, 0.19032137, 0.6996228, 0.3338527, 0.8442094, - 0.96965003, 1.1726775, 0., 0.28037727 + 0.6914956, + 0.5294584, + 0.19032137, + 0.6996228, + 0.3338527, + 0.8442094, + 0.96965003, + 1.1726775, + 0.0, + 0.28037727, ] np.testing.assert_allclose(out[10, 100, 500:510], expect, rtol=1e-05) @@ -1201,8 +1261,16 @@ class TestRandomValue(unittest.TestCase): self.assertEqual(np.sum(index2), 8582219962) self.assertEqual(np.sum(out), 16778396.563660286) expect = [ - 1.28587354, 0.15563703, 0., 0.28799703, 0., 0., 0., 0.54964, - 0.51355682, 0.33818988 + 1.28587354, + 0.15563703, + 0.0, + 0.28799703, + 0.0, + 0.0, + 0.0, + 0.54964, + 0.51355682, + 0.33818988, ] np.testing.assert_allclose(out[20, 100, 500:510], expect, rtol=1e-05) @@ -1212,7 +1280,7 @@ class TestRandomValue(unittest.TestCase): self.assertEqual(np.sum(index0), 130086900) self.assertEqual(np.sum(index1), 4291190105) self.assertEqual(np.sum(index2), 4292243807) - expect = [0., 0., 0., 0., 0., 0., 0., 0., 4., 4.] + expect = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 4.0, 4.0] np.testing.assert_allclose(out[0, 100, 500:510], expect, rtol=1e-05) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py b/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py index 30109d430be..0d435691560 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py @@ -22,46 +22,51 @@ from paddle.fluid.framework import _test_eager_guard class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=False, - dtype='float32', - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=False, + dtype='float32', + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=param_attr, - bias_attr=bias_attr, - use_cudnn=use_cudnn, - dtype=dtype, - act=act) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=param_attr, + bias_attr=bias_attr, + use_cudnn=use_cudnn, + dtype=dtype, + act=act, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -70,39 +75,45 @@ class SimpleImgConvPool(fluid.dygraph.Layer): class MNIST(fluid.dygraph.Layer): - def __init__(self, dtype="float32"): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(num_channels=3, - num_filters=20, - filter_size=5, - pool_size=2, - pool_stride=2, - act="relu", - dtype=dtype, - use_cudnn=True) - - self._simple_img_conv_pool_2 = SimpleImgConvPool(num_channels=20, - num_filters=50, - filter_size=5, - pool_size=2, - pool_stride=2, - act="relu", - dtype=dtype, - use_cudnn=True) + self._simple_img_conv_pool_1 = SimpleImgConvPool( + num_channels=3, + num_filters=20, + filter_size=5, + pool_size=2, + pool_stride=2, + act="relu", + dtype=dtype, + use_cudnn=True, + ) + + self._simple_img_conv_pool_2 = SimpleImgConvPool( + num_channels=20, + num_filters=50, + filter_size=5, + pool_size=2, + pool_stride=2, + act="relu", + dtype=dtype, + use_cudnn=True, + ) self.pool_2_shape = 50 * 53 * 53 SIZE = 10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 self._linear = Linear( self.pool_2_shape, 10, param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer(loc=0.0, - scale=scale)), + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), act="softmax", - dtype=dtype) + dtype=dtype, + ) def forward(self, inputs, label): x = self._simple_img_conv_pool_1(inputs) @@ -115,7 +126,6 @@ class MNIST(fluid.dygraph.Layer): class TestMnist(unittest.TestCase): - def func_mnist_fp16(self): if not fluid.is_compiled_with_cuda(): return diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_mode_of_unittest.py b/python/paddle/fluid/tests/unittests/test_dygraph_mode_of_unittest.py index ce8bb122f63..8da813c00ef 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_mode_of_unittest.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_mode_of_unittest.py @@ -17,11 +17,10 @@ import paddle class TestDygraphModeOfUnittest(unittest.TestCase): - def test_dygraph_mode(self): self.assertTrue( paddle.in_dynamic_mode(), - 'Default Mode of Unittest should be dygraph mode, but get static mode.' + 'Default Mode of Unittest should be dygraph mode, but get static mode.', ) diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py b/python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py index 559424c7fb4..4486da7eb30 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py @@ -27,43 +27,48 @@ SEED = 123123111 class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=False, - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=False, + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=None, - bias_attr=None, - use_cudnn=use_cudnn) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=None, + bias_attr=None, + use_cudnn=use_cudnn, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -72,33 +77,30 @@ class SimpleImgConvPool(fluid.dygraph.Layer): class MNIST(fluid.dygraph.Layer): - def __init__(self): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(1, - 20, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_1 = SimpleImgConvPool( + 1, 20, 5, 2, 2, act="relu" + ) - self._simple_img_conv_pool_2 = SimpleImgConvPool(20, - 50, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_2 = SimpleImgConvPool( + 20, 50, 5, 2, 2, act="relu" + ) self.pool_2_shape = 50 * 4 * 4 - SIZE = 100 #10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 - self._fc = Linear(self.pool_2_shape, - SIZE, - param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=scale)), - act="softmax") + SIZE = 100 # 10 + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 + self._fc = Linear( + self.pool_2_shape, + SIZE, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + act="softmax", + ) def forward(self, inputs): x = self._simple_img_conv_pool_1(inputs) @@ -109,7 +111,6 @@ class MNIST(fluid.dygraph.Layer): class TestDygraphMultiForward(unittest.TestCase): - def test_mnist_forward_float32(self): epoch_num = 1 @@ -117,21 +118,25 @@ class TestDygraphMultiForward(unittest.TestCase): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) mnist = MNIST() - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=mnist.parameters()) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=mnist.parameters() + ) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) dy_param_init_value = {} mnist.eval() for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader()): - dy_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(128, 1) + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(128, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -150,18 +155,21 @@ class TestDygraphMultiForward(unittest.TestCase): with new_program_scope(): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) mnist = MNIST() sgd = SGDOptimizer(learning_rate=1e-3) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) @@ -173,38 +181,42 @@ class TestDygraphMultiForward(unittest.TestCase): for param in mnist.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader()): - static_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([128, 1]) + static_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([128, 1]) + ) fetch_list = [avg_loss.name] - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_out = out[0] - np.testing.assert_allclose(dy_x_data.all(), - static_x_data.all(), - rtol=1e-05) + np.testing.assert_allclose( + dy_x_data.all(), static_x_data.all(), rtol=1e-05 + ) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py b/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py index 9c3e6e1529e..b865a1b1799 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py @@ -21,14 +21,13 @@ from paddle.nn.utils import spectral_norm class TestDygraphSpectralNorm(unittest.TestCase): - def setUp(self): self.init_test_case() self.set_data() def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 12, 12]], ) + self.data_desc = (['x', [2, 12, 12]],) self.n_power_iterations = 1 self.eps = 1e-12 self.dim = None @@ -38,8 +37,9 @@ class TestDygraphSpectralNorm(unittest.TestCase): for desc in self.data_desc: data_name = desc[0] data_shape = desc[1] - data_value = np.random.random(size=[self.batch_size] + - data_shape).astype('float32') + data_value = np.random.random( + size=[self.batch_size] + data_shape + ).astype('float32') self.data[data_name] = data_value def spectral_normalize(self, weight, u, v, dim, power_iters, eps): @@ -68,18 +68,27 @@ class TestDygraphSpectralNorm(unittest.TestCase): linear = paddle.nn.Conv2D(2, 1, 3) before_weight = linear.weight.numpy().copy() if self.dim == None: - if isinstance(linear, (nn.Conv1DTranspose, nn.Conv2DTranspose, - nn.Conv3DTranspose, nn.Linear)): + if isinstance( + linear, + ( + nn.Conv1DTranspose, + nn.Conv2DTranspose, + nn.Conv3DTranspose, + nn.Linear, + ), + ): self.dim = 1 else: self.dim = 0 else: self.dim = (self.dim + len(before_weight)) % len(before_weight) - sn = spectral_norm(linear, - n_power_iterations=self.n_power_iterations, - eps=self.eps, - dim=self.dim) + sn = spectral_norm( + linear, + n_power_iterations=self.n_power_iterations, + eps=self.eps, + dim=self.dim, + ) u = sn.weight_u.numpy().copy() v = sn.weight_v.numpy().copy() outputs = [] @@ -88,52 +97,47 @@ class TestDygraphSpectralNorm(unittest.TestCase): outputs.append(output.numpy()) self.actual_outputs = linear.weight.numpy() - expect_output = self.spectral_normalize(before_weight, u, v, self.dim, - self.n_power_iterations, - self.eps) + expect_output = self.spectral_normalize( + before_weight, u, v, self.dim, self.n_power_iterations, self.eps + ) for expect, actual in zip(expect_output, self.actual_outputs): - np.testing.assert_allclose(np.array(actual), - np.array(expect), - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + np.array(actual), np.array(expect), rtol=1e-05, atol=0.001 + ) class TestDygraphWeightNormCase(TestDygraphSpectralNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.n_power_iterations = 1 self.eps = 1e-12 self.dim = None class TestDygraphWeightNormWithIterations(TestDygraphSpectralNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.n_power_iterations = 2 self.eps = 1e-12 self.dim = None class TestDygraphWeightNormWithDim(TestDygraphSpectralNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.n_power_iterations = 1 self.eps = 1e-12 self.dim = 1 class TestDygraphWeightNormWithEps(TestDygraphSpectralNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.n_power_iterations = 1 self.eps = 1e-10 self.dim = None diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py b/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py index 270a1afe04b..ebae140f88c 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py @@ -22,14 +22,13 @@ from paddle.nn.utils import weight_norm, remove_weight_norm class TestDygraphWeightNorm(unittest.TestCase): - def setUp(self): self.init_test_case() self.set_data() def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = None def set_data(self): @@ -37,8 +36,9 @@ class TestDygraphWeightNorm(unittest.TestCase): for desc in self.data_desc: data_name = desc[0] data_shape = desc[1] - data_value = np.random.random(size=[self.batch_size] + - data_shape).astype('float32') + data_value = np.random.random( + size=[self.batch_size] + data_shape + ).astype('float32') self.data[data_name] = data_value def norm_except_dim(self, w, dim=None): @@ -88,14 +88,22 @@ class TestDygraphWeightNorm(unittest.TestCase): perm[dim] = 0 p_transposed = np.transpose(v, perm) transposed_shape = p_transposed.shape - transposed_shape_numel = reduce(lambda x, y: x * y, - transposed_shape) + transposed_shape_numel = reduce( + lambda x, y: x * y, transposed_shape + ) p_matrix = np.reshape( - p_transposed, (p_transposed.shape[0], - transposed_shape_numel // p_transposed.shape[0])) - v_norm = v / np.expand_dims(np.expand_dims( - np.linalg.norm(p_matrix, axis=1, keepdims=True), axis=0), - axis=(ndims - 1)) + p_transposed, + ( + p_transposed.shape[0], + transposed_shape_numel // p_transposed.shape[0], + ), + ) + v_norm = v / np.expand_dims( + np.expand_dims( + np.linalg.norm(p_matrix, axis=1, keepdims=True), axis=0 + ), + axis=(ndims - 1), + ) v_norm = np.reshape(v_norm, transposed_shape) v_norm = np.transpose(v_norm, perm) g = np.squeeze(g, axis=1) @@ -103,9 +111,10 @@ class TestDygraphWeightNorm(unittest.TestCase): eaxis = 2 elif dim == 2: eaxis = 1 - g_mul = np.expand_dims(np.expand_dims(np.expand_dims(g, axis=0), - axis=eaxis), - axis=(ndims - 1)) + g_mul = np.expand_dims( + np.expand_dims(np.expand_dims(g, axis=0), axis=eaxis), + axis=(ndims - 1), + ) w = g_mul * v_norm return g, v @@ -129,52 +138,46 @@ class TestDygraphWeightNorm(unittest.TestCase): expect_output = self.weight_normalize(before_weight, self.dim) for expect, actual in zip(expect_output, self.actual_outputs): - np.testing.assert_allclose(np.array(actual), - expect, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + np.array(actual), expect, rtol=1e-05, atol=0.001 + ) class TestDygraphWeightNormCase1(TestDygraphWeightNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = 0 class TestDygraphWeightNormCase2(TestDygraphWeightNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = 1 class TestDygraphWeightNormCase3(TestDygraphWeightNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = 3 class TestDygraphWeightNormCase4(TestDygraphWeightNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = -3 class TestDygraphRemoveWeightNorm(unittest.TestCase): - def setUp(self): self.init_test_case() def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = None def test_check_output(self): @@ -184,10 +187,9 @@ class TestDygraphRemoveWeightNorm(unittest.TestCase): wn = weight_norm(linear, dim=self.dim) rwn = remove_weight_norm(linear) after_weight = linear.weight - np.testing.assert_allclose(before_weight.numpy(), - after_weight.numpy(), - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + before_weight.numpy(), after_weight.numpy(), rtol=1e-05, atol=0.001 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py index de03749769d..e4f0a3a173a 100644 --- a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py @@ -29,20 +29,21 @@ numpy.random.seed(2020) class TestDynamicRNN(unittest.TestCase): - def setUp(self): self.word_dict_len = 5147 self.BATCH_SIZE = 2 reader = fake_imdb_reader(self.word_dict_len, self.BATCH_SIZE * 100) self.train_data = paddle.batch(reader, batch_size=self.BATCH_SIZE) - def _train(self, - main_program, - startup_program, - feed_list, - fetch_list, - is_nested=False, - max_iters=1): + def _train( + self, + main_program, + startup_program, + feed_list, + fetch_list, + is_nested=False, + max_iters=1, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_program) @@ -50,10 +51,12 @@ class TestDynamicRNN(unittest.TestCase): data = next(self.train_data()) for iter_id in range(max_iters): - fetch_outs = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=fetch_list, - return_numpy=False) + fetch_outs = exe.run( + main_program, + feed=feeder.feed(data), + fetch_list=fetch_list, + return_numpy=False, + ) if len(fetch_list) == 3: rnn_in_seq = fetch_outs[0] rnn_out_seq = fetch_outs[1] @@ -65,9 +68,9 @@ class TestDynamicRNN(unittest.TestCase): loss_i = numpy.array(fetch_outs[2]) elif len(fetch_list) == 1: loss_i = numpy.array(fetch_outs[0]) - #print(loss_i) + # print(loss_i) - self.assertEqual((1, ), loss_i.shape) + self.assertEqual((1,), loss_i.shape) self.assertFalse(numpy.isnan(loss_i)) if iter_id == 0: loss_0 = loss_i @@ -81,13 +84,12 @@ class TestDynamicRNN(unittest.TestCase): startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - sentence = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=1) - sent_emb = fluid.layers.embedding(input=sentence, - size=[self.word_dict_len, 32], - dtype='float32') + sentence = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=1 + ) + sent_emb = fluid.layers.embedding( + input=sentence, size=[self.word_dict_len, 32], dtype='float32' + ) rank_table = lod_rank_table(x=sent_emb) sent_emb_array = lod_tensor_to_array(x=sent_emb, table=rank_table) @@ -100,7 +102,8 @@ class TestDynamicRNN(unittest.TestCase): input=fluid.layers.array_read(array=sent_emb_array, i=i), value=0, shape=[-1, 100], - dtype='float32') + dtype='float32', + ) boot_mem.stop_gradient = False mem_array = fluid.layers.array_write(x=boot_mem, i=i) @@ -127,8 +130,9 @@ class TestDynamicRNN(unittest.TestCase): logits = fluid.layers.fc(input=last, size=1, act=None) label = fluid.layers.data(name='label', shape=[1], dtype='float32') - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, - label=label) + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logits, label=label + ) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(1e-4) sgd.minimize(loss=loss) @@ -136,12 +140,14 @@ class TestDynamicRNN(unittest.TestCase): # Check for lod_level set in compile-time. self.assertEqual(sent_emb.lod_level, result_all_timesteps.lod_level) - self._train(main_program=main_program, - startup_program=startup_program, - feed_list=[sentence, label], - fetch_list=[sent_emb, result_all_timesteps, loss], - is_nested=False, - max_iters=1) + self._train( + main_program=main_program, + startup_program=startup_program, + feed_list=[sentence, label], + fetch_list=[sent_emb, result_all_timesteps, loss], + is_nested=False, + max_iters=1, + ) def test_train_dynamic_rnn(self): main_program = fluid.Program() @@ -149,13 +155,12 @@ class TestDynamicRNN(unittest.TestCase): main_program.random_seed = 10 startup_program.random_seed = 10 with fluid.program_guard(main_program, startup_program): - sentence = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=1) - sent_emb = fluid.layers.embedding(input=sentence, - size=[self.word_dict_len, 32], - dtype='float32') + sentence = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=1 + ) + sent_emb = fluid.layers.embedding( + input=sentence, size=[self.word_dict_len, 32], dtype='float32' + ) drnn = fluid.layers.DynamicRNN() with drnn.block(): @@ -170,8 +175,9 @@ class TestDynamicRNN(unittest.TestCase): logits = fluid.layers.fc(input=last, size=1, act=None) label = fluid.layers.data(name='label', shape=[1], dtype='float32') - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, - label=label) + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logits, label=label + ) loss = paddle.mean(loss) sgd = fluid.optimizer.Adam(1e-3) sgd.minimize(loss=loss) @@ -179,12 +185,14 @@ class TestDynamicRNN(unittest.TestCase): # Check for lod_level set in compile-time. self.assertEqual(sent_emb.lod_level, drnn_result.lod_level) - self._train(main_program=main_program, - startup_program=startup_program, - feed_list=[sentence, label], - fetch_list=[sent_emb, drnn_result, loss], - is_nested=False, - max_iters=100) + self._train( + main_program=main_program, + startup_program=startup_program, + feed_list=[sentence, label], + fetch_list=[sent_emb, drnn_result, loss], + is_nested=False, + max_iters=100, + ) def _fake_reader(self): seq_len, label = [[2, 2]], [0, 1] @@ -205,30 +213,30 @@ class TestDynamicRNN(unittest.TestCase): main_program.random_seed = 10 startup_program.random_seed = 10 with fluid.program_guard(main_program, startup_program): - sentence = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=2) - label = fluid.layers.data(name='label', - shape=[1], - dtype='float32', - lod_level=1) + sentence = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=2 + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='float32', lod_level=1 + ) drnn0 = fluid.layers.DynamicRNN() with drnn0.block(): in_0 = drnn0.step_input(sentence) assert in_0.lod_level == 1, "the lod level of in_ should be 1" - sentence_emb = fluid.layers.embedding(input=in_0, - size=[len(word_dict), 32], - dtype='float32') - out_0 = fluid.layers.fc(input=sentence_emb, - size=100, - act='tanh') + sentence_emb = fluid.layers.embedding( + input=in_0, size=[len(word_dict), 32], dtype='float32' + ) + out_0 = fluid.layers.fc( + input=sentence_emb, size=100, act='tanh' + ) drnn1 = fluid.layers.DynamicRNN() with drnn1.block(): in_1 = drnn1.step_input(out_0) - assert in_1.lod_level == 0, "the lod level of in_1 should be 0" + assert ( + in_1.lod_level == 0 + ), "the lod level of in_1 should be 0" out_1 = fluid.layers.fc(input=[in_1], size=100, act='tanh') drnn1.output(out_1) @@ -238,20 +246,23 @@ class TestDynamicRNN(unittest.TestCase): last = drnn0() logits = fluid.layers.fc(input=last, size=1, act=None) - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, - label=label) + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logits, label=label + ) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(1e-3) sgd.minimize(loss=loss) train_data_orig = self.train_data self.train_data = paddle.batch(self._fake_reader, batch_size=2) - self._train(main_program=main_program, - startup_program=startup_program, - feed_list=[sentence, label], - fetch_list=[loss], - is_nested=True, - max_iters=100) + self._train( + main_program=main_program, + startup_program=startup_program, + feed_list=[sentence, label], + fetch_list=[loss], + is_nested=True, + max_iters=100, + ) self.train_data = train_data_orig # this unit test is just used to the two layer nested dyn_rnn. @@ -264,14 +275,12 @@ class TestDynamicRNN(unittest.TestCase): main_program.random_seed = 10 startup_program.random_seed = 10 with fluid.program_guard(main_program, startup_program): - sentence = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=2) - label = fluid.layers.data(name='label', - shape=[1], - dtype='float32', - lod_level=1) + sentence = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=2 + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='float32', lod_level=1 + ) drnn0 = fluid.layers.DynamicRNN() with drnn0.block(): @@ -279,14 +288,19 @@ class TestDynamicRNN(unittest.TestCase): sentence_emb = fluid.layers.embedding( input=in_0, size=[len(word_dict), hidden_size], - dtype='float32') - input_forward_proj = fluid.layers.fc(input=sentence_emb, - size=hidden_size * 4, - act=None, - bias_attr=False) - forward, _ = fluid.layers.dynamic_lstm(input=input_forward_proj, - size=hidden_size * 4, - use_peepholes=False) + dtype='float32', + ) + input_forward_proj = fluid.layers.fc( + input=sentence_emb, + size=hidden_size * 4, + act=None, + bias_attr=False, + ) + forward, _ = fluid.layers.dynamic_lstm( + input=input_forward_proj, + size=hidden_size * 4, + use_peepholes=False, + ) drnn1 = fluid.layers.DynamicRNN() with drnn1.block(): @@ -299,33 +313,34 @@ class TestDynamicRNN(unittest.TestCase): last = drnn0() logits = fluid.layers.fc(input=last, size=1, act=None) - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, - label=label) + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logits, label=label + ) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(1e-3) sgd.minimize(loss=loss) train_data_orig = self.train_data self.train_data = paddle.batch(self._fake_reader, batch_size=2) - self._train(main_program=main_program, - startup_program=startup_program, - feed_list=[sentence, label], - fetch_list=[loss], - is_nested=True, - max_iters=100) + self._train( + main_program=main_program, + startup_program=startup_program, + feed_list=[sentence, label], + fetch_list=[loss], + is_nested=True, + max_iters=100, + ) self.train_data = train_data_orig class TestDynamicRNNErrors(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): init = fluid.layers.zeros(shape=[1], dtype='float32') shape = 'shape' - sentence = fluid.data(name='sentence', - shape=[None, 32], - dtype='float32', - lod_level=1) + sentence = fluid.data( + name='sentence', shape=[None, 32], dtype='float32', lod_level=1 + ) # The type of Input(shape) in API(memory) must be list or tuple def input_shape_type_of_memory(): @@ -341,9 +356,9 @@ class TestDynamicRNNErrors(unittest.TestCase): with drnn.block(): word = drnn.step_input(sentence) memory = drnn.memory(shape=[10], dtype='float32', value=0) - hidden = fluid.layers.fc(input=[word, memory], - size=10, - act='tanh') + hidden = fluid.layers.fc( + input=[word, memory], size=10, act='tanh' + ) out = numpy.ones(1).astype('float32') drnn.update_memory(ex_mem=memory, new_mem=hidden) drnn.output(hidden, out) diff --git a/python/paddle/fluid/tests/unittests/test_dynamic_rnn_stop_gradient.py b/python/paddle/fluid/tests/unittests/test_dynamic_rnn_stop_gradient.py index 0a186b0c8af..3922f6a8c22 100644 --- a/python/paddle/fluid/tests/unittests/test_dynamic_rnn_stop_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_dynamic_rnn_stop_gradient.py @@ -24,16 +24,15 @@ def build_and_run_program(place, batch_size, beam_size, stop_gradient=False): np.random.seed(2) x = layers.assign( - np.random.rand(batch_size, beam_size, 32).astype("float32")) + np.random.rand(batch_size, beam_size, 32).astype("float32") + ) indices = fluid.data(shape=[None, beam_size], dtype="int64", name="indices") - step_idx = layers.fill_constant(shape=[1], - dtype="int64", - value=0, - force_cpu=True) - max_len = layers.fill_constant(shape=[1], - dtype="int64", - value=10, - force_cpu=True) + step_idx = layers.fill_constant( + shape=[1], dtype="int64", value=0, force_cpu=True + ) + max_len = layers.fill_constant( + shape=[1], dtype="int64", value=10, force_cpu=True + ) cond = layers.less_than(x=step_idx, y=max_len) while_op = layers.While(cond) scores = layers.array_write(x, step_idx) @@ -44,7 +43,8 @@ def build_and_run_program(place, batch_size, beam_size, stop_gradient=False): bs.stop_gradient = stop_gradient batch_pos = layers.expand( layers.unsqueeze(layers.range(0, bs, 1, dtype=bs.dtype), [1]), - [1, beam_size]) + [1, beam_size], + ) topk_coordinates = layers.stack([batch_pos, indices], axis=2) topk_coordinates.stop_gradient = stop_gradient score = layers.gather_nd(x, topk_coordinates) @@ -58,17 +58,15 @@ def build_and_run_program(place, batch_size, beam_size, stop_gradient=False): opt = fluid.optimizer.Adam(0.01) opt.minimize(loss) exe = fluid.Executor(place) - data = np.random.random_integers(low=0, - high=beam_size - 1, - size=(batch_size, - beam_size)).astype("int64") - loss_val, = exe.run(feed={"indices": data}, fetch_list=[loss]) + data = np.random.random_integers( + low=0, high=beam_size - 1, size=(batch_size, beam_size) + ).astype("int64") + (loss_val,) = exe.run(feed={"indices": data}, fetch_list=[loss]) return loss_val class TestDynRNNStopGradient(unittest.TestCase): - def setUp(self): self.batch_size = 20 self.beam_size = 64 @@ -76,10 +74,12 @@ class TestDynRNNStopGradient(unittest.TestCase): def run_main(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): with fluid.scope_guard(fluid.Scope()): - value1 = build_and_run_program(place, self.batch_size, - self.beam_size, False) - value2 = build_and_run_program(place, self.batch_size, - self.beam_size, True) + value1 = build_and_run_program( + place, self.batch_size, self.beam_size, False + ) + value2 = build_and_run_program( + place, self.batch_size, self.beam_size, True + ) np.testing.assert_array_equal(value1, value2) diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py index 7b7921d91c2..01cfedf063e 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py @@ -22,7 +22,6 @@ from decorator_helper import prog_scope class Memory(object): - def __init__(self, shape, dtype='float32'): self.ex = np.zeros(shape=shape, dtype=dtype) self.cur = None @@ -45,7 +44,6 @@ class Memory(object): class Output(object): - def __init__(self): self.outs = [] @@ -60,7 +58,6 @@ class Output(object): class BaseRNN(object): - def __init__(self, ins, mems, params, outs, num_seq=5, max_seq_len=15): self.num_seq = num_seq self.inputs = collections.defaultdict(list) @@ -159,8 +156,10 @@ class BaseRNN(object): def get_numeric_gradient_of_param(self, param_name, delta=0.001): p = self.params[param_name] if len(p.shape) != 2: - raise ValueError("Not support get numeric gradient of an parameter," - " which is not matrix") + raise ValueError( + "Not support get numeric gradient of an parameter," + " which is not matrix" + ) g = np.zeros(shape=p.shape, dtype=p.dtype) for i in range(p.shape[0]): @@ -174,10 +173,9 @@ class BaseRNN(object): g[i][j] = (pos - neg) / (delta * 2) return g - def get_numeric_gradient_of_input(self, - input_name, - delta=0.001, - return_one_tensor=True): + def get_numeric_gradient_of_input( + self, input_name, delta=0.001, return_one_tensor=True + ): ipt = self.inputs[input_name] grad = [] @@ -213,7 +211,6 @@ class BaseRNN(object): class SeedFixedTestCase(unittest.TestCase): - @classmethod def setUpClass(cls): """Fix random seeds to remove randomness from tests""" @@ -238,17 +235,18 @@ class TestSimpleMul(SeedFixedTestCase): OUT_NAME = 'Out' class SimpleMul(BaseRNN): - def __init__(self): base = TestSimpleMul - super(base.SimpleMul, - self).__init__({base.DATA_NAME: { - 'shape': [base.DATA_WIDTH] - }}, {}, { - base.PARAM_NAME: { - 'shape': [base.DATA_WIDTH, base.HIDDEN_WIDTH] - } - }, [base.OUT_NAME]) + super(base.SimpleMul, self).__init__( + {base.DATA_NAME: {'shape': [base.DATA_WIDTH]}}, + {}, + { + base.PARAM_NAME: { + 'shape': [base.DATA_WIDTH, base.HIDDEN_WIDTH] + } + }, + [base.OUT_NAME], + ) def step(self, X, W, Out): Out.out(np.matmul(X, W)) @@ -258,19 +256,21 @@ class TestSimpleMul(SeedFixedTestCase): @prog_scope() def test_forward_backward(self): py_rnn = TestSimpleMul.SimpleMul() - dat = fluid.layers.data(name=self.DATA_NAME, - shape=[self.DATA_WIDTH], - lod_level=1) + dat = fluid.layers.data( + name=self.DATA_NAME, shape=[self.DATA_WIDTH], lod_level=1 + ) dat.stop_gradient = False rnn = fluid.layers.DynamicRNN() with rnn.block(): d = rnn.step_input(dat) - o = fluid.layers.fc(input=d, - param_attr=self.PARAM_NAME, - bias_attr=False, - size=self.HIDDEN_WIDTH, - act=None) + o = fluid.layers.fc( + input=d, + param_attr=self.PARAM_NAME, + bias_attr=False, + size=self.HIDDEN_WIDTH, + act=None, + ) rnn.output(o) out = rnn() @@ -283,18 +283,24 @@ class TestSimpleMul(SeedFixedTestCase): out, w_g, i_g = list( map( np.array, - exe.run(feed=py_rnn.to_feed(cpu), - fetch_list=[ - out, self.PARAM_NAME + "@GRAD", - self.DATA_NAME + "@GRAD" - ], - return_numpy=False))) + exe.run( + feed=py_rnn.to_feed(cpu), + fetch_list=[ + out, + self.PARAM_NAME + "@GRAD", + self.DATA_NAME + "@GRAD", + ], + return_numpy=False, + ), + ) + ) out_by_python = py_rnn.exe()[self.OUT_NAME] np.testing.assert_allclose(out, out_by_python, rtol=1e-05) w_g_num = py_rnn.get_numeric_gradient_of_param(self.PARAM_NAME) np.testing.assert_allclose(w_g_num, w_g, rtol=0.05) i_g_num = py_rnn.get_numeric_gradient_of_input( - input_name=self.DATA_NAME) + input_name=self.DATA_NAME + ) i_g_num = i_g_num.reshape(i_g.shape) np.testing.assert_allclose(i_g_num, i_g, rtol=0.05) @@ -306,23 +312,24 @@ class TestSimpleMulWithMemory(SeedFixedTestCase): PARAM_NAME = 'W' class SimpleMulWithMemory(BaseRNN): - def __init__(self): super(TestSimpleMulWithMemory.SimpleMulWithMemory, self).__init__( { TestSimpleMulWithMemory.DATA_NAME: { 'shape': [TestSimpleMulWithMemory.DATA_WIDTH] } - }, {'Mem': { - 'shape': [TestSimpleMulWithMemory.HIDDEN_WIDTH] - }}, { + }, + {'Mem': {'shape': [TestSimpleMulWithMemory.HIDDEN_WIDTH]}}, + { TestSimpleMulWithMemory.PARAM_NAME: { 'shape': [ TestSimpleMulWithMemory.DATA_WIDTH, - TestSimpleMulWithMemory.HIDDEN_WIDTH + TestSimpleMulWithMemory.HIDDEN_WIDTH, ] } - }, ['Out']) + }, + ['Out'], + ) def step(self, X, Mem, W, Out): o = np.matmul(X, W) @@ -337,19 +344,21 @@ class TestSimpleMulWithMemory(SeedFixedTestCase): @prog_scope() def test_forward_backward(self): py_rnn = TestSimpleMulWithMemory.SimpleMulWithMemory() - data = fluid.layers.data(name=self.DATA_NAME, - shape=[self.DATA_WIDTH], - lod_level=1) + data = fluid.layers.data( + name=self.DATA_NAME, shape=[self.DATA_WIDTH], lod_level=1 + ) data.stop_gradient = False rnn = fluid.layers.DynamicRNN() with rnn.block(): d = rnn.step_input(data) mem = rnn.memory(value=0.0, shape=[self.HIDDEN_WIDTH]) - hidden = fluid.layers.fc(input=d, - size=self.HIDDEN_WIDTH, - param_attr=self.PARAM_NAME, - bias_attr=False, - act=None) + hidden = fluid.layers.fc( + input=d, + size=self.HIDDEN_WIDTH, + param_attr=self.PARAM_NAME, + bias_attr=False, + act=None, + ) o = fluid.layers.elementwise_add(x=hidden, y=mem) rnn.update_memory(mem, o) rnn.output(o) @@ -365,13 +374,18 @@ class TestSimpleMulWithMemory(SeedFixedTestCase): last_np, w_g, i_g = list( map( np.array, - exe.run(feed=feed, - fetch_list=[ - last, self.PARAM_NAME + "@GRAD", - self.DATA_NAME + "@GRAD" - ], - return_numpy=False))) - last_by_py, = list(py_rnn.exe().values()) + exe.run( + feed=feed, + fetch_list=[ + last, + self.PARAM_NAME + "@GRAD", + self.DATA_NAME + "@GRAD", + ], + return_numpy=False, + ), + ) + ) + (last_by_py,) = list(py_rnn.exe().values()) w_g_num = py_rnn.get_numeric_gradient_of_param(self.PARAM_NAME) np.testing.assert_allclose(last_np, last_by_py, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py index 3bbf9ff3266..2f0a9924756 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py @@ -27,7 +27,6 @@ np.random.seed(1) class TestDyRnnStaticInput(unittest.TestCase): - def setUp(self): self._delta = 0.005 self._max_sequence_len = 3 @@ -58,14 +57,14 @@ class TestDyRnnStaticInput(unittest.TestCase): self.static_input_tensor.set(self.static_input_data, self.place) def fetch_value(self, var): - fetch_outs = self.exe.run(feed={ - 'x_tensor': - self.x_tensor, - 'static_input_tensor': - self.static_input_tensor - }, - fetch_list=[var], - return_numpy=False) + fetch_outs = self.exe.run( + feed={ + 'x_tensor': self.x_tensor, + 'static_input_tensor': self.static_input_tensor, + }, + fetch_list=[var], + return_numpy=False, + ) return self._lodtensor_to_ndarray(fetch_outs[0]) def _lodtensor_to_ndarray(self, lod_tensor): @@ -76,24 +75,28 @@ class TestDyRnnStaticInput(unittest.TestCase): return ndarray, lod_tensor.recursive_sequence_lengths() def build_graph(self, only_forward=False): - x_tensor = fluid.layers.data(name='x_tensor', - shape=[self.x_tensor_dim], - dtype='float32', - lod_level=1) + x_tensor = fluid.layers.data( + name='x_tensor', + shape=[self.x_tensor_dim], + dtype='float32', + lod_level=1, + ) x_tensor.stop_gradient = False static_input_tensor = fluid.layers.data( name='static_input_tensor', shape=[self.static_input_tensor_dim], dtype='float32', - lod_level=1) + lod_level=1, + ) static_input_tensor.stop_gradient = False if only_forward: static_input_out_array = self._program.global_block().create_var( name='static_input_out_array', type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype='float32') + dtype='float32', + ) static_input_out_array.stop_gradient = True rnn = fluid.layers.DynamicRNN() @@ -101,25 +104,30 @@ class TestDyRnnStaticInput(unittest.TestCase): step_x = rnn.step_input(x_tensor) step_static_input = rnn.static_input(static_input_tensor) if only_forward: - fluid.layers.array_write(x=step_static_input, - i=rnn.step_idx, - array=static_input_out_array) - last = fluid.layers.sequence_pool(input=step_static_input, - pool_type='last') - projected = fluid.layers.fc(input=[step_x, last], - size=self.output_dim) + fluid.layers.array_write( + x=step_static_input, + i=rnn.step_idx, + array=static_input_out_array, + ) + last = fluid.layers.sequence_pool( + input=step_static_input, pool_type='last' + ) + projected = fluid.layers.fc( + input=[step_x, last], size=self.output_dim + ) rnn.output(projected) if only_forward: static_input_step_outs = [] - step_idx = fluid.layers.fill_constant(shape=[1], - dtype='int64', - value=0) + step_idx = fluid.layers.fill_constant( + shape=[1], dtype='int64', value=0 + ) step_idx.stop_gradient = True for i in range(self._max_sequence_len): - step_out = fluid.layers.array_read(static_input_out_array, - step_idx) + step_out = fluid.layers.array_read( + static_input_out_array, step_idx + ) step_out.stop_gradient = True static_input_step_outs.append(step_out) fluid.layers.increment(x=step_idx, value=1.0, in_place=True) @@ -131,7 +139,8 @@ class TestDyRnnStaticInput(unittest.TestCase): loss = paddle.mean(last) append_backward(loss) static_input_grad = self._program.global_block().var( - framework.grad_var_name('static_input_tensor')) + framework.grad_var_name('static_input_tensor') + ) return static_input_grad, loss def get_expected_static_step_outs(self): @@ -145,8 +154,10 @@ class TestDyRnnStaticInput(unittest.TestCase): cur_offset = 0 for i in range(len(static_lod[0])): static_sliced.append( - self.static_input_data[cur_offset:(cur_offset + - static_lod[0][i])]) + self.static_input_data[ + cur_offset : (cur_offset + static_lod[0][i]) + ] + ) cur_offset += static_lod[0][i] static_seq_len = static_lod[0] static_reordered = [] @@ -170,7 +181,8 @@ class TestDyRnnStaticInput(unittest.TestCase): static_step_lods.append([lod]) end = total_len static_step_outs.append( - np.array(static_reordered[:end]).astype('float32')) + np.array(static_reordered[:end]).astype('float32') + ) return static_step_outs, static_step_lods @@ -203,13 +215,14 @@ class TestDyRnnStaticInput(unittest.TestCase): y_neg = self.fetch_value(loss)[0][0] self.static_input_tensor._set_float_element(i, origin) numeric_gradients.ravel()[i] = (y_pos - y_neg) / self._delta / 2 - np.testing.assert_allclose(actual_gradients, - numeric_gradients, - rtol=0.001) + np.testing.assert_allclose( + actual_gradients, numeric_gradients, rtol=0.001 + ) np.testing.assert_allclose( actual_lod, self.static_input_tensor.recursive_sequence_lengths(), - rtol=1e-05) + rtol=1e-05, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_delete_vars.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_delete_vars.py index b6052d33c95..eebc321b1a8 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_delete_vars.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_delete_vars.py @@ -39,8 +39,10 @@ def simple_fc_net(): hidden, size=200, act='tanh', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) prediction = fluid.layers.fc(hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) loss = paddle.mean(loss) @@ -65,7 +67,6 @@ def get_persistables_and_non_persistables(prog, fetch_list): class TestExecutor(unittest.TestCase): - def test_executor_main(self): places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): @@ -87,12 +88,13 @@ class TestExecutor(unittest.TestCase): def prepare_feed(self, image, label, dev_cnt=1): batch_size = 32 * dev_cnt - image_shape = (batch_size, ) + tuple(image.shape[1:]) - label_shape = (batch_size, ) + tuple(label.shape[1:]) + image_shape = (batch_size,) + tuple(image.shape[1:]) + label_shape = (batch_size,) + tuple(label.shape[1:]) image_np = np.random.random(size=image_shape).astype('float32') - label_np = np.random.random_integers(low=0, high=9, - size=label_shape).astype('int64') + label_np = np.random.random_integers( + low=0, high=9, size=label_shape + ).astype('int64') return image_np, label_np @@ -113,10 +115,16 @@ class TestExecutor(unittest.TestCase): if t._is_initialized(): outline_np_vars.append(name) - print('Non-alive persistable vars {} in {}'.format( - outline_p_vars, persitables)) - print('Alive non-persistable vars {} in {}'.format( - outline_np_vars, non_persistables)) + print( + 'Non-alive persistable vars {} in {}'.format( + outline_p_vars, persitables + ) + ) + print( + 'Alive non-persistable vars {} in {}'.format( + outline_np_vars, non_persistables + ) + ) self.assertEqual(len(outline_p_vars), 0) self.assertEqual(len(outline_np_vars), 0) @@ -130,12 +138,14 @@ class TestExecutor(unittest.TestCase): image, label, loss = simple_fc_net() loss.persistable = False persistables, non_persistables = get_persistables_and_non_persistables( - fluid.default_main_program(), [loss.name]) + fluid.default_main_program(), [loss.name] + ) print('Non-persistable var number {}'.format(len(non_persistables))) print(non_persistables) - self.assert_gc_vars(fluid.default_main_program(), [loss.name], - non_persistables) + self.assert_gc_vars( + fluid.default_main_program(), [loss.name], non_persistables + ) exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) @@ -147,23 +157,34 @@ class TestExecutor(unittest.TestCase): for _ in range(10): image_np, label_np = self.prepare_feed(image, label) fluid.global_scope().var(image.name).get_tensor().set( - image_np, self.place) + image_np, self.place + ) fluid.global_scope().var(label.name).get_tensor().set( - label_np, self.place) + label_np, self.place + ) # exe.run would not create local scope # so that we can detect whether gc clears temporary variables - exe.run(fluid.default_main_program().desc, fluid.global_scope(), 0, - False, True, [loss.name]) - self.assertScopeVar(fluid.global_scope(), persistables, - non_persistables) + exe.run( + fluid.default_main_program().desc, + fluid.global_scope(), + 0, + False, + True, + [loss.name], + ) + self.assertScopeVar( + fluid.global_scope(), persistables, non_persistables + ) def pe_main(self): image, label, loss = simple_fc_net() loss.persistable = False persistables, non_persistables = get_persistables_and_non_persistables( - fluid.default_main_program(), [loss.name]) - self.assert_gc_vars(fluid.default_main_program(), [loss.name], - non_persistables) + fluid.default_main_program(), [loss.name] + ) + self.assert_gc_vars( + fluid.default_main_program(), [loss.name], non_persistables + ) exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) @@ -176,11 +197,14 @@ class TestExecutor(unittest.TestCase): build_strategy.enable_inplace = False prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name, exec_strategy=exec_strategy) + fluid.default_main_program() + ).with_data_parallel(loss_name=loss.name, exec_strategy=exec_strategy) - dev_cnt = fluid.core.get_cuda_device_count() if isinstance(self.place, fluid.CUDAPlace) \ + dev_cnt = ( + fluid.core.get_cuda_device_count() + if isinstance(self.place, fluid.CUDAPlace) else int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + ) for idx in range(10): image_np, label_np = self.prepare_feed(image, label, dev_cnt) diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py index 6e1ed17a3b6..86952d74e39 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py @@ -40,10 +40,9 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): reader = fake_imdb_reader(word_dict_size, batch_size * 40) train_reader = paddle.batch(reader, batch_size=batch_size) - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") @@ -54,8 +53,9 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() feeder = fluid.DataFeeder(feed_list=[data, label], place=place) - reader = feeder.decorate_reader(train_reader, - multi_devices=use_parallel_executor) + reader = feeder.decorate_reader( + train_reader, multi_devices=use_parallel_executor + ) exe = fluid.Executor(place) fluid.default_startup_program().random_seed = 1 @@ -65,8 +65,8 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): train_cp = fluid.default_main_program() if use_parallel_executor: train_cp = compiler.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=cost.name) + fluid.default_main_program() + ).with_data_parallel(loss_name=cost.name) fetch_list = [cost.name] else: fetch_list = [cost] @@ -74,16 +74,17 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): for pass_id in range(pass_num): batch_id = 0 for data in reader(): - exe.run(train_cp, - feed=data, - fetch_list=fetch_list if batch_id % 4 == 0 else []) + exe.run( + train_cp, + feed=data, + fetch_list=fetch_list if batch_id % 4 == 0 else [], + ) batch_id += 1 if batch_id > 16: break class TestBase(unittest.TestCase): - def setUp(self): self.net = None @@ -93,9 +94,11 @@ class TestBase(unittest.TestCase): for use_cuda in [True, False]: for use_parallel_executor in [False, True]: - print('network: {}, use_cuda: {}, use_parallel_executor: {}'. - format(self.net.__name__, use_cuda, - use_parallel_executor)) + print( + 'network: {}, use_cuda: {}, use_parallel_executor: {}'.format( + self.net.__name__, use_cuda, use_parallel_executor + ) + ) with fluid.program_guard(fluid.Program(), fluid.Program()): with fluid.scope_guard(core.Scope()): train(self.net, use_cuda, use_parallel_executor) diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_gru_net.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_gru_net.py index ac501a43ca7..f785936dcb9 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_gru_net.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_gru_net.py @@ -20,18 +20,21 @@ import paddle.fluid as fluid fluid.core._set_eager_deletion_mode(0.0, 1.0, True) -def gru_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=400.0): +def gru_net( + data, + label, + dict_dim, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, + emb_lr=400.0, +): emb = fluid.layers.embedding( input=data, size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) + param_attr=fluid.ParamAttr(learning_rate=emb_lr), + ) fc0 = fluid.layers.fc(input=emb, size=hid_dim * 3) gru_h = fluid.layers.dynamic_gru(input=fc0, size=hid_dim, is_reverse=False) gru_max = fluid.layers.sequence_pool(input=gru_h, pool_type='max') @@ -44,7 +47,6 @@ def gru_net(data, class GRUTest(TestBase): - def setUp(self): self.net = gru_net diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_lstm_net.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_lstm_net.py index bb6f6082015..7709460111a 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_lstm_net.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_lstm_net.py @@ -20,22 +20,25 @@ import unittest fluid.core._set_eager_deletion_mode(0.0, 1.0, True) -def lstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): +def lstm_net( + data, + label, + dict_dim, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, + emb_lr=30.0, +): emb = fluid.layers.embedding( input=data, size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) + param_attr=fluid.ParamAttr(learning_rate=emb_lr), + ) fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - lstm_h, c = fluid.layers.dynamic_lstm(input=fc0, - size=hid_dim * 4, - is_reverse=False) + lstm_h, c = fluid.layers.dynamic_lstm( + input=fc0, size=hid_dim * 4, is_reverse=False + ) lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') lstm_max_tanh = fluid.layers.tanh(lstm_max) fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') @@ -46,7 +49,6 @@ def lstm_net(data, class LSTMTest(TestBase): - def setUp(self): self.net = lstm_net diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_padding_rnn.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_padding_rnn.py index e31595a4b01..e48a8056d03 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_padding_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_padding_rnn.py @@ -28,7 +28,6 @@ os.environ["CPU_NUM"] = "1" class RNNConfig(object): - def __init__(self, model_type, rnn_model): self.model_type = model_type self.rnn_model = rnn_model @@ -95,26 +94,30 @@ class RNNConfig(object): # Fake data reader for test class Reader(object): - def get_data_iter(self, rnn_config): for i in range(rnn_config.max_epoch): - x = np.zeros(shape=(rnn_config.batch_size, rnn_config.num_steps), - dtype='int64') - y = np.ones(shape=(rnn_config.batch_size, rnn_config.num_steps), - dtype='int64') + x = np.zeros( + shape=(rnn_config.batch_size, rnn_config.num_steps), + dtype='int64', + ) + y = np.ones( + shape=(rnn_config.batch_size, rnn_config.num_steps), + dtype='int64', + ) yield (x, y) # Model from PaddleNLP/models/language_model/lm_model.py in Paddle Models repo -def lm_model(hidden_size, - vocab_size, - batch_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None, - rnn_model='static'): - +def lm_model( + hidden_size, + vocab_size, + batch_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + rnn_model='static', +): def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): weight_1_arr = [] weight_2_arr = [] @@ -128,23 +131,24 @@ def lm_model(hidden_size, dtype="float32", name="fc_weight1_" + str(i), default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) weight_1_arr.append(weight_1) bias_1 = layers.create_parameter( [hidden_size * 4], dtype="float32", name="fc_bias1_" + str(i), - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) bias_arr.append(bias_1) - pre_hidden = layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) + pre_hidden = layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) pre_hidden = layers.reshape(pre_hidden, shape=[-1, hidden_size]) pre_cell = layers.reshape(pre_cell, shape=[-1, hidden_size]) hidden_array.append(pre_hidden) @@ -165,25 +169,31 @@ def lm_model(hidden_size, gate_input = layers.matmul(x=nn, y=weight_1) gate_input = layers.elementwise_add(gate_input, bias) - i = layers.slice(gate_input, - axes=[1], - starts=[0], - ends=[hidden_size]) - j = layers.slice(gate_input, - axes=[1], - starts=[hidden_size], - ends=[hidden_size * 2]) - f = layers.slice(gate_input, - axes=[1], - starts=[hidden_size * 2], - ends=[hidden_size * 3]) - o = layers.slice(gate_input, - axes=[1], - starts=[hidden_size * 3], - ends=[hidden_size * 4]) + i = layers.slice( + gate_input, axes=[1], starts=[0], ends=[hidden_size] + ) + j = layers.slice( + gate_input, + axes=[1], + starts=[hidden_size], + ends=[hidden_size * 2], + ) + f = layers.slice( + gate_input, + axes=[1], + starts=[hidden_size * 2], + ends=[hidden_size * 3], + ) + o = layers.slice( + gate_input, + axes=[1], + starts=[hidden_size * 3], + ends=[hidden_size * 4], + ) c = pre_cell * layers.sigmoid(f) + layers.sigmoid( - i) * layers.tanh(j) + i + ) * layers.tanh(j) m = layers.tanh(c) * layers.sigmoid(o) rnn.update_memory(pre_hidden, m) @@ -198,7 +208,8 @@ def lm_model(hidden_size, input = layers.dropout( input, dropout_prob=dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn.step_output(input) rnnout = rnn() @@ -211,15 +222,13 @@ def lm_model(hidden_size, c = rnnout[i * 2 + 1] m.stop_gradient = True c.stop_gradient = True - last_h = layers.slice(m, - axes=[0], - starts=[num_steps - 1], - ends=[num_steps]) + last_h = layers.slice( + m, axes=[0], starts=[num_steps - 1], ends=[num_steps] + ) last_hidden_array.append(last_h) - last_c = layers.slice(c, - axes=[0], - starts=[num_steps - 1], - ends=[num_steps]) + last_c = layers.slice( + c, axes=[0], starts=[num_steps - 1], ends=[num_steps] + ) last_cell_array.append(last_c) real_res = layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = layers.concat(last_hidden_array, 0) @@ -227,10 +236,9 @@ def lm_model(hidden_size, return real_res, last_hidden, last_cell - def encoder_static(input_embedding, - len=3, - init_hidden=None, - init_cell=None): + def encoder_static( + input_embedding, len=3, init_hidden=None, init_cell=None + ): weight_1_arr = [] weight_2_arr = [] @@ -244,36 +252,37 @@ def lm_model(hidden_size, dtype="float32", name="fc_weight1_" + str(i), default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) weight_1_arr.append(weight_1) bias_1 = layers.create_parameter( [hidden_size * 4], dtype="float32", name="fc_bias1_" + str(i), - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) bias_arr.append(bias_1) - pre_hidden = layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = layers.reshape(pre_hidden, - shape=[-1, hidden_size], - inplace=True) - pre_cell = layers.reshape(pre_cell, - shape=[-1, hidden_size], - inplace=True) + pre_hidden = layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = layers.reshape( + pre_hidden, shape=[-1, hidden_size], inplace=True + ) + pre_cell = layers.reshape( + pre_cell, shape=[-1, hidden_size], inplace=True + ) hidden_array.append(pre_hidden) cell_array.append(pre_cell) res = [] - sliced_inputs = layers.split(input_embedding, - num_or_sections=len, - dim=1) + sliced_inputs = layers.split( + input_embedding, num_or_sections=len, dim=1 + ) for index in range(len): input = sliced_inputs[index] @@ -291,7 +300,8 @@ def lm_model(hidden_size, i, j, f, o = layers.split(gate_input, num_or_sections=4, dim=-1) c = pre_cell * layers.sigmoid(f) + layers.sigmoid( - i) * layers.tanh(j) + i + ) * layers.tanh(j) m = layers.tanh(c) * layers.sigmoid(o) hidden_array[k] = m @@ -302,55 +312,67 @@ def lm_model(hidden_size, input = layers.dropout( input, dropout_prob=dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append(input) last_hidden = layers.concat(hidden_array, 1) - last_hidden = layers.reshape(last_hidden, - shape=[-1, num_layers, hidden_size], - inplace=True) + last_hidden = layers.reshape( + last_hidden, shape=[-1, num_layers, hidden_size], inplace=True + ) last_hidden = layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = layers.concat(cell_array, 1) - last_cell = layers.reshape(last_cell, - shape=[-1, num_layers, hidden_size]) + last_cell = layers.reshape( + last_cell, shape=[-1, num_layers, hidden_size] + ) last_cell = layers.transpose(x=last_cell, perm=[1, 0, 2]) real_res = layers.concat(res, 0) - real_res = layers.reshape(real_res, - shape=[len, -1, hidden_size], - inplace=True) + real_res = layers.reshape( + real_res, shape=[len, -1, hidden_size], inplace=True + ) real_res = layers.transpose(x=real_res, perm=[1, 0, 2]) return real_res, last_hidden, last_cell batch_size_each = batch_size - x = layers.data(name="x", - shape=[batch_size_each, num_steps, 1], - dtype='int64', - append_batch_size=False) - y = layers.data(name="y", - shape=[batch_size_each * num_steps, 1], - dtype='int64', - append_batch_size=False) - - init_hidden = layers.data(name="init_hidden", - shape=[num_layers, batch_size_each, hidden_size], - dtype='float32', - append_batch_size=False) - init_cell = layers.data(name="init_cell", - shape=[num_layers, batch_size_each, hidden_size], - dtype='float32', - append_batch_size=False) + x = layers.data( + name="x", + shape=[batch_size_each, num_steps, 1], + dtype='int64', + append_batch_size=False, + ) + y = layers.data( + name="y", + shape=[batch_size_each * num_steps, 1], + dtype='int64', + append_batch_size=False, + ) + + init_hidden = layers.data( + name="init_hidden", + shape=[num_layers, batch_size_each, hidden_size], + dtype='float32', + append_batch_size=False, + ) + init_cell = layers.data( + name="init_cell", + shape=[num_layers, batch_size_each, hidden_size], + dtype='float32', + append_batch_size=False, + ) init_cell.persistable = True init_hidden.persistable = True - init_hidden_reshape = layers.reshape(init_hidden, - shape=[num_layers, -1, hidden_size]) - init_cell_reshape = layers.reshape(init_cell, - shape=[num_layers, -1, hidden_size]) + init_hidden_reshape = layers.reshape( + init_hidden, shape=[num_layers, -1, hidden_size] + ) + init_cell_reshape = layers.reshape( + init_cell, shape=[num_layers, -1, hidden_size] + ) x_emb = layers.embedding( input=x, @@ -359,29 +381,36 @@ def lm_model(hidden_size, is_sparse=False, param_attr=fluid.ParamAttr( name='embedding_para', - initializer=fluid.initializer.UniformInitializer(low=-init_scale, - high=init_scale))) - - x_emb = layers.reshape(x_emb, - shape=[-1, num_steps, hidden_size], - inplace=True) + initializer=fluid.initializer.UniformInitializer( + low=-init_scale, high=init_scale + ), + ), + ) + + x_emb = layers.reshape( + x_emb, shape=[-1, num_steps, hidden_size], inplace=True + ) if dropout != None and dropout > 0.0: - x_emb = layers.dropout(x_emb, - dropout_prob=dropout, - dropout_implementation='upscale_in_train') + x_emb = layers.dropout( + x_emb, + dropout_prob=dropout, + dropout_implementation='upscale_in_train', + ) if rnn_model == "padding": rnn_out, last_hidden, last_cell = padding_rnn( x_emb, len=num_steps, init_hidden=init_hidden_reshape, - init_cell=init_cell_reshape) + init_cell=init_cell_reshape, + ) elif rnn_model == "static": rnn_out, last_hidden, last_cell = encoder_static( x_emb, len=num_steps, init_hidden=init_hidden_reshape, - init_cell=init_cell_reshape) + init_cell=init_cell_reshape, + ) elif rnn_model == "cudnn": x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) rnn_out, last_hidden, last_cell = layers.lstm( @@ -393,44 +422,61 @@ def lm_model(hidden_size, num_layers, is_bidirec=False, default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) rnn_out = layers.transpose(rnn_out, perm=[1, 0, 2]) elif rnn_model == "basic_lstm": - rnn_out, last_hidden, last_cell = basic_lstm( x_emb, init_hidden, init_cell, hidden_size, \ - num_layers=num_layers, batch_first=True, dropout_prob=dropout, \ - param_attr = ParamAttr( initializer=fluid.initializer.UniformInitializer(low=-init_scale, high=init_scale) ), \ - bias_attr = ParamAttr( initializer = fluid.initializer.Constant(0.0) ), \ - forget_bias = 0.0) + rnn_out, last_hidden, last_cell = basic_lstm( + x_emb, + init_hidden, + init_cell, + hidden_size, + num_layers=num_layers, + batch_first=True, + dropout_prob=dropout, + param_attr=ParamAttr( + initializer=fluid.initializer.UniformInitializer( + low=-init_scale, high=init_scale + ) + ), + bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0)), + forget_bias=0.0, + ) else: print("type not support") return - rnn_out = layers.reshape(rnn_out, - shape=[-1, num_steps, hidden_size], - inplace=True) + rnn_out = layers.reshape( + rnn_out, shape=[-1, num_steps, hidden_size], inplace=True + ) softmax_weight = layers.create_parameter( [hidden_size, vocab_size], dtype="float32", name="softmax_weight", default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) softmax_bias = layers.create_parameter( [vocab_size], dtype="float32", name='softmax_bias', default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) projection = layers.matmul(rnn_out, softmax_weight) projection = layers.elementwise_add(projection, softmax_bias) - projection = layers.reshape(projection, - shape=[-1, vocab_size], - inplace=True) + projection = layers.reshape( + projection, shape=[-1, vocab_size], inplace=True + ) - loss = layers.softmax_with_cross_entropy(logits=projection, - label=y, - soft_label=False) + loss = layers.softmax_with_cross_entropy( + logits=projection, label=y, soft_label=False + ) loss = layers.reshape(loss, shape=[-1, num_steps], inplace=True) loss = layers.reduce_mean(loss, dim=[0]) @@ -452,7 +498,6 @@ def lm_model(hidden_size, class PaddingRNNTestBase(unittest.TestCase): - def setUp(self): self.reader = Reader() self.device_count = 1 @@ -485,64 +530,91 @@ class PaddingRNNTestBase(unittest.TestCase): self.startup_program = fluid.Program() with fluid.program_guard(self.main_program, self.startup_program): with fluid.unique_name.guard(): - res_vars = lm_model(config.hidden_size, - config.vocab_size, - config.batch_size, - num_layers=config.num_layers, - num_steps=config.num_steps, - init_scale=config.init_scale, - dropout=config.dropout, - rnn_model=config.rnn_model) - self.loss, self.last_hidden, self.last_cell, self.feed_order = res_vars + res_vars = lm_model( + config.hidden_size, + config.vocab_size, + config.batch_size, + num_layers=config.num_layers, + num_steps=config.num_steps, + init_scale=config.init_scale, + dropout=config.dropout, + rnn_model=config.rnn_model, + ) + ( + self.loss, + self.last_hidden, + self.last_cell, + self.feed_order, + ) = res_vars fluid.clip.set_gradient_clip( clip=fluid.clip.GradientClipByGlobalNorm( - clip_norm=config.max_grad_norm)) + clip_norm=config.max_grad_norm + ) + ) self.learning_rate = fluid.layers.create_global_var( name="learning_rate", shape=[1], value=1.0, dtype='float32', - persistable=True) + persistable=True, + ) optimizer = fluid.optimizer.SGD( - learning_rate=self.learning_rate) + learning_rate=self.learning_rate + ) optimizer.minimize(self.loss) self.exe.run(self.startup_program) if parallel: self.train_program = fluid.compiler.CompiledProgram( - self.main_program).with_data_parallel( - loss_name=self.loss.name, - build_strategy=self.build_strategy, - exec_strategy=self.exec_strategy) + self.main_program + ).with_data_parallel( + loss_name=self.loss.name, + build_strategy=self.build_strategy, + exec_strategy=self.exec_strategy, + ) else: self.train_program = self.main_program def _generate_init_data(self): - init_hidden = np.zeros((self.config.num_layers, self.config.batch_size, - self.config.hidden_size), - dtype='float32') - init_cell = np.zeros((self.config.num_layers, self.config.batch_size, - self.config.hidden_size), - dtype='float32') + init_hidden = np.zeros( + ( + self.config.num_layers, + self.config.batch_size, + self.config.hidden_size, + ), + dtype='float32', + ) + init_cell = np.zeros( + ( + self.config.num_layers, + self.config.batch_size, + self.config.hidden_size, + ), + dtype='float32', + ) return init_hidden, init_cell def _generate_new_lr(self, epoch_id=0, device_count=1): - new_lr = self.config.base_learning_rate * (self.config.lr_decay**max( - epoch_id + 1 - self.config.epoch_start_decay, 0.0)) + new_lr = self.config.base_learning_rate * ( + self.config.lr_decay + ** max(epoch_id + 1 - self.config.epoch_start_decay, 0.0) + ) lr = np.ones((self.device_count), dtype='float32') * new_lr return lr - def _prepare_input(self, - batch, - init_hidden=None, - init_cell=None, - epoch_id=0, - with_lr=True, - device_count=1): + def _prepare_input( + self, + batch, + init_hidden=None, + init_cell=None, + epoch_id=0, + with_lr=True, + device_count=1, + ): x, y = batch x = x.reshape((-1, self.config.num_steps, 1)) y = y.reshape((-1, 1)) @@ -573,16 +645,20 @@ class PaddingRNNTestBase(unittest.TestCase): init_cell=init_cell, epoch_id=epoch_id, with_lr=True, - device_count=self.device_count) - - fetch_outs = self.exe.run(self.train_program, - feed=input_data_feed, - fetch_list=[ - self.loss.name, "learning_rate", - self.last_hidden.name, - self.last_cell.name - ], - use_program_cache=use_program_cache) + device_count=self.device_count, + ) + + fetch_outs = self.exe.run( + self.train_program, + feed=input_data_feed, + fetch_list=[ + self.loss.name, + "learning_rate", + self.last_hidden.name, + self.last_cell.name, + ], + use_program_cache=use_program_cache, + ) cost_train = np.array(fetch_outs[0]) lr = np.array(fetch_outs[1]) @@ -607,9 +683,9 @@ class PaddingRNNTestBase(unittest.TestCase): ppl = np.append(ppl, train_ppl) return ppl - def compare_padding_static_mode(self, - parallel=True, - use_program_cache=True): + def compare_padding_static_mode( + self, parallel=True, use_program_cache=True + ): ''' Test that train ppl of padding mode is same to that of static mode ''' @@ -623,7 +699,6 @@ class PaddingRNNTestBase(unittest.TestCase): class EagerDeletionPaddingRNNTest(PaddingRNNTestBase): - def test_padding_mode_no_eager_deletion(self): ''' Test that train ppl of padding mode is same to that of static mode without eager deletion diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py index 018ab42c50a..b67a72a5da2 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py @@ -33,7 +33,6 @@ fluid.core._set_eager_deletion_mode(0.0, 1.0, True) class PyRNNBase(object): - def __init__(self, input_shape, output_shape): self.x = np.ones(shape=input_shape).astype("float32") self.y = np.zeros(shape=output_shape).astype("float32") @@ -51,13 +50,13 @@ class PyRNNBase(object): class PySimpleRNN1(PyRNNBase): - def __init__(self, input_shape, output_shape): super(PySimpleRNN1, self).__init__(input_shape, output_shape) seq_len, batch_size, input_dim = input_shape - self.h_boot = np.random.normal(size=(batch_size, - input_dim)).astype("float32") + self.h_boot = np.random.normal(size=(batch_size, input_dim)).astype( + "float32" + ) self.scale = 1.0 / 2.0 men_dim = (seq_len, batch_size, input_dim) @@ -73,7 +72,6 @@ class PySimpleRNN1(PyRNNBase): class PySimpleRNN2(PyRNNBase): - def __init__(self, input_shape, output_shape): super(PySimpleRNN2, self).__init__(input_shape, output_shape) @@ -94,7 +92,7 @@ class PySimpleRNN2(PyRNNBase): hU = np.matmul(pre_mem, self.U).astype("float32") def py_sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) self.mems[step_id] = py_sigmoid(xW + hU) self.y[step_id] = self.mems[step_id] @@ -140,14 +138,16 @@ class EagerDeletionRecurrentOpTest1(unittest.TestCase): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False rnn = layers.StaticRNN() @@ -155,8 +155,10 @@ class EagerDeletionRecurrentOpTest1(unittest.TestCase): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) - h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), - scale=self.py_rnn.scale) + h = layers.scale( + x=layers.elementwise_add(x=h_pre, y=x_t), + scale=self.py_rnn.scale, + ) rnn.update_memory(h_pre, h) rnn.output(h) @@ -164,17 +166,18 @@ class EagerDeletionRecurrentOpTest1(unittest.TestCase): return rnn() def forward(self): - gc_vars = core._get_eager_deletion_vars(self.main_program.desc, - [self.output.name]) + gc_vars = core._get_eager_deletion_vars( + self.main_program.desc, [self.output.name] + ) self.assertEqual(len(gc_vars), self.main_program.num_blocks) self.feed_map = { x: create_tensor(getattr(self.py_rnn, x), self.place) for x in self.data_field } exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=[self.output]) + out = exe.run( + self.main_program, feed=self.feed_map, fetch_list=[self.output] + ) return out[0] @@ -189,14 +192,17 @@ class EagerDeletionRecurrentOpTest1(unittest.TestCase): ] gc_vars = core._get_eager_deletion_vars( - self.main_program.desc, [var.name for var in fetch_list]) + self.main_program.desc, [var.name for var in fetch_list] + ) self.assertEqual(len(gc_vars), self.main_program.num_blocks) exe = Executor(self.place) - return exe.run(self.main_program, - feed=self.feed_map, - fetch_list=fetch_list, - return_numpy=False) + return exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=fetch_list, + return_numpy=False, + ) def test_backward(self, rtol=0.01): self.check_forward() @@ -213,10 +219,18 @@ class EagerDeletionRecurrentOpTest1(unittest.TestCase): num_grad[idx], ana_grad[idx], rtol=rtol, - err_msg='num_grad (' + name + ') has diff at ' + - str(self.place) + '\nExpect ' + str(num_grad[idx]) + '\n' + - 'But Got' + str(ana_grad[idx]) + ' in class ' + - self.__class__.__name__) + err_msg='num_grad (' + + name + + ') has diff at ' + + str(self.place) + + '\nExpect ' + + str(num_grad[idx]) + + '\n' + + 'But Got' + + str(ana_grad[idx]) + + ' in class ' + + self.__class__.__name__, + ) def check_forward(self): pd_output = self.forward() @@ -277,14 +291,16 @@ class EagerDeletionRecurrentOpTest2(EagerDeletionRecurrentOpTest1): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False rnn = layers.StaticRNN() @@ -297,15 +313,19 @@ class EagerDeletionRecurrentOpTest2(EagerDeletionRecurrentOpTest1): size=self.input_dim, param_attr=ParamAttr( name='W', - initializer=fluid.initializer.ConstantInitializer(1.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(1.0), + ), + bias_attr=False, + ) temp_r = layers.fc( input=h_pre, size=self.input_dim, param_attr=ParamAttr( name='U', - initializer=fluid.initializer.ConstantInitializer(0.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(0.0), + ), + bias_attr=False, + ) h = layers.sigmoid(x=layers.elementwise_add(x=temp_l, y=temp_r)) @@ -334,16 +354,18 @@ class EagerDeletionRecurrentOpMultipleMemoryTest(EagerDeletionRecurrentOpTest1): ''' class PySimpleRNN3(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(EagerDeletionRecurrentOpMultipleMemoryTest.PySimpleRNN3, - self).__init__(input_shape, output_shape) + super( + EagerDeletionRecurrentOpMultipleMemoryTest.PySimpleRNN3, self + ).__init__(input_shape, output_shape) seq_len, batch_size, input_dim = input_shape - self.h_boot1 = np.random.normal(size=(batch_size, - input_dim)).astype("float32") - self.h_boot2 = np.random.normal(size=(batch_size, - input_dim)).astype("float32") + self.h_boot1 = np.random.normal( + size=(batch_size, input_dim) + ).astype("float32") + self.h_boot2 = np.random.normal( + size=(batch_size, input_dim) + ).astype("float32") men_dim = (seq_len, batch_size, input_dim) self.mems1 = np.zeros(shape=men_dim).astype("float32") @@ -372,26 +394,33 @@ class EagerDeletionRecurrentOpMultipleMemoryTest(EagerDeletionRecurrentOpTest1): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = EagerDeletionRecurrentOpMultipleMemoryTest.PySimpleRNN3( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot1 = layers.data(shape=[self.batch_size, self.input_dim], - dtype='float32', - name='h_boot1', - append_batch_size=False) + h_boot1 = layers.data( + shape=[self.batch_size, self.input_dim], + dtype='float32', + name='h_boot1', + append_batch_size=False, + ) h_boot1.stop_gradient = False - h_boot2 = layers.data(shape=[self.batch_size, self.input_dim], - dtype='float32', - name='h_boot2', - append_batch_size=False) + h_boot2 = layers.data( + shape=[self.batch_size, self.input_dim], + dtype='float32', + name='h_boot2', + append_batch_size=False, + ) h_boot2.stop_gradient = False rnn = layers.StaticRNN() @@ -426,10 +455,10 @@ class EagerDeletionRecurrentOpNoMemBootTest(EagerDeletionRecurrentOpTest1): ''' class PySimpleRNN4(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(EagerDeletionRecurrentOpNoMemBootTest.PySimpleRNN4, - self).__init__(input_shape, output_shape) + super( + EagerDeletionRecurrentOpNoMemBootTest.PySimpleRNN4, self + ).__init__(input_shape, output_shape) men_dim = input_shape self.mems = np.zeros(shape=men_dim).astype("float32") @@ -453,16 +482,19 @@ class EagerDeletionRecurrentOpNoMemBootTest(EagerDeletionRecurrentOpTest1): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = EagerDeletionRecurrentOpNoMemBootTest.PySimpleRNN4( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False rnn = layers.StaticRNN() @@ -496,10 +528,10 @@ class EagerDeletionTwoRecurrentOpsTest(EagerDeletionRecurrentOpTest1): ''' class PySimpleRNN5(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(EagerDeletionTwoRecurrentOpsTest.PySimpleRNN5, - self).__init__(input_shape, output_shape) + super(EagerDeletionTwoRecurrentOpsTest.PySimpleRNN5, self).__init__( + input_shape, output_shape + ) self.mem_0 = np.zeros(shape=input_shape).astype("float32") self.mem_1 = np.zeros(shape=input_shape).astype("float32") self.rnn_0_output = np.zeros(shape=input_shape).astype("float32") @@ -508,13 +540,15 @@ class EagerDeletionTwoRecurrentOpsTest(EagerDeletionRecurrentOpTest1): # First Rnn for step in range(self.x.shape[0]): x_t = self.x[step] - pre_mem = np.zeros_like(x_t) if step == 0 else self.mem_0[step - - 1] + pre_mem = ( + np.zeros_like(x_t) if step == 0 else self.mem_0[step - 1] + ) self.mem_0[step] = x_t + pre_mem self.rnn_0_output[step] = self.mem_0[step] # Second RNN - pre_mem = np.zeros_like(x) if step_id == 0 else self.mem_1[step_id - - 1] + pre_mem = ( + np.zeros_like(x) if step_id == 0 else self.mem_1[step_id - 1] + ) self.mem_1[step_id] = x + np.sum(self.rnn_0_output) self.y[step_id] = self.mem_1[step_id] + pre_mem @@ -530,16 +564,19 @@ class EagerDeletionTwoRecurrentOpsTest(EagerDeletionRecurrentOpTest1): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = EagerDeletionTwoRecurrentOpsTest.PySimpleRNN5( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False rnn_0 = layers.StaticRNN() @@ -563,8 +600,9 @@ class EagerDeletionTwoRecurrentOpsTest(EagerDeletionRecurrentOpTest1): return rnn_1() -class EagerDeletionRecurrentOpParallelExecutorTest(EagerDeletionRecurrentOpTest1 - ): +class EagerDeletionRecurrentOpParallelExecutorTest( + EagerDeletionRecurrentOpTest1 +): ''' Test RNNOp with ParallelExecutor equation: @@ -586,10 +624,12 @@ class EagerDeletionRecurrentOpParallelExecutorTest(EagerDeletionRecurrentOpTest1 build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = True exec_strategy = fluid.ExecutionStrategy() - parallel_exe = fluid.ParallelExecutor(use_cuda=False, - main_program=self.main_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) + parallel_exe = fluid.ParallelExecutor( + use_cuda=False, + main_program=self.main_program, + build_strategy=build_strategy, + exec_strategy=exec_strategy, + ) out = parallel_exe.run(feed=self.feed_map, fetch_list=[self.output]) return out[0] @@ -606,20 +646,23 @@ class EagerDeletionRecurrentOpParallelExecutorTest(EagerDeletionRecurrentOpTest1 build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = True exec_strategy = fluid.ExecutionStrategy() - parallel_exe = fluid.ParallelExecutor(use_cuda=False, - loss_name=self.output.name, - main_program=self.main_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - return parallel_exe.run(feed=self.feed_map, - fetch_list=fetch_list, - return_numpy=False) + parallel_exe = fluid.ParallelExecutor( + use_cuda=False, + loss_name=self.output.name, + main_program=self.main_program, + build_strategy=build_strategy, + exec_strategy=exec_strategy, + ) + return parallel_exe.run( + feed=self.feed_map, fetch_list=fetch_list, return_numpy=False + ) class EagerDeletionFarwardOnlyRnnAndBackwardRnnTest( - EagerDeletionRecurrentOpTest1): + EagerDeletionRecurrentOpTest1 +): ''' - Test one forward only RNN and one backward RNN in one program + Test one forward only RNN and one backward RNN in one program ''' def setUp(self): @@ -635,11 +678,12 @@ class EagerDeletionFarwardOnlyRnnAndBackwardRnnTest( shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', - append_batch_size=False) + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False forward_only_rnn = layers.StaticRNN() @@ -647,8 +691,10 @@ class EagerDeletionFarwardOnlyRnnAndBackwardRnnTest( h_pre = forward_only_rnn.memory(init=h_boot) x_t = forward_only_rnn.step_input(x) - h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), - scale=self.py_rnn.scale) + h = layers.scale( + x=layers.elementwise_add(x=h_pre, y=x_t), + scale=self.py_rnn.scale, + ) forward_only_rnn.update_memory(h_pre, h) forward_only_rnn.output(h) @@ -661,8 +707,10 @@ class EagerDeletionFarwardOnlyRnnAndBackwardRnnTest( h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) - h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), - scale=self.py_rnn.scale) + h = layers.scale( + x=layers.elementwise_add(x=h_pre, y=x_t), + scale=self.py_rnn.scale, + ) rnn.update_memory(h_pre, h) rnn.output(h) @@ -675,9 +723,11 @@ class EagerDeletionFarwardOnlyRnnAndBackwardRnnTest( for x in self.data_field } exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=[self.forward_only_output, self.output]) + out = exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=[self.forward_only_output, self.output], + ) return out[0], out[1] @@ -691,7 +741,6 @@ class EagerDeletionFarwardOnlyRnnAndBackwardRnnTest( class RecurrentNet(paddle.nn.Layer): - def __init__(self): super(RecurrentNet, self).__init__() self.cell = paddle.nn.SimpleRNNCell(16, 32) @@ -703,7 +752,6 @@ class RecurrentNet(paddle.nn.Layer): class TestDy2StRecurrentOpBackward(unittest.TestCase): - def setUp(self): paddle.disable_static() paddle.seed(100) diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_while_op.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_while_op.py index 5aa5ff4eecc..1793d69f48f 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_while_op.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_while_op.py @@ -32,7 +32,6 @@ fluid.core._set_eager_deletion_mode(0.0, 1.0, True) class TestEagerDeletionWhileOpBase(unittest.TestCase): - def test_main(self): places = [ core.CPUPlace(), @@ -51,29 +50,30 @@ class TestEagerDeletionWhileOpBase(unittest.TestCase): self.with_data_parallel = with_data_parallel if not core.is_compiled_with_cuda() and isinstance( - self.place, core.CUDAPlace): + self.place, core.CUDAPlace + ): return if isinstance(self.place, core.CUDAPlace): - device_cnt = core.get_cuda_device_count( - ) if self.with_data_parallel else 1 + device_cnt = ( + core.get_cuda_device_count() if self.with_data_parallel else 1 + ) else: - device_cnt = int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count()) - ) if self.with_data_parallel else 1 - - d0 = layers.data("d0", - shape=[10], - append_batch_size=False, - dtype='float32') - d1 = layers.data("d1", - shape=[10], - append_batch_size=False, - dtype='float32') - d2 = layers.data("d2", - shape=[10], - append_batch_size=False, - dtype='float32') + device_cnt = ( + int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + if self.with_data_parallel + else 1 + ) + + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32' + ) + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32' + ) + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32' + ) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = True @@ -136,7 +136,8 @@ class TestEagerDeletionWhileOpBase(unittest.TestCase): optim.minimize(loss) gc_vars = core._get_eager_deletion_vars( - fluid.default_main_program().desc, [loss.name]) + fluid.default_main_program().desc, [loss.name] + ) self.assertEqual(len(gc_vars), 5) exe = Executor(self.place) @@ -145,8 +146,8 @@ class TestEagerDeletionWhileOpBase(unittest.TestCase): prog = fluid.default_main_program() if self.with_data_parallel: prog = compiler.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name) + fluid.default_main_program() + ).with_data_parallel(loss_name=loss.name) for _ in range(5): d = [] @@ -157,13 +158,11 @@ class TestEagerDeletionWhileOpBase(unittest.TestCase): else: d.append(numpy.array([tmp] * device_cnt)) - outs = exe.run(program=prog, - feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2] - }, - fetch_list=[sum_result]) + outs = exe.run( + program=prog, + feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, + fetch_list=[sum_result], + ) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_eager_run_program.py b/python/paddle/fluid/tests/unittests/test_eager_run_program.py index df151f27ec2..847bdc3d1a8 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_run_program.py +++ b/python/paddle/fluid/tests/unittests/test_eager_run_program.py @@ -15,11 +15,18 @@ import paddle import numpy as np from paddle import _legacy_C_ops -from paddle.fluid.framework import _test_eager_guard, Variable, _in_legacy_dygraph +from paddle.fluid.framework import ( + _test_eager_guard, + Variable, + _in_legacy_dygraph, +) from paddle.fluid import core from paddle.fluid.layers.utils import _hash_with_id from paddle.fluid.dygraph.base import switch_to_static_graph -from paddle.fluid.executor import _is_enable_standalone_executor, _is_dy2st_enable_standalone_executor +from paddle.fluid.executor import ( + _is_enable_standalone_executor, + _is_dy2st_enable_standalone_executor, +) import unittest @@ -60,11 +67,21 @@ def _create_out(var): var_desc = var.desc varbase = None if _in_legacy_dygraph(): - var_base = core.VarBase(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var_base = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) else: - var_base = core.eager.Tensor(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var_base = core.eager.Tensor( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) return var_base @@ -72,16 +89,17 @@ def _create_out(var): def _add_build_strategy_for(input_program, start_op_index, end_op_index): compiled_program = paddle.static.CompiledProgram( core.Graph(input_program.desc, start_op_index, end_op_index), - build_strategy=paddle.static.BuildStrategy()) - compiled_program._compile(core.Scope(), - paddle.framework._current_expected_place()) + build_strategy=paddle.static.BuildStrategy(), + ) + compiled_program._compile( + core.Scope(), paddle.framework._current_expected_place() + ) ir_graph = paddle.fluid.framework.IrGraph(compiled_program._graph) builded_program = ir_graph.to_program() return builded_program class TestRunProgram(unittest.TestCase): - def test_eager(self): paddle.set_device('cpu') paddle.enable_static() @@ -95,12 +113,13 @@ class TestRunProgram(unittest.TestCase): main_program = paddle.static.default_main_program() program = _append_backward_desc(main_program, [out]) forward_program = _add_build_strategy_for( - program, 0, - main_program.desc.block(0).op_size()) + program, 0, main_program.desc.block(0).op_size() + ) backward_program = _add_build_strategy_for( program, main_program.desc.block(0).op_size() + 2, - program.desc.block(0).op_size()) + program.desc.block(0).op_size(), + ) paddle.disable_static('cpu') # step 2: call run_program in eager mode @@ -120,31 +139,52 @@ class TestRunProgram(unittest.TestCase): scope = core.Scope() attrs = [ 'global_block', - program.desc.block(0), 'start_op_index', 0, 'end_op_index', - main_program.desc.block(0).op_size(), 'is_test', False, + program.desc.block(0), + 'start_op_index', + 0, + 'end_op_index', + main_program.desc.block(0).op_size(), + 'is_test', + False, 'program_id', - _hash_with_id(program) + _hash_with_id(program), ] - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: attrs.extend( - ('forward_global_block', forward_program.desc.block(0), - 'backward_global_block', backward_program.desc.block(0))) - - _legacy_C_ops.run_program([x_t, y_t], [fake_var], [out_t], [scope], - [fake_var], None, *attrs) + ( + 'forward_global_block', + forward_program.desc.block(0), + 'backward_global_block', + backward_program.desc.block(0), + ) + ) + + _legacy_C_ops.run_program( + [x_t, y_t], + [fake_var], + [out_t], + [scope], + [fake_var], + None, + *attrs + ) loss = paddle.mean(out_t) loss.backward() np.testing.assert_array_equal(np.ones([2, 2]) * 4, out_t.numpy()) np.testing.assert_array_equal( - np.ones([2, 4]) * 0.5, x_t.grad.numpy()) + np.ones([2, 4]) * 0.5, x_t.grad.numpy() + ) np.testing.assert_array_equal( - np.ones([4, 2]) * 0.5, y_t.grad.numpy()) + np.ones([4, 2]) * 0.5, y_t.grad.numpy() + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_eager_trace_op.py b/python/paddle/fluid/tests/unittests/test_eager_trace_op.py index 771395cd223..6f93c4bd19d 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_trace_op.py +++ b/python/paddle/fluid/tests/unittests/test_eager_trace_op.py @@ -19,26 +19,25 @@ from paddle.fluid.framework import _test_eager_guard class TestEagerTraceOp(unittest.TestCase): - def test_branches(self): with _test_eager_guard(): data = np.random.random([1, 1]).astype(np.float32) x = paddle.to_tensor(data) paddle.fluid.framework._dygraph_tracer().trace_op( - 'broadcast_tensors', { - 'X': [x, x], - 'Out': [x, x] - }, {'Out': [x, x]}, {}) + 'broadcast_tensors', + {'X': [x, x], 'Out': [x, x]}, + {'Out': [x, x]}, + {}, + ) paddle.fluid.framework._dygraph_tracer().trace_op( - 'scale', {'X': x}, {'Out': x}, {'scale': 0.5}) + 'scale', {'X': x}, {'Out': x}, {'scale': 0.5} + ) scale = paddle.to_tensor(np.random.random([1]).astype(np.float32)) paddle.fluid.framework._dygraph_tracer().trace_op( - 'instance_norm', { - 'Scale': [scale], - 'X': [x] - }, {'Y': [x]}, {}) + 'instance_norm', {'Scale': [scale], 'X': [x]}, {'Y': [x]}, {} + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py index 0a245d851cf..c737ed2e41c 100644 --- a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py @@ -18,23 +18,26 @@ from op_test import OpTest import paddle -def python_edit_distance(input, - label, - input_length=None, - label_length=None, - normalized=True, - ignored_tokens=None): +def python_edit_distance( + input, + label, + input_length=None, + label_length=None, + normalized=True, + ignored_tokens=None, +): return paddle.nn.functional.loss.edit_distance( input, label, normalized=normalized, ignored_tokens=ignored_tokens, input_length=input_length, - label_length=label_length) + label_length=label_length, + ) def Levenshtein(hyp, ref): - """ Compute the Levenshtein distance between two strings. + """Compute the Levenshtein distance between two strings. :param hyp: hypothesis string in index :type hyp: list @@ -65,7 +68,6 @@ def Levenshtein(hyp, ref): class TestEditDistanceOp(OpTest): - def setUp(self): self.op_type = "edit_distance" self.python_api = python_edit_distance @@ -85,8 +87,9 @@ class TestEditDistanceOp(OpTest): x2_offset = 0 for i in range(0, num_strs): distance[i] = Levenshtein( - hyp=x1[x1_offset:(x1_offset + self.x1_lod[i])], - ref=x2[x2_offset:(x2_offset + self.x2_lod[i])]) + hyp=x1[x1_offset : (x1_offset + self.x1_lod[i])], + ref=x2[x2_offset : (x2_offset + self.x2_lod[i])], + ) x1_offset += self.x1_lod[i] x2_offset += self.x2_lod[i] if normalized is True: @@ -102,7 +105,6 @@ class TestEditDistanceOp(OpTest): class TestEditDistanceOpNormalizedCase0(OpTest): - def reset_config(self): pass @@ -130,8 +132,9 @@ class TestEditDistanceOpNormalizedCase0(OpTest): x2_offset = 0 for i in range(0, num_strs): distance[i] = Levenshtein( - hyp=self.x1[x1_offset:(x1_offset + self.x1_lod[i])], - ref=self.x2[x2_offset:(x2_offset + self.x2_lod[i])]) + hyp=self.x1[x1_offset : (x1_offset + self.x1_lod[i])], + ref=self.x2[x2_offset : (x2_offset + self.x2_lod[i])], + ) x1_offset += self.x1_lod[i] x2_offset += self.x2_lod[i] if normalized is True: @@ -141,7 +144,7 @@ class TestEditDistanceOpNormalizedCase0(OpTest): self.attrs = {'normalized': normalized} self.inputs = { 'Hyps': (self.x1, [self.x1_lod]), - 'Refs': (self.x2, [self.x2_lod]) + 'Refs': (self.x2, [self.x2_lod]), } self.outputs = {'Out': distance, 'SequenceNum': sequence_num} @@ -152,21 +155,18 @@ class TestEditDistanceOpNormalizedCase0(OpTest): class TestEditDistanceOpNormalizedCase1(TestEditDistanceOpNormalizedCase0): - def reset_config(self): self.x1_lod = [0, 6, 0] self.x2_lod = [2, 1, 2] class TestEditDistanceOpNormalizedCase2(TestEditDistanceOpNormalizedCase0): - def reset_config(self): self.x1_lod = [0, 0, 6] self.x2_lod = [2, 2, 1] class TestEditDistanceOpNormalizedTensor(OpTest): - def reset_config(self): self.x1 = np.array([[10, 3, 0, 0], [6, 5, 8, 2]], dtype=np.int64) self.x2 = np.array([[10, 4, 0], [6, 7, 8]], dtype=np.int64) @@ -185,8 +185,10 @@ class TestEditDistanceOpNormalizedTensor(OpTest): sequence_num = np.array(num_strs).astype("int64") for i in range(0, num_strs): - distance[i] = Levenshtein(hyp=self.x1[i][0:self.x1_lod[i]], - ref=self.x2[i][0:self.x2_lod[i]]) + distance[i] = Levenshtein( + hyp=self.x1[i][0 : self.x1_lod[i]], + ref=self.x2[i][0 : self.x2_lod[i]], + ) if normalized is True: len_ref = self.x2_lod[i] distance[i] = distance[i] / len_ref @@ -196,7 +198,7 @@ class TestEditDistanceOpNormalizedTensor(OpTest): 'Hyps': self.x1, 'Refs': self.x2, 'HypsLength': self.x1_lod, - 'RefsLength': self.x2_lod + 'RefsLength': self.x2_lod, } self.outputs = {'Out': distance, 'SequenceNum': sequence_num} diff --git a/python/paddle/fluid/tests/unittests/test_egr_code_generate_api.py b/python/paddle/fluid/tests/unittests/test_egr_code_generate_api.py index 3ed4962cff6..aea56537654 100644 --- a/python/paddle/fluid/tests/unittests/test_egr_code_generate_api.py +++ b/python/paddle/fluid/tests/unittests/test_egr_code_generate_api.py @@ -19,7 +19,6 @@ import unittest class EagerOpAPIGenerateTestCase(unittest.TestCase): - def test_elementwise_add(self): with _test_eager_guard(): paddle.set_device("cpu") @@ -35,8 +34,9 @@ class EagerOpAPIGenerateTestCase(unittest.TestCase): def test_sum(self): with _test_eager_guard(): - x_data = np.array([[0.2, 0.3, 0.5, 0.9], [0.1, 0.2, 0.6, - 0.7]]).astype('float32') + x_data = np.array( + [[0.2, 0.3, 0.5, 0.9], [0.1, 0.2, 0.6, 0.7]] + ).astype('float32') x = paddle.to_tensor(x_data, 'float32') out = paddle.sum(x, axis=0) out_arr = out.numpy() @@ -61,8 +61,8 @@ class EagerOpAPIGenerateTestCase(unittest.TestCase): out = paddle.nn.functional.sigmoid(x) out_arr = out.numpy() out_arr_expected = np.array( - [0.40131234, 0.450166, 0.52497919, - 0.57444252]).astype('float32') + [0.40131234, 0.450166, 0.52497919, 0.57444252] + ).astype('float32') np.testing.assert_allclose(out_arr, out_arr_expected, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_egr_python_api.py b/python/paddle/fluid/tests/unittests/test_egr_python_api.py index 82881ee0da6..eee9e8eac4b 100644 --- a/python/paddle/fluid/tests/unittests/test_egr_python_api.py +++ b/python/paddle/fluid/tests/unittests/test_egr_python_api.py @@ -15,13 +15,18 @@ import paddle.fluid.core as core import paddle import numpy as np -from paddle.fluid.framework import EagerParamBase, _current_expected_place, _disable_legacy_dygraph, _test_eager_guard, in_dygraph_mode +from paddle.fluid.framework import ( + EagerParamBase, + _current_expected_place, + _disable_legacy_dygraph, + _test_eager_guard, + in_dygraph_mode, +) import unittest import copy class EagerScaleTestCase(unittest.TestCase): - def test_scale_base(self): with _test_eager_guard(): paddle.set_device("cpu") @@ -40,8 +45,9 @@ class EagerScaleTestCase(unittest.TestCase): paddle.set_device("cpu") input_data = np.ones([4, 16, 16, 32]).astype('float32') - data_eager = paddle.to_tensor(input_data, 'float32', - core.CPUPlace(), False) + data_eager = paddle.to_tensor( + input_data, 'float32', core.CPUPlace(), False + ) grad_data = np.ones([4, 16, 16, 32]).astype('float32') grad_eager = paddle.to_tensor(grad_data, 'float32', core.CPUPlace()) @@ -59,32 +65,34 @@ class EagerScaleTestCase(unittest.TestCase): paddle.set_device("cpu") input_data = np.ones([4, 16, 16, 32]).astype('float32') - data_eager = paddle.to_tensor(input_data, 'float32', - core.CPUPlace(), False) + data_eager = paddle.to_tensor( + input_data, 'float32', core.CPUPlace(), False + ) grad_data = np.ones([4, 16, 16, 32]).astype('float32') grad_data2 = np.ones([4, 16]).astype('float32') grad_eager = paddle.to_tensor(grad_data, 'float32', core.CPUPlace()) - grad_eager2 = paddle.to_tensor(grad_data2, 'float32', - core.CPUPlace()) + grad_eager2 = paddle.to_tensor( + grad_data2, 'float32', core.CPUPlace() + ) data_eager.retain_grads() out_eager = core.eager.scale(data_eager, 1.0, 0.9, True, True) self.assertIsNone(data_eager.grad) with self.assertRaisesRegexp( - AssertionError, - "The type of grad_tensor must be paddle.Tensor"): + AssertionError, "The type of grad_tensor must be paddle.Tensor" + ): out_eager.backward(grad_data, False) with self.assertRaisesRegexp( - AssertionError, - "Tensor shape not match, Tensor of grad_tensor /*"): + AssertionError, + "Tensor shape not match, Tensor of grad_tensor /*", + ): out_eager.backward(grad_eager2, False) class EagerDtypeTestCase(unittest.TestCase): - def check_to_tesnsor_and_numpy(self, dtype, proto_dtype): with _test_eager_guard(): arr = np.random.random([4, 16, 16, 32]).astype(dtype) @@ -103,14 +111,15 @@ class EagerDtypeTestCase(unittest.TestCase): self.check_to_tesnsor_and_numpy('float16', core.VarDesc.VarType.FP16) self.check_to_tesnsor_and_numpy('float32', core.VarDesc.VarType.FP32) self.check_to_tesnsor_and_numpy('float64', core.VarDesc.VarType.FP64) - self.check_to_tesnsor_and_numpy('complex64', - core.VarDesc.VarType.COMPLEX64) - self.check_to_tesnsor_and_numpy('complex128', - core.VarDesc.VarType.COMPLEX128) + self.check_to_tesnsor_and_numpy( + 'complex64', core.VarDesc.VarType.COMPLEX64 + ) + self.check_to_tesnsor_and_numpy( + 'complex128', core.VarDesc.VarType.COMPLEX128 + ) class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): - def constructor(self, place): egr_tensor = core.eager.Tensor() self.assertEqual(egr_tensor.persistable, False) @@ -119,17 +128,22 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor.stop_gradient, True) - egr_tensor0 = core.eager.Tensor(core.VarDesc.VarType.FP32, - [4, 16, 16, 32], "test_eager_tensor", - core.VarDesc.VarType.LOD_TENSOR, True) + egr_tensor0 = core.eager.Tensor( + core.VarDesc.VarType.FP32, + [4, 16, 16, 32], + "test_eager_tensor", + core.VarDesc.VarType.LOD_TENSOR, + True, + ) self.assertEqual(egr_tensor0.persistable, True) self.assertEqual(egr_tensor0.name, "test_eager_tensor") self.assertEqual(egr_tensor0.shape, [4, 16, 16, 32]) self.assertEqual(egr_tensor0.dtype, core.VarDesc.VarType.FP32) arr0 = np.random.rand(4, 16, 16, 32).astype('float32') - egr_tensor1 = core.eager.Tensor(arr0, place, True, False, - "numpy_tensor1", False) + egr_tensor1 = core.eager.Tensor( + arr0, place, True, False, "numpy_tensor1", False + ) self.assertEqual(egr_tensor1.persistable, True) self.assertEqual(egr_tensor1.name, "numpy_tensor1") self.assertEqual(egr_tensor1.shape, [4, 16, 16, 32]) @@ -139,8 +153,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): np.testing.assert_array_equal(egr_tensor1.numpy(), arr0) arr1 = np.random.randint(100, size=(4, 16, 16, 32), dtype=np.int64) - egr_tensor2 = core.eager.Tensor(arr1, place, False, True, - "numpy_tensor2", True) + egr_tensor2 = core.eager.Tensor( + arr1, place, False, True, "numpy_tensor2", True + ) self.assertEqual(egr_tensor2.persistable, False) self.assertEqual(egr_tensor2.name, "numpy_tensor2") self.assertEqual(egr_tensor2.shape, [4, 16, 16, 32]) @@ -158,7 +173,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor3.stop_gradient, True) self.assertTrue( egr_tensor3.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor3.numpy(), arr2) egr_tensor3.stop_gradient = False @@ -170,7 +187,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor4.stop_gradient, True) self.assertTrue( egr_tensor4.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor4.numpy(), egr_tensor3.numpy()) arr4 = np.random.rand(4, 16, 16, 32).astype('float32') @@ -253,16 +272,18 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(zero_dim_param.shape, []) with self.assertRaisesRegexp( - ValueError, "The shape of Parameter should not be None"): + ValueError, "The shape of Parameter should not be None" + ): eager_param = EagerParamBase(shape=None, dtype="float32") with self.assertRaisesRegexp( - ValueError, "The dtype of Parameter should not be None"): + ValueError, "The dtype of Parameter should not be None" + ): eager_param = EagerParamBase(shape=[1, 1], dtype=None) with self.assertRaisesRegexp( - ValueError, - "Each dimension of shape for Parameter must be greater than 0, but received /*" + ValueError, + "Each dimension of shape for Parameter must be greater than 0, but received /*", ): eager_param = EagerParamBase(shape=[-1], dtype="float32") @@ -271,19 +292,19 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): eager_param.trainable = False self.assertFalse(eager_param.trainable) with self.assertRaisesRegexp( - ValueError, - "The type of trainable MUST be bool, but the type is /*"): + ValueError, "The type of trainable MUST be bool, but the type is /*" + ): eager_param.trainable = "False" - eager_param_2 = EagerParamBase(shape=paddle.shape( - paddle.to_tensor([1, 2, 3, 4])), - dtype="float32") + eager_param_2 = EagerParamBase( + shape=paddle.shape(paddle.to_tensor([1, 2, 3, 4])), dtype="float32" + ) self.assertTrue(eager_param_2.trainable) eager_param_2.trainable = False self.assertFalse(eager_param_2.trainable) with self.assertRaisesRegexp( - ValueError, - "The type of trainable MUST be bool, but the type is /*"): + ValueError, "The type of trainable MUST be bool, but the type is /*" + ): eager_param_2.trainable = "False" def test_constructor(self): @@ -306,7 +327,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor0.shape, [4, 16, 16, 32]) self.assertTrue( egr_tensor0.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) self.assertEqual(egr_tensor0.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor0.stop_gradient, True) @@ -326,9 +349,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor2.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor2.stop_gradient, True) - egr_tensor3 = core.eager.Tensor(arr, - place=place, - name="new_eager_tensor") + egr_tensor3 = core.eager.Tensor( + arr, place=place, name="new_eager_tensor" + ) self.assertEqual(egr_tensor3.persistable, False) self.assertTrue("new_eager_tensor" in egr_tensor3.name) self.assertEqual(egr_tensor3.shape, [4, 16, 16, 32]) @@ -336,10 +359,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor3.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor3.stop_gradient, True) - egr_tensor4 = core.eager.Tensor(arr, - place=place, - persistable=True, - name="new_eager_tensor") + egr_tensor4 = core.eager.Tensor( + arr, place=place, persistable=True, name="new_eager_tensor" + ) self.assertEqual(egr_tensor4.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor4.name) self.assertEqual(egr_tensor4.shape, [4, 16, 16, 32]) @@ -347,11 +369,13 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor4.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor4.stop_gradient, True) - egr_tensor5 = core.eager.Tensor(arr, - core.CPUPlace(), - persistable=True, - name="new_eager_tensor", - zero_copy=True) + egr_tensor5 = core.eager.Tensor( + arr, + core.CPUPlace(), + persistable=True, + name="new_eager_tensor", + zero_copy=True, + ) self.assertEqual(egr_tensor5.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor5.name) self.assertEqual(egr_tensor5.shape, [4, 16, 16, 32]) @@ -359,11 +383,13 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor5.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor5.stop_gradient, True) - egr_tensor6 = core.eager.Tensor(arr, - place=core.CPUPlace(), - persistable=True, - name="new_eager_tensor", - zero_copy=True) + egr_tensor6 = core.eager.Tensor( + arr, + place=core.CPUPlace(), + persistable=True, + name="new_eager_tensor", + zero_copy=True, + ) self.assertEqual(egr_tensor6.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor6.name) self.assertEqual(egr_tensor6.shape, [4, 16, 16, 32]) @@ -371,11 +397,13 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor6.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor6.stop_gradient, True) - egr_tensor7 = core.eager.Tensor(arr, - place=place, - persistable=True, - name="new_eager_tensor", - zero_copy=True) + egr_tensor7 = core.eager.Tensor( + arr, + place=place, + persistable=True, + name="new_eager_tensor", + zero_copy=True, + ) self.assertEqual(egr_tensor7.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor7.name) self.assertEqual(egr_tensor7.shape, [4, 16, 16, 32]) @@ -383,12 +411,14 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor7.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor7.stop_gradient, True) - egr_tensor8 = core.eager.Tensor(arr, - place=place, - persistable=True, - name="new_eager_tensor", - zero_copy=True, - stop_gradient=False) + egr_tensor8 = core.eager.Tensor( + arr, + place=place, + persistable=True, + name="new_eager_tensor", + zero_copy=True, + stop_gradient=False, + ) self.assertEqual(egr_tensor8.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor8.name) self.assertEqual(egr_tensor8.shape, [4, 16, 16, 32]) @@ -396,12 +426,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor8.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor8.stop_gradient, False) - egr_tensor9 = core.eager.Tensor(arr, - place, - True, - True, - "new_eager_tensor", - stop_gradient=False) + egr_tensor9 = core.eager.Tensor( + arr, place, True, True, "new_eager_tensor", stop_gradient=False + ) self.assertEqual(egr_tensor9.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor9.name) self.assertEqual(egr_tensor9.shape, [4, 16, 16, 32]) @@ -409,12 +436,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor9.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor9.stop_gradient, False) - egr_tensor10 = core.eager.Tensor(arr, - place, - True, - True, - name="new_eager_tensor", - stop_gradient=False) + egr_tensor10 = core.eager.Tensor( + arr, place, True, True, name="new_eager_tensor", stop_gradient=False + ) self.assertEqual(egr_tensor10.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor10.name) self.assertEqual(egr_tensor10.shape, [4, 16, 16, 32]) @@ -422,12 +446,14 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor10.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor10.stop_gradient, False) - egr_tensor11 = core.eager.Tensor(arr, - place, - True, - zero_copy=True, - name="new_eager_tensor", - stop_gradient=False) + egr_tensor11 = core.eager.Tensor( + arr, + place, + True, + zero_copy=True, + name="new_eager_tensor", + stop_gradient=False, + ) self.assertEqual(egr_tensor11.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor11.name) self.assertEqual(egr_tensor11.shape, [4, 16, 16, 32]) @@ -435,12 +461,14 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor11.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor11.stop_gradient, False) - egr_tensor12 = core.eager.Tensor(arr, - place, - persistable=True, - zero_copy=True, - name="new_eager_tensor", - stop_gradient=False) + egr_tensor12 = core.eager.Tensor( + arr, + place, + persistable=True, + zero_copy=True, + name="new_eager_tensor", + stop_gradient=False, + ) self.assertEqual(egr_tensor12.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor12.name) self.assertEqual(egr_tensor12.shape, [4, 16, 16, 32]) @@ -448,12 +476,14 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor12.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor12.stop_gradient, False) - egr_tensor13 = core.eager.Tensor(value=arr, - place=place, - persistable=True, - zero_copy=True, - name="new_eager_tensor", - stop_gradient=False) + egr_tensor13 = core.eager.Tensor( + value=arr, + place=place, + persistable=True, + zero_copy=True, + name="new_eager_tensor", + stop_gradient=False, + ) self.assertEqual(egr_tensor13.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor13.name) self.assertEqual(egr_tensor13.shape, [4, 16, 16, 32]) @@ -462,11 +492,13 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor13.stop_gradient, False) # special case - egr_tensor14 = core.eager.Tensor(dtype=core.VarDesc.VarType.FP32, - dims=[4, 16, 16, 32], - name="special_eager_tensor", - type=core.VarDesc.VarType.LOD_TENSOR, - persistable=True) + egr_tensor14 = core.eager.Tensor( + dtype=core.VarDesc.VarType.FP32, + dims=[4, 16, 16, 32], + name="special_eager_tensor", + type=core.VarDesc.VarType.LOD_TENSOR, + persistable=True, + ) self.assertEqual(egr_tensor14.persistable, True) self.assertEqual(egr_tensor14.name, "special_eager_tensor") self.assertEqual(egr_tensor14.shape, [4, 16, 16, 32]) @@ -481,11 +513,14 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor15.stop_gradient, True) self.assertTrue( egr_tensor15.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor15.numpy(), egr_tensor4.numpy()) - egr_tensor16 = core.eager.Tensor(value=egr_tensor4, - name="new_eager_tensor") + egr_tensor16 = core.eager.Tensor( + value=egr_tensor4, name="new_eager_tensor" + ) self.assertEqual(egr_tensor16.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor16.name) self.assertEqual(egr_tensor16.shape, egr_tensor4.shape) @@ -493,7 +528,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor16.stop_gradient, True) self.assertTrue( egr_tensor16.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor16.numpy(), egr_tensor4.numpy()) egr_tensor17 = core.eager.Tensor( @@ -547,7 +584,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor20.stop_gradient, True) self.assertTrue( egr_tensor20.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor20.numpy(), x) egr_tensor21 = core.eager.Tensor(value=t, place=place) @@ -577,9 +616,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertTrue(egr_tensor23.place._equals(place)) np.testing.assert_array_equal(egr_tensor23.numpy(), x) - egr_tensor24 = core.eager.Tensor(value=t, - place=place, - name="from_framework_tensor") + egr_tensor24 = core.eager.Tensor( + value=t, place=place, name="from_framework_tensor" + ) self.assertEqual(egr_tensor24.persistable, False) self.assertTrue("from_framework_tensor" in egr_tensor24.name) self.assertEqual(egr_tensor24.shape, [3, 3]) @@ -609,15 +648,18 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): arr = np.ones([4, 16, 16, 32]).astype('float32') arr1 = np.zeros([4, 16]).astype('float32') arr2 = np.ones([4, 16, 16, 32]).astype('float32') + np.ones( - [4, 16, 16, 32]).astype('float32') - tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, - core.CPUPlace()) + [4, 16, 16, 32] + ).astype('float32') + tensor = paddle.to_tensor( + arr, core.VarDesc.VarType.FP32, core.CPUPlace() + ) self.assertEqual(tensor.stop_gradient, True) tensor.stop_gradient = False print("Set persistable") tensor.persistable = False - tensor1 = paddle.to_tensor(arr1, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor1 = paddle.to_tensor( + arr1, core.VarDesc.VarType.FP32, core.CPUPlace() + ) tensor1.persistable = True self.assertEqual(tensor1.stop_gradient, True) np.testing.assert_array_equal(tensor.numpy(), arr) @@ -629,8 +671,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): np.testing.assert_array_equal(tensor.numpy(), arr1) print("Test _copy_to") - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CPUPlace() + ) np.testing.assert_array_equal(tensor2.numpy(), arr2) self.assertTrue(tensor2.place.is_cpu_place()) tensor2.persistable = True @@ -656,8 +699,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): tensor10 = paddle.to_tensor([1, 2, 3], place='gpu_pinned') tensor11 = tensor10._copy_to(core.CUDAPlace(0), True) - np.testing.assert_array_equal(tensor10.numpy(), - tensor11.numpy()) + np.testing.assert_array_equal( + tensor10.numpy(), tensor11.numpy() + ) else: tensor3 = tensor2._copy_to(core.CPUPlace(), True) np.testing.assert_array_equal(tensor3.numpy(), arr2) @@ -676,18 +720,22 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): arr = np.ones([4, 16, 16, 32]).astype('float32') arr1 = np.zeros([4, 16]).astype('float32') arr2 = np.ones([4, 16, 16, 32]).astype('float32') + np.ones( - [4, 16, 16, 32]).astype('float32') + [4, 16, 16, 32] + ).astype('float32') tensor = None tensor2 = None - tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor = paddle.to_tensor( + arr, core.VarDesc.VarType.FP32, core.CPUPlace() + ) tensor3 = core.eager.Tensor(value=tensor, place=core.CPUPlace()) if core.is_compiled_with_cuda(): - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CUDAPlace(0)) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CUDAPlace(0) + ) else: - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CPUPlace() + ) np.testing.assert_array_equal(tensor.numpy(), arr) np.testing.assert_array_equal(tensor2.numpy(), arr2) tensor2._share_buffer_to(tensor) @@ -704,18 +752,22 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): arr = np.ones([4, 16, 16, 32]).astype('float32') arr1 = np.zeros([4, 16]).astype('float32') arr2 = np.ones([4, 16, 16, 32]).astype('float32') + np.ones( - [4, 16, 16, 32]).astype('float32') + [4, 16, 16, 32] + ).astype('float32') tensor = None tensor2 = None - tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor = paddle.to_tensor( + arr, core.VarDesc.VarType.FP32, core.CPUPlace() + ) tensor3 = core.eager.Tensor() if core.is_compiled_with_cuda(): - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CUDAPlace(0)) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CUDAPlace(0) + ) else: - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CPUPlace() + ) np.testing.assert_array_equal(tensor.numpy(), arr) np.testing.assert_array_equal(tensor2.numpy(), arr2) tensor2._share_underline_tensor_to(tensor) @@ -732,8 +784,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): with _test_eager_guard(): paddle.set_device("cpu") arr = np.ones([4, 16, 16, 32]).astype('float32') - tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor = paddle.to_tensor( + arr, core.VarDesc.VarType.FP32, core.CPUPlace() + ) self.assertEqual(tensor.shape, [4, 16, 16, 32]) tensor.name = 'tensor_name_test' self.assertEqual(tensor.name, 'tensor_name_test') @@ -764,14 +817,14 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): paddle.set_device("gpu:0") with paddle.fluid.framework._dygraph_place_guard(core.CPUPlace()): self.assertTrue( - isinstance(_current_expected_place(), - type(core.CPUPlace()))) + isinstance(_current_expected_place(), type(core.CPUPlace())) + ) else: paddle.set_device("cpu") with paddle.fluid.framework._dygraph_place_guard(core.CPUPlace()): self.assertTrue( - isinstance(_current_expected_place(), - type(core.CPUPlace()))) + isinstance(_current_expected_place(), type(core.CPUPlace())) + ) def test_value(self): with _test_eager_guard(): @@ -783,13 +836,19 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): self.assertEqual(egr_tensor0.shape, [4, 16, 16, 32]) self.assertTrue( egr_tensor0.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) self.assertEqual(egr_tensor0.dtype, core.VarDesc.VarType.FP64) self.assertEqual(egr_tensor0.stop_gradient, True) - self.assertTrue(egr_tensor0.value().get_tensor()._dtype(), - core.VarDesc.VarType.FP64) - self.assertTrue(egr_tensor0.value().get_tensor()._place(), - paddle.fluid.framework._current_expected_place()) + self.assertTrue( + egr_tensor0.value().get_tensor()._dtype(), + core.VarDesc.VarType.FP64, + ) + self.assertTrue( + egr_tensor0.value().get_tensor()._place(), + paddle.fluid.framework._current_expected_place(), + ) self.assertTrue(egr_tensor0.value().get_tensor()._is_initialized()) def test_set_value(self): @@ -813,8 +872,9 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): def test_sharding_related_api(self): with _test_eager_guard(): arr0 = np.random.rand(4, 16, 16, 32).astype('float32') - egr_tensor1 = core.eager.Tensor(arr0, core.CPUPlace(), True, False, - "numpy_tensor1", False) + egr_tensor1 = core.eager.Tensor( + arr0, core.CPUPlace(), True, False, "numpy_tensor1", False + ) self.assertEqual(egr_tensor1._numel(), 32768) self.assertEqual(egr_tensor1._slice(0, 2)._numel(), 16384) @@ -839,7 +899,6 @@ class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): class EagerParamBaseUsageTestCase(unittest.TestCase): - def test_print(self): with _test_eager_guard(): linear = paddle.nn.Linear(3, 3, bias_attr=False) @@ -850,10 +909,12 @@ class EagerParamBaseUsageTestCase(unittest.TestCase): linear = paddle.nn.Linear(1, 3) linear_copy = copy.deepcopy(linear) linear_copy2 = linear.weight._copy_to(core.CPUPlace(), True) - np.testing.assert_array_equal(linear.weight.numpy(), - linear_copy.weight.numpy()) - np.testing.assert_array_equal(linear.weight.numpy(), - linear_copy2.numpy()) + np.testing.assert_array_equal( + linear.weight.numpy(), linear_copy.weight.numpy() + ) + np.testing.assert_array_equal( + linear.weight.numpy(), linear_copy2.numpy() + ) def func_fp16_initilaizer(self): paddle.set_default_dtype("float16") @@ -862,22 +923,25 @@ class EagerParamBaseUsageTestCase(unittest.TestCase): 1, 3, bias_attr=False, - weight_attr=paddle.fluid.initializer.Uniform()) + weight_attr=paddle.fluid.initializer.Uniform(), + ) linear3 = paddle.nn.Linear( 1, 3, bias_attr=False, - weight_attr=paddle.fluid.initializer.TruncatedNormalInitializer()) + weight_attr=paddle.fluid.initializer.TruncatedNormalInitializer(), + ) linear4 = paddle.nn.Linear( 1, 3, bias_attr=False, - weight_attr=paddle.fluid.initializer.MSRAInitializer()) + weight_attr=paddle.fluid.initializer.MSRAInitializer(), + ) res = [ linear1.weight.numpy(), linear2.weight.numpy(), linear3.weight.numpy(), - linear4.weight.numpy() + linear4.weight.numpy(), ] paddle.set_default_dtype("float32") return res @@ -896,7 +960,8 @@ class EagerParamBaseUsageTestCase(unittest.TestCase): def func_layer_helper_base(self, value): base = paddle.fluid.layer_helper_base.LayerHelperBase( - "test_layer", "test_layer") + "test_layer", "test_layer" + ) return base.to_variable(value).numpy() def func_base_to_variable(self, value): @@ -945,7 +1010,6 @@ class EagerParamBaseUsageTestCase(unittest.TestCase): class EagerGuardTestCase(unittest.TestCase): - def test__test_eager_guard(self): tracer = paddle.fluid.dygraph.tracer.Tracer() with _test_eager_guard(tracer): diff --git a/python/paddle/fluid/tests/unittests/test_egr_string_tensor_api.py b/python/paddle/fluid/tests/unittests/test_egr_string_tensor_api.py index e0a1058e618..b25e40149ba 100644 --- a/python/paddle/fluid/tests/unittests/test_egr_string_tensor_api.py +++ b/python/paddle/fluid/tests/unittests/test_egr_string_tensor_api.py @@ -19,13 +19,15 @@ import unittest class EagerStringTensorTestCase(unittest.TestCase): - def setUp(self): - self.str_arr = np.array([ - ["15.4寸笔记本的键盘确实爽，基本跟台式机差不多了，蛮喜欢数字小键盘，输数字特方便，样子也很美观，做工也相当不错" - ], # From ChnSentiCorp - ["One of the very best Three Stooges shorts ever."] - ]) # From IMDB + self.str_arr = np.array( + [ + [ + "15.4寸笔记本的键盘确实爽，基本跟台式机差不多了，蛮喜欢数字小键盘，输数字特方便，样子也很美观，做工也相当不错" + ], # From ChnSentiCorp + ["One of the very best Three Stooges shorts ever."], + ] + ) # From IMDB def test_constructor_with_args(self): with _test_eager_guard(): @@ -38,8 +40,9 @@ class EagerStringTensorTestCase(unittest.TestCase): ST2 = core.eager.StringTensor(shape, "ST2") # constructor 2 self.assertEqual(ST2.name, "ST2") self.assertEqual(ST2.shape, shape) - np.testing.assert_array_equal(ST2.numpy(), - np.empty(shape, dtype=np.unicode_)) + np.testing.assert_array_equal( + ST2.numpy(), np.empty(shape, dtype=np.unicode_) + ) ST3 = core.eager.StringTensor(self.str_arr, "ST3") # constructor 3 self.assertEqual(ST3.name, "ST3") @@ -68,15 +71,18 @@ class EagerStringTensorTestCase(unittest.TestCase): def test_constructor_with_kwargs(self): with _test_eager_guard(): shape = [2, 3] - ST1 = core.eager.StringTensor(dims=shape, - name="ST1") # constructor 2 + ST1 = core.eager.StringTensor( + dims=shape, name="ST1" + ) # constructor 2 self.assertEqual(ST1.name, "ST1") self.assertEqual(ST1.shape, shape) - np.testing.assert_array_equal(ST1.numpy(), - np.empty(shape, dtype=np.unicode_)) + np.testing.assert_array_equal( + ST1.numpy(), np.empty(shape, dtype=np.unicode_) + ) - ST2 = core.eager.StringTensor(self.str_arr, - name="ST2") # constructor 3 + ST2 = core.eager.StringTensor( + self.str_arr, name="ST2" + ) # constructor 3 self.assertEqual(ST2.name, "ST2") self.assertEqual(ST2.shape, list(self.str_arr.shape)) np.testing.assert_array_equal(ST2.numpy(), self.str_arr) @@ -86,8 +92,9 @@ class EagerStringTensorTestCase(unittest.TestCase): self.assertEqual(ST3.shape, list(self.str_arr.shape)) np.testing.assert_array_equal(ST3.numpy(), self.str_arr) - ST4 = core.eager.StringTensor(value=ST2, - name="ST4") # constructor 6 + ST4 = core.eager.StringTensor( + value=ST2, name="ST4" + ) # constructor 6 self.assertEqual(ST4.name, "ST4") self.assertEqual(ST4.shape, list(self.str_arr.shape)) np.testing.assert_array_equal(ST4.numpy(), self.str_arr) diff --git a/python/paddle/fluid/tests/unittests/test_eig_op.py b/python/paddle/fluid/tests/unittests/test_eig_op.py index d8af7048c2a..338eb3512e5 100644 --- a/python/paddle/fluid/tests/unittests/test_eig_op.py +++ b/python/paddle/fluid/tests/unittests/test_eig_op.py @@ -22,9 +22,9 @@ import unittest # cast output to complex for numpy.linalg.eig def cast_to_complex(input, output): - if (input.dtype == np.float32): + if input.dtype == np.float32: output = output.astype(np.complex64) - elif (input.dtype == np.float64): + elif input.dtype == np.float64: output = output.astype(np.complex128) return output @@ -57,7 +57,6 @@ def eig_backward(w, v, grad_w, grad_v): class TestEigOp(OpTest): - def setUp(self): paddle.enable_static() paddle.device.set_device("cpu") @@ -72,8 +71,10 @@ class TestEigOp(OpTest): self.set_dims() self.x = np.random.random(self.shape).astype(self.dtype) self.out = np.linalg.eig(self.x) - self.out = (cast_to_complex(self.x, self.out[0]), - cast_to_complex(self.x, self.out[1])) + self.out = ( + cast_to_complex(self.x, self.out[0]), + cast_to_complex(self.x, self.out[1]), + ) # for the real input, a customized checker is needed def checker(self, outs): @@ -84,13 +85,17 @@ class TestEigOp(OpTest): length_w = len(expect_out_w) act_w_real = np.sort( - np.array([np.abs(actual_out_w[i].real) for i in range(length_w)])) + np.array([np.abs(actual_out_w[i].real) for i in range(length_w)]) + ) act_w_imag = np.sort( - np.array([np.abs(actual_out_w[i].imag) for i in range(length_w)])) + np.array([np.abs(actual_out_w[i].imag) for i in range(length_w)]) + ) exp_w_real = np.sort( - np.array([np.abs(expect_out_w[i].real) for i in range(length_w)])) + np.array([np.abs(expect_out_w[i].real) for i in range(length_w)]) + ) exp_w_imag = np.sort( - np.array([np.abs(expect_out_w[i].imag) for i in range(length_w)])) + np.array([np.abs(expect_out_w[i].imag) for i in range(length_w)]) + ) for i in range(length_w): np.testing.assert_allclose( @@ -98,25 +103,37 @@ class TestEigOp(OpTest): exp_w_real[i], rtol=1e-06, atol=1e-05, - err_msg='The eigenvalues real part have diff: \nExpected ' + - str(act_w_real[i]) + '\n' + 'But got: ' + str(exp_w_real[i])) + err_msg='The eigenvalues real part have diff: \nExpected ' + + str(act_w_real[i]) + + '\n' + + 'But got: ' + + str(exp_w_real[i]), + ) np.testing.assert_allclose( act_w_imag[i], exp_w_imag[i], rtol=1e-06, atol=1e-05, - err_msg='The eigenvalues image part have diff: \nExpected ' + - str(act_w_imag[i]) + '\n' + 'But got: ' + str(exp_w_imag[i])) + err_msg='The eigenvalues image part have diff: \nExpected ' + + str(act_w_imag[i]) + + '\n' + + 'But got: ' + + str(exp_w_imag[i]), + ) length_v = len(expect_out_v) act_v_real = np.sort( - np.array([np.abs(actual_out_v[i].real) for i in range(length_v)])) + np.array([np.abs(actual_out_v[i].real) for i in range(length_v)]) + ) act_v_imag = np.sort( - np.array([np.abs(actual_out_v[i].imag) for i in range(length_v)])) + np.array([np.abs(actual_out_v[i].imag) for i in range(length_v)]) + ) exp_v_real = np.sort( - np.array([np.abs(expect_out_v[i].real) for i in range(length_v)])) + np.array([np.abs(expect_out_v[i].real) for i in range(length_v)]) + ) exp_v_imag = np.sort( - np.array([np.abs(expect_out_v[i].imag) for i in range(length_v)])) + np.array([np.abs(expect_out_v[i].imag) for i in range(length_v)]) + ) for i in range(length_v): np.testing.assert_allclose( @@ -124,15 +141,23 @@ class TestEigOp(OpTest): exp_v_real[i], rtol=1e-06, atol=1e-05, - err_msg='The eigenvectors real part have diff: \nExpected ' + - str(act_v_real[i]) + '\n' + 'But got: ' + str(exp_v_real[i])) + err_msg='The eigenvectors real part have diff: \nExpected ' + + str(act_v_real[i]) + + '\n' + + 'But got: ' + + str(exp_v_real[i]), + ) np.testing.assert_allclose( act_v_imag[i], exp_v_imag[i], rtol=1e-06, atol=1e-05, - err_msg='The eigenvectors image part have diff: \nExpected ' + - str(act_v_imag[i]) + '\n' + 'But got: ' + str(exp_v_imag[i])) + err_msg='The eigenvectors image part have diff: \nExpected ' + + str(act_v_imag[i]) + + '\n' + + 'But got: ' + + str(exp_v_imag[i]), + ) def set_dtype(self): self.dtype = np.complex64 @@ -149,32 +174,34 @@ class TestEigOp(OpTest): gtype = np.complex128 self.grad_w = np.ones(self.out[0].shape, gtype) self.grad_v = np.ones(self.out[1].shape, gtype) - self.grad_x = eig_backward(self.out[0], self.out[1], self.grad_w, - self.grad_v) + self.grad_x = eig_backward( + self.out[0], self.out[1], self.grad_w, self.grad_v + ) def test_check_output(self): - self.check_output_with_place_customized(checker=self.checker, - place=core.CPUPlace()) + self.check_output_with_place_customized( + checker=self.checker, place=core.CPUPlace() + ) def test_check_grad(self): self.init_grad() - self.check_grad(['X'], ['Eigenvalues', 'Eigenvectors'], - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_w, self.grad_v]) + self.check_grad( + ['X'], + ['Eigenvalues', 'Eigenvectors'], + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_w, self.grad_v], + ) class TestComplex128(TestEigOp): - def set_dtype(self): self.dtype = np.complex128 @skip_check_grad_ci( - reason= - "For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" + reason="For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" ) class TestDouble(TestEigOp): - def set_dtype(self): self.dtype = np.float64 @@ -183,11 +210,9 @@ class TestDouble(TestEigOp): @skip_check_grad_ci( - reason= - "For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" + reason="For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" ) class TestEigBatchMarices(TestEigOp): - def set_dtype(self): self.dtype = np.float64 @@ -199,11 +224,9 @@ class TestEigBatchMarices(TestEigOp): @skip_check_grad_ci( - reason= - "For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" + reason="For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" ) class TestFloat(TestEigOp): - def set_dtype(self): self.dtype = np.float32 @@ -212,7 +235,6 @@ class TestFloat(TestEigOp): class TestEigStatic(TestEigOp): - def test_check_output_with_place(self): paddle.enable_static() place = core.CPUPlace() @@ -223,28 +245,36 @@ class TestEigStatic(TestEigOp): act_val, act_vec = paddle.linalg.eig(input) exe = fluid.Executor(place) - fetch_val, fetch_vec = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[act_val, act_vec]) + fetch_val, fetch_vec = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[act_val, act_vec], + ) np.testing.assert_allclose( expect_val, fetch_val, rtol=1e-06, atol=1e-06, - err_msg='The eigen values have diff: \nExpected ' + - str(expect_val) + '\n' + 'But got: ' + str(fetch_val)) + err_msg='The eigen values have diff: \nExpected ' + + str(expect_val) + + '\n' + + 'But got: ' + + str(fetch_val), + ) np.testing.assert_allclose( np.abs(expect_vec), np.abs(fetch_vec), rtol=1e-06, atol=1e-06, - err_msg='The eigen vectors have diff: \nExpected ' + - str(np.abs(expect_vec)) + '\n' + 'But got: ' + - str(np.abs(fetch_vec))) + err_msg='The eigen vectors have diff: \nExpected ' + + str(np.abs(expect_vec)) + + '\n' + + 'But got: ' + + str(np.abs(fetch_vec)), + ) class TestEigDyGraph(unittest.TestCase): - def test_check_output_with_place(self): input_np = np.random.random([3, 3]).astype('complex') expect_val, expect_vec = np.linalg.eig(input_np) @@ -260,16 +290,23 @@ class TestEigDyGraph(unittest.TestCase): fetch_val.numpy(), rtol=1e-06, atol=1e-06, - err_msg='The eigen values have diff: \nExpected ' + - str(expect_val) + '\n' + 'But got: ' + str(fetch_val)) + err_msg='The eigen values have diff: \nExpected ' + + str(expect_val) + + '\n' + + 'But got: ' + + str(fetch_val), + ) np.testing.assert_allclose( np.abs(expect_vec), np.abs(fetch_vec.numpy()), rtol=1e-06, atol=1e-06, - err_msg='The eigen vectors have diff: \nExpected ' + - str(np.abs(expect_vec)) + '\n' + 'But got: ' + - str(np.abs(fetch_vec.numpy()))) + err_msg='The eigen vectors have diff: \nExpected ' + + str(np.abs(expect_vec)) + + '\n' + + 'But got: ' + + str(np.abs(fetch_vec.numpy())), + ) def test_check_grad(self): test_shape = [3, 3] @@ -289,17 +326,20 @@ class TestEigDyGraph(unittest.TestCase): w, v = paddle.linalg.eig(x) (w.sum() + v.sum()).backward() - np.testing.assert_allclose(np.abs(x.grad.numpy()), - np.abs(grad_x), - rtol=1e-05, - atol=1e-05, - err_msg='The grad x have diff: \nExpected ' + - str(np.abs(grad_x)) + '\n' + 'But got: ' + - str(np.abs(x.grad.numpy()))) + np.testing.assert_allclose( + np.abs(x.grad.numpy()), + np.abs(grad_x), + rtol=1e-05, + atol=1e-05, + err_msg='The grad x have diff: \nExpected ' + + str(np.abs(grad_x)) + + '\n' + + 'But got: ' + + str(np.abs(x.grad.numpy())), + ) class TestEigWrongDimsError(unittest.TestCase): - def test_error(self): paddle.device.set_device("cpu") paddle.disable_static() @@ -309,7 +349,6 @@ class TestEigWrongDimsError(unittest.TestCase): class TestEigNotSquareError(unittest.TestCase): - def test_error(self): paddle.device.set_device("cpu") paddle.disable_static() @@ -319,7 +358,6 @@ class TestEigNotSquareError(unittest.TestCase): class TestEigUnsupportedDtypeError(unittest.TestCase): - def test_error(self): paddle.device.set_device("cpu") paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_eigh_op.py b/python/paddle/fluid/tests/unittests/test_eigh_op.py index 6118b761283..928234a138f 100644 --- a/python/paddle/fluid/tests/unittests/test_eigh_op.py +++ b/python/paddle/fluid/tests/unittests/test_eigh_op.py @@ -57,8 +57,8 @@ def valid_single_eigh_result(A, eigh_value, eigh_vector, uplo): # ||A - Q*T*Q'|| / (N*||A||) < rtol np.testing.assert_array_less( - np.linalg.norm(residual, np.inf) / (N * np.linalg.norm(A, np.inf)), - rtol) + np.linalg.norm(residual, np.inf) / (N * np.linalg.norm(A, np.inf)), rtol + ) # ||I - Q*Q'|| / M < rtol residual = np.eye(M) - eigh_vector @ np.linalg.inv(eigh_vector) @@ -66,7 +66,6 @@ def valid_single_eigh_result(A, eigh_value, eigh_vector, uplo): class TestEighOp(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "eigh" @@ -94,13 +93,11 @@ class TestEighOp(OpTest): class TestEighUPLOCase(TestEighOp): - def init_config(self): self.UPLO = 'U' class TestEighGPUCase(unittest.TestCase): - def setUp(self): self.x_shape = [32, 32] self.dtype = "float32" @@ -113,19 +110,22 @@ class TestEighGPUCase(unittest.TestCase): paddle.disable_static(place=paddle.CUDAPlace(0)) input_real_data = paddle.to_tensor(self.x_np) actual_w, actual_v = paddle.linalg.eigh(input_real_data, self.UPLO) - valid_eigh_result(self.x_np, actual_w.numpy(), actual_v.numpy(), - self.UPLO) + valid_eigh_result( + self.x_np, actual_w.numpy(), actual_v.numpy(), self.UPLO + ) class TestEighAPI(unittest.TestCase): - def setUp(self): self.init_input_data() self.UPLO = 'L' self.rtol = 1e-5 # for test_eigh_grad self.atol = 1e-5 # for test_eigh_grad - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) np.random.seed(123) def init_input_shape(self): @@ -136,26 +136,31 @@ class TestEighAPI(unittest.TestCase): self.dtype = "float32" self.real_data = np.random.random(self.x_shape).astype(self.dtype) complex_data = np.random.random(self.x_shape).astype( - self.dtype) + 1J * np.random.random(self.x_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.x_shape).astype(self.dtype) self.trans_dims = list(range(len(self.x_shape) - 2)) + [ - len(self.x_shape) - 1, len(self.x_shape) - 2 + len(self.x_shape) - 1, + len(self.x_shape) - 2, ] - #build a random conjugate matrix + # build a random conjugate matrix self.complex_symm = np.divide( - complex_data + np.conj(complex_data.transpose(self.trans_dims)), 2) + complex_data + np.conj(complex_data.transpose(self.trans_dims)), 2 + ) def check_static_float_result(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - input_x = paddle.static.data('input_x', - shape=self.x_shape, - dtype=self.dtype) + input_x = paddle.static.data( + 'input_x', shape=self.x_shape, dtype=self.dtype + ) output_w, output_v = paddle.linalg.eigh(input_x) exe = paddle.static.Executor(self.place) - actual_w, actual_v = exe.run(main_prog, - feed={"input_x": self.real_data}, - fetch_list=[output_w, output_v]) + actual_w, actual_v = exe.run( + main_prog, + feed={"input_x": self.real_data}, + fetch_list=[output_w, output_v], + ) valid_eigh_result(self.real_data, actual_w, actual_v, self.UPLO) def check_static_complex_result(self): @@ -163,14 +168,16 @@ class TestEighAPI(unittest.TestCase): startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): x_dtype = np.complex64 if self.dtype == "float32" else np.complex128 - input_x = paddle.static.data('input_x', - shape=self.x_shape, - dtype=x_dtype) + input_x = paddle.static.data( + 'input_x', shape=self.x_shape, dtype=x_dtype + ) output_w, output_v = paddle.linalg.eigh(input_x) exe = paddle.static.Executor(self.place) - actual_w, actual_v = exe.run(main_prog, - feed={"input_x": self.complex_symm}, - fetch_list=[output_w, output_v]) + actual_w, actual_v = exe.run( + main_prog, + feed={"input_x": self.complex_symm}, + fetch_list=[output_w, output_v], + ) valid_eigh_result(self.complex_symm, actual_w, actual_v, self.UPLO) def test_in_static_mode(self): @@ -182,61 +189,62 @@ class TestEighAPI(unittest.TestCase): paddle.disable_static() input_real_data = paddle.to_tensor(self.real_data) actual_w, actual_v = paddle.linalg.eigh(input_real_data) - valid_eigh_result(self.real_data, actual_w.numpy(), actual_v.numpy(), - self.UPLO) + valid_eigh_result( + self.real_data, actual_w.numpy(), actual_v.numpy(), self.UPLO + ) input_complex_data = paddle.to_tensor(self.complex_symm) actual_w, actual_v = paddle.linalg.eigh(input_complex_data) - valid_eigh_result(self.complex_symm, actual_w.numpy(), actual_v.numpy(), - self.UPLO) + valid_eigh_result( + self.complex_symm, actual_w.numpy(), actual_v.numpy(), self.UPLO + ) def test_eigh_grad(self): paddle.disable_static() x = paddle.to_tensor(self.complex_symm, stop_gradient=False) w, v = paddle.linalg.eigh(x) (w.sum() + paddle.abs(v).sum()).backward() - np.testing.assert_allclose(abs(x.grad.numpy()), - abs(x.grad.numpy().conj().transpose( - self.trans_dims)), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + abs(x.grad.numpy()), + abs(x.grad.numpy().conj().transpose(self.trans_dims)), + rtol=self.rtol, + atol=self.atol, + ) class TestEighBatchAPI(TestEighAPI): - def init_input_shape(self): self.x_shape = [2, 5, 5] class TestEighAPIError(unittest.TestCase): - def test_error(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - #input maxtrix must greater than 2 dimensions - input_x = paddle.static.data(name='x_1', - shape=[12], - dtype='float32') + # input maxtrix must greater than 2 dimensions + input_x = paddle.static.data( + name='x_1', shape=[12], dtype='float32' + ) self.assertRaises(ValueError, paddle.linalg.eigh, input_x) - #input matrix must be square matrix - input_x = paddle.static.data(name='x_2', - shape=[12, 32], - dtype='float32') + # input matrix must be square matrix + input_x = paddle.static.data( + name='x_2', shape=[12, 32], dtype='float32' + ) self.assertRaises(ValueError, paddle.linalg.eigh, input_x) - #uplo must be in 'L' or 'U' - input_x = paddle.static.data(name='x_3', - shape=[4, 4], - dtype="float32") + # uplo must be in 'L' or 'U' + input_x = paddle.static.data( + name='x_3', shape=[4, 4], dtype="float32" + ) uplo = 'R' self.assertRaises(ValueError, paddle.linalg.eigh, input_x, uplo) - #x_data cannot be integer - input_x = paddle.static.data(name='x_4', - shape=[4, 4], - dtype="int32") + # x_data cannot be integer + input_x = paddle.static.data( + name='x_4', shape=[4, 4], dtype="int32" + ) self.assertRaises(TypeError, paddle.linalg.eigh, input_x) diff --git a/python/paddle/fluid/tests/unittests/test_eigvals_op.py b/python/paddle/fluid/tests/unittests/test_eigvals_op.py index a361ad24024..df8d6b001cf 100644 --- a/python/paddle/fluid/tests/unittests/test_eigvals_op.py +++ b/python/paddle/fluid/tests/unittests/test_eigvals_op.py @@ -23,7 +23,7 @@ np.set_printoptions(threshold=np.inf) def np_eigvals(a): res = np.linalg.eigvals(a) - if (a.dtype == np.float32 or a.dtype == np.complex64): + if a.dtype == np.float32 or a.dtype == np.complex64: res = res.astype(np.complex64) else: res = res.astype(np.complex128) @@ -32,7 +32,6 @@ def np_eigvals(a): class TestEigvalsOp(OpTest): - def setUp(self): np.random.seed(0) paddle.enable_static() @@ -53,111 +52,121 @@ class TestEigvalsOp(OpTest): self.input_dims = (5, 5) def set_input_data(self): - if (self.dtype == np.float32 or self.dtype == np.float64): + if self.dtype == np.float32 or self.dtype == np.float64: self.input_data = np.random.random(self.input_dims).astype( - self.dtype) + self.dtype + ) else: - self.input_data = (np.random.random(self.input_dims) + - np.random.random(self.input_dims) * 1j).astype( - self.dtype) + self.input_data = ( + np.random.random(self.input_dims) + + np.random.random(self.input_dims) * 1j + ).astype(self.dtype) def test_check_output(self): self.__class__.no_need_check_grad = True - self.check_output_with_place_customized(checker=self.verify_output, - place=core.CPUPlace()) + self.check_output_with_place_customized( + checker=self.verify_output, place=core.CPUPlace() + ) def verify_output(self, outs): actual_outs = np.sort(np.array(outs[0])) expect_outs = np.sort(np.array(self.outputs['Out'])) self.assertTrue( - actual_outs.shape == expect_outs.shape, "Output shape has diff.\n" - "Expect shape " + str(expect_outs.shape) + "\n" + "But Got" + - str(actual_outs.shape) + " in class " + self.__class__.__name__) + actual_outs.shape == expect_outs.shape, + "Output shape has diff.\n" + "Expect shape " + + str(expect_outs.shape) + + "\n" + + "But Got" + + str(actual_outs.shape) + + " in class " + + self.__class__.__name__, + ) n_dim = actual_outs.shape[-1] - for actual_row, expect_row in zip(actual_outs.reshape((-1, n_dim)), - expect_outs.reshape((-1, n_dim))): - is_mapped_index = np.zeros((n_dim, )) + for actual_row, expect_row in zip( + actual_outs.reshape((-1, n_dim)), expect_outs.reshape((-1, n_dim)) + ): + is_mapped_index = np.zeros((n_dim,)) for i in range(n_dim): is_mapped = False for j in range(n_dim): if is_mapped_index[j] == 0 and np.isclose( - np.array(actual_row[i]), - np.array(expect_row[j]), - atol=1e-5): + np.array(actual_row[i]), + np.array(expect_row[j]), + atol=1e-5, + ): is_mapped_index[j] = True is_mapped = True break self.assertTrue( is_mapped, - "Output has diff in class " + self.__class__.__name__ + - "\nExpect " + str(expect_outs) + "\n" + "But Got" + - str(actual_outs) + "\nThe data " + str(actual_row[i]) + - " in " + str(actual_row) + " mismatch.") + "Output has diff in class " + + self.__class__.__name__ + + "\nExpect " + + str(expect_outs) + + "\n" + + "But Got" + + str(actual_outs) + + "\nThe data " + + str(actual_row[i]) + + " in " + + str(actual_row) + + " mismatch.", + ) class TestEigvalsOpFloat64(TestEigvalsOp): - def set_dtype(self): self.dtype = np.float64 class TestEigvalsOpComplex64(TestEigvalsOp): - def set_dtype(self): self.dtype = np.complex64 class TestEigvalsOpComplex128(TestEigvalsOp): - def set_dtype(self): self.dtype = np.complex128 class TestEigvalsOpLargeScare(TestEigvalsOp): - def set_input_dims(self): self.input_dims = (128, 128) class TestEigvalsOpLargeScareFloat64(TestEigvalsOpLargeScare): - def set_dtype(self): self.dtype = np.float64 class TestEigvalsOpLargeScareComplex64(TestEigvalsOpLargeScare): - def set_dtype(self): self.dtype = np.complex64 class TestEigvalsOpLargeScareComplex128(TestEigvalsOpLargeScare): - def set_dtype(self): self.dtype = np.complex128 class TestEigvalsOpBatch1(TestEigvalsOp): - def set_input_dims(self): self.input_dims = (1, 2, 3, 4, 4) class TestEigvalsOpBatch2(TestEigvalsOp): - def set_input_dims(self): self.input_dims = (3, 1, 4, 5, 5) class TestEigvalsOpBatch3(TestEigvalsOp): - def set_input_dims(self): self.input_dims = (6, 2, 9, 6, 6) class TestEigvalsAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) @@ -183,42 +192,62 @@ class TestEigvalsAPI(unittest.TestCase): self.dtype = np.float32 def set_input_data(self): - if (self.dtype == np.float32 or self.dtype == np.float64): + if self.dtype == np.float32 or self.dtype == np.float64: self.input_data = np.random.random(self.input_dims).astype( - self.dtype) + self.dtype + ) else: - self.input_data = (np.random.random(self.input_dims) + - np.random.random(self.input_dims) * 1j).astype( - self.dtype) + self.input_data = ( + np.random.random(self.input_dims) + + np.random.random(self.input_dims) * 1j + ).astype(self.dtype) def verify_output(self, actural_outs, expect_outs): actual_outs = np.array(actural_outs) expect_outs = np.array(expect_outs) self.assertTrue( - actual_outs.shape == expect_outs.shape, "Output shape has diff." - "\nExpect shape " + str(expect_outs.shape) + "\n" + "But Got" + - str(actual_outs.shape) + " in class " + self.__class__.__name__) + actual_outs.shape == expect_outs.shape, + "Output shape has diff." + "\nExpect shape " + + str(expect_outs.shape) + + "\n" + + "But Got" + + str(actual_outs.shape) + + " in class " + + self.__class__.__name__, + ) n_dim = actual_outs.shape[-1] - for actual_row, expect_row in zip(actual_outs.reshape((-1, n_dim)), - expect_outs.reshape((-1, n_dim))): - is_mapped_index = np.zeros((n_dim, )) + for actual_row, expect_row in zip( + actual_outs.reshape((-1, n_dim)), expect_outs.reshape((-1, n_dim)) + ): + is_mapped_index = np.zeros((n_dim,)) for i in range(n_dim): is_mapped = False for j in range(n_dim): if is_mapped_index[j] == 0 and np.isclose( - np.array(actual_row[i]), - np.array(expect_row[j]), - atol=1e-5): + np.array(actual_row[i]), + np.array(expect_row[j]), + atol=1e-5, + ): is_mapped_index[j] = True is_mapped = True break self.assertTrue( is_mapped, - "Output has diff in class " + self.__class__.__name__ + - "\nExpect " + str(expect_outs) + "\n" + "But Got" + - str(actual_outs) + "\nThe data " + str(actual_row[i]) + - " in " + str(actual_row) + " mismatch.") + "Output has diff in class " + + self.__class__.__name__ + + "\nExpect " + + str(expect_outs) + + "\n" + + "But Got" + + str(actual_outs) + + "\nThe data " + + str(actual_row[i]) + + " in " + + str(actual_row) + + " mismatch.", + ) def run_dygraph(self, place): paddle.disable_static() @@ -241,24 +270,28 @@ class TestEigvalsAPI(unittest.TestCase): def run_static(self, place): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - small_input_tensor = paddle.static.data(name='small_x', - shape=self.small_dims, - dtype=self.dtype) - large_input_tensor = paddle.static.data(name='large_x', - shape=self.large_dims, - dtype=self.dtype) - batch_input_tensor = paddle.static.data(name='batch_x', - shape=self.batch_dims, - dtype=self.dtype) - - small_outs = paddle.linalg.eigvals(small_input_tensor, - name='small_x') - large_outs = paddle.linalg.eigvals(large_input_tensor, - name='large_x') - batch_outs = paddle.linalg.eigvals(batch_input_tensor, - name='batch_x') + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + small_input_tensor = paddle.static.data( + name='small_x', shape=self.small_dims, dtype=self.dtype + ) + large_input_tensor = paddle.static.data( + name='large_x', shape=self.large_dims, dtype=self.dtype + ) + batch_input_tensor = paddle.static.data( + name='batch_x', shape=self.batch_dims, dtype=self.dtype + ) + + small_outs = paddle.linalg.eigvals( + small_input_tensor, name='small_x' + ) + large_outs = paddle.linalg.eigvals( + large_input_tensor, name='large_x' + ) + batch_outs = paddle.linalg.eigvals( + batch_input_tensor, name='batch_x' + ) exe = paddle.static.Executor(place) @@ -266,9 +299,10 @@ class TestEigvalsAPI(unittest.TestCase): feed={ "small_x": self.small_input, "large_x": self.large_input, - "batch_x": self.batch_input + "batch_x": self.batch_input, }, - fetch_list=[small_outs, large_outs, batch_outs]) + fetch_list=[small_outs, large_outs, batch_outs], + ) np_outs = np_eigvals(self.small_input) self.verify_output(paddle_outs[0], np_outs) @@ -281,7 +315,7 @@ class TestEigvalsAPI(unittest.TestCase): def test_cases(self): places = [core.CPUPlace()] - #if core.is_compiled_with_cuda(): + # if core.is_compiled_with_cuda(): # places.append(core.CUDAPlace(0)) for place in places: self.run_dygraph(place) @@ -301,19 +335,16 @@ class TestEigvalsAPI(unittest.TestCase): class TestEigvalsAPIFloat64(TestEigvalsAPI): - def set_dtype(self): self.dtype = np.float64 class TestEigvalsAPIComplex64(TestEigvalsAPI): - def set_dtype(self): self.dtype = np.complex64 class TestEigvalsAPIComplex128(TestEigvalsAPI): - def set_dtype(self): self.dtype = np.complex128 diff --git a/python/paddle/fluid/tests/unittests/test_eigvalsh_op.py b/python/paddle/fluid/tests/unittests/test_eigvalsh_op.py index 08d3da2103c..2dc378a0810 100644 --- a/python/paddle/fluid/tests/unittests/test_eigvalsh_op.py +++ b/python/paddle/fluid/tests/unittests/test_eigvalsh_op.py @@ -34,7 +34,11 @@ def valid_eigenvalues(actual, expected): FP32_MAX_RELATIVE_ERR = 5e-5 FP64_MAX_RELATIVE_ERR = 1e-14 - rtol = FP32_MAX_RELATIVE_ERR if actual.dtype == np.single else FP64_MAX_RELATIVE_ERR + rtol = ( + FP32_MAX_RELATIVE_ERR + if actual.dtype == np.single + else FP64_MAX_RELATIVE_ERR + ) diff = np.abs(expected - actual) max_diff = np.max(diff) @@ -44,7 +48,6 @@ def valid_eigenvalues(actual, expected): class TestEigvalshOp(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "eigvalsh" @@ -75,13 +78,11 @@ class TestEigvalshOp(OpTest): class TestEigvalshUPLOCase(TestEigvalshOp): - def init_config(self): self.UPLO = 'U' class TestEigvalshGPUCase(unittest.TestCase): - def setUp(self): self.x_shape = [32, 32] self.dtype = "float32" @@ -98,14 +99,16 @@ class TestEigvalshGPUCase(unittest.TestCase): class TestEigvalshAPI(unittest.TestCase): - def setUp(self): self.dtype = "float32" self.UPLO = 'L' self.rtol = 1e-5 # test_eigvalsh_grad self.atol = 1e-5 # test_eigvalsh_grad - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) np.random.seed(123) self.init_input_shape() self.init_input_data() @@ -116,25 +119,30 @@ class TestEigvalshAPI(unittest.TestCase): def init_input_data(self): self.real_data = np.random.random(self.x_shape).astype(self.dtype) complex_data = np.random.random(self.x_shape).astype( - self.dtype) + 1J * np.random.random(self.x_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.x_shape).astype(self.dtype) self.trans_dims = list(range(len(self.x_shape) - 2)) + [ - len(self.x_shape) - 1, len(self.x_shape) - 2 + len(self.x_shape) - 1, + len(self.x_shape) - 2, ] self.complex_symm = np.divide( - complex_data + np.conj(complex_data.transpose(self.trans_dims)), 2) + complex_data + np.conj(complex_data.transpose(self.trans_dims)), 2 + ) def check_static_float_result(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - input_x = paddle.static.data('input_x', - shape=self.x_shape, - dtype=self.dtype) + input_x = paddle.static.data( + 'input_x', shape=self.x_shape, dtype=self.dtype + ) output_w = paddle.linalg.eigvalsh(input_x) exe = paddle.static.Executor(self.place) - actual_w = exe.run(main_prog, - feed={"input_x": self.real_data}, - fetch_list=[output_w]) + actual_w = exe.run( + main_prog, + feed={"input_x": self.real_data}, + fetch_list=[output_w], + ) expected_w = np.linalg.eigvalsh(self.real_data) compare_result(actual_w[0], expected_w) @@ -144,14 +152,16 @@ class TestEigvalshAPI(unittest.TestCase): startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): x_dtype = np.complex64 if self.dtype == "float32" else np.complex128 - input_x = paddle.static.data('input_x', - shape=self.x_shape, - dtype=x_dtype) + input_x = paddle.static.data( + 'input_x', shape=self.x_shape, dtype=x_dtype + ) output_w = paddle.linalg.eigvalsh(input_x) exe = paddle.static.Executor(self.place) - actual_w = exe.run(main_prog, - feed={"input_x": self.complex_symm}, - fetch_list=[output_w]) + actual_w = exe.run( + main_prog, + feed={"input_x": self.complex_symm}, + fetch_list=[output_w], + ) expected_w = np.linalg.eigvalsh(self.complex_symm) compare_result(actual_w[0], expected_w) @@ -177,48 +187,47 @@ class TestEigvalshAPI(unittest.TestCase): x = paddle.to_tensor(self.complex_symm, stop_gradient=False) w = paddle.linalg.eigvalsh(x) (w.sum()).backward() - np.testing.assert_allclose(abs(x.grad.numpy()), - abs(x.grad.numpy().conj().transpose( - self.trans_dims)), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + abs(x.grad.numpy()), + abs(x.grad.numpy().conj().transpose(self.trans_dims)), + rtol=self.rtol, + atol=self.atol, + ) class TestEigvalshBatchAPI(TestEigvalshAPI): - def init_input_shape(self): self.x_shape = [2, 5, 5] class TestEigvalshAPIError(unittest.TestCase): - def test_error(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - #input maxtrix must greater than 2 dimensions - input_x = paddle.static.data(name='x_1', - shape=[12], - dtype='float32') + # input maxtrix must greater than 2 dimensions + input_x = paddle.static.data( + name='x_1', shape=[12], dtype='float32' + ) self.assertRaises(ValueError, paddle.linalg.eigvalsh, input_x) - #input matrix must be square matrix - input_x = paddle.static.data(name='x_2', - shape=[12, 32], - dtype='float32') + # input matrix must be square matrix + input_x = paddle.static.data( + name='x_2', shape=[12, 32], dtype='float32' + ) self.assertRaises(ValueError, paddle.linalg.eigvalsh, input_x) - #uplo must be in 'L' or 'U' - input_x = paddle.static.data(name='x_3', - shape=[4, 4], - dtype="float32") + # uplo must be in 'L' or 'U' + input_x = paddle.static.data( + name='x_3', shape=[4, 4], dtype="float32" + ) uplo = 'R' self.assertRaises(ValueError, paddle.linalg.eigvalsh, input_x, uplo) - #x_data cannot be integer - input_x = paddle.static.data(name='x_4', - shape=[4, 4], - dtype="int32") + # x_data cannot be integer + input_x = paddle.static.data( + name='x_4', shape=[4, 4], dtype="int32" + ) self.assertRaises(TypeError, paddle.linalg.eigvalsh, input_x) diff --git a/python/paddle/fluid/tests/unittests/test_einsum.py b/python/paddle/fluid/tests/unittests/test_einsum.py index 40583c01f34..2e8a0a6c7cb 100644 --- a/python/paddle/fluid/tests/unittests/test_einsum.py +++ b/python/paddle/fluid/tests/unittests/test_einsum.py @@ -23,85 +23,109 @@ os.environ['FLAGS_new_einsum'] = "0" class TestErrors(unittest.TestCase): - def setUp(self): pass def test_diagonalize_errors(self): a = np.arange(4 * 3 * 4 * 4).reshape(4, 3, 4, 4).astype('float') a = paddle.to_tensor(a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('...ii->...i', a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('i...i', a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('i...i->i...', a) def test_param_errors(self): a = np.arange(4 * 3 * 4 * 4).reshape(4, 3, 4, 4).astype('float') a = paddle.to_tensor(a) - with self.assertRaisesRegex(AssertionError, - ('At least one operand is expected.')): + with self.assertRaisesRegex( + AssertionError, ('At least one operand is expected.') + ): paddle.einsum('ijk') with self.assertRaisesRegex( - AssertionError, - ('Invalid equation: multiple `->` were found.')): + AssertionError, ('Invalid equation: multiple `->` were found.') + ): paddle.einsum('i -> j -> k', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 2, " - "but found 3 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 2, " + "but found 3 segments in the label equation." + ), + ): paddle.einsum('i,j,k', a, a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 2, " - "but found 1 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 2, " + "but found 1 segments in the label equation." + ), + ): paddle.einsum('ij -> k', a, a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 1, " - "but found 2 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 1, " + "but found 2 segments in the label equation." + ), + ): paddle.einsum('i, -> k', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the label string '' misses dimensions.")): + AssertionError, + ("Invalid equation: the label string '' misses dimensions."), + ): paddle.einsum('->', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the label string 'i' misses dimensions.")): + AssertionError, + ("Invalid equation: the label string 'i' misses dimensions."), + ): paddle.einsum('i', a) with self.assertRaisesRegex( - AssertionError, ("Invalid equation: _ is not a valid label, " - "which should be letters.")): + AssertionError, + ( + "Invalid equation: _ is not a valid label, " + "which should be letters." + ), + ): paddle.einsum('i_', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: `.` is found outside of an ellipsis.")): + AssertionError, + ("Invalid equation: `.` is found outside of an ellipsis."), + ): paddle.einsum('i..j', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: `.` is found outside of an ellipsis.")): + AssertionError, + ("Invalid equation: `.` is found outside of an ellipsis."), + ): paddle.einsum('...k...', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: missing ellipsis in output labels.")): + AssertionError, + ("Invalid equation: missing ellipsis in output labels."), + ): paddle.einsum('i...->i', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: duplicate output labels are found.")): + AssertionError, + ("Invalid equation: duplicate output labels are found."), + ): paddle.einsum('i...->i...i', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid operands: label i " - "corresponds to non-broadcastable dimensions.")): + AssertionError, + ( + "Invalid operands: label i " + "corresponds to non-broadcastable dimensions." + ), + ): paddle.einsum('ij...,ji...', a, a) class TestEinsum(unittest.TestCase): - @classmethod def setUpClass(cls): np.random.seed(12345) @@ -132,15 +156,17 @@ class TestEinsum(unittest.TestCase): return core.CUDAPlace(0) return core.CPUPlace() - def check_output_equal(self, actual, expect, rtol=1.e-5, atol=1.e-8): + def check_output_equal(self, actual, expect, rtol=1.0e-5, atol=1.0e-8): error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' - np.testing.assert_allclose(actual, - expect, - rtol=rtol, - atol=atol, - err_msg=error_msg.format( - paddle.get_device(), expect, actual, - self.__class__.__name__)) + np.testing.assert_allclose( + actual, + expect, + rtol=rtol, + atol=atol, + err_msg=error_msg.format( + paddle.get_device(), expect, actual, self.__class__.__name__ + ), + ) def setUp(self): self.sample = {"paradigm": "i->", "data": ["x"]} @@ -153,7 +179,8 @@ class TestEinsum(unittest.TestCase): equation = self.sample["paradigm"] with paddle.fluid.dygraph.guard( - self._get_place(force_to_use_cpu=False)): + self._get_place(force_to_use_cpu=False) + ): pd_operands = [paddle.to_tensor(operand) for operand in operands] result = paddle.einsum(equation, *pd_operands) self.check_output_equal(result.numpy(), expected_result) @@ -165,163 +192,136 @@ class TestEinsum(unittest.TestCase): class TestEinsumVectorDot(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,i->", "data": ["x", "x"]} class TestEinsumVectorMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,i->i", "data": ["x", "x"]} class TestEinsumVectorOuter(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,j->ij", "data": ["x", "y"]} class TestEinsumMatrixTranspose(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->ji", "data": ["A"]} class TestEinsumMatrixRowSum(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->j", "data": ["A"]} class TestEinsumMatrixColSum(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->i", "data": ["A"]} class TestEinsumMatrixEleMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,ij->ij", "data": ["A", "A"]} class TestEinsumDegenerateMatrixVecMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,j", "data": ["a", "b"]} class TestEinsumMatrixVecMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,j->i", "data": ["A", "x"]} class TestEinsumMatrixMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,kj->ik", "data": ["A", "B"]} class TestEinsumMatrixOuter(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,kl->ijkl", "data": ["A", "C"]} class TestEinsumTensorBMM(TestEinsum): - def setUp(self): self.sample = {"paradigm": "bij,bjk->bik", "data": ["D", "E"]} class TestEinsumTensorContract1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->i", "data": ["D", "A"]} class TestEinsumTensorContract2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,lk->ijl", "data": ["D", "B"]} class TestEinsumTensorContract3(TestEinsum): - def setUp(self): self.sample = {"paradigm": "abcd,dfg->abcfg", "data": ["F", "D"]} class TestEinsumTensorContract4(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->ik", "data": ["D", "A"]} class TestEinsumTensorContract5(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->ij", "data": ["D", "A"]} class TestEinsumTensorContract6(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ik, ijk->j", "data": ["A", "G"]} class TestEinsumTensorContract7(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk, ik->jk", "data": ["G", "A"]} class TestEinsumEllipsis1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i...->...", "data": ["G"]} class TestEinsumEllipsis2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,...i->j...", "data": ["A", "H"]} class TestEinsumEllipsis3(TestEinsum): - def setUp(self): self.sample = {"paradigm": "k...,jk", "data": ["F", "I"]} class TestEinsumTestEinsumBilinear(TestEinsum): - def setUp(self): self.sample = {"paradigm": "bn,anm,bm->ba", "data": ["B", "E", "I"]} class TestEinsumTestEinsumOthers1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijkl, lmn->kmn", "data": ["F", "H"]} class TestEinsumTestEinsumOthers2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijkl, lmn->ijn", "data": ["F", "H"]} class TestEinsumBatch1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "blq,bhlk->bhlqk", "data": ["J", "K"]} class TestNumpyTests(unittest.TestCase): - def setUp(self): pass @@ -333,20 +333,23 @@ class TestNumpyTests(unittest.TestCase): return core.CUDAPlace(0) return core.CPUPlace() - def check_output_equal(self, actual, expect, rtol=1.e-5, atol=1.e-8): + def check_output_equal(self, actual, expect, rtol=1.0e-5, atol=1.0e-8): error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' - np.testing.assert_allclose(actual, - expect, - rtol=rtol, - atol=atol, - err_msg=error_msg.format( - paddle.get_device(), expect, actual, - self.__class__.__name__)) + np.testing.assert_allclose( + actual, + expect, + rtol=rtol, + atol=atol, + err_msg=error_msg.format( + paddle.get_device(), expect, actual, self.__class__.__name__ + ), + ) def check_output(self, eqn, *ops): expect = np.einsum(eqn, *ops) with paddle.fluid.dygraph.guard( - self._get_place(force_to_use_cpu=False)): + self._get_place(force_to_use_cpu=False) + ): pd_operands = [paddle.to_tensor(op) for op in ops] actual = paddle.einsum(eqn, *pd_operands) self.check_output_equal(actual.numpy(), expect) @@ -421,8 +424,8 @@ class TestNumpyTests(unittest.TestCase): q = np.ones((1, 2)).astype('float') self.check_output('ij,ij->j', p, q) - x = np.array([2., 3.]).astype('float') - y = np.array([4.]).astype('float') + x = np.array([2.0, 3.0]).astype('float') + y = np.array([4.0]).astype('float') self.check_output("i, i", x, y) p = np.ones((1, 5)) / 2 @@ -453,21 +456,21 @@ class TestNumpyTests(unittest.TestCase): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): - a = paddle.static.data(name='a', - shape=[3, None, None, None], - dtype='float') - b = paddle.static.data(name='b', - shape=[2, None, None, None], - dtype='float') - c = paddle.static.data(name='c', - shape=[None, None, 2, None], - dtype='float') - d = paddle.static.data(name='d', - shape=[None, None, 5], - dtype='float') - e = paddle.static.data(name='e', - shape=[None, 2, None], - dtype='float') + a = paddle.static.data( + name='a', shape=[3, None, None, None], dtype='float' + ) + b = paddle.static.data( + name='b', shape=[2, None, None, None], dtype='float' + ) + c = paddle.static.data( + name='c', shape=[None, None, 2, None], dtype='float' + ) + d = paddle.static.data( + name='d', shape=[None, None, 5], dtype='float' + ) + e = paddle.static.data( + name='e', shape=[None, 2, None], dtype='float' + ) outs = [] outs.append(paddle.einsum("ibnd,jbnd->bnij", a, b)) diff --git a/python/paddle/fluid/tests/unittests/test_einsum_op.py b/python/paddle/fluid/tests/unittests/test_einsum_op.py index f3f7b909366..9db367a2335 100644 --- a/python/paddle/fluid/tests/unittests/test_einsum_op.py +++ b/python/paddle/fluid/tests/unittests/test_einsum_op.py @@ -19,7 +19,6 @@ from op_test import OpTest class TestEinsumBinary(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "einsum" @@ -34,12 +33,15 @@ class TestEinsumBinary(OpTest): self.inputs = {"Operands": self.operands} self.attrs = {"equation": self.equation} self.outputs = { - 'Out': - out, - "InnerCache": [('cache_' + str(i), np.array([1.0])) - for i in range(len(self.operands))], - "XShape": [('xshape_' + str(i), np.array([1.0])) - for i in range(len(self.operands))], + 'Out': out, + "InnerCache": [ + ('cache_' + str(i), np.array([1.0])) + for i in range(len(self.operands)) + ], + "XShape": [ + ('xshape_' + str(i), np.array([1.0])) + for i in range(len(self.operands)) + ], } def init_input(self): @@ -62,7 +64,6 @@ class TestEinsumBinary(OpTest): class TestEinsum1(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(20, 3, 3), (20, 3, 3)] self.types = [np.float64, np.float64] @@ -70,7 +71,6 @@ class TestEinsum1(TestEinsumBinary): class TestEinsum2(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(20, 3, 3), (20, 3, 3)] self.types = [np.float64, np.float64] @@ -78,7 +78,6 @@ class TestEinsum2(TestEinsumBinary): class TestEinsum3(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 10), (10, 10)] self.types = [np.float64, np.float64] @@ -86,7 +85,6 @@ class TestEinsum3(TestEinsumBinary): class TestEinsumWithReduction(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 3, 5), (5, 30)] self.types = [np.float64, np.float64] @@ -94,7 +92,6 @@ class TestEinsumWithReduction(TestEinsumBinary): class TestEinsumWithReduction1(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 3, 3, 5), (10, 5, 10, 10)] self.types = [np.float64, np.float64] @@ -102,7 +99,6 @@ class TestEinsumWithReduction1(TestEinsumBinary): class TestEinsumWithUnary(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 10, 3, 5)] self.types = [np.float64] @@ -110,7 +106,6 @@ class TestEinsumWithUnary(TestEinsumBinary): class TestEinsumWithUnary1(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(5, 10, 3, 3), (3, 6, 3, 10)] self.types = [np.float64, np.float64] @@ -118,7 +113,6 @@ class TestEinsumWithUnary1(TestEinsumBinary): class TestEinsumWithBroadcast1(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(5, 10, 3, 3)] self.types = [np.float64] @@ -126,7 +120,6 @@ class TestEinsumWithBroadcast1(TestEinsumBinary): class TestEinsumWithBroadcast2(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 11), (3, 4, 5, 10)] self.types = [np.float64, np.float64] @@ -134,7 +127,6 @@ class TestEinsumWithBroadcast2(TestEinsumBinary): class TestEinsumWithBroadcast3(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 3, 2, 3, 4), (12, 10)] self.types = [np.float64, np.float64] @@ -142,7 +134,6 @@ class TestEinsumWithBroadcast3(TestEinsumBinary): class TestEinsumWithBroadcast4(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 3, 2, 3, 4), (12, 10)] self.types = [np.float64, np.float64] @@ -150,7 +141,6 @@ class TestEinsumWithBroadcast4(TestEinsumBinary): class TestEinsumWithBroadcast5(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(3, 2, 2, 10), (10, 3, 2, 2)] self.types = [np.float64, np.float64] @@ -158,7 +148,6 @@ class TestEinsumWithBroadcast5(TestEinsumBinary): class TestEinsumWithBroadcast6(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(100), (100)] self.types = [np.float64, np.float64] diff --git a/python/paddle/fluid/tests/unittests/test_einsum_v2.py b/python/paddle/fluid/tests/unittests/test_einsum_v2.py index 0cfe6bdece9..e7b041124c2 100644 --- a/python/paddle/fluid/tests/unittests/test_einsum_v2.py +++ b/python/paddle/fluid/tests/unittests/test_einsum_v2.py @@ -31,91 +31,117 @@ def error_trans(func, *args, **kargs): out = func(*args, **kargs) except ValueError as e: if "Same label have different shapes" in str(e): - raise AssertionError("Invalid operands: label i " - "corresponds to non-broadcastable dimensions.") + raise AssertionError( + "Invalid operands: label i " + "corresponds to non-broadcastable dimensions." + ) class TestErrors(unittest.TestCase): - def setUp(self): pass def test_diagonalize_errors(self): a = np.arange(4 * 3 * 4 * 4).reshape(4, 3, 4, 4).astype('float') a = paddle.to_tensor(a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('...ii->...i', a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('i...i', a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('i...i->i...', a) def test_param_errors(self): a = np.arange(4 * 3 * 4 * 4).reshape(4, 3, 4, 4).astype('float') a = paddle.to_tensor(a) with self.assertRaisesRegex( - AssertionError, - ("Required at least one operand in Einsum API, but received 0 ")): + AssertionError, + ("Required at least one operand in Einsum API, but received 0 "), + ): paddle.einsum('ijk') with self.assertRaisesRegex( - AssertionError, - ('Invalid equation: multiple `->` were found.')): + AssertionError, ('Invalid equation: multiple `->` were found.') + ): paddle.einsum('i -> j -> k', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 2, " - "but found 3 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 2, " + "but found 3 segments in the label equation." + ), + ): paddle.einsum('i,j,k', a, a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 2, " - "but found 1 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 2, " + "but found 1 segments in the label equation." + ), + ): paddle.einsum('ij -> k', a, a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 1, " - "but found 2 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 1, " + "but found 2 segments in the label equation." + ), + ): paddle.einsum('i, -> k', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the label string '' misses dimensions.")): + AssertionError, + ("Invalid equation: the label string '' misses dimensions."), + ): paddle.einsum('->', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the label string 'i' misses dimensions.")): + AssertionError, + ("Invalid equation: the label string 'i' misses dimensions."), + ): paddle.einsum('i', a) with self.assertRaisesRegex( - AssertionError, ("Invalid equation: _ is not a valid label, " - "which should be letters.")): + AssertionError, + ( + "Invalid equation: _ is not a valid label, " + "which should be letters." + ), + ): paddle.einsum('i_', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: `.` is found outside of an ellipsis.")): + AssertionError, + ("Invalid equation: `.` is found outside of an ellipsis."), + ): paddle.einsum('i..j', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: `.` is found outside of an ellipsis.")): + AssertionError, + ("Invalid equation: `.` is found outside of an ellipsis."), + ): paddle.einsum('...k...', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: missing ellipsis in output labels.")): + AssertionError, + ("Invalid equation: missing ellipsis in output labels."), + ): paddle.einsum('i...->i', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: duplicate output labels are found.")): + AssertionError, + ("Invalid equation: duplicate output labels are found."), + ): paddle.einsum('i...->i...i', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid operands: label i " - "corresponds to non-broadcastable dimensions.")): + AssertionError, + ( + "Invalid operands: label i " + "corresponds to non-broadcastable dimensions." + ), + ): error_trans(paddle.einsum, 'ij...,ji...', a, a) class TestEinsum(unittest.TestCase): - @classmethod def setUpClass(cls): np.random.seed(12345) @@ -146,15 +172,17 @@ class TestEinsum(unittest.TestCase): return core.CUDAPlace(0) return core.CPUPlace() - def check_output_equal(self, actual, expect, rtol=1.e-5, atol=1.e-8): + def check_output_equal(self, actual, expect, rtol=1.0e-5, atol=1.0e-8): error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' - np.testing.assert_allclose(actual, - expect, - rtol=rtol, - atol=atol, - err_msg=error_msg.format( - paddle.get_device(), expect, actual, - self.__class__.__name__)) + np.testing.assert_allclose( + actual, + expect, + rtol=rtol, + atol=atol, + err_msg=error_msg.format( + paddle.get_device(), expect, actual, self.__class__.__name__ + ), + ) def setUp(self): self.sample = {"paradigm": "i->", "data": ["x"]} @@ -167,7 +195,8 @@ class TestEinsum(unittest.TestCase): equation = self.sample["paradigm"] with paddle.fluid.dygraph.guard( - self._get_place(force_to_use_cpu=False)): + self._get_place(force_to_use_cpu=False) + ): pd_operands = [paddle.to_tensor(operand) for operand in operands] result = paddle.einsum(equation, *pd_operands) self.check_output_equal(result.numpy(), expected_result) @@ -179,163 +208,136 @@ class TestEinsum(unittest.TestCase): class TestEinsumVectorDot(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,i->", "data": ["x", "x"]} class TestEinsumVectorMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,i->i", "data": ["x", "x"]} class TestEinsumVectorOuter(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,j->ij", "data": ["x", "y"]} class TestEinsumMatrixTranspose(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->ji", "data": ["A"]} class TestEinsumMatrixRowSum(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->j", "data": ["A"]} class TestEinsumMatrixColSum(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->i", "data": ["A"]} class TestEinsumMatrixEleMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,ij->ij", "data": ["A", "A"]} class TestEinsumDegenerateMatrixVecMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,j", "data": ["a", "b"]} class TestEinsumMatrixVecMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,j->i", "data": ["A", "x"]} class TestEinsumMatrixMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,kj->ik", "data": ["A", "B"]} class TestEinsumMatrixOuter(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,kl->ijkl", "data": ["A", "C"]} class TestEinsumTensorBMM(TestEinsum): - def setUp(self): self.sample = {"paradigm": "bij,bjk->bik", "data": ["D", "E"]} class TestEinsumTensorContract1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->i", "data": ["D", "A"]} class TestEinsumTensorContract2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,lk->ijl", "data": ["D", "B"]} class TestEinsumTensorContract3(TestEinsum): - def setUp(self): self.sample = {"paradigm": "abcd,dfg->abcfg", "data": ["F", "D"]} class TestEinsumTensorContract4(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->ik", "data": ["D", "A"]} class TestEinsumTensorContract5(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->ij", "data": ["D", "A"]} class TestEinsumTensorContract6(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ik, ijk->j", "data": ["A", "G"]} class TestEinsumTensorContract7(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk, ik->jk", "data": ["G", "A"]} class TestEinsumEllipsis1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i...->...", "data": ["G"]} class TestEinsumEllipsis2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,...i->j...", "data": ["A", "H"]} class TestEinsumEllipsis3(TestEinsum): - def setUp(self): self.sample = {"paradigm": "k...,jk", "data": ["F", "I"]} class TestEinsumTestEinsumBilinear(TestEinsum): - def setUp(self): self.sample = {"paradigm": "bn,anm,bm->ba", "data": ["B", "E", "I"]} class TestEinsumTestEinsumOthers1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijkl, lmn->kmn", "data": ["F", "H"]} class TestEinsumTestEinsumOthers2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijkl, lmn->ijn", "data": ["F", "H"]} class TestEinsumBatch1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "blq,bhlk->bhlqk", "data": ["J", "K"]} class TestNumpyTests(unittest.TestCase): - def setUp(self): pass @@ -347,20 +349,23 @@ class TestNumpyTests(unittest.TestCase): return core.CUDAPlace(0) return core.CPUPlace() - def check_output_equal(self, actual, expect, rtol=1.e-5, atol=1.e-8): + def check_output_equal(self, actual, expect, rtol=1.0e-5, atol=1.0e-8): error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' - np.testing.assert_allclose(actual, - expect, - rtol=rtol, - atol=atol, - err_msg=error_msg.format( - paddle.get_device(), expect, actual, - self.__class__.__name__)) + np.testing.assert_allclose( + actual, + expect, + rtol=rtol, + atol=atol, + err_msg=error_msg.format( + paddle.get_device(), expect, actual, self.__class__.__name__ + ), + ) def check_output(self, eqn, *ops): expect = np.einsum(eqn, *ops) with paddle.fluid.dygraph.guard( - self._get_place(force_to_use_cpu=False)): + self._get_place(force_to_use_cpu=False) + ): pd_operands = [paddle.to_tensor(op) for op in ops] actual = paddle.einsum(eqn, *pd_operands) self.check_output_equal(actual.numpy(), expect) @@ -432,20 +437,20 @@ class TestNumpyTests(unittest.TestCase): self.check_output("i,i", a, a) # TODO(@xiongkun): explict broadcast in EinsumOp is not supported, it's not recommend to use einsum like this. - #p = np.ones((10, 2)).astype('float') - #q = np.ones((1, 2)).astype('float') - #self.check_output('ij,ij->j', p, q) + # p = np.ones((10, 2)).astype('float') + # q = np.ones((1, 2)).astype('float') + # self.check_output('ij,ij->j', p, q) # TODO(@xiongkun): explict-label-broadcast in EinsumOp is not supported, it's not recommend to use einsum like this. - #x = np.array([2., 3.]).astype('float') - #y = np.array([4.]).astype('float') - #self.check_output("i, i", x, y) + # x = np.array([2., 3.]).astype('float') + # y = np.array([4.]).astype('float') + # self.check_output("i, i", x, y) # TODO(@xiongkun): explict-label-broadcast in EinsumOp is not supported, it's not recommend to use einsum like this. - #p = np.ones((1, 5)) / 2 - #q = np.ones((5, 5)) / 2 - #self.check_output("...ij,...jk->...ik", p, p) - #self.check_output("...ij,...jk->...ik", p, q) + # p = np.ones((1, 5)) / 2 + # q = np.ones((5, 5)) / 2 + # self.check_output("...ij,...jk->...ik", p, p) + # self.check_output("...ij,...jk->...ik", p, q) x = np.eye(2).astype('float') y = np.ones(2).astype('float') @@ -456,11 +461,11 @@ class TestNumpyTests(unittest.TestCase): def test_large_nops(self): pass # TODO(@xiongkun): explict broadcast in EinsumOp is not supported, it's not recommend to use einsum like this. - #a = np.arange(4 * 3 * 1 * 4).reshape(4, 3, 1, 4).astype('float') - #self.check_output('a...b,b...c,c...d', a, a, a) - #self.check_output('a...b,b...c,c...a', a, a, a) - #self.check_output('a...b,b...c,c...a', a, a, a) - #self.check_output('...ab,...ba,...ab,...ab', a, a, a, a) + # a = np.arange(4 * 3 * 1 * 4).reshape(4, 3, 1, 4).astype('float') + # self.check_output('a...b,b...c,c...d', a, a, a) + # self.check_output('a...b,b...c,c...a', a, a, a) + # self.check_output('a...b,b...c,c...a', a, a, a) + # self.check_output('...ab,...ba,...ab,...ab', a, a, a, a) def test_static_graph(self): paddle.enable_static() @@ -472,21 +477,21 @@ class TestNumpyTests(unittest.TestCase): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): - a = paddle.static.data(name='a', - shape=[3, None, None, None], - dtype='float') - b = paddle.static.data(name='b', - shape=[2, None, None, None], - dtype='float') - c = paddle.static.data(name='c', - shape=[None, None, 2, None], - dtype='float') - d = paddle.static.data(name='d', - shape=[None, None, 5], - dtype='float') - e = paddle.static.data(name='e', - shape=[None, 2, None], - dtype='float') + a = paddle.static.data( + name='a', shape=[3, None, None, None], dtype='float' + ) + b = paddle.static.data( + name='b', shape=[2, None, None, None], dtype='float' + ) + c = paddle.static.data( + name='c', shape=[None, None, 2, None], dtype='float' + ) + d = paddle.static.data( + name='d', shape=[None, None, 5], dtype='float' + ) + e = paddle.static.data( + name='e', shape=[None, 2, None], dtype='float' + ) outs = [] outs.append(paddle.einsum("ibnd,jbnd->bnij", a, b)) @@ -514,7 +519,6 @@ class TestNumpyTests(unittest.TestCase): class TestStaticGraphShape(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -528,9 +532,11 @@ class TestStaticGraphShape(unittest.TestCase): self.assertEqual(C.shape, (-1, 384)) -@unittest.skipIf(not core.is_compiled_with_cuda() - or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA or not support the bfloat16") +@unittest.skipIf( + not core.is_compiled_with_cuda() + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "core is not compiled with CUDA or not support the bfloat16", +) class TestBF16(unittest.TestCase): """ EinsumOp support bfloat16 type, add unittest here for the correctness. @@ -539,8 +545,7 @@ class TestBF16(unittest.TestCase): def test_shape(self): cuda_major = paddle.version.cuda().split('.')[0].strip() if int(cuda_major) >= 11: - """ MatmulKernel support bfloat16 only if cuda_major > 11.0. - """ + """MatmulKernel support bfloat16 only if cuda_major > 11.0.""" A = paddle.to_tensor(np.array([1.0, 2.0])).astype(paddle.bfloat16) A = A.cuda() B = paddle.to_tensor(np.array([2.0, 3.0])).astype(paddle.bfloat16) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py index f99f8ea1c8e..ae516bc44ca 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py @@ -16,14 +16,17 @@ import unittest import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + skip_check_grad_ci, + convert_float_to_uint16, +) import paddle.fluid as fluid from paddle.fluid import Program, program_guard from paddle.fluid.framework import _test_eager_guard class TestElementwiseAddOp(OpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -37,47 +40,55 @@ class TestElementwiseAddOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} def check_eager(self): - return (self.use_mkldnn == False and self.axis == -1) + return self.use_mkldnn == False and self.axis == -1 def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output(check_dygraph=(self.use_mkldnn == False), - check_eager=self.check_eager()) + self.check_output( + check_dygraph=(self.use_mkldnn == False), + check_eager=self.check_eager(), + ) def test_check_grad_normal(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return - self.check_grad(['X', 'Y'], - 'Out', - check_dygraph=(self.use_mkldnn == False), - check_eager=self.check_eager()) + self.check_grad( + ['X', 'Y'], + 'Out', + check_dygraph=(self.use_mkldnn == False), + check_eager=self.check_eager(), + ) def test_check_grad_ingore_x(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - check_dygraph=(self.use_mkldnn == False), - check_eager=self.check_eager()) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + check_dygraph=(self.use_mkldnn == False), + check_eager=self.check_eager(), + ) def test_check_grad_ingore_y(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - check_dygraph=(self.use_mkldnn == False), - check_eager=self.check_eager()) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + check_dygraph=(self.use_mkldnn == False), + check_eager=self.check_eager(), + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -91,10 +102,10 @@ class TestElementwiseAddOp(OpTest): self.axis = -1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16ElementwiseAddOp(TestElementwiseAddOp): - def init_dtype(self): self.dtype = np.float16 @@ -104,16 +115,17 @@ class TestFP16ElementwiseAddOp(TestElementwiseAddOp): place = core.CUDAPlace(0) if core.is_float16_supported(place): self.check_output_with_place( - place, atol=1e-3, check_dygraph=(self.use_mkldnn == False)) + place, atol=1e-3, check_dygraph=(self.use_mkldnn == False) + ) @unittest.skipIf( - not core.is_compiled_with_cuda() or core.cudnn_version() < 8100 + not core.is_compiled_with_cuda() + or core.cudnn_version() < 8100 or paddle.device.cuda.get_device_capability()[0] < 8, - "only support compiled with CUDA and cudnn version need larger than 8.1.0 and device's compute capability is at least 8.0" + "only support compiled with CUDA and cudnn version need larger than 8.1.0 and device's compute capability is at least 8.0", ) class TestBF16ElementwiseAddOp(OpTest): - def setUp(self): self.op_type = "elementwise_add" self.dtype = np.uint16 @@ -125,9 +137,12 @@ class TestBF16ElementwiseAddOp(OpTest): self.axis = -1 self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(convert_float_to_uint16(self.x)), - 'Y': OpTest.np_dtype_to_fluid_dtype(convert_float_to_uint16(self.y)) + 'X': OpTest.np_dtype_to_fluid_dtype( + convert_float_to_uint16(self.x) + ), + 'Y': OpTest.np_dtype_to_fluid_dtype( + convert_float_to_uint16(self.y) + ), } self.attrs = {'axis': self.axis, 'use_mkldnn': False} self.outputs = {'Out': convert_float_to_uint16(self.out)} @@ -142,23 +157,21 @@ class TestBF16ElementwiseAddOp(OpTest): def test_check_grad_ingore_x(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - no_grad_set=set("X"), - check_eager=False) + self.check_grad_with_place( + place, ['Y'], 'Out', no_grad_set=set("X"), check_eager=False + ) def test_check_grad_ingore_y(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - no_grad_set=set('Y'), - check_eager=False) + self.check_grad_with_place( + place, ['X'], 'Out', no_grad_set=set('Y'), check_eager=False + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -166,9 +179,9 @@ class TestElementwiseAddOp_scalar(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -176,9 +189,9 @@ class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -186,9 +199,9 @@ class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -196,23 +209,20 @@ class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestFP16ElementwiseAddOp_Vector(TestFP16ElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -223,7 +233,6 @@ class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -234,7 +243,6 @@ class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -245,7 +253,6 @@ class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -256,7 +263,6 @@ class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -264,7 +270,6 @@ class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -272,7 +277,6 @@ class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 1).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -283,7 +287,6 @@ class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -294,7 +297,6 @@ class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -305,7 +307,6 @@ class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -316,7 +317,6 @@ class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -324,7 +324,6 @@ class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -332,7 +331,6 @@ class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -340,7 +338,6 @@ class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) @@ -348,7 +345,6 @@ class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -356,7 +352,6 @@ class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -367,7 +362,6 @@ class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -378,9 +372,9 @@ class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -391,9 +385,9 @@ class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -404,7 +398,6 @@ class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -415,7 +408,6 @@ class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -426,7 +418,6 @@ class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -437,7 +428,6 @@ class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -448,7 +438,6 @@ class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -459,7 +448,6 @@ class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -470,7 +458,6 @@ class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 1, 12).astype(self.dtype) self.y = np.random.rand(10, 2, 12).astype(self.dtype) @@ -481,14 +468,15 @@ class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, y1) # the input dtype of elementwise_add must be float16 or float32 or float64 or int32 or int64 @@ -499,7 +487,6 @@ class TestElementwiseAddOpError(unittest.TestCase): class TestAddApi(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.add(x, y, name) @@ -517,7 +504,7 @@ class TestAddApi(unittest.TestCase): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -527,7 +514,7 @@ class TestAddApi(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -538,18 +525,16 @@ class TestAddApi(unittest.TestCase): y = fluid.dygraph.to_variable(np_y) z = self._executed_api(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) class TestAddInplaceApi(TestAddApi): - def _executed_api(self, x, y, name=None): return x.add_(y, name) class TestAddInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float') self.y_numpy = np.random.rand(3, 4).astype('float') @@ -566,21 +551,18 @@ class TestAddInplaceBroadcastSuccess(unittest.TestCase): class TestAddInplaceBroadcastSuccess2(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') self.y_numpy = np.random.rand(3, 1).astype('float') class TestAddInplaceBroadcastSuccess3(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') class TestAddInplaceBroadcastError(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(3, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') @@ -599,21 +581,18 @@ class TestAddInplaceBroadcastError(unittest.TestCase): class TestAddInplaceBroadcastError2(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestAddInplaceBroadcastError3(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestComplexElementwiseAddOp(OpTest): - def setUp(self): self.op_type = "elementwise_add" self.dtype = np.float64 @@ -623,7 +602,7 @@ class TestComplexElementwiseAddOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -633,14 +612,17 @@ class TestComplexElementwiseAddOp(OpTest): def init_input_output(self): self.x = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.y = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = self.x + self.y def init_grad_input_output(self): - self.grad_out = np.ones( - self.shape, self.dtype) + 1J * np.ones(self.shape, self.dtype) + self.grad_out = np.ones(self.shape, self.dtype) + 1j * np.ones( + self.shape, self.dtype + ) self.grad_x = self.grad_out self.grad_y = self.grad_out @@ -648,43 +630,49 @@ class TestComplexElementwiseAddOp(OpTest): self.check_output(check_eager=False) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) class TestRealComplexElementwiseAddOp(TestComplexElementwiseAddOp): - def init_input_output(self): self.x = np.random.random(self.shape).astype(self.dtype) self.y = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = self.x + self.y def init_grad_input_output(self): - self.grad_out = np.ones( - self.shape, self.dtype) + 1J * np.ones(self.shape, self.dtype) + self.grad_out = np.ones(self.shape, self.dtype) + 1j * np.ones( + self.shape, self.dtype + ) self.grad_x = np.real(self.grad_out) self.grad_y = self.grad_out class TestBoolAddFloatElementwiseAddop(unittest.TestCase): - def test_static_add(self): paddle.enable_static() a = 1.5 diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py index 3cea2fdf5cd..16bf0df5af3 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py @@ -21,7 +21,6 @@ from paddle.fluid import core class ElementwiseDivOp(OpTest): - def setUp(self): self.op_type = "elementwise_div" self.python_api = paddle.divide @@ -34,8 +33,9 @@ class ElementwiseDivOp(OpTest): out = self.compute_output(x, y).astype(self.val_dtype) grad_out = np.ones(out.shape).astype(self.val_dtype) grad_x = self.compute_gradient_x(grad_out, y).astype(self.val_dtype) - grad_y = self.compute_gradient_y(grad_out, out, - y).astype(self.val_dtype) + grad_y = self.compute_gradient_y(grad_out, out, y).astype( + self.val_dtype + ) # Convert np.float32 data to np.uint16 for bfloat16 Paddle OP if self.dtype == np.uint16: @@ -84,28 +84,26 @@ class ElementwiseDivOp(OpTest): def test_check_gradient(self): check_list = [] - check_list.append({ - 'grad': ['X', 'Y'], - 'no_grad': None, - 'val_grad': [self.grad_x, self.grad_y] - }) - check_list.append({ - 'grad': ['Y'], - 'no_grad': set('X'), - 'val_grad': [self.grad_y] - }) - check_list.append({ - 'grad': ['X'], - 'no_grad': set('Y'), - 'val_grad': [self.grad_x] - }) + check_list.append( + { + 'grad': ['X', 'Y'], + 'no_grad': None, + 'val_grad': [self.grad_x, self.grad_y], + } + ) + check_list.append( + {'grad': ['Y'], 'no_grad': set('X'), 'val_grad': [self.grad_y]} + ) + check_list.append( + {'grad': ['X'], 'no_grad': set('Y'), 'val_grad': [self.grad_x]} + ) for check_option in check_list: check_args = [check_option['grad'], 'Out'] check_kwargs = { 'no_grad_set': check_option['no_grad'], 'user_defined_grads': check_option['val_grad'], 'user_defined_grad_outputs': [self.grad_out], - 'check_dygraph': self.check_dygraph + 'check_dygraph': self.check_dygraph, } if self.place is None: self.check_grad(*check_args, **check_kwargs) @@ -114,11 +112,12 @@ class ElementwiseDivOp(OpTest): self.check_grad_with_place(*check_args, **check_kwargs) -@unittest.skipIf(not core.is_compiled_with_cuda() - or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA or not support the bfloat16") +@unittest.skipIf( + not core.is_compiled_with_cuda() + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "core is not compiled with CUDA or not support the bfloat16", +) class TestElementwiseDivOpBF16(ElementwiseDivOp): - def init_args(self): # In due to output data type inconsistence of bfloat16 paddle op, we disable the dygraph check. self.check_dygraph = False @@ -134,9 +133,9 @@ class TestElementwiseDivOpBF16(ElementwiseDivOp): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseDivOpScalar(ElementwiseDivOp): - def init_shape(self): self.x_shape = [20, 3, 4] self.y_shape = [1] @@ -146,14 +145,12 @@ class TestElementwiseDivOpScalar(ElementwiseDivOp): class TestElementwiseDivOpVector(ElementwiseDivOp): - def init_shape(self): self.x_shape = [100] self.y_shape = [100] class TestElementwiseDivOpBroadcast0(ElementwiseDivOp): - def init_shape(self): self.x_shape = [100, 3, 4] self.y_shape = [100] @@ -170,7 +167,6 @@ class TestElementwiseDivOpBroadcast0(ElementwiseDivOp): class TestElementwiseDivOpBroadcast1(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 100, 4] self.y_shape = [100] @@ -187,7 +183,6 @@ class TestElementwiseDivOpBroadcast1(ElementwiseDivOp): class TestElementwiseDivOpBroadcast2(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 3, 100] self.y_shape = [100] @@ -203,7 +198,6 @@ class TestElementwiseDivOpBroadcast2(ElementwiseDivOp): class TestElementwiseDivOpBroadcast3(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 10, 12, 5] self.y_shape = [10, 12] @@ -216,12 +210,12 @@ class TestElementwiseDivOpBroadcast3(ElementwiseDivOp): return grad_out / y.reshape(1, 10, 12, 1) def compute_gradient_y(self, grad_out, out, y): - return np.sum(-1 * grad_out * out / y.reshape(1, 10, 12, 1), - axis=(0, 3)) + return np.sum( + -1 * grad_out * out / y.reshape(1, 10, 12, 1), axis=(0, 3) + ) class TestElementwiseDivOpBroadcast4(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 3, 50] self.y_shape = [2, 1, 50] @@ -231,7 +225,6 @@ class TestElementwiseDivOpBroadcast4(ElementwiseDivOp): class TestElementwiseDivOpBroadcast5(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 3, 4, 20] self.y_shape = [2, 3, 1, 20] @@ -241,7 +234,6 @@ class TestElementwiseDivOpBroadcast5(ElementwiseDivOp): class TestElementwiseDivOpCommonuse1(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 3, 100] self.y_shape = [1, 1, 100] @@ -251,7 +243,6 @@ class TestElementwiseDivOpCommonuse1(ElementwiseDivOp): class TestElementwiseDivOpCommonuse2(ElementwiseDivOp): - def init_shape(self): self.x_shape = [30, 3, 1, 5] self.y_shape = [30, 1, 4, 1] @@ -264,7 +255,6 @@ class TestElementwiseDivOpCommonuse2(ElementwiseDivOp): class TestElementwiseDivOpXsizeLessThanYsize(ElementwiseDivOp): - def init_shape(self): self.x_shape = [10, 12] self.y_shape = [2, 3, 10, 12] @@ -275,7 +265,6 @@ class TestElementwiseDivOpXsizeLessThanYsize(ElementwiseDivOp): class TestElementwiseDivOpInt(ElementwiseDivOp): - def init_dtype(self): self.dtype = np.int32 self.val_dtype = np.int32 @@ -287,32 +276,30 @@ class TestElementwiseDivOpInt(ElementwiseDivOp): return x // y -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestElementwiseDivOpFp16(ElementwiseDivOp): - def init_dtype(self): self.dtype = np.float16 self.val_dtype = np.float16 class TestElementwiseDivBroadcast(unittest.TestCase): - def test_shape_with_batch_sizes(self): with fluid.program_guard(fluid.Program()): - x_var = fluid.data(name='x', - dtype='float32', - shape=[None, 3, None, None]) - one = 2. + x_var = fluid.data( + name='x', dtype='float32', shape=[None, 3, None, None] + ) + one = 2.0 out = one / x_var exe = fluid.Executor(fluid.CPUPlace()) x = np.random.uniform(0.1, 0.6, (1, 3, 32, 32)).astype("float32") - out_result, = exe.run(feed={'x': x}, fetch_list=[out]) + (out_result,) = exe.run(feed={'x': x}, fetch_list=[out]) self.assertEqual((out_result == (2 / x)).all(), True) class TestDivideOp(unittest.TestCase): - def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="float32") @@ -329,12 +316,11 @@ class TestDivideOp(unittest.TestCase): y = paddle.to_tensor(np_y) z = paddle.divide(x, y) np_z = z.numpy() - z_expected = np.array([2., 0.6, 2.]) + z_expected = np.array([2.0, 0.6, 2.0]) self.assertEqual((np_z == z_expected).all(), True) class TestComplexElementwiseDivOp(OpTest): - def setUp(self): self.op_type = "elementwise_div" self.python_api = paddle.divide @@ -344,7 +330,7 @@ class TestComplexElementwiseDivOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -353,17 +339,18 @@ class TestComplexElementwiseDivOp(OpTest): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) - self.y = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + self.x = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) + self.y = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) self.out = self.x / self.y def init_grad_input_output(self): - self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1J * np.ones( - (2, 3, 4, 5), self.dtype) + self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1j * np.ones( + (2, 3, 4, 5), self.dtype + ) self.grad_x = self.grad_out / np.conj(self.y) self.grad_y = -self.grad_out * np.conj(self.x / self.y / self.y) @@ -371,38 +358,44 @@ class TestComplexElementwiseDivOp(OpTest): self.check_output(check_eager=False) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) class TestRealComplexElementwiseDivOp(TestComplexElementwiseDivOp): - def init_input_output(self): self.x = np.random.random((2, 3, 4, 5)).astype(self.dtype) - self.y = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + self.y = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) self.out = self.x / self.y def init_grad_input_output(self): - self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1J * np.ones( - (2, 3, 4, 5), self.dtype) + self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1j * np.ones( + (2, 3, 4, 5), self.dtype + ) self.grad_x = np.real(self.grad_out / np.conj(self.y)) self.grad_y = -self.grad_out * np.conj(self.x / self.y / self.y) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py index 746bb888669..de058ed2b3b 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py @@ -22,7 +22,6 @@ import random class TestElementwiseModOp(OpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -38,7 +37,7 @@ class TestElementwiseModOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -59,7 +58,6 @@ class TestElementwiseModOp(OpTest): class TestElementwiseModOp_scalar(TestElementwiseModOp): - def init_input_output(self): scale_x = random.randint(0, 100000000) scale_y = random.randint(1, 100000000) @@ -69,7 +67,6 @@ class TestElementwiseModOp_scalar(TestElementwiseModOp): class TestElementwiseModOpInverse(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.uniform(0, 10000, [10]).astype(self.dtype) self.y = np.random.uniform(0, 1000, [10, 10]).astype(self.dtype) @@ -77,7 +74,6 @@ class TestElementwiseModOpInverse(TestElementwiseModOp): class TestFloorDivideOp(unittest.TestCase): - def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="int64") diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py index 823714a9a59..91401ff574f 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py @@ -20,16 +20,12 @@ import paddle.fluid as fluid class TestElementWiseAddOp(unittest.TestCase): - def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) def check_forward_backward(self): - def test_with_place(place): out_grad = np.random.random_sample(self.x.shape).astype(np.float32) x_grad = out_grad @@ -52,25 +48,29 @@ class TestElementWiseAddOp(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) - elementwise_add_op = block.append_op(type="elementwise_add", - inputs={ - "X": block.var('x'), - "Y": block.var('y'), - }, - outputs={ - "Out": - block.var('out'), - }, - attrs={ - "axis": self.axis, - }) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) + elementwise_add_op = block.append_op( + type="elementwise_add", + inputs={ + "X": block.var('x'), + "Y": block.var('y'), + }, + outputs={ + "Out": block.var('out'), + }, + attrs={ + "axis": self.axis, + }, + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - elementwise_add_op.desc, set(), []) + elementwise_add_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -83,18 +83,20 @@ class TestElementWiseAddOp(unittest.TestCase): grad_var.set_dtype(core.VarDesc.VarType.FP32) exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'y', 'out@GRAD'] - }, - fetch_list=['x@GRAD', 'y@GRAD']) + out = exe.run( + program, + feed={ + name: var_dict[name] for name in ['x', 'y', 'out@GRAD'] + }, + fetch_list=['x@GRAD', 'y@GRAD'], + ) self.__assert_close(x_grad, out[0], "x@GRAD") self.__assert_close(y_grad, out[1], "y@GRAD", atol=1.4) places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "elementwise_add"): + "elementwise_add" + ): places.append(core.CUDAPlace(0)) for place in places: diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py index 7789f872d4c..aebf1c3f4fe 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py @@ -26,7 +26,6 @@ def Heaviside_grad(x, y, dout): class TestElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_heaviside" x = np.random.random((13, 17)).astype("float64") @@ -48,7 +47,6 @@ class TestElementwiseOp(OpTest): class TestHeavisideBroadcast(unittest.TestCase): - def setUp(self): self.input_1 = np.random.rand(2, 100, 13, 17).astype("float32") self.input_2 = np.random.rand(100, 13, 17).astype("float32") @@ -87,7 +85,6 @@ class TestHeavisideBroadcast(unittest.TestCase): class TestHeavisideAPI_float64(unittest.TestCase): - def setUp(self): self.x_np = np.random.random((13, 17)).astype("float64") self.y_np = np.random.random((13, 17)).astype("float64") @@ -95,45 +92,49 @@ class TestHeavisideAPI_float64(unittest.TestCase): self.dtype = "float64" def test_static(self): - for use_cuda in ([False, True] - if paddle.device.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if paddle.device.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() prog = paddle.static.Program() with paddle.static.program_guard(prog): - x = paddle.static.data(name=f"x_{self.dtype}", - shape=[13, 17], - dtype=self.dtype) - y = paddle.static.data(name=f"y_{self.dtype}", - shape=[13, 17], - dtype=self.dtype) + x = paddle.static.data( + name=f"x_{self.dtype}", shape=[13, 17], dtype=self.dtype + ) + y = paddle.static.data( + name=f"y_{self.dtype}", shape=[13, 17], dtype=self.dtype + ) out = paddle.heaviside(x, y) exe = paddle.static.Executor(place=place) - res, = exe.run(prog, - feed={ - f"x_{self.dtype}": self.x_np, - f"y_{self.dtype}": self.y_np - }, - fetch_list=out, - use_prune=True) + (res,) = exe.run( + prog, + feed={ + f"x_{self.dtype}": self.x_np, + f"y_{self.dtype}": self.y_np, + }, + fetch_list=out, + use_prune=True, + ) np.testing.assert_allclose(res, self.out_np, rtol=1e-05) def test_dygraph(self): - for use_cuda in ([False, True] - if paddle.device.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if paddle.device.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) - result = paddle.heaviside(paddle.to_tensor(self.x_np), - paddle.to_tensor(self.y_np)) + result = paddle.heaviside( + paddle.to_tensor(self.x_np), paddle.to_tensor(self.y_np) + ) np.testing.assert_allclose(result.numpy(), self.out_np, rtol=1e-05) class TestHeavisideAPI_float32(TestHeavisideAPI_float64): - def setUp(self): self.x_np = np.random.random((13, 17)).astype("float32") self.y_np = np.random.random((13, 17)).astype("float32") @@ -142,7 +143,6 @@ class TestHeavisideAPI_float32(TestHeavisideAPI_float64): class TestHeavisideAPI_int64(TestHeavisideAPI_float64): - def setUp(self): self.x_np = np.random.random((13, 17)).astype("int64") self.y_np = np.random.random((13, 17)).astype("int64") @@ -151,7 +151,6 @@ class TestHeavisideAPI_int64(TestHeavisideAPI_float64): class TestHeavisideAPI_int32(TestHeavisideAPI_float64): - def setUp(self): self.x_np = np.random.random((13, 17)).astype("int32") self.y_np = np.random.random((13, 17)).astype("int32") @@ -160,14 +159,13 @@ class TestHeavisideAPI_int32(TestHeavisideAPI_float64): class TestHeavisideAPI_float16(OpTest): - def setUp(self): self.dtype = np.float16 self.op_type = "elementwise_heaviside" self.python_api = paddle.heaviside self.inputs = { 'X': np.random.uniform(1, 2, [20, 5]).astype("float16"), - 'Y': np.random.uniform(1, 2, [20, 5]).astype("float16") + 'Y': np.random.uniform(1, 2, [20, 5]).astype("float16"), } self.outputs = {'Out': np.heaviside(self.inputs['X'], self.inputs['Y'])} @@ -175,16 +173,17 @@ class TestHeavisideAPI_float16(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=Heaviside_grad( - self.inputs['X'], self.inputs['Y'], - 1 / self.inputs['X'].size), - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=Heaviside_grad( + self.inputs['X'], self.inputs['Y'], 1 / self.inputs['X'].size + ), + check_eager=True, + ) class TestHeavisideError(unittest.TestCase): - def test_input(self): paddle.disable_static() @@ -199,8 +198,9 @@ class TestHeavisideError(unittest.TestCase): self.assertRaises(ValueError, test_input_y) def test_input_xy(): - paddle.heaviside(paddle.randn([100], 'float32'), - paddle.randn([100], 'float64')) + paddle.heaviside( + paddle.randn([100], 'float32'), paddle.randn([100], 'float64') + ) self.assertRaises(ValueError, test_input_xy) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py index d44eac529b4..018a44c2be9 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py @@ -20,7 +20,6 @@ import paddle class TestElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum @@ -46,25 +45,25 @@ class TestElementwiseOp(OpTest): self.check_grad(['X', 'Y'], 'Out', check_eager=True) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) - - -@unittest.skipIf(core.is_compiled_with_cuda() and ( - core.cudnn_version() < 8100 - or paddle.device.cuda.get_device_capability()[0] < 8 -), "run test when gpu is availble and the minimum cudnn version is 8.1.0 and gpu's compute capability is at least 8.0." - ) + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y') + ) + + +@unittest.skipIf( + core.is_compiled_with_cuda() + and ( + core.cudnn_version() < 8100 + or paddle.device.cuda.get_device_capability()[0] < 8 + ), + "run test when gpu is availble and the minimum cudnn version is 8.1.0 and gpu's compute capability is at least 8.0.", +) class TestElementwiseBF16Op(OpTest): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum @@ -77,7 +76,7 @@ class TestElementwiseBF16Op(OpTest): y = x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype(np.float32) self.inputs = { 'X': convert_float_to_uint16(x), - 'Y': convert_float_to_uint16(y) + 'Y': convert_float_to_uint16(y), } self.outputs = {'Out': convert_float_to_uint16(np.maximum(x, y))} @@ -101,9 +100,9 @@ class TestElementwiseBF16Op(OpTest): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMaxOp_scalar(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum @@ -114,97 +113,98 @@ class TestElementwiseMaxOp_scalar(TestElementwiseOp): class TestElementwiseMaxOp_Vector(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum - x = np.random.random((100, )).astype("float64") - sgn = np.random.choice([-1, 1], (100, )).astype("float64") - y = x + sgn * np.random.uniform(0.1, 1, (100, )).astype("float64") + x = np.random.random((100,)).astype("float64") + sgn = np.random.choice([-1, 1], (100,)).astype("float64") + y = x + sgn * np.random.uniform(0.1, 1, (100,)).astype("float64") self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': np.maximum(self.inputs['X'], self.inputs['Y'])} class TestElementwiseMaxOp_broadcast_0(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (100, 5, 2)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 0} self.outputs = { - 'Out': np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseMaxOp_broadcast_1(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseMaxOp_broadcast_2(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (1, 3, 100)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseMaxOp_broadcast_3(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (2, 50, 2, 1)).astype(np.float64) sgn = np.random.choice([-1, 1], (50, 2)).astype(np.float64) - y = x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (50, 2)).astype(np.float64) + y = x[0, :, :, 0] + sgn * np.random.uniform(1, 2, (50, 2)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], self.inputs['Y'].reshape(1, 50, 2, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 50, 2, 1) + ) } class TestElementwiseMaxOp_broadcast_4(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(np.float64) sgn = np.random.choice([-1, 1], (2, 3, 1, 5)).astype(np.float64) - y = x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 5)).astype(np.float64) + y = x + sgn * np.random.uniform(1, 2, (2, 3, 1, 5)).astype(np.float64) self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': np.maximum(self.inputs['X'], self.inputs['Y'])} diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py index e4c08742b01..5a2cdc691fa 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum @@ -49,22 +48,20 @@ class TestElementwiseOp(OpTest): self.check_grad(['X', 'Y'], 'Out', check_eager=True) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y') + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOp_scalar(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum @@ -75,104 +72,104 @@ class TestElementwiseMinOp_scalar(TestElementwiseOp): class TestElementwiseMinOp_Vector(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum - x = np.random.random((100, )).astype("float64") - sgn = np.random.choice([-1, 1], (100, )).astype("float64") - y = x + sgn * np.random.uniform(0.1, 1, (100, )).astype("float64") + x = np.random.random((100,)).astype("float64") + sgn = np.random.choice([-1, 1], (100,)).astype("float64") + y = x + sgn * np.random.uniform(0.1, 1, (100,)).astype("float64") self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': np.minimum(self.inputs['X'], self.inputs['Y'])} class TestElementwiseMinOp_broadcast_0(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (100, 3, 2)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 0} self.outputs = { - 'Out': np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseMinOp_broadcast_1(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseMinOp_broadcast_2(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseMinOp_broadcast_3(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (2, 25, 4, 1)).astype(np.float64) sgn = np.random.choice([-1, 1], (25, 4)).astype(np.float64) - y = x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (25, 4)).astype(np.float64) + y = x[0, :, :, 0] + sgn * np.random.uniform(1, 2, (25, 4)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], self.inputs['Y'].reshape(1, 25, 4, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 25, 4, 1) + ) } class TestElementwiseMinOp_broadcast_4(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (2, 10, 2, 5)).astype(np.float64) sgn = np.random.choice([-1, 1], (2, 10, 1, 5)).astype(np.float64) - y = x + sgn * \ - np.random.uniform(1, 2, (2, 10, 1, 5)).astype(np.float64) + y = x + sgn * np.random.uniform(1, 2, (2, 10, 1, 5)).astype(np.float64) self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': np.minimum(self.inputs['X'], self.inputs['Y'])} class TestElementwiseMinOpFP16(unittest.TestCase): - def get_out_and_grad(self, x_np, y_np, axis, place, use_fp32=False): assert x_np.dtype == np.float16 assert y_np.dtype == np.float16 @@ -188,8 +185,11 @@ class TestElementwiseMinOpFP16(unittest.TestCase): y.stop_gradient = False z = fluid.layers.elementwise_min(x, y, axis) x_g, y_g = paddle.grad([z], [x, y]) - return z.numpy().astype(dtype), x_g.numpy().astype( - dtype), y_g.numpy().astype(dtype) + return ( + z.numpy().astype(dtype), + x_g.numpy().astype(dtype), + y_g.numpy().astype(dtype), + ) def check_main(self, x_shape, y_shape, axis=-1): if not paddle.is_compiled_with_cuda(): @@ -201,8 +201,9 @@ class TestElementwiseMinOpFP16(unittest.TestCase): x_np = np.random.random(size=x_shape).astype(np.float16) y_np = np.random.random(size=y_shape).astype(np.float16) - z_1, x_g_1, y_g_1 = self.get_out_and_grad(x_np, y_np, axis, place, - False) + z_1, x_g_1, y_g_1 = self.get_out_and_grad( + x_np, y_np, axis, place, False + ) z_2, x_g_2, y_g_2 = self.get_out_and_grad(x_np, y_np, axis, place, True) np.testing.assert_array_equal(z_1, z_2) np.testing.assert_array_equal(x_g_1, x_g_2) @@ -210,11 +211,11 @@ class TestElementwiseMinOpFP16(unittest.TestCase): def test_main(self): self.check_main((13, 17), (13, 17)) - self.check_main((10, 3, 4), (1, )) - self.check_main((100, ), (100, )) - self.check_main((100, 3, 2), (100, ), 0) - self.check_main((2, 100, 3), (100, ), 1) - self.check_main((2, 3, 100), (100, )) + self.check_main((10, 3, 4), (1,)) + self.check_main((100,), (100,)) + self.check_main((100, 3, 2), (100,), 0) + self.check_main((2, 100, 3), (100,), 1) + self.check_main((2, 3, 100), (100,)) self.check_main((2, 25, 4, 1), (25, 4), 1) self.check_main((2, 10, 2, 5), (2, 10, 1, 5)) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py index 8c4ba799171..8969d76ce51 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py @@ -22,7 +22,6 @@ import random class TestElementwiseModOp(OpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -37,7 +36,7 @@ class TestElementwiseModOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -61,7 +60,6 @@ class TestElementwiseModOp(OpTest): class TestElementwiseModOp_scalar(TestElementwiseModOp): - def init_input_output(self): scale_x = random.randint(0, 100000000) scale_y = random.randint(1, 100000000) @@ -71,7 +69,6 @@ class TestElementwiseModOp_scalar(TestElementwiseModOp): class TestElementwiseModOpFloat(TestElementwiseModOp): - def init_dtype(self): self.dtype = np.float32 @@ -88,7 +85,6 @@ class TestElementwiseModOpFloat(TestElementwiseModOp): class TestElementwiseModOpFp16(TestElementwiseModOp): - def init_dtype(self): self.dtype = np.float16 @@ -105,13 +101,11 @@ class TestElementwiseModOpFp16(TestElementwiseModOp): class TestElementwiseModOpDouble(TestElementwiseModOpFloat): - def init_dtype(self): self.dtype = np.float64 class TestRemainderOp(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.remainder(x, y, name) @@ -135,7 +129,7 @@ class TestRemainderOp(unittest.TestCase): self.assertEqual((np_z == z_expected).all(), True) np_x = np.array([-3.3, 11.5, -2, 3.5]) - np_y = np.array([-1.2, 2., 3.3, -2.3]) + np_y = np.array([-1.2, 2.0, 3.3, -2.3]) x = paddle.to_tensor(np_x) y = paddle.to_tensor(np_y) z = x % y @@ -152,13 +146,11 @@ class TestRemainderOp(unittest.TestCase): class TestRemainderInplaceOp(TestRemainderOp): - def _executed_api(self, x, y, name=None): return x.remainder_(y, name) class TestRemainderInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float') self.y_numpy = np.random.rand(3, 4).astype('float') @@ -174,17 +166,17 @@ class TestRemainderInplaceBroadcastSuccess(unittest.TestCase): paddle.enable_static() -class TestRemainderInplaceBroadcastSuccess2(TestRemainderInplaceBroadcastSuccess - ): - +class TestRemainderInplaceBroadcastSuccess2( + TestRemainderInplaceBroadcastSuccess +): def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') self.y_numpy = np.random.rand(3, 1).astype('float') -class TestRemainderInplaceBroadcastSuccess3(TestRemainderInplaceBroadcastSuccess - ): - +class TestRemainderInplaceBroadcastSuccess3( + TestRemainderInplaceBroadcastSuccess +): def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py index 05ed9cc9ded..cc3cd9be823 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py @@ -20,11 +20,14 @@ import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid import Program, program_guard -from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + skip_check_grad_ci, + convert_float_to_uint16, +) class ElementwiseMulOp(OpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -39,7 +42,7 @@ class ElementwiseMulOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} @@ -50,23 +53,27 @@ class ElementwiseMulOp(OpTest): def test_check_grad_normal(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['X', 'Y'], - 'Out', - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + ['X', 'Y'], 'Out', check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad_ingore_x(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_ingore_y(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + check_dygraph=(self.use_mkldnn == False), + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -81,7 +88,6 @@ class ElementwiseMulOp(OpTest): class TestBF16ElementwiseMulOp(OpTest): - def setUp(self): self.op_type = "elementwise_mul" self.dtype = np.uint16 @@ -93,9 +99,12 @@ class TestBF16ElementwiseMulOp(OpTest): self.axis = -1 self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(convert_float_to_uint16(self.x)), - 'Y': OpTest.np_dtype_to_fluid_dtype(convert_float_to_uint16(self.y)) + 'X': OpTest.np_dtype_to_fluid_dtype( + convert_float_to_uint16(self.x) + ), + 'Y': OpTest.np_dtype_to_fluid_dtype( + convert_float_to_uint16(self.y) + ), } self.outputs = {'Out': convert_float_to_uint16(self.out)} self.attrs = {'axis': self.axis, 'use_mkldnn': False} @@ -114,33 +123,31 @@ class TestBF16ElementwiseMulOp(OpTest): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float64), - 'Y': np.random.rand(1).astype(np.float64) + 'Y': np.random.rand(1).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.random((100, )).astype("float64"), - 'Y': np.random.random((100, )).astype("float64") + 'X': np.random.random((100,)).astype("float64"), + 'Y': np.random.random((100,)).astype("float64"), } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} self.init_kernel_type() class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -151,12 +158,11 @@ class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.attrs = {'axis': 1} @@ -167,12 +173,11 @@ class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.outputs = { @@ -182,12 +187,11 @@ class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float64), - 'Y': np.random.rand(10, 12).astype(np.float64) + 'Y': np.random.rand(10, 12).astype(np.float64), } self.attrs = {'axis': 1} @@ -198,68 +202,63 @@ class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 2, 11).astype(np.float64), - 'Y': np.random.rand(10, 1, 11).astype(np.float64) + 'Y': np.random.rand(10, 1, 11).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 4, 2, 3).astype(np.float64), - 'Y': np.random.rand(10, 4, 1, 3).astype(np.float64) + 'Y': np.random.rand(10, 4, 1, 3).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestElementwiseMulOpFp16(ElementwiseMulOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float64), - 'Y': np.random.rand(1, 1, 100).astype(np.float64) + 'Y': np.random.rand(1, 1, 100).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(30, 3, 1, 5).astype(np.float64), - 'Y': np.random.rand(30, 1, 4, 1).astype(np.float64) + 'Y': np.random.rand(30, 1, 4, 1).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 10).astype(np.float64), - 'Y': np.random.rand(2, 2, 10, 10).astype(np.float64) + 'Y': np.random.rand(2, 2, 10, 10).astype(np.float64), } self.attrs = {'axis': 2} @@ -271,14 +270,15 @@ class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): class TestElementwiseMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_mul must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.elementwise_mul, x1, y1) # the input dtype of elementwise_mul must be float16 or float32 or float64 or int32 or int64 @@ -289,7 +289,6 @@ class TestElementwiseMulOpError(unittest.TestCase): class TestComplexElementwiseMulOp(OpTest): - def setUp(self): self.op_type = "elementwise_mul" self.init_base_dtype() @@ -298,7 +297,7 @@ class TestComplexElementwiseMulOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -307,17 +306,18 @@ class TestComplexElementwiseMulOp(OpTest): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) - self.y = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + self.x = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) + self.y = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) self.out = self.x * self.y def init_grad_input_output(self): - self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1J * np.ones( - (2, 3, 4, 5), self.dtype) + self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1j * np.ones( + (2, 3, 4, 5), self.dtype + ) self.grad_x = self.grad_out * np.conj(self.y) self.grad_y = self.grad_out * np.conj(self.x) @@ -325,38 +325,44 @@ class TestComplexElementwiseMulOp(OpTest): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) class TestRealComplexElementwiseMulOp(TestComplexElementwiseMulOp): - def init_input_output(self): self.x = np.random.random((2, 3, 4, 5)).astype(self.dtype) - self.y = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + self.y = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) self.out = self.x * self.y def init_grad_input_output(self): - self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1J * np.ones( - (2, 3, 4, 5), self.dtype) + self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1j * np.ones( + (2, 3, 4, 5), self.dtype + ) self.grad_x = np.real(self.grad_out * np.conj(self.y)) self.grad_y = self.grad_out * np.conj(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py b/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py index 875da9df79b..3f816b61650 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py @@ -25,7 +25,6 @@ from decorator_helper import prog_scope class TestElementwiseMulDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -41,11 +40,9 @@ class TestElementwiseMulDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -57,7 +54,6 @@ class TestElementwiseMulDoubleGradCheck(unittest.TestCase): class TestElementwiseMulBroadcastDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -73,11 +69,9 @@ class TestElementwiseMulBroadcastDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -89,7 +83,6 @@ class TestElementwiseMulBroadcastDoubleGradCheck(unittest.TestCase): class TestElementwiseAddDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -105,11 +98,9 @@ class TestElementwiseAddDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -121,7 +112,6 @@ class TestElementwiseAddDoubleGradCheck(unittest.TestCase): class TestElementwiseAddBroadcastDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -137,11 +127,9 @@ class TestElementwiseAddBroadcastDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -153,7 +141,6 @@ class TestElementwiseAddBroadcastDoubleGradCheck(unittest.TestCase): class TestElementwiseSubDoubleGradCheck(unittest.TestCase): - def subtract_wrapper(self, x): return paddle.subtract(x[0], x[1]) @@ -172,16 +159,16 @@ class TestElementwiseSubDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.subtract_wrapper, - [x, y], - out, - x_init=[x_arr, y_arr], - place=place) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.subtract_wrapper, + [x, y], + out, + x_init=[x_arr, y_arr], + place=place, + ) def test_grad(self): paddle.enable_static() @@ -193,7 +180,6 @@ class TestElementwiseSubDoubleGradCheck(unittest.TestCase): class TestElementwiseSubBroadcastDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -209,11 +195,9 @@ class TestElementwiseSubBroadcastDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -225,7 +209,6 @@ class TestElementwiseSubBroadcastDoubleGradCheck(unittest.TestCase): class TestElementwiseDivDoubleGradCheck(unittest.TestCase): - def divide_wrapper(self, x): return paddle.divide(x[0], x[1]) @@ -245,18 +228,17 @@ class TestElementwiseDivDoubleGradCheck(unittest.TestCase): y_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr[np.abs(y_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps, - atol=1e-3) - gradient_checker.double_grad_check_for_dygraph(self.divide_wrapper, - [x, y], - out, - x_init=[x_arr, y_arr], - place=place, - atol=1e-3) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps, atol=1e-3 + ) + gradient_checker.double_grad_check_for_dygraph( + self.divide_wrapper, + [x, y], + out, + x_init=[x_arr, y_arr], + place=place, + atol=1e-3, + ) def test_grad(self): paddle.enable_static() @@ -268,7 +250,6 @@ class TestElementwiseDivDoubleGradCheck(unittest.TestCase): class TestElementwiseDivBroadcastDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -285,12 +266,9 @@ class TestElementwiseDivBroadcastDoubleGradCheck(unittest.TestCase): y_arr = np.random.uniform(-1, 1, shape[1:-1]).astype(dtype) y_arr[np.abs(y_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps, - atol=1e-3) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps, atol=1e-3 + ) def test_grad(self): paddle.enable_static() @@ -302,7 +280,6 @@ class TestElementwiseDivBroadcastDoubleGradCheck(unittest.TestCase): class TestElementwiseAddTripleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -318,11 +295,9 @@ class TestElementwiseAddTripleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -334,7 +309,6 @@ class TestElementwiseAddTripleGradCheck(unittest.TestCase): class TestElementwiseAddBroadcastTripleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -350,11 +324,9 @@ class TestElementwiseAddBroadcastTripleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -366,7 +338,6 @@ class TestElementwiseAddBroadcastTripleGradCheck(unittest.TestCase): class TestElementwiseMulTripleGradCheck(unittest.TestCase): - def multiply_wrapper(self, x): return paddle.multiply(x[0], x[1]) @@ -385,17 +356,17 @@ class TestElementwiseMulTripleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.multiply_wrapper, - [x, y], - out, - x_init=[x_arr, y_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.multiply_wrapper, + [x, y], + out, + x_init=[x_arr, y_arr], + place=place, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -408,7 +379,6 @@ class TestElementwiseMulTripleGradCheck(unittest.TestCase): class TestElementwiseMulBroadcastTripleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -424,11 +394,9 @@ class TestElementwiseMulBroadcastTripleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py index 4050e4a4c60..53cb18f8aa3 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py @@ -26,13 +26,12 @@ def pow_grad(x, y, dout): class TestElementwisePowOp(OpTest): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(1, 2, [20, 5]).astype("float64"), - 'Y': np.random.uniform(1, 2, [20, 5]).astype("float64") + 'Y': np.random.uniform(1, 2, [20, 5]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @@ -50,75 +49,70 @@ class TestElementwisePowOp(OpTest): class TestElementwisePowOp_big_shape_1(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(1, 2, [10, 10]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [10, 10]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [10, 10]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_big_shape_2(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(1, 2, [10, 10]).astype("float64"), - 'Y': np.random.uniform(0.2, 2, [10, 10]).astype("float64") + 'Y': np.random.uniform(0.2, 2, [10, 10]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwisePowOp_scalar(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [3, 3, 4]).astype(np.float64), - 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float64) + 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float64), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_tensor(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [100]).astype("float64"), - 'Y': np.random.uniform(1, 3, [100]).astype("float64") + 'Y': np.random.uniform(1, 3, [100]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_broadcast_0(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 1, 100]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_broadcast_1(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 100, 1]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float64"), } self.attrs = {'axis': 1} self.outputs = { @@ -127,51 +121,49 @@ class TestElementwisePowOp_broadcast_1(TestElementwisePowOp): class TestElementwisePowOp_broadcast_2(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [100, 3, 1]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float64"), } self.attrs = {'axis': 0} self.outputs = { - 'Out': np.power(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwisePowOp_broadcast_3(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 20, 5, 1]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [20, 5]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [20, 5]).astype("float64"), } self.attrs = {'axis': 1} self.outputs = { - 'Out': np.power(self.inputs['X'], - self.inputs['Y'].reshape(1, 20, 5, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(1, 20, 5, 1) + ) } class TestElementwisePowOp_broadcast_4(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 10, 3, 5]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOpInt(OpTest): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow @@ -186,7 +178,6 @@ class TestElementwisePowOpInt(OpTest): class TestElementwisePowGradOpInt(unittest.TestCase): - def setUp(self): self.x = np.asarray([1, 3, 6]) self.y = np.asarray([1, 1, 1]) @@ -194,11 +185,13 @@ class TestElementwisePowGradOpInt(unittest.TestCase): # dout = 1 self.grad_res = np.asarray([1, 1, 1]) # dx = dout * y * pow(x, y-1) - self.grad_x = self.grad_res * self.y * (self.x - **(self.y - 1)).astype("int") + self.grad_x = ( + self.grad_res * self.y * (self.x ** (self.y - 1)).astype("int") + ) # dy = dout * log(x) * pow(x, y) - self.grad_y = (self.grad_res * np.log(self.x) * - (self.x**self.y)).astype("int") + self.grad_y = ( + self.grad_res * np.log(self.x) * (self.x**self.y) + ).astype("int") def test_grad(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -220,13 +213,12 @@ class TestElementwisePowGradOpInt(unittest.TestCase): class TestElementwisePowOpFP16(OpTest): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(1, 2, [20, 5]).astype("float16"), - 'Y': np.random.uniform(1, 2, [20, 5]).astype("float16") + 'Y': np.random.uniform(1, 2, [20, 5]).astype("float16"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @@ -237,12 +229,14 @@ class TestElementwisePowOpFP16(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=pow_grad(self.inputs['X'], - self.inputs['Y'], - 1 / self.inputs['X'].size), - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=pow_grad( + self.inputs['X'], self.inputs['Y'], 1 / self.inputs['X'].size + ), + check_eager=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py index 9c8a0002bd0..f8f050d6f6b 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py @@ -21,12 +21,11 @@ from paddle.fluid.framework import _test_eager_guard class TestElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype("float64"), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} @@ -37,20 +36,17 @@ class TestElementwiseOp(OpTest): self.check_grad(['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y') + ) class TestBF16ElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_sub" self.dtype = np.uint16 @@ -60,7 +56,7 @@ class TestBF16ElementwiseOp(OpTest): self.inputs = { 'X': convert_float_to_uint16(x), - 'Y': convert_float_to_uint16(y) + 'Y': convert_float_to_uint16(y), } self.outputs = {'Out': convert_float_to_uint16(out)} @@ -78,36 +74,34 @@ class TestBF16ElementwiseOp(OpTest): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseSubOp_scalar(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float64), - 'Y': np.random.rand(1).astype(np.float64) + 'Y': np.random.rand(1).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_Vector(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { - 'X': np.random.random((100, )).astype("float64"), - 'Y': np.random.random((100, )).astype("float64") + 'X': np.random.random((100,)).astype("float64"), + 'Y': np.random.random((100,)).astype("float64"), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_broadcast_0(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(100, 3, 2).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.attrs = {'axis': 0} @@ -117,12 +111,11 @@ class TestElementwiseSubOp_broadcast_0(TestElementwiseOp): class TestElementwiseSubOp_broadcast_1(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.attrs = {'axis': 1} @@ -132,12 +125,11 @@ class TestElementwiseSubOp_broadcast_1(TestElementwiseOp): class TestElementwiseSubOp_broadcast_2(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.outputs = { @@ -146,12 +138,11 @@ class TestElementwiseSubOp_broadcast_2(TestElementwiseOp): class TestElementwiseSubOp_broadcast_3(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float64), - 'Y': np.random.rand(10, 12).astype(np.float64) + 'Y': np.random.rand(10, 12).astype(np.float64), } self.attrs = {'axis': 1} @@ -161,45 +152,41 @@ class TestElementwiseSubOp_broadcast_3(TestElementwiseOp): class TestElementwiseSubOp_broadcast_4(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 5, 3, 12).astype(np.float64), - 'Y': np.random.rand(2, 5, 1, 12).astype(np.float64) + 'Y': np.random.rand(2, 5, 1, 12).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_1(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float64), - 'Y': np.random.rand(1, 1, 100).astype(np.float64) + 'Y': np.random.rand(1, 1, 100).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_2(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 3, 1, 4).astype(np.float64), - 'Y': np.random.rand(10, 1, 12, 1).astype(np.float64) + 'Y': np.random.rand(10, 1, 12, 1).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_xsize_lessthan_ysize(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 12).astype(np.float64), - 'Y': np.random.rand(2, 3, 10, 12).astype(np.float64) + 'Y': np.random.rand(2, 3, 10, 12).astype(np.float64), } self.attrs = {'axis': 2} @@ -210,7 +197,6 @@ class TestElementwiseSubOp_xsize_lessthan_ysize(TestElementwiseOp): class TestComplexElementwiseSubOp(OpTest): - def setUp(self): self.op_type = "elementwise_sub" self.dtype = np.float64 @@ -220,7 +206,7 @@ class TestComplexElementwiseSubOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -230,14 +216,17 @@ class TestComplexElementwiseSubOp(OpTest): def init_input_output(self): self.x = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.y = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = self.x - self.y def init_grad_input_output(self): - self.grad_out = np.ones( - self.shape, self.dtype) + 1J * np.ones(self.shape, self.dtype) + self.grad_out = np.ones(self.shape, self.dtype) + 1j * np.ones( + self.shape, self.dtype + ) self.grad_x = self.grad_out self.grad_y = -self.grad_out @@ -245,43 +234,49 @@ class TestComplexElementwiseSubOp(OpTest): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) class TestRealComplexElementwiseSubOp(TestComplexElementwiseSubOp): - def init_input_output(self): self.x = np.random.random(self.shape).astype(self.dtype) self.y = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = self.x - self.y def init_grad_input_output(self): - self.grad_out = np.ones( - self.shape, self.dtype) + 1J * np.ones(self.shape, self.dtype) + self.grad_out = np.ones(self.shape, self.dtype) + 1j * np.ones( + self.shape, self.dtype + ) self.grad_x = np.real(self.grad_out) self.grad_y = -self.grad_out class TestSubtractApi(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.subtract(x, y, name) @@ -299,7 +294,7 @@ class TestSubtractApi(unittest.TestCase): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -308,7 +303,7 @@ class TestSubtractApi(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([1., -2., 2.]) + z_expected = np.array([1.0, -2.0, 2.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -319,18 +314,16 @@ class TestSubtractApi(unittest.TestCase): y = fluid.dygraph.to_variable(np_y) z = self._executed_api(x, y) np_z = z.numpy() - z_expected = np.array([1., -2., 2.]) + z_expected = np.array([1.0, -2.0, 2.0]) self.assertEqual((np_z == z_expected).all(), True) class TestSubtractInplaceApi(TestSubtractApi): - def _executed_api(self, x, y, name=None): return x.subtract_(y, name) class TestSubtractInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float') self.y_numpy = np.random.rand(3, 4).astype('float') @@ -347,21 +340,18 @@ class TestSubtractInplaceBroadcastSuccess(unittest.TestCase): class TestSubtractInplaceBroadcastSuccess2(TestSubtractInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') self.y_numpy = np.random.rand(3, 1).astype('float') class TestSubtractInplaceBroadcastSuccess3(TestSubtractInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') class TestSubtractInplaceBroadcastError(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(3, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') @@ -380,21 +370,18 @@ class TestSubtractInplaceBroadcastError(unittest.TestCase): class TestSubtractInplaceBroadcastError2(TestSubtractInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestSubtractInplaceBroadcastError3(TestSubtractInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestFloatElementwiseSubop(unittest.TestCase): - def func_dygraph_sub(self): paddle.disable_static() @@ -407,26 +394,23 @@ class TestFloatElementwiseSubop(unittest.TestCase): # normal case: tensor - tensor expect_out = np_a - np_b actual_out = tensor_a - tensor_b - np.testing.assert_allclose(actual_out, - expect_out, - rtol=1e-07, - atol=1e-07) + np.testing.assert_allclose( + actual_out, expect_out, rtol=1e-07, atol=1e-07 + ) # normal case: tensor - scalar expect_out = np_a - 1 actual_out = tensor_a - 1 - np.testing.assert_allclose(actual_out, - expect_out, - rtol=1e-07, - atol=1e-07) + np.testing.assert_allclose( + actual_out, expect_out, rtol=1e-07, atol=1e-07 + ) # normal case: scalar - tenor expect_out = 1 - np_a actual_out = 1 - tensor_a - np.testing.assert_allclose(actual_out, - expect_out, - rtol=1e-07, - atol=1e-07) + np.testing.assert_allclose( + actual_out, expect_out, rtol=1e-07, atol=1e-07 + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_ema.py b/python/paddle/fluid/tests/unittests/test_ema.py index 1e2434a6a52..fe4178c8259 100644 --- a/python/paddle/fluid/tests/unittests/test_ema.py +++ b/python/paddle/fluid/tests/unittests/test_ema.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class TestExponentialMovingAverage(unittest.TestCase): - def setUp(self): self._places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): @@ -32,19 +31,21 @@ class TestExponentialMovingAverage(unittest.TestCase): with fluid.program_guard(self._train_program, self._startup_prog): with fluid.unique_name.guard(): data = fluid.data(name='x', shape=[-1, 5], dtype='float32') - hidden = fluid.layers.fc(input=data, - size=10, - param_attr=self._param_name) + hidden = fluid.layers.fc( + input=data, size=10, param_attr=self._param_name + ) cost = paddle.mean(hidden) self._test_program = fluid.default_main_program().clone( - for_test=True) + for_test=True + ) optimizer = fluid.optimizer.Adam(learning_rate=0.001) optimizer.minimize(cost) self._ema = fluid.optimizer.ExponentialMovingAverage( - self._ema_decay) + self._ema_decay + ) self._ema.update() def train(self, place): @@ -55,16 +56,19 @@ class TestExponentialMovingAverage(unittest.TestCase): for pass_id in range(2): for batch_id in range(3): data = np.random.random(size=(10, 5)).astype('float32') - tmp_param = np.array(fluid.global_scope().find_var( - self._param_name).get_tensor()) + tmp_param = np.array( + fluid.global_scope().find_var(self._param_name).get_tensor() + ) exe.run(program=self._train_program, feed={'x': data}) - tmp_param = np.array(fluid.global_scope().find_var( - self._param_name).get_tensor()) + tmp_param = np.array( + fluid.global_scope().find_var(self._param_name).get_tensor() + ) params.append(tmp_param) with self._ema.apply(exe): - final_ema = np.array(fluid.global_scope().find_var( - self._param_name).get_tensor()) + final_ema = np.array( + fluid.global_scope().find_var(self._param_name).get_tensor() + ) data = np.random.random(size=(10, 5)).astype('float32') exe.run(program=self._test_program, feed={'x': data}) return params, final_ema @@ -75,9 +79,11 @@ class TestExponentialMovingAverage(unittest.TestCase): manu_ema = np.zeros_like(final_ema) if len(params) > 0: for param in params: - manu_ema = self._ema_decay * manu_ema + ( - 1 - self._ema_decay) * param - manu_ema = manu_ema / (1.0 - self._ema_decay**len(params)) + manu_ema = ( + self._ema_decay * manu_ema + + (1 - self._ema_decay) * param + ) + manu_ema = manu_ema / (1.0 - self._ema_decay ** len(params)) np.testing.assert_allclose(manu_ema, final_ema, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_ema_fleet.py b/python/paddle/fluid/tests/unittests/test_ema_fleet.py index 206e3cf7362..8a3a6993d05 100644 --- a/python/paddle/fluid/tests/unittests/test_ema_fleet.py +++ b/python/paddle/fluid/tests/unittests/test_ema_fleet.py @@ -24,7 +24,6 @@ def gen_data(): class TestFleetStaticEMA(unittest.TestCase): - def setUp(self): self._places = [paddle.CPUPlace()] if paddle.device.is_compiled_with_cuda(): @@ -41,17 +40,19 @@ class TestFleetStaticEMA(unittest.TestCase): with static.program_guard(self._train_program, self._startup_prog): with utils.unique_name.guard(): data = static.data(name='x', shape=[-1, 5], dtype='float32') - hidden = static.nn.fc(x=data, - size=10, - weight_attr=self._param_name) + hidden = static.nn.fc( + x=data, size=10, weight_attr=self._param_name + ) cost = paddle.mean(hidden) self._test_program = static.default_main_program().clone( - for_test=True) + for_test=True + ) optimizer = paddle.optimizer.Adam(learning_rate=0.001) optimizer = paddle.distributed.fleet.distributed_optimizer( - optimizer, strategy) + optimizer, strategy + ) optimizer.minimize(cost) self._ema = static.ExponentialMovingAverage(self._ema_decay) @@ -65,13 +66,19 @@ class TestFleetStaticEMA(unittest.TestCase): for pass_id in range(2): for batch_id in range(3): exe.run(program=self._train_program, feed={'x': gen_data()}) - tmp_param = np.array(static.global_scope().find_var( - self._param_name).get_tensor()) + tmp_param = np.array( + static.global_scope() + .find_var(self._param_name) + .get_tensor() + ) params.append(tmp_param) with self._ema.apply(exe, restore): - final_ema = np.array(static.global_scope().find_var( - self._param_name).get_tensor()) + final_ema = np.array( + static.global_scope() + .find_var(self._param_name) + .get_tensor() + ) exe.run(program=self._test_program, feed={'x': gen_data()}) if not restore: self._ema.restore(exe) @@ -85,9 +92,11 @@ class TestFleetStaticEMA(unittest.TestCase): manu_ema = np.zeros_like(final_ema) if len(params) > 0: for param in params: - manu_ema = self._ema_decay * manu_ema + ( - 1 - self._ema_decay) * param - manu_ema = manu_ema / (1.0 - self._ema_decay**len(params)) + manu_ema = ( + self._ema_decay * manu_ema + + (1 - self._ema_decay) * param + ) + manu_ema = manu_ema / (1.0 - self._ema_decay ** len(params)) np.testing.assert_allclose(manu_ema, final_ema, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_embedding_id_stop_gradient.py b/python/paddle/fluid/tests/unittests/test_embedding_id_stop_gradient.py index db3af060d26..68f9696ffe2 100644 --- a/python/paddle/fluid/tests/unittests/test_embedding_id_stop_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_embedding_id_stop_gradient.py @@ -19,7 +19,6 @@ import unittest class TestEmbeddingIdStopGradientBase(unittest.TestCase): - def setUp(self): self.reshape_times = 1 self.iteration = 10 @@ -70,17 +69,15 @@ class TestEmbeddingIdStopGradientBase(unittest.TestCase): fetch_val = None for _ in range(self.iteration): - fetch_val = exe.run(feed={ - x_1.name: x1_data, - x_2.name: x2_data - }, - fetch_list=[emb])[0] + fetch_val = exe.run( + feed={x_1.name: x1_data, x_2.name: x2_data}, + fetch_list=[emb], + )[0] return fetch_val class TestEmbeddingIdStopGradient2(TestEmbeddingIdStopGradientBase): - def setUp(self): self.reshape_times = 100 self.iteration = 10 diff --git a/python/paddle/fluid/tests/unittests/test_empty_like_op.py b/python/paddle/fluid/tests/unittests/test_empty_like_op.py index 6c1d179fa4e..4ce4ab6a6d5 100644 --- a/python/paddle/fluid/tests/unittests/test_empty_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_empty_like_op.py @@ -21,37 +21,43 @@ from paddle.static import program_guard, Program class TestEmptyLikeAPICommon(unittest.TestCase): - def __check_out__(self, out): data_type = convert_dtype(out.dtype) self.assertEqual( - data_type, self.dst_dtype, - 'dtype should be %s, but get %s' % (self.dst_dtype, data_type)) + data_type, + self.dst_dtype, + 'dtype should be %s, but get %s' % (self.dst_dtype, data_type), + ) shape = out.shape self.assertTupleEqual( - shape, self.dst_shape, - 'shape should be %s, but get %s' % (self.dst_shape, shape)) + shape, + self.dst_shape, + 'shape should be %s, but get %s' % (self.dst_shape, shape), + ) if data_type in ['float32', 'float64', 'int32', 'int64']: max_value = np.nanmax(out) min_value = np.nanmin(out) always_non_full_zero = max_value >= min_value always_full_zero = max_value == 0.0 and min_value == 0.0 - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = out.size true_num = np.sum(out == True) false_num = np.sum(out == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: self.assertTrue(False, 'invalid data type') class TestEmptyLikeAPI(TestEmptyLikeAPICommon): - def setUp(self): self.init_config() @@ -69,7 +75,6 @@ class TestEmptyLikeAPI(TestEmptyLikeAPICommon): class TestEmptyLikeAPI2(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("float64") self.dtype = self.x.dtype @@ -78,7 +83,6 @@ class TestEmptyLikeAPI2(TestEmptyLikeAPI): class TestEmptyLikeAPI3(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("int") self.dtype = self.x.dtype @@ -87,7 +91,6 @@ class TestEmptyLikeAPI3(TestEmptyLikeAPI): class TestEmptyLikeAPI4(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("int64") self.dtype = self.x.dtype @@ -96,7 +99,6 @@ class TestEmptyLikeAPI4(TestEmptyLikeAPI): class TestEmptyLikeAPI5(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("bool") self.dtype = self.x.dtype @@ -105,7 +107,6 @@ class TestEmptyLikeAPI5(TestEmptyLikeAPI): class TestEmptyLikeAPI6(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("float64") self.dtype = "float32" @@ -114,7 +115,6 @@ class TestEmptyLikeAPI6(TestEmptyLikeAPI): class TestEmptyLikeAPI7(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("int") self.dtype = "float32" @@ -123,7 +123,6 @@ class TestEmptyLikeAPI7(TestEmptyLikeAPI): class TestEmptyLikeAPI8(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("int64") self.dtype = "float32" @@ -132,7 +131,6 @@ class TestEmptyLikeAPI8(TestEmptyLikeAPI): class TestEmptyLikeAPI9(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("bool") self.dtype = "float32" @@ -141,7 +139,6 @@ class TestEmptyLikeAPI9(TestEmptyLikeAPI): class TestEmptyLikeAPI10(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("float32") self.dtype = "bool" @@ -150,7 +147,6 @@ class TestEmptyLikeAPI10(TestEmptyLikeAPI): class TestEmptyLikeAPI_Static(TestEmptyLikeAPICommon): - def setUp(self): self.init_config() @@ -164,14 +160,17 @@ class TestEmptyLikeAPI_Static(TestEmptyLikeAPICommon): with program_guard(train_program, startup_program): x = np.random.random(self.x_shape).astype(dtype) - data_x = paddle.static.data('x', - shape=self.data_x_shape, - dtype=dtype) + data_x = paddle.static.data( + 'x', shape=self.data_x_shape, dtype=dtype + ) out = paddle.empty_like(data_x) - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) res = exe.run(train_program, feed={'x': x}, fetch_list=[out]) @@ -187,16 +186,13 @@ class TestEmptyLikeAPI_Static(TestEmptyLikeAPICommon): class TestEmptyLikeAPI_Static2(TestEmptyLikeAPI_Static): - def init_config(self): self.x_shape = (3, 200, 3) self.data_x_shape = [-1, 200, 3] class TestEmptyError(unittest.TestCase): - def test_attr(self): - def test_dtype(): x = np.random.random((200, 3)).astype("float64") dtype = 'uint8' diff --git a/python/paddle/fluid/tests/unittests/test_empty_op.py b/python/paddle/fluid/tests/unittests/test_empty_op.py index d9994110f1d..11b66325c1f 100644 --- a/python/paddle/fluid/tests/unittests/test_empty_op.py +++ b/python/paddle/fluid/tests/unittests/test_empty_op.py @@ -1,4 +1,4 @@ -#Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ # Situation 1: Attr(shape) is a list(without tensor) class TestEmptyOp(OpTest): - def setUp(self): self.op_type = "empty" self.init_config() @@ -38,14 +37,18 @@ class TestEmptyOp(OpTest): always_full_zero = max_value == 0.0 and min_value == 0.0 always_non_full_zero = max_value >= min_value - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = outs[0].size true_num = np.sum(outs[0] == True) false_num = np.sum(outs[0] == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: self.assertTrue(False, 'invalid data type') @@ -59,7 +62,6 @@ class TestEmptyOp(OpTest): class TestEmptyOp2(TestEmptyOp): - def init_config(self): shape = [500, 3] dtype = 'float64' @@ -70,7 +72,6 @@ class TestEmptyOp2(TestEmptyOp): class TestEmptyOp3(TestEmptyOp): - def init_config(self): shape = [500, 3] dtype = 'int32' @@ -81,7 +82,6 @@ class TestEmptyOp3(TestEmptyOp): class TestEmptyOp4(TestEmptyOp): - def init_config(self): shape = [500, 3] dtype = 'int64' @@ -92,7 +92,6 @@ class TestEmptyOp4(TestEmptyOp): class TestEmptyOp5(TestEmptyOp): - def init_config(self): shape = [500, 3] dtype = 'bool' @@ -104,7 +103,6 @@ class TestEmptyOp5(TestEmptyOp): # Situation 2: shape is a tensor class TestEmptyOp_ShapeTensor(OpTest): - def setUp(self): self.op_type = "empty" self.init_config() @@ -128,21 +126,24 @@ class TestEmptyOp_ShapeTensor(OpTest): always_full_zero = max_value == 0.0 and min_value == 0.0 always_non_full_zero = max_value >= min_value - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = outs[0].size true_num = np.sum(outs[0] == True) false_num = np.sum(outs[0] == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: self.assertTrue(False, 'invalid data type') # Situation 3: Attr(shape) is a list(with tensor) class TestEmptyOp_ShapeTensorList(OpTest): - def setUp(self): self.op_type = "empty" self.init_config() @@ -156,8 +157,9 @@ class TestEmptyOp_ShapeTensorList(OpTest): shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape, 'dtype': dtype_inner} @@ -174,27 +176,32 @@ class TestEmptyOp_ShapeTensorList(OpTest): always_full_zero = max_value == 0.0 and min_value == 0.0 always_non_full_zero = max_value >= min_value - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = outs[0].size true_num = np.sum(outs[0] == True) false_num = np.sum(outs[0] == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: self.assertTrue(False, 'invalid data type') class TestEmptyAPI(unittest.TestCase): - def __check_out__(self, out, dtype='float32'): max_value = np.nanmax(np.array(out)) min_value = np.nanmin(np.array(out)) always_non_full_zero = max_value >= min_value always_full_zero = max_value == 0.0 and min_value == 0.0 - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) def test_dygraph_api_out(self): paddle.disable_static() @@ -233,15 +240,15 @@ class TestEmptyAPI(unittest.TestCase): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 3) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 3) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - shape_tensor_unknown = fluid.data(name="shape_tensor_unknown", - shape=[-1], - dtype="int64") + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + shape_tensor_unknown = fluid.data( + name="shape_tensor_unknown", shape=[-1], dtype="int64" + ) out_1 = paddle.empty(shape=[200, 3], dtype=dtype) out_2 = paddle.empty(shape=shape_tensor_int32, dtype=dtype) @@ -259,7 +266,8 @@ class TestEmptyAPI(unittest.TestCase): "shape_tensor_int64": np.array([200, 3]).astype("int64"), "shape_tensor_unknown": np.array([200, 3]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6], + ) self.__check_out__(res_1, dtype) self.__check_out__(res_2, dtype) @@ -270,9 +278,7 @@ class TestEmptyAPI(unittest.TestCase): class TestEmptyError(unittest.TestCase): - def test_attr(self): - def test_dtype(): shape = [200, 3] dtype = 'uint8' diff --git a/python/paddle/fluid/tests/unittests/test_entry_attr.py b/python/paddle/fluid/tests/unittests/test_entry_attr.py index eb5bd30d1c2..cfbff2113c3 100644 --- a/python/paddle/fluid/tests/unittests/test_entry_attr.py +++ b/python/paddle/fluid/tests/unittests/test_entry_attr.py @@ -18,14 +18,18 @@ paddle.enable_static() import unittest import paddle.fluid as fluid -from paddle.distributed import ProbabilityEntry, CountFilterEntry, ShowClickEntry +from paddle.distributed import ( + ProbabilityEntry, + CountFilterEntry, + ShowClickEntry, +) class EntryAttrChecks(unittest.TestCase): - def base(self): with self.assertRaises(NotImplementedError): from paddle.distributed.entry_attr import EntryAttr + base = EntryAttr() base._to_attr() @@ -62,18 +66,21 @@ class EntryAttrChecks(unittest.TestCase): with fluid.scope_guard(scope): with fluid.program_guard(prog): - input = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) + input = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) prob = ProbabilityEntry(0.5) emb = paddle.static.nn.sparse_embedding( input=input, size=[100, 10], is_test=False, entry=prob, - param_attr=fluid.ParamAttr(name="deep_embedding")) + param_attr=fluid.ParamAttr(name="deep_embedding"), + ) pool = fluid.layers.sequence_pool(input=emb, pool_type="sum") predict = fluid.layers.fc(input=pool, size=2, act='softmax') @@ -92,7 +99,6 @@ class EntryAttrChecks(unittest.TestCase): class TestEntryAttrs(EntryAttrChecks): - def test_base(self): self.base() diff --git a/python/paddle/fluid/tests/unittests/test_entry_attr2.py b/python/paddle/fluid/tests/unittests/test_entry_attr2.py index b65fd12eba7..f4f098798ac 100644 --- a/python/paddle/fluid/tests/unittests/test_entry_attr2.py +++ b/python/paddle/fluid/tests/unittests/test_entry_attr2.py @@ -21,24 +21,26 @@ import paddle.fluid as fluid class EntryAttrChecks(unittest.TestCase): - def embedding_layer(self): prog = fluid.Program() scope = fluid.core.Scope() with fluid.scope_guard(scope): with fluid.program_guard(prog): - input = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) + input = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) emb = fluid.layers.embedding( input=input, size=[100, 10], is_sparse=True, is_distributed=True, - param_attr=fluid.ParamAttr(name="deep_embedding")) + param_attr=fluid.ParamAttr(name="deep_embedding"), + ) pool = fluid.layers.sequence_pool(input=emb, pool_type="sum") predict = fluid.layers.fc(input=pool, size=2, act='softmax') @@ -53,7 +55,6 @@ class EntryAttrChecks(unittest.TestCase): class TestEntryAttrs(EntryAttrChecks): - def test_embedding_layer(self): self.embedding_layer() diff --git a/python/paddle/fluid/tests/unittests/test_erf_op.py b/python/paddle/fluid/tests/unittests/test_erf_op.py index 338ca508db6..089fdc0a0b4 100644 --- a/python/paddle/fluid/tests/unittests/test_erf_op.py +++ b/python/paddle/fluid/tests/unittests/test_erf_op.py @@ -23,7 +23,6 @@ import paddle.fluid.dygraph as dg class TestErfOp(OpTest): - def setUp(self): self.op_type = "erf" self.dtype = self._init_dtype() @@ -44,7 +43,6 @@ class TestErfOp(OpTest): class TestErfLayer(unittest.TestCase): - def _test_case(self, place): x = np.random.uniform(-1, 1, size=(11, 17)).astype(np.float64) y_ref = erf(x) diff --git a/python/paddle/fluid/tests/unittests/test_erfinv_op.py b/python/paddle/fluid/tests/unittests/test_erfinv_op.py index 8822b3e0765..e605d8e0e62 100644 --- a/python/paddle/fluid/tests/unittests/test_erfinv_op.py +++ b/python/paddle/fluid/tests/unittests/test_erfinv_op.py @@ -24,7 +24,6 @@ np.random.seed(0) class TestErfinv(OpTest): - def setUp(self): self.op_type = "erfinv" self.python_api = paddle.erfinv @@ -33,8 +32,9 @@ class TestErfinv(OpTest): self.x = np.random.uniform(-1, 1, size=self.shape).astype(self.dtype) self.res_ref = erfinv(self.x).astype(self.dtype) self.grad_out = np.ones(self.shape, self.dtype) - self.gradient = np.sqrt(np.pi) / 2 * np.exp(np.square( - self.res_ref)) * self.grad_out + self.gradient = ( + np.sqrt(np.pi) / 2 * np.exp(np.square(self.res_ref)) * self.grad_out + ) self.inputs = {'X': self.x} self.outputs = {'Out': self.res_ref} @@ -45,20 +45,20 @@ class TestErfinv(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.gradient], - user_defined_grad_outputs=self.grad_out) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.gradient], + user_defined_grad_outputs=self.grad_out, + ) class TestErfinvFP32(TestErfinv): - def init_dtype(self): self.dtype = np.float32 class TestErfinvAPI(unittest.TestCase): - def init_dtype(self): self.dtype = 'float32' @@ -86,7 +86,6 @@ class TestErfinvAPI(unittest.TestCase): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) @@ -98,7 +97,6 @@ class TestErfinvAPI(unittest.TestCase): run(place) def test_inplace_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_exception.py b/python/paddle/fluid/tests/unittests/test_exception.py index 6d39f9f5f41..bb53ae95028 100644 --- a/python/paddle/fluid/tests/unittests/test_exception.py +++ b/python/paddle/fluid/tests/unittests/test_exception.py @@ -21,7 +21,6 @@ import paddle.fluid.core as core class TestException(unittest.TestCase): - def test_exception(self): exception = None try: @@ -34,7 +33,6 @@ class TestException(unittest.TestCase): class TestExceptionNoCStack(unittest.TestCase): - def setUp(self): paddle.enable_static() # test no C++ stack format @@ -57,12 +55,11 @@ class TestExceptionNoCStack(unittest.TestCase): y = numpy.random.random(size=(8, 1)).astype('float32') with self.assertRaises(ValueError): - exe.run(fluid.default_main_program(), - feed={ - 'X': x, - 'Y': y - }, - fetch_list=[avg_loss.name]) + exe.run( + fluid.default_main_program(), + feed={'X': x, 'Y': y}, + fetch_list=[avg_loss.name], + ) def test_exception_in_dynamic_mode(self): place = fluid.CPUPlace() diff --git a/python/paddle/fluid/tests/unittests/test_executor_and_mul.py b/python/paddle/fluid/tests/unittests/test_executor_and_mul.py index a9cd5c4f587..f85f7b97ec8 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_and_mul.py +++ b/python/paddle/fluid/tests/unittests/test_executor_and_mul.py @@ -20,17 +20,15 @@ from paddle.fluid.layers import mul, data, zeros, array_write, increment class TestExecutor(unittest.TestCase): - def test_mul(self): i = zeros(shape=[1], dtype='int64') a = data(name='a', shape=[784], dtype='float32') array = array_write(x=a, i=i) i = increment(i) - b = data(name='b', - shape=[784, 100], - dtype='float32', - append_batch_size=False) + b = data( + name='b', shape=[784, 100], dtype='float32', append_batch_size=False + ) array_write(x=b, i=i, array=array) i = increment(i) @@ -41,11 +39,9 @@ class TestExecutor(unittest.TestCase): b_np = np.random.random((784, 100)).astype('float32') exe = Executor() - res, res_array = exe.run(feed={ - 'a': a_np, - 'b': b_np - }, - fetch_list=[out, array]) + res, res_array = exe.run( + feed={'a': a_np, 'b': b_np}, fetch_list=[out, array] + ) self.assertEqual((100, 100), res.shape) np.testing.assert_allclose(res, np.dot(a_np, b_np), rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_executor_and_use_program_cache.py b/python/paddle/fluid/tests/unittests/test_executor_and_use_program_cache.py index 34f9674f27d..2412794929c 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_and_use_program_cache.py +++ b/python/paddle/fluid/tests/unittests/test_executor_and_use_program_cache.py @@ -21,16 +21,17 @@ from test_eager_deletion_padding_rnn import RNNConfig, PaddingRNNTestBase class TestExecutor(unittest.TestCase): - def test_mul(self): main_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): a = fluid.layers.data(name='a', shape=[784], dtype='float32') - b = fluid.layers.data(name='b', - shape=[784, 100], - dtype='float32', - append_batch_size=False) + b = fluid.layers.data( + name='b', + shape=[784, 100], + dtype='float32', + append_batch_size=False, + ) output = fluid.layers.mul(x=a, y=b) # Compute with numpy @@ -47,13 +48,12 @@ class TestExecutor(unittest.TestCase): run_time = 0.0 for i in range(max_iters): begin = time.time() - outs = exe.run(program=main_program, - feed={ - 'a': a_np, - 'b': b_np - }, - fetch_list=[output.name], - use_program_cache=use_program_cache) + outs = exe.run( + program=main_program, + feed={'a': a_np, 'b': b_np}, + fetch_list=[output.name], + use_program_cache=use_program_cache, + ) end = time.time() run_time += end - begin out = outs[0] @@ -62,29 +62,31 @@ class TestExecutor(unittest.TestCase): return run_time max_iters = 3 - run_time_with_cache = _train(use_program_cache=True, - max_iters=max_iters) + run_time_with_cache = _train( + use_program_cache=True, max_iters=max_iters + ) print("run time with program cache: %f" % run_time_with_cache) - run_time_without_cache = _train(use_program_cache=False, - max_iters=max_iters) + run_time_without_cache = _train( + use_program_cache=False, max_iters=max_iters + ) print("run time without program cache: %f" % run_time_without_cache) - run_time_with_cache = _train(use_program_cache=True, - max_iters=max_iters) + run_time_with_cache = _train( + use_program_cache=True, max_iters=max_iters + ) print("run time with program cache: %f" % run_time_with_cache) - run_time_with_cache = _train(use_program_cache=True, - max_iters=max_iters) + run_time_with_cache = _train( + use_program_cache=True, max_iters=max_iters + ) print("run time with program cache: %f" % run_time_with_cache) class ExecutorPaddingRNNTest(PaddingRNNTestBase): - - def train_and_save_inference_program(self, - rnn_model="static", - parallel=True, - use_program_cache=True): + def train_and_save_inference_program( + self, rnn_model="static", parallel=True, use_program_cache=True + ): config = RNNConfig("test", rnn_model) with fluid.scope_guard(fluid.Scope()): self.train(config, parallel, use_program_cache) @@ -94,55 +96,75 @@ class ExecutorPaddingRNNTest(PaddingRNNTestBase): target_vars=[self.loss, self.last_hidden, self.last_cell], executor=self.exe, dirname="padding_rnn." + rnn_model + ".inference_model", - params_filename="__params__") + params_filename="__params__", + ) def test_inference_output(self): for rnn_model in ["static", "padding"]: # Set parallel to False to use the default executor. - self.train_and_save_inference_program(rnn_model=rnn_model, - parallel=True, - use_program_cache=True) + self.train_and_save_inference_program( + rnn_model=rnn_model, parallel=True, use_program_cache=True + ) - x_np = np.random.random((self.config.batch_size, - self.config.num_steps, 1)).astype("int64") + x_np = np.random.random( + (self.config.batch_size, self.config.num_steps, 1) + ).astype("int64") y_np = np.random.random( - (self.config.batch_size * self.config.num_steps, - 1)).astype("int64") + (self.config.batch_size * self.config.num_steps, 1) + ).astype("int64") init_hidden_np = np.random.random( - (self.config.num_layers, self.config.batch_size, - self.config.hidden_size)).astype("float32") + ( + self.config.num_layers, + self.config.batch_size, + self.config.hidden_size, + ) + ).astype("float32") init_cell_np = np.random.random( - (self.config.num_layers, self.config.batch_size, - self.config.hidden_size)).astype("float32") + ( + self.config.num_layers, + self.config.batch_size, + self.config.hidden_size, + ) + ).astype("float32") for use_program_cache in [False, True]: with fluid.scope_guard(fluid.Scope()): - save_dirname = "padding_rnn." + rnn_model + ".inference_model" - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model( - save_dirname, self.exe, params_filename="__params__") - - results = self.exe.run(program=inference_program, - feed={ - "x": x_np, - "y": y_np, - "init_hidden": init_hidden_np, - "init_cell": init_cell_np - }, - fetch_list=fetch_targets, - use_program_cache=use_program_cache) + save_dirname = ( + "padding_rnn." + rnn_model + ".inference_model" + ) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + save_dirname, self.exe, params_filename="__params__" + ) + + results = self.exe.run( + program=inference_program, + feed={ + "x": x_np, + "y": y_np, + "init_hidden": init_hidden_np, + "init_cell": init_cell_np, + }, + fetch_list=fetch_targets, + use_program_cache=use_program_cache, + ) if use_program_cache is True: results_with_cache = results else: results_without_cache = results - self.assertEqual(len(results_with_cache), - len(results_without_cache)) + self.assertEqual( + len(results_with_cache), len(results_without_cache) + ) for i in range(len(results_with_cache)): - self.assertEqual(results_with_cache[i].shape, - results_without_cache[i].shape) - np.testing.assert_allclose(results_with_cache[i], - results_without_cache[i], - rtol=1e-05) + self.assertEqual( + results_with_cache[i].shape, results_without_cache[i].shape + ) + np.testing.assert_allclose( + results_with_cache[i], results_without_cache[i], rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_executor_check_feed.py b/python/paddle/fluid/tests/unittests/test_executor_check_feed.py index cfcd81a0a35..77122f46bdc 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_check_feed.py +++ b/python/paddle/fluid/tests/unittests/test_executor_check_feed.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class TestExecutor(unittest.TestCase): - def net(self): lr = fluid.data(name="lr", shape=[1], dtype='float32') x = fluid.data(name="x", shape=[None, 1], dtype='float32') @@ -48,13 +47,12 @@ class TestExecutor(unittest.TestCase): y_true = [[2.0], [4.0], [6.0], [8.0]] a = 0 with self.assertRaises(ValueError): - exe.run(feed={ - 'x': train_data, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False, - use_prune=True) + exe.run( + feed={'x': train_data, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + use_prune=True, + ) def test_compiled_program_check_feed(self): main_program = fluid.Program() @@ -67,19 +65,19 @@ class TestExecutor(unittest.TestCase): lr, cost = self.net() exe.run(startup_program) compiled_prog = fluid.CompiledProgram( - main_program).with_data_parallel(loss_name=cost.name) + main_program + ).with_data_parallel(loss_name=cost.name) train_data = [[1.0], [2.0], [3.0], [4.0]] y_true = [[2.0], [4.0], [6.0], [8.0]] a = 0 with self.assertRaises(ValueError): - exe.run(compiled_prog, - feed={ - 'x': train_data, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False, - use_prune=True) + exe.run( + compiled_prog, + feed={'x': train_data, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + use_prune=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_executor_check_fetch_list.py b/python/paddle/fluid/tests/unittests/test_executor_check_fetch_list.py index e91bdf976d3..d6711a612d6 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_check_fetch_list.py +++ b/python/paddle/fluid/tests/unittests/test_executor_check_fetch_list.py @@ -20,7 +20,6 @@ import unittest class TestCheckFetchList(unittest.TestCase): - def setUp(self): paddle.enable_static() self.feed = {"x": np.array([[0], [0], [1], [0]], dtype='float32')} @@ -32,10 +31,9 @@ class TestCheckFetchList(unittest.TestCase): main_program = paddle.static.Program() with paddle.static.program_guard(main_program): x = paddle.static.data(name='x', shape=[4, 1], dtype='float32') - output = paddle.unique_consecutive(x, - return_inverse=True, - return_counts=True, - axis=0) + output = paddle.unique_consecutive( + x, return_inverse=True, return_counts=True, axis=0 + ) self.main_program = main_program self.fetch_list = output @@ -46,22 +44,23 @@ class TestCheckFetchList(unittest.TestCase): self.main_program, feed=self.feed, fetch_list=[self.fetch_list], # support single list/tuple - return_numpy=True) + return_numpy=True, + ) np.testing.assert_array_equal(res[0], self.expected) def test_with_error(self): with self.assertRaises(TypeError): fetch_list = [23] - res = self.exe.run(self.main_program, - feed=self.feed, - fetch_list=fetch_list) + res = self.exe.run( + self.main_program, feed=self.feed, fetch_list=fetch_list + ) with self.assertRaises(TypeError): fetch_list = [(self.fetch_list[0], 32)] - res = self.exe.run(self.main_program, - feed=self.feed, - fetch_list=fetch_list) + res = self.exe.run( + self.main_program, feed=self.feed, fetch_list=fetch_list + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_executor_feed_non_tensor.py b/python/paddle/fluid/tests/unittests/test_executor_feed_non_tensor.py index 6441ba66e57..41b4211a8bc 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_feed_non_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_executor_feed_non_tensor.py @@ -20,7 +20,6 @@ import paddle.fluid as fluid class TestExecutor(unittest.TestCase): - def net(self): lr = fluid.data(name="lr", shape=[1], dtype='float32') x = fluid.data(name="x", shape=[None, 1], dtype='float32') @@ -45,18 +44,18 @@ class TestExecutor(unittest.TestCase): exe = fluid.Executor(cpu) lr, cost = self.net() exe.run(startup_program) - train_data = numpy.array([[1.0], [2.0], [3.0], - [4.0]]).astype('float32') - y_true = numpy.array([[2.0], [4.0], [6.0], - [8.0]]).astype('float32') + train_data = numpy.array([[1.0], [2.0], [3.0], [4.0]]).astype( + 'float32' + ) + y_true = numpy.array([[2.0], [4.0], [6.0], [8.0]]).astype( + 'float32' + ) a = 0.01 - _lr, _ = exe.run(feed={ - 'x': train_data, - 'y': y_true, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False) + _lr, _ = exe.run( + feed={'x': train_data, 'y': y_true, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + ) self.assertEqual(_lr._dtype(), lr.dtype) self.assertEqual(_lr._dtype(), fluid.core.VarDesc.VarType.FP32) self.assertEqual(type(a), float) @@ -71,18 +70,18 @@ class TestExecutor(unittest.TestCase): exe = fluid.Executor(cpu) lr, cost = self.net() exe.run(startup_program) - train_data = numpy.array([[1.0], [2.0], [3.0], - [4.0]]).astype('float32') - y_true = numpy.array([[2.0], [4.0], [6.0], - [8.0]]).astype('float32') + train_data = numpy.array([[1.0], [2.0], [3.0], [4.0]]).astype( + 'float32' + ) + y_true = numpy.array([[2.0], [4.0], [6.0], [8.0]]).astype( + 'float32' + ) a = 0 - _lr, _ = exe.run(feed={ - 'x': train_data, - 'y': y_true, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False) + _lr, _ = exe.run( + feed={'x': train_data, 'y': y_true, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + ) self.assertEqual(_lr._dtype(), lr.dtype) self.assertEqual(_lr._dtype(), fluid.core.VarDesc.VarType.FP32) self.assertEqual(type(a), int) @@ -100,13 +99,11 @@ class TestExecutor(unittest.TestCase): train_data = [[1.0], [2.0], [3.0], [4.0]] y_true = [[2.0], [4.0], [6.0], [8.0]] a = 0 - _lr, _ = exe.run(feed={ - 'x': train_data, - 'y': y_true, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False) + _lr, _ = exe.run( + feed={'x': train_data, 'y': y_true, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + ) self.assertEqual(_lr._dtype(), lr.dtype) self.assertEqual(_lr._dtype(), fluid.core.VarDesc.VarType.FP32) self.assertEqual(type(y_true), list) @@ -122,37 +119,39 @@ class TestExecutor(unittest.TestCase): exe = fluid.Executor(cpu) exe.run(startup_program) compiled_prog = fluid.CompiledProgram( - main_program).with_data_parallel(loss_name=cost.name) - train_data = numpy.array([[1.0], [2.0], [3.0], - [4.0]]).astype('float32') - y_true = numpy.array([[2.0], [4.0], [6.0], - [8.0]]).astype('float32') + main_program + ).with_data_parallel(loss_name=cost.name) + train_data = numpy.array([[1.0], [2.0], [3.0], [4.0]]).astype( + 'float32' + ) + y_true = numpy.array([[2.0], [4.0], [6.0], [8.0]]).astype( + 'float32' + ) a = 0.01 - _lr, _ = exe.run(compiled_prog, - feed={ - 'x': train_data, - 'y': y_true, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False) + _lr, _ = exe.run( + compiled_prog, + feed={'x': train_data, 'y': y_true, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + ) self.assertEqual(_lr._dtype(), lr.dtype) self.assertEqual(_lr._dtype(), fluid.core.VarDesc.VarType.FP32) self.assertEqual(type(a), float) class TestAsLodTensor(unittest.TestCase): - def test_as_lodtensor_int32(self): cpu = fluid.CPUPlace() - tensor = fluid.executor._as_lodtensor(1.0, cpu, - fluid.core.VarDesc.VarType.INT32) + tensor = fluid.executor._as_lodtensor( + 1.0, cpu, fluid.core.VarDesc.VarType.INT32 + ) self.assertEqual(tensor._dtype(), fluid.core.VarDesc.VarType.INT32) def test_as_lodtensor_fp64(self): cpu = fluid.CPUPlace() - tensor = fluid.executor._as_lodtensor(1, cpu, - fluid.core.VarDesc.VarType.FP64) + tensor = fluid.executor._as_lodtensor( + 1, cpu, fluid.core.VarDesc.VarType.FP64 + ) self.assertEqual(tensor._dtype(), fluid.core.VarDesc.VarType.FP64) def test_as_lodtensor_assertion_error(self): @@ -161,25 +160,37 @@ class TestAsLodTensor(unittest.TestCase): def test_as_lodtensor_type_error(self): cpu = fluid.CPUPlace() - self.assertRaises(TypeError, fluid.executor._as_lodtensor, {"a": 1}, - cpu, fluid.core.VarDesc.VarType.INT32) + self.assertRaises( + TypeError, + fluid.executor._as_lodtensor, + {"a": 1}, + cpu, + fluid.core.VarDesc.VarType.INT32, + ) def test_as_lodtensor_list(self): cpu = fluid.CPUPlace() - tensor = fluid.executor._as_lodtensor([1, 2], cpu, - fluid.core.VarDesc.VarType.FP64) + tensor = fluid.executor._as_lodtensor( + [1, 2], cpu, fluid.core.VarDesc.VarType.FP64 + ) self.assertEqual(tensor._dtype(), fluid.core.VarDesc.VarType.FP64) def test_as_lodtensor_tuple(self): cpu = fluid.CPUPlace() - tensor = fluid.executor._as_lodtensor((1, 2), cpu, - fluid.core.VarDesc.VarType.FP64) + tensor = fluid.executor._as_lodtensor( + (1, 2), cpu, fluid.core.VarDesc.VarType.FP64 + ) self.assertEqual(tensor._dtype(), fluid.core.VarDesc.VarType.FP64) def test_as_lodtensor_nested_list(self): cpu = fluid.CPUPlace() - self.assertRaises(TypeError, fluid.executor._as_lodtensor, - [[1], [1, 2]], cpu, fluid.core.VarDesc.VarType.INT32) + self.assertRaises( + TypeError, + fluid.executor._as_lodtensor, + [[1], [1, 2]], + cpu, + fluid.core.VarDesc.VarType.INT32, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_executor_return_tensor_not_overwriting.py b/python/paddle/fluid/tests/unittests/test_executor_return_tensor_not_overwriting.py index fbb45e9c906..7a3063025d5 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_return_tensor_not_overwriting.py +++ b/python/paddle/fluid/tests/unittests/test_executor_return_tensor_not_overwriting.py @@ -20,7 +20,6 @@ from op_test import OpTest, skip_check_grad_ci @skip_check_grad_ci(reason="Not op test but call the method of class OpTest.") class TestExecutorReturnTensorNotOverwritingWithOptest(OpTest): - def setUp(self): pass @@ -30,7 +29,7 @@ class TestExecutorReturnTensorNotOverwritingWithOptest(OpTest): self.out = np.add(self.x, self.y) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.op_type = "elementwise_add" @@ -44,7 +43,7 @@ class TestExecutorReturnTensorNotOverwritingWithOptest(OpTest): self.out = np.dot(self.x, self.y) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.op_type = "mul" @@ -67,7 +66,6 @@ class TestExecutorReturnTensorNotOverwritingWithOptest(OpTest): class TestExecutorReturnTensorNotOverOverwritingWithLayers(unittest.TestCase): - def setUp(self): pass @@ -78,7 +76,8 @@ class TestExecutorReturnTensorNotOverOverwritingWithLayers(unittest.TestCase): program = fluid.default_main_program() if parallel: program = fluid.CompiledProgram(program).with_data_parallel( - places=place) + places=place + ) exe = fluid.Executor(place) out = exe.run(program, fetch_list=[out], return_numpy=False) return out @@ -90,7 +89,8 @@ class TestExecutorReturnTensorNotOverOverwritingWithLayers(unittest.TestCase): program = fluid.default_main_program() if parallel: program = fluid.CompiledProgram(program).with_data_parallel( - places=place) + places=place + ) exe = fluid.Executor(place) out = exe.run(program, fetch_list=[out], return_numpy=False) return out diff --git a/python/paddle/fluid/tests/unittests/test_expand_as_op.py b/python/paddle/fluid/tests/unittests/test_expand_as_op.py index 9815f587ddb..827f7a73a75 100755 --- a/python/paddle/fluid/tests/unittests/test_expand_as_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_as_op.py @@ -29,7 +29,6 @@ def bcast(x, target_tensor): class TestExpandAsOpRank1(OpTest): - def setUp(self): self.op_type = "expand_as" x = np.random.rand(100).astype("float64") @@ -48,7 +47,6 @@ class TestExpandAsOpRank1(OpTest): class TestExpandAsOpRank2(OpTest): - def setUp(self): self.op_type = "expand_as" x = np.random.rand(10, 12).astype("float64") @@ -67,7 +65,6 @@ class TestExpandAsOpRank2(OpTest): class TestExpandAsOpRank3(OpTest): - def setUp(self): self.op_type = "expand_as" x = np.random.rand(2, 3, 20).astype("float64") @@ -86,7 +83,6 @@ class TestExpandAsOpRank3(OpTest): class TestExpandAsOpRank4(OpTest): - def setUp(self): self.op_type = "expand_as" x = np.random.rand(1, 1, 7, 16).astype("float64") @@ -106,9 +102,9 @@ class TestExpandAsOpRank4(OpTest): # Test dygraph API class TestExpandAsDygraphAPI(unittest.TestCase): - def test_api(self): import paddle + paddle.disable_static() np_data_x = np.array([1, 2, 3]).astype('int32') np_data_y = np.array([1, 2, 3, 1, 2, 3]).astype('int32') @@ -122,29 +118,28 @@ class TestExpandAsDygraphAPI(unittest.TestCase): # Test python API class TestExpandAsAPI(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([48, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[48, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[48, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = fluid.layers.expand_as(x, target_tensor=y) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (4, 1))) diff --git a/python/paddle/fluid/tests/unittests/test_expand_as_v2_op.py b/python/paddle/fluid/tests/unittests/test_expand_as_v2_op.py index e0506f8eb52..5f4c04470ff 100755 --- a/python/paddle/fluid/tests/unittests/test_expand_as_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_as_v2_op.py @@ -20,7 +20,6 @@ import paddle.fluid as fluid class TestExpandAsBasic(OpTest): - def setUp(self): self.op_type = "expand_as_v2" self.python_api = paddle.expand_as @@ -40,7 +39,6 @@ class TestExpandAsBasic(OpTest): class TestExpandAsOpRank2(TestExpandAsBasic): - def setUp(self): self.op_type = "expand_as_v2" self.python_api = paddle.expand_as @@ -54,7 +52,6 @@ class TestExpandAsOpRank2(TestExpandAsBasic): class TestExpandAsOpRank3(TestExpandAsBasic): - def setUp(self): self.op_type = "expand_as_v2" self.python_api = paddle.expand_as @@ -68,7 +65,6 @@ class TestExpandAsOpRank3(TestExpandAsBasic): class TestExpandAsOpRank4(TestExpandAsBasic): - def setUp(self): self.op_type = "expand_as_v2" self.python_api = paddle.expand_as @@ -100,7 +96,6 @@ class TestExpandAsOpRank5(TestExpandAsBasic): class TestExpandAsV2Error(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): x1 = fluid.layers.data(name='x1', shape=[4], dtype="uint8") @@ -113,29 +108,28 @@ class TestExpandAsV2Error(unittest.TestCase): # Test python API class TestExpandAsV2API(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([2, 12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[2, 12, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[2, 12, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = paddle.expand_as(x, y=y) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (2, 1, 1))) diff --git a/python/paddle/fluid/tests/unittests/test_expand_op.py b/python/paddle/fluid/tests/unittests/test_expand_op.py index 36be4ab1a9b..fd3dac2472e 100644 --- a/python/paddle/fluid/tests/unittests/test_expand_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_op.py @@ -22,12 +22,12 @@ import paddle # Situation 1: expand_times is a list(without tensor) class TestExpandOpRank1(OpTest): - def setUp(self): self.op_type = "expand" self.init_data() - self.dtype = "float32" if fluid.core.is_compiled_with_rocm( - ) else "float64" + self.dtype = ( + "float32" if fluid.core.is_compiled_with_rocm() else "float64" + ) self.inputs = {'X': np.random.random(self.ori_shape).astype(self.dtype)} self.attrs = {'expand_times': self.expand_times} @@ -46,35 +46,30 @@ class TestExpandOpRank1(OpTest): class TestExpandOpRank2_Corner(TestExpandOpRank1): - def init_data(self): self.ori_shape = [120] self.expand_times = [2] class TestExpandOpRank2(TestExpandOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [2, 3] class TestExpandOpRank3_Corner(TestExpandOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.expand_times = (1, 1, 1) class TestExpandOpRank3(TestExpandOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.expand_times = (2, 1, 4) class TestExpandOpRank4(TestExpandOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.expand_times = (3, 2, 1, 2) @@ -82,17 +77,18 @@ class TestExpandOpRank4(TestExpandOpRank1): # Situation 2: expand_times is a list(with tensor) class TestExpandOpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "expand" self.init_data() - self.dtype = "float32" if fluid.core.is_compiled_with_rocm( - ) else "float64" + self.dtype = ( + "float32" if fluid.core.is_compiled_with_rocm() else "float64" + ) expand_times_tensor = [] for index, ele in enumerate(self.expand_times): - expand_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -115,7 +111,6 @@ class TestExpandOpRank1_tensor_attr(OpTest): class TestExpandOpRank2_Corner_tensor_attr(TestExpandOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -123,7 +118,6 @@ class TestExpandOpRank2_Corner_tensor_attr(TestExpandOpRank1_tensor_attr): class TestExpandOpRank2_attr_tensor(TestExpandOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [2, 3] @@ -132,12 +126,12 @@ class TestExpandOpRank2_attr_tensor(TestExpandOpRank1_tensor_attr): # Situation 3: expand_times is a tensor class TestExpandOpRank1_tensor(OpTest): - def setUp(self): self.op_type = "expand" self.init_data() - self.dtype = "float32" if fluid.core.is_compiled_with_rocm( - ) else "float64" + self.dtype = ( + "float32" if fluid.core.is_compiled_with_rocm() else "float64" + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -159,7 +153,6 @@ class TestExpandOpRank1_tensor(OpTest): class TestExpandOpRank2_tensor(TestExpandOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [2, 3] @@ -167,7 +160,6 @@ class TestExpandOpRank2_tensor(TestExpandOpRank1_tensor): # Situation 4: input x is Integer class TestExpandOpInteger(OpTest): - def setUp(self): self.op_type = "expand" self.inputs = { @@ -183,7 +175,6 @@ class TestExpandOpInteger(OpTest): # Situation 5: input x is Bool class TestExpandOpBoolean(OpTest): - def setUp(self): self.op_type = "expand" self.inputs = {'X': np.random.randint(2, size=(2, 4, 5)).astype("bool")} @@ -197,7 +188,6 @@ class TestExpandOpBoolean(OpTest): # Situation 56: input x is Integer class TestExpandOpInt64_t(OpTest): - def setUp(self): self.op_type = "expand" self.inputs = { @@ -212,11 +202,11 @@ class TestExpandOpInt64_t(OpTest): class TestExpandError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) expand_times = [2, 2] self.assertRaises(TypeError, fluid.layers.expand, x1, expand_times) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -228,18 +218,16 @@ class TestExpandError(unittest.TestCase): # Test python API class TestExpandAPI(unittest.TestCase): - def test_api(self): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) positive_2 = fluid.layers.fill_constant([1], "int32", 2) - expand_times = fluid.layers.data(name="expand_times", - shape=[2], - append_batch_size=False) + expand_times = fluid.layers.data( + name="expand_times", shape=[2], append_batch_size=False + ) out_1 = fluid.layers.expand(x, expand_times=[2, 3]) out_2 = fluid.layers.expand(x, expand_times=[positive_2, 3]) @@ -248,28 +236,24 @@ class TestExpandAPI(unittest.TestCase): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_times": - np.array([1, 3]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={"x": input, "expand_times": np.array([1, 3]).astype("int32")}, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (2, 3))) assert np.array_equal(res_2, np.tile(input, (2, 3))) assert np.array_equal(res_3, np.tile(input, (1, 3))) class TestExpandDygraphAPI(unittest.TestCase): - def test_expand_times_is_tensor(self): with paddle.fluid.dygraph.guard(): a = paddle.rand([2, 5]) b = paddle.fluid.layers.expand(a, expand_times=[2, 3]) - c = paddle.fluid.layers.expand(a, - expand_times=paddle.to_tensor( - [2, 3], dtype='int32')) + c = paddle.fluid.layers.expand( + a, expand_times=paddle.to_tensor([2, 3], dtype='int32') + ) np.testing.assert_array_equal(b.numpy(), np.tile(a.numpy(), [2, 3])) np.testing.assert_array_equal(c.numpy(), np.tile(a.numpy(), [2, 3])) diff --git a/python/paddle/fluid/tests/unittests/test_expand_v2_op.py b/python/paddle/fluid/tests/unittests/test_expand_v2_op.py index d89332f393a..da27fb397cc 100644 --- a/python/paddle/fluid/tests/unittests/test_expand_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_v2_op.py @@ -26,7 +26,6 @@ import paddle.fluid.layers as layers # Situation 1: shape is a list(without tensor) class TestExpandV2OpRank1(OpTest): - def setUp(self): self.op_type = "expand_v2" self.init_data() @@ -50,7 +49,6 @@ class TestExpandV2OpRank1(OpTest): class TestExpandV2OpRank2_DimExpanding(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] @@ -58,7 +56,6 @@ class TestExpandV2OpRank2_DimExpanding(TestExpandV2OpRank1): class TestExpandV2OpRank2(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = [1, 140] self.shape = [12, 140] @@ -66,7 +63,6 @@ class TestExpandV2OpRank2(TestExpandV2OpRank1): class TestExpandV2OpRank3_Corner(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) @@ -74,7 +70,6 @@ class TestExpandV2OpRank3_Corner(TestExpandV2OpRank1): class TestExpandV2OpRank4(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) @@ -83,14 +78,14 @@ class TestExpandV2OpRank4(TestExpandV2OpRank1): # Situation 2: shape is a list(with tensor) class TestExpandV2OpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "expand_v2" self.init_data() expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float64"), @@ -114,7 +109,6 @@ class TestExpandV2OpRank1_tensor_attr(OpTest): class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -124,7 +118,6 @@ class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpRank1_tensor_attr): # Situation 3: shape is a tensor class TestExpandV2OpRank1_tensor(OpTest): - def setUp(self): self.op_type = "expand_v2" self.init_data() @@ -151,7 +144,6 @@ class TestExpandV2OpRank1_tensor(OpTest): # Situation 4: input x is Integer class TestExpandV2OpInteger(OpTest): - def setUp(self): self.op_type = "expand_v2" self.inputs = { @@ -167,7 +159,6 @@ class TestExpandV2OpInteger(OpTest): # Situation 5: input x is Bool class TestExpandV2OpBoolean(OpTest): - def setUp(self): self.op_type = "expand_v2" self.inputs = {'X': np.random.randint(2, size=(2, 4, 5)).astype("bool")} @@ -181,7 +172,6 @@ class TestExpandV2OpBoolean(OpTest): # Situation 56: input x is Integer class TestExpandV2OpInt64_t(OpTest): - def setUp(self): self.op_type = "expand_v2" self.inputs = { @@ -196,11 +186,11 @@ class TestExpandV2OpInt64_t(OpTest): class TestExpandV2Error(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) shape = [2, 2] self.assertRaises(TypeError, paddle.tensor.expand, x1, shape) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -212,19 +202,19 @@ class TestExpandV2Error(unittest.TestCase): # Test python API class TestExpandV2API(unittest.TestCase): - def test_api(self): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.expand(x, shape=[12, 14]) out_2 = paddle.expand(x, shape=[positive_2, 14]) @@ -233,28 +223,28 @@ class TestExpandV2API(unittest.TestCase): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) assert np.array_equal(res_3, np.tile(input, (1, 1))) class TestExpandInferShape(unittest.TestCase): - def test_shape_with_var(self): with program_guard(Program(), Program()): x = paddle.static.data(shape=[-1, 1, 3], name='x') fake_var = paddle.randn([2, 3]) target_shape = [ - -1, paddle.shape(fake_var)[0], - paddle.shape(fake_var)[1] + -1, + paddle.shape(fake_var)[0], + paddle.shape(fake_var)[1], ] out = paddle.expand(x, shape=target_shape) self.assertListEqual(list(out.shape), [-1, -1, -1]) @@ -262,7 +252,6 @@ class TestExpandInferShape(unittest.TestCase): # Test python Dygraph API class TestExpandV2DygraphAPI(unittest.TestCase): - def test_expand_times_is_tensor(self): with paddle.fluid.dygraph.guard(): with _test_eager_guard(): @@ -278,15 +267,16 @@ class TestExpandV2DygraphAPI(unittest.TestCase): np_array = np.array([2, 5]) expand_2 = paddle.expand(a, shape=np_array) - np.testing.assert_array_equal(egr_expand_1.numpy(), - egr_expand_2.numpy()) + np.testing.assert_array_equal( + egr_expand_1.numpy(), egr_expand_2.numpy() + ) np.testing.assert_array_equal(expand_1.numpy(), expand_2.numpy()) - np.testing.assert_array_equal(expand_1.numpy(), - egr_expand_1.numpy()) + np.testing.assert_array_equal( + expand_1.numpy(), egr_expand_1.numpy() + ) class TestExpandDoubleGradCheck(unittest.TestCase): - def expand_wrapper(self, x): return paddle.expand(x[0], [2, 3]) @@ -301,17 +291,13 @@ class TestExpandDoubleGradCheck(unittest.TestCase): out = paddle.expand(data, [2, 3]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.expand_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.expand_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -323,7 +309,6 @@ class TestExpandDoubleGradCheck(unittest.TestCase): class TestExpandTripleGradCheck(unittest.TestCase): - def expand_wrapper(self, x): return paddle.expand(x[0], [2, 3]) @@ -338,17 +323,13 @@ class TestExpandTripleGradCheck(unittest.TestCase): out = paddle.expand(data, [2, 3]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.expand_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.expand_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_exponential_op.py b/python/paddle/fluid/tests/unittests/test_exponential_op.py index 54a6befc8ed..458be9f3635 100644 --- a/python/paddle/fluid/tests/unittests/test_exponential_op.py +++ b/python/paddle/fluid/tests/unittests/test_exponential_op.py @@ -21,7 +21,6 @@ paddle.seed(100) class TestExponentialOp1(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "exponential" @@ -43,7 +42,7 @@ class TestExponentialOp1(OpTest): hist1 = hist1.astype("float32") hist1 = hist1 / float(outs[0].size) - data_np = np.random.exponential(1. / self.lam, [1024, 1024]) + data_np = np.random.exponential(1.0 / self.lam, [1024, 1024]) hist2, _ = np.histogram(data_np, range=(0, 5)) hist2 = hist2.astype("float32") hist2 = hist2 / float(data_np.size) @@ -57,34 +56,36 @@ class TestExponentialOp1(OpTest): user_defined_grads=[np.zeros([1024, 1024], dtype=self.dtype)], user_defined_grad_outputs=[ np.random.rand(1024, 1024).astype(self.dtype) - ]) + ], + ) class TestExponentialOp2(TestExponentialOp1): - def config(self): self.lam = 0.25 self.dtype = "float32" class TestExponentialAPI(unittest.TestCase): - def test_static(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - x_np = np.full([10, 10], -1.) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + x_np = np.full([10, 10], -1.0) x = paddle.static.data(name="X", shape=[10, 10], dtype='float64') x.exponential_(1.0) exe = paddle.static.Executor() - out = exe.run(paddle.static.default_main_program(), - feed={"X": x_np}, - fetch_list=[x]) + out = exe.run( + paddle.static.default_main_program(), + feed={"X": x_np}, + fetch_list=[x], + ) self.assertTrue(np.min(out) >= 0) def test_dygraph(self): paddle.disable_static() - x = paddle.full([10, 10], -1., dtype='float32') + x = paddle.full([10, 10], -1.0, dtype='float32') x.stop_gradient = False y = 2 * x y.exponential_(0.5) @@ -113,47 +114,94 @@ class TestExponentialAPI(unittest.TestCase): x.exponential_(1.0) x_np = x.numpy() expect = [ - 0.80073667, 0.2249291, 0.07734892, 1.25392, 0.14013891, 0.45736602, - 1.9735607, 0.30490234, 0.57100505, 0.8115938 + 0.80073667, + 0.2249291, + 0.07734892, + 1.25392, + 0.14013891, + 0.45736602, + 1.9735607, + 0.30490234, + 0.57100505, + 0.8115938, ] np.testing.assert_allclose(x_np[0, 0, 0, 0:10], expect, rtol=1e-05) expect = [ - 1.4296371e+00, 9.5411777e-01, 5.2575850e-01, 2.4805880e-01, - 1.2322118e-04, 8.4604341e-01, 2.1111444e-01, 1.4143821e+00, - 2.8194717e-01, 1.1360573e+00 + 1.4296371e00, + 9.5411777e-01, + 5.2575850e-01, + 2.4805880e-01, + 1.2322118e-04, + 8.4604341e-01, + 2.1111444e-01, + 1.4143821e00, + 2.8194717e-01, + 1.1360573e00, ] - np.testing.assert_allclose(x_np[16, 1, 300, 200:210], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[16, 1, 300, 200:210], expect, rtol=1e-05 + ) expect = [ - 1.3448033, 0.35146526, 1.7380928, 0.32012638, 0.10396296, - 0.51344526, 0.15308502, 0.18712929, 0.03888268, 0.20771872 + 1.3448033, + 0.35146526, + 1.7380928, + 0.32012638, + 0.10396296, + 0.51344526, + 0.15308502, + 0.18712929, + 0.03888268, + 0.20771872, ] - np.testing.assert_allclose(x_np[32, 1, 600, 500:510], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[32, 1, 600, 500:510], expect, rtol=1e-05 + ) expect = [ - 0.5107464, 0.20970327, 2.1986802, 1.580056, 0.31036147, 0.43966478, - 0.9056133, 0.30119267, 1.4797124, 1.4319834 + 0.5107464, + 0.20970327, + 2.1986802, + 1.580056, + 0.31036147, + 0.43966478, + 0.9056133, + 0.30119267, + 1.4797124, + 1.4319834, ] - np.testing.assert_allclose(x_np[48, 2, 900, 800:810], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[48, 2, 900, 800:810], expect, rtol=1e-05 + ) expect = [ - 3.4640615, 1.1019983, 0.41195083, 0.22681557, 0.291846, 0.53617656, - 1.5791925, 2.4645927, 0.04094889, 0.9057725 + 3.4640615, + 1.1019983, + 0.41195083, + 0.22681557, + 0.291846, + 0.53617656, + 1.5791925, + 2.4645927, + 0.04094889, + 0.9057725, ] - np.testing.assert_allclose(x_np[63, 2, 1023, 1000:1010], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[63, 2, 1023, 1000:1010], expect, rtol=1e-05 + ) x = paddle.empty([10, 10], dtype="float32") x.exponential_(3.0) x_np = x.numpy() expect = [ - 0.02831675, 0.1691551, 0.6798956, 0.69347525, 0.0243443, 0.22180498, - 0.30574575, 0.9839696, 0.2834912, 0.59420055 + 0.02831675, + 0.1691551, + 0.6798956, + 0.69347525, + 0.0243443, + 0.22180498, + 0.30574575, + 0.9839696, + 0.2834912, + 0.59420055, ] np.testing.assert_allclose(x_np[5, 0:10], expect, rtol=1e-05) @@ -161,51 +209,115 @@ class TestExponentialAPI(unittest.TestCase): x.exponential_(0.25) x_np = x.numpy() expect = [ - 10.0541229, 12.67860643, 1.09850734, 7.35289643, 2.65471225, - 3.86217432, 2.97902086, 2.92744479, 2.67927152, 0.19667352 + 10.0541229, + 12.67860643, + 1.09850734, + 7.35289643, + 2.65471225, + 3.86217432, + 2.97902086, + 2.92744479, + 2.67927152, + 0.19667352, ] np.testing.assert_allclose(x_np[0, 0, 0, 100:110], expect, rtol=1e-05) expect = [ - 0.68328125, 3.1454553, 0.92158376, 1.95842188, 1.05296941, - 12.93242051, 5.20255978, 3.3588624, 1.57377174, 5.73194183 + 0.68328125, + 3.1454553, + 0.92158376, + 1.95842188, + 1.05296941, + 12.93242051, + 5.20255978, + 3.3588624, + 1.57377174, + 5.73194183, ] np.testing.assert_allclose(x_np[4, 0, 300, 190:200], expect, rtol=1e-05) expect = [ - 1.37973974, 3.45036798, 7.94625406, 1.62610973, 0.31032122, - 4.13596493, 1.98494535, 1.13207041, 8.30592769, 2.81460147 + 1.37973974, + 3.45036798, + 7.94625406, + 1.62610973, + 0.31032122, + 4.13596493, + 1.98494535, + 1.13207041, + 8.30592769, + 2.81460147, ] np.testing.assert_allclose(x_np[8, 1, 600, 300:310], expect, rtol=1e-05) expect = [ - 2.27710811, 12.25003028, 2.96409124, 4.72405788, 0.67917249, - 4.35856718, 0.46870976, 2.31120149, 9.61595826, 4.64446271 + 2.27710811, + 12.25003028, + 2.96409124, + 4.72405788, + 0.67917249, + 4.35856718, + 0.46870976, + 2.31120149, + 9.61595826, + 4.64446271, ] - np.testing.assert_allclose(x_np[12, 1, 900, 500:510], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[12, 1, 900, 500:510], expect, rtol=1e-05 + ) expect = [ - 0.95883744, 1.57316361, 15.22524512, 20.49559882, 13.70008548, - 3.29430143, 3.90390424, 0.9146657, 0.80972249, 0.33376219 + 0.95883744, + 1.57316361, + 15.22524512, + 20.49559882, + 13.70008548, + 3.29430143, + 3.90390424, + 0.9146657, + 0.80972249, + 0.33376219, ] - np.testing.assert_allclose(x_np[15, 1, 1023, 750:760], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[15, 1, 1023, 750:760], expect, rtol=1e-05 + ) x = paddle.empty([512, 768], dtype="float64") x.exponential_(0.3) x_np = x.numpy() expect = [ - 8.79266704, 4.79596009, 2.75480243, 6.04670011, 0.35379556, - 0.76864868, 3.17428251, 0.26556859, 12.22485885, 10.51690383 + 8.79266704, + 4.79596009, + 2.75480243, + 6.04670011, + 0.35379556, + 0.76864868, + 3.17428251, + 0.26556859, + 12.22485885, + 10.51690383, ] np.testing.assert_allclose(x_np[0, 200:210], expect, rtol=1e-05) expect = [ - 5.6341126, 0.52243418, 5.36410796, 6.83672002, 11.9243311, - 5.85985566, 5.75169548, 0.13877972, 6.1348385, 3.82436519 + 5.6341126, + 0.52243418, + 5.36410796, + 6.83672002, + 11.9243311, + 5.85985566, + 5.75169548, + 0.13877972, + 6.1348385, + 3.82436519, ] np.testing.assert_allclose(x_np[300, 400:410], expect, rtol=1e-05) expect = [ - 4.94883581, 0.56345306, 0.85841585, 1.92287801, 6.10036656, - 1.19524847, 3.64735434, 5.19618716, 2.57467974, 3.49152791 + 4.94883581, + 0.56345306, + 0.85841585, + 1.92287801, + 6.10036656, + 1.19524847, + 3.64735434, + 5.19618716, + 2.57467974, + 3.49152791, ] np.testing.assert_allclose(x_np[500, 700:710], expect, rtol=1e-05) @@ -213,8 +325,16 @@ class TestExponentialAPI(unittest.TestCase): x.exponential_(4.0) x_np = x.numpy() expect = [ - 0.15713826, 0.56395964, 0.0680941, 0.00316643, 0.27046853, - 0.19852724, 0.12776634, 0.09642974, 0.51977551, 1.33739699 + 0.15713826, + 0.56395964, + 0.0680941, + 0.00316643, + 0.27046853, + 0.19852724, + 0.12776634, + 0.09642974, + 0.51977551, + 1.33739699, ] np.testing.assert_allclose(x_np[5, 0:10], expect, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_eye_op.py b/python/paddle/fluid/tests/unittests/test_eye_op.py index c67388c1339..e61037ec1af 100644 --- a/python/paddle/fluid/tests/unittests/test_eye_op.py +++ b/python/paddle/fluid/tests/unittests/test_eye_op.py @@ -26,7 +26,6 @@ from test_attribute_var import UnittestBase class TestEyeOp(OpTest): - def setUp(self): ''' Test eye op with specified shape @@ -38,7 +37,7 @@ class TestEyeOp(OpTest): self.attrs = { 'num_rows': 219, 'num_columns': 319, - 'dtype': framework.convert_np_dtype_to_dtype_(np.int32) + 'dtype': framework.convert_np_dtype_to_dtype_(np.int32), } self.outputs = {'Out': np.eye(219, 319, dtype=np.int32)} @@ -47,7 +46,6 @@ class TestEyeOp(OpTest): class TestEyeOp1(OpTest): - def setUp(self): ''' Test eye op with default parameters @@ -64,7 +62,6 @@ class TestEyeOp1(OpTest): class TestEyeOp2(OpTest): - def setUp(self): ''' Test eye op with specified shape @@ -81,13 +78,12 @@ class TestEyeOp2(OpTest): class API_TestTensorEye(unittest.TestCase): - def test_out(self): with paddle.static.program_guard(paddle.static.Program()): data = paddle.eye(10) place = fluid.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, dtype="float32") self.assertEqual((result == expected_result).all(), True) @@ -95,7 +91,7 @@ class API_TestTensorEye(unittest.TestCase): data = paddle.eye(10, num_columns=7, dtype="float64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, 7, dtype="float64") self.assertEqual((result == expected_result).all(), True) @@ -103,7 +99,7 @@ class API_TestTensorEye(unittest.TestCase): data = paddle.eye(10, dtype="int64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, dtype="int64") self.assertEqual((result == expected_result).all(), True) @@ -125,8 +121,9 @@ class API_TestTensorEye(unittest.TestCase): result = tmp_result expected_result = np.stack(result, axis=0) paddle.enable_static() - self.assertEqual(out.numpy().shape == np.array(expected_result).shape, - True) + self.assertEqual( + out.numpy().shape == np.array(expected_result).shape, True + ) self.assertEqual((out.numpy() == expected_result).all(), True) paddle.disable_static() @@ -141,8 +138,9 @@ class API_TestTensorEye(unittest.TestCase): result = tmp_result expected_result = np.stack(result, axis=0) paddle.enable_static() - self.assertEqual(out.numpy().shape == np.array(expected_result).shape, - True) + self.assertEqual( + out.numpy().shape == np.array(expected_result).shape, True + ) self.assertEqual((out.numpy() == expected_result).all(), True) def test_errors(self): @@ -165,7 +163,6 @@ class API_TestTensorEye(unittest.TestCase): class TestEyeRowsCol(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -191,8 +188,9 @@ class TestEyeRowsCol(UnittestBase): res = exe.run(fetch_list=[tmp, out]) gt = np.eye(3, 10) np.testing.assert_allclose(res[0], gt) - paddle.static.save_inference_model(self.save_path, [x], [tmp, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [tmp, out], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() np.testing.assert_allclose(infer_outs[0], gt) @@ -215,7 +213,6 @@ class TestEyeRowsCol(UnittestBase): class TestEyeRowsCol2(TestEyeRowsCol): - def call_func(self, x): rows = paddle.assign(3) cols = paddle.assign(10) diff --git a/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py b/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py index 239f5a93b1f..b56ff6edbfe 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py @@ -47,11 +47,9 @@ def channel_wise_quantize_max_abs(x, quant_bit=8, quant_axis=0): return y, scales -def channel_wise_dequantize_max_abs(x, - scales, - quant_bits, - quant_axis, - activation_scale=None): +def channel_wise_dequantize_max_abs( + x, scales, quant_bits, quant_axis, activation_scale=None +): assert quant_axis in [0, 1], "The quant_axis should be 0 or 1." if isinstance(quant_bits, list): @@ -72,7 +70,6 @@ def channel_wise_dequantize_max_abs(x, class TestFakeChannelWiseDequantizeMaxAbsOpTwoScales(OpTest): - def set_args(self): self.quant_bits = [8, 8] self.activation_scale = 0.7861 @@ -86,15 +83,19 @@ class TestFakeChannelWiseDequantizeMaxAbsOpTwoScales(OpTest): self.op_type = "fake_channel_wise_dequantize_max_abs" x = np.random.randn(4, 3, 64, 64).astype(self.dtype) yq, scales = channel_wise_quantize_max_abs(x, self.quant_bits[0], 1) - ydq = channel_wise_dequantize_max_abs(yq, scales, self.quant_bits, 1, - self.activation_scale) + ydq = channel_wise_dequantize_max_abs( + yq, scales, self.quant_bits, 1, self.activation_scale + ) self.inputs = { - 'X': - yq, - 'Scales': - [("scales0", np.array(scales).astype(self.dtype)), - ("scales1", np.array([self.activation_scale]).astype(self.dtype))] + 'X': yq, + 'Scales': [ + ("scales0", np.array(scales).astype(self.dtype)), + ( + "scales1", + np.array([self.activation_scale]).astype(self.dtype), + ), + ], } self.attrs = {'quant_bits': self.quant_bits} self.outputs = {'Out': ydq} @@ -104,8 +105,8 @@ class TestFakeChannelWiseDequantizeMaxAbsOpTwoScales(OpTest): class TestFakeChannelWiseDequantizeMaxAbsOpTwoScalesFloat16( - TestFakeChannelWiseDequantizeMaxAbsOpTwoScales): - + TestFakeChannelWiseDequantizeMaxAbsOpTwoScales +): def set_dtype(self): self.dtype = np.float16 @@ -114,7 +115,6 @@ class TestFakeChannelWiseDequantizeMaxAbsOpTwoScalesFloat16( class TestFakeChannelWiseDequantizeMaxAbsOpOneScale(OpTest): - def set_args(self): self.quant_bits = [8] self.quant_axis = 0 @@ -127,18 +127,20 @@ class TestFakeChannelWiseDequantizeMaxAbsOpOneScale(OpTest): self.set_dtype() self.op_type = "fake_channel_wise_dequantize_max_abs" x = np.random.randn(4, 3, 64, 64).astype(self.dtype) - yq, scales = channel_wise_quantize_max_abs(x, self.quant_bits[0], - self.quant_axis) - ydq = channel_wise_dequantize_max_abs(yq, scales, self.quant_bits, - self.quant_axis) + yq, scales = channel_wise_quantize_max_abs( + x, self.quant_bits[0], self.quant_axis + ) + ydq = channel_wise_dequantize_max_abs( + yq, scales, self.quant_bits, self.quant_axis + ) self.inputs = { 'X': yq, - 'Scales': [("scales0", np.array(scales).astype(self.dtype))] + 'Scales': [("scales0", np.array(scales).astype(self.dtype))], } self.attrs = { 'quant_bits': self.quant_bits, - 'quant_axis': self.quant_axis + 'quant_axis': self.quant_axis, } self.outputs = {'Out': ydq} @@ -147,16 +149,16 @@ class TestFakeChannelWiseDequantizeMaxAbsOpOneScale(OpTest): class TestFakeChannelWiseDequantizeMaxAbsOpOneScale1( - TestFakeChannelWiseDequantizeMaxAbsOpOneScale): - + TestFakeChannelWiseDequantizeMaxAbsOpOneScale +): def set_args(self): self.quant_bits = [8] self.quant_axis = 1 class TestFakeChannelWiseDequantizeMaxAbsOpOneScaleFloat16( - TestFakeChannelWiseDequantizeMaxAbsOpOneScale): - + TestFakeChannelWiseDequantizeMaxAbsOpOneScale +): def set_dtype(self): self.dtype = np.float16 @@ -165,8 +167,8 @@ class TestFakeChannelWiseDequantizeMaxAbsOpOneScaleFloat16( class TestFakeChannelWiseDequantizeMaxAbsOpOneScale1Float16( - TestFakeChannelWiseDequantizeMaxAbsOpOneScale1): - + TestFakeChannelWiseDequantizeMaxAbsOpOneScale1 +): def set_dtype(self): self.dtype = np.float16 @@ -175,7 +177,6 @@ class TestFakeChannelWiseDequantizeMaxAbsOpOneScale1Float16( class TestFakeDequantizeMaxAbsOp(OpTest): - def set_args(self): self.num_bits = 8 self.max_range = math.pow(2, self.num_bits - 1) - 1 @@ -200,20 +201,17 @@ class TestFakeDequantizeMaxAbsOp(OpTest): class TestFakeDequantizeMaxAbsOpDouble(TestFakeDequantizeMaxAbsOp): - def set_dtype(self): self.dtype = np.float64 class TestFakeDequantizeMaxAbsOp5Bits(TestFakeDequantizeMaxAbsOp): - def set_args(self): self.num_bits = 5 self.max_range = math.pow(2, self.num_bits - 1) - 1 class TestFakeDequantizeMaxAbsOpFloat16(TestFakeDequantizeMaxAbsOp): - def set_dtype(self): self.dtype = np.float16 @@ -222,7 +220,6 @@ class TestFakeDequantizeMaxAbsOpFloat16(TestFakeDequantizeMaxAbsOp): class TestChannelWiseDequantizeOp(OpTest): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -232,17 +229,19 @@ class TestChannelWiseDequantizeOp(OpTest): self.set_args() self.op_type = "dequantize_linear" x = np.random.randn(4, 3, 64, 64).astype(self.data_type) - yq, scale = channel_wise_quantize_max_abs(x, self.bit_length, - self.quant_axis) - ydq = channel_wise_dequantize_max_abs(yq, scale, self.bit_length, - self.quant_axis) + yq, scale = channel_wise_quantize_max_abs( + x, self.bit_length, self.quant_axis + ) + ydq = channel_wise_dequantize_max_abs( + yq, scale, self.bit_length, self.quant_axis + ) scale = np.array(scale).astype(self.data_type) zero_point = np.zeros(scale.shape, dtype="int32") print('TestChannelWiseDequantizeOp:') self.inputs = {'X': yq, 'Scale': scale, 'ZeroPoint': zero_point} self.attrs = { 'bit_length': self.bit_length, - 'quant_axis': self.quant_axis + 'quant_axis': self.quant_axis, } self.outputs = {'Y': ydq} @@ -251,7 +250,6 @@ class TestChannelWiseDequantizeOp(OpTest): class TestChannelWiseDequantizeOp1(TestChannelWiseDequantizeOp): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -259,7 +257,6 @@ class TestChannelWiseDequantizeOp1(TestChannelWiseDequantizeOp): class TestDequantizeOp(OpTest): - def set_args(self): self.bit_length = 8 self.quant_axis = -1 @@ -278,7 +275,7 @@ class TestDequantizeOp(OpTest): self.inputs = {'X': yq, 'Scale': scale, 'ZeroPoint': zero_point} self.attrs = { 'bit_length': self.bit_length, - 'quant_axis': self.quant_axis + 'quant_axis': self.quant_axis, } self.outputs = {'Y': ydq} @@ -287,7 +284,6 @@ class TestDequantizeOp(OpTest): class TestDequantizeOpDouble(TestDequantizeOp): - def set_args(self): self.bit_length = 8 self.max_range = math.pow(2, self.bit_length - 1) - 1 @@ -296,7 +292,6 @@ class TestDequantizeOpDouble(TestDequantizeOp): class TestDequantizeOp5Bits(TestDequantizeOp): - def set_args(self): self.bit_length = 5 self.max_range = math.pow(2, self.bit_length - 1) - 1 diff --git a/python/paddle/fluid/tests/unittests/test_fake_init_op.py b/python/paddle/fluid/tests/unittests/test_fake_init_op.py index 6ff7abe0c22..4a18c09b389 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_init_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_init_op.py @@ -19,14 +19,14 @@ from paddle.fluid.op import Operator class TestFakeInitOpSelectedRows(unittest.TestCase): - def check_with_place(self, place, is_selected_rows): scope = core.Scope() out_var_name = 'Out' if is_selected_rows: - out_tensor = scope.var( - out_var_name).get_selected_rows().get_tensor() + out_tensor = ( + scope.var(out_var_name).get_selected_rows().get_tensor() + ) else: out_tensor = scope.var(out_var_name).get_tensor() diff --git a/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py b/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py index 49d6329373b..5bcb0eec733 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py @@ -38,16 +38,13 @@ def get_compute_type(dtype): class TestFakeQuantizeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_quantize_abs_max' self.attrs = {'bit_length': 8} - def _fake_quantize_abs_max(self, - dtype, - input_shape, - distribution, - round_type='TiesAwayFromZero'): + def _fake_quantize_abs_max( + self, dtype, input_shape, distribution, round_type='TiesAwayFromZero' + ): input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) scale = np.max(np.abs(input_data)) @@ -55,12 +52,14 @@ class TestFakeQuantizeAbsMaxOp(OpTest): inv_scale = 1.0 / (scale + 1e-6) if scale < 1e-30 else 1.0 / scale if round_type == 'TiesToEven': round_out = np.round( - input_data.astype(compute_type) * inv_scale * bnt) + input_data.astype(compute_type) * inv_scale * bnt + ) output_data = np.clip(round_out, -bnt - 1, bnt) self.attrs['round_type'] = 0 else: output_data = round_c( - input_data.astype(compute_type) * inv_scale * bnt) + input_data.astype(compute_type) * inv_scale * bnt + ) self.attrs['round_type'] = 1 self.inputs = {'X': input_data} self.outputs = {'Out': output_data, 'OutScale': scale} @@ -71,9 +70,9 @@ class TestFakeQuantizeAbsMaxOp(OpTest): self._fake_quantize_abs_max(np.float32, (124, 240), np.random.random) def test_fake_quantize_abs_max_round1(self): - self._fake_quantize_abs_max(np.float32, (124, 240), - np.random.random, - round_type='TiesToEven') + self._fake_quantize_abs_max( + np.float32, (124, 240), np.random.random, round_type='TiesToEven' + ) def test_fake_quantize_abs_max_float16(self): self._fake_quantize_abs_max(np.float16, (124, 240), np.random.random) @@ -82,41 +81,45 @@ class TestFakeQuantizeAbsMaxOp(OpTest): self._fake_quantize_abs_max(np.float32, (10, 10), np.zeros) def test_fake_quantize_abs_max_underflow2(self): - self._fake_quantize_abs_max(np.float32, (10, 10), - lambda shape: np.full(shape, 1e-40)) + self._fake_quantize_abs_max( + np.float32, (10, 10), lambda shape: np.full(shape, 1e-40) + ) class TestFakeChannelWiseQuantizeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_channel_wise_quantize_abs_max' self.attrs = {'bit_length': 8} - def _fake_channel_wise_quantize_abs_max(self, - dtype, - input_shape, - quant_axis, - distribution, - round_type='TiesToEven'): + def _fake_channel_wise_quantize_abs_max( + self, + dtype, + input_shape, + quant_axis, + distribution, + round_type='TiesToEven', + ): assert quant_axis in [0, 1], 'quant_axis should be 0 or 1.' input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 - compute_axis = tuple(i for i in range(len(input_shape)) - if i != quant_axis) + compute_axis = tuple( + i for i in range(len(input_shape)) if i != quant_axis + ) scale_broadcast = np.amax(input_data, axis=compute_axis, keepdims=True) if round_type == 'TiesToEven': round_out = np.round( - input_data.astype(compute_type) / scale_broadcast * bnt) + input_data.astype(compute_type) / scale_broadcast * bnt + ) output_data = np.clip(round_out, -bnt - 1, bnt) self.attrs['round_type'] = 0 else: - output_data = round_c(bnt * input_data.astype(compute_type) / - scale_broadcast) + output_data = round_c( + bnt * input_data.astype(compute_type) / scale_broadcast + ) self.attrs['round_type'] = 1 if quant_axis == 1: - scale_broadcast = np.transpose(scale_broadcast, - (1, ) + compute_axis) + scale_broadcast = np.transpose(scale_broadcast, (1,) + compute_axis) scale = scale_broadcast.reshape(input_shape[quant_axis], -1)[:, 0] self.inputs = {'X': input_data} self.outputs = {'Out': output_data, 'OutScale': scale} @@ -126,35 +129,41 @@ class TestFakeChannelWiseQuantizeAbsMaxOp(OpTest): def test_fake_channel_wise_quantize_abs_max(self): dtype_options = [np.float32, np.float16] - input_shape_quant_axis_options = [[(20, 15, 6, 6), 0], - [(20, 15, 6, 6), 1], [(30, 30), 0], - [(30, 30), 1]] + input_shape_quant_axis_options = [ + [(20, 15, 6, 6), 0], + [(20, 15, 6, 6), 1], + [(30, 30), 0], + [(30, 30), 1], + ] round_type_options = ['TiesToEven', 'TiesAwayFromZero'] for dtype, input_shape_quant_axis, round_type in itertools.product( - dtype_options, input_shape_quant_axis_options, - round_type_options): + dtype_options, input_shape_quant_axis_options, round_type_options + ): input_shape, quant_axis = input_shape_quant_axis - with self.subTest(dtype=dtype, - input_shape=input_shape, - quant_axis=quant_axis, - round_type=round_type): + with self.subTest( + dtype=dtype, + input_shape=input_shape, + quant_axis=quant_axis, + round_type=round_type, + ): self._fake_channel_wise_quantize_abs_max( - dtype, input_shape, quant_axis, np.random.random, - round_type) + dtype, input_shape, quant_axis, np.random.random, round_type + ) class TestFakeQuantizeRangeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_quantize_range_abs_max' self.attrs = {'bit_length': 5, 'window_size': 1} - def _fake_quantize_range_abs_max(self, - dtype, - input_shape, - distribution, - is_test=False, - round_type='TiesToEven'): + def _fake_quantize_range_abs_max( + self, + dtype, + input_shape, + distribution, + is_test=False, + round_type='TiesToEven', + ): input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 @@ -165,7 +174,8 @@ class TestFakeQuantizeRangeAbsMaxOp(OpTest): out_scale[0] = in_scale[0] = out_scale[0] - 1.0 if round_type == 'TiesToEven': round_out = np.round( - input_data.astype(compute_type) / out_scale[0] * bnt) + input_data.astype(compute_type) / out_scale[0] * bnt + ) self.attrs['round_type'] = 0 output_data = np.clip(round_out, -bnt - 1, bnt) else: @@ -174,17 +184,18 @@ class TestFakeQuantizeRangeAbsMaxOp(OpTest): else: clip_data = input_data output_data = round_c( - clip_data.astype(compute_type) / out_scale[0] * bnt) + clip_data.astype(compute_type) / out_scale[0] * bnt + ) self.attrs['round_type'] = 1 self.inputs = { 'X': input_data, 'Iter': np.zeros(1).astype(np.int64), - 'InScale': in_scale + 'InScale': in_scale, } self.outputs = { 'Out': output_data, 'OutScale': out_scale[0], - 'OutScales': out_scale + 'OutScales': out_scale, } self.dtype = dtype self.attrs['is_test'] = is_test @@ -195,20 +206,22 @@ class TestFakeQuantizeRangeAbsMaxOp(OpTest): is_test_options = [False, True] round_type_options = ['TiesToEven', 'TiesAwayFromZero'] for dtype, is_test, round_type in itertools.product( - dtype_options, is_test_options, round_type_options): + dtype_options, is_test_options, round_type_options + ): self.attrs['bit_length'] = 8 if is_test else 5 - with self.subTest(dtype=dtype, - is_test=is_test, - round_type=round_type): + with self.subTest( + dtype=dtype, is_test=is_test, round_type=round_type + ): self._fake_quantize_range_abs_max( - dtype, (8, 16, 6, 6), + dtype, + (8, 16, 6, 6), lambda shape: (np.random.random(shape) - 0.4) * 10, is_test=is_test, - round_type=round_type) + round_type=round_type, + ) class TestMovingAverageAbsMaxScaleOp(OpTest): - def setUp(self): self.op_type = 'moving_average_abs_max_scale' self.attrs = {'moving_rate': float(0.9), 'is_test': False} @@ -218,41 +231,44 @@ class TestMovingAverageAbsMaxScaleOp(OpTest): in_accum = np.ones(1).astype(dtype) in_state = np.ones(1).astype(dtype) out_accum = self.attrs['moving_rate'] * in_accum[0] + np.max( - np.abs(input_data)) + np.abs(input_data) + ) out_state = self.attrs['moving_rate'] * in_state[0] + 1.0 out_scale = out_accum / out_state self.inputs = { 'X': input_data, 'InAccum': in_accum, - 'InState': in_state + 'InState': in_state, } self.outputs = { 'Out': input_data, 'OutAccum': out_accum, 'OutState': out_state, - 'OutScale': out_scale + 'OutScale': out_scale, } self.dtype = dtype self.check_output() def test_moving_average_abs_max(self): - self._moving_average_abs_max_scale(np.float32, (8, 16, 7, 7), - np.random.random) + self._moving_average_abs_max_scale( + np.float32, (8, 16, 7, 7), np.random.random + ) class TestFakeQuantizeMovingAverageAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_quantize_moving_average_abs_max' self.attrs = {'bit_length': 5, 'moving_rate': 0.9, 'is_test': False} - def _fake_quantize_moving_average_abs_max(self, - dtype, - input_shape, - distribution, - dequantize=False, - with_gradient=False, - round_type='TiesAwayFromZero'): + def _fake_quantize_moving_average_abs_max( + self, + dtype, + input_shape, + distribution, + dequantize=False, + with_gradient=False, + round_type='TiesAwayFromZero', + ): input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 @@ -263,17 +279,20 @@ class TestFakeQuantizeMovingAverageAbsMaxOp(OpTest): out_state = np.zeros(1).astype(dtype) out_scale = np.zeros(1).astype(dtype) out_accum[0] = self.attrs['moving_rate'] * in_accum[0] + np.max( - np.abs(input_data)) + np.abs(input_data) + ) out_state[0] = self.attrs['moving_rate'] * in_state[0] + 1.0 out_scale = out_accum / out_state if round_type == 'TiesToEven': round_out = np.round( - input_data.astype(compute_type) / out_scale * bnt) + input_data.astype(compute_type) / out_scale * bnt + ) quant_data = np.clip(round_out, -bnt - 1, bnt) self.attrs['round_type'] = 0 else: quant_data = round_c( - input_data.astype(compute_type) / out_scale * bnt) + input_data.astype(compute_type) / out_scale * bnt + ) self.attrs['round_type'] = 1 if dequantize: output_data = (quant_data * out_scale / bnt).astype(dtype) @@ -284,13 +303,13 @@ class TestFakeQuantizeMovingAverageAbsMaxOp(OpTest): 'X': input_data, 'InScale': in_scale, 'InAccum': in_accum, - 'InState': in_state + 'InState': in_state, } self.outputs = { 'Out': output_data, 'OutAccum': out_accum, 'OutState': out_state, - 'OutScale': out_scale + 'OutScale': out_scale, } self.dtype = dtype self.check_output() @@ -301,36 +320,38 @@ class TestFakeQuantizeMovingAverageAbsMaxOp(OpTest): self.check_grad(['X'], 'Out', user_defined_grads=gradient) def test_fake_quantize_moving_average_abs_max(self): - self._fake_quantize_moving_average_abs_max(np.float32, (8, 16, 7, 7), - np.random.random) + self._fake_quantize_moving_average_abs_max( + np.float32, (8, 16, 7, 7), np.random.random + ) def test_fake_quantize_moving_average_abs_max_float16(self): - self._fake_quantize_moving_average_abs_max(np.float16, (8, 16, 7, 7), - np.random.random) + self._fake_quantize_moving_average_abs_max( + np.float16, (8, 16, 7, 7), np.random.random + ) def test_fake_quantize_moving_average_abs_max_round1(self): - self._fake_quantize_moving_average_abs_max(np.float32, (8, 16, 7, 7), - np.random.random, - round_type='TiesToEven') + self._fake_quantize_moving_average_abs_max( + np.float32, (8, 16, 7, 7), np.random.random, round_type='TiesToEven' + ) def test_fake_quantize_dequantize_moving_average_abs_max(self): - self._fake_quantize_moving_average_abs_max(np.float32, (8, 16, 7, 7), - np.random.random, - dequantize=True, - with_gradient=True) + self._fake_quantize_moving_average_abs_max( + np.float32, + (8, 16, 7, 7), + np.random.random, + dequantize=True, + with_gradient=True, + ) class TestFakeQuantizeDequantizeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_quantize_dequantize_abs_max' self.attrs = {'bit_length': 8} - def _fake_quantize_dequantize_abs_max(self, - dtype, - input_shape, - distribution, - round_type='TiesAwayFromZero'): + def _fake_quantize_dequantize_abs_max( + self, dtype, input_shape, distribution, round_type='TiesAwayFromZero' + ): input_data = distribution(input_shape).astype(dtype) scale = np.max(np.abs(input_data)).astype(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 @@ -344,7 +365,7 @@ class TestFakeQuantizeDequantizeAbsMaxOp(OpTest): self.inputs = {'X': input_data} self.outputs = { 'Out': output_data, - 'OutScale': np.array(scale).astype(dtype) + 'OutScale': np.array(scale).astype(dtype), } self.dtype = dtype self.check_output() @@ -352,47 +373,53 @@ class TestFakeQuantizeDequantizeAbsMaxOp(OpTest): self.check_grad(['X'], 'Out', user_defined_grads=gradient) def test_fake_quantize_dequantize_abs_max(self): - self._fake_quantize_dequantize_abs_max(np.float32, (124, 240), - np.random.random) + self._fake_quantize_dequantize_abs_max( + np.float32, (124, 240), np.random.random + ) def test_fake_quantize_dequantize_abs_max_round1(self): - self._fake_quantize_dequantize_abs_max(np.float32, (124, 240), - np.random.random, - round_type='TiesToEven') + self._fake_quantize_dequantize_abs_max( + np.float32, (124, 240), np.random.random, round_type='TiesToEven' + ) class TestChannelWiseFakeQuantizeDequantizeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_channel_wise_quantize_dequantize_abs_max' self.attrs = {'bit_length': 8} - def _fake_channel_wise_quantize_dequantize_abs_max(self, - dtype, - input_shape, - quant_axis, - distribution, - round_type='TiesToEven'): + def _fake_channel_wise_quantize_dequantize_abs_max( + self, + dtype, + input_shape, + quant_axis, + distribution, + round_type='TiesToEven', + ): assert quant_axis in [0, 1], 'quant_axis should be 0 or 1.' input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 output_data = input_data.copy().astype(compute_type) - compute_axis = tuple(i for i in range(len(input_shape)) - if i != quant_axis) + compute_axis = tuple( + i for i in range(len(input_shape)) if i != quant_axis + ) scale_broadcast = np.amax(input_data, axis=compute_axis, keepdims=True) if round_type == 'TiesToEven': round_out = np.round(bnt * output_data / scale_broadcast) - output_data = np.clip(round_out, -bnt - 1, - bnt) * scale_broadcast / bnt + output_data = ( + np.clip(round_out, -bnt - 1, bnt) * scale_broadcast / bnt + ) self.attrs['round_type'] = 0 else: - output_data = round_c( - bnt * output_data / scale_broadcast) * scale_broadcast / bnt + output_data = ( + round_c(bnt * output_data / scale_broadcast) + * scale_broadcast + / bnt + ) self.attrs['round_type'] = 1 if quant_axis == 1: - scale_broadcast = np.transpose(scale_broadcast, - (1, ) + compute_axis) + scale_broadcast = np.transpose(scale_broadcast, (1,) + compute_axis) scale = scale_broadcast.reshape(input_shape[quant_axis], -1)[:, 0] self.inputs = {'X': input_data} self.outputs = {'Out': output_data, 'OutScale': scale} @@ -403,22 +430,29 @@ class TestChannelWiseFakeQuantizeDequantizeAbsMaxOp(OpTest): self.check_grad(['X'], 'Out', user_defined_grads=gradient) def test_channel_wise_fake_quant_dequant_abs_max(self): - input_shape_quant_axis_options = [[(3, 4, 64, 64), 0], - [(15, 20, 5, 5), 1], [(30, 15), 0], - [(30, 15), 1]] + input_shape_quant_axis_options = [ + [(3, 4, 64, 64), 0], + [(15, 20, 5, 5), 1], + [(30, 15), 0], + [(30, 15), 1], + ] round_type_options = ['TiesToEven', 'TiesAwayFromZero'] for input_shape_quant_axis, round_type in itertools.product( - input_shape_quant_axis_options, round_type_options): + input_shape_quant_axis_options, round_type_options + ): input_shape, quant_axis = input_shape_quant_axis - with self.subTest(input_shape=input_shape, - quant_axis=quant_axis, - round_type=round_type): + with self.subTest( + input_shape=input_shape, + quant_axis=quant_axis, + round_type=round_type, + ): self._fake_channel_wise_quantize_dequantize_abs_max( np.float32, input_shape, quant_axis, np.random.random, - round_type=round_type) + round_type=round_type, + ) def quantize_max_abs(x, max_range): @@ -446,7 +480,6 @@ def channel_wise_quantize_max_abs(x, quant_bit=8, quant_axis=0): class TestChannelWiseQuantizeOp(OpTest): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -456,15 +489,16 @@ class TestChannelWiseQuantizeOp(OpTest): self.set_args() self.op_type = "quantize_linear" x = np.random.randn(4, 3, 64, 64).astype(self.data_type) - yq, scale = channel_wise_quantize_max_abs(x, self.bit_length, - self.quant_axis) + yq, scale = channel_wise_quantize_max_abs( + x, self.bit_length, self.quant_axis + ) scale = np.array(scale).astype(self.data_type) zero_point = np.zeros(scale.shape, dtype="int32") self.inputs = {'X': x, 'Scale': scale, 'ZeroPoint': zero_point} self.attrs = { 'bit_length': self.bit_length, - 'quant_axis': self.quant_axis + 'quant_axis': self.quant_axis, } self.outputs = {'Y': yq} @@ -473,7 +507,6 @@ class TestChannelWiseQuantizeOp(OpTest): class TestChannelWiseQuantizeOp1(TestChannelWiseQuantizeOp): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -481,7 +514,6 @@ class TestChannelWiseQuantizeOp1(TestChannelWiseQuantizeOp): class TestChannelWiseQuantizeOpTrain(OpTest): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -492,8 +524,9 @@ class TestChannelWiseQuantizeOpTrain(OpTest): self.set_args() self.op_type = "quantize_linear" x = np.random.randn(4, 3, 64, 64).astype(self.data_type) - yq, scale = channel_wise_quantize_max_abs(x, self.bit_length, - self.quant_axis) + yq, scale = channel_wise_quantize_max_abs( + x, self.bit_length, self.quant_axis + ) scale = np.array(scale).astype(self.data_type) zero_point = np.zeros(scale.shape, dtype="int32") @@ -501,7 +534,7 @@ class TestChannelWiseQuantizeOpTrain(OpTest): self.attrs = { 'bit_length': self.bit_length, 'quant_axis': self.quant_axis, - 'is_test': self.is_test + 'is_test': self.is_test, } self.outputs = {'Y': yq, 'OutScale': scale} @@ -510,7 +543,6 @@ class TestChannelWiseQuantizeOpTrain(OpTest): class TestquantizeOp(OpTest): - def set_args(self): self.bit_length = 8 self.quant_axis = -1 @@ -537,7 +569,6 @@ class TestquantizeOp(OpTest): class TestquantizeOpTrain(TestquantizeOp): - def set_args(self): self.bit_length = 8 self.quant_axis = -1 @@ -552,7 +583,7 @@ class TestquantizeOpTrain(TestquantizeOp): 'bit_length': self.bit_length, 'quant_axis': self.quant_axis, 'moving_rate': 0.9, - 'is_test': self.is_test + 'is_test': self.is_test, } x = np.random.randn(31, 65).astype(self.data_type) @@ -563,7 +594,8 @@ class TestquantizeOpTrain(TestquantizeOp): out_accum = np.zeros(1).astype(self.data_type) out_state = np.zeros(1).astype(self.data_type) out_accum[0] = self.attrs['moving_rate'] * in_accum[0] + np.max( - np.abs(x)) + np.abs(x) + ) out_state[0] = self.attrs['moving_rate'] * in_state[0] + 1.0 out_scale = out_accum / out_state diff --git a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py index 44aa0225059..b21f25b75a3 100755 --- a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py +++ b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py @@ -38,8 +38,13 @@ def to_string_tensor(string_values, name): string_values(list[string]): The value will be setted to the tensor. name(string): The name of the tensor. """ - tensor = paddle.Tensor(core.VarDesc.VarType.STRING, [], name, - core.VarDesc.VarType.STRINGS, False) + tensor = paddle.Tensor( + core.VarDesc.VarType.STRING, + [], + name, + core.VarDesc.VarType.STRINGS, + False, + ) tensor.value().set_string_list(string_values) return tensor @@ -54,31 +59,42 @@ def to_map_tensor(string_dict, name): string_dict(dict): The value will be setted to the tensor. name(string): The name of the tensor. """ - tensor = paddle.Tensor(core.VarDesc.VarType.RAW, [], name, - core.VarDesc.VarType.VOCAB, True) + tensor = paddle.Tensor( + core.VarDesc.VarType.RAW, [], name, core.VarDesc.VarType.VOCAB, True + ) tensor.value().set_vocab(string_dict) return tensor class FasterTokenizer(nn.Layer): - def __init__(self, vocab_dict): super(FasterTokenizer, self).__init__() vocab_tensor = to_map_tensor(vocab_dict, "vocab") self.register_buffer("vocab", vocab_tensor, persistable=True) - def forward(self, - text, - text_pair=None, - do_lower_case=True, - max_seq_len=-1, - is_split_into_words=False, - pad_to_max_seq_len=False): + def forward( + self, + text, + text_pair=None, + do_lower_case=True, + max_seq_len=-1, + is_split_into_words=False, + pad_to_max_seq_len=False, + ): if _non_static_mode(): input_ids, seg_ids = _legacy_C_ops.faster_tokenizer( - self.vocab, text, text_pair, "do_lower_case", do_lower_case, - "max_seq_len", max_seq_len, "pad_to_max_seq_len", - pad_to_max_seq_len, "is_split_into_words", is_split_into_words) + self.vocab, + text, + text_pair, + "do_lower_case", + do_lower_case, + "max_seq_len", + max_seq_len, + "pad_to_max_seq_len", + pad_to_max_seq_len, + "is_split_into_words", + is_split_into_words, + ) return input_ids, seg_ids attrs = { @@ -91,33 +107,27 @@ class FasterTokenizer(nn.Layer): input_ids = helper.create_variable_for_type_inference(dtype="int64") seg_ids = helper.create_variable_for_type_inference(dtype="int64") if text_pair is None: - helper.append_op(type='faster_tokenizer', - inputs={ - 'Vocab': self.vocab, - 'Text': text - }, - outputs={ - 'InputIds': input_ids, - 'SegmentIds': seg_ids - }, - attrs=attrs) + helper.append_op( + type='faster_tokenizer', + inputs={'Vocab': self.vocab, 'Text': text}, + outputs={'InputIds': input_ids, 'SegmentIds': seg_ids}, + attrs=attrs, + ) else: - helper.append_op(type='faster_tokenizer', - inputs={ - 'Vocab': self.vocab, - 'Text': text, - 'TextPair': text_pair - }, - outputs={ - 'InputIds': input_ids, - 'SegmentIds': seg_ids - }, - attrs=attrs) + helper.append_op( + type='faster_tokenizer', + inputs={ + 'Vocab': self.vocab, + 'Text': text, + 'TextPair': text_pair, + }, + outputs={'InputIds': input_ids, 'SegmentIds': seg_ids}, + attrs=attrs, + ) return input_ids, seg_ids class Predictor(object): - def __init__(self, model_dir): model_file = os.path.join(model_dir, "inference.pdmodel") params_file = os.path.join(model_dir, "inference.pdiparams") @@ -152,7 +162,6 @@ class Predictor(object): class TestBertTokenizerOp(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese") @@ -177,11 +186,12 @@ class TestBertTokenizerOp(unittest.TestCase): '很好的地理位置，一蹋糊涂的服务，萧条的酒店。', ' 选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。酒店装修一般，' '但还算整洁。 泳池在大堂的屋顶，因此很小，不过女儿倒是喜欢。 包的早餐是西式的，还算丰富。 服务吗，一般', - 'Test bert tokenizer. The first text.' + 'Test bert tokenizer. The first text.', ] self.text_pairs = [ - '非常不错，服务很好，位于市中心区，交通方便，不过价格也高！', '房间太小。其他的都一般。。。。。。。。。', - 'Test bert tokenizer. The second text.' + '非常不错，服务很好，位于市中心区，交通方便，不过价格也高！', + '房间太小。其他的都一般。。。。。。。。。', + 'Test bert tokenizer. The second text.', ] self.texts_tensor = to_string_tensor(self.texts, "texts") self.text_pairs_tensor = to_string_tensor(self.text_pairs, "text_pairs") @@ -198,7 +208,8 @@ class TestBertTokenizerOp(unittest.TestCase): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -206,15 +217,16 @@ class TestBertTokenizerOp(unittest.TestCase): text=self.text, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) # case 2: only one text and one text_pair (batch_size = 1) input_ids, token_type_ids = self.faster_tokenizer( @@ -223,7 +235,8 @@ class TestBertTokenizerOp(unittest.TestCase): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -232,15 +245,16 @@ class TestBertTokenizerOp(unittest.TestCase): text_pair=self.text_pair, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) # case 3: only texts (batch_size = 3) input_ids, token_type_ids = self.faster_tokenizer( @@ -248,7 +262,8 @@ class TestBertTokenizerOp(unittest.TestCase): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -256,16 +271,16 @@ class TestBertTokenizerOp(unittest.TestCase): self.texts, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = [i["input_ids"] for i in encoded_inputs] py_token_type_ids = [i["token_type_ids"] for i in encoded_inputs] py_input_ids = np.array(py_input_ids).reshape([3, -1]) py_token_type_ids = np.array(py_token_type_ids).reshape([3, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) # case 4: texts and text pairs (batch_size = 3) input_ids, token_type_ids = self.faster_tokenizer( @@ -274,7 +289,8 @@ class TestBertTokenizerOp(unittest.TestCase): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -283,16 +299,16 @@ class TestBertTokenizerOp(unittest.TestCase): self.text_pairs, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = [i["input_ids"] for i in encoded_inputs] py_token_type_ids = [i["token_type_ids"] for i in encoded_inputs] py_input_ids = np.array(py_input_ids).reshape([3, -1]) py_token_type_ids = np.array(py_token_type_ids).reshape([3, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) def test_padding(self): with _test_eager_guard(): @@ -311,7 +327,8 @@ class TestBertTokenizerOp(unittest.TestCase): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -319,15 +336,16 @@ class TestBertTokenizerOp(unittest.TestCase): self.text, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) # case 2: only one text and one text_pair (batch_size = 1) input_ids, token_type_ids = self.faster_tokenizer( @@ -336,7 +354,8 @@ class TestBertTokenizerOp(unittest.TestCase): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -345,15 +364,16 @@ class TestBertTokenizerOp(unittest.TestCase): self.text_pair, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) def test_no_padding(self): with _test_eager_guard(): @@ -367,19 +387,21 @@ class TestBertTokenizerOp(unittest.TestCase): input_ids, token_type_ids = self.faster_tokenizer( self.text_tensor, do_lower_case=self.bert_tokenizer.do_lower_case, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() encoded_inputs = self.bert_tokenizer( - list(self.text[0]), is_split_into_words=self.is_split_into_words) + list(self.text[0]), is_split_into_words=self.is_split_into_words + ) py_input_ids = np.array(encoded_inputs["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array(encoded_inputs["token_type_ids"]).reshape( - [1, -1]) + [1, -1] + ) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) def test_is_split_into_words(self): with _test_eager_guard(): @@ -398,8 +420,10 @@ class TestBertTokenizerOp(unittest.TestCase): self.faster_tokenizer, input_spec=[ paddle.static.InputSpec( - shape=[None], dtype=core.VarDesc.VarType.STRINGS), # texts - ]) + shape=[None], dtype=core.VarDesc.VarType.STRINGS + ), # texts + ], + ) # Save in static graph model. paddle.jit.save(static_model, self.inference_path) predictor = Predictor(self.save_path) @@ -408,19 +432,19 @@ class TestBertTokenizerOp(unittest.TestCase): encoded_inputs = self.bert_tokenizer(self.text) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) def test_feed_string_var(self): self.init_data() paddle.enable_static() - x = paddle.static.data(name="x", - shape=[-1], - dtype=core.VarDesc.VarType.STRINGS) + x = paddle.static.data( + name="x", shape=[-1], dtype=core.VarDesc.VarType.STRINGS + ) exe = paddle.static.Executor(paddle.framework.CPUPlace()) exe.run(paddle.static.default_main_program(), feed={'x': self.text}) paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_fc_op.py b/python/paddle/fluid/tests/unittests/test_fc_op.py index ac69b8e8c6f..8003c0f99e8 100644 --- a/python/paddle/fluid/tests/unittests/test_fc_op.py +++ b/python/paddle/fluid/tests/unittests/test_fc_op.py @@ -43,7 +43,6 @@ def fc_refer(matrix, with_bias, with_relu=False): class MatrixGenerate: - def __init__(self, mb, ic, oc, h, w, bias_dims=2): self.input = np.random.random((mb, ic, h, w)).astype("float32") self.weights = np.random.random((ic * h * w, oc)).astype("float32") @@ -54,7 +53,6 @@ class MatrixGenerate: class TestFCOp(OpTest): - def config(self): self.with_bias = True self.with_relu = True @@ -68,7 +66,7 @@ class TestFCOp(OpTest): self.inputs = { 'Input': self.matrix.input, 'W': self.matrix.weights, - 'Bias': self.matrix.bias + 'Bias': self.matrix.bias, } else: self.inputs = {'Input': self.matrix.input, 'W': self.matrix.weights} @@ -88,7 +86,6 @@ class TestFCOp(OpTest): class TestFCOpNoBias1(TestFCOp): - def config(self): self.with_bias = False self.with_relu = False @@ -96,7 +93,6 @@ class TestFCOpNoBias1(TestFCOp): class TestFCOpNoBias2(TestFCOp): - def config(self): self.with_bias = False self.with_relu = False @@ -104,7 +100,6 @@ class TestFCOpNoBias2(TestFCOp): class TestFCOpNoBias4(TestFCOp): - def config(self): self.with_bias = False self.with_relu = False @@ -112,7 +107,6 @@ class TestFCOpNoBias4(TestFCOp): class TestFCOpWithBias1(TestFCOp): - def config(self): self.with_bias = True self.with_relu = False @@ -120,7 +114,6 @@ class TestFCOpWithBias1(TestFCOp): class TestFCOpWithBias2(TestFCOp): - def config(self): self.with_bias = True self.with_relu = True @@ -128,7 +121,6 @@ class TestFCOpWithBias2(TestFCOp): class TestFCOpWithBias3(TestFCOp): - def config(self): self.with_bias = True self.with_relu = True @@ -136,7 +128,6 @@ class TestFCOpWithBias3(TestFCOp): class TestFCOpWithPadding(TestFCOp): - def config(self): self.with_bias = True self.with_relu = True @@ -144,9 +135,7 @@ class TestFCOpWithPadding(TestFCOp): class TestFcOp_NumFlattenDims_NegOne(unittest.TestCase): - def test_api(self): - def run_program(num_flatten_dims): paddle.seed(SEED) np.random.seed(SEED) @@ -155,17 +144,22 @@ class TestFcOp_NumFlattenDims_NegOne(unittest.TestCase): with program_guard(main_program, startup_program): input = np.random.random([2, 2, 25]).astype("float32") - x = fluid.layers.data(name="x", - shape=[2, 2, 25], - append_batch_size=False, - dtype="float32") - - out = paddle.static.nn.fc(x=x, - size=1, - num_flatten_dims=num_flatten_dims) - - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + x = fluid.layers.data( + name="x", + shape=[2, 2, 25], + append_batch_size=False, + dtype="float32", + ) + + out = paddle.static.nn.fc( + x=x, size=1, num_flatten_dims=num_flatten_dims + ) + + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = fluid.Executor(place=place) exe.run(startup_program) out = exe.run(main_program, feed={"x": input}, fetch_list=[out]) @@ -177,7 +171,6 @@ class TestFcOp_NumFlattenDims_NegOne(unittest.TestCase): class TestFCOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((2, 4)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_feed_data_check_shape_type.py b/python/paddle/fluid/tests/unittests/test_feed_data_check_shape_type.py index be58e03f661..3b885f26683 100644 --- a/python/paddle/fluid/tests/unittests/test_feed_data_check_shape_type.py +++ b/python/paddle/fluid/tests/unittests/test_feed_data_check_shape_type.py @@ -27,8 +27,8 @@ np.random.seed(123) class TestFeedData(unittest.TestCase): ''' - Test paddle.fluid.data feeds with different shape and types. - Note: paddle.fluid.data is not paddle.fluid.layers.data. + Test paddle.fluid.data feeds with different shape and types. + Note: paddle.fluid.data is not paddle.fluid.layers.data. ''' def setUp(self): @@ -38,16 +38,22 @@ class TestFeedData(unittest.TestCase): self.iterations = 5 def _get_device_count(self, use_cuda): - return core.get_cuda_device_count() if use_cuda else int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + return ( + core.get_cuda_device_count() + if use_cuda + else int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + ) def _get_feed_batch_size(self, use_cuda, use_parallel_executor): """ Returns actual fed data size. We should multiple the number of devices when it is using ParallelExecutor """ - return self.data_batch_size * self._get_device_count( - use_cuda) if use_parallel_executor else self.data_batch_size + return ( + self.data_batch_size * self._get_device_count(use_cuda) + if use_parallel_executor + else self.data_batch_size + ) def _simple_fc_net(self, in_size, label_size, class_num, hidden_sizes): in_data = fluid.data(name="data", dtype='float32', shape=in_size) @@ -59,26 +65,32 @@ class TestFeedData(unittest.TestCase): predict_label = fluid.layers.fc(hidden, size=class_num, act='softmax') loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) return in_data, label, loss def test(self): - for use_cuda in [True, False - ] if core.is_compiled_with_cuda() else [False]: + for use_cuda in ( + [True, False] if core.is_compiled_with_cuda() else [False] + ): for use_parallel_executor in [False, True]: print('Test Parameters:'), - print({ - 'use_cuda': use_cuda, - 'use_parallel_executor': use_parallel_executor, - }) + print( + { + 'use_cuda': use_cuda, + 'use_parallel_executor': use_parallel_executor, + } + ) # Test feeding without error - self._test_feed_data_match_shape_type(use_cuda, - use_parallel_executor) - self._test_feed_data_contains_neg_one(use_cuda, - use_parallel_executor) + self._test_feed_data_match_shape_type( + use_cuda, use_parallel_executor + ) + self._test_feed_data_contains_neg_one( + use_cuda, use_parallel_executor + ) self._test_feed_lod_tensor(use_cuda, use_parallel_executor) # Test exception message when feeding with error @@ -86,74 +98,110 @@ class TestFeedData(unittest.TestCase): error_shape_list = [self.data_batch_size, 3, 4, 5] with self.assertRaises(ValueError) as shape_mismatch_err: - self._test_feed_data_shape_mismatch(use_cuda, - use_parallel_executor) + self._test_feed_data_shape_mismatch( + use_cuda, use_parallel_executor + ) self.assertEqual( str(shape_mismatch_err.exception), "The fed Variable %r should have dimensions = %r, " - "shape = %r, but received fed shape %r on each device" % - (u'data', len(in_shape_tuple), in_shape_tuple, - error_shape_list)) + "shape = %r, but received fed shape %r on each device" + % ( + u'data', + len(in_shape_tuple), + in_shape_tuple, + error_shape_list, + ), + ) with self.assertRaises(ValueError) as dtype_mismatch_err: - self._test_feed_data_dtype_mismatch(use_cuda, - use_parallel_executor) + self._test_feed_data_dtype_mismatch( + use_cuda, use_parallel_executor + ) self.assertEqual( str(dtype_mismatch_err.exception), "The data type of fed Variable %r must be 'int64', but " - "received 'float64'" % (u'label')) + "received 'float64'" % (u'label'), + ) def _test_feed_data_dtype_mismatch(self, use_cuda, use_parallel_executor): - feed_batch_size = self._get_feed_batch_size(use_cuda, - use_parallel_executor) + feed_batch_size = self._get_feed_batch_size( + use_cuda, use_parallel_executor + ) in_size = [self.data_batch_size, 3, 4, 5] feed_in_data = np.random.uniform( - size=[feed_batch_size, 3, 4, 5]).astype(np.float32) + size=[feed_batch_size, 3, 4, 5] + ).astype(np.float32) label_size = [self.data_batch_size, 1] - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[feed_batch_size, - 1]).astype(np.float64) - self._feed_data_in_executor(in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[feed_batch_size, 1] + ).astype(np.float64) + self._feed_data_in_executor( + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ) def _test_feed_data_shape_mismatch(self, use_cuda, use_parallel_executor): batch_size = self._get_feed_batch_size(use_cuda, use_parallel_executor) in_size = [None, 3, 4, 8] feed_in_data = np.random.uniform(size=[batch_size, 3, 4, 5]).astype( - np.float32) + np.float32 + ) label_size = [-1, 1] - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[batch_size, 1]).astype(np.int64) - self._feed_data_in_executor(in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[batch_size, 1] + ).astype(np.int64) + self._feed_data_in_executor( + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ) def _test_feed_data_contains_neg_one(self, use_cuda, use_parallel_executor): batch_size = self._get_feed_batch_size(use_cuda, use_parallel_executor) in_size = [-1, 3, 4, 5] feed_in_data = np.random.uniform(size=[batch_size, 3, 4, 5]).astype( - np.float32) + np.float32 + ) label_size = (None, 1) - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[batch_size, 1]).astype(np.int64) - self._feed_data_in_executor(in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[batch_size, 1] + ).astype(np.int64) + self._feed_data_in_executor( + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ) def _test_feed_data_match_shape_type(self, use_cuda, use_parallel_executor): - feed_batch_size = self._get_feed_batch_size(use_cuda, - use_parallel_executor) + feed_batch_size = self._get_feed_batch_size( + use_cuda, use_parallel_executor + ) in_size = [self.data_batch_size, 3, 4, 5] feed_in_data = np.random.uniform( - size=[feed_batch_size, 3, 4, 5]).astype(np.float32) + size=[feed_batch_size, 3, 4, 5] + ).astype(np.float32) label_size = [self.data_batch_size, 1] - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[feed_batch_size, - 1]).astype(np.int64) - self._feed_data_in_executor(in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[feed_batch_size, 1] + ).astype(np.int64) + self._feed_data_in_executor( + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ) def _test_feed_lod_tensor(self, use_cuda, use_parallel_executor): device_count = self._get_device_count(use_cuda) @@ -164,33 +212,46 @@ class TestFeedData(unittest.TestCase): sum_length = int((device_count + 1) * device_count / 2) feed_in_data = np.random.uniform(size=[sum_length, 3, 4, 5]).astype( - np.float32) + np.float32 + ) feed_data_tensor = fluid.LoDTensor() feed_data_tensor.set(feed_in_data, fluid.CPUPlace()) feed_data_tensor.set_recursive_sequence_lengths(sequence_lengths) label_size = [device_count, 1] feed_label_tensor = fluid.LoDTensor() - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[sum_length, 1]).astype(np.int64) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[sum_length, 1] + ).astype(np.int64) feed_label_tensor.set(feed_label, fluid.CPUPlace()) feed_label_tensor.set_recursive_sequence_lengths(sequence_lengths) - self._feed_data_in_executor(in_size, label_size, feed_data_tensor, - feed_label_tensor, use_cuda, - use_parallel_executor) - - def _feed_data_in_executor(self, in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor): + self._feed_data_in_executor( + in_size, + label_size, + feed_data_tensor, + feed_label_tensor, + use_cuda, + use_parallel_executor, + ) + + def _feed_data_in_executor( + self, + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ): startup_program = fluid.Program() main_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - in_data, label, loss = self._simple_fc_net(in_size, label_size, - self.class_num, - self.hidden_sizes) + in_data, label, loss = self._simple_fc_net( + in_size, label_size, self.class_num, self.hidden_sizes + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -200,15 +261,15 @@ class TestFeedData(unittest.TestCase): train_program = main_program if use_parallel_executor: train_program = compiler.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name) + main_program + ).with_data_parallel(loss_name=loss.name) for i in range(self.iterations): - fetches = exe.run(train_program, - feed={ - in_data.name: feed_in_data, - label.name: feed_label - }, - fetch_list=[loss.name]) + fetches = exe.run( + train_program, + feed={in_data.name: feed_in_data, label.name: feed_label}, + fetch_list=[loss.name], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py index 1a6d780fb61..4054af8fb51 100644 --- a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py +++ b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py @@ -18,7 +18,6 @@ import numpy as np class TestFeedFetch(unittest.TestCase): - def test_feed_fetch(self): scope = core.Scope() place = core.CPUPlace() diff --git a/python/paddle/fluid/tests/unittests/test_fetch_handler.py b/python/paddle/fluid/tests/unittests/test_fetch_handler.py index 562c79cc6eb..6a8b1f689fd 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_handler.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_handler.py @@ -22,7 +22,6 @@ import paddle.fluid as fluid class TestFetchHandler(unittest.TestCase): - @unittest.skip(reason="Skip unstable ci") def test_fetch_handler(self): place = core.CPUPlace() @@ -36,7 +35,6 @@ class TestFetchHandler(unittest.TestCase): var_emb3 = block.create_var(name='emb3', type=core.VarDesc.VarType.FP32) class FH(fluid.executor.FetchHandler): - def handler(self, fetch_dict): assert len(fetch_dict) == 1 @@ -49,14 +47,13 @@ class TestFetchHandler(unittest.TestCase): time.sleep(3) fm.stop() - default_fh = fluid.executor.FetchHandler(var_dict={ - 'emb': var_emb, - 'emb2': None, - 'emb3': var_emb3 - }, - period_secs=1) + default_fh = fluid.executor.FetchHandler( + var_dict={'emb': var_emb, 'emb2': None, 'emb3': var_emb3}, + period_secs=1, + ) default_fm = fluid.trainer_factory.FetchHandlerMonitor( - scope, default_fh) + scope, default_fh + ) default_fm.start() time.sleep(5) default_fm.stop() diff --git a/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py index 2b59c88948d..c096a4c8a6e 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py @@ -21,7 +21,6 @@ from simple_nets import simple_fc_net_with_inputs, simple_fc_net class TestFetchLoDTensorArray(unittest.TestCase): - def build_program(self, main_program, startup_program): with fluid.unique_name.guard(): with fluid.program_guard(main_program, startup_program): @@ -59,27 +58,34 @@ class TestFetchLoDTensorArray(unittest.TestCase): build_strategy = fluid.BuildStrategy() binary = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) device_num = fluid.core.get_cuda_device_count() if use_cuda else 2 for _ in range(3): - loss_v, array_v = exe.run(binary, - feed=feed_dict, - fetch_list=[loss, array], - return_merged=False) + loss_v, array_v = exe.run( + binary, + feed=feed_dict, + fetch_list=[loss, array], + return_merged=False, + ) self.assertEqual(np.array(loss_v).shape, (device_num, 1)) self.assertEqual( - np.array(array_v[0][0]).shape, (batch_size / device_num, 784)) + np.array(array_v[0][0]).shape, (batch_size / device_num, 784) + ) self.assertEqual( - np.array(array_v[0][1]).shape, (batch_size / device_num, 1)) - self.assertEqual(np.array(array_v[0][2]).shape, (1, )) + np.array(array_v[0][1]).shape, (batch_size / device_num, 1) + ) + self.assertEqual(np.array(array_v[0][2]).shape, (1,)) for _ in range(3): - loss_v, array_v = exe.run(binary, - feed=feed_dict, - fetch_list=[loss, array], - return_merged=True) - self.assertEqual(np.array(loss_v).shape, (device_num, )) + loss_v, array_v = exe.run( + binary, + feed=feed_dict, + fetch_list=[loss, array], + return_merged=True, + ) + self.assertEqual(np.array(loss_v).shape, (device_num,)) self.assertEqual(np.array(array_v[0]).shape, (batch_size, 784)) self.assertEqual(np.array(array_v[1]).shape, (batch_size, 1)) np.testing.assert_allclose(loss_v, array_v[2], rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py b/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py index e9ce5294c37..0da628db92c 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py @@ -22,23 +22,26 @@ os.environ["CPU_NUM"] = "2" class TestFetchUnmerged(unittest.TestCase): - def conv_net(self, img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=8, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=8, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=16, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=16, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) hidden = fluid.layers.fc(input=conv_pool_2, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -48,12 +51,12 @@ class TestFetchUnmerged(unittest.TestCase): def build_program(self, main, startup, is_test): with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss, prediction = self.conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.001) @@ -63,8 +66,9 @@ class TestFetchUnmerged(unittest.TestCase): def fetch_unmerged(self, use_cuda=True): main_program = fluid.Program() startup_program = fluid.Program() - feeds, loss, prediction = self.build_program(main_program, - startup_program, False) + feeds, loss, prediction = self.build_program( + main_program, startup_program, False + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -72,34 +76,41 @@ class TestFetchUnmerged(unittest.TestCase): build_strategy = fluid.BuildStrategy() binary = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) iters = 2 batch_size = 16 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) device_num = fluid.core.get_cuda_device_count() if use_cuda else 2 for _ in range(iters): data = next(train_reader()) - loss_v, prediction_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss, prediction], - return_merged=False) + loss_v, prediction_v = exe.run( + binary, + feed=feeder.feed(data), + fetch_list=[loss, prediction], + return_merged=False, + ) self.assertEqual(np.array(loss_v).shape, (device_num, 1)) self.assertEqual( np.array(prediction_v).shape, - (device_num, batch_size / device_num, 10)) + (device_num, batch_size / device_num, 10), + ) for _ in range(iters): data = next(train_reader()) - loss_v, prediction_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss, prediction], - return_merged=True) - self.assertEqual(np.array(loss_v).shape, (device_num, )) + loss_v, prediction_v = exe.run( + binary, + feed=feeder.feed(data), + fetch_list=[loss, prediction], + return_merged=True, + ) + self.assertEqual(np.array(loss_v).shape, (device_num,)) self.assertEqual(np.array(prediction_v).shape, (batch_size, 10)) def test_fetch_unmerged(self): diff --git a/python/paddle/fluid/tests/unittests/test_fetch_var.py b/python/paddle/fluid/tests/unittests/test_fetch_var.py index 1ce597ef98a..98b06b31368 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_var.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_var.py @@ -19,7 +19,6 @@ import unittest class TestFetchVar(unittest.TestCase): - def set_input(self): self.val = np.array([1, 3, 5]).astype(np.int32) @@ -35,7 +34,6 @@ class TestFetchVar(unittest.TestCase): class TestFetchNullVar(TestFetchVar): - def set_input(self): self.val = np.array([]).astype(np.int32) diff --git a/python/paddle/fluid/tests/unittests/test_fill_any_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_any_like_op.py index fae7b3c2b0a..1739361080d 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_any_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_any_like_op.py @@ -20,7 +20,6 @@ from op_test import OpTest, convert_float_to_uint16 class TestFillAnyLikeOp(OpTest): - def setUp(self): self.op_type = "fill_any_like" self.dtype = np.int32 @@ -38,16 +37,15 @@ class TestFillAnyLikeOp(OpTest): class TestFillAnyLikeOpFloat32(TestFillAnyLikeOp): - def init(self): self.dtype = np.float32 self.value = 0.0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFillAnyLikeOpBfloat16(OpTest): - def setUp(self): self.op_type = "fill_any_like" self.dtype = np.uint16 @@ -55,8 +53,9 @@ class TestFillAnyLikeOpBfloat16(OpTest): self.inputs = {'X': np.random.random((219, 232)).astype(np.float32)} self.attrs = {'value': self.value, 'dtype': core.VarDesc.VarType.BF16} self.outputs = { - 'Out': - convert_float_to_uint16(self.value * np.ones_like(self.inputs["X"])) + 'Out': convert_float_to_uint16( + self.value * np.ones_like(self.inputs["X"]) + ) } def test_check_output(self): @@ -65,25 +64,21 @@ class TestFillAnyLikeOpBfloat16(OpTest): class TestFillAnyLikeOpValue1(TestFillAnyLikeOp): - def init(self): self.value = 1.0 class TestFillAnyLikeOpValue2(TestFillAnyLikeOp): - def init(self): self.value = 1e-10 class TestFillAnyLikeOpValue3(TestFillAnyLikeOp): - def init(self): self.value = 1e-100 class TestFillAnyLikeOpType(TestFillAnyLikeOp): - def setUp(self): self.op_type = "fill_any_like" self.dtype = np.int32 @@ -92,16 +87,15 @@ class TestFillAnyLikeOpType(TestFillAnyLikeOp): self.inputs = {'X': np.random.random((219, 232)).astype(self.dtype)} self.attrs = { 'value': self.value, - 'dtype': int(core.VarDesc.VarType.FP32) + 'dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.value * np.ones_like(self.inputs["X"]).astype(np.float32) + 'Out': self.value + * np.ones_like(self.inputs["X"]).astype(np.float32) } class TestFillAnyLikeOpFloat16(TestFillAnyLikeOp): - def init(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/test_fill_any_op.py b/python/paddle/fluid/tests/unittests/test_fill_any_op.py index 7dbdfe8d58a..76daed5a5d8 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_any_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_any_op.py @@ -19,7 +19,6 @@ from op_test import OpTest class TestFillAnyOp(OpTest): - def setUp(self): self.op_type = "fill_any" self.dtype = 'float64' @@ -28,11 +27,11 @@ class TestFillAnyOp(OpTest): self.inputs = {'X': np.random.random((20, 30)).astype(self.dtype)} self.attrs = { 'value_float': float(self.value), - 'value_int': int(self.value) + 'value_int': int(self.value), } self.outputs = { - 'Out': - self.value * np.ones_like(self.inputs["X"]).astype(self.dtype) + 'Out': self.value + * np.ones_like(self.inputs["X"]).astype(self.dtype) } def init(self): @@ -46,34 +45,29 @@ class TestFillAnyOp(OpTest): class TestFillAnyOpFloat32(TestFillAnyOp): - def init(self): self.dtype = np.float32 self.value = 0.0 class TestFillAnyOpFloat16(TestFillAnyOp): - def init(self): self.dtype = np.float16 class TestFillAnyOpvalue1(TestFillAnyOp): - def init(self): self.dtype = np.float32 self.value = 111111555 class TestFillAnyOpvalue2(TestFillAnyOp): - def init(self): self.dtype = np.float32 self.value = 11111.1111 class TestFillAnyInplace(unittest.TestCase): - def test_fill_any_version(self): with paddle.fluid.dygraph.guard(): var = paddle.to_tensor(np.ones((4, 2, 3)).astype(np.float32)) @@ -91,7 +85,8 @@ class TestFillAnyInplace(unittest.TestCase): def test_fill_any_eqaul(self): with paddle.fluid.dygraph.guard(): tensor = paddle.to_tensor( - np.random.random((20, 30)).astype(np.float32)) + np.random.random((20, 30)).astype(np.float32) + ) target = tensor.numpy() target[...] = 1 @@ -100,7 +95,7 @@ class TestFillAnyInplace(unittest.TestCase): def test_backward(self): with paddle.fluid.dygraph.guard(): - x = paddle.full([10, 10], -1., dtype='float32') + x = paddle.full([10, 10], -1.0, dtype='float32') x.stop_gradient = False y = 2 * x y.fill_(1) diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py index ebe0ed179d2..339fafbd0b4 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py @@ -21,16 +21,18 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ paddle.enable_static() -def fill_constant_batch_size_like(input, - shape, - value, - data_type, - input_dim_idx=0, - output_dim_idx=0, - force_cpu=False): +def fill_constant_batch_size_like( + input, + shape, + value, + data_type, + input_dim_idx=0, + output_dim_idx=0, + force_cpu=False, +): return paddle.fluid.layers.fill_constant_batch_size_like( - input, shape, data_type, value, input_dim_idx, output_dim_idx, - force_cpu) + input, shape, data_type, value, input_dim_idx, output_dim_idx, force_cpu + ) class TestFillConstatnBatchSizeLike1(OpTest): @@ -51,7 +53,7 @@ class TestFillConstatnBatchSizeLike1(OpTest): 'value': self.value, 'input_dim_idx': self.input_dim_idx, 'output_dim_idx': self.output_dim_idx, - 'force_cpu': self.force_cpu + 'force_cpu': self.force_cpu, } def init_data(self): diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_op.py b/python/paddle/fluid/tests/unittests/test_fill_constant_op.py index a031385aa0a..838efff490a 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_op.py @@ -26,10 +26,8 @@ from paddle.fluid import Program, program_guard # Situation 1: Attr(shape) is a list(without tensor) class TestFillConstantOp1(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.inputs = {} @@ -41,10 +39,8 @@ class TestFillConstantOp1(OpTest): class TestFillConstantOp2(OpTest): - def setUp(self): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' self.op_type = "fill_constant" self.inputs = {} @@ -56,10 +52,8 @@ class TestFillConstantOp2(OpTest): class TestFillConstantOp3(OpTest): - def setUp(self): - '''Test fill_constant op with specified int64 value - ''' + '''Test fill_constant op with specified int64 value''' self.op_type = "fill_constant" self.inputs = {} @@ -71,10 +65,8 @@ class TestFillConstantOp3(OpTest): class TestFillConstantOp4(OpTest): - def setUp(self): - '''Test fill_constant op with specified int value - ''' + '''Test fill_constant op with specified int value''' self.op_type = "fill_constant" self.inputs = {} @@ -85,20 +77,19 @@ class TestFillConstantOp4(OpTest): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFillConstantBF16Op(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.dtype = np.uint16 self.inputs = {} self.attrs = { 'shape': [123, 92], 'value': 3.8, - 'dtype': core.VarDesc.VarType.BF16 + 'dtype': core.VarDesc.VarType.BF16, } self.outputs = {'Out': convert_float_to_uint16(np.full((123, 92), 3.8))} @@ -108,17 +99,15 @@ class TestFillConstantBF16Op(OpTest): class TestFillConstantOpWithSelectedRows(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() # create Out Variable out = scope.var('Out').get_selected_rows() # create and run fill_constant_op operator - fill_constant_op = Operator("fill_constant", - shape=[123, 92], - value=3.8, - Out='Out') + fill_constant_op = Operator( + "fill_constant", shape=[123, 92], value=3.8, Out='Out' + ) fill_constant_op.run(scope, place) # get result from Out @@ -138,16 +127,15 @@ class TestFillConstantOpWithSelectedRows(unittest.TestCase): # Situation 2: Attr(shape) is a list(with tensor) class TestFillConstantOp1_ShapeTensorList(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape, 'value': self.value} @@ -163,16 +151,15 @@ class TestFillConstantOp1_ShapeTensorList(OpTest): class TestFillConstantOp2_ShapeTensorList(OpTest): - def setUp(self): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' self.op_type = "fill_constant" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape} @@ -187,7 +174,6 @@ class TestFillConstantOp2_ShapeTensorList(OpTest): class TestFillConstantOp3_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): - def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -195,7 +181,6 @@ class TestFillConstantOp3_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): class TestFillConstantOp4_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): - def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -204,10 +189,8 @@ class TestFillConstantOp4_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): # Situation 3: shape is a tensor class TestFillConstantOp1_ShapeTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() @@ -225,16 +208,14 @@ class TestFillConstantOp1_ShapeTensor(OpTest): # Situation 4: value is a tensor class TestFillConstantOp1_ValueTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype("float32") + 'ValueTensor': np.array([self.value]).astype("float32"), } self.attrs = {'value': self.value + 1.0} self.outputs = {'Out': np.full(self.shape, self.value)} @@ -250,16 +231,14 @@ class TestFillConstantOp1_ValueTensor(OpTest): # Situation 5: value is a tensor class TestFillConstantOp2_ValueTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype("int32") + 'ValueTensor': np.array([self.value]).astype("int32"), } self.attrs = {'value': self.value, 'dtype': 2} self.outputs = {'Out': np.full(self.shape, self.value)} @@ -275,56 +254,55 @@ class TestFillConstantOp2_ValueTensor(OpTest): # Test python API class TestFillConstantAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 2) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - - out_1 = fluid.layers.fill_constant(shape=[1, 2], - dtype="float32", - value=1.1) - - out_2 = fluid.layers.fill_constant(shape=[1, positive_2_int32], - dtype="float32", - value=1.1) - - out_3 = fluid.layers.fill_constant(shape=[1, positive_2_int64], - dtype="float32", - value=1.1) - - out_4 = fluid.layers.fill_constant(shape=shape_tensor_int32, - dtype="float32", - value=1.1) - - out_5 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype="float32", - value=1.1) - - out_6 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=1.1) - - val1 = fluid.layers.fill_constant(shape=[1], - dtype=np.float32, - value=1.1) - val2 = fluid.layers.fill_constant(shape=[1], - dtype=np.float64, - value=1.1) - out_7 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=val1) - - out_8 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=val2) + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + + out_1 = fluid.layers.fill_constant( + shape=[1, 2], dtype="float32", value=1.1 + ) + + out_2 = fluid.layers.fill_constant( + shape=[1, positive_2_int32], dtype="float32", value=1.1 + ) + + out_3 = fluid.layers.fill_constant( + shape=[1, positive_2_int64], dtype="float32", value=1.1 + ) + + out_4 = fluid.layers.fill_constant( + shape=shape_tensor_int32, dtype="float32", value=1.1 + ) + + out_5 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype="float32", value=1.1 + ) + + out_6 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=1.1 + ) + + val1 = fluid.layers.fill_constant( + shape=[1], dtype=np.float32, value=1.1 + ) + val2 = fluid.layers.fill_constant( + shape=[1], dtype=np.float64, value=1.1 + ) + out_7 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=val1 + ) + + out_8 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=val2 + ) exe = fluid.Executor(place=fluid.CPUPlace()) res_1, res_2, res_3, res_4, res_5, res_6, res_7, res_8 = exe.run( @@ -333,7 +311,8 @@ class TestFillConstantAPI(unittest.TestCase): "shape_tensor_int32": np.array([1, 2]).astype("int32"), "shape_tensor_int64": np.array([1, 2]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.array_equal(res_1, np.full([1, 2], 1.1, dtype="float32")) assert np.array_equal(res_2, np.full([1, 2], 1.1, dtype="float32")) @@ -346,7 +325,6 @@ class TestFillConstantAPI(unittest.TestCase): class TestFillConstantImperative(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): data1 = np.array([1, 2]).astype('int32') @@ -355,26 +333,30 @@ class TestFillConstantImperative(unittest.TestCase): shape = fluid.dygraph.to_variable(data1) val = fluid.dygraph.to_variable(data2) value = fluid.dygraph.to_variable(data3) - res1 = fluid.layers.fill_constant(shape=[1, 2], - dtype='float32', - value=1.1) - res2 = fluid.layers.fill_constant(shape=shape, - dtype='float32', - value=1.1) - res3 = fluid.layers.fill_constant(shape=shape, - dtype='float32', - value=val) - res4 = fluid.layers.fill_constant(shape=shape, - dtype='int32', - value=value) - assert np.array_equal(res1.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res2.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res3.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res4.numpy(), - np.full([1, 2], 88, dtype="int32")) + res1 = fluid.layers.fill_constant( + shape=[1, 2], dtype='float32', value=1.1 + ) + res2 = fluid.layers.fill_constant( + shape=shape, dtype='float32', value=1.1 + ) + res3 = fluid.layers.fill_constant( + shape=shape, dtype='float32', value=val + ) + res4 = fluid.layers.fill_constant( + shape=shape, dtype='int32', value=value + ) + assert np.array_equal( + res1.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res2.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res3.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res4.numpy(), np.full([1, 2], 88, dtype="int32") + ) def test_nan(self): with fluid.dygraph.guard(): @@ -394,42 +376,49 @@ class TestFillConstantImperative(unittest.TestCase): class TestFillConstantOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage + # for ci coverage x1 = fluid.layers.data(name='x1', shape=[1], dtype="int16") - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1], - value=5, - dtype='uint4') - - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1.1], - value=5, - dtype='float32', - out=x1) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1], + value=5, + dtype='uint4', + ) + + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1.1], + value=5, + dtype='float32', + out=x1, + ) # The argument dtype of fill_constant_op must be one of bool, float16, - #float32, float64, uint8, int16, int32 or int64 + # float32, float64, uint8, int16, int32 or int64 x2 = fluid.layers.data(name='x2', shape=[1], dtype="int32") - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1], - value=5, - dtype='float64', - out=x2) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1], + value=5, + dtype='float64', + out=x2, + ) x3 = np.random.randn(100, 100).astype('int32') - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[100, 100], - value=5, - dtype='float64', - out=x3) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[100, 100], + value=5, + dtype='float64', + out=x3, + ) # The argument shape's type of fill_constant_op must be list, tuple or Variable. def test_shape_type(): @@ -439,39 +428,37 @@ class TestFillConstantOpError(unittest.TestCase): # The shape dtype of fill_constant_op must be int32 or int64. def test_shape_tensor_dtype(): - shape = fluid.data(name="shape_tensor", - shape=[2], - dtype="float32") - fluid.layers.fill_constant(shape=shape, - dtype="float32", - value=1) + shape = fluid.data( + name="shape_tensor", shape=[2], dtype="float32" + ) + fluid.layers.fill_constant( + shape=shape, dtype="float32", value=1 + ) self.assertRaises(TypeError, test_shape_tensor_dtype) def test_shape_tensor_list_dtype(): - shape = fluid.data(name="shape_tensor_list", - shape=[1], - dtype="bool") - fluid.layers.fill_constant(shape=[shape, 2], - dtype="float32", - value=1) + shape = fluid.data( + name="shape_tensor_list", shape=[1], dtype="bool" + ) + fluid.layers.fill_constant( + shape=[shape, 2], dtype="float32", value=1 + ) self.assertRaises(TypeError, test_shape_tensor_list_dtype) class TestFillConstantOp_ValueTensorBf16(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { - "ShapeTensor": - np.array(self.shape).astype("int32"), - 'ValueTensor': - convert_float_to_uint16(np.array([self.value]).astype("float32")) + "ShapeTensor": np.array(self.shape).astype("int32"), + 'ValueTensor': convert_float_to_uint16( + np.array([self.value]).astype("float32") + ), } self.attrs = {'value': self.value, 'dtype': core.VarDesc.VarType.BF16} self.outputs = {'Out': np.full(self.shape, self.value)} diff --git a/python/paddle/fluid/tests/unittests/test_fill_diagonal_tensor_op.py b/python/paddle/fluid/tests/unittests/test_fill_diagonal_tensor_op.py index 9823f5e2585..f412a161eb6 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_diagonal_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_diagonal_tensor_op.py @@ -34,13 +34,15 @@ def fill_diagonal_ndarray(x, value, offset=0, dim1=0, dim2=1): diagonal = np.lib.stride_tricks.as_strided( x[:, offset:] if dim_sum == 1 else x[:, :, offset:], shape=(shape[dim3], diagdim), - strides=(strides[dim3], strides[dim1] + strides[dim2])) + strides=(strides[dim3], strides[dim1] + strides[dim2]), + ) else: diagdim = min(shape[dim2], shape[dim1] + offset) diagonal = np.lib.stride_tricks.as_strided( x[-offset:, :] if dim_sum in [1, 2] else x[:, -offset:], shape=(shape[dim3], diagdim), - strides=(strides[dim3], strides[dim1] + strides[dim2])) + strides=(strides[dim3], strides[dim1] + strides[dim2]), + ) diagonal[...] = value return x @@ -79,13 +81,12 @@ def fill_gt(x, y, offset, dim1, dim2): class TensorFillDiagTensor_Test(OpTest): - def setUp(self): self.op_type = "fill_diagonal_tensor" self.python_api = paddle.tensor.manipulation.fill_diagonal_tensor self.init_kernel_type() x = np.random.random((10, 10)).astype(self.dtype) - y = np.random.random((10, )).astype(self.dtype) + y = np.random.random((10,)).astype(self.dtype) dim1 = 0 dim2 = 1 offset = 0 @@ -106,7 +107,6 @@ class TensorFillDiagTensor_Test(OpTest): class TensorFillDiagTensor_Test2(TensorFillDiagTensor_Test): - def setUp(self): self.op_type = "fill_diagonal_tensor" self.python_api = paddle.tensor.manipulation.fill_diagonal_tensor @@ -127,7 +127,6 @@ class TensorFillDiagTensor_Test2(TensorFillDiagTensor_Test): class TensorFillDiagTensor_Test3(TensorFillDiagTensor_Test): - def setUp(self): self.op_type = "fill_diagonal_tensor" self.python_api = paddle.tensor.manipulation.fill_diagonal_tensor diff --git a/python/paddle/fluid/tests/unittests/test_fill_op.py b/python/paddle/fluid/tests/unittests/test_fill_op.py index 0cfa7ecbed0..95a432136ca 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_op.py @@ -20,7 +20,6 @@ from paddle.fluid.op import Operator class TestFillOp1(OpTest): - def setUp(self): self.op_type = "fill" val = np.random.random(size=[100, 200]) @@ -29,7 +28,7 @@ class TestFillOp1(OpTest): 'value': val.flatten().tolist(), 'shape': [100, 200], 'dtype': int(core.VarDesc.VarType.FP64), - 'force_cpu': False + 'force_cpu': False, } self.outputs = {'Out': val.astype('float64')} @@ -38,7 +37,6 @@ class TestFillOp1(OpTest): class TestFillOp2(OpTest): - def setUp(self): self.op_type = "fill" val = np.random.random(size=[100, 200]) @@ -47,7 +45,7 @@ class TestFillOp2(OpTest): 'value': val.flatten().tolist(), 'shape': [100, 200], 'dtype': int(core.VarDesc.VarType.FP64), - 'force_cpu': True + 'force_cpu': True, } self.outputs = {'Out': val.astype('float64')} @@ -56,7 +54,6 @@ class TestFillOp2(OpTest): class TestFillOp3(unittest.TestCase): - def check_with_place(self, place, f_cpu): scope = core.Scope() # create Out Variable @@ -64,12 +61,14 @@ class TestFillOp3(unittest.TestCase): # create and run fill_op operator val = np.random.random(size=[300, 200]) - fill_op = Operator("fill", - value=val.flatten(), - shape=[300, 200], - dtype=int(core.VarDesc.VarType.FP32), - force_cpu=f_cpu, - Out='Out') + fill_op = Operator( + "fill", + value=val.flatten(), + shape=[300, 200], + dtype=int(core.VarDesc.VarType.FP32), + force_cpu=f_cpu, + Out='Out', + ) fill_op.run(scope, place) # get result from Out diff --git a/python/paddle/fluid/tests/unittests/test_fill_zeros_like2_op.py b/python/paddle/fluid/tests/unittests/test_fill_zeros_like2_op.py index a298bc5c58b..f1a1039ea01 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_zeros_like2_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_zeros_like2_op.py @@ -20,7 +20,6 @@ from op_test import OpTest class TestFillZerosLike2Op(OpTest): - def setUp(self): self.op_type = "fill_zeros_like2" self.dtype = np.float32 @@ -37,21 +36,17 @@ class TestFillZerosLike2Op(OpTest): class TestFillZerosLike2OpFp16(TestFillZerosLike2Op): - def init_dtype(self): self.dtype = np.float16 class TestFillZerosLike2OpFp64(TestFillZerosLike2Op): - def init_dtype(self): self.dtype = np.float64 class TestZerosError(unittest.TestCase): - def test_errors(self): - def test_zeros_like_type_error(): with fluid.program_guard(fluid.Program(), fluid.Program()): fluid.layers.zeros_like([10], dtype="float") diff --git a/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py index e6515e426e4..7317973e774 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py @@ -18,7 +18,6 @@ from op_test import OpTest class TestFillZerosLikeOp(OpTest): - def setUp(self): self.op_type = "fill_zeros_like" self.dtype = np.float32 @@ -34,7 +33,6 @@ class TestFillZerosLikeOp(OpTest): class TestFillZerosLikeOpFp16(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/test_filter_by_instag_op.py b/python/paddle/fluid/tests/unittests/test_filter_by_instag_op.py index b60dcbe9c89..e1136d677a1 100644 --- a/python/paddle/fluid/tests/unittests/test_filter_by_instag_op.py +++ b/python/paddle/fluid/tests/unittests/test_filter_by_instag_op.py @@ -16,11 +16,11 @@ import unittest import numpy as np from op_test import OpTest + """This is Test Case 1""" class TestFilterByInstagOp(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' x1 = np.zeros((36, 4), dtype=np.float64) @@ -48,8 +48,9 @@ class TestFilterByInstagOp(OpTest): out[ln, k] = cur ln += 1 - mmap = np.array([[0, 1, 2], [2, 6, 4], [6, 15, 6], [12, 28, - 8]]).astype('int64') + mmap = np.array([[0, 1, 2], [2, 6, 4], [6, 15, 6], [12, 28, 8]]).astype( + 'int64' + ) mmap_lod = [[1, 1, 1, 1]] loss_weight = np.array([[1], [1], [1], [1]]).astype('double') @@ -62,7 +63,7 @@ class TestFilterByInstagOp(OpTest): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': True, 'out_val_if_empty': 0} @@ -71,16 +72,15 @@ class TestFilterByInstagOp(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['Ins'], - 'Out', - no_grad_set=set(['Ins_tag', 'Filter_tag'])) + self.check_grad( + ['Ins'], 'Out', no_grad_set=set(['Ins_tag', 'Filter_tag']) + ) """This is Test Case 2""" class TestFilterByInstagOp2(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -110,7 +110,7 @@ class TestFilterByInstagOp2(OpTest): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': True, 'out_val_if_empty': 0} @@ -118,16 +118,15 @@ class TestFilterByInstagOp2(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['Ins'], - 'Out', - no_grad_set=set(['Ins_tag', 'Filter_tag'])) + self.check_grad( + ['Ins'], 'Out', no_grad_set=set(['Ins_tag', 'Filter_tag']) + ) """This is Test Case 3""" class TestFilterByInstagOp3(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -154,7 +153,7 @@ class TestFilterByInstagOp3(OpTest): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': True, 'out_val_if_empty': 0} @@ -162,16 +161,15 @@ class TestFilterByInstagOp3(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['Ins'], - 'Out', - no_grad_set=set(['Ins_tag', 'Filter_tag'])) + self.check_grad( + ['Ins'], 'Out', no_grad_set=set(['Ins_tag', 'Filter_tag']) + ) """This is Test Case 4""" class TestFilterByInstagOp4(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -197,7 +195,7 @@ class TestFilterByInstagOp4(OpTest): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': False, 'out_val_if_empty': 0} @@ -205,13 +203,12 @@ class TestFilterByInstagOp4(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['Ins'], - 'Out', - no_grad_set=set(['Ins_tag', 'Filter_tag'])) + self.check_grad( + ['Ins'], 'Out', no_grad_set=set(['Ins_tag', 'Filter_tag']) + ) class TestFilterByInstagOp6(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -237,7 +234,7 @@ class TestFilterByInstagOp6(OpTest): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': False, 'out_val_if_empty': 0} @@ -249,7 +246,6 @@ class TestFilterByInstagOp6(OpTest): class TestFilterByInstagOp7(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -275,7 +271,7 @@ class TestFilterByInstagOp7(OpTest): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': False, 'out_val_if_empty': 0} diff --git a/python/paddle/fluid/tests/unittests/test_flatten2_op.py b/python/paddle/fluid/tests/unittests/test_flatten2_op.py index cbfceb720ad..fe5aad118af 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten2_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten2_op.py @@ -20,7 +20,6 @@ from op_test import OpTest class TestFlattenOp(OpTest): - def setUp(self): self.op_type = "flatten2" self.init_test_case() @@ -28,7 +27,7 @@ class TestFlattenOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def test_check_output(self): @@ -47,7 +46,6 @@ class TestFlattenOp(OpTest): class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.axis = 0 @@ -55,7 +53,6 @@ class TestFlattenOp1(TestFlattenOp): class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -65,7 +62,6 @@ class TestFlattenOpWithDefaultAxis(TestFlattenOp): class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 @@ -73,7 +69,6 @@ class TestFlattenOpSixDims(TestFlattenOp): class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): - def execute_api(self, x, axis=1): return fluid.layers.flatten(x, axis=axis) @@ -81,15 +76,14 @@ class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): paddle.enable_static() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[-1, 3, -1, -1], - dtype='float32') + x = paddle.static.data( + name="x", shape=[-1, 3, -1, -1], dtype='float32' + ) out = self.execute_api(x, axis=2) self.assertTrue((-1, -1) == out.shape) class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_data = np.random.random((3, 2, 4, 5)).astype("float64") @@ -102,9 +96,9 @@ class TestFlatten2OpError(unittest.TestCase): def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = fluid.layers.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2 = fluid.layers.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) fluid.layers.flatten(x2, axis=1) self.assertRaises(TypeError, test_type) diff --git a/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py b/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py index b938adfbf14..8d1bcc6254d 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py @@ -19,7 +19,6 @@ from op_test import OpTest class TestFlattenOp(OpTest): - def setUp(self): self.python_api = paddle.flatten self.python_out_sig = ["Out"] @@ -31,7 +30,7 @@ class TestFlattenOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def test_check_output(self): @@ -44,17 +43,16 @@ class TestFlattenOp(OpTest): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 self.stop_axis = -1 - self.new_shape = (120) + self.new_shape = 120 def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 1 @@ -64,12 +62,11 @@ class TestFlattenOp_1(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_2(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -79,12 +76,11 @@ class TestFlattenOp_2(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_3(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -94,12 +90,11 @@ class TestFlattenOp_3(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_4(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = -2 @@ -109,12 +104,11 @@ class TestFlattenOp_4(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_5(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 2 @@ -124,12 +118,11 @@ class TestFlattenOp_5(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.start_axis = 3 @@ -139,50 +132,63 @@ class TestFlattenOpSixDims(TestFlattenOp): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_ValueError1(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) out = paddle.flatten(x_var, start_axis=2, stop_axis=1) self.assertRaises(ValueError, test_ValueError1) def test_ValueError2(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=10, stop_axis=1) self.assertRaises(ValueError, test_ValueError2) def test_ValueError3(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=2, stop_axis=10) self.assertRaises(ValueError, test_ValueError3) def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x2 = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x2 = x2.astype('float16') - x2_var = paddle.fluid.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2_var = paddle.fluid.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) paddle.flatten(x2_var) self.assertRaises(TypeError, test_type) @@ -194,7 +200,6 @@ class TestFlatten2OpError(unittest.TestCase): class TestStaticFlattenPythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -204,9 +209,9 @@ class TestStaticFlattenPythonAPI(unittest.TestCase): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[2, 3, 4, 4], - dtype='float32') + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32' + ) out = self.execute_api(x, start_axis=-2, stop_axis=-1) exe = paddle.static.Executor(place=paddle.CPUPlace()) @@ -215,7 +220,6 @@ class TestStaticFlattenPythonAPI(unittest.TestCase): class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -223,25 +227,30 @@ class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): paddle.enable_static() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[-1, 3, -1, -1], - dtype='float32') + x = paddle.static.data( + name="x", shape=[-1, 3, -1, -1], dtype='float32' + ) out = self.execute_api(x, start_axis=2, stop_axis=3) self.assertTrue((-1, 3, -1) == out.shape) class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): - def execute_api(self, x, start_axis=0, stop_axis=-1): return x.flatten_(start_axis, stop_axis) class TestFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_InputError(): @@ -260,11 +269,17 @@ class TestFlattenPython(unittest.TestCase): class TestDygraphInplaceFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_Negative(): diff --git a/python/paddle/fluid/tests/unittests/test_flatten_op.py b/python/paddle/fluid/tests/unittests/test_flatten_op.py index 9a371465f33..a1108d3392c 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten_op.py @@ -19,7 +19,6 @@ from op_test import OpTest class TestFlattenOp(OpTest): - def setUp(self): self.op_type = "flatten" self.init_test_case() @@ -43,7 +42,6 @@ class TestFlattenOp(OpTest): class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 2, 10) self.axis = 0 @@ -51,7 +49,6 @@ class TestFlattenOp1(TestFlattenOp): class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -61,7 +58,6 @@ class TestFlattenOpWithDefaultAxis(TestFlattenOp): class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/test_fleet.py b/python/paddle/fluid/tests/unittests/test_fleet.py index a06f2647509..a9a75868ee3 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet.py +++ b/python/paddle/fluid/tests/unittests/test_fleet.py @@ -27,7 +27,8 @@ class TestFleet1(unittest.TestCase): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" @@ -42,24 +43,38 @@ class TestFleet1(unittest.TestCase): os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" role_maker = GeneralRoleMaker() - #role_maker.generate_role() + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) - emb = fluid.layers.embedding(input=show, size=[1, 1], \ - is_sparse=True, is_distributed=True, \ - param_attr=fluid.ParamAttr(name="embedding")) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + emb = fluid.layers.embedding( + input=show, + size=[1, 1], + is_sparse=True, + is_distributed=True, + param_attr=fluid.ParamAttr(name="embedding"), + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow = fluid.layers.data_norm(input=bow, epsilon=1e-4, name="norm") fc = fluid.layers.fc(input=bow, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: @@ -70,7 +85,8 @@ class TestFleet1(unittest.TestCase): "embedding": { "sparse_accessor_class": "DownpourSparseValueAccessor" } - }) + }, + ) adam.minimize([cost], [scope]) fleet.run_server() except: diff --git a/python/paddle/fluid/tests/unittests/test_fleet_api_input.py b/python/paddle/fluid/tests/unittests/test_fleet_api_input.py index 56b06e3b8f2..ac77fc42efd 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_api_input.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_api_input.py @@ -15,19 +15,24 @@ import unittest import paddle import paddle.fluid as fluid -from paddle.fluid.transpiler.distribute_transpiler import DistributeTranspilerConfig +from paddle.fluid.transpiler.distribute_transpiler import ( + DistributeTranspilerConfig, +) from paddle.fluid.incubate.fleet.base.role_maker import UserDefinedRoleMaker -from paddle.fluid.incubate.fleet.base.role_maker import UserDefinedCollectiveRoleMaker +from paddle.fluid.incubate.fleet.base.role_maker import ( + UserDefinedCollectiveRoleMaker, +) from paddle.fluid.incubate.fleet.base.role_maker import Role import paddle.fluid.incubate.fleet.base.role_maker as role_maker -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) from paddle.fluid.incubate.fleet.parameter_server import TranspilerOptimizer from paddle.fluid.incubate.fleet.collective import CollectiveOptimizer from dist_simnet_bow import train_network class DistributeTranspilerConfigTest(unittest.TestCase): - def set_runtime_split_send_recv(self, config, value): config.runtime_split_send_recv = value @@ -37,10 +42,12 @@ class DistributeTranspilerConfigTest(unittest.TestCase): def testConfig(self): config = DistributeTranspilerConfig() self.assertRaises(Exception, self.set_sync_mode, config, None) - self.assertRaises(Exception, self.set_runtime_split_send_recv, config, - None) - self.assertRaises(Exception, self.set_runtime_split_send_recv, config, - True) + self.assertRaises( + Exception, self.set_runtime_split_send_recv, config, None + ) + self.assertRaises( + Exception, self.set_runtime_split_send_recv, config, True + ) self.set_sync_mode(config, False) self.assertFalse(config.sync_mode) self.set_runtime_split_send_recv(config, True) @@ -48,7 +55,6 @@ class DistributeTranspilerConfigTest(unittest.TestCase): class FleetTest(unittest.TestCase): - def testInvalidInputs(self): self.assertRaises(Exception, fleet.split_files, "files") self.assertRaises(Exception, fleet.init, "pserver") @@ -61,40 +67,50 @@ class FleetTest(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) pe = fluid.ParallelExecutor(use_cuda=False, loss_name=loss.name) - self.assertRaises(Exception, - fleet.save_inference_model, - dirname='/tmp/', - feeded_var_names=['X'], - target_vars=[loss], - executor=pe) - self.assertRaises(Exception, - fleet.save_inference_model, - dirname='/tmp/', - feeded_var_names=['X'], - target_vars=[loss], - executor="executor") + self.assertRaises( + Exception, + fleet.save_inference_model, + dirname='/tmp/', + feeded_var_names=['X'], + target_vars=[loss], + executor=pe, + ) + self.assertRaises( + Exception, + fleet.save_inference_model, + dirname='/tmp/', + feeded_var_names=['X'], + target_vars=[loss], + executor="executor", + ) compiled_prog = fluid.compiler.CompiledProgram( - fluid.default_main_program()) - self.assertRaises(Exception, - fleet.save_inference_model, - dirname='/tmp/', - feeded_var_names=['X'], - target_vars=[loss], - executor=exe, - main_program=compiled_prog) - self.assertRaises(Exception, - fleet.save_persistables, - executor=pe, - dirname='/tmp/') - self.assertRaises(Exception, - fleet.save_persistables, - executor="executor", - dirname='/tmp/') - self.assertRaises(Exception, - fleet.save_persistables, - executor=exe, - dirname='/tmp/', - main_program=compiled_prog) + fluid.default_main_program() + ) + self.assertRaises( + Exception, + fleet.save_inference_model, + dirname='/tmp/', + feeded_var_names=['X'], + target_vars=[loss], + executor=exe, + main_program=compiled_prog, + ) + self.assertRaises( + Exception, fleet.save_persistables, executor=pe, dirname='/tmp/' + ) + self.assertRaises( + Exception, + fleet.save_persistables, + executor="executor", + dirname='/tmp/', + ) + self.assertRaises( + Exception, + fleet.save_persistables, + executor=exe, + dirname='/tmp/', + main_program=compiled_prog, + ) self.assertRaises(Exception, fleet._transpile, "config") def set_program(self, avg_cost, strategy): @@ -108,7 +124,8 @@ class FleetTest(unittest.TestCase): current_id=0, role=role_maker.Role.SERVER, worker_num=2, - server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"]) + server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"], + ) # for test optimizer without init(role) # fleet.init(role) batch_size = 128 @@ -127,7 +144,8 @@ class FleetTest(unittest.TestCase): current_id=0, role=role_maker.Role.SERVER, worker_num=2, - server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"]) + server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"], + ) # for test optimizer without init(role) fleet.init(role) batch_size = 128 @@ -145,119 +163,120 @@ class FleetTest(unittest.TestCase): class TranspilerOptimizerTest(unittest.TestCase): - def testInvalidInputs(self): self.assertRaises(Exception, TranspilerOptimizer, "Adam", None) - self.assertRaises(Exception, TranspilerOptimizer, - fluid.optimizer.Adam(0.001), "strategy") + self.assertRaises( + Exception, + TranspilerOptimizer, + fluid.optimizer.Adam(0.001), + "strategy", + ) transpiler = TranspilerOptimizer(fluid.optimizer.Adam(0.001)) self.assertRaises(Exception, transpiler.minimize, loss=[]) data = fluid.layers.data(name='X', shape=[1], dtype='float32') hidden = fluid.layers.fc(input=data, size=10) loss = paddle.mean(hidden) - self.assertRaises(Exception, - transpiler.minimize, - loss=loss.name, - startup_program=[]) + self.assertRaises( + Exception, transpiler.minimize, loss=loss.name, startup_program=[] + ) class UserDefinedRoleMakerTest(unittest.TestCase): - - def createRoleMaker(self, - current_id=0, - role=Role.WORKER, - worker_num=1, - server_endpoints=["127.0.0.1:8080"]): - role = UserDefinedRoleMaker(current_id, role, worker_num, - server_endpoints) + def createRoleMaker( + self, + current_id=0, + role=Role.WORKER, + worker_num=1, + server_endpoints=["127.0.0.1:8080"], + ): + role = UserDefinedRoleMaker( + current_id, role, worker_num, server_endpoints + ) def testRoleMaker(self): self.createRoleMaker() # test all invalid server_endpoints self.assertRaises( - Exception, self.createRoleMaker, - server_endpoints=None) # server_endpoints must be as list + Exception, self.createRoleMaker, server_endpoints=None + ) # server_endpoints must be as list + self.assertRaises( + Exception, self.createRoleMaker, server_endpoints=[] + ) # server_endpoints can't be empty + self.assertRaises( + Exception, self.createRoleMaker, server_endpoints=[3, []] + ) # element in server_endpoints must be as string self.assertRaises( - Exception, self.createRoleMaker, - server_endpoints=[]) # server_endpoints can't be empty - self.assertRaises(Exception, - self.createRoleMaker, - server_endpoints=[ - 3, [] - ]) # element in server_endpoints must be as string - self.assertRaises(Exception, - self.createRoleMaker, - server_endpoints=[ - "127.0.0.1:8080", "127.0.0.1:8080" - ]) # element in server_endpoints can't be duplicate + Exception, + self.createRoleMaker, + server_endpoints=["127.0.0.1:8080", "127.0.0.1:8080"], + ) # element in server_endpoints can't be duplicate # test all invalid current_id - self.assertRaises(Exception, self.createRoleMaker, - current_id="0") # current_id must be as int self.assertRaises( - Exception, self.createRoleMaker, - current_id=-1) # current_id must be greater than or equal to 0 + Exception, self.createRoleMaker, current_id="0" + ) # current_id must be as int + self.assertRaises( + Exception, self.createRoleMaker, current_id=-1 + ) # current_id must be greater than or equal to 0 self.assertRaises( Exception, self.createRoleMaker, current_id=1, role=Role.SERVER, - server_endpoints=["127.0.0.1:8080"] + server_endpoints=["127.0.0.1:8080"], ) # if role is server, current_id must be less than len(server_endpoints) # test all invalid worker_num - self.assertRaises(Exception, self.createRoleMaker, - worker_num="1") # worker_num must be as int - self.assertRaises(Exception, self.createRoleMaker, - worker_num=0) # worker_num must be greater than 0 + self.assertRaises( + Exception, self.createRoleMaker, worker_num="1" + ) # worker_num must be as int + self.assertRaises( + Exception, self.createRoleMaker, worker_num=0 + ) # worker_num must be greater than 0 # test all invalid role self.assertRaises( - Exception, self.createRoleMaker, - role=3) # role must be as Role(Role.WORKER=1, Role.SERVER=2) + Exception, self.createRoleMaker, role=3 + ) # role must be as Role(Role.WORKER=1, Role.SERVER=2) class UserDefinedCollectiveRoleMakerTest(unittest.TestCase): - - def createRoleMaker(self, - current_id=0, - worker_endpoints=["127.0.0.1:8080"]): + def createRoleMaker( + self, current_id=0, worker_endpoints=["127.0.0.1:8080"] + ): role = UserDefinedCollectiveRoleMaker(current_id, worker_endpoints) def testRoleMaker(self): self.createRoleMaker() # test all invalid worker_endpoints self.assertRaises( - Exception, self.createRoleMaker, - worker_endpoints=None) # worker_endpoints must be as list + Exception, self.createRoleMaker, worker_endpoints=None + ) # worker_endpoints must be as list + self.assertRaises( + Exception, self.createRoleMaker, worker_endpoints=[] + ) # worker_endpoints can't be empty + self.assertRaises( + Exception, self.createRoleMaker, worker_endpoints=[3, []] + ) # element worker_endpoints must be as string self.assertRaises( - Exception, self.createRoleMaker, - worker_endpoints=[]) # worker_endpoints can't be empty - self.assertRaises(Exception, - self.createRoleMaker, - worker_endpoints=[ - 3, [] - ]) # element worker_endpoints must be as string - self.assertRaises(Exception, - self.createRoleMaker, - worker_endpoints=[ - "127.0.0.1:8080", "127.0.0.1:8080" - ]) # element in worker_endpoints can't be duplicate + Exception, + self.createRoleMaker, + worker_endpoints=["127.0.0.1:8080", "127.0.0.1:8080"], + ) # element in worker_endpoints can't be duplicate # test all invalid current_id - self.assertRaises(Exception, self.createRoleMaker, - current_id="0") # current_id must be as int self.assertRaises( - Exception, self.createRoleMaker, - current_id=-1) # current_id must be greater than or equal to 0 + Exception, self.createRoleMaker, current_id="0" + ) # current_id must be as int + self.assertRaises( + Exception, self.createRoleMaker, current_id=-1 + ) # current_id must be greater than or equal to 0 self.assertRaises( Exception, self.createRoleMaker, current_id=1, - worker_endpoints=[ - "127.0.0.1:8080" - ]) # current_id must be less than len(worker_endpoints) + worker_endpoints=["127.0.0.1:8080"], + ) # current_id must be less than len(worker_endpoints) class CollectiveOptimizerTest(unittest.TestCase): - def test_ds_as_None(self): optimizer = fluid.optimizer.AdamOptimizer() dist_optimizer = CollectiveOptimizer(optimizer, strategy=None) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_ascend_utils.py b/python/paddle/fluid/tests/unittests/test_fleet_ascend_utils.py index 7ee6ee83f22..03260616cda 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_ascend_utils.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_ascend_utils.py @@ -18,30 +18,22 @@ import unittest import paddle.distributed.fleet.ascend_utils as ascend_utils RANK_TABLE_JSON = { - "status": - "completed", - "version": - "1.0", - "server_count": - "1", - "server_list": [{ - "server_id": - "127.0.0.1", - "device": [{ - "device_id": "0", - "device_ip": "192.1.184.23", - "rank_id": "0" - }, { - "device_id": "1", - "device_ip": "192.2.21.93", - "rank_id": "1" - }] - }] + "status": "completed", + "version": "1.0", + "server_count": "1", + "server_list": [ + { + "server_id": "127.0.0.1", + "device": [ + {"device_id": "0", "device_ip": "192.1.184.23", "rank_id": "0"}, + {"device_id": "1", "device_ip": "192.2.21.93", "rank_id": "1"}, + ], + } + ], } class TestAscendUtil(unittest.TestCase): - def test_get_cloud_cluster(self): cluster, pod = ascend_utils.get_cloud_cluster() self.assertTrue(cluster) @@ -51,7 +43,8 @@ class TestAscendUtil(unittest.TestCase): json.dump(RANK_TABLE_JSON, f) rank_table_file = "./rank_table_file.json" cluster, pod = ascend_utils.get_cloud_cluster( - rank_table_file=rank_table_file) + rank_table_file=rank_table_file + ) self.assertTrue(cluster) self.assertTrue(pod) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_auto.py b/python/paddle/fluid/tests/unittests/test_fleet_auto.py index 29710dc5885..608b948d35f 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_auto.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_auto.py @@ -21,26 +21,27 @@ paddle.enable_static() class TestDistributedStrategyAuto(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_distributed_strategy_auto(self): fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base.py b/python/paddle/fluid/tests/unittests/test_fleet_base.py index aba343b1f29..49dd5d6928b 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base.py @@ -22,13 +22,13 @@ import numpy as np class TestFleetBase(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36000" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36002" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36002" def test_init(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) @@ -59,8 +59,9 @@ class TestFleetBase(unittest.TestCase): def test_worker_endpoints(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - self.assertEqual("127.0.0.1:36000", - fleet.worker_endpoints(to_string=True)) + self.assertEqual( + "127.0.0.1:36000", fleet.worker_endpoints(to_string=True) + ) self.assertEqual(["127.0.0.1:36000"], fleet.worker_endpoints()) def test_server_num(self): @@ -90,10 +91,13 @@ class TestFleetBase(unittest.TestCase): role = role_maker.PaddleCloudRoleMaker() fleet.init(role) if fleet.is_server(): - self.assertEqual("127.0.0.1:36001,127.0.0.2:36002", - fleet.server_endpoints(to_string=True)) - self.assertEqual(["127.0.0.1:36001", "127.0.0.2:36002"], - fleet.server_endpoints()) + self.assertEqual( + "127.0.0.1:36001,127.0.0.2:36002", + fleet.server_endpoints(to_string=True), + ) + self.assertEqual( + ["127.0.0.1:36001", "127.0.0.2:36002"], fleet.server_endpoints() + ) def test_is_server(self): os.environ["TRAINING_ROLE"] = "PSERVER" @@ -139,14 +143,15 @@ class TestFleetBase(unittest.TestCase): def test_exception(self): import paddle.distributed.fleet as fleet + self.assertRaises(Exception, fleet.init_worker) class TestFleetDygraph(unittest.TestCase): - def setUp(self): os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213,127.0.0.1:36214" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36213,127.0.0.1:36214" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ["PADDLE_TRAINER_ID"] = "0" @@ -156,8 +161,9 @@ class TestFleetDygraph(unittest.TestCase): value = np.arange(26).reshape(2, 13).astype("float32") a = fluid.dygraph.to_variable(value) layer = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.Adam(learning_rate=0.01, - parameters=layer.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=0.01, parameters=layer.parameters() + ) # remove init cause this UT cannot launch distributed task adam = fleet.distributed_optimizer(adam) try: @@ -169,7 +175,7 @@ class TestFleetDygraph(unittest.TestCase): lr = 0.001 adam.set_lr(lr) cur_lr = adam.get_lr() - assert (lr == cur_lr) + assert lr == cur_lr state_dict = adam.state_dict() adam.set_state_dict(state_dict) @@ -177,22 +183,20 @@ class TestFleetDygraph(unittest.TestCase): class TestFleetBaseSingleError(unittest.TestCase): - def setUp(self): os.environ.pop("PADDLE_TRAINER_ENDPOINTS") def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_run_collective_minimize(self): - def test_single_error(): - input_x = paddle.static.data(name="x", - shape=[-1, 32], - dtype='float32') + input_x = paddle.static.data( + name="x", shape=[-1, 32], dtype='float32' + ) input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64') fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh') @@ -202,8 +206,10 @@ class TestFleetBaseSingleError(unittest.TestCase): fleet.init(is_collective=True) # in non_distributed mode(use `python` to launch), raise error if has multi cards - if fluid.core.is_compiled_with_cuda( - ) and fluid.core.get_cuda_device_count() > 1: + if ( + fluid.core.is_compiled_with_cuda() + and fluid.core.get_cuda_device_count() > 1 + ): self.assertRaises(ValueError, test_single_error) else: test_single_error() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base_2.py b/python/paddle/fluid/tests/unittests/test_fleet_base_2.py index ed914d28665..084718c5407 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base_2.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base_2.py @@ -23,12 +23,11 @@ import paddle.fluid as fluid class TestFleetBase(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36000" os.environ["PADDLE_TRAINERS_NUM"] = "1" - #os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ + # os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ # "127.0.0.1:36001,127.0.0.2:36001" def test_ps_minimize(self): @@ -38,23 +37,24 @@ class TestFleetBase(unittest.TestCase): os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["PADDLE_TRAINER_ID"] = "1" - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_slot = paddle.fluid.layers.data(name="slot", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_slot = paddle.fluid.layers.data( + name="slot", shape=[1], dtype='int64' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') - emb = paddle.fluid.layers.embedding(input=input_slot, - size=[10, 9], - is_sparse=True) + emb = paddle.fluid.layers.embedding( + input=input_slot, size=[10, 9], is_sparse=True + ) input_x = paddle.concat(x=[input_x, emb], axis=1) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) role = fleet.PaddleCloudRoleMaker(is_collective=False) @@ -73,16 +73,17 @@ class TestFleetBase(unittest.TestCase): exe.run(paddle.static.default_startup_program()) pe = fluid.ParallelExecutor(use_cuda=False, loss_name=avg_cost.name) compiled_prog = fluid.compiler.CompiledProgram( - fluid.default_main_program()) + fluid.default_main_program() + ) temp_dir = tempfile.TemporaryDirectory() fleet.init_worker() - fleet.fleet.save(dirname=temp_dir.name, - feed=['x', 'y'], - fetch=[avg_cost]) - fleet.fleet.save(dirname=temp_dir.name, - feed=[input_x, input_y], - fetch=[avg_cost]) + fleet.fleet.save( + dirname=temp_dir.name, feed=['x', 'y'], fetch=[avg_cost] + ) + fleet.fleet.save( + dirname=temp_dir.name, feed=[input_x, input_y], fetch=[avg_cost] + ) fleet.fleet.save(dirname=temp_dir.name) fleet.load_model(path=temp_dir.name, mode=0) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base_3.py b/python/paddle/fluid/tests/unittests/test_fleet_base_3.py index 40ee3836dd2..d9014b2c6f8 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base_3.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base_3.py @@ -22,25 +22,26 @@ paddle.enable_static() class TestFleetBase_1(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_collective_minimize(self): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) role = role_maker.PaddleCloudRoleMaker(is_collective=True) @@ -52,25 +53,26 @@ class TestFleetBase_1(unittest.TestCase): class TestFleetBase(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_fleet_get_applied_optimizer(self): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) fleet.init(is_collective=True) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base_4.py b/python/paddle/fluid/tests/unittests/test_fleet_base_4.py index 5d70193212c..986a48162b2 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base_4.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base_4.py @@ -21,13 +21,13 @@ paddle.enable_static() class TestFleetBase(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_fleet_init(self): diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base_single.py b/python/paddle/fluid/tests/unittests/test_fleet_base_single.py index 776a286d97d..13318d2d3df 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base_single.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base_single.py @@ -28,7 +28,6 @@ import paddle.nn as nn class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear1 = nn.Linear(10, 10) @@ -39,7 +38,6 @@ class LinearNet(nn.Layer): class TestFleetDygraphSingle(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -52,8 +50,9 @@ class TestFleetDygraphSingle(unittest.TestCase): layer = LinearNet() loss_fn = nn.MSELoss() - adam = paddle.optimizer.Adam(learning_rate=0.001, - parameters=layer.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=0.001, parameters=layer.parameters() + ) adam = fleet.distributed_optimizer(adam) dp_layer = fleet.distributed_model(layer) @@ -68,14 +67,13 @@ class TestFleetDygraphSingle(unittest.TestCase): class TestFleetBaseSingleRunCollective(unittest.TestCase): - def setUp(self): pass def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_run_collective_minimize(self): @@ -93,8 +91,11 @@ class TestFleetBaseSingleRunCollective(unittest.TestCase): optimizer = fleet.distributed_optimizer(optimizer) optimizer.minimize(avg_cost) - place = fluid.CUDAPlace( - 0) if paddle.fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(paddle.static.default_startup_program()) @@ -105,14 +106,13 @@ class TestFleetBaseSingleRunCollective(unittest.TestCase): class TestFleetBaseSingleRunPS(unittest.TestCase): - def setUp(self): pass def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_run_ps_minimize(self): @@ -139,11 +139,15 @@ class TestFleetBaseSingleRunPS(unittest.TestCase): exe.run(paddle.static.default_startup_program()) step = 10 for i in range(step): - cost_val = exe.run(program=fluid.default_main_program(), - feed=self.gen_data(), - fetch_list=[avg_cost.name]) - print("worker_index: %d, step%d cost = %f" % - (fleet.worker_index(), i, cost_val[0])) + cost_val = exe.run( + program=fluid.default_main_program(), + feed=self.gen_data(), + fetch_list=[avg_cost.name], + ) + print( + "worker_index: %d, step%d cost = %f" + % (fleet.worker_index(), i, cost_val[0]) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fleet_elastic_collective.py b/python/paddle/fluid/tests/unittests/test_fleet_elastic_collective.py index 8d31fc4efb5..2d9a9268ee3 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_elastic_collective.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_elastic_collective.py @@ -25,11 +25,11 @@ print("test") class TestCollectiveLauncher(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.code_path = os.path.join(self.temp_dir.name, - "fake_python_for_elastic.py") + self.code_path = os.path.join( + self.temp_dir.name, "fake_python_for_elastic.py" + ) with open(self.code_path, "w") as f: f.write(fake_python_code) @@ -37,7 +37,6 @@ class TestCollectiveLauncher(unittest.TestCase): self.temp_dir.cleanup() def test_launch(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -76,7 +75,6 @@ class TestCollectiveLauncher(unittest.TestCase): pass def test_stop(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_elastic_init.py b/python/paddle/fluid/tests/unittests/test_fleet_elastic_init.py index e32f74ffb14..5bd3859013c 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_elastic_init.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_elastic_init.py @@ -19,9 +19,7 @@ from paddle.distributed.fleet.launch_utils import DistributeMode class TestElasticInit(unittest.TestCase): - def setUp(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_elastic_manager.py b/python/paddle/fluid/tests/unittests/test_fleet_elastic_manager.py index 3c07edc7803..a4ad3e1a7c4 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_elastic_manager.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_elastic_manager.py @@ -17,17 +17,17 @@ import unittest from paddle.distributed.fleet.elastic.manager import ElasticManager from paddle.distributed.fleet.elastic.manager import LauncherInterface -from paddle.distributed.fleet.elastic.manager import ELASTIC_AUTO_PARALLEL_EXIT_CODE +from paddle.distributed.fleet.elastic.manager import ( + ELASTIC_AUTO_PARALLEL_EXIT_CODE, +) -class MockLease(): - +class MockLease: def refresh(self): pass class MockEtcdClient: - def __init__(self, lease=None): self._lease = lease @@ -66,12 +66,10 @@ class MockEtcdClient: class TestElasticManager(unittest.TestCase): - def setUp(self): self.etcd_client = MockEtcdClient() def test_elastic_manager_init(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -87,8 +85,7 @@ class TestElasticManager(unittest.TestCase): args = Argument() - class _MockLease(): - + class _MockLease: def refresh(self): raise ValueError("valid error, this only for unittest") @@ -96,7 +93,6 @@ class TestElasticManager(unittest.TestCase): elastic = ElasticManager(args, etcd_client=etcd_client) def test_match_faulttolerance(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -117,7 +113,8 @@ class TestElasticManager(unittest.TestCase): hosts = ["10.10.10.1:6001", "10.10.10.2:6001"] os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" self.assertEqual(elastic._match(hosts), True) @@ -127,7 +124,6 @@ class TestElasticManager(unittest.TestCase): self.assertEqual(elastic._match(hosts), False) def test_match_elastic(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -146,16 +142,20 @@ class TestElasticManager(unittest.TestCase): args.ips = "10.10.10.1,10.10.10.2,10.10.10.3,10.10.10.4" os.environ['FLAGS_START_PORT'] = "6001" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001,10.10.10.4:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001,10.10.10.4:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001,10.10.10.4:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001,10.10.10.4:6001" elastic = ElasticManager(args, self.etcd_client) hosts = ["10.10.10.1:6001", "10.10.10.2:6001"] self.assertEqual(elastic._match(hosts), False) hosts = [ - "10.10.10.1:6001", "10.10.10.2:6001", "10.10.10.3:6001", - "10.10.10.4:6001" + "10.10.10.1:6001", + "10.10.10.2:6001", + "10.10.10.3:6001", + "10.10.10.4:6001", ] self.assertEqual(elastic._match(hosts), True) @@ -167,19 +167,20 @@ class TestElasticManager(unittest.TestCase): args.ips = "10.10.10.1,10.10.10.2" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" elastic = ElasticManager(args, self.etcd_client) hosts = ["10.10.10.1:6001", "10.10.10.2:6001"] self.assertEqual(elastic._match(hosts), True) # TODO test timeout - #time.sleep(60) - #self.assertEqual(elastic._match(hosts), True) + # time.sleep(60) + # self.assertEqual(elastic._match(hosts), True) def test_update_hosts_for_faulttolerance(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -198,9 +199,11 @@ class TestElasticManager(unittest.TestCase): os.environ['PADDLE_ELASTIC_NP'] = "2" os.environ['PADDLE_TRAINERS'] = "10.10.10.1,10.10.10.2" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" elastic = ElasticManager(args, self.etcd_client) # add 10.10.10.3:6001 os.environ['PADDLE_TRAINER_ID'] = "0" @@ -244,30 +247,38 @@ class TestElasticManager(unittest.TestCase): os.environ['FLAGS_START_PORT'] = "6001" os.environ['PADDLE_TRAINERS'] = "10.10.10.1,10.10.10.2" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" elastic = ElasticManager(args, self.etcd_client) # add 10.10.10.3:6001 elastic.curr_host = "10.10.10.1:6001" elastic.hosts = [ - "10.10.10.1:6001", "10.10.10.2:6001", "10.10.10.3:6001" + "10.10.10.1:6001", + "10.10.10.2:6001", + "10.10.10.3:6001", ] elastic._update_hosts() - #self.assertEqual(elastic.all_host_endpoints, + # self.assertEqual(elastic.all_host_endpoints, # ["10.10.10.1:6001", "10.10.10.2:6001", "10.10.10.3:6001"]) - self.assertEqual(os.getenv('PADDLE_TRAINERS'), - "10.10.10.1,10.10.10.2,10.10.10.3") + self.assertEqual( + os.getenv('PADDLE_TRAINERS'), "10.10.10.1,10.10.10.2,10.10.10.3" + ) ####################### # elastic, scale in # ####################### os.environ[ - 'PADDLE_TRAINERS'] = "10.10.10.0,10.10.10.1,10.10.10.2,10.10.10.3" + 'PADDLE_TRAINERS' + ] = "10.10.10.0,10.10.10.1,10.10.10.2,10.10.10.3" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.0:6000,10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.0:6000,10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.0:6000,10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.0:6000,10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001" os.environ['POD_IP'] = "10.10.10.1" os.environ['TRAINER_PORTS_NUM'] = "4" os.environ['PADDLE_TRAINER_ID'] = "1" @@ -277,23 +288,31 @@ class TestElasticManager(unittest.TestCase): # remove 10.10.10.1:6001 elastic.curr_host = "10.10.10.1:6001" elastic.hosts = [ - "10.10.10.1:6001", "10.10.10.2:6001", "10.10.10.3:6001" + "10.10.10.1:6001", + "10.10.10.2:6001", + "10.10.10.3:6001", ] elastic._update_hosts() - #self.assertEqual(elastic.all_host_endpoints, + # self.assertEqual(elastic.all_host_endpoints, # ["10.10.10.3:6001", "10.10.10.1:6001", "10.10.10.2:6001"]) - self.assertEqual(os.getenv('PADDLE_TRAINERS'), - "10.10.10.3,10.10.10.1,10.10.10.2") - self.assertEqual(os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS'), - "10.10.10.3:6001,10.10.10.1:6001,10.10.10.2:6001") + self.assertEqual( + os.getenv('PADDLE_TRAINERS'), "10.10.10.3,10.10.10.1,10.10.10.2" + ) + self.assertEqual( + os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS'), + "10.10.10.3:6001,10.10.10.1:6001,10.10.10.2:6001", + ) ############ os.environ[ - 'PADDLE_TRAINERS'] = "10.10.10.1,10.10.10.1,10.10.10.1,10.10.10.1" + 'PADDLE_TRAINERS' + ] = "10.10.10.1,10.10.10.1,10.10.10.1,10.10.10.1" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.1:6002,10.10.10.1:6003,10.10.10.1:6004" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.1:6002,10.10.10.1:6003,10.10.10.1:6004" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.1:6002,10.10.10.1:6003,10.10.10.1:6004" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.1:6002,10.10.10.1:6003,10.10.10.1:6004" os.environ['POD_IP'] = "10.10.10.1" os.environ['TRAINER_PORTS_NUM'] = "4" os.environ['PADDLE_PORT'] = "6001" @@ -304,14 +323,15 @@ class TestElasticManager(unittest.TestCase): os.environ['PADDLE_TRAINER_ID'] = "-1" elastic.hosts = ["10.10.10.1:6001", "10.10.10.1:6003"] elastic._update_hosts() - #self.assertEqual(elastic.all_host_endpoints, + # self.assertEqual(elastic.all_host_endpoints, # ["10.10.10.1:6001", "10.10.10.1:6001"]) self.assertEqual(os.getenv('PADDLE_TRAINERS'), "10.10.10.1,10.10.10.1") - self.assertEqual(os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS'), - "10.10.10.1:6001,10.10.10.1:6003") + self.assertEqual( + os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS'), + "10.10.10.1:6001,10.10.10.1:6003", + ) def test_exit(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -330,7 +350,6 @@ class TestElasticManager(unittest.TestCase): elastic.exit() def test_pre_hook(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -353,7 +372,6 @@ class TestElasticManager(unittest.TestCase): elastic.pre_hook() def test_watch(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -369,7 +387,6 @@ class TestElasticManager(unittest.TestCase): elastic_pre_hook = None class ElasticLauncher: - def watch(self): return ELASTIC_AUTO_PARALLEL_EXIT_CODE @@ -383,14 +400,11 @@ class TestElasticManager(unittest.TestCase): elastic.watch() def test_launcher_interface_check_procs(self): - class Proc: - def poll(self): return ELASTIC_AUTO_PARALLEL_EXIT_CODE class ProcList: - def __init__(self): self.proc = Proc() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_run.py b/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_run.py index 9bde09ab724..4e37988f767 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_run.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_run.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestDistModelRun(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -34,8 +33,9 @@ class TestDistModelRun(unittest.TestCase): def test_dist_model_run(self): # step 0: declare folder to save the model and params - path_prefix = os.path.join(self.temp_dir.name, - "dist_model_run_test/inf") + path_prefix = os.path.join( + self.temp_dir.name, "dist_model_run_test/inf" + ) # step 1: saving the inference model and params x = paddle.static.data(name='x', shape=[28, 28], dtype='float32') @@ -47,12 +47,11 @@ class TestDistModelRun(unittest.TestCase): exe.run(paddle.static.default_startup_program()) x_data = np.random.randn(28, 28).astype('float32') y_data = np.random.randint(0, 9, size=[28, 1]).astype('int64') - exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[avg_loss], + ) paddle.static.save_inference_model(path_prefix, [x, y], [avg_loss], exe) print('save model to', path_prefix) @@ -74,21 +73,23 @@ class TestDistModelRun(unittest.TestCase): print("dist model rst:", dist_model_rst) # step 4: use framework's api to inference with fake data - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.load_inference_model(path_prefix, exe)) - results = exe.run(inference_program, - feed={ - 'x': x_tensor, - 'y': y_tensor - }, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(path_prefix, exe) + results = exe.run( + inference_program, + feed={'x': x_tensor, 'y': y_tensor}, + fetch_list=fetch_targets, + ) load_inference_model_rst = results[0] print("load inference model api rst:", load_inference_model_rst) # step 5: compare two results - np.testing.assert_allclose(dist_model_rst, - load_inference_model_rst, - rtol=1e-05) + np.testing.assert_allclose( + dist_model_rst, load_inference_model_rst, rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_tensor.py b/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_tensor.py index 98affdfa540..85da732ab5b 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_tensor.py @@ -22,53 +22,67 @@ paddle.enable_static() class TestDistModelTensor(unittest.TestCase): - def test_dist_model_tensor(self): tensor_32 = np.random.randint(10, 20, size=[20, 2]).astype('int32') dist_tensor32 = DistModelTensor(tensor_32, '32_tensor') self.assertEqual(dist_tensor32.dtype, DistModelDataType.INT32) - self.assertEqual(dist_tensor32.data.tolist('int32'), - tensor_32.ravel().tolist()) + self.assertEqual( + dist_tensor32.data.tolist('int32'), tensor_32.ravel().tolist() + ) # the length is how many byte the data contains self.assertEqual(dist_tensor32.data.length(), 40 * 4) self.assertEqual(dist_tensor32.name, '32_tensor') dist_tensor32.data.reset(tensor_32) - self.assertEqual(dist_tensor32.as_ndarray().ravel().tolist(), - tensor_32.ravel().tolist()) + self.assertEqual( + dist_tensor32.as_ndarray().ravel().tolist(), + tensor_32.ravel().tolist(), + ) tensor_64 = np.random.randint(10, 20, size=[20, 2]).astype('int64') dist_tensor64 = DistModelTensor(tensor_64, '64_tensor') self.assertEqual(dist_tensor64.dtype, DistModelDataType.INT64) - self.assertEqual(dist_tensor64.data.tolist('int64'), - tensor_64.ravel().tolist()) + self.assertEqual( + dist_tensor64.data.tolist('int64'), tensor_64.ravel().tolist() + ) self.assertEqual(dist_tensor64.data.length(), 40 * 8) self.assertEqual(dist_tensor64.name, '64_tensor') dist_tensor64.data.reset(tensor_64) - self.assertEqual(dist_tensor64.as_ndarray().ravel().tolist(), - tensor_64.ravel().tolist()) + self.assertEqual( + dist_tensor64.as_ndarray().ravel().tolist(), + tensor_64.ravel().tolist(), + ) tensor_float = np.random.randn(20, 2).astype('float32') dist_tensor_float = DistModelTensor(tensor_float, 'float_tensor') self.assertEqual(dist_tensor_float.dtype, DistModelDataType.FLOAT32) - self.assertEqual(dist_tensor_float.data.tolist('float32'), - tensor_float.ravel().tolist()) + self.assertEqual( + dist_tensor_float.data.tolist('float32'), + tensor_float.ravel().tolist(), + ) self.assertEqual(dist_tensor_float.data.length(), 40 * 4) self.assertEqual(dist_tensor_float.name, 'float_tensor') dist_tensor_float.data.reset(tensor_float) - self.assertEqual(dist_tensor_float.as_ndarray().ravel().tolist(), - tensor_float.ravel().tolist()) + self.assertEqual( + dist_tensor_float.as_ndarray().ravel().tolist(), + tensor_float.ravel().tolist(), + ) tensor_float_16 = np.random.randn(20, 2).astype('float16') - dist_tensor_float_16 = DistModelTensor(tensor_float_16, - 'float_tensor_16') + dist_tensor_float_16 = DistModelTensor( + tensor_float_16, 'float_tensor_16' + ) self.assertEqual(dist_tensor_float_16.dtype, DistModelDataType.FLOAT16) - self.assertEqual(dist_tensor_float_16.data.tolist('float16'), - tensor_float_16.ravel().tolist()) + self.assertEqual( + dist_tensor_float_16.data.tolist('float16'), + tensor_float_16.ravel().tolist(), + ) self.assertEqual(dist_tensor_float_16.data.length(), 40 * 2) self.assertEqual(dist_tensor_float_16.name, 'float_tensor_16') dist_tensor_float_16.data.reset(tensor_float_16) - self.assertEqual(dist_tensor_float_16.as_ndarray().ravel().tolist(), - tensor_float_16.ravel().tolist()) + self.assertEqual( + dist_tensor_float_16.as_ndarray().ravel().tolist(), + tensor_float_16.ravel().tolist(), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor.py b/python/paddle/fluid/tests/unittests/test_fleet_executor.py index f8469c8ed26..502f03784cb 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor.py @@ -21,21 +21,21 @@ paddle.enable_static() class TestFleetExecutor(unittest.TestCase): - def fake_fleet_opt(self): # TODO: Fake for coverage will be removed in the future import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() strategy.sharding_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": 1 + "pp_degree": 1, } strategy.pipeline_configs = {"accumulate_steps": 1} fleet_opt = { "dist_strategy": strategy.sharding_configs, "num_micro_batches": strategy.pipeline_configs["accumulate_steps"], - "scheduler": "1F1B" + "scheduler": "1F1B", } return fleet_opt @@ -43,12 +43,12 @@ class TestFleetExecutor(unittest.TestCase): exe = paddle.static.Executor(place) empty_program = paddle.static.Program() with fluid.program_guard(empty_program, empty_program): - x = fluid.layers.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = fluid.layers.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = fluid.layers.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = fluid.layers.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) z = x + y a = 2 * x + 3 * y loss = paddle.mean(a) @@ -57,23 +57,24 @@ class TestFleetExecutor(unittest.TestCase): steps_per_pass = 10 bd = [steps_per_pass * p for p in passes] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] - lr_val = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr) + lr_val = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr + ) opt = paddle.optimizer.AdamW( learning_rate=lr_val, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) opt.minimize(loss) # TODO: section_program will be removed in the future empty_program._pipeline_opt = { "fleet_opt": self.fake_fleet_opt(), - "section_program": empty_program + "section_program": empty_program, } - res = exe.run(empty_program, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[z.name, a.name]) + res = exe.run( + empty_program, + feed={'x': x_data, 'y': y_data}, + fetch_list=[z.name, a.name], + ) return res def test_executor_on_single_device(self): diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_multi_devices.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_multi_devices.py index c21549c3ce3..5c635e4ce48 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_multi_devices.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_multi_devices.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestFleetExecutor(unittest.TestCase): - def run_fleet_executor(self, place, fleet_opt=dict()): exe = paddle.static.Executor(place) empty_program = paddle.static.Program() @@ -30,24 +29,25 @@ class TestFleetExecutor(unittest.TestCase): x = fluid.layers.data(name='x', shape=[1], dtype=paddle.float32) empty_program._pipeline_opt = { "fleet_opt": fleet_opt, - "section_program": empty_program + "section_program": empty_program, } exe.run(empty_program, feed={'x': [1]}) def test_dist_executor_on_multi_devices(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005,127.0.0.1:7006,127.0.0.1:7007" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005,127.0.0.1:7006,127.0.0.1:7007" strategy = fleet.DistributedStrategy() strategy.sharding_configs = { "dp_degree": 2, "mp_degree": 2, - "pp_degree": 2 + "pp_degree": 2, } strategy.pipeline_configs = {"accumulate_steps": 8} fleet_opt = { "dist_strategy": strategy.sharding_configs, - "num_micro_batches": strategy.pipeline_configs["accumulate_steps"] + "num_micro_batches": strategy.pipeline_configs["accumulate_steps"], } if fluid.is_compiled_with_cuda(): # TODO: Distribute test case is not supported for executor can not stop diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_origin_scheduler.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_origin_scheduler.py index 9a71937ddb2..cda6be8f0a6 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_origin_scheduler.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_origin_scheduler.py @@ -21,21 +21,21 @@ paddle.enable_static() class TestFleetExecutor(unittest.TestCase): - def fake_fleet_opt(self): # TODO: Fake for coverage will be removed in the future import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() strategy.sharding_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": 1 + "pp_degree": 1, } strategy.pipeline_configs = {"accumulate_steps": 1} fleet_opt = { "dist_strategy": strategy.sharding_configs, "num_micro_batches": strategy.pipeline_configs["accumulate_steps"], - "scheduler": "Origin" + "scheduler": "Origin", } return fleet_opt @@ -43,12 +43,12 @@ class TestFleetExecutor(unittest.TestCase): exe = paddle.static.Executor(place) empty_program = paddle.static.Program() with fluid.program_guard(empty_program, empty_program): - x = fluid.layers.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = fluid.layers.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = fluid.layers.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = fluid.layers.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) z = x + y a = 2 * x + 3 * y loss = paddle.mean(a) @@ -57,23 +57,24 @@ class TestFleetExecutor(unittest.TestCase): steps_per_pass = 10 bd = [steps_per_pass * p for p in passes] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] - lr_val = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr) + lr_val = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr + ) opt = paddle.optimizer.AdamW( learning_rate=lr_val, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) opt.minimize(loss) # TODO: section_program will be removed in the future empty_program._pipeline_opt = { "fleet_opt": self.fake_fleet_opt(), - "section_program": empty_program + "section_program": empty_program, } - res = exe.run(empty_program, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[z.name, a.name]) + res = exe.run( + empty_program, + feed={'x': x_data, 'y': y_data}, + fetch_list=[z.name, a.name], + ) return res def test_executor_on_single_device(self): diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_task_node.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_task_node.py index 07ecf85c3db..03d6be10111 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_task_node.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_task_node.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestFleetExecutorTaskNode(unittest.TestCase): - def test_task_node(self): program = paddle.static.Program() task_node_0 = core.TaskNode(program.desc, 0, 1, 1) @@ -31,16 +30,19 @@ class TestFleetExecutorTaskNode(unittest.TestCase): self.assertEqual(task_node_1.task_id(), 1) self.assertEqual(task_node_2.task_id(), 2) self.assertTrue( - task_node_0.add_downstream_task(task_node_1.task_id(), 1)) + task_node_0.add_downstream_task(task_node_1.task_id(), 1) + ) self.assertTrue(task_node_1.add_upstream_task(task_node_0.task_id(), 1)) def test_lazy_task_node(self): program = paddle.static.Program() - task = TaskNode(program=program, - rank=0, - max_run_times=1, - max_slot_times=1, - lazy_initialize=True) + task = TaskNode( + program=program, + rank=0, + max_run_times=1, + max_slot_times=1, + lazy_initialize=True, + ) task_node = task.task_node() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_utils.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_utils.py index cbc54e8c70f..fb003f450f6 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_utils.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_utils.py @@ -20,25 +20,27 @@ paddle.enable_static() class TestFleetExecutorUtils(unittest.TestCase): - def test_construct_program(self): # TODO(liyurui): These functions are not ready now. strategy = paddle.distributed.fleet.DistributedStrategy() strategy.sharding_configs = { "dp_degree": 2, "mp_degree": 2, - "pp_degree": 2 + "pp_degree": 2, } fleet_executor_utils = FleetExecutorUtils( dist_strategy=strategy.sharding_configs, rank=0, nrank=1, - max_run_times=1) + max_run_times=1, + ) op_list = {"lr": [], "fwd": [], "bwd": [], "opt": []} program_map = fleet_executor_utils.convert_op_list_to_program( - op_list, paddle.static.Program()) + op_list, paddle.static.Program() + ) task_node_map = fleet_executor_utils.construct_task_nodes_1f1b( - program_map) + program_map + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_with_task_nodes.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_with_task_nodes.py index f80f998c047..802ce0cbfe3 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_with_task_nodes.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_with_task_nodes.py @@ -22,17 +22,16 @@ paddle.enable_static() class TestFleetExecutor(unittest.TestCase): - def run_fleet_executor(self, place, x_data, y_data): exe = paddle.static.Executor(place) empty_program = paddle.static.Program() with fluid.program_guard(empty_program, empty_program): - x = fluid.layers.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = fluid.layers.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = fluid.layers.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = fluid.layers.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) z = x + y a = 2 * x + 3 * y loss = paddle.mean(a) @@ -41,11 +40,13 @@ class TestFleetExecutor(unittest.TestCase): steps_per_pass = 10 bd = [steps_per_pass * p for p in passes] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] - lr_val = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr) + lr_val = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr + ) opt = paddle.optimizer.AdamW( learning_rate=lr_val, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) opt.minimize(loss) # TODO: section_program will be removed in the future task_node = TaskNode( @@ -55,22 +56,20 @@ class TestFleetExecutor(unittest.TestCase): node_type="Compute", max_run_times=1, max_slot_times=1, - lazy_initialize=True) + lazy_initialize=True, + ) empty_program._pipeline_opt = { "fleet_opt": { 'tasks': [task_node], - 'task_id_to_rank': { - task_node.task_id(): 0 - } + 'task_id_to_rank': {task_node.task_id(): 0}, }, - "section_program": empty_program + "section_program": empty_program, } - res = exe.run(empty_program, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[z.name, a.name]) + res = exe.run( + empty_program, + feed={'x': x_data, 'y': y_data}, + fetch_list=[z.name, a.name], + ) return res def test_executor_on_single_device(self): diff --git a/python/paddle/fluid/tests/unittests/test_fleet_gradient_scale.py b/python/paddle/fluid/tests/unittests/test_fleet_gradient_scale.py index 6aefb2926e4..36e823a2bc9 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_gradient_scale.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_gradient_scale.py @@ -22,7 +22,6 @@ import os class TestGradientScale(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" @@ -30,18 +29,19 @@ class TestGradientScale(unittest.TestCase): def mlp(self, input_x, input_y, hid_dim=128, label_dim=2): fc_1 = paddle.static.nn.fc(x=input_x, size=hid_dim, activation='tanh') fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim, activation='tanh') - prediction = paddle.static.nn.fc(x=[fc_2], - size=label_dim, - activation='softmax') - cost = paddle.nn.functional.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.static.nn.fc( + x=[fc_2], size=label_dim, activation='softmax' + ) + cost = paddle.nn.functional.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) return avg_cost def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_gpu(self): @@ -53,16 +53,17 @@ class TestGradientScale(unittest.TestCase): strategy.gradient_scale_configs = {'scale_strategy': 'sum'} with fluid.program_guard(main_program, startup_program): with fluid.unique_name.guard(): - input_x = paddle.static.data(name="x", - shape=[None, 32], - dtype='float32') - input_y = paddle.static.data(name="y", - shape=[None, 1], - dtype='int64') + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32' + ) + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64' + ) cost = self.mlp(input_x=input_x, input_y=input_y) output_name = cost.name - optimizer = fleet.distributed_optimizer(fluid.optimizer.Adam(), - strategy) + optimizer = fleet.distributed_optimizer( + fluid.optimizer.Adam(), strategy + ) optimizer.minimize(cost) final_strategy = fleet._final_strategy() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_metric.py b/python/paddle/fluid/tests/unittests/test_fleet_metric.py index f4af98e649d..c8fd7e0d2e5 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_metric.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_metric.py @@ -32,7 +32,6 @@ class TestFleetMetric(unittest.TestCase): """Set up, set envs.""" class FakeUtil(UtilBase): - def __init__(self, fake_fleet): super(FakeUtil, self).__init__() self.fleet = fake_fleet @@ -79,16 +78,20 @@ class TestFleetMetric(unittest.TestCase): train = fluid.Program() startup = fluid.Program() with fluid.program_guard(train, startup): - t = fluid.layers.create_global_var(shape=[1, 1], - value=1, - dtype='int64', - persistable=True, - force_cpu=True) - t1 = fluid.layers.create_global_var(shape=[1, 1], - value=1, - dtype='int64', - persistable=True, - force_cpu=True) + t = fluid.layers.create_global_var( + shape=[1, 1], + value=1, + dtype='int64', + persistable=True, + force_cpu=True, + ) + t1 = fluid.layers.create_global_var( + shape=[1, 1], + value=1, + dtype='int64', + persistable=True, + force_cpu=True, + ) place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.Scope() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py index 81daea3af93..8c92efe3236 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py @@ -26,7 +26,8 @@ class TestFleet1(unittest.TestCase): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" @@ -41,33 +42,48 @@ class TestFleet1(unittest.TestCase): os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" role_maker = GeneralRoleMaker() - #role_maker.generate_role() + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) - emb = fluid.layers.embedding(input=show, size=[1, 1], \ - is_sparse=True, is_distributed=True, \ - param_attr=fluid.ParamAttr(name="embedding")) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + emb = fluid.layers.embedding( + input=show, + size=[1, 1], + is_sparse=True, + is_distributed=True, + param_attr=fluid.ParamAttr(name="embedding"), + ) fc = fluid.layers.fc(input=emb, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: adam = fluid.optimizer.Adam(learning_rate=0.000005) - adam = fleet.distributed_optimizer(adam, - strategy={ - "embedding": { - "sparse_accessor_class": - "DownpourCtrAccessor" - } - }) + adam = fleet.distributed_optimizer( + adam, + strategy={ + "embedding": { + "sparse_accessor_class": "DownpourCtrAccessor" + } + }, + ) adam.minimize([cost], [scope]) fleet.run_server() except: diff --git a/python/paddle/fluid/tests/unittests/test_fleet_ps.py b/python/paddle/fluid/tests/unittests/test_fleet_ps.py index 687c9a6eaf5..f5f7f81311c 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_ps.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_ps.py @@ -14,15 +14,17 @@ import unittest from paddle.fluid.framework import default_main_program -from paddle.fluid.incubate.fleet.parameter_server.ir.pserver_pass import _get_optimizer_input_shape +from paddle.fluid.incubate.fleet.parameter_server.ir.pserver_pass import ( + _get_optimizer_input_shape, +) main_program = default_main_program() class TestFleetPS(unittest.TestCase): - def test_version(self): from paddle.fluid.incubate.fleet.parameter_server import version + transpiler = version.is_transpiler() self.assertEqual(transpiler, True) @@ -43,8 +45,9 @@ class TestFleetPS(unittest.TestCase): for attrs in optimizers: op_type, varkey, orig_shape, param_shape = attrs - new_shape = _get_optimizer_input_shape(op_type, varkey, orig_shape, - param_shape) + new_shape = _get_optimizer_input_shape( + op_type, varkey, orig_shape, param_shape + ) self.assertListEqual(new_shape, param_shape) optimizers = [] @@ -52,8 +55,9 @@ class TestFleetPS(unittest.TestCase): for attrs in optimizers: op_type, varkey, orig_shape, param_shape = attrs - new_shape = _get_optimizer_input_shape(op_type, varkey, orig_shape, - param_shape) + new_shape = _get_optimizer_input_shape( + op_type, varkey, orig_shape, param_shape + ) self.assertListEqual(new_shape, orig_shape) with self.assertRaises(ValueError): @@ -62,8 +66,9 @@ class TestFleetPS(unittest.TestCase): for attrs in optimizers: op_type, varkey, orig_shape, param_shape = attrs - _get_optimizer_input_shape(op_type, varkey, orig_shape, - param_shape) + _get_optimizer_input_shape( + op_type, varkey, orig_shape, param_shape + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fleet_pyramid_hash.py b/python/paddle/fluid/tests/unittests/test_fleet_pyramid_hash.py index d22fc3a1b8c..fb4ed901d81 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_pyramid_hash.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_pyramid_hash.py @@ -15,12 +15,15 @@ import unittest import paddle.fluid as fluid import paddle.fluid.incubate.fleet.base.role_maker as role_maker -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, +) class TestPyramidHashOpApi(unittest.TestCase): - def test_dist_geo_server_transpiler(self): num_voc = 128 embed_dim = 64 @@ -49,7 +52,8 @@ class TestPyramidHashOpApi(unittest.TestCase): ), param_attr_bl=None, distribute_update_vars=["PyramidHash_emb_0"], - name=None) + name=None, + ) cost = fluid.layers.reduce_sum(hash_embd) @@ -57,7 +61,8 @@ class TestPyramidHashOpApi(unittest.TestCase): current_id=0, role=role_maker.Role.SERVER, worker_num=2, - server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"]) + server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"], + ) fleet.init(role) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py index 9824b59028f..05732d3cc86 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py @@ -27,7 +27,8 @@ class TestCloudRoleMaker(unittest.TestCase): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_tr_rolemaker(self): """Test tr rolenamer.""" @@ -70,20 +71,30 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" role_maker = GeneralRoleMaker() - #print("init rolemaker") - #role_maker.generate_role() + # print("init rolemaker") + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="float32", lod_level=1, append_batch_size=False) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="float32", + lod_level=1, + append_batch_size=False, + ) fc = fluid.layers.fc(input=show, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: @@ -95,8 +106,10 @@ class TestCloudRoleMaker(unittest.TestCase): print("do not support pslib test, skip") return fleet.clear_one_table(0) - from paddle.fluid.incubate.fleet.base.role_maker import \ - MPISymetricRoleMaker + from paddle.fluid.incubate.fleet.base.role_maker import ( + MPISymetricRoleMaker, + ) + try: role = MPISymetricRoleMaker() role._all_reduce([1], [2]) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py index b60c43a209a..7d03b00fbdd 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py @@ -34,7 +34,9 @@ class TestCloudRoleMaker2(unittest.TestCase): def test_pslib_2(self): """Test cases for pslib.""" import paddle.fluid as fluid - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, + ) from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import RoleMakerBase @@ -58,11 +60,21 @@ class TestCloudRoleMaker2(unittest.TestCase): startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="float32", lod_level=1, append_batch_size=False) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="float32", + lod_level=1, + append_batch_size=False, + ) fc = fluid.layers.fc(input=show, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: @@ -159,8 +171,9 @@ class TestCloudRoleMaker2(unittest.TestCase): role23._get_size() role23._get_size() - path = os.path.join(self.temp_dir.name, - "test_fleet_gloo_role_maker_1.txt") + path = os.path.join( + self.temp_dir.name, "test_fleet_gloo_role_maker_1.txt" + ) with open(path, "w") as f: data = "1 1 1 1\n" f.write(data) @@ -172,7 +185,7 @@ class TestCloudRoleMaker2(unittest.TestCase): dataset.get_memory_data_size(fleet) dataset.get_shuffle_data_size(fleet) - class TmpClass(): + class TmpClass: """ dummy tmp class """ @@ -262,13 +275,16 @@ class TestCloudRoleMaker2(unittest.TestCase): tmp.all_reduce_worker([], []) tmp.barrier_worker() from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker + tmp = RoleMakerBase() tmp.all_gather(1) tmp.all_reduce_worker([], []) tmp.barrier_worker() tmp.barrier_all() - from paddle.fluid.incubate.fleet.base.role_maker import \ - MPISymetricRoleMaker + from paddle.fluid.incubate.fleet.base.role_maker import ( + MPISymetricRoleMaker, + ) + tmp1 = MPISymetricRoleMaker() tmp1.all_gather(1) tmp1.all_gather(1) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py index 2e639a81679..0ce1b0743aa 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py @@ -26,7 +26,8 @@ class TestCloudRoleMaker(unittest.TestCase): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" @@ -40,22 +41,34 @@ class TestCloudRoleMaker(unittest.TestCase): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" - role_maker = GeneralRoleMaker(init_timeout_seconds=100, - run_timeout_seconds=100, - http_ip_port="127.0.0.1:36003") - #role_maker.generate_role() + role_maker = GeneralRoleMaker( + init_timeout_seconds=100, + run_timeout_seconds=100, + http_ip_port="127.0.0.1:36003", + ) + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="float32", lod_level=1, append_batch_size=False) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="float32", + lod_level=1, + append_batch_size=False, + ) fc = fluid.layers.fc(input=show, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: @@ -72,6 +85,7 @@ class TestCloudRoleMaker(unittest.TestCase): return from paddle.fluid.incubate.fleet.base.role_maker import MockBarrier + mb = MockBarrier() mb.barrier() mb.barrier_all() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py index 14d5277446b..2825a1232fc 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py @@ -26,11 +26,13 @@ class TestCloudRoleMaker(unittest.TestCase): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" import threading + try: from paddle.distributed.fleet.utils.http_server import KVHandler from paddle.distributed.fleet.utils.http_server import KVServer @@ -39,7 +41,7 @@ class TestCloudRoleMaker(unittest.TestCase): print("warning: no fleet, skip test_pslib_4") return - class FakeStream(): + class FakeStream: """ it is a fake stream only for test. """ @@ -113,6 +115,7 @@ class TestCloudRoleMaker(unittest.TestCase): fake end header, it will do nothing. """ pass + except: print("warning: no KVHandler, skip test_pslib_4") return @@ -130,6 +133,7 @@ class TestCloudRoleMaker(unittest.TestCase): self.delete_kv = {} self.kv_lock = threading.Lock() self.kv = {} + except: print("warning: no KVHTTPServer, skip test_pslib_4") return @@ -147,6 +151,7 @@ class TestCloudRoleMaker(unittest.TestCase): self.listen_thread = None self.size = {} self.size["a"] = 999 + except: print("warning: no KVServer, skip test_pslib_4") return diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_init.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_init.py index dfc88afaf1f..f7f2b11acdc 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_init.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_init.py @@ -25,7 +25,8 @@ class TestPSCloudRoleMakerCase1(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" def test_paddle_trainers_num(self): # PADDLE_TRAINERS_NUM @@ -40,7 +41,8 @@ class TestPSCloudRoleMakerCase2(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) def test_training_role(self): @@ -56,7 +58,8 @@ class TestPSCloudRoleMakerCase3(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'TRAINER' @@ -73,7 +76,8 @@ class TestPSCloudRoleMakerCase4(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'PSERVER' @@ -90,7 +94,8 @@ class TestPSCloudRoleMakerCase5(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'PSERVER' os.environ["PADDLE_PORT"] = str(4001) @@ -108,9 +113,11 @@ class TestPSCloudRoleMakerCase6(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ[ - "PADDLE_HETER_TRAINER_IP_PORT_LIST"] = "127.0.0.1:4003,127.0.0.1:4004" + "PADDLE_HETER_TRAINER_IP_PORT_LIST" + ] = "127.0.0.1:4003,127.0.0.1:4004" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'HETER_TRAINER' @@ -127,9 +134,11 @@ class TestPSCloudRoleMakerCase7(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ[ - "PADDLE_HETER_TRAINER_IP_PORT_LIST"] = "127.0.0.1:4003,127.0.0.1:4004" + "PADDLE_HETER_TRAINER_IP_PORT_LIST" + ] = "127.0.0.1:4003,127.0.0.1:4004" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'HETER_TRAINER' os.environ["PADDLE_PORT"] = str(4003) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_runtime.py b/python/paddle/fluid/tests/unittests/test_fleet_runtime.py index 21a0afab012..4a415f92ca5 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_runtime.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_runtime.py @@ -17,9 +17,9 @@ import paddle class TestFleetRuntime(unittest.TestCase): - def test_fleet_runtime_base(self): import paddle.distributed.fleet.runtime + base = paddle.distributed.fleet.runtime.runtime_base.RuntimeBase() base._run_worker() base._init_server() @@ -30,7 +30,9 @@ class TestFleetRuntime(unittest.TestCase): def test_fleet_collective_runtime(self): import paddle.distributed.fleet.runtime - collective_runtime = paddle.distributed.fleet.runtime.CollectiveRuntime( + + collective_runtime = ( + paddle.distributed.fleet.runtime.CollectiveRuntime() ) collective_runtime._init_worker() collective_runtime._run_worker() @@ -42,20 +44,26 @@ class TestFleetRuntime(unittest.TestCase): def test_fleet_ps_runtime(self): ps_runtime = paddle.distributed.fleet.runtime.ParameterServerRuntime() - self.assertRaises(Exception, ps_runtime._get_optimizer_status, - "test_op", None) + self.assertRaises( + Exception, ps_runtime._get_optimizer_status, "test_op", None + ) reshaped_names, origin_names = ps_runtime._get_optimizer_status( - "adam", "param") + "adam", "param" + ) self.assertTrue( - len(reshaped_names) == 2 and reshaped_names[0] == 'param_moment1_0' - and reshaped_names[1] == 'param_moment2_0') + len(reshaped_names) == 2 + and reshaped_names[0] == 'param_moment1_0' + and reshaped_names[1] == 'param_moment2_0' + ) self.assertTrue( len(origin_names) == 2 and origin_names[0] == 'param_beta1_pow_acc_0' - and origin_names[1] == 'param_beta2_pow_acc_0') + and origin_names[1] == 'param_beta2_pow_acc_0' + ) reshaped_names, origin_names = ps_runtime._get_optimizer_status( - "sgd", "param") + "sgd", "param" + ) self.assertTrue(len(reshaped_names) == 0 and len(origin_names) == 0) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py index 09cc08dc327..d1f4df65328 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py @@ -26,7 +26,8 @@ class TestFleet1(unittest.TestCase): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" @@ -41,22 +42,36 @@ class TestFleet1(unittest.TestCase): os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" role_maker = GeneralRoleMaker() - #role_maker.generate_role() + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) - emb = fluid.layers.embedding(input=show, size=[1, 1], \ - is_sparse=True, is_distributed=True, \ - param_attr=fluid.ParamAttr(name="embedding")) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + emb = fluid.layers.embedding( + input=show, + size=[1, 1], + is_sparse=True, + is_distributed=True, + param_attr=fluid.ParamAttr(name="embedding"), + ) fc = fluid.layers.fc(input=emb, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_util.py b/python/paddle/fluid/tests/unittests/test_fleet_util.py index 1b6f2cf8552..ff12ef64b5f 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_util.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_util.py @@ -31,6 +31,7 @@ class TestFleetUtil(unittest.TestCase): def test_util_base(self): import paddle.distributed.fleet as fleet + util = fleet.UtilBase() strategy = fleet.DistributedStrategy() util._set_strategy(strategy) @@ -39,6 +40,7 @@ class TestFleetUtil(unittest.TestCase): def test_util_factory(self): import paddle.distributed.fleet as fleet + factory = fleet.base.util_factory.UtilFactory() strategy = fleet.DistributedStrategy() role_maker = None # should be fleet.PaddleCloudRoleMaker() @@ -53,6 +55,7 @@ class TestFleetUtil(unittest.TestCase): def test_get_util(self): import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) self.assertNotEqual(fleet.util, None) @@ -61,7 +64,6 @@ class TestFleetUtil(unittest.TestCase): import paddle.distributed.fleet as fleet class UserDefinedUtil(fleet.UtilBase): - def __init__(self): super(UserDefinedUtil, self).__init__() @@ -69,6 +71,7 @@ class TestFleetUtil(unittest.TestCase): return 10 import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) my_util = UserDefinedUtil() @@ -87,8 +90,9 @@ class TestFleetUtil(unittest.TestCase): fleet.util._set_file_system(fs) def download_files(self): - path = download(self.proto_data_url, self.module_name, - self.proto_data_md5) + path = download( + self.proto_data_url, self.module_name, self.proto_data_md5 + ) print('data is downloaded at ' + path) tar = tarfile.open(path) unzip_folder = tempfile.mkdtemp() @@ -97,6 +101,7 @@ class TestFleetUtil(unittest.TestCase): def test_get_file_shard(self): import paddle.distributed.fleet as fleet + self.assertRaises(Exception, fleet.util.get_file_shard, "files") role = role_maker.UserDefinedRoleMaker( @@ -105,7 +110,8 @@ class TestFleetUtil(unittest.TestCase): current_id=0, role=role_maker.Role.WORKER, worker_endpoints=["127.0.0.1:6003", "127.0.0.1:6004"], - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) files = fleet.util.get_file_shard(["1", "2", "3"]) @@ -113,21 +119,27 @@ class TestFleetUtil(unittest.TestCase): def test_program_type_trans(self): import paddle.distributed.fleet as fleet + data_dir = self.download_files() program_dir = os.path.join(data_dir, self.pruned_dir) text_program = "pruned_main_program.pbtxt" binary_program = "pruned_main_program.bin" - text_to_binary = fleet.util._program_type_trans(program_dir, - text_program, True) - binary_to_text = fleet.util._program_type_trans(program_dir, - binary_program, False) + text_to_binary = fleet.util._program_type_trans( + program_dir, text_program, True + ) + binary_to_text = fleet.util._program_type_trans( + program_dir, binary_program, False + ) self.assertTrue( - os.path.exists(os.path.join(program_dir, text_to_binary))) + os.path.exists(os.path.join(program_dir, text_to_binary)) + ) self.assertTrue( - os.path.exists(os.path.join(program_dir, binary_to_text))) + os.path.exists(os.path.join(program_dir, binary_to_text)) + ) def test_prams_check(self): import paddle.distributed.fleet as fleet + data_dir = self.download_files() class config: @@ -139,7 +151,7 @@ class TestFleetUtil(unittest.TestCase): feed_config.feeded_vars_types = [np.float32, np.float32] feed_config.feeded_vars_filelist = [ os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_1")), - os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_2")) + os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_2")), ] fetch_config = config() @@ -155,7 +167,9 @@ class TestFleetUtil(unittest.TestCase): conf.save_params_filename = None # test saved var's shape - conf.dump_program_filename = "pruned_main_program.save_var_shape_not_match" + conf.dump_program_filename = ( + "pruned_main_program.save_var_shape_not_match" + ) self.assertRaises(Exception, fleet.util._params_check) @@ -164,10 +178,13 @@ class TestFleetUtil(unittest.TestCase): results = fleet.util._params_check(conf) self.assertTrue(len(results) == 1) np.testing.assert_array_almost_equal( - results[0], np.array([[3.0590223e-07]], dtype=np.float32)) + results[0], np.array([[3.0590223e-07]], dtype=np.float32) + ) # test feed_var's shape - conf.dump_program_filename = "pruned_main_program.feed_var_shape_not_match" + conf.dump_program_filename = ( + "pruned_main_program.feed_var_shape_not_match" + ) self.assertRaises(Exception, fleet.util._params_check) # test correct case with feed_vars_filelist @@ -175,7 +192,8 @@ class TestFleetUtil(unittest.TestCase): results = fleet.util._params_check(conf) self.assertTrue(len(results) == 1) np.testing.assert_array_almost_equal( - results[0], np.array([[3.0590223e-07]], dtype=np.float32)) + results[0], np.array([[3.0590223e-07]], dtype=np.float32) + ) # test correct case without feed_vars_filelist conf.feed_config.feeded_vars_filelist = None @@ -189,6 +207,7 @@ class TestFleetUtil(unittest.TestCase): def test_proto_check(self): import paddle.distributed.fleet as fleet + data_dir = self.download_files() class config: @@ -196,14 +215,17 @@ class TestFleetUtil(unittest.TestCase): conf = config() conf.train_prog_path = os.path.join( - data_dir, os.path.join(self.train_dir, "join_main_program.pbtxt")) + data_dir, os.path.join(self.train_dir, "join_main_program.pbtxt") + ) conf.is_text_train_program = True # test not match conf.pruned_prog_path = os.path.join( data_dir, - os.path.join(self.pruned_dir, - "pruned_main_program.save_var_shape_not_match")) + os.path.join( + self.pruned_dir, "pruned_main_program.save_var_shape_not_match" + ), + ) conf.is_text_pruned_program = True conf.draw = False res = fleet.util._proto_check(conf) @@ -211,8 +233,8 @@ class TestFleetUtil(unittest.TestCase): # test match conf.pruned_prog_path = os.path.join( - data_dir, os.path.join(self.pruned_dir, - "pruned_main_program.pbtxt")) + data_dir, os.path.join(self.pruned_dir, "pruned_main_program.pbtxt") + ) if sys.platform == 'win32' or sys.platform == 'sys.platform': conf.draw = False else: @@ -223,13 +245,15 @@ class TestFleetUtil(unittest.TestCase): def test_visualize(self): import paddle.distributed.fleet as fleet + if sys.platform == 'win32' or sys.platform == 'sys.platform': pass else: data_dir = self.download_files() program_path = os.path.join( - data_dir, os.path.join(self.train_dir, - "join_main_program.pbtxt")) + data_dir, + os.path.join(self.train_dir, "join_main_program.pbtxt"), + ) is_text = True program = fleet.util._load_program(program_path, is_text) output_dir = os.path.join(data_dir, self.train_dir) @@ -237,10 +261,14 @@ class TestFleetUtil(unittest.TestCase): fleet.util._visualize_graphviz(program, output_dir, output_filename) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename + ".dot"))) + os.path.join(output_dir, output_filename + ".dot") + ) + ) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename + ".pdf"))) + os.path.join(output_dir, output_filename + ".pdf") + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_flip.py b/python/paddle/fluid/tests/unittests/test_flip.py index ddf3534ad33..f5943db87e0 100644 --- a/python/paddle/fluid/tests/unittests/test_flip.py +++ b/python/paddle/fluid/tests/unittests/test_flip.py @@ -41,13 +41,15 @@ class TestFlipOp_API(unittest.TestCase): exe = fluid.Executor(place) exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[3, 2, 1], [6, 5, 4]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='flip output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='flip output is wrong, out =' + str(out_np), + ) def test_dygraph(self): img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) @@ -60,11 +62,11 @@ class TestFlipOp_API(unittest.TestCase): self.assertTrue( (ret.numpy() == out_ref).all(), - msg='flip output is wrong, out =' + str(ret.numpy())) + msg='flip output is wrong, out =' + str(ret.numpy()), + ) class TestFlipOp(OpTest): - def setUp(self): self.op_type = 'flip' self.python_api = paddle.tensor.flip @@ -96,49 +98,42 @@ class TestFlipOp(OpTest): class TestFlipOpAxis1(TestFlipOp): - def init_test_case(self): self.in_shape = (2, 4, 4) self.axis = [0] class TestFlipOpAxis2(TestFlipOp): - def init_test_case(self): self.in_shape = (4, 4, 6, 3) self.axis = [0, 2] class TestFlipOpAxis3(TestFlipOp): - def init_test_case(self): self.in_shape = (4, 3, 1) self.axis = [0, 1, 2] class TestFlipOpAxis4(TestFlipOp): - def init_test_case(self): self.in_shape = (6, 4, 2, 2) self.axis = [0, 1, 2, 3] class TestFlipOpEmptyAxis(TestFlipOp): - def init_test_case(self): self.in_shape = (6, 4, 2, 2) self.axis = [] class TestFlipOpNegAxis(TestFlipOp): - def init_test_case(self): self.in_shape = (6, 4, 2, 2) self.axis = [-1] class TestFlipDoubleGradCheck(unittest.TestCase): - def flip_wrapper(self, x): return paddle.flip(x[0], [0, 1]) @@ -153,17 +148,13 @@ class TestFlipDoubleGradCheck(unittest.TestCase): out = paddle.flip(data, [0, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.flip_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.flip_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -175,7 +166,6 @@ class TestFlipDoubleGradCheck(unittest.TestCase): class TestFlipTripleGradCheck(unittest.TestCase): - def flip_wrapper(self, x): return paddle.flip(x[0], [0, 1]) @@ -190,17 +180,13 @@ class TestFlipTripleGradCheck(unittest.TestCase): out = paddle.flip(data, [0, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.flip_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.flip_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_fmax_op.py b/python/paddle/fluid/tests/unittests/test_fmax_op.py index 16e46275479..986417fede6 100644 --- a/python/paddle/fluid/tests/unittests/test_fmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_fmax_op.py @@ -44,56 +44,56 @@ class ApiFMaxTest(unittest.TestCase): def test_static_api(self): """test_static_api""" paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_fmax = paddle.fmax(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_fmax]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_fmax], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_fmax = paddle.fmax(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_fmax]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_fmax], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_fmax = paddle.fmax(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_fmax]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_fmax], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_fmax = paddle.fmax(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_fmax]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_fmax], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): @@ -151,19 +151,23 @@ class TestElementwiseFmaxOp(OpTest): def test_check_grad_ingore_x(self): """test_check_grad_ingore_x""" - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): """test_check_grad_ingore_y""" - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + check_eager=True, + ) class TestElementwiseFmax2Op(OpTest): @@ -194,19 +198,23 @@ class TestElementwiseFmax2Op(OpTest): def test_check_grad_ingore_x(self): """test_check_grad_ingore_x""" - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): """test_check_grad_ingore_y""" - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + check_eager=True, + ) class TestElementwiseFmax3Op(OpTest): diff --git a/python/paddle/fluid/tests/unittests/test_fmin_op.py b/python/paddle/fluid/tests/unittests/test_fmin_op.py index daad19a4059..e1a9cf7a912 100644 --- a/python/paddle/fluid/tests/unittests/test_fmin_op.py +++ b/python/paddle/fluid/tests/unittests/test_fmin_op.py @@ -46,56 +46,56 @@ class ApiFMinTest(unittest.TestCase): def test_static_api(self): """test_static_api""" paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_fmin = paddle.fmin(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_fmin]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_fmin], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_fmin = paddle.fmin(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_fmin]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_fmin], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_fmin = paddle.fmin(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_fmin]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_fmin], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_fmin = paddle.fmin(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_fmin]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_fmin], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): @@ -153,19 +153,23 @@ class TestElementwiseFminOp(OpTest): def test_check_grad_ingore_x(self): """test_check_grad_ingore_x""" - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): """test_check_grad_ingore_y""" - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + check_eager=True, + ) class TestElementwiseFmin2Op(OpTest): @@ -196,19 +200,23 @@ class TestElementwiseFmin2Op(OpTest): def test_check_grad_ingore_x(self): """test_check_grad_ingore_x""" - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): """test_check_grad_ingore_y""" - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + check_eager=True, + ) class TestElementwiseFmin3Op(OpTest): diff --git a/python/paddle/fluid/tests/unittests/test_fold_op.py b/python/paddle/fluid/tests/unittests/test_fold_op.py index 62833173909..b0c9a757585 100644 --- a/python/paddle/fluid/tests/unittests/test_fold_op.py +++ b/python/paddle/fluid/tests/unittests/test_fold_op.py @@ -42,34 +42,64 @@ class TestFoldOp(OpTest): def calc_fold(self): output_shape = [0] * 4 output_shape[0] = self.batch_size - output_shape[1] = int(self.input_channels / - (self.kernel_sizes[0] * self.kernel_sizes[1])) + output_shape[1] = int( + self.input_channels / (self.kernel_sizes[0] * self.kernel_sizes[1]) + ) output_shape[2] = self.output_sizes[0] output_shape[3] = self.output_sizes[1] dkernel_h = self.dilations[0] * (self.kernel_sizes[0] - 1) + 1 dkernel_w = self.dilations[1] * (self.kernel_sizes[1] - 1) + 1 - col_height = int((self.output_sizes[0] + self.paddings[0] + - self.paddings[2] - dkernel_h) / self.strides[0]) + 1 - col_width = int((self.output_sizes[1] + self.paddings[1] + - self.paddings[3] - dkernel_w) / self.strides[1]) + 1 + col_height = ( + int( + ( + self.output_sizes[0] + + self.paddings[0] + + self.paddings[2] + - dkernel_h + ) + / self.strides[0] + ) + + 1 + ) + col_width = ( + int( + ( + self.output_sizes[1] + + self.paddings[1] + + self.paddings[3] + - dkernel_w + ) + / self.strides[1] + ) + + 1 + ) output = np.zeros(output_shape).astype(np.float64) ############ calculate output ############## for b in range(output_shape[0]): for c in range(self.input_channels): w_offset = int(c % self.kernel_sizes[1]) h_offset = int( - (c / self.kernel_sizes[1]) % self.kernel_sizes[0]) + (c / self.kernel_sizes[1]) % self.kernel_sizes[0] + ) c_out = int(c / self.kernel_sizes[0] / self.kernel_sizes[1]) for h in range(col_height): - h_out = int(h * self.strides[0] - self.paddings[0] + - h_offset * self.dilations[0]) + h_out = int( + h * self.strides[0] + - self.paddings[0] + + h_offset * self.dilations[0] + ) for w in range(col_width): - w_out = int(w * self.strides[1] - self.paddings[1] + - w_offset * self.dilations[1]) + w_out = int( + w * self.strides[1] + - self.paddings[1] + + w_offset * self.dilations[1] + ) if (h_out >= 0 and h_out < self.output_sizes[0]) and ( - w_out >= 0 and w_out < self.output_sizes[1]): - output[b, c_out, h_out, - w_out] += self.x[b, c, w + col_width * h] + w_out >= 0 and w_out < self.output_sizes[1] + ): + output[b, c_out, h_out, w_out] += self.x[ + b, c, w + col_width * h + ] self.outputs = output @@ -82,7 +112,7 @@ class TestFoldOp(OpTest): 'paddings': self.paddings, 'dilations': self.dilations, 'strides': self.strides, - 'output_sizes': self.output_sizes + 'output_sizes': self.output_sizes, } self.outputs = {'Y': self.outputs} @@ -100,7 +130,7 @@ class TestFoldOp(OpTest): class TestFoldAPI(TestFoldOp): - #This is for test on paddle.nn.Fold + # This is for test on paddle.nn.Fold def setUp(self): self.op_type = 'fold' @@ -117,19 +147,19 @@ class TestFoldAPI(TestFoldOp): m = paddle.nn.Fold(**self.attrs) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - self.outputs['Y'], - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), self.outputs['Y'], rtol=1e-05 + ) def test_info(self): str(paddle.nn.Fold(**self.attrs)) class TestFoldOpError(unittest.TestCase): - def test_errors(self): from paddle.nn.functional import fold from paddle.fluid.framework import Program, program_guard + with program_guard(Program(), Program()): def test_input_shape(): @@ -145,59 +175,67 @@ class TestFoldOpError(unittest.TestCase): def test_padding_shape(): # padding_size must be 2 or 4 x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[2, 3], - kernel_sizes=[2, 2], - paddings=[2, 2, 3]) + out = fold( + x, + output_sizes=[2, 3], + kernel_sizes=[2, 2], + paddings=[2, 2, 3], + ) def test_dilations_shape(): # dialtions_size must be 2 x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[2, 3], - kernel_sizes=[2, 2], - dilations=[2, 2, 3]) + out = fold( + x, + output_sizes=[2, 3], + kernel_sizes=[2, 2], + dilations=[2, 2, 3], + ) def test_strides_shape(): # strids_size must be 2 x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[2, 3], - kernel_sizes=[2, 2], - strides=[2, 2, 3]) + out = fold( + x, + output_sizes=[2, 3], + kernel_sizes=[2, 2], + strides=[2, 2, 3], + ) def test_output_size(): # im_h * im_w must be L x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[6, 6], - kernel_sizes=[2, 2], - strides=[1, 1]) + out = fold( + x, output_sizes=[6, 6], kernel_sizes=[2, 2], strides=[1, 1] + ) def test_output_size_2(): # out_size must GT 1 x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[0.1, 0.2], - kernel_sizes=[2, 2], - strides=[1, 1]) + out = fold( + x, + output_sizes=[0.1, 0.2], + kernel_sizes=[2, 2], + strides=[1, 1], + ) def test_block_h_w(): # test_block_h_w GT 0 x = paddle.randn(shape=[2, 1, 1], dtype="float32") - out = fold(x, - output_sizes=[1, 1], - kernel_sizes=[2, 2], - strides=1) + out = fold( + x, output_sizes=[1, 1], kernel_sizes=[2, 2], strides=1 + ) def test_GT_0(): x = paddle.randn(shape=[2, 1, 1], dtype="float32") - out = fold(x, - output_sizes=[0, 0], - kernel_sizes=[0, 0], - dilations=0, - paddings=[0, 0], - strides=0) + out = fold( + x, + output_sizes=[0, 0], + kernel_sizes=[0, 0], + dilations=0, + paddings=[0, 0], + strides=0, + ) self.assertRaises(AssertionError, test_input_shape) self.assertRaises(AssertionError, test_kernel_shape) diff --git a/python/paddle/fluid/tests/unittests/test_frac_api.py b/python/paddle/fluid/tests/unittests/test_frac_api.py index c3f826bed0e..b455dbbdb4d 100644 --- a/python/paddle/fluid/tests/unittests/test_frac_api.py +++ b/python/paddle/fluid/tests/unittests/test_frac_api.py @@ -34,8 +34,11 @@ class TestFracAPI(unittest.TestCase): def setUp(self): self.set_dtype() self.x_np = np.random.uniform(-3, 3, [2, 3]).astype(self.dtype) - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_api_static(self): paddle.enable_static() @@ -46,7 +49,7 @@ class TestFracAPI(unittest.TestCase): if fluid.core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - res, = exe.run(feed={'X': self.x_np}, fetch_list=[out]) + (res,) = exe.run(feed={'X': self.x_np}, fetch_list=[out]) out_ref = ref_frac(self.x_np) np.testing.assert_allclose(out_ref, res, rtol=1e-05) @@ -97,8 +100,11 @@ class TestFracError(unittest.TestCase): def setUp(self): self.x_np = np.random.uniform(-3, 3, [2, 3]).astype('int16') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_error(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_frame_op.py b/python/paddle/fluid/tests/unittests/test_frame_op.py index 7f38b952665..fc0c092e91e 100644 --- a/python/paddle/fluid/tests/unittests/test_frame_op.py +++ b/python/paddle/fluid/tests/unittests/test_frame_op.py @@ -44,7 +44,6 @@ def frame_from_librosa(x, frame_length, hop_length, axis=-1): class TestFrameOp(OpTest): - def setUp(self): self.op_type = "frame" self.python_api = paddle.signal.frame @@ -57,7 +56,7 @@ class TestFrameOp(OpTest): } def initTestCase(self): - input_shape = (150, ) + input_shape = (150,) input_type = 'float64' attrs = { 'frame_length': 50, @@ -78,9 +77,8 @@ class TestFrameOp(OpTest): class TestCase1(TestFrameOp): - def initTestCase(self): - input_shape = (150, ) + input_shape = (150,) input_type = 'float64' attrs = { 'frame_length': 50, @@ -91,7 +89,6 @@ class TestCase1(TestFrameOp): class TestCase2(TestFrameOp): - def initTestCase(self): input_shape = (8, 150) input_type = 'float64' @@ -104,7 +101,6 @@ class TestCase2(TestFrameOp): class TestCase3(TestFrameOp): - def initTestCase(self): input_shape = (150, 8) input_type = 'float64' @@ -117,7 +113,6 @@ class TestCase3(TestFrameOp): class TestCase4(TestFrameOp): - def initTestCase(self): input_shape = (4, 2, 150) input_type = 'float64' @@ -130,7 +125,6 @@ class TestCase4(TestFrameOp): class TestCase5(TestFrameOp): - def initTestCase(self): input_shape = (150, 4, 2) input_type = 'float64' diff --git a/python/paddle/fluid/tests/unittests/test_framework_debug_str.py b/python/paddle/fluid/tests/unittests/test_framework_debug_str.py index d398b66f8fe..6a1e72c072a 100644 --- a/python/paddle/fluid/tests/unittests/test_framework_debug_str.py +++ b/python/paddle/fluid/tests/unittests/test_framework_debug_str.py @@ -17,7 +17,6 @@ from paddle.fluid.framework import Program class TestDebugStringFramework(unittest.TestCase): - def test_debug_str(self): p = Program() p.current_block().create_var(name='t', shape=[0, 1]) diff --git a/python/paddle/fluid/tests/unittests/test_frexp_api.py b/python/paddle/fluid/tests/unittests/test_frexp_api.py index a7c4fa6e2f7..9604080ffd4 100644 --- a/python/paddle/fluid/tests/unittests/test_frexp_api.py +++ b/python/paddle/fluid/tests/unittests/test_frexp_api.py @@ -18,13 +18,15 @@ import paddle.fluid class TestFrexpAPI(unittest.TestCase): - def setUp(self): np.random.seed(1024) self.rtol = 1e-5 self.atol = 1e-8 - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.set_input() def set_input(self): @@ -35,8 +37,9 @@ class TestFrexpAPI(unittest.TestCase): # 开启静态图模式 paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - input_data = paddle.fluid.data('X', self.x_np.shape, - self.x_np.dtype) + input_data = paddle.fluid.data( + 'X', self.x_np.shape, self.x_np.dtype + ) out = paddle.frexp(input_data) # 计算静态图结果 exe = paddle.static.Executor(self.place) @@ -65,7 +68,7 @@ class TestFrexpAPI(unittest.TestCase): class TestSplitsFloat32Case1(TestFrexpAPI): """ - Test num_or_sections which is an integer and data type is float32. + Test num_or_sections which is an integer and data type is float32. """ def set_input(self): @@ -74,7 +77,7 @@ class TestSplitsFloat32Case1(TestFrexpAPI): class TestSplitsFloat64Case1(TestFrexpAPI): """ - Test num_or_sections which is an integer and data type is float64. + Test num_or_sections which is an integer and data type is float64. """ def set_input(self): @@ -83,7 +86,7 @@ class TestSplitsFloat64Case1(TestFrexpAPI): class TestSplitsFloat64Case2(TestFrexpAPI): """ - Test num_or_sections which is an integer and data type is float64. + Test num_or_sections which is an integer and data type is float64. """ def set_input(self): diff --git a/python/paddle/fluid/tests/unittests/test_fs_interface.py b/python/paddle/fluid/tests/unittests/test_fs_interface.py index e4682e5f731..4705e44eba6 100644 --- a/python/paddle/fluid/tests/unittests/test_fs_interface.py +++ b/python/paddle/fluid/tests/unittests/test_fs_interface.py @@ -19,7 +19,6 @@ from paddle.distributed.fleet.utils.fs import FS class FSTest(unittest.TestCase): - def _test_method(self, func): args = inspect.getfullargspec(func).args diff --git a/python/paddle/fluid/tests/unittests/test_fsp_op.py b/python/paddle/fluid/tests/unittests/test_fsp_op.py index bac40a98d4b..3a1de0833a6 100644 --- a/python/paddle/fluid/tests/unittests/test_fsp_op.py +++ b/python/paddle/fluid/tests/unittests/test_fsp_op.py @@ -27,16 +27,18 @@ def fsp_matrix(a, b): a_t = a.transpose([0, 2, 3, 1]) a_t = a_t.reshape([batch, h * w, a_channel]) b_t = b.transpose([0, 2, 3, 1]).reshape([batch, h * w, b_channel]) - a_r = a_t.repeat(b_channel, - axis=1).reshape([batch, h * w, b_channel, - a_channel]).transpose([0, 1, 3, 2]) - b_r = b_t.repeat(a_channel, - axis=1).reshape([batch, h * w, a_channel, b_channel]) + a_r = ( + a_t.repeat(b_channel, axis=1) + .reshape([batch, h * w, b_channel, a_channel]) + .transpose([0, 1, 3, 2]) + ) + b_r = b_t.repeat(a_channel, axis=1).reshape( + [batch, h * w, a_channel, b_channel] + ) return np.mean(a_r * b_r, axis=1) class TestFSPOp(OpTest): - def setUp(self): self.op_type = "fsp" self.initTestCase() @@ -59,25 +61,24 @@ class TestFSPOp(OpTest): class BadInputTest(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): data = fluid.layers.data(name='data', shape=[3, 32, 32]) feature_map_0 = [1, 2, 3] - feature_map_1 = fluid.layers.conv2d(data, - num_filters=2, - filter_size=3) + feature_map_1 = fluid.layers.conv2d( + data, num_filters=2, filter_size=3 + ) loss = fluid.layers.fsp_matrix(feature_map_0, feature_map_1) self.assertRaises(TypeError, test_bad_x) def test_bad_y(): data = fluid.layers.data(name='data', shape=[3, 32, 32]) - feature_map_0 = fluid.layers.conv2d(data, - num_filters=2, - filter_size=3) + feature_map_0 = fluid.layers.conv2d( + data, num_filters=2, filter_size=3 + ) feature_map_1 = [1, 2, 3] loss = fluid.layers.fsp_matrix(feature_map_0, feature_map_1) diff --git a/python/paddle/fluid/tests/unittests/test_ftrl_op.py b/python/paddle/fluid/tests/unittests/test_ftrl_op.py index 7feabc4be60..e346b4f2d11 100644 --- a/python/paddle/fluid/tests/unittests/test_ftrl_op.py +++ b/python/paddle/fluid/tests/unittests/test_ftrl_op.py @@ -29,24 +29,38 @@ def ftrl_step(param, grad, rows, sq_accum, lin_accum, lr, l1, l2, lr_power): new_accum = sq_accum_hit + grad * grad if lr_power == -0.5: - lin_accum_updated = lin_accum_hit + grad - ( - (np.sqrt(new_accum) - np.sqrt(sq_accum_hit)) / lr) * param_hit + lin_accum_updated = ( + lin_accum_hit + + grad + - ((np.sqrt(new_accum) - np.sqrt(sq_accum_hit)) / lr) * param_hit + ) else: - lin_accum_updated = lin_accum_hit + grad - ( - (np.power(new_accum, -lr_power) - np.power(sq_accum_hit, -lr_power)) - / lr) * param_hit + lin_accum_updated = ( + lin_accum_hit + + grad + - ( + ( + np.power(new_accum, -lr_power) + - np.power(sq_accum_hit, -lr_power) + ) + / lr + ) + * param_hit + ) x = l1 * np.sign(lin_accum_updated) - lin_accum_updated if lr_power == -0.5: y = (np.sqrt(new_accum) / lr) + (2 * l2) pre_shrink = x / y param_updated = np.where( - np.abs(lin_accum_updated) > l1, pre_shrink, 0.0) + np.abs(lin_accum_updated) > l1, pre_shrink, 0.0 + ) else: y = (np.power(new_accum, -lr_power) / lr) + (2 * l2) pre_shrink = x / y param_updated = np.where( - np.abs(lin_accum_updated) > l1, pre_shrink, 0.0) + np.abs(lin_accum_updated) > l1, pre_shrink, 0.0 + ) sq_accum_updated = sq_accum_hit + grad * grad @@ -63,7 +77,6 @@ def ftrl_step(param, grad, rows, sq_accum, lin_accum, lr, l1, l2, lr_power): class TestFTRLOp(OpTest): - def setUp(self): self.op_type = "ftrl" rows = 102 @@ -81,22 +94,23 @@ class TestFTRLOp(OpTest): 'SquaredAccumulator': sq_accum, 'LinearAccumulator': linear_accum, 'Grad': g, - 'LearningRate': lr + 'LearningRate': lr, } self.attrs = { 'l1': l1, 'l2': l2, 'lr_power': lr_power, - 'learning_rate': lr + 'learning_rate': lr, } param_out, sq_accum_out, lin_accum_out = ftrl_step( - w, g, range(rows), sq_accum, linear_accum, lr, l1, l2, lr_power) + w, g, range(rows), sq_accum, linear_accum, lr, l1, l2, lr_power + ) self.outputs = { 'ParamOut': param_out, 'SquaredAccumOut': sq_accum_out, - 'LinearAccumOut': lin_accum_out + 'LinearAccumOut': lin_accum_out, } def test_check_output(self): @@ -104,7 +118,6 @@ class TestFTRLOp(OpTest): class TestSparseFTRLOp(unittest.TestCase): - def setUp(self): self.lr_power = -0.5 @@ -150,22 +163,32 @@ class TestSparseFTRLOp(unittest.TestCase): # calculate ground-truth answer param_out, sq_accum_out, lin_accum_out = ftrl_step( - param_array, grad_array, rows, sq_accum_array, lin_accum_array, lr, - l1, l2, lr_power) + param_array, + grad_array, + rows, + sq_accum_array, + lin_accum_array, + lr, + l1, + l2, + lr_power, + ) # create and run operator - op = Operator("ftrl", - Param='Param', - Grad='Grad', - ParamOut='Param', - SquaredAccumulator='SquaredAccumulator', - SquaredAccumOut='SquaredAccumulator', - LinearAccumulator='LinearAccumulator', - LinearAccumOut='LinearAccumulator', - LearningRate='LearningRate', - l1=l1, - l2=l2, - lr_power=lr_power) + op = Operator( + "ftrl", + Param='Param', + Grad='Grad', + ParamOut='Param', + SquaredAccumulator='SquaredAccumulator', + SquaredAccumOut='SquaredAccumulator', + LinearAccumulator='LinearAccumulator', + LinearAccumOut='LinearAccumulator', + LearningRate='LearningRate', + l1=l1, + l2=l2, + lr_power=lr_power, + ) op.run(scope, place) @@ -176,15 +199,15 @@ class TestSparseFTRLOp(unittest.TestCase): for i in range(height): for j in range(row_numel): - self.assertAlmostEqual(param_out[i][j], - param_array[i][j], - places=4) - self.assertAlmostEqual(sq_accum_out[i][j], - sq_accum_array[i][j], - places=4) - self.assertAlmostEqual(lin_accum_out[i][j], - lin_accum_array[i][j], - places=4) + self.assertAlmostEqual( + param_out[i][j], param_array[i][j], places=4 + ) + self.assertAlmostEqual( + sq_accum_out[i][j], sq_accum_array[i][j], places=4 + ) + self.assertAlmostEqual( + lin_accum_out[i][j], lin_accum_array[i][j], places=4 + ) def init_kernel(self): pass @@ -198,12 +221,12 @@ class TestSparseFTRLOp(unittest.TestCase): class TestSparseFTRLOp2(TestSparseFTRLOp): - def init_kernel(self): self.lr_power = -0.6 if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_full_like_op.py b/python/paddle/fluid/tests/unittests/test_full_like_op.py index fe565140e70..c24ee6791b1 100644 --- a/python/paddle/fluid/tests/unittests/test_full_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_full_like_op.py @@ -23,16 +23,16 @@ from paddle.fluid.framework import _test_eager_guard class TestFullOp(unittest.TestCase): - """ Test fill_any_like op(whose API is full_like) for attr out. """ + """Test fill_any_like op(whose API is full_like) for attr out.""" def test_attr_tensor_API(self): startup_program = Program() train_program = Program() with program_guard(train_program, startup_program): fill_value = 2.0 - input = paddle.fluid.data(name='input', - dtype='float32', - shape=[2, 3]) + input = paddle.fluid.data( + name='input', dtype='float32', shape=[2, 3] + ) output = paddle.full_like(input, fill_value) output_dtype = paddle.full_like(input, fill_value, dtype='float32') @@ -44,14 +44,15 @@ class TestFullOp(unittest.TestCase): img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) - self.assertTrue(not (out_np - np.full_like(img, fill_value)).any(), - msg="full_like output is wrong, out = " + - str(out_np)) + self.assertTrue( + not (out_np - np.full_like(img, fill_value)).any(), + msg="full_like output is wrong, out = " + str(out_np), + ) def test_full_like_imperative(self): paddle.disable_static() @@ -73,24 +74,25 @@ class TestFullOp(unittest.TestCase): class TestFullOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage + # for ci coverage - input_data = paddle.fluid.data(name='input', - dtype='float32', - shape=[2, 3]) + input_data = paddle.fluid.data( + name='input', dtype='float32', shape=[2, 3] + ) output = paddle.full_like(input_data, 2.0) def test_input_dtype(): paddle.full_like - self.assertRaises(TypeError, - paddle.full_like, - x=input_data, - fill_value=2, - dtype='uint4') + self.assertRaises( + TypeError, + paddle.full_like, + x=input_data, + fill_value=2, + dtype='uint4', + ) class TestFullLikeOp1(OpTest): @@ -107,7 +109,7 @@ class TestFullLikeOp1(OpTest): self.outputs = {'Out': out} self.attrs = { 'value': self.fill_value, - 'dtype': convert_np_dtype_to_dtype_(self.dtype) + 'dtype': convert_np_dtype_to_dtype_(self.dtype), } def init_data(self): @@ -120,7 +122,6 @@ class TestFullLikeOp1(OpTest): class TestFullLikeOp2(TestFullLikeOp1): - def init_data(self): self.fill_value = 1000 self.shape = [1024, 1024] @@ -128,25 +129,26 @@ class TestFullLikeOp2(TestFullLikeOp1): class TestFullLikeOp3(TestFullLikeOp1): - def init_data(self): self.fill_value = 8888 self.shape = [5000, 5000] self.dtype = np.int64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFullLikeOp4(unittest.TestCase): - def test_skip_data_transform(self): paddle.disable_static() with _test_eager_guard(): - x = paddle.to_tensor([1., 2., 3., 4.], - place=paddle.CUDAPinnedPlace()) - out = paddle.full_like(x, 1.) + x = paddle.to_tensor( + [1.0, 2.0, 3.0, 4.0], place=paddle.CUDAPinnedPlace() + ) + out = paddle.full_like(x, 1.0) self.assertTrue( - (out.numpy() == np.ones([4]).astype(np.float32)).all(), True) + (out.numpy() == np.ones([4]).astype(np.float32)).all(), True + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_full_op.py b/python/paddle/fluid/tests/unittests/test_full_op.py index 1f20cfd6c72..a8c54188e48 100644 --- a/python/paddle/fluid/tests/unittests/test_full_op.py +++ b/python/paddle/fluid/tests/unittests/test_full_op.py @@ -23,45 +23,44 @@ from paddle.fluid.framework import _test_eager_guard # Test python API class TestFullAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 2) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) out_1 = paddle.full(shape=[1, 2], dtype="float32", fill_value=1.1) - out_2 = paddle.full(shape=[1, positive_2_int32], - dtype="float32", - fill_value=1.1) + out_2 = paddle.full( + shape=[1, positive_2_int32], dtype="float32", fill_value=1.1 + ) - out_3 = paddle.full(shape=[1, positive_2_int64], - dtype="float32", - fill_value=1.1) + out_3 = paddle.full( + shape=[1, positive_2_int64], dtype="float32", fill_value=1.1 + ) - out_4 = paddle.full(shape=shape_tensor_int32, - dtype="float32", - fill_value=1.2) + out_4 = paddle.full( + shape=shape_tensor_int32, dtype="float32", fill_value=1.2 + ) - out_5 = paddle.full(shape=shape_tensor_int64, - dtype="float32", - fill_value=1.1) + out_5 = paddle.full( + shape=shape_tensor_int64, dtype="float32", fill_value=1.1 + ) - out_6 = paddle.full(shape=shape_tensor_int64, - dtype=np.float32, - fill_value=1.1) + out_6 = paddle.full( + shape=shape_tensor_int64, dtype=np.float32, fill_value=1.1 + ) val = fluid.layers.fill_constant(shape=[1], dtype=np.float32, value=1.1) - out_7 = paddle.full(shape=shape_tensor_int64, - dtype=np.float32, - fill_value=val) + out_7 = paddle.full( + shape=shape_tensor_int64, dtype=np.float32, fill_value=val + ) exe = fluid.Executor(place=fluid.CPUPlace()) res_1, res_2, res_3, res_4, res_5, res_6, res_7 = exe.run( @@ -70,7 +69,8 @@ class TestFullAPI(unittest.TestCase): "shape_tensor_int32": np.array([1, 2]).astype("int32"), "shape_tensor_int64": np.array([1, 2]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, np.full([1, 2], 1.1, dtype="float32")) assert np.array_equal(res_2, np.full([1, 2], 1.1, dtype="float32")) @@ -85,89 +85,104 @@ class TestFullAPI(unittest.TestCase): with _test_eager_guard(): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 2) - positive_4_int64 = fluid.layers.fill_constant([1], "int64", 4, - True) - - out_1 = paddle.full(shape=[1, 2], - dtype="float32", - fill_value=1.1) - - out_2 = paddle.full(shape=[1, positive_2_int32.item()], - dtype="float32", - fill_value=1.1) - - out_3 = paddle.full(shape=[1, positive_2_int64.item()], - dtype="float32", - fill_value=1.1) - - out_4 = paddle.full(shape=[1, 2], - dtype="float32", - fill_value=1.2) - - out_5 = paddle.full(shape=[1, 2], - dtype="float32", - fill_value=1.1) - - out_6 = paddle.full(shape=[1, 2], - dtype=np.float32, - fill_value=1.1) - - val = fluid.layers.fill_constant(shape=[1], - dtype=np.float32, - value=1.1) - out_7 = paddle.full(shape=[1, 2], - dtype=np.float32, - fill_value=val) - - out_8 = paddle.full(shape=positive_2_int32, - dtype="float32", - fill_value=1.1) - - out_9 = paddle.full(shape=[ - positive_2_int32, positive_2_int64, positive_4_int64 - ], - dtype="float32", - fill_value=1.1) + positive_4_int64 = fluid.layers.fill_constant( + [1], "int64", 4, True + ) + + out_1 = paddle.full( + shape=[1, 2], dtype="float32", fill_value=1.1 + ) + + out_2 = paddle.full( + shape=[1, positive_2_int32.item()], + dtype="float32", + fill_value=1.1, + ) + + out_3 = paddle.full( + shape=[1, positive_2_int64.item()], + dtype="float32", + fill_value=1.1, + ) + + out_4 = paddle.full( + shape=[1, 2], dtype="float32", fill_value=1.2 + ) + + out_5 = paddle.full( + shape=[1, 2], dtype="float32", fill_value=1.1 + ) + + out_6 = paddle.full( + shape=[1, 2], dtype=np.float32, fill_value=1.1 + ) + + val = fluid.layers.fill_constant( + shape=[1], dtype=np.float32, value=1.1 + ) + out_7 = paddle.full( + shape=[1, 2], dtype=np.float32, fill_value=val + ) + + out_8 = paddle.full( + shape=positive_2_int32, dtype="float32", fill_value=1.1 + ) + + out_9 = paddle.full( + shape=[ + positive_2_int32, + positive_2_int64, + positive_4_int64, + ], + dtype="float32", + fill_value=1.1, + ) # test for numpy.float64 as fill_value - out_10 = paddle.full_like(out_7, - dtype=np.float32, - fill_value=np.abs(1.1)) - - assert np.array_equal(out_1, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_2, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_3, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_4, - np.full([1, 2], 1.2, dtype="float32")) - assert np.array_equal(out_5, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_6, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_7, - np.full([1, 2], 1.1, dtype="float32")) + out_10 = paddle.full_like( + out_7, dtype=np.float32, fill_value=np.abs(1.1) + ) + + assert np.array_equal( + out_1, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_2, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_3, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_4, np.full([1, 2], 1.2, dtype="float32") + ) + assert np.array_equal( + out_5, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_6, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_7, np.full([1, 2], 1.1, dtype="float32") + ) assert np.array_equal(out_8, np.full([2], 1.1, dtype="float32")) - assert np.array_equal(out_9, - np.full([2, 2, 4], 1.1, dtype="float32")) - assert np.array_equal(out_10, - np.full([1, 2], 1.1, dtype="float32")) + assert np.array_equal( + out_9, np.full([2, 2, 4], 1.1, dtype="float32") + ) + assert np.array_equal( + out_10, np.full([1, 2], 1.1, dtype="float32") + ) class TestFullOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage - self.assertRaises(TypeError, - paddle.full, - shape=[1], - fill_value=5, - dtype='uint4') + # for ci coverage + self.assertRaises( + TypeError, paddle.full, shape=[1], fill_value=5, dtype='uint4' + ) # The argument dtype of full must be one of bool, float16, - #float32, float64, uint8, int16, int32 or int64 + # float32, float64, uint8, int16, int32 or int64 # The argument shape's type of full_op must be list, tuple or Variable. def test_shape_type(): @@ -177,17 +192,17 @@ class TestFullOpError(unittest.TestCase): # The shape dtype of full op must be int32 or int64. def test_shape_tensor_dtype(): - shape = fluid.data(name="shape_tensor", - shape=[2], - dtype="float32") + shape = fluid.data( + name="shape_tensor", shape=[2], dtype="float32" + ) paddle.full(shape=shape, dtype="float32", fill_value=1) self.assertRaises(TypeError, test_shape_tensor_dtype) def test_shape_tensor_list_dtype(): - shape = fluid.data(name="shape_tensor_list", - shape=[1], - dtype="bool") + shape = fluid.data( + name="shape_tensor_list", shape=[1], dtype="bool" + ) paddle.full(shape=[shape, 2], dtype="float32", fill_value=1) self.assertRaises(TypeError, test_shape_tensor_list_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_function_hook.py b/python/paddle/fluid/tests/unittests/test_function_hook.py index b884d62e7d3..7ba230233a4 100644 --- a/python/paddle/fluid/tests/unittests/test_function_hook.py +++ b/python/paddle/fluid/tests/unittests/test_function_hook.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestCapture: - def __init__(self): self.list = [] @@ -40,7 +39,6 @@ def grad_hook(grad): class TestBakcwardFunctionHookError(unittest.TestCase): - def func_hook(self): input_data = np.ones([4, 4]).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv1d.py b/python/paddle/fluid/tests/unittests/test_functional_conv1d.py index 251ac52451d..6869c511b9f 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv1d.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv1d.py @@ -21,7 +21,6 @@ from unittest import TestCase class TestFunctionalConv1DError(TestCase): - def setUp(self): self.input = [] self.filter = [] @@ -36,16 +35,21 @@ class TestFunctionalConv1DError(TestCase): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv1d(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv1d( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_exception(self): with self.assertRaises(ValueError): @@ -53,7 +57,6 @@ class TestFunctionalConv1DError(TestCase): class TestFunctionalConv1DErrorCase1(TestFunctionalConv1DError): - def setUp(self): self.input = np.random.randn(1, 3, 3) self.filter = np.random.randn(3, 3, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv1d_transpose.py b/python/paddle/fluid/tests/unittests/test_functional_conv1d_transpose.py index 98daff216a7..431c940f0b4 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv1d_transpose.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv1d_transpose.py @@ -21,7 +21,6 @@ from unittest import TestCase class TestFunctionalConv1DError(TestCase): - def setUp(self): self.input = [] self.filter = [] @@ -36,16 +35,21 @@ class TestFunctionalConv1DError(TestCase): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv1d_transpose(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv1d_transpose( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_exception(self): with self.assertRaises(ValueError): @@ -53,7 +57,6 @@ class TestFunctionalConv1DError(TestCase): class TestFunctionalConv1DErrorCase1(TestFunctionalConv1DError): - def setUp(self): self.input = np.random.randn(1, 3, 3) self.filter = np.random.randn(3, 3, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv2d.py b/python/paddle/fluid/tests/unittests/test_functional_conv2d.py index 6c0f526f236..1a92a9babb7 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv2d.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv2d.py @@ -41,27 +41,34 @@ class TestFunctionalConv2D(TestCase): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 2 + filter_shape = (self.filter_shape,) * 2 else: filter_shape = tuple(self.filter_shape) self.weight = np.random.uniform( - -1, 1, (self.out_channels, self.in_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.out_channels, self.in_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - self.channel_last = (self.data_format == "NHWC") + self.channel_last = self.data_format == "NHWC" if self.channel_last: - self.input_shape = (self.batch_size, ) + self.spatial_shape + ( - self.in_channels, ) + self.input_shape = ( + (self.batch_size,) + self.spatial_shape + (self.in_channels,) + ) else: - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) def static_graph_case_1(self): main = fluid.Program() @@ -69,11 +76,17 @@ class TestFunctionalConv2D(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) y = fluid.layers.conv2d( x, self.out_channels, @@ -84,12 +97,14 @@ class TestFunctionalConv2D(TestCase): groups=self.groups, param_attr=I.NumpyArrayInitializer(self.weight), bias_attr=False - if self.no_bias else I.NumpyArrayInitializer(self.bias), + if self.no_bias + else I.NumpyArrayInitializer(self.bias), act=self.act, - data_format=self.data_format) + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def static_graph_case_2(self): @@ -98,24 +113,32 @@ class TestFunctionalConv2D(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight.shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight.shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias.shape, dtype=self.dtype) - y = F.conv2d(x, - weight, - None if self.no_bias else bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d( + x, + weight, + None if self.no_bias else bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) @@ -125,7 +148,7 @@ class TestFunctionalConv2D(TestCase): feed_dict = {"input": self.input, "weight": self.weight} if not self.no_bias: feed_dict["bias"] = self.bias - out, = exe.run(main, feed=feed_dict, fetch_list=[y]) + (out,) = exe.run(main, feed=feed_dict, fetch_list=[y]) return out def dygraph_case(self): @@ -133,14 +156,16 @@ class TestFunctionalConv2D(TestCase): x = dg.to_variable(self.input) weight = dg.to_variable(self.weight) bias = None if self.no_bias else dg.to_variable(self.bias) - y = F.conv2d(x, - weight, - bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d( + x, + weight, + bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) @@ -160,8 +185,9 @@ class TestFunctionalConv2D(TestCase): self.place = fluid.CPUPlace() self._test_identity() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu(self): self.place = fluid.CUDAPlace(0) self._test_identity() @@ -191,12 +217,14 @@ class TestFunctionalConv2DError(TestCase): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 2 + filter_shape = (self.filter_shape,) * 2 else: filter_shape = tuple(self.filter_shape) - self.weight_shape = (self.out_channels, - self.in_channels // self.groups) + filter_shape - self.bias_shape = (self.out_channels, ) + self.weight_shape = ( + self.out_channels, + self.in_channels // self.groups, + ) + filter_shape + self.bias_shape = (self.out_channels,) def static_graph_case(self): main = fluid.Program() @@ -205,28 +233,35 @@ class TestFunctionalConv2DError(TestCase): with fluid.program_guard(main, start): self.channel_last = self.data_format == "NHWC" if self.channel_last: - x = x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias_shape, dtype=self.dtype) - y = F.conv2d(x, - weight, - None if self.no_bias else bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d( + x, + weight, + None if self.no_bias else bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) class TestFunctionalConv2DCase2(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -242,7 +277,6 @@ class TestFunctionalConv2DCase2(TestFunctionalConv2D): class TestFunctionalConv2DCase3(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -258,7 +292,6 @@ class TestFunctionalConv2DCase3(TestFunctionalConv2D): class TestFunctionalConv2DCase4(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -274,7 +307,6 @@ class TestFunctionalConv2DCase4(TestFunctionalConv2D): class TestFunctionalConv2DCase5(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -290,7 +322,6 @@ class TestFunctionalConv2DCase5(TestFunctionalConv2D): class TestFunctionalConv2DCase6(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -306,7 +337,6 @@ class TestFunctionalConv2DCase6(TestFunctionalConv2D): class TestFunctionalConv2DCase7(TestFunctionalConv2D): - def setUp(self): self.in_channels = 6 self.out_channels = 8 @@ -322,7 +352,6 @@ class TestFunctionalConv2DCase7(TestFunctionalConv2D): class TestFunctionalConv2DCase8(TestFunctionalConv2D): - def setUp(self): self.in_channels = 6 self.out_channels = 12 @@ -338,7 +367,6 @@ class TestFunctionalConv2DCase8(TestFunctionalConv2D): class TestFunctionalConv2DErrorCase2(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -354,7 +382,6 @@ class TestFunctionalConv2DErrorCase2(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase3(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -370,7 +397,6 @@ class TestFunctionalConv2DErrorCase3(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase4(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 4 self.out_channels = 3 @@ -386,7 +412,6 @@ class TestFunctionalConv2DErrorCase4(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase7(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -402,7 +427,6 @@ class TestFunctionalConv2DErrorCase7(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase8(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -418,7 +442,6 @@ class TestFunctionalConv2DErrorCase8(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase9(TestFunctionalConv2DError): - def setUp(self): self.in_channels = -5 self.out_channels = 5 @@ -434,7 +457,6 @@ class TestFunctionalConv2DErrorCase9(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase10(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -450,7 +472,6 @@ class TestFunctionalConv2DErrorCase10(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase11(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -466,7 +487,6 @@ class TestFunctionalConv2DErrorCase11(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase12(TestCase): - def setUp(self): self.input = np.array([]) self.filter = np.array([]) @@ -485,38 +505,45 @@ class TestFunctionalConv2DErrorCase12(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): x = fluid.data("input", self.input.shape, dtype=paddle.float32) - y = fluid.layers.conv2d(x, - self.num_filters, - self.filter_size, - stride=self.stride, - padding=self.padding, - dilation=self.dilation, - groups=self.groups, - param_attr=I.NumpyArrayInitializer( - self.filter), - bias_attr=False if self.bias is None - else I.NumpyArrayInitializer(self.bias), - act=None, - data_format=self.data_format) + y = fluid.layers.conv2d( + x, + self.num_filters, + self.filter_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + param_attr=I.NumpyArrayInitializer(self.filter), + bias_attr=False + if self.bias is None + else I.NumpyArrayInitializer(self.bias), + act=None, + data_format=self.data_format, + ) exe = fluid.Executor() exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv2d(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv2d( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_dygraph_exception(self): with self.assertRaises(ValueError): @@ -528,7 +555,6 @@ class TestFunctionalConv2DErrorCase12(TestCase): class TestFunctionalConv2DErrorCase13(TestFunctionalConv2DErrorCase12): - def setUp(self): self.input = np.random.randn(1, 3, 3, 3) self.filter = np.random.randn(3, 3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv2d_transpose.py b/python/paddle/fluid/tests/unittests/test_functional_conv2d_transpose.py index dce6a37c6bb..d377be24ddf 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv2d_transpose.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv2d_transpose.py @@ -43,27 +43,34 @@ class TestFunctionalConv2D(TestCase): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 2 + filter_shape = (self.filter_shape,) * 2 else: filter_shape = tuple(self.filter_shape) self.weight = np.random.uniform( - -1, 1, (self.in_channels, self.out_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.in_channels, self.out_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - self.channel_last = (self.data_format == "NHWC") + self.channel_last = self.data_format == "NHWC" if self.channel_last: - self.input_shape = (self.batch_size, ) + self.spatial_shape + ( - self.in_channels, ) + self.input_shape = ( + (self.batch_size,) + self.spatial_shape + (self.in_channels,) + ) else: - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) def static_graph_case_1(self): main = fluid.Program() @@ -71,11 +78,17 @@ class TestFunctionalConv2D(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) y = fluid.layers.conv2d_transpose( x, self.out_channels, @@ -87,11 +100,13 @@ class TestFunctionalConv2D(TestCase): groups=self.groups, param_attr=I.NumpyArrayInitializer(self.weight), bias_attr=False - if self.no_bias else I.NumpyArrayInitializer(self.bias), - data_format=self.data_format) + if self.no_bias + else I.NumpyArrayInitializer(self.bias), + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def static_graph_case_2(self): @@ -100,31 +115,39 @@ class TestFunctionalConv2D(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight.shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight.shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias.shape, dtype=self.dtype) - y = F.conv2d_transpose(x, - weight, - None if self.no_bias else bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d_transpose( + x, + weight, + None if self.no_bias else bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) feed_dict = {"input": self.input, "weight": self.weight} if not self.no_bias: feed_dict["bias"] = self.bias - out, = exe.run(main, feed=feed_dict, fetch_list=[y]) + (out,) = exe.run(main, feed=feed_dict, fetch_list=[y]) return out def dygraph_case(self): @@ -132,15 +155,17 @@ class TestFunctionalConv2D(TestCase): x = dg.to_variable(self.input) weight = dg.to_variable(self.weight) bias = None if self.no_bias else dg.to_variable(self.bias) - y = F.conv2d_transpose(x, - weight, - bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d_transpose( + x, + weight, + bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) out = y.numpy() return out @@ -160,14 +185,16 @@ class TestFunctionalConv2D(TestCase): with _test_eager_guard(): self.test_identity_cpu() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu(self): self.place = fluid.CUDAPlace(0) self._test_identity() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu_check_eager(self): with _test_eager_guard(): self.test_identity_gpu() @@ -198,12 +225,14 @@ class TestFunctionalConv2DError(TestCase): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 2 + filter_shape = (self.filter_shape,) * 2 else: filter_shape = tuple(self.filter_shape) - self.weight_shape = (self.in_channels, - self.out_channels // self.groups) + filter_shape - self.bias_shape = (self.out_channels, ) + self.weight_shape = ( + self.in_channels, + self.out_channels // self.groups, + ) + filter_shape + self.bias_shape = (self.out_channels,) def static_graph_case(self): main = fluid.Program() @@ -212,29 +241,36 @@ class TestFunctionalConv2DError(TestCase): with fluid.program_guard(main, start): self.channel_last = self.data_format == "NHWC" if self.channel_last: - x = x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias_shape, dtype=self.dtype) - y = F.conv2d_transpose(x, - weight, - None if self.no_bias else bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d_transpose( + x, + weight, + None if self.no_bias else bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) class TestFunctionalConv2DCase2(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -248,7 +284,6 @@ class TestFunctionalConv2DCase2(TestFunctionalConv2D): class TestFunctionalConv2DCase3(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -262,7 +297,6 @@ class TestFunctionalConv2DCase3(TestFunctionalConv2D): class TestFunctionalConv2DCase4(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -276,7 +310,6 @@ class TestFunctionalConv2DCase4(TestFunctionalConv2D): class TestFunctionalConv2DCase5(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -290,7 +323,6 @@ class TestFunctionalConv2DCase5(TestFunctionalConv2D): class TestFunctionalConv2DCase6(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -304,7 +336,6 @@ class TestFunctionalConv2DCase6(TestFunctionalConv2D): class TestFunctionalConv2DCase7(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 4 @@ -318,7 +349,6 @@ class TestFunctionalConv2DCase7(TestFunctionalConv2D): class TestFunctionalConv2DCase8(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 4 @@ -333,7 +363,6 @@ class TestFunctionalConv2DCase8(TestFunctionalConv2D): class TestFunctionalConv2DCase9(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -347,7 +376,6 @@ class TestFunctionalConv2DCase9(TestFunctionalConv2D): class TestFunctionalConv2DCase10(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -361,7 +389,6 @@ class TestFunctionalConv2DCase10(TestFunctionalConv2D): class TestFunctionalConv2DCase11(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -375,7 +402,6 @@ class TestFunctionalConv2DCase11(TestFunctionalConv2D): class TestFunctionalConv2DCase12(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -389,7 +415,6 @@ class TestFunctionalConv2DCase12(TestFunctionalConv2D): class TestFunctionalConv2DErrorCase2(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -403,7 +428,6 @@ class TestFunctionalConv2DErrorCase2(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase3(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -417,7 +441,6 @@ class TestFunctionalConv2DErrorCase3(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase4(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -431,7 +454,6 @@ class TestFunctionalConv2DErrorCase4(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase5(TestFunctionalConv2DError): - def setUp(self): self.in_channels = -2 self.out_channels = 5 @@ -445,7 +467,6 @@ class TestFunctionalConv2DErrorCase5(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase7(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 4 self.out_channels = 5 @@ -460,7 +481,6 @@ class TestFunctionalConv2DErrorCase7(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase8(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 4 self.out_channels = 5 @@ -474,7 +494,6 @@ class TestFunctionalConv2DErrorCase8(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase9(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -488,7 +507,6 @@ class TestFunctionalConv2DErrorCase9(TestFunctionalConv2DError): class TestFunctionalConv2DErrorCase10(TestCase): - def setUp(self): self.input = np.array([]) self.filter = np.array([]) @@ -507,38 +525,45 @@ class TestFunctionalConv2DErrorCase10(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): x = fluid.data("input", self.input.shape, dtype=paddle.float32) - y = fluid.layers.conv2d(x, - self.num_filters, - self.filter_size, - stride=self.stride, - padding=self.padding, - dilation=self.dilation, - groups=self.groups, - param_attr=I.NumpyArrayInitializer( - self.filter), - bias_attr=False if self.bias is None - else I.NumpyArrayInitializer(self.bias), - act=None, - data_format=self.data_format) + y = fluid.layers.conv2d( + x, + self.num_filters, + self.filter_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + param_attr=I.NumpyArrayInitializer(self.filter), + bias_attr=False + if self.bias is None + else I.NumpyArrayInitializer(self.bias), + act=None, + data_format=self.data_format, + ) exe = fluid.Executor() exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv2d_transpose(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv2d_transpose( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_dygraph_exception(self): with self.assertRaises(ValueError): @@ -554,7 +579,6 @@ class TestFunctionalConv2DErrorCase10(TestCase): class TestFunctionalConv2DErrorCase11(TestFunctionalConv2DErrorCase10): - def setUp(self): self.input = np.random.randn(1, 3, 3, 3) self.filter = np.random.randn(3, 3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv3d.py b/python/paddle/fluid/tests/unittests/test_functional_conv3d.py index 9ecbf2bf46c..d2d7074a4b1 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv3d.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv3d.py @@ -41,27 +41,34 @@ class TestFunctionalConv3D(TestCase): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 3 + filter_shape = (self.filter_shape,) * 3 else: filter_shape = tuple(self.filter_shape) self.weight = np.random.uniform( - -1, 1, (self.out_channels, self.in_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.out_channels, self.in_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - self.channel_last = (self.data_format == "NDHWC") + self.channel_last = self.data_format == "NDHWC" if self.channel_last: - self.input_shape = (self.batch_size, ) + self.spatial_shape + ( - self.in_channels, ) + self.input_shape = ( + (self.batch_size,) + self.spatial_shape + (self.in_channels,) + ) else: - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) def static_graph_case_1(self): main = fluid.Program() @@ -69,11 +76,17 @@ class TestFunctionalConv3D(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = fluid.data("input", (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) y = fluid.layers.conv3d( x, self.out_channels, @@ -84,12 +97,14 @@ class TestFunctionalConv3D(TestCase): groups=self.groups, param_attr=I.NumpyArrayInitializer(self.weight), bias_attr=False - if self.no_bias else I.NumpyArrayInitializer(self.bias), + if self.no_bias + else I.NumpyArrayInitializer(self.bias), act=self.act, - data_format=self.data_format) + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def static_graph_case_2(self): @@ -98,25 +113,32 @@ class TestFunctionalConv3D(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = x = fluid.data("input", - (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight.shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight.shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias.shape, dtype=self.dtype) - y = F.conv3d(x, - weight, - None if self.no_bias else bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d( + x, + weight, + None if self.no_bias else bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) @@ -126,7 +148,7 @@ class TestFunctionalConv3D(TestCase): feed_dict = {"input": self.input, "weight": self.weight} if not self.no_bias: feed_dict["bias"] = self.bias - out, = exe.run(main, feed=feed_dict, fetch_list=[y]) + (out,) = exe.run(main, feed=feed_dict, fetch_list=[y]) return out def dygraph_case(self): @@ -134,14 +156,16 @@ class TestFunctionalConv3D(TestCase): x = dg.to_variable(self.input) weight = dg.to_variable(self.weight) bias = None if self.no_bias else dg.to_variable(self.bias) - y = F.conv3d(x, - weight, - bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d( + x, + weight, + bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) @@ -161,8 +185,9 @@ class TestFunctionalConv3D(TestCase): self.place = fluid.CPUPlace() self._test_identity() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu(self): self.place = fluid.CUDAPlace(0) self._test_identity() @@ -192,12 +217,14 @@ class TestFunctionalConv3DError(TestCase): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 3 + filter_shape = (self.filter_shape,) * 3 else: filter_shape = tuple(self.filter_shape) - self.weight_shape = (self.out_channels, - self.in_channels // self.groups) + filter_shape - self.bias_shape = (self.out_channels, ) + self.weight_shape = ( + self.out_channels, + self.in_channels // self.groups, + ) + filter_shape + self.bias_shape = (self.out_channels,) def static_graph_case(self): main = fluid.Program() @@ -206,32 +233,38 @@ class TestFunctionalConv3DError(TestCase): with fluid.program_guard(main, start): self.channel_last = self.data_format == "NDHWC" if self.channel_last: - x = x = fluid.data("input", - (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias_shape, dtype=self.dtype) - y = F.conv3d(x, - weight, - None if self.no_bias else bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d( + x, + weight, + None if self.no_bias else bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) class TestFunctionalConv3DCase2(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -246,7 +279,6 @@ class TestFunctionalConv3DCase2(TestFunctionalConv3D): class TestFunctionalConv3DCase3(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -261,7 +293,6 @@ class TestFunctionalConv3DCase3(TestFunctionalConv3D): class TestFunctionalConv3DCase4(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -276,7 +307,6 @@ class TestFunctionalConv3DCase4(TestFunctionalConv3D): class TestFunctionalConv3DCase5(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -291,7 +321,6 @@ class TestFunctionalConv3DCase5(TestFunctionalConv3D): class TestFunctionalConv3DCase6(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -306,7 +335,6 @@ class TestFunctionalConv3DCase6(TestFunctionalConv3D): class TestFunctionalConv3DCase7(TestFunctionalConv3D): - def setUp(self): self.in_channels = 6 self.out_channels = 8 @@ -321,7 +349,6 @@ class TestFunctionalConv3DCase7(TestFunctionalConv3D): class TestFunctionalConv3DCase8(TestFunctionalConv3D): - def setUp(self): self.in_channels = 6 self.out_channels = 12 @@ -337,7 +364,6 @@ class TestFunctionalConv3DCase8(TestFunctionalConv3D): class TestFunctionalConv3DErrorCase2(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -352,7 +378,6 @@ class TestFunctionalConv3DErrorCase2(TestFunctionalConv3DError): class TestFunctionalConv3DErrorCase3(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -367,7 +392,6 @@ class TestFunctionalConv3DErrorCase3(TestFunctionalConv3DError): class TestFunctionalConv3DErrorCase4(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 4 self.out_channels = 3 @@ -382,7 +406,6 @@ class TestFunctionalConv3DErrorCase4(TestFunctionalConv3DError): class TestFunctionalConv3DErrorCase7(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -397,7 +420,6 @@ class TestFunctionalConv3DErrorCase7(TestFunctionalConv3DError): class TestFunctionalConv3DErrorCase8(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -412,7 +434,6 @@ class TestFunctionalConv3DErrorCase8(TestFunctionalConv3DError): class TestFunctionalConv3DErrorCase9(TestFunctionalConv3DError): - def setUp(self): self.in_channels = -5 self.out_channels = 5 @@ -427,7 +448,6 @@ class TestFunctionalConv3DErrorCase9(TestFunctionalConv3DError): class TestFunctionalConv3DErrorCase10(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -442,7 +462,6 @@ class TestFunctionalConv3DErrorCase10(TestFunctionalConv3DError): class TestFunctionalConv3DErrorCase11(TestCase): - def setUp(self): self.input = np.array([]) self.filter = np.array([]) @@ -461,38 +480,45 @@ class TestFunctionalConv3DErrorCase11(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): x = fluid.data("input", self.input.shape, dtype=paddle.float32) - y = fluid.layers.conv3d(x, - self.num_filters, - self.filter_size, - stride=self.stride, - padding=self.padding, - dilation=self.dilation, - groups=self.groups, - param_attr=I.NumpyArrayInitializer( - self.filter), - bias_attr=False if self.bias is None - else I.NumpyArrayInitializer(self.bias), - act=None, - data_format=self.data_format) + y = fluid.layers.conv3d( + x, + self.num_filters, + self.filter_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + param_attr=I.NumpyArrayInitializer(self.filter), + bias_attr=False + if self.bias is None + else I.NumpyArrayInitializer(self.bias), + act=None, + data_format=self.data_format, + ) exe = fluid.Executor() exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv3d(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv3d( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_dygraph_exception(self): with self.assertRaises(ValueError): @@ -504,7 +530,6 @@ class TestFunctionalConv3DErrorCase11(TestCase): class TestFunctionalConv3DErrorCase12(TestFunctionalConv3DErrorCase11): - def setUp(self): self.input = np.random.randn(1, 3, 3, 3, 3) self.filter = np.random.randn(3, 3, 1, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv3d_transpose.py b/python/paddle/fluid/tests/unittests/test_functional_conv3d_transpose.py index 0190779a021..d9ee0495359 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv3d_transpose.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv3d_transpose.py @@ -43,27 +43,34 @@ class TestFunctionalConv3DTranspose(TestCase): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 3 + filter_shape = (self.filter_shape,) * 3 else: filter_shape = tuple(self.filter_shape) self.weight = np.random.uniform( - -1, 1, (self.in_channels, self.out_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.in_channels, self.out_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - self.channel_last = (self.data_format == "NDHWC") + self.channel_last = self.data_format == "NDHWC" if self.channel_last: - self.input_shape = (self.batch_size, ) + self.spatial_shape + ( - self.in_channels, ) + self.input_shape = ( + (self.batch_size,) + self.spatial_shape + (self.in_channels,) + ) else: - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) def static_graph_case_1(self): main = fluid.Program() @@ -71,11 +78,17 @@ class TestFunctionalConv3DTranspose(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = fluid.data("input", (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) y = fluid.layers.conv3d_transpose( x, self.out_channels, @@ -87,12 +100,14 @@ class TestFunctionalConv3DTranspose(TestCase): groups=self.groups, param_attr=I.NumpyArrayInitializer(self.weight), bias_attr=False - if self.no_bias else I.NumpyArrayInitializer(self.bias), + if self.no_bias + else I.NumpyArrayInitializer(self.bias), act=self.act, - data_format=self.data_format) + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def static_graph_case_2(self): @@ -101,26 +116,33 @@ class TestFunctionalConv3DTranspose(TestCase): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = x = fluid.data("input", - (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight.shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight.shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias.shape, dtype=self.dtype) - y = F.conv3d_transpose(x, - weight, - None if self.no_bias else bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d_transpose( + x, + weight, + None if self.no_bias else bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) exe = fluid.Executor(self.place) @@ -128,7 +150,7 @@ class TestFunctionalConv3DTranspose(TestCase): feed_dict = {"input": self.input, "weight": self.weight} if not self.no_bias: feed_dict["bias"] = self.bias - out, = exe.run(main, feed=feed_dict, fetch_list=[y]) + (out,) = exe.run(main, feed=feed_dict, fetch_list=[y]) return out def dygraph_case(self): @@ -136,15 +158,17 @@ class TestFunctionalConv3DTranspose(TestCase): x = dg.to_variable(self.input) weight = dg.to_variable(self.weight) bias = None if self.no_bias else dg.to_variable(self.bias) - y = F.conv3d_transpose(x, - weight, - bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d_transpose( + x, + weight, + bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) out = y.numpy() @@ -166,14 +190,16 @@ class TestFunctionalConv3DTranspose(TestCase): with _test_eager_guard(): self.test_identity_cpu() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu(self): self.place = fluid.CUDAPlace(0) self._test_identity() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu_check_eager(self): with _test_eager_guard(): self.test_identity_gpu() @@ -204,12 +230,14 @@ class TestFunctionalConv3DTransposeError(TestCase): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 3 + filter_shape = (self.filter_shape,) * 3 else: filter_shape = tuple(self.filter_shape) - self.weight_shape = (self.in_channels, - self.out_channels // self.groups) + filter_shape - self.bias_shape = (self.out_channels, ) + self.weight_shape = ( + self.in_channels, + self.out_channels // self.groups, + ) + filter_shape + self.bias_shape = (self.out_channels,) def static_graph_case(self): main = fluid.Program() @@ -218,32 +246,38 @@ class TestFunctionalConv3DTransposeError(TestCase): with fluid.program_guard(main, start): self.channel_last = self.data_format == "NDHWC" if self.channel_last: - x = x = fluid.data("input", - (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias_shape, dtype=self.dtype) - y = F.conv3d_transpose(x, - weight, - None if self.no_bias else bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d_transpose( + x, + weight, + None if self.no_bias else bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) class TestFunctionalConv3DTransposeCase2(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -258,7 +292,6 @@ class TestFunctionalConv3DTransposeCase2(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase3(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -273,7 +306,6 @@ class TestFunctionalConv3DTransposeCase3(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase4(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -288,7 +320,6 @@ class TestFunctionalConv3DTransposeCase4(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase5(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -303,7 +334,6 @@ class TestFunctionalConv3DTransposeCase5(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase6(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 4 @@ -318,7 +348,6 @@ class TestFunctionalConv3DTransposeCase6(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase7(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 4 @@ -334,7 +363,6 @@ class TestFunctionalConv3DTransposeCase7(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase8(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -349,7 +377,6 @@ class TestFunctionalConv3DTransposeCase8(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase9(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -364,7 +391,6 @@ class TestFunctionalConv3DTransposeCase9(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase10(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -379,7 +405,6 @@ class TestFunctionalConv3DTransposeCase10(TestFunctionalConv3DTranspose): class TestFunctionalConv3DTransposeCase11(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -393,9 +418,9 @@ class TestFunctionalConv3DTransposeCase11(TestFunctionalConv3DTranspose): self.data_format = "NCDHW" -class TestFunctionalConv3DTransposeErrorCase2(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase2( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -409,9 +434,9 @@ class TestFunctionalConv3DTransposeErrorCase2(TestFunctionalConv3DTransposeError self.data_format = "NDHWC" -class TestFunctionalConv3DTransposeErrorCase3(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase3( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -425,9 +450,9 @@ class TestFunctionalConv3DTransposeErrorCase3(TestFunctionalConv3DTransposeError self.data_format = "NDHWC" -class TestFunctionalConv3DTransposeErrorCase4(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase4( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -441,9 +466,9 @@ class TestFunctionalConv3DTransposeErrorCase4(TestFunctionalConv3DTransposeError self.data_format = "NCDHW" -class TestFunctionalConv3DTransposeErrorCase5(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase5( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = -2 self.out_channels = 5 @@ -457,9 +482,9 @@ class TestFunctionalConv3DTransposeErrorCase5(TestFunctionalConv3DTransposeError self.data_format = "NCDHW" -class TestFunctionalConv3DTransposeErrorCase7(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase7( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 4 self.out_channels = 5 @@ -474,9 +499,9 @@ class TestFunctionalConv3DTransposeErrorCase7(TestFunctionalConv3DTransposeError self.data_format = "NCDHW" -class TestFunctionalConv3DTransposeErrorCase8(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase8( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 4 self.out_channels = 5 @@ -490,9 +515,9 @@ class TestFunctionalConv3DTransposeErrorCase8(TestFunctionalConv3DTransposeError self.data_format = "not_valid" -class TestFunctionalConv3DTransposeErrorCase9(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase9( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -507,7 +532,6 @@ class TestFunctionalConv3DTransposeErrorCase9(TestFunctionalConv3DTransposeError class TestFunctionalConv3DTransposeErrorCase10(TestCase): - def setUp(self): self.input = np.array([]) self.filter = np.array([]) @@ -535,29 +559,36 @@ class TestFunctionalConv3DTransposeErrorCase10(TestCase): dilation=self.dilation, groups=self.groups, param_attr=I.NumpyArrayInitializer(self.filter), - bias_attr=False if self.bias is None else - I.NumpyArrayInitializer(self.bias), + bias_attr=False + if self.bias is None + else I.NumpyArrayInitializer(self.bias), act=None, - data_format=self.data_format) + data_format=self.data_format, + ) exe = fluid.Executor() exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv3d_transpose(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv3d_transpose( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_dygraph_exception(self): with self.assertRaises(ValueError): @@ -573,8 +604,8 @@ class TestFunctionalConv3DTransposeErrorCase10(TestCase): class TestFunctionalConv3DTransposeErrorCase11( - TestFunctionalConv3DTransposeErrorCase10): - + TestFunctionalConv3DTransposeErrorCase10 +): def setUp(self): self.input = np.random.randn(1, 3, 3, 3, 3) self.filter = np.random.randn(3, 3, 1, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py index 67729d6633d..9431bcac4e3 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py @@ -26,18 +26,19 @@ paddle.enable_static() class TestFuseAllReduceOpsBase(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - def compare_fuse_all_reduce_ops(self, - model, - use_device, - init_feed_dict=None, - get_data_from_feeder=None, - optimizer=None, - fuse_all_optimizer_ops=False): + def compare_fuse_all_reduce_ops( + self, + model, + use_device, + init_feed_dict=None, + get_data_from_feeder=None, + optimizer=None, + fuse_all_optimizer_ops=False, + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return if use_device == DeviceType.XPU and not core.is_compiled_with_xpu(): @@ -48,22 +49,32 @@ class TestFuseAllReduceOpsBase(TestParallelExecutorBase): img, label = init_feed_dict() feed_dict_data = {"image": img, "label": label} - not_fuse_op_first_loss, not_fuse_op_last_loss, _ = self.check_network_convergence( + ( + not_fuse_op_first_loss, + not_fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, feed_dict=feed_dict_data, get_data_from_feeder=get_data_from_feeder, use_device=use_device, fuse_all_reduce_ops=False, fuse_all_optimizer_ops=fuse_all_optimizer_ops, - optimizer=optimizer) - fuse_op_first_loss, fuse_op_last_loss, _ = self.check_network_convergence( + optimizer=optimizer, + ) + ( + fuse_op_first_loss, + fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, feed_dict=feed_dict_data, get_data_from_feeder=get_data_from_feeder, use_device=use_device, fuse_all_reduce_ops=True, fuse_all_optimizer_ops=fuse_all_optimizer_ops, - optimizer=optimizer) + optimizer=optimizer, + ) for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) @@ -73,18 +84,20 @@ class TestFuseAllReduceOpsBase(TestParallelExecutorBase): def optimizer(self, learning_rate=1e-3): optimizer = fluid.optimizer.SGD( learning_rate=learning_rate, - regularization=fluid.regularizer.L2Decay(1e-3)) + regularization=fluid.regularizer.L2Decay(1e-3), + ) return optimizer class TestFuseAllReduceOps(TestFuseAllReduceOpsBase): - def _decorate_compare_fused_all_reduce(self, model, use_device): - self.compare_fuse_all_reduce_ops(model, - use_device, - init_feed_dict=init_data, - optimizer=self.optimizer, - fuse_all_optimizer_ops=True) + self.compare_fuse_all_reduce_ops( + model, + use_device, + init_feed_dict=init_data, + optimizer=self.optimizer, + fuse_all_optimizer_ops=True, + ) def test_simple_fc_with_fuse_all_reduce(self): self._decorate_compare_fused_all_reduce(simple_fc_net, DeviceType.CUDA) @@ -92,27 +105,29 @@ class TestFuseAllReduceOps(TestFuseAllReduceOpsBase): self._decorate_compare_fused_all_reduce(simple_fc_net, DeviceType.CPU) def test_batchnorm_fc_with_fuse_all_reduce(self): - self._decorate_compare_fused_all_reduce(fc_with_batchnorm, - DeviceType.CUDA) + self._decorate_compare_fused_all_reduce( + fc_with_batchnorm, DeviceType.CUDA + ) # TODO(wangxi): xpu batch_norm op only support dim = 4 # self._decorate_compare_fused_all_reduce(fc_with_batchnorm, # DeviceType.XPU) - self._decorate_compare_fused_all_reduce(fc_with_batchnorm, - DeviceType.CPU) + self._decorate_compare_fused_all_reduce( + fc_with_batchnorm, DeviceType.CPU + ) class TestFuseAllReduceOpsAndOptiOps(TestFuseAllReduceOps): - def _decorate_compare_fused_all_reduce(self, model, use_device): - self.compare_fuse_all_reduce_ops(model, - use_device, - init_feed_dict=init_data, - optimizer=self.optimizer, - fuse_all_optimizer_ops=True) + self.compare_fuse_all_reduce_ops( + model, + use_device, + init_feed_dict=init_data, + optimizer=self.optimizer, + fuse_all_optimizer_ops=True, + ) class TestFuseAllReduceOpsWithSparseGrad(TestFuseAllReduceOpsBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -132,7 +147,8 @@ class TestFuseAllReduceOpsWithSparseGrad(TestFuseAllReduceOpsBase): model, use_device, get_data_from_feeder=self.get_data_from_feeder, - optimizer=self.optimizer) + optimizer=self.optimizer, + ) def test_simple_bow_net_with_fuse_all_reduce(self): model = partial(bow_net, dict_dim=self.word_dict_len, is_sparse=True) diff --git a/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py index d3a18ad28ca..5018b7bd5e4 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py @@ -18,41 +18,47 @@ import unittest class TestFuseBatchNormActPass(unittest.TestCase): - def build_program(self, main_program, startup_program, use_cuda, seed=1): with fluid.program_guard(main_program, startup_program): x = fluid.layers.data(name='x', shape=[1, 28, 28], dtype='float32') y = fluid.layers.data(name="y", shape=[1], dtype='int64') - hidden1 = fluid.layers.conv2d(input=x, - filter_size=3, - num_filters=16, - stride=1, - padding=1, - act=None, - bias_attr=False, - data_format='NHWC') + hidden1 = fluid.layers.conv2d( + input=x, + filter_size=3, + num_filters=16, + stride=1, + padding=1, + act=None, + bias_attr=False, + data_format='NHWC', + ) param_attr = fluid.ParamAttr( name='batch_norm_w', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) bias_attr = fluid.ParamAttr( name='batch_norm_b', - initializer=fluid.initializer.Constant(value=0.0)) - hidden2 = fluid.layers.batch_norm(input=hidden1, - param_attr=param_attr, - bias_attr=bias_attr, - act='relu', - data_layout='NHWC') + initializer=fluid.initializer.Constant(value=0.0), + ) + hidden2 = fluid.layers.batch_norm( + input=hidden1, + param_attr=param_attr, + bias_attr=bias_attr, + act='relu', + data_layout='NHWC', + ) hidden3 = fluid.layers.fc(input=hidden2, size=32, act='relu') - hidden4 = fluid.layers.batch_norm(input=hidden3, - act='relu', - data_layout='NHWC') + hidden4 = fluid.layers.batch_norm( + input=hidden3, act='relu', data_layout='NHWC' + ) prediction = fluid.layers.fc(input=hidden4, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=y) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(learning_rate=0.001) if use_cuda: sgd = fluid.contrib.mixed_precision.decorate( - sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0) + sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0 + ) sgd.minimize(loss) return x, y, loss @@ -71,36 +77,40 @@ class TestFuseBatchNormActPass(unittest.TestCase): build_strategy = fluid.BuildStrategy() build_strategy.fuse_bn_act_ops = False binary = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) + loss_name=loss.name, build_strategy=build_strategy + ) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) loss_vals = [] scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) loss_vals.append(loss_v[0][0]) # open fused_bn_act_ops build_strategy_fused = fluid.BuildStrategy() build_strategy_fused.fuse_bn_act_ops = True binary_fused = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy_fused) - train_reader_fused = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) + loss_name=loss.name, build_strategy=build_strategy_fused + ) + train_reader_fused = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) loss_vals_fused = [] scope_fused = fluid.Scope() with fluid.scope_guard(scope_fused): exe.run(startup_program) for _ in range(iters): data = next(train_reader_fused()) - loss_v = exe.run(binary_fused, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary_fused, feed=feeder.feed(data), fetch_list=[loss] + ) loss_vals_fused.append(loss_v[0][0]) # check loss diff --git a/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py index 67c4879a05a..47c2a1a5f16 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py @@ -21,130 +21,151 @@ from paddle.fluid import core paddle.enable_static() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Paddle core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Paddle core is not compiled with CUDA" +) class TestFusedBnAddActAPI(unittest.TestCase): - def setUp(self): self.conv_param_attr1 = fluid.ParamAttr( name='conv2d_1.weight', initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) self.conv_param_attr2 = fluid.ParamAttr( name='conv2d_2.weight', initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) self.bn_param_attr1 = fluid.ParamAttr( name='batch_norm_w_1', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) self.bn_bias_attr1 = fluid.ParamAttr( name='batch_norm_b_1', - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0), + ) self.bn_param_attr2 = fluid.ParamAttr( name='batch_norm_w_2', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) self.bn_bias_attr2 = fluid.ParamAttr( name='batch_norm_b_2', - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0), + ) self.fc_param_attr = fluid.ParamAttr( name='fc.weight', - initializer=fluid.initializer.Xavier(uniform=False)) + initializer=fluid.initializer.Xavier(uniform=False), + ) - def build_fused_program(self, - main_program, - startup_program, - use_cuda, - seed=1): + def build_fused_program( + self, main_program, startup_program, use_cuda, seed=1 + ): with fluid.program_guard(main_program, startup_program): x = fluid.layers.data(name='x', shape=[1, 28, 28], dtype='float32') y = fluid.layers.data(name="y", shape=[1], dtype='int64') - conv1_1 = fluid.layers.conv2d(input=x, - filter_size=3, - num_filters=32, - stride=1, - padding=1, - act=None, - param_attr=self.conv_param_attr1, - bias_attr=False, - data_format='NHWC') - conv1_2 = fluid.layers.conv2d(input=x, - filter_size=3, - num_filters=32, - stride=1, - padding=1, - act=None, - param_attr=self.conv_param_attr2, - bias_attr=False, - data_format='NHWC') - bn = fluid.layers.batch_norm(input=conv1_1, - param_attr=self.bn_param_attr1, - bias_attr=self.bn_bias_attr1, - act=None, - data_layout='NHWC') + conv1_1 = fluid.layers.conv2d( + input=x, + filter_size=3, + num_filters=32, + stride=1, + padding=1, + act=None, + param_attr=self.conv_param_attr1, + bias_attr=False, + data_format='NHWC', + ) + conv1_2 = fluid.layers.conv2d( + input=x, + filter_size=3, + num_filters=32, + stride=1, + padding=1, + act=None, + param_attr=self.conv_param_attr2, + bias_attr=False, + data_format='NHWC', + ) + bn = fluid.layers.batch_norm( + input=conv1_1, + param_attr=self.bn_param_attr1, + bias_attr=self.bn_bias_attr1, + act=None, + data_layout='NHWC', + ) fused_bn_add_act = fluid.contrib.layers.fused_bn_add_act( conv1_2, bn, param_attr=self.bn_param_attr2, - bias_attr=self.bn_bias_attr2) - prediction = fluid.layers.fc(input=fused_bn_add_act, - size=10, - act='softmax', - param_attr=self.fc_param_attr) + bias_attr=self.bn_bias_attr2, + ) + prediction = fluid.layers.fc( + input=fused_bn_add_act, + size=10, + act='softmax', + param_attr=self.fc_param_attr, + ) loss = fluid.layers.cross_entropy(input=prediction, label=y) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(learning_rate=0.001) sgd = fluid.contrib.mixed_precision.decorate( - sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0) + sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0 + ) sgd.minimize(loss) return x, y, loss - def build_origin_program(self, - main_program, - startup_program, - use_cuda, - seed=1): + def build_origin_program( + self, main_program, startup_program, use_cuda, seed=1 + ): with fluid.program_guard(main_program, startup_program): x = fluid.layers.data(name='x', shape=[1, 28, 28], dtype='float32') y = fluid.layers.data(name="y", shape=[1], dtype='int64') - conv1_1 = fluid.layers.conv2d(input=x, - filter_size=3, - num_filters=32, - stride=1, - padding=1, - act=None, - param_attr=self.conv_param_attr1, - bias_attr=False, - data_format='NHWC') - bn1 = fluid.layers.batch_norm(input=conv1_1, - param_attr=self.bn_param_attr1, - bias_attr=self.bn_bias_attr1, - act=None, - data_layout='NHWC') - conv1_2 = fluid.layers.conv2d(input=conv1_1, - filter_size=1, - num_filters=32, - stride=1, - act=None, - param_attr=self.conv_param_attr2, - bias_attr=False, - data_format='NHWC') - bn2 = fluid.layers.batch_norm(input=conv1_1, - param_attr=self.bn_param_attr2, - bias_attr=self.bn_bias_attr2, - act=None, - data_layout='NHWC') + conv1_1 = fluid.layers.conv2d( + input=x, + filter_size=3, + num_filters=32, + stride=1, + padding=1, + act=None, + param_attr=self.conv_param_attr1, + bias_attr=False, + data_format='NHWC', + ) + bn1 = fluid.layers.batch_norm( + input=conv1_1, + param_attr=self.bn_param_attr1, + bias_attr=self.bn_bias_attr1, + act=None, + data_layout='NHWC', + ) + conv1_2 = fluid.layers.conv2d( + input=conv1_1, + filter_size=1, + num_filters=32, + stride=1, + act=None, + param_attr=self.conv_param_attr2, + bias_attr=False, + data_format='NHWC', + ) + bn2 = fluid.layers.batch_norm( + input=conv1_1, + param_attr=self.bn_param_attr2, + bias_attr=self.bn_bias_attr2, + act=None, + data_layout='NHWC', + ) out = bn1 + bn2 out = fluid.layers.relu(out) - prediction = fluid.layers.fc(input=out, - size=10, - act='softmax', - param_attr=self.fc_param_attr) + prediction = fluid.layers.fc( + input=out, size=10, act='softmax', param_attr=self.fc_param_attr + ) loss = fluid.layers.cross_entropy(input=prediction, label=y) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(learning_rate=0.001) sgd = fluid.contrib.mixed_precision.decorate( - sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0) + sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0 + ) sgd.minimize(loss) return loss @@ -158,12 +179,14 @@ class TestFusedBnAddActAPI(unittest.TestCase): # build_fused_program: turn on fuse_bn_add_act_ops main_program = fluid.Program() startup_program = fluid.Program() - loss = self.build_origin_program(main_program, startup_program, - use_cuda) + loss = self.build_origin_program( + main_program, startup_program, use_cuda + ) build_strategy_fused = fluid.BuildStrategy() build_strategy_fused.fuse_bn_add_act_ops = True binary_fused = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy_fused) + loss_name=loss.name, build_strategy=build_strategy_fused + ) exe = fluid.Executor(place) loss_vals_fused = [] x_data = [] @@ -176,30 +199,27 @@ class TestFusedBnAddActAPI(unittest.TestCase): y = np.random.random((batch_size, 1)).astype("int64") x_data.append(x) y_data.append(y) - loss_v = exe.run(binary_fused, - feed={ - "x": x, - "y": y - }, - fetch_list=[loss]) + loss_v = exe.run( + binary_fused, feed={"x": x, "y": y}, fetch_list=[loss] + ) loss_vals_fused.append(loss_v[0][0]) # build_origin_program: turn off fused_bn_act_ops build_strategy = fluid.BuildStrategy() build_strategy.fuse_bn_add_act_ops = False binary = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy_fused) + loss_name=loss.name, build_strategy=build_strategy_fused + ) loss_vals = [] scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) for i in range(iters): - loss_v = exe.run(binary, - feed={ - "x": x_data[i], - "y": y_data[i] - }, - fetch_list=[loss]) + loss_v = exe.run( + binary, + feed={"x": x_data[i], "y": y_data[i]}, + fetch_list=[loss], + ) loss_vals.append(loss_v[0][0]) # check loss @@ -215,9 +235,9 @@ class TestFusedBnAddActAPI(unittest.TestCase): main_program = fluid.Program() startup_program = fluid.Program() place = fluid.CUDAPlace(0) - x, y, loss = self.build_fused_program(main_program, - startup_program, - use_cuda=True) + x, y, loss = self.build_fused_program( + main_program, startup_program, use_cuda=True + ) exe = fluid.Executor(place) scope = fluid.Scope() with fluid.scope_guard(scope): @@ -225,12 +245,9 @@ class TestFusedBnAddActAPI(unittest.TestCase): for _ in range(5): x = np.random.random((4, 1, 28, 28)).astype("float32") y = np.random.random((4, 1)).astype("int64") - loss_v = exe.run(main_program, - feed={ - "x": x, - "y": y - }, - fetch_list=[loss]) + loss_v = exe.run( + main_program, feed={"x": x, "y": y}, fetch_list=[loss] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py index 15b79bf0a7f..bc052ec1cf7 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py @@ -21,7 +21,6 @@ import os class TestMNIST(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -34,7 +33,8 @@ class TestMNIST(TestParallelExecutorBase): def _optimizer(learning_rate=1e-6): optimizer = fluid.optimizer.SGD( learning_rate=learning_rate, - regularization=fluid.regularizer.L2Decay(1e-6)) + regularization=fluid.regularizer.L2Decay(1e-6), + ) return optimizer # NOTE(dzh): @@ -42,28 +42,32 @@ class TestMNIST(TestParallelExecutorBase): # FIXME (liuwei12) # the new memory optimize strategy will crash this unittest # add enable_inplace=False here to force pass the unittest - not_fuse_op_first_loss, not_fuse_op_last_loss, _ = self.check_network_convergence( + ( + not_fuse_op_first_loss, + not_fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, fuse_elewise_add_act_ops=False, use_ir_memory_optimize=False, enable_inplace=False, - optimizer=_optimizer) - fuse_op_first_loss, fuse_op_last_loss, _ = self.check_network_convergence( + optimizer=_optimizer, + ) + ( + fuse_op_first_loss, + fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, fuse_elewise_add_act_ops=True, use_ir_memory_optimize=False, enable_inplace=False, - optimizer=_optimizer) + optimizer=_optimizer, + ) for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) @@ -75,13 +79,16 @@ class TestMNIST(TestParallelExecutorBase): self._compare_fuse_elewise_add_act_ops(simple_fc_net, DeviceType.CPU) def test_batchnorm_fc_with_fuse_op(self): - self._compare_fuse_elewise_add_act_ops(fc_with_batchnorm, - DeviceType.CUDA) - self._compare_fuse_elewise_add_act_ops(fc_with_batchnorm, - DeviceType.CPU) + self._compare_fuse_elewise_add_act_ops( + fc_with_batchnorm, DeviceType.CUDA + ) + self._compare_fuse_elewise_add_act_ops( + fc_with_batchnorm, DeviceType.CPU + ) if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py index 1d7dc0e70d4..3d95b77bae6 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py @@ -45,7 +45,6 @@ def verify_node_count(graph, node_name, target_count): class MultiFCLayer(paddle.nn.Layer): - def __init__(self, hidden, Activation): super(MultiFCLayer, self).__init__() self.linear1 = paddle.nn.Linear(hidden, 4 * hidden) @@ -72,10 +71,10 @@ class MultiFCLayer(paddle.nn.Layer): return output -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueFWDBase(unittest.TestCase): - def setUp(self): self.batch = 64 self.seqlen = 128 @@ -87,29 +86,42 @@ class TestFuseGemmEpilogueFWDBase(unittest.TestCase): self.startup_prog = paddle.static.Program() with paddle.static.program_guard(self.main_prog, self.startup_prog): - data = paddle.static.data(name="_data", - shape=[-1, self.seqlen, self.hidden], - dtype='float32') - matmul_y = paddle.static.data(name="_matmul_y", - shape=[1, self.hidden, self.hidden], - dtype='float32') - ele_y = paddle.static.data(name="_ele_y", - shape=[ - self.hidden, - ], - dtype='float32') + data = paddle.static.data( + name="_data", + shape=[-1, self.seqlen, self.hidden], + dtype='float32', + ) + matmul_y = paddle.static.data( + name="_matmul_y", + shape=[1, self.hidden, self.hidden], + dtype='float32', + ) + ele_y = paddle.static.data( + name="_ele_y", + shape=[ + self.hidden, + ], + dtype='float32', + ) multi_layer = MultiFCLayer(self.hidden, self._get_act_type()[0]) with paddle.static.amp.fp16_guard(): out = multi_layer(data, matmul_y, ele_y) self.loss = paddle.mean(out) - self.data_arr = np.random.random( - (self.batch, self.seqlen, self.hidden)).astype("float32") - 0.5 - self.matmul_y_arr = np.random.random( - (1, self.hidden, self.hidden)).astype("float32") - 0.5 - self.ele_y_arr = np.random.random( - (self.hidden, )).astype("float32") - 0.5 + self.data_arr = ( + np.random.random((self.batch, self.seqlen, self.hidden)).astype( + "float32" + ) + - 0.5 + ) + self.matmul_y_arr = ( + np.random.random((1, self.hidden, self.hidden)).astype("float32") + - 0.5 + ) + self.ele_y_arr = ( + np.random.random((self.hidden,)).astype("float32") - 0.5 + ) self.place = paddle.CUDAPlace(0) self.exe = paddle.static.Executor(self.place) @@ -120,37 +132,45 @@ class TestFuseGemmEpilogueFWDBase(unittest.TestCase): self.feed = { "_data": self.data_arr, "_matmul_y": self.matmul_y_arr, - "_ele_y": self.ele_y_arr + "_ele_y": self.ele_y_arr, } - self.reference = self.exe.run(self.main_prog, - feed=self.feed, - fetch_list=[self.loss.name]) + self.reference = self.exe.run( + self.main_prog, feed=self.feed, fetch_list=[self.loss.name] + ) - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def _test_output(self): build_strategy = paddle.static.BuildStrategy() build_strategy.fuse_gemm_epilogue = True program = paddle.static.CompiledProgram(self.main_prog) - program = program.with_data_parallel(loss_name=self.loss.name, - build_strategy=build_strategy, - places=paddle.static.cuda_places()) - - result = self.exe.run(program, - feed=self.feed, - fetch_list=[self.loss.name]) + program = program.with_data_parallel( + loss_name=self.loss.name, + build_strategy=build_strategy, + places=paddle.static.cuda_places(), + ) + + result = self.exe.run( + program, feed=self.feed, fetch_list=[self.loss.name] + ) self.assertTrue( compare(self.reference, result, self.atol, self.rtol), - "[{}] outputs are miss-matched.".format(type(self).__name__)) + "[{}] outputs are miss-matched.".format(type(self).__name__), + ) self.assertTrue( verify_node_count(program._graph, "fused_gemm_epilogue", 3), - "[{}] The number of fused_gemm_epilogue is miss-matched in the computing graph." - .format(type(self).__name__)) + "[{}] The number of fused_gemm_epilogue is miss-matched in the computing graph.".format( + type(self).__name__ + ), + ) act_fwd_name = self._get_act_type()[1] self.assertTrue( verify_node_count(program._graph, act_fwd_name, 1), - "[{}] The number of {} is miss-matched in the computing graph.". - format(type(self).__name__, act_fwd_name)) + "[{}] The number of {} is miss-matched in the computing graph.".format( + type(self).__name__, act_fwd_name + ), + ) def _pre_test_hooks(self): self.atol = 1e-4 @@ -160,10 +180,10 @@ class TestFuseGemmEpilogueFWDBase(unittest.TestCase): return paddle.nn.ReLU, "relu" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueReluFWDFP32(TestFuseGemmEpilogueFWDBase): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 @@ -175,27 +195,28 @@ class TestFuseGemmEpilogueReluFWDFP32(TestFuseGemmEpilogueFWDBase): self._test_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueReluFWDFP16(TestFuseGemmEpilogueReluFWDFP32): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 fp16_var_list = paddle.static.amp.cast_model_to_fp16(self.main_prog) paddle.static.amp.cast_parameters_to_fp16( - self.place, self.main_prog, to_fp16_var_names=fp16_var_list) + self.place, self.main_prog, to_fp16_var_names=fp16_var_list + ) self.data_arr = self.data_arr.astype("float16") self.matmul_y_arr = self.matmul_y_arr.astype("float16") self.ele_y_arr = self.ele_y_arr.astype("float16") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGeluFWDFP32(TestFuseGemmEpilogueFWDBase): - def _pre_test_hooks(self): self.atol = 1e-4 self.rtol = 1e-3 @@ -207,27 +228,28 @@ class TestFuseGemmEpilogueGeluFWDFP32(TestFuseGemmEpilogueFWDBase): self._test_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGeluFWDFP16(TestFuseGemmEpilogueGeluFWDFP32): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 fp16_var_list = paddle.static.amp.cast_model_to_fp16(self.main_prog) paddle.static.amp.cast_parameters_to_fp16( - self.place, self.main_prog, to_fp16_var_names=fp16_var_list) + self.place, self.main_prog, to_fp16_var_names=fp16_var_list + ) self.data_arr = self.data_arr.astype("float16") self.matmul_y_arr = self.matmul_y_arr.astype("float16") self.ele_y_arr = self.ele_y_arr.astype("float16") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueBWDBase(unittest.TestCase): - def setUp(self): self.batch = 64 self.seqlen = 128 @@ -239,17 +261,23 @@ class TestFuseGemmEpilogueBWDBase(unittest.TestCase): self.startup_prog = paddle.static.Program() with paddle.static.program_guard(self.main_prog, self.startup_prog): - data = paddle.static.data(name="_data", - shape=[-1, self.seqlen, self.hidden], - dtype='float32') - matmul_y = paddle.static.data(name="_matmul_y", - shape=[1, self.hidden, self.hidden], - dtype='float32') - ele_y = paddle.static.data(name="_ele_y", - shape=[ - self.hidden, - ], - dtype='float32') + data = paddle.static.data( + name="_data", + shape=[-1, self.seqlen, self.hidden], + dtype='float32', + ) + matmul_y = paddle.static.data( + name="_matmul_y", + shape=[1, self.hidden, self.hidden], + dtype='float32', + ) + ele_y = paddle.static.data( + name="_ele_y", + shape=[ + self.hidden, + ], + dtype='float32', + ) multi_layer = MultiFCLayer(self.hidden, self._get_act_type()[0]) with paddle.static.amp.fp16_guard(): @@ -257,12 +285,19 @@ class TestFuseGemmEpilogueBWDBase(unittest.TestCase): self.loss = paddle.mean(out) paddle.static.append_backward(loss=self.loss) - self.data_arr = np.random.random( - (self.batch, self.seqlen, self.hidden)).astype("float32") - 0.5 - self.matmul_y_arr = np.random.random( - (1, self.hidden, self.hidden)).astype("float32") - 0.5 - self.ele_y_arr = np.random.random( - (self.hidden, )).astype("float32") - 0.5 + self.data_arr = ( + np.random.random((self.batch, self.seqlen, self.hidden)).astype( + "float32" + ) + - 0.5 + ) + self.matmul_y_arr = ( + np.random.random((1, self.hidden, self.hidden)).astype("float32") + - 0.5 + ) + self.ele_y_arr = ( + np.random.random((self.hidden,)).astype("float32") - 0.5 + ) self.place = paddle.CUDAPlace(0) self.exe = paddle.static.Executor(self.place) @@ -273,7 +308,7 @@ class TestFuseGemmEpilogueBWDBase(unittest.TestCase): self.feed = { "_data": self.data_arr, "_matmul_y": self.matmul_y_arr, - "_ele_y": self.ele_y_arr + "_ele_y": self.ele_y_arr, } self.fetch = [ @@ -283,46 +318,58 @@ class TestFuseGemmEpilogueBWDBase(unittest.TestCase): '{}.w_0@GRAD'.format(multi_layer.linear2.full_name()), '{}.b_0@GRAD'.format(multi_layer.linear2.full_name()), '{}.w_0@GRAD'.format(multi_layer.linear3.full_name()), - '{}.b_0@GRAD'.format(multi_layer.linear3.full_name()) + '{}.b_0@GRAD'.format(multi_layer.linear3.full_name()), ] - self.outs_ref = self.exe.run(self.main_prog, - feed=self.feed, - fetch_list=self.fetch) + self.outs_ref = self.exe.run( + self.main_prog, feed=self.feed, fetch_list=self.fetch + ) - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def _test_output(self): build_strategy = paddle.static.BuildStrategy() build_strategy.fuse_gemm_epilogue = True program = paddle.static.CompiledProgram(self.main_prog) - program = program.with_data_parallel(loss_name=self.loss.name, - build_strategy=build_strategy, - places=paddle.static.cuda_places()) + program = program.with_data_parallel( + loss_name=self.loss.name, + build_strategy=build_strategy, + places=paddle.static.cuda_places(), + ) outs_res = self.exe.run(program, feed=self.feed, fetch_list=self.fetch) for ref, res in zip(self.outs_ref, outs_res): self.assertTrue( compare(ref, res, self.atol, self.rtol), - "[{}] output is miss-matched.".format(type(self).__name__)) + "[{}] output is miss-matched.".format(type(self).__name__), + ) self.assertTrue( verify_node_count(program._graph, "fused_gemm_epilogue", 3), - "[{}] The number of fused_gemm_epilogue is miss-matched in the computing graph." - .format(type(self).__name__)) + "[{}] The number of fused_gemm_epilogue is miss-matched in the computing graph.".format( + type(self).__name__ + ), + ) self.assertTrue( verify_node_count(program._graph, "fused_gemm_epilogue_grad", 3), - "[{}] The number of fused_gemm_epilogue_grad is miss-matched in the computing graph." - .format(type(self).__name__)) + "[{}] The number of fused_gemm_epilogue_grad is miss-matched in the computing graph.".format( + type(self).__name__ + ), + ) _, act_fwd_name, act_bwd_name = self._get_act_type() self.assertTrue( verify_node_count(program._graph, act_fwd_name, 1), - "[{}] The number of {} is miss-matched in the computing graph.". - format(type(self).__name__, act_fwd_name)) + "[{}] The number of {} is miss-matched in the computing graph.".format( + type(self).__name__, act_fwd_name + ), + ) self.assertTrue( verify_node_count(program._graph, act_bwd_name, 2), - "[{}] The number of {} is miss-matched in the computing graph.". - format(type(self).__name__, act_bwd_name)) + "[{}] The number of {} is miss-matched in the computing graph.".format( + type(self).__name__, act_bwd_name + ), + ) def _pre_test_hooks(self): self.atol = 1e-4 @@ -332,10 +379,10 @@ class TestFuseGemmEpilogueBWDBase(unittest.TestCase): return paddle.nn.ReLU, "relu", "relu_grad" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueReLUBWDFP32(TestFuseGemmEpilogueBWDBase): - def _pre_test_hooks(self): self.atol = 1e-4 self.rtol = 1e-3 @@ -347,27 +394,28 @@ class TestFuseGemmEpilogueReLUBWDFP32(TestFuseGemmEpilogueBWDBase): self._test_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueReLUBWDFP16(TestFuseGemmEpilogueReLUBWDFP32): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 fp16_var_list = paddle.static.amp.cast_model_to_fp16(self.main_prog) paddle.static.amp.cast_parameters_to_fp16( - self.place, self.main_prog, to_fp16_var_names=fp16_var_list) + self.place, self.main_prog, to_fp16_var_names=fp16_var_list + ) self.data_arr = self.data_arr.astype("float16") self.matmul_y_arr = self.matmul_y_arr.astype("float16") self.ele_y_arr = self.ele_y_arr.astype("float16") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGeLUBWDFP32(TestFuseGemmEpilogueBWDBase): - def _pre_test_hooks(self): self.atol = 5e-4 self.rtol = 1e-3 @@ -379,17 +427,18 @@ class TestFuseGemmEpilogueGeLUBWDFP32(TestFuseGemmEpilogueBWDBase): self._test_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGeLUBWDFP16(TestFuseGemmEpilogueGeLUBWDFP32): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 fp16_var_list = paddle.static.amp.cast_model_to_fp16(self.main_prog) paddle.static.amp.cast_parameters_to_fp16( - self.place, self.main_prog, to_fp16_var_names=fp16_var_list) + self.place, self.main_prog, to_fp16_var_names=fp16_var_list + ) self.data_arr = self.data_arr.astype("float16") self.matmul_y_arr = self.matmul_y_arr.astype("float16") diff --git a/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py index 5021870e9b6..9fd6fcc276e 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py @@ -24,7 +24,6 @@ import os class TestFuseOptimizationOps(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -33,72 +32,84 @@ class TestFuseOptimizationOps(TestParallelExecutorBase): img, label = init_data() return {"image": img, "label": label} - def _compare_fused_optimizer_ops(self, - model, - use_device, - feed_dict=None, - get_data_from_feeder=None, - optimizer=fluid.optimizer.Adam): + def _compare_fused_optimizer_ops( + self, + model, + use_device, + feed_dict=None, + get_data_from_feeder=None, + optimizer=fluid.optimizer.Adam, + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return - not_fuse_op_first_loss, not_fuse_op_last_loss, _ = self.check_network_convergence( + ( + not_fuse_op_first_loss, + not_fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, feed_dict=feed_dict, get_data_from_feeder=get_data_from_feeder, use_device=use_device, fuse_all_optimizer_ops=False, - optimizer=optimizer) - fuse_op_first_loss, fuse_op_last_loss, _ = self.check_network_convergence( + optimizer=optimizer, + ) + ( + fuse_op_first_loss, + fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, feed_dict=feed_dict, get_data_from_feeder=get_data_from_feeder, use_device=use_device, fuse_all_optimizer_ops=True, - optimizer=optimizer) + optimizer=optimizer, + ) for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) for loss in zip(not_fuse_op_last_loss, fuse_op_last_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) - def _decorate_compare_fused_optimizer_ops(self, model, use_device, - optimizer): - self._compare_fused_optimizer_ops(model, - use_device, - feed_dict=self._get_feed_dict(), - optimizer=optimizer) + def _decorate_compare_fused_optimizer_ops( + self, model, use_device, optimizer + ): + self._compare_fused_optimizer_ops( + model, + use_device, + feed_dict=self._get_feed_dict(), + optimizer=optimizer, + ) class TestFuseAdamOps(TestFuseOptimizationOps): - def optimizer(self, learning_rate=1e-4): return fluid.optimizer.Adam(learning_rate=learning_rate) def test_batchnorm_fc_with_fuse_op(self): - self._decorate_compare_fused_optimizer_ops(fc_with_batchnorm, - DeviceType.CUDA, - optimizer=self.optimizer) - self._decorate_compare_fused_optimizer_ops(fc_with_batchnorm, - DeviceType.CPU, - optimizer=self.optimizer) + self._decorate_compare_fused_optimizer_ops( + fc_with_batchnorm, DeviceType.CUDA, optimizer=self.optimizer + ) + self._decorate_compare_fused_optimizer_ops( + fc_with_batchnorm, DeviceType.CPU, optimizer=self.optimizer + ) class TestFuseSGDOps(TestFuseAdamOps): - def optimizer(self, learning_rate=1e-3): return fluid.optimizer.SGD(learning_rate=learning_rate) class TestFuseMomentumOps(TestFuseAdamOps): - def optimizer(self, learning_rate=1e-3): - return fluid.optimizer.Momentum(learning_rate=learning_rate, - momentum=0.1) + return fluid.optimizer.Momentum( + learning_rate=learning_rate, momentum=0.1 + ) class TestSpareFuseAdamOps(TestFuseOptimizationOps): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -113,85 +124,87 @@ class TestSpareFuseAdamOps(TestFuseOptimizationOps): feeder = fluid.DataFeeder(feed_list=["words", "label"], place=place) return feeder.feed(self.train_data) - def _decorate_compare_fused_optimizer_ops(self, model, use_device, - optimizer): + def _decorate_compare_fused_optimizer_ops( + self, model, use_device, optimizer + ): self._compare_fused_optimizer_ops( model, use_device, get_data_from_feeder=self._get_data_from_feeder, - optimizer=optimizer) + optimizer=optimizer, + ) def optimizer(self, learning_rate=1e-4): return fluid.optimizer.Adam(learning_rate=learning_rate) def test_simple_bow_net_with_fuse_op(self): model = partial(bow_net, dict_dim=self.word_dict_len, is_sparse=True) - self._decorate_compare_fused_optimizer_ops(model, - DeviceType.CUDA, - optimizer=self.optimizer) - self._decorate_compare_fused_optimizer_ops(model, - DeviceType.CPU, - optimizer=self.optimizer) + self._decorate_compare_fused_optimizer_ops( + model, DeviceType.CUDA, optimizer=self.optimizer + ) + self._decorate_compare_fused_optimizer_ops( + model, DeviceType.CPU, optimizer=self.optimizer + ) class TestSpareFuseSGDOps(TestSpareFuseAdamOps): - def optimizer(self, learning_rate=1e-3): return fluid.optimizer.SGD(learning_rate=learning_rate) class TestSpareFuseMomentumOps(TestSpareFuseAdamOps): - def optimizer(self, learning_rate=1e-3): - return fluid.optimizer.Momentum(learning_rate=learning_rate, - momentum=0.1) + return fluid.optimizer.Momentum( + learning_rate=learning_rate, momentum=0.1 + ) class TestPassConflictBase(TestFuseAdamOps): - - def _compare_fused_optimizer_ops(self, - model, - use_device, - feed_dict=None, - get_data_from_feeder=None, - optimizer=fluid.optimizer.Adam): + def _compare_fused_optimizer_ops( + self, + model, + use_device, + feed_dict=None, + get_data_from_feeder=None, + optimizer=fluid.optimizer.Adam, + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return - self.check_pass_conflict(model, - feed_dict=feed_dict, - get_data_from_feeder=get_data_from_feeder, - use_device=use_device, - fuse_all_optimizer_ops=True, - optimizer=optimizer, - enable_sequential_execution=True) + self.check_pass_conflict( + model, + feed_dict=feed_dict, + get_data_from_feeder=get_data_from_feeder, + use_device=use_device, + fuse_all_optimizer_ops=True, + optimizer=optimizer, + enable_sequential_execution=True, + ) class TestFuseAdamOpsPassConflict(TestPassConflictBase): - def optimizer(self, learning_rate=1e-4): return fluid.optimizer.Adam(learning_rate=learning_rate) def test_batchnorm_fc_with_fuse_op(self): - self._decorate_compare_fused_optimizer_ops(fc_with_batchnorm, - DeviceType.CPU, - optimizer=self.optimizer) - self._decorate_compare_fused_optimizer_ops(fc_with_batchnorm, - DeviceType.CUDA, - optimizer=self.optimizer) + self._decorate_compare_fused_optimizer_ops( + fc_with_batchnorm, DeviceType.CPU, optimizer=self.optimizer + ) + self._decorate_compare_fused_optimizer_ops( + fc_with_batchnorm, DeviceType.CUDA, optimizer=self.optimizer + ) class TestFuseSGDOpsPassConflict(TestFuseAdamOpsPassConflict): - def optimizer(self, learning_rate=1e-3): return fluid.optimizer.SGD(learning_rate=learning_rate) class TestFuseMomentumOpsPassConflict(TestFuseAdamOpsPassConflict): - def optimizer(self, learning_rate=1e-3): - return fluid.optimizer.Momentum(learning_rate=learning_rate, - momentum=0.1) + return fluid.optimizer.Momentum( + learning_rate=learning_rate, momentum=0.1 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py index a93c1380c0b..686a5c1e410 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py @@ -26,27 +26,27 @@ def norm(*args, **kargs): def sep_conv(input, channel, stride, filter, dilation=1, act=None): # with scope('depthwise'): - input = fluid.layers.conv2d(input, - input.shape[1], - filter, - stride, - groups=input.shape[1], - padding=(filter // 2) * dilation, - dilation=dilation, - use_cudnn=False, - bias_attr=False) + input = fluid.layers.conv2d( + input, + input.shape[1], + filter, + stride, + groups=input.shape[1], + padding=(filter // 2) * dilation, + dilation=dilation, + use_cudnn=False, + bias_attr=False, + ) input = norm(input) - if act: input = act(input) + if act: + input = act(input) # with scope('pointwise'): - input = fluid.layers.conv2d(input, - channel, - 1, - 1, - groups=1, - padding=0, - bias_attr=False) + input = fluid.layers.conv2d( + input, channel, 1, 1, groups=1, padding=0, bias_attr=False + ) input = norm(input) - if act: input = act(input) + if act: + input = act(input) return input @@ -65,7 +65,6 @@ def simple_depthwise_net(use_feed): class TestMNIST(TestParallelExecutorBase): - def _init_data(self, random=True): np.random.seed(5) if random: @@ -83,31 +82,36 @@ class TestMNIST(TestParallelExecutorBase): def _optimizer(learning_rate=1e-6): optimizer = fluid.optimizer.SGD( learning_rate=learning_rate, - regularization=fluid.regularizer.L2Decay(1e-6)) + regularization=fluid.regularizer.L2Decay(1e-6), + ) return optimizer if only_forward: _optimizer = None - fuse_op_first_loss, fuse_op_last_loss, _ = self.check_network_convergence( + ( + fuse_op_first_loss, + fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, fuse_relu_depthwise_conv=True, use_ir_memory_optimize=True, - optimizer=_optimizer) - not_fuse_op_first_loss, not_fuse_op_last_loss, _ = self.check_network_convergence( + optimizer=_optimizer, + ) + ( + not_fuse_op_first_loss, + not_fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, fuse_relu_depthwise_conv=False, - optimizer=_optimizer) + optimizer=_optimizer, + ) for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) diff --git a/python/paddle/fluid/tests/unittests/test_fused_attention_op.py b/python/paddle/fluid/tests/unittests/test_fused_attention_op.py index 1e27005499a..6e3c0014422 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_attention_op.py @@ -30,7 +30,6 @@ default_main_program().random_seed = 42 class TestFusedAttentionOp(OpTest): - def setUp(self): self.config() self.generate_input_data() @@ -50,22 +49,30 @@ class TestFusedAttentionOp(OpTest): self.__class__.op_type = "fused_attention" # use autograd to check grad in this unittest. self.__class__.no_need_check_grad = True - self.q_proj = Linear(self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.k_proj = Linear(self.kdim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.v_proj = Linear(self.vdim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.out_proj = Linear(self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) + self.q_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.k_proj = Linear( + self.kdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.v_proj = Linear( + self.vdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.out_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) paddle.set_default_dtype(np.float32) self.norm1 = LayerNorm(self.embed_dim) self.norm2 = LayerNorm(self.embed_dim) @@ -92,40 +99,57 @@ class TestFusedAttentionOp(OpTest): self.weight_attr = None self.bias_attr = None self.kdim, self.vdim = self.embed_dim, self.embed_dim - self.key_length, self.value_length = self.query_length, self.query_length + self.key_length, self.value_length = ( + self.query_length, + self.query_length, + ) def generate_input_data(self): - self.query = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.query = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) out_seq_len = self.key_length if self.has_cache_kv: assert self.training is False, ValueError( - 'cache_kv can only used in inference') - self.cache_kv = np.random.rand(2, self.batch_size, self.num_heads, - self.cache_length, - self.head_dim).astype(self.x_type) + 'cache_kv can only used in inference' + ) + self.cache_kv = np.random.rand( + 2, + self.batch_size, + self.num_heads, + self.cache_length, + self.head_dim, + ).astype(self.x_type) out_seq_len += self.cache_length else: self.cache_kv = None if self.has_attn_mask: # [B, n_head, seq_len, out_seq_len] - self.attn_mask = np.ones((self.batch_size, self.num_heads, - self.query_length, out_seq_len), - dtype=self.attn_mask_type) + self.attn_mask = np.ones( + ( + self.batch_size, + self.num_heads, + self.query_length, + out_seq_len, + ), + dtype=self.attn_mask_type, + ) if self.attn_mask_type == np.int64: self.attn_mask = np.tril(self.attn_mask) elif self.attn_mask_type == np.float64: self.attn_mask = (np.tril(self.attn_mask) - 1.0) * 1e9 else: raise ValueError( - "'attn_mask_type' should be 'int64' or 'float64'.") + "'attn_mask_type' should be 'int64' or 'float64'." + ) else: self.attn_mask = None self.key, self.value = self.query, self.query - self.dout = np.random.random((self.batch_size, self.query_length, - self.embed_dim)).astype(self.x_type) + self.dout = np.random.random( + (self.batch_size, self.query_length, self.embed_dim) + ).astype(self.x_type) def GetBaselineOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -167,10 +191,9 @@ class TestFusedAttentionOp(OpTest): # [B, n_head, seq_len, head_dim] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, out_seq_len] - qk_out = layers.matmul(x=q_out, - y=k_out, - transpose_y=True, - alpha=self.head_dim**-0.5) + qk_out = layers.matmul( + x=q_out, y=k_out, transpose_y=True, alpha=self.head_dim**-0.5 + ) if attn_mask is not None: attn_mask = _convert_attention_mask(attn_mask, qk_out.dtype) @@ -180,10 +203,12 @@ class TestFusedAttentionOp(OpTest): softmax_out = F.softmax(qk_out) if self.dropout_prob: - dropout_out = F.dropout(softmax_out, - self.dropout_prob, - training=self.training, - mode="upscale_in_train") + dropout_out = F.dropout( + softmax_out, + self.dropout_prob, + training=self.training, + mode="upscale_in_train", + ) # [B, n_head, seq_len, out_seq_len] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, head_dim] qktv_out = tensor.matmul(dropout_out, v_out) @@ -192,7 +217,8 @@ class TestFusedAttentionOp(OpTest): fmha_out = tensor.transpose(qktv_out, perm=[0, 2, 1, 3]) out_linear_in = tensor.reshape( - x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]]) + x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]] + ) out = self.out_proj(out_linear_in) residual_out = residual + self.dropout(out) @@ -204,37 +230,47 @@ class TestFusedAttentionOp(OpTest): if self.has_cache_kv: return final_out - paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)], - retain_graph=True) + paddle.autograd.backward( + [final_out], [paddle.to_tensor(self.dout)], retain_graph=True + ) return final_out, tensor_query.grad def GetFusedAttentionOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) - q_proj_weight = paddle.to_tensor(self.q_proj.weight, - stop_gradient=False) - k_proj_weight = paddle.to_tensor(self.k_proj.weight, - stop_gradient=False) - v_proj_weight = paddle.to_tensor(self.v_proj.weight, - stop_gradient=False) - out_linear_weight = paddle.to_tensor(self.out_proj.weight, - stop_gradient=False) + q_proj_weight = paddle.to_tensor( + self.q_proj.weight, stop_gradient=False + ) + k_proj_weight = paddle.to_tensor( + self.k_proj.weight, stop_gradient=False + ) + v_proj_weight = paddle.to_tensor( + self.v_proj.weight, stop_gradient=False + ) + out_linear_weight = paddle.to_tensor( + self.out_proj.weight, stop_gradient=False + ) if self.bias_attr is False: qkv_bias_tensor = None out_linear_bias = None else: - q_proj_bias = paddle.to_tensor(self.q_proj.bias, - stop_gradient=False) - k_proj_bias = paddle.to_tensor(self.k_proj.bias, - stop_gradient=False) - v_proj_bias = paddle.to_tensor(self.v_proj.bias, - stop_gradient=False) + q_proj_bias = paddle.to_tensor( + self.q_proj.bias, stop_gradient=False + ) + k_proj_bias = paddle.to_tensor( + self.k_proj.bias, stop_gradient=False + ) + v_proj_bias = paddle.to_tensor( + self.v_proj.bias, stop_gradient=False + ) qkv_bias = np.concatenate( - (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy())) + (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy()) + ) qkv_bias = qkv_bias.reshape((3, self.num_heads, self.head_dim)) qkv_bias_tensor = paddle.to_tensor(qkv_bias, stop_gradient=False) - out_linear_bias = paddle.to_tensor(self.out_proj.bias, - stop_gradient=False) + out_linear_bias = paddle.to_tensor( + self.out_proj.bias, stop_gradient=False + ) ln1_scale = paddle.to_tensor(self.norm1.weight, stop_gradient=False) ln1_bias = paddle.to_tensor(self.norm1.bias, stop_gradient=False) @@ -245,9 +281,11 @@ class TestFusedAttentionOp(OpTest): k_proj_weight = k_proj_weight.numpy().transpose((1, 0)) v_proj_weight = v_proj_weight.numpy().transpose((1, 0)) qkv_weight = np.concatenate( - (q_proj_weight, k_proj_weight, v_proj_weight)) + (q_proj_weight, k_proj_weight, v_proj_weight) + ) qkv_weight = qkv_weight.reshape( - (3, self.num_heads, self.head_dim, self.embed_dim)) + (3, self.num_heads, self.head_dim, self.embed_dim) + ) x = paddle.to_tensor(self.query, stop_gradient=False) cache_kv = None @@ -264,47 +302,56 @@ class TestFusedAttentionOp(OpTest): if attn_mask is not None: attn_mask = _convert_attention_mask(attn_mask, x.dtype) final_out = incubate_f.fused_multi_head_attention( - x, qkv_weight_tensor, out_linear_weight, self.pre_layer_norm, - ln1_scale, ln1_bias, ln2_scale, ln2_bias, epsilon, qkv_bias_tensor, - out_linear_bias, cache_kv, attn_mask, self.dropout_prob, - self.attn_dropout_prob, ln2_epsilon) + x, + qkv_weight_tensor, + out_linear_weight, + self.pre_layer_norm, + ln1_scale, + ln1_bias, + ln2_scale, + ln2_bias, + epsilon, + qkv_bias_tensor, + out_linear_bias, + cache_kv, + attn_mask, + self.dropout_prob, + self.attn_dropout_prob, + ln2_epsilon, + ) if self.has_cache_kv: return final_out[0], final_out[1] - paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)], - retain_graph=True) + paddle.autograd.backward( + [final_out], [paddle.to_tensor(self.dout)], retain_graph=True + ) return final_out, x.grad def test_fused_attention_op(self): final_out_ref, x_grad_ref = self.GetBaselineOut() final_out, x_grad = self.GetFusedAttentionOut() - np.testing.assert_allclose(final_out_ref, - final_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(x_grad_ref, - x_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out.numpy(), rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + x_grad_ref, x_grad.numpy(), rtol=self.rtol, atol=self.atol + ) class TestFusedAttentionOpBiasIsNone(TestFusedAttentionOp): - def config(self): super().config() self.bias_attr = False class TestFusedAttentionOpPreLn(TestFusedAttentionOp): - def config(self): super().config() self.pre_layer_norm = True class TestFusedAttentionOpNoneAttnMask(TestFusedAttentionOp): - def config(self): super().config() self.pre_layer_norm = True @@ -312,7 +359,6 @@ class TestFusedAttentionOpNoneAttnMask(TestFusedAttentionOp): class TestFusedAttentionOpFp16(TestFusedAttentionOp): - def config(self): super().config() self.x_type = np.float16 @@ -320,18 +366,15 @@ class TestFusedAttentionOpFp16(TestFusedAttentionOp): def test_fused_attention_op(self): final_out_ref, x_grad_ref = self.GetBaselineOut() final_out, x_grad = self.GetFusedAttentionOut() - np.testing.assert_allclose(final_out_ref, - final_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(x_grad_ref, - x_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out.numpy(), rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + x_grad_ref, x_grad.numpy(), rtol=self.rtol, atol=self.atol + ) class TestFusedAttentionOpCacheKV(TestFusedAttentionOp): - def config(self): super().config() self.has_cache_kv = True @@ -343,10 +386,9 @@ class TestFusedAttentionOpCacheKV(TestFusedAttentionOp): with paddle.no_grad(): final_out_ref = self.GetBaselineOut() final_out, cache_kv_out = self.GetFusedAttentionOut() - np.testing.assert_allclose(final_out_ref, - final_out.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out.numpy(), rtol=self.rtol, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fused_attention_op_api.py b/python/paddle/fluid/tests/unittests/test_fused_attention_op_api.py index ee8efdc81d0..3a3872bb672 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_attention_op_api.py +++ b/python/paddle/fluid/tests/unittests/test_fused_attention_op_api.py @@ -39,8 +39,9 @@ def softmax(x): def batch_matmul(x, y): assert x.shape[0] == y.shape[0] assert x.shape[1] == y.shape[1] - retval = np.zeros((x.shape[0], x.shape[1], x.shape[2], y.shape[3]), - dtype=np.float64) + retval = np.zeros( + (x.shape[0], x.shape[1], x.shape[2], y.shape[3]), dtype=np.float64 + ) for i in range(x.shape[0]): for j in range(x.shape[1]): retval[i, j, :, :] = np.matmul(x[i, j, :, :], y[i, j, :, :]) @@ -52,24 +53,34 @@ def layer_norm(x, has_scale, has_bias, weight, bias, epsilon=1e-05): x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model - x1_up = (x - mu) + x1_up = x - mu x1_down_1 = sigma_squar + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) x1 = x1_up / x1_down x_scaled = x1 - if (has_scale): + if has_scale: x_scaled = weight * x1 x_scaled_bias = x_scaled - if (has_bias): + if has_bias: x_scaled_bias = x_scaled + bias x_scaled_bias = x_scaled_bias.reshape((batch_size, src_len, d_model)) return x_scaled_bias -def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, - ln_2_scale, ln_2_bias, qkv_weight, qkv_bias, - out_linear_weight, out_linear_bias): +def compute_reference( + pre_layer_norm, + query, + attn_mask, + ln_scale, + ln_bias, + ln_2_scale, + ln_2_bias, + qkv_weight, + qkv_bias, + out_linear_weight, + out_linear_bias, +): batch_size = query.shape[0] seq_len = query.shape[1] embed_dim = query.shape[2] @@ -87,11 +98,13 @@ def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, qkv_weight = qkv_weight.transpose((3, 0, 1, 2)) qkv_weight = qkv_weight.reshape( qkv_weight.shape[0], - qkv_weight.shape[1] * qkv_weight.shape[2] * qkv_weight.shape[3]) + qkv_weight.shape[1] * qkv_weight.shape[2] * qkv_weight.shape[3], + ) if qkv_bias is not None: - qkv_bias = qkv_bias.reshape(qkv_bias.shape[0] * qkv_bias.shape[1] * - qkv_bias.shape[2]) + qkv_bias = qkv_bias.reshape( + qkv_bias.shape[0] * qkv_bias.shape[1] * qkv_bias.shape[2] + ) if pre_layer_norm: ln_out = ln_out.reshape(batch_size * seq_len, embed_dim) qkv = fc(ln_out, qkv_weight) @@ -109,22 +122,25 @@ def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, qkv_bias_out = qkv query = query.reshape(batch_size, seq_len, embed_dim) - qkv_bias_out = qkv_bias_out.reshape(batch_size, seq_len, 3, num_head, - head_dim) + qkv_bias_out = qkv_bias_out.reshape( + batch_size, seq_len, 3, num_head, head_dim + ) # q*k^t qkv_bias_out = qkv_bias_out.transpose( - (2, 0, 1, 3, 4)) # 3, batch_size, seq_len, num_head, head_dim + (2, 0, 1, 3, 4) + ) # 3, batch_size, seq_len, num_head, head_dim qkv_bias_out = qkv_bias_out.transpose( - (0, 1, 3, 2, 4)) # 3, batch_size, num_head, seq_len, head_dim + (0, 1, 3, 2, 4) + ) # 3, batch_size, num_head, seq_len, head_dim q = qkv_bias_out[0:1, ::] q = q.reshape(batch_size, num_head, seq_len, head_dim) - k = qkv_bias_out[1:2, ::] #[1, batch_size, num_head, seq_len, head_dim] + k = qkv_bias_out[1:2, ::] # [1, batch_size, num_head, seq_len, head_dim] k = k.reshape(batch_size, num_head, seq_len, head_dim) v = qkv_bias_out[2::] v = v.reshape(batch_size, num_head, seq_len, head_dim) - k = k.transpose([0, 1, 3, 2]) #[batch_size, num_head, head_dim, seq_len] + k = k.transpose([0, 1, 3, 2]) # [batch_size, num_head, head_dim, seq_len] qkt = batch_matmul(q, k / np.sqrt(head_dim, dtype=np.float64)) if attn_mask is not None: @@ -139,11 +155,13 @@ def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, attn_heads = batch_matmul(softmax_out, v) attn_heads = attn_heads.transpose( - (0, 2, 1, 3)) # [batch_size, seq_len, num_head, head_dim] + (0, 2, 1, 3) + ) # [batch_size, seq_len, num_head, head_dim] # out_linear - out_linear_input = attn_heads.reshape(batch_size, seq_len, - num_head * head_dim) + out_linear_input = attn_heads.reshape( + batch_size, seq_len, num_head * head_dim + ) out_linear_out = fc(out_linear_input, out_linear_weight) # bias add, dropout, residual add, layer_norm. @@ -155,13 +173,16 @@ def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, out_linear_bias_dropout_residual_out = query + out_linear_bias_dropout_out if not pre_layer_norm: out_linear_bias_dropout_residual_out = layer_norm( - out_linear_bias_dropout_residual_out, True, has_bias, ln_2_scale, - ln_2_bias) + out_linear_bias_dropout_residual_out, + True, + has_bias, + ln_2_scale, + ln_2_bias, + ) return out_linear_bias_dropout_residual_out class TestFusedAttentionAPI(unittest.TestCase): - def setUp(self): self.setXType() self.setPreLn() @@ -209,22 +230,33 @@ class TestFusedAttentionAPI(unittest.TestCase): self.weight_attr = None self.kdim, self.vdim = self.embed_dim, self.embed_dim - self.key_length, self.value_length = self.query_length, self.query_length + self.key_length, self.value_length = ( + self.query_length, + self.query_length, + ) def generate_input_data(self): - self.query = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.query = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) if self.has_attn_mask: - self.attn_mask = np.ones((self.batch_size, self.num_heads, - self.query_length, self.key_length), - dtype=self.attn_mask_type) + self.attn_mask = np.ones( + ( + self.batch_size, + self.num_heads, + self.query_length, + self.key_length, + ), + dtype=self.attn_mask_type, + ) if self.attn_mask_type == np.int64: self.attn_mask = np.tril(self.attn_mask) elif self.attn_mask_type == np.float64: self.attn_mask = (np.tril(self.attn_mask) - 1.0) * 1e9 else: raise ValueError( - "'attn_mask_type' should be 'int64' or 'float64'.") + "'attn_mask_type' should be 'int64' or 'float64'." + ) else: self.attn_mask = None self.key, self.value = self.query, self.query @@ -235,18 +267,34 @@ class TestFusedAttentionAPI(unittest.TestCase): else: attn_mask_tensor = None fused_attn = FusedMultiHeadAttention( - self.embed_dim, self.num_heads, self.dropout_prob, - self.attn_dropout_prob, self.kdim, self.vdim, self.pre_layer_norm, - self.need_weight, self.weight_attr, self.bias_attr, - self.weight_attr, self.bias_attr, self.weight_attr, self.bias_attr, - self.weight_attr, self.bias_attr) + self.embed_dim, + self.num_heads, + self.dropout_prob, + self.attn_dropout_prob, + self.kdim, + self.vdim, + self.pre_layer_norm, + self.need_weight, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + ) if self.bias_attr is not False: - qkv_bias = np.random.random( - fused_attn.qkv_bias.shape).astype('float32') + qkv_bias = np.random.random(fused_attn.qkv_bias.shape).astype( + 'float32' + ) fused_attn.qkv_bias.set_value(paddle.to_tensor(qkv_bias)) - out = fused_attn(paddle.to_tensor(self.query), - paddle.to_tensor(self.query), - paddle.to_tensor(self.query), attn_mask_tensor) + out = fused_attn( + paddle.to_tensor(self.query), + paddle.to_tensor(self.query), + paddle.to_tensor(self.query), + attn_mask_tensor, + ) fused_attn_qkv_bias = None fused_attn_linear_bias = None @@ -263,38 +311,58 @@ class TestFusedAttentionAPI(unittest.TestCase): fused_attn_ln_bias = fused_attn.ln_bias.numpy() ref_out = compute_reference( - self.pre_layer_norm, self.query, self.attn_mask, + self.pre_layer_norm, + self.query, + self.attn_mask, fused_attn.pre_ln_scale.numpy() if self.pre_layer_norm else None, fused_attn_pre_ln_bias, fused_attn.ln_scale.numpy() if not self.pre_layer_norm else None, fused_attn_ln_bias, - fused_attn.qkv_weight.numpy(), fused_attn_qkv_bias, - fused_attn.linear_weight.numpy(), fused_attn_linear_bias) - np.testing.assert_allclose(ref_out, - out.numpy(), - rtol=self.rtol, - atol=self.atol) + fused_attn.qkv_weight.numpy(), + fused_attn_qkv_bias, + fused_attn.linear_weight.numpy(), + fused_attn_linear_bias, + ) + np.testing.assert_allclose( + ref_out, out.numpy(), rtol=self.rtol, atol=self.atol + ) def run_static(self): fused_attn = FusedMultiHeadAttention( - self.embed_dim, self.num_heads, self.dropout_prob, - self.attn_dropout_prob, self.kdim, self.vdim, self.pre_layer_norm, - self.need_weight, self.weight_attr, self.bias_attr, - self.weight_attr, self.bias_attr, self.weight_attr, self.bias_attr, - self.weight_attr, self.bias_attr) + self.embed_dim, + self.num_heads, + self.dropout_prob, + self.attn_dropout_prob, + self.kdim, + self.vdim, + self.pre_layer_norm, + self.need_weight, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + ) x = paddle.static.data( name='X', shape=[self.batch_size, self.query_length, self.embed_dim], - dtype=self.x_type) + dtype=self.x_type, + ) if self.has_attn_mask: - attn_mask = paddle.static.data(name='SrcMask', - shape=[ - self.batch_size, self.num_heads, - self.query_length, - self.key_length - ], - dtype=self.attn_mask_type) + attn_mask = paddle.static.data( + name='SrcMask', + shape=[ + self.batch_size, + self.num_heads, + self.query_length, + self.key_length, + ], + dtype=self.attn_mask_type, + ) final_out = fused_attn(x, x, x, attn_mask) else: final_out = fused_attn(x, x, x) @@ -314,35 +382,38 @@ class TestFusedAttentionAPI(unittest.TestCase): if self.pre_layer_norm: out, qkv_weight, out_linear_weight, ln_scale = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.query, - "SrcMask": self.attn_mask - }, + feed={"X": self.query, "SrcMask": self.attn_mask}, fetch_list=[ final_out, fused_attn.qkv_weight, fused_attn.linear_weight, fused_attn.pre_ln_scale, - ]) + ], + ) else: out, qkv_weight, out_linear_weight, ln_2_scale = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.query, - "SrcMask": self.attn_mask - }, + feed={"X": self.query, "SrcMask": self.attn_mask}, fetch_list=[ - final_out, fused_attn.qkv_weight, - fused_attn.linear_weight, fused_attn.ln_scale - ]) + final_out, + fused_attn.qkv_weight, + fused_attn.linear_weight, + fused_attn.ln_scale, + ], + ) else: if self.pre_layer_norm: - out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_scale, ln_bias = exe.run( + ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_scale, + ln_bias, + ) = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.query, - "SrcMask": self.attn_mask - }, + feed={"X": self.query, "SrcMask": self.attn_mask}, fetch_list=[ final_out, fused_attn.qkv_weight, @@ -351,20 +422,30 @@ class TestFusedAttentionAPI(unittest.TestCase): fused_attn.linear_bias, fused_attn.pre_ln_scale, fused_attn.pre_ln_bias, - ]) + ], + ) else: - out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_2_scale, ln_2_bias = exe.run( + ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_2_scale, + ln_2_bias, + ) = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.query, - "SrcMask": self.attn_mask - }, + feed={"X": self.query, "SrcMask": self.attn_mask}, fetch_list=[ - final_out, fused_attn.qkv_weight, - fused_attn.qkv_bias, fused_attn.linear_weight, - fused_attn.linear_bias, fused_attn.ln_scale, - fused_attn.ln_bias - ]) + final_out, + fused_attn.qkv_weight, + fused_attn.qkv_bias, + fused_attn.linear_weight, + fused_attn.linear_bias, + fused_attn.ln_scale, + fused_attn.ln_bias, + ], + ) else: if self.bias_attr is False: if self.pre_layer_norm: @@ -378,7 +459,8 @@ class TestFusedAttentionAPI(unittest.TestCase): fused_attn.qkv_weight, fused_attn.linear_weight, fused_attn.pre_ln_scale, - ]) + ], + ) else: out, qkv_weight, out_linear_weight, ln_2_scale = exe.run( paddle.static.default_main_program(), @@ -386,12 +468,23 @@ class TestFusedAttentionAPI(unittest.TestCase): "X": self.query, }, fetch_list=[ - final_out, fused_attn.qkv_weight, - fused_attn.linear_weight, fused_attn.ln_scale - ]) + final_out, + fused_attn.qkv_weight, + fused_attn.linear_weight, + fused_attn.ln_scale, + ], + ) else: if self.pre_layer_norm: - out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_scale, ln_bias = exe.run( + ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_scale, + ln_bias, + ) = exe.run( paddle.static.default_main_program(), feed={ "X": self.query, @@ -404,30 +497,71 @@ class TestFusedAttentionAPI(unittest.TestCase): fused_attn.linear_bias, fused_attn.pre_ln_scale, fused_attn.pre_ln_bias, - ]) + ], + ) else: - out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_2_scale, ln_2_bias = exe.run( + ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_2_scale, + ln_2_bias, + ) = exe.run( paddle.static.default_main_program(), feed={ "X": self.query, }, fetch_list=[ - final_out, fused_attn.qkv_weight, - fused_attn.qkv_bias, fused_attn.linear_weight, - fused_attn.linear_bias, fused_attn.ln_scale, - fused_attn.ln_bias - ]) - return out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_scale, ln_bias, ln_2_scale, ln_2_bias + final_out, + fused_attn.qkv_weight, + fused_attn.qkv_bias, + fused_attn.linear_weight, + fused_attn.linear_bias, + fused_attn.ln_scale, + fused_attn.ln_bias, + ], + ) + return ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_scale, + ln_bias, + ln_2_scale, + ln_2_bias, + ) def test_static_api(self): paddle.enable_static() with paddle.static.program_guard(Program()): - out, qkv_weight, qkv_bias, linear_weight, linear_bias, ln_scale, ln_bias, ln_2_scale, ln_2_bias = self.run_static( - ) - ref_out = compute_reference(self.pre_layer_norm, self.query, - self.attn_mask, ln_scale, ln_bias, - ln_2_scale, ln_2_bias, qkv_weight, qkv_bias, - linear_weight, linear_bias) + ( + out, + qkv_weight, + qkv_bias, + linear_weight, + linear_bias, + ln_scale, + ln_bias, + ln_2_scale, + ln_2_bias, + ) = self.run_static() + ref_out = compute_reference( + self.pre_layer_norm, + self.query, + self.attn_mask, + ln_scale, + ln_bias, + ln_2_scale, + ln_2_bias, + qkv_weight, + qkv_bias, + linear_weight, + linear_bias, + ) np.testing.assert_allclose(ref_out, out, rtol=self.rtol, atol=self.atol) def test_dynamic_api(self): @@ -436,7 +570,6 @@ class TestFusedAttentionAPI(unittest.TestCase): class TestFusedAttentionAPINoneAttnMask(TestFusedAttentionAPI): - def setAttnMask(self): self.has_attn_mask = False @@ -445,7 +578,6 @@ class TestFusedAttentionAPINoneAttnMask(TestFusedAttentionAPI): class TestFusedAttentionAPIBiasIsNone(TestFusedAttentionAPI): - def setBiasAttr(self): self.bias_attr = False diff --git a/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op.py b/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op.py index 7572a1dfe14..8ae92fe194f 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op.py @@ -28,7 +28,6 @@ default_main_program().random_seed = 42 class TestFusedBiasDropoutResidualLayerNormOp(OpTest): - def setUp(self): self.config() self.generate_input_data() @@ -53,23 +52,28 @@ class TestFusedBiasDropoutResidualLayerNormOp(OpTest): self.bias_attr = None def generate_input_data(self): - self.x = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) - self.residual = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.x = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) + self.residual = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) self.linear_bias = np.random.rand(self.embed_dim).astype(self.x_type) - self.dout = np.random.random((self.batch_size, self.query_length, - self.embed_dim)).astype(self.x_type) + self.dout = np.random.random( + (self.batch_size, self.query_length, self.embed_dim) + ).astype(self.x_type) if self.bias_attr is False: self.tensor_linear_bias = None else: - self.tensor_linear_bias = paddle.to_tensor(self.linear_bias, - stop_gradient=False) + self.tensor_linear_bias = paddle.to_tensor( + self.linear_bias, stop_gradient=False + ) self.tensor_x = paddle.to_tensor(self.x, stop_gradient=False) - self.tensor_residual = paddle.to_tensor(self.residual, - stop_gradient=False) + self.tensor_residual = paddle.to_tensor( + self.residual, stop_gradient=False + ) def GetBaselineOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -82,14 +86,20 @@ class TestFusedBiasDropoutResidualLayerNormOp(OpTest): residual_out = self.tensor_residual + self.dropout(out) final_out = self.norm1(residual_out) - paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)], - retain_graph=True) + paddle.autograd.backward( + [final_out], [paddle.to_tensor(self.dout)], retain_graph=True + ) if self.tensor_linear_bias is not None: tensor_linear_bias_grad = self.tensor_linear_bias.grad else: tensor_linear_bias_grad = None - return final_out, self.tensor_x.grad, self.tensor_residual.grad, tensor_linear_bias_grad + return ( + final_out, + self.tensor_x.grad, + self.tensor_residual.grad, + tensor_linear_bias_grad, + ) def GetFusedBiasDropoutResidualLayerNormOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -99,52 +109,71 @@ class TestFusedBiasDropoutResidualLayerNormOp(OpTest): epsilon = 1e-05 final_out = incubate_f.fused_bias_dropout_residual_layer_norm( - self.tensor_x, self.tensor_residual, self.tensor_linear_bias, - ln_scale, ln_bias, self.dropout_prob, epsilon) + self.tensor_x, + self.tensor_residual, + self.tensor_linear_bias, + ln_scale, + ln_bias, + self.dropout_prob, + epsilon, + ) - paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)], - retain_graph=True) + paddle.autograd.backward( + [final_out], [paddle.to_tensor(self.dout)], retain_graph=True + ) if self.tensor_linear_bias is not None: tensor_linear_bias_grad = self.tensor_linear_bias.grad else: tensor_linear_bias_grad = None - return final_out, self.tensor_x.grad, self.tensor_residual.grad, tensor_linear_bias_grad + return ( + final_out, + self.tensor_x.grad, + self.tensor_residual.grad, + tensor_linear_bias_grad, + ) def test_fused_op(self): - out_ref, x_grad_ref, residual_grad_ref, linear_bias_grad_ref = self.GetBaselineOut( + ( + out_ref, + x_grad_ref, + residual_grad_ref, + linear_bias_grad_ref, + ) = self.GetBaselineOut() + ( + out, + x_grad, + residual_grad, + linear_bias_grad, + ) = self.GetFusedBiasDropoutResidualLayerNormOut() + np.testing.assert_allclose( + out_ref, out.numpy(), rtol=1e-5, atol=self.atol + ) + np.testing.assert_allclose( + x_grad_ref, x_grad.numpy(), rtol=1e-5, atol=self.atol ) - out, x_grad, residual_grad, linear_bias_grad = self.GetFusedBiasDropoutResidualLayerNormOut( + np.testing.assert_allclose( + residual_grad_ref, residual_grad.numpy(), rtol=1e-5, atol=self.atol ) - np.testing.assert_allclose(out_ref, - out.numpy(), - rtol=1e-5, - atol=self.atol) - np.testing.assert_allclose(x_grad_ref, - x_grad.numpy(), - rtol=1e-5, - atol=self.atol) - np.testing.assert_allclose(residual_grad_ref, - residual_grad.numpy(), - rtol=1e-5, - atol=self.atol) if linear_bias_grad_ref is not None: - np.testing.assert_allclose(linear_bias_grad_ref, - linear_bias_grad.numpy(), - rtol=1e-5, - atol=self.atol) + np.testing.assert_allclose( + linear_bias_grad_ref, + linear_bias_grad.numpy(), + rtol=1e-5, + atol=self.atol, + ) class TestFusedBiasDropoutResidualLayerNormOpBiasIsNone( - TestFusedBiasDropoutResidualLayerNormOp): - + TestFusedBiasDropoutResidualLayerNormOp +): def config(self): super().config() self.bias_attr = False class TestFusedBiasDropoutResidualLayerNormOpFp16( - TestFusedBiasDropoutResidualLayerNormOp): - + TestFusedBiasDropoutResidualLayerNormOp +): def config(self): super().config() self.x_type = np.float16 diff --git a/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op_api.py b/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op_api.py index aa54860e099..261a2686b27 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op_api.py +++ b/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op_api.py @@ -15,7 +15,9 @@ import numpy as np import paddle -from paddle.incubate.nn.layer.fused_transformer import FusedBiasDropoutResidualLayerNorm +from paddle.incubate.nn.layer.fused_transformer import ( + FusedBiasDropoutResidualLayerNorm, +) from paddle.static import Program import unittest @@ -25,16 +27,16 @@ def layer_norm(x, has_scale, has_bias, weight, bias, epsilon=1e-05): x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model - x1_up = (x - mu) + x1_up = x - mu x1_down_1 = sigma_squar + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) x1 = x1_up / x1_down x_scaled = x1 - if (has_scale): + if has_scale: x_scaled = weight * x1 x_scaled_bias = x_scaled - if (has_bias): + if has_bias: x_scaled_bias = x_scaled + bias x_scaled_bias = x_scaled_bias.reshape((batch_size, src_len, d_model)) return x_scaled_bias @@ -56,12 +58,12 @@ def compute_reference(x, residual, ln_scale, ln_bias, linear_bias): linear_bias_dropout_out = linear_bias_out linear_bias_dropout_residual_out = residual + linear_bias_dropout_out linear_bias_dropout_residual_ln_out = layer_norm( - linear_bias_dropout_residual_out, True, has_bias, ln_scale, ln_bias) + linear_bias_dropout_residual_out, True, has_bias, ln_scale, ln_bias + ) return linear_bias_dropout_residual_ln_out class TestFusedBiasDropoutResidualLayerNormAPI(unittest.TestCase): - def setUp(self): self.setXType() self.setBiasAttr() @@ -84,50 +86,56 @@ class TestFusedBiasDropoutResidualLayerNormAPI(unittest.TestCase): self.weight_attr = None def generate_input_data(self): - self.x = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) - self.residual = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.x = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) + self.residual = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) def run_imperative(self): fused_bias_dropout_residual_ln = FusedBiasDropoutResidualLayerNorm( - self.embed_dim, self.dropout_prob, self.weight_attr, self.bias_attr) + self.embed_dim, self.dropout_prob, self.weight_attr, self.bias_attr + ) linear_bias = None if self.bias_attr is not False: linear_bias = np.random.random( - fused_bias_dropout_residual_ln.linear_bias.shape).astype( - 'float32') + fused_bias_dropout_residual_ln.linear_bias.shape + ).astype('float32') fused_bias_dropout_residual_ln.linear_bias.set_value( - paddle.to_tensor(linear_bias)) - out = fused_bias_dropout_residual_ln(paddle.to_tensor(self.x), - paddle.to_tensor(self.residual)) + paddle.to_tensor(linear_bias) + ) + out = fused_bias_dropout_residual_ln( + paddle.to_tensor(self.x), paddle.to_tensor(self.residual) + ) ln_bias = None if self.bias_attr is not False: ln_bias = fused_bias_dropout_residual_ln.ln_bias.numpy() - ln_scale = fused_bias_dropout_residual_ln.ln_scale.numpy(), - ref_out = compute_reference(self.x, self.residual, ln_scale, ln_bias, - linear_bias) - np.testing.assert_allclose(ref_out, - out.numpy(), - rtol=1e-5, - atol=self.atol) + ln_scale = (fused_bias_dropout_residual_ln.ln_scale.numpy(),) + ref_out = compute_reference( + self.x, self.residual, ln_scale, ln_bias, linear_bias + ) + np.testing.assert_allclose( + ref_out, out.numpy(), rtol=1e-5, atol=self.atol + ) def run_static(self): - fused_op = FusedBiasDropoutResidualLayerNorm(self.embed_dim, - self.dropout_prob, - self.weight_attr, - self.bias_attr) + fused_op = FusedBiasDropoutResidualLayerNorm( + self.embed_dim, self.dropout_prob, self.weight_attr, self.bias_attr + ) x = paddle.static.data( name='X', shape=[self.batch_size, self.query_length, self.embed_dim], - dtype=self.x_type) + dtype=self.x_type, + ) residual = paddle.static.data( name='Residual', shape=[self.batch_size, self.query_length, self.embed_dim], - dtype=self.x_type) + dtype=self.x_type, + ) final_out = fused_op(x, residual) place = paddle.CUDAPlace(0) @@ -137,31 +145,31 @@ class TestFusedBiasDropoutResidualLayerNormAPI(unittest.TestCase): linear_bias = None ln_bias = None if self.bias_attr is False: - out, ln_scale = exe.run(paddle.static.default_main_program(), - feed={ - "X": self.x, - "Residual": self.residual - }, - fetch_list=[final_out, fused_op.ln_scale]) + out, ln_scale = exe.run( + paddle.static.default_main_program(), + feed={"X": self.x, "Residual": self.residual}, + fetch_list=[final_out, fused_op.ln_scale], + ) else: out, linear_bias, ln_scale, ln_bias = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.x, - "Residual": self.residual - }, + feed={"X": self.x, "Residual": self.residual}, fetch_list=[ - final_out, fused_op.linear_bias, fused_op.ln_scale, - fused_op.ln_bias - ]) + final_out, + fused_op.linear_bias, + fused_op.ln_scale, + fused_op.ln_bias, + ], + ) return out, linear_bias, ln_scale, ln_bias def test_static_api(self): paddle.enable_static() with paddle.static.program_guard(Program()): out, linear_bias, ln_scale, ln_bias = self.run_static() - ref_out = compute_reference(self.x, self.residual, ln_scale, ln_bias, - linear_bias) + ref_out = compute_reference( + self.x, self.residual, ln_scale, ln_bias, linear_bias + ) np.testing.assert_allclose(ref_out, out, rtol=1e-5, atol=self.atol) def test_dynamic_api(self): @@ -170,8 +178,8 @@ class TestFusedBiasDropoutResidualLayerNormAPI(unittest.TestCase): class TestFusedBiasDropoutResidualLayerNormAPIBiasIsNone( - TestFusedBiasDropoutResidualLayerNormAPI): - + TestFusedBiasDropoutResidualLayerNormAPI +): def setBiasAttr(self): self.bias_attr = False diff --git a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py index d44c6720d8b..cc672f5a16f 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py @@ -32,14 +32,10 @@ from op_test import OpTest # TestFusedElementwiseActivationOp_channelwise_add -def create_test_class(test_case, - callback, - attrs, - dtype=np.float32, - grad_chek=True): - +def create_test_class( + test_case, callback, attrs, dtype=np.float32, grad_chek=True +): class TestFusedElementwiseActivationOp_base(OpTest): - def setUp(self): self.op_type = "fused_elemwise_activation" self.dtype = dtype @@ -54,12 +50,12 @@ def create_test_class(test_case, self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } if self.attrs["save_intermediate_out"]: self.outputs = { 'Out': self.out, - "IntermediateOut": self.intermediate_out + "IntermediateOut": self.intermediate_out, } else: self.outputs = {'Out': self.out} @@ -70,8 +66,9 @@ def create_test_class(test_case, self.axis = -1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y + ) def init_attr(self): self.attrs = { @@ -101,133 +98,152 @@ def create_test_class(test_case, if not grad_chek: return if self.attrs["save_intermediate_out"]: - self.check_grad(['Y'], ['Out'], - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], + ['Out'], + max_relative_error=0.005, + no_grad_set=set("X"), + ) else: - self.check_grad(['Y'], ['Out'], - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], + ['Out'], + max_relative_error=0.005, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if not grad_chek: return if self.attrs["save_intermediate_out"]: - self.check_grad(['X'], ['Out'], - max_relative_error=0.005, - no_grad_set=set("Y")) + self.check_grad( + ['X'], + ['Out'], + max_relative_error=0.005, + no_grad_set=set("Y"), + ) else: - self.check_grad(['X'], ['Out'], - max_relative_error=0.005, - no_grad_set=set("Y")) + self.check_grad( + ['X'], + ['Out'], + max_relative_error=0.005, + no_grad_set=set("Y"), + ) class TestFusedElementwiseActivationOp_scalar( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) class TestFusedElementwiseActivationOp_scalar2( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) class TestFusedElementwiseActivationOp_Vector( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): - self.x = np.random.random((32, )).astype(self.dtype) - self.y = np.random.random((32, )).astype(self.dtype) + self.x = np.random.random((32,)).astype(self.dtype) + self.y = np.random.random((32,)).astype(self.dtype) class TestFusedElementwiseActivationOp_broadcast_0( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(2).astype(self.dtype) self.axis = 0 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(2, 1, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(2, 1, 1) + ) class TestFusedElementwiseActivationOp_broadcast_1( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(3).astype(self.dtype) self.axis = 1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 3, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 3, 1) + ) class TestFusedElementwiseActivationOp_broadcast_2( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(4).astype(self.dtype) def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 1, 4)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 1, 4) + ) class TestFusedElementwiseActivationOp_broadcast_3( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4, 5).astype(self.dtype) self.y = np.random.rand(3, 4).astype(self.dtype) self.axis = 1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 3, 4, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 3, 4, 1) + ) class TestFusedElementwiseActivationOp_broadcast_4( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4, 5).astype(self.dtype) self.y = np.random.rand(2, 1).astype(self.dtype) self.axis = 0 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(2, 1, 1, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(2, 1, 1, 1) + ) class TestFusedElementwiseActivationOp_rowwise_add_0( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(3, 4).astype(self.dtype) self.axis = 1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 3, 4)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 3, 4) + ) class TestFusedElementwiseActivationOp_rowwise_add_1( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) self.axis = 1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 1) + ) class TestFusedElementwiseActivationOp_channelwise_add( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(3, 20, 20).astype(self.dtype) self.y = np.random.rand(3, 1, 1).astype(self.dtype) @@ -236,36 +252,59 @@ def create_test_class(test_case, TestFusedElementwiseActivationOp_scalar.__name__ = test_case + "_scalar" TestFusedElementwiseActivationOp_scalar2.__name__ = test_case + "_scalar2" TestFusedElementwiseActivationOp_Vector.__name__ = test_case + "_Vector" - TestFusedElementwiseActivationOp_broadcast_0.__name__ = test_case + "_broadcast_0" - TestFusedElementwiseActivationOp_broadcast_1.__name__ = test_case + "_broadcast_1" - TestFusedElementwiseActivationOp_broadcast_2.__name__ = test_case + "_broadcast_2" - TestFusedElementwiseActivationOp_broadcast_3.__name__ = test_case + "_broadcast_3" - TestFusedElementwiseActivationOp_broadcast_4.__name__ = test_case + "_broadcast_4" - TestFusedElementwiseActivationOp_rowwise_add_0.__name__ = test_case + "_rowwise_add_0" - TestFusedElementwiseActivationOp_rowwise_add_1.__name__ = test_case + "_rowwise_add_1" - TestFusedElementwiseActivationOp_channelwise_add.__name__ = test_case + "_channelwise_add" + TestFusedElementwiseActivationOp_broadcast_0.__name__ = ( + test_case + "_broadcast_0" + ) + TestFusedElementwiseActivationOp_broadcast_1.__name__ = ( + test_case + "_broadcast_1" + ) + TestFusedElementwiseActivationOp_broadcast_2.__name__ = ( + test_case + "_broadcast_2" + ) + TestFusedElementwiseActivationOp_broadcast_3.__name__ = ( + test_case + "_broadcast_3" + ) + TestFusedElementwiseActivationOp_broadcast_4.__name__ = ( + test_case + "_broadcast_4" + ) + TestFusedElementwiseActivationOp_rowwise_add_0.__name__ = ( + test_case + "_rowwise_add_0" + ) + TestFusedElementwiseActivationOp_rowwise_add_1.__name__ = ( + test_case + "_rowwise_add_1" + ) + TestFusedElementwiseActivationOp_channelwise_add.__name__ = ( + test_case + "_channelwise_add" + ) globals()[test_case + "_base"] = TestFusedElementwiseActivationOp_base globals()[test_case + "_scalar"] = TestFusedElementwiseActivationOp_scalar globals()[test_case + "_scalar2"] = TestFusedElementwiseActivationOp_scalar2 globals()[test_case + "_Vector"] = TestFusedElementwiseActivationOp_Vector - globals()[test_case + - "_broadcast_0"] = TestFusedElementwiseActivationOp_broadcast_0 - globals()[test_case + - "_broadcast_1"] = TestFusedElementwiseActivationOp_broadcast_1 - globals()[test_case + - "_broadcast_2"] = TestFusedElementwiseActivationOp_broadcast_2 - globals()[test_case + - "_broadcast_3"] = TestFusedElementwiseActivationOp_broadcast_3 - globals()[test_case + - "_broadcast_4"] = TestFusedElementwiseActivationOp_broadcast_4 - globals()[test_case + - "_rowwise_add_0"] = TestFusedElementwiseActivationOp_rowwise_add_0 - globals()[test_case + - "_rowwise_add_1"] = TestFusedElementwiseActivationOp_rowwise_add_1 globals()[ - test_case + - "_channelwise_add"] = TestFusedElementwiseActivationOp_channelwise_add + test_case + "_broadcast_0" + ] = TestFusedElementwiseActivationOp_broadcast_0 + globals()[ + test_case + "_broadcast_1" + ] = TestFusedElementwiseActivationOp_broadcast_1 + globals()[ + test_case + "_broadcast_2" + ] = TestFusedElementwiseActivationOp_broadcast_2 + globals()[ + test_case + "_broadcast_3" + ] = TestFusedElementwiseActivationOp_broadcast_3 + globals()[ + test_case + "_broadcast_4" + ] = TestFusedElementwiseActivationOp_broadcast_4 + globals()[ + test_case + "_rowwise_add_0" + ] = TestFusedElementwiseActivationOp_rowwise_add_0 + globals()[ + test_case + "_rowwise_add_1" + ] = TestFusedElementwiseActivationOp_rowwise_add_1 + globals()[ + test_case + "_channelwise_add" + ] = TestFusedElementwiseActivationOp_channelwise_add def scale_add_func(x, y, x_bcast, y_bcast, scale, mode=0): @@ -338,97 +377,129 @@ for mode in {0, 1}: gelu_add_func = partial(gelu_add_func, mode=mode) for save_intermediate_out in {True, False}: - suffix = ("_save_intermediate_out" if save_intermediate_out else "") \ - + ("_mode_"+ str(mode)) + suffix = ("_save_intermediate_out" if save_intermediate_out else "") + ( + "_mode_" + str(mode) + ) create_test_class( - 'scale_add' + suffix, scale_add_func, { + 'scale_add' + suffix, + scale_add_func, + { 'scale': scale, 'functor_list': ["scale", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'add_scale' + suffix, add_scale_func, { + 'add_scale' + suffix, + add_scale_func, + { 'scale': scale, 'functor_list': ["elementwise_add", "scale"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'add_relu' + suffix, add_relu_func, { + 'add_relu' + suffix, + add_relu_func, + { 'functor_list': ["elementwise_add", "relu"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'relu_add' + suffix, relu_add_func, { + 'relu_add' + suffix, + relu_add_func, + { 'functor_list': ["relu", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'mul_scale' + suffix, mul_scale_func, { + 'mul_scale' + suffix, + mul_scale_func, + { 'scale': scale, 'functor_list': ["elementwise_mul", "scale"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'gelu_add' + suffix, gelu_add_func, { + 'gelu_add' + suffix, + gelu_add_func, + { 'functor_list': ["gelu", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) if core.is_compiled_with_cuda(): create_test_class( 'scale_add_fp16' + suffix, - scale_add_func, { + scale_add_func, + { 'scale': scale, 'functor_list': ["scale", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'add_scale_fp16' + suffix, - add_scale_func, { + add_scale_func, + { 'scale': scale, 'functor_list': ["elementwise_add", "scale"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'add_relu_fp16' + suffix, - add_relu_func, { + add_relu_func, + { 'functor_list': ["elementwise_add", "relu"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'relu_add_fp16' + suffix, - relu_add_func, { + relu_add_func, + { 'functor_list': ["relu", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'mul_scale_fp16' + suffix, - mul_scale_func, { + mul_scale_func, + { 'scale': scale, 'functor_list': ["elementwise_mul", "scale"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'gelu_add_fp16' + suffix, - gelu_add_func, { + gelu_add_func, + { 'functor_list': ["gelu", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fused_emb_seq_pool_op.py b/python/paddle/fluid/tests/unittests/test_fused_emb_seq_pool_op.py index 5963ef07e92..392c04f730a 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_emb_seq_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_emb_seq_pool_op.py @@ -19,27 +19,34 @@ from op_test import OpTest, skip_check_grad_ci import paddle.version as ver -@skip_check_grad_ci(reason="check_grad is called when ver.mkl() == ON" - "and 'Linux' in platform.platform().") +@skip_check_grad_ci( + reason="check_grad is called when ver.mkl() == ON" + "and 'Linux' in platform.platform()." +) class TestFusedEmbeddingSeqPoolOp(OpTest): - def setUp(self): self.op_type = "fused_embedding_seq_pool" self.emb_size = 6 self.table = np.random.random((17, self.emb_size)).astype("float64") - self.ids = np.array([[[4], [3]], [[4], [3]], [[2], [1]], - [[16], [1]]]).astype("int64") + self.ids = np.array( + [[[4], [3]], [[4], [3]], [[2], [1]], [[16], [1]]] + ).astype("int64") ids_expand = np.expand_dims(self.ids, axis=1) self.lod = [[3, 1]] self.attrs = {'is_sparse': True} self.inputs = {'W': self.table, 'Ids': (ids_expand, self.lod)} self.outputs = { - 'Out': - np.reshape( - np.array([ - self.table[[4, 3]] + self.table[[4, 3]] + - self.table[[2, 1]], self.table[[16, 1]] - ]), [len(self.lod[0]), 2 * self.emb_size]) + 'Out': np.reshape( + np.array( + [ + self.table[[4, 3]] + + self.table[[4, 3]] + + self.table[[2, 1]], + self.table[[16, 1]], + ] + ), + [len(self.lod[0]), 2 * self.emb_size], + ) } def test_check_output(self): @@ -50,14 +57,12 @@ class TestFusedEmbeddingSeqPoolOp(OpTest): # TODO(wangzhongpu): support lod in dygraph mode if ver.mkl() == "ON" and 'Linux' in platform.platform(): self.attrs = {'is_sparse': False} - self.check_grad(['W'], - 'Out', - no_grad_set=['Ids'], - check_dygraph=False) + self.check_grad( + ['W'], 'Out', no_grad_set=['Ids'], check_dygraph=False + ) class TestLookupTableOpWithPadding(TestFusedEmbeddingSeqPoolOp): - def test_check_output(self): if ver.mkl() == "ON" and 'Linux' in platform.platform(): ids = np.squeeze(self.ids, axis=2) @@ -65,18 +70,20 @@ class TestLookupTableOpWithPadding(TestFusedEmbeddingSeqPoolOp): output = list() index = 0 for count in self.lod[0]: - arr = ids[index:count + index] - out = np.reshape(self.table[arr.flatten()], - [arr.shape[0], arr.shape[1], self.emb_size]) + arr = ids[index : count + index] + out = np.reshape( + self.table[arr.flatten()], + [arr.shape[0], arr.shape[1], self.emb_size], + ) idx = np.argwhere(arr == padding_idx) for item in idx: out[item[0], item[1], :] = np.zeros(self.emb_size) output.append(np.sum(out, 0)) index += count self.outputs = { - 'Out': - np.reshape(np.array(output), - [len(self.lod[0]), 2 * self.emb_size]) + 'Out': np.reshape( + np.array(output), [len(self.lod[0]), 2 * self.emb_size] + ) } self.attrs = {'padding_idx': int(padding_idx)} # TODO(wangzhongpu): support lod in dygraph mode @@ -88,30 +95,28 @@ class TestLookupTableOpWithPadding(TestFusedEmbeddingSeqPoolOp): padding_idx = np.random.choice(ids.flatten(), 1)[0] self.attrs = {'padding_idx': int(padding_idx), 'is_sparse': False} # TODO(wangzhongpu): support lod in dygraph mode - self.check_grad(['W'], - 'Out', - no_grad_set=['Ids'], - check_dygraph=False) + self.check_grad( + ['W'], 'Out', no_grad_set=['Ids'], check_dygraph=False + ) class TestFusedEmbeddingSeqPoolApi(unittest.TestCase): - def test_api(self): if ver.mkl() == "ON" and 'Linux' in platform.platform(): import paddle.fluid as fluid dict_size = 20 - data_t = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=1) + data_t = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=1 + ) padding_idx = np.random.randint(1, 10) out = fluid.contrib.fused_embedding_seq_pool( input=data_t, size=[dict_size, 32], param_attr='w', padding_idx=padding_idx, - is_sparse=False) + is_sparse=False, + ) place = fluid.CPUPlace() exe = fluid.Executor(place) diff --git a/python/paddle/fluid/tests/unittests/test_fused_embedding_fc_lstm_op.py b/python/paddle/fluid/tests/unittests/test_fused_embedding_fc_lstm_op.py index 4d1b38e54ee..54fc4d77b79 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_embedding_fc_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_embedding_fc_lstm_op.py @@ -23,30 +23,41 @@ def fc(x, w, b): def fused_embedded_fc_lstm( - ids, # T x 1 - lod, # 1 x N - embeddings=None, # Dict_size x M - wx=None, # M x 4D - bx=None, # 1 x 4D - h0=None, # N x D - c0=None, # N x D - w_h=None, # D x 4D - w_b=None, # 1 x 4D - w_c=None, # 1 x 3D - is_reverse=False, - act_gate=None, - act_cell=None, - act_cand=None): + ids, # T x 1 + lod, # 1 x N + embeddings=None, # Dict_size x M + wx=None, # M x 4D + bx=None, # 1 x 4D + h0=None, # N x D + c0=None, # N x D + w_h=None, # D x 4D + w_b=None, # 1 x 4D + w_c=None, # 1 x 3D + is_reverse=False, + act_gate=None, + act_cell=None, + act_cand=None, +): # Make a lookup for embeddings and pass result into lstm reference T = ids.shape[0] M = embeddings.shape[1] x = embeddings[ids].reshape([T, M]) - return lstm(fc(x, wx, bx), lod, h0, c0, w_h, w_b, w_c, is_reverse, act_gate, - act_cell, act_cand) + return lstm( + fc(x, wx, bx), + lod, + h0, + c0, + w_h, + w_b, + w_c, + is_reverse, + act_gate, + act_cell, + act_cand, + ) class TestFusionLSTMOp(OpTest): - def set_conf(self): pass @@ -76,22 +87,24 @@ class TestFusionLSTMOp(OpTest): b = np.random.normal(size=(1, 7 * self.D)).astype('float32') else: b = np.random.normal(size=(1, 4 * self.D)).astype('float32') - w_b = np.copy(b[:, 0:4 * self.D]) - w_c = b[:, 4 * self.D:] if self.use_peepholes else None + w_b = np.copy(b[:, 0 : 4 * self.D]) + w_c = b[:, 4 * self.D :] if self.use_peepholes else None # low is 0 , high is voc_size - 1 - ids = np.random.randint(low=0, high=self.dict_size - 1, - size=(T, 1)).astype("int64") + ids = np.random.randint( + low=0, high=self.dict_size - 1, size=(T, 1) + ).astype("int64") # embeddings as they were trained , so each entry is of M size - embeddings = np.random.random( - (self.dict_size, self.M)).astype("float32") + embeddings = np.random.random((self.dict_size, self.M)).astype( + "float32" + ) # multiply embeddings via Weights fc_embeddings = np.dot(embeddings, wx) # bias should be manually added into the bias of this fused embedding fc LSTM - b[0, 0:4 * self.D] += bx[0, :] - combined_biases = b[:, 0:4 * self.D] + b[0, 0 : 4 * self.D] += bx[0, :] + combined_biases = b[:, 0 : 4 * self.D] # So let broadcast it , so they can be added ones = np.ones([self.dict_size, 1]) broadcasted_biases = np.dot(ones, combined_biases) @@ -107,17 +120,28 @@ class TestFusionLSTMOp(OpTest): wh = np.random.normal(size=(self.D, 4 * self.D)).astype('float32') - h, c = fused_embedded_fc_lstm(ids, self.lod, embeddings, wx, bx, h0, c0, - wh, w_b, w_c, self.is_reverse, - ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + h, c = fused_embedded_fc_lstm( + ids, + self.lod, + embeddings, + wx, + bx, + h0, + c0, + wh, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = { 'Ids': (ids, self.lod), 'Embeddings': fc_embeddings, 'WeightH': wh, - 'Bias': b + 'Bias': b, } if self.has_initial_state: @@ -133,7 +157,7 @@ class TestFusionLSTMOp(OpTest): 'is_reverse': self.is_reverse, 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, - 'candidate_activation': self.act_cand + 'candidate_activation': self.act_cand, } def test_check_output(self): @@ -143,74 +167,63 @@ class TestFusionLSTMOp(OpTest): class TestFusionLSTMOpInit(TestFusionLSTMOp): - def set_conf(self): self.has_initial_state = True class TestFusionLSTMOpReverse(TestFusionLSTMOp): - def set_conf(self): self.is_reverse = True class TestFusionLSTMOpInitReverse(TestFusionLSTMOp): - def set_conf(self): self.has_initial_state = True self.is_reverse = True class TestFusionLSTMOpMD1(TestFusionLSTMOp): - def set_conf(self): self.M = 36 self.D = 8 class TestFusionLSTMOpMD2(TestFusionLSTMOp): - def set_conf(self): self.M = 8 self.D = 8 class TestFusionLSTMOpMD3(TestFusionLSTMOp): - def set_conf(self): self.M = 15 self.D = 3 class TestFusionLSTMOpBS1(TestFusionLSTMOp): - def set_conf(self): self.lod = [[3]] self.D = 16 class TestFusionLSTMOpPeepholes(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True class TestFusionLSTMOpPeepholesInit(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True class TestFusionLSTMOpPeepholesReverse(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.is_reverse = True class TestFusionLSTMOpPeepholesInitReverse(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True @@ -218,7 +231,6 @@ class TestFusionLSTMOpPeepholesInitReverse(TestFusionLSTMOp): class TestFusionLSTMOpPeepholesBS1(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.lod = [[2]] diff --git a/python/paddle/fluid/tests/unittests/test_fused_fc_elementwise_layernorm_op.py b/python/paddle/fluid/tests/unittests/test_fused_fc_elementwise_layernorm_op.py index fa9948128de..09797b03849 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_fc_elementwise_layernorm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_fc_elementwise_layernorm_op.py @@ -22,10 +22,10 @@ from test_layer_norm_op import _reference_layer_norm_naive np.random.random(123) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Paddle core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Paddle core is not compiled with CUDA" +) class TestFusedFCElementwiseLayerNormOp(OpTest): - def config(self): self.matrix = MatrixGenerate(1, 10, 15, 3, 3, 2) self.y_shape = [1, 15] @@ -44,11 +44,12 @@ class TestFusedFCElementwiseLayerNormOp(OpTest): y = np.random.random_sample(self.y_shape).astype(np.float32) add_out = fc_out + y # layer_norm - scale_shape = [np.prod(self.y_shape[self.begin_norm_axis:])] + scale_shape = [np.prod(self.y_shape[self.begin_norm_axis :])] scale = np.random.random_sample(scale_shape).astype(np.float32) bias_1 = np.random.random_sample(scale_shape).astype(np.float32) out, mean, variance = _reference_layer_norm_naive( - add_out, scale, bias_1, epsilon, self.begin_norm_axis) + add_out, scale, bias_1, epsilon, self.begin_norm_axis + ) self.inputs = { "X": self.matrix.input, @@ -56,12 +57,12 @@ class TestFusedFCElementwiseLayerNormOp(OpTest): "Bias0": self.matrix.bias, "Y": y, "Scale": scale, - "Bias1": bias_1 + "Bias1": bias_1, } self.attrs = { "activation_type": "relu", "epsilon": epsilon, - "begin_norm_axis": self.begin_norm_axis + "begin_norm_axis": self.begin_norm_axis, } self.outputs = {"Out": out, "Mean": mean, "Variance": variance} @@ -71,7 +72,6 @@ class TestFusedFCElementwiseLayerNormOp(OpTest): class TestFusedFCElementwiseLayerNormOp2(TestFusedFCElementwiseLayerNormOp): - def config(self): self.matrix = MatrixGenerate(4, 5, 6, 2, 2, 1) self.y_shape = [4, 6] diff --git a/python/paddle/fluid/tests/unittests/test_fused_feedforward_op.py b/python/paddle/fluid/tests/unittests/test_fused_feedforward_op.py index 85b7ee9285d..71bf2bad2cc 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_feedforward_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_feedforward_op.py @@ -25,7 +25,6 @@ from paddle.fluid.framework import default_main_program class TestFusedFFNOp(OpTest): - def getDtype(self): self.dtype = "float32" self.layer_norm_dtype = "float32" @@ -54,7 +53,7 @@ class TestFusedFFNOp(OpTest): def setUp(self): paddle.disable_static() self.__class__.op_type = "fused_feedforward" - #check grad in test_out_and_grad() + # check grad in test_out_and_grad() self.__class__.no_need_check_grad = True self.getDtype() self.getShape() @@ -66,17 +65,23 @@ class TestFusedFFNOp(OpTest): self.bias_attr = None self.weight_attrs = transformer._convert_param_attr_to_list( - self.weight_attr, 2) + self.weight_attr, 2 + ) self.bias_attrs = transformer._convert_param_attr_to_list( - self.bias_attr, 2) - self.linear1 = Linear(self.d_model, - self.dim_feedforward, - self.weight_attrs[1], - bias_attr=self.bias_attrs[1]) - self.linear2 = Linear(self.dim_feedforward, - self.d_model, - self.weight_attrs[1], - bias_attr=self.bias_attrs[1]) + self.bias_attr, 2 + ) + self.linear1 = Linear( + self.d_model, + self.dim_feedforward, + self.weight_attrs[1], + bias_attr=self.bias_attrs[1], + ) + self.linear2 = Linear( + self.dim_feedforward, + self.d_model, + self.weight_attrs[1], + bias_attr=self.bias_attrs[1], + ) paddle.set_default_dtype(self.layer_norm_dtype) self.norm1 = LayerNorm(self.d_model) @@ -86,10 +91,12 @@ class TestFusedFFNOp(OpTest): self.dropout2 = Dropout(0.0, mode="upscale_in_train") self.activation = getattr(F, self.act_method) - self.src = np.random.random((self.batch_size, self.query_length, - self.d_model)).astype(self.dtype) - self.dout = np.random.random((self.batch_size, self.query_length, - self.d_model)).astype(self.dtype) + self.src = np.random.random( + (self.batch_size, self.query_length, self.d_model) + ).astype(self.dtype) + self.dout = np.random.random( + (self.batch_size, self.query_length, self.d_model) + ).astype(self.dtype) def Base(self): paddle.disable_static() @@ -98,46 +105,54 @@ class TestFusedFFNOp(OpTest): if self.pre_layer_norm: ln1_out = self.norm1(tensor_src) linear2_out = self.linear2( - self.dropout(self.activation(self.linear1(ln1_out)))) + self.dropout(self.activation(self.linear1(ln1_out))) + ) dropout2_out = residual + self.dropout2(linear2_out) - paddle.autograd.backward([dropout2_out], - [paddle.to_tensor(self.dout)], True) + paddle.autograd.backward( + [dropout2_out], [paddle.to_tensor(self.dout)], True + ) return dropout2_out, tensor_src.grad else: linear2_out = self.linear2( - self.dropout(self.activation(self.linear1(tensor_src)))) + self.dropout(self.activation(self.linear1(tensor_src))) + ) dropout2_out = residual + self.dropout2(linear2_out) dropout2_out = self.norm2(dropout2_out) - paddle.autograd.backward([dropout2_out], - [paddle.to_tensor(self.dout)], True) + paddle.autograd.backward( + [dropout2_out], [paddle.to_tensor(self.dout)], True + ) return dropout2_out, tensor_src.grad def FusedFFN(self): paddle.disable_static() - linear1_weight = paddle.to_tensor(self.linear1.weight, - stop_gradient=False) + linear1_weight = paddle.to_tensor( + self.linear1.weight, stop_gradient=False + ) linear1_bias = paddle.to_tensor(self.linear1.bias, stop_gradient=False) - linear2_weight = paddle.to_tensor(self.linear2.weight, - stop_gradient=False) + linear2_weight = paddle.to_tensor( + self.linear2.weight, stop_gradient=False + ) linear2_bias = paddle.to_tensor(self.linear2.bias, stop_gradient=False) ln1_scale = paddle.to_tensor(self.norm1.weight, stop_gradient=False) ln1_bias = paddle.to_tensor(self.norm1.bias, stop_gradient=False) ln2_scale = paddle.to_tensor(self.norm2.weight, stop_gradient=False) ln2_bias = paddle.to_tensor(self.norm2.bias, stop_gradient=False) x = paddle.to_tensor(self.src, stop_gradient=False) - out = incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - linear1_bias, - linear2_bias, - ln1_scale, - ln1_bias, - ln2_scale, - ln2_bias, - 0.0, - 0.0, - activation=self.act_method, - pre_layer_norm=self.pre_layer_norm) + out = incubate_f.fused_feedforward( + x, + linear1_weight, + linear2_weight, + linear1_bias, + linear2_bias, + ln1_scale, + ln1_bias, + ln2_scale, + ln2_bias, + 0.0, + 0.0, + activation=self.act_method, + pre_layer_norm=self.pre_layer_norm, + ) paddle.autograd.backward([out], [paddle.to_tensor(self.dout)]) return out, x.grad @@ -145,18 +160,18 @@ class TestFusedFFNOp(OpTest): default_main_program().random_seed = 42 base_out, base_grad = self.Base() fused_out, fused_grad = self.FusedFFN() - np.testing.assert_allclose(base_out.numpy(), - fused_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(base_grad.numpy(), - fused_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + base_out.numpy(), fused_out.numpy(), rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + base_grad.numpy(), + fused_grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) class TestFusedFFNOpFp16(TestFusedFFNOp): - def getDtype(self): self.dtype = "float16" self.layer_norm_dtype = "float32" @@ -173,20 +188,17 @@ class TestFusedFFNOpFp16(TestFusedFFNOp): class TestFusedFFNOpFp64(TestFusedFFNOp): - def getDtype(self): self.dtype = "float64" self.layer_norm_dtype = "float64" class TestFusedFFNOpActivation(TestFusedFFNOp): - def getActivation(self): self.act_method = "relu" class TestFusedFFNOpNormalizeBefore(TestFusedFFNOp): - def getNormalizeBefore(self): self.pre_layer_norm = True @@ -198,7 +210,6 @@ class TestFusedFFNOpNormalizeBefore(TestFusedFFNOp): class APITestStaticFusedFFN(unittest.TestCase): - def test_static(self): paddle.enable_static() default_main_program().random_seed = 42 @@ -208,36 +219,39 @@ class APITestStaticFusedFFN(unittest.TestCase): d_model = 8 dim_feedforward = 8 - x = paddle.static.data(name='x', - shape=[batch_size, d_model, dim_feedforward], - dtype=dtype) - linear1_weight = paddle.static.data(name='linear1_weight', - shape=[d_model, dim_feedforward], - dtype=dtype) - linear1_bias = paddle.static.data(name='linear1_bias', - shape=[dim_feedforward]) - linear2_weight = paddle.static.data(name='linear2_weight', - shape=[dim_feedforward, d_model], - dtype=dtype) + x = paddle.static.data( + name='x', shape=[batch_size, d_model, dim_feedforward], dtype=dtype + ) + linear1_weight = paddle.static.data( + name='linear1_weight', shape=[d_model, dim_feedforward], dtype=dtype + ) + linear1_bias = paddle.static.data( + name='linear1_bias', shape=[dim_feedforward] + ) + linear2_weight = paddle.static.data( + name='linear2_weight', shape=[dim_feedforward, d_model], dtype=dtype + ) linear2_bias = paddle.static.data(name='linear2_bias', shape=[d_model]) ln1_scale = paddle.static.data(name='ln1_scale', shape=[d_model]) ln1_bias = paddle.static.data(name='ln1_scale', shape=[d_model]) ln2_scale = paddle.static.data(name='ln2_scale', shape=[d_model]) ln2_bias = paddle.static.data(name='ln2_scale', shape=[d_model]) - fused_out = incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - linear1_bias, - linear2_bias, - ln1_scale, - ln1_bias, - ln2_scale, - ln2_bias, - 0.0, - 0.0, - activation="relu", - pre_layer_norm=False) + fused_out = incubate_f.fused_feedforward( + x, + linear1_weight, + linear2_weight, + linear1_bias, + linear2_bias, + ln1_scale, + ln1_bias, + ln2_scale, + ln2_bias, + 0.0, + 0.0, + activation="relu", + pre_layer_norm=False, + ) ######base ffn###### linear1_out = F.linear(x, linear1_weight, linear1_bias) @@ -245,21 +259,26 @@ class APITestStaticFusedFFN(unittest.TestCase): dropout1_out = F.dropout(x=act_out, p=0.0, training=False) linear2_out = F.linear(dropout1_out, linear2_weight, linear2_bias) dropout2_out = x + F.dropout(x=linear2_out, p=0.0, training=False) - ln_out = F.layer_norm(dropout2_out, - normalized_shape=list([d_model]), - weight=ln2_scale, - bias=ln2_bias) + ln_out = F.layer_norm( + dropout2_out, + normalized_shape=list([d_model]), + weight=ln2_scale, + bias=ln2_bias, + ) ######base ffn###### exe = paddle.static.Executor(paddle.CUDAPlace(0)) x_data = np.random.random( - (batch_size, d_model, dim_feedforward)).astype(dtype) + (batch_size, d_model, dim_feedforward) + ).astype(dtype) linear1_weight_data = np.random.random( - (d_model, dim_feedforward)).astype(dtype) + (d_model, dim_feedforward) + ).astype(dtype) linear1_bias_data = np.zeros((dim_feedforward)).astype(dtype) linear2_weight_data = np.random.random( - (dim_feedforward, d_model)).astype(dtype) + (dim_feedforward, d_model) + ).astype(dtype) linear2_bias_data = np.zeros((d_model)).astype(dtype) ln1_scale_data = np.ones((d_model)).astype(layer_norm_dtype) @@ -271,94 +290,92 @@ class APITestStaticFusedFFN(unittest.TestCase): real_res = [] for res in res_list: - fetch = exe.run(feed={ - 'x': x_data, - 'linear1_weight': linear1_weight_data, - 'linear1_bias': linear1_bias_data, - 'linear2_weight': linear2_weight_data, - 'linear2_bias': linear2_bias_data, - 'ln1_scale': ln1_scale_data, - 'ln1_bias': ln1_bias_data, - 'ln2_scale': ln2_scale_data, - 'ln2_bias': ln2_bias_data - }, - fetch_list=[res]) + fetch = exe.run( + feed={ + 'x': x_data, + 'linear1_weight': linear1_weight_data, + 'linear1_bias': linear1_bias_data, + 'linear2_weight': linear2_weight_data, + 'linear2_bias': linear2_bias_data, + 'ln1_scale': ln1_scale_data, + 'ln1_bias': ln1_bias_data, + 'ln2_scale': ln2_scale_data, + 'ln2_bias': ln2_bias_data, + }, + fetch_list=[res], + ) real_res.append(fetch) - np.testing.assert_allclose(real_res[0], - real_res[1], - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + real_res[0], real_res[1], rtol=1e-05, atol=0.001 + ) class TestFusedFFNOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): def test_dtype(): - x = paddle.static.data(name='x', - shape=[1, 10, 10], - dtype="int32") - linear1_weight = paddle.static.data(name='linear1_weight', - shape=[1, 10, 10], - dtype="float32") - linear2_weight = paddle.static.data(name='linear2_weight', - shape=[1, 10, 10], - dtype="float32") + x = paddle.static.data( + name='x', shape=[1, 10, 10], dtype="int32" + ) + linear1_weight = paddle.static.data( + name='linear1_weight', shape=[1, 10, 10], dtype="float32" + ) + linear2_weight = paddle.static.data( + name='linear2_weight', shape=[1, 10, 10], dtype="float32" + ) incubate_f.fused_feedforward(x, linear1_weight, linear2_weight) self.assertRaises(TypeError, test_dtype) def test_dropout_rate_type(): - x = paddle.static.data(name='x1', - shape=[1, 10, 10], - dtype="float32") - linear1_weight = paddle.static.data(name='linear1_weight1', - shape=[10, 10], - dtype="float32") - linear2_weight = paddle.static.data(name='linear2_weight1', - shape=[10, 10], - dtype="float32") - incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - dropout1_rate="a") + x = paddle.static.data( + name='x1', shape=[1, 10, 10], dtype="float32" + ) + linear1_weight = paddle.static.data( + name='linear1_weight1', shape=[10, 10], dtype="float32" + ) + linear2_weight = paddle.static.data( + name='linear2_weight1', shape=[10, 10], dtype="float32" + ) + incubate_f.fused_feedforward( + x, linear1_weight, linear2_weight, dropout1_rate="a" + ) self.assertRaises(TypeError, test_dropout_rate_type) def test_dropout_rate_value(): - x = paddle.static.data(name='x2', - shape=[1, 10, 10], - dtype="float32") - linear1_weight = paddle.static.data(name='linear1_weight2', - shape=[10, 10], - dtype="float32") - linear2_weight = paddle.static.data(name='linear2_weight2', - shape=[10, 10], - dtype="float32") - incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - dropout2_rate=-1) + x = paddle.static.data( + name='x2', shape=[1, 10, 10], dtype="float32" + ) + linear1_weight = paddle.static.data( + name='linear1_weight2', shape=[10, 10], dtype="float32" + ) + linear2_weight = paddle.static.data( + name='linear2_weight2', shape=[10, 10], dtype="float32" + ) + incubate_f.fused_feedforward( + x, linear1_weight, linear2_weight, dropout2_rate=-1 + ) self.assertRaises(ValueError, test_dropout_rate_value) def test_dropout_mode(): - x = paddle.static.data(name='x3', - shape=[1, 10, 10], - dtype="float32") - linear1_weight = paddle.static.data(name='linear1_weight3', - shape=[10, 10], - dtype="float32") - linear2_weight = paddle.static.data(name='linear2_weight3', - shape=[10, 10], - dtype="float32") - incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - mode='test') + x = paddle.static.data( + name='x3', shape=[1, 10, 10], dtype="float32" + ) + linear1_weight = paddle.static.data( + name='linear1_weight3', shape=[10, 10], dtype="float32" + ) + linear2_weight = paddle.static.data( + name='linear2_weight3', shape=[10, 10], dtype="float32" + ) + incubate_f.fused_feedforward( + x, linear1_weight, linear2_weight, mode='test' + ) self.assertRaises(ValueError, test_dropout_mode) diff --git a/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py b/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py index 2baab279462..1eed59690d8 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py @@ -28,10 +28,10 @@ from paddle import _legacy_C_ops from paddle.fluid import core -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Paddle is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Paddle is not compiled with CUDA" +) class TestFusedGateAttentionOp(OpTest): - def setUp(self): self.__class__.op_type = "fused_gate_attention" # use autograd to check grad in this unittest. @@ -55,7 +55,6 @@ class TestFusedGateAttentionOp(OpTest): self.bias_attr = True def generate_input_data(self): - def _random(shape): if self.dtype == "bfloat16": data = np.random.random(shape).astype("float32") @@ -65,7 +64,8 @@ class TestFusedGateAttentionOp(OpTest): np.random.seed(123) self.query = _random( - (self.batch_size, self.msa_len, self.res_len, self.q_dim)) + (self.batch_size, self.msa_len, self.res_len, self.q_dim) + ) self.q_weight = _random((self.q_dim, self.num_heads, self.head_dim)) self.k_weight = _random((self.kv_dim, self.num_heads, self.head_dim)) self.v_weight = _random((self.kv_dim, self.num_heads, self.head_dim)) @@ -78,15 +78,18 @@ class TestFusedGateAttentionOp(OpTest): self.qkv_weight = np.stack([q_weight_t, k_weight_t, v_weight_t]) else: self.key = _random( - (self.batch_size, self.msa_len, self.m_size, self.kv_dim)) + (self.batch_size, self.msa_len, self.m_size, self.kv_dim) + ) self.qkv_weight = None self.attn_mask = _random( - (self.batch_size, self.msa_len, 1, 1, self.m_size)) + (self.batch_size, self.msa_len, 1, 1, self.m_size) + ) if self.bias_attr: self.nonbatched_bias = _random( - (self.batch_size, 1, self.num_heads, self.res_len, self.m_size)) + (self.batch_size, 1, self.num_heads, self.res_len, self.m_size) + ) if self.has_gating: self.gating_w = _random((self.q_dim, self.num_heads, self.head_dim)) @@ -96,12 +99,17 @@ class TestFusedGateAttentionOp(OpTest): self.output_b = _random((self.out_dim)) self.dout = _random( - (self.batch_size, self.msa_len, self.res_len, self.q_dim)) + (self.batch_size, self.msa_len, self.res_len, self.q_dim) + ) def collect_outputs(self, query, key, softmax_out, fmha_out, gate_out, out): outputs = [ - softmax_out, fmha_out, gate_out if self.has_gating else None, out, - query.grad, None if self.merge_qkv else key.grad + softmax_out, + fmha_out, + gate_out if self.has_gating else None, + out, + query.grad, + None if self.merge_qkv else key.grad, ] return outputs @@ -109,14 +117,17 @@ class TestFusedGateAttentionOp(OpTest): paddle.disable_static(place=paddle.CUDAPlace(0)) query = paddle.to_tensor(self.query, stop_gradient=False) - key = query if self.merge_qkv else paddle.to_tensor(self.key, - stop_gradient=False) + key = ( + query + if self.merge_qkv + else paddle.to_tensor(self.key, stop_gradient=False) + ) q_weight = paddle.to_tensor(self.q_weight, stop_gradient=False) k_weight = paddle.to_tensor(self.k_weight, stop_gradient=False) v_weight = paddle.to_tensor(self.v_weight, stop_gradient=False) src_mask = paddle.to_tensor(self.attn_mask, stop_gradient=True) - c = self.head_dim**(-0.5) + c = self.head_dim ** (-0.5) # [batch_size, msa_len, res_len, q_dim], [q_dim, num_heads, head_dim] # -> [batch_size, msa_len, res_len, num_heads, head_dim] q = paddle.einsum('nbqa,ahc->nbqhc', query, q_weight) * c @@ -134,8 +145,9 @@ class TestFusedGateAttentionOp(OpTest): # -> [batch_size, msa_len, num_heads, res_len, m_size] logits = logits + src_mask if self.bias_attr: - nonbatched_bias = paddle.to_tensor(self.nonbatched_bias, - stop_gradient=False) + nonbatched_bias = paddle.to_tensor( + self.nonbatched_bias, stop_gradient=False + ) # [batch_size, msa_len, num_heads, res_len, m_size], [batch_size, 1, num_heads, res_len, m_size] # -> [batch_size, msa_len, num_heads, res_len, m_size] logits = logits + nonbatched_bias @@ -157,14 +169,22 @@ class TestFusedGateAttentionOp(OpTest): # gate_values = paddle.einsum('nbqc,chv->nbqhv', query, # gating_w) + gating_b gating_w_2d = paddle.reshape( - gating_w, shape=[self.q_dim, self.num_heads * self.head_dim]) + gating_w, shape=[self.q_dim, self.num_heads * self.head_dim] + ) gate_values_4d = paddle.matmul(query, gating_w_2d) - gate_values = paddle.reshape( - gate_values_4d, - shape=[ - self.batch_size, self.msa_len, self.res_len, self.num_heads, - self.head_dim - ]) + gating_b + gate_values = ( + paddle.reshape( + gate_values_4d, + shape=[ + self.batch_size, + self.msa_len, + self.res_len, + self.num_heads, + self.head_dim, + ], + ) + + gating_b + ) gate_values = nn.functional.sigmoid(gate_values) gate_out = fmha_out * gate_values else: @@ -181,20 +201,32 @@ class TestFusedGateAttentionOp(OpTest): gate_out, shape=[ self.batch_size * self.msa_len * self.res_len, - self.num_heads * self.head_dim - ]) + self.num_heads * self.head_dim, + ], + ) output_w_2d = paddle.reshape( - output_w, shape=[self.num_heads * self.head_dim, self.out_dim]) + output_w, shape=[self.num_heads * self.head_dim, self.out_dim] + ) out_2d = paddle.matmul(gate_out_2d, output_w_2d) - out = paddle.reshape( - out_2d, - shape=[self.batch_size, self.msa_len, self.res_len, self.out_dim - ]) + output_b - - paddle.autograd.backward([out], [paddle.to_tensor(self.dout)], - retain_graph=True) - return self.collect_outputs(query, key, softmax_out, fmha_out, gate_out, - out) + out = ( + paddle.reshape( + out_2d, + shape=[ + self.batch_size, + self.msa_len, + self.res_len, + self.out_dim, + ], + ) + + output_b + ) + + paddle.autograd.backward( + [out], [paddle.to_tensor(self.dout)], retain_graph=True + ) + return self.collect_outputs( + query, key, softmax_out, fmha_out, gate_out, out + ) def get_fused_gate_attention_out(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -216,8 +248,9 @@ class TestFusedGateAttentionOp(OpTest): src_mask = paddle.to_tensor(self.attn_mask, stop_gradient=True) if self.bias_attr: - nonbatched_bias = paddle.to_tensor(self.nonbatched_bias, - stop_gradient=False) + nonbatched_bias = paddle.to_tensor( + self.nonbatched_bias, stop_gradient=False + ) else: nonbatched_bias = None if self.has_gating: @@ -230,18 +263,42 @@ class TestFusedGateAttentionOp(OpTest): output_w = paddle.to_tensor(self.output_w, stop_gradient=False) output_b = paddle.to_tensor(self.output_b, stop_gradient=False) - _, _, _, _, softmax_out, fmha_out, gate_out, out = _legacy_C_ops.fused_gate_attention( - query, key, q_weight, k_weight, v_weight, qkv_weight, - nonbatched_bias, src_mask, gating_w, gating_b, output_w, output_b, - 'has_gating', self.has_gating, 'merge_qkv', self.merge_qkv) - - paddle.autograd.backward([out], [paddle.to_tensor(self.dout)], - retain_graph=True) - return self.collect_outputs(query, key, softmax_out, fmha_out, gate_out, - out) + ( + _, + _, + _, + _, + softmax_out, + fmha_out, + gate_out, + out, + ) = _legacy_C_ops.fused_gate_attention( + query, + key, + q_weight, + k_weight, + v_weight, + qkv_weight, + nonbatched_bias, + src_mask, + gating_w, + gating_b, + output_w, + output_b, + 'has_gating', + self.has_gating, + 'merge_qkv', + self.merge_qkv, + ) + + paddle.autograd.backward( + [out], [paddle.to_tensor(self.dout)], retain_graph=True + ) + return self.collect_outputs( + query, key, softmax_out, fmha_out, gate_out, out + ) def check(self, ref, out, atol, rtol, check_equal, name): - def _convert(value): if self.dtype == "bfloat16": return convert_uint16_to_float(value) @@ -250,19 +307,25 @@ class TestFusedGateAttentionOp(OpTest): if check_equal: self.assertTrue( np.equal(_convert(ref), _convert(out)).all(), - "Checking < {} > failed!".format(name)) + "Checking < {} > failed!".format(name), + ) else: np.testing.assert_allclose( _convert(ref), _convert(out), atol=atol, rtol=rtol, - err_msg="Checking < {} > failed!".format(name)) + err_msg="Checking < {} > failed!".format(name), + ) def check_output_and_grad(self, atol, rtol): output_names = [ - "softmax_out", "fmha_out", "gate_out", "out", "query_grad", - "key_grad" + "softmax_out", + "fmha_out", + "gate_out", + "out", + "query_grad", + "key_grad", ] outputs_ref = self.get_reference_out() outputs_fused = self.get_fused_gate_attention_out() @@ -278,22 +341,26 @@ class TestFusedGateAttentionOp(OpTest): # that in fused ops, check_equal is set to False and we use allclose # to check the correctness. check_equal = False - self.check(ref_res.numpy(), fused_res.numpy(), atol, rtol, - check_equal, output_names[i]) + self.check( + ref_res.numpy(), + fused_res.numpy(), + atol, + rtol, + check_equal, + output_names[i], + ) def test_output_and_grad(self): self.check_output_and_grad(atol=1e-5, rtol=1e-6) class TestMergeQKVLargeBatchSizeCase(TestFusedGateAttentionOp): - def config(self): super().config() self.batch_size = 2 class TestSeparatedQKVCase(TestFusedGateAttentionOp): - def config(self): self.dtype = "float32" self.has_gating = False @@ -310,7 +377,6 @@ class TestSeparatedQKVCase(TestFusedGateAttentionOp): class TestMergeQKVNoBiasGatingCase(TestFusedGateAttentionOp): - def config(self): super().config() self.has_gating = False @@ -318,7 +384,6 @@ class TestMergeQKVNoBiasGatingCase(TestFusedGateAttentionOp): class TestMergeQKVFp16Case(TestFusedGateAttentionOp): - def config(self): super().config() self.dtype = "float16" @@ -330,7 +395,6 @@ class TestMergeQKVFp16Case(TestFusedGateAttentionOp): class TestMergeQKVLargeBatchSizeFp16Case(TestMergeQKVFp16Case): - def config(self): super().config() self.batch_size = 2 @@ -338,10 +402,9 @@ class TestMergeQKVLargeBatchSizeFp16Case(TestMergeQKVFp16Case): @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11000, - "core is not compiled with CUDA and cuda version need larger than or equal to 11.3" + "core is not compiled with CUDA and cuda version need larger than or equal to 11.3", ) class TestMergeQKVBF16Case(TestFusedGateAttentionOp): - def config(self): super().config() self.dtype = "bfloat16" @@ -351,7 +414,6 @@ class TestMergeQKVBF16Case(TestFusedGateAttentionOp): class TestMergeQKVLargeBatchSizeBF16Case(TestMergeQKVBF16Case): - def config(self): super().config() self.batch_size = 2 diff --git a/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_grad_op.py b/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_grad_op.py index ceefa72631c..322e63dd4fd 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_grad_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_grad_op.py @@ -29,10 +29,10 @@ def get_outputs(DOut, X, Y): @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYBiasFP16(OpTest): - def setUp(self): self.op_type = "fused_gemm_epilogue_grad" self.place = core.CUDAPlace(0) @@ -41,13 +41,14 @@ class TestFuseGemmEpilogueGradOpDXYBiasFP16(OpTest): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - DX, DY, DBias = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + DX, DY, DBias = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DX': DX, 'DY': DY, 'DBias': DBias} def init_dtype_type(self): @@ -56,38 +57,41 @@ class TestFuseGemmEpilogueGradOpDXYBiasFP16(OpTest): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYBiasFP32( - TestFuseGemmEpilogueGradOpDXYBiasFP16): - + TestFuseGemmEpilogueGradOpDXYBiasFP16 +): def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYBiasFP64( - TestFuseGemmEpilogueGradOpDXYBiasFP16): - + TestFuseGemmEpilogueGradOpDXYBiasFP16 +): def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDYBiasFP16(OpTest): - def setUp(self): self.op_type = "fused_gemm_epilogue_grad" self.place = core.CUDAPlace(0) @@ -96,13 +100,14 @@ class TestFuseGemmEpilogueGradOpDYBiasFP16(OpTest): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - _, DY, DBias = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + _, DY, DBias = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DY': DY, 'DBias': DBias} def init_dtype_type(self): @@ -111,38 +116,41 @@ class TestFuseGemmEpilogueGradOpDYBiasFP16(OpTest): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") -class TestFuseGemmEpilogueGradOpDYBiasFP32(TestFuseGemmEpilogueGradOpDYBiasFP16 - ): - +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) +class TestFuseGemmEpilogueGradOpDYBiasFP32( + TestFuseGemmEpilogueGradOpDYBiasFP16 +): def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") -class TestFuseGemmEpilogueGradOpDYBiasFP64(TestFuseGemmEpilogueGradOpDYBiasFP16 - ): - +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) +class TestFuseGemmEpilogueGradOpDYBiasFP64( + TestFuseGemmEpilogueGradOpDYBiasFP16 +): def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDYFP16(OpTest): - def setUp(self): self.op_type = "fused_gemm_epilogue_grad" self.place = core.CUDAPlace(0) @@ -151,13 +159,14 @@ class TestFuseGemmEpilogueGradOpDYFP16(OpTest): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - _, DY, _ = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + _, DY, _ = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DY': DY} def init_dtype_type(self): @@ -166,36 +175,37 @@ class TestFuseGemmEpilogueGradOpDYFP16(OpTest): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDYFP32(TestFuseGemmEpilogueGradOpDYFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDYFP64(TestFuseGemmEpilogueGradOpDYFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYFP16(OpTest): - def setUp(self): self.op_type = "fused_gemm_epilogue_grad" self.place = core.CUDAPlace(0) @@ -204,13 +214,14 @@ class TestFuseGemmEpilogueGradOpDXYFP16(OpTest): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - DX, DY, _ = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + DX, DY, _ = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DX': DX, 'DY': DY} def init_dtype_type(self): @@ -219,26 +230,27 @@ class TestFuseGemmEpilogueGradOpDXYFP16(OpTest): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYFP32(TestFuseGemmEpilogueGradOpDXYFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYFP64(TestFuseGemmEpilogueGradOpDXYFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_op.py b/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_op.py index 66ea9d7fa83..a3af9cc194f 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_op.py @@ -28,8 +28,11 @@ def is_fused_gemm_epilogue_supported(): def gelu(x): - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * (1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + ) return y_ref.astype(x.dtype) @@ -54,10 +57,10 @@ class TestFuseGemmBase(OpTest): @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -66,12 +69,12 @@ class TestFuseGemmEpilogueOpReluMMFP16(TestFuseGemmBase): self.inputs = { 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], - 'relu') + 'Out': get_output( + self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], 'relu' + ) } self.attrs = {"activation": 'relu'} @@ -81,36 +84,37 @@ class TestFuseGemmEpilogueOpReluMMFP16(TestFuseGemmBase): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP32(TestFuseGemmEpilogueOpReluMMFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP64(TestFuseGemmEpilogueOpReluMMFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -119,12 +123,15 @@ class TestFuseGemmEpilogueOpReluMTMFP16(TestFuseGemmBase): self.inputs = { 'X': np.random.random((4, 8)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'].T, self.inputs['Y'], - self.inputs['Bias'], 'relu') + 'Out': get_output( + self.inputs['X'].T, + self.inputs['Y'], + self.inputs['Bias'], + 'relu', + ) } self.attrs = {'trans_x': True, "activation": 'relu'} @@ -134,36 +141,37 @@ class TestFuseGemmEpilogueOpReluMTMFP16(TestFuseGemmBase): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP32(TestFuseGemmEpilogueOpReluMTMFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP64(TestFuseGemmEpilogueOpReluMTMFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMTFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -172,12 +180,15 @@ class TestFuseGemmEpilogueOpReluMMTFP16(TestFuseGemmBase): self.inputs = { 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((128, 4)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'], self.inputs['Y'].T, - self.inputs['Bias'], 'relu') + 'Out': get_output( + self.inputs['X'], + self.inputs['Y'].T, + self.inputs['Bias'], + 'relu', + ) } self.attrs = {'trans_y': True, "activation": 'relu'} @@ -187,36 +198,37 @@ class TestFuseGemmEpilogueOpReluMMTFP16(TestFuseGemmBase): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMTFP32(TestFuseGemmEpilogueOpReluMMTFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMTFP64(TestFuseGemmEpilogueOpReluMMTFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMTFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -225,12 +237,15 @@ class TestFuseGemmEpilogueOpReluMTMTFP16(TestFuseGemmBase): self.inputs = { 'X': np.random.random((4, 8)).astype(self.dtype) - 0.5, 'Y': np.random.random((128, 4)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'].T, self.inputs['Y'].T, - self.inputs['Bias'], 'relu') + 'Out': get_output( + self.inputs['X'].T, + self.inputs['Y'].T, + self.inputs['Bias'], + 'relu', + ) } self.attrs = {'trans_x': True, 'trans_y': True, "activation": 'relu'} @@ -240,36 +255,37 @@ class TestFuseGemmEpilogueOpReluMTMTFP16(TestFuseGemmBase): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMTFP32(TestFuseGemmEpilogueOpReluMTMTFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMTFP64(TestFuseGemmEpilogueOpReluMTMTFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP16MultiDimX(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -278,12 +294,15 @@ class TestFuseGemmEpilogueOpReluMMFP16MultiDimX(TestFuseGemmBase): self.inputs = { 'X': np.random.random((2, 2, 8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'].reshape((-1, 4)), self.inputs['Y'], - self.inputs['Bias'], 'relu').reshape((2, 2, 8, 128)) + 'Out': get_output( + self.inputs['X'].reshape((-1, 4)), + self.inputs['Y'], + self.inputs['Bias'], + 'relu', + ).reshape((2, 2, 8, 128)) } self.attrs = {"activation": 'relu'} @@ -293,38 +312,41 @@ class TestFuseGemmEpilogueOpReluMMFP16MultiDimX(TestFuseGemmBase): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP32MultiDimX( - TestFuseGemmEpilogueOpReluMMFP16MultiDimX): - + TestFuseGemmEpilogueOpReluMMFP16MultiDimX +): def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP64MultiDimX( - TestFuseGemmEpilogueOpReluMMFP16MultiDimX): - + TestFuseGemmEpilogueOpReluMMFP16MultiDimX +): def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP16MultiDimX(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -333,12 +355,15 @@ class TestFuseGemmEpilogueOpReluMTMFP16MultiDimX(TestFuseGemmBase): self.inputs = { 'X': np.random.random((4, 2, 2, 8)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'].reshape((4, -1)).T, self.inputs['Y'], - self.inputs['Bias'], 'relu').reshape((2, 2, 8, 128)) + 'Out': get_output( + self.inputs['X'].reshape((4, -1)).T, + self.inputs['Y'], + self.inputs['Bias'], + 'relu', + ).reshape((2, 2, 8, 128)) } self.attrs = {'trans_x': True, "activation": 'relu'} @@ -348,38 +373,41 @@ class TestFuseGemmEpilogueOpReluMTMFP16MultiDimX(TestFuseGemmBase): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP32MultiDimX( - TestFuseGemmEpilogueOpReluMTMFP16MultiDimX): - + TestFuseGemmEpilogueOpReluMTMFP16MultiDimX +): def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP64MultiDimX( - TestFuseGemmEpilogueOpReluMTMFP16MultiDimX): - + TestFuseGemmEpilogueOpReluMTMFP16MultiDimX +): def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpGeluMMFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -388,15 +416,15 @@ class TestFuseGemmEpilogueOpGeluMMFP16(TestFuseGemmBase): self.inputs = { 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'gelu'} self.outputs = { - 'Out': - get_output(self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], - 'gelu') + 'Out': get_output( + self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], 'gelu' + ) } def init_dtype_type(self): @@ -405,36 +433,37 @@ class TestFuseGemmEpilogueOpGeluMMFP16(TestFuseGemmBase): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpGeluMMFP32(TestFuseGemmEpilogueOpGeluMMFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpGeluMMFP64(TestFuseGemmEpilogueOpGeluMMFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpNoneMMFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -443,15 +472,15 @@ class TestFuseGemmEpilogueOpNoneMMFP16(TestFuseGemmBase): self.inputs = { 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} self.outputs = { - 'Out': - get_output(self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], - 'none') + 'Out': get_output( + self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], 'none' + ) } def init_dtype_type(self): @@ -460,26 +489,27 @@ class TestFuseGemmEpilogueOpNoneMMFP16(TestFuseGemmBase): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpNoneMMFP32(TestFuseGemmEpilogueOpNoneMMFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpNoneMMFP64(TestFuseGemmEpilogueOpNoneMMFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @@ -522,9 +552,9 @@ def matmul_grad(x, y, bias, dz, trans_x, trans_y): @unittest.skipIf( not is_fused_gemm_epilogue_supported(), - "fused_gemm_epilogue is only supported when CUDA version >= 11.6") + "fused_gemm_epilogue is only supported when CUDA version >= 11.6", +) class TestEagerFusedGemmEpilogue(unittest.TestCase): - def setUp(self): paddle.set_device('gpu') @@ -532,22 +562,22 @@ class TestEagerFusedGemmEpilogue(unittest.TestCase): paddle.disable_static() x_np = np.random.random((8, 4)).astype(np.float64) - 0.5 y_np = np.random.random((4, 128)).astype(np.float64) - 0.5 - bias_np = np.random.random((128, )).astype(np.float64) - 0.5 + bias_np = np.random.random((128,)).astype(np.float64) - 0.5 x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) bias = paddle.to_tensor(bias_np) x.stop_gradient = False y.stop_gradient = False - out1 = core.eager.ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, - 'activation', 'none') - out2 = core.eager.ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, - 'activation', 'relu') - out3 = core.eager.ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, - 'activation', 'gelu') + out1 = core.eager.ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'none' + ) + out2 = core.eager.ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'relu' + ) + out3 = core.eager.ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'gelu' + ) out_np1 = get_output(x_np, y_np, bias_np, 'none') out_np2 = get_output(x_np, y_np, bias_np, 'relu') @@ -557,13 +587,16 @@ class TestEagerFusedGemmEpilogue(unittest.TestCase): np.testing.assert_allclose(out2, out_np2, rtol=1e-05) np.testing.assert_allclose(out3, out_np3, rtol=1e-05) - out_grad_np1 = np.random.randint(low=-20, high=20, - size=out_np1.shape).astype(np.float64) - paddle.autograd.backward(out1, - grad_tensors=[paddle.to_tensor(out_grad_np1)]) + out_grad_np1 = np.random.randint( + low=-20, high=20, size=out_np1.shape + ).astype(np.float64) + paddle.autograd.backward( + out1, grad_tensors=[paddle.to_tensor(out_grad_np1)] + ) x_grad_np, y_grad_np, bias_grad_np = matmul_grad( - x_np, y_np, bias_np, out_grad_np1, False, False) + x_np, y_np, bias_np, out_grad_np1, False, False + ) np.testing.assert_allclose(x.grad.numpy(), x_grad_np, rtol=1e-05) self.assertEqual(y_grad_np.shape, y_np.shape) np.testing.assert_allclose(y.grad.numpy(), y_grad_np, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py b/python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py index b26b5e2f046..a4d152c75b9 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py +++ b/python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py @@ -64,9 +64,9 @@ def matmul_grad(x, y, bias, dz, trans_x, trans_y): @unittest.skipIf( not is_fused_matmul_bias_supported(), - "fused_gemm_epilogue is only supported when CUDA version >= 11.6") + "fused_gemm_epilogue is only supported when CUDA version >= 11.6", +) class TestFusedMatmulBias(unittest.TestCase): - def setUp(self): paddle.set_device('gpu') @@ -103,7 +103,8 @@ class TestFusedMatmulBias(unittest.TestCase): paddle.autograd.backward(z, grad_tensors=[paddle.to_tensor(z_grad_np)]) x_grad_np, y_grad_np, bias_grad_np = matmul_grad( - x_np, y_np, bias_np, z_grad_np, trans_x, trans_y) + x_np, y_np, bias_np, z_grad_np, trans_x, trans_y + ) np.testing.assert_array_equal(x.grad.numpy(), x_grad_np) self.assertEqual(y_grad_np.shape, y_np.shape) np.testing.assert_array_equal(y.grad.numpy(), y_grad_np) @@ -118,8 +119,9 @@ class TestFusedMatmulBias(unittest.TestCase): for trans_x in [False, True]: for trans_y in [False, True]: for need_bias in [False, True]: - self.rand_test_base(m, n, k, trans_x, trans_y, need_bias, - dtype, seed) + self.rand_test_base( + m, n, k, trans_x, trans_y, need_bias, dtype, seed + ) def test_fp32(self): self.rand_test(30, 40, 50, np.float32) @@ -130,9 +132,9 @@ class TestFusedMatmulBias(unittest.TestCase): @unittest.skipIf( not is_fused_matmul_bias_supported(), - "fused_gemm_epilogue is only supported when CUDA version >= 11.6") + "fused_gemm_epilogue is only supported when CUDA version >= 11.6", +) class TestFusedLinear(unittest.TestCase): - def check_fused_linear(self, transpose): x = paddle.randn([30, 40]) linear = FusedLinear(40, 50, transpose_weight=transpose) @@ -149,9 +151,9 @@ class TestFusedLinear(unittest.TestCase): @unittest.skipIf( not is_fused_matmul_bias_supported(), - "fused_gemm_epilogue is only supported when CUDA version >= 11.6") + "fused_gemm_epilogue is only supported when CUDA version >= 11.6", +) class TestStaticGraph(unittest.TestCase): - def test_static_graph(self): paddle.enable_static() x = paddle.static.data(name='x', dtype='float32', shape=[-1, 100]) diff --git a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_int8_op.py b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_int8_op.py index 3384e86668e..e3fb41412fd 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_int8_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_int8_op.py @@ -68,27 +68,69 @@ def fused_multi_transformer_int8( ffn1_in_scale=[], ffn2_in_scale=[], ): - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer cache_kv_out, final_out = _legacy_C_ops.fused_multi_transformer_int8( - x, ln_scales, ln_biases, qkv_weights, qkv_biases, cache_kvs, time_step, - attn_mask, linear_weights, linear_biases, ffn_ln_scales, ffn_ln_biases, - ffn1_weights, ffn1_biases, ffn2_weights, ffn2_biases, qkv_out_scales, - out_linear_out_scales, ffn1_out_scales, ffn2_out_scales, cache_kvs, - 'num_head', num_head, 'dim_head', dim_head, 'dim_ffn', dim_ffn, - 'qkv_in_scale', qkv_in_scale, 'out_linear_in_scale', - out_linear_in_scale, 'ffn1_in_scale', ffn1_in_scale, 'ffn2_in_scale', - ffn2_in_scale, 'pre_layer_norm', pre_layer_norm, 'epsilon', epsilon, - 'dropout_rate', dropout_rate, 'is_test', not training, - 'dropout_implementation', mode, 'act_method', activation, 'trans_qkvw', - trans_qkvw, 'ring_id', ring_id) + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + cache_kvs, + time_step, + attn_mask, + linear_weights, + linear_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + qkv_out_scales, + out_linear_out_scales, + ffn1_out_scales, + ffn2_out_scales, + cache_kvs, + 'num_head', + num_head, + 'dim_head', + dim_head, + 'dim_ffn', + dim_ffn, + 'qkv_in_scale', + qkv_in_scale, + 'out_linear_in_scale', + out_linear_in_scale, + 'ffn1_in_scale', + ffn1_in_scale, + 'ffn2_in_scale', + ffn2_in_scale, + 'pre_layer_norm', + pre_layer_norm, + 'epsilon', + epsilon, + 'dropout_rate', + dropout_rate, + 'is_test', + not training, + 'dropout_implementation', + mode, + 'act_method', + activation, + 'trans_qkvw', + trans_qkvw, + 'ring_id', + ring_id, + ) if cache_kvs is not None: return final_out, cache_kv_out return final_out class TestFusedMultiTransformerInt8Op(unittest.TestCase): - def setUp(self): self.config() self.generate_input_data() @@ -110,12 +152,12 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): self.__class__.no_need_check_grad = True paddle.set_default_dtype(np.float32) - self.norm = LayerNorm(self.embed_dim, - weight_attr=False, - bias_attr=False) - self.ffn_norm = LayerNorm(self.embed_dim, - weight_attr=False, - bias_attr=False) + self.norm = LayerNorm( + self.embed_dim, weight_attr=False, bias_attr=False + ) + self.ffn_norm = LayerNorm( + self.embed_dim, weight_attr=False, bias_attr=False + ) paddle.set_default_dtype(self.x_type) self.dropout = Dropout(self.dropout_prob, mode="upscale_in_train") @@ -127,7 +169,7 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): self.x_type = np.float32 self.attn_mask_type = np.float64 - #self.attn_mask_type = np.bool + # self.attn_mask_type = np.bool self.pre_layer_norm = True self.has_attn_mask = True @@ -154,31 +196,38 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): self.weight_attr = None self.bias_attr = None self.kdim, self.vdim = self.embed_dim, self.embed_dim - self.key_length, self.value_length = self.query_length, self.query_length + self.key_length, self.value_length = ( + self.query_length, + self.query_length, + ) def generate_input_data(self): - self.query = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) - q_weight = np.random.randint(-64, 64, [self.embed_dim, self.embed_dim], - np.int32).astype('float64') - k_weight = np.random.randint(-64, 64, [self.kdim, self.embed_dim], - np.int32).astype('float64') - v_weight = np.random.randint(-64, 64, [self.vdim, self.embed_dim], - np.int32).astype('float64') + self.query = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) + q_weight = np.random.randint( + -64, 64, [self.embed_dim, self.embed_dim], np.int32 + ).astype('float64') + k_weight = np.random.randint( + -64, 64, [self.kdim, self.embed_dim], np.int32 + ).astype('float64') + v_weight = np.random.randint( + -64, 64, [self.vdim, self.embed_dim], np.int32 + ).astype('float64') self.q_weight_tensor = paddle.to_tensor(q_weight) self.k_weight_tensor = paddle.to_tensor(k_weight) self.v_weight_tensor = paddle.to_tensor(v_weight) - out_weight = np.random.randint(-64, 64, - [self.embed_dim, self.embed_dim], - np.int32).astype('float64') - ffn1_weight = np.random.randint(-64, 64, - [self.embed_dim, 4 * self.embed_dim], - np.int32).astype('float64') - ffn2_weight = np.random.randint(-64, 64, - [4 * self.embed_dim, self.embed_dim], - np.int32).astype('float64') + out_weight = np.random.randint( + -64, 64, [self.embed_dim, self.embed_dim], np.int32 + ).astype('float64') + ffn1_weight = np.random.randint( + -64, 64, [self.embed_dim, 4 * self.embed_dim], np.int32 + ).astype('float64') + ffn2_weight = np.random.randint( + -64, 64, [4 * self.embed_dim, self.embed_dim], np.int32 + ).astype('float64') self.out_weight_tensor = paddle.to_tensor(out_weight) self.ffn1_weight_tensor = paddle.to_tensor(ffn1_weight) @@ -193,12 +242,14 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): self.v_proj_bias_tensor = paddle.to_tensor(v_proj_bias) out_linear_proj_bias = np.random.rand(self.embed_dim).astype( - self.x_type) + self.x_type + ) ffn1_proj_bias = np.random.rand(4 * self.embed_dim).astype(self.x_type) ffn2_proj_bias = np.random.rand(self.embed_dim).astype(self.x_type) self.out_linear_proj_bias_tensor = paddle.to_tensor( - out_linear_proj_bias) + out_linear_proj_bias + ) self.ffn1_proj_bias_tensor = paddle.to_tensor(ffn1_proj_bias) self.ffn2_proj_bias_tensor = paddle.to_tensor(ffn2_proj_bias) @@ -214,9 +265,13 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): self.ffn2_out_scales = [] if self.has_cache_kv: - self.cache_kv = np.random.rand(2, self.batch_size, self.num_heads, - self.cache_length, - self.head_dim).astype(self.x_type) + self.cache_kv = np.random.rand( + 2, + self.batch_size, + self.num_heads, + self.cache_length, + self.head_dim, + ).astype(self.x_type) if self.gen_cache_kv: self.cache_kv[:] = 0 @@ -229,7 +284,8 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): # [B, n_head, seq_len, out_seq_len] self.attn_mask = np.ones( (self.batch_size, 1, self.query_length, out_seq_len), - dtype=self.attn_mask_type) + dtype=self.attn_mask_type, + ) if self.attn_mask_type == np.int64: self.attn_mask = np.tril(self.attn_mask) elif self.attn_mask_type == np.float64: @@ -246,7 +302,8 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): self.attn_mask = np.tril(self.attn_mask) else: raise ValueError( - "'attn_mask_type' should be 'int64' or 'float64'.") + "'attn_mask_type' should be 'int64' or 'float64'." + ) else: self.attn_mask = None @@ -289,8 +346,11 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): q = paddle.nn.functional.linear(ln1_out, self.q_weight_tensor) # de quant q = paddle.cast( - paddle.cast(q, 'float32') * self.qkv_in_scales[i] / - self.qkv_out_scales[i], self.x_type) + paddle.cast(q, 'float32') + * self.qkv_in_scales[i] + / self.qkv_out_scales[i], + self.x_type, + ) q = q + self.q_proj_bias_tensor q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) @@ -298,13 +358,19 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): k = paddle.nn.functional.linear(ln1_out, self.k_weight_tensor) k = paddle.cast( - paddle.cast(k, 'float32') * self.qkv_in_scales[i] / - self.qkv_out_scales[i], self.x_type) + paddle.cast(k, 'float32') + * self.qkv_in_scales[i] + / self.qkv_out_scales[i], + self.x_type, + ) k = k + self.k_proj_bias_tensor v = paddle.nn.functional.linear(ln1_out, self.v_weight_tensor) v = paddle.cast( - paddle.cast(v, 'float32') * self.qkv_in_scales[i] / - self.qkv_out_scales[i], self.x_type) + paddle.cast(v, 'float32') + * self.qkv_in_scales[i] + / self.qkv_out_scales[i], + self.x_type, + ) v = v + self.v_proj_bias_tensor k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) @@ -332,10 +398,9 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): # [B, n_head, seq_len, head_dim] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, out_seq_len] - qk_out = layers.matmul(x=q_out, - y=k_out, - transpose_y=True, - alpha=self.head_dim**-0.5) + qk_out = layers.matmul( + x=q_out, y=k_out, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.debug: print('qk out is') @@ -355,10 +420,12 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): print('softmax out is') print(softmax_out[0][0][0]) if self.dropout_prob: - dropout_out = F.dropout(softmax_out, - self.dropout_prob, - training=self.training, - mode="upscale_in_train") + dropout_out = F.dropout( + softmax_out, + self.dropout_prob, + training=self.training, + mode="upscale_in_train", + ) # [B, n_head, seq_len, out_seq_len] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, head_dim] qktv_out = tensor.matmul(dropout_out, v_out) @@ -370,23 +437,30 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): print('fmha out is') print(fmha_out[0][0][0]) out_linear_in = tensor.reshape( - x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]]) + x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]] + ) - max_v = paddle.max(paddle.abs(paddle.cast(out_linear_in, - 'float32')))[0] + max_v = paddle.max( + paddle.abs(paddle.cast(out_linear_in, 'float32')) + )[0] # self.out_linear_in_scales.append(127.0 / max_v) self.out_linear_in_scales.append(max_v) self.out_linear_out_scales.append((127.0 * 127.0)) - out_linear_in = self.fake_quant(out_linear_in, - self.out_linear_in_scales[i]) + out_linear_in = self.fake_quant( + out_linear_in, self.out_linear_in_scales[i] + ) - out = paddle.nn.functional.linear(out_linear_in, - self.out_weight_tensor) + out = paddle.nn.functional.linear( + out_linear_in, self.out_weight_tensor + ) out = paddle.cast( - paddle.cast(out, 'float32') * self.out_linear_in_scales[i] / - self.out_linear_out_scales[i], self.x_type) + paddle.cast(out, 'float32') + * self.out_linear_in_scales[i] + / self.out_linear_out_scales[i], + self.x_type, + ) out = out + self.out_linear_proj_bias_tensor @@ -400,18 +474,23 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): if self.pre_layer_norm: ffn_ln_out = self.ffn_norm(attn_out) - max_v = paddle.max(paddle.abs(paddle.cast(ffn_ln_out, - 'float32')))[0] + max_v = paddle.max(paddle.abs(paddle.cast(ffn_ln_out, 'float32')))[ + 0 + ] self.ffn1_in_scales.append(max_v) self.ffn1_out_scales.append((127.0 * 127.0)) ffn_ln_out = self.fake_quant(ffn_ln_out, self.ffn1_in_scales[i]) - ffn1_out = paddle.nn.functional.linear(ffn_ln_out, - self.ffn1_weight_tensor) + ffn1_out = paddle.nn.functional.linear( + ffn_ln_out, self.ffn1_weight_tensor + ) ffn1_out = paddle.cast( - paddle.cast(ffn1_out, 'float32') * self.ffn1_in_scales[i] / - self.ffn1_out_scales[i], self.x_type) + paddle.cast(ffn1_out, 'float32') + * self.ffn1_in_scales[i] + / self.ffn1_out_scales[i], + self.x_type, + ) ffn1_out = ffn1_out + self.ffn1_proj_bias_tensor ffn1_out = self.dropout(self.activation(ffn1_out)) @@ -423,12 +502,16 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): # print('ffn2_in_scales ', i, self.ffn2_in_scales[i]) ffn1_out = self.fake_quant(ffn1_out, self.ffn2_in_scales[i]) - ffn2_out = paddle.nn.functional.linear(ffn1_out, - self.ffn2_weight_tensor) + ffn2_out = paddle.nn.functional.linear( + ffn1_out, self.ffn2_weight_tensor + ) ffn2_out = paddle.cast( - paddle.cast(ffn2_out, 'float32') * self.ffn2_in_scales[i] / - self.ffn2_out_scales[i], self.x_type) + paddle.cast(ffn2_out, 'float32') + * self.ffn2_in_scales[i] + / self.ffn2_out_scales[i], + self.x_type, + ) ffn2_out = ffn2_out + self.ffn2_proj_bias_tensor residual_out = attn_out + self.dropout(ffn2_out) @@ -456,26 +539,35 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): k_proj_weight = self.k_weight_tensor.numpy().transpose((1, 0)) v_proj_weight = self.v_weight_tensor.numpy().transpose((1, 0)) qkv_weight = np.concatenate( - (q_proj_weight, k_proj_weight, v_proj_weight)) + (q_proj_weight, k_proj_weight, v_proj_weight) + ) qkv_weight = qkv_weight.reshape( - (3, self.num_heads, self.head_dim, self.embed_dim)) + (3, self.num_heads, self.head_dim, self.embed_dim) + ) qkv_weight_tensor = paddle.to_tensor(qkv_weight) qkv_weight_tensor = paddle.cast(qkv_weight_tensor, 'int8') out_weight_tensor = paddle.cast( paddle.to_tensor(self.out_weight_tensor.numpy().transpose((1, 0))), - 'int8') + 'int8', + ) ffn1_weight_tensor = paddle.cast( paddle.to_tensor(self.ffn1_weight_tensor.numpy().transpose((1, 0))), - 'int8') + 'int8', + ) ffn2_weight_tensor = paddle.cast( paddle.to_tensor(self.ffn2_weight_tensor.numpy().transpose((1, 0))), - 'int8') + 'int8', + ) qkv_bias = np.concatenate( - (self.q_proj_bias_tensor.numpy(), self.k_proj_bias_tensor.numpy(), - self.v_proj_bias_tensor.numpy())) + ( + self.q_proj_bias_tensor.numpy(), + self.k_proj_bias_tensor.numpy(), + self.v_proj_bias_tensor.numpy(), + ) + ) qkv_bias = qkv_bias.reshape((3, self.num_heads, self.head_dim)) qkv_bias_tensor = paddle.to_tensor(qkv_bias) @@ -486,11 +578,16 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): cache_kvs = [] max_seq_length = (self.cache_length + 128) // 128 * 128 - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) elems = 4 if self.x_type is np.float16: @@ -502,25 +599,36 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): # [B, num_head, 128, head_dim] # cache_k_tmp = self.cache_kv[0, :] # [B, num_head, 128, head_dim / 4, 4] - cache_k_tmp = self.cache_kv[0].reshape([ - self.batch_size, self.num_heads, self.cache_length, v_elems, - elems - ]) + cache_k_tmp = self.cache_kv[0].reshape( + [ + self.batch_size, + self.num_heads, + self.cache_length, + v_elems, + elems, + ] + ) # [B, num_head, head_dim / 4, 128, 4] cache_k_tmp = cache_k_tmp.transpose([0, 1, 3, 2, 4]) - cache_kv[0, :].reshape([ - self.batch_size, self.num_heads, v_elems, max_seq_length, elems - ])[:, :, :, :self.cache_length, :] = cache_k_tmp - - cache_kv[1, :, :, :self.cache_length, :] = self.cache_kv[1] + cache_kv[0, :].reshape( + [ + self.batch_size, + self.num_heads, + v_elems, + max_seq_length, + elems, + ] + )[:, :, :, : self.cache_length, :] = cache_k_tmp + + cache_kv[1, :, :, : self.cache_length, :] = self.cache_kv[1] if self.gen_cache_kv: assert self.query_length == self.cache_length cache_kv[:] = 0 else: - time_step = paddle.to_tensor([self.cache_length], - dtype='int32', - place=paddle.CPUPlace()) + time_step = paddle.to_tensor( + [self.cache_length], dtype='int32', place=paddle.CPUPlace() + ) if self.has_attn_mask: attn_mask = paddle.to_tensor(self.attn_mask, stop_gradient=True) else: @@ -542,14 +650,18 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): ffn1_in_scale = [] ffn2_in_scale = [] - qkv_out_scales_tensor = paddle.ones([self.layers, 3 * self.embed_dim], - 'float32') + qkv_out_scales_tensor = paddle.ones( + [self.layers, 3 * self.embed_dim], 'float32' + ) out_linear_out_scales_tensor = paddle.ones( - [self.layers, self.embed_dim], 'float32') - ffn1_out_scales_tensor = paddle.ones([self.layers, 4 * self.embed_dim], - 'float32') - ffn2_out_scales_tensor = paddle.ones([self.layers, self.embed_dim], - 'float32') + [self.layers, self.embed_dim], 'float32' + ) + ffn1_out_scales_tensor = paddle.ones( + [self.layers, 4 * self.embed_dim], 'float32' + ) + ffn2_out_scales_tensor = paddle.ones( + [self.layers, self.embed_dim], 'float32' + ) for i in range(self.layers): qkv_weights.append(qkv_weight_tensor) @@ -612,7 +724,8 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): qkv_in_scale=qkv_in_scale, out_linear_in_scale=out_linear_in_scale, ffn1_in_scale=ffn1_in_scale, - ffn2_in_scale=ffn2_in_scale) + ffn2_in_scale=ffn2_in_scale, + ) if self.has_cache_kv: return final_out[0], final_out[1] @@ -634,9 +747,11 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): if self.debug: print("cache_k out timestep=128") - print(cache_kv_out[0].reshape( - [2, bsz, num_head, v_elems, max_seq_len, - elems])[0, 0, 0, :, self.cache_length, :]) + print( + cache_kv_out[0].reshape( + [2, bsz, num_head, v_elems, max_seq_len, elems] + )[0, 0, 0, :, self.cache_length, :] + ) print("cache_v out timestep=128") print(cache_kv_out[0][1, 0, 0, self.cache_length, :]) @@ -649,33 +764,31 @@ class TestFusedMultiTransformerInt8Op(unittest.TestCase): cache_k = cache_kv_out[i][0, :] cache_k = cache_k.reshape( - [bsz, num_head, v_elems, max_seq_len, elems]) - cache_k = cache_k[:, :, :, :self.cache_length, :] + [bsz, num_head, v_elems, max_seq_len, elems] + ) + cache_k = cache_k[:, :, :, : self.cache_length, :] cache_k = cache_k.transpose([0, 1, 3, 2, 4]) cache_k = cache_k.reshape( - [bsz, num_head, self.cache_length, head_dim]) - - cache_v = cache_kv_out[i][1, :, :, :self.cache_length, :] - - np.testing.assert_allclose(cache_k_ref, - cache_k, - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(cache_v_ref, - cache_v, - rtol=self.rtol, - atol=self.atol) + [bsz, num_head, self.cache_length, head_dim] + ) + + cache_v = cache_kv_out[i][1, :, :, : self.cache_length, :] + + np.testing.assert_allclose( + cache_k_ref, cache_k, rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + cache_v_ref, cache_v, rtol=self.rtol, atol=self.atol + ) if i == 0: break - np.testing.assert_allclose(final_out_ref, - final_out, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out, rtol=self.rtol, atol=self.atol + ) class TestFusedMultiTransformerInt8OpFp16(TestFusedMultiTransformerInt8Op): - def config(self): super().config() self.x_type = np.float16 @@ -683,7 +796,6 @@ class TestFusedMultiTransformerInt8OpFp16(TestFusedMultiTransformerInt8Op): class TestFusedMultiTransformerInt8OpCacheKV(TestFusedMultiTransformerInt8Op): - def config(self): super().config() super().generate_input_data() @@ -693,9 +805,9 @@ class TestFusedMultiTransformerInt8OpCacheKV(TestFusedMultiTransformerInt8Op): self.layers = 3 # odd layers -class TestFusedMultiTransformerInt8OpCacheKVFp16(TestFusedMultiTransformerInt8Op - ): - +class TestFusedMultiTransformerInt8OpCacheKVFp16( + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -704,9 +816,9 @@ class TestFusedMultiTransformerInt8OpCacheKVFp16(TestFusedMultiTransformerInt8Op self.x_type = np.float16 -class TestFusedMultiTransformerInt8OpGenCacheKV(TestFusedMultiTransformerInt8Op - ): - +class TestFusedMultiTransformerInt8OpGenCacheKV( + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -714,8 +826,8 @@ class TestFusedMultiTransformerInt8OpGenCacheKV(TestFusedMultiTransformerInt8Op class TestFusedMultiTransformerInt8OpGenCacheKVFp16( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -725,8 +837,8 @@ class TestFusedMultiTransformerInt8OpGenCacheKVFp16( class TestFusedMultiTransformerInt8OpPostLayerNormFp16( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.x_type = np.float16 @@ -735,8 +847,8 @@ class TestFusedMultiTransformerInt8OpPostLayerNormFp16( class TestFusedMultiTransformerInt8OpCacheKVPostLayerNorm( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -747,8 +859,8 @@ class TestFusedMultiTransformerInt8OpCacheKVPostLayerNorm( class TestFusedMultiTransformerInt8OpCacheKVPostLayerNormFp16( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -759,8 +871,8 @@ class TestFusedMultiTransformerInt8OpCacheKVPostLayerNormFp16( class TestFusedMultiTransformerInt8OpGenCacheKVPostLayerNorm( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -769,8 +881,8 @@ class TestFusedMultiTransformerInt8OpGenCacheKVPostLayerNorm( class TestFusedMultiTransformerInt8OpGenCacheKVPostLayerNormFp16( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True diff --git a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py index dcfb7294040..0daf8342294 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py @@ -32,7 +32,6 @@ default_main_program().random_seed = 42 class TestFusedMultiTransformerOp(OpTest): - def setUp(self): self.config() self.generate_input_data() @@ -54,34 +53,47 @@ class TestFusedMultiTransformerOp(OpTest): self.__class__.no_need_check_grad = False bias_attr = paddle.fluid.ParamAttr( - initializer=paddle.fluid.initializer.Constant(value=0.0005)) - self.q_proj = Linear(self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=bias_attr) - #bias_attr=self.bias_attr) - - self.k_proj = Linear(self.kdim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.v_proj = Linear(self.vdim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.out_proj = Linear(self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - - self.ffn1_proj = Linear(self.embed_dim, - 4 * self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.ffn2_proj = Linear(4 * self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) + initializer=paddle.fluid.initializer.Constant(value=0.0005) + ) + self.q_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=bias_attr, + ) + # bias_attr=self.bias_attr) + + self.k_proj = Linear( + self.kdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.v_proj = Linear( + self.vdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.out_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + + self.ffn1_proj = Linear( + self.embed_dim, + 4 * self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.ffn2_proj = Linear( + 4 * self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) paddle.set_default_dtype(np.float32) self.norm = LayerNorm(self.embed_dim) @@ -97,7 +109,7 @@ class TestFusedMultiTransformerOp(OpTest): self.x_type = np.float32 self.attn_mask_type = np.float64 - #self.attn_mask_type = np.bool + # self.attn_mask_type = np.bool self.pre_layer_norm = True self.has_attn_mask = True @@ -126,18 +138,27 @@ class TestFusedMultiTransformerOp(OpTest): self.weight_attr = None self.bias_attr = None self.kdim, self.vdim = self.embed_dim, self.embed_dim - self.key_length, self.value_length = self.query_length, self.query_length + self.key_length, self.value_length = ( + self.query_length, + self.query_length, + ) def generate_input_data(self): - self.query = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.query = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) out_seq_len = self.key_length if self.has_cache_kv: assert self.training is False, ValueError( - 'cache_kv can only used in inference') - self.cache_kv = np.random.rand(2, self.batch_size, self.num_heads, - self.cache_length, - self.head_dim).astype(self.x_type) + 'cache_kv can only used in inference' + ) + self.cache_kv = np.random.rand( + 2, + self.batch_size, + self.num_heads, + self.cache_length, + self.head_dim, + ).astype(self.x_type) if self.gen_cache_kv: self.cache_kv[:] = 0 else: @@ -148,14 +169,19 @@ class TestFusedMultiTransformerOp(OpTest): if self.has_pre_cache: out_seq_len += self.pre_cache_num self.pre_cache_kv = np.random.rand( - 2, self.batch_size, self.num_heads, self.pre_cache_num, - self.head_dim).astype(self.x_type) + 2, + self.batch_size, + self.num_heads, + self.pre_cache_num, + self.head_dim, + ).astype(self.x_type) if self.has_attn_mask: # [B, n_head, seq_len, out_seq_len] self.attn_mask = np.ones( (self.batch_size, 1, self.query_length, out_seq_len), - dtype=self.attn_mask_type) + dtype=self.attn_mask_type, + ) if self.attn_mask_type == np.int64: self.attn_mask = np.tril(self.attn_mask) elif self.attn_mask_type == np.float64: @@ -172,13 +198,15 @@ class TestFusedMultiTransformerOp(OpTest): self.attn_mask = np.tril(self.attn_mask) else: raise ValueError( - "'attn_mask_type' should be 'int64' or 'float64'.") + "'attn_mask_type' should be 'int64' or 'float64'." + ) else: self.attn_mask = None self.key, self.value = self.query, self.query - self.dout = np.random.random((self.batch_size, self.query_length, - self.embed_dim)).astype(self.x_type) + self.dout = np.random.random( + (self.batch_size, self.query_length, self.embed_dim) + ).astype(self.x_type) def GetBaselineOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -190,8 +218,9 @@ class TestFusedMultiTransformerOp(OpTest): cache_kv = paddle.to_tensor(self.cache_kv, stop_gradient=False) if self.has_pre_cache: - pre_cache_kv = paddle.to_tensor(self.pre_cache_kv, - stop_gradient=False) + pre_cache_kv = paddle.to_tensor( + self.pre_cache_kv, stop_gradient=False + ) if self.has_attn_mask: attn_mask = paddle.to_tensor(self.attn_mask, stop_gradient=False) @@ -241,10 +270,9 @@ class TestFusedMultiTransformerOp(OpTest): # [B, n_head, seq_len, head_dim] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, out_seq_len] - qk_out = layers.matmul(x=q_out, - y=k_out, - transpose_y=True, - alpha=self.head_dim**-0.5) + qk_out = layers.matmul( + x=q_out, y=k_out, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.debug: print('qk out is') @@ -264,10 +292,12 @@ class TestFusedMultiTransformerOp(OpTest): print('softmax out is') print(softmax_out[0][0][0]) if self.dropout_prob: - dropout_out = F.dropout(softmax_out, - self.dropout_prob, - training=self.training, - mode="upscale_in_train") + dropout_out = F.dropout( + softmax_out, + self.dropout_prob, + training=self.training, + mode="upscale_in_train", + ) # [B, n_head, seq_len, out_seq_len] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, head_dim] qktv_out = tensor.matmul(dropout_out, v_out) @@ -279,7 +309,8 @@ class TestFusedMultiTransformerOp(OpTest): print('fmha out is') print(fmha_out[0][0][0]) out_linear_in = tensor.reshape( - x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]]) + x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]] + ) out = self.out_proj(out_linear_in) residual_out = residual + self.dropout(out) @@ -309,53 +340,69 @@ class TestFusedMultiTransformerOp(OpTest): def GetFusedMultiTransformerOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) - q_proj_weight = paddle.to_tensor(self.q_proj.weight, - stop_gradient=False) - k_proj_weight = paddle.to_tensor(self.k_proj.weight, - stop_gradient=False) - v_proj_weight = paddle.to_tensor(self.v_proj.weight, - stop_gradient=False) - out_linear_weight = paddle.to_tensor(self.out_proj.weight, - stop_gradient=False) - ffn1_weight = paddle.to_tensor(self.ffn1_proj.weight, - stop_gradient=False) - ffn2_weight = paddle.to_tensor(self.ffn2_proj.weight, - stop_gradient=False) + q_proj_weight = paddle.to_tensor( + self.q_proj.weight, stop_gradient=False + ) + k_proj_weight = paddle.to_tensor( + self.k_proj.weight, stop_gradient=False + ) + v_proj_weight = paddle.to_tensor( + self.v_proj.weight, stop_gradient=False + ) + out_linear_weight = paddle.to_tensor( + self.out_proj.weight, stop_gradient=False + ) + ffn1_weight = paddle.to_tensor( + self.ffn1_proj.weight, stop_gradient=False + ) + ffn2_weight = paddle.to_tensor( + self.ffn2_proj.weight, stop_gradient=False + ) if self.bias_attr is False: qkv_bias_tensor = None out_linear_bias = None else: - q_proj_bias = paddle.to_tensor(self.q_proj.bias, - stop_gradient=False) - k_proj_bias = paddle.to_tensor(self.k_proj.bias, - stop_gradient=False) - v_proj_bias = paddle.to_tensor(self.v_proj.bias, - stop_gradient=False) + q_proj_bias = paddle.to_tensor( + self.q_proj.bias, stop_gradient=False + ) + k_proj_bias = paddle.to_tensor( + self.k_proj.bias, stop_gradient=False + ) + v_proj_bias = paddle.to_tensor( + self.v_proj.bias, stop_gradient=False + ) qkv_bias = np.concatenate( - (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy())) + (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy()) + ) qkv_bias = qkv_bias.reshape((3, self.num_heads, self.head_dim)) qkv_bias_tensor = paddle.to_tensor(qkv_bias, stop_gradient=False) - out_linear_bias = paddle.to_tensor(self.out_proj.bias, - stop_gradient=False) - ffn1_bias = paddle.to_tensor(self.ffn1_proj.bias, - stop_gradient=False) - ffn2_bias = paddle.to_tensor(self.ffn2_proj.bias, - stop_gradient=False) + out_linear_bias = paddle.to_tensor( + self.out_proj.bias, stop_gradient=False + ) + ffn1_bias = paddle.to_tensor( + self.ffn1_proj.bias, stop_gradient=False + ) + ffn2_bias = paddle.to_tensor( + self.ffn2_proj.bias, stop_gradient=False + ) ln_scale = paddle.to_tensor(self.norm.weight, stop_gradient=False) ln_bias = paddle.to_tensor(self.norm.bias, stop_gradient=False) - ffn_ln_scale = paddle.to_tensor(self.ffn_norm.weight, - stop_gradient=False) + ffn_ln_scale = paddle.to_tensor( + self.ffn_norm.weight, stop_gradient=False + ) ffn_ln_bias = paddle.to_tensor(self.ffn_norm.bias, stop_gradient=False) q_proj_weight = q_proj_weight.numpy().transpose((1, 0)) k_proj_weight = k_proj_weight.numpy().transpose((1, 0)) v_proj_weight = v_proj_weight.numpy().transpose((1, 0)) qkv_weight = np.concatenate( - (q_proj_weight, k_proj_weight, v_proj_weight)) + (q_proj_weight, k_proj_weight, v_proj_weight) + ) qkv_weight = qkv_weight.reshape( - (3, self.num_heads, self.head_dim, self.embed_dim)) + (3, self.num_heads, self.head_dim, self.embed_dim) + ) x = paddle.to_tensor(self.query, stop_gradient=False) cache_kvs, cache_kv = None, None @@ -365,11 +412,16 @@ class TestFusedMultiTransformerOp(OpTest): cache_kvs = [] max_seq_length = (self.cache_length + 128) // 128 * 128 - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) elems = 4 if self.x_type is np.float16: @@ -381,35 +433,52 @@ class TestFusedMultiTransformerOp(OpTest): # [B, num_head, 128, head_dim] # cache_k_tmp = self.cache_kv[0, :] # [B, num_head, 128, head_dim / 4, 4] - cache_k_tmp = self.cache_kv[0].reshape([ - self.batch_size, self.num_heads, self.cache_length, v_elems, - elems - ]) + cache_k_tmp = self.cache_kv[0].reshape( + [ + self.batch_size, + self.num_heads, + self.cache_length, + v_elems, + elems, + ] + ) # [B, num_head, head_dim / 4, 128, 4] cache_k_tmp = cache_k_tmp.transpose([0, 1, 3, 2, 4]) - cache_kv[0, :].reshape([ - self.batch_size, self.num_heads, v_elems, max_seq_length, elems - ])[:, :, :, :self.cache_length, :] = cache_k_tmp - - cache_kv[1, :, :, :self.cache_length, :] = self.cache_kv[1] + cache_kv[0, :].reshape( + [ + self.batch_size, + self.num_heads, + v_elems, + max_seq_length, + elems, + ] + )[:, :, :, : self.cache_length, :] = cache_k_tmp + + cache_kv[1, :, :, : self.cache_length, :] = self.cache_kv[1] if self.gen_cache_kv: assert self.query_length == self.cache_length cache_kv[:] = 0 else: - time_step = paddle.to_tensor([self.cache_length], - dtype='int32', - place=paddle.CPUPlace()) + time_step = paddle.to_tensor( + [self.cache_length], dtype='int32', place=paddle.CPUPlace() + ) if self.has_pre_cache: cache_kvs = [] - max_seq_length = (self.cache_length + - 128) // 128 * 128 + self.pre_cache_num - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + max_seq_length = ( + self.cache_length + 128 + ) // 128 * 128 + self.pre_cache_num + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) pre_caches = [] if self.has_attn_mask: @@ -443,35 +512,40 @@ class TestFusedMultiTransformerOp(OpTest): ffn_ln_scales.append(ffn_ln_scale) ffn_ln_biases.append(ffn_ln_bias) if self.has_cache_kv: - cache_kvs.append(paddle.to_tensor(cache_kv, - stop_gradient=False)) + cache_kvs.append( + paddle.to_tensor(cache_kv, stop_gradient=False) + ) if self.has_pre_cache: - cache_kvs.append(paddle.to_tensor(cache_kv, - stop_gradient=False)) + cache_kvs.append( + paddle.to_tensor(cache_kv, stop_gradient=False) + ) pre_caches.append( - paddle.to_tensor(self.pre_cache_kv, stop_gradient=False)) - - final_out = fused_multi_transformer(x, - ln_scales, - ln_biases, - qkv_weights, - qkv_biases, - out_weights, - out_biases, - ffn_ln_scales, - ffn_ln_biases, - ffn1_weights, - ffn1_biases, - ffn2_weights, - ffn2_biases, - pre_layer_norm=self.pre_layer_norm, - epsilon=epsilon, - cache_kvs=cache_kvs, - pre_caches=pre_caches, - time_step=time_step, - attn_mask=attn_mask, - dropout_rate=self.dropout_prob, - training=self.training) + paddle.to_tensor(self.pre_cache_kv, stop_gradient=False) + ) + + final_out = fused_multi_transformer( + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + out_weights, + out_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + pre_layer_norm=self.pre_layer_norm, + epsilon=epsilon, + cache_kvs=cache_kvs, + pre_caches=pre_caches, + time_step=time_step, + attn_mask=attn_mask, + dropout_rate=self.dropout_prob, + training=self.training, + ) if self.has_cache_kv: return final_out[0], final_out[1] @@ -492,11 +566,16 @@ class TestFusedMultiTransformerOp(OpTest): cache_kvs = [] max_seq_length = (self.cache_length + 128) // 128 * 128 - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) elems = 4 if self.x_type is np.float16: @@ -504,37 +583,53 @@ class TestFusedMultiTransformerOp(OpTest): assert self.head_dim % elems == 0 v_elems = self.head_dim // elems - cache_k_tmp = self.cache_kv[0].reshape([ - self.batch_size, self.num_heads, self.cache_length, v_elems, - elems - ]) + cache_k_tmp = self.cache_kv[0].reshape( + [ + self.batch_size, + self.num_heads, + self.cache_length, + v_elems, + elems, + ] + ) # [B, num_head, head_dim / 4, 128, 4] cache_k_tmp = cache_k_tmp.transpose([0, 1, 3, 2, 4]) - cache_kv[0, :].reshape([ - self.batch_size, self.num_heads, v_elems, max_seq_length, elems - ])[:, :, :, :self.cache_length, :] = cache_k_tmp - - cache_kv[1, :, :, :self.cache_length, :] = self.cache_kv[1] + cache_kv[0, :].reshape( + [ + self.batch_size, + self.num_heads, + v_elems, + max_seq_length, + elems, + ] + )[:, :, :, : self.cache_length, :] = cache_k_tmp + + cache_kv[1, :, :, : self.cache_length, :] = self.cache_kv[1] if self.gen_cache_kv: assert self.query_length == self.cache_length cache_kv[:] = 0 else: - time_step = layers.fill_constant(shape=[1], - dtype="int32", - value=0, - force_cpu=True) + time_step = layers.fill_constant( + shape=[1], dtype="int32", value=0, force_cpu=True + ) time_step_feed = self.cache_length if self.has_pre_cache: cache_kvs = [] - max_seq_length = (self.cache_length + - 128) // 128 * 128 + self.pre_cache_num - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + max_seq_length = ( + self.cache_length + 128 + ) // 128 * 128 + self.pre_cache_num + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) pre_caches = [] attn_mask = None @@ -585,35 +680,43 @@ class TestFusedMultiTransformerOp(OpTest): ffn1_weight_attrs=ffn1_weights_attr, ffn1_bias_attrs=ffn1_biases_attr, ffn2_weight_attrs=ffn2_weights_attr, - ffn2_bias_attrs=ffn2_biases_attr) + ffn2_bias_attrs=ffn2_biases_attr, + ) transformer.eval() for i in range(self.layers): if self.has_cache_kv: cache_kvs.append( - layers.fill_constant(shape=cache_kv.shape, - dtype=cache_kv.dtype, - value=0)) + layers.fill_constant( + shape=cache_kv.shape, dtype=cache_kv.dtype, value=0 + ) + ) cache_kvs_feed.append(cache_kv) if self.has_pre_cache: cache_kvs.append( - layers.fill_constant(shape=cache_kv.shape, - dtype=cache_kv.dtype, - value=0)) + layers.fill_constant( + shape=cache_kv.shape, dtype=cache_kv.dtype, value=0 + ) + ) cache_kvs_feed.append(cache_kv) pre_caches.append( - layers.fill_constant(shape=self.pre_cache_kv.shape, - dtype=self.pre_cache_kv.dtype, - value=0)) + layers.fill_constant( + shape=self.pre_cache_kv.shape, + dtype=self.pre_cache_kv.dtype, + value=0, + ) + ) pre_caches_feed.append(self.pre_cache_kv) - final_out = transformer(x, - attn_mask=attn_mask, - caches=cache_kvs, - pre_caches=pre_caches, - time_step=time_step)[0] + final_out = transformer( + x, + attn_mask=attn_mask, + caches=cache_kvs, + pre_caches=pre_caches, + time_step=time_step, + )[0] exe = paddle.static.Executor(place=paddle.CUDAPlace(0)) exe.run(paddle.static.default_startup_program()) feed_data = { @@ -621,11 +724,13 @@ class TestFusedMultiTransformerOp(OpTest): 'cache_kvs': cache_kvs_feed, 'pre_caches': pre_caches_feed, 'time_step': time_step_feed, - 'attn_mask': attn_mask + 'attn_mask': attn_mask, } - out = exe.run(paddle.fluid.default_main_program(), - feed=feed_data, - fetch_list=[final_out]) + out = exe.run( + paddle.fluid.default_main_program(), + feed=feed_data, + fetch_list=[final_out], + ) paddle.disable_static() return out[0] @@ -644,9 +749,11 @@ class TestFusedMultiTransformerOp(OpTest): if self.debug: print("cache_k out timestep=128") - print(cache_kv_out[0].reshape( - [2, bsz, num_head, v_elems, max_seq_len, - elems])[0, 0, 0, :, self.cache_length, :]) + print( + cache_kv_out[0].reshape( + [2, bsz, num_head, v_elems, max_seq_len, elems] + )[0, 0, 0, :, self.cache_length, :] + ) print("cache_v out timestep=128") print(cache_kv_out[0][1, 0, 0, self.cache_length, :]) @@ -659,33 +766,31 @@ class TestFusedMultiTransformerOp(OpTest): cache_k = cache_kv_out[i][0, :] cache_k = cache_k.reshape( - [bsz, num_head, v_elems, max_seq_len, elems]) - cache_k = cache_k[:, :, :, :self.cache_length, :] + [bsz, num_head, v_elems, max_seq_len, elems] + ) + cache_k = cache_k[:, :, :, : self.cache_length, :] cache_k = cache_k.transpose([0, 1, 3, 2, 4]) cache_k = cache_k.reshape( - [bsz, num_head, self.cache_length, head_dim]) - - cache_v = cache_kv_out[i][1, :, :, :self.cache_length, :] - - np.testing.assert_allclose(cache_k_ref, - cache_k, - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(cache_v_ref, - cache_v, - rtol=self.rtol, - atol=self.atol) + [bsz, num_head, self.cache_length, head_dim] + ) + + cache_v = cache_kv_out[i][1, :, :, : self.cache_length, :] + + np.testing.assert_allclose( + cache_k_ref, cache_k, rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + cache_v_ref, cache_v, rtol=self.rtol, atol=self.atol + ) if i == 0: break - np.testing.assert_allclose(final_out_ref, - final_out, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out, rtol=self.rtol, atol=self.atol + ) class TestFusedMultiTransformerOpFp16(TestFusedMultiTransformerOp): - def config(self): super().config() self.x_type = np.float16 @@ -693,7 +798,6 @@ class TestFusedMultiTransformerOpFp16(TestFusedMultiTransformerOp): class TestFusedMultiTransformerOpCacheKV(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_cache_kv = True @@ -703,7 +807,6 @@ class TestFusedMultiTransformerOpCacheKV(TestFusedMultiTransformerOp): class TestFusedMultiTransformerOpCacheKVFp16(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_cache_kv = True @@ -713,7 +816,6 @@ class TestFusedMultiTransformerOpCacheKVFp16(TestFusedMultiTransformerOp): class TestFusedMultiTransformerOpGenCacheKV(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_cache_kv = True @@ -721,7 +823,6 @@ class TestFusedMultiTransformerOpGenCacheKV(TestFusedMultiTransformerOp): class TestFusedMultiTransformerOpGenCacheKVFp16(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_cache_kv = True @@ -731,7 +832,6 @@ class TestFusedMultiTransformerOpGenCacheKVFp16(TestFusedMultiTransformerOp): class TestFusedMultiTransformerOpPostLayerNormFp16(TestFusedMultiTransformerOp): - def config(self): super().config() self.x_type = np.float16 @@ -740,8 +840,8 @@ class TestFusedMultiTransformerOpPostLayerNormFp16(TestFusedMultiTransformerOp): class TestFusedMultiTransformerOpCacheKVPostLayerNorm( - TestFusedMultiTransformerOp): - + TestFusedMultiTransformerOp +): def config(self): super().config() self.has_cache_kv = True @@ -752,8 +852,8 @@ class TestFusedMultiTransformerOpCacheKVPostLayerNorm( class TestFusedMultiTransformerOpCacheKVPostLayerNormFp16( - TestFusedMultiTransformerOp): - + TestFusedMultiTransformerOp +): def config(self): super().config() self.has_cache_kv = True @@ -764,8 +864,8 @@ class TestFusedMultiTransformerOpCacheKVPostLayerNormFp16( class TestFusedMultiTransformerOpGenCacheKVPostLayerNorm( - TestFusedMultiTransformerOp): - + TestFusedMultiTransformerOp +): def config(self): super().config() self.has_cache_kv = True @@ -774,8 +874,8 @@ class TestFusedMultiTransformerOpGenCacheKVPostLayerNorm( class TestFusedMultiTransformerOpGenCacheKVPostLayerNormFp16( - TestFusedMultiTransformerOp): - + TestFusedMultiTransformerOp +): def config(self): super().config() self.has_cache_kv = True @@ -786,7 +886,6 @@ class TestFusedMultiTransformerOpGenCacheKVPostLayerNormFp16( class TestFusedMultiTransformerOpPreCache(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_pre_cache = True @@ -794,29 +893,31 @@ class TestFusedMultiTransformerOpPreCache(TestFusedMultiTransformerOp): class TestFusedMultiTransformerOpPreCacheStatic(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_pre_cache = True self.has_attn_mask = False self.x_type = np.float32 self.weight_attr = paddle.ParamAttr( - initializer=paddle.fluid.initializer.Constant(0.)) + initializer=paddle.fluid.initializer.Constant(0.0) + ) self.bias_attr = paddle.ParamAttr( - initializer=paddle.fluid.initializer.Constant(0.0005)) + initializer=paddle.fluid.initializer.Constant(0.0005) + ) self.ln_w_attr = paddle.ParamAttr( - initializer=paddle.fluid.initializer.Constant(1.)) + initializer=paddle.fluid.initializer.Constant(1.0) + ) self.ln_b_attr = paddle.ParamAttr( - initializer=paddle.fluid.initializer.Constant(0.)) + initializer=paddle.fluid.initializer.Constant(0.0) + ) def test_fused_multi_transformer_op(self): final_out_ref = self.GetBaselineOut() final_out = self.GetFusedMultiTransformerOutStatic() - np.testing.assert_allclose(final_out_ref, - final_out, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out, rtol=self.rtol, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fused_multihead_matmul_op.py b/python/paddle/fluid/tests/unittests/test_fused_multihead_matmul_op.py index 717a98abd50..e2b53903b6d 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_multihead_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_multihead_matmul_op.py @@ -24,15 +24,15 @@ def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Paddle core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Paddle core is not compiled with CUDA" +) class TestFusedMultiheadMatmulOp(OpTest): - def config(self): self.seq_len = 128 self.size_per_head = 64 @@ -45,13 +45,15 @@ class TestFusedMultiheadMatmulOp(OpTest): self.config() h = self.seq_len w = self.head_number * self.size_per_head - self.Input = np.random.random( - (self.batch_size, h, w)).astype("float32") - 0.5 + self.Input = ( + np.random.random((self.batch_size, h, w)).astype("float32") - 0.5 + ) self.WQ = np.random.random((w, w)).astype("float32") self.KQ = np.random.random((w, w)).astype("float32") self.VQ = np.random.random((w, w)).astype("float32") self.CombinedW = np.hstack((self.WQ, self.KQ, self.VQ)).reshape( - (w, 3, w)) + (w, 3, w) + ) self.Q = np.dot(self.Input, self.WQ) self.K = np.dot(self.Input, self.KQ) self.V = np.dot(self.Input, self.VQ) @@ -61,18 +63,32 @@ class TestFusedMultiheadMatmulOp(OpTest): self.BiasV = np.random.random((1, w)).astype("float32") self.CombinedB = np.vstack((self.BiasQ, self.BiasK, self.BiasV)) self.BiasQK = np.random.random( - (self.batch_size, self.head_number, self.seq_len, - self.seq_len)).astype("float32") + (self.batch_size, self.head_number, self.seq_len, self.seq_len) + ).astype("float32") # Compute Q path fc_q = self.Q + self.BiasQ - reshape_q = np.reshape(fc_q, (self.batch_size, self.seq_len, - self.head_number, self.size_per_head)) + reshape_q = np.reshape( + fc_q, + ( + self.batch_size, + self.seq_len, + self.head_number, + self.size_per_head, + ), + ) transpose_q = np.transpose(reshape_q, (0, 2, 1, 3)) scale_q = self.scale * transpose_q # Compute K path fc_k = self.K + self.BiasK - reshape_k = np.reshape(fc_k, (self.batch_size, self.seq_len, - self.head_number, self.size_per_head)) + reshape_k = np.reshape( + fc_k, + ( + self.batch_size, + self.seq_len, + self.head_number, + self.size_per_head, + ), + ) transpose_k = np.transpose(reshape_k, (0, 2, 3, 1)) # Compute Q*K @@ -81,8 +97,15 @@ class TestFusedMultiheadMatmulOp(OpTest): softmax_qk = np.apply_along_axis(stable_softmax, 3, eltadd_qk) # Compute V path fc_v = self.V + self.BiasV - reshape_v = np.reshape(fc_v, (self.batch_size, self.seq_len, - self.head_number, self.size_per_head)) + reshape_v = np.reshape( + fc_v, + ( + self.batch_size, + self.seq_len, + self.head_number, + self.size_per_head, + ), + ) transpose_v = np.transpose(reshape_v, (0, 2, 1, 3)) # Compute QK*V @@ -94,14 +117,14 @@ class TestFusedMultiheadMatmulOp(OpTest): "Input": self.Input, "W": self.CombinedW, "Bias": self.CombinedB, - "BiasQK": self.BiasQK + "BiasQK": self.BiasQK, } self.attrs = { "transpose_Q": False, "transpose_K": True, "transpose_V": False, "head_number": self.head_number, - "alpha": self.scale + "alpha": self.scale, } self.outputs = {"Out": reshape_qkv} @@ -111,7 +134,6 @@ class TestFusedMultiheadMatmulOp(OpTest): class TestFusedMultiHeadMatmulOp2(TestFusedMultiheadMatmulOp): - def config(self): self.seq_len = 256 self.size_per_head = 32 diff --git a/python/paddle/fluid/tests/unittests/test_fused_token_prune_op.py b/python/paddle/fluid/tests/unittests/test_fused_token_prune_op.py index adfb9e09177..22d0fe4ba76 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_token_prune_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_token_prune_op.py @@ -18,10 +18,10 @@ from op_test import OpTest from paddle.framework import core -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFusedTokenPruneOp(OpTest): - def setDtype(self): self.dtype = np.float32 @@ -30,19 +30,21 @@ class TestFusedTokenPruneOp(OpTest): attn = np.array(attn, dtype=self.dtype) attn = np.expand_dims(attn, axis=0) self.attn = np.expand_dims( - attn, axis=0) # [1,1,2,2] bsz = 1, nd_head=1, max_seq_len=2 + attn, axis=0 + ) # [1,1,2,2] bsz = 1, nd_head=1, max_seq_len=2 mask = [[1, 1], [-1, -1]] mask = np.array(mask, dtype=self.dtype) mask = np.expand_dims(mask, axis=0) self.mask = np.expand_dims(mask, axis=0) # same as attn x = [[1, 2, 3], [4, 5, 6]] x = np.array(x, dtype=self.dtype) - self.x = np.expand_dims(x, - axis=0) # [1, 2, 3] bsz = 1, max_seq_len=2, c=3 + self.x = np.expand_dims( + x, axis=0 + ) # [1, 2, 3] bsz = 1, max_seq_len=2, c=3 new_mask = [[1]] new_mask = np.array(new_mask, dtype=self.dtype) new_mask = np.expand_dims(new_mask, axis=0) - self.new_mask = np.expand_dims(new_mask, axis=0) #[1, 1, 1, 1] + self.new_mask = np.expand_dims(new_mask, axis=0) # [1, 1, 1, 1] out_slimmedx_py = [[[1, 2, 3]]] self.out_slimmedx_py = np.array(out_slimmedx_py, dtype=self.dtype) @@ -58,49 +60,68 @@ class TestFusedTokenPruneOp(OpTest): 'Attn': self.attn, 'Mask': self.mask, 'X': self.x, - 'NewMask': self.new_mask + 'NewMask': self.new_mask, } self.outputs = { 'SlimmedX': self.out_slimmedx_py, - 'CLSInds': self.out_cls_inds_py + 'CLSInds': self.out_cls_inds_py, } def test_check_output(self): self.check_output_with_place(core.CUDAPlace(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFusedTokenPruneOpFloat64(TestFusedTokenPruneOp): - def setDtype(self): self.dtype = np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFusedTokenPruneOp2(TestFusedTokenPruneOp): - def setInouts(self): - attn = [[[[1, 2, 3, 4], [4, 3, 2, 1], [5, 9, 5, 4], [9, 6, 5, 4]], - [[8, 5, 2, 0], [1, 0, 2, 3], [2, 2, 3, 2], [7, 4, 1, 8]]]] + attn = [ + [ + [[1, 2, 3, 4], [4, 3, 2, 1], [5, 9, 5, 4], [9, 6, 5, 4]], + [[8, 5, 2, 0], [1, 0, 2, 3], [2, 2, 3, 2], [7, 4, 1, 8]], + ] + ] self.attn = np.array( - attn, - dtype=self.dtype) # [1,2,4,4] bsz = 1, nd_head=2, max_seq_len=4 - mask = [[[[-1, -1, -1, 1], [-1, -1, 1, 1], [-1, -1, 1, 1], - [-1, -1, 1, 1]], - [[-1, -1, 1, 1], [-1, -1, 1, 1], [-1, -1, 1, 1], - [-1, -1, 1, 1]]]] + attn, dtype=self.dtype + ) # [1,2,4,4] bsz = 1, nd_head=2, max_seq_len=4 + mask = [ + [ + [ + [-1, -1, -1, 1], + [-1, -1, 1, 1], + [-1, -1, 1, 1], + [-1, -1, 1, 1], + ], + [ + [-1, -1, 1, 1], + [-1, -1, 1, 1], + [-1, -1, 1, 1], + [-1, -1, 1, 1], + ], + ] + ] self.mask = np.array(mask, dtype=self.dtype) # same as attn - x = [[[1.1, 1.1, 1.1], [2.2, 2.2, 2.2], [3.3, 3.3, 3.3], - [4.4, 4.4, 4.4]]] + x = [ + [[1.1, 1.1, 1.1], [2.2, 2.2, 2.2], [3.3, 3.3, 3.3], [4.4, 4.4, 4.4]] + ] self.x = np.array( - x, dtype=self.dtype) # [1, 4, 3] bsz = 1, max_seq_len=4, c=3 - self.new_mask = np.random.rand(1, 2, 2, - 2).astype(self.dtype) #[1, 2, 2, 2] + x, dtype=self.dtype + ) # [1, 4, 3] bsz = 1, max_seq_len=4, c=3 + self.new_mask = np.random.rand(1, 2, 2, 2).astype( + self.dtype + ) # [1, 2, 2, 2] - out_slimmedx_py = [[[1.1, 1.1, 1.1], [4.4, 4.4, 4.4]]] #[1, 2, 3] + out_slimmedx_py = [[[1.1, 1.1, 1.1], [4.4, 4.4, 4.4]]] # [1, 2, 3] self.out_slimmedx_py = np.array(out_slimmedx_py, dtype=self.dtype) out_cls_inds_py = [[0, 3]] diff --git a/python/paddle/fluid/tests/unittests/test_fused_transformer_encoder_layer.py b/python/paddle/fluid/tests/unittests/test_fused_transformer_encoder_layer.py index 50632b3f76f..a8a349265b4 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_transformer_encoder_layer.py +++ b/python/paddle/fluid/tests/unittests/test_fused_transformer_encoder_layer.py @@ -21,7 +21,6 @@ import unittest class TestFusedTransformerEncoderLayer(unittest.TestCase): - def setActivation(self): self.activation = 'gelu' @@ -61,8 +60,8 @@ class TestFusedTransformerEncoderLayer(unittest.TestCase): def fused_weight(self, weight, num_head): a = paddle.transpose(weight, perm=[1, 0]) return paddle.reshape( - a, shape=[1, num_head, - int(a.shape[0] / num_head), a.shape[1]]) + a, shape=[1, num_head, int(a.shape[0] / num_head), a.shape[1]] + ) def fused_qkv(self, q, k, v, num_head): fq = self.fused_weight(q, num_head) @@ -75,16 +74,29 @@ class TestFusedTransformerEncoderLayer(unittest.TestCase): return default_main_program().random_seed = 42 base_encoder = TransformerEncoderLayer( - self.d_model, self.nhead, self.dim_feedforward, self.dropout_rate, - self.activation, self.attn_dropout_rate, self.act_dropout_rate, - self.pre_layer_norm) - src = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.dtype) + self.d_model, + self.nhead, + self.dim_feedforward, + self.dropout_rate, + self.activation, + self.attn_dropout_rate, + self.act_dropout_rate, + self.pre_layer_norm, + ) + src = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.dtype) if self.has_attn_mask: - attn_mask = np.ones((self.batch_size, self.num_heads, - self.query_length, self.key_length), - dtype=self.attn_mask_type) + attn_mask = np.ones( + ( + self.batch_size, + self.num_heads, + self.query_length, + self.key_length, + ), + dtype=self.attn_mask_type, + ) attn_mask_tensor = paddle.to_tensor(attn_mask) else: attn_mask = None @@ -92,14 +104,21 @@ class TestFusedTransformerEncoderLayer(unittest.TestCase): dout = np.random.random(src.shape).astype(self.dtype) - base_out = base_encoder(paddle.to_tensor(src, stop_gradient=False), - attn_mask_tensor) + base_out = base_encoder( + paddle.to_tensor(src, stop_gradient=False), attn_mask_tensor + ) paddle.autograd.backward([base_out], [paddle.to_tensor(dout)], True) fused_encoder = FusedTransformerEncoderLayer( - self.d_model, self.nhead, self.dim_feedforward, self.dropout_rate, - self.activation, self.attn_dropout_rate, self.act_dropout_rate, - self.pre_layer_norm) + self.d_model, + self.nhead, + self.dim_feedforward, + self.dropout_rate, + self.activation, + self.attn_dropout_rate, + self.act_dropout_rate, + self.pre_layer_norm, + ) fused_encoder.ffn._linear1_weight.set_value(base_encoder.linear1.weight) fused_encoder.ffn._linear1_bias.set_value(base_encoder.linear1.bias) @@ -113,17 +132,22 @@ class TestFusedTransformerEncoderLayer(unittest.TestCase): fused_encoder.ffn._ln2_bias.set_value(base_encoder.norm2.bias) fused_encoder.fused_attn.linear_weight.set_value( - base_encoder.self_attn.out_proj.weight) + base_encoder.self_attn.out_proj.weight + ) fused_encoder.fused_attn.linear_bias.set_value( - base_encoder.self_attn.out_proj.bias) + base_encoder.self_attn.out_proj.bias + ) if self.pre_layer_norm: fused_encoder.fused_attn.pre_ln_scale.set_value( - base_encoder.norm1.weight) + base_encoder.norm1.weight + ) fused_encoder.fused_attn.pre_ln_bias.set_value( - base_encoder.norm1.bias) + base_encoder.norm1.bias + ) else: fused_encoder.fused_attn.ln_scale.set_value( - base_encoder.norm1.weight) + base_encoder.norm1.weight + ) fused_encoder.fused_attn.ln_bias.set_value(base_encoder.norm1.bias) q = base_encoder.self_attn.q_proj.weight @@ -138,59 +162,74 @@ class TestFusedTransformerEncoderLayer(unittest.TestCase): tmp = paddle.concat(x=[q_bias, k_bias, v_bias], axis=0) qkv_bias = paddle.reshape( tmp, - shape=[3, self.num_heads, - int(tmp.shape[0] / 3 / self.num_heads)]) + shape=[3, self.num_heads, int(tmp.shape[0] / 3 / self.num_heads)], + ) fused_encoder.fused_attn.qkv_bias.set_value(qkv_bias) - fused_out = fused_encoder(paddle.to_tensor(src, stop_gradient=False), - attn_mask_tensor) + fused_out = fused_encoder( + paddle.to_tensor(src, stop_gradient=False), attn_mask_tensor + ) paddle.autograd.backward([fused_out], [paddle.to_tensor(dout)], True) correct_ffn_str = 'd_model={}, dim_feedforward={}, dropout_rate={}, epsilon={}, activation={}, act_dropout_rate={}, normalize_before={}, dtype={}'.format( - self.d_model, self.dim_feedforward, self.dropout_rate, - fused_encoder.ffn._epsilon, self.activation, self.dropout_rate, - self.pre_layer_norm, self.dtype) + self.d_model, + self.dim_feedforward, + self.dropout_rate, + fused_encoder.ffn._epsilon, + self.activation, + self.dropout_rate, + self.pre_layer_norm, + self.dtype, + ) self.assertTrue(fused_encoder.ffn.extra_repr(), correct_ffn_str) correct_attn_str = 'embed_dim={}, num_heads={}, dropout_rate={}, attn_dropout_rate={}, epsilon={}, kdim={}, vdim={}, normalize_before={}, need_weights={}, dtype={}'.format( - self.embed_dim, self.num_heads, self.dropout_rate, - self.dropout_rate, fused_encoder.fused_attn._epsilon, None, None, - self.pre_layer_norm, False, self.dtype) + self.embed_dim, + self.num_heads, + self.dropout_rate, + self.dropout_rate, + fused_encoder.fused_attn._epsilon, + None, + None, + self.pre_layer_norm, + False, + self.dtype, + ) self.assertTrue(fused_encoder.fused_attn.extra_repr(), correct_attn_str) - np.testing.assert_allclose(fused_out.numpy(), - base_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(fused_out.grad.numpy(), - base_out.grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + fused_out.numpy(), base_out.numpy(), rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + fused_out.grad.numpy(), + base_out.grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) class TestFusedTransformerEncoderLayerAct(TestFusedTransformerEncoderLayer): - def setActivation(self): self.activation = 'relu' class TestFusedTransformerEncoderLayerPreLayerNorm( - TestFusedTransformerEncoderLayer): - + TestFusedTransformerEncoderLayer +): def setPreLayerNorm(self): self.pre_layer_norm = True class TestFusedTransformerEncoderLayerAttnMaskIsNone( - TestFusedTransformerEncoderLayer): - + TestFusedTransformerEncoderLayer +): def setAttnMask(self): self.has_attn_mask = False class TestFusedTransformerEncoderLayerPreLnTrueAttnMaskIsNone( - TestFusedTransformerEncoderLayer): - + TestFusedTransformerEncoderLayer +): def setPreLayerNorm(self): self.pre_layer_norm = True diff --git a/python/paddle/fluid/tests/unittests/test_fused_transformer_with_amp_decorator.py b/python/paddle/fluid/tests/unittests/test_fused_transformer_with_amp_decorator.py index f0173d9632f..212db2406aa 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_transformer_with_amp_decorator.py +++ b/python/paddle/fluid/tests/unittests/test_fused_transformer_with_amp_decorator.py @@ -14,12 +14,14 @@ import paddle import paddle.nn as nn -from paddle.incubate.nn.layer.fused_transformer import FusedMultiHeadAttention, FusedFeedForward +from paddle.incubate.nn.layer.fused_transformer import ( + FusedMultiHeadAttention, + FusedFeedForward, +) import unittest class PreModel(nn.Layer): - def __init__(self): super(PreModel, self).__init__() self.attn = FusedMultiHeadAttention( @@ -27,9 +29,9 @@ class PreModel(nn.Layer): num_heads=16, normalize_before=False, ) - self.ffn = FusedFeedForward(d_model=1024, - dim_feedforward=4096, - normalize_before=False) + self.ffn = FusedFeedForward( + d_model=1024, dim_feedforward=4096, normalize_before=False + ) def forward(self, x): x = self.attn(x) @@ -37,7 +39,6 @@ class PreModel(nn.Layer): class PostModel(nn.Layer): - def __init__(self): super(PostModel, self).__init__() self.attn = FusedMultiHeadAttention( @@ -45,9 +46,9 @@ class PostModel(nn.Layer): num_heads=16, normalize_before=True, ) - self.ffn = FusedFeedForward(d_model=1024, - dim_feedforward=4096, - normalize_before=True) + self.ffn = FusedFeedForward( + d_model=1024, dim_feedforward=4096, normalize_before=True + ) def forward(self, x): x = self.attn(x) @@ -55,19 +56,18 @@ class PostModel(nn.Layer): class TestFusedTransformerWithAmpDecorator(unittest.TestCase): - def get_model(self): self.pre_model = PreModel() self.post_model = PostModel() def test_run(self): self.get_model() - pre_model = paddle.amp.decorate(models=self.pre_model, - level='O2', - save_dtype='float32') - post_model = paddle.amp.decorate(models=self.post_model, - level='O2', - save_dtype='float32') + pre_model = paddle.amp.decorate( + models=self.pre_model, level='O2', save_dtype='float32' + ) + post_model = paddle.amp.decorate( + models=self.post_model, level='O2', save_dtype='float32' + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py b/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py index e2aa2fc901e..68777ca2567 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py @@ -20,29 +20,31 @@ from paddle.fluid.tests.unittests.test_fusion_lstm_op import fc, ACTIVATION def fusion_gru( - x, # T x M - lod, # 1 x N - h0, # N x D - wx, # M x 3D - wh, # D x 3D - bias, # 1 x 3D + x, # T x M + lod, # 1 x N + h0, # N x D + wx, # M x 3D + wh, # D x 3D + bias, # 1 x 3D + is_reverse, + origin_mode, + act_state, + act_gate, +): + return gru( + fc(x, wx, bias), + lod, + h0, + wh, + np.zeros((1, wh.shape[1]), dtype='float32'), is_reverse, - origin_mode, act_state, - act_gate): - return gru(fc(x, wx, bias), - lod, - h0, - wh, - np.zeros((1, wh.shape[1]), dtype='float32'), - is_reverse, - act_state, - act_gate, - origin_mode=origin_mode) + act_gate, + origin_mode=origin_mode, + ) class TestFusionGRUOp(OpTest): - def set_confs(self): pass @@ -66,17 +68,29 @@ class TestFusionGRUOp(OpTest): x = np.random.rand(T, self.M).astype('float32') wx = np.random.rand(self.M, 3 * self.D).astype('float32') wh = np.random.rand(self.D, 3 * self.D).astype('float32') - bias = np.random.rand( - 1, 3 * self.D).astype('float32') if self.with_bias else np.zeros( - (1, 3 * self.D), dtype='float32') - h0 = np.random.rand( - N, self.D).astype('float32') if self.with_h0 else np.zeros( - (N, self.D), dtype='float32') - - _, _, _, hidden = fusion_gru(x, self.lod, h0, wx, wh, bias, - self.is_reverse, self.origin_mode, - ACTIVATION[self.act_state], - ACTIVATION[self.act_gate]) + bias = ( + np.random.rand(1, 3 * self.D).astype('float32') + if self.with_bias + else np.zeros((1, 3 * self.D), dtype='float32') + ) + h0 = ( + np.random.rand(N, self.D).astype('float32') + if self.with_h0 + else np.zeros((N, self.D), dtype='float32') + ) + + _, _, _, hidden = fusion_gru( + x, + self.lod, + h0, + wx, + wh, + bias, + self.is_reverse, + self.origin_mode, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate], + ) self.inputs = {'X': (x, self.lod), 'WeightX': wx, 'WeightH': wh} @@ -93,7 +107,7 @@ class TestFusionGRUOp(OpTest): 'gate_activation': self.act_gate, 'is_reverse': self.is_reverse, 'origin_mode': self.origin_mode, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def test_check_output(self): @@ -103,46 +117,39 @@ class TestFusionGRUOp(OpTest): class TestFusionGRUOpNoInitial(TestFusionGRUOp): - def set_confs(self): self.with_h0 = False class TestFusionGRUOpNoBias(TestFusionGRUOp): - def set_confs(self): self.with_bias = False class TestFusionGRUOpReverse(TestFusionGRUOp): - def set_confs(self): self.is_reverse = True class TestFusionGRUOpMD1(TestFusionGRUOp): - def set_confs(self): self.M = 36 self.D = 8 class TestFusionGRUOpMD2(TestFusionGRUOp): - def set_confs(self): self.M = 8 self.D = 8 class TestFusionGRUOpMD3(TestFusionGRUOp): - def set_confs(self): self.M = 17 self.D = 15 class TestFusionGRUOpBS1(TestFusionGRUOp): - def set_confs(self): self.lod = [[3]] self.D = 16 @@ -150,5 +157,6 @@ class TestFusionGRUOpBS1(TestFusionGRUOp): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py b/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py index 9b1d9dfffc6..fcde3d2f5db 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py @@ -23,25 +23,36 @@ def fc(x, w, b): def fusion_lstm( - x, # T x M - lod, # 1 x N - wx=None, # M x 4D - bx=None, # 1 x 4D - h0=None, # N x D - c0=None, # N x D - w_h=None, # D x 4D - w_b=None, # 1 x 4D - w_c=None, # 1 x 3D - is_reverse=False, - act_gate=None, - act_cell=None, - act_cand=None): - return lstm(fc(x, wx, bx), lod, h0, c0, w_h, w_b, w_c, is_reverse, act_gate, - act_cell, act_cand) + x, # T x M + lod, # 1 x N + wx=None, # M x 4D + bx=None, # 1 x 4D + h0=None, # N x D + c0=None, # N x D + w_h=None, # D x 4D + w_b=None, # 1 x 4D + w_c=None, # 1 x 3D + is_reverse=False, + act_gate=None, + act_cell=None, + act_cand=None, +): + return lstm( + fc(x, wx, bx), + lod, + h0, + c0, + w_h, + w_b, + w_c, + is_reverse, + act_gate, + act_cell, + act_cand, + ) class TestFusionLSTMOp(OpTest): - def set_conf(self): pass @@ -76,24 +87,36 @@ class TestFusionLSTMOp(OpTest): b = np.random.normal(size=(1, 7 * self.D)).astype('float32') else: b = np.random.normal(size=(1, 4 * self.D)).astype('float32') - w_b = np.copy(b[:, 0:4 * self.D]) - w_c = b[:, 4 * self.D:] if self.use_peepholes else None + w_b = np.copy(b[:, 0 : 4 * self.D]) + w_c = b[:, 4 * self.D :] if self.use_peepholes else None # this is the weight of fc wx = np.random.normal(size=(self.M, 4 * self.D)).astype('float32') # this is the bias of fc # and it should be manually added into the bias of this fusion LSTM bx = np.random.normal(size=(1, 4 * self.D)).astype('float32') - b[0, 0:4 * self.D] += bx[0, :] - h, c = fusion_lstm(x, self.lod, wx, bx, h0, c0, wh, w_b, w_c, - self.is_reverse, ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], ACTIVATION[self.act_cand]) + b[0, 0 : 4 * self.D] += bx[0, :] + h, c = fusion_lstm( + x, + self.lod, + wx, + bx, + h0, + c0, + wh, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = { 'X': (x, self.lod), 'WeightX': wx, 'WeightH': wh, - 'Bias': b + 'Bias': b, } if self.has_initial_state: @@ -110,7 +133,7 @@ class TestFusionLSTMOp(OpTest): 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, 'candidate_activation': self.act_cand, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def test_check_output(self): @@ -120,74 +143,63 @@ class TestFusionLSTMOp(OpTest): class TestFusionLSTMOpInit(TestFusionLSTMOp): - def set_conf(self): self.has_initial_state = True class TestFusionLSTMOpReverse(TestFusionLSTMOp): - def set_conf(self): self.is_reverse = True class TestFusionLSTMOpInitReverse(TestFusionLSTMOp): - def set_conf(self): self.has_initial_state = True self.is_reverse = True class TestFusionLSTMOpMD1(TestFusionLSTMOp): - def set_conf(self): self.M = 36 self.D = 8 class TestFusionLSTMOpMD2(TestFusionLSTMOp): - def set_conf(self): self.M = 8 self.D = 8 class TestFusionLSTMOpMD3(TestFusionLSTMOp): - def set_conf(self): self.M = 15 self.D = 3 class TestFusionLSTMOpBS1(TestFusionLSTMOp): - def set_conf(self): self.lod = [[3]] self.D = 16 class TestFusionLSTMOpPeepholes(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True class TestFusionLSTMOpPeepholesInit(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True class TestFusionLSTMOpPeepholesReverse(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.is_reverse = True class TestFusionLSTMOpPeepholesInitReverse(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True @@ -195,7 +207,6 @@ class TestFusionLSTMOpPeepholesInitReverse(TestFusionLSTMOp): class TestFusionLSTMOpPeepholesBS1(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.lod = [[2]] @@ -204,5 +215,6 @@ class TestFusionLSTMOpPeepholesBS1(TestFusionLSTMOp): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fusion_repeated_fc_relu_op.py b/python/paddle/fluid/tests/unittests/test_fusion_repeated_fc_relu_op.py index c1b117dfe57..c58570d9de9 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_repeated_fc_relu_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_repeated_fc_relu_op.py @@ -19,7 +19,6 @@ from test_fc_op import fc_refer, MatrixGenerate class TestFusionRepeatedFCReluOp(OpTest): - def setUp(self): self.bs = 3 self.ic = 9 @@ -28,7 +27,7 @@ class TestFusionRepeatedFCReluOp(OpTest): self.set_conf() self.op_type = 'fusion_repeated_fc_relu' sz = len(self.oc) - ics = [self.ic] + self.oc[0:sz - 1] + ics = [self.ic] + self.oc[0 : sz - 1] assert len(ics) == len(self.oc) weights = [] biases = [] @@ -38,23 +37,32 @@ class TestFusionRepeatedFCReluOp(OpTest): matrix = MatrixGenerate(self.bs, ics[i], self.oc[i], 1, 1) inp = np.reshape(matrix.input, [self.bs, ics[i]]) weights.append( - ('W_{0}'.format(i), np.reshape(matrix.weights, - [ics[i], self.oc[i]]))) + ( + 'W_{0}'.format(i), + np.reshape(matrix.weights, [ics[i], self.oc[i]]), + ) + ) biases.append(('B_{0}'.format(i), matrix.bias)) outs.append( - np.reshape(np.maximum(fc_refer(matrix, True), 0), - [self.bs, self.oc[i]])) + np.reshape( + np.maximum(fc_refer(matrix, True), 0), [self.bs, self.oc[i]] + ) + ) for i in range(sz - 1): matrix = MatrixGenerate(self.bs, ics[i + 1], self.oc[i + 1], 1, 1) matrix.input = np.reshape(outs[i], [self.bs, ics[i + 1], 1, 1]) out = fc_refer(matrix, True) - weights.append(('W_{0}'.format(i + 1), - np.reshape(matrix.weights, - [ics[i + 1], self.oc[i + 1]]))) + weights.append( + ( + 'W_{0}'.format(i + 1), + np.reshape(matrix.weights, [ics[i + 1], self.oc[i + 1]]), + ) + ) biases.append(('B_{0}'.format(i + 1), matrix.bias)) outs.append( - np.reshape(np.maximum(out, 0), [self.bs, self.oc[i + 1]])) + np.reshape(np.maximum(out, 0), [self.bs, self.oc[i + 1]]) + ) relu_outs = [] for i in range(sz - 1): @@ -76,7 +84,6 @@ class TestFusionRepeatedFCReluOp(OpTest): class TestFusionRepeatedFCReluOpBS1(TestFusionRepeatedFCReluOp): - def set_conf(self): self.bs = 1 self.oc = [4, 2, 7, 5, 512, 1024] diff --git a/python/paddle/fluid/tests/unittests/test_fusion_seqconv_eltadd_relu_op.py b/python/paddle/fluid/tests/unittests/test_fusion_seqconv_eltadd_relu_op.py index cceced75adc..91ed3f56442 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_seqconv_eltadd_relu_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_seqconv_eltadd_relu_op.py @@ -19,7 +19,6 @@ from sequence.test_sequence_conv import seqconv class TestSeqConvEltAddRelu(OpTest): - def set_conf(self): pass @@ -38,8 +37,8 @@ class TestSeqConvEltAddRelu(OpTest): T = sum(self.lod[0]) x = np.random.uniform(-1, 1, [T, self.in_fea_size]).astype('float32') w = np.random.uniform( - -1, 1, [self.in_fea_size * self.context_length, self.out_fea_size - ]).astype('float32') + -1, 1, [self.in_fea_size * self.context_length, self.out_fea_size] + ).astype('float32') b = np.random.uniform(-2, 1, [1, self.out_fea_size]).astype('float32') out = seqconv(x, self.lod, w, self.context_length, self.context_start) out = np.maximum(out + b, 0) @@ -48,7 +47,7 @@ class TestSeqConvEltAddRelu(OpTest): self.attrs = { 'contextStart': self.context_start, 'contextLength': self.context_length, - 'contextStride': self.context_stride + 'contextStride': self.context_stride, } self.outputs = {'Out': out} @@ -57,19 +56,16 @@ class TestSeqConvEltAddRelu(OpTest): class TestSeqConvEltAddReluBS1(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[10]] class TestSeqConvEltAddReluBS1Case2(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[2]] class TestSeqConvEltAddReluCase1(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[3, 5, 1, 6]] self.context_length = 3 @@ -77,7 +73,6 @@ class TestSeqConvEltAddReluCase1(TestSeqConvEltAddRelu): class TestSeqConvEltAddReluCase2(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[10, 1, 2, 4, 1, 5, 6]] self.in_fea_size = 2 @@ -86,7 +81,6 @@ class TestSeqConvEltAddReluCase2(TestSeqConvEltAddRelu): class TestSeqConvEltAddReluCase3(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[10, 1, 2, 4, 1, 5, 6]] self.context_length = 5 diff --git a/python/paddle/fluid/tests/unittests/test_fusion_seqexpand_concat_fc_op.py b/python/paddle/fluid/tests/unittests/test_fusion_seqexpand_concat_fc_op.py index be3065ec86a..10a8623ec9c 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_seqexpand_concat_fc_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_seqexpand_concat_fc_op.py @@ -45,7 +45,6 @@ def fusion_seqexpand_concat_fc(xs, lod, w, b, fc_act): class TestFusionSeqExpandConcatFCOp(OpTest): - def set_conf(self): pass @@ -65,19 +64,24 @@ class TestFusionSeqExpandConcatFCOp(OpTest): x0 = np.random.normal(size=(T, self.inputs_M[0])).astype('float32') xs = [x0] for i in range(num_inputs - 1): - xi = np.random.normal(size=(bs, - self.inputs_M[i + 1])).astype('float32') + xi = np.random.normal(size=(bs, self.inputs_M[i + 1])).astype( + 'float32' + ) xs.append(xi) # fc weight and bias - w = np.random.normal(size=(sum(self.inputs_M), - self.D)).astype('float32') - b = np.random.normal( - size=(1, self.D)).astype('float32') if self.with_bias else np.zeros( - (1, self.D)).astype('float32') - - out = fusion_seqexpand_concat_fc(xs, self.lod, w, b, - ACTIVATION[self.fc_act]) + w = np.random.normal(size=(sum(self.inputs_M), self.D)).astype( + 'float32' + ) + b = ( + np.random.normal(size=(1, self.D)).astype('float32') + if self.with_bias + else np.zeros((1, self.D)).astype('float32') + ) + + out = fusion_seqexpand_concat_fc( + xs, self.lod, w, b, ACTIVATION[self.fc_act] + ) self.inputs = {'X': [('x0', (x0, self.lod))], 'FCWeight': w} normal_lod = [[1] * bs] @@ -95,47 +99,40 @@ class TestFusionSeqExpandConcatFCOp(OpTest): class TestFusionSECFCOpNonBias(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.with_bias = False class TestFusionSECFCOpNonAct(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.fc_act = 'identity' class TestFusionSECFCOpMD1(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.inputs_M = [3, 4, 2, 1, 5] self.D = 8 class TestFusionSECFCOpMD2(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.lod = [[5, 6]] self.inputs_M = [1, 1] class TestFusionSECFCOpBS1_1(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.lod = [[1]] self.inputs_M = [3, 4, 2] class TestFusionSECFCOpBS1_2(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.lod = [[1]] self.inputs_M = [3, 4] class TestFusionSECFCOpBS1_3(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.lod = [[5]] self.inputs_M = [6, 3] diff --git a/python/paddle/fluid/tests/unittests/test_fusion_seqpool_concat_op.py b/python/paddle/fluid/tests/unittests/test_fusion_seqpool_concat_op.py index c634d8d2b58..ceb69451c9b 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_seqpool_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_seqpool_concat_op.py @@ -16,11 +16,14 @@ import unittest import numpy as np from op_test import OpTest from test_reorder_lod_tensor import convert_to_offset -from sequence.test_sequence_pool import compute_seqpool_sum, compute_seqpool_avg, compute_seqpool_sqrt +from sequence.test_sequence_pool import ( + compute_seqpool_sum, + compute_seqpool_avg, + compute_seqpool_sqrt, +) class TestFusionSeqPoolConcatOp(OpTest): - def setUp(self): self.w = 11 self.lods = [[[2, 3, 5]], [[1, 5, 2]]] @@ -34,8 +37,9 @@ class TestFusionSeqPoolConcatOp(OpTest): i = 0 for lod in self.lods: assert bs == len(lod[0]), 'All lod size should be equal' - x = np.random.uniform(0.1, 1, - [sum(lod[0]), self.w]).astype('float32') + x = np.random.uniform(0.1, 1, [sum(lod[0]), self.w]).astype( + 'float32' + ) offset = convert_to_offset(lod) out = np.zeros((bs, self.w)).astype('float32') if self.pooltype == "SUM": @@ -68,26 +72,22 @@ class TestFusionSeqPoolConcatOp(OpTest): class TestFusionSeqPoolConcatOpCase1(TestFusionSeqPoolConcatOp): - def set_conf(self): self.lods = [[[1]]] class TestFusionSeqPoolConcatOpCase2(TestFusionSeqPoolConcatOp): - def set_conf(self): self.lods = [[[1]], [[1]], [[1]]] class TestFusionSeqPoolConcatOpCase3(TestFusionSeqPoolConcatOp): - def set_conf(self): self.lods = [[[1, 3, 4, 6]]] self.w = 10 class TestFusionSeqPoolConcatOpCase4(TestFusionSeqPoolConcatOp): - def set_conf(self): self.lods = [[[2, 13, 4]], [[1, 1, 1]], [[5, 3, 1]], [[9, 10, 3]]] self.w = 3 @@ -95,14 +95,11 @@ class TestFusionSeqPoolConcatOpCase4(TestFusionSeqPoolConcatOp): ## test avg pool and sqrt def create_test_avg_sqrt_class(parent): - class TestSeqPoolAvgCase(parent): - def set_pooltype(self): self.pooltype = "AVERAGE" class TestSeqPoolSqrtCase(parent): - def set_pooltype(self): self.pooltype = "SQRT" diff --git a/python/paddle/fluid/tests/unittests/test_fusion_seqpool_cvm_concat_op.py b/python/paddle/fluid/tests/unittests/test_fusion_seqpool_cvm_concat_op.py index be298c15535..9d9324cefe1 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_seqpool_cvm_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_seqpool_cvm_concat_op.py @@ -16,12 +16,15 @@ import unittest import numpy as np from op_test import OpTest from test_reorder_lod_tensor import convert_to_offset -from sequence.test_sequence_pool import compute_seqpool_sum, compute_seqpool_avg, compute_seqpool_sqrt +from sequence.test_sequence_pool import ( + compute_seqpool_sum, + compute_seqpool_avg, + compute_seqpool_sqrt, +) from test_cvm_op import cvm_compute class TestFusionSeqPoolCVMConcatOp(OpTest): - def setUp(self): self.w = 11 self.use_cvm = True @@ -38,8 +41,9 @@ class TestFusionSeqPoolCVMConcatOp(OpTest): i = 0 for lod in self.lods: assert bs == len(lod[0]), 'All lod size should be equal' - x = np.random.uniform(0.1, 1, - [sum(lod[0]), self.w]).astype('float32') + x = np.random.uniform(0.1, 1, [sum(lod[0]), self.w]).astype( + 'float32' + ) offset = convert_to_offset(lod) out = np.zeros((bs, self.w)).astype('float32') if self.pooltype == "SUM": @@ -75,26 +79,22 @@ class TestFusionSeqPoolCVMConcatOp(OpTest): class TestFusionSeqPoolCVMConcatOpCase1(TestFusionSeqPoolCVMConcatOp): - def set_conf(self): self.lods = [[[1]]] class TestFusionSeqPoolCVMConcatOpCase2(TestFusionSeqPoolCVMConcatOp): - def set_conf(self): self.lods = [[[1]], [[1]], [[1]]] class TestFusionSeqPoolCVMConcatOpCase3(TestFusionSeqPoolCVMConcatOp): - def set_conf(self): self.lods = [[[1, 3, 4, 6]]] self.w = 10 class TestFusionSeqPoolCVMConcatOpCase4(TestFusionSeqPoolCVMConcatOp): - def set_conf(self): self.lods = [[[2, 13, 4]], [[1, 1, 1]], [[5, 3, 1]], [[9, 10, 3]]] self.w = 3 @@ -102,14 +102,11 @@ class TestFusionSeqPoolCVMConcatOpCase4(TestFusionSeqPoolCVMConcatOp): ## test avg pool and sqrt def create_test_avg_sqrt_class(parent): - class TestSeqPoolAvgCase(parent): - def set_pooltype(self): self.pooltype = "AVERAGE" class TestSeqPoolSqrtCase(parent): - def set_pooltype(self): self.pooltype = "SQRT" diff --git a/python/paddle/fluid/tests/unittests/test_fusion_squared_mat_sub_op.py b/python/paddle/fluid/tests/unittests/test_fusion_squared_mat_sub_op.py index a5dd503e43a..2a6098a8a6a 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_squared_mat_sub_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_squared_mat_sub_op.py @@ -18,7 +18,6 @@ from op_test import OpTest class TestFusionSquaredMatSubOp(OpTest): - def setUp(self): self.op_type = 'fusion_squared_mat_sub' self.m = 11 @@ -31,8 +30,8 @@ class TestFusionSquaredMatSubOp(OpTest): self.inputs = {'X': matx, 'Y': maty} self.outputs = { - 'Out': - (np.dot(matx, maty)**2 - np.dot(matx**2, maty**2)) * self.scalar + 'Out': (np.dot(matx, maty) ** 2 - np.dot(matx**2, maty**2)) + * self.scalar } self.attrs = { 'scalar': self.scalar, @@ -46,7 +45,6 @@ class TestFusionSquaredMatSubOp(OpTest): class TestFusionSquaredMatSubOpCase1(TestFusionSquaredMatSubOp): - def set_conf(self): self.scalar = -0.3 diff --git a/python/paddle/fluid/tests/unittests/test_fusion_transpose_flatten_concat_op.py b/python/paddle/fluid/tests/unittests/test_fusion_transpose_flatten_concat_op.py index f60357fdfa9..768bbec552d 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_transpose_flatten_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_transpose_flatten_concat_op.py @@ -18,10 +18,10 @@ from op_test import OpTest import paddle.fluid.core as core -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFusionTransposeFlattenConcationOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "fusion_transpose_flatten_concat" @@ -34,8 +34,10 @@ class TestFusionTransposeFlattenConcationOp(OpTest): ins.append(("x%d" % i, a)) b = a.transpose(self.trans_axis) - flat_shape = (np.prod(b.shape[:self.flatten_axis]), - np.prod(b.shape[self.flatten_axis:])) + flat_shape = ( + np.prod(b.shape[: self.flatten_axis]), + np.prod(b.shape[self.flatten_axis :]), + ) c = b.reshape(flat_shape) flats.append(c) out = np.concatenate(flats, axis=self.concat_axis) @@ -44,7 +46,7 @@ class TestFusionTransposeFlattenConcationOp(OpTest): self.attrs = { 'trans_axis': list(self.trans_axis), 'flatten_axis': self.flatten_axis, - 'concat_axis': self.concat_axis + 'concat_axis': self.concat_axis, } self.outputs = {'Out': out} @@ -59,10 +61,10 @@ class TestFusionTransposeFlattenConcationOp(OpTest): self.concat_axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase1(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 4, 18, 17), (3, 8, 18, 7), (6, 12, 9, 5)] self.trans_axis = (0, 2, 3, 1) @@ -70,10 +72,10 @@ class TestCase1(TestFusionTransposeFlattenConcationOp): self.concat_axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase2(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 8, 20, 17), (3, 8, 19, 17), (3, 8, 40, 17)] self.trans_axis = (0, 2, 3, 1) @@ -81,10 +83,10 @@ class TestCase2(TestFusionTransposeFlattenConcationOp): self.concat_axis = 0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase3(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 8, 20, 17), (3, 8, 19, 17), (3, 8, 40, 17)] self.trans_axis = (0, 3, 2, 1) @@ -92,10 +94,10 @@ class TestCase3(TestFusionTransposeFlattenConcationOp): self.concat_axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase4(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 8, 9, 17), (8, 3, 9, 17), (4, 6, 9, 17)] self.trans_axis = (0, 2, 1, 3) @@ -103,10 +105,10 @@ class TestCase4(TestFusionTransposeFlattenConcationOp): self.concat_axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase5(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 8, 9, 17, 2), (3, 8, 2, 17, 9), (3, 17, 9, 8, 2)] self.trans_axis = (0, 2, 1, 4, 3) diff --git a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py index 17d5fb11ea9..b098dcf4e16 100644 --- a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py +++ b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py @@ -20,7 +20,6 @@ import unittest class GastNodeTransformer(gast.NodeTransformer): - def __init__(self, root): self.root = root @@ -122,7 +121,6 @@ def code_ast(source): class TestPythonCompatibility(unittest.TestCase): - def _check_compatibility(self, source, target): source_dump = code_gast_ast(source) target_dump = code_ast(target) diff --git a/python/paddle/fluid/tests/unittests/test_gather_nd_op.py b/python/paddle/fluid/tests/unittests/test_gather_nd_op.py index 5154d7feb77..176012e96a2 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_nd_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_nd_op.py @@ -39,7 +39,6 @@ class TestGatherNdOpWithEmptyIndex(OpTest): class TestGatherNdOpWithIndex1(OpTest): - def setUp(self): self.op_type = "gather_nd" self.python_api = paddle.gather_nd @@ -55,7 +54,7 @@ class TestGatherNdOpWithIndex1(OpTest): class TestGatherNdOpWithLowIndex(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.op_type = "gather_nd" @@ -65,7 +64,9 @@ class TestGatherNdOpWithLowIndex(OpTest): self.inputs = {'X': xnp, 'Index': index} - self.outputs = {'Out': xnp[tuple(index.T)]} #[[14, 25, 1], [76, 22, 3]] + self.outputs = { + 'Out': xnp[tuple(index.T)] + } # [[14, 25, 1], [76, 22, 3]] def test_check_output(self): self.check_output(check_eager=False) @@ -75,7 +76,7 @@ class TestGatherNdOpWithLowIndex(OpTest): class TestGatherNdOpIndex1(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.op_type = "gather_nd" @@ -95,7 +96,7 @@ class TestGatherNdOpIndex1(OpTest): class TestGatherNdOpWithSameIndexAsX(OpTest): - #Index has same rank as X's rank + # Index has same rank as X's rank def setUp(self): self.op_type = "gather_nd" @@ -104,7 +105,7 @@ class TestGatherNdOpWithSameIndexAsX(OpTest): index = np.array([[1, 1], [2, 1]]).astype("int64") self.inputs = {'X': xnp, 'Index': index} - self.outputs = {'Out': xnp[tuple(index.T)]} #[25, 22] + self.outputs = {'Out': xnp[tuple(index.T)]} # [25, 22] def test_check_output(self): self.check_output(check_eager=False) @@ -114,7 +115,7 @@ class TestGatherNdOpWithSameIndexAsX(OpTest): class TestGatherNdOpWithHighRankSame(OpTest): - #Both Index and X have high rank, and Rank(Index) = Rank(X) + # Both Index and X have high rank, and Rank(Index) = Rank(X) def setUp(self): self.op_type = "gather_nd" @@ -134,7 +135,7 @@ class TestGatherNdOpWithHighRankSame(OpTest): class TestGatherNdOpWithHighRankDiff(OpTest): - #Both Index and X have high rank, and Rank(Index) < Rank(X) + # Both Index and X have high rank, and Rank(Index) < Rank(X) def setUp(self): self.op_type = "gather_nd" @@ -154,13 +155,12 @@ class TestGatherNdOpWithHighRankDiff(OpTest): self.check_grad(['X'], 'Out', check_eager=False) -#Test Python API +# Test Python API class TestGatherNdOpAPI(unittest.TestCase): - def test_case1(self): - x1 = fluid.layers.data(name='x1', - shape=[30, 40, 50, 60], - dtype='float32') + x1 = fluid.layers.data( + name='x1', shape=[30, 40, 50, 60], dtype='float32' + ) index1 = fluid.layers.data(name='index1', shape=[2, 4], dtype='int32') output1 = fluid.layers.gather_nd(x1, index1) @@ -175,23 +175,20 @@ class TestGatherNdOpAPI(unittest.TestCase): output3 = fluid.layers.gather_nd(x3, index3, name="gather_nd_layer") -#Test Raise Index Error +# Test Raise Index Error class TestGatherNdOpRaise(unittest.TestCase): - def test_check_raise(self): - def check_raise_is_test(): try: - x = fluid.layers.data(name='x', - shape=[3, 4, 5], - dtype='float32') - index = fluid.layers.data(name='index', - shape=[2, 10], - dtype='int32') + x = fluid.layers.data( + name='x', shape=[3, 4, 5], dtype='float32' + ) + index = fluid.layers.data( + name='index', shape=[2, 10], dtype='int32' + ) output = fluid.layers.gather_nd(x, index) except Exception as e: - t = \ - "Input(Index).shape[-1] should be no greater than Input(X).rank" + t = "Input(Index).shape[-1] should be no greater than Input(X).rank" if t in str(e): raise IndexError @@ -199,17 +196,17 @@ class TestGatherNdOpRaise(unittest.TestCase): class TestGatherNdError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') index = paddle.fluid.data(shape=shape, dtype='bool', name='index') - index_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = paddle.fluid.data( + shape=shape, dtype='float32', name='index_float' + ) np_x = np.random.random(shape).astype('float32') np_index = np.array(np.random.randint(2, size=shape, dtype=bool)) @@ -230,7 +227,6 @@ class TestGatherNdError(unittest.TestCase): class TestGatherNdAPI2(unittest.TestCase): - def test_static(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[-1, 2], dtype='float64') @@ -240,11 +236,9 @@ class TestGatherNdAPI2(unittest.TestCase): exe = fluid.Executor(place) input = np.array([[1, 2], [3, 4], [5, 6]]) index_1 = np.array([[1]]) - result, = exe.run(feed={ - "data1": input, - "index": index_1 - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"data1": input, "index": index_1}, fetch_list=[out] + ) expected_output = np.array([[3, 4]]) np.testing.assert_allclose(result, expected_output, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_gather_op.py b/python/paddle/fluid/tests/unittests/test_gather_op.py index 9e2dab26081..c6301d3e472 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_op.py @@ -29,7 +29,6 @@ def gather_numpy(x, index, axis): class TestGatherOp(OpTest): - def setUp(self): self.op_type = "gather" self.python_api = paddle.gather @@ -37,7 +36,7 @@ class TestGatherOp(OpTest): xnp = np.random.random(self.x_shape).astype(self.x_type) self.inputs = { 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) + 'Index': np.array(self.index).astype(self.index_type), } self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} @@ -58,31 +57,28 @@ class TestGatherOp(OpTest): class TestCase1(TestGatherOp): - def config(self): """ For one dimension input """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float64" self.index = [1, 3, 5] self.index_type = "int32" class TestCase2(TestGatherOp): - def config(self): """ For int64_t index type """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float64" self.index = [1, 3, 5] self.index_type = "int64" class TestCase3(TestGatherOp): - def config(self): """ For other input type @@ -94,7 +90,6 @@ class TestCase3(TestGatherOp): class TestCase4(TestGatherOp): - def config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': False} @@ -104,7 +99,6 @@ class TestCase4(TestGatherOp): class TestCase5(TestGatherOp): - def config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': False} @@ -114,7 +108,6 @@ class TestCase5(TestGatherOp): class TestCase6(TestGatherOp): - def config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': True} @@ -124,7 +117,6 @@ class TestCase6(TestGatherOp): class TestGatherBF16Op(OpTest): - def setUp(self): self.op_type = "gather" self.python_api = paddle.gather @@ -136,7 +128,7 @@ class TestGatherBF16Op(OpTest): self.inputs = { 'X': convert_float_to_uint16(xnp), 'Index': index_np, - 'Axis': axis_np + 'Axis': axis_np, } out = gather_numpy(self.inputs['X'], index_np, axis_np[0]) self.outputs = {'Out': out} @@ -159,7 +151,6 @@ class TestGatherBF16Op(OpTest): class TestGatherOp1(OpTest): - def setUp(self): self.op_type = "gather" self.python_api = paddle.gather @@ -190,7 +181,6 @@ class TestGatherOp1(OpTest): class TestGatherOp2(TestGatherOp1): - def config(self): """ For multi-dimension input @@ -204,7 +194,6 @@ class TestGatherOp2(TestGatherOp1): class TestGatherOp3(TestGatherOp1): - def config(self): """ For multi-dimension input @@ -218,7 +207,6 @@ class TestGatherOp3(TestGatherOp1): class TestGatherOp4(TestGatherOp1): - def config(self): """ For multi-dimension input @@ -233,7 +221,6 @@ class TestGatherOp4(TestGatherOp1): class API_TestGather(unittest.TestCase): - def test_out1(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[-1, 2], dtype='float64') @@ -243,17 +230,16 @@ class API_TestGather(unittest.TestCase): exe = fluid.Executor(place) input = np.array([[1, 2], [3, 4], [5, 6]]) index_1 = np.array([1, 2]) - result, = exe.run(feed={ - "data1": input, - "index": index_1 - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"data1": input, "index": index_1}, fetch_list=[out] + ) expected_output = np.array([[3, 4], [5, 6]]) np.testing.assert_allclose(result, expected_output, rtol=1e-05) def test_out2(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data('x', shape=[-1, 2], dtype='float64') index = paddle.fluid.data('index', shape=[-1, 1], dtype='int32') axis = paddle.fluid.data('axis', shape=[1], dtype='int32') @@ -263,18 +249,15 @@ class API_TestGather(unittest.TestCase): x_np = np.array([[1, 2], [3, 4], [5, 6]]).astype('float64') index_np = np.array([1, 1]).astype('int32') axis_np = np.array([1]).astype('int32') - result, = exe.run(feed={ - "x": x_np, - "index": index_np, - 'axis': axis_np - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"x": x_np, "index": index_np, 'axis': axis_np}, + fetch_list=[out], + ) expected_output = gather_numpy(x_np, index_np, axis_np[0]) np.testing.assert_allclose(result, expected_output, rtol=1e-05) class API_TestDygraphGather(unittest.TestCase): - def test_out1(self): paddle.disable_static() input_1 = np.array([[1, 2], [3, 4], [5, 6]]) @@ -319,18 +302,20 @@ class API_TestDygraphGather(unittest.TestCase): def test_dygraph(): with fluid.dygraph.guard(): - gpu_out = paddle.gather(paddle.to_tensor(x), - paddle.to_tensor(index)) + gpu_out = paddle.gather( + paddle.to_tensor(x), paddle.to_tensor(index) + ) return gpu_out.numpy() @switch_to_static_graph def test_static_graph(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_t = paddle.static.data(name="x", dtype=x.dtype, shape=x.shape) - index_t = paddle.static.data(name="index", - dtype=index.dtype, - shape=index.shape) + index_t = paddle.static.data( + name="index", dtype=index.dtype, shape=index.shape + ) out_t = paddle.gather(x_t, index_t) feed = {x_t.name: x, index_t.name: index} fetch = [out_t] @@ -343,18 +328,18 @@ class API_TestDygraphGather(unittest.TestCase): class TestGathertError(unittest.TestCase): - def test_error1(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='int8', name='x') axis = paddle.fluid.data(shape=[1], dtype='float32', name='axis') index = paddle.fluid.data(shape=shape, dtype='int32', name='index') - index_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = paddle.fluid.data( + shape=shape, dtype='float32', name='index_float' + ) def test_x_type(): paddle.gather(x, index) @@ -382,9 +367,9 @@ class TestGathertError(unittest.TestCase): shape = [8, 9, 6] x = fluid.data(shape=shape, dtype='int8', name='x') index = fluid.data(shape=shape, dtype='int32', name='mask') - index_float = fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = fluid.data( + shape=shape, dtype='float32', name='index_float' + ) def test_x_type(): paddle.fluid.layers.gather(x, index) @@ -398,7 +383,6 @@ class TestGathertError(unittest.TestCase): class TestCheckOutType(unittest.TestCase): - def test_out_type(self): data = paddle.static.data(shape=[16, 10], dtype='int64', name='x') index = paddle.static.data(shape=[4], dtype='int64', name='index') diff --git a/python/paddle/fluid/tests/unittests/test_gather_tree_op.py b/python/paddle/fluid/tests/unittests/test_gather_tree_op.py index 9d0d193e496..18fb61c69f1 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_tree_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_tree_op.py @@ -21,17 +21,16 @@ from paddle.fluid.framework import program_guard, Program class TestGatherTreeOp(OpTest): - def setUp(self): self.op_type = "gather_tree" self.python_api = paddle.nn.functional.gather_tree max_length, batch_size, beam_size = 5, 2, 2 - ids = np.random.randint(0, - high=10, - size=(max_length, batch_size, beam_size)) - parents = np.random.randint(0, - high=beam_size, - size=(max_length, batch_size, beam_size)) + ids = np.random.randint( + 0, high=10, size=(max_length, batch_size, beam_size) + ) + parents = np.random.randint( + 0, high=beam_size, size=(max_length, batch_size, beam_size) + ) self.inputs = {"Ids": ids, "Parents": parents} self.outputs = {'Out': self.backtrace(ids, parents)} @@ -44,8 +43,9 @@ class TestGatherTreeOp(OpTest): (max_length, batch_size, beam_size) = ids.shape for batch in range(batch_size): for beam in range(beam_size): - out[max_length - 1, batch, beam] = ids[max_length - 1, batch, - beam] + out[max_length - 1, batch, beam] = ids[ + max_length - 1, batch, beam + ] parent = parents[max_length - 1, batch, beam] for step in range(max_length - 2, -1, -1): out[step, batch, beam] = ids[step, batch, parent] @@ -54,41 +54,46 @@ class TestGatherTreeOp(OpTest): class TestGatherTreeOpAPI(unittest.TestCase): - def test_case(self): paddle.enable_static() - ids = fluid.layers.data(name='ids', - shape=[5, 2, 2], - dtype='int64', - append_batch_size=False) - parents = fluid.layers.data(name='parents', - shape=[5, 2, 2], - dtype='int64', - append_batch_size=False) + ids = fluid.layers.data( + name='ids', shape=[5, 2, 2], dtype='int64', append_batch_size=False + ) + parents = fluid.layers.data( + name='parents', + shape=[5, 2, 2], + dtype='int64', + append_batch_size=False, + ) final_sequences = fluid.layers.gather_tree(ids, parents) paddle.disable_static() def test_case2(self): - ids = paddle.to_tensor([[[2, 2], [6, 1]], [[3, 9], [6, 1]], - [[0, 1], [9, 0]]]) - parents = paddle.to_tensor([[[0, 0], [1, 1]], [[1, 0], [1, 0]], - [[0, 0], [0, 1]]]) + ids = paddle.to_tensor( + [[[2, 2], [6, 1]], [[3, 9], [6, 1]], [[0, 1], [9, 0]]] + ) + parents = paddle.to_tensor( + [[[0, 0], [1, 1]], [[1, 0], [1, 0]], [[0, 0], [0, 1]]] + ) final_sequences = paddle.nn.functional.gather_tree(ids, parents) class TestGatherTreeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): - ids = fluid.layers.data(name='ids', - shape=[5, 2, 2], - dtype='int64', - append_batch_size=False) - parents = fluid.layers.data(name='parents', - shape=[5, 2, 2], - dtype='int64', - append_batch_size=False) + ids = fluid.layers.data( + name='ids', + shape=[5, 2, 2], + dtype='int64', + append_batch_size=False, + ) + parents = fluid.layers.data( + name='parents', + shape=[5, 2, 2], + dtype='int64', + append_batch_size=False, + ) def test_Variable_ids(): # the input type must be Variable @@ -106,38 +111,46 @@ class TestGatherTreeOpError(unittest.TestCase): def test_type_ids(): # dtype must be int32 or int64 - bad_ids = fluid.layers.data(name='bad_ids', - shape=[5, 2, 2], - dtype='float32', - append_batch_size=False) + bad_ids = fluid.layers.data( + name='bad_ids', + shape=[5, 2, 2], + dtype='float32', + append_batch_size=False, + ) fluid.layers.gather_tree(bad_ids, parents) self.assertRaises(TypeError, test_type_ids) def test_type_parents(): # dtype must be int32 or int64 - bad_parents = fluid.layers.data(name='bad_parents', - shape=[5, 2, 2], - dtype='float32', - append_batch_size=False) + bad_parents = fluid.layers.data( + name='bad_parents', + shape=[5, 2, 2], + dtype='float32', + append_batch_size=False, + ) fluid.layers.gather_tree(ids, bad_parents) self.assertRaises(TypeError, test_type_parents) def test_ids_ndim(): - bad_ids = fluid.layers.data(name='bad_test_ids', - shape=[5, 2], - dtype='int64', - append_batch_size=False) + bad_ids = fluid.layers.data( + name='bad_test_ids', + shape=[5, 2], + dtype='int64', + append_batch_size=False, + ) paddle.nn.functional.gather_tree(bad_ids, parents) self.assertRaises(ValueError, test_ids_ndim) def test_parents_ndim(): - bad_parents = fluid.layers.data(name='bad_test_parents', - shape=[5, 2], - dtype='int64', - append_batch_size=False) + bad_parents = fluid.layers.data( + name='bad_test_parents', + shape=[5, 2], + dtype='int64', + append_batch_size=False, + ) paddle.nn.functional.gather_tree(ids, bad_parents) self.assertRaises(ValueError, test_parents_ndim) diff --git a/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py b/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py index 16e34dc6789..c3ecf7303c1 100644 --- a/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py @@ -23,7 +23,6 @@ import paddle class TestGaussianRandomOp(OpTest): - def setUp(self): self.op_type = "gaussian_random" self.python_api = paddle.normal @@ -35,7 +34,7 @@ class TestGaussianRandomOp(OpTest): "mean": self.mean, "std": self.std, "seed": 10, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } paddle.seed(10) @@ -43,7 +42,7 @@ class TestGaussianRandomOp(OpTest): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def test_check_output(self): self.check_output_customized(self.verify_output) @@ -64,10 +63,10 @@ class TestGaussianRandomOp(OpTest): np.testing.assert_allclose(hist, hist2, rtol=0, atol=0.01) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestGaussianRandomBF16Op(OpTest): - def setUp(self): self.op_type = "gaussian_random" self.python_api = paddle.normal @@ -80,7 +79,7 @@ class TestGaussianRandomBF16Op(OpTest): "std": self.std, "seed": 10, "dtype": paddle.fluid.core.VarDesc.VarType.BF16, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } paddle.seed(10) @@ -88,11 +87,12 @@ class TestGaussianRandomBF16Op(OpTest): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def test_check_output(self): - self.check_output_with_place_customized(self.verify_output, - place=core.CUDAPlace(0)) + self.check_output_with_place_customized( + self.verify_output, place=core.CUDAPlace(0) + ) def test_eager(self): with _test_eager_guard(): @@ -112,7 +112,6 @@ class TestGaussianRandomBF16Op(OpTest): class TestMeanStdAreInt(TestGaussianRandomOp): - def set_attrs(self): self.mean = 1 self.std = 2 @@ -120,23 +119,22 @@ class TestMeanStdAreInt(TestGaussianRandomOp): # Situation 2: Attr(shape) is a list(with tensor) class TestGaussianRandomOp_ShapeTensorList(TestGaussianRandomOp): - def setUp(self): - '''Test gaussian_random op with specified value - ''' + '''Test gaussian_random op with specified value''' self.op_type = "gaussian_random" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.attrs = { 'shape': self.infer_shape, 'mean': self.mean, 'std': self.std, 'seed': self.seed, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.inputs = {"ShapeTensorList": shape_tensor_list} @@ -154,9 +152,9 @@ class TestGaussianRandomOp_ShapeTensorList(TestGaussianRandomOp): self.check_output_customized(self.verify_output) -class TestGaussianRandomOp2_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList - ): - +class TestGaussianRandomOp2_ShapeTensorList( + TestGaussianRandomOp_ShapeTensorList +): def init_data(self): self.shape = [123, 92] self.infer_shape = [-1, -1] @@ -166,9 +164,9 @@ class TestGaussianRandomOp2_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList self.seed = 10 -class TestGaussianRandomOp3_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList - ): - +class TestGaussianRandomOp3_ShapeTensorList( + TestGaussianRandomOp_ShapeTensorList +): def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -178,9 +176,9 @@ class TestGaussianRandomOp3_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList self.seed = 10 -class TestGaussianRandomOp4_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList - ): - +class TestGaussianRandomOp4_ShapeTensorList( + TestGaussianRandomOp_ShapeTensorList +): def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -192,10 +190,8 @@ class TestGaussianRandomOp4_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList # Situation 3: shape is a tensor class TestGaussianRandomOp1_ShapeTensor(TestGaussianRandomOp): - def setUp(self): - '''Test gaussian_random op with specified value - ''' + '''Test gaussian_random op with specified value''' self.op_type = "gaussian_random" self.init_data() self.use_mkldnn = False @@ -205,7 +201,7 @@ class TestGaussianRandomOp1_ShapeTensor(TestGaussianRandomOp): 'mean': self.mean, 'std': self.std, 'seed': self.seed, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': np.zeros((123, 92), dtype='float32')} @@ -219,54 +215,61 @@ class TestGaussianRandomOp1_ShapeTensor(TestGaussianRandomOp): # Test python API class TestGaussianRandomAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2000) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 500) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - - out_1 = fluid.layers.gaussian_random(shape=[2000, 500], - dtype="float32", - mean=0.0, - std=1.0, - seed=10) - - out_2 = fluid.layers.gaussian_random(shape=[2000, positive_2_int32], - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_3 = fluid.layers.gaussian_random(shape=[2000, positive_2_int64], - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_4 = fluid.layers.gaussian_random(shape=shape_tensor_int32, - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_5 = fluid.layers.gaussian_random(shape=shape_tensor_int64, - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_6 = fluid.layers.gaussian_random(shape=shape_tensor_int64, - dtype=np.float32, - mean=0., - std=1.0, - seed=10) + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + + out_1 = fluid.layers.gaussian_random( + shape=[2000, 500], dtype="float32", mean=0.0, std=1.0, seed=10 + ) + + out_2 = fluid.layers.gaussian_random( + shape=[2000, positive_2_int32], + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_3 = fluid.layers.gaussian_random( + shape=[2000, positive_2_int64], + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_4 = fluid.layers.gaussian_random( + shape=shape_tensor_int32, + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_5 = fluid.layers.gaussian_random( + shape=shape_tensor_int64, + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_6 = fluid.layers.gaussian_random( + shape=shape_tensor_int64, + dtype=np.float32, + mean=0.0, + std=1.0, + seed=10, + ) exe = fluid.Executor(place=fluid.CPUPlace()) res_1, res_2, res_3, res_4, res_5, res_6 = exe.run( @@ -275,20 +278,21 @@ class TestGaussianRandomAPI(unittest.TestCase): "shape_tensor_int32": np.array([2000, 500]).astype("int32"), "shape_tensor_int64": np.array([2000, 500]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6], + ) self.assertAlmostEqual(np.mean(res_1), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_1), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_1), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_2), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_2), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_2), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_3), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_3), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_3), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_4), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_5), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_5), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_5), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_5), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_5), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_6), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_6), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_6), 1.0, delta=0.1) def test_default_dtype(self): paddle.disable_static() @@ -316,7 +320,6 @@ class TestGaussianRandomAPI(unittest.TestCase): class TestStandardNormalDtype(unittest.TestCase): - def test_default_dtype(self): paddle.disable_static() @@ -343,7 +346,6 @@ class TestStandardNormalDtype(unittest.TestCase): class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -356,9 +358,9 @@ class TestRandomValue(unittest.TestCase): def _check_random_value(dtype, expect, expect_mean, expect_std): x = paddle.randn([32, 3, 1024, 1024], dtype=dtype) actual = x.numpy() - np.testing.assert_allclose(actual[2, 1, 512, 1000:1010], - expect, - rtol=1e-05) + np.testing.assert_allclose( + actual[2, 1, 512, 1000:1010], expect, rtol=1e-05 + ) self.assertTrue(np.mean(actual), expect_mean) self.assertTrue(np.std(actual), expect_std) @@ -367,22 +369,42 @@ class TestRandomValue(unittest.TestCase): paddle.set_device('gpu') paddle.seed(2021) expect = [ - -0.79037829, -0.54411126, -0.32266671, 0.35791815, 1.44169267, - -0.87785644, -1.23909874, -2.18194139, 0.49489656, 0.40703062 + -0.79037829, + -0.54411126, + -0.32266671, + 0.35791815, + 1.44169267, + -0.87785644, + -1.23909874, + -2.18194139, + 0.49489656, + 0.40703062, ] - expect_mean = -0.0000053026194133403266873214888799115129813799285329878330230713 + expect_mean = ( + -0.0000053026194133403266873214888799115129813799285329878330230713 + ) expect_std = 0.99999191058126390974081232343451119959354400634765625 - _check_random_value(core.VarDesc.VarType.FP64, expect, expect_mean, - expect_std) + _check_random_value( + core.VarDesc.VarType.FP64, expect, expect_mean, expect_std + ) expect = [ - -0.7988942, 1.8644791, 0.02782744, 1.3692524, 0.6419724, 0.12436751, - 0.12058455, -1.9984808, 1.5635862, 0.18506318 + -0.7988942, + 1.8644791, + 0.02782744, + 1.3692524, + 0.6419724, + 0.12436751, + 0.12058455, + -1.9984808, + 1.5635862, + 0.18506318, ] expect_mean = -0.00004762359094456769526004791259765625 expect_std = 0.999975681304931640625 - _check_random_value(core.VarDesc.VarType.FP32, expect, expect_mean, - expect_std) + _check_random_value( + core.VarDesc.VarType.FP32, expect, expect_mean, expect_std + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_gcd.py b/python/paddle/fluid/tests/unittests/test_gcd.py index d5ef16cea42..7a2d02dcb81 100644 --- a/python/paddle/fluid/tests/unittests/test_gcd.py +++ b/python/paddle/fluid/tests/unittests/test_gcd.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestGcdAPI(unittest.TestCase): - def setUp(self): self.x_np = 12 self.y_np = 20 @@ -37,32 +36,34 @@ class TestGcdAPI(unittest.TestCase): y = fluid.data(name='input2', dtype='int32', shape=self.y_shape) out = paddle.gcd(x, y) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(fluid.default_main_program(), - feed={ - 'input1': self.x_np, - 'input2': self.y_np - }, - fetch_list=[out]) - self.assertTrue((np.array(res[0]) == np.gcd(self.x_np, - self.y_np)).all()) + res = exe.run( + fluid.default_main_program(), + feed={'input1': self.x_np, 'input2': self.y_np}, + fetch_list=[out], + ) + self.assertTrue( + (np.array(res[0]) == np.gcd(self.x_np, self.y_np)).all() + ) def test_dygraph(self): paddle.disable_static() x = paddle.to_tensor(self.x_np) y = paddle.to_tensor(self.y_np) result = paddle.gcd(x, y) - np.testing.assert_allclose(np.gcd(self.x_np, self.y_np), - result.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np.gcd(self.x_np, self.y_np), result.numpy(), rtol=1e-05 + ) paddle.enable_static() class TestGcdAPI2(TestGcdAPI): - def setUp(self): self.x_np = np.arange(6).astype(np.int32) self.y_np = np.array([20]).astype(np.int32) @@ -71,7 +72,6 @@ class TestGcdAPI2(TestGcdAPI): class TestGcdAPI3(TestGcdAPI): - def setUp(self): self.x_np = 0 self.y_np = 20 @@ -80,7 +80,6 @@ class TestGcdAPI3(TestGcdAPI): class TestGcdAPI4(TestGcdAPI): - def setUp(self): self.x_np = 0 self.y_np = 0 @@ -89,7 +88,6 @@ class TestGcdAPI4(TestGcdAPI): class TestGcdAPI5(TestGcdAPI): - def setUp(self): self.x_np = 12 self.y_np = -20 diff --git a/python/paddle/fluid/tests/unittests/test_gelu_op.py b/python/paddle/fluid/tests/unittests/test_gelu_op.py index 6af627be73b..203e2517cd4 100644 --- a/python/paddle/fluid/tests/unittests/test_gelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_gelu_op.py @@ -24,15 +24,20 @@ from paddle.fluid.framework import _test_eager_guard def gelu(x, approximate): if approximate: - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * ( + 1.0 + + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))) + ) + ) else: y_ref = 0.5 * x * (1 + erf(x / np.sqrt(2))) return y_ref.astype(x.dtype) class TestGeluOp(unittest.TestCase): - def _test_case1_cpu(self, approximate): x = np.random.uniform(-1, 1, size=(11, 17)).astype(np.float32) y_ref = gelu(x, approximate) @@ -87,10 +92,9 @@ class TestGeluOp(unittest.TestCase): y_ref, x_g_ref = run_gelu_op(True) np.testing.assert_allclose(y_ref, y_fast_math, rtol=1e-05, atol=0.0005) - np.testing.assert_allclose(x_g_ref, - x_g_fast_math, - rtol=1e-05, - atol=0.0005) + np.testing.assert_allclose( + x_g_ref, x_g_fast_math, rtol=1e-05, atol=0.0005 + ) def test_fast_math_eager(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_generate_mask_labels_op.py b/python/paddle/fluid/tests/unittests/test_generate_mask_labels_op.py index 18a13197044..a2121ee3ef5 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_mask_labels_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_mask_labels_op.py @@ -16,6 +16,7 @@ import unittest import numpy as np import math from op_test import OpTest + ''' # Equivalent code rles = mask_util.frPyObjects([segm], im_h, im_w) @@ -34,15 +35,17 @@ def decode(cnts, m): def poly2mask(xy, k, h, w): - scale = 5. + scale = 5.0 x = [int(scale * p + 0.5) for p in xy[::2]] x = x + [x[0]] y = [int(scale * p + 0.5) for p in xy[1::2]] y = y + [y[0]] - m = sum([ - int(max(abs(x[j] - x[j + 1]), abs(y[j] - y[j + 1]))) + int(1) - for j in range(k) - ]) + m = sum( + [ + int(max(abs(x[j] - x[j + 1]), abs(y[j] - y[j + 1]))) + int(1) + for j in range(k) + ] + ) u, v = [], [] for j in range(k): @@ -58,20 +61,22 @@ def poly2mask(xy, k, h, w): ys, ye = ye, ys if dx >= dy: - if (dx == 0): assert ye - ys == 0 + if dx == 0: + assert ye - ys == 0 s = 0 if dx == 0 else float(ye - ys) / dx else: - if (dy == 0): assert xe - xs == 0 + if dy == 0: + assert xe - xs == 0 s = 0 if dy == 0 else float(xe - xs) / dy if dx >= dy: ts = [dx - d if flip else d for d in range(dx + 1)] u.extend([xs + t for t in ts]) - v.extend([int(ys + s * t + .5) for t in ts]) + v.extend([int(ys + s * t + 0.5) for t in ts]) else: ts = [dy - d if flip else d for d in range(dy + 1)] v.extend([t + ys for t in ts]) - u.extend([int(xs + s * t + .5) for t in ts]) + u.extend([int(xs + s * t + 0.5) for t in ts]) k = len(u) x = np.zeros((k), np.int_) @@ -80,11 +85,11 @@ def poly2mask(xy, k, h, w): for j in range(1, k): if u[j] != u[j - 1]: xd = float(u[j] if (u[j] < u[j - 1]) else (u[j] - 1)) - xd = (xd + .5) / scale - .5 - if (math.floor(xd) != xd or xd < 0 or xd > (w - 1)): + xd = (xd + 0.5) / scale - 0.5 + if math.floor(xd) != xd or xd < 0 or xd > (w - 1): continue yd = float(v[j] if v[j] < v[j - 1] else v[j - 1]) - yd = (yd + .5) / scale - .5 + yd = (yd + 0.5) / scale - 0.5 yd = math.ceil(0 if yd < 0 else (h if yd > h else yd)) x[m] = int(xd) y[m] = int(yd) @@ -93,21 +98,21 @@ def poly2mask(xy, k, h, w): a = [int(x[i] * h + y[i]) for i in range(k)] a.append(h * w) a.sort() - b = [0] + a[:len(a) - 1] + b = [0] + a[: len(a) - 1] a = [c - d for (c, d) in zip(a, b)] k += 1 b = [0 for i in range(k)] b[0] = a[0] m, j = 1, 1 - while (j < k): + while j < k: if a[j] > 0: b[m] = a[j] m += 1 j += 1 else: j += 1 - if (j < k): + if j < k: b[m - 1] += a[j] j += 1 mask = decode(b, m) @@ -134,19 +139,28 @@ def bbox_overlaps(boxes, query_boxes): K = query_boxes.shape[0] overlaps = np.zeros((N, K), dtype=boxes.dtype) for k in range(K): - box_area = (query_boxes[k, 2] - query_boxes[k, 0] + 1) *\ - (query_boxes[k, 3] - query_boxes[k, 1] + 1) + box_area = (query_boxes[k, 2] - query_boxes[k, 0] + 1) * ( + query_boxes[k, 3] - query_boxes[k, 1] + 1 + ) for n in range(N): - iw = min(boxes[n, 2], query_boxes[k, 2]) -\ - max(boxes[n, 0], query_boxes[k, 0]) + 1 + iw = ( + min(boxes[n, 2], query_boxes[k, 2]) + - max(boxes[n, 0], query_boxes[k, 0]) + + 1 + ) if iw > 0: - ih = min(boxes[n, 3], query_boxes[k, 3]) -\ - max(boxes[n, 1], query_boxes[k, 1]) + 1 + ih = ( + min(boxes[n, 3], query_boxes[k, 3]) + - max(boxes[n, 1], query_boxes[k, 1]) + + 1 + ) if ih > 0: ua = float( - (boxes[n, 2] - boxes[n, 0] + 1) *\ - (boxes[n, 3] - boxes[n, 1] + 1) +\ - box_area - iw * ih) + (boxes[n, 2] - boxes[n, 0] + 1) + * (boxes[n, 3] - boxes[n, 1] + 1) + + box_area + - iw * ih + ) overlaps[n, k] = iw * ih / ua return overlaps @@ -191,7 +205,8 @@ def expand_mask_targets(masks, mask_class_labels, resolution, num_classes): # Target values of -1 are "don't care" / ignore labels mask_targets = -np.ones( - (masks.shape[0], num_classes * resolution**2), dtype=np.int32) + (masks.shape[0], num_classes * resolution**2), dtype=np.int32 + ) for i in range(masks.shape[0]): cls = int(mask_class_labels[i]) start = resolution**2 * cls @@ -203,9 +218,18 @@ def expand_mask_targets(masks, mask_class_labels, resolution, num_classes): return mask_targets -def generate_mask_labels(num_classes, im_info, gt_classes, is_crowd, - label_int32, gt_polys, resolution, rois, roi_lod, - gt_lod): +def generate_mask_labels( + num_classes, + im_info, + gt_classes, + is_crowd, + label_int32, + gt_polys, + resolution, + rois, + roi_lod, + gt_lod, +): mask_rois = [] roi_has_mask_int32 = [] mask_int32 = [] @@ -215,9 +239,16 @@ def generate_mask_labels(num_classes, im_info, gt_classes, is_crowd, roi_e = roi_lod[i + 1] gt_s = gt_lod[i] gt_e = gt_lod[i + 1] - mask_blob = _sample_mask(num_classes, im_info[i], gt_classes[gt_s:gt_e], - is_crowd[gt_s:gt_e], label_int32[roi_s:roi_e], - gt_polys[i], resolution, rois[roi_s:roi_e]) + mask_blob = _sample_mask( + num_classes, + im_info[i], + gt_classes[gt_s:gt_e], + is_crowd[gt_s:gt_e], + label_int32[roi_s:roi_e], + gt_polys[i], + resolution, + rois[roi_s:roi_e], + ) new_lod.append(mask_blob['mask_rois'].shape[0]) mask_rois.append(mask_blob['mask_rois']) roi_has_mask_int32.append(mask_blob['roi_has_mask_int32']) @@ -226,14 +257,15 @@ def generate_mask_labels(num_classes, im_info, gt_classes, is_crowd, def _sample_mask( - num_classes, - im_info, - gt_classes, - is_crowd, - label_int32, - gt_polys, # [[[], []], []] - resolution, - rois): + num_classes, + im_info, + gt_classes, + is_crowd, + label_int32, + gt_polys, # [[[], []], []] + resolution, + rois, +): mask_blob = {} im_scale = im_info[2] sample_boxes = rois @@ -248,7 +280,8 @@ def _sample_mask( masks = np.zeros((fg_inds.shape[0], resolution**2), dtype=np.int32) rois_fg = sample_boxes[fg_inds] overlaps_bbfg_bbpolys = bbox_overlaps( - rois_fg.astype(np.float32), boxes_from_polys.astype(np.float32)) + rois_fg.astype(np.float32), boxes_from_polys.astype(np.float32) + ) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) for i in range(rois_fg.shape[0]): fg_polys_ind = fg_polys_inds[i] @@ -261,10 +294,11 @@ def _sample_mask( bg_inds = np.where(label_int32 == 0)[0] rois_fg = sample_boxes[bg_inds[0]].reshape((1, -1)) masks = -np.ones((1, resolution**2), dtype=np.int32) - mask_class_labels = np.zeros((1, )) + mask_class_labels = np.zeros((1,)) roi_has_mask = np.append(roi_has_mask, 0) - masks = expand_mask_targets(masks, mask_class_labels, resolution, - num_classes) + masks = expand_mask_targets( + masks, mask_class_labels, resolution, num_classes + ) rois_fg *= im_scale mask_blob['mask_rois'] = rois_fg mask_blob['roi_has_mask_int32'] = roi_has_mask @@ -280,7 +314,6 @@ def trans_lod(lod): class TestGenerateMaskLabels(OpTest): - def set_data(self): self.init_test_case() self.make_generate_proposal_labels_out() @@ -293,16 +326,16 @@ class TestGenerateMaskLabels(OpTest): 'IsCrowd': (self.is_crowd.astype(np.int32), self.gt_lod), 'LabelsInt32': (self.label_int32.astype(np.int32), self.rois_lod), 'GtSegms': (self.gt_polys.astype(np.float32), self.masks_lod), - 'Rois': (self.rois.astype(np.float32), self.rois_lod) + 'Rois': (self.rois.astype(np.float32), self.rois_lod), } self.attrs = { 'num_classes': self.num_classes, - 'resolution': self.resolution + 'resolution': self.resolution, } self.outputs = { 'MaskRois': (self.mask_rois, [self.new_lod]), 'RoiHasMaskInt32': (self.roi_has_mask_int32, [self.new_lod]), - 'MaskInt32': (self.mask_int32, [self.new_lod]) + 'MaskInt32': (self.mask_int32, [self.new_lod]), } def init_test_case(self): @@ -358,9 +391,9 @@ class TestGenerateMaskLabels(OpTest): lod1.append(poly_num) pts = [] for j in range(poly_num): - poly_size = np.random.randint(min_poly_size, - max_poly_size, - size=1)[0] + poly_size = np.random.randint( + min_poly_size, max_poly_size, size=1 + )[0] x = np.random.rand(poly_size, 1) * w y = np.random.rand(poly_size, 1) * h xy = np.concatenate((x, y), axis=1) @@ -392,18 +425,27 @@ class TestGenerateMaskLabels(OpTest): def init_test_output(self): roi_lod = trans_lod(self.rois_lod[0]) gt_lod = trans_lod(self.gt_lod[0]) - outs = generate_mask_labels(self.num_classes, self.im_info, - self.gt_classes, self.is_crowd, - self.label_int32, self.gt_polys_list, - self.resolution, self.rois, roi_lod, gt_lod) + outs = generate_mask_labels( + self.num_classes, + self.im_info, + self.gt_classes, + self.is_crowd, + self.label_int32, + self.gt_polys_list, + self.resolution, + self.rois, + roi_lod, + gt_lod, + ) self.mask_rois = outs[0] self.roi_has_mask_int32 = outs[1] self.mask_int32 = outs[2] self.new_lod = outs[3] self.mask_rois = np.vstack(self.mask_rois) - self.roi_has_mask_int32 = np.hstack(self.roi_has_mask_int32)[:, - np.newaxis] + self.roi_has_mask_int32 = np.hstack(self.roi_has_mask_int32)[ + :, np.newaxis + ] self.mask_int32 = np.vstack(self.mask_int32) def setUp(self): diff --git a/python/paddle/fluid/tests/unittests/test_generate_proposal_labels_op.py b/python/paddle/fluid/tests/unittests/test_generate_proposal_labels_op.py index 4b5a5ebc2ba..8027234ce5b 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_proposal_labels_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_proposal_labels_op.py @@ -17,22 +17,24 @@ import numpy as np from op_test import OpTest -def generate_proposal_labels_in_python(rpn_rois, - gt_classes, - is_crowd, - gt_boxes, - im_info, - batch_size_per_im, - fg_fraction, - fg_thresh, - bg_thresh_hi, - bg_thresh_lo, - bbox_reg_weights, - class_nums, - use_random, - is_cls_agnostic, - is_cascade_rcnn, - max_overlaps=None): +def generate_proposal_labels_in_python( + rpn_rois, + gt_classes, + is_crowd, + gt_boxes, + im_info, + batch_size_per_im, + fg_fraction, + fg_thresh, + bg_thresh_hi, + bg_thresh_lo, + bbox_reg_weights, + class_nums, + use_random, + is_cls_agnostic, + is_cascade_rcnn, + max_overlaps=None, +): rois = [] labels_int32 = [] bbox_targets = [] @@ -41,16 +43,29 @@ def generate_proposal_labels_in_python(rpn_rois, max_overlap_with_gt = [] lod = [] assert len(rpn_rois) == len( - im_info), 'batch size of rpn_rois and ground_truth is not matched' + im_info + ), 'batch size of rpn_rois and ground_truth is not matched' for im_i in range(len(im_info)): max_overlap = max_overlaps[im_i] if is_cascade_rcnn else None - frcn_blobs = _sample_rois(rpn_rois[im_i], gt_classes[im_i], - is_crowd[im_i], gt_boxes[im_i], im_info[im_i], - batch_size_per_im, fg_fraction, fg_thresh, - bg_thresh_hi, bg_thresh_lo, bbox_reg_weights, - class_nums, use_random, is_cls_agnostic, - is_cascade_rcnn, max_overlap) + frcn_blobs = _sample_rois( + rpn_rois[im_i], + gt_classes[im_i], + is_crowd[im_i], + gt_boxes[im_i], + im_info[im_i], + batch_size_per_im, + fg_fraction, + fg_thresh, + bg_thresh_hi, + bg_thresh_lo, + bbox_reg_weights, + class_nums, + use_random, + is_cls_agnostic, + is_cascade_rcnn, + max_overlap, + ) lod.append(frcn_blobs['rois'].shape[0]) rois.append(frcn_blobs['rois']) labels_int32.append(frcn_blobs['labels_int32']) @@ -59,7 +74,15 @@ def generate_proposal_labels_in_python(rpn_rois, bbox_outside_weights.append(frcn_blobs['bbox_outside_weights']) max_overlap_with_gt.append(frcn_blobs['max_overlap']) - return rois, labels_int32, bbox_targets, bbox_inside_weights, bbox_outside_weights, max_overlap_with_gt, lod + return ( + rois, + labels_int32, + bbox_targets, + bbox_inside_weights, + bbox_outside_weights, + max_overlap_with_gt, + lod, + ) def filter_roi(rois, max_overlap): @@ -71,16 +94,30 @@ def filter_roi(rois, max_overlap): return np.zeros((1, 4)).astype('float32') -def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, - batch_size_per_im, fg_fraction, fg_thresh, bg_thresh_hi, - bg_thresh_lo, bbox_reg_weights, class_nums, use_random, - is_cls_agnostic, is_cascade_rcnn, max_overlap): +def _sample_rois( + rpn_rois, + gt_classes, + is_crowd, + gt_boxes, + im_info, + batch_size_per_im, + fg_fraction, + fg_thresh, + bg_thresh_hi, + bg_thresh_lo, + bbox_reg_weights, + class_nums, + use_random, + is_cls_agnostic, + is_cascade_rcnn, + max_overlap, +): rois_per_image = int(batch_size_per_im) fg_rois_per_im = int(np.round(fg_fraction * rois_per_image)) # Roidb im_scale = im_info[2] - inv_im_scale = 1. / im_scale + inv_im_scale = 1.0 / im_scale rpn_rois = rpn_rois * inv_im_scale if is_cascade_rcnn: @@ -97,12 +134,14 @@ def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, # Boxes which with non-zero overlap with gt boxes overlapped_boxes_ind = np.where(overlaps_max > 0)[0] overlapped_boxes_gt_classes = gt_classes[ - overlaps_argmax[overlapped_boxes_ind]] + overlaps_argmax[overlapped_boxes_ind] + ] gt_overlaps[ - overlapped_boxes_ind, - overlapped_boxes_gt_classes] = overlaps_max[overlapped_boxes_ind] + overlapped_boxes_ind, overlapped_boxes_gt_classes + ] = overlaps_max[overlapped_boxes_ind] box_to_gt_ind_map[overlapped_boxes_ind] = overlaps_argmax[ - overlapped_boxes_ind] + overlapped_boxes_ind + ] crowd_ind = np.where(is_crowd)[0] gt_overlaps[crowd_ind] = -1.0 @@ -112,8 +151,9 @@ def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, if is_cascade_rcnn: # Cascade RCNN Decode Filter fg_inds = np.where(max_overlaps >= fg_thresh)[0] - bg_inds = np.where((max_overlaps < bg_thresh_hi) - & (max_overlaps >= bg_thresh_lo))[0] + bg_inds = np.where( + (max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo) + )[0] fg_rois_per_this_image = fg_inds.shape[0] bg_rois_per_this_image = bg_inds.shape[0] else: @@ -122,21 +162,23 @@ def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, fg_rois_per_this_image = np.minimum(fg_rois_per_im, fg_inds.shape[0]) # Sample foreground if there are too many if (fg_inds.shape[0] > fg_rois_per_this_image) and use_random: - fg_inds = np.random.choice(fg_inds, - size=fg_rois_per_this_image, - replace=False) + fg_inds = np.random.choice( + fg_inds, size=fg_rois_per_this_image, replace=False + ) fg_inds = fg_inds[:fg_rois_per_this_image] # Background - bg_inds = np.where((max_overlaps < bg_thresh_hi) - & (max_overlaps >= bg_thresh_lo))[0] + bg_inds = np.where( + (max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo) + )[0] bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image - bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, - bg_inds.shape[0]) + bg_rois_per_this_image = np.minimum( + bg_rois_per_this_image, bg_inds.shape[0] + ) # Sample background if there are too many if (bg_inds.shape[0] > bg_rois_per_this_image) and use_random: - bg_inds = np.random.choice(bg_inds, - size=bg_rois_per_this_image, - replace=False) + bg_inds = np.random.choice( + bg_inds, size=bg_rois_per_this_image, replace=False + ) bg_inds = bg_inds[:bg_rois_per_this_image] keep_inds = np.append(fg_inds, bg_inds) @@ -146,22 +188,27 @@ def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, sampled_max_overlap = max_overlaps[keep_inds] sampled_gts = gt_boxes[box_to_gt_ind_map[keep_inds]] sampled_gts[fg_rois_per_this_image:, :] = gt_boxes[0] - bbox_label_targets = _compute_targets(sampled_boxes, sampled_gts, - sampled_labels, bbox_reg_weights) + bbox_label_targets = _compute_targets( + sampled_boxes, sampled_gts, sampled_labels, bbox_reg_weights + ) bbox_targets, bbox_inside_weights = _expand_bbox_targets( - bbox_label_targets, class_nums, is_cls_agnostic) - bbox_outside_weights = np.array(bbox_inside_weights > 0, - dtype=bbox_inside_weights.dtype) + bbox_label_targets, class_nums, is_cls_agnostic + ) + bbox_outside_weights = np.array( + bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype + ) # Scale rois sampled_rois = sampled_boxes * im_scale # Faster RCNN blobs - frcn_blobs = dict(rois=sampled_rois, - labels_int32=sampled_labels, - bbox_targets=bbox_targets, - bbox_inside_weights=bbox_inside_weights, - bbox_outside_weights=bbox_outside_weights, - max_overlap=sampled_max_overlap) + frcn_blobs = dict( + rois=sampled_rois, + labels_int32=sampled_labels, + bbox_targets=bbox_targets, + bbox_inside_weights=bbox_inside_weights, + bbox_outside_weights=bbox_outside_weights, + max_overlap=sampled_max_overlap, + ) return frcn_blobs @@ -195,12 +242,13 @@ def _compute_targets(roi_boxes, gt_boxes, labels, bbox_reg_weights): targets = np.zeros(roi_boxes.shape) bbox_reg_weights = np.asarray(bbox_reg_weights) - targets = _box_to_delta(ex_boxes=roi_boxes, - gt_boxes=gt_boxes, - weights=bbox_reg_weights) + targets = _box_to_delta( + ex_boxes=roi_boxes, gt_boxes=gt_boxes, weights=bbox_reg_weights + ) - return np.hstack([labels[:, np.newaxis], targets]).astype(np.float32, - copy=False) + return np.hstack([labels[:, np.newaxis], targets]).astype( + np.float32, copy=False + ) def _box_to_delta(ex_boxes, gt_boxes, weights): @@ -230,8 +278,12 @@ def _expand_bbox_targets(bbox_targets_input, class_nums, is_cls_agnostic): # class_labels = [1 if ll > 0 else 0 for ll in class_labels] # class_labels = np.array(class_labels, dtype=np.int32) # class_nums = 2 - bbox_targets = np.zeros((class_labels.shape[0], - 4 * class_nums if not is_cls_agnostic else 4 * 2)) + bbox_targets = np.zeros( + ( + class_labels.shape[0], + 4 * class_nums if not is_cls_agnostic else 4 * 2, + ) + ) bbox_inside_weights = np.zeros(bbox_targets.shape) for ind in fg_inds: class_label = int(class_labels[ind]) if not is_cls_agnostic else 1 @@ -243,9 +295,8 @@ def _expand_bbox_targets(bbox_targets_input, class_nums, is_cls_agnostic): class TestGenerateProposalLabelsOp(OpTest): - def set_data(self): - #self.use_random = False + # self.use_random = False self.init_use_random() self.init_test_params() self.init_test_input() @@ -260,8 +311,10 @@ class TestGenerateProposalLabelsOp(OpTest): 'ImInfo': self.im_info, } if self.max_overlaps is not None: - self.inputs['MaxOverlap'] = (self.max_overlaps[0], - self.rpn_rois_lod) + self.inputs['MaxOverlap'] = ( + self.max_overlaps[0], + self.rpn_rois_lod, + ) self.attrs = { 'batch_size_per_im': self.batch_size_per_im, @@ -273,7 +326,7 @@ class TestGenerateProposalLabelsOp(OpTest): 'class_nums': self.class_nums, 'use_random': self.use_random, 'is_cls_agnostic': self.is_cls_agnostic, - 'is_cascade_rcnn': self.is_cascade_rcnn + 'is_cascade_rcnn': self.is_cascade_rcnn, } self.outputs = { 'Rois': (self.rois, [self.lod]), @@ -291,7 +344,9 @@ class TestGenerateProposalLabelsOp(OpTest): self.op_type = 'generate_proposal_labels' self.set_data() - def init_test_cascade(self, ): + def init_test_cascade( + self, + ): self.is_cascade_rcnn = False self.max_overlaps = None @@ -317,28 +372,46 @@ class TestGenerateProposalLabelsOp(OpTest): for i in range(len(images_shape)): self.im_info[i, 0] = images_shape[i][0] self.im_info[i, 1] = images_shape[i][1] - self.im_info[i, 2] = 0.8 #scale + self.im_info[i, 2] = 0.8 # scale self.rpn_rois, self.rpn_rois_lod = _generate_proposals( - images_shape, proposal_nums) + images_shape, proposal_nums + ) ground_truth, self.gts_lod = _generate_groundtruth( - images_shape, self.class_nums, gt_nums) + images_shape, self.class_nums, gt_nums + ) self.gt_classes = [gt['gt_classes'] for gt in ground_truth] self.gt_boxes = [gt['boxes'] for gt in ground_truth] self.is_crowd = [gt['is_crowd'] for gt in ground_truth] def init_test_output(self): - self.rois, self.labels_int32, self.bbox_targets, \ - self.bbox_inside_weights, self.bbox_outside_weights, \ - self.max_overlap_with_gt, \ - self.lod = generate_proposal_labels_in_python( - self.rpn_rois, self.gt_classes, self.is_crowd, self.gt_boxes, self.im_info, - self.batch_size_per_im, self.fg_fraction, - self.fg_thresh, self.bg_thresh_hi, self.bg_thresh_lo, - self.bbox_reg_weights, self.class_nums, self.use_random, - self.is_cls_agnostic, self.is_cascade_rcnn, self.max_overlaps - ) + ( + self.rois, + self.labels_int32, + self.bbox_targets, + self.bbox_inside_weights, + self.bbox_outside_weights, + self.max_overlap_with_gt, + self.lod, + ) = generate_proposal_labels_in_python( + self.rpn_rois, + self.gt_classes, + self.is_crowd, + self.gt_boxes, + self.im_info, + self.batch_size_per_im, + self.fg_fraction, + self.fg_thresh, + self.bg_thresh_hi, + self.bg_thresh_lo, + self.bbox_reg_weights, + self.class_nums, + self.use_random, + self.is_cls_agnostic, + self.is_cascade_rcnn, + self.max_overlaps, + ) self.rois = np.vstack(self.rois) self.labels_int32 = np.hstack(self.labels_int32) self.labels_int32 = self.labels_int32[:, np.newaxis] @@ -349,19 +422,17 @@ class TestGenerateProposalLabelsOp(OpTest): class TestCascade(TestGenerateProposalLabelsOp): - def init_test_cascade(self): self.is_cascade_rcnn = True roi_num = len(self.rpn_rois[0]) self.max_overlaps = [] max_overlap = np.random.rand(roi_num).astype('float32') # Make GT samples with overlap = 1 - max_overlap[max_overlap > 0.9] = 1. + max_overlap[max_overlap > 0.9] = 1.0 self.max_overlaps.append(max_overlap) class TestUseRandom(TestGenerateProposalLabelsOp): - def init_use_random(self): self.use_random = True self.is_cascade_rcnn = False @@ -384,7 +455,6 @@ class TestUseRandom(TestGenerateProposalLabelsOp): class TestClsAgnostic(TestCascade): - def init_test_params(self): self.batch_size_per_im = 512 self.fg_fraction = 0.25 @@ -397,7 +467,6 @@ class TestClsAgnostic(TestCascade): class TestOnlyGT(TestCascade): - def init_test_input(self): np.random.seed(0) gt_nums = 6 # Keep same with batch_size_per_im for unittest @@ -407,10 +476,11 @@ class TestOnlyGT(TestCascade): for i in range(len(images_shape)): self.im_info[i, 0] = images_shape[i][0] self.im_info[i, 1] = images_shape[i][1] - self.im_info[i, 2] = 0.8 #scale + self.im_info[i, 2] = 0.8 # scale ground_truth, self.gts_lod = _generate_groundtruth( - images_shape, self.class_nums, gt_nums) + images_shape, self.class_nums, gt_nums + ) self.gt_classes = [gt['gt_classes'] for gt in ground_truth] self.gt_boxes = [gt['boxes'] for gt in ground_truth] @@ -420,7 +490,6 @@ class TestOnlyGT(TestCascade): class TestOnlyGT2(TestCascade): - def init_test_cascade(self): self.is_cascade_rcnn = True roi_num = len(self.rpn_rois[0]) @@ -447,13 +516,15 @@ def _generate_groundtruth(images_shape, class_nums, gt_nums): num_gts = 0 for i, image_shape in enumerate(images_shape): # Avoid background - gt_classes = np.random.randint(low=1, high=class_nums, - size=gt_nums).astype(np.int32) + gt_classes = np.random.randint( + low=1, high=class_nums, size=gt_nums + ).astype(np.int32) gt_boxes = _generate_boxes(image_shape, gt_nums) is_crowd = np.zeros((gt_nums), dtype=np.int32) is_crowd[0] = 1 ground_truth.append( - dict(gt_classes=gt_classes, boxes=gt_boxes, is_crowd=is_crowd)) + dict(gt_classes=gt_classes, boxes=gt_boxes, is_crowd=is_crowd) + ) num_gts += len(gt_classes) gts_lod.append(num_gts) return ground_truth, [gts_lod] @@ -467,8 +538,12 @@ def _generate_boxes(image_size, box_nums): wh = xywh[:, [2, 3]] * (image_size - xy1) xy2 = xy1 + wh boxes = np.hstack([xy1, xy2]) - boxes[:, [0, 2]] = np.minimum(width - 1., np.maximum(0., boxes[:, [0, 2]])) - boxes[:, [1, 3]] = np.minimum(height - 1., np.maximum(0., boxes[:, [1, 3]])) + boxes[:, [0, 2]] = np.minimum( + width - 1.0, np.maximum(0.0, boxes[:, [0, 2]]) + ) + boxes[:, [1, 3]] = np.minimum( + height - 1.0, np.maximum(0.0, boxes[:, [1, 3]]) + ) return boxes.astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py b/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py index d5fbca5be9e..8a23973ede3 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py @@ -21,9 +21,18 @@ from test_anchor_generator_op import anchor_generator_in_python import copy -def generate_proposals_in_python(scores, bbox_deltas, im_info, anchors, - variances, pre_nms_topN, post_nms_topN, - nms_thresh, min_size, eta): +def generate_proposals_in_python( + scores, + bbox_deltas, + im_info, + anchors, + variances, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, +): all_anchors = anchors.reshape(-1, 4) rois = np.empty((0, 5), dtype=np.float32) roi_probs = np.empty((0, 1), dtype=np.float32) @@ -34,9 +43,17 @@ def generate_proposals_in_python(scores, bbox_deltas, im_info, anchors, num_images = scores.shape[0] for img_idx in range(num_images): img_i_boxes, img_i_probs = proposal_for_one_image( - im_info[img_idx, :], all_anchors, variances, - bbox_deltas[img_idx, :, :, :], scores[img_idx, :, :, :], - pre_nms_topN, post_nms_topN, nms_thresh, min_size, eta) + im_info[img_idx, :], + all_anchors, + variances, + bbox_deltas[img_idx, :, :, :], + scores[img_idx, :, :, :], + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + ) rois_num.append(img_i_probs.shape[0]) rpn_rois.append(img_i_boxes) rpn_roi_probs.append(img_i_probs) @@ -44,9 +61,18 @@ def generate_proposals_in_python(scores, bbox_deltas, im_info, anchors, return rpn_rois, rpn_roi_probs, rois_num -def proposal_for_one_image(im_info, all_anchors, variances, bbox_deltas, scores, - pre_nms_topN, post_nms_topN, nms_thresh, min_size, - eta): +def proposal_for_one_image( + im_info, + all_anchors, + variances, + bbox_deltas, + scores, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, +): # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # - bbox deltas will be (4 * A, H, W) format from conv output @@ -94,10 +120,9 @@ def proposal_for_one_image(im_info, all_anchors, variances, bbox_deltas, scores, # take post_nms_topN (e.g. 1000) # return the top proposals if nms_thresh > 0: - keep = nms(boxes=proposals, - scores=scores, - nms_threshold=nms_thresh, - eta=eta) + keep = nms( + boxes=proposals, scores=scores, nms_threshold=nms_thresh, eta=eta + ) if post_nms_topN > 0 and post_nms_topN < len(keep): keep = keep[:post_nms_topN] proposals = proposals[keep, :] @@ -111,10 +136,10 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): Decode proposals by anchors and bbox_deltas from RPN """ offset = 1 if pixel_offset else 0 - #proposals: xmin, ymin, xmax, ymax + # proposals: xmin, ymin, xmax, ymax proposals = np.zeros_like(bbox_deltas, dtype=np.float32) - #anchor_loc: width, height, center_x, center_y + # anchor_loc: width, height, center_x, center_y anchor_loc = np.zeros_like(bbox_deltas, dtype=np.float32) anchor_loc[:, 0] = all_anchors[:, 2] - all_anchors[:, 0] + offset @@ -122,34 +147,52 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): anchor_loc[:, 2] = all_anchors[:, 0] + 0.5 * anchor_loc[:, 0] anchor_loc[:, 3] = all_anchors[:, 1] + 0.5 * anchor_loc[:, 1] - #predicted bbox: bbox_center_x, bbox_center_y, bbox_width, bbox_height + # predicted bbox: bbox_center_x, bbox_center_y, bbox_width, bbox_height pred_bbox = np.zeros_like(bbox_deltas, dtype=np.float32) if variances is not None: for i in range(bbox_deltas.shape[0]): - pred_bbox[i, 0] = variances[i, 0] * bbox_deltas[i, 0] * anchor_loc[ - i, 0] + anchor_loc[i, 2] - pred_bbox[i, 1] = variances[i, 1] * bbox_deltas[i, 1] * anchor_loc[ - i, 1] + anchor_loc[i, 3] - pred_bbox[i, 2] = math.exp( - min(variances[i, 2] * bbox_deltas[i, 2], math.log( - 1000 / 16.0))) * anchor_loc[i, 0] - pred_bbox[i, 3] = math.exp( - min(variances[i, 3] * bbox_deltas[i, 3], math.log( - 1000 / 16.0))) * anchor_loc[i, 1] + pred_bbox[i, 0] = ( + variances[i, 0] * bbox_deltas[i, 0] * anchor_loc[i, 0] + + anchor_loc[i, 2] + ) + pred_bbox[i, 1] = ( + variances[i, 1] * bbox_deltas[i, 1] * anchor_loc[i, 1] + + anchor_loc[i, 3] + ) + pred_bbox[i, 2] = ( + math.exp( + min( + variances[i, 2] * bbox_deltas[i, 2], + math.log(1000 / 16.0), + ) + ) + * anchor_loc[i, 0] + ) + pred_bbox[i, 3] = ( + math.exp( + min( + variances[i, 3] * bbox_deltas[i, 3], + math.log(1000 / 16.0), + ) + ) + * anchor_loc[i, 1] + ) else: for i in range(bbox_deltas.shape[0]): - pred_bbox[i, - 0] = bbox_deltas[i, 0] * anchor_loc[i, 0] + anchor_loc[i, - 2] - pred_bbox[i, - 1] = bbox_deltas[i, 1] * anchor_loc[i, 1] + anchor_loc[i, - 3] - pred_bbox[i, 2] = math.exp( - min(bbox_deltas[i, 2], math.log(1000 / 16.0))) * anchor_loc[i, - 0] - pred_bbox[i, 3] = math.exp( - min(bbox_deltas[i, 3], math.log(1000 / 16.0))) * anchor_loc[i, - 1] + pred_bbox[i, 0] = ( + bbox_deltas[i, 0] * anchor_loc[i, 0] + anchor_loc[i, 2] + ) + pred_bbox[i, 1] = ( + bbox_deltas[i, 1] * anchor_loc[i, 1] + anchor_loc[i, 3] + ) + pred_bbox[i, 2] = ( + math.exp(min(bbox_deltas[i, 2], math.log(1000 / 16.0))) + * anchor_loc[i, 0] + ) + pred_bbox[i, 3] = ( + math.exp(min(bbox_deltas[i, 3], math.log(1000 / 16.0))) + * anchor_loc[i, 1] + ) proposals[:, 0] = pred_bbox[:, 0] - pred_bbox[:, 2] / 2 proposals[:, 1] = pred_bbox[:, 1] - pred_bbox[:, 3] / 2 proposals[:, 2] = pred_bbox[:, 0] + pred_bbox[:, 2] / 2 - offset @@ -161,33 +204,33 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): def clip_tiled_boxes(boxes, im_shape, pixel_offset=True): """Clip boxes to image boundaries. im_shape is [height, width] and boxes has shape (N, 4 * num_tiled_boxes).""" - assert boxes.shape[1] % 4 == 0, \ - 'boxes.shape[1] is {:d}, but must be divisible by 4.'.format( + assert ( + boxes.shape[1] % 4 == 0 + ), 'boxes.shape[1] is {:d}, but must be divisible by 4.'.format( boxes.shape[1] ) offset = 1 if pixel_offset else 0 # x1 >= 0 - boxes[:, - 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - offset), - 0) + boxes[:, 0::4] = np.maximum( + np.minimum(boxes[:, 0::4], im_shape[1] - offset), 0 + ) # y1 >= 0 - boxes[:, - 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - offset), - 0) + boxes[:, 1::4] = np.maximum( + np.minimum(boxes[:, 1::4], im_shape[0] - offset), 0 + ) # x2 < im_shape[1] - boxes[:, - 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - offset), - 0) + boxes[:, 2::4] = np.maximum( + np.minimum(boxes[:, 2::4], im_shape[1] - offset), 0 + ) # y2 < im_shape[0] - boxes[:, - 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - offset), - 0) + boxes[:, 3::4] = np.maximum( + np.minimum(boxes[:, 3::4], im_shape[0] - offset), 0 + ) return boxes def filter_boxes(boxes, min_size, im_info, pixel_offset=True): - """Only keep boxes with both sides >= min_size and center within the image. - """ + """Only keep boxes with both sides >= min_size and center within the image.""" # Scale min_size to match image scale im_scale = im_info[2] min_size = max(min_size, 1.0) @@ -197,11 +240,14 @@ def filter_boxes(boxes, min_size, im_info, pixel_offset=True): if pixel_offset: ws_orig_scale = (boxes[:, 2] - boxes[:, 0]) / im_scale + 1 hs_orig_scale = (boxes[:, 3] - boxes[:, 1]) / im_scale + 1 - x_ctr = boxes[:, 0] + ws / 2. - y_ctr = boxes[:, 1] + hs / 2. - keep = np.where((ws_orig_scale >= min_size) - & (hs_orig_scale >= min_size) & (x_ctr < im_info[1]) - & (y_ctr < im_info[0]))[0] + x_ctr = boxes[:, 0] + ws / 2.0 + y_ctr = boxes[:, 1] + hs / 2.0 + keep = np.where( + (ws_orig_scale >= min_size) + & (hs_orig_scale >= min_size) + & (x_ctr < im_info[1]) + & (y_ctr < im_info[0]) + )[0] else: keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep @@ -263,9 +309,9 @@ def nms(boxes, scores, nms_threshold, eta=1.0, pixel_offset=True): for k in range(len(selected_indices)): if keep: kept_idx = selected_indices[k] - overlap = iou(boxes[idx], - boxes[kept_idx], - pixel_offset=pixel_offset) + overlap = iou( + boxes[idx], boxes[kept_idx], pixel_offset=pixel_offset + ) keep = True if overlap <= adaptive_threshold else False else: break @@ -277,7 +323,6 @@ def nms(boxes, scores, nms_threshold, eta=1.0, pixel_offset=True): class TestGenerateProposalsOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -287,7 +332,7 @@ class TestGenerateProposalsOp(OpTest): 'BboxDeltas': self.bbox_deltas, 'ImInfo': self.im_info.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -295,7 +340,7 @@ class TestGenerateProposalsOp(OpTest): 'post_nms_topN': self.post_nms_topN, 'nms_thresh': self.nms_thresh, 'min_size': self.min_size, - 'eta': self.eta + 'eta': self.eta, } self.outputs = { @@ -315,7 +360,7 @@ class TestGenerateProposalsOp(OpTest): self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 3.0 - self.eta = 1. + self.eta = 1.0 def init_test_input(self): batch_size = 1 @@ -323,30 +368,47 @@ class TestGenerateProposalsOp(OpTest): layer_h = 16 layer_w = 16 input_feat = np.random.random( - (batch_size, input_channels, layer_h, layer_w)).astype('float32') + (batch_size, input_channels, layer_h, layer_w) + ).astype('float32') self.anchors, self.variances = anchor_generator_in_python( input_feat=input_feat, - anchor_sizes=[16., 32.], + anchor_sizes=[16.0, 32.0], aspect_ratios=[0.5, 1.0], variances=[1.0, 1.0, 1.0, 1.0], stride=[16.0, 16.0], - offset=0.5) - self.im_info = np.array([[64., 64., 8.]]) #im_height, im_width, scale + offset=0.5, + ) + self.im_info = np.array( + [[64.0, 64.0, 8.0]] + ) # im_height, im_width, scale num_anchors = self.anchors.shape[2] self.scores = np.random.random( - (batch_size, num_anchors, layer_h, layer_w)).astype('float32') + (batch_size, num_anchors, layer_h, layer_w) + ).astype('float32') self.bbox_deltas = np.random.random( - (batch_size, num_anchors * 4, layer_h, layer_w)).astype('float32') + (batch_size, num_anchors * 4, layer_h, layer_w) + ).astype('float32') def init_test_output(self): - self.rpn_rois, self.rpn_roi_probs, self.rois_num = generate_proposals_in_python( - self.scores, self.bbox_deltas, self.im_info, self.anchors, - self.variances, self.pre_nms_topN, self.post_nms_topN, - self.nms_thresh, self.min_size, self.eta) + ( + self.rpn_rois, + self.rpn_roi_probs, + self.rois_num, + ) = generate_proposals_in_python( + self.scores, + self.bbox_deltas, + self.im_info, + self.anchors, + self.variances, + self.pre_nms_topN, + self.post_nms_topN, + self.nms_thresh, + self.min_size, + self.eta, + ) class TestGenerateProposalsOutLodOp(TestGenerateProposalsOp): - def set_data(self): self.init_test_params() self.init_test_input() @@ -356,7 +418,7 @@ class TestGenerateProposalsOutLodOp(TestGenerateProposalsOp): 'BboxDeltas': self.bbox_deltas, 'ImInfo': self.im_info.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -365,24 +427,23 @@ class TestGenerateProposalsOutLodOp(TestGenerateProposalsOp): 'nms_thresh': self.nms_thresh, 'min_size': self.min_size, 'eta': self.eta, - 'return_rois_num': True + 'return_rois_num': True, } self.outputs = { 'RpnRois': (self.rpn_rois[0], [self.rois_num]), 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.rois_num]), - 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)) + 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)), } class TestGenerateProposalsOpNoBoxLeft(TestGenerateProposalsOp): - def init_test_params(self): self.pre_nms_topN = 12000 # train 12000, test 2000 self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 1000.0 - self.eta = 1. + self.eta = 1.0 if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_generate_proposals_v2_op.py b/python/paddle/fluid/tests/unittests/test_generate_proposals_v2_op.py index 06db10c5bd5..f0b5d561141 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_proposals_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_proposals_v2_op.py @@ -34,7 +34,11 @@ def python_generate_proposals_v2( pixel_offset=False, return_rois_num=True, ): - rpn_rois, rpn_roi_probs, rpn_rois_num = paddle.vision.ops.generate_proposals( + ( + rpn_rois, + rpn_roi_probs, + rpn_rois_num, + ) = paddle.vision.ops.generate_proposals( scores, bbox_deltas, img_size, @@ -46,13 +50,24 @@ def python_generate_proposals_v2( min_size=min_size, eta=eta, pixel_offset=pixel_offset, - return_rois_num=return_rois_num) + return_rois_num=return_rois_num, + ) return rpn_rois, rpn_roi_probs -def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, - variances, pre_nms_topN, post_nms_topN, - nms_thresh, min_size, eta, pixel_offset): +def generate_proposals_v2_in_python( + scores, + bbox_deltas, + im_shape, + anchors, + variances, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, +): all_anchors = anchors.reshape(-1, 4) rois = np.empty((0, 5), dtype=np.float32) roi_probs = np.empty((0, 1), dtype=np.float32) @@ -63,10 +78,18 @@ def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, num_images = scores.shape[0] for img_idx in range(num_images): img_i_boxes, img_i_probs = proposal_for_one_image( - im_shape[img_idx, :], all_anchors, variances, - bbox_deltas[img_idx, :, :, :], scores[img_idx, :, :, :], - pre_nms_topN, post_nms_topN, nms_thresh, min_size, eta, - pixel_offset) + im_shape[img_idx, :], + all_anchors, + variances, + bbox_deltas[img_idx, :, :, :], + scores[img_idx, :, :, :], + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, + ) rois_num.append(img_i_probs.shape[0]) rpn_rois.append(img_i_boxes) rpn_roi_probs.append(img_i_probs) @@ -74,9 +97,19 @@ def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, return rpn_rois, rpn_roi_probs, rois_num -def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, - scores, pre_nms_topN, post_nms_topN, nms_thresh, - min_size, eta, pixel_offset): +def proposal_for_one_image( + im_shape, + all_anchors, + variances, + bbox_deltas, + scores, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, +): # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # - bbox deltas will be (4 * A, H, W) format from conv output @@ -124,11 +157,13 @@ def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, # take post_nms_topN (e.g. 1000) # return the top proposals if nms_thresh > 0: - keep = nms(boxes=proposals, - scores=scores, - nms_threshold=nms_thresh, - eta=eta, - pixel_offset=pixel_offset) + keep = nms( + boxes=proposals, + scores=scores, + nms_threshold=nms_thresh, + eta=eta, + pixel_offset=pixel_offset, + ) if post_nms_topN > 0 and post_nms_topN < len(keep): keep = keep[:post_nms_topN] proposals = proposals[keep, :] @@ -138,25 +173,27 @@ def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, def filter_boxes(boxes, min_size, im_shape, pixel_offset=True): - """Only keep boxes with both sides >= min_size and center within the image. - """ + """Only keep boxes with both sides >= min_size and center within the image.""" # Scale min_size to match image scale min_size = max(min_size, 1.0) offset = 1 if pixel_offset else 0 ws = boxes[:, 2] - boxes[:, 0] + offset hs = boxes[:, 3] - boxes[:, 1] + offset if pixel_offset: - x_ctr = boxes[:, 0] + ws / 2. - y_ctr = boxes[:, 1] + hs / 2. - keep = np.where((ws >= min_size) & (hs >= min_size) - & (x_ctr < im_shape[1]) & (y_ctr < im_shape[0]))[0] + x_ctr = boxes[:, 0] + ws / 2.0 + y_ctr = boxes[:, 1] + hs / 2.0 + keep = np.where( + (ws >= min_size) + & (hs >= min_size) + & (x_ctr < im_shape[1]) + & (y_ctr < im_shape[0]) + )[0] else: keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep class TestGenerateProposalsV2Op(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -166,7 +203,7 @@ class TestGenerateProposalsV2Op(OpTest): 'BboxDeltas': self.bbox_deltas, 'ImShape': self.im_shape.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -196,7 +233,7 @@ class TestGenerateProposalsV2Op(OpTest): self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 3.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = True def init_test_input(self): @@ -205,26 +242,43 @@ class TestGenerateProposalsV2Op(OpTest): layer_h = 16 layer_w = 16 input_feat = np.random.random( - (batch_size, input_channels, layer_h, layer_w)).astype('float32') + (batch_size, input_channels, layer_h, layer_w) + ).astype('float32') self.anchors, self.variances = anchor_generator_in_python( input_feat=input_feat, - anchor_sizes=[16., 32.], + anchor_sizes=[16.0, 32.0], aspect_ratios=[0.5, 1.0], variances=[1.0, 1.0, 1.0, 1.0], stride=[16.0, 16.0], - offset=0.5) + offset=0.5, + ) self.im_shape = np.array([[64, 64]]).astype('float32') num_anchors = self.anchors.shape[2] self.scores = np.random.random( - (batch_size, num_anchors, layer_h, layer_w)).astype('float32') + (batch_size, num_anchors, layer_h, layer_w) + ).astype('float32') self.bbox_deltas = np.random.random( - (batch_size, num_anchors * 4, layer_h, layer_w)).astype('float32') + (batch_size, num_anchors * 4, layer_h, layer_w) + ).astype('float32') def init_test_output(self): - self.rpn_rois, self.rpn_roi_probs, self.rois_num = generate_proposals_v2_in_python( - self.scores, self.bbox_deltas, self.im_shape, self.anchors, - self.variances, self.pre_nms_topN, self.post_nms_topN, - self.nms_thresh, self.min_size, self.eta, self.pixel_offset) + ( + self.rpn_rois, + self.rpn_roi_probs, + self.rois_num, + ) = generate_proposals_v2_in_python( + self.scores, + self.bbox_deltas, + self.im_shape, + self.anchors, + self.variances, + self.pre_nms_topN, + self.post_nms_topN, + self.nms_thresh, + self.min_size, + self.eta, + self.pixel_offset, + ) # class TestGenerateProposalsV2OpNoBoxLeft(TestGenerateProposalsV2Op): diff --git a/python/paddle/fluid/tests/unittests/test_generator.py b/python/paddle/fluid/tests/unittests/test_generator.py index 445c7b050d0..5adef46fc5d 100644 --- a/python/paddle/fluid/tests/unittests/test_generator.py +++ b/python/paddle/fluid/tests/unittests/test_generator.py @@ -33,9 +33,9 @@ class TestGenerator(unittest.TestCase): def test_basic_generator_error(self): if paddle.fluid.core.is_compiled_with_cuda(): - self.assertRaises(ValueError, - generator.Generator, - place=paddle.CUDAPlace(0)) + self.assertRaises( + ValueError, generator.Generator, place=paddle.CUDAPlace(0) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_generator_dataloader.py b/python/paddle/fluid/tests/unittests/test_generator_dataloader.py index dbc8169bd11..67b2c37accb 100644 --- a/python/paddle/fluid/tests/unittests/test_generator_dataloader.py +++ b/python/paddle/fluid/tests/unittests/test_generator_dataloader.py @@ -41,15 +41,16 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): with fluid.unique_name.guard(): with fluid.program_guard(main_prog, startup_prog): - image = fluid.layers.data(name='image', - shape=[784], - dtype='float32') + image = fluid.layers.data( + name='image', shape=[784], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') py_reader = fluid.io.DataLoader.from_generator( feed_list=[image, label], capacity=4, iterable=not use_legacy_py_reader, - use_double_buffer=use_double_buffer) + use_double_buffer=use_double_buffer, + ) hidden = image for hidden_size in [10, 20, 30]: hidden = fluid.layers.fc( @@ -57,13 +58,16 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): size=hidden_size, act='tanh', bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) + initializer=fluid.initializer.Constant(value=1.0) + ), + ) - predict_label = fluid.layers.fc(hidden, - size=CLASS_NUM, - act='softmax') + predict_label = fluid.layers.fc( + hidden, size=CLASS_NUM, act='softmax' + ) loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -71,28 +75,35 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): class TestBase(unittest.TestCase): - - def run_main(self, use_legacy_py_reader, with_data_parallel, places, - use_double_buffer): + def run_main( + self, + use_legacy_py_reader, + with_data_parallel, + places, + use_double_buffer, + ): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, py_reader, loss = simple_fc_net( - places, use_legacy_py_reader, use_double_buffer) + places, use_legacy_py_reader, use_double_buffer + ) reader = paddle.batch(random_reader, batch_size=BATCH_SIZE) ps = places if use_double_buffer else fluid.cpu_places(len(places)) py_reader.set_sample_list_generator( - reader, places=ps if py_reader.iterable else None) + reader, places=ps if py_reader.iterable else None + ) exe = fluid.Executor(place=places[0]) exe.run(startup_prog) prog = fluid.CompiledProgram(main_prog) if with_data_parallel: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step = 0 step_list = [] @@ -104,9 +115,11 @@ class TestBase(unittest.TestCase): py_reader.start() while True: try: - L, = exe.run(program=prog, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 except fluid.core.EOFException: @@ -118,7 +131,8 @@ class TestBase(unittest.TestCase): step = 0 for d in py_reader(): assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -126,10 +140,12 @@ class TestBase(unittest.TestCase): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(ps[i]) assert label._place()._equals(ps[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -137,7 +153,7 @@ class TestBase(unittest.TestCase): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } return ret @@ -167,16 +183,17 @@ class TestBase(unittest.TestCase): use_legacy_py_reader=use_legacy_py_reader, with_data_parallel=with_data_parallel, places=p, - use_double_buffer=use_double_buffer) + use_double_buffer=use_double_buffer, + ) results.append(ret) if not use_double_buffer: diff = np.max( - np.abs(results[0]['loss'] - results[1]['loss'])) + np.abs(results[0]['loss'] - results[1]['loss']) + ) self.assertLess(diff, 1e-3) class TestDataLoaderBaseAbstract(unittest.TestCase): - def test_main(self): loader = DataLoaderBase() try: diff --git a/python/paddle/fluid/tests/unittests/test_get_all_op_or_kernel_names.py b/python/paddle/fluid/tests/unittests/test_get_all_op_or_kernel_names.py index 999912494ad..47af7355cb7 100644 --- a/python/paddle/fluid/tests/unittests/test_get_all_op_or_kernel_names.py +++ b/python/paddle/fluid/tests/unittests/test_get_all_op_or_kernel_names.py @@ -38,15 +38,15 @@ class TestGetAllRegisteredOpKernels(unittest.TestCase): class TestGetAllOpNames(unittest.TestCase): - def test_get_all_op_names(self): all_op_names = core.get_all_op_names() all_op_with_phi_kernels = core.get_all_op_names("phi") all_op_with_fluid_kernels = core.get_all_op_names("fluid") self.assertTrue( - len(all_op_names) > len( - set(all_op_with_phi_kernels) | set(all_op_with_fluid_kernels))) + len(all_op_names) + > len(set(all_op_with_phi_kernels) | set(all_op_with_fluid_kernels)) + ) self.assertTrue("scale" in all_op_with_phi_kernels) self.assertTrue("scale" in all_op_with_phi_kernels) diff --git a/python/paddle/fluid/tests/unittests/test_get_device_properties.py b/python/paddle/fluid/tests/unittests/test_get_device_properties.py index e29b565ff45..3fea41dbadd 100644 --- a/python/paddle/fluid/tests/unittests/test_get_device_properties.py +++ b/python/paddle/fluid/tests/unittests/test_get_device_properties.py @@ -18,7 +18,6 @@ from paddle.device.cuda import device_count, get_device_properties class TestGetDeviceProperties(unittest.TestCase): - def test_get_device_properties_default(self): if core.is_compiled_with_cuda(): props = get_device_properties() @@ -44,7 +43,6 @@ class TestGetDeviceProperties(unittest.TestCase): class TestGetDevicePropertiesError(unittest.TestCase): - def test_error_api(self): if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_get_inputs_outputs_in_block.py b/python/paddle/fluid/tests/unittests/test_get_inputs_outputs_in_block.py index d19ccea8ea0..b4196794032 100644 --- a/python/paddle/fluid/tests/unittests/test_get_inputs_outputs_in_block.py +++ b/python/paddle/fluid/tests/unittests/test_get_inputs_outputs_in_block.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestGetInputsOutputsInBlock(unittest.TestCase): - def test_ordered(self): # Program variable names may be different when test order is different # This helper makes the test ordered. @@ -49,7 +48,8 @@ class TestGetInputsOutputsInBlock(unittest.TestCase): sub_block = main_program.block(1) inner_inputs, inner_outputs = utils.get_inputs_outputs_in_block( - sub_block) + sub_block + ) # 'assign_0.tmp_0', 'assign_1.tmp_0' are name of i and ten in program self.assertTrue(inner_inputs == {'assign_0.tmp_0', 'assign_1.tmp_0'}) # 'tmp_0', 'assign_0.tmp_0' are name of i < ten and i in program @@ -66,7 +66,8 @@ class TestGetInputsOutputsInBlock(unittest.TestCase): sub_block = main_program.block(1) inner_inputs, inner_outputs = utils.get_inputs_outputs_in_block( - sub_block) + sub_block + ) #'fill_constant_1.tmp_0', 'tmp_3' are names of a, c self.assertTrue(inner_inputs == {'fill_constant_1.tmp_0', 'tmp_3'}) #'_generated_var_1', is name of a + c diff --git a/python/paddle/fluid/tests/unittests/test_get_places_op.py b/python/paddle/fluid/tests/unittests/test_get_places_op.py index 93b03d029cc..d51e36f2514 100644 --- a/python/paddle/fluid/tests/unittests/test_get_places_op.py +++ b/python/paddle/fluid/tests/unittests/test_get_places_op.py @@ -20,7 +20,6 @@ import unittest class TestGetPlaces(unittest.TestCase): - @prog_scope() def check_get_cpu_places(self): places = get_places() diff --git a/python/paddle/fluid/tests/unittests/test_get_set_flags.py b/python/paddle/fluid/tests/unittests/test_get_set_flags.py index 80300eb7dfc..dad58ae08ba 100644 --- a/python/paddle/fluid/tests/unittests/test_get_set_flags.py +++ b/python/paddle/fluid/tests/unittests/test_get_set_flags.py @@ -17,11 +17,10 @@ import unittest as unittest class TestGetAndSetFlags(unittest.TestCase): - def test_api(self): flags = { 'FLAGS_eager_delete_tensor_gb': 1.0, - 'FLAGS_check_nan_inf': True + 'FLAGS_check_nan_inf': True, } fluid.set_flags(flags) @@ -38,7 +37,6 @@ class TestGetAndSetFlags(unittest.TestCase): class TestGetAndSetFlagsErrors(unittest.TestCase): - def test_errors(self): flags_list = ['FLAGS_eager_delete_tensor_gb', 'FLAGS_check_nan_inf'] flag = 1 diff --git a/python/paddle/fluid/tests/unittests/test_get_tensor_from_selected_rows_op.py b/python/paddle/fluid/tests/unittests/test_get_tensor_from_selected_rows_op.py index 425cca109f0..17bf702b886 100644 --- a/python/paddle/fluid/tests/unittests/test_get_tensor_from_selected_rows_op.py +++ b/python/paddle/fluid/tests/unittests/test_get_tensor_from_selected_rows_op.py @@ -40,7 +40,6 @@ class TestGetTensorFromSelectedRowsError(unittest.TestCase): class TestGetTensorFromSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_global_var_getter_setter.py b/python/paddle/fluid/tests/unittests/test_global_var_getter_setter.py index 90fe9988ac2..3394a08de8b 100644 --- a/python/paddle/fluid/tests/unittests/test_global_var_getter_setter.py +++ b/python/paddle/fluid/tests/unittests/test_global_var_getter_setter.py @@ -17,7 +17,6 @@ import unittest as unittest class VarInfo(object): - def __init__(self, var_name, var_type, writable): self.name = var_name self.type = var_type @@ -25,7 +24,6 @@ class VarInfo(object): class TestGlobalVarGetterSetter(unittest.TestCase): - def test_main(self): var_infos = [ VarInfo("FLAGS_free_idle_chunk", bool, False), diff --git a/python/paddle/fluid/tests/unittests/test_glu.py b/python/paddle/fluid/tests/unittests/test_glu.py index c8f0098456c..25f1975db0c 100644 --- a/python/paddle/fluid/tests/unittests/test_glu.py +++ b/python/paddle/fluid/tests/unittests/test_glu.py @@ -32,7 +32,6 @@ def glu(x, dim=-1): class TestGLUCase(unittest.TestCase): - def setUp(self): self.x = np.random.randn(5, 20) self.dim = -1 @@ -53,7 +52,6 @@ class TestGLUCase(unittest.TestCase): class TestGLUV2(unittest.TestCase): - def setUp(self): self.x = np.random.randn(5, 20) self.dim = -1 diff --git a/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py b/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py index fc130426aef..ac786dea529 100644 --- a/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py +++ b/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py @@ -21,7 +21,6 @@ from paddle.fluid import core class TestGPUPackagePaddle(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -34,8 +33,9 @@ class TestGPUPackagePaddle(unittest.TestCase): os.environ['HIP_VISIBLE_DEVICES'] = '' else: os.environ['CUDA_VISIBLE_DEVICES'] = '' - test_file = os.path.join(self.temp_dir.name, - 'test_no_gpu_run_rand.py') + test_file = os.path.join( + self.temp_dir.name, 'test_no_gpu_run_rand.py' + ) with open(test_file, 'w') as wb: cmd_test = """ import paddle @@ -48,10 +48,12 @@ assert x.place.is_gpu_place() is False, "There is no CUDA device, but Tensor's p _python = sys.executable ps_cmd = '{} {}'.format(_python, test_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=os.environ) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=os.environ, + ) stdout, stderr = ps_proc.communicate() assert 'CPU device will be used by default' in str( diff --git a/python/paddle/fluid/tests/unittests/test_grad_clip_minimize.py b/python/paddle/fluid/tests/unittests/test_grad_clip_minimize.py index fc8c1c946bb..ad1fc3ad183 100644 --- a/python/paddle/fluid/tests/unittests/test_grad_clip_minimize.py +++ b/python/paddle/fluid/tests/unittests/test_grad_clip_minimize.py @@ -19,11 +19,14 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.base import to_variable -from paddle.fluid.clip import GradientClipByValue, GradientClipByNorm, GradientClipByGlobalNorm +from paddle.fluid.clip import ( + GradientClipByValue, + GradientClipByNorm, + GradientClipByGlobalNorm, +) class TestGradClipByGlobalNorm(unittest.TestCase): - def init_value(self): self.max_global_norm = 5.0 self.init_scale = 1.0 @@ -35,10 +38,15 @@ class TestGradClipByGlobalNorm(unittest.TestCase): self.para_and_grad = [] for i in range(10): self.para_and_grad.append( - (np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'), - np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'))) + ( + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + ) + ) def get_numpy_global_norm_result(self): gloabl_norm = 0.0 @@ -98,7 +106,6 @@ class TestGradClipByGlobalNorm(unittest.TestCase): class TestGradClipByNorm(unittest.TestCase): - def init_value(self): self.max_norm = 5.0 self.init_scale = 1.0 @@ -110,10 +117,15 @@ class TestGradClipByNorm(unittest.TestCase): self.para_and_grad = [] for i in range(10): self.para_and_grad.append( - (np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'), - np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'))) + ( + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + ) + ) def get_numpy_norm_result(self): @@ -169,7 +181,6 @@ class TestGradClipByNorm(unittest.TestCase): class TestGradClipByValue(unittest.TestCase): - def init_value(self): self.max_value = 0.8 self.min_value = -0.1 @@ -182,10 +193,15 @@ class TestGradClipByValue(unittest.TestCase): self.para_and_grad = [] for i in range(10): self.para_and_grad.append( - (np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'), - np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'))) + ( + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + ) + ) def get_numpy_clip_result(self): @@ -197,8 +213,9 @@ class TestGradClipByValue(unittest.TestCase): def get_dygrap_clip_result(self): with fluid.dygraph.guard(): - value_clip = GradientClipByValue(max=self.max_value, - min=self.min_value) + value_clip = GradientClipByValue( + max=self.max_value, min=self.min_value + ) p_g_var = [] for p, g in self.para_and_grad: new_p = to_variable(p) diff --git a/python/paddle/fluid/tests/unittests/test_gradient_clip.py b/python/paddle/fluid/tests/unittests/test_gradient_clip.py index 825e84f9f6d..1687d7e5983 100644 --- a/python/paddle/fluid/tests/unittests/test_gradient_clip.py +++ b/python/paddle/fluid/tests/unittests/test_gradient_clip.py @@ -23,21 +23,17 @@ from paddle.fluid.clip import _allow_pure_fp16_global_norm_clip paddle.enable_static() -def bow_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): +def bow_net( + data, label, dict_dim, emb_dim=128, hid_dim=128, hid_dim2=96, class_dim=2 +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - emb = fluid.layers.embedding(input=data, - is_sparse=True, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=True, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -50,7 +46,6 @@ def bow_net(data, class TestGradientClip(unittest.TestCase): - def setUp(self): self.word_dict_len = 5147 self.BATCH_SIZE = 2 @@ -74,8 +69,9 @@ class TestGradientClip(unittest.TestCase): def check_gradient_clip(self, place, dtype='float32'): prog = fluid.Program() startup_program = fluid.Program() - with fluid.program_guard(main_program=prog, - startup_program=startup_program): + with fluid.program_guard( + main_program=prog, startup_program=startup_program + ): image = fluid.data(name="a", shape=[-1, 784], dtype='float32') label = fluid.data(name="b", shape=[-1, 1], dtype='int64') if dtype != 'float32': @@ -96,8 +92,9 @@ class TestGradientClip(unittest.TestCase): p_g = sorted(p_g, key=lambda x: x[0].name) p_g_clip = sorted(p_g_clip, key=lambda x: x[0].name) - with fluid.program_guard(main_program=prog_clip, - startup_program=startup_program): + with fluid.program_guard( + main_program=prog_clip, startup_program=startup_program + ): p_g_clip = self.clip_gradient(p_g_clip) grad_list = [elem[1] for elem in p_g] @@ -110,20 +107,20 @@ class TestGradientClip(unittest.TestCase): data = next(train_reader()) out = exe.run(prog, feed=feeder.feed(data), fetch_list=grad_list) - out_clip = exe.run(prog_clip, - feed=feeder.feed(data), - fetch_list=grad_clip_list) + out_clip = exe.run( + prog_clip, feed=feeder.feed(data), fetch_list=grad_clip_list + ) self.check_clip_result(out, out_clip) def check_sparse_gradient_clip(self, place): prog = fluid.Program() startup_program = fluid.Program() - with fluid.program_guard(main_program=prog, - startup_program=startup_program): - data = fluid.data(name="words", - shape=[-1, 1], - dtype="int64", - lod_level=1) + with fluid.program_guard( + main_program=prog, startup_program=startup_program + ): + data = fluid.data( + name="words", shape=[-1, 1], dtype="int64", lod_level=1 + ) label = fluid.data(name="label", shape=[-1, 1], dtype="int64") cost = bow_net(data, label, self.word_dict_len) @@ -135,7 +132,7 @@ class TestGradientClip(unittest.TestCase): data = next(self.train_data()) val = exe.run(prog, feed=feeder.feed(data), fetch_list=[cost])[0] - self.assertEqual((1, ), val.shape) + self.assertEqual((1,), val.shape) self.assertFalse(np.isnan(val)) def backward_and_optimize(self, cost): @@ -143,7 +140,6 @@ class TestGradientClip(unittest.TestCase): class TestGradientClipByGlobalNorm(TestGradientClip): - def init(self): self.clip_norm = 0.2 @@ -163,13 +159,13 @@ class TestGradientClipByGlobalNorm(TestGradientClip): v, rtol=1e-05, atol=1e-08, - err_msg= - 'gradient clip by global norm has wrong results!, \nu={}\nv={}\ndiff={}' - .format(u, v, u - v)) + err_msg='gradient clip by global norm has wrong results!, \nu={}\nv={}\ndiff={}'.format( + u, v, u - v + ), + ) # test whether the output is right when use 'set_gradient_clip' def test_old_gradient_clip(self): - def func(params_grads): clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=self.clip_norm) fluid.clip.set_gradient_clip(clip) @@ -180,7 +176,6 @@ class TestGradientClipByGlobalNorm(TestGradientClip): # test whether the output is right when use grad_clip def test_new_gradient_clip(self): - def func(params_grads): clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=self.clip_norm) return clip(params_grads) @@ -190,7 +185,6 @@ class TestGradientClipByGlobalNorm(TestGradientClip): # test whether the output is right when use grad_clip under float64 def test_new_gradient_clip_fp64(self): - def func(params_grads): clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=self.clip_norm) return clip(params_grads) @@ -200,12 +194,12 @@ class TestGradientClipByGlobalNorm(TestGradientClip): # invoke 'set_gradient_clip' in a wrong order def test_wrong_API_order(self): - def backward_func(cost): clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0) fluid.clip.set_gradient_clip(clip) - sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.01, - grad_clip=clip) + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=0.01, grad_clip=clip + ) # if 'set_gradient_clip' and 'optimize(grad_clip)' together, 'set_gradient_clip' will be ineffective sgd_optimizer.minimize(cost) # 'set_gradient_clip' must before 'minimize', otherwise, 'set_gradient_clip' will be ineffective @@ -219,43 +213,74 @@ class TestGradientClipByGlobalNorm(TestGradientClip): def test_tpyeError(self): # the type of optimizer(grad_clip=) must be an instance of GradientClipBase's derived class with self.assertRaises(TypeError): - sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.1, - grad_clip="test") + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=0.1, grad_clip="test" + ) # if grad is None or not need clip def test_none_grad_fp32(self): ops = self._test_none_grad_helper("float32") - self.assertListEqual(ops, [ - 'squared_l2_norm', 'squared_l2_norm', 'sum', 'sqrt', - 'fill_constant', 'elementwise_max', 'elementwise_div', - 'elementwise_mul', 'elementwise_mul' - ]) + self.assertListEqual( + ops, + [ + 'squared_l2_norm', + 'squared_l2_norm', + 'sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + ], + ) def test_none_grad_fp16(self): ops = self._test_none_grad_helper("float16") - self.assertListEqual(ops, [ - 'square', 'reduce_sum', 'square', 'reduce_sum', 'sum', 'cast', - 'sqrt', 'fill_constant', 'elementwise_max', 'elementwise_div', - 'cast', 'elementwise_mul', 'cast', 'elementwise_mul' - ]) + self.assertListEqual( + ops, + [ + 'square', + 'reduce_sum', + 'square', + 'reduce_sum', + 'sum', + 'cast', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'cast', + 'elementwise_mul', + 'cast', + 'elementwise_mul', + ], + ) def _test_none_grad_helper(self, dtype): prog = fluid.Program() startup_program = fluid.Program() - with fluid.program_guard(main_program=prog, - startup_program=startup_program): + with fluid.program_guard( + main_program=prog, startup_program=startup_program + ): clip = fluid.clip.GradientClipByGlobalNorm(self.clip_norm) - x = fluid.default_main_program().global_block().create_parameter( - name="x", shape=[2, 3], dtype=dtype) - y = fluid.default_main_program().global_block().create_parameter( - name="y", shape=[2, 3], dtype=dtype) + x = ( + fluid.default_main_program() + .global_block() + .create_parameter(name="x", shape=[2, 3], dtype=dtype) + ) + y = ( + fluid.default_main_program() + .global_block() + .create_parameter(name="y", shape=[2, 3], dtype=dtype) + ) # (x, None) should not be returned params_grads = [(x, None), (x, y), (y, x)] params_grads = clip(params_grads) self.assertTrue( len(params_grads) == 2, - "ClipByGlobalNorm: when grad is None, it shouldn't be returned by gradient clip!" + "ClipByGlobalNorm: when grad is None, it shouldn't be returned by gradient clip!", ) ops = [op.type for op in x.block.ops] @@ -263,7 +288,6 @@ class TestGradientClipByGlobalNorm(TestGradientClip): class TestGradientClipByNorm(TestGradientClip): - def init(self): self.clip_norm = 0.2 @@ -277,11 +301,11 @@ class TestGradientClipByNorm(TestGradientClip): v, rtol=1e-05, atol=1e-08, - err_msg='gradient clip by norm has wrong results!') + err_msg='gradient clip by norm has wrong results!', + ) # test whether the output is right when use grad_clip def test_gradient_clip(self): - def func(params_grads): clip = fluid.clip.GradientClipByNorm(clip_norm=self.clip_norm) return clip(params_grads) @@ -292,25 +316,35 @@ class TestGradientClipByNorm(TestGradientClip): # if grad is None or not need clip def test_none_grad(self): clip = fluid.clip.GradientClipByNorm(self.clip_norm) - x = fluid.default_main_program().global_block().create_parameter( - name="x", shape=[2, 3], dtype="float32", need_clip=False) - y = fluid.default_main_program().global_block().create_parameter( - name="y", shape=[2, 3], dtype="float32", need_clip=False) + x = ( + fluid.default_main_program() + .global_block() + .create_parameter( + name="x", shape=[2, 3], dtype="float32", need_clip=False + ) + ) + y = ( + fluid.default_main_program() + .global_block() + .create_parameter( + name="y", shape=[2, 3], dtype="float32", need_clip=False + ) + ) # (x, None) should not be returned params_grads = [(x, None), (x, y)] params_grads = clip(params_grads) self.assertTrue( len(clip(params_grads)) == 1, - "ClipGradByNorm: when grad is None, it shouldn't be returned by gradient clip!" + "ClipGradByNorm: when grad is None, it shouldn't be returned by gradient clip!", ) self.assertTrue( params_grads[0][1].name == 'y', - "ClipGradByNorm: grad should not be clipped when filtered out!") + "ClipGradByNorm: grad should not be clipped when filtered out!", + ) class TestGradientClipByValue(TestGradientClip): - def init(self): self.max = 0.2 self.min = 0.1 @@ -325,11 +359,11 @@ class TestGradientClipByValue(TestGradientClip): v, rtol=1e-06, atol=1e-08, - err_msg='gradient clip by value has wrong results!') + err_msg='gradient clip by value has wrong results!', + ) # test whether the output is right when use grad_clip def test_gradient_clip(self): - def func(params_grads): clip = fluid.clip.GradientClipByValue(max=self.max, min=self.min) return clip(params_grads) @@ -340,37 +374,49 @@ class TestGradientClipByValue(TestGradientClip): # if grad is None or not need clip def test_none_grad(self): clip = fluid.clip.GradientClipByValue(self.max, self.min) - x = fluid.default_main_program().global_block().create_parameter( - name="x", shape=[2, 3], dtype="float32", need_clip=False) - y = fluid.default_main_program().global_block().create_parameter( - name="y", shape=[2, 3], dtype="float32", need_clip=False) + x = ( + fluid.default_main_program() + .global_block() + .create_parameter( + name="x", shape=[2, 3], dtype="float32", need_clip=False + ) + ) + y = ( + fluid.default_main_program() + .global_block() + .create_parameter( + name="y", shape=[2, 3], dtype="float32", need_clip=False + ) + ) # (x, None) should not be returned params_grads = [(x, None), (x, y)] params_grads = clip(params_grads) self.assertTrue( len(clip(params_grads)) == 1, - "ClipGradByValue: when grad is None, it shouldn't be returned by gradient clip!" + "ClipGradByValue: when grad is None, it shouldn't be returned by gradient clip!", ) self.assertTrue( params_grads[0][1].name == 'y', - "ClipGradByValue: grad should not be clipped when filtered out!") + "ClipGradByValue: grad should not be clipped when filtered out!", + ) class TestDygraphGradientClip(unittest.TestCase): - def test_gradient_clip(self): with fluid.dygraph.guard(): linear = fluid.dygraph.Linear(5, 5) - inputs = fluid.layers.uniform_random([16, 5], min=-10, - max=10).astype('float32') + inputs = fluid.layers.uniform_random( + [16, 5], min=-10, max=10 + ).astype('float32') out = linear(fluid.dygraph.to_variable(inputs)) loss = fluid.layers.reduce_mean(out) loss.backward() sgd_optimizer = fluid.optimizer.SGD( learning_rate=0.0, parameter_list=linear.parameters(), - grad_clip=fluid.clip.GradientClipByGlobalNorm(0.1)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(0.1), + ) self.check_clip_result(loss, sgd_optimizer) def check_clip_result(self, loss, optimizer): @@ -378,20 +424,23 @@ class TestDygraphGradientClip(unittest.TestCase): class TestDygraphGradientClipByGlobalNorm(TestDygraphGradientClip): - def setUp(self): self.clip_norm = 0.8 self.clip1 = fluid.clip.GradientClipByGlobalNorm( - clip_norm=self.clip_norm) + clip_norm=self.clip_norm + ) self.clip2 = fluid.clip.GradientClipByGlobalNorm( - clip_norm=self.clip_norm) + clip_norm=self.clip_norm + ) def check_clip_result(self, loss, optimizer): # if grad is None - x = fluid.dygraph.to_variable(np.array([2, 3]).astype("float32"), - name="x") - y = fluid.dygraph.to_variable(np.array([3, 4]).astype("float32"), - name="y") + x = fluid.dygraph.to_variable( + np.array([2, 3]).astype("float32"), name="x" + ) + y = fluid.dygraph.to_variable( + np.array([3, 4]).astype("float32"), name="y" + ) assert len(self.clip1([(x, x), (x, y), (x, None)])) == 2 # get params and grads from network opt, params_grads = optimizer.minimize(loss) @@ -416,11 +465,11 @@ class TestDygraphGradientClipByGlobalNorm(TestDygraphGradientClip): self.assertTrue( np.isclose(a=a, b=b, rtol=1e-6, atol=1e-8), "gradient clip by global norm has wrong results, expetcd:%f, but received:%f" - % (a, b)) + % (a, b), + ) class TestDygraphGradientClipByNorm(TestDygraphGradientClip): - def setUp(self): self.clip_norm = 0.8 self.clip = fluid.clip.GradientClipByNorm(clip_norm=self.clip_norm) @@ -445,11 +494,11 @@ class TestDygraphGradientClipByNorm(TestDygraphGradientClip): self.assertTrue( np.isclose(a=a, b=b, rtol=1e-6, atol=1e-8), "gradient clip by norm has wrong results, expetcd:%f, but received:%f" - % (a, b)) + % (a, b), + ) class TestDygraphGradientClipByValue(TestDygraphGradientClip): - def setUp(self): self.max = 0.2 self.min = 0.1 @@ -472,11 +521,11 @@ class TestDygraphGradientClipByValue(TestDygraphGradientClip): v, rtol=1e-06, atol=1e-08, - err_msg='gradient clip by value has wrong results!') + err_msg='gradient clip by value has wrong results!', + ) class SimpleNet(paddle.nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.linear = paddle.nn.Linear(5, 5) @@ -489,19 +538,21 @@ class SimpleNet(paddle.nn.Layer): class TestDygraphGradientClipFP16(unittest.TestCase): - def test_gradient_clip(self): if fluid.core.is_compiled_with_cuda(): with fluid.dygraph.guard(): paddle.seed(10) model = SimpleNet() sgd_optimizer = paddle.optimizer.SGD( - learning_rate=0.0, parameters=model.parameters()) + learning_rate=0.0, parameters=model.parameters() + ) model, sgd_optimizer = paddle.amp.decorate( - models=model, optimizers=sgd_optimizer, level='O2') + models=model, optimizers=sgd_optimizer, level='O2' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - inputs = fluid.layers.uniform_random([1, 5], min=-10, - max=10).astype('float32') + inputs = fluid.layers.uniform_random( + [1, 5], min=-10, max=10 + ).astype('float32') with paddle.amp.auto_cast(level='O2'): out = model(fluid.dygraph.to_variable(inputs)) loss = fluid.layers.reduce_mean(out) @@ -540,15 +591,16 @@ class TestDygraphGradientClipFP16(unittest.TestCase): self.assertTrue( np.isclose(a=a, b=b, rtol=1e-3, atol=1e-8), "gradient clip by global norm has wrong results, expetcd:%f, but received:%f" - % (a, b)) + % (a, b), + ) class TestDygraphGradientClipFP64(unittest.TestCase): - def test_gradient_clip(self): with fluid.dygraph.guard(): - inputs = fluid.layers.uniform_random([16, 5], min=-10, - max=10).astype('float64') + inputs = fluid.layers.uniform_random( + [16, 5], min=-10, max=10 + ).astype('float64') linear = fluid.dygraph.Linear(5, 5, dtype="float64") out = linear(fluid.dygraph.to_variable(inputs)) loss = fluid.layers.reduce_mean(out) @@ -586,11 +638,11 @@ class TestDygraphGradientClipFP64(unittest.TestCase): self.assertTrue( np.isclose(a=a, b=b, rtol=1e-6, atol=1e-8), "gradient clip by global norm has wrong results, expetcd:%f, but received:%f" - % (a, b)) + % (a, b), + ) class TestPureFP16ClipGradByGlobalNorm(unittest.TestCase): - def check_main(self, expected_has_cast_op): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -601,12 +653,12 @@ class TestPureFP16ClipGradByGlobalNorm(unittest.TestCase): param_and_grads = [] main_block = main_prog.global_block() for name, shape in zip(names, shapes): - p = main_block.create_parameter(name=name, - shape=shape, - dtype='float16') - g = main_block.create_parameter(name=p.name + '@GRAD', - shape=p.shape, - dtype=p.dtype) + p = main_block.create_parameter( + name=name, shape=shape, dtype='float16' + ) + g = main_block.create_parameter( + name=p.name + '@GRAD', shape=p.shape, dtype=p.dtype + ) param_and_grads.append((p, g)) clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) diff --git a/python/paddle/fluid/tests/unittests/test_graph_khop_sampler.py b/python/paddle/fluid/tests/unittests/test_graph_khop_sampler.py index 57b8209d8d6..b2f57af35de 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_khop_sampler.py +++ b/python/paddle/fluid/tests/unittests/test_graph_khop_sampler.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class TestGraphKhopSampler(unittest.TestCase): - def setUp(self): num_nodes = 20 edges = np.random.randint(num_nodes, size=(100, 2)) @@ -42,8 +41,9 @@ class TestGraphKhopSampler(unittest.TestCase): self.row = sorted_edges[:, 0].astype("int64") self.colptr = colptr.astype("int64") self.sorted_eid = sorted_eid.astype("int64") - self.nodes = np.unique(np.random.randint(num_nodes, - size=5)).astype("int64") + self.nodes = np.unique(np.random.randint(num_nodes, size=5)).astype( + "int64" + ) self.sample_sizes = [5, 5] self.dst_src_dict = dst_src_dict @@ -53,10 +53,14 @@ class TestGraphKhopSampler(unittest.TestCase): colptr = paddle.to_tensor(self.colptr) nodes = paddle.to_tensor(self.nodes) - edge_src, edge_dst, sample_index, reindex_nodes = \ - paddle.incubate.graph_khop_sampler(row, colptr, - nodes, self.sample_sizes, - return_eids=False) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + ) = paddle.incubate.graph_khop_sampler( + row, colptr, nodes, self.sample_sizes, return_eids=False + ) # Reindex edge_src and edge_dst to original index. edge_src = edge_src.reshape([-1]) edge_dst = edge_dst.reshape([-1]) @@ -72,10 +76,13 @@ class TestGraphKhopSampler(unittest.TestCase): continue # Ensure no repetitive sample neighbors. self.assertTrue( - edge_src_n.shape[0] == paddle.unique(edge_src_n).shape[0]) + edge_src_n.shape[0] == paddle.unique(edge_src_n).shape[0] + ) # Ensure the correct sample size. - self.assertTrue(edge_src_n.shape[0] == self.sample_sizes[0] - or edge_src_n.shape[0] == len(self.dst_src_dict[n])) + self.assertTrue( + edge_src_n.shape[0] == self.sample_sizes[0] + or edge_src_n.shape[0] == len(self.dst_src_dict[n]) + ) in_neighbors = np.isin(edge_src_n.numpy(), self.dst_src_dict[n]) # Ensure the correct sample neighbors. self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) @@ -91,22 +98,35 @@ class TestGraphKhopSampler(unittest.TestCase): row = None if fluid.framework.in_dygraph_mode(): row = paddle.fluid.core.eager.to_uva_tensor( - self.row.astype(self.row.dtype), 0) + self.row.astype(self.row.dtype), 0 + ) sorted_eid = paddle.fluid.core.eager.to_uva_tensor( - self.sorted_eid.astype(self.sorted_eid.dtype), 0) + self.sorted_eid.astype(self.sorted_eid.dtype), 0 + ) else: row = paddle.fluid.core.to_uva_tensor( - self.row.astype(self.row.dtype)) + self.row.astype(self.row.dtype) + ) sorted_eid = paddle.fluid.core.to_uva_tensor( - self.sorted_eid.astype(self.sorted_eid.dtype)) + self.sorted_eid.astype(self.sorted_eid.dtype) + ) colptr = paddle.to_tensor(self.colptr) nodes = paddle.to_tensor(self.nodes) - edge_src, edge_dst, sample_index, reindex_nodes, edge_eids = \ - paddle.incubate.graph_khop_sampler(row, colptr, - nodes, self.sample_sizes, - sorted_eids=sorted_eid, - return_eids=True) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + edge_eids, + ) = paddle.incubate.graph_khop_sampler( + row, + colptr, + nodes, + self.sample_sizes, + sorted_eids=sorted_eid, + return_eids=True, + ) edge_src = edge_src.reshape([-1]) edge_dst = edge_dst.reshape([-1]) sample_index = sample_index.reshape([-1]) @@ -120,10 +140,12 @@ class TestGraphKhopSampler(unittest.TestCase): if edge_src_n.shape[0] == 0: continue self.assertTrue( - edge_src_n.shape[0] == paddle.unique(edge_src_n).shape[0]) + edge_src_n.shape[0] == paddle.unique(edge_src_n).shape[0] + ) self.assertTrue( edge_src_n.shape[0] == self.sample_sizes[0] - or edge_src_n.shape[0] == len(self.dst_src_dict[n])) + or edge_src_n.shape[0] == len(self.dst_src_dict[n]) + ) in_neighbors = np.isin(edge_src_n.numpy(), self.dst_src_dict[n]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) @@ -135,31 +157,40 @@ class TestGraphKhopSampler(unittest.TestCase): def test_sample_result_static_with_eids(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - sorted_eids = paddle.static.data(name="eids", - shape=self.sorted_eid.shape, - dtype=self.sorted_eid.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) - - edge_src, edge_dst, sample_index, reindex_nodes, edge_eids = \ - paddle.incubate.graph_khop_sampler(row, colptr, - nodes, self.sample_sizes, - sorted_eids, True) + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + sorted_eids = paddle.static.data( + name="eids", + shape=self.sorted_eid.shape, + dtype=self.sorted_eid.dtype, + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) + + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + edge_eids, + ) = paddle.incubate.graph_khop_sampler( + row, colptr, nodes, self.sample_sizes, sorted_eids, True + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'eids': self.sorted_eid, - 'colptr': self.colptr, - 'nodes': self.nodes - }, - fetch_list=[edge_src, edge_dst, sample_index]) + ret = exe.run( + feed={ + 'row': self.row, + 'eids': self.sorted_eid, + 'colptr': self.colptr, + 'nodes': self.nodes, + }, + fetch_list=[edge_src, edge_dst, sample_index], + ) edge_src, edge_dst, sample_index = ret edge_src = edge_src.reshape([-1]) @@ -175,35 +206,44 @@ class TestGraphKhopSampler(unittest.TestCase): if edge_src_n.shape[0] == 0: continue self.assertTrue( - edge_src_n.shape[0] == np.unique(edge_src_n).shape[0]) + edge_src_n.shape[0] == np.unique(edge_src_n).shape[0] + ) self.assertTrue( edge_src_n.shape[0] == self.sample_sizes[0] - or edge_src_n.shape[0] == len(self.dst_src_dict[n])) + or edge_src_n.shape[0] == len(self.dst_src_dict[n]) + ) in_neighbors = np.isin(edge_src_n, self.dst_src_dict[n]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_static_without_eids(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) - edge_src, edge_dst, sample_index, reindex_nodes = \ - paddle.incubate.graph_khop_sampler(row, colptr, - nodes, self.sample_sizes) + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + ) = paddle.incubate.graph_khop_sampler( + row, colptr, nodes, self.sample_sizes + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes - }, - fetch_list=[edge_src, edge_dst, sample_index]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + }, + fetch_list=[edge_src, edge_dst, sample_index], + ) edge_src, edge_dst, sample_index = ret edge_src = edge_src.reshape([-1]) edge_dst = edge_dst.reshape([-1]) @@ -218,10 +258,12 @@ class TestGraphKhopSampler(unittest.TestCase): if edge_src_n.shape[0] == 0: continue self.assertTrue( - edge_src_n.shape[0] == np.unique(edge_src_n).shape[0]) + edge_src_n.shape[0] == np.unique(edge_src_n).shape[0] + ) self.assertTrue( edge_src_n.shape[0] == self.sample_sizes[0] - or edge_src_n.shape[0] == len(self.dst_src_dict[n])) + or edge_src_n.shape[0] == len(self.dst_src_dict[n]) + ) in_neighbors = np.isin(edge_src_n, self.dst_src_dict[n]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) diff --git a/python/paddle/fluid/tests/unittests/test_graph_reindex.py b/python/paddle/fluid/tests/unittests/test_graph_reindex.py index db1d7d3cff9..318deda61f2 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_reindex.py +++ b/python/paddle/fluid/tests/unittests/test_graph_reindex.py @@ -18,7 +18,6 @@ import paddle class TestGraphReindex(unittest.TestCase): - def setUp(self): self.x = np.arange(5).astype("int64") self.neighbors = np.random.randint(100, size=20).astype("int64") @@ -32,7 +31,8 @@ class TestGraphReindex(unittest.TestCase): self.out_nodes = np.array(out_nodes, dtype="int64") reindex_dict = {node: ind for ind, node in enumerate(self.out_nodes)} self.reindex_src = np.array( - [reindex_dict[node] for node in self.neighbors]) + [reindex_dict[node] for node in self.neighbors] + ) reindex_dst = [] for node, c in zip(self.x, self.count): for i in range(c): @@ -48,16 +48,21 @@ class TestGraphReindex(unittest.TestCase): value_buffer = paddle.full([self.num_nodes], -1, dtype="int32") index_buffer = paddle.full([self.num_nodes], -1, dtype="int32") - reindex_src, reindex_dst, out_nodes = \ - paddle.incubate.graph_reindex(x, neighbors, count) + reindex_src, reindex_dst, out_nodes = paddle.incubate.graph_reindex( + x, neighbors, count + ) np.testing.assert_allclose(self.reindex_src, reindex_src, rtol=1e-05) np.testing.assert_allclose(self.reindex_dst, reindex_dst, rtol=1e-05) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) - reindex_src, reindex_dst, out_nodes = \ - paddle.incubate.graph_reindex(x, neighbors, count, - value_buffer, index_buffer, - flag_buffer_hashtable=True) + reindex_src, reindex_dst, out_nodes = paddle.incubate.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + flag_buffer_hashtable=True, + ) np.testing.assert_allclose(self.reindex_src, reindex_src, rtol=1e-05) np.testing.assert_allclose(self.reindex_dst, reindex_dst, rtol=1e-05) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) @@ -70,20 +75,21 @@ class TestGraphReindex(unittest.TestCase): count = paddle.to_tensor(self.count) count = paddle.concat([count, count]) - reindex_src, reindex_dst, out_nodes = \ - paddle.incubate.graph_reindex(x, neighbors, count) - np.testing.assert_allclose(self.reindex_src, - reindex_src[:self.neighbors.shape[0]], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_src, - reindex_src[self.neighbors.shape[0]:], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst[:self.neighbors.shape[0]], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst[self.neighbors.shape[0]:], - rtol=1e-05) + reindex_src, reindex_dst, out_nodes = paddle.incubate.graph_reindex( + x, neighbors, count + ) + np.testing.assert_allclose( + self.reindex_src, reindex_src[: self.neighbors.shape[0]], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_src, reindex_src[self.neighbors.shape[0] :], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst[: self.neighbors.shape[0]], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst[self.neighbors.shape[0] :], rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) def test_heter_reindex_result_v2(self): @@ -111,10 +117,11 @@ class TestGraphReindex(unittest.TestCase): reindex_dst.append(reindex_dict[node]) reindex_dst = np.array(reindex_dst, dtype="int64") - reindex_src_, reindex_dst_, out_nodes_ = \ - paddle.incubate.graph_reindex(paddle.to_tensor(x), - paddle.to_tensor(neighbors), - paddle.to_tensor(counts)) + reindex_src_, reindex_dst_, out_nodes_ = paddle.incubate.graph_reindex( + paddle.to_tensor(x), + paddle.to_tensor(neighbors), + paddle.to_tensor(counts), + ) np.testing.assert_allclose(reindex_src, reindex_src_, rtol=1e-05) np.testing.assert_allclose(reindex_dst, reindex_dst_, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_, rtol=1e-05) @@ -122,66 +129,89 @@ class TestGraphReindex(unittest.TestCase): def test_reindex_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - x = paddle.static.data(name="x", - shape=self.x.shape, - dtype=self.x.dtype) - neighbors = paddle.static.data(name="neighbors", - shape=self.neighbors.shape, - dtype=self.neighbors.dtype) - count = paddle.static.data(name="count", - shape=self.count.shape, - dtype=self.count.dtype) - value_buffer = paddle.static.data(name="value_buffer", - shape=[self.num_nodes], - dtype="int32") - index_buffer = paddle.static.data(name="index_buffer", - shape=[self.num_nodes], - dtype="int32") - - reindex_src_1, reindex_dst_1, out_nodes_1 = \ - paddle.incubate.graph_reindex(x, neighbors, count) - reindex_src_2, reindex_dst_2, out_nodes_2 = \ - paddle.incubate.graph_reindex(x, neighbors, count, - value_buffer, index_buffer, - flag_buffer_hashtable=True) + x = paddle.static.data( + name="x", shape=self.x.shape, dtype=self.x.dtype + ) + neighbors = paddle.static.data( + name="neighbors", + shape=self.neighbors.shape, + dtype=self.neighbors.dtype, + ) + count = paddle.static.data( + name="count", shape=self.count.shape, dtype=self.count.dtype + ) + value_buffer = paddle.static.data( + name="value_buffer", shape=[self.num_nodes], dtype="int32" + ) + index_buffer = paddle.static.data( + name="index_buffer", shape=[self.num_nodes], dtype="int32" + ) + + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + ) = paddle.incubate.graph_reindex(x, neighbors, count) + ( + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = paddle.incubate.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + flag_buffer_hashtable=True, + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'x': - self.x, - 'neighbors': - self.neighbors, - 'count': - self.count, - 'value_buffer': - np.full([self.num_nodes], -1, dtype="int32"), - 'index_buffer': - np.full([self.num_nodes], -1, dtype="int32") - }, - fetch_list=[ - reindex_src_1, reindex_dst_1, out_nodes_1, - reindex_src_2, reindex_dst_2, out_nodes_2 - ]) - reindex_src_1, reindex_dst_1, out_nodes_1, reindex_src_2, \ - reindex_dst_2, out_nodes_2 = ret - np.testing.assert_allclose(self.reindex_src, - reindex_src_1, - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst_1, - rtol=1e-05) + ret = exe.run( + feed={ + 'x': self.x, + 'neighbors': self.neighbors, + 'count': self.count, + 'value_buffer': np.full( + [self.num_nodes], -1, dtype="int32" + ), + 'index_buffer': np.full( + [self.num_nodes], -1, dtype="int32" + ), + }, + fetch_list=[ + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ], + ) + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = ret + np.testing.assert_allclose( + self.reindex_src, reindex_src_1, rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst_1, rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes_1, rtol=1e-05) - np.testing.assert_allclose(self.reindex_src, - reindex_src_2, - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst_2, - rtol=1e-05) + np.testing.assert_allclose( + self.reindex_src, reindex_src_2, rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst_2, rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes_2, rtol=1e-05) class TestGeometricGraphReindex(unittest.TestCase): - def setUp(self): self.x = np.arange(5).astype("int64") self.neighbors = np.random.randint(100, size=20).astype("int64") @@ -195,7 +225,8 @@ class TestGeometricGraphReindex(unittest.TestCase): self.out_nodes = np.array(out_nodes, dtype="int64") reindex_dict = {node: ind for ind, node in enumerate(self.out_nodes)} self.reindex_src = np.array( - [reindex_dict[node] for node in self.neighbors]) + [reindex_dict[node] for node in self.neighbors] + ) reindex_dst = [] for node, c in zip(self.x, self.count): for i in range(c): @@ -211,15 +242,16 @@ class TestGeometricGraphReindex(unittest.TestCase): value_buffer = paddle.full([self.num_nodes], -1, dtype="int32") index_buffer = paddle.full([self.num_nodes], -1, dtype="int32") - reindex_src, reindex_dst, out_nodes = \ - paddle.geometric.reindex_graph(x, neighbors, count) + reindex_src, reindex_dst, out_nodes = paddle.geometric.reindex_graph( + x, neighbors, count + ) np.testing.assert_allclose(self.reindex_src, reindex_src, rtol=1e-05) np.testing.assert_allclose(self.reindex_dst, reindex_dst, rtol=1e-05) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) - reindex_src, reindex_dst, out_nodes = \ - paddle.geometric.reindex_graph(x, neighbors, count, - value_buffer, index_buffer) + reindex_src, reindex_dst, out_nodes = paddle.geometric.reindex_graph( + x, neighbors, count, value_buffer, index_buffer + ) np.testing.assert_allclose(self.reindex_src, reindex_src, rtol=1e-05) np.testing.assert_allclose(self.reindex_dst, reindex_dst, rtol=1e-05) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) @@ -232,20 +264,21 @@ class TestGeometricGraphReindex(unittest.TestCase): count = paddle.to_tensor(self.count) count = paddle.concat([count, count]) - reindex_src, reindex_dst, out_nodes = \ - paddle.geometric.reindex_graph(x, neighbors, count) - np.testing.assert_allclose(self.reindex_src, - reindex_src[:self.neighbors.shape[0]], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_src, - reindex_src[self.neighbors.shape[0]:], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst[:self.neighbors.shape[0]], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst[self.neighbors.shape[0]:], - rtol=1e-05) + reindex_src, reindex_dst, out_nodes = paddle.geometric.reindex_graph( + x, neighbors, count + ) + np.testing.assert_allclose( + self.reindex_src, reindex_src[: self.neighbors.shape[0]], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_src, reindex_src[self.neighbors.shape[0] :], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst[: self.neighbors.shape[0]], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst[self.neighbors.shape[0] :], rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) def test_heter_reindex_result_v2(self): @@ -273,10 +306,11 @@ class TestGeometricGraphReindex(unittest.TestCase): reindex_dst.append(reindex_dict[node]) reindex_dst = np.array(reindex_dst, dtype="int64") - reindex_src_, reindex_dst_, out_nodes_ = \ - paddle.geometric.reindex_graph(paddle.to_tensor(x), - paddle.to_tensor(neighbors), - paddle.to_tensor(counts)) + reindex_src_, reindex_dst_, out_nodes_ = paddle.geometric.reindex_graph( + paddle.to_tensor(x), + paddle.to_tensor(neighbors), + paddle.to_tensor(counts), + ) np.testing.assert_allclose(reindex_src, reindex_src_, rtol=1e-05) np.testing.assert_allclose(reindex_dst, reindex_dst_, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_, rtol=1e-05) @@ -308,9 +342,13 @@ class TestGeometricGraphReindex(unittest.TestCase): neighbors = [paddle.to_tensor(neighbors1), paddle.to_tensor(neighbors2)] count = [paddle.to_tensor(count1), paddle.to_tensor(count2)] - reindex_src_, reindex_dst_, out_nodes_ = \ - paddle.geometric.reindex_heter_graph(paddle.to_tensor(x), - neighbors, count) + ( + reindex_src_, + reindex_dst_, + out_nodes_, + ) = paddle.geometric.reindex_heter_graph( + paddle.to_tensor(x), neighbors, count + ) np.testing.assert_allclose(reindex_src, reindex_src_, rtol=1e-05) np.testing.assert_allclose(reindex_dst, reindex_dst_, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_, rtol=1e-05) @@ -318,60 +356,80 @@ class TestGeometricGraphReindex(unittest.TestCase): def test_reindex_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - x = paddle.static.data(name="x", - shape=self.x.shape, - dtype=self.x.dtype) - neighbors = paddle.static.data(name="neighbors", - shape=self.neighbors.shape, - dtype=self.neighbors.dtype) - count = paddle.static.data(name="count", - shape=self.count.shape, - dtype=self.count.dtype) - value_buffer = paddle.static.data(name="value_buffer", - shape=[self.num_nodes], - dtype="int32") - index_buffer = paddle.static.data(name="index_buffer", - shape=[self.num_nodes], - dtype="int32") - - reindex_src_1, reindex_dst_1, out_nodes_1 = \ - paddle.geometric.reindex_graph(x, neighbors, count) - reindex_src_2, reindex_dst_2, out_nodes_2 = \ - paddle.geometric.reindex_graph(x, neighbors, count, - value_buffer, index_buffer) + x = paddle.static.data( + name="x", shape=self.x.shape, dtype=self.x.dtype + ) + neighbors = paddle.static.data( + name="neighbors", + shape=self.neighbors.shape, + dtype=self.neighbors.dtype, + ) + count = paddle.static.data( + name="count", shape=self.count.shape, dtype=self.count.dtype + ) + value_buffer = paddle.static.data( + name="value_buffer", shape=[self.num_nodes], dtype="int32" + ) + index_buffer = paddle.static.data( + name="index_buffer", shape=[self.num_nodes], dtype="int32" + ) + + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + ) = paddle.geometric.reindex_graph(x, neighbors, count) + ( + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = paddle.geometric.reindex_graph( + x, neighbors, count, value_buffer, index_buffer + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'x': - self.x, - 'neighbors': - self.neighbors, - 'count': - self.count, - 'value_buffer': - np.full([self.num_nodes], -1, dtype="int32"), - 'index_buffer': - np.full([self.num_nodes], -1, dtype="int32") - }, - fetch_list=[ - reindex_src_1, reindex_dst_1, out_nodes_1, - reindex_src_2, reindex_dst_2, out_nodes_2 - ]) - reindex_src_1, reindex_dst_1, out_nodes_1, reindex_src_2, \ - reindex_dst_2, out_nodes_2 = ret - np.testing.assert_allclose(self.reindex_src, - reindex_src_1, - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst_1, - rtol=1e-05) + ret = exe.run( + feed={ + 'x': self.x, + 'neighbors': self.neighbors, + 'count': self.count, + 'value_buffer': np.full( + [self.num_nodes], -1, dtype="int32" + ), + 'index_buffer': np.full( + [self.num_nodes], -1, dtype="int32" + ), + }, + fetch_list=[ + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ], + ) + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = ret + np.testing.assert_allclose( + self.reindex_src, reindex_src_1, rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst_1, rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes_1, rtol=1e-05) - np.testing.assert_allclose(self.reindex_src, - reindex_src_2, - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst_2, - rtol=1e-05) + np.testing.assert_allclose( + self.reindex_src, reindex_src_2, rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst_2, rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes_2, rtol=1e-05) def test_heter_reindex_result_static(self): @@ -401,48 +459,69 @@ class TestGeometricGraphReindex(unittest.TestCase): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[5], dtype="int64") - neighbors1 = paddle.static.data(name="neighbors1", - shape=[20], - dtype="int64") + neighbors1 = paddle.static.data( + name="neighbors1", shape=[20], dtype="int64" + ) count1 = paddle.static.data(name="count1", shape=[5], dtype="int32") - neighbors2 = paddle.static.data(name="neighbors2", - shape=[20], - dtype="int64") + neighbors2 = paddle.static.data( + name="neighbors2", shape=[20], dtype="int64" + ) count2 = paddle.static.data(name="count2", shape=[5], dtype="int32") - value_buffer = paddle.static.data(name="value_buffer", - shape=[5], - dtype="int32") - index_buffer = paddle.static.data(name="index_buffer", - shape=[5], - dtype="int32") - - reindex_src_1, reindex_dst_1, out_nodes_1 = \ - paddle.geometric.reindex_heter_graph(x, - [neighbors1, neighbors2], - [count1, count2]) - reindex_src_2, reindex_dst_2, out_nodes_2 = \ - paddle.geometric.reindex_heter_graph(x, - [neighbors1, neighbors2], - [count1, count2], - value_buffer, index_buffer) + value_buffer = paddle.static.data( + name="value_buffer", shape=[5], dtype="int32" + ) + index_buffer = paddle.static.data( + name="index_buffer", shape=[5], dtype="int32" + ) + + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + ) = paddle.geometric.reindex_heter_graph( + x, [neighbors1, neighbors2], [count1, count2] + ) + ( + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = paddle.geometric.reindex_heter_graph( + x, + [neighbors1, neighbors2], + [count1, count2], + value_buffer, + index_buffer, + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'x': np_x, - 'neighbors1': np_neighbors1, - 'count1': np_count1, - 'neighbors2': np_neighbors2, - 'count2': np_count2, - 'value_buffer': np.full([5], -1, dtype="int32"), - 'index_buffer': np.full([5], -1, dtype="int32") - }, - fetch_list=[ - reindex_src_1, reindex_dst_1, out_nodes_1, - reindex_src_2, reindex_dst_2, out_nodes_2 - ]) - - reindex_src_1, reindex_dst_1, out_nodes_1, reindex_src_2, \ - reindex_dst_2, out_nodes_2 = ret + ret = exe.run( + feed={ + 'x': np_x, + 'neighbors1': np_neighbors1, + 'count1': np_count1, + 'neighbors2': np_neighbors2, + 'count2': np_count2, + 'value_buffer': np.full([5], -1, dtype="int32"), + 'index_buffer': np.full([5], -1, dtype="int32"), + }, + fetch_list=[ + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ], + ) + + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = ret np.testing.assert_allclose(reindex_src, reindex_src_1, rtol=1e-05) np.testing.assert_allclose(reindex_dst, reindex_dst_1, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_1, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_graph_sample_neighbors.py b/python/paddle/fluid/tests/unittests/test_graph_sample_neighbors.py index 557d41993b8..20366b80e2a 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_sample_neighbors.py +++ b/python/paddle/fluid/tests/unittests/test_graph_sample_neighbors.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class TestGraphSampleNeighbors(unittest.TestCase): - def setUp(self): num_nodes = 20 edges = np.random.randint(num_nodes, size=(100, 2)) @@ -40,8 +39,9 @@ class TestGraphSampleNeighbors(unittest.TestCase): self.row = sorted_edges[:, 0].astype("int64") self.colptr = colptr.astype("int64") - self.nodes = np.unique(np.random.randint(num_nodes, - size=5)).astype("int64") + self.nodes = np.unique(np.random.randint(num_nodes, size=5)).astype( + "int64" + ) self.sample_size = 5 self.dst_src_dict = dst_src_dict @@ -52,24 +52,29 @@ class TestGraphSampleNeighbors(unittest.TestCase): nodes = paddle.to_tensor(self.nodes) out_neighbors, out_count = paddle.incubate.graph_sample_neighbors( - row, colptr, nodes, sample_size=self.sample_size) + row, colptr, nodes, sample_size=self.sample_size + ) out_count_cumsum = paddle.cumsum(out_count) for i in range(len(out_count)): if i == 0: - neighbors = out_neighbors[0:out_count_cumsum[i]] + neighbors = out_neighbors[0 : out_count_cumsum[i]] else: neighbors = out_neighbors[ - out_count_cumsum[i - 1]:out_count_cumsum[i]] + out_count_cumsum[i - 1] : out_count_cumsum[i] + ] # Ensure the correct sample size. self.assertTrue( out_count[i] == self.sample_size - or out_count[i] == len(self.dst_src_dict[self.nodes[i]])) + or out_count[i] == len(self.dst_src_dict[self.nodes[i]]) + ) # Ensure no repetitive sample neighbors. self.assertTrue( - neighbors.shape[0] == paddle.unique(neighbors).shape[0]) + neighbors.shape[0] == paddle.unique(neighbors).shape[0] + ) # Ensure the correct sample neighbors. - in_neighbors = np.isin(neighbors.numpy(), - self.dst_src_dict[self.nodes[i]]) + in_neighbors = np.isin( + neighbors.numpy(), self.dst_src_dict[self.nodes[i]] + ) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_fisher_yates_sampling(self): @@ -86,57 +91,69 @@ class TestGraphSampleNeighbors(unittest.TestCase): nodes, perm_buffer=perm_buffer, sample_size=self.sample_size, - flag_perm_buffer=True) + flag_perm_buffer=True, + ) out_count_cumsum = paddle.cumsum(out_count) for i in range(len(out_count)): if i == 0: - neighbors = out_neighbors[0:out_count_cumsum[i]] + neighbors = out_neighbors[0 : out_count_cumsum[i]] else: neighbors = out_neighbors[ - out_count_cumsum[i - 1]:out_count_cumsum[i]] + out_count_cumsum[i - 1] : out_count_cumsum[i] + ] # Ensure the correct sample size. self.assertTrue( out_count[i] == self.sample_size - or out_count[i] == len(self.dst_src_dict[self.nodes[i]])) + or out_count[i] == len(self.dst_src_dict[self.nodes[i]]) + ) # Ensure no repetitive sample neighbors. self.assertTrue( - neighbors.shape[0] == paddle.unique(neighbors).shape[0]) + neighbors.shape[0] == paddle.unique(neighbors).shape[0] + ) # Ensure the correct sample neighbors. - in_neighbors = np.isin(neighbors.numpy(), - self.dst_src_dict[self.nodes[i]]) + in_neighbors = np.isin( + neighbors.numpy(), self.dst_src_dict[self.nodes[i]] + ) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) out_neighbors, out_count = paddle.incubate.graph_sample_neighbors( - row, colptr, nodes, sample_size=self.sample_size) + row, colptr, nodes, sample_size=self.sample_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes - }, - fetch_list=[out_neighbors, out_count]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + }, + fetch_list=[out_neighbors, out_count], + ) out_neighbors, out_count = ret out_count_cumsum = np.cumsum(out_count) out_neighbors = np.split(out_neighbors, out_count_cumsum)[:-1] - for neighbors, node, count in zip(out_neighbors, self.nodes, - out_count): - self.assertTrue(count == self.sample_size - or count == len(self.dst_src_dict[node])) + for neighbors, node, count in zip( + out_neighbors, self.nodes, out_count + ): + self.assertTrue( + count == self.sample_size + or count == len(self.dst_src_dict[node]) + ) self.assertTrue( - neighbors.shape[0] == np.unique(neighbors).shape[0]) + neighbors.shape[0] == np.unique(neighbors).shape[0] + ) in_neighbors = np.isin(neighbors, self.dst_src_dict[node]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) @@ -147,18 +164,22 @@ class TestGraphSampleNeighbors(unittest.TestCase): nodes = paddle.to_tensor(self.nodes) def check_eid_error(): - paddle.incubate.graph_sample_neighbors(row, - colptr, - nodes, - sample_size=self.sample_size, - return_eids=True) + paddle.incubate.graph_sample_neighbors( + row, + colptr, + nodes, + sample_size=self.sample_size, + return_eids=True, + ) def check_perm_buffer_error(): - paddle.incubate.graph_sample_neighbors(row, - colptr, - nodes, - sample_size=self.sample_size, - flag_perm_buffer=True) + paddle.incubate.graph_sample_neighbors( + row, + colptr, + nodes, + sample_size=self.sample_size, + flag_perm_buffer=True, + ) self.assertRaises(ValueError, check_eid_error) self.assertRaises(ValueError, check_perm_buffer_error) @@ -171,15 +192,24 @@ class TestGraphSampleNeighbors(unittest.TestCase): eids = paddle.to_tensor(self.edges_id) perm_buffer = paddle.to_tensor(self.edges_id) - out_neighbors, out_count, out_eids = paddle.incubate.graph_sample_neighbors( + ( + out_neighbors, + out_count, + out_eids, + ) = paddle.incubate.graph_sample_neighbors( row, colptr, nodes, eids=eids, sample_size=self.sample_size, - return_eids=True) + return_eids=True, + ) - out_neighbors, out_count, out_eids = paddle.incubate.graph_sample_neighbors( + ( + out_neighbors, + out_count, + out_eids, + ) = paddle.incubate.graph_sample_neighbors( row, colptr, nodes, @@ -187,42 +217,49 @@ class TestGraphSampleNeighbors(unittest.TestCase): perm_buffer=perm_buffer, sample_size=self.sample_size, return_eids=True, - flag_perm_buffer=True) + flag_perm_buffer=True, + ) paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) - eids = paddle.static.data(name="eids", - shape=self.edges_id.shape, - dtype=self.nodes.dtype) - - out_neighbors, out_count, out_eids = paddle.incubate.graph_sample_neighbors( + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) + eids = paddle.static.data( + name="eids", shape=self.edges_id.shape, dtype=self.nodes.dtype + ) + + ( + out_neighbors, + out_count, + out_eids, + ) = paddle.incubate.graph_sample_neighbors( row, colptr, nodes, eids, sample_size=self.sample_size, - return_eids=True) + return_eids=True, + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes, - 'eids': self.edges_id - }, - fetch_list=[out_neighbors, out_count, out_eids]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + 'eids': self.edges_id, + }, + fetch_list=[out_neighbors, out_count, out_eids], + ) class TestGeometricGraphSampleNeighbors(unittest.TestCase): - def setUp(self): num_nodes = 20 edges = np.random.randint(num_nodes, size=(100, 2)) @@ -243,8 +280,9 @@ class TestGeometricGraphSampleNeighbors(unittest.TestCase): self.row = sorted_edges[:, 0].astype("int64") self.colptr = colptr.astype("int64") - self.nodes = np.unique(np.random.randint(num_nodes, - size=5)).astype("int64") + self.nodes = np.unique(np.random.randint(num_nodes, size=5)).astype( + "int64" + ) self.sample_size = 5 self.dst_src_dict = dst_src_dict @@ -255,24 +293,29 @@ class TestGeometricGraphSampleNeighbors(unittest.TestCase): nodes = paddle.to_tensor(self.nodes) out_neighbors, out_count = paddle.geometric.sample_neighbors( - row, colptr, nodes, sample_size=self.sample_size) + row, colptr, nodes, sample_size=self.sample_size + ) out_count_cumsum = paddle.cumsum(out_count) for i in range(len(out_count)): if i == 0: - neighbors = out_neighbors[0:out_count_cumsum[i]] + neighbors = out_neighbors[0 : out_count_cumsum[i]] else: neighbors = out_neighbors[ - out_count_cumsum[i - 1]:out_count_cumsum[i]] + out_count_cumsum[i - 1] : out_count_cumsum[i] + ] # Ensure the correct sample size. self.assertTrue( out_count[i] == self.sample_size - or out_count[i] == len(self.dst_src_dict[self.nodes[i]])) + or out_count[i] == len(self.dst_src_dict[self.nodes[i]]) + ) # Ensure no repetitive sample neighbors. self.assertTrue( - neighbors.shape[0] == paddle.unique(neighbors).shape[0]) + neighbors.shape[0] == paddle.unique(neighbors).shape[0] + ) # Ensure the correct sample neighbors. - in_neighbors = np.isin(neighbors.numpy(), - self.dst_src_dict[self.nodes[i]]) + in_neighbors = np.isin( + neighbors.numpy(), self.dst_src_dict[self.nodes[i]] + ) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_fisher_yates_sampling(self): @@ -288,57 +331,69 @@ class TestGeometricGraphSampleNeighbors(unittest.TestCase): colptr, nodes, perm_buffer=perm_buffer, - sample_size=self.sample_size) + sample_size=self.sample_size, + ) out_count_cumsum = paddle.cumsum(out_count) for i in range(len(out_count)): if i == 0: - neighbors = out_neighbors[0:out_count_cumsum[i]] + neighbors = out_neighbors[0 : out_count_cumsum[i]] else: neighbors = out_neighbors[ - out_count_cumsum[i - 1]:out_count_cumsum[i]] + out_count_cumsum[i - 1] : out_count_cumsum[i] + ] # Ensure the correct sample size. self.assertTrue( out_count[i] == self.sample_size - or out_count[i] == len(self.dst_src_dict[self.nodes[i]])) + or out_count[i] == len(self.dst_src_dict[self.nodes[i]]) + ) # Ensure no repetitive sample neighbors. self.assertTrue( - neighbors.shape[0] == paddle.unique(neighbors).shape[0]) + neighbors.shape[0] == paddle.unique(neighbors).shape[0] + ) # Ensure the correct sample neighbors. - in_neighbors = np.isin(neighbors.numpy(), - self.dst_src_dict[self.nodes[i]]) + in_neighbors = np.isin( + neighbors.numpy(), self.dst_src_dict[self.nodes[i]] + ) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) out_neighbors, out_count = paddle.geometric.sample_neighbors( - row, colptr, nodes, sample_size=self.sample_size) + row, colptr, nodes, sample_size=self.sample_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes - }, - fetch_list=[out_neighbors, out_count]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + }, + fetch_list=[out_neighbors, out_count], + ) out_neighbors, out_count = ret out_count_cumsum = np.cumsum(out_count) out_neighbors = np.split(out_neighbors, out_count_cumsum)[:-1] - for neighbors, node, count in zip(out_neighbors, self.nodes, - out_count): - self.assertTrue(count == self.sample_size - or count == len(self.dst_src_dict[node])) + for neighbors, node, count in zip( + out_neighbors, self.nodes, out_count + ): self.assertTrue( - neighbors.shape[0] == np.unique(neighbors).shape[0]) + count == self.sample_size + or count == len(self.dst_src_dict[node]) + ) + self.assertTrue( + neighbors.shape[0] == np.unique(neighbors).shape[0] + ) in_neighbors = np.isin(neighbors, self.dst_src_dict[node]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) @@ -349,11 +404,13 @@ class TestGeometricGraphSampleNeighbors(unittest.TestCase): nodes = paddle.to_tensor(self.nodes) def check_eid_error(): - paddle.geometric.sample_neighbors(row, - colptr, - nodes, - sample_size=self.sample_size, - return_eids=True) + paddle.geometric.sample_neighbors( + row, + colptr, + nodes, + sample_size=self.sample_size, + return_eids=True, + ) self.assertRaises(ValueError, check_eid_error) @@ -371,7 +428,8 @@ class TestGeometricGraphSampleNeighbors(unittest.TestCase): nodes, eids=eids, sample_size=self.sample_size, - return_eids=True) + return_eids=True, + ) out_neighbors, out_count, out_eids = paddle.geometric.sample_neighbors( row, @@ -380,38 +438,46 @@ class TestGeometricGraphSampleNeighbors(unittest.TestCase): eids=eids, perm_buffer=perm_buffer, sample_size=self.sample_size, - return_eids=True) + return_eids=True, + ) paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) - eids = paddle.static.data(name="eids", - shape=self.edges_id.shape, - dtype=self.nodes.dtype) - - out_neighbors, out_count, out_eids = paddle.geometric.sample_neighbors( + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) + eids = paddle.static.data( + name="eids", shape=self.edges_id.shape, dtype=self.nodes.dtype + ) + + ( + out_neighbors, + out_count, + out_eids, + ) = paddle.geometric.sample_neighbors( row, colptr, nodes, sample_size=self.sample_size, eids=eids, - return_eids=True) + return_eids=True, + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes, - 'eids': self.edges_id - }, - fetch_list=[out_neighbors, out_count, out_eids]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + 'eids': self.edges_id, + }, + fetch_list=[out_neighbors, out_count, out_eids], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_graph_send_recv_op.py b/python/paddle/fluid/tests/unittests/test_graph_send_recv_op.py index 40d1673438f..a5344a96a31 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_send_recv_op.py +++ b/python/paddle/fluid/tests/unittests/test_graph_send_recv_op.py @@ -21,18 +21,15 @@ from paddle.fluid.framework import _test_eager_guard from op_test import OpTest -def graph_send_recv_wrapper(x, - src_index, - dst_index, - reduce_op="sum", - out_size=None, - name=None): - return paddle.geometric.send_u_recv(x, src_index, dst_index, - reduce_op.lower(), out_size, name) +def graph_send_recv_wrapper( + x, src_index, dst_index, reduce_op="sum", out_size=None, name=None +): + return paddle.geometric.send_u_recv( + x, src_index, dst_index, reduce_op.lower(), out_size, name + ) class TestGraphSendRecvMaxOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_recv_wrapper @@ -48,21 +45,20 @@ class TestGraphSendRecvMaxOp(OpTest): self.attrs = {'reduce_op': 'MAX'} out, self.gradient = compute_graph_send_recv_for_min_max( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out} def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.gradient], - check_eager=True) + self.check_grad( + ['X'], 'Out', user_defined_grads=[self.gradient], check_eager=True + ) class TestGraphSendRecvMinOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_recv_wrapper @@ -78,7 +74,8 @@ class TestGraphSendRecvMinOp(OpTest): self.attrs = {'reduce_op': 'MIN'} out, self.gradient = compute_graph_send_recv_for_min_max( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out} @@ -86,14 +83,12 @@ class TestGraphSendRecvMinOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.gradient], - check_eager=True) + self.check_grad( + ['X'], 'Out', user_defined_grads=[self.gradient], check_eager=True + ) class TestGraphSendRecvSumOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_recv_wrapper @@ -120,7 +115,6 @@ class TestGraphSendRecvSumOp(OpTest): class TestGraphSendRecvMeanOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_recv_wrapper @@ -136,7 +130,8 @@ class TestGraphSendRecvMeanOp(OpTest): self.attrs = {'reduce_op': 'MEAN'} out, dst_count = compute_graph_send_recv_for_sum_mean( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out, 'Dst_count': dst_count} @@ -197,8 +192,9 @@ def compute_graph_send_recv_for_min_max(inputs, attributes): results[s_id, :] += gather_x[index, :] first_set.add(s_id) else: - results[s_id, :] = np.maximum(results[s_id, :], - gather_x[index, :]) + results[s_id, :] = np.maximum( + results[s_id, :], gather_x[index, :] + ) elif reduce_op == "MIN": first_set = set() for index, s_id in enumerate(dst_index): @@ -206,8 +202,9 @@ def compute_graph_send_recv_for_min_max(inputs, attributes): results[s_id, :] += gather_x[index, :] first_set.add(s_id) else: - results[s_id, :] = np.minimum(results[s_id, :], - gather_x[index, :]) + results[s_id, :] = np.minimum( + results[s_id, :], gather_x[index, :] + ) else: raise ValueError("Invalid reduce_op, only MAX, MIN supported!") @@ -216,14 +213,14 @@ def compute_graph_send_recv_for_min_max(inputs, attributes): for i in range(index_size): forward_src_idx = src_index[i] forward_dst_idx = dst_index[i] - gradient[forward_src_idx] += 1 * (x[forward_src_idx] - == results[forward_dst_idx]) + gradient[forward_src_idx] += 1 * ( + x[forward_src_idx] == results[forward_dst_idx] + ) return results, gradient / results.size class API_GraphSendRecvOpTest(unittest.TestCase): - def test_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -231,53 +228,64 @@ class API_GraphSendRecvOpTest(unittest.TestCase): src_index = paddle.static.data(name="src", shape=[4], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[4], dtype="int32") - res_sum = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "sum") - res_mean = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "mean") - res_max = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "max") - res_min = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "min") + res_sum = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "sum" + ) + res_mean = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "mean" + ) + res_max = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "max" + ) + res_min = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "min" + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype='float32') data2 = np.array([0, 1, 2, 0], dtype="int32") data3 = np.array([1, 2, 1, 0], dtype="int32") - np_sum = np.array([[0, 2, 3], [2, 8, 10], [1, 4, 5]], - dtype="float32") - np_mean = np.array([[0, 2, 3], [1, 4, 5], [1, 4, 5]], - dtype="float32") - np_max = np.array([[0, 2, 3], [2, 6, 7], [1, 4, 5]], - dtype="float32") - np_min = np.array([[0, 2, 3], [0, 2, 3], [1, 4, 5]], - dtype="float32") - - ret = exe.run(feed={ - 'x': data1, - 'src': data2, - 'dst': data3 - }, - fetch_list=[res_sum, res_mean, res_max, res_min]) + np_sum = np.array( + [[0, 2, 3], [2, 8, 10], [1, 4, 5]], dtype="float32" + ) + np_mean = np.array( + [[0, 2, 3], [1, 4, 5], [1, 4, 5]], dtype="float32" + ) + np_max = np.array( + [[0, 2, 3], [2, 6, 7], [1, 4, 5]], dtype="float32" + ) + np_min = np.array( + [[0, 2, 3], [0, 2, 3], [1, 4, 5]], dtype="float32" + ) + + ret = exe.run( + feed={'x': data1, 'src': data2, 'dst': data3}, + fetch_list=[res_sum, res_mean, res_max, res_min], + ) for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): np.testing.assert_allclose(np_res, ret_res, rtol=1e-05, atol=1e-06) def test_dygraph(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_sum = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "sum") - res_mean = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "mean") - res_max = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "max") - res_min = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "min") + res_sum = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "sum" + ) + res_mean = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "mean" + ) + res_max = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "max" + ) + res_min = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "min" + ) np_sum = np.array([[0, 2, 3], [2, 8, 10], [1, 4, 5]], dtype="float32") np_mean = np.array([[0, 2, 3], [1, 4, 5], [1, 4, 5]], dtype="float32") @@ -291,18 +299,23 @@ class API_GraphSendRecvOpTest(unittest.TestCase): def test_int32_input(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), - dtype="int32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), dtype="int32" + ) src_index = paddle.to_tensor(np.array([0, 1, 2, 0, 1]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0, 1]), dtype="int32") - res_sum = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "sum") - res_mean = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "mean") - res_max = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "max") - res_min = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "min") + res_sum = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "sum" + ) + res_mean = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "mean" + ) + res_max = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "max" + ) + res_min = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "min" + ) np_sum = np.array([[0, 2, 3], [3, 12, 14], [1, 4, 5]], dtype="int32") np_mean = np.array([[0, 2, 3], [1, 4, 4], [1, 4, 5]], dtype="int32") @@ -316,23 +329,24 @@ class API_GraphSendRecvOpTest(unittest.TestCase): def test_set_outsize_gpu(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), dtype="float32" + ) src_index = paddle.to_tensor(np.array([0, 0, 1]), dtype="int32") dst_index = paddle.to_tensor(np.array([0, 1, 1]), dtype="int32") res = paddle.incubate.graph_send_recv(x, src_index, dst_index, "sum") out_size = paddle.max(dst_index) + 1 res_set_outsize = paddle.incubate.graph_send_recv( - x, src_index, dst_index, "sum", out_size) + x, src_index, dst_index, "sum", out_size + ) np_res = np.array([[0, 2, 3], [1, 6, 8], [0, 0, 0]], dtype="float32") np_res_set_outsize = np.array([[0, 2, 3], [1, 6, 8]], dtype="float32") np.testing.assert_allclose(np_res, res, rtol=1e-05, atol=1e-06) - np.testing.assert_allclose(np_res_set_outsize, - res_set_outsize, - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res_set_outsize, res_set_outsize, rtol=1e-05, atol=1e-06 + ) def test_out_size_tensor_static(self): paddle.enable_static() @@ -340,12 +354,13 @@ class API_GraphSendRecvOpTest(unittest.TestCase): x = paddle.static.data(name="x", shape=[3, 3], dtype="float32") src_index = paddle.static.data(name="src", shape=[3], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[3], dtype="int32") - out_size = paddle.static.data(name="out_size", - shape=[1], - dtype="int32") + out_size = paddle.static.data( + name="out_size", shape=[1], dtype="int32" + ) - res_sum = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "sum", out_size) + res_sum = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "sum", out_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]], dtype='float32') @@ -355,13 +370,15 @@ class API_GraphSendRecvOpTest(unittest.TestCase): np_sum = np.array([[0, 2, 3], [1, 6, 8]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'src': data2, - 'dst': data3, - 'out_size': data4, - }, - fetch_list=[res_sum]) + ret = exe.run( + feed={ + 'x': data1, + 'src': data2, + 'dst': data3, + 'out_size': data4, + }, + fetch_list=[res_sum], + ) np.testing.assert_allclose(np_sum, ret[0], rtol=1e-05, atol=1e-06) def test_api_eager_dygraph(self): @@ -372,7 +389,6 @@ class API_GraphSendRecvOpTest(unittest.TestCase): class API_GeometricSendURecvTest(unittest.TestCase): - def test_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -380,43 +396,50 @@ class API_GeometricSendURecvTest(unittest.TestCase): src_index = paddle.static.data(name="src", shape=[4], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[4], dtype="int32") - res_sum = paddle.geometric.send_u_recv(x, src_index, dst_index, - "sum") - res_mean = paddle.geometric.send_u_recv(x, src_index, dst_index, - "mean") - res_max = paddle.geometric.send_u_recv(x, src_index, dst_index, - "max") - res_min = paddle.geometric.send_u_recv(x, src_index, dst_index, - "min") + res_sum = paddle.geometric.send_u_recv( + x, src_index, dst_index, "sum" + ) + res_mean = paddle.geometric.send_u_recv( + x, src_index, dst_index, "mean" + ) + res_max = paddle.geometric.send_u_recv( + x, src_index, dst_index, "max" + ) + res_min = paddle.geometric.send_u_recv( + x, src_index, dst_index, "min" + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype='float32') data2 = np.array([0, 1, 2, 0], dtype="int32") data3 = np.array([1, 2, 1, 0], dtype="int32") - np_sum = np.array([[0, 2, 3], [2, 8, 10], [1, 4, 5]], - dtype="float32") - np_mean = np.array([[0, 2, 3], [1, 4, 5], [1, 4, 5]], - dtype="float32") - np_max = np.array([[0, 2, 3], [2, 6, 7], [1, 4, 5]], - dtype="float32") - np_min = np.array([[0, 2, 3], [0, 2, 3], [1, 4, 5]], - dtype="float32") - - ret = exe.run(feed={ - 'x': data1, - 'src': data2, - 'dst': data3 - }, - fetch_list=[res_sum, res_mean, res_max, res_min]) + np_sum = np.array( + [[0, 2, 3], [2, 8, 10], [1, 4, 5]], dtype="float32" + ) + np_mean = np.array( + [[0, 2, 3], [1, 4, 5], [1, 4, 5]], dtype="float32" + ) + np_max = np.array( + [[0, 2, 3], [2, 6, 7], [1, 4, 5]], dtype="float32" + ) + np_min = np.array( + [[0, 2, 3], [0, 2, 3], [1, 4, 5]], dtype="float32" + ) + + ret = exe.run( + feed={'x': data1, 'src': data2, 'dst': data3}, + fetch_list=[res_sum, res_mean, res_max, res_min], + ) for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): np.testing.assert_allclose(np_res, ret_res, rtol=1e-05, atol=1e-06) def test_dygraph(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") res_sum = paddle.geometric.send_u_recv(x, src_index, dst_index, "sum") @@ -436,8 +459,9 @@ class API_GeometricSendURecvTest(unittest.TestCase): def test_int32_input(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), - dtype="int32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), dtype="int32" + ) src_index = paddle.to_tensor(np.array([0, 1, 2, 0, 1]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0, 1]), dtype="int32") res_sum = paddle.geometric.send_u_recv(x, src_index, dst_index, "sum") @@ -457,23 +481,24 @@ class API_GeometricSendURecvTest(unittest.TestCase): def test_set_outsize_gpu(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), dtype="float32" + ) src_index = paddle.to_tensor(np.array([0, 0, 1]), dtype="int32") dst_index = paddle.to_tensor(np.array([0, 1, 1]), dtype="int32") res = paddle.geometric.send_u_recv(x, src_index, dst_index, "sum") out_size = paddle.max(dst_index) + 1 - res_set_outsize = paddle.geometric.send_u_recv(x, src_index, dst_index, - "sum", out_size) + res_set_outsize = paddle.geometric.send_u_recv( + x, src_index, dst_index, "sum", out_size + ) np_res = np.array([[0, 2, 3], [1, 6, 8], [0, 0, 0]], dtype="float32") np_res_set_outsize = np.array([[0, 2, 3], [1, 6, 8]], dtype="float32") np.testing.assert_allclose(np_res, res, rtol=1e-05, atol=1e-06) - np.testing.assert_allclose(np_res_set_outsize, - res_set_outsize, - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res_set_outsize, res_set_outsize, rtol=1e-05, atol=1e-06 + ) def test_out_size_tensor_static(self): paddle.enable_static() @@ -481,12 +506,13 @@ class API_GeometricSendURecvTest(unittest.TestCase): x = paddle.static.data(name="x", shape=[3, 3], dtype="float32") src_index = paddle.static.data(name="src", shape=[3], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[3], dtype="int32") - out_size = paddle.static.data(name="out_size", - shape=[1], - dtype="int32") + out_size = paddle.static.data( + name="out_size", shape=[1], dtype="int32" + ) - res_sum = paddle.geometric.send_u_recv(x, src_index, dst_index, - "sum", out_size) + res_sum = paddle.geometric.send_u_recv( + x, src_index, dst_index, "sum", out_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]], dtype='float32') @@ -496,13 +522,15 @@ class API_GeometricSendURecvTest(unittest.TestCase): np_sum = np.array([[0, 2, 3], [1, 6, 8]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'src': data2, - 'dst': data3, - 'out_size': data4, - }, - fetch_list=[res_sum]) + ret = exe.run( + feed={ + 'x': data1, + 'src': data2, + 'dst': data3, + 'out_size': data4, + }, + fetch_list=[res_sum], + ) np.testing.assert_allclose(np_sum, ret[0], rtol=1e-05, atol=1e-06) def test_api_eager_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_graph_send_ue_recv_op.py b/python/paddle/fluid/tests/unittests/test_graph_send_ue_recv_op.py index bf74885ba5a..c3bb651a3de 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_send_ue_recv_op.py +++ b/python/paddle/fluid/tests/unittests/test_graph_send_ue_recv_op.py @@ -40,7 +40,6 @@ def get_broadcast_shape(shp1, shp2): class BroadCastInfo(object): - def __init__(self, x_shape, y_shape): self.x_shape = x_shape self.y_shape = y_shape @@ -71,16 +70,24 @@ class BroadCastInfo(object): lhs_offset = [0] rhs_offset = [0] for j in range(0, max_ndim): - dl = 1 if (len(self.x_shape) - 1 - j) < 1 \ - else self.x_shape[len(self.x_shape) - 1 - j] - dr = 1 if (len(self.y_shape) - 1 - j) < 1 \ - else self.y_shape[len(self.y_shape) - 1 - j] + dl = ( + 1 + if (len(self.x_shape) - 1 - j) < 1 + else self.x_shape[len(self.x_shape) - 1 - j] + ) + dr = ( + 1 + if (len(self.y_shape) - 1 - j) < 1 + else self.y_shape[len(self.y_shape) - 1 - j] + ) for i in range(1, max(dl, dr)): for k in range(0, out_len): - lhs_offset.append(lhs_offset[k] + i * - (i < dl) * stride_l) - rhs_offset.append(rhs_offset[k] + i * - (i < dr) * stride_r) + lhs_offset.append( + lhs_offset[k] + i * (i < dl) * stride_l + ) + rhs_offset.append( + rhs_offset[k] + i * (i < dr) * stride_r + ) out_len *= max(dl, dr) stride_l *= dl @@ -176,16 +183,18 @@ def compute_graph_send_ue_recv_for_max_min(inputs, attributes): results[s_id, :] += x_compute_y[index, :] first_set.add(s_id) else: - results[s_id, :] = np.maximum(results[s_id, :], - x_compute_y[index, :]) + results[s_id, :] = np.maximum( + results[s_id, :], x_compute_y[index, :] + ) elif reduce_op == 'MIN': for index, s_id in enumerate(dst_index): if s_id not in first_set: results[s_id, :] += x_compute_y[index, :] first_set.add(s_id) else: - results[s_id, :] = np.minimum(results[s_id, :], - x_compute_y[index, :]) + results[s_id, :] = np.minimum( + results[s_id, :], x_compute_y[index, :] + ) else: raise ValueError("Invalid reduce_op, only MAX, MIN supported!") @@ -220,9 +229,11 @@ def compute_graph_send_ue_recv_for_max_min(inputs, attributes): out_add_1 = int(j % out_off.shape[1]) val = x_off[x_add_0][x_add_1] + y_off[y_add_0][y_add_1] x_grad_off[x_add_0][x_add_1] += 1 * ( - val == out_off[out_add_0][out_add_1]) + val == out_off[out_add_0][out_add_1] + ) y_grad_off[y_add_0][y_add_1] += 1 * ( - val == out_off[out_add_0][out_add_1]) + val == out_off[out_add_0][out_add_1] + ) elif message_op == 'MUL': if len(x_off.shape) == 1 and len(y_off.shape) == 1: val = x_off[x_add] * y_off[y_add] @@ -237,33 +248,45 @@ def compute_graph_send_ue_recv_for_max_min(inputs, attributes): out_add_0 = int(j / out_off.shape[1]) out_add_1 = int(j % out_off.shape[1]) val = x_off[x_add_0][x_add_1] * y_off[y_add_0][y_add_1] - x_grad_off[x_add_0][x_add_1] += 1 * ( - val == out_off[out_add_0][out_add_1] - ) * y_off[y_add_0][y_add_1] - y_grad_off[y_add_0][y_add_1] += 1 * ( - val == out_off[out_add_0][out_add_1] - ) * x_off[x_add_0][x_add_1] + x_grad_off[x_add_0][x_add_1] += ( + 1 + * (val == out_off[out_add_0][out_add_1]) + * y_off[y_add_0][y_add_1] + ) + y_grad_off[y_add_0][y_add_1] += ( + 1 + * (val == out_off[out_add_0][out_add_1]) + * x_off[x_add_0][x_add_1] + ) gradients = [x_gradient / results.size, y_gradient / results.size] return results, gradients -def graph_send_ue_recv_wrapper(x, - y, - src_index, - dst_index, - message_op="add", - reduce_op="sum", - out_size=None, - name=None): - return paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - message_op.lower(), reduce_op.lower(), - out_size, name) +def graph_send_ue_recv_wrapper( + x, + y, + src_index, + dst_index, + message_op="add", + reduce_op="sum", + out_size=None, + name=None, +): + return paddle.geometric.send_ue_recv( + x, + y, + src_index, + dst_index, + message_op.lower(), + reduce_op.lower(), + out_size, + name, + ) class TestGraphSendUERecvSumOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_ue_recv_wrapper @@ -274,7 +297,7 @@ class TestGraphSendUERecvSumOp(OpTest): 'X': self.x, 'Y': self.y, 'Src_index': self.src_index, - 'Dst_index': self.dst_index + 'Dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op, 'reduce_op': 'SUM'} @@ -298,7 +321,6 @@ class TestGraphSendUERecvSumOp(OpTest): class TestSumCase1(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -309,7 +331,6 @@ class TestSumCase1(TestGraphSendUERecvSumOp): class TestSumCase2(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -320,7 +341,6 @@ class TestSumCase2(TestGraphSendUERecvSumOp): class TestSumCase3(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -331,7 +351,6 @@ class TestSumCase3(TestGraphSendUERecvSumOp): class TestSumCase4(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -342,7 +361,6 @@ class TestSumCase4(TestGraphSendUERecvSumOp): class TestSumCase5(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -353,7 +371,6 @@ class TestSumCase5(TestGraphSendUERecvSumOp): class TestSumCase6(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -364,7 +381,6 @@ class TestSumCase6(TestGraphSendUERecvSumOp): class TestSumCase7(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -375,7 +391,6 @@ class TestSumCase7(TestGraphSendUERecvSumOp): class TestGraphSendUERecvMeanOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_ue_recv_wrapper @@ -386,12 +401,13 @@ class TestGraphSendUERecvMeanOp(OpTest): 'X': self.x, 'Y': self.y, 'Src_index': self.src_index, - 'Dst_index': self.dst_index + 'Dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op, 'reduce_op': 'MEAN'} out, dst_count = compute_graph_send_ue_recv_for_mean( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out, 'Dst_count': dst_count} @@ -411,7 +427,6 @@ class TestGraphSendUERecvMeanOp(OpTest): class TestMeanCase1(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -422,7 +437,6 @@ class TestMeanCase1(TestGraphSendUERecvMeanOp): class TestMeanCase2(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -433,7 +447,6 @@ class TestMeanCase2(TestGraphSendUERecvMeanOp): class TestMeanCase3(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -444,7 +457,6 @@ class TestMeanCase3(TestGraphSendUERecvMeanOp): class TestMeanCase4(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -455,7 +467,6 @@ class TestMeanCase4(TestGraphSendUERecvMeanOp): class TestMeanCase5(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -466,7 +477,6 @@ class TestMeanCase5(TestGraphSendUERecvMeanOp): class TestMeanCase6(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -477,7 +487,6 @@ class TestMeanCase6(TestGraphSendUERecvMeanOp): class TestMeanCase7(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -488,7 +497,6 @@ class TestMeanCase7(TestGraphSendUERecvMeanOp): class TestGraphSendUERecvMaxOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_ue_recv_wrapper @@ -499,12 +507,13 @@ class TestGraphSendUERecvMaxOp(OpTest): 'X': self.x, 'Y': self.y, 'Src_index': self.src_index, - 'Dst_index': self.dst_index + 'Dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op, 'reduce_op': 'MAX'} out, self.gradients = compute_graph_send_ue_recv_for_max_min( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out} @@ -520,14 +529,15 @@ class TestGraphSendUERecvMaxOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=self.gradients, - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=self.gradients, + check_eager=True, + ) class TestMaxCase1(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -538,7 +548,6 @@ class TestMaxCase1(TestGraphSendUERecvMaxOp): class TestMaxCase2(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -549,7 +558,6 @@ class TestMaxCase2(TestGraphSendUERecvMaxOp): class TestMaxCase3(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -560,7 +568,6 @@ class TestMaxCase3(TestGraphSendUERecvMaxOp): class TestMaxCase4(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -571,7 +578,6 @@ class TestMaxCase4(TestGraphSendUERecvMaxOp): class TestMaxCase5(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -582,7 +588,6 @@ class TestMaxCase5(TestGraphSendUERecvMaxOp): class TestMaxCase6(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -593,7 +598,6 @@ class TestMaxCase6(TestGraphSendUERecvMaxOp): class TestMaxCase7(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -604,7 +608,6 @@ class TestMaxCase7(TestGraphSendUERecvMaxOp): class TestGraphSendUERecvMinOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_ue_recv_wrapper @@ -615,12 +618,13 @@ class TestGraphSendUERecvMinOp(OpTest): 'X': self.x, 'Y': self.y, 'Src_index': self.src_index, - 'Dst_index': self.dst_index + 'Dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op, 'reduce_op': 'MIN'} out, self.gradients = compute_graph_send_ue_recv_for_max_min( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out} @@ -636,14 +640,15 @@ class TestGraphSendUERecvMinOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=self.gradients, - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=self.gradients, + check_eager=True, + ) class TestMinCase1(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -654,7 +659,6 @@ class TestMinCase1(TestGraphSendUERecvMinOp): class TestMinCase2(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -665,7 +669,6 @@ class TestMinCase2(TestGraphSendUERecvMinOp): class TestMinCase3(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -676,7 +679,6 @@ class TestMinCase3(TestGraphSendUERecvMinOp): class TestMinCase4(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -687,7 +689,6 @@ class TestMinCase4(TestGraphSendUERecvMinOp): class TestMinCase5(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -698,7 +699,6 @@ class TestMinCase5(TestGraphSendUERecvMinOp): class TestMinCase6(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -709,7 +709,6 @@ class TestMinCase6(TestGraphSendUERecvMinOp): class TestMinCase7(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -720,23 +719,27 @@ class TestMinCase7(TestGraphSendUERecvMinOp): class API_GeometricSendUERecvTest(unittest.TestCase): - def test_compute_all_with_sum(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) y = paddle.ones(shape=[4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "sum") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "sub", "sum") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "sum") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "div", "sum") + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "sum" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "sum" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "sum" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "sum" + ) res = [res_add, res_sub, res_mul, res_div] np_add = np.array([[1, 3, 4], [4, 10, 12], [2, 5, 6]], dtype="float32") @@ -750,25 +753,32 @@ class API_GeometricSendUERecvTest(unittest.TestCase): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_mean(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) y = paddle.ones(shape=[4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "mean") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "sub", "mean") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "mean") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "div", "mean") + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "mean" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "mean" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "mean" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "mean" + ) res = [res_add, res_sub, res_mul, res_div] np_add = np.array([[1, 3, 4], [2, 5, 6], [2, 5, 6]], dtype="float32") @@ -782,25 +792,32 @@ class API_GeometricSendUERecvTest(unittest.TestCase): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_max(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) y = paddle.ones(shape=[4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "max") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "sub", "max") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "max") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "div", "max") + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "max" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "max" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "max" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "max" + ) res = [res_add, res_sub, res_mul, res_div] np_add = np.array([[1, 3, 4], [3, 7, 8], [2, 5, 6]], dtype="float32") @@ -815,73 +832,91 @@ class API_GeometricSendUERecvTest(unittest.TestCase): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_max_fp16(self): paddle.disable_static() if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, - 7]]), - dtype="float16") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float16" + ) y = paddle.ones(shape=[4, 1], dtype="float16") - src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), - dtype="int32") - dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), - dtype="int32") - - res_add = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "add", "max") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "sub", "max") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "mul", "max") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "div", "max") + src_index = paddle.to_tensor( + np.array([0, 1, 2, 0]), dtype="int32" + ) + dst_index = paddle.to_tensor( + np.array([1, 2, 1, 0]), dtype="int32" + ) + + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "max" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "max" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "max" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "max" + ) res = [res_add, res_sub, res_mul, res_div] - np_add = np.array([[1, 3, 4], [3, 7, 8], [2, 5, 6]], - dtype="float16") - np_sub = np.array([[-1, 1, 2], [1, 5, 6], [0, 3, 4]], - dtype="float16") - np_mul = np.array([[0, 2, 3], [2, 6, 7], [1, 4, 5]], - dtype="float16") - np_div = np.array([[0, 2, 3], [2, 6, 7], [1, 4, 5]], - dtype="float16") - - np.testing.assert_allclose(np_sub, - res_sub, - rtol=1e-05, - atol=1e-06) - for np_res, paddle_res in zip([np_add, np_sub, np_mul, np_div], - res): + np_add = np.array( + [[1, 3, 4], [3, 7, 8], [2, 5, 6]], dtype="float16" + ) + np_sub = np.array( + [[-1, 1, 2], [1, 5, 6], [0, 3, 4]], dtype="float16" + ) + np_mul = np.array( + [[0, 2, 3], [2, 6, 7], [1, 4, 5]], dtype="float16" + ) + np_div = np.array( + [[0, 2, 3], [2, 6, 7], [1, 4, 5]], dtype="float16" + ) + + np.testing.assert_allclose( + np_sub, res_sub, rtol=1e-05, atol=1e-06 + ) + for np_res, paddle_res in zip( + [np_add, np_sub, np_mul, np_div], res + ): np.testing.assert_allclose( np_res, paddle_res, rtol=1e-05, atol=1e-06, - err_msg= - 'two value is {}\n{}, check diff!' - .format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_min(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) y = paddle.ones(shape=[4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "min") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "sub", "min") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "min") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "div", "min") + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "min" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "min" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "min" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "min" + ) res = [res_add, res_sub, res_mul, res_div] np_add = np.array([[1, 3, 4], [1, 3, 4], [2, 5, 6]], dtype="float32") @@ -895,71 +930,90 @@ class API_GeometricSendUERecvTest(unittest.TestCase): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_min_fp16(self): paddle.disable_static() if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, - 7]]), - dtype="float16") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float16" + ) y = paddle.ones(shape=[4, 1], dtype="float16") - src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), - dtype="int32") - dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), - dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "add", "min") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "sub", "min") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "mul", "min") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "div", "min") + src_index = paddle.to_tensor( + np.array([0, 1, 2, 0]), dtype="int32" + ) + dst_index = paddle.to_tensor( + np.array([1, 2, 1, 0]), dtype="int32" + ) + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "min" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "min" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "min" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "min" + ) res = [res_add, res_sub, res_mul, res_div] - np_add = np.array([[1, 3, 4], [1, 3, 4], [2, 5, 6]], - dtype="float16") - np_sub = np.array([[-1, 1, 2], [-1, 1, 2], [0, 3, 4]], - dtype="float16") - np_mul = np.array([[0, 2, 3], [0, 2, 3], [1, 4, 5]], - dtype="float16") - np_div = np.array([[0, 2, 3], [0, 2, 3], [1, 4, 5]], - dtype="float16") - - for np_res, paddle_res in zip([np_add, np_sub, np_mul, np_div], - res): + np_add = np.array( + [[1, 3, 4], [1, 3, 4], [2, 5, 6]], dtype="float16" + ) + np_sub = np.array( + [[-1, 1, 2], [-1, 1, 2], [0, 3, 4]], dtype="float16" + ) + np_mul = np.array( + [[0, 2, 3], [0, 2, 3], [1, 4, 5]], dtype="float16" + ) + np_div = np.array( + [[0, 2, 3], [0, 2, 3], [1, 4, 5]], dtype="float16" + ) + + for np_res, paddle_res in zip( + [np_add, np_sub, np_mul, np_div], res + ): np.testing.assert_allclose( np_res, paddle_res, rtol=1e-05, atol=1e-06, - err_msg= - 'two value is {}\n{}, check diff!' - .format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_reshape_lhs_rhs(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) x = x.reshape(shape=[3, 3, 1]) y = paddle.ones([4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "min") - np_add = np.array([[1, 3, 4], [1, 3, 4], [2, 5, 6]], - dtype="float16").reshape([3, 3, 1]) + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "min" + ) + np_add = np.array( + [[1, 3, 4], [1, 3, 4], [2, 5, 6]], dtype="float16" + ).reshape([3, 3, 1]) np.testing.assert_allclose( np_add, res_add, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_add, res_add)) + err_msg='two value is {}\n{}, check diff!'.format( + np_add, res_add + ), + ) def test_out_size_tensor_static(self): paddle.enable_static() @@ -968,12 +1022,13 @@ class API_GeometricSendUERecvTest(unittest.TestCase): y = paddle.static.data(name="y", shape=[3], dtype="float32") src_index = paddle.static.data(name="src", shape=[3], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[3], dtype="int32") - out_size = paddle.static.data(name="out_size", - shape=[1], - dtype="int32") + out_size = paddle.static.data( + name="out_size", shape=[1], dtype="int32" + ) - res_sum = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "sum", out_size) + res_sum = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "sum", out_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]], dtype="float32") @@ -984,21 +1039,25 @@ class API_GeometricSendUERecvTest(unittest.TestCase): np_sum = np.array([[0, 2, 3], [3, 16, 21]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'y': data2, - 'src': data3, - 'dst': data4, - 'out_size': data5, - }, - fetch_list=[res_sum]) + ret = exe.run( + feed={ + 'x': data1, + 'y': data2, + 'src': data3, + 'dst': data4, + 'out_size': data5, + }, + fetch_list=[res_sum], + ) np.testing.assert_allclose( np_sum, ret[0], rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_sum, ret[0])) + err_msg='two value is {}\n{}, check diff!'.format( + np_sum, ret[0] + ), + ) def test_api_eager_dygraph(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_graph_send_uv_op.py b/python/paddle/fluid/tests/unittests/test_graph_send_uv_op.py index df22555d357..9dd252b17c5 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_send_uv_op.py +++ b/python/paddle/fluid/tests/unittests/test_graph_send_uv_op.py @@ -40,12 +40,12 @@ def compute_graph_send_uv(inputs, attributes): def graph_send_uv_wrapper(x, y, src_index, dst_index, message_op="add"): - return paddle.geometric.send_uv(x, y, src_index, dst_index, - message_op.lower()) + return paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op.lower() + ) class TestGraphSendUVOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_uv_wrapper @@ -56,7 +56,7 @@ class TestGraphSendUVOp(OpTest): 'x': self.x, 'y': self.y, 'src_index': self.src_index, - 'dst_index': self.dst_index + 'dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op} out = compute_graph_send_uv(self.inputs, self.attrs) @@ -78,7 +78,6 @@ class TestGraphSendUVOp(OpTest): class TestCase1(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((10, 20)).astype("float64") @@ -89,7 +88,6 @@ class TestCase1(TestGraphSendUVOp): class TestCase2(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((100, 20)).astype("float64") @@ -100,7 +98,6 @@ class TestCase2(TestGraphSendUVOp): class TestCase3(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((100, 20)).astype("float64") self.y = np.random.random((100, 1)).astype("float64") @@ -111,7 +108,6 @@ class TestCase3(TestGraphSendUVOp): class TestCase4(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((100, 20)).astype("float64") @@ -122,7 +118,6 @@ class TestCase4(TestGraphSendUVOp): class TestCase5(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((100, 20)).astype("float64") self.y = np.random.random((100, 1)).astype("float64") @@ -133,7 +128,6 @@ class TestCase5(TestGraphSendUVOp): class TestCase6(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((10, 10, 1)).astype("float64") self.y = np.random.random((10, 10, 10)) @@ -144,7 +138,6 @@ class TestCase6(TestGraphSendUVOp): class TestCase7(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((10, 10, 1)).astype("float64") self.y = np.random.random((10, 10, 10)) @@ -155,7 +148,6 @@ class TestCase7(TestGraphSendUVOp): class API_GeometricSendUVTest(unittest.TestCase): - def test_compute_all_dygraph(self): paddle.disable_static() x = paddle.to_tensor([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype="float32") @@ -163,37 +155,33 @@ class API_GeometricSendUVTest(unittest.TestCase): src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="add") - res_sub = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="sub") - res_mul = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="mul") - res_div = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="div") + res_add = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="add" + ) + res_sub = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="sub" + ) + res_mul = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="mul" + ) + res_div = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="div" + ) res = [res_add, res_sub, res_mul, res_div] - np_add = np.array([[2, 5, 7], [5, 9, 11], [4, 9, 11], [1, 3, 5]], - dtype="float32") - np_sub = np.array([[-2, -1, -1], [-3, -1, -1], [0, 3, 3], [-1, 1, 1]], - dtype="float32") - np_mul = np.array([[0, 6, 12], [4, 20, 30], [4, 18, 28], [0, 2, 6]], - dtype="float32") + np_add = np.array( + [[2, 5, 7], [5, 9, 11], [4, 9, 11], [1, 3, 5]], dtype="float32" + ) + np_sub = np.array( + [[-2, -1, -1], [-3, -1, -1], [0, 3, 3], [-1, 1, 1]], dtype="float32" + ) + np_mul = np.array( + [[0, 6, 12], [4, 20, 30], [4, 18, 28], [0, 2, 6]], dtype="float32" + ) np_div = np.array( [[0, 2 / 3, 0.75], [0.25, 0.8, 5 / 6], [1, 2, 7 / 4], [0, 2, 1.5]], - dtype="float32") + dtype="float32", + ) for np_res, paddle_res in zip([np_add, np_sub, np_mul, np_div], res): np.testing.assert_allclose( @@ -201,8 +189,10 @@ class API_GeometricSendUVTest(unittest.TestCase): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_static(self): paddle.enable_static() @@ -211,26 +201,18 @@ class API_GeometricSendUVTest(unittest.TestCase): y = paddle.static.data(name="y", shape=[3, 3], dtype="float32") src_index = paddle.static.data(name="src", shape=[4], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[4], dtype="int32") - res_add = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="add") - res_sub = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="sub") - res_mul = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="mul") - res_div = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="div") + res_add = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="add" + ) + res_sub = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="sub" + ) + res_mul = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="mul" + ) + res_div = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="div" + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype="float32") @@ -238,33 +220,48 @@ class API_GeometricSendUVTest(unittest.TestCase): data3 = np.array([0, 1, 2, 0], dtype="int32") data4 = np.array([1, 2, 1, 0], dtype="int32") - np_add = np.array([[2, 5, 7], [5, 9, 11], [4, 9, 11], [1, 3, 5]], - dtype="float32") + np_add = np.array( + [[2, 5, 7], [5, 9, 11], [4, 9, 11], [1, 3, 5]], dtype="float32" + ) np_sub = np.array( [[-2, -1, -1], [-3, -1, -1], [0, 3, 3], [-1, 1, 1]], - dtype="float32") - np_mul = np.array([[0, 6, 12], [4, 20, 30], [4, 18, 28], [0, 2, 6]], - dtype="float32") - np_div = np.array([[0, 2 / 3, 0.75], [0.25, 0.8, 5 / 6], - [1, 2, 7 / 4], [0, 2, 1.5]], - dtype="float32") - - ret = exe.run(feed={ - 'x': data1, - 'y': data2, - 'src': data3, - 'dst': data4, - }, - fetch_list=[res_add, res_sub, res_mul, res_div]) - for np_res, paddle_res in zip([np_add, np_sub, np_mul, np_div], - ret): + dtype="float32", + ) + np_mul = np.array( + [[0, 6, 12], [4, 20, 30], [4, 18, 28], [0, 2, 6]], + dtype="float32", + ) + np_div = np.array( + [ + [0, 2 / 3, 0.75], + [0.25, 0.8, 5 / 6], + [1, 2, 7 / 4], + [0, 2, 1.5], + ], + dtype="float32", + ) + + ret = exe.run( + feed={ + 'x': data1, + 'y': data2, + 'src': data3, + 'dst': data4, + }, + fetch_list=[res_add, res_sub, res_mul, res_div], + ) + for np_res, paddle_res in zip( + [np_add, np_sub, np_mul, np_div], ret + ): np.testing.assert_allclose( np_res, paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!' - .format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_api_eager_dygraph(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_grid_sample_function.py b/python/paddle/fluid/tests/unittests/test_grid_sample_function.py index 7255f82bc07..75001091855 100644 --- a/python/paddle/fluid/tests/unittests/test_grid_sample_function.py +++ b/python/paddle/fluid/tests/unittests/test_grid_sample_function.py @@ -21,14 +21,15 @@ import unittest class GridSampleTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - x_shape=[2, 2, 3, 3], - grid_shape=[2, 3, 3, 2], - mode="bilinear", - padding_mode="zeros", - align_corners=False): + def __init__( + self, + methodName='runTest', + x_shape=[2, 2, 3, 3], + grid_shape=[2, 3, 3, 2], + mode="bilinear", + padding_mode="zeros", + align_corners=False, + ): super(GridSampleTestCase, self).__init__(methodName) self.padding_mode = padding_mode self.x_shape = x_shape @@ -49,25 +50,29 @@ class GridSampleTestCase(unittest.TestCase): with fluid.program_guard(main, start): x = fluid.data("x", self.x_shape, dtype=self.dtype) grid = fluid.data("grid", self.grid_shape, dtype=self.dtype) - y_var = F.grid_sample(x, - grid, - mode=self.mode, - padding_mode=self.padding_mode, - align_corners=self.align_corners) + y_var = F.grid_sample( + x, + grid, + mode=self.mode, + padding_mode=self.padding_mode, + align_corners=self.align_corners, + ) feed_dict = {"x": self.x, "grid": self.grid} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def dynamic_functional(self): x_t = paddle.to_tensor(self.x) grid_t = paddle.to_tensor(self.grid) - y_t = F.grid_sample(x_t, - grid_t, - mode=self.mode, - padding_mode=self.padding_mode, - align_corners=self.align_corners) + y_t = F.grid_sample( + x_t, + grid_t, + mode=self.mode, + padding_mode=self.padding_mode, + align_corners=self.align_corners, + ) y_np = y_t.numpy() return y_np @@ -87,7 +92,6 @@ class GridSampleTestCase(unittest.TestCase): class GridSampleErrorTestCase(GridSampleTestCase): - def runTest(self): place = fluid.CPUPlace() with self.assertRaises(ValueError): @@ -97,22 +101,30 @@ class GridSampleErrorTestCase(GridSampleTestCase): def add_cases(suite): suite.addTest(GridSampleTestCase(methodName='runTest')) suite.addTest( - GridSampleTestCase(methodName='runTest', - mode='bilinear', - padding_mode='reflection', - align_corners=True)) + GridSampleTestCase( + methodName='runTest', + mode='bilinear', + padding_mode='reflection', + align_corners=True, + ) + ) suite.addTest( - GridSampleTestCase(methodName='runTest', - mode='bilinear', - padding_mode='zeros', - align_corners=True)) + GridSampleTestCase( + methodName='runTest', + mode='bilinear', + padding_mode='zeros', + align_corners=True, + ) + ) def add_error_cases(suite): suite.addTest( - GridSampleErrorTestCase(methodName='runTest', padding_mode="VALID")) + GridSampleErrorTestCase(methodName='runTest', padding_mode="VALID") + ) suite.addTest( - GridSampleErrorTestCase(methodName='runTest', align_corners="VALID")) + GridSampleErrorTestCase(methodName='runTest', align_corners="VALID") + ) suite.addTest(GridSampleErrorTestCase(methodName='runTest', mode="VALID")) @@ -124,7 +136,6 @@ def load_tests(loader, standard_tests, pattern): class TestGridSampleAPI(unittest.TestCase): - def test_errors(self): with self.assertRaises(ValueError): x = paddle.randn([1, 1, 3, 3]) diff --git a/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py b/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py index c162da61528..0245383045b 100644 --- a/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py +++ b/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py @@ -25,12 +25,15 @@ def AffineGrid(theta, grid_shape): n = grid_shape[0] h = grid_shape[1] w = grid_shape[2] - h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, - axis=0).T[:, :, np.newaxis] - w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, - axis=0)[:, :, np.newaxis] - grid = np.concatenate([w_idx, h_idx, np.ones([h, w, 1])], - axis=2) # h * w * 3 + h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, axis=0).T[ + :, :, np.newaxis + ] + w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, axis=0)[ + :, :, np.newaxis + ] + grid = np.concatenate( + [w_idx, h_idx, np.ones([h, w, 1])], axis=2 + ) # h * w * 3 grid = np.repeat(grid[np.newaxis, :], n, axis=0) # n * h * w *3 ret = np.zeros([n, h * w, 2]) @@ -50,13 +53,17 @@ def getGridPointValue(data, x, y): out_H = x.shape[1] out_W = x.shape[2] - #out = np.zeros(data_shape, dtype='float64') + # out = np.zeros(data_shape, dtype='float64') out = np.zeros([N, C, out_H, out_W], dtype='float64') for i in range(N): for j in range(out_H): for k in range(out_W): - if y[i, j, k] < 0 or y[i, j, k] > in_H - 1 or x[ - i, j, k] < 0 or x[i, j, k] > in_W - 1: + if ( + y[i, j, k] < 0 + or y[i, j, k] > in_H - 1 + or x[i, j, k] < 0 + or x[i, j, k] > in_W - 1 + ): out[i, :, j, k] = 0 else: out[i, :, j, k] = data[i, :, y[i, j, k], x[i, j, k]] @@ -69,26 +76,24 @@ def AffineGrid3D(theta, grid_shape): d = grid_shape[1] h = grid_shape[2] w = grid_shape[3] - d_idx = np.repeat(np.repeat(np.linspace(-1, 1, d)[:, np.newaxis, - np.newaxis], - h, - axis=1), - w, - axis=2)[:, :, :, np.newaxis] - h_idx = np.repeat(np.repeat(np.linspace(-1, 1, h)[np.newaxis, :, - np.newaxis], - w, - axis=2), - d, - axis=0)[:, :, :, np.newaxis] - w_idx = np.repeat(np.repeat(np.linspace(-1, 1, w)[np.newaxis, - np.newaxis, :], - h, - axis=1), - d, - axis=0)[:, :, :, np.newaxis] + d_idx = np.repeat( + np.repeat(np.linspace(-1, 1, d)[:, np.newaxis, np.newaxis], h, axis=1), + w, + axis=2, + )[:, :, :, np.newaxis] + h_idx = np.repeat( + np.repeat(np.linspace(-1, 1, h)[np.newaxis, :, np.newaxis], w, axis=2), + d, + axis=0, + )[:, :, :, np.newaxis] + w_idx = np.repeat( + np.repeat(np.linspace(-1, 1, w)[np.newaxis, np.newaxis, :], h, axis=1), + d, + axis=0, + )[:, :, :, np.newaxis] grid = np.concatenate( - [w_idx, h_idx, d_idx, np.ones([d, h, w, 1])], axis=3) # d * h * w * 4 + [w_idx, h_idx, d_idx, np.ones([d, h, w, 1])], axis=3 + ) # d * h * w * 4 grid = np.repeat(grid[np.newaxis, :], n, axis=0) # n * d * h * w *4 ret = np.zeros([n, d * h * w, 3]) theta = theta.transpose([0, 2, 1]) @@ -114,13 +119,19 @@ def getGridPointValue3D(data, x, y, z): for j in range(out_D): for k in range(out_H): for l in range(out_W): - if y[i, j, k, l] < 0 or y[i, j, k, l] > in_H - 1 or x[ - i, j, k, l] < 0 or x[i, j, k, l] > in_W - 1 or z[ - i, j, k, l] < 0 or z[i, j, k, l] > in_D - 1: + if ( + y[i, j, k, l] < 0 + or y[i, j, k, l] > in_H - 1 + or x[i, j, k, l] < 0 + or x[i, j, k, l] > in_W - 1 + or z[i, j, k, l] < 0 + or z[i, j, k, l] > in_D - 1 + ): out[i, :, j, k, l] = 0 else: - out[i, :, j, k, l] = data[i, :, z[i, j, k, l], - y[i, j, k, l], x[i, j, k, l]] + out[i, :, j, k, l] = data[ + i, :, z[i, j, k, l], y[i, j, k, l], x[i, j, k, l] + ] return out @@ -133,27 +144,28 @@ def unnormalizeAndClip(grid_slice, max_val, align_corners, padding_mode): if align_corners: grid_slice = 0.5 * ((grid_slice.astype('float64') + 1.0) * max_val) else: - grid_slice = 0.5 * ((grid_slice.astype('float64') + 1.0) * - (max_val + 1)) - 0.5 + grid_slice = ( + 0.5 * ((grid_slice.astype('float64') + 1.0) * (max_val + 1)) - 0.5 + ) if padding_mode == "border": grid_slice = clip(grid_slice, 0, max_val) elif padding_mode == "reflection": double_range = 2 * max_val if align_corners else (max_val + 1) * 2 - grid_abs = np.abs(grid_slice) if align_corners else np.abs(grid_slice + - 0.5) + grid_abs = ( + np.abs(grid_slice) if align_corners else np.abs(grid_slice + 0.5) + ) extra = grid_abs - np.floor(grid_abs / double_range) * double_range grid_slice = np.minimum(extra, double_range - extra) - grid_slice = grid_slice if align_corners else clip( - grid_slice - 0.5, 0, max_val) + grid_slice = ( + grid_slice if align_corners else clip(grid_slice - 0.5, 0, max_val) + ) return grid_slice -def GridSampler(data, - grid, - align_corners=True, - mode="bilinear", - padding_mode="zeros"): +def GridSampler( + data, grid, align_corners=True, mode="bilinear", padding_mode="zeros" +): dims = data.shape N = dims[0] in_C = dims[1] @@ -177,14 +189,18 @@ def GridSampler(data, y0 = np.floor(y).astype('int32') y1 = y0 + 1 - wa = np.tile(((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wb = np.tile(((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wc = np.tile(((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wd = np.tile(((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) + wa = np.tile( + ((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wb = np.tile( + ((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wc = np.tile( + ((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wd = np.tile( + ((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) va = getGridPointValue(data, x0, y0) vb = getGridPointValue(data, x0, y1) @@ -199,11 +215,9 @@ def GridSampler(data, return out -def GridSampler3D(data, - grid, - align_corners=True, - mode="bilinear", - padding_mode="zeros"): +def GridSampler3D( + data, grid, align_corners=True, mode="bilinear", padding_mode="zeros" +): dims = data.shape N = dims[0] in_C = dims[1] @@ -235,22 +249,54 @@ def GridSampler3D(data, z0 = np.floor(z).astype('int32') z1 = z0 + 1 - w_tnw = np.tile(((x1 - x) * (y1 - y) * (z1 - z)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_tne = np.tile(((x - x0) * (y1 - y) * (z1 - z)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_tsw = np.tile(((x1 - x) * (y - y0) * (z1 - z)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_tse = np.tile(((x - x0) * (y - y0) * (z1 - z)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_bnw = np.tile(((x1 - x) * (y1 - y) * (z - z0)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_bne = np.tile(((x - x0) * (y1 - y) * (z - z0)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_bsw = np.tile(((x1 - x) * (y - y0) * (z - z0)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_bse = np.tile(((x - x0) * (y - y0) * (z - z0)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) + w_tnw = np.tile( + ((x1 - x) * (y1 - y) * (z1 - z)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_tne = np.tile( + ((x - x0) * (y1 - y) * (z1 - z)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_tsw = np.tile( + ((x1 - x) * (y - y0) * (z1 - z)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_tse = np.tile( + ((x - x0) * (y - y0) * (z1 - z)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_bnw = np.tile( + ((x1 - x) * (y1 - y) * (z - z0)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_bne = np.tile( + ((x - x0) * (y1 - y) * (z - z0)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_bsw = np.tile( + ((x1 - x) * (y - y0) * (z - z0)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_bse = np.tile( + ((x - x0) * (y - y0) * (z - z0)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) v_tnw = getGridPointValue3D(data, x0, y0, z0) v_tne = getGridPointValue3D(data, x1, y0, z0) @@ -261,9 +307,16 @@ def GridSampler3D(data, v_bsw = getGridPointValue3D(data, x0, y1, z1) v_bse = getGridPointValue3D(data, x1, y1, z1) - out = (w_tnw * v_tnw + w_tne * v_tne + w_tsw * v_tsw + w_tse * v_tse + - w_bnw * v_bnw + w_bne * v_bne + w_bsw * v_bsw + - w_bse * v_bse).astype('float64') + out = ( + w_tnw * v_tnw + + w_tne * v_tne + + w_tsw * v_tsw + + w_tse * v_tse + + w_bnw * v_bnw + + w_bne * v_bne + + w_bsw * v_bsw + + w_bse * v_bse + ).astype('float64') elif mode == "nearest": x = np.round(x).astype('int32') @@ -274,7 +327,6 @@ def GridSampler3D(data, class TestGridSamplerOp(OpTest): - def setUp(self): self.use_cudnn = False self.numeric_grad_delta = 0.0001 @@ -298,12 +350,12 @@ class TestGridSamplerOp(OpTest): 'use_cudnn': self.use_cudnn, "align_corners": self.align_corners, "padding_mode": self.padding_mode, - "mode": self.mode + "mode": self.mode, } self.outputs = { - 'Output': - GridSampler(x, grid, self.align_corners, self.mode, - self.padding_mode) + 'Output': GridSampler( + x, grid, self.align_corners, self.mode, self.padding_mode + ) } else: for i in range(self.theta_shape[0]): @@ -316,23 +368,25 @@ class TestGridSamplerOp(OpTest): 'use_cudnn': self.use_cudnn, "align_corners": self.align_corners, "padding_mode": self.padding_mode, - "mode": self.mode + "mode": self.mode, } self.outputs = { - 'Output': - GridSampler3D(x, grid, self.align_corners, self.mode, - self.padding_mode) + 'Output': GridSampler3D( + x, grid, self.align_corners, self.mode, self.padding_mode + ) } def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Grid'], - 'Output', - max_relative_error=0.01, - numeric_grad_delta=self.numeric_grad_delta, - check_eager=True) + self.check_grad( + ['X', 'Grid'], + 'Output', + max_relative_error=0.01, + numeric_grad_delta=self.numeric_grad_delta, + check_eager=True, + ) def initTestCase(self): self.x_shape = (2, 3, 8, 8) @@ -345,7 +399,6 @@ class TestGridSamplerOp(OpTest): class Case1(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -356,7 +409,6 @@ class Case1(TestGridSamplerOp): class Case1_(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -367,7 +419,6 @@ class Case1_(TestGridSamplerOp): class Case2(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -378,7 +429,6 @@ class Case2(TestGridSamplerOp): class Case3(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -389,7 +439,6 @@ class Case3(TestGridSamplerOp): class Case4(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -400,10 +449,11 @@ class Case4(TestGridSamplerOp): self.numeric_grad_delta = 0.0001 -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class LargeInputCase(TestGridSamplerOp): - def get_places(self): places = [] if core.is_compiled_with_cuda(): @@ -423,10 +473,11 @@ class LargeInputCase(TestGridSamplerOp): pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class Case5(LargeInputCase): - def initTestCase(self): self.no_need_check_grad = True self.x_shape = (2, 3, 128, 128) @@ -439,7 +490,6 @@ class Case5(LargeInputCase): class Case6(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6, 7) self.grid_shape = (2, 8, 9, 10, 3) @@ -451,7 +501,6 @@ class Case6(TestGridSamplerOp): class Case6_(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 4, 5, 6) self.grid_shape = (2, 7, 8, 9, 3) @@ -463,7 +512,6 @@ class Case6_(TestGridSamplerOp): class Case7(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 4, 5, 6) self.grid_shape = (2, 7, 8, 9, 3) @@ -475,7 +523,6 @@ class Case7(TestGridSamplerOp): class Case8(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 4, 5, 6) self.grid_shape = (2, 7, 8, 9, 3) @@ -487,7 +534,6 @@ class Case8(TestGridSamplerOp): class Case9(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 4, 5, 6) self.grid_shape = (2, 7, 8, 9, 3) @@ -498,10 +544,11 @@ class Case9(TestGridSamplerOp): self.numeric_grad_delta = 0.000001 -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class LargeInput3DCase(TestGridSamplerOp): - def get_places(self): places = [] if core.is_compiled_with_cuda(): @@ -523,10 +570,11 @@ class LargeInput3DCase(TestGridSamplerOp): pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class Case10(LargeInput3DCase): - def initTestCase(self): self.no_need_check_grad = True self.x_shape = (2, 3, 24, 24, 12) diff --git a/python/paddle/fluid/tests/unittests/test_group_norm_op.py b/python/paddle/fluid/tests/unittests/test_group_norm_op.py index 3ae2bf0a391..480efd5686e 100644 --- a/python/paddle/fluid/tests/unittests/test_group_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_group_norm_op.py @@ -32,14 +32,14 @@ def group_norm_naive(x, scale, bias, epsilon, groups, data_layout): var = np.var(x, axis=1, keepdims=True) output = (x - mean) / np.sqrt(var + epsilon) output = output.reshape((N, C, H, W)) * scale.reshape( - (-1, 1, 1)) + bias.reshape((-1, 1, 1)) + (-1, 1, 1) + ) + bias.reshape((-1, 1, 1)) if data_layout == "NHWC": output = np.transpose(output, (0, 2, 3, 1)) # NCHW => NHWC return output, mean.reshape((N, G)), var.reshape((N, G)) class TestGroupNormOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -51,9 +51,9 @@ class TestGroupNormOpError(unittest.TestCase): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[2, 100, 3, 5], - dtype='int32') + x2 = fluid.layers.data( + name='x2', shape=[2, 100, 3, 5], dtype='int32' + ) groups = 2 fluid.layers.group_norm(x2, groups) @@ -61,7 +61,6 @@ class TestGroupNormOpError(unittest.TestCase): class TestGroupNormOp(OpTest): - def setUp(self): self.op_type = "group_norm" self.data_format = "NCHW" @@ -76,15 +75,19 @@ class TestGroupNormOp(OpTest): input = np.transpose(input, (0, 2, 3, 1)) scale = np.random.random([self.shape[1]]).astype(self.dtype) bias = np.random.random([self.shape[1]]).astype(self.dtype) - output, mean, var = group_norm_naive(input, scale, bias, - self.attrs['epsilon'], - self.attrs['groups'], - self.data_format) + output, mean, var = group_norm_naive( + input, + scale, + bias, + self.attrs['epsilon'], + self.attrs['groups'], + self.data_format, + ) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(input), 'Scale': OpTest.np_dtype_to_fluid_dtype(scale), - 'Bias': OpTest.np_dtype_to_fluid_dtype(bias) + 'Bias': OpTest.np_dtype_to_fluid_dtype(bias), } self.outputs = {'Y': output, 'Mean': mean, 'Variance': var} self.attrs['data_layout'] = self.data_format @@ -106,28 +109,37 @@ class TestGroupNormOp(OpTest): # Set to inplace_atol to 0, which means the absolute error is 0, and the # relative error is 1e-05 in numpy.allclose by default. # Reference: https://docs.scipy.org/doc/numpy/reference/generated/numpy.allclose.html - self.check_output_with_place(place, - atol=atol, - inplace_atol=inplace_atol) + self.check_output_with_place( + place, atol=atol, inplace_atol=inplace_atol + ) def do_compare_between_place(self): - if not core.is_compiled_with_cuda(): return + if not core.is_compiled_with_cuda(): + return place = core.CPUPlace() place2 = core.CUDAPlace(0) self.scope = core.Scope() op_inputs = self.inputs if hasattr(self, "inputs") else dict() op_outputs = self.outputs if hasattr(self, "outputs") else dict() op_attrs = self.attrs if hasattr(self, "attrs") else dict() - self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs, - op_attrs) + self.op = create_op( + self.scope, self.op_type, op_inputs, op_outputs, op_attrs + ) inputs_to_check = set(['X', 'Scale', 'Bias']) output_names = 'Y' - cpu_grads = self._get_gradient(inputs_to_check, place, output_names, - None) - gpu_grads = self._get_gradient(inputs_to_check, place2, output_names, - None) - self._assert_is_close(cpu_grads, gpu_grads, inputs_to_check, 0.005, - "Gradient Check On %s" % str(place)) + cpu_grads = self._get_gradient( + inputs_to_check, place, output_names, None + ) + gpu_grads = self._get_gradient( + inputs_to_check, place2, output_names, None + ) + self._assert_is_close( + cpu_grads, + gpu_grads, + inputs_to_check, + 0.005, + "Gradient Check On %s" % str(place), + ) def test_check_grad(self): if self.compare_between_place: @@ -149,44 +161,37 @@ class TestGroupNormOp(OpTest): class TestGroupNormOp1(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 class TestGroupNormOp2(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 class TestGroupNormOpBigEps1(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.attrs['epsilon'] = 0.5 class TestGroupNormOpBigEps2(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.attrs['epsilon'] = 0.5 class TestGroupNormOpBigEps3(TestGroupNormOp): - def init_test_case(self): self.attrs['epsilon'] = 0.5 @skip_check_grad_ci( - reason= - '''This test case is used to ensure whether the gradient checking results between CPU and GPU + reason='''This test case is used to ensure whether the gradient checking results between CPU and GPU are consistent when using the same inputs, thus, it doesn't need to call check_grad.''' ) class TestGroupNormOpLargeData(TestGroupNormOp): - def init_test_case(self): self.shape = (2, 32, 64, 64) self.attrs['groups'] = 8 @@ -194,21 +199,18 @@ class TestGroupNormOpLargeData(TestGroupNormOp): class TestGroupNormOp1_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.data_format = "NHWC" class TestGroupNormOp2_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.data_format = "NHWC" class TestGroupNormOpBigEps1_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.attrs['epsilon'] = 0.5 @@ -216,7 +218,6 @@ class TestGroupNormOpBigEps1_With_NHWC(TestGroupNormOp): class TestGroupNormOpBigEps2_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.attrs['epsilon'] = 0.5 @@ -224,19 +225,16 @@ class TestGroupNormOpBigEps2_With_NHWC(TestGroupNormOp): class TestGroupNormOpBigEps3_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['epsilon'] = 0.5 self.data_format = "NHWC" @skip_check_grad_ci( - reason= - '''This test case is used to ensure whether the gradient checking results between CPU and GPU + reason='''This test case is used to ensure whether the gradient checking results between CPU and GPU are consistent when using the same inputs, thus, it doesn't need to call check_grad.''' ) class TestGroupNormOpLargeData_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.shape = (2, 64, 32, 32) # NCHW self.attrs['groups'] = 8 @@ -245,16 +243,15 @@ class TestGroupNormOpLargeData_With_NHWC(TestGroupNormOp): class TestGroupNormAPI_With_NHWC(unittest.TestCase): - def test_case1(self): data1 = fluid.data(name='data1', shape=[None, 3, 3, 4], dtype='float64') - out1 = fluid.layers.group_norm(input=data1, - groups=2, - data_layout="NHWC") + out1 = fluid.layers.group_norm( + input=data1, groups=2, data_layout="NHWC" + ) data2 = fluid.data(name='data2', shape=[None, 4, 3, 3], dtype='float64') - out2 = fluid.layers.group_norm(input=data2, - groups=2, - data_layout="NCHW") + out2 = fluid.layers.group_norm( + input=data2, groups=2, data_layout="NCHW" + ) data1_np = np.random.random((2, 3, 3, 4)).astype("float64") data2_np = np.random.random((2, 4, 3, 3)).astype("float64") @@ -263,25 +260,18 @@ class TestGroupNormAPI_With_NHWC(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2], - return_numpy=True) - expect_res1 = group_norm_naive(data1_np, - scale, - bias, - epsilon=1e-5, - groups=2, - data_layout="NHWC") - expect_res2 = group_norm_naive(data2_np, - scale, - bias, - epsilon=1e-5, - groups=2, - data_layout="NCHW") + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2], + return_numpy=True, + ) + expect_res1 = group_norm_naive( + data1_np, scale, bias, epsilon=1e-5, groups=2, data_layout="NHWC" + ) + expect_res2 = group_norm_naive( + data2_np, scale, bias, epsilon=1e-5, groups=2, data_layout="NCHW" + ) np.testing.assert_allclose(results[0], expect_res1[0], rtol=1e-05) np.testing.assert_allclose(results[1], expect_res2[0], rtol=1e-05) @@ -292,15 +282,14 @@ class TestGroupNormException(unittest.TestCase): data = fluid.data(name='data', shape=[None, 3, 3, 4], dtype="float64") def attr_data_format(): - out = fluid.layers.group_norm(input=data, - groups=2, - data_layout="NDHW") + out = fluid.layers.group_norm( + input=data, groups=2, data_layout="NDHW" + ) self.assertRaises(ValueError, attr_data_format) class TestGroupNormEager(unittest.TestCase): - def test_dygraph_api(self): self.dtype = np.float64 self.shape = (8, 32, 32) @@ -315,13 +304,17 @@ class TestGroupNormEager(unittest.TestCase): with _test_eager_guard(): tensor_eager_1 = fluid.dygraph.to_variable(input) tensor_eager_1.stop_gradient = False - groupNorm_eager = fluid.dygraph.nn.GroupNorm(channels=32, - groups=4) + groupNorm_eager = fluid.dygraph.nn.GroupNorm( + channels=32, groups=4 + ) ret2 = groupNorm_eager(tensor_eager_1) ret2.backward() - self.assertEqual(( - tensor_1.grad.numpy() == tensor_eager_1.grad.numpy()).all(), - True) + self.assertEqual( + ( + tensor_1.grad.numpy() == tensor_eager_1.grad.numpy() + ).all(), + True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_group_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_group_norm_op_v2.py index 52889f757df..63b69e6107b 100644 --- a/python/paddle/fluid/tests/unittests/test_group_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_group_norm_op_v2.py @@ -33,18 +33,23 @@ def group_norm_naive_for_general_dimension(x, scale, bias, epsilon, groups): var = np.var(x, axis=1, keepdims=True) output = (x - mean) / np.sqrt(var + epsilon) output = output.reshape(input_shape) * scale.reshape( - (-1, 1, 1)) + bias.reshape((-1, 1, 1)) + (-1, 1, 1) + ) + bias.reshape((-1, 1, 1)) return output class TestDygraphGroupNormv2(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("group_norm"): places.append(fluid.CUDAPlace(0)) - shapes = [[2, 2, 2, 2], [2, 2, 4], [4, 2], [4, 2, 6, 6, 2], - [2, 2, 2, 2, 2, 2]] + shapes = [ + [2, 2, 2, 2], + [2, 2, 4], + [4, 2], + [4, 2, 6, 6, 2], + [2, 2, 2, 2, 2, 2], + ] for p in places: def compute_v1(x): @@ -61,18 +66,20 @@ class TestDygraphGroupNormv2(unittest.TestCase): def test_weight_bias_false(): with fluid.dygraph.guard(p): - gn = paddle.nn.GroupNorm(num_channels=2, - num_groups=2, - weight_attr=False, - bias_attr=False) + gn = paddle.nn.GroupNorm( + num_channels=2, + num_groups=2, + weight_attr=False, + bias_attr=False, + ) def test_nn_exception(): with fluid.dygraph.guard(p): def attr_data_format(): - out = paddle.nn.GroupNorm(num_groups=2, - num_channels=2, - data_format="NHWC") + out = paddle.nn.GroupNorm( + num_groups=2, num_channels=2, data_format="NHWC" + ) self.assertRaises(ValueError, attr_data_format) @@ -92,8 +99,13 @@ class TestDygraphGroupNormv2(unittest.TestCase): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("group_norm"): places.append(fluid.CUDAPlace(0)) - shapes = [[2, 6, 2, 2], [2, 6, 4], [4, 6], [4, 6, 6, 6, 2], - [4, 6, 2, 2, 2, 2]] + shapes = [ + [2, 6, 2, 2], + [2, 6, 4], + [4, 6], + [4, 6, 6, 6, 2], + [4, 6, 2, 2, 2, 2], + ] for p in places: exe = fluid.Executor(p) @@ -127,11 +139,15 @@ class TestDygraphGroupNormv2(unittest.TestCase): class TestGroupNormAPIV2_With_General_Dimensions(unittest.TestCase): - def test_numerical_accuracy(self): paddle.disable_static() - shapes = [(2, 6), (2, 6, 4), (2, 6, 4, 4), (2, 6, 6, 6, 2), - (2, 6, 6, 6, 2, 3)] + shapes = [ + (2, 6), + (2, 6, 4), + (2, 6, 4, 4), + (2, 6, 6, 6, 2), + (2, 6, 6, 6, 2, 3), + ] np.random.seed(10) places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("group_norm"): @@ -143,9 +159,11 @@ class TestGroupNormAPIV2_With_General_Dimensions(unittest.TestCase): bias = np.array([0]).astype("float32") data = np.random.random(shape).astype("float32") expect_res1 = group_norm_naive_for_general_dimension( - data, scale, bias, epsilon=1e-5, groups=6) + data, scale, bias, epsilon=1e-5, groups=6 + ) expect_res2 = group_norm_naive_for_general_dimension( - data, scale, bias, epsilon=1e-5, groups=2) + data, scale, bias, epsilon=1e-5, groups=2 + ) gn1 = paddle.nn.GroupNorm(num_channels=6, num_groups=6) gn2 = paddle.nn.GroupNorm(num_channels=6, num_groups=2) @@ -161,9 +179,7 @@ class TestGroupNormAPIV2_With_General_Dimensions(unittest.TestCase): class TestGroupNormDimException(unittest.TestCase): - def test_exception(self): - def test_empty_input_static_API(): x = paddle.to_tensor([], dtype='float32') paddle.static.nn.group_norm(x, 3) @@ -171,7 +187,7 @@ class TestGroupNormDimException(unittest.TestCase): self.assertRaises(ValueError, test_empty_input_static_API) def test_one_dim_input_static_API(): - x = paddle.randn((3, ), dtype='float32') + x = paddle.randn((3,), dtype='float32') paddle.static.nn.group_norm(x, 3) self.assertRaises(ValueError, test_one_dim_input_static_API) diff --git a/python/paddle/fluid/tests/unittests/test_gru_op.py b/python/paddle/fluid/tests/unittests/test_gru_op.py index 4e51c1da0e8..d2753dfd205 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_op.py @@ -22,17 +22,17 @@ from paddle.fluid import Program, program_guard def gru( - input, # T x 3D - lod, # 1 x N - h0, # N x D - weight, # D x 3D - bias, # 1 x 3D - is_reverse, - act_state, - act_gate, - dtype='float32', - origin_mode=False): - + input, # T x 3D + lod, # 1 x N + h0, # N x D + weight, # D x 3D + bias, # 1 x 3D + is_reverse, + act_state, + act_gate, + dtype='float32', + origin_mode=False, +): def _seq_to_batch(lod, is_reverse): idx_in_seq_list = [] seq_lens = lod[0] @@ -41,16 +41,19 @@ def gru( seq_starts.append(seq_starts[-1] + seq_lens[i]) sorted_seqs = sorted( list(range(len(seq_lens))), - key=functools.cmp_to_key(lambda x, y: seq_lens[y] - seq_lens[x])) + key=functools.cmp_to_key(lambda x, y: seq_lens[y] - seq_lens[x]), + ) num_batch = seq_lens[sorted_seqs[0]] for batch_idx in range(num_batch): idx_in_seq = [] for i in range(len(seq_lens)): if seq_lens[sorted_seqs[i]] <= batch_idx: break - idx = (seq_starts[sorted_seqs[i] + 1] - 1 - - batch_idx) if is_reverse else ( - seq_starts[sorted_seqs[i]] + batch_idx) + idx = ( + (seq_starts[sorted_seqs[i] + 1] - 1 - batch_idx) + if is_reverse + else (seq_starts[sorted_seqs[i]] + batch_idx) + ) idx_in_seq.append(idx) idx_in_seq_list.append(idx_in_seq) return idx_in_seq_list, sorted_seqs @@ -59,13 +62,13 @@ def gru( T = x.shape[0] D = w.shape[0] g = x + np.tile(b, (T, 1)) - w_u_r = w.flatten()[:D * D * 2].reshape((D, D * 2)) - u_r = act_gate(np.dot(h_p, w_u_r) + g[:, :D * 2]) + w_u_r = w.flatten()[: D * D * 2].reshape((D, D * 2)) + u_r = act_gate(np.dot(h_p, w_u_r) + g[:, : D * 2]) u = u_r[:, :D] - r = u_r[:, D:D * 2] + r = u_r[:, D : D * 2] r_h_p = r * h_p - w_c = w.flatten()[D * D * 2:].reshape((D, D)) - c = act_state(np.dot(r_h_p, w_c) + g[:, D * 2:]) + w_c = w.flatten()[D * D * 2 :].reshape((D, D)) + c = act_state(np.dot(r_h_p, w_c) + g[:, D * 2 :]) g = np.hstack((u_r, c)) if origin_mode: h = (1 - u) * c + u * h_p @@ -90,7 +93,7 @@ def gru( x = input[idx_in_seq_list[batch_idx]] g, r_h_p, h = _step(x, h_p, weight, bias, act_state, act_gate) if batch_idx < (max_seq_len - 1): - h_p = h[:len(idx_in_seq_list[batch_idx + 1])] + h_p = h[: len(idx_in_seq_list[batch_idx + 1])] start_idx = end_idx end_idx = start_idx + len(idx_in_seq_list[batch_idx]) batch_gate[start_idx:end_idx] = g @@ -101,7 +104,6 @@ def gru( class TestGRUOp(OpTest): - def set_confs(self): pass @@ -126,17 +128,29 @@ class TestGRUOp(OpTest): N = len(self.lod[0]) input = np.random.rand(T, 3 * self.D).astype(self.dtype) weight = np.random.rand(self.D, 3 * self.D).astype(self.dtype) - bias = np.random.rand(1, 3 * self.D).astype( - self.dtype) if self.with_bias else np.zeros( - (1, 3 * self.D), dtype=self.dtype) - h0 = np.random.rand(N, self.D).astype( - self.dtype) if self.with_h0 else np.zeros( - (N, self.D), dtype=self.dtype) + bias = ( + np.random.rand(1, 3 * self.D).astype(self.dtype) + if self.with_bias + else np.zeros((1, 3 * self.D), dtype=self.dtype) + ) + h0 = ( + np.random.rand(N, self.D).astype(self.dtype) + if self.with_h0 + else np.zeros((N, self.D), dtype=self.dtype) + ) batch_gate, batch_reset_hidden_prev, batch_hidden, hidden = gru( - input, self.lod, h0, weight, bias, self.is_reverse, - ACTIVATION[self.act_state], ACTIVATION[self.act_gate], self.dtype, - self.origin_mode) + input, + self.lod, + h0, + weight, + bias, + self.is_reverse, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate], + self.dtype, + self.origin_mode, + ) self.inputs = {'Input': (input, self.lod), 'Weight': weight} if self.with_bias: @@ -157,45 +171,41 @@ class TestGRUOp(OpTest): 'gate_activation': self.act_gate, 'is_reverse': self.is_reverse, 'origin_mode': self.origin_mode, - 'is_test': self.is_test + 'is_test': self.is_test, } def test_check_output(self): self.check_output(atol=1e-8, check_dygraph=False) def test_check_grad(self): - self.check_grad(['Input', 'H0', 'Weight', 'Bias'], ['Hidden'], - check_dygraph=False) + self.check_grad( + ['Input', 'H0', 'Weight', 'Bias'], ['Hidden'], check_dygraph=False + ) class TestGRUOriginMode(TestGRUOp): - def set_confs(self): self.origin_mode = True class TestGRUOp2(TestGRUOp): - def set_confs(self): self.dtype = 'float64' class TestGRUOp2Len0(TestGRUOp): - def set_confs(self): self.lod = [[2, 0, 4]] self.dtype = 'float64' class TestGRUOp2OriginMode(TestGRUOp): - def set_confs(self): self.dtype = 'float64' self.origin_mode = True class TestGRUOp2OriginModeLen0(TestGRUOp): - def set_confs(self): self.lod = [[0, 3, 4]] self.dtype = 'float64' @@ -203,7 +213,6 @@ class TestGRUOp2OriginModeLen0(TestGRUOp): class TestGRUOp2OriginModeLastLen0(TestGRUOp): - def set_confs(self): self.lod = [[0, 3, 0]] self.dtype = 'float64' @@ -211,40 +220,37 @@ class TestGRUOp2OriginModeLastLen0(TestGRUOp): class TestGRUOpNoInitial(TestGRUOp): - def set_confs(self): self.with_h0 = False def test_check_grad(self): - self.check_grad(['Input', 'Weight', 'Bias'], ['Hidden'], - check_dygraph=False) + self.check_grad( + ['Input', 'Weight', 'Bias'], ['Hidden'], check_dygraph=False + ) class TestGRUOpNoBias(TestGRUOp): - def set_confs(self): self.with_bias = False def test_check_grad(self): - self.check_grad(['Input', 'H0', 'Weight'], ['Hidden'], - check_dygraph=False) + self.check_grad( + ['Input', 'H0', 'Weight'], ['Hidden'], check_dygraph=False + ) class TestGRUOpReverse(TestGRUOp): - def set_confs(self): self.is_reverse = True class TestGRUOpReverseOriginMode(TestGRUOp): - def set_confs(self): self.is_reverse = True self.origin_mode = True class TestGRUOpInference(TestGRUOp): - def set_is_test(self): self.is_test = True @@ -260,7 +266,6 @@ class TestGRUOpInference(TestGRUOp): class TestGruOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -271,9 +276,9 @@ class TestGruOpError(unittest.TestCase): self.assertRaises(TypeError, test_Variable) def test_h_0(): - in_data = fluid.data(name="input", - shape=[None, 1536], - dtype="float32") + in_data = fluid.data( + name="input", shape=[None, 1536], dtype="float32" + ) h = fluid.data(name="h", shape=[None, 512], dtype="int32") fluid.layers.dynamic_gru(input=in_data, size=512, h_0=h) diff --git a/python/paddle/fluid/tests/unittests/test_gru_rnn_op.py b/python/paddle/fluid/tests/unittests/test_gru_rnn_op.py index 0a10eb74f55..89379ac87f3 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_rnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_rnn_op.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestGRUOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(self.num_layers): @@ -45,13 +44,16 @@ class TestGRUOp(OpTest): def setUp(self): self.op_type = "rnn" self.dtype = "float32" if core.is_compiled_with_rocm() else "float64" - self.sequence_length = None if core.is_compiled_with_rocm( - ) else np.array([12, 11, 10, 9, 8, 7, 6, 5], dtype=np.int32) + self.sequence_length = ( + None + if core.is_compiled_with_rocm() + else np.array([12, 11, 10, 9, 8, 7, 6, 5], dtype=np.int32) + ) self.num_layers = 1 self.is_bidirec = False self.is_test = False self.mode = "GRU" - self.dropout = 0. + self.dropout = 0.0 seq_length = 12 batch_size = 8 input_size = 4 @@ -61,10 +63,9 @@ class TestGRUOp(OpTest): self.direction_num = 2 if self.is_bidirec else 1 direction = "bidirectional" if self.is_bidirec else "forward" - input = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_length, batch_size, - input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, high=0.1, size=(seq_length, batch_size, input_size) + ).astype(self.dtype) if self.sequence_length is not None: input[3][1:][:] = 0 @@ -72,13 +73,15 @@ class TestGRUOp(OpTest): input[2][3:][:] = 0 input[1][4:][:] = 0 - rnn1 = GRU(input_size, - self.hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - dtype=self.dtype) + rnn1 = GRU( + input_size, + self.hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) @@ -92,8 +95,9 @@ class TestGRUOp(OpTest): self._get_places = rocm_rnn_get_place - init_h = np.zeros((self.num_layers * self.direction_num, batch_size, - self.hidden_size)).astype(self.dtype) + init_h = np.zeros( + (self.num_layers * self.direction_num, batch_size, self.hidden_size) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") @@ -101,7 +105,7 @@ class TestGRUOp(OpTest): 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -116,13 +120,13 @@ class TestGRUOp(OpTest): 'hidden_size': self.hidden_size, 'num_layers': self.num_layers, 'is_test': self.is_test, - 'mode': self.mode + 'mode': self.mode, } self.outputs = { 'Out': output, 'State': [('last_hidden', last_hidden)], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def set_attrs(self): @@ -140,27 +144,23 @@ class TestGRUOp(OpTest): class TestGRUOp1(TestGRUOp): - def set_attrs(self): self.sequence_length = None class TestGRUOp2(TestGRUOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True class TestGRUOp3(TestGRUOp): - def set_attrs(self): self.sequence_length = None self.is_test = True class TestGRUOp4(TestGRUOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True @@ -168,7 +168,6 @@ class TestGRUOp4(TestGRUOp): class TestGRUOpAvx(TestGRUOp): - def set_attrs(self): self.dtype = "float32" self.hidden_size = 8 diff --git a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py index 96c27741610..7111e4f8b9c 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py @@ -23,14 +23,14 @@ from paddle.fluid.framework import program_guard, Program class TestGRUUnitAPIError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): D = 5 layer = fluid.dygraph.nn.GRUUnit(size=D * 3) # the input must be Variable. - x0 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x0 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, layer, x0) # the input dtype must be float32 or float64 x = fluid.data(name='x', shape=[-1, D * 3], dtype='float16') @@ -50,11 +50,11 @@ def identity(x): def sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) def tanh(x): - return 2. * sigmoid(2. * x) - 1. + return 2.0 * sigmoid(2.0 * x) - 1.0 def relu(x): @@ -62,21 +62,22 @@ def relu(x): class TestGRUUnitOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size = 5 hidden_dim = 40 - input = fluid.data(name='input', - shape=[None, hidden_dim * 3], - dtype='float32') - pre_hidden = fluid.data(name='pre_hidden', - shape=[None, hidden_dim], - dtype='float32') + input = fluid.data( + name='input', shape=[None, hidden_dim * 3], dtype='float32' + ) + pre_hidden = fluid.data( + name='pre_hidden', shape=[None, hidden_dim], dtype='float32' + ) np_input = np.random.uniform( - -0.1, 0.1, (batch_size, hidden_dim * 3)).astype('float64') + -0.1, 0.1, (batch_size, hidden_dim * 3) + ).astype('float64') np_pre_hidden = np.random.uniform( - -0.1, 0.1, (batch_size, hidden_dim)).astype('float64') + -0.1, 0.1, (batch_size, hidden_dim) + ).astype('float64') def test_input_Variable(): gru_unit(np_input, pre_hidden, hidden_dim * 3) @@ -89,17 +90,21 @@ class TestGRUUnitOpError(unittest.TestCase): self.assertRaises(TypeError, test_pre_hidden_Variable) def test_input_type(): - error_input = fluid.data(name='error_input', - shape=[None, hidden_dim * 3], - dtype='int32') + error_input = fluid.data( + name='error_input', + shape=[None, hidden_dim * 3], + dtype='int32', + ) gru_unit(error_input, pre_hidden, hidden_dim * 3) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[None, hidden_dim], - dtype='int32') + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[None, hidden_dim], + dtype='int32', + ) gru_unit(input, error_pre_hidden, hidden_dim * 3) self.assertRaises(TypeError, test_pre_hidden_type) @@ -120,21 +125,22 @@ class TestGRUUnitOp(OpTest): frame_size = self.frame_size self.op_type = 'gru_unit' self.inputs = { - 'Input': - np.random.uniform(-0.1, 0.1, - (batch_size, frame_size * 3)).astype(self.dtype), - 'HiddenPrev': - np.random.uniform(-0.1, 0.1, - (batch_size, frame_size)).astype(self.dtype), - 'Weight': - np.random.uniform(-1. / math.sqrt(frame_size), - 1. / math.sqrt(frame_size), - (frame_size, frame_size * 3)).astype(self.dtype), + 'Input': np.random.uniform( + -0.1, 0.1, (batch_size, frame_size * 3) + ).astype(self.dtype), + 'HiddenPrev': np.random.uniform( + -0.1, 0.1, (batch_size, frame_size) + ).astype(self.dtype), + 'Weight': np.random.uniform( + -1.0 / math.sqrt(frame_size), + 1.0 / math.sqrt(frame_size), + (frame_size, frame_size * 3), + ).astype(self.dtype), } self.attrs = { 'activation': GRUActivationType.tanh, 'gate_activation': GRUActivationType.sigmoid, - 'origin_mode': origin_mode + 'origin_mode': origin_mode, } def set_outputs(self, origin_mode=False): @@ -144,21 +150,27 @@ class TestGRUUnitOp(OpTest): x = self.inputs['Input'] h_p = self.inputs['HiddenPrev'] w = self.inputs['Weight'] - b = self.inputs['Bias'] if 'Bias' in self.inputs else np.zeros( - (1, frame_size * 3)) + b = ( + self.inputs['Bias'] + if 'Bias' in self.inputs + else np.zeros((1, frame_size * 3)) + ) g = x + np.tile(b, (batch_size, 1)) - w_u_r = w.flatten()[:frame_size * frame_size * 2].reshape( - (frame_size, frame_size * 2)) - u_r = self.activate[self.attrs['gate_activation']](np.dot(h_p, w_u_r) + - g[:, :frame_size * - 2]) + w_u_r = w.flatten()[: frame_size * frame_size * 2].reshape( + (frame_size, frame_size * 2) + ) + u_r = self.activate[self.attrs['gate_activation']]( + np.dot(h_p, w_u_r) + g[:, : frame_size * 2] + ) u = u_r[:, :frame_size] - r = u_r[:, frame_size:frame_size * 2] + r = u_r[:, frame_size : frame_size * 2] r_h_p = r * h_p - w_c = w.flatten()[frame_size * frame_size * 2:].reshape( - (frame_size, frame_size)) - c = self.activate[self.attrs['activation']](np.dot(r_h_p, w_c) + - g[:, frame_size * 2:]) + w_c = w.flatten()[frame_size * frame_size * 2 :].reshape( + (frame_size, frame_size) + ) + c = self.activate[self.attrs['activation']]( + np.dot(r_h_p, w_c) + g[:, frame_size * 2 :] + ) g = np.hstack((u_r, c)) if origin_mode: h = (1 - u) * c + u * h_p @@ -167,12 +179,13 @@ class TestGRUUnitOp(OpTest): self.outputs = { 'Gate': g.astype(self.dtype), 'ResetHiddenPrev': r_h_p.astype(self.dtype), - 'Hidden': h.astype(self.dtype) + 'Hidden': h.astype(self.dtype), } def setUp(self): - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.set_inputs() self.set_outputs() @@ -184,41 +197,44 @@ class TestGRUUnitOp(OpTest): class TestGRUUnitOpOriginMode(TestGRUUnitOp): - def setUp(self): - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.set_inputs(origin_mode=True) self.set_outputs(origin_mode=True) class TestGRUUnitOpWithBias(TestGRUUnitOp): - def set_inputs(self, origin_mode=False): batch_size = self.batch_size frame_size = self.frame_size super(TestGRUUnitOpWithBias, self).set_inputs() self.inputs['Bias'] = np.random.uniform( - -0.1, 0.1, (1, frame_size * 3)).astype(self.dtype) + -0.1, 0.1, (1, frame_size * 3) + ).astype(self.dtype) self.attrs = { 'activation': GRUActivationType.identity, 'gate_activation': GRUActivationType.sigmoid, - 'origin_mode': origin_mode + 'origin_mode': origin_mode, } def test_check_grad(self): self.check_grad(['Input', 'HiddenPrev', 'Weight', 'Bias'], ['Hidden']) def test_check_grad_ingore_input(self): - self.check_grad(['HiddenPrev', 'Weight', 'Bias'], ['Hidden'], - no_grad_set=set('Input')) + self.check_grad( + ['HiddenPrev', 'Weight', 'Bias'], + ['Hidden'], + no_grad_set=set('Input'), + ) class TestGRUUnitOpWithBiasOriginMode(TestGRUUnitOpWithBias): - def setUp(self): - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.set_inputs(origin_mode=True) self.set_outputs(origin_mode=True) diff --git a/python/paddle/fluid/tests/unittests/test_gumbel_softmax_op.py b/python/paddle/fluid/tests/unittests/test_gumbel_softmax_op.py index 90de83fce24..de7f5d46352 100644 --- a/python/paddle/fluid/tests/unittests/test_gumbel_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_gumbel_softmax_op.py @@ -21,7 +21,6 @@ paddle.enable_static() class TestGumbelSoftmaxOp(OpTest): - def init_attrs(self): self.shape = [20, 10] self.attrs = {"hard": True, "axis": -1} @@ -51,7 +50,6 @@ class TestGumbelSoftmaxOp(OpTest): class TestGumbelSoftmaxOp2(TestGumbelSoftmaxOp): - def init_attrs(self): self.shape = [20, 10] self.attrs = {"hard": True, "axis": 0} @@ -60,7 +58,6 @@ class TestGumbelSoftmaxOp2(TestGumbelSoftmaxOp): class TestGumbelSoftmaxOp3(TestGumbelSoftmaxOp): - def init_attrs(self): self.shape = [100] self.attrs = {"hard": True, "axis": -1} @@ -69,7 +66,6 @@ class TestGumbelSoftmaxOp3(TestGumbelSoftmaxOp): class TestGumbelSoftmaxOp4(TestGumbelSoftmaxOp): - def init_attrs(self): self.shape = [20, 10, 5] self.attrs = {"hard": True, "axis": -1} @@ -78,7 +74,6 @@ class TestGumbelSoftmaxOp4(TestGumbelSoftmaxOp): class TestGumbelSoftmaxOp5(TestGumbelSoftmaxOp): - def init_attrs(self): self.shape = [20, 10, 5] self.attrs = {"hard": True, "axis": 1} @@ -87,7 +82,6 @@ class TestGumbelSoftmaxOp5(TestGumbelSoftmaxOp): class TestGumbelSoftmaxOpSampleDistribution(OpTest): - def softmax(self, x): x_row_max = x.max(axis=-1) x_row_max = x_row_max.reshape(list(x.shape)[:-1] + [1]) @@ -139,7 +133,6 @@ class TestGumbelSoftmaxOpSampleDistribution(OpTest): class TestGumbelSoftmaxOpGrad(unittest.TestCase): - def init_attrs(self): self.shape = [20, 10] self.dtype = "float64" @@ -159,22 +152,22 @@ class TestGumbelSoftmaxOpGrad(unittest.TestCase): out_hard.sum().backward() out_soft.sum().backward() - np.testing.assert_allclose(x_hard.grad.numpy(), - x_soft.grad.numpy(), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + x_hard.grad.numpy(), x_soft.grad.numpy(), rtol=1e-5, atol=1e-8 + ) paddle.enable_static() class TestGumbelSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] - self.x = np.random.uniform(-1., 1., self.x_shape).astype(np.float32) + self.x = np.random.uniform(-1.0, 1.0, self.x_shape).astype(np.float32) self.count_expected = 24 - self.place = paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_check_api(self): # test static api @@ -201,13 +194,13 @@ class TestGumbelSoftmaxAPI(unittest.TestCase): class TestGumbelSoftmaxOpError(unittest.TestCase): - def test_errors(self): paddle.disable_static() def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros((100, 784)), - [[10, 10, 10, 70]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace() + ) paddle.nn.functional.gumbel_softmax(x1) self.assertRaises(ValueError, test_Variable) @@ -234,9 +227,9 @@ class TestGumbelSoftmaxOpError(unittest.TestCase): def test_dtype(): with paddle.static.program_guard(paddle.static.Program()): - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) paddle.nn.functional.gumbel_softmax(x_int32) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_hash_op.py b/python/paddle/fluid/tests/unittests/test_hash_op.py index fe076290948..0070a3fadc7 100644 --- a/python/paddle/fluid/tests/unittests/test_hash_op.py +++ b/python/paddle/fluid/tests/unittests/test_hash_op.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class TestHashOp(OpTest): - def setUp(self): self.op_type = "hash" self.init_test_case() @@ -31,9 +30,16 @@ class TestHashOp(OpTest): np.random.seed(1) self.in_seq = np.random.randint(0, 10, (8, 1)).astype("int32") self.lod = [[2, 6]] - self.out_seq = [[[3481], [7475]], [[1719], [5986]], [[8473], [694]], - [[3481], [7475]], [[4372], [9456]], [[4372], [9456]], - [[6897], [3218]], [[9038], [7951]]] + self.out_seq = [ + [[3481], [7475]], + [[1719], [5986]], + [[8473], [694]], + [[3481], [7475]], + [[4372], [9456]], + [[4372], [9456]], + [[6897], [3218]], + [[9038], [7951]], + ] self.out_seq = np.array(self.out_seq) def test_check_output(self): @@ -41,7 +47,6 @@ class TestHashOp(OpTest): class TestHashNotLoDOp(TestHashOp): - def setUp(self): self.op_type = "hash" self.init_test_case() @@ -52,9 +57,16 @@ class TestHashNotLoDOp(TestHashOp): def init_test_case(self): np.random.seed(1) self.in_seq = np.random.randint(0, 10, (8, 1)).astype("int32") - self.out_seq = [[[3481], [7475]], [[1719], [5986]], [[8473], [694]], - [[3481], [7475]], [[4372], [9456]], [[4372], [9456]], - [[6897], [3218]], [[9038], [7951]]] + self.out_seq = [ + [[3481], [7475]], + [[1719], [5986]], + [[8473], [694]], + [[3481], [7475]], + [[4372], [9456]], + [[4372], [9456]], + [[6897], [3218]], + [[9038], [7951]], + ] self.out_seq = np.array(self.out_seq) def test_check_output(self): @@ -98,15 +110,15 @@ class TestHashOp3(TestHashOp): def init_test_case(self): self.in_seq = np.array([10, 5]).reshape((2, 1)).astype("int64") - self.out_seq = np.array([1204014882, 393011615, 3586283837, - 2814821595]).reshape((2, 2, 1)) + self.out_seq = np.array( + [1204014882, 393011615, 3586283837, 2814821595] + ).reshape((2, 2, 1)) def test_check_output(self): self.check_output() class TestHashOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_data = np.random.randint(0, 10, (8, 1)).astype("int32") @@ -119,30 +131,27 @@ class TestHashOpError(unittest.TestCase): def test_type(): # dtype must be int32, int64. - x2 = fluid.layers.data(name='x2', - shape=[1], - dtype="float32", - lod_level=1) + x2 = fluid.layers.data( + name='x2', shape=[1], dtype="float32", lod_level=1 + ) fluid.layers.hash(input=x2, hash_size=2**32) self.assertRaises(TypeError, test_type) def test_hash_size_type(): # hash_size dtype must be int32, int64. - x3 = fluid.layers.data(name='x3', - shape=[1], - dtype="int32", - lod_level=1) + x3 = fluid.layers.data( + name='x3', shape=[1], dtype="int32", lod_level=1 + ) fluid.layers.hash(input=x3, hash_size=1024.5) self.assertRaises(TypeError, test_hash_size_type) def test_num_hash_type(): # num_hash dtype must be int32, int64. - x4 = fluid.layers.data(name='x4', - shape=[1], - dtype="int32", - lod_level=1) + x4 = fluid.layers.data( + name='x4', shape=[1], dtype="int32", lod_level=1 + ) fluid.layers.hash(input=x4, hash_size=2**32, num_hash=2.5) self.assertRaises(TypeError, test_num_hash_type) diff --git a/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py b/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py index 8088ca7c363..68e7b4210a6 100644 --- a/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py +++ b/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py @@ -21,8 +21,9 @@ np.random.seed(42) def calc_hinge_embedding_loss(input, label, margin=1.0, reduction='mean'): - result = np.where(label == -1., np.maximum(0., margin - input), 0.) + \ - np.where(label == 1., input, 0.) + result = np.where( + label == -1.0, np.maximum(0.0, margin - input), 0.0 + ) + np.where(label == 1.0, input, 0.0) if reduction == 'none': return result elif reduction == 'sum': @@ -32,13 +33,12 @@ def calc_hinge_embedding_loss(input, label, margin=1.0, reduction='mean'): class TestFunctionalHingeEmbeddingLoss(unittest.TestCase): - def setUp(self): self.margin = 1.0 self.shape = (10, 10, 5) self.input_np = np.random.random(size=self.shape).astype(np.float64) # get label elem in {1., -1.} - self.label_np = 2 * np.random.randint(0, 2, size=self.shape) - 1. + self.label_np = 2 * np.random.randint(0, 2, size=self.shape) - 1.0 def run_dynamic_check(self, place=paddle.CPUPlace()): paddle.disable_static(place=place) @@ -50,45 +50,45 @@ class TestFunctionalHingeEmbeddingLoss(unittest.TestCase): np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) - dy_result = paddle.nn.functional.hinge_embedding_loss(input, - label, - reduction='sum') - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction='sum') + dy_result = paddle.nn.functional.hinge_embedding_loss( + input, label, reduction='sum' + ) + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction='sum' + ) np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) - dy_result = paddle.nn.functional.hinge_embedding_loss(input, - label, - reduction='none') - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction='none') + dy_result = paddle.nn.functional.hinge_embedding_loss( + input, label, reduction='none' + ) + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction='none' + ) np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, self.shape) def run_static_check(self, place=paddle.CPUPlace): paddle.enable_static() for reduction in ['none', 'mean', 'sum']: - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction=reduction) + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction=reduction + ) with program_guard(Program(), Program()): - input = paddle.static.data(name="input", - shape=self.shape, - dtype=paddle.float64) - label = paddle.static.data(name="label", - shape=self.shape, - dtype=paddle.float64) + input = paddle.static.data( + name="input", shape=self.shape, dtype=paddle.float64 + ) + label = paddle.static.data( + name="label", shape=self.shape, dtype=paddle.float64 + ) st_result = paddle.nn.functional.hinge_embedding_loss( - input, label, reduction=reduction) + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - result_numpy, = exe.run(feed={ - "input": self.input_np, - "label": self.label_np - }, - fetch_list=[st_result]) + (result_numpy,) = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[st_result], + ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) def test_cpu(self): @@ -103,22 +103,21 @@ class TestFunctionalHingeEmbeddingLoss(unittest.TestCase): # test case the raise message def test_reduce_errors(self): - def test_value_error(): loss = paddle.nn.functional.hinge_embedding_loss( - self.input_np, self.label_np, reduction='reduce_mean') + self.input_np, self.label_np, reduction='reduce_mean' + ) self.assertRaises(ValueError, test_value_error) class TestClassHingeEmbeddingLoss(unittest.TestCase): - def setUp(self): self.margin = 1.0 self.shape = (10, 10, 5) self.input_np = np.random.random(size=self.shape).astype(np.float64) # get label elem in {1., -1.} - self.label_np = 2 * np.random.randint(0, 2, size=self.shape) - 1. + self.label_np = 2 * np.random.randint(0, 2, size=self.shape) - 1.0 def run_dynamic_check(self, place=paddle.CPUPlace()): paddle.disable_static(place=place) @@ -131,45 +130,47 @@ class TestClassHingeEmbeddingLoss(unittest.TestCase): self.assertTrue(dy_result.shape, [1]) hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( - reduction='sum') + reduction='sum' + ) dy_result = hinge_embedding_loss(input, label) - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction='sum') + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction='sum' + ) np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( - reduction='none') + reduction='none' + ) dy_result = hinge_embedding_loss(input, label) - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction='none') + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction='none' + ) np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, self.shape) def run_static_check(self, place=paddle.CPUPlace): paddle.enable_static() for reduction in ['none', 'mean', 'sum']: - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction=reduction) + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction=reduction + ) with program_guard(Program(), Program()): - input = paddle.static.data(name="input", - shape=self.shape, - dtype=paddle.float64) - label = paddle.static.data(name="label", - shape=self.shape, - dtype=paddle.float64) + input = paddle.static.data( + name="input", shape=self.shape, dtype=paddle.float64 + ) + label = paddle.static.data( + name="label", shape=self.shape, dtype=paddle.float64 + ) hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( - reduction=reduction) + reduction=reduction + ) st_result = hinge_embedding_loss(input, label) exe = paddle.static.Executor(place) - result_numpy, = exe.run(feed={ - "input": self.input_np, - "label": self.label_np - }, - fetch_list=[st_result]) + (result_numpy,) = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[st_result], + ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) def test_cpu(self): @@ -184,10 +185,10 @@ class TestClassHingeEmbeddingLoss(unittest.TestCase): # test case the raise message def test_reduce_errors(self): - def test_value_error(): hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( - reduction='reduce_mean') + reduction='reduce_mean' + ) loss = hinge_embedding_loss(self.input_np, self.label_np) self.assertRaises(ValueError, test_value_error) diff --git a/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py b/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py index c2fbd5d9150..1b320f86449 100644 --- a/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py @@ -18,7 +18,6 @@ from op_test import OpTest class TestHingeLossOp(OpTest): - def setUp(self): self.op_type = 'hinge_loss' samples_num = 100 diff --git a/python/paddle/fluid/tests/unittests/test_histogram_op.py b/python/paddle/fluid/tests/unittests/test_histogram_op.py index 725b6c02362..516bc53a34a 100644 --- a/python/paddle/fluid/tests/unittests/test_histogram_op.py +++ b/python/paddle/fluid/tests/unittests/test_histogram_op.py @@ -36,14 +36,15 @@ class TestHistogramOpAPI(unittest.TestCase): exe = fluid.Executor(place) exe.run(startup_program) img = np.array([[2, 4, 2], [2, 5, 4]]).astype(np.int64) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) actual = np.array(res[0]) expected = np.array([0, 3, 0, 2, 1]).astype(np.int64) self.assertTrue( (actual == expected).all(), - msg='histogram output is wrong, out =' + str(actual)) + msg='histogram output is wrong, out =' + str(actual), + ) def test_dygraph(self): with fluid.dygraph.guard(): @@ -53,15 +54,18 @@ class TestHistogramOpAPI(unittest.TestCase): expected = np.array([0, 3, 0, 2, 1]).astype(np.int64) self.assertTrue( (actual.numpy() == expected).all(), - msg='histogram output is wrong, out =' + str(actual.numpy())) + msg='histogram output is wrong, out =' + str(actual.numpy()), + ) with _test_eager_guard(): inputs_np = np.array([[2, 4, 2], [2, 5, 4]]).astype(np.int64) inputs = paddle.to_tensor(inputs_np) actual = paddle.histogram(inputs, bins=5, min=1, max=5) - self.assertTrue((actual.numpy() == expected).all(), - msg='histogram output is wrong, out =' + - str(actual.numpy())) + self.assertTrue( + (actual.numpy() == expected).all(), + msg='histogram output is wrong, out =' + + str(actual.numpy()), + ) class TestHistogramOpError(unittest.TestCase): @@ -79,9 +83,9 @@ class TestHistogramOpError(unittest.TestCase): """Test bins should be greater than or equal to 1.""" def net_func(): - input_value = paddle.fluid.layers.fill_constant(shape=[3, 4], - dtype='float32', - value=3.0) + input_value = paddle.fluid.layers.fill_constant( + shape=[3, 4], dtype='float32', value=3.0 + ) paddle.histogram(input=input_value, bins=-1, min=1, max=5) with self.assertRaises(IndexError): @@ -91,9 +95,9 @@ class TestHistogramOpError(unittest.TestCase): """Test max must be larger or equal to min.""" def net_func(): - input_value = paddle.fluid.layers.fill_constant(shape=[3, 4], - dtype='float32', - value=3.0) + input_value = paddle.fluid.layers.fill_constant( + shape=[3, 4], dtype='float32', value=3.0 + ) paddle.histogram(input=input_value, bins=1, min=5, max=1) with self.assertRaises(ValueError): @@ -103,9 +107,9 @@ class TestHistogramOpError(unittest.TestCase): """Test range of min, max is not finite""" def net_func(): - input_value = paddle.fluid.layers.fill_constant(shape=[3, 4], - dtype='float32', - value=3.0) + input_value = paddle.fluid.layers.fill_constant( + shape=[3, 4], dtype='float32', value=3.0 + ) paddle.histogram(input=input_value, bins=1, min=-np.inf, max=5) with self.assertRaises(TypeError): @@ -114,24 +118,17 @@ class TestHistogramOpError(unittest.TestCase): def test_type_errors(self): with program_guard(Program()): # The input type must be Variable. - self.assertRaises(TypeError, - paddle.histogram, - 1, - bins=5, - min=1, - max=5) + self.assertRaises( + TypeError, paddle.histogram, 1, bins=5, min=1, max=5 + ) # The input type must be 'int32', 'int64', 'float32', 'float64' x_bool = fluid.data(name='x_bool', shape=[4, 3], dtype='bool') - self.assertRaises(TypeError, - paddle.histogram, - x_bool, - bins=5, - min=1, - max=5) + self.assertRaises( + TypeError, paddle.histogram, x_bool, bins=5, min=1, max=5 + ) class TestHistogramOp(OpTest): - def setUp(self): self.op_type = "histogram" self.init_test_case() @@ -139,9 +136,9 @@ class TestHistogramOp(OpTest): self.python_api = paddle.histogram self.inputs = {"X": np_input} self.init_attrs() - Out, _ = np.histogram(np_input, - bins=self.bins, - range=(self.min, self.max)) + Out, _ = np.histogram( + np_input, bins=self.bins, range=(self.min, self.max) + ) self.outputs = {"Out": Out.astype(np.int64)} def init_test_case(self): diff --git a/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py b/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py index 66cb6ac427a..a905b9c2ae0 100644 --- a/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py +++ b/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py @@ -31,7 +31,6 @@ def find_latest_set(num): class CodeTable(object): - def __init__(self, num_classes, code): self.c = num_classes + code @@ -46,7 +45,6 @@ class CodeTable(object): class CodeTableWithCustomTree(object): - def __init__(self, path_table, path_code, index): self.ptable_ = path_table self.pcode_ = path_code @@ -129,8 +127,9 @@ def hsigmoid_grad(x, w, label, bias, num_classes): return [dx, dw, db] -def hsigmoidWithCustomTree(x, w, path_table, path_code, label, bias, - num_classes): +def hsigmoidWithCustomTree( + x, w, path_table, path_code, label, bias, num_classes +): batch_size = x.shape[0] code_length = len(path_table[0]) code_table = [0 for _ in range(code_length)] @@ -169,26 +168,36 @@ def hsigmoidWithCustomTree(x, w, path_table, path_code, label, bias, return pre_output, out -def python_api(input, - weight, - label, - path_table=None, - path_code=None, - bias=None, - num_classes=-1, - is_sparse=False, - remote_prefetch=False): - assert is_sparse == remote_prefetch, "is_sparse is equal to remote_prefetch in dygraph." - return paddle.nn.functional.hsigmoid_loss(input, label, num_classes, weight, - bias, path_table, path_code, - is_sparse) +def python_api( + input, + weight, + label, + path_table=None, + path_code=None, + bias=None, + num_classes=-1, + is_sparse=False, + remote_prefetch=False, +): + assert ( + is_sparse == remote_prefetch + ), "is_sparse is equal to remote_prefetch in dygraph." + return paddle.nn.functional.hsigmoid_loss( + input, + label, + num_classes, + weight, + bias, + path_table, + path_code, + is_sparse, + ) python_out_sig = ["Out"] class TestHSigmoidOp(OpTest): - def setUp(self): self.op_type = "hierarchical_sigmoid" self.python_api = python_api @@ -196,12 +205,15 @@ class TestHSigmoidOp(OpTest): num_classes = 101 feature_size = 5 batch_size = 20 - x = np.random.uniform(-1, 1, - (batch_size, feature_size)).astype('float64') - w = np.random.uniform(-1, 1, - (num_classes - 1, feature_size)).astype('float64') - label = np.random.randint(0, num_classes, - (batch_size, 1)).astype('int64') + x = np.random.uniform(-1, 1, (batch_size, feature_size)).astype( + 'float64' + ) + w = np.random.uniform(-1, 1, (num_classes - 1, feature_size)).astype( + 'float64' + ) + label = np.random.randint(0, num_classes, (batch_size, 1)).astype( + 'int64' + ) bias = np.random.uniform(-1, 1, (num_classes - 1, 1)).astype('float64') self.attrs = {'num_classes': num_classes, 'is_sparse': False} self.inputs = {'X': x, 'W': w, 'Label': label, 'Bias': bias} @@ -213,35 +225,48 @@ class TestHSigmoidOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'W', 'Bias'], ['Out'], - user_defined_grads=self.user_grads, - check_eager=True) + self.check_grad( + ['X', 'W', 'Bias'], + ['Out'], + user_defined_grads=self.user_grads, + check_eager=True, + ) @skip_check_grad_ci( - reason= - "For 'TestHSigmoidOpSparse', check_grad is separately calculated by 'TestHSigmoidOpWithSparseGrad'." + reason="For 'TestHSigmoidOpSparse', check_grad is separately calculated by 'TestHSigmoidOpWithSparseGrad'." ) class TestHSigmoidOpSparse(OpTest): - def setUp(self): self.op_type = "hierarchical_sigmoid" self.python_api = python_api self.python_out_sig = python_out_sig - num_classes = 6 #using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample + num_classes = 6 # using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample feature_size = 8 batch_size = 4 x = np.random.random((batch_size, feature_size)) w = np.random.random((num_classes - 1, feature_size)) label = np.array([0, 1, 4, 5]).astype('int64') - path_table = np.array([ - (0, 2, -1, -1, -1), (0, 1, 3, -1, -1), (0, 1, 4, -1, -1), - (0, 2, -1, -1, -1) - ]).astype( - 'int64') #np.array to store 1,2,5,6s' non-leaf path(root -> leaf) - path_code = np.array([(0, 0, -1, -1, -1), (1, 1, 1, -1, -1), - (1, 0, 0, -1, -1), (0, 1, -1, -1, -1) - ]).astype('int64') #np.array to store + path_table = np.array( + [ + (0, 2, -1, -1, -1), + (0, 1, 3, -1, -1), + (0, 1, 4, -1, -1), + (0, 2, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store 1,2,5,6s' non-leaf path(root -> leaf) + path_code = np.array( + [ + (0, 0, -1, -1, -1), + (1, 1, 1, -1, -1), + (1, 0, 0, -1, -1), + (0, 1, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store bias = np.random.random((num_classes - 1, 1)) self.attrs = {'num_classes': num_classes, 'is_sparse': True} self.inputs = { @@ -250,10 +275,11 @@ class TestHSigmoidOpSparse(OpTest): 'PathTable': path_table, 'PathCode': path_code, 'Label': label, - 'Bias': bias + 'Bias': bias, } - pre_output, out = hsigmoidWithCustomTree(x, w, path_table, path_code, - label, bias, num_classes) + pre_output, out = hsigmoidWithCustomTree( + x, w, path_table, path_code, label, bias, num_classes + ) self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): @@ -261,15 +287,14 @@ class TestHSigmoidOpSparse(OpTest): class TestHSigmoidOpWithSparseGrad(unittest.TestCase): - def hs_net_conf(self, is_sparse): input_word = fluid.layers.data(name="x", shape=[1], dtype='int64') - path_table = fluid.layers.data(name='path_table', - shape=[3], - dtype='int64') - path_code = fluid.layers.data(name='path_code', - shape=[3], - dtype='int64') + path_table = fluid.layers.data( + name='path_table', shape=[3], dtype='int64' + ) + path_code = fluid.layers.data( + name='path_code', shape=[3], dtype='int64' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') data_list = [input_word, path_table, path_code, label] @@ -278,17 +303,21 @@ class TestHSigmoidOpWithSparseGrad(unittest.TestCase): input=input_word, is_sparse=is_sparse, size=[3, 3], - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(3)))) - - cost = fluid.layers.hsigmoid(input=emb, - label=label, - bias_attr=True, - num_classes=3, - path_table=path_table, - path_code=path_code, - is_custom=True, - is_sparse=is_sparse) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal(scale=1 / math.sqrt(3)) + ), + ) + + cost = fluid.layers.hsigmoid( + input=emb, + label=label, + bias_attr=True, + num_classes=3, + path_table=path_table, + path_code=path_code, + is_custom=True, + is_sparse=is_sparse, + ) avg_cost = fluid.layers.reduce_mean(cost) @@ -315,45 +344,61 @@ class TestHSigmoidOpWithSparseGrad(unittest.TestCase): exe.run(start_up) result = list() for i in range(10): - data = [([[x[i % 2]]], [list(path_table[i % 2])], - [list(path_code[i % 2])], [label[i % 2]])] - - loss_val = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[loss]) + data = [ + ( + [[x[i % 2]]], + [list(path_table[i % 2])], + [list(path_code[i % 2])], + [label[i % 2]], + ) + ] + + loss_val = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[loss] + ) result.append(loss_val) return result def test_hs_grad_with_sparse(self): dense_result = self.training_test(is_sparse=False) sparse_result = self.training_test(is_sparse=True) - assert (dense_result == sparse_result) + assert dense_result == sparse_result @skip_check_grad_ci( - reason= - "[skip shape check] The huffman tree is structed separately. It will be complicated if use large shape." + reason="[skip shape check] The huffman tree is structed separately. It will be complicated if use large shape." ) class TestHSigmoidOpWithCostumTree(OpTest): - def setUp(self): self.op_type = "hierarchical_sigmoid" self.python_api = python_api self.python_out_sig = python_out_sig - num_classes = 6 #using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample + num_classes = 6 # using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample feature_size = 8 batch_size = 4 x = np.random.uniform(-1, 1, (batch_size, feature_size)) w = np.random.uniform(-1, 1, (num_classes - 1, feature_size)) label = np.array([0, 1, 4, 5]).astype('int64') - path_table = np.array([ - (0, 2, -1, -1, -1), (0, 1, 3, -1, -1), (0, 1, 4, -1, -1), - (0, 2, -1, -1, -1) - ]).astype( - 'int64') #np.array to store 1,2,5,6s' non-leaf path(root -> leaf) - path_code = np.array([(0, 0, -1, -1, -1), (1, 1, 1, -1, -1), - (1, 0, 0, -1, -1), (0, 1, -1, -1, -1) - ]).astype('int64') #np.array to store + path_table = np.array( + [ + (0, 2, -1, -1, -1), + (0, 1, 3, -1, -1), + (0, 1, 4, -1, -1), + (0, 2, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store 1,2,5,6s' non-leaf path(root -> leaf) + path_code = np.array( + [ + (0, 0, -1, -1, -1), + (1, 1, 1, -1, -1), + (1, 0, 0, -1, -1), + (0, 1, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store bias = np.random.random((num_classes - 1, 1)) self.attrs = {'num_classes': num_classes, 'is_sparse': False} self.inputs = { @@ -362,45 +407,59 @@ class TestHSigmoidOpWithCostumTree(OpTest): 'PathTable': path_table, 'PathCode': path_code, 'Label': label, - 'Bias': bias + 'Bias': bias, } - pre_output, out = hsigmoidWithCustomTree(x, w, path_table, path_code, - label, bias, num_classes) + pre_output, out = hsigmoidWithCustomTree( + x, w, path_table, path_code, label, bias, num_classes + ) self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['Bias', 'X', 'W'], ['Out'], - no_grad_set=set('Label'), - check_eager=True) + self.check_grad( + ['Bias', 'X', 'W'], + ['Out'], + no_grad_set=set('Label'), + check_eager=True, + ) @skip_check_grad_ci( - reason= - "[skip shape check] The huffman tree is structed separately. It will be complicated if use large shape." + reason="[skip shape check] The huffman tree is structed separately. It will be complicated if use large shape." ) class TestHSigmoidOpWithCostumTreeWithoutBias(OpTest): - def setUp(self): self.op_type = "hierarchical_sigmoid" self.python_api = python_api self.python_out_sig = python_out_sig - num_classes = 6 #using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample + num_classes = 6 # using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample feature_size = 8 batch_size = 4 x = np.random.uniform(-1, 1, (batch_size, feature_size)) w = np.random.uniform(-1, 1, (num_classes - 1, feature_size)) label = np.array([0, 1, 4, 5]).astype('int64') - path_table = np.array([ - (0, 2, -1, -1, -1), (0, 1, 3, -1, -1), (0, 1, 4, -1, -1), - (0, 2, -1, -1, -1) - ]).astype( - 'int64') #np.array to store 1,2,5,6s' non-leaf path(root -> leaf) - path_code = np.array([(0, 0, -1, -1, -1), (1, 1, 1, -1, -1), - (1, 0, 0, -1, -1), (0, 1, -1, -1, -1) - ]).astype('int64') #np.array to store + path_table = np.array( + [ + (0, 2, -1, -1, -1), + (0, 1, 3, -1, -1), + (0, 1, 4, -1, -1), + (0, 2, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store 1,2,5,6s' non-leaf path(root -> leaf) + path_code = np.array( + [ + (0, 0, -1, -1, -1), + (1, 1, 1, -1, -1), + (1, 0, 0, -1, -1), + (0, 1, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store # bias = np.random.random((num_classes - 1, 1)).astype("float32") self.attrs = {'num_classes': num_classes, 'is_sparse': False} self.inputs = { @@ -410,22 +469,24 @@ class TestHSigmoidOpWithCostumTreeWithoutBias(OpTest): 'PathCode': path_code, 'Label': label, } - pre_output, out = hsigmoidWithCustomTree(x=x, - w=w, - path_table=path_table, - path_code=path_code, - label=label, - bias=None, - num_classes=num_classes) + pre_output, out = hsigmoidWithCustomTree( + x=x, + w=w, + path_table=path_table, + path_code=path_code, + label=label, + bias=None, + num_classes=num_classes, + ) self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'W'], ['Out'], - no_grad_set=set('Label'), - check_eager=True) + self.check_grad( + ['X', 'W'], ['Out'], no_grad_set=set('Label'), check_eager=True + ) class TestHSigmoidLossAPI(unittest.TestCase): @@ -441,27 +502,38 @@ class TestHSigmoidLossAPI(unittest.TestCase): paddle.set_default_dtype(self.dtype) self.x_np = np.random.uniform( - -1, 1, [self.batch_size, self.feature_size]).astype(self.dtype) - self.labels_np = np.random.randint(self.num_classes, - size=(self.batch_size, 1), - dtype='int64') + -1, 1, [self.batch_size, self.feature_size] + ).astype(self.dtype) + self.labels_np = np.random.randint( + self.num_classes, size=(self.batch_size, 1), dtype='int64' + ) self.weight_np = np.random.uniform( - -1, 1, [self.num_classes - 1, self.feature_size]).astype(self.dtype) - self.bias_np = np.random.uniform( - -1, 1, (self.num_classes - 1, )).astype(self.dtype) + -1, 1, [self.num_classes - 1, self.feature_size] + ).astype(self.dtype) + self.bias_np = np.random.uniform(-1, 1, (self.num_classes - 1,)).astype( + self.dtype + ) self.path_table_np = None self.path_code_np = None - _, self.out_np = hsigmoid(self.x_np, self.weight_np, self.labels_np, - self.bias_np, self.num_classes) + _, self.out_np = hsigmoid( + self.x_np, + self.weight_np, + self.labels_np, + self.bias_np, + self.num_classes, + ) self.set_attrs() if self.is_custom: - _, self.out_np = hsigmoidWithCustomTree(self.x_np, self.weight_np, - self.path_table_np, - self.path_code_np, - self.labels_np, - self.bias_np.reshape(-1, 1), - self.num_classes) + _, self.out_np = hsigmoidWithCustomTree( + self.x_np, + self.weight_np, + self.path_table_np, + self.path_code_np, + self.labels_np, + self.bias_np.reshape(-1, 1), + self.num_classes, + ) def set_attrs(self): pass @@ -477,13 +549,19 @@ class TestHSigmoidLossAPI(unittest.TestCase): if self.is_custom: path_table = paddle.to_tensor(self.path_table_np) path_code = paddle.to_tensor(self.path_code_np) - out1 = F.hsigmoid_loss(x, labels, self.num_classes, weight, bias, - path_table, path_code) + out1 = F.hsigmoid_loss( + x, labels, self.num_classes, weight, bias, path_table, path_code + ) weight_attr = I.NumpyArrayInitializer(self.weight_np) bias_attr = I.NumpyArrayInitializer(self.bias_np) - m = paddle.nn.HSigmoidLoss(self.feature_size, self.num_classes, - weight_attr, bias_attr, self.is_custom) + m = paddle.nn.HSigmoidLoss( + self.feature_size, + self.num_classes, + weight_attr, + bias_attr, + self.is_custom, + ) out2 = m(x, labels, path_table, path_code) for out in [out1, out2]: @@ -497,23 +575,34 @@ class TestHSigmoidLossAPI(unittest.TestCase): x = paddle.static.data('x', [-1, self.feature_size]) labels = paddle.static.data('labels', [-1, 1], 'int64') weight = paddle.static.data('weight', [-1, self.feature_size]) - bias = paddle.static.data('bias', [ - -1, - ]) + bias = paddle.static.data( + 'bias', + [ + -1, + ], + ) path_table = None path_code = None if self.is_custom: path_table = paddle.static.data('path_table', [-1, -1], 'int64') path_code = paddle.static.data('path_code', [-1, -1], 'int64') - out1 = F.hsigmoid_loss(x, labels, self.num_classes, weight, bias, - path_table, path_code) + out1 = F.hsigmoid_loss( + x, labels, self.num_classes, weight, bias, path_table, path_code + ) weight_attr = paddle.framework.ParamAttr( - initializer=I.NumpyArrayInitializer(self.weight_np)) + initializer=I.NumpyArrayInitializer(self.weight_np) + ) bias_attr = paddle.framework.ParamAttr( - initializer=I.NumpyArrayInitializer(self.bias_np)) - m = paddle.nn.HSigmoidLoss(self.feature_size, self.num_classes, - weight_attr, bias_attr, self.is_custom) + initializer=I.NumpyArrayInitializer(self.bias_np) + ) + m = paddle.nn.HSigmoidLoss( + self.feature_size, + self.num_classes, + weight_attr, + bias_attr, + self.is_custom, + ) out2 = m(x, labels, path_table, path_code) exe = paddle.static.Executor(self.place) @@ -522,14 +611,14 @@ class TestHSigmoidLossAPI(unittest.TestCase): 'x': self.x_np, 'labels': self.labels_np, 'weight': self.weight_np, - 'bias': self.bias_np + 'bias': self.bias_np, } if self.is_custom: feed_dict["path_code"] = self.path_code_np feed_dict["path_table"] = self.path_table_np - ret1, ret2 = exe.run(train_program, - feed=feed_dict, - fetch_list=[out1, out2]) + ret1, ret2 = exe.run( + train_program, feed=feed_dict, fetch_list=[out1, out2] + ) for ret in [ret1, ret2]: np.testing.assert_allclose(self.out_np, ret, rtol=1e-05) @@ -547,9 +636,17 @@ class TestHSigmoidLossAPI(unittest.TestCase): path_code = fluid.data('path_code', [-1, -1], 'int64') weight_attr = I.NumpyArrayInitializer(self.weight_np) bias_attr = I.NumpyArrayInitializer(self.bias_np) - out = fluid.layers.hsigmoid(x, labels, self.num_classes, - weight_attr, bias_attr, 'out', - path_table, path_code, self.is_custom) + out = fluid.layers.hsigmoid( + x, + labels, + self.num_classes, + weight_attr, + bias_attr, + 'out', + path_table, + path_code, + self.is_custom, + ) exe = fluid.Executor(self.place) exe.run(startup_program) @@ -557,13 +654,14 @@ class TestHSigmoidLossAPI(unittest.TestCase): if self.is_custom: feed_dict["path_code"] = self.path_code_np feed_dict["path_table"] = self.path_table_np - ret, = exe.run(train_program, feed=feed_dict, fetch_list=[out]) + (ret,) = exe.run(train_program, feed=feed_dict, fetch_list=[out]) np.testing.assert_allclose(ret, self.out_np, rtol=1e-05) def test_errors(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): # test paddle.nn.HSigmoidLoss self.assertRaises(ValueError, paddle.nn.HSigmoidLoss, 6, 1) @@ -574,46 +672,52 @@ class TestHSigmoidLossAPI(unittest.TestCase): bias = paddle.static.data('bias', [7]) x_int32 = paddle.static.data('x_int32', [4, 6], 'int32') - self.assertRaises(TypeError, F.hsigmoid_loss, x_int32, label, 8, - weight) + self.assertRaises( + TypeError, F.hsigmoid_loss, x_int32, label, 8, weight + ) - label_float32 = paddle.static.data('label_float32', [4, 1], - 'float32') - self.assertRaises(TypeError, F.hsigmoid_loss, x, label_float32, 8, - weight) + label_float32 = paddle.static.data( + 'label_float32', [4, 1], 'float32' + ) + self.assertRaises( + TypeError, F.hsigmoid_loss, x, label_float32, 8, weight + ) weight_int32 = paddle.static.data('weight_int32', [7, 6], 'int32') - self.assertRaises(TypeError, F.hsigmoid_loss, x, label, 8, - weight_int32) + self.assertRaises( + TypeError, F.hsigmoid_loss, x, label, 8, weight_int32 + ) bias_int32 = paddle.static.data('bias_int32', [7], 'int32') - self.assertRaises(TypeError, - F.hsigmoid_loss, - x, - label, - 8, - weight, - bias=bias_int32) - - path_table_int32 = paddle.static.data('path_table_int32', [7], - 'int32') - self.assertRaises(TypeError, - F.hsigmoid_loss, - x, - label, - 8, - weight, - path_table=path_table_int32) - - path_code_int32 = paddle.static.data('path_code_int32', [7], - 'int32') - self.assertRaises(TypeError, - F.hsigmoid_loss, - x, - label, - 8, - weight, - path_code=path_code_int32) + self.assertRaises( + TypeError, F.hsigmoid_loss, x, label, 8, weight, bias=bias_int32 + ) + + path_table_int32 = paddle.static.data( + 'path_table_int32', [7], 'int32' + ) + self.assertRaises( + TypeError, + F.hsigmoid_loss, + x, + label, + 8, + weight, + path_table=path_table_int32, + ) + + path_code_int32 = paddle.static.data( + 'path_code_int32', [7], 'int32' + ) + self.assertRaises( + TypeError, + F.hsigmoid_loss, + x, + label, + 8, + weight, + path_code=path_code_int32, + ) # test paddle.nn.HSigmoidLoss paddle.disable_static(self.place) @@ -636,8 +740,9 @@ class TestHSigmoidLossAPI(unittest.TestCase): self.assertRaises(TypeError, fluid.layers.hsigmoid, 1, label, 2) # The input dtype must be float16, float32, float64. x_int32 = fluid.data(name='x_int32', shape=[4, 3], dtype='int32') - self.assertRaises(TypeError, fluid.layers.hsigmoid, x_int32, label, - 2) + self.assertRaises( + TypeError, fluid.layers.hsigmoid, x_int32, label, 2 + ) # support the input dtype is float32 x_fp32 = fluid.data(name='x_fp32', shape=[4, 3], dtype='float32') fluid.layers.hsigmoid(x_fp32, label, 2) @@ -646,20 +751,30 @@ class TestHSigmoidLossAPI(unittest.TestCase): self.assertRaises(TypeError, fluid.layers.hsigmoid, x_fp32, 1, 2) # The label dtype must be int64. label_int32 = fluid.data('label_int32', [4, 1], 'int32') - self.assertRaises(TypeError, fluid.layers.hsigmoid, x_fp32, - label_int32, 2) + self.assertRaises( + TypeError, fluid.layers.hsigmoid, x_fp32, label_int32, 2 + ) class TestHSigmoidLossAPICustom(TestHSigmoidLossAPI): - def set_attrs(self): self.is_custom = True - self.path_table_np = np.array([(0, 2, -1, -1, -1), (0, 1, 3, -1, -1), - (0, 1, 4, -1, -1), - (0, 2, -1, -1, -1)]).astype(np.int64) - self.path_code_np = np.array([(0, 0, -1, -1, -1), (1, 1, 1, -1, -1), - (1, 0, 0, -1, -1), - (0, 1, -1, -1, -1)]).astype(np.int64) + self.path_table_np = np.array( + [ + (0, 2, -1, -1, -1), + (0, 1, 3, -1, -1), + (0, 1, 4, -1, -1), + (0, 2, -1, -1, -1), + ] + ).astype(np.int64) + self.path_code_np = np.array( + [ + (0, 0, -1, -1, -1), + (1, 1, 1, -1, -1), + (1, 0, 0, -1, -1), + (0, 1, -1, -1, -1), + ] + ).astype(np.int64) def test_errors(self): pass diff --git a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py index d679d2b719f..778fedfd4aa 100644 --- a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py @@ -29,7 +29,6 @@ def huber_loss_forward(val, delta): class TestHuberLossOp(OpTest): - def setUp(self): self.op_type = 'huber_loss' self.python_api = paddle.fluid.layers.huber_loss @@ -38,16 +37,17 @@ class TestHuberLossOp(OpTest): self.init_input() shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - self.delta).astype('float32') + loss = np.vectorize(huber_loss_forward)(residual, self.delta).astype( + 'float32' + ) self.attrs = {'delta': self.delta} self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def init_input(self): shape = self.set_shape() self.inputs = { - 'X': np.random.uniform(0, 1., shape).astype('float32'), - 'Y': np.random.uniform(0, 1., shape).astype('float32'), + 'X': np.random.uniform(0, 1.0, shape).astype('float32'), + 'Y': np.random.uniform(0, 1.0, shape).astype('float32'), } def set_shape(self): @@ -60,38 +60,32 @@ class TestHuberLossOp(OpTest): self.check_grad(['X', 'Y'], 'Out', check_eager=True) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.008, - no_grad_set=set("residual")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.008, no_grad_set=set("residual") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.008, - no_grad_set=set('residual')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.008, no_grad_set=set('residual') + ) def TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (64) + return 64 def TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (6, 6) def TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (6, 6, 1) class TestHuberLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input and label must be Variable @@ -106,10 +100,12 @@ class TestHuberLossOpError(unittest.TestCase): # the dtype of input and label must be float32 or float64 xw2 = fluid.data(name='xw2', shape=[None, 6], dtype="int32") lw2 = fluid.data(name='lw2', shape=[None, 6], dtype="int32") - self.assertRaises(TypeError, fluid.layers.huber_loss, xw2, lr, - delta) - self.assertRaises(TypeError, fluid.layers.huber_loss, xr, lw2, - delta) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xw2, lr, delta + ) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xr, lw2, delta + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_hybrid_parallel_topology.py b/python/paddle/fluid/tests/unittests/test_hybrid_parallel_topology.py index 00b1f10ede8..d9210c128dc 100644 --- a/python/paddle/fluid/tests/unittests/test_hybrid_parallel_topology.py +++ b/python/paddle/fluid/tests/unittests/test_hybrid_parallel_topology.py @@ -18,7 +18,6 @@ import numpy as np class TestCommunicateTopology(unittest.TestCase): - def test_topology(self): topo = fleet.CommunicateTopology(["dp", "mp", "pp"], [2, 2, 2]) @@ -33,8 +32,9 @@ class TestCommunicateTopology(unittest.TestCase): # test get_hybrid_group_names parallel_names = ["dp", "mp", "pp"] - np.testing.assert_array_equal(parallel_names, - topo.get_hybrid_group_names()) + np.testing.assert_array_equal( + parallel_names, topo.get_hybrid_group_names() + ) # test get_dims np.testing.assert_array_equal(2, topo.get_dim("dp")) @@ -78,29 +78,64 @@ class TestCommunicateTopology(unittest.TestCase): self.assertEqual(topo.get_dim_size("pp"), 2) def test_topology_4D(self): - topo = fleet.CommunicateTopology(["dp", "pp", "sharding", "mp"], - [2, 2, 2, 2]) + topo = fleet.CommunicateTopology( + ["dp", "pp", "sharding", "mp"], [2, 2, 2, 2] + ) # test get_comm_list - dp_comm_list = [[0, 8], [1, 9], [2, 10], [3, 11], [4, 12], [5, 13], - [6, 14], [7, 15]] - mp_comm_list = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], - [12, 13], [14, 15]] - pp_comm_list = [[0, 4], [1, 5], [2, 6], [3, 7], [8, 12], [9, 13], - [10, 14], [11, 15]] - sharding_comm_list = [[0, 2], [1, 3], [4, 6], [5, 7], [8, 10], [9, 11], - [12, 14], [13, 15]] + dp_comm_list = [ + [0, 8], + [1, 9], + [2, 10], + [3, 11], + [4, 12], + [5, 13], + [6, 14], + [7, 15], + ] + mp_comm_list = [ + [0, 1], + [2, 3], + [4, 5], + [6, 7], + [8, 9], + [10, 11], + [12, 13], + [14, 15], + ] + pp_comm_list = [ + [0, 4], + [1, 5], + [2, 6], + [3, 7], + [8, 12], + [9, 13], + [10, 14], + [11, 15], + ] + sharding_comm_list = [ + [0, 2], + [1, 3], + [4, 6], + [5, 7], + [8, 10], + [9, 11], + [12, 14], + [13, 15], + ] np.testing.assert_array_equal(dp_comm_list, topo.get_comm_list("dp")) np.testing.assert_array_equal(mp_comm_list, topo.get_comm_list("mp")) np.testing.assert_array_equal(pp_comm_list, topo.get_comm_list("pp")) - np.testing.assert_array_equal(sharding_comm_list, - topo.get_comm_list("sharding")) + np.testing.assert_array_equal( + sharding_comm_list, topo.get_comm_list("sharding") + ) # test get_hybrid_group_names parallel_names = ["dp", "pp", "sharding", "mp"] - np.testing.assert_array_equal(parallel_names, - topo.get_hybrid_group_names()) + np.testing.assert_array_equal( + parallel_names, topo.get_hybrid_group_names() + ) # test get_dims np.testing.assert_array_equal(2, topo.get_dim("dp")) @@ -149,20 +184,27 @@ class TestCommunicateTopology(unittest.TestCase): # test get_axis_list self.assertEqual(topo.get_axis_list("dp", 0), [0, 1, 2, 3, 4, 5, 6, 7]) - self.assertEqual(topo.get_axis_list("dp", 1), - [8, 9, 10, 11, 12, 13, 14, 15]) - self.assertEqual(topo.get_axis_list("mp", 0), - [0, 2, 4, 6, 8, 10, 12, 14]) - self.assertEqual(topo.get_axis_list("mp", 1), - [1, 3, 5, 7, 9, 11, 13, 15]) - self.assertEqual(topo.get_axis_list("pp", 0), - [0, 1, 2, 3, 8, 9, 10, 11]) - self.assertEqual(topo.get_axis_list("pp", 1), - [4, 5, 6, 7, 12, 13, 14, 15]) - self.assertEqual(topo.get_axis_list("sharding", 0), - [0, 1, 4, 5, 8, 9, 12, 13]) - self.assertEqual(topo.get_axis_list("sharding", 1), - [2, 3, 6, 7, 10, 11, 14, 15]) + self.assertEqual( + topo.get_axis_list("dp", 1), [8, 9, 10, 11, 12, 13, 14, 15] + ) + self.assertEqual( + topo.get_axis_list("mp", 0), [0, 2, 4, 6, 8, 10, 12, 14] + ) + self.assertEqual( + topo.get_axis_list("mp", 1), [1, 3, 5, 7, 9, 11, 13, 15] + ) + self.assertEqual( + topo.get_axis_list("pp", 0), [0, 1, 2, 3, 8, 9, 10, 11] + ) + self.assertEqual( + topo.get_axis_list("pp", 1), [4, 5, 6, 7, 12, 13, 14, 15] + ) + self.assertEqual( + topo.get_axis_list("sharding", 0), [0, 1, 4, 5, 8, 9, 12, 13] + ) + self.assertEqual( + topo.get_axis_list("sharding", 1), [2, 3, 6, 7, 10, 11, 14, 15] + ) # test get_dim_size self.assertEqual(topo.get_dim_size("dp"), 2) diff --git a/python/paddle/fluid/tests/unittests/test_identity_loss_op.py b/python/paddle/fluid/tests/unittests/test_identity_loss_op.py index de81c16da7d..0d808e7be34 100644 --- a/python/paddle/fluid/tests/unittests/test_identity_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_identity_loss_op.py @@ -21,7 +21,6 @@ from op_test import OpTest class TestIdentityLossOp(OpTest): - def setUp(self): self.max_relative_error = 0.006 self.python_api = paddle.incubate.identity_loss @@ -61,34 +60,29 @@ class TestIdentityLossOp(OpTest): class TestCase1(TestIdentityLossOp): - def initTestCase(self): self.shape = (8, 16, 8) self.reduction = 0 class TestCase2(TestIdentityLossOp): - def initTestCase(self): self.shape = (8, 16) self.reduction = 1 class TestCase3(TestIdentityLossOp): - def initTestCase(self): self.shape = (4, 8, 16) self.reduction = 2 class TestIdentityLossFloat32(TestIdentityLossOp): - def set_attrs(self): self.dtype = 'float32' class TestIdentityLossOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -100,8 +94,9 @@ class TestIdentityLossOpError(unittest.TestCase): self.assertRaises(Exception, test_int) def test_string(): - paddle.incubate.identity_loss(x=input_data, - reduction="wrongkey") + paddle.incubate.identity_loss( + x=input_data, reduction="wrongkey" + ) self.assertRaises(Exception, test_string) @@ -114,7 +109,6 @@ class TestIdentityLossOpError(unittest.TestCase): class TestIdentityLossAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) @@ -138,13 +132,14 @@ class TestIdentityLossAPI(unittest.TestCase): out4 = paddle.incubate.identity_loss(x, reduction=2) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': self.x}, - fetch_list=[out1, out2, out3, out4]) + res = exe.run( + feed={'X': self.x}, fetch_list=[out1, out2, out3, out4] + ) ref = [ self.identity_loss_ref(self.x, 2), self.identity_loss_ref(self.x, 0), self.identity_loss_ref(self.x, 1), - self.identity_loss_ref(self.x, 2) + self.identity_loss_ref(self.x, 2), ] for out, out_ref in zip(res, ref): np.testing.assert_allclose(out, out_ref, rtol=0.0001) @@ -172,8 +167,9 @@ class TestIdentityLossAPI(unittest.TestCase): x = paddle.to_tensor(x) self.assertRaises(Exception, paddle.incubate.identity_loss, x, -1) self.assertRaises(Exception, paddle.incubate.identity_loss, x, 3) - self.assertRaises(Exception, paddle.incubate.identity_loss, x, - "wrongkey") + self.assertRaises( + Exception, paddle.incubate.identity_loss, x, "wrongkey" + ) paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('X', [10, 12], 'int32') diff --git a/python/paddle/fluid/tests/unittests/test_identity_op.py b/python/paddle/fluid/tests/unittests/test_identity_op.py index f257db187eb..f798b421cfb 100644 --- a/python/paddle/fluid/tests/unittests/test_identity_op.py +++ b/python/paddle/fluid/tests/unittests/test_identity_op.py @@ -18,7 +18,6 @@ import paddle class TestIdentityAPI(unittest.TestCase): - def setUp(self): self.shape = [4, 4] self.x = np.random.random((4, 4)).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py b/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py index 9debbccdb3d..bacb13a0981 100644 --- a/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py +++ b/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py @@ -18,21 +18,27 @@ import numpy as np class TestIInfoAndFInfoAPI(unittest.TestCase): - def test_invalid_input(self): for dtype in [ - paddle.float16, paddle.float32, paddle.float64, paddle.bfloat16, - paddle.complex64, paddle.complex128, paddle.bool + paddle.float16, + paddle.float32, + paddle.float64, + paddle.bfloat16, + paddle.complex64, + paddle.complex128, + paddle.bool, ]: with self.assertRaises(ValueError): _ = paddle.iinfo(dtype) def test_iinfo(self): - for paddle_dtype, np_dtype in [(paddle.int64, np.int64), - (paddle.int32, np.int32), - (paddle.int16, np.int16), - (paddle.int8, np.int8), - (paddle.uint8, np.uint8)]: + for paddle_dtype, np_dtype in [ + (paddle.int64, np.int64), + (paddle.int32, np.int32), + (paddle.int16, np.int16), + (paddle.int8, np.int8), + (paddle.uint8, np.uint8), + ]: xinfo = paddle.iinfo(paddle_dtype) xninfo = np.iinfo(np_dtype) self.assertEqual(xinfo.bits, xninfo.bits) diff --git a/python/paddle/fluid/tests/unittests/test_im2sequence_op.py b/python/paddle/fluid/tests/unittests/test_im2sequence_op.py index 5a90f79dd7e..24a5e696a10 100644 --- a/python/paddle/fluid/tests/unittests/test_im2sequence_op.py +++ b/python/paddle/fluid/tests/unittests/test_im2sequence_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -39,26 +39,58 @@ def get_output_shape(attrs, in_shape, img_real_size): imgreal_w = img_real_size[index, 1] / out_stride[1] else: imgreal_w = img_real_size[index, 0] / out_stride[1] + 1 - output_height[0,index] = \ - 1 + \ - (imgreal_h + paddings[0] + paddings[2] - kernels[0] + strides[0] - 1) / \ - strides[0] - - output_width[0,index] = \ - 1 + \ - (imgreal_w + paddings[1] + paddings[3] - kernels[1] + strides[1] - 1) / \ - strides[1] + output_height[0, index] = ( + 1 + + ( + imgreal_h + + paddings[0] + + paddings[2] + - kernels[0] + + strides[0] + - 1 + ) + / strides[0] + ) + + output_width[0, index] = ( + 1 + + ( + imgreal_w + + paddings[1] + + paddings[3] + - kernels[1] + + strides[1] + - 1 + ) + / strides[1] + ) else: for index in range(batchsize): - output_height[0,index] = \ - 1 + \ - (img_height + paddings[0] + paddings[2] - kernels[0] + strides[0] - 1) / \ - strides[0] - - output_width[0,index] = \ - 1 + \ - (img_width + paddings[1] + paddings[3] - kernels[1] + strides[1] - 1) / \ - strides[1] + output_height[0, index] = ( + 1 + + ( + img_height + + paddings[0] + + paddings[2] + - kernels[0] + + strides[0] + - 1 + ) + / strides[0] + ) + + output_width[0, index] = ( + 1 + + ( + img_width + + paddings[1] + + paddings[3] + - kernels[1] + + strides[1] + - 1 + ) + / strides[1] + ) return output_height, output_width @@ -80,50 +112,70 @@ def im2col(attrs, im, col): for channel in range(0, input_channels): for filter_row_idx in range(0, filter_height): for filter_col_idx in range(0, filter_width): - im_row_offset = col_row_idx * stride_height \ - + filter_row_idx - padding_height - - im_col_offset = col_col_idx * stride_width \ - + filter_col_idx - padding_width - - if (im_row_offset < 0 or im_row_offset >= input_height - or im_col_offset < 0 - or im_col_offset >= input_width): - col[col_row_idx][col_col_idx][channel][\ - filter_row_idx][filter_col_idx] = 0.0 + im_row_offset = ( + col_row_idx * stride_height + + filter_row_idx + - padding_height + ) + + im_col_offset = ( + col_col_idx * stride_width + + filter_col_idx + - padding_width + ) + + if ( + im_row_offset < 0 + or im_row_offset >= input_height + or im_col_offset < 0 + or im_col_offset >= input_width + ): + col[col_row_idx][col_col_idx][channel][ + filter_row_idx + ][filter_col_idx] = 0.0 else: - im_offset = (channel * input_height + im_row_offset \ - ) * input_width + im_col_offset + im_offset = ( + channel * input_height + im_row_offset + ) * input_width + im_col_offset - col[col_row_idx][col_col_idx][channel][\ - filter_row_idx][filter_col_idx] = im[channel][ \ - im_row_offset][im_col_offset] + col[col_row_idx][col_col_idx][channel][ + filter_row_idx + ][filter_col_idx] = im[channel][im_row_offset][ + im_col_offset + ] def Im2Sequence(inputs, img_real_size, attrs): - output_height, output_width = get_output_shape(attrs, inputs.shape, - img_real_size) + output_height, output_width = get_output_shape( + attrs, inputs.shape, img_real_size + ) img_channels = inputs.shape[1] batch_size = inputs.shape[0] out = [] for index in range(batch_size): - tmp = np.zeros([ - output_height[0, index], output_width[0, index], img_channels, - attrs['kernels'][0], attrs['kernels'][1] - ]).astype("float32") + tmp = np.zeros( + [ + output_height[0, index], + output_width[0, index], + img_channels, + attrs['kernels'][0], + attrs['kernels'][1], + ] + ).astype("float32") out.append(tmp) for index in range(len(inputs)): im2col(attrs, inputs[index], out[index]) - out[index] = out[index].reshape([ - output_height[0, index] * output_width[0, index], - img_channels * attrs['kernels'][0] * attrs['kernels'][1] - ]) + out[index] = out[index].reshape( + [ + output_height[0, index] * output_width[0, index], + img_channels * attrs['kernels'][0] * attrs['kernels'][1], + ] + ) out = np.concatenate(out, axis=0) return out class TestBlockExpandOp(OpTest): - def config(self): self.batch_size = 1 self.img_channels = 3 @@ -138,9 +190,16 @@ class TestBlockExpandOp(OpTest): def setUp(self): self.config() self.op_type = "im2sequence" - x = np.random.uniform(0.1, 1, [ - self.batch_size, self.img_channels, self.img_height, self.img_width - ]).astype("float32") + x = np.random.uniform( + 0.1, + 1, + [ + self.batch_size, + self.img_channels, + self.img_height, + self.img_width, + ], + ).astype("float32") real_size = np.array([]).astype("float32") out = Im2Sequence(x, real_size, self.attrs) self.inputs = {'X': x} @@ -154,7 +213,6 @@ class TestBlockExpandOp(OpTest): class TestBlockExpandOpCase2(TestBlockExpandOp): - def config(self): self.batch_size = 2 self.img_channels = 3 @@ -168,7 +226,6 @@ class TestBlockExpandOpCase2(TestBlockExpandOp): class TestBlockExpandOpCase3(TestBlockExpandOp): - def config(self): self.batch_size = 6 self.img_channels = 1 @@ -182,7 +239,6 @@ class TestBlockExpandOpCase3(TestBlockExpandOp): class TestBlockExpandOpCase4(TestBlockExpandOp): - def config(self): self.batch_size = 6 self.img_channels = 2 @@ -196,11 +252,9 @@ class TestBlockExpandOpCase4(TestBlockExpandOp): @skip_check_grad_ci( - reason= - "Since 'real_size' is used just in forward computation, we don't test the gradient here." + reason="Since 'real_size' is used just in forward computation, we don't test the gradient here." ) class TestBlockExpandOpCase5(OpTest): - def config(self): self.batch_size = 1 self.img_channels = 3 @@ -217,9 +271,16 @@ class TestBlockExpandOpCase5(OpTest): def setUp(self): self.config() self.op_type = "im2sequence" - x = np.random.uniform(0.1, 1, [ - self.batch_size, self.img_channels, self.img_height, self.img_width - ]).astype("float32") + x = np.random.uniform( + 0.1, + 1, + [ + self.batch_size, + self.img_channels, + self.img_height, + self.img_width, + ], + ).astype("float32") out = np.array(Im2Sequence(x, self.real_size, self.attrs)) self.inputs = {'X': x, 'Y': self.real_size} self.outputs = {'Out': out} @@ -229,7 +290,6 @@ class TestBlockExpandOpCase5(OpTest): class TestBlockExpandOpCase6(TestBlockExpandOpCase5): - def config(self): self.batch_size = 3 self.img_channels = 1 @@ -245,7 +305,6 @@ class TestBlockExpandOpCase6(TestBlockExpandOpCase5): class TestBlockExpandOpCase7(TestBlockExpandOpCase6): - def config(self): self.batch_size = 2 self.img_channels = 2 diff --git a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py index fc3fee1ec65..fbcf634634b 100644 --- a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py +++ b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py @@ -20,26 +20,27 @@ from paddle.fluid.framework import Program def conv_block(input, num_filter, groups, dropouts): - return nets.img_conv_group(input=input, - pool_size=2, - pool_stride=2, - conv_num_filter=[num_filter] * groups, - conv_filter_size=3, - conv_act='relu', - conv_with_batchnorm=True, - conv_batchnorm_drop_rate=dropouts, - pool_type='max') + return nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max', + ) class TestLayer(unittest.TestCase): - def test_batch_norm_layer(self): main_program = Program() startup_program = Program() with fluid.program_guard(main_program, startup_program): - images = fluid.layers.data(name='pixel', - shape=[3, 48, 48], - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32' + ) hidden1 = fluid.layers.batch_norm(input=images) hidden2 = fluid.layers.fc(input=hidden1, size=128, act='relu') fluid.layers.batch_norm(input=hidden2) @@ -50,9 +51,9 @@ class TestLayer(unittest.TestCase): main_program = Program() startup_program = Program() with fluid.program_guard(main_program, startup_program): - images = fluid.layers.data(name='pixel', - shape=[3, 48, 48], - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32' + ) fluid.layers.dropout(x=images, dropout_prob=0.5) print(str(main_program)) @@ -62,9 +63,9 @@ class TestLayer(unittest.TestCase): startup_program = Program() with fluid.program_guard(main_program, startup_program): - images = fluid.layers.data(name='pixel', - shape=[3, 48, 48], - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32' + ) conv1 = conv_block(images, 64, 2, [0.3, 0]) conv_block(conv1, 256, 3, [0.4, 0.4, 0]) @@ -74,12 +75,12 @@ class TestLayer(unittest.TestCase): main_program = Program() startup_program = Program() with fluid.program_guard(main_program, startup_program): - image1 = fluid.layers.data(name='pixel1', - shape=[3, 48, 48], - dtype='float32') - image2 = fluid.layers.data(name='pixel2', - shape=[3, 48, 48], - dtype='float32') + image1 = fluid.layers.data( + name='pixel1', shape=[3, 48, 48], dtype='float32' + ) + image2 = fluid.layers.data( + name='pixel2', shape=[3, 48, 48], dtype='float32' + ) fluid.layers.elementwise_add(x=image1, y=image2, act='relu') print(main_program) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_auto_prune.py b/python/paddle/fluid/tests/unittests/test_imperative_auto_prune.py index 0cd97cbf328..3cda7d5d216 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_auto_prune.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_auto_prune.py @@ -20,19 +20,20 @@ from paddle.fluid.framework import _test_eager_guard class AutoPruneLayer0(fluid.Layer): - def __init__(self, input_size): super(AutoPruneLayer0, self).__init__() self.linear1 = fluid.dygraph.Linear( input_size, 5, param_attr=fluid.initializer.ConstantInitializer(value=2), - bias_attr=False) + bias_attr=False, + ) self.linear2 = fluid.dygraph.Linear( 5, 5, param_attr=fluid.initializer.ConstantInitializer(value=2), - bias_attr=False) + bias_attr=False, + ) def forward(self, x, y): a = self.linear1(x) @@ -43,19 +44,20 @@ class AutoPruneLayer0(fluid.Layer): class AutoPruneLayer1(fluid.Layer): - def __init__(self, input_size): super(AutoPruneLayer1, self).__init__() self.linear1 = fluid.dygraph.Linear( input_size, 5, param_attr=fluid.initializer.ConstantInitializer(value=2), - bias_attr=False) + bias_attr=False, + ) self.linear2 = fluid.dygraph.Linear( 5, 5, param_attr=fluid.initializer.ConstantInitializer(value=2), - bias_attr=False) + bias_attr=False, + ) def forward(self, x, y): a = self.linear1(x) @@ -67,7 +69,6 @@ class AutoPruneLayer1(fluid.Layer): class AutoPruneLayer2(fluid.Layer): - def __init__(self, input_size): super(AutoPruneLayer2, self).__init__() self.linear = fluid.dygraph.Linear(input_size, 10, act=None) @@ -85,16 +86,15 @@ class AutoPruneLayer2(fluid.Layer): class AutoPruneLayer3(fluid.Layer): - def __init__(self, input_size): super(AutoPruneLayer3, self).__init__() self.linear = fluid.dygraph.Linear(input_size, 20, act=None) def forward(self, x, label, test_num): feature = self.linear(x) - part1, part2 = fluid.layers.split(feature, - num_or_sections=[10, 10], - dim=1) + part1, part2 = fluid.layers.split( + feature, num_or_sections=[10, 10], dim=1 + ) # Note that: part2 is not used. loss = fluid.layers.cross_entropy(input=part1, label=label) loss = paddle.mean(loss) @@ -105,7 +105,6 @@ class AutoPruneLayer3(fluid.Layer): class MyLayer(fluid.Layer): - def __init__(self, input_size, vocab_size, size, dtype="float32"): super(MyLayer, self).__init__(dtype=dtype) self.embed0 = fluid.Embedding(size=(vocab_size, size)) @@ -128,7 +127,6 @@ class MyLayer(fluid.Layer): class MyLayer2(fluid.Layer): - def __init__(self, input_size, vocab_size, size, dtype="float32"): super(MyLayer2, self).__init__(dtype=dtype) self.embed0 = fluid.Embedding(size=(vocab_size, size)) @@ -140,8 +138,9 @@ class MyLayer2(fluid.Layer): # mind the difference with MyLayer # In this example, the forward method involes all params loss = fluid.layers.reduce_mean( - self.linear_0(self.embed0(indices)) + - self.linear_1(self.embed1(indices))) + self.linear_0(self.embed0(indices)) + + self.linear_1(self.embed1(indices)) + ) return loss def linear0(self, x): @@ -154,7 +153,6 @@ class MyLayer2(fluid.Layer): class TestImperativeAutoPrune(unittest.TestCase): - def func_auto_prune(self): with fluid.dygraph.guard(): case1 = AutoPruneLayer0(input_size=5) @@ -312,12 +310,15 @@ class TestImperativeAutoPrune(unittest.TestCase): out2.backward() optimizer = fluid.optimizer.SGD( learning_rate=0.003, - parameter_list=(linear.parameters() + linear2.parameters())) + parameter_list=(linear.parameters() + linear2.parameters()), + ) optimizer.minimize(out2) - np.testing.assert_array_equal(linear2_origin, - linear2.weight.numpy()) + np.testing.assert_array_equal( + linear2_origin, linear2.weight.numpy() + ) self.assertFalse( - np.array_equal(linear_origin, linear.weight.numpy())) + np.array_equal(linear_origin, linear.weight.numpy()) + ) def test_auto_prune8(self): with _test_eager_guard(): @@ -342,10 +343,12 @@ class TestImperativeAutoPrune(unittest.TestCase): out2.backward() optimizer = fluid.optimizer.SGD( learning_rate=0.003, - parameter_list=(linear.parameters() + linear2.parameters())) + parameter_list=(linear.parameters() + linear2.parameters()), + ) optimizer.minimize(out2) - np.testing.assert_array_equal(linear2_origin, - linear2.weight.numpy()) + np.testing.assert_array_equal( + linear2_origin, linear2.weight.numpy() + ) np.testing.assert_array_equal(linear_origin, linear.weight.numpy()) try: linear2.weight.gradient() @@ -371,7 +374,7 @@ class TestImperativeAutoPrune(unittest.TestCase): out2 = linear2(b) out1.stop_gradient = True out = fluid.layers.concat(input=[out1, out2, c], axis=1) - #TODO(jiabin): In Eager Mode we don't actually need sort_sum_gradient, this test should be removed when we don't support fluid anymore. + # TODO(jiabin): In Eager Mode we don't actually need sort_sum_gradient, this test should be removed when we don't support fluid anymore. fluid.set_flags({'FLAGS_sort_sum_gradient': True}) out.backward() self.assertTrue(linear.weight.gradient() is None) @@ -387,8 +390,9 @@ class TestImperativeAutoPrune(unittest.TestCase): size = 20 batch_size = 16 - indices = np.random.randint(low=0, high=100, - size=(batch_size, 1)).astype("int64") + indices = np.random.randint( + low=0, high=100, size=(batch_size, 1) + ).astype("int64") embed = np.random.randn(batch_size, size).astype("float32") place = fluid.CPUPlace() @@ -396,7 +400,8 @@ class TestImperativeAutoPrune(unittest.TestCase): model = MyLayer(size, vocab_size, size) grad_clip = fluid.clip.GradientClipByGlobalNorm(0.001) optimizer = fluid.optimizer.AdamOptimizer( - 0.001, parameter_list=model.parameters(), grad_clip=grad_clip) + 0.001, parameter_list=model.parameters(), grad_clip=grad_clip + ) indices = fluid.dygraph.to_variable(indices) embed = fluid.dygraph.to_variable(embed) dummy_loss = model(embed) @@ -414,7 +419,8 @@ class TestImperativeAutoPrune(unittest.TestCase): model = MyLayer2(size, vocab_size, size) grad_clip = fluid.clip.GradientClipByGlobalNorm(0.001) optimizer = fluid.optimizer.AdamOptimizer( - 0.001, parameter_list=model.parameters(), grad_clip=grad_clip) + 0.001, parameter_list=model.parameters(), grad_clip=grad_clip + ) indices = fluid.dygraph.to_variable(indices) emebd = fluid.dygraph.to_variable(embed) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_basic.py b/python/paddle/fluid/tests/unittests/test_imperative_basic.py index 3c8088a1055..62afffb59ab 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_basic.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_basic.py @@ -27,7 +27,6 @@ from paddle.fluid.framework import _in_legacy_dygraph, _test_eager_guard class MyLayer(fluid.Layer): - def __init__(self): super(MyLayer, self).__init__() @@ -40,23 +39,28 @@ class MyLayer(fluid.Layer): class MLP(fluid.Layer): - def __init__(self, input_size): super(MLP, self).__init__() self._linear1 = Linear( input_size, 3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + ) self._linear2 = Linear( 3, 4, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + ) def forward(self, inputs): x = self._linear1(inputs) @@ -66,7 +70,6 @@ class MLP(fluid.Layer): class SimpleRNNCell(fluid.Layer): - def __init__(self, step_input_size, hidden_size, output_size, param_attr): super(SimpleRNNCell, self).__init__() self.step_input_size = step_input_size @@ -79,18 +82,24 @@ class SimpleRNNCell(fluid.Layer): h2h_param_shape = [self.hidden_size, self.hidden_size] h2o_param_shape = [self.output_size, self.hidden_size] self._i2h_w = None - self._i2h_w = self.create_parameter(attr=self.param_attr, - shape=i2h_param_shape, - dtype=self._dtype, - is_bias=False) - self._h2h_w = self.create_parameter(attr=self.param_attr, - shape=h2h_param_shape, - dtype=self._dtype, - is_bias=False) - self._h2o_w = self.create_parameter(attr=self.param_attr, - shape=h2o_param_shape, - dtype=self._dtype, - is_bias=False) + self._i2h_w = self.create_parameter( + attr=self.param_attr, + shape=i2h_param_shape, + dtype=self._dtype, + is_bias=False, + ) + self._h2h_w = self.create_parameter( + attr=self.param_attr, + shape=h2h_param_shape, + dtype=self._dtype, + is_bias=False, + ) + self._h2o_w = self.create_parameter( + attr=self.param_attr, + shape=h2o_param_shape, + dtype=self._dtype, + is_bias=False, + ) def forward(self, input, pre_hidden): tmp_i2h = paddle.fluid.layers.nn.mul(input, self._i2h_w) @@ -104,29 +113,33 @@ class SimpleRNNCell(fluid.Layer): class SimpleRNN(fluid.Layer): - def __init__(self): super(SimpleRNN, self).__init__() self.seq_len = 4 self._cell = SimpleRNNCell( - 3, 3, 3, - fluid.ParamAttr(initializer=fluid.initializer.Constant(value=0.1))) + 3, + 3, + 3, + fluid.ParamAttr(initializer=fluid.initializer.Constant(value=0.1)), + ) def forward(self, inputs): outs = list() pre_hiddens = list() - init_hidden = self.create_parameter(attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.1)), - shape=[1, 3], - dtype='float32', - is_bias=False) + init_hidden = self.create_parameter( + attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + shape=[1, 3], + dtype='float32', + is_bias=False, + ) pre_hidden = init_hidden for i in range(self.seq_len): - input = fluid.layers.slice(inputs, - axes=[1], - starts=[i], - ends=[i + 1]) + input = fluid.layers.slice( + inputs, axes=[1], starts=[i], ends=[i + 1] + ) input = fluid.layers.reshape(input, shape=[1, 3]) out_softmax, pre_hidden = self._cell(input, pre_hidden) outs.append(out_softmax) @@ -135,7 +148,6 @@ class SimpleRNN(fluid.Layer): class TestImperative(unittest.TestCase): - def functional_dygraph_context(self): self.assertFalse(fluid.dygraph.enabled()) fluid.enable_dygraph() @@ -220,8 +232,9 @@ class TestImperative(unittest.TestCase): t = fluid.Tensor() t.set(x, fluid.CPUPlace()) if not _in_legacy_dygraph(): - egr_tmp = fluid.core.eager.Tensor(value=x, - place=fluid.core.CPUPlace()) + egr_tmp = fluid.core.eager.Tensor( + value=x, place=fluid.core.CPUPlace() + ) egr_tmp2 = fluid.core.eager.Tensor(y, fluid.core.CPUPlace()) egr_tmp3 = paddle.to_tensor(x) egr_tmp4 = fluid.core.eager.Tensor(y) @@ -359,13 +372,13 @@ class TestImperative(unittest.TestCase): cur_block = cur_program.current_block() # Normally, we don't allow tensor with -1 shape being created in dygraph mode, this test is not good. if _in_legacy_dygraph(): - new_variable = cur_block.create_var(name="X", - shape=[-1, 23, 48], - dtype='float32') + new_variable = cur_block.create_var( + name="X", shape=[-1, 23, 48], dtype='float32' + ) else: - new_variable = cur_block.create_var(name="X", - shape=[1, 23, 48], - dtype='float32') + new_variable = cur_block.create_var( + name="X", shape=[1, 23, 48], dtype='float32' + ) try: new_variable.numpy() except Exception as e: @@ -400,13 +413,13 @@ class TestImperative(unittest.TestCase): cur_block = cur_program.current_block() # Normally, we don't allow tensor with -1 shape being created in dygraph mode, this test is not good. if _in_legacy_dygraph(): - new_variable = cur_block.create_var(name="X", - shape=[-1, 23, 48], - dtype='float32') + new_variable = cur_block.create_var( + name="X", shape=[-1, 23, 48], dtype='float32' + ) else: - new_variable = cur_block.create_var(name="X", - shape=[1, 23, 48], - dtype='float32') + new_variable = cur_block.create_var( + name="X", shape=[1, 23, 48], dtype='float32' + ) try: new_variable.gradient() except Exception as e: @@ -464,19 +477,24 @@ class TestImperative(unittest.TestCase): dy_grad2 = l2._x_for_debug.gradient() with new_program_scope(): - inp = fluid.layers.data(name="inp", - shape=[3], - append_batch_size=False) + inp = fluid.layers.data( + name="inp", shape=[3], append_batch_size=False + ) l = MyLayer() x = l(inp)[0] param_grads = fluid.backward.append_backward( - x, parameter_list=[l._x_for_debug.name])[0] - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + x, parameter_list=[l._x_for_debug.name] + )[0] + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) static_out, static_grad = exe.run( feed={inp.name: np_inp}, - fetch_list=[x.name, param_grads[1].name]) + fetch_list=[x.name, param_grads[1].name], + ) np.testing.assert_array_equal(dy_out, static_out) np.testing.assert_array_equal(dy_grad, static_grad) @@ -510,20 +528,25 @@ class TestImperative(unittest.TestCase): dy_grad2 = mlp2._linear1.weight.gradient() with new_program_scope(): - inp = fluid.layers.data(name="inp", - shape=[2, 2], - append_batch_size=False) + inp = fluid.layers.data( + name="inp", shape=[2, 2], append_batch_size=False + ) mlp = MLP(input_size=2) out = mlp(inp) param_grads = fluid.backward.append_backward( - out, parameter_list=[mlp._linear1.weight.name])[0] - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + out, parameter_list=[mlp._linear1.weight.name] + )[0] + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(fluid.default_startup_program()) static_out, static_grad = exe.run( feed={inp.name: np_inp}, - fetch_list=[out.name, param_grads[1].name]) + fetch_list=[out.name, param_grads[1].name], + ) np.testing.assert_allclose(dy_out, static_out, rtol=1e-05) np.testing.assert_allclose(dy_grad, static_grad, rtol=1e-05) @@ -548,28 +571,27 @@ class TestImperative(unittest.TestCase): self.func_mlp() def test_gradient_accumulation(self): - def test_single_api(sort_sum_gradient): fluid.set_flags({'FLAGS_sort_sum_gradient': sort_sum_gradient}) - x = paddle.to_tensor(5., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) for i in range(10): y = paddle.pow(x, 4.0) y.backward() self.assertEqual(x.grad.numpy(), (i + 1) * 500) x.clear_gradient() - self.assertEqual(x.grad.numpy(), 0.) + self.assertEqual(x.grad.numpy(), 0.0) for i in range(10): y = paddle.pow(x, 4.0) y.backward() self.assertEqual(x.grad.numpy(), (i + 1) * 500) x.clear_grad() - self.assertEqual(x.grad.numpy(), 0.) + self.assertEqual(x.grad.numpy(), 0.0) def test_simple_net(sort_sum_gradient): fluid.set_flags({'FLAGS_sort_sum_gradient': sort_sum_gradient}) - x = paddle.to_tensor(5., stop_gradient=False) - y = paddle.to_tensor(2., stop_gradient=False) - z = paddle.to_tensor(3., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) + y = paddle.to_tensor(2.0, stop_gradient=False) + z = paddle.to_tensor(3.0, stop_gradient=False) def fun(x, y, z): loss1 = x * x * y @@ -618,59 +640,72 @@ class TestImperative(unittest.TestCase): # generate the gradient of each step mlp2 = MLP(input_size=input_size) - expected_weight1_grad = 0. - expected_bias1_grad = 0. - expected_weight2_grad = 0. - expected_bias2_grad = 0. + expected_weight1_grad = 0.0 + expected_bias1_grad = 0.0 + expected_weight2_grad = 0.0 + expected_bias2_grad = 0.0 for batch_id in range(100): x = paddle.uniform([10, input_size]) detach_x = x.detach() clear_loss = mlp2(detach_x) clear_loss.backward() - expected_weight1_grad = (expected_weight1_grad + - mlp2._linear1.weight.grad.numpy()) - expected_bias1_grad = (expected_bias1_grad + - mlp2._linear1.bias.grad.numpy()) - expected_weight2_grad = (expected_weight2_grad + - mlp2._linear2.weight.grad.numpy()) - expected_bias2_grad = (expected_bias2_grad + - mlp2._linear2.bias.grad.numpy()) + expected_weight1_grad = ( + expected_weight1_grad + mlp2._linear1.weight.grad.numpy() + ) + expected_bias1_grad = ( + expected_bias1_grad + mlp2._linear1.bias.grad.numpy() + ) + expected_weight2_grad = ( + expected_weight2_grad + mlp2._linear2.weight.grad.numpy() + ) + expected_bias2_grad = ( + expected_bias2_grad + mlp2._linear2.bias.grad.numpy() + ) loss = mlp1(x) loss.backward() np.testing.assert_array_equal(loss.grad.numpy(), [1]) - np.testing.assert_allclose(mlp1._linear1.weight.grad.numpy(), - expected_weight1_grad, - rtol=1e-05) - np.testing.assert_allclose(mlp1._linear1.bias.grad.numpy(), - expected_bias1_grad, - rtol=1e-05) - np.testing.assert_allclose(mlp1._linear2.weight.grad.numpy(), - expected_weight2_grad, - rtol=1e-05) - np.testing.assert_allclose(mlp1._linear2.bias.grad.numpy(), - expected_bias2_grad, - rtol=1e-05) + np.testing.assert_allclose( + mlp1._linear1.weight.grad.numpy(), + expected_weight1_grad, + rtol=1e-05, + ) + np.testing.assert_allclose( + mlp1._linear1.bias.grad.numpy(), + expected_bias1_grad, + rtol=1e-05, + ) + np.testing.assert_allclose( + mlp1._linear2.weight.grad.numpy(), + expected_weight2_grad, + rtol=1e-05, + ) + np.testing.assert_allclose( + mlp1._linear2.bias.grad.numpy(), + expected_bias2_grad, + rtol=1e-05, + ) mlp2.clear_gradients() np.testing.assert_array_equal(clear_loss.grad.numpy(), [1]) if ((batch_id + 1) % 10) % 2 == 0: mlp1.clear_gradients() - expected_weight1_grad = 0. - expected_bias1_grad = 0. - expected_weight2_grad = 0. - expected_bias2_grad = 0. + expected_weight1_grad = 0.0 + expected_bias1_grad = 0.0 + expected_weight2_grad = 0.0 + expected_bias2_grad = 0.0 elif ((batch_id + 1) % 10) % 2 == 1: mlp1.clear_gradients() mlp1._linear1.weight._set_grad_ivar( - paddle.ones([input_size, 3])) + paddle.ones([input_size, 3]) + ) mlp1._linear2.weight._set_grad_ivar(paddle.ones([3, 4])) - expected_weight1_grad = 1. - expected_bias1_grad = 0. - expected_weight2_grad = 1. - expected_bias2_grad = 0. + expected_weight1_grad = 1.0 + expected_bias1_grad = 0.0 + expected_weight2_grad = 1.0 + expected_bias2_grad = 0.0 with fluid.dygraph.guard(): test_single_api(False) @@ -696,19 +731,25 @@ class TestImperative(unittest.TestCase): # static graph with new_program_scope(): - inp_data1 = fluid.layers.data(name='inp1', - shape=[3, 3], - dtype=np.float32) - inp_data2 = fluid.layers.data(name='inp2', - shape=[3, 3], - dtype=np.float32) + inp_data1 = fluid.layers.data( + name='inp1', shape=[3, 3], dtype=np.float32 + ) + inp_data2 = fluid.layers.data( + name='inp2', shape=[3, 3], dtype=np.float32 + ) a = fluid.layers.expand( - fluid.layers.reshape(fluid.layers.reduce_sum(inp_data1), - [1, 1]), [4, 1]) + fluid.layers.reshape( + fluid.layers.reduce_sum(inp_data1), [1, 1] + ), + [4, 1], + ) b = fluid.layers.expand( - fluid.layers.reshape(fluid.layers.reduce_sum(inp_data2), - [1, 1]), [4, 1]) + fluid.layers.reshape( + fluid.layers.reduce_sum(inp_data2), [1, 1] + ), + [4, 1], + ) cond = fluid.layers.less_than(x=a, y=b) ie = fluid.layers.IfElse(cond) @@ -725,14 +766,16 @@ class TestImperative(unittest.TestCase): ie.output(d3) out = ie() - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) - static_result = exe.run(fluid.default_main_program(), - feed={ - 'inp1': np_inp1, - 'inp2': np_inp2 - }, - fetch_list=out)[0] + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + static_result = exe.run( + fluid.default_main_program(), + feed={'inp1': np_inp1, 'inp2': np_inp2}, + fetch_list=out, + )[0] np.testing.assert_allclose(dygraph_result, static_result, rtol=1e-05) def test_dygraph_vs_static(self): @@ -741,8 +784,14 @@ class TestImperative(unittest.TestCase): self.func_dygraph_vs_static() def func_rnn(self): - np_inp = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0], - [10.0, 11.0, 12.0]]) + np_inp = np.array( + [ + [1.0, 2.0, 3.0], + [4.0, 5.0, 6.0], + [7.0, 8.0, 9.0], + [10.0, 11.0, 12.0], + ] + ) np_inp = np_inp.reshape((1, 4, 3)) np_inp = np_inp.astype(np.float32) with fluid.dygraph.guard(): @@ -769,20 +818,28 @@ class TestImperative(unittest.TestCase): dy_grad_i2h2 = simple_rnn2._cell._i2h_w.gradient() with new_program_scope(): - inp = fluid.layers.data(name="inp", - shape=[1, 4, 3], - append_batch_size=False) + inp = fluid.layers.data( + name="inp", shape=[1, 4, 3], append_batch_size=False + ) simple_rnn = SimpleRNN() outs, pre_hiddens = simple_rnn(inp) param_grads = fluid.backward.append_backward(outs[3]) exe = fluid.Executor(fluid.CPUPlace()) exe.run(fluid.default_startup_program()) - static_out, static_grad_h2o, static_grad_h2h, static_grad_i2h = exe.run( + ( + static_out, + static_grad_h2o, + static_grad_h2h, + static_grad_i2h, + ) = exe.run( feed={inp.name: np_inp}, fetch_list=[ - outs[3].name, param_grads[0][1].name, - param_grads[1][1].name, param_grads[2][1].name - ]) + outs[3].name, + param_grads[0][1].name, + param_grads[1][1].name, + param_grads[2][1].name, + ], + ) np.testing.assert_array_equal(dy_out, static_out) np.testing.assert_array_equal(dy_grad_h2o, static_grad_h2o) @@ -825,7 +882,6 @@ class TestImperative(unittest.TestCase): class TestDygraphUtils(unittest.TestCase): - def func_append_activation_in_dygraph_exception(self): with new_program_scope(): np_inp = np.random.random(size=(10, 20, 30)).astype(np.float32) @@ -883,9 +939,9 @@ class TestDygraphUtils(unittest.TestCase): def func_append_activation_in_dygraph_use_mkldnn(self): a_np = np.random.uniform(-2, 2, (10, 20, 30)).astype(np.float32) - helper = LayerHelper(fluid.unique_name.generate("test"), - act="relu", - use_mkldnn=True) + helper = LayerHelper( + fluid.unique_name.generate("test"), act="relu", use_mkldnn=True + ) func = helper.append_activation with fluid.dygraph.guard(): a = paddle.to_tensor(a_np) @@ -945,12 +1001,12 @@ class TestDygraphUtils(unittest.TestCase): class TestDygraphGuardWithError(unittest.TestCase): - def func_without_guard(self): with fluid.dygraph.guard(): x = paddle.to_tensor(np.zeros([10, 10])) - with self.assertRaisesRegexp(TypeError, - "Please use `with fluid.dygraph.guard()"): + with self.assertRaisesRegexp( + TypeError, "Please use `with fluid.dygraph.guard()" + ): y = fluid.layers.matmul(x, x) def test_without_guard(self): @@ -960,16 +1016,17 @@ class TestDygraphGuardWithError(unittest.TestCase): class TestMetaclass(unittest.TestCase): - def func_metaclass(self): self.assertEqual(type(MyLayer).__name__, 'type') self.assertNotEqual(type(MyLayer).__name__, 'pybind11_type') if not _in_legacy_dygraph(): self.assertEqual( - type(paddle.fluid.core.eager.Tensor).__name__, 'type') + type(paddle.fluid.core.eager.Tensor).__name__, 'type' + ) else: self.assertEqual( - type(paddle.fluid.core.VarBase).__name__, 'pybind11_type') + type(paddle.fluid.core.VarBase).__name__, 'pybind11_type' + ) def test_metaclass(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_layerdict.py b/python/paddle/fluid/tests/unittests/test_imperative_container_layerdict.py index 2e7bd103ec5..d81cef00f7b 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_layerdict.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_layerdict.py @@ -19,12 +19,13 @@ from paddle.fluid.framework import _test_eager_guard class TestLayerDict(unittest.TestCase): - def func_layer_dict(self): - layers = OrderedDict([ - ('conv1d', paddle.nn.Conv1D(3, 2, 3)), - ('conv2d', paddle.nn.Conv2D(3, 2, 3)), - ]) + layers = OrderedDict( + [ + ('conv1d', paddle.nn.Conv1D(3, 2, 3)), + ('conv2d', paddle.nn.Conv2D(3, 2, 3)), + ] + ) layers_dicts = paddle.nn.LayerDict(sublayers=layers) @@ -58,10 +59,12 @@ class TestLayerDict(unittest.TestCase): layers_dicts['linear'] = layers['linear'] check_layer_dict() - sublayer = OrderedDict([ - ('sigmod', paddle.nn.Sigmoid()), - ('relu', paddle.nn.ReLU()), - ]) + sublayer = OrderedDict( + [ + ('sigmod', paddle.nn.Sigmoid()), + ('relu', paddle.nn.ReLU()), + ] + ) layers.update(sublayer) layers_dicts.update(sublayer) check_layer_dict() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py b/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py index d4d2bd0e519..89d92b38205 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class MyLayer(fluid.Layer): - def __init__(self, layerlist): super(MyLayer, self).__init__() self.layerlist = layerlist @@ -32,53 +31,58 @@ class MyLayer(fluid.Layer): class TestImperativeContainer(unittest.TestCase): - def fluid_dygraph_list(self): return fluid.dygraph.LayerList( - [fluid.dygraph.Linear(2**i, 2**(i + 1)) for i in range(6)]) + [fluid.dygraph.Linear(2**i, 2 ** (i + 1)) for i in range(6)] + ) def paddle_imperative_list(self): return paddle.nn.LayerList( - [fluid.dygraph.Linear(2**i, 2**(i + 1)) for i in range(6)]) + [fluid.dygraph.Linear(2**i, 2 ** (i + 1)) for i in range(6)] + ) def layer_list(self, use_fluid_api): data_np = np.random.uniform(-1, 1, [5, 1]).astype('float32') with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(data_np) - layerlist = self.fluid_dygraph_list( - ) if use_fluid_api else self.paddle_imperative_list() + layerlist = ( + self.fluid_dygraph_list() + if use_fluid_api + else self.paddle_imperative_list() + ) size = len(layerlist) model = MyLayer(layerlist) res1 = model(x) self.assertListEqual(res1.shape, [5, 2**size]) - model.layerlist[size - 1] = fluid.dygraph.Linear(2**(size - 1), 5) + model.layerlist[size - 1] = fluid.dygraph.Linear(2 ** (size - 1), 5) res2 = model(x) self.assertListEqual(res2.shape, [5, 5]) del model.layerlist[size - 1] res3 = model(x) - self.assertListEqual(res3.shape, [5, 2**(size - 1)]) - model.layerlist.append(fluid.dygraph.Linear(2**(size - 1), 3)) + self.assertListEqual(res3.shape, [5, 2 ** (size - 1)]) + model.layerlist.append(fluid.dygraph.Linear(2 ** (size - 1), 3)) res4 = model(x) self.assertListEqual(res4.shape, [5, 3]) res4.backward() model2 = MyLayer(layerlist[:-1]) res5 = model2(x) - self.assertListEqual(res5.shape, [5, 2**(size - 1)]) + self.assertListEqual(res5.shape, [5, 2 ** (size - 1)]) del model2.layerlist[1:] res6 = model2(x) - self.assertListEqual(res6.shape, [5, 2**(0 + 1)]) + self.assertListEqual(res6.shape, [5, 2 ** (0 + 1)]) res6.backward() model3 = MyLayer(layerlist[:-2]) model3.layerlist.append(fluid.dygraph.Linear(3, 1)) - model3.layerlist.insert(size - 2, - fluid.dygraph.Linear(2**(size - 2), 3)) + model3.layerlist.insert( + size - 2, fluid.dygraph.Linear(2 ** (size - 2), 3) + ) res7 = model3(x) self.assertListEqual(res7.shape, [5, 1]) to_be_extended = [ - fluid.dygraph.Linear(3**i, 3**(i + 1)) for i in range(3) + fluid.dygraph.Linear(3**i, 3 ** (i + 1)) for i in range(3) ] model3.layerlist.extend(to_be_extended) res8 = model3(x) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py b/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py index 2101e84858a..55e666c10aa 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py @@ -21,24 +21,26 @@ from paddle.fluid.framework import _test_eager_guard class MyLayer(fluid.Layer): - def __init__(self, num_stacked_param, use_fluid_api): super(MyLayer, self).__init__() # create ParameterList with iterable Parameters - self.params = self.fluid_dygraph_ParameterList( - num_stacked_param - ) if use_fluid_api else self.paddle_imperative_ParameterList( - num_stacked_param) + self.params = ( + self.fluid_dygraph_ParameterList(num_stacked_param) + if use_fluid_api + else self.paddle_imperative_ParameterList(num_stacked_param) + ) def fluid_dygraph_ParameterList(self, num_stacked_param): return fluid.dygraph.ParameterList( - [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] * - num_stacked_param) + [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] + * num_stacked_param + ) def paddle_imperative_ParameterList(self, num_stacked_param): return paddle.nn.ParameterList( - [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] * - num_stacked_param) + [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] + * num_stacked_param + ) def forward(self, x): for i, p in enumerate(self.params): @@ -47,7 +49,6 @@ class MyLayer(fluid.Layer): class TestImperativeContainerParameterList(unittest.TestCase): - def paramter_list(self, use_fluid_api): data_np = np.random.uniform(-1, 1, [5, 2]).astype('float32') with fluid.dygraph.guard(): @@ -61,11 +62,13 @@ class TestImperativeContainerParameterList(unittest.TestCase): loss.backward() model.params[num_stacked_param - 1] = fluid.layers.create_parameter( - shape=[2, 3], dtype='float32') + shape=[2, 3], dtype='float32' + ) res = model(x) self.assertListEqual(res.shape, [5, 3]) model.params.append( - fluid.layers.create_parameter(shape=[3, 4], dtype='float32')) + fluid.layers.create_parameter(shape=[3, 4], dtype='float32') + ) self.assertEqual(len(model.params), num_stacked_param + 1) res = model(x) self.assertListEqual(res.shape, [5, 4]) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_sequential.py b/python/paddle/fluid/tests/unittests/test_imperative_container_sequential.py index c0b4ed44e5b..c1b0a42675f 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_sequential.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_sequential.py @@ -19,13 +19,13 @@ from paddle.fluid.framework import _test_eager_guard class TestImperativeContainerSequential(unittest.TestCase): - def func_sequential(self): data = np.random.uniform(-1, 1, [5, 10]).astype('float32') with fluid.dygraph.guard(): data = fluid.dygraph.to_variable(data) - model1 = fluid.dygraph.Sequential(fluid.Linear(10, 1), - fluid.Linear(1, 2)) + model1 = fluid.dygraph.Sequential( + fluid.Linear(10, 1), fluid.Linear(1, 2) + ) res1 = model1(data) self.assertListEqual(res1.shape, [5, 2]) model1[1] = fluid.Linear(1, 3) @@ -64,8 +64,9 @@ class TestImperativeContainerSequential(unittest.TestCase): data = np.random.uniform(-1, 1, [5, 10]).astype('float32') with fluid.dygraph.guard(): data = fluid.dygraph.to_variable(data) - model1 = fluid.dygraph.Sequential(fluid.Linear(10, 1), - fluid.Linear(1, 2)) + model1 = fluid.dygraph.Sequential( + fluid.Linear(10, 1), fluid.Linear(1, 2) + ) res1 = model1(data) self.assertListEqual(res1.shape, [5, 2]) model1[1] = fluid.Linear(1, 3) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_base.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_base.py index 5ae1b3f0014..da15dcf29b5 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_base.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_base.py @@ -26,7 +26,6 @@ def get_random_images_and_labels(image_shape, label_shape): def sample_generator_creator(batch_size, batch_num): - def __reader__(): for _ in range(batch_num * batch_size): image, label = get_random_images_and_labels([784], [1]) @@ -36,7 +35,6 @@ def sample_generator_creator(batch_size, batch_num): class TestDygraphDataLoader(unittest.TestCase): - def setUp(self): self.batch_size = 8 self.batch_num = 4 @@ -53,13 +51,14 @@ class TestDygraphDataLoader(unittest.TestCase): def func_test_single_process_loader(self): with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - iterable=False, - use_multiprocess=False) - loader.set_sample_generator(sample_generator_creator( - self.batch_size, self.batch_num), - batch_size=self.batch_size, - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, iterable=False, use_multiprocess=False + ) + loader.set_sample_generator( + sample_generator_creator(self.batch_size, self.batch_num), + batch_size=self.batch_size, + places=fluid.CPUPlace(), + ) self.iter_loader_data(loader) def test_single_process_loader(self): @@ -69,12 +68,14 @@ class TestDygraphDataLoader(unittest.TestCase): def func_test_multi_process_loader(self): with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - use_multiprocess=True) - loader.set_sample_generator(sample_generator_creator( - self.batch_size, self.batch_num), - batch_size=self.batch_size, - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, use_multiprocess=True + ) + loader.set_sample_generator( + sample_generator_creator(self.batch_size, self.batch_num), + batch_size=self.batch_size, + places=fluid.CPUPlace(), + ) self.iter_loader_data(loader) def test_multi_process_loader(self): @@ -85,9 +86,10 @@ class TestDygraphDataLoader(unittest.TestCase): def func_test_generator_no_places(self): with fluid.dygraph.guard(): loader = fluid.io.DataLoader.from_generator(capacity=self.capacity) - loader.set_sample_generator(sample_generator_creator( - self.batch_size, self.batch_num), - batch_size=self.batch_size) + loader.set_sample_generator( + sample_generator_creator(self.batch_size, self.batch_num), + batch_size=self.batch_size, + ) self.iter_loader_data(loader) def test_generator_no_places(self): @@ -98,13 +100,14 @@ class TestDygraphDataLoader(unittest.TestCase): def func_test_set_pin_memory(self): with fluid.dygraph.guard(): use_pinned_memory(False) - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - iterable=False, - use_multiprocess=False) - loader.set_sample_generator(sample_generator_creator( - self.batch_size, self.batch_num), - batch_size=self.batch_size, - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, iterable=False, use_multiprocess=False + ) + loader.set_sample_generator( + sample_generator_creator(self.batch_size, self.batch_num), + batch_size=self.batch_size, + places=fluid.CPUPlace(), + ) self.iter_loader_data(loader) use_pinned_memory(True) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exception.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exception.py index 1389b93c38e..eac20ea03c7 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exception.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exception.py @@ -27,7 +27,6 @@ def get_random_images_and_labels(image_shape, label_shape): class TestDygraphDataLoaderWithException(unittest.TestCase): - def setUp(self): self.batch_size = 8 self.batch_num = 4 @@ -36,8 +35,9 @@ class TestDygraphDataLoaderWithException(unittest.TestCase): def func_test_not_capacity(self): with fluid.dygraph.guard(): - with self.assertRaisesRegexp(ValueError, - "Please give value to capacity."): + with self.assertRaisesRegexp( + ValueError, "Please give value to capacity." + ): fluid.io.DataLoader.from_generator() def test_not_capacity(self): @@ -46,9 +46,7 @@ class TestDygraphDataLoaderWithException(unittest.TestCase): self.func_test_not_capacity() def func_test_single_process_with_thread_expection(self): - def error_sample_genarator(batch_num): - def __reader__(): for _ in range(batch_num): yield [[[1, 2], [1]]] @@ -56,11 +54,12 @@ class TestDygraphDataLoaderWithException(unittest.TestCase): return __reader__ with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - iterable=False, - use_multiprocess=False) - loader.set_batch_generator(error_sample_genarator(self.batch_num), - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, iterable=False, use_multiprocess=False + ) + loader.set_batch_generator( + error_sample_genarator(self.batch_num), places=fluid.CPUPlace() + ) exception = None try: for _ in loader(): @@ -76,9 +75,7 @@ class TestDygraphDataLoaderWithException(unittest.TestCase): self.func_test_single_process_with_thread_expection() def func_test_multi_process_with_process_expection(self): - def error_sample_genarator(batch_num): - def __reader__(): for _ in range(batch_num): yield [[[1, 2], [1]]] @@ -86,10 +83,12 @@ class TestDygraphDataLoaderWithException(unittest.TestCase): return __reader__ with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - use_multiprocess=True) - loader.set_batch_generator(error_sample_genarator(self.batch_num), - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, use_multiprocess=True + ) + loader.set_batch_generator( + error_sample_genarator(self.batch_num), places=fluid.CPUPlace() + ) exception = None try: for _ in loader(): @@ -104,24 +103,25 @@ class TestDygraphDataLoaderWithException(unittest.TestCase): self.func_test_multi_process_with_process_expection() def func_test_multi_process_with_get_timeout(self): - def slow_batch_generator_creator(batch_size, batch_num): - def __reader__(): for _ in range(batch_num): time.sleep(80) batch_image, batch_label = get_random_images_and_labels( - [batch_size, 784], [batch_size, 1]) + [batch_size, 784], [batch_size, 1] + ) yield batch_image, batch_label return __reader__ with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - use_multiprocess=True) - loader.set_batch_generator(slow_batch_generator_creator( - self.batch_size, self.batch_num), - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, use_multiprocess=True + ) + loader.set_batch_generator( + slow_batch_generator_creator(self.batch_size, self.batch_num), + places=fluid.CPUPlace(), + ) exception = None try: for _ in range(self.epoch_num): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exit_func.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exit_func.py index ec59d473c13..ff9ae7343b3 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exit_func.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exit_func.py @@ -21,14 +21,17 @@ from paddle.fluid.framework import _test_eager_guard import queue -from paddle.fluid.reader import multiprocess_queue_set, _cleanup, CleanupFuncRegistrar +from paddle.fluid.reader import ( + multiprocess_queue_set, + _cleanup, + CleanupFuncRegistrar, +) # NOTE: These special functions cannot be detected by the existing coverage mechanism, # so the following unittests are added for these internal functions. class TestDygraphDataLoaderCleanUpFunc(unittest.TestCase): - def setUp(self): self.capacity = 10 @@ -65,8 +68,9 @@ class TestRegisterExitFunc(unittest.TestCase): self.func_test_not_callable_func() def func_test_old_handler_for_sigint(self): - CleanupFuncRegistrar.register(function=self.none_func, - signals=[signal.SIGINT]) + CleanupFuncRegistrar.register( + function=self.none_func, signals=[signal.SIGINT] + ) def test_old_handler_for_sigint(self): with _test_eager_guard(): @@ -78,8 +82,9 @@ class TestRegisterExitFunc(unittest.TestCase): def __test_process__(): pass - CleanupFuncRegistrar.register(function=self.none_func, - signals=[signal.SIGCHLD]) + CleanupFuncRegistrar.register( + function=self.none_func, signals=[signal.SIGCHLD] + ) exception = None try: diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_fds_clear.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_fds_clear.py index f0bdc8dacba..355f35ce5e9 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_fds_clear.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_fds_clear.py @@ -26,25 +26,24 @@ def get_random_images_and_labels(image_shape, label_shape): def batch_generator_creator(batch_size, batch_num): - def __reader__(): for _ in range(batch_num): batch_image, batch_label = get_random_images_and_labels( - [batch_size, 784], [batch_size, 1]) + [batch_size, 784], [batch_size, 1] + ) yield batch_image, batch_label return __reader__ class RandomDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([784]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') return image, label def __len__(self): @@ -52,7 +51,6 @@ class RandomDataset(Dataset): class TestDygraphDataLoaderMmapFdsClear(unittest.TestCase): - def setUp(self): self.batch_size = 8 self.batch_num = 100 @@ -60,11 +58,13 @@ class TestDygraphDataLoaderMmapFdsClear(unittest.TestCase): self.capacity = 50 def prepare_data_loader(self): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - use_multiprocess=True) - loader.set_batch_generator(batch_generator_creator( - self.batch_size, self.batch_num), - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, use_multiprocess=True + ) + loader.set_batch_generator( + batch_generator_creator(self.batch_size, self.batch_num), + places=fluid.CPUPlace(), + ) return loader def run_one_epoch_with_break(self, loader): @@ -102,16 +102,17 @@ class TestDygraphDataLoaderMmapFdsClear(unittest.TestCase): class TestMultiProcessDataLoaderMmapFdsClear(TestDygraphDataLoaderMmapFdsClear): - def prepare_data_loader(self): place = fluid.CPUPlace() with fluid.dygraph.guard(place): dataset = RandomDataset(self.batch_size * self.batch_num) - loader = DataLoader(dataset, - places=place, - batch_size=self.batch_size, - drop_last=True, - num_workers=2) + loader = DataLoader( + dataset, + places=place, + batch_size=self.batch_size, + drop_last=True, + num_workers=2, + ) return loader diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py index 94322a67959..c03b9b3bbf1 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py @@ -29,11 +29,11 @@ def get_random_images_and_labels(image_shape, label_shape): def batch_generator_creator(batch_size, batch_num): - def __reader__(): for _ in range(batch_num): batch_image, batch_label = get_random_images_and_labels( - [batch_size, 784], [batch_size, 1]) + [batch_size, 784], [batch_size, 1] + ) yield batch_image, batch_label return __reader__ @@ -42,7 +42,6 @@ def batch_generator_creator(batch_size, batch_num): # NOTE: coverage CI can't cover child process code, so need these test. # Here test child process loop function in main process class TestDygraphDataLoaderProcess(unittest.TestCase): - def setUp(self): self.batch_size = 8 self.batch_num = 4 @@ -60,10 +59,12 @@ class TestDygraphDataLoaderProcess(unittest.TestCase): with fluid.dygraph.guard(): loader = fluid.io.DataLoader.from_generator( - capacity=self.batch_num + 1, use_multiprocess=True) - loader.set_batch_generator(batch_generator_creator( - self.batch_size, self.batch_num), - places=fluid.CPUPlace()) + capacity=self.batch_num + 1, use_multiprocess=True + ) + loader.set_batch_generator( + batch_generator_creator(self.batch_size, self.batch_num), + places=fluid.CPUPlace(), + ) loader._data_queue = queue.Queue(self.batch_num + 1) _reader_process_loop(loader._batch_reader, loader._data_queue) # For clean memory mapped files @@ -73,8 +74,9 @@ class TestDygraphDataLoaderProcess(unittest.TestCase): util_queue.put(data) # Clean up memory mapped files - clear_process = multiprocessing.Process(target=__clear_process__, - args=(util_queue, )) + clear_process = multiprocessing.Process( + target=__clear_process__, args=(util_queue,) + ) clear_process.start() def test_reader_process_loop(self): @@ -83,9 +85,7 @@ class TestDygraphDataLoaderProcess(unittest.TestCase): self.func_test_reader_process_loop() def func_test_reader_process_loop_simple_none(self): - def none_sample_genarator(batch_num): - def __reader__(): for _ in range(batch_num): yield None @@ -94,9 +94,11 @@ class TestDygraphDataLoaderProcess(unittest.TestCase): with fluid.dygraph.guard(): loader = fluid.io.DataLoader.from_generator( - capacity=self.batch_num + 1, use_multiprocess=True) - loader.set_batch_generator(none_sample_genarator(self.batch_num), - places=fluid.CPUPlace()) + capacity=self.batch_num + 1, use_multiprocess=True + ) + loader.set_batch_generator( + none_sample_genarator(self.batch_num), places=fluid.CPUPlace() + ) loader._data_queue = queue.Queue(self.batch_num + 1) exception = None try: diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_parallel.py b/python/paddle/fluid/tests/unittests/test_imperative_data_parallel.py index 4d61d2a67fd..9118788617d 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_parallel.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_parallel.py @@ -23,7 +23,6 @@ import paddle.fluid.core as core class MLP(fluid.Layer): - def __init__(self, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -37,7 +36,6 @@ class MLP(fluid.Layer): class TestDataParallelStateDict(unittest.TestCase): - def test_data_parallel_state_dict(self): with fluid.dygraph.guard(): strategy = dygraph.parallel.prepare_context() @@ -48,13 +46,17 @@ class TestDataParallelStateDict(unittest.TestCase): parallel_state = parallel_mlp.state_dict() base_para = {} - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) for k, v in single_state.items(): self.assertTrue(k in parallel_state) - np.testing.assert_array_equal(v.numpy(), - parallel_state[k].numpy()) + np.testing.assert_array_equal( + v.numpy(), parallel_state[k].numpy() + ) base_para[k] = v.numpy() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_decorator.py b/python/paddle/fluid/tests/unittests/test_imperative_decorator.py index f783b18b2b2..42407c0b36f 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_decorator.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_decorator.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import _test_eager_guard class TestTracerMode(unittest.TestCase): - def setUp(self): self.init_mode = True @@ -60,8 +59,9 @@ class TestTracerMode(unittest.TestCase): decorated_func = fluid.dygraph.no_grad(need_no_grad_func) self.assertTrue( - str(inspect.getfullargspec(decorated_func)) == str( - inspect.getfullargspec(need_no_grad_func))) + str(inspect.getfullargspec(decorated_func)) + == str(inspect.getfullargspec(need_no_grad_func)) + ) self.assertEqual(self.tracer._train_mode, self.init_mode) @@ -79,13 +79,11 @@ class TestTracerMode(unittest.TestCase): class TestTracerMode2(TestTracerMode): - def setUp(self): self.init_mode = False class TestNoGradClass(unittest.TestCase): - @paddle.no_grad() def no_grad_func(self, a): self.assertEqual(self.tracer._train_mode, True) @@ -105,8 +103,10 @@ class TestNoGradClass(unittest.TestCase): return a + b decorated_func = paddle.no_grad()(need_no_grad_func) - self.assertEqual(str(inspect.getfullargspec(decorated_func)), - str(inspect.getfullargspec(need_no_grad_func))) + self.assertEqual( + str(inspect.getfullargspec(decorated_func)), + str(inspect.getfullargspec(need_no_grad_func)), + ) def test_gen(): for i in range(3): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py b/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py index 3da576045c5..361792f5077 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py @@ -28,7 +28,6 @@ from paddle.fluid.framework import _test_eager_guard class DMF(fluid.Layer): - def __init__(self): super(DMF, self).__init__() self._user_latent = Linear(1000, 256) @@ -41,15 +40,23 @@ class DMF(fluid.Layer): self._user_layers.append( self.add_sublayer( 'user_layer_%d' % i, - Linear(256 if i == 0 else self._hid_sizes[i - 1], - self._hid_sizes[i], - act='relu'))) + Linear( + 256 if i == 0 else self._hid_sizes[i - 1], + self._hid_sizes[i], + act='relu', + ), + ) + ) self._item_layers.append( self.add_sublayer( 'item_layer_%d' % i, - Linear(256 if i == 0 else self._hid_sizes[i - 1], - self._hid_sizes[i], - act='relu'))) + Linear( + 256 if i == 0 else self._hid_sizes[i - 1], + self._hid_sizes[i], + act='relu', + ), + ) + ) def forward(self, users, items): users = self._user_latent(users) @@ -62,7 +69,6 @@ class DMF(fluid.Layer): class MLP(fluid.Layer): - def __init__(self): super(MLP, self).__init__() self._user_latent = Linear(1000, 256) @@ -73,22 +79,26 @@ class MLP(fluid.Layer): self._match_layers.append( self.add_sublayer( 'match_layer_%d' % i, - Linear(256 * 2 if i == 0 else self._hid_sizes[i - 1], - self._hid_sizes[i], - act='relu'))) + Linear( + 256 * 2 if i == 0 else self._hid_sizes[i - 1], + self._hid_sizes[i], + act='relu', + ), + ) + ) def forward(self, users, items): users = self._user_latent(users) items = self._item_latent(items) - match_vec = fluid.layers.concat([users, items], - axis=len(users.shape) - 1) + match_vec = fluid.layers.concat( + [users, items], axis=len(users.shape) - 1 + ) for l in self._match_layers: match_vec = l(match_vec) return match_vec class DeepCF(fluid.Layer): - def __init__(self, num_users, num_items, matrix): super(DeepCF, self).__init__() self._num_users = num_users @@ -98,7 +108,8 @@ class DeepCF(fluid.Layer): shape=matrix.shape, dtype=matrix.dtype, is_bias=False, - default_initializer=fluid.initializer.NumpyArrayInitializer(matrix)) + default_initializer=fluid.initializer.NumpyArrayInitializer(matrix), + ) self._rating_matrix.stop_gradient = True self._mlp = MLP() @@ -110,20 +121,21 @@ class DeepCF(fluid.Layer): # items_emb = self._item_emb(items) users_emb = fluid.layers.gather(self._rating_matrix, users) items_emb = fluid.layers.gather( - fluid.layers.transpose(self._rating_matrix, [1, 0]), items) + fluid.layers.transpose(self._rating_matrix, [1, 0]), items + ) users_emb.stop_gradient = True items_emb.stop_gradient = True mlp_predictive = self._mlp(users_emb, items_emb) dmf_predictive = self._dmf(users_emb, items_emb) - predictive = fluid.layers.concat([mlp_predictive, dmf_predictive], - axis=len(mlp_predictive.shape) - 1) + predictive = fluid.layers.concat( + [mlp_predictive, dmf_predictive], axis=len(mlp_predictive.shape) - 1 + ) prediction = self._match_fc(predictive) return prediction class TestDygraphDeepCF(unittest.TestCase): - def setUp(self): # Can use Amusic dataset as the DeepCF describes. self.data_path = os.environ.get('DATA_PATH', '') @@ -152,9 +164,14 @@ class TestDygraphDeepCF(unittest.TestCase): users_np = np.array(user_ids, dtype=np.int32)[indices] items_np = np.array(item_ids, dtype=np.int32)[indices] labels_np = np.array(labels, dtype=np.float32)[indices] - return np.expand_dims(users_np, -1), \ - np.expand_dims(items_np, -1), \ - np.expand_dims(labels_np, -1), NUM_USERS, NUM_ITEMS, matrix + return ( + np.expand_dims(users_np, -1), + np.expand_dims(items_np, -1), + np.expand_dims(labels_np, -1), + NUM_USERS, + NUM_ITEMS, + matrix, + ) def load_data(self): sys.stderr.write('loading from %s\n' % self.data_path) @@ -194,18 +211,35 @@ class TestDygraphDeepCF(unittest.TestCase): users_np = np.array(user_ids, dtype=np.int32)[indices] items_np = np.array(item_ids, dtype=np.int32)[indices] labels_np = np.array(labels, dtype=np.float32)[indices] - return np.expand_dims(users_np, -1), \ - np.expand_dims(items_np, -1), \ - np.expand_dims(labels_np, -1), num_users, num_items, matrix + return ( + np.expand_dims(users_np, -1), + np.expand_dims(items_np, -1), + np.expand_dims(labels_np, -1), + num_users, + num_items, + matrix, + ) def test_deefcf(self): seed = 90 if self.data_path: - (users_np, items_np, labels_np, num_users, num_items, - matrix) = self.load_data() + ( + users_np, + items_np, + labels_np, + num_users, + num_items, + matrix, + ) = self.load_data() else: - (users_np, items_np, labels_np, num_users, num_items, - matrix) = self.get_data() + ( + users_np, + items_np, + labels_np, + num_users, + num_items, + matrix, + ) = self.get_data() paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) startup = fluid.Program() @@ -220,28 +254,39 @@ class TestDygraphDeepCF(unittest.TestCase): deepcf = DeepCF(num_users, num_items, matrix) prediction = deepcf(users, items) loss = fluid.layers.reduce_sum( - fluid.layers.log_loss(prediction, labels)) + fluid.layers.log_loss(prediction, labels) + ) adam = fluid.optimizer.AdamOptimizer(0.01) adam.minimize(loss) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(startup) for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, self.batch_size * self.num_batches, - self.batch_size): + for slice in range( + 0, self.batch_size * self.num_batches, self.batch_size + ): if slice + self.batch_size >= users_np.shape[0]: break static_loss = exe.run( main, feed={ - users.name: users_np[slice:slice + self.batch_size], - items.name: items_np[slice:slice + self.batch_size], - labels.name: - labels_np[slice:slice + self.batch_size] + users.name: users_np[ + slice : slice + self.batch_size + ], + items.name: items_np[ + slice : slice + self.batch_size + ], + labels.name: labels_np[ + slice : slice + self.batch_size + ], }, - fetch_list=[loss])[0] + fetch_list=[loss], + )[0] sys.stderr.write('static loss %s\n' % static_loss) with fluid.dygraph.guard(): @@ -250,21 +295,27 @@ class TestDygraphDeepCF(unittest.TestCase): deepcf = DeepCF(num_users, num_items, matrix) adam = fluid.optimizer.AdamOptimizer( - 0.01, parameter_list=deepcf.parameters()) + 0.01, parameter_list=deepcf.parameters() + ) for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, self.batch_size * self.num_batches, - self.batch_size): + for slice in range( + 0, self.batch_size * self.num_batches, self.batch_size + ): if slice + self.batch_size >= users_np.shape[0]: break prediction = deepcf( - to_variable(users_np[slice:slice + self.batch_size]), - to_variable(items_np[slice:slice + self.batch_size])) + to_variable(users_np[slice : slice + self.batch_size]), + to_variable(items_np[slice : slice + self.batch_size]), + ) loss = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction, - to_variable(labels_np[slice:slice + - self.batch_size]))) + to_variable( + labels_np[slice : slice + self.batch_size] + ), + ) + ) loss.backward() adam.minimize(loss) deepcf.clear_gradients() @@ -277,28 +328,35 @@ class TestDygraphDeepCF(unittest.TestCase): deepcf2 = DeepCF(num_users, num_items, matrix) adam2 = fluid.optimizer.AdamOptimizer( - 0.01, parameter_list=deepcf2.parameters()) + 0.01, parameter_list=deepcf2.parameters() + ) fluid.set_flags({'FLAGS_sort_sum_gradient': True}) for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, self.batch_size * self.num_batches, - self.batch_size): + for slice in range( + 0, self.batch_size * self.num_batches, self.batch_size + ): if slice + self.batch_size >= users_np.shape[0]: break prediction2 = deepcf2( - to_variable(users_np[slice:slice + self.batch_size]), - to_variable(items_np[slice:slice + self.batch_size])) + to_variable(users_np[slice : slice + self.batch_size]), + to_variable(items_np[slice : slice + self.batch_size]), + ) loss2 = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction2, - to_variable(labels_np[slice:slice + - self.batch_size]))) + to_variable( + labels_np[slice : slice + self.batch_size] + ), + ) + ) loss2.backward() adam2.minimize(loss2) deepcf2.clear_gradients() dy_loss2 = loss2.numpy() - sys.stderr.write('dynamic loss: %s %s\n' % - (slice, dy_loss2)) + sys.stderr.write( + 'dynamic loss: %s %s\n' % (slice, dy_loss2) + ) with fluid.dygraph.guard(): with _test_eager_guard(): @@ -309,30 +367,39 @@ class TestDygraphDeepCF(unittest.TestCase): deepcf = DeepCF(num_users, num_items, matrix) adam = fluid.optimizer.AdamOptimizer( - 0.01, parameter_list=deepcf.parameters()) + 0.01, parameter_list=deepcf.parameters() + ) for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, self.batch_size * self.num_batches, - self.batch_size): + for slice in range( + 0, self.batch_size * self.num_batches, self.batch_size + ): if slice + self.batch_size >= users_np.shape[0]: break prediction = deepcf( - to_variable(users_np[slice:slice + - self.batch_size]), - to_variable(items_np[slice:slice + - self.batch_size])) + to_variable( + users_np[slice : slice + self.batch_size] + ), + to_variable( + items_np[slice : slice + self.batch_size] + ), + ) loss = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction, - to_variable(labels_np[slice:slice + - self.batch_size]))) + to_variable( + labels_np[slice : slice + self.batch_size] + ), + ) + ) loss.backward() adam.minimize(loss) deepcf.clear_gradients() eager_loss = loss.numpy() - sys.stderr.write('eager loss: %s %s\n' % - (slice, eager_loss)) + sys.stderr.write( + 'eager loss: %s %s\n' % (slice, eager_loss) + ) self.assertEqual(static_loss, dy_loss) self.assertEqual(static_loss, dy_loss2) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_double_grad.py b/python/paddle/fluid/tests/unittests/test_imperative_double_grad.py index 1d77c513239..140f1191c96 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_double_grad.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_double_grad.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import _test_eager_guard def _dygraph_guard_(func): - def __impl__(*args, **kwargs): if fluid._non_static_mode(): return func(*args, **kwargs) @@ -43,7 +42,6 @@ def random_var(size, low=-1, high=1, dtype='float32'): class TestEagerGrad(TestCase): - def func_simple_example_eager_grad(self): np.random.seed(2021) paddle.set_device('cpu') @@ -184,8 +182,9 @@ class TestEagerGrad(TestCase): out4 = paddle.mean(out3) egr_dout2, egr_dout3 = paddle.grad([out4], [out2, out3]) - np.testing.assert_array_equal(dout2_record_by_hook[0].numpy(), - np.array([1.0, 2.0])) + np.testing.assert_array_equal( + dout2_record_by_hook[0].numpy(), np.array([1.0, 2.0]) + ) x1 = paddle.to_tensor([1.0, 2.0]) x1.stop_gradient = False @@ -205,27 +204,30 @@ class TestEagerGrad(TestCase): class TestDygraphDoubleGrad(TestCase): - def setUp(self): self.sort_sum_gradient = False self.shape = [5, 10] - def grad(self, - outputs, - inputs, - grad_outputs=None, - no_grad_vars=None, - retain_graph=None, - create_graph=False, - allow_unused=False): + def grad( + self, + outputs, + inputs, + grad_outputs=None, + no_grad_vars=None, + retain_graph=None, + create_graph=False, + allow_unused=False, + ): fluid.set_flags({'FLAGS_sort_sum_gradient': self.sort_sum_gradient}) - return fluid.dygraph.grad(outputs=outputs, - inputs=inputs, - grad_outputs=grad_outputs, - no_grad_vars=no_grad_vars, - retain_graph=retain_graph, - create_graph=create_graph, - allow_unused=allow_unused) + return fluid.dygraph.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=grad_outputs, + no_grad_vars=no_grad_vars, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=allow_unused, + ) @dygraph_guard def func_exception(self): @@ -247,12 +249,16 @@ class TestDygraphDoubleGrad(TestCase): self.grad([random_var(shape)], [1]) with self.assertRaises(AssertionError): - self.grad([random_var(shape), random_var(shape)], - [random_var(shape)], [random_var(shape)]) + self.grad( + [random_var(shape), random_var(shape)], + [random_var(shape)], + [random_var(shape)], + ) with self.assertRaises(AssertionError): - self.grad([random_var(shape)], [random_var(shape)], - no_grad_vars=[1]) + self.grad( + [random_var(shape)], [random_var(shape)], no_grad_vars=[1] + ) with self.assertRaises(AssertionError): self.grad([random_var(shape)], [random_var(shape)], no_grad_vars=1) @@ -269,31 +275,33 @@ class TestDygraphDoubleGrad(TestCase): y = x + 1 for create_graph in [False, True]: - dx, = self.grad([x], [x], - create_graph=create_graph, - retain_graph=True) + (dx,) = self.grad( + [x], [x], create_graph=create_graph, retain_graph=True + ) self.assertEqual(dx.shape, x.shape) self.assertTrue(np.all(dx.numpy() == 1)) self.assertNotEqual(dx.stop_gradient, create_graph) - dx_mul_2, = self.grad([y, x], [x], - create_graph=create_graph, - retain_graph=True) + (dx_mul_2,) = self.grad( + [y, x], [x], create_graph=create_graph, retain_graph=True + ) self.assertEqual(dx_mul_2.shape, x.shape) self.assertTrue(np.all(dx_mul_2.numpy() == 2)) self.assertNotEqual(dx_mul_2.stop_gradient, create_graph) - none_grad, = self.grad([x], [y], - create_graph=create_graph, - allow_unused=True) + (none_grad,) = self.grad( + [x], [y], create_graph=create_graph, allow_unused=True + ) self.assertTrue(none_grad is None) - grad_with_none_and_not_none, = self.grad([x, y], [y], - create_graph=create_graph) + (grad_with_none_and_not_none,) = self.grad( + [x, y], [y], create_graph=create_graph + ) self.assertTrue(grad_with_none_and_not_none.shape, x.shape) self.assertTrue(np.all(grad_with_none_and_not_none.numpy() == 1)) - self.assertNotEqual(grad_with_none_and_not_none.stop_gradient, - create_graph) + self.assertNotEqual( + grad_with_none_and_not_none.stop_gradient, create_graph + ) def test_simple_example(self): with _test_eager_guard(): @@ -315,15 +323,20 @@ class TestDygraphDoubleGrad(TestCase): w_mean = fluid.layers.reduce_mean(w) del y1, z, w - dx_actual, = self.grad([w_mean], [x], - create_graph=True, - no_grad_vars=[y2]) + (dx_actual,) = self.grad( + [w_mean], [x], create_graph=True, no_grad_vars=[y2] + ) self.assertFalse(y2.stop_gradient) self.assertFalse(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + y2.numpy()) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 + / float(numel) + * (np.maximum(x_np, 0) + y2.numpy()) + * (x_np > 0) + * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) @@ -340,11 +353,12 @@ class TestDygraphDoubleGrad(TestCase): half_numel = int(numel / 2) half_x_positive = np.random.uniform(low=1, high=2, size=[half_numel]) - half_x_negative = np.random.uniform(low=-2, - high=-1, - size=[numel - half_numel]) - x_np = np.array(list(half_x_positive) + - list(half_x_negative)).astype('float32') + half_x_negative = np.random.uniform( + low=-2, high=-1, size=[numel - half_numel] + ) + x_np = np.array(list(half_x_positive) + list(half_x_negative)).astype( + 'float32' + ) np.random.shuffle(x_np) x = fluid.dygraph.to_variable(x_np) @@ -359,8 +373,9 @@ class TestDygraphDoubleGrad(TestCase): relu_x_np = np.maximum(x_np, alpha * x_np).astype('float32') relu_x_grad_np = ((x_np > 0) + (x_np < 0) * alpha).astype('float32') dy_expected = (relu_x_np * relu_x_grad_np * 2).astype('float32') - dz_expected = (np.power(relu_x_np, 3) * relu_x_grad_np * - 4).astype('float32') + dz_expected = (np.power(relu_x_np, 3) * relu_x_grad_np * 4).astype( + 'float32' + ) random_grad_y = random_var(y.shape, low=1, high=2) random_grad_z = random_var(z.shape, low=1, high=2) @@ -373,31 +388,39 @@ class TestDygraphDoubleGrad(TestCase): for grad_y in [random_grad_y]: for grad_z in [random_grad_z]: for create_graph in [False, True]: - dx_actual, = self.grad(outputs=[y, z], - inputs=[x], - grad_outputs=[grad_y, grad_z], - create_graph=create_graph, - retain_graph=True) + (dx_actual,) = self.grad( + outputs=[y, z], + inputs=[x], + grad_outputs=[grad_y, grad_z], + create_graph=create_graph, + retain_graph=True, + ) - grad_y_np = ones_grad_y if grad_y is None else grad_y.numpy( + grad_y_np = ( + ones_grad_y if grad_y is None else grad_y.numpy() ) - grad_z_np = ones_grad_z if grad_z is None else grad_z.numpy( + grad_z_np = ( + ones_grad_z if grad_z is None else grad_z.numpy() ) - dx_expected = dy_expected * grad_y_np + dz_expected * grad_z_np - np.testing.assert_allclose(dx_actual.numpy(), - dx_expected, - rtol=1e-05) + dx_expected = ( + dy_expected * grad_y_np + dz_expected * grad_z_np + ) + np.testing.assert_allclose( + dx_actual.numpy(), dx_expected, rtol=1e-05 + ) if grad_y is not None: self.assertTrue(grad_y.stop_gradient) - np.testing.assert_array_equal(grad_y.numpy(), - original_random_grad_y) + np.testing.assert_array_equal( + grad_y.numpy(), original_random_grad_y + ) if grad_z is not None: self.assertTrue(grad_z.stop_gradient) - np.testing.assert_array_equal(grad_z.numpy(), - original_random_grad_z) + np.testing.assert_array_equal( + grad_z.numpy(), original_random_grad_z + ) def test_none_one_initial_gradient(self): with _test_eager_guard(): @@ -418,35 +441,39 @@ class TestDygraphDoubleGrad(TestCase): w_mean = fluid.layers.reduce_mean(w) del y, z, w - dx_actual, = self.grad([w_mean], [x], create_graph=True) + (dx_actual,) = self.grad([w_mean], [x], create_graph=True) del w_mean self.assertFalse(dx_actual.stop_gradient) # Theoritical result based on math calculation - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * (x_np > 0) * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) loss = fluid.layers.reduce_mean(dx_actual * dx_actual + x * x) loss.backward(retain_graph=True) x_grad_actual = x.gradient() - x_grad_expected = (2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 2 / float(numel))).astype('float32') + x_grad_expected = ( + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 2 / float(numel)) + ).astype('float32') np.testing.assert_allclose(x_grad_actual, x_grad_expected, rtol=1e-05) for i in range(5): loss.backward(retain_graph=True) x_grad_actual = x.gradient() - x_grad_expected = ( - i + 2) * (2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 2 / float(numel))).astype('float32') - np.testing.assert_allclose(x_grad_actual, - x_grad_expected, - rtol=1e-05) + x_grad_expected = (i + 2) * ( + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 2 / float(numel)) + ).astype('float32') + np.testing.assert_allclose( + x_grad_actual, x_grad_expected, rtol=1e-05 + ) def test_example_with_gradient_accumulation_and_create_graph(self): with _test_eager_guard(): @@ -468,25 +495,35 @@ class TestDygraphDoubleGrad(TestCase): w_mean = fluid.layers.reduce_mean(w) del y1, z, w - dx_actual, = self.grad([w_mean], [x], - retain_graph=True, - create_graph=True, - no_grad_vars=[y2]) + (dx_actual,) = self.grad( + [w_mean], + [x], + retain_graph=True, + create_graph=True, + no_grad_vars=[y2], + ) self.assertFalse(y2.stop_gradient) self.assertFalse(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + y2.numpy()) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 + / float(numel) + * (np.maximum(x_np, 0) + y2.numpy()) + * (x_np > 0) + * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) loss = fluid.layers.reduce_mean(dx_actual * dx_actual + x * x) loss.backward() x_grad_actual = x.gradient() - x_grad_expected = (2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 4 / float(numel))).astype('float32') + x_grad_expected = ( + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 4 / float(numel)) + ).astype('float32') np.testing.assert_allclose(x_grad_actual, x_grad_expected, rtol=1e-05) def test_example_with_gradient_accumulation_and_no_grad_vars(self): @@ -508,13 +545,14 @@ class TestDygraphDoubleGrad(TestCase): w_mean = fluid.layers.reduce_mean(w) del y, z, w - dx_actual, = self.grad([w_mean], [x], create_graph=False) + (dx_actual,) = self.grad([w_mean], [x], create_graph=False) del w_mean self.assertTrue(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * (x_np > 0) * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) @@ -532,17 +570,18 @@ class TestDygraphDoubleGrad(TestCase): class TestDygraphDoubleGradSortGradient(TestDygraphDoubleGrad): - def setUp(self): self.sort_sum_gradient = True self.shape = [5, 10] class TestDygraphDoubleGradVisitedUniq(TestCase): - def func_compare(self): - value = np.random.uniform(-0.5, 0.5, 100).reshape(10, 2, - 5).astype("float32") + value = ( + np.random.uniform(-0.5, 0.5, 100) + .reshape(10, 2, 5) + .astype("float32") + ) def model_f(input): linear = fluid.dygraph.Linear(5, 3, bias_attr=False) @@ -563,11 +602,13 @@ class TestDygraphDoubleGradVisitedUniq(TestCase): out = model_f(a) - dx = fluid.dygraph.grad(outputs=[out], - inputs=[a], - create_graph=False, - only_inputs=True, - allow_unused=False) + dx = fluid.dygraph.grad( + outputs=[out], + inputs=[a], + create_graph=False, + only_inputs=True, + allow_unused=False, + ) grad_1 = dx[0].numpy() @@ -591,17 +632,15 @@ class TestDygraphDoubleGradVisitedUniq(TestCase): class TestRaiseNoDoubleGradOp(TestCase): - def raise_no_grad_op(self): with fluid.dygraph.guard(): x = fluid.layers.ones(shape=[2, 3, 2, 2], dtype='float32') x.stop_gradient = False y = paddle.fluid.layers.group_norm(x, groups=1) - dx = fluid.dygraph.grad(outputs=[y], - inputs=[x], - create_graph=True, - retain_graph=True)[0] + dx = fluid.dygraph.grad( + outputs=[y], inputs=[x], create_graph=True, retain_graph=True + )[0] loss = fluid.layers.reduce_mean(dx) loss.backward() @@ -611,7 +650,6 @@ class TestRaiseNoDoubleGradOp(TestCase): class TestDoubleGradResNet(TestCase): - def setUp(self): paddle.seed(123) paddle.framework.random._manual_program_seed(123) @@ -626,7 +664,8 @@ class TestDoubleGradResNet(TestCase): egr_out = model(egr_data) egr_preds = paddle.argmax(egr_out, axis=1) egr_label_onehot = paddle.nn.functional.one_hot( - paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1]) + paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1] + ) egr_target = paddle.sum(egr_out * egr_label_onehot, axis=1) egr_g = paddle.grad(outputs=egr_target, inputs=egr_out)[0] @@ -638,8 +677,9 @@ class TestDoubleGradResNet(TestCase): data.stop_gradient = False out = model(data) preds = paddle.argmax(out, axis=1) - label_onehot = paddle.nn.functional.one_hot(paddle.to_tensor(preds), - num_classes=out.shape[1]) + label_onehot = paddle.nn.functional.one_hot( + paddle.to_tensor(preds), num_classes=out.shape[1] + ) target = paddle.sum(out * label_onehot, axis=1) g = paddle.grad(outputs=target, inputs=out)[0] @@ -658,7 +698,8 @@ class TestDoubleGradResNet(TestCase): egr_out = model(egr_data) egr_preds = paddle.argmax(egr_out, axis=1) egr_label_onehot = paddle.nn.functional.one_hot( - paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1]) + paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1] + ) egr_target = paddle.sum(egr_out * egr_label_onehot, axis=1) egr_g = paddle.grad(outputs=egr_target, inputs=egr_out)[0] @@ -670,8 +711,9 @@ class TestDoubleGradResNet(TestCase): data.stop_gradient = False out = model(data) preds = paddle.argmax(out, axis=1) - label_onehot = paddle.nn.functional.one_hot(paddle.to_tensor(preds), - num_classes=out.shape[1]) + label_onehot = paddle.nn.functional.one_hot( + paddle.to_tensor(preds), num_classes=out.shape[1] + ) target = paddle.sum(out * label_onehot, axis=1) g = paddle.grad(outputs=target, inputs=out)[0] @@ -683,24 +725,23 @@ class TestDoubleGradResNet(TestCase): class TestDoubleGradBasics(TestCase): - def test_matmul(self): input_numpy = np.ones([3, 3]) * 2 with _test_eager_guard(): - x = paddle.to_tensor(input_numpy, - stop_gradient=False, - dtype='float32') - y = paddle.to_tensor(input_numpy, - stop_gradient=False, - dtype='float32') - grad_out = paddle.to_tensor(np.ones([3, 3]), - stop_gradient=False, - dtype='float32') + x = paddle.to_tensor( + input_numpy, stop_gradient=False, dtype='float32' + ) + y = paddle.to_tensor( + input_numpy, stop_gradient=False, dtype='float32' + ) + grad_out = paddle.to_tensor( + np.ones([3, 3]), stop_gradient=False, dtype='float32' + ) out = paddle.matmul(x, y, False, False) - new_x_g, new_y_g = paddle.grad([out], [x, y], [grad_out], - retain_graph=True, - create_graph=True) + new_x_g, new_y_g = paddle.grad( + [out], [x, y], [grad_out], retain_graph=True, create_graph=True + ) new_x_g.backward() out_ref = np.ones([3, 3]) * 12.0 @@ -718,8 +759,9 @@ class TestDoubleGradBasics(TestCase): np.testing.assert_array_equal(y.grad.numpy(), y_grad_ref) grad_out_grad_ref = np.ones([3, 3]) * 6.0 - np.testing.assert_array_equal(grad_out.grad.numpy(), - grad_out_grad_ref) + np.testing.assert_array_equal( + grad_out.grad.numpy(), grad_out_grad_ref + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_framework.py b/python/paddle/fluid/tests/unittests/test_imperative_framework.py index 3a1e7c9cd12..dc23f8b0a97 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_framework.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_framework.py @@ -20,23 +20,28 @@ from paddle.fluid.framework import _test_eager_guard class MLP(fluid.Layer): - def __init__(self, input_size): super(MLP, self).__init__() self._linear1 = fluid.dygraph.Linear( input_size, 3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + ) self._linear2 = fluid.dygraph.Linear( 3, 4, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + ) def forward(self, inputs): x = self._linear1(inputs) @@ -46,19 +51,18 @@ class MLP(fluid.Layer): class TestDygraphFramework(unittest.TestCase): - def func_test_dygraph_backward(self): with new_program_scope(): mlp = MLP(input_size=2) - var_inp = fluid.layers.data("input", - shape=[2, 2], - dtype="float32", - append_batch_size=False) + var_inp = fluid.layers.data( + "input", shape=[2, 2], dtype="float32", append_batch_size=False + ) out = mlp(var_inp) try: out.backward() raise AssertionError( - "backward should not be usable in static graph mode") + "backward should not be usable in static graph mode" + ) except AssertionError as e: self.assertTrue((e is not None)) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_gan.py b/python/paddle/fluid/tests/unittests/test_imperative_gan.py index 9edecc89989..8cfe9befb62 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_gan.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_gan.py @@ -26,7 +26,6 @@ from paddle.fluid.framework import _test_eager_guard class Discriminator(fluid.Layer): - def __init__(self): super(Discriminator, self).__init__() self._fc1 = Linear(1, 32, act='elu') @@ -39,7 +38,6 @@ class Discriminator(fluid.Layer): class Generator(fluid.Layer): - def __init__(self): super(Generator, self).__init__() self._fc1 = Linear(2, 64, act='elu') @@ -54,7 +52,6 @@ class Generator(fluid.Layer): class TestDygraphGAN(unittest.TestCase): - def func_test_gan_float32(self): seed = 90 paddle.seed(1) @@ -64,34 +61,38 @@ class TestDygraphGAN(unittest.TestCase): generate_p = fluid.Program() scope = fluid.core.Scope() - with new_program_scope(main=discriminate_p, - startup=startup, - scope=scope): + with new_program_scope( + main=discriminate_p, startup=startup, scope=scope + ): discriminator = Discriminator() generator = Generator() - img = fluid.layers.data(name="img", - shape=[2, 1], - append_batch_size=False) - noise = fluid.layers.data(name="noise", - shape=[2, 2], - append_batch_size=False) + img = fluid.layers.data( + name="img", shape=[2, 1], append_batch_size=False + ) + noise = fluid.layers.data( + name="noise", shape=[2, 2], append_batch_size=False + ) d_real = discriminator(img) d_loss_real = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( x=d_real, - label=fluid.layers.fill_constant(shape=[2, 1], - dtype='float32', - value=1.0))) + label=fluid.layers.fill_constant( + shape=[2, 1], dtype='float32', value=1.0 + ), + ) + ) d_fake = discriminator(generator(noise)) d_loss_fake = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( x=d_fake, - label=fluid.layers.fill_constant(shape=[2, 1], - dtype='float32', - value=0.0))) + label=fluid.layers.fill_constant( + shape=[2, 1], dtype='float32', value=0.0 + ), + ) + ) d_loss = d_loss_real + d_loss_fake @@ -102,42 +103,47 @@ class TestDygraphGAN(unittest.TestCase): discriminator = Discriminator() generator = Generator() - noise = fluid.layers.data(name="noise", - shape=[2, 2], - append_batch_size=False) + noise = fluid.layers.data( + name="noise", shape=[2, 2], append_batch_size=False + ) d_fake = discriminator(generator(noise)) g_loss = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( x=d_fake, - label=fluid.layers.fill_constant(shape=[2, 1], - dtype='float32', - value=1.0))) + label=fluid.layers.fill_constant( + shape=[2, 1], dtype='float32', value=1.0 + ), + ) + ) sgd = SGDOptimizer(learning_rate=1e-3) sgd.minimize(g_loss) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) static_params = dict() with fluid.scope_guard(scope): img = np.ones([2, 1], np.float32) noise = np.ones([2, 2], np.float32) exe.run(startup) - static_d_loss = exe.run(discriminate_p, - feed={ - 'img': img, - 'noise': noise - }, - fetch_list=[d_loss])[0] - static_g_loss = exe.run(generate_p, - feed={'noise': noise}, - fetch_list=[g_loss])[0] + static_d_loss = exe.run( + discriminate_p, + feed={'img': img, 'noise': noise}, + fetch_list=[d_loss], + )[0] + static_g_loss = exe.run( + generate_p, feed={'noise': noise}, fetch_list=[g_loss] + )[0] # generate_p contains all parameters needed. for param in generate_p.global_block().all_parameters(): static_params[param.name] = np.array( - scope.find_var(param.name).get_tensor()) + scope.find_var(param.name).get_tensor() + ) dy_params = dict() with fluid.dygraph.guard(): @@ -146,20 +152,28 @@ class TestDygraphGAN(unittest.TestCase): discriminator = Discriminator() generator = Generator() - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=(discriminator.parameters() + - generator.parameters())) + sgd = SGDOptimizer( + learning_rate=1e-3, + parameter_list=( + discriminator.parameters() + generator.parameters() + ), + ) d_real = discriminator(to_variable(np.ones([2, 1], np.float32))) d_loss_real = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_real, label=to_variable(np.ones([2, 1], np.float32)))) + x=d_real, label=to_variable(np.ones([2, 1], np.float32)) + ) + ) d_fake = discriminator( - generator(to_variable(np.ones([2, 2], np.float32)))) + generator(to_variable(np.ones([2, 2], np.float32))) + ) d_loss_fake = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_fake, label=to_variable(np.zeros([2, 1], np.float32)))) + x=d_fake, label=to_variable(np.zeros([2, 1], np.float32)) + ) + ) d_loss = d_loss_real + d_loss_fake d_loss.backward() @@ -168,10 +182,13 @@ class TestDygraphGAN(unittest.TestCase): generator.clear_gradients() d_fake = discriminator( - generator(to_variable(np.ones([2, 2], np.float32)))) + generator(to_variable(np.ones([2, 2], np.float32))) + ) g_loss = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_fake, label=to_variable(np.ones([2, 1], np.float32)))) + x=d_fake, label=to_variable(np.ones([2, 1], np.float32)) + ) + ) g_loss.backward() sgd.minimize(g_loss) for p in discriminator.parameters(): @@ -189,20 +206,28 @@ class TestDygraphGAN(unittest.TestCase): paddle.framework.random._manual_program_seed(1) discriminator2 = Discriminator() generator2 = Generator() - sgd2 = SGDOptimizer(learning_rate=1e-3, - parameter_list=(discriminator2.parameters() + - generator2.parameters())) + sgd2 = SGDOptimizer( + learning_rate=1e-3, + parameter_list=( + discriminator2.parameters() + generator2.parameters() + ), + ) d_real2 = discriminator2(to_variable(np.ones([2, 1], np.float32))) d_loss_real2 = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_real2, label=to_variable(np.ones([2, 1], np.float32)))) + x=d_real2, label=to_variable(np.ones([2, 1], np.float32)) + ) + ) d_fake2 = discriminator2( - generator2(to_variable(np.ones([2, 2], np.float32)))) + generator2(to_variable(np.ones([2, 2], np.float32))) + ) d_loss_fake2 = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_fake2, label=to_variable(np.zeros([2, 1], np.float32)))) + x=d_fake2, label=to_variable(np.zeros([2, 1], np.float32)) + ) + ) d_loss2 = d_loss_real2 + d_loss_fake2 d_loss2.backward() @@ -211,10 +236,13 @@ class TestDygraphGAN(unittest.TestCase): generator2.clear_gradients() d_fake2 = discriminator2( - generator2(to_variable(np.ones([2, 2], np.float32)))) + generator2(to_variable(np.ones([2, 2], np.float32))) + ) g_loss2 = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_fake2, label=to_variable(np.ones([2, 1], np.float32)))) + x=d_fake2, label=to_variable(np.ones([2, 1], np.float32)) + ) + ) g_loss2.backward() sgd2.minimize(g_loss2) for p in discriminator2.parameters(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py index f001e7c4e0c..1d2e61b0863 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py @@ -30,7 +30,6 @@ def gen_data(): class GraphConv(fluid.Layer): - def __init__(self, name_scope, in_features, out_features): super(GraphConv, self).__init__(name_scope) @@ -39,10 +38,11 @@ class GraphConv(fluid.Layer): self.weight = self.create_parameter( attr=None, dtype='float32', - shape=[self._in_features, self._out_features]) - self.bias = self.create_parameter(attr=None, - dtype='float32', - shape=[self._out_features]) + shape=[self._in_features, self._out_features], + ) + self.bias = self.create_parameter( + attr=None, dtype='float32', shape=[self._out_features] + ) def forward(self, features, adj): support = fluid.layers.matmul(features, self.weight) @@ -51,7 +51,6 @@ class GraphConv(fluid.Layer): class GCN(fluid.Layer): - def __init__(self, name_scope, num_hidden): super(GCN, self).__init__(name_scope) self.gc = GraphConv(self.full_name(), num_hidden, 32) @@ -63,7 +62,6 @@ class GCN(fluid.Layer): class TestDygraphGNN(unittest.TestCase): - def func_gnn_float32(self): paddle.seed(90) paddle.framework.random._manual_program_seed(90) @@ -72,19 +70,25 @@ class TestDygraphGNN(unittest.TestCase): scope = fluid.core.Scope() with new_program_scope(main=main, startup=startup, scope=scope): - features = fluid.layers.data(name='features', - shape=[1, 100, 50], - dtype='float32', - append_batch_size=False) + features = fluid.layers.data( + name='features', + shape=[1, 100, 50], + dtype='float32', + append_batch_size=False, + ) # Use selected rows when it's supported. - adj = fluid.layers.data(name='adj', - shape=[1, 100, 100], - dtype='float32', - append_batch_size=False) - labels = fluid.layers.data(name='labels', - shape=[100, 1], - dtype='int64', - append_batch_size=False) + adj = fluid.layers.data( + name='adj', + shape=[1, 100, 100], + dtype='float32', + append_batch_size=False, + ) + labels = fluid.layers.data( + name='labels', + shape=[100, 1], + dtype='int64', + append_batch_size=False, + ) model = GCN('test_gcn', 50) logits = model(features, adj) @@ -96,21 +100,24 @@ class TestDygraphGNN(unittest.TestCase): adam = AdamOptimizer(learning_rate=1e-3) adam.minimize(loss) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(startup) - static_loss = exe.run(feed={ - 'features': - np.ones([1, 100, 50], dtype=np.float32), - 'adj': - np.ones([1, 100, 100], dtype=np.float32), - 'labels': - np.ones([100, 1], dtype=np.int64) - }, - fetch_list=[loss])[0] + static_loss = exe.run( + feed={ + 'features': np.ones([1, 100, 50], dtype=np.float32), + 'adj': np.ones([1, 100, 100], dtype=np.float32), + 'labels': np.ones([100, 1], dtype=np.int64), + }, + fetch_list=[loss], + )[0] static_weight = np.array( - scope.find_var(model.gc.weight.name).get_tensor()) + scope.find_var(model.gc.weight.name).get_tensor() + ) with fluid.dygraph.guard(): paddle.seed(90) @@ -127,11 +134,13 @@ class TestDygraphGNN(unittest.TestCase): # In other example, it's nll with log_softmax. However, paddle's # log_loss only supports binary classification now. loss = fluid.layers.softmax_with_cross_entropy( - logits, to_variable(labels)) + logits, to_variable(labels) + ) loss = fluid.layers.reduce_sum(loss) loss.backward() - adam = AdamOptimizer(learning_rate=1e-3, - parameter_list=model.parameters()) + adam = AdamOptimizer( + learning_rate=1e-3, parameter_list=model.parameters() + ) adam.minimize(loss) model.clear_gradients() @@ -153,24 +162,26 @@ class TestDygraphGNN(unittest.TestCase): # In other example, it's nll with log_softmax. However, paddle's # log_loss only supports binary classification now. loss2 = fluid.layers.softmax_with_cross_entropy( - logits2, to_variable(labels2)) + logits2, to_variable(labels2) + ) loss2 = fluid.layers.reduce_sum(loss2) loss2.backward() - adam2 = AdamOptimizer(learning_rate=1e-3, - parameter_list=model2.parameters()) + adam2 = AdamOptimizer( + learning_rate=1e-3, parameter_list=model2.parameters() + ) adam2.minimize(loss2) model2.clear_gradients() loss2_value = loss2.numpy() model2_gc_weight_value = model2.gc.weight.numpy() self.assertEqual(static_loss, loss_value) - np.testing.assert_allclose(static_weight, - model_gc_weight_value, - rtol=1e-05) + np.testing.assert_allclose( + static_weight, model_gc_weight_value, rtol=1e-05 + ) self.assertEqual(static_loss, loss2_value) - np.testing.assert_allclose(static_weight, - model2_gc_weight_value, - rtol=1e-05) + np.testing.assert_allclose( + static_weight, model2_gc_weight_value, rtol=1e-05 + ) sys.stderr.write('%s %s\n' % (static_loss, loss_value)) def test_gnn_float32(self): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_group.py b/python/paddle/fluid/tests/unittests/test_imperative_group.py index f67592312c8..672dee430f8 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_group.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_group.py @@ -17,11 +17,14 @@ import unittest import paddle import paddle.fluid.core as core -from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph, in_dygraph_mode +from paddle.fluid.framework import ( + _test_eager_guard, + _in_legacy_dygraph, + in_dygraph_mode, +) class TestDataParallelGroup(unittest.TestCase): - def create_varbase(self, dtype, shape): return paddle.rand(shape=shape, dtype=dtype) @@ -38,8 +41,9 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [2, 100])) var_list.append(self.create_varbase("float32", [2, 50])) var_list.append(self.create_varbase("float32", [2, 25])) - res = self.assign_group_by_size(var_list, [False, False, False, False], - [400]) + res = self.assign_group_by_size( + var_list, [False, False, False, False], [400] + ) self.assertEqual([[0], [1], [2], [3]], res) def test_construct_group1(self): @@ -52,7 +56,8 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [False, False, False, False, False, False], [400]) + var_list, [False, False, False, False, False, False], [400] + ) self.assertEqual([[0, 2], [1, 3], [4], [5]], res) def test_construct_group2(self): @@ -62,8 +67,9 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [2, 50])) var_list.append(self.create_varbase("float32", [2, 50])) var_list.append(self.create_varbase("float32", [2, 50])) - res = self.assign_group_by_size(var_list, [False, False, False, False], - [400, 800]) + res = self.assign_group_by_size( + var_list, [False, False, False, False], [400, 800] + ) self.assertEqual([[0], [1, 2], [3]], res) def test_construct_group3(self): @@ -76,7 +82,8 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [False, False, False, False, False, False], [200, 400]) + var_list, [False, False, False, False, False, False], [200, 400] + ) self.assertEqual([[0], [1], [2, 4], [3, 5]], res) def test_construct_group4(self): @@ -89,7 +96,8 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [False, False, False, False, False, False], [0]) + var_list, [False, False, False, False, False, False], [0] + ) self.assertEqual([[0], [1], [2], [3], [4], [5]], res) def test_construct_group5(self): @@ -102,23 +110,27 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [False, False, False, False, False, False], [10000]) + var_list, [False, False, False, False, False, False], [10000] + ) self.assertEqual([[0, 2, 4], [1, 3, 5]], res) def test_construct_group6(self): # multi dtype & limit capability & multi tensor type var_list = [] - var_list.append(self.create_varbase( - "float32", - [1, 50], - )) + var_list.append( + self.create_varbase( + "float32", + [1, 50], + ) + ) var_list.append(self.create_varbase("float64", [1, 25])) var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [True, False, False, False, False, True], [400]) + var_list, [True, False, False, False, False, True], [400] + ) self.assertEqual([[0], [1, 3], [2, 4], [5]], res) def test_construct_group7(self): @@ -131,7 +143,8 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [True, False, False, False, False, True], [200, 400]) + var_list, [True, False, False, False, False, True], [200, 400] + ) self.assertEqual([[0], [1], [2], [3], [4], [5]], res) def test_construct_group8(self): @@ -141,8 +154,9 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [2, 100])) var_list.append(self.create_varbase("float32", [2, 50])) var_list.append(self.create_varbase("float32", [2, 25])) - res = self.assign_group_by_size(var_list, [False, False, False, False], - [400], [3, 0, 1, 2]) + res = self.assign_group_by_size( + var_list, [False, False, False, False], [400], [3, 0, 1, 2] + ) self.assertEqual([[3, 0], [1], [2]], res) def test_construct_group9(self): @@ -152,8 +166,9 @@ class TestDataParallelGroup(unittest.TestCase): var_list.append(self.create_varbase("float32", [2, 25])) var_list.append(self.create_varbase("float32", [2, 25])) var_list.append(self.create_varbase("float32", [2, 1000])) - res = self.assign_group_by_size(var_list, [False, False, False, True], - [300], [1, 0, 2, 3]) + res = self.assign_group_by_size( + var_list, [False, False, False, True], [300], [1, 0, 2, 3] + ) self.assertEqual([[1, 0], [3], [2]], res) def test_construct_group_in_legacy_mode(self): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_hook_for_layer.py b/python/paddle/fluid/tests/unittests/test_imperative_hook_for_layer.py index 1f60bf20e09..462f614defa 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_hook_for_layer.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_hook_for_layer.py @@ -60,27 +60,37 @@ class Test_Forward_Hook(unittest.TestCase): fluid.default_main_program().random_seed = seed fluid.set_flags({'FLAGS_sort_sum_gradient': True}) - input_word = np.array( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, - 8]).reshape(6, 3).astype('int64') + input_word = ( + np.array( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8] + ) + .reshape(6, 3) + .astype('int64') + ) input_word1 = input_word * 2 input_word = input_word.reshape((-1, 3, 1)) input_word1 = input_word1.reshape((-1, 3, 1)) - y_data = np.array( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, - 9]).reshape(6, 3).astype('int64') + y_data = ( + np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ) + .reshape(6, 3) + .astype('int64') + ) y_data = y_data.reshape((-1, 1)) input = base.to_variable(input_word) input1 = base.to_variable(input_word1) y = base.to_variable(y_data) - simplenet = SimpleNet(hidden_size=20, - vocab_size=32, - num_steps=3, - init_scale=0.1, - is_sparse=False, - dtype="float32") + simplenet = SimpleNet( + hidden_size=20, + vocab_size=32, + num_steps=3, + init_scale=0.1, + is_sparse=False, + dtype="float32", + ) # origin, don't register any hook outs_origin = simplenet(input, y) @@ -88,29 +98,35 @@ class Test_Forward_Hook(unittest.TestCase): # register forward_pre_hook forward_pre_hook_handle1 = simplenet.register_forward_pre_hook( - forward_pre_hook1) + forward_pre_hook1 + ) outs_pre_hook = simplenet(input, y) - np.testing.assert_array_equal(outs_pre_hook.numpy(), - outs_origin1.numpy()) + np.testing.assert_array_equal( + outs_pre_hook.numpy(), outs_origin1.numpy() + ) # remove forward_pre_hook forward_pre_hook_handle1.remove() outs_pre_hook = simplenet(input, y) - np.testing.assert_array_equal(outs_pre_hook.numpy(), - outs_origin.numpy()) + np.testing.assert_array_equal( + outs_pre_hook.numpy(), outs_origin.numpy() + ) # register forward_posst_hook - forward_post_hook_handle1 = simplenet.register_forward_post_hook( - forward_post_hook1) + forward_post_hook_handle1 = ( + simplenet.register_forward_post_hook(forward_post_hook1) + ) outs_forward_hook = simplenet(input, y) - np.testing.assert_array_equal(outs_forward_hook.numpy(), - outs_origin.numpy() * 2) + np.testing.assert_array_equal( + outs_forward_hook.numpy(), outs_origin.numpy() * 2 + ) # remove forward_post_hook forward_post_hook_handle1.remove() outs_forward_hook = simplenet(input, y) - np.testing.assert_array_equal(outs_forward_hook.numpy(), - outs_origin.numpy()) + np.testing.assert_array_equal( + outs_forward_hook.numpy(), outs_origin.numpy() + ) # test forward_pre_hook and forward_post_hook that don't have return value def func_forward_hook(self): @@ -129,24 +145,34 @@ class Test_Forward_Hook(unittest.TestCase): global call_forward_post_hook global call_forward_pre_hook - input_word = np.array( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, - 8]).reshape(6, 3).astype('int64') + input_word = ( + np.array( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8] + ) + .reshape(6, 3) + .astype('int64') + ) input_word = input_word.reshape((-1, 3, 1)) - y_data = np.array( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, - 9]).reshape(6, 3).astype('int64') + y_data = ( + np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ) + .reshape(6, 3) + .astype('int64') + ) y_data = y_data.reshape((-1, 1)) input = base.to_variable(input_word) y = base.to_variable(y_data) - simplenet = SimpleNet(hidden_size=20, - vocab_size=32, - num_steps=3, - init_scale=0.1, - is_sparse=False, - dtype="float32") + simplenet = SimpleNet( + hidden_size=20, + vocab_size=32, + num_steps=3, + init_scale=0.1, + is_sparse=False, + dtype="float32", + ) # origin, don't register any hook outs_origin = simplenet(input, y) @@ -155,9 +181,11 @@ class Test_Forward_Hook(unittest.TestCase): # register forward_post_hook and forward_pre_hook forward_post_hook_handle = simplenet.register_forward_post_hook( - forward_post_hook) + forward_post_hook + ) forward_pre_hook_handle = simplenet.register_forward_pre_hook( - forward_pre_hook) + forward_pre_hook + ) outs_hook = simplenet(input, y) self.assertTrue(call_forward_post_hook) self.assertTrue(call_forward_pre_hook) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_layer_apply.py b/python/paddle/fluid/tests/unittests/test_imperative_layer_apply.py index 58e56fc6a6d..40e24abeccd 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_layer_apply.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_layer_apply.py @@ -23,21 +23,25 @@ from paddle.fluid.framework import _test_eager_guard class LeNetDygraph(fluid.dygraph.Layer): - def __init__(self, num_classes=10, classifier_activation='softmax'): super(LeNetDygraph, self).__init__() self.num_classes = num_classes - self.features = nn.Sequential(nn.Conv2D(1, 6, 3, stride=1, padding=1), - nn.ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2), - nn.Conv2D(6, 16, 5, stride=1, padding=0), - nn.ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2)) + self.features = nn.Sequential( + nn.Conv2D(1, 6, 3, stride=1, padding=1), + nn.ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + nn.Conv2D(6, 16, 5, stride=1, padding=0), + nn.ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + ) if num_classes > 0: - self.fc = nn.Sequential(nn.Linear(400, 120), nn.Linear(120, 84), - nn.Linear(84, 10), - nn.Softmax()) #Todo: accept any activation + self.fc = nn.Sequential( + nn.Linear(400, 120), + nn.Linear(120, 84), + nn.Linear(84, 10), + nn.Softmax(), + ) # Todo: accept any activation def forward(self, inputs): x = self.features(inputs) @@ -50,27 +54,26 @@ class LeNetDygraph(fluid.dygraph.Layer): def init_weights(layer): if type(layer) == nn.Linear: - new_weight = paddle.fluid.layers.fill_constant(layer.weight.shape, - layer.weight.dtype, - value=0.9) + new_weight = paddle.fluid.layers.fill_constant( + layer.weight.shape, layer.weight.dtype, value=0.9 + ) layer.weight.set_value(new_weight) - new_bias = paddle.fluid.layers.fill_constant(layer.bias.shape, - layer.bias.dtype, - value=-0.1) + new_bias = paddle.fluid.layers.fill_constant( + layer.bias.shape, layer.bias.dtype, value=-0.1 + ) layer.bias.set_value(new_bias) elif type(layer) == nn.Conv2D: - new_weight = paddle.fluid.layers.fill_constant(layer.weight.shape, - layer.weight.dtype, - value=0.7) + new_weight = paddle.fluid.layers.fill_constant( + layer.weight.shape, layer.weight.dtype, value=0.7 + ) layer.weight.set_value(new_weight) - new_bias = paddle.fluid.layers.fill_constant(layer.bias.shape, - layer.bias.dtype, - value=-0.2) + new_bias = paddle.fluid.layers.fill_constant( + layer.bias.shape, layer.bias.dtype, value=-0.2 + ) layer.bias.set_value(new_bias) class TestLayerApply(unittest.TestCase): - def func_apply_init_weight(self): with fluid.dygraph.guard(): net = LeNetDygraph() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_layer_children.py b/python/paddle/fluid/tests/unittests/test_imperative_layer_children.py index ded833bd9a8..4eeaf8da857 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_layer_children.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_layer_children.py @@ -23,15 +23,16 @@ from paddle.fluid.framework import _test_eager_guard class LeNetDygraph(fluid.dygraph.Layer): - def __init__(self): super(LeNetDygraph, self).__init__() - self.features = nn.Sequential(nn.Conv2D(1, 6, 3, stride=1, padding=1), - nn.ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2), - nn.Conv2D(6, 16, 5, stride=1, padding=0), - nn.ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2)) + self.features = nn.Sequential( + nn.Conv2D(1, 6, 3, stride=1, padding=1), + nn.ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + nn.Conv2D(6, 16, 5, stride=1, padding=0), + nn.ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + ) def forward(self, inputs): x = self.features(inputs) @@ -39,7 +40,6 @@ class LeNetDygraph(fluid.dygraph.Layer): class TestLayerChildren(unittest.TestCase): - def func_apply_init_weight(self): with fluid.dygraph.guard(): net = LeNetDygraph() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_layer_trainable.py b/python/paddle/fluid/tests/unittests/test_imperative_layer_trainable.py index c359d99c819..b0dcfd653fb 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_layer_trainable.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_layer_trainable.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestImperativeLayerTrainable(unittest.TestCase): - def func_set_trainable(self): with fluid.dygraph.guard(): label = np.random.uniform(-1, 1, [10, 10]).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_layers.py b/python/paddle/fluid/tests/unittests/test_imperative_layers.py index e2b9855af41..e0cd3d4f88f 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_layers.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_layers.py @@ -18,7 +18,6 @@ from paddle.fluid.framework import _test_eager_guard class TestLayerPrint(unittest.TestCase): - def func_test_layer_str(self): module = nn.ELU(0.2) self.assertEqual(str(module), 'ELU(alpha=0.2)') @@ -39,13 +38,14 @@ class TestLayerPrint(unittest.TestCase): self.assertEqual(str(module), 'Tanh(name=Tanh)') module = nn.Hardtanh(name="Hardtanh") - self.assertEqual(str(module), - 'Hardtanh(min=-1.0, max=1.0, name=Hardtanh)') + self.assertEqual( + str(module), 'Hardtanh(min=-1.0, max=1.0, name=Hardtanh)' + ) module = nn.PReLU(1, 0.25, name="PReLU", data_format="NCHW") self.assertEqual( str(module), - 'PReLU(num_parameters=1, data_format=NCHW, init=0.25, dtype=float32, name=PReLU)' + 'PReLU(num_parameters=1, data_format=NCHW, init=0.25, dtype=float32, name=PReLU)', ) module = nn.ReLU() @@ -57,7 +57,8 @@ class TestLayerPrint(unittest.TestCase): module = nn.SELU() self.assertEqual( str(module), - 'SELU(scale=1.0507009873554805, alpha=1.6732632423543772)') + 'SELU(scale=1.0507009873554805, alpha=1.6732632423543772)', + ) module = nn.LeakyReLU() self.assertEqual(str(module), 'LeakyReLU(negative_slope=0.01)') @@ -101,32 +102,35 @@ class TestLayerPrint(unittest.TestCase): module = nn.Linear(2, 4, name='linear') self.assertEqual( str(module), - 'Linear(in_features=2, out_features=4, dtype=float32, name=linear)') + 'Linear(in_features=2, out_features=4, dtype=float32, name=linear)', + ) module = nn.Upsample(size=[12, 12]) self.assertEqual( str(module), - 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=NCHW)' + 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=NCHW)', ) module = nn.UpsamplingNearest2D(size=[12, 12]) self.assertEqual( - str(module), 'UpsamplingNearest2D(size=[12, 12], data_format=NCHW)') + str(module), 'UpsamplingNearest2D(size=[12, 12], data_format=NCHW)' + ) module = nn.UpsamplingBilinear2D(size=[12, 12]) self.assertEqual( - str(module), - 'UpsamplingBilinear2D(size=[12, 12], data_format=NCHW)') + str(module), 'UpsamplingBilinear2D(size=[12, 12], data_format=NCHW)' + ) module = nn.Bilinear(in1_features=5, in2_features=4, out_features=1000) self.assertEqual( str(module), - 'Bilinear(in1_features=5, in2_features=4, out_features=1000, dtype=float32)' + 'Bilinear(in1_features=5, in2_features=4, out_features=1000, dtype=float32)', ) module = nn.Dropout(p=0.5) - self.assertEqual(str(module), - 'Dropout(p=0.5, axis=None, mode=upscale_in_train)') + self.assertEqual( + str(module), 'Dropout(p=0.5, axis=None, mode=upscale_in_train)' + ) module = nn.Dropout2D(p=0.5) self.assertEqual(str(module), 'Dropout2D(p=0.5, data_format=NCHW)') @@ -140,22 +144,24 @@ class TestLayerPrint(unittest.TestCase): module = nn.Pad1D(padding=[1, 2], mode='constant') self.assertEqual( str(module), - 'Pad1D(padding=[1, 2], mode=constant, value=0.0, data_format=NCL)') + 'Pad1D(padding=[1, 2], mode=constant, value=0.0, data_format=NCL)', + ) module = nn.Pad2D(padding=[1, 0, 1, 2], mode='constant') self.assertEqual( str(module), - 'Pad2D(padding=[1, 0, 1, 2], mode=constant, value=0.0, data_format=NCHW)' + 'Pad2D(padding=[1, 0, 1, 2], mode=constant, value=0.0, data_format=NCHW)', ) module = nn.ZeroPad2D(padding=[1, 0, 1, 2]) - self.assertEqual(str(module), - 'ZeroPad2D(padding=[1, 0, 1, 2], data_format=NCHW)') + self.assertEqual( + str(module), 'ZeroPad2D(padding=[1, 0, 1, 2], data_format=NCHW)' + ) module = nn.Pad3D(padding=[1, 0, 1, 2, 0, 0], mode='constant') self.assertEqual( str(module), - 'Pad3D(padding=[1, 0, 1, 2, 0, 0], mode=constant, value=0.0, data_format=NCDHW)' + 'Pad3D(padding=[1, 0, 1, 2, 0, 0], mode=constant, value=0.0, data_format=NCDHW)', ) module = nn.CosineSimilarity(axis=0) @@ -165,107 +171,127 @@ class TestLayerPrint(unittest.TestCase): self.assertEqual(str(module), 'Embedding(10, 3, sparse=True)') module = nn.Conv1D(3, 2, 3) - self.assertEqual(str(module), - 'Conv1D(3, 2, kernel_size=[3], data_format=NCL)') + self.assertEqual( + str(module), 'Conv1D(3, 2, kernel_size=[3], data_format=NCL)' + ) module = nn.Conv1DTranspose(2, 1, 2) self.assertEqual( str(module), - 'Conv1DTranspose(2, 1, kernel_size=[2], data_format=NCL)') + 'Conv1DTranspose(2, 1, kernel_size=[2], data_format=NCL)', + ) module = nn.Conv2D(4, 6, (3, 3)) - self.assertEqual(str(module), - 'Conv2D(4, 6, kernel_size=[3, 3], data_format=NCHW)') + self.assertEqual( + str(module), 'Conv2D(4, 6, kernel_size=[3, 3], data_format=NCHW)' + ) module = nn.Conv2DTranspose(4, 6, (3, 3)) self.assertEqual( str(module), - 'Conv2DTranspose(4, 6, kernel_size=[3, 3], data_format=NCHW)') + 'Conv2DTranspose(4, 6, kernel_size=[3, 3], data_format=NCHW)', + ) module = nn.Conv3D(4, 6, (3, 3, 3)) self.assertEqual( str(module), - 'Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') + 'Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)', + ) module = nn.Conv3DTranspose(4, 6, (3, 3, 3)) self.assertEqual( str(module), - 'Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') + 'Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)', + ) module = nn.PairwiseDistance() self.assertEqual(str(module), 'PairwiseDistance(p=2.0)') module = nn.InstanceNorm1D(2) - self.assertEqual(str(module), - 'InstanceNorm1D(num_features=2, epsilon=1e-05)') + self.assertEqual( + str(module), 'InstanceNorm1D(num_features=2, epsilon=1e-05)' + ) module = nn.InstanceNorm2D(2) - self.assertEqual(str(module), - 'InstanceNorm2D(num_features=2, epsilon=1e-05)') + self.assertEqual( + str(module), 'InstanceNorm2D(num_features=2, epsilon=1e-05)' + ) module = nn.InstanceNorm3D(2) - self.assertEqual(str(module), - 'InstanceNorm3D(num_features=2, epsilon=1e-05)') + self.assertEqual( + str(module), 'InstanceNorm3D(num_features=2, epsilon=1e-05)' + ) module = nn.GroupNorm(num_channels=6, num_groups=6) self.assertEqual( str(module), - 'GroupNorm(num_groups=6, num_channels=6, epsilon=1e-05)') + 'GroupNorm(num_groups=6, num_channels=6, epsilon=1e-05)', + ) module = nn.LayerNorm([2, 2, 3]) self.assertEqual( - str(module), 'LayerNorm(normalized_shape=[2, 2, 3], epsilon=1e-05)') + str(module), 'LayerNorm(normalized_shape=[2, 2, 3], epsilon=1e-05)' + ) module = nn.BatchNorm1D(1) self.assertEqual( str(module), - 'BatchNorm1D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCL)' + 'BatchNorm1D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCL)', ) module = nn.BatchNorm2D(1) self.assertEqual( str(module), - 'BatchNorm2D(num_features=1, momentum=0.9, epsilon=1e-05)') + 'BatchNorm2D(num_features=1, momentum=0.9, epsilon=1e-05)', + ) module = nn.BatchNorm3D(1) self.assertEqual( str(module), - 'BatchNorm3D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCDHW)' + 'BatchNorm3D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCDHW)', ) module = nn.SyncBatchNorm(2) self.assertEqual( str(module), - 'SyncBatchNorm(num_features=2, momentum=0.9, epsilon=1e-05)') + 'SyncBatchNorm(num_features=2, momentum=0.9, epsilon=1e-05)', + ) module = nn.LocalResponseNorm(size=5) self.assertEqual( str(module), - 'LocalResponseNorm(size=5, alpha=0.0001, beta=0.75, k=1.0)') + 'LocalResponseNorm(size=5, alpha=0.0001, beta=0.75, k=1.0)', + ) module = nn.AvgPool1D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'AvgPool1D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'AvgPool1D(kernel_size=2, stride=2, padding=0)' + ) module = nn.AvgPool2D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'AvgPool2D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'AvgPool2D(kernel_size=2, stride=2, padding=0)' + ) module = nn.AvgPool3D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'AvgPool3D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'AvgPool3D(kernel_size=2, stride=2, padding=0)' + ) module = nn.MaxPool1D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'MaxPool1D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'MaxPool1D(kernel_size=2, stride=2, padding=0)' + ) module = nn.MaxPool2D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'MaxPool2D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'MaxPool2D(kernel_size=2, stride=2, padding=0)' + ) module = nn.MaxPool3D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'MaxPool3D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'MaxPool3D(kernel_size=2, stride=2, padding=0)' + ) module = nn.AdaptiveAvgPool1D(output_size=16) self.assertEqual(str(module), 'AdaptiveAvgPool1D(output_size=16)') @@ -277,16 +303,19 @@ class TestLayerPrint(unittest.TestCase): self.assertEqual(str(module), 'AdaptiveAvgPool3D(output_size=3)') module = nn.AdaptiveMaxPool1D(output_size=16, return_mask=True) - self.assertEqual(str(module), - 'AdaptiveMaxPool1D(output_size=16, return_mask=True)') + self.assertEqual( + str(module), 'AdaptiveMaxPool1D(output_size=16, return_mask=True)' + ) module = nn.AdaptiveMaxPool2D(output_size=3, return_mask=True) - self.assertEqual(str(module), - 'AdaptiveMaxPool2D(output_size=3, return_mask=True)') + self.assertEqual( + str(module), 'AdaptiveMaxPool2D(output_size=3, return_mask=True)' + ) module = nn.AdaptiveMaxPool3D(output_size=3, return_mask=True) - self.assertEqual(str(module), - 'AdaptiveMaxPool3D(output_size=3, return_mask=True)') + self.assertEqual( + str(module), 'AdaptiveMaxPool3D(output_size=3, return_mask=True)' + ) module = nn.SimpleRNNCell(16, 32) self.assertEqual(str(module), 'SimpleRNNCell(16, 32)') @@ -303,49 +332,56 @@ class TestLayerPrint(unittest.TestCase): module = nn.SimpleRNN(16, 32, 2) self.assertEqual( str(module), - 'SimpleRNN(16, 32, num_layers=2\n (0): RNN(\n (cell): SimpleRNNCell(16, 32)\n )\n (1): RNN(\n (cell): SimpleRNNCell(32, 32)\n )\n)' + 'SimpleRNN(16, 32, num_layers=2\n (0): RNN(\n (cell): SimpleRNNCell(16, 32)\n )\n (1): RNN(\n (cell): SimpleRNNCell(32, 32)\n )\n)', ) module = nn.LSTM(16, 32, 2) self.assertEqual( str(module), - 'LSTM(16, 32, num_layers=2\n (0): RNN(\n (cell): LSTMCell(16, 32)\n )\n (1): RNN(\n (cell): LSTMCell(32, 32)\n )\n)' + 'LSTM(16, 32, num_layers=2\n (0): RNN(\n (cell): LSTMCell(16, 32)\n )\n (1): RNN(\n (cell): LSTMCell(32, 32)\n )\n)', ) module = nn.GRU(16, 32, 2) self.assertEqual( str(module), - 'GRU(16, 32, num_layers=2\n (0): RNN(\n (cell): GRUCell(16, 32)\n )\n (1): RNN(\n (cell): GRUCell(32, 32)\n )\n)' + 'GRU(16, 32, num_layers=2\n (0): RNN(\n (cell): GRUCell(16, 32)\n )\n (1): RNN(\n (cell): GRUCell(32, 32)\n )\n)', ) module1 = nn.Sequential( - ('conv1', nn.Conv2D(1, 20, 5)), ('relu1', nn.ReLU()), - ('conv2', nn.Conv2D(20, 64, 5)), ('relu2', nn.ReLU())) + ('conv1', nn.Conv2D(1, 20, 5)), + ('relu1', nn.ReLU()), + ('conv2', nn.Conv2D(20, 64, 5)), + ('relu2', nn.ReLU()), + ) self.assertEqual( str(module1), - 'Sequential(\n '\ - '(conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n '\ - '(relu1): ReLU()\n '\ - '(conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n '\ - '(relu2): ReLU()\n)' + 'Sequential(\n ' + '(conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n ' + '(relu1): ReLU()\n ' + '(conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n ' + '(relu2): ReLU()\n)', ) module2 = nn.Sequential( nn.Conv3DTranspose(4, 6, (3, 3, 3)), nn.AvgPool3D(kernel_size=2, stride=2, padding=0), - nn.Tanh(name="Tanh"), module1, nn.Conv3D(4, 6, (3, 3, 3)), - nn.MaxPool3D(kernel_size=2, stride=2, padding=0), nn.GELU(True)) + nn.Tanh(name="Tanh"), + module1, + nn.Conv3D(4, 6, (3, 3, 3)), + nn.MaxPool3D(kernel_size=2, stride=2, padding=0), + nn.GELU(True), + ) self.assertEqual( str(module2), - 'Sequential(\n '\ - '(0): Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ - '(1): AvgPool3D(kernel_size=2, stride=2, padding=0)\n '\ - '(2): Tanh(name=Tanh)\n '\ - '(3): Sequential(\n (conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n (relu1): ReLU()\n'\ - ' (conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n (relu2): ReLU()\n )\n '\ - '(4): Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ - '(5): MaxPool3D(kernel_size=2, stride=2, padding=0)\n '\ - '(6): GELU(approximate=True)\n)' + 'Sequential(\n ' + '(0): Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n ' + '(1): AvgPool3D(kernel_size=2, stride=2, padding=0)\n ' + '(2): Tanh(name=Tanh)\n ' + '(3): Sequential(\n (conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n (relu1): ReLU()\n' + ' (conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n (relu2): ReLU()\n )\n ' + '(4): Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n ' + '(5): MaxPool3D(kernel_size=2, stride=2, padding=0)\n ' + '(6): GELU(approximate=True)\n)', ) def test_layer_str(self): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py index e61671e39f1..27a27c3ce70 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py @@ -15,14 +15,23 @@ import unittest import paddle.fluid as fluid import paddle.fluid.framework as framework -from paddle.fluid.dygraph.nn import BatchNorm, Conv2D, Conv3D, Embedding, GroupNorm, LayerNorm, Linear, NCE, PRelu +from paddle.fluid.dygraph.nn import ( + BatchNorm, + Conv2D, + Conv3D, + Embedding, + GroupNorm, + LayerNorm, + Linear, + NCE, + PRelu, +) import numpy as np import os import tempfile class TestDygraphLoadStatic(unittest.TestCase): - def testLoadStaticModel(self): # static mode temp_dir = tempfile.TemporaryDirectory() @@ -32,30 +41,26 @@ class TestDygraphLoadStatic(unittest.TestCase): fc_out1 = fluid.layers.fc(a, 10) fc_out2 = fluid.layers.fc(a, 20) - conv_out_1 = fluid.layers.conv2d(conv_in, - num_filters=10, - filter_size=5, - act="relu") - conv_out_2 = fluid.layers.conv2d(conv_in, - num_filters=10, - filter_size=5, - act="relu") - - conv3d_in = fluid.data(name='conv3d_in', - shape=[None, 3, 12, 32, 32], - dtype='float32') - conv3d_out_1 = fluid.layers.conv3d(input=conv3d_in, - num_filters=2, - filter_size=3, - act="relu") - conv3d_out_2 = fluid.layers.conv3d(input=conv3d_in, - num_filters=2, - filter_size=3, - act="relu") - - batchnorm_in = fluid.data(name="batchnorm_in", - shape=[None, 10], - dtype='float32') + conv_out_1 = fluid.layers.conv2d( + conv_in, num_filters=10, filter_size=5, act="relu" + ) + conv_out_2 = fluid.layers.conv2d( + conv_in, num_filters=10, filter_size=5, act="relu" + ) + + conv3d_in = fluid.data( + name='conv3d_in', shape=[None, 3, 12, 32, 32], dtype='float32' + ) + conv3d_out_1 = fluid.layers.conv3d( + input=conv3d_in, num_filters=2, filter_size=3, act="relu" + ) + conv3d_out_2 = fluid.layers.conv3d( + input=conv3d_in, num_filters=2, filter_size=3, act="relu" + ) + + batchnorm_in = fluid.data( + name="batchnorm_in", shape=[None, 10], dtype='float32' + ) batchnorm_out_1 = fluid.layers.batch_norm(batchnorm_in) batchnorm_out_2 = fluid.layers.batch_norm(batchnorm_in) @@ -68,53 +73,56 @@ class TestDygraphLoadStatic(unittest.TestCase): layernorm_2 = fluid.layers.layer_norm(layernorm) nce_in = fluid.data(name="nce_in", shape=[None, 100], dtype='float32') - nce_label = fluid.data(name="nce_label", - shape=[None, 10], - dtype='int64') + nce_label = fluid.data( + name="nce_label", shape=[None, 10], dtype='int64' + ) nce_out_1 = fluid.layers.nce(nce_in, nce_label, 10000) nce_out_2 = fluid.layers.nce(nce_in, nce_label, 10000) - prelu_in = fluid.data(name="prelu_in", - shape=[None, 5, 10, 10], - dtype='float32') + prelu_in = fluid.data( + name="prelu_in", shape=[None, 5, 10, 10], dtype='float32' + ) prelu_out_1 = fluid.layers.prelu(prelu_in, "channel") prelu_out_2 = fluid.layers.prelu(prelu_in, "channel") - bilinear_tensor_pro_x = fluid.data("t1", - shape=[None, 5], - dtype="float32") - bilinear_tensor_pro_y = fluid.data("t2", - shape=[None, 4], - dtype="float32") + bilinear_tensor_pro_x = fluid.data( + "t1", shape=[None, 5], dtype="float32" + ) + bilinear_tensor_pro_y = fluid.data( + "t2", shape=[None, 4], dtype="float32" + ) bilinear_tensor_pro_out_1 = fluid.layers.bilinear_tensor_product( - x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000) + x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000 + ) bilinear_tensor_pro_out_2 = fluid.layers.bilinear_tensor_product( - x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000) - - conv2d_trans_in = fluid.data(name="conv2d_trans_in", - shape=[None, 10, 10, 10]) - - conv2d_trans_out_1 = fluid.layers.conv2d_transpose(conv2d_trans_in, - num_filters=10, - filter_size=5, - act="relu") - conv2d_trans_out_2 = fluid.layers.conv2d_transpose(conv2d_trans_in, - num_filters=10, - filter_size=5, - act="relu") - - conv3d_trans_in = fluid.data(name='conv3d_trans_in', - shape=[None, 3, 12, 32, 32], - dtype='float32') + x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000 + ) + + conv2d_trans_in = fluid.data( + name="conv2d_trans_in", shape=[None, 10, 10, 10] + ) + + conv2d_trans_out_1 = fluid.layers.conv2d_transpose( + conv2d_trans_in, num_filters=10, filter_size=5, act="relu" + ) + conv2d_trans_out_2 = fluid.layers.conv2d_transpose( + conv2d_trans_in, num_filters=10, filter_size=5, act="relu" + ) + + conv3d_trans_in = fluid.data( + name='conv3d_trans_in', shape=[None, 3, 12, 32, 32], dtype='float32' + ) conv3d_trans_out_1 = fluid.layers.conv3d_transpose( - input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu") + input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu" + ) conv3d_trans_out_2 = fluid.layers.conv3d_transpose( - input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu") + input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu" + ) - groupnorm_in = fluid.data(name='groupnorm_in', - shape=[None, 8, 32, 32], - dtype='float32') + groupnorm_in = fluid.data( + name='groupnorm_in', shape=[None, 8, 32, 32], dtype='float32' + ) groupnorm_out1 = fluid.layers.group_norm(input=groupnorm_in, groups=4) groupnorm_out2 = fluid.layers.group_norm(input=groupnorm_in, groups=4) ''' @@ -123,39 +131,47 @@ class TestDygraphLoadStatic(unittest.TestCase): spe_norm_out_2 = fluid.layers.spectral_norm(weight=spec_norm, dim=1, power_iters=2) ''' - nodes_vector = fluid.data(name='vectors', - shape=[None, 10, 5], - dtype='float32') - edge_set = fluid.data(name='edge_set', - shape=[None, 10, 2], - dtype='float32') - tree_conv_out1 = fluid.contrib.layers.tree_conv(nodes_vector, edge_set, - 6, 1, 2) - tree_conv_out2 = fluid.contrib.layers.tree_conv(nodes_vector, edge_set, - 6, 1, 2) - - para1 = fluid.layers.create_parameter([100, 100], - 'float32', - name="weight_test_1") - para2 = fluid.layers.create_parameter([20, 200], - 'float32', - name="weight_test_2") + nodes_vector = fluid.data( + name='vectors', shape=[None, 10, 5], dtype='float32' + ) + edge_set = fluid.data( + name='edge_set', shape=[None, 10, 2], dtype='float32' + ) + tree_conv_out1 = fluid.contrib.layers.tree_conv( + nodes_vector, edge_set, 6, 1, 2 + ) + tree_conv_out2 = fluid.contrib.layers.tree_conv( + nodes_vector, edge_set, 6, 1, 2 + ) + + para1 = fluid.layers.create_parameter( + [100, 100], 'float32', name="weight_test_1" + ) + para2 = fluid.layers.create_parameter( + [20, 200], 'float32', name="weight_test_2" + ) para_list = fluid.default_main_program().list_vars() - exe = fluid.Executor(fluid.CPUPlace( - ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not fluid.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) out = exe.run(framework.default_startup_program()) - fluid.save(framework.default_main_program(), - os.path.join(temp_dir.name, "test_1")) + fluid.save( + framework.default_main_program(), + os.path.join(temp_dir.name, "test_1"), + ) para_dict = fluid.load_program_state( - os.path.join(temp_dir.name, "test_1")) + os.path.join(temp_dir.name, "test_1") + ) new_dict = {} for k, v in para_dict.items(): - #print( k, v.shape ) + # print( k, v.shape ) if k.startswith("fc"): name = k.replace("fc", "linear", 1) new_dict[name] = v @@ -165,30 +181,31 @@ class TestDygraphLoadStatic(unittest.TestCase): with fluid.dygraph.guard(): class MyTest(fluid.dygraph.Layer): - def __init__(self): super(MyTest, self).__init__() self.linear1 = Linear(10, 10) self.lienar2 = Linear(10, 20) - self.conv2d_1 = Conv2D(num_channels=10, - num_filters=10, - filter_size=5, - act="relu") - self.conv2d_2 = Conv2D(num_channels=10, - num_filters=10, - filter_size=5, - act="relu") - - self.conv3d_1 = Conv3D(num_channels=3, - num_filters=2, - filter_size=3, - act="relu") - self.conv3d_2 = Conv3D(num_channels=3, - num_filters=2, - filter_size=3, - act="relu") + self.conv2d_1 = Conv2D( + num_channels=10, + num_filters=10, + filter_size=5, + act="relu", + ) + self.conv2d_2 = Conv2D( + num_channels=10, + num_filters=10, + filter_size=5, + act="relu", + ) + + self.conv3d_1 = Conv3D( + num_channels=3, num_filters=2, filter_size=3, act="relu" + ) + self.conv3d_2 = Conv3D( + num_channels=3, num_filters=2, filter_size=3, act="relu" + ) self.batch_norm_1 = BatchNorm(10) self.batch_norm_2 = BatchNorm(10) @@ -208,12 +225,12 @@ class TestDygraphLoadStatic(unittest.TestCase): self.group_norm1 = GroupNorm(8, 4) self.gourp_norm2 = GroupNorm(8, 4) - self.w_1 = self.create_parameter([100, 100], - dtype='float32', - attr="weight_test_1") - self.w_2 = self.create_parameter([20, 200], - dtype='float32', - attr="weight_test_2") + self.w_1 = self.create_parameter( + [100, 100], dtype='float32', attr="weight_test_1" + ) + self.w_2 = self.create_parameter( + [20, 200], dtype='float32', attr="weight_test_2" + ) my_test = MyTest() my_test.set_dict(new_dict, use_structured_name=False) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py b/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py index a359d43d9c6..3e74ba69d52 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py @@ -26,14 +26,15 @@ from paddle.fluid.framework import _test_eager_guard class SimpleNet(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype='float32'): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype='float32', + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -46,24 +47,31 @@ class SimpleNet(fluid.Layer): param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) projection = fluid.layers.matmul( - x_emb, fluid.layers.transpose(self.embedding.weight, perm=[1, 0])) + x_emb, fluid.layers.transpose(self.embedding.weight, perm=[1, 0]) + ) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -72,7 +80,6 @@ class SimpleNet(fluid.Layer): class TestDygraphSimpleNet(unittest.TestCase): - def func_simple_net(self): for is_sparse in [True, False]: dtype_list = ["float32"] @@ -105,22 +112,27 @@ class TestDygraphSimpleNet(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - simple_net = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse, - dtype=dtype) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=simple_net.parameters()) + simple_net = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + dtype=dtype, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, + parameter_list=simple_net.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None helper = DyGraphProgramDescTracerTestHelper(self) fluid.set_flags( - {'FLAGS_sort_sum_gradient': is_sort_sum_gradient}) + {'FLAGS_sort_sum_gradient': is_sort_sum_gradient} + ) for i in range(batch_num): x_data = np.arange(12).reshape(4, 3).astype('int64') @@ -147,17 +159,19 @@ class TestDygraphSimpleNet(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - simple_net = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - is_sparse=is_sparse, - dtype=dtype) + simple_net = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + is_sparse=is_sparse, + dtype=dtype, + ) exe = fluid.Executor(place) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype=dtype) static_loss = simple_net(x, y) @@ -168,8 +182,10 @@ class TestDygraphSimpleNet(unittest.TestCase): for param in simple_net.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_loss_value = None @@ -180,22 +196,22 @@ class TestDygraphSimpleNet(unittest.TestCase): y_data = y_data.reshape((-1, 1)) fetch_list = [static_loss] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_data, "y": y_data}, + fetch_list=fetch_list, + ) static_loss_value = out[0] if i == batch_num - 1: for k in range(3, len(out)): - static_param_updated[static_param_name_list[ - k - 1]] = out[k] + static_param_updated[ + static_param_name_list[k - 1] + ] = out[k] - np.testing.assert_allclose(static_loss_value, - dy_loss_value, - rtol=0.001) + np.testing.assert_allclose( + static_loss_value, dy_loss_value, rtol=0.001 + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, dy_param_init[key]) for key, value in static_param_updated.items(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_mnist.py b/python/paddle/fluid/tests/unittests/test_imperative_mnist.py index 22954d11e28..41b0b365e7a 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_mnist.py @@ -26,43 +26,48 @@ from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=False, - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=False, + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=None, - bias_attr=None, - use_cudnn=use_cudnn) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=None, + bias_attr=None, + use_cudnn=use_cudnn, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -71,33 +76,30 @@ class SimpleImgConvPool(fluid.dygraph.Layer): class MNIST(fluid.dygraph.Layer): - def __init__(self): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(1, - 20, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_1 = SimpleImgConvPool( + 1, 20, 5, 2, 2, act="relu" + ) - self._simple_img_conv_pool_2 = SimpleImgConvPool(20, - 50, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_2 = SimpleImgConvPool( + 20, 50, 5, 2, 2, act="relu" + ) self.pool_2_shape = 50 * 4 * 4 SIZE = 10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 - self._fc = Linear(self.pool_2_shape, - 10, - param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=scale)), - act="softmax") + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 + self._fc = Linear( + self.pool_2_shape, + 10, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + act="softmax", + ) def forward(self, inputs): x = self._simple_img_conv_pool_1(inputs) @@ -108,9 +110,7 @@ class MNIST(fluid.dygraph.Layer): class TestImperativeMnist(unittest.TestCase): - def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): image = np.array(item[0]).reshape(1, 28, 28) @@ -132,16 +132,19 @@ class TestImperativeMnist(unittest.TestCase): fluid.default_main_program().random_seed = seed mnist = MNIST() - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=mnist.parameters()) + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=mnist.parameters() + ) batch_py_reader = fluid.io.PyReader(capacity=1) batch_py_reader.decorate_sample_list_generator( - paddle.batch(self.reader_decorator( - paddle.dataset.mnist.train()), - batch_size=batch_size, - drop_last=True), - places=fluid.CPUPlace()) + paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=batch_size, + drop_last=True, + ), + places=fluid.CPUPlace(), + ) mnist.train() dy_param_init_value = {} @@ -159,12 +162,14 @@ class TestImperativeMnist(unittest.TestCase): if batch_id % 10 == 0 and _in_legacy_dygraph(): cost, traced_layer = paddle.jit.TracedLayer.trace( - mnist, inputs=img) + mnist, inputs=img + ) if program is not None: self.assertTrue(program, traced_layer.program) program = traced_layer.program traced_layer.save_inference_model( - './infer_imperative_mnist') + './infer_imperative_mnist' + ) else: cost = mnist(img) @@ -193,18 +198,23 @@ class TestImperativeMnist(unittest.TestCase): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) mnist = MNIST() sgd = SGDOptimizer(learning_rate=1e-3) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size, - drop_last=True) - - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + train_reader = paddle.batch( + paddle.dataset.mnist.train(), + batch_size=batch_size, + drop_last=True, + ) + + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) @@ -217,8 +227,10 @@ class TestImperativeMnist(unittest.TestCase): for param in mnist.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -227,12 +239,14 @@ class TestImperativeMnist(unittest.TestCase): for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: break - static_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape( - [batch_size, 1]) + static_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([batch_size, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) @@ -240,35 +254,34 @@ class TestImperativeMnist(unittest.TestCase): if traced_layer is not None: traced_layer([static_x_data]) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_out = out[0] for i in range(1, len(out)): - static_param_value[static_param_name_list[i - - 1]] = out[i] + static_param_value[static_param_name_list[i - 1]] = out[ + i + ] - np.testing.assert_allclose(dy_x_data.all(), - static_x_data.all(), - rtol=1e-05) + np.testing.assert_allclose( + dy_x_data.all(), static_x_data.all(), rtol=1e-05 + ) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-05 + ) def test_mnist_float32(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_mnist_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_mnist_sorted_gradient.py index 214214f8710..1a077311e05 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_mnist_sorted_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_mnist_sorted_gradient.py @@ -26,7 +26,6 @@ from paddle.fluid.framework import _test_eager_guard class TestImperativeMnistSortGradient(unittest.TestCase): - def func_test_mnist_sort_gradient_float32(self): seed = 90 epoch_num = 1 @@ -37,21 +36,25 @@ class TestImperativeMnistSortGradient(unittest.TestCase): fluid.set_flags({'FLAGS_sort_sum_gradient': True}) mnist2 = MNIST() - sgd2 = SGDOptimizer(learning_rate=1e-3, - parameter_list=mnist2.parameters()) - train_reader2 = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + sgd2 = SGDOptimizer( + learning_rate=1e-3, parameter_list=mnist2.parameters() + ) + train_reader2 = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) mnist2.train() dy_param_init_value2 = {} for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader2()): - dy_x_data2 = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data2 = np.array([x[1] for x in data - ]).astype('int64').reshape(128, 1) + dy_x_data2 = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data2 = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(128, 1) + ) img2 = to_variable(dy_x_data2) label2 = to_variable(y_data2) @@ -81,18 +84,21 @@ class TestImperativeMnistSortGradient(unittest.TestCase): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) mnist = MNIST() sgd = SGDOptimizer(learning_rate=1e-3) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) @@ -105,53 +111,57 @@ class TestImperativeMnistSortGradient(unittest.TestCase): for param in mnist.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader()): - static_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([128, 1]) + static_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([128, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_out = out[0] for i in range(1, len(out)): - static_param_value[static_param_name_list[i - - 1]] = out[i] + static_param_value[static_param_name_list[i - 1]] = out[ + i + ] if batch_id == 20: break - np.testing.assert_allclose(dy_x_data2.all(), - static_x_data.all(), - rtol=1e-05) + np.testing.assert_allclose( + dy_x_data2.all(), static_x_data.all(), rtol=1e-05 + ) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value2[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value2[key], rtol=1e-05 + ) np.testing.assert_allclose(static_out, dy_out2, rtol=1e-05) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - dy_param_value2[key], - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + value, dy_param_value2[key], rtol=1e-05, atol=1e-05 + ) def test_mnist_sort_gradient_float32(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_named_members.py b/python/paddle/fluid/tests/unittests/test_imperative_named_members.py index 57fad812bf8..7043eb0c2e7 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_named_members.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_named_members.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class MyLayer(fluid.Layer): - def __init__(self, num_channel, dim, num_filter=5): super(MyLayer, self).__init__() self.fc = fluid.dygraph.Linear(dim, dim) @@ -33,7 +32,6 @@ class MyLayer(fluid.Layer): class TestImperativeNamedSubLayers(unittest.TestCase): - def func_test_named_sublayers(self): with fluid.dygraph.guard(): fc1 = fluid.Linear(10, 3) @@ -45,21 +43,22 @@ class TestImperativeNamedSubLayers(unittest.TestCase): expected_sublayers = [fc1, fc2, custom, custom.fc, custom.conv] self.assertEqual(len(list_named_sublayers), len(expected_sublayers)) - for (name, - sublayer), expected_sublayer in zip(list_named_sublayers, - expected_sublayers): + for (name, sublayer), expected_sublayer in zip( + list_named_sublayers, expected_sublayers + ): self.assertEqual(sublayer, expected_sublayer) list_sublayers = list(model.sublayers()) self.assertEqual(len(list_named_sublayers), len(list_sublayers)) - for (name, - sublayer), expected_sublayer in zip(list_named_sublayers, - list_sublayers): + for (name, sublayer), expected_sublayer in zip( + list_named_sublayers, list_sublayers + ): self.assertEqual(sublayer, expected_sublayer) self.assertListEqual( [l for _, l in list(model.named_sublayers(include_self=True))], - [model] + expected_sublayers) + [model] + expected_sublayers, + ) def test_named_sublayers(self): with _test_eager_guard(): @@ -68,7 +67,6 @@ class TestImperativeNamedSubLayers(unittest.TestCase): class TestImperativeNamedParameters(unittest.TestCase): - def func_test_named_parameters(self): with fluid.dygraph.guard(): fc1 = fluid.Linear(10, 3) @@ -80,7 +78,8 @@ class TestImperativeNamedParameters(unittest.TestCase): expected_named_parameters = list() for prefix, layer in model.named_sublayers(): for name, param in layer.named_parameters( - include_sublayers=False): + include_sublayers=False + ): full_name = prefix + ('.' if prefix else '') + name expected_named_parameters.append((full_name, param)) @@ -95,7 +94,6 @@ class TestImperativeNamedParameters(unittest.TestCase): with fluid.dygraph.guard(): class Mymodel(fluid.dygraph.Layer): - def __init__(self): super(Mymodel, self).__init__() self.linear1 = fluid.dygraph.Linear(10, 10) @@ -103,28 +101,42 @@ class TestImperativeNamedParameters(unittest.TestCase): self.conv2d = fluid.dygraph.Conv2D(3, 2, 3) self.embedding = fluid.dygraph.Embedding(size=[128, 16]) self.h_0 = fluid.dygraph.to_variable( - np.zeros([10, 10]).astype('float32')) - self.weight = self.create_parameter(shape=[2, 3], - attr=fluid.ParamAttr(), - dtype="float32", - is_bias=False) + np.zeros([10, 10]).astype('float32') + ) + self.weight = self.create_parameter( + shape=[2, 3], + attr=fluid.ParamAttr(), + dtype="float32", + is_bias=False, + ) model = Mymodel() expected_members = dir(model) - self.assertTrue("linear1" in expected_members, - "model should contain Layer: linear1") - self.assertTrue("linear2" in expected_members, - "model should contain Layer: linear2") - self.assertTrue("conv2d" in expected_members, - "model should contain Layer: conv2d") - self.assertTrue("embedding" in expected_members, - "model should contain Layer: embedding") - self.assertTrue("h_0" in expected_members, - "model should contain buffer: h_0") - self.assertTrue("weight" in expected_members, - "model should contain parameter: weight") + self.assertTrue( + "linear1" in expected_members, + "model should contain Layer: linear1", + ) + self.assertTrue( + "linear2" in expected_members, + "model should contain Layer: linear2", + ) + self.assertTrue( + "conv2d" in expected_members, + "model should contain Layer: conv2d", + ) + self.assertTrue( + "embedding" in expected_members, + "model should contain Layer: embedding", + ) + self.assertTrue( + "h_0" in expected_members, "model should contain buffer: h_0" + ) + self.assertTrue( + "weight" in expected_members, + "model should contain parameter: weight", + ) def test_dir_layer(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py b/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py index a93471a09c9..4e333f7ca4e 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py @@ -21,15 +21,16 @@ from paddle.fluid.framework import _in_legacy_dygraph, _test_eager_guard class TestImperativeNumpyBridge(unittest.TestCase): - def func_tensor_from_numpy(self): data_np = np.array([[2, 3, 1]]).astype('float32') with fluid.dygraph.guard(fluid.CPUPlace()): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") var = fluid.dygraph.to_variable(data_np, zero_copy=True) - assert "Currently, zero_copy is not supported, and it will be discarded." in str( - w[-1].message) + assert ( + "Currently, zero_copy is not supported, and it will be discarded." + in str(w[-1].message) + ) # Temporally diable zero_copy # var = fluid.dygraph.to_variable(data_np, zero_copy=True) # np.testing.assert_array_equal(var.numpy(), data_np) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_ocr_attention_model.py b/python/paddle/fluid/tests/unittests/test_imperative_ocr_attention_model.py index eb582310df4..7971bdd34cd 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_ocr_attention_model.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_ocr_attention_model.py @@ -17,7 +17,14 @@ import numpy as np import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear, BatchNorm, Embedding, GRUUnit +from paddle.fluid.dygraph.nn import ( + Conv2D, + Pool2D, + Linear, + BatchNorm, + Embedding, + GRUUnit, +) from paddle.fluid.dygraph.base import to_variable from test_imperative_base import new_program_scope from paddle.fluid.framework import _test_eager_guard @@ -27,6 +34,7 @@ class Config(object): ''' config for training ''' + # encoder rnn hidden_size encoder_size = 8 # decoder size for decoder stage @@ -58,53 +66,62 @@ class Config(object): class ConvBNPool(fluid.dygraph.Layer): - - def __init__(self, - group, - out_ch, - channels, - act="relu", - is_test=False, - pool=True, - use_cudnn=True): + def __init__( + self, + group, + out_ch, + channels, + act="relu", + is_test=False, + pool=True, + use_cudnn=True, + ): super(ConvBNPool, self).__init__() self.group = group self.pool = pool filter_size = 3 - conv_std_0 = (2.0 / (filter_size**2 * channels[0]))**0.5 + conv_std_0 = (2.0 / (filter_size**2 * channels[0])) ** 0.5 conv_param_0 = fluid.ParamAttr( - initializer=fluid.initializer.Normal(0.0, conv_std_0)) + initializer=fluid.initializer.Normal(0.0, conv_std_0) + ) - conv_std_1 = (2.0 / (filter_size**2 * channels[1]))**0.5 + conv_std_1 = (2.0 / (filter_size**2 * channels[1])) ** 0.5 conv_param_1 = fluid.ParamAttr( - initializer=fluid.initializer.Normal(0.0, conv_std_1)) - - self.conv_0_layer = Conv2D(channels[0], - out_ch[0], - 3, - padding=1, - param_attr=conv_param_0, - bias_attr=False, - act=None, - use_cudnn=use_cudnn) + initializer=fluid.initializer.Normal(0.0, conv_std_1) + ) + + self.conv_0_layer = Conv2D( + channels[0], + out_ch[0], + 3, + padding=1, + param_attr=conv_param_0, + bias_attr=False, + act=None, + use_cudnn=use_cudnn, + ) self.bn_0_layer = BatchNorm(out_ch[0], act=act, is_test=is_test) - self.conv_1_layer = Conv2D(out_ch[0], - num_filters=out_ch[1], - filter_size=3, - padding=1, - param_attr=conv_param_1, - bias_attr=False, - act=None, - use_cudnn=use_cudnn) + self.conv_1_layer = Conv2D( + out_ch[0], + num_filters=out_ch[1], + filter_size=3, + padding=1, + param_attr=conv_param_1, + bias_attr=False, + act=None, + use_cudnn=use_cudnn, + ) self.bn_1_layer = BatchNorm(out_ch[1], act=act, is_test=is_test) if self.pool: - self.pool_layer = Pool2D(pool_size=2, - pool_type='max', - pool_stride=2, - use_cudnn=use_cudnn, - ceil_mode=True) + self.pool_layer = Pool2D( + pool_size=2, + pool_type='max', + pool_stride=2, + use_cudnn=use_cudnn, + ceil_mode=True, + ) def forward(self, inputs): conv_0 = self.conv_0_layer(inputs) @@ -118,22 +135,25 @@ class ConvBNPool(fluid.dygraph.Layer): class OCRConv(fluid.dygraph.Layer): - def __init__(self, is_test=False, use_cudnn=True): super(OCRConv, self).__init__() - self.conv_bn_pool_1 = ConvBNPool(2, [8, 8], [1, 8], - is_test=is_test, - use_cudnn=use_cudnn) - self.conv_bn_pool_2 = ConvBNPool(2, [8, 8], [8, 8], - is_test=is_test, - use_cudnn=use_cudnn) - self.conv_bn_pool_3 = ConvBNPool(2, [8, 8], [8, 8], - is_test=is_test, - use_cudnn=use_cudnn) - self.conv_bn_pool_4 = ConvBNPool(2, [16, 16], [8, 16], - is_test=is_test, - pool=False, - use_cudnn=use_cudnn) + self.conv_bn_pool_1 = ConvBNPool( + 2, [8, 8], [1, 8], is_test=is_test, use_cudnn=use_cudnn + ) + self.conv_bn_pool_2 = ConvBNPool( + 2, [8, 8], [8, 8], is_test=is_test, use_cudnn=use_cudnn + ) + self.conv_bn_pool_3 = ConvBNPool( + 2, [8, 8], [8, 8], is_test=is_test, use_cudnn=use_cudnn + ) + self.conv_bn_pool_4 = ConvBNPool( + 2, + [16, 16], + [8, 16], + is_test=is_test, + pool=False, + use_cudnn=use_cudnn, + ) def forward(self, inputs): inputs_1 = self.conv_bn_pool_1(inputs) @@ -145,24 +165,27 @@ class OCRConv(fluid.dygraph.Layer): class DynamicGRU(fluid.dygraph.Layer): - - def __init__(self, - size, - param_attr=None, - bias_attr=None, - is_reverse=False, - gate_activation='sigmoid', - candidate_activation='tanh', - h_0=None, - origin_mode=False): + def __init__( + self, + size, + param_attr=None, + bias_attr=None, + is_reverse=False, + gate_activation='sigmoid', + candidate_activation='tanh', + h_0=None, + origin_mode=False, + ): super(DynamicGRU, self).__init__() - self.gru_unit = GRUUnit(size * 3, - param_attr=param_attr, - bias_attr=bias_attr, - activation=candidate_activation, - gate_activation=gate_activation, - origin_mode=origin_mode) + self.gru_unit = GRUUnit( + size * 3, + param_attr=param_attr, + bias_attr=bias_attr, + activation=candidate_activation, + gate_activation=gate_activation, + origin_mode=origin_mode, + ) self.h_0 = h_0 self.is_reverse = is_reverse @@ -174,15 +197,16 @@ class DynamicGRU(fluid.dygraph.Layer): for i in range(inputs.shape[1]): if self.is_reverse: i = inputs.shape[1] - 1 - i - input_ = fluid.layers.slice(inputs, - axes=[1], - starts=[i], - ends=[i + 1]) - input_ = fluid.layers.reshape(input_, [-1, input_.shape[2]], - inplace=False) + input_ = fluid.layers.slice( + inputs, axes=[1], starts=[i], ends=[i + 1] + ) + input_ = fluid.layers.reshape( + input_, [-1, input_.shape[2]], inplace=False + ) hidden, reset, gate = self.gru_unit(input_, hidden) - hidden_ = fluid.layers.reshape(hidden, [-1, 1, hidden.shape[1]], - inplace=False) + hidden_ = fluid.layers.reshape( + hidden, [-1, 1, hidden.shape[1]], inplace=False + ) if self.is_reverse: res = [hidden_] + res else: @@ -192,75 +216,85 @@ class DynamicGRU(fluid.dygraph.Layer): class EncoderNet(fluid.dygraph.Layer): - - def __init__(self, - rnn_hidden_size=Config.encoder_size, - is_test=False, - use_cudnn=True): + def __init__( + self, rnn_hidden_size=Config.encoder_size, is_test=False, use_cudnn=True + ): super(EncoderNet, self).__init__() self.rnn_hidden_size = rnn_hidden_size para_attr = fluid.ParamAttr( - initializer=fluid.initializer.Normal(0.0, 0.02)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Normal( - 0.0, 0.02), - learning_rate=2.0) + initializer=fluid.initializer.Normal(0.0, 0.02) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02), learning_rate=2.0 + ) if fluid.framework._non_static_mode(): - h_0 = np.zeros((Config.batch_size, rnn_hidden_size), - dtype="float32") + h_0 = np.zeros( + (Config.batch_size, rnn_hidden_size), dtype="float32" + ) h_0 = to_variable(h_0) else: h_0 = fluid.layers.fill_constant( shape=[Config.batch_size, rnn_hidden_size], dtype='float32', - value=0) + value=0, + ) self.ocr_convs = OCRConv(is_test=is_test, use_cudnn=use_cudnn) - self.fc_1_layer = Linear(32, - rnn_hidden_size * 3, - param_attr=para_attr, - bias_attr=False) - self.fc_2_layer = Linear(32, - rnn_hidden_size * 3, - param_attr=para_attr, - bias_attr=False) - self.gru_forward_layer = DynamicGRU(size=rnn_hidden_size, - h_0=h_0, - param_attr=para_attr, - bias_attr=bias_attr, - candidate_activation='relu') - self.gru_backward_layer = DynamicGRU(size=rnn_hidden_size, - h_0=h_0, - param_attr=para_attr, - bias_attr=bias_attr, - candidate_activation='relu', - is_reverse=True) - - self.encoded_proj_fc = Linear(rnn_hidden_size * 2, - Config.decoder_size, - bias_attr=False) + self.fc_1_layer = Linear( + 32, rnn_hidden_size * 3, param_attr=para_attr, bias_attr=False + ) + self.fc_2_layer = Linear( + 32, rnn_hidden_size * 3, param_attr=para_attr, bias_attr=False + ) + self.gru_forward_layer = DynamicGRU( + size=rnn_hidden_size, + h_0=h_0, + param_attr=para_attr, + bias_attr=bias_attr, + candidate_activation='relu', + ) + self.gru_backward_layer = DynamicGRU( + size=rnn_hidden_size, + h_0=h_0, + param_attr=para_attr, + bias_attr=bias_attr, + candidate_activation='relu', + is_reverse=True, + ) + + self.encoded_proj_fc = Linear( + rnn_hidden_size * 2, Config.decoder_size, bias_attr=False + ) def forward(self, inputs): conv_features = self.ocr_convs(inputs) - #sliced_feature = fluid.layers.im2sequence( + # sliced_feature = fluid.layers.im2sequence( # input=conv_features, # stride=[1, 1], # filter_size=[conv_features.shape[2], 1]) - transpose_conv_features = fluid.layers.transpose(conv_features, - perm=[0, 3, 1, 2]) - sliced_feature = fluid.layers.reshape(transpose_conv_features, [ - -1, 8, - transpose_conv_features.shape[2] * transpose_conv_features.shape[3] - ], - inplace=False) + transpose_conv_features = fluid.layers.transpose( + conv_features, perm=[0, 3, 1, 2] + ) + sliced_feature = fluid.layers.reshape( + transpose_conv_features, + [ + -1, + 8, + transpose_conv_features.shape[2] + * transpose_conv_features.shape[3], + ], + inplace=False, + ) fc_1 = self.fc_1_layer(sliced_feature) fc_2 = self.fc_2_layer(sliced_feature) gru_forward = self.gru_forward_layer(fc_1) gru_backward = self.gru_backward_layer(fc_2) - encoded_vector = fluid.layers.concat(input=[gru_forward, gru_backward], - axis=2) + encoded_vector = fluid.layers.concat( + input=[gru_forward, gru_backward], axis=2 + ) encoded_proj = self.encoded_proj_fc(encoded_vector) @@ -268,79 +302,80 @@ class EncoderNet(fluid.dygraph.Layer): class SimpleAttention(fluid.dygraph.Layer): - def __init__(self, decoder_size): super(SimpleAttention, self).__init__() - self.fc_1 = Linear(decoder_size, - decoder_size, - act=None, - bias_attr=False) + self.fc_1 = Linear( + decoder_size, decoder_size, act=None, bias_attr=False + ) self.fc_2 = Linear(decoder_size, 1, act=None, bias_attr=False) def forward(self, encoder_vec, encoder_proj, decoder_state): decoder_state_fc = self.fc_1(decoder_state) decoder_state_proj_reshape = fluid.layers.reshape( - decoder_state_fc, [-1, 1, decoder_state_fc.shape[1]], inplace=False) + decoder_state_fc, [-1, 1, decoder_state_fc.shape[1]], inplace=False + ) decoder_state_expand = fluid.layers.expand( - decoder_state_proj_reshape, [1, encoder_proj.shape[1], 1]) - concated = fluid.layers.elementwise_add(encoder_proj, - decoder_state_expand) + decoder_state_proj_reshape, [1, encoder_proj.shape[1], 1] + ) + concated = fluid.layers.elementwise_add( + encoder_proj, decoder_state_expand + ) concated = fluid.layers.tanh(x=concated) attention_weight = self.fc_2(concated) weights_reshape = fluid.layers.reshape( x=attention_weight, shape=[attention_weight.shape[0], attention_weight.shape[1]], - inplace=False) + inplace=False, + ) weights_reshape = fluid.layers.softmax(weights_reshape) - scaled = fluid.layers.elementwise_mul(x=encoder_vec, - y=weights_reshape, - axis=0) + scaled = fluid.layers.elementwise_mul( + x=encoder_vec, y=weights_reshape, axis=0 + ) context = fluid.layers.reduce_sum(scaled, dim=1) return context class GRUDecoderWithAttention(fluid.dygraph.Layer): - def __init__(self, decoder_size, num_classes): super(GRUDecoderWithAttention, self).__init__() self.simple_attention = SimpleAttention(decoder_size) - self.fc_1_layer = Linear(Config.encoder_size * 2, - decoder_size * 3, - bias_attr=False) - self.fc_2_layer = Linear(decoder_size, - decoder_size * 3, - bias_attr=False) - self.gru_unit = GRUUnit(size=decoder_size * 3, - param_attr=None, - bias_attr=None) - self.out_layer = Linear(decoder_size, - num_classes + 2, - bias_attr=None, - act='softmax') + self.fc_1_layer = Linear( + Config.encoder_size * 2, decoder_size * 3, bias_attr=False + ) + self.fc_2_layer = Linear( + decoder_size, decoder_size * 3, bias_attr=False + ) + self.gru_unit = GRUUnit( + size=decoder_size * 3, param_attr=None, bias_attr=None + ) + self.out_layer = Linear( + decoder_size, num_classes + 2, bias_attr=None, act='softmax' + ) self.decoder_size = decoder_size - def forward(self, target_embedding, encoder_vec, encoder_proj, - decoder_boot): + def forward( + self, target_embedding, encoder_vec, encoder_proj, decoder_boot + ): res = [] hidden_mem = decoder_boot for i in range(target_embedding.shape[1]): - current_word = fluid.layers.slice(target_embedding, - axes=[1], - starts=[i], - ends=[i + 1]) - current_word = fluid.layers.reshape(current_word, - [-1, current_word.shape[2]], - inplace=False) - - context = self.simple_attention(encoder_vec, encoder_proj, - hidden_mem) + current_word = fluid.layers.slice( + target_embedding, axes=[1], starts=[i], ends=[i + 1] + ) + current_word = fluid.layers.reshape( + current_word, [-1, current_word.shape[2]], inplace=False + ) + + context = self.simple_attention( + encoder_vec, encoder_proj, hidden_mem + ) fc_1 = self.fc_1_layer(context) fc_2 = self.fc_2_layer(current_word) decoder_inputs = fluid.layers.elementwise_add(x=fc_1, y=fc_2) @@ -356,45 +391,48 @@ class GRUDecoderWithAttention(fluid.dygraph.Layer): class OCRAttention(fluid.dygraph.Layer): - def __init__(self): super(OCRAttention, self).__init__() self.encoder_net = EncoderNet() - self.fc = Linear(Config.encoder_size, - Config.decoder_size, - bias_attr=False, - act='relu') + self.fc = Linear( + Config.encoder_size, + Config.decoder_size, + bias_attr=False, + act='relu', + ) self.embedding = Embedding( - [Config.num_classes + 2, Config.word_vector_dim], dtype='float32') + [Config.num_classes + 2, Config.word_vector_dim], dtype='float32' + ) self.gru_decoder_with_attention = GRUDecoderWithAttention( - Config.decoder_size, Config.num_classes) + Config.decoder_size, Config.num_classes + ) def forward(self, inputs, label_in): gru_backward, encoded_vector, encoded_proj = self.encoder_net(inputs) - backward_first = fluid.layers.slice(gru_backward, - axes=[1], - starts=[0], - ends=[1]) - backward_first = fluid.layers.reshape(backward_first, - [-1, backward_first.shape[2]], - inplace=False) + backward_first = fluid.layers.slice( + gru_backward, axes=[1], starts=[0], ends=[1] + ) + backward_first = fluid.layers.reshape( + backward_first, [-1, backward_first.shape[2]], inplace=False + ) decoder_boot = self.fc(backward_first) label_in = fluid.layers.reshape(label_in, [-1], inplace=False) trg_embedding = self.embedding(label_in) trg_embedding = fluid.layers.reshape( - trg_embedding, [-1, Config.max_length, trg_embedding.shape[1]], - inplace=False) + trg_embedding, + [-1, Config.max_length, trg_embedding.shape[1]], + inplace=False, + ) - prediction = self.gru_decoder_with_attention(trg_embedding, - encoded_vector, - encoded_proj, decoder_boot) + prediction = self.gru_decoder_with_attention( + trg_embedding, encoded_vector, encoded_proj, decoder_boot + ) return prediction class TestDygraphOCRAttention(unittest.TestCase): - def test_ocr_test(self): seed = 90 epoch_num = 1 @@ -403,26 +441,41 @@ class TestDygraphOCRAttention(unittest.TestCase): else: batch_num = 2 np.random.seed = seed - image_np = np.random.randn(Config.batch_size, Config.DATA_SHAPE[0], - Config.DATA_SHAPE[1], - Config.DATA_SHAPE[2]).astype('float32') - label_in_np = np.arange(0, Config.max_length, - dtype='int64').reshape([1, Config.max_length]) + image_np = np.random.randn( + Config.batch_size, + Config.DATA_SHAPE[0], + Config.DATA_SHAPE[1], + Config.DATA_SHAPE[2], + ).astype('float32') + label_in_np = np.arange(0, Config.max_length, dtype='int64').reshape( + [1, Config.max_length] + ) for i in range(2, Config.batch_size + 1): label_in_np = np.vstack( - (label_in_np, - np.arange((i - 1) * Config.max_length, - i * Config.max_length, - dtype='int64').reshape([1, Config.max_length]))) + ( + label_in_np, + np.arange( + (i - 1) * Config.max_length, + i * Config.max_length, + dtype='int64', + ).reshape([1, Config.max_length]), + ) + ) - label_out_np = np.arange(0, Config.max_length, - dtype='int64').reshape([1, Config.max_length]) + label_out_np = np.arange(0, Config.max_length, dtype='int64').reshape( + [1, Config.max_length] + ) for i in range(2, Config.batch_size + 1): label_out_np = np.vstack( - (label_out_np, - np.arange((i - 1) * Config.max_length, - i * Config.max_length, - dtype='int64').reshape([1, Config.max_length]))) + ( + label_out_np, + np.arange( + (i - 1) * Config.max_length, + i * Config.max_length, + dtype='int64', + ).reshape([1, Config.max_length]), + ) + ) def run_dygraph(): fluid.set_flags({'FLAGS_sort_sum_gradient': True}) @@ -432,11 +485,13 @@ class TestDygraphOCRAttention(unittest.TestCase): if Config.learning_rate_decay == "piecewise_decay": learning_rate = fluid.layers.piecewise_decay( - [50000], [Config.LR, Config.LR * 0.01]) + [50000], [Config.LR, Config.LR * 0.01] + ) else: learning_rate = Config.LR optimizer = fluid.optimizer.SGD( - learning_rate=0.001, parameter_list=ocr_attention.parameters()) + learning_rate=0.001, parameter_list=ocr_attention.parameters() + ) dy_param_init_value = {} for param in ocr_attention.parameters(): dy_param_init_value[param.name] = param.numpy() @@ -447,12 +502,15 @@ class TestDygraphOCRAttention(unittest.TestCase): label_out.stop_gradient = True img = to_variable(image_np) dy_prediction = ocr_attention(img, label_in) - label_out = fluid.layers.reshape(label_out, [-1, 1], - inplace=False) + label_out = fluid.layers.reshape( + label_out, [-1, 1], inplace=False + ) dy_prediction = fluid.layers.reshape( - dy_prediction, [label_out.shape[0], -1], inplace=False) - loss = fluid.layers.cross_entropy(input=dy_prediction, - label=label_out) + dy_prediction, [label_out.shape[0], -1], inplace=False + ) + loss = fluid.layers.cross_entropy( + input=dy_prediction, label=label_out + ) avg_loss = fluid.layers.reduce_sum(loss) dy_out = avg_loss.numpy() @@ -466,9 +524,11 @@ class TestDygraphOCRAttention(unittest.TestCase): for param in ocr_attention.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + core.grad_var_suffix() + ] = np_array optimizer.minimize(avg_loss) ocr_attention.clear_gradients() @@ -483,45 +543,52 @@ class TestDygraphOCRAttention(unittest.TestCase): with fluid.dygraph.guard(): with _test_eager_guard(): - eager_out, eager_param_init_value, eager_param_value = run_dygraph( - ) + ( + eager_out, + eager_param_init_value, + eager_param_value, + ) = run_dygraph() with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) ocr_attention = OCRAttention() if Config.learning_rate_decay == "piecewise_decay": learning_rate = fluid.layers.piecewise_decay( - [50000], [Config.LR, Config.LR * 0.01]) + [50000], [Config.LR, Config.LR * 0.01] + ) else: learning_rate = Config.LR optimizer = fluid.optimizer.SGD(learning_rate=0.001) - images = fluid.layers.data(name='pixel', - shape=Config.DATA_SHAPE, - dtype='float32') - static_label_in = fluid.layers.data(name='label_in', - shape=[1], - dtype='int64', - lod_level=0) - static_label_out = fluid.layers.data(name='label_out', - shape=[1], - dtype='int64', - lod_level=0) + images = fluid.layers.data( + name='pixel', shape=Config.DATA_SHAPE, dtype='float32' + ) + static_label_in = fluid.layers.data( + name='label_in', shape=[1], dtype='int64', lod_level=0 + ) + static_label_out = fluid.layers.data( + name='label_out', shape=[1], dtype='int64', lod_level=0 + ) static_label_out.stop_gradient = True static_label_out.trainable = False static_prediction = ocr_attention(images, static_label_in) static_prediction = fluid.layers.reshape( - static_prediction, shape=[-1, Config.num_classes + 2]) + static_prediction, shape=[-1, Config.num_classes + 2] + ) - cost = fluid.layers.cross_entropy(input=static_prediction, - label=static_label_out) + cost = fluid.layers.cross_entropy( + input=static_prediction, label=static_label_out + ) static_avg_loss = fluid.layers.reduce_sum(cost) # param_grad_list = fluid.backward.append_backward(static_avg_loss) optimizer.minimize(static_avg_loss) @@ -532,11 +599,14 @@ class TestDygraphOCRAttention(unittest.TestCase): for param in ocr_attention.parameters(): static_param_name_list.append(param.name) if param.trainable: - static_grad_name_list.append(param.name + - core.grad_var_suffix()) + static_grad_name_list.append( + param.name + core.grad_var_suffix() + ) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -549,24 +619,30 @@ class TestDygraphOCRAttention(unittest.TestCase): static_label_in = label_in_np static_label_out = label_out_np static_label_out = static_label_out.reshape((-1, 1)) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": image_np, - "label_in": static_label_in, - "label_out": static_label_out - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "pixel": image_np, + "label_in": static_label_in, + "label_out": static_label_out, + }, + fetch_list=fetch_list, + ) static_param_value = {} static_grad_value = {} static_out = out[0] for i in range(1, len(static_param_name_list) + 1): - static_param_value[static_param_name_list[i - - 1]] = out[i] + static_param_value[static_param_name_list[i - 1]] = out[ + i + ] grad_start_pos = len(static_param_name_list) + 1 - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + for i in range( + grad_start_pos, + len(static_grad_name_list) + grad_start_pos, + ): + static_grad_value[ + static_grad_name_list[i - grad_start_pos] + ] = out[i] np.testing.assert_allclose(static_out, dy_out, rtol=1e-05, atol=1e-8) @@ -574,10 +650,9 @@ class TestDygraphOCRAttention(unittest.TestCase): np.testing.assert_array_equal(value, dy_param_init_value[key]) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-8 + ) # check eager here np.testing.assert_allclose(static_out, eager_out, rtol=1e-05, atol=1e-8) @@ -586,10 +661,9 @@ class TestDygraphOCRAttention(unittest.TestCase): np.testing.assert_array_equal(value, eager_param_init_value[key]) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - eager_param_value[key], - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + value, eager_param_value[key], rtol=1e-05, atol=1e-8 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py b/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py index ba1f73bcbcb..0219cc99479 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py @@ -19,8 +19,28 @@ import itertools import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.optimizer import SGDOptimizer, Adam, MomentumOptimizer, LarsMomentumOptimizer, AdagradOptimizer, AdamaxOptimizer, DpsgdOptimizer, DecayedAdagradOptimizer, AdadeltaOptimizer, RMSPropOptimizer, FtrlOptimizer, LambOptimizer -from paddle.fluid.optimizer import ModelAverage, DGCMomentumOptimizer, ExponentialMovingAverage, PipelineOptimizer, LookaheadOptimizer, RecomputeOptimizer +from paddle.fluid.optimizer import ( + SGDOptimizer, + Adam, + MomentumOptimizer, + LarsMomentumOptimizer, + AdagradOptimizer, + AdamaxOptimizer, + DpsgdOptimizer, + DecayedAdagradOptimizer, + AdadeltaOptimizer, + RMSPropOptimizer, + FtrlOptimizer, + LambOptimizer, +) +from paddle.fluid.optimizer import ( + ModelAverage, + DGCMomentumOptimizer, + ExponentialMovingAverage, + PipelineOptimizer, + LookaheadOptimizer, + RecomputeOptimizer, +) from paddle.fluid.dygraph import Linear from test_imperative_base import new_program_scope from paddle.fluid.framework import _test_eager_guard @@ -30,7 +50,6 @@ from paddle.fluid.framework import _test_eager_guard class MLP(fluid.Layer): - def __init__(self, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -44,7 +63,6 @@ class MLP(fluid.Layer): class TestImperativeOptimizerBase(unittest.TestCase): - def setUp(self): self.batch_num = 20 @@ -55,7 +73,6 @@ class TestImperativeOptimizerBase(unittest.TestCase): raise NotImplementedError() def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): image = np.array(item[0]).reshape(1, 784) @@ -68,8 +85,11 @@ class TestImperativeOptimizerBase(unittest.TestCase): seed = 90 batch_size = 128 if place == None: - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): try: @@ -77,7 +97,8 @@ class TestImperativeOptimizerBase(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) mlp = MLP() optimizer = self.get_optimizer_dygraph( - parameter_list=mlp.parameters()) + parameter_list=mlp.parameters() + ) except Exception as e: assert str(e) == exception_message @@ -86,8 +107,11 @@ class TestImperativeOptimizerBase(unittest.TestCase): batch_size = 128 if place == None: - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) with fluid.dygraph.guard(place): paddle.seed(seed) @@ -95,15 +119,18 @@ class TestImperativeOptimizerBase(unittest.TestCase): mlp = MLP() optimizer = self.get_optimizer_dygraph( - parameter_list=mlp.parameters()) + parameter_list=mlp.parameters() + ) batch_py_reader = fluid.io.PyReader(capacity=1) batch_py_reader.decorate_sample_list_generator( - paddle.batch(self.reader_decorator( - paddle.dataset.mnist.train()), - batch_size=batch_size, - drop_last=True), - places=fluid.CPUPlace()) + paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=batch_size, + drop_last=True, + ), + places=fluid.CPUPlace(), + ) dy_param_init_value = {} for batch_id, data in enumerate(batch_py_reader()): @@ -135,20 +162,23 @@ class TestImperativeOptimizerBase(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) if place == None: - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = fluid.Executor(place) mlp = MLP() optimizer = self.get_optimizer() - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') img = fluid.layers.reshape(img, shape=[batch_size, 784]) cost = mlp(img) @@ -161,8 +191,10 @@ class TestImperativeOptimizerBase(unittest.TestCase): for param in mlp.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -172,18 +204,21 @@ class TestImperativeOptimizerBase(unittest.TestCase): break static_x_data = np.array( - [x[0].reshape(1, 28, 28) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([128, 1]) + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([128, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_out = out[0] @@ -191,43 +226,48 @@ class TestImperativeOptimizerBase(unittest.TestCase): static_param_value[static_param_name_list[i - 1]] = out[i] for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) if core.is_compiled_with_rocm(): - np.testing.assert_allclose(static_out, - dy_out, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + static_out, dy_out, rtol=1e-05, atol=0.001 + ) else: np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) for key, value in static_param_value.items(): if core.is_compiled_with_rocm(): - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=0.001 + ) else: - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05 + ) class TestImperativeOptimizerPiecewiseDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): bd = [3, 6, 9] - optimizer = SGDOptimizer(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=[0.1 * (0.1**i) for i in range(len(bd) + 1)]), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, + values=[0.1 * (0.1**i) for i in range(len(bd) + 1)], + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): bd = [3, 6, 9] - optimizer = SGDOptimizer(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=[0.1 * (0.1**i) for i in range(len(bd) + 1)])) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, + values=[0.1 * (0.1**i) for i in range(len(bd) + 1)], + ) + ) return optimizer def func_test_sgd(self): @@ -240,22 +280,27 @@ class TestImperativeOptimizerPiecewiseDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerNaturalExpDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.natural_exp_decay( - learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.natural_exp_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = SGDOptimizer( - learning_rate=fluid.layers.natural_exp_decay(learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.natural_exp_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ) + ) return optimizer def func_test_sgd(self): @@ -268,22 +313,27 @@ class TestImperativeOptimizerNaturalExpDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerExponentialDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.exponential_decay( - learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.exponential_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = SGDOptimizer( - learning_rate=fluid.layers.exponential_decay(learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ) + ) return optimizer def func_test_sgd(self): @@ -296,22 +346,27 @@ class TestImperativeOptimizerExponentialDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerInverseTimeDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = Adam(learning_rate=fluid.layers.inverse_time_decay( - learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True), - parameter_list=parameter_list) + optimizer = Adam( + learning_rate=fluid.layers.inverse_time_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = Adam( - learning_rate=fluid.layers.inverse_time_decay(learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.inverse_time_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ) + ) return optimizer def func_test_adam(self): @@ -324,16 +379,21 @@ class TestImperativeOptimizerInverseTimeDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerPolynomialDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.polynomial_decay( - learning_rate=0.1, decay_steps=5, cycle=self.cycle), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.polynomial_decay( + learning_rate=0.1, decay_steps=5, cycle=self.cycle + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = SGDOptimizer(learning_rate=fluid.layers.polynomial_decay( - learning_rate=0.1, decay_steps=5, cycle=self.cycle)) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.polynomial_decay( + learning_rate=0.1, decay_steps=5, cycle=self.cycle + ) + ) return optimizer def func_test_sgd_cycle(self): @@ -356,16 +416,21 @@ class TestImperativeOptimizerPolynomialDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerCosineDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.cosine_decay( - learning_rate=0.1, step_each_epoch=10000, epochs=120), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.cosine_decay( + learning_rate=0.1, step_each_epoch=10000, epochs=120 + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = SGDOptimizer(learning_rate=fluid.layers.cosine_decay( - learning_rate=0.1, step_each_epoch=10000, epochs=120)) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.cosine_decay( + learning_rate=0.1, step_each_epoch=10000, epochs=120 + ) + ) return optimizer def func_test_sgd(self): @@ -378,16 +443,21 @@ class TestImperativeOptimizerCosineDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerNoamDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.noam_decay( - d_model=512, warmup_steps=8000), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.noam_decay( + d_model=512, warmup_steps=8000 + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = SGDOptimizer(learning_rate=fluid.layers.noam_decay( - d_model=512, warmup_steps=8000)) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.noam_decay( + d_model=512, warmup_steps=8000 + ) + ) return optimizer def func_test_sgd(self): @@ -400,7 +470,6 @@ class TestImperativeOptimizerNoamDecay(TestImperativeOptimizerBase): class TestOptimizerLearningRate(unittest.TestCase): - def func_test_constant_lr(self): with fluid.dygraph.guard(): a = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -413,13 +482,13 @@ class TestOptimizerLearningRate(unittest.TestCase): loss = fluid.layers.reduce_mean(b) - adam = fluid.optimizer.Adam(0.001, - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + 0.001, parameter_list=linear.parameters() + ) - np.testing.assert_allclose(adam.current_step_lr(), - 0.001, - rtol=1e-06, - atol=0.0) + np.testing.assert_allclose( + adam.current_step_lr(), 0.001, rtol=1e-06, atol=0.0 + ) for i in range(10): adam.minimize(loss) @@ -447,14 +516,14 @@ class TestOptimizerLearningRate(unittest.TestCase): bd = [2, 4, 6, 8] value = [0.2, 0.4, 0.6, 0.8, 1.0] - adam = fluid.optimizer.Adam(fluid.dygraph.PiecewiseDecay( - bd, value, 0), - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + fluid.dygraph.PiecewiseDecay(bd, value, 0), + parameter_list=linear.parameters(), + ) - np.testing.assert_allclose(adam.current_step_lr(), - 0.2, - rtol=1e-06, - atol=0.0) + np.testing.assert_allclose( + adam.current_step_lr(), 0.2, rtol=1e-06, atol=0.0 + ) ret = [0.2, 0.2, 0.4, 0.4, 0.6, 0.6, 0.8, 0.8, 1.0, 1.0, 1.0, 1.0] for i in range(12): @@ -481,17 +550,19 @@ class TestOptimizerLearningRate(unittest.TestCase): loss = fluid.layers.reduce_mean(b) base_lr = 1.0 - adam = fluid.optimizer.Adam(fluid.dygraph.NaturalExpDecay( - learning_rate=base_lr, - decay_steps=3, - decay_rate=0.5, - staircase=True), - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + fluid.dygraph.NaturalExpDecay( + learning_rate=base_lr, + decay_steps=3, + decay_rate=0.5, + staircase=True, + ), + parameter_list=linear.parameters(), + ) - np.testing.assert_allclose(adam.current_step_lr(), - 1.0, - rtol=1e-06, - atol=0.0) + np.testing.assert_allclose( + adam.current_step_lr(), 1.0, rtol=1e-06, atol=0.0 + ) ret = [1.0, 1.0, 1.0, np.exp(-0.5), np.exp(-0.5)] for i in range(5): @@ -526,21 +597,24 @@ class TestOptimizerLearningRate(unittest.TestCase): lr = adam.current_step_lr() np.testing.assert_allclose(lr, lr_list[i], rtol=1e-06, atol=0.0) - lr_var = fluid.layers.create_global_var(shape=[1], - value=0.7, - dtype='float32') + lr_var = fluid.layers.create_global_var( + shape=[1], value=0.7, dtype='float32' + ) adam.set_lr(lr_var) adam.minimize(loss) lr = adam.current_step_lr() np.testing.assert_allclose(lr, 0.7, rtol=1e-06, atol=0.0) with self.assertRaises(RuntimeError): - adam = fluid.optimizer.Adam(fluid.dygraph.NaturalExpDecay( - learning_rate=0.1, - decay_steps=3, - decay_rate=0.5, - staircase=True), - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + fluid.dygraph.NaturalExpDecay( + learning_rate=0.1, + decay_steps=3, + decay_rate=0.5, + staircase=True, + ), + parameter_list=linear.parameters(), + ) adam.set_lr(0.01) def test_set_lr(self): @@ -550,11 +624,10 @@ class TestOptimizerLearningRate(unittest.TestCase): class TestImperativeMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = MomentumOptimizer(learning_rate=0.001, - momentum=0.9, - parameter_list=parameter_list) + optimizer = MomentumOptimizer( + learning_rate=0.001, momentum=0.9, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -571,11 +644,10 @@ class TestImperativeMomentumOptimizer(TestImperativeOptimizerBase): class TestImperativeLarsMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = LarsMomentumOptimizer(learning_rate=0.001, - momentum=0.9, - parameter_list=parameter_list) + optimizer = LarsMomentumOptimizer( + learning_rate=0.001, momentum=0.9, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -592,10 +664,10 @@ class TestImperativeLarsMomentumOptimizer(TestImperativeOptimizerBase): class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdagradOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = AdagradOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -612,10 +684,10 @@ class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase): class TestImperativeAdamaxOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdamaxOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = AdamaxOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -632,21 +704,21 @@ class TestImperativeAdamaxOptimizer(TestImperativeOptimizerBase): class TestImperativeDpsgdOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DpsgdOptimizer(learning_rate=0.01, - clip=10.0, - batch_size=16.0, - sigma=1.0, - parameter_list=parameter_list) + optimizer = DpsgdOptimizer( + learning_rate=0.01, + clip=10.0, + batch_size=16.0, + sigma=1.0, + parameter_list=parameter_list, + ) optimizer._seed = 100 return optimizer def get_optimizer(self): - optimizer = DpsgdOptimizer(learning_rate=0.01, - clip=10.0, - batch_size=16.0, - sigma=1.0) + optimizer = DpsgdOptimizer( + learning_rate=0.01, clip=10.0, batch_size=16.0, sigma=1.0 + ) optimizer._seed = 100 return optimizer @@ -660,10 +732,10 @@ class TestImperativeDpsgdOptimizer(TestImperativeOptimizerBase): class TestImperativeDecayedAdagradOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DecayedAdagradOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = DecayedAdagradOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -680,18 +752,19 @@ class TestImperativeDecayedAdagradOptimizer(TestImperativeOptimizerBase): class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdadeltaOptimizer(learning_rate=0.0003, - epsilon=1.0e-6, - rho=0.95, - parameter_list=parameter_list) + optimizer = AdadeltaOptimizer( + learning_rate=0.0003, + epsilon=1.0e-6, + rho=0.95, + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = AdadeltaOptimizer(learning_rate=0.0003, - epsilon=1.0e-6, - rho=0.95) + optimizer = AdadeltaOptimizer( + learning_rate=0.0003, epsilon=1.0e-6, rho=0.95 + ) return optimizer def func_test_adadelta(self): @@ -704,10 +777,10 @@ class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase): class TestImperativeRMSPropOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = RMSPropOptimizer(learning_rate=0.1, - parameter_list=parameter_list) + optimizer = RMSPropOptimizer( + learning_rate=0.1, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -724,10 +797,10 @@ class TestImperativeRMSPropOptimizer(TestImperativeOptimizerBase): class TestImperativeFtrlOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = FtrlOptimizer(learning_rate=0.1, - parameter_list=parameter_list) + optimizer = FtrlOptimizer( + learning_rate=0.1, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -748,16 +821,18 @@ def exclude_fn(param): class TestImperativeLambOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = LambOptimizer(learning_rate=0.002, - exclude_from_weight_decay_fn=exclude_fn, - parameter_list=parameter_list) + optimizer = LambOptimizer( + learning_rate=0.002, + exclude_from_weight_decay_fn=exclude_fn, + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = LambOptimizer(learning_rate=0.002, - exclude_from_weight_decay_fn=exclude_fn) + optimizer = LambOptimizer( + learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn + ) return optimizer # should fix: may fail in CI-windows @@ -766,11 +841,10 @@ class TestImperativeLambOptimizer(TestImperativeOptimizerBase): class TestImperativeModelAverage(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = ModelAverage(0.15, - min_average_window=10000, - max_average_window=12500) + optimizer = ModelAverage( + 0.15, min_average_window=10000, max_average_window=12500 + ) return optimizer def func_test_modelaverage(self): @@ -784,13 +858,14 @@ class TestImperativeModelAverage(TestImperativeOptimizerBase): class TestImperativeDGCMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DGCMomentumOptimizer(learning_rate=0.0001, - momentum=0.9, - rampup_step=1000, - rampup_begin_step=1252, - sparsity=[0.999, 0.999]) + optimizer = DGCMomentumOptimizer( + learning_rate=0.0001, + momentum=0.9, + rampup_step=1000, + rampup_begin_step=1252, + sparsity=[0.999, 0.999], + ) return optimizer def func_test_dgcmomentum(self): @@ -804,13 +879,14 @@ class TestImperativeDGCMomentumOptimizer(TestImperativeOptimizerBase): class TestImperativeExponentialMovingAverage(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = ExponentialMovingAverage(0.999) return optimizer def func_test_exponentialmoving(self): - exception_message = "In dygraph, don't support ExponentialMovingAverage." + exception_message = ( + "In dygraph, don't support ExponentialMovingAverage." + ) self._check_exception(exception_message) def test_exponentialmoving(self): @@ -820,10 +896,10 @@ class TestImperativeExponentialMovingAverage(TestImperativeOptimizerBase): class TestImperativePipelineOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = fluid.optimizer.SGD(learning_rate=0.5, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.5, parameter_list=parameter_list + ) optimizer = PipelineOptimizer(optimizer) return optimizer @@ -838,10 +914,10 @@ class TestImperativePipelineOptimizer(TestImperativeOptimizerBase): class TestImperativeLookaheadOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = fluid.optimizer.SGD(learning_rate=0.5, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.5, parameter_list=parameter_list + ) optimizer = LookaheadOptimizer(optimizer, alpha=0.5, k=5) return optimizer @@ -856,10 +932,10 @@ class TestImperativeLookaheadOptimizer(TestImperativeOptimizerBase): class TestImperativeRecomputeOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = fluid.optimizer.SGD(learning_rate=0.5, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.5, parameter_list=parameter_list + ) optimizer = RecomputeOptimizer(optimizer) return optimizer @@ -874,16 +950,17 @@ class TestImperativeRecomputeOptimizer(TestImperativeOptimizerBase): class TestImperativeOptimizerList(unittest.TestCase): - def func_test_parameter_list(self): with fluid.dygraph.guard(): linear_1 = Linear(10, 10) linear_2 = Linear(10, 10) - sgd = SGDOptimizer(1.0, - parameter_list=itertools.chain( - linear_1.parameters(), - linear_2.parameters())) + sgd = SGDOptimizer( + 1.0, + parameter_list=itertools.chain( + linear_1.parameters(), linear_2.parameters() + ), + ) in_np = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") in_data = fluid.dygraph.to_variable(in_np) @@ -895,8 +972,9 @@ class TestImperativeOptimizerList(unittest.TestCase): sgd.minimize(loss) self.assertTrue( - len(sgd._parameter_list) == len(linear_1.parameters() + - linear_2.parameters())) + len(sgd._parameter_list) + == len(linear_1.parameters() + linear_2.parameters()) + ) def test_parameter_list(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py b/python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py index 3b0ba3572cc..251ac5b6b93 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py @@ -19,8 +19,25 @@ import itertools import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.optimizer import MomentumOptimizer, LarsMomentumOptimizer, AdagradOptimizer, AdamaxOptimizer, DpsgdOptimizer, DecayedAdagradOptimizer, AdadeltaOptimizer, RMSPropOptimizer, FtrlOptimizer -from paddle.fluid.optimizer import ModelAverage, DGCMomentumOptimizer, ExponentialMovingAverage, PipelineOptimizer, LookaheadOptimizer, RecomputeOptimizer +from paddle.fluid.optimizer import ( + MomentumOptimizer, + LarsMomentumOptimizer, + AdagradOptimizer, + AdamaxOptimizer, + DpsgdOptimizer, + DecayedAdagradOptimizer, + AdadeltaOptimizer, + RMSPropOptimizer, + FtrlOptimizer, +) +from paddle.fluid.optimizer import ( + ModelAverage, + DGCMomentumOptimizer, + ExponentialMovingAverage, + PipelineOptimizer, + LookaheadOptimizer, + RecomputeOptimizer, +) from paddle.fluid.dygraph import Linear from test_imperative_base import new_program_scope from paddle.fluid.framework import _test_eager_guard @@ -30,7 +47,6 @@ from paddle.fluid.framework import _test_eager_guard class MLP(fluid.Layer): - def __init__(self, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -44,7 +60,6 @@ class MLP(fluid.Layer): class TestImperativeOptimizerBase(unittest.TestCase): - def setUp(self): self.batch_num = 20 @@ -55,7 +70,6 @@ class TestImperativeOptimizerBase(unittest.TestCase): raise NotImplementedError() def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): image = np.array(item[0]).reshape(1, 784) @@ -68,8 +82,11 @@ class TestImperativeOptimizerBase(unittest.TestCase): seed = 90 batch_size = 128 if place == None: - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) try: paddle.disable_static() @@ -77,7 +94,8 @@ class TestImperativeOptimizerBase(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) mlp = MLP() optimizer = self.get_optimizer_dygraph( - parameter_list=mlp.parameters()) + parameter_list=mlp.parameters() + ) except Exception as e: assert str(e) == exception_message finally: @@ -88,8 +106,11 @@ class TestImperativeOptimizerBase(unittest.TestCase): batch_size = 128 if place == None: - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) paddle.disable_static(place) paddle.seed(seed) @@ -99,11 +120,14 @@ class TestImperativeOptimizerBase(unittest.TestCase): optimizer = self.get_optimizer_dygraph(parameter_list=mlp.parameters()) batch_py_reader = fluid.io.PyReader(capacity=1) - batch_py_reader.decorate_sample_list_generator(paddle.batch( - self.reader_decorator(paddle.dataset.mnist.train()), - batch_size=batch_size, - drop_last=True), - places=fluid.CPUPlace()) + batch_py_reader.decorate_sample_list_generator( + paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=batch_size, + drop_last=True, + ), + places=fluid.CPUPlace(), + ) dy_param_init_value = {} for batch_id, data in enumerate(batch_py_reader()): @@ -126,10 +150,13 @@ class TestImperativeOptimizerBase(unittest.TestCase): avg_loss.backward() optimizer.minimize(avg_loss) - if isinstance(optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): - if isinstance(optimizer._learning_rate, - paddle.optimizer.lr.ReduceOnPlateau): + if isinstance( + optimizer._learning_rate, paddle.optimizer.lr.LRScheduler + ): + if isinstance( + optimizer._learning_rate, + paddle.optimizer.lr.ReduceOnPlateau, + ): optimizer._learning_rate.step(avg_loss) else: optimizer._learning_rate.step() @@ -144,20 +171,23 @@ class TestImperativeOptimizerBase(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) if place == None: - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = fluid.Executor(place) mlp = MLP() optimizer = self.get_optimizer() - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') img = fluid.layers.reshape(img, shape=[batch_size, 784]) cost = mlp(img) @@ -170,8 +200,10 @@ class TestImperativeOptimizerBase(unittest.TestCase): for param in mlp.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -181,22 +213,28 @@ class TestImperativeOptimizerBase(unittest.TestCase): break static_x_data = np.array( - [x[0].reshape(1, 28, 28) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([128, 1]) + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([128, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) - if isinstance(optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): - if isinstance(optimizer._learning_rate, - paddle.optimizer.lr.ReduceOnPlateau): + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) + if isinstance( + optimizer._learning_rate, paddle.optimizer.lr.LRScheduler + ): + if isinstance( + optimizer._learning_rate, + paddle.optimizer.lr.ReduceOnPlateau, + ): optimizer._learning_rate.step(out[0]) else: optimizer._learning_rate.step() @@ -207,39 +245,38 @@ class TestImperativeOptimizerBase(unittest.TestCase): static_param_value[static_param_name_list[i - 1]] = out[i] for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) if core.is_compiled_with_rocm(): - np.testing.assert_allclose(static_out, - dy_out, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + static_out, dy_out, rtol=1e-05, atol=0.001 + ) else: np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) for key, value in static_param_value.items(): if core.is_compiled_with_rocm(): - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=0.001 + ) else: - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05 + ) class TestImperativeOptimizerPiecewiseDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): bd = [3, 6, 9] optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.PiecewiseDecay( boundaries=bd, - values=[0.1 * (0.1**i) for i in range(len(bd) + 1)]), - parameters=parameter_list) + values=[0.1 * (0.1**i) for i in range(len(bd) + 1)], + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): @@ -247,7 +284,9 @@ class TestImperativeOptimizerPiecewiseDecay(TestImperativeOptimizerBase): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.PiecewiseDecay( boundaries=bd, - values=[0.1 * (0.1**i) for i in range(len(bd) + 1)])) + values=[0.1 * (0.1**i) for i in range(len(bd) + 1)], + ) + ) return optimizer def func_test_sgd(self): @@ -260,18 +299,21 @@ class TestImperativeOptimizerPiecewiseDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerNaturalExpDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.NaturalExpDecay(learning_rate=0.5, - gamma=0.9), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.NaturalExpDecay( + learning_rate=0.5, gamma=0.9 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.NaturalExpDecay(learning_rate=0.5, - gamma=0.9)) + learning_rate=paddle.optimizer.lr.NaturalExpDecay( + learning_rate=0.5, gamma=0.9 + ) + ) return optimizer def func_test_sgd(self): @@ -284,18 +326,21 @@ class TestImperativeOptimizerNaturalExpDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerExponentialDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.ExponentialDecay( - learning_rate=0.5, gamma=0.9), - parameters=parameter_list) + learning_rate=0.5, gamma=0.9 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.ExponentialDecay( - learning_rate=0.5, gamma=0.9)) + learning_rate=0.5, gamma=0.9 + ) + ) return optimizer def func_test_sgd(self): @@ -308,18 +353,21 @@ class TestImperativeOptimizerExponentialDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerInverseTimeDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.Adam( learning_rate=paddle.optimizer.lr.InverseTimeDecay( - learning_rate=0.5, gamma=0.9), - parameters=parameter_list) + learning_rate=0.5, gamma=0.9 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.Adam( learning_rate=paddle.optimizer.lr.InverseTimeDecay( - learning_rate=0.5, gamma=0.9)) + learning_rate=0.5, gamma=0.9 + ) + ) return optimizer def func_test_adam(self): @@ -332,19 +380,21 @@ class TestImperativeOptimizerInverseTimeDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerPolynomialDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.PolynomialDecay(learning_rate=0.5, - decay_steps=5, - cycle=self.cycle), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.PolynomialDecay( + learning_rate=0.5, decay_steps=5, cycle=self.cycle + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.PolynomialDecay( - learning_rate=0.5, decay_steps=5, cycle=self.cycle)) + learning_rate=0.5, decay_steps=5, cycle=self.cycle + ) + ) return optimizer def func_test_sgd_cycle(self): @@ -367,18 +417,21 @@ class TestImperativeOptimizerPolynomialDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerCosineAnnealingDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.CosineAnnealingDecay( - learning_rate=0.5, T_max=5), - parameters=parameter_list) + learning_rate=0.5, T_max=5 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.CosineAnnealingDecay( - learning_rate=0.5, T_max=5)) + learning_rate=0.5, T_max=5 + ) + ) return optimizer def func_test_sgd(self): @@ -391,19 +444,21 @@ class TestImperativeOptimizerCosineAnnealingDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerNoamDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100)) + learning_rate=paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100 + ) + ) return optimizer def func_test_sgd(self): @@ -416,18 +471,21 @@ class TestImperativeOptimizerNoamDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerLambdaDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.LambdaDecay( - learning_rate=0.5, lr_lambda=lambda epoch: 0.9**epoch), - parameters=parameter_list) + learning_rate=0.5, lr_lambda=lambda epoch: 0.9**epoch + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.LambdaDecay( - learning_rate=0.5, lr_lambda=lambda epoch: 0.9**epoch)) + learning_rate=0.5, lr_lambda=lambda epoch: 0.9**epoch + ) + ) return optimizer def func_test_sgd(self): @@ -440,23 +498,25 @@ class TestImperativeOptimizerLambdaDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerLinearWarmup(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.LinearWarmup(learning_rate=0.5, - warmup_steps=20, - start_lr=0, - end_lr=0.5), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.LinearWarmup( + learning_rate=0.5, warmup_steps=20, start_lr=0, end_lr=0.5 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.LinearWarmup(learning_rate=0.5, - warmup_steps=20, - start_lr=0, - end_lr=0.5, - verbose=True)) + learning_rate=paddle.optimizer.lr.LinearWarmup( + learning_rate=0.5, + warmup_steps=20, + start_lr=0, + end_lr=0.5, + verbose=True, + ) + ) return optimizer def func_test_sgd(self): @@ -469,18 +529,21 @@ class TestImperativeOptimizerLinearWarmup(TestImperativeOptimizerBase): class TestImperativeOptimizerMultiStepDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.MultiStepDecay( - learning_rate=0.5, milestones=[2, 4, 6], gamma=0.8), - parameters=parameter_list) + learning_rate=0.5, milestones=[2, 4, 6], gamma=0.8 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.MultiStepDecay( - learning_rate=0.5, milestones=[2, 4, 6], gamma=0.8)) + learning_rate=0.5, milestones=[2, 4, 6], gamma=0.8 + ) + ) return optimizer def func_test_sgd(self): @@ -493,19 +556,21 @@ class TestImperativeOptimizerMultiStepDecay(TestImperativeOptimizerBase): class TestImperativeOptimizerStepLR(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.StepDecay(learning_rate=0.5, - step_size=5, - gamma=0.8), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.StepDecay( + learning_rate=0.5, step_size=5, gamma=0.8 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.StepDecay( - learning_rate=0.5, step_size=5, gamma=0.8)) + learning_rate=0.5, step_size=5, gamma=0.8 + ) + ) return optimizer def func_test_sgd(self): @@ -518,18 +583,19 @@ class TestImperativeOptimizerStepLR(TestImperativeOptimizerBase): class TestImperativeOptimizerReduceOnPlateau(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.ReduceOnPlateau( - learning_rate=0.5), - parameters=parameter_list) + learning_rate=0.5 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.ReduceOnPlateau( - learning_rate=0.5)) + learning_rate=paddle.optimizer.lr.ReduceOnPlateau(learning_rate=0.5) + ) return optimizer def func_test_sgd(self): @@ -542,7 +608,6 @@ class TestImperativeOptimizerReduceOnPlateau(TestImperativeOptimizerBase): class TestOptimizerLearningRate(unittest.TestCase): - def func_test_constant_lr(self): with fluid.dygraph.guard(): a = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -557,10 +622,9 @@ class TestOptimizerLearningRate(unittest.TestCase): adam = paddle.optimizer.Adam(0.001, parameters=linear.parameters()) - np.testing.assert_allclose(adam.get_lr(), - 0.001, - rtol=1e-06, - atol=0.0) + np.testing.assert_allclose( + adam.get_lr(), 0.001, rtol=1e-06, atol=0.0 + ) for i in range(10): adam.minimize(loss) @@ -589,8 +653,9 @@ class TestOptimizerLearningRate(unittest.TestCase): value = [0.2, 0.4, 0.6, 0.8, 1.0] scheduler = paddle.optimizer.lr.PiecewiseDecay(bd, value) - adam = paddle.optimizer.Adam(scheduler, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + scheduler, parameters=linear.parameters() + ) np.testing.assert_allclose(adam.get_lr(), 0.2, rtol=1e-06, atol=0.0) @@ -618,8 +683,9 @@ class TestOptimizerLearningRate(unittest.TestCase): base_lr = 1.0 scheduler = paddle.optimizer.lr.NaturalExpDecay(1.0, gamma=0.5) - adam = paddle.optimizer.Adam(scheduler, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + scheduler, parameters=linear.parameters() + ) np.testing.assert_allclose(adam.get_lr(), 1.0, rtol=1e-06, atol=0.0) @@ -657,16 +723,18 @@ class TestOptimizerLearningRate(unittest.TestCase): np.testing.assert_allclose(lr, lr_list[i], rtol=1e-06, atol=0.0) with self.assertRaises(TypeError): - lr_var = fluid.layers.create_global_var(shape=[1], - value=0.7, - dtype='float32') + lr_var = fluid.layers.create_global_var( + shape=[1], value=0.7, dtype='float32' + ) adam.set_lr(lr_var) with self.assertRaises(RuntimeError): adam = paddle.optimizer.Adam( - paddle.optimizer.lr.NaturalExpDecay(learning_rate=0.1, - gamma=0.5), - parameters=linear.parameters()) + paddle.optimizer.lr.NaturalExpDecay( + learning_rate=0.1, gamma=0.5 + ), + parameters=linear.parameters(), + ) adam.set_lr(0.01) def test_set_lr(self): @@ -676,11 +744,10 @@ class TestOptimizerLearningRate(unittest.TestCase): class TestImperativeMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = MomentumOptimizer(learning_rate=0.001, - momentum=0.9, - parameter_list=parameter_list) + optimizer = MomentumOptimizer( + learning_rate=0.001, momentum=0.9, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -697,11 +764,10 @@ class TestImperativeMomentumOptimizer(TestImperativeOptimizerBase): class TestImperativeLarsMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = LarsMomentumOptimizer(learning_rate=0.001, - momentum=0.9, - parameter_list=parameter_list) + optimizer = LarsMomentumOptimizer( + learning_rate=0.001, momentum=0.9, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -718,10 +784,10 @@ class TestImperativeLarsMomentumOptimizer(TestImperativeOptimizerBase): class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdagradOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = AdagradOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -738,10 +804,10 @@ class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase): class TestImperativeAdamaxOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdamaxOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = AdamaxOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -758,21 +824,21 @@ class TestImperativeAdamaxOptimizer(TestImperativeOptimizerBase): class TestImperativeDpsgdOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DpsgdOptimizer(learning_rate=0.01, - clip=10.0, - batch_size=16.0, - sigma=1.0, - parameter_list=parameter_list) + optimizer = DpsgdOptimizer( + learning_rate=0.01, + clip=10.0, + batch_size=16.0, + sigma=1.0, + parameter_list=parameter_list, + ) optimizer._seed = 100 return optimizer def get_optimizer(self): - optimizer = DpsgdOptimizer(learning_rate=0.01, - clip=10.0, - batch_size=16.0, - sigma=1.0) + optimizer = DpsgdOptimizer( + learning_rate=0.01, clip=10.0, batch_size=16.0, sigma=1.0 + ) optimizer._seed = 100 return optimizer @@ -786,10 +852,10 @@ class TestImperativeDpsgdOptimizer(TestImperativeOptimizerBase): class TestImperativeDecayedAdagradOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DecayedAdagradOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = DecayedAdagradOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -806,18 +872,19 @@ class TestImperativeDecayedAdagradOptimizer(TestImperativeOptimizerBase): class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdadeltaOptimizer(learning_rate=0.0003, - epsilon=1.0e-6, - rho=0.95, - parameter_list=parameter_list) + optimizer = AdadeltaOptimizer( + learning_rate=0.0003, + epsilon=1.0e-6, + rho=0.95, + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = AdadeltaOptimizer(learning_rate=0.0003, - epsilon=1.0e-6, - rho=0.95) + optimizer = AdadeltaOptimizer( + learning_rate=0.0003, epsilon=1.0e-6, rho=0.95 + ) return optimizer def func_test_adadelta(self): @@ -830,10 +897,10 @@ class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase): class TestImperativeRMSPropOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = RMSPropOptimizer(learning_rate=0.1, - parameter_list=parameter_list) + optimizer = RMSPropOptimizer( + learning_rate=0.1, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -850,10 +917,10 @@ class TestImperativeRMSPropOptimizer(TestImperativeOptimizerBase): class TestImperativeFtrlOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = FtrlOptimizer(learning_rate=0.1, - parameter_list=parameter_list) + optimizer = FtrlOptimizer( + learning_rate=0.1, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -874,17 +941,18 @@ def exclude_fn(param): class TestImperativeLambOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.Lamb( learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn, - parameters=parameter_list) + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.Lamb( - learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn) + learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn + ) return optimizer # should fix: may fail in CI-windows @@ -893,11 +961,10 @@ class TestImperativeLambOptimizer(TestImperativeOptimizerBase): class TestImperativeModelAverage(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = ModelAverage(0.15, - min_average_window=10000, - max_average_window=12500) + optimizer = ModelAverage( + 0.15, min_average_window=10000, max_average_window=12500 + ) return optimizer def func_test_modelaverage(self): @@ -911,13 +978,14 @@ class TestImperativeModelAverage(TestImperativeOptimizerBase): class TestImperativeDGCMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DGCMomentumOptimizer(learning_rate=0.0001, - momentum=0.9, - rampup_step=1000, - rampup_begin_step=1252, - sparsity=[0.999, 0.999]) + optimizer = DGCMomentumOptimizer( + learning_rate=0.0001, + momentum=0.9, + rampup_step=1000, + rampup_begin_step=1252, + sparsity=[0.999, 0.999], + ) return optimizer def func_test_dgcmomentum(self): @@ -931,13 +999,14 @@ class TestImperativeDGCMomentumOptimizer(TestImperativeOptimizerBase): class TestImperativeExponentialMovingAverage(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = ExponentialMovingAverage(0.999) return optimizer def func_test_exponentialmoving(self): - exception_message = "In dygraph, don't support ExponentialMovingAverage." + exception_message = ( + "In dygraph, don't support ExponentialMovingAverage." + ) self._check_exception(exception_message) def test_exponentialmoving(self): @@ -947,10 +1016,10 @@ class TestImperativeExponentialMovingAverage(TestImperativeOptimizerBase): class TestImperativePipelineOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = paddle.optimizer.SGD(learning_rate=0.5, - parameters=parameter_list) + optimizer = paddle.optimizer.SGD( + learning_rate=0.5, parameters=parameter_list + ) optimizer = PipelineOptimizer(optimizer) return optimizer @@ -965,10 +1034,10 @@ class TestImperativePipelineOptimizer(TestImperativeOptimizerBase): class TestImperativeLookaheadOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = paddle.optimizer.SGD(learning_rate=0.5, - parameters=parameter_list) + optimizer = paddle.optimizer.SGD( + learning_rate=0.5, parameters=parameter_list + ) optimizer = LookaheadOptimizer(optimizer, alpha=0.5, k=5) return optimizer @@ -983,10 +1052,10 @@ class TestImperativeLookaheadOptimizer(TestImperativeOptimizerBase): class TestImperativeRecomputeOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = paddle.optimizer.SGD(learning_rate=0.5, - parameters=parameter_list) + optimizer = paddle.optimizer.SGD( + learning_rate=0.5, parameters=parameter_list + ) optimizer = RecomputeOptimizer(optimizer) return optimizer @@ -1001,16 +1070,17 @@ class TestImperativeRecomputeOptimizer(TestImperativeOptimizerBase): class TestImperativeOptimizerList(unittest.TestCase): - def func_test_parameter_list(self): with fluid.dygraph.guard(): linear_1 = Linear(10, 10) linear_2 = Linear(10, 10) - sgd = paddle.optimizer.SGD(1.0, - parameters=itertools.chain( - linear_1.parameters(), - linear_2.parameters())) + sgd = paddle.optimizer.SGD( + 1.0, + parameters=itertools.chain( + linear_1.parameters(), linear_2.parameters() + ), + ) in_np = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") in_data = fluid.dygraph.to_variable(in_np) @@ -1022,8 +1092,9 @@ class TestImperativeOptimizerList(unittest.TestCase): sgd.minimize(loss) self.assertTrue( - len(sgd._parameter_list) == len(linear_1.parameters() + - linear_2.parameters())) + len(sgd._parameter_list) + == len(linear_1.parameters() + linear_2.parameters()) + ) def test_parameter_list(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_parallel_coalesce_split.py b/python/paddle/fluid/tests/unittests/test_imperative_parallel_coalesce_split.py index 7c6d13399c0..300a195f6dc 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_parallel_coalesce_split.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_parallel_coalesce_split.py @@ -20,11 +20,14 @@ import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid.dygraph.parallel import DataParallel from paddle.fluid.dygraph.base import to_variable -from paddle.fluid.dygraph.parallel import _coalesce_tensors, _split_tensors, _reshape_inplace +from paddle.fluid.dygraph.parallel import ( + _coalesce_tensors, + _split_tensors, + _reshape_inplace, +) class MyLayer(fluid.Layer): - def __init__(self, name_scope): super(MyLayer, self).__init__(name_scope) @@ -36,7 +39,6 @@ class MyLayer(fluid.Layer): class TestImperativeParallelCoalesceSplit(unittest.TestCase): - def test_coalesce_split(self): with fluid.dygraph.guard(): test_layer = MyLayer("test_layer") @@ -47,8 +49,9 @@ class TestImperativeParallelCoalesceSplit(unittest.TestCase): vars = [] vars.append(to_variable(np.random.random([2, 3]).astype("float32"))) vars.append(to_variable(np.random.random([4, 9]).astype("float32"))) - vars.append(to_variable( - np.random.random([10, 1]).astype("float32"))) + vars.append( + to_variable(np.random.random([10, 1]).astype("float32")) + ) var_groups = OrderedDict() var_groups.setdefault(0, vars) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_partitial_backward.py b/python/paddle/fluid/tests/unittests/test_imperative_partitial_backward.py index f966b2ba9a8..7f723a5d665 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_partitial_backward.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_partitial_backward.py @@ -19,7 +19,6 @@ from paddle.fluid.framework import _test_eager_guard class TestImperativePartitialBackward(unittest.TestCase): - def func_partitial_backward(self): with fluid.dygraph.guard(): x = np.random.randn(2, 4, 5).astype("float32") @@ -39,11 +38,14 @@ class TestImperativePartitialBackward(unittest.TestCase): self.assertIsNone(param._grad_ivar()) optimizer = fluid.optimizer.AdamOptimizer( - parameter_list=(linear1.parameters() + linear2.parameters())) + parameter_list=(linear1.parameters() + linear2.parameters()) + ) _, params_grads = optimizer.minimize(loss) - self.assertListEqual(sorted([p.name for p in linear1.parameters()]), - sorted([p_g[0].name for p_g in params_grads])) + self.assertListEqual( + sorted([p.name for p in linear1.parameters()]), + sorted([p_g[0].name for p_g in params_grads]), + ) linear1.clear_gradients() linear2.clear_gradients() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py index 0588016ce27..ecf81ed5433 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py @@ -28,13 +28,9 @@ from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -56,19 +52,26 @@ class SimpleLSTMRNN(fluid.Layer): weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -76,29 +79,29 @@ class SimpleLSTMRNN(fluid.Layer): self.hidden_array = [] for i in range(self._num_layers): - pre_hidden = fluid.layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = fluid.layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = fluid.layers.reshape(pre_hidden, - shape=[-1, self._hidden_size]) - pre_cell = fluid.layers.reshape(pre_cell, - shape=[-1, self._hidden_size]) + pre_hidden = fluid.layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = fluid.layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = fluid.layers.reshape( + pre_hidden, shape=[-1, self._hidden_size] + ) + pre_cell = fluid.layers.reshape( + pre_cell, shape=[-1, self._hidden_size] + ) self.hidden_array.append(pre_hidden) self.cell_array.append(pre_cell) res = [] for index in range(self._num_steps): - self._input = fluid.layers.slice(input_embedding, - axes=[1], - starts=[index], - ends=[index + 1]) - self._input = fluid.layers.reshape(self._input, - shape=[-1, self._hidden_size]) + self._input = fluid.layers.slice( + input_embedding, axes=[1], starts=[index], ends=[index + 1] + ) + self._input = fluid.layers.reshape( + self._input, shape=[-1, self._hidden_size] + ) for k in range(self._num_layers): pre_hidden = self.hidden_array[k] pre_cell = self.cell_array[k] @@ -109,11 +112,12 @@ class SimpleLSTMRNN(fluid.Layer): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) self.hidden_array[k] = m self.cell_array[k] = c @@ -123,33 +127,39 @@ class SimpleLSTMRNN(fluid.Layer): self._input = fluid.layers.dropout( self._input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append( - fluid.layers.reshape(self._input, - shape=[1, -1, self._hidden_size])) + fluid.layers.reshape( + self._input, shape=[1, -1, self._hidden_size] + ) + ) real_res = fluid.layers.concat(res, 0) real_res = fluid.layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = fluid.layers.concat(self.hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(self.cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -157,11 +167,13 @@ class PtbModel(fluid.Layer): self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', @@ -169,46 +181,60 @@ class PtbModel(fluid.Layer): param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.drop_out, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) rnn_out = fluid.layers.reshape( - rnn_out, shape=[-1, self.num_steps, self.hidden_size]) + rnn_out, shape=[-1, self.num_steps, self.hidden_size] + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -217,7 +243,6 @@ class PtbModel(fluid.Layer): class TestDygraphPtbRnn(unittest.TestCase): - def func_test_ptb_rnn(self): for is_sparse in [True, False]: self.ptb_rnn_cpu_float32(is_sparse) @@ -242,15 +267,18 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -265,27 +293,32 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) if i % 5 == 0 and _in_legacy_dygraph(): outs, traced_layer = TracedLayer.trace( - ptb_model, [x, y, init_hidden, init_cell]) + ptb_model, [x, y, init_hidden, init_cell] + ) outs_static = traced_layer([x, y, init_hidden, init_cell]) helper.assertEachVar(outs, outs_static) if program is not None: self.assertTrue( - is_equal_program(traced_layer.program, program)) + is_equal_program(traced_layer.program, program) + ) program = traced_layer.program traced_layer.save_inference_model( - './infe_imperative_ptb_rnn', feed=list(range(4))) + './infe_imperative_ptb_rnn', feed=list(range(4)) + ) else: outs = ptb_model(x, y, init_hidden, init_cell) @@ -308,29 +341,35 @@ class TestDygraphPtbRnn(unittest.TestCase): with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse) - - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + ) + + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -338,8 +377,10 @@ class TestDygraphPtbRnn(unittest.TestCase): for param in ptb_model.parameters(): static_param_name_list.append(param.name) - out = exe.run(framework.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + framework.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_loss_value = None @@ -351,33 +392,40 @@ class TestDygraphPtbRnn(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] if i == batch_num - 1: for k in range(3, len(out)): - static_param_updated[static_param_name_list[k - - 3]] = out[k] + static_param_updated[ + static_param_name_list[k - 3] + ] = out[k] np.testing.assert_array_equal(static_loss_value, dy_loss_value) - np.testing.assert_array_equal(static_last_cell_value, - dy_last_cell_value) - np.testing.assert_array_equal(static_last_hidden_value, - dy_last_hidden_value) + np.testing.assert_array_equal( + static_last_cell_value, dy_last_cell_value + ) + np.testing.assert_array_equal( + static_last_hidden_value, dy_last_hidden_value + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, dy_param_init[key]) for key, value in static_param_updated.items(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn_sorted_gradient.py index 286049500c2..38c8b9ca73d 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn_sorted_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn_sorted_gradient.py @@ -26,7 +26,6 @@ from paddle.fluid.framework import _test_eager_guard class TestDygraphPtbRnnSortGradient(unittest.TestCase): - def func_ptb_rnn_sort_gradient(self): for is_sparse in [True, False]: self.ptb_rnn_sort_gradient_cpu_float32(is_sparse) @@ -47,15 +46,18 @@ class TestDygraphPtbRnnSortGradient(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -68,15 +70,18 @@ class TestDygraphPtbRnnSortGradient(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -95,29 +100,35 @@ class TestDygraphPtbRnnSortGradient(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse) - - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + ) + + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps, 1], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps, 1], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -125,8 +136,10 @@ class TestDygraphPtbRnnSortGradient(unittest.TestCase): for param in ptb_model.parameters(): static_param_name_list.append(param.name) - out = exe.run(framework.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + framework.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_loss_value = None @@ -138,33 +151,40 @@ class TestDygraphPtbRnnSortGradient(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] if i == batch_num - 1: for k in range(3, len(out)): - static_param_updated[static_param_name_list[k - - 3]] = out[k] + static_param_updated[ + static_param_name_list[k - 3] + ] = out[k] np.testing.assert_array_equal(static_loss_value, dy_loss_value) - np.testing.assert_array_equal(static_last_cell_value, - dy_last_cell_value) - np.testing.assert_array_equal(static_last_hidden_value, - dy_last_hidden_value) + np.testing.assert_array_equal( + static_last_cell_value, dy_last_cell_value + ) + np.testing.assert_array_equal( + static_last_hidden_value, dy_last_hidden_value + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, dy_param_init[key]) for key, value in static_param_updated.items(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_recurrent_usage.py b/python/paddle/fluid/tests/unittests/test_imperative_recurrent_usage.py index 622d174eefc..123803731da 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_recurrent_usage.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_recurrent_usage.py @@ -23,7 +23,6 @@ import numpy as np class RecurrentTest(fluid.Layer): - def __init__(self, name_scope): super(RecurrentTest, self).__init__(name_scope) @@ -34,7 +33,6 @@ class RecurrentTest(fluid.Layer): class TestRecurrentFeed(unittest.TestCase): - def test_recurrent_feed(self): seed = 90 @@ -84,28 +82,33 @@ class TestRecurrentFeed(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - in1 = fluid.layers.data(name="inp1", - shape=[2, 2], - append_batch_size=False) - in2 = fluid.layers.data(name="inp2", - shape=[2, 2], - append_batch_size=False) + in1 = fluid.layers.data( + name="inp1", shape=[2, 2], append_batch_size=False + ) + in2 = fluid.layers.data( + name="inp2", shape=[2, 2], append_batch_size=False + ) rt1 = RecurrentTest("RecurrentTest") static_sum_out, static_out = rt1(in1, in2) fluid.backward.append_backward(static_sum_out) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) - - static_dout = fluid.default_main_program().block( - 0)._find_var_recursive(static_out.name + "@GRAD") + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + + static_dout = ( + fluid.default_main_program() + .block(0) + ._find_var_recursive(static_out.name + "@GRAD") + ) fetch_list = [static_sum_out, static_out, static_dout] for i in range(3): - out = exe.run(fluid.default_main_program(), - feed={ - "inp1": original_np1, - "inp2": original_np2 - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"inp1": original_np1, "inp2": original_np2}, + fetch_list=fetch_list, + ) static_out_value = out[1] static_sum_out = out[0] static_dout = out[2] diff --git a/python/paddle/fluid/tests/unittests/test_imperative_reinforcement.py b/python/paddle/fluid/tests/unittests/test_imperative_reinforcement.py index a831737d752..5e52815bbb3 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_reinforcement.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_reinforcement.py @@ -25,7 +25,6 @@ from paddle.fluid.framework import _test_eager_guard class Policy(fluid.dygraph.Layer): - def __init__(self, input_size): super(Policy, self).__init__() @@ -46,7 +45,6 @@ class Policy(fluid.dygraph.Layer): class TestImperativeMnist(unittest.TestCase): - def test_mnist_float32(self): seed = 90 epoch_num = 1 @@ -83,8 +81,9 @@ class TestImperativeMnist(unittest.TestCase): loss_probs = fluid.layers.elementwise_mul(dy_reward, loss_probs) loss = fluid.layers.reduce_sum(loss_probs) - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=policy.parameters()) + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=policy.parameters() + ) dy_param_init_value = {} @@ -108,29 +107,35 @@ class TestImperativeMnist(unittest.TestCase): with fluid.dygraph.guard(): with _test_eager_guard(): - eager_out, eager_param_init_value, eager_param_value = run_dygraph( - ) + ( + eager_out, + eager_param_init_value, + eager_param_value, + ) = run_dygraph() with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) policy = Policy(input_size=4) st_sgd = SGDOptimizer(learning_rate=1e-3) - st_state = fluid.layers.data(name='st_state', - shape=[4], - dtype='float32') - st_reward = fluid.layers.data(name='st_reward', - shape=[1], - dtype='float32') - st_mask = fluid.layers.data(name='st_mask', - shape=[2], - dtype='float32') + st_state = fluid.layers.data( + name='st_state', shape=[4], dtype='float32' + ) + st_reward = fluid.layers.data( + name='st_reward', shape=[1], dtype='float32' + ) + st_mask = fluid.layers.data( + name='st_mask', shape=[2], dtype='float32' + ) st_loss_probs = policy(st_state) @@ -139,7 +144,8 @@ class TestImperativeMnist(unittest.TestCase): st_loss_probs = fluid.layers.reduce_sum(st_loss_probs, dim=-1) st_loss_probs = fluid.layers.elementwise_mul( - st_reward, st_loss_probs) + st_reward, st_loss_probs + ) st_loss = fluid.layers.reduce_sum(st_loss_probs) st_sgd.minimize(st_loss) @@ -150,8 +156,10 @@ class TestImperativeMnist(unittest.TestCase): for param in policy.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -159,13 +167,11 @@ class TestImperativeMnist(unittest.TestCase): fetch_list = [st_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "st_state": state, - "st_reward": reward, - "st_mask": mask - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"st_state": state, "st_reward": reward, "st_mask": mask}, + fetch_list=fetch_list, + ) static_param_value = {} static_out = out[0] diff --git a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py index 55a51347283..657246795f7 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py @@ -26,7 +26,7 @@ from utils import DyGraphProgramDescTracerTestHelper, is_equal_program from paddle.fluid.dygraph import TracedLayer from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph -#NOTE(zhiqiu): run with FLAGS_cudnn_deterministic=1 +# NOTE(zhiqiu): run with FLAGS_cudnn_deterministic=1 batch_size = 8 train_parameters = { @@ -37,7 +37,7 @@ train_parameters = { "name": "piecewise_decay", "batch_size": batch_size, "epochs": [30, 60, 90], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "batch_size": batch_size, "lr": 0.1, @@ -60,8 +60,9 @@ def optimizer_setting(params, parameter_list=None): lr = [] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] if fluid._non_static_mode(): - optimizer = fluid.optimizer.SGD(learning_rate=0.01, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.01, parameter_list=parameter_list + ) else: optimizer = fluid.optimizer.SGD(learning_rate=0.01) # TODO(minqiyang): Add learning rate scheduler support to dygraph mode @@ -76,26 +77,29 @@ def optimizer_setting(params, parameter_list=None): class ConvBNLayer(fluid.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None, - use_cudnn=False): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + use_cudnn=False, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=False, - use_cudnn=use_cudnn) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False, + use_cudnn=use_cudnn, + ) self._batch_norm = BatchNorm(num_filters, act=act) @@ -107,38 +111,42 @@ class ConvBNLayer(fluid.Layer): class BottleneckBlock(fluid.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - shortcut=True, - use_cudnn=False): + def __init__( + self, num_channels, num_filters, stride, shortcut=True, use_cudnn=False + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu', - use_cudnn=use_cudnn) - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - act='relu', - use_cudnn=use_cudnn) - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act=None, - use_cudnn=use_cudnn) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu', + use_cudnn=use_cudnn, + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu', + use_cudnn=use_cudnn, + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act=None, + use_cudnn=use_cudnn, + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride, - use_cudnn=use_cudnn) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + use_cudnn=use_cudnn, + ) self.shortcut = shortcut @@ -159,14 +167,16 @@ class BottleneckBlock(fluid.Layer): class ResNet(fluid.Layer): - def __init__(self, layers=50, class_dim=102, use_cudnn=True): super(ResNet, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: depth = [3, 4, 6, 3] @@ -177,16 +187,17 @@ class ResNet(fluid.Layer): num_channels = [64, 256, 512, 1024] num_filters = [64, 128, 256, 512] - self.conv = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu', - use_cudnn=use_cudnn) - self.pool2d_max = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + use_cudnn=use_cudnn, + ) + self.pool2d_max = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] for block in range(len(depth)): @@ -194,22 +205,27 @@ class ResNet(fluid.Layer): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels[block] - if i == 0 else num_filters[block] * 4, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - shortcut=shortcut, - use_cudnn=use_cudnn)) + BottleneckBlock( + num_channels=num_channels[block] + if i == 0 + else num_filters[block] * 4, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + shortcut=shortcut, + use_cudnn=use_cudnn, + ), + ) self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) self.pool2d_avg_output = num_filters[-1] * 4 * 1 * 1 import math + stdv = 1.0 / math.sqrt(2048 * 1.0) self.out = Linear( @@ -217,7 +233,9 @@ class ResNet(fluid.Layer): class_dim, act='softmax', param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) def forward(self, inputs): y = self.conv(inputs) @@ -231,9 +249,7 @@ class ResNet(fluid.Layer): class TestDygraphResnet(unittest.TestCase): - def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): doc = np.array(item[0]).reshape(3, 224, 224) @@ -255,13 +271,15 @@ class TestDygraphResnet(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) resnet = ResNet() - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) dy_param_init_value = {} for param in resnet.parameters(): @@ -274,10 +292,14 @@ class TestDygraphResnet(unittest.TestCase): if batch_id >= batch_num: break - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -288,10 +310,12 @@ class TestDygraphResnet(unittest.TestCase): out, traced_layer = TracedLayer.trace(resnet, img) if program is not None: self.assertTrue( - is_equal_program(program, traced_layer.program)) + is_equal_program(program, traced_layer.program) + ) traced_layer.save_inference_model( - './infer_imperative_resnet') + './infer_imperative_resnet' + ) program = traced_layer.program else: @@ -322,9 +346,11 @@ class TestDygraphResnet(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + core.grad_var_suffix() + ] = np_array optimizer.minimize(avg_loss) resnet.clear_gradients() @@ -337,8 +363,11 @@ class TestDygraphResnet(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) resnet = ResNet() optimizer = optimizer_setting(train_parameters) @@ -346,11 +375,12 @@ class TestDygraphResnet(unittest.TestCase): np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) - img = fluid.layers.data(name='pixel', - shape=[3, 224, 224], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -365,11 +395,14 @@ class TestDygraphResnet(unittest.TestCase): static_param_name_list.append(param.name) for param in resnet.parameters(): if param.trainable: - static_grad_name_list.append(param.name + - core.grad_var_suffix()) + static_grad_name_list.append( + param.name + core.grad_var_suffix() + ) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -379,9 +412,13 @@ class TestDygraphResnet(unittest.TestCase): break static_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([batch_size, 1]) + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([batch_size, 1]) + ) if traced_layer is not None: traced_layer([static_x_data]) @@ -389,26 +426,30 @@ class TestDygraphResnet(unittest.TestCase): fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) fetch_list.extend(static_grad_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_grad_value = {} static_out = out[0] param_start_pos = 1 grad_start_pos = len(static_param_name_list) + param_start_pos - for i in range(param_start_pos, - len(static_param_name_list) + param_start_pos): - static_param_value[static_param_name_list[ - i - param_start_pos]] = out[i] - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + for i in range( + param_start_pos, + len(static_param_name_list) + param_start_pos, + ): + static_param_value[ + static_param_name_list[i - param_start_pos] + ] = out[i] + for i in range( + grad_start_pos, len(static_grad_name_list) + grad_start_pos + ): + static_grad_value[ + static_grad_name_list[i - grad_start_pos] + ] = out[i] print("static", static_out) print("dygraph", dy_out) @@ -417,9 +458,9 @@ class TestDygraphResnet(unittest.TestCase): self.assertEqual(len(dy_param_init_value), len(static_param_init_value)) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any())) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py index e28f2c14086..9e203092dc7 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py @@ -32,7 +32,7 @@ train_parameters = { "name": "piecewise_decay", "batch_size": batch_size, "epochs": [30, 60, 90], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "batch_size": batch_size, "lr": 0.1, @@ -55,8 +55,9 @@ def optimizer_setting(params, parameter_list=None): lr = [] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] if fluid._non_static_mode(): - optimizer = fluid.optimizer.SGD(learning_rate=0.01, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.01, parameter_list=parameter_list + ) else: optimizer = fluid.optimizer.SGD(learning_rate=0.01) # TODO(minqiyang): Add learning rate scheduler support to dygraph mode @@ -71,7 +72,6 @@ def optimizer_setting(params, parameter_list=None): class TestDygraphResnetSortGradient(unittest.TestCase): - def func_test_resnet_sort_gradient_float32(self): seed = 90 @@ -83,14 +83,17 @@ class TestDygraphResnetSortGradient(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) resnet = ResNet() - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) import random + random.seed = seed train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) dy_param_init_value = {} for param in resnet.parameters(): @@ -100,10 +103,14 @@ class TestDygraphResnetSortGradient(unittest.TestCase): if batch_id >= batch_num: break - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -126,9 +133,11 @@ class TestDygraphResnetSortGradient(unittest.TestCase): for param in resnet.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + core.grad_var_suffix() + ] = np_array optimizer.minimize(avg_loss) resnet.clear_gradients() @@ -141,22 +150,27 @@ class TestDygraphResnetSortGradient(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) resnet = ResNet() optimizer = optimizer_setting(train_parameters) np.random.seed(seed) import random + random.seed = seed train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) - img = fluid.layers.data(name='pixel', - shape=[3, 224, 224], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -171,11 +185,14 @@ class TestDygraphResnetSortGradient(unittest.TestCase): static_param_name_list.append(param.name) for param in resnet.parameters(): if param.trainable: - static_grad_name_list.append(param.name + - core.grad_var_suffix()) + static_grad_name_list.append( + param.name + core.grad_var_suffix() + ) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -185,42 +202,50 @@ class TestDygraphResnetSortGradient(unittest.TestCase): break static_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([batch_size, 1]) + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([batch_size, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) fetch_list.extend(static_grad_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_grad_value = {} static_out = out[0] param_start_pos = 1 grad_start_pos = len(static_param_name_list) + param_start_pos - for i in range(param_start_pos, - len(static_param_name_list) + param_start_pos): - static_param_value[static_param_name_list[ - i - param_start_pos]] = out[i] - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + for i in range( + param_start_pos, + len(static_param_name_list) + param_start_pos, + ): + static_param_value[ + static_param_name_list[i - param_start_pos] + ] = out[i] + for i in range( + grad_start_pos, len(static_grad_name_list) + grad_start_pos + ): + static_grad_value[ + static_grad_name_list[i - grad_start_pos] + ] = out[i] np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) self.assertEqual(len(dy_param_init_value), len(static_param_init_value)) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any())) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_save_load.py b/python/paddle/fluid/tests/unittests/test_imperative_save_load.py index 7933d4f1545..55cc00b12ab 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_save_load.py @@ -26,13 +26,9 @@ from paddle.fluid.framework import _test_eager_guard class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -51,19 +47,26 @@ class SimpleLSTMRNN(fluid.Layer): weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -71,29 +74,29 @@ class SimpleLSTMRNN(fluid.Layer): self.hidden_array = [] for i in range(self._num_layers): - pre_hidden = fluid.layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = fluid.layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = fluid.layers.reshape(pre_hidden, - shape=[-1, self._hidden_size]) - pre_cell = fluid.layers.reshape(pre_cell, - shape=[-1, self._hidden_size]) + pre_hidden = fluid.layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = fluid.layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = fluid.layers.reshape( + pre_hidden, shape=[-1, self._hidden_size] + ) + pre_cell = fluid.layers.reshape( + pre_cell, shape=[-1, self._hidden_size] + ) self.hidden_array.append(pre_hidden) self.cell_array.append(pre_cell) res = [] for index in range(self._num_steps): - self._input = fluid.layers.slice(input_embedding, - axes=[1], - starts=[index], - ends=[index + 1]) - self._input = fluid.layers.reshape(self._input, - shape=[-1, self._hidden_size]) + self._input = fluid.layers.slice( + input_embedding, axes=[1], starts=[index], ends=[index + 1] + ) + self._input = fluid.layers.reshape( + self._input, shape=[-1, self._hidden_size] + ) for k in range(self._num_layers): pre_hidden = self.hidden_array[k] pre_cell = self.cell_array[k] @@ -104,11 +107,12 @@ class SimpleLSTMRNN(fluid.Layer): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) self.hidden_array[k] = m self.cell_array[k] = c @@ -118,32 +122,38 @@ class SimpleLSTMRNN(fluid.Layer): self._input = fluid.layers.dropout( self._input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append( - fluid.layers.reshape(self._input, - shape=[1, -1, self._hidden_size])) + fluid.layers.reshape( + self._input, shape=[1, -1, self._hidden_size] + ) + ) real_res = fluid.layers.concat(res, 0) real_res = fluid.layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = fluid.layers.concat(self.hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(self.cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -151,11 +161,13 @@ class PtbModel(fluid.Layer): self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', @@ -163,48 +175,62 @@ class PtbModel(fluid.Layer): param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.drop_out, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) rnn_out = fluid.layers.reshape( - rnn_out, shape=[-1, self.num_steps, self.hidden_size]) + rnn_out, shape=[-1, self.num_steps, self.hidden_size] + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -213,7 +239,6 @@ class PtbModel(fluid.Layer): class TestDygraphPtbRnn(unittest.TestCase): - def func_setUp(self): seed = 90 hidden_size = 10 @@ -228,11 +253,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -242,11 +269,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -258,15 +291,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -312,11 +348,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -326,11 +364,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -342,15 +386,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -382,8 +429,9 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -420,11 +468,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -434,11 +484,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -450,15 +506,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -487,8 +546,9 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -525,11 +585,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -539,11 +601,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -555,15 +623,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -596,8 +667,9 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -634,18 +706,25 @@ class TestDygraphPtbRnn(unittest.TestCase): with fluid.dygraph.guard(): # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) - - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=0.0, - beta1=0.8, - beta2=0.6, - parameter_list=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) + + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=0.0, + beta1=0.8, + beta2=0.6, + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -660,15 +739,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -677,15 +759,18 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) state_dict = ptb_model.state_dict() @@ -709,11 +794,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [0.0] @@ -724,12 +811,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 0.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=0.0, - beta1=0.8, - beta2=0.6, - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=0.0, + beta1=0.8, + beta2=0.6, + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -745,15 +837,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -762,15 +857,18 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) # check parameter @@ -797,11 +895,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [0.0] @@ -812,13 +912,19 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 0.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - beta1=0.8, - beta2=0.6, - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + beta1=0.8, + beta2=0.6, + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -844,15 +950,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -861,15 +970,18 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) # check parameter @@ -888,15 +1000,18 @@ class TestDygraphPtbRnn(unittest.TestCase): fluid.save_dygraph(state_dict, os.path.join('saved_dy', 'emb_dy')) para_state_dict, opti_state_dict = fluid.load_dygraph( - os.path.join('saved_dy', 'emb_dy')) + os.path.join('saved_dy', 'emb_dy') + ) self.assertTrue(opti_state_dict == None) para_state_dict, opti_state_dict = fluid.load_dygraph( - os.path.join('saved_dy', 'emb_dy.pdparams')) + os.path.join('saved_dy', 'emb_dy.pdparams') + ) para_state_dict, opti_state_dict = fluid.load_dygraph( - os.path.join('saved_dy', 'emb_dy.pdopt')) + os.path.join('saved_dy', 'emb_dy.pdopt') + ) def func_test_load_compatible_with_keep_name_table(self): with fluid.dygraph.guard(): @@ -905,7 +1020,8 @@ class TestDygraphPtbRnn(unittest.TestCase): fluid.save_dygraph(state_dict, os.path.join('saved_dy', 'emb_dy')) para_state_dict, opti_state_dict = fluid.load_dygraph( - os.path.join('saved_dy', 'emb_dy'), keep_name_table=True) + os.path.join('saved_dy', 'emb_dy'), keep_name_table=True + ) self.assertTrue(para_state_dict != None) self.assertTrue(opti_state_dict == None) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py b/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py index 0e051381adf..795870e3d01 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py @@ -28,13 +28,9 @@ import tempfile class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -53,19 +49,26 @@ class SimpleLSTMRNN(fluid.Layer): weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -73,29 +76,29 @@ class SimpleLSTMRNN(fluid.Layer): self.hidden_array = [] for i in range(self._num_layers): - pre_hidden = fluid.layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = fluid.layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = fluid.layers.reshape(pre_hidden, - shape=[-1, self._hidden_size]) - pre_cell = fluid.layers.reshape(pre_cell, - shape=[-1, self._hidden_size]) + pre_hidden = fluid.layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = fluid.layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = fluid.layers.reshape( + pre_hidden, shape=[-1, self._hidden_size] + ) + pre_cell = fluid.layers.reshape( + pre_cell, shape=[-1, self._hidden_size] + ) self.hidden_array.append(pre_hidden) self.cell_array.append(pre_cell) res = [] for index in range(self._num_steps): - self._input = fluid.layers.slice(input_embedding, - axes=[1], - starts=[index], - ends=[index + 1]) - self._input = fluid.layers.reshape(self._input, - shape=[-1, self._hidden_size]) + self._input = fluid.layers.slice( + input_embedding, axes=[1], starts=[index], ends=[index + 1] + ) + self._input = fluid.layers.reshape( + self._input, shape=[-1, self._hidden_size] + ) for k in range(self._num_layers): pre_hidden = self.hidden_array[k] pre_cell = self.cell_array[k] @@ -106,11 +109,12 @@ class SimpleLSTMRNN(fluid.Layer): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) self.hidden_array[k] = m self.cell_array[k] = c @@ -120,32 +124,38 @@ class SimpleLSTMRNN(fluid.Layer): self._input = fluid.layers.dropout( self._input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append( - fluid.layers.reshape(self._input, - shape=[1, -1, self._hidden_size])) + fluid.layers.reshape( + self._input, shape=[1, -1, self._hidden_size] + ) + ) real_res = fluid.layers.concat(res, 0) real_res = fluid.layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = fluid.layers.concat(self.hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(self.cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -153,11 +163,13 @@ class PtbModel(fluid.Layer): self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', @@ -165,48 +177,62 @@ class PtbModel(fluid.Layer): param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.drop_out, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) rnn_out = fluid.layers.reshape( - rnn_out, shape=[-1, self.num_steps, self.hidden_size]) + rnn_out, shape=[-1, self.num_steps, self.hidden_size] + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -215,7 +241,6 @@ class PtbModel(fluid.Layer): class TestDygraphPtbRnn(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -236,11 +261,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -250,12 +277,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, parameters=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -267,15 +299,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -298,8 +333,10 @@ class TestDygraphPtbRnn(unittest.TestCase): else: self.base_opti[k] = v - paddle.save(self.opti_dict, - os.path.join(self.temp_dir.name, "test_dy_v2.pdopt")) + paddle.save( + self.opti_dict, + os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"), + ) self.state_dict = ptb_model.state_dict() @@ -308,8 +345,10 @@ class TestDygraphPtbRnn(unittest.TestCase): np_t = v.numpy() self.model_base[k] = np_t - paddle.save(self.state_dict, - os.path.join(self.temp_dir.name, "test_dy_v2.pdparams")) + paddle.save( + self.state_dict, + os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"), + ) def func_testLoadAndSetVarBase(self): seed = 90 @@ -325,11 +364,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -339,12 +380,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, parameters=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -356,15 +402,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -388,16 +437,19 @@ class TestDygraphPtbRnn(unittest.TestCase): self.assertTrue(np.sum(np.abs(v.numpy())) == 0) para_state_dict = paddle.load( - os.path.join(self.temp_dir.name, "test_dy_v2.pdparams")) + os.path.join(self.temp_dir.name, "test_dy_v2.pdparams") + ) opti_state_dict = paddle.load( - os.path.join(self.temp_dir.name, "test_dy_v2.pdopt")) + os.path.join(self.temp_dir.name, "test_dy_v2.pdopt") + ) adam.set_state_dict(opti_state_dict) opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -434,11 +486,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -448,12 +502,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, parameters=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -465,15 +524,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -503,8 +565,9 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -541,11 +604,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -555,12 +620,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, parameters=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -572,15 +642,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -614,8 +687,9 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -654,18 +728,25 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) - - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=0.0, - beta1=0.8, - beta2=0.6, - parameters=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) + + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=0.0, + beta1=0.8, + beta2=0.6, + parameters=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -680,15 +761,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -697,15 +781,18 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) state_dict = ptb_model.state_dict() @@ -729,11 +816,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [0.0] @@ -744,12 +833,17 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 0.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=0.0, - beta1=0.8, - beta2=0.6, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=0.0, + beta1=0.8, + beta2=0.6, + parameters=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -757,7 +851,8 @@ class TestDygraphPtbRnn(unittest.TestCase): last_cell = None state_dict, opti_dict = fluid.load_dygraph( - os.path.join(self.temp_dir.name, "test_dy_v2")) + os.path.join(self.temp_dir.name, "test_dy_v2") + ) adam.set_state_dict(opti_dict) ptb_model.set_dict(state_dict) @@ -766,15 +861,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -783,15 +881,18 @@ class TestDygraphPtbRnn(unittest.TestCase): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) # check parameter @@ -817,11 +918,13 @@ class TestDygraphPtbRnn(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [0.0] @@ -832,14 +935,20 @@ class TestDygraphPtbRnn(unittest.TestCase): new_lr = 0.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - beta1=0.8, - beta2=0.6, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, + beta1=0.8, + beta2=0.6, + parameters=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -865,15 +974,18 @@ class TestDygraphPtbRnn(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() scheduler.step() @@ -884,14 +996,17 @@ class TestDygraphPtbRnn(unittest.TestCase): for k, v in opti_dict.items(): if k == "LR_Scheduler": np.testing.assert_array_equal( - v['last_epoch'], self.base_opti[k]['last_epoch'] + 1) + v['last_epoch'], self.base_opti[k]['last_epoch'] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) # check parameter @@ -909,10 +1024,12 @@ class TestDygraphPtbRnn(unittest.TestCase): state_dict = emb.state_dict() paddle.save( state_dict, - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + ) para_state_dict = paddle.load( - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams') + ) def func_test_no_state_in_input_dict(self): with fluid.dygraph.guard(): @@ -920,10 +1037,12 @@ class TestDygraphPtbRnn(unittest.TestCase): state_dict = emb.state_dict() paddle.save( state_dict, - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + ) para_state_dict = paddle.load( - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams') + ) para_state_dict.pop('weight') emb.set_state_dict(para_state_dict) @@ -934,14 +1053,16 @@ class TestDygraphPtbRnn(unittest.TestCase): state_dict = emb.state_dict() paddle.save( state_dict, - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + ) - para_state_dict = paddle.load(os.path.join(self.temp_dir.name, - 'saved_dy', - 'emb_dy.pdparams'), - return_numpy=True) + para_state_dict = paddle.load( + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + return_numpy=True, + ) para_state_dict['weight'] = np.expand_dims( - para_state_dict['weight'], axis=-1) + para_state_dict['weight'], axis=-1 + ) emb.set_state_dict(para_state_dict) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py b/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py index c04f3adcadd..1f995817c0f 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py @@ -35,7 +35,7 @@ train_parameters = { "name": "piecewise_decay", "batch_size": batch_size, "epochs": [30, 60, 90], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "batch_size": batch_size, "lr": 0.1, @@ -51,15 +51,16 @@ def optimizer_setting(params, parameter_list=None): else: total_images = params["total_images"] # TODO(Yancey1989): using lr decay if it is ready. - #batch_size = ls["batch_size"] - #step = int(total_images / batch_size + 1) + # batch_size = ls["batch_size"] + # step = int(total_images / batch_size + 1) - #bd = [step * e for e in ls["epochs"]] - #base_lr = params["lr"] - #lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] + # bd = [step * e for e in ls["epochs"]] + # base_lr = params["lr"] + # lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] if fluid._non_static_mode(): - optimizer = fluid.optimizer.SGD(learning_rate=0.01, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.01, parameter_list=parameter_list + ) else: optimizer = fluid.optimizer.SGD(learning_rate=0.01) @@ -67,24 +68,27 @@ def optimizer_setting(params, parameter_list=None): class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=None) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=None, + ) self._batch_norm = BatchNorm(num_filters, act=act) @@ -96,7 +100,6 @@ class ConvBNLayer(fluid.dygraph.Layer): class SqueezeExcitation(fluid.dygraph.Layer): - def __init__(self, num_channels, reduction_ratio): super(SqueezeExcitation, self).__init__() @@ -105,15 +108,19 @@ class SqueezeExcitation(fluid.dygraph.Layer): self._squeeze = Linear( num_channels, num_channels // reduction_ratio, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - act='relu') + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + act='relu', + ) self._excitation = Linear( num_channels // reduction_ratio, num_channels, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - act='sigmoid') + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + act='sigmoid', + ) def forward(self, input): y = self._pool(input) @@ -125,37 +132,45 @@ class SqueezeExcitation(fluid.dygraph.Layer): class BottleneckBlock(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - cardinality, - reduction_ratio, - shortcut=True): + def __init__( + self, + num_channels, + num_filters, + stride, + cardinality, + reduction_ratio, + shortcut=True, + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1) - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - groups=cardinality) - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act='relu') - - self.scale = SqueezeExcitation(num_channels=num_filters * 4, - reduction_ratio=reduction_ratio) + self.conv0 = ConvBNLayer( + num_channels=num_channels, num_filters=num_filters, filter_size=1 + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act='relu', + ) + + self.scale = SqueezeExcitation( + num_channels=num_filters * 4, reduction_ratio=reduction_ratio + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -180,67 +195,76 @@ class BottleneckBlock(fluid.dygraph.Layer): class SeResNeXt(fluid.dygraph.Layer): - def __init__(self, layers=50, class_dim=102): super(SeResNeXt, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 101: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 23, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 152: cardinality = 64 reduction_ratio = 16 depth = [3, 8, 36, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - self.conv1 = ConvBNLayer(num_channels=64, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - self.conv2 = ConvBNLayer(num_channels=64, - num_filters=128, - filter_size=3, - stride=1, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=3, + stride=2, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=64, + num_filters=64, + filter_size=3, + stride=2, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=64, + num_filters=128, + filter_size=3, + stride=1, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] num_channels = 64 @@ -251,20 +275,24 @@ class SeResNeXt(fluid.dygraph.Layer): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=cardinality, - reduction_ratio=reduction_ratio, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio, + shortcut=shortcut, + ), + ) num_channels = bottleneck_block._num_channels_out self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) import math + stdv = 1.0 / math.sqrt(2048 * 1.0) self.pool2d_avg_output = num_filters[-1] * 4 * 1 * 1 @@ -274,7 +302,9 @@ class SeResNeXt(fluid.dygraph.Layer): class_dim, act='softmax', param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) def forward(self, inputs): if self.layers == 50 or self.layers == 101: @@ -295,9 +325,7 @@ class SeResNeXt(fluid.dygraph.Layer): class TestImperativeResneXt(unittest.TestCase): - def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): doc = np.array(item[0]).reshape(3, 224, 224) @@ -319,16 +347,21 @@ class TestImperativeResneXt(unittest.TestCase): se_resnext = SeResNeXt() optimizer = optimizer_setting( - train_parameters, parameter_list=se_resnext.parameters()) + train_parameters, parameter_list=se_resnext.parameters() + ) np.random.seed(seed) batch_py_reader = fluid.io.PyReader(capacity=1) batch_py_reader.decorate_sample_list_generator( - paddle.batch(self.reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True), - places=fluid.CPUPlace()) + paddle.batch( + self.reader_decorator( + paddle.dataset.flowers.train(use_xmap=False) + ), + batch_size=batch_size, + drop_last=True, + ), + places=fluid.CPUPlace(), + ) dy_param_init_value = {} for param in se_resnext.parameters(): @@ -346,8 +379,9 @@ class TestImperativeResneXt(unittest.TestCase): out = se_resnext(img) softmax_out = fluid.layers.softmax(out, use_cudnn=False) - loss = fluid.layers.cross_entropy(input=softmax_out, - label=label) + loss = fluid.layers.cross_entropy( + input=softmax_out, label=label + ) avg_loss = paddle.mean(x=loss) dy_out = avg_loss.numpy() @@ -362,9 +396,11 @@ class TestImperativeResneXt(unittest.TestCase): for param in se_resnext.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + core.grad_var_suffix() + ] = np_array optimizer.minimize(avg_loss) se_resnext.clear_gradients() @@ -373,23 +409,39 @@ class TestImperativeResneXt(unittest.TestCase): for param in se_resnext.parameters(): dy_param_value[param.name] = param.numpy() - return dy_out, dy_param_init_value, dy_param_value, dy_grad_value + return ( + dy_out, + dy_param_init_value, + dy_param_value, + dy_grad_value, + ) with fluid.dygraph.guard(): - dy_out, dy_param_init_value, dy_param_value, dy_grad_value = run_dygraph( - ) + ( + dy_out, + dy_param_init_value, + dy_param_value, + dy_grad_value, + ) = run_dygraph() with fluid.dygraph.guard(): with _test_eager_guard(): - eager_out, eager_param_init_value, eager_param_value, eager_grad_value = run_dygraph( - ) + ( + eager_out, + eager_param_init_value, + eager_param_value, + eager_grad_value, + ) = run_dygraph() with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) se_resnext = SeResNeXt() optimizer = optimizer_setting(train_parameters) @@ -398,11 +450,12 @@ class TestImperativeResneXt(unittest.TestCase): train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size, - drop_last=True) + drop_last=True, + ) - img = fluid.layers.data(name='pixel', - shape=[3, 224, 224], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = se_resnext(img) softmax_out = fluid.layers.softmax(out, use_cudnn=False) @@ -418,11 +471,14 @@ class TestImperativeResneXt(unittest.TestCase): static_param_name_list.append(param.name) for param in se_resnext.parameters(): if param.trainable: - static_grad_name_list.append(param.name + - core.grad_var_suffix()) + static_grad_name_list.append( + param.name + core.grad_var_suffix() + ) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -431,47 +487,54 @@ class TestImperativeResneXt(unittest.TestCase): if batch_id >= batch_num and batch_num != -1: break - static_x_data = np.array([ - x[0].reshape(3, 224, 224) for x in data - ]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape( - [batch_size, 1]) + static_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([batch_size, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) fetch_list.extend(static_grad_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_grad_value = {} static_out = out[0] param_start_pos = 1 - grad_start_pos = len( - static_param_name_list) + param_start_pos + grad_start_pos = ( + len(static_param_name_list) + param_start_pos + ) + for i in range( + param_start_pos, + len(static_param_name_list) + param_start_pos, + ): + static_param_value[ + static_param_name_list[i - param_start_pos] + ] = out[i] for i in range( - param_start_pos, - len(static_param_name_list) + param_start_pos): - static_param_value[static_param_name_list[ - i - param_start_pos]] = out[i] - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + grad_start_pos, + len(static_grad_name_list) + grad_start_pos, + ): + static_grad_value[ + static_grad_name_list[i - grad_start_pos] + ] = out[i] np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) self.assertEqual(len(dy_param_init_value), len(static_param_init_value)) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any())) @@ -491,13 +554,14 @@ class TestImperativeResneXt(unittest.TestCase): # check eager np.testing.assert_allclose(static_out, eager_out, rtol=1e-05) - self.assertEqual(len(eager_param_init_value), - len(static_param_init_value)) + self.assertEqual( + len(eager_param_init_value), len(static_param_init_value) + ) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - eager_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, eager_param_init_value[key], rtol=1e-05 + ) self.assertEqual(len(eager_grad_value), len(static_grad_value)) @@ -506,9 +570,9 @@ class TestImperativeResneXt(unittest.TestCase): self.assertEqual(len(eager_param_value), len(static_param_value)) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - eager_param_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, eager_param_value[key], rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_selected_rows.py b/python/paddle/fluid/tests/unittests/test_imperative_selected_rows.py index 602d2e07ce5..93a04a36115 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_selected_rows.py @@ -23,13 +23,14 @@ from paddle.fluid.framework import _test_eager_guard class SimpleNet(paddle.nn.Layer): - def __init__(self, vocab_size, hidden_size, dtype): super(SimpleNet, self).__init__() - self.emb = fluid.dygraph.Embedding(size=[vocab_size, hidden_size], - dtype=dtype, - param_attr='emb.w', - is_sparse=True) + self.emb = fluid.dygraph.Embedding( + size=[vocab_size, hidden_size], + dtype=dtype, + param_attr='emb.w', + is_sparse=True, + ) def forward(self, input): input_emb = self.emb(input) @@ -37,7 +38,6 @@ class SimpleNet(paddle.nn.Layer): class TestSimpleNet(unittest.TestCase): - def func_selectedrows_gradient1(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -48,16 +48,18 @@ class TestSimpleNet(unittest.TestCase): for sort_sum_gradient in [True, False]: paddle.disable_static(place) fluid.set_flags( - {'FLAGS_sort_sum_gradient': sort_sum_gradient}) + {'FLAGS_sort_sum_gradient': sort_sum_gradient} + ) # grad_clip = fluid.clip.GradientClipByGlobalNorm(5.0) input_word = np.array([[1, 2], [2, 1]]).astype('int64') input = paddle.to_tensor(input_word) simplenet = SimpleNet(20, 32, dtype) - adam = SGDOptimizer(learning_rate=0.001, - parameter_list=simplenet.parameters() - ) # grad_clip=grad_clip + adam = SGDOptimizer( + learning_rate=0.001, + parameter_list=simplenet.parameters(), + ) # grad_clip=grad_clip input_emb, emb = simplenet(input) self.assertTrue(emb.weight.gradient() is None) @@ -90,16 +92,19 @@ class TestSimpleNet(unittest.TestCase): for sort_sum_gradient in [True, False]: with fluid.dygraph.guard(place): fluid.set_flags( - {'FLAGS_sort_sum_gradient': sort_sum_gradient}) + {'FLAGS_sort_sum_gradient': sort_sum_gradient} + ) grad_clip = fluid.clip.GradientClipByGlobalNorm(5.0) input_word = np.array([[1, 2], [2, 1]]).astype('int64') input = to_variable(input_word) simplenet = SimpleNet(20, 32, "float32") - adam = SGDOptimizer(learning_rate=0.001, - parameter_list=simplenet.parameters(), - grad_clip=grad_clip) + adam = SGDOptimizer( + learning_rate=0.001, + parameter_list=simplenet.parameters(), + grad_clip=grad_clip, + ) input_emb, emb = simplenet(input) self.assertTrue(emb.weight.gradient() is None) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_selected_rows_to_lod_tensor.py b/python/paddle/fluid/tests/unittests/test_imperative_selected_rows_to_lod_tensor.py index 35d863b774b..023753322d1 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_selected_rows_to_lod_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_selected_rows_to_lod_tensor.py @@ -26,14 +26,15 @@ from paddle.fluid.framework import _test_eager_guard class SimpleNet(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype='float32'): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype='float32', + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -46,31 +47,40 @@ class SimpleNet(fluid.Layer): param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.hidden_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) fc = fluid.layers.matmul(x_emb, self.softmax_weight) fc = fluid.layers.elementwise_add(fc, self.softmax_bias) projection = fluid.layers.matmul( - fc, fluid.layers.transpose(self.embedding.weight, perm=[1, 0])) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + fc, fluid.layers.transpose(self.embedding.weight, perm=[1, 0]) + ) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -79,7 +89,6 @@ class SimpleNet(fluid.Layer): class TestDygraphSimpleNet(unittest.TestCase): - def func_simple_net(self): for is_sparse in [True, False]: dtype_list = ["float32"] @@ -113,21 +122,26 @@ class TestDygraphSimpleNet(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - simple_net = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse, - dtype=dtype) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=simple_net.parameters()) + simple_net = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + dtype=dtype, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, + parameter_list=simple_net.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None fluid.set_flags( - {'FLAGS_sort_sum_gradient': is_sort_sum_gradient}) + {'FLAGS_sort_sum_gradient': is_sort_sum_gradient} + ) for i in range(batch_num): x_data = np.arange(12).reshape(4, 3).astype('int64') @@ -154,17 +168,19 @@ class TestDygraphSimpleNet(unittest.TestCase): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - simple_net = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - is_sparse=is_sparse, - dtype=dtype) + simple_net = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + is_sparse=is_sparse, + dtype=dtype, + ) exe = fluid.Executor(place) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype=dtype) static_loss = simple_net(x, y) @@ -175,8 +191,10 @@ class TestDygraphSimpleNet(unittest.TestCase): for param in simple_net.parameters(): static_param_name_list.append(param.name) - out = exe.run(framework.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + framework.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_loss_value = None @@ -187,18 +205,18 @@ class TestDygraphSimpleNet(unittest.TestCase): y_data = y_data.reshape((-1, 1)) fetch_list = [static_loss] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_data, "y": y_data}, + fetch_list=fetch_list, + ) static_loss_value = out[0] if i == batch_num - 1: for k in range(3, len(out)): - static_param_updated[static_param_name_list[ - k - 1]] = out[k] + static_param_updated[ + static_param_name_list[k - 1] + ] = out[k] np.testing.assert_array_equal(static_loss_value, dy_loss_value) for key, value in static_param_init.items(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_signal_handler.py b/python/paddle/fluid/tests/unittests/test_imperative_signal_handler.py index 977cb9cc257..488500cb380 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_signal_handler.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_signal_handler.py @@ -38,9 +38,7 @@ def set_child_signal_handler(self, child_pid): class DygraphDataLoaderSingalHandler(unittest.TestCase): - def func_child_process_exit_with_error(self): - def __test_process__(): core._set_process_signal_handler() sys.exit(1) @@ -73,7 +71,6 @@ class DygraphDataLoaderSingalHandler(unittest.TestCase): self.func_child_process_exit_with_error() def func_child_process_killed_by_sigsegv(self): - def __test_process__(): core._set_process_signal_handler() os.kill(os.getpid(), signal.SIGSEGV) @@ -106,7 +103,6 @@ class DygraphDataLoaderSingalHandler(unittest.TestCase): self.func_child_process_killed_by_sigsegv() def func_child_process_killed_by_sigbus(self): - def __test_process__(): core._set_process_signal_handler() os.kill(os.getpid(), signal.SIGBUS) @@ -139,7 +135,6 @@ class DygraphDataLoaderSingalHandler(unittest.TestCase): self.func_child_process_killed_by_sigbus() def func_child_process_killed_by_sigterm(self): - def __test_process__(): core._set_process_signal_handler() time.sleep(10) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py index 4b394ba87fd..3a43709997e 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py @@ -24,7 +24,6 @@ if fluid.is_compiled_with_cuda(): class Config(object): - def __init__(self, place, sort_sum_gradient=True): self.place = place @@ -60,7 +59,6 @@ class Config(object): def create_mnist_dataset(cfg): - def create_target_label(label): return label # return (label + 1) % cfg.c_dim # fake label target @@ -107,7 +105,6 @@ def create_mnist_dataset(cfg): class InstanceNorm(fluid.dygraph.Layer): - def __init__(self, num_channels, epsilon=1e-5): super(InstanceNorm, self).__init__() self.epsilon = epsilon @@ -117,36 +114,40 @@ class InstanceNorm(fluid.dygraph.Layer): def forward(self, input): if fluid._non_static_mode(): - out, _, _ = _legacy_C_ops.instance_norm(input, self.scale, - self.bias, 'epsilon', - self.epsilon) + out, _, _ = _legacy_C_ops.instance_norm( + input, self.scale, self.bias, 'epsilon', self.epsilon + ) return out else: return fluid.layers.instance_norm( input, epsilon=self.epsilon, param_attr=fluid.ParamAttr(self.scale.name), - bias_attr=fluid.ParamAttr(self.bias.name)) + bias_attr=fluid.ParamAttr(self.bias.name), + ) class Conv2DLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters=64, - filter_size=7, - stride=1, - padding=0, - norm=None, - use_bias=False, - relufactor=None): + def __init__( + self, + num_channels, + num_filters=64, + filter_size=7, + stride=1, + padding=0, + norm=None, + use_bias=False, + relufactor=None, + ): super(Conv2DLayer, self).__init__() - self._conv = fluid.dygraph.Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=padding, - bias_attr=None if use_bias else False) + self._conv = fluid.dygraph.Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + bias_attr=None if use_bias else False, + ) if norm is not None: self._norm = InstanceNorm(num_filters) @@ -168,16 +169,17 @@ class Conv2DLayer(fluid.dygraph.Layer): class Deconv2DLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters=64, - filter_size=7, - stride=1, - padding=0, - norm=None, - use_bias=False, - relufactor=None): + def __init__( + self, + num_channels, + num_filters=64, + filter_size=7, + stride=1, + padding=0, + norm=None, + use_bias=False, + relufactor=None, + ): super(Deconv2DLayer, self).__init__() self._deconv = fluid.dygraph.Conv2DTranspose( @@ -186,7 +188,8 @@ class Deconv2DLayer(fluid.dygraph.Layer): filter_size=filter_size, stride=stride, padding=padding, - bias_attr=None if use_bias else False) + bias_attr=None if use_bias else False, + ) if norm is not None: self._norm = InstanceNorm(num_filters) @@ -208,24 +211,27 @@ class Deconv2DLayer(fluid.dygraph.Layer): class ResidualBlock(fluid.dygraph.Layer): - def __init__(self, num_channels, num_filters): super(ResidualBlock, self).__init__() - self._conv0 = Conv2DLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=3, - stride=1, - padding=1, - norm=True, - relufactor=0) - - self._conv1 = Conv2DLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=1, - padding=1, - norm=True, - relufactor=None) + self._conv0 = Conv2DLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=3, + stride=1, + padding=1, + norm=True, + relufactor=0, + ) + + self._conv1 = Conv2DLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=1, + padding=1, + norm=True, + relufactor=None, + ) def forward(self, input): conv0 = self._conv0(input) @@ -234,27 +240,30 @@ class ResidualBlock(fluid.dygraph.Layer): class Generator(fluid.dygraph.Layer): - def __init__(self, cfg, num_channels=3): super(Generator, self).__init__() - conv_base = Conv2DLayer(num_channels=cfg.c_dim + num_channels, - num_filters=cfg.g_base_dims, - filter_size=7, - stride=1, - padding=3, - norm=True, - relufactor=0) + conv_base = Conv2DLayer( + num_channels=cfg.c_dim + num_channels, + num_filters=cfg.g_base_dims, + filter_size=7, + stride=1, + padding=3, + norm=True, + relufactor=0, + ) sub_layers = [conv_base] cur_channels = cfg.g_base_dims for i in range(2): - sub_layer = Conv2DLayer(num_channels=cur_channels, - num_filters=cur_channels * 2, - filter_size=4, - stride=2, - padding=1, - norm=True, - relufactor=0) + sub_layer = Conv2DLayer( + num_channels=cur_channels, + num_filters=cur_channels * 2, + filter_size=4, + stride=2, + padding=1, + norm=True, + relufactor=0, + ) cur_channels *= 2 sub_layers.append(sub_layer) @@ -264,8 +273,9 @@ class Generator(fluid.dygraph.Layer): repeat_num = cfg.g_repeat_num sub_layers = [] for i in range(repeat_num): - res_block = ResidualBlock(num_channels=cur_channels, - num_filters=cfg.g_base_dims * 4) + res_block = ResidualBlock( + num_channels=cur_channels, num_filters=cfg.g_base_dims * 4 + ) sub_layers.append(res_block) self._res_block = fluid.dygraph.Sequential(*sub_layers) @@ -273,32 +283,38 @@ class Generator(fluid.dygraph.Layer): cur_channels = cfg.g_base_dims * 4 sub_layers = [] for i in range(2): - rate = 2**(1 - i) - deconv = Deconv2DLayer(num_channels=cur_channels, - num_filters=cfg.g_base_dims * rate, - filter_size=4, - stride=2, - padding=1, - relufactor=0, - norm=True) + rate = 2 ** (1 - i) + deconv = Deconv2DLayer( + num_channels=cur_channels, + num_filters=cfg.g_base_dims * rate, + filter_size=4, + stride=2, + padding=1, + relufactor=0, + norm=True, + ) cur_channels = cfg.g_base_dims * rate sub_layers.append(deconv) self._deconv = fluid.dygraph.Sequential(*sub_layers) - self._conv1 = Conv2DLayer(num_channels=cur_channels, - num_filters=3, - filter_size=7, - stride=1, - padding=3, - relufactor=None) + self._conv1 = Conv2DLayer( + num_channels=cur_channels, + num_filters=3, + filter_size=7, + stride=1, + padding=3, + relufactor=None, + ) def forward(self, input, label_trg): shape = input.shape - label_trg_e = fluid.layers.reshape(label_trg, - [-1, label_trg.shape[1], 1, 1]) + label_trg_e = fluid.layers.reshape( + label_trg, [-1, label_trg.shape[1], 1, 1] + ) label_trg_e = fluid.layers.expand( - x=label_trg_e, expand_times=[1, 1, shape[2], shape[3]]) + x=label_trg_e, expand_times=[1, 1, shape[2], shape[3]] + ) input1 = fluid.layers.concat([input, label_trg_e], 1) @@ -311,28 +327,31 @@ class Generator(fluid.dygraph.Layer): class Discriminator(fluid.dygraph.Layer): - def __init__(self, cfg, num_channels=3): super(Discriminator, self).__init__() cur_dim = cfg.d_base_dims - conv_base = Conv2DLayer(num_channels=num_channels, - num_filters=cur_dim, - filter_size=4, - stride=2, - padding=1, - relufactor=0.2) + conv_base = Conv2DLayer( + num_channels=num_channels, + num_filters=cur_dim, + filter_size=4, + stride=2, + padding=1, + relufactor=0.2, + ) repeat_num = cfg.d_repeat_num sub_layers = [conv_base] for i in range(1, repeat_num): - sub_layer = Conv2DLayer(num_channels=cur_dim, - num_filters=cur_dim * 2, - filter_size=4, - stride=2, - padding=1, - relufactor=0.2) + sub_layer = Conv2DLayer( + num_channels=cur_dim, + num_filters=cur_dim * 2, + filter_size=4, + stride=2, + padding=1, + relufactor=0.2, + ) cur_dim *= 2 sub_layers.append(sub_layer) @@ -340,15 +359,17 @@ class Discriminator(fluid.dygraph.Layer): kernel_size = int(cfg.image_size / np.power(2, repeat_num)) - self._conv1 = Conv2DLayer(num_channels=cur_dim, - num_filters=1, - filter_size=3, - stride=1, - padding=1) + self._conv1 = Conv2DLayer( + num_channels=cur_dim, + num_filters=1, + filter_size=3, + stride=1, + padding=1, + ) - self._conv2 = Conv2DLayer(num_channels=cur_dim, - num_filters=cfg.c_dim, - filter_size=kernel_size) + self._conv2 = Conv2DLayer( + num_channels=cur_dim, num_filters=cfg.c_dim, filter_size=kernel_size + ) def forward(self, input): conv = self._conv0(input) @@ -359,38 +380,41 @@ class Discriminator(fluid.dygraph.Layer): def loss_cls(cls, label, cfg): cls_shape = cls.shape - cls = fluid.layers.reshape(cls, - [-1, cls_shape[1] * cls_shape[2] * cls_shape[3]]) - return fluid.layers.reduce_sum( - fluid.layers.sigmoid_cross_entropy_with_logits(cls, - label)) / cfg.batch_size + cls = fluid.layers.reshape( + cls, [-1, cls_shape[1] * cls_shape[2] * cls_shape[3]] + ) + return ( + fluid.layers.reduce_sum( + fluid.layers.sigmoid_cross_entropy_with_logits(cls, label) + ) + / cfg.batch_size + ) def calc_gradients(outputs, inputs, no_grad_set): if fluid._non_static_mode(): - return fluid.dygraph.grad(outputs=outputs, - inputs=inputs, - no_grad_vars=no_grad_set, - create_graph=True) + return fluid.dygraph.grad( + outputs=outputs, + inputs=inputs, + no_grad_vars=no_grad_set, + create_graph=True, + ) else: - return fluid.gradients(targets=outputs, - inputs=inputs, - no_grad_set=no_grad_set) + return fluid.gradients( + targets=outputs, inputs=inputs, no_grad_set=no_grad_set + ) def gradient_penalty(f, real, fake, no_grad_set, cfg): - def _interpolate(a, b): shape = [a.shape[0]] - alpha = fluid.layers.uniform_random_batch_size_like(input=a, - shape=shape, - min=0.1, - max=1.0, - seed=cfg.seed) + alpha = fluid.layers.uniform_random_batch_size_like( + input=a, shape=shape, min=0.1, max=1.0, seed=cfg.seed + ) inner = fluid.layers.elementwise_mul( - b, 1.0 - alpha, axis=0) + fluid.layers.elementwise_mul( - a, alpha, axis=0) + b, 1.0 - alpha, axis=0 + ) + fluid.layers.elementwise_mul(a, alpha, axis=0) return inner x = _interpolate(real, fake) @@ -398,9 +422,9 @@ def gradient_penalty(f, real, fake, no_grad_set, cfg): if isinstance(pred, tuple): pred = pred[0] - gradient = calc_gradients(outputs=[pred], - inputs=[x], - no_grad_set=no_grad_set) + gradient = calc_gradients( + outputs=[pred], inputs=[x], no_grad_set=no_grad_set + ) if gradient is None: return None @@ -409,22 +433,26 @@ def gradient_penalty(f, real, fake, no_grad_set, cfg): grad_shape = gradient.shape gradient = fluid.layers.reshape( - gradient, [-1, grad_shape[1] * grad_shape[2] * grad_shape[3]]) + gradient, [-1, grad_shape[1] * grad_shape[2] * grad_shape[3]] + ) epsilon = 1e-16 norm = fluid.layers.sqrt( - fluid.layers.reduce_sum(fluid.layers.square(gradient), dim=1) + epsilon) + fluid.layers.reduce_sum(fluid.layers.square(gradient), dim=1) + epsilon + ) gp = fluid.layers.reduce_mean(fluid.layers.square(norm - 1.0)) return gp -def get_generator_loss(image_real, label_org, label_trg, generator, - discriminator, cfg): +def get_generator_loss( + image_real, label_org, label_trg, generator, discriminator, cfg +): fake_img = generator(image_real, label_trg) rec_img = generator(fake_img, label_org) g_loss_rec = fluid.layers.reduce_mean( - fluid.layers.abs(fluid.layers.elementwise_sub(image_real, rec_img))) + fluid.layers.abs(fluid.layers.elementwise_sub(image_real, rec_img)) + ) pred_fake, cls_fake = discriminator(fake_img) @@ -434,8 +462,9 @@ def get_generator_loss(image_real, label_org, label_trg, generator, return g_loss -def get_discriminator_loss(image_real, label_org, label_trg, generator, - discriminator, cfg): +def get_discriminator_loss( + image_real, label_org, label_trg, generator, discriminator, cfg +): fake_img = generator(image_real, label_trg) pred_real, cls_real = discriminator(image_real) pred_fake, _ = discriminator(fake_img) @@ -444,8 +473,13 @@ def get_discriminator_loss(image_real, label_org, label_trg, generator, d_loss_real = -paddle.mean(pred_real) d_loss = d_loss_real + d_loss_fake + d_loss_cls - d_loss_gp = gradient_penalty(discriminator, image_real, fake_img, - set(discriminator.parameters()), cfg) + d_loss_gp = gradient_penalty( + discriminator, + image_real, + fake_img, + set(discriminator.parameters()), + cfg, + ) if d_loss_gp is not None: d_loss += cfg.lambda_gp * d_loss_gp @@ -457,21 +491,22 @@ def build_optimizer(layer, cfg, loss=None): beta1 = 0.5 beta2 = 0.999 if fluid._non_static_mode(): - return fluid.optimizer.Adam(learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - parameter_list=layer.parameters()) + return fluid.optimizer.Adam( + learning_rate=learning_rate, + beta1=beta1, + beta2=beta2, + parameter_list=layer.parameters(), + ) else: - optimizer = fluid.optimizer.Adam(learning_rate=learning_rate, - beta1=beta1, - beta2=beta2) + optimizer = fluid.optimizer.Adam( + learning_rate=learning_rate, beta1=beta1, beta2=beta2 + ) optimizer.minimize(loss, parameter_list=layer.parameters()) return optimizer class DyGraphTrainModel(object): - def __init__(self, cfg): paddle.seed(1) paddle.framework.random._manual_program_seed(1) @@ -498,18 +533,28 @@ class DyGraphTrainModel(object): label_org = fluid.dygraph.to_variable(label_org) label_trg = fluid.dygraph.to_variable(label_trg) - g_loss = get_generator_loss(image_real, label_org, label_trg, - self.generator, self.discriminator, - self.cfg) + g_loss = get_generator_loss( + image_real, + label_org, + label_trg, + self.generator, + self.discriminator, + self.cfg, + ) g_loss.backward() if self.g_optimizer: self.g_optimizer.minimize(g_loss) self.clear_gradients() - d_loss = get_discriminator_loss(image_real, label_org, label_trg, - self.generator, self.discriminator, - self.cfg) + d_loss = get_discriminator_loss( + image_real, + label_org, + label_trg, + self.generator, + self.discriminator, + self.cfg, + ) d_loss.backward() if self.d_optimizer: self.d_optimizer.minimize(d_loss) @@ -520,7 +565,6 @@ class DyGraphTrainModel(object): class StaticGraphTrainModel(object): - def __init__(self, cfg): self.cfg = cfg @@ -528,13 +572,14 @@ class StaticGraphTrainModel(object): image_real = fluid.data( shape=[None, 3, cfg.image_size, cfg.image_size], dtype='float32', - name='image_real') - label_org = fluid.data(shape=[None, cfg.c_dim], - dtype='float32', - name='label_org') - label_trg = fluid.data(shape=[None, cfg.c_dim], - dtype='float32', - name='label_trg') + name='image_real', + ) + label_org = fluid.data( + shape=[None, cfg.c_dim], dtype='float32', name='label_org' + ) + label_trg = fluid.data( + shape=[None, cfg.c_dim], dtype='float32', name='label_trg' + ) return image_real, label_org, label_trg paddle.seed(cfg.seed) @@ -547,8 +592,14 @@ class StaticGraphTrainModel(object): image_real, label_org, label_trg = create_data_layer() generator = Generator(cfg) discriminator = Discriminator(cfg) - g_loss = get_generator_loss(image_real, label_org, label_trg, - generator, discriminator, cfg) + g_loss = get_generator_loss( + image_real, + label_org, + label_trg, + generator, + discriminator, + cfg, + ) build_optimizer(generator, cfg, loss=g_loss) self.dis_program = fluid.Program() @@ -558,9 +609,14 @@ class StaticGraphTrainModel(object): image_real, label_org, label_trg = create_data_layer() generator = Generator(cfg) discriminator = Discriminator(cfg) - d_loss = get_discriminator_loss(image_real, label_org, - label_trg, generator, - discriminator, cfg) + d_loss = get_discriminator_loss( + image_real, + label_org, + label_trg, + generator, + discriminator, + cfg, + ) build_optimizer(discriminator, cfg, loss=d_loss) self.executor = fluid.Executor(cfg.place) @@ -577,20 +633,19 @@ class StaticGraphTrainModel(object): feed = { 'image_real': image_real, 'label_org': label_org, - 'label_trg': label_trg + 'label_trg': label_trg, } with fluid.scope_guard(self.scope): - g_loss_val = self.executor.run(self.gen_program, - feed=feed, - fetch_list=[self.g_loss])[0] - d_loss_val = self.executor.run(self.dis_program, - feed=feed, - fetch_list=[self.d_loss])[0] + g_loss_val = self.executor.run( + self.gen_program, feed=feed, fetch_list=[self.g_loss] + )[0] + d_loss_val = self.executor.run( + self.dis_program, feed=feed, fetch_list=[self.d_loss] + )[0] return g_loss_val[0], d_loss_val[0] class TestStarGANWithGradientPenalty(unittest.TestCase): - def func_main(self): self.place_test(fluid.CPUPlace()) @@ -606,8 +661,9 @@ class TestStarGANWithGradientPenalty(unittest.TestCase): fluid_dygraph_loss = [] with fluid.dygraph.guard(cfg.place): fluid_dygraph_model = DyGraphTrainModel(cfg) - for batch_id, (image_real, label_org, - label_trg) in enumerate(dataset()): + for batch_id, (image_real, label_org, label_trg) in enumerate( + dataset() + ): loss = fluid_dygraph_model.run(image_real, label_org, label_trg) fluid_dygraph_loss.append(loss) @@ -615,15 +671,17 @@ class TestStarGANWithGradientPenalty(unittest.TestCase): with _test_eager_guard(): with fluid.dygraph.guard(cfg.place): eager_dygraph_model = DyGraphTrainModel(cfg) - for batch_id, (image_real, label_org, - label_trg) in enumerate(dataset()): - loss = eager_dygraph_model.run(image_real, label_org, - label_trg) + for batch_id, (image_real, label_org, label_trg) in enumerate( + dataset() + ): + loss = eager_dygraph_model.run( + image_real, label_org, label_trg + ) eager_dygraph_loss.append(loss) - for (g_loss_f, d_loss_f), (g_loss_e, - d_loss_e) in zip(fluid_dygraph_loss, - eager_dygraph_loss): + for (g_loss_f, d_loss_f), (g_loss_e, d_loss_e) in zip( + fluid_dygraph_loss, eager_dygraph_loss + ): self.assertEqual(g_loss_f, g_loss_e) self.assertEqual(d_loss_f, d_loss_e) @@ -632,7 +690,6 @@ class TestStarGANWithGradientPenalty(unittest.TestCase): class TestStarGANWithGradientPenaltyLegacy(unittest.TestCase): - def func_main(self): self.place_test(fluid.CPUPlace()) @@ -647,21 +704,24 @@ class TestStarGANWithGradientPenaltyLegacy(unittest.TestCase): static_graph_model = StaticGraphTrainModel(cfg) static_loss = [] - for batch_id, (image_real, label_org, - label_trg) in enumerate(dataset()): + for batch_id, (image_real, label_org, label_trg) in enumerate( + dataset() + ): loss = static_graph_model.run(image_real, label_org, label_trg) static_loss.append(loss) dygraph_loss = [] with fluid.dygraph.guard(cfg.place): dygraph_model = DyGraphTrainModel(cfg) - for batch_id, (image_real, label_org, - label_trg) in enumerate(dataset()): + for batch_id, (image_real, label_org, label_trg) in enumerate( + dataset() + ): loss = dygraph_model.run(image_real, label_org, label_trg) dygraph_loss.append(loss) - for (g_loss_s, d_loss_s), (g_loss_d, - d_loss_d) in zip(static_loss, dygraph_loss): + for (g_loss_s, d_loss_s), (g_loss_d, d_loss_d) in zip( + static_loss, dygraph_loss + ): self.assertEqual(g_loss_s, g_loss_d) self.assertEqual(d_loss_s, d_loss_d) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_static_runner_mnist.py b/python/paddle/fluid/tests/unittests/test_imperative_static_runner_mnist.py index cc204e6b3d4..825144cd16a 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_static_runner_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_static_runner_mnist.py @@ -25,19 +25,23 @@ LOADED_VAR_SUFFIX = ".load_0" def convolutional_neural_network(img): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') return prediction @@ -55,7 +59,6 @@ def static_train_net(img, label): class TestImperativeStaticModelRunnerMnist(unittest.TestCase): - def setUp(self): self.seed = 90 self.epoch_num = 1 @@ -63,7 +66,6 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): self.batch_num = 50 def reader_decorator(self, reader): - def _reader_impl(): for item in reader(): image = np.array(item[0]).reshape(1, 28, 28) @@ -77,44 +79,58 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): startup_program = fluid.default_startup_program() main_program = fluid.default_main_program() - img = fluid.data(name='img', - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 1, 28, 28], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') prediction, avg_loss = static_train_net(img, label) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) exe.run(startup_program) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=100), - batch_size=self.batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.mnist.train(), buf_size=100 + ), + batch_size=self.batch_size, + ) for _ in range(0, self.epoch_num): for batch_id, data in enumerate(train_reader()): - exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_loss]) + exe.run( + main_program, + feed=feeder.feed(data), + fetch_list=[avg_loss], + ) if batch_id > self.batch_num: break - fluid.io.save_inference_model(self.save_dirname, ["img"], - [prediction], - exe, - model_filename=self.model_filename, - params_filename=self.params_filename, - clip_extra=False) + fluid.io.save_inference_model( + self.save_dirname, + ["img"], + [prediction], + exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + clip_extra=False, + ) def load_and_train_dygraph(self): - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = self.seed fluid.default_main_program().random_seed = self.seed @@ -123,22 +139,26 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): mnist = fluid.dygraph.static_runner.StaticModelRunner( model_dir=self.save_dirname, model_filename=self.model_filename, - params_filename=self.params_filename) + params_filename=self.params_filename, + ) suffix_varname_dict = mnist._program_holder_dict[ - 'forward']._suffix_varname_dict + 'forward' + ]._suffix_varname_dict dict_old_new = {v: k for k, v in suffix_varname_dict.items()} dy_param_init_value = {} for param in mnist.parameters(): dy_param_init_value[param.name] = param.numpy() - sgd = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=mnist.parameters()) + sgd = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=mnist.parameters() + ) - train_reader = paddle.batch(self.reader_decorator( - paddle.dataset.mnist.train()), - batch_size=self.batch_size, - drop_last=True) + train_reader = paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=self.batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator(capacity=10) train_loader.set_sample_list_generator(train_reader, places=place) @@ -169,58 +189,71 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): for param in mnist.parameters(): dy_param_value[param.name] = param.numpy() - return dy_x_data, dy_out, dy_param_init_value, dy_param_value, dict_old_new + return ( + dy_x_data, + dy_out, + dy_param_init_value, + dy_param_value, + dict_old_new, + ) def load_and_train_static(self): with new_program_scope(): fluid.default_startup_program().random_seed = self.seed fluid.default_main_program().random_seed = self.seed - img = fluid.data(name='img', - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 1, 28, 28], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') prediction, avg_loss = static_train_net(img, label) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fluid.io.load_params(exe, - self.save_dirname, - main_program=fluid.default_main_program(), - filename=self.params_filename) + fluid.io.load_params( + exe, + self.save_dirname, + main_program=fluid.default_main_program(), + filename=self.params_filename, + ) static_param_init_value = {} static_param_name_list = [] for param in fluid.default_main_program().all_parameters(): static_param_name_list.append(param.name) static_param_init_value[param.name] = fluid.executor._fetch_var( - param.name) + param.name + ) - train_reader = paddle.batch(self.reader_decorator( - paddle.dataset.mnist.train()), - batch_size=self.batch_size, - drop_last=True) + train_reader = paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=self.batch_size, + drop_last=True, + ) for epoch in range(self.epoch_num): for batch_id, data in enumerate(train_reader()): static_x_data = np.array([x[0] for x in data]) - y_data = np.array([x[1] for x in data - ]).reshape([self.batch_size, 1]) + y_data = np.array([x[1] for x in data]).reshape( + [self.batch_size, 1] + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "img": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"img": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) if batch_id >= self.batch_num: break @@ -230,21 +263,31 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): for i in range(1, len(out)): static_param_value[static_param_name_list[i - 1]] = out[i] - return static_x_data, static_out, static_param_init_value, static_param_value + return ( + static_x_data, + static_out, + static_param_init_value, + static_param_value, + ) def load_and_infer_dygraph(self): - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): fluid.default_main_program().random_seed = self.seed mnist = fluid.dygraph.static_runner.StaticModelRunner( - model_dir=self.save_dirname, model_filename=self.model_filename) - - train_reader = paddle.batch(self.reader_decorator( - paddle.dataset.mnist.test()), - batch_size=self.batch_size, - drop_last=True) + model_dir=self.save_dirname, model_filename=self.model_filename + ) + + train_reader = paddle.batch( + self.reader_decorator(paddle.dataset.mnist.test()), + batch_size=self.batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator(capacity=10) train_loader.set_sample_list_generator(train_reader, places=place) @@ -264,24 +307,33 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): def load_and_infer_static(self): with new_program_scope(): - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - [infer_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.save_dirname, exe) + [ + infer_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(self.save_dirname, exe) infer_program.random_seed = self.seed - train_reader = paddle.batch(self.reader_decorator( - paddle.dataset.mnist.test()), - batch_size=self.batch_size, - drop_last=True) + train_reader = paddle.batch( + self.reader_decorator(paddle.dataset.mnist.test()), + batch_size=self.batch_size, + drop_last=True, + ) for batch_id, data in enumerate(train_reader()): static_x_data = np.array([x[0] for x in data]) - out = exe.run(infer_program, - feed={feed_target_names[0]: static_x_data}, - fetch_list=fetch_targets) + out = exe.run( + infer_program, + feed={feed_target_names[0]: static_x_data}, + fetch_list=fetch_targets, + ) if batch_id >= 1: break @@ -300,11 +352,20 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): # Phase 2. load model & train dygraph - dy_x_data, dy_out, dy_param_init_value, dy_param_value, dict_old_new_init= \ - self.load_and_train_dygraph() - - static_x_data, static_out, static_param_init_value, static_param_value = \ - self.load_and_train_static() + ( + dy_x_data, + dy_out, + dy_param_init_value, + dy_param_value, + dict_old_new_init, + ) = self.load_and_train_dygraph() + + ( + static_x_data, + static_out, + static_param_init_value, + static_param_value, + ) = self.load_and_train_static() # Phase 3. compare np.testing.assert_array_equal(static_x_data, dy_x_data) @@ -318,10 +379,9 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): for key, value in static_param_value.items(): key = dict_old_new_init[key] - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-4 + ) def test_mnist_train_with_params_filename(self): self.save_dirname = "mnist.inference.model" @@ -331,11 +391,20 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): self.train_and_save_model() # Phase 2. load model & train dygraph - dy_x_data, dy_out, dy_param_init_value, dy_param_value, dict_old_new_init= \ - self.load_and_train_dygraph() - - static_x_data, static_out, static_param_init_value, static_param_value = \ - self.load_and_train_static() + ( + dy_x_data, + dy_out, + dy_param_init_value, + dy_param_value, + dict_old_new_init, + ) = self.load_and_train_dygraph() + + ( + static_x_data, + static_out, + static_param_init_value, + static_param_value, + ) = self.load_and_train_static() # Phase 3. compare np.testing.assert_array_equal(static_x_data, dy_x_data) @@ -348,10 +417,9 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): for key, value in static_param_value.items(): key = dict_old_new_init[key] - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-4 + ) def test_mnist_infer_no_params_filename(self): self.save_dirname = "mnist.inference.model.noname" @@ -361,11 +429,9 @@ class TestImperativeStaticModelRunnerMnist(unittest.TestCase): self.train_and_save_model() # Phase 2. load model & train dygraph - dy_x_data, dy_out = \ - self.load_and_infer_dygraph() + dy_x_data, dy_out = self.load_and_infer_dygraph() - static_x_data, static_out = \ - self.load_and_infer_static() + static_x_data, static_out = self.load_and_infer_static() # Phase 3. compare np.testing.assert_array_equal(static_x_data, dy_x_data) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_static_runner_while.py b/python/paddle/fluid/tests/unittests/test_imperative_static_runner_while.py index b12d8e6a3db..d163350f136 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_static_runner_while.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_static_runner_while.py @@ -27,7 +27,6 @@ LOADED_VAR_SUFFIX = ".load_0" def while_softmax_regression(img): - def cond(i, times, pred): return i < times @@ -39,14 +38,13 @@ def while_softmax_regression(img): i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0) times = fluid.layers.fill_constant(shape=[1], dtype='int64', value=5) pred = fluid.layers.fc(input=img, size=10, act='softmax') - i, times, pred = fluid.layers.while_loop(cond=cond, - body=body, - loop_vars=[i, times, pred]) + i, times, pred = fluid.layers.while_loop( + cond=cond, body=body, loop_vars=[i, times, pred] + ) return pred class TestImperativeStaticModelRunnerWhile(unittest.TestCase): - def setUp(self): self.seed = 90 self.batch_size = 32 @@ -56,7 +54,6 @@ class TestImperativeStaticModelRunnerWhile(unittest.TestCase): self.params_filename = None def _random_batch_reader(self): - def _get_random_images_and_labels(image_shape, label_shape): image = np.random.random(size=image_shape).astype('float32') label = np.random.random(size=label_shape).astype('int64') @@ -65,7 +62,8 @@ class TestImperativeStaticModelRunnerWhile(unittest.TestCase): def __reader__(): for _ in range(self.batch_num): batch_image, batch_label = _get_random_images_and_labels( - [self.batch_size, 784], [self.batch_size, 1]) + [self.batch_size, 784], [self.batch_size, 1] + ) yield batch_image, batch_label return __reader__ @@ -85,29 +83,39 @@ class TestImperativeStaticModelRunnerWhile(unittest.TestCase): optimizer = fluid.optimizer.SGD(learning_rate=0.001) optimizer.minimize(avg_loss) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup_program) - loader = fluid.io.DataLoader.from_generator(feed_list=[img, label], - capacity=5, - iterable=True) + loader = fluid.io.DataLoader.from_generator( + feed_list=[img, label], capacity=5, iterable=True + ) loader.set_batch_generator(self._random_batch_reader(), places=place) for data in loader(): exe.run(main_program, feed=data, fetch_list=[avg_loss]) - fluid.io.save_inference_model(self.save_dirname, ["img"], [pred], - exe, - model_filename=self.model_filename, - params_filename=self.params_filename, - clip_extra=False) + fluid.io.save_inference_model( + self.save_dirname, + ["img"], + [pred], + exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + clip_extra=False, + ) def load_and_train_dygraph(self): - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = self.seed fluid.default_main_program().random_seed = self.seed @@ -115,18 +123,21 @@ class TestImperativeStaticModelRunnerWhile(unittest.TestCase): fluid.set_flags({'FLAGS_sort_sum_gradient': True}) while_net = fluid.dygraph.static_runner.StaticModelRunner( - self.save_dirname) + self.save_dirname + ) dy_param_init_value = {} for param in while_net.parameters(): dy_param_init_value[param.name] = param.numpy() - sgd = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=while_net.parameters()) + sgd = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=while_net.parameters() + ) train_loader = fluid.io.DataLoader.from_generator(capacity=10) - train_loader.set_batch_generator(self._random_batch_reader(), - places=place) + train_loader.set_batch_generator( + self._random_batch_reader(), places=place + ) while_net.train() @@ -168,37 +179,46 @@ class TestImperativeStaticModelRunnerWhile(unittest.TestCase): optimizer = fluid.optimizer.SGD(learning_rate=0.001) optimizer.minimize(avg_loss) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fluid.io.load_params(exe, - self.save_dirname, - main_program=fluid.default_main_program(), - filename=self.params_filename) + fluid.io.load_params( + exe, + self.save_dirname, + main_program=fluid.default_main_program(), + filename=self.params_filename, + ) static_param_init_value = {} static_param_name_list = [] for param in fluid.default_main_program().all_parameters(): static_param_name_list.append(param.name) static_param_init_value[param.name] = fluid.executor._fetch_var( - param.name) + param.name + ) - loader = fluid.io.DataLoader.from_generator(feed_list=[img, label], - capacity=5, - iterable=True) - loader.set_batch_generator(self._random_batch_reader(), - places=place) + loader = fluid.io.DataLoader.from_generator( + feed_list=[img, label], capacity=5, iterable=True + ) + loader.set_batch_generator( + self._random_batch_reader(), places=place + ) for data in loader(): fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed=data, - fetch_list=[avg_loss]) + out = exe.run( + fluid.default_main_program(), + feed=data, + fetch_list=[avg_loss], + ) static_param_value = {} static_out = out[0] @@ -213,17 +233,24 @@ class TestImperativeStaticModelRunnerWhile(unittest.TestCase): # # Phase 2. load model & train dygraph with unique_name.guard(): - dy_out, dy_param_init_value, dy_param_value = \ - self.load_and_train_dygraph() + ( + dy_out, + dy_param_init_value, + dy_param_value, + ) = self.load_and_train_dygraph() with unique_name.guard(): - static_out, static_param_init_value, static_param_value = \ - self.load_and_train_static() + ( + static_out, + static_param_init_value, + static_param_value, + ) = self.load_and_train_static() # Phase 3. compare with unique_name.guard(): dict_old_new_init = rename_var_with_generator( - static_param_init_value.keys()) + static_param_init_value.keys() + ) for key, value in static_param_init_value.items(): key = dict_old_new_init[key] np.testing.assert_array_equal(value, dy_param_init_value[key]) @@ -232,10 +259,9 @@ class TestImperativeStaticModelRunnerWhile(unittest.TestCase): for key, value in static_param_value.items(): key += LOADED_VAR_SUFFIX - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_tensor_clear_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_tensor_clear_gradient.py index 55879293734..0445e09232f 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_tensor_clear_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_tensor_clear_gradient.py @@ -21,7 +21,6 @@ import numpy as np def _dygraph_guard_(func): - def __impl__(*args, **kwargs): if fluid._non_static_mode(): return func(*args, **kwargs) @@ -36,7 +35,6 @@ dygraph_guard = wrap_decorator(_dygraph_guard_) class TestDygraphClearGradient(TestCase): - def setUp(self): self.input_shape = [10, 2] diff --git a/python/paddle/fluid/tests/unittests/test_imperative_thread_local_has_grad.py b/python/paddle/fluid/tests/unittests/test_imperative_thread_local_has_grad.py index 5e8a54ca390..7acbc2d339c 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_thread_local_has_grad.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_thread_local_has_grad.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class SimpleNet(nn.Layer): - def __init__(self, in_dim, out_dim): super(SimpleNet, self).__init__() self.fc = nn.Linear(in_dim, out_dim) @@ -32,7 +31,6 @@ class SimpleNet(nn.Layer): class TestCases(unittest.TestCase): - @paddle.no_grad() def thread_1_main(self): time.sleep(8) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_trace_non_persistable_inputs.py b/python/paddle/fluid/tests/unittests/test_imperative_trace_non_persistable_inputs.py index a346bb22985..66a1e7f8411 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_trace_non_persistable_inputs.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_trace_non_persistable_inputs.py @@ -19,12 +19,12 @@ import os class SimpleFCLayer(fluid.dygraph.Layer): - def __init__(self, feature_size, batch_size, fc_size): super(SimpleFCLayer, self).__init__() self._linear = fluid.dygraph.Linear(feature_size, fc_size) self._offset = fluid.dygraph.to_variable( - np.random.random((batch_size, fc_size)).astype('float32')) + np.random.random((batch_size, fc_size)).astype('float32') + ) def forward(self, x): fc = self._linear(x) @@ -32,7 +32,6 @@ class SimpleFCLayer(fluid.dygraph.Layer): class TestTracedLayerRecordNonPersistableInput(unittest.TestCase): - def test_main(self): if fluid.framework.in_dygraph_mode(): return @@ -42,21 +41,28 @@ class TestTracedLayerRecordNonPersistableInput(unittest.TestCase): batch_size = 4 fc_size = 2 layer = SimpleFCLayer(feature_size, batch_size, fc_size) - optimizer = fluid.optimizer.SGD(learning_rate=1e-3, - parameter_list=layer.parameters()) - - expected_persistable_vars = set([ - layer._linear.weight.name, layer._linear.bias.name, - layer._offset.name - ]) + optimizer = fluid.optimizer.SGD( + learning_rate=1e-3, parameter_list=layer.parameters() + ) + + expected_persistable_vars = set( + [ + layer._linear.weight.name, + layer._linear.bias.name, + layer._offset.name, + ] + ) for _ in range(10): in_x = fluid.dygraph.to_variable( - np.random.random( - (batch_size, feature_size)).astype('float32')) + np.random.random((batch_size, feature_size)).astype( + 'float32' + ) + ) if traced_layer is None: dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - layer, [in_x]) + layer, [in_x] + ) else: dygraph_out = layer(in_x) dygraph_out_numpy = dygraph_out.numpy() @@ -79,11 +85,15 @@ class TestTracedLayerRecordNonPersistableInput(unittest.TestCase): self.assertEqual(actual_persistable_vars, expected_persistable_vars) traced_layer.save_inference_model( - path='./traced_layer_test_non_persistable_vars') - self.assertTrue('traced_layer_test_non_persistable_vars.pdmodel' in - os.listdir('./')) - self.assertTrue('traced_layer_test_non_persistable_vars.pdiparams' in - os.listdir('./')) + path='./traced_layer_test_non_persistable_vars' + ) + self.assertTrue( + 'traced_layer_test_non_persistable_vars.pdmodel' in os.listdir('./') + ) + self.assertTrue( + 'traced_layer_test_non_persistable_vars.pdiparams' + in os.listdir('./') + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py index 3ba02fe8daf..22189d88d31 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py @@ -149,12 +149,15 @@ def position_encoding_init(n_position, d_pos_vec): channels = d_pos_vec position = np.arange(n_position) num_timescales = channels // 2 - log_timescale_increment = (np.log(float(1e4) / float(1)) / - (num_timescales - 1)) - inv_timescales = np.exp( - np.arange(num_timescales)) * -log_timescale_increment + log_timescale_increment = np.log(float(1e4) / float(1)) / ( + num_timescales - 1 + ) + inv_timescales = ( + np.exp(np.arange(num_timescales)) * -log_timescale_increment + ) scaled_time = np.expand_dims(position, 1) * np.expand_dims( - inv_timescales, 0) + inv_timescales, 0 + ) signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1) signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant') position_enc = signal @@ -164,21 +167,27 @@ def position_encoding_init(n_position, d_pos_vec): def create_data(is_static=False): if is_static: return [ - src_word_np, src_pos_np, src_slf_attn_bias_np, trg_word_np, - trg_pos_np, trg_slf_attn_bias_np, trg_src_attn_bias_np, lbl_word_np, - lbl_weight_np + src_word_np, + src_pos_np, + src_slf_attn_bias_np, + trg_word_np, + trg_pos_np, + trg_slf_attn_bias_np, + trg_src_attn_bias_np, + lbl_word_np, + lbl_weight_np, ] else: enc_inputs = [ to_variable(src_word_np, name='src_word'), to_variable(src_pos_np, name='src_pos'), - to_variable(src_slf_attn_bias_np, name='src_slf_attn_bias') + to_variable(src_slf_attn_bias_np, name='src_slf_attn_bias'), ] dec_inputs = [ to_variable(trg_word_np, name='trg_word'), to_variable(trg_pos_np, name='trg_pos'), to_variable(trg_slf_attn_bias_np, name='trg_slf_attn_bias'), - to_variable(trg_src_attn_bias_np, name='trg_src_attn_bias') + to_variable(trg_src_attn_bias_np, name='trg_src_attn_bias'), ] label = to_variable(lbl_word_np, name='lbl_word') weight = to_variable(lbl_weight_np, name='lbl_weight') @@ -187,11 +196,18 @@ def create_data(is_static=False): def create_feed_dict_list(data, init=False): if init: - data_input_names = encoder_data_input_fields + \ - decoder_data_input_fields[:-1] + label_data_input_fields + pos_enc_param_names + data_input_names = ( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + + pos_enc_param_names + ) else: - data_input_names = encoder_data_input_fields + \ - decoder_data_input_fields[:-1] + label_data_input_fields + data_input_names = ( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + ) feed_dict_list = dict() for i in range(len(data_input_names)): feed_dict_list[data_input_names[i]] = data[i] @@ -204,12 +220,15 @@ def make_all_inputs(input_fields): """ inputs = [] for input_field in input_fields: - input_var = fluid.layers.data(name=input_field, - shape=input_descs[input_field][0], - dtype=input_descs[input_field][1], - lod_level=input_descs[input_field][2] if - len(input_descs[input_field]) == 3 else 0, - append_batch_size=False) + input_var = fluid.layers.data( + name=input_field, + shape=input_descs[input_field][0], + dtype=input_descs[input_field][1], + lod_level=input_descs[input_field][2] + if len(input_descs[input_field]) == 3 + else 0, + append_batch_size=False, + ) inputs.append(input_var) return inputs @@ -234,12 +253,17 @@ input_descs = { # encoder. # The actual data shape of src_slf_attn_bias is: # [batch_size, n_head, max_src_len_in_batch, max_src_len_in_batch] - "src_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "src_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # The actual data shape of trg_word is: # [batch_size, max_trg_len_in_batch] - "trg_word": [(batch_size, seq_len), "int64", - 2], # lod_level is only used in fast decoder. + "trg_word": [ + (batch_size, seq_len), + "int64", + 2, + ], # lod_level is only used in fast decoder. # The actual data shape of trg_pos is: # [batch_size, max_trg_len_in_batch] "trg_pos": [(batch_size, seq_len), "int64"], @@ -247,14 +271,18 @@ input_descs = { # subsequent words in the decoder. # The actual data shape of trg_slf_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_trg_len_in_batch] - "trg_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used to remove attention weights on paddings of the source # input in the encoder-decoder attention. # The actual data shape of trg_src_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_src_len_in_batch] - "trg_src_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_src_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used in independent decoder program for inference. # The actual data shape of enc_output is: # [batch_size, max_src_len_in_batch, d_model] @@ -270,7 +298,7 @@ input_descs = { "init_score": [(batch_size, 1), "float32", 2], # This input is used in beam-search decoder for the first gather # (cell states updation) - "init_idx": [(batch_size, ), "int32"], + "init_idx": [(batch_size,), "int32"], } # Names of word embedding table which might be reused for weight sharing. @@ -318,44 +346,52 @@ sync = False batch_num = 5 np.random.seed(90) -src_word_np = np.arange(1, TrainTaskConfig.batch_size * seq_len + 1).reshape( - [TrainTaskConfig.batch_size, seq_len]).astype('int64') -src_pos_np = np.random.randint(1, - seq_len, - size=(TrainTaskConfig.batch_size, seq_len), - dtype='int64') -src_slf_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, - ModelHyperParams.n_head, seq_len, - seq_len).astype('float32') - -trg_word_np = np.arange(1, TrainTaskConfig.batch_size * seq_len + 1).reshape( - [TrainTaskConfig.batch_size, seq_len]).astype('int64') -trg_pos_np = np.random.randint(1, - seq_len, - size=(TrainTaskConfig.batch_size, seq_len), - dtype='int64') -trg_slf_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, - ModelHyperParams.n_head, seq_len, - seq_len).astype('float32') -trg_src_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, - ModelHyperParams.n_head, seq_len, - seq_len).astype('float32') - -lbl_word_np = np.random.randint(1, - ModelHyperParams.src_vocab_size - 1, - size=(TrainTaskConfig.batch_size * seq_len, 1), - dtype='int64') -lbl_weight_np = np.random.randn(TrainTaskConfig.batch_size * seq_len, - 1).astype('float32') - -pos_inp1 = position_encoding_init(ModelHyperParams.max_length, - ModelHyperParams.d_model) -pos_inp2 = position_encoding_init(ModelHyperParams.max_length, - ModelHyperParams.d_model) +src_word_np = ( + np.arange(1, TrainTaskConfig.batch_size * seq_len + 1) + .reshape([TrainTaskConfig.batch_size, seq_len]) + .astype('int64') +) +src_pos_np = np.random.randint( + 1, seq_len, size=(TrainTaskConfig.batch_size, seq_len), dtype='int64' +) +src_slf_attn_bias_np = np.random.randn( + TrainTaskConfig.batch_size, ModelHyperParams.n_head, seq_len, seq_len +).astype('float32') + +trg_word_np = ( + np.arange(1, TrainTaskConfig.batch_size * seq_len + 1) + .reshape([TrainTaskConfig.batch_size, seq_len]) + .astype('int64') +) +trg_pos_np = np.random.randint( + 1, seq_len, size=(TrainTaskConfig.batch_size, seq_len), dtype='int64' +) +trg_slf_attn_bias_np = np.random.randn( + TrainTaskConfig.batch_size, ModelHyperParams.n_head, seq_len, seq_len +).astype('float32') +trg_src_attn_bias_np = np.random.randn( + TrainTaskConfig.batch_size, ModelHyperParams.n_head, seq_len, seq_len +).astype('float32') + +lbl_word_np = np.random.randint( + 1, + ModelHyperParams.src_vocab_size - 1, + size=(TrainTaskConfig.batch_size * seq_len, 1), + dtype='int64', +) +lbl_weight_np = np.random.randn(TrainTaskConfig.batch_size * seq_len, 1).astype( + 'float32' +) +pos_inp1 = position_encoding_init( + ModelHyperParams.max_length, ModelHyperParams.d_model +) +pos_inp2 = position_encoding_init( + ModelHyperParams.max_length, ModelHyperParams.d_model +) -class PrePostProcessLayer(Layer): +class PrePostProcessLayer(Layer): def __init__(self, d_model, process_cmd, shape_len=None): super(PrePostProcessLayer, self).__init__() for cmd in process_cmd: @@ -363,11 +399,14 @@ class PrePostProcessLayer(Layer): self._layer_norm = LayerNorm( normalized_shape=d_model, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.)), + initializer=fluid.initializer.Constant(1.0) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.))) + initializer=fluid.initializer.Constant(0.0) + ), + ) - def forward(self, prev_out, out, process_cmd, dropout_rate=0.): + def forward(self, prev_out, out, process_cmd, dropout_rate=0.0): for cmd in process_cmd: if cmd == "a": # add residual connection out = out + prev_out if prev_out is not None else out @@ -379,12 +418,12 @@ class PrePostProcessLayer(Layer): out, dropout_prob=dropout_rate, seed=ModelHyperParams.dropout_seed, - is_test=False) + is_test=False, + ) return out class PositionwiseFeedForwardLayer(Layer): - def __init__(self, d_inner_hid, d_hid, dropout_rate): super(PositionwiseFeedForwardLayer, self).__init__() self._i2h = Linear(d_hid, d_inner_hid, act="relu") @@ -394,25 +433,28 @@ class PositionwiseFeedForwardLayer(Layer): def forward(self, x): hidden = self._i2h(x) if self._dropout_rate: - hidden = fluid.layers.dropout(hidden, - dropout_prob=self._dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) + hidden = fluid.layers.dropout( + hidden, + dropout_prob=self._dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) out = self._h2o(hidden) return out class MultiHeadAttentionLayer(Layer): - - def __init__(self, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - cache=None, - gather_idx=None, - static_kv=False): + def __init__( + self, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, + cache=None, + gather_idx=None, + static_kv=False, + ): super(MultiHeadAttentionLayer, self).__init__() self._n_head = n_head self._d_key = d_key @@ -435,20 +477,25 @@ class MultiHeadAttentionLayer(Layer): # split head reshaped_q = fluid.layers.reshape( - x=q, shape=[0, 0, self._n_head, self._d_key], inplace=False) + x=q, shape=[0, 0, self._n_head, self._d_key], inplace=False + ) transpose_q = fluid.layers.transpose(x=reshaped_q, perm=[0, 2, 1, 3]) reshaped_k = fluid.layers.reshape( - x=k, shape=[0, 0, self._n_head, self._d_key], inplace=False) + x=k, shape=[0, 0, self._n_head, self._d_key], inplace=False + ) transpose_k = fluid.layers.transpose(x=reshaped_k, perm=[0, 2, 1, 3]) reshaped_v = fluid.layers.reshape( - x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False) + x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False + ) transpose_v = fluid.layers.transpose(x=reshaped_v, perm=[0, 2, 1, 3]) # scale dot product attention - product = fluid.layers.matmul(x=transpose_q, - y=transpose_k, - transpose_y=True, - alpha=self._d_model**-0.5) + product = fluid.layers.matmul( + x=transpose_q, + y=transpose_k, + transpose_y=True, + alpha=self._d_model**-0.5, + ) if attn_bias is not None: product += attn_bias weights = fluid.layers.softmax(product) @@ -457,7 +504,8 @@ class MultiHeadAttentionLayer(Layer): weights, dropout_prob=self._dropout_rate, seed=ModelHyperParams.dropout_seed, - is_test=False) + is_test=False, + ) out = fluid.layers.matmul(weights_droped, transpose_v) else: out = fluid.layers.matmul(weights, transpose_v) @@ -469,7 +517,8 @@ class MultiHeadAttentionLayer(Layer): final_out = fluid.layers.reshape( x=trans_x, shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], - inplace=False) + inplace=False, + ) # fc to output proj_out = self._proj_fc(final_out) @@ -477,118 +526,150 @@ class MultiHeadAttentionLayer(Layer): class EncoderSubLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderSubLayer, self).__init__() self._preprocess_cmd = preprocess_cmd self._postprocess_cmd = postprocess_cmd self._prepostprocess_dropout = prepostprocess_dropout - self._preprocess_layer = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + self._preprocess_layer = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) self._multihead_attention_layer = MultiHeadAttentionLayer( - d_key, d_value, d_model, n_head, attention_dropout) - self._postprocess_layer = PrePostProcessLayer(d_model, - self._postprocess_cmd, - None) - self._preprocess_layer2 = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + d_key, d_value, d_model, n_head, attention_dropout + ) + self._postprocess_layer = PrePostProcessLayer( + d_model, self._postprocess_cmd, None + ) + self._preprocess_layer2 = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) self._positionwise_feed_forward = PositionwiseFeedForwardLayer( - d_inner_hid, d_model, relu_dropout) - self._postprocess_layer2 = PrePostProcessLayer(d_model, - self._postprocess_cmd, - None) + d_inner_hid, d_model, relu_dropout + ) + self._postprocess_layer2 = PrePostProcessLayer( + d_model, self._postprocess_cmd, None + ) def forward(self, enc_input, attn_bias): pre_process_multihead = self._preprocess_layer( - None, enc_input, self._preprocess_cmd, self._prepostprocess_dropout) - attn_output = self._multihead_attention_layer(pre_process_multihead, - None, None, attn_bias) - attn_output = self._postprocess_layer(enc_input, attn_output, - self._postprocess_cmd, - self._prepostprocess_dropout) + None, enc_input, self._preprocess_cmd, self._prepostprocess_dropout + ) + attn_output = self._multihead_attention_layer( + pre_process_multihead, None, None, attn_bias + ) + attn_output = self._postprocess_layer( + enc_input, + attn_output, + self._postprocess_cmd, + self._prepostprocess_dropout, + ) pre_process2_output = self._preprocess_layer2( - None, attn_output, self._preprocess_cmd, - self._prepostprocess_dropout) + None, + attn_output, + self._preprocess_cmd, + self._prepostprocess_dropout, + ) ffd_output = self._positionwise_feed_forward(pre_process2_output) - return self._postprocess_layer2(attn_output, ffd_output, - self._postprocess_cmd, - self._prepostprocess_dropout) + return self._postprocess_layer2( + attn_output, + ffd_output, + self._postprocess_cmd, + self._prepostprocess_dropout, + ) class EncoderLayer(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderLayer, self).__init__() self._preprocess_cmd = preprocess_cmd self._encoder_sublayers = list() self._prepostprocess_dropout = prepostprocess_dropout self._n_layer = n_layer - self._preprocess_layer = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + self._preprocess_layer = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) for i in range(n_layer): self._encoder_sublayers.append( self.add_sublayer( 'esl_%d' % i, - EncoderSubLayer(n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd))) + EncoderSubLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ), + ) + ) def forward(self, enc_input, attn_bias): for i in range(self._n_layer): enc_output = self._encoder_sublayers[i](enc_input, attn_bias) enc_input = enc_output - return self._preprocess_layer(None, enc_output, self._preprocess_cmd, - self._prepostprocess_dropout) + return self._preprocess_layer( + None, enc_output, self._preprocess_cmd, self._prepostprocess_dropout + ) class PrepareEncoderDecoderLayer(Layer): - - def __init__(self, - src_vocab_size, - src_emb_dim, - src_max_len, - dropout_rate, - is_sparse=False, - word_emb_param_name=None, - pos_enc_param_name=None): + def __init__( + self, + src_vocab_size, + src_emb_dim, + src_max_len, + dropout_rate, + is_sparse=False, + word_emb_param_name=None, + pos_enc_param_name=None, + ): super(PrepareEncoderDecoderLayer, self).__init__() self._src_max_len = src_max_len self._src_emb_dim = src_emb_dim self._src_vocab_size = src_vocab_size self._dropout_rate = dropout_rate - self._input_emb = Embedding(size=[src_vocab_size, src_emb_dim], - is_sparse=is_sparse, - padding_idx=0, - param_attr=fluid.ParamAttr( - name=word_emb_param_name, - initializer=fluid.initializer.Normal( - 0., src_emb_dim**-0.5))) + self._input_emb = Embedding( + size=[src_vocab_size, src_emb_dim], + is_sparse=is_sparse, + padding_idx=0, + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0.0, src_emb_dim**-0.5), + ), + ) if pos_enc_param_name is pos_enc_param_names[0]: pos_inp = pos_inp1 @@ -600,7 +681,9 @@ class PrepareEncoderDecoderLayer(Layer): param_attr=fluid.ParamAttr( name=pos_enc_param_name, initializer=fluid.initializer.NumpyArrayInitializer(pos_inp), - trainable=False)) + trainable=False, + ), + ) # use in dygraph_mode to fit different length batch # self._pos_emb._w = to_variable( @@ -608,37 +691,44 @@ class PrepareEncoderDecoderLayer(Layer): def forward(self, src_word, src_pos): src_word_emb = self._input_emb(src_word) - src_word_emb = fluid.layers.scale(x=src_word_emb, - scale=self._src_emb_dim**0.5) + src_word_emb = fluid.layers.scale( + x=src_word_emb, scale=self._src_emb_dim**0.5 + ) # # TODO change this to fit dynamic length input src_pos_emb = self._pos_emb(src_pos) src_pos_emb.stop_gradient = True enc_input = src_word_emb + src_pos_emb - return fluid.layers.dropout( - enc_input, - dropout_prob=self._dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) if self._dropout_rate else enc_input + return ( + fluid.layers.dropout( + enc_input, + dropout_prob=self._dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) + if self._dropout_rate + else enc_input + ) class WrapEncoderLayer(Layer): - - def __init__(self, - src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=False): + def __init__( + self, + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=False, + ): """ The wrapper assembles together all needed layers for the encoder. """ @@ -651,11 +741,21 @@ class WrapEncoderLayer(Layer): prepostprocess_dropout, is_sparse=is_sparse, word_emb_param_name=word_emb_param_names[0], - pos_enc_param_name=pos_enc_param_names[0]) - self._encoder = EncoderLayer(n_layer, n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd) + pos_enc_param_name=pos_enc_param_names[0], + ) + self._encoder = EncoderLayer( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ) def forward(self, enc_inputs): src_word, src_pos, src_slf_attn_bias = enc_inputs @@ -665,26 +765,28 @@ class WrapEncoderLayer(Layer): class DecoderSubLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None, - gather_idx=None): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + cache=None, + gather_idx=None, + ): super(DecoderSubLayer, self).__init__() self._postprocess_cmd = postprocess_cmd self._preprocess_cmd = preprocess_cmd self._prepostprcess_dropout = prepostprocess_dropout - self._pre_process_layer = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + self._pre_process_layer = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._multihead_attention_layer = MultiHeadAttentionLayer( d_key, d_value, @@ -692,11 +794,14 @@ class DecoderSubLayer(Layer): n_head, attention_dropout, cache=cache, - gather_idx=gather_idx) - self._post_process_layer = PrePostProcessLayer(d_model, postprocess_cmd, - None) - self._pre_process_layer2 = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + gather_idx=gather_idx, + ) + self._post_process_layer = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) + self._pre_process_layer2 = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._multihead_attention_layer2 = MultiHeadAttentionLayer( d_key, d_value, @@ -705,63 +810,86 @@ class DecoderSubLayer(Layer): attention_dropout, cache=cache, gather_idx=gather_idx, - static_kv=True) - self._post_process_layer2 = PrePostProcessLayer(d_model, - postprocess_cmd, None) - self._pre_process_layer3 = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + static_kv=True, + ) + self._post_process_layer2 = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) + self._pre_process_layer3 = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._positionwise_feed_forward_layer = PositionwiseFeedForwardLayer( - d_inner_hid, d_model, relu_dropout) - self._post_process_layer3 = PrePostProcessLayer(d_model, - postprocess_cmd, None) + d_inner_hid, d_model, relu_dropout + ) + self._post_process_layer3 = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) def forward(self, dec_input, enc_output, slf_attn_bias, dec_enc_attn_bias): - pre_process_rlt = self._pre_process_layer(None, dec_input, - self._preprocess_cmd, - self._prepostprcess_dropout) + pre_process_rlt = self._pre_process_layer( + None, dec_input, self._preprocess_cmd, self._prepostprcess_dropout + ) slf_attn_output = self._multihead_attention_layer( - pre_process_rlt, None, None, slf_attn_bias) + pre_process_rlt, None, None, slf_attn_bias + ) slf_attn_output_pp = self._post_process_layer( - dec_input, slf_attn_output, self._postprocess_cmd, - self._prepostprcess_dropout) - pre_process_rlt2 = self._pre_process_layer2(None, slf_attn_output_pp, - self._preprocess_cmd, - self._prepostprcess_dropout) + dec_input, + slf_attn_output, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) + pre_process_rlt2 = self._pre_process_layer2( + None, + slf_attn_output_pp, + self._preprocess_cmd, + self._prepostprcess_dropout, + ) enc_attn_output_pp = self._multihead_attention_layer2( - pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias) - enc_attn_output = self._post_process_layer2(slf_attn_output_pp, - enc_attn_output_pp, - self._postprocess_cmd, - self._prepostprcess_dropout) - pre_process_rlt3 = self._pre_process_layer3(None, enc_attn_output, - self._preprocess_cmd, - self._prepostprcess_dropout) + pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias + ) + enc_attn_output = self._post_process_layer2( + slf_attn_output_pp, + enc_attn_output_pp, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) + pre_process_rlt3 = self._pre_process_layer3( + None, + enc_attn_output, + self._preprocess_cmd, + self._prepostprcess_dropout, + ) ffd_output = self._positionwise_feed_forward_layer(pre_process_rlt3) - dec_output = self._post_process_layer3(enc_attn_output, ffd_output, - self._postprocess_cmd, - self._prepostprcess_dropout) + dec_output = self._post_process_layer3( + enc_attn_output, + ffd_output, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) return dec_output class DecoderLayer(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=None, - gather_idx=None): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + caches=None, + gather_idx=None, + ): super(DecoderLayer, self).__init__() - self._pre_process_layer = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + self._pre_process_layer = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._decoder_sub_layers = list() self._n_layer = n_layer self._preprocess_cmd = preprocess_cmd @@ -770,53 +898,62 @@ class DecoderLayer(Layer): self._decoder_sub_layers.append( self.add_sublayer( 'dsl_%d' % i, - DecoderSubLayer(n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None if caches is None else caches[i], - gather_idx=gather_idx))) - - def forward(self, dec_input, enc_output, dec_slf_attn_bias, - dec_enc_attn_bias): + DecoderSubLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + cache=None if caches is None else caches[i], + gather_idx=gather_idx, + ), + ) + ) + + def forward( + self, dec_input, enc_output, dec_slf_attn_bias, dec_enc_attn_bias + ): for i in range(self._n_layer): - tmp_dec_output = self._decoder_sub_layers[i](dec_input, enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias) + tmp_dec_output = self._decoder_sub_layers[i]( + dec_input, enc_output, dec_slf_attn_bias, dec_enc_attn_bias + ) dec_input = tmp_dec_output - dec_output = self._pre_process_layer(None, tmp_dec_output, - self._preprocess_cmd, - self._prepostprocess_dropout) + dec_output = self._pre_process_layer( + None, + tmp_dec_output, + self._preprocess_cmd, + self._prepostprocess_dropout, + ) return dec_output class WrapDecoderLayer(Layer): - - def __init__(self, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - caches=None, - gather_idx=None, - is_sparse=False): + def __init__( + self, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + caches=None, + gather_idx=None, + is_sparse=False, + ): """ The wrapper assembles together all needed layers for the encoder. """ @@ -829,20 +966,23 @@ class WrapDecoderLayer(Layer): prepostprocess_dropout, is_sparse=is_sparse, word_emb_param_name=word_emb_param_names[1], - pos_enc_param_name=pos_enc_param_names[1]) - self._decoder_layer = DecoderLayer(n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=caches, - gather_idx=gather_idx) + pos_enc_param_name=pos_enc_param_names[1], + ) + self._decoder_layer = DecoderLayer( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + caches=caches, + gather_idx=gather_idx, + ) self._weight_sharing = weight_sharing if not weight_sharing: self._fc = Linear(d_model, trg_vocab_size, bias_attr=False) @@ -850,17 +990,20 @@ class WrapDecoderLayer(Layer): def forward(self, dec_inputs=None, enc_output=None): trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias = dec_inputs dec_input = self._prepare_decoder_layer(trg_word, trg_pos) - dec_output = self._decoder_layer(dec_input, enc_output, - trg_slf_attn_bias, trg_src_attn_bias) + dec_output = self._decoder_layer( + dec_input, enc_output, trg_slf_attn_bias, trg_src_attn_bias + ) dec_output_reshape = fluid.layers.reshape( - dec_output, shape=[-1, dec_output.shape[-1]], inplace=False) + dec_output, shape=[-1, dec_output.shape[-1]], inplace=False + ) if self._weight_sharing: predict = fluid.layers.matmul( x=dec_output_reshape, y=self._prepare_decoder_layer._input_emb.weight, - transpose_y=True) + transpose_y=True, + ) else: predict = self._fc(dec_output_reshape) @@ -872,81 +1015,91 @@ class WrapDecoderLayer(Layer): class TransFormer(Layer): - - def __init__(self, - src_vocab_size, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - label_smooth_eps, - use_py_reader=False, - is_test=False, - is_sparse=False): + def __init__( + self, + src_vocab_size, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + label_smooth_eps, + use_py_reader=False, + is_test=False, + is_sparse=False, + ): super(TransFormer, self).__init__() self._label_smooth_eps = label_smooth_eps self._trg_vocab_size = trg_vocab_size if weight_sharing: - assert src_vocab_size == trg_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) - self._wrap_encoder_layer = WrapEncoderLayer(src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=is_sparse) - self._wrap_decoder_layer = WrapDecoderLayer(trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=is_sparse) + assert ( + src_vocab_size == trg_vocab_size + ), "Vocabularies in source and target should be same for weight sharing." + self._wrap_encoder_layer = WrapEncoderLayer( + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=is_sparse, + ) + self._wrap_decoder_layer = WrapDecoderLayer( + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=is_sparse, + ) if weight_sharing: - self._wrap_decoder_layer._prepare_decoder_layer._input_emb.weight = self._wrap_encoder_layer._prepare_encoder_layer._input_emb.weight + self._wrap_decoder_layer._prepare_decoder_layer._input_emb.weight = ( + self._wrap_encoder_layer._prepare_encoder_layer._input_emb.weight + ) def forward(self, enc_inputs, dec_inputs, label, weights): enc_output = self._wrap_encoder_layer(enc_inputs) predict = self._wrap_decoder_layer(dec_inputs, enc_output) if self._label_smooth_eps: label_out = fluid.layers.label_smooth( - label=fluid.layers.one_hot(input=label, - depth=self._trg_vocab_size), - epsilon=self._label_smooth_eps) + label=fluid.layers.one_hot( + input=label, depth=self._trg_vocab_size + ), + epsilon=self._label_smooth_eps, + ) cost = fluid.layers.softmax_with_cross_entropy( logits=predict, label=label_out, - soft_label=True if self._label_smooth_eps else False) + soft_label=True if self._label_smooth_eps else False, + ) weighted_cost = cost * weights sum_cost = fluid.layers.reduce_sum(weighted_cost) token_num = fluid.layers.reduce_sum(weights) @@ -956,7 +1109,6 @@ class TransFormer(Layer): class TestDygraphTransformerSortGradient(unittest.TestCase): - def test_transformer_sort_gradient(self): for is_sparse in [True, False]: self.transformer_sort_gradient_float32(is_sparse) @@ -969,28 +1121,31 @@ class TestDygraphTransformerSortGradient(unittest.TestCase): fluid.set_flags({'FLAGS_new_executor_use_inplace': False}) paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - transformer = TransFormer(ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - use_py_reader=use_py_reader, - is_test=False, - is_sparse=is_sparse) + transformer = TransFormer( + ModelHyperParams.src_vocab_size, + ModelHyperParams.trg_vocab_size, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.prepostprocess_dropout, + ModelHyperParams.attention_dropout, + ModelHyperParams.relu_dropout, + ModelHyperParams.preprocess_cmd, + ModelHyperParams.postprocess_cmd, + ModelHyperParams.weight_sharing, + TrainTaskConfig.label_smooth_eps, + use_py_reader=use_py_reader, + is_test=False, + is_sparse=is_sparse, + ) if sync: lr_decay = fluid.layers.learning_rate_scheduler.noam_decay( - ModelHyperParams.d_model, TrainTaskConfig.warmup_steps) + ModelHyperParams.d_model, TrainTaskConfig.warmup_steps + ) with fluid.default_main_program()._lr_schedule_guard(): learning_rate = lr_decay * TrainTaskConfig.learning_rate optimizer = fluid.optimizer.Adam( @@ -998,11 +1153,12 @@ class TestDygraphTransformerSortGradient(unittest.TestCase): beta1=TrainTaskConfig.beta1, beta2=TrainTaskConfig.beta2, epsilon=TrainTaskConfig.eps, - parameter_list=transformer.parameters()) + parameter_list=transformer.parameters(), + ) else: optimizer = fluid.optimizer.SGD( - learning_rate=0.003, - parameter_list=transformer.parameters()) + learning_rate=0.003, parameter_list=transformer.parameters() + ) dy_param_init = dict() dy_param_updated = dict() @@ -1013,20 +1169,23 @@ class TestDygraphTransformerSortGradient(unittest.TestCase): enc_inputs, dec_inputs, label, weights = create_data() if False: outs, traced_layer = TracedLayer.trace( - transformer, [enc_inputs, dec_inputs, label, weights]) + transformer, [enc_inputs, dec_inputs, label, weights] + ) ins_static = enc_inputs + dec_inputs + [label, weights] outs_static = traced_layer(ins_static) helper.assertEachVar(outs, outs_static) if program is not None: self.assertTrue( - is_equal_program(program, traced_layer.program)) + is_equal_program(program, traced_layer.program) + ) program = traced_layer.program traced_layer.save_inference_model( './infer_imperative_transformer', feed=list(range(len(ins_static))), - fetch=list(range(len(outs_static)))) + fetch=list(range(len(outs_static))), + ) else: outs = transformer(enc_inputs, dec_inputs, label, weights) @@ -1049,61 +1208,88 @@ class TestDygraphTransformerSortGradient(unittest.TestCase): dy_predict_value = dy_predict.numpy() dy_token_num_value = dy_token_num.numpy() - return dy_avg_cost_value, dy_sum_cost_value, dy_predict_value, dy_token_num_value, \ - dy_param_init, dy_param_updated + return ( + dy_avg_cost_value, + dy_sum_cost_value, + dy_predict_value, + dy_token_num_value, + dy_param_init, + dy_param_updated, + ) with guard(): fluid.set_flags({'FLAGS_sort_sum_gradient': True}) if _in_legacy_dygraph(): - dy_avg_cost_value, dy_sum_cost_value, dy_predict_value, dy_token_num_value, \ - dy_param_init, dy_param_updated = run_dygraph() + ( + dy_avg_cost_value, + dy_sum_cost_value, + dy_predict_value, + dy_token_num_value, + dy_param_init, + dy_param_updated, + ) = run_dygraph() with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - transformer = TransFormer(ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - use_py_reader=use_py_reader, - is_test=False, - is_sparse=is_sparse) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + transformer = TransFormer( + ModelHyperParams.src_vocab_size, + ModelHyperParams.trg_vocab_size, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.prepostprocess_dropout, + ModelHyperParams.attention_dropout, + ModelHyperParams.relu_dropout, + ModelHyperParams.preprocess_cmd, + ModelHyperParams.postprocess_cmd, + ModelHyperParams.weight_sharing, + TrainTaskConfig.label_smooth_eps, + use_py_reader=use_py_reader, + is_test=False, + is_sparse=is_sparse, + ) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) optimizer = fluid.optimizer.SGD(learning_rate=0.003) - data_input_names = encoder_data_input_fields + decoder_data_input_fields[: - -1] + label_data_input_fields + data_input_names = ( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + ) all_inputs = make_all_inputs(data_input_names) enc_inputs_len = len(encoder_data_input_fields) dec_inputs_len = len(decoder_data_input_fields[:-1]) enc_inputs = all_inputs[0:enc_inputs_len] - dec_inputs = all_inputs[enc_inputs_len:enc_inputs_len + - dec_inputs_len] + dec_inputs = all_inputs[ + enc_inputs_len : enc_inputs_len + dec_inputs_len + ] label = all_inputs[-2] weights = all_inputs[-1] static_param_updated = dict() static_param_init = dict() static_param_name_list = list() - static_sum_cost, static_avg_cost, static_predict, static_token_num = transformer( - enc_inputs, dec_inputs, label, weights) + ( + static_sum_cost, + static_avg_cost, + static_predict, + static_token_num, + ) = transformer(enc_inputs, dec_inputs, label, weights) optimizer.minimize(static_avg_cost) for param in transformer.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_sum_cost_value = None @@ -1113,31 +1299,40 @@ class TestDygraphTransformerSortGradient(unittest.TestCase): for i in range(batch_num): feed_dict = create_feed_dict_list(create_data(True)) fetch_list = [ - static_sum_cost, static_avg_cost, static_predict, - static_token_num + static_sum_cost, + static_avg_cost, + static_predict, + static_token_num, ] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed=feed_dict, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed=feed_dict, + fetch_list=fetch_list, + ) static_sum_cost_value = out[0] static_avg_cost_value = out[1] static_predict_value = out[2] static_token_num_value = out[3] if i == batch_num - 1: for k in range(4, len(out)): - static_param_updated[static_param_name_list[k - - 4]] = out[k] + static_param_updated[ + static_param_name_list[k - 4] + ] = out[k] if _in_legacy_dygraph(): - np.testing.assert_array_equal(static_avg_cost_value, - dy_avg_cost_value) - np.testing.assert_array_equal(static_sum_cost_value, - dy_sum_cost_value) - np.testing.assert_array_equal(static_predict_value, - dy_predict_value) - np.testing.assert_array_equal(static_token_num_value, - dy_token_num_value) + np.testing.assert_array_equal( + static_avg_cost_value, dy_avg_cost_value + ) + np.testing.assert_array_equal( + static_sum_cost_value, dy_sum_cost_value + ) + np.testing.assert_array_equal( + static_predict_value, dy_predict_value + ) + np.testing.assert_array_equal( + static_token_num_value, dy_token_num_value + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, dy_param_init[key]) @@ -1147,33 +1342,45 @@ class TestDygraphTransformerSortGradient(unittest.TestCase): # compare eager result with imperative result with guard(): fluid.set_flags({'FLAGS_sort_sum_gradient': False}) - dy_avg_cost_value, dy_sum_cost_value, dy_predict_value, dy_token_num_value, \ - dy_param_init, dy_param_updated = run_dygraph() + ( + dy_avg_cost_value, + dy_sum_cost_value, + dy_predict_value, + dy_token_num_value, + dy_param_init, + dy_param_updated, + ) = run_dygraph() with guard(): with _test_eager_guard(): - eager_avg_cost_value, eager_sum_cost_value, eager_predict_value, eager_token_num_value, \ - eager_param_init, eager_param_updated = run_dygraph() - np.testing.assert_allclose(dy_avg_cost_value, - eager_avg_cost_value, - rtol=1e-05) - np.testing.assert_allclose(dy_sum_cost_value, - eager_sum_cost_value, - rtol=1e-05) - - np.testing.assert_allclose(dy_predict_value, - eager_predict_value, - rtol=1e-05) - np.testing.assert_allclose(dy_token_num_value, - eager_token_num_value, - rtol=1e-05) + ( + eager_avg_cost_value, + eager_sum_cost_value, + eager_predict_value, + eager_token_num_value, + eager_param_init, + eager_param_updated, + ) = run_dygraph() + np.testing.assert_allclose( + dy_avg_cost_value, eager_avg_cost_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dy_sum_cost_value, eager_sum_cost_value, rtol=1e-05 + ) + + np.testing.assert_allclose( + dy_predict_value, eager_predict_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dy_token_num_value, eager_token_num_value, rtol=1e-05 + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, eager_param_init[key]) for key, value in dy_param_updated.items(): - np.testing.assert_allclose(value, - eager_param_updated[key], - rtol=1e-05) + np.testing.assert_allclose( + value, eager_param_updated[key], rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_triple_grad.py b/python/paddle/fluid/tests/unittests/test_imperative_triple_grad.py index 7f0ea807271..ebebe754b21 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_triple_grad.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_triple_grad.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard def _dygraph_guard_(func): - def __impl__(*args, **kwargs): if fluid._non_static_mode(): return func(*args, **kwargs) @@ -43,36 +42,37 @@ def random_var(size, low=-1, high=1, dtype='float32'): class TestDygraphTripleGradMatmul(TestCase): - def test_matmul_triple_grad(self): input_numpy = np.ones([3, 3]) * 2 with _test_eager_guard(): - x = paddle.to_tensor(input_numpy, - stop_gradient=False, - dtype='float32') - y = paddle.to_tensor(input_numpy, - stop_gradient=False, - dtype='float32') + x = paddle.to_tensor( + input_numpy, stop_gradient=False, dtype='float32' + ) + y = paddle.to_tensor( + input_numpy, stop_gradient=False, dtype='float32' + ) out = paddle.matmul(x, y, False, False) - new_out_g = paddle.to_tensor(np.ones([3, 3]), - stop_gradient=False, - dtype='float32') - new_x_g, new_y_g = paddle.grad([out], [x, y], [new_out_g], - retain_graph=True, - create_graph=True) - - new_x_g_g = paddle.to_tensor(np.ones([3, 3]), - stop_gradient=False, - dtype='float32') - new_y_g_g = paddle.to_tensor(np.ones([3, 3]), - stop_gradient=False, - dtype='float32') - new_a, new_b, new_c = paddle.grad([new_x_g, new_y_g], - [x, y, new_out_g], - [new_x_g_g, new_y_g_g], - retain_graph=True, - create_graph=True) + new_out_g = paddle.to_tensor( + np.ones([3, 3]), stop_gradient=False, dtype='float32' + ) + new_x_g, new_y_g = paddle.grad( + [out], [x, y], [new_out_g], retain_graph=True, create_graph=True + ) + + new_x_g_g = paddle.to_tensor( + np.ones([3, 3]), stop_gradient=False, dtype='float32' + ) + new_y_g_g = paddle.to_tensor( + np.ones([3, 3]), stop_gradient=False, dtype='float32' + ) + new_a, new_b, new_c = paddle.grad( + [new_x_g, new_y_g], + [x, y, new_out_g], + [new_x_g_g, new_y_g_g], + retain_graph=True, + create_graph=True, + ) new_a.backward() @@ -108,27 +108,30 @@ class TestDygraphTripleGradMatmul(TestCase): class TestDygraphTripleGrad(TestCase): - def setUp(self): self.sort_sum_gradient = False self.shape = [5, 5] - def grad(self, - outputs, - inputs, - grad_outputs=None, - no_grad_vars=None, - retain_graph=None, - create_graph=False, - allow_unused=False): + def grad( + self, + outputs, + inputs, + grad_outputs=None, + no_grad_vars=None, + retain_graph=None, + create_graph=False, + allow_unused=False, + ): fluid.set_flags({'FLAGS_sort_sum_gradient': self.sort_sum_gradient}) - return fluid.dygraph.grad(outputs=outputs, - inputs=inputs, - grad_outputs=grad_outputs, - no_grad_vars=no_grad_vars, - retain_graph=retain_graph, - create_graph=create_graph, - allow_unused=allow_unused) + return fluid.dygraph.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=grad_outputs, + no_grad_vars=no_grad_vars, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=allow_unused, + ) @dygraph_guard def func_exception(self): @@ -150,12 +153,16 @@ class TestDygraphTripleGrad(TestCase): self.grad([random_var(shape)], [1]) with self.assertRaises(AssertionError): - self.grad([random_var(shape), random_var(shape)], - [random_var(shape)], [random_var(shape)]) + self.grad( + [random_var(shape), random_var(shape)], + [random_var(shape)], + [random_var(shape)], + ) with self.assertRaises(AssertionError): - self.grad([random_var(shape)], [random_var(shape)], - no_grad_vars=[1]) + self.grad( + [random_var(shape)], [random_var(shape)], no_grad_vars=[1] + ) with self.assertRaises(AssertionError): self.grad([random_var(shape)], [random_var(shape)], no_grad_vars=1) @@ -178,34 +185,41 @@ class TestDygraphTripleGrad(TestCase): out = fluid.layers.sigmoid(paddle.matmul(x, y) + z) out_np = out.numpy() - dx_actual, = self.grad([out], [x], create_graph=True) + (dx_actual,) = self.grad([out], [x], create_graph=True) # Theoritical result based on math calculation dout = np.ones(self.shape).astype('float32') - dx_expected = np.matmul(dout * out_np * (1 - out_np), - np.transpose(y_np)) + dx_expected = np.matmul( + dout * out_np * (1 - out_np), np.transpose(y_np) + ) np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) - ddx_actual, = self.grad([dx_actual], [x], create_graph=True) + (ddx_actual,) = self.grad([dx_actual], [x], create_graph=True) # Theoritical result based on math calculation DDY = np.zeros(self.shape).astype('float32') DDX = np.ones(self.shape).astype('float32') - double_grad_tmp1 = np.matmul(dout * out_np * (1 - out_np), - np.transpose(DDY)) + double_grad_tmp1 = np.matmul( + dout * out_np * (1 - out_np), np.transpose(DDY) + ) double_grad_tmp2 = np.matmul(DDX, y_np) + np.matmul(x_np, DDY) double_grad_tmp3 = ( - 1 - 2 * out_np) * dout * double_grad_tmp2 * out_np * (1 - out_np) - ddx_expected = double_grad_tmp1 + np.matmul(double_grad_tmp3, - np.transpose(y_np)) + (1 - 2 * out_np) * dout * double_grad_tmp2 * out_np * (1 - out_np) + ) + ddx_expected = double_grad_tmp1 + np.matmul( + double_grad_tmp3, np.transpose(y_np) + ) np.testing.assert_allclose(ddx_actual.numpy(), ddx_expected, rtol=1e-05) # Theoritical result based on math calculation d_ddout = np.zeros(self.shape).astype('float32') tmp0 = np.matmul(DDX, y_np) + np.matmul(x_np, DDY) tmp1 = (1 - 2 * out_np) * ((1 - 2 * out_np) * dout * tmp0 * tmp0) - tmp2 = tmp0 * (1 - 2 * out_np) * d_ddout - 2 * dout * ( - 1 - out_np) * out_np * tmp0 * tmp0 - dddx_expected = np.matmul(((tmp1 + tmp2) * out_np * (1 - out_np)), - np.transpose(y_np)) + tmp2 = ( + tmp0 * (1 - 2 * out_np) * d_ddout + - 2 * dout * (1 - out_np) * out_np * tmp0 * tmp0 + ) + dddx_expected = np.matmul( + ((tmp1 + tmp2) * out_np * (1 - out_np)), np.transpose(y_np) + ) ddx_actual.backward() dddx_grad_actual = x.gradient() @@ -222,29 +236,32 @@ class TestDygraphTripleGrad(TestCase): class TestDygraphTripleGradBradcastCase(TestCase): - def setUp(self): self.sort_sum_gradient = False self.x_shape = [3, 2, 2] self.y_shape = [1, 2, 2] self.z_shape = [2, 2] - def grad(self, - outputs, - inputs, - grad_outputs=None, - no_grad_vars=None, - retain_graph=None, - create_graph=False, - allow_unused=False): + def grad( + self, + outputs, + inputs, + grad_outputs=None, + no_grad_vars=None, + retain_graph=None, + create_graph=False, + allow_unused=False, + ): fluid.set_flags({'FLAGS_sort_sum_gradient': self.sort_sum_gradient}) - return fluid.dygraph.grad(outputs=outputs, - inputs=inputs, - grad_outputs=grad_outputs, - no_grad_vars=no_grad_vars, - retain_graph=retain_graph, - create_graph=create_graph, - allow_unused=allow_unused) + return fluid.dygraph.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=grad_outputs, + no_grad_vars=no_grad_vars, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=allow_unused, + ) @dygraph_guard def func_example_with_gradient_and_create_graph(self): @@ -264,34 +281,42 @@ class TestDygraphTripleGradBradcastCase(TestCase): out = fluid.layers.sigmoid(paddle.matmul(x, y) + z) out_np = out.numpy() - dx_actual, = self.grad([out], [x], create_graph=True) + (dx_actual,) = self.grad([out], [x], create_graph=True) # Theoritical result based on math calculation dout = np.ones(self.x_shape).astype('float32') - dx_expected = np.matmul(dout * out_np * (1 - out_np), - np.transpose(y_np, axes=(0, 2, 1))) + dx_expected = np.matmul( + dout * out_np * (1 - out_np), np.transpose(y_np, axes=(0, 2, 1)) + ) np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) - ddx_actual, = self.grad([dx_actual], [x], create_graph=True) + (ddx_actual,) = self.grad([dx_actual], [x], create_graph=True) # Theoritical result based on math calculation DDY = np.zeros(self.y_shape).astype('float32') DDX = np.ones(self.x_shape).astype('float32') - double_grad_tmp1 = np.matmul(dout * out_np * (1 - out_np), - np.transpose(DDY, axes=(0, 2, 1))) + double_grad_tmp1 = np.matmul( + dout * out_np * (1 - out_np), np.transpose(DDY, axes=(0, 2, 1)) + ) double_grad_tmp2 = np.matmul(DDX, y_np) + np.matmul(x_np, DDY) double_grad_tmp3 = ( - 1 - 2 * out_np) * dout * double_grad_tmp2 * out_np * (1 - out_np) + (1 - 2 * out_np) * dout * double_grad_tmp2 * out_np * (1 - out_np) + ) ddx_expected = double_grad_tmp1 + np.matmul( - double_grad_tmp3, np.transpose(y_np, axes=(0, 2, 1))) + double_grad_tmp3, np.transpose(y_np, axes=(0, 2, 1)) + ) np.testing.assert_allclose(ddx_actual.numpy(), ddx_expected, rtol=1e-05) # Theoritical result based on math calculation d_ddout = np.zeros(self.x_shape).astype('float32') tmp0 = np.matmul(DDX, y_np) + np.matmul(x_np, DDY) tmp1 = (1 - 2 * out_np) * ((1 - 2 * out_np) * dout * tmp0 * tmp0) - tmp2 = tmp0 * (1 - 2 * out_np) * d_ddout - 2 * dout * ( - 1 - out_np) * out_np * tmp0 * tmp0 - dddx_expected = np.matmul(((tmp1 + tmp2) * out_np * (1 - out_np)), - np.transpose(y_np, axes=(0, 2, 1))) + tmp2 = ( + tmp0 * (1 - 2 * out_np) * d_ddout + - 2 * dout * (1 - out_np) * out_np * tmp0 * tmp0 + ) + dddx_expected = np.matmul( + ((tmp1 + tmp2) * out_np * (1 - out_np)), + np.transpose(y_np, axes=(0, 2, 1)), + ) ddx_actual.backward() dddx_grad_actual = x.gradient() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_using_non_zero_gpu.py b/python/paddle/fluid/tests/unittests/test_imperative_using_non_zero_gpu.py index 2cc157ae050..9d7ece63360 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_using_non_zero_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_using_non_zero_gpu.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestImperativeUsingNonZeroGpu(unittest.TestCase): - def run_main(self, np_arr, place): with guard(place): var = to_variable(np_arr) diff --git a/python/paddle/fluid/tests/unittests/test_increment.py b/python/paddle/fluid/tests/unittests/test_increment.py index 34c7af4ac08..4e435e55794 100755 --- a/python/paddle/fluid/tests/unittests/test_increment.py +++ b/python/paddle/fluid/tests/unittests/test_increment.py @@ -20,12 +20,11 @@ import paddle.fluid as fluid class TestIncrement(unittest.TestCase): - def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.layers.fill_constant(shape=[1], - dtype='int64', - value=5) + input = fluid.layers.fill_constant( + shape=[1], dtype='int64', value=5 + ) expected_result = np.array([8], dtype='int64') output = paddle.tensor.math.increment(input, value=3) @@ -41,7 +40,6 @@ class TestIncrement(unittest.TestCase): class TestInplaceApiWithDataTransform(unittest.TestCase): - def test_increment(self): if fluid.core.is_compiled_with_cuda(): paddle.enable_static() @@ -50,7 +48,7 @@ class TestInplaceApiWithDataTransform(unittest.TestCase): with paddle.fluid.device_guard("cpu"): x = paddle.increment(x) exe = paddle.static.Executor(paddle.CUDAPlace(0)) - a, = exe.run(paddle.static.default_main_program(), fetch_list=[x]) + (a,) = exe.run(paddle.static.default_main_program(), fetch_list=[x]) paddle.disable_static() self.assertEqual(a[0], 1) diff --git a/python/paddle/fluid/tests/unittests/test_index_add_op.py b/python/paddle/fluid/tests/unittests/test_index_add_op.py index f3b799f0d35..c54479b3529 100644 --- a/python/paddle/fluid/tests/unittests/test_index_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_index_add_op.py @@ -19,8 +19,9 @@ from op_test import OpTest from paddle.fluid import Program -def compute_index_add_ref(axis, x_shape, x_np, add_value_shape, add_value_np, - index_size, index_np): +def compute_index_add_ref( + axis, x_shape, x_np, add_value_shape, add_value_np, index_size, index_np +): if axis < 0: axis = axis + len(x_shape) if axis != 0: @@ -28,11 +29,13 @@ def compute_index_add_ref(axis, x_shape, x_np, add_value_shape, add_value_np, x_reshape = [outer_loop] + list(x_shape[axis:]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) - add_value_reshape = [np.prod(add_value_shape[:axis]).astype(int) - ] + list(add_value_shape[axis:]) + add_value_reshape = [ + np.prod(add_value_shape[:axis]).astype(int) + ] + list(add_value_shape[axis:]) - add_value_np_reshape = np.reshape(add_value_np, - tuple(add_value_reshape)) + add_value_np_reshape = np.reshape( + add_value_np, tuple(add_value_reshape) + ) else: x_np_reshape = x_np add_value_np_reshape = add_value_np @@ -54,23 +57,29 @@ def raw_index_add(x, index, value, axis): class TestIndexAddOp(OpTest): - def setUp(self): self.python_api = raw_index_add self.op_type = "index_add" self.init_dtype_type() - index_np = np.random.randint(low=0, - high=self.x_shape[self.axis], - size=self.index_size) + index_np = np.random.randint( + low=0, high=self.x_shape[self.axis], size=self.index_size + ) x_np = np.random.random(self.x_shape).astype(self.x_type) add_value_np = np.random.random(self.add_value_shape).astype( - self.x_type) + self.x_type + ) self.inputs = {'X': x_np, 'Index': index_np, 'AddValue': add_value_np} self.attrs = {'axis': self.axis} - out = compute_index_add_ref(self.axis, self.x_shape, x_np, - self.add_value_shape, add_value_np, - self.index_size, index_np) + out = compute_index_add_ref( + self.axis, + self.x_shape, + x_np, + self.add_value_shape, + add_value_np, + self.index_size, + index_np, + ) self.outputs = {'Out': out} def init_dtype_type(self): @@ -89,7 +98,6 @@ class TestIndexAddOp(OpTest): class TestIndexAddAPI(unittest.TestCase): - def setUp(self): self.setType() self.setPlace() @@ -126,11 +134,11 @@ class TestIndexAddAPI(unittest.TestCase): self.x_np = np.random.random(self.x_shape).astype(self.x_type) self.add_value_np = np.random.random(self.add_value_shape).astype( - self.x_type) - self.index_np = np.random.randint(low=0, - high=self.x_shape[axis], - size=self.index_size).astype( - self.index_type) + self.x_type + ) + self.index_np = np.random.randint( + low=0, high=self.x_shape[axis], size=self.index_size + ).astype(self.index_type) if self.check_backward: self.dout_np = np.random.random(self.x_shape).astype(self.x_type) @@ -154,36 +162,47 @@ class TestIndexAddAPI(unittest.TestCase): add_value = paddle.to_tensor(self.add_value_np, stop_gradient=False) out = paddle.index_add(input_tensor, index, self.axis, add_value) - ref_out = compute_index_add_ref(self.axis, self.x_shape, self.x_np, - self.add_value_shape, self.add_value_np, - self.index_size, self.index_np) - np.testing.assert_allclose(ref_out, - out.numpy(), - rtol=self.rtol, - atol=self.atol) + ref_out = compute_index_add_ref( + self.axis, + self.x_shape, + self.x_np, + self.add_value_shape, + self.add_value_np, + self.index_size, + self.index_np, + ) + np.testing.assert_allclose( + ref_out, out.numpy(), rtol=self.rtol, atol=self.atol + ) if self.check_backward: dout_tensor = paddle.to_tensor(self.dout_np) paddle.autograd.backward([out], [dout_tensor], retain_graph=True) - ref_x_grad, ref_add_value_grad = self.compute_index_add_backward_ref( + ( + ref_x_grad, + ref_add_value_grad, + ) = self.compute_index_add_backward_ref() + np.testing.assert_allclose( + ref_x_grad, + input_tensor.grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) + np.testing.assert_allclose( + ref_add_value_grad, + add_value.grad.numpy(), + rtol=self.rtol, + atol=self.atol, ) - np.testing.assert_allclose(ref_x_grad, - input_tensor.grad.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(ref_add_value_grad, - add_value.grad.numpy(), - rtol=self.rtol, - atol=self.atol) def run_static(self, device): x = paddle.static.data(name='X', shape=self.x_shape, dtype=self.x_type) - index = paddle.static.data(name='Index', - shape=self.index_shape, - dtype=self.index_type) - add_value = paddle.static.data(name='AddValue', - shape=self.add_value_shape, - dtype=self.x_type) + index = paddle.static.data( + name='Index', shape=self.index_shape, dtype=self.index_type + ) + add_value = paddle.static.data( + name='AddValue', shape=self.add_value_shape, dtype=self.x_type + ) out = paddle.index_add(x, index, self.axis, add_value) @@ -193,19 +212,22 @@ class TestIndexAddAPI(unittest.TestCase): place = paddle.CUDAPlace(0) else: raise TypeError( - "paddle.index_add api only support cpu and gpu device now.") + "paddle.index_add api only support cpu and gpu device now." + ) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) - res = exe.run(paddle.static.default_main_program(), - feed={ - "X": self.x_np, - "Index": self.index_np, - "AddValue": self.add_value_np, - }, - fetch_list=[out.name], - return_numpy=False) + res = exe.run( + paddle.static.default_main_program(), + feed={ + "X": self.x_np, + "Index": self.index_np, + "AddValue": self.add_value_np, + }, + fetch_list=[out.name], + return_numpy=False, + ) return res def test_static(self): @@ -213,14 +235,18 @@ class TestIndexAddAPI(unittest.TestCase): for device in self.place: with paddle.static.program_guard(Program()): out = self.run_static(device) - ref_out = compute_index_add_ref(self.axis, self.x_shape, self.x_np, - self.add_value_shape, - self.add_value_np, self.index_size, - self.index_np) - np.testing.assert_allclose(ref_out, - np.array(out[0]), - rtol=self.rtol, - atol=self.atol) + ref_out = compute_index_add_ref( + self.axis, + self.x_shape, + self.x_np, + self.add_value_shape, + self.add_value_np, + self.index_size, + self.index_np, + ) + np.testing.assert_allclose( + ref_out, np.array(out[0]), rtol=self.rtol, atol=self.atol + ) def test_dynamic(self): paddle.disable_static() @@ -229,14 +255,12 @@ class TestIndexAddAPI(unittest.TestCase): class TestIndexAddAPIMoreType(TestIndexAddAPI): - def setType(self): self.x_type = np.float64 self.index_type = np.int64 class TestIndexAddAPICase2(TestIndexAddAPI): - def config(self): self.axis = 1 self.x_shape = (100, 100, 5) @@ -245,7 +269,6 @@ class TestIndexAddAPICase2(TestIndexAddAPI): class TestIndexAddAPICase3(TestIndexAddAPI): - def config(self): self.axis = 2 self.x_shape = (100, 100, 25) @@ -254,16 +277,14 @@ class TestIndexAddAPICase3(TestIndexAddAPI): class TestIndexAddAPICase4(TestIndexAddAPI): - def config(self): self.axis = 0 - self.x_shape = (10, ) + self.x_shape = (10,) self.index_size = 4 - self.add_value_shape = (4, ) + self.add_value_shape = (4,) class TestIndexAddAPICase5(TestIndexAddAPI): - def config(self): self.axis = -1 self.x_shape = (10, 10) diff --git a/python/paddle/fluid/tests/unittests/test_index_sample_op.py b/python/paddle/fluid/tests/unittests/test_index_sample_op.py index 7c4be22305b..550ddfe344f 100644 --- a/python/paddle/fluid/tests/unittests/test_index_sample_op.py +++ b/python/paddle/fluid/tests/unittests/test_index_sample_op.py @@ -20,16 +20,14 @@ from op_test import OpTest class TestIndexSampleOp(OpTest): - def setUp(self): self.op_type = "index_sample" self.python_api = paddle.index_sample self.config() xnp = np.random.random(self.x_shape).astype(self.x_type) - indexnp = np.random.randint(low=0, - high=self.x_shape[1], - size=self.index_shape).astype( - self.index_type) + indexnp = np.random.randint( + low=0, high=self.x_shape[1], size=self.index_shape + ).astype(self.index_type) self.inputs = {'X': xnp, 'Index': indexnp} index_array = [] for i in range(self.index_shape[0]): @@ -56,7 +54,6 @@ class TestIndexSampleOp(OpTest): class TestCase1(TestIndexSampleOp): - def config(self): """ For one dimension input @@ -68,7 +65,6 @@ class TestCase1(TestIndexSampleOp): class TestCase2(TestIndexSampleOp): - def config(self): """ For int64_t index type @@ -80,7 +76,6 @@ class TestCase2(TestIndexSampleOp): class TestCase3(TestIndexSampleOp): - def config(self): """ For int index type @@ -92,7 +87,6 @@ class TestCase3(TestIndexSampleOp): class TestCase4(TestIndexSampleOp): - def config(self): """ For int64 index type @@ -104,7 +98,6 @@ class TestCase4(TestIndexSampleOp): class TestIndexSampleShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() # create x value @@ -115,8 +108,9 @@ class TestIndexSampleShape(unittest.TestCase): # create index value index_shape = (2, 3) index_type = "int32" - index_np = np.random.randint(low=0, high=x_shape[1], - size=index_shape).astype(index_type) + index_np = np.random.randint( + low=0, high=x_shape[1], size=index_shape + ).astype(index_type) x = fluid.data(name='x', shape=[-1, 5], dtype='float64') index = fluid.data(name='index', shape=[-1, 3], dtype='int32') @@ -131,18 +125,24 @@ class TestIndexSampleShape(unittest.TestCase): class TestIndexSampleDynamic(unittest.TestCase): - def test_result(self): with fluid.dygraph.guard(): - x = paddle.to_tensor([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]], - dtype='float32') - index = paddle.to_tensor([[0, 1, 2], [1, 2, 3], [0, 0, 0]], - dtype='int32') + x = paddle.to_tensor( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + dtype='float32', + ) + index = paddle.to_tensor( + [[0, 1, 2], [1, 2, 3], [0, 0, 0]], dtype='int32' + ) out_z1 = paddle.index_sample(x, index) - except_output = np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0], - [9.0, 9.0, 9.0]]) + except_output = np.array( + [[1.0, 2.0, 3.0], [6.0, 7.0, 8.0], [9.0, 9.0, 9.0]] + ) assert out_z1.numpy().all() == except_output.all() diff --git a/python/paddle/fluid/tests/unittests/test_index_select_op.py b/python/paddle/fluid/tests/unittests/test_index_select_op.py index 2d73a29e2c9..edb2dc71182 100644 --- a/python/paddle/fluid/tests/unittests/test_index_select_op.py +++ b/python/paddle/fluid/tests/unittests/test_index_select_op.py @@ -21,19 +21,18 @@ from paddle.fluid import Program, program_guard class TestIndexSelectOp(OpTest): - def setUp(self): self.python_api = paddle.index_select self.op_type = "index_select" self.init_dtype_type() - index_np = np.random.randint(low=0, - high=self.x_shape[self.dim], - size=self.index_size) + index_np = np.random.randint( + low=0, high=self.x_shape[self.dim], size=self.index_size + ) x_np = np.random.random(self.x_shape).astype(self.x_type) self.inputs = {'X': x_np, 'Index': index_np} self.attrs = {'dim': self.dim} - outer_loop = np.prod(self.x_shape[:self.dim]) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + outer_loop = np.prod(self.x_shape[: self.dim]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] for i in range(outer_loop): @@ -61,7 +60,6 @@ class TestIndexSelectOp(OpTest): class TestIndexSelectOpCase2(TestIndexSelectOp): - def init_dtype_type(self): self.x_type = np.float32 self.index_type = np.int32 @@ -71,7 +69,6 @@ class TestIndexSelectOpCase2(TestIndexSelectOp): class TestIndexSelectOpCaseSingleThread(TestIndexSelectOp): - def init_dtype_type(self): if fluid.is_compiled_with_cuda(): fluid.set_flags({'FLAGS_cudnn_deterministic': True}) @@ -83,10 +80,14 @@ class TestIndexSelectOpCaseSingleThread(TestIndexSelectOp): class TestIndexSelectAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]]) + self.data_x = np.array( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] + ) self.data_index = np.array([0, 1, 1]).astype('int32') def test_index_select_api(self): @@ -95,39 +96,37 @@ class TestIndexSelectAPI(unittest.TestCase): # case 1: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='index', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='index', shape=[3], dtype='int32', append_batch_size=False + ) z = paddle.index_select(x, index, axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 2: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='index', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='index', shape=[3], dtype='int32', append_batch_size=False + ) z = paddle.index_select(x, index) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) def test_dygraph_api(self): @@ -138,8 +137,9 @@ class TestIndexSelectAPI(unittest.TestCase): index = fluid.dygraph.to_variable(self.data_index) z = paddle.index_select(x, index) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) # case 2: @@ -148,8 +148,9 @@ class TestIndexSelectAPI(unittest.TestCase): index = fluid.dygraph.to_variable(self.data_index) z = paddle.index_select(x, index, axis=1) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_infer_no_need_buffer_slots.py b/python/paddle/fluid/tests/unittests/test_infer_no_need_buffer_slots.py index 51f6cd53ca3..659aa36c334 100644 --- a/python/paddle/fluid/tests/unittests/test_infer_no_need_buffer_slots.py +++ b/python/paddle/fluid/tests/unittests/test_infer_no_need_buffer_slots.py @@ -20,12 +20,17 @@ import paddle.fluid.core as core class TestInferNoNeedBufferSlots(unittest.TestCase): - def net(self): - x1 = fluid.default_main_program().global_block().create_var( - dtype="float32", shape=[1], lod_level=0, name="x1") - x2 = fluid.default_main_program().global_block().create_var( - dtype="float32", shape=[1], lod_level=0, name="x2") + x1 = ( + fluid.default_main_program() + .global_block() + .create_var(dtype="float32", shape=[1], lod_level=0, name="x1") + ) + x2 = ( + fluid.default_main_program() + .global_block() + .create_var(dtype="float32", shape=[1], lod_level=0, name="x2") + ) x = fluid.layers.elementwise_add(x1, x2) return x @@ -52,18 +57,27 @@ class TestInferNoNeedBufferSlots(unittest.TestCase): if idx == 0: # elementwise_add op self.assertEqual( - core.infer_no_need_buffer_slots(op.type, inputs, outputs, - attrs), set([])) + core.infer_no_need_buffer_slots( + op.type, inputs, outputs, attrs + ), + set([]), + ) elif idx == 1: # fill constant op self.assertEqual( - core.infer_no_need_buffer_slots(op.type, inputs, outputs, - attrs), set([])) + core.infer_no_need_buffer_slots( + op.type, inputs, outputs, attrs + ), + set([]), + ) else: # elementwise_add_grad op self.assertEqual( - core.infer_no_need_buffer_slots(op.type, inputs, outputs, - attrs), set(['Y', 'X'])) + core.infer_no_need_buffer_slots( + op.type, inputs, outputs, attrs + ), + set(['Y', 'X']), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_infer_shape.py b/python/paddle/fluid/tests/unittests/test_infer_shape.py index f4c718bf80a..df2ba25afbc 100644 --- a/python/paddle/fluid/tests/unittests/test_infer_shape.py +++ b/python/paddle/fluid/tests/unittests/test_infer_shape.py @@ -18,7 +18,6 @@ import paddle.fluid.core as core class TestInferShape(unittest.TestCase): - def test_sum_op(self): prog = core.ProgramDesc() self.assertIsNotNone(prog) diff --git a/python/paddle/fluid/tests/unittests/test_inference_api.py b/python/paddle/fluid/tests/unittests/test_inference_api.py index 27ddf815aa0..e62b258fefe 100644 --- a/python/paddle/fluid/tests/unittests/test_inference_api.py +++ b/python/paddle/fluid/tests/unittests/test_inference_api.py @@ -25,37 +25,46 @@ from paddle.inference import get_trt_compile_version, get_trt_runtime_version class TestInferenceApi(unittest.TestCase): - def test_inference_api(self): tensor32 = np.random.randint(10, 20, size=[20, 2]).astype('int32') paddletensor32 = PaddleTensor(tensor32) dtype32 = paddletensor32.dtype self.assertEqual(dtype32, PaddleDType.INT32) - self.assertEqual(paddletensor32.data.tolist('int32'), - tensor32.ravel().tolist()) + self.assertEqual( + paddletensor32.data.tolist('int32'), tensor32.ravel().tolist() + ) paddletensor32.data.reset(tensor32) - self.assertEqual(paddletensor32.as_ndarray().ravel().tolist(), - tensor32.ravel().tolist()) + self.assertEqual( + paddletensor32.as_ndarray().ravel().tolist(), + tensor32.ravel().tolist(), + ) tensor64 = np.random.randint(10, 20, size=[20, 2]).astype('int64') paddletensor64 = PaddleTensor(tensor64) dtype64 = paddletensor64.dtype self.assertEqual(dtype64, PaddleDType.INT64) - self.assertEqual(paddletensor64.data.tolist('int64'), - tensor64.ravel().tolist()) + self.assertEqual( + paddletensor64.data.tolist('int64'), tensor64.ravel().tolist() + ) paddletensor64.data.reset(tensor64) - self.assertEqual(paddletensor64.as_ndarray().ravel().tolist(), - tensor64.ravel().tolist()) + self.assertEqual( + paddletensor64.as_ndarray().ravel().tolist(), + tensor64.ravel().tolist(), + ) tensor_float = np.random.randn(20, 2).astype('float32') paddletensor_float = PaddleTensor(tensor_float) dtype_float = paddletensor_float.dtype self.assertEqual(dtype_float, PaddleDType.FLOAT32) - self.assertEqual(paddletensor_float.data.tolist('float32'), - tensor_float.ravel().tolist()) + self.assertEqual( + paddletensor_float.data.tolist('float32'), + tensor_float.ravel().tolist(), + ) paddletensor_float.data.reset(tensor_float) - self.assertEqual(paddletensor_float.as_ndarray().ravel().tolist(), - tensor_float.ravel().tolist()) + self.assertEqual( + paddletensor_float.as_ndarray().ravel().tolist(), + tensor_float.ravel().tolist(), + ) def get_sample_model(): @@ -66,24 +75,26 @@ def get_sample_model(): startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): data = fluid.data(name="data", shape=[-1, 6, 64, 64], dtype="float32") - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - groups=1, - padding=0, - bias_attr=False, - act=None) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + groups=1, + padding=0, + bias_attr=False, + act=None, + ) exe.run(startup_program) - serialized_program = paddle.static.serialize_program(data, - conv_out, - program=main_program) + serialized_program = paddle.static.serialize_program( + data, conv_out, program=main_program + ) serialized_params = paddle.static.serialize_persistables( - data, conv_out, executor=exe, program=main_program) + data, conv_out, executor=exe, program=main_program + ) return serialized_program, serialized_params class TestInferenceBaseAPI(unittest.TestCase): - def get_config(self, model, params): config = Config() config.set_model_buffer(model, len(model), params, len(params)) diff --git a/python/paddle/fluid/tests/unittests/test_inference_model_io.py b/python/paddle/fluid/tests/unittests/test_inference_model_io.py index f63368dffc8..bf9ac53f99c 100644 --- a/python/paddle/fluid/tests/unittests/test_inference_model_io.py +++ b/python/paddle/fluid/tests/unittests/test_inference_model_io.py @@ -28,13 +28,16 @@ import paddle.fluid.layers as layers import paddle.fluid.optimizer as optimizer from paddle.fluid.compiler import CompiledProgram from paddle.fluid.framework import Program, program_guard -from paddle.fluid.io import save_inference_model, load_inference_model, save_persistables +from paddle.fluid.io import ( + save_inference_model, + load_inference_model, + save_persistables, +) paddle.enable_static() class InferModel(object): - def __init__(self, list): self.program = list[0] self.feed_var_names = list[1] @@ -42,7 +45,6 @@ class InferModel(object): class TestBook(unittest.TestCase): - def test_fit_line_inference_model(self): root_path = tempfile.TemporaryDirectory() MODEL_DIR = os.path.join(root_path.name, "inference_model") @@ -69,31 +71,36 @@ class TestBook(unittest.TestCase): exe.run(init_program, feed={}, fetch_list=[]) for i in range(100): - tensor_x = np.array([[1, 1], [1, 2], [3, 4], [5, - 2]]).astype("float32") + tensor_x = np.array([[1, 1], [1, 2], [3, 4], [5, 2]]).astype( + "float32" + ) tensor_y = np.array([[-2], [-3], [-7], [-7]]).astype("float32") - exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost]) + exe.run( + program, + feed={'x': tensor_x, 'y': tensor_y}, + fetch_list=[avg_cost], + ) # Separated model and unified model save_inference_model(MODEL_DIR, ["x", "y"], [avg_cost], exe, program) - save_inference_model(UNI_MODEL_DIR, ["x", "y"], [avg_cost], exe, - program, 'model', 'params') + save_inference_model( + UNI_MODEL_DIR, + ["x", "y"], + [avg_cost], + exe, + program, + 'model', + 'params', + ) main_program = program.clone()._prune_with_input( - feeded_var_names=["x", "y"], targets=[avg_cost]) + feeded_var_names=["x", "y"], targets=[avg_cost] + ) params_str = save_persistables(exe, None, main_program, None) - expected = exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost])[0] + expected = exe.run( + program, feed={'x': tensor_x, 'y': tensor_y}, fetch_list=[avg_cost] + )[0] importlib.reload(executor) # reload to build a new scope @@ -101,15 +108,18 @@ class TestBook(unittest.TestCase): with open(os.path.join(UNI_MODEL_DIR, 'model'), "rb") as f: model_str = f.read() model_1 = InferModel( - load_inference_model(None, exe, model_str, params_str)) + load_inference_model(None, exe, model_str, params_str) + ) for model in [model_0, model_1]: - outs = exe.run(model.program, - feed={ - model.feed_var_names[0]: tensor_x, - model.feed_var_names[1]: tensor_y - }, - fetch_list=model.fetch_vars) + outs = exe.run( + model.program, + feed={ + model.feed_var_names[0]: tensor_x, + model.feed_var_names[1]: tensor_y, + }, + fetch_list=model.fetch_vars, + ) actual = outs[0] self.assertEqual(model.feed_var_names, ["x", "y"]) @@ -119,12 +129,17 @@ class TestBook(unittest.TestCase): root_path.cleanup() - self.assertRaises(ValueError, fluid.io.load_inference_model, None, exe, - model_str, None) + self.assertRaises( + ValueError, + fluid.io.load_inference_model, + None, + exe, + model_str, + None, + ) class TestSaveInferenceModel(unittest.TestCase): - def test_save_inference_model(self): root_path = tempfile.TemporaryDirectory() MODEL_DIR = os.path.join(root_path.name, "inference_model2") @@ -160,8 +175,9 @@ class TestSaveInferenceModel(unittest.TestCase): y = layers.data(name='y', shape=[1], dtype='int32') predict = fluid.layers.fc(input=x, size=2, act='softmax') acc = fluid.layers.accuracy(input=predict, label=y) - auc_var, batch_auc_var, auc_states = fluid.layers.auc(input=predict, - label=y) + auc_var, batch_auc_var, auc_states = fluid.layers.auc( + input=predict, label=y + ) cost = fluid.layers.cross_entropy(input=predict, label=y) avg_cost = paddle.mean(x=cost) @@ -170,8 +186,9 @@ class TestSaveInferenceModel(unittest.TestCase): exe.run(init_program, feed={}, fetch_list=[]) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - save_inference_model(MODEL_DIR, ["x", "y"], [avg_cost], exe, - program) + save_inference_model( + MODEL_DIR, ["x", "y"], [avg_cost], exe, program + ) root_path.cleanup() expected_warn = "please ensure that you have set the auc states to zeros before saving inference model" self.assertTrue(len(w) > 0) @@ -179,7 +196,6 @@ class TestSaveInferenceModel(unittest.TestCase): class TestInstance(unittest.TestCase): - def test_save_inference_model(self): root_path = tempfile.TemporaryDirectory() MODEL_DIR = os.path.join(root_path.name, "inference_model3") @@ -203,16 +219,19 @@ class TestInstance(unittest.TestCase): # will print warning message cp_prog = CompiledProgram(program).with_data_parallel( - loss_name=avg_cost.name) + loss_name=avg_cost.name + ) save_inference_model(MODEL_DIR, ["x", "y"], [avg_cost], exe, cp_prog) - self.assertRaises(TypeError, save_inference_model, - [MODEL_DIR, ["x", "y"], [avg_cost], [], cp_prog]) + self.assertRaises( + TypeError, + save_inference_model, + [MODEL_DIR, ["x", "y"], [avg_cost], [], cp_prog], + ) root_path.cleanup() class TestSaveInferenceModelNew(unittest.TestCase): - def test_save_and_load_inference_model(self): root_path = tempfile.TemporaryDirectory() MODEL_DIR = os.path.join(root_path.name, "inference_model5") @@ -239,86 +258,143 @@ class TestSaveInferenceModelNew(unittest.TestCase): tensor_x = np.array([[1, 1], [1, 2], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7]]).astype("float32") for i in range(3): - exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost]) - - self.assertRaises(ValueError, paddle.static.save_inference_model, None, - ['x', 'y'], [avg_cost], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR + "/", [x, y], [avg_cost], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR, ['x', 'y'], [avg_cost], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR, 'x', [avg_cost], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR, [x, y], ['avg_cost'], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR, [x, y], 'avg_cost', exe) + exe.run( + program, + feed={'x': tensor_x, 'y': tensor_y}, + fetch_list=[avg_cost], + ) + + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + None, + ['x', 'y'], + [avg_cost], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR + "/", + [x, y], + [avg_cost], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR, + ['x', 'y'], + [avg_cost], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR, + 'x', + [avg_cost], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR, + [x, y], + ['avg_cost'], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR, + [x, y], + 'avg_cost', + exe, + ) model_path = MODEL_DIR + "_isdir.pdmodel" os.makedirs(model_path) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR + "_isdir", [x, y], [avg_cost], exe) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR + "_isdir", + [x, y], + [avg_cost], + exe, + ) os.rmdir(model_path) params_path = MODEL_DIR + "_isdir.pdmodel" os.makedirs(params_path) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR + "_isdir", [x, y], [avg_cost], exe) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR + "_isdir", + [x, y], + [avg_cost], + exe, + ) os.rmdir(params_path) - paddle.static.io.save_inference_model(MODEL_DIR, [x, y], [avg_cost], - exe) + paddle.static.io.save_inference_model( + MODEL_DIR, [x, y], [avg_cost], exe + ) self.assertTrue(os.path.exists(MODEL_DIR + ".pdmodel")) self.assertTrue(os.path.exists(MODEL_DIR + ".pdiparams")) - expected = exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost])[0] + expected = exe.run( + program, feed={'x': tensor_x, 'y': tensor_y}, fetch_list=[avg_cost] + )[0] importlib.reload(executor) # reload to build a new scope - self.assertRaises(ValueError, paddle.static.load_inference_model, None, - exe) - self.assertRaises(ValueError, paddle.static.load_inference_model, - MODEL_DIR + "/", exe) - self.assertRaises(ValueError, paddle.static.load_inference_model, - [MODEL_DIR], exe) - self.assertRaises(ValueError, - paddle.static.load_inference_model, - MODEL_DIR, - exe, - pserver_endpoints=None) - self.assertRaises(ValueError, - paddle.static.load_inference_model, - MODEL_DIR, - exe, - unsupported_param=None) - self.assertRaises((TypeError, ValueError), - paddle.static.load_inference_model, - None, - exe, - model_filename="illegal", - params_filename="illegal") - - model = InferModel(paddle.static.io.load_inference_model( - MODEL_DIR, exe)) + self.assertRaises( + ValueError, paddle.static.load_inference_model, None, exe + ) + self.assertRaises( + ValueError, paddle.static.load_inference_model, MODEL_DIR + "/", exe + ) + self.assertRaises( + ValueError, paddle.static.load_inference_model, [MODEL_DIR], exe + ) + self.assertRaises( + ValueError, + paddle.static.load_inference_model, + MODEL_DIR, + exe, + pserver_endpoints=None, + ) + self.assertRaises( + ValueError, + paddle.static.load_inference_model, + MODEL_DIR, + exe, + unsupported_param=None, + ) + self.assertRaises( + (TypeError, ValueError), + paddle.static.load_inference_model, + None, + exe, + model_filename="illegal", + params_filename="illegal", + ) + + model = InferModel( + paddle.static.io.load_inference_model(MODEL_DIR, exe) + ) root_path.cleanup() - outs = exe.run(model.program, - feed={ - model.feed_var_names[0]: tensor_x, - model.feed_var_names[1]: tensor_y - }, - fetch_list=model.fetch_vars) + outs = exe.run( + model.program, + feed={ + model.feed_var_names[0]: tensor_x, + model.feed_var_names[1]: tensor_y, + }, + fetch_list=model.fetch_vars, + ) actual = outs[0] self.assertEqual(model.feed_var_names, ["x", "y"]) @@ -359,12 +435,11 @@ class TestSaveInferenceModelNew(unittest.TestCase): tensor_x = np.array([[1, 1], [1, 2], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7]]).astype("float32") for i in range(3): - exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost]) + exe.run( + program, + feed={'x': tensor_x, 'y': tensor_y}, + fetch_list=[avg_cost], + ) # test if return type of serialize_program is bytes res1 = paddle.static.io.serialize_program([x, y], [avg_cost]) @@ -375,8 +450,13 @@ class TestSaveInferenceModelNew(unittest.TestCase): # test if variables in program is empty res = paddle.static.io._serialize_persistables(Program(), None) self.assertEqual(res, None) - self.assertRaises(TypeError, paddle.static.io.deserialize_persistables, - None, None, None) + self.assertRaises( + TypeError, + paddle.static.io.deserialize_persistables, + None, + None, + None, + ) def test_normalize_program(self): init_program = fluid.default_startup_program() @@ -402,34 +482,40 @@ class TestSaveInferenceModelNew(unittest.TestCase): tensor_x = np.array([[1, 1], [1, 2], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7]]).astype("float32") for i in range(3): - exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost]) + exe.run( + program, + feed={'x': tensor_x, 'y': tensor_y}, + fetch_list=[avg_cost], + ) # test if return type of serialize_program is bytes res = paddle.static.normalize_program(program, [x, y], [avg_cost]) self.assertTrue(isinstance(res, Program)) # test program type - self.assertRaises(TypeError, paddle.static.normalize_program, None, - [x, y], [avg_cost]) + self.assertRaises( + TypeError, paddle.static.normalize_program, None, [x, y], [avg_cost] + ) # test feed_vars type - self.assertRaises(TypeError, paddle.static.normalize_program, program, - 'x', [avg_cost]) + self.assertRaises( + TypeError, paddle.static.normalize_program, program, 'x', [avg_cost] + ) # test fetch_vars type - self.assertRaises(TypeError, paddle.static.normalize_program, program, - [x, y], 'avg_cost') + self.assertRaises( + TypeError, + paddle.static.normalize_program, + program, + [x, y], + 'avg_cost', + ) class TestLoadInferenceModelError(unittest.TestCase): - def test_load_model_not_exist(self): place = core.CPUPlace() exe = executor.Executor(place) - self.assertRaises(ValueError, load_inference_model, - './test_not_exist_dir', exe) + self.assertRaises( + ValueError, load_inference_model, './test_not_exist_dir', exe + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_initializer.py b/python/paddle/fluid/tests/unittests/test_initializer.py index 24a384b3aa0..4454d7a9bb8 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer.py +++ b/python/paddle/fluid/tests/unittests/test_initializer.py @@ -27,9 +27,11 @@ DELTA = 0.00001 def check_cast_op(op): - return op.type == 'cast' and \ - op.attr('in_dtype') == VarDesc.VarType.FP32 and \ - op.attr('out_dtype') in [VarDesc.VarType.FP16, VarDesc.VarType.BF16] + return ( + op.type == 'cast' + and op.attr('in_dtype') == VarDesc.VarType.FP32 + and op.attr('out_dtype') in [VarDesc.VarType.FP16, VarDesc.VarType.BF16] + ) def output_hist(out): @@ -41,21 +43,21 @@ def output_hist(out): class TestConstantInitializer(unittest.TestCase): - def test_calculate_gain(self): self.assertEqual(paddle.nn.initializer.calculate_gain('sigmoid'), 1) self.assertEqual(paddle.nn.initializer.calculate_gain('linear'), 1) self.assertEqual(paddle.nn.initializer.calculate_gain('conv2d'), 1) self.assertEqual(paddle.nn.initializer.calculate_gain('tanh'), 5.0 / 3) - self.assertEqual(paddle.nn.initializer.calculate_gain('relu'), - math.sqrt(2.0)) - self.assertEqual(paddle.nn.initializer.calculate_gain('leaky_relu', 1), - 1) + self.assertEqual( + paddle.nn.initializer.calculate_gain('relu'), math.sqrt(2.0) + ) + self.assertEqual( + paddle.nn.initializer.calculate_gain('leaky_relu', 1), 1 + ) self.assertEqual(paddle.nn.initializer.calculate_gain('selu'), 3.0 / 4) def test_constant_initializer_default_value(self, dtype="float32"): - """Test the constant initializer with default value - """ + """Test the constant initializer with default value""" program = framework.Program() block = program.global_block() for _ in range(2): @@ -64,7 +66,8 @@ class TestConstantInitializer(unittest.TestCase): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.ConstantInitializer()) + initializer=initializer.ConstantInitializer(), + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -73,8 +76,7 @@ class TestConstantInitializer(unittest.TestCase): return block def test_constant_initializer(self, dtype="float32"): - """Test constant initializer with supplied value - """ + """Test constant initializer with supplied value""" program = framework.Program() block = program.global_block() for _ in range(2): @@ -83,7 +85,8 @@ class TestConstantInitializer(unittest.TestCase): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.ConstantInitializer(2.3)) + initializer=initializer.ConstantInitializer(2.3), + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -92,32 +95,31 @@ class TestConstantInitializer(unittest.TestCase): return block def test_constant_initializer_fp16(self): - """Test constant initializer with float16 - """ + """Test constant initializer with float16""" self.test_constant_initializer_default_value("float16") self.test_constant_initializer("float16") def test_constant_initializer_bf16(self): """Test constant initializer with bfloat16 - No cast operator has been added here + No cast operator has been added here """ self.test_constant_initializer_default_value("uint16") self.test_constant_initializer("uint16") class TestUniformInitializer(unittest.TestCase): - def test_uniform_initializer_default_value(self, dtype="float32"): - """Test the uniform initializer with default value - """ + """Test the uniform initializer with default value""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer()) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -128,40 +130,42 @@ class TestUniformInitializer(unittest.TestCase): return block def test_uniform_initializer_random_seed(self): - """Test the uniform initializer with manually setting seed - """ + """Test the uniform initializer with manually setting seed""" program = framework.Program() program.random_seed = 123 block = program.global_block() for _ in range(2): - block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="param1", - initializer=initializer.UniformInitializer()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param1", + initializer=initializer.UniformInitializer(), + ) block.create_parameter( dtype="float32", shape=[5, 10], lod_level=0, name="param2", - initializer=initializer.UniformInitializer(seed=456)) + initializer=initializer.UniformInitializer(seed=456), + ) init_op = block.ops[1] self.assertEqual(init_op.attr("seed"), 456) init_op1 = block.ops[0] self.assertEqual(init_op1.attr("seed"), 123) def test_uniform_initializer(self, dtype="float32"): - """Test uniform initializer with supplied attributes - """ + """Test uniform initializer with supplied attributes""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer( - -4.2, 3.1, 123)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(-4.2, 3.1, 123), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -172,17 +176,17 @@ class TestUniformInitializer(unittest.TestCase): return block def test_uniform_initializer_two_op(self, dtype="float32"): - """Test uniform initializer with supplied attributes - """ + """Test uniform initializer with supplied attributes""" program = framework.Program() block = program.global_block() for i in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer( - -4.2, float(i), 123)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(-4.2, float(i), 123), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op0 = block.ops[0] @@ -193,8 +197,7 @@ class TestUniformInitializer(unittest.TestCase): return block def test_uniform_initializer_fp16(self): - """Test uniform initializer with float16 - """ + """Test uniform initializer with float16""" block = self.test_uniform_initializer_default_value("float16") self.assertTrue(check_cast_op(block.ops[1])) block = self.test_uniform_initializer(dtype="float16") @@ -204,7 +207,7 @@ class TestUniformInitializer(unittest.TestCase): def test_uniform_initializer_bf16(self): """Test uniform initializer with bfloat16 - No cast operator has been added here + No cast operator has been added here """ block = self.test_uniform_initializer_default_value("uint16") block = self.test_uniform_initializer(dtype="uint16") @@ -212,18 +215,18 @@ class TestUniformInitializer(unittest.TestCase): class TestNormalInitializer(unittest.TestCase): - def test_normal_initializer_default_value(self): - """Test the normal initializer with default value - """ + """Test the normal initializer with default value""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.NormalInitializer()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -232,17 +235,17 @@ class TestNormalInitializer(unittest.TestCase): self.assertEqual(init_op.attr('seed'), 0) def test_normal_initializer(self, dtype="float32"): - """Test normal initializer with supplied attributes - """ + """Test normal initializer with supplied attributes""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.NormalInitializer( - 2.3, 1.9, 123)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer(2.3, 1.9, 123), + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -253,21 +256,18 @@ class TestNormalInitializer(unittest.TestCase): return block def test_normal_initializer_fp16(self): - """Test normal initializer with float16 - """ + """Test normal initializer with float16""" self.test_normal_initializer("float16") def test_normal_initializer_bf16(self): - """Test normal initializer with bfloat16 - """ + """Test normal initializer with bfloat16""" self.test_normal_initializer("uint16") class TestXavierInitializer(unittest.TestCase): - def test_uniform_xavier_initializer(self): """Test Xavier initializer with uniform distribution on - for matrix multiply. + for matrix multiply. """ program = framework.Program() block = program.global_block() @@ -277,7 +277,8 @@ class TestXavierInitializer(unittest.TestCase): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.XavierInitializer()) + initializer=initializer.XavierInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -288,7 +289,7 @@ class TestXavierInitializer(unittest.TestCase): def test_uniform_xavier_initializer_conv(self): """Test Xavier initializer with uniform distribution on - for convolutions. + for convolutions. """ program = framework.Program() block = program.global_block() @@ -298,20 +299,22 @@ class TestXavierInitializer(unittest.TestCase): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.XavierInitializer()) + initializer=initializer.XavierInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') receptive_field_size = float(15 * 20) limit = np.sqrt( - 6.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) + 6.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size) + ) self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) def test_normal_xavier_initializer(self): """Test Xavier initializer with normal distribution on - for matrix multiply. + for matrix multiply. """ program = framework.Program() block = program.global_block() @@ -321,7 +324,8 @@ class TestXavierInitializer(unittest.TestCase): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.XavierInitializer(uniform=False)) + initializer=initializer.XavierInitializer(uniform=False), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -332,7 +336,7 @@ class TestXavierInitializer(unittest.TestCase): def test_normal_xavier_initializer_conv(self): """Test Xavier initializer with normal distribution on - for convolutions. + for convolutions. """ program = framework.Program() block = program.global_block() @@ -342,36 +346,40 @@ class TestXavierInitializer(unittest.TestCase): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.XavierInitializer(uniform=False)) + initializer=initializer.XavierInitializer(uniform=False), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') receptive_field_size = float(15 * 20) std = np.sqrt( - 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) + 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size) + ) self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) - def test_xavier_initializer_supplied_arguments(self, - dtype="float32", - uniform=True): - """Test the Xavier initializer with supplied arguments - """ + def test_xavier_initializer_supplied_arguments( + self, dtype="float32", uniform=True + ): + """Test the Xavier initializer with supplied arguments""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.XavierInitializer( - uniform=uniform, - fan_in=12, - fan_out=23, - seed=134)) - num_ops = 2 if (dtype == "float16" or - (dtype == "uint16" and not uniform)) else 1 + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer( + uniform=uniform, fan_in=12, fan_out=23, seed=134 + ), + ) + num_ops = ( + 2 + if (dtype == "float16" or (dtype == "uint16" and not uniform)) + else 1 + ) self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] if uniform: @@ -385,25 +393,24 @@ class TestXavierInitializer(unittest.TestCase): return block def test_xavier_initializer_fp16(self): - """Test the Xavier initializer with float16 - """ + """Test the Xavier initializer with float16""" block = self.test_xavier_initializer_supplied_arguments("float16") def test_xavier_initializer_bf16(self): - """Test the Xavier initializer with bfloat16 - """ + """Test the Xavier initializer with bfloat16""" block_uniform = self.test_xavier_initializer_supplied_arguments( - "uint16") + "uint16" + ) self.assertEqual(len(block_uniform.ops), 1) block_gaussian = self.test_xavier_initializer_supplied_arguments( - "uint16", False) + "uint16", False + ) class TestMSRAInitializer(unittest.TestCase): - def test_uniform_msra_initializer(self): """Test MSRA initializer with uniform distribution on - for matrix multiply. + for matrix multiply. """ program = framework.Program() block = program.global_block() @@ -413,7 +420,8 @@ class TestMSRAInitializer(unittest.TestCase): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.MSRAInitializer()) + initializer=initializer.MSRAInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -424,7 +432,7 @@ class TestMSRAInitializer(unittest.TestCase): def test_uniform_msra_initializer_conv(self): """Test MSRA initializer with uniform distribution on - for convolutions. + for convolutions. """ program = framework.Program() block = program.global_block() @@ -434,7 +442,8 @@ class TestMSRAInitializer(unittest.TestCase): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.MSRAInitializer()) + initializer=initializer.MSRAInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -446,7 +455,7 @@ class TestMSRAInitializer(unittest.TestCase): def test_normal_msra_initializer(self): """Test MSRA initializer with normal distribution on - for matrix multiply. + for matrix multiply. """ program = framework.Program() block = program.global_block() @@ -456,7 +465,8 @@ class TestMSRAInitializer(unittest.TestCase): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.MSRAInitializer(uniform=False)) + initializer=initializer.MSRAInitializer(uniform=False), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -467,7 +477,7 @@ class TestMSRAInitializer(unittest.TestCase): def test_normal_msra_initializer_conv(self): """Test MSRA initializer with normal distribution on - for convolutions. + for convolutions. """ program = framework.Program() block = program.global_block() @@ -477,7 +487,8 @@ class TestMSRAInitializer(unittest.TestCase): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.MSRAInitializer(uniform=False)) + initializer=initializer.MSRAInitializer(uniform=False), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -488,17 +499,17 @@ class TestMSRAInitializer(unittest.TestCase): self.assertEqual(init_op.attr('seed'), 0) def test_msra_initializer_supplied_arguments(self, dtype="float32"): - """Test the MSRA initializer with supplied arguments - """ + """Test the MSRA initializer with supplied arguments""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.MSRAInitializer( - fan_in=12, seed=134)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.MSRAInitializer(fan_in=12, seed=134), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -510,22 +521,18 @@ class TestMSRAInitializer(unittest.TestCase): return block def test_msra_initializer_fp16(self): - """Test the MSRA initializer with float16 - """ + """Test the MSRA initializer with float16""" block = self.test_msra_initializer_supplied_arguments("float16") self.assertTrue(check_cast_op(block.ops[1])) def test_msra_initializer_bf16(self): - """Test the MSRA initializer with bfloat16 - """ + """Test the MSRA initializer with bfloat16""" block = self.test_msra_initializer_supplied_arguments("uint16") class TestBilinearInitializer(unittest.TestCase): - def test_bilinear_initializer(self, dtype="float32"): - """Test the bilinear initializer with supplied arguments - """ + """Test the bilinear initializer with supplied arguments""" program = framework.Program() block = program.global_block() for _ in range(2): @@ -534,7 +541,8 @@ class TestBilinearInitializer(unittest.TestCase): shape=[8, 1, 3, 3], lod_level=0, name="param", - initializer=initializer.BilinearInitializer()) + initializer=initializer.BilinearInitializer(), + ) num_ops = 2 if dtype in ["float16", "uint16", "float64"] else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -545,14 +553,12 @@ class TestBilinearInitializer(unittest.TestCase): self.test_bilinear_initializer(dtype='float64') def test_bilinear_initializer_fp16(self): - """Test the bilinear initializer with supplied arguments - """ + """Test the bilinear initializer with supplied arguments""" block = self.test_bilinear_initializer("float16") self.assertTrue(check_cast_op(block.ops[1])) def test_bilinear_initializer_bf16(self): - """Test the bilinear initializer with supplied arguments - """ + """Test the bilinear initializer with supplied arguments""" block = self.test_bilinear_initializer("uint16") self.assertTrue(check_cast_op(block.ops[1])) @@ -561,33 +567,37 @@ class TestBilinearInitializer(unittest.TestCase): class TestBilinearInitializerDygraphAPI(unittest.TestCase): - def func_test_case(self): factor = 2 C = 2 B = 8 H = W = 32 - w_attr = paddle.ParamAttr(learning_rate=0., - regularizer=L2Decay(0.), - initializer=initializer.BilinearInitializer()) + w_attr = paddle.ParamAttr( + learning_rate=0.0, + regularizer=L2Decay(0.0), + initializer=initializer.BilinearInitializer(), + ) data = paddle.rand([B, 3, H, W], dtype='float32') - conv_up = paddle.nn.Conv2DTranspose(3, - out_channels=C, - kernel_size=2 * factor - factor % 2, - padding=int( - math.ceil((factor - 1) / 2.)), - stride=factor, - weight_attr=w_attr, - bias_attr=False) + conv_up = paddle.nn.Conv2DTranspose( + 3, + out_channels=C, + kernel_size=2 * factor - factor % 2, + padding=int(math.ceil((factor - 1) / 2.0)), + stride=factor, + weight_attr=w_attr, + bias_attr=False, + ) x = conv_up(data) return x def func_test_case_fp16(self): paddle.set_default_dtype("float16") paddle.seed(1234) - w_attr = paddle.ParamAttr(learning_rate=0., - regularizer=L2Decay(0.), - initializer=initializer.BilinearInitializer()) + w_attr = paddle.ParamAttr( + learning_rate=0.0, + regularizer=L2Decay(0.0), + initializer=initializer.BilinearInitializer(), + ) conv2d = paddle.nn.Conv2D(1, 2, 3, weight_attr=w_attr) paddle.set_default_dtype("float32") return conv2d.weight @@ -610,11 +620,10 @@ class TestBilinearInitializerDygraphAPI(unittest.TestCase): class TestNumpyArrayInitializer(unittest.TestCase): - def test_numpy_array_initializer(self, dtype="float32"): - """Test the numpy array initializer with supplied arguments - """ + """Test the numpy array initializer with supplied arguments""" import numpy + program = framework.Program() block = program.global_block() np_array = numpy.random.random((10000)).astype(dtype) @@ -624,7 +633,8 @@ class TestNumpyArrayInitializer(unittest.TestCase): shape=np_array.shape, lod_level=0, name="param", - initializer=initializer.NumpyArrayInitializer(np_array)) + initializer=initializer.NumpyArrayInitializer(np_array), + ) num_ops = 2 if dtype in ["float16", "uint16"] else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -633,23 +643,19 @@ class TestNumpyArrayInitializer(unittest.TestCase): return block def test_numpy_array_initializer_fp16(self): - """Test the numpy array initializer with float16 - """ + """Test the numpy array initializer with float16""" block = self.test_numpy_array_initializer("float16") self.assertTrue(block.ops[1]) def test_numpy_array_initializer_bf16(self): - """Test the numpy array initializer with bfloat16 - """ + """Test the numpy array initializer with bfloat16""" block = self.test_numpy_array_initializer("uint16") self.assertTrue(block.ops[1]) class TestSetGlobalInitializer(unittest.TestCase): - def test_set_global_weight_initilizer(self): - """Test Set Global Param initilizer with UniformInitializer - """ + """Test Set Global Param initilizer with UniformInitializer""" main_prog = framework.Program() startup_prog = framework.Program() fluid.set_global_initializer(initializer.Uniform(low=-0.5, high=0.5)) @@ -674,13 +680,13 @@ class TestSetGlobalInitializer(unittest.TestCase): fluid.set_global_initializer(None) def test_set_global_bias_initilizer(self): - """Test Set Global Bias initilizer with NormalInitializer - """ + """Test Set Global Bias initilizer with NormalInitializer""" main_prog = framework.Program() startup_prog = framework.Program() - fluid.set_global_initializer(initializer.Uniform(low=-0.5, high=0.5), - bias_init=initializer.Normal(loc=0.0, - scale=2.0)) + fluid.set_global_initializer( + initializer.Uniform(low=-0.5, high=0.5), + bias_init=initializer.Normal(loc=0.0, scale=2.0), + ) with fluid.program_guard(main_prog, startup_prog): x = fluid.data(name="x", shape=[1, 3, 32, 32]) # default initilizer of bias in layers.conv2d is ConstantInitializer @@ -705,7 +711,6 @@ class TestSetGlobalInitializer(unittest.TestCase): class TestUniformInitializerDygraph(unittest.TestCase): - def func_uniform_initializer(self, dtype="float32"): """ In dygraph mode, we can use initializer directly to initialize a tensor. @@ -714,15 +719,16 @@ class TestUniformInitializerDygraph(unittest.TestCase): tensor = paddle.zeros([1024, 1024, 16]) tensor.stop_gradient = False - np.testing.assert_allclose(np.zeros((1024, 1024, 16)), - tensor.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np.zeros((1024, 1024, 16)), tensor.numpy(), rtol=1e-05 + ) uniform_ = paddle.nn.initializer.Uniform() uniform_(tensor) - self.assertEqual(tensor.stop_gradient, - False) # stop_gradient is not changed + self.assertEqual( + tensor.stop_gradient, False + ) # stop_gradient is not changed hist, prob = output_hist(tensor.numpy()) @@ -737,7 +743,6 @@ class TestUniformInitializerDygraph(unittest.TestCase): class TestXavierInitializerDygraph(unittest.TestCase): - def func_xvarier_initializer(self, dtype="float32"): """ In dygraph mode, we can use initializer directly to initialize a tensor. @@ -747,15 +752,16 @@ class TestXavierInitializerDygraph(unittest.TestCase): tensor = paddle.zeros([1024, 1024, 16]) tensor.stop_gradient = False - xavier_ = paddle.fluid.initializer.XavierInitializer(uniform=False, - fan_in=3, - fan_out=5) + xavier_ = paddle.fluid.initializer.XavierInitializer( + uniform=False, fan_in=3, fan_out=5 + ) xavier_(tensor) hist, _ = output_hist(tensor.numpy()) hist2, _ = output_hist( - np.random.normal(0, np.sqrt(2.0 / (3 + 5)), [1024, 1024, 16])) + np.random.normal(0, np.sqrt(2.0 / (3 + 5)), [1024, 1024, 16]) + ) np.testing.assert_allclose(hist, hist2, rtol=0, atol=0.01) paddle.enable_static() @@ -767,7 +773,6 @@ class TestXavierInitializerDygraph(unittest.TestCase): class TestMSRAInitializerDygraph(unittest.TestCase): - def func_msra_initializer(self, dtype="float32"): """ In dygraph mode, we can use initializer directly to initialize a tensor. @@ -777,14 +782,16 @@ class TestMSRAInitializerDygraph(unittest.TestCase): tensor = paddle.zeros([1024, 1024, 16]) tensor.stop_gradient = False - msra_ = paddle.fluid.initializer.MSRAInitializer(uniform=False, - fan_in=4) + msra_ = paddle.fluid.initializer.MSRAInitializer( + uniform=False, fan_in=4 + ) msra_(tensor) hist, _ = output_hist(tensor.numpy()) hist2, _ = output_hist( - np.random.normal(0, np.sqrt(2.0 / (4)), [1024, 1024, 16])) + np.random.normal(0, np.sqrt(2.0 / (4)), [1024, 1024, 16]) + ) np.testing.assert_allclose(hist, hist2, rtol=0, atol=0.01) paddle.enable_static() @@ -796,7 +803,6 @@ class TestMSRAInitializerDygraph(unittest.TestCase): class TesetconsistencyOfDynamicAndStaticGraph(unittest.TestCase): - def func_order(self): paddle.set_device('cpu') SEED = 123 @@ -805,23 +811,26 @@ class TesetconsistencyOfDynamicAndStaticGraph(unittest.TestCase): learning_rate=1.0, trainable=False, regularizer=None, - initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, - std=2.0)) + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0 + ), + ) bias_attr = paddle.framework.ParamAttr( name="linear_bias", learning_rate=1.0, trainable=False, regularizer=None, - initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, - std=2.0)) + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0 + ), + ) def run_dynamic_graph(): paddle.disable_static() paddle.seed(SEED) - linear = paddle.nn.Linear(1, - 1, - weight_attr=weight_attr, - bias_attr=bias_attr) + linear = paddle.nn.Linear( + 1, 1, weight_attr=weight_attr, bias_attr=bias_attr + ) return linear.weight.numpy(), linear.bias.numpy() paddle.enable_static() @@ -829,12 +838,13 @@ class TesetconsistencyOfDynamicAndStaticGraph(unittest.TestCase): paddle.enable_static() exe = paddle.static.Executor(paddle.CPUPlace()) paddle.seed(SEED) - linear = paddle.nn.Linear(1, - 1, - weight_attr=weight_attr, - bias_attr=bias_attr) - res = exe.run(paddle.static.default_startup_program(), - fetch_list=['linear_weight', 'linear_bias']) + linear = paddle.nn.Linear( + 1, 1, weight_attr=weight_attr, bias_attr=bias_attr + ) + res = exe.run( + paddle.static.default_startup_program(), + fetch_list=['linear_weight', 'linear_bias'], + ) return res[0], res[1] dynamic_res = run_dynamic_graph() @@ -857,7 +867,8 @@ class TestOrthogonalInitializer1(unittest.TestCase): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal(gain=3.0)) + initializer=paddle.nn.initializer.Orthogonal(gain=3.0) + ) self.dtype = "float64" self.in_features = 10 self.out_features = 15 @@ -865,10 +876,9 @@ class TestOrthogonalInitializer1(unittest.TestCase): def check_result(self, a, b): np.testing.assert_array_equal(a, b) - np.testing.assert_allclose(np.matmul(a, a.T), - 9 * np.eye(10), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + np.matmul(a, a.T), 9 * np.eye(10), rtol=1e-5, atol=1e-8 + ) def func_orthogonal(self): self.config() @@ -876,9 +886,9 @@ class TestOrthogonalInitializer1(unittest.TestCase): paddle.disable_static() paddle.seed(2021) - linear = paddle.nn.Linear(self.in_features, - self.out_features, - weight_attr=self.weight_attr) + linear = paddle.nn.Linear( + self.in_features, self.out_features, weight_attr=self.weight_attr + ) res_dygraph = linear.weight.numpy() paddle.enable_static() @@ -886,9 +896,11 @@ class TestOrthogonalInitializer1(unittest.TestCase): start_prog = paddle.static.Program() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): - linear = paddle.nn.Linear(self.in_features, - self.out_features, - weight_attr=self.weight_attr) + linear = paddle.nn.Linear( + self.in_features, + self.out_features, + weight_attr=self.weight_attr, + ) block = start_prog.global_block() self.assertEqual(len(block.ops), self.num_ops) @@ -919,7 +931,8 @@ class TestOrthogonalInitializer2(TestOrthogonalInitializer1): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal(gain=2.0)) + initializer=paddle.nn.initializer.Orthogonal(gain=2.0) + ) self.dtype = "float64" self.in_features = 15 self.out_features = 10 @@ -927,10 +940,9 @@ class TestOrthogonalInitializer2(TestOrthogonalInitializer1): def check_result(self, a, b): np.testing.assert_array_equal(a, b) - np.testing.assert_allclose(np.matmul(a.T, a), - 4 * np.eye(10), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + np.matmul(a.T, a), 4 * np.eye(10), rtol=1e-5, atol=1e-8 + ) # 2-D Parameter with shape: [10, 10] @@ -941,7 +953,8 @@ class TestOrthogonalInitializer3(TestOrthogonalInitializer1): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal()) + initializer=paddle.nn.initializer.Orthogonal() + ) self.dtype = "float32" self.in_features = 10 self.out_features = 10 @@ -949,14 +962,12 @@ class TestOrthogonalInitializer3(TestOrthogonalInitializer1): def check_result(self, a, b): np.testing.assert_array_equal(a, b) - np.testing.assert_allclose(np.matmul(a.T, a), - np.eye(10), - rtol=1e-05, - atol=1e-06) - np.testing.assert_allclose(np.matmul(a, a.T), - np.eye(10), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np.matmul(a.T, a), np.eye(10), rtol=1e-05, atol=1e-06 + ) + np.testing.assert_allclose( + np.matmul(a, a.T), np.eye(10), rtol=1e-05, atol=1e-06 + ) def test_error(self): self.config() @@ -972,7 +983,8 @@ class TestOrthogonalInitializer4(unittest.TestCase): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal(gain=3.0)) + initializer=paddle.nn.initializer.Orthogonal(gain=3.0) + ) self.dtype = "float64" self.in_features = 4 self.out_features = 6 @@ -981,10 +993,9 @@ class TestOrthogonalInitializer4(unittest.TestCase): def check_result(self, a, b): np.testing.assert_array_equal(a, b) a = a.reshape(6, -1) - np.testing.assert_allclose(np.matmul(a, a.T), - 9 * np.eye(6), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + np.matmul(a, a.T), 9 * np.eye(6), rtol=1e-5, atol=1e-8 + ) def func_orthogonal(self): self.config() @@ -992,10 +1003,12 @@ class TestOrthogonalInitializer4(unittest.TestCase): paddle.disable_static() paddle.seed(2021) - conv2d = paddle.nn.Conv2D(self.in_features, - self.out_features, - self.kernel_size, - weight_attr=self.weight_attr) + conv2d = paddle.nn.Conv2D( + self.in_features, + self.out_features, + self.kernel_size, + weight_attr=self.weight_attr, + ) res_dygraph = conv2d.weight.numpy() paddle.enable_static() @@ -1003,13 +1016,17 @@ class TestOrthogonalInitializer4(unittest.TestCase): start_prog = paddle.static.Program() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): - conv2d = paddle.nn.Conv2D(self.in_features, - self.out_features, - self.kernel_size, - weight_attr=self.weight_attr) + conv2d = paddle.nn.Conv2D( + self.in_features, + self.out_features, + self.kernel_size, + weight_attr=self.weight_attr, + ) exe = paddle.static.Executor() - res_static = exe.run(paddle.static.default_startup_program(), - fetch_list=[conv2d.weight])[0] + res_static = exe.run( + paddle.static.default_startup_program(), + fetch_list=[conv2d.weight], + )[0] self.check_result(res_dygraph, res_static) def test_orthogonal(self): @@ -1026,7 +1043,8 @@ class TestOrthogonalInitializer5(TestOrthogonalInitializer4): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal(gain=2.0)) + initializer=paddle.nn.initializer.Orthogonal(gain=2.0) + ) self.dtype = "float64" self.in_features = 4 self.out_features = 50 @@ -1035,10 +1053,9 @@ class TestOrthogonalInitializer5(TestOrthogonalInitializer4): def check_result(self, a, b): np.testing.assert_array_equal(a, b) a = a.reshape(50, -1) - np.testing.assert_allclose(np.matmul(a.T, a), - 4 * np.eye(36), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + np.matmul(a.T, a), 4 * np.eye(36), rtol=1e-5, atol=1e-8 + ) # 4-D Parameter with shape: [36, 4, 3, 3] @@ -1049,7 +1066,8 @@ class TestOrthogonalInitializer6(TestOrthogonalInitializer4): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal()) + initializer=paddle.nn.initializer.Orthogonal() + ) self.dtype = "float32" self.in_features = 4 self.out_features = 36 @@ -1058,29 +1076,29 @@ class TestOrthogonalInitializer6(TestOrthogonalInitializer4): def check_result(self, a, b): np.testing.assert_array_equal(a, b) a = a.reshape(36, -1) - np.testing.assert_allclose(np.matmul(a.T, a), - np.eye(36), - rtol=1e-05, - atol=1e-06) - np.testing.assert_allclose(np.matmul(a, a.T), - np.eye(36), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np.matmul(a.T, a), np.eye(36), rtol=1e-05, atol=1e-06 + ) + np.testing.assert_allclose( + np.matmul(a, a.T), np.eye(36), rtol=1e-05, atol=1e-06 + ) # initialize Conv1D weight class TestDiracInitializer1(unittest.TestCase): - def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Dirac()) + initializer=paddle.nn.initializer.Dirac() + ) self.dtype = "float64" self.in_channels = 3 self.out_channels = 2 self.kernel_size = 3 self.input_shape = [8, self.in_channels, 10] self.conv_layer = paddle.nn.Conv1D - self.num_ops = 8 #fill_constant*2, reshape*2, assign_value*2, scatter, cast + self.num_ops = ( + 8 # fill_constant*2, reshape*2, assign_value*2, scatter, cast + ) def check_result(self, w_dygraph, w_static, conv_in, conv_out): np.testing.assert_array_equal(w_dygraph, w_static) @@ -1091,10 +1109,12 @@ class TestDiracInitializer1(unittest.TestCase): paddle.set_default_dtype(self.dtype) paddle.disable_static() - conv = self.conv_layer(self.in_channels, - self.out_channels, - self.kernel_size, - weight_attr=self.weight_attr) + conv = self.conv_layer( + self.in_channels, + self.out_channels, + self.kernel_size, + weight_attr=self.weight_attr, + ) weight_dygraph = conv.weight.numpy() paddle.enable_static() @@ -1102,10 +1122,12 @@ class TestDiracInitializer1(unittest.TestCase): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): inp = paddle.rand(self.input_shape) - conv = self.conv_layer(self.in_channels, - self.out_channels, - self.kernel_size, - weight_attr=self.weight_attr) + conv = self.conv_layer( + self.in_channels, + self.out_channels, + self.kernel_size, + weight_attr=self.weight_attr, + ) output = conv(inp) block = start_prog.global_block() @@ -1124,8 +1146,9 @@ class TestDiracInitializer1(unittest.TestCase): conv_output = fetch[1] weight_static = fetch[2] - self.check_result(weight_dygraph, weight_static, conv_input, - conv_output) + self.check_result( + weight_dygraph, weight_static, conv_input, conv_output + ) def test_dirac(self): with framework._test_eager_guard(): @@ -1135,10 +1158,10 @@ class TestDiracInitializer1(unittest.TestCase): # initialize Conv2D weight class TestDiracInitializer2(TestDiracInitializer1): - def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Dirac(groups=1)) + initializer=paddle.nn.initializer.Dirac(groups=1) + ) self.dtype = "float64" self.in_channels = 4 self.out_channels = 8 @@ -1149,18 +1172,20 @@ class TestDiracInitializer2(TestDiracInitializer1): def check_result(self, w_dygraph, w_static, conv_in, conv_out): np.testing.assert_array_equal(w_dygraph, w_static) - np.testing.assert_array_equal(conv_out[:, 0:4, :, :], conv_in[:, :, 1:9, - 1:9]) - np.testing.assert_array_equal(conv_out[:, 4:8, :, :], - np.zeros([8, 4, 8, 8])) + np.testing.assert_array_equal( + conv_out[:, 0:4, :, :], conv_in[:, :, 1:9, 1:9] + ) + np.testing.assert_array_equal( + conv_out[:, 4:8, :, :], np.zeros([8, 4, 8, 8]) + ) # initialize Conv3D weight class TestDiracInitializer3(TestDiracInitializer1): - def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Dirac(groups=2)) + initializer=paddle.nn.initializer.Dirac(groups=2) + ) self.dtype = "float32" self.in_channels = 5 self.out_channels = 10 @@ -1171,10 +1196,12 @@ class TestDiracInitializer3(TestDiracInitializer1): def check_result(self, w_dygraph, w_static, conv_in, conv_out): np.testing.assert_array_equal(w_dygraph, w_static) - np.testing.assert_array_equal(conv_out[:, 0:5, :, :, :], - conv_in[:, :, 1:9, 1:9, 1:9]) - np.testing.assert_array_equal(conv_out[:, 5:10, :, :, :], - conv_in[:, :, 1:9, 1:9, 1:9]) + np.testing.assert_array_equal( + conv_out[:, 0:5, :, :, :], conv_in[:, :, 1:9, 1:9, 1:9] + ) + np.testing.assert_array_equal( + conv_out[:, 5:10, :, :, :], conv_in[:, :, 1:9, 1:9, 1:9] + ) def test_error(self): self.config() diff --git a/python/paddle/fluid/tests/unittests/test_initializer_nn.py b/python/paddle/fluid/tests/unittests/test_initializer_nn.py index 54063f74a6e..c9b81720ba9 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer_nn.py +++ b/python/paddle/fluid/tests/unittests/test_initializer_nn.py @@ -32,26 +32,28 @@ def get_uniform_min_and_max(weight): def check_cast_op(op): - return op.type == 'cast' and \ - op.attr('in_dtype') == VarDesc.VarType.FP32 and \ - op.attr('out_dtype') in [VarDesc.VarType.FP16, VarDesc.VarType.BF16] + return ( + op.type == 'cast' + and op.attr('in_dtype') == VarDesc.VarType.FP32 + and op.attr('out_dtype') in [VarDesc.VarType.FP16, VarDesc.VarType.BF16] + ) class TestConstantInitializer(unittest.TestCase): - - def static_test_constant_initializer_common(self, - init_inst, - dtype="float32", - value_target=0.0): + def static_test_constant_initializer_common( + self, init_inst, dtype="float32", value_target=0.0 + ): paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=init_inst) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=init_inst, + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -61,46 +63,45 @@ class TestConstantInitializer(unittest.TestCase): return block def test_constant_initializer_default_value_static(self, dtype="float32"): - """Test the constant initializer with default value in static graph - """ + """Test the constant initializer with default value in static graph""" block = self.static_test_constant_initializer_common( - init_inst=initializer.Constant(), dtype=dtype, value_target=0.0) + init_inst=initializer.Constant(), dtype=dtype, value_target=0.0 + ) return block def test_constant_initializer_default_value_dygraph(self, dtype="float32"): - """Test constant initializer with supplied value in dygraph - """ + """Test constant initializer with supplied value in dygraph""" with fluid.dygraph.guard(): linear = nn.Linear(2, 4, weight_attr=nn.initializer.Constant()) mat_target = np.ones((2, 4), dtype=dtype) * 0.0 mat_linear = linear.weight.numpy() mismatch = np.sum( - (mat_target - mat_linear) * (mat_target - mat_linear)) + (mat_target - mat_linear) * (mat_target - mat_linear) + ) self.assertAlmostEqual(mismatch, 0.0, delta=DELTA) def test_constant_initializer_static(self, dtype="float32"): - """Test constant initializer with supplied value in static graph - """ + """Test constant initializer with supplied value in static graph""" block = self.static_test_constant_initializer_common( - init_inst=initializer.Constant(2.3), dtype=dtype, value_target=2.3) + init_inst=initializer.Constant(2.3), dtype=dtype, value_target=2.3 + ) return block def test_constant_initializer_dygraph(self, dtype="float32"): - """Test constant initializer with supplied value in dygraph - """ + """Test constant initializer with supplied value in dygraph""" with fluid.dygraph.guard(): - linear = nn.Linear(2, - 4, - weight_attr=nn.initializer.Constant(value=2.0)) + linear = nn.Linear( + 2, 4, weight_attr=nn.initializer.Constant(value=2.0) + ) mat_target = np.ones((2, 4), dtype=dtype) * 2.0 mat_linear = linear.weight.numpy() mismatch = np.sum( - (mat_target - mat_linear) * (mat_target - mat_linear)) + (mat_target - mat_linear) * (mat_target - mat_linear) + ) self.assertAlmostEqual(mismatch, 0.0, delta=DELTA) def test_constant_initializer_fp16(self): - """Test constant initializer with float16 - """ + """Test constant initializer with float16""" block = self.test_constant_initializer_default_value_static("float16") block = self.test_constant_initializer_static("float16") self.test_constant_initializer_default_value_dygraph("float16") @@ -108,29 +109,30 @@ class TestConstantInitializer(unittest.TestCase): def test_constant_initializer_bf16(self): """Test constant initializer with bfloat16 - No cast operator has been added here + No cast operator has been added here """ - self.test_constant_initializer_default_value_static("uint16") #bfloat16 - self.test_constant_initializer_static("uint16") #bfloat16 + self.test_constant_initializer_default_value_static( + "uint16" + ) # bfloat16 + self.test_constant_initializer_static("uint16") # bfloat16 class TestKaimingInitializer(unittest.TestCase): - - def static_test_kaiming_initializer_common(self, - init_inst, - dtype="float32", - uniform=False, - is_conv=False): + def static_test_kaiming_initializer_common( + self, init_inst, dtype="float32", uniform=False, is_conv=False + ): paddle.enable_static() program = framework.Program() block = program.global_block() shape_mat = [5, 10, 15, 20] if is_conv else [5, 10] for _ in range(2): - param = block.create_parameter(dtype="float32", - shape=shape_mat, - lod_level=0, - name="param", - initializer=init_inst) + param = block.create_parameter( + dtype="float32", + shape=shape_mat, + lod_level=0, + name="param", + initializer=init_inst, + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] if uniform: @@ -153,75 +155,76 @@ class TestKaimingInitializer(unittest.TestCase): self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) paddle.disable_static() - def dygraph_test_kaiming_initializer_common(self, - init_inst, - dtype="float32", - uniform=False): + def dygraph_test_kaiming_initializer_common( + self, init_inst, dtype="float32", uniform=False + ): linear = nn.Linear(40, 20, weight_attr=init_inst) def test_kaiming_dygraph(self): self.dygraph_test_kaiming_initializer_common( init_inst=initializer.KaimingUniform(), dtype="float32", - uniform=True) + uniform=True, + ) self.dygraph_test_kaiming_initializer_common( init_inst=initializer.KaimingNormal(), dtype="float32", - uniform=False) + uniform=False, + ) def test_kaiming_uniform_initializer_static(self): - """Test Kaiming unorm initializer for matrix multiply. - """ + """Test Kaiming unorm initializer for matrix multiply.""" self.static_test_kaiming_initializer_common( init_inst=initializer.KaimingUniform(), dtype="float32", uniform=True, - is_conv=False) + is_conv=False, + ) def test_kaiming_uniform_initializer_conv_static(self): - """Test Kaiming unorm initializer for convolutions. - """ + """Test Kaiming unorm initializer for convolutions.""" self.static_test_kaiming_initializer_common( init_inst=initializer.KaimingUniform(), dtype="float32", uniform=True, - is_conv=True) + is_conv=True, + ) def test_kaiming_normal_initializer_static(self): - """Test Kaiming normal initializer for matrix multiply. - """ + """Test Kaiming normal initializer for matrix multiply.""" self.static_test_kaiming_initializer_common( init_inst=initializer.KaimingNormal(), dtype="float32", uniform=False, - is_conv=False) + is_conv=False, + ) def test_kaiming_normal_initializer_conv_static(self): - """Test Kaiming normal initializer for convolutions. - """ + """Test Kaiming normal initializer for convolutions.""" self.static_test_kaiming_initializer_common( init_inst=initializer.KaimingNormal(), dtype="float32", uniform=False, - is_conv=True) + is_conv=True, + ) class TestUniform(unittest.TestCase): - def test_uniform_common(self, dtype="float32", seed=0): - """Test the uniform initializer with default value - """ + """Test the uniform initializer with default value""" paddle.enable_static() program = framework.Program() program.random_seed = seed block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Uniform()) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -234,24 +237,23 @@ class TestUniform(unittest.TestCase): return block - def test_uniform_initializer_default_value(self, - dtype="float32", - seed=0, - min_value=-1.0, - max_vlaue=1.0): - """Test the uniform initializer with default value - """ + def test_uniform_initializer_default_value( + self, dtype="float32", seed=0, min_value=-1.0, max_vlaue=1.0 + ): + """Test the uniform initializer with default value""" paddle.enable_static() program = framework.Program() program.random_seed = seed block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Uniform()) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -264,25 +266,23 @@ class TestUniform(unittest.TestCase): return block - def test_uniform_initializer(self, - dtype="float32", - seed=0, - min_value=-4.2, - max_vlaue=3.1): - """Test uniform initializer with supplied attributes - """ + def test_uniform_initializer( + self, dtype="float32", seed=0, min_value=-4.2, max_vlaue=3.1 + ): + """Test uniform initializer with supplied attributes""" paddle.enable_static() program = framework.Program() program.random_seed = seed block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Uniform( - min_value, max_vlaue)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(min_value, max_vlaue), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -294,25 +294,23 @@ class TestUniform(unittest.TestCase): return block - def test_uniform_initializer_two_op(self, - dtype="float32", - seed=123, - min_value=-4.2, - max_vlaue=0.0): - """Test uniform initializer with supplied attributes - """ + def test_uniform_initializer_two_op( + self, dtype="float32", seed=123, min_value=-4.2, max_vlaue=0.0 + ): + """Test uniform initializer with supplied attributes""" paddle.enable_static() program = framework.Program() program.random_seed = seed block = program.global_block() for i in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Uniform( - min_value, float(i))) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(min_value, float(i)), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op0 = block.ops[0] @@ -326,8 +324,7 @@ class TestUniform(unittest.TestCase): return block def test_uniform_initializer_fp16(self): - """Test uniform initializer with float16 - """ + """Test uniform initializer with float16""" block = self.test_uniform_initializer_default_value("float16") self.assertTrue(check_cast_op(block.ops[1])) block = self.test_uniform_initializer(dtype="float16") @@ -336,44 +333,47 @@ class TestUniform(unittest.TestCase): self.assertTrue(check_cast_op(block.ops[1])) def test_uniform_initializer_bf16(self): - """Test uniform initializer with bfloat16 - """ - block = self.test_uniform_initializer_default_value("uint16") #bfloat16 - block = self.test_uniform_initializer(dtype="uint16") #bfloat16 - block = self.test_uniform_initializer_two_op("uint16") #bfloat16 + """Test uniform initializer with bfloat16""" + block = self.test_uniform_initializer_default_value( + "uint16" + ) # bfloat16 + block = self.test_uniform_initializer(dtype="uint16") # bfloat16 + block = self.test_uniform_initializer_two_op("uint16") # bfloat16 def test_uniform_initializer_dygraph(self): - """Test uniform initializer in dygraph model. - """ + """Test uniform initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5)) + initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) min_value, max_value = get_uniform_min_and_max(linear.weight.numpy()) - self.assertTrue(min_value >= -0.5, - 'min value {} should >= -0.5'.format(min_value)) - self.assertTrue(max_value <= 0.5, - 'max value {} should <= 0.5'.format(max_value)) + self.assertTrue( + min_value >= -0.5, 'min value {} should >= -0.5'.format(min_value) + ) + self.assertTrue( + max_value <= 0.5, 'max value {} should <= 0.5'.format(max_value) + ) class TestNormal(unittest.TestCase): - def test_normal_initializer_default_value(self): - """Test the normal initializer with default value - """ + """Test the normal initializer with default value""" paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Normal()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Normal(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -384,18 +384,19 @@ class TestNormal(unittest.TestCase): paddle.disable_static() def test_normal_initializer(self, dtype="float32"): - """Test normal initializer with supplied attributes - """ + """Test normal initializer with supplied attributes""" paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Normal(2.3, 1.9)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Normal(2.3, 1.9), + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -408,41 +409,39 @@ class TestNormal(unittest.TestCase): return block def test_normal_initializer_fp16(self): - """Test normal initializer with float16 - """ + """Test normal initializer with float16""" block = self.test_normal_initializer("float16") def test_normal_initializer_bf16(self): - """Test normal initializer with bfloat16 - """ - block = self.test_normal_initializer("uint16") #bfloat16 + """Test normal initializer with bfloat16""" + block = self.test_normal_initializer("uint16") # bfloat16 def test_normal_initializer_dygraph(self): - """Test normal initializer in dygraph model. - """ + """Test normal initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0)) + initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) class TestTruncatedNormal(unittest.TestCase): - def test_truncated_normal_initializer_default_value(self): - """Test the truncated normal initializer with default value - """ + """Test the truncated normal initializer with default value""" paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.TruncatedNormal()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.TruncatedNormal(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'truncated_gaussian_random') @@ -453,19 +452,19 @@ class TestTruncatedNormal(unittest.TestCase): paddle.disable_static() def test_truncated_normal_initializer(self, dtype="float32"): - """Test truncated normal initializer with supplied attributes - """ + """Test truncated normal initializer with supplied attributes""" paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.TruncatedNormal( - 2.3, 1.9)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.TruncatedNormal(2.3, 1.9), + ) num_ops = 2 if dtype in ["float16", "uint16"] else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -478,38 +477,36 @@ class TestTruncatedNormal(unittest.TestCase): return block def test_truncated_normal_initializer_fp16(self): - """Test truncated normal initializer with float16 - """ + """Test truncated normal initializer with float16""" paddle.enable_static() block = self.test_truncated_normal_initializer("float16") self.assertTrue(check_cast_op(block.ops[1])) def test_truncated_normal_initializer_bf16(self): - """Test truncated normal initializer with bfloat16 - """ + """Test truncated normal initializer with bfloat16""" paddle.enable_static() - block = self.test_truncated_normal_initializer("uint16") #bfloat16 + block = self.test_truncated_normal_initializer("uint16") # bfloat16 self.assertTrue(check_cast_op(block.ops[1])) def test_truncated_normal_initializer_dygraph(self): - """Test truncated normal initializer in dygraph model. - """ + """Test truncated normal initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, - std=2.0)) + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0 + ), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) class TestXavierUniform(unittest.TestCase): - def test_xavier_uniform_initializer(self): """Test Xavier initializer with uniform distribution on - for matrix multiply. + for matrix multiply. """ paddle.enable_static() @@ -521,7 +518,8 @@ class TestXavierUniform(unittest.TestCase): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.XavierUniform()) + initializer=initializer.XavierUniform(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -534,7 +532,7 @@ class TestXavierUniform(unittest.TestCase): def test_xavier_uniform_initializer_conv(self): """Test Xavier initializer with uniform distribution on - for convolutions. + for convolutions. """ paddle.enable_static() @@ -546,33 +544,34 @@ class TestXavierUniform(unittest.TestCase): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.XavierUniform()) + initializer=initializer.XavierUniform(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') receptive_field_size = float(15 * 20) limit = np.sqrt( - 6.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) + 6.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size) + ) self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) def test_xavier_uniform_initializer_dygraph(self): - """Test xavier uniform initializer in dygraph model. - """ + """Test xavier uniform initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.XavierUniform()) + initializer=paddle.nn.initializer.XavierUniform(), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) class TestXavierNormal(unittest.TestCase): - def test_xavier_normal_initializer(self): """Test Xavier initializer with normal distribution on - for matrix multiply. + for matrix multiply. """ paddle.enable_static() @@ -584,7 +583,8 @@ class TestXavierNormal(unittest.TestCase): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.XavierNormal()) + initializer=initializer.XavierNormal(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -597,7 +597,7 @@ class TestXavierNormal(unittest.TestCase): def test_xavier_normal_initializer_conv(self): """Test Xavier initializer with normal distribution on - for convolutions. + for convolutions. """ paddle.enable_static() @@ -609,13 +609,15 @@ class TestXavierNormal(unittest.TestCase): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.XavierNormal()) + initializer=initializer.XavierNormal(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') receptive_field_size = float(15 * 20) std = np.sqrt( - 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) + 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size) + ) self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) @@ -623,33 +625,34 @@ class TestXavierNormal(unittest.TestCase): paddle.disable_static() def test_xavier_normal_initializer_dygraph(self): - """Test xavier normal initializer in dygraph model. - """ + """Test xavier normal initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.XavierNormal()) + initializer=paddle.nn.initializer.XavierNormal(), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) class TestAssign(unittest.TestCase): - def test_assign_initializer(self, dtype="float32"): - """Test the numpy array initializer with supplied arguments - """ + """Test the numpy array initializer with supplied arguments""" paddle.enable_static() import numpy + program = framework.Program() block = program.global_block() np_array = numpy.random.random((10000)).astype(dtype) for _ in range(2): - block.create_parameter(dtype=np_array.dtype, - shape=np_array.shape, - lod_level=0, - name="param", - initializer=initializer.Assign(np_array)) + block.create_parameter( + dtype=np_array.dtype, + shape=np_array.shape, + lod_level=0, + name="param", + initializer=initializer.Assign(np_array), + ) num_ops = 2 if dtype in ["float16", "uint16"] else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -661,61 +664,59 @@ class TestAssign(unittest.TestCase): return block def test_assign_initializer_fp16(self): - """Test the numpy array initializer with float16 - """ + """Test the numpy array initializer with float16""" block = self.test_assign_initializer("float16") self.assertTrue(block.ops[1]) def test_assign_initializer_bf16(self): - """Test the numpy array initializer with bfloat16 - """ - block = self.test_assign_initializer("uint16") #bfloat16 + """Test the numpy array initializer with bfloat16""" + block = self.test_assign_initializer("uint16") # bfloat16 self.assertTrue(block.ops[1]) def test_assign_initializer_dygraph_1(self): - """Test assign initializer in dygraph model. - """ + """Test assign initializer in dygraph model.""" paddle.disable_static() weight_attr_1 = paddle.framework.ParamAttr( name="linear_weight_1", - initializer=paddle.nn.initializer.Assign(np.array([2, 2]))) + initializer=paddle.nn.initializer.Assign(np.array([2, 2])), + ) linear_1 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_1) self.assertTrue((linear_1.weight.numpy() == [2.0, 2.0]).all(), '') def test_assign_initializer_dygraph_2(self): - """Test assign initializer in dygraph model. - """ + """Test assign initializer in dygraph model.""" paddle.disable_static() weight_attr_2 = paddle.framework.ParamAttr( name="linear_weight_2", - initializer=paddle.nn.initializer.Assign([2, 2])) + initializer=paddle.nn.initializer.Assign([2, 2]), + ) linear_2 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_2) self.assertTrue((linear_2.weight.numpy() == [2.0, 2.0]).all(), '') def test_assign_initializer_dygraph_3(self): - """Test assign initializer in dygraph model. - """ + """Test assign initializer in dygraph model.""" paddle.disable_static() weight_attr_3 = paddle.framework.ParamAttr( name="linear_weight_3", - initializer=paddle.nn.initializer.Assign(paddle.full([2], 2))) + initializer=paddle.nn.initializer.Assign(paddle.full([2], 2)), + ) linear_3 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_3) self.assertTrue((linear_3.weight.numpy() == [2.0, 2.0]).all(), '') def test_assign_initializer_dygraph_4(self): - """Test assign initializer in dygraph model. - """ + """Test assign initializer in dygraph model.""" paddle.disable_static() weight_attr_4 = paddle.framework.ParamAttr( name="linear_weight_4", - initializer=paddle.nn.initializer.Assign((2, 2))) + initializer=paddle.nn.initializer.Assign((2, 2)), + ) linear_4 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_4) self.assertTrue((linear_4.weight.numpy() == [2.0, 2.0]).all(), '') diff --git a/python/paddle/fluid/tests/unittests/test_inner.py b/python/paddle/fluid/tests/unittests/test_inner.py index cf631101e8e..3e30e9fa228 100644 --- a/python/paddle/fluid/tests/unittests/test_inner.py +++ b/python/paddle/fluid/tests/unittests/test_inner.py @@ -22,27 +22,28 @@ from paddle.fluid.framework import _test_eager_guard class TestMultiplyApi(unittest.TestCase): - def _run_static_graph_case(self, x_data, y_data): with program_guard(Program(), Program()): paddle.enable_static() - x = paddle.static.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = paddle.static.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = paddle.static.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = paddle.static.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) res = paddle.inner(x, y) - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - outs = exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[res]) + outs = exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[res], + ) res = outs[0] return res @@ -94,17 +95,21 @@ class TestMultiplyApi(unittest.TestCase): # test dynamic computation graph: 2-d array Complex x_data = np.random.rand(20, 50).astype( - np.float64) + 1J * np.random.rand(20, 50).astype(np.float64) - y_data = np.random.rand(50).astype( - np.float64) + 1J * np.random.rand(50).astype(np.float64) + np.float64 + ) + 1j * np.random.rand(20, 50).astype(np.float64) + y_data = np.random.rand(50).astype(np.float64) + 1j * np.random.rand( + 50 + ).astype(np.float64) res = self._run_dynamic_graph_case(x_data, y_data) np.testing.assert_allclose(res, np.inner(x_data, y_data), rtol=1e-05) # test dynamic computation graph: 3-d array Complex x_data = np.random.rand(5, 10, 10).astype( - np.float64) + 1J * np.random.rand(5, 10, 10).astype(np.float64) - y_data = np.random.rand(2, 10).astype( - np.float64) + 1J * np.random.rand(2, 10).astype(np.float64) + np.float64 + ) + 1j * np.random.rand(5, 10, 10).astype(np.float64) + y_data = np.random.rand(2, 10).astype(np.float64) + 1j * np.random.rand( + 2, 10 + ).astype(np.float64) res = self._run_dynamic_graph_case(x_data, y_data) np.testing.assert_allclose(res, np.inner(x_data, y_data), rtol=1e-05) @@ -115,7 +120,6 @@ class TestMultiplyApi(unittest.TestCase): class TestMultiplyError(unittest.TestCase): - def func_test_errors(self): # test static computation graph: dtype can not be int8 paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_inplace.py b/python/paddle/fluid/tests/unittests/test_inplace.py index 0d88a35ff24..2496ee37a05 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace.py +++ b/python/paddle/fluid/tests/unittests/test_inplace.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class TestInplace(unittest.TestCase): - def func_test_forward_version(self): with paddle.fluid.dygraph.guard(): var = paddle.to_tensor(np.ones((4, 2, 3)).astype(np.float32)) @@ -60,9 +59,11 @@ class TestInplace(unittest.TestCase): loss = paddle.nn.functional.relu(var_c + var_d) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): loss.backward() def test_backward_error(self): @@ -101,7 +102,9 @@ class TestInplace(unittest.TestCase): var_b[1:2] = 3 # var_b is modified inplace before using it - var_c = var_b + var_b # Here, the grad op of sum doesn't use the value of var_b + var_c = ( + var_b + var_b + ) # Here, the grad op of sum doesn't use the value of var_b loss = var_c.sum() var_b[1:2] = 3 # var_b is modified inplace after using it @@ -115,7 +118,6 @@ class TestInplace(unittest.TestCase): class TestDygraphInplace(unittest.TestCase): - def setUp(self): self.init_data() self.set_np_compare_func() @@ -138,7 +140,7 @@ class TestDygraphInplace(unittest.TestCase): inplace_var = self.inplace_api_processing(var) self.assertTrue(id(var) == id(inplace_var)) - inplace_var[0] = 2. + inplace_var[0] = 2.0 np.testing.assert_array_equal(var.numpy(), inplace_var.numpy()) def test_inplace_api(self): @@ -154,7 +156,7 @@ class TestDygraphInplace(unittest.TestCase): inplace_var = self.inplace_api_processing(var) self.assertEqual(var.inplace_version, 1) - inplace_var[0] = 2. + inplace_var[0] = 2.0 self.assertEqual(var.inplace_version, 2) inplace_var = self.inplace_api_processing(inplace_var) @@ -195,9 +197,11 @@ class TestDygraphInplace(unittest.TestCase): loss = paddle.nn.functional.relu(var_c) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): loss.backward() def test_backward_error(self): @@ -215,7 +219,8 @@ class TestDygraphInplace(unittest.TestCase): var_b = var_a**2 var_c = self.inplace_api_processing( - var_b) # var_b is modified inplace before using it + var_b + ) # var_b is modified inplace before using it # Here, the gradient computation will use the value of var_b var_d = var_c**2 @@ -252,9 +257,12 @@ class TestDygraphInplace(unittest.TestCase): var_b = var_a**2 var_c = self.inplace_api_processing( - var_b) # var_b is modified inplace before using it + var_b + ) # var_b is modified inplace before using it - var_d = var_c + var_c # Here, the grad op of sum doesn't use the value of var_b + var_d = ( + var_c + var_c + ) # Here, the grad op of sum doesn't use the value of var_b loss = var_d.sum() loss.backward() @@ -268,7 +276,9 @@ class TestDygraphInplace(unittest.TestCase): var_c = self.non_inplace_api_processing(var_b) - var_d = var_c + var_c # Here, the grad op of sum doesn't use the value of var_b + var_d = ( + var_c + var_c + ) # Here, the grad op of sum doesn't use the value of var_b loss = var_d.sum() loss.backward() @@ -282,7 +292,6 @@ class TestDygraphInplace(unittest.TestCase): class TestDygraphInplaceUnsqueeze(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.unsqueeze(var, -1) @@ -291,7 +300,6 @@ class TestDygraphInplaceUnsqueeze(TestDygraphInplace): class TestDygraphInplaceReshape(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.reshape(var, [-1]) @@ -300,7 +308,6 @@ class TestDygraphInplaceReshape(TestDygraphInplace): class TestDygraphInplaceReshapeTensor(TestDygraphInplace): - def non_inplace_api_processing(self, var): shape = paddle.to_tensor(-1) return paddle.reshape(var, shape) @@ -311,7 +318,6 @@ class TestDygraphInplaceReshapeTensor(TestDygraphInplace): class TestDygraphInplaceFlatten(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.flatten() @@ -320,28 +326,28 @@ class TestDygraphInplaceFlatten(TestDygraphInplace): class TestDygraphInplaceScatter(TestDygraphInplace): - def init_data(self): self.input_var_numpy = np.array([[1, 1], [2, 2], [3, 3]]) self.dtype = "float32" def non_inplace_api_processing(self, var): index = paddle.to_tensor([2, 1, 0, 1], dtype='int64') - updates = paddle.to_tensor([[1, 1], [2, 2], [3, 3], [4, 4]], - dtype='float32') + updates = paddle.to_tensor( + [[1, 1], [2, 2], [3, 3], [4, 4]], dtype='float32' + ) return paddle.scatter(var, index, updates, overwrite=False) def inplace_api_processing(self, var): index = paddle.to_tensor([2, 1, 0, 1], dtype='int64') - updates = paddle.to_tensor([[1, 1], [2, 2], [3, 3], [4, 4]], - dtype='float32') + updates = paddle.to_tensor( + [[1, 1], [2, 2], [3, 3], [4, 4]], dtype='float32' + ) return paddle.scatter_(var, index, updates, overwrite=False) class TestDygraphInplaceElu(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.nn.functional.elu(var) @@ -350,7 +356,6 @@ class TestDygraphInplaceElu(TestDygraphInplace): class TestDygraphInplaceRelu(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.nn.functional.relu(var) @@ -359,7 +364,6 @@ class TestDygraphInplaceRelu(TestDygraphInplace): class TestDygraphInplaceSoftmax(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.nn.functional.softmax(var) @@ -368,7 +372,6 @@ class TestDygraphInplaceSoftmax(TestDygraphInplace): class TestDygraphInplaceTanh(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.tanh(var) @@ -377,7 +380,6 @@ class TestDygraphInplaceTanh(TestDygraphInplace): class TestDygraphInplaceCeil(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.ceil() @@ -386,7 +388,6 @@ class TestDygraphInplaceCeil(TestDygraphInplace): class TestDygraphInplaceFloor(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.floor() @@ -395,7 +396,6 @@ class TestDygraphInplaceFloor(TestDygraphInplace): class TestDygraphInplaceExp(TestDygraphInplace): - def set_np_compare_func(self): self.np_compare = np.allclose @@ -407,7 +407,6 @@ class TestDygraphInplaceExp(TestDygraphInplace): class TestDygraphInplaceReciprocal(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.reciprocal() @@ -416,7 +415,6 @@ class TestDygraphInplaceReciprocal(TestDygraphInplace): class TestDygraphInplaceRound(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.round() @@ -425,7 +423,6 @@ class TestDygraphInplaceRound(TestDygraphInplace): class TestDygraphInplaceSqrt(TestDygraphInplace): - def init_data(self): self.input_var_numpy = np.random.uniform(0, 5, [10, 20, 1]) self.dtype = "float32" @@ -438,7 +435,6 @@ class TestDygraphInplaceSqrt(TestDygraphInplace): class TestDygraphInplaceRsqrt(TestDygraphInplaceSqrt): - def non_inplace_api_processing(self, var): return var.rsqrt() @@ -447,7 +443,6 @@ class TestDygraphInplaceRsqrt(TestDygraphInplaceSqrt): class TestDygraphInplaceClip(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.clip(0.6, 1.5) @@ -456,7 +451,6 @@ class TestDygraphInplaceClip(TestDygraphInplace): class TestDygraphInplaceScale(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.scale(scale=2.0, bias=3.0) @@ -465,7 +459,6 @@ class TestDygraphInplaceScale(TestDygraphInplace): class TestDygraphInplaceAdd(TestDygraphInplace): - def init_data(self): self.input_var_numpy = np.random.rand(2, 3, 4) self.dtype = "float32" @@ -481,7 +474,6 @@ class TestDygraphInplaceAdd(TestDygraphInplace): class TestDygraphInplaceSubtract(TestDygraphInplaceAdd): - def non_inplace_api_processing(self, var): input_var_2 = paddle.to_tensor(self.input_var_numpy_2) return var.subtract(input_var_2) @@ -492,7 +484,6 @@ class TestDygraphInplaceSubtract(TestDygraphInplaceAdd): class TestDygraphInplaceRemainder(TestDygraphInplaceAdd): - def non_inplace_api_processing(self, var): input_var_2 = paddle.to_tensor(self.input_var_numpy_2) return var.remainder(input_var_2) @@ -515,7 +506,6 @@ class TestDygraphInplaceRemainder(TestDygraphInplaceAdd): class TestLossIsInplaceVar(unittest.TestCase): - def func_test_loss_is_inplace_var(self): with paddle.fluid.dygraph.guard(): var_a = paddle.ones((2, 2)) @@ -546,7 +536,6 @@ class TestLossIsInplaceVar(unittest.TestCase): class TestContinuouslyInplace(unittest.TestCase): - def func_test_continuously_inplace(self): a = paddle.rand([2, 3]) a.stop_gradient = False @@ -565,7 +554,6 @@ class TestContinuouslyInplace(unittest.TestCase): class TestGetitemBeforeInplace(unittest.TestCase): - def test_getitem_before_inplace(self): with _test_eager_guard(): a = paddle.ones(shape=[4, 2, 3], dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_inplace_abn_op.py b/python/paddle/fluid/tests/unittests/test_inplace_abn_op.py index 998480a9193..1048c6710d2 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_abn_op.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_abn_op.py @@ -22,7 +22,6 @@ import paddle class TestInplaceANBOpTraining(unittest.TestCase): - def setUp(self): self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.N = 4 @@ -31,26 +30,30 @@ class TestInplaceANBOpTraining(unittest.TestCase): self.W = 9 self.dshape = [self.N, self.C, self.H, self.W] - def build_program(self, - place, - layout, - seed, - only_forward=False, - activation="identity", - alpha=1.0, - use_cuda=False, - inplace=False): + def build_program( + self, + place, + layout, + seed, + only_forward=False, + activation="identity", + alpha=1.0, + use_cuda=False, + inplace=False, + ): main = fluid.Program() startup = fluid.Program() main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - data = fluid.layers.data(name='input', - shape=self.dshape, - dtype=self.dtype, - append_batch_size=False, - stop_gradient=False) + data = fluid.layers.data( + name='input', + shape=self.dshape, + dtype=self.dtype, + append_batch_size=False, + stop_gradient=False, + ) if inplace: bn = fluid.layers.inplace_abn( data, @@ -61,7 +64,8 @@ class TestInplaceANBOpTraining(unittest.TestCase): moving_variance_name='bn_moving_variance', data_layout=layout, is_test=only_forward, - act_alpha=alpha) + act_alpha=alpha, + ) else: bn = fluid.layers.batch_norm( data, @@ -71,7 +75,8 @@ class TestInplaceANBOpTraining(unittest.TestCase): moving_variance_name='bn_moving_variance', data_layout=layout, is_test=only_forward, - in_place=inplace) + in_place=inplace, + ) if activation == 'leaky_relu': bn = fluid.layers.leaky_relu(bn, alpha) if activation == 'elu': @@ -80,7 +85,7 @@ class TestInplaceANBOpTraining(unittest.TestCase): # NOTE: in inplace mode input and output of bn # may have same name, multiply 1. to generate # a new Variable for fetch - bn = bn * 1. + bn = bn * 1.0 sigmoid = fluid.layers.sigmoid(bn) out = fluid.layers.reduce_sum(sigmoid) @@ -92,23 +97,28 @@ class TestInplaceANBOpTraining(unittest.TestCase): def compare(self, place, layout, only_forward, activation, alpha, use_cuda): seed = 10 os.environ['FLAGS_cudnn_deterministic'] = "1" - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 fetch_outs = [] fetch_names = [] for inplace in [False, True]: - main, startup, outs = self.build_program(place, - layout, - seed, - only_forward, - activation, - alpha, - inplace=inplace) + main, startup, outs = self.build_program( + place, + layout, + seed, + only_forward, + activation, + alpha, + inplace=inplace, + ) exe = fluid.Executor(place) exe.run(startup) fetch_name = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ @@ -124,50 +134,69 @@ class TestInplaceANBOpTraining(unittest.TestCase): fv.persistable = True build_strategy = fluid.BuildStrategy() - build_strategy.sync_batch_norm = use_cuda and \ - fluid.core.get_cuda_device_count() > 1 + build_strategy.sync_batch_norm = ( + use_cuda and fluid.core.get_cuda_device_count() > 1 + ) build_strategy.enable_inplace = inplace exec_strategy = fluid.ExecutionStrategy() exec_strategy.num_threads = 1 if os.name == 'nt' else 0 comp_prog1 = compiler.CompiledProgram(main).with_data_parallel( outs[0].name if not only_forward else None, build_strategy=build_strategy, - exec_strategy=exec_strategy) - bn_fetches = exe.run(program=main, - feed={'input': data}, - fetch_list=fetch_name) + exec_strategy=exec_strategy, + ) + bn_fetches = exe.run( + program=main, feed={'input': data}, fetch_list=fetch_name + ) fetch_outs.append(bn_fetches) fetch_names.append(fetch_name) - for bn_val, inplace_abn_val, name1, name2 in zip(*(fetch_outs + - fetch_names)): + for bn_val, inplace_abn_val, name1, name2 in zip( + *(fetch_outs + fetch_names) + ): np.testing.assert_allclose( bn_val, inplace_abn_val, rtol=1e-05, atol=0.01, - err_msg='Output (' + name1 + ':' + name2 + - ') has diff on {} with {} layout and {} activation. \n'.format( - place, layout, activation) + '\nBN ' + str(bn_val) + - '\n' + 'Inplace ABN ' + str(inplace_abn_val)) + err_msg='Output (' + + name1 + + ':' + + name2 + + ') has diff on {} with {} layout and {} activation. \n'.format( + place, layout, activation + ) + + '\nBN ' + + str(bn_val) + + '\n' + + 'Inplace ABN ' + + str(inplace_abn_val), + ) def test_op(self): use_cudas = [False, True] if core.is_compiled_with_cuda() else [False] - #use_cudas = [False] + # use_cudas = [False] for use_cuda in use_cudas: place = core.CUDAPlace(0) if use_cuda else core.CPUPlace() layouts = ["NCHW", "NHWC"] for layout in layouts: - for activation, alpha in zip([None, 'elu', 'leaky_relu'], - [0., 1., 0.02]): + for activation, alpha in zip( + [None, 'elu', 'leaky_relu'], [0.0, 1.0, 0.02] + ): for infer_only in [True, False]: - self.compare(place, layout, infer_only, activation, - alpha, use_cuda) + self.compare( + place, + layout, + infer_only, + activation, + alpha, + use_cuda, + ) def test_all_branches(self): seed = 10 os.environ['FLAGS_cudnn_deterministic'] = "1" - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 use_cudas = [False, True] if core.is_compiled_with_cuda() else [False] alpha = 0.1 layouts = ["NCHW", "NHWC"] @@ -176,8 +205,15 @@ class TestInplaceANBOpTraining(unittest.TestCase): for layout in layouts: for activation in ['identity', 'leaky_relu']: main, startup, outs = self.build_program( - place, layout, seed, False, activation, alpha, use_cuda, - True) + place, + layout, + seed, + False, + activation, + alpha, + use_cuda, + True, + ) exe = fluid.Executor(place) exe.run(startup) exe.run(program=main, feed={'input': data}) diff --git a/python/paddle/fluid/tests/unittests/test_inplace_addto_strategy.py b/python/paddle/fluid/tests/unittests/test_inplace_addto_strategy.py index 825fcbd0620..4f0aebf6c3c 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_addto_strategy.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_addto_strategy.py @@ -20,27 +20,31 @@ import numpy as np class ConvBNLayer(fluid.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - data_format="NCHW"): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + data_format="NCHW", + ): super(ConvBNLayer, self).__init__() - self._conv = paddle.nn.Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False, - data_format=data_format) - - self._batch_norm = paddle.nn.BatchNorm(num_filters, - data_layout=data_format) + self._conv = paddle.nn.Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + data_format=data_format, + ) + + self._batch_norm = paddle.nn.BatchNorm( + num_filters, data_layout=data_format + ) def forward(self, inputs): y = self._conv(inputs) @@ -57,10 +61,12 @@ def create_program(data_format="NCHW"): if data_format == "NHWC": x = paddle.transpose(x, [0, 2, 3, 1]) x = fluid.layers.prelu(x, mode="channel") - conv = ConvBNLayer(num_channels=3, - num_filters=3, - filter_size=1, - data_format=data_format) + conv = ConvBNLayer( + num_channels=3, + num_filters=3, + filter_size=1, + data_format=data_format, + ) y = conv(x) + x loss = fluid.layers.reduce_sum(y) @@ -72,9 +78,7 @@ def create_program(data_format="NCHW"): class TestInplaceAddto(unittest.TestCase): - def check_result(self, data_format="NCHW"): - def run_program(enable_addto): np.random.seed(10) paddle.seed(10) @@ -83,22 +87,27 @@ class TestInplaceAddto(unittest.TestCase): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) fluid.set_flags({"FLAGS_max_inplace_grad_add": 2}) loss, main, startup, w = create_program(data_format=data_format) - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) strategy = fluid.BuildStrategy() strategy.enable_addto = enable_addto compiled = fluid.CompiledProgram(main).with_data_parallel( - loss_name=loss.name, build_strategy=strategy) + loss_name=loss.name, build_strategy=strategy + ) exe.run(startup) - img = np.random.uniform(-128, 128, - [8, 3, 224, 224]).astype(np.float32) + img = np.random.uniform(-128, 128, [8, 3, 224, 224]).astype( + np.float32 + ) for i in range(10): - res = exe.run(compiled, - feed={'img': img}, - fetch_list=[loss.name, w.name]) + res = exe.run( + compiled, feed={'img': img}, fetch_list=[loss.name, w.name] + ) return res res1, w1 = run_program(True) diff --git a/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py b/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py index 7f4bf6a5fef..aa7e837b6cd 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py @@ -21,7 +21,6 @@ paddle.disable_static() def clear_grad(w, a): - @paddle.no_grad() def warp(*_): assert w.grad is not None @@ -32,7 +31,6 @@ def clear_grad(w, a): class TestInplaceAndClearGradient(unittest.TestCase): - def test(self): paddle.set_device('cpu') diff --git a/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py b/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py index 09e8c611236..d8f8b5dbc72 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py @@ -21,7 +21,6 @@ from paddle.static import Program, program_guard # In static mode, inplace strategy will not be used in Inplace APIs. class TestStaticAutoGeneratedAPI(unittest.TestCase): - def setUp(self): paddle.enable_static() self.init_data() @@ -48,23 +47,22 @@ class TestStaticAutoGeneratedAPI(unittest.TestCase): out = self.executed_paddle_api(x) exe = paddle.static.Executor(place=paddle.CPUPlace()) - fetch_x, fetch_out = exe.run(main_prog, - feed={"x": self.np_x}, - fetch_list=[x, out]) + fetch_x, fetch_out = exe.run( + main_prog, feed={"x": self.np_x}, fetch_list=[x, out] + ) np.testing.assert_array_equal(fetch_x, self.np_x) self.assertTrue( - self.np_compare(fetch_out, self.executed_numpy_api(self.np_x))) + self.np_compare(fetch_out, self.executed_numpy_api(self.np_x)) + ) class TestStaticInplaceAutoGeneratedAPI(TestStaticAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.ceil_() class TestStaticFloorAPI(TestStaticAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.floor() @@ -73,13 +71,11 @@ class TestStaticFloorAPI(TestStaticAutoGeneratedAPI): class TestStaticInplaceFloorAPI(TestStaticFloorAPI): - def executed_paddle_api(self, x): return x.floor_() class TestStaticExpAPI(TestStaticAutoGeneratedAPI): - def set_np_compare_func(self): self.np_compare = np.allclose @@ -91,13 +87,11 @@ class TestStaticExpAPI(TestStaticAutoGeneratedAPI): class TestStaticInplaceExpAPI(TestStaticExpAPI): - def executed_paddle_api(self, x): return x.exp_() class TestStaticReciprocalAPI(TestStaticAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.reciprocal() @@ -106,13 +100,11 @@ class TestStaticReciprocalAPI(TestStaticAutoGeneratedAPI): class TestStaticInplaceReciprocalAPI(TestStaticReciprocalAPI): - def executed_paddle_api(self, x): return x.reciprocal_() class TestStaticRoundAPI(TestStaticAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.round() @@ -121,13 +113,11 @@ class TestStaticRoundAPI(TestStaticAutoGeneratedAPI): class TestStaticInplaceRoundAPI(TestStaticRoundAPI): - def executed_paddle_api(self, x): return x.round_() class TestStaticSqrtAPI(TestStaticAutoGeneratedAPI): - def init_data(self): self.dtype = 'float32' self.shape = [10, 20] @@ -144,13 +134,11 @@ class TestStaticSqrtAPI(TestStaticAutoGeneratedAPI): class TestStaticInplaceSqrtAPI(TestStaticSqrtAPI): - def executed_paddle_api(self, x): return x.sqrt_() class TestStaticRsqrtAPI(TestStaticSqrtAPI): - def executed_paddle_api(self, x): return x.rsqrt() @@ -159,14 +147,12 @@ class TestStaticRsqrtAPI(TestStaticSqrtAPI): class TestStaticInplaceRsqrtAPI(TestStaticRsqrtAPI): - def executed_paddle_api(self, x): return x.rsqrt_() # In dygraph mode, inplace strategy will be used in Inplace APIs. class TestDygraphAutoGeneratedAPI(unittest.TestCase): - def setUp(self): paddle.disable_static() self.init_data() @@ -191,17 +177,16 @@ class TestDygraphAutoGeneratedAPI(unittest.TestCase): out = self.executed_paddle_api(x) self.assertTrue( - self.np_compare(out.numpy(), self.executed_numpy_api(self.np_x))) + self.np_compare(out.numpy(), self.executed_numpy_api(self.np_x)) + ) class TestDygraphInplaceAutoGeneratedAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.ceil_() class TestDygraphFloorAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.floor() @@ -210,13 +195,11 @@ class TestDygraphFloorAPI(TestDygraphAutoGeneratedAPI): class TestDygraphInplaceFloorAPI(TestDygraphFloorAPI): - def executed_paddle_api(self, x): return x.floor_() class TestDygraphExpAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.exp() @@ -228,13 +211,11 @@ class TestDygraphExpAPI(TestDygraphAutoGeneratedAPI): class TestDygraphInplaceExpAPI(TestDygraphExpAPI): - def executed_paddle_api(self, x): return x.exp_() class TestDygraphReciprocalAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.reciprocal() @@ -243,13 +224,11 @@ class TestDygraphReciprocalAPI(TestDygraphAutoGeneratedAPI): class TestDygraphInplaceReciprocalAPI(TestDygraphReciprocalAPI): - def executed_paddle_api(self, x): return x.reciprocal_() class TestDygraphRoundAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.round() @@ -258,13 +237,11 @@ class TestDygraphRoundAPI(TestDygraphAutoGeneratedAPI): class TestDygraphInplaceRoundAPI(TestDygraphRoundAPI): - def executed_paddle_api(self, x): return x.round_() class TestDygraphSqrtAPI(TestDygraphAutoGeneratedAPI): - def init_data(self): self.dtype = 'float32' self.shape = [10, 20] @@ -281,22 +258,19 @@ class TestDygraphSqrtAPI(TestDygraphAutoGeneratedAPI): class TestDygraphInplaceSqrtAPI(TestDygraphSqrtAPI): - def executed_paddle_api(self, x): return x.sqrt_() class TestDygraphRsqrtAPI(TestDygraphSqrtAPI): - def executed_paddle_api(self, x): return x.rsqrt() def executed_numpy_api(self, x): - return 1. / np.sqrt(x) + return 1.0 / np.sqrt(x) class TestDygraphInplaceRsqrtAPI(TestDygraphRsqrtAPI): - def executed_paddle_api(self, x): return x.rsqrt_() diff --git a/python/paddle/fluid/tests/unittests/test_inplace_softmax_with_cross_entropy.py b/python/paddle/fluid/tests/unittests/test_inplace_softmax_with_cross_entropy.py index 759454a2887..8e9b154a41c 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_softmax_with_cross_entropy.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_softmax_with_cross_entropy.py @@ -18,40 +18,41 @@ import unittest class TestSoftmaxWithXe(unittest.TestCase): - def setUp(self): self.initParameter() - self.m, self.n = np.random.random_integers(low=100, high=2000, - size=[2]).astype('int64') + self.m, self.n = np.random.random_integers( + low=100, high=2000, size=[2] + ).astype('int64') def initParameter(self): self.dtype = 'float32' self.soft_label = False - def softmax_with_xe(self, - x, - y, - place, - inplace=True, - numeric_stable_mode=True): + def softmax_with_xe( + self, x, y, place, inplace=True, numeric_stable_mode=True + ): m, n = x.shape with fluid.program_guard(fluid.Program(), fluid.Program()): with fluid.scope_guard(fluid.Scope()): - x_d = fluid.layers.data(name='x', - shape=[m, n], - dtype=self.dtype, - append_batch_size=False) + x_d = fluid.layers.data( + name='x', + shape=[m, n], + dtype=self.dtype, + append_batch_size=False, + ) y_d = fluid.layers.data( name='y', shape=[m, 1] if not self.soft_label else [m, n], dtype='int64' if not self.soft_label else self.dtype, - append_batch_size=False) + append_batch_size=False, + ) z_d, s_d = fluid.layers.softmax_with_cross_entropy( x_d, y_d, soft_label=self.soft_label, return_softmax=True, - numeric_stable_mode=numeric_stable_mode) + numeric_stable_mode=numeric_stable_mode, + ) exe = fluid.Executor(place) @@ -60,19 +61,18 @@ class TestSoftmaxWithXe(unittest.TestCase): build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = inplace prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - build_strategy=build_strategy, places=place) + fluid.default_main_program() + ).with_data_parallel( + build_strategy=build_strategy, places=place + ) fetch_list = [z_d.name, s_d.name] print('Inplace is {}'.format("ON" if inplace else "OFF")) - z, s = exe.run(prog, - feed={ - x_d.name: x, - y_d.name: y - }, - fetch_list=fetch_list) + z, s = exe.run( + prog, feed={x_d.name: x, y_d.name: y}, fetch_list=fetch_list + ) return z, s def main_with_place(self, place): @@ -82,39 +82,32 @@ class TestSoftmaxWithXe(unittest.TestCase): for a, b in x_range: x = ((b - a) * x + a).astype(self.dtype) if not self.soft_label: - y = np.random.random_integers(size=[self.m, 1], - low=0, - high=self.n - 1).astype('int64') + y = np.random.random_integers( + size=[self.m, 1], low=0, high=self.n - 1 + ).astype('int64') else: y = np.random.random(size=[self.m, self.n]).astype(self.dtype) - norm_y = np.broadcast_to(np.reshape(np.sum(y, axis=1), [-1, 1]), - y.shape) + norm_y = np.broadcast_to( + np.reshape(np.sum(y, axis=1), [-1, 1]), y.shape + ) y = y / norm_y - z1, s1 = self.softmax_with_xe(x, - y, - place, - inplace=False, - numeric_stable_mode=False) - z2, s2 = self.softmax_with_xe(x, - y, - place, - inplace=True, - numeric_stable_mode=False) + z1, s1 = self.softmax_with_xe( + x, y, place, inplace=False, numeric_stable_mode=False + ) + z2, s2 = self.softmax_with_xe( + x, y, place, inplace=True, numeric_stable_mode=False + ) self.assertTrue((z1 == z2).all()) self.assertTrue((s1 == s2).all()) - z1, s1 = self.softmax_with_xe(x, - y, - place, - inplace=False, - numeric_stable_mode=True) - z2, s2 = self.softmax_with_xe(x, - y, - place, - inplace=True, - numeric_stable_mode=True) + z1, s1 = self.softmax_with_xe( + x, y, place, inplace=False, numeric_stable_mode=True + ) + z2, s2 = self.softmax_with_xe( + x, y, place, inplace=True, numeric_stable_mode=True + ) self.assertTrue((z1 == z2).all()) self.assertTrue((s1 == s2).all()) @@ -125,21 +118,18 @@ class TestSoftmaxWithXe(unittest.TestCase): class TestSoftmaxWithXe1(TestSoftmaxWithXe): - def initParameter(self): self.dtype = 'float32' self.soft_label = True class TestSoftmaxWithXe2(TestSoftmaxWithXe): - def initParameter(self): self.dtype = 'float64' self.soft_label = False class TestSoftmaxWithXe3(TestSoftmaxWithXe): - def initParameter(self): self.dtype = 'float64' self.soft_label = True diff --git a/python/paddle/fluid/tests/unittests/test_input_spec.py b/python/paddle/fluid/tests/unittests/test_input_spec.py index bb31c4e7f51..9d805d64e93 100644 --- a/python/paddle/fluid/tests/unittests/test_input_spec.py +++ b/python/paddle/fluid/tests/unittests/test_input_spec.py @@ -21,15 +21,17 @@ import paddle import paddle.fluid as fluid from paddle.static import InputSpec from paddle.fluid.framework import convert_np_dtype_to_dtype_ -from paddle.fluid.dygraph.dygraph_to_static.utils import _compatible_non_tensor_spec +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + _compatible_non_tensor_spec, +) class TestInputSpec(unittest.TestCase): - def test_default(self): tensor_spec = InputSpec([3, 4]) - self.assertEqual(tensor_spec.dtype, - convert_np_dtype_to_dtype_('float32')) + self.assertEqual( + tensor_spec.dtype, convert_np_dtype_to_dtype_('float32') + ) self.assertEqual(tensor_spec.name, None) def test_from_tensor(self): @@ -45,15 +47,17 @@ class TestInputSpec(unittest.TestCase): def test_from_numpy(self): x_numpy = np.ones([10, 12]) x_np_spec = InputSpec.from_numpy(x_numpy) - self.assertEqual(x_np_spec.dtype, - convert_np_dtype_to_dtype_(x_numpy.dtype)) + self.assertEqual( + x_np_spec.dtype, convert_np_dtype_to_dtype_(x_numpy.dtype) + ) self.assertEqual(x_np_spec.shape, x_numpy.shape) self.assertEqual(x_np_spec.name, None) x_numpy2 = np.array([1, 2, 3, 4]).astype('int64') x_np_spec2 = InputSpec.from_numpy(x_numpy2, name='x_np_int64') - self.assertEqual(x_np_spec2.dtype, - convert_np_dtype_to_dtype_(x_numpy2.dtype)) + self.assertEqual( + x_np_spec2.dtype, convert_np_dtype_to_dtype_(x_numpy2.dtype) + ) self.assertEqual(x_np_spec2.shape, x_numpy2.shape) self.assertEqual(x_np_spec2.name, 'x_np_int64') @@ -84,7 +88,7 @@ class TestInputSpec(unittest.TestCase): # unbatch unbatch_spec = batch_tensor_spec.unbatch() - self.assertEqual(unbatch_spec.shape, (10, )) + self.assertEqual(unbatch_spec.shape, (10,)) # 1. `unbatch` requires len(shape) > 1 with self.assertRaises(ValueError): @@ -116,7 +120,6 @@ class TestInputSpec(unittest.TestCase): class NetWithNonTensorSpec(paddle.nn.Layer): - def __init__(self, in_num, out_num): super(NetWithNonTensorSpec, self).__init__() self.linear_1 = paddle.nn.Linear(in_num, out_num) @@ -157,7 +160,6 @@ class NetWithNonTensorSpec(paddle.nn.Layer): class TestNetWithNonTensorSpec(unittest.TestCase): - def setUp(self): self.in_num = 16 self.out_num = 16 @@ -243,7 +245,6 @@ class TestNetWithNonTensorSpec(unittest.TestCase): class NetWithNonTensorSpecPrune(paddle.nn.Layer): - def __init__(self, in_num, out_num): super(NetWithNonTensorSpecPrune, self).__init__() self.linear_1 = paddle.nn.Linear(in_num, out_num) @@ -263,7 +264,6 @@ class NetWithNonTensorSpecPrune(paddle.nn.Layer): class TestNetWithNonTensorSpecWithPrune(unittest.TestCase): - def setUp(self): self.in_num = 16 self.out_num = 16 @@ -311,7 +311,6 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase): class UnHashableObject: - def __init__(self, val): self.val = val @@ -320,7 +319,6 @@ class UnHashableObject: class TestCompatibleNonTensorSpec(unittest.TestCase): - def test_case(self): self.assertTrue(_compatible_non_tensor_spec([1, 2, 3], [1, 2, 3])) self.assertFalse(_compatible_non_tensor_spec([1, 2, 3], [1, 2])) @@ -328,8 +326,10 @@ class TestCompatibleNonTensorSpec(unittest.TestCase): # not supported unhashable object. self.assertTrue( - _compatible_non_tensor_spec(UnHashableObject(1), - UnHashableObject(1))) + _compatible_non_tensor_spec( + UnHashableObject(1), UnHashableObject(1) + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_install_check.py b/python/paddle/fluid/tests/unittests/test_install_check.py index e51079278db..15f2b5f3b7e 100644 --- a/python/paddle/fluid/tests/unittests/test_install_check.py +++ b/python/paddle/fluid/tests/unittests/test_install_check.py @@ -18,7 +18,6 @@ import os class TestInstallCheck(unittest.TestCase): - def test_paddle_fluid(self): paddle.fluid.install_check.run_check() diff --git a/python/paddle/fluid/tests/unittests/test_instance_norm_op.py b/python/paddle/fluid/tests/unittests/test_instance_norm_op.py index 88bed96b352..50278505f98 100644 --- a/python/paddle/fluid/tests/unittests/test_instance_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_instance_norm_op.py @@ -62,10 +62,19 @@ def _reference_instance_norm_grad(x, d_y, scale, mean, var, epsilon): scale_tile = np.reshape(scale, (1, c, 1, 1)) scale_tile = np.tile(scale_tile, (n, 1, h, w)) - d_x = scale_tile * var_inv * ( - d_y - np.mean(d_y, axis=(2, 3), keepdims=True) - - (x - mean_tile) * var_inv * - np.mean(d_y * (x - mean_tile) * var_inv, axis=(2, 3), keepdims=True)) + d_x = ( + scale_tile + * var_inv + * ( + d_y + - np.mean(d_y, axis=(2, 3), keepdims=True) + - (x - mean_tile) + * var_inv + * np.mean( + d_y * (x - mean_tile) * var_inv, axis=(2, 3), keepdims=True + ) + ) + ) return d_x, d_scale, d_bias @@ -76,7 +85,6 @@ def _cal_mean_variance(x, epsilon, mean_shape): class TestInstanceNormOpTraining(unittest.TestCase): - def setUp(self): self.epsilon = 1e-5 self.init_test_case() @@ -85,23 +93,24 @@ class TestInstanceNormOpTraining(unittest.TestCase): self.shape = [2, 3, 4, 5] self.no_grad_set = set() self.fetch_list = [ - 'y', 'saved_mean', 'saved_variance', 'x@GRAD', 'scale@GRAD', - 'bias@GRAD' + 'y', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) def set_global_mean_var(self, mean_shape, x): mean, variance = _cal_mean_variance(x, self.epsilon, mean_shape) return mean, variance def test_forward_backward(self): - def test_with_place(place, shape): epsilon = self.epsilon n, c, h, w = shape[0], shape[1], shape[2], shape[3] @@ -116,12 +125,14 @@ class TestInstanceNormOpTraining(unittest.TestCase): d_y = np.random.random_sample(shape).astype(np.float32) y, saved_mean, variance_tmp = _reference_instance_norm_naive( - x, scale, bias, epsilon, mean, variance) + x, scale, bias, epsilon, mean, variance + ) saved_variance = 1 / np.sqrt(variance_tmp + epsilon) d_x, d_scale, d_bias = _reference_instance_norm_grad( - x, d_y, scale, saved_mean, saved_variance, epsilon) + x, d_y, scale, saved_mean, saved_variance, epsilon + ) var_dict = locals() var_dict['y@GRAD'] = d_y @@ -130,7 +141,12 @@ class TestInstanceNormOpTraining(unittest.TestCase): var_dict['bias@GRAD'] = d_bias var_names = [ - 'x', 'scale', 'bias', 'y', 'saved_mean', 'saved_variance' + 'x', + 'scale', + 'bias', + 'y', + 'saved_mean', + 'saved_variance', ] ground_truth = {name: var_dict[name] for name in var_names} @@ -138,31 +154,33 @@ class TestInstanceNormOpTraining(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) - in_op = block.append_op(type="instance_norm", - inputs={ - "X": block.var("x"), - "Scale": block.var("scale"), - "Bias": block.var("bias"), - }, - outputs={ - "Y": - block.var("y"), - "SavedMean": - block.var("saved_mean"), - "SavedVariance": - block.var("saved_variance") - }, - attrs={ - "epsilon": epsilon, - }) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) + in_op = block.append_op( + type="instance_norm", + inputs={ + "X": block.var("x"), + "Scale": block.var("scale"), + "Bias": block.var("bias"), + }, + outputs={ + "Y": block.var("y"), + "SavedMean": block.var("saved_mean"), + "SavedVariance": block.var("saved_variance"), + }, + attrs={ + "epsilon": epsilon, + }, + ) block.create_var(name="y@GRAD", dtype='float32', shape=y.shape) grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - in_op.desc, self.no_grad_set, []) + in_op.desc, self.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -177,12 +195,14 @@ class TestInstanceNormOpTraining(unittest.TestCase): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'scale', 'bias', 'y@GRAD'] - }, - fetch_list=self.fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in ['x', 'scale', 'bias', 'y@GRAD'] + }, + fetch_list=self.fetch_list, + ) for id, name in enumerate(self.fetch_list): self.__assert_close(var_dict[name], out[id], name) @@ -191,14 +211,14 @@ class TestInstanceNormOpTraining(unittest.TestCase): places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): places.append(core.CUDAPlace(0)) for place in places: test_with_place(place, self.shape) class TestInstanceNormOpTrainingCase1(TestInstanceNormOpTraining): - def init_test_case(self): self.shape = [2, 3, 4, 5] self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -206,7 +226,6 @@ class TestInstanceNormOpTrainingCase1(TestInstanceNormOpTraining): class TestInstanceNormOpTrainingCase2(TestInstanceNormOpTraining): - def init_test_case(self): self.shape = [20, 50, 4, 5] self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -214,12 +233,12 @@ class TestInstanceNormOpTrainingCase2(TestInstanceNormOpTraining): class TestInstanceNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of instance_norm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.instance_norm, x1) # the input dtype of instance_norm must be float32 or float64 @@ -228,24 +247,22 @@ class TestInstanceNormOpError(unittest.TestCase): class TestInstanceNormOpErrorCase1(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the first dimension of input for instance_norm must between [2d, 5d] - x = fluid.layers.data(name='x', - shape=[3], - dtype="float32", - append_batch_size=False) + x = fluid.layers.data( + name='x', shape=[3], dtype="float32", append_batch_size=False + ) self.assertRaises(ValueError, paddle.static.nn.instance_norm, x) class TestElasticNormOp(unittest.TestCase): - def init_test_case(self): self.epsilon = 1e-5 self.places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): self.places.append(core.CUDAPlace(0)) def test_norm(self): @@ -258,20 +275,19 @@ class TestElasticNormOp(unittest.TestCase): scale = np.ones(scale_shape).astype(np.float32) bias = np.zeros(scale_shape).astype(np.float32) mean, variance = _cal_mean_variance(inputs, self.epsilon, mean_shape) - out_np, _, _ = _reference_instance_norm_naive(inputs, scale, bias, - self.epsilon, mean, - variance) + out_np, _, _ = _reference_instance_norm_naive( + inputs, scale, bias, self.epsilon, mean, variance + ) for place in self.places: with fluid.dygraph.guard(place): - instance_norm = fluid.dygraph.InstanceNorm(5, - param_attr=False, - bias_attr=False) + instance_norm = fluid.dygraph.InstanceNorm( + 5, param_attr=False, bias_attr=False + ) outputs = instance_norm(to_variable(inputs)) - np.testing.assert_allclose(outputs.numpy(), - out_np, - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + outputs.numpy(), out_np, rtol=1e-05, atol=1e-06 + ) def test_eager_api(self): with _test_eager_guard(): @@ -279,12 +295,12 @@ class TestElasticNormOp(unittest.TestCase): class TestElasticNormOpCase2(unittest.TestCase): - def init_test_case(self): self.epsilon = 1e-5 self.places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): self.places.append(core.CUDAPlace(0)) def test_norm(self): @@ -297,20 +313,19 @@ class TestElasticNormOpCase2(unittest.TestCase): scale = np.ones(scale_shape).astype(np.float32) bias = np.zeros(scale_shape).astype(np.float32) mean, variance = _cal_mean_variance(inputs, self.epsilon, mean_shape) - out_np, _, _ = _reference_instance_norm_naive(inputs, scale, bias, - self.epsilon, mean, - variance) + out_np, _, _ = _reference_instance_norm_naive( + inputs, scale, bias, self.epsilon, mean, variance + ) for place in self.places: with fluid.dygraph.guard(place): - instance_norm = fluid.dygraph.InstanceNorm(3, - param_attr=True, - bias_attr=True) + instance_norm = fluid.dygraph.InstanceNorm( + 3, param_attr=True, bias_attr=True + ) outputs = instance_norm(to_variable(inputs)) - np.testing.assert_allclose(outputs.numpy(), - out_np, - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + outputs.numpy(), out_np, rtol=1e-05, atol=1e-06 + ) def test_eager_api(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_instance_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_instance_norm_op_v2.py index cbb91795a50..66bbc3a3387 100644 --- a/python/paddle/fluid/tests/unittests/test_instance_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_instance_norm_op_v2.py @@ -23,11 +23,11 @@ import paddle class TestInstanceNorm(unittest.TestCase): - def test_error(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): places.append(fluid.CUDAPlace(0)) for p in places: @@ -48,9 +48,9 @@ class TestInstanceNorm(unittest.TestCase): def weight_bias_false(): x_data_4 = np.random.random(size=(2, 1, 3, 3)).astype('float32') - instance_norm3d = paddle.nn.InstanceNorm3D(1, - weight_attr=False, - bias_attr=False) + instance_norm3d = paddle.nn.InstanceNorm3D( + 1, weight_attr=False, bias_attr=False + ) with fluid.dygraph.guard(p): weight_bias_false() @@ -61,7 +61,8 @@ class TestInstanceNorm(unittest.TestCase): def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): places.append(fluid.CUDAPlace(0)) for p in places: shape = [4, 10, 4, 4] @@ -86,7 +87,8 @@ class TestInstanceNorm(unittest.TestCase): def test_static(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): places.append(fluid.CUDAPlace(0)) for p in places: exe = fluid.Executor(p) diff --git a/python/paddle/fluid/tests/unittests/test_inverse_op.py b/python/paddle/fluid/tests/unittests/test_inverse_op.py index b155df0498d..f06d32170ba 100644 --- a/python/paddle/fluid/tests/unittests/test_inverse_op.py +++ b/python/paddle/fluid/tests/unittests/test_inverse_op.py @@ -21,7 +21,6 @@ from op_test import OpTest class TestInverseOp(OpTest): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -46,7 +45,6 @@ class TestInverseOp(OpTest): class TestInverseOpBatched(TestInverseOp): - def config(self): self.matrix_shape = [8, 4, 4] self.dtype = "float64" @@ -54,35 +52,30 @@ class TestInverseOpBatched(TestInverseOp): class TestInverseOpLarge(TestInverseOp): - def config(self): self.matrix_shape = [32, 32] self.dtype = "float64" self.python_api = paddle.tensor.math.inverse def test_grad(self): - self.check_grad(['Input'], - 'Output', - max_relative_error=1e-6, - check_eager=True) + self.check_grad( + ['Input'], 'Output', max_relative_error=1e-6, check_eager=True + ) class TestInverseOpFP32(TestInverseOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float32" self.python_api = paddle.tensor.math.inverse def test_grad(self): - self.check_grad(['Input'], - 'Output', - max_relative_error=1e-2, - check_eager=True) + self.check_grad( + ['Input'], 'Output', max_relative_error=1e-2, check_eager=True + ) class TestInverseOpBatchedFP32(TestInverseOpFP32): - def config(self): self.matrix_shape = [8, 4, 4] self.dtype = "float32" @@ -90,7 +83,6 @@ class TestInverseOpBatchedFP32(TestInverseOpFP32): class TestInverseOpLargeFP32(TestInverseOpFP32): - def config(self): self.matrix_shape = [32, 32] self.dtype = "float32" @@ -98,7 +90,6 @@ class TestInverseOpLargeFP32(TestInverseOpFP32): class TestInverseAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -113,12 +104,14 @@ class TestInverseAPI(unittest.TestCase): result_np = np.linalg.inv(input_np) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) - np.testing.assert_allclose(fetches[0], - np.linalg.inv(input_np), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.inv(input_np), rtol=1e-05 + ) def test_static(self): for place in self.places: @@ -130,13 +123,12 @@ class TestInverseAPI(unittest.TestCase): input_np = np.random.random([4, 4]).astype("float64") input = fluid.dygraph.to_variable(input_np) result = paddle.inverse(input) - np.testing.assert_allclose(result.numpy(), - np.linalg.inv(input_np), - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), np.linalg.inv(input_np), rtol=1e-05 + ) class TestInverseAPIError(unittest.TestCase): - def test_errors(self): input_np = np.random.random([4, 4]).astype("float64") @@ -159,7 +151,6 @@ class TestInverseAPIError(unittest.TestCase): class TestInverseSingularAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -174,9 +165,11 @@ class TestInverseSingularAPI(unittest.TestCase): exe = fluid.Executor(place) try: - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) except RuntimeError as ex: print("The mat is singular") except ValueError as ex: diff --git a/python/paddle/fluid/tests/unittests/test_io_save_load.py b/python/paddle/fluid/tests/unittests/test_io_save_load.py index f5feb8b78d6..80c2ae1f92f 100644 --- a/python/paddle/fluid/tests/unittests/test_io_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_io_save_load.py @@ -22,7 +22,6 @@ import os class TestSaveLoadAPIError(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.save_dir = os.path.join(self.temp_dir.name, "fake_dir") @@ -51,16 +50,18 @@ class TestSaveLoadAPIError(unittest.TestCase): exe = fluid.Executor(place) # case 1: main_program type error when vars None with self.assertRaises(TypeError): - fluid.io.load_vars(executor=exe, - dirname=self.save_dir, - main_program="program") + fluid.io.load_vars( + executor=exe, dirname=self.save_dir, main_program="program" + ) # case 2: main_program type error when vars not None with self.assertRaises(TypeError): - fluid.io.load_vars(executor=exe, - dirname=self.save_dir, - main_program="program", - vars="vars") + fluid.io.load_vars( + executor=exe, + dirname=self.save_dir, + main_program="program", + vars="vars", + ) def test_load_vars_error(self): with _test_eager_guard(): @@ -69,7 +70,6 @@ class TestSaveLoadAPIError(unittest.TestCase): class TestSaveInferenceModelAPIError(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -87,13 +87,15 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) with self.assertRaisesRegexp( - ValueError, "not involved in the target_vars calculation"): - fluid.io.save_inference_model(dirname=os.path.join( - self.temp_dir.name, 'model'), - feeded_var_names=['x', 'y'], - target_vars=[z], - executor=exe, - main_program=main_prog) + ValueError, "not involved in the target_vars calculation" + ): + fluid.io.save_inference_model( + dirname=os.path.join(self.temp_dir.name, 'model'), + feeded_var_names=['x', 'y'], + target_vars=[z], + executor=exe, + main_program=main_prog, + ) def test_useless_feeded_var_names(self): with _test_eager_guard(): @@ -102,7 +104,6 @@ class TestSaveInferenceModelAPIError(unittest.TestCase): class TestWhenTrainWithNoGrad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py index 4d7c66993c0..4324280efcb 100644 --- a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py +++ b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py @@ -18,7 +18,6 @@ from op_test import OpTest class TestIOUSimilarityOp(OpTest): - def test_check_output(self): self.check_output(check_dygraph=False) @@ -34,7 +33,9 @@ class TestIOUSimilarityOp(OpTest): self.attrs = {"box_normalized": self.box_normalized} self.outputs = {'Out': self.output} - def _compute_iou(self, ): + def _compute_iou( + self, + ): for row in range(self.boxes1.shape[0]): for col in range(self.boxes2.shape[0]): xmin1, ymin1, xmax1, ymax1 = self.boxes1[row] @@ -64,7 +65,6 @@ class TestIOUSimilarityOp(OpTest): class TestIOUSimilarityOpWithLoD(TestIOUSimilarityOp): - def test_check_output(self): self.check_output(check_dygraph=False) @@ -81,7 +81,6 @@ class TestIOUSimilarityOpWithLoD(TestIOUSimilarityOp): class TestIOUSimilarityOpWithBoxNormalized(TestIOUSimilarityOp): - def test_check_output(self): self.check_output(check_dygraph=False) diff --git a/python/paddle/fluid/tests/unittests/test_ir_graph.py b/python/paddle/fluid/tests/unittests/test_ir_graph.py index 1951db0395f..a8effa96920 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_graph.py +++ b/python/paddle/fluid/tests/unittests/test_ir_graph.py @@ -24,8 +24,9 @@ class TestIRGraph(unittest.TestCase): def test_nodes(self): graph = build_graph() self.assertTrue( - {node.name() - for node in graph.nodes()} == {"x1", "x2", "out", "sum"}) + {node.name() for node in graph.nodes()} + == {"x1", "x2", "out", "sum"} + ) def test_has_set_get(self): graph = build_graph() @@ -85,8 +86,9 @@ class TestIRGraph(unittest.TestCase): graph = build_graph() nodes = graph.release_nodes() self.assertTrue(len(graph.nodes()) == 0) - self.assertTrue({node.name() - for node in nodes} == {"x1", "x2", "out", "sum"}) + self.assertTrue( + {node.name() for node in nodes} == {"x1", "x2", "out", "sum"} + ) def test_remove_node(self): graph = build_graph() @@ -94,8 +96,9 @@ class TestIRGraph(unittest.TestCase): for node in nodes: if node.name() == "sum": break - self.assertTrue({node.name() - for node in nodes} == {"x1", "x2", "out", "sum"}) + self.assertTrue( + {node.name() for node in nodes} == {"x1", "x2", "out", "sum"} + ) nodes.remove(node) self.assertTrue({node.name() for node in nodes} == {"x1", "x2", "out"}) diff --git a/python/paddle/fluid/tests/unittests/test_ir_inplace_pass.py b/python/paddle/fluid/tests/unittests/test_ir_inplace_pass.py index 3fde50653b6..62ea08307ce 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_inplace_pass.py +++ b/python/paddle/fluid/tests/unittests/test_ir_inplace_pass.py @@ -31,8 +31,10 @@ def fc_with_batchnorm(use_feed): hidden, size=200, act='tanh', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) hidden = fluid.layers.batch_norm(input=hidden) prediction = fluid.layers.fc(hidden, size=10, act='softmax') @@ -42,7 +44,6 @@ def fc_with_batchnorm(use_feed): class TestIrInplace(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -56,13 +57,11 @@ class TestIrInplace(TestParallelExecutorBase): label = np.ones(shape=[32, 1], dtype='int64') self.check_network_convergence( fc_with_batchnorm, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=DeviceType.CUDA, use_ir_memory_optimize=ir_memory_optimize, - enable_inplace=enable_inplace) + enable_inplace=enable_inplace, + ) def test_fc_with_batchnorm(self, delta=1e-3): loss00 = self._fc_with_batchnorm(False, False) diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_ifelse_op.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_ifelse_op.py index 9cc78982aab..657c267767d 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_ifelse_op.py +++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_ifelse_op.py @@ -29,11 +29,9 @@ from paddle.fluid.optimizer import MomentumOptimizer class TestIrMemoryOptimizeIfElseOp(unittest.TestCase): - - def check_network_convergence(self, - use_cuda=True, - use_mem_opt=False, - iter_num=5): + def check_network_convergence( + self, use_cuda=True, use_mem_opt=False, iter_num=5 + ): paddle.seed(100) paddle.framework.random._manual_program_seed(100) prog = Program() @@ -65,14 +63,17 @@ class TestIrMemoryOptimizeIfElseOp(unittest.TestCase): optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=200) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=200 + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) exec_strategy = fluid.ExecutionStrategy() - exec_strategy._use_device = core.DeviceType.CUDA if use_cuda else core.DeviceType.CPU + exec_strategy._use_device = ( + core.DeviceType.CUDA if use_cuda else core.DeviceType.CPU + ) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = use_mem_opt @@ -81,7 +82,8 @@ class TestIrMemoryOptimizeIfElseOp(unittest.TestCase): train_cp = train_cp.with_data_parallel( loss_name=avg_loss.name, exec_strategy=exec_strategy, - build_strategy=build_strategy) + build_strategy=build_strategy, + ) fetch_list = [avg_loss.name] exe.run(startup_prog) @@ -94,12 +96,11 @@ class TestIrMemoryOptimizeIfElseOp(unittest.TestCase): y_data = np.array([x[1] for x in data]).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) - outs = exe.run(train_cp, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + outs = exe.run( + train_cp, + feed={'x': x_data, 'y': y_data}, + fetch_list=[avg_loss], + ) loop += 1 ret.append(outs[0]) diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_nlp.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_nlp.py index ed4ab08a506..f661db6d15e 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_nlp.py +++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_nlp.py @@ -20,23 +20,26 @@ import unittest from ir_memory_optimize_net_base import TestIrMemOptBase -def lstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): +def lstm_net( + data, + label, + dict_dim, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, + emb_lr=30.0, +): emb = fluid.layers.embedding( input=data, size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) + param_attr=fluid.ParamAttr(learning_rate=emb_lr), + ) fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - lstm_h, c = fluid.layers.dynamic_lstm(input=fc0, - size=hid_dim * 4, - is_reverse=False) + lstm_h, c = fluid.layers.dynamic_lstm( + input=fc0, size=hid_dim * 4, is_reverse=False + ) lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') lstm_max_tanh = fluid.layers.tanh(lstm_max) fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') @@ -47,7 +50,6 @@ def lstm_net(data, class TestIrMemOptRNN(TestIrMemOptBase): - def setUp(self): self.network = lstm_net diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py index d26a0a67894..d21156b43e8 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py +++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py @@ -52,7 +52,6 @@ def fc_with_inplace_net(use_feed): class TestMNIST(TestParallelExecutorBase): - def _dummy_data(self): np.random.seed(5) img = np.random.random(size=[32, 784]).astype(np.float32) @@ -66,20 +65,16 @@ class TestMNIST(TestParallelExecutorBase): img, label = self._dummy_data() first_loss0, last_loss0, _ = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_ir_memory_optimize=False) + use_ir_memory_optimize=False, + ) first_loss1, last_loss1, _ = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_ir_memory_optimize=True) + use_ir_memory_optimize=True, + ) for loss in zip(first_loss0, first_loss1): self.assertAlmostEqual(loss[0], loss[1], delta=1e-6) for loss in zip(last_loss0, last_loss1): diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_transformer.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_transformer.py index 8b7fe96805f..8d1191fddd5 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_transformer.py @@ -25,7 +25,6 @@ from test_parallel_executor_transformer import get_feed_data_reader, transformer # NOTE(dzhwinter): test diferent strategy colisions. # open the eager delete tensor strategy by default. class TestTransformerWithIR(TestParallelExecutorBase): - def test_main(self): if core.is_compiled_with_cuda(): # check python transpiler @@ -34,14 +33,16 @@ class TestTransformerWithIR(TestParallelExecutorBase): use_device=DeviceType.CUDA, feed_data_reader=get_feed_data_reader(), use_ir_memory_optimize=False, - iter=2) + iter=2, + ) # check IR memory optimize self.check_network_convergence( transformer, use_device=DeviceType.CUDA, feed_data_reader=get_feed_data_reader(), use_ir_memory_optimize=True, - iter=2) + iter=2, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_is_complex.py b/python/paddle/fluid/tests/unittests/test_is_complex.py index a441bd86296..fd2e58eff63 100644 --- a/python/paddle/fluid/tests/unittests/test_is_complex.py +++ b/python/paddle/fluid/tests/unittests/test_is_complex.py @@ -18,7 +18,6 @@ import unittest class TestIsComplex(unittest.TestCase): - def test_for_integer(self): x = paddle.arange(10) self.assertFalse(paddle.is_complex(x)) diff --git a/python/paddle/fluid/tests/unittests/test_is_empty_op.py b/python/paddle/fluid/tests/unittests/test_is_empty_op.py index edbc60c68e5..d6235f7abc4 100644 --- a/python/paddle/fluid/tests/unittests/test_is_empty_op.py +++ b/python/paddle/fluid/tests/unittests/test_is_empty_op.py @@ -19,7 +19,6 @@ import paddle class TestEmpty(OpTest): - def setUp(self): self.op_type = "is_empty" self.inputs = {'X': np.array([1, 2, 3])} @@ -30,7 +29,6 @@ class TestEmpty(OpTest): class TestNotEmpty(TestEmpty): - def setUp(self): self.op_type = "is_empty" self.inputs = {'X': np.array([])} @@ -38,11 +36,11 @@ class TestNotEmpty(TestEmpty): class TestIsEmptyOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_data = np.random.random((3, 2)).astype("float64") def test_Variable(): @@ -53,25 +51,24 @@ class TestIsEmptyOpError(unittest.TestCase): def test_type(): # dtype must be float32, float64, int32, int64 - x3 = paddle.static.data(name="x3", - shape=[4, 32, 32], - dtype="bool") + x3 = paddle.static.data( + name="x3", shape=[4, 32, 32], dtype="bool" + ) res = paddle.is_empty(x=x3) self.assertRaises(TypeError, test_type) def test_name_type(): # name type must be string. - x4 = paddle.static.data(name="x4", - shape=[3, 2], - dtype="float32") + x4 = paddle.static.data( + name="x4", shape=[3, 2], dtype="float32" + ) res = paddle.is_empty(x=x4, name=1) self.assertRaises(TypeError, test_name_type) class TestIsEmptyOpDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() input = paddle.rand(shape=[4, 32, 32], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_is_integer.py b/python/paddle/fluid/tests/unittests/test_is_integer.py index a933e9fac66..9e0eae91292 100644 --- a/python/paddle/fluid/tests/unittests/test_is_integer.py +++ b/python/paddle/fluid/tests/unittests/test_is_integer.py @@ -18,7 +18,6 @@ import unittest class TestIsInteger(unittest.TestCase): - def test_for_integer(self): x = paddle.arange(10) self.assertTrue(paddle.is_integer(x)) diff --git a/python/paddle/fluid/tests/unittests/test_is_tensor.py b/python/paddle/fluid/tests/unittests/test_is_tensor.py index deef1225cd0..b91b5aaf3e7 100644 --- a/python/paddle/fluid/tests/unittests/test_is_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_is_tensor.py @@ -19,24 +19,20 @@ DELTA = 0.00001 class TestIsTensorApi(unittest.TestCase): - def test_is_tensor_real(self, dtype="float32"): - """Test is_tensor api with a real tensor - """ + """Test is_tensor api with a real tensor""" paddle.disable_static() x = paddle.rand([3, 2, 4], dtype=dtype) self.assertTrue(paddle.is_tensor(x)) def test_is_tensor_list(self, dtype="float32"): - """Test is_tensor api with a list - """ + """Test is_tensor api with a list""" paddle.disable_static() x = [1, 2, 3] self.assertFalse(paddle.is_tensor(x)) def test_is_tensor_number(self, dtype="float32"): - """Test is_tensor api with a number - """ + """Test is_tensor api with a number""" paddle.disable_static() x = 5 self.assertFalse(paddle.is_tensor(x)) diff --git a/python/paddle/fluid/tests/unittests/test_isclose_op.py b/python/paddle/fluid/tests/unittests/test_isclose_op.py index 04b7fbe54e7..5d53c337ce1 100644 --- a/python/paddle/fluid/tests/unittests/test_isclose_op.py +++ b/python/paddle/fluid/tests/unittests/test_isclose_op.py @@ -19,9 +19,8 @@ import paddle class TestIscloseOp(OpTest): - def set_args(self): - self.input = np.array([10000., 1e-07]).astype("float32") + self.input = np.array([10000.0, 1e-07]).astype("float32") self.other = np.array([10000.1, 1e-08]).astype("float32") self.rtol = np.array([1e-05]).astype("float64") self.atol = np.array([1e-08]).astype("float64") @@ -36,18 +35,21 @@ class TestIscloseOp(OpTest): 'Input': self.input, 'Other': self.other, "Rtol": self.rtol, - "Atol": self.atol + "Atol": self.atol, } self.attrs = {'equal_nan': self.equal_nan} self.outputs = { - 'Out': - np.array([ - np.isclose(self.inputs['Input'], - self.inputs['Other'], - rtol=self.rtol, - atol=self.atol, - equal_nan=self.equal_nan) - ]) + 'Out': np.array( + [ + np.isclose( + self.inputs['Input'], + self.inputs['Other'], + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, + ) + ] + ) } def test_check_output(self): @@ -55,9 +57,7 @@ class TestIscloseOp(OpTest): class TestIscloseOpException(TestIscloseOp): - def test_check_output(self): - def test_rtol_num(): self.inputs['Rtol'] = np.array([1e-05, 1e-05]).astype("float64") self.inputs['Atol'] = np.array([1e-08]).astype("float64") @@ -88,9 +88,8 @@ class TestIscloseOpException(TestIscloseOp): class TestIscloseOpSmallNum(TestIscloseOp): - def set_args(self): - self.input = np.array([10000., 1e-08]).astype("float32") + self.input = np.array([10000.0, 1e-08]).astype("float32") self.other = np.array([10000.1, 1e-09]).astype("float32") self.rtol = np.array([1e-05]).astype("float64") self.atol = np.array([1e-08]).astype("float64") @@ -98,7 +97,6 @@ class TestIscloseOpSmallNum(TestIscloseOp): class TestIscloseOpNanFalse(TestIscloseOp): - def set_args(self): self.input = np.array([1.0, float('nan')]).astype("float32") self.other = np.array([1.0, float('nan')]).astype("float32") @@ -108,7 +106,6 @@ class TestIscloseOpNanFalse(TestIscloseOp): class TestIscloseOpNanTrue(TestIscloseOp): - def set_args(self): self.input = np.array([1.0, float('nan')]).astype("float32") self.other = np.array([1.0, float('nan')]).astype("float32") @@ -118,7 +115,6 @@ class TestIscloseOpNanTrue(TestIscloseOp): class TestIscloseStatic(unittest.TestCase): - def test_api_case(self): paddle.enable_static() x_data = np.random.rand(10, 10) @@ -127,24 +123,23 @@ class TestIscloseStatic(unittest.TestCase): if paddle.fluid.core.is_compiled_with_cuda(): places.append(paddle.fluid.CUDAPlace(0)) for place in places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='float64') result = paddle.isclose(x, y) exe = paddle.fluid.Executor(place) - fetches = exe.run(paddle.fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data - }, - fetch_list=[result]) + fetches = exe.run( + paddle.fluid.default_main_program(), + feed={"x": x_data, "y": y_data}, + fetch_list=[result], + ) expected_out = np.isclose(x_data, y_data) self.assertTrue((fetches[0] == expected_out).all(), True) class TestIscloseDygraph(unittest.TestCase): - def test_api_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -162,13 +157,13 @@ class TestIscloseDygraph(unittest.TestCase): class TestIscloseError(unittest.TestCase): - def test_input_dtype(self): paddle.enable_static() def test_x_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float16') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='float64') result = paddle.isclose(x, y) @@ -176,8 +171,9 @@ class TestIscloseError(unittest.TestCase): self.assertRaises(TypeError, test_x_dtype) def test_y_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='int32') result = paddle.isclose(x, y) @@ -206,7 +202,6 @@ class TestIscloseError(unittest.TestCase): class TestIscloseOpFloat32(TestIscloseOp): - def set_args(self): self.input = np.array([10.1]).astype("float32") self.other = np.array([10]).astype("float32") @@ -216,7 +211,6 @@ class TestIscloseOpFloat32(TestIscloseOp): class TestIscloseOpFloat64(TestIscloseOp): - def set_args(self): self.input = np.array([10.1]).astype("float64") self.other = np.array([10]).astype("float64") @@ -229,7 +223,6 @@ class TestIscloseOpFloat64(TestIscloseOp): class TestIscloseOpLargeDimInput(TestIscloseOp): - def set_args(self): self.input = np.array(np.zeros([2048, 1024])).astype("float64") self.other = np.array(np.zeros([2048, 1024])).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/test_isfinite_op.py b/python/paddle/fluid/tests/unittests/test_isfinite_op.py index f2fa252ff0a..e2fa9f67b53 100644 --- a/python/paddle/fluid/tests/unittests/test_isfinite_op.py +++ b/python/paddle/fluid/tests/unittests/test_isfinite_op.py @@ -21,7 +21,6 @@ from op_test import OpTest class TestInf(OpTest): - def setUp(self): self.op_type = "isinf" self.dtype = np.float32 @@ -42,9 +41,7 @@ class TestInf(OpTest): class TestRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.isfinite([10]) @@ -57,16 +54,15 @@ class TestRaiseError(unittest.TestCase): self.assertRaises(TypeError, test_dtype) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16Inf(TestInf): - def init_dtype(self): self.dtype = np.float16 class TestNAN(OpTest): - def setUp(self): self.op_type = "isnan" self.dtype = np.float32 @@ -86,16 +82,15 @@ class TestNAN(OpTest): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16NAN(TestNAN): - def init_dtype(self): self.dtype = np.float16 class TestIsfinite(OpTest): - def setUp(self): self.op_type = "isfinite" self.dtype = np.float32 @@ -116,16 +111,15 @@ class TestIsfinite(OpTest): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16Isfinite(TestIsfinite): - def init_dtype(self): self.dtype = np.float16 class BadInputTest(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/test_isfinite_v2_op.py b/python/paddle/fluid/tests/unittests/test_isfinite_v2_op.py index 252e43b3423..e3d094890ac 100644 --- a/python/paddle/fluid/tests/unittests/test_isfinite_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_isfinite_v2_op.py @@ -31,9 +31,9 @@ def run_static(x_np, dtype, op_str, use_gpu=False): x = paddle.fluid.data(name='x', shape=x_np.shape, dtype=dtype) res = getattr(paddle.tensor, op_str)(x) exe.run(startup_program) - static_result = exe.run(main_program, - feed={'x': x_np}, - fetch_list=[res]) + static_result = exe.run( + main_program, feed={'x': x_np}, fetch_list=[res] + ) return static_result @@ -59,15 +59,16 @@ def run_eager(x_np, op_str, use_gpu=True): return dygraph_result -def np_data_generator(low, high, np_shape, type, sv_list, op_str, *args, - **kwargs): +def np_data_generator( + low, high, np_shape, type, sv_list, op_str, *args, **kwargs +): x_np = np.random.uniform(low, high, np_shape).astype(getattr(np, type)) # x_np.shape[0] >= len(sv_list) if type in ['float16', 'float32', 'float64']: for i, v in enumerate(sv_list): x_np[i] = v ori_shape = x_np.shape - x_np = x_np.reshape((np.product(ori_shape), )) + x_np = x_np.reshape((np.product(ori_shape),)) np.random.shuffle(x_np) x_np = x_np.reshape(ori_shape) result_np = getattr(np, op_str)(x_np) @@ -80,35 +81,35 @@ TEST_META_DATA = [ 'high': 1, 'np_shape': [8, 17, 5, 6, 7], 'type': 'float16', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, { 'low': 0.1, 'high': 1, 'np_shape': [11, 17], 'type': 'float32', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, { 'low': 0.1, 'high': 1, 'np_shape': [2, 3, 4, 5], 'type': 'float64', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, { 'low': 0, 'high': 100, 'np_shape': [11, 17, 10], 'type': 'int32', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, { 'low': 0, 'high': 999, 'np_shape': [132], 'type': 'int64', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, ] @@ -127,7 +128,6 @@ def test(test_case, op_str, use_gpu=False): class TestCPUNormal(unittest.TestCase): - def test_inf(self): test(self, 'isinf') @@ -139,7 +139,6 @@ class TestCPUNormal(unittest.TestCase): class TestCUDANormal(unittest.TestCase): - def test_inf(self): test(self, 'isinf', True) @@ -151,7 +150,6 @@ class TestCUDANormal(unittest.TestCase): class TestError(unittest.TestCase): - def test_bad_input(self): paddle.enable_static() with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/test_jit_layer.py b/python/paddle/fluid/tests/unittests/test_jit_layer.py index 5a03e0ac3b8..1713e61f9de 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_layer.py +++ b/python/paddle/fluid/tests/unittests/test_jit_layer.py @@ -20,13 +20,14 @@ import numpy as np from paddle.static import InputSpec from paddle.fluid.framework import _dygraph_place_guard from paddle.jit.layer import Layer -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) paddle.seed(1) class Net(paddle.nn.Layer): - def __init__(self): super(Net, self).__init__() self.fc1 = paddle.nn.Linear(4, 4) @@ -50,7 +51,6 @@ class Net(paddle.nn.Layer): class TestMultiLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -81,20 +81,19 @@ class TestMultiLoad(unittest.TestCase): class SaveLinear(paddle.nn.Layer): - def __init__(self): super().__init__() self.linear = paddle.nn.Linear(80, 80) @paddle.jit.to_static( - input_spec=[InputSpec(shape=[None, 80], dtype='float32')]) + input_spec=[InputSpec(shape=[None, 80], dtype='float32')] + ) def forward(self, x): out = self.linear(x) return out class TestMKLOutput(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/python/paddle/fluid/tests/unittests/test_jit_pre_save_hooks.py b/python/paddle/fluid/tests/unittests/test_jit_pre_save_hooks.py index 1d3a35b6173..9b9ce8297ba 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_pre_save_hooks.py +++ b/python/paddle/fluid/tests/unittests/test_jit_pre_save_hooks.py @@ -16,15 +16,17 @@ import unittest import paddle -from paddle.fluid.dygraph.jit import _run_save_pre_hooks, _clear_save_pre_hooks, _register_save_pre_hook +from paddle.fluid.dygraph.jit import ( + _run_save_pre_hooks, + _clear_save_pre_hooks, + _register_save_pre_hook, +) _counter = 0 class TestPreSaveHooks(unittest.TestCase): - def test_pre_save_hook_functions(self): - def fake_func(*args, **kwgs): global _counter _counter += 1 @@ -32,13 +34,15 @@ class TestPreSaveHooks(unittest.TestCase): remove_handler = _register_save_pre_hook(fake_func) self.assertEqual(len(paddle.fluid.dygraph.jit._save_pre_hooks), 1) self.assertTrue( - paddle.fluid.dygraph.jit._save_pre_hooks[0] is fake_func) + paddle.fluid.dygraph.jit._save_pre_hooks[0] is fake_func + ) # Test of avoiding redundancy hanging remove_handler = _register_save_pre_hook(fake_func) self.assertEqual(len(paddle.fluid.dygraph.jit._save_pre_hooks), 1) self.assertTrue( - paddle.fluid.dygraph.jit._save_pre_hooks[0] is fake_func) + paddle.fluid.dygraph.jit._save_pre_hooks[0] is fake_func + ) remove_handler.remove() self.assertEqual(len(paddle.fluid.dygraph.jit._save_pre_hooks), 0) diff --git a/python/paddle/fluid/tests/unittests/test_jit_save_load.py b/python/paddle/fluid/tests/unittests/test_jit_save_load.py index efb5a196f81..2f6405e7aa6 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_jit_save_load.py @@ -34,7 +34,6 @@ SEED = 10 def random_batch_reader(input_size, label_size): - def _get_random_inputs_and_labels(input_size, label_size): np.random.seed(SEED) input = np.random.random(size=input_size).astype('float32') @@ -44,14 +43,14 @@ def random_batch_reader(input_size, label_size): def __reader__(): for _ in range(BATCH_NUM): batch_input, batch_label = _get_random_inputs_and_labels( - [BATCH_SIZE, input_size], [BATCH_SIZE, label_size]) + [BATCH_SIZE, input_size], [BATCH_SIZE, label_size] + ) yield batch_input, batch_label return __reader__ class LinearNet(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNet, self).__init__() self._linear = Linear(in_size, out_size) @@ -62,7 +61,6 @@ class LinearNet(fluid.dygraph.Layer): class LinearNetWithInputSpec(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithInputSpec, self).__init__() self._linear = Linear(in_size, out_size) @@ -73,7 +71,6 @@ class LinearNetWithInputSpec(fluid.dygraph.Layer): class LinearNetNotDeclarative(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetNotDeclarative, self).__init__() self._linear = Linear(in_size, out_size) @@ -83,15 +80,16 @@ class LinearNetNotDeclarative(fluid.dygraph.Layer): class LinerNetWithLabel(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinerNetWithLabel, self).__init__() self._linear = Linear(in_size, out_size) - @declarative(input_spec=[ - InputSpec(shape=[None, 784], dtype='float32', name="image"), - InputSpec(shape=[None, 1], dtype='int64', name="label") - ]) + @declarative( + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image"), + InputSpec(shape=[None, 1], dtype='int64', name="label"), + ] + ) def forward(self, x, label): out = self._linear(x) loss = fluid.layers.cross_entropy(out, label) @@ -100,15 +98,16 @@ class LinerNetWithLabel(paddle.nn.Layer): class LinerNetWithPruneInput(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinerNetWithPruneInput, self).__init__() self._linear = Linear(in_size, out_size) - @declarative(input_spec=[ - InputSpec(shape=[None, 784], dtype='float32', name="image"), - InputSpec(shape=[None, 1], dtype='int64', name="label") - ]) + @declarative( + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image"), + InputSpec(shape=[None, 1], dtype='int64', name="label"), + ] + ) def forward(self, x, label): out = self._linear(x) loss = fluid.layers.cross_entropy(out, label) @@ -117,22 +116,22 @@ class LinerNetWithPruneInput(paddle.nn.Layer): class LinerNetWithUselessInput(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinerNetWithUselessInput, self).__init__() self._linear = Linear(in_size, out_size) - @declarative(input_spec=[ - InputSpec(shape=[None, 784], dtype='float32', name="image"), - InputSpec(shape=[None, 1], dtype='int64', name="label") - ]) + @declarative( + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image"), + InputSpec(shape=[None, 1], dtype='int64', name="label"), + ] + ) def forward(self, x, label): out = self._linear(x) return out class LinearNetReturnLoss(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetReturnLoss, self).__init__() self._linear = Linear(in_size, out_size) @@ -146,16 +145,17 @@ class LinearNetReturnLoss(fluid.dygraph.Layer): class LinearNetMultiInput(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetMultiInput, self).__init__() self._linear1 = Linear(in_size, out_size) self._linear2 = Linear(in_size, out_size) - @declarative(input_spec=[ - InputSpec([None, 8], dtype='float32'), - InputSpec([None, 8], dtype='float32') - ]) + @declarative( + input_spec=[ + InputSpec([None, 8], dtype='float32'), + InputSpec([None, 8], dtype='float32'), + ] + ) def forward(self, x, y): x_out = self._linear1(x) y_out = self._linear2(y) @@ -164,14 +164,17 @@ class LinearNetMultiInput(fluid.dygraph.Layer): class LinearNetMultiInput1(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetMultiInput1, self).__init__() self._linear1 = Linear(in_size, out_size) self._linear2 = Linear(in_size, out_size) - @declarative(input_spec=(InputSpec([None, 8], dtype='float32'), - InputSpec([None, 8], dtype='float32'))) + @declarative( + input_spec=( + InputSpec([None, 8], dtype='float32'), + InputSpec([None, 8], dtype='float32'), + ) + ) def forward(self, x, y): x_out = self._linear1(x) y_out = self._linear2(y) @@ -180,7 +183,6 @@ class LinearNetMultiInput1(fluid.dygraph.Layer): class MultiLoadingLinearNet(fluid.dygraph.Layer): - def __init__(self, size, model_path): super(MultiLoadingLinearNet, self).__init__() self._linear = Linear(size, size) @@ -197,7 +199,6 @@ class MultiLoadingLinearNet(fluid.dygraph.Layer): class LinearNetReturnHidden(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetReturnHidden, self).__init__() self._linear_1 = Linear(in_size, out_size) @@ -212,7 +213,6 @@ class LinearNetReturnHidden(fluid.dygraph.Layer): class LinearNetWithNestOut(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithNestOut, self).__init__() self._linear_1 = Linear(in_size, out_size) @@ -228,18 +228,16 @@ class LinearNetWithNestOut(fluid.dygraph.Layer): class LinearNetWithDictInput(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithDictInput, self).__init__() self._linear = Linear(in_size, out_size) - @paddle.jit.to_static(input_spec=[{ - 'img': - InputSpec(shape=[None, 8], dtype='float32', name='img') - }, { - 'label': - InputSpec(shape=[None, 1], dtype='int64', name='label') - }]) + @paddle.jit.to_static( + input_spec=[ + {'img': InputSpec(shape=[None, 8], dtype='float32', name='img')}, + {'label': InputSpec(shape=[None, 1], dtype='int64', name='label')}, + ] + ) def forward(self, img, label): out = self._linear(img['img']) # not return loss to avoid prune output @@ -248,7 +246,6 @@ class LinearNetWithDictInput(paddle.nn.Layer): class LinearNetWithDictInputNoPrune(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithDictInputNoPrune, self).__init__() self._linear = Linear(in_size, out_size) @@ -259,7 +256,6 @@ class LinearNetWithDictInputNoPrune(paddle.nn.Layer): class EmptyLayer(paddle.nn.Layer): - def __init__(self): super(EmptyLayer, self).__init__() @@ -269,7 +265,6 @@ class EmptyLayer(paddle.nn.Layer): class NoParamLayer(paddle.nn.Layer): - def __init__(self): super(NoParamLayer, self).__init__() @@ -279,7 +274,6 @@ class NoParamLayer(paddle.nn.Layer): class LinearNetWithMultiStaticFunc(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithMultiStaticFunc, self).__init__() self._linear_0 = Linear(in_size, out_size) @@ -301,12 +295,14 @@ class LinearNetWithMultiStaticFunc(fluid.dygraph.Layer): def train(layer, input_size=784, label_size=1): # create optimizer - sgd = fluid.optimizer.SGDOptimizer(learning_rate=0.01, - parameter_list=layer.parameters()) + sgd = fluid.optimizer.SGDOptimizer( + learning_rate=0.01, parameter_list=layer.parameters() + ) # create data loader train_loader = fluid.io.DataLoader.from_generator(capacity=5) - train_loader.set_batch_generator(random_batch_reader( - input_size, label_size)) + train_loader.set_batch_generator( + random_batch_reader(input_size, label_size) + ) # train for data in train_loader(): img, label = data @@ -325,12 +321,14 @@ def train(layer, input_size=784, label_size=1): def train_with_label(layer, input_size=784, label_size=1): # create optimizer - sgd = fluid.optimizer.SGDOptimizer(learning_rate=0.01, - parameter_list=layer.parameters()) + sgd = fluid.optimizer.SGDOptimizer( + learning_rate=0.01, parameter_list=layer.parameters() + ) # create data loader train_loader = fluid.io.DataLoader.from_generator(capacity=5) - train_loader.set_batch_generator(random_batch_reader( - input_size, label_size)) + train_loader.set_batch_generator( + random_batch_reader(input_size, label_size) + ) # train for data in train_loader(): img, label = data @@ -345,11 +343,11 @@ def train_with_label(layer, input_size=784, label_size=1): class TestJitSaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "test_jit_save_load/model") + self.model_path = os.path.join( + self.temp_dir.name, "test_jit_save_load/model" + ) # enable dygraph mode fluid.enable_dygraph() # config seed @@ -364,9 +362,9 @@ class TestJitSaveLoad(unittest.TestCase): example_inputs, layer, _ = train(layer) final_model_path = model_path if model_path else self.model_path orig_input_types = [type(x) for x in example_inputs] - paddle.jit.save(layer=layer, - path=final_model_path, - input_spec=example_inputs) + paddle.jit.save( + layer=layer, path=final_model_path, input_spec=example_inputs + ) new_input_types = [type(x) for x in example_inputs] self.assertEqual(orig_input_types, new_input_types) return layer @@ -385,10 +383,11 @@ class TestJitSaveLoad(unittest.TestCase): infer_layer.eval() # inference & compare x = fluid.dygraph.to_variable( - np.random.random((1, 784)).astype('float32')) + np.random.random((1, 784)).astype('float32') + ) np.testing.assert_array_equal( - train_layer(x).numpy(), - infer_layer(x).numpy()) + train_layer(x).numpy(), infer_layer(x).numpy() + ) def load_and_finetune(self, train_layer, load_train_layer): train_layer.train() @@ -396,8 +395,9 @@ class TestJitSaveLoad(unittest.TestCase): # train & compare img0, _, train_loss = train(train_layer) img1, _, load_train_loss = train(load_train_layer) - np.testing.assert_array_equal(train_loss.numpy(), - load_train_loss.numpy()) + np.testing.assert_array_equal( + train_loss.numpy(), load_train_loss.numpy() + ) def load_dygraph_state_dict(self, train_layer): train_layer.eval() @@ -411,26 +411,28 @@ class TestJitSaveLoad(unittest.TestCase): new_layer.eval() # inference & compare x = fluid.dygraph.to_variable( - np.random.random((1, 784)).astype('float32')) + np.random.random((1, 784)).astype('float32') + ) np.testing.assert_array_equal( - train_layer(x).numpy(), - new_layer(x).numpy()) + train_layer(x).numpy(), new_layer(x).numpy() + ) def test_load_dygraph_no_path(self): - model_path = os.path.join(self.temp_dir.name, - "test_jit_save_load.no_path/model_path") + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_load.no_path/model_path" + ) with self.assertRaises(ValueError): model_dict, _ = fluid.dygraph.load_dygraph(model_path) def test_jit_load_no_path(self): - path = os.path.join(self.temp_dir.name, - "test_jit_save_load.no_path/model_path") + path = os.path.join( + self.temp_dir.name, "test_jit_save_load.no_path/model_path" + ) with self.assertRaises(ValueError): loaded_layer = paddle.jit.load(path) class TestSaveLoadWithNestOut(unittest.TestCase): - def setUp(self): # enable dygraph mode fluid.enable_dygraph() @@ -441,7 +443,8 @@ class TestSaveLoadWithNestOut(unittest.TestCase): def test_nest_output(self): x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) net = LinearNetWithNestOut(8, 8) dy_outs = flatten(net(x)) @@ -455,13 +458,12 @@ class TestSaveLoadWithNestOut(unittest.TestCase): self.assertTrue(len(dy_outs) == 4) for dy_out, load_out in zip(dy_outs, load_outs): - np.testing.assert_allclose(dy_out.numpy(), - load_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + dy_out.numpy(), load_out.numpy(), rtol=1e-05 + ) class TestSaveLoadWithDictInput(unittest.TestCase): - def test_dict_input(self): # NOTE: This net cannot be executed, it is just # a special case for exporting models in model validation @@ -473,17 +475,17 @@ class TestSaveLoadWithDictInput(unittest.TestCase): # {'label': var label : fluid.VarType.LOD_TENSOR.shape(-1, 1).astype(VarType.INT64)}) self.assertEqual(len(net.forward.concrete_program.inputs), 3) temp_dir = tempfile.TemporaryDirectory() - path = os.path.join(temp_dir.name, - "test_jit_save_load_with_dict_input/model") + path = os.path.join( + temp_dir.name, "test_jit_save_load_with_dict_input/model" + ) # prune inputs - paddle.jit.save(layer=net, - path=path, - input_spec=[{ - 'img': - InputSpec(shape=[None, 8], - dtype='float32', - name='img') - }]) + paddle.jit.save( + layer=net, + path=path, + input_spec=[ + {'img': InputSpec(shape=[None, 8], dtype='float32', name='img')} + ], + ) img = paddle.randn(shape=[4, 8], dtype='float32') loaded_net = paddle.jit.load(path) @@ -496,25 +498,27 @@ class TestSaveLoadWithDictInput(unittest.TestCase): class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): - def test_dict_input(self): net = LinearNetWithDictInputNoPrune(8, 8) temp_dir = tempfile.TemporaryDirectory() path = os.path.join( - temp_dir.name, "test_jit_save_load_with_dict_input_no_prune/model") + temp_dir.name, "test_jit_save_load_with_dict_input_no_prune/model" + ) # prune inputs - paddle.jit.save(layer=net, - path=path, - input_spec=[{ - 'img': - InputSpec(shape=[None, 8], - dtype='float32', - name='img'), - 'img2': - InputSpec(shape=[None, 8], - dtype='float32', - name='img2') - }]) + paddle.jit.save( + layer=net, + path=path, + input_spec=[ + { + 'img': InputSpec( + shape=[None, 8], dtype='float32', name='img' + ), + 'img2': InputSpec( + shape=[None, 8], dtype='float32', name='img2' + ), + } + ], + ) img = paddle.randn(shape=[4, 8], dtype='float32') img2 = paddle.randn(shape=[4, 8], dtype='float32') @@ -526,7 +530,6 @@ class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): class TestSaveLoadWithInputSpec(unittest.TestCase): - def setUp(self): # enable dygraph mode fluid.enable_dygraph() @@ -538,11 +541,13 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): def test_with_input_spec(self): net = LinearNetReturnLoss(8, 8) # set x.shape = [None, 8] - net.forward = declarative(net.forward, - input_spec=[InputSpec([None, 8], name='x')]) + net.forward = declarative( + net.forward, input_spec=[InputSpec([None, 8], name='x')] + ) - model_path = os.path.join(self.temp_dir.name, - "input_spec.output_spec/model") + model_path = os.path.join( + self.temp_dir.name, "input_spec.output_spec/model" + ) # check inputs and outputs self.assertTrue(len(net.forward.inputs) == 1) input_x = net.forward.inputs[0] @@ -556,14 +561,16 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): # 2. load to infer infer_layer = paddle.jit.load(model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) pred = infer_layer(x) def test_multi_in_out(self): net = LinearNetMultiInput(8, 8) - model_path = os.path.join(self.temp_dir.name, - "multi_inout.output_spec1/model") + model_path = os.path.join( + self.temp_dir.name, "multi_inout.output_spec1/model" + ) # 1. check inputs and outputs self.assertTrue(len(net.forward.inputs) == 2) input_x = net.forward.inputs[0] @@ -578,15 +585,18 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): # 3. load to infer infer_layer = paddle.jit.load(model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) y = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) # 4. predict pred_x, pred_y = infer_layer(x, y) # 1. prune y and loss - model_path = os.path.join(self.temp_dir.name, - "multi_inout.output_spec2/model") + model_path = os.path.join( + self.temp_dir.name, "multi_inout.output_spec2/model" + ) output_spec = net.forward.outputs[:1] paddle.jit.save(net, model_path, [input_x], output_spec=output_spec) # 2. load again @@ -600,8 +610,9 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): def test_multi_in_out1(self): net = LinearNetMultiInput1(8, 8) - model_path = os.path.join(self.temp_dir.name, - "multi_inout1.output_spec1/model") + model_path = os.path.join( + self.temp_dir.name, "multi_inout1.output_spec1/model" + ) # 1. check inputs and outputs self.assertTrue(len(net.forward.inputs) == 2) input_x = net.forward.inputs[0] @@ -616,17 +627,20 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): # 3. load to infer infer_layer = paddle.jit.load(model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) y = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) # 4. predict pred_x, pred_y = infer_layer(x, y) # 1. prune y and loss - model_path = os.path.join(self.temp_dir.name, - "multi_inout1.output_spec2/model") + model_path = os.path.join( + self.temp_dir.name, "multi_inout1.output_spec2/model" + ) output_spec = net.forward.outputs[:1] - paddle.jit.save(net, model_path, (input_x, ), output_spec=output_spec) + paddle.jit.save(net, model_path, (input_x,), output_spec=output_spec) # 2. load again infer_layer2 = paddle.jit.load(model_path) # 3. predict @@ -637,7 +651,6 @@ class TestSaveLoadWithInputSpec(unittest.TestCase): class TestJitSaveLoadConfig(unittest.TestCase): - def setUp(self): # enable dygraph mode fluid.enable_dygraph() @@ -652,30 +665,36 @@ class TestJitSaveLoadConfig(unittest.TestCase): def test_output_spec(self): train_layer = LinearNetReturnLoss(8, 8) adam = fluid.optimizer.AdamOptimizer( - learning_rate=0.1, parameter_list=train_layer.parameters()) + learning_rate=0.1, parameter_list=train_layer.parameters() + ) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) for i in range(10): out, loss = train_layer(x) loss.backward() adam.minimize(loss) train_layer.clear_gradients() - model_path = os.path.join(self.temp_dir.name, - "save_load_config.output_spec") + model_path = os.path.join( + self.temp_dir.name, "save_load_config.output_spec" + ) output_spec = [out] - paddle.jit.save(layer=train_layer, - path=model_path, - input_spec=[x], - output_spec=output_spec) + paddle.jit.save( + layer=train_layer, + path=model_path, + input_spec=[x], + output_spec=output_spec, + ) train_layer.eval() infer_layer = paddle.jit.load(model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) np.testing.assert_array_equal( - train_layer(x)[0].numpy(), - infer_layer(x).numpy()) + train_layer(x)[0].numpy(), infer_layer(x).numpy() + ) def test_save_no_support_config_error(self): layer = LinearNet(784, 1) @@ -700,12 +719,12 @@ class TestJitSaveLoadConfig(unittest.TestCase): class TestJitMultipleLoading(unittest.TestCase): - def setUp(self): self.linear_size = 4 self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "jit_multi_load/model") + self.model_path = os.path.join( + self.temp_dir.name, "jit_multi_load/model" + ) # enable dygraph mode fluid.enable_dygraph() # config seed @@ -720,13 +739,14 @@ class TestJitMultipleLoading(unittest.TestCase): def train_and_save_orig_model(self): layer = LinearNet(self.linear_size, self.linear_size) example_inputs, layer, _ = train(layer, self.linear_size, 1) - paddle.jit.save(layer=layer, - path=self.model_path, - input_spec=example_inputs) + paddle.jit.save( + layer=layer, path=self.model_path, input_spec=example_inputs + ) def test_load_model_retransform_inference(self): - multi_loaded_layer = MultiLoadingLinearNet(self.linear_size, - self.model_path) + multi_loaded_layer = MultiLoadingLinearNet( + self.linear_size, self.model_path + ) state_dict = multi_loaded_layer.state_dict() name_set = set() for _, var in state_dict.items(): @@ -735,12 +755,12 @@ class TestJitMultipleLoading(unittest.TestCase): class TestJitPruneModelAndLoad(unittest.TestCase): - def setUp(self): self.linear_size = 4 self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "jit_prune_model_and_load/model") + self.model_path = os.path.join( + self.temp_dir.name, "jit_prune_model_and_load/model" + ) # enable dygraph mode fluid.enable_dygraph() # config seed @@ -753,9 +773,11 @@ class TestJitPruneModelAndLoad(unittest.TestCase): def train_and_save(self): train_layer = LinearNetReturnHidden(8, 8) adam = fluid.optimizer.AdamOptimizer( - learning_rate=0.1, parameter_list=train_layer.parameters()) + learning_rate=0.1, parameter_list=train_layer.parameters() + ) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) for i in range(10): hidden, loss = train_layer(x) loss.backward() @@ -763,10 +785,12 @@ class TestJitPruneModelAndLoad(unittest.TestCase): train_layer.clear_gradients() output_spec = [hidden] - paddle.jit.save(layer=train_layer, - path=self.model_path, - input_spec=[x], - output_spec=output_spec) + paddle.jit.save( + layer=train_layer, + path=self.model_path, + input_spec=[x], + output_spec=output_spec, + ) return train_layer @@ -777,10 +801,11 @@ class TestJitPruneModelAndLoad(unittest.TestCase): infer_layer = paddle.jit.load(self.model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) np.testing.assert_array_equal( - train_layer(x)[0].numpy(), - infer_layer(x).numpy()) + train_layer(x)[0].numpy(), infer_layer(x).numpy() + ) def test_load_var_not_in_extra_var_info(self): self.train_and_save() @@ -798,7 +823,6 @@ class TestJitPruneModelAndLoad(unittest.TestCase): class TestJitSaveMultiCases(unittest.TestCase): - def setUp(self): # enable dygraph mode fluid.enable_dygraph() @@ -810,11 +834,9 @@ class TestJitSaveMultiCases(unittest.TestCase): def tearDown(self): self.temp_dir.cleanup() - def verify_inference_correctness(self, - layer, - model_path, - with_label_and_loss=False, - with_label=False): + def verify_inference_correctness( + self, layer, model_path, with_label_and_loss=False, with_label=False + ): layer.eval() loaded_layer = paddle.jit.load(model_path) loaded_layer.eval() @@ -834,17 +856,19 @@ class TestJitSaveMultiCases(unittest.TestCase): np.testing.assert_array_equal( pred, loaded_pred, - err_msg= - 'Result diff when load and inference:\nlayer result:\n{}\nloaded layer result:\n{}' - .format(pred, loaded_pred)) + err_msg='Result diff when load and inference:\nlayer result:\n{}\nloaded layer result:\n{}'.format( + pred, loaded_pred + ), + ) def test_no_prune_to_static_after_train(self): layer = LinearNet(784, 1) train(layer) - model_path = os.path.join(self.temp_dir.name, - "test_no_prune_to_static_after_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_to_static_after_train/model" + ) paddle.jit.save(layer, model_path) self.verify_inference_correctness(layer, model_path) @@ -852,8 +876,9 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_no_prune_to_static_no_train(self): layer = LinearNetWithInputSpec(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_no_prune_to_static_no_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_to_static_no_train/model" + ) paddle.jit.save(layer, model_path) self.verify_inference_correctness(layer, model_path) @@ -864,11 +889,13 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) model_path = os.path.join( - self.temp_dir.name, "test_no_prune_no_to_static_after_train/model") + self.temp_dir.name, "test_no_prune_no_to_static_after_train/model" + ) paddle.jit.save( layer, model_path, - input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) + input_spec=[InputSpec(shape=[None, 784], dtype='float32')], + ) self.verify_inference_correctness(layer, model_path) @@ -879,7 +906,8 @@ class TestJitSaveMultiCases(unittest.TestCase): model_path = os.path.join( self.temp_dir.name, - "test_no_prune_no_to_static_after_train_with_examples/model") + "test_no_prune_no_to_static_after_train_with_examples/model", + ) paddle.jit.save(layer=layer, path=model_path, input_spec=example_inputs) self.verify_inference_correctness(layer, model_path) @@ -887,12 +915,14 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_no_prune_no_to_static_no_train(self): layer = LinearNetNotDeclarative(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_no_prune_no_to_static_no_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_no_to_static_no_train/model" + ) paddle.jit.save( layer, model_path, - input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) + input_spec=[InputSpec(shape=[None, 784], dtype='float32')], + ) self.verify_inference_correctness(layer, model_path) @@ -901,54 +931,57 @@ class TestJitSaveMultiCases(unittest.TestCase): out = train_with_label(layer) - model_path = os.path.join(self.temp_dir.name, - "test_prune_to_static_after_train/model") - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ], - output_spec=[out]) - - self.verify_inference_correctness(layer, - model_path, - with_label_and_loss=True) + model_path = os.path.join( + self.temp_dir.name, "test_prune_to_static_after_train/model" + ) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + output_spec=[out], + ) + + self.verify_inference_correctness( + layer, model_path, with_label_and_loss=True + ) def test_prune_to_static_no_train(self): layer = LinerNetWithLabel(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_prune_to_static_no_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_prune_to_static_no_train/model" + ) # TODO: no train, cannot get output_spec var here # now only can use index output_spec = layer.forward.outputs[:1] - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ], - output_spec=output_spec) - - self.verify_inference_correctness(layer, - model_path, - with_label_and_loss=True) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + output_spec=output_spec, + ) + + self.verify_inference_correctness( + layer, model_path, with_label_and_loss=True + ) def test_prune_input_to_static_no_train(self): layer = LinerNetWithPruneInput(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_prune_input_to_static_no_train/model") - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ]) + model_path = os.path.join( + self.temp_dir.name, "test_prune_input_to_static_no_train/model" + ) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + ) self.verify_inference_correctness(layer, model_path, with_label=True) @@ -957,14 +990,15 @@ class TestJitSaveMultiCases(unittest.TestCase): model_path = os.path.join( self.temp_dir.name, - "test_prune_useless_input_to_static_no_train/model") - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ]) + "test_prune_useless_input_to_static_no_train/model", + ) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + ) self.verify_inference_correctness(layer, model_path, with_label=True) @@ -974,18 +1008,20 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) model_path = os.path.join( - self.temp_dir.name, "test_no_prune_input_spec_name_warning/model") + self.temp_dir.name, "test_no_prune_input_spec_name_warning/model" + ) paddle.jit.save( layer, model_path, - input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name='feed_input') - ]) + input_spec=[InputSpec(shape=[None, 784], dtype='float32')], + ) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name='feed_input') + ], + ) self.verify_inference_correctness(layer, model_path) @@ -995,7 +1031,8 @@ class TestJitSaveMultiCases(unittest.TestCase): train(layer) model_path = os.path.join( - self.temp_dir.name, "test_not_prune_output_spec_name_warning/model") + self.temp_dir.name, "test_not_prune_output_spec_name_warning/model" + ) out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) paddle.jit.save(layer, model_path, output_spec=[out]) @@ -1004,47 +1041,52 @@ class TestJitSaveMultiCases(unittest.TestCase): def test_prune_input_spec_name_error(self): layer = LinerNetWithLabel(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_prune_input_spec_name_error/model") + model_path = os.path.join( + self.temp_dir.name, "test_prune_input_spec_name_error/model" + ) with self.assertRaises(ValueError): paddle.jit.save( layer, model_path, - input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) + input_spec=[InputSpec(shape=[None, 784], dtype='float32')], + ) with self.assertRaises(ValueError): - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name='feed_input') - ]) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec( + shape=[None, 784], dtype='float32', name='feed_input' + ) + ], + ) def test_prune_output_spec_name_error(self): layer = LinerNetWithLabel(784, 1) train_with_label(layer) - model_path = os.path.join(self.temp_dir.name, - "test_prune_to_static_after_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_prune_to_static_after_train/model" + ) out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) with self.assertRaises(ValueError): - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ], - output_spec=[out]) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + output_spec=[out], + ) class TestJitSaveLoadEmptyLayer(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "jit_save_load_empty_layer/model") + self.model_path = os.path.join( + self.temp_dir.name, "jit_save_load_empty_layer/model" + ) # enable dygraph mode paddle.disable_static() @@ -1062,11 +1104,11 @@ class TestJitSaveLoadEmptyLayer(unittest.TestCase): class TestJitSaveLoadNoParamLayer(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "jit_save_load_no_param_layer/model") + self.model_path = os.path.join( + self.temp_dir.name, "jit_save_load_no_param_layer/model" + ) # enable dygraph mode paddle.disable_static() @@ -1085,7 +1127,6 @@ class TestJitSaveLoadNoParamLayer(unittest.TestCase): class TestJitSaveLoadMultiMethods(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -1096,7 +1137,8 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): def test_jit_save_load_inference(self): model_path_inference = os.path.join( - self.temp_dir.name, "jit_save_load_multi_methods/model") + self.temp_dir.name, "jit_save_load_multi_methods/model" + ) IMAGE_SIZE = 224 layer = LinearNetWithMultiStaticFunc(IMAGE_SIZE, 10) inps = paddle.randn([1, IMAGE_SIZE]) @@ -1108,21 +1150,26 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): load_net = paddle.jit.load(model_path_inference) for func, result in result_origin.items(): self.assertTrue( - float((result - - getattr(load_net, func, None)(inps)).abs().max()) < 1e-5) + float( + (result - getattr(load_net, func, None)(inps)).abs().max() + ) + < 1e-5 + ) def test_jit_save_load_multi_methods_inputspec(self): - model_path = os.path.join(self.temp_dir.name, - 'jit_save_load_multi_methods/model') + model_path = os.path.join( + self.temp_dir.name, 'jit_save_load_multi_methods/model' + ) layer = LinearNetWithMultiStaticFunc(784, 1) with self.assertRaises(ValueError): - paddle.jit.save(layer, - model_path, - input_spec=[InputSpec(shape=[None, 784])]) + paddle.jit.save( + layer, model_path, input_spec=[InputSpec(shape=[None, 784])] + ) def test_parse_name(self): - model_path_inference = os.path.join(self.temp_dir.name, - "jit_save_load_parse_name/model") + model_path_inference = os.path.join( + self.temp_dir.name, "jit_save_load_parse_name/model" + ) IMAGE_SIZE = 224 layer = LinearNet(IMAGE_SIZE, 1) inps = paddle.randn([1, IMAGE_SIZE]) @@ -1135,7 +1182,6 @@ class TestJitSaveLoadMultiMethods(unittest.TestCase): class LayerSaved(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LayerSaved, self).__init__() self.hidden = 100 @@ -1157,7 +1203,6 @@ class LayerSaved(paddle.nn.Layer): class Net(paddle.nn.Layer): - def __init__(self): super().__init__() self.fc1 = paddle.nn.Linear(4, 4) @@ -1210,7 +1255,6 @@ class Net(paddle.nn.Layer): class NetTensor(paddle.nn.Layer): - def __init__(self): super().__init__() self.fc1 = paddle.nn.Linear(4, 4) @@ -1231,7 +1275,6 @@ class NetTensor(paddle.nn.Layer): class TestJitSaveCombineProperty(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -1241,17 +1284,19 @@ class TestJitSaveCombineProperty(unittest.TestCase): self.temp_dir.cleanup() def test_jit_save_combine_property(self): - model_path = os.path.join(self.temp_dir.name, - "test_jit_save_combine/model") + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_combine/model" + ) # Use new namespace with unique_name.guard(): net = Net() - #save + # save paddle.jit.save(net, model_path, combine_params=True) def test_jit_save_tensor_property(self): - model_path = os.path.join(self.temp_dir.name, - "test_jit_save_combine/model") + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_combine/model" + ) # Use new namespace with unique_name.guard(): net = NetTensor() @@ -1260,7 +1305,6 @@ class TestJitSaveCombineProperty(unittest.TestCase): class LayerLoadFinetune(paddle.nn.Layer): - def __init__(self, in_size, out_size, load_path): super(LayerLoadFinetune, self).__init__() # Test duplicate name @@ -1294,7 +1338,6 @@ class LayerLoadFinetune(paddle.nn.Layer): class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -1305,32 +1348,39 @@ class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase): def test_save_load_finetune_load(self): model_path = os.path.join( - self.temp_dir.name, "test_jit_save_load_save_without_running/model") + self.temp_dir.name, "test_jit_save_load_save_without_running/model" + ) IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) # Use new namespace with unique_name.guard(): layer_save = LayerSaved(IMAGE_SIZE, IMAGE_SIZE) - #save - paddle.jit.save(layer_save, - model_path, - input_spec=[ - paddle.static.InputSpec(shape=[None, IMAGE_SIZE], - dtype='float32') - ]) + # save + paddle.jit.save( + layer_save, + model_path, + input_spec=[ + paddle.static.InputSpec( + shape=[None, IMAGE_SIZE], dtype='float32' + ) + ], + ) result_00 = layer_save(inps0) result_01 = layer_save(inps1) - #load and save without running + # load and save without running with unique_name.guard(): layer_load = paddle.jit.load(model_path) - paddle.jit.save(layer_load, - model_path, - input_spec=[ - paddle.static.InputSpec( - shape=[None, IMAGE_SIZE], dtype='float32') - ]) - #reload + paddle.jit.save( + layer_load, + model_path, + input_spec=[ + paddle.static.InputSpec( + shape=[None, IMAGE_SIZE], dtype='float32' + ) + ], + ) + # reload layer_reload = paddle.jit.load(model_path) result_10 = layer_reload(inps0) result_11 = layer_reload(inps1) @@ -1340,7 +1390,6 @@ class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase): class TestJitSaveLoadFinetuneLoad(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -1350,8 +1399,9 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase): self.temp_dir.cleanup() def test_save_load_finetune_load(self): - model_path = os.path.join(self.temp_dir.name, - "test_jit_save_load_finetune_load/model") + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_load_finetune_load/model" + ) IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) @@ -1359,18 +1409,18 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase): with unique_name.guard(): layer_save = LayerSaved(IMAGE_SIZE, IMAGE_SIZE) layer_save(inps0) - #save + # save paddle.jit.save(layer_save, model_path) - #load + # load with unique_name.guard(): layer_load = LayerLoadFinetune(IMAGE_SIZE, IMAGE_SIZE, model_path) - #train + # train train(layer_load, input_size=IMAGE_SIZE) result_00 = layer_load(inps0) result_01 = layer_load(inps1) - #save + # save paddle.jit.save(layer_load, model_path) - #load + # load layer_finetune = paddle.jit.load(model_path) result_10 = layer_finetune(inps0) result_11 = layer_finetune(inps1) @@ -1385,7 +1435,6 @@ class TestJitSaveLoadFinetuneLoad(unittest.TestCase): # So divided into three TestCase: TestJitSaveLoadFunctionCase1, # TestJitSaveLoadFunctionCase2, TestJitSaveLoadFunctionCase3. class TestJitSaveLoadFunctionCase1(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1394,13 +1443,13 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase): self.temp_dir.cleanup() def test_jit_save_load_static_function(self): - @paddle.jit.to_static def fun(inputs): return paddle.tanh(inputs) - path = os.path.join(self.temp_dir.name, - 'test_jit_save_load_function_1/func') + path = os.path.join( + self.temp_dir.name, 'test_jit_save_load_function_1/func' + ) inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1412,7 +1461,6 @@ class TestJitSaveLoadFunctionCase1(unittest.TestCase): class TestJitSaveLoadFunctionCase2(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1421,15 +1469,17 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase): self.temp_dir.cleanup() def test_jit_save_load_function_input_spec(self): - - @paddle.jit.to_static(input_spec=[ - InputSpec(shape=[None, 6], dtype='float32', name='x'), - ]) + @paddle.jit.to_static( + input_spec=[ + InputSpec(shape=[None, 6], dtype='float32', name='x'), + ] + ) def fun(inputs): return paddle.nn.functional.relu(inputs) - path = os.path.join(self.temp_dir.name, - 'test_jit_save_load_function_2/func') + path = os.path.join( + self.temp_dir.name, 'test_jit_save_load_function_2/func' + ) inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1440,7 +1490,6 @@ class TestJitSaveLoadFunctionCase2(unittest.TestCase): class TestJitSaveLoadFunctionCase3(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1449,22 +1498,22 @@ class TestJitSaveLoadFunctionCase3(unittest.TestCase): self.temp_dir.cleanup() def test_jit_save_load_function_function(self): - def fun(inputs): return paddle.tanh(inputs) - path = os.path.join(self.temp_dir.name, - 'test_jit_save_load_function_3/func') + path = os.path.join( + self.temp_dir.name, 'test_jit_save_load_function_3/func' + ) inps = paddle.rand([3, 6]) origin = fun(inps) - paddle.jit.save(fun, - path, - input_spec=[ - InputSpec(shape=[None, 6], - dtype='float32', - name='x'), - ]) + paddle.jit.save( + fun, + path, + input_spec=[ + InputSpec(shape=[None, 6], dtype='float32', name='x'), + ], + ) load_func = paddle.jit.load(path) load_result = load_func(inps) @@ -1472,7 +1521,6 @@ class TestJitSaveLoadFunctionCase3(unittest.TestCase): class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1481,9 +1529,7 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): self.temp_dir.cleanup() def test_jit_save_load_function(self): - class LinearNet(paddle.nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = paddle.nn.Linear(5, 6) @@ -1499,11 +1545,13 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): inps = paddle.rand([3, 5]) origin = layer.anothor_forward(inps) - func = paddle.jit.to_static(layer.anothor_forward, - [paddle.static.InputSpec(shape=[-1, 5])]) + func = paddle.jit.to_static( + layer.anothor_forward, [paddle.static.InputSpec(shape=[-1, 5])] + ) path = os.path.join( self.temp_dir.name, - 'test_jit_save_load_function_with_params_case1/func') + 'test_jit_save_load_function_with_params_case1/func', + ) paddle.jit.save(func, path) load_func = paddle.jit.load(path) @@ -1512,7 +1560,6 @@ class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1521,9 +1568,7 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): self.temp_dir.cleanup() def test_jit_save_load_function(self): - class LinearNet(paddle.nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = paddle.nn.Linear(5, 6) @@ -1541,19 +1586,20 @@ class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): path = os.path.join( self.temp_dir.name, - 'test_jit_save_load_function_with_params_case2/func') + 'test_jit_save_load_function_with_params_case2/func', + ) paddle.jit.save(layer.anothor_forward, path) origin_result = layer.anothor_forward(inps) load_func = paddle.jit.load(path) load_result = load_func(inps) - np.testing.assert_array_equal(origin_result.numpy(), - load_result.numpy()) + np.testing.assert_array_equal( + origin_result.numpy(), load_result.numpy() + ) class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1562,9 +1608,7 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): self.temp_dir.cleanup() def test_jit_save_load_function(self): - class LinearNet(paddle.nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = paddle.nn.Linear(5, 6) @@ -1583,7 +1627,8 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): path = os.path.join( self.temp_dir.name, - 'test_jit_save_load_function_with_params_case3/func') + 'test_jit_save_load_function_with_params_case3/func', + ) paddle.jit.save(layer.anothor_forward, path) load_func = paddle.jit.load(path) @@ -1592,7 +1637,6 @@ class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): class TestJitSaveLoadDataParallel(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -1610,18 +1654,20 @@ class TestJitSaveLoadDataParallel(unittest.TestCase): np.testing.assert_array_equal( pred, loaded_pred, - err_msg= - 'Result diff when load and inference:\nlayer result:\n{}\nloaded layer result:\n{}' - .format(pred, loaded_pred)) + err_msg='Result diff when load and inference:\nlayer result:\n{}\nloaded layer result:\n{}'.format( + pred, loaded_pred + ), + ) def test_jit_save_data_parallel_with_inputspec(self): layer = LinearNetNotDeclarative(784, 1) layer = paddle.DataParallel(layer) - path = os.path.join(self.temp_dir.name, - "jit_save_data_parallel_with_inputspec/model") - paddle.jit.save(layer=layer, - path=path, - input_spec=[InputSpec(shape=[None, 784])]) + path = os.path.join( + self.temp_dir.name, "jit_save_data_parallel_with_inputspec/model" + ) + paddle.jit.save( + layer=layer, path=path, input_spec=[InputSpec(shape=[None, 784])] + ) self.verify_inference_correctness(layer, path) @@ -1629,8 +1675,9 @@ class TestJitSaveLoadDataParallel(unittest.TestCase): layer = LinearNetWithInputSpec(784, 1) layer = paddle.DataParallel(layer) - path = os.path.join(self.temp_dir.name, - "jit_save_data_parallel_with_to_static/model") + path = os.path.join( + self.temp_dir.name, "jit_save_data_parallel_with_to_static/model" + ) paddle.jit.save(layer, path) self.verify_inference_correctness(layer, path) @@ -1641,16 +1688,17 @@ class InputSepcLayer(paddle.nn.Layer): A layer with InputSpec to test InputSpec compatibility ''' - @paddle.jit.to_static(input_spec=[ - InputSpec(shape=[None, 8], dtype='float32', name='x'), - InputSpec(shape=[None, 1], dtype='float64', name='y') - ]) + @paddle.jit.to_static( + input_spec=[ + InputSpec(shape=[None, 8], dtype='float32', name='x'), + InputSpec(shape=[None, 1], dtype='float64', name='y'), + ] + ) def forward(self, x, y): return x, y class TestInputSpecCompatibility(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -1662,15 +1710,18 @@ class TestInputSpecCompatibility(unittest.TestCase): input_y = paddle.uniform([8, 1], dtype='float64') expected_result = expect_layer(input_x, input_y) test_result = test_layer(input_x, input_y) - np.testing.assert_allclose(expected_result[0].numpy(), - test_result[0].numpy()) - np.testing.assert_allclose(expected_result[1].numpy(), - test_result[1].numpy()) + np.testing.assert_allclose( + expected_result[0].numpy(), test_result[0].numpy() + ) + np.testing.assert_allclose( + expected_result[1].numpy(), test_result[1].numpy() + ) def test_jit_save_compatible_input_sepc(self): layer = InputSepcLayer() - save_dir = os.path.join(self.temp_dir.name, - "jit_save_compatible_input_spec") + save_dir = os.path.join( + self.temp_dir.name, "jit_save_compatible_input_spec" + ) path = save_dir + "/model" paddle.jit.save(layer=layer, path=path) @@ -1678,68 +1729,74 @@ class TestInputSpecCompatibility(unittest.TestCase): self._assert_input_spec_layer_return(layer, no_input_spec_layer) shutil.rmtree(save_dir) - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[None, 8], - dtype='float32', - name='x'), - InputSpec(shape=[None, 1], - dtype='float64', - name='y') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[None, 8], dtype='float32', name='x'), + InputSpec(shape=[None, 1], dtype='float64', name='y'), + ], + ) same_input_spec_layer = paddle.jit.load(path) self._assert_input_spec_layer_return(layer, same_input_spec_layer) shutil.rmtree(save_dir) - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[8, 8], dtype='float32'), - InputSpec(shape=[8, -1], dtype='float64') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[8, 8], dtype='float32'), + InputSpec(shape=[8, -1], dtype='float64'), + ], + ) compatible_input_spec_layer = paddle.jit.load(path) self._assert_input_spec_layer_return(layer, compatible_input_spec_layer) shutil.rmtree(save_dir) def test_jit_save_incompatible_input_sepc(self): layer = InputSepcLayer() - save_dir = os.path.join(self.temp_dir.name, - "jit_save_compatible_input_spec") + save_dir = os.path.join( + self.temp_dir.name, "jit_save_compatible_input_spec" + ) path = save_dir + "/model" with self.assertRaises(ValueError): # type mismatch - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[None, 8], dtype='float64'), - InputSpec(shape=[None, 1], dtype='float64') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[None, 8], dtype='float64'), + InputSpec(shape=[None, 1], dtype='float64'), + ], + ) with self.assertRaises(ValueError): # shape len mismatch - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[None, 8, 1], dtype='float32'), - InputSpec(shape=[None, 1], dtype='float64') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[None, 8, 1], dtype='float32'), + InputSpec(shape=[None, 1], dtype='float64'), + ], + ) with self.assertRaises(ValueError): # shape mismatch - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[None, 8], dtype='float32'), - InputSpec(shape=[None, 2], dtype='float64') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[None, 8], dtype='float32'), + InputSpec(shape=[None, 2], dtype='float64'), + ], + ) if os.path.exists(save_dir): shutil.rmtree(save_dir) class NotJitForward(paddle.nn.Layer): - def __init__(self): super(NotJitForward, self).__init__() @@ -1748,7 +1805,6 @@ class NotJitForward(paddle.nn.Layer): class TestNotJitForward(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py b/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py index 6358854255d..9fd0868f1e4 100644 --- a/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py @@ -36,7 +36,6 @@ def kldiv_loss(x, target, reduction): class TestKLDivLossOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'kldiv_loss' @@ -57,10 +56,9 @@ class TestKLDivLossOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Loss', - no_grad_set=set(["Target"]), - check_eager=True) + self.check_grad( + ['X'], 'Loss', no_grad_set=set(["Target"]), check_eager=True + ) def initTestCase(self): self.x_shape = (4, 5, 5) @@ -68,28 +66,24 @@ class TestKLDivLossOp(OpTest): class TestKLDivLossOp2(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (3, 2, 7, 7) self.reduction = 'none' class TestKLDivLossOp3(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 7, 9) self.reduction = 'mean' class TestKLDivLossOp4(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (5, 20) self.reduction = 'sum' class TestKLDivLossDygraph(unittest.TestCase): - def run_kl_loss(self, reduction, shape=(5, 20)): x = np.random.uniform(-10, 10, shape).astype('float64') target = np.random.uniform(-10, 10, shape).astype('float64') @@ -97,8 +91,9 @@ class TestKLDivLossDygraph(unittest.TestCase): with paddle.fluid.dygraph.guard(): kldiv_criterion = paddle.nn.KLDivLoss(reduction) - pred_loss = kldiv_criterion(paddle.to_tensor(x), - paddle.to_tensor(target)) + pred_loss = kldiv_criterion( + paddle.to_tensor(x), paddle.to_tensor(target) + ) np.testing.assert_allclose(pred_loss.numpy(), gt_loss, rtol=1e-05) def test_kl_loss_batchmean(self): @@ -126,7 +121,6 @@ class TestKLDivLossDygraph(unittest.TestCase): class TestKLDivLossTypePromotion(unittest.TestCase): - def test_kl_div_promotion(self): with paddle.fluid.dygraph.guard(): diff --git a/python/paddle/fluid/tests/unittests/test_kron_op.py b/python/paddle/fluid/tests/unittests/test_kron_op.py index b29df857b62..a2f1657a06e 100644 --- a/python/paddle/fluid/tests/unittests/test_kron_op.py +++ b/python/paddle/fluid/tests/unittests/test_kron_op.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import _test_eager_guard class TestKronOp(OpTest): - def setUp(self): self.op_type = "kron" self.python_api = paddle.kron @@ -51,7 +50,6 @@ class TestKronOp(OpTest): class TestKronOp2(TestKronOp): - def setUp(self): self.op_type = "kron" self.python_api = paddle.kron @@ -64,7 +62,6 @@ class TestKronOp2(TestKronOp): class TestKronOp3(TestKronOp): - def setUp(self): self.op_type = "kron" self.python_api = paddle.kron @@ -77,7 +74,6 @@ class TestKronOp3(TestKronOp): class TestKronLayer(unittest.TestCase): - def test_case(self): a = np.random.randn(10, 10).astype(np.float64) b = np.random.randn(10, 10).astype(np.float64) @@ -104,7 +100,7 @@ class TestKronLayer(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(start) - c, = exe.run(main, feed={'a': a, 'b': b}, fetch_list=[out_var]) + (c,) = exe.run(main, feed={'a': a, 'b': b}, fetch_list=[out_var]) np.testing.assert_allclose(c, np.kron(a, b)) def test_api_eager_dygraph(self): @@ -114,7 +110,6 @@ class TestKronLayer(unittest.TestCase): class TestComplexKronOp(OpTest): - def setUp(self): self.op_type = "kron" self.python_api = paddle.kron @@ -127,7 +122,7 @@ class TestComplexKronOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -137,14 +132,17 @@ class TestComplexKronOp(OpTest): def init_input_output(self): self.x = np.random.random(self.x_shape).astype( - self.dtype) + 1J * np.random.random(self.x_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.x_shape).astype(self.dtype) self.y = np.random.random(self.y_shape).astype( - self.dtype) + 1J * np.random.random(self.y_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.y_shape).astype(self.dtype) self.out = np.kron(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones(self.out_shape, self.dtype) + 1J * np.ones( - self.out_shape, self.dtype) + self.grad_out = np.ones(self.out_shape, self.dtype) + 1j * np.ones( + self.out_shape, self.dtype + ) self.grad_x = self.get_grad_x_by_numpy() self.grad_y = self.get_grad_y_by_numpy() @@ -157,7 +155,8 @@ class TestComplexKronOp(OpTest): idx_i = x_i * self.y_shape[0] + i idx_j = x_j * self.y_shape[1] + j grad_x[x_i][x_j] += self.grad_out[idx_i][ - idx_j] * np.conj(self.y[i][j]) + idx_j + ] * np.conj(self.y[i][j]) return grad_x def get_grad_y_by_numpy(self): @@ -169,47 +168,55 @@ class TestComplexKronOp(OpTest): idx_i = x_i * self.y_shape[0] + y_i idx_j = x_j * self.y_shape[1] + y_j grad_y[y_i][y_j] += self.grad_out[idx_i][ - idx_j] * np.conj(self.x[x_i][x_j]) + idx_j + ] * np.conj(self.x[x_i][x_j]) return grad_y def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestKronOpTypePromotion(TestComplexKronOp): - def init_input_output(self): self.x = np.random.random(self.x_shape).astype(self.dtype) self.y = np.random.random(self.y_shape).astype( - self.dtype) + 1J * np.random.random(self.y_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.y_shape).astype(self.dtype) self.out = np.kron(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones(self.out_shape, self.dtype) + 1J * np.ones( - self.out_shape, self.dtype) + self.grad_out = np.ones(self.out_shape, self.dtype) + 1j * np.ones( + self.out_shape, self.dtype + ) self.grad_x = self.get_grad_x_by_numpy().real self.grad_y = self.get_grad_y_by_numpy() diff --git a/python/paddle/fluid/tests/unittests/test_kthvalue_op.py b/python/paddle/fluid/tests/unittests/test_kthvalue_op.py index 823efffd3ba..f3f5ccf1879 100644 --- a/python/paddle/fluid/tests/unittests/test_kthvalue_op.py +++ b/python/paddle/fluid/tests/unittests/test_kthvalue_op.py @@ -33,7 +33,6 @@ def cal_kthvalue(x, k, axis, keepdim=False): class TestKthvalueOp(OpTest): - def init_args(self): self.k = 5 self.axis = -1 @@ -46,9 +45,9 @@ class TestKthvalueOp(OpTest): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis} - output, indices = cal_kthvalue(self.input_data, - k=self.k, - axis=self.axis) + output, indices = cal_kthvalue( + self.input_data, k=self.k, axis=self.axis + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -61,7 +60,6 @@ class TestKthvalueOp(OpTest): class TestKthvalueOpWithKeepdim(OpTest): - def init_args(self): self.k = 2 self.axis = 1 @@ -74,10 +72,9 @@ class TestKthvalueOpWithKeepdim(OpTest): self.input_data = np.random.random((1, 3, 2, 4, 10)) self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'keepdim': True} - output, indices = cal_kthvalue(self.input_data, - k=self.k, - axis=self.axis, - keepdim=True) + output, indices = cal_kthvalue( + self.input_data, k=self.k, axis=self.axis, keepdim=True + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -90,7 +87,6 @@ class TestKthvalueOpWithKeepdim(OpTest): class TestKthvalueOpKernels(unittest.TestCase): - def setUp(self): self.axises = [2, -1] @@ -107,9 +103,9 @@ class TestKthvalueOpKernels(unittest.TestCase): value_expect, indice_expect = cal_kthvalue(inputs, k, axis) v, inds = paddle.kthvalue(tensor, k, axis) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) - np.testing.assert_allclose(inds.numpy(), - indice_expect, - rtol=1e-05) + np.testing.assert_allclose( + inds.numpy(), indice_expect, rtol=1e-05 + ) def test_gpu_kernel(): shape = (2, 30, 250) @@ -121,9 +117,9 @@ class TestKthvalueOpKernels(unittest.TestCase): value_expect, indice_expect = cal_kthvalue(inputs, k, axis) v, inds = paddle.kthvalue(tensor, k, axis) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) - np.testing.assert_allclose(inds.numpy(), - indice_expect, - rtol=1e-05) + np.testing.assert_allclose( + inds.numpy(), indice_expect, rtol=1e-05 + ) test_cpu_kernel() if fluid.core.is_compiled_with_cuda(): @@ -131,13 +127,11 @@ class TestKthvalueOpKernels(unittest.TestCase): class TestKthvalueOpWithNaN(unittest.TestCase): - def setUp(self): paddle.disable_static() self.x = paddle.uniform([2, 200, 10], dtype='float32') def test_errors(self): - def test_nan_in_cpu_kernel(): paddle.set_device('cpu') nan_position = 100 @@ -160,7 +154,6 @@ class TestKthvalueOpWithNaN(unittest.TestCase): class TestKthvalueOpErrors(unittest.TestCase): - def setUp(self): self.x = paddle.uniform([2, 10, 20, 25], dtype='float32') @@ -184,7 +177,6 @@ class TestKthvalueOpErrors(unittest.TestCase): class TestModeOpInStatic(unittest.TestCase): - def setUp(self): np.random.seed(666) self.input_data = np.random.random((2, 20, 1, 2, 80)).astype(np.float64) @@ -192,16 +184,18 @@ class TestModeOpInStatic(unittest.TestCase): def test_run_static(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[2, 20, 1, 2, 80], - dtype="float64") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[2, 20, 1, 2, 80], dtype="float64" + ) result = paddle.kthvalue(input_tensor, self.k, axis=1) expect_value = cal_kthvalue(self.input_data, self.k, axis=1)[0] exe = paddle.static.Executor(paddle.CPUPlace()) - paddle_result = exe.run(feed={"x": self.input_data}, - fetch_list=[result])[0] + paddle_result = exe.run( + feed={"x": self.input_data}, fetch_list=[result] + )[0] np.testing.assert_allclose(paddle_result, expect_value, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_l1_loss.py b/python/paddle/fluid/tests/unittests/test_l1_loss.py index bd68a125e3a..00b716ce871 100644 --- a/python/paddle/fluid/tests/unittests/test_l1_loss.py +++ b/python/paddle/fluid/tests/unittests/test_l1_loss.py @@ -19,7 +19,6 @@ import unittest class TestFunctionalL1Loss(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10, 5)).astype(np.float32) self.label_np = np.random.random(size=(10, 10, 5)).astype(np.float32) @@ -43,12 +42,12 @@ class TestFunctionalL1Loss(unittest.TestCase): self.assertTrue(dy_result.shape, [10, 10, 5]) def run_static(self, use_gpu=False): - input = paddle.fluid.data(name='input', - shape=[10, 10, 5], - dtype='float32') - label = paddle.fluid.data(name='label', - shape=[10, 10, 5], - dtype='float32') + input = paddle.fluid.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=[10, 10, 5], dtype='float32' + ) result0 = paddle.nn.functional.l1_loss(input, label) result1 = paddle.nn.functional.l1_loss(input, label, reduction='sum') result2 = paddle.nn.functional.l1_loss(input, label, reduction='none') @@ -57,11 +56,10 @@ class TestFunctionalL1Loss(unittest.TestCase): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - static_result = exe.run(feed={ - "input": self.input_np, - "label": self.label_np - }, - fetch_list=[result0, result1, result2]) + static_result = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[result0, result1, result2], + ) expected = np.mean(np.abs(self.input_np - self.label_np)) np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) @@ -93,23 +91,21 @@ class TestFunctionalL1Loss(unittest.TestCase): # test case the raise message def test_errors(self): - def test_value_error(): - input = paddle.fluid.data(name='input', - shape=[10, 10, 5], - dtype='float32') - label = paddle.fluid.data(name='label', - shape=[10, 10, 5], - dtype='float32') - loss = paddle.nn.functional.l1_loss(input, - label, - reduction='reduce_mean') + input = paddle.fluid.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=[10, 10, 5], dtype='float32' + ) + loss = paddle.nn.functional.l1_loss( + input, label, reduction='reduce_mean' + ) self.assertRaises(ValueError, test_value_error) class TestClassL1Loss(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10, 5)).astype(np.float32) self.label_np = np.random.random(size=(10, 10, 5)).astype(np.float32) @@ -136,12 +132,12 @@ class TestClassL1Loss(unittest.TestCase): self.assertTrue(dy_result.shape, [10, 10, 5]) def run_static(self, use_gpu=False): - input = paddle.fluid.data(name='input', - shape=[10, 10, 5], - dtype='float32') - label = paddle.fluid.data(name='label', - shape=[10, 10, 5], - dtype='float32') + input = paddle.fluid.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=[10, 10, 5], dtype='float32' + ) l1_loss = paddle.nn.loss.L1Loss() result0 = l1_loss(input, label) l1_loss = paddle.nn.loss.L1Loss(reduction='sum') @@ -154,11 +150,10 @@ class TestClassL1Loss(unittest.TestCase): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - static_result = exe.run(feed={ - "input": self.input_np, - "label": self.label_np - }, - fetch_list=[result0, result1, result2]) + static_result = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[result0, result1, result2], + ) expected = np.mean(np.abs(self.input_np - self.label_np)) np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) @@ -189,7 +184,6 @@ class TestClassL1Loss(unittest.TestCase): # test case the raise message def test_errors(self): - def test_value_error(): loss = paddle.nn.loss.L1Loss(reduction="reduce_mean") diff --git a/python/paddle/fluid/tests/unittests/test_l1_norm_op.py b/python/paddle/fluid/tests/unittests/test_l1_norm_op.py index bc568dd5414..03d69e23347 100644 --- a/python/paddle/fluid/tests/unittests/test_l1_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_l1_norm_op.py @@ -18,8 +18,7 @@ from op_test import OpTest class TestL1NormOp(OpTest): - """Test l1_norm - """ + """Test l1_norm""" def setUp(self): self.op_type = "l1_norm" diff --git a/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py b/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py index c5859b73116..12b2f0a4714 100644 --- a/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py +++ b/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py @@ -21,13 +21,14 @@ import unittest class LabelSmoothTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - label_shape=(20, 1), - prior_dist=None, - epsilon=0.1, - dtype="float32"): + def __init__( + self, + methodName='runTest', + label_shape=(20, 1), + prior_dist=None, + epsilon=0.1, + dtype="float32", + ): super(LabelSmoothTestCase, self).__init__(methodName) self.label_shape = label_shape @@ -44,17 +45,19 @@ class LabelSmoothTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - label_var = fluid.data("input", - self.label_shape, - dtype=self.dtype) - y_var = fluid.layers.label_smooth(label_var, - prior_dist=self.prior_dist, - epsilon=self.epsilon, - dtype=self.dtype) + label_var = fluid.data( + "input", self.label_shape, dtype=self.dtype + ) + y_var = fluid.layers.label_smooth( + label_var, + prior_dist=self.prior_dist, + epsilon=self.epsilon, + dtype=self.dtype, + ) feed_dict = {"input": self.label} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -63,24 +66,24 @@ class LabelSmoothTestCase(unittest.TestCase): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - label_var = fluid.data("input", - self.label_shape, - dtype=self.dtype) - y_var = F.label_smooth(label_var, - prior_dist=self.prior_dist, - epsilon=self.epsilon) + label_var = fluid.data( + "input", self.label_shape, dtype=self.dtype + ) + y_var = F.label_smooth( + label_var, prior_dist=self.prior_dist, epsilon=self.epsilon + ) feed_dict = {"input": self.label} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_dygraph_layer(self): paddle.disable_static() label_var = dg.to_variable(self.label) - y_var = F.label_smooth(label_var, - prior_dist=self.prior_dist, - epsilon=self.epsilon) + y_var = F.label_smooth( + label_var, prior_dist=self.prior_dist, epsilon=self.epsilon + ) y_np = y_var.numpy() return y_np @@ -101,7 +104,6 @@ class LabelSmoothTestCase(unittest.TestCase): class LabelSmoothErrorTestCase(LabelSmoothTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -112,7 +114,8 @@ class LabelSmoothErrorTestCase(LabelSmoothTestCase): def add_cases(suite): suite.addTest(LabelSmoothTestCase(methodName='runTest')) suite.addTest( - LabelSmoothTestCase(methodName='runTest', label_shape=[2, 3, 1])) + LabelSmoothTestCase(methodName='runTest', label_shape=[2, 3, 1]) + ) def add_error_cases(suite): diff --git a/python/paddle/fluid/tests/unittests/test_label_smooth_op.py b/python/paddle/fluid/tests/unittests/test_label_smooth_op.py index 0fe5862f2c9..9166d07c741 100644 --- a/python/paddle/fluid/tests/unittests/test_label_smooth_op.py +++ b/python/paddle/fluid/tests/unittests/test_label_smooth_op.py @@ -19,7 +19,6 @@ import paddle class TestLabelSmoothOp(OpTest): - def config(self): self.op_type = "label_smooth" self.python_api = paddle.nn.functional.label_smooth @@ -32,7 +31,8 @@ class TestLabelSmoothOp(OpTest): def setUp(self): self.config() smoothed_label = ( - 1 - self.epsilon) * self.label + self.epsilon / self.label_dim + 1 - self.epsilon + ) * self.label + self.epsilon / self.label_dim self.inputs = {'X': self.label} self.attrs = {'epsilon': self.epsilon} self.outputs = {'Out': smoothed_label} @@ -45,7 +45,6 @@ class TestLabelSmoothOp(OpTest): class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): - def setUp(self): self.config() dist = np.random.random((1, self.label_dim)) @@ -56,23 +55,25 @@ class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): class TestLabelSmoothOp3D(TestLabelSmoothOp): - def setUp(self): super(TestLabelSmoothOp3D, self).setUp() self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) class TestLabelSmoothOpWithPriorDist3D(TestLabelSmoothOpWithPriorDist): - def setUp(self): super(TestLabelSmoothOpWithPriorDist3D, self).setUp() self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lamb_op.py b/python/paddle/fluid/tests/unittests/test_lamb_op.py index eaa8db70313..9c3bbe6d830 100644 --- a/python/paddle/fluid/tests/unittests/test_lamb_op.py +++ b/python/paddle/fluid/tests/unittests/test_lamb_op.py @@ -23,18 +23,16 @@ paddle.enable_static() class TestLambOp1(OpTest): - def set_attrs(self): self.attrs = { 'epsilon': 1e-4, 'beta1': 0.78, 'beta2': 0.836, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } def setUp(self): - '''Test Lamb Op with supplied attributes - ''' + '''Test Lamb Op with supplied attributes''' self.op_type = "lamb" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -53,19 +51,23 @@ class TestLambOp1(OpTest): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } - - param_out, moment1_out, moment2_out, \ - beta1_pow_out, beta2_pow_out = lamb_step(self.inputs, self.attrs) + ( + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + ) = lamb_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } def test_check_output(self): @@ -73,38 +75,41 @@ class TestLambOp1(OpTest): class TestLambOp2(TestLambOp1): - def set_attrs(self): self.attrs = { 'epsilon': 1e-8, 'beta1': 0.9, 'beta2': 0.999, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } class TestLambOpMultipleSteps(TestLambOp1): - def set_attrs(self): self.attrs = { 'epsilon': 1e-8, 'beta1': 0.9, 'beta2': 0.999, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } self.num_steps = 10 def test_check_output(self): for i in range(self.num_steps): - param_out, moment1_out, moment2_out, \ - beta1_pow_out, beta2_pow_out = lamb_step(self.inputs, self.attrs) + ( + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + ) = lamb_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } # Verify output for this step @@ -120,8 +125,9 @@ class TestLambOpMultipleSteps(TestLambOp1): self.inputs['Beta2Pow'] = beta2_pow_out # Randomize gradient for next step - self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + self.inputs['Grad'] = np.random.uniform(-1, 1, (102, 105)).astype( + "float32" + ) def lamb_step(inputs, attributes): @@ -152,14 +158,16 @@ def lamb_step(inputs, attributes): moment2_unbiased = moment2_out / (1 - beta2_pow) r_1 = np.linalg.norm(param) - r_2 = np.linalg.norm(moment1_unbiased / - (np.sqrt(moment2_unbiased) + epsilon) + - weight_decay * param) + r_2 = np.linalg.norm( + moment1_unbiased / (np.sqrt(moment2_unbiased) + epsilon) + + weight_decay * param + ) lr_t = lr * r_1 / r_2 - param_out = param - lr_t * (moment1_unbiased / - (np.sqrt(moment2_unbiased) + epsilon) + - weight_decay * param) + param_out = param - lr_t * ( + moment1_unbiased / (np.sqrt(moment2_unbiased) + epsilon) + + weight_decay * param + ) beta1_pow_out = beta1_pow * beta1 beta2_pow_out = beta2_pow * beta2 @@ -195,25 +203,32 @@ def lamb_step_sparse(inputs, attributes, height, rows, row_numel, np_grad): moment2_unbiased = np.zeros(shape=[height, row_numel]) def update_mom(row_id, update_value): - moment1_out[row_id] = beta1 * moment1[row_id] + (1 - - beta1) * update_value - moment2_out[row_id] = beta2 * moment2[row_id] + ( - 1 - beta2) * np.square(update_value) - - moment1_out[row_id] = beta1 * moment1[row_id] + (1 - - beta1) * update_value - moment2_out[row_id] = beta2 * moment2[row_id] + ( - 1 - beta2) * np.square(update_value) + moment1_out[row_id] = ( + beta1 * moment1[row_id] + (1 - beta1) * update_value + ) + moment2_out[row_id] = beta2 * moment2[row_id] + (1 - beta2) * np.square( + update_value + ) + + moment1_out[row_id] = ( + beta1 * moment1[row_id] + (1 - beta1) * update_value + ) + moment2_out[row_id] = beta2 * moment2[row_id] + (1 - beta2) * np.square( + update_value + ) def update_param(): r_1 = np.linalg.norm(param) - r_2 = np.linalg.norm(moment1_out / (np.sqrt(moment2_out) + epsilon) + - weight_decay * param) + r_2 = np.linalg.norm( + moment1_out / (np.sqrt(moment2_out) + epsilon) + + weight_decay * param + ) lr_t = lr * r_1 / r_2 - param_out = param - lr_t * (moment1_out / - (np.sqrt(moment2_out) + epsilon) + - weight_decay * param) + param_out = param - lr_t * ( + moment1_out / (np.sqrt(moment2_out) + epsilon) + + weight_decay * param + ) for row_id in range(param_out.shape[0]): update_value = np.zeros(np_grad[0].shape).astype("float32") @@ -229,7 +244,6 @@ def lamb_step_sparse(inputs, attributes, height, rows, row_numel, np_grad): class TestSparseLambOp(unittest.TestCase): - def setup(self, scope, place): beta1 = 0.78 beta2 = 0.836 @@ -246,14 +260,14 @@ class TestSparseLambOp(unittest.TestCase): "Moment2": np.full((height, row_numel), 5.0).astype("float32"), 'Beta1Pow': np.array([beta1]).astype("float32"), 'Beta2Pow': np.array([beta2]).astype("float32"), - "LearningRate": np.full((1), 2.0).astype("float32") + "LearningRate": np.full((1), 2.0).astype("float32"), } self.init_output = np.full((height, row_numel), 0.0).astype("float32") self.attrs = { 'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2, - 'weight_decay': 0.05 + 'weight_decay': 0.05, } grad_selected_rows = scope.var('Grad').get_selected_rows() @@ -269,13 +283,14 @@ class TestSparseLambOp(unittest.TestCase): self.sparse_inputs = ["Grad"] param_out, mom1, mom2, beta1_pow_out, beta2_pow_out = lamb_step_sparse( - self.dense_inputs, self.attrs, height, rows, row_numel, np_array) + self.dense_inputs, self.attrs, height, rows, row_numel, np_array + ) self.outputs = { "ParamOut": param_out, "Moment1Out": mom1, "Moment2Out": mom2, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } def check_with_place(self, place): diff --git a/python/paddle/fluid/tests/unittests/test_lambv2_op.py b/python/paddle/fluid/tests/unittests/test_lambv2_op.py index 1e01f3a4ce3..475866fb903 100644 --- a/python/paddle/fluid/tests/unittests/test_lambv2_op.py +++ b/python/paddle/fluid/tests/unittests/test_lambv2_op.py @@ -22,39 +22,40 @@ import paddle.fluid.layers as layers class LAMBOptimizer(paddle.optimizer.Lamb): - def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, fluid.framework.Block) block.program._use_lamb = True - m = moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) + m = moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) v = self._get_accumulator(self._moment2_acc_str, param_and_grad[0]) - beta_1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta_2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - - beta_1 = layers.fill_constant(dtype='float32', - shape=[1], - value=self._beta1, - name='lamb_beta_1') - beta_2 = layers.fill_constant(dtype='float32', - shape=[1], - value=self._beta2, - name='lamb_beta_2') - epsilon = layers.fill_constant(dtype='float32', - shape=[1], - value=self._epsilon, - name='epsilon') + beta_1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta_2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + + beta_1 = layers.fill_constant( + dtype='float32', shape=[1], value=self._beta1, name='lamb_beta_1' + ) + beta_2 = layers.fill_constant( + dtype='float32', shape=[1], value=self._beta2, name='lamb_beta_2' + ) + epsilon = layers.fill_constant( + dtype='float32', shape=[1], value=self._epsilon, name='epsilon' + ) one = paddle.ones(shape=[1]).astype('float32') zero = paddle.zeros(shape=[1]).astype('float32') next_m = paddle.multiply(m, beta_1) + paddle.multiply( - param_and_grad[1], one - beta_1) + param_and_grad[1], one - beta_1 + ) next_v = paddle.multiply(v, beta_2) + paddle.multiply( - paddle.pow(param_and_grad[1], 2), one - beta_2) + paddle.pow(param_and_grad[1], 2), one - beta_2 + ) beta1_correction = one - beta_1_pow_acc beta2_correction = one - beta_2_pow_acc @@ -64,8 +65,10 @@ class LAMBOptimizer(paddle.optimizer.Lamb): update = next_m_unbiased / (paddle.sqrt(next_v_unbiased) + epsilon) - if self._exclude_from_weight_decay_fn is not None and self._exclude_from_weight_decay_fn( - param_and_grad[0]): + if ( + self._exclude_from_weight_decay_fn is not None + and self._exclude_from_weight_decay_fn(param_and_grad[0]) + ): self._lamb_weight_decay = 0.0 update += self._lamb_weight_decay * param_and_grad[0] @@ -76,8 +79,11 @@ class LAMBOptimizer(paddle.optimizer.Lamb): ratio = paddle.where( paddle.greater_than(w_norm, zero), - paddle.where(paddle.greater_than(g_norm, zero), (w_norm / g_norm), - one), one) + paddle.where( + paddle.greater_than(g_norm, zero), (w_norm / g_norm), one + ), + one, + ) update_with_lr = ratio * learning_rate * update next_param = param_and_grad[0] - update_with_lr @@ -92,16 +98,15 @@ class LAMBOptimizer(paddle.optimizer.Lamb): class TestLambOpV2(unittest.TestCase): - def test_lamb_op(self): shape = [2, 4, 8, 8] data = paddle.to_tensor(np.random.random(size=shape).astype("float32")) conv = paddle.nn.Conv2D(4, 6, (3, 3)) data = conv(data) loss = paddle.mean(data) - opt = paddle.optimizer.Lamb(learning_rate=1e-5, - epsilon=1e-8, - parameters=conv.parameters()) + opt = paddle.optimizer.Lamb( + learning_rate=1e-5, epsilon=1e-8, parameters=conv.parameters() + ) loss.backward() opt.minimize(loss) @@ -109,7 +114,6 @@ class TestLambOpV2(unittest.TestCase): class TestLambOpWithCombinedOp(unittest.TestCase): - def test_lamb_op_with_multi_steps(self): paddle.enable_static() @@ -140,12 +144,11 @@ class TestLambOpWithCombinedOp(unittest.TestCase): executor = fluid.Executor(place) executor.run(startup_program) - output = executor.run(program=main_program, - feed={ - 'X': feed_x, - 'Y': feed_y - }, - fetch_list=[avg_loss.name]) + output = executor.run( + program=main_program, + feed={'X': feed_x, 'Y': feed_y}, + fetch_list=[avg_loss.name], + ) main = fluid.Program() startup = fluid.Program() @@ -156,18 +159,16 @@ class TestLambOpWithCombinedOp(unittest.TestCase): exe = fluid.Executor(place) exe.run(startup) - out = exe.run(program=main, - feed={ - 'X': feed_x, - 'Y': feed_y - }, - fetch_list=[loss.name]) + out = exe.run( + program=main, + feed={'X': feed_x, 'Y': feed_y}, + fetch_list=[loss.name], + ) np.testing.assert_allclose(out, output, rtol=1e-05) class TestLambOpV2Group(TestLambOpV2): - def test_lamb_op(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -175,16 +176,19 @@ class TestLambOpV2Group(TestLambOpV2): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Lamb(learning_rate=0.01, - parameters=[{ - 'params': linear_1.parameters() - }, { - 'params': linear_2.parameters(), - 'lamb_weight_decay': 0.001, - 'beta1': 0.9, - 'beta2': 0.99 - }], - lamb_weight_decay=0.01) + adam = paddle.optimizer.Lamb( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'lamb_weight_decay': 0.001, + 'beta1': 0.9, + 'beta2': 0.99, + }, + ], + lamb_weight_decay=0.01, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -193,16 +197,15 @@ class TestLambOpV2Group(TestLambOpV2): class TestLambOpMultiPrecision(unittest.TestCase): - def check_main(self, x_np, place, multi_precision=False, seed=10, n=10): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): paddle.seed(seed) with paddle.static.amp.fp16_guard(): - x = paddle.static.data(name='x', - shape=[None, 10], - dtype='float32') + x = paddle.static.data( + name='x', shape=[None, 10], dtype='float32' + ) linear = paddle.nn.Linear(10, 2) hidden = linear(x) loss = paddle.mean(hidden) @@ -210,9 +213,9 @@ class TestLambOpMultiPrecision(unittest.TestCase): original_optimizer = paddle.optimizer.Lamb(learning_rate=1e-3) original_optimizer._multi_precision = multi_precision if multi_precision: - optimizer = paddle.static.amp.decorate(original_optimizer, - use_pure_fp16=True, - use_fp16_guard=True) + optimizer = paddle.static.amp.decorate( + original_optimizer, use_pure_fp16=True, use_fp16_guard=True + ) else: optimizer = original_optimizer optimizer.minimize(loss) @@ -233,8 +236,9 @@ class TestLambOpMultiPrecision(unittest.TestCase): if multi_precision: params[0] = np.array(params[0]) params[1] = np.array(params[1]) - np.testing.assert_array_equal(params[0], - params[1].astype(np.float16)) + np.testing.assert_array_equal( + params[0], params[1].astype(np.float16) + ) return params[0].astype(np.float32) else: self.assertTrue(params[0] is not None) @@ -250,9 +254,9 @@ class TestLambOpMultiPrecision(unittest.TestCase): weight_np, bias_np = None, None for i in range(n): feed_dict = {x.name: x_np} - weight_np, bias_np = exe.run(main_prog, - feed=feed_dict, - fetch_list=[weight, bias]) + weight_np, bias_np = exe.run( + main_prog, feed=feed_dict, fetch_list=[weight, bias] + ) weight_np = weight_np.astype('float32') bias_np = bias_np.astype('float32') np.testing.assert_array_equal(weight_np, get_parameter(weight)) diff --git a/python/paddle/fluid/tests/unittests/test_launch_coverage.py b/python/paddle/fluid/tests/unittests/test_launch_coverage.py index 00db6590d6f..1f00ea9af55 100644 --- a/python/paddle/fluid/tests/unittests/test_launch_coverage.py +++ b/python/paddle/fluid/tests/unittests/test_launch_coverage.py @@ -15,7 +15,11 @@ import unittest from argparse import ArgumentParser, REMAINDER -from paddle.distributed.utils.launch_utils import _print_arguments, get_gpus, get_cluster_from_args +from paddle.distributed.utils.launch_utils import ( + _print_arguments, + get_gpus, + get_cluster_from_args, +) from paddle.distributed.fleet.launch_utils import find_free_ports @@ -32,72 +36,75 @@ PADDLE_CURRENT_ENDPOINT PADDLE_TRAINERS_NUM PADDLE_TRAINER_ENDPOINTS POD_IP (current node ip address, not needed for local training) -''') +''' + ) - #Optional arguments for the launch helper + # Optional arguments for the launch helper parser.add_argument( "--cluster_node_ips", type=str, default="127.0.0.1", - help="Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17..") - parser.add_argument("--node_ip", - type=str, - default="127.0.0.1", - help="The current node ip. ") + help="Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17..", + ) + parser.add_argument( + "--node_ip", type=str, default="127.0.0.1", help="The current node ip. " + ) parser.add_argument( "--use_paddlecloud", action='store_true', - help= - "wheter to use paddlecloud platform to run your multi-process job. If false, no need to set this argument." + help="wheter to use paddlecloud platform to run your multi-process job. If false, no need to set this argument.", + ) + parser.add_argument( + "--started_port", + type=int, + default=None, + help="The trainer's started port on a single node", ) - parser.add_argument("--started_port", - type=int, - default=None, - help="The trainer's started port on a single node") - parser.add_argument("--print_config", - type=bool, - default=True, - help="Print the config or not") + parser.add_argument( + "--print_config", + type=bool, + default=True, + help="Print the config or not", + ) parser.add_argument( "--selected_gpus", type=str, default=None, - help= - "It's for gpu training and the training process will run on the selected_gpus," - "each process is bound to a single GPU. And if it's not set, this module will use all the gpu cards for training." + help="It's for gpu training and the training process will run on the selected_gpus," + "each process is bound to a single GPU. And if it's not set, this module will use all the gpu cards for training.", ) parser.add_argument( "--log_level", type=int, - default= - 20, # logging.INFO, details are here:https://docs.python.org/3/library/logging.html#levels - help="Logging level, default is logging.INFO") + default=20, # logging.INFO, details are here:https://docs.python.org/3/library/logging.html#levels + help="Logging level, default is logging.INFO", + ) parser.add_argument( "--log_dir", type=str, - help= - "The path for each process's log.If it's not set, the log will printed to default pipe." + help="The path for each process's log.If it's not set, the log will printed to default pipe.", ) - #positional - parser.add_argument("training_script", - type=str, - help="The full path to the single GPU training " - "program/script to be launched in parallel, " - "followed by all the arguments for the " - "training script") + # positional + parser.add_argument( + "training_script", + type=str, + help="The full path to the single GPU training " + "program/script to be launched in parallel, " + "followed by all the arguments for the " + "training script", + ) - #rest from the training program + # rest from the training program parser.add_argument('training_script_args', nargs=REMAINDER) return parser.parse_args() class TestCoverage(unittest.TestCase): - def test_gpus(self): args = _parse_args() diff --git a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py index e49cf0b24e7..17d2730b3e8 100644 --- a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py @@ -23,7 +23,9 @@ import paddle.nn.functional as F from functools import reduce from op_test import _set_use_system_allocator from paddle.fluid import Program, program_guard -from paddle.fluid.contrib.mixed_precision.fp16_utils import _keep_layer_norm_scale_bias_to_fp32 +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _keep_layer_norm_scale_bias_to_fp32, +) paddle.enable_static() @@ -35,13 +37,14 @@ _set_use_system_allocator(True) def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): x_shape = x.shape N = reduce(mul, x_shape[0:begin_norm_axis], 1) - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) x.shape = [N, D] mean = np.mean(x, axis=1) var = np.var(x, axis=1) + epsilon - output = np.divide((x - mean.reshape([N, 1])), - (np.sqrt(var)).reshape([N, 1])) + output = np.divide( + (x - mean.reshape([N, 1])), (np.sqrt(var)).reshape([N, 1]) + ) if scale is not None: output = scale.reshape([1, D]) * output if beta is not None: @@ -51,16 +54,12 @@ def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): return output, mean, var -def _reference_layer_norm_grad(x, - grad_y, - scale, - bias, - mean, - var, - begin_norm_axis=1): +def _reference_layer_norm_grad( + x, grad_y, scale, bias, mean, var, begin_norm_axis=1 +): x_shape = x.shape N = reduce(mul, x_shape[0:begin_norm_axis], 1) - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) if scale is not None: scale_shape = scale.shape @@ -75,38 +74,39 @@ def _reference_layer_norm_grad(x, d_bias = None # d_scale if scale is not None: - d_scale = np.sum(((x - mean) * np.sqrt(1 / var)) * grad_y, - axis=0).reshape([1, D]) + d_scale = np.sum( + ((x - mean) * np.sqrt(1 / var)) * grad_y, axis=0 + ).reshape([1, D]) else: d_scale = None # dx if scale is not None: dx_end = scale * np.sqrt(1.0 / var) * grad_y - d_mean_0 = np.sum(-np.sqrt(1.0 / var) * grad_y * scale, - axis=1).reshape([N, 1 - ]) # the second part equals to zero. + d_mean_0 = np.sum(-np.sqrt(1.0 / var) * grad_y * scale, axis=1).reshape( + [N, 1] + ) # the second part equals to zero. d_mean = 1.0 / D * d_mean_0 - d_std = np.sum(-(1.0 / var) * - (x - mean) * grad_y * scale, axis=1).reshape([N, 1]) * ( - 1.0 / D * np.sqrt(1.0 / var).reshape([N, 1]) * - (x - mean)) + d_std = np.sum( + -(1.0 / var) * (x - mean) * grad_y * scale, axis=1 + ).reshape([N, 1]) * ( + 1.0 / D * np.sqrt(1.0 / var).reshape([N, 1]) * (x - mean) + ) else: dx_end = 1.0 * np.sqrt(1.0 / var) * grad_y - d_mean_0 = np.sum(-np.sqrt(1.0 / var) * grad_y * 1.0, - axis=1).reshape([N, 1 - ]) # the second part equals to zero. + d_mean_0 = np.sum(-np.sqrt(1.0 / var) * grad_y * 1.0, axis=1).reshape( + [N, 1] + ) # the second part equals to zero. d_mean = 1.0 / D * d_mean_0 - d_std = np.sum(-(1.0 / var) * - (x - mean) * grad_y * 1.0, axis=1).reshape([N, 1]) * ( - 1.0 / D * np.sqrt(1.0 / var).reshape([N, 1]) * - (x - mean)) + d_std = np.sum( + -(1.0 / var) * (x - mean) * grad_y * 1.0, axis=1 + ).reshape([N, 1]) * ( + 1.0 / D * np.sqrt(1.0 / var).reshape([N, 1]) * (x - mean) + ) grad_x = dx_end + d_mean + d_std grad_x.shape, x.shape, grad_y.shape = x_shape, x_shape, x_shape - var.shape, mean.shape = [ - N, - ], [ + var.shape, mean.shape = [N,], [ N, ] @@ -116,45 +116,53 @@ def _reference_layer_norm_grad(x, class TestLayerNormOp(unittest.TestCase): - def setUp(self): self.use_cudnn = True def __assert_close(self, tensor, np_array, msg, atol=1e-4): self.assertTrue(np.allclose(np.array(tensor), np_array, atol=atol), msg) - def check_forward_backward(self, - shape, - begin_norm_axis, - has_scale=True, - has_bias=True, - y_grad_scale=1.0, - use_mkldnn=False): - - def test_with_place(place, - shape, - begin_norm_axis, - use_mkldnn=use_mkldnn): + def check_forward_backward( + self, + shape, + begin_norm_axis, + has_scale=True, + has_bias=True, + y_grad_scale=1.0, + use_mkldnn=False, + ): + def test_with_place( + place, shape, begin_norm_axis, use_mkldnn=use_mkldnn + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) x = np.random.random_sample(x_shape).astype(np.float32) - scale = np.random.random_sample(scale_shape).astype( - np.float32) if has_scale else None - bias = np.random.random_sample(scale_shape).astype( - np.float32) if has_bias else None + scale = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_scale + else None + ) + bias = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_bias + else None + ) y_grad = (np.random.random_sample(x_shape) * y_grad_scale).astype( - np.float32) + np.float32 + ) # reference forward & backward y, mean, variance = _reference_layer_norm_naive( - x, scale, bias, epsilon, begin_norm_axis) + x, scale, bias, epsilon, begin_norm_axis + ) x_grad, scale_grad, bias_grad = _reference_layer_norm_grad( - x, y_grad, scale, bias, mean, variance, begin_norm_axis) + x, y_grad, scale, bias, mean, variance, begin_norm_axis + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -169,9 +177,11 @@ class TestLayerNormOp(unittest.TestCase): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = {"X": block.var('x')} fetch_list = [ 'y', @@ -191,17 +201,20 @@ class TestLayerNormOp(unittest.TestCase): outputs={ "Y": block.var('y'), "Mean": block.var('mean'), # share the same memory - "Variance": - block.var('variance'), # share the same memory + "Variance": block.var( + 'variance' + ), # share the same memory }, attrs={ "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, - "use_mkldnn": use_mkldnn - }) + "use_mkldnn": use_mkldnn, + }, + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - layer_norm_op.desc, set(), []) + layer_norm_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -215,12 +228,14 @@ class TestLayerNormOp(unittest.TestCase): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'scale', 'bias', 'y@GRAD'] - }, - fetch_list=fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in ['x', 'scale', 'bias', 'y@GRAD'] + }, + fetch_list=fetch_list, + ) # print(y) # print(out[0]) self.__assert_close(y, out[0], "y") @@ -228,17 +243,25 @@ class TestLayerNormOp(unittest.TestCase): self.__assert_close(variance, out[2], "variance", 1e-3) self.__assert_close(x_grad, out[3], "x_grad") if has_scale: - self.__assert_close(scale_grad, - out[fetch_list.index('scale@GRAD')], - "scale_grad", 1e-3) + self.__assert_close( + scale_grad, + out[fetch_list.index('scale@GRAD')], + "scale_grad", + 1e-3, + ) if has_bias: - self.__assert_close(bias_grad, - out[fetch_list.index('bias@GRAD')], - "bias_grad") + self.__assert_close( + bias_grad, + out[fetch_list.index('bias@GRAD')], + "bias_grad", + ) places = [core.CPUPlace()] - if core.is_compiled_with_cuda() and core.op_support_gpu( - "layer_norm") and self.use_cudnn: + if ( + core.is_compiled_with_cuda() + and core.op_support_gpu("layer_norm") + and self.use_cudnn + ): places.append(core.CUDAPlace(0)) for place in places: @@ -247,87 +270,108 @@ class TestLayerNormOp(unittest.TestCase): def test_check_forward_backward_with_scale_and_bias(self): self.check_forward_backward(shape=[1, 3, 4, 5], begin_norm_axis=1) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=1) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=True) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=True, - has_bias=False) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=False) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=True, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=True, + has_bias=False, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=False, + ) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=3) - self.check_forward_backward(shape=[92, 513, 129], - begin_norm_axis=2, - y_grad_scale=0.1) + self.check_forward_backward( + shape=[92, 513, 129], begin_norm_axis=2, y_grad_scale=0.1 + ) self.check_forward_backward(shape=[3, 34, 1134], begin_norm_axis=2) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=False, - has_bias=True, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=True, - has_bias=False, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=False, - has_bias=False, - y_grad_scale=0.1) - self.check_forward_backward(shape=[512, 1024], - begin_norm_axis=1, - has_scale=True, - has_bias=True) - self.check_forward_backward(shape=[1, 128, 256, 256], - begin_norm_axis=3, - has_scale=True, - has_bias=True) - self.check_forward_backward(shape=[1, 256, 384], - begin_norm_axis=2, - has_scale=True, - has_bias=True) + self.check_forward_backward( + shape=[92, 513, 1134], begin_norm_axis=2, y_grad_scale=0.1 + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=False, + has_bias=True, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=True, + has_bias=False, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=False, + has_bias=False, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[512, 1024], begin_norm_axis=1, has_scale=True, has_bias=True + ) + self.check_forward_backward( + shape=[1, 128, 256, 256], + begin_norm_axis=3, + has_scale=True, + has_bias=True, + ) + self.check_forward_backward( + shape=[1, 256, 384], + begin_norm_axis=2, + has_scale=True, + has_bias=True, + ) class TestLayerNormAPI(unittest.TestCase): - def test_case(self): - x = fluid.layers.data(name='x', - shape=[64, 32, 256], - dtype='float32', - append_batch_size=False) - x = fluid.layers.layer_norm(x, - scale=True, - shift=True, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None) - x = fluid.layers.layer_norm(x, - scale=False, - shift=False, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None) - x = fluid.layers.layer_norm(x, - scale=False, - shift=False, - begin_norm_axis=1, - epsilon=1e-05, - param_attr="scale", - bias_attr="shift") + x = fluid.layers.data( + name='x', + shape=[64, 32, 256], + dtype='float32', + append_batch_size=False, + ) + x = fluid.layers.layer_norm( + x, + scale=True, + shift=True, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + ) + x = fluid.layers.layer_norm( + x, + scale=False, + shift=False, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + ) + x = fluid.layers.layer_norm( + x, + scale=False, + shift=False, + begin_norm_axis=1, + epsilon=1e-05, + param_attr="scale", + bias_attr="shift", + ) class TestDygraphLayerNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): paddle.enable_static() @@ -344,7 +388,6 @@ class TestDygraphLayerNormAPIError(unittest.TestCase): class TestFP16ScaleBiasLayerNorm(unittest.TestCase): - def check_main(self, x_np, weight_np, bias_np, dtype): paddle.disable_static() @@ -375,9 +418,11 @@ class TestFP16ScaleBiasLayerNorm(unittest.TestCase): bias_np = np.random.random([20]).astype('float16') y_np_1, x_g_np_1, w_g_np_1, b_g_np_1 = self.check_main( - x_np, weight_np, bias_np, 'float16') + x_np, weight_np, bias_np, 'float16' + ) y_np_2, x_g_np_2, w_g_np_2, b_g_np_2 = self.check_main( - x_np, weight_np, bias_np, 'float32') + x_np, weight_np, bias_np, 'float32' + ) def assert_equal(x, y): np.testing.assert_array_equal(x, y) @@ -389,7 +434,6 @@ class TestFP16ScaleBiasLayerNorm(unittest.TestCase): class TestBF16ScaleBiasLayerNorm(unittest.TestCase): - def check_main(self, x_np, weight_np, bias_np, dtype): paddle.disable_static() @@ -416,18 +460,22 @@ class TestBF16ScaleBiasLayerNorm(unittest.TestCase): return y_np, x_g_np, w_g_np, b_g_np def test_main(self): - if (not core.is_compiled_with_cuda()) or ( - core.cudnn_version() < - 8100) or (paddle.device.cuda.get_device_capability()[0] < 8): + if ( + (not core.is_compiled_with_cuda()) + or (core.cudnn_version() < 8100) + or (paddle.device.cuda.get_device_capability()[0] < 8) + ): return x_np = np.random.random([10, 20]).astype('float32') weight_np = np.random.random([20]).astype('float32') bias_np = np.random.random([20]).astype('float32') y_np_1, x_g_np_1, w_g_np_1, b_g_np_1 = self.check_main( - x_np, weight_np, bias_np, 'float32') + x_np, weight_np, bias_np, 'float32' + ) y_np_2, x_g_np_2, w_g_np_2, b_g_np_2 = self.check_main( - x_np, weight_np, bias_np, 'bfloat16') + x_np, weight_np, bias_np, 'bfloat16' + ) def assert_equal(x, y): np.testing.assert_allclose(x, y, rtol=1e-05, atol=0.1) @@ -439,7 +487,6 @@ class TestBF16ScaleBiasLayerNorm(unittest.TestCase): class TestGetSetKeepLayerNormScaleBiasFP32Flag(unittest.TestCase): - def test_main(self): self.assertTrue(_keep_layer_norm_scale_bias_to_fp32()) _keep_layer_norm_scale_bias_to_fp32(False) diff --git a/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py index 8d6dfab7199..6e9e4931465 100644 --- a/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py @@ -23,7 +23,6 @@ import paddle class TestDygraphLayerNormv2(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("layer_norm"): @@ -114,7 +113,6 @@ class TestDygraphLayerNormv2(unittest.TestCase): class TestLayerNormFunction(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("layer_norm"): @@ -162,10 +160,12 @@ class TestLayerNormFunction(unittest.TestCase): y4 = compute_v4(x) np.testing.assert_allclose(y3, y4, rtol=1e-05) - self.assertRaises(ValueError, - paddle.nn.functional.layer_norm, - x=x, - normalized_shape=1.0) + self.assertRaises( + ValueError, + paddle.nn.functional.layer_norm, + x=x, + normalized_shape=1.0, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index 176a8385c3c..14754e347f9 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -36,7 +36,6 @@ from paddle.fluid.framework import _test_eager_guard class LayerTest(unittest.TestCase): - @classmethod def setUpClass(cls): cls.seed = 111 @@ -61,38 +60,36 @@ class LayerTest(unittest.TestCase): paddle.framework.random._manual_program_seed(self.seed) yield - def get_static_graph_result(self, - feed, - fetch_list, - with_lod=False, - force_to_use_cpu=False): + def get_static_graph_result( + self, feed, fetch_list, with_lod=False, force_to_use_cpu=False + ): exe = fluid.Executor(self._get_place(force_to_use_cpu)) exe.run(fluid.default_startup_program()) - return exe.run(fluid.default_main_program(), - feed=feed, - fetch_list=fetch_list, - return_numpy=(not with_lod)) + return exe.run( + fluid.default_main_program(), + feed=feed, + fetch_list=fetch_list, + return_numpy=(not with_lod), + ) @contextlib.contextmanager def dynamic_graph(self, force_to_use_cpu=False): with fluid.dygraph.guard( - self._get_place(force_to_use_cpu=force_to_use_cpu)): + self._get_place(force_to_use_cpu=force_to_use_cpu) + ): paddle.seed(self.seed) paddle.framework.random._manual_program_seed(self.seed) yield class TestLayer(LayerTest): - def test_custom_layer_with_kwargs(self): - class CustomLayer(fluid.Layer): - def __init__(self, input_size, linear1_size=4): super(CustomLayer, self).__init__() - self.linear1 = nn.Linear(input_size, - linear1_size, - bias_attr=False) + self.linear1 = nn.Linear( + input_size, linear1_size, bias_attr=False + ) self.linear2 = nn.Linear(linear1_size, 1, bias_attr=False) def forward(self, x, do_linear2=False): @@ -121,37 +118,37 @@ class TestLayer(LayerTest): def test_dropout(self): inp = np.ones([3, 32, 32], dtype='float32') with self.static_graph(): - t = layers.data(name='data', - shape=[3, 32, 32], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 32, 32], + dtype='float32', + append_batch_size=False, + ) dropout = nn.Dropout(p=0.35, seed=1, is_test=False) ret = dropout(t) - ret2 = fluid.layers.dropout(t, - dropout_prob=0.35, - seed=1, - is_test=False) + ret2 = fluid.layers.dropout( + t, dropout_prob=0.35, seed=1, is_test=False + ) static_ret, static_ret2 = self.get_static_graph_result( - feed={'data': inp}, fetch_list=[ret, ret2]) + feed={'data': inp}, fetch_list=[ret, ret2] + ) with self.dynamic_graph(): with _test_eager_guard(): t = base.to_variable(inp) dropout = nn.Dropout(p=0.35, seed=1, is_test=False) dy_eager_ret = dropout(t) - dy_eager_ret2 = fluid.layers.dropout(t, - dropout_prob=0.35, - seed=1, - is_test=False) + dy_eager_ret2 = fluid.layers.dropout( + t, dropout_prob=0.35, seed=1, is_test=False + ) dy_eager_ret_value = dy_eager_ret.numpy() dy_eager_ret2_value = dy_eager_ret2.numpy() t = base.to_variable(inp) dropout = nn.Dropout(p=0.35, seed=1, is_test=False) dy_ret = dropout(t) - dy_ret2 = fluid.layers.dropout(t, - dropout_prob=0.35, - seed=1, - is_test=False) + dy_ret2 = fluid.layers.dropout( + t, dropout_prob=0.35, seed=1, is_test=False + ) dy_ret_value = dy_ret.numpy() dy_ret2_value = dy_ret2.numpy() @@ -165,28 +162,34 @@ class TestLayer(LayerTest): def test_linear(self): inp = np.ones([3, 32, 32], dtype='float32') with self.static_graph(): - t = layers.data(name='data', - shape=[3, 32, 32], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 32, 32], + dtype='float32', + append_batch_size=False, + ) linear = nn.Linear( - 32, 4, bias_attr=fluid.initializer.ConstantInitializer(value=1)) + 32, 4, bias_attr=fluid.initializer.ConstantInitializer(value=1) + ) ret = linear(t) - static_ret = self.get_static_graph_result(feed={'data': inp}, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={'data': inp}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): t = base.to_variable(inp) linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) dy_eager_ret = linear(t) dy_eager_ret_value = dy_eager_ret.numpy() t = base.to_variable(inp) linear = nn.Linear( - 32, 4, bias_attr=fluid.initializer.ConstantInitializer(value=1)) + 32, 4, bias_attr=fluid.initializer.ConstantInitializer(value=1) + ) dy_ret = linear(t) dy_ret_value = dy_ret.numpy() @@ -201,7 +204,8 @@ class TestLayer(LayerTest): linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) linear_ret1 = linear(inp) self.assertRaises(TypeError, test_Variable) @@ -213,7 +217,8 @@ class TestLayer(LayerTest): linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) linear_ret2 = linear(inp) self.assertRaises(TypeError, test_type) @@ -221,14 +226,17 @@ class TestLayer(LayerTest): def test_Flatten(self): inp = np.ones([3, 4, 4, 5], dtype='float32') with self.static_graph(): - t = layers.data(name='data', - shape=[3, 4, 4, 5], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 4, 4, 5], + dtype='float32', + append_batch_size=False, + ) flatten = nn.Flatten() ret = flatten(t) - static_ret = self.get_static_graph_result(feed={'data': inp}, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={'data': inp}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): t = base.to_variable(inp) @@ -252,7 +260,8 @@ class TestLayer(LayerTest): linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) linear_ret1 = linear(inp) self.assertRaises(TypeError, test_Variable) @@ -264,7 +273,8 @@ class TestLayer(LayerTest): linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) linear_ret2 = linear(inp) self.assertRaises(TypeError, test_type) @@ -272,41 +282,51 @@ class TestLayer(LayerTest): def test_layer_norm(self): inp = np.ones([3, 32, 32], dtype='float32') with self.static_graph(): - t = layers.data(name='data', - shape=[3, 32, 32], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 32, 32], + dtype='float32', + append_batch_size=False, + ) ret = layers.layer_norm( t, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') - static_ret = self.get_static_graph_result(feed={'data': inp}, - fetch_list=[ret])[0] + act='sigmoid', + ) + static_ret = self.get_static_graph_result( + feed={'data': inp}, fetch_list=[ret] + )[0] with self.static_graph(): - t = layers.data(name='data', - shape=[3, 32, 32], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 32, 32], + dtype='float32', + append_batch_size=False, + ) lm = nn.LayerNorm( normalized_shape=[32, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) ret = lm(t) - static_ret2 = self.get_static_graph_result(feed={'data': inp}, - fetch_list=[ret])[0] + static_ret2 = self.get_static_graph_result( + feed={'data': inp}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): lm = nn.LayerNorm( normalized_shape=[32, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) dy_eager_ret = lm(base.to_variable(inp)) dy_eager_ret_value = dy_eager_ret.numpy() lm = nn.LayerNorm( normalized_shape=[32, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) dy_ret = lm(base.to_variable(inp)) dy_ret_value = dy_ret.numpy() @@ -318,7 +338,8 @@ class TestLayer(LayerTest): scale=False, param_attr=fluid.initializer.ConstantInitializer(value=1), bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) lm(base.to_variable(inp)) self.assertFalse(hasattr(lm, "_scale_w")) @@ -330,7 +351,8 @@ class TestLayer(LayerTest): scale=False, param_attr=fluid.initializer.ConstantInitializer(value=1), bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) lm(base.to_variable(inp)) self.assertFalse(hasattr(lm, "_scale_w")) @@ -345,14 +367,16 @@ class TestLayer(LayerTest): lm = nn.LayerNorm( normalized_shape=[16, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) with self.assertRaises(ValueError): lm(base.to_variable(inp)) lm = nn.LayerNorm( normalized_shape=[16, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) with self.assertRaises(ValueError): lm(base.to_variable(inp)) @@ -364,7 +388,8 @@ class TestLayer(LayerTest): ret = my_sync_bn(t) static_ret = self.get_static_graph_result( feed={'t': np.ones([3, 3, 5, 5], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -385,8 +410,8 @@ class TestLayer(LayerTest): t = layers.data(name='t', shape=[3, 3], dtype='float32') ret = layers.relu(t) static_ret = self.get_static_graph_result( - feed={'t': np.ones([3, 3], - dtype='float32')}, fetch_list=[ret])[0] + feed={'t': np.ones([3, 3], dtype='float32')}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -406,20 +431,21 @@ class TestLayer(LayerTest): t = layers.data(name='t', shape=[3, 3], dtype='float32') t2 = layers.data(name='t2', shape=[3, 3], dtype='float32') ret = layers.matmul(t, t2) - static_ret = self.get_static_graph_result(feed={ - 't': - np.ones([3, 3], dtype='float32'), - 't2': - np.ones([3, 3], dtype='float32') - }, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={ + 't': np.ones([3, 3], dtype='float32'), + 't2': np.ones([3, 3], dtype='float32'), + }, + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): t = np.ones([3, 3], dtype='float32') t2 = np.ones([3, 3], dtype='float32') - dy_eager_ret = layers.matmul(base.to_variable(t), - base.to_variable(t2)) + dy_eager_ret = layers.matmul( + base.to_variable(t), base.to_variable(t2) + ) dy_eager_ret_value = dy_eager_ret.numpy() t = np.ones([3, 3], dtype='float32') @@ -436,49 +462,55 @@ class TestLayer(LayerTest): ret = layers.conv2d(input=images, num_filters=3, filter_size=[2, 2]) static_ret = self.get_static_graph_result( feed={'pixel': np.ones([2, 3, 5, 5], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.static_graph(): images = layers.data(name='pixel', shape=[3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) ret = conv2d(images) static_ret2 = self.get_static_graph_result( feed={'pixel': np.ones([2, 3, 5, 5], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) dy_eager_ret = conv2d(base.to_variable(images)) dy_eager_ret_value = dy_eager_ret.numpy() images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) dy_ret = conv2d(base.to_variable(images)) dy_ret_value = dy_ret.numpy() with self.dynamic_graph(): with _test_eager_guard(): images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2], - bias_attr=False) + conv2d = nn.Conv2D( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + bias_attr=False, + ) dy_ret = conv2d(base.to_variable(images)) self.assertTrue(conv2d.bias is None) images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2], - bias_attr=False) + conv2d = nn.Conv2D( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + bias_attr=False, + ) dy_ret = conv2d(base.to_variable(images)) self.assertTrue(conv2d.bias is None) @@ -486,9 +518,9 @@ class TestLayer(LayerTest): # the input of Conv2D must be Variable. def test_Variable(): images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) conv2d_ret1 = conv2d(images) self.assertRaises(TypeError, test_Variable) @@ -496,12 +528,12 @@ class TestLayer(LayerTest): # the input dtype of Conv2D must be float16 or float32 or float64 # float16 only can be set on GPU place def test_type(): - images = layers.data(name='pixel', - shape=[3, 5, 5], - dtype='int32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + images = layers.data( + name='pixel', shape=[3, 5, 5], dtype='int32' + ) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) conv2d_ret2 = conv2d(images) self.assertRaises(TypeError, test_type) @@ -516,26 +548,33 @@ class TestLayer(LayerTest): custom_weight = np.random.randn(3, 3, 2, 2).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - conv2d1 = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) - conv2d2 = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2], - param_attr=weight_attr) + custom_weight + ) + ) + conv2d1 = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) + conv2d2 = nn.Conv2D( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + param_attr=weight_attr, + ) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) conv2d1_weight_np = conv2d1.weight.numpy() conv2d1_bias = conv2d1.bias self.assertFalse( - np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy())) + np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy()) + ) conv2d2.weight.set_value(conv2d1_weight_np) - np.testing.assert_array_equal(conv2d1_weight_np, - conv2d2.weight.numpy()) + np.testing.assert_array_equal( + conv2d1_weight_np, conv2d2.weight.numpy() + ) conv2d2.bias.set_value(conv2d1_bias) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) @@ -543,22 +582,29 @@ class TestLayer(LayerTest): conv2d2.weight = conv2d1.weight conv2d2.bias = conv2d1.bias - np.testing.assert_array_equal(conv2d1.weight.numpy(), - conv2d2.weight.numpy()) - np.testing.assert_array_equal(conv2d1.bias.numpy(), - conv2d2.bias.numpy()) + np.testing.assert_array_equal( + conv2d1.weight.numpy(), conv2d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv2d1.bias.numpy(), conv2d2.bias.numpy() + ) images = np.ones([2, 3, 5, 5], dtype='float32') custom_weight = np.random.randn(3, 3, 2, 2).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) - conv2d1 = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) - conv2d2 = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2], - param_attr=weight_attr) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) + conv2d1 = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) + conv2d2 = nn.Conv2D( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + param_attr=weight_attr, + ) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) @@ -566,10 +612,12 @@ class TestLayer(LayerTest): conv2d1_weight_np = conv2d1.weight.numpy() conv2d1_bias = conv2d1.bias self.assertFalse( - np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy())) + np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy()) + ) conv2d2.weight.set_value(conv2d1_weight_np) - np.testing.assert_array_equal(conv2d1_weight_np, - conv2d2.weight.numpy()) + np.testing.assert_array_equal( + conv2d1_weight_np, conv2d2.weight.numpy() + ) conv2d2.bias.set_value(conv2d1_bias) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) @@ -577,10 +625,12 @@ class TestLayer(LayerTest): conv2d2.weight = conv2d1.weight conv2d2.bias = conv2d1.bias - np.testing.assert_array_equal(conv2d1.weight.numpy(), - conv2d2.weight.numpy()) - np.testing.assert_array_equal(conv2d1.bias.numpy(), - conv2d2.bias.numpy()) + np.testing.assert_array_equal( + conv2d1.weight.numpy(), conv2d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv2d1.bias.numpy(), conv2d2.bias.numpy() + ) def test_gru_unit(self): lod = [[2, 4, 3]] @@ -595,114 +645,132 @@ class TestLayer(LayerTest): x = layers.data(name='x', shape=[-1, D * 3], dtype='float32') hidden = layers.data(name='hidden', shape=[-1, D], dtype='float32') updated_hidden, reset_hidden_pre, gate = layers.gru_unit( - input=x, hidden=hidden, size=D * 3) + input=x, hidden=hidden, size=D * 3 + ) static_ret = self.get_static_graph_result( - feed={ - 'x': input, - 'hidden': hidden_input - }, - fetch_list=[updated_hidden, reset_hidden_pre, gate]) + feed={'x': input, 'hidden': hidden_input}, + fetch_list=[updated_hidden, reset_hidden_pre, gate], + ) with self.static_graph(): x = layers.data(name='x', shape=[-1, D * 3], dtype='float32') hidden = layers.data(name='hidden', shape=[-1, D], dtype='float32') updated_hidden, reset_hidden_pre, gate = layers.gru_unit( - input=x, hidden=hidden, size=D * 3) + input=x, hidden=hidden, size=D * 3 + ) gru = nn.GRUUnit(size=D * 3) updated_hidden, reset_hidden_pre, gate = gru(x, hidden) static_ret2 = self.get_static_graph_result( - feed={ - 'x': input, - 'hidden': hidden_input - }, - fetch_list=[updated_hidden, reset_hidden_pre, gate]) + feed={'x': input, 'hidden': hidden_input}, + fetch_list=[updated_hidden, reset_hidden_pre, gate], + ) with self.dynamic_graph(): with _test_eager_guard(): gru = nn.GRUUnit(size=D * 3) - dy_eager_ret = gru(base.to_variable(input), - base.to_variable(hidden_input)) + dy_eager_ret = gru( + base.to_variable(input), base.to_variable(hidden_input) + ) dy_eager_ret_value = [] for i in range(len(static_ret)): dy_eager_ret_value.append(dy_eager_ret[i].numpy()) gru = nn.GRUUnit(size=D * 3) - dy_ret = gru(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret = gru( + base.to_variable(input), base.to_variable(hidden_input) + ) dy_ret_value = [] for i in range(len(static_ret)): dy_ret_value.append(dy_ret[i].numpy()) for i in range(len(static_ret)): - np.testing.assert_allclose(static_ret[i], - static_ret2[i], - rtol=1e-05) - np.testing.assert_allclose(static_ret[i], - dy_ret_value[i], - rtol=1e-05) - np.testing.assert_allclose(static_ret[i], - dy_eager_ret_value[i], - rtol=1e-05) + np.testing.assert_allclose( + static_ret[i], static_ret2[i], rtol=1e-05 + ) + np.testing.assert_allclose( + static_ret[i], dy_ret_value[i], rtol=1e-05 + ) + np.testing.assert_allclose( + static_ret[i], dy_eager_ret_value[i], rtol=1e-05 + ) with self.dynamic_graph(): with _test_eager_guard(): custom_weight = np.random.randn(D, D * 3).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) + custom_weight + ) + ) gru1 = nn.GRUUnit(size=D * 3) gru2 = nn.GRUUnit(size=D * 3, param_attr=weight_attr) - dy_ret1 = gru1(base.to_variable(input), - base.to_variable(hidden_input)) - dy_ret2 = gru2(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret1 = gru1( + base.to_variable(input), base.to_variable(hidden_input) + ) + dy_ret2 = gru2( + base.to_variable(input), base.to_variable(hidden_input) + ) self.assertFalse( - np.array_equal(gru1.weight.numpy(), gru2.weight.numpy())) + np.array_equal(gru1.weight.numpy(), gru2.weight.numpy()) + ) for o1, o2 in zip(dy_ret1, dy_ret2): self.assertFalse(np.array_equal(o1.numpy(), o2.numpy())) gru2.weight.set_value(gru1.weight.numpy()) gru2.bias.set_value(gru1.bias) - dy_ret1 = gru1(base.to_variable(input), - base.to_variable(hidden_input)) - dy_ret2 = gru2(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret1 = gru1( + base.to_variable(input), base.to_variable(hidden_input) + ) + dy_ret2 = gru2( + base.to_variable(input), base.to_variable(hidden_input) + ) for o1, o2 in zip(dy_ret1, dy_ret2): np.testing.assert_array_equal(o1.numpy(), o2.numpy()) gru2.weight = gru1.weight gru2.bias = gru1.bias - np.testing.assert_array_equal(gru1.weight.numpy(), - gru2.weight.numpy()) - np.testing.assert_array_equal(gru1.bias.numpy(), - gru2.bias.numpy()) + np.testing.assert_array_equal( + gru1.weight.numpy(), gru2.weight.numpy() + ) + np.testing.assert_array_equal( + gru1.bias.numpy(), gru2.bias.numpy() + ) custom_weight = np.random.randn(D, D * 3).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) gru1 = nn.GRUUnit(size=D * 3) gru2 = nn.GRUUnit(size=D * 3, param_attr=weight_attr) - dy_ret1 = gru1(base.to_variable(input), - base.to_variable(hidden_input)) - dy_ret2 = gru2(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret1 = gru1( + base.to_variable(input), base.to_variable(hidden_input) + ) + dy_ret2 = gru2( + base.to_variable(input), base.to_variable(hidden_input) + ) self.assertFalse( - np.array_equal(gru1.weight.numpy(), gru2.weight.numpy())) + np.array_equal(gru1.weight.numpy(), gru2.weight.numpy()) + ) for o1, o2 in zip(dy_ret1, dy_ret2): self.assertFalse(np.array_equal(o1.numpy(), o2.numpy())) gru2.weight.set_value(gru1.weight.numpy()) gru2.bias.set_value(gru1.bias) - dy_ret1 = gru1(base.to_variable(input), - base.to_variable(hidden_input)) - dy_ret2 = gru2(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret1 = gru1( + base.to_variable(input), base.to_variable(hidden_input) + ) + dy_ret2 = gru2( + base.to_variable(input), base.to_variable(hidden_input) + ) for o1, o2 in zip(dy_ret1, dy_ret2): np.testing.assert_array_equal(o1.numpy(), o2.numpy()) gru2.weight = gru1.weight gru2.bias = gru1.bias - np.testing.assert_array_equal(gru1.weight.numpy(), - gru2.weight.numpy()) + np.testing.assert_array_equal( + gru1.weight.numpy(), gru2.weight.numpy() + ) np.testing.assert_array_equal(gru1.bias.numpy(), gru2.bias.numpy()) def test_elementwise_math(self): @@ -727,15 +795,10 @@ class TestLayer(LayerTest): ret = layers.elementwise_sub(ret, t5) ret = layers.elementwise_mul(ret, t6) - static_ret = self.get_static_graph_result(feed={ - 't': n, - 't2': n2, - 't3': n3, - 't4': n4, - 't5': n5, - 't6': n6 - }, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={'t': n, 't2': n2, 't3': n3, 't4': n4, 't5': n5, 't6': n6}, + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -762,10 +825,12 @@ class TestLayer(LayerTest): with self.dynamic_graph(): with _test_eager_guard(): - min_eager_ret = layers.elementwise_min(to_variable(n), - to_variable(n2)) - max_eager_ret = layers.elementwise_max(to_variable(n), - to_variable(n2)) + min_eager_ret = layers.elementwise_min( + to_variable(n), to_variable(n2) + ) + max_eager_ret = layers.elementwise_max( + to_variable(n), to_variable(n2) + ) min_eager_ret_value = min_eager_ret.numpy() max_eager_ret_value = max_eager_ret.numpy() @@ -786,39 +851,46 @@ class TestLayer(LayerTest): else: place = core.CPUPlace() with self.static_graph(): - seq = layers.data(name='seq_in', - shape=[3, 4], - dtype='float32', - lod_level=1, - append_batch_size=False) + seq = layers.data( + name='seq_in', + shape=[3, 4], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) out = layers.sequence_conv(seq, 2, act='sigmoid') - static_rlt = self.get_static_graph_result(feed={ - "seq_in": - fluid.create_lod_tensor(data=inp_np, - recursive_seq_lens=[[1, 1, 1]], - place=place) - }, - fetch_list=[out], - with_lod=True)[0] + static_rlt = self.get_static_graph_result( + feed={ + "seq_in": fluid.create_lod_tensor( + data=inp_np, recursive_seq_lens=[[1, 1, 1]], place=place + ) + }, + fetch_list=[out], + with_lod=True, + )[0] with self.static_graph(): - seq = layers.data(name='seq_in', - shape=[3, 4], - dtype='float32', - lod_level=1, - append_batch_size=False) + seq = layers.data( + name='seq_in', + shape=[3, 4], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) seq_conv = nn.SequenceConv('seq_conv', num_filters=2, act='sigmoid') out = seq_conv(seq) - static_rlt2 = self.get_static_graph_result(feed={ - "seq_in": - fluid.create_lod_tensor(data=inp_np, - recursive_seq_lens=[[1, 1, 1]], - place=place) - }, - fetch_list=[out], - with_lod=True)[0] - np.testing.assert_array_equal(np.array(static_rlt), - np.array(static_rlt2)) + static_rlt2 = self.get_static_graph_result( + feed={ + "seq_in": fluid.create_lod_tensor( + data=inp_np, recursive_seq_lens=[[1, 1, 1]], place=place + ) + }, + fetch_list=[out], + with_lod=True, + )[0] + np.testing.assert_array_equal( + np.array(static_rlt), np.array(static_rlt2) + ) def test_conv2d_transpose(self): inp_np = np.arange(0, 24).reshape([2, 3, 2, 2]).astype('float32') @@ -829,9 +901,11 @@ class TestLayer(LayerTest): num_filters=10, filter_size=27, act='sigmoid', - bias_attr=fluid.initializer.ConstantInitializer(value=1)) - static_rlt = self.get_static_graph_result(feed={'pixel': inp_np}, - fetch_list=[out])[0] + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) + static_rlt = self.get_static_graph_result( + feed={'pixel': inp_np}, fetch_list=[out] + )[0] with self.static_graph(): img = layers.data(name='pixel', shape=[3, 2, 2], dtype='float32') conv2d_transpose = nn.Conv2DTranspose( @@ -839,10 +913,12 @@ class TestLayer(LayerTest): num_filters=10, filter_size=27, act='sigmoid', - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) out = conv2d_transpose(img) - static_rlt2 = self.get_static_graph_result(feed={'pixel': inp_np}, - fetch_list=[out])[0] + static_rlt2 = self.get_static_graph_result( + feed={'pixel': inp_np}, fetch_list=[out] + )[0] with self.dynamic_graph(): with _test_eager_guard(): conv2d_transpose = nn.Conv2DTranspose( @@ -850,7 +926,8 @@ class TestLayer(LayerTest): num_filters=10, filter_size=27, act='sigmoid', - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) dy_eager_rlt = conv2d_transpose(base.to_variable(inp_np)) dy_eager_rlt_value = dy_eager_rlt.numpy() @@ -859,7 +936,8 @@ class TestLayer(LayerTest): num_filters=10, filter_size=27, act='sigmoid', - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) dy_rlt = conv2d_transpose(base.to_variable(inp_np)) dy_rlt_value = dy_rlt.numpy() np.testing.assert_allclose(static_rlt2, static_rlt, rtol=1e-05) @@ -872,26 +950,33 @@ class TestLayer(LayerTest): custom_weight = np.random.randn(3, 3, 2, 2).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - conv2d1 = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2]) - conv2d2 = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2], - param_attr=weight_attr) + custom_weight + ) + ) + conv2d1 = nn.Conv2DTranspose( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) + conv2d2 = nn.Conv2DTranspose( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + param_attr=weight_attr, + ) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) conv2d1_weight_np = conv2d1.weight.numpy() conv2d1_bias = conv2d1.bias self.assertFalse( - np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy())) + np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy()) + ) conv2d2.weight.set_value(conv2d1_weight_np) - np.testing.assert_array_equal(conv2d1_weight_np, - conv2d2.weight.numpy()) + np.testing.assert_array_equal( + conv2d1_weight_np, conv2d2.weight.numpy() + ) conv2d2.bias.set_value(conv2d1_bias) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) @@ -899,22 +984,29 @@ class TestLayer(LayerTest): conv2d2.weight = conv2d1.weight conv2d2.bias = conv2d1.bias - np.testing.assert_array_equal(conv2d1.weight.numpy(), - conv2d2.weight.numpy()) - np.testing.assert_array_equal(conv2d1.bias.numpy(), - conv2d2.bias.numpy()) + np.testing.assert_array_equal( + conv2d1.weight.numpy(), conv2d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv2d1.bias.numpy(), conv2d2.bias.numpy() + ) images = np.ones([2, 3, 5, 5], dtype='float32') custom_weight = np.random.randn(3, 3, 2, 2).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) - conv2d1 = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2]) - conv2d2 = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2], - param_attr=weight_attr) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) + conv2d1 = nn.Conv2DTranspose( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) + conv2d2 = nn.Conv2DTranspose( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + param_attr=weight_attr, + ) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) @@ -922,10 +1014,12 @@ class TestLayer(LayerTest): conv2d1_weight_np = conv2d1.weight.numpy() conv2d1_bias = conv2d1.bias self.assertFalse( - np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy())) + np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy()) + ) conv2d2.weight.set_value(conv2d1_weight_np) - np.testing.assert_array_equal(conv2d1_weight_np, - conv2d2.weight.numpy()) + np.testing.assert_array_equal( + conv2d1_weight_np, conv2d2.weight.numpy() + ) conv2d2.bias.set_value(conv2d1_bias) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) @@ -933,19 +1027,21 @@ class TestLayer(LayerTest): conv2d2.weight = conv2d1.weight conv2d2.bias = conv2d1.bias - np.testing.assert_array_equal(conv2d1.weight.numpy(), - conv2d2.weight.numpy()) - np.testing.assert_array_equal(conv2d1.bias.numpy(), - conv2d2.bias.numpy()) + np.testing.assert_array_equal( + conv2d1.weight.numpy(), conv2d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv2d1.bias.numpy(), conv2d2.bias.numpy() + ) with self.static_graph(): # the input of Conv2DTranspose must be Variable. def test_Variable(): images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2DTranspose( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) conv2d_ret1 = conv2d(images) self.assertRaises(TypeError, test_Variable) @@ -953,12 +1049,12 @@ class TestLayer(LayerTest): # the input dtype of Conv2DTranspose must be float16 or float32 or float64 # float16 only can be set on GPU place def test_type(): - images = layers.data(name='pixel', - shape=[3, 5, 5], - dtype='int32') - conv2d = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + images = layers.data( + name='pixel', shape=[3, 5, 5], dtype='int32' + ) + conv2d = nn.Conv2DTranspose( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) conv2d_ret2 = conv2d(images) self.assertRaises(TypeError, test_type) @@ -968,48 +1064,42 @@ class TestLayer(LayerTest): inp_np_y = np.array([[4, 5, 6]]).astype('float32') with self.static_graph(): - data_x = layers.data(name='x', - shape=[1, 3], - dtype="float32", - append_batch_size=False) - data_y = layers.data(name='y', - shape=[1, 3], - dtype="float32", - append_batch_size=False) + data_x = layers.data( + name='x', shape=[1, 3], dtype="float32", append_batch_size=False + ) + data_y = layers.data( + name='y', shape=[1, 3], dtype="float32", append_batch_size=False + ) out = layers.bilinear_tensor_product( data_x, data_y, 6, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) - static_rlt = self.get_static_graph_result(feed={ - 'x': inp_np_x, - 'y': inp_np_y - }, - fetch_list=[out])[0] + static_rlt = self.get_static_graph_result( + feed={'x': inp_np_x, 'y': inp_np_y}, fetch_list=[out] + )[0] with self.static_graph(): - data_x = layers.data(name='x', - shape=[1, 3], - dtype="float32", - append_batch_size=False) - data_y = layers.data(name='y', - shape=[1, 3], - dtype="float32", - append_batch_size=False) + data_x = layers.data( + name='x', shape=[1, 3], dtype="float32", append_batch_size=False + ) + data_y = layers.data( + name='y', shape=[1, 3], dtype="float32", append_batch_size=False + ) btp = nn.BilinearTensorProduct( 3, 3, 6, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) out = btp(data_x, data_y) - static_rlt2 = self.get_static_graph_result(feed={ - 'x': inp_np_x, - 'y': inp_np_y - }, - fetch_list=[out])[0] + static_rlt2 = self.get_static_graph_result( + feed={'x': inp_np_x, 'y': inp_np_y}, fetch_list=[out] + )[0] with self.dynamic_graph(): with _test_eager_guard(): btp = nn.BilinearTensorProduct( @@ -1017,9 +1107,11 @@ class TestLayer(LayerTest): 3, 6, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') - dy_eager_rlt = btp(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + act='sigmoid', + ) + dy_eager_rlt = btp( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) dy_eager_rlt_value = dy_eager_rlt.numpy() btp = nn.BilinearTensorProduct( @@ -1027,41 +1119,39 @@ class TestLayer(LayerTest): 3, 6, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) dy_rlt = btp(base.to_variable(inp_np_x), base.to_variable(inp_np_y)) dy_rlt_value = dy_rlt.numpy() with self.dynamic_graph(): with _test_eager_guard(): btp2 = nn.BilinearTensorProduct(3, 3, 6, act='sigmoid') - dy_eager_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + dy_eager_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) dy_eager_rlt2_value = dy_eager_rlt2.numpy() btp2 = nn.BilinearTensorProduct(3, 3, 6, act='sigmoid') - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) dy_rlt2_value = dy_rlt2.numpy() with self.static_graph(): - data_x2 = layers.data(name='x', - shape=[1, 3], - dtype="float32", - append_batch_size=False) - data_y2 = layers.data(name='y', - shape=[1, 3], - dtype="float32", - append_batch_size=False) - out2 = layers.bilinear_tensor_product(data_x2, - data_y2, - 6, - act='sigmoid') - - static_rlt3 = self.get_static_graph_result(feed={ - 'x': inp_np_x, - 'y': inp_np_y - }, - fetch_list=[out2])[0] + data_x2 = layers.data( + name='x', shape=[1, 3], dtype="float32", append_batch_size=False + ) + data_y2 = layers.data( + name='y', shape=[1, 3], dtype="float32", append_batch_size=False + ) + out2 = layers.bilinear_tensor_product( + data_x2, data_y2, 6, act='sigmoid' + ) + + static_rlt3 = self.get_static_graph_result( + feed={'x': inp_np_x, 'y': inp_np_y}, fetch_list=[out2] + )[0] np.testing.assert_array_equal(dy_rlt2_value, static_rlt3) np.testing.assert_array_equal(dy_eager_rlt2_value, static_rlt3) @@ -1074,87 +1164,108 @@ class TestLayer(LayerTest): custom_weight = np.random.randn(6, 3, 3).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) + custom_weight + ) + ) btp1 = nn.BilinearTensorProduct(3, 3, 6, act='sigmoid') - btp2 = nn.BilinearTensorProduct(3, - 3, - 6, - act='sigmoid', - param_attr=weight_attr) - dy_rlt1 = btp1(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + btp2 = nn.BilinearTensorProduct( + 3, 3, 6, act='sigmoid', param_attr=weight_attr + ) + dy_rlt1 = btp1( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) self.assertFalse( - np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy())) + np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) + ) btp2.weight.set_value(btp1.weight.numpy()) btp2.bias.set_value(btp1.bias) - dy_rlt1 = btp1(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + dy_rlt1 = btp1( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) np.testing.assert_array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) btp2.weight = btp1.weight btp2.bias = btp1.bias - np.testing.assert_array_equal(btp1.weight.numpy(), - btp2.weight.numpy()) - np.testing.assert_array_equal(btp1.bias.numpy(), - btp2.bias.numpy()) + np.testing.assert_array_equal( + btp1.weight.numpy(), btp2.weight.numpy() + ) + np.testing.assert_array_equal( + btp1.bias.numpy(), btp2.bias.numpy() + ) custom_weight = np.random.randn(6, 3, 3).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) btp1 = nn.BilinearTensorProduct(3, 3, 6, act='sigmoid') - btp2 = nn.BilinearTensorProduct(3, - 3, - 6, - act='sigmoid', - param_attr=weight_attr) - dy_rlt1 = btp1(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + btp2 = nn.BilinearTensorProduct( + 3, 3, 6, act='sigmoid', param_attr=weight_attr + ) + dy_rlt1 = btp1( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) self.assertFalse(np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy())) btp2.weight.set_value(btp1.weight.numpy()) btp2.bias.set_value(btp1.bias) - dy_rlt1 = btp1(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + dy_rlt1 = btp1( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) np.testing.assert_array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) btp2.weight = btp1.weight btp2.bias = btp1.bias - np.testing.assert_array_equal(btp1.weight.numpy(), - btp2.weight.numpy()) + np.testing.assert_array_equal( + btp1.weight.numpy(), btp2.weight.numpy() + ) np.testing.assert_array_equal(btp1.bias.numpy(), btp2.bias.numpy()) def prelu_test(self, mode): inp_np = np.ones([5, 200, 100, 100]).astype('float32') with self.static_graph(): - data_t = layers.data(name="input", - shape=[5, 200, 100, 100], - dtype="float32", - append_batch_size=False) - out = layers.prelu(data_t, - mode, - param_attr=ParamAttr(initializer=Constant(1.0))) - static_rlt = self.get_static_graph_result(feed={"input": inp_np}, - fetch_list=[out])[0] + data_t = layers.data( + name="input", + shape=[5, 200, 100, 100], + dtype="float32", + append_batch_size=False, + ) + out = layers.prelu( + data_t, mode, param_attr=ParamAttr(initializer=Constant(1.0)) + ) + static_rlt = self.get_static_graph_result( + feed={"input": inp_np}, fetch_list=[out] + )[0] with self.static_graph(): - data_t = layers.data(name="input", - shape=[5, 200, 100, 100], - dtype="float32", - append_batch_size=False) - prelu = nn.PRelu(mode=mode, - channel=inp_np.shape[1], - input_shape=data_t.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + data_t = layers.data( + name="input", + shape=[5, 200, 100, 100], + dtype="float32", + append_batch_size=False, + ) + prelu = nn.PRelu( + mode=mode, + channel=inp_np.shape[1], + input_shape=data_t.shape, + param_attr=ParamAttr(initializer=Constant(1.0)), + ) out = prelu(data_t) - static_rlt2 = self.get_static_graph_result(feed={"input": inp_np}, - fetch_list=[out])[0] + static_rlt2 = self.get_static_graph_result( + feed={"input": inp_np}, fetch_list=[out] + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -1162,14 +1273,17 @@ class TestLayer(LayerTest): mode=mode, channel=inp_np.shape[1], input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + param_attr=ParamAttr(initializer=Constant(1.0)), + ) dy_eager_rlt = prelu(base.to_variable(inp_np)) dy_eager_rlt_value = dy_eager_rlt.numpy() - prelu = nn.PRelu(mode=mode, - channel=inp_np.shape[1], - input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + prelu = nn.PRelu( + mode=mode, + channel=inp_np.shape[1], + input_shape=inp_np.shape, + param_attr=ParamAttr(initializer=Constant(1.0)), + ) dy_rlt = prelu(base.to_variable(inp_np)) dy_rlt_value = dy_rlt.numpy() @@ -1185,42 +1299,51 @@ class TestLayer(LayerTest): mode=mode, channel=inp_np.shape[1], input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(2.0))) + param_attr=ParamAttr(initializer=Constant(2.0)), + ) prelu2 = nn.PRelu( mode=mode, channel=inp_np.shape[1], input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + param_attr=ParamAttr(initializer=Constant(1.0)), + ) dy_rlt1 = prelu1(inp) dy_rlt2 = prelu2(inp) self.assertFalse( - np.array_equal(prelu1.weight.numpy(), - prelu2.weight.numpy())) + np.array_equal(prelu1.weight.numpy(), prelu2.weight.numpy()) + ) self.assertFalse( - np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy())) + np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) + ) prelu2.weight.set_value(prelu1.weight.numpy()) dy_rlt1 = prelu1(inp) dy_rlt2 = prelu2(inp) np.testing.assert_array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) prelu2.weight = prelu1.weight - np.testing.assert_array_equal(prelu1.weight.numpy(), - prelu2.weight.numpy()) + np.testing.assert_array_equal( + prelu1.weight.numpy(), prelu2.weight.numpy() + ) inp_np = np.random.randn(5, 200, 100, 100).astype("float32") inp = base.to_variable(inp_np) - prelu1 = nn.PRelu(mode=mode, - channel=inp_np.shape[1], - input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(2.0))) - prelu2 = nn.PRelu(mode=mode, - channel=inp_np.shape[1], - input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + prelu1 = nn.PRelu( + mode=mode, + channel=inp_np.shape[1], + input_shape=inp_np.shape, + param_attr=ParamAttr(initializer=Constant(2.0)), + ) + prelu2 = nn.PRelu( + mode=mode, + channel=inp_np.shape[1], + input_shape=inp_np.shape, + param_attr=ParamAttr(initializer=Constant(1.0)), + ) dy_rlt1 = prelu1(inp) dy_rlt2 = prelu2(inp) self.assertFalse( - np.array_equal(prelu1.weight.numpy(), prelu2.weight.numpy())) + np.array_equal(prelu1.weight.numpy(), prelu2.weight.numpy()) + ) self.assertFalse(np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy())) prelu2.weight.set_value(prelu1.weight.numpy()) dy_rlt1 = prelu1(inp) @@ -1228,8 +1351,9 @@ class TestLayer(LayerTest): np.testing.assert_array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) prelu2.weight = prelu1.weight - np.testing.assert_array_equal(prelu1.weight.numpy(), - prelu2.weight.numpy()) + np.testing.assert_array_equal( + prelu1.weight.numpy(), prelu2.weight.numpy() + ) def test_prelu(self): self.prelu_test("channel") @@ -1241,31 +1365,37 @@ class TestLayer(LayerTest): dict_size = 20 with self.static_graph(): data_t = layers.data(name='word', shape=[1], dtype='int64') - emb = layers.embedding(input=data_t, - size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) - static_rlt = self.get_static_graph_result(feed={'word': inp_word}, - fetch_list=[emb])[0] + emb = layers.embedding( + input=data_t, + size=[dict_size, 32], + param_attr='emb.w', + is_sparse=False, + ) + static_rlt = self.get_static_graph_result( + feed={'word': inp_word}, fetch_list=[emb] + )[0] with self.static_graph(): data_t = layers.data(name='word', shape=[1], dtype='int64') - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) emb_rlt = emb2(data_t) - static_rlt2 = self.get_static_graph_result(feed={'word': inp_word}, - fetch_list=[emb_rlt])[0] + static_rlt2 = self.get_static_graph_result( + feed={'word': inp_word}, fetch_list=[emb_rlt] + )[0] with self.dynamic_graph(): with _test_eager_guard(): - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr='eager_emb.w', - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], + param_attr='eager_emb.w', + is_sparse=False, + ) dy_eager_rlt = emb2(base.to_variable(inp_word)) dy_eager_rlt_value = dy_eager_rlt.numpy() - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) dy_rlt = emb2(base.to_variable(inp_word)) dy_rlt_value = dy_rlt.numpy() @@ -1278,33 +1408,43 @@ class TestLayer(LayerTest): custom_weight = np.random.randn(dict_size, 32).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) + custom_weight + ) + ) emb1 = nn.Embedding(size=[dict_size, 32], is_sparse=False) - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr=weight_attr, - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], + param_attr=weight_attr, + is_sparse=False, + ) rep1 = emb1(base.to_variable(inp_word)) rep2 = emb2(base.to_variable(inp_word)) self.assertFalse( - np.array_equal(emb1.weight.numpy(), custom_weight)) - np.testing.assert_array_equal(emb2.weight.numpy(), - custom_weight) + np.array_equal(emb1.weight.numpy(), custom_weight) + ) + np.testing.assert_array_equal( + emb2.weight.numpy(), custom_weight + ) self.assertFalse(np.array_equal(rep1.numpy(), rep2.numpy())) emb2.weight.set_value(emb1.weight.numpy()) rep2 = emb2(base.to_variable(inp_word)) np.testing.assert_array_equal(rep1.numpy(), rep2.numpy()) emb2.weight = emb1.weight - np.testing.assert_array_equal(emb1.weight.numpy(), - emb2.weight.numpy()) + np.testing.assert_array_equal( + emb1.weight.numpy(), emb2.weight.numpy() + ) custom_weight = np.random.randn(dict_size, 32).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) emb1 = nn.Embedding(size=[dict_size, 32], is_sparse=False) - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr=weight_attr, - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], param_attr=weight_attr, is_sparse=False + ) rep1 = emb1(base.to_variable(inp_word)) rep2 = emb2(base.to_variable(inp_word)) self.assertFalse(np.array_equal(emb1.weight.numpy(), custom_weight)) @@ -1315,8 +1455,9 @@ class TestLayer(LayerTest): np.testing.assert_array_equal(rep1.numpy(), rep2.numpy()) emb2.weight = emb1.weight - np.testing.assert_array_equal(emb1.weight.numpy(), - emb2.weight.numpy()) + np.testing.assert_array_equal( + emb1.weight.numpy(), emb2.weight.numpy() + ) def test_nce(self): window_size = 5 @@ -1329,54 +1470,61 @@ class TestLayer(LayerTest): words = [] for i in range(window_size): words.append( - layers.data(name='word_{0}'.format(i), - shape=[None], - dtype='int64')) - sample_weights = layers.fill_constant(shape=[5, 1], - dtype='float32', - value=1) + layers.data( + name='word_{0}'.format(i), shape=[None], dtype='int64' + ) + ) + sample_weights = layers.fill_constant( + shape=[5, 1], dtype='float32', value=1 + ) embs = [] for i in range(window_size): if i == label_word: continue - emb = fluid.embedding(input=words[i], - size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + emb = fluid.embedding( + input=words[i], + size=[dict_size, 32], + param_attr='emb.w', + is_sparse=False, + ) embs.append(emb) embs = layers.concat(input=embs, axis=1) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) - nce_loss = layers.nce(input=embs, - label=wl, - num_total_classes=dict_size, - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='nce.w', - bias_attr='nce.b', - sample_weight=sample_weights) + nce_loss = layers.nce( + input=embs, + label=wl, + num_total_classes=dict_size, + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='nce.w', + bias_attr='nce.b', + sample_weight=sample_weights, + ) feed_dict = dict() for i in range(window_size): feed_dict['word_{0}'.format(i)] = inp_word[i] - static_rlt = self.get_static_graph_result(feed=feed_dict, - fetch_list=[nce_loss])[0] + static_rlt = self.get_static_graph_result( + feed=feed_dict, fetch_list=[nce_loss] + )[0] with self.static_graph(): words = [] for i in range(window_size): words.append( - layers.data(name='word_{0}'.format(i), - shape=[None], - dtype='int64')) - sample_weights = layers.fill_constant(shape=[5, 1], - dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + layers.data( + name='word_{0}'.format(i), shape=[None], dtype='int64' + ) + ) + sample_weights = layers.fill_constant( + shape=[5, 1], dtype='float32', value=1 + ) + emb = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) embs2 = [] for i in range(window_size): @@ -1387,15 +1535,17 @@ class TestLayer(LayerTest): embs2.append(emb_rlt) embs2 = layers.concat(input=embs2, axis=1) - nce = nn.NCE(num_total_classes=dict_size, - dim=embs2.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='nce.w', - bias_attr='nce.b', - sample_weight=sample_weights) + nce = nn.NCE( + num_total_classes=dict_size, + dim=embs2.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='nce.w', + bias_attr='nce.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) nce_loss2 = nce(embs2, wl) @@ -1403,21 +1553,23 @@ class TestLayer(LayerTest): for i in range(len(words)): feed_dict['word_{0}'.format(i)] = inp_word[i] - static_rlt2 = self.get_static_graph_result(feed=feed_dict, - fetch_list=[nce_loss2 - ])[0] + static_rlt2 = self.get_static_graph_result( + feed=feed_dict, fetch_list=[nce_loss2] + )[0] with self.dynamic_graph(): with _test_eager_guard(): words = [] for i in range(window_size): words.append(base.to_variable(inp_word[i])) - sample_weights = layers.fill_constant(shape=[5, 1], - dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='eager_emb.w', - is_sparse=False) + sample_weights = layers.fill_constant( + shape=[5, 1], dtype='float32', value=1 + ) + emb = nn.Embedding( + size=[dict_size, 32], + param_attr='eager_emb.w', + is_sparse=False, + ) embs3 = [] for i in range(window_size): @@ -1427,18 +1579,20 @@ class TestLayer(LayerTest): emb_rlt = emb(words[i]) embs3.append(emb_rlt) - embs3 = layers.concat(input=embs3, - axis=fluid.dygraph.to_variable( - np.array([1]))) - nce = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='eager_nce.w', - bias_attr='eager_nce.b', - sample_weight=sample_weights) + embs3 = layers.concat( + input=embs3, axis=fluid.dygraph.to_variable(np.array([1])) + ) + nce = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='eager_nce.w', + bias_attr='eager_nce.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) dy_eager_rlt = nce(embs3, wl) @@ -1447,12 +1601,12 @@ class TestLayer(LayerTest): words = [] for i in range(window_size): words.append(base.to_variable(inp_word[i])) - sample_weights = layers.fill_constant(shape=[5, 1], - dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + sample_weights = layers.fill_constant( + shape=[5, 1], dtype='float32', value=1 + ) + emb = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) embs3 = [] for i in range(window_size): @@ -1462,17 +1616,20 @@ class TestLayer(LayerTest): emb_rlt = emb(words[i]) embs3.append(emb_rlt) - embs3 = layers.concat(input=embs3, - axis=fluid.dygraph.to_variable(np.array([1]))) - nce = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='nce.w', - bias_attr='nce.b', - sample_weight=sample_weights) + embs3 = layers.concat( + input=embs3, axis=fluid.dygraph.to_variable(np.array([1])) + ) + nce = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='nce.w', + bias_attr='nce.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) dy_rlt = nce(embs3, wl) @@ -1484,21 +1641,27 @@ class TestLayer(LayerTest): with self.dynamic_graph(): with _test_eager_guard(): - custom_weight = np.random.randn(dict_size, - 128).astype("float32") + custom_weight = np.random.randn(dict_size, 128).astype( + "float32" + ) weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) + custom_weight + ) + ) words = [] for i in range(window_size): words.append(base.to_variable(inp_word[i])) sample_weights = layers.fill_constant( shape=fluid.dygraph.to_variable(np.array([5, 1])), dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='eager_emb.w', - is_sparse=False) + value=1, + ) + emb = nn.Embedding( + size=[dict_size, 32], + param_attr='eager_emb.w', + is_sparse=False, + ) embs3 = [] for i in range(window_size): @@ -1509,58 +1672,70 @@ class TestLayer(LayerTest): embs3.append(emb_rlt) embs3 = layers.concat(input=embs3, axis=1) - nce1 = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='eager_nce1.w', - bias_attr='eager_nce1.b', - sample_weight=sample_weights) - - nce2 = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr=weight_attr, - bias_attr='eager_nce2.b', - sample_weight=sample_weights) + nce1 = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='eager_nce1.w', + bias_attr='eager_nce1.b', + sample_weight=sample_weights, + ) + + nce2 = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr=weight_attr, + bias_attr='eager_nce2.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) nce1_loss = nce1(embs3, wl) nce2_loss = nce2(embs3, wl) self.assertFalse( - np.array_equal(nce1_loss.numpy(), nce2_loss.numpy())) + np.array_equal(nce1_loss.numpy(), nce2_loss.numpy()) + ) nce2.weight.set_value(nce1.weight.numpy()) nce2.bias.set_value(nce1.bias) nce1_loss = nce1(embs3, wl) nce2_loss = nce2(embs3, wl) - np.testing.assert_array_equal(nce1_loss.numpy(), - nce2_loss.numpy()) + np.testing.assert_array_equal( + nce1_loss.numpy(), nce2_loss.numpy() + ) nce2.weight = nce1.weight nce2.bias = nce1.bias - np.testing.assert_array_equal(nce1.weight.numpy(), - nce2.weight.numpy()) - np.testing.assert_array_equal(nce1.bias.numpy(), - nce2.bias.numpy()) + np.testing.assert_array_equal( + nce1.weight.numpy(), nce2.weight.numpy() + ) + np.testing.assert_array_equal( + nce1.bias.numpy(), nce2.bias.numpy() + ) custom_weight = np.random.randn(dict_size, 128).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) words = [] for i in range(window_size): words.append(base.to_variable(inp_word[i])) sample_weights = layers.fill_constant( shape=fluid.dygraph.to_variable(np.array([5, 1])), dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + value=1, + ) + emb = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) embs3 = [] for i in range(window_size): @@ -1571,31 +1746,36 @@ class TestLayer(LayerTest): embs3.append(emb_rlt) embs3 = layers.concat(input=embs3, axis=1) - nce1 = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='nce1.w', - bias_attr='nce1.b', - sample_weight=sample_weights) - - nce2 = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr=weight_attr, - bias_attr='nce2.b', - sample_weight=sample_weights) + nce1 = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='nce1.w', + bias_attr='nce1.b', + sample_weight=sample_weights, + ) + + nce2 = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr=weight_attr, + bias_attr='nce2.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) nce1_loss = nce1(embs3, wl) nce2_loss = nce2(embs3, wl) self.assertFalse( - np.array_equal(nce1_loss.numpy(), nce2_loss.numpy())) + np.array_equal(nce1_loss.numpy(), nce2_loss.numpy()) + ) nce2.weight.set_value(nce1.weight.numpy()) nce2.bias.set_value(nce1.bias) nce1_loss = nce1(embs3, wl) @@ -1604,46 +1784,54 @@ class TestLayer(LayerTest): nce2.weight = nce1.weight nce2.bias = nce1.bias - np.testing.assert_array_equal(nce1.weight.numpy(), - nce2.weight.numpy()) + np.testing.assert_array_equal( + nce1.weight.numpy(), nce2.weight.numpy() + ) np.testing.assert_array_equal(nce1.bias.numpy(), nce2.bias.numpy()) def test_one_hot(self): with self.dynamic_graph(): with _test_eager_guard(): - label = fluid.dygraph.to_variable(np.array([[1], [1], [3], - [0]])) + label = fluid.dygraph.to_variable( + np.array([[1], [1], [3], [0]]) + ) one_hot_label1 = fluid.layers.one_hot(input=label, depth=4) one_hot_label2 = fluid.layers.one_hot( - input=label, depth=fluid.dygraph.to_variable(np.array([4]))) - np.testing.assert_array_equal(one_hot_label1.numpy(), - one_hot_label2.numpy()) + input=label, depth=fluid.dygraph.to_variable(np.array([4])) + ) + np.testing.assert_array_equal( + one_hot_label1.numpy(), one_hot_label2.numpy() + ) label = fluid.dygraph.to_variable(np.array([[1], [1], [3], [0]])) one_hot_label1 = fluid.layers.one_hot(input=label, depth=4) one_hot_label2 = fluid.layers.one_hot( - input=label, depth=fluid.dygraph.to_variable(np.array([4]))) - np.testing.assert_array_equal(one_hot_label1.numpy(), - one_hot_label2.numpy()) + input=label, depth=fluid.dygraph.to_variable(np.array([4])) + ) + np.testing.assert_array_equal( + one_hot_label1.numpy(), one_hot_label2.numpy() + ) def test_split(self): with self.dynamic_graph(): with _test_eager_guard(): input = fluid.dygraph.to_variable(np.random.random((3, 8, 5))) x0, x1 = fluid.layers.split(input, num_or_sections=2, dim=1) - x00, x11 = fluid.layers.split(input, - num_or_sections=2, - dim=fluid.dygraph.to_variable( - np.array([1]))) + x00, x11 = fluid.layers.split( + input, + num_or_sections=2, + dim=fluid.dygraph.to_variable(np.array([1])), + ) np.testing.assert_array_equal(x0.numpy(), x00.numpy()) np.testing.assert_array_equal(x1.numpy(), x11.numpy()) input = fluid.dygraph.to_variable(np.random.random((3, 8, 5))) x0, x1 = fluid.layers.split(input, num_or_sections=2, dim=1) - x00, x11 = fluid.layers.split(input, - num_or_sections=2, - dim=fluid.dygraph.to_variable( - np.array([1]))) + x00, x11 = fluid.layers.split( + input, + num_or_sections=2, + dim=fluid.dygraph.to_variable(np.array([1])), + ) np.testing.assert_array_equal(x0.numpy(), x00.numpy()) np.testing.assert_array_equal(x1.numpy(), x11.numpy()) @@ -1653,40 +1841,48 @@ class TestLayer(LayerTest): input = fluid.dygraph.to_variable(np.random.random((13, 11))) top5_values1, top5_indices1 = layers.topk(input, k=5) top5_values2, top5_indices2 = layers.topk( - input, k=fluid.dygraph.to_variable(np.array([5]))) - np.testing.assert_array_equal(top5_values1.numpy(), - top5_values2.numpy()) - np.testing.assert_array_equal(top5_indices1.numpy(), - top5_indices2.numpy()) + input, k=fluid.dygraph.to_variable(np.array([5])) + ) + np.testing.assert_array_equal( + top5_values1.numpy(), top5_values2.numpy() + ) + np.testing.assert_array_equal( + top5_indices1.numpy(), top5_indices2.numpy() + ) input = fluid.dygraph.to_variable(np.random.random((13, 11))) top5_values1, top5_indices1 = layers.topk(input, k=5) top5_values2, top5_indices2 = layers.topk( - input, k=fluid.dygraph.to_variable(np.array([5]))) - np.testing.assert_array_equal(top5_values1.numpy(), - top5_values2.numpy()) - np.testing.assert_array_equal(top5_indices1.numpy(), - top5_indices2.numpy()) + input, k=fluid.dygraph.to_variable(np.array([5])) + ) + np.testing.assert_array_equal( + top5_values1.numpy(), top5_values2.numpy() + ) + np.testing.assert_array_equal( + top5_indices1.numpy(), top5_indices2.numpy() + ) def test_conv3d(self): with self.static_graph(): - images = layers.data(name='pixel', - shape=[3, 6, 6, 6], - dtype='float32') + images = layers.data( + name='pixel', shape=[3, 6, 6, 6], dtype='float32' + ) ret = layers.conv3d(input=images, num_filters=3, filter_size=2) static_ret = self.get_static_graph_result( feed={'pixel': np.ones([2, 3, 6, 6, 6], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.static_graph(): - images = layers.data(name='pixel', - shape=[3, 6, 6, 6], - dtype='float32') + images = layers.data( + name='pixel', shape=[3, 6, 6, 6], dtype='float32' + ) conv3d = nn.Conv3D(num_channels=3, num_filters=3, filter_size=2) ret = conv3d(images) static_ret2 = self.get_static_graph_result( feed={'pixel': np.ones([2, 3, 6, 6, 6], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -1710,26 +1906,33 @@ class TestLayer(LayerTest): custom_weight = np.random.randn(3, 3, 2, 2, 2).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - conv3d1 = nn.Conv3D(num_channels=3, - num_filters=3, - filter_size=2) - conv3d2 = nn.Conv3D(num_channels=3, - num_filters=3, - filter_size=2, - param_attr=weight_attr) + custom_weight + ) + ) + conv3d1 = nn.Conv3D( + num_channels=3, num_filters=3, filter_size=2 + ) + conv3d2 = nn.Conv3D( + num_channels=3, + num_filters=3, + filter_size=2, + param_attr=weight_attr, + ) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) conv3d1_weight_np = conv3d1.weight.numpy() conv3d1_bias = conv3d1.bias self.assertFalse( - np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy())) + np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy()) + ) conv3d2.weight.set_value(conv3d1_weight_np) - np.testing.assert_array_equal(conv3d1_weight_np, - conv3d2.weight.numpy()) + np.testing.assert_array_equal( + conv3d1_weight_np, conv3d2.weight.numpy() + ) conv3d1.bias.set_value(conv3d1_bias) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) @@ -1737,20 +1940,27 @@ class TestLayer(LayerTest): conv3d2.weight = conv3d1.weight conv3d2.bias = conv3d1.bias - np.testing.assert_array_equal(conv3d1.weight.numpy(), - conv3d2.weight.numpy()) - np.testing.assert_array_equal(conv3d1.bias.numpy(), - conv3d2.bias.numpy()) + np.testing.assert_array_equal( + conv3d1.weight.numpy(), conv3d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv3d1.bias.numpy(), conv3d2.bias.numpy() + ) images = np.ones([2, 3, 6, 6, 6], dtype='float32') custom_weight = np.random.randn(3, 3, 2, 2, 2).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) conv3d1 = nn.Conv3D(num_channels=3, num_filters=3, filter_size=2) - conv3d2 = nn.Conv3D(num_channels=3, - num_filters=3, - filter_size=2, - param_attr=weight_attr) + conv3d2 = nn.Conv3D( + num_channels=3, + num_filters=3, + filter_size=2, + param_attr=weight_attr, + ) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) @@ -1758,10 +1968,12 @@ class TestLayer(LayerTest): conv3d1_weight_np = conv3d1.weight.numpy() conv3d1_bias = conv3d1.bias self.assertFalse( - np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy())) + np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy()) + ) conv3d2.weight.set_value(conv3d1_weight_np) - np.testing.assert_array_equal(conv3d1_weight_np, - conv3d2.weight.numpy()) + np.testing.assert_array_equal( + conv3d1_weight_np, conv3d2.weight.numpy() + ) conv3d1.bias.set_value(conv3d1_bias) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) @@ -1769,10 +1981,12 @@ class TestLayer(LayerTest): conv3d2.weight = conv3d1.weight conv3d2.bias = conv3d1.bias - np.testing.assert_array_equal(conv3d1.weight.numpy(), - conv3d2.weight.numpy()) - np.testing.assert_array_equal(conv3d1.bias.numpy(), - conv3d2.bias.numpy()) + np.testing.assert_array_equal( + conv3d1.weight.numpy(), conv3d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv3d1.bias.numpy(), conv3d2.bias.numpy() + ) def test_row_conv(self): input = np.arange(15).reshape([3, 5]).astype('float32') @@ -1782,37 +1996,43 @@ class TestLayer(LayerTest): place = core.CPUPlace() with self.static_graph(): - x = layers.data(name='X', - shape=[3, 5], - dtype='float32', - lod_level=1, - append_batch_size=False) + x = layers.data( + name='X', + shape=[3, 5], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) ret = layers.row_conv(input=x, future_context_size=2) - static_ret = self.get_static_graph_result(feed={ - 'X': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.static_graph(): - x = layers.data(name='X', - shape=[3, 5], - dtype='float32', - lod_level=1, - append_batch_size=False) + x = layers.data( + name='X', + shape=[3, 5], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) rowConv = nn.RowConv('RowConv', future_context_size=2) ret = rowConv(x) - static_ret2 = self.get_static_graph_result(feed={ - 'X': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret2 = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] # TODO: dygraph can't support LODTensor @@ -1829,52 +2049,61 @@ class TestLayer(LayerTest): input = np.random.random(shape).astype('float32') with self.static_graph(): - X = fluid.layers.data(name='X', - shape=shape, - dtype='float32', - lod_level=1, - append_batch_size=False) + X = fluid.layers.data( + name='X', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False, + ) ret = layers.group_norm( input=X, groups=2, param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5), - bias_attr=fluid.initializer.ConstantInitializer(value=1)) - static_ret = self.get_static_graph_result(feed={ - 'X': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) + static_ret = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.static_graph(): - X = fluid.layers.data(name='X', - shape=shape, - dtype='float32', - lod_level=1, - append_batch_size=False) + X = fluid.layers.data( + name='X', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False, + ) groupNorm = nn.GroupNorm( channels=shape[1], groups=2, param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5), - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) ret = groupNorm(X) - static_ret2 = self.get_static_graph_result(feed={ - 'X': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret2 = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.dynamic_graph(): groupNorm = nn.GroupNorm( channels=shape[1], groups=2, param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5), - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) dy_ret = groupNorm(base.to_variable(input)) dy_rlt_value = dy_ret.numpy() @@ -1897,23 +2126,23 @@ class TestLayer(LayerTest): input = np.random.random(shape).astype('float32') with self.static_graph(): - X = fluid.layers.data(name='X', - shape=shape, - dtype='float32', - append_batch_size=False) + X = fluid.layers.data( + name='X', shape=shape, dtype='float32', append_batch_size=False + ) ret = layers.instance_norm(input=X) - static_ret = self.get_static_graph_result(feed={'X': input}, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={'X': input}, fetch_list=[ret] + )[0] with self.static_graph(): - X = fluid.layers.data(name='X', - shape=shape, - dtype='float32', - append_batch_size=False) + X = fluid.layers.data( + name='X', shape=shape, dtype='float32', append_batch_size=False + ) instanceNorm = nn.InstanceNorm(num_channels=shape[1]) ret = instanceNorm(X) - static_ret2 = self.get_static_graph_result(feed={'X': input}, - fetch_list=[ret])[0] + static_ret2 = self.get_static_graph_result( + feed={'X': input}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -1968,37 +2197,43 @@ class TestLayer(LayerTest): input = np.random.random(shape).astype('float32') with self.static_graph(): - Weight = fluid.layers.data(name='Weight', - shape=shape, - dtype='float32', - lod_level=1, - append_batch_size=False) + Weight = fluid.layers.data( + name='Weight', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False, + ) ret = layers.spectral_norm(weight=Weight, dim=1, power_iters=2) - static_ret = self.get_static_graph_result(feed={ - 'Weight': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1]], - place=place), - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret = self.get_static_graph_result( + feed={ + 'Weight': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place + ), + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.static_graph(): - Weight = fluid.layers.data(name='Weight', - shape=shape, - dtype='float32', - lod_level=1, - append_batch_size=False) + Weight = fluid.layers.data( + name='Weight', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False, + ) spectralNorm = nn.SpectralNorm(shape, dim=1, power_iters=2) ret = spectralNorm(Weight) - static_ret2 = self.get_static_graph_result(feed={ - 'Weight': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret2 = self.get_static_graph_result( + feed={ + 'Weight': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -2024,77 +2259,85 @@ class TestLayer(LayerTest): adj = np.tile(adj, (1, 1, 1)) vectors = np.random.random((1, 10, 5)).astype('float32') with self.static_graph(): - NodesVector = fluid.layers.data(name='NodesVector', - shape=(1, 10, 5), - dtype='float32', - lod_level=1, - append_batch_size=False) - EdgeSet = fluid.layers.data(name='EdgeSet', - shape=(1, 9, 2), - dtype='int32', - lod_level=1, - append_batch_size=False) - ret = fluid.contrib.layers.tree_conv(nodes_vector=NodesVector, - edge_set=EdgeSet, - output_size=6, - num_filters=1, - max_depth=2) - static_ret = self.get_static_graph_result(feed={ - 'NodesVector': - fluid.create_lod_tensor(data=vectors, - recursive_seq_lens=[[1]], - place=place), - 'EdgeSet': - fluid.create_lod_tensor(data=adj, - recursive_seq_lens=[[1]], - place=place) - }, - fetch_list=[ret], - with_lod=False)[0] + NodesVector = fluid.layers.data( + name='NodesVector', + shape=(1, 10, 5), + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + EdgeSet = fluid.layers.data( + name='EdgeSet', + shape=(1, 9, 2), + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + ret = fluid.contrib.layers.tree_conv( + nodes_vector=NodesVector, + edge_set=EdgeSet, + output_size=6, + num_filters=1, + max_depth=2, + ) + static_ret = self.get_static_graph_result( + feed={ + 'NodesVector': fluid.create_lod_tensor( + data=vectors, recursive_seq_lens=[[1]], place=place + ), + 'EdgeSet': fluid.create_lod_tensor( + data=adj, recursive_seq_lens=[[1]], place=place + ), + }, + fetch_list=[ret], + with_lod=False, + )[0] with self.static_graph(): - NodesVector = fluid.layers.data(name='NodesVector', - shape=(1, 10, 5), - dtype='float32', - lod_level=1, - append_batch_size=False) - EdgeSet = fluid.layers.data(name='EdgeSet', - shape=(1, 9, 2), - dtype='int32', - lod_level=1, - append_batch_size=False) - treeConv = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2) + NodesVector = fluid.layers.data( + name='NodesVector', + shape=(1, 10, 5), + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + EdgeSet = fluid.layers.data( + name='EdgeSet', + shape=(1, 9, 2), + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + treeConv = nn.TreeConv( + feature_size=5, output_size=6, num_filters=1, max_depth=2 + ) ret = treeConv(NodesVector, EdgeSet) - static_ret2 = self.get_static_graph_result(feed={ - 'NodesVector': - fluid.create_lod_tensor(data=vectors, - recursive_seq_lens=[[1]], - place=place), - 'EdgeSet': - fluid.create_lod_tensor(data=adj, - recursive_seq_lens=[[1]], - place=place) - }, - fetch_list=[ret], - with_lod=False)[0] + static_ret2 = self.get_static_graph_result( + feed={ + 'NodesVector': fluid.create_lod_tensor( + data=vectors, recursive_seq_lens=[[1]], place=place + ), + 'EdgeSet': fluid.create_lod_tensor( + data=adj, recursive_seq_lens=[[1]], place=place + ), + }, + fetch_list=[ret], + with_lod=False, + )[0] with self.dynamic_graph(): with _test_eager_guard(): - treeConv = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2) - dy_eager_ret = treeConv(base.to_variable(vectors), - base.to_variable(adj)) + treeConv = nn.TreeConv( + feature_size=5, output_size=6, num_filters=1, max_depth=2 + ) + dy_eager_ret = treeConv( + base.to_variable(vectors), base.to_variable(adj) + ) dy_eager_rlt_value = dy_eager_ret.numpy() - treeConv = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2) + treeConv = nn.TreeConv( + feature_size=5, output_size=6, num_filters=1, max_depth=2 + ) dy_ret = treeConv(base.to_variable(vectors), base.to_variable(adj)) dy_rlt_value = dy_ret.numpy() @@ -2107,107 +2350,135 @@ class TestLayer(LayerTest): custom_weight = np.random.randn(5, 3, 6, 1).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - treeConv1 = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2, - bias_attr='eager_tc1_b') - treeConv2 = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2, - param_attr=weight_attr, - bias_attr='eager_tc2_b') - dy_ret1 = treeConv1(base.to_variable(vectors), - base.to_variable(adj)) - dy_ret2 = treeConv2(base.to_variable(vectors), - base.to_variable(adj)) + custom_weight + ) + ) + treeConv1 = nn.TreeConv( + feature_size=5, + output_size=6, + num_filters=1, + max_depth=2, + bias_attr='eager_tc1_b', + ) + treeConv2 = nn.TreeConv( + feature_size=5, + output_size=6, + num_filters=1, + max_depth=2, + param_attr=weight_attr, + bias_attr='eager_tc2_b', + ) + dy_ret1 = treeConv1( + base.to_variable(vectors), base.to_variable(adj) + ) + dy_ret2 = treeConv2( + base.to_variable(vectors), base.to_variable(adj) + ) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) treeConv2.weight.set_value(treeConv1.weight.numpy()) treeConv2.bias.set_value(treeConv1.bias) - dy_ret1 = treeConv1(base.to_variable(vectors), - base.to_variable(adj)) - dy_ret2 = treeConv2(base.to_variable(vectors), - base.to_variable(adj)) + dy_ret1 = treeConv1( + base.to_variable(vectors), base.to_variable(adj) + ) + dy_ret2 = treeConv2( + base.to_variable(vectors), base.to_variable(adj) + ) np.testing.assert_array_equal(dy_ret1.numpy(), dy_ret2.numpy()) treeConv2.weight = treeConv1.weight treeConv2.bias = treeConv1.bias - np.testing.assert_array_equal(treeConv1.weight.numpy(), - treeConv2.weight.numpy()) - np.testing.assert_array_equal(treeConv1.bias.numpy(), - treeConv2.bias.numpy()) + np.testing.assert_array_equal( + treeConv1.weight.numpy(), treeConv2.weight.numpy() + ) + np.testing.assert_array_equal( + treeConv1.bias.numpy(), treeConv2.bias.numpy() + ) custom_weight = np.random.randn(5, 3, 6, 1).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) - treeConv1 = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2, - bias_attr='tc1_b') - treeConv2 = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2, - param_attr=weight_attr, - bias_attr='tc2_b') - dy_ret1 = treeConv1(base.to_variable(vectors), - base.to_variable(adj)) - dy_ret2 = treeConv2(base.to_variable(vectors), - base.to_variable(adj)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) + treeConv1 = nn.TreeConv( + feature_size=5, + output_size=6, + num_filters=1, + max_depth=2, + bias_attr='tc1_b', + ) + treeConv2 = nn.TreeConv( + feature_size=5, + output_size=6, + num_filters=1, + max_depth=2, + param_attr=weight_attr, + bias_attr='tc2_b', + ) + dy_ret1 = treeConv1( + base.to_variable(vectors), base.to_variable(adj) + ) + dy_ret2 = treeConv2( + base.to_variable(vectors), base.to_variable(adj) + ) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) treeConv2.weight.set_value(treeConv1.weight.numpy()) treeConv2.bias.set_value(treeConv1.bias) - dy_ret1 = treeConv1(base.to_variable(vectors), - base.to_variable(adj)) - dy_ret2 = treeConv2(base.to_variable(vectors), - base.to_variable(adj)) + dy_ret1 = treeConv1( + base.to_variable(vectors), base.to_variable(adj) + ) + dy_ret2 = treeConv2( + base.to_variable(vectors), base.to_variable(adj) + ) np.testing.assert_array_equal(dy_ret1.numpy(), dy_ret2.numpy()) treeConv2.weight = treeConv1.weight treeConv2.bias = treeConv1.bias - np.testing.assert_array_equal(treeConv1.weight.numpy(), - treeConv2.weight.numpy()) - np.testing.assert_array_equal(treeConv1.bias.numpy(), - treeConv2.bias.numpy()) + np.testing.assert_array_equal( + treeConv1.weight.numpy(), treeConv2.weight.numpy() + ) + np.testing.assert_array_equal( + treeConv1.bias.numpy(), treeConv2.bias.numpy() + ) def test_conv3d_transpose(self): - input_array = np.arange(0, 48).reshape([2, 3, 2, 2, - 2]).astype('float32') + input_array = ( + np.arange(0, 48).reshape([2, 3, 2, 2, 2]).astype('float32') + ) with self.static_graph(): img = layers.data(name='pixel', shape=[3, 2, 2, 2], dtype='float32') - out = layers.conv3d_transpose(input=img, - num_filters=12, - filter_size=12, - use_cudnn=False) + out = layers.conv3d_transpose( + input=img, num_filters=12, filter_size=12, use_cudnn=False + ) static_rlt = self.get_static_graph_result( - feed={'pixel': input_array}, fetch_list=[out])[0] + feed={'pixel': input_array}, fetch_list=[out] + )[0] with self.static_graph(): img = layers.data(name='pixel', shape=[3, 2, 2, 2], dtype='float32') - conv3d_transpose = nn.Conv3DTranspose(num_channels=3, - num_filters=12, - filter_size=12, - use_cudnn=False) + conv3d_transpose = nn.Conv3DTranspose( + num_channels=3, num_filters=12, filter_size=12, use_cudnn=False + ) out = conv3d_transpose(img) static_rlt2 = self.get_static_graph_result( - feed={'pixel': input_array}, fetch_list=[out])[0] + feed={'pixel': input_array}, fetch_list=[out] + )[0] with self.dynamic_graph(): with _test_eager_guard(): - conv3d_transpose = nn.Conv3DTranspose(num_channels=3, - num_filters=12, - filter_size=12, - use_cudnn=False) + conv3d_transpose = nn.Conv3DTranspose( + num_channels=3, + num_filters=12, + filter_size=12, + use_cudnn=False, + ) dy_eager_rlt = conv3d_transpose(base.to_variable(input_array)) dy_eager_rlt_value = dy_eager_rlt.numpy() - conv3d_transpose = nn.Conv3DTranspose(num_channels=3, - num_filters=12, - filter_size=12, - use_cudnn=False) + conv3d_transpose = nn.Conv3DTranspose( + num_channels=3, num_filters=12, filter_size=12, use_cudnn=False + ) dy_rlt = conv3d_transpose(base.to_variable(input_array)) dy_rlt_value = dy_rlt.numpy() np.testing.assert_allclose(static_rlt2, static_rlt, rtol=1e-05) @@ -2220,30 +2491,39 @@ class TestLayer(LayerTest): custom_weight = np.random.randn(3, 3, 2, 2, 2).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - conv3d1 = nn.Conv3DTranspose(num_channels=3, - num_filters=3, - filter_size=2, - bias_attr='eager_conv3d1_b', - use_cudnn=False) - conv3d2 = nn.Conv3DTranspose(num_channels=3, - num_filters=3, - filter_size=2, - param_attr=weight_attr, - bias_attr='eager_conv3d2_b', - use_cudnn=False) + custom_weight + ) + ) + conv3d1 = nn.Conv3DTranspose( + num_channels=3, + num_filters=3, + filter_size=2, + bias_attr='eager_conv3d1_b', + use_cudnn=False, + ) + conv3d2 = nn.Conv3DTranspose( + num_channels=3, + num_filters=3, + filter_size=2, + param_attr=weight_attr, + bias_attr='eager_conv3d2_b', + use_cudnn=False, + ) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) conv3d1_weight_np = conv3d1.weight.numpy() conv3d1_bias = conv3d1.bias self.assertFalse( - np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy())) + np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy()) + ) conv3d2.weight.set_value(conv3d1_weight_np) - np.testing.assert_array_equal(conv3d1_weight_np, - conv3d2.weight.numpy()) + np.testing.assert_array_equal( + conv3d1_weight_np, conv3d2.weight.numpy() + ) conv3d1.bias.set_value(conv3d1_bias) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) @@ -2251,26 +2531,35 @@ class TestLayer(LayerTest): conv3d2.weight = conv3d1.weight conv3d2.bias = conv3d1.bias - np.testing.assert_array_equal(conv3d1.weight.numpy(), - conv3d2.weight.numpy()) - np.testing.assert_array_equal(conv3d1.bias.numpy(), - conv3d2.bias.numpy()) + np.testing.assert_array_equal( + conv3d1.weight.numpy(), conv3d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv3d1.bias.numpy(), conv3d2.bias.numpy() + ) images = np.ones([2, 3, 6, 6, 6], dtype='float32') custom_weight = np.random.randn(3, 3, 2, 2, 2).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) - conv3d1 = nn.Conv3DTranspose(num_channels=3, - num_filters=3, - filter_size=2, - bias_attr='conv3d1_b', - use_cudnn=False) - conv3d2 = nn.Conv3DTranspose(num_channels=3, - num_filters=3, - filter_size=2, - param_attr=weight_attr, - bias_attr='conv3d2_b', - use_cudnn=False) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) + conv3d1 = nn.Conv3DTranspose( + num_channels=3, + num_filters=3, + filter_size=2, + bias_attr='conv3d1_b', + use_cudnn=False, + ) + conv3d2 = nn.Conv3DTranspose( + num_channels=3, + num_filters=3, + filter_size=2, + param_attr=weight_attr, + bias_attr='conv3d2_b', + use_cudnn=False, + ) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) @@ -2278,10 +2567,12 @@ class TestLayer(LayerTest): conv3d1_weight_np = conv3d1.weight.numpy() conv3d1_bias = conv3d1.bias self.assertFalse( - np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy())) + np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy()) + ) conv3d2.weight.set_value(conv3d1_weight_np) - np.testing.assert_array_equal(conv3d1_weight_np, - conv3d2.weight.numpy()) + np.testing.assert_array_equal( + conv3d1_weight_np, conv3d2.weight.numpy() + ) conv3d1.bias.set_value(conv3d1_bias) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) @@ -2289,10 +2580,12 @@ class TestLayer(LayerTest): conv3d2.weight = conv3d1.weight conv3d2.bias = conv3d1.bias - np.testing.assert_array_equal(conv3d1.weight.numpy(), - conv3d2.weight.numpy()) - np.testing.assert_array_equal(conv3d1.bias.numpy(), - conv3d2.bias.numpy()) + np.testing.assert_array_equal( + conv3d1.weight.numpy(), conv3d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv3d1.bias.numpy(), conv3d2.bias.numpy() + ) def test_eye_op(self): np_eye = np.eye(3, 2) @@ -2304,12 +2597,12 @@ class TestLayer(LayerTest): with self.dynamic_graph(): with _test_eager_guard(): eager_eye_tensor = layers.eye(num_rows=3, num_columns=2) - eager_eye_tensor_rlt1 = layers.eye(num_rows=3, - num_columns=2, - batch_shape=[3]) - eager_eye_tensor_rlt2 = layers.eye(num_rows=3, - num_columns=2, - batch_shape=[4, 3]) + eager_eye_tensor_rlt1 = layers.eye( + num_rows=3, num_columns=2, batch_shape=[3] + ) + eager_eye_tensor_rlt2 = layers.eye( + num_rows=3, num_columns=2, batch_shape=[4, 3] + ) eager_diag_tensor = layers.eye(20) eager_eye_tensor_value = eager_eye_tensor.numpy() eager_eye_tensor_rlt1_value = eager_eye_tensor_rlt1.numpy() @@ -2317,12 +2610,12 @@ class TestLayer(LayerTest): eager_diag_tensor_value = eager_diag_tensor.numpy() eye_tensor = layers.eye(num_rows=3, num_columns=2) - eye_tensor_rlt1 = layers.eye(num_rows=3, - num_columns=2, - batch_shape=[3]) - eye_tensor_rlt2 = layers.eye(num_rows=3, - num_columns=2, - batch_shape=[4, 3]) + eye_tensor_rlt1 = layers.eye( + num_rows=3, num_columns=2, batch_shape=[3] + ) + eye_tensor_rlt2 = layers.eye( + num_rows=3, num_columns=2, batch_shape=[4, 3] + ) diag_tensor = layers.eye(20) eye_tensor_value = eye_tensor.numpy() eye_tensor_rlt1_value = eye_tensor_rlt1.numpy() @@ -2330,23 +2623,23 @@ class TestLayer(LayerTest): diag_tensor_value = diag_tensor.numpy() np.testing.assert_allclose(eager_eye_tensor_value, np_eye, rtol=1e-05) - np.testing.assert_allclose(eager_eye_tensor_rlt1_value, - stack_rlt1, - rtol=1e-05) - np.testing.assert_allclose(eager_eye_tensor_rlt2_value, - stack_rlt2, - rtol=1e-05) - np.testing.assert_allclose(eager_diag_tensor_value, - np.eye(20), - rtol=1e-05) + np.testing.assert_allclose( + eager_eye_tensor_rlt1_value, stack_rlt1, rtol=1e-05 + ) + np.testing.assert_allclose( + eager_eye_tensor_rlt2_value, stack_rlt2, rtol=1e-05 + ) + np.testing.assert_allclose( + eager_diag_tensor_value, np.eye(20), rtol=1e-05 + ) np.testing.assert_allclose(eye_tensor_value, np_eye, rtol=1e-05) - np.testing.assert_allclose(eye_tensor_rlt1_value, - stack_rlt1, - rtol=1e-05) - np.testing.assert_allclose(eye_tensor_rlt2_value, - stack_rlt2, - rtol=1e-05) + np.testing.assert_allclose( + eye_tensor_rlt1_value, stack_rlt1, rtol=1e-05 + ) + np.testing.assert_allclose( + eye_tensor_rlt2_value, stack_rlt2, rtol=1e-05 + ) np.testing.assert_allclose(diag_tensor_value, np.eye(20), rtol=1e-05) with self.assertRaises(TypeError): @@ -2406,11 +2699,9 @@ class TestLayer(LayerTest): a = layers.data(name='a', shape=[1], dtype='int64') b = layers.data(name='b', shape=[1], dtype='int64') cond = layers.less_than(x=a, y=b) - static_ret = self.get_static_graph_result(feed={ - "a": value_a, - "b": value_b - }, - fetch_list=[cond])[0] + static_ret = self.get_static_graph_result( + feed={"a": value_a, "b": value_b}, fetch_list=[cond] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da = base.to_variable(value_a) @@ -2432,11 +2723,9 @@ class TestLayer(LayerTest): a1 = layers.data(name='a1', shape=[1], dtype='int64') b1 = layers.data(name='b1', shape=[1], dtype='int64') cond1 = layers.less_equal(x=a1, y=b1) - static_ret1 = self.get_static_graph_result(feed={ - "a1": value_a, - "b1": value_b - }, - fetch_list=[cond1])[0] + static_ret1 = self.get_static_graph_result( + feed={"a1": value_a, "b1": value_b}, fetch_list=[cond1] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da1 = base.to_variable(value_a) @@ -2453,16 +2742,14 @@ class TestLayer(LayerTest): for i in range(len(static_ret1)): self.assertTrue(dcond1.numpy()[i] == static_ret1[i]) - #greater than + # greater than with self.static_graph(): a2 = layers.data(name='a2', shape=[1], dtype='int64') b2 = layers.data(name='b2', shape=[1], dtype='int64') cond2 = layers.greater_than(x=a2, y=b2) - static_ret2 = self.get_static_graph_result(feed={ - "a2": value_a, - "b2": value_b - }, - fetch_list=[cond2])[0] + static_ret2 = self.get_static_graph_result( + feed={"a2": value_a, "b2": value_b}, fetch_list=[cond2] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da2 = base.to_variable(value_a) @@ -2479,16 +2766,14 @@ class TestLayer(LayerTest): for i in range(len(static_ret2)): self.assertTrue(dcond2.numpy()[i] == static_ret2[i]) - #greater equal + # greater equal with self.static_graph(): a3 = layers.data(name='a3', shape=[1], dtype='int64') b3 = layers.data(name='b3', shape=[1], dtype='int64') cond3 = layers.greater_equal(x=a3, y=b3) - static_ret3 = self.get_static_graph_result(feed={ - "a3": value_a, - "b3": value_b - }, - fetch_list=[cond3])[0] + static_ret3 = self.get_static_graph_result( + feed={"a3": value_a, "b3": value_b}, fetch_list=[cond3] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da3 = base.to_variable(value_a) @@ -2510,11 +2795,9 @@ class TestLayer(LayerTest): a4 = layers.data(name='a4', shape=[1], dtype='int64') b4 = layers.data(name='b4', shape=[1], dtype='int64') cond4 = layers.equal(x=a4, y=b4) - static_ret4 = self.get_static_graph_result(feed={ - "a4": value_a, - "b4": value_b - }, - fetch_list=[cond4])[0] + static_ret4 = self.get_static_graph_result( + feed={"a4": value_a, "b4": value_b}, fetch_list=[cond4] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da4 = base.to_variable(value_a) @@ -2536,11 +2819,9 @@ class TestLayer(LayerTest): a5 = layers.data(name='a5', shape=[1], dtype='int64') b5 = layers.data(name='b5', shape=[1], dtype='int64') cond5 = layers.equal(x=a5, y=b5) - static_ret5 = self.get_static_graph_result(feed={ - "a5": value_a, - "b5": value_b - }, - fetch_list=[cond5])[0] + static_ret5 = self.get_static_graph_result( + feed={"a5": value_a, "b5": value_b}, fetch_list=[cond5] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da5 = base.to_variable(value_a) @@ -2558,7 +2839,6 @@ class TestLayer(LayerTest): self.assertTrue(dcond5.numpy()[i] == static_ret5[i]) def test_cond(self): - def less_than_branch(a, b): return fluid.layers.elementwise_add(a, b) @@ -2566,16 +2846,22 @@ class TestLayer(LayerTest): return fluid.layers.elementwise_sub(a, b) with self.static_graph(): - a = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=0.1) - b = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=0.23) - out = fluid.layers.cond(a >= b, lambda: greater_equal_branch(a, b), - lambda: less_than_branch(a, b)) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + a = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=0.1 + ) + b = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=0.23 + ) + out = fluid.layers.cond( + a >= b, + lambda: greater_equal_branch(a, b), + lambda: less_than_branch(a, b), + ) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(fetch_list=[out]) static_res = ret[0] @@ -2584,15 +2870,23 @@ class TestLayer(LayerTest): with _test_eager_guard(): a = fluid.dygraph.to_variable(np.array([0.1]).astype('float32')) b = fluid.dygraph.to_variable( - np.array([0.23]).astype('float32')) - out = layers.cond(a < b, lambda: less_than_branch(a, b), - lambda: greater_equal_branch(a, b)) - out2 = layers.cond(a >= b, lambda: greater_equal_branch(a, b), - lambda: less_than_branch(a, b)) + np.array([0.23]).astype('float32') + ) + out = layers.cond( + a < b, + lambda: less_than_branch(a, b), + lambda: greater_equal_branch(a, b), + ) + out2 = layers.cond( + a >= b, + lambda: greater_equal_branch(a, b), + lambda: less_than_branch(a, b), + ) eager_dynamic_res = out.numpy() eager_dynamic_res2 = out2.numpy() - np.testing.assert_array_equal(eager_dynamic_res, - eager_dynamic_res2) + np.testing.assert_array_equal( + eager_dynamic_res, eager_dynamic_res2 + ) with self.assertRaises(TypeError): layers.cond(a < b, 'str', 'str') with self.assertRaises(TypeError): @@ -2600,10 +2894,16 @@ class TestLayer(LayerTest): a = fluid.dygraph.to_variable(np.array([0.1]).astype('float32')) b = fluid.dygraph.to_variable(np.array([0.23]).astype('float32')) - out = layers.cond(a < b, lambda: less_than_branch(a, b), - lambda: greater_equal_branch(a, b)) - out2 = layers.cond(a >= b, lambda: greater_equal_branch(a, b), - lambda: less_than_branch(a, b)) + out = layers.cond( + a < b, + lambda: less_than_branch(a, b), + lambda: greater_equal_branch(a, b), + ) + out2 = layers.cond( + a >= b, + lambda: greater_equal_branch(a, b), + lambda: less_than_branch(a, b), + ) dynamic_res = out.numpy() dynamic_res2 = out2.numpy() np.testing.assert_array_equal(dynamic_res, dynamic_res2) @@ -2616,7 +2916,6 @@ class TestLayer(LayerTest): np.testing.assert_array_equal(static_res, eager_dynamic_res) def test_case(self): - def fn_1(): return layers.fill_constant(shape=[1, 2], dtype='float32', value=1) @@ -2635,12 +2934,16 @@ class TestLayer(LayerTest): pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_3 = layers.equal(x, y) # false: 0.3 == 0.1 - out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], - default=fn_3) + out_1 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3 + ) out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) static_res1, static_res2 = exe.run(fetch_list=[out_1, out_2]) @@ -2654,11 +2957,12 @@ class TestLayer(LayerTest): pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_3 = layers.equal(x, y) # false: 0.3 == 0.1 - out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), - (pred_2, fn_2)], - default=fn_3) - out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, - fn_3)]) + out_1 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3 + ) + out_2 = layers.case( + pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)] + ) eager_dynamic_res1 = out_1.numpy() eager_dynamic_res2 = out_2.numpy() @@ -2670,8 +2974,9 @@ class TestLayer(LayerTest): pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_3 = layers.equal(x, y) # false: 0.3 == 0.1 - out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], - default=fn_3) + out_1 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3 + ) out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)]) dynamic_res1 = out_1.numpy() dynamic_res2 = out_2.numpy() @@ -2682,7 +2987,6 @@ class TestLayer(LayerTest): np.testing.assert_array_equal(static_res2, eager_dynamic_res2) def test_switch_case(self): - def fn_1(): return layers.fill_constant(shape=[1, 2], dtype='float32', value=1) @@ -2696,46 +3000,54 @@ class TestLayer(LayerTest): index_1 = layers.fill_constant(shape=[1], dtype='int32', value=1) index_2 = layers.fill_constant(shape=[1], dtype='int32', value=2) - out_1 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2 - }, - default=fn_3) - out_2 = layers.switch_case(branch_index=index_2, - branch_fns=[(1, fn_1), (2, fn_2)], - default=fn_3) - out_3 = layers.switch_case(branch_index=index_2, - branch_fns=[(0, fn_1), (4, fn_2), - (7, fn_3)]) - - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + out_1 = layers.switch_case( + branch_index=index_1, + branch_fns={1: fn_1, 2: fn_2}, + default=fn_3, + ) + out_2 = layers.switch_case( + branch_index=index_2, + branch_fns=[(1, fn_1), (2, fn_2)], + default=fn_3, + ) + out_3 = layers.switch_case( + branch_index=index_2, + branch_fns=[(0, fn_1), (4, fn_2), (7, fn_3)], + ) + + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) static_res1, static_res2, static_res3 = exe.run( - fetch_list=[out_1, out_2, out_3]) + fetch_list=[out_1, out_2, out_3] + ) with self.dynamic_graph(): with _test_eager_guard(): - index_1 = layers.fill_constant(shape=[1], - dtype='int32', - value=1) - index_2 = layers.fill_constant(shape=[1], - dtype='int32', - value=2) - - out_1 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2 - }, - default=fn_3) - out_2 = layers.switch_case(branch_index=index_2, - branch_fns=[(1, fn_1), (2, fn_2)], - default=fn_3) - out_3 = layers.switch_case(branch_index=index_2, - branch_fns=[(0, fn_1), (4, fn_2), - (7, fn_3)]) + index_1 = layers.fill_constant( + shape=[1], dtype='int32', value=1 + ) + index_2 = layers.fill_constant( + shape=[1], dtype='int32', value=2 + ) + + out_1 = layers.switch_case( + branch_index=index_1, + branch_fns={1: fn_1, 2: fn_2}, + default=fn_3, + ) + out_2 = layers.switch_case( + branch_index=index_2, + branch_fns=[(1, fn_1), (2, fn_2)], + default=fn_3, + ) + out_3 = layers.switch_case( + branch_index=index_2, + branch_fns=[(0, fn_1), (4, fn_2), (7, fn_3)], + ) eager_dynamic_res1 = out_1.numpy() eager_dynamic_res2 = out_2.numpy() @@ -2744,18 +3056,20 @@ class TestLayer(LayerTest): index_1 = layers.fill_constant(shape=[1], dtype='int32', value=1) index_2 = layers.fill_constant(shape=[1], dtype='int32', value=2) - out_1 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2 - }, - default=fn_3) - out_2 = layers.switch_case(branch_index=index_2, - branch_fns=[(1, fn_1), (2, fn_2)], - default=fn_3) - out_3 = layers.switch_case(branch_index=index_2, - branch_fns=[(0, fn_1), (4, fn_2), - (7, fn_3)]) + out_1 = layers.switch_case( + branch_index=index_1, + branch_fns={1: fn_1, 2: fn_2}, + default=fn_3, + ) + out_2 = layers.switch_case( + branch_index=index_2, + branch_fns=[(1, fn_1), (2, fn_2)], + default=fn_3, + ) + out_3 = layers.switch_case( + branch_index=index_2, + branch_fns=[(0, fn_1), (4, fn_2), (7, fn_3)], + ) dynamic_res1 = out_1.numpy() dynamic_res2 = out_2.numpy() @@ -2772,32 +3086,32 @@ class TestLayer(LayerTest): with self.static_graph(): x = fluid.layers.data(name="x1", shape=[6, 5, 8]) - dim1 = fluid.layers.data(name="dim1", - shape=[1], - append_batch_size=False) - dim2 = fluid.layers.data(name="dim2", - shape=[1], - append_batch_size=False) + dim1 = fluid.layers.data( + name="dim1", shape=[1], append_batch_size=False + ) + dim2 = fluid.layers.data( + name="dim2", shape=[1], append_batch_size=False + ) crop_shape1 = (1, 2, 4, 4) - crop_shape2 = fluid.layers.data(name="crop_shape", - shape=[4], - append_batch_size=False) + crop_shape2 = fluid.layers.data( + name="crop_shape", shape=[4], append_batch_size=False + ) crop_shape3 = [-1, dim1, dim2, 4] crop_offsets1 = [0, 0, 1, 0] - crop_offsets2 = fluid.layers.data(name="crop_offset", - shape=[4], - append_batch_size=False) + crop_offsets2 = fluid.layers.data( + name="crop_offset", shape=[4], append_batch_size=False + ) crop_offsets3 = [0, dim1, dim2, 0] - out1 = fluid.layers.crop_tensor(x, - shape=crop_shape1, - offsets=crop_offsets1) - out2 = fluid.layers.crop_tensor(x, - shape=crop_shape2, - offsets=crop_offsets2) - out3 = fluid.layers.crop_tensor(x, - shape=crop_shape3, - offsets=crop_offsets3) + out1 = fluid.layers.crop_tensor( + x, shape=crop_shape1, offsets=crop_offsets1 + ) + out2 = fluid.layers.crop_tensor( + x, shape=crop_shape2, offsets=crop_offsets2 + ) + out3 = fluid.layers.crop_tensor( + x, shape=crop_shape3, offsets=crop_offsets3 + ) self.assertIsNotNone(out1) self.assertIsNotNone(out2) @@ -2806,10 +3120,9 @@ class TestLayer(LayerTest): def test_shard_index(self): with self.static_graph(): x = fluid.layers.data(name="label", shape=[4, 1], dtype='int64') - shard_label = fluid.layers.shard_index(input=x, - index_num=20, - nshards=2, - shard_id=0) + shard_label = fluid.layers.shard_index( + input=x, index_num=20, nshards=2, shard_id=0 + ) self.assertIsNotNone(shard_label) @@ -2828,11 +3141,9 @@ class TestLayer(LayerTest): exe.run(fluid.default_startup_program()) # x = np.random.rand(3, 32, 32).astype("float32") # y = np.array([[1], [0], [1]]) - static_out = exe.run(feed={ - "input": x, - "label": y - }, - fetch_list=result[0]) + static_out = exe.run( + feed={"input": x, "label": y}, fetch_list=result[0] + ) with self.dynamic_graph(force_to_use_cpu=True): data = base.to_variable(x) @@ -2845,15 +3156,19 @@ class TestLayer(LayerTest): class TestBook(LayerTest): - def setUp(self): self.only_static_set = set({"make_word_embedding"}) - self.not_compare_static_dygraph_set = set({ - "make_gaussian_random", "make_gaussian_random_batch_size_like", - "make_kldiv_loss", "make_prelu", - "make_sampled_softmax_with_cross_entropy", "make_sampling_id", - "make_uniform_random_batch_size_like" - }) + self.not_compare_static_dygraph_set = set( + { + "make_gaussian_random", + "make_gaussian_random_batch_size_like", + "make_kldiv_loss", + "make_prelu", + "make_sampled_softmax_with_cross_entropy", + "make_sampling_id", + "make_uniform_random_batch_size_like", + } + ) self.all_close_compare = set({"make_spectral_norm"}) def func_all_layers(self): @@ -2877,7 +3192,8 @@ class TestBook(LayerTest): static_result = self.get_static_graph_result( feed=self._feed_dict, fetch_list=fetch_list, - force_to_use_cpu=self._force_to_use_cpu) + force_to_use_cpu=self._force_to_use_cpu, + ) else: assert method.__name__ in ('make_get_places') @@ -2898,7 +3214,9 @@ class TestBook(LayerTest): rtol=1e-05, atol=0, err_msg='Result of function [{}] compare failed'.format( - method.__name__)) + method.__name__ + ), + ) continue if method.__name__ not in self.not_compare_static_dygraph_set: @@ -2906,7 +3224,9 @@ class TestBook(LayerTest): static_result[0], dy_result_value, err_msg='Result of function [{}] not equal'.format( - method.__name__)) + method.__name__ + ), + ) def test_all_layers(self): with _test_eager_guard(): @@ -2922,95 +3242,111 @@ class TestBook(LayerTest): elif dtype == 'float64': return np.random.random(shape).astype(dtype) elif dtype == 'int32': - return np.random.randint(self._low_data_bound, - self._high_data_bound, shape).astype(dtype) + return np.random.randint( + self._low_data_bound, self._high_data_bound, shape + ).astype(dtype) elif dtype == 'int64': - return np.random.randint(self._low_data_bound, - self._high_data_bound, shape).astype(dtype) - - def _get_data(self, - name, - shape, - dtype, - set_feed_dict=True, - append_batch_size=True): + return np.random.randint( + self._low_data_bound, self._high_data_bound, shape + ).astype(dtype) + + def _get_data( + self, name, shape, dtype, set_feed_dict=True, append_batch_size=True + ): if base.enabled(): - return base.to_variable(value=self._get_np_data( - shape, dtype, append_batch_size), - name=name, - zero_copy=False) + return base.to_variable( + value=self._get_np_data(shape, dtype, append_batch_size), + name=name, + zero_copy=False, + ) else: if set_feed_dict: self._feed_dict[name] = self._get_np_data( - shape, dtype, append_batch_size) - return layers.data(name=name, - shape=shape, - dtype=dtype, - append_batch_size=append_batch_size) + shape, dtype, append_batch_size + ) + return layers.data( + name=name, + shape=shape, + dtype=dtype, + append_batch_size=append_batch_size, + ) def make_sampled_softmax_with_cross_entropy(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): logits = self._get_data(name='Logits', shape=[256], dtype='float32') label = self._get_data(name='Label', shape=[1], dtype='int64') num_samples = 25 output = layers.sampled_softmax_with_cross_entropy( - logits, label, num_samples) - return (output) + logits, label, num_samples + ) + return output def make_fit_a_line(self): - with program_guard(fluid.default_main_program(), - startup_program=fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), + startup_program=fluid.default_startup_program(), + ): x = self._get_data(name='x', shape=[13], dtype='float32') y_predict = layers.fc(input=x, size=1, act=None) y = self._get_data(name='y', shape=[1], dtype='float32') cost = layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - return (avg_cost) + return avg_cost def make_recognize_digits_mlp(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): # Change g_program, so the rest layers use `g_program` images = self._get_data(name='pixel', shape=[784], dtype='float32') label = self._get_data(name='label', shape=[1], dtype='int64') hidden1 = layers.fc(input=images, size=128, act='relu') hidden2 = layers.fc(input=hidden1, size=64, act='relu') - predict = layers.fc(input=[hidden2, hidden1], - size=10, - act='softmax', - param_attr=["sftmax.w1", "sftmax.w2"]) + predict = layers.fc( + input=[hidden2, hidden1], + size=10, + act='softmax', + param_attr=["sftmax.w1", "sftmax.w2"], + ) cost = layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(cost) - return (avg_cost) + return avg_cost def make_conv2d_transpose(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): img = self._get_data(name='pixel', shape=[3, 2, 2], dtype='float32') - return layers.conv2d_transpose(input=img, - num_filters=10, - output_size=28) + return layers.conv2d_transpose( + input=img, num_filters=10, output_size=28 + ) def make_recognize_digits_conv(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - images = self._get_data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + images = self._get_data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = self._get_data(name='label', shape=[1], dtype='int64') - conv_pool_1 = nets.simple_img_conv_pool(input=images, - filter_size=5, - num_filters=2, - pool_size=2, - pool_stride=2, - act="relu") - conv_pool_2 = nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=4, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = nets.simple_img_conv_pool( + input=images, + filter_size=5, + num_filters=2, + pool_size=2, + pool_stride=2, + act="relu", + ) + conv_pool_2 = nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=4, + pool_size=2, + pool_stride=2, + act="relu", + ) predict = layers.fc(input=conv_pool_2, size=10, act="softmax") cost = layers.cross_entropy(input=predict, label=label) @@ -3018,185 +3354,212 @@ class TestBook(LayerTest): return avg_cost def make_word_embedding(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): dict_size = 10000 embed_size = 32 first_word = self._get_data(name='firstw', shape=[1], dtype='int64') - second_word = self._get_data(name='secondw', - shape=[1], - dtype='int64') + second_word = self._get_data( + name='secondw', shape=[1], dtype='int64' + ) third_word = self._get_data(name='thirdw', shape=[1], dtype='int64') forth_word = self._get_data(name='forthw', shape=[1], dtype='int64') next_word = self._get_data(name='nextw', shape=[1], dtype='int64') - embed_first = layers.embedding(input=first_word, - size=[dict_size, embed_size], - dtype='float32', - param_attr='shared_w') - embed_second = layers.embedding(input=second_word, - size=[dict_size, embed_size], - dtype='float32', - param_attr='shared_w') - - embed_third = layers.embedding(input=third_word, - size=[dict_size, embed_size], - dtype='float32', - param_attr='shared_w') - embed_forth = layers.embedding(input=forth_word, - size=[dict_size, embed_size], - dtype='float32', - param_attr='shared_w') + embed_first = layers.embedding( + input=first_word, + size=[dict_size, embed_size], + dtype='float32', + param_attr='shared_w', + ) + embed_second = layers.embedding( + input=second_word, + size=[dict_size, embed_size], + dtype='float32', + param_attr='shared_w', + ) + + embed_third = layers.embedding( + input=third_word, + size=[dict_size, embed_size], + dtype='float32', + param_attr='shared_w', + ) + embed_forth = layers.embedding( + input=forth_word, + size=[dict_size, embed_size], + dtype='float32', + param_attr='shared_w', + ) concat_embed = layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], - axis=1) + axis=1, + ) hidden1 = layers.fc(input=concat_embed, size=256, act='sigmoid') - predict_word = layers.fc(input=hidden1, - size=dict_size, - act='softmax') + predict_word = layers.fc( + input=hidden1, size=dict_size, act='softmax' + ) cost = layers.cross_entropy(input=predict_word, label=next_word) avg_cost = paddle.mean(cost) - return (avg_cost) + return avg_cost def make_sigmoid_cross_entropy(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): dat = self._get_data(name='data', shape=[10], dtype='float32') lbl = self._get_data(name='label', shape=[10], dtype='float32') ignore_index = -1 - return (layers.sigmoid_cross_entropy_with_logits( - x=dat, label=lbl, ignore_index=ignore_index)) + return layers.sigmoid_cross_entropy_with_logits( + x=dat, label=lbl, ignore_index=ignore_index + ) def make_hsigmoid(self): self._force_to_use_cpu = True with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): x = self._get_data(name='x', shape=[2], dtype='float32') y = self._get_data(name='y', shape=[2], dtype='int64') - return (layers.hsigmoid(input=x, label=y, num_classes=2)) + return layers.hsigmoid(input=x, label=y, num_classes=2) # test hsigmod with custom tree structure program2 = Program() with program_guard(program2): x2 = self._get_data(name='x2', shape=[4, 8], dtype='float32') y2 = self._get_data(name='y2', shape=[4], dtype='int64') - path_table = self._get_data(name='path_table', - shape=[4, 6], - dtype='int64') - path_code = self._get_data(name='path_code', - shape=[4, 6], - dtype='int64') - return (layers.hsigmoid(input=x2, - label=y2, - num_classes=6, - path_table=path_table, - path_code=path_code, - is_custom=True)) + path_table = self._get_data( + name='path_table', shape=[4, 6], dtype='int64' + ) + path_code = self._get_data( + name='path_code', shape=[4, 6], dtype='int64' + ) + return layers.hsigmoid( + input=x2, + label=y2, + num_classes=6, + path_table=path_table, + path_code=path_code, + is_custom=True, + ) def make_pool2d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 224, 224], dtype='float32') - return (layers.pool2d(x, - pool_size=[5, 3], - pool_stride=[1, 2], - pool_padding=(2, 1))) + return layers.pool2d( + x, pool_size=[5, 3], pool_stride=[1, 2], pool_padding=(2, 1) + ) def make_pool2d_infershape(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): theta = self._get_data("theta", shape=[2, 3], dtype='float32') x = fluid.layers.affine_grid(theta, out_shape=[2, 3, 244, 244]) - return (layers.pool2d(x, - pool_size=[5, 3], - pool_stride=[1, 2], - pool_padding=(2, 1))) + return layers.pool2d( + x, pool_size=[5, 3], pool_stride=[1, 2], pool_padding=(2, 1) + ) def make_pool3d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[3, 244, 244, 244], - dtype='float32') - return (layers.pool3d(x, - pool_size=[5, 3, 2], - pool_stride=[1, 2, 3], - pool_padding=(2, 1, 1))) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', shape=[3, 244, 244, 244], dtype='float32' + ) + return layers.pool3d( + x, + pool_size=[5, 3, 2], + pool_stride=[1, 2, 3], + pool_padding=(2, 1, 1), + ) def make_adaptive_pool2d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 224, 224], dtype='float32') - return (layers.adaptive_pool2d(x, [3, 3], pool_type='avg')) + return layers.adaptive_pool2d(x, [3, 3], pool_type='avg') pool, mask = layers.adaptive_pool2d(x, [3, 3], require_index=True) - return (pool) - return (mask) - return (layers.adaptive_pool2d(x, 3, pool_type='avg')) + return pool + return mask + return layers.adaptive_pool2d(x, 3, pool_type='avg') pool, mask = layers.adaptive_pool2d(x, 3, require_index=True) - return (pool) - return (mask) + return pool + return mask def make_adaptive_pool3d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[3, 244, 224, 224], - dtype='float32') - return (layers.adaptive_pool3d(x, [3, 3, 3], pool_type='avg')) - pool, mask = layers.adaptive_pool3d(x, [3, 3, 3], - require_index=True) - return (pool) - return (mask) - return (layers.adaptive_pool3d(x, 3, pool_type='avg')) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', shape=[3, 244, 224, 224], dtype='float32' + ) + return layers.adaptive_pool3d(x, [3, 3, 3], pool_type='avg') + pool, mask = layers.adaptive_pool3d( + x, [3, 3, 3], require_index=True + ) + return pool + return mask + return layers.adaptive_pool3d(x, 3, pool_type='avg') pool, mask = layers.adaptive_pool3d(x, 3, require_index=True) - return (pool) - return (mask) + return pool + return mask def make_lstm_unit(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x_t_data = self._get_data(name='x_t_data', - shape=[10, 10], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x_t_data = self._get_data( + name='x_t_data', shape=[10, 10], dtype='float32' + ) x_t = layers.fc(input=x_t_data, size=10) - prev_hidden_data = self._get_data(name='prev_hidden_data', - shape=[10, 30], - dtype='float32') + prev_hidden_data = self._get_data( + name='prev_hidden_data', shape=[10, 30], dtype='float32' + ) prev_hidden = layers.fc(input=prev_hidden_data, size=30) - prev_cell_data = self._get_data(name='prev_cell', - shape=[10, 30], - dtype='float32') + prev_cell_data = self._get_data( + name='prev_cell', shape=[10, 30], dtype='float32' + ) prev_cell = layers.fc(input=prev_cell_data, size=30) - return (layers.lstm_unit(x_t=x_t, - hidden_t_prev=prev_hidden, - cell_t_prev=prev_cell)) + return layers.lstm_unit( + x_t=x_t, hidden_t_prev=prev_hidden, cell_t_prev=prev_cell + ) def make_softmax(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name='data', shape=[10], dtype='float32') hid = layers.fc(input=data, size=20) - return (layers.softmax(hid, axis=1)) + return layers.softmax(hid, axis=1) def make_space_to_depth(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 9, 6, 6], - append_batch_size=False, - dtype='float32') - return (layers.space_to_depth(data, 3)) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', + shape=[32, 9, 6, 6], + append_batch_size=False, + dtype='float32', + ) + return layers.space_to_depth(data, 3) def make_lrn(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name='data', shape=[6, 2, 2], dtype='float32') - return (layers.lrn(data)) + return layers.lrn(data) def make_get_places(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): get_places(device_count=1) @prog_scope() @@ -3205,9 +3568,10 @@ class TestBook(LayerTest): words = [] for i in range(window_size): words.append( - self._get_data(name='word_{0}'.format(i), - shape=[1], - dtype='int64')) + self._get_data( + name='word_{0}'.format(i), shape=[1], dtype='int64' + ) + ) dict_size = 10000 label_word = int(window_size // 2) + 1 @@ -3217,38 +3581,45 @@ class TestBook(LayerTest): if i == label_word: continue - emb = layers.embedding(input=words[i], - size=[dict_size, 32], - param_attr='emb.w', - is_sparse=True) + emb = layers.embedding( + input=words[i], + size=[dict_size, 32], + param_attr='emb.w', + is_sparse=True, + ) embs.append(emb) embs = layers.concat(input=embs, axis=1) - loss = layers.nce(input=embs, - label=words[label_word], - num_total_classes=dict_size, - param_attr='nce.w', - bias_attr='nce.b') + loss = layers.nce( + input=embs, + label=words[label_word], + num_total_classes=dict_size, + param_attr='nce.w', + bias_attr='nce.b', + ) avg_loss = paddle.mean(loss) - return (avg_loss) + return avg_loss def make_multiplex(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x1 = self._get_data(name='x1', shape=[4], dtype='float32') x2 = self._get_data(name='x2', shape=[4], dtype='float32') index = self._get_data(name='index', shape=[1], dtype='int32') out = layers.multiplex(inputs=[x1, x2], index=index) - return (out) + return out def make_softmax_with_cross_entropy(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[16], dtype='float32') y = self._get_data(name='label', shape=[1], dtype='int64') loss, softmax = layers.softmax_with_cross_entropy( - x, y, return_softmax=True) + x, y, return_softmax=True + ) self.assertIsNotNone(loss) self.assertIsNotNone(softmax) @@ -3267,39 +3638,41 @@ class TestBook(LayerTest): self.assertIsNotNone(loss2) self.assertIsNotNone(loss3) self.assertIsNotNone(loss4) - return (loss4) + return loss4 def make_smooth_l1(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[4], dtype='float32') y = self._get_data(name='label', shape=[4], dtype='float32') loss = layers.smooth_l1(x, y) - return (loss) + return loss def make_scatter(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[3, 3], - append_batch_size=False, - dtype='float32') - idx = self._get_data(name='idx', - shape=[2], - append_batch_size=False, - dtype='int32') - updates = self._get_data(name='updates', - shape=[2, 3], - append_batch_size=False, - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', shape=[3, 3], append_batch_size=False, dtype='float32' + ) + idx = self._get_data( + name='idx', shape=[2], append_batch_size=False, dtype='int32' + ) + updates = self._get_data( + name='updates', + shape=[2, 3], + append_batch_size=False, + dtype='float32', + ) out = layers.scatter(input=x, index=idx, updates=updates) - return (out) + return out def make_one_hot(self): with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): label = self._get_data(name="label", shape=[1], dtype="int32") one_hot_label = layers.one_hot(input=label, depth=10) - return (one_hot_label) + return one_hot_label def make_label_smooth(self): # TODO(minqiyang): support gpu ut @@ -3307,321 +3680,370 @@ class TestBook(LayerTest): with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): label = self._get_data(name="label", shape=[1], dtype="int32") one_hot_label = layers.one_hot(input=label, depth=10) - smooth_label = layers.label_smooth(label=one_hot_label, - epsilon=0.1, - dtype="int32") - return (smooth_label) + smooth_label = layers.label_smooth( + label=one_hot_label, epsilon=0.1, dtype="int32" + ) + return smooth_label def make_topk(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name="label", shape=[200], dtype="float32") values, indices = layers.topk(data, k=5) - return (values) - return (indices) + return values + return indices def make_resize_bilinear(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6], dtype="float32") output = layers.resize_bilinear(x, out_shape=[12, 12]) - return (output) + return output def make_resize_bilinear_by_scale(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6], dtype="float32") output = layers.resize_bilinear(x, scale=1.5) - return (output) + return output def make_resize_nearest(self): try: - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x1', shape=[3, 9, 6], dtype="float32") output = layers.resize_nearest(x, out_shape=[12, 12]) except ValueError: pass try: - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x2', - shape=[3, 9, 6, 7], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x2', shape=[3, 9, 6, 7], dtype="float32" + ) output = layers.resize_nearest(x, out_shape=[12, 12, 12]) except ValueError: pass - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6], dtype="float32") output = layers.resize_nearest(x, out_shape=[12, 12]) - return (output) + return output def make_resize_nearest_by_scale(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x1', shape=[3, 9, 6], dtype="float32") output = layers.resize_nearest(x, scale=1.8) - return (output) + return output def make_resize_trilinear(self): try: - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x2', shape=[3, 9, 6], dtype="float32") output = layers.resize_trilinear(x, out_shape=[12, 12, 12]) except ValueError: pass try: - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[3, 9, 6, 7], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', shape=[3, 9, 6, 7], dtype="float32" + ) output = layers.resize_trilinear(x, out_shape=[12, 12]) except ValueError: pass - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6, 7], dtype="float32") output = layers.resize_trilinear(x, out_shape=[12, 12, 12]) - return (output) + return output def make_resize_trilinear_by_scale(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6, 7], dtype="float32") output = layers.resize_trilinear(x, scale=2.1) - return (output) + return output def make_polygon_box_transform(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[8, 4, 4], dtype="float32") output = layers.polygon_box_transform(input=x) - return (output) + return output def make_l2_normalize(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[8, 7, 10], dtype="float32") output = layers.l2_normalize(x, axis=1) return output def make_crop(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 5], dtype="float32") y = self._get_data(name='y', shape=[2, 3], dtype="float32") output = layers.crop(x, shape=y) - return (output) + return output def make_mean_iou(self): with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): x = self._get_data(name='x', shape=[16], dtype='int32') y = self._get_data(name='label', shape=[16], dtype='int32') iou = layers.mean_iou(x, y, self._high_data_bound) - return (iou) + return iou def make_argsort(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name='x', shape=[2, 3, 3], dtype="float32") out, ids = layers.argsort(input=data, axis=1) - return (out) - return (ids) + return out + return ids def make_rank_loss(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - label = self._get_data(name='label', - append_batch_size=False, - shape=[16, 1], - dtype="float32") - left = self._get_data(name='left', - append_batch_size=False, - shape=[16, 1], - dtype="float32") - right = self._get_data(name='right', - append_batch_size=False, - shape=[16, 1], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + label = self._get_data( + name='label', + append_batch_size=False, + shape=[16, 1], + dtype="float32", + ) + left = self._get_data( + name='left', + append_batch_size=False, + shape=[16, 1], + dtype="float32", + ) + right = self._get_data( + name='right', + append_batch_size=False, + shape=[16, 1], + dtype="float32", + ) out = layers.rank_loss(label, left, right, name="rank_loss") - return (out) + return out def make_shape(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[3, 100, 100], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[3, 100, 100], dtype="float32" + ) out = layers.shape(input) - return (out) + return out def make_pad2d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[3, 100, 100], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[3, 100, 100], dtype="float32" + ) paddings = layers.fill_constant(shape=[4], dtype='int32', value=1) - out = layers.pad2d(input, - paddings=[1, 2, 3, 4], - mode='reflect', - data_format='NCHW', - name="shape") - out_1 = layers.pad2d(input, - paddings=paddings, - mode='reflect', - data_format='NCHW', - name="shape") - return (out) - return (out_1) + out = layers.pad2d( + input, + paddings=[1, 2, 3, 4], + mode='reflect', + data_format='NCHW', + name="shape", + ) + out_1 = layers.pad2d( + input, + paddings=paddings, + mode='reflect', + data_format='NCHW', + name="shape", + ) + return out + return out_1 def make_prelu(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[5, 200, 100, 100], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[5, 200, 100, 100], dtype="float32" + ) mode = 'channel' - out = layers.prelu(input, - mode, - param_attr=ParamAttr(initializer=Constant(1.0)), - name='prelu') - return (out) + out = layers.prelu( + input, + mode, + param_attr=ParamAttr(initializer=Constant(1.0)), + name='prelu', + ) + return out def make_soft_relu(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.soft_relu(input, threshold=30.0, name='soft_relu') - return (out) + return out def make_sigmoid(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sigmoid(input, name='sigmoid') - return (out) + return out def make_exp(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.exp(input, name='exp') - return (out) + return out def make_tanh(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.tanh(input, name='tanh') - return (out) + return out def make_tanh_shrink(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.tanh_shrink(input, name='tanh_shrink') - return (out) + return out def make_sqrt(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sqrt(input, name='sqrt') - return (out) + return out def make_abs(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.abs(input, name='abs') - return (out) + return out def make_ceil(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.ceil(input, name='ceil') - return (out) + return out def make_floor(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.floor(input, name='floor') - return (out) + return out def make_cos(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.cos(input, name='cos') - return (out) + return out def make_sin(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sin(input, name='sin') - return (out) + return out def make_round(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.round(input, name='round') - return (out) + return out def make_reciprocal(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.reciprocal(input, name='reciprocal') - return (out) + return out def make_square(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.square(input, name='square') - return (out) + return out def make_softplus(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softplus(input, name='softplus') - return (out) + return out def make_softsign(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softsign(input, name='softsign') - return (out) + return out def make_mish(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.mish(input, name='mish') - return (out) + return out def make_cross_entropy(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="x", shape=[30, 10], dtype="float32") label = self._get_data(name="label", shape=[30, 1], dtype="int64") mode = 'channel' out = layers.cross_entropy(x, label, False, 4) - return (out) + return out def make_bpr_loss(self): self._force_to_use_cpu = True @@ -3629,173 +4051,196 @@ class TestBook(LayerTest): x = self._get_data(name="x", shape=[30, 10], dtype="float32") label = self._get_data(name="label", shape=[30, 1], dtype="int64") out = layers.bpr_loss(x, label) - return (out) + return out def make_expand(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="input", shape=[10], dtype='int32') out = layers.expand(x, [1, 2]) return out def make_uniform_random_batch_size_like(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[13, 11], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32' + ) out = layers.uniform_random_batch_size_like(input, [-1, 11]) - return (out) + return out def make_gaussian_random(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): out = layers.gaussian_random(shape=[20, 30]) - return (out) + return out def make_sampling_id(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name="X", - shape=[13, 11], - dtype='float32', - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name="X", + shape=[13, 11], + dtype='float32', + append_batch_size=False, + ) out = layers.sampling_id(x) - return (out) + return out def make_gaussian_random_batch_size_like(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[13, 11], - dtype='float32') - - out = layers.gaussian_random_batch_size_like(input, - shape=[-1, 11], - mean=1.0, - std=2.0) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32' + ) + + out = layers.gaussian_random_batch_size_like( + input, shape=[-1, 11], mean=1.0, std=2.0 + ) + return out def make_sum(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[13, 11], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32' + ) out = layers.sum(input) - return (out) + return out def make_slice(self): starts = [1, 0, 2] ends = [3, 3, 4] axes = [0, 1, 2] - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[3, 4, 5, 6], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[3, 4, 5, 6], dtype='float32' + ) out = layers.slice(input, axes=axes, starts=starts, ends=ends) return out def make_scale_variable(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[3, 4, 5, 6], - dtype='float32') - scale_var = self._get_data(name="scale", - shape=[1], - dtype='float32', - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[3, 4, 5, 6], dtype='float32' + ) + scale_var = self._get_data( + name="scale", + shape=[1], + dtype='float32', + append_batch_size=False, + ) out = layers.scale(input, scale=scale_var) return out def make_softshrink(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softshrink(input, alpha=0.3) - return (out) + return out def make_iou_similarity(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="x", shape=[4], dtype="float32") y = self._get_data(name="y", shape=[4], dtype="float32") out = layers.iou_similarity(x, y, name='iou_similarity') - return (out) + return out def make_grid_sampler(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 5, 7], dtype='float32') grid = self._get_data(name='grid', shape=[5, 7, 2], dtype='float32') out = layers.grid_sampler(x, grid) - return (out) + return out def make_bilinear_tensor_product_layer(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name='data', shape=[4], dtype="float32") theta = self._get_data(name="theta", shape=[5], dtype="float32") out = layers.bilinear_tensor_product(data, theta, 6) - return (out) + return out def make_batch_norm(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 128, 128], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32" + ) out = layers.batch_norm(data) - return (out) + return out def make_batch_norm_momentum_variable(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 128, 128], - dtype="float32") - momentum = self._get_data(name='momentum', - shape=[1], - dtype='float32', - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32" + ) + momentum = self._get_data( + name='momentum', + shape=[1], + dtype='float32', + append_batch_size=False, + ) out = layers.batch_norm(data, momentum=momentum) - return (out) + return out def make_inplace_abn(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 128, 128], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32" + ) out = layers.inplace_abn(data, act='leaky_relu', act_alpha=0.2) - return (out) + return out def make_inplace_abn_momentum_variable(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 128, 128], - dtype="float32") - momentum = self._get_data(name='momentum', - shape=[1], - dtype='float32', - append_batch_size=False) - out = layers.inplace_abn(data, - momentum=momentum, - act='elu', - act_alpha=2.0) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32" + ) + momentum = self._get_data( + name='momentum', + shape=[1], + dtype='float32', + append_batch_size=False, + ) + out = layers.inplace_abn( + data, momentum=momentum, act='elu', act_alpha=2.0 + ) + return out def make_range(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): layers.range(0, 10, 2, 'int32') layers.range(0.1, 10.0, 0.2, 'float32') layers.range(0.1, 10.0, 0.2, 'float64') @@ -3806,87 +4251,101 @@ class TestBook(LayerTest): return y def make_spectral_norm(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - weight = self._get_data(name='weight', - shape=[2, 3, 32, 32], - dtype="float32", - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + weight = self._get_data( + name='weight', + shape=[2, 3, 32, 32], + dtype="float32", + append_batch_size=False, + ) out = layers.spectral_norm(weight, dim=1, power_iters=1) - return (out) + return out def make_kldiv_loss(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[32, 128, 128], - dtype="float32", - append_batch_size=False) - target = self._get_data(name='target', - shape=[32, 128, 128], - dtype="float32", - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', + shape=[32, 128, 128], + dtype="float32", + append_batch_size=False, + ) + target = self._get_data( + name='target', + shape=[32, 128, 128], + dtype="float32", + append_batch_size=False, + ) loss = layers.kldiv_loss(x=x, target=target, reduction='batchmean') - return (loss) + return loss def make_temporal_shift(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") out = layers.temporal_shift(x, seg_num=2, shift_ratio=0.2) - return (out) + return out def make_shuffle_channel(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") out = layers.shuffle_channel(x, group=4) - return (out) + return out def make_fsp_matrix(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") y = self._get_data(name="Y", shape=[8, 4, 4], dtype="float32") out = layers.fsp_matrix(x, y) - return (out) + return out def make_pixel_shuffle(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[9, 4, 4], dtype="float32") out = layers.pixel_shuffle(x, upscale_factor=3) - return (out) + return out def make_mse_loss(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[1], dtype="float32") y = self._get_data(name="Y", shape=[1], dtype="float32") out = layers.mse_loss(input=x, label=y) - return (out) + return out def make_square_error_cost(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[1], dtype="float32") y = self._get_data(name="Y", shape=[1], dtype="float32") out = layers.square_error_cost(input=x, label=y) - return (out) + return out def test_dynamic_lstmp(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): hidden_dim, proj_dim = 16, 8 - seq_data = layers.data(name='seq_data', - shape=[10, 10], - dtype='float32', - lod_level=1) + seq_data = layers.data( + name='seq_data', shape=[10, 10], dtype='float32', lod_level=1 + ) fc_out = layers.fc(input=seq_data, size=4 * hidden_dim) self.assertIsNotNone( - layers.dynamic_lstmp(input=fc_out, - size=4 * hidden_dim, - proj_size=proj_dim)) + layers.dynamic_lstmp( + input=fc_out, size=4 * hidden_dim, proj_size=proj_dim + ) + ) def test_linear_chain_crf(self): with self.static_graph(): @@ -3894,63 +4353,71 @@ class TestBook(LayerTest): feature = layers.data(name='feature', shape=[784], dtype='float32') label = layers.data(name='label', shape=[1], dtype='int64') emission = layers.fc(input=feature, size=10) - crf = layers.linear_chain_crf(input=emission, - label=label, - param_attr=ParamAttr(name="crfw")) - crf_decode = layers.crf_decoding(input=emission, - param_attr=ParamAttr(name="crfw")) + crf = layers.linear_chain_crf( + input=emission, label=label, param_attr=ParamAttr(name="crfw") + ) + crf_decode = layers.crf_decoding( + input=emission, param_attr=ParamAttr(name="crfw") + ) self.assertFalse(crf is None) self.assertFalse(crf_decode is None) - return layers.chunk_eval(input=crf_decode, - label=label, - chunk_scheme="IOB", - num_chunk_types=(label_dict_len - 1) // 2) + return layers.chunk_eval( + input=crf_decode, + label=label, + chunk_scheme="IOB", + num_chunk_types=(label_dict_len - 1) // 2, + ) def test_linear_chain_crf_padding(self): with self.static_graph(): label_dict_len, max_len = 10, 20 - feature = layers.data(name='feature', - shape=[max_len, 784], - dtype='float32') + feature = layers.data( + name='feature', shape=[max_len, 784], dtype='float32' + ) label = layers.data(name='label', shape=[max_len], dtype='int64') length = layers.data(name='length', shape=[1], dtype='int64') emission = layers.fc(input=feature, size=10, num_flatten_dims=2) - crf = layers.linear_chain_crf(input=emission, - label=label, - length=length, - param_attr=ParamAttr(name="crfw")) - crf_decode = layers.crf_decoding(input=emission, - length=length, - param_attr=ParamAttr(name="crfw")) + crf = layers.linear_chain_crf( + input=emission, + label=label, + length=length, + param_attr=ParamAttr(name="crfw"), + ) + crf_decode = layers.crf_decoding( + input=emission, length=length, param_attr=ParamAttr(name="crfw") + ) self.assertFalse(crf is None) self.assertFalse(crf_decode is None) - return layers.chunk_eval(input=crf_decode, - label=label, - seq_length=length, - chunk_scheme="IOB", - num_chunk_types=(label_dict_len - 1) // 2) + return layers.chunk_eval( + input=crf_decode, + label=label, + seq_length=length, + chunk_scheme="IOB", + num_chunk_types=(label_dict_len - 1) // 2, + ) def test_im2sequence(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[3, 128, 128], dtype='float32') y = layers.data(name='y', shape=[], dtype='float32') - output = layers.im2sequence(input=x, - input_image_size=y, - stride=[1, 1], - filter_size=[2, 2], - out_stride=[1, 1]) - return (output) + output = layers.im2sequence( + input=x, + input_image_size=y, + stride=[1, 1], + filter_size=[2, 2], + out_stride=[1, 1], + ) + return output def test_lod_reset(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): # case 1 x = layers.data(name='x', shape=[10], dtype='float32') - y = layers.data(name='y', - shape=[10, 20], - dtype='float32', - lod_level=2) + y = layers.data( + name='y', shape=[10, 20], dtype='float32', lod_level=2 + ) z = layers.lod_reset(x=x, y=y) self.assertTrue(z.lod_level == 2) # case 2 @@ -3982,153 +4449,152 @@ class TestBook(LayerTest): strides = [1, 1, 1] with self.static_graph(): x = layers.data(name="x", shape=[245, 30, 30], dtype="float32") - out = layers.strided_slice(x, - axes=axes, - starts=starts, - ends=ends, - strides=strides) + out = layers.strided_slice( + x, axes=axes, starts=starts, ends=ends, strides=strides + ) return out def test_fill_constant_batch_size_like(self): with self.static_graph(): - like = fluid.layers.fill_constant(shape=[1, 200], - value=10, - dtype='int64') - out = layers.fill_constant_batch_size_like(input=like, - shape=[2, 3300], - value=1315454564656, - dtype='int64') + like = fluid.layers.fill_constant( + shape=[1, 200], value=10, dtype='int64' + ) + out = layers.fill_constant_batch_size_like( + input=like, shape=[2, 3300], value=1315454564656, dtype='int64' + ) return out def test_psroi_pool(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name="x", shape=[245, 30, 30], dtype="float32") - rois = layers.data(name="rois", - shape=[4], - dtype="float32", - lod_level=1) + rois = layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1 + ) output = layers.psroi_pool(x, rois, 5, 0.25, 7, 7) - return (output) + return output def test_sequence_expand(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[10], dtype='float32') - y = layers.data(name='y', - shape=[10, 20], - dtype='float32', - lod_level=2) - return (layers.sequence_expand(x=x, y=y, ref_level=1)) + y = layers.data( + name='y', shape=[10, 20], dtype='float32', lod_level=2 + ) + return layers.sequence_expand(x=x, y=y, ref_level=1) def test_sequence_reshape(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[8], dtype='float32', lod_level=1) out = layers.sequence_reshape(input=x, new_dim=16) - return (out) + return out def test_sequence_unpad(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[10, 5], dtype='float32') length = layers.data(name='length', shape=[], dtype='int64') - return (layers.sequence_unpad(x=x, length=length)) + return layers.sequence_unpad(x=x, length=length) def test_sequence_softmax(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - seq_data = layers.data(name='seq_data', - shape=[10, 10], - dtype='float32', - lod_level=1) + seq_data = layers.data( + name='seq_data', shape=[10, 10], dtype='float32', lod_level=1 + ) seq = layers.fc(input=seq_data, size=20) - return (layers.sequence_softmax(seq)) + return layers.sequence_softmax(seq) def test_sequence_unsqueeze(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[8, 2], dtype='float32') out = layers.unsqueeze(input=x, axes=[1]) - return (out) + return out def test_sequence_scatter(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - x = layers.data(name='x', - shape=[3, 6], - append_batch_size=False, - dtype='float32') - idx = layers.data(name='idx', - shape=[12, 1], - append_batch_size=False, - dtype='int32', - lod_level=1) - updates = layers.data(name='updates', - shape=[12, 1], - append_batch_size=False, - dtype='float32', - lod_level=1) + x = layers.data( + name='x', shape=[3, 6], append_batch_size=False, dtype='float32' + ) + idx = layers.data( + name='idx', + shape=[12, 1], + append_batch_size=False, + dtype='int32', + lod_level=1, + ) + updates = layers.data( + name='updates', + shape=[12, 1], + append_batch_size=False, + dtype='float32', + lod_level=1, + ) out = layers.sequence_scatter(input=x, index=idx, updates=updates) - return (out) + return out def test_sequence_slice(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): import numpy as np - seqs = layers.data(name='x', - shape=[10, 5], - dtype='float32', - lod_level=1) + + seqs = layers.data( + name='x', shape=[10, 5], dtype='float32', lod_level=1 + ) offset = layers.assign(input=np.array([[0, 1]]).astype('int32')) length = layers.assign(input=np.array([[2, 1]]).astype('int32')) - out = layers.sequence_slice(input=seqs, - offset=offset, - length=length) - return (out) + out = layers.sequence_slice( + input=seqs, offset=offset, length=length + ) + return out def test_filter_by_instag(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - x1 = layers.data(name='Ins', - shape=[32, 1], - dtype='float32', - lod_level=0) - x2 = layers.data(name='Ins_tag', - shape=[32, 1], - dtype='int64', - lod_level=0, - stop_gradient=True) - x3 = layers.create_global_var(shape=[1, 1], - value=20, - dtype='int64', - persistable=True, - force_cpu=True, - name='Filter_tag') + x1 = layers.data( + name='Ins', shape=[32, 1], dtype='float32', lod_level=0 + ) + x2 = layers.data( + name='Ins_tag', + shape=[32, 1], + dtype='int64', + lod_level=0, + stop_gradient=True, + ) + x3 = layers.create_global_var( + shape=[1, 1], + value=20, + dtype='int64', + persistable=True, + force_cpu=True, + name='Filter_tag', + ) out1, out2 = layers.filter_by_instag(x1, x2, x3, is_lod=True) def test_shuffle_batch(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - x = layers.data(name='X', - shape=[4, 50], - dtype='float32', - lod_level=0) + x = layers.data( + name='X', shape=[4, 50], dtype='float32', lod_level=0 + ) out1 = fluid.contrib.layers.shuffle_batch(x) default_main_program().random_seed = 1000 out2 = fluid.contrib.layers.shuffle_batch(x) self.assertIsNotNone(out1) self.assertIsNotNone(out2) - return (out1) + return out1 def test_partial_sum(self): with self.static_graph(): x = fluid.data(name="x", shape=[None, 3], dtype="float32") y = fluid.data(name="y", shape=[None, 3], dtype="float32") - sum = fluid.contrib.layers.partial_sum([x, y], - start_index=0, - length=2) - return (sum) + sum = fluid.contrib.layers.partial_sum( + [x, y], start_index=0, length=2 + ) + return sum def test_batch_fc(self): with self.static_graph(): @@ -4139,21 +4605,24 @@ class TestBook(LayerTest): param_attr=fluid.ParamAttr( learning_rate=1.0, name="w_0", - initializer=fluid.initializer.Xavier(uniform=False)), + initializer=fluid.initializer.Xavier(uniform=False), + ), bias_size=[16, 10], bias_attr=fluid.ParamAttr( learning_rate=1.0, name="b_0", - initializer=fluid.initializer.Xavier(uniform=False)), - act="relu") - return (out) + initializer=fluid.initializer.Xavier(uniform=False), + ), + act="relu", + ) + return out def test_rank_attention(self): with self.static_graph(): input = fluid.data(name="input", shape=[None, 2], dtype="float32") - rank_offset = fluid.data(name="rank_offset", - shape=[None, 7], - dtype="int32") + rank_offset = fluid.data( + name="rank_offset", shape=[None, 7], dtype="int32" + ) out = fluid.contrib.layers.rank_attention( input=input, rank_offset=rank_offset, @@ -4161,9 +4630,11 @@ class TestBook(LayerTest): rank_param_attr=fluid.ParamAttr( learning_rate=1.0, name="ubm_rank_param.w_0", - initializer=fluid.initializer.Xavier(uniform=False)), - max_rank=3) - return (out) + initializer=fluid.initializer.Xavier(uniform=False), + ), + max_rank=3, + ) + return out def test_roi_pool(self): x_np = np.random.rand(2, 3, 8, 8).astype('float32') @@ -4175,35 +4646,27 @@ class TestBook(LayerTest): rois = layers.data(name="rois", shape=[4], dtype="float32") rois_num = fluid.data(name="rois_num", shape=[None], dtype="int32") output = layers.roi_pool(x, rois, 4, 4, 0.5, rois_num=rois_num) - static_res = self.get_static_graph_result(feed={ - 'x': x_np, - 'rois': rois_np, - 'rois_num': rois_num_np - }, - fetch_list=[output])[0] + static_res = self.get_static_graph_result( + feed={'x': x_np, 'rois': rois_np, 'rois_num': rois_num_np}, + fetch_list=[output], + )[0] with self.dynamic_graph(): with _test_eager_guard(): x_dy = base.to_variable(x_np) rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - dy_eager_res = layers.roi_pool(x_dy, - rois_dy, - 4, - 4, - 0.5, - rois_num=rois_num_dy) + dy_eager_res = layers.roi_pool( + x_dy, rois_dy, 4, 4, 0.5, rois_num=rois_num_dy + ) dy_eager_res_value = dy_eager_res[0].numpy() x_dy = base.to_variable(x_np) rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - dy_res = layers.roi_pool(x_dy, - rois_dy, - 4, - 4, - 0.5, - rois_num=rois_num_dy) + dy_res = layers.roi_pool( + x_dy, rois_dy, 4, 4, 0.5, rois_num=rois_num_dy + ) dy_res_value = dy_res[0].numpy() np.testing.assert_array_equal(static_res, dy_res_value) np.testing.assert_array_equal(static_res, dy_eager_res_value) @@ -4224,37 +4687,27 @@ class TestBook(LayerTest): rois = layers.data(name="rois", shape=[4], dtype="float32") rois_num = fluid.data(name="rois_num", shape=[None], dtype="int32") output = layers.roi_align(x, rois, 4, 4, 0.5, 2, rois_num=rois_num) - static_res = self.get_static_graph_result(feed={ - 'x': x_np, - 'rois': rois_np, - 'rois_num': rois_num_np - }, - fetch_list=[output])[0] + static_res = self.get_static_graph_result( + feed={'x': x_np, 'rois': rois_np, 'rois_num': rois_num_np}, + fetch_list=[output], + )[0] with self.dynamic_graph(): with _test_eager_guard(): x_dy = base.to_variable(x_np) rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - dy_eager_res = layers.roi_align(x_dy, - rois_dy, - 4, - 4, - 0.5, - 2, - rois_num=rois_num_dy) + dy_eager_res = layers.roi_align( + x_dy, rois_dy, 4, 4, 0.5, 2, rois_num=rois_num_dy + ) dy_eager_res_value = dy_eager_res.numpy() x_dy = base.to_variable(x_np) rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - dy_res = layers.roi_align(x_dy, - rois_dy, - 4, - 4, - 0.5, - 2, - rois_num=rois_num_dy) + dy_res = layers.roi_align( + x_dy, rois_dy, 4, 4, 0.5, 2, rois_num=rois_num_dy + ) dy_res_value = dy_res.numpy() np.testing.assert_array_equal(static_res, dy_eager_res_value) np.testing.assert_array_equal(static_res, dy_res_value) @@ -4266,18 +4719,16 @@ class TestBook(LayerTest): label_np = np.random.randint(0, num_classes, [2, 3, 1], dtype=np.int64) with self.static_graph(): - input_ = layers.data(name="input", - shape=[None, 3, num_classes], - dtype="float32") - label_ = layers.data(name="label", - shape=[None, 3, 1], - dtype="int64") + input_ = layers.data( + name="input", shape=[None, 3, num_classes], dtype="float32" + ) + label_ = layers.data( + name="label", shape=[None, 3, 1], dtype="int64" + ) output = layers.dice_loss(input_, label_, eps) - static_res = self.get_static_graph_result(feed={ - 'input': input_np, - 'label': label_np - }, - fetch_list=[output])[0] + static_res = self.get_static_graph_result( + feed={'input': input_np, 'label': label_np}, fetch_list=[output] + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -4297,46 +4748,47 @@ class TestBook(LayerTest): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") - rois = layers.data(name="rois", - shape=[8], - dtype="float32", - lod_level=1) + rois = layers.data( + name="rois", shape=[8], dtype="float32", lod_level=1 + ) output = layers.roi_perspective_transform(x, rois, 7, 7, 0.6) - return (output) + return output def test_row_conv(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[16], dtype='float32', lod_level=1) out = layers.row_conv(input=x, future_context_size=2) - return (out) + return out def test_simple_conv2d(self): # TODO(minqiyang): dygraph do not support layers with param now with self.static_graph(): - images = layers.data(name='pixel', - shape=[3, 48, 48], - dtype='float32') - return layers.conv2d(input=images, - num_filters=3, - filter_size=[4, 4]) + images = layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32' + ) + return layers.conv2d( + input=images, num_filters=3, filter_size=[4, 4] + ) def test_squeeze(self): # TODO(minqiyang): dygraph do not support layers with param now with self.static_graph(): x = layers.data(name='x', shape=[1, 1, 4], dtype='float32') out = layers.squeeze(input=x, axes=[2]) - return (out) + return out def test_flatten(self): # TODO(minqiyang): dygraph do not support op without kernel now with self.static_graph(): - x = layers.data(name='x', - append_batch_size=False, - shape=[4, 4, 3], - dtype="float32") + x = layers.data( + name='x', + append_batch_size=False, + shape=[4, 4, 3], + dtype="float32", + ) out = layers.flatten(x, axis=1, name="flatten") - return (out) + return out def test_linspace(self): program = Program() @@ -4347,211 +4799,270 @@ class TestBook(LayerTest): def test_deformable_conv(self): with self.static_graph(): - input = layers.data(name='input', - append_batch_size=False, - shape=[2, 3, 32, 32], - dtype="float32") - offset = layers.data(name='offset', - append_batch_size=False, - shape=[2, 18, 32, 32], - dtype="float32") - mask = layers.data(name='mask', - append_batch_size=False, - shape=[2, 9, 32, 32], - dtype="float32") - out = layers.deformable_conv(input=input, - offset=offset, - mask=mask, - num_filters=2, - filter_size=3, - padding=1) - return (out) + input = layers.data( + name='input', + append_batch_size=False, + shape=[2, 3, 32, 32], + dtype="float32", + ) + offset = layers.data( + name='offset', + append_batch_size=False, + shape=[2, 18, 32, 32], + dtype="float32", + ) + mask = layers.data( + name='mask', + append_batch_size=False, + shape=[2, 9, 32, 32], + dtype="float32", + ) + out = layers.deformable_conv( + input=input, + offset=offset, + mask=mask, + num_filters=2, + filter_size=3, + padding=1, + ) + return out def test_deformable_conv2(self): with self.static_graph(): - input = fluid.data(name='input', - shape=[None, 3, None, None], - dtype="float32") - offset = fluid.data(name='offset', - shape=[None, 18, None, None], - dtype="float32") - mask = fluid.data(name='mask', - shape=[None, 9, None, None], - dtype="float32") - out = layers.deformable_conv(input=input, - offset=offset, - mask=mask, - num_filters=2, - filter_size=3, - padding=1) - return (out) + input = fluid.data( + name='input', shape=[None, 3, None, None], dtype="float32" + ) + offset = fluid.data( + name='offset', shape=[None, 18, None, None], dtype="float32" + ) + mask = fluid.data( + name='mask', shape=[None, 9, None, None], dtype="float32" + ) + out = layers.deformable_conv( + input=input, + offset=offset, + mask=mask, + num_filters=2, + filter_size=3, + padding=1, + ) + return out def test_unfold(self): with self.static_graph(): x = layers.data(name='x', shape=[3, 20, 20], dtype='float32') out = layers.unfold(x, [3, 3], 1, 1, 1) - return (out) + return out def test_partial_concat(self): with self.static_graph(): x = fluid.data(name="x", shape=[None, 3], dtype="float32") y = fluid.data(name="y", shape=[None, 3], dtype="float32") - concat1 = fluid.contrib.layers.partial_concat([x, y], - start_index=0, - length=2) - concat2 = fluid.contrib.layers.partial_concat(x, - start_index=0, - length=-1) + concat1 = fluid.contrib.layers.partial_concat( + [x, y], start_index=0, length=2 + ) + concat2 = fluid.contrib.layers.partial_concat( + x, start_index=0, length=-1 + ) return concat1, concat2 def test_deform_roi_pooling(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = layers.data(name='input', - shape=[2, 3, 32, 32], - dtype='float32', - append_batch_size=False) - rois = layers.data(name="rois", - shape=[4], - dtype='float32', - lod_level=1) - trans = layers.data(name="trans", - shape=[2, 3, 32, 32], - dtype='float32', - append_batch_size=False) - out = layers.deformable_roi_pooling(input=input, - rois=rois, - trans=trans, - no_trans=False, - spatial_scale=1.0, - group_size=(1, 1), - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - trans_std=0.1) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = layers.data( + name='input', + shape=[2, 3, 32, 32], + dtype='float32', + append_batch_size=False, + ) + rois = layers.data( + name="rois", shape=[4], dtype='float32', lod_level=1 + ) + trans = layers.data( + name="trans", + shape=[2, 3, 32, 32], + dtype='float32', + append_batch_size=False, + ) + out = layers.deformable_roi_pooling( + input=input, + rois=rois, + trans=trans, + no_trans=False, + spatial_scale=1.0, + group_size=(1, 1), + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + trans_std=0.1, + ) + return out def test_deformable_conv_v1(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = layers.data(name='input', - append_batch_size=False, - shape=[2, 3, 32, 32], - dtype="float32") - offset = layers.data(name='offset', - append_batch_size=False, - shape=[2, 18, 32, 32], - dtype="float32") - out = layers.deformable_conv(input=input, - offset=offset, - mask=None, - num_filters=2, - filter_size=3, - padding=1, - modulated=False) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = layers.data( + name='input', + append_batch_size=False, + shape=[2, 3, 32, 32], + dtype="float32", + ) + offset = layers.data( + name='offset', + append_batch_size=False, + shape=[2, 18, 32, 32], + dtype="float32", + ) + out = layers.deformable_conv( + input=input, + offset=offset, + mask=None, + num_filters=2, + filter_size=3, + padding=1, + modulated=False, + ) + return out def test_retinanet_target_assign(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - bbox_pred = layers.data(name='bbox_pred', - shape=[1, 100, 4], - append_batch_size=False, - dtype='float32') - cls_logits = layers.data(name='cls_logits', - shape=[1, 100, 10], - append_batch_size=False, - dtype='float32') - anchor_box = layers.data(name='anchor_box', - shape=[100, 4], - append_batch_size=False, - dtype='float32') - anchor_var = layers.data(name='anchor_var', - shape=[100, 4], - append_batch_size=False, - dtype='float32') - gt_boxes = layers.data(name='gt_boxes', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - gt_labels = layers.data(name='gt_labels', - shape=[10, 1], - append_batch_size=False, - dtype='int32') - is_crowd = layers.data(name='is_crowd', - shape=[1], - append_batch_size=False, - dtype='int32') - im_info = layers.data(name='im_info', - shape=[1, 3], - append_batch_size=False, - dtype='float32') - return (layers.retinanet_target_assign(bbox_pred, cls_logits, - anchor_box, anchor_var, - gt_boxes, gt_labels, - is_crowd, im_info, 10)) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + bbox_pred = layers.data( + name='bbox_pred', + shape=[1, 100, 4], + append_batch_size=False, + dtype='float32', + ) + cls_logits = layers.data( + name='cls_logits', + shape=[1, 100, 10], + append_batch_size=False, + dtype='float32', + ) + anchor_box = layers.data( + name='anchor_box', + shape=[100, 4], + append_batch_size=False, + dtype='float32', + ) + anchor_var = layers.data( + name='anchor_var', + shape=[100, 4], + append_batch_size=False, + dtype='float32', + ) + gt_boxes = layers.data( + name='gt_boxes', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + gt_labels = layers.data( + name='gt_labels', + shape=[10, 1], + append_batch_size=False, + dtype='int32', + ) + is_crowd = layers.data( + name='is_crowd', + shape=[1], + append_batch_size=False, + dtype='int32', + ) + im_info = layers.data( + name='im_info', + shape=[1, 3], + append_batch_size=False, + dtype='float32', + ) + return layers.retinanet_target_assign( + bbox_pred, + cls_logits, + anchor_box, + anchor_var, + gt_boxes, + gt_labels, + is_crowd, + im_info, + 10, + ) def test_sigmoid_focal_loss(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = layers.data(name='data', - shape=[10, 80], - append_batch_size=False, - dtype='float32') - label = layers.data(name='label', - shape=[10, 1], - append_batch_size=False, - dtype='int32') - fg_num = layers.data(name='fg_num', - shape=[1], - append_batch_size=False, - dtype='int32') - out = fluid.layers.sigmoid_focal_loss(x=input, - label=label, - fg_num=fg_num, - gamma=2., - alpha=0.25) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = layers.data( + name='data', + shape=[10, 80], + append_batch_size=False, + dtype='float32', + ) + label = layers.data( + name='label', + shape=[10, 1], + append_batch_size=False, + dtype='int32', + ) + fg_num = layers.data( + name='fg_num', shape=[1], append_batch_size=False, dtype='int32' + ) + out = fluid.layers.sigmoid_focal_loss( + x=input, label=label, fg_num=fg_num, gamma=2.0, alpha=0.25 + ) + return out def test_addmm(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = layers.data(name='input_data', - shape=[3, 3], - append_batch_size=False, - dtype='float32') - x = layers.data(name='x', - shape=[3, 2], - append_batch_size=False, - dtype='float32') - y = layers.data(name='y', - shape=[2, 3], - append_batch_size=False, - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = layers.data( + name='input_data', + shape=[3, 3], + append_batch_size=False, + dtype='float32', + ) + x = layers.data( + name='x', shape=[3, 2], append_batch_size=False, dtype='float32' + ) + y = layers.data( + name='y', shape=[2, 3], append_batch_size=False, dtype='float32' + ) out = paddle.addmm(input=input, x=x, y=y) - return (out) + return out def test_retinanet_detection_output(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - bboxes = layers.data(name='bboxes', - shape=[1, 21, 4], - append_batch_size=False, - dtype='float32') - scores = layers.data(name='scores', - shape=[1, 21, 10], - append_batch_size=False, - dtype='float32') - anchors = layers.data(name='anchors', - shape=[21, 4], - append_batch_size=False, - dtype='float32') - im_info = layers.data(name="im_info", - shape=[1, 3], - append_batch_size=False, - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + bboxes = layers.data( + name='bboxes', + shape=[1, 21, 4], + append_batch_size=False, + dtype='float32', + ) + scores = layers.data( + name='scores', + shape=[1, 21, 10], + append_batch_size=False, + dtype='float32', + ) + anchors = layers.data( + name='anchors', + shape=[21, 4], + append_batch_size=False, + dtype='float32', + ) + im_info = layers.data( + name="im_info", + shape=[1, 3], + append_batch_size=False, + dtype='float32', + ) nmsed_outs = layers.retinanet_detection_output( bboxes=[bboxes, bboxes], scores=[scores, scores], @@ -4561,38 +5072,39 @@ class TestBook(LayerTest): nms_top_k=1000, keep_top_k=100, nms_threshold=0.3, - nms_eta=1.) - return (nmsed_outs) + nms_eta=1.0, + ) + return nmsed_outs def test_warpctc_with_padding(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - input_length = layers.data(name='logits_length', - shape=[11], - dtype='int64') - label_length = layers.data(name='labels_length', - shape=[12], - dtype='int64') + input_length = layers.data( + name='logits_length', shape=[11], dtype='int64' + ) + label_length = layers.data( + name='labels_length', shape=[12], dtype='int64' + ) label = layers.data(name='label', shape=[12, 1], dtype='int32') - predict = layers.data(name='predict', - shape=[4, 4, 8], - dtype='float32') - output = layers.warpctc(input=predict, - label=label, - input_length=input_length, - label_length=label_length) - return (output) + predict = layers.data( + name='predict', shape=[4, 4, 8], dtype='float32' + ) + output = layers.warpctc( + input=predict, + label=label, + input_length=input_length, + label_length=label_length, + ) + return output def test_edit_distance(self): with self.static_graph(): - predict = layers.data(name='predict', - shape=[-1, 1], - dtype='int64', - lod_level=1) - label = layers.data(name='label', - shape=[-1, 1], - dtype='int64', - lod_level=1) + predict = layers.data( + name='predict', shape=[-1, 1], dtype='int64', lod_level=1 + ) + label = layers.data( + name='label', shape=[-1, 1], dtype='int64', lod_level=1 + ) evaluator = fluid.evaluator.EditDistance(predict, label) return evaluator.metrics @@ -4600,15 +5112,15 @@ class TestBook(LayerTest): input_size = 128 hidden_size = 256 with self.static_graph(): - input = fluid.data(name="input", - shape=[None, None, input_size], - dtype='float32') - pre_hidden = fluid.data(name="pre_hidden", - shape=[None, hidden_size], - dtype='float32') - sequence_length = fluid.data(name="sequence_length", - shape=[None], - dtype='int32') + input = fluid.data( + name="input", shape=[None, None, input_size], dtype='float32' + ) + pre_hidden = fluid.data( + name="pre_hidden", shape=[None, hidden_size], dtype='float32' + ) + sequence_length = fluid.data( + name="sequence_length", shape=[None], dtype='int32' + ) for bidirectional in [True, False]: for batch_first in [True, False]: @@ -4620,30 +5132,35 @@ class TestBook(LayerTest): sequence_length=sequence_length, dropout_prob=0.5, bidirectional=bidirectional, - batch_first=batch_first) + batch_first=batch_first, + ) class TestMetricsDetectionMap(unittest.TestCase): - def test_detection_map(self): program = fluid.Program() with program_guard(program): - detect_res = fluid.layers.data(name='detect_res', - shape=[10, 6], - append_batch_size=False, - dtype='float32') - label = fluid.layers.data(name='label', - shape=[10, 1], - append_batch_size=False, - dtype='float32') - box = fluid.layers.data(name='bbox', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - map_eval = fluid.metrics.DetectionMAP(detect_res, - label, - box, - class_num=21) + detect_res = fluid.layers.data( + name='detect_res', + shape=[10, 6], + append_batch_size=False, + dtype='float32', + ) + label = fluid.layers.data( + name='label', + shape=[10, 1], + append_batch_size=False, + dtype='float32', + ) + box = fluid.layers.data( + name='bbox', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + map_eval = fluid.metrics.DetectionMAP( + detect_res, label, box, class_num=21 + ) cur_map, accm_map = map_eval.get_map_var() self.assertIsNotNone(cur_map) self.assertIsNotNone(accm_map) @@ -4651,11 +5168,11 @@ class TestMetricsDetectionMap(unittest.TestCase): class ExampleNet(paddle.nn.Layer): - def __init__(self): super(ExampleNet, self).__init__() self.weight = self.create_parameter( - shape=[1, 1], attr=paddle.ParamAttr(trainable=False)) + shape=[1, 1], attr=paddle.ParamAttr(trainable=False) + ) def forward(self): # only for test parameter trainable attr @@ -4663,7 +5180,6 @@ class ExampleNet(paddle.nn.Layer): class TestLayerParameterTrainableSet(unittest.TestCase): - def test_layer_parameter_set(self): with fluid.dygraph.guard(): net = ExampleNet() @@ -4671,7 +5187,6 @@ class TestLayerParameterTrainableSet(unittest.TestCase): class TestLayerTrainingAttribute(unittest.TestCase): - def test_set_train_eval_in_dynamic_mode(self): with fluid.dygraph.guard(): net = paddle.nn.Dropout() @@ -4689,7 +5204,6 @@ class TestLayerTrainingAttribute(unittest.TestCase): class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() self._linear = paddle.nn.Linear(1, 1) @@ -4702,7 +5216,6 @@ class MyLayer(paddle.nn.Layer): class MySuperLayer(paddle.nn.Layer): - def __init__(self): super(MySuperLayer, self).__init__() self._mylayer = MyLayer() @@ -4713,7 +5226,6 @@ class MySuperLayer(paddle.nn.Layer): class TestSubLayerCount(unittest.TestCase): - def test_sublayer(self): with fluid.dygraph.guard(): mySuperlayer = MySuperLayer() diff --git a/python/paddle/fluid/tests/unittests/test_layout_autotune.py b/python/paddle/fluid/tests/unittests/test_layout_autotune.py index b7af0464ba9..95618edcc38 100644 --- a/python/paddle/fluid/tests/unittests/test_layout_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_layout_autotune.py @@ -23,7 +23,6 @@ import paddle.nn.functional as F class SimpleNet(paddle.nn.Layer): - def __init__(self, data_format="NCHW", class_num=2): super(SimpleNet, self).__init__() self.conv = paddle.nn.Conv2D(3, 8, (3, 3)) @@ -44,7 +43,6 @@ class SimpleNet(paddle.nn.Layer): class LayoutAutoTune(unittest.TestCase): - def test_config(self): paddle.fluid.core.enable_layout_autotune() if self.use_autoune(): @@ -59,9 +57,8 @@ class LayoutAutoTune(unittest.TestCase): def use_autoune(self): if paddle.is_compiled_with_cuda(): paddle.incubate.autotune.set_config( - config={"layout": { - "enable": True - }}) + config={"layout": {"enable": True}} + ) return paddle.fluid.core.use_layout_autotune() else: config = {"layout": {"enable": False}} @@ -75,11 +72,12 @@ class LayoutAutoTune(unittest.TestCase): def train(self, data_format): model = SimpleNet(data_format="NCHW", class_num=2) data = paddle.rand([1, 3, 16, 16]) - if (data_format == "NHWC"): + if data_format == "NHWC": data = paddle.rand([1, 16, 16, 3]) label_data = paddle.randint(0, 1, shape=[1, 1], dtype="int64") - optimizer = paddle.optimizer.SGD(learning_rate=0.0001, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.0001, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler() for i in range(2): with paddle.amp.auto_cast(level="O2"): @@ -101,8 +99,9 @@ class LayoutAutoTune(unittest.TestCase): conv = paddle.nn.Conv2D(3, 8, (3, 3)) data = paddle.rand([1, 3, 16, 14]) label_data = paddle.randint(0, 1, shape=[1, 1], dtype="int64") - optimizer = paddle.optimizer.SGD(learning_rate=0.0001, - parameters=conv.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.0001, parameters=conv.parameters() + ) scaler = paddle.amp.GradScaler() with paddle.amp.auto_cast(level="O2"): conv_out = conv(data) @@ -170,7 +169,6 @@ class LayoutAutoTune(unittest.TestCase): class TestAutoTuneAPI(unittest.TestCase): - def test_set_config_warnings(self): with warnings.catch_warnings(record=True) as w: config = {"layout": {"enable": 1}} diff --git a/python/paddle/fluid/tests/unittests/test_lazy_init.py b/python/paddle/fluid/tests/unittests/test_lazy_init.py index 79b3429b10e..914fc141261 100644 --- a/python/paddle/fluid/tests/unittests/test_lazy_init.py +++ b/python/paddle/fluid/tests/unittests/test_lazy_init.py @@ -17,12 +17,18 @@ import unittest import numpy as np from paddle import LazyGuard from paddle.nn import Linear, Layer -from paddle.nn.initializer import Constant, Normal, TruncatedNormal, Uniform, XavierNormal, XavierUniform +from paddle.nn.initializer import ( + Constant, + Normal, + TruncatedNormal, + Uniform, + XavierNormal, + XavierUniform, +) from paddle.fluid import unique_name class TestInitializerBase(unittest.TestCase): - def setUp(self): self.set_initializer() self.set_param_attr() @@ -34,11 +40,13 @@ class TestInitializerBase(unittest.TestCase): self.b_initializer = Constant(0.3) def set_param_attr(self): - self.weight_attr = paddle.ParamAttr(name="weight", - initializer=self.w_initializer) + self.weight_attr = paddle.ParamAttr( + name="weight", initializer=self.w_initializer + ) - self.bias_attr = paddle.ParamAttr(name="bias", - initializer=self.b_initializer) + self.bias_attr = paddle.ParamAttr( + name="bias", initializer=self.b_initializer + ) def set_init_ops(self): self.init_ops = ['fill_constant', 'fill_constant'] @@ -48,29 +56,26 @@ class TestInitializerBase(unittest.TestCase): def test_wrapper(self): with LazyGuard(): - fc = Linear(10, - 10, - weight_attr=self.weight_attr, - bias_attr=self.bias_attr) + fc = Linear( + 10, 10, weight_attr=self.weight_attr, bias_attr=self.bias_attr + ) program = fc._startup_program() print(program) self.check_program(program) def check_program(self, program): self.assertEqual(program.block(0).var("weight").shape, (10, 10)) - self.assertEqual(program.block(0).var("bias").shape, (10, )) + self.assertEqual(program.block(0).var("bias").shape, (10,)) ops = [op.type for op in program.block(0).ops] self.assertEqual(ops, self.init_ops) class TestDygraphLazy(TestInitializerBase): - def test_wrapper(self): with LazyGuard(): - fc = Linear(10, - 10, - weight_attr=self.weight_attr, - bias_attr=self.bias_attr) + fc = Linear( + 10, 10, weight_attr=self.weight_attr, bias_attr=self.bias_attr + ) self.check_data(fc) @@ -86,14 +91,15 @@ class TestDygraphLazy(TestInitializerBase): out = model(x) self.assertEqual(out.shape, [2, 10]) - np.testing.assert_allclose(model.weight.numpy(), - np.ones([10, 10], dtype=np.float32) * 0.6) - np.testing.assert_allclose(model.bias.numpy(), - np.ones([10], dtype=np.float32) * 0.3) + np.testing.assert_allclose( + model.weight.numpy(), np.ones([10, 10], dtype=np.float32) * 0.6 + ) + np.testing.assert_allclose( + model.bias.numpy(), np.ones([10], dtype=np.float32) * 0.3 + ) class NestModel(Layer): - def __init__(self, base_model): super(NestModel, self).__init__() self.base_model = base_model @@ -106,13 +112,11 @@ class NestModel(Layer): class TestNestModelLazy(TestInitializerBase): - def test_wrapper(self): with LazyGuard(): - base_model = Linear(10, - 10, - weight_attr=self.weight_attr, - bias_attr=self.bias_attr) + base_model = Linear( + 10, 10, weight_attr=self.weight_attr, bias_attr=self.bias_attr + ) nest_model = NestModel(base_model) self.check_data(nest_model) @@ -130,16 +134,19 @@ class TestNestModelLazy(TestInitializerBase): out = model(x) self.assertEqual(out.shape, [2, 10]) - np.testing.assert_allclose(model.base_model.weight.numpy(), - np.ones([10, 10], dtype=np.float32) * 0.6) - np.testing.assert_allclose(model.base_model.bias.numpy(), - np.ones([10], dtype=np.float32) * 0.3) + np.testing.assert_allclose( + model.base_model.weight.numpy(), + np.ones([10, 10], dtype=np.float32) * 0.6, + ) + np.testing.assert_allclose( + model.base_model.bias.numpy(), np.ones([10], dtype=np.float32) * 0.3 + ) def check_program(self, model): # verify nest_model startup_program whole_program = model._startup_program() self.assertEqual(whole_program.block(0).var("weight").shape, (10, 10)) - self.assertEqual(whole_program.block(0).var("bias").shape, (10, )) + self.assertEqual(whole_program.block(0).var("bias").shape, (10,)) ops = [op.type for op in whole_program.block(0).ops] init_ops = self.init_ops + ['uniform_random', 'fill_constant'] self.assertEqual(ops, init_ops) @@ -147,13 +154,12 @@ class TestNestModelLazy(TestInitializerBase): # verify base_model startup_program sub_program = model.base_model._startup_program() self.assertEqual(sub_program.block(0).var("weight").shape, (10, 10)) - self.assertEqual(sub_program.block(0).var("bias").shape, (10, )) + self.assertEqual(sub_program.block(0).var("bias").shape, (10,)) ops = [op.type for op in sub_program.block(0).ops] self.assertEqual(ops, self.init_ops) class TestUniform(TestInitializerBase): - def set_initializer(self): self.w_initializer = Uniform() self.b_initializer = Uniform() @@ -163,7 +169,6 @@ class TestUniform(TestInitializerBase): class TestNormal(TestInitializerBase): - def set_initializer(self): self.w_initializer = Normal() self.b_initializer = Normal() @@ -173,26 +178,24 @@ class TestNormal(TestInitializerBase): class TestTruncatedNormal(TestInitializerBase): - def set_initializer(self): self.w_initializer = TruncatedNormal() self.b_initializer = TruncatedNormal() def set_init_ops(self): self.init_ops = [ - 'truncated_gaussian_random', 'truncated_gaussian_random' + 'truncated_gaussian_random', + 'truncated_gaussian_random', ] class TestXavierNormal(TestNormal): - def set_initializer(self): self.w_initializer = XavierNormal() self.b_initializer = XavierNormal() class TestXavierUniform(TestUniform): - def set_initializer(self): self.w_initializer = XavierUniform() self.b_initializer = XavierUniform() diff --git a/python/paddle/fluid/tests/unittests/test_lbfgs.py b/python/paddle/fluid/tests/unittests/test_lbfgs.py index 36b01d05e78..21d96d17c2a 100644 --- a/python/paddle/fluid/tests/unittests/test_lbfgs.py +++ b/python/paddle/fluid/tests/unittests/test_lbfgs.py @@ -44,37 +44,35 @@ def test_static_graph_H0(func, x0, H0, dtype='float32'): startup = paddle.static.Program() with paddle.static.program_guard(main, startup): X = paddle.static.data(name='x', shape=[x0.shape[0]], dtype=dtype) - H = paddle.static.data(name='h', - shape=[H0.shape[0], H0.shape[1]], - dtype=dtype) - Y = minimize_lbfgs(func, - X, - initial_inverse_hessian_estimate=H, - dtype=dtype) + H = paddle.static.data( + name='h', shape=[H0.shape[0], H0.shape[1]], dtype=dtype + ) + Y = minimize_lbfgs( + func, X, initial_inverse_hessian_estimate=H, dtype=dtype + ) exe = paddle.static.Executor() exe.run(startup) return exe.run(main, feed={'x': x0, 'h': H0}, fetch_list=[Y]) -def test_dynamic_graph(func, - x0, - H0=None, - line_search_fn='strong_wolfe', - dtype='float32'): +def test_dynamic_graph( + func, x0, H0=None, line_search_fn='strong_wolfe', dtype='float32' +): paddle.disable_static() x0 = paddle.to_tensor(x0) if H0 is not None: H0 = paddle.to_tensor(H0) - return minimize_lbfgs(func, - x0, - initial_inverse_hessian_estimate=H0, - line_search_fn=line_search_fn, - dtype=dtype) + return minimize_lbfgs( + func, + x0, + initial_inverse_hessian_estimate=H0, + line_search_fn=line_search_fn, + dtype=dtype, + ) class TestLbfgs(unittest.TestCase): - def test_quadratic_nd(self): for dimension in [1, 10]: minimum = np.random.random(size=[dimension]).astype('float32') @@ -84,7 +82,8 @@ class TestLbfgs(unittest.TestCase): minimum_ = paddle.assign(minimum) scale_ = paddle.assign(scale) return paddle.sum( - paddle.multiply(scale_, (F.square_error_cost(x, minimum_)))) + paddle.multiply(scale_, (F.square_error_cost(x, minimum_))) + ) x0 = np.random.random(size=[dimension]).astype('float32') results = test_static_graph(func, x0) @@ -99,16 +98,17 @@ class TestLbfgs(unittest.TestCase): def func(x): # df = 3(x - 1.01)(x - 0.99) # f = x^3 - 3x^2 + 3*1.01*0.99x - return x * x * x / 3.0 - ( - extream_point[0] + extream_point[1] - ) * x * x / 2 + extream_point[0] * extream_point[1] * x + return ( + x * x * x / 3.0 + - (extream_point[0] + extream_point[1]) * x * x / 2 + + extream_point[0] * extream_point[1] * x + ) x0 = np.array([-1.7]).astype('float32') results = test_static_graph(func, x0) self.assertFalse(results[0][0]) def test_multi_minima(self): - def func(x): # df = 12(x + 1.1)(x - 0.2)(x - 0.8) # f = 3*x^4+0.4*x^3-5.46*x^2+2.112*x @@ -131,7 +131,7 @@ class TestLbfgs(unittest.TestCase): # f(x, y) = (a - x)^2 + b (y - x^2)^2 # minimum = (a, a^2) x, y = position[0], position[1] - c = (a - x)**2 + b * (y - x**2)**2 + c = (a - x) ** 2 + b * (y - x**2) ** 2 # the return cant be np array[1], or in jacobin will cause flat error return c[0] @@ -141,7 +141,6 @@ class TestLbfgs(unittest.TestCase): np.testing.assert_allclose(minimum, results[2], rtol=1e-05) def test_exception(self): - def func(x): return paddle.dot(x, x) @@ -150,11 +149,9 @@ class TestLbfgs(unittest.TestCase): # test dtype is not float32 or float64 x1 = np.random.random(size=[2]).astype('int32') - self.assertRaises(ValueError, - test_static_graph, - func, - x1, - dtype='int32') + self.assertRaises( + ValueError, test_static_graph, func, x1, dtype='int32' + ) # test initial_inverse_hessian_estimate is good results = test_static_graph_H0(func, x0, H0, dtype='float32') @@ -164,12 +161,9 @@ class TestLbfgs(unittest.TestCase): # test initial_inverse_hessian_estimate is bad and float64 x2 = np.random.random(size=[2]).astype('float64') H1 = np.array([[1.0, 2.0], [3.0, 1.0]]).astype('float64') - self.assertRaises(ValueError, - test_static_graph_H0, - func, - x2, - H0=H1, - dtype='float64') + self.assertRaises( + ValueError, test_static_graph_H0, func, x2, H0=H1, dtype='float64' + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lcm.py b/python/paddle/fluid/tests/unittests/test_lcm.py index 6f113c13a01..83f3f82d7e7 100644 --- a/python/paddle/fluid/tests/unittests/test_lcm.py +++ b/python/paddle/fluid/tests/unittests/test_lcm.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestLcmAPI(unittest.TestCase): - def setUp(self): self.x_np = 12 self.y_np = 20 @@ -37,32 +36,34 @@ class TestLcmAPI(unittest.TestCase): x2 = fluid.data(name='input2', dtype='int32', shape=self.y_shape) out = paddle.lcm(x1, x2) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(fluid.default_main_program(), - feed={ - 'input1': self.x_np, - 'input2': self.y_np - }, - fetch_list=[out]) - self.assertTrue((np.array(res[0]) == np.lcm(self.x_np, - self.y_np)).all()) + res = exe.run( + fluid.default_main_program(), + feed={'input1': self.x_np, 'input2': self.y_np}, + fetch_list=[out], + ) + self.assertTrue( + (np.array(res[0]) == np.lcm(self.x_np, self.y_np)).all() + ) def test_dygraph(self): paddle.disable_static() x1 = paddle.to_tensor(self.x_np) x2 = paddle.to_tensor(self.y_np) result = paddle.lcm(x1, x2) - np.testing.assert_allclose(np.lcm(self.x_np, self.y_np), - result.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np.lcm(self.x_np, self.y_np), result.numpy(), rtol=1e-05 + ) paddle.enable_static() class TestLcmAPI2(TestLcmAPI): - def setUp(self): self.x_np = np.arange(6).astype(np.int32) self.y_np = np.array([20]).astype(np.int32) @@ -71,7 +72,6 @@ class TestLcmAPI2(TestLcmAPI): class TestLcmAPI3(TestLcmAPI): - def setUp(self): self.x_np = 0 self.y_np = 20 @@ -80,7 +80,6 @@ class TestLcmAPI3(TestLcmAPI): class TestLcmAPI4(TestLcmAPI): - def setUp(self): self.x_np = 0 self.y_np = 0 @@ -89,7 +88,6 @@ class TestLcmAPI4(TestLcmAPI): class TestLcmAPI5(TestLcmAPI): - def setUp(self): self.x_np = 12 self.y_np = -20 diff --git a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py index bd73f628a02..ffe45f3ec31 100644 --- a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py +++ b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py @@ -23,45 +23,41 @@ import paddle.fluid.framework as framework import paddle.fluid.core as core -def exponential_decay(learning_rate, - global_step, - decay_steps, - decay_rate, - staircase=False): +def exponential_decay( + learning_rate, global_step, decay_steps, decay_rate, staircase=False +): exponent = global_step / decay_steps if staircase: exponent = math.floor(exponent) return learning_rate * decay_rate**exponent -def natural_exp_decay(learning_rate, - global_step, - decay_steps, - decay_rate, - staircase=False): +def natural_exp_decay( + learning_rate, global_step, decay_steps, decay_rate, staircase=False +): exponent = float(global_step) / float(decay_steps) if staircase: exponent = math.floor(exponent) return learning_rate * math.exp(-1 * decay_rate * exponent) -def inverse_time_decay(learning_rate, - global_step, - decay_steps, - decay_rate, - staircase=False): +def inverse_time_decay( + learning_rate, global_step, decay_steps, decay_rate, staircase=False +): temp = float(global_step) / float(decay_steps) if staircase: temp = math.floor(temp) return learning_rate / (1 + decay_rate * temp) -def polynomial_decay(learning_rate, - global_step, - decay_steps, - end_learning_rate=0.0001, - power=1.0, - cycle=False): +def polynomial_decay( + learning_rate, + global_step, + decay_steps, + end_learning_rate=0.0001, + power=1.0, + cycle=False, +): if cycle: div = math.ceil(global_step / float(decay_steps)) if div == 0: @@ -69,8 +65,9 @@ def polynomial_decay(learning_rate, decay_steps = decay_steps * div else: global_step = min(global_step, decay_steps) - return (learning_rate - end_learning_rate) * \ - ((1 - float(global_step) / float(decay_steps)) ** power) + end_learning_rate + return (learning_rate - end_learning_rate) * ( + (1 - float(global_step) / float(decay_steps)) ** power + ) + end_learning_rate def piecewise_decay(global_step, boundaries, values): @@ -83,8 +80,9 @@ def piecewise_decay(global_step, boundaries, values): def cosine_decay(global_step, learning_rate, step_each_epoch, epochs): cur_epoch = math.floor(global_step / step_each_epoch) - decayed_lr = learning_rate * 0.5 * (math.cos(cur_epoch * math.pi / epochs) + - 1) + decayed_lr = ( + learning_rate * 0.5 * (math.cos(cur_epoch * math.pi / epochs) + 1) + ) return decayed_lr @@ -119,7 +117,6 @@ def lambda_decay(global_step, learning_rate, lr_lambda): class TestLearningRateDecayDygraph(unittest.TestCase): - def test_LR_state_dict(self): with fluid.dygraph.guard(): x = np.random.uniform(-1, 1, [3, 10]).astype("float32") @@ -130,17 +127,24 @@ class TestLearningRateDecayDygraph(unittest.TestCase): learning_rate=0.1, decay_steps=10000, decay_rate=0.5, - staircase=True) + staircase=True, + ) Step_scheduler = fluid.dygraph.StepDecay(0.5, step_size=3) Reducelr_scheduler = fluid.dygraph.ReduceLROnPlateau( - learning_rate=1.0, decay_rate=0.5, patience=5, cooldown=3) - - adam1 = fluid.optimizer.Adam(learning_rate=Exponential_scheduler, - parameter_list=linear.parameters()) - adam2 = fluid.optimizer.Adam(learning_rate=Step_scheduler, - parameter_list=linear.parameters()) - adam3 = fluid.optimizer.Adam(learning_rate=Reducelr_scheduler, - parameter_list=linear.parameters()) + learning_rate=1.0, decay_rate=0.5, patience=5, cooldown=3 + ) + + adam1 = fluid.optimizer.Adam( + learning_rate=Exponential_scheduler, + parameter_list=linear.parameters(), + ) + adam2 = fluid.optimizer.Adam( + learning_rate=Step_scheduler, parameter_list=linear.parameters() + ) + adam3 = fluid.optimizer.Adam( + learning_rate=Reducelr_scheduler, + parameter_list=linear.parameters(), + ) print(adam3.state_dict()) for epoch in range(10): @@ -161,59 +165,76 @@ class TestLearningRateDecayDygraph(unittest.TestCase): learning_rate=0.1, decay_steps=10000, decay_rate=0.5, - staircase=True) + staircase=True, + ) Step_scheduler_test = fluid.dygraph.StepDecay(0.5, step_size=3) Reducelr_scheduler_test = fluid.dygraph.ReduceLROnPlateau( - learning_rate=1.0, decay_rate=0.5, patience=5, cooldown=3) + learning_rate=1.0, decay_rate=0.5, patience=5, cooldown=3 + ) fluid.dygraph.save_dygraph(adam1.state_dict(), "save_path") _, opt_state = fluid.dygraph.load_dygraph("save_path") adam_test = fluid.optimizer.Adam( learning_rate=Exponential_scheduler_test, - parameter_list=linear.parameters()) + parameter_list=linear.parameters(), + ) adam_test.set_dict(opt_state) self.assertEqual( adam_test._learning_rate.step_num, adam1._learning_rate.step_num, - "epoch_num is different before and after set_dict") + "epoch_num is different before and after set_dict", + ) fluid.dygraph.save_dygraph(adam2.state_dict(), "save_path") _, opt_state = fluid.dygraph.load_dygraph("save_path") - adam_test = fluid.optimizer.Adam(learning_rate=Step_scheduler_test, - parameter_list=linear.parameters()) + adam_test = fluid.optimizer.Adam( + learning_rate=Step_scheduler_test, + parameter_list=linear.parameters(), + ) adam_test.set_dict(opt_state) self.assertEqual( adam_test._learning_rate.epoch_num, adam2._learning_rate.epoch_num, - "epoch_num is different before and after set_dict") + "epoch_num is different before and after set_dict", + ) self.assertEqual( - adam_test._learning_rate(), adam2._learning_rate(), - "current learning rate is different before and after set_dict") + adam_test._learning_rate(), + adam2._learning_rate(), + "current learning rate is different before and after set_dict", + ) fluid.dygraph.save_dygraph(adam3.state_dict(), "save_path") _, opt_state = fluid.dygraph.load_dygraph("save_path") adam_test = fluid.optimizer.Adam( learning_rate=Reducelr_scheduler_test, - parameter_list=linear.parameters()) + parameter_list=linear.parameters(), + ) adam_test.set_dict(opt_state) self.assertEqual( adam_test._learning_rate.best_loss, adam3._learning_rate.best_loss.numpy()[0], - "best_loss is different before and after set_dict") + "best_loss is different before and after set_dict", + ) self.assertEqual( adam_test._learning_rate.cooldown_counter, adam3._learning_rate.cooldown_counter, - "cooldown_counter is different before and after set_dict") + "cooldown_counter is different before and after set_dict", + ) self.assertEqual( adam_test._learning_rate.num_bad_epochs, adam3._learning_rate.num_bad_epochs, - "num_bad_epochs is different before and after set_dict") - self.assertEqual(adam_test._learning_rate.epoch_num, - adam3._learning_rate.epoch_num, - "epoch is different before and after set_dict") + "num_bad_epochs is different before and after set_dict", + ) + self.assertEqual( + adam_test._learning_rate.epoch_num, + adam3._learning_rate.epoch_num, + "epoch is different before and after set_dict", + ) self.assertEqual( - adam_test._learning_rate(), adam3._learning_rate(), - "current learning rate is different before and after set_dict") + adam_test._learning_rate(), + adam3._learning_rate(), + "current learning rate is different before and after set_dict", + ) def test_NoamDecay(self): with fluid.dygraph.guard(): @@ -223,42 +244,47 @@ class TestLearningRateDecayDygraph(unittest.TestCase): lr = fluid.layers.noam_decay(d_model, warmup_steps, learning_rate) for step in range(5): step += 1 - right_result = noam_decay(step, d_model, warmup_steps, - learning_rate) + right_result = noam_decay( + step, d_model, warmup_steps, learning_rate + ) fluid_result = lr() self.assertAlmostEqual( right_result, fluid_result[0], - msg= - 'Failed lr scheduler in step {0}, Python result is {1}, Fluid result is {2}' - .format(step, right_result, fluid_result[0])) + msg='Failed lr scheduler in step {0}, Python result is {1}, Fluid result is {2}'.format( + step, right_result, fluid_result[0] + ), + ) def test_LinearLrWarmup(self): with fluid.dygraph.guard(): - lr = fluid.layers.polynomial_decay(learning_rate=1.0, - decay_steps=10, - end_learning_rate=0.0, - power=1.0) - lr = fluid.layers.linear_lr_warmup(learning_rate=lr, - warmup_steps=2, - start_lr=0.0, - end_lr=1.0) + lr = fluid.layers.polynomial_decay( + learning_rate=1.0, + decay_steps=10, + end_learning_rate=0.0, + power=1.0, + ) + lr = fluid.layers.linear_lr_warmup( + learning_rate=lr, warmup_steps=2, start_lr=0.0, end_lr=1.0 + ) right_result = [0.5, 0.9, 0.8, 0.7, 0.6] for i in range(5): t = lr() - np.testing.assert_allclose(t.numpy()[0].item(), - right_result[i], - rtol=1e-05) + np.testing.assert_allclose( + t.numpy()[0].item(), right_result[i], rtol=1e-05 + ) with self.assertRaises(TypeError): - lr = fluid.layers.linear_lr_warmup(learning_rate="fake_lr", - warmup_steps=2, - start_lr=0.0, - end_lr=1.0) + lr = fluid.layers.linear_lr_warmup( + learning_rate="fake_lr", + warmup_steps=2, + start_lr=0.0, + end_lr=1.0, + ) def test_MultiStepDecay(self): with fluid.dygraph.guard(): @@ -267,30 +293,36 @@ class TestLearningRateDecayDygraph(unittest.TestCase): decay_rate = 0.2 linear = fluid.dygraph.Linear(10, 10) - scheduler = fluid.dygraph.MultiStepDecay(learning_rate, milestones, - decay_rate) + scheduler = fluid.dygraph.MultiStepDecay( + learning_rate, milestones, decay_rate + ) adam = fluid.optimizer.AdamOptimizer( - learning_rate=scheduler, parameter_list=linear.parameters()) + learning_rate=scheduler, parameter_list=linear.parameters() + ) for epoch in range(10): - right_result = multi_step_decay(epoch, learning_rate, - milestones, decay_rate) + right_result = multi_step_decay( + epoch, learning_rate, milestones, decay_rate + ) fluid_result = adam.current_step_lr() scheduler.epoch() self.assertAlmostEqual( right_result, fluid_result, - msg= - 'Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}' - .format(epoch, right_result, fluid_result)) + msg='Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}'.format( + epoch, right_result, fluid_result + ), + ) with self.assertRaises(ValueError): - lr = fluid.dygraph.MultiStepDecay(learning_rate, [30, 50, 20], - 0.1) + lr = fluid.dygraph.MultiStepDecay( + learning_rate, [30, 50, 20], 0.1 + ) with self.assertRaises(ValueError): - lr = fluid.dygraph.MultiStepDecay(learning_rate, [20, 30, 50], - 1) + lr = fluid.dygraph.MultiStepDecay( + learning_rate, [20, 30, 50], 1 + ) with self.assertRaises(TypeError): lr = fluid.dygraph.MultiStepDecay("test", [20, 30, 50]) @@ -303,19 +335,22 @@ class TestLearningRateDecayDygraph(unittest.TestCase): learning_rate = 0.5 step_size = 3 decay_rate = 0.2 - scheduler = fluid.dygraph.StepDecay(learning_rate, step_size, - decay_rate) + scheduler = fluid.dygraph.StepDecay( + learning_rate, step_size, decay_rate + ) for epoch in range(10): - right_result = step_decay(epoch, learning_rate, step_size, - decay_rate) + right_result = step_decay( + epoch, learning_rate, step_size, decay_rate + ) fluid_result = scheduler().numpy()[0] scheduler.epoch() self.assertAlmostEqual( right_result, fluid_result, - msg= - 'Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}' - .format(epoch, right_result, fluid_result)) + msg='Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}'.format( + epoch, right_result, fluid_result + ), + ) with self.assertRaises(TypeError): lr = fluid.dygraph.StepDecay(learning_rate, "test", 0.1) @@ -330,8 +365,9 @@ class TestLearningRateDecayDygraph(unittest.TestCase): scheduler = fluid.dygraph.LambdaDecay(learning_rate, lr_lambda) linear = fluid.dygraph.nn.Linear(10, 10) - adam = fluid.optimizer.Adam(scheduler, - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + scheduler, parameter_list=linear.parameters() + ) for epoch in range(30): right_result = lambda_decay(epoch, learning_rate, lr_lambda) @@ -340,26 +376,28 @@ class TestLearningRateDecayDygraph(unittest.TestCase): self.assertAlmostEqual( right_result, fluid_result, - msg= - 'Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}' - .format(epoch, right_result, fluid_result)) + msg='Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}'.format( + epoch, right_result, fluid_result + ), + ) with self.assertRaises(TypeError): lr = fluid.dygraph.LambdaDecay(learning_rate, "test") class TestLearningRateDecay(unittest.TestCase): - def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) for place in places: - self.check_decay_with_place(place, python_decay_fn, fluid_decay_fn, - kwargs) + self.check_decay_with_place( + place, python_decay_fn, fluid_decay_fn, kwargs + ) - def check_decay_with_place(self, place, python_decay_fn, fluid_decay_fn, - kwargs): + def check_decay_with_place( + self, place, python_decay_fn, fluid_decay_fn, kwargs + ): main_prog = fluid.Program() startup_prog = fluid.Program() @@ -375,23 +413,27 @@ class TestLearningRateDecay(unittest.TestCase): # Step of NoamDecay starts from 1. if python_decay_fn.__name__ == 'noam_decay': step += 1 - lr_val, = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) - python_decayed_lr = python_decay_fn(global_step=float(step), - **kwargs) + (lr_val,) = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) + python_decayed_lr = python_decay_fn( + global_step=float(step), **kwargs + ) self.assertAlmostEqual( python_decayed_lr, lr_val[0], - msg= - 'Failed lr scheduler is {0}, step {1}, Python result is {2}, Fluid result is {3}' - .format(python_decay_fn.__name__, str(step), - str(python_decayed_lr), str(lr_val[0]))) + msg='Failed lr scheduler is {0}, step {1}, Python result is {2}, Fluid result is {3}'.format( + python_decay_fn.__name__, + str(step), + str(python_decayed_lr), + str(lr_val[0]), + ), + ) def test_decay(self): common_kwargs_true = { "learning_rate": 1.0, "decay_steps": 5, "decay_rate": 0.5, - "staircase": True + "staircase": True, } common_kwargs_false = copy.deepcopy(common_kwargs_true) common_kwargs_false["staircase"] = False @@ -402,37 +444,47 @@ class TestLearningRateDecay(unittest.TestCase): (natural_exp_decay, layers.natural_exp_decay, common_kwargs_true), (natural_exp_decay, layers.natural_exp_decay, common_kwargs_false), (inverse_time_decay, layers.inverse_time_decay, common_kwargs_true), - (inverse_time_decay, layers.inverse_time_decay, - common_kwargs_false), - (polynomial_decay, layers.polynomial_decay, { - "learning_rate": 1.0, - "decay_steps": 5, - "cycle": True - }), - (polynomial_decay, layers.polynomial_decay, { - "learning_rate": 1.0, - "decay_steps": 5, - "cycle": False - }), - (piecewise_decay, layers.piecewise_decay, { - "boundaries": [3, 6, 9], - "values": [0.1, 0.2, 0.3, 0.4] - }), - (cosine_decay, layers.cosine_decay, { - "learning_rate": 0.1, - "step_each_epoch": 100, - "epochs": 120 - }), - (noam_decay, layers.noam_decay, { - "d_model": 0.01, - "warmup_steps": 200, - "learning_rate": 2.0 - }) + ( + inverse_time_decay, + layers.inverse_time_decay, + common_kwargs_false, + ), + ( + polynomial_decay, + layers.polynomial_decay, + {"learning_rate": 1.0, "decay_steps": 5, "cycle": True}, + ), + ( + polynomial_decay, + layers.polynomial_decay, + {"learning_rate": 1.0, "decay_steps": 5, "cycle": False}, + ), + ( + piecewise_decay, + layers.piecewise_decay, + {"boundaries": [3, 6, 9], "values": [0.1, 0.2, 0.3, 0.4]}, + ), + ( + cosine_decay, + layers.cosine_decay, + {"learning_rate": 0.1, "step_each_epoch": 100, "epochs": 120}, + ), + ( + noam_decay, + layers.noam_decay, + {"d_model": 0.01, "warmup_steps": 200, "learning_rate": 2.0}, + ), ] for py_decay_fn, fluid_decay_fn, kwargs in decay_fns: - print("class=" + self.__class__.__name__ + " decay_fn=" + - py_decay_fn.__name__ + " kwargs=" + str(kwargs)) + print( + "class=" + + self.__class__.__name__ + + " decay_fn=" + + py_decay_fn.__name__ + + " kwargs=" + + str(kwargs) + ) main_program = framework.Program() startup_program = framework.Program() with framework.program_guard(main_program, startup_program): @@ -440,19 +492,20 @@ class TestLearningRateDecay(unittest.TestCase): class TestLinearWamrupLearningRateDecay(unittest.TestCase): - - def check_decay_with_place(self, place, python_decay_fn, fluid_decay_fn, - kwargs): + def check_decay_with_place( + self, place, python_decay_fn, fluid_decay_fn, kwargs + ): main_prog = fluid.Program() startup_prog = fluid.Program() warmup_steps = 10 - start_lr = 0.1 / 3. + start_lr = 0.1 / 3.0 end_lr = 0.1 with fluid.program_guard(main_prog, startup_prog): - decayed_lr = layers.linear_lr_warmup(fluid_decay_fn(**kwargs), - warmup_steps, start_lr, end_lr) + decayed_lr = layers.linear_lr_warmup( + fluid_decay_fn(**kwargs), warmup_steps, start_lr, end_lr + ) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -462,24 +515,28 @@ class TestLinearWamrupLearningRateDecay(unittest.TestCase): # Step of NoamDecay starts from 1. if fluid_decay_fn.__name__ == 'noam_decay': step += 1 - lr_val, = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) + (lr_val,) = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) if step < warmup_steps: - python_decayed_lr = linear_lr_warmup(float(step), warmup_steps, - start_lr, end_lr) + python_decayed_lr = linear_lr_warmup( + float(step), warmup_steps, start_lr, end_lr + ) else: - python_decayed_lr = python_decay_fn(global_step=float(step), - **kwargs) + python_decayed_lr = python_decay_fn( + global_step=float(step), **kwargs + ) self.assertAlmostEqual( python_decayed_lr, lr_val[0], - msg= - 'Test {0} Failed, step {1}, Python result is {2}, Fluid result is {3}' - .format(python_decay_fn.__name__, str(step), - str(python_decayed_lr), str(lr_val[0]))) + msg='Test {0} Failed, step {1}, Python result is {2}, Fluid result is {3}'.format( + python_decay_fn.__name__, + str(step), + str(python_decayed_lr), + str(lr_val[0]), + ), + ) class TestLinearWamrupLearningRateDecayWithScalarInput(unittest.TestCase): - def run_scalar_lr(self, place, lr, start_lr, end_lr): main_prog = fluid.Program() startup_prog = fluid.Program() @@ -487,27 +544,30 @@ class TestLinearWamrupLearningRateDecayWithScalarInput(unittest.TestCase): warmup_steps = 10 with fluid.program_guard(main_prog, startup_prog): - decayed_lr = layers.linear_lr_warmup(lr, warmup_steps, start_lr, - end_lr) + decayed_lr = layers.linear_lr_warmup( + lr, warmup_steps, start_lr, end_lr + ) exe = fluid.Executor(place) exe.run(startup_prog) for step in range(20): - lr_val, = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) + (lr_val,) = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) if step < warmup_steps: - expected_lr = linear_lr_warmup(float(step), warmup_steps, - start_lr, end_lr) + expected_lr = linear_lr_warmup( + float(step), warmup_steps, start_lr, end_lr + ) else: expected_lr = lr self.assertAlmostEqual( expected_lr, lr_val[0], msg='Test failed, step {0}, expected {1}, but got {2}'.format( - step, expected_lr, lr_val[0])) + step, expected_lr, lr_val[0] + ), + ) def test_scalar_lr(self): - def run_places(lr, start_lr, end_lr): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -517,13 +577,13 @@ class TestLinearWamrupLearningRateDecayWithScalarInput(unittest.TestCase): # float lr = 0.2 - start_lr = 0.1 / 3. + start_lr = 0.1 / 3.0 end_lr = 0.2 run_places(lr, start_lr, end_lr) # int end_lr - lr = 2. - start_lr = 0.1 / 3. + lr = 2.0 + start_lr = 0.1 / 3.0 end_lr = 1 run_places(lr, start_lr, end_lr) diff --git a/python/paddle/fluid/tests/unittests/test_lerp_op.py b/python/paddle/fluid/tests/unittests/test_lerp_op.py index f7f096a8d5a..1cae3fbaba9 100644 --- a/python/paddle/fluid/tests/unittests/test_lerp_op.py +++ b/python/paddle/fluid/tests/unittests/test_lerp_op.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import unittest import numpy as np from op_test import OpTest @@ -25,14 +23,13 @@ np.random.seed(0) class TestLerp(OpTest): - def setUp(self): self.op_type = "lerp" self.python_api = paddle.lerp self.init_dtype() self.init_shape() - x = np.arange(1., 101.).astype(self.dtype).reshape(self.shape) - y = np.full(100, 10.).astype(self.dtype).reshape(self.shape) + x = np.arange(1.0, 101.0).astype(self.dtype).reshape(self.shape) + y = np.full(100, 10.0).astype(self.dtype).reshape(self.shape) w = np.asarray([0.5]).astype(self.dtype) self.inputs = {'X': x, 'Y': y, 'Weight': w} self.outputs = {'Out': x + w * (y - x)} @@ -51,72 +48,64 @@ class TestLerp(OpTest): class TestLerpWithDim2(TestLerp): - def init_shape(self): self.shape = [2, 50] class TestLerpWithDim3(TestLerp): - def init_shape(self): self.shape = [2, 2, 25] class TestLerpWithDim4(TestLerp): - def init_shape(self): self.shape = [2, 2, 5, 5] class TestLerpWithDim5(TestLerp): - def init_shape(self): self.shape = [2, 1, 2, 5, 5] class TestLerpWithDim6(TestLerp): - def init_shape(self): self.shape = [2, 1, 2, 5, 1, 5] class TestLerpBroadXY(TestLerp): - def setUp(self): self.op_type = "lerp" self.python_api = paddle.lerp self.init_dtype() self.init_shape() - x = np.arange(1., 201.).astype(self.dtype).reshape([2, 1, 2, 50]) - y = np.full(200, 10.).astype(self.dtype).reshape([2, 2, 1, 50]) + x = np.arange(1.0, 201.0).astype(self.dtype).reshape([2, 1, 2, 50]) + y = np.full(200, 10.0).astype(self.dtype).reshape([2, 2, 1, 50]) w = np.asarray([0.5]).astype(self.dtype) self.inputs = {'X': x, 'Y': y, 'Weight': w} self.outputs = {'Out': x + w * (y - x)} class TestLerpBroadWToXY(TestLerp): - def setUp(self): self.op_type = "lerp" self.python_api = paddle.lerp self.init_dtype() self.init_shape() x = np.full(600, 2.5).astype(self.dtype).reshape([50, 2, 2, 3]) - y = np.full(600, 1.).astype(self.dtype).reshape([50, 2, 2, 3]) + y = np.full(600, 1.0).astype(self.dtype).reshape([50, 2, 2, 3]) w = np.random.random([3]).astype(self.dtype) self.inputs = {'X': x, 'Y': y, 'Weight': w} self.outputs = {'Out': x + w * (y - x)} class TestLerpAPI(unittest.TestCase): - def init_dtype(self): self.dtype = 'float32' def setUp(self): self.init_dtype() - self.x = np.arange(1., 5.).astype(self.dtype) - self.y = np.full(4, 10.).astype(self.dtype) + self.x = np.arange(1.0, 5.0).astype(self.dtype) + self.y = np.full(4, 10.0).astype(self.dtype) self.w = np.asarray([0.75]).astype(self.dtype) self.res_ref = self.x + self.w * (self.y - self.x) self.place = [paddle.CPUPlace()] @@ -132,10 +121,12 @@ class TestLerpAPI(unittest.TestCase): y = paddle.fluid.data('y', [1, 4], dtype=self.dtype) out = paddle.lerp(x, y, 0.5) exe = paddle.static.Executor(place) - res = exe.run(feed={ - 'x': self.x.reshape([1, 4]), - 'y': self.y.reshape([1, 4]), - }) + res = exe.run( + feed={ + 'x': self.x.reshape([1, 4]), + 'y': self.y.reshape([1, 4]), + } + ) for r in res: np.testing.assert_allclose(self.res_ref, r, rtol=1e-05) @@ -143,7 +134,6 @@ class TestLerpAPI(unittest.TestCase): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) @@ -157,7 +147,6 @@ class TestLerpAPI(unittest.TestCase): run(place) def test_inplace_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) @@ -170,7 +159,6 @@ class TestLerpAPI(unittest.TestCase): run(place) def test_inplace_api_exception(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) @@ -185,8 +173,8 @@ class TestLerpAPI(unittest.TestCase): def test_x_broadcast_y(self): paddle.disable_static() - x = np.arange(1., 21.).astype(self.dtype).reshape([2, 2, 5]) - y = np.full(30, 10.).astype(self.dtype).reshape([3, 2, 1, 5]) + x = np.arange(1.0, 21.0).astype(self.dtype).reshape([2, 2, 5]) + y = np.full(30, 10.0).astype(self.dtype).reshape([3, 2, 1, 5]) out = paddle.lerp(paddle.to_tensor(x), paddle.to_tensor(y), 0.5) res_ref = x + 0.5 * (y - x) np.testing.assert_allclose(res_ref, out.numpy(), rtol=1e-05) @@ -194,11 +182,12 @@ class TestLerpAPI(unittest.TestCase): def test_x_y_broadcast_w(self): paddle.disable_static() - x = np.arange(11., 21.).astype(self.dtype).reshape([2, 5]) + x = np.arange(11.0, 21.0).astype(self.dtype).reshape([2, 5]) y = np.full(20, 7.5).astype(self.dtype).reshape([2, 2, 5]) w = np.full(40, 0.225).astype(self.dtype).reshape([2, 2, 2, 5]) - out = paddle.lerp(paddle.to_tensor(x), paddle.to_tensor(y), - paddle.to_tensor(w)) + out = paddle.lerp( + paddle.to_tensor(x), paddle.to_tensor(y), paddle.to_tensor(w) + ) res_ref = x + w * (y - x) np.testing.assert_allclose(res_ref, out.numpy(), rtol=1e-05) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_lgamma_op.py b/python/paddle/fluid/tests/unittests/test_lgamma_op.py index 13e420abad3..54211aef14f 100644 --- a/python/paddle/fluid/tests/unittests/test_lgamma_op.py +++ b/python/paddle/fluid/tests/unittests/test_lgamma_op.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestLgammaOp(OpTest): - def setUp(self): self.op_type = 'lgamma' self.python_api = paddle.lgamma @@ -48,19 +47,16 @@ class TestLgammaOp(OpTest): class TestLgammaOpFp32(TestLgammaOp): - def init_dtype_type(self): self.dtype = np.float32 def test_check_grad_normal(self): - self.check_grad(['X'], - 'Out', - numeric_grad_delta=0.005, - check_eager=True) + self.check_grad( + ['X'], 'Out', numeric_grad_delta=0.005, check_eager=True + ) class TestLgammaOpApi(unittest.TestCase): - def test_lgamma(self): paddle.disable_static() self.dtype = "float32" diff --git a/python/paddle/fluid/tests/unittests/test_limit_by_capacity_op.py b/python/paddle/fluid/tests/unittests/test_limit_by_capacity_op.py index 3e604d25657..02c71ea3a41 100644 --- a/python/paddle/fluid/tests/unittests/test_limit_by_capacity_op.py +++ b/python/paddle/fluid/tests/unittests/test_limit_by_capacity_op.py @@ -43,17 +43,17 @@ def all_close(exp, out, n_worker): return np.allclose(exp.sum(0), out.sum(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLimitByCapacityInt64API(unittest.TestCase): - def init_test_case(self): - self.expert_count = np.random.randint(0, - 1000, - size=(len(self.capacity) * - self.n_worker)) - self.out = limit_by_capacity(self.expert_count, self.capacity, - self.n_worker) + self.expert_count = np.random.randint( + 0, 1000, size=(len(self.capacity) * self.n_worker) + ) + self.out = limit_by_capacity( + self.expert_count, self.capacity, self.n_worker + ) self.expert_count = self.expert_count.astype("int64") self.capacity = self.capacity.astype("int64") self.place = paddle.CUDAPlace(0) @@ -66,19 +66,23 @@ class TestLimitByCapacityInt64API(unittest.TestCase): def test_static_api(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - capacity = paddle.static.data('capacity', - shape=self.capacity.shape, - dtype="int64") + capacity = paddle.static.data( + 'capacity', shape=self.capacity.shape, dtype="int64" + ) expert_count_tensor = paddle.static.data( - 'ExpertCount', shape=self.expert_count.shape, dtype="int64") - out = utils._limit_by_capacity(expert_count_tensor, capacity, - self.n_worker) + 'ExpertCount', shape=self.expert_count.shape, dtype="int64" + ) + out = utils._limit_by_capacity( + expert_count_tensor, capacity, self.n_worker + ) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'capacity': self.capacity, - 'ExpertCount': self.expert_count, - }, - fetch_list=out) + res = exe.run( + feed={ + 'capacity': self.capacity, + 'ExpertCount': self.expert_count, + }, + fetch_list=out, + ) assert all_close(self.out, res[0], self.n_worker) @@ -86,8 +90,9 @@ class TestLimitByCapacityInt64API(unittest.TestCase): paddle.disable_static(self.place) capacity = paddle.to_tensor(self.capacity) expert_count_tensor = paddle.to_tensor(self.expert_count) - out = utils._limit_by_capacity(expert_count_tensor, capacity, - self.n_worker) + out = utils._limit_by_capacity( + expert_count_tensor, capacity, self.n_worker + ) assert all_close(self.out, out.numpy(), self.n_worker) def test_dygraph_api(self): @@ -96,10 +101,10 @@ class TestLimitByCapacityInt64API(unittest.TestCase): self.func_dygraph_api() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLimitByCapacityInt64API_SmallWorker(TestLimitByCapacityInt64API): - def setUp(self): self.capacity = np.array([100, 12000, 1200, 0, 4700, 1000, 57, 200]) self.n_worker = 1 diff --git a/python/paddle/fluid/tests/unittests/test_linalg_cond.py b/python/paddle/fluid/tests/unittests/test_linalg_cond.py index 175d44cb1ce..4724bbc6d08 100644 --- a/python/paddle/fluid/tests/unittests/test_linalg_cond.py +++ b/python/paddle/fluid/tests/unittests/test_linalg_cond.py @@ -31,9 +31,9 @@ def test_static_assert_true(self, x_list, p_list): exe = static.Executor() result = exe.run(feed={"X": x}, fetch_list=[output]) expected_output = np.linalg.cond(x, p) - np.testing.assert_allclose(result[0], - expected_output, - rtol=5e-5) + np.testing.assert_allclose( + result[0], expected_output, rtol=5e-5 + ) def test_dygraph_assert_true(self, x_list, p_list): @@ -42,9 +42,9 @@ def test_dygraph_assert_true(self, x_list, p_list): input_tensor = paddle.to_tensor(x) output = paddle.linalg.cond(input_tensor, p) expected_output = np.linalg.cond(x, p) - np.testing.assert_allclose(output.numpy(), - expected_output, - rtol=5e-5) + np.testing.assert_allclose( + output.numpy(), expected_output, rtol=5e-5 + ) def gen_input(): @@ -81,7 +81,6 @@ def gen_empty_input(): class API_TestStaticCond(unittest.TestCase): - def test_out(self): paddle.enable_static() # test calling results of 'cond' in static mode @@ -91,7 +90,6 @@ class API_TestStaticCond(unittest.TestCase): class API_TestDygraphCond(unittest.TestCase): - def func_out(self): paddle.disable_static() # test calling results of 'cond' in dynamic mode @@ -106,14 +104,13 @@ class API_TestDygraphCond(unittest.TestCase): class TestCondAPIError(unittest.TestCase): - def func_dygraph_api_error(self): paddle.disable_static() # test raising errors when 'cond' is called in dygraph mode p_list_error = ('fro_', '_nuc', -0.7, 0, 1.5, 3) x_list_n_n, x_list_m_n = gen_input() for p in p_list_error: - for x in (x_list_n_n + x_list_m_n): + for x in x_list_n_n + x_list_m_n: x_tensor = paddle.to_tensor(x) self.assertRaises(ValueError, paddle.linalg.cond, x_tensor, p) @@ -133,7 +130,7 @@ class TestCondAPIError(unittest.TestCase): p_list_error = ('f ro', 'fre', 'NUC', -1.6, 0, 5) x_list_n_n, x_list_m_n = gen_input() for p in p_list_error: - for x in (x_list_n_n + x_list_m_n): + for x in x_list_n_n + x_list_m_n: with static.program_guard(static.Program(), static.Program()): x_data = static.data("X", shape=x.shape, dtype=x.dtype) self.assertRaises(ValueError, paddle.linalg.cond, x_data, p) @@ -149,13 +146,13 @@ class TestCondAPIError(unittest.TestCase): paddle.enable_static() x_list_n_n, x_list_m_n = gen_empty_input() - for p in (p_list_n_n + p_list_m_n): + for p in p_list_n_n + p_list_m_n: for x in x_list_n_n: with static.program_guard(static.Program(), static.Program()): x_data = static.data("X", shape=x.shape, dtype=x.dtype) self.assertRaises(ValueError, paddle.linalg.cond, x_data, p) - for p in (p_list_n_n + p_list_m_n): + for p in p_list_n_n + p_list_m_n: for x in x_list_n_n: with static.program_guard(static.Program(), static.Program()): x_data = static.data("X", shape=x.shape, dtype=x.dtype) @@ -163,7 +160,6 @@ class TestCondAPIError(unittest.TestCase): class TestCondEmptyTensorInput(unittest.TestCase): - def func_dygraph_empty_tensor_input(self): paddle.disable_static() # test calling results of 'cond' when input is an empty tensor in dynamic mode diff --git a/python/paddle/fluid/tests/unittests/test_linalg_lstsq_op.py b/python/paddle/fluid/tests/unittests/test_linalg_lstsq_op.py index f4180941be2..f627057ed24 100644 --- a/python/paddle/fluid/tests/unittests/test_linalg_lstsq_op.py +++ b/python/paddle/fluid/tests/unittests/test_linalg_lstsq_op.py @@ -20,7 +20,6 @@ import paddle.fluid.core as core class LinalgLstsqTestCase(unittest.TestCase): - def setUp(self): self.devices = ["cpu"] self.init_config() @@ -39,15 +38,17 @@ class LinalgLstsqTestCase(unittest.TestCase): def generate_input(self): self._input_data_1 = np.random.random(self._input_shape_1).astype( - self.dtype) + self.dtype + ) self._input_data_2 = np.random.random(self._input_shape_2).astype( - self.dtype) + self.dtype + ) def generate_output(self): if len(self._input_shape_1) == 2: - out = np.linalg.lstsq(self._input_data_1, - self._input_data_2, - rcond=self.rcond) + out = np.linalg.lstsq( + self._input_data_1, self._input_data_2, rcond=self.rcond + ) self._output_solution = out[0] self._output_residuals = out[1] self._output_rank = out[2] @@ -58,9 +59,11 @@ class LinalgLstsqTestCase(unittest.TestCase): self._output_rank = [] self._output_sg_values = [] for i in range(self._input_shape_1[0]): - out = np.linalg.lstsq(self._input_data_1[i], - self._input_data_2[i], - rcond=self.rcond) + out = np.linalg.lstsq( + self._input_data_1[i], + self._input_data_2[i], + rcond=self.rcond, + ) self._output_solution.append(out[0]) self._output_residuals.append(out[1]) self._output_rank.append(out[2]) @@ -72,16 +75,15 @@ class LinalgLstsqTestCase(unittest.TestCase): for dev in self.devices: paddle.set_device(dev) place = paddle.CPUPlace() if dev == "cpu" else paddle.CUDAPlace(0) - x = paddle.to_tensor(self._input_data_1, - place=place, - dtype=self.dtype) - y = paddle.to_tensor(self._input_data_2, - place=place, - dtype=self.dtype) - results = paddle.linalg.lstsq(x, - y, - rcond=self.rcond, - driver=self.driver) + x = paddle.to_tensor( + self._input_data_1, place=place, dtype=self.dtype + ) + y = paddle.to_tensor( + self._input_data_2, place=place, dtype=self.dtype + ) + results = paddle.linalg.lstsq( + x, y, rcond=self.rcond, driver=self.driver + ) self._result_solution = results[0].numpy() self._result_residuals = results[1].numpy() self._result_rank = results[2].numpy() @@ -94,16 +96,15 @@ class LinalgLstsqTestCase(unittest.TestCase): for dev in self.devices: paddle.set_device(dev) place = paddle.CPUPlace() if dev == "cpu" else paddle.CUDAPlace(0) - x = paddle.to_tensor(self._input_data_1, - place=place, - dtype=self.dtype) - y = paddle.to_tensor(self._input_data_2, - place=place, - dtype=self.dtype) - results = paddle.linalg.lstsq(x, - y, - rcond=self.rcond, - driver=self.driver) + x = paddle.to_tensor( + self._input_data_1, place=place, dtype=self.dtype + ) + y = paddle.to_tensor( + self._input_data_2, place=place, dtype=self.dtype + ) + results = paddle.linalg.lstsq( + x, y, rcond=self.rcond, driver=self.driver + ) self._result_solution = results[0].numpy() self._result_residuals = results[1].numpy() self._result_rank = results[2].numpy() @@ -116,23 +117,25 @@ class LinalgLstsqTestCase(unittest.TestCase): paddle.set_device(dev) place = fluid.CPUPlace() if dev == "cpu" else fluid.CUDAPlace(0) with fluid.program_guard(fluid.Program(), fluid.Program()): - x = paddle.fluid.data(name="x", - shape=self._input_shape_1, - dtype=self._input_data_1.dtype) - y = paddle.fluid.data(name="y", - shape=self._input_shape_2, - dtype=self._input_data_2.dtype) - results = paddle.linalg.lstsq(x, - y, - rcond=self.rcond, - driver=self.driver) + x = paddle.fluid.data( + name="x", + shape=self._input_shape_1, + dtype=self._input_data_1.dtype, + ) + y = paddle.fluid.data( + name="y", + shape=self._input_shape_2, + dtype=self._input_data_2.dtype, + ) + results = paddle.linalg.lstsq( + x, y, rcond=self.rcond, driver=self.driver + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": self._input_data_1, - "y": self._input_data_2 - }, - fetch_list=[results]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": self._input_data_1, "y": self._input_data_2}, + fetch_list=[results], + ) self._result_solution = fetches[0] self._result_residuals = fetches[1] self._result_rank = fetches[2] @@ -141,44 +144,53 @@ class LinalgLstsqTestCase(unittest.TestCase): def assert_np_close(self): if len(self._input_shape_1) == 2: - np.testing.assert_allclose(self._result_solution, - self._output_solution, - rtol=1e-3) - if self._input_shape_1[-2] > self._input_shape_1[ - -1] and self._output_rank == self._input_shape_1[-1]: - np.testing.assert_allclose(self._result_residuals, - self._output_residuals, - rtol=1e-5) + np.testing.assert_allclose( + self._result_solution, self._output_solution, rtol=1e-3 + ) + if ( + self._input_shape_1[-2] > self._input_shape_1[-1] + and self._output_rank == self._input_shape_1[-1] + ): + np.testing.assert_allclose( + self._result_residuals, self._output_residuals, rtol=1e-5 + ) if self.driver in ("gelsy", "gelsd", "gelss"): - np.testing.assert_allclose(self._result_rank, - self._output_rank, - rtol=1e-5) + np.testing.assert_allclose( + self._result_rank, self._output_rank, rtol=1e-5 + ) if self.driver in ("gelsd", "gelss"): - np.testing.assert_allclose(self._result_sg_values, - self._output_sg_values, - rtol=1e-5) + np.testing.assert_allclose( + self._result_sg_values, self._output_sg_values, rtol=1e-5 + ) else: for i in range(len(self._output_solution)): - np.testing.assert_allclose(self._result_solution[i], - self._output_solution[i], - rtol=1e-3) - if self._input_shape_1[-2] > self._input_shape_1[ - -1] and self._output_rank[i] == self._input_shape_1[-1]: - np.testing.assert_allclose(self._result_residuals[i], - self._output_residuals[i], - rtol=1e-5) + np.testing.assert_allclose( + self._result_solution[i], + self._output_solution[i], + rtol=1e-3, + ) + if ( + self._input_shape_1[-2] > self._input_shape_1[-1] + and self._output_rank[i] == self._input_shape_1[-1] + ): + np.testing.assert_allclose( + self._result_residuals[i], + self._output_residuals[i], + rtol=1e-5, + ) if self.driver in ("gelsy", "gelsd", "gelss"): - np.testing.assert_allclose(self._result_rank[i], - self._output_rank[i], - rtol=1e-5) + np.testing.assert_allclose( + self._result_rank[i], self._output_rank[i], rtol=1e-5 + ) if self.driver in ("gelsd", "gelss"): - np.testing.assert_allclose(self._result_sg_values[i], - self._output_sg_values[i], - rtol=1e-5) + np.testing.assert_allclose( + self._result_sg_values[i], + self._output_sg_values[i], + rtol=1e-5, + ) class LinalgLstsqTestCase1(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = 1e-15 @@ -188,7 +200,6 @@ class LinalgLstsqTestCase1(LinalgLstsqTestCase): class LinalgLstsqTestCase2(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 @@ -198,7 +209,6 @@ class LinalgLstsqTestCase2(LinalgLstsqTestCase): class LinalgLstsqTestCase3(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 @@ -208,7 +218,6 @@ class LinalgLstsqTestCase3(LinalgLstsqTestCase): class LinalgLstsqTestCaseRcond(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-7 @@ -218,7 +227,6 @@ class LinalgLstsqTestCaseRcond(LinalgLstsqTestCase): class LinalgLstsqTestCaseGelsFloat32(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = None @@ -228,7 +236,6 @@ class LinalgLstsqTestCaseGelsFloat32(LinalgLstsqTestCase): class LinalgLstsqTestCaseGelsFloat64(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = None @@ -238,7 +245,6 @@ class LinalgLstsqTestCaseGelsFloat64(LinalgLstsqTestCase): class LinalgLstsqTestCaseGelssFloat64(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = None @@ -248,7 +254,6 @@ class LinalgLstsqTestCaseGelssFloat64(LinalgLstsqTestCase): class LinalgLstsqTestCaseGelsyFloat32(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = 1e-15 @@ -258,7 +263,6 @@ class LinalgLstsqTestCaseGelsyFloat32(LinalgLstsqTestCase): class LinalgLstsqTestCaseBatch1(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = 1e-15 @@ -268,7 +272,6 @@ class LinalgLstsqTestCaseBatch1(LinalgLstsqTestCase): class LinalgLstsqTestCaseBatch2(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 @@ -278,7 +281,6 @@ class LinalgLstsqTestCaseBatch2(LinalgLstsqTestCase): class LinalgLstsqTestCaseLarge1(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 @@ -288,7 +290,6 @@ class LinalgLstsqTestCaseLarge1(LinalgLstsqTestCase): class LinalgLstsqTestCaseLarge2(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 diff --git a/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py b/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py index 03e671d8832..8c39cbb7c95 100644 --- a/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py +++ b/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py @@ -20,7 +20,6 @@ import paddle.fluid.core as core class LinalgPinvTestCase(unittest.TestCase): - def setUp(self): self.init_config() self.generate_input() @@ -33,11 +32,13 @@ class LinalgPinvTestCase(unittest.TestCase): self._input_shape = (5, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) def generate_output(self): - self._output_data = np.linalg.pinv(self._input_data, \ - rcond=self.rcond, hermitian=self.hermitian) + self._output_data = np.linalg.pinv( + self._input_data, rcond=self.rcond, hermitian=self.hermitian + ) def init_config(self): self.dtype = 'float64' @@ -48,9 +49,9 @@ class LinalgPinvTestCase(unittest.TestCase): for place in self.places: paddle.disable_static(place) x = paddle.to_tensor(self._input_data, place=place) - out = paddle.linalg.pinv(x, - rcond=self.rcond, - hermitian=self.hermitian).numpy() + out = paddle.linalg.pinv( + x, rcond=self.rcond, hermitian=self.hermitian + ).numpy() if (np.abs(out - self._output_data) < 1e-6).any(): pass else: @@ -65,16 +66,20 @@ class LinalgPinvTestCase(unittest.TestCase): places.append(fluid.CUDAPlace(0)) for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): - x = paddle.fluid.data(name="input", - shape=self._input_shape, - dtype=self._input_data.dtype) - out = paddle.linalg.pinv(x, - rcond=self.rcond, - hermitian=self.hermitian) + x = paddle.fluid.data( + name="input", + shape=self._input_shape, + dtype=self._input_data.dtype, + ) + out = paddle.linalg.pinv( + x, rcond=self.rcond, hermitian=self.hermitian + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": self._input_data}, - fetch_list=[out]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": self._input_data}, + fetch_list=[out], + ) if (np.abs(fetches[0] - self._output_data) < 1e-6).any(): pass else: @@ -84,12 +89,12 @@ class LinalgPinvTestCase(unittest.TestCase): def test_grad(self): for place in self.places: - x = paddle.to_tensor(self._input_data, - place=place, - stop_gradient=False) - out = paddle.linalg.pinv(x, - rcond=self.rcond, - hermitian=self.hermitian) + x = paddle.to_tensor( + self._input_data, place=place, stop_gradient=False + ) + out = paddle.linalg.pinv( + x, rcond=self.rcond, hermitian=self.hermitian + ) try: out.backward() x_grad = x.grad @@ -99,75 +104,75 @@ class LinalgPinvTestCase(unittest.TestCase): class LinalgPinvTestCase1(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (4, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCase2(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (5, 4) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatch1(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatch2(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 4, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatch3(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 4) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatch4(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 6, 5, 4) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatchBig(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (2, 200, 300) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseFP32(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) def init_config(self): self.dtype = 'float32' @@ -176,12 +181,12 @@ class LinalgPinvTestCaseFP32(LinalgPinvTestCase): class LinalgPinvTestCaseRcond(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) def init_config(self): self.dtype = 'float64' @@ -190,12 +195,12 @@ class LinalgPinvTestCaseRcond(LinalgPinvTestCase): class LinalgPinvTestCaseHermitian1(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (5, 5) np.random.seed(123) - x = np.random.random(self._input_shape).astype(self.dtype) + \ - 1J * np.random.random(self._input_shape).astype(self.dtype) + x = np.random.random(self._input_shape).astype( + self.dtype + ) + 1j * np.random.random(self._input_shape).astype(self.dtype) self._input_data = x + x.transpose().conj() def init_config(self): @@ -205,12 +210,12 @@ class LinalgPinvTestCaseHermitian1(LinalgPinvTestCase): class LinalgPinvTestCaseHermitian2(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) - x = np.random.random(self._input_shape).astype(self.dtype) + \ - 1J * np.random.random(self._input_shape).astype(self.dtype) + x = np.random.random(self._input_shape).astype( + self.dtype + ) + 1j * np.random.random(self._input_shape).astype(self.dtype) self._input_data = x + x.transpose((0, 2, 1)).conj() def init_config(self): @@ -220,12 +225,12 @@ class LinalgPinvTestCaseHermitian2(LinalgPinvTestCase): class LinalgPinvTestCaseHermitian3(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) - x = np.random.random(self._input_shape).astype(self.dtype) + \ - 1J * np.random.random(self._input_shape).astype(self.dtype) + x = np.random.random(self._input_shape).astype( + self.dtype + ) + 1j * np.random.random(self._input_shape).astype(self.dtype) self._input_data = x + x.transpose((0, 2, 1)).conj() def init_config(self): @@ -235,7 +240,6 @@ class LinalgPinvTestCaseHermitian3(LinalgPinvTestCase): class LinalgPinvTestCaseHermitian4(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (5, 5) np.random.seed(123) @@ -249,7 +253,6 @@ class LinalgPinvTestCaseHermitian4(LinalgPinvTestCase): class LinalgPinvTestCaseHermitian5(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) @@ -263,7 +266,6 @@ class LinalgPinvTestCaseHermitian5(LinalgPinvTestCase): class LinalgPinvTestCaseHermitianFP32(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) diff --git a/python/paddle/fluid/tests/unittests/test_linear.py b/python/paddle/fluid/tests/unittests/test_linear.py index eaf2cb528cc..0cec0463542 100644 --- a/python/paddle/fluid/tests/unittests/test_linear.py +++ b/python/paddle/fluid/tests/unittests/test_linear.py @@ -21,14 +21,16 @@ import paddle.nn.functional as F class LinearTestCase(unittest.TestCase): - def setUp(self): self.dtype = 'float32' self.input = np.ones((3, 1, 2)).astype(self.dtype) self.weight = np.ones((2, 2)).astype(self.dtype) self.bias = np.ones((2)).astype(self.dtype) - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def functional(self, place): paddle.disable_static(place) @@ -46,17 +48,18 @@ class LinearTestCase(unittest.TestCase): learning_rate=1.0, trainable=False, regularizer=None, - initializer=paddle.fluid.initializer.ConstantInitializer(value=1.0)) + initializer=paddle.fluid.initializer.ConstantInitializer(value=1.0), + ) bias_attr = fluid.ParamAttr( name="linear_bias", learning_rate=1.0, trainable=False, regularizer=None, - initializer=paddle.fluid.initializer.ConstantInitializer(value=1.0)) - linear = paddle.nn.Linear(2, - 2, - weight_attr=weight_attr, - bias_attr=bias_attr) + initializer=paddle.fluid.initializer.ConstantInitializer(value=1.0), + ) + linear = paddle.nn.Linear( + 2, 2, weight_attr=weight_attr, bias_attr=bias_attr + ) y = linear(input) return y.numpy() @@ -75,18 +78,21 @@ class LinearTestCase(unittest.TestCase): if not paddle.is_compiled_with_cuda(): return paddle.seed(100) - linear = paddle.nn.Linear(2, - 3, - weight_attr=paddle.nn.initializer.Normal( - 0, 1.)) + linear = paddle.nn.Linear( + 2, 3, weight_attr=paddle.nn.initializer.Normal(0, 1.0) + ) paddle.nn.utils._stride_column(linear.weight) - expect = [[1.4349908, -0.8099171, -2.64788], - [-1.4981681, -1.1784115, -0.023253186]] + expect = [ + [1.4349908, -0.8099171, -2.64788], + [-1.4981681, -1.1784115, -0.023253186], + ] np.testing.assert_allclose(linear.weight.numpy(), expect, rtol=1e-05) linear = paddle.nn.Linear(2, 3) - expect = [[0.73261100, 0.43836895, 0.07908206], - [0.85075015, -1.04724526, 0.64371765]] + expect = [ + [0.73261100, 0.43836895, 0.07908206], + [0.85075015, -1.04724526, 0.64371765], + ] np.testing.assert_allclose(linear.weight.numpy(), expect, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py index 526601bbe68..203a4ac62ca 100755 --- a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py @@ -20,9 +20,16 @@ from op_test import OpTest class LinearChainCrfForward(object): - - def __init__(self, seq_start_positions, emission_weights, emission_row_max, - emission_exps, transition_weights, transition_exps, labels): + def __init__( + self, + seq_start_positions, + emission_weights, + emission_row_max, + emission_exps, + transition_weights, + transition_exps, + labels, + ): self.tag_num = emission_weights.shape[1] self.seq_num = len(seq_start_positions) - 1 @@ -46,8 +53,9 @@ class LinearChainCrfForward(object): # The output of linear chain crf operator. # alpha is a memo table in dynamic programming to calculate # nomalization factor. - self.alpha = np.zeros((seq_start_positions[-1], self.tag_num), - dtype="float64") + self.alpha = np.zeros( + (seq_start_positions[-1], self.tag_num), dtype="float64" + ) self.log_likelihood = np.zeros((self.seq_num, 1)) def _l1_norm(self, x): @@ -57,7 +65,7 @@ class LinearChainCrfForward(object): def _forward_a_sequence(self, x, x_row_max, x_exps, label, alpha): seq_len = x_row_max.shape[0] - log_likelihood = 0. + log_likelihood = 0.0 for i in range(self.tag_num): alpha[0, i] = self.a_exps[i] * x_exps[0, i] @@ -66,22 +74,21 @@ class LinearChainCrfForward(object): # calculate the unnormalized logits of the normalization factor. for k in range(1, seq_len): for i in range(self.tag_num): - s = 0. + s = 0.0 for j in range(self.tag_num): s += alpha[k - 1, j] * self.w_exps[j, i] alpha[k, i] = x_exps[k, i] * s log_likelihood -= x_row_max[k] + np.log(self._l1_norm(alpha[k, :])) - s = 0. + s = 0.0 for i in range(self.tag_num): s += alpha[-1, i] * self.b_exps[i] log_likelihood -= np.log(s) # calculate the nominator part. - log_likelihood += (self.a[label[0]] + x[0, label[0]] + - self.b[label[-1]]) + log_likelihood += self.a[label[0]] + x[0, label[0]] + self.b[label[-1]] for k in range(1, seq_len): - log_likelihood += (x[k, label[k]] + self.w[label[k - 1], label[k]]) + log_likelihood += x[k, label[k]] + self.w[label[k - 1], label[k]] return -log_likelihood def crf_forward_compute(self): @@ -91,14 +98,16 @@ class LinearChainCrfForward(object): if start >= end: continue self.log_likelihood[i] = self._forward_a_sequence( - self.x[start:end, :], self.x_row_max[start:end, :], - self.x_exps[start:end, :], self.labels[start:end, :], - self.alpha[start:end, :]) + self.x[start:end, :], + self.x_row_max[start:end, :], + self.x_exps[start:end, :], + self.labels[start:end, :], + self.alpha[start:end, :], + ) return self.alpha, self.log_likelihood class TestLinearChainCrfOp(OpTest): - def set_test_data(self): # TODO(caoying) Fix the unittest by: add the boundary cases when # sequence lengths are 1, 2, and 3. @@ -114,34 +123,41 @@ class TestLinearChainCrfOp(OpTest): lod[-1].append(random.randint(1, MAX_SEQ_LEN)) seq_start_pos.append(seq_start_pos[-1] + lod[-1][-1]) emission = np.random.uniform( - -1, 1, [seq_start_pos[-1], TAG_NUM]).astype("float64") + -1, 1, [seq_start_pos[-1], TAG_NUM] + ).astype("float64") emission_row_max = np.amax(emission, axis=1, keepdims=True) emission_exps = np.exp(emission - emission_row_max) - transition = np.random.uniform(-0.5, 0.5, - [TAG_NUM + 2, TAG_NUM]).astype("float64") + transition = np.random.uniform( + -0.5, 0.5, [TAG_NUM + 2, TAG_NUM] + ).astype("float64") transition_exps = np.exp(transition) - labels = np.random.randint(low=0, - high=TAG_NUM, - size=(seq_start_pos[-1], 1), - dtype="int64") + labels = np.random.randint( + low=0, high=TAG_NUM, size=(seq_start_pos[-1], 1), dtype="int64" + ) self.inputs = { "Emission": (emission, lod), "Transition": transition, - "Label": (labels, lod) + "Label": (labels, lod), } - crf = LinearChainCrfForward(seq_start_pos, emission, emission_row_max, - emission_exps, transition, transition_exps, - labels) + crf = LinearChainCrfForward( + seq_start_pos, + emission, + emission_row_max, + emission_exps, + transition, + transition_exps, + labels, + ) alpha, log_likelihood = crf.crf_forward_compute() self.outputs = { "Alpha": alpha, "EmissionExps": emission_exps, "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood + "LogLikelihood": log_likelihood, } def setUp(self): @@ -155,20 +171,19 @@ class TestLinearChainCrfOp(OpTest): self.check_grad(["Emission", "Transition"], "LogLikelihood") def test_check_grad_ignore_transition(self): - self.check_grad(["Emission"], - "LogLikelihood", - no_grad_set=set("Transition")) + self.check_grad( + ["Emission"], "LogLikelihood", no_grad_set=set("Transition") + ) class TestLinearChainCrfPaddingTensor(OpTest): - def seq_pad(self, data, length): max_len = np.max(length) shape = [len(length), max_len] + list(data.shape[1:]) padded = np.zeros(shape).astype(data.dtype) offset = 0 for i, l in enumerate(length): - padded[i, 0:l] = data[offset:offset + l] + padded[i, 0:l] = data[offset : offset + l] offset += l return padded @@ -179,7 +194,7 @@ class TestLinearChainCrfPaddingTensor(OpTest): padded = np.ones(shape).astype(data.dtype) offset = 0 for i, l in enumerate(length): - padded[i, 0:l] = data[offset:offset + l] + padded[i, 0:l] = data[offset : offset + l] offset += l return padded @@ -196,32 +211,39 @@ class TestLinearChainCrfPaddingTensor(OpTest): lod[-1].append(random.randint(1, MAX_SEQ_LEN)) seq_start_pos.append(seq_start_pos[-1] + lod[-1][-1]) emission = np.random.uniform( - -1, 1, [seq_start_pos[-1], TAG_NUM]).astype("float64") + -1, 1, [seq_start_pos[-1], TAG_NUM] + ).astype("float64") emission_row_max = np.amax(emission, axis=1, keepdims=True) emission_exps = np.exp(emission - emission_row_max) - transition = np.random.uniform(-0.5, 0.5, - [TAG_NUM + 2, TAG_NUM]).astype("float64") + transition = np.random.uniform( + -0.5, 0.5, [TAG_NUM + 2, TAG_NUM] + ).astype("float64") transition_exps = np.exp(transition) - labels = np.random.randint(low=0, - high=TAG_NUM, - size=(seq_start_pos[-1], 1), - dtype="int64") + labels = np.random.randint( + low=0, high=TAG_NUM, size=(seq_start_pos[-1], 1), dtype="int64" + ) self.inputs = { "Emission": self.seq_pad(emission, lod[0]), "Transition": transition, "Label": self.seq_pad(labels, lod[0]), - "Length": np.array(lod).astype("int64") + "Length": np.array(lod).astype("int64"), } - crf = LinearChainCrfForward(seq_start_pos, emission, emission_row_max, - emission_exps, transition, transition_exps, - labels) + crf = LinearChainCrfForward( + seq_start_pos, + emission, + emission_row_max, + emission_exps, + transition, + transition_exps, + labels, + ) alpha, log_likelihood = crf.crf_forward_compute() self.outputs = { "Alpha": self.seq_pad(alpha, lod[0]), "EmissionExps": self.seq_pad_exps(emission_exps, lod[0]), "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood + "LogLikelihood": log_likelihood, } def setUp(self): @@ -235,9 +257,9 @@ class TestLinearChainCrfPaddingTensor(OpTest): self.check_grad(["Emission", "Transition"], "LogLikelihood") def test_check_grad_ignore_transition(self): - self.check_grad(["Emission"], - "LogLikelihood", - no_grad_set=set("Transition")) + self.check_grad( + ["Emission"], "LogLikelihood", no_grad_set=set("Transition") + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_linear_interp_op.py b/python/paddle/fluid/tests/unittests/test_linear_interp_op.py index cd5d799a5bd..f52e597d665 100755 --- a/python/paddle/fluid/tests/unittests/test_linear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_interp_op.py @@ -23,13 +23,15 @@ from paddle.fluid import Program, program_guard from paddle.nn.functional import interpolate -def linear_interp_np(input, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): +def linear_interp_np( + input, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): if data_layout == "NHWC": input = np.transpose(input, (0, 2, 1)) # NHWC => NCHW if out_size is not None: @@ -40,7 +42,7 @@ def linear_interp_np(input, ratio_w = 0.0 if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -48,22 +50,23 @@ def linear_interp_np(input, out = np.zeros((batch_size, channel, out_w)) for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, - j] = w2lambda * input[:, :, w] + w1lambda * input[:, :, w + wid] + out[:, :, j] = ( + w2lambda * input[:, :, w] + w1lambda * input[:, :, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 1)) # NCHW => NHWC @@ -72,7 +75,6 @@ def linear_interp_np(input, class TestLinearInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -91,9 +93,15 @@ class TestLinearInterpOp(OpTest): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = linear_interp_np( + input_np, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -106,7 +114,7 @@ class TestLinearInterpOp(OpTest): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -123,59 +131,63 @@ class TestLinearInterpOp(OpTest): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 1 class TestLinearInterpOpDataLayout(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 1 self.data_layout = 'NHWC' class TestLinearInterpOpAlignMode(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 0 class TestLinearInterpOpScale(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 self.scale = 0.5 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 0 class TestLinearInterpOpSizeTensor(TestLinearInterpOp): - def setUp(self): self.out_size = None self.actual_shape = None @@ -196,9 +208,15 @@ class TestLinearInterpOpSizeTensor(TestLinearInterpOp): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = linear_interp_np( + input_np, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None and self.shape_by_1Dtensor: @@ -208,8 +226,9 @@ class TestLinearInterpOpSizeTensor(TestLinearInterpOp): else: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs = { @@ -218,78 +237,88 @@ class TestLinearInterpOpSizeTensor(TestLinearInterpOp): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} class TestResizeLinearAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[1, 3, 64], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[1], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[1], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = fluid.layers.resize_linear(x, - out_shape=[ - 128, - ], - align_mode=1, - align_corners=False) - out2 = fluid.layers.resize_linear(x, - out_shape=[128], - align_mode=1, - align_corners=False) - out3 = fluid.layers.resize_linear(x, - out_shape=shape_tensor, - align_mode=1, - align_corners=False) - out4 = fluid.layers.resize_linear(x, - out_shape=[ - 128, - ], - actual_shape=actual_size, - align_mode=1, - align_corners=False) - out5 = fluid.layers.resize_linear(x, - scale=scale_tensor, - align_mode=1, - align_corners=False) - - out6 = interpolate(x, - scale_factor=scale_tensor, - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') - out7 = interpolate(x, - size=[ - 128, - ], - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') - out8 = interpolate(x, - size=shape_tensor, - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = fluid.layers.resize_linear( + x, + out_shape=[ + 128, + ], + align_mode=1, + align_corners=False, + ) + out2 = fluid.layers.resize_linear( + x, out_shape=[128], align_mode=1, align_corners=False + ) + out3 = fluid.layers.resize_linear( + x, out_shape=shape_tensor, align_mode=1, align_corners=False + ) + out4 = fluid.layers.resize_linear( + x, + out_shape=[ + 128, + ], + actual_shape=actual_size, + align_mode=1, + align_corners=False, + ) + out5 = fluid.layers.resize_linear( + x, scale=scale_tensor, align_mode=1, align_corners=False + ) + + out6 = interpolate( + x, + scale_factor=scale_tensor, + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) + out7 = interpolate( + x, + size=[ + 128, + ], + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) + out8 = interpolate( + x, + size=shape_tensor, + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) x_data = np.random.random((1, 3, 64)).astype("float32") dim_data = np.array([128]).astype("int32") - shape_data = np.array([ - 128, - ]).astype("int32") - actual_size_data = np.array([ - 128, - ]).astype("int32") + shape_data = np.array( + [ + 128, + ] + ).astype("int32") + actual_size_data = np.array( + [ + 128, + ] + ).astype("int32") scale_data = np.array([2.0]).astype("float32") if core.is_compiled_with_cuda(): @@ -305,46 +334,45 @@ class TestResizeLinearAPI(unittest.TestCase): "dim": dim_data, "shape_tensor": shape_data, "actual_size": actual_size_data, - "scale_tensor": scale_data + "scale_tensor": scale_data, }, fetch_list=[out1, out2, out3, out4, out5, out6, out7, out8], - return_numpy=True) + return_numpy=True, + ) - expect_res = linear_interp_np(x_data, - out_w=128, - align_mode=1, - align_corners=False) + expect_res = linear_interp_np( + x_data, out_w=128, align_mode=1, align_corners=False + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) class TestLinearInterpOpAPI2_0(unittest.TestCase): - def test_case(self): # dygraph x_data = np.random.random((1, 3, 128)).astype("float32") - us_1 = paddle.nn.Upsample(size=[ - 64, - ], - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') + us_1 = paddle.nn.Upsample( + size=[ + 64, + ], + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) interp = us_1(x) - expect = linear_interp_np(x_data, - out_w=64, - align_mode=1, - align_corners=False) + expect = linear_interp_np( + x_data, out_w=64, align_mode=1, align_corners=False + ) np.testing.assert_allclose(interp.numpy(), expect, rtol=1e-05) class TestResizeLinearOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -357,9 +385,14 @@ class TestResizeLinearOpUint8(OpTest): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode) + output_np = linear_interp_np( + input_np, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -369,7 +402,7 @@ class TestResizeLinearOpUint8(OpTest): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } self.outputs = {'Out': output_np} @@ -383,42 +416,48 @@ class TestResizeLinearOpUint8(OpTest): self.interp_method = 'linear' self.input_shape = [2, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = True self.align_mode = 1 class TestLinearInterpOpException(unittest.TestCase): - def test_exception(self): - def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out = fluid.layers.resize_linear(x1, - out_shape=[ - 256, - ], - data_format='NCW') + out = fluid.layers.resize_linear( + x1, + out_shape=[ + 256, + ], + data_format='NCW', + ) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out = fluid.layers.resize_linear(x2, - out_shape=[ - 256, - ], - data_format='NHWCD') + out = fluid.layers.resize_linear( + x2, + out_shape=[ + 256, + ], + data_format='NHWCD', + ) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out = fluid.layers.resize_linear(x3, - out_shape=[ - 256, - 256, - ], - data_format='NHWC') + out = fluid.layers.resize_linear( + x3, + out_shape=[ + 256, + 256, + ], + data_format='NHWC', + ) self.assertRaises(ValueError, input_shape_error) self.assertRaises(ValueError, data_format_error) @@ -426,36 +465,41 @@ class TestLinearInterpOpException(unittest.TestCase): class TestLinearInterpOpError(unittest.TestCase): - def test_error(self): with program_guard(Program(), Program()): def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out1 = paddle.nn.Upsample(size=[ - 256, - ], - data_format='NCW', - mode='linear') + out1 = paddle.nn.Upsample( + size=[ + 256, + ], + data_format='NCW', + mode='linear', + ) out1_res = out1(x1) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out2 = paddle.nn.Upsample(size=[ - 256, - ], - data_format='NHWCD', - mode='linear') + out2 = paddle.nn.Upsample( + size=[ + 256, + ], + data_format='NHWCD', + mode='linear', + ) out2_res = out2(x2) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out3 = paddle.nn.Upsample(size=[ - 256, - 256, - ], - data_format='NHWC', - mode='linear') + out3 = paddle.nn.Upsample( + size=[ + 256, + 256, + ], + data_format='NHWC', + mode='linear', + ) out3_res = out3(x3) self.assertRaises(ValueError, input_shape_error) diff --git a/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py index 9fbbc8ceaa0..30e7d2ed97b 100755 --- a/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py @@ -23,18 +23,20 @@ from paddle.fluid import Program, program_guard from paddle.nn.functional import interpolate -def linear_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='NCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='linear', - align_corners=True, - align_mode=0): +def linear_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='NCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='linear', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -44,21 +46,35 @@ def linear_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.linear_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, scale, - interp_method, align_corners, align_mode) - - -def linear_interp_np(input, - out_w, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): + return paddle._C_ops.linear_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) + + +def linear_interp_np( + input, + out_w, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): if data_layout == "NHWC": input = np.transpose(input, (0, 2, 1)) # NHWC => NCHW if out_size is not None: @@ -69,7 +85,7 @@ def linear_interp_np(input, ratio_w = 0.0 if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -80,22 +96,23 @@ def linear_interp_np(input, out = np.zeros((batch_size, channel, out_w)) for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, - j] = w2lambda * input[:, :, w] + w1lambda * input[:, :, w + wid] + out[:, :, j] = ( + w2lambda * input[:, :, w] + w1lambda * input[:, :, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 1)) # NCHW => NHWC @@ -104,7 +121,6 @@ def linear_interp_np(input, class TestLinearInterpOp(OpTest): - def setUp(self): self.python_api = linear_interp_test self.out_size = None @@ -129,9 +145,16 @@ class TestLinearInterpOp(OpTest): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.scale, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = linear_interp_np( + input_np, + out_w, + self.scale, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -143,7 +166,7 @@ class TestLinearInterpOp(OpTest): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale > 0: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -165,58 +188,62 @@ class TestLinearInterpOp(OpTest): self.input_shape = [1, 3, 100] self.out_w = 50 self.scale = 0.5 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 1 class TestLinearInterpOpDataLayout(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 100, 3] self.out_w = 50 self.scale = 0.5 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 1 self.data_layout = 'NHWC' class TestLinearInterpOpAlignMode(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 self.scale = 0.5 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 0 class TestLinearInterpOpScale(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 self.scale = 0.8 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 0 class TestLinearInterpOpSizeTensor(TestLinearInterpOp): - def setUp(self): self.python_api = linear_interp_test self.out_size = None @@ -242,9 +269,16 @@ class TestLinearInterpOpSizeTensor(TestLinearInterpOp): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, 0, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = linear_interp_np( + input_np, + out_w, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None and self.shape_by_1Dtensor: @@ -254,8 +288,9 @@ class TestLinearInterpOpSizeTensor(TestLinearInterpOp): else: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs = { @@ -263,7 +298,7 @@ class TestLinearInterpOpSizeTensor(TestLinearInterpOp): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale > 0: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -275,72 +310,82 @@ class TestLinearInterpOpSizeTensor(TestLinearInterpOp): class TestResizeLinearAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[1, 3, 64], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[1], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[1], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = fluid.layers.resize_linear(x, - out_shape=[ - 128, - ], - align_mode=1, - align_corners=False) - out2 = fluid.layers.resize_linear(x, - out_shape=[128], - align_mode=1, - align_corners=False) - out3 = fluid.layers.resize_linear(x, - out_shape=shape_tensor, - align_mode=1, - align_corners=False) - out4 = fluid.layers.resize_linear(x, - out_shape=[ - 128, - ], - actual_shape=actual_size, - align_mode=1, - align_corners=False) - out5 = fluid.layers.resize_linear(x, - scale=scale_tensor, - align_mode=1, - align_corners=False) - - out6 = interpolate(x, - scale_factor=scale_tensor, - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') - out7 = interpolate(x, - size=[ - 128, - ], - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') - out8 = interpolate(x, - size=shape_tensor, - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = fluid.layers.resize_linear( + x, + out_shape=[ + 128, + ], + align_mode=1, + align_corners=False, + ) + out2 = fluid.layers.resize_linear( + x, out_shape=[128], align_mode=1, align_corners=False + ) + out3 = fluid.layers.resize_linear( + x, out_shape=shape_tensor, align_mode=1, align_corners=False + ) + out4 = fluid.layers.resize_linear( + x, + out_shape=[ + 128, + ], + actual_shape=actual_size, + align_mode=1, + align_corners=False, + ) + out5 = fluid.layers.resize_linear( + x, scale=scale_tensor, align_mode=1, align_corners=False + ) + + out6 = interpolate( + x, + scale_factor=scale_tensor, + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) + out7 = interpolate( + x, + size=[ + 128, + ], + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) + out8 = interpolate( + x, + size=shape_tensor, + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) x_data = np.random.random((1, 3, 64)).astype("float32") dim_data = np.array([128]).astype("int32") - shape_data = np.array([ - 128, - ]).astype("int32") - actual_size_data = np.array([ - 128, - ]).astype("int32") + shape_data = np.array( + [ + 128, + ] + ).astype("int32") + actual_size_data = np.array( + [ + 128, + ] + ).astype("int32") scale_data = np.array([2.0]).astype("float32") if core.is_compiled_with_cuda(): @@ -356,44 +401,43 @@ class TestResizeLinearAPI(unittest.TestCase): "dim": dim_data, "shape_tensor": shape_data, "actual_size": actual_size_data, - "scale_tensor": scale_data + "scale_tensor": scale_data, }, fetch_list=[out1, out2, out3, out4, out5, out6, out7, out8], - return_numpy=True) + return_numpy=True, + ) - expect_res = linear_interp_np(x_data, - out_w=128, - align_mode=1, - align_corners=False) + expect_res = linear_interp_np( + x_data, out_w=128, align_mode=1, align_corners=False + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) class TestLinearInterpOpAPI2_0(unittest.TestCase): - def test_case(self): # dygraph x_data = np.random.random((1, 3, 128)).astype("float32") - us_1 = paddle.nn.Upsample(size=[64], - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') + us_1 = paddle.nn.Upsample( + size=[64], + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) interp = us_1(x) - expect = linear_interp_np(x_data, - out_w=64, - align_mode=1, - align_corners=False) + expect = linear_interp_np( + x_data, out_w=64, align_mode=1, align_corners=False + ) np.testing.assert_allclose(interp.numpy(), expect, rtol=1e-05) class TestResizeLinearOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -410,9 +454,15 @@ class TestResizeLinearOpUint8(OpTest): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, 0, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode) + output_np = linear_interp_np( + input_np, + out_w, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -421,7 +471,7 @@ class TestResizeLinearOpUint8(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } if self.scale > 0: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -441,42 +491,48 @@ class TestResizeLinearOpUint8(OpTest): self.interp_method = 'linear' self.input_shape = [2, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = True self.align_mode = 1 class TestLinearInterpOpException(unittest.TestCase): - def test_exception(self): - def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out = fluid.layers.resize_linear(x1, - out_shape=[ - 256, - ], - data_format='NCW') + out = fluid.layers.resize_linear( + x1, + out_shape=[ + 256, + ], + data_format='NCW', + ) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out = fluid.layers.resize_linear(x2, - out_shape=[ - 256, - ], - data_format='NHWCD') + out = fluid.layers.resize_linear( + x2, + out_shape=[ + 256, + ], + data_format='NHWCD', + ) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out = fluid.layers.resize_linear(x3, - out_shape=[ - 256, - 256, - ], - data_format='NHWC') + out = fluid.layers.resize_linear( + x3, + out_shape=[ + 256, + 256, + ], + data_format='NHWC', + ) self.assertRaises(ValueError, input_shape_error) self.assertRaises(ValueError, data_format_error) @@ -484,29 +540,28 @@ class TestLinearInterpOpException(unittest.TestCase): class TestLinearInterpOpError(unittest.TestCase): - def test_error(self): with program_guard(Program(), Program()): def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out1 = paddle.nn.Upsample(size=[256], - data_format='NCW', - mode='linear') + out1 = paddle.nn.Upsample( + size=[256], data_format='NCW', mode='linear' + ) out1_res = out1(x1) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out2 = paddle.nn.Upsample(size=[256], - data_format='NHWCD', - mode='linear') + out2 = paddle.nn.Upsample( + size=[256], data_format='NHWCD', mode='linear' + ) out2_res = out2(x2) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out3 = paddle.nn.Upsample(size=[256, 256], - data_format='NHWC', - mode='linear') + out3 = paddle.nn.Upsample( + size=[256, 256], data_format='NHWC', mode='linear' + ) out3_res = out3(x3) self.assertRaises(ValueError, input_shape_error) @@ -514,10 +569,10 @@ class TestLinearInterpOpError(unittest.TestCase): self.assertRaises(ValueError, out_shape_error) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLinearInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 64] @@ -531,12 +586,14 @@ class TestLinearInterpOpForFloat16(unittest.TestCase): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - scale_factor=self.scale, - mode=self.interp_method, - align_mode=self.align_mode, - align_corners=self.align_corners, - data_format=self.data_layout) + y = interpolate( + x, + scale_factor=self.scale, + mode=self.interp_method, + align_mode=self.align_mode, + align_corners=self.align_corners, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_linspace.py b/python/paddle/fluid/tests/unittests/test_linspace.py index c38d5b1fe59..3549d46ec9e 100644 --- a/python/paddle/fluid/tests/unittests/test_linspace.py +++ b/python/paddle/fluid/tests/unittests/test_linspace.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import _test_eager_guard class TestLinspaceOpCommonCase(OpTest): - def setUp(self): self.op_type = "linspace" self.python_api = paddle.linspace @@ -31,7 +30,7 @@ class TestLinspaceOpCommonCase(OpTest): self.inputs = { 'Start': np.array([0]).astype(dtype), 'Stop': np.array([10]).astype(dtype), - 'Num': np.array([11]).astype('int32') + 'Num': np.array([11]).astype('int32'), } self.attrs = {'dtype': int(core.VarDesc.VarType.FP32)} @@ -42,7 +41,6 @@ class TestLinspaceOpCommonCase(OpTest): class TestLinspaceOpReverseCase(OpTest): - def setUp(self): self.op_type = "linspace" self.python_api = paddle.linspace @@ -50,7 +48,7 @@ class TestLinspaceOpReverseCase(OpTest): self.inputs = { 'Start': np.array([10]).astype(dtype), 'Stop': np.array([0]).astype(dtype), - 'Num': np.array([11]).astype('int32') + 'Num': np.array([11]).astype('int32'), } self.attrs = {'dtype': int(core.VarDesc.VarType.FP32)} @@ -61,7 +59,6 @@ class TestLinspaceOpReverseCase(OpTest): class TestLinspaceOpNumOneCase(OpTest): - def setUp(self): self.op_type = "linspace" self.python_api = paddle.linspace @@ -69,7 +66,7 @@ class TestLinspaceOpNumOneCase(OpTest): self.inputs = { 'Start': np.array([10]).astype(dtype), 'Stop': np.array([0]).astype(dtype), - 'Num': np.array([1]).astype('int32') + 'Num': np.array([1]).astype('int32'), } self.attrs = {'dtype': int(core.VarDesc.VarType.FP32)} @@ -80,7 +77,6 @@ class TestLinspaceOpNumOneCase(OpTest): class TestLinspaceAPI(unittest.TestCase): - def test_variable_input1(self): start = paddle.full(shape=[1], fill_value=0, dtype='float32') stop = paddle.full(shape=[1], fill_value=10, dtype='float32') @@ -106,17 +102,16 @@ class TestLinspaceAPI(unittest.TestCase): out_2 = paddle.linspace(0, 10, 5, dtype=np.float32) out_3 = paddle.linspace(0, 10, 5, dtype=core.VarDesc.VarType.FP32) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), fetch_list=[out_1, out_2, out_3] + ) assert np.array_equal(res_1, res_2) def test_name(self): with paddle.static.program_guard(paddle.static.Program()): - out = paddle.linspace(0, - 10, - 5, - dtype='float32', - name='linspace_res') + out = paddle.linspace( + 0, 10, 5, dtype='float32', name='linspace_res' + ) assert 'linspace_res' in out.name def test_imperative(self): @@ -139,7 +134,6 @@ class TestLinspaceAPI(unittest.TestCase): class TestLinspaceOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): diff --git a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py index 897689d5597..3bac2d40ae3 100644 --- a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py +++ b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py @@ -50,18 +50,21 @@ def run_pserver(use_cuda, sync_mode, ip, port, trainers, trainer_id): config = fluid.DistributeTranspilerConfig() config.sync_mode = sync_mode t = fluid.DistributeTranspiler(config=config) - t.transpile(trainer_id, - pservers=pserver_endpoints, - trainers=trainers, - sync_mode=sync_mode) + t.transpile( + trainer_id, + pservers=pserver_endpoints, + trainers=trainers, + sync_mode=sync_mode, + ) pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) exe.run(pserver_startup) exe.run(pserver_prog) -def run_pserver_with_empty_block(use_cuda, sync_mode, ip, port, trainers, - trainer_id): +def run_pserver_with_empty_block( + use_cuda, sync_mode, ip, port, trainers, trainer_id +): remove_ps_flag(os.getpid()) x = fluid.layers.data(name='x', shape=[1], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None, bias_attr=False) @@ -87,15 +90,17 @@ def run_pserver_with_empty_block(use_cuda, sync_mode, ip, port, trainers, config.slice_var_up = False t = fluid.DistributeTranspiler(config=config) - t.transpile(trainer_id, - pservers=pserver_endpoints, - trainers=trainers, - sync_mode=sync_mode) + t.transpile( + trainer_id, + pservers=pserver_endpoints, + trainers=trainers, + sync_mode=sync_mode, + ) pserver_prog = t.get_pserver_program(ps2) # pserver2 have no parameter - assert (len(pserver_prog.blocks) == 2) - assert (len(pserver_prog.blocks[1].ops) == 0) + assert len(pserver_prog.blocks) == 2 + assert len(pserver_prog.blocks[1].ops) == 0 pserver_startup = t.get_startup_program(ps2, pserver_prog) exe.run(pserver_startup) @@ -108,7 +113,6 @@ def gen_complete_file_flag(flag_file): class TestListenAndServOp(unittest.TestCase): - def setUp(self): self.ps_timeout = 200 self.ip = "127.0.0.1" @@ -117,9 +121,17 @@ class TestListenAndServOp(unittest.TestCase): self.trainer_id = 0 def _start_pserver(self, use_cuda, sync_mode, pserver_func): - p = Process(target=pserver_func, - args=(use_cuda, sync_mode, self.ip, self.port, - self.trainers, self.trainer_id)) + p = Process( + target=pserver_func, + args=( + use_cuda, + sync_mode, + self.ip, + self.port, + self.trainers, + self.trainer_id, + ), + ) p.daemon = True p.start() return p diff --git a/python/paddle/fluid/tests/unittests/test_load_op.py b/python/paddle/fluid/tests/unittests/test_load_op.py index 1b81f3b8f40..ed123e06a03 100644 --- a/python/paddle/fluid/tests/unittests/test_load_op.py +++ b/python/paddle/fluid/tests/unittests/test_load_op.py @@ -21,8 +21,7 @@ import tempfile class TestLoadOp(unittest.TestCase): - """ Test load operator. - """ + """Test load operator.""" def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -37,13 +36,17 @@ class TestLoadOp(unittest.TestCase): param_attr=fluid.ParamAttr( name='w', initializer=fluid.initializer.NumpyArrayInitializer( - self.ones))) + self.ones + ), + ), + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) - fluid.io.save_persistables(exe, - dirname=os.path.join(self.temp_dir.name, - "./model"), - main_program=main_prog) + fluid.io.save_persistables( + exe, + dirname=os.path.join(self.temp_dir.name, "./model"), + main_program=main_prog, + ) def tearDown(self): self.temp_dir.cleanup() @@ -53,8 +56,9 @@ class TestLoadOp(unittest.TestCase): start_prog = fluid.Program() with fluid.program_guard(main_prog, start_prog): var = layers.create_tensor(dtype='float32') - layers.load(var, - file_path=os.path.join(self.temp_dir.name, './model/w')) + layers.load( + var, file_path=os.path.join(self.temp_dir.name, './model/w') + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) diff --git a/python/paddle/fluid/tests/unittests/test_load_op_xpu.py b/python/paddle/fluid/tests/unittests/test_load_op_xpu.py index 957fe5aa420..05ad3dc7762 100644 --- a/python/paddle/fluid/tests/unittests/test_load_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/test_load_op_xpu.py @@ -21,11 +21,11 @@ import paddle.fluid.layers as layers import paddle -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestLoadOpXpu(unittest.TestCase): - """ Test load operator. - """ + """Test load operator.""" def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -41,12 +41,15 @@ class TestLoadOpXpu(unittest.TestCase): param_attr=fluid.ParamAttr( name='w', initializer=fluid.initializer.NumpyArrayInitializer( - self.ones))) + self.ones + ), + ), + ) exe = fluid.Executor(fluid.XPUPlace(0)) exe.run(start_prog) - fluid.io.save_persistables(exe, - dirname=self.model_path, - main_program=main_prog) + fluid.io.save_persistables( + exe, dirname=self.model_path, main_program=main_prog + ) def tearDown(self): self.temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py index 4d47c94f2fa..6bace212953 100644 --- a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py +++ b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py @@ -24,19 +24,23 @@ import tempfile def convolutional_neural_network(img): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') return prediction @@ -54,7 +58,6 @@ def static_train_net(img, label): class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.seed = 90 @@ -72,30 +75,38 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): startup_program = fluid.default_startup_program() main_program = fluid.default_main_program() - img = fluid.data(name='img', - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 1, 28, 28], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') prediction, avg_loss = static_train_net(img, label) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) exe.run(startup_program) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=100), - batch_size=self.batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.mnist.train(), buf_size=100 + ), + batch_size=self.batch_size, + ) for _ in range(0, self.epoch_num): for batch_id, data in enumerate(train_reader()): - exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_loss]) + exe.run( + main_program, + feed=feeder.feed(data), + fetch_list=[avg_loss], + ) if batch_id > self.batch_num: break @@ -103,18 +114,22 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): static_param_dict = {} for param in fluid.default_main_program().all_parameters(): static_param_dict[param.name] = fluid.executor._fetch_var( - param.name) + param.name + ) if only_params: - fluid.io.save_params(exe, - self.save_dirname, - filename=self.params_filename) + fluid.io.save_params( + exe, self.save_dirname, filename=self.params_filename + ) else: fluid.io.save_inference_model( - self.save_dirname, ["img"], [prediction], + self.save_dirname, + ["img"], + [prediction], exe, model_filename=self.model_filename, - params_filename=self.params_filename) + params_filename=self.params_filename, + ) return static_param_dict @@ -124,7 +139,8 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): def test_load_default(self): self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.default") + self.temp_dir.name, "static_mnist.load_state_dict.default" + ) self.model_filename = None self.params_filename = None orig_param_dict = self.train_and_save_model() @@ -137,38 +153,45 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): def test_load_with_model_filename(self): self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.model_filename") + self.temp_dir.name, "static_mnist.load_state_dict.model_filename" + ) self.model_filename = "static_mnist.model" self.params_filename = None orig_param_dict = self.train_and_save_model() load_param_dict, _ = fluid.load_dygraph( - self.save_dirname, model_filename=self.model_filename) + self.save_dirname, model_filename=self.model_filename + ) self.check_load_state_dict(orig_param_dict, load_param_dict) - new_load_param_dict = paddle.load(self.save_dirname, - model_filename=self.model_filename) + new_load_param_dict = paddle.load( + self.save_dirname, model_filename=self.model_filename + ) self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_param_filename(self): self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.param_filename") + self.temp_dir.name, "static_mnist.load_state_dict.param_filename" + ) self.model_filename = None self.params_filename = "static_mnist.params" orig_param_dict = self.train_and_save_model() load_param_dict, _ = fluid.load_dygraph( - self.save_dirname, params_filename=self.params_filename) + self.save_dirname, params_filename=self.params_filename + ) self.check_load_state_dict(orig_param_dict, load_param_dict) - new_load_param_dict = paddle.load(self.save_dirname, - params_filename=self.params_filename) + new_load_param_dict = paddle.load( + self.save_dirname, params_filename=self.params_filename + ) self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_model_and_param_filename(self): self.save_dirname = os.path.join( self.temp_dir.name, - "static_mnist.load_state_dict.model_and_param_filename") + "static_mnist.load_state_dict.model_and_param_filename", + ) self.model_filename = "static_mnist.model" self.params_filename = "static_mnist.params" orig_param_dict = self.train_and_save_model() @@ -176,17 +199,21 @@ class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): load_param_dict, _ = fluid.load_dygraph( self.save_dirname, params_filename=self.params_filename, - model_filename=self.model_filename) + model_filename=self.model_filename, + ) self.check_load_state_dict(orig_param_dict, load_param_dict) - new_load_param_dict = paddle.load(self.save_dirname, - params_filename=self.params_filename, - model_filename=self.model_filename) + new_load_param_dict = paddle.load( + self.save_dirname, + params_filename=self.params_filename, + model_filename=self.model_filename, + ) self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_state_dict_from_save_params(self): self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.save_params") + self.temp_dir.name, "static_mnist.load_state_dict.save_params" + ) self.params_filename = None orig_param_dict = self.train_and_save_model(True) diff --git a/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py b/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py index 2d9bd1ac650..756b2ab7732 100644 --- a/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py +++ b/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py @@ -20,7 +20,6 @@ from paddle.fluid.executor import Executor class TestLoadVarsShapeCheck(unittest.TestCase): - def setUp(self): self.model_path = "./model_temp/" diff --git a/python/paddle/fluid/tests/unittests/test_locality_aware_nms_op.py b/python/paddle/fluid/tests/unittests/test_locality_aware_nms_op.py index b79b1bff8b9..a09707c5767 100644 --- a/python/paddle/fluid/tests/unittests/test_locality_aware_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_locality_aware_nms_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -25,13 +25,15 @@ def weight_merge(box1, box2, score1, score2): box2[i] = (box1[i] * score1 + box2[i] * score2) / (score1 + score2) -def nms(boxes, - scores, - score_threshold, - nms_threshold, - top_k=200, - normalized=True, - eta=1.0): +def nms( + boxes, + scores, + score_threshold, + nms_threshold, + top_k=200, + normalized=True, + eta=1.0, +): """Apply non-maximum suppression at test time to avoid detecting too many overlapping bounding boxes for a given object. Args: @@ -48,11 +50,13 @@ def nms(boxes, """ index = -1 for i in range(boxes.shape[0]): - if index > -1 and iou(boxes[i], boxes[index], - normalized) > nms_threshold: + if ( + index > -1 + and iou(boxes[i], boxes[index], normalized) > nms_threshold + ): weight_merge(boxes[i], boxes[index], scores[i], scores[index]) scores[index] += scores[i] - scores[i] = score_threshold - 1. + scores[i] = score_threshold - 1.0 else: index = i @@ -90,8 +94,17 @@ def nms(boxes, return selected_indices -def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, - nms_top_k, keep_top_k, normalized, shared): +def multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared, +): if shared: class_num = scores.shape[0] priorbox_num = scores.shape[1] @@ -102,13 +115,26 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, selected_indices = {} num_det = 0 for c in range(class_num): - if c == background: continue + if c == background: + continue if shared: - indices = nms(boxes, scores[c], score_threshold, nms_threshold, - nms_top_k, normalized) + indices = nms( + boxes, + scores[c], + score_threshold, + nms_threshold, + nms_top_k, + normalized, + ) else: - indices = nms(boxes[:, c, :], scores[:, c], score_threshold, - nms_threshold, nms_top_k, normalized) + indices = nms( + boxes[:, c, :], + scores[:, c], + score_threshold, + nms_threshold, + nms_top_k, + normalized, + ) selected_indices[c] = indices num_det += len(indices) @@ -121,9 +147,9 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, else: score_index.append((scores[idx][c], c, idx)) - sorted_score_index = sorted(score_index, - key=lambda tup: tup[0], - reverse=True) + sorted_score_index = sorted( + score_index, key=lambda tup: tup[0], reverse=True + ) sorted_score_index = sorted_score_index[:keep_top_k] selected_indices = {} @@ -139,29 +165,33 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, return selected_indices, num_det -def batched_multiclass_nms(boxes, - scores, - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized=True): +def batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized=True, +): batch_size = scores.shape[0] num_boxes = scores.shape[2] det_outs = [] lod = [] for n in range(batch_size): - nmsed_outs, nmsed_num = multiclass_nms(boxes[n], - scores[n], - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized, - shared=True) + nmsed_outs, nmsed_num = multiclass_nms( + boxes[n], + scores[n], + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared=True, + ) lod.append(nmsed_num) if nmsed_num == 0: @@ -170,19 +200,25 @@ def batched_multiclass_nms(boxes, for c, indices in nmsed_outs.items(): for idx in indices: xmin, ymin, xmax, ymax = boxes[n][idx][:] - tmp_det_out.append([ - c, scores[n][c][idx], xmin, ymin, xmax, ymax, - idx + n * num_boxes - ]) - sorted_det_out = sorted(tmp_det_out, - key=lambda tup: tup[0], - reverse=False) + tmp_det_out.append( + [ + c, + scores[n][c][idx], + xmin, + ymin, + xmax, + ymax, + idx + n * num_boxes, + ] + ) + sorted_det_out = sorted( + tmp_det_out, key=lambda tup: tup[0], reverse=False + ) det_outs.extend(sorted_det_out) return det_outs, lod class TestLocalAwareNMSOp(OpTest): - def set_argument(self): self.score_threshold = 0.01 @@ -203,7 +239,7 @@ class TestLocalAwareNMSOp(OpTest): def softmax(x): # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -217,10 +253,15 @@ class TestLocalAwareNMSOp(OpTest): boxes_copy = copy.deepcopy(boxes) scores_copy = copy.deepcopy(scores) - det_outs, lod = batched_multiclass_nms(boxes_copy, scores_copy, - background, score_threshold, - nms_threshold, nms_top_k, - keep_top_k) + det_outs, lod = batched_multiclass_nms( + boxes_copy, + scores_copy, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) lod = [1] if not det_outs else lod det_outs = [[-1, 0]] if not det_outs else det_outs @@ -245,13 +286,11 @@ class TestLocalAwareNMSOp(OpTest): class TestLocalAwareNMSOpNoBoxes(TestLocalAwareNMSOp): - def set_argument(self): self.score_threshold = 2.0 class TestLocalAwareNMSOp4Points(OpTest): - def set_argument(self): self.score_threshold = 0.01 @@ -266,41 +305,93 @@ class TestLocalAwareNMSOp4Points(OpTest): nms_threshold = 0.3 score_threshold = self.score_threshold - scores = np.array([[[0.76319082, 0.73770091]], - [[0.68513154, 0.45952697]]]) - boxes = np.array([[[ - 0.42078365, 0.58117018, 2.92776169, 3.28557757, 4.24344318, - 0.92196165, 2.72370856, -1.66141214 - ], - [ - 0.13856006, 1.86871034, 2.81287224, 3.61381734, - 4.5505249, 0.51766346, 2.75630304, -1.91459389 - ]], - [[ - 1.57533883, 1.3217477, 3.07904942, 3.89512545, - 4.78680923, 1.96914586, 3.539482, -1.59739244 - ], - [ - 0.55084125, 1.71596215, 2.52476074, 3.18940435, - 5.09035159, 0.91959482, 3.71442385, -0.57299128 - ]]]) - - det_outs = np.array([[ - 0., 1.5008917, 0.28206837, 1.2140071, 2.8712926, 3.4469104, - 4.3943763, 0.7232457, 2.7397292, -1.7858533 - ], - [ - 0., 1.1446586, 1.1640508, 1.4800063, 2.856528, - 3.6118112, 4.908667, 1.5478, 3.609713, - -1.1861432 - ]]) + scores = np.array( + [[[0.76319082, 0.73770091]], [[0.68513154, 0.45952697]]] + ) + boxes = np.array( + [ + [ + [ + 0.42078365, + 0.58117018, + 2.92776169, + 3.28557757, + 4.24344318, + 0.92196165, + 2.72370856, + -1.66141214, + ], + [ + 0.13856006, + 1.86871034, + 2.81287224, + 3.61381734, + 4.5505249, + 0.51766346, + 2.75630304, + -1.91459389, + ], + ], + [ + [ + 1.57533883, + 1.3217477, + 3.07904942, + 3.89512545, + 4.78680923, + 1.96914586, + 3.539482, + -1.59739244, + ], + [ + 0.55084125, + 1.71596215, + 2.52476074, + 3.18940435, + 5.09035159, + 0.91959482, + 3.71442385, + -0.57299128, + ], + ], + ] + ) + + det_outs = np.array( + [ + [ + 0.0, + 1.5008917, + 0.28206837, + 1.2140071, + 2.8712926, + 3.4469104, + 4.3943763, + 0.7232457, + 2.7397292, + -1.7858533, + ], + [ + 0.0, + 1.1446586, + 1.1640508, + 1.4800063, + 2.856528, + 3.6118112, + 4.908667, + 1.5478, + 3.609713, + -1.1861432, + ], + ] + ) lod = [1, 1] nmsed_outs = det_outs.astype('float32') self.op_type = 'locality_aware_nms' self.inputs = { 'BBoxes': boxes.astype('float32'), - 'Scores': scores.astype('float32') + 'Scores': scores.astype('float32'), } self.outputs = {'Out': (nmsed_outs, [lod])} self.attrs = { @@ -309,7 +400,7 @@ class TestLocalAwareNMSOp4Points(OpTest): 'nms_top_k': nms_top_k, 'keep_top_k': keep_top_k, 'background_label': -1, - 'normalized': False + 'normalized': False, } def test_check_output(self): @@ -317,75 +408,151 @@ class TestLocalAwareNMSOp4Points(OpTest): class TestLocalityAwareNMSAPI(unittest.TestCase): - def test_api(self): boxes = fluid.data(name='bboxes', shape=[None, 81, 8], dtype='float32') scores = fluid.data(name='scores', shape=[None, 1, 81], dtype='float32') - fluid.layers.locality_aware_nms(bboxes=boxes, - scores=scores, - score_threshold=0.5, - nms_top_k=400, - nms_threshold=0.3, - keep_top_k=200, - normalized=False) + fluid.layers.locality_aware_nms( + bboxes=boxes, + scores=scores, + score_threshold=0.5, + nms_top_k=400, + nms_threshold=0.3, + keep_top_k=200, + normalized=False, + ) class TestLocalityAwareNMSError(unittest.TestCase): - def test_error(self): boxes = fluid.data(name='bboxes', shape=[None, 81, 8], dtype='float32') scores = fluid.data(name='scores', shape=[None, 1, 81], dtype='float32') - boxes_int = fluid.data(name='bboxes_int', - shape=[None, 81, 8], - dtype='int32') - scores_int = fluid.data(name='scores_int', - shape=[None, 1, 81], - dtype='int32') + boxes_int = fluid.data( + name='bboxes_int', shape=[None, 81, 8], dtype='int32' + ) + scores_int = fluid.data( + name='scores_int', shape=[None, 1, 81], dtype='int32' + ) boxes_tmp = [1, 2] scores_tmp = [1, 2] # type of boxes and scores must be variable - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes_tmp, - scores, 0.5, 400, 200) - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores_tmp, 0.5, 400, 200) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes_tmp, + scores, + 0.5, + 400, + 200, + ) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores_tmp, + 0.5, + 400, + 200, + ) # dtype of boxes and scores must in ['float32', 'float64'] - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes_int, - scores, 0.5, 400, 200) - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores_int, 0.5, 400, 200) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes_int, + scores, + 0.5, + 400, + 200, + ) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores_int, + 0.5, + 400, + 200, + ) score_threshold = int(1) # type of score_threshold must be float - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, score_threshold, 400, 200) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + score_threshold, + 400, + 200, + ) nms_top_k = 400.5 # type of num_top_k must be int - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, nms_top_k, 200) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + nms_top_k, + 200, + ) keep_top_k = 200.5 # type of keep_top_k must be int - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, 400, keep_top_k) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + 400, + keep_top_k, + ) nms_threshold = int(0) # type of nms_threshold must be int - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, 400, 200, nms_threshold) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + 400, + 200, + nms_threshold, + ) nms_eta = int(1) # type of nms_eta must be float - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, 400, 200, 0.5, nms_eta) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + 400, + 200, + 0.5, + nms_eta, + ) bg_label = 1.5 # type of background_label must be int - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, 400, 200, 0.5, 1.0, bg_label) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + 400, + 200, + 0.5, + 1.0, + bg_label, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lod_append_op.py b/python/paddle/fluid/tests/unittests/test_lod_append_op.py index 23d828b3cd7..721a247d5d6 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_append_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_append_op.py @@ -1,4 +1,4 @@ -#Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,15 +19,13 @@ from paddle.fluid import Program class TestLoDAppendAPI(unittest.TestCase): - def test_api(self, use_cuda=False): main_program = Program() with fluid.program_guard(main_program): x = fluid.layers.data(name='x', shape=[6], dtype='float32') - level = fluid.layers.data(name='level', - shape=[3], - dtype='int32', - lod_level=0) + level = fluid.layers.data( + name='level', shape=[3], dtype='int32', lod_level=0 + ) result = fluid.layers.lod_append(x, level) x_i = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0]).astype("float32") @@ -38,35 +36,32 @@ class TestLoDAppendAPI(unittest.TestCase): return place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - [out] = exe.run(fluid.default_main_program(), - feed={ - 'x': x_i, - 'level': level_i - }, - fetch_list=[result], - return_numpy=False) + [out] = exe.run( + fluid.default_main_program(), + feed={'x': x_i, 'level': level_i}, + fetch_list=[result], + return_numpy=False, + ) self.assertEqual(out.recursive_sequence_lengths(), [[2, 4]]) class TestLodAppendOpError(unittest.TestCase): - def test_error(self): # The input(x) must be Variable. x1 = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float64") level1 = [0, 2, 4] self.assertRaises(TypeError, fluid.layers.lod_append, x1, level1) - #The input(level) must be Variable or list. + # The input(level) must be Variable or list. x2 = fluid.layers.data(name='x2', shape=[4], dtype='float32') self.assertRaises(ValueError, fluid.layers.lod_append, x2, 2) # Input(x) dtype must be float32 or float64 or int32 or int64 for dtype in ["bool", "float16"]: x3 = fluid.layers.data(name='x3_' + dtype, shape=[4], dtype=dtype) - level3 = fluid.layers.data(name='level3' + dtype, - shape=[4], - dtype='int32', - lod_level=2) + level3 = fluid.layers.data( + name='level3' + dtype, shape=[4], dtype='int32', lod_level=2 + ) self.assertRaises(TypeError, fluid.layers.lod_append, x3, level3) diff --git a/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py b/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py index 4129c59edca..4b702d0a0f0 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py @@ -24,7 +24,6 @@ import numpy class TestLoDArrayLength(unittest.TestCase): - def test_array_length(self): tmp = layers.zeros(shape=[10], dtype='int32') i = layers.fill_constant(shape=[1], dtype='int64', value=10) @@ -37,17 +36,15 @@ class TestLoDArrayLength(unittest.TestCase): class TestLoDArrayLengthOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage + # for ci coverage x1 = numpy.random.randn(2, 4).astype('int32') self.assertRaises(TypeError, fluid.layers.array_length, array=x1) class TestArrayLengthApi(unittest.TestCase): - def test_api(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py index 7a6b6857ffb..88ad04a52c2 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py +++ b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py @@ -21,7 +21,6 @@ import unittest class TestLoDRankTable(unittest.TestCase): - def test_lod_rank_table(self): x = data(name='x', shape=[100]) cpu = core.CPUPlace() @@ -32,8 +31,9 @@ class TestLoDRankTable(unittest.TestCase): tensor = core.LoDTensor() tensor.set(numpy.random.random(size=(17, 100)), cpu) - tensor.set_recursive_sequence_lengths([[1, 2], [5, 1, 1], - [3, 1, 5, 1, 3, 3, 1]]) + tensor.set_recursive_sequence_lengths( + [[1, 2], [5, 1, 1], [3, 1, 5, 1, 3, 3, 1]] + ) exe.run(scope=scope, feed={'x': tensor}) var = scope.find_var(rank_table.name) table = var.get_lod_rank_table() @@ -41,7 +41,6 @@ class TestLoDRankTable(unittest.TestCase): class TestLoDRankTableError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = numpy.random.random((2, 4)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py index 8c814e984e2..5e9bd45907a 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py @@ -20,7 +20,6 @@ from paddle.fluid import Program, program_guard class TestLodResetOpByAttr(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -44,7 +43,6 @@ class TestLodResetOpByAttr(OpTest): class TestLodResetOpByInput(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -55,7 +53,7 @@ class TestLodResetOpByInput(OpTest): target_lod = [4, 3, 3] self.inputs = { 'X': (x, lod), - 'Y': np.array([target_offset_lod]).astype('int32') + 'Y': np.array([target_offset_lod]).astype('int32'), } self.outputs = {'Out': (x, [target_lod])} @@ -69,7 +67,6 @@ class TestLodResetOpByInput(OpTest): class TestLodResetOpBoth(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -79,7 +76,7 @@ class TestLodResetOpBoth(OpTest): target_lod_in = [4, 3, 3] self.inputs = { 'X': (x, lod), - 'Y': np.array(target_offset_lod_in).astype('int32') + 'Y': np.array(target_offset_lod_in).astype('int32'), } self.attrs = {'target_lod': target_offset_lod_attr} self.outputs = {'Out': (x, [target_lod_in])} @@ -94,7 +91,6 @@ class TestLodResetOpBoth(OpTest): class TestLodResetOpYIsLoDTensor(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -114,7 +110,6 @@ class TestLodResetOpYIsLoDTensor(OpTest): class TestLodAppendOpByAttr(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -138,7 +133,6 @@ class TestLodAppendOpByAttr(OpTest): class TestLodResetOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input must be Variable. @@ -148,13 +142,12 @@ class TestLodResetOpError(unittest.TestCase): # Input(x) dtype must be float32 or float64 or int32 or int64 for dtype in ["bool", "float16"]: - x2 = fluid.layers.data(name='x2' + dtype, - shape=[4], - dtype=dtype) - y2 = fluid.layers.data(name='y2' + dtype, - shape=[4], - dtype='int32', - lod_level=2) + x2 = fluid.layers.data( + name='x2' + dtype, shape=[4], dtype=dtype + ) + y2 = fluid.layers.data( + name='y2' + dtype, shape=[4], dtype='int32', lod_level=2 + ) self.assertRaises(TypeError, fluid.layers.lod_reset, x2, y2) diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py index 685c48e9687..c6656ff1caa 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py @@ -19,7 +19,6 @@ import numpy as np class TestLoDTensorArray(unittest.TestCase): - def test_get_set(self): scope = core.Scope() arr = scope.var('tmp_lod_tensor_array') @@ -49,7 +48,6 @@ class TestLoDTensorArray(unittest.TestCase): class TestCreateArray(unittest.TestCase): - def setUp(self): self.place = paddle.CPUPlace() self.shapes = [[10, 4], [8, 12], [1]] @@ -60,7 +58,8 @@ class TestCreateArray(unittest.TestCase): np.random.random(shape).astype('float32') for shape in self.shapes ] array = paddle.tensor.create_array( - 'float32', [paddle.to_tensor(x) for x in init_data]) + 'float32', [paddle.to_tensor(x) for x in init_data] + ) for res, gt in zip(array, init_data): np.testing.assert_array_equal(res, gt) @@ -82,8 +81,9 @@ class TestCreateArray(unittest.TestCase): # test error with nest list with self.assertRaises(TypeError): - paddle.tensor.create_array('float32', - [init_data[0], [init_data[1]]]) + paddle.tensor.create_array( + 'float32', [init_data[0], [init_data[1]]] + ) # test error with not variable with self.assertRaises(TypeError): diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py index 9cc9a5540aa..418c95901ed 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py @@ -28,7 +28,6 @@ from paddle.fluid.layers.control_flow import array_to_lod_tensor class TestCPULoDTensorArrayOps(unittest.TestCase): - def place(self): return core.CPUPlace() @@ -40,10 +39,12 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): np.array(x).astype('int32') for x in [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]] ] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=[] * 6, - expect_max_len=6) + self.main( + tensor=tensor, + expect_array=expect, + expect_lod=[] * 6, + expect_max_len=6, + ) def test_lod_tensor_to_array_level_0_empty_seq(self): tensor = core.LoDTensor() @@ -53,10 +54,12 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): np.array(x).astype('int32') for x in [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]] ] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=[] * 6, - expect_max_len=6) + self.main( + tensor=tensor, + expect_array=expect, + expect_lod=[] * 6, + expect_max_len=6, + ) def test_lod_tensor_to_array_level_1(self): tensor = core.LoDTensor() @@ -66,65 +69,82 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): expect = [ np.array([9, 10, 0, 1, 2], dtype='int32'), np.array([11, 12, 13, 14, 15, 16, 3, 4, 5, 6, 7, 8], dtype='int32'), - np.array([17, 18, 19], dtype='int32') + np.array([17, 18, 19], dtype='int32'), ] lod = [[[2, 3]], [[6, 6]], [[3]]] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=lod, - expect_max_len=3) + self.main( + tensor=tensor, expect_array=expect, expect_lod=lod, expect_max_len=3 + ) def test_lod_tensor_to_array_level_1_empty_seq(self): tensor = core.LoDTensor() tensor.set(np.arange(31).reshape(31, 1).astype('int32'), self.place()) tensor.set_recursive_sequence_lengths( - [[3, 2, 4, 2], [3, 4, 4, 0, 1, 5, 2, 2, 2, 7, 1]]) + [[3, 2, 4, 2], [3, 4, 4, 0, 1, 5, 2, 2, 2, 7, 1]] + ) expect = [ - np.array(item, dtype='int32') for item in [[ - 12, 13, 14, 15, 16, 0, 1, 2, 23, 24, 25, 26, 27, 28, 29 - ], [17, 18, 3, 4, 5, 6, 11, 30], [19, 20, 7, 8, 9, 10], [21, 22]] + np.array(item, dtype='int32') + for item in [ + [12, 13, 14, 15, 16, 0, 1, 2, 23, 24, 25, 26, 27, 28, 29], + [17, 18, 3, 4, 5, 6, 11, 30], + [19, 20, 7, 8, 9, 10], + [21, 22], + ] ] lod = [[[5, 3, 0, 7]], [[2, 4, 1, 1]], [[2, 4]], [[2]]] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=lod, - expect_max_len=4) + self.main( + tensor=tensor, expect_array=expect, expect_lod=lod, expect_max_len=4 + ) def test_lod_tensor_to_array_level_2(self): tensor = core.LoDTensor() tensor.set(np.arange(50).reshape(50, 1).astype('int32'), self.place()) tensor.set_recursive_sequence_lengths( - [[2, 3, 1], [2, 3, 1, 4, 2, 1], - [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4]]) + [ + [2, 3, 1], + [2, 3, 1, 4, 2, 1], + [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4], + ] + ) expect = [ np.array(item, dtype='int32') - for item in [[21, 0, 1, 2, 3, 4, 5, 6, 46, 47, 48, 49], - list(range(22, 39)) + list(range(7, 21)), - list(range(39, 46))] + for item in [ + [21, 0, 1, 2, 3, 4, 5, 6, 46, 47, 48, 49], + list(range(22, 39)) + list(range(7, 21)), + list(range(39, 46)), + ] ] - lod = [[[1, 2, 1], [1, 3, 4, 4]], [[4, 3], [1, 4, 4, 8, 4, 6, 4]], - [[2], [6, 1]]] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=lod, - expect_max_len=3) + lod = [ + [[1, 2, 1], [1, 3, 4, 4]], + [[4, 3], [1, 4, 4, 8, 4, 6, 4]], + [[2], [6, 1]], + ] + self.main( + tensor=tensor, expect_array=expect, expect_lod=lod, expect_max_len=3 + ) def test_lod_tensor_to_array_level_2_skip_level(self): tensor = core.LoDTensor() tensor.set(np.arange(50).reshape(50, 1).astype('int32'), self.place()) tensor.set_recursive_sequence_lengths( - [[2, 3, 1], [2, 3, 1, 4, 2, 1], - [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4]]) - self.main(tensor=tensor, - expect_array=None, - expect_lod=None, - expect_max_len=4, - level=1) + [ + [2, 3, 1], + [2, 3, 1, 4, 2, 1], + [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4], + ] + ) + self.main( + tensor=tensor, + expect_array=None, + expect_lod=None, + expect_max_len=4, + level=1, + ) def main(self, tensor, expect_array, expect_lod, expect_max_len, level=0): place = self.place() @@ -151,37 +171,38 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): self.assertEqual( np.array(scope.find_var(max_len.name).get_tensor())[0], - expect_max_len) + expect_max_len, + ) def check_array_same(self, array, expect_tensor, expect_lod): self.assertEqual(len(expect_tensor), len(array)) for i, exp in enumerate(zip(expect_tensor, expect_lod)): exp_tensor, exp_lod = exp exp_tensor = np.expand_dims(exp_tensor, axis=1) - np.testing.assert_allclose(exp_tensor, - np.array(array[i]), - rtol=1e-05) + np.testing.assert_allclose( + exp_tensor, np.array(array[i]), rtol=1e-05 + ) self.assertEqual(exp_lod, array[i].recursive_sequence_lengths()) def check_tensor_same(self, actual, expect): - np.testing.assert_allclose(np.array(actual), - np.array(expect), - rtol=1e-05) - self.assertEqual(actual.recursive_sequence_lengths(), - expect.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual), np.array(expect), rtol=1e-05 + ) + self.assertEqual( + actual.recursive_sequence_lengths(), + expect.recursive_sequence_lengths(), + ) class TestCPULoDTensorArrayOpGrad(unittest.TestCase): - def test_grad(self): place = core.CPUPlace() program = Program() with program_guard(program): - x = layers.data(name='x', - shape=[1], - dtype='float32', - stop_gradient=False) + x = layers.data( + name='x', shape=[1], dtype='float32', stop_gradient=False + ) table = lod_rank_table(x, level=0) array = lod_tensor_to_array(x, table) result = array_to_lod_tensor(array, table) @@ -198,10 +219,13 @@ class TestCPULoDTensorArrayOpGrad(unittest.TestCase): exe = Executor(place) g_out = [ - np.array(item).sum() for item in exe.run(program, - feed={'x': tensor}, - fetch_list=[g_vars], - return_numpy=False) + np.array(item).sum() + for item in exe.run( + program, + feed={'x': tensor}, + fetch_list=[g_vars], + return_numpy=False, + ) ] g_out_sum = np.array(g_out).sum() @@ -209,7 +233,6 @@ class TestCPULoDTensorArrayOpGrad(unittest.TestCase): class TestLoDTensorArrayError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = np.random.random((10)).astype("float32") @@ -242,7 +265,6 @@ class TestLoDTensorArrayError(unittest.TestCase): class TestArrayLoDTensorError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = np.random.random((10)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_log_loss_op.py b/python/paddle/fluid/tests/unittests/test_log_loss_op.py index 28681da7882..35d31128732 100644 --- a/python/paddle/fluid/tests/unittests/test_log_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_log_loss_op.py @@ -23,7 +23,6 @@ def sigmoid_array(x): class TestLogLossOp(OpTest): - def setUp(self): self.op_type = 'log_loss' samples_num = 100 @@ -38,8 +37,9 @@ class TestLogLossOp(OpTest): } self.attrs = {'epsilon': epsilon} - loss = -labels * np.log(predicted + epsilon) - ( - 1 - labels) * np.log(1 - predicted + epsilon) + loss = -labels * np.log(predicted + epsilon) - (1 - labels) * np.log( + 1 - predicted + epsilon + ) self.outputs = {'Loss': loss} def test_check_output(self): @@ -50,7 +50,6 @@ class TestLogLossOp(OpTest): class TestLogLossOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/test_log_softmax.py b/python/paddle/fluid/tests/unittests/test_log_softmax.py index 8f1fb2c1da2..4be4319d805 100644 --- a/python/paddle/fluid/tests/unittests/test_log_softmax.py +++ b/python/paddle/fluid/tests/unittests/test_log_softmax.py @@ -23,7 +23,7 @@ np.random.seed(10) def ref_log_softmax(x): - shiftx = (x - np.max(x)) + shiftx = x - np.max(x) out = shiftx - np.log(np.exp(shiftx).sum()) return out @@ -33,14 +33,14 @@ def ref_log_softmax_grad(x, axis): axis += len(x.shape) out = np.apply_along_axis(ref_log_softmax, axis, x) axis_dim = x.shape[axis] - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) dx = dout - np.exp(out) * dout.copy().sum(axis=axis, keepdims=True).repeat( - axis_dim, axis=axis) + axis_dim, axis=axis + ) return dx class TestLogSoftmaxOp(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.python_api = F.log_softmax @@ -49,7 +49,7 @@ class TestLogSoftmaxOp(OpTest): self.axis = -1 self.set_attrs() - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -64,27 +64,25 @@ class TestLogSoftmaxOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], ['Out'], - user_defined_grads=[self.x_grad], - check_eager=True) + self.check_grad( + ['X'], ['Out'], user_defined_grads=[self.x_grad], check_eager=True + ) class TestLogSoftmaxShape(TestLogSoftmaxOp): - def set_attrs(self): self.shape = [12, 10] class TestLogSoftmaxAxis(TestLogSoftmaxOp): - def set_attrs(self): self.axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLogSoftmaxBF16Op(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.python_api = F.log_softmax @@ -92,7 +90,7 @@ class TestLogSoftmaxBF16Op(OpTest): self.shape = [2, 3, 4, 5] self.axis = -1 - x = np.random.uniform(0.1, 1., self.shape).astype(np.float32) + x = np.random.uniform(0.1, 1.0, self.shape).astype(np.float32) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -106,19 +104,24 @@ class TestLogSoftmaxBF16Op(OpTest): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], ['Out'], - user_defined_grads=[self.x_grad], - check_eager=True) + self.check_grad_with_place( + place, + ['X'], + ['Out'], + user_defined_grads=[self.x_grad], + check_eager=True, + ) class TestNNLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] - self.x = np.random.uniform(-1., 1., self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.x = np.random.uniform(-1.0, 1.0, self.x_shape).astype(np.float32) + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def check_api(self, axis=-1): ref_out = np.apply_along_axis(ref_log_softmax, axis, self.x) @@ -145,13 +148,14 @@ class TestNNLogSoftmaxAPI(unittest.TestCase): class TestNNFunctionalLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def check_api(self, axis=-1, dtype=None): x = self.x.copy() diff --git a/python/paddle/fluid/tests/unittests/test_logcumsumexp_op.py b/python/paddle/fluid/tests/unittests/test_logcumsumexp_op.py index 6064303a521..96f0e4bff6a 100644 --- a/python/paddle/fluid/tests/unittests/test_logcumsumexp_op.py +++ b/python/paddle/fluid/tests/unittests/test_logcumsumexp_op.py @@ -26,11 +26,13 @@ def np_naive_logcumsumexp(x: np.ndarray, axis: Optional[int] = None): return np.log(np.cumsum(np.exp(x), axis=axis)) -def np_logcumsumexp(x: np.ndarray, - axis: Optional[int] = None, - flatten: Optional[bool] = None, - reverse: bool = False, - exclusive: bool = False): +def np_logcumsumexp( + x: np.ndarray, + axis: Optional[int] = None, + flatten: Optional[bool] = None, + reverse: bool = False, + exclusive: bool = False, +): # `flatten` aligns with c++ op if flatten: assert axis in [0, None] @@ -76,23 +78,30 @@ def np_logcumsumexp_grad( log_grad_negative = np.where(dout < 0, np.log(-dout), np.finfo(x.dtype).min) output_pos = np.exp( - np_logcumsumexp(log_grad_positive - out, - axis=axis, - flatten=flatten, - reverse=not reverse, - exclusive=exclusive).reshape(x.shape) + x) + np_logcumsumexp( + log_grad_positive - out, + axis=axis, + flatten=flatten, + reverse=not reverse, + exclusive=exclusive, + ).reshape(x.shape) + + x + ) output_neg = np.exp( - np_logcumsumexp(log_grad_negative - out, - axis=axis, - flatten=flatten, - reverse=not reverse, - exclusive=exclusive).reshape(x.shape) + x) + np_logcumsumexp( + log_grad_negative - out, + axis=axis, + flatten=flatten, + reverse=not reverse, + exclusive=exclusive, + ).reshape(x.shape) + + x + ) return output_pos - output_neg class TestLogcumsumexp(unittest.TestCase): - def run_imperative(self): data_np = np.arange(12, dtype=np.float32).reshape(3, 4) data = paddle.to_tensor(data_np) @@ -146,14 +155,16 @@ class TestLogcumsumexp(unittest.TestCase): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - out = exe.run(feed={'X': data_np}, - fetch_list=[ - y.name, - y2.name, - y3.name, - y4.name, - y5.name, - ]) + out = exe.run( + feed={'X': data_np}, + fetch_list=[ + y.name, + y2.name, + y3.name, + y4.name, + y5.name, + ], + ) z = np_logcumsumexp(data_np) np.testing.assert_allclose(z, out[0], rtol=1e-05) @@ -202,9 +213,7 @@ class TestLogcumsumexp(unittest.TestCase): class BaseTestCases: - class BaseOpTest(OpTest): - def setUp(self): self.op_type = "logcumsumexp" input, attrs = self.input_and_attrs() @@ -218,56 +227,55 @@ class BaseTestCases: self.check_output() def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ - np_logcumsumexp_grad(self.inputs['X'], - 1 / self.inputs['X'].size, - **self.attrs) - ]) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ + np_logcumsumexp_grad( + self.inputs['X'], + 1 / self.inputs['X'].size, + **self.attrs + ) + ], + ) def input_and_attrs(self): raise NotImplementedError() class TestLogcumsumexpOp1(BaseTestCases.BaseOpTest): - def input_and_attrs(self): return np.arange(100, dtype=np.float64).reshape(10, 10), { 'axis': 0, 'flatten': True, - 'reverse': True + 'reverse': True, } class TestLogcumsumexpOp2(BaseTestCases.BaseOpTest): - def input_and_attrs(self): return np.arange(100, dtype=np.float64).reshape(10, 10), { 'axis': 1, - 'reverse': True + 'reverse': True, } class TestLogcumsumexpOp3(BaseTestCases.BaseOpTest): - def input_and_attrs(self): return np.arange(100, dtype=np.float64).reshape(10, 10), {'axis': 1} class TestLogcumsumexpOp4(BaseTestCases.BaseOpTest): - def input_and_attrs(self): return np.arange(100, dtype=np.float64).reshape(10, 10), { 'axis': 0, 'flatten': True, 'reverse': True, - 'exclusive': True + 'exclusive': True, } class TestLogcumsumexpFP16(unittest.TestCase): - def check_main(self, x_np, dtype, axis=None): paddle.disable_static() x = paddle.to_tensor(x_np.astype(dtype)) diff --git a/python/paddle/fluid/tests/unittests/test_logical_op.py b/python/paddle/fluid/tests/unittests/test_logical_op.py index c36445e7b07..31490961c84 100755 --- a/python/paddle/fluid/tests/unittests/test_logical_op.py +++ b/python/paddle/fluid/tests/unittests/test_logical_op.py @@ -20,87 +20,41 @@ from paddle.framework import _non_static_mode from paddle.fluid.framework import _test_eager_guard SUPPORTED_DTYPES = [ - bool, np.int8, np.int16, np.int32, np.int64, np.float32, np.float64 + bool, + np.int8, + np.int16, + np.int32, + np.int64, + np.float32, + np.float64, ] -TEST_META_OP_DATA = [{ - 'op_str': 'logical_and', - 'binary_op': True -}, { - 'op_str': 'logical_or', - 'binary_op': True -}, { - 'op_str': 'logical_xor', - 'binary_op': True -}, { - 'op_str': 'logical_not', - 'binary_op': False -}] +TEST_META_OP_DATA = [ + {'op_str': 'logical_and', 'binary_op': True}, + {'op_str': 'logical_or', 'binary_op': True}, + {'op_str': 'logical_xor', 'binary_op': True}, + {'op_str': 'logical_not', 'binary_op': False}, +] TEST_META_SHAPE_DATA = { - 'XDimLargerThanYDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 5] - }, - 'XDimLargerThanYDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 1] - }, - 'XDimLargerThanYDim3': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [1, 4, 1] - }, - 'XDimLargerThanYDim4': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4, 1] - }, - 'XDimLargerThanYDim5': { - 'x_shape': [2, 3, 1, 5], - 'y_shape': [3, 1, 1] - }, - 'XDimLessThanYDim1': { - 'x_shape': [4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim2': { - 'x_shape': [1, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim3': { - 'x_shape': [3, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim4': { - 'x_shape': [3, 1, 1], - 'y_shape': [2, 3, 1, 5] - }, - 'XDimLessThanYDim5': { - 'x_shape': [4, 5], - 'y_shape': [2, 3, 4, 5] - }, - 'Axis1InLargerDim': { - 'x_shape': [1, 4, 5], - 'y_shape': [2, 3, 1, 5] - }, - 'EqualDim1': { - 'x_shape': [10, 7], - 'y_shape': [10, 7] - }, - 'EqualDim2': { - 'x_shape': [1, 1, 4, 5], - 'y_shape': [2, 3, 1, 5] - } + 'XDimLargerThanYDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 5]}, + 'XDimLargerThanYDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 1]}, + 'XDimLargerThanYDim3': {'x_shape': [2, 3, 4, 5], 'y_shape': [1, 4, 1]}, + 'XDimLargerThanYDim4': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4, 1]}, + 'XDimLargerThanYDim5': {'x_shape': [2, 3, 1, 5], 'y_shape': [3, 1, 1]}, + 'XDimLessThanYDim1': {'x_shape': [4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim2': {'x_shape': [1, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim3': {'x_shape': [3, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim4': {'x_shape': [3, 1, 1], 'y_shape': [2, 3, 1, 5]}, + 'XDimLessThanYDim5': {'x_shape': [4, 5], 'y_shape': [2, 3, 4, 5]}, + 'Axis1InLargerDim': {'x_shape': [1, 4, 5], 'y_shape': [2, 3, 1, 5]}, + 'EqualDim1': {'x_shape': [10, 7], 'y_shape': [10, 7]}, + 'EqualDim2': {'x_shape': [1, 1, 4, 5], 'y_shape': [2, 3, 1, 5]}, } TEST_META_WRONG_SHAPE_DATA = { - 'ErrorDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4] - }, - 'ErrorDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 3] - } + 'ErrorDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4]}, + 'ErrorDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 3]}, } @@ -175,16 +129,20 @@ def test(unit_test, use_gpu=False, test_error=False): META_DATA = dict(TEST_META_WRONG_SHAPE_DATA) for shape_data in META_DATA.values(): for data_type in SUPPORTED_DTYPES: - meta_data['x_np'] = np_data_generator(shape_data['x_shape'], - dtype=data_type) - meta_data['y_np'] = np_data_generator(shape_data['y_shape'], - dtype=data_type) + meta_data['x_np'] = np_data_generator( + shape_data['x_shape'], dtype=data_type + ) + meta_data['y_np'] = np_data_generator( + shape_data['y_shape'], dtype=data_type + ) if meta_data['binary_op'] and test_error: # catch C++ Exception - unit_test.assertRaises(BaseException, run_static, - **meta_data) - unit_test.assertRaises(BaseException, run_dygraph, - **meta_data) + unit_test.assertRaises( + BaseException, run_static, **meta_data + ) + unit_test.assertRaises( + BaseException, run_dygraph, **meta_data + ) continue static_result = run_static(**meta_data) dygraph_result = run_dygraph(**meta_data) @@ -195,12 +153,12 @@ def test(unit_test, use_gpu=False, test_error=False): np_result = np_op(meta_data['x_np']) unit_test.assertTrue((static_result == np_result).all()) unit_test.assertTrue( - (dygraph_result.numpy() == np_result).all()) + (dygraph_result.numpy() == np_result).all() + ) unit_test.assertTrue((eager_result.numpy() == np_result).all()) def test_type_error(unit_test, use_gpu, type_str_map): - def check_type(op_str, x, y, binary_op): op = getattr(paddle, op_str) error_type = ValueError @@ -235,24 +193,24 @@ def test_type_error(unit_test, use_gpu, type_str_map): startup_program = paddle.static.Program() main_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.static.data(name='x', - shape=[10], - dtype=type_str_map['x']) - y = paddle.static.data(name='y', - shape=[10], - dtype=type_str_map['y']) + x = paddle.static.data( + name='x', shape=[10], dtype=type_str_map['x'] + ) + y = paddle.static.data( + name='y', shape=[10], dtype=type_str_map['y'] + ) check_type(meta_data['op_str'], x, y, binary_op) def type_map_factory(): - return [{ - 'x': x_type, - 'y': y_type - } for x_type in SUPPORTED_DTYPES for y_type in SUPPORTED_DTYPES] + return [ + {'x': x_type, 'y': y_type} + for x_type in SUPPORTED_DTYPES + for y_type in SUPPORTED_DTYPES + ] class TestCPU(unittest.TestCase): - def test(self): test(self) @@ -266,7 +224,6 @@ class TestCPU(unittest.TestCase): class TestCUDA(unittest.TestCase): - def test(self): test(self, True) diff --git a/python/paddle/fluid/tests/unittests/test_logit_op.py b/python/paddle/fluid/tests/unittests/test_logit_op.py index 7240fad1a44..74f645fb68d 100644 --- a/python/paddle/fluid/tests/unittests/test_logit_op.py +++ b/python/paddle/fluid/tests/unittests/test_logit_op.py @@ -22,22 +22,23 @@ np.random.seed(10) def logit(x, eps): - x_min = np.minimum(x, 1. - eps) + x_min = np.minimum(x, 1.0 - eps) x_max = np.maximum(x_min, eps) - return np.log(x_max / (1. - x_max)) + return np.log(x_max / (1.0 - x_max)) def logit_grad(x, eps=1e-8): - tmp_x = np.select([x < eps, x > (1. - eps)], [x * 0., x * 0.], default=-1.0) - x_1 = 1. - x + tmp_x = np.select( + [x < eps, x > (1.0 - eps)], [x * 0.0, x * 0.0], default=-1.0 + ) + x_1 = 1.0 - x _x = np.select([tmp_x == -1.0], [np.reciprocal(x * x_1)], default=0.0) - dout = np.full_like(x, fill_value=1. / _x.size) + dout = np.full_like(x, fill_value=1.0 / _x.size) dx = dout * _x return dx class TestLogitOp(OpTest): - def setUp(self): self.op_type = 'logit' self.python_api = paddle.logit @@ -45,7 +46,7 @@ class TestLogitOp(OpTest): self.shape = [120] self.eps = 1e-8 self.set_attrs() - x = np.random.uniform(-1., 1., self.shape).astype(self.dtype) + x = np.random.uniform(-1.0, 1.0, self.shape).astype(self.dtype) out = logit(x, self.eps) self.x_grad = logit_grad(x, self.eps) self.inputs = {'X': x} @@ -59,31 +60,30 @@ class TestLogitOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], ['Out'], - user_defined_grads=[self.x_grad], - check_eager=True) + self.check_grad( + ['X'], ['Out'], user_defined_grads=[self.x_grad], check_eager=True + ) class TestLogitShape(TestLogitOp): - def set_attrs(self): self.shape = [2, 60] class TestLogitEps(TestLogitOp): - def set_attrs(self): self.eps = 1e-8 class TestLogitAPI(unittest.TestCase): - def setUp(self): self.x_shape = [120] - self.x = np.random.uniform(0., 1., self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.x = np.random.uniform(0.0, 1.0, self.x_shape).astype(np.float32) + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def check_api(self, eps=1e-8): ref_out = logit(self.x, eps) diff --git a/python/paddle/fluid/tests/unittests/test_logspace.py b/python/paddle/fluid/tests/unittests/test_logspace.py index 8b5501a26ae..170d056895a 100644 --- a/python/paddle/fluid/tests/unittests/test_logspace.py +++ b/python/paddle/fluid/tests/unittests/test_logspace.py @@ -19,7 +19,6 @@ import paddle class TestLogspaceOpCommonCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -38,7 +37,6 @@ class TestLogspaceOpCommonCase(OpTest): class TestLogspaceOpReverseCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -46,7 +44,7 @@ class TestLogspaceOpReverseCase(OpTest): 'Start': np.array([10]).astype(dtype), 'Stop': np.array([0]).astype(dtype), 'Num': np.array([11]).astype('int32'), - 'Base': np.array([2]).astype(dtype) + 'Base': np.array([2]).astype(dtype), } self.attrs = {'dtype': int(paddle.float32)} @@ -57,7 +55,6 @@ class TestLogspaceOpReverseCase(OpTest): class TestLogspaceOpNumOneCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -65,7 +62,7 @@ class TestLogspaceOpNumOneCase(OpTest): 'Start': np.array([10]).astype(dtype), 'Stop': np.array([0]).astype(dtype), 'Num': np.array([1]).astype('int32'), - 'Base': np.array([2]).astype(dtype) + 'Base': np.array([2]).astype(dtype), } self.attrs = {'dtype': int(paddle.float32)} @@ -76,7 +73,6 @@ class TestLogspaceOpNumOneCase(OpTest): class TestLogspaceOpMinusBaseCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -95,7 +91,6 @@ class TestLogspaceOpMinusBaseCase(OpTest): class TestLogspaceOpZeroBaseCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -114,7 +109,6 @@ class TestLogspaceOpZeroBaseCase(OpTest): class TestLogspaceAPI(unittest.TestCase): - def test_variable_input1(self): paddle.enable_static() prog = paddle.static.Program() @@ -156,12 +150,9 @@ class TestLogspaceAPI(unittest.TestCase): def test_name(self): with paddle.static.program_guard(paddle.static.Program()): - out = paddle.logspace(0, - 10, - 5, - 2, - dtype='float32', - name='logspace_res') + out = paddle.logspace( + 0, 10, 5, 2, dtype='float32', name='logspace_res' + ) assert 'logspace_res' in out.name def test_imperative(self): @@ -179,7 +170,6 @@ class TestLogspaceAPI(unittest.TestCase): class TestLogspaceOpError(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): @@ -209,9 +199,9 @@ class TestLogspaceOpError(unittest.TestCase): self.assertRaises(TypeError, test_num_type) def test_start_dtype(): - start = paddle.static.data(shape=[1], - dtype="float64", - name="start") + start = paddle.static.data( + shape=[1], dtype="float64", name="start" + ) paddle.logspace(start, 10, 1, 2, dtype="float32") self.assertRaises(ValueError, test_start_dtype) @@ -223,17 +213,17 @@ class TestLogspaceOpError(unittest.TestCase): self.assertRaises(ValueError, test_end_dtype) def test_num_dtype(): - num = paddle.static.data(shape=[1], - dtype="float32", - name="step") + num = paddle.static.data( + shape=[1], dtype="float32", name="step" + ) paddle.logspace(0, 10, num, 2, dtype="float32") self.assertRaises(TypeError, test_num_dtype) def test_base_dtype(): - base = paddle.static.data(shape=[1], - dtype="float64", - name="end") + base = paddle.static.data( + shape=[1], dtype="float64", name="end" + ) paddle.logspace(0, 10, 1, base, dtype="float32") self.assertRaises(ValueError, test_base_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_logsumexp.py b/python/paddle/fluid/tests/unittests/test_logsumexp.py index ca732d6b0a7..ac29603713e 100644 --- a/python/paddle/fluid/tests/unittests/test_logsumexp.py +++ b/python/paddle/fluid/tests/unittests/test_logsumexp.py @@ -21,7 +21,7 @@ from op_test import OpTest def ref_logsumexp(x, axis=None, keepdim=False, reduce_all=False): if isinstance(axis, int): - axis = (axis, ) + axis = (axis,) elif isinstance(axis, list): axis = tuple(axis) if reduce_all: @@ -53,7 +53,6 @@ def logsumexp_ref_grad(x): class TestLogsumexp(OpTest): - def setUp(self): self.op_type = 'logsumexp' self.python_api = logsumexp_wrapper @@ -73,7 +72,7 @@ class TestLogsumexp(OpTest): self.attrs = { 'axis': self.axis, 'keepdim': self.keepdim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.user_defined_grads = None self.user_defined_grad_outputs = None @@ -90,10 +89,12 @@ class TestLogsumexp(OpTest): def test_check_grad(self): self.check_grad( - ['X'], ['Out'], + ['X'], + ['Out'], user_defined_grads=self.user_defined_grads, user_defined_grad_outputs=self.user_defined_grad_outputs, - check_eager=True) + check_eager=True, + ) def calc_grad(self): dy = np.ones(1, dtype=self.dtype) @@ -103,19 +104,16 @@ class TestLogsumexp(OpTest): class TestLogsumexp_shape(TestLogsumexp): - def set_attrs(self): self.shape = [4, 5, 6] class TestLogsumexp_axis(TestLogsumexp): - def set_attrs(self): self.axis = [0, -1] class TestLogsumexp_axis_all(TestLogsumexp): - def set_attrs(self): self.axis = [0, 1, 2, 3] @@ -126,13 +124,11 @@ class TestLogsumexp_axis_all(TestLogsumexp): class TestLogsumexp_keepdim(TestLogsumexp): - def set_attrs(self): self.keepdim = True class TestLogsumexp_reduce_all(TestLogsumexp): - def set_attrs(self): self.reduce_all = True @@ -143,7 +139,6 @@ class TestLogsumexp_reduce_all(TestLogsumexp): class TestLogsumexp_FP32(TestLogsumexp): - def set_attrs(self): self.dtype = 'float32' @@ -154,10 +149,10 @@ class TestLogsumexp_FP32(TestLogsumexp): np.testing.assert_allclose(x_grad, ref_x_grad, rtol=1e-08, atol=1e-08) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLogsumexp_FP16(TestLogsumexp): - def set_attrs(self): self.dtype = 'float16' @@ -169,10 +164,9 @@ class TestLogsumexp_FP16(TestLogsumexp): tensor_x = paddle.to_tensor(x) out_pad = logsumexp_wrapper(tensor_x) paddle.enable_static() - np.testing.assert_allclose(out_pad.numpy(), - out_ref, - rtol=1e-03, - atol=1e-08) + np.testing.assert_allclose( + out_pad.numpy(), out_ref, rtol=1e-03, atol=1e-08 + ) def test_check_grad(self): self.__class__.dtype = self.dtype @@ -184,7 +178,6 @@ class TestLogsumexp_FP16(TestLogsumexp): class TestLogsumexpError(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): self.assertRaises(TypeError, paddle.logsumexp, 1) @@ -193,12 +186,14 @@ class TestLogsumexpError(unittest.TestCase): class TestLogsumexpAPI(unittest.TestCase): - def setUp(self): self.shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def api_case(self, axis=None, keepdim=False): out_ref = ref_logsumexp(self.x, axis, keepdim) diff --git a/python/paddle/fluid/tests/unittests/test_lookahead.py b/python/paddle/fluid/tests/unittests/test_lookahead.py index 10c2ef78b86..6f7c052f992 100644 --- a/python/paddle/fluid/tests/unittests/test_lookahead.py +++ b/python/paddle/fluid/tests/unittests/test_lookahead.py @@ -25,7 +25,6 @@ SGD_LR = 1.0 class TestLookAhead(unittest.TestCase): - def test_lookahead_static(self): paddle.enable_static() place = fluid.CPUPlace() @@ -41,7 +40,8 @@ class TestLookAhead(unittest.TestCase): optimizer = paddle.optimizer.SGD(learning_rate=SGD_LR) lookahead = paddle.incubate.optimizer.LookAhead( - optimizer, alpha=LOOKAHEAD_ALPHA, k=LOOKAHEAD_K) + optimizer, alpha=LOOKAHEAD_ALPHA, k=LOOKAHEAD_K + ) lookahead.minimize(loss) exe.run(startup) @@ -49,21 +49,24 @@ class TestLookAhead(unittest.TestCase): fast_param = None for i in range(10): if (i + 1) % LOOKAHEAD_K == 0: - slow_param = slow_param + LOOKAHEAD_ALPHA * (fast_param - - slow_param) + slow_param = slow_param + LOOKAHEAD_ALPHA * ( + fast_param - slow_param + ) x = np.random.random(size=(10, 1)).astype('float32') - latest_b, b_grad = exe.run(program=train_program, - feed={'X': x}, - fetch_list=[ - 'fc_0.b_0', - 'fc_0.b_0@GRAD', - ]) + latest_b, b_grad = exe.run( + program=train_program, + feed={'X': x}, + fetch_list=[ + 'fc_0.b_0', + 'fc_0.b_0@GRAD', + ], + ) if i == 0: slow_param = latest_b if (i + 1) % LOOKAHEAD_K == 0: - self.assertAlmostEqual(slow_param.all(), - latest_b.all(), - delta=5e-3) + self.assertAlmostEqual( + slow_param.all(), latest_b.all(), delta=5e-3 + ) fast_param = latest_b - SGD_LR * b_grad def func_test_look_ahead_dygraph(self): @@ -76,21 +79,20 @@ class TestLookAhead(unittest.TestCase): # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, - (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( + 'int64' + ) return image, label def __len__(self): return self.num_samples class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -110,34 +112,41 @@ class TestLookAhead(unittest.TestCase): out = layer(image) loss = loss_fn(out, label) loss.backward() - fast_param = (layer.bias.numpy() - - SGD_LR * layer.bias.grad.numpy()) + fast_param = ( + layer.bias.numpy() - SGD_LR * layer.bias.grad.numpy() + ) opt.step() if idx == 1: slow_param = fast_param if idx % LOOKAHEAD_K == 0: slow_param = slow_param + LOOKAHEAD_ALPHA * ( - fast_param - slow_param) - self.assertAlmostEqual(np.mean(slow_param), - np.mean(layer.bias.numpy()), - delta=5e-3) + fast_param - slow_param + ) + self.assertAlmostEqual( + np.mean(slow_param), + np.mean(layer.bias.numpy()), + delta=5e-3, + ) opt.clear_grad() layer = LinearNet() loss_fn = nn.CrossEntropyLoss() - optimizer = paddle.optimizer.SGD(learning_rate=SGD_LR, - parameters=layer.parameters()) - lookahead = paddle.incubate.optimizer.LookAhead(optimizer, - alpha=LOOKAHEAD_ALPHA, - k=LOOKAHEAD_K) + optimizer = paddle.optimizer.SGD( + learning_rate=SGD_LR, parameters=layer.parameters() + ) + lookahead = paddle.incubate.optimizer.LookAhead( + optimizer, alpha=LOOKAHEAD_ALPHA, k=LOOKAHEAD_K + ) # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) train(layer, loader, loss_fn, lookahead) diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_bf16_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_bf16_op.py index 8c6b3421e24..4c572622abe 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_bf16_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_bf16_op.py @@ -14,10 +14,12 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import (OpTest, - convert_float_to_uint16, - convert_uint16_to_float, - skip_check_grad_ci) +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + convert_uint16_to_float, + skip_check_grad_ci, +) import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator @@ -26,8 +28,11 @@ from paddle import enable_static def _lookup(weights, ids, flat_ids, op_version="lookup_table"): w_shape = weights.shape - out_shape = list(ids.shape[:-1]) if op_version == "lookup_table" else list( - ids.shape) + out_shape = ( + list(ids.shape[:-1]) + if op_version == "lookup_table" + else list(ids.shape) + ) out_shape.append(w_shape[-1]) out = weights[flat_ids].reshape(out_shape) return out @@ -36,8 +41,11 @@ def _lookup(weights, ids, flat_ids, op_version="lookup_table"): def _get_grad(weights, ids, flat_ids, op_version="lookup_table"): w_shape = weights.shape w_grad = np.zeros((w_shape), dtype=weights.dtype) - out_shape = list(ids.shape[:-1]) if op_version == "lookup_table" else list( - ids.shape) + out_shape = ( + list(ids.shape[:-1]) + if op_version == "lookup_table" + else list(ids.shape) + ) out_grad_shape = (np.prod(out_shape), w_shape[-1]) out_grad = weights[flat_ids].reshape(out_grad_shape) for i, idx in enumerate(flat_ids): @@ -45,10 +53,10 @@ def _get_grad(weights, ids, flat_ids, op_version="lookup_table"): return w_grad -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16Op(OpTest): - def init_test(self): self.op_type = "lookup_table" self.ids_shape = (4, 1) @@ -62,11 +70,13 @@ class TestLookupTableBF16Op(OpTest): self.flat_ids = self.ids.flatten() self.w_bf16 = convert_float_to_uint16(table) - self.out_bf16 = _lookup(self.w_bf16, self.ids, self.flat_ids, - self.op_type) + self.out_bf16 = _lookup( + self.w_bf16, self.ids, self.flat_ids, self.op_type + ) self.out_fp32 = _lookup(table, self.ids, self.flat_ids, self.op_type) - self.w_grad_fp32 = _get_grad(table, self.ids, self.flat_ids, - self.op_type) + self.w_grad_fp32 = _get_grad( + table, self.ids, self.flat_ids, self.op_type + ) self.inputs = {'W': self.w_bf16, 'Ids': self.ids} self.outputs = {'Out': self.out_fp32} @@ -75,36 +85,40 @@ class TestLookupTableBF16Op(OpTest): self.check_output_with_place(core.CPUPlace(), check_dygraph=False) def test_check_grad(self): - self.check_grad_with_place(core.CPUPlace(), ['W'], - 'Out', - no_grad_set=set('Ids'), - check_dygraph=False, - max_relative_error=1.5e-2, - user_defined_grads=[self.w_grad_fp32], - user_defined_grad_outputs=[self.out_bf16]) - - -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") + self.check_grad_with_place( + core.CPUPlace(), + ['W'], + 'Out', + no_grad_set=set('Ids'), + check_dygraph=False, + max_relative_error=1.5e-2, + user_defined_grads=[self.w_grad_fp32], + user_defined_grad_outputs=[self.out_bf16], + ) + + +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpIds4D(TestLookupTableBF16Op): - def init_test(self): self.op_type = "lookup_table" self.ids_shape = (2, 4, 5, 1) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpWIsSelectedRows(unittest.TestCase): - def init_test(self): self.op_type = "lookup_table" self.ids_shape = (10, 1) def setUp(self): self.init_test() - self.ids = np.random.randint(low=0, high=15, - size=self.ids_shape).astype("int64") + self.ids = np.random.randint( + low=0, high=15, size=self.ids_shape + ).astype("int64") self.flat_ids = self.ids.flatten() self.w_fp32 = np.random.random((15, 32)).astype("float32") self.w_bf16 = convert_float_to_uint16(self.w_fp32) @@ -144,11 +158,12 @@ class TestLookupTableBF16OpWIsSelectedRows(unittest.TestCase): self._check_output(ref, result_array) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpWIsSelectedRows4DIds( - TestLookupTableBF16OpWIsSelectedRows): - + TestLookupTableBF16OpWIsSelectedRows +): def init_test(self): self.op_type = "lookup_table" self.ids_shape = (3, 4, 5, 1) @@ -161,11 +176,12 @@ class TestLookupTableBF16OpWIsSelectedRows4DIds( @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") + "test the gradient here." +) +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpWithPadding(TestLookupTableBF16Op): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -177,11 +193,12 @@ class TestLookupTableBF16OpWithPadding(TestLookupTableBF16Op): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") + "test the gradient here." +) +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpIds4DPadding(TestLookupTableBF16OpIds4D): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -202,8 +219,9 @@ class TestEmbeddingLayerBF16ConstantInitializer(unittest.TestCase): def setUp(self): self.ids_shape = [4, 1] self.w_shape = [10, 64] - self.ids = np.random.randint(low=0, high=9, - size=self.ids_shape).astype("int64") + self.ids = np.random.randint(low=0, high=9, size=self.ids_shape).astype( + "int64" + ) self.flat_ids = self.ids.flatten() self.value = 3.0 self.w_fp32 = np.full(self.w_shape, self.value) @@ -214,18 +232,20 @@ class TestEmbeddingLayerBF16ConstantInitializer(unittest.TestCase): with fluid.program_guard(self.prog, self.startup_prog): x = fluid.layers.data(name='x', shape=self.ids_shape, dtype='int64') - self.emb = fluid.layers.embedding(input=x, - size=self.w_shape, - param_attr=fluid.ParamAttr( - name="emb_weight", - initializer=self.initializer), - is_sparse=False, - dtype="uint16") # bfloat16 + self.emb = fluid.layers.embedding( + input=x, + size=self.w_shape, + param_attr=fluid.ParamAttr( + name="emb_weight", initializer=self.initializer + ), + is_sparse=False, + dtype="uint16", + ) # bfloat16 exe = fluid.Executor(self.place) exe.run(self.startup_prog) - self.result = exe.run(self.prog, - feed={'x': self.ids}, - fetch_list=['emb_weight', self.emb]) + self.result = exe.run( + self.prog, feed={'x': self.ids}, fetch_list=['emb_weight', self.emb] + ) def test_embedding_weights(self): result = convert_uint16_to_float(self.result[0]) diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_dequant_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_dequant_op.py index bd15a3ac1cb..42a491344c6 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_dequant_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_dequant_op.py @@ -19,7 +19,6 @@ import struct class TestLookupTableDequantOp(OpTest): - def setUp(self): self.op_type = "lookup_table_dequant" table = np.random.random((17, 32)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py index 2a57ca7e596..02fccaaca22 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py @@ -23,7 +23,6 @@ import paddle.nn.functional as F class TestLookupTableOp(OpTest): - def setUp(self): self.op_type = "lookup_table" table = np.random.random((17, 31)).astype("float64") @@ -40,12 +39,12 @@ class TestLookupTableOp(OpTest): class TestLookupTableOpWithTensorIds(OpTest): - def setUp(self): self.op_type = "lookup_table" table = np.random.random((17, 31)).astype("float64") - ids = np.random.randint(low=0, high=17, - size=(2, 4, 5, 1)).astype("int64") + ids = np.random.randint(low=0, high=17, size=(2, 4, 5, 1)).astype( + "int64" + ) self.inputs = {'W': table, 'Ids': ids} self.outputs = {'Out': table[ids.flatten()].reshape((2, 4, 5, 31))} @@ -59,9 +58,9 @@ class TestLookupTableOpWithTensorIds(OpTest): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithPadding(TestLookupTableOp): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -73,9 +72,9 @@ class TestLookupTableOpWithPadding(TestLookupTableOp): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -86,7 +85,6 @@ class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): class TestLookupTableWIsSelectedRows(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([[0], [4], [3], [5]]).astype("int64") @@ -139,13 +137,14 @@ class TestLookupTableWIsSelectedRows(unittest.TestCase): self.check_with_place(place) -class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows - ): - +class TestLookupTableWithTensorIdsWIsSelectedRows( + TestLookupTableWIsSelectedRows +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3, 1)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3, 1)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -155,7 +154,6 @@ class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows class TestEmbedOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.randint(0, 10, (4, 1)).astype("int64") @@ -176,9 +174,9 @@ class TestEmbedOpError(unittest.TestCase): def test_param_dtype(): # dtype must be float32 or float64 input2 = fluid.data(name='x2', shape=[4, 1], dtype='int64') - fluid.layers.embedding(input=input2, - size=(10, 64), - dtype='int64') + fluid.layers.embedding( + input=input2, size=(10, 64), dtype='int64' + ) self.assertRaises(TypeError, test_param_dtype) @@ -187,11 +185,11 @@ class TestEmbedOpError(unittest.TestCase): class TestLookupTableOpInt8(OpTest): - def setUp(self): self.op_type = "lookup_table" - table = np.random.randint(low=-128, high=127, - size=(17, 31)).astype("int8") + table = np.random.randint(low=-128, high=127, size=(17, 31)).astype( + "int8" + ) ids = np.random.randint(0, 17, 4).astype("int64") ids_expand = np.expand_dims(ids, axis=1) self.inputs = {'W': table, 'Ids': ids_expand} @@ -207,13 +205,14 @@ class TestLookupTableOpInt8(OpTest): class TestLookupTableOpWithTensorIdsInt8(OpTest): - def setUp(self): self.op_type = "lookup_table" - table = np.random.randint(low=-128, high=127, - size=(17, 31)).astype("int8") - ids = np.random.randint(low=0, high=17, - size=(2, 4, 5, 1)).astype("int64") + table = np.random.randint(low=-128, high=127, size=(17, 31)).astype( + "int8" + ) + ids = np.random.randint(low=0, high=17, size=(2, 4, 5, 1)).astype( + "int64" + ) self.inputs = {'W': table, 'Ids': ids} self.outputs = {'Out': table[ids.flatten()].reshape((2, 4, 5, 31))} @@ -227,7 +226,6 @@ class TestLookupTableOpWithTensorIdsInt8(OpTest): class TestLookupTableOpWithPaddingInt8(TestLookupTableOpInt8): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -242,8 +240,8 @@ class TestLookupTableOpWithPaddingInt8(TestLookupTableOpInt8): class TestLookupTableOpWithTensorIdsAndPaddingInt8( - TestLookupTableOpWithTensorIdsInt8): - + TestLookupTableOpWithTensorIdsInt8 +): def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -259,7 +257,6 @@ class TestLookupTableOpWithTensorIdsAndPaddingInt8( class TestLookupTableWIsSelectedRowsInt8(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([[0], [4], [3], [5]]).astype("int64") @@ -313,12 +310,13 @@ class TestLookupTableWIsSelectedRowsInt8(unittest.TestCase): class TestLookupTableWithTensorIdsWIsSelectedRowsInt8( - TestLookupTableWIsSelectedRowsInt8): - + TestLookupTableWIsSelectedRowsInt8 +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3, 1)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3, 1)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -329,11 +327,11 @@ class TestLookupTableWithTensorIdsWIsSelectedRowsInt8( @skip_check_grad_ci(reason="Int16 type only be used in test and inference.") class TestLookupTableOpInt16(OpTest): - def setUp(self): self.op_type = "lookup_table" - table = np.random.randint(low=-128, high=127, - size=(17, 31)).astype("int16") + table = np.random.randint(low=-128, high=127, size=(17, 31)).astype( + "int16" + ) ids = np.random.randint(0, 17, 4).astype("int64") ids_expand = np.expand_dims(ids, axis=1) self.inputs = {'W': table, 'Ids': ids_expand} @@ -345,13 +343,14 @@ class TestLookupTableOpInt16(OpTest): @skip_check_grad_ci(reason="Int16 type only be used in test and inference.") class TestLookupTableOpWithTensorIdsInt16(OpTest): - def setUp(self): self.op_type = "lookup_table" - table = np.random.randint(low=-128, high=127, - size=(17, 31)).astype("int16") - ids = np.random.randint(low=0, high=17, - size=(2, 4, 5, 1)).astype("int64") + table = np.random.randint(low=-128, high=127, size=(17, 31)).astype( + "int16" + ) + ids = np.random.randint(low=0, high=17, size=(2, 4, 5, 1)).astype( + "int64" + ) self.inputs = {'W': table, 'Ids': ids} self.outputs = {'Out': table[ids.flatten()].reshape((2, 4, 5, 31))} @@ -361,7 +360,6 @@ class TestLookupTableOpWithTensorIdsInt16(OpTest): @skip_check_grad_ci(reason="Int16 type only be used in test and inference.") class TestLookupTableOpWithPaddingInt16(TestLookupTableOpInt16): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -372,8 +370,8 @@ class TestLookupTableOpWithPaddingInt16(TestLookupTableOpInt16): @skip_check_grad_ci(reason="Int16 type only be used in test and inference.") class TestLookupTableOpWithTensorIdsAndPaddingInt16( - TestLookupTableOpWithTensorIdsInt16): - + TestLookupTableOpWithTensorIdsInt16 +): def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -384,7 +382,6 @@ class TestLookupTableOpWithTensorIdsAndPaddingInt16( class TestLookupTableWIsSelectedRowsInt16(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([[0], [4], [3], [5]]).astype("int64") @@ -437,12 +434,13 @@ class TestLookupTableWIsSelectedRowsInt16(unittest.TestCase): class TestLookupTableWithTensorIdsWIsSelectedRowsInt16( - TestLookupTableWIsSelectedRowsInt16): - + TestLookupTableWIsSelectedRowsInt16 +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3, 1)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3, 1)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -452,13 +450,14 @@ class TestLookupTableWithTensorIdsWIsSelectedRowsInt16( class TestOutDtype(unittest.TestCase): - def test_dtype(self): api_fn = F.embedding - check_out_dtype(api_fn, - in_specs=[([10, 16], 'int64'), ([100, 64], )], - expect_dtypes=['float32', 'float64'], - target_index=1) + check_out_dtype( + api_fn, + in_specs=[([10, 16], 'int64'), ([100, 64],)], + expect_dtypes=['float32', 'float64'], + target_index=1, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_v2_bf16_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_v2_bf16_op.py index a479693d518..e0dbf743430 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_v2_bf16_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_v2_bf16_op.py @@ -17,23 +17,24 @@ import numpy as np import paddle from paddle.fluid.tests.unittests.op_test import convert_uint16_to_float from paddle.fluid.tests.unittests.test_lookup_table_bf16_op import ( - _lookup, TestLookupTableBF16Op, TestLookupTableBF16OpIds4D, + _lookup, + TestLookupTableBF16Op, + TestLookupTableBF16OpIds4D, TestLookupTableBF16OpWIsSelectedRows, - TestLookupTableBF16OpWIsSelectedRows4DIds) + TestLookupTableBF16OpWIsSelectedRows4DIds, +) import paddle.fluid as fluid import paddle.fluid.core as core class TestLookupTableV2BF16Op(TestLookupTableBF16Op): - def init_test(self): self.op_type = "lookup_table_v2" - self.ids_shape = (4) + self.ids_shape = 4 self.mkldnn_data_type = "bfloat16" class TestLookupTableV2BF16OpIds4D(TestLookupTableBF16OpIds4D): - def init_test(self): self.op_type = "lookup_table_v2" self.ids_shape = (2, 4, 5) @@ -41,23 +42,22 @@ class TestLookupTableV2BF16OpIds4D(TestLookupTableBF16OpIds4D): class TestLookupTableV2BF16OpWIsSelectedRows( - TestLookupTableBF16OpWIsSelectedRows): - + TestLookupTableBF16OpWIsSelectedRows +): def init_test(self): self.op_type = "lookup_table_v2" - self.ids_shape = (10) + self.ids_shape = 10 class TestLookupTableV2BF16OpWIsSelectedRows4DIds( - TestLookupTableBF16OpWIsSelectedRows4DIds): - + TestLookupTableBF16OpWIsSelectedRows4DIds +): def init_test(self): self.op_type = "lookup_table_v2" self.ids_shape = (3, 4, 5) class TestLookupTableBF16OpWithPadding(TestLookupTableV2BF16Op): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -67,7 +67,6 @@ class TestLookupTableBF16OpWithPadding(TestLookupTableV2BF16Op): class TestLookupTableBF16OpIds4DPadding(TestLookupTableV2BF16OpIds4D): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -89,8 +88,9 @@ class TestEmbeddingLayerBF16ConstantInitializer(unittest.TestCase): self.op_type = "lookup_table_v2" self.ids_shape = [4] self.w_shape = [10, 64] - self.ids = np.random.randint(low=0, high=9, - size=self.ids_shape).astype("int64") + self.ids = np.random.randint(low=0, high=9, size=self.ids_shape).astype( + "int64" + ) self.flat_ids = self.ids.flatten() self.value = 3.0 self.w_fp32 = np.full(self.w_shape, self.value) @@ -101,18 +101,20 @@ class TestEmbeddingLayerBF16ConstantInitializer(unittest.TestCase): with fluid.program_guard(self.prog, self.startup_prog): x = fluid.layers.data(name='x', shape=self.ids_shape, dtype='int64') - self.emb = fluid.input.embedding(input=x, - size=self.w_shape, - param_attr=fluid.ParamAttr( - name="emb_weight", - initializer=self.initializer), - is_sparse=False, - dtype="uint16") # bfloat16 + self.emb = fluid.input.embedding( + input=x, + size=self.w_shape, + param_attr=fluid.ParamAttr( + name="emb_weight", initializer=self.initializer + ), + is_sparse=False, + dtype="uint16", + ) # bfloat16 exe = fluid.Executor(self.place) exe.run(self.startup_prog) - self.result = exe.run(self.prog, - feed={'x': self.ids}, - fetch_list=['emb_weight', self.emb]) + self.result = exe.run( + self.prog, feed={'x': self.ids}, fetch_list=['emb_weight', self.emb] + ) def test_embedding_weights(self): result = convert_uint16_to_float(self.result[0]) diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_v2_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_v2_op.py index 4919c9e8c00..0f73c052bb5 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_v2_op.py @@ -24,7 +24,6 @@ from paddle.fluid import Program, program_guard class TestStaticGraphSupportMultipleInt(unittest.TestCase): - def test_main(self): dtypes = ['uint8', 'int8', 'int16', 'int32', 'int64'] if paddle.in_dynamic_mode(): @@ -33,8 +32,9 @@ class TestStaticGraphSupportMultipleInt(unittest.TestCase): else: disable_static = False for i, dtype in enumerate(dtypes): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.static.data(name='x', shape=[-1, 7, 30], dtype=dtype) emb = paddle.nn.Embedding(10, 20) y = emb(x) @@ -44,7 +44,6 @@ class TestStaticGraphSupportMultipleInt(unittest.TestCase): class TestLookupTableOp(OpTest): - def setUp(self): self.op_type = "lookup_table_v2" self.python_api = paddle.nn.functional.embedding @@ -64,25 +63,21 @@ class TestLookupTableOp(OpTest): class TestLookupTableOpInt16(OpTest): - def id_dtype(self): return "int16" class TestLookupTableOpInt8(OpTest): - def id_dtype(self): return "int8" class TestLookupTableOpUInt8(OpTest): - def id_dtype(self): return "uint8" class TestLookupTableOpWithTensorIds(OpTest): - def setUp(self): self.op_type = "lookup_table_v2" table = np.random.random((17, 31)).astype("float64") @@ -100,9 +95,9 @@ class TestLookupTableOpWithTensorIds(OpTest): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithPadding(TestLookupTableOp): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -114,9 +109,9 @@ class TestLookupTableOpWithPadding(TestLookupTableOp): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -127,7 +122,6 @@ class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): class TestLookupTableWIsSelectedRows(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([0, 4, 3, 5]).astype("int32") @@ -180,13 +174,14 @@ class TestLookupTableWIsSelectedRows(unittest.TestCase): self.check_with_place(place) -class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows - ): - +class TestLookupTableWithTensorIdsWIsSelectedRows( + TestLookupTableWIsSelectedRows +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -196,7 +191,6 @@ class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows class TestLookupTableIsSparse(unittest.TestCase): - def init_data(self): self.x_data = np.array([[1, 3, 0, 4, 7]]).astype("int64") self.y_data = np.array([[0.1, 0.3, 0, 0.4, 0.7]]).astype("float32") @@ -214,8 +208,11 @@ class TestLookupTableIsSparse(unittest.TestCase): name="emb_weight", learning_rate=10, initializer=fluid.initializer.NumpyArrayInitializer( - self.w_data)), - is_sparse=is_sparse) + self.w_data + ), + ), + is_sparse=is_sparse, + ) y = fluid.layers.reduce_sum(emb, dim=-1) loss = fluid.layers.square_error_cost(input=y, label=y_) @@ -227,12 +224,11 @@ class TestLookupTableIsSparse(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': self.x_data, - 'y_': self.y_data - }, - fetch_list=['emb_weight'], - return_numpy=False) + ret = exe.run( + feed={'x': self.x_data, 'y_': self.y_data}, + fetch_list=['emb_weight'], + return_numpy=False, + ) return np.array(ret[0]) def test_w_grad(self): @@ -242,14 +238,12 @@ class TestLookupTableIsSparse(unittest.TestCase): self.check_grad(w_grad, w_grad_with_sparse) def check_grad(self, w_grad1, w_grad2, tolerance=1e-6): - np.testing.assert_allclose(w_grad1, - w_grad2, - rtol=tolerance, - atol=tolerance) + np.testing.assert_allclose( + w_grad1, w_grad2, rtol=tolerance, atol=tolerance + ) class TestLookupTableApi(unittest.TestCase): - def test_api(self): x = fluid.layers.data(name='x', shape=[20], dtype='int64') emb = fluid.embedding(input=x, size=[128, 64]) @@ -259,15 +253,16 @@ class TestLookupTableApi(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_data, - }, - fetch_list=[emb], - return_numpy=False) + ret = exe.run( + feed={ + 'x': x_data, + }, + fetch_list=[emb], + return_numpy=False, + ) class TestEmbedOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.randint(0, 10, (4, 6)).astype("int64") diff --git a/python/paddle/fluid/tests/unittests/test_lr_scheduler.py b/python/paddle/fluid/tests/unittests/test_lr_scheduler.py index 2d76f141f9f..55d26bc5093 100644 --- a/python/paddle/fluid/tests/unittests/test_lr_scheduler.py +++ b/python/paddle/fluid/tests/unittests/test_lr_scheduler.py @@ -21,9 +21,9 @@ import paddle.fluid as fluid import paddle.fluid.core as core -def reduce_lr_on_plateau(decay_rate, threshold, cooldown, patience, m, n, loss, - var_list): - +def reduce_lr_on_plateau( + decay_rate, threshold, cooldown, patience, m, n, loss, var_list +): def is_better(current, best, m, n): if m == 'min' and n == 'rel': return current < best - best * threshold @@ -53,7 +53,6 @@ def reduce_lr_on_plateau(decay_rate, threshold, cooldown, patience, m, n, loss, class TestReduceOnPlateauDecay(object): - def test_ReduceLR(self): # the decay rate must be less than 1.0 with self.assertRaises(ValueError): @@ -63,8 +62,9 @@ class TestReduceOnPlateauDecay(object): paddle.optimizer.lr.ReduceOnPlateau(learning_rate=1.0, mode="test") # the threshold_mode must be "rel" or "abs" with self.assertRaises(ValueError): - paddle.optimizer.lr.ReduceOnPlateau(learning_rate=1.0, - threshold_mode="test") + paddle.optimizer.lr.ReduceOnPlateau( + learning_rate=1.0, threshold_mode="test" + ) with self.assertRaises(TypeError): paddle.optimizer.lr.ReduceOnPlateau(learning_rate="test") with self.assertRaises(TypeError): @@ -75,8 +75,9 @@ class TestReduceOnPlateauDecay(object): places.append(paddle.CUDAPlace(0)) for place in places: - for m, n in zip(['min', 'max', 'min', 'max'], - ['rel', 'rel', 'abs', 'abs']): + for m, n in zip( + ['min', 'max', 'min', 'max'], ['rel', 'rel', 'abs', 'abs'] + ): kwargs = { 'learning_rate': 1.0, 'mode': m, @@ -107,10 +108,9 @@ class TestReduceOnPlateauDecay(object): main_prog = paddle.static.Program() start_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): - x = fluid.layers.create_global_var([1], - 1, - 'float32', - persistable=True) + x = fluid.layers.create_global_var( + [1], 1, 'float32', persistable=True + ) paddle.increment(x) loss = paddle.sin(x) scheduler = paddle.optimizer.lr.ReduceOnPlateau(**kwargs) @@ -124,12 +124,19 @@ class TestReduceOnPlateauDecay(object): for epoch in range(20): for batch_id in range(1): - out, actual_lr = exe.run(main_prog, - fetch_list=[loss.name, lr_var.name]) + out, actual_lr = exe.run( + main_prog, fetch_list=[loss.name, lr_var.name] + ) expected_lr = reduce_lr_on_plateau( - kwargs['factor'], kwargs['threshold'], kwargs['cooldown'], - kwargs['patience'], kwargs['mode'], - kwargs['threshold_mode'], out[0], var_list) + kwargs['factor'], + kwargs['threshold'], + kwargs['cooldown'], + kwargs['patience'], + kwargs['mode'], + kwargs['threshold_mode'], + out[0], + var_list, + ) scheduler.step(out[0]) actual_lr = scheduler() @@ -137,12 +144,19 @@ class TestReduceOnPlateauDecay(object): for epoch in range(10): for batch_id in range(1): - out, actual_lr = exe.run(test_prog, - fetch_list=[loss.name, lr_var.name]) + out, actual_lr = exe.run( + test_prog, fetch_list=[loss.name, lr_var.name] + ) expected_lr = reduce_lr_on_plateau( - kwargs['factor'], kwargs['threshold'], kwargs['cooldown'], - kwargs['patience'], kwargs['mode'], - kwargs['threshold_mode'], out[0], var_list) + kwargs['factor'], + kwargs['threshold'], + kwargs['cooldown'], + kwargs['patience'], + kwargs['mode'], + kwargs['threshold_mode'], + out[0], + var_list, + ) scheduler.step(out[0]) actual_lr = scheduler() self.assertEqual(actual_lr, np.array(expected_lr)) @@ -158,8 +172,9 @@ class TestReduceOnPlateauDecay(object): linear = paddle.nn.Linear(10, 10) scheduler = paddle.optimizer.lr.ReduceOnPlateau(**kwargs) - adam = paddle.optimizer.Adam(learning_rate=scheduler, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=scheduler, parameters=linear.parameters() + ) for epoch in range(20): for batch_id in range(1): @@ -174,17 +189,25 @@ class TestReduceOnPlateauDecay(object): current_lr = adam.get_lr() # get lr form python expected_lr = reduce_lr_on_plateau( - kwargs['factor'], kwargs['threshold'], kwargs['cooldown'], - kwargs['patience'], kwargs['mode'], kwargs['threshold_mode'], - loss, var_list) + kwargs['factor'], + kwargs['threshold'], + kwargs['cooldown'], + kwargs['patience'], + kwargs['mode'], + kwargs['threshold_mode'], + loss, + var_list, + ) self.assertEqual(current_lr, expected_lr) state_dict = adam.state_dict() scheduler1 = paddle.optimizer.lr.ReduceOnPlateau(**kwargs) - adam1 = paddle.optimizer.Adam(learning_rate=scheduler1, - parameters=linear.parameters()) + adam1 = paddle.optimizer.Adam( + learning_rate=scheduler1, parameters=linear.parameters() + ) adam1.set_state_dict(state_dict) - self.assertEqual(scheduler.cooldown_counter, - scheduler1.cooldown_counter) + self.assertEqual( + scheduler.cooldown_counter, scheduler1.cooldown_counter + ) self.assertEqual(scheduler.best.numpy()[0], scheduler1.best) self.assertEqual(scheduler.num_bad_epochs, scheduler1.num_bad_epochs) self.assertEqual(scheduler.last_epoch, scheduler1.last_epoch) @@ -231,13 +254,15 @@ def inverse_time_lr(epoch_num, learning_rate, gamma, verbose=False): return learning_rate / (1 + gamma * epoch_num) -def polynomial_lr(epoch_num, - learning_rate, - decay_steps, - end_lr=0.0001, - power=1.0, - cycle=False, - verbose=False): +def polynomial_lr( + epoch_num, + learning_rate, + decay_steps, + end_lr=0.0001, + power=1.0, + cycle=False, + verbose=False, +): if cycle: div = math.ceil(epoch_num / float(decay_steps)) @@ -247,53 +272,58 @@ def polynomial_lr(epoch_num, else: epoch_num = min(epoch_num, decay_steps) return (learning_rate - end_lr) * ( - (1 - float(epoch_num) / float(decay_steps))**power) + end_lr + (1 - float(epoch_num) / float(decay_steps)) ** power + ) + end_lr def get_lr(self): if self.last_epoch == 0: return self.base_lr elif (self.last_epoch - 1 - self.T_max) % (2 * self.T_max) == 0: - return self.last_lr + (self.base_lr - self.eta_min) * ( - 1 - math.cos(math.pi / self.T_max)) / 2 + return ( + self.last_lr + + (self.base_lr - self.eta_min) + * (1 - math.cos(math.pi / self.T_max)) + / 2 + ) return (1 + math.cos(math.pi * self.last_epoch / self.T_max)) / ( - 1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max)) * ( - self.last_lr - self.eta_min) + self.eta_min + 1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max) + ) * (self.last_lr - self.eta_min) + self.eta_min cosine_annealing_lr_current = None -def cosine_annealing_lr(epoch_num, - learning_rate, - T_max, - eta_min=0, - verbose=False): +def cosine_annealing_lr( + epoch_num, learning_rate, T_max, eta_min=0, verbose=False +): global cosine_annealing_lr_current if epoch_num == 0: cosine_annealing_lr_current = learning_rate elif (epoch_num - 1 - T_max) % (2 * T_max) == 0: - cosine_annealing_lr_current = cosine_annealing_lr_current + ( - learning_rate - eta_min) * (1 - - math.cos(math.pi / float(T_max))) / 2 + cosine_annealing_lr_current = ( + cosine_annealing_lr_current + + (learning_rate - eta_min) + * (1 - math.cos(math.pi / float(T_max))) + / 2 + ) else: cosine_annealing_lr_current = ( - 1 + math.cos(math.pi * epoch_num / float(T_max))) / ( - 1 + math.cos(math.pi * (epoch_num - 1) / float(T_max))) * ( - cosine_annealing_lr_current - eta_min) + eta_min + 1 + math.cos(math.pi * epoch_num / float(T_max)) + ) / (1 + math.cos(math.pi * (epoch_num - 1) / float(T_max))) * ( + cosine_annealing_lr_current - eta_min + ) + eta_min return cosine_annealing_lr_current -def linear_warmup_lr(epoch_num, - learning_rate, - warmup_steps, - start_lr, - end_lr, - verbose=False): +def linear_warmup_lr( + epoch_num, learning_rate, warmup_steps, start_lr, end_lr, verbose=False +): tmp = epoch_num - warmup_steps if tmp < 0: - return start_lr + (end_lr - start_lr) * (float(epoch_num) / - float(warmup_steps)) + return start_lr + (end_lr - start_lr) * ( + float(epoch_num) / float(warmup_steps) + ) elif paddle.in_dynamic_mode(): if tmp < 3: return 0.5 @@ -305,35 +335,36 @@ def linear_warmup_lr(epoch_num, return 0.5 -def multi_step_lr(epoch_num, - learning_rate, - milestones, - gamma=0.1, - verbose=False): +def multi_step_lr( + epoch_num, learning_rate, milestones, gamma=0.1, verbose=False +): for i in range(len(milestones)): if epoch_num < milestones[i]: return learning_rate * (gamma**i) - return learning_rate * (gamma**len(milestones)) + return learning_rate * (gamma ** len(milestones)) def step_lr(epoch_num, learning_rate, step_size, gamma=0.1, verbose=False): return learning_rate * math.pow(gamma, epoch_num // step_size) -def one_cycle_lr(epoch_num, - max_learning_rate, - total_steps, - divide_factor=25, - end_learning_rate=0.0001, - phase_pct=0.3, - anneal_strategy='cos', - three_phase=False, - verbose=False): +def one_cycle_lr( + epoch_num, + max_learning_rate, + total_steps, + divide_factor=25, + end_learning_rate=0.0001, + phase_pct=0.3, + anneal_strategy='cos', + three_phase=False, + verbose=False, +): initial_lr = max_learning_rate / divide_factor if three_phase: _end_steps = [ float(phase_pct * total_steps) - 1, - float(2 * phase_pct * total_steps) - 2, total_steps - 1 + float(2 * phase_pct * total_steps) - 2, + total_steps - 1, ] _schedule_phases = [ { @@ -367,6 +398,7 @@ def one_cycle_lr(epoch_num, def anneal_func(start, end, pct): cos_out = math.cos(math.pi * pct) + 1 return end + (start - end) / 2.0 * cos_out + else: def anneal_func(start, end, pct): @@ -384,24 +416,26 @@ def one_cycle_lr(epoch_num, return computed_lr -def cyclic_lr(epoch_num, - base_learning_rate, - max_learning_rate, - step_size_up, - step_size_down, - mode, - exp_gamma=0.1, - scale_fn=None, - scale_mode='cycle', - verbose=False): +def cyclic_lr( + epoch_num, + base_learning_rate, + max_learning_rate, + step_size_up, + step_size_down, + mode, + exp_gamma=0.1, + scale_fn=None, + scale_mode='cycle', + verbose=False, +): total_steps = step_size_up + step_size_down step_ratio = step_size_up / total_steps def triangular(x): - return 1. + return 1.0 def triangular2(x): - return 1 / (2.**(x - 1)) + return 1 / (2.0 ** (x - 1)) def exp_range(x): return exp_gamma**x @@ -419,7 +453,7 @@ def cyclic_lr(epoch_num, cycle = math.floor(1 + epoch_num / total_steps) iterations = epoch_num - x = 1. + epoch_num / total_steps - cycle + x = 1.0 + epoch_num / total_steps - cycle if x <= step_ratio: scale_factor = x / step_ratio @@ -432,7 +466,6 @@ def cyclic_lr(epoch_num, class TestLRScheduler(unittest.TestCase): - def _test_static(self, python_func, paddle_api, kwarg, place): scheduler = paddle_api(**kwarg) adam = paddle.optimizer.Adam(learning_rate=scheduler) @@ -456,7 +489,8 @@ class TestLRScheduler(unittest.TestCase): out = exe.run( main_prog, feed={'x': np.random.randn(3, 4, 5).astype('float32')}, - fetch_list=lr_var.name) + fetch_list=lr_var.name, + ) self.assertEqual(out, np.array(python_func(num, **kwarg))) scheduler.step() num += 1 @@ -466,22 +500,26 @@ class TestLRScheduler(unittest.TestCase): out = exe.run( test_prog, feed={'x': np.random.randn(3, 4, 5).astype('float32')}, - fetch_list=lr_var.name) + fetch_list=lr_var.name, + ) self.assertEqual(out, np.array(python_func(num, **kwarg))) scheduler.step() num += 1 if isinstance(place, paddle.CPUPlace): compiled_train_prog = paddle.static.CompiledProgram( - main_prog).with_data_parallel(loss_name=loss.name, - places=fluid.cpu_places(4)) + main_prog + ).with_data_parallel( + loss_name=loss.name, places=fluid.cpu_places(4) + ) for epoch in range(5): python_result = python_func(num, **kwarg) for batch_id in range(2): _ = exe.run( compiled_train_prog, feed={'x': np.random.randn(12, 4, 5).astype('float32')}, - fetch_list=lr_var.name) + fetch_list=lr_var.name, + ) scopes = compiled_train_prog._executor.local_scopes() out = np.array(scopes[0].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) @@ -495,17 +533,20 @@ class TestLRScheduler(unittest.TestCase): num += 1 compiled_test_prog = paddle.static.CompiledProgram( - test_prog).with_data_parallel( - loss_name=loss.name, - share_vars_from=compiled_train_prog, - places=fluid.cpu_places(4)) + test_prog + ).with_data_parallel( + loss_name=loss.name, + share_vars_from=compiled_train_prog, + places=fluid.cpu_places(4), + ) for epoch in range(5): python_result = python_func(num, **kwarg) for batch_id in range(2): _ = exe.run( compiled_test_prog, feed={'x': np.random.randn(12, 4, 5).astype('float32')}, - fetch_list=lr_var.name) + fetch_list=lr_var.name, + ) scopes = compiled_test_prog._executor.local_scopes() out = np.array(scopes[0].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) @@ -524,10 +565,12 @@ class TestLRScheduler(unittest.TestCase): linear = paddle.nn.Linear(10, 10) if paddle_api.__name__ == "LinearWarmup": kwarg['learning_rate'] = paddle.optimizer.lr.PiecewiseDecay( - [3, 6], [0.5, 0.2, 0.1]) + [3, 6], [0.5, 0.2, 0.1] + ) scheduler = paddle_api(**kwarg) - adam = paddle.optimizer.Adam(learning_rate=scheduler, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=scheduler, parameters=linear.parameters() + ) for epoch in range(20): for batch_id in range(2): x = paddle.to_tensor(x) @@ -545,15 +588,20 @@ class TestLRScheduler(unittest.TestCase): self.assertAlmostEqual(current_lr, expected_lr) state_dict = adam.state_dict() scheduler1 = paddle.optimizer.lr.LinearWarmup(**kwarg) - adam1 = paddle.optimizer.Adam(learning_rate=scheduler1, - parameters=linear.parameters()) + adam1 = paddle.optimizer.Adam( + learning_rate=scheduler1, parameters=linear.parameters() + ) adam1.set_state_dict(state_dict) self.assertEqual(scheduler.last_epoch, scheduler1.last_epoch) self.assertEqual(scheduler.last_lr, scheduler1.last_lr) - self.assertEqual(scheduler.learning_rate.last_lr, - scheduler1.learning_rate.last_lr) - self.assertEqual(scheduler.learning_rate.last_epoch, - scheduler1.learning_rate.last_epoch) + self.assertEqual( + scheduler.learning_rate.last_lr, + scheduler1.learning_rate.last_lr, + ) + self.assertEqual( + scheduler.learning_rate.last_epoch, + scheduler1.learning_rate.last_epoch, + ) scheduler.step() else: self.assertEqual(current_lr, expected_lr) @@ -563,266 +611,355 @@ class TestLRScheduler(unittest.TestCase): with self.assertRaises(NotImplementedError): paddle.optimizer.lr.LRScheduler().step() with self.assertRaises(TypeError): - paddle.optimizer.lr.MultiStepDecay(learning_rate="test", - milestones=[1, 2, 3]) + paddle.optimizer.lr.MultiStepDecay( + learning_rate="test", milestones=[1, 2, 3] + ) with self.assertRaises(TypeError): - paddle.optimizer.lr.MultiStepDecay(learning_rate=0.5, - milestones='test') + paddle.optimizer.lr.MultiStepDecay( + learning_rate=0.5, milestones='test' + ) with self.assertRaises(ValueError): - paddle.optimizer.lr.MultiStepDecay(learning_rate=0.5, - milestones=[3, 2, 1]) + paddle.optimizer.lr.MultiStepDecay( + learning_rate=0.5, milestones=[3, 2, 1] + ) with self.assertRaises(ValueError): - paddle.optimizer.lr.MultiStepDecay(learning_rate=0.5, - milestones=[1, 2, 3], - gamma=2) + paddle.optimizer.lr.MultiStepDecay( + learning_rate=0.5, milestones=[1, 2, 3], gamma=2 + ) # check type of max_learning_rate with self.assertRaises(TypeError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate='test', - total_steps=20) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate='test', total_steps=20 + ) # check value of max_learning_rate with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=-1.5, - total_steps=20) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=-1.5, total_steps=20 + ) # check type of end_learning_rate with self.assertRaises(TypeError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=20, - end_learning_rate='test') + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps=20, end_learning_rate='test' + ) # check value of end_learning_rate with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=20, - end_learning_rate=-1) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps=20, end_learning_rate=-1 + ) # check type of total_steps with self.assertRaises(TypeError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps='test') + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps='test' + ) # check value of total_steps with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=-10) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps=-10 + ) # check value of anneal_strategy with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=20, - anneal_strategy='test') + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps=20, anneal_strategy='test' + ) # check value of phase_pct when three_phase is True with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=20, - phase_pct=0.6, - three_phase=True) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, + total_steps=20, + phase_pct=0.6, + three_phase=True, + ) # check type of max_learning_rate with self.assertRaises(TypeError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate='test', - step_size_up=10) + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate='test', + step_size_up=10, + ) # check value of max_learning_rate with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=-1, - step_size_up=10) + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, max_learning_rate=-1, step_size_up=10 + ) # check type of step_size_up with self.assertRaises(TypeError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up='test') + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up='test', + ) # check value of step_size_up with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=-1) + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, max_learning_rate=1.0, step_size_up=-1 + ) # check type of step_size_down with self.assertRaises(TypeError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=500, - step_size_down='test') + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up=500, + step_size_down='test', + ) # check type of step_size_down with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=500, - step_size_down=-1) + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up=500, + step_size_down=-1, + ) # check value of mode with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=500, - step_size_down=500, - mode='test') + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up=500, + step_size_down=500, + mode='test', + ) # check type value of scale_mode with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=500, - step_size_down=-1, - scale_mode='test') + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up=500, + step_size_down=-1, + scale_mode='test', + ) func_api_kwargs = [ - (noam_lr, paddle.optimizer.lr.NoamDecay, { - "d_model": 0.01, - "warmup_steps": 100, - "verbose": False - }), - (piecewise_lr, paddle.optimizer.lr.PiecewiseDecay, { - "boundaries": [3, 6, 9, 15, 20], - "values": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6], - "verbose": False - }), - (natural_exp_lr, paddle.optimizer.lr.NaturalExpDecay, { - "learning_rate": 0.5, - "gamma": 0.1, - "verbose": True - }), - (inverse_time_lr, paddle.optimizer.lr.InverseTimeDecay, { - "learning_rate": 0.5, - "gamma": 0.1, - "verbose": False - }), - (polynomial_lr, paddle.optimizer.lr.PolynomialDecay, { - "learning_rate": 0.5, - "decay_steps": 20, - "end_lr": 0, - "power": 1.0, - "cycle": False - }), - (polynomial_lr, paddle.optimizer.lr.PolynomialDecay, { - "learning_rate": 0.5, - "decay_steps": 20, - "end_lr": 0, - "power": 1.0, - "cycle": True, - "verbose": False - }), - (linear_warmup_lr, paddle.optimizer.lr.LinearWarmup, { - 'learning_rate': 0.5, - 'warmup_steps': 10, - 'start_lr': 0, - 'end_lr': 0.5 - }), - (exponential_lr, paddle.optimizer.lr.ExponentialDecay, { - "learning_rate": 0.5, - "gamma": 0.9, - "verbose": False - }), - (multi_step_lr, paddle.optimizer.lr.MultiStepDecay, { - "learning_rate": 0.5, - "milestones": [3, 6, 9, 15, 20], - "gamma": 0.8 - }), - (step_lr, paddle.optimizer.lr.StepDecay, { - "learning_rate": 0.5, - "step_size": 2, - "gamma": 0.8, - "verbose": False - }), - (lambda_lr, paddle.optimizer.lr.LambdaDecay, { - "learning_rate": 0.5, - "lr_lambda": lambda x: 0.95**x, - "verbose": True - }), - (multiplicative_lr, paddle.optimizer.lr.MultiplicativeDecay, { - "learning_rate": 0.5, - "lr_lambda": lambda x: 0.95, - "verbose": True - }), - (cosine_annealing_lr, paddle.optimizer.lr.CosineAnnealingDecay, { - "learning_rate": 0.5, - "T_max": 10, - "verbose": False - }), - (one_cycle_lr, paddle.optimizer.lr.OneCycleLR, { - "max_learning_rate": 0.1, - "total_steps": 20, - "divide_factor": 5, - "end_learning_rate": 0.0001, - "anneal_strategy": 'cos', - "phase_pct": 0.3, - "three_phase": False, - }), - (one_cycle_lr, paddle.optimizer.lr.OneCycleLR, { - "max_learning_rate": 0.5, - "total_steps": 20, - "divide_factor": 10, - "end_learning_rate": 0.001, - "anneal_strategy": 'linear', - "phase_pct": 0.4, - "three_phase": False, - }), - (one_cycle_lr, paddle.optimizer.lr.OneCycleLR, { - "max_learning_rate": 1.0, - "total_steps": 20, - "divide_factor": 9, - "end_learning_rate": 0.0001, - "anneal_strategy": 'cos', - "phase_pct": 0.3, - "three_phase": True, - }), - (one_cycle_lr, paddle.optimizer.lr.OneCycleLR, { - "max_learning_rate": 0.3, - "total_steps": 20, - "divide_factor": 25, - "end_learning_rate": 0.0005, - "anneal_strategy": 'linear', - "phase_pct": 0.2, - "three_phase": True, - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'triangular', - "exp_gamma": 1., - "scale_fn": None, - "scale_mode": 'cycle', - "verbose": False - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'triangular2', - "exp_gamma": 1., - "scale_fn": None, - "scale_mode": 'cycle', - "verbose": False - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'exp_range', - "exp_gamma": 0.8, - "scale_fn": None, - "scale_mode": 'cycle', - "verbose": False - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'exp_range', - "exp_gamma": 1., - "scale_fn": lambda x: 0.95**x, - "scale_mode": 'cycle', - "verbose": False - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'exp_range', - "exp_gamma": 1., - "scale_fn": lambda x: 0.95, - "scale_mode": 'iterations', - "verbose": False - }) + ( + noam_lr, + paddle.optimizer.lr.NoamDecay, + {"d_model": 0.01, "warmup_steps": 100, "verbose": False}, + ), + ( + piecewise_lr, + paddle.optimizer.lr.PiecewiseDecay, + { + "boundaries": [3, 6, 9, 15, 20], + "values": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6], + "verbose": False, + }, + ), + ( + natural_exp_lr, + paddle.optimizer.lr.NaturalExpDecay, + {"learning_rate": 0.5, "gamma": 0.1, "verbose": True}, + ), + ( + inverse_time_lr, + paddle.optimizer.lr.InverseTimeDecay, + {"learning_rate": 0.5, "gamma": 0.1, "verbose": False}, + ), + ( + polynomial_lr, + paddle.optimizer.lr.PolynomialDecay, + { + "learning_rate": 0.5, + "decay_steps": 20, + "end_lr": 0, + "power": 1.0, + "cycle": False, + }, + ), + ( + polynomial_lr, + paddle.optimizer.lr.PolynomialDecay, + { + "learning_rate": 0.5, + "decay_steps": 20, + "end_lr": 0, + "power": 1.0, + "cycle": True, + "verbose": False, + }, + ), + ( + linear_warmup_lr, + paddle.optimizer.lr.LinearWarmup, + { + 'learning_rate': 0.5, + 'warmup_steps': 10, + 'start_lr': 0, + 'end_lr': 0.5, + }, + ), + ( + exponential_lr, + paddle.optimizer.lr.ExponentialDecay, + {"learning_rate": 0.5, "gamma": 0.9, "verbose": False}, + ), + ( + multi_step_lr, + paddle.optimizer.lr.MultiStepDecay, + { + "learning_rate": 0.5, + "milestones": [3, 6, 9, 15, 20], + "gamma": 0.8, + }, + ), + ( + step_lr, + paddle.optimizer.lr.StepDecay, + { + "learning_rate": 0.5, + "step_size": 2, + "gamma": 0.8, + "verbose": False, + }, + ), + ( + lambda_lr, + paddle.optimizer.lr.LambdaDecay, + { + "learning_rate": 0.5, + "lr_lambda": lambda x: 0.95**x, + "verbose": True, + }, + ), + ( + multiplicative_lr, + paddle.optimizer.lr.MultiplicativeDecay, + { + "learning_rate": 0.5, + "lr_lambda": lambda x: 0.95, + "verbose": True, + }, + ), + ( + cosine_annealing_lr, + paddle.optimizer.lr.CosineAnnealingDecay, + {"learning_rate": 0.5, "T_max": 10, "verbose": False}, + ), + ( + one_cycle_lr, + paddle.optimizer.lr.OneCycleLR, + { + "max_learning_rate": 0.1, + "total_steps": 20, + "divide_factor": 5, + "end_learning_rate": 0.0001, + "anneal_strategy": 'cos', + "phase_pct": 0.3, + "three_phase": False, + }, + ), + ( + one_cycle_lr, + paddle.optimizer.lr.OneCycleLR, + { + "max_learning_rate": 0.5, + "total_steps": 20, + "divide_factor": 10, + "end_learning_rate": 0.001, + "anneal_strategy": 'linear', + "phase_pct": 0.4, + "three_phase": False, + }, + ), + ( + one_cycle_lr, + paddle.optimizer.lr.OneCycleLR, + { + "max_learning_rate": 1.0, + "total_steps": 20, + "divide_factor": 9, + "end_learning_rate": 0.0001, + "anneal_strategy": 'cos', + "phase_pct": 0.3, + "three_phase": True, + }, + ), + ( + one_cycle_lr, + paddle.optimizer.lr.OneCycleLR, + { + "max_learning_rate": 0.3, + "total_steps": 20, + "divide_factor": 25, + "end_learning_rate": 0.0005, + "anneal_strategy": 'linear', + "phase_pct": 0.2, + "three_phase": True, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'triangular', + "exp_gamma": 1.0, + "scale_fn": None, + "scale_mode": 'cycle', + "verbose": False, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'triangular2', + "exp_gamma": 1.0, + "scale_fn": None, + "scale_mode": 'cycle', + "verbose": False, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'exp_range', + "exp_gamma": 0.8, + "scale_fn": None, + "scale_mode": 'cycle', + "verbose": False, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'exp_range', + "exp_gamma": 1.0, + "scale_fn": lambda x: 0.95**x, + "scale_mode": 'cycle', + "verbose": False, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'exp_range', + "exp_gamma": 1.0, + "scale_fn": lambda x: 0.95, + "scale_mode": 'iterations', + "verbose": False, + }, + ), ] for python_func, paddle_api, kwarg in func_api_kwargs: @@ -838,14 +975,17 @@ class TestLRScheduler(unittest.TestCase): paddle.enable_static() def test_linear_warmp(self): - natural_lr = paddle.optimizer.lr.NaturalExpDecay(learning_rate=0.5, - gamma=0.1) + natural_lr = paddle.optimizer.lr.NaturalExpDecay( + learning_rate=0.5, gamma=0.1 + ) natural_lr_warmup = paddle.optimizer.lr.LinearWarmup( - learning_rate=natural_lr, warmup_steps=10, start_lr=0.0, end_lr=0.1) + learning_rate=natural_lr, warmup_steps=10, start_lr=0.0, end_lr=0.1 + ) for idx in range(30): if idx >= 10: - self.assertEqual(natural_lr_warmup.get_lr(), - natural_lr.get_lr()) + self.assertEqual( + natural_lr_warmup.get_lr(), natural_lr.get_lr() + ) natural_lr.step() natural_lr_warmup.step() diff --git a/python/paddle/fluid/tests/unittests/test_lrn_op.py b/python/paddle/fluid/tests/unittests/test_lrn_op.py index c87d3067cc4..60446284600 100644 --- a/python/paddle/fluid/tests/unittests/test_lrn_op.py +++ b/python/paddle/fluid/tests/unittests/test_lrn_op.py @@ -22,7 +22,6 @@ from paddle.fluid import Program, program_guard class TestLRNOp(OpTest): - def get_input(self): r''' TODO(gongweibao): why it's grad diff is so large? x = np.ndarray( @@ -64,7 +63,7 @@ class TestLRNOp(OpTest): 'k': self.k, 'alpha': self.alpha, 'beta': self.beta, - 'data_format': self.data_format + 'data_format': self.data_format, } return attrs @@ -103,13 +102,11 @@ class TestLRNOp(OpTest): class TestLRNOpAttrDataFormat(TestLRNOp): - def init_test_case(self): self.data_format = 'NHWC' class TestLRNAPI(unittest.TestCase): - def test_case(self): data1 = fluid.data(name='data1', shape=[2, 4, 5, 5], dtype='float32') data2 = fluid.data(name='data2', shape=[2, 5, 5, 4], dtype='float32') @@ -124,23 +121,22 @@ class TestLRNAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2], - return_numpy=True) - - np.testing.assert_allclose(results[0], - np.transpose(results[1], (0, 3, 1, 2)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2], + return_numpy=True, + ) + + np.testing.assert_allclose( + results[0], np.transpose(results[1], (0, 3, 1, 2)), rtol=1e-05 + ) def test_exception(self): input1 = fluid.data(name="input1", shape=[2, 4, 5, 5], dtype="float32") - input2 = fluid.data(name="input2", - shape=[2, 4, 5, 5, 5], - dtype="float32") + input2 = fluid.data( + name="input2", shape=[2, 4, 5, 5, 5], dtype="float32" + ) def _attr_data_fromat(): out = fluid.layers.lrn(input1, data_format='NDHW') @@ -153,7 +149,6 @@ class TestLRNAPI(unittest.TestCase): class TestLRNOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input must be float32 @@ -162,7 +157,6 @@ class TestLRNOpError(unittest.TestCase): class TestLocalResponseNormFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -174,84 +168,81 @@ class TestLocalResponseNormFAPI(unittest.TestCase): in_np1 = np.random.random([3, 40, 40]).astype("float32") in_np2 = np.transpose(in_np1, (0, 2, 1)) - input1 = fluid.data(name="input1", - shape=[3, 40, 40], - dtype="float32") - input2 = fluid.data(name="input2", - shape=[3, 40, 40], - dtype="float32") - res1 = paddle.nn.functional.local_response_norm(x=input1, - size=5, - data_format='NCL') - res2 = paddle.nn.functional.local_response_norm(x=input2, - size=5, - data_format='NLC') + input1 = fluid.data( + name="input1", shape=[3, 40, 40], dtype="float32" + ) + input2 = fluid.data( + name="input2", shape=[3, 40, 40], dtype="float32" + ) + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCL' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NLC' + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input1": in_np1, - "input2": in_np2 - }, - fetch_list=[res1, res2]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) fetches1_tran = np.transpose(fetches[1], (0, 2, 1)) np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) def check_static_4d_input(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input1 = fluid.data(name="input1", - shape=[3, 3, 40, 40], - dtype="float32") - input2 = fluid.data(name="input2", - shape=[3, 40, 40, 3], - dtype="float32") - - res1 = paddle.nn.functional.local_response_norm(x=input1, - size=5, - data_format='NCHW') - res2 = paddle.nn.functional.local_response_norm(x=input2, - size=5, - data_format='NHWC') + input1 = fluid.data( + name="input1", shape=[3, 3, 40, 40], dtype="float32" + ) + input2 = fluid.data( + name="input2", shape=[3, 40, 40, 3], dtype="float32" + ) + + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NHWC' + ) in_np1 = np.random.random([3, 3, 40, 40]).astype("float32") in_np2 = np.transpose(in_np1, (0, 2, 3, 1)) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input1": in_np1, - "input2": in_np2 - }, - fetch_list=[res1, res2]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) fetches1_tran = np.transpose(fetches[1], (0, 3, 1, 2)) np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) def check_static_5d_input(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input1 = fluid.data(name="input1", - shape=[3, 3, 3, 40, 40], - dtype="float32") - input2 = fluid.data(name="input2", - shape=[3, 3, 40, 40, 3], - dtype="float32") - res1 = paddle.nn.functional.local_response_norm(x=input1, - size=5, - data_format='NCDHW') - res2 = paddle.nn.functional.local_response_norm(x=input2, - size=5, - data_format='NDHWC') + input1 = fluid.data( + name="input1", shape=[3, 3, 3, 40, 40], dtype="float32" + ) + input2 = fluid.data( + name="input2", shape=[3, 3, 40, 40, 3], dtype="float32" + ) + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCDHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NDHWC' + ) in_np1 = np.random.random([3, 3, 3, 40, 40]).astype("float32") in_np2 = np.transpose(in_np1, (0, 2, 3, 4, 1)) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input1": in_np1, - "input2": in_np2 - }, - fetch_list=[res1, res2]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) fetches1_tran = np.transpose(fetches[1], (0, 4, 1, 2, 3)) np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) @@ -270,12 +261,12 @@ class TestLocalResponseNormFAPI(unittest.TestCase): in1 = paddle.to_tensor(in_np1) in2 = paddle.to_tensor(in_np2) - res1 = paddle.nn.functional.local_response_norm(x=in1, - size=5, - data_format='NCL') - res2 = paddle.nn.functional.local_response_norm(x=in2, - size=5, - data_format='NLC') + res1 = paddle.nn.functional.local_response_norm( + x=in1, size=5, data_format='NCL' + ) + res2 = paddle.nn.functional.local_response_norm( + x=in2, size=5, data_format='NLC' + ) res2_tran = np.transpose(res2.numpy(), (0, 2, 1)) np.testing.assert_allclose(res1.numpy(), res2_tran, rtol=1e-05) @@ -288,12 +279,12 @@ class TestLocalResponseNormFAPI(unittest.TestCase): in1 = paddle.to_tensor(in_np1) in2 = paddle.to_tensor(in_np2) - res1 = paddle.nn.functional.local_response_norm(x=in1, - size=5, - data_format='NCHW') - res2 = paddle.nn.functional.local_response_norm(x=in2, - size=5, - data_format='NHWC') + res1 = paddle.nn.functional.local_response_norm( + x=in1, size=5, data_format='NCHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=in2, size=5, data_format='NHWC' + ) res2_tran = np.transpose(res2.numpy(), (0, 3, 1, 2)) np.testing.assert_allclose(res1.numpy(), res2_tran, rtol=1e-05) @@ -306,12 +297,12 @@ class TestLocalResponseNormFAPI(unittest.TestCase): in1 = paddle.to_tensor(in_np1) in2 = paddle.to_tensor(in_np2) - res1 = paddle.nn.functional.local_response_norm(x=in1, - size=5, - data_format='NCDHW') - res2 = paddle.nn.functional.local_response_norm(x=in2, - size=5, - data_format='NDHWC') + res1 = paddle.nn.functional.local_response_norm( + x=in1, size=5, data_format='NCDHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=in2, size=5, data_format='NDHWC' + ) res2_tran = np.transpose(res2.numpy(), (0, 4, 1, 2, 3)) np.testing.assert_allclose(res1.numpy(), res2_tran, rtol=1e-05) @@ -324,14 +315,14 @@ class TestLocalResponseNormFAPI(unittest.TestCase): class TestLocalResponseNormFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of lrn must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) paddle.nn.functional.local_response_norm(x1, size=5) self.assertRaises(TypeError, test_Variable) @@ -344,9 +335,9 @@ class TestLocalResponseNormFAPIError(unittest.TestCase): def test_dataformat(): x = fluid.data(name='x', shape=[3, 4, 5, 6], dtype="float32") - paddle.nn.functional.local_response_norm(x, - size=5, - data_format="NCTHW") + paddle.nn.functional.local_response_norm( + x, size=5, data_format="NCTHW" + ) self.assertRaises(ValueError, test_dataformat) @@ -364,7 +355,6 @@ class TestLocalResponseNormFAPIError(unittest.TestCase): class TestLocalResponseNormCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_lstm_cudnn_op.py b/python/paddle/fluid/tests/unittests/test_lstm_cudnn_op.py index 53949bbb738..3be8fa8e3c1 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_cudnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_cudnn_op.py @@ -33,7 +33,6 @@ EXP_MAX_INPUT = 40.0 class RandomWeight: - def __init__(self): pass @@ -43,34 +42,29 @@ class RandomWeight: self.input_size = input_size self.dtype = dtype - self.weight_ih = np.random.uniform(low=-std, - high=std, - size=(4 * self.hidden_size, - self.input_size)).astype(dtype) + self.weight_ih = np.random.uniform( + low=-std, high=std, size=(4 * self.hidden_size, self.input_size) + ).astype(dtype) self.weight_hh = np.random.uniform( - low=-std, high=std, - size=(4 * self.hidden_size, self.hidden_size)).astype(dtype) - self.bias_ih = np.random.uniform(low=-std, - high=std, - size=(4 * - self.hidden_size)).astype(dtype) - self.bias_hh = np.random.uniform(low=-std, - high=std, - size=(4 * - self.hidden_size)).astype(dtype) + low=-std, high=std, size=(4 * self.hidden_size, self.hidden_size) + ).astype(dtype) + self.bias_ih = np.random.uniform( + low=-std, high=std, size=(4 * self.hidden_size) + ).astype(dtype) + self.bias_hh = np.random.uniform( + low=-std, high=std, size=(4 * self.hidden_size) + ).astype(dtype) weight = RandomWeight() class LayerMixin(object): - def __call__(self, *args, **kwargs): return self.forward(*args, **kwargs) class LayerListMixin(LayerMixin): - def __init__(self, layers=None): self._layers = list(layers) if layers else [] @@ -82,7 +76,6 @@ class LayerListMixin(LayerMixin): class LSTMCell(LayerMixin): - def __init__(self, input_size, hidden_size, bias=True): self.input_size = input_size self.hidden_size = hidden_size @@ -145,12 +138,14 @@ def update_state(mask, new, old): return tuple(map(lambda x, y: np.where(mask, x, y), new, old)) -def rnn(cell, - inputs, - initial_states, - sequence_length=None, - time_major=False, - is_reverse=False): +def rnn( + cell, + inputs, + initial_states, + sequence_length=None, + time_major=False, + is_reverse=False, +): if not time_major: inputs = np.transpose(inputs, [1, 0, 2]) if is_reverse: @@ -172,7 +167,7 @@ def rnn(cell, if mask is not None: m_t = mask[t] y, new_state = cell(x_t, state) - y = np.where(m_t, y, 0.) + y = np.where(m_t, y, 0.0) outputs.append(y) state = update_state(m_t, new_state, state) else: @@ -190,25 +185,27 @@ def rnn(cell, return outputs, final_state -def birnn(cell_fw, - cell_bw, - inputs, - initial_states, - sequence_length=None, - time_major=False): +def birnn( + cell_fw, + cell_bw, + inputs, + initial_states, + sequence_length=None, + time_major=False, +): states_fw, states_bw = initial_states - outputs_fw, states_fw = rnn(cell_fw, - inputs, - states_fw, - sequence_length, - time_major=time_major) - - outputs_bw, states_bw = rnn(cell_bw, - inputs, - states_bw, - sequence_length, - time_major=time_major, - is_reverse=True) + outputs_fw, states_fw = rnn( + cell_fw, inputs, states_fw, sequence_length, time_major=time_major + ) + + outputs_bw, states_bw = rnn( + cell_bw, + inputs, + states_bw, + sequence_length, + time_major=time_major, + is_reverse=True, + ) outputs = np.concatenate((outputs_fw, outputs_bw), -1) final_states = (states_fw, states_bw) @@ -271,7 +268,6 @@ def concat_states(states, bidirectional=False, state_components=1): class RNN(LayerMixin): - def __init__(self, cell, is_reverse=False, time_major=False): super(RNN, self).__init__() self.cell = cell @@ -282,59 +278,69 @@ class RNN(LayerMixin): self.time_major = time_major def forward(self, inputs, initial_states=None, sequence_length=None): - final_outputs, final_states = rnn(self.cell, - inputs, - initial_states=initial_states, - sequence_length=sequence_length, - time_major=self.time_major, - is_reverse=self.is_reverse) + final_outputs, final_states = rnn( + self.cell, + inputs, + initial_states=initial_states, + sequence_length=sequence_length, + time_major=self.time_major, + is_reverse=self.is_reverse, + ) return final_outputs, final_states class BiRNN(LayerMixin): - def __init__(self, cell_fw, cell_bw, time_major=False): super(BiRNN, self).__init__() self.cell_fw = cell_fw self.cell_bw = cell_bw self.time_major = time_major - def forward(self, - inputs, - initial_states=None, - sequence_length=None, - **kwargs): + def forward( + self, inputs, initial_states=None, sequence_length=None, **kwargs + ): if isinstance(initial_states, (list, tuple)): - assert len(initial_states) == 2, \ - "length of initial_states should be 2 when it is a list/tuple" + assert ( + len(initial_states) == 2 + ), "length of initial_states should be 2 when it is a list/tuple" else: initial_states = [initial_states, initial_states] - outputs, final_states = birnn(self.cell_fw, self.cell_bw, inputs, - initial_states, sequence_length, - self.time_major) + outputs, final_states = birnn( + self.cell_fw, + self.cell_bw, + inputs, + initial_states, + sequence_length, + self.time_major, + ) return outputs, final_states class RNNMixin(LayerListMixin): - def forward(self, inputs, initial_states=None, sequence_length=None): batch_index = 1 if self.time_major else 0 batch_size = inputs.shape[batch_index] dtype = inputs.dtype if initial_states is None: - state_shape = (self.num_layers * self.num_directions, batch_size, - self.hidden_size) + state_shape = ( + self.num_layers * self.num_directions, + batch_size, + self.hidden_size, + ) if self.state_components == 1: initial_states = np.zeros(state_shape, dtype) else: - initial_states = tuple([ - np.zeros(state_shape, dtype) - for _ in range(self.state_components) - ]) - - states = split_states(initial_states, self.num_directions == 2, - self.state_components) + initial_states = tuple( + [ + np.zeros(state_shape, dtype) + for _ in range(self.state_components) + ] + ) + + states = split_states( + initial_states, self.num_directions == 2, self.state_components + ) final_states = [] for i, rnn_layer in enumerate(self): @@ -344,20 +350,22 @@ class RNNMixin(LayerListMixin): final_states.append(final_state) inputs = outputs - final_states = concat_states(final_states, self.num_directions == 2, - self.state_components) + final_states = concat_states( + final_states, self.num_directions == 2, self.state_components + ) return outputs, final_states class LSTM(RNNMixin): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - dropout=0., - time_major=False): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + dropout=0.0, + time_major=False, + ): super(LSTM, self).__init__() if direction in ["forward", "backward"]: @@ -378,7 +386,8 @@ class LSTM(RNNMixin): else: raise ValueError( "direction should be forward, backward or bidirectional, " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.input_size = input_size self.hidden_size = hidden_size @@ -389,10 +398,10 @@ class LSTM(RNNMixin): self.state_components = 2 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNLstmOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(2 * self.num_layers): @@ -404,8 +413,11 @@ class TestCUDNNLstmOp(OpTest): def setUp(self): self.op_type = "cudnn_lstm" self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.sequence_length = None if core.is_compiled_with_rocm( - ) else np.array([12, 11, 10, 9, 8], dtype=np.int32) + self.sequence_length = ( + None + if core.is_compiled_with_rocm() + else np.array([12, 11, 10, 9, 8], dtype=np.int32) + ) self.num_layers = 1 self.set_attrs() @@ -414,24 +426,26 @@ class TestCUDNNLstmOp(OpTest): input_size = 21 hidden_size = 21 - input = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_length, batch_size, - input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, high=0.1, size=(seq_length, batch_size, input_size) + ).astype(self.dtype) input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 weight.updata_weight(hidden_size, input_size, self.dtype) - rnn1 = LSTM(input_size, - hidden_size, - num_layers=self.num_layers, - time_major=True, - direction="forward") - - output, (last_hidden, - last_cell) = rnn1(input, sequence_length=self.sequence_length) + rnn1 = LSTM( + input_size, + hidden_size, + num_layers=self.num_layers, + time_major=True, + direction="forward", + ) + + output, (last_hidden, last_cell) = rnn1( + input, sequence_length=self.sequence_length + ) flat_w = [] num = 0 @@ -455,10 +469,12 @@ class TestCUDNNLstmOp(OpTest): bias_hh = weight.bias_hh flat_w.append(("bias" + str(num), bias_hh)) num += 1 - init_h = np.zeros( - (self.num_layers, batch_size, hidden_size)).astype(self.dtype) - init_c = np.zeros( - (self.num_layers, batch_size, hidden_size)).astype(self.dtype) + init_h = np.zeros((self.num_layers, batch_size, hidden_size)).astype( + self.dtype + ) + init_c = np.zeros((self.num_layers, batch_size, hidden_size)).astype( + self.dtype + ) state_out = np.ndarray((300)).astype("uint8") if core.is_compiled_with_rocm(): @@ -473,7 +489,7 @@ class TestCUDNNLstmOp(OpTest): 'WeightList': flat_w, 'InitH': init_h, 'InitC': init_c, - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -494,7 +510,7 @@ class TestCUDNNLstmOp(OpTest): "LastH": last_hidden, 'LastC': last_cell, 'Reserve': np.ndarray((400)).astype("uint8"), - 'StateOut': state_out + 'StateOut': state_out, } def set_attrs(self): @@ -503,26 +519,29 @@ class TestCUDNNLstmOp(OpTest): def test_output_with_place(self): place = core.CUDAPlace(0) if core.is_compiled_with_rocm(): - self.check_output_with_place(place, - atol=1e-5, - no_check_set=['Reserve', 'StateOut']) + self.check_output_with_place( + place, atol=1e-5, no_check_set=['Reserve', 'StateOut'] + ) else: - self.check_output_with_place(place, - no_check_set=['Reserve', 'StateOut']) + self.check_output_with_place( + place, no_check_set=['Reserve', 'StateOut'] + ) def test_grad_with_place(self): place = core.CUDAPlace(0) var_name_list = self.get_weight_names() for var_name in var_name_list: self.check_grad_with_place( - place, set(['Input', var_name, 'InitH', 'InitC']), - ['Out', 'LastH', 'LastC']) + place, + set(['Input', var_name, 'InitH', 'InitC']), + ['Out', 'LastH', 'LastC'], + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNlstmAPI(unittest.TestCase): - def test_lstm(self): seq_len = 20 batch_size = 5 @@ -530,31 +549,41 @@ class TestCUDNNlstmAPI(unittest.TestCase): dropout_prob = 0.0 num_layers = 1 dtype = 'float32' if core.is_compiled_with_rocm() else 'float64' - input = fluid.data(name='input', - shape=[seq_len, batch_size, hidden_size], - dtype=dtype) - init_h = layers.fill_constant([num_layers, batch_size, hidden_size], - dtype, 0.0) - init_c = layers.fill_constant([num_layers, batch_size, hidden_size], - dtype, 0.0) - rnn_out, last_h, last_c = layers.lstm(input, init_h, init_c, seq_len, - hidden_size, num_layers, - dropout_prob, False) + input = fluid.data( + name='input', shape=[seq_len, batch_size, hidden_size], dtype=dtype + ) + init_h = layers.fill_constant( + [num_layers, batch_size, hidden_size], dtype, 0.0 + ) + init_c = layers.fill_constant( + [num_layers, batch_size, hidden_size], dtype, 0.0 + ) + rnn_out, last_h, last_c = layers.lstm( + input, + init_h, + init_c, + seq_len, + hidden_size, + num_layers, + dropout_prob, + False, + ) exe = fluid.Executor(fluid.CUDAPlace(0)) exe.run(fluid.default_startup_program()) - input_i = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_len, batch_size, - hidden_size)).astype("float64") - out = exe.run(fluid.default_main_program(), - feed={'input': input_i}, - fetch_list=[rnn_out, last_h, last_c, 'cudnn_lstm_0.w_0']) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + input_i = np.random.uniform( + low=-0.1, high=0.1, size=(seq_len, batch_size, hidden_size) + ).astype("float64") + out = exe.run( + fluid.default_main_program(), + feed={'input': input_i}, + fetch_list=[rnn_out, last_h, last_c, 'cudnn_lstm_0.w_0'], + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNlstmAPI(unittest.TestCase): - def test_lstm(self): seq_len = 20 batch_size = 5 @@ -562,25 +591,36 @@ class TestCUDNNlstmAPI(unittest.TestCase): dropout_prob = 0.0 num_layers = 2 dtype = 'float32' if core.is_compiled_with_rocm() else 'float64' - input = fluid.data(name='input', - shape=[seq_len, batch_size, hidden_size], - dtype=dtype) - init_h = layers.fill_constant([num_layers, batch_size, hidden_size], - dtype, 0.0) - init_c = layers.fill_constant([num_layers, batch_size, hidden_size], - dtype, 0.0) - rnn_out, last_h, last_c = layers.lstm(input, init_h, init_c, seq_len, - hidden_size, num_layers, - dropout_prob, False, True) + input = fluid.data( + name='input', shape=[seq_len, batch_size, hidden_size], dtype=dtype + ) + init_h = layers.fill_constant( + [num_layers, batch_size, hidden_size], dtype, 0.0 + ) + init_c = layers.fill_constant( + [num_layers, batch_size, hidden_size], dtype, 0.0 + ) + rnn_out, last_h, last_c = layers.lstm( + input, + init_h, + init_c, + seq_len, + hidden_size, + num_layers, + dropout_prob, + False, + True, + ) exe = fluid.Executor(fluid.CUDAPlace(0)) exe.run(fluid.default_startup_program()) - input_i = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_len, batch_size, - hidden_size)).astype(dtype) - out = exe.run(fluid.default_main_program(), - feed={'input': input_i}, - fetch_list=[rnn_out, last_h, last_c, 'cudnn_lstm_0.w_0']) + input_i = np.random.uniform( + low=-0.1, high=0.1, size=(seq_len, batch_size, hidden_size) + ).astype(dtype) + out = exe.run( + fluid.default_main_program(), + feed={'input': input_i}, + fetch_list=[rnn_out, last_h, last_c, 'cudnn_lstm_0.w_0'], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lstm_op.py b/python/paddle/fluid/tests/unittests/test_lstm_op.py index db701a7b733..aeb9b28c860 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_op.py @@ -33,13 +33,13 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. + return (2.0 / (1.0 + np.exp(y))) - 1.0 def relu(x): @@ -50,23 +50,23 @@ ACTIVATION = { 'identity': identity, 'sigmoid': sigmoid, 'tanh': tanh, - 'relu': relu + 'relu': relu, } def lstm( - input, # T x 4D - lod, # 1 x N - h0=None, # N x D - c0=None, # N x D - w_h=None, # D x 4D - w_b=None, # 1 x 4D - w_c=None, # 1 x 3D - is_reverse=False, - act_gate=None, - act_cell=None, - act_cand=None): - + input, # T x 4D + lod, # 1 x N + h0=None, # N x D + c0=None, # N x D + w_h=None, # D x 4D + w_b=None, # 1 x 4D + w_c=None, # 1 x 3D + is_reverse=False, + act_gate=None, + act_cell=None, + act_cand=None, +): def _step(x, w_h, w_c, h_pre, c_pre, act_gate, act_cell, act_cand): g = np.dot(h_pre, w_h) # 1 x 4D g = g + x @@ -108,13 +108,14 @@ def lstm( for i in range(batch_size): # compute one sequence seq_len = lod[0][i] - x = input[offset[i]:offset[i + 1], :] + x = input[offset[i] : offset[i + 1], :] h_pre = h0[i] # 1 x D c_pre = c0[i] # 1 x D for j in range(seq_len): # compute one step - h_pre, c_pre = _step(x[j], w_h, w_c, h_pre, c_pre, act_gate, - act_cell, act_cand) + h_pre, c_pre = _step( + x[j], w_h, w_c, h_pre, c_pre, act_gate, act_cell, act_cand + ) hidden.append(h_pre.flatten()) cell.append(c_pre.flatten()) @@ -130,7 +131,6 @@ def lstm( class LstmUnitTestError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size = 20 @@ -138,77 +138,123 @@ class LstmUnitTestError(unittest.TestCase): dropout_prob = 0.2 hidden_size = 150 num_layers = 1 - input = fluid.data(name='input', - shape=[batch_size, seq_len, hidden_size], - dtype='float32') - pre_hidden = fill_constant([num_layers, batch_size, hidden_size], - 'float32', 0.0) - pre_cell = fill_constant([num_layers, batch_size, hidden_size], - 'float32', 0.0) + input = fluid.data( + name='input', + shape=[batch_size, seq_len, hidden_size], + dtype='float32', + ) + pre_hidden = fill_constant( + [num_layers, batch_size, hidden_size], 'float32', 0.0 + ) + pre_cell = fill_constant( + [num_layers, batch_size, hidden_size], 'float32', 0.0 + ) np_input = np.random.uniform( - -0.1, 0.1, (batch_size, seq_len, hidden_size)).astype('float64') + -0.1, 0.1, (batch_size, seq_len, hidden_size) + ).astype('float64') np_pre_hidden = np.random.uniform( - -0.1, 0.1, - (num_layers, batch_size, hidden_size)).astype('float64') + -0.1, 0.1, (num_layers, batch_size, hidden_size) + ).astype('float64') np_pre_cell = np.random.uniform( - -0.1, 0.1, - (num_layers, batch_size, hidden_size)).astype('float64') + -0.1, 0.1, (num_layers, batch_size, hidden_size) + ).astype('float64') def test_input_Variable(): - LSTM(np_input, pre_hidden, pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + LSTM( + np_input, + pre_hidden, + pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_input_Variable) def test_pre_hidden_Variable(): - LSTM(np_input, np_pre_hidden, pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + LSTM( + np_input, + np_pre_hidden, + pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_pre_hidden_Variable) def test_pre_cell_Variable(): - LSTM(np_input, pre_hidden, np_pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + LSTM( + np_input, + pre_hidden, + np_pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_pre_cell_Variable) def test_input_type(): - error_input = fluid.data(name='error_input', - shape=[None, hidden_size * 3], - dtype='int32') - LSTM(error_input, pre_hidden, pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + error_input = fluid.data( + name='error_input', + shape=[None, hidden_size * 3], + dtype='int32', + ) + LSTM( + error_input, + pre_hidden, + pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[None, hidden_size], - dtype='int32') - LSTM(input, error_pre_hidden, pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[None, hidden_size], + dtype='int32', + ) + LSTM( + input, + error_pre_hidden, + pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_pre_hidden_type) def test_pre_cell_type(): - error_pre_cell = fluid.data(name='error_pre_cell', - shape=[None, hidden_size], - dtype='int32') - LSTM(input, pre_hidden, error_pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + error_pre_cell = fluid.data( + name='error_pre_cell', + shape=[None, hidden_size], + dtype='int32', + ) + LSTM( + input, + pre_hidden, + error_pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_pre_cell_type) class TestLstmOp(OpTest): - def set_is_test(self): self.is_test = False @@ -247,11 +293,21 @@ class TestLstmOp(OpTest): else: b = np.random.normal(size=(1, 4 * self.D)).astype('float64') - w_b = b[:, 0:4 * self.D] - w_c = b[:, 4 * self.D:] if self.use_peepholes else None - h, c = lstm(x, self.lod, h0, c0, w, w_b, w_c, self.is_reverse, - ACTIVATION[self.act_gate], ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + w_b = b[:, 0 : 4 * self.D] + w_c = b[:, 4 * self.D :] if self.use_peepholes else None + h, c = lstm( + x, + self.lod, + h0, + c0, + w, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = {'Input': (x, self.lod), 'Weight': w} @@ -271,7 +327,7 @@ class TestLstmOp(OpTest): 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, 'candidate_activation': self.act_cand, - 'is_test': self.is_test + 'is_test': self.is_test, } def test_check_output(self): @@ -281,33 +337,33 @@ class TestLstmOp(OpTest): # TODO(qingqing) remove folowing lines after the check_grad is refined. N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'Bias'], ['Hidden'], - max_relative_error=5e-4, - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'Bias'], + ['Hidden'], + max_relative_error=5e-4, + check_dygraph=False, + ) class TestLstmOpCase1(TestLstmOp): - def set_lod(self): self.lod = [[0, 3, 2]] class TestLstmOpCase2(TestLstmOp): - def set_lod(self): self.lod = [[0, 3, 0]] class TestLstmOpCase3(TestLstmOp): - def set_lod(self): self.lod = [[2, 0, 4]] class TestLstmOpInference(TestLstmOp): - def set_is_test(self): self.is_test = True @@ -317,43 +373,42 @@ class TestLstmOpInference(TestLstmOp): class TestLstmOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): input_data = np.random.random((1, 2048)).astype("float32") - fluid.layers.dynamic_lstm(input=input_data, - size=2048, - use_peepholes=False) + fluid.layers.dynamic_lstm( + input=input_data, size=2048, use_peepholes=False + ) self.assertRaises(TypeError, test_Variable) def test_h_0(): - in_data = fluid.data(name="input", - shape=[None, 2048], - dtype="float32") + in_data = fluid.data( + name="input", shape=[None, 2048], dtype="float32" + ) h = fluid.data(name="h", shape=[None, 512], dtype="int32") c = fluid.data(name="c", shape=[None, 512], dtype="float32") - fluid.layers.dynamic_lstm(input=in_data, - size=2048, - use_peepholes=False, - h_0=h, - c_0=c) + fluid.layers.dynamic_lstm( + input=in_data, size=2048, use_peepholes=False, h_0=h, c_0=c + ) self.assertRaises(TypeError, test_h_0) def test_c_0(): - in_data_ = fluid.data(name="input_", - shape=[None, 2048], - dtype="float32") + in_data_ = fluid.data( + name="input_", shape=[None, 2048], dtype="float32" + ) h_ = fluid.data(name="h_", shape=[None, 512], dtype="float32") c_ = fluid.data(name="c_", shape=[None, 512], dtype="int32") - fluid.layers.dynamic_lstm(input=in_data_, - size=2048, - use_peepholes=False, - h_0=h_, - c_0=c_) + fluid.layers.dynamic_lstm( + input=in_data_, + size=2048, + use_peepholes=False, + h_0=h_, + c_0=c_, + ) self.assertRaises(TypeError, test_c_0) diff --git a/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py b/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py index b4e6690117d..d46e8e4719e 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py @@ -21,35 +21,39 @@ from paddle.fluid.framework import program_guard, Program def sigmoid_np(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) def tanh_np(x): - return 2 * sigmoid_np(2. * x) - 1. + return 2 * sigmoid_np(2.0 * x) - 1.0 class LstmUnitTestError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size, dict_dim, emb_dim, hidden_dim = 32, 128, 64, 512 - data = fluid.data(name='step_data', - shape=[batch_size], - dtype='int64') + data = fluid.data( + name='step_data', shape=[batch_size], dtype='int64' + ) inputs = fluid.embedding(input=data, size=[dict_dim, emb_dim]) - pre_hidden = fluid.data(name='pre_hidden', - shape=[batch_size, hidden_dim], - dtype='float32') - pre_cell = fluid.data(name='pre_cell', - shape=[batch_size, hidden_dim], - dtype='float32') + pre_hidden = fluid.data( + name='pre_hidden', + shape=[batch_size, hidden_dim], + dtype='float32', + ) + pre_cell = fluid.data( + name='pre_cell', shape=[batch_size, hidden_dim], dtype='float32' + ) np_input = np.random.uniform( - -0.1, 0.1, (batch_size, emb_dim)).astype('float64') + -0.1, 0.1, (batch_size, emb_dim) + ).astype('float64') np_pre_hidden = np.random.uniform( - -0.1, 0.1, (batch_size, hidden_dim)).astype('float64') + -0.1, 0.1, (batch_size, hidden_dim) + ).astype('float64') np_pre_cell = np.random.uniform( - -0.1, 0.1, (batch_size, hidden_dim)).astype('float64') + -0.1, 0.1, (batch_size, hidden_dim) + ).astype('float64') def test_input_Variable(): lstm_unit(np_input, pre_hidden, pre_cell) @@ -67,42 +71,48 @@ class LstmUnitTestError(unittest.TestCase): self.assertRaises(TypeError, test_pre_cell_Variable) def test_input_type(): - error_input = fluid.data(name='error_input', - shape=[batch_size, emb_dim], - dtype='int32') + error_input = fluid.data( + name='error_input', + shape=[batch_size, emb_dim], + dtype='int32', + ) lstm_unit(error_input, pre_hidden, pre_cell) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[batch_size, hidden_dim], - dtype='int32') + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[batch_size, hidden_dim], + dtype='int32', + ) lstm_unit(inputs, error_pre_hidden, pre_cell) self.assertRaises(TypeError, test_pre_hidden_type) def test_pre_cell_type(): - error_pre_cell = fluid.data(name='error_pre_cell', - shape=[batch_size, hidden_dim], - dtype='int32') + error_pre_cell = fluid.data( + name='error_pre_cell', + shape=[batch_size, hidden_dim], + dtype='int32', + ) lstm_unit(inputs, pre_hidden, error_pre_cell) self.assertRaises(TypeError, test_pre_cell_type) class LstmUnitTest(OpTest): - def setUp(self): self.op_type = "lstm_unit" x_np = np.random.normal(size=(15, 160)).astype("float64") c_np = np.random.normal(size=(15, 40)).astype("float64") i_np, f_np, o_np, j_np = np.split(x_np, 4, axis=1) - forget_bias_np = 0. - self.attrs = {'forget_bias': 0.} + forget_bias_np = 0.0 + self.attrs = {'forget_bias': 0.0} new_c = c_np * sigmoid_np(f_np + forget_bias_np) + sigmoid_np( - i_np) * tanh_np(j_np) + i_np + ) * tanh_np(j_np) new_h = tanh_np(new_c) * sigmoid_np(o_np) self.inputs = {'X': x_np, 'C_prev': c_np} diff --git a/python/paddle/fluid/tests/unittests/test_lstmp_op.py b/python/paddle/fluid/tests/unittests/test_lstmp_op.py index 721a61a7e7d..730af353dce 100644 --- a/python/paddle/fluid/tests/unittests/test_lstmp_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstmp_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -22,30 +22,42 @@ ACTIVATION = { 'identity': LstmTest.identity, 'sigmoid': LstmTest.sigmoid, 'tanh': LstmTest.tanh, - 'relu': LstmTest.relu + 'relu': LstmTest.relu, } # LSTM with recurrent projection Layer def lstmp( - input, # T x 4D - lod, # 1 x N - h0=None, # N x D - c0=None, # N x D - w_r=None, # P x 4D - w_rh=None, # D x P - w_b=None, # 1 x 4D - w_c=None, # 1 x 3D - is_reverse=False, - proj_clip=0.0, - cell_clip=0.0, - act_gate=None, - act_cell=None, - act_cand=None, - act_proj=None): - - def _step(x, w_r, w_rh, w_c, r_pre, c_pre, proj_clip, cell_clip, act_gate, - act_cell, act_cand, act_proj): + input, # T x 4D + lod, # 1 x N + h0=None, # N x D + c0=None, # N x D + w_r=None, # P x 4D + w_rh=None, # D x P + w_b=None, # 1 x 4D + w_c=None, # 1 x 3D + is_reverse=False, + proj_clip=0.0, + cell_clip=0.0, + act_gate=None, + act_cell=None, + act_cand=None, + act_proj=None, +): + def _step( + x, + w_r, + w_rh, + w_c, + r_pre, + c_pre, + proj_clip, + cell_clip, + act_gate, + act_cell, + act_cand, + act_proj, + ): g = np.dot(r_pre, w_r) # 1 x 4D g = g + x g = np.reshape(g, (1, g.size)) @@ -103,14 +115,25 @@ def lstmp( for i in range(batch_size): # compute one sequence seq_len = lod[0][i] - x = input[offset[i]:offset[i + 1], :] + x = input[offset[i] : offset[i + 1], :] r_pre = h0[i] c_pre = c0[i] # 1 x D for j in range(seq_len): # compute one step - r_pre, c_pre = _step(x[j], w_r, w_rh, w_c, r_pre, c_pre, proj_clip, - cell_clip, act_gate, act_cell, act_cand, - act_proj) + r_pre, c_pre = _step( + x[j], + w_r, + w_rh, + w_c, + r_pre, + c_pre, + proj_clip, + cell_clip, + act_gate, + act_cell, + act_cand, + act_proj, + ) projection.append(r_pre.flatten()) cell.append(c_pre.flatten()) @@ -126,7 +149,6 @@ def lstmp( class TestLstmpOp(LstmTest.TestLstmOp): - def reset_argument(self): pass @@ -154,15 +176,28 @@ class TestLstmpOp(LstmTest.TestLstmOp): else: b = np.random.normal(size=(1, 4 * self.D)).astype('float64') - w_b = b[:, 0:4 * self.D] - w_c = b[:, 4 * self.D:] if self.use_peepholes else None + w_b = b[:, 0 : 4 * self.D] + w_c = b[:, 4 * self.D :] if self.use_peepholes else None w_rh = np.random.normal(size=(self.D, self.P)).astype('float64') proj_clip = 0.1 cell_clip = 0.1 - r, c = lstmp(x, self.lod, h0, c0, w, w_rh, w_b, w_c, self.is_reverse, - proj_clip, cell_clip, ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], ACTIVATION[self.act_cand], - ACTIVATION[self.act_proj]) + r, c = lstmp( + x, + self.lod, + h0, + c0, + w, + w_rh, + w_b, + w_c, + self.is_reverse, + proj_clip, + cell_clip, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ACTIVATION[self.act_proj], + ) self.inputs = {'Input': (x, self.lod), 'Weight': w, 'ProjWeight': w_rh} @@ -184,7 +219,7 @@ class TestLstmpOp(LstmTest.TestLstmOp): 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, 'candidate_activation': self.act_cand, - 'proj_activation': self.act_proj + 'proj_activation': self.act_proj, } def test_check_output(self): @@ -195,16 +230,18 @@ class TestLstmpOp(LstmTest.TestLstmOp): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'ProjWeight', 'Bias'], - ['Projection'], - numeric_grad_delta=0.0000005, - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'ProjWeight', 'Bias'], + ['Projection'], + numeric_grad_delta=0.0000005, + check_dygraph=False, + ) class TestLstmpOpHasInitial(TestLstmpOp): - def reset_argument(self): self.has_initial_state = True @@ -213,162 +250,187 @@ class TestLstmpOpHasInitial(TestLstmpOp): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'ProjWeight', 'Bias', 'H0', 'C0'], - ['Projection'], - numeric_grad_delta=0.0000005, - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'ProjWeight', 'Bias', 'H0', 'C0'], + ['Projection'], + numeric_grad_delta=0.0000005, + check_dygraph=False, + ) def test_check_grad_ingore_bias(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'ProjWeight', 'Weight'], ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('Bias'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'ProjWeight', 'Weight'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('Bias'), + check_dygraph=False, + ) def test_check_grad_ingore_weight(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'ProjWeight', 'Bias'], ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('Weight'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'ProjWeight', 'Bias'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('Weight'), + check_dygraph=False, + ) def test_check_grad_ingore_proj_weight(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'Bias'], ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('ProjWeight'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'Bias'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('ProjWeight'), + check_dygraph=False, + ) def test_check_grad_ingore_input(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Weight', 'ProjWeight', 'Bias'], ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('Input'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Weight', 'ProjWeight', 'Bias'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('Input'), + check_dygraph=False, + ) def test_check_grad_ingore_h0(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'ProjWeight', 'Bias', 'C0'], - ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('H0'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'ProjWeight', 'Bias', 'C0'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('H0'), + check_dygraph=False, + ) def test_check_grad_ingore_c0(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'ProjWeight', 'Bias', 'H0'], - ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('C0'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'ProjWeight', 'Bias', 'H0'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('C0'), + check_dygraph=False, + ) class TestLstmpOpRerverse(TestLstmpOp): - def reset_argument(self): self.is_reverse = True class TestLstmpOpNotUsePeepholes(TestLstmpOp): - def reset_argument(self): self.use_peepholes = False class TestLstmpOpLinearProjection(TestLstmpOp): - def reset_argument(self): self.act_proj = 'identity' class TestLstmpOpLen0Case1(TestLstmpOp): - def reset_argument(self): self.lod = [[0, 4, 0]] class TestLstmpOpLen0Case2(TestLstmpOp): - def reset_argument(self): self.lod = [[2, 0, 3]] class TestLstmpOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): input_data = np.random.random((1, 2048)).astype("float32") - fluid.layers.dynamic_lstmp(input=input_data, - size=2048, - proj_size=256, - use_peepholes=False, - is_reverse=True, - cell_activation="tanh", - proj_activation="tanh") + fluid.layers.dynamic_lstmp( + input=input_data, + size=2048, + proj_size=256, + use_peepholes=False, + is_reverse=True, + cell_activation="tanh", + proj_activation="tanh", + ) self.assertRaises(TypeError, test_Variable) def test_h_0(): - in_data = fluid.data(name="input", - shape=[None, 2048], - dtype="float32") + in_data = fluid.data( + name="input", shape=[None, 2048], dtype="float32" + ) h = fluid.data(name="h", shape=[None, 512], dtype="int32") c = fluid.data(name="c", shape=[None, 512], dtype="float32") - fluid.layers.dynamic_lstmp(input=in_data, - size=2048, - proj_size=256, - use_peepholes=False, - is_reverse=True, - cell_activation="tanh", - proj_activation="tanh", - h_0=h, - c_0=c) + fluid.layers.dynamic_lstmp( + input=in_data, + size=2048, + proj_size=256, + use_peepholes=False, + is_reverse=True, + cell_activation="tanh", + proj_activation="tanh", + h_0=h, + c_0=c, + ) self.assertRaises(TypeError, test_h_0) def test_c_0(): - in_data_ = fluid.data(name="input_", - shape=[None, 2048], - dtype="float32") + in_data_ = fluid.data( + name="input_", shape=[None, 2048], dtype="float32" + ) h_ = fluid.data(name="h_", shape=[None, 512], dtype="float32") c_ = fluid.data(name="c_", shape=[None, 512], dtype="int32") - fluid.layers.dynamic_lstmp(input=in_data_, - size=2048, - proj_size=256, - use_peepholes=False, - is_reverse=True, - cell_activation="tanh", - proj_activation="tanh", - h_0=h_, - c_0=c_) + fluid.layers.dynamic_lstmp( + input=in_data_, + size=2048, + proj_size=256, + use_peepholes=False, + is_reverse=True, + cell_activation="tanh", + proj_activation="tanh", + h_0=h_, + c_0=c_, + ) self.assertRaises(TypeError, test_c_0) diff --git a/python/paddle/fluid/tests/unittests/test_lu_op.py b/python/paddle/fluid/tests/unittests/test_lu_op.py index 0c24bcd6865..d76fdf16ad9 100644 --- a/python/paddle/fluid/tests/unittests/test_lu_op.py +++ b/python/paddle/fluid/tests/unittests/test_lu_op.py @@ -43,8 +43,11 @@ def scipy_lu(A, pivot): PP.append(P) PL.append(L) PU.append(U) - return np.array(PP).reshape(preshape + pshape), np.array(PL).reshape( - preshape + lshape), np.array(PU).reshape(preshape + ushape) + return ( + np.array(PP).reshape(preshape + pshape), + np.array(PL).reshape(preshape + lshape), + np.array(PU).reshape(preshape + ushape), + ) def Pmat_to_perm(Pmat_org, cut): @@ -66,9 +69,15 @@ def Pmat_to_perm(Pmat_org, cut): sP[idx, :] = tmp permmat.append(permlst) - Pivot = np.array(permmat).reshape(list(shape[:-2]) + [ - rows, - ]) + 1 + Pivot = ( + np.array(permmat).reshape( + list(shape[:-2]) + + [ + rows, + ] + ) + + 1 + ) return Pivot[..., :cut] @@ -112,17 +121,20 @@ class TestLUOp(OpTest): ushape = np.array(sU.shape) lpad = (len(sL.shape) - 2) * [(0, 0)] + list( - ((0, (ashape - lshape)[-2]), (0, (ashape - lshape)[-1]))) + ((0, (ashape - lshape)[-2]), (0, (ashape - lshape)[-1])) + ) upad = (len(sU.shape) - 2) * [(0, 0)] + list( - ((0, (ashape - ushape)[-2]), (0, (ashape - ushape)[-1]))) + ((0, (ashape - ushape)[-2]), (0, (ashape - ushape)[-1])) + ) NsL = np.pad(sL, lpad) NsU = np.pad(sU, upad) NLU = NsL + NsU self.output = NLU self.Pivots = Pmat_to_perm(sP, min(ashape[-2], ashape[-1])) - self.Infos = np.zeros( - self.x_shape[:-2]) if len(X.shape) > 2 else np.array([0]) + self.Infos = ( + np.zeros(self.x_shape[:-2]) if len(X.shape) > 2 else np.array([0]) + ) def setUp(self): self.op_type = "lu" @@ -136,7 +148,7 @@ class TestLUOp(OpTest): self.outputs = { 'Out': self.output, 'Pivots': self.Pivots, - 'Infos': self.Infos + 'Infos': self.Infos, } def test_check_output(self): @@ -173,9 +185,7 @@ class TestLUOp3(TestLUOp): class TestLUAPI(unittest.TestCase): - def test_dygraph(self): - def run_lu_dygraph(shape, dtype): if dtype == "float32": np_dtype = np.float32 @@ -217,7 +227,7 @@ class TestLUAPI(unittest.TestCase): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] dtypes = ["float32", "float64"] for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): @@ -251,28 +261,29 @@ class TestLUAPI(unittest.TestCase): ushape = np.array(sU.shape) lpad = (len(sL.shape) - 2) * [(0, 0)] + list( - ((0, (ashape - lshape)[-2]), (0, - (ashape - lshape)[-1]))) + ((0, (ashape - lshape)[-2]), (0, (ashape - lshape)[-1])) + ) upad = (len(sU.shape) - 2) * [(0, 0)] + list( - ((0, (ashape - ushape)[-2]), (0, - (ashape - ushape)[-1]))) + ((0, (ashape - ushape)[-2]), (0, (ashape - ushape)[-1])) + ) NsL = np.pad(sL, lpad) NsU = np.pad(sU, upad) NLU = NsL + NsU - x = paddle.fluid.data(name="input", - shape=shape, - dtype=dtype) + x = paddle.fluid.data( + name="input", shape=shape, dtype=dtype + ) lu, p = paddle.linalg.lu(x, pivot=pivot) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[lu, p]) - np.testing.assert_allclose(fetches[0], - NLU, - rtol=1e-05, - atol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[lu, p], + ) + np.testing.assert_allclose( + fetches[0], NLU, rtol=1e-05, atol=1e-05 + ) tensor_shapes = [ (3, 5), @@ -283,7 +294,7 @@ class TestLUAPI(unittest.TestCase): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] dtypes = ["float32", "float64"] for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): diff --git a/python/paddle/fluid/tests/unittests/test_lu_unpack_op.py b/python/paddle/fluid/tests/unittests/test_lu_unpack_op.py index 1ec36680fa4..0c00fbea820 100644 --- a/python/paddle/fluid/tests/unittests/test_lu_unpack_op.py +++ b/python/paddle/fluid/tests/unittests/test_lu_unpack_op.py @@ -48,9 +48,11 @@ def scipy_lu_unpack(A): Llst.append(L) Ulst.append(U) - return np.array(Plst).reshape(preshape + pshape), np.array( - Llst).reshape(preshape + lshape), np.array(Ulst).reshape(preshape + - ushape) + return ( + np.array(Plst).reshape(preshape + pshape), + np.array(Llst).reshape(preshape + lshape), + np.array(Ulst).reshape(preshape + ushape), + ) def Pmat_to_perm(Pmat_org, cut): @@ -72,9 +74,15 @@ def Pmat_to_perm(Pmat_org, cut): sP[idx, :] = tmp permmat.append(permlst) - Pivot = np.array(permmat).reshape(list(shape[:-2]) + [ - rows, - ]) + 1 + Pivot = ( + np.array(permmat).reshape( + list(shape[:-2]) + + [ + rows, + ] + ) + + 1 + ) return Pivot[..., :cut] @@ -132,21 +140,23 @@ class TestLU_UnpackOp(OpTest): place = fluid.CPUPlace() if core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) - xv = paddle.fluid.data(name="input", - shape=self.x_shape, - dtype=self.dtype) + xv = paddle.fluid.data( + name="input", shape=self.x_shape, dtype=self.dtype + ) lu, p = paddle.linalg.lu(xv) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": x}, - fetch_list=[lu, p]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": x}, + fetch_list=[lu, p], + ) lu, pivots = fetches[0], fetches[1] self.inputs = {'X': lu, 'Pivots': pivots} self.attrs = { 'unpack_ludata': self.unpack_ludata, - 'unpack_pivots': self.unpack_pivots + 'unpack_pivots': self.unpack_pivots, } self.set_output(x) self.outputs = { @@ -189,12 +199,10 @@ class TestLU_UnpackOp3(TestLU_UnpackOp): class TestLU_UnpackAPI(unittest.TestCase): - def setUp(self): np.random.seed(2022) def test_dygraph(self): - def run_lu_unpack_dygraph(shape, dtype): if dtype == "float32": np_dtype = np.float32 @@ -229,7 +237,7 @@ class TestLU_UnpackAPI(unittest.TestCase): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] dtypes = ["float32", "float64"] for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): @@ -255,27 +263,26 @@ class TestLU_UnpackAPI(unittest.TestCase): with fluid.program_guard(fluid.Program(), fluid.Program()): sP, sL, sU = scipy_lu_unpack(a) - x = paddle.fluid.data(name="input", - shape=shape, - dtype=dtype) + x = paddle.fluid.data( + name="input", shape=shape, dtype=dtype + ) lu, p = paddle.linalg.lu(x) pP, pL, pU = paddle.linalg.lu_unpack(lu, p) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[pP, pL, pU]) - np.testing.assert_allclose(fetches[0], - sP, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(fetches[1], - sL, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(fetches[2], - sU, - rtol=1e-05, - atol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[pP, pL, pU], + ) + np.testing.assert_allclose( + fetches[0], sP, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + fetches[1], sL, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + fetches[2], sU, rtol=1e-05, atol=1e-05 + ) tensor_shapes = [ (3, 5), @@ -286,7 +293,7 @@ class TestLU_UnpackAPI(unittest.TestCase): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] dtypes = ["float32", "float64"] for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): diff --git a/python/paddle/fluid/tests/unittests/test_manual_seed.py b/python/paddle/fluid/tests/unittests/test_manual_seed.py index 7d9762c788a..ac697a49dbd 100644 --- a/python/paddle/fluid/tests/unittests/test_manual_seed.py +++ b/python/paddle/fluid/tests/unittests/test_manual_seed.py @@ -20,7 +20,6 @@ import numpy as np class TestManualSeed(unittest.TestCase): - def test_seed(self): fluid.enable_dygraph() diff --git a/python/paddle/fluid/tests/unittests/test_margin_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_margin_cross_entropy_op.py index 6473ab7568f..d8a5f653a9a 100644 --- a/python/paddle/fluid/tests/unittests/test_margin_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_margin_cross_entropy_op.py @@ -21,20 +21,15 @@ from paddle.fluid import Program, program_guard def stable_softmax_comm(x): - shiftx = (x - np.max(x)) + shiftx = x - np.max(x) deno = np.log(np.sum(np.exp(shiftx))) comm = shiftx - deno return comm -def margin_cross_entropy(logits, - label, - axis, - margin1, - margin2, - margin3, - scale, - reduction=None): +def margin_cross_entropy( + logits, label, axis, margin1, margin2, margin3, scale, reduction=None +): one_hot_label = np.zeros_like(logits, dtype=logits.dtype) for i, lb in enumerate(label): one_hot_label[i, lb] = 1.0 @@ -61,16 +56,18 @@ def margin_cross_entropy(logits, return loss, softmax -def python_api(logits, - label, - return_softmax=False, - ring_id=0, - rank=0, - nrank=0, - margin1=1.0, - margin2=0.5, - margin3=0.0, - scale=64.0): +def python_api( + logits, + label, + return_softmax=False, + ring_id=0, + rank=0, + nrank=0, + margin1=1.0, + margin2=0.5, + margin3=0.0, + scale=64.0, +): return paddle.nn.functional.margin_cross_entropy( logits, label, @@ -80,13 +77,14 @@ def python_api(logits, margin3=margin3, scale=scale, group=None, - reduction=None) + reduction=None, + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOp(OpTest): - def initParams(self): self.python_api = python_api self.op_type = "margin_cross_entropy" @@ -111,26 +109,35 @@ class TestMarginCrossEntropyOp(OpTest): self.init_dtype() datas = np.random.uniform( - -0.99, 0.99, [self.batch_dim, self.feat_dim]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.feat_dim] + ).astype(self.dtype) datas = datas / np.sqrt(np.sum(np.square(datas), axis=1, keepdims=True)) weights = np.random.uniform( - -0.99, 0.99, [self.feat_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.feat_dim, self.num_class] + ).astype(self.dtype) weights = weights / np.sqrt( - np.sum(np.square(weights), axis=0, keepdims=True)) + np.sum(np.square(weights), axis=0, keepdims=True) + ) logits = np.matmul(datas, weights) - labels = np.random.randint(0, - self.num_class, (self.batch_dim, ), - dtype="int64") + labels = np.random.randint( + 0, self.num_class, (self.batch_dim,), dtype="int64" + ) - loss, softmax = margin_cross_entropy(logits, labels, self.axis, - self.margin1, self.margin2, - self.margin3, self.scale) + loss, softmax = margin_cross_entropy( + logits, + labels, + self.axis, + self.margin1, + self.margin2, + self.margin3, + self.scale, + ) self.inputs = {"Logits": logits, "Label": labels} self.outputs = { "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { 'margin1': self.margin1, @@ -140,55 +147,61 @@ class TestMarginCrossEntropyOp(OpTest): } def test_check_output(self): - self.check_output_with_place(core.CUDAPlace(0), - atol=1e-5, - check_eager=True) + self.check_output_with_place( + core.CUDAPlace(0), atol=1e-5, check_eager=True + ) def test_check_grad(self): - self.check_grad_with_place(core.CUDAPlace(0), ["Logits"], - "Loss", - check_eager=True) + self.check_grad_with_place( + core.CUDAPlace(0), ["Logits"], "Loss", check_eager=True + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpFP32(TestMarginCrossEntropyOp): - def init_dtype(self): self.dtype = np.float32 def test_check_grad(self): - self.check_grad_with_place(core.CUDAPlace(0), ["Logits"], - "Loss", - numeric_grad_delta=5e-2, - max_relative_error=5e-2, - check_eager=True) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + self.check_grad_with_place( + core.CUDAPlace(0), + ["Logits"], + "Loss", + numeric_grad_delta=5e-2, + max_relative_error=5e-2, + check_eager=True, + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpFP16(TestMarginCrossEntropyOp): - def init_dtype(self): self.dtype = np.float16 def test_check_output(self): - self.check_output_with_place(core.CUDAPlace(0), - atol=5e-2, - check_eager=True) + self.check_output_with_place( + core.CUDAPlace(0), atol=5e-2, check_eager=True + ) def test_check_grad(self): - self.check_grad_with_place(core.CUDAPlace(0), ["Logits"], - "Loss", - numeric_grad_delta=6e-1, - max_relative_error=6e-1, - check_eager=True) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + self.check_grad_with_place( + core.CUDAPlace(0), + ["Logits"], + "Loss", + numeric_grad_delta=6e-1, + max_relative_error=6e-1, + check_eager=True, + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpCosFace(TestMarginCrossEntropyOp): - def init_loss_params(self): self.margin1 = 1.0 self.margin2 = 0.0 @@ -196,10 +209,10 @@ class TestMarginCrossEntropyOpCosFace(TestMarginCrossEntropyOp): self.scale = 2.0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpSphereFace(TestMarginCrossEntropyOp): - def init_loss_params(self): self.margin1 = 1.35 self.margin2 = 0.0 @@ -208,28 +221,27 @@ class TestMarginCrossEntropyOpSphereFace(TestMarginCrossEntropyOp): class TestMarginCrossEntropyOpCPU(TestMarginCrossEntropyOp): - def test_check_output(self): try: - self.check_output_with_place(core.CPUPlace(), - atol=1e-5, - check_eager=True) + self.check_output_with_place( + core.CPUPlace(), atol=1e-5, check_eager=True + ) except RuntimeError: pass def test_check_grad(self): try: - self.check_grad_with_place(core.CPUPlace(), ["Logits"], - "Loss", - check_eager=True) + self.check_grad_with_place( + core.CPUPlace(), ["Logits"], "Loss", check_eager=True + ) except RuntimeError: pass -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpV2(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -268,31 +280,42 @@ class TestMarginCrossEntropyOpV2(unittest.TestCase): def check_static_result(self, place): with program_guard(Program(), Program()): datas = np.random.uniform( - -0.99, 0.99, [self.batch_dim, self.feat_dim]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.feat_dim] + ).astype(self.dtype) datas = datas / np.sqrt( - np.sum(np.square(datas), axis=1, keepdims=True)) + np.sum(np.square(datas), axis=1, keepdims=True) + ) weights = np.random.uniform( - -0.99, 0.99, [self.feat_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.feat_dim, self.num_class] + ).astype(self.dtype) weights = weights / np.sqrt( - np.sum(np.square(weights), axis=0, keepdims=True)) + np.sum(np.square(weights), axis=0, keepdims=True) + ) logits_np = np.matmul(datas, weights) - labels_np = np.random.randint(0, - self.num_class, (self.batch_dim, ), - dtype="int64") - - loss_np, softmax_np = margin_cross_entropy(logits_np, labels_np, - self.axis, self.margin1, - self.margin2, - self.margin3, self.scale, - self.reduction) - - logits = paddle.static.data(name='logits', - shape=[self.batch_dim, self.num_class], - dtype=self.dtype) - label = paddle.static.data(name='label', - shape=[self.batch_dim], - dtype="int64") + labels_np = np.random.randint( + 0, self.num_class, (self.batch_dim,), dtype="int64" + ) + + loss_np, softmax_np = margin_cross_entropy( + logits_np, + labels_np, + self.axis, + self.margin1, + self.margin2, + self.margin3, + self.scale, + self.reduction, + ) + + logits = paddle.static.data( + name='logits', + shape=[self.batch_dim, self.num_class], + dtype=self.dtype, + ) + label = paddle.static.data( + name='label', shape=[self.batch_dim], dtype="int64" + ) loss, softmax = paddle.nn.functional.margin_cross_entropy( logits, label, @@ -301,16 +324,15 @@ class TestMarginCrossEntropyOpV2(unittest.TestCase): margin3=self.margin3, scale=self.scale, return_softmax=True, - reduction=self.reduction) + reduction=self.reduction, + ) exe = paddle.fluid.Executor(place) - [loss_res, - softmax_res] = exe.run(paddle.fluid.default_main_program(), - feed={ - 'logits': logits_np, - 'label': labels_np - }, - fetch_list=[loss, softmax]) + [loss_res, softmax_res] = exe.run( + paddle.fluid.default_main_program(), + feed={'logits': logits_np, 'label': labels_np}, + fetch_list=[loss, softmax], + ) np.testing.assert_allclose(loss_res, loss_np) np.testing.assert_allclose(softmax_res, softmax_np) @@ -321,24 +343,33 @@ class TestMarginCrossEntropyOpV2(unittest.TestCase): def check_dynamic_result(self, place): with paddle.fluid.dygraph.guard(place): datas = np.random.uniform( - -0.99, 0.99, [self.batch_dim, self.feat_dim]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.feat_dim] + ).astype(self.dtype) datas = datas / np.sqrt( - np.sum(np.square(datas), axis=1, keepdims=True)) + np.sum(np.square(datas), axis=1, keepdims=True) + ) weights = np.random.uniform( - -0.99, 0.99, [self.feat_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.feat_dim, self.num_class] + ).astype(self.dtype) weights = weights / np.sqrt( - np.sum(np.square(weights), axis=0, keepdims=True)) + np.sum(np.square(weights), axis=0, keepdims=True) + ) logits_np = np.matmul(datas, weights) - labels_np = np.random.randint(0, - self.num_class, (self.batch_dim, ), - dtype="int64") - - loss_np, softmax_np = margin_cross_entropy(logits_np, labels_np, - self.axis, self.margin1, - self.margin2, - self.margin3, self.scale, - self.reduction) + labels_np = np.random.randint( + 0, self.num_class, (self.batch_dim,), dtype="int64" + ) + + loss_np, softmax_np = margin_cross_entropy( + logits_np, + labels_np, + self.axis, + self.margin1, + self.margin2, + self.margin3, + self.scale, + self.reduction, + ) logits = paddle.to_tensor(logits_np, dtype=self.dtype) labels = paddle.to_tensor(labels_np, dtype="int64") @@ -351,7 +382,8 @@ class TestMarginCrossEntropyOpV2(unittest.TestCase): margin3=self.margin3, scale=self.scale, return_softmax=True, - reduction=self.reduction) + reduction=self.reduction, + ) loss_res = loss.numpy() softmax_res = softmax.numpy() @@ -359,26 +391,26 @@ class TestMarginCrossEntropyOpV2(unittest.TestCase): np.testing.assert_allclose(softmax_res, softmax_np) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpV3(TestMarginCrossEntropyOpV2): - def init_reduction(self): self.reduction = 'mean' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpV4(TestMarginCrossEntropyOpV2): - def init_reduction(self): self.reduction = 'sum' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpAPIError(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -408,17 +440,15 @@ class TestMarginCrossEntropyOpAPIError(unittest.TestCase): self.dtype = np.float64 def test_dynamic_errors(self): - def test_dim(): for place in self.places: with paddle.fluid.dygraph.guard(place): - labels_np = np.random.randint(0, - self.num_class, - (self.batch_dim, 2), - dtype="int64") + labels_np = np.random.randint( + 0, self.num_class, (self.batch_dim, 2), dtype="int64" + ) logits_np = np.random.uniform( - -0.99, 0.99, - [self.batch_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.num_class] + ).astype(self.dtype) labels = paddle.to_tensor(labels_np) logits = paddle.to_tensor(logits_np) @@ -430,17 +460,18 @@ class TestMarginCrossEntropyOpAPIError(unittest.TestCase): margin3=self.margin3, scale=self.scale, return_softmax=True, - reduction=None) + reduction=None, + ) def test_label_type(): for place in self.places: with paddle.fluid.dygraph.guard(place): - labels_np = np.random.uniform(0, self.num_class, - (self.batch_dim, 1)).astype( - self.dtype) + labels_np = np.random.uniform( + 0, self.num_class, (self.batch_dim, 1) + ).astype(self.dtype) logits_np = np.random.uniform( - -0.99, 0.99, - [self.batch_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.num_class] + ).astype(self.dtype) labels = paddle.to_tensor(labels_np) logits = paddle.to_tensor(logits_np) @@ -452,18 +483,18 @@ class TestMarginCrossEntropyOpAPIError(unittest.TestCase): margin3=self.margin3, scale=self.scale, return_softmax=True, - reduction=None) + reduction=None, + ) def test_group_value(): for place in self.places: with paddle.fluid.dygraph.guard(place): - labels_np = np.random.randint(0, - self.num_class, - (self.batch_dim, ), - dtype="int64") + labels_np = np.random.randint( + 0, self.num_class, (self.batch_dim,), dtype="int64" + ) logits_np = np.random.uniform( - -0.99, 0.99, - [self.batch_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.num_class] + ).astype(self.dtype) labels = paddle.to_tensor(labels_np) logits = paddle.to_tensor(logits_np) @@ -476,7 +507,8 @@ class TestMarginCrossEntropyOpAPIError(unittest.TestCase): scale=self.scale, return_softmax=True, reduction=None, - group=True) + group=True, + ) self.assertRaises(ValueError, test_dim) self.assertRaises(NotImplementedError, test_label_type) diff --git a/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py b/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py index 95919ff5329..f68b995a1ff 100644 --- a/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py @@ -19,20 +19,21 @@ from paddle import fluid class TestMarginRankLossOp(OpTest): - def setUp(self): self.op_type = "margin_rank_loss" batch_size = 5 margin = 0.5 # labels_{i} = {-1, 1} - label = 2 * np.random.randint( - 0, 2, size=(batch_size, 1)).astype("float32") - 1 + label = ( + 2 * np.random.randint(0, 2, size=(batch_size, 1)).astype("float32") + - 1 + ) x1 = np.random.random((batch_size, 1)).astype("float32") x2 = np.random.random((batch_size, 1)).astype("float32") # loss = max(0, -label * (x1 - x2) + margin) loss = -label * (x1 - x2) + margin loss = np.where(loss > 0, loss, 0) - act = np.where(loss > 0, 1., 0.) + act = np.where(loss > 0, 1.0, 0.0) self.attrs = {'margin': margin} self.inputs = {'Label': label, 'X1': x1, 'X2': x2} @@ -52,13 +53,17 @@ class TestMarginRankLossOp(OpTest): class TestMarginRankLossLayer(unittest.TestCase): - def setUp(self): self.batch_size = 5 self.margin = 0.5 # labels_{i} = {-1, 1} - self.label = 2 * np.random.randint( - 0, 2, size=(self.batch_size, 1)).astype("float32") - 1 + self.label = ( + 2 + * np.random.randint(0, 2, size=(self.batch_size, 1)).astype( + "float32" + ) + - 1 + ) self.x1 = np.random.random((self.batch_size, 1)).astype("float32") self.x2 = np.random.random((self.batch_size, 1)).astype("float32") # loss = max(0, -label * (x1 - x2) + margin) @@ -86,13 +91,11 @@ class TestMarginRankLossLayer(unittest.TestCase): exe = fluid.Executor(place) exe.run(start) - out_np, = exe.run(main, - feed={ - "label": self.label, - "x1": self.x1, - "x2": self.x2 - }, - fetch_list=[out]) + (out_np,) = exe.run( + main, + feed={"label": self.label, "x1": self.x1, "x2": self.x2}, + fetch_list=[out], + ) np.testing.assert_allclose(out_np, self.loss) diff --git a/python/paddle/fluid/tests/unittests/test_marker_op.py b/python/paddle/fluid/tests/unittests/test_marker_op.py index f0cb6ec1bbb..ed19915c41e 100644 --- a/python/paddle/fluid/tests/unittests/test_marker_op.py +++ b/python/paddle/fluid/tests/unittests/test_marker_op.py @@ -17,14 +17,13 @@ from paddle.distributed.fleet.meta_optimizers.common import OpRole class TestMarkerOp(OpTest): - def setUp(self): self.op_type = "marker" self.inputs = {} self.attrs = { 'marker_role': 'forward', 'marker_pos': 'B', - 'op_role': OpRole.Forward + 'op_role': OpRole.Forward, } self.outputs = {} diff --git a/python/paddle/fluid/tests/unittests/test_masked_select_op.py b/python/paddle/fluid/tests/unittests/test_masked_select_op.py index 4de07192dd0..7774eb23dc1 100644 --- a/python/paddle/fluid/tests/unittests/test_masked_select_op.py +++ b/python/paddle/fluid/tests/unittests/test_masked_select_op.py @@ -27,7 +27,6 @@ def np_masked_select(x, mask): class TestMaskedSelectOp(OpTest): - def setUp(self): self.init() self.op_type = "masked_select" @@ -49,19 +48,16 @@ class TestMaskedSelectOp(OpTest): class TestMaskedSelectOp1(TestMaskedSelectOp): - def init(self): self.shape = (6, 8, 9, 18) class TestMaskedSelectOp2(TestMaskedSelectOp): - def init(self): - self.shape = (168, ) + self.shape = (168,) class TestMaskedSelectAPI(unittest.TestCase): - def test_imperative_mode(self): paddle.disable_static() shape = (88, 6, 8) @@ -86,27 +82,26 @@ class TestMaskedSelectAPI(unittest.TestCase): exe = paddle.static.Executor(place=paddle.CPUPlace()) - res, = exe.run(paddle.static.default_main_program(), - feed={ - "x": np_x, - "mask": np_mask - }, - fetch_list=[out]) + (res,) = exe.run( + paddle.static.default_main_program(), + feed={"x": np_x, "mask": np_mask}, + fetch_list=[out], + ) np.testing.assert_allclose(res, np_out, rtol=1e-05) class TestMaskedSelectError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') mask = paddle.fluid.data(shape=shape, dtype='bool', name='mask') - mask_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='mask_float') + mask_float = paddle.fluid.data( + shape=shape, dtype='float32', name='mask_float' + ) np_x = np.random.random(shape).astype('float32') np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) diff --git a/python/paddle/fluid/tests/unittests/test_match_matrix_tensor_op.py b/python/paddle/fluid/tests/unittests/test_match_matrix_tensor_op.py index 56e6ab07a30..8b46676704e 100644 --- a/python/paddle/fluid/tests/unittests/test_match_matrix_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_match_matrix_tensor_op.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class TestMatchMatrixTensorOp(OpTest): - def setUp(self): self.init_op_type() self.set_data() @@ -56,8 +55,8 @@ class TestMatchMatrixTensorOp(OpTest): for idx in range(len(x_lod[0])): x_len = x_lod[0][idx] y_len = y_lod[0][idx] - x_sub = x_data[x_offset:(x_offset + x_len), :] - y_sub = y_data[y_offset:(y_offset + y_len), :] + x_sub = x_data[x_offset : (x_offset + x_len), :] + y_sub = y_data[y_offset : (y_offset + y_len), :] tmp_sub = np.dot(x_sub, w_data) tmp = np.vstack((tmp, tmp_sub.reshape(tmp_sub.size, 1))) @@ -77,7 +76,6 @@ class TestMatchMatrixTensorOp(OpTest): class TestMatchMatrixTensorOpCase1(TestMatchMatrixTensorOp): - def set_data(self): ix, iy, h, dim_t = [5, 8, 25, 4] x_lod = [[5]] @@ -86,7 +84,6 @@ class TestMatchMatrixTensorOpCase1(TestMatchMatrixTensorOp): class TestMatchMatrixTensorOpCase2(TestMatchMatrixTensorOp): - def set_data(self): ix, iy, h, dim_t = [105, 120, 1, 4] x_lod = [[30, 45, 30]] @@ -95,7 +92,6 @@ class TestMatchMatrixTensorOpCase2(TestMatchMatrixTensorOp): class TestMatchMatrixTensorOpCase3(TestMatchMatrixTensorOp): - def set_data(self): ix, iy, h, dim_t = [5, 9, 32, 1] x_lod = [[1, 2, 2]] @@ -104,7 +100,6 @@ class TestMatchMatrixTensorOpCase3(TestMatchMatrixTensorOp): class TestMatchMatrixTensorOpCase4(TestMatchMatrixTensorOp): - def set_data(self): ix, iy, h, dim_t = [8, 12, 16, 5] x_lod = [[1, 2, 3, 1, 1]] @@ -114,9 +109,9 @@ class TestMatchMatrixTensorOpCase4(TestMatchMatrixTensorOp): def test_api(self): x_lod_tensor = fluid.layers.data(name='x', shape=[10], lod_level=1) y_lod_tensor = fluid.layers.data(name='y', shape=[10], lod_level=1) - out, out_tmp = fluid.contrib.match_matrix_tensor(x=x_lod_tensor, - y=y_lod_tensor, - channel_num=3) + out, out_tmp = fluid.contrib.match_matrix_tensor( + x=x_lod_tensor, y=y_lod_tensor, channel_num=3 + ) place = fluid.CPUPlace() x_data = np.random.rand(7, 10).astype('float32') @@ -126,12 +121,9 @@ class TestMatchMatrixTensorOpCase4(TestMatchMatrixTensorOp): exe = fluid.Executor(place=place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x, - 'y': y - }, - fetch_list=[out], - return_numpy=False) + ret = exe.run( + feed={'x': x, 'y': y}, fetch_list=[out], return_numpy=False + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch.py b/python/paddle/fluid/tests/unittests/test_math_op_patch.py index 9e36c3b382f..9efdb268a4b 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch.py @@ -21,7 +21,6 @@ import numpy as np class TestMathOpPatches(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -36,9 +35,9 @@ class TestMathOpPatches(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, c_np, d_np = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b, c, d]) + b_np, c_np, d_np = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b, c, d] + ) np.testing.assert_allclose(a_np + 10, b_np, rtol=1e-05) ab_np = np.concatenate([a_np, b_np], axis=1) np.testing.assert_allclose(ab_np + 10, c_np, rtol=1e-05) @@ -52,9 +51,9 @@ class TestMathOpPatches(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + b_np = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np + 10, b_np, rtol=1e-05) @prog_scope() @@ -64,9 +63,9 @@ class TestMathOpPatches(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np - 10, b_np, rtol=1e-05) @prog_scope() @@ -76,9 +75,9 @@ class TestMathOpPatches(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(10 - a_np, b_np, rtol=1e-05) @prog_scope() @@ -88,9 +87,9 @@ class TestMathOpPatches(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np * 10, b_np, rtol=1e-05) @prog_scope() @@ -100,9 +99,9 @@ class TestMathOpPatches(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(10 * a_np, b_np, rtol=1e-05) @prog_scope() @@ -112,9 +111,9 @@ class TestMathOpPatches(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np / 10, b_np, rtol=1e-05) @prog_scope() @@ -125,9 +124,9 @@ class TestMathOpPatches(unittest.TestCase): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') + 1e-2 - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(10 / a_np, b_np, rtol=1e-05) @prog_scope() @@ -139,12 +138,11 @@ class TestMathOpPatches(unittest.TestCase): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') b_np = np.random.random(size=[10, 1]).astype('float32') + 1e-2 - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - 'b': b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, 'b': b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np / b_np, c_np, rtol=1e-05) @prog_scope() @@ -156,12 +154,11 @@ class TestMathOpPatches(unittest.TestCase): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') b_np = np.random.random(size=[10, 1]).astype('float32') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - 'b': b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, 'b': b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np * b_np, c_np, rtol=1e-05) @prog_scope() @@ -173,12 +170,11 @@ class TestMathOpPatches(unittest.TestCase): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') b_np = np.random.random(size=[10, 1]).astype('float32') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - 'b': b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, 'b': b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np + b_np, c_np, rtol=1e-05) @prog_scope() @@ -190,12 +186,11 @@ class TestMathOpPatches(unittest.TestCase): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') b_np = np.random.random(size=[10, 1]).astype('float32') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - 'b': b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, 'b': b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np - b_np, c_np, rtol=1e-05) @prog_scope() @@ -205,9 +200,9 @@ class TestMathOpPatches(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.array([3, 4, 10, 14, 9, 18]).astype('int64') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) b_np_actual = (a_np / 7).astype('float32') np.testing.assert_allclose(b_np, b_np_actual, rtol=1e-05) @@ -216,19 +211,18 @@ class TestMathOpPatches(unittest.TestCase): def test_equal(self): a = fluid.layers.data(name="a", shape=[1], dtype='float32') b = fluid.layers.data(name="b", shape=[1], dtype='float32') - c = (a == b) + c = a == b place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.array([3, 4, 10, 14, 9, 18]).astype('float32') b_np = np.array([3, 4, 11, 15, 8, 18]).astype('float32') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - "b": b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, "b": b_np}, + fetch_list=[c], + ) np.testing.assert_array_equal(c_np, a_np == b_np) self.assertEqual(c.dtype, fluid.core.VarDesc.VarType.BOOL) @@ -240,19 +234,18 @@ class TestMathOpPatches(unittest.TestCase): one = fluid.layers.ones(shape=[1], dtype='int32') zero = fluid.layers.zeros(shape=[1], dtype='int32') - cond = (one == zero) + cond = one == zero c = fluid.layers.cond(cond, lambda: a + b, lambda: a - b) place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.array([3, 4, 10, 14, 9, 18]).astype('float') b_np = np.array([3, 4, 11, 15, 8, 18]).astype('float') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - "b": b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, "b": b_np}, + fetch_list=[c], + ) np.testing.assert_array_equal(c_np, a_np - b_np) @@ -264,9 +257,9 @@ class TestMathOpPatches(unittest.TestCase): exe = fluid.Executor(place) a_np = np.random.uniform(-1, 1, size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(-a_np, b_np, rtol=1e-05) @prog_scope() @@ -277,9 +270,9 @@ class TestMathOpPatches(unittest.TestCase): exe = fluid.Executor(place) a_np = np.random.uniform(-1, 1, size=[10, 1]).astype('float64') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np.astype('float32'), b_np, rtol=1e-05) def test_bitwise_and(self): @@ -292,12 +285,11 @@ class TestMathOpPatches(unittest.TestCase): z = x & y exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": y_np}, + fetch_list=[z], + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -311,12 +303,11 @@ class TestMathOpPatches(unittest.TestCase): z = x | y exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": y_np}, + fetch_list=[z], + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -330,12 +321,11 @@ class TestMathOpPatches(unittest.TestCase): z = x ^ y exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": y_np}, + fetch_list=[z], + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -347,9 +337,9 @@ class TestMathOpPatches(unittest.TestCase): z = ~x exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={"x": x_np}, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), feed={"x": x_np}, fetch_list=[z] + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -361,9 +351,9 @@ class TestMathOpPatches(unittest.TestCase): z = x.T exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={"x": x_np}, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), feed={"x": x_np}, fetch_list=[z] + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -382,12 +372,11 @@ class TestMathOpPatches(unittest.TestCase): b_np = np.random.uniform(-1, 1, size=[3, 5]).astype('float32') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - c_np, = exe.run(paddle.static.default_main_program(), - feed={ - "a": a_np, - "b": b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + paddle.static.default_main_program(), + feed={"a": a_np, "b": b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np @ b_np, c_np, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py b/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py index 5c66d4cd6b5..5330dc8b4f8 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestMathOpPatchesVarBase(unittest.TestCase): - def setUp(self): self.shape = [10, 1024] self.dtype = np.float32 @@ -75,7 +74,7 @@ class TestMathOpPatchesVarBase(unittest.TestCase): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) res = a / b - #NOTE: Not sure why array_equal fails on windows, allclose is acceptable + # NOTE: Not sure why array_equal fails on windows, allclose is acceptable np.testing.assert_allclose(res.numpy(), a_np / b_np, rtol=1e-05) def test_div(self): @@ -244,8 +243,8 @@ class TestMathOpPatchesVarBase(unittest.TestCase): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) c = fluid.dygraph.to_variable(c_np) - res1 = (a == b) - res2 = (a == c) + res1 = a == b + res2 = a == c np.testing.assert_array_equal(res1.numpy(), a_np == b_np) np.testing.assert_array_equal(res2.numpy(), a_np == c_np) @@ -262,8 +261,8 @@ class TestMathOpPatchesVarBase(unittest.TestCase): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) c = fluid.dygraph.to_variable(c_np) - res1 = (a != b) - res2 = (a != c) + res1 = a != b + res2 = a != c np.testing.assert_array_equal(res1.numpy(), a_np != b_np) np.testing.assert_array_equal(res2.numpy(), a_np != c_np) @@ -278,7 +277,7 @@ class TestMathOpPatchesVarBase(unittest.TestCase): with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) - res = (a < b) + res = a < b np.testing.assert_array_equal(res.numpy(), a_np < b_np) def test_less_than(self): @@ -292,7 +291,7 @@ class TestMathOpPatchesVarBase(unittest.TestCase): with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) - res = (a <= b) + res = a <= b np.testing.assert_array_equal(res.numpy(), a_np <= b_np) def test_less_equal(self): @@ -306,7 +305,7 @@ class TestMathOpPatchesVarBase(unittest.TestCase): with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) - res = (a > b) + res = a > b np.testing.assert_array_equal(res.numpy(), a_np > b_np) def test_greater_than(self): @@ -320,7 +319,7 @@ class TestMathOpPatchesVarBase(unittest.TestCase): with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) - res = (a >= b) + res = a >= b np.testing.assert_array_equal(res.numpy(), a_np >= b_np) def test_greater_equal(self): @@ -386,10 +385,12 @@ class TestMathOpPatchesVarBase(unittest.TestCase): x = fluid.layers.ones((2, 2), dtype="float32") y = t * x - np.testing.assert_allclose(y.numpy(), - t * np.ones((2, 2), dtype='float32'), - rtol=1e-05, - atol=0.0) + np.testing.assert_allclose( + y.numpy(), + t * np.ones((2, 2), dtype='float32'), + rtol=1e-05, + atol=0.0, + ) def test_np_left_mul(self): with _test_eager_guard(): @@ -483,92 +484,112 @@ class TestMathOpPatchesVarBase(unittest.TestCase): np.testing.assert_array_equal(x.abs().numpy(), paddle.abs(x).numpy()) m = x.abs() np.testing.assert_array_equal(m.sqrt().numpy(), paddle.sqrt(m).numpy()) - np.testing.assert_array_equal(m.rsqrt().numpy(), - paddle.rsqrt(m).numpy()) + np.testing.assert_array_equal( + m.rsqrt().numpy(), paddle.rsqrt(m).numpy() + ) np.testing.assert_array_equal(x.ceil().numpy(), paddle.ceil(x).numpy()) - np.testing.assert_array_equal(x.floor().numpy(), - paddle.floor(x).numpy()) + np.testing.assert_array_equal( + x.floor().numpy(), paddle.floor(x).numpy() + ) np.testing.assert_array_equal(x.cos().numpy(), paddle.cos(x).numpy()) np.testing.assert_array_equal(x.acos().numpy(), paddle.acos(x).numpy()) np.testing.assert_array_equal(x.asin().numpy(), paddle.asin(x).numpy()) np.testing.assert_array_equal(x.sin().numpy(), paddle.sin(x).numpy()) np.testing.assert_array_equal(x.sinh().numpy(), paddle.sinh(x).numpy()) np.testing.assert_array_equal(x.cosh().numpy(), paddle.cosh(x).numpy()) - np.testing.assert_array_equal(x.round().numpy(), - paddle.round(x).numpy()) - np.testing.assert_array_equal(x.reciprocal().numpy(), - paddle.reciprocal(x).numpy()) - np.testing.assert_array_equal(x.square().numpy(), - paddle.square(x).numpy()) + np.testing.assert_array_equal( + x.round().numpy(), paddle.round(x).numpy() + ) + np.testing.assert_array_equal( + x.reciprocal().numpy(), paddle.reciprocal(x).numpy() + ) + np.testing.assert_array_equal( + x.square().numpy(), paddle.square(x).numpy() + ) np.testing.assert_array_equal(x.rank().numpy(), paddle.rank(x).numpy()) np.testing.assert_array_equal(x[0].t().numpy(), paddle.t(x[0]).numpy()) - np.testing.assert_array_equal(x.asinh().numpy(), - paddle.asinh(x).numpy()) + np.testing.assert_array_equal( + x.asinh().numpy(), paddle.asinh(x).numpy() + ) ### acosh(x) = nan, need to change input t_np = np.random.uniform(1, 2, [2, 3]).astype(self.dtype) t = paddle.to_tensor(t_np) - np.testing.assert_array_equal(t.acosh().numpy(), - paddle.acosh(t).numpy()) - np.testing.assert_array_equal(x.atanh().numpy(), - paddle.atanh(x).numpy()) - d = paddle.to_tensor([[1.2285208, 1.3491015, 1.4899898], - [1.30058, 1.0688717, 1.4928783], - [1.0958099, 1.3724753, 1.8926544]]) + np.testing.assert_array_equal( + t.acosh().numpy(), paddle.acosh(t).numpy() + ) + np.testing.assert_array_equal( + x.atanh().numpy(), paddle.atanh(x).numpy() + ) + d = paddle.to_tensor( + [ + [1.2285208, 1.3491015, 1.4899898], + [1.30058, 1.0688717, 1.4928783], + [1.0958099, 1.3724753, 1.8926544], + ] + ) d = d.matmul(d.t()) # ROCM not support cholesky if not fluid.core.is_compiled_with_rocm(): - np.testing.assert_array_equal(d.cholesky().numpy(), - paddle.cholesky(d).numpy()) + np.testing.assert_array_equal( + d.cholesky().numpy(), paddle.cholesky(d).numpy() + ) - np.testing.assert_array_equal(x.is_empty().numpy(), - paddle.is_empty(x).numpy()) - np.testing.assert_array_equal(x.isfinite().numpy(), - paddle.isfinite(x).numpy()) np.testing.assert_array_equal( - x.cast('int32').numpy(), - paddle.cast(x, 'int32').numpy()) + x.is_empty().numpy(), paddle.is_empty(x).numpy() + ) + np.testing.assert_array_equal( + x.isfinite().numpy(), paddle.isfinite(x).numpy() + ) np.testing.assert_array_equal( - x.expand([3, 2, 3]).numpy(), - paddle.expand(x, [3, 2, 3]).numpy()) + x.cast('int32').numpy(), paddle.cast(x, 'int32').numpy() + ) np.testing.assert_array_equal( - x.tile([2, 2]).numpy(), - paddle.tile(x, [2, 2]).numpy()) - np.testing.assert_array_equal(x.flatten().numpy(), - paddle.flatten(x).numpy()) + x.expand([3, 2, 3]).numpy(), paddle.expand(x, [3, 2, 3]).numpy() + ) + np.testing.assert_array_equal( + x.tile([2, 2]).numpy(), paddle.tile(x, [2, 2]).numpy() + ) + np.testing.assert_array_equal( + x.flatten().numpy(), paddle.flatten(x).numpy() + ) index = paddle.to_tensor([0, 1]) np.testing.assert_array_equal( - x.gather(index).numpy(), - paddle.gather(x, index).numpy()) + x.gather(index).numpy(), paddle.gather(x, index).numpy() + ) index = paddle.to_tensor([[0, 1], [1, 2]]) np.testing.assert_array_equal( - x.gather_nd(index).numpy(), - paddle.gather_nd(x, index).numpy()) + x.gather_nd(index).numpy(), paddle.gather_nd(x, index).numpy() + ) np.testing.assert_array_equal( - x.reverse([0, 1]).numpy(), - paddle.reverse(x, [0, 1]).numpy()) + x.reverse([0, 1]).numpy(), paddle.reverse(x, [0, 1]).numpy() + ) np.testing.assert_array_equal( - a.reshape([3, 2]).numpy(), - paddle.reshape(a, [3, 2]).numpy()) + a.reshape([3, 2]).numpy(), paddle.reshape(a, [3, 2]).numpy() + ) np.testing.assert_array_equal( x.slice([0, 1], [0, 0], [1, 2]).numpy(), - paddle.slice(x, [0, 1], [0, 0], [1, 2]).numpy()) + paddle.slice(x, [0, 1], [0, 0], [1, 2]).numpy(), + ) np.testing.assert_array_equal( - x.split(2)[0].numpy(), - paddle.split(x, 2)[0].numpy()) + x.split(2)[0].numpy(), paddle.split(x, 2)[0].numpy() + ) m = paddle.to_tensor( - np.random.uniform(-1, 1, [1, 6, 1, 1]).astype(self.dtype)) + np.random.uniform(-1, 1, [1, 6, 1, 1]).astype(self.dtype) + ) np.testing.assert_array_equal( - m.squeeze([]).numpy(), - paddle.squeeze(m, []).numpy()) + m.squeeze([]).numpy(), paddle.squeeze(m, []).numpy() + ) np.testing.assert_array_equal( - m.squeeze([1, 2]).numpy(), - paddle.squeeze(m, [1, 2]).numpy()) + m.squeeze([1, 2]).numpy(), paddle.squeeze(m, [1, 2]).numpy() + ) m = paddle.to_tensor([2, 3, 3, 1, 5, 3], 'float32') - np.testing.assert_array_equal(m.unique()[0].numpy(), - paddle.unique(m)[0].numpy()) + np.testing.assert_array_equal( + m.unique()[0].numpy(), paddle.unique(m)[0].numpy() + ) np.testing.assert_array_equal( m.unique(return_counts=True)[1], - paddle.unique(m, return_counts=True)[1]) + paddle.unique(m, return_counts=True)[1], + ) np.testing.assert_array_equal(x.flip([0]), paddle.flip(x, [0])) np.testing.assert_array_equal(x.unbind(0), paddle.unbind(x, 0)) np.testing.assert_array_equal(x.roll(1), paddle.roll(x, 1)) @@ -582,84 +603,86 @@ class TestMathOpPatchesVarBase(unittest.TestCase): # 2. Binary operation np.testing.assert_array_equal( - x.divide(y).numpy(), - paddle.divide(x, y).numpy()) + x.divide(y).numpy(), paddle.divide(x, y).numpy() + ) np.testing.assert_array_equal( x.matmul(y, True, False).numpy(), - paddle.matmul(x, y, True, False).numpy()) + paddle.matmul(x, y, True, False).numpy(), + ) np.testing.assert_array_equal( x.norm(p='fro', axis=[0, 1]).numpy(), - paddle.norm(x, p='fro', axis=[0, 1]).numpy()) + paddle.norm(x, p='fro', axis=[0, 1]).numpy(), + ) np.testing.assert_array_equal( - x.dist(y).numpy(), - paddle.dist(x, y).numpy()) + x.dist(y).numpy(), paddle.dist(x, y).numpy() + ) np.testing.assert_array_equal( - x.cross(y).numpy(), - paddle.cross(x, y).numpy()) + x.cross(y).numpy(), paddle.cross(x, y).numpy() + ) m = x.expand([2, 2, 3]) n = y.expand([2, 2, 3]).transpose([0, 2, 1]) np.testing.assert_array_equal( - m.bmm(n).numpy(), - paddle.bmm(m, n).numpy()) + m.bmm(n).numpy(), paddle.bmm(m, n).numpy() + ) np.testing.assert_array_equal( - x.histogram(5, -1, 1).numpy(), - paddle.histogram(x, 5, -1, 1).numpy()) + x.histogram(5, -1, 1).numpy(), paddle.histogram(x, 5, -1, 1).numpy() + ) np.testing.assert_array_equal( - x.equal(y).numpy(), - paddle.equal(x, y).numpy()) + x.equal(y).numpy(), paddle.equal(x, y).numpy() + ) np.testing.assert_array_equal( - x.greater_equal(y).numpy(), - paddle.greater_equal(x, y).numpy()) + x.greater_equal(y).numpy(), paddle.greater_equal(x, y).numpy() + ) np.testing.assert_array_equal( - x.greater_than(y).numpy(), - paddle.greater_than(x, y).numpy()) + x.greater_than(y).numpy(), paddle.greater_than(x, y).numpy() + ) np.testing.assert_array_equal( - x.less_equal(y).numpy(), - paddle.less_equal(x, y).numpy()) + x.less_equal(y).numpy(), paddle.less_equal(x, y).numpy() + ) np.testing.assert_array_equal( - x.less_than(y).numpy(), - paddle.less_than(x, y).numpy()) + x.less_than(y).numpy(), paddle.less_than(x, y).numpy() + ) np.testing.assert_array_equal( - x.not_equal(y).numpy(), - paddle.not_equal(x, y).numpy()) + x.not_equal(y).numpy(), paddle.not_equal(x, y).numpy() + ) np.testing.assert_array_equal( - x.equal_all(y).numpy(), - paddle.equal_all(x, y).numpy()) + x.equal_all(y).numpy(), paddle.equal_all(x, y).numpy() + ) np.testing.assert_array_equal( - x.allclose(y).numpy(), - paddle.allclose(x, y).numpy()) + x.allclose(y).numpy(), paddle.allclose(x, y).numpy() + ) m = x.expand([2, 2, 3]) np.testing.assert_array_equal( - x.expand_as(m).numpy(), - paddle.expand_as(x, m).numpy()) + x.expand_as(m).numpy(), paddle.expand_as(x, m).numpy() + ) index = paddle.to_tensor([2, 1, 0]) np.testing.assert_array_equal( - a.scatter(index, b).numpy(), - paddle.scatter(a, index, b).numpy()) + a.scatter(index, b).numpy(), paddle.scatter(a, index, b).numpy() + ) # 3. Bool tensor operation x = paddle.to_tensor([[True, False], [True, False]]) y = paddle.to_tensor([[False, False], [False, True]]) np.testing.assert_array_equal( - x.logical_and(y).numpy(), - paddle.logical_and(x, y).numpy()) + x.logical_and(y).numpy(), paddle.logical_and(x, y).numpy() + ) np.testing.assert_array_equal( - x.logical_not(y).numpy(), - paddle.logical_not(x, y).numpy()) + x.logical_not(y).numpy(), paddle.logical_not(x, y).numpy() + ) np.testing.assert_array_equal( - x.logical_or(y).numpy(), - paddle.logical_or(x, y).numpy()) + x.logical_or(y).numpy(), paddle.logical_or(x, y).numpy() + ) np.testing.assert_array_equal( - x.logical_xor(y).numpy(), - paddle.logical_xor(x, y).numpy()) + x.logical_xor(y).numpy(), paddle.logical_xor(x, y).numpy() + ) np.testing.assert_array_equal( - x.logical_and(y).numpy(), - paddle.logical_and(x, y).numpy()) + x.logical_and(y).numpy(), paddle.logical_and(x, y).numpy() + ) a = paddle.to_tensor([[1, 2], [3, 4]]) b = paddle.to_tensor([[4, 3], [2, 1]]) np.testing.assert_array_equal( - x.where(a, b).numpy(), - paddle.where(x, a, b).numpy()) + x.where(a, b).numpy(), paddle.where(x, a, b).numpy() + ) x_np = np.random.randn(3, 6, 9, 7) x = paddle.to_tensor(x_np) @@ -724,7 +747,7 @@ class TestMathOpPatchesVarBase(unittest.TestCase): a_np = np.random.random(self.shape).astype(self.dtype) with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) - res = 1J * a + res = 1j * a np.testing.assert_array_equal(res.numpy(), 1j * a_np) def test_complex_scalar(self): diff --git a/python/paddle/fluid/tests/unittests/test_matmul_op.py b/python/paddle/fluid/tests/unittests/test_matmul_op.py index b8ca9b9dc1a..62c7e9b7bb7 100644 --- a/python/paddle/fluid/tests/unittests/test_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_matmul_op.py @@ -86,7 +86,6 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): class Generator(object): - def setUp(self): self.op_type = "matmul" X = np.random.random(self.shape_X).astype("float32") @@ -95,7 +94,7 @@ class Generator(object): self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, - 'transpose_Y': self.transpose_Y + 'transpose_Y': self.transpose_Y, } self.outputs = {'Out': Out} @@ -106,39 +105,37 @@ class Generator(object): self.check_grad(['X', 'Y'], 'Out', max_relative_error=1e-3) def test_check_grad_ignore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=1e-3, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=1e-3, no_grad_set=set("X") + ) def test_check_grad_ignore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=1e-3, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=1e-3, no_grad_set=set('Y') + ) class TestMatmulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The inputs type of matmul_op must be Variable. input1 = 12 self.assertRaises(TypeError, fluid.layers.matmul, input1, input1) # The inputs dtype of matmul_op must be float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[10, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[10, 10], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.matmul, input2, input2) - input3 = fluid.layers.data(name='input3', - shape=[2, 2], - dtype="float16") + input3 = fluid.layers.data( + name='input3', shape=[2, 2], dtype="float16" + ) fluid.layers.matmul(input3, input3) # Negative dimension generation def generate_negative_dims(in_shape): from itertools import combinations + size = len(in_shape) indexs = list() shapes = list() @@ -146,7 +143,8 @@ def generate_negative_dims(in_shape): indexs.extend(list(combinations([j for j in range(size)], i + 1))) for idx in indexs: shapes.append( - [in_shape[i] if i not in idx else -1 for i in range(size)]) + [in_shape[i] if i not in idx else -1 for i in range(size)] + ) return shapes @@ -160,51 +158,61 @@ def test_negative_dims_program(obj): with program_guard(Program(), Program()): x = fluid.data(name='x', shape=shape_x, dtype='float32') y = fluid.data(name='y', shape=shape_y, dtype='float32') - output = fluid.layers.matmul(x, y, obj.transpose_X, - obj.transpose_Y) + output = fluid.layers.matmul( + x, y, obj.transpose_X, obj.transpose_Y + ) obj.assertEqual(len(Ref.shape), len(output.shape)) for idx in range(len(Ref.shape)): if output.shape[idx] != -1: obj.assertEqual(Ref.shape[idx], output.shape[idx]) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(fluid.default_main_program(), - feed={ - 'x': X, - 'y': Y - }, - fetch_list=[output]) + (res,) = exe.run( + fluid.default_main_program(), + feed={'x': X, 'y': Y}, + fetch_list=[output], + ) np.allclose(res, Ref, atol=1e-5) # Generate program api cases for all negative possibilities def api_test(dim_x, dim_y, trans_x, trans_y): - test_name = ('TestMatMulAPI_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( - dim_x, dim_y, trans_x, trans_y)) - shape_x, shape_y = generate_compatible_shapes(dim_x, dim_y, trans_x, - trans_y) + test_name = 'TestMatMulAPI_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim_x, dim_y, trans_x, trans_y + ) + shape_x, shape_y = generate_compatible_shapes( + dim_x, dim_y, trans_x, trans_y + ) globals()[test_name] = type( - test_name, (unittest.TestCase, ), { + test_name, + (unittest.TestCase,), + { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': trans_x, 'transpose_Y': trans_y, 'test_propram': test_negative_dims_program, - }) + }, + ) # Generate operators cases for all possibilities def inject_test(dim_x, dim_y, trans_x, trans_y): - test_name = ('TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( - dim_x, dim_y, trans_x, trans_y)) - shape_x, shape_y = generate_compatible_shapes(dim_x, dim_y, trans_x, - trans_y) + test_name = 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim_x, dim_y, trans_x, trans_y + ) + shape_x, shape_y = generate_compatible_shapes( + dim_x, dim_y, trans_x, trans_y + ) globals()[test_name] = type( - test_name, (Generator, OpTest), { + test_name, + (Generator, OpTest), + { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': trans_x, 'transpose_Y': trans_y, - }) + }, + ) for dim_X in (1, 2, 3): @@ -216,8 +224,9 @@ for dim_X in (1, 2, 3): # Test case more batch_size and N, M, K -def generate_compatible_shapes_batch(dim_X, dim_Y, transpose_X, transpose_Y, - batch_size): +def generate_compatible_shapes_batch( + dim_X, dim_Y, transpose_X, transpose_Y, batch_size +): BATCH_SIZE = 2 M = 3 N = 4 @@ -278,20 +287,25 @@ for dim in [4]: for transpose_Y in [False, True]: test_name = ( 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( - dim, dim, transpose_X, transpose_Y)) + dim, dim, transpose_X, transpose_Y + ) + ) shape_X, shape_Y = generate_compatible_shapes_ndim( - dim, transpose_X, transpose_Y) + dim, transpose_X, transpose_Y + ) globals()[test_name] = type( - test_name, (Generator, OpTest), { + test_name, + (Generator, OpTest), + { 'shape_X': shape_X, 'shape_Y': shape_Y, 'transpose_X': transpose_X, 'transpose_Y': transpose_Y, - }) + }, + ) class API_TestMm(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2], dtype="float64") @@ -302,8 +316,9 @@ class API_TestMm(unittest.TestCase): data1 = np.random.rand(2) data2 = np.random.rand(2) np_res = exe.run(feed={'x': data1, 'y': data2}, fetch_list=[result]) - expected_result = np.matmul(data1.reshape(1, 2), - data2.reshape(2, 1)) + expected_result = np.matmul( + data1.reshape(1, 2), data2.reshape(2, 1) + ) np.testing.assert_allclose( np_res, @@ -311,7 +326,9 @@ class API_TestMm(unittest.TestCase): rtol=1e-05, atol=1e-05, err_msg='two value is {}\n{}, check diff!'.format( - np_res, expected_result)) + np_res, expected_result + ), + ) def test_dygraph_without_out(self): device = fluid.CPUPlace() @@ -326,7 +343,6 @@ class API_TestMm(unittest.TestCase): class Test_API_Matmul(unittest.TestCase): - def test_dygraph_without_out(self): device = fluid.CPUPlace() with fluid.dygraph.guard(device): @@ -340,9 +356,7 @@ class Test_API_Matmul(unittest.TestCase): class API_TestMmError(unittest.TestCase): - def test_errors(self): - def test_error1(): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.data(name="data1", shape=[10, 2], dtype="float32") @@ -353,24 +367,24 @@ class API_TestMmError(unittest.TestCase): def test_error2(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[-1, 10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 2, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 2, 10], dtype="float32" + ) paddle.mm(data1, data2) test_error2() def test_error3(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[10, 10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[3, 2, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[10, 10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[3, 2, 10], dtype="float32" + ) paddle.mm(data1, data2) self.assertRaises(ValueError, test_error3) diff --git a/python/paddle/fluid/tests/unittests/test_matmul_op_with_head.py b/python/paddle/fluid/tests/unittests/test_matmul_op_with_head.py index 77e3c7f4c1f..bd6b1877ecd 100644 --- a/python/paddle/fluid/tests/unittests/test_matmul_op_with_head.py +++ b/python/paddle/fluid/tests/unittests/test_matmul_op_with_head.py @@ -59,16 +59,18 @@ def matmul_head(X, Y, head_number=1): sub_y_height = Y.shape[-2] // head_number if np.ndim(X) == 2: for i in range(0, head_number): - x.append(X[:, i * sub_x_width:i * sub_x_width + sub_x_width]) - y.append(Y[i * sub_y_height:i * sub_y_height + sub_y_height, :]) + x.append(X[:, i * sub_x_width : i * sub_x_width + sub_x_width]) + y.append(Y[i * sub_y_height : i * sub_y_height + sub_y_height, :]) for i in range(0, head_number): z.append(np.matmul(x[i], y[i])) Z = np.concatenate((z), axis=1) elif np.ndim(X) == 3: for i in range(0, head_number): - x.append(X[:, :, i * sub_x_width:i * sub_x_width + sub_x_width]) - y.append(Y[:, i * sub_y_height:i * sub_y_height + sub_y_height, :]) + x.append(X[:, :, i * sub_x_width : i * sub_x_width + sub_x_width]) + y.append( + Y[:, i * sub_y_height : i * sub_y_height + sub_y_height, :] + ) for i in range(0, head_number): z.append(np.matmul(x[i], y[i])) Z = np.concatenate((z), axis=2) @@ -87,11 +89,9 @@ def transpose_mat(X): return X -def reference_matmul_mul_head(X, - Y, - head_number=1, - transpose_X=False, - transpose_Y=False): +def reference_matmul_mul_head( + X, Y, head_number=1, transpose_X=False, transpose_Y=False +): """Reference forward implementation using np.matmul.""" # np.matmul does not support the transpose flags, so we manually # transpose X and Y appropriately. @@ -112,19 +112,19 @@ def reference_matmul_mul_head(X, # Generator for multiple head class GeneratorMulHead(object): - def setUp(self): self.op_type = "matmul" X = np.random.random(self.shape_X).astype("float32") Y = np.random.random(self.shape_Y).astype("float32") - Out = reference_matmul_mul_head(X, Y, 4, self.transpose_X, - self.transpose_Y) + Out = reference_matmul_mul_head( + X, Y, 4, self.transpose_X, self.transpose_Y + ) self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, 'transpose_Y': self.transpose_Y, - 'head_number': self.head_number + 'head_number': self.head_number, } self.outputs = {'Out': Out} @@ -135,17 +135,23 @@ class GeneratorMulHead(object): def inject_test_multiple_head(dim_x, dim_y, trans_x, trans_y, head_number): test_name = ( 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}_head_{}'.format( - dim_x, dim_y, trans_x, trans_y, head_number)) + dim_x, dim_y, trans_x, trans_y, head_number + ) + ) shape_x, shape_y = generate_compatible_shapes_mul_head( - dim_x, dim_y, trans_x, trans_y) + dim_x, dim_y, trans_x, trans_y + ) globals()[test_name] = type( - test_name, (GeneratorMulHead, OpTest), { + test_name, + (GeneratorMulHead, OpTest), + { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': trans_x, 'transpose_Y': trans_y, - 'head_number': head_number - }) + 'head_number': head_number, + }, + ) def matmul_head2(X, Y, head_number=1): @@ -154,20 +160,21 @@ def matmul_head2(X, Y, head_number=1): z = [] sub_x_width = X.shape[-1] // head_number sub_y_width = Y.shape[-1] // head_number - assert (sub_x_width == Y.shape[-2] - ), "Error: incompatible head number or matrix size!" + assert ( + sub_x_width == Y.shape[-2] + ), "Error: incompatible head number or matrix size!" if np.ndim(X) == 2: for i in range(0, head_number): - x.append(X[:, i * sub_x_width:i * sub_x_width + sub_x_width]) - y.append(Y[:, i * sub_y_width:i * sub_y_width + sub_y_width]) + x.append(X[:, i * sub_x_width : i * sub_x_width + sub_x_width]) + y.append(Y[:, i * sub_y_width : i * sub_y_width + sub_y_width]) for i in range(0, head_number): z.append(np.matmul(x[i], y[i])) Z = np.concatenate((z), axis=1) elif np.ndim(X) == 3: for i in range(0, head_number): - x.append(X[:, :, i * sub_x_width:i * sub_x_width + sub_x_width]) - y.append(Y[:, :, i * sub_y_width:i * sub_y_width + sub_y_width]) + x.append(X[:, :, i * sub_x_width : i * sub_x_width + sub_x_width]) + y.append(Y[:, :, i * sub_y_width : i * sub_y_width + sub_y_width]) for i in range(0, head_number): z.append(np.matmul(x[i], y[i])) Z = np.concatenate((z), axis=2) @@ -176,11 +183,9 @@ def matmul_head2(X, Y, head_number=1): return Z -def reference_matmul_mul_head2(X, - Y, - head_number=1, - transpose_X=False, - transpose_Y=False): +def reference_matmul_mul_head2( + X, Y, head_number=1, transpose_X=False, transpose_Y=False +): """Reference forward implementation using np.matmul.""" # np.matmul does not support the transpose flags, so we manually # transpose X and Y appropriately. @@ -199,8 +204,9 @@ def reference_matmul_mul_head2(X, return Out -def generate_compatible_shapes_mul_head2(dim_X, dim_Y, transpose_X, - transpose_Y): +def generate_compatible_shapes_mul_head2( + dim_X, dim_Y, transpose_X, transpose_Y +): BATCH_SIZE = 2 # Assume head number H is 4. We need make sure K1/H = M2 M1 = 3 @@ -227,38 +233,36 @@ def generate_compatible_shapes_mul_head2(dim_X, dim_Y, transpose_X, # Generator for multiple head, case 2 when width of X is not same as height of Y class GeneratorMulHead2(object): - def setUp(self): self.op_type = "matmul" X = np.zeros(self.shape_X) Y = np.zeros(self.shape_Y) if len(self.shape_X) == 2: - X = np.arange(0, - self.shape_X[-1] * self.shape_X[-2], - dtype=np.float32).reshape(self.shape_X) - Y = np.arange(0, - self.shape_Y[-1] * self.shape_Y[-2], - dtype=np.float32).reshape(self.shape_Y) + X = np.arange( + 0, self.shape_X[-1] * self.shape_X[-2], dtype=np.float32 + ).reshape(self.shape_X) + Y = np.arange( + 0, self.shape_Y[-1] * self.shape_Y[-2], dtype=np.float32 + ).reshape(self.shape_Y) else: for i in range(0, len(self.shape_X) - 1): - X[i, :, :] = np.arange(0, - self.shape_X[-1] * self.shape_X[-2], - dtype=np.float32).reshape( - list(self.shape_X)[-2:]) - Y[i, :, :] = np.arange(0, - self.shape_Y[-1] * self.shape_Y[-2], - dtype=np.float32).reshape( - list(self.shape_Y)[-2:]) - - Out = reference_matmul_mul_head2(X, Y, 4, self.transpose_X, - self.transpose_Y) + X[i, :, :] = np.arange( + 0, self.shape_X[-1] * self.shape_X[-2], dtype=np.float32 + ).reshape(list(self.shape_X)[-2:]) + Y[i, :, :] = np.arange( + 0, self.shape_Y[-1] * self.shape_Y[-2], dtype=np.float32 + ).reshape(list(self.shape_Y)[-2:]) + + Out = reference_matmul_mul_head2( + X, Y, 4, self.transpose_X, self.transpose_Y + ) self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, 'transpose_Y': self.transpose_Y, - 'head_number': self.head_number + 'head_number': self.head_number, } self.outputs = {'Out': Out} @@ -269,26 +273,32 @@ class GeneratorMulHead2(object): def inject_test_multiple_head2(dim_x, dim_y, trans_x, trans_y, head_number): test_name = ( 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}_head2_{}'.format( - dim_x, dim_y, trans_x, trans_y, head_number)) + dim_x, dim_y, trans_x, trans_y, head_number + ) + ) shape_x, shape_y = generate_compatible_shapes_mul_head2( - dim_x, dim_y, trans_x, trans_y) + dim_x, dim_y, trans_x, trans_y + ) globals()[test_name] = type( - test_name, (GeneratorMulHead2, OpTest), { + test_name, + (GeneratorMulHead2, OpTest), + { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': trans_x, 'transpose_Y': trans_y, - 'head_number': head_number - }) + 'head_number': head_number, + }, + ) -#test case for multiple head +# test case for multiple head for dim in (2, 3): for transose_x in (False, True): for transose_y in (False, True): inject_test_multiple_head(dim, dim, transose_x, transose_y, 4) -#test case for multiple head when X.width != Y.height +# test case for multiple head when X.width != Y.height for dim in (2, 3): for transose_x in (False, True): for transose_y in (False, True): diff --git a/python/paddle/fluid/tests/unittests/test_matmul_v2_op.py b/python/paddle/fluid/tests/unittests/test_matmul_v2_op.py index 4caffe5019f..9af6d6598d2 100644 --- a/python/paddle/fluid/tests/unittests/test_matmul_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_matmul_v2_op.py @@ -29,7 +29,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): # transpose X and Y appropriately. if transpose_X: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -38,7 +38,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -60,8 +60,8 @@ class TestMatMulV2Op(OpTest): """ def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -106,10 +106,9 @@ class TestMatMulV2Op(OpTest): def test_check_grad(self): if core.is_compiled_with_rocm(): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=1e-2, - check_eager=False) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=1e-2, check_eager=False + ) else: self.check_grad(['X', 'Y'], 'Out', check_eager=False) @@ -120,7 +119,7 @@ class TestMatMulOp2(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.trans_x = False self.trans_y = True @@ -132,7 +131,7 @@ class TestMatMulOp3(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False @@ -144,7 +143,7 @@ class TestMatMulOp4(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -157,7 +156,7 @@ class TestMatMulOp5(TestMatMulV2Op): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False @@ -169,7 +168,7 @@ class TestMatMulOp6(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 102, 1) - self.y_shape = (102, ) + self.y_shape = (102,) self.trans_x = True self.trans_y = False @@ -181,7 +180,7 @@ class TestMatMulOp7(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 1, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -288,7 +287,7 @@ class TestMatMulOp16(TestMatMulV2Op): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -301,7 +300,7 @@ class TestMatMulOp17(TestMatMulV2Op): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False @@ -330,15 +329,14 @@ class TestMatMulOpBroadcast2(TestMatMulV2Op): self.trans_y = True -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=1.0): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -346,18 +344,20 @@ def create_test_fp16_class(parent, atol=0.001, max_relative_error=1.0): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_output_with_place(place, - atol=atol, - check_eager=False) + self.check_output_with_place( + place, atol=atol, check_eager=False + ) def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): self.check_grad_with_place( - place, ['X', 'Y'], + place, + ['X', 'Y'], 'Out', max_relative_error=max_relative_error, - check_eager=False) + check_eager=False, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name @@ -382,24 +382,25 @@ create_test_fp16_class(TestMatMulOp15) create_test_fp16_class(TestMatMulOp16) create_test_fp16_class(TestMatMulOp17) -#--------------------test matmul bf16-------------------- +# --------------------test matmul bf16-------------------- def create_test_bf16_class(parent, atol=0.01): - @unittest.skipIf( not core.is_compiled_with_cuda() or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA and not support the bfloat16") + "core is not compiled with CUDA and not support the bfloat16", + ) class TestMatMulOpBf16Case(parent): - def get_numeric_grad(self, place, check_name): scope = core.Scope() self._check_grad_helper() - op = create_op(scope, self.op_type, self.inputs, self.outputs, - self.attrs) - return get_numeric_gradient(place, scope, op, self.inputs_fp32, - check_name, ['Out']) + op = create_op( + scope, self.op_type, self.inputs, self.outputs, self.attrs + ) + return get_numeric_gradient( + place, scope, op, self.inputs_fp32, check_name, ['Out'] + ) def init_kernel_type(self): self.dtype = np.uint16 @@ -411,18 +412,24 @@ def create_test_bf16_class(parent, atol=0.01): def test_check_grad_x(self): place = core.CUDAPlace(0) numeric_grads = self.get_numeric_grad(place, 'X') - self.check_grad_with_place(place, ['X'], - 'Out', - no_grad_set=set(['Y']), - user_defined_grads=[numeric_grads]) + self.check_grad_with_place( + place, + ['X'], + 'Out', + no_grad_set=set(['Y']), + user_defined_grads=[numeric_grads], + ) def test_check_grad_y(self): place = core.CUDAPlace(0) numeric_grads = self.get_numeric_grad(place, 'Y') - self.check_grad_with_place(place, ['Y'], - 'Out', - no_grad_set=set(['X']), - user_defined_grads=[numeric_grads]) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + no_grad_set=set(['X']), + user_defined_grads=[numeric_grads], + ) def test_check_grad(self): pass @@ -452,7 +459,6 @@ create_test_bf16_class(TestMatMulOp17) class TestMatMulV2API(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -469,12 +475,11 @@ class TestMatMulV2API(unittest.TestCase): y_np = np.random.random([3, 4]).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": x_np, - "input_y": y_np - }, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np, "input_y": y_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: @@ -506,7 +511,8 @@ class TestMatMulV2API(unittest.TestCase): if core.is_float16_supported(place): with fluid.dygraph.guard(place): paddle.set_flags( - {'FLAGS_gemm_use_half_precision_compute_type': False}) + {'FLAGS_gemm_use_half_precision_compute_type': False} + ) input_x = np.random.random([2, 8, 16]).astype("float16") input_y = np.random.random([2, 16, 8]).astype("float16") for i in range(0, 16, 2): @@ -522,7 +528,8 @@ class TestMatMulV2API(unittest.TestCase): self.assertTrue(np.isfinite(result_np)[0, 0, 0]) np.testing.assert_array_equal(result_np, result.numpy()) paddle.set_flags( - {'FLAGS_gemm_use_half_precision_compute_type': True}) + {'FLAGS_gemm_use_half_precision_compute_type': True} + ) def test_compute_type_fp16_nan(self): if core.is_compiled_with_cuda(): @@ -530,7 +537,8 @@ class TestMatMulV2API(unittest.TestCase): if core.is_float16_supported(place): with fluid.dygraph.guard(place): paddle.set_flags( - {'FLAGS_gemm_use_half_precision_compute_type': True}) + {'FLAGS_gemm_use_half_precision_compute_type': True} + ) input_x = np.random.random([2, 8, 16]).astype("float16") input_y = np.random.random([2, 16, 8]).astype("float16") for i in range(0, 16, 2): @@ -543,10 +551,12 @@ class TestMatMulV2API(unittest.TestCase): result = paddle.matmul(x, y) result_np = np.matmul(input_x, input_y) self.assertFalse( - paddle.isfinite(result)[0, 0, 0]) # contains nan/inf + paddle.isfinite(result)[0, 0, 0] + ) # contains nan/inf self.assertTrue(np.isfinite(result_np)[0, 0, 0]) paddle.set_flags( - {'FLAGS_gemm_use_half_precision_compute_type': False}) + {'FLAGS_gemm_use_half_precision_compute_type': False} + ) def test_api_eager_dygraph(self): with _test_eager_guard(): @@ -555,7 +565,6 @@ class TestMatMulV2API(unittest.TestCase): class TestComplexMatMulOp(OpTest): - def setUp(self): self.op_type = "matmul_v2" self.init_base_dtype() @@ -564,7 +573,7 @@ class TestComplexMatMulOp(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -573,17 +582,18 @@ class TestComplexMatMulOp(OpTest): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (10, 10)).astype(self.dtype) + 1J * np.random.random( - (10, 10)).astype(self.dtype) - self.y = np.random.random( - (10, 10)).astype(self.dtype) + 1J * np.random.random( - (10, 10)).astype(self.dtype) + self.x = np.random.random((10, 10)).astype( + self.dtype + ) + 1j * np.random.random((10, 10)).astype(self.dtype) + self.y = np.random.random((10, 10)).astype( + self.dtype + ) + 1j * np.random.random((10, 10)).astype(self.dtype) self.out = np.dot(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones((10, 10), self.dtype) + 1J * np.ones( - (10, 10), self.dtype) + self.grad_out = np.ones((10, 10), self.dtype) + 1j * np.ones( + (10, 10), self.dtype + ) self.grad_x = np.matmul(self.grad_out, np.conj(self.y).T) self.grad_y = np.matmul(np.conj(self.x).T, self.grad_out) @@ -591,31 +601,36 @@ class TestComplexMatMulOp(OpTest): self.check_output(check_eager=False) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) class TestComplexMatMulOpBroadcast(OpTest): - def setUp(self): self.op_type = "matmul_v2" self.init_base_dtype() @@ -624,7 +639,7 @@ class TestComplexMatMulOpBroadcast(OpTest): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -633,61 +648,68 @@ class TestComplexMatMulOpBroadcast(OpTest): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (10, 2, 5)).astype(self.dtype) + 1J * np.random.random( - (10, 2, 5)).astype(self.dtype) - self.y = np.random.random( - (5, 20)).astype(self.dtype) + 1J * np.random.random( - (5, 20)).astype(self.dtype) + self.x = np.random.random((10, 2, 5)).astype( + self.dtype + ) + 1j * np.random.random((10, 2, 5)).astype(self.dtype) + self.y = np.random.random((5, 20)).astype( + self.dtype + ) + 1j * np.random.random((5, 20)).astype(self.dtype) self.out = np.dot(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones((10, 2, 20), self.dtype) + 1J * np.ones( - (10, 2, 20), self.dtype) + self.grad_out = np.ones((10, 2, 20), self.dtype) + 1j * np.ones( + (10, 2, 20), self.dtype + ) self.grad_x = np.matmul(self.grad_out, np.conj(self.y).T) - self.grad_y = np.sum(np.matmul( - np.conj(self.x).transpose(0, 2, 1), self.grad_out), - axis=0) + self.grad_y = np.sum( + np.matmul(np.conj(self.x).transpose(0, 2, 1), self.grad_out), axis=0 + ) def test_check_output(self): self.check_output(check_eager=False) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) class TestMatMulTypePromotion(TestComplexMatMulOp): - def init_input_output(self): self.x = np.random.random((10, 10)).astype(self.dtype) - self.y = np.random.random( - (10, 10)).astype(self.dtype) + 1J * np.random.random( - (10, 10)).astype(self.dtype) + self.y = np.random.random((10, 10)).astype( + self.dtype + ) + 1j * np.random.random((10, 10)).astype(self.dtype) self.out = np.dot(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones((10, 10), self.dtype) + 1J * np.ones( - (10, 10), self.dtype) + self.grad_out = np.ones((10, 10), self.dtype) + 1j * np.ones( + (10, 10), self.dtype + ) self.grad_x = np.matmul(self.grad_out, np.conj(self.y).T).real self.grad_y = np.matmul(np.conj(self.x).T, self.grad_out) diff --git a/python/paddle/fluid/tests/unittests/test_matrix_nms_op.py b/python/paddle/fluid/tests/unittests/test_matrix_nms_op.py index d0b9c302128..fe43d42cb45 100644 --- a/python/paddle/fluid/tests/unittests/test_matrix_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_matrix_nms_op.py @@ -21,22 +21,34 @@ from paddle.fluid import Program, program_guard import paddle -def python_matrix_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - post_threshold, - use_gaussian=False, - gaussian_sigma=2., - background_label=0, - normalized=True, - return_index=True, - return_rois_num=True): +def python_matrix_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + post_threshold, + use_gaussian=False, + gaussian_sigma=2.0, + background_label=0, + normalized=True, + return_index=True, + return_rois_num=True, +): out, rois_num, index = paddle.vision.ops.matrix_nms( - bboxes, scores, score_threshold, post_threshold, nms_top_k, keep_top_k, - use_gaussian, gaussian_sigma, background_label, normalized, - return_index, return_rois_num) + bboxes, + scores, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + use_gaussian, + gaussian_sigma, + background_label, + normalized, + return_index, + return_rois_num, + ) if not return_index: index = None if not return_rois_num: @@ -47,7 +59,7 @@ def python_matrix_nms(bboxes, def softmax(x): # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -61,18 +73,20 @@ def iou_matrix(a, b, norm=True): area_i = np.prod(br_i - tl_i + pad, axis=2) * (tl_i < br_i).all(axis=2) area_a = np.prod(a[:, 2:] - a[:, :2] + pad, axis=1) area_b = np.prod(b[:, 2:] - b[:, :2] + pad, axis=1) - area_o = (area_a[:, np.newaxis] + area_b - area_i) + area_o = area_a[:, np.newaxis] + area_b - area_i return area_i / (area_o + 1e-10) -def matrix_nms(boxes, - scores, - score_threshold, - post_threshold=0., - nms_top_k=400, - normalized=True, - use_gaussian=False, - gaussian_sigma=2.): +def matrix_nms( + boxes, + scores, + score_threshold, + post_threshold=0.0, + nms_top_k=400, + normalized=True, + use_gaussian=False, + gaussian_sigma=2.0, +): all_scores = copy.deepcopy(scores) all_scores = all_scores.flatten() selected_indices = np.where(all_scores > score_threshold)[0] @@ -99,7 +113,7 @@ def matrix_nms(boxes, decay = decay.min(0) decayed_scores = sorted_scores * decay - if post_threshold > 0.: + if post_threshold > 0.0: inds = np.where(decayed_scores > post_threshold)[0] selected_boxes = selected_boxes[inds, :] decayed_scores = decayed_scores[inds] @@ -108,9 +122,18 @@ def matrix_nms(boxes, return decayed_scores, selected_boxes, sorted_indices -def multiclass_nms(boxes, scores, background, score_threshold, post_threshold, - nms_top_k, keep_top_k, normalized, use_gaussian, - gaussian_sigma): +def multiclass_nms( + boxes, + scores, + background, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + normalized, + use_gaussian, + gaussian_sigma, +): all_boxes = [] all_cls = [] all_scores = [] @@ -119,8 +142,15 @@ def multiclass_nms(boxes, scores, background, score_threshold, post_threshold, if c == background: continue decayed_scores, selected_boxes, indices = matrix_nms( - boxes, scores[c], score_threshold, post_threshold, nms_top_k, - normalized, use_gaussian, gaussian_sigma) + boxes, + scores[c], + score_threshold, + post_threshold, + nms_top_k, + normalized, + use_gaussian, + gaussian_sigma, + ) all_cls.append(np.full(len(decayed_scores), c, decayed_scores.dtype)) all_boxes.append(selected_boxes) all_scores.append(decayed_scores) @@ -131,7 +161,8 @@ def multiclass_nms(boxes, scores, background, score_threshold, post_threshold, all_scores = np.concatenate(all_scores) all_indices = np.concatenate(all_indices) all_pred = np.concatenate( - (all_cls[:, np.newaxis], all_scores[:, np.newaxis], all_boxes), axis=1) + (all_cls[:, np.newaxis], all_scores[:, np.newaxis], all_boxes), axis=1 + ) num_det = len(all_pred) if num_det == 0: @@ -149,25 +180,35 @@ def multiclass_nms(boxes, scores, background, score_threshold, post_threshold, return all_pred, all_indices -def batched_multiclass_nms(boxes, - scores, - background, - score_threshold, - post_threshold, - nms_top_k, - keep_top_k, - normalized=True, - use_gaussian=False, - gaussian_sigma=2.): +def batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + normalized=True, + use_gaussian=False, + gaussian_sigma=2.0, +): batch_size = scores.shape[0] det_outs = [] index_outs = [] lod = [] for n in range(batch_size): - nmsed_outs, indices = multiclass_nms(boxes[n], scores[n], background, - score_threshold, post_threshold, - nms_top_k, keep_top_k, normalized, - use_gaussian, gaussian_sigma) + nmsed_outs, indices = multiclass_nms( + boxes[n], + scores[n], + background, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + normalized, + use_gaussian, + gaussian_sigma, + ) nmsed_num = len(nmsed_outs) lod.append(nmsed_num) if nmsed_num == 0: @@ -182,9 +223,8 @@ def batched_multiclass_nms(boxes, class TestMatrixNMSOp(OpTest): - def set_argument(self): - self.post_threshold = 0. + self.post_threshold = 0.0 self.use_gaussian = False def setUp(self): @@ -202,7 +242,7 @@ class TestMatrixNMSOp(OpTest): use_gaussian = False if hasattr(self, 'use_gaussian'): use_gaussian = self.use_gaussian - gaussian_sigma = 2. + gaussian_sigma = 2.0 scores = np.random.random((N * M, C)).astype('float32') @@ -215,8 +255,17 @@ class TestMatrixNMSOp(OpTest): boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5 det_outs, index_outs, lod = batched_multiclass_nms( - boxes, scores, background, score_threshold, post_threshold, - nms_top_k, keep_top_k, True, use_gaussian, gaussian_sigma) + boxes, + scores, + background, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + True, + use_gaussian, + gaussian_sigma, + ) empty = len(det_outs) == 0 det_outs = np.array([], dtype=np.float32) if empty else det_outs @@ -228,7 +277,7 @@ class TestMatrixNMSOp(OpTest): self.outputs = { 'Out': nmsed_outs, 'Index': index_outs[:, None], - 'RoisNum': np.array(lod).astype('int32') + 'RoisNum': np.array(lod).astype('int32'), } self.attrs = { 'score_threshold': score_threshold, @@ -246,20 +295,17 @@ class TestMatrixNMSOp(OpTest): class TestMatrixNMSOpNoOutput(TestMatrixNMSOp): - def set_argument(self): self.post_threshold = 2.0 class TestMatrixNMSOpGaussian(TestMatrixNMSOp): - def set_argument(self): - self.post_threshold = 0. + self.post_threshold = 0.0 self.use_gaussian = True class TestMatrixNMSError(unittest.TestCase): - def test_errors(self): M = 1200 N = 7 @@ -268,7 +314,7 @@ class TestMatrixNMSError(unittest.TestCase): nms_top_k = 400 keep_top_k = 200 score_threshold = 0.01 - post_threshold = 0. + post_threshold = 0.0 boxes_np = np.random.random((M, C, BOX_SIZE)).astype('float32') scores = np.random.random((N * M, C)).astype('float32') @@ -277,52 +323,62 @@ class TestMatrixNMSError(unittest.TestCase): scores_np = np.transpose(scores, (0, 2, 1)) with program_guard(Program(), Program()): - boxes_data = fluid.data(name='bboxes', - shape=[M, C, BOX_SIZE], - dtype='float32') - scores_data = fluid.data(name='scores', - shape=[N, C, M], - dtype='float32') + boxes_data = fluid.data( + name='bboxes', shape=[M, C, BOX_SIZE], dtype='float32' + ) + scores_data = fluid.data( + name='scores', shape=[N, C, M], dtype='float32' + ) def test_bboxes_Variable(): # the bboxes type must be Variable - fluid.layers.matrix_nms(bboxes=boxes_np, - scores=scores_data, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) - paddle.vision.ops.matrix_nms(bboxes=boxes_np, - scores=scores_data, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + fluid.layers.matrix_nms( + bboxes=boxes_np, + scores=scores_data, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) + paddle.vision.ops.matrix_nms( + bboxes=boxes_np, + scores=scores_data, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) def test_scores_Variable(): # the scores type must be Variable - fluid.layers.matrix_nms(bboxes=boxes_data, - scores=scores_np, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) - paddle.vision.ops.matrix_nms(bboxes=boxes_data, - scores=scores_np, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + fluid.layers.matrix_nms( + bboxes=boxes_data, + scores=scores_np, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) + paddle.vision.ops.matrix_nms( + bboxes=boxes_data, + scores=scores_np, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) def test_empty(): # when all score are lower than threshold try: - fluid.layers.matrix_nms(bboxes=boxes_data, - scores=scores_data, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + fluid.layers.matrix_nms( + bboxes=boxes_data, + scores=scores_data, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) except Exception as e: self.fail(e) try: @@ -332,19 +388,22 @@ class TestMatrixNMSError(unittest.TestCase): score_threshold=score_threshold, post_threshold=post_threshold, nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + keep_top_k=keep_top_k, + ) except Exception as e: self.fail(e) def test_coverage(): # cover correct workflow try: - fluid.layers.matrix_nms(bboxes=boxes_data, - scores=scores_data, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + fluid.layers.matrix_nms( + bboxes=boxes_data, + scores=scores_data, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) except Exception as e: self.fail(e) try: @@ -354,7 +413,8 @@ class TestMatrixNMSError(unittest.TestCase): score_threshold=score_threshold, post_threshold=post_threshold, nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + keep_top_k=keep_top_k, + ) except Exception as e: self.fail(e) diff --git a/python/paddle/fluid/tests/unittests/test_matrix_power_op.py b/python/paddle/fluid/tests/unittests/test_matrix_power_op.py index af400b14615..0632af3a3a3 100644 --- a/python/paddle/fluid/tests/unittests/test_matrix_power_op.py +++ b/python/paddle/fluid/tests/unittests/test_matrix_power_op.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestMatrixPowerOp(OpTest): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -45,14 +44,12 @@ class TestMatrixPowerOp(OpTest): self.check_output() def test_grad(self): - self.check_grad(["X"], - "Out", - numeric_grad_delta=1e-5, - max_relative_error=1e-7) + self.check_grad( + ["X"], "Out", numeric_grad_delta=1e-5, max_relative_error=1e-7 + ) class TestMatrixPowerOpN1(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -60,7 +57,6 @@ class TestMatrixPowerOpN1(TestMatrixPowerOp): class TestMatrixPowerOpN2(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -68,7 +64,6 @@ class TestMatrixPowerOpN2(TestMatrixPowerOp): class TestMatrixPowerOpN3(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -76,7 +71,6 @@ class TestMatrixPowerOpN3(TestMatrixPowerOp): class TestMatrixPowerOpN4(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -84,7 +78,6 @@ class TestMatrixPowerOpN4(TestMatrixPowerOp): class TestMatrixPowerOpN5(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -92,7 +85,6 @@ class TestMatrixPowerOpN5(TestMatrixPowerOp): class TestMatrixPowerOpN6(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -100,7 +92,6 @@ class TestMatrixPowerOpN6(TestMatrixPowerOp): class TestMatrixPowerOpN10(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -108,21 +99,18 @@ class TestMatrixPowerOpN10(TestMatrixPowerOp): class TestMatrixPowerOpNMinus(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" self.n = -1 def test_grad(self): - self.check_grad(["X"], - "Out", - numeric_grad_delta=1e-5, - max_relative_error=1e-6) + self.check_grad( + ["X"], "Out", numeric_grad_delta=1e-5, max_relative_error=1e-6 + ) class TestMatrixPowerOpNMinus2(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -130,7 +118,6 @@ class TestMatrixPowerOpNMinus2(TestMatrixPowerOpNMinus): class TestMatrixPowerOpNMinus3(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -138,7 +125,6 @@ class TestMatrixPowerOpNMinus3(TestMatrixPowerOpNMinus): class TestMatrixPowerOpNMinus4(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -146,7 +132,6 @@ class TestMatrixPowerOpNMinus4(TestMatrixPowerOpNMinus): class TestMatrixPowerOpNMinus5(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -154,7 +139,6 @@ class TestMatrixPowerOpNMinus5(TestMatrixPowerOpNMinus): class TestMatrixPowerOpNMinus6(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -162,21 +146,18 @@ class TestMatrixPowerOpNMinus6(TestMatrixPowerOpNMinus): class TestMatrixPowerOpNMinus10(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" self.n = -10 def test_grad(self): - self.check_grad(["X"], - "Out", - numeric_grad_delta=1e-5, - max_relative_error=1e-6) + self.check_grad( + ["X"], "Out", numeric_grad_delta=1e-5, max_relative_error=1e-6 + ) class TestMatrixPowerOpBatched1(TestMatrixPowerOp): - def config(self): self.matrix_shape = [8, 4, 4] self.dtype = "float64" @@ -184,7 +165,6 @@ class TestMatrixPowerOpBatched1(TestMatrixPowerOp): class TestMatrixPowerOpBatched2(TestMatrixPowerOp): - def config(self): self.matrix_shape = [2, 6, 4, 4] self.dtype = "float64" @@ -192,7 +172,6 @@ class TestMatrixPowerOpBatched2(TestMatrixPowerOp): class TestMatrixPowerOpBatched3(TestMatrixPowerOp): - def config(self): self.matrix_shape = [2, 6, 4, 4] self.dtype = "float64" @@ -200,7 +179,6 @@ class TestMatrixPowerOpBatched3(TestMatrixPowerOp): class TestMatrixPowerOpBatchedLong(TestMatrixPowerOp): - def config(self): self.matrix_shape = [1, 2, 3, 4, 4, 3, 3] self.dtype = "float64" @@ -208,7 +186,6 @@ class TestMatrixPowerOpBatchedLong(TestMatrixPowerOp): class TestMatrixPowerOpLarge1(TestMatrixPowerOp): - def config(self): self.matrix_shape = [32, 32] self.dtype = "float64" @@ -216,7 +193,6 @@ class TestMatrixPowerOpLarge1(TestMatrixPowerOp): class TestMatrixPowerOpLarge2(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -224,7 +200,6 @@ class TestMatrixPowerOpLarge2(TestMatrixPowerOp): class TestMatrixPowerOpFP32(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float32" @@ -235,7 +210,6 @@ class TestMatrixPowerOpFP32(TestMatrixPowerOp): class TestMatrixPowerOpBatchedFP32(TestMatrixPowerOpFP32): - def config(self): self.matrix_shape = [2, 8, 4, 4] self.dtype = "float32" @@ -243,7 +217,6 @@ class TestMatrixPowerOpBatchedFP32(TestMatrixPowerOpFP32): class TestMatrixPowerOpLarge1FP32(TestMatrixPowerOpFP32): - def config(self): self.matrix_shape = [32, 32] self.dtype = "float32" @@ -251,7 +224,6 @@ class TestMatrixPowerOpLarge1FP32(TestMatrixPowerOpFP32): class TestMatrixPowerOpLarge2FP32(TestMatrixPowerOpFP32): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float32" @@ -259,7 +231,6 @@ class TestMatrixPowerOpLarge2FP32(TestMatrixPowerOpFP32): class TestMatrixPowerOpFP32Minus(TestMatrixPowerOpFP32): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float32" @@ -267,7 +238,6 @@ class TestMatrixPowerOpFP32Minus(TestMatrixPowerOpFP32): class TestMatrixPowerAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -282,12 +252,14 @@ class TestMatrixPowerAPI(unittest.TestCase): result_np = np.linalg.matrix_power(input_np, -2) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input_x": input_np}, - fetch_list=[result]) - np.testing.assert_allclose(fetches[0], - np.linalg.matrix_power(input_np, -2), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": input_np}, + fetch_list=[result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.matrix_power(input_np, -2), rtol=1e-05 + ) def test_static(self): for place in self.places: @@ -299,13 +271,14 @@ class TestMatrixPowerAPI(unittest.TestCase): input_np = np.random.random([4, 4]).astype("float64") input = paddle.to_tensor(input_np) result = paddle.linalg.matrix_power(input, -2) - np.testing.assert_allclose(result.numpy(), - np.linalg.matrix_power(input_np, -2), - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), + np.linalg.matrix_power(input_np, -2), + rtol=1e-05, + ) class TestMatrixPowerAPIError(unittest.TestCase): - def test_errors(self): input_np = np.random.random([4, 4]).astype("float64") @@ -314,9 +287,9 @@ class TestMatrixPowerAPIError(unittest.TestCase): # n must be int for n in [2.0, '2', -2.0]: - input = fluid.data(name="input_float32", - shape=[4, 4], - dtype='float32') + input = fluid.data( + name="input_float32", shape=[4, 4], dtype='float32' + ) self.assertRaises(TypeError, paddle.linalg.matrix_power, input, n) # The data type of input must be float32 or float64. @@ -339,7 +312,6 @@ class TestMatrixPowerAPIError(unittest.TestCase): class TestMatrixPowerSingularAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -354,9 +326,11 @@ class TestMatrixPowerSingularAPI(unittest.TestCase): exe = fluid.Executor(place) try: - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) except RuntimeError as ex: print("The mat is singular") except ValueError as ex: diff --git a/python/paddle/fluid/tests/unittests/test_matrix_rank_op.py b/python/paddle/fluid/tests/unittests/test_matrix_rank_op.py index 3c3dba88004..72c104ad85c 100644 --- a/python/paddle/fluid/tests/unittests/test_matrix_rank_op.py +++ b/python/paddle/fluid/tests/unittests/test_matrix_rank_op.py @@ -31,7 +31,6 @@ def matrix_rank_wraper(x, tol=None, use_default_tol=True, hermitian=False): class TestMatrixRankOP(OpTest): - def setUp(self): self.python_api = matrix_rank_wraper self.op_type = "matrix_rank" @@ -58,91 +57,90 @@ class TestMatrixRankOP(OpTest): class TestMatrixRankOP1(TestMatrixRankOP): - def init_data(self): self.x = np.eye(3, k=1, dtype=np.float64) self.tol_tensor = None self.tol = None self.use_default_tol = True self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP2(TestMatrixRankOP): - def init_data(self): self.x = np.random.rand(3, 4, 5, 6).astype(np.float32) self.tol_tensor = np.random.random([3, 4]).astype(self.x.dtype) self.tol = None self.use_default_tol = False self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP3(TestMatrixRankOP): - def init_data(self): self.x = np.eye(200, dtype=np.float64) self.tol_tensor = None self.tol = None self.use_default_tol = True self.hermitian = True - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP4(TestMatrixRankOP): - def init_data(self): self.x = np.random.rand(1, 10).astype(np.float32) self.tol_tensor = None self.tol = None self.use_default_tol = True self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP5(TestMatrixRankOP): - def init_data(self): self.x = np.random.rand(5, 1).astype(np.float64) self.tol_tensor = np.random.random([1, 4]).astype(self.x.dtype) self.tol = None self.use_default_tol = False self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP6(TestMatrixRankOP): - def init_data(self): self.x = np.random.rand(3, 4, 5, 6).astype(np.float32) self.tol_tensor = None self.tol = None self.use_default_tol = False self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP7(TestMatrixRankOP): - def init_data(self): self.x = np.eye(200, dtype=np.float64) self.tol_tensor = np.random.random([200, 200]).astype(self.x.dtype) self.tol = None self.use_default_tol = True self.hermitian = True - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankAPI(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() @@ -177,51 +175,54 @@ class TestMatrixRankAPI(unittest.TestCase): with fluid.program_guard(fluid.Program(), fluid.Program()): x_np = np.random.rand(3, 4, 7, 7).astype(np.float64) tol_np = np.random.random([3, 4]).astype(np.float32) - x_pd = paddle.fluid.data(name="X", - shape=[3, 4, 7, 7], - dtype='float64') - tol_pd = paddle.fluid.data(name="TolTensor", - shape=[3, 4], - dtype='float32') + x_pd = paddle.fluid.data( + name="X", shape=[3, 4, 7, 7], dtype='float64' + ) + tol_pd = paddle.fluid.data( + name="TolTensor", shape=[3, 4], dtype='float32' + ) rank_np = np.linalg.matrix_rank(x_np, tol_np, hermitian=False) - rank_pd = paddle.linalg.matrix_rank(x_pd, - tol_pd, - hermitian=False) + rank_pd = paddle.linalg.matrix_rank( + x_pd, tol_pd, hermitian=False + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "X": x_np, - "TolTensor": tol_np - }, - fetch_list=[rank_pd]) + fetches = exe.run( + fluid.default_main_program(), + feed={"X": x_np, "TolTensor": tol_np}, + fetch_list=[rank_pd], + ) np.testing.assert_allclose(fetches[0], rank_np, rtol=1e-05) for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): x_np = np.random.rand(3, 4, 7, 7).astype(np.float64) - x_pd = paddle.fluid.data(name="X", - shape=[3, 4, 7, 7], - dtype='float64') + x_pd = paddle.fluid.data( + name="X", shape=[3, 4, 7, 7], dtype='float64' + ) rank_np = np.linalg.matrix_rank(x_np, hermitian=True) rank_pd = paddle.linalg.matrix_rank(x_pd, hermitian=True) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"X": x_np}, - fetch_list=[rank_pd]) + fetches = exe.run( + fluid.default_main_program(), + feed={"X": x_np}, + fetch_list=[rank_pd], + ) np.testing.assert_allclose(fetches[0], rank_np, rtol=1e-05) for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): x_np = np.random.rand(3, 4, 7, 7).astype(np.float64) - x_pd = paddle.fluid.data(name="X", - shape=[3, 4, 7, 7], - dtype='float64') + x_pd = paddle.fluid.data( + name="X", shape=[3, 4, 7, 7], dtype='float64' + ) rank_np = np.linalg.matrix_rank(x_np, 0.1, hermitian=False) rank_pd = paddle.linalg.matrix_rank(x_pd, 0.1, hermitian=False) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"X": x_np}, - fetch_list=[rank_pd]) + fetches = exe.run( + fluid.default_main_program(), + feed={"X": x_np}, + fetch_list=[rank_pd], + ) np.testing.assert_allclose(fetches[0], rank_np, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_max_min_amax_amin_op.py b/python/paddle/fluid/tests/unittests/test_max_min_amax_amin_op.py index ee316bb0431..e323d91ba56 100644 --- a/python/paddle/fluid/tests/unittests/test_max_min_amax_amin_op.py +++ b/python/paddle/fluid/tests/unittests/test_max_min_amax_amin_op.py @@ -22,12 +22,14 @@ paddle.enable_static() class TestMaxMinAmaxAminAPI(unittest.TestCase): - def setUp(self): self.init_case() self.cal_np_out_and_gradient() - self.place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def init_case(self): self.x_np = np.array([[0.2, 0.3, 0.5, 0.9], [0.1, 0.2, 0.6, 0.7]]) @@ -40,7 +42,6 @@ class TestMaxMinAmaxAminAPI(unittest.TestCase): # its gradient check is not supported by unittest framework, # thus we calculate the gradient by numpy function. def cal_np_out_and_gradient(self): - def _cal_np_out_and_gradient(func): if func == 'amax': out = np.amax(self.x_np, axis=self.axis, keepdims=self.keepdim) @@ -51,8 +52,10 @@ class TestMaxMinAmaxAminAPI(unittest.TestCase): elif func == 'min': out = np.min(self.x_np, axis=self.axis, keepdims=self.keepdim) else: - print('This unittest only test amax/amin/max/min, but now is', - func) + print( + 'This unittest only test amax/amin/max/min, but now is', + func, + ) self.np_out[func] = out grad = np.zeros(self.shape) out_b = np.broadcast_to(out.view(), self.shape) @@ -86,7 +89,6 @@ class TestMaxMinAmaxAminAPI(unittest.TestCase): # We check the output between paddle API and numpy in static graph. def test_static_graph(self): - def _test_static_graph(func): startup_program = fluid.Program() train_program = fluid.Program() @@ -96,9 +98,11 @@ class TestMaxMinAmaxAminAPI(unittest.TestCase): out = self._choose_paddle_func(func, x) exe = fluid.Executor(self.place) - res = exe.run(fluid.default_main_program(), - feed={'input': self.x_np}, - fetch_list=[out]) + res = exe.run( + fluid.default_main_program(), + feed={'input': self.x_np}, + fetch_list=[out], + ) self.assertTrue((np.array(res[0]) == self.np_out[func]).all()) _test_static_graph('amax') @@ -109,19 +113,18 @@ class TestMaxMinAmaxAminAPI(unittest.TestCase): # As dygraph is easy to compute gradient, we check the gradient between # paddle API and numpy in dygraph. def test_dygraph(self): - def _test_dygraph(func): paddle.disable_static() - x = paddle.to_tensor(self.x_np, - dtype=self.dtype, - stop_gradient=False) + x = paddle.to_tensor( + self.x_np, dtype=self.dtype, stop_gradient=False + ) out = self._choose_paddle_func(func, x) grad_tensor = paddle.ones_like(x) paddle.autograd.backward([out], [grad_tensor], True) - np.testing.assert_allclose(self.np_out[func], - out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + self.np_out[func], out.numpy(), rtol=1e-05 + ) np.testing.assert_allclose(self.np_grad[func], x.grad, rtol=1e-05) paddle.enable_static() @@ -130,10 +133,10 @@ class TestMaxMinAmaxAminAPI(unittest.TestCase): _test_dygraph('max') _test_dygraph('min') - # test two minimum or maximum elements -class TestMaxMinAmaxAminAPI2(TestMaxMinAmaxAminAPI): + +class TestMaxMinAmaxAminAPI2(TestMaxMinAmaxAminAPI): def init_case(self): self.x_np = np.array([[0.2, 0.3, 0.9, 0.9], [0.1, 0.1, 0.6, 0.7]]) self.shape = [2, 4] @@ -144,7 +147,6 @@ class TestMaxMinAmaxAminAPI2(TestMaxMinAmaxAminAPI): # test different axis class TestMaxMinAmaxAminAPI3(TestMaxMinAmaxAminAPI): - def init_case(self): self.x_np = np.array([[0.2, 0.3, 0.9, 0.9], [0.1, 0.1, 0.6, 0.7]]) self.shape = [2, 4] @@ -155,7 +157,6 @@ class TestMaxMinAmaxAminAPI3(TestMaxMinAmaxAminAPI): # test keepdim = True class TestMaxMinAmaxAminAPI4(TestMaxMinAmaxAminAPI): - def init_case(self): self.x_np = np.array([[0.2, 0.3, 0.9, 0.9], [0.1, 0.1, 0.6, 0.7]]) self.shape = [2, 4] @@ -166,10 +167,10 @@ class TestMaxMinAmaxAminAPI4(TestMaxMinAmaxAminAPI): # test axis is tuple class TestMaxMinAmaxAminAPI5(TestMaxMinAmaxAminAPI): - def init_case(self): - self.x_np = np.array([[[1, 2], [3, 4]], [[5, 6], [7, - 8]]]).astype(np.int32) + self.x_np = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]).astype( + np.int32 + ) self.shape = [2, 2, 2] self.dtype = 'int32' self.axis = (0, 1) @@ -178,7 +179,6 @@ class TestMaxMinAmaxAminAPI5(TestMaxMinAmaxAminAPI): # test multiple minimum or maximum elements class TestMaxMinAmaxAminAPI6(TestMaxMinAmaxAminAPI): - def init_case(self): self.x_np = np.array([[0.2, 0.9, 0.9, 0.9], [0.9, 0.9, 0.2, 0.2]]) self.shape = [2, 4] diff --git a/python/paddle/fluid/tests/unittests/test_max_op.py b/python/paddle/fluid/tests/unittests/test_max_op.py index cf52cdb8d47..181343bd5f6 100644 --- a/python/paddle/fluid/tests/unittests/test_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_max_op.py @@ -22,7 +22,6 @@ from test_sum_op import TestReduceOPTensorAxisBase class ApiMaxTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -31,39 +30,43 @@ class ApiMaxTest(unittest.TestCase): def test_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="float32") result_max = paddle.max(x=data, axis=1) exe = paddle.static.Executor(self.place) input_data = np.random.rand(10, 10).astype(np.float32) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_max]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_max]) self.assertEqual((res == np.max(input_data, axis=1)).all(), True) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="int64") result_max = paddle.max(x=data, axis=0) exe = paddle.static.Executor(self.place) input_data = np.random.randint(10, size=(10, 10)).astype(np.int64) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_max]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_max]) self.assertEqual((res == np.max(input_data, axis=0)).all(), True) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="int64") result_max = paddle.max(x=data, axis=(0, 1)) exe = paddle.static.Executor(self.place) input_data = np.random.randint(10, size=(10, 10)).astype(np.int64) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_max]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_max]) self.assertEqual((res == np.max(input_data, axis=(0, 1))).all(), True) def test_errors(self): paddle.enable_static() def test_input_type(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = np.random.rand(10, 10) result_max = paddle.max(x=data, axis=0) @@ -105,17 +108,17 @@ class ApiMaxTest(unittest.TestCase): class TestOutDtype(unittest.TestCase): - def test_max(self): api_fn = paddle.max shape = [10, 16] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64', 'int32', 'int64']) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64', 'int32', 'int64'], + ) class TestMaxWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.max self.np_api = np.max @@ -125,7 +128,6 @@ class TestMaxWithTensorAxis1(TestReduceOPTensorAxisBase): class TestMaxWithTensorAxis2(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.max self.np_api = np.max @@ -134,7 +136,7 @@ class TestMaxWithTensorAxis2(TestReduceOPTensorAxisBase): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] diff --git a/python/paddle/fluid/tests/unittests/test_maximum_op.py b/python/paddle/fluid/tests/unittests/test_maximum_op.py index 98c7aaa262f..0e5ce1c9ff9 100644 --- a/python/paddle/fluid/tests/unittests/test_maximum_op.py +++ b/python/paddle/fluid/tests/unittests/test_maximum_op.py @@ -19,7 +19,6 @@ import paddle.fluid.core as core class ApiMaximumTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -40,56 +39,56 @@ class ApiMaximumTest(unittest.TestCase): def test_static_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_max = paddle.maximum(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_max = paddle.maximum(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.maximum(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.maximum(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 019d49072c4..9334c37f94b 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -28,14 +28,15 @@ np.random.seed(1) def maxout_forward_naive(x, groups, channel_axis): s0, s1, s2, s3 = x.shape if channel_axis == 1: - return np.ndarray([s0, s1 // groups, groups, s2, s3], \ - buffer = x, dtype=x.dtype).max(axis=2) - return np.ndarray([s0, s1, s2, s3 // groups, groups], \ - buffer = x, dtype=x.dtype).max(axis=4) + return np.ndarray( + [s0, s1 // groups, groups, s2, s3], buffer=x, dtype=x.dtype + ).max(axis=2) + return np.ndarray( + [s0, s1, s2, s3 // groups, groups], buffer=x, dtype=x.dtype + ).max(axis=4) class TestMaxOutOp(OpTest): - def setUp(self): self.op_type = "maxout" self.python_api = paddle.nn.functional.maxout @@ -63,25 +64,21 @@ class TestMaxOutOp(OpTest): class TestMaxOutOpAxis0(TestMaxOutOp): - def set_attrs(self): self.axis = -1 class TestMaxOutOpAxis1(TestMaxOutOp): - def set_attrs(self): self.axis = 3 class TestMaxOutOpFP32(TestMaxOutOp): - def set_attrs(self): self.dtype = 'float32' class TestMaxOutOpGroups(TestMaxOutOp): - def set_attrs(self): self.groups = 3 @@ -92,8 +89,11 @@ class TestMaxoutAPI(unittest.TestCase): self.x_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float64) self.groups = 2 self.axis = 1 - self.place=paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -142,9 +142,9 @@ class TestMaxoutAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.maxout, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 4, 6, 8], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 4, 6, 8], dtype='int32' + ) self.assertRaises(TypeError, F.maxout, x_int32) x_float32 = paddle.fluid.data(name='x_float32', shape=[2, 4, 6, 8]) diff --git a/python/paddle/fluid/tests/unittests/test_mean_iou.py b/python/paddle/fluid/tests/unittests/test_mean_iou.py index d55d1507f96..3c78395755f 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_iou.py +++ b/python/paddle/fluid/tests/unittests/test_mean_iou.py @@ -18,8 +18,9 @@ from op_test import OpTest import paddle.fluid as fluid -def compute_mean_iou(predictions, labels, num_classes, in_wrongs, in_corrects, - in_mean_ious): +def compute_mean_iou( + predictions, labels, num_classes, in_wrongs, in_corrects, in_mean_ious +): assert predictions.shape == labels.shape predictions = predictions.flatten() labels = labels.flatten() @@ -40,8 +41,9 @@ def compute_mean_iou(predictions, labels, num_classes, in_wrongs, in_corrects, denominator = out_wrong + out_correct valid_count = (denominator != 0).sum() - denominator = np.where(denominator > 0, denominator, - np.ones(denominator.shape)) + denominator = np.where( + denominator > 0, denominator, np.ones(denominator.shape) + ) mean_iou = (out_correct / denominator).sum() / valid_count for _, in_mean_iou in in_mean_ious: @@ -50,48 +52,67 @@ def compute_mean_iou(predictions, labels, num_classes, in_wrongs, in_corrects, class TestMeanIOUOp(OpTest): - def setUp(self): self.config() self.op_type = "mean_iou" - predictions = np.random.randint(0, self.num_classes, - self.image_size).astype("int32") - labels = np.random.randint(0, self.num_classes, - self.image_size).astype("int32") + predictions = np.random.randint( + 0, self.num_classes, self.image_size + ).astype("int32") + labels = np.random.randint(0, self.num_classes, self.image_size).astype( + "int32" + ) in_wrongs = [] for i in range(self.in_wrong_num): in_wrongs.append( - ("in_wrong_%d" % i, - np.random.randint(0, 10, [self.num_classes]).astype("int32"))) + ( + "in_wrong_%d" % i, + np.random.randint(0, 10, [self.num_classes]).astype( + "int32" + ), + ) + ) in_corrects = [] for i in range(self.in_correct_num): in_corrects.append( - ("in_correct_%d" % i, - np.random.randint(0, 10, [self.num_classes]).astype("int32"))) + ( + "in_correct_%d" % i, + np.random.randint(0, 10, [self.num_classes]).astype( + "int32" + ), + ) + ) in_mean_ious = [] for i in range(self.in_mean_iou_num): - in_mean_ious.append(("in_mean_iou_%d" % i, - np.random.uniform(0, 1, - [1]).astype("float32"))) + in_mean_ious.append( + ( + "in_mean_iou_%d" % i, + np.random.uniform(0, 1, [1]).astype("float32"), + ) + ) self.inputs = { 'Predictions': predictions, 'Labels': labels, 'InWrongs': in_wrongs, 'InCorrects': in_corrects, - 'InMeanIou': in_mean_ious + 'InMeanIou': in_mean_ious, } self.attrs = {'num_classes': int(self.num_classes)} mean_iou, out_wrong, out_correct = compute_mean_iou( - predictions, labels, self.num_classes, in_wrongs, in_corrects, - in_mean_ious) + predictions, + labels, + self.num_classes, + in_wrongs, + in_corrects, + in_mean_ious, + ) self.outputs = { 'OutMeanIou': mean_iou, 'OutWrong': out_wrong, - 'OutCorrect': out_correct + 'OutCorrect': out_correct, } def config(self): @@ -106,7 +127,6 @@ class TestMeanIOUOp(OpTest): class TestCase1(TestMeanIOUOp): - def config(self): self.num_classes = 5 self.image_size = [100, 128] @@ -121,14 +141,15 @@ class TestCase1(TestMeanIOUOp): class TestMeanIOUOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.mean_iou, x1, y1) # The input dtype of accuracy_op must be float32 or float64. x2 = fluid.layers.data(name='x2', shape=[4], dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_mean_op.py b/python/paddle/fluid/tests/unittests/test_mean_op.py index 6ba980879a6..ed9313b0546 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_op.py +++ b/python/paddle/fluid/tests/unittests/test_mean_op.py @@ -40,7 +40,6 @@ def reduce_mean_wrapper(x, axis=0, keepdim=False, reduce_all=False): class TestMeanOp(OpTest): - def setUp(self): self.op_type = "mean" self.python_api = paddle.mean @@ -60,27 +59,26 @@ class TestMeanOp(OpTest): class TestMeanOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of mean_op must be Variable. input1 = 12 self.assertRaises(TypeError, paddle.mean, input1) # The input dtype of mean_op must be float16, float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, paddle.mean, input2) - input3 = fluid.layers.data(name='input3', - shape=[4], - dtype="float16") + input3 = fluid.layers.data( + name='input3', shape=[4], dtype="float16" + ) fluid.layers.softmax(input3) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16MeanOp(TestMeanOp): - def init_dtype_type(self): self.dtype = np.float16 self.__class__.no_need_check_grad = True @@ -100,13 +98,13 @@ class TestFP16MeanOp(TestMeanOp): y = paddle.mean(x) dx = paddle.grad(y, x)[0].numpy() dx_expected = self.dtype(1.0 / np.prod(x_np.shape)) * np.ones( - x_np.shape).astype(self.dtype) + x_np.shape + ).astype(self.dtype) np.testing.assert_array_equal(dx, dx_expected) @OpTestTool.skip_if_not_cpu_bf16() class TestBF16MeanOp(TestMeanOp): - def init_dtype_type(self): self.dtype = np.uint16 @@ -136,7 +134,6 @@ def ref_reduce_mean_grad(x, axis, dtype): class TestReduceMeanOp(OpTest): - def setUp(self): self.op_type = 'reduce_mean' self.python_api = reduce_mean_wrapper @@ -157,7 +154,7 @@ class TestReduceMeanOp(OpTest): self.attrs = { 'dim': self.axis, 'keep_dim': self.keepdim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } if self.dtype == 'float16': @@ -187,18 +184,17 @@ class TestReduceMeanOp(OpTest): return with fluid.dygraph.guard(place=place): x = paddle.tensor(self.inputs['X']) - y = paddle.mean(x, - axis=self.attrs['dim'], - keepdim=self.attrs['keep_dim']) + y = paddle.mean( + x, axis=self.attrs['dim'], keepdim=self.attrs['keep_dim'] + ) dx = paddle.grad(y, x)[0].numpy() - dx_expected = ref_reduce_mean_grad(self.inputs['X'], - self.attrs['dim'], - self.dtype) + dx_expected = ref_reduce_mean_grad( + self.inputs['X'], self.attrs['dim'], self.dtype + ) np.testing.assert_array_equal(dx, dx_expected) class TestReduceMeanOpDefaultAttrs(TestReduceMeanOp): - def setUp(self): self.op_type = 'reduce_mean' self.python_api = reduce_mean_wrapper @@ -212,104 +208,88 @@ class TestReduceMeanOpDefaultAttrs(TestReduceMeanOp): class TestReduceMeanOpFloat32(TestReduceMeanOp): - def set_attrs(self): self.dtype = 'float32' class TestReduceMeanOpFloat16(TestReduceMeanOp): - def set_attrs(self): self.dtype = 'float16' class TestReduceMeanOpShape1D(TestReduceMeanOp): - def set_attrs(self): self.shape = [100] class TestReduceMeanOpShape1DFP16(TestReduceMeanOp): - def set_attrs(self): self.shape = [100] self.dtype = 'float16' class TestReduceMeanOpShape6D(TestReduceMeanOp): - def set_attrs(self): self.shape = [2, 3, 4, 5, 6, 7] class TestReduceMeanOpShape6DFP16(TestReduceMeanOp): - def set_attrs(self): self.shape = [2, 3, 4, 5, 6, 7] self.dtype = 'float16' class TestReduceMeanOpAxisAll(TestReduceMeanOp): - def set_attrs(self): self.axis = [0, 1, 2, 3] class TestReduceMeanOpAxisAllFP16(TestReduceMeanOp): - def set_attrs(self): self.axis = [0, 1, 2, 3] self.dtype = 'float16' class TestReduceMeanOpAxisTuple(TestReduceMeanOp): - def set_attrs(self): self.axis = (0, 1, 2) class TestReduceMeanOpAxisTupleFP16(TestReduceMeanOp): - def set_attrs(self): self.axis = (0, 1, 2) self.dtype = 'float16' class TestReduceMeanOpAxisNegative(TestReduceMeanOp): - def set_attrs(self): self.axis = [-2, -1] class TestReduceMeanOpAxisNegativeFP16(TestReduceMeanOp): - def set_attrs(self): self.axis = [-2, -1] self.dtype = 'float16' class TestReduceMeanOpKeepdimTrue1(TestReduceMeanOp): - def set_attrs(self): self.keepdim = True class TestReduceMeanOpKeepdimTrue1FP16(TestReduceMeanOp): - def set_attrs(self): self.keepdim = True self.dtype = 'float16' class TestReduceMeanOpKeepdimTrue2(TestReduceMeanOp): - def set_attrs(self): self.axis = [0, 1, 2, 3] self.keepdim = True class TestReduceMeanOpKeepdimTrue2FP16(TestReduceMeanOp): - def set_attrs(self): self.axis = [0, 1, 2, 3] self.keepdim = True @@ -317,13 +297,11 @@ class TestReduceMeanOpKeepdimTrue2FP16(TestReduceMeanOp): class TestReduceMeanOpReduceAllTrue(TestReduceMeanOp): - def set_attrs(self): self.reduce_all = True class TestReduceMeanOpReduceAllTrueFP16(TestReduceMeanOp): - def set_attrs(self): self.reduce_all = True self.dtype = 'float16' @@ -335,8 +313,11 @@ class TestMeanAPI(unittest.TestCase): def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_api_static(self): paddle.enable_static() @@ -350,8 +331,9 @@ class TestMeanAPI(unittest.TestCase): out5 = paddle.mean(x, tuple(axis)) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': self.x}, - fetch_list=[out1, out2, out3, out4, out5]) + res = exe.run( + feed={'X': self.x}, fetch_list=[out1, out2, out3, out4, out5] + ) out_ref = np.mean(self.x) for out in res: np.testing.assert_allclose(out, out_ref, rtol=0.0001) @@ -393,9 +375,9 @@ class TestMeanAPI(unittest.TestCase): x_np = np.random.rand(10, 10).astype(np.float32) x = fluid.dygraph.to_variable(x_np) out = fluid.layers.reduce_mean(input=x, dim=1) - np.testing.assert_allclose(out.numpy(), - np.mean(x_np, axis=1), - rtol=1e-05) + np.testing.assert_allclose( + out.numpy(), np.mean(x_np, axis=1), rtol=1e-05 + ) def test_errors(self): paddle.disable_static() @@ -410,7 +392,6 @@ class TestMeanAPI(unittest.TestCase): class TestMeanWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.mean self.np_api = np.mean @@ -420,7 +401,6 @@ class TestMeanWithTensorAxis1(TestReduceOPTensorAxisBase): class TestMeanWithTensorAxis2(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.mean self.np_api = np.mean @@ -429,12 +409,11 @@ class TestMeanWithTensorAxis2(TestReduceOPTensorAxisBase): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] class TestMeanDoubleGradCheck(unittest.TestCase): - def mean_wrapper(self, x): return paddle.mean(x[0]) @@ -449,17 +428,13 @@ class TestMeanDoubleGradCheck(unittest.TestCase): out = paddle.mean(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.mean_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.mean_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -471,7 +446,6 @@ class TestMeanDoubleGradCheck(unittest.TestCase): class TestMeanTripleGradCheck(unittest.TestCase): - def mean_wrapper(self, x): return paddle.mean(x[0]) @@ -486,17 +460,13 @@ class TestMeanTripleGradCheck(unittest.TestCase): out = paddle.mean(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.mean_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.mean_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_median.py b/python/paddle/fluid/tests/unittests/test_median.py index 8f33e4713d2..c717293b4cd 100644 --- a/python/paddle/fluid/tests/unittests/test_median.py +++ b/python/paddle/fluid/tests/unittests/test_median.py @@ -21,7 +21,6 @@ DELTA = 1e-6 class TestMedian(unittest.TestCase): - def check_numpy_res(self, np1, np2): self.assertEqual(np1.shape, np2.shape) mismatch = np.sum((np1 - np2) * (np1 - np2)) @@ -56,8 +55,11 @@ class TestMedian(unittest.TestCase): w = 4 l = 2 x = np.arange(h * w * l).reshape([h, w, l]) - lis_tests = [[x, axis, keepdims] for axis in [-1, 0, 1, 2, None] - for keepdims in [False, True]] + lis_tests = [ + [x, axis, keepdims] + for axis in [-1, 0, 1, 2, None] + for keepdims in [False, True] + ] for lis_test in lis_tests: self.static_single_test_median(lis_test) @@ -67,8 +69,11 @@ class TestMedian(unittest.TestCase): w = 4 l = 2 x = np.arange(h * w * l).reshape([h, w, l]) - lis_tests = [[x, axis, keepdims] for axis in [-1, 0, 1, 2, None] - for keepdims in [False, True]] + lis_tests = [ + [x, axis, keepdims] + for axis in [-1, 0, 1, 2, None] + for keepdims in [False, True] + ] for lis_test in lis_tests: self.dygraph_single_test_median(lis_test) diff --git a/python/paddle/fluid/tests/unittests/test_memcpy_op.py b/python/paddle/fluid/tests/unittests/test_memcpy_op.py index 8ab07ad9583..65a246d9636 100755 --- a/python/paddle/fluid/tests/unittests/test_memcpy_op.py +++ b/python/paddle/fluid/tests/unittests/test_memcpy_op.py @@ -21,7 +21,6 @@ from paddle.fluid import Program, program_guard class TestMemcpy_FillConstant(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = Program() @@ -33,20 +32,25 @@ class TestMemcpy_FillConstant(unittest.TestCase): shape=[10, 10], dtype='float32', persistable=False, - stop_gradient=True) - gpu_var = main_program.global_block().create_var(name=gpu_var_name, - shape=[10, 10], - dtype='float32', - persistable=False, - stop_gradient=True) - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": gpu_var_name}, - attrs={ - "shape": [10, 10], - "dtype": gpu_var.dtype, - "value": 1.0, - "place_type": 1 - }) + stop_gradient=True, + ) + gpu_var = main_program.global_block().create_var( + name=gpu_var_name, + shape=[10, 10], + dtype='float32', + persistable=False, + stop_gradient=True, + ) + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": gpu_var_name}, + attrs={ + "shape": [10, 10], + "dtype": gpu_var.dtype, + "value": 1.0, + "place_type": 1, + }, + ) main_program.global_block().append_op( type="fill_constant", outputs={"Out": pinned_var_name}, @@ -54,35 +58,40 @@ class TestMemcpy_FillConstant(unittest.TestCase): "shape": [10, 10], "dtype": gpu_var.dtype, "value": 0.0, - "place_type": 2 - }) + "place_type": 2, + }, + ) return main_program, gpu_var, pinned_var def test_gpu_copy_to_pinned(self): main_program, gpu_var, pinned_var = self.get_prog() - main_program.global_block().append_op(type='memcpy', - inputs={'X': gpu_var}, - outputs={'Out': pinned_var}, - attrs={'dst_place_type': 2}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': gpu_var}, + outputs={'Out': pinned_var}, + attrs={'dst_place_type': 2}, + ) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - gpu_, pinned_ = exe.run(main_program, - feed={}, - fetch_list=[gpu_var.name, pinned_var.name]) + gpu_, pinned_ = exe.run( + main_program, feed={}, fetch_list=[gpu_var.name, pinned_var.name] + ) np.testing.assert_allclose(gpu_, pinned_, rtol=1e-05) np.testing.assert_allclose(pinned_, np.ones((10, 10)), rtol=1e-05) def test_pinned_copy_gpu(self): main_program, gpu_var, pinned_var = self.get_prog() - main_program.global_block().append_op(type='memcpy', - inputs={'X': pinned_var}, - outputs={'Out': gpu_var}, - attrs={'dst_place_type': 1}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': pinned_var}, + outputs={'Out': gpu_var}, + attrs={'dst_place_type': 1}, + ) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - gpu_, pinned_ = exe.run(main_program, - feed={}, - fetch_list=[gpu_var.name, pinned_var.name]) + gpu_, pinned_ = exe.run( + main_program, feed={}, fetch_list=[gpu_var.name, pinned_var.name] + ) np.testing.assert_allclose(gpu_, pinned_, rtol=1e-05) np.testing.assert_allclose(gpu_, np.zeros((10, 10)), rtol=1e-05) @@ -98,13 +107,15 @@ class TestMemcpy_FillConstant(unittest.TestCase): shape=[1], dtype='bool', persistable=False, - stop_gradient=True) + stop_gradient=True, + ) gpu_var = main_program.global_block().create_var( name=gpu_var_name, shape=[1], dtype='bool', persistable=False, - stop_gradient=True) + stop_gradient=True, + ) main_program.global_block().append_op( type="fill_constant", outputs={"Out": gpu_var_name}, @@ -112,8 +123,9 @@ class TestMemcpy_FillConstant(unittest.TestCase): "shape": [1], "dtype": gpu_var.dtype, "value": False, - "place_type": 1 - }) + "place_type": 1, + }, + ) main_program.global_block().append_op( type="fill_constant", outputs={"Out": pinned_var_name}, @@ -121,18 +133,23 @@ class TestMemcpy_FillConstant(unittest.TestCase): "shape": [1], "dtype": gpu_var.dtype, "value": True, - "place_type": 2 - }) + "place_type": 2, + }, + ) - main_program.global_block().append_op(type='memcpy', - inputs={'X': pinned_var}, - outputs={'Out': gpu_var}, - attrs={'dst_place_type': 1}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': pinned_var}, + outputs={'Out': gpu_var}, + attrs={'dst_place_type': 1}, + ) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - gpu_, pinned_ = exe.run(main_program, - feed={}, - fetch_list=[gpu_var.name, pinned_var.name]) + gpu_, pinned_ = exe.run( + main_program, + feed={}, + fetch_list=[gpu_var.name, pinned_var.name], + ) expect_value = np.array([1]).astype('bool') np.testing.assert_array_equal(gpu_, expect_value) else: @@ -140,7 +157,6 @@ class TestMemcpy_FillConstant(unittest.TestCase): class TestMemcpyOPError(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = Program() @@ -150,7 +166,8 @@ class TestMemcpyOPError(unittest.TestCase): shape=[10, 10], dtype='float32', persistable=False, - stop_gradient=True) + stop_gradient=True, + ) main_program.global_block().append_op( type="fill_constant", outputs={"Out": "tensor@Pinned_0"}, @@ -158,41 +175,47 @@ class TestMemcpyOPError(unittest.TestCase): "shape": [10, 10], "dtype": pinned_var.dtype, "value": 0.0, - "place_type": 2 - }) + "place_type": 2, + }, + ) return main_program, pinned_var def test_SELECTED_ROWS(self): main_program, pinned_var = self.get_prog() - selected_row_var = main_program.global_block().create_var( \ - name="selected_row_0", dtype="float32", persistable=False, \ - type=fluid.core.VarDesc.VarType.SELECTED_ROWS, stop_gradient=True) - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": selected_row_var}, - attrs={ - "shape": - selected_row_var.shape, - "dtype": - selected_row_var.dtype, - "value": 1.0, - "place_type": 1 - }) + selected_row_var = main_program.global_block().create_var( + name="selected_row_0", + dtype="float32", + persistable=False, + type=fluid.core.VarDesc.VarType.SELECTED_ROWS, + stop_gradient=True, + ) + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": selected_row_var}, + attrs={ + "shape": selected_row_var.shape, + "dtype": selected_row_var.dtype, + "value": 1.0, + "place_type": 1, + }, + ) with self.assertRaises(RuntimeError): main_program.global_block().append_op( type='memcpy', inputs={'X': selected_row_var}, outputs={'Out': pinned_var}, - attrs={'dst_place_type': 2}) + attrs={'dst_place_type': 2}, + ) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) selected_row_var_, pinned_ = exe.run( main_program, feed={}, - fetch_list=[selected_row_var.name, pinned_var.name]) + fetch_list=[selected_row_var.name, pinned_var.name], + ) class TestMemcpyApi(unittest.TestCase): - def test_api(self): a = paddle.ones([1024, 1024]) b = paddle.tensor.creation._memcpy(a, paddle.CUDAPinnedPlace()) diff --git a/python/paddle/fluid/tests/unittests/test_memory_analysis.py b/python/paddle/fluid/tests/unittests/test_memory_analysis.py index 1672e7371cb..8f38433d20c 100644 --- a/python/paddle/fluid/tests/unittests/test_memory_analysis.py +++ b/python/paddle/fluid/tests/unittests/test_memory_analysis.py @@ -14,12 +14,14 @@ import unittest import paddle -from paddle.fluid.memory_analysis import pre_allocate_memory, get_max_memory_info +from paddle.fluid.memory_analysis import ( + pre_allocate_memory, + get_max_memory_info, +) from simple_nets import simple_fc_net class TestMemoryAnalysis(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -28,18 +30,19 @@ class TestMemoryAnalysis(unittest.TestCase): optimizer = paddle.optimizer.Adam(learning_rate=1e-3) optimizer.minimize(loss) main_prog = paddle.static.default_main_program() - max_tmp_mem_1, max_persitable_mem_1 = get_max_memory_info(main_prog, - batch_size=32) + max_tmp_mem_1, max_persitable_mem_1 = get_max_memory_info( + main_prog, batch_size=32 + ) self.assertGreater(max_tmp_mem_1, 0) self.assertGreater(max_persitable_mem_1, 0) - max_tmp_mem_2, max_persitable_mem_2 = get_max_memory_info(main_prog, - batch_size=64) + max_tmp_mem_2, max_persitable_mem_2 = get_max_memory_info( + main_prog, batch_size=64 + ) self.assertEqual(max_persitable_mem_1, max_persitable_mem_2) self.assertLess(max_tmp_mem_1, max_tmp_mem_2) class TestPreAllocateMemory(unittest.TestCase): - def setUp(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_memory_reuse_exclude_feed_var.py b/python/paddle/fluid/tests/unittests/test_memory_reuse_exclude_feed_var.py index 7b335bf83d6..839c21b151a 100644 --- a/python/paddle/fluid/tests/unittests/test_memory_reuse_exclude_feed_var.py +++ b/python/paddle/fluid/tests/unittests/test_memory_reuse_exclude_feed_var.py @@ -18,15 +18,14 @@ import unittest class TestMemoryReuseExcludeFeedVar(unittest.TestCase): - def setUp(self): self.image_shape = [28, 28] self.iteration = 10 def main_impl(self, place): - image = fluid.layers.data(name='image', - shape=self.image_shape, - dtype='float32') + image = fluid.layers.data( + name='image', shape=self.image_shape, dtype='float32' + ) relu_image = fluid.layers.relu(image) loss = fluid.layers.reduce_mean(relu_image) @@ -38,12 +37,13 @@ class TestMemoryReuseExcludeFeedVar(unittest.TestCase): exe.run(fluid.default_startup_program()) compiled_prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + fluid.default_main_program() + ).with_data_parallel(loss_name=loss.name, build_strategy=build_strategy) image_tensor = fluid.LoDTensor() - np_image = np.random.uniform(low=-10, high=10, - size=self.image_shape).astype('float32') + np_image = np.random.uniform( + low=-10, high=10, size=self.image_shape + ).astype('float32') image_tensor.set(np_image, place) feed_dict = [{image.name: image_tensor}] diff --git a/python/paddle/fluid/tests/unittests/test_memory_usage.py b/python/paddle/fluid/tests/unittests/test_memory_usage.py index 2ddfd1ceb02..8c41d0853fc 100644 --- a/python/paddle/fluid/tests/unittests/test_memory_usage.py +++ b/python/paddle/fluid/tests/unittests/test_memory_usage.py @@ -20,8 +20,10 @@ import unittest def train_simulator(test_batch_size=10): if test_batch_size <= 0: - raise ValueError("batch_size should be a positive integeral value, " - "but got batch_size={}".format(test_batch_size)) + raise ValueError( + "batch_size should be a positive integeral value, " + "but got batch_size={}".format(test_batch_size) + ) x = fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None) @@ -35,14 +37,16 @@ def train_simulator(test_batch_size=10): # Calculate memory usage in current network config lower_usage, upper_usage, unit = fluid.contrib.memory_usage( - fluid.default_main_program(), batch_size=test_batch_size) + fluid.default_main_program(), batch_size=test_batch_size + ) - print("memory usage is about %.3f - %.3f %s" % - (lower_usage, upper_usage, unit)) + print( + "memory usage is about %.3f - %.3f %s" + % (lower_usage, upper_usage, unit) + ) class TestMemoryUsage(unittest.TestCase): - def test_with_unit_B(self): with self.program_scope_guard(): train_simulator() diff --git a/python/paddle/fluid/tests/unittests/test_merge_selectedrows_op.py b/python/paddle/fluid/tests/unittests/test_merge_selectedrows_op.py index 65fd3b9e405..a22571289b2 100644 --- a/python/paddle/fluid/tests/unittests/test_merge_selectedrows_op.py +++ b/python/paddle/fluid/tests/unittests/test_merge_selectedrows_op.py @@ -19,7 +19,6 @@ from paddle.fluid.op import Operator class TestMergeSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_merged_adam_op.py b/python/paddle/fluid/tests/unittests/test_merged_adam_op.py index c6aadeda5fc..49d1cf4bd79 100644 --- a/python/paddle/fluid/tests/unittests/test_merged_adam_op.py +++ b/python/paddle/fluid/tests/unittests/test_merged_adam_op.py @@ -19,20 +19,22 @@ from paddle import _C_ops, _legacy_C_ops from paddle.fluid.framework import in_dygraph_mode -def run_adam_op(params, - grads, - lrs, - moment1s, - moment2s, - beta1_pows, - beta2_pows, - master_params, - epsilon, - beta1, - beta2, - place, - multi_precision=False, - use_merged=False): +def run_adam_op( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + epsilon, + beta1, + beta2, + place, + multi_precision=False, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(lrs) assert len(params) == len(moment1s) @@ -57,25 +59,71 @@ def run_adam_op(params, if not use_merged: for i in range(len(param_vars)): _, _, _, _, _, _ = _legacy_C_ops.adam( - param_vars[i], grad_vars[i], lr_vars[i], moment1_vars[i], - moment2_vars[i], beta1_pow_vars[i], beta2_pow_vars[i], - master_param_vars[i], param_vars[i], moment1_vars[i], - moment2_vars[i], beta1_pow_vars[i], beta2_pow_vars[i], - master_param_vars[i], 'epsilon', epsilon, 'beta1', beta1, - 'beta2', beta2, 'multi_precision', multi_precision) + param_vars[i], + grad_vars[i], + lr_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + param_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) else: if in_dygraph_mode(): _, _, _, _, _, _ = _C_ops.merged_adam_( - param_vars, grad_vars, lr_vars, moment1_vars, moment2_vars, - beta1_pow_vars, beta2_pow_vars, master_param_vars, beta1, beta2, - epsilon, multi_precision, False) + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + beta1, + beta2, + epsilon, + multi_precision, + False, + ) else: _, _, _, _, _, _ = _legacy_C_ops.merged_adam( - param_vars, grad_vars, lr_vars, moment1_vars, moment2_vars, - beta1_pow_vars, beta2_pow_vars, master_param_vars, param_vars, - moment1_vars, moment2_vars, beta1_pow_vars, beta2_pow_vars, - master_param_vars, 'epsilon', epsilon, 'beta1', beta1, 'beta2', - beta2, 'multi_precision', multi_precision) + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + param_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) outputs = { 'ParamOut': param_vars, @@ -83,14 +131,13 @@ def run_adam_op(params, 'Moment2Out': moment2_vars, 'Beta1PowOut': beta1_pow_vars, 'Beta2PowOut': beta2_pow_vars, - 'MasterParamOut': master_param_vars + 'MasterParamOut': master_param_vars, } return outputs class TestMergedAdam(unittest.TestCase): - def setUp(self): paddle.disable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -111,27 +158,46 @@ class TestMergedAdam(unittest.TestCase): beta1_pows = self.gen_rand_data([[1], [1], [1], [1]], mp_dtype) beta2_pows = self.gen_rand_data([[1], [1], [1], [1]], mp_dtype) master_params = [p.astype(mp_dtype) for p in params] - return params, grads, lrs, moment1s, moment2s, beta1_pows, beta2_pows, master_params + return ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) def check_with_place(self, place, multi_precision): - params, grads, lrs, moment1s, moment2s, beta1_pows, beta2_pows, master_params = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): - return run_adam_op(params=params, - grads=grads, - lrs=lrs, - moment1s=moment1s, - moment2s=moment2s, - beta1_pows=beta1_pows, - beta2_pows=beta2_pows, - master_params=master_params, - epsilon=0.9, - beta1=0.9, - beta2=0.99, - place=place, - multi_precision=multi_precision, - use_merged=use_merged) + return run_adam_op( + params=params, + grads=grads, + lrs=lrs, + moment1s=moment1s, + moment2s=moment2s, + beta1_pows=beta1_pows, + beta2_pows=beta2_pows, + master_params=master_params, + epsilon=0.9, + beta1=0.9, + beta2=0.99, + place=place, + multi_precision=multi_precision, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -144,10 +210,9 @@ class TestMergedAdam(unittest.TestCase): if place == 'gpu': np.testing.assert_array_equal(value1[i], value2[i]) else: - np.testing.assert_allclose(value1[i], - value2[i], - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + value1[i], value2[i], rtol=1e-05, atol=1e-07 + ) def get_places(self): places = ['cpu'] diff --git a/python/paddle/fluid/tests/unittests/test_merged_momentum_op.py b/python/paddle/fluid/tests/unittests/test_merged_momentum_op.py index 466ee3afd58..dcba88284fc 100644 --- a/python/paddle/fluid/tests/unittests/test_merged_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_merged_momentum_op.py @@ -19,16 +19,18 @@ from paddle.fluid.layer_helper import LayerHelper from collections import OrderedDict -def run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False): +def run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -45,48 +47,70 @@ def run_momentum_op(params, } param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -96,8 +120,9 @@ def run_momentum_op(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -108,10 +133,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars[i] outputs['MasterParamOut'] = master_param_vars[i] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -123,10 +147,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -134,17 +157,19 @@ def run_momentum_op(params, return exe.run(main, feed=feed_dict, fetch_list=fetch_list) -def run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False, - use_nesterov=True): +def run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, + use_nesterov=True, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -156,48 +181,70 @@ def run_momentum_op2(params, helper = LayerHelper(op_type, **locals()) param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -207,8 +254,9 @@ def run_momentum_op2(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -227,10 +275,9 @@ def run_momentum_op2(params, 'regularization_method': 'l2_decay', 'regularization_coeff': 2.0, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -243,22 +290,18 @@ def run_momentum_op2(params, inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars attrs = { - 'mu': - mu, - 'multi_precision': - multi_precision, - 'rescale_grad': - rescale_grad, - 'use_nesterov': - use_nesterov, - 'regularization_method': - ['l2_decay' for i in range(len(param_vars))], + 'mu': mu, + 'multi_precision': multi_precision, + 'rescale_grad': rescale_grad, + 'use_nesterov': use_nesterov, + 'regularization_method': [ + 'l2_decay' for i in range(len(param_vars)) + ], 'regularization_coeff': [2.0 for i in range(len(param_vars))], } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -267,7 +310,6 @@ def run_momentum_op2(params, class TestMergedMomentum(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -280,8 +322,11 @@ class TestMergedMomentum(unittest.TestCase): def prepare_data(self, shapes, multi_precision, seed, place): np.random.seed(seed) mp_dtype = np.float32 - dtype = np.float16 if multi_precision and isinstance( - place, paddle.CUDAPlace) else np.float32 + dtype = ( + np.float16 + if multi_precision and isinstance(place, paddle.CUDAPlace) + else np.float32 + ) params = self.gen_rand_data(shapes, dtype) grads = self.gen_rand_data(shapes, dtype) velocitys = self.gen_rand_data(shapes, mp_dtype) @@ -293,21 +338,28 @@ class TestMergedMomentum(unittest.TestCase): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): # FIXME(zengjinle): CPU Momentum Op does not support rescale_grad rescale_grad = 1.0 if isinstance(place, paddle.CPUPlace) else 0.01 - return run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged) + return run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -331,7 +383,6 @@ class TestMergedMomentum(unittest.TestCase): class TestMergedMomentum2(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -343,8 +394,11 @@ class TestMergedMomentum2(unittest.TestCase): def prepare_data(self, shapes, multi_precision, seed, place): np.random.seed(seed) mp_dtype = np.float32 - dtype = np.float16 if multi_precision and isinstance( - place, paddle.CUDAPlace) else np.float32 + dtype = ( + np.float16 + if multi_precision and isinstance(place, paddle.CUDAPlace) + else np.float32 + ) params = self.gen_rand_data(shapes, dtype) grads = self.gen_rand_data(shapes, dtype) velocitys = self.gen_rand_data(shapes, mp_dtype) @@ -356,22 +410,29 @@ class TestMergedMomentum2(unittest.TestCase): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_nesterov, use_merged): # FIXME(zengjinle): CPU Momentum Op does not support rescale_grad rescale_grad = 1.0 if isinstance(place, paddle.CPUPlace) else 0.01 - return run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged, - use_nesterov=use_nesterov) + return run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + use_nesterov=use_nesterov, + ) outs1 = run_op(use_nesterov=True, use_merged=True) outs2 = run_op(use_nesterov=True, use_merged=False) diff --git a/python/paddle/fluid/tests/unittests/test_meshgrid_op.py b/python/paddle/fluid/tests/unittests/test_meshgrid_op.py index f49ac0d1d52..3c3da16c958 100644 --- a/python/paddle/fluid/tests/unittests/test_meshgrid_op.py +++ b/python/paddle/fluid/tests/unittests/test_meshgrid_op.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestMeshgridOp(OpTest): - def setUp(self): self.op_type = "meshgrid" self.dtype = self.get_dtype() @@ -46,7 +45,7 @@ class TestMeshgridOp(OpTest): ins = [] outs = [] for i in range(len(self.shape)): - ins.append(np.random.random((self.shape[i], )).astype(self.dtype)) + ins.append(np.random.random((self.shape[i],)).astype(self.dtype)) for i in range(len(self.shape)): out_reshape = [1] * len(self.shape) @@ -60,23 +59,29 @@ class TestMeshgridOp(OpTest): class TestMeshgridOp2(TestMeshgridOp): - def get_x_shape(self): return [100, 300] class TestMeshgridOp3(unittest.TestCase): - def test_api(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -85,28 +90,34 @@ class TestMeshgridOp3(unittest.TestCase): exe = fluid.Executor(place=fluid.CPUPlace()) grid_x, grid_y = paddle.tensor.meshgrid(x, y) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp4(unittest.TestCase): - def test_list_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -115,29 +126,35 @@ class TestMeshgridOp4(unittest.TestCase): exe = fluid.Executor(place=fluid.CPUPlace()) grid_x, grid_y = paddle.tensor.meshgrid([x, y]) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp5(unittest.TestCase): - def test_tuple_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -146,26 +163,32 @@ class TestMeshgridOp5(unittest.TestCase): exe = fluid.Executor(place=fluid.CPUPlace()) grid_x, grid_y = paddle.tensor.meshgrid((x, y)) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp6(unittest.TestCase): - def test_api_with_dygraph(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) @@ -181,14 +204,21 @@ class TestMeshgridOp6(unittest.TestCase): class TestMeshgridOp7(unittest.TestCase): - def test_api_with_dygraph_list_input(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) @@ -204,14 +234,21 @@ class TestMeshgridOp7(unittest.TestCase): class TestMeshgridOp8(unittest.TestCase): - def test_api_with_dygraph_tuple_input(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) @@ -227,14 +264,21 @@ class TestMeshgridOp8(unittest.TestCase): class TestMeshgridEager(unittest.TestCase): - def test_dygraph_api(self): - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_1 = fluid.dygraph.to_variable(input_1) @@ -250,15 +294,22 @@ class TestMeshgridEager(unittest.TestCase): tensor_eager_1.stop_gradient = False tensor_eager_2.stop_gradient = False res_eager_1, res_eager_2 = paddle.tensor.meshgrid( - (tensor_eager_1, tensor_eager_2)) + (tensor_eager_1, tensor_eager_2) + ) sum_eager = paddle.add_n([res_eager_1, res_eager_2]) sum_eager.backward() - self.assertEqual(( - tensor_1.grad.numpy() == tensor_eager_1.grad.numpy()).all(), - True) - self.assertEqual(( - tensor_2.grad.numpy() == tensor_eager_2.grad.numpy()).all(), - True) + self.assertEqual( + ( + tensor_1.grad.numpy() == tensor_eager_1.grad.numpy() + ).all(), + True, + ) + self.assertEqual( + ( + tensor_2.grad.numpy() == tensor_eager_2.grad.numpy() + ).all(), + True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_min_op.py b/python/paddle/fluid/tests/unittests/test_min_op.py index 1f4cd926241..a81b63dbb58 100644 --- a/python/paddle/fluid/tests/unittests/test_min_op.py +++ b/python/paddle/fluid/tests/unittests/test_min_op.py @@ -22,7 +22,6 @@ from test_sum_op import TestReduceOPTensorAxisBase class ApiMinTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -31,39 +30,43 @@ class ApiMinTest(unittest.TestCase): def test_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="float32") result_min = paddle.min(x=data, axis=1) exe = paddle.static.Executor(self.place) input_data = np.random.rand(10, 10).astype(np.float32) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_min]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_min]) self.assertEqual((res == np.min(input_data, axis=1)).all(), True) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="int64") result_min = paddle.min(x=data, axis=0) exe = paddle.static.Executor(self.place) input_data = np.random.randint(10, size=(10, 10)).astype(np.int64) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_min]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_min]) self.assertEqual((res == np.min(input_data, axis=0)).all(), True) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="int64") result_min = paddle.min(x=data, axis=(0, 1)) exe = paddle.static.Executor(self.place) input_data = np.random.randint(10, size=(10, 10)).astype(np.int64) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_min]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_min]) self.assertEqual((res == np.min(input_data, axis=(0, 1))).all(), True) def test_errors(self): paddle.enable_static() def test_input_type(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = np.random.rand(10, 10) result_min = paddle.min(x=data, axis=0) @@ -84,17 +87,17 @@ class ApiMinTest(unittest.TestCase): class TestOutDtype(unittest.TestCase): - def test_min(self): api_fn = paddle.min shape = [10, 16] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64', 'int32', 'int64']) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64', 'int32', 'int64'], + ) class TestMinWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.min self.np_api = np.min @@ -104,7 +107,6 @@ class TestMinWithTensorAxis1(TestReduceOPTensorAxisBase): class TestMinWithTensorAxis2(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.min self.np_api = np.min @@ -113,7 +115,7 @@ class TestMinWithTensorAxis2(TestReduceOPTensorAxisBase): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] self.keepdim = True diff --git a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py index 67329652726..7afb4860211 100644 --- a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py +++ b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py @@ -18,26 +18,25 @@ from op_test import OpTest class TestMineHardExamplesOp(OpTest): - def set_data(self): self.init_test_data() self.inputs = { 'ClsLoss': self.cls_loss, 'LocLoss': self.loc_loss, 'MatchIndices': self.match_indices, - 'MatchDist': self.match_dis + 'MatchDist': self.match_dis, } self.attrs = { 'neg_pos_ratio': self.neg_pos_ratio, 'neg_overlap': self.neg_overlap, 'sample_size': self.sample_size, - 'mining_type': self.mining_type + 'mining_type': self.mining_type, } self.outputs = { 'NegIndices': (self.neg_indices, self.neg_indices_lod), - 'UpdatedMatchIndices': self.updated_match_indices + 'UpdatedMatchIndices': self.updated_match_indices, } def test_check_output(self): @@ -55,17 +54,21 @@ class TestMineHardExamplesOp(OpTest): self.neg_overlap = 0.5 self.sample_size = 0 self.mining_type = "max_negative" - self.cls_loss = np.array([[0.1, 0.1, 0.3], [0.3, 0.1, - 0.1]]).astype('float64') + self.cls_loss = np.array([[0.1, 0.1, 0.3], [0.3, 0.1, 0.1]]).astype( + 'float64' + ) - self.loc_loss = np.array([[0.1, 0.2, 0.3], [0.3, 0.4, - 0.1]]).astype('float64') + self.loc_loss = np.array([[0.1, 0.2, 0.3], [0.3, 0.4, 0.1]]).astype( + 'float64' + ) - self.match_dis = np.array([[0.2, 0.4, 0.8], [0.1, 0.9, - 0.3]]).astype('float64') + self.match_dis = np.array([[0.2, 0.4, 0.8], [0.1, 0.9, 0.3]]).astype( + 'float64' + ) - self.match_indices = np.array([[0, -1, -1], [-1, 0, - -1]]).astype('int32') + self.match_indices = np.array([[0, -1, -1], [-1, 0, -1]]).astype( + 'int32' + ) self.updated_match_indices = self.match_indices @@ -74,23 +77,26 @@ class TestMineHardExamplesOp(OpTest): class TestMineHardExamplesOpHardExample(TestMineHardExamplesOp): - def init_test_data(self): super(TestMineHardExamplesOpHardExample, self).init_test_data() self.mining_type = "hard_example" self.sample_size = 2 - self.cls_loss = np.array([[0.5, 0.1, 0.3], [0.3, 0.1, - 0.1]]).astype('float64') + self.cls_loss = np.array([[0.5, 0.1, 0.3], [0.3, 0.1, 0.1]]).astype( + 'float64' + ) - self.loc_loss = np.array([[0.2, 0.2, 0.3], [0.3, 0.1, - 0.2]]).astype('float64') + self.loc_loss = np.array([[0.2, 0.2, 0.3], [0.3, 0.1, 0.2]]).astype( + 'float64' + ) - self.match_indices = np.array([[0, -1, -1], [-1, 0, - -1]]).astype('int32') + self.match_indices = np.array([[0, -1, -1], [-1, 0, -1]]).astype( + 'int32' + ) - self.updated_match_indices = np.array([[0, -1, -1], - [-1, -1, -1]]).astype('int32') + self.updated_match_indices = np.array( + [[0, -1, -1], [-1, -1, -1]] + ).astype('int32') self.neg_indices_lod = [[1, 2]] self.neg_indices = np.array([[2], [0], [2]]).astype('int32') diff --git a/python/paddle/fluid/tests/unittests/test_minimum_op.py b/python/paddle/fluid/tests/unittests/test_minimum_op.py index 4dbdfdcad4c..81771950c08 100644 --- a/python/paddle/fluid/tests/unittests/test_minimum_op.py +++ b/python/paddle/fluid/tests/unittests/test_minimum_op.py @@ -19,7 +19,6 @@ import paddle.fluid.core as core class ApiMinimumTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -40,56 +39,56 @@ class ApiMinimumTest(unittest.TestCase): def test_static_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_max = paddle.minimum(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_max = paddle.minimum(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.minimum(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.minimum(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): diff --git a/python/paddle/fluid/tests/unittests/test_minus_op.py b/python/paddle/fluid/tests/unittests/test_minus_op.py index aaa8744061c..40b4ddd3b57 100644 --- a/python/paddle/fluid/tests/unittests/test_minus_op.py +++ b/python/paddle/fluid/tests/unittests/test_minus_op.py @@ -19,12 +19,11 @@ import paddle class TestMinusOp(OpTest): - def setUp(self): self.op_type = "minus" self.inputs = { 'X': np.random.random((32, 84)).astype("float32"), - 'Y': np.random.random((32, 84)).astype("float32") + 'Y': np.random.random((32, 84)).astype("float32"), } self.outputs = {'Out': (self.inputs['X'] - self.inputs['Y'])} diff --git a/python/paddle/fluid/tests/unittests/test_mix_precision_all_reduce_fuse.py b/python/paddle/fluid/tests/unittests/test_mix_precision_all_reduce_fuse.py index bc57d38b77c..62243890305 100644 --- a/python/paddle/fluid/tests/unittests/test_mix_precision_all_reduce_fuse.py +++ b/python/paddle/fluid/tests/unittests/test_mix_precision_all_reduce_fuse.py @@ -36,21 +36,25 @@ def conv_net(use_feed): img = fluid.layers.data(name='image', shape=img_shape, dtype='float16') label = fluid.layers.data(name='label', shape=[1], dtype='int64') - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) conv_pool_1 = fluid.layers.cast(conv_pool_1, np.float32) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) hidden = fluid.layers.cast(conv_pool_2, np.float32) return loss_net(hidden, label) @@ -61,11 +65,10 @@ def _optimizer(learning_rate=1e-6): class TestResnet(TestParallelExecutorBase): - def check_model(self, use_device): - img, label = init_data(batch_size=batch_size, - img_shape=img_shape, - label_range=9) + img, label = init_data( + batch_size=batch_size, img_shape=img_shape, label_range=9 + ) img = np.float16(img) feed_dict = {"image": img, "label": label} @@ -75,7 +78,8 @@ class TestResnet(TestParallelExecutorBase): iter=10, use_device=use_device, fuse_all_reduce_ops=True, - optimizer=_optimizer) + optimizer=_optimizer, + ) def test_model(self): if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_mode_op.py b/python/paddle/fluid/tests/unittests/test_mode_op.py index 8b2dcdf5295..bec5707ac95 100644 --- a/python/paddle/fluid/tests/unittests/test_mode_op.py +++ b/python/paddle/fluid/tests/unittests/test_mode_op.py @@ -40,7 +40,7 @@ def cal_mode(a, axis, keepdim=False): if axis < 0: axis = len(a.shape) + axis in_dims = list(range(a.ndim)) - a_view = np.transpose(a, in_dims[:axis] + in_dims[axis + 1:] + [axis]) + a_view = np.transpose(a, in_dims[:axis] + in_dims[axis + 1 :] + [axis]) inds = np.ndindex(a_view.shape[:-1]) modes = np.empty(a_view.shape[:-1], dtype=a.dtype) indexes = np.empty(a_view.shape[:-1], dtype=np.int64) @@ -55,7 +55,6 @@ def cal_mode(a, axis, keepdim=False): class TestModeOp(OpTest): - def init_args(self): self.axis = 1 @@ -81,7 +80,6 @@ class TestModeOp(OpTest): class TestModeOpLastdim(OpTest): - def init_args(self): self.axis = -1 @@ -107,14 +105,12 @@ class TestModeOpLastdim(OpTest): class TestModeOpKernels(unittest.TestCase): - def setUp(self): self.axises = [-1, 1] np.random.seed(666) self.inputs = np.ceil(np.random.rand(2, 10, 10) * 1000) def test_mode_op(self): - def test_cpu_kernel(): paddle.set_device('cpu') tensor = paddle.to_tensor(self.inputs) @@ -123,9 +119,9 @@ class TestModeOpKernels(unittest.TestCase): v, inds = paddle.mode(tensor, axis) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) - value_expect, indice_expect = cal_mode(self.inputs, - axis, - keepdim=True) + value_expect, indice_expect = cal_mode( + self.inputs, axis, keepdim=True + ) v, inds = paddle.mode(tensor, axis, keepdim=True) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) @@ -137,9 +133,9 @@ class TestModeOpKernels(unittest.TestCase): v, inds = paddle.mode(tensor, axis) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) - value_expect, indice_expect = cal_mode(self.inputs, - axis, - keepdim=True) + value_expect, indice_expect = cal_mode( + self.inputs, axis, keepdim=True + ) v, inds = paddle.mode(tensor, axis, keepdim=True) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) @@ -150,7 +146,6 @@ class TestModeOpKernels(unittest.TestCase): class TestModeOpErrors(unittest.TestCase): - def setUp(self): self.x = paddle.uniform([2, 10, 20, 25], dtype='float32') @@ -161,25 +156,27 @@ class TestModeOpErrors(unittest.TestCase): class TestModeOpInStatic(unittest.TestCase): - def setUp(self): np.random.seed(666) - self.input_data = np.ceil(np.random.random((2, 10, 10)) * 1000, - dtype=np.float64) + self.input_data = np.ceil( + np.random.random((2, 10, 10)) * 1000, dtype=np.float64 + ) def test_run_static(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[2, 10, 10], - dtype="float64") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[2, 10, 10], dtype="float64" + ) result = paddle.mode(input_tensor, axis=1) expect_value = cal_mode(self.input_data, axis=1)[0] exe = paddle.static.Executor(paddle.CPUPlace()) - paddle_result = exe.run(feed={"x": self.input_data}, - fetch_list=[result])[0] + paddle_result = exe.run( + feed={"x": self.input_data}, fetch_list=[result] + )[0] np.testing.assert_allclose(paddle_result, expect_value, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_modelaverage.py b/python/paddle/fluid/tests/unittests/test_modelaverage.py index 474fa692b47..0ab8aad4271 100644 --- a/python/paddle/fluid/tests/unittests/test_modelaverage.py +++ b/python/paddle/fluid/tests/unittests/test_modelaverage.py @@ -20,7 +20,6 @@ import paddle.nn as nn class TestModelAverage(unittest.TestCase): - def test_model_average_static(self): paddle.enable_static() place = fluid.CPUPlace() @@ -35,50 +34,75 @@ class TestModelAverage(unittest.TestCase): hidden = fluid.layers.fc(input=data, size=10) loss = paddle.mean(hidden) test_program = train_program.clone() - optimizer = paddle.optimizer.Momentum(learning_rate=0.2, - momentum=0.1) + optimizer = paddle.optimizer.Momentum( + learning_rate=0.2, momentum=0.1 + ) optimizer.minimize(loss) # build ModelAverage optimizer model_average = paddle.incubate.optimizer.ModelAverage( - 0.15, min_average_window=2, max_average_window=10) + 0.15, min_average_window=2, max_average_window=10 + ) exe.run(startup) for i in range(10): x = np.random.random(size=(10, 1)).astype('float32') - latest_b, sum_1, sum_2, sum_3, num_accumulates, old_num_accumulates, num_updates = exe.run( + ( + latest_b, + sum_1, + sum_2, + sum_3, + num_accumulates, + old_num_accumulates, + num_updates, + ) = exe.run( program=train_program, feed={'X': x}, fetch_list=[ - 'fc_0.b_0', 'fc_0.b_0_sum_1_0', 'fc_0.b_0_sum_2_0', - 'fc_0.b_0_sum_3_0', 'fc_0.b_0_num_accumulates_0', - 'fc_0.b_0_old_num_accumulates_0', 'fc_0.b_0_num_updates_0' - ]) + 'fc_0.b_0', + 'fc_0.b_0_sum_1_0', + 'fc_0.b_0_sum_2_0', + 'fc_0.b_0_sum_3_0', + 'fc_0.b_0_num_accumulates_0', + 'fc_0.b_0_old_num_accumulates_0', + 'fc_0.b_0_num_updates_0', + ], + ) self.assertTrue( - np.equal(sum_1, np.zeros(shape=[10], dtype='float32')).all()) + np.equal(sum_1, np.zeros(shape=[10], dtype='float32')).all() + ) self.assertTrue( - np.equal(sum_2, np.zeros(shape=[10], dtype='float32')).all()) + np.equal(sum_2, np.zeros(shape=[10], dtype='float32')).all() + ) self.assertTrue( - np.equal(num_accumulates, np.array([0], dtype='int64')).all()) + np.equal(num_accumulates, np.array([0], dtype='int64')).all() + ) self.assertTrue( - np.equal(old_num_accumulates, np.array([2], dtype='int64')).all()) + np.equal(old_num_accumulates, np.array([2], dtype='int64')).all() + ) self.assertTrue( - np.equal(num_updates, np.array([10], dtype='int64')).all()) + np.equal(num_updates, np.array([10], dtype='int64')).all() + ) - average_b = (sum_1 + sum_2 + sum_3) / (num_accumulates + - old_num_accumulates) + average_b = (sum_1 + sum_2 + sum_3) / ( + num_accumulates + old_num_accumulates + ) # apply ModelAverage with model_average.apply(exe): x = np.random.random(size=(10, 1)).astype('float32') - outs, b = exe.run(program=test_program, - feed={'X': x}, - fetch_list=[loss.name, 'fc_0.b_0']) + outs, b = exe.run( + program=test_program, + feed={'X': x}, + fetch_list=[loss.name, 'fc_0.b_0'], + ) self.assertAlmostEqual(np.mean(average_b), np.mean(b)) x = np.random.random(size=(10, 1)).astype('float32') - outs, b = exe.run(program=test_program, - feed={'X': x}, - fetch_list=[loss.name, 'fc_0.b_0']) + outs, b = exe.run( + program=test_program, + feed={'X': x}, + fetch_list=[loss.name, 'fc_0.b_0'], + ) self.assertAlmostEqual(np.mean(latest_b), np.mean(b)) def test_model_average_dygraph(self): @@ -91,21 +115,20 @@ class TestModelAverage(unittest.TestCase): # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, - (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( + 'int64' + ) return image, label def __len__(self): return self.num_samples class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -131,23 +154,30 @@ class TestModelAverage(unittest.TestCase): sum_2 = model_average._get_accumulator('sum_2', layer.bias) sum_3 = model_average._get_accumulator('sum_3', layer.bias) num_accumulates = model_average._get_accumulator( - 'num_accumulates', layer.bias) + 'num_accumulates', layer.bias + ) old_num_accumulates = model_average._get_accumulator( - 'old_num_accumulates', layer.bias) + 'old_num_accumulates', layer.bias + ) num_updates = model_average._get_accumulator( - 'num_updates', layer.bias) + 'num_updates', layer.bias + ) - return ((sum_1 + sum_2 + sum_3) / - (num_accumulates + old_num_accumulates)).numpy() + return ( + (sum_1 + sum_2 + sum_3) + / (num_accumulates + old_num_accumulates) + ).numpy() def evaluate(layer, loader, loss_fn, check_param): for batch_id, (image, label) in enumerate(loader()): out = layer(image) loss = loss_fn(out, label) loss.backward() - self.assertAlmostEqual(np.mean(layer.bias.numpy()), - np.mean(check_param), - delta=5e-3) + self.assertAlmostEqual( + np.mean(layer.bias.numpy()), + np.mean(check_param), + delta=5e-3, + ) # print("Evaluate batch {}: loss = {}, bias = {}".format( # batch_id, np.mean(loss.numpy()), layer.bias.numpy())) @@ -155,36 +185,42 @@ class TestModelAverage(unittest.TestCase): layer = LinearNet() loss_fn = nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Momentum(learning_rate=0.2, - momentum=0.1, - parameters=layer.parameters()) + optimizer = paddle.optimizer.Momentum( + learning_rate=0.2, momentum=0.1, parameters=layer.parameters() + ) # build ModelAverage optimizer model_average = paddle.incubate.optimizer.ModelAverage( 0.15, parameters=layer.parameters(), min_average_window=2, - max_average_window=10) + max_average_window=10, + ) # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) - eval_loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=1) + loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) + eval_loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=1, + ) # train check_param = train(layer, loader, loss_fn, optimizer, model_average) # print(check_param) with model_average.apply(need_restore=False): evaluate(layer, eval_loader, loss_fn, check_param) - check_param = (model_average._get_accumulator('restore', - layer.bias)).numpy() + check_param = ( + model_average._get_accumulator('restore', layer.bias) + ).numpy() # print(check_param) # print("\nEvaluate With Restored Paramters") model_average.restore() diff --git a/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py index 1e81a6ae6b3..99ce37834b5 100644 --- a/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py @@ -19,27 +19,29 @@ from op_test import OpTest def modified_huber_loss_forward(val): if val < -1: - return -4. * val + return -4.0 * val elif val < 1: - return (1. - val) * (1. - val) + return (1.0 - val) * (1.0 - val) else: - return 0. + return 0.0 class TestModifiedHuberLossOp(OpTest): - def setUp(self): self.op_type = 'modified_huber_loss' samples_num = 100 - x_np = np.random.uniform(-2., 2., (samples_num, 1)).astype('float32') - y_np = np.random.choice([0, 1], samples_num).reshape( - (samples_num, 1)).astype('float32') - product_res = x_np * (2. * y_np - 1.) + x_np = np.random.uniform(-2.0, 2.0, (samples_num, 1)).astype('float32') + y_np = ( + np.random.choice([0, 1], samples_num) + .reshape((samples_num, 1)) + .astype('float32') + ) + product_res = x_np * (2.0 * y_np - 1.0) # keep away from the junction of piecewise function for pos, val in np.ndenumerate(product_res): - while abs(val - 1.) < 0.05: - x_np[pos] = np.random.uniform(-2., 2.) + while abs(val - 1.0) < 0.05: + x_np[pos] = np.random.uniform(-2.0, 2.0) y_np[pos] = np.random.choice([0, 1]) product_res[pos] = x_np[pos] * (2 * y_np[pos] - 1) val = product_res[pos] @@ -49,7 +51,7 @@ class TestModifiedHuberLossOp(OpTest): self.outputs = { 'IntermediateVal': product_res.astype('float32'), - 'Out': loss.reshape((samples_num, 1)).astype('float32') + 'Out': loss.reshape((samples_num, 1)).astype('float32'), } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_momentum_op.py b/python/paddle/fluid/tests/unittests/test_momentum_op.py index e16dfa969f1..991291e36fb 100644 --- a/python/paddle/fluid/tests/unittests/test_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_momentum_op.py @@ -23,14 +23,16 @@ import numpy from paddle.fluid.framework import _test_eager_guard -def calculate_momentum_by_numpy(param, - grad, - mu, - velocity, - use_nesterov, - learning_rate, - regularization_method=None, - regularization_coeff=1.0): +def calculate_momentum_by_numpy( + param, + grad, + mu, + velocity, + use_nesterov, + learning_rate, + regularization_method=None, + regularization_coeff=1.0, +): if regularization_method == "l2_decay": grad = grad + regularization_coeff * param @@ -42,8 +44,9 @@ def calculate_momentum_by_numpy(param, else: velocity_out = mu * velocity + grad if use_nesterov: - param_out = param - grad * learning_rate - \ - velocity_out * mu * learning_rate + param_out = ( + param - grad * learning_rate - velocity_out * mu * learning_rate + ) else: param_out = param - learning_rate * velocity_out @@ -51,7 +54,6 @@ def calculate_momentum_by_numpy(param, class TestMomentumOp1(OpTest): - def setUp(self): self.op_type = "momentum" self.dtype = np.float32 @@ -68,7 +70,7 @@ class TestMomentumOp1(OpTest): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu} @@ -79,7 +81,8 @@ class TestMomentumOp1(OpTest): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -91,7 +94,6 @@ class TestMomentumOp1(OpTest): class TestMomentumOpFp16(TestMomentumOp1): - def init_dtype(self): self.dtype = np.float16 @@ -100,8 +102,7 @@ class TestMomentumOpFp16(TestMomentumOp1): class TestMomentumOp2(OpTest): - '''Test Momentum with default values for attributes - ''' + '''Test Momentum with default values for attributes''' def setUp(self): self.op_type = "momentum" @@ -117,7 +118,7 @@ class TestMomentumOp2(OpTest): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu, 'use_nesterov': use_nesterov} @@ -128,7 +129,8 @@ class TestMomentumOp2(OpTest): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -136,10 +138,10 @@ class TestMomentumOp2(OpTest): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLarsMomentumOpWithMP(OpTest): - def setUp(self): self.config() self.op_type = "lars_momentum" @@ -166,11 +168,16 @@ class TestLarsMomentumOpWithMP(OpTest): fp32_grad = grad.astype("float32") pnorm = np.sqrt(np.square(master_param).sum()) gnorm = np.sqrt(np.square(fp32_grad).sum()) - local_lr = learning_rate * lars_coeff * pnorm / ( - gnorm + lars_weight_decay * pnorm) + local_lr = ( + learning_rate + * lars_coeff + * pnorm + / (gnorm + lars_weight_decay * pnorm) + ) fp32_grad = fp32_grad * rescale_grad velocity_out = mu * velocity + local_lr * ( - fp32_grad + lars_weight_decay * master_param) + fp32_grad + lars_weight_decay * master_param + ) p_new = master_param - velocity_out param_out = p_new.astype("float16") master_param_out = p_new @@ -183,7 +190,8 @@ class TestLarsMomentumOpWithMP(OpTest): param_outs.append(("SubParam_out_" + str(i), param_out)) master_params.append(("SubMasterParam_" + str(i), master_param)) master_param_outs.append( - ("SubMasterParamOut_" + str(i), master_param_out)) + ("SubMasterParamOut_" + str(i), master_param_out) + ) self.inputs = { 'Param': params, @@ -198,13 +206,13 @@ class TestLarsMomentumOpWithMP(OpTest): 'lars_coeff': lars_coeff, 'lars_weight_decay': [lars_weight_decay], 'multi_precision': True, - 'rescale_grad': rescale_grad + 'rescale_grad': rescale_grad, } self.outputs = { 'ParamOut': param_outs, 'VelocityOut': velocity_outs, - 'MasterParamOut': master_param_outs + 'MasterParamOut': master_param_outs, } def test_check_output(self): @@ -219,7 +227,6 @@ class TestLarsMomentumOpWithMP(OpTest): class TestLarsMomentumOp(OpTest): - def setUp(self): self.config() self.op_type = "lars_momentum" @@ -240,10 +247,15 @@ class TestLarsMomentumOp(OpTest): learning_rate = np.array([0.001]).astype("float32") pnorm = np.sqrt(np.square(param).sum()) gnorm = np.sqrt(np.square(grad).sum()) - local_lr = learning_rate * lars_coeff * pnorm / ( - gnorm + lars_weight_decay * param) + local_lr = ( + learning_rate + * lars_coeff + * pnorm + / (gnorm + lars_weight_decay * param) + ) velocity_out = mu * velocity + local_lr * ( - grad + lars_weight_decay * param) + grad + lars_weight_decay * param + ) param_out = param - velocity_out params.append(("SubParam_" + str(i), param)) @@ -257,13 +269,13 @@ class TestLarsMomentumOp(OpTest): 'Param': params, 'Grad': grads, 'Velocity': velocitys, - 'LearningRate': learning_rates + 'LearningRate': learning_rates, } self.attrs = { 'mu': mu, 'lars_coeff': lars_coeff, - 'lars_weight_decay': [lars_weight_decay] + 'lars_weight_decay': [lars_weight_decay], } self.outputs = {'ParamOut': param_outs, 'VelocityOut': velocity_outs} @@ -276,7 +288,6 @@ class TestLarsMomentumOp(OpTest): class TestSparseMomentumOp(unittest.TestCase): - def setUp(self): self.use_nesterov = False self.regularization_method = "" @@ -315,8 +326,9 @@ class TestSparseMomentumOp(unittest.TestCase): velocity_np_array = np.ones((height, row_numel)).astype("float32") velocity.set(velocity_np_array, place) velocity_out = scope.var('VelocityOut').get_tensor() - velocity_out_np_array = np.full((height, row_numel), - 0.0).astype("float32") + velocity_out_np_array = np.full((height, row_numel), 0.0).astype( + "float32" + ) velocity_out.set(velocity_out_np_array, place) # create and initialize LearningRate Variable @@ -325,17 +337,19 @@ class TestSparseMomentumOp(unittest.TestCase): lr.set(lr_array, place) # create and run operator - op = Operator("momentum", - Param='Param', - Grad='Grad', - Velocity='Velocity', - ParamOut='ParamOut', - VelocityOut='VelocityOut', - LearningRate='LearningRate', - mu=mu, - use_nesterov=use_nesterov, - regularization_method=regularization_method, - regularization_coeff=regularization_coeff) + op = Operator( + "momentum", + Param='Param', + Grad='Grad', + Velocity='Velocity', + ParamOut='ParamOut', + VelocityOut='VelocityOut', + LearningRate='LearningRate', + mu=mu, + use_nesterov=use_nesterov, + regularization_method=regularization_method, + regularization_coeff=regularization_coeff, + ) op.run(scope, place) # get and compare result @@ -358,7 +372,8 @@ class TestSparseMomentumOp(unittest.TestCase): use_nesterov=use_nesterov, learning_rate=lr_array, regularization_method=regularization_method, - regularization_coeff=regularization_coeff) + regularization_coeff=regularization_coeff, + ) self.assertTrue((_velocity_out == velocity_out_np_array).all()) self.assertTrue((_param_out == param_out_np_array).all()) @@ -375,13 +390,11 @@ class TestSparseMomentumOp(unittest.TestCase): class TestSparseMomentumOp2(TestSparseMomentumOp): - def init_kernel(self): self.use_nesterov = True class TestSparseMomentumOpWithMultiPrecision(unittest.TestCase): - def setUp(self): self.init_args() self.regularization_method = "" @@ -425,8 +438,9 @@ class TestSparseMomentumOpWithMultiPrecision(unittest.TestCase): velocity_np_array = np.ones((height, row_numel)).astype("float32") velocity.set(velocity_np_array, place) velocity_out = scope.var('VelocityOut').get_tensor() - velocity_out_np_array = np.full((height, row_numel), - 0.0).astype("float32") + velocity_out_np_array = np.full((height, row_numel), 0.0).astype( + "float32" + ) velocity_out.set(velocity_out_np_array, place) # create and initialize LearningRate Variable @@ -435,21 +449,23 @@ class TestSparseMomentumOpWithMultiPrecision(unittest.TestCase): lr.set(lr_array, place) # create and run operator - op = Operator("momentum", - Param='Param', - Grad='Grad', - Velocity='Velocity', - MasterParam='MasterParam', - ParamOut='ParamOut', - VelocityOut='VelocityOut', - MasterParamOut='MasterParamOut', - LearningRate='LearningRate', - mu=mu, - use_nesterov=use_nesterov, - regularization_method=regularization_method, - regularization_coeff=regularization_coeff, - multi_precision=True, - rescale_grad=1.0) + op = Operator( + "momentum", + Param='Param', + Grad='Grad', + Velocity='Velocity', + MasterParam='MasterParam', + ParamOut='ParamOut', + VelocityOut='VelocityOut', + MasterParamOut='MasterParamOut', + LearningRate='LearningRate', + mu=mu, + use_nesterov=use_nesterov, + regularization_method=regularization_method, + regularization_coeff=regularization_coeff, + multi_precision=True, + rescale_grad=1.0, + ) op.run(scope, place) # get and compare result @@ -470,7 +486,8 @@ class TestSparseMomentumOpWithMultiPrecision(unittest.TestCase): use_nesterov=use_nesterov, learning_rate=lr_array, regularization_method=regularization_method, - regularization_coeff=regularization_coeff) + regularization_coeff=regularization_coeff, + ) self.assertTrue((_velocity_out == velocity_out_np_array).all()) self.assertTrue((_param_out == param_out_np_array).all()) @@ -484,23 +501,22 @@ class TestSparseMomentumOpWithMultiPrecision(unittest.TestCase): class TestSparseMomentumOpWithMultiPrecision2( - TestSparseMomentumOpWithMultiPrecision): - + TestSparseMomentumOpWithMultiPrecision +): def init_args(self): self.use_nesterov = True class TestMomentumV2(unittest.TestCase): - def test_momentum_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - momentum=0.9, - parameters=linear.parameters()) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, momentum=0.9, parameters=linear.parameters() + ) out = linear(a) out.backward() adam.step() @@ -517,13 +533,15 @@ class TestMomentumV2(unittest.TestCase): cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - rms_optimizer = paddle.optimizer.Momentum(learning_rate=0.1, - momentum=0.9) + rms_optimizer = paddle.optimizer.Momentum( + learning_rate=0.1, momentum=0.9 + ) rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -531,9 +549,9 @@ class TestMomentumV2(unittest.TestCase): exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list) def test_raise_error(self): - self.assertRaises(ValueError, - paddle.optimizer.Momentum, - learning_rate=None) + self.assertRaises( + ValueError, paddle.optimizer.Momentum, learning_rate=None + ) self.assertRaises(ValueError, paddle.optimizer.Momentum, momentum=None) def test_api_eager_dygraph(self): @@ -543,7 +561,6 @@ class TestMomentumV2(unittest.TestCase): class TestMomentumOpWithDecay(OpTest): - def setUp(self): self.op_type = "momentum" self.dtype = np.float32 @@ -565,14 +582,14 @@ class TestMomentumOpWithDecay(OpTest): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = { 'mu': mu, 'use_nesterov': use_nesterov, 'regularization_method': regularization_method, - 'regularization_coeff': regularization_coeff + 'regularization_coeff': regularization_coeff, } grad = grad + regularization_coeff * param @@ -583,7 +600,8 @@ class TestMomentumOpWithDecay(OpTest): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -596,7 +614,6 @@ class TestMomentumOpWithDecay(OpTest): class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): - def init_config(self): self.dtype = np.float16 @@ -606,13 +623,11 @@ class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): class TestMomentumOpWithDecay2(TestMomentumOpWithDecay): - def init_config(self): self.use_nesterov = False class TestSparseMomentumOpWithDecay(TestSparseMomentumOp): - def setUp(self): self.use_nesterov = False self.regularization_method = 'l2_decay' @@ -620,13 +635,11 @@ class TestSparseMomentumOpWithDecay(TestSparseMomentumOp): class TestSparseMomentumOpWithDecay2(TestSparseMomentumOpWithDecay): - def init_kernel(self): self.use_nesterov = True class TestMomentumOpWithDecayAPI(unittest.TestCase): - def _test_momentum_dygraph_common(self, regularization): paddle.disable_static() inp = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -639,13 +652,16 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): learning_rate=0.01, momentum=0.9, parameter_list=linear.parameters(), - regularization=regularization) + regularization=regularization, + ) momentum.minimize(loss) def test_momentum_dygraph_1(self): self._test_momentum_dygraph_common( regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ) + ) def test_momentum_static(self): paddle.enable_static() @@ -659,12 +675,14 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): avg_cost = paddle.mean(cost) momentum_optimizer = paddle.fluid.contrib.optimizer.Momentum( - learning_rate=0.1, momentum=0.9) + learning_rate=0.1, momentum=0.9 + ) momentum_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -673,23 +691,23 @@ class TestMomentumOpWithDecayAPI(unittest.TestCase): class TestFusedMomentumWithDecayAPI(unittest.TestCase): - def get_program(self, weight_attr, bias_attr=False): main_program = paddle.static.Program() startup_program = paddle.static.Program() - with paddle.static.program_guard(main_program=main_program, - startup_program=startup_program): + with paddle.static.program_guard( + main_program=main_program, startup_program=startup_program + ): x = paddle.static.data(name='x', shape=[10, 10]) - linear = paddle.nn.Linear(10, - 10, - weight_attr=weight_attr, - bias_attr=bias_attr) + linear = paddle.nn.Linear( + 10, 10, weight_attr=weight_attr, bias_attr=bias_attr + ) out = linear(x) loss = paddle.mean(out) optimizer = paddle.optimizer.Momentum( learning_rate=0.01, momentum=0.9, - weight_decay=paddle.regularizer.L2Decay(0.5)) + weight_decay=paddle.regularizer.L2Decay(0.5), + ) optimizer.minimize(loss) return main_program @@ -698,7 +716,8 @@ class TestFusedMomentumWithDecayAPI(unittest.TestCase): weight_attr = paddle.ParamAttr( name="weight", initializer=paddle.nn.initializer.Constant(value=0.5), - regularizer=paddle.regularizer.L2Decay(0.1)) + regularizer=paddle.regularizer.L2Decay(0.1), + ) program = self.get_program(weight_attr, bias_attr=False) ops = program.global_block().ops @@ -713,11 +732,13 @@ class TestFusedMomentumWithDecayAPI(unittest.TestCase): weight_attr = paddle.ParamAttr( name="weight", initializer=paddle.nn.initializer.Constant(value=0.5), - regularizer=paddle.regularizer.L1Decay(0.1)) + regularizer=paddle.regularizer.L1Decay(0.1), + ) bias_attr = paddle.ParamAttr( name="bias", - initializer=paddle.nn.initializer.Constant(value=0.), - regularizer=None) + initializer=paddle.nn.initializer.Constant(value=0.0), + regularizer=None, + ) program = self.get_program(weight_attr, bias_attr) ops = program.global_block().ops @@ -732,8 +753,9 @@ class TestFusedMomentumWithDecayAPI(unittest.TestCase): self.assertEqual(ops[-1].attr('regularization_coeff'), 0) if 'bias' in ops[-2].input('Param'): self.assertEqual(ops[-2].attr('regularization_method'), 'l2_decay') - self.assertEqual(ops[-2].attr('regularization_coeff'), - np.float32(0.5)) + self.assertEqual( + ops[-2].attr('regularization_coeff'), np.float32(0.5) + ) def test_param_has_no_regularizer(self): paddle.enable_static() @@ -747,11 +769,11 @@ class TestFusedMomentumWithDecayAPI(unittest.TestCase): class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): - def __update_params(self, momentum, linear): for i in range(10): - inp = paddle.full(shape=[2, 2], fill_value=i, - dtype='float32').astype("float32") + inp = paddle.full( + shape=[2, 2], fill_value=i, dtype='float32' + ).astype("float32") inp = paddle.to_tensor(inp) out = linear(inp) loss = paddle.mean(out) @@ -766,32 +788,39 @@ class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_old = paddle.fluid.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_old.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_old, linear=linear_old) linear_new = paddle.nn.Linear( 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_new = paddle.fluid.contrib.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_new.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_new, linear=linear_new) self.assertEqual( (linear_old.weight.numpy() == linear_new.weight.numpy()).all(), True, - 'the param weight updated by two Momentum optimizers should equal') + 'the param weight updated by two Momentum optimizers should equal', + ) def test_vs(self, place=fluid.CPUPlace()): places = [fluid.CPUPlace()] @@ -803,7 +832,6 @@ class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): class TestMomentumV2Group(TestMomentumV2): - def test_momentum_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -811,22 +839,20 @@ class TestMomentumV2Group(TestMomentumV2): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - 'learning_rate': - 0.1, - 'momentum': - 0.99 - }], - weight_decay=0.1, - momentum=0.9) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + }, + ], + weight_decay=0.1, + momentum=0.9, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -835,13 +861,14 @@ class TestMomentumV2Group(TestMomentumV2): class TestMultiTensorMomentumDygraph(unittest.TestCase): - - def _momentum_optimize_dygraph(self, - place, - use_param_attr=False, - use_param_group=False, - use_amp=False, - use_multi_tensor=False): + def _momentum_optimize_dygraph( + self, + place, + use_param_attr=False, + use_param_group=False, + use_amp=False, + use_multi_tensor=False, + ): paddle.disable_static() paddle.seed(10) paddle.set_device(place) @@ -849,7 +876,8 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): weight_attr = paddle.ParamAttr( learning_rate=0.5, regularizer=paddle.regularizer.L2Decay(1.0), - trainable=True) + trainable=True, + ) if use_param_attr: model = paddle.nn.Linear(5, 5, weight_attr) else: @@ -858,17 +886,21 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): optimizer = paddle.optimizer.Momentum( parameters=model.parameters(), use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + multi_precision=use_amp, + ) else: optimizer = paddle.optimizer.Momentum( - parameters=[{ - 'params': model.parameters(), - 'weight_decay': 0.001, - 'learning_rate': 0.1, - 'momentum': 0.99 - }], + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + } + ], use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + multi_precision=use_amp, + ) for idx in range(5): if place == 'gpu' and use_amp == True: model = paddle.amp.decorate(models=model, level='O2') @@ -898,9 +930,11 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): def _check_with_place_amp(self, place, use_amp): output1, params1 = self._momentum_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=True) + place=place, use_amp=use_amp, use_multi_tensor=True + ) output2, params2 = self._momentum_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=False) + place=place, use_amp=use_amp, use_multi_tensor=False + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): @@ -911,12 +945,14 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): place=place, use_amp=use_amp, use_param_attr=True, - use_multi_tensor=True) + use_multi_tensor=True, + ) output2, params2 = self._momentum_optimize_dygraph( place=place, use_amp=use_amp, use_param_attr=True, - use_multi_tensor=False) + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -926,12 +962,14 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): place=place, use_amp=use_amp, use_param_group=True, - use_multi_tensor=True) + use_multi_tensor=True, + ) output2, params2 = self._momentum_optimize_dygraph( place=place, use_amp=use_amp, use_param_group=True, - use_multi_tensor=False) + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -950,11 +988,9 @@ class TestMultiTensorMomentumDygraph(unittest.TestCase): class TestMultiTensorMomentumStatic(unittest.TestCase): - - def _momentum_optimize_static(self, - place, - use_amp=False, - use_multi_tensor=False): + def _momentum_optimize_static( + self, place, use_amp=False, use_multi_tensor=False + ): paddle.enable_static() paddle.seed(10) np.random.seed(10) @@ -963,24 +999,26 @@ class TestMultiTensorMomentumStatic(unittest.TestCase): exe = paddle.static.Executor(place=place) train_program = paddle.static.Program() startup_program = paddle.static.Program() - optimizer = paddle.optimizer.Momentum(multi_precision=use_amp, - use_multi_tensor=use_multi_tensor) + optimizer = paddle.optimizer.Momentum( + multi_precision=use_amp, use_multi_tensor=use_multi_tensor + ) if use_amp: optimizer = paddle.static.amp.decorate( optimizer, init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if use_amp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -992,9 +1030,9 @@ class TestMultiTensorMomentumStatic(unittest.TestCase): x = numpy.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -1005,12 +1043,12 @@ class TestMultiTensorMomentumStatic(unittest.TestCase): return places def _check_with_place_amp(self, place, use_amp): - output1 = self._momentum_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=True) - output2 = self._momentum_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=False) + output1 = self._momentum_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=True + ) + output2 = self._momentum_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=False + ) for idx in range(len(output1)): np.testing.assert_allclose(output1[idx], output2[idx], rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_monitor.py b/python/paddle/fluid/tests/unittests/test_monitor.py index 8649f11821e..dafb23c7d41 100644 --- a/python/paddle/fluid/tests/unittests/test_monitor.py +++ b/python/paddle/fluid/tests/unittests/test_monitor.py @@ -27,7 +27,7 @@ import tempfile class TestDatasetWithStat(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def setUp(self): self.use_data_loader = False @@ -53,10 +53,9 @@ class TestDatasetWithStat(unittest.TestCase): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) embs = [] @@ -78,7 +77,8 @@ class TestDatasetWithStat(unittest.TestCase): exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) @@ -86,11 +86,13 @@ class TestDatasetWithStat(unittest.TestCase): else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset, - fetch_list=[embs[0], embs[1]], - fetch_info=["emb0", "emb1"], - print_period=1) + exe.train_from_dataset( + fluid.default_main_program(), + dataset, + fetch_list=[embs[0], embs[1]], + fetch_info=["emb0", "emb1"], + print_period=1, + ) except Exception as e: self.assertTrue(False) diff --git a/python/paddle/fluid/tests/unittests/test_mse_loss.py b/python/paddle/fluid/tests/unittests/test_mse_loss.py index ef2a494c219..b0b59694406 100644 --- a/python/paddle/fluid/tests/unittests/test_mse_loss.py +++ b/python/paddle/fluid/tests/unittests/test_mse_loss.py @@ -22,7 +22,6 @@ from paddle.fluid.executor import Executor class TestMseLoss(unittest.TestCase): - def test_mse_loss(self): input_val = np.random.uniform(0.1, 0.5, (2, 3)).astype("float32") label_val = np.random.uniform(0.1, 0.5, (2, 3)).astype("float32") @@ -34,24 +33,22 @@ class TestMseLoss(unittest.TestCase): label_var = fluid.data(name="label", shape=[-1, 3], dtype="float32") output = layers.mse_loss(input=input_var, label=label_var) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) - result, = exe.run(fluid.default_main_program(), - feed={ - "input": input_val, - "label": label_val - }, - fetch_list=[output]) + (result,) = exe.run( + fluid.default_main_program(), + feed={"input": input_val, "label": label_val}, + fetch_list=[output], + ) np.testing.assert_allclose(np_result, result, rtol=1e-05) class TestMseInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): input = [256, 3] label = fluid.data(name='label1', shape=[None, 3], dtype='float32') @@ -68,7 +65,6 @@ class TestMseInvalidInput(unittest.TestCase): class TestNNMseLoss(unittest.TestCase): - def test_NNMseLoss_mean(self): for dim in [[10, 10], [2, 10, 10], [3, 3, 10, 10]]: input_np = np.random.uniform(0.1, 0.5, dim).astype("float32") @@ -76,30 +72,34 @@ class TestNNMseLoss(unittest.TestCase): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.layers.data(name='input', - shape=dim, - dtype='float32') - label = fluid.layers.data(name='label', - shape=dim, - dtype='float32') + input = fluid.layers.data( + name='input', shape=dim, dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.loss.MSELoss() ret = mse_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[ret]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[ret], + ) with fluid.dygraph.guard(): mse_loss = paddle.nn.loss.MSELoss() - dy_ret = mse_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = mse_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_result = dy_ret.numpy() sub = input_np - label_np @@ -116,30 +116,34 @@ class TestNNMseLoss(unittest.TestCase): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.layers.data(name='input', - shape=dim, - dtype='float32') - label = fluid.layers.data(name='label', - shape=dim, - dtype='float32') + input = fluid.layers.data( + name='input', shape=dim, dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.loss.MSELoss(reduction='sum') ret = mse_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[ret]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[ret], + ) with fluid.dygraph.guard(): mse_loss = paddle.nn.loss.MSELoss(reduction='sum') - dy_ret = mse_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = mse_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_result = dy_ret.numpy() sub = input_np - label_np @@ -156,34 +160,38 @@ class TestNNMseLoss(unittest.TestCase): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.layers.data(name='input', - shape=dim, - dtype='float32') - label = fluid.layers.data(name='label', - shape=dim, - dtype='float32') + input = fluid.layers.data( + name='input', shape=dim, dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.loss.MSELoss(reduction='none') ret = mse_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[ret]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[ret], + ) with fluid.dygraph.guard(): mse_loss = paddle.nn.loss.MSELoss(reduction='none') - dy_ret = mse_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = mse_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_result = dy_ret.numpy() sub = input_np - label_np - expected = (sub * sub) + expected = sub * sub np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) @@ -191,7 +199,6 @@ class TestNNMseLoss(unittest.TestCase): class TestNNFunctionalMseLoss(unittest.TestCase): - def test_NNFunctionalMseLoss_mean(self): for dim in [[10, 10], [2, 10, 10], [3, 3, 10, 10]]: input_np = np.random.uniform(0.1, 0.5, dim).astype("float32") @@ -199,30 +206,32 @@ class TestNNFunctionalMseLoss(unittest.TestCase): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=dim, - dtype='float32') - target = paddle.fluid.data(name='target', - shape=dim, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=dim, dtype='float32' + ) + target = paddle.fluid.data( + name='target', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.functional.mse_loss(input, target, 'mean') exe = paddle.static.Executor(place) exe.run(startup_prog) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "target": target_np - }, - fetch_list=[mse_loss]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "target": target_np}, + fetch_list=[mse_loss], + ) paddle.disable_static() - dy_ret = paddle.nn.functional.mse_loss(paddle.to_tensor(input_np), - paddle.to_tensor(target_np), - 'mean') + dy_ret = paddle.nn.functional.mse_loss( + paddle.to_tensor(input_np), paddle.to_tensor(target_np), 'mean' + ) dy_result = dy_ret.numpy() sub = input_np - target_np @@ -239,30 +248,32 @@ class TestNNFunctionalMseLoss(unittest.TestCase): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=dim, - dtype='float32') - target = paddle.fluid.data(name='target', - shape=dim, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=dim, dtype='float32' + ) + target = paddle.fluid.data( + name='target', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.functional.mse_loss(input, target, 'sum') exe = paddle.static.Executor(place) exe.run(startup_prog) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "target": target_np - }, - fetch_list=[mse_loss]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "target": target_np}, + fetch_list=[mse_loss], + ) paddle.disable_static() - dy_ret = paddle.nn.functional.mse_loss(paddle.to_tensor(input_np), - paddle.to_tensor(target_np), - 'sum') + dy_ret = paddle.nn.functional.mse_loss( + paddle.to_tensor(input_np), paddle.to_tensor(target_np), 'sum' + ) dy_result = dy_ret.numpy() sub = input_np - target_np @@ -279,30 +290,32 @@ class TestNNFunctionalMseLoss(unittest.TestCase): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=dim, - dtype='float32') - target = paddle.fluid.data(name='target', - shape=dim, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=dim, dtype='float32' + ) + target = paddle.fluid.data( + name='target', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.functional.mse_loss(input, target, 'none') exe = paddle.static.Executor(place) exe.run(startup_prog) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "target": target_np - }, - fetch_list=[mse_loss]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "target": target_np}, + fetch_list=[mse_loss], + ) paddle.disable_static() - dy_ret = paddle.nn.functional.mse_loss(paddle.to_tensor(input_np), - paddle.to_tensor(target_np), - 'none') + dy_ret = paddle.nn.functional.mse_loss( + paddle.to_tensor(input_np), paddle.to_tensor(target_np), 'none' + ) dy_result = dy_ret.numpy() sub = input_np - target_np diff --git a/python/paddle/fluid/tests/unittests/test_mul_nn_grad.py b/python/paddle/fluid/tests/unittests/test_mul_nn_grad.py index 22000cb1808..94e12e337ec 100644 --- a/python/paddle/fluid/tests/unittests/test_mul_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_mul_nn_grad.py @@ -26,7 +26,6 @@ paddle.enable_static() class TestMulGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): prog = fluid.Program() @@ -45,7 +44,6 @@ class TestMulGradCheck(unittest.TestCase): class TestMulDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -62,11 +60,9 @@ class TestMulDoubleGradCheck(unittest.TestCase): x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -77,7 +73,6 @@ class TestMulDoubleGradCheck(unittest.TestCase): class TestMatmulDoubleGradCheck(unittest.TestCase): - def setUp(self): self.init_test() @@ -92,25 +87,21 @@ class TestMatmulDoubleGradCheck(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = layers.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = layers.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = layers.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = layers.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = layers.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = layers.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -121,7 +112,6 @@ class TestMatmulDoubleGradCheck(unittest.TestCase): def TestMatmulDoubleGradCheckCase1(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 3] self.y_shape = [3, 2] @@ -130,7 +120,6 @@ def TestMatmulDoubleGradCheckCase1(TestMatmulDoubleGradCheck): def TestMatmulDoubleGradCheckCase2(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 4, 3] self.y_shape = [2, 4, 5] @@ -139,7 +128,6 @@ def TestMatmulDoubleGradCheckCase2(TestMatmulDoubleGradCheck): def TestMatmulDoubleGradCheckCase3(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 3, 4, 5] self.y_shape = [2, 3, 3, 5] @@ -148,7 +136,6 @@ def TestMatmulDoubleGradCheckCase3(TestMatmulDoubleGradCheck): def TestMatmulDoubleGradCheckCase4(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 3, 4] self.y_shape = [4, 3] diff --git a/python/paddle/fluid/tests/unittests/test_mul_op.py b/python/paddle/fluid/tests/unittests/test_mul_op.py index 0236f34ea8b..5469caed157 100644 --- a/python/paddle/fluid/tests/unittests/test_mul_op.py +++ b/python/paddle/fluid/tests/unittests/test_mul_op.py @@ -24,14 +24,13 @@ from paddle.fluid import Program, program_guard class TestMulOp(OpTest): - def setUp(self): self.op_type = "mul" self.dtype = np.float64 self.init_dtype_type() self.inputs = { 'X': np.random.random((20, 5)).astype(self.dtype), - 'Y': np.random.random((5, 21)).astype(self.dtype) + 'Y': np.random.random((5, 21)).astype(self.dtype), } self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} @@ -45,27 +44,26 @@ class TestMulOp(OpTest): self.check_grad(['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.5, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.5, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y') + ) class TestMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of mul_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - x2 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + x2 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.mul, x1, x2) # The input dtype of mul_op must be float32 or float64. x3 = fluid.layers.data(name='x3', shape=[4], dtype="int32") @@ -74,21 +72,22 @@ class TestMulOpError(unittest.TestCase): class TestMulOp2(OpTest): - def setUp(self): self.op_type = "mul" self.dtype = np.float64 self.init_dtype_type() self.inputs = { 'X': np.random.random((3, 4, 2, 9)).astype(self.dtype), - 'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.dtype) + 'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.dtype), } self.attrs = { 'x_num_col_dims': 2, 'y_num_col_dims': 2, } - result = np.dot(self.inputs['X'].reshape(3 * 4, 2 * 9), - self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3)) + result = np.dot( + self.inputs['X'].reshape(3 * 4, 2 * 9), + self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3), + ) result = result.reshape(3, 4, 1, 2, 3) self.outputs = {'Out': result} @@ -102,22 +101,20 @@ class TestMulOp2(OpTest): self.check_grad(['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.5, - no_grad_set=set('X')) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set('X') + ) def test_check_grad_ignore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.5, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y') + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16MulOp1(TestMulOp): - def init_dtype_type(self): self.dtype = np.float16 @@ -129,31 +126,37 @@ class TestFP16MulOp1(TestMulOp): def test_check_grad_normal(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.5 + ) def test_check_grad_ingore_x(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.5, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.5, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.5, - no_grad_set=set('Y')) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.5, + no_grad_set=set('Y'), + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16MulOp2(TestMulOp2): - def init_dtype_type(self): self.dtype = np.float16 @@ -165,25 +168,31 @@ class TestFP16MulOp2(TestMulOp2): def test_check_grad_normal(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.9) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.9 + ) def test_check_grad_ingore_x(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.5, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.5, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.9, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.9, + no_grad_set=set('Y'), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_multi_dot_op.py b/python/paddle/fluid/tests/unittests/test_multi_dot_op.py index ce188cac5c7..96d8c7572ee 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_dot_op.py +++ b/python/paddle/fluid/tests/unittests/test_multi_dot_op.py @@ -23,10 +23,9 @@ from paddle.fluid.framework import _test_eager_guard paddle.enable_static() -#the unittest of multi_dot -#compare the result of paddle multi_dot and numpy multi_dot +# the unittest of multi_dot +# compare the result of paddle multi_dot and numpy multi_dot class TestMultiDotOp(OpTest): - def setUp(self): self.op_type = "multi_dot" self.python_api = paddle.linalg.multi_dot @@ -50,9 +49,8 @@ class TestMultiDotOp(OpTest): self.check_grad(['x1'], 'Out', check_eager=True) -#(A*B)*C +# (A*B)*C class TestMultiDotOp3Mat(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((2, 10)).astype(self.dtype) self.B = np.random.random((10, 4)).astype(self.dtype) @@ -66,9 +64,8 @@ class TestMultiDotOp3Mat(TestMultiDotOp): self.check_grad(['x2'], 'Out', check_eager=True) -#A*(B*C) +# A*(B*C) class TestMultiDotOp3Mat2(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((3, 4)).astype(self.dtype) self.B = np.random.random((4, 8)).astype(self.dtype) @@ -83,15 +80,18 @@ class TestMultiDotOp3Mat2(TestMultiDotOp): class TestMultiDotOp4Mat(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((8, 6)).astype(self.dtype) self.B = np.random.random((6, 3)).astype(self.dtype) self.C = np.random.random((3, 4)).astype(self.dtype) self.D = np.random.random((4, 5)).astype(self.dtype) self.inputs = { - 'X': [('x0', self.A), ('x1', self.B), ('x2', self.C), - ('x3', self.D)] + 'X': [ + ('x0', self.A), + ('x1', self.B), + ('x2', self.C), + ('x3', self.D), + ] } self.outputs = {'Out': multi_dot([self.A, self.B, self.C, self.D])} @@ -103,7 +103,6 @@ class TestMultiDotOp4Mat(TestMultiDotOp): class TestMultiDotOpFirst1D(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((4)).astype(self.dtype) self.B = np.random.random((4, 3)).astype(self.dtype) @@ -112,7 +111,6 @@ class TestMultiDotOpFirst1D(TestMultiDotOp): class TestMultiDotOp3MatFirst1D(TestMultiDotOp3Mat): - def get_inputs_and_outputs(self): self.A = np.random.random((4)).astype(self.dtype) self.B = np.random.random((4, 3)).astype(self.dtype) @@ -122,21 +120,23 @@ class TestMultiDotOp3MatFirst1D(TestMultiDotOp3Mat): class TestMultiDotOp4MatFirst1D(TestMultiDotOp4Mat): - def get_inputs_and_outputs(self): self.A = np.random.random((4)).astype(self.dtype) self.B = np.random.random((4, 3)).astype(self.dtype) self.C = np.random.random((3, 4)).astype(self.dtype) self.D = np.random.random((4, 5)).astype(self.dtype) self.inputs = { - 'X': [('x0', self.A), ('x1', self.B), ('x2', self.C), - ('x3', self.D)] + 'X': [ + ('x0', self.A), + ('x1', self.B), + ('x2', self.C), + ('x3', self.D), + ] } self.outputs = {'Out': multi_dot([self.A, self.B, self.C, self.D])} class TestMultiDotOpLast1D(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((3, 6)).astype(self.dtype) self.B = np.random.random((6)).astype(self.dtype) @@ -145,7 +145,6 @@ class TestMultiDotOpLast1D(TestMultiDotOp): class TestMultiDotOp3MatLast1D(TestMultiDotOp3Mat): - def get_inputs_and_outputs(self): self.A = np.random.random((2, 4)).astype(self.dtype) self.B = np.random.random((4, 3)).astype(self.dtype) @@ -160,32 +159,33 @@ class TestMultiDotOp3MatLast1D(TestMultiDotOp3Mat): class TestMultiDotOp4MatLast1D(TestMultiDotOp4Mat): - def get_inputs_and_outputs(self): self.A = np.random.random((2, 3)).astype(self.dtype) self.B = np.random.random((3, 2)).astype(self.dtype) self.C = np.random.random((2, 3)).astype(self.dtype) self.D = np.random.random((3)).astype(self.dtype) self.inputs = { - 'X': [('x0', self.A), ('x1', self.B), ('x2', self.C), - ('x3', self.D)] + 'X': [ + ('x0', self.A), + ('x1', self.B), + ('x2', self.C), + ('x3', self.D), + ] } self.outputs = {'Out': multi_dot([self.A, self.B, self.C, self.D])} class TestMultiDotOpFirstAndLast1D(TestMultiDotOp): - def get_inputs_and_outputs(self): - self.A = np.random.random((4, )).astype(self.dtype) + self.A = np.random.random((4,)).astype(self.dtype) self.B = np.random.random((4)).astype(self.dtype) self.inputs = {'X': [('x0', self.A), ('x1', self.B)]} self.outputs = {'Out': multi_dot([self.A, self.B])} class TestMultiDotOp3MatFirstAndLast1D(TestMultiDotOp3Mat): - def get_inputs_and_outputs(self): - self.A = np.random.random((6, )).astype(self.dtype) + self.A = np.random.random((6,)).astype(self.dtype) self.B = np.random.random((6, 4)).astype(self.dtype) self.C = np.random.random((4)).astype(self.dtype) self.inputs = {'X': [('x0', self.A), ('x1', self.B), ('x2', self.C)]} @@ -193,52 +193,57 @@ class TestMultiDotOp3MatFirstAndLast1D(TestMultiDotOp3Mat): class TestMultiDotOp4MatFirstAndLast1D(TestMultiDotOp4Mat): - def get_inputs_and_outputs(self): - self.A = np.random.random((3, )).astype(self.dtype) + self.A = np.random.random((3,)).astype(self.dtype) self.B = np.random.random((3, 4)).astype(self.dtype) self.C = np.random.random((4, 2)).astype(self.dtype) self.D = np.random.random((2)).astype(self.dtype) self.inputs = { - 'X': [('x0', self.A), ('x1', self.B), ('x2', self.C), - ('x3', self.D)] + 'X': [ + ('x0', self.A), + ('x1', self.B), + ('x2', self.C), + ('x3', self.D), + ] } self.outputs = {'Out': multi_dot([self.A, self.B, self.C, self.D])} #####python API test####### class TestMultiDotOpError(unittest.TestCase): - def test_errors(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): # The inputs type of multi_dot must be list matrix. input1 = 12 - self.assertRaises(TypeError, paddle.linalg.multi_dot, - [input1, input1]) + self.assertRaises( + TypeError, paddle.linalg.multi_dot, [input1, input1] + ) # The inputs dtype of multi_dot must be float64, float64 or float16. - input2 = paddle.static.data(name='input2', - shape=[10, 10], - dtype="int32") - self.assertRaises(TypeError, paddle.linalg.multi_dot, - [input2, input2]) + input2 = paddle.static.data( + name='input2', shape=[10, 10], dtype="int32" + ) + self.assertRaises( + TypeError, paddle.linalg.multi_dot, [input2, input2] + ) # the number of tensor must be larger than 1 x0 = paddle.static.data(name='x0', shape=[3, 2], dtype="float64") self.assertRaises(ValueError, paddle.linalg.multi_dot, [x0]) - #the first tensor must be 1D or 2D + # the first tensor must be 1D or 2D x1 = paddle.static.data(name='x1', shape=[3, 2, 3], dtype="float64") x2 = paddle.static.data(name='x2', shape=[3, 2], dtype="float64") self.assertRaises(ValueError, paddle.linalg.multi_dot, [x1, x2]) - #the last tensor must be 1D or 2D + # the last tensor must be 1D or 2D x3 = paddle.static.data(name='x3', shape=[3, 2], dtype="float64") x4 = paddle.static.data(name='x4', shape=[3, 2, 2], dtype="float64") self.assertRaises(ValueError, paddle.linalg.multi_dot, [x3, x4]) - #the tensor must be 2D, except first and last tensor + # the tensor must be 2D, except first and last tensor x5 = paddle.static.data(name='x5', shape=[3, 2], dtype="float64") x6 = paddle.static.data(name='x6', shape=[2], dtype="float64") x7 = paddle.static.data(name='x7', shape=[2, 2], dtype="float64") @@ -246,7 +251,6 @@ class TestMultiDotOpError(unittest.TestCase): class APITestMultiDot(unittest.TestCase): - def test_out(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -256,11 +260,9 @@ class APITestMultiDot(unittest.TestCase): exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.random.rand(3, 2).astype("float64") data2 = np.random.rand(2, 3).astype("float64") - np_res, = exe.run(feed={ - 'x0': data1, - 'x1': data2 - }, - fetch_list=[result]) + (np_res,) = exe.run( + feed={'x0': data1, 'x1': data2}, fetch_list=[result] + ) expected_result = np.linalg.multi_dot([data1, data2]) np.testing.assert_allclose( @@ -269,7 +271,9 @@ class APITestMultiDot(unittest.TestCase): rtol=1e-05, atol=1e-05, err_msg='two value is {}\n{}, check diff!'.format( - np_res, expected_result)) + np_res, expected_result + ), + ) def test_dygraph_without_out(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_multi_label_soft_margin_loss.py b/python/paddle/fluid/tests/unittests/test_multi_label_soft_margin_loss.py index b26007a83ab..b5236553ca6 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_label_soft_margin_loss.py +++ b/python/paddle/fluid/tests/unittests/test_multi_label_soft_margin_loss.py @@ -24,7 +24,8 @@ def call_MultiLabelSoftMarginLoss_layer( reduction='mean', ): multilabel_margin_loss = paddle.nn.MultiLabelSoftMarginLoss( - weight=weight, reduction=reduction) + weight=weight, reduction=reduction + ) res = multilabel_margin_loss( input=input, label=label, @@ -47,55 +48,52 @@ def call_MultiLabelSoftMarginLoss_functional( return res -def test_static(place, - input_np, - label_np, - weight_np=None, - reduction='mean', - functional=False): +def test_static( + place, + input_np, + label_np, + weight_np=None, + reduction='mean', + functional=False, +): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype='float64') - label = paddle.static.data(name='label', - shape=label_np.shape, - dtype='float64') + input = paddle.static.data( + name='input', shape=input_np.shape, dtype='float64' + ) + label = paddle.static.data( + name='label', shape=label_np.shape, dtype='float64' + ) feed_dict = { "input": input_np, "label": label_np, } weight = None if weight_np is not None: - weight = paddle.static.data(name='weight', - shape=weight_np.shape, - dtype='float64') + weight = paddle.static.data( + name='weight', shape=weight_np.shape, dtype='float64' + ) feed_dict['weight'] = weight_np if functional: - res = call_MultiLabelSoftMarginLoss_functional(input=input, - label=label, - weight=weight, - reduction=reduction) + res = call_MultiLabelSoftMarginLoss_functional( + input=input, label=label, weight=weight, reduction=reduction + ) else: - res = call_MultiLabelSoftMarginLoss_layer(input=input, - label=label, - weight=weight, - reduction=reduction) + res = call_MultiLabelSoftMarginLoss_layer( + input=input, label=label, weight=weight, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, feed=feed_dict, fetch_list=[res]) + (static_result,) = exe.run(prog, feed=feed_dict, fetch_list=[res]) return static_result -def test_dygraph(place, - input_np, - label_np, - weight=None, - reduction='mean', - functional=False): +def test_dygraph( + place, input_np, label_np, weight=None, reduction='mean', functional=False +): with paddle.fluid.dygraph.base.guard(): input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) @@ -104,12 +102,12 @@ def test_dygraph(place, if functional: dy_res = call_MultiLabelSoftMarginLoss_functional( - input=input, label=label, weight=weight, reduction=reduction) + input=input, label=label, weight=weight, reduction=reduction + ) else: - dy_res = call_MultiLabelSoftMarginLoss_layer(input=input, - label=label, - weight=weight, - reduction=reduction) + dy_res = call_MultiLabelSoftMarginLoss_layer( + input=input, label=label, weight=weight, reduction=reduction + ) dy_result = dy_res.numpy() return dy_result @@ -120,7 +118,6 @@ def calc_multilabel_margin_loss( weight=None, reduction="mean", ): - def LogSigmoid(x): return np.log(1 / (1 + np.exp(-x))) @@ -140,7 +137,6 @@ def calc_multilabel_margin_loss( class TestMultiLabelMarginLoss(unittest.TestCase): - def test_MultiLabelSoftMarginLoss(self): input = np.random.uniform(0.1, 0.8, size=(5, 5)).astype(np.float64) label = np.random.randint(0, 2, size=(5, 5)).astype(np.float64) @@ -151,52 +147,64 @@ class TestMultiLabelMarginLoss(unittest.TestCase): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - expected = calc_multilabel_margin_loss(input=input, - label=label, - reduction=reduction) - - dy_result = test_dygraph(place=place, - input_np=input, - label_np=label, - reduction=reduction) - - static_result = test_static(place=place, - input_np=input, - label_np=label, - reduction=reduction) + expected = calc_multilabel_margin_loss( + input=input, label=label, reduction=reduction + ) + + dy_result = test_dygraph( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + ) + + static_result = test_static( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place=place, - input_np=input, - label_np=label, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input_np=input, - label_np=label, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + static_functional = test_static( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_MultiLabelSoftMarginLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.MultiLabelSoftMarginLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.MultiLabelSoftMarginLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.multi_label_soft_margin_loss, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.multi_label_soft_margin_loss, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def test_MultiLabelSoftMarginLoss_weights(self): @@ -205,37 +213,44 @@ class TestMultiLabelMarginLoss(unittest.TestCase): weight = np.random.randint(0, 2, size=(5, 5)).astype(np.float64) place = 'cpu' reduction = 'mean' - expected = calc_multilabel_margin_loss(input=input, - label=label, - weight=weight, - reduction=reduction) - - dy_result = test_dygraph(place=place, - input_np=input, - label_np=label, - weight=weight, - reduction=reduction) - - static_result = test_static(place=place, - input_np=input, - label_np=label, - weight_np=weight, - reduction=reduction) + expected = calc_multilabel_margin_loss( + input=input, label=label, weight=weight, reduction=reduction + ) + + dy_result = test_dygraph( + place=place, + input_np=input, + label_np=label, + weight=weight, + reduction=reduction, + ) + + static_result = test_static( + place=place, + input_np=input, + label_np=label, + weight_np=weight, + reduction=reduction, + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place=place, - input_np=input, - label_np=label, - weight_np=weight, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input_np=input, - label_np=label, - weight=weight, - reduction=reduction, - functional=True) + static_functional = test_static( + place=place, + input_np=input, + label_np=label, + weight_np=weight, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input_np=input, + label_np=label, + weight=weight, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-05) np.testing.assert_allclose(static_functional, dy_functional, rtol=1e-05) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) @@ -245,10 +260,12 @@ class TestMultiLabelMarginLoss(unittest.TestCase): input = paddle.to_tensor([[0.1, 0.3], [1, 2]], dtype='float32') label = paddle.to_tensor([[0.2, 0.1]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.multi_label_soft_margin_loss, - input=input, - label=label) + self.assertRaises( + ValueError, + paddle.nn.functional.multi_label_soft_margin_loss, + input=input, + label=label, + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py index bd03b646ddc..cde6c8daf96 100644 --- a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -18,42 +18,70 @@ import copy from op_test import OpTest import paddle import paddle.fluid as fluid -from paddle.fluid import Program, program_guard, in_dygraph_mode, _non_static_mode +from paddle.fluid import ( + Program, + program_guard, + in_dygraph_mode, + _non_static_mode, +) from paddle.fluid.layer_helper import LayerHelper from paddle import _C_ops, _legacy_C_ops -def multiclass_nms3(bboxes, - scores, - rois_num=None, - score_threshold=0.3, - nms_top_k=1000, - keep_top_k=100, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=-1, - return_index=True, - return_rois_num=True, - name=None): +def multiclass_nms3( + bboxes, + scores, + rois_num=None, + score_threshold=0.3, + nms_top_k=1000, + keep_top_k=100, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=-1, + return_index=True, + return_rois_num=True, + name=None, +): helper = LayerHelper('multiclass_nms3', **locals()) if in_dygraph_mode(): - attrs = (score_threshold, nms_top_k, keep_top_k, nms_threshold, - normalized, nms_eta, background_label) + attrs = ( + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold, + normalized, + nms_eta, + background_label, + ) output, index, nms_rois_num = _C_ops.multiclass_nms3( - bboxes, scores, rois_num, *attrs) + bboxes, scores, rois_num, *attrs + ) if not return_index: index = None return output, index, nms_rois_num elif _non_static_mode(): - attrs = ('background_label', background_label, 'score_threshold', - score_threshold, 'nms_top_k', nms_top_k, 'nms_threshold', - nms_threshold, 'keep_top_k', keep_top_k, 'nms_eta', nms_eta, - 'normalized', normalized) + attrs = ( + 'background_label', + background_label, + 'score_threshold', + score_threshold, + 'nms_top_k', + nms_top_k, + 'nms_threshold', + nms_threshold, + 'keep_top_k', + keep_top_k, + 'nms_eta', + nms_eta, + 'normalized', + normalized, + ) output, index, nms_rois_num = _legacy_C_ops.multiclass_nms3( - bboxes, scores, rois_num, *attrs) + bboxes, scores, rois_num, *attrs + ) if not return_index: index = None return output, index, nms_rois_num @@ -70,21 +98,24 @@ def multiclass_nms3(bboxes, if return_rois_num: nms_rois_num = helper.create_variable_for_type_inference( - dtype='int32') + dtype='int32' + ) outputs['NmsRoisNum'] = nms_rois_num - helper.append_op(type="multiclass_nms3", - inputs=inputs, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs=outputs) + helper.append_op( + type="multiclass_nms3", + inputs=inputs, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs=outputs, + ) output.stop_gradient = True index.stop_gradient = True if not return_index: @@ -98,14 +129,13 @@ def multiclass_nms3(bboxes, def softmax(x): # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) def iou(box_a, box_b, norm): - """Apply intersection-over-union overlap between box_a and box_b - """ + """Apply intersection-over-union overlap between box_a and box_b""" xmin_a = min(box_a[0], box_a[2]) ymin_a = min(box_a[1], box_a[3]) xmax_a = max(box_a[0], box_a[2]) @@ -116,10 +146,12 @@ def iou(box_a, box_b, norm): xmax_b = max(box_b[0], box_b[2]) ymax_b = max(box_b[1], box_b[3]) - area_a = (ymax_a - ymin_a + (norm == False)) * (xmax_a - xmin_a + - (norm == False)) - area_b = (ymax_b - ymin_b + (norm == False)) * (xmax_b - xmin_b + - (norm == False)) + area_a = (ymax_a - ymin_a + (norm == False)) * ( + xmax_a - xmin_a + (norm == False) + ) + area_b = (ymax_b - ymin_b + (norm == False)) * ( + xmax_b - xmin_b + (norm == False) + ) if area_a <= 0 and area_b <= 0: return 0.0 @@ -128,21 +160,24 @@ def iou(box_a, box_b, norm): xb = min(xmax_a, xmax_b) yb = min(ymax_a, ymax_b) - inter_area = max(xb - xa + - (norm == False), 0.0) * max(yb - ya + (norm == False), 0.0) + inter_area = max(xb - xa + (norm == False), 0.0) * max( + yb - ya + (norm == False), 0.0 + ) iou_ratio = inter_area / (area_a + area_b - inter_area) return iou_ratio -def nms(boxes, - scores, - score_threshold, - nms_threshold, - top_k=200, - normalized=True, - eta=1.0): +def nms( + boxes, + scores, + score_threshold, + nms_threshold, + top_k=200, + normalized=True, + eta=1.0, +): """Apply non-maximum suppression at test time to avoid detecting too many overlapping bounding boxes for a given object. Args: @@ -189,8 +224,17 @@ def nms(boxes, return selected_indices -def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, - nms_top_k, keep_top_k, normalized, shared): +def multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared, +): if shared: class_num = scores.shape[0] priorbox_num = scores.shape[1] @@ -201,13 +245,26 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, selected_indices = {} num_det = 0 for c in range(class_num): - if c == background: continue + if c == background: + continue if shared: - indices = nms(boxes, scores[c], score_threshold, nms_threshold, - nms_top_k, normalized) + indices = nms( + boxes, + scores[c], + score_threshold, + nms_threshold, + nms_top_k, + normalized, + ) else: - indices = nms(boxes[:, c, :], scores[:, c], score_threshold, - nms_threshold, nms_top_k, normalized) + indices = nms( + boxes[:, c, :], + scores[:, c], + score_threshold, + nms_threshold, + nms_top_k, + normalized, + ) selected_indices[c] = indices num_det += len(indices) @@ -220,9 +277,9 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, else: score_index.append((scores[idx][c], c, idx)) - sorted_score_index = sorted(score_index, - key=lambda tup: tup[0], - reverse=True) + sorted_score_index = sorted( + score_index, key=lambda tup: tup[0], reverse=True + ) sorted_score_index = sorted_score_index[:keep_top_k] selected_indices = {} @@ -238,9 +295,17 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, return selected_indices, num_det -def lod_multiclass_nms(boxes, scores, background, score_threshold, - nms_threshold, nms_top_k, keep_top_k, box_lod, - normalized): +def lod_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + box_lod, + normalized, +): num_class = boxes.shape[1] det_outs = [] lod = [] @@ -249,19 +314,21 @@ def lod_multiclass_nms(boxes, scores, background, score_threshold, if box_lod[0][n] == 0: lod.append(0) continue - box = boxes[head:head + box_lod[0][n]] - score = scores[head:head + box_lod[0][n]] + box = boxes[head : head + box_lod[0][n]] + score = scores[head : head + box_lod[0][n]] offset = head head = head + box_lod[0][n] - nmsed_outs, nmsed_num = multiclass_nms(box, - score, - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized, - shared=False) + nmsed_outs, nmsed_num = multiclass_nms( + box, + score, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared=False, + ) lod.append(nmsed_num) if nmsed_num == 0: @@ -270,41 +337,52 @@ def lod_multiclass_nms(boxes, scores, background, score_threshold, for c, indices in nmsed_outs.items(): for idx in indices: xmin, ymin, xmax, ymax = box[idx, c, :] - tmp_det_out.append([ - c, score[idx][c], xmin, ymin, xmax, ymax, - offset * num_class + idx * num_class + c - ]) - sorted_det_out = sorted(tmp_det_out, - key=lambda tup: tup[0], - reverse=False) + tmp_det_out.append( + [ + c, + score[idx][c], + xmin, + ymin, + xmax, + ymax, + offset * num_class + idx * num_class + c, + ] + ) + sorted_det_out = sorted( + tmp_det_out, key=lambda tup: tup[0], reverse=False + ) det_outs.extend(sorted_det_out) return det_outs, lod -def batched_multiclass_nms(boxes, - scores, - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized=True): +def batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized=True, +): batch_size = scores.shape[0] num_boxes = scores.shape[2] det_outs = [] index_outs = [] lod = [] for n in range(batch_size): - nmsed_outs, nmsed_num = multiclass_nms(boxes[n], - scores[n], - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized, - shared=True) + nmsed_outs, nmsed_num = multiclass_nms( + boxes[n], + scores[n], + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared=True, + ) lod.append(nmsed_num) if nmsed_num == 0: @@ -313,19 +391,25 @@ def batched_multiclass_nms(boxes, for c, indices in nmsed_outs.items(): for idx in indices: xmin, ymin, xmax, ymax = boxes[n][idx][:] - tmp_det_out.append([ - c, scores[n][c][idx], xmin, ymin, xmax, ymax, - idx + n * num_boxes - ]) - sorted_det_out = sorted(tmp_det_out, - key=lambda tup: tup[0], - reverse=False) + tmp_det_out.append( + [ + c, + scores[n][c][idx], + xmin, + ymin, + xmax, + ymax, + idx + n * num_boxes, + ] + ) + sorted_det_out = sorted( + tmp_det_out, key=lambda tup: tup[0], reverse=False + ) det_outs.extend(sorted_det_out) return det_outs, lod class TestMulticlassNMSOp(OpTest): - def set_argument(self): self.score_threshold = 0.01 @@ -351,9 +435,15 @@ class TestMulticlassNMSOp(OpTest): boxes[:, :, 0:2] = boxes[:, :, 0:2] * 0.5 boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5 - det_outs, lod = batched_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k) + det_outs, lod = batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) lod = [1] if not det_outs else lod det_outs = [[-1, 0]] if not det_outs else det_outs det_outs = np.array(det_outs) @@ -377,7 +467,6 @@ class TestMulticlassNMSOp(OpTest): class TestMulticlassNMSOpNoOutput(TestMulticlassNMSOp): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 @@ -385,7 +474,6 @@ class TestMulticlassNMSOpNoOutput(TestMulticlassNMSOp): class TestMulticlassNMSLoDInput(OpTest): - def set_argument(self): self.score_threshold = 0.01 @@ -412,13 +500,21 @@ class TestMulticlassNMSLoDInput(OpTest): boxes[:, :, 2] = boxes[:, :, 2] * 10 + 10 boxes[:, :, 3] = boxes[:, :, 3] * 10 + 10 - det_outs, lod = lod_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k, box_lod, - normalized) + det_outs, lod = lod_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + box_lod, + normalized, + ) det_outs = np.array(det_outs).astype('float32') - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms' self.inputs = { 'BBoxes': (boxes, box_lod), @@ -440,7 +536,6 @@ class TestMulticlassNMSLoDInput(OpTest): class TestMulticlassNMSNoBox(TestMulticlassNMSLoDInput): - def setUp(self): self.set_argument() M = 1200 @@ -464,13 +559,21 @@ class TestMulticlassNMSNoBox(TestMulticlassNMSLoDInput): boxes[:, :, 2] = boxes[:, :, 2] * 10 + 10 boxes[:, :, 3] = boxes[:, :, 3] * 10 + 10 - det_outs, lod = lod_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k, box_lod, - normalized) + det_outs, lod = lod_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + box_lod, + normalized, + ) det_outs = np.array(det_outs).astype('float32') - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms' self.inputs = { 'BBoxes': (boxes, box_lod), @@ -489,7 +592,6 @@ class TestMulticlassNMSNoBox(TestMulticlassNMSLoDInput): class TestIOU(unittest.TestCase): - def test_iou(self): box1 = np.array([4.0, 3.0, 7.0, 5.0]).astype('float32') box2 = np.array([3.0, 4.0, 6.0, 8.0]).astype('float32') @@ -500,7 +602,6 @@ class TestIOU(unittest.TestCase): class TestMulticlassNMS2Op(TestMulticlassNMSOp): - def setUp(self): self.set_argument() N = 7 @@ -523,20 +624,28 @@ class TestMulticlassNMS2Op(TestMulticlassNMSOp): boxes[:, :, 0:2] = boxes[:, :, 0:2] * 0.5 boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5 - det_outs, lod = batched_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k) + det_outs, lod = batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) det_outs = np.array(det_outs) - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs - index_outs = det_outs[:, - -1:].astype('int') if len(det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) + index_outs = ( + det_outs[:, -1:].astype('int') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms2' self.inputs = {'BBoxes': boxes, 'Scores': scores} self.outputs = { 'Out': (nmsed_outs, [lod]), - 'Index': (index_outs, [lod]) + 'Index': (index_outs, [lod]), } self.attrs = { 'background_label': 0, @@ -553,7 +662,6 @@ class TestMulticlassNMS2Op(TestMulticlassNMSOp): class TestMulticlassNMS2OpNoOutput(TestMulticlassNMS2Op): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 @@ -561,7 +669,6 @@ class TestMulticlassNMS2OpNoOutput(TestMulticlassNMS2Op): class TestMulticlassNMS2LoDInput(TestMulticlassNMSLoDInput): - def setUp(self): self.set_argument() M = 1200 @@ -585,16 +692,25 @@ class TestMulticlassNMS2LoDInput(TestMulticlassNMSLoDInput): boxes[:, :, 2] = boxes[:, :, 2] * 10 + 10 boxes[:, :, 3] = boxes[:, :, 3] * 10 + 10 - det_outs, lod = lod_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k, box_lod, - normalized) + det_outs, lod = lod_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + box_lod, + normalized, + ) det_outs = np.array(det_outs) - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs - index_outs = det_outs[:, - -1:].astype('int') if len(det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) + index_outs = ( + det_outs[:, -1:].astype('int') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms2' self.inputs = { 'BBoxes': (boxes, box_lod), @@ -602,7 +718,7 @@ class TestMulticlassNMS2LoDInput(TestMulticlassNMSLoDInput): } self.outputs = { 'Out': (nmsed_outs, [lod]), - 'Index': (index_outs, [lod]) + 'Index': (index_outs, [lod]), } self.attrs = { 'background_label': 0, @@ -620,7 +736,6 @@ def test_check_output(self): class TestMulticlassNMS2LoDNoOutput(TestMulticlassNMS2LoDInput): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 @@ -628,7 +743,6 @@ class TestMulticlassNMS2LoDNoOutput(TestMulticlassNMS2LoDInput): class TestMulticlassNMSError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): M = 1200 @@ -642,12 +756,12 @@ class TestMulticlassNMSError(unittest.TestCase): scores = np.reshape(scores, (N, M, C)) scores_np = np.transpose(scores, (0, 2, 1)) - boxes_data = fluid.data(name='bboxes', - shape=[M, C, BOX_SIZE], - dtype='float32') - scores_data = fluid.data(name='scores', - shape=[N, C, M], - dtype='float32') + boxes_data = fluid.data( + name='bboxes', shape=[M, C, BOX_SIZE], dtype='float32' + ) + scores_data = fluid.data( + name='scores', shape=[N, C, M], dtype='float32' + ) def test_bboxes_Variable(): # the bboxes type must be Variable @@ -662,7 +776,6 @@ class TestMulticlassNMSError(unittest.TestCase): class TestMulticlassNMS3Op(TestMulticlassNMS2Op): - def setUp(self): self.python_api = multiclass_nms3 self.set_argument() @@ -686,21 +799,29 @@ class TestMulticlassNMS3Op(TestMulticlassNMS2Op): boxes[:, :, 0:2] = boxes[:, :, 0:2] * 0.5 boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5 - det_outs, lod = batched_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k) + det_outs, lod = batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) det_outs = np.array(det_outs) - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs - index_outs = det_outs[:, - -1:].astype('int') if len(det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) + index_outs = ( + det_outs[:, -1:].astype('int') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms3' self.inputs = {'BBoxes': boxes, 'Scores': scores} self.outputs = { 'Out': nmsed_outs, 'Index': index_outs, - 'NmsRoisNum': np.array(lod).astype('int32') + 'NmsRoisNum': np.array(lod).astype('int32'), } self.attrs = { 'background_label': 0, @@ -717,7 +838,6 @@ class TestMulticlassNMS3Op(TestMulticlassNMS2Op): class TestMulticlassNMS3OpNoOutput(TestMulticlassNMS3Op): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 diff --git a/python/paddle/fluid/tests/unittests/test_multihead_attention.py b/python/paddle/fluid/tests/unittests/test_multihead_attention.py index 589b62d7641..8c6b3f96f31 100644 --- a/python/paddle/fluid/tests/unittests/test_multihead_attention.py +++ b/python/paddle/fluid/tests/unittests/test_multihead_attention.py @@ -19,42 +19,44 @@ import numpy as np class TestMultiheadAttention(unittest.TestCase): - def gen_random_input(self): - """Generate random input data. - """ + """Generate random input data.""" # batch_size, max_sequence_length, hidden dimension self.input_shape = (3, 13, 16) self.queries = np.random.random(size=self.input_shape).astype("float32") self.keys = np.random.random(size=self.input_shape).astype("float32") def set_program(self): - """Build the test program. - """ - queries = fluid.layers.data(name="queries", - shape=self.input_shape, - dtype="float32", - append_batch_size=False) + """Build the test program.""" + queries = fluid.layers.data( + name="queries", + shape=self.input_shape, + dtype="float32", + append_batch_size=False, + ) queries.stop_gradient = False - keys = fluid.layers.data(name="keys", - shape=self.input_shape, - dtype="float32", - append_batch_size=False) + keys = fluid.layers.data( + name="keys", + shape=self.input_shape, + dtype="float32", + append_batch_size=False, + ) keys.stop_gradient = False - contexts = fluid.nets.scaled_dot_product_attention(queries=queries, - keys=keys, - values=keys, - num_heads=8, - dropout_rate=0.) + contexts = fluid.nets.scaled_dot_product_attention( + queries=queries, + keys=keys, + values=keys, + num_heads=8, + dropout_rate=0.0, + ) out = fluid.layers.reduce_sum(contexts, dim=None) fluid.backward.append_backward(loss=out) self.fetch_list = [contexts] def run_program(self): - """Run the test program. - """ + """Run the test program.""" places = [core.CPUPlace()] if core.is_compiled_with_cuda(): places.append(core.CUDAPlace(0)) @@ -64,15 +66,16 @@ class TestMultiheadAttention(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - output = exe.run(fluid.default_main_program(), - feed=self.inputs, - fetch_list=self.fetch_list, - return_numpy=True) + output = exe.run( + fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=True, + ) self.op_output = output def set_inputs(self, place): - """Set the randomly generated data to the test program. - """ + """Set the randomly generated data to the test program.""" self.inputs = {} queries = fluid.Tensor() queries.set(self.queries, place) @@ -89,7 +92,7 @@ class TestMultiheadAttention(unittest.TestCase): self.set_program() self.run_program() - #fixme(caoying) add more meaningfull unittest. + # fixme(caoying) add more meaningfull unittest. if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_multimarginloss.py b/python/paddle/fluid/tests/unittests/test_multimarginloss.py index 1eff1deb692..3a54f752eb2 100644 --- a/python/paddle/fluid/tests/unittests/test_multimarginloss.py +++ b/python/paddle/fluid/tests/unittests/test_multimarginloss.py @@ -26,10 +26,9 @@ def call_MultiMarginLoss_layer( weight=None, reduction='mean', ): - triplet_margin_loss = paddle.nn.MultiMarginLoss(p=p, - margin=margin, - weight=weight, - reduction=reduction) + triplet_margin_loss = paddle.nn.MultiMarginLoss( + p=p, margin=margin, weight=weight, reduction=reduction + ) res = triplet_margin_loss( input=input, label=label, @@ -45,82 +44,92 @@ def call_MultiMarginLoss_functional( weight=None, reduction='mean', ): - res = paddle.nn.functional.multi_margin_loss(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + res = paddle.nn.functional.multi_margin_loss( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) return res -def test_static(place, - input_np, - label_np, - p=1, - margin=1.0, - weight_np=None, - reduction='mean', - functional=False): +def test_static( + place, + input_np, + label_np, + p=1, + margin=1.0, + weight_np=None, + reduction='mean', + functional=False, +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype=input_np.dtype) - label = paddle.static.data(name='label', - shape=label_np.shape, - dtype=label_np.dtype) + input = paddle.static.data( + name='input', shape=input_np.shape, dtype=input_np.dtype + ) + label = paddle.static.data( + name='label', shape=label_np.shape, dtype=label_np.dtype + ) feed_dict = { "input": input_np, "label": label_np, } weight = None if weight_np is not None: - weight = paddle.static.data(name='weight', - shape=weight_np.shape, - dtype=weight_np.dtype) + weight = paddle.static.data( + name='weight', shape=weight_np.shape, dtype=weight_np.dtype + ) feed_dict['weight'] = weight_np if functional: - res = call_MultiMarginLoss_functional(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + res = call_MultiMarginLoss_functional( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) else: - res = call_MultiMarginLoss_layer(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + res = call_MultiMarginLoss_layer( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) exe = paddle.static.Executor(place) static_result = exe.run(prog, feed=feed_dict, fetch_list=[res]) return static_result[0] -def test_static_data_shape(place, - input_np, - label_np, - wrong_label_shape=None, - weight_np=None, - wrong_weight_shape=None, - functional=False): +def test_static_data_shape( + place, + input_np, + label_np, + wrong_label_shape=None, + weight_np=None, + wrong_weight_shape=None, + functional=False, +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype=input_np.dtype) + input = paddle.static.data( + name='input', shape=input_np.shape, dtype=input_np.dtype + ) if wrong_label_shape is None: label_shape = label_np.shape else: label_shape = wrong_label_shape - label = paddle.static.data(name='label', - shape=label_shape, - dtype=label_np.dtype) + label = paddle.static.data( + name='label', shape=label_shape, dtype=label_np.dtype + ) feed_dict = { "input": input_np, "label": label_np, @@ -131,9 +140,9 @@ def test_static_data_shape(place, weight_shape = weight_np.shape else: weight_shape = wrong_weight_shape - weight = paddle.static.data(name='weight', - shape=weight_shape, - dtype=weight_np.dtype) + weight = paddle.static.data( + name='weight', shape=weight_shape, dtype=weight_np.dtype + ) feed_dict['weight'] = weight_np if functional: res = call_MultiMarginLoss_functional( @@ -153,14 +162,16 @@ def test_static_data_shape(place, return static_result -def test_dygraph(place, - input, - label, - p=1, - margin=1.0, - weight=None, - reduction='mean', - functional=False): +def test_dygraph( + place, + input, + label, + p=1, + margin=1.0, + weight=None, + reduction='mean', + functional=False, +): paddle.disable_static() input = paddle.to_tensor(input) label = paddle.to_tensor(label) @@ -168,19 +179,23 @@ def test_dygraph(place, if weight is not None: weight = paddle.to_tensor(weight) if functional: - dy_res = call_MultiMarginLoss_functional(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + dy_res = call_MultiMarginLoss_functional( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) else: - dy_res = call_MultiMarginLoss_layer(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + dy_res = call_MultiMarginLoss_layer( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -194,16 +209,22 @@ def calc_multi_margin_loss( weight=None, reduction='mean', ): - index_sample = np.array([input[i, label[i]] - for i in range(label.size)]).reshape(-1, 1) + index_sample = np.array( + [input[i, label[i]] for i in range(label.size)] + ).reshape(-1, 1) if weight is None: - expected = np.mean(np.maximum(margin + input - index_sample, 0.0)**p, - axis=1) - margin**p / input.shape[1] + expected = ( + np.mean(np.maximum(margin + input - index_sample, 0.0) ** p, axis=1) + - margin**p / input.shape[1] + ) else: - weight = np.array([weight[label[i]] - for i in range(label.size)]).reshape(-1, 1) - expected = np.mean(np.maximum(weight * (margin + input - index_sample), 0.0) ** p, axis=1) - weight*(margin ** p / \ - input.shape[1]) + weight = np.array( + [weight[label[i]] for i in range(label.size)] + ).reshape(-1, 1) + expected = np.mean( + np.maximum(weight * (margin + input - index_sample), 0.0) ** p, + axis=1, + ) - weight * (margin**p / input.shape[1]) if reduction == 'mean': expected = np.mean(expected) @@ -216,14 +237,14 @@ def calc_multi_margin_loss( class TestMultiMarginLoss(unittest.TestCase): - def test_MultiMarginLoss(self): batch_size = 5 num_classes = 2 shape = (batch_size, num_classes) input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) - label = np.random.uniform(0, input.shape[1], - size=(batch_size, )).astype(np.int64) + label = np.random.uniform(0, input.shape[1], size=(batch_size,)).astype( + np.int64 + ) places = [paddle.CPUPlace()] if paddle.device.is_compiled_with_cuda(): @@ -231,9 +252,9 @@ class TestMultiMarginLoss(unittest.TestCase): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - expected = calc_multi_margin_loss(input=input, - label=label, - reduction=reduction) + expected = calc_multi_margin_loss( + input=input, label=label, reduction=reduction + ) dy_result = test_dygraph( place=place, @@ -251,32 +272,40 @@ class TestMultiMarginLoss(unittest.TestCase): np.testing.assert_allclose(static_result, expected) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected) - static_functional = test_static(place=place, - input_np=input, - label_np=label, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - label=label, - reduction=reduction, - functional=True) + static_functional = test_static( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + label=label, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected) def test_MultiMarginLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.MultiMarginLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.MultiMarginLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([0], dtype='int32') - self.assertRaises(ValueError, - paddle.nn.functional.multi_margin_loss, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.multi_margin_loss, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def test_MultiMarginLoss_dimension(self): @@ -308,12 +337,12 @@ class TestMultiMarginLoss(unittest.TestCase): reduction = 'mean' place = paddle.CPUPlace() input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) - label = np.random.uniform(0, input.shape[1], - size=(batch_size, )).astype(np.int64) - expected = calc_multi_margin_loss(input=input, - p=p, - label=label, - reduction=reduction) + label = np.random.uniform(0, input.shape[1], size=(batch_size,)).astype( + np.int64 + ) + expected = calc_multi_margin_loss( + input=input, p=p, label=label, reduction=reduction + ) dy_result = test_dygraph( place=place, @@ -333,18 +362,22 @@ class TestMultiMarginLoss(unittest.TestCase): np.testing.assert_allclose(static_result, expected) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected) - static_functional = test_static(place=place, - p=p, - input_np=input, - label_np=label, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - p=p, - input=input, - label=label, - reduction=reduction, - functional=True) + static_functional = test_static( + place=place, + p=p, + input_np=input, + label_np=label, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + p=p, + input=input, + label=label, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected) @@ -356,14 +389,13 @@ class TestMultiMarginLoss(unittest.TestCase): reduction = 'mean' place = paddle.CPUPlace() input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) - label = np.random.uniform(0, input.shape[1], - size=(batch_size, )).astype(np.int64) - weight = np.random.uniform(0, 2, - size=(num_classes, )).astype(np.float64) - expected = calc_multi_margin_loss(input=input, - label=label, - weight=weight, - reduction=reduction) + label = np.random.uniform(0, input.shape[1], size=(batch_size,)).astype( + np.int64 + ) + weight = np.random.uniform(0, 2, size=(num_classes,)).astype(np.float64) + expected = calc_multi_margin_loss( + input=input, label=label, weight=weight, reduction=reduction + ) dy_result = test_dygraph( place=place, @@ -383,18 +415,22 @@ class TestMultiMarginLoss(unittest.TestCase): np.testing.assert_allclose(static_result, expected) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected) - static_functional = test_static(place=place, - input_np=input, - label_np=label, - weight_np=weight, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - label=label, - weight=weight, - reduction=reduction, - functional=True) + static_functional = test_static( + place=place, + input_np=input, + label_np=label, + weight_np=weight, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + label=label, + weight=weight, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected) @@ -405,10 +441,10 @@ class TestMultiMarginLoss(unittest.TestCase): shape = (batch_size, num_classes) place = paddle.CPUPlace() input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) - label = np.random.uniform(0, input.shape[1], - size=(batch_size, )).astype(np.int64) - weight = np.random.uniform(0, 2, - size=(num_classes, )).astype(np.float64) + label = np.random.uniform(0, input.shape[1], size=(batch_size,)).astype( + np.int64 + ) + weight = np.random.uniform(0, 2, size=(num_classes,)).astype(np.float64) self.assertRaises( ValueError, @@ -416,7 +452,7 @@ class TestMultiMarginLoss(unittest.TestCase): place=place, input_np=input, label_np=label, - wrong_label_shape=(10, ), + wrong_label_shape=(10,), functional=True, ) self.assertRaises( @@ -425,7 +461,7 @@ class TestMultiMarginLoss(unittest.TestCase): place=place, input_np=input, label_np=label, - wrong_label_shape=(10, ), + wrong_label_shape=(10,), functional=False, ) self.assertRaises( @@ -435,7 +471,7 @@ class TestMultiMarginLoss(unittest.TestCase): input_np=input, label_np=label, weight_np=weight, - wrong_weight_shape=(3, ), + wrong_weight_shape=(3,), functional=True, ) self.assertRaises( @@ -445,7 +481,7 @@ class TestMultiMarginLoss(unittest.TestCase): input_np=input, label_np=label, weight_np=weight, - wrong_weight_shape=(3, ), + wrong_weight_shape=(3,), functional=False, ) diff --git a/python/paddle/fluid/tests/unittests/test_multinomial_op.py b/python/paddle/fluid/tests/unittests/test_multinomial_op.py index 5df70185dda..f79dee65250 100644 --- a/python/paddle/fluid/tests/unittests/test_multinomial_op.py +++ b/python/paddle/fluid/tests/unittests/test_multinomial_op.py @@ -43,7 +43,6 @@ def sample_output_two_dimension(out, shape): class TestMultinomialOp(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "multinomial" @@ -66,16 +65,16 @@ class TestMultinomialOp(OpTest): # normalize the input to get the probability prob = self.input_np / self.input_np.sum(axis=-1, keepdims=True) sample_prob = self.sample_output(np.array(outs[0])) - np.testing.assert_allclose(sample_prob, - prob, - rtol=0, - atol=0.01, - err_msg='sample_prob: ' + str(sample_prob) + - '\nprob: ' + str(prob)) + np.testing.assert_allclose( + sample_prob, + prob, + rtol=0, + atol=0.01, + err_msg='sample_prob: ' + str(sample_prob) + '\nprob: ' + str(prob), + ) class TestMultinomialOp2(TestMultinomialOp): - def init_data(self): # input probability is a matrix self.input_np = np.random.rand(3, 4) @@ -87,7 +86,6 @@ class TestMultinomialOp2(TestMultinomialOp): class TestMultinomialOp3(TestMultinomialOp): - def init_data(self): # replacement is False. number of samples must be less than number of categories. self.input_np = np.random.rand(1000) @@ -98,12 +96,13 @@ class TestMultinomialOp3(TestMultinomialOp): out = np.array(outs[0]) unique_out = np.unique(out) self.assertEqual( - len(unique_out), 100, - "replacement is False. categories can't be sampled repeatedly") + len(unique_out), + 100, + "replacement is False. categories can't be sampled repeatedly", + ) class TestMultinomialApi(unittest.TestCase): - def test_dygraph(self): # input probability is a vector, and replacement is True paddle.disable_static() @@ -114,12 +113,13 @@ class TestMultinomialApi(unittest.TestCase): sample_prob = sample_output_one_dimension(out.numpy(), 4) prob = x_numpy / x_numpy.sum(axis=-1, keepdims=True) - np.testing.assert_allclose(sample_prob, - prob, - rtol=0, - atol=0.01, - err_msg='sample_prob: ' + str(sample_prob) + - '\nprob: ' + str(prob)) + np.testing.assert_allclose( + sample_prob, + prob, + rtol=0, + atol=0.01, + err_msg='sample_prob: ' + str(sample_prob) + '\nprob: ' + str(prob), + ) def test_dygraph2(self): # input probability is a matrix, and replacement is True @@ -130,12 +130,13 @@ class TestMultinomialApi(unittest.TestCase): sample_prob = sample_output_two_dimension(out.numpy(), [3, 4]) prob = x_numpy / x_numpy.sum(axis=-1, keepdims=True) - np.testing.assert_allclose(sample_prob, - prob, - rtol=0, - atol=0.01, - err_msg='sample_prob: ' + str(sample_prob) + - '\nprob: ' + str(prob)) + np.testing.assert_allclose( + sample_prob, + prob, + rtol=0, + atol=0.01, + err_msg='sample_prob: ' + str(sample_prob) + '\nprob: ' + str(prob), + ) paddle.enable_static() def test_dygraph3(self): @@ -147,8 +148,10 @@ class TestMultinomialApi(unittest.TestCase): unique_out = np.unique(out.numpy()) self.assertEqual( - len(unique_out), 100, - "replacement is False. categories can't be sampled repeatedly") + len(unique_out), + 100, + "replacement is False. categories can't be sampled repeatedly", + ) paddle.enable_static() def test_dygraph4(self): @@ -178,16 +181,16 @@ class TestMultinomialApi(unittest.TestCase): sample_prob = sample_output_one_dimension(out, 4) prob = x_np / x_np.sum(axis=-1, keepdims=True) - np.testing.assert_allclose(sample_prob, - prob, - rtol=0, - atol=0.01, - err_msg='sample_prob: ' + str(sample_prob) + - '\nprob: ' + str(prob)) + np.testing.assert_allclose( + sample_prob, + prob, + rtol=0, + atol=0.01, + err_msg='sample_prob: ' + str(sample_prob) + '\nprob: ' + str(prob), + ) class TestMultinomialAlias(unittest.TestCase): - def test_alias(self): paddle.disable_static() x = paddle.rand([4]) @@ -197,12 +200,10 @@ class TestMultinomialAlias(unittest.TestCase): class TestMultinomialError(unittest.TestCase): - def setUp(self): paddle.disable_static() def test_num_sample(self): - def test_num_sample_less_than_0(): x = paddle.rand([4]) paddle.multinomial(x, num_samples=-2) @@ -210,7 +211,6 @@ class TestMultinomialError(unittest.TestCase): self.assertRaises(ValueError, test_num_sample_less_than_0) def test_replacement_False(self): - def test_samples_larger_than_categories(): x = paddle.rand([4]) paddle.multinomial(x, num_samples=5, replacement=False) @@ -218,7 +218,6 @@ class TestMultinomialError(unittest.TestCase): self.assertRaises(ValueError, test_samples_larger_than_categories) def test_input_probs_dim(self): - def test_dim_larger_than_2(): x = paddle.rand([2, 3, 3]) paddle.multinomial(x) @@ -233,7 +232,7 @@ class TestMultinomialError(unittest.TestCase): self.assertRaises(ValueError, test_dim_less_than_1) with self.assertRaises(ValueError): - y = paddle.multinomial(paddle.to_tensor([1., 2., -3.])) + y = paddle.multinomial(paddle.to_tensor([1.0, 2.0, -3.0])) with self.assertRaises(ValueError): prob = paddle.rand([20, 1000]) @@ -242,7 +241,6 @@ class TestMultinomialError(unittest.TestCase): class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -294,7 +292,6 @@ class TestRandomValue(unittest.TestCase): class TestMultinomialTensorNumSamples(UnittestBase): - def init_info(self): self.shapes = [[3, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -326,8 +323,9 @@ class TestMultinomialTensorNumSamples(UnittestBase): exe = paddle.static.Executor() exe.run(starup_prog) res = exe.run(fetch_list=[feat, out]) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) np.testing.assert_equal(res[1].shape, (3, 3)) # Test for Inference Predictor diff --git a/python/paddle/fluid/tests/unittests/test_multiplex_op.py b/python/paddle/fluid/tests/unittests/test_multiplex_op.py index 10ad40082f7..75a08f99ecc 100644 --- a/python/paddle/fluid/tests/unittests/test_multiplex_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiplex_op.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestMultiplexOp(OpTest): - def setUp(self): self.op_type = "multiplex" rows = 4 @@ -34,7 +33,7 @@ class TestMultiplexOp(OpTest): ins4 = np.random.random((rows, 25)).astype("float64") self.inputs = { 'Ids': index, - 'X': [('x1', ins1), ('x2', ins2), ('x3', ins3), ('x4', ins4)] + 'X': [('x1', ins1), ('x2', ins2), ('x3', ins3), ('x4', ins4)], } # multiplex output output = np.zeros_like(ins1) @@ -60,7 +59,6 @@ class TestMultiplexOp(OpTest): class TestMultiplexOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): x1 = fluid.data(name='x1', shape=[None, 2], dtype='int64') @@ -86,16 +84,15 @@ class TestMultiplexOpError(unittest.TestCase): self.assertRaises(TypeError, test_type) def test_type2(): - index2 = fluid.data(name='index2', - shape=[None, 1], - dtype='int16') + index2 = fluid.data( + name='index2', shape=[None, 1], dtype='int16' + ) paddle.multiplex(inputs=[x1, x2], index=index2) self.assertRaises(TypeError, test_type2) class TestMultiplexODygrap(unittest.TestCase): - def test_multiplex_dygraph(self): paddle.disable_static() img1 = np.array([[1, 2], [3, 4]]).astype(np.float32) @@ -118,18 +115,25 @@ class TestMultiplexODygrap(unittest.TestCase): with _test_eager_guard(): inputs_eager = [paddle.to_tensor(img1), paddle.to_tensor(img2)] index_eager = paddle.to_tensor( - np.array([[1], [0]]).astype(np.int32)) + np.array([[1], [0]]).astype(np.int32) + ) inputs_eager[0].stop_gradient = False inputs_eager[1].stop_gradient = False res_eager = paddle.multiplex(inputs_eager, index_eager) res_eager.backward() self.assertEqual((res.numpy() == res_eager.numpy()).all(), True) self.assertEqual( - (inputs[0].grad.numpy() == inputs_eager[0].grad.numpy() - ).all(), True) + ( + inputs[0].grad.numpy() == inputs_eager[0].grad.numpy() + ).all(), + True, + ) self.assertEqual( - (inputs[1].grad.numpy() == inputs_eager[1].grad.numpy() - ).all(), True) + ( + inputs[1].grad.numpy() == inputs_eager[1].grad.numpy() + ).all(), + True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_multiply.py b/python/paddle/fluid/tests/unittests/test_multiply.py index e32245990a8..a2c213ade9c 100755 --- a/python/paddle/fluid/tests/unittests/test_multiply.py +++ b/python/paddle/fluid/tests/unittests/test_multiply.py @@ -23,27 +23,28 @@ from paddle.fluid.framework import _test_eager_guard class TestMultiplyApi(unittest.TestCase): - def _run_static_graph_case(self, x_data, y_data): with program_guard(Program(), Program()): paddle.enable_static() - x = paddle.static.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = paddle.static.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = paddle.static.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = paddle.static.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) res = tensor.multiply(x, y) - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - outs = exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[res]) + outs = exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[res], + ) res = outs[0] return res @@ -112,7 +113,6 @@ class TestMultiplyApi(unittest.TestCase): class TestMultiplyError(unittest.TestCase): - def func_test_errors(self): # test static computation graph: dtype can not be int8 paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py index 4cf28bbd30b..f3872a388c0 100755 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py @@ -17,15 +17,20 @@ import numpy as np import paddle import paddle.fluid as fluid -from paddle.io import Dataset, IterableDataset, TensorDataset, \ - ComposeDataset, ChainDataset, DataLoader +from paddle.io import ( + Dataset, + IterableDataset, + TensorDataset, + ComposeDataset, + ChainDataset, + DataLoader, +) from paddle.fluid.framework import _test_eager_guard IMAGE_SIZE = 32 class RandomDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -35,12 +40,11 @@ class RandomDataset(Dataset): def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') return image, label class RandomIterableDataset(IterableDataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -48,12 +52,11 @@ class RandomIterableDataset(IterableDataset): for i in range(self.sample_num): np.random.seed(i) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') yield image, label class TestTensorDataset(unittest.TestCase): - def run_main(self, num_workers, places): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -66,21 +69,25 @@ class TestTensorDataset(unittest.TestCase): dataset = TensorDataset([input, label]) assert len(dataset) == 16 - dataloader = DataLoader(dataset, - places=place, - num_workers=num_workers, - batch_size=1, - drop_last=True) + dataloader = DataLoader( + dataset, + places=place, + num_workers=num_workers, + batch_size=1, + drop_last=True, + ) for i, (input, label) in enumerate(dataloader()): assert len(input) == 1 assert len(label) == 1 assert input.shape == [1, 3, 4] assert label.shape == [1, 1] - assert isinstance(input, - (fluid.core.VarBase, fluid.core.eager.Tensor)) - assert isinstance(label, - (fluid.core.VarBase, fluid.core.eager.Tensor)) + assert isinstance( + input, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) + assert isinstance( + label, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) assert np.allclose(input.numpy(), input_np[i]) assert np.allclose(label.numpy(), label_np[i]) @@ -98,7 +105,6 @@ class TestTensorDataset(unittest.TestCase): class TestComposeDataset(unittest.TestCase): - def func_test_main(self): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -124,7 +130,6 @@ class TestComposeDataset(unittest.TestCase): class TestRandomSplitApi(unittest.TestCase): - def func_test_main(self): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -151,7 +156,6 @@ class TestRandomSplitApi(unittest.TestCase): class TestRandomSplitError(unittest.TestCase): - def func_test_errors(self): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -167,7 +171,6 @@ class TestRandomSplitError(unittest.TestCase): class TestSubsetDataset(unittest.TestCase): - def run_main(self, num_workers, places): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -184,11 +187,13 @@ class TestSubsetDataset(unittest.TestCase): assert len(dataset) == 5 def prepare_dataloader(dataset): - return DataLoader(dataset, - places=places, - num_workers=num_workers, - batch_size=1, - drop_last=True) + return DataLoader( + dataset, + places=places, + num_workers=num_workers, + batch_size=1, + drop_last=True, + ) dataloader = prepare_dataloader(dataset) dataloader_even = prepare_dataloader(even_subset) @@ -199,10 +204,12 @@ class TestSubsetDataset(unittest.TestCase): assert len(label) == 1 assert input.shape == [1, 3, 4] assert label.shape == [1, 1] - assert isinstance(input, - (fluid.core.VarBase, fluid.core.eager.Tensor)) - assert isinstance(label, - (fluid.core.VarBase, fluid.core.eager.Tensor)) + assert isinstance( + input, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) + assert isinstance( + label, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) elements_list = list() for _, (input, label) in enumerate(dataloader()): @@ -237,7 +244,6 @@ class TestSubsetDataset(unittest.TestCase): class TestChainDataset(unittest.TestCase): - def run_main(self, num_workers, places): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -275,7 +281,6 @@ class TestChainDataset(unittest.TestCase): class NumpyMixTensorDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -285,12 +290,11 @@ class NumpyMixTensorDataset(Dataset): def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') return paddle.to_tensor(image, place=paddle.CPUPlace()), label class TestNumpyMixTensorDataset(TestTensorDataset): - def run_main(self, num_workers, places): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -298,25 +302,28 @@ class TestNumpyMixTensorDataset(TestTensorDataset): with fluid.dygraph.guard(place): dataset = NumpyMixTensorDataset(16) assert len(dataset) == 16 - dataloader = DataLoader(dataset, - places=place, - num_workers=num_workers, - batch_size=1, - drop_last=True) + dataloader = DataLoader( + dataset, + places=place, + num_workers=num_workers, + batch_size=1, + drop_last=True, + ) for i, (input, label) in enumerate(dataloader()): assert len(input) == 1 assert len(label) == 1 assert input.shape == [1, IMAGE_SIZE] assert label.shape == [1, 1] - assert isinstance(input, - (fluid.core.VarBase, fluid.core.eager.Tensor)) - assert isinstance(label, - (fluid.core.VarBase, fluid.core.eager.Tensor)) + assert isinstance( + input, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) + assert isinstance( + label, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) class ComplextDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -324,18 +331,19 @@ class ComplextDataset(Dataset): return self.sample_num def __getitem__(self, idx): - return (3.1, 'abc', - paddle.to_tensor(np.random.random([IMAGE_SIZE - ]).astype('float32'), - place=paddle.CPUPlace()), - [1, np.random.random([2]).astype('float32')], { - 'a': 2.0, - 'b': np.random.random([2]).astype('float32') - }) + return ( + 3.1, + 'abc', + paddle.to_tensor( + np.random.random([IMAGE_SIZE]).astype('float32'), + place=paddle.CPUPlace(), + ), + [1, np.random.random([2]).astype('float32')], + {'a': 2.0, 'b': np.random.random([2]).astype('float32')}, + ) class TestComplextDataset(unittest.TestCase): - def run_main(self, num_workers): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -343,11 +351,13 @@ class TestComplextDataset(unittest.TestCase): with fluid.dygraph.guard(place): dataset = ComplextDataset(16) assert len(dataset) == 16 - dataloader = DataLoader(dataset, - places=place, - num_workers=num_workers, - batch_size=2, - drop_last=True) + dataloader = DataLoader( + dataset, + places=place, + num_workers=num_workers, + batch_size=2, + drop_last=True, + ) for i, data in enumerate(dataloader()): assert len(data) == 5 @@ -380,7 +390,6 @@ class TestComplextDataset(unittest.TestCase): class SingleFieldDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -392,7 +401,6 @@ class SingleFieldDataset(Dataset): class TestSingleFieldDataset(unittest.TestCase): - def init_dataset(self): self.sample_num = 16 self.dataset = SingleFieldDataset(self.sample_num) @@ -403,15 +411,18 @@ class TestSingleFieldDataset(unittest.TestCase): place = paddle.CPUPlace() with fluid.dygraph.guard(place): self.init_dataset() - dataloader = DataLoader(self.dataset, - places=place, - num_workers=num_workers, - batch_size=2, - drop_last=True) + dataloader = DataLoader( + self.dataset, + places=place, + num_workers=num_workers, + batch_size=2, + drop_last=True, + ) for i, data in enumerate(dataloader()): - assert isinstance(data, - (fluid.core.VarBase, fluid.core.eager.Tensor)) + assert isinstance( + data, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) assert data.shape == [2, 2, 3] def func_test_main(self): @@ -425,7 +436,6 @@ class TestSingleFieldDataset(unittest.TestCase): class SingleFieldIterableDataset(IterableDataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -435,22 +445,23 @@ class SingleFieldIterableDataset(IterableDataset): class TestSingleFieldIterableDataset(TestSingleFieldDataset): - def init_dataset(self): self.sample_num = 16 self.dataset = SingleFieldIterableDataset(self.sample_num) class TestDataLoaderGenerateStates(unittest.TestCase): - def setUp(self): self.inputs = [(0, 1), (0, 2), (1, 3)] - self.outputs = [[1835504127, 1731038949, 1320224556, 2330041505], - [2834126987, 2358157858, 1860244682, 1437227251], - [457190280, 2660306227, 859341110, 354512857]] + self.outputs = [ + [1835504127, 1731038949, 1320224556, 2330041505], + [2834126987, 2358157858, 1860244682, 1437227251], + [457190280, 2660306227, 859341110, 354512857], + ] def func_test_main(self): from paddle.fluid.dataloader.worker import _generate_states + for inp, outp in zip(self.inputs, self.outputs): out = _generate_states(*inp) assert out == outp @@ -462,16 +473,18 @@ class TestDataLoaderGenerateStates(unittest.TestCase): class TestDatasetWithDropLast(unittest.TestCase): - def run_main(self, dataset, num_samples, batch_size): for num_workers in [0, 1]: for drop_last in [True, False]: - steps = (num_samples + (1 - int(drop_last)) * \ - (batch_size - 1)) // batch_size - dataloader = DataLoader(dataset, - batch_size=batch_size, - drop_last=drop_last, - num_workers=num_workers) + steps = ( + num_samples + (1 - int(drop_last)) * (batch_size - 1) + ) // batch_size + dataloader = DataLoader( + dataset, + batch_size=batch_size, + drop_last=drop_last, + num_workers=num_workers, + ) datas = [] for data in dataloader: datas.append(data) diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dynamic.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dynamic.py index d0e592d488c..e4d678d8d17 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dynamic.py @@ -21,35 +21,52 @@ import paddle.fluid as fluid from paddle.io import DataLoader from paddle.fluid.dygraph.nn import Linear -from test_multiprocess_dataloader_static import RandomDataset, RandomBatchedDataset, prepare_places -from test_multiprocess_dataloader_static import EPOCH_NUM, BATCH_SIZE, IMAGE_SIZE, SAMPLE_NUM, CLASS_NUM +from test_multiprocess_dataloader_static import ( + RandomDataset, + RandomBatchedDataset, + prepare_places, +) +from test_multiprocess_dataloader_static import ( + EPOCH_NUM, + BATCH_SIZE, + IMAGE_SIZE, + SAMPLE_NUM, + CLASS_NUM, +) class SimpleFCNet(fluid.dygraph.Layer): - def __init__(self): super(SimpleFCNet, self).__init__() - param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)) + param_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ) self._fcs = [] in_channel = IMAGE_SIZE for hidden_size in [10, 20, 30]: self._fcs.append( - Linear(in_channel, - hidden_size, - act='tanh', - param_attr=param_attr, - bias_attr=bias_attr)) + Linear( + in_channel, + hidden_size, + act='tanh', + param_attr=param_attr, + bias_attr=bias_attr, + ) + ) in_channel = hidden_size self._fcs.append( - Linear(in_channel, - CLASS_NUM, - act='softmax', - param_attr=param_attr, - bias_attr=bias_attr)) + Linear( + in_channel, + CLASS_NUM, + act='softmax', + param_attr=param_attr, + bias_attr=bias_attr, + ) + ) def forward(self, image): out = image @@ -59,7 +76,6 @@ class SimpleFCNet(fluid.dygraph.Layer): class TestDygraphDataLoader(unittest.TestCase): - def run_main(self, num_workers, places, persistent_workers): fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 @@ -68,11 +84,13 @@ class TestDygraphDataLoader(unittest.TestCase): optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters()) dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - num_workers=num_workers, - batch_size=BATCH_SIZE, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + num_workers=num_workers, + batch_size=BATCH_SIZE, + drop_last=True, + persistent_workers=persistent_workers, + ) assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) step_list = [] @@ -96,7 +114,7 @@ class TestDygraphDataLoader(unittest.TestCase): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret @@ -107,21 +125,27 @@ class TestDygraphDataLoader(unittest.TestCase): for persistent_workers in [False, True]: results = [] for num_workers in [0, 2]: - print(self.__class__.__name__, p, num_workers, - persistent_workers) + print( + self.__class__.__name__, + p, + num_workers, + persistent_workers, + ) sys.stdout.flush() - ret = self.run_main(num_workers=num_workers, - places=p, - persistent_workers=persistent_workers) + ret = self.run_main( + num_workers=num_workers, + places=p, + persistent_workers=persistent_workers, + ) results.append(ret) diff = np.max( - np.abs(results[0]['loss'] - results[1]['loss']) / - np.abs(results[0]['loss'])) + np.abs(results[0]['loss'] - results[1]['loss']) + / np.abs(results[0]['loss']) + ) self.assertLess(diff, 1e-2) class TestDygraphDataLoaderWithBatchedDataset(TestDygraphDataLoader): - def run_main(self, num_workers, places, persistent_workers): fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 @@ -130,11 +154,13 @@ class TestDygraphDataLoaderWithBatchedDataset(TestDygraphDataLoader): optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters()) dataset = RandomBatchedDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - num_workers=num_workers, - batch_size=None, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + num_workers=num_workers, + batch_size=None, + drop_last=True, + persistent_workers=persistent_workers, + ) assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) step_list = [] @@ -158,7 +184,7 @@ class TestDygraphDataLoaderWithBatchedDataset(TestDygraphDataLoader): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py index ea4b9363bfc..f60c73f4577 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py @@ -23,14 +23,13 @@ from paddle.fluid.dataloader.dataloader_iter import _worker_loop class RandomDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([784]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') return image, label def __len__(self): @@ -38,7 +37,6 @@ class RandomDataset(Dataset): class TestDataLoaderAssert(unittest.TestCase): - def test_main(self): place = fluid.cpu_places()[0] with fluid.dygraph.guard(place): @@ -61,9 +59,9 @@ class TestDataLoaderAssert(unittest.TestCase): # num_workers < 0 try: - loader = DataLoader(dataset=dataset, - places=place, - num_workers=-1) + loader = DataLoader( + dataset=dataset, places=place, num_workers=-1 + ) self.assertTrue(False) except AssertionError: pass @@ -77,27 +75,28 @@ class TestDataLoaderAssert(unittest.TestCase): # set batch_sampler and shuffle/batch_size/drop_last try: - loader = DataLoader(dataset=dataset, - places=place, - batch_sampler=batch_sampler, - shuffle=True, - drop_last=True) + loader = DataLoader( + dataset=dataset, + places=place, + batch_sampler=batch_sampler, + shuffle=True, + drop_last=True, + ) self.assertTrue(False) except AssertionError: pass # set batch_sampler correctly try: - loader = DataLoader(dataset=dataset, - places=place, - batch_sampler=batch_sampler) + loader = DataLoader( + dataset=dataset, places=place, batch_sampler=batch_sampler + ) self.assertTrue(True) except AssertionError: self.assertTrue(False) class TestDatasetRuntimeError(unittest.TestCase): - def test_main(self): dataset = Dataset() @@ -141,10 +140,10 @@ class TestDatasetRuntimeError(unittest.TestCase): # CI Converage cannot record stub in subprocess, # HACK a _worker_loop in main process call here -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestDataLoaderWorkerLoop(unittest.TestCase): - def run_without_worker_done(self, use_shared_memory=True): try: place = fluid.cpu_places()[0] @@ -161,12 +160,15 @@ class TestDataLoaderWorkerLoop(unittest.TestCase): np.stack(s, axis=0) for s in list(zip(*sample_list)) ] - loader = DataLoader(dataset, - num_workers=1, - places=place, - use_shared_memory=use_shared_memory) - assert loader.num_workers > 0, \ - "go to AssertionError and pass in Mac and Windows" + loader = DataLoader( + dataset, + num_workers=1, + places=place, + use_shared_memory=use_shared_memory, + ) + assert ( + loader.num_workers > 0 + ), "go to AssertionError and pass in Mac and Windows" loader = iter(loader) print("loader length", len(loader)) indices_queue = multiprocessing.Queue() @@ -174,16 +176,28 @@ class TestDataLoaderWorkerLoop(unittest.TestCase): indices_queue.put([i, i + 10]) indices_queue.put(None) base_seed = 1234 - _worker_loop(loader._dataset, 0, indices_queue, - loader._data_queue, loader._workers_done_event, - True, _collate_fn, True, _init_fn, 0, 1, - loader._use_shared_memory, base_seed) + _worker_loop( + loader._dataset, + 0, + indices_queue, + loader._data_queue, + loader._workers_done_event, + True, + _collate_fn, + True, + _init_fn, + 0, + 1, + loader._use_shared_memory, + base_seed, + ) self.assertTrue(False) except AssertionError: pass except Exception as e: print("Exception", e) import sys + sys.stdout.flush() self.assertTrue(False) @@ -203,12 +217,15 @@ class TestDataLoaderWorkerLoop(unittest.TestCase): np.stack(s, axis=0) for s in list(zip(*sample_list)) ] - loader = DataLoader(dataset, - num_workers=1, - places=place, - use_shared_memory=use_shared_memory) - assert loader.num_workers > 0, \ - "go to AssertionError and pass in Mac and Windows" + loader = DataLoader( + dataset, + num_workers=1, + places=place, + use_shared_memory=use_shared_memory, + ) + assert ( + loader.num_workers > 0 + ), "go to AssertionError and pass in Mac and Windows" loader = iter(loader) print("loader length", len(loader)) indices_queue = multiprocessing.Queue() @@ -217,10 +234,21 @@ class TestDataLoaderWorkerLoop(unittest.TestCase): indices_queue.put(None) loader._workers_done_event.set() base_seed = 1234 - _worker_loop(loader._dataset, 0, indices_queue, - loader._data_queue, loader._workers_done_event, - True, _collate_fn, True, _init_fn, 0, 1, - loader._use_shared_memory, base_seed) + _worker_loop( + loader._dataset, + 0, + indices_queue, + loader._data_queue, + loader._workers_done_event, + True, + _collate_fn, + True, + _init_fn, + 0, + 1, + loader._use_shared_memory, + base_seed, + ) self.assertTrue(True) except AssertionError: pass diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_dynamic.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_dynamic.py index 98c8a19d520..fe5cf6e27a8 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_dynamic.py @@ -21,35 +21,52 @@ import paddle.fluid as fluid from paddle.io import DataLoader from paddle.fluid.dygraph.nn import Linear -from test_multiprocess_dataloader_iterable_dataset_static import RandomDataset, RandomBatchedDataset, prepare_places -from test_multiprocess_dataloader_iterable_dataset_static import EPOCH_NUM, BATCH_SIZE, IMAGE_SIZE, SAMPLE_NUM, CLASS_NUM +from test_multiprocess_dataloader_iterable_dataset_static import ( + RandomDataset, + RandomBatchedDataset, + prepare_places, +) +from test_multiprocess_dataloader_iterable_dataset_static import ( + EPOCH_NUM, + BATCH_SIZE, + IMAGE_SIZE, + SAMPLE_NUM, + CLASS_NUM, +) class SimpleFCNet(fluid.dygraph.Layer): - def __init__(self): super(SimpleFCNet, self).__init__() - param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)) + param_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ) self._fcs = [] in_channel = IMAGE_SIZE for hidden_size in [10, 20, 30]: self._fcs.append( - Linear(in_channel, - hidden_size, - act='tanh', - param_attr=param_attr, - bias_attr=bias_attr)) + Linear( + in_channel, + hidden_size, + act='tanh', + param_attr=param_attr, + bias_attr=bias_attr, + ) + ) in_channel = hidden_size self._fcs.append( - Linear(in_channel, - CLASS_NUM, - act='softmax', - param_attr=param_attr, - bias_attr=bias_attr)) + Linear( + in_channel, + CLASS_NUM, + act='softmax', + param_attr=param_attr, + bias_attr=bias_attr, + ) + ) def forward(self, image): out = image @@ -59,7 +76,6 @@ class SimpleFCNet(fluid.dygraph.Layer): class TestDygraphDataLoader(unittest.TestCase): - def run_main(self, num_workers, places, persistent_workers): fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 @@ -68,11 +84,13 @@ class TestDygraphDataLoader(unittest.TestCase): optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters()) dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - num_workers=num_workers, - batch_size=BATCH_SIZE, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + num_workers=num_workers, + batch_size=BATCH_SIZE, + drop_last=True, + persistent_workers=persistent_workers, + ) step_list = [] loss_list = [] @@ -95,7 +113,7 @@ class TestDygraphDataLoader(unittest.TestCase): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret @@ -106,19 +124,26 @@ class TestDygraphDataLoader(unittest.TestCase): for persistent_workers in [False, True]: results = [] for num_workers in [0, 2]: - print(self.__class__.__name__, p, num_workers, - persistent_workers) + print( + self.__class__.__name__, + p, + num_workers, + persistent_workers, + ) sys.stdout.flush() - ret = self.run_main(num_workers=num_workers, - places=p, - persistent_workers=persistent_workers) + ret = self.run_main( + num_workers=num_workers, + places=p, + persistent_workers=persistent_workers, + ) results.append(ret) - assert results[0]['loss'].shape[0] * 2 == results[1][ - 'loss'].shape[0] + assert ( + results[0]['loss'].shape[0] * 2 + == results[1]['loss'].shape[0] + ) class TestDygraphDataLoaderWithBatchedDataset(TestDygraphDataLoader): - def run_main(self, num_workers, places, persistent_workers): fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 @@ -127,11 +152,13 @@ class TestDygraphDataLoaderWithBatchedDataset(TestDygraphDataLoader): optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters()) dataset = RandomBatchedDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - num_workers=num_workers, - batch_size=None, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + num_workers=num_workers, + batch_size=None, + drop_last=True, + persistent_workers=persistent_workers, + ) step_list = [] loss_list = [] @@ -154,7 +181,7 @@ class TestDygraphDataLoaderWithBatchedDataset(TestDygraphDataLoader): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_split.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_split.py index aef0c8c7d08..57ae8bf46f8 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_split.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_split.py @@ -21,7 +21,6 @@ from paddle.io import DataLoader, IterableDataset, get_worker_info class RangeIterableDatasetSplit(IterableDataset): - def __init__(self, start, end): self.start = start self.end = end @@ -34,7 +33,9 @@ class RangeIterableDatasetSplit(IterableDataset): else: per_worker = int( math.ceil( - (self.end - self.start) / float(worker_info.num_workers))) + (self.end - self.start) / float(worker_info.num_workers) + ) + ) worker_id = worker_info.id iter_start = self.start + worker_id * per_worker iter_end = min(iter_start + per_worker, self.end) @@ -44,16 +45,17 @@ class RangeIterableDatasetSplit(IterableDataset): class TestDynamicDataLoaderIterSplit(unittest.TestCase): - def test_main(self): place = fluid.CPUPlace() with fluid.dygraph.guard(place): dataset = RangeIterableDatasetSplit(0, 10) - dataloader = DataLoader(dataset, - places=place, - num_workers=2, - batch_size=1, - drop_last=True) + dataloader = DataLoader( + dataset, + places=place, + num_workers=2, + batch_size=1, + drop_last=True, + ) rets = [] for d in dataloader: @@ -63,7 +65,6 @@ class TestDynamicDataLoaderIterSplit(unittest.TestCase): class RangeIterableDataset(IterableDataset): - def __init__(self, start, end): self.start = start self.end = end @@ -74,7 +75,6 @@ class RangeIterableDataset(IterableDataset): class TestDynamicDataLoaderIterInitFuncSplit(unittest.TestCase): - def test_main(self): place = fluid.CPUPlace() with fluid.dygraph.guard(place): @@ -87,18 +87,21 @@ class TestDynamicDataLoaderIterInitFuncSplit(unittest.TestCase): start = dataset.start end = dataset.end num_per_worker = int( - math.ceil((end - start) / float(worker_info.num_workers))) + math.ceil((end - start) / float(worker_info.num_workers)) + ) worker_id = worker_info.id dataset.start = start + worker_id * num_per_worker dataset.end = min(dataset.start + num_per_worker, end) - dataloader = DataLoader(dataset, - places=place, - num_workers=1, - batch_size=1, - drop_last=True, - worker_init_fn=worker_spliter) + dataloader = DataLoader( + dataset, + places=place, + num_workers=1, + batch_size=1, + drop_last=True, + worker_init_fn=worker_spliter, + ) rets = [] for d in dataloader: diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_static.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_static.py index 4c78edf3ade..49f12ea657e 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_static.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_static.py @@ -28,7 +28,6 @@ CLASS_NUM = 10 class RandomDataset(IterableDataset): - def __init__(self, sample_num, class_num): self.sample_num = sample_num self.class_num = class_num @@ -37,8 +36,9 @@ class RandomDataset(IterableDataset): for i in range(self.sample_num): np.random.seed(i) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, - (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype( + 'int64' + ) yield image, label @@ -50,29 +50,36 @@ def simple_fc_net_static(): with fluid.unique_name.guard(): with fluid.program_guard(main_prog, startup_prog): - image = fluid.data(name='image', - shape=[None, IMAGE_SIZE], - dtype='float32') + image = fluid.data( + name='image', shape=[None, IMAGE_SIZE], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') hidden = image - param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)) + param_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ) for hidden_size in [10, 20, 30]: - hidden = fluid.layers.fc(hidden, - size=hidden_size, - act='tanh', - param_attr=param_attr, - bias_attr=bias_attr) - - predict_label = fluid.layers.fc(hidden, - size=CLASS_NUM, - act='softmax', - param_attr=param_attr, - bias_attr=bias_attr) + hidden = fluid.layers.fc( + hidden, + size=hidden_size, + act='tanh', + param_attr=param_attr, + bias_attr=bias_attr, + ) + + predict_label = fluid.layers.fc( + hidden, + size=CLASS_NUM, + act='softmax', + param_attr=param_attr, + bias_attr=bias_attr, + ) loss = fluid.layers.reduce_mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -96,21 +103,22 @@ def prepare_places(with_data_parallel, with_cpu=False, with_gpu=True): class TestStaticDataLoader(unittest.TestCase): - def run_main(self, num_workers, places, persistent_workers): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, image, label, loss = simple_fc_net_static() dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - places=places, - num_workers=num_workers, - batch_size=BATCH_SIZE, - return_list=False, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + places=places, + num_workers=num_workers, + batch_size=BATCH_SIZE, + return_list=False, + drop_last=True, + persistent_workers=persistent_workers, + ) # assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) exe = fluid.Executor(place=places[0]) @@ -118,8 +126,9 @@ class TestStaticDataLoader(unittest.TestCase): prog = fluid.CompiledProgram(main_prog) if len(places) > 1: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step_list = [] loss_list = [] @@ -128,7 +137,8 @@ class TestStaticDataLoader(unittest.TestCase): step = 0 for d in dataloader: assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -136,10 +146,12 @@ class TestStaticDataLoader(unittest.TestCase): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(places[i]) assert label._place()._equals(places[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -148,7 +160,7 @@ class TestStaticDataLoader(unittest.TestCase): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret @@ -158,19 +170,26 @@ class TestStaticDataLoader(unittest.TestCase): for persistent_workers in [False, True]: results = [] for num_workers in [0, 2]: - print(self.__class__.__name__, p, num_workers, - persistent_workers) + print( + self.__class__.__name__, + p, + num_workers, + persistent_workers, + ) sys.stdout.flush() - ret = self.run_main(num_workers=num_workers, - places=p, - persistent_workers=persistent_workers) + ret = self.run_main( + num_workers=num_workers, + places=p, + persistent_workers=persistent_workers, + ) results.append(ret) - assert results[0]['loss'].shape[0] * 2 == results[1][ - 'loss'].shape[0] + assert ( + results[0]['loss'].shape[0] * 2 + == results[1]['loss'].shape[0] + ) class RandomBatchedDataset(IterableDataset): - def __init__(self, sample_num, class_num): self.sample_num = sample_num // BATCH_SIZE self.class_num = class_num @@ -182,37 +201,40 @@ class RandomBatchedDataset(IterableDataset): labels = [] for _ in range(BATCH_SIZE): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, - (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype( + 'int64' + ) images.append(image) labels.append(label) yield np.stack(images, axis=0), np.stack(labels, axis=0) class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): - def run_main(self, num_workers, places, persistent_workers): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, image, label, loss = simple_fc_net_static() dataset = RandomBatchedDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - places=places, - num_workers=num_workers, - batch_size=None, - return_list=False, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + places=places, + num_workers=num_workers, + batch_size=None, + return_list=False, + drop_last=True, + persistent_workers=persistent_workers, + ) exe = fluid.Executor(place=places[0]) exe.run(startup_prog) prog = fluid.CompiledProgram(main_prog) if len(places) > 1: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step_list = [] loss_list = [] @@ -221,7 +243,8 @@ class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): step = 0 for d in dataloader: assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -229,10 +252,12 @@ class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(places[i]) assert label._place()._equals(places[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -241,7 +266,7 @@ class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_static.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_static.py index 6a44c9f4843..853d7ec1924 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_static.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_static.py @@ -29,7 +29,6 @@ CLASS_NUM = 10 class RandomDataset(Dataset): - def __init__(self, sample_num, class_num): self.sample_num = sample_num self.class_num = class_num @@ -37,7 +36,7 @@ class RandomDataset(Dataset): def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -52,29 +51,36 @@ def simple_fc_net_static(): with fluid.unique_name.guard(): with fluid.program_guard(main_prog, startup_prog): - image = fluid.data(name='image', - shape=[None, IMAGE_SIZE], - dtype='float32') + image = fluid.data( + name='image', shape=[None, IMAGE_SIZE], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') hidden = image - param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)) + param_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ) for hidden_size in [10, 20, 30]: - hidden = fluid.layers.fc(hidden, - size=hidden_size, - act='tanh', - param_attr=param_attr, - bias_attr=bias_attr) - - predict_label = fluid.layers.fc(hidden, - size=CLASS_NUM, - act='softmax', - param_attr=param_attr, - bias_attr=bias_attr) + hidden = fluid.layers.fc( + hidden, + size=hidden_size, + act='tanh', + param_attr=param_attr, + bias_attr=bias_attr, + ) + + predict_label = fluid.layers.fc( + hidden, + size=CLASS_NUM, + act='softmax', + param_attr=param_attr, + bias_attr=bias_attr, + ) loss = fluid.layers.reduce_mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -98,21 +104,22 @@ def prepare_places(with_data_parallel, with_cpu=False, with_gpu=True): class TestStaticDataLoader(unittest.TestCase): - def run_main(self, num_workers, places, persistent_workers, use_pe=True): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, image, label, loss = simple_fc_net_static() dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - places=places, - num_workers=num_workers, - batch_size=BATCH_SIZE, - return_list=False, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + places=places, + num_workers=num_workers, + batch_size=BATCH_SIZE, + return_list=False, + drop_last=True, + persistent_workers=persistent_workers, + ) assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) exe = fluid.Executor(place=places[0]) @@ -121,8 +128,9 @@ class TestStaticDataLoader(unittest.TestCase): if use_pe: prog = fluid.CompiledProgram(main_prog) if len(places) > 1: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) else: prog = main_prog @@ -133,7 +141,8 @@ class TestStaticDataLoader(unittest.TestCase): step = 0 for d in dataloader: assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -141,10 +150,12 @@ class TestStaticDataLoader(unittest.TestCase): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(places[i]) assert label._place()._equals(places[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -153,7 +164,7 @@ class TestStaticDataLoader(unittest.TestCase): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret @@ -163,35 +174,43 @@ class TestStaticDataLoader(unittest.TestCase): for persistent_workers in [True, False]: results = [] for num_workers in [0, 2]: - print(self.__class__.__name__, p, num_workers, - persistent_workers) + print( + self.__class__.__name__, + p, + num_workers, + persistent_workers, + ) sys.stdout.flush() - ret = self.run_main(num_workers=num_workers, - places=p, - persistent_workers=persistent_workers) + ret = self.run_main( + num_workers=num_workers, + places=p, + persistent_workers=persistent_workers, + ) results.append(ret) diff = np.max( - np.abs(results[0]['loss'] - results[1]['loss']) / - np.abs(results[0]['loss'])) + np.abs(results[0]['loss'] - results[1]['loss']) + / np.abs(results[0]['loss']) + ) self.assertLess(diff, 1e-2) class TestStaticDataLoaderReturnList(unittest.TestCase): - def run_single_place(self, num_workers): scope = fluid.Scope() - image = fluid.data(name='image', - shape=[None, IMAGE_SIZE], - dtype='float32') + image = fluid.data( + name='image', shape=[None, IMAGE_SIZE], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') with fluid.scope_guard(scope): dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - num_workers=num_workers, - batch_size=BATCH_SIZE, - drop_last=True, - return_list=True) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + num_workers=num_workers, + batch_size=BATCH_SIZE, + drop_last=True, + return_list=True, + ) for d in dataloader: assert isinstance(d, list) @@ -201,19 +220,21 @@ class TestStaticDataLoaderReturnList(unittest.TestCase): def run_multi_place(self, num_workers): scope = fluid.Scope() - image = fluid.data(name='image', - shape=[None, IMAGE_SIZE], - dtype='float32') + image = fluid.data( + name='image', shape=[None, IMAGE_SIZE], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') with fluid.scope_guard(scope): dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - num_workers=num_workers, - batch_size=BATCH_SIZE, - places=[fluid.CPUPlace()] * 2, - drop_last=True, - return_list=True) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + num_workers=num_workers, + batch_size=BATCH_SIZE, + places=[fluid.CPUPlace()] * 2, + drop_last=True, + return_list=True, + ) for d in dataloader: assert isinstance(d, list) @@ -229,7 +250,6 @@ class TestStaticDataLoaderReturnList(unittest.TestCase): class RandomBatchedDataset(Dataset): - def __init__(self, sample_num, class_num): self.sample_num = int(sample_num / BATCH_SIZE) self.class_num = class_num @@ -240,8 +260,9 @@ class RandomBatchedDataset(Dataset): labels = [] for _ in range(BATCH_SIZE): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, - (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype( + 'int64' + ) images.append(image) labels.append(label) return np.stack(images, axis=0), np.stack(labels, axis=0) @@ -251,21 +272,22 @@ class RandomBatchedDataset(Dataset): class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): - def run_main(self, num_workers, places, persistent_workers): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, image, label, loss = simple_fc_net_static() dataset = RandomBatchedDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - places=places, - num_workers=num_workers, - batch_size=None, - return_list=False, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + places=places, + num_workers=num_workers, + batch_size=None, + return_list=False, + drop_last=True, + persistent_workers=persistent_workers, + ) assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) exe = fluid.Executor(place=places[0]) @@ -273,8 +295,9 @@ class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): prog = fluid.CompiledProgram(main_prog) if len(places) > 1: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step_list = [] loss_list = [] @@ -283,7 +306,8 @@ class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): step = 0 for d in dataloader: assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -291,10 +315,12 @@ class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(places[i]) assert label._place()._equals(places[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -303,7 +329,7 @@ class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py b/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py index e1033ae881e..3eae95ef34b 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py @@ -24,7 +24,6 @@ class ReaderException(Exception): class TestMultiprocessReaderExceptionWithQueueSuccess(unittest.TestCase): - def setUp(self): self.use_pipe = False self.raise_exception = False @@ -40,12 +39,12 @@ class TestMultiprocessReaderExceptionWithQueueSuccess(unittest.TestCase): batch_size = 4 def fake_reader(): - def __impl__(): for _ in range(sample_num): if not self.raise_exception: - yield list(np.random.uniform(low=-1, high=1, - size=[10])), + yield list( + np.random.uniform(low=-1, high=1, size=[10]) + ), else: raise ValueError() @@ -53,23 +52,28 @@ class TestMultiprocessReaderExceptionWithQueueSuccess(unittest.TestCase): with fluid.program_guard(fluid.Program(), fluid.Program()): image = fluid.data(name='image', dtype='float32', shape=[None, 10]) - reader = fluid.io.DataLoader.from_generator(feed_list=[image], - capacity=2, - iterable=iterable) + reader = fluid.io.DataLoader.from_generator( + feed_list=[image], capacity=2, iterable=iterable + ) image_p_1 = image + 1 decorated_reader = multiprocess_reader( - [fake_reader(), fake_reader()], use_pipe=self.use_pipe) + [fake_reader(), fake_reader()], use_pipe=self.use_pipe + ) if isinstance(place, fluid.CUDAPlace): - reader.set_sample_generator(decorated_reader, - batch_size=batch_size, - places=fluid.cuda_places(0)) + reader.set_sample_generator( + decorated_reader, + batch_size=batch_size, + places=fluid.cuda_places(0), + ) else: - reader.set_sample_generator(decorated_reader, - batch_size=batch_size, - places=fluid.cpu_places(1)) + reader.set_sample_generator( + decorated_reader, + batch_size=batch_size, + places=fluid.cpu_places(1), + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -117,24 +121,24 @@ class TestMultiprocessReaderExceptionWithQueueSuccess(unittest.TestCase): class TestMultiprocessReaderExceptionWithQueueFailed( - TestMultiprocessReaderExceptionWithQueueSuccess): - + TestMultiprocessReaderExceptionWithQueueSuccess +): def setUp(self): self.use_pipe = False self.raise_exception = True class TestMultiprocessReaderExceptionWithPipeSuccess( - TestMultiprocessReaderExceptionWithQueueSuccess): - + TestMultiprocessReaderExceptionWithQueueSuccess +): def setUp(self): self.use_pipe = True self.raise_exception = False class TestMultiprocessReaderExceptionWithPipeFailed( - TestMultiprocessReaderExceptionWithQueueSuccess): - + TestMultiprocessReaderExceptionWithQueueSuccess +): def setUp(self): self.use_pipe = True self.raise_exception = True diff --git a/python/paddle/fluid/tests/unittests/test_mv_op.py b/python/paddle/fluid/tests/unittests/test_mv_op.py index fbfd18a4778..d2381a21e99 100644 --- a/python/paddle/fluid/tests/unittests/test_mv_op.py +++ b/python/paddle/fluid/tests/unittests/test_mv_op.py @@ -1,4 +1,4 @@ -#Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ from op_test import OpTest class TestMVOp(OpTest): - def setUp(self): self.op_type = "mv" self.python_api = paddle.mv @@ -40,7 +39,6 @@ class TestMVOp(OpTest): class TestMVAPI(unittest.TestCase): - def test_dygraph_api_out(self): paddle.disable_static() @@ -68,12 +66,12 @@ class TestMVAPI(unittest.TestCase): self.input_vec = np.random.rand(100).astype("float64") with program_guard(train_program, startup_program): - data_x = paddle.static.data("x", - shape=[5, 100], - dtype="float64") - data_vec = paddle.static.data("vec", - shape=[100], - dtype="float64") + data_x = paddle.static.data( + "x", shape=[5, 100], dtype="float64" + ) + data_vec = paddle.static.data( + "vec", shape=[100], dtype="float64" + ) data_x.stop_gradient = x_stop_gradient data_vec.stop_gradient = vec_stop_gradient @@ -82,19 +80,16 @@ class TestMVAPI(unittest.TestCase): self.place = paddle.CPUPlace() exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "vec": self.input_vec - }, - fetch_list=[result_vec]) + (res,) = exe.run( + feed={"x": self.input_x, "vec": self.input_vec}, + fetch_list=[result_vec], + ) z_expected = np.array(np.dot(self.input_x, self.input_vec)) np.testing.assert_allclose(res, z_expected, rtol=1e-05) class TestMVError(unittest.TestCase): - def test_input(self): - def test_shape(): paddle.enable_static() @@ -102,9 +97,9 @@ class TestMVError(unittest.TestCase): self.input_vec = np.random.rand(100).astype("float64") data_x = paddle.static.data("x", shape=[5, 100], dtype="float64") - data_vec = paddle.static.data("vec", - shape=[100, 2], - dtype="float64") + data_vec = paddle.static.data( + "vec", shape=[100, 2], dtype="float64" + ) result_vec = paddle.mv(data_x, data_vec) self.assertRaises(ValueError, test_shape) diff --git a/python/paddle/fluid/tests/unittests/test_naive_best_fit_gpu_memory_limit.py b/python/paddle/fluid/tests/unittests/test_naive_best_fit_gpu_memory_limit.py index 6994bf30523..c6846d7068e 100644 --- a/python/paddle/fluid/tests/unittests/test_naive_best_fit_gpu_memory_limit.py +++ b/python/paddle/fluid/tests/unittests/test_naive_best_fit_gpu_memory_limit.py @@ -23,7 +23,6 @@ if fluid.is_compiled_with_cuda(): class TestBase(unittest.TestCase): - def setUp(self): if fluid.is_compiled_with_cuda(): self._limit = fluid.core.globals()['FLAGS_gpu_memory_limit_mb'] @@ -36,8 +35,10 @@ class TestBase(unittest.TestCase): place = fluid.CUDAPlace(0) t = fluid.LoDTensor() - t.set(np.ndarray([int(self._limit / 2), other_dim], dtype='float32'), - place) + t.set( + np.ndarray([int(self._limit / 2), other_dim], dtype='float32'), + place, + ) del t t = fluid.LoDTensor() diff --git a/python/paddle/fluid/tests/unittests/test_name_scope.py b/python/paddle/fluid/tests/unittests/test_name_scope.py index a0e37384eb2..475ebec9ece 100644 --- a/python/paddle/fluid/tests/unittests/test_name_scope.py +++ b/python/paddle/fluid/tests/unittests/test_name_scope.py @@ -17,7 +17,6 @@ import paddle.fluid as fluid class TestNameScope(unittest.TestCase): - def test_name_scope(self): with fluid.name_scope("s1"): a = fluid.layers.data(name='data', shape=[1], dtype='int32') diff --git a/python/paddle/fluid/tests/unittests/test_nan_inf.py b/python/paddle/fluid/tests/unittests/test_nan_inf.py index 65a537981ad..f062adb6a7a 100644 --- a/python/paddle/fluid/tests/unittests/test_nan_inf.py +++ b/python/paddle/fluid/tests/unittests/test_nan_inf.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestNanInf(unittest.TestCase): - def setUp(self): self._python_interp = sys.executable if os.getenv('WITH_COVERAGE', 'OFF') == 'ON': @@ -33,10 +32,12 @@ class TestNanInf(unittest.TestCase): def check_nan_inf(self): cmd = self._python_interp - proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.env) + proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.env, + ) out, err = proc.communicate() returncode = proc.returncode @@ -49,7 +50,8 @@ class TestNanInf(unittest.TestCase): assert (out + err).find('find nan or inf==='.encode()) != -1 else: assert (out + err).find( - 'There are `nan` or `inf` in tensor'.encode()) != -1 + 'There are `nan` or `inf` in tensor'.encode() + ) != -1 def test_nan_inf_in_static_mode(self): self._python_interp += " check_nan_inf_base.py" @@ -61,7 +63,6 @@ class TestNanInf(unittest.TestCase): class TestNanInfEnv(TestNanInf): - def setUp(self): super(TestNanInfEnv, self).setUp() # windows python have some bug with env, so need use str to pass ci @@ -69,7 +70,8 @@ class TestNanInfEnv(TestNanInf): self.env[str("PADDLE_INF_NAN_SKIP_OP")] = str("mul") self.env[str("PADDLE_INF_NAN_SKIP_ROLE")] = str("loss") self.env[str("PADDLE_INF_NAN_SKIP_VAR")] = str( - "elementwise_add:fc_0.tmp_1") + "elementwise_add:fc_0.tmp_1" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_nanmean_api.py b/python/paddle/fluid/tests/unittests/test_nanmean_api.py index 3155140518d..bcc9ac5c77a 100644 --- a/python/paddle/fluid/tests/unittests/test_nanmean_api.py +++ b/python/paddle/fluid/tests/unittests/test_nanmean_api.py @@ -27,10 +27,14 @@ class TestNanmeanAPI(unittest.TestCase): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) self.x[0, :, :, :] = np.nan - self.x_grad = np.array([[np.nan, np.nan, 3.], [0., np.nan, - 2.]]).astype(np.float32) - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.x_grad = np.array( + [[np.nan, np.nan, 3.0], [0.0, np.nan, 2.0]] + ).astype(np.float32) + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_api_static(self): paddle.enable_static() @@ -43,8 +47,9 @@ class TestNanmeanAPI(unittest.TestCase): out4 = paddle.nanmean(x, axis) out5 = paddle.nanmean(x, tuple(axis)) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': self.x}, - fetch_list=[out1, out2, out3, out4, out5]) + res = exe.run( + feed={'X': self.x}, fetch_list=[out1, out2, out3, out4, out5] + ) out_ref = np.nanmean(self.x) for out in res: np.testing.assert_allclose(out, out_ref, rtol=0.0001) @@ -100,9 +105,9 @@ class TestNanmeanAPI(unittest.TestCase): sum_dx_ref = np.prod(y.shape) if np.isnan(y.numpy()).sum(): sum_dx_ref -= np.isnan(y.numpy()).sum() - cnt = paddle.sum(~paddle.isnan(x_tensor), - axis=axis, - keepdim=keepdim) + cnt = paddle.sum( + ~paddle.isnan(x_tensor), axis=axis, keepdim=keepdim + ) if (cnt == 0).sum(): dx[np.isnan(dx)] = 0 sum_dx = dx.sum() diff --git a/python/paddle/fluid/tests/unittests/test_nanmedian.py b/python/paddle/fluid/tests/unittests/test_nanmedian.py index 76bea458813..952263ae94b 100644 --- a/python/paddle/fluid/tests/unittests/test_nanmedian.py +++ b/python/paddle/fluid/tests/unittests/test_nanmedian.py @@ -21,20 +21,19 @@ np.random.seed(102) class TestNanmedian(unittest.TestCase): - def setUp(self): - single_axis_shape = (120) + single_axis_shape = 120 multi_axis_shape = (2, 3, 4, 5) self.fake_data = { - "single_axis_normal": - np.random.uniform(-1, 1, single_axis_shape).astype(np.float32), - "multi_axis_normal": - np.random.uniform(-1, 1, multi_axis_shape).astype(np.float32), - "single_axis_all_nan": - np.full(single_axis_shape, np.nan), - "multi_axis_all_nan": - np.full(multi_axis_shape, np.nan), + "single_axis_normal": np.random.uniform( + -1, 1, single_axis_shape + ).astype(np.float32), + "multi_axis_normal": np.random.uniform( + -1, 1, multi_axis_shape + ).astype(np.float32), + "single_axis_all_nan": np.full(single_axis_shape, np.nan), + "multi_axis_all_nan": np.full(multi_axis_shape, np.nan), } single_partial_nan = self.fake_data["single_axis_normal"].copy() @@ -59,11 +58,22 @@ class TestNanmedian(unittest.TestCase): col_data[:, :, 2, 3:] = np.nan self.fake_data["col_nan_odd"] = col_data - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.axis_candiate_list = [ - None, 0, 2, -1, -2, (1, 2), [0, -1], [0, 1, 3], (1, 2, 3), - [0, 2, 1, 3] + None, + 0, + 2, + -1, + -2, + (1, 2), + [0, -1], + [0, 1, 3], + (1, 2, 3), + [0, 2, 1, 3], ] def test_api_static(self): @@ -79,8 +89,9 @@ class TestNanmedian(unittest.TestCase): out4 = paddle.nanmedian(x, axis=axis, keepdim=True) out5 = paddle.nanmedian(x, axis=tuple(axis), keepdim=True) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': data}, - fetch_list=[out1, out2, out3, out4, out5]) + res = exe.run( + feed={'X': data}, fetch_list=[out1, out2, out3, out4, out5] + ) for out in res: np.testing.assert_allclose(np_res, out, rtol=1e-05, equal_nan=True) @@ -109,23 +120,22 @@ class TestNanmedian(unittest.TestCase): continue np_res = np.nanmedian(data, keepdims=keep_dim) - pd_res = paddle.nanmedian(paddle.to_tensor(data), - keepdim=keep_dim) - np.testing.assert_allclose(np_res, - pd_res.numpy(), - rtol=1e-05, - equal_nan=True) + pd_res = paddle.nanmedian( + paddle.to_tensor(data), keepdim=keep_dim + ) + np.testing.assert_allclose( + np_res, pd_res.numpy(), rtol=1e-05, equal_nan=True + ) def test_axis_case(data, axis): - pd_res = paddle.nanmedian(paddle.to_tensor(data), - axis=axis, - keepdim=False) + pd_res = paddle.nanmedian( + paddle.to_tensor(data), axis=axis, keepdim=False + ) axis = clean_axis_numpy(axis, len(data.shape)) np_res = np.nanmedian(data, axis=axis, keepdims=False) - np.testing.assert_allclose(np_res, - pd_res.numpy(), - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + np_res, pd_res.numpy(), rtol=1e-05, equal_nan=True + ) for name, data in self.fake_data.items(): test_data_case(data) diff --git a/python/paddle/fluid/tests/unittests/test_nansum_api.py b/python/paddle/fluid/tests/unittests/test_nansum_api.py index c0fe5da0cbc..9abc0e4e4e7 100644 --- a/python/paddle/fluid/tests/unittests/test_nansum_api.py +++ b/python/paddle/fluid/tests/unittests/test_nansum_api.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class API_Test_Nansum(unittest.TestCase): - def test_static_graph(self): paddle.enable_static() startup_program = fluid.Program() @@ -36,12 +35,14 @@ class API_Test_Nansum(unittest.TestCase): exe = fluid.Executor(place) exe.run(startup_program) - x = np.array([[float('nan'), 3, 5, 9], [1, 2, - float('-nan'), - 7]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': x}, - fetch_list=[out1, out2, out3, out4]) + x = np.array( + [[float('nan'), 3, 5, 9], [1, 2, float('-nan'), 7]] + ).astype(np.float32) + res = exe.run( + train_program, + feed={'input': x}, + fetch_list=[out1, out2, out3, out4], + ) out1_np = np.array(res[0]) out2_np = np.array(res[1]) @@ -52,14 +53,22 @@ class API_Test_Nansum(unittest.TestCase): out3_ref = np.array([17, 10]).astype(np.float32) out4_ref = np.array([[17], [10]]).astype(np.float32) - self.assertTrue((out1_np == out1_ref).all(), - msg='nansum output is wrong, out =' + str(out1_np)) - self.assertTrue((out2_np == out2_ref).all(), - msg='nansum output is wrong, out =' + str(out2_np)) - self.assertTrue((out3_np == out3_ref).all(), - msg='nansum output is wrong, out =' + str(out3_np)) - self.assertTrue((out4_np == out4_ref).all(), - msg='nansum output is wrong, out =' + str(out4_np)) + self.assertTrue( + (out1_np == out1_ref).all(), + msg='nansum output is wrong, out =' + str(out1_np), + ) + self.assertTrue( + (out2_np == out2_ref).all(), + msg='nansum output is wrong, out =' + str(out2_np), + ) + self.assertTrue( + (out3_np == out3_ref).all(), + msg='nansum output is wrong, out =' + str(out3_np), + ) + self.assertTrue( + (out4_np == out4_ref).all(), + msg='nansum output is wrong, out =' + str(out4_np), + ) def test_error_api(self): paddle.enable_static() @@ -79,8 +88,9 @@ class API_Test_Nansum(unittest.TestCase): self.assertRaises(TypeError, run2) def test_dygraph(self): - x = np.array([[float('nan'), 3, 5, 9], [1, 2, float('-nan'), - 7]]).astype(np.float32) + x = np.array( + [[float('nan'), 3, 5, 9], [1, 2, float('-nan'), 7]] + ).astype(np.float32) with fluid.dygraph.guard(): inputs = fluid.dygraph.to_variable(x) out = paddle.nansum(inputs) @@ -88,7 +98,8 @@ class API_Test_Nansum(unittest.TestCase): self.assertTrue( (out.numpy() == out_ref).all(), - msg='nansum output is wrong, out =' + str(out.numpy())) + msg='nansum output is wrong, out =' + str(out.numpy()), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_nce.py b/python/paddle/fluid/tests/unittests/test_nce.py index 484dbfabd9d..885d12c6fa2 100644 --- a/python/paddle/fluid/tests/unittests/test_nce.py +++ b/python/paddle/fluid/tests/unittests/test_nce.py @@ -22,8 +22,9 @@ from paddle.fluid import Program, program_guard from op_test import OpTest -def nce(input, weight, bias, sample_weight, labels, num_classes, - num_sample_class): +def nce( + input, weight, bias, sample_weight, labels, num_classes, num_sample_class +): samples = [] sample_labels = [] batch_size = input.shape[0] @@ -54,23 +55,34 @@ def nce(input, weight, bias, sample_weight, labels, num_classes, o = sample_out[i] cost = -np.log(o / (o + b)) if samples[i][2] else -np.log(b / (o + b)) out[samples[i][0]] += cost * samples[i][3] - return (out[:, np.newaxis], - np.array(sample_out).reshape(batch_size, - num_sample_class + num_true_class), - np.array(sample_labels).reshape(batch_size, - num_sample_class + num_true_class)) + return ( + out[:, np.newaxis], + np.array(sample_out).reshape( + batch_size, num_sample_class + num_true_class + ), + np.array(sample_labels).reshape( + batch_size, num_sample_class + num_true_class + ), + ) class TestNCE(OpTest): - - def generate_data(self, dim, batch_size, num_classes, num_true_class, - num_neg_samples, is_sparse): + def generate_data( + self, + dim, + batch_size, + num_classes, + num_true_class, + num_neg_samples, + is_sparse, + ): input = np.random.randn(batch_size, dim).astype(np.float32) weight = np.random.randn(num_classes, dim).astype(np.float32) bias = np.random.randn(num_classes).astype(np.float32) sample_weight = np.random.randn(batch_size).astype(np.float32) - labels = np.random.randint(0, num_classes, - (batch_size, num_true_class)).astype("int64") + labels = np.random.randint( + 0, num_classes, (batch_size, num_true_class) + ).astype("int64") self.attrs = { 'num_total_classes': num_classes, 'num_neg_samples': num_neg_samples, @@ -78,14 +90,14 @@ class TestNCE(OpTest): 'seed': 0, 'sampler': 0, 'is_sparse': is_sparse, - 'is_test': self.is_test + 'is_test': self.is_test, } self.inputs = { 'Input': input, 'Label': labels, 'Weight': weight, 'Bias': bias, - 'SampleWeight': sample_weight + 'SampleWeight': sample_weight, } def set_is_test(self): @@ -95,17 +107,22 @@ class TestNCE(OpTest): self.generate_data(5, 25, 100, 1, 2, False) def compute(self): - out = nce(self.inputs['Input'], self.inputs['Weight'], - self.inputs['Bias'], self.inputs['SampleWeight'], - self.inputs['Label'], self.attrs['num_total_classes'], - self.attrs['num_neg_samples']) + out = nce( + self.inputs['Input'], + self.inputs['Weight'], + self.inputs['Bias'], + self.inputs['SampleWeight'], + self.inputs['Label'], + self.attrs['num_total_classes'], + self.attrs['num_neg_samples'], + ) if self.is_test: self.outputs = {'Cost': out[0]} else: self.outputs = { 'Cost': out[0], 'SampleLogits': out[1], - 'SampleLabels': out[2] + 'SampleLabels': out[2], } def setUp(self): @@ -118,13 +135,12 @@ class TestNCE(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(["Input", "Weight", "Bias"], - "Cost", - max_relative_error=0.02) + self.check_grad( + ["Input", "Weight", "Bias"], "Cost", max_relative_error=0.02 + ) class TestNCECase1Tensor(TestNCE): - def set_data(self): self.generate_data(10, 20, 100, 2, 5, False) @@ -139,7 +155,6 @@ class TestNCETensorIsTest(TestNCE): class TestNCECase1SelectedRows(unittest.TestCase): - def setUp(self): self.base_lr = 0.0001 self.batch_size = 8 @@ -163,33 +178,51 @@ class TestNCECase1SelectedRows(unittest.TestCase): optimizer = fluid.optimizer.SGD(learning_rate=self.base_lr) return optimizer - def train_network(self, num_total_classes, num_neg_samples, sampler, - custom_dist, is_sparse): + def train_network( + self, + num_total_classes, + num_neg_samples, + sampler, + custom_dist, + is_sparse, + ): input = fluid.layers.data(name="input", shape=[10], dtype="float32") label = fluid.layers.data(name="label", shape=[1], dtype="int64") - w_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 10], - dtype='float32', - name='nce_w', - initializer=initializer.ConstantInitializer()) - b_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 1], - dtype='float32', - name='nce_b', - initializer=initializer.ConstantInitializer()) - - cost = fluid.layers.nce(input=input, - label=label, - num_total_classes=num_total_classes, - sampler=sampler, - custom_dist=custom_dist, - sample_weight=None, - param_attr='nce_w', - bias_attr='nce_b', - seed=1, - num_neg_samples=num_neg_samples, - is_sparse=is_sparse) + w_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 10], + dtype='float32', + name='nce_w', + initializer=initializer.ConstantInitializer(), + ) + ) + b_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 1], + dtype='float32', + name='nce_b', + initializer=initializer.ConstantInitializer(), + ) + ) + + cost = fluid.layers.nce( + input=input, + label=label, + num_total_classes=num_total_classes, + sampler=sampler, + custom_dist=custom_dist, + sample_weight=None, + param_attr='nce_w', + bias_attr='nce_b', + seed=1, + num_neg_samples=num_neg_samples, + is_sparse=is_sparse, + ) avg_cost = paddle.mean(cost) # optimizer optimizer = self.get_optimizer() @@ -210,15 +243,19 @@ class TestNCECase1SelectedRows(unittest.TestCase): dense_startup_program = fluid.framework.Program() dense_train_program = fluid.framework.Program() with fluid.scope_guard(dense_scope): - with fluid.program_guard(dense_train_program, - dense_startup_program): - cost, feeds = self.train_network(20, 5, "custom_dist", - nid_freq_arr.tolist(), False) + with fluid.program_guard( + dense_train_program, dense_startup_program + ): + cost, feeds = self.train_network( + 20, 5, "custom_dist", nid_freq_arr.tolist(), False + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) exe.run(dense_startup_program) - loss_val = exe.run(dense_train_program, - feed=feeder.feed(data), - fetch_list=[cost.name]) + loss_val = exe.run( + dense_train_program, + feed=feeder.feed(data), + fetch_list=[cost.name], + ) rets.append(np.mean(loss_val)) # for sparse @@ -226,106 +263,113 @@ class TestNCECase1SelectedRows(unittest.TestCase): sparse_startup_program = fluid.framework.Program() sparse_train_program = fluid.framework.Program() with fluid.scope_guard(sparse_scope): - with fluid.program_guard(sparse_train_program, - sparse_startup_program): - cost, feeds = self.train_network(20, 5, "custom_dist", - nid_freq_arr.tolist(), True) + with fluid.program_guard( + sparse_train_program, sparse_startup_program + ): + cost, feeds = self.train_network( + 20, 5, "custom_dist", nid_freq_arr.tolist(), True + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) exe.run(sparse_startup_program) - loss_val = exe.run(sparse_train_program, - feed=feeder.feed(data), - fetch_list=[cost.name]) + loss_val = exe.run( + sparse_train_program, + feed=feeder.feed(data), + fetch_list=[cost.name], + ) rets.append(np.mean(loss_val)) self.assertEqual(rets[0], rets[1]) class TestNCE_OpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - input1 = fluid.create_lod_tensor(np.array([0.0, 3.0, 2.0, 4.0]), - [[1, 1, 2]], fluid.CPUPlace()) - label1 = fluid.layers.data(name='label1', - shape=[-1, 4], - dtype="int64") + input1 = fluid.create_lod_tensor( + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) + label1 = fluid.layers.data( + name='label1', shape=[-1, 4], dtype="int64" + ) # the input(input) of nce layer must be Variable. self.assertRaises(TypeError, fluid.layers.nce, input1, label1, 5) - input2 = fluid.layers.data(name='input2', - shape=[-1, 4], - dtype="float32") - label2 = fluid.create_lod_tensor(np.array([0.0, 3.0, 2.0, 4.0]), - [[1, 1, 2]], fluid.CPUPlace()) + input2 = fluid.layers.data( + name='input2', shape=[-1, 4], dtype="float32" + ) + label2 = fluid.create_lod_tensor( + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) # the input(label) of nce layer must be Variable. self.assertRaises(TypeError, fluid.layers.nce, input2, label2, 5) - input3 = fluid.layers.data(name='input3', - shape=[-1, 4], - dtype="float16") - label3 = fluid.layers.data(name='label3', - shape=[-1, 1], - dtype="int64") + input3 = fluid.layers.data( + name='input3', shape=[-1, 4], dtype="float16" + ) + label3 = fluid.layers.data( + name='label3', shape=[-1, 1], dtype="int64" + ) # the data type of input(input) must be float32 or float64. self.assertRaises(TypeError, fluid.layers.nce, input3, label3, 5) - input4 = fluid.layers.data(name='input4', - shape=[-1, 4], - dtype="float32") - label4 = fluid.layers.data(name='label4', - shape=[-1, 1], - dtype="int32") + input4 = fluid.layers.data( + name='input4', shape=[-1, 4], dtype="float32" + ) + label4 = fluid.layers.data( + name='label4', shape=[-1, 1], dtype="int32" + ) # the data type of input(label) must be int64. self.assertRaises(TypeError, fluid.layers.nce, input4, label4, 5) class TestDygraphNCE_OpError(unittest.TestCase): - def test_NCE_errors(self): with program_guard(Program(), Program()): nce = fluid.NCE(20, 5) - input1 = fluid.create_lod_tensor(np.array([0.0, 3.0, 2.0, 4.0]), - [[1, 1, 2]], fluid.CPUPlace()) - label1 = fluid.layers.data(name='label1', - shape=[-1, 4], - dtype="int64") + input1 = fluid.create_lod_tensor( + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) + label1 = fluid.layers.data( + name='label1', shape=[-1, 4], dtype="int64" + ) # the input(input) of NCE layer must be Variable. self.assertRaises(TypeError, nce, input1, label1) - input2 = fluid.layers.data(name='input2', - shape=[-1, 4], - dtype="float32") - label2 = fluid.create_lod_tensor(np.array([0.0, 3.0, 2.0, 4.0]), - [[1, 1, 2]], fluid.CPUPlace()) + input2 = fluid.layers.data( + name='input2', shape=[-1, 4], dtype="float32" + ) + label2 = fluid.create_lod_tensor( + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) # the input(label) of NCE layer must be Variable. self.assertRaises(TypeError, nce, input2, label2) - input3 = fluid.layers.data(name='input3', - shape=[-1, 4], - dtype="float16") - label3 = fluid.layers.data(name='label3', - shape=[-1, 1], - dtype="int64") + input3 = fluid.layers.data( + name='input3', shape=[-1, 4], dtype="float16" + ) + label3 = fluid.layers.data( + name='label3', shape=[-1, 1], dtype="int64" + ) # the data type of input(input) must be float32 or float64. self.assertRaises(TypeError, nce, input3, label3) - input4 = fluid.layers.data(name='input4', - shape=[-1, 4], - dtype="float32") - label4 = fluid.layers.data(name='label4', - shape=[-1, 1], - dtype="int32") + input4 = fluid.layers.data( + name='input4', shape=[-1, 4], dtype="float32" + ) + label4 = fluid.layers.data( + name='label4', shape=[-1, 1], dtype="int32" + ) # the data type of input(label) must be int64. self.assertRaises(TypeError, nce, input4, label4) - input5 = fluid.layers.data(name='input5', - shape=[-1, 4], - dtype="float32") - label5 = fluid.layers.data(name='label5', - shape=[-1, 1], - dtype="int64") + input5 = fluid.layers.data( + name='input5', shape=[-1, 4], dtype="float32" + ) + label5 = fluid.layers.data( + name='label5', shape=[-1, 1], dtype="int64" + ) sample_weight = fluid.create_lod_tensor( - np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace()) + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) # the sample_weight of nce must be Variable or None. self.assertRaises(TypeError, nce, input5, label5, sample_weight) diff --git a/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py b/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py index 42b59e57653..f8a997e4fba 100755 --- a/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py @@ -19,13 +19,15 @@ import paddle.fluid.core as core import paddle.fluid as fluid -def nearest_neighbor_interp_np(X, - out_h, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp_np( + X, + out_h, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -38,13 +40,13 @@ def nearest_neighbor_interp_np(X, n, c, in_h, in_w = X.shape ratio_h = ratio_w = 0.0 - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -71,7 +73,6 @@ def nearest_neighbor_interp_np(X, class TestNearestInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -95,10 +96,15 @@ class TestNearestInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -112,7 +118,7 @@ class TestNearestInterpOp(OpTest): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -120,136 +126,126 @@ class TestNearestInterpOp(OpTest): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") self.align_corners = True class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 4, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 8]).astype("int32") self.align_corners = True self.data_layout = "NHWC" class TestNearestInterpOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "nearest_interp" self.check_eager = True - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale > 0: out_h = int(self.input_shape[2] * self.scale) @@ -258,9 +254,14 @@ class TestNearestInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -270,67 +271,62 @@ class TestNearestInterpOpUint8(OpTest): 'out_w': self.out_w, 'scale': self.scale, 'interp_method': self.interp_method, - 'align_corners': self.align_corners + 'align_corners': self.align_corners, } self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=self.check_eager) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 3, 9, 6] self.out_h = 10 self.out_w = 9 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase1Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 80 self.out_w = 40 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase2Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 5 self.out_w = 13 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 15]).astype("int32") self.align_corners = True class TestNearestInterpWithoutCorners(TestNearestInterpOp): - def set_align_corners(self): self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -342,19 +338,17 @@ class TestNearestNeighborInterpScale2(TestNearestInterpOp): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 1. + self.scale = 1.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestInterpOp_attr_tensor(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -389,59 +383,62 @@ class TestNearestInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.check_eager = False self.attrs['out_h'] = self.out_h self.attrs['out_w'] = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 5, 4, 4] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True @@ -449,7 +446,6 @@ class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -462,7 +458,6 @@ class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): class TestNearestAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 6, 3], dtype="float32") @@ -470,18 +465,18 @@ class TestNearestAPI(unittest.TestCase): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) - out1 = fluid.layers.resize_nearest(y, - out_shape=[12, 12], - data_format='NHWC') + out1 = fluid.layers.resize_nearest( + y, out_shape=[12, 12], data_format='NHWC' + ) out2 = fluid.layers.resize_nearest(x, out_shape=[12, dim]) out3 = fluid.layers.resize_nearest(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_nearest(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_nearest( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_nearest(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -496,39 +491,39 @@ class TestNearestAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = nearest_neighbor_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 1)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = nearest_neighbor_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 1)), rtol=1e-05 + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res, rtol=1e-05) class TestNearestInterpException(unittest.TestCase): - def test_exception(self): input = fluid.data(name="input", shape=[1, 3, 6, 6], dtype="float32") def attr_data_format(): # for 4-D input, data_format can only be NCHW or NHWC - out = fluid.layers.resize_nearest(input, - out_shape=[4, 8], - data_format='NDHWC') + out = fluid.layers.resize_nearest( + input, out_shape=[4, 8], data_format='NDHWC' + ) def attr_scale_type(): out = fluid.layers.resize_nearest(input, scale='scale') @@ -543,5 +538,6 @@ class TestNearestInterpException(unittest.TestCase): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py index c3157a6e9a1..fb1e1b08cea 100755 --- a/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py @@ -23,18 +23,20 @@ from paddle.nn.functional import interpolate paddle.enable_static() -def nearest_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='NCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='nearest', - align_corners=True, - align_mode=0): +def nearest_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='NCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='nearest', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -44,23 +46,36 @@ def nearest_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.nearest_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, scale, - interp_method, align_corners, - align_mode) - - -def nearest_neighbor_interp_np(X, - out_h, - out_w, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): + return paddle._C_ops.nearest_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) + + +def nearest_neighbor_interp_np( + X, + out_h, + out_w, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -73,16 +88,16 @@ def nearest_neighbor_interp_np(X, n, c, in_h, in_w = X.shape ratio_h = ratio_w = 0.0 - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -110,17 +125,19 @@ def nearest_neighbor_interp_np(X, return out.astype(X.dtype) -def nearest_neighbor_interp3d_np(X, - out_d, - out_h, - out_w, - scale_d=0, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp3d_np( + X, + out_d, + out_h, + out_w, + scale_d=0, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -135,24 +152,24 @@ def nearest_neighbor_interp3d_np(X, n, c, in_d, in_h, in_w = X.shape ratio_d = ratio_h = ratio_w = 0.0 - if (out_d > 1): - if (align_corners): + if out_d > 1: + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: if scale_d > 0: ratio_d = 1.0 / scale_d else: ratio_d = 1.0 * in_d / out_d - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -184,7 +201,6 @@ def nearest_neighbor_interp3d_np(X, class TestNearestInterpOp(OpTest): - def setUp(self): self.python_api = nearest_interp_test self.out_size = None @@ -240,15 +256,30 @@ class TestNearestInterpOp(OpTest): if len(self.input_shape) == 4: output_np = nearest_neighbor_interp_np( - input_np, out_h, out_w, scale_h, scale_w, self.out_size, - self.actual_shape, self.align_corners, self.data_layout) + input_np, + out_h, + out_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) elif len(self.input_shape) == 5: - output_np = nearest_neighbor_interp3d_np(input_np, out_d, out_h, - out_w, scale_d, scale_h, - scale_w, self.out_size, - self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp3d_np( + input_np, + out_d, + out_h, + out_w, + scale_d, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -261,7 +292,7 @@ class TestNearestInterpOp(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } else: self.attrs = { @@ -269,7 +300,7 @@ class TestNearestInterpOp(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -297,7 +328,6 @@ class TestNearestInterpOp(OpTest): class TestNearestNeighborInterpCase1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 1, 7, 8] @@ -309,7 +339,6 @@ class TestNearestNeighborInterpCase1(TestNearestInterpOp): class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] @@ -320,7 +349,6 @@ class TestNearestNeighborInterpCase2(TestNearestInterpOp): class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] @@ -331,7 +359,6 @@ class TestNearestNeighborInterpCase3(TestNearestInterpOp): class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] @@ -343,7 +370,6 @@ class TestNearestNeighborInterpCase4(TestNearestInterpOp): class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] @@ -355,7 +381,6 @@ class TestNearestNeighborInterpCase5(TestNearestInterpOp): class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] @@ -367,7 +392,6 @@ class TestNearestNeighborInterpCase6(TestNearestInterpOp): class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] @@ -378,7 +402,6 @@ class TestNearestNeighborInterpSame(TestNearestInterpOp): class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -390,7 +413,6 @@ class TestNearestNeighborInterpActualShape(TestNearestInterpOp): class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 4, 4, 5] @@ -403,15 +425,15 @@ class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): class TestNearestInterpOpUint8(OpTest): - def setUp(self): self.python_api = nearest_interp_test self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "nearest_interp_v2" - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -428,9 +450,16 @@ class TestNearestInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -438,7 +467,7 @@ class TestNearestInterpOpUint8(OpTest): 'out_h': self.out_h, 'out_w': self.out_w, 'interp_method': self.interp_method, - 'align_corners': self.align_corners + 'align_corners': self.align_corners, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -450,9 +479,9 @@ class TestNearestInterpOpUint8(OpTest): self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=True) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=True + ) def init_test_case(self): self.interp_method = 'nearest' @@ -464,7 +493,6 @@ class TestNearestInterpOpUint8(OpTest): class TestNearestNeighborInterpCase1Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] @@ -475,7 +503,6 @@ class TestNearestNeighborInterpCase1Uint8(TestNearestInterpOpUint8): class TestNearestNeighborInterpCase2Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] @@ -487,25 +514,22 @@ class TestNearestNeighborInterpCase2Uint8(TestNearestInterpOpUint8): class TestNearestInterpWithoutCorners(TestNearestInterpOp): - def set_align_corners(self): self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -517,7 +541,6 @@ class TestNearestNeighborInterpScale2(TestNearestInterpOp): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] @@ -529,7 +552,6 @@ class TestNearestNeighborInterpScale3(TestNearestInterpOp): class TestNearestNeighbor3DInterp(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 4, 7, 5] @@ -542,7 +564,6 @@ class TestNearestNeighbor3DInterp(TestNearestInterpOp): class TestNearestInterpOp_attr_tensor(OpTest): - def setUp(self): self.python_api = nearest_interp_test self.out_size = None @@ -581,8 +602,9 @@ class TestNearestInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -594,9 +616,16 @@ class TestNearestInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -617,7 +646,6 @@ class TestNearestInterpOp_attr_tensor(OpTest): # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] @@ -630,7 +658,6 @@ class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -644,7 +671,6 @@ class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -657,7 +683,6 @@ class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): class TestNearestAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 6, 3], dtype="float32") @@ -665,18 +690,18 @@ class TestNearestAPI(unittest.TestCase): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) - out1 = fluid.layers.resize_nearest(y, - out_shape=[12, 12], - data_format='NHWC') + out1 = fluid.layers.resize_nearest( + y, out_shape=[12, 12], data_format='NHWC' + ) out2 = fluid.layers.resize_nearest(x, out_shape=[12, dim]) out3 = fluid.layers.resize_nearest(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_nearest(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_nearest( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_nearest(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -691,33 +716,34 @@ class TestNearestAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = nearest_neighbor_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 1)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = nearest_neighbor_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 1)), rtol=1e-05 + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res, rtol=1e-05) class TestNearestInterpOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -727,21 +753,22 @@ class TestNearestInterpOpAPI_dy(unittest.TestCase): scale_np = np.array([2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = nearest_neighbor_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="nearest", - align_corners=False) + expect_res = nearest_neighbor_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="nearest", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestNearestInterp3DOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -751,30 +778,30 @@ class TestNearestInterp3DOpAPI_dy(unittest.TestCase): scale_np = np.array([2, 2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = nearest_neighbor_interp3d_np(input_data, - out_d=12, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="nearest", - align_corners=False, - data_format="NCDHW") + expect_res = nearest_neighbor_interp3d_np( + input_data, out_d=12, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="nearest", + align_corners=False, + data_format="NCDHW", + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestNearestInterpException(unittest.TestCase): - def test_exception(self): import paddle + input = fluid.data(name="input", shape=[1, 3, 6, 6], dtype="float32") def attr_data_format(): # for 4-D input, data_format can only be NCHW or NHWC - out = fluid.layers.resize_nearest(input, - out_shape=[4, 8], - data_format='NDHWC') + out = fluid.layers.resize_nearest( + input, out_shape=[4, 8], data_format='NDHWC' + ) def attr_scale_type(): out = fluid.layers.resize_nearest(input, scale='scale') @@ -788,9 +815,9 @@ class TestNearestInterpException(unittest.TestCase): def mode_error(): x = paddle.randn([1, 3]) - out = paddle.nn.functional.interpolate(x, - scale_factor='scale', - mode="BILINEAR") + out = paddle.nn.functional.interpolate( + x, scale_factor='scale', mode="BILINEAR" + ) self.assertRaises(ValueError, attr_data_format) self.assertRaises(TypeError, attr_scale_type) @@ -799,10 +826,10 @@ class TestNearestInterpException(unittest.TestCase): self.assertRaises(ValueError, mode_error) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNearestInterp3DOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 2, 6, 6, 6] @@ -815,11 +842,13 @@ class TestNearestInterp3DOpForFloat16(unittest.TestCase): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - scale_factor=self.scale, - mode=self.interp_method, - align_corners=self.align_corners, - data_format=self.data_layout) + y = interpolate( + x, + scale_factor=self.scale, + mode=self.interp_method, + align_corners=self.align_corners, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') @@ -838,10 +867,10 @@ class TestNearestInterp3DOpForFloat16(unittest.TestCase): np.testing.assert_allclose(x_g_np_1, x_g_np_2) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNearestInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 2, 6, 6] @@ -853,10 +882,12 @@ class TestNearestInterpOpForFloat16(unittest.TestCase): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - scale_factor=self.scale, - mode=self.interp_method, - align_corners=self.align_corners) + y = interpolate( + x, + scale_factor=self.scale, + mode=self.interp_method, + align_corners=self.align_corners, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_neg_op.py b/python/paddle/fluid/tests/unittests/test_neg_op.py index 77c961408a5..260d0fa35dc 100644 --- a/python/paddle/fluid/tests/unittests/test_neg_op.py +++ b/python/paddle/fluid/tests/unittests/test_neg_op.py @@ -18,7 +18,6 @@ import paddle class TestNegOp(unittest.TestCase): - def setUp(self): self.init_dtype_type() self.input = (np.random.random((32, 8)) * 100).astype(self.dtype) @@ -30,9 +29,9 @@ class TestNegOp(unittest.TestCase): input = paddle.to_tensor(self.input) dy_result = paddle.neg(input) expected_result = np.negative(self.input) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) def run_static(self, use_gpu=False): input = paddle.fluid.data(name='input', shape=[32, 8], dtype=self.dtype) @@ -66,31 +65,26 @@ class TestNegOp(unittest.TestCase): class TestNegOpFp32(TestNegOp): - def init_dtype_type(self): self.dtype = np.float32 class TestNegOpInt64(TestNegOp): - def init_dtype_type(self): self.dtype = np.int64 class TestNegOpInt32(TestNegOp): - def init_dtype_type(self): self.dtype = np.int32 class TestNegOpInt16(TestNegOp): - def init_dtype_type(self): self.dtype = np.int16 class TestNegOpInt8(TestNegOp): - def init_dtype_type(self): self.dtype = np.int8 diff --git a/python/paddle/fluid/tests/unittests/test_network_with_dtype.py b/python/paddle/fluid/tests/unittests/test_network_with_dtype.py index 203dcdc38cb..f230cc66c20 100644 --- a/python/paddle/fluid/tests/unittests/test_network_with_dtype.py +++ b/python/paddle/fluid/tests/unittests/test_network_with_dtype.py @@ -22,7 +22,6 @@ BATCH_SIZE = 20 class TestNetWithDtype(unittest.TestCase): - def setUp(self): self.dtype = "float64" self.init_dtype() @@ -40,8 +39,9 @@ class TestNetWithDtype(unittest.TestCase): sgd_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=BATCH_SIZE + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(startup) diff --git a/python/paddle/fluid/tests/unittests/test_newprofiler.py b/python/paddle/fluid/tests/unittests/test_newprofiler.py index afa09462cb6..0d42de0be7c 100755 --- a/python/paddle/fluid/tests/unittests/test_newprofiler.py +++ b/python/paddle/fluid/tests/unittests/test_newprofiler.py @@ -25,62 +25,67 @@ from paddle.io import Dataset, DataLoader class TestProfiler(unittest.TestCase): - def tearDown(self): self.temp_dir.cleanup() def test_profiler(self): - def my_trace_back(prof): - path = os.path.join(self.temp_dir.name, - './test_profiler_chrometracing') + path = os.path.join( + self.temp_dir.name, './test_profiler_chrometracing' + ) profiler.export_chrome_tracing(path)(prof) path = os.path.join(self.temp_dir.name, './test_profiler_pb') profiler.export_protobuf(path)(prof) self.temp_dir = tempfile.TemporaryDirectory() x_value = np.random.randn(2, 3, 3) - x = paddle.to_tensor(x_value, - stop_gradient=False, - place=paddle.CPUPlace()) + x = paddle.to_tensor( + x_value, stop_gradient=False, place=paddle.CPUPlace() + ) y = x / 2.0 ones_like_y = paddle.ones_like(y) - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], ) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + ) as prof: y = x / 2.0 prof = None self.assertEqual(utils._is_profiler_used, False) with profiler.RecordEvent(name='test'): y = x / 2.0 - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=(1, 2)) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], scheduler=(1, 2) + ) as prof: self.assertEqual(utils._is_profiler_used, True) with profiler.RecordEvent(name='test'): y = x / 2.0 prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=profiler.make_scheduler(closed=0, - ready=1, - record=1, - repeat=1), - on_trace_ready=my_trace_back) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + scheduler=profiler.make_scheduler( + closed=0, ready=1, record=1, repeat=1 + ), + on_trace_ready=my_trace_back, + ) as prof: y = x / 2.0 prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=profiler.make_scheduler(closed=0, - ready=0, - record=2, - repeat=1), - on_trace_ready=my_trace_back) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + scheduler=profiler.make_scheduler( + closed=0, ready=0, record=2, repeat=1 + ), + on_trace_ready=my_trace_back, + ) as prof: for i in range(3): y = x / 2.0 prof.step() prof = None with profiler.Profiler( - targets=[profiler.ProfilerTarget.CPU], - scheduler=lambda x: profiler.ProfilerState.RECORD_AND_RETURN, - on_trace_ready=my_trace_back) as prof: + targets=[profiler.ProfilerTarget.CPU], + scheduler=lambda x: profiler.ProfilerState.RECORD_AND_RETURN, + on_trace_ready=my_trace_back, + ) as prof: for i in range(2): y = x / 2.0 prof.step() @@ -107,35 +112,39 @@ class TestProfiler(unittest.TestCase): prof = None with profiler.Profiler( - targets=[profiler.ProfilerTarget.CPU], - scheduler=lambda x: profiler.ProfilerState.RECORD_AND_RETURN, - on_trace_ready=my_trace_back) as prof: + targets=[profiler.ProfilerTarget.CPU], + scheduler=lambda x: profiler.ProfilerState.RECORD_AND_RETURN, + on_trace_ready=my_trace_back, + ) as prof: for i in range(2): y = x / 2.0 prof.step() prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=my_sheduler, - on_trace_ready=my_trace_back) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + scheduler=my_sheduler, + on_trace_ready=my_trace_back, + ) as prof: for i in range(5): y = x / 2.0 prof.step() prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=my_sheduler1) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], scheduler=my_sheduler1 + ) as prof: for i in range(5): y = x / 2.0 prof.step() prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=profiler.make_scheduler(closed=1, - ready=1, - record=2, - repeat=1, - skip_first=1), - on_trace_ready=my_trace_back, - profile_memory=True, - record_shapes=True) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + scheduler=profiler.make_scheduler( + closed=1, ready=1, record=2, repeat=1, skip_first=1 + ), + on_trace_ready=my_trace_back, + profile_memory=True, + record_shapes=True, + ) as prof: for i in range(5): y = x / 2.0 paddle.grad(outputs=y, inputs=[x], grad_outputs=ones_like_y) @@ -148,13 +157,12 @@ class TestProfiler(unittest.TestCase): prof = None dataset = RandomDataset(10 * 4) simple_net = SimpleNet() - opt = paddle.optimizer.SGD(learning_rate=1e-3, - parameters=simple_net.parameters()) - loader = DataLoader(dataset, - batch_size=4, - shuffle=True, - drop_last=True, - num_workers=2) + opt = paddle.optimizer.SGD( + learning_rate=1e-3, parameters=simple_net.parameters() + ) + loader = DataLoader( + dataset, batch_size=4, shuffle=True, drop_last=True, num_workers=2 + ) prof = profiler.Profiler(on_trace_ready=lambda prof: None) prof.start() for i, (image, label) in enumerate(loader()): @@ -171,8 +179,9 @@ class TestProfiler(unittest.TestCase): dataset = RandomDataset(10 * 4) simple_net = SimpleNet() loader = DataLoader(dataset, batch_size=4, shuffle=True, drop_last=True) - opt = paddle.optimizer.Adam(learning_rate=1e-3, - parameters=simple_net.parameters()) + opt = paddle.optimizer.Adam( + learning_rate=1e-3, parameters=simple_net.parameters() + ) prof = profiler.Profiler(on_trace_ready=lambda prof: None) prof.start() for i, (image, label) in enumerate(loader()): @@ -187,19 +196,17 @@ class TestProfiler(unittest.TestCase): class TestNvprof(unittest.TestCase): - def test_nvprof(self): for i in range(10): paddle.fluid.profiler._nvprof_range(i, 10, 20) x_value = np.random.randn(2, 3, 3) - x = paddle.to_tensor(x_value, - stop_gradient=False, - place=paddle.CPUPlace()) + x = paddle.to_tensor( + x_value, stop_gradient=False, place=paddle.CPUPlace() + ) y = x / 2.0 class TestGetProfiler(unittest.TestCase): - def test_getprofiler(self): config_content = ''' { @@ -222,11 +229,12 @@ class TestGetProfiler(unittest.TestCase): filehandle.write(config_content) filehandle.flush() import paddle.profiler.profiler as profiler + profiler = profiler.get_profiler(filehandle.name) x_value = np.random.randn(2, 3, 3) - x = paddle.to_tensor(x_value, - stop_gradient=False, - place=paddle.CPUPlace()) + x = paddle.to_tensor( + x_value, stop_gradient=False, place=paddle.CPUPlace() + ) with profiler: for i in range(5): y = x / 2.0 @@ -262,6 +270,7 @@ class TestGetProfiler(unittest.TestCase): filehandle.write(config_content) filehandle.flush() import paddle.profiler.profiler as profiler + try: profiler = profiler.get_profiler(filehandle.name) except: @@ -299,6 +308,7 @@ class TestGetProfiler(unittest.TestCase): filehandle.write(config_content) filehandle.flush() import paddle.profiler.profiler as profiler + profiler = profiler.get_profiler(filehandle.name) # test exception @@ -334,20 +344,21 @@ class TestGetProfiler(unittest.TestCase): filehandle.write(config_content) filehandle.flush() import paddle.profiler.profiler as profiler + profiler = profiler.get_profiler(filehandle.name) # test path error import paddle.profiler.profiler as profiler + profiler = profiler.get_profiler('nopath.json') class RandomDataset(Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([100]).astype('float32') - label = np.random.randint(0, 10 - 1, (1, )).astype('int64') + label = np.random.randint(0, 10 - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -355,7 +366,6 @@ class RandomDataset(Dataset): class SimpleNet(nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.fc = nn.Linear(100, 10) @@ -365,19 +375,20 @@ class SimpleNet(nn.Layer): class TestTimerOnly(unittest.TestCase): - def test_with_dataloader(self): - def train(step_num_samples=None): dataset = RandomDataset(20 * 4) simple_net = SimpleNet() - opt = paddle.optimizer.SGD(learning_rate=1e-3, - parameters=simple_net.parameters()) - loader = DataLoader(dataset, - batch_size=4, - shuffle=True, - drop_last=True, - num_workers=2) + opt = paddle.optimizer.SGD( + learning_rate=1e-3, parameters=simple_net.parameters() + ) + loader = DataLoader( + dataset, + batch_size=4, + shuffle=True, + drop_last=True, + num_workers=2, + ) step_info = '' p = profiler.Profiler(timer_only=True) p.start() diff --git a/python/paddle/fluid/tests/unittests/test_newprofiler_helper.py b/python/paddle/fluid/tests/unittests/test_newprofiler_helper.py index e03ee36cad1..706ae74113f 100755 --- a/python/paddle/fluid/tests/unittests/test_newprofiler_helper.py +++ b/python/paddle/fluid/tests/unittests/test_newprofiler_helper.py @@ -18,7 +18,6 @@ import paddle.profiler.statistic_helper as statistic_helper class TestStatisticHelper(unittest.TestCase): - def test_sum_ranges_case1(self): src = [(1, 3), (4, 10), (11, 15)] self.assertEqual(statistic_helper.sum_ranges(src), 12) diff --git a/python/paddle/fluid/tests/unittests/test_nll_loss.py b/python/paddle/fluid/tests/unittests/test_nll_loss.py index b8158bc0e6b..a598c501ec0 100644 --- a/python/paddle/fluid/tests/unittests/test_nll_loss.py +++ b/python/paddle/fluid/tests/unittests/test_nll_loss.py @@ -20,11 +20,9 @@ from op_test import OpTest from paddle.fluid.framework import _test_eager_guard -def nll_loss_1d(logs, - targets, - weight=None, - reduction='mean', - ignore_index=-100): +def nll_loss_1d( + logs, targets, weight=None, reduction='mean', ignore_index=-100 +): input_shape = logs.shape N = input_shape[0] C = input_shape[1] @@ -41,17 +39,16 @@ def nll_loss_1d(logs, if reduction == 'sum': return np.sum(out), np.array([total_weight]).astype('float64') elif reduction == 'mean': - return out.sum() / total_weight, np.array([total_weight - ]).astype('float64') + return out.sum() / total_weight, np.array([total_weight]).astype( + 'float64' + ) elif reduction == 'none': return out -def nll_loss_2d(logs, - targets, - weight=None, - reduction='mean', - ignore_index=-100): +def nll_loss_2d( + logs, targets, weight=None, reduction='mean', ignore_index=-100 +): input_shape = logs.shape N = input_shape[0] H = input_shape[2] @@ -71,24 +68,27 @@ def nll_loss_2d(logs, if reduction == 'sum': return np.sum(out), np.array([total_weight]).astype('float64') elif reduction == 'mean': - return out.sum() / total_weight, np.array([total_weight - ]).astype('float64') + return out.sum() / total_weight, np.array([total_weight]).astype( + 'float64' + ) elif reduction == 'none': return out class TestNLLLoss(unittest.TestCase): - def test_NLLLoss_1D_mean(self): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[10, 10], dtype='float64') label = fluid.data(name='label', shape=[10], dtype='int64') @@ -96,24 +96,25 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss() - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() with fluid.dygraph.guard(): with _test_eager_guard(): nll_loss = paddle.nn.loss.NLLLoss() - eager_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + eager_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) eager_result = eager_res.numpy() expected = nll_loss_1d(input_np, label_np)[0] @@ -126,12 +127,15 @@ class TestNLLLoss(unittest.TestCase): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[10, 10], dtype='float64') label = fluid.data(name='label', shape=[10], dtype='int64') @@ -139,17 +143,17 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss(reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() with _test_eager_guard(): @@ -172,12 +176,15 @@ class TestNLLLoss(unittest.TestCase): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) - weight_np = np.random.random(size=(10, )).astype(np.float64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) + weight_np = np.random.random(size=(10,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[10, 10], dtype='float64') @@ -187,26 +194,32 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() with _test_eager_guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - eager_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + eager_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) loss = eager_res.sum() loss.backward() eager_result = eager_res.numpy() @@ -222,12 +235,15 @@ class TestNLLLoss(unittest.TestCase): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) - weight_np = np.random.random(size=(10, )).astype(np.float64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) + weight_np = np.random.random(size=(10,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[10, 10], dtype='float64') @@ -237,24 +253,27 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='sum' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() - expected = nll_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='sum')[0] + expected = nll_loss_1d( + input_np, label_np, weight=weight_np, reduction='sum' + )[0] np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -264,8 +283,8 @@ class TestNLLLoss(unittest.TestCase): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) - weight_np = np.random.random(size=(10, )).astype(np.float64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) + weight_np = np.random.random(size=(10,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() place = fluid.CPUPlace() @@ -277,19 +296,23 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_1d(input_np, label_np, weight=weight_np)[0] @@ -301,8 +324,8 @@ class TestNLLLoss(unittest.TestCase): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) - weight_np = np.random.random(size=(10, )).astype(np.float64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) + weight_np = np.random.random(size=(10,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() place = fluid.CPUPlace() @@ -314,24 +337,27 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='none') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='none' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() - expected = nll_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='none') + expected = nll_loss_1d( + input_np, label_np, weight=weight_np, reduction='none' + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -344,29 +370,32 @@ class TestNLLLoss(unittest.TestCase): label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') nll_loss = paddle.nn.loss.NLLLoss() res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss() - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_2d(input_np, label_np)[0] @@ -382,29 +411,32 @@ class TestNLLLoss(unittest.TestCase): label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') nll_loss = paddle.nn.loss.NLLLoss(reduction='sum') res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss(reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_2d(input_np, label_np, reduction='sum')[0] @@ -418,16 +450,19 @@ class TestNLLLoss(unittest.TestCase): input_np = np.random.random(size=(5, 3, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') @@ -435,19 +470,23 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_2d(input_np, label_np, weight=weight_np)[0] @@ -461,14 +500,14 @@ class TestNLLLoss(unittest.TestCase): input_np = np.random.random(size=(5, 3, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') @@ -476,19 +515,23 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_2d(input_np, label_np, weight=weight_np)[0] @@ -502,15 +545,18 @@ class TestNLLLoss(unittest.TestCase): input_np = np.random.random(size=(5, 3, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') @@ -518,25 +564,28 @@ class TestNLLLoss(unittest.TestCase): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='sum' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() - expected = nll_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='sum')[0] + expected = nll_loss_2d( + input_np, label_np, weight=weight_np, reduction='sum' + )[0] np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -549,35 +598,39 @@ class TestNLLLoss(unittest.TestCase): label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') nll_loss = paddle.nn.loss.NLLLoss() res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss() - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) expected = nll_loss_2d(input_np_reshape, label_np_reshape)[0] @@ -590,45 +643,53 @@ class TestNLLLoss(unittest.TestCase): input_np = np.random.random(size=(5, 3, 5, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') nll_loss = paddle.nn.loss.NLLLoss(weight=weight) res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) - expected = nll_loss_2d(input_np_reshape, - label_np_reshape, - weight=weight_np)[0] + expected = nll_loss_2d( + input_np_reshape, label_np_reshape, weight=weight_np + )[0] np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -639,46 +700,56 @@ class TestNLLLoss(unittest.TestCase): input_np = np.random.random(size=(5, 3, 5, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') nll_loss = paddle.nn.loss.NLLLoss(weight=weight, reduction='sum') res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='sum' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) - expected = nll_loss_2d(input_np_reshape, - label_np_reshape, - weight=weight_np, - reduction='sum')[0] + expected = nll_loss_2d( + input_np_reshape, + label_np_reshape, + weight=weight_np, + reduction='sum', + )[0] np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -689,47 +760,57 @@ class TestNLLLoss(unittest.TestCase): input_np = np.random.random(size=(5, 3, 5, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') nll_loss = paddle.nn.loss.NLLLoss(weight=weight, reduction='none') res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='none') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='none' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - out_shape = (input_shape[0], ) + input_shape[2:] - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + out_shape = (input_shape[0],) + input_shape[2:] + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) - expected = nll_loss_2d(input_np_reshape, - label_np_reshape, - weight=weight_np, - reduction='none') + expected = nll_loss_2d( + input_np_reshape, + label_np_reshape, + weight=weight_np, + reduction='none', + ) expected = np.reshape(expected, out_shape) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -740,45 +821,52 @@ class TestNLLLoss(unittest.TestCase): input_np = np.random.random(size=(5, 3, 5, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') nll_loss = paddle.nn.loss.NLLLoss(weight=weight, reduction='none') res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='none') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='none' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - out_shape = (input_shape[0], ) + input_shape[2:] - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + out_shape = (input_shape[0],) + input_shape[2:] + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) - expected = nll_loss_2d(input_np_reshape, - label_np_reshape, - weight=weight_np, - reduction='none') + expected = nll_loss_2d( + input_np_reshape, + label_np_reshape, + weight=weight_np, + reduction='none', + ) expected = np.reshape(expected, out_shape) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -786,7 +874,6 @@ class TestNLLLoss(unittest.TestCase): class TestNLLLossOp1DWithReduce(OpTest): - def setUp(self): self.init_test_case() self.op_type = "nll_loss" @@ -796,20 +883,23 @@ class TestNLLLossOp1DWithReduce(OpTest): self.python_api = paddle.nn.functional.nll_loss self.python_out_sig = ["Out"] np.random.seed(200) - input_np = np.random.uniform(0.1, 0.8, - self.input_shape).astype("float64") + input_np = np.random.uniform(0.1, 0.8, self.input_shape).astype( + "float64" + ) np.random.seed(200) - label_np = np.random.randint(0, self.input_shape[1], - self.label_shape).astype("int64") + label_np = np.random.randint( + 0, self.input_shape[1], self.label_shape + ).astype("int64") output_np, total_weight_np = nll_loss_1d(input_np, label_np) self.inputs = {'X': input_np, 'Label': label_np} if self.with_weight: np.random.seed(200) - weight_np = np.random.uniform(0.1, 0.8, - self.input_shape[1]).astype("float64") - output_np, total_weight_np = nll_loss_1d(input_np, - label_np, - weight=weight_np) + weight_np = np.random.uniform(0.1, 0.8, self.input_shape[1]).astype( + "float64" + ) + output_np, total_weight_np = nll_loss_1d( + input_np, label_np, weight=weight_np + ) self.inputs['Weight'] = weight_np self.outputs = {'Out': output_np, 'Total_weight': total_weight_np} @@ -836,7 +926,6 @@ class TestNLLLossOp1DWithReduce(OpTest): class TestNLLLossOp1DNoReduce(OpTest): - def setUp(self): self.init_test_case() self.op_type = "nll_loss" @@ -844,22 +933,24 @@ class TestNLLLossOp1DNoReduce(OpTest): self.python_out_sig = ["Out"] self.with_weight = False np.random.seed(200) - input_np = np.random.uniform(0.1, 0.8, - self.input_shape).astype("float64") + input_np = np.random.uniform(0.1, 0.8, self.input_shape).astype( + "float64" + ) np.random.seed(200) - label_np = np.random.randint(0, self.input_shape[1], - self.label_shape).astype("int64") + label_np = np.random.randint( + 0, self.input_shape[1], self.label_shape + ).astype("int64") output_np = nll_loss_1d(input_np, label_np, reduction='none') total_weight_np = np.array([0]).astype('float64') self.inputs = {'X': input_np, 'Label': label_np} if self.with_weight: np.random.seed(200) - weight_np = np.random.uniform(0.1, 0.8, - self.input_shape[1]).astype("float64") - output_np, total_weight_np = nll_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='none') + weight_np = np.random.uniform(0.1, 0.8, self.input_shape[1]).astype( + "float64" + ) + output_np, total_weight_np = nll_loss_1d( + input_np, label_np, weight=weight_np, reduction='none' + ) self.inputs['Weight'] = weight_np self.outputs = {'Out': output_np, 'Total_weight': total_weight_np} @@ -886,7 +977,6 @@ class TestNLLLossOp1DNoReduce(OpTest): class TestNLLLossOp2DWithReduce(OpTest): - def setUp(self): self.init_test_case() self.op_type = "nll_loss" @@ -894,20 +984,23 @@ class TestNLLLossOp2DWithReduce(OpTest): self.python_out_sig = ["Out"] self.with_weight = False np.random.seed(200) - input_np = np.random.uniform(0.1, 0.8, - self.input_shape).astype("float64") + input_np = np.random.uniform(0.1, 0.8, self.input_shape).astype( + "float64" + ) np.random.seed(200) - label_np = np.random.randint(0, self.input_shape[1], - self.label_shape).astype("int64") + label_np = np.random.randint( + 0, self.input_shape[1], self.label_shape + ).astype("int64") output_np, total_weight_np = nll_loss_2d(input_np, label_np) self.inputs = {'X': input_np, 'Label': label_np} if self.with_weight: np.random.seed(200) - weight_np = np.random.uniform(0.1, 0.8, - self.input_shape[1]).astype("float64") - output_np, total_weight_np = nll_loss_2d(input_np, - label_np, - weight=weight_np) + weight_np = np.random.uniform(0.1, 0.8, self.input_shape[1]).astype( + "float64" + ) + output_np, total_weight_np = nll_loss_2d( + input_np, label_np, weight=weight_np + ) self.inputs['Weight'] = weight_np self.outputs = {'Out': output_np, 'Total_weight': total_weight_np} @@ -934,7 +1027,6 @@ class TestNLLLossOp2DWithReduce(OpTest): class TestNLLLossOp2DNoReduce(OpTest): - def setUp(self): self.init_test_case() self.op_type = "nll_loss" @@ -942,22 +1034,24 @@ class TestNLLLossOp2DNoReduce(OpTest): self.python_out_sig = ["Out"] self.with_weight = False np.random.seed(200) - input_np = np.random.uniform(0.1, 0.8, - self.input_shape).astype("float64") + input_np = np.random.uniform(0.1, 0.8, self.input_shape).astype( + "float64" + ) np.random.seed(200) - label_np = np.random.randint(0, self.input_shape[1], - self.label_shape).astype("int64") + label_np = np.random.randint( + 0, self.input_shape[1], self.label_shape + ).astype("int64") output_np = nll_loss_2d(input_np, label_np, reduction='none') total_weight_np = np.array([0]).astype('float64') self.inputs = {'X': input_np, 'Label': label_np} if self.with_weight: np.random.seed(200) - weight_np = np.random.uniform(0.1, 0.8, - self.input_shape[1]).astype("float64") - output_np, total_weight_np = nll_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='none') + weight_np = np.random.uniform(0.1, 0.8, self.input_shape[1]).astype( + "float64" + ) + output_np, total_weight_np = nll_loss_2d( + input_np, label_np, weight=weight_np, reduction='none' + ) self.inputs['Weight'] = weight_np self.outputs = {'Out': output_np, 'Total_weight': total_weight_np} @@ -984,7 +1078,6 @@ class TestNLLLossOp2DNoReduce(OpTest): class TestNLLLossName(unittest.TestCase): - def test_name(self): prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -998,22 +1091,26 @@ class TestNLLLossName(unittest.TestCase): class TestNLLLossInvalidArgs(unittest.TestCase): - def test_x_dim_value_error(self): - def test_x_dim_lt_2(): prog = paddle.static.Program() startup_prog = paddle.static.Program() place = paddle.CPUPlace() with paddle.static.program_guard(prog, startup_prog): - x = paddle.fluid.data(name='x', shape=[ - 10, - ], dtype='float64') - label = paddle.fluid.data(name='label', - shape=[ - 10, - ], - dtype='float64') + x = paddle.fluid.data( + name='x', + shape=[ + 10, + ], + dtype='float64', + ) + label = paddle.fluid.data( + name='label', + shape=[ + 10, + ], + dtype='float64', + ) nll_loss = paddle.nn.loss.NLLLoss() res = nll_loss(x, label) @@ -1021,8 +1118,8 @@ class TestNLLLossInvalidArgs(unittest.TestCase): def test_x_dim_imperative_lt_2(): with fluid.dygraph.guard(): - x_np = np.random.random(size=(5, )).astype(np.float64) - label_np = np.random.randint(0, 10, size=(5, )).astype(np.int64) + x_np = np.random.random(size=(5,)).astype(np.float64) + label_np = np.random.randint(0, 10, size=(5,)).astype(np.int64) x = paddle.to_tensor(x_np) label = paddle.to_tensor(label_np) nll_loss = paddle.nn.loss.NLLLoss() @@ -1031,16 +1128,15 @@ class TestNLLLossInvalidArgs(unittest.TestCase): self.assertRaises(ValueError, test_x_dim_imperative_lt_2) def test_reduction_value_error(self): - def test_NLLLoss_reduction_not_sum_mean_none(): prog = paddle.static.Program() startup_prog = paddle.static.Program() place = paddle.CPUPlace() with paddle.static.program_guard(prog, startup_prog): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') - label = paddle.fluid.data(name='label', - shape=[10], - dtype='int64') + label = paddle.fluid.data( + name='label', shape=[10], dtype='int64' + ) nll_loss = paddle.nn.loss.NLLLoss(reduction='') res = nll_loss(x, label) @@ -1049,14 +1145,15 @@ class TestNLLLossInvalidArgs(unittest.TestCase): def test_NLLLoss_reduction_imperative_not_sum_mean_none(): with fluid.dygraph.guard(): x_np = np.random.random(size=(5, 3)).astype(np.float64) - label_np = np.random.randint(0, 3, size=(5, )).astype(np.int64) + label_np = np.random.randint(0, 3, size=(5,)).astype(np.int64) x = paddle.to_tensor(x_np) label = paddle.to_tensor(label_np) nll_loss = paddle.nn.loss.NLLLoss(reduction='') res = nll_loss(x, label) - self.assertRaises(ValueError, - test_NLLLoss_reduction_imperative_not_sum_mean_none) + self.assertRaises( + ValueError, test_NLLLoss_reduction_imperative_not_sum_mean_none + ) def test_nll_loss_function_reduction_not_sum_mean_none(): prog = paddle.static.Program() @@ -1064,25 +1161,27 @@ class TestNLLLossInvalidArgs(unittest.TestCase): place = paddle.CPUPlace() with paddle.static.program_guard(prog, startup_prog): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') - label = paddle.fluid.data(name='label', - shape=[10], - dtype='int64') + label = paddle.fluid.data( + name='label', shape=[10], dtype='int64' + ) res = paddle.nn.functional.nll_loss(x, label, reduction='') - self.assertRaises(ValueError, - test_nll_loss_function_reduction_not_sum_mean_none) + self.assertRaises( + ValueError, test_nll_loss_function_reduction_not_sum_mean_none + ) def test_nll_loss_function_reduction_imperative_not_sum_mean_none(): with fluid.dygraph.guard(): x_np = np.random.random(size=(5, 3)).astype(np.float64) - label_np = np.random.randint(0, 3, size=(5, )).astype(np.int64) + label_np = np.random.randint(0, 3, size=(5,)).astype(np.int64) x = paddle.to_tensor(x_np) label = paddle.to_tensor(label_np) res = paddle.nn.functional.nll_loss(x, label, reduction='') self.assertRaises( ValueError, - test_nll_loss_function_reduction_imperative_not_sum_mean_none) + test_nll_loss_function_reduction_imperative_not_sum_mean_none, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_nms_op.py b/python/paddle/fluid/tests/unittests/test_nms_op.py index cbd24d4ddf2..80f2da01325 100755 --- a/python/paddle/fluid/tests/unittests/test_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_nms_op.py @@ -19,8 +19,7 @@ import paddle def iou(box_a, box_b): - """Apply intersection-over-union overlap between box_a and box_b - """ + """Apply intersection-over-union overlap between box_a and box_b""" xmin_a = min(box_a[0], box_a[2]) ymin_a = min(box_a[1], box_a[3]) xmax_a = max(box_a[0], box_a[2]) @@ -69,7 +68,6 @@ def nms(boxes, nms_threshold): class TestNMSOp(OpTest): - def setUp(self): self.op_type = 'nms' self.python_api = paddle.vision.ops.nms diff --git a/python/paddle/fluid/tests/unittests/test_nn_dice_loss.py b/python/paddle/fluid/tests/unittests/test_nn_dice_loss.py index f5a7d2ed4ed..f1e1f6f6e4b 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_dice_loss.py +++ b/python/paddle/fluid/tests/unittests/test_nn_dice_loss.py @@ -22,7 +22,6 @@ eps = 1e-6 class TestDiceLossValue(unittest.TestCase): - def test_dice_loss(self): input_ = paddle.rand([2, 3, num_classes]) label_ = paddle.randint(0, num_classes, [2, 3, 1], dtype=paddle.int64) @@ -40,23 +39,21 @@ class TestDiceLossValue(unittest.TestCase): class TestDiceLossInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_dtype(): input_ = paddle.rand([2, 3, num_classes], dtype=paddle.float32) - label_ = paddle.randint(0, - num_classes, [2, 3, 1], - dtype=paddle.int64) + label_ = paddle.randint( + 0, num_classes, [2, 3, 1], dtype=paddle.int64 + ) nn.dice_loss(input_, label_.astype(paddle.float32)) self.assertRaises(AssertionError, test_invalid_dtype) def test_zero_shape_input(): input_ = paddle.rand([0, 3, num_classes], dtype=paddle.float32) - label_ = paddle.randint(0, - num_classes, [0, 3, 1], - dtype=paddle.int64) + label_ = paddle.randint( + 0, num_classes, [0, 3, 1], dtype=paddle.int64 + ) nn.dice_loss(input_, label_) self.assertRaises(AssertionError, test_zero_shape_input) diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py index d658fcec836..a7062e60025 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py @@ -22,7 +22,6 @@ paddle.disable_static() class EmbeddingDygraph(unittest.TestCase): - def func_1(self): x_data = np.arange(3, 6).reshape((3, 1)).astype(np.int64) paddle.disable_static(paddle.CPUPlace()) @@ -33,8 +32,9 @@ class EmbeddingDygraph(unittest.TestCase): w0 = np.full(shape=(10, 3), fill_value=2).astype(np.float32) embedding.weight.set_value(w0) - adam = paddle.optimizer.Adam(parameters=[embedding.weight], - learning_rate=0.01) + adam = paddle.optimizer.Adam( + parameters=[embedding.weight], learning_rate=0.01 + ) adam.clear_grad() out = embedding(x) diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py index bb1541beb1f..b5c354c7486 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py @@ -19,33 +19,36 @@ import paddle.nn.functional as functional class EmbeddingStatic(unittest.TestCase): - def test_1(self): prog = fluid.Program() with fluid.program_guard(prog): def test_bad_x(): initializer = fluid.initializer.NumpyArrayInitializer( - np.random.random(size=(128, 100))) - - param_attr = fluid.ParamAttr(name="emb_weight", - learning_rate=0.5, - initializer=initializer, - trainable=True) - - weight = prog.global_block().create_parameter((128, 100), - attr=param_attr, - dtype="float32") - - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="int64") - - emb = functional.embedding(x=label, - weight=weight, - sparse=True, - name="embedding") + np.random.random(size=(128, 100)) + ) + + param_attr = fluid.ParamAttr( + name="emb_weight", + learning_rate=0.5, + initializer=initializer, + trainable=True, + ) + + weight = prog.global_block().create_parameter( + (128, 100), attr=param_attr, dtype="float32" + ) + + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="int64", + ) + + emb = functional.embedding( + x=label, weight=weight, sparse=True, name="embedding" + ) test_bad_x() @@ -55,27 +58,34 @@ class EmbeddingStatic(unittest.TestCase): def test_bad_x(): initializer = fluid.initializer.NumpyArrayInitializer( - np.random.random(size=(128, 100))) - - param_attr = fluid.ParamAttr(name="emb_weight", - learning_rate=0.5, - initializer=initializer, - trainable=True) - - weight = prog.global_block().create_parameter((128, 100), - attr=param_attr, - dtype="float32") - - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="int32") - - emb = functional.embedding(x=label, - weight=weight, - padding_idx=129, - sparse=True, - name="embedding") + np.random.random(size=(128, 100)) + ) + + param_attr = fluid.ParamAttr( + name="emb_weight", + learning_rate=0.5, + initializer=initializer, + trainable=True, + ) + + weight = prog.global_block().create_parameter( + (128, 100), attr=param_attr, dtype="float32" + ) + + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="int32", + ) + + emb = functional.embedding( + x=label, + weight=weight, + padding_idx=129, + sparse=True, + name="embedding", + ) with self.assertRaises(ValueError): test_bad_x() diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_hot_op.py b/python/paddle/fluid/tests/unittests/test_nn_functional_hot_op.py index 27a7af94485..f7126a99881 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_hot_op.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_hot_op.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import Program, program_guard class TestOneHotOp(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -47,7 +46,6 @@ class TestOneHotOp(OpTest): class TestOneHotOp_attr(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -56,8 +54,9 @@ class TestOneHotOp_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -71,7 +70,6 @@ class TestOneHotOp_attr(OpTest): class TestOneHotOp_default_dtype(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -95,7 +93,6 @@ class TestOneHotOp_default_dtype(OpTest): class TestOneHotOp_default_dtype_attr(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -104,8 +101,9 @@ class TestOneHotOp_default_dtype_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -119,7 +117,6 @@ class TestOneHotOp_default_dtype_attr(OpTest): class TestOneHotOp_exception(unittest.TestCase): - def setUp(self): self.op_type = 'one_hot_v2' self.depth = 10 @@ -135,30 +132,34 @@ class TestOneHotOp_exception(unittest.TestCase): def test_check_output(self): program = Program() with program_guard(program): - x = fluid.layers.data(name='x', - shape=[self.dimension], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='x', shape=[self.dimension], dtype='float32', lod_level=1 + ) block = program.current_block() - one_hot_out = block.create_var(name="one_hot_out", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype='float32') - block.append_op(type='one_hot', - inputs={'X': x}, - attrs={'depth': self.depth}, - outputs={'Out': one_hot_out}) + one_hot_out = block.create_var( + name="one_hot_out", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype='float32', + ) + block.append_op( + type='one_hot', + inputs={'X': x}, + attrs={'depth': self.depth}, + outputs={'Out': one_hot_out}, + ) exe = fluid.Executor(self.place) def run(): - exe.run(feed={'x': self.x}, - fetch_list=[one_hot_out], - return_numpy=False) + exe.run( + feed={'x': self.x}, + fetch_list=[one_hot_out], + return_numpy=False, + ) self.assertRaises(ValueError, run) class TestOneHotOpApi(unittest.TestCase): - def test_api(self): num_classes = 10 self._run(num_classes) @@ -169,40 +170,45 @@ class TestOneHotOpApi(unittest.TestCase): def test_api_with_dygraph(self): num_classes = 10 - label = np.array([ - np.random.randint(0, num_classes - 1) for i in range(6) - ]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, num_classes - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(): one_hot_label = functional.one_hot( - x=fluid.dygraph.to_variable(label), num_classes=num_classes) + x=fluid.dygraph.to_variable(label), num_classes=num_classes + ) def _run(self, num_classes): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = functional.one_hot(x=label, num_classes=num_classes) place = fluid.CPUPlace() - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) class BadInputTestOnehotV2(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="float32") + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="float32", + ) one_hot_label = functional.one_hot(x=label, num_classes=4) self.assertRaises(TypeError, test_bad_x) diff --git a/python/paddle/fluid/tests/unittests/test_nn_grad.py b/python/paddle/fluid/tests/unittests/test_nn_grad.py index 7b806491bc5..4b3d92c92dc 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_nn_grad.py @@ -26,28 +26,25 @@ paddle.enable_static() class TestSliceOpDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): self.config() - out = fluid.layers.slice(self.inputs, - axes=self.axes, - starts=self.starts, - ends=self.ends) - gradient_checker.double_grad_check([self.inputs], - out, - x_init=self.x_arr, - place=place) + out = fluid.layers.slice( + self.inputs, axes=self.axes, starts=self.starts, ends=self.ends + ) + gradient_checker.double_grad_check( + [self.inputs], out, x_init=self.x_arr, place=place + ) def config(self): self.starts = [1, 0, -1] self.ends = [3, 3, 6] self.axes = [0, 1, 2] self.x_arr = np.random.random([3, 4, 5, 2]).astype("float64") - self.inputs = layers.create_parameter(dtype="float64", - shape=[3, 4, 5, 2], - name='x') + self.inputs = layers.create_parameter( + dtype="float64", shape=[3, 4, 5, 2], name='x' + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -58,19 +55,17 @@ class TestSliceOpDoubleGradCheck(unittest.TestCase): class TestSliceOpDoubleGradCheckCase3(TestSliceOpDoubleGradCheck): - def config(self): self.starts = [1, -1, 1] self.ends = [3, 3, 3] self.axes = [0, 1, 2] self.x_arr = np.random.random([3, 3, 3]).astype("float64") - self.inputs = layers.create_parameter(dtype="float64", - shape=[3, 3, 3], - name='x3') + self.inputs = layers.create_parameter( + dtype="float64", shape=[3, 3, 3], name='x3' + ) class TestReduceMeanWithDimDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [7, 11] @@ -82,11 +77,9 @@ class TestReduceMeanWithDimDoubleGradCheck(unittest.TestCase): y = layers.reduce_mean(x, dim=0) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -97,7 +90,6 @@ class TestReduceMeanWithDimDoubleGradCheck(unittest.TestCase): class TestReduceSumWithDimDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [7, 11] @@ -109,11 +101,9 @@ class TestReduceSumWithDimDoubleGradCheck(unittest.TestCase): y = layers.reduce_sum(x, dim=0) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -124,7 +114,6 @@ class TestReduceSumWithDimDoubleGradCheck(unittest.TestCase): class TestReshapeDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): x_shape = [3, 12] @@ -137,11 +126,9 @@ class TestReshapeDoubleGradCheck(unittest.TestCase): out = layers.expand(x, expand_times) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -152,7 +139,6 @@ class TestReshapeDoubleGradCheck(unittest.TestCase): class TestExpandDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): x_shape = [3, 12] @@ -165,11 +151,9 @@ class TestExpandDoubleGradCheck(unittest.TestCase): out = layers.reshape(x, new_shape) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -180,7 +164,6 @@ class TestExpandDoubleGradCheck(unittest.TestCase): class TestTileDoubleGradCheck(unittest.TestCase): - def tile_wrapper(self, x): return paddle.tile(x[0], [4, 9]) @@ -196,15 +179,12 @@ class TestTileDoubleGradCheck(unittest.TestCase): out = paddle.tile(x, repeat_times) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.tile_wrapper, [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.tile_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -215,7 +195,6 @@ class TestTileDoubleGradCheck(unittest.TestCase): class TestExpandV2DoubleGradCheck(unittest.TestCase): - def expand_wrapper(self, x): return paddle.expand(x[0], [4, 12]) @@ -231,15 +210,12 @@ class TestExpandV2DoubleGradCheck(unittest.TestCase): out = paddle.expand(x, new_shape) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.expand_wrapper, [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.expand_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -250,7 +226,6 @@ class TestExpandV2DoubleGradCheck(unittest.TestCase): class TestSqueezeDoubleGradCheck(unittest.TestCase): - def squeeze_warpper(self, x): axes = [0, 2] return paddle.squeeze(x[0], axes) @@ -267,16 +242,12 @@ class TestSqueezeDoubleGradCheck(unittest.TestCase): out = paddle.squeeze(x, axes) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.squeeze_warpper, - [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.squeeze_warpper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -287,7 +258,6 @@ class TestSqueezeDoubleGradCheck(unittest.TestCase): class TestUnsqueezeDoubleGradCheck(unittest.TestCase): - def unsqueeze_wrapper(self, x): axes = [1, 2] return paddle.unsqueeze(x[0], axes) @@ -304,16 +274,12 @@ class TestUnsqueezeDoubleGradCheck(unittest.TestCase): out = paddle.unsqueeze(x, axes) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.unsqueeze_wrapper, - [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.unsqueeze_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -324,9 +290,8 @@ class TestUnsqueezeDoubleGradCheck(unittest.TestCase): class TestClipDoubleGradCheck(unittest.TestCase): - def clip_wrapper(self, x): - return paddle.clip(x[0], min=-1., max=1.) + return paddle.clip(x[0], min=-1.0, max=1.0) @prog_scope() def func(self, place): @@ -335,14 +300,13 @@ class TestClipDoubleGradCheck(unittest.TestCase): x = layers.data('x', x_shape, False, dtype) x.persistable = True - out = paddle.clip(x, min=-1., max=1.) - x_arr = np.random.uniform(-5., 5., x_shape).astype(dtype) + out = paddle.clip(x, min=-1.0, max=1.0) + x_arr = np.random.uniform(-5.0, 5.0, x_shape).astype(dtype) gradient_checker.double_grad_check([x], out, x_init=x_arr, place=place) - gradient_checker.double_grad_check_for_dygraph(self.clip_wrapper, [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.clip_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -353,7 +317,6 @@ class TestClipDoubleGradCheck(unittest.TestCase): class TestTransposeDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): x_shape = [3, 40] @@ -376,7 +339,6 @@ class TestTransposeDoubleGradCheck(unittest.TestCase): class TestTransposeDoubleGradCheckCase1(unittest.TestCase): - @prog_scope() def func(self, place): x_shape = [2, 3, 4, 5] @@ -399,7 +361,6 @@ class TestTransposeDoubleGradCheckCase1(unittest.TestCase): class TestConstantPadDoubleGradCheck(unittest.TestCase): - def pad_wrapper(self, x): pad = [1, 1, 1, 1] return paddle.nn.functional.pad(x[0], pad) @@ -416,15 +377,12 @@ class TestConstantPadDoubleGradCheck(unittest.TestCase): out = paddle.nn.functional.pad(x, pad) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.pad_wrapper, [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.pad_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -435,7 +393,6 @@ class TestConstantPadDoubleGradCheck(unittest.TestCase): class TestConstantPadDoubleGradCheckCase1(TestConstantPadDoubleGradCheck): - @prog_scope() def func(self, place): x_shape = [2, 3, 4, 5] @@ -451,7 +408,6 @@ class TestConstantPadDoubleGradCheckCase1(TestConstantPadDoubleGradCheck): class TestConcatDoubleGradCheck(unittest.TestCase): - def concat_wrapper(self, x): return paddle.concat(x, axis=0) @@ -469,15 +425,16 @@ class TestConcatDoubleGradCheck(unittest.TestCase): x2_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) x1_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x1, x2], - out, - x_init=[x1_arr, x2_arr], - place=place) - gradient_checker.double_grad_check_for_dygraph(self.concat_wrapper, - [x1, x2], - out, - x_init=[x1_arr, x2_arr], - place=place) + gradient_checker.double_grad_check( + [x1, x2], out, x_init=[x1_arr, x2_arr], place=place + ) + gradient_checker.double_grad_check_for_dygraph( + self.concat_wrapper, + [x1, x2], + out, + x_init=[x1_arr, x2_arr], + place=place, + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -488,23 +445,22 @@ class TestConcatDoubleGradCheck(unittest.TestCase): class TestAvgPool2DDoubleGradCheckCase1(unittest.TestCase): - @prog_scope() def func(self, place): - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) input_NCHW.persistable = True y = layers.pool2d(input_NCHW, pool_size=2, pool_type="avg") x_arr = np.random.uniform(-1, 1, [2, 3, 5, 5]).astype(np.float32) - gradient_checker.double_grad_check([input_NCHW], - y, - x_init=x_arr, - place=place, - eps=0.05) + gradient_checker.double_grad_check( + [input_NCHW], y, x_init=x_arr, place=place, eps=0.05 + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -515,36 +471,33 @@ class TestAvgPool2DDoubleGradCheckCase1(unittest.TestCase): class TestAvgPool2DDoubleGradCheckCase2(unittest.TestCase): - def pool2d_wrapper(self, x): - return paddle.nn.functional.avg_pool2d(x[0], - kernel_size=2, - data_format="NHWC") + return paddle.nn.functional.avg_pool2d( + x[0], kernel_size=2, data_format="NHWC" + ) @prog_scope() def func(self, place): - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) input_NHWC.persistable = True - y = paddle.nn.functional.avg_pool2d(input_NHWC, - kernel_size=2, - data_format="NHWC") + y = paddle.nn.functional.avg_pool2d( + input_NHWC, kernel_size=2, data_format="NHWC" + ) x_arr = np.random.uniform(-1, 1, [2, 5, 5, 3]).astype(np.float32) - gradient_checker.double_grad_check([input_NHWC], - y, - x_init=x_arr, - place=place, - eps=0.05) + gradient_checker.double_grad_check( + [input_NHWC], y, x_init=x_arr, place=place, eps=0.05 + ) - gradient_checker.double_grad_check_for_dygraph(self.pool2d_wrapper, - [input_NHWC], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.pool2d_wrapper, [input_NHWC], y, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -555,35 +508,32 @@ class TestAvgPool2DDoubleGradCheckCase2(unittest.TestCase): class TestAvgPool2DDoubleGradCheckCase3(unittest.TestCase): - def pool2d_wrapper(self, x): - return paddle.nn.functional.avg_pool2d(x[0], - kernel_size=2, - padding=[1, 1]) + return paddle.nn.functional.avg_pool2d( + x[0], kernel_size=2, padding=[1, 1] + ) @prog_scope() def func(self, place): - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) input_NCHW.persistable = True - y = paddle.nn.functional.avg_pool2d(input_NCHW, - kernel_size=2, - padding=[1, 1]) + y = paddle.nn.functional.avg_pool2d( + input_NCHW, kernel_size=2, padding=[1, 1] + ) x_arr = np.random.uniform(-1, 1, [2, 3, 5, 5]).astype(np.float32) - gradient_checker.double_grad_check([input_NCHW], - y, - x_init=x_arr, - place=place, - eps=0.05) - gradient_checker.double_grad_check_for_dygraph(self.pool2d_wrapper, - [input_NCHW], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [input_NCHW], y, x_init=x_arr, place=place, eps=0.05 + ) + gradient_checker.double_grad_check_for_dygraph( + self.pool2d_wrapper, [input_NCHW], y, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -594,32 +544,29 @@ class TestAvgPool2DDoubleGradCheckCase3(unittest.TestCase): class TestAvgPool2DDoubleGradCheckCase4(unittest.TestCase): - def pool2d_wrapper(self, x): return paddle.nn.functional.avg_pool2d(x[0], kernel_size=[4, 4]) @prog_scope() def func(self, place): - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) input_NCHW.persistable = True y = layers.pool2d(input_NCHW, pool_size=[4, 4], pool_type="avg") y = paddle.nn.functional.avg_pool2d(input_NCHW, kernel_size=[4, 4]) x_arr = np.random.uniform(-1, 1, [2, 3, 5, 5]).astype(np.float32) - gradient_checker.double_grad_check([input_NCHW], - y, - x_init=x_arr, - place=place, - eps=0.05) - gradient_checker.double_grad_check_for_dygraph(self.pool2d_wrapper, - [input_NCHW], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [input_NCHW], y, x_init=x_arr, place=place, eps=0.05 + ) + gradient_checker.double_grad_check_for_dygraph( + self.pool2d_wrapper, [input_NCHW], y, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_nn_margin_rank_loss.py b/python/paddle/fluid/tests/unittests/test_nn_margin_rank_loss.py index 2f407edc722..2ddfd653a40 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_margin_rank_loss.py +++ b/python/paddle/fluid/tests/unittests/test_nn_margin_rank_loss.py @@ -32,14 +32,13 @@ def calc_margin_rank_loss(x, y, label, margin=0.0, reduction='none'): def create_test_case(margin, reduction): - class MarginRankingLossCls(unittest.TestCase): - def setUp(self): self.x_data = np.random.rand(10, 10).astype("float64") self.y_data = np.random.rand(10, 10).astype("float64") - self.label_data = np.random.choice([-1, 1], - size=[10, 10]).astype("float64") + self.label_data = np.random.choice([-1, 1], size=[10, 10]).astype( + "float64" + ) self.places = [] self.places.append(fluid.CPUPlace()) if core.is_compiled_with_cuda(): @@ -47,59 +46,69 @@ def create_test_case(margin, reduction): def run_static_functional_api(self, place): paddle.enable_static() - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - self.label_data, - margin=margin, - reduction=reduction) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + self.label_data, + margin=margin, + reduction=reduction, + ) with program_guard(Program(), Program()): - x = paddle.static.data(name="x", - shape=[10, 10], - dtype="float64") - y = paddle.static.data(name="y", - shape=[10, 10], - dtype="float64") - label = paddle.static.data(name="label", - shape=[10, 10], - dtype="float64") + x = paddle.static.data( + name="x", shape=[10, 10], dtype="float64" + ) + y = paddle.static.data( + name="y", shape=[10, 10], dtype="float64" + ) + label = paddle.static.data( + name="label", shape=[10, 10], dtype="float64" + ) result = paddle.nn.functional.margin_ranking_loss( - x, y, label, margin, reduction) + x, y, label, margin, reduction + ) exe = paddle.static.Executor(place) - result_numpy, = exe.run(feed={ - "x": self.x_data, - "y": self.y_data, - "label": self.label_data - }, - fetch_list=[result]) + (result_numpy,) = exe.run( + feed={ + "x": self.x_data, + "y": self.y_data, + "label": self.label_data, + }, + fetch_list=[result], + ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) def run_static_api(self, place): paddle.enable_static() - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - self.label_data, - margin=margin, - reduction=reduction) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + self.label_data, + margin=margin, + reduction=reduction, + ) with program_guard(Program(), Program()): - x = paddle.static.data(name="x", - shape=[10, 10], - dtype="float64") - y = paddle.static.data(name="y", - shape=[10, 10], - dtype="float64") - label = paddle.static.data(name="label", - shape=[10, 10], - dtype="float64") + x = paddle.static.data( + name="x", shape=[10, 10], dtype="float64" + ) + y = paddle.static.data( + name="y", shape=[10, 10], dtype="float64" + ) + label = paddle.static.data( + name="label", shape=[10, 10], dtype="float64" + ) margin_rank_loss = paddle.nn.loss.MarginRankingLoss( - margin=margin, reduction=reduction) + margin=margin, reduction=reduction + ) result = margin_rank_loss(x, y, label) exe = paddle.static.Executor(place) - result_numpy, = exe.run(feed={ - "x": self.x_data, - "y": self.y_data, - "label": self.label_data - }, - fetch_list=[result]) + (result_numpy,) = exe.run( + feed={ + "x": self.x_data, + "y": self.y_data, + "label": self.label_data, + }, + fetch_list=[result], + ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) self.assertTrue('loss' in result.name) @@ -110,12 +119,15 @@ def create_test_case(margin, reduction): label = paddle.to_tensor(self.label_data) result = paddle.nn.functional.margin_ranking_loss( - x, y, label, margin, reduction) - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - self.label_data, - margin=margin, - reduction=reduction) + x, y, label, margin, reduction + ) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + self.label_data, + margin=margin, + reduction=reduction, + ) np.testing.assert_allclose(result.numpy(), expected, rtol=1e-05) def run_dynamic_api(self, place): @@ -124,13 +136,16 @@ def create_test_case(margin, reduction): y = paddle.to_tensor(self.y_data) label = paddle.to_tensor(self.label_data) margin_rank_loss = paddle.nn.loss.MarginRankingLoss( - margin=margin, reduction=reduction) + margin=margin, reduction=reduction + ) result = margin_rank_loss(x, y, label) - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - self.label_data, - margin=margin, - reduction=reduction) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + self.label_data, + margin=margin, + reduction=reduction, + ) np.testing.assert_allclose(result.numpy(), expected, rtol=1e-05) def run_dynamic_broadcast_api(self, place): @@ -140,13 +155,16 @@ def create_test_case(margin, reduction): y = paddle.to_tensor(self.y_data) label = paddle.to_tensor(label_data) margin_rank_loss = paddle.nn.loss.MarginRankingLoss( - margin=margin, reduction=reduction) + margin=margin, reduction=reduction + ) result = margin_rank_loss(x, y, label) - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - label_data, - margin=margin, - reduction=reduction) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + label_data, + margin=margin, + reduction=reduction, + ) np.testing.assert_allclose(result.numpy(), expected, rtol=1e-05) def test_case(self): @@ -172,21 +190,22 @@ class MarginRakingLossError(unittest.TestCase): paddle.enable_static() def test_errors(self): - def test_margin_value_error(): margin_rank_loss = paddle.nn.loss.MarginRankingLoss( - margin=0.1, reduction="reduce_mean") + margin=0.1, reduction="reduce_mean" + ) self.assertRaises(ValueError, test_margin_value_error) def test_functional_margin_value_error(): x = paddle.static.data(name="x", shape=[10, 10], dtype="float64") y = paddle.static.data(name="y", shape=[10, 10], dtype="float64") - label = paddle.static.data(name="label", - shape=[10, 10], - dtype="float64") + label = paddle.static.data( + name="label", shape=[10, 10], dtype="float64" + ) result = paddle.nn.functional.margin_ranking_loss( - x, y, label, margin=0.1, reduction="reduction_mean") + x, y, label, margin=0.1, reduction="reduction_mean" + ) self.assertRaises(ValueError, test_functional_margin_value_error) diff --git a/python/paddle/fluid/tests/unittests/test_nn_matmul_v2_grad.py b/python/paddle/fluid/tests/unittests/test_nn_matmul_v2_grad.py index 567ac1404c6..0c3bca6ba0a 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_matmul_v2_grad.py +++ b/python/paddle/fluid/tests/unittests/test_nn_matmul_v2_grad.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestMatmulDoubleGradCheck(unittest.TestCase): - def setUp(self): self.init_test() @@ -40,25 +39,21 @@ class TestMatmulDoubleGradCheck(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -69,7 +64,6 @@ class TestMatmulDoubleGradCheck(unittest.TestCase): class TestMatmulDoubleGradCheckCase1(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 3] self.y_shape = [3, 2] @@ -85,7 +79,6 @@ class TestMatmulDoubleGradCheckCase1(TestMatmulDoubleGradCheck): class TestMatmulDoubleGradCheck2(unittest.TestCase): - def setUp(self): self.init_test() @@ -100,25 +93,21 @@ class TestMatmulDoubleGradCheck2(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -129,7 +118,6 @@ class TestMatmulDoubleGradCheck2(unittest.TestCase): class TestMatmulDoubleGradCheckCase3(unittest.TestCase): - def setUp(self): self.init_test() @@ -144,25 +132,21 @@ class TestMatmulDoubleGradCheckCase3(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -173,7 +157,6 @@ class TestMatmulDoubleGradCheckCase3(unittest.TestCase): class TestMatmulTripleGradCheckDotCase(unittest.TestCase): - def setUp(self): self.init_test() @@ -190,21 +173,19 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) out = paddle.matmul(x, y, self.transpose_x, self.transpose_y, name='out') np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): @@ -216,7 +197,6 @@ def test_grad(self): class TestMatmulTripleGradCheckNormalCase1(unittest.TestCase): - def setUp(self): self.init_test() @@ -231,25 +211,21 @@ class TestMatmulTripleGradCheckNormalCase1(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -260,7 +236,6 @@ class TestMatmulTripleGradCheckNormalCase1(unittest.TestCase): class TestMatmulTripleGradCheckNormalCase2(unittest.TestCase): - def setUp(self): self.init_test() @@ -275,25 +250,21 @@ class TestMatmulTripleGradCheckNormalCase2(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -304,7 +275,6 @@ class TestMatmulTripleGradCheckNormalCase2(unittest.TestCase): class TestMatmulTripleGradCheckNormalCase3(unittest.TestCase): - def setUp(self): self.init_test() @@ -319,25 +289,21 @@ class TestMatmulTripleGradCheckNormalCase3(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -348,7 +314,6 @@ class TestMatmulTripleGradCheckNormalCase3(unittest.TestCase): class TestMatmulTripleGradCheckNormalCase4(unittest.TestCase): - def setUp(self): self.init_test() @@ -363,25 +328,21 @@ class TestMatmulTripleGradCheckNormalCase4(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -392,7 +353,6 @@ class TestMatmulTripleGradCheckNormalCase4(unittest.TestCase): class TestMatmulTripleGradCheckBroadcastCase1(unittest.TestCase): - def setUp(self): self.init_test() @@ -407,25 +367,21 @@ class TestMatmulTripleGradCheckBroadcastCase1(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -436,7 +392,6 @@ class TestMatmulTripleGradCheckBroadcastCase1(unittest.TestCase): class TestMatmulTripleGradCheckBroadcastCase2(unittest.TestCase): - def setUp(self): self.init_test() @@ -451,25 +406,21 @@ class TestMatmulTripleGradCheckBroadcastCase2(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -480,7 +431,6 @@ class TestMatmulTripleGradCheckBroadcastCase2(unittest.TestCase): class TestMatmulTripleGradCheckBroadcastCase3(unittest.TestCase): - def setUp(self): self.init_test() @@ -495,25 +445,21 @@ class TestMatmulTripleGradCheckBroadcastCase3(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -524,7 +470,6 @@ class TestMatmulTripleGradCheckBroadcastCase3(unittest.TestCase): class TestMatmulTripleGradCheckBroadcastCase4(unittest.TestCase): - def setUp(self): self.init_test() @@ -539,25 +484,21 @@ class TestMatmulTripleGradCheckBroadcastCase4(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -568,7 +509,6 @@ class TestMatmulTripleGradCheckBroadcastCase4(unittest.TestCase): class TestMatmulTripleGradCheckBroadcastCase5(unittest.TestCase): - def setUp(self): self.init_test() @@ -583,25 +523,21 @@ class TestMatmulTripleGradCheckBroadcastCase5(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -612,7 +548,6 @@ class TestMatmulTripleGradCheckBroadcastCase5(unittest.TestCase): class TestMatmulTripleGradCheckSpecialCase1(unittest.TestCase): - def setUp(self): self.init_test() @@ -627,25 +562,21 @@ class TestMatmulTripleGradCheckSpecialCase1(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -656,7 +587,6 @@ class TestMatmulTripleGradCheckSpecialCase1(unittest.TestCase): class TestMatmulTripleGradCheckSpecialCase2(unittest.TestCase): - def setUp(self): self.init_test() @@ -671,25 +601,21 @@ class TestMatmulTripleGradCheckSpecialCase2(unittest.TestCase): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_nn_quant_functional_layers.py b/python/paddle/fluid/tests/unittests/test_nn_quant_functional_layers.py index 9ef7e19fbe0..6edc691f23a 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_quant_functional_layers.py +++ b/python/paddle/fluid/tests/unittests/test_nn_quant_functional_layers.py @@ -18,8 +18,7 @@ import paddle class TestFunctionalLayers(unittest.TestCase): - """ - """ + """ """ def setUp(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_nn_sigmoid_op.py b/python/paddle/fluid/tests/unittests/test_nn_sigmoid_op.py index 7af7b96c999..45840d20319 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_sigmoid_op.py +++ b/python/paddle/fluid/tests/unittests/test_nn_sigmoid_op.py @@ -23,7 +23,6 @@ import paddle.nn.functional as functional class TestNNSigmoidAPI(unittest.TestCase): - def setUp(self): self.init_data() @@ -69,7 +68,6 @@ class TestNNSigmoidAPI(unittest.TestCase): class TestNNFunctionalSigmoidAPI(unittest.TestCase): - def setUp(self): self.init_data() diff --git a/python/paddle/fluid/tests/unittests/test_nonzero_api.py b/python/paddle/fluid/tests/unittests/test_nonzero_api.py index cd4d227e823..77363d1d3e7 100644 --- a/python/paddle/fluid/tests/unittests/test_nonzero_api.py +++ b/python/paddle/fluid/tests/unittests/test_nonzero_api.py @@ -20,7 +20,6 @@ from paddle.fluid import Program, program_guard class TestNonZeroAPI(unittest.TestCase): - def test_nonzero_api_as_tuple(self): data = np.array([[True, False], [False, True]]) with program_guard(Program(), Program()): @@ -31,9 +30,9 @@ class TestNonZeroAPI(unittest.TestCase): z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0, 0], [1, 1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -45,9 +44,9 @@ class TestNonZeroAPI(unittest.TestCase): self.assertEqual(len(y), 1) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0], [1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -57,9 +56,9 @@ class TestNonZeroAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[-1, 2]) y = paddle.nonzero(x) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': data}, - fetch_list=[y.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[y.name], return_numpy=False + ) expect_out = np.array([[0, 0], [1, 1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -68,9 +67,9 @@ class TestNonZeroAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[-1]) y = paddle.nonzero(x) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': data}, - fetch_list=[y.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[y.name], return_numpy=False + ) expect_out = np.array([[0], [1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_norm_all.py b/python/paddle/fluid/tests/unittests/test_norm_all.py index f9a62d576c8..17047d349cd 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_all.py +++ b/python/paddle/fluid/tests/unittests/test_norm_all.py @@ -23,18 +23,25 @@ from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph # hack method for test p_norm final state -def p_norm_python_api(x, - p=2.0, - axis=-1, - epsilon=1e-12, - keepdim=False, - as_vector=False): +def p_norm_python_api( + x, p=2.0, axis=-1, epsilon=1e-12, keepdim=False, as_vector=False +): if in_dygraph_mode(): return _C_ops.p_norm(x, p, axis, epsilon, keepdim, as_vector) if _in_legacy_dygraph(): - return _legacy_C_ops.p_norm(x, 'axis', axis, 'porder', float(p), - 'keepdim', keepdim, 'epsilon', epsilon, - 'as_vector', as_vector) + return _legacy_C_ops.p_norm( + x, + 'axis', + axis, + 'porder', + float(p), + 'keepdim', + keepdim, + 'epsilon', + epsilon, + 'as_vector', + as_vector, + ) def p_norm(x, axis, porder, keepdims=False, reduce_all=False): @@ -76,10 +83,13 @@ def p_norm(x, axis, porder, keepdims=False, reduce_all=False): def numpy_frobenius_norm(x, axis=None, keepdims=False): - if isinstance(axis, list): axis = tuple(axis) - if axis is None: x = x.reshape(1, x.size) - r = np.linalg.norm(x, ord='fro', axis=axis, - keepdims=keepdims).astype(x.dtype) + if isinstance(axis, list): + axis = tuple(axis) + if axis is None: + x = x.reshape(1, x.size) + r = np.linalg.norm(x, ord='fro', axis=axis, keepdims=keepdims).astype( + x.dtype + ) return r @@ -88,19 +98,18 @@ def frobenius_norm(x, dim, keep_dim, reduce_all): class TestFrobeniusNormOp(OpTest): - def setUp(self): self.python_api = frobenius_norm self.op_type = "frobenius_norm" self.init_test_case() x = (np.random.random(self.shape) + 1.0).astype(self.dtype) norm = numpy_frobenius_norm(x, self.axis, self.keepdim) - self.reduce_all = (len(self.axis) == len(self.shape)) + self.reduce_all = len(self.axis) == len(self.shape) self.inputs = {'X': x} self.attrs = { 'dim': list(self.axis), 'keep_dim': self.keepdim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.outputs = {'Out': norm} @@ -118,7 +127,6 @@ class TestFrobeniusNormOp(OpTest): class TestFrobeniusNormOp2(TestFrobeniusNormOp): - def init_test_case(self): self.shape = [5, 5, 5] self.axis = (0, 1) @@ -130,7 +138,6 @@ class TestFrobeniusNormOp2(TestFrobeniusNormOp): class TestPnormOp(OpTest): - def setUp(self): self.op_type = "p_norm" self.python_api = p_norm_python_api @@ -143,7 +150,7 @@ class TestPnormOp(OpTest): 'axis': self.axis, 'keepdim': self.keepdim, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } self.outputs = {'Out': norm} self.gradient = self.calc_gradient() @@ -169,7 +176,7 @@ class TestPnormOp(OpTest): 'axis': self.axis, 'keepdim': self.keepdim, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } x = self.inputs["X"] porder = self.attrs["porder"] @@ -180,22 +187,21 @@ class TestPnormOp(OpTest): if porder == 0: grad = np.zeros(x.shape).astype(x.dtype) elif porder in [float("inf"), float("-inf")]: - norm = p_norm(x, - axis=axis, - porder=porder, - keepdims=True, - reduce_all=asvector) + norm = p_norm( + x, axis=axis, porder=porder, keepdims=True, reduce_all=asvector + ) x_abs = np.abs(x) grad = np.sign(x) grad[x_abs != norm] = 0.0 else: - norm = p_norm(x, - axis=axis, - porder=porder, - keepdims=True, - reduce_all=asvector) - grad = np.power(norm, 1 - porder) * np.power( - np.abs(x), porder - 1) * np.sign(x) + norm = p_norm( + x, axis=axis, porder=porder, keepdims=True, reduce_all=asvector + ) + grad = ( + np.power(norm, 1 - porder) + * np.power(np.abs(x), porder - 1) + * np.sign(x) + ) numel = 1 for s in x.shape: @@ -206,7 +212,6 @@ class TestPnormOp(OpTest): class TestPnormOp2(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -221,7 +226,6 @@ class TestPnormOp2(TestPnormOp): class TestPnormOp3(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -236,7 +240,6 @@ class TestPnormOp3(TestPnormOp): class TestPnormOp4(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -251,7 +254,6 @@ class TestPnormOp4(TestPnormOp): class TestPnormOp5(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -266,7 +268,6 @@ class TestPnormOp5(TestPnormOp): class TestPnormOp6(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = -1 @@ -280,10 +281,10 @@ class TestPnormOp6(TestPnormOp): self.check_grad(['X'], 'Out', user_defined_grads=self.gradient) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPnormOpFP16(TestPnormOp): - def init_test_case(self): self.shape = [2, 3, 4, 5] self.axis = 1 @@ -301,15 +302,15 @@ class TestPnormOpFP16(TestPnormOp): def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X'], - 'Out', - user_defined_grads=self.gradient) + self.check_grad_with_place( + place, ['X'], 'Out', user_defined_grads=self.gradient + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPnormOpFP161(TestPnormOpFP16): - def init_test_case(self): self.shape = [2, 3, 4, 5] self.axis = -1 @@ -320,17 +321,18 @@ class TestPnormOpFP161(TestPnormOpFP16): self.asvector = True -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPnormBF16Op(OpTest): - def setUp(self): self.op_type = "p_norm" self.python_api = p_norm_python_api self.init_test_case() self.x = (np.random.random(self.shape) + 0.5).astype(np.float32) - self.norm = p_norm(self.x, self.axis, self.porder, self.keepdim, - self.asvector) + self.norm = p_norm( + self.x, self.axis, self.porder, self.keepdim, self.asvector + ) self.gradient = self.calc_gradient() self.inputs = {'X': convert_float_to_uint16(self.x)} self.attrs = { @@ -338,7 +340,7 @@ class TestPnormBF16Op(OpTest): 'axis': self.axis, 'keepdim': self.keepdim, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } self.outputs = {'Out': convert_float_to_uint16(self.norm)} @@ -348,10 +350,13 @@ class TestPnormBF16Op(OpTest): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad_with_place( + place, + ['X'], + 'Out', + user_defined_grads=self.gradient, + check_eager=True, + ) def init_test_case(self): self.shape = [2, 3, 4, 5] @@ -368,7 +373,7 @@ class TestPnormBF16Op(OpTest): 'axis': self.axis, 'keepdim': self.keepdim, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } x = self.x porder = self.attrs["porder"] @@ -379,22 +384,21 @@ class TestPnormBF16Op(OpTest): if porder == 0: grad = np.zeros(x.shape).astype(x.dtype) elif porder in [float("inf"), float("-inf")]: - norm = p_norm(x, - axis=axis, - porder=porder, - keepdims=True, - reduce_all=asvector) + norm = p_norm( + x, axis=axis, porder=porder, keepdims=True, reduce_all=asvector + ) x_abs = np.abs(x) grad = np.sign(x) grad[x_abs != norm] = 0.0 else: - norm = p_norm(x, - axis=axis, - porder=porder, - keepdims=True, - reduce_all=asvector) - grad = np.power(norm, 1 - porder) * np.power( - np.abs(x), porder - 1) * np.sign(x) + norm = p_norm( + x, axis=axis, porder=porder, keepdims=True, reduce_all=asvector + ) + grad = ( + np.power(norm, 1 - porder) + * np.power(np.abs(x), porder - 1) + * np.sign(x) + ) numel = 1 for s in x.shape: @@ -411,15 +415,19 @@ def run_fro(self, p, axis, shape_x, dtype, keep_dim, check_dim=False): place = fluid.CPUPlace() exe = fluid.Executor(place) np_input = (np.random.rand(*shape_x) + 1.0).astype(dtype) - expected_result = numpy_frobenius_norm(np_input, - axis=axis, - keepdims=keep_dim) - result, = exe.run(feed={"X": np_input}, fetch_list=[out]) + expected_result = numpy_frobenius_norm( + np_input, axis=axis, keepdims=keep_dim + ) + (result,) = exe.run(feed={"X": np_input}, fetch_list=[out]) self.assertEqual((np.abs(result - expected_result) < 1e-6).all(), True) if keep_dim and check_dim: self.assertEqual( - (np.abs(np.array(result.shape) - np.array(expected_result.shape)) < - 1e-6).all(), True) + ( + np.abs(np.array(result.shape) - np.array(expected_result.shape)) + < 1e-6 + ).all(), + True, + ) def run_pnorm(self, p, axis, shape_x, dtype, keep_dim, check_dim=False): @@ -429,16 +437,19 @@ def run_pnorm(self, p, axis, shape_x, dtype, keep_dim, check_dim=False): place = fluid.CPUPlace() exe = fluid.Executor(place) np_input = (np.random.rand(*shape_x) + 1.0).astype(dtype) - expected_result = p_norm(np_input, - porder=p, - axis=axis, - keepdims=keep_dim).astype(dtype) - result, = exe.run(feed={"X": np_input}, fetch_list=[out]) + expected_result = p_norm( + np_input, porder=p, axis=axis, keepdims=keep_dim + ).astype(dtype) + (result,) = exe.run(feed={"X": np_input}, fetch_list=[out]) self.assertEqual((np.abs(result - expected_result) < 1e-6).all(), True) if keep_dim and check_dim: self.assertEqual( - (np.abs(np.array(result.shape) - np.array(expected_result.shape)) < - 1e-6).all(), True) + ( + np.abs(np.array(result.shape) - np.array(expected_result.shape)) + < 1e-6 + ).all(), + True, + ) def run_graph(self, p, axis, shape_x, dtype): @@ -447,7 +458,7 @@ def run_graph(self, p, axis, shape_x, dtype): np_input = np.arange(24).astype('float32') - 12 np_input = np_input.reshape(shape) x = paddle.to_tensor(np_input) - #[[[-12. -11. -10. -9.] [ -8. -7. -6. -5.] [ -4. -3. -2. -1.]] + # [[[-12. -11. -10. -9.] [ -8. -7. -6. -5.] [ -4. -3. -2. -1.]] # [[ 0. 1. 2. 3.] [ 4. 5. 6. 7.] [ 8. 9. 10. 11.]]] out_pnorm = paddle.norm(x, p=2, axis=-1) @@ -458,135 +469,166 @@ def run_graph(self, p, axis, shape_x, dtype): # compute 2-order norm along [0,1] dimension. out_pnorm = paddle.norm(x, p=2, axis=[0, 1]) out_pnorm = paddle.norm(x, p=2) - #out_pnorm = [17.43559577 16.91153453 16.73320053 16.91153453] + # out_pnorm = [17.43559577 16.91153453 16.73320053 16.91153453] # compute inf-order norm out_pnorm = paddle.norm(x, p=np.inf) - #out_pnorm = [12.] + # out_pnorm = [12.] out_pnorm = paddle.norm(x, p=np.inf, axis=0) - #out_pnorm = [[0. 1. 2. 3.] [4. 5. 6. 5.] [4. 3. 2. 1.]] + # out_pnorm = [[0. 1. 2. 3.] [4. 5. 6. 5.] [4. 3. 2. 1.]] # compute -inf-order norm out_pnorm = paddle.norm(x, p=-np.inf) - #out_pnorm = [0.] + # out_pnorm = [0.] out_pnorm = paddle.norm(x, p=-np.inf, axis=0) # out_fro = [17.43559577 16.91153453 16.73320053 16.91153453] paddle.enable_static() class API_NormTest(unittest.TestCase): - def test_basic(self): keep_dims = {False, True} for keep in keep_dims: - run_fro(self, - p='fro', - axis=None, - shape_x=[2, 3, 4], - dtype="float32", - keep_dim=keep) - run_fro(self, - p='fro', - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=2, - axis=None, - shape_x=[3, 4], - dtype="float32", - keep_dim=keep) - run_pnorm(self, - p=2, - axis=1, - shape_x=[3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=np.inf, - axis=0, - shape_x=[2, 3, 4], - dtype="float32", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=np.inf, - axis=None, - shape_x=[2, 3, 4], - dtype="float32", - keep_dim=keep) - run_pnorm(self, - p=-np.inf, - axis=0, - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=-np.inf, - axis=None, - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep) - run_pnorm(self, - p=0, - axis=1, - shape_x=[3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - - run_pnorm(self, - p=1, - axis=1, - shape_x=[3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=0, - axis=None, - shape_x=[3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=2, - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=2, - axis=-1, - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=1, - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=np.inf, - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=-np.inf, - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) + run_fro( + self, + p='fro', + axis=None, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep, + ) + run_fro( + self, + p='fro', + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=2, + axis=None, + shape_x=[3, 4], + dtype="float32", + keep_dim=keep, + ) + run_pnorm( + self, + p=2, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=np.inf, + axis=0, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=np.inf, + axis=None, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep, + ) + run_pnorm( + self, + p=-np.inf, + axis=0, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=-np.inf, + axis=None, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + ) + run_pnorm( + self, + p=0, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + + run_pnorm( + self, + p=1, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=0, + axis=None, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=2, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=2, + axis=-1, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=1, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=np.inf, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=-np.inf, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) def test_dygraph(self): run_graph(self, p='fro', axis=None, shape_x=[2, 3, 4], dtype="float32") @@ -609,8 +651,9 @@ class API_NormTest(unittest.TestCase): self.assertRaises(TypeError, err_dtype, "fro", [2, 2], "int64") self.assertRaises(ValueError, paddle.norm, "inf", [2], "int64") out = fluid.data(name="out", shape=[1], dtype="int64") - self.assertRaises(TypeError, err_dtype, "fro", [2, 2], "float64", - out) + self.assertRaises( + TypeError, err_dtype, "fro", [2, 2], "float64", out + ) self.assertRaises(TypeError, err_dtype, 2, [10], "int64") self.assertRaises(TypeError, err_dtype, 2, [10], "float64", out) @@ -620,11 +663,9 @@ class API_NormTest(unittest.TestCase): self.assertRaises(ValueError, paddle.norm, data, p=[1], axis=-1) self.assertRaises(ValueError, paddle.norm, 0, [1, 0], "float64") data = fluid.data(name="data_3d", shape=[2, 2, 2], dtype="float64") - self.assertRaises(ValueError, - paddle.norm, - data, - p='unspport', - axis=[-3, -2, -1]) + self.assertRaises( + ValueError, paddle.norm, data, p='unspport', axis=[-3, -2, -1] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_norm_nn_grad.py b/python/paddle/fluid/tests/unittests/test_norm_nn_grad.py index 28622d44d18..26dea91aecc 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_norm_nn_grad.py @@ -25,7 +25,6 @@ from decorator_helper import prog_scope class TestInstanceNormDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): prog = fluid.Program() @@ -38,12 +37,9 @@ class TestInstanceNormDoubleGradCheck(unittest.TestCase): x = layers.create_parameter(dtype=dtype, shape=shape, name='x') z = fluid.layers.instance_norm(input=x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -55,8 +51,8 @@ class TestInstanceNormDoubleGradCheck(unittest.TestCase): class TestInstanceNormDoubleGradCheckWithoutParamBias( - TestInstanceNormDoubleGradCheck): - + TestInstanceNormDoubleGradCheck +): @prog_scope() def func(self, place): prog = fluid.Program() @@ -67,20 +63,16 @@ class TestInstanceNormDoubleGradCheckWithoutParamBias( eps = 0.005 atol = 1e-4 x = layers.create_parameter(dtype=dtype, shape=shape, name='x') - z = fluid.layers.instance_norm(input=x, - param_attr=False, - bias_attr=False) + z = fluid.layers.instance_norm( + input=x, param_attr=False, bias_attr=False + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) class TestInstanceNormDoubleGradEagerCheck(unittest.TestCase): - def instance_norm_wrapper(self, x): return paddle.nn.functional.instance_norm(x[0]) @@ -97,19 +89,18 @@ class TestInstanceNormDoubleGradEagerCheck(unittest.TestCase): z = paddle.nn.functional.instance_norm(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) # check for static mode - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) # check for eager mode gradient_checker.double_grad_check_for_dygraph( - self.instance_norm_wrapper, [x], + self.instance_norm_wrapper, + [x], z, x_init=x_arr, atol=atol, - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -121,8 +112,8 @@ class TestInstanceNormDoubleGradEagerCheck(unittest.TestCase): class TestInstanceNormDoubleGradEagerCheckWithParams( - TestInstanceNormDoubleGradEagerCheck): - + TestInstanceNormDoubleGradEagerCheck +): def instance_norm_wrapper(self, x): instance_norm = paddle.nn.InstanceNorm2D(3) return instance_norm(x[0]) @@ -140,23 +131,21 @@ class TestInstanceNormDoubleGradEagerCheckWithParams( z = paddle.nn.InstanceNorm2D(3)(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) # check for static mode - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) # check for eager mode gradient_checker.double_grad_check_for_dygraph( - self.instance_norm_wrapper, [x], + self.instance_norm_wrapper, + [x], z, x_init=x_arr, atol=atol, - place=place) + place=place, + ) class TestBatchNormDoubleGradCheck(unittest.TestCase): - def setUp(self): self.init_test() @@ -170,7 +159,8 @@ class TestBatchNormDoubleGradCheck(unittest.TestCase): batch_norm = paddle.nn.BatchNorm2D( self.shape[self.channel_index], data_format=self.data_layout, - use_global_stats=self.use_global_stats) + use_global_stats=self.use_global_stats, + ) return batch_norm(x[0]) @prog_scope() @@ -182,22 +172,23 @@ class TestBatchNormDoubleGradCheck(unittest.TestCase): eps = 0.005 atol = 1e-4 x = layers.create_parameter(dtype=dtype, shape=self.shape, name='x') - z = fluid.layers.batch_norm(input=x, - data_layout=self.data_layout, - use_global_stats=self.use_global_stats) + z = fluid.layers.batch_norm( + input=x, + data_layout=self.data_layout, + use_global_stats=self.use_global_stats, + ) x_arr = np.random.uniform(-1, 1, self.shape).astype(dtype) - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.batch_norm_wrapper, [x], + self.batch_norm_wrapper, + [x], z, x_init=x_arr, atol=atol, - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -209,7 +200,6 @@ class TestBatchNormDoubleGradCheck(unittest.TestCase): class TestBatchNormDoubleGradCheckCase1(TestBatchNormDoubleGradCheck): - def init_test(self): self.data_layout = 'NHWC' self.use_global_stats = False @@ -218,7 +208,6 @@ class TestBatchNormDoubleGradCheckCase1(TestBatchNormDoubleGradCheck): class TestBatchNormDoubleGradCheckCase2(TestBatchNormDoubleGradCheck): - def init_test(self): self.data_layout = 'NCHW' self.use_global_stats = True @@ -227,7 +216,6 @@ class TestBatchNormDoubleGradCheckCase2(TestBatchNormDoubleGradCheck): class TestBatchNormDoubleGradCheckCase3(TestBatchNormDoubleGradCheck): - def init_test(self): self.data_layout = 'NHWC' self.use_global_stats = True @@ -236,7 +224,6 @@ class TestBatchNormDoubleGradCheckCase3(TestBatchNormDoubleGradCheck): class TestBatchNormDoubleGradCheckCase4(TestBatchNormDoubleGradCheck): - def init_test(self): self.data_layout = 'NCHW' self.use_global_stats = False @@ -247,12 +234,12 @@ class TestBatchNormDoubleGradCheckCase4(TestBatchNormDoubleGradCheck): batch_norm = paddle.nn.BatchNorm3D( self.shape[self.channel_index], data_format=self.data_layout, - use_global_stats=self.use_global_stats) + use_global_stats=self.use_global_stats, + ) return batch_norm(x[0]) class TestBatchNormDoubleGradCheckCase5(TestBatchNormDoubleGradCheck): - @prog_scope() def func(self, place): prog = fluid.Program() @@ -261,25 +248,30 @@ class TestBatchNormDoubleGradCheckCase5(TestBatchNormDoubleGradCheck): dtype = "float32" eps = 0.005 atol = 2e-4 - chn = self.shape[1] if self.data_layout == 'NCHW' else self.shape[-1] + chn = ( + self.shape[1] if self.data_layout == 'NCHW' else self.shape[-1] + ) x = layers.create_parameter(dtype=dtype, shape=self.shape, name='x') - z = fluid.layers.batch_norm(input=x, - data_layout=self.data_layout, - use_global_stats=self.use_global_stats) + z = fluid.layers.batch_norm( + input=x, + data_layout=self.data_layout, + use_global_stats=self.use_global_stats, + ) x_arr = np.random.uniform(-1, 1, self.shape).astype(dtype) w, b = prog.global_block().all_parameters()[1:3] w_arr = np.ones(chn).astype(dtype) b_arr = np.zeros(chn).astype(dtype) - gradient_checker.double_grad_check([x, w, b], - z, - x_init=[x_arr, w_arr, b_arr], - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, w, b], + z, + x_init=[x_arr, w_arr, b_arr], + atol=atol, + place=place, + eps=eps, + ) class TestBatchNormDoubleGradCheckCase6(TestBatchNormDoubleGradCheckCase5): - def init_test(self): self.data_layout = 'NCHW' self.use_global_stats = True diff --git a/python/paddle/fluid/tests/unittests/test_norm_op.py b/python/paddle/fluid/tests/unittests/test_norm_op.py index a5c643f4612..d5b360cb729 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_norm_op.py @@ -28,7 +28,6 @@ def l2_norm(x, axis, epsilon): class TestNormOp(OpTest): - def setUp(self): self.op_type = "norm" self.python_api = paddle.fluid.layers.l2_normalize @@ -56,7 +55,6 @@ class TestNormOp(OpTest): class TestNormOp2(TestNormOp): - def init_test_case(self): self.shape = [5, 3, 9, 7] self.axis = 0 @@ -64,17 +62,17 @@ class TestNormOp2(TestNormOp): class TestNormOp3(TestNormOp): - def init_test_case(self): self.shape = [5, 3, 2, 7] self.axis = -1 self.epsilon = 1e-8 -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestNormOp4(TestNormOp): - def init_test_case(self): self.shape = [128, 1024, 14, 14] self.axis = 2 @@ -84,10 +82,11 @@ class TestNormOp4(TestNormOp): pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestNormOp5(TestNormOp): - def init_test_case(self): self.shape = [2048, 2048] self.axis = 1 @@ -98,7 +97,6 @@ class TestNormOp5(TestNormOp): class TestNormOp6(TestNormOp): - def init_dtype(self): self.dtype = "float32" @@ -106,10 +104,10 @@ class TestNormOp6(TestNormOp): self.check_grad(['X'], 'Out', max_relative_error=0.008) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNormOp7(TestNormOp): - def init_dtype(self): self.dtype = "float16" @@ -117,14 +115,13 @@ class TestNormOp7(TestNormOp): self.check_output_with_place(fluid.core.CUDAPlace(0), atol=5e-2) def test_check_grad(self): - self.check_grad_with_place(fluid.core.CUDAPlace(0), ['X'], - 'Out', - max_relative_error=0.05) + self.check_grad_with_place( + fluid.core.CUDAPlace(0), ['X'], 'Out', max_relative_error=0.05 + ) @skip_check_grad_ci(reason="skip check grad for test mode.") class TestNormTestOp(OpTest): - def setUp(self): self.op_type = "norm" self.init_test_case() @@ -134,7 +131,7 @@ class TestNormTestOp(OpTest): self.attrs = { 'epsilon': self.epsilon, 'axis': self.axis, - 'is_test': True + 'is_test': True, } self.outputs = {'Out': y} @@ -151,7 +148,6 @@ class TestNormTestOp(OpTest): class API_NormTest(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): @@ -164,5 +160,6 @@ class API_NormTest(unittest.TestCase): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_normal.py b/python/paddle/fluid/tests/unittests/test_normal.py index 3458bbdf7fd..d24f5f02d2b 100644 --- a/python/paddle/fluid/tests/unittests/test_normal.py +++ b/python/paddle/fluid/tests/unittests/test_normal.py @@ -22,7 +22,6 @@ paddle.seed(10) class TestNormalAPI(unittest.TestCase): - def setUp(self): self.mean = 1.0 self.std = 0.0 @@ -30,9 +29,11 @@ class TestNormalAPI(unittest.TestCase): self.repeat_num = 2000 self.set_attrs() self.dtype = self.get_dtype() - self.place=paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def set_attrs(self): self.shape = [8, 12] @@ -59,27 +60,32 @@ class TestNormalAPI(unittest.TestCase): ret_all_shape = copy.deepcopy(shape) ret_all_shape.insert(0, self.repeat_num) ret_all = np.zeros(ret_all_shape, self.dtype) - if isinstance(self.mean, np.ndarray) \ - and isinstance(self.std, np.ndarray): + if isinstance(self.mean, np.ndarray) and isinstance( + self.std, np.ndarray + ): with paddle.static.program_guard(paddle.static.Program()): - mean = paddle.fluid.data('Mean', self.mean.shape, - self.mean.dtype) + mean = paddle.fluid.data( + 'Mean', self.mean.shape, self.mean.dtype + ) std = paddle.fluid.data('Std', self.std.shape, self.std.dtype) out = paddle.normal(mean, std, self.shape) exe = paddle.static.Executor(self.place) for i in range(self.repeat_num): - ret = exe.run(feed={ - 'Mean': self.mean, - 'Std': self.std.reshape(shape) - }, - fetch_list=[out]) + ret = exe.run( + feed={ + 'Mean': self.mean, + 'Std': self.std.reshape(shape), + }, + fetch_list=[out], + ) ret_all[i] = ret[0] return ret_all elif isinstance(self.mean, np.ndarray): with paddle.static.program_guard(paddle.static.Program()): - mean = paddle.fluid.data('Mean', self.mean.shape, - self.mean.dtype) + mean = paddle.fluid.data( + 'Mean', self.mean.shape, self.mean.dtype + ) out = paddle.normal(mean, self.std, self.shape) exe = paddle.static.Executor(self.place) @@ -114,10 +120,16 @@ class TestNormalAPI(unittest.TestCase): ret_all_shape.insert(0, self.repeat_num) ret_all = np.zeros(ret_all_shape, self.dtype) - mean = paddle.to_tensor(self.mean) \ - if isinstance(self.mean, np.ndarray) else self.mean - std = paddle.to_tensor(self.std) \ - if isinstance(self.std, np.ndarray) else self.std + mean = ( + paddle.to_tensor(self.mean) + if isinstance(self.mean, np.ndarray) + else self.mean + ) + std = ( + paddle.to_tensor(self.std) + if isinstance(self.std, np.ndarray) + else self.std + ) for i in range(self.repeat_num): out = paddle.normal(mean, std, self.shape) ret_all[i] = out.numpy() @@ -134,42 +146,43 @@ class TestNormalAPI(unittest.TestCase): ret = ret.flatten().reshape([self.repeat_num, -1]) mean = np.mean(ret, axis=0) std = np.std(ret, axis=0) - mean_ref=self.mean.flatten() \ - if isinstance(self.mean, np.ndarray) else self.mean - std_ref=self.std.flatten() \ - if isinstance(self.std, np.ndarray) else self.std + mean_ref = ( + self.mean.flatten() + if isinstance(self.mean, np.ndarray) + else self.mean + ) + std_ref = ( + self.std.flatten() + if isinstance(self.std, np.ndarray) + else self.std + ) np.testing.assert_allclose(mean_ref, mean, rtol=0.2, atol=0.2) np.testing.assert_allclose(std_ref, std, rtol=0.2, atol=0.2) class TestNormalAPI_mean_is_tensor(TestNormalAPI): - def set_attrs(self): self.mean = np.random.uniform(-2, -1, [2, 3, 4, 5]).astype('float64') class TestNormalAPI_std_is_tensor(TestNormalAPI): - def set_attrs(self): self.std = np.random.uniform(0.7, 1, [2, 3, 17]).astype('float64') class TestNormalAPI_mean_std_are_tensor(TestNormalAPI): - def set_attrs(self): self.mean = np.random.uniform(1, 2, [1, 100]).astype('float64') self.std = np.random.uniform(0.5, 1, [1, 100]).astype('float64') class TestNormalAPI_mean_std_are_tensor_with_different_dtype(TestNormalAPI): - def set_attrs(self): self.mean = np.random.uniform(1, 2, [100]).astype('float64') self.std = np.random.uniform(1, 2, [100]).astype('float32') class TestNormalAlias(unittest.TestCase): - def test_alias(self): paddle.disable_static() shape = [1, 2, 3] @@ -180,7 +193,6 @@ class TestNormalAlias(unittest.TestCase): class TestNormalErrors(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): mean = [1, 2, 3] diff --git a/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py b/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py index e35e0283332..5a5b4f4eeb0 100644 --- a/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py +++ b/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py @@ -22,18 +22,19 @@ class TestNormalization(unittest.TestCase): data_desc = {"name": "input", "shape": (2, 3, 7)} def gen_random_input(self): - """Generate random input data. - """ - self.data = np.random.random( - size=self.data_desc["shape"]).astype("float32") + """Generate random input data.""" + self.data = np.random.random(size=self.data_desc["shape"]).astype( + "float32" + ) def set_program(self, axis, epsilon): - """Build the test program. - """ - data = fluid.layers.data(name=self.data_desc["name"], - shape=self.data_desc["shape"], - dtype="float32", - append_batch_size=False) + """Build the test program.""" + data = fluid.layers.data( + name=self.data_desc["name"], + shape=self.data_desc["shape"], + dtype="float32", + append_batch_size=False, + ) data.stop_gradient = False l2_norm = fluid.layers.l2_normalize(x=data, axis=axis, epsilon=epsilon) out = fluid.layers.reduce_sum(l2_norm, dim=None) @@ -42,8 +43,7 @@ class TestNormalization(unittest.TestCase): self.fetch_list = [l2_norm] def run_program(self): - """Run the test program. - """ + """Run the test program.""" places = [core.CPUPlace()] if core.is_compiled_with_cuda(): places.append(core.CUDAPlace(0)) @@ -52,33 +52,33 @@ class TestNormalization(unittest.TestCase): self.set_inputs(place) exe = fluid.Executor(place) - output, = exe.run(fluid.default_main_program(), - feed=self.inputs, - fetch_list=self.fetch_list, - return_numpy=True) + (output,) = exe.run( + fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=True, + ) self.op_output = output def set_inputs(self, place): - """Set the randomly generated data to the test program. - """ + """Set the randomly generated data to the test program.""" self.inputs = {} tensor = fluid.Tensor() tensor.set(self.data, place) self.inputs[self.data_desc["name"]] = tensor def l2_normalize(self, data, axis, epsilon): - """ Compute the groundtruth. - """ + """Compute the groundtruth.""" output = data / np.broadcast_to( np.sqrt(np.sum(np.square(data), axis=axis, keepdims=True)), - data.shape) + data.shape, + ) return output def test_l2_normalize(self): - """ Test the python wrapper for l2_normalize. - """ + """Test the python wrapper for l2_normalize.""" axis = 1 - #TODO(caoying) epsilon is not supported due to lack of a maximum_op. + # TODO(caoying) epsilon is not supported due to lack of a maximum_op. epsilon = 1e-6 self.gen_random_input() @@ -89,10 +89,9 @@ class TestNormalization(unittest.TestCase): expect_output = self.l2_normalize(self.data, axis, epsilon) # check output - np.testing.assert_allclose(self.op_output, - expect_output, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + self.op_output, expect_output, rtol=1e-05, atol=0.001 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_normalize.py b/python/paddle/fluid/tests/unittests/test_normalize.py index bb8b4881d3f..e4fed2904e9 100644 --- a/python/paddle/fluid/tests/unittests/test_normalize.py +++ b/python/paddle/fluid/tests/unittests/test_normalize.py @@ -28,7 +28,6 @@ def p_normalize(x, axis=1, p=2, epsilon=1e-12, keepdims=True): class TestNNFunctionalNormalize(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10)).astype(np.float32) self.input_np2 = np.array([0.0, 0.0]).astype(np.float32) @@ -66,11 +65,10 @@ class TestNNFunctionalNormalize(unittest.TestCase): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - static_result = exe.run(feed={ - "input": self.input_np, - "input2": self.input_np2 - }, - fetch_list=[result0, result1, result2, result4]) + static_result = exe.run( + feed={"input": self.input_np, "input2": self.input_np2}, + fetch_list=[result0, result1, result2, result4], + ) np.testing.assert_allclose(static_result[0], self.expected0, rtol=1e-05) np.testing.assert_allclose(static_result[1], self.expected1, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_npair_loss_op.py b/python/paddle/fluid/tests/unittests/test_npair_loss_op.py index 557c5bac866..f90b3f4cdff 100644 --- a/python/paddle/fluid/tests/unittests/test_npair_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_npair_loss_op.py @@ -20,13 +20,13 @@ from paddle.fluid import Program, program_guard def npairloss(anchor, positive, labels, l2_reg=0.002): - def softmax_cross_entropy_with_logits(logits, labels): logits = np.exp(logits) logits = logits / np.sum(logits, axis=1).reshape(-1, 1) - return np.mean(-np.sum(labels * np.log(logits), axis=1), - dtype=np.float32) + return np.mean( + -np.sum(labels * np.log(logits), axis=1), dtype=np.float32 + ) batch_size = labels.shape[0] @@ -35,27 +35,26 @@ def npairloss(anchor, positive, labels, l2_reg=0.002): labels = labels / np.sum(labels, axis=1, keepdims=True) l2loss = np.mean(np.sum(np.power(anchor, 2), 1)) + np.mean( - np.sum(np.power(positive, 2), 1)) + np.sum(np.power(positive, 2), 1) + ) l2loss = (l2loss * 0.25 * l2_reg).astype(np.float32) similarity_matrix = np.matmul(anchor, positive.transpose()) celoss = np.mean( - softmax_cross_entropy_with_logits(similarity_matrix, labels)) + softmax_cross_entropy_with_logits(similarity_matrix, labels) + ) return l2loss + celoss class TestNpairLossOp(unittest.TestCase): - def setUp(self): self.dtype = np.float32 def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) def test_npair_loss(self): reg_lambda = 0.002 @@ -65,81 +64,97 @@ class TestNpairLossOp(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - embeddings_anchor = np.random.rand(num_data, - feat_dim).astype(np.float32) - embeddings_positive = np.random.rand(num_data, - feat_dim).astype(np.float32) - row_labels = np.random.randint(0, num_classes, - size=(num_data)).astype(np.float32) - out_loss = npairloss(embeddings_anchor, - embeddings_positive, - row_labels, - l2_reg=reg_lambda) - - anc = fluid.layers.data(dtype='float32', - name='anc', - shape=embeddings_anchor.shape, - append_batch_size=False) - pos = fluid.layers.data(dtype='float32', - name='pos', - shape=embeddings_positive.shape, - append_batch_size=False) - lab = fluid.layers.data(dtype='float32', - name='lab', - shape=row_labels.shape, - append_batch_size=False) - - npair_loss_op = fluid.layers.npair_loss(anchor=anc, - positive=pos, - labels=lab, - l2_reg=reg_lambda) - out_tensor = exe.run(feed={ - 'anc': embeddings_anchor, - 'pos': embeddings_positive, - 'lab': row_labels - }, - fetch_list=[npair_loss_op.name]) - - self.__assert_close(out_tensor, - out_loss, - "inference output are different at " + str(place) + - ", " + str(np.dtype('float32')) + - str(np.array(out_tensor)) + str(out_loss), - atol=1e-3) + embeddings_anchor = np.random.rand(num_data, feat_dim).astype( + np.float32 + ) + embeddings_positive = np.random.rand(num_data, feat_dim).astype( + np.float32 + ) + row_labels = np.random.randint(0, num_classes, size=(num_data)).astype( + np.float32 + ) + out_loss = npairloss( + embeddings_anchor, + embeddings_positive, + row_labels, + l2_reg=reg_lambda, + ) + + anc = fluid.layers.data( + dtype='float32', + name='anc', + shape=embeddings_anchor.shape, + append_batch_size=False, + ) + pos = fluid.layers.data( + dtype='float32', + name='pos', + shape=embeddings_positive.shape, + append_batch_size=False, + ) + lab = fluid.layers.data( + dtype='float32', + name='lab', + shape=row_labels.shape, + append_batch_size=False, + ) + + npair_loss_op = fluid.layers.npair_loss( + anchor=anc, positive=pos, labels=lab, l2_reg=reg_lambda + ) + out_tensor = exe.run( + feed={ + 'anc': embeddings_anchor, + 'pos': embeddings_positive, + 'lab': row_labels, + }, + fetch_list=[npair_loss_op.name], + ) + + self.__assert_close( + out_tensor, + out_loss, + "inference output are different at " + + str(place) + + ", " + + str(np.dtype('float32')) + + str(np.array(out_tensor)) + + str(out_loss), + atol=1e-3, + ) class TestNpairLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): anchor_np = np.random.random((2, 4)).astype("float32") positive_np = np.random.random((2, 4)).astype("float32") labels_np = np.random.random((2)).astype("float32") - anchor_data = fluid.data(name='anchor', - shape=[2, 4], - dtype='float32') - positive_data = fluid.data(name='positive', - shape=[2, 4], - dtype='float32') + anchor_data = fluid.data( + name='anchor', shape=[2, 4], dtype='float32' + ) + positive_data = fluid.data( + name='positive', shape=[2, 4], dtype='float32' + ) labels_data = fluid.data(name='labels', shape=[2], dtype='float32') def test_anchor_Variable(): # the anchor type must be Variable - fluid.layers.npair_loss(anchor=anchor_np, - positive=positive_data, - labels=labels_data) + fluid.layers.npair_loss( + anchor=anchor_np, positive=positive_data, labels=labels_data + ) def test_positive_Variable(): # the positive type must be Variable - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_np, - labels=labels_data) + fluid.layers.npair_loss( + anchor=anchor_data, positive=positive_np, labels=labels_data + ) def test_labels_Variable(): # the labels type must be Variable - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_data, - labels=labels_np) + fluid.layers.npair_loss( + anchor=anchor_data, positive=positive_data, labels=labels_np + ) self.assertRaises(TypeError, test_anchor_Variable) self.assertRaises(TypeError, test_positive_Variable) @@ -147,30 +162,34 @@ class TestNpairLossOpError(unittest.TestCase): def test_anchor_type(): # dtype must be float32 or float64 - anchor_data1 = fluid.data(name='anchor1', - shape=[2, 4], - dtype='int32') - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_data, - labels=labels_np) + anchor_data1 = fluid.data( + name='anchor1', shape=[2, 4], dtype='int32' + ) + fluid.layers.npair_loss( + anchor=anchor_data, positive=positive_data, labels=labels_np + ) def test_positive_type(): # dtype must be float32 or float64 - positive_data1 = fluid.data(name='positive1', - shape=[2, 4], - dtype='int32') - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_data1, - labels=labels_np) + positive_data1 = fluid.data( + name='positive1', shape=[2, 4], dtype='int32' + ) + fluid.layers.npair_loss( + anchor=anchor_data, + positive=positive_data1, + labels=labels_np, + ) def test_labels_type(): # dtype must be float32 or float64 - labels_data1 = fluid.data(name='labels1', - shape=[2], - dtype='int32') - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_data, - labels=labels_data1) + labels_data1 = fluid.data( + name='labels1', shape=[2], dtype='int32' + ) + fluid.layers.npair_loss( + anchor=anchor_data, + positive=positive_data, + labels=labels_data1, + ) self.assertRaises(TypeError, test_anchor_type) self.assertRaises(TypeError, test_positive_type) diff --git a/python/paddle/fluid/tests/unittests/test_number_count_op.py b/python/paddle/fluid/tests/unittests/test_number_count_op.py index 58e24e6692a..a31fb1a5978 100644 --- a/python/paddle/fluid/tests/unittests/test_number_count_op.py +++ b/python/paddle/fluid/tests/unittests/test_number_count_op.py @@ -22,17 +22,17 @@ from paddle.fluid.framework import _test_eager_guard def count(x, upper_num): - res = np.zeros((upper_num, )).astype(int) + res = np.zeros((upper_num,)).astype(int) for i in x.reshape(-1): if i >= 0 and i < len(res): res[i] += 1 return res -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNumberCountOpInt64(op_test.OpTest): - def setUp(self): upper_num = 16 self.op_type = "number_count" @@ -45,14 +45,15 @@ class TestNumberCountOpInt64(op_test.OpTest): self.check_output_with_place(paddle.CUDAPlace(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNumberCountAPI(unittest.TestCase): - def setUp(self): self.upper_num = 320 - self.x = np.random.randint(-1, self.upper_num, - size=(6000, 200)).astype('int64') + self.x = np.random.randint(-1, self.upper_num, size=(6000, 200)).astype( + 'int64' + ) self.out = count(self.x, self.upper_num) self.place = paddle.CUDAPlace(0) diff --git a/python/paddle/fluid/tests/unittests/test_numel_op.py b/python/paddle/fluid/tests/unittests/test_numel_op.py index 7bb76c531a3..4bb359a7bd2 100644 --- a/python/paddle/fluid/tests/unittests/test_numel_op.py +++ b/python/paddle/fluid/tests/unittests/test_numel_op.py @@ -20,7 +20,6 @@ import paddle class TestNumelOp(OpTest): - def setUp(self): self.op_type = "size" self.init() @@ -38,19 +37,16 @@ class TestNumelOp(OpTest): class TestNumelOp1(TestNumelOp): - def init(self): self.shape = (11, 66) class TestNumelOp2(TestNumelOp): - def init(self): - self.shape = (0, ) + self.shape = (0,) class TestNumelAPI(unittest.TestCase): - def test_numel_static(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -64,17 +60,19 @@ class TestNumelAPI(unittest.TestCase): out_1 = paddle.numel(x_1) out_2 = paddle.numel(x_2) exe = paddle.static.Executor(place=paddle.CPUPlace()) - res_1, res_2 = exe.run(feed={ - "x_1": input_1, - "x_2": input_2, - }, - fetch_list=[out_1, out_2]) - assert (np.array_equal(res_1, - np.array([np.size(input_1) - ]).astype("int64"))) - assert (np.array_equal(res_2, - np.array([np.size(input_2) - ]).astype("int64"))) + res_1, res_2 = exe.run( + feed={ + "x_1": input_1, + "x_2": input_2, + }, + fetch_list=[out_1, out_2], + ) + assert np.array_equal( + res_1, np.array([np.size(input_1)]).astype("int64") + ) + assert np.array_equal( + res_2, np.array([np.size(input_2)]).astype("int64") + ) def test_numel_imperative(self): paddle.disable_static(paddle.CPUPlace()) @@ -84,8 +82,8 @@ class TestNumelAPI(unittest.TestCase): x_2 = paddle.to_tensor(input_2) out_1 = paddle.numel(x_1) out_2 = paddle.numel(x_2) - assert (np.array_equal(out_1.numpy().item(0), np.size(input_1))) - assert (np.array_equal(out_2.numpy().item(0), np.size(input_2))) + assert np.array_equal(out_1.numpy().item(0), np.size(input_1)) + assert np.array_equal(out_2.numpy().item(0), np.size(input_2)) paddle.enable_static() def test_error(self): diff --git a/python/paddle/fluid/tests/unittests/test_one_hot_op.py b/python/paddle/fluid/tests/unittests/test_one_hot_op.py index 55abf26ea23..4ca0017052e 100644 --- a/python/paddle/fluid/tests/unittests/test_one_hot_op.py +++ b/python/paddle/fluid/tests/unittests/test_one_hot_op.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import Program, program_guard class TestOneHotOp(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -32,8 +31,9 @@ class TestOneHotOp(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -47,7 +47,6 @@ class TestOneHotOp(OpTest): class TestOneHotOp_attr(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -56,8 +55,9 @@ class TestOneHotOp_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -71,7 +71,6 @@ class TestOneHotOp_attr(OpTest): class TestOneHotOp_default_dtype(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -81,8 +80,9 @@ class TestOneHotOp_default_dtype(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -96,7 +96,6 @@ class TestOneHotOp_default_dtype(OpTest): class TestOneHotOp_default_dtype_attr(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -105,8 +104,9 @@ class TestOneHotOp_default_dtype_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -120,7 +120,6 @@ class TestOneHotOp_default_dtype_attr(OpTest): class TestOneHotOp_out_of_range(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -128,8 +127,9 @@ class TestOneHotOp_out_of_range(OpTest): x = [np.random.choice([-1, depth]) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) self.inputs = {'X': (x, x_lod)} self.attrs = {'depth': depth, 'allow_out_of_range': True} @@ -140,7 +140,6 @@ class TestOneHotOp_out_of_range(OpTest): class TestOneHotOp_exception(unittest.TestCase): - def setUp(self): self.op_type = 'one_hot' self.depth = 10 @@ -156,46 +155,54 @@ class TestOneHotOp_exception(unittest.TestCase): def test_check_output(self): program = Program() with program_guard(program): - x = fluid.layers.data(name='x', - shape=[self.dimension], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='x', shape=[self.dimension], dtype='float32', lod_level=1 + ) block = program.current_block() - one_hot_out = block.create_var(name="one_hot_out", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype='float32') - block.append_op(type='one_hot', - inputs={'X': x}, - attrs={'depth': self.depth}, - outputs={'Out': one_hot_out}) + one_hot_out = block.create_var( + name="one_hot_out", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype='float32', + ) + block.append_op( + type='one_hot', + inputs={'X': x}, + attrs={'depth': self.depth}, + outputs={'Out': one_hot_out}, + ) exe = fluid.Executor(self.place) def run(): - exe.run(feed={'x': self.x}, - fetch_list=[one_hot_out], - return_numpy=False) + exe.run( + feed={'x': self.x}, + fetch_list=[one_hot_out], + return_numpy=False, + ) self.assertRaises(ValueError, run) class TestOneHotOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input must be Variable in_w = np.random.random((4, 1)).astype("int32") self.assertRaises(TypeError, fluid.layers.one_hot, in_w) # the input must be int32 or int 64 - in_w2 = fluid.layers.data(name="in_w2", - shape=[4, 1], - append_batch_size=False, - dtype="float32") + in_w2 = fluid.layers.data( + name="in_w2", + shape=[4, 1], + append_batch_size=False, + dtype="float32", + ) self.assertRaises(TypeError, fluid.layers.one_hot, in_w2) # the depth must be int, long or Variable - in_r = fluid.layers.data(name="in_r", - shape=[4, 1], - append_batch_size=False, - dtype="int32") + in_r = fluid.layers.data( + name="in_r", + shape=[4, 1], + append_batch_size=False, + dtype="int32", + ) depth_w = np.array([4]) self.assertRaises(TypeError, fluid.layers.one_hot, in_r, 4.1) self.assertRaises(TypeError, fluid.layers.one_hot, in_r, depth_w) diff --git a/python/paddle/fluid/tests/unittests/test_one_hot_v2_op.py b/python/paddle/fluid/tests/unittests/test_one_hot_v2_op.py index 284957a5318..23d277c04e5 100644 --- a/python/paddle/fluid/tests/unittests/test_one_hot_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_one_hot_v2_op.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import Program, program_guard, _test_eager_guard class TestOneHotOp(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -46,7 +45,6 @@ class TestOneHotOp(OpTest): class TestOneHotOp_attr(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -55,8 +53,9 @@ class TestOneHotOp_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -70,7 +69,6 @@ class TestOneHotOp_attr(OpTest): class TestOneHotOp_default_dtype(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -94,7 +92,6 @@ class TestOneHotOp_default_dtype(OpTest): class TestOneHotOp_default_dtype_attr(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -103,8 +100,9 @@ class TestOneHotOp_default_dtype_attr(OpTest): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -118,7 +116,6 @@ class TestOneHotOp_default_dtype_attr(OpTest): class TestOneHotOp_exception(unittest.TestCase): - def setUp(self): self.op_type = 'one_hot_v2' self.depth = 10 @@ -134,30 +131,34 @@ class TestOneHotOp_exception(unittest.TestCase): def test_check_output(self): program = Program() with program_guard(program): - x = fluid.layers.data(name='x', - shape=[self.dimension], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='x', shape=[self.dimension], dtype='float32', lod_level=1 + ) block = program.current_block() - one_hot_out = block.create_var(name="one_hot_out", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype='float32') - block.append_op(type='one_hot', - inputs={'X': x}, - attrs={'depth': self.depth}, - outputs={'Out': one_hot_out}) + one_hot_out = block.create_var( + name="one_hot_out", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype='float32', + ) + block.append_op( + type='one_hot', + inputs={'X': x}, + attrs={'depth': self.depth}, + outputs={'Out': one_hot_out}, + ) exe = fluid.Executor(self.place) def run(): - exe.run(feed={'x': self.x}, - fetch_list=[one_hot_out], - return_numpy=False) + exe.run( + feed={'x': self.x}, + fetch_list=[one_hot_out], + return_numpy=False, + ) self.assertRaises(ValueError, run) class TestOneHotOpApi(unittest.TestCase): - def test_api(self): depth = 10 self._run(depth) @@ -168,45 +169,53 @@ class TestOneHotOpApi(unittest.TestCase): def test_api_with_dygraph(self): depth = 10 - label = np.array([np.random.randint(0, depth - 1) - for i in range(6)]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, depth - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(): one_hot_label = fluid.one_hot( - input=fluid.dygraph.to_variable(label), depth=depth) + input=fluid.dygraph.to_variable(label), depth=depth + ) one_hot_label = paddle.nn.functional.one_hot( - fluid.dygraph.to_variable(label), depth) + fluid.dygraph.to_variable(label), depth + ) with _test_eager_guard(): one_hot_label = paddle.nn.functional.one_hot( - paddle.to_tensor(label), depth) + paddle.to_tensor(label), depth + ) def _run(self, depth): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = fluid.one_hot(input=label, depth=depth) place = fluid.CPUPlace() - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) class BadInputTestOnehotV2(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="float32") + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="float32", + ) one_hot_label = fluid.one_hot(input=label, depth=4) self.assertRaises(TypeError, test_bad_x) diff --git a/python/paddle/fluid/tests/unittests/test_ones_like.py b/python/paddle/fluid/tests/unittests/test_ones_like.py index 8c0ceadbf58..fe7a66acecd 100644 --- a/python/paddle/fluid/tests/unittests/test_ones_like.py +++ b/python/paddle/fluid/tests/unittests/test_ones_like.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ class TestOnesLikeAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = paddle.fluid.data('x', [3, 4]) @@ -31,7 +30,6 @@ class TestOnesLikeAPIError(unittest.TestCase): class TestOnesLikeAPI(unittest.TestCase): - def test_api(self): shape = [3, 4] startup_program = Program() @@ -46,25 +44,33 @@ class TestOnesLikeAPI(unittest.TestCase): out4 = ones_like(x, 'int32') out5 = ones_like(x, 'int64') - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - outs = exe.run(train_program, - feed={'X': np.ones(shape).astype('float32')}, - fetch_list=[out1, out2, out3, out4, out5]) + outs = exe.run( + train_program, + feed={'X': np.ones(shape).astype('float32')}, + fetch_list=[out1, out2, out3, out4, out5], + ) for i, dtype in enumerate( - [np.float32, np.bool_, np.float64, np.int32, np.int64]): + [np.float32, np.bool_, np.float64, np.int32, np.int64] + ): self.assertEqual(outs[i].dtype, dtype) self.assertEqual((outs[i] == np.ones(shape, dtype)).all(), True) class TestOnesLikeImpeartive(unittest.TestCase): - def test_out(self): shape = [3, 4] - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) x = paddle.to_tensor(np.ones(shape)) for dtype in [np.bool_, np.float32, np.float64, np.int32, np.int64]: @@ -80,11 +86,13 @@ class TestOnesLikeImpeartive(unittest.TestCase): class TestOnesAPI(unittest.TestCase): - def test_api(self): shape = [3, 4] - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) for dtype in [np.float32, np.float64, np.int32, np.int64]: diff --git a/python/paddle/fluid/tests/unittests/test_ones_op.py b/python/paddle/fluid/tests/unittests/test_ones_op.py index ac02627f9cc..a82842b3c72 100644 --- a/python/paddle/fluid/tests/unittests/test_ones_op.py +++ b/python/paddle/fluid/tests/unittests/test_ones_op.py @@ -21,13 +21,12 @@ import numpy as np class ApiOnesTest(unittest.TestCase): - def test_paddle_ones(self): with paddle.static.program_guard(paddle.static.Program()): ones = paddle.ones(shape=[10]) place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[ones]) + (result,) = exe.run(fetch_list=[ones]) expected_result = np.ones(10, dtype="float32") self.assertEqual((result == expected_result).all(), True) @@ -35,7 +34,7 @@ class ApiOnesTest(unittest.TestCase): ones = paddle.ones(shape=[10], dtype="float64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[ones]) + (result,) = exe.run(fetch_list=[ones]) expected_result = np.ones(10, dtype="float64") self.assertEqual((result == expected_result).all(), True) @@ -43,7 +42,7 @@ class ApiOnesTest(unittest.TestCase): ones = paddle.ones(shape=[10], dtype="int64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[ones]) + (result,) = exe.run(fetch_list=[ones]) expected_result = np.ones(10, dtype="int64") self.assertEqual((result == expected_result).all(), True) @@ -52,15 +51,13 @@ class ApiOnesTest(unittest.TestCase): ones = fluid.layers.ones(shape=[10], dtype="int64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[ones]) + (result,) = exe.run(fetch_list=[ones]) expected_result = np.ones(10, dtype="int64") self.assertEqual((result == expected_result).all(), True) class ApiOnesZerosError(unittest.TestCase): - def test_errors(self): - def test_error1(): with paddle.static.program_guard(paddle.static.Program()): ones = paddle.ones(shape=10, dtype="int64") diff --git a/python/paddle/fluid/tests/unittests/test_onnx_export.py b/python/paddle/fluid/tests/unittests/test_onnx_export.py index 6cb4ed2fd92..4a4e2600044 100644 --- a/python/paddle/fluid/tests/unittests/test_onnx_export.py +++ b/python/paddle/fluid/tests/unittests/test_onnx_export.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class LinearNet(paddle.nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = paddle.nn.Linear(128, 10) @@ -30,7 +29,6 @@ class LinearNet(paddle.nn.Layer): class Logic(paddle.nn.Layer): - def __init__(self): super(Logic, self).__init__() @@ -42,10 +40,10 @@ class Logic(paddle.nn.Layer): class TestExportWithTensor(unittest.TestCase): - def func_with_tensor(self): - self.x_spec = paddle.static.InputSpec(shape=[None, 128], - dtype='float32') + self.x_spec = paddle.static.InputSpec( + shape=[None, 128], dtype='float32' + ) model = LinearNet() paddle.onnx.export(model, 'linear_net', input_spec=[self.x_spec]) @@ -56,7 +54,6 @@ class TestExportWithTensor(unittest.TestCase): class TestExportWithTensor1(unittest.TestCase): - def func_with_tensor(self): self.x = paddle.to_tensor(np.random.random((1, 128))) model = LinearNet() @@ -69,17 +66,15 @@ class TestExportWithTensor1(unittest.TestCase): class TestExportPrunedGraph(unittest.TestCase): - def func_prune_graph(self): model = Logic() self.x = paddle.to_tensor(np.array([1])) self.y = paddle.to_tensor(np.array([-1])) paddle.jit.to_static(model) out = model(self.x, self.y, z=True) - paddle.onnx.export(model, - 'pruned', - input_spec=[self.x], - output_spec=[out]) + paddle.onnx.export( + model, 'pruned', input_spec=[self.x], output_spec=[out] + ) def test_prune_graph(self): # test eager diff --git a/python/paddle/fluid/tests/unittests/test_op_function_generator.py b/python/paddle/fluid/tests/unittests/test_op_function_generator.py index a8e4e7d3e82..3163a768888 100644 --- a/python/paddle/fluid/tests/unittests/test_op_function_generator.py +++ b/python/paddle/fluid/tests/unittests/test_op_function_generator.py @@ -22,7 +22,6 @@ from paddle import _legacy_C_ops class TestTracedLayer(fluid.dygraph.Layer): - def __init__(self, name_scope): super(TestTracedLayer, self).__init__(name_scope) @@ -31,7 +30,6 @@ class TestTracedLayer(fluid.dygraph.Layer): class TestVariable(unittest.TestCase): - def setUp(self): self.shape = [512, 768] self.dtype = np.float32 @@ -100,7 +98,8 @@ class TestVariable(unittest.TestCase): a = np.random.uniform(-1, 1, self.shape).astype(self.dtype) x = fluid.dygraph.to_variable(a) res_dygraph, static_layer = TracedLayer.trace( - layer, inputs=x) # dygraph out + layer, inputs=x + ) # dygraph out res_static_graph = static_layer([x])[0] np.testing.assert_array_equal(res_dygraph.numpy(), res_static_graph) diff --git a/python/paddle/fluid/tests/unittests/test_op_name_conflict.py b/python/paddle/fluid/tests/unittests/test_op_name_conflict.py index 86f82f0a9e2..10e9b4d3028 100644 --- a/python/paddle/fluid/tests/unittests/test_op_name_conflict.py +++ b/python/paddle/fluid/tests/unittests/test_op_name_conflict.py @@ -18,7 +18,6 @@ import unittest class TestOpNameConflict(unittest.TestCase): - def test_conflict(self): main = fluid.Program() startup = fluid.Program() @@ -34,12 +33,14 @@ class TestOpNameConflict(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) - m_v, n_v, p_v = exe.run(feed={ - "x": np.ones((1), "float32") * 2, - "y": np.ones((1), "float32") * 3, - "z": np.ones((1), "float32") * 5 - }, - fetch_list=[m, n, p]) + m_v, n_v, p_v = exe.run( + feed={ + "x": np.ones((1), "float32") * 2, + "y": np.ones((1), "float32") * 3, + "z": np.ones((1), "float32") * 5, + }, + fetch_list=[m, n, p], + ) self.assertEqual(m_v[0], 5.0) self.assertEqual(n_v[0], 8.0) @@ -50,37 +51,42 @@ class TestOpNameConflict(unittest.TestCase): startup = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( - ) else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - data = fluid.data(name='data', - shape=[None, 1, 2, 2], - dtype='float32') - tensor = fluid.data(name='tensor', - shape=[None, 32, 64], - dtype='float32') - x = fluid.data(name='x', - shape=[None, 1], - dtype='float32', - lod_level=1) + data = fluid.data( + name='data', shape=[None, 1, 2, 2], dtype='float32' + ) + tensor = fluid.data( + name='tensor', shape=[None, 32, 64], dtype='float32' + ) + x = fluid.data( + name='x', shape=[None, 1], dtype='float32', lod_level=1 + ) input_scale = fluid.layers.create_parameter( shape=[1], dtype="float32", - default_initializer=fluid.initializer.Constant(2.0)) + default_initializer=fluid.initializer.Constant(2.0), + ) input_bias = fluid.layers.create_parameter( shape=[1], dtype="float32", - default_initializer=fluid.initializer.Constant(0.5)) - out_affine = fluid.layers.affine_channel(data, - scale=input_scale, - bias=input_bias) - out_similarity = fluid.layers.similarity_focus(input=data, - axis=1, - indexes=[0]) + default_initializer=fluid.initializer.Constant(0.5), + ) + out_affine = fluid.layers.affine_channel( + data, scale=input_scale, bias=input_bias + ) + out_similarity = fluid.layers.similarity_focus( + input=data, axis=1, indexes=[0] + ) position_tensor = fluid.layers.add_position_encoding( - input=tensor, alpha=1.0, beta=1.0) + input=tensor, alpha=1.0, beta=1.0 + ) x_reversed = fluid.layers.sequence_reverse(x) exe.run(fluid.default_startup_program()) @@ -88,21 +94,24 @@ class TestOpNameConflict(unittest.TestCase): x_d = fluid.create_lod_tensor( np.array([[1.1], [2.2], [3.3], [4.4]]).astype('float32'), - [[1, 3]], place) - outs = exe.run(test_program, - fetch_list=[ - out_affine, out_similarity, position_tensor, - x_reversed - ], - feed={ - data.name: - np.ones([1, 1, 2, 2]).astype('float32'), - tensor.name: - np.ones([1, 32, 64]).astype('float32'), - x.name: - x_d - }, - return_numpy=False) + [[1, 3]], + place, + ) + outs = exe.run( + test_program, + fetch_list=[ + out_affine, + out_similarity, + position_tensor, + x_reversed, + ], + feed={ + data.name: np.ones([1, 1, 2, 2]).astype('float32'), + tensor.name: np.ones([1, 32, 64]).astype('float32'), + x.name: x_d, + }, + return_numpy=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_op_support_gpu.py b/python/paddle/fluid/tests/unittests/test_op_support_gpu.py index b13301517b3..fe7fd6ab3d5 100644 --- a/python/paddle/fluid/tests/unittests/test_op_support_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_op_support_gpu.py @@ -17,10 +17,10 @@ import paddle.fluid.core as core class TestOpSupportGPU(unittest.TestCase): - def test_case(self): - self.assertEqual(core.is_compiled_with_cuda(), - core.op_support_gpu("sum")) + self.assertEqual( + core.is_compiled_with_cuda(), core.op_support_gpu("sum") + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_op_version.py b/python/paddle/fluid/tests/unittests/test_op_version.py index 49b3c69b890..47f49d21864 100644 --- a/python/paddle/fluid/tests/unittests/test_op_version.py +++ b/python/paddle/fluid/tests/unittests/test_op_version.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class OpLastCheckpointCheckerTest(unittest.TestCase): - def __init__(self, methodName='runTest'): super(OpLastCheckpointCheckerTest, self).__init__(methodName) self.checker = utils.OpLastCheckpointChecker() @@ -27,8 +26,9 @@ class OpLastCheckpointCheckerTest(unittest.TestCase): def test_op_attr_info(self): update_type = fluid.core.OpUpdateType.kNewAttr - info_list = self.checker.filter_updates(self.fake_op, update_type, - 'STRINGS') + info_list = self.checker.filter_updates( + self.fake_op, update_type, 'STRINGS' + ) self.assertTrue(info_list) self.assertEqual(info_list[0].name(), 'STRINGS') self.assertEqual(info_list[0].default_value(), ['str1', 'str2']) @@ -36,8 +36,9 @@ class OpLastCheckpointCheckerTest(unittest.TestCase): def test_op_input_output_info(self): update_type = fluid.core.OpUpdateType.kNewInput - info_list = self.checker.filter_updates(self.fake_op, update_type, - 'NewInput') + info_list = self.checker.filter_updates( + self.fake_op, update_type, 'NewInput' + ) self.assertTrue(info_list) self.assertEqual(info_list[0].name(), 'NewInput') self.assertEqual(info_list[0].remark(), 'NewInput_') @@ -50,7 +51,6 @@ class OpLastCheckpointCheckerTest(unittest.TestCase): class OpVersionTest(unittest.TestCase): - def __init__(self, methodName='runTest'): super(OpVersionTest, self).__init__(methodName) self.vmap = fluid.core.get_op_version_map() @@ -72,11 +72,13 @@ class OpVersionTest(unittest.TestCase): true_l = [2.56, 1.28] self.assertEqual(len(true_l), len(desc_2[1].info().default_value())) for i in range(len(true_l)): - self.assertAlmostEqual(desc_2[1].info().default_value()[i], - true_l[i], 2) + self.assertAlmostEqual( + desc_2[1].info().default_value()[i], true_l[i], 2 + ) self.assertEqual(desc_2[2].info().default_value(), [10, 100]) - self.assertEqual(desc_2[3].info().default_value(), - [10000001, -10000001]) + self.assertEqual( + desc_2[3].info().default_value(), [10000001, -10000001] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_operator.py b/python/paddle/fluid/tests/unittests/test_operator.py index 62db058682b..0b7d9a4c1bb 100644 --- a/python/paddle/fluid/tests/unittests/test_operator.py +++ b/python/paddle/fluid/tests/unittests/test_operator.py @@ -21,7 +21,6 @@ import paddle.fluid.proto.framework_pb2 as framework_pb2 class TestGetAllProtos(unittest.TestCase): - def test_all(self): all_protos = op.get_all_op_protos() self.assertNotEqual(0, len(all_protos)) @@ -31,7 +30,6 @@ class TestGetAllProtos(unittest.TestCase): class TestOpDescCreationMethod(unittest.TestCase): - def test_plain_input_output(self): op_proto = framework_pb2.OpProto() op_proto.type = "test" @@ -112,10 +110,9 @@ class TestOpDescCreationMethod(unittest.TestCase): expected1.type = 'fc' self.assertEqual(expected1, generated1) - generated2 = method(X=['x1', 'x2', 'x3'], - b='b', - W=['w1', 'w2', 'w3'], - Y='y') + generated2 = method( + X=['x1', 'x2', 'x3'], b='b', W=['w1', 'w2', 'w3'], Y='y' + ) expected2 = framework_pb2.OpDesc() tmp = expected2.inputs.add() @@ -163,14 +160,16 @@ class TestOpDescCreationMethod(unittest.TestCase): method = op.OpDescCreationMethod(op_proto) - generated = method(X="a", - int_attr=10, - float_attr=3.2, - float64_attr=np.finfo("float64").max, - string_attr="test_str", - ints_attr=[0, 1, 2, 3, 4], - floats_attr=[0.2, 3.2, 4.5], - strings_attr=["a", "b", "c"]) + generated = method( + X="a", + int_attr=10, + float_attr=3.2, + float64_attr=np.finfo("float64").max, + string_attr="test_str", + ints_attr=[0, 1, 2, 3, 4], + floats_attr=[0.2, 3.2, 4.5], + strings_attr=["a", "b", "c"], + ) expected = framework_pb2.OpDesc() expected.type = "test" @@ -218,13 +217,13 @@ class TestOpDescCreationMethod(unittest.TestCase): class TestOpCreations(unittest.TestCase): - def test_all(self): add_op = op.Operator("sum", X=["a", "b"], Out="z") self.assertIsNotNone(add_op) # Invoke C++ DebugString() - self.assertEqual('Op(sum), inputs:{X[a, b]}, outputs:{Out[z]}.', - str(add_op)) + self.assertEqual( + 'Op(sum), inputs:{X[a, b]}, outputs:{Out[z]}.', str(add_op) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_operator_desc.py b/python/paddle/fluid/tests/unittests/test_operator_desc.py index eeb7dfe95db..af3eecec826 100644 --- a/python/paddle/fluid/tests/unittests/test_operator_desc.py +++ b/python/paddle/fluid/tests/unittests/test_operator_desc.py @@ -22,7 +22,6 @@ main_program = default_startup_program() class TestOperator(unittest.TestCase): - def test_error_type(self): block = main_program._create_block() try: @@ -30,37 +29,34 @@ class TestOperator(unittest.TestCase): self.assertFail() except ValueError as v_err: self.assertEqual( - str(v_err), - "`type` to initialized an Operator can not be None.") + str(v_err), "`type` to initialized an Operator can not be None." + ) try: block.append_op(type="no_such_op") self.assertFail() except ValueError as a_err: self.assertEqual( - str(a_err), "Operator \"no_such_op\" has not been registered.") + str(a_err), "Operator \"no_such_op\" has not been registered." + ) def test_op_desc_creation(self): program = Program() block = program.current_block() - mul_x = block.create_var(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mul_op = block.append_op(type="mul", - inputs={ - "X": [mul_x], - "Y": mul_y - }, - outputs={"Out": [mul_out]}, - attrs={"x_num_col_dims": 1}) + mul_x = block.create_var( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mul_op = block.append_op( + type="mul", + inputs={"X": [mul_x], "Y": mul_y}, + outputs={"Out": [mul_out]}, + attrs={"x_num_col_dims": 1}, + ) self.assertNotEqual(str(mul_op), "") self.assertEqual(mul_op.type, "mul") @@ -71,10 +67,19 @@ class TestOperator(unittest.TestCase): self.assertEqual(mul_op.output("Out"), ["mul.out"]) self.assertEqual( set(mul_op.attr_names), - set([ - "x_num_col_dims", "y_num_col_dims", "op_role", "op_role_var", - "op_namescope", "op_callstack", "op_device", "with_quant_attr" - ])) + set( + [ + "x_num_col_dims", + "y_num_col_dims", + "op_role", + "op_role_var", + "op_namescope", + "op_callstack", + "op_device", + "with_quant_attr", + ] + ), + ) self.assertEqual(mul_op.has_attr("x_num_col_dims"), True) self.assertEqual(mul_op.attr_type("x_num_col_dims"), core.AttrType.INT) self.assertEqual(mul_op.attr("x_num_col_dims"), 1) @@ -89,25 +94,23 @@ class TestOperator(unittest.TestCase): def test_mult_input(self): program = Program() block = program.current_block() - sum_x1 = block.create_var(dtype="int", - shape=[3, 4], - lod_level=0, - name="sum.x1") - sum_x2 = block.create_var(dtype="int", - shape=[3, 4], - lod_level=0, - name="sum.x2") - sum_x3 = block.create_var(dtype="int", - shape=[3, 4], - lod_level=0, - name="sum.x3") - sum_out = block.create_var(dtype="int", - shape=[3, 4], - lod_level=0, - name="sum.out") - sum_op = block.append_op(type="sum", - inputs={"X": [sum_x1, sum_x2, sum_x3]}, - outputs={"Out": sum_out}) + sum_x1 = block.create_var( + dtype="int", shape=[3, 4], lod_level=0, name="sum.x1" + ) + sum_x2 = block.create_var( + dtype="int", shape=[3, 4], lod_level=0, name="sum.x2" + ) + sum_x3 = block.create_var( + dtype="int", shape=[3, 4], lod_level=0, name="sum.x3" + ) + sum_out = block.create_var( + dtype="int", shape=[3, 4], lod_level=0, name="sum.out" + ) + sum_op = block.append_op( + type="sum", + inputs={"X": [sum_x1, sum_x2, sum_x3]}, + outputs={"Out": sum_out}, + ) self.assertEqual(sum_op.type, "sum") self.assertEqual(sum_op.input_names, ["X"]) self.assertEqual(sum_op.input("X"), ["sum.x1", "sum.x2", "sum.x3"]) diff --git a/python/paddle/fluid/tests/unittests/test_ops_nms.py b/python/paddle/fluid/tests/unittests/test_ops_nms.py index 4004f29b446..573231a8a72 100644 --- a/python/paddle/fluid/tests/unittests/test_ops_nms.py +++ b/python/paddle/fluid/tests/unittests/test_ops_nms.py @@ -46,10 +46,12 @@ def multiclass_nms(boxes, scores, category_idxs, iou_threshold, top_k): cur_category_scores = scores[cur_category_boxes_idxs] cur_category_sorted_indices = np.argsort(-cur_category_scores) cur_category_sorted_boxes = cur_category_boxes[ - cur_category_sorted_indices] + cur_category_sorted_indices + ] - cur_category_keep_boxes_sub_idxs = cur_category_sorted_indices[nms( - cur_category_sorted_boxes, iou_threshold)] + cur_category_keep_boxes_sub_idxs = cur_category_sorted_indices[ + nms(cur_category_sorted_boxes, iou_threshold) + ] mask[cur_category_boxes_idxs[cur_category_keep_boxes_sub_idxs]] = True @@ -72,7 +74,6 @@ def gen_args(num_boxes, dtype): class TestOpsNMS(unittest.TestCase): - def setUp(self): self.num_boxes = 64 self.threshold = 0.5 @@ -91,78 +92,100 @@ class TestOpsNMS(unittest.TestCase): for device in self.devices: for dtype in self.dtypes: boxes, scores, category_idxs, categories = gen_args( - self.num_boxes, dtype) + self.num_boxes, dtype + ) paddle.set_device(device) - out = paddle.vision.ops.nms(paddle.to_tensor(boxes), - self.threshold, - paddle.to_tensor(scores)) - out = paddle.vision.ops.nms(paddle.to_tensor(boxes), - self.threshold) + out = paddle.vision.ops.nms( + paddle.to_tensor(boxes), + self.threshold, + paddle.to_tensor(scores), + ) + out = paddle.vision.ops.nms( + paddle.to_tensor(boxes), self.threshold + ) out_py = nms(boxes, self.threshold) np.testing.assert_array_equal( out.numpy(), out_py, - err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py)) + err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py), + ) def test_multiclass_nms_dynamic(self): for device in self.devices: for dtype in self.dtypes: boxes, scores, category_idxs, categories = gen_args( - self.num_boxes, dtype) + self.num_boxes, dtype + ) paddle.set_device(device) - out = paddle.vision.ops.nms(paddle.to_tensor(boxes), - self.threshold, - paddle.to_tensor(scores), - paddle.to_tensor(category_idxs), - categories, self.topk) - out_py = multiclass_nms(boxes, scores, category_idxs, - self.threshold, self.topk) + out = paddle.vision.ops.nms( + paddle.to_tensor(boxes), + self.threshold, + paddle.to_tensor(scores), + paddle.to_tensor(category_idxs), + categories, + self.topk, + ) + out_py = multiclass_nms( + boxes, scores, category_idxs, self.threshold, self.topk + ) np.testing.assert_array_equal( out.numpy(), out_py, - err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py)) + err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py), + ) def test_multiclass_nms_static(self): for device in self.devices: for dtype in self.dtypes: paddle.enable_static() boxes, scores, category_idxs, categories = gen_args( - self.num_boxes, dtype) - boxes_static = paddle.static.data(shape=boxes.shape, - dtype=boxes.dtype, - name="boxes") - scores_static = paddle.static.data(shape=scores.shape, - dtype=scores.dtype, - name="scores") + self.num_boxes, dtype + ) + boxes_static = paddle.static.data( + shape=boxes.shape, dtype=boxes.dtype, name="boxes" + ) + scores_static = paddle.static.data( + shape=scores.shape, dtype=scores.dtype, name="scores" + ) category_idxs_static = paddle.static.data( shape=category_idxs.shape, dtype=category_idxs.dtype, - name="category_idxs") - out = paddle.vision.ops.nms(boxes_static, self.threshold, - scores_static, category_idxs_static, - categories, self.topk) + name="category_idxs", + ) + out = paddle.vision.ops.nms( + boxes_static, + self.threshold, + scores_static, + category_idxs_static, + categories, + self.topk, + ) place = paddle.CPUPlace() if device == 'gpu': place = paddle.CUDAPlace(0) exe = paddle.static.Executor(place) - out = exe.run(paddle.static.default_main_program(), - feed={ - 'boxes': boxes, - 'scores': scores, - 'category_idxs': category_idxs - }, - fetch_list=[out]) + out = exe.run( + paddle.static.default_main_program(), + feed={ + 'boxes': boxes, + 'scores': scores, + 'category_idxs': category_idxs, + }, + fetch_list=[out], + ) paddle.disable_static() - out_py = multiclass_nms(boxes, scores, category_idxs, - self.threshold, self.topk) + out_py = multiclass_nms( + boxes, scores, category_idxs, self.threshold, self.topk + ) out = np.array(out) out = np.squeeze(out) np.testing.assert_array_equal( out, out_py, - err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py)) + err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py), + ) def test_multiclass_nms_dynamic_to_static(self): for device in self.devices: @@ -173,10 +196,14 @@ class TestOpsNMS(unittest.TestCase): scores = np.arange(0, 64).astype('float32') categories = np.array([0, 1, 2, 3]) category_idxs = categories.repeat(16) - out = paddle.vision.ops.nms(x, 0.1, - paddle.to_tensor(scores), - paddle.to_tensor(category_idxs), - categories, 10) + out = paddle.vision.ops.nms( + x, + 0.1, + paddle.to_tensor(scores), + paddle.to_tensor(category_idxs), + categories, + 10, + ) return out boxes = np.random.rand(64, 4).astype('float32') @@ -188,9 +215,9 @@ class TestOpsNMS(unittest.TestCase): fun, self.path, input_spec=[ - paddle.static.InputSpec(shape=[None, 4], - dtype='float32', - name='x') + paddle.static.InputSpec( + shape=[None, 4], dtype='float32', name='x' + ) ], ) load_func = paddle.jit.load(self.path) @@ -199,20 +226,23 @@ class TestOpsNMS(unittest.TestCase): origin, res, err_msg='origin out: {}\n inference model out: {}\n'.format( - origin, res)) + origin, res + ), + ) def test_matrix_nms_dynamic(self): for device in self.devices: for dtype in self.dtypes: boxes, scores, category_idxs, categories = gen_args( - self.num_boxes, dtype) + self.num_boxes, dtype + ) scores = np.random.rand(1, 4, self.num_boxes).astype(dtype) paddle.set_device(device) out = paddle.vision.ops.matrix_nms( paddle.to_tensor(boxes).unsqueeze(0), paddle.to_tensor(scores), self.threshold, - post_threshold=0., + post_threshold=0.0, nms_top_k=400, keep_top_k=100, ) diff --git a/python/paddle/fluid/tests/unittests/test_optimizer.py b/python/paddle/fluid/tests/unittests/test_optimizer.py index 47d42e354ca..17ad33f67ab 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer.py @@ -22,7 +22,11 @@ import paddle.fluid.optimizer as optimizer import paddle.fluid.core as core import numpy as np from paddle.fluid.backward import append_backward -from paddle.fluid.framework import Program, program_guard, convert_np_dtype_to_dtype_ +from paddle.fluid.framework import ( + Program, + program_guard, + convert_np_dtype_to_dtype_, +) from paddle.fluid.framework import _test_eager_guard import paddle from paddle.io import Dataset @@ -30,40 +34,36 @@ import numpy class TestOptimizer(unittest.TestCase): - def test_sgd_optimizer(self): - def check_sgd_optimizer(optimizer_attr): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr=optimizer_attr) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr=optimizer_attr, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01) opts, _ = sgd_optimizer.minimize(mean_out, init_program) return opts @@ -78,40 +78,36 @@ class TestOptimizer(unittest.TestCase): class TestOptimizerBackwardApplygrad(unittest.TestCase): - def test_sgd_optimizer(self): - def check_sgd_optimizer(optimizer_attr): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr=optimizer_attr) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr=optimizer_attr, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01) with framework.program_guard(program, init_program): p_g = sgd_optimizer.backward(mean_out) @@ -128,9 +124,7 @@ class TestOptimizerBackwardApplygrad(unittest.TestCase): class TestMomentumOptimizer(unittest.TestCase): - class MockMomentum(optimizer.MomentumOptimizer): - def get_accumulators(self): return self._accumulators @@ -141,36 +135,35 @@ class TestMomentumOptimizer(unittest.TestCase): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) learning_rate = 0.01 - momentum_optimizer = self.MockMomentum(learning_rate=learning_rate, - momentum=0.2) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + momentum_optimizer = self.MockMomentum( + learning_rate=learning_rate, momentum=0.2 + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) @@ -201,37 +194,35 @@ class TestMomentumOptimizer(unittest.TestCase): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - momentum_optimizer = self.MockMomentum(learning_rate=learning_rate, - momentum=0.2, - use_nesterov=True) + momentum_optimizer = self.MockMomentum( + learning_rate=learning_rate, momentum=0.2, use_nesterov=True + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) @@ -260,9 +251,7 @@ class TestMomentumOptimizer(unittest.TestCase): class TestAdagradOptimizer(unittest.TestCase): - class MockAdagrad(optimizer.AdagradOptimizer): - def get_accumulators(self): return self._accumulators @@ -273,36 +262,35 @@ class TestAdagradOptimizer(unittest.TestCase): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - adagrad_optimizer = self.MockAdagrad(learning_rate=learning_rate, - epsilon=1.0e-6) + adagrad_optimizer = self.MockAdagrad( + learning_rate=learning_rate, epsilon=1.0e-6 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adagrad_optimizer.get_accumulators()), 0) @@ -329,9 +317,7 @@ class TestAdagradOptimizer(unittest.TestCase): class TestAdamOptimizer(unittest.TestCase): - class MockAdam(optimizer.AdamOptimizer): - def get_accumulators(self): return self._accumulators @@ -345,37 +331,35 @@ class TestAdamOptimizer(unittest.TestCase): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - adam_optimizer = self.MockAdam(learning_rate=learning_rate, - beta1=0.9, - beta2=0.999) + adam_optimizer = self.MockAdam( + learning_rate=learning_rate, beta1=0.9, beta2=0.999 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adam_optimizer.get_accumulators()), 0) @@ -404,9 +388,7 @@ class TestAdamOptimizer(unittest.TestCase): class TestAdamaxOptimizer(unittest.TestCase): - class MockAdamax(optimizer.AdamaxOptimizer): - def get_accumulators(self): return self._accumulators @@ -420,37 +402,35 @@ class TestAdamaxOptimizer(unittest.TestCase): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - adamax_optimizer = self.MockAdamax(learning_rate=learning_rate, - beta1=0.9, - beta2=0.999) + adamax_optimizer = self.MockAdamax( + learning_rate=learning_rate, beta1=0.9, beta2=0.999 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adamax_optimizer.get_accumulators()), 0) @@ -479,61 +459,56 @@ class TestAdamaxOptimizer(unittest.TestCase): class TestDpsgdOptimizer(unittest.TestCase): - def test_dpsgd_optimizer(self): - def check_dpsgd_optimizer(optimizer_attr): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr=optimizer_attr) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) - dpsgd_optimizer = optimizer.DpsgdOptimizer(learning_rate=0.01, - clip=100.0, - batch_size=16.0, - sigma=0.0) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr=optimizer_attr, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) + dpsgd_optimizer = optimizer.DpsgdOptimizer( + learning_rate=0.01, clip=100.0, batch_size=16.0, sigma=0.0 + ) opts, _ = dpsgd_optimizer.minimize(mean_out, init_program) return opts - opts = check_dpsgd_optimizer({ - 'learning_rate': 1.1, - 'clip': 100.0, - 'batch_size': 16.0, - 'sigma': 4.0 - }) + opts = check_dpsgd_optimizer( + { + 'learning_rate': 1.1, + 'clip': 100.0, + 'batch_size': 16.0, + 'sigma': 4.0, + } + ) self.assertEqual(len(opts), 2) self.assertEqual([op.type for op in opts], ["scale", "dpsgd"]) class TestDecayedAdagradOptimizer(unittest.TestCase): - class MockDecayedAdagrad(optimizer.DecayedAdagradOptimizer): - def get_accumulators(self): return self._accumulators @@ -544,36 +519,35 @@ class TestDecayedAdagradOptimizer(unittest.TestCase): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 decayed_adagrad_optimizer = self.MockDecayedAdagrad( - learning_rate=learning_rate, decay=0.95, epsilon=1.0e-6) + learning_rate=learning_rate, decay=0.95, epsilon=1.0e-6 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(decayed_adagrad_optimizer.get_accumulators()), 0) @@ -586,7 +560,8 @@ class TestDecayedAdagradOptimizer(unittest.TestCase): accumulators = decayed_adagrad_optimizer.get_accumulators() self.assertEqual(len(accumulators), 1) self.assertTrue( - decayed_adagrad_optimizer.get_moment_str() in accumulators) + decayed_adagrad_optimizer.get_moment_str() in accumulators + ) moment_acc = accumulators[decayed_adagrad_optimizer.get_moment_str()] self.assertEqual(len(moment_acc), 1) self.assertTrue(mul_x.name in moment_acc) @@ -601,9 +576,7 @@ class TestDecayedAdagradOptimizer(unittest.TestCase): class TestFtrlOptimizer(unittest.TestCase): - class MockFtrl(optimizer.FtrlOptimizer): - def get_accumulators(self): return self._accumulators @@ -617,38 +590,35 @@ class TestFtrlOptimizer(unittest.TestCase): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - ftrl_optimizer = self.MockFtrl(learning_rate=learning_rate, - l1=0.0, - l2=0.0, - lr_power=-0.5) + ftrl_optimizer = self.MockFtrl( + learning_rate=learning_rate, l1=0.0, l2=0.0, lr_power=-0.5 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(ftrl_optimizer.get_accumulators()), 0) @@ -677,44 +647,40 @@ class TestFtrlOptimizer(unittest.TestCase): class TestLookaheadOptimizer(unittest.TestCase): - def test_lookahead_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() init_block = init_program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - init_mul_x = init_block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + init_mul_x = init_block.create_parameter( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) sgd = optimizer.SGD(learning_rate=0.01) lookahead = optimizer.LookaheadOptimizer(sgd, alpha=0.5, k=5) @@ -725,109 +691,98 @@ class TestLookaheadOptimizer(unittest.TestCase): class TestRecomputeOptimizer(unittest.TestCase): - def net(self, return_input=False, with_dropout=False, with_seed=False): program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") + mul_x = block.create_parameter( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) if with_dropout is True: - mul_out_drop = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out.dropout") - mul_out_mask = block.create_var(dtype="uint8", - shape=[5, 8], - lod_level=0, - name="mul.out.mask") + mul_out_drop = block.create_var( + dtype="float32", + shape=[5, 8], + lod_level=0, + name="mul.out.dropout", + ) + mul_out_mask = block.create_var( + dtype="uint8", shape=[5, 8], lod_level=0, name="mul.out.mask" + ) if with_seed is True: - seed_out = block.create_var(dtype="int32", - shape=[1], - name="seed.out") - - b1 = block.create_parameter(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b1") - b1_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b1_out") - b2 = block.create_parameter(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b2") - b2_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b2_out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) + seed_out = block.create_var( + dtype="int32", shape=[1], name="seed.out" + ) + + b1 = block.create_parameter( + dtype="float32", shape=[5, 8], lod_level=0, name="b1" + ) + b1_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="b1_out" + ) + b2 = block.create_parameter( + dtype="float32", shape=[5, 8], lod_level=0, name="b2" + ) + b2_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="b2_out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) if with_dropout is True: dropout_inputs = {'X': [mul_out]} if with_seed is True: - block.append_op(type='seed', - outputs={'Out': seed_out}, - attrs={ - 'deterministic': True, - 'rng_name': 'rng0', - 'force_cpu': True - }) + block.append_op( + type='seed', + outputs={'Out': seed_out}, + attrs={ + 'deterministic': True, + 'rng_name': 'rng0', + 'force_cpu': True, + }, + ) dropout_inputs = {'X': [mul_out], 'Seed': [seed_out]} - block.append_op(type='dropout', - inputs=dropout_inputs, - outputs={ - 'Out': [mul_out_drop], - 'Mask': [mul_out_mask] - }, - attrs={ - 'dropout_prob': 0.5, - }) - block.append_op(type="elementwise_add", - inputs={ - "X": mul_out_drop, - "Y": b1 - }, - outputs={"Out": b1_out}) + block.append_op( + type='dropout', + inputs=dropout_inputs, + outputs={'Out': [mul_out_drop], 'Mask': [mul_out_mask]}, + attrs={ + 'dropout_prob': 0.5, + }, + ) + block.append_op( + type="elementwise_add", + inputs={"X": mul_out_drop, "Y": b1}, + outputs={"Out": b1_out}, + ) else: - block.append_op(type="elementwise_add", - inputs={ - "X": mul_out, - "Y": b1 - }, - outputs={"Out": b1_out}) - - block.append_op(type="elementwise_add", - inputs={ - "X": b1_out, - "Y": b2 - }, - outputs={"Out": b2_out}) - block.append_op(type="mean", - inputs={"X": b2_out}, - outputs={"Out": mean_out}) + block.append_op( + type="elementwise_add", + inputs={"X": mul_out, "Y": b1}, + outputs={"Out": b1_out}, + ) + + block.append_op( + type="elementwise_add", + inputs={"X": b1_out, "Y": b2}, + outputs={"Out": b2_out}, + ) + block.append_op( + type="mean", inputs={"X": b2_out}, outputs={"Out": mean_out} + ) if return_input == True: return mul_x, mul_out, b1_out, b2_out, mean_out @@ -836,124 +791,225 @@ class TestRecomputeOptimizer(unittest.TestCase): def test_no_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 12) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_one_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_str_checkpoints(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out.name]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_multi_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([mul_out, b2_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add", - "elementwise_add_grad", "elementwise_add_grad", "mul_grad", "sgd", - "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_adjacent_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([mul_out, b1_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 12) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_out_of_order_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b2_out, mul_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add", - "elementwise_add_grad", "elementwise_add_grad", "mul_grad", "sgd", - "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_input_as_checkpoints(self): mul_x, mul_out, b1_out, b2_out, mean_out = self.net(return_input=True) self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([mul_x, b2_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 14) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "mul", "elementwise_add", - "elementwise_add_grad", "elementwise_add_grad", "mul_grad", "sgd", - "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "mul", + "elementwise_add", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_apply_gradients(self): mul_out, b1_out, b2_out, mean_out = self.net() @@ -961,10 +1017,12 @@ class TestRecomputeOptimizer(unittest.TestCase): recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out]) # apply backward - params_grads = recompute_optimizer.backward(mean_out, - startup_program=None, - parameter_list=None, - no_grad_set=None) + params_grads = recompute_optimizer.backward( + mean_out, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ) # apply gradient program = mean_out.block.program @@ -972,11 +1030,24 @@ class TestRecomputeOptimizer(unittest.TestCase): optimize_ops = recompute_optimizer.apply_gradients(params_grads) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_load(self): mul_out, b1_out, b2_out, mean_out = self.net() @@ -989,7 +1060,8 @@ class TestRecomputeOptimizer(unittest.TestCase): except NotImplementedError as e: self.assertEqual( "load function is not supported by Recompute Optimizer for now", - str(e)) + str(e), + ) def test_dropout(self): """ @@ -1000,40 +1072,81 @@ class TestRecomputeOptimizer(unittest.TestCase): self.assertEqual(len(mean_out.block.ops), 5) self.assertEqual( [op.type for op in mean_out.block.ops], - ["mul", "dropout", "elementwise_add", "elementwise_add", "mean"]) + ["mul", "dropout", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 17) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "seed", "dropout", "elementwise_add", "elementwise_add", - "mean", "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "dropout", "elementwise_add_grad", "dropout_grad", "mul_grad", - "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "seed", + "dropout", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "dropout", + "elementwise_add_grad", + "dropout_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_dropout_with_determinate_seed(self): - mul_out, b1_out, b2_out, mean_out = self.net(with_dropout=True, - with_seed=True) + mul_out, b1_out, b2_out, mean_out = self.net( + with_dropout=True, with_seed=True + ) self.assertEqual(len(mean_out.block.ops), 6) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "seed", "dropout", "elementwise_add", "elementwise_add", - "mean" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "seed", + "dropout", + "elementwise_add", + "elementwise_add", + "mean", + ], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 17) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "seed", "dropout", "elementwise_add", "elementwise_add", - "mean", "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "dropout", "elementwise_add_grad", "dropout_grad", "mul_grad", - "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "seed", + "dropout", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "dropout", + "elementwise_add_grad", + "dropout_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_dropout_with_seed(self): """ @@ -1044,16 +1157,16 @@ class TestRecomputeOptimizer(unittest.TestCase): def gen_data(): return { "x": np.random.random(size=(100, 3)).astype('float32'), - "y": np.random.randint(2, size=(100, 1)).astype('int64') + "y": np.random.randint(2, size=(100, 1)).astype('int64'), } def mlp(input_x, input_y): - drop_res = fluid.layers.dropout(input_x, - dropout_prob=0.5, - name="dropout_with_seed_cpu") - prediction = fluid.layers.fc(input=[drop_res], - size=2, - act='softmax') + drop_res = fluid.layers.dropout( + input_x, dropout_prob=0.5, name="dropout_with_seed_cpu" + ) + prediction = fluid.layers.fc( + input=[drop_res], size=2, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=input_y) sum_cost = fluid.layers.reduce_mean(cost) return drop_res, prediction, sum_cost @@ -1063,9 +1176,9 @@ class TestRecomputeOptimizer(unittest.TestCase): scope = fluid.Scope() with fluid.scope_guard(scope): with program_guard(main_program, startup_program): - input_x = fluid.layers.data(name="x", - shape=[3], - dtype='float32') + input_x = fluid.layers.data( + name="x", shape=[3], dtype='float32' + ) input_y = fluid.layers.data(name="y", shape=[1], dtype='int64') drop_res, prediction, cost = mlp(input_x, input_y) sgd = fluid.optimizer.Adam(learning_rate=0.01) @@ -1077,19 +1190,21 @@ class TestRecomputeOptimizer(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) feed_data = gen_data() - drop_vec = exe.run(feed=feed_data, - program=fluid.default_main_program(), - fetch_list=[ - "dropout_with_seed_cpu.tmp_1", - "dropout_with_seed_cpu.tmp_1.subprog_0" - ]) + drop_vec = exe.run( + feed=feed_data, + program=fluid.default_main_program(), + fetch_list=[ + "dropout_with_seed_cpu.tmp_1", + "dropout_with_seed_cpu.tmp_1.subprog_0", + ], + ) self.assertEqual(drop_vec[0].tolist(), drop_vec[1].tolist()) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestRecomputeOptimizerCUDA(unittest.TestCase): - def test_dropout_with_seed(self): """ when we recompute a dropout op, make sure that the recomputed one @@ -1099,16 +1214,16 @@ class TestRecomputeOptimizerCUDA(unittest.TestCase): def gen_data(): return { "x": np.random.random(size=(100, 3)).astype('float32'), - "y": np.random.randint(2, size=(100, 1)).astype('int64') + "y": np.random.randint(2, size=(100, 1)).astype('int64'), } def mlp(input_x, input_y): - drop_res = fluid.layers.dropout(input_x, - dropout_prob=0.5, - name="dropout_with_seed_gpu") - prediction = fluid.layers.fc(input=[drop_res], - size=2, - act='softmax') + drop_res = fluid.layers.dropout( + input_x, dropout_prob=0.5, name="dropout_with_seed_gpu" + ) + prediction = fluid.layers.fc( + input=[drop_res], size=2, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=input_y) sum_cost = fluid.layers.reduce_mean(cost) return drop_res, prediction, sum_cost @@ -1118,9 +1233,9 @@ class TestRecomputeOptimizerCUDA(unittest.TestCase): scope = fluid.Scope() with fluid.scope_guard(scope): with program_guard(main_program, startup_program): - input_x = fluid.layers.data(name="x", - shape=[3], - dtype='float32') + input_x = fluid.layers.data( + name="x", shape=[3], dtype='float32' + ) input_y = fluid.layers.data(name="y", shape=[1], dtype='int64') drop_res, prediction, cost = mlp(input_x, input_y) sgd = fluid.optimizer.Adam(learning_rate=0.01) @@ -1132,70 +1247,67 @@ class TestRecomputeOptimizerCUDA(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) feed_data = gen_data() - drop_vec = exe.run(feed=feed_data, - program=fluid.default_main_program(), - fetch_list=[ - "dropout_with_seed_gpu.tmp_1", - "dropout_with_seed_gpu.tmp_1.subprog_0" - ]) + drop_vec = exe.run( + feed=feed_data, + program=fluid.default_main_program(), + fetch_list=[ + "dropout_with_seed_gpu.tmp_1", + "dropout_with_seed_gpu.tmp_1.subprog_0", + ], + ) self.assertEqual(drop_vec[0].tolist(), drop_vec[1].tolist()) class TestGradientMergeOptimizer(unittest.TestCase): - def net(self): program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - b1 = block.create_parameter(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b1") - b1_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b1_out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="elementwise_add", - inputs={ - "X": mul_out, - "Y": b1 - }, - outputs={"Out": b1_out}) - block.append_op(type="mean", - inputs={"X": b1_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + b1 = block.create_parameter( + dtype="float32", shape=[5, 8], lod_level=0, name="b1" + ) + b1_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="b1_out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="elementwise_add", + inputs={"X": mul_out, "Y": b1}, + outputs={"Out": b1_out}, + ) + block.append_op( + type="mean", inputs={"X": b1_out}, outputs={"Out": mean_out} + ) return mean_out - def test_program_desc(self, ): + def test_program_desc( + self, + ): cost = self.net() main_program = cost.block.program init_program = framework.Program() self.assertEqual(main_program.num_blocks, 1) self.assertEqual(len(cost.block.ops), 3) - self.assertEqual([op.type for op in cost.block.ops], - ["mul", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in cost.block.ops], + ["mul", "elementwise_add", "mean"], + ) opt = optimizer.SGD(learning_rate=1.0) opt = optimizer.GradientMergeOptimizer(opt, k_steps=4) @@ -1222,13 +1334,15 @@ class TestGradientMergeOptimizer(unittest.TestCase): 'elementwise_add', 'elementwise_add', 'conditional_block', - ]) + ], + ) # optimize block self.assertEqual(len(main_program.block(1).ops), 6) self.assertEqual( [op.type for op in main_program.block(1).ops], - ['scale', 'scale', 'sgd', 'sgd', 'fill_constant', 'fill_constant']) + ['scale', 'scale', 'sgd', 'sgd', 'fill_constant', 'fill_constant'], + ) class TestOptimizerDtype(unittest.TestCase): @@ -1238,9 +1352,7 @@ class TestOptimizerDtype(unittest.TestCase): ''' def check_with_dtype(self, dtype): - class MyLayer(paddle.nn.Layer): - def __init__(self, dtype): super(MyLayer, self).__init__() self._w = self.create_parameter([2, 3], dtype=dtype) @@ -1287,12 +1399,14 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): numpy.random.seed(100) class SimpleNet(paddle.nn.Layer): - def __init__(self, input_size, output_size): super(SimpleNet, self).__init__() - self.linears = paddle.nn.LayerList([ - paddle.nn.Linear(input_size, output_size) for i in range(1) - ]) + self.linears = paddle.nn.LayerList( + [ + paddle.nn.Linear(input_size, output_size) + for i in range(1) + ] + ) def forward(self, x): for i, l in enumerate(self.linears): @@ -1305,7 +1419,6 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): nums_batch = 10 class RandomDataset(Dataset): - def __init__(self, num_samples): self.num_samples = num_samples @@ -1318,17 +1431,21 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): return self.num_samples dataset = RandomDataset(nums_batch * batch_size) - loader = paddle.io.DataLoader(dataset, - batch_size=batch_size, - shuffle=False, - drop_last=True, - num_workers=0) + loader = paddle.io.DataLoader( + dataset, + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, + ) mse = paddle.nn.MSELoss() model = SimpleNet(input_size, output_size) # 定义模型 - optimizer = paddle.optimizer.Momentum(learning_rate=0.0001, - parameters=model.parameters(), - multi_precision=True) # 定义优化器 + optimizer = paddle.optimizer.Momentum( + learning_rate=0.0001, + parameters=model.parameters(), + multi_precision=True, + ) # 定义优化器 scaler = paddle.amp.GradScaler(init_loss_scaling=1024) model = paddle.amp.decorate(models=model, level='O2') @@ -1356,9 +1473,11 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): if core.is_compiled_with_cuda(): with fluid.dygraph.guard(): out_use_state_dict = self.check_with_opt_state_dict( - use_save_load=True) + use_save_load=True + ) out_no_state_dict = self.check_with_opt_state_dict( - use_save_load=False) + use_save_load=False + ) np.testing.assert_array_equal(out_use_state_dict, out_no_state_dict) diff --git a/python/paddle/fluid/tests/unittests/test_optimizer_for_varbase.py b/python/paddle/fluid/tests/unittests/test_optimizer_for_varbase.py index 06cdc599f6d..fd73c042cb2 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer_for_varbase.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer_for_varbase.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph class TestOptimizerForVarBase(unittest.TestCase): - def setUp(self): self.lr = 0.01 @@ -32,16 +31,16 @@ class TestOptimizerForVarBase(unittest.TestCase): z = x + y - opt = optimizer(learning_rate=self.lr, - parameters=[x], - weight_decay=0.01) + opt = optimizer( + learning_rate=self.lr, parameters=[x], weight_decay=0.01 + ) z.backward() opt.step() - np.testing.assert_allclose(x.numpy(), - np.full([2, 3], -self.lr), - rtol=1e-05) + np.testing.assert_allclose( + x.numpy(), np.full([2, 3], -self.lr), rtol=1e-05 + ) def run_optimizer_minimize_with_varbase_list_input(self, optimizer): x = paddle.zeros([2, 3]) @@ -55,9 +54,9 @@ class TestOptimizerForVarBase(unittest.TestCase): z.backward() opt.minimize(z) - np.testing.assert_allclose(x.numpy(), - np.full([2, 3], -self.lr), - rtol=1e-05) + np.testing.assert_allclose( + x.numpy(), np.full([2, 3], -self.lr), rtol=1e-05 + ) def func_test_adam_with_varbase_list_input(self): self.run_optimizer_step_with_varbase_list_input(optimizer.Adam) @@ -130,17 +129,20 @@ class TestOptimizerForVarBase(unittest.TestCase): shape=[5, 10], lod_level=0, name="x", - optimize_attr={'learning_rate': 1.0}) + optimize_attr={'learning_rate': 1.0}, + ) else: x = paddle.fluid.framework.EagerParamBase( dtype="float32", shape=[5, 10], lod_level=0, name="x", - optimize_attr={'learning_rate': 1.0}) + optimize_attr={'learning_rate': 1.0}, + ) x.value().get_tensor().set( np.random.random((5, 10)).astype('float32'), - paddle.fluid.framework._current_expected_place()) + paddle.fluid.framework._current_expected_place(), + ) y = paddle.ones([5, 10]) z = x + y @@ -160,17 +162,20 @@ class TestOptimizerForVarBase(unittest.TestCase): shape=[5, 10], lod_level=0, name="x", - optimize_attr={'learning_rate': 0.12}) + optimize_attr={'learning_rate': 0.12}, + ) else: x = paddle.fluid.framework.EagerParamBase( dtype="float32", shape=[5, 10], lod_level=0, name="x", - optimize_attr={'learning_rate': 0.12}) + optimize_attr={'learning_rate': 0.12}, + ) x.value().get_tensor().set( np.random.random((5, 10)).astype('float32'), - paddle.fluid.framework._current_expected_place()) + paddle.fluid.framework._current_expected_place(), + ) y = paddle.ones([5, 10]) z = x + y diff --git a/python/paddle/fluid/tests/unittests/test_optimizer_grad.py b/python/paddle/fluid/tests/unittests/test_optimizer_grad.py index a29fb920d78..87e39a1efb0 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer_grad.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer_grad.py @@ -43,8 +43,8 @@ class SimpleNetWithCond(object): def _init_param(self): self.x = np.ones(self.shape).astype('float32') - self.y = np.ones(self.shape).astype('float32') * 2. - self.z = np.ones(self.shape).astype('float32') * 3. + self.y = np.ones(self.shape).astype('float32') * 2.0 + self.z = np.ones(self.shape).astype('float32') * 3.0 def _calc_gradient(self, cond_i): """ @@ -81,39 +81,42 @@ class SimpleNetWithCond(object): dtype="float32", shape=self.shape, attr=fluid.ParamAttr(learning_rate=self.param_lr, name="param_x"), - default_initializer=fluid.initializer.NumpyArrayInitializer(self.x)) + default_initializer=fluid.initializer.NumpyArrayInitializer(self.x), + ) param_y = fluid.layers.create_parameter( dtype="float32", shape=self.shape, attr=fluid.ParamAttr(learning_rate=self.param_lr, name="param_y"), - default_initializer=fluid.initializer.NumpyArrayInitializer(self.y)) + default_initializer=fluid.initializer.NumpyArrayInitializer(self.y), + ) param_z = fluid.layers.create_parameter( dtype="float32", shape=self.shape, attr=fluid.ParamAttr(learning_rate=self.param_lr, name="param_z"), - default_initializer=fluid.initializer.NumpyArrayInitializer(self.z)) + default_initializer=fluid.initializer.NumpyArrayInitializer(self.z), + ) sum_xy = fluid.layers.elementwise_add(param_x, param_y, name='sum_xy') sub_yz = fluid.layers.elementwise_sub(param_y, param_z, name='sub_yz') useless = fluid.layers.fc(param_x, size=1, name='fc_useless') def cond_true(): - cond_yz = fluid.layers.elementwise_add(param_y, - param_z, - name='sum_cond_yz') + cond_yz = fluid.layers.elementwise_add( + param_y, param_z, name='sum_cond_yz' + ) # param_y will not be updated param_y.stop_gradient = self.y_no_grad - cond_res = fluid.layers.elementwise_add(cond_yz, - param_z, - name='sum_cond_true') + cond_res = fluid.layers.elementwise_add( + cond_yz, param_z, name='sum_cond_true' + ) cond_useless = fluid.layers.elementwise_mul(param_x, param_y) return cond_res def cond_false(): - cond_res = fluid.layers.elementwise_add(param_y, - param_z, - name='sum_cond_false') + cond_res = fluid.layers.elementwise_add( + param_y, param_z, name='sum_cond_false' + ) cond_useless = fluid.layers.elementwise_mul(param_z, param_z) return cond_res @@ -123,18 +126,23 @@ class SimpleNetWithCond(object): mean_out = paddle.mean(sum_all) if use_bf16: import paddle.static.amp as amp + self.optimizer = amp.bf16.decorate_bf16( self.optimizer, amp_lists=amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'elementwise_add'}), + custom_fp32_list={'elementwise_add'} + ), use_bf16_guard=False, - use_pure_bf16=True) + use_pure_bf16=True, + ) self.optimizer.minimize(mean_out) - fetch_list = ["param_x", "param_z"] if self.y_no_grad else [ - "param_x", "param_y", "param_z" - ] + fetch_list = ( + ["param_x", "param_z"] + if self.y_no_grad + else ["param_x", "param_y", "param_z"] + ) fetch_list += [_append_grad_suffix_(param) for param in fetch_list] return fetch_list, self.optimizer @@ -200,8 +208,9 @@ class TestOptimizer(unittest.TestCase): for param_lr in self.param_lr: for cond_i in self.cond_i: for y_no_grad in self.y_no_grad: - self.attr[ - 'lr'] = param_lr * self.optimizer._learning_rate + self.attr['lr'] = ( + param_lr * self.optimizer._learning_rate + ) self._init_param_attr() main_program = fluid.Program() @@ -209,11 +218,15 @@ class TestOptimizer(unittest.TestCase): with fluid.program_guard(main_program, init_program): # reset optimizer._accumulators to avoid duplicate name in loop. self.optimizer._accumulators = defaultdict( - lambda: dict()) - test_net = self.NetClass(self.optimizer, param_lr, - y_no_grad) - fetch_list, decorated_optimizer = test_net.build_net( - cond_i, use_bf16) + lambda: dict() + ) + test_net = self.NetClass( + self.optimizer, param_lr, y_no_grad + ) + ( + fetch_list, + decorated_optimizer, + ) = test_net.build_net(cond_i, use_bf16) if use_bf16: self.optimizer = decorated_optimizer @@ -225,24 +238,28 @@ class TestOptimizer(unittest.TestCase): # Train 2 steps to check validity for batch_i in range(2): - res = exe.run(main_program, - fetch_list=fetch_list) + res = exe.run( + main_program, fetch_list=fetch_list + ) gt_grads = test_net._calc_gradient(cond_i) gt_params = self._apply_optimize( - test_net, gt_grads) + test_net, gt_grads + ) param_grads = gt_params + gt_grads for i in range(len(res)): np.testing.assert_allclose( - res[i], param_grads[i]) + res[i], param_grads[i] + ) -@unittest.skipIf(not fluid.core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not fluid.core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestSGDOptimizer(TestOptimizer): - def test_optimizer_multiblock_except(self): - with self.assertRaisesRegexp(ValueError, - "var param_y not in this block"): + with self.assertRaisesRegexp( + ValueError, "var param_y not in this block" + ): self._check_grads(use_bf16=True) @@ -256,10 +273,9 @@ class TestAdamOptimizer(TestOptimizer): def setUp(self): self._init_config() beta1, beta2, epsilon = 0.9, 0.999, 1e-8 - self.optimizer = optimizer.AdamOptimizer(learning_rate=0.01, - beta1=beta1, - beta2=beta2, - epsilon=epsilon) + self.optimizer = optimizer.AdamOptimizer( + learning_rate=0.01, beta1=beta1, beta2=beta2, epsilon=epsilon + ) self.attr = { "beta1": beta1, "beta2": beta2, @@ -267,7 +283,7 @@ class TestAdamOptimizer(TestOptimizer): "beta2_pow": beta2, "moment1": np.zeros(SHAPE).astype("float32"), "moment2": np.zeros(SHAPE).astype("float32"), - "epsilon": epsilon + "epsilon": epsilon, } def _apply_gradient(self, param, grad, name): @@ -280,13 +296,14 @@ class TestAdamOptimizer(TestOptimizer): beta1_pow, beta2_pow = attr['beta1_pow'], attr['beta2_pow'] epsilon = attr['epsilon'] - moment1_out = beta1 * moment1 + (1. - beta1) * grad - moment2_out = beta2 * moment2 + (1. - beta2) * np.square(grad) + moment1_out = beta1 * moment1 + (1.0 - beta1) * grad + moment2_out = beta2 * moment2 + (1.0 - beta2) * np.square(grad) - lr = attr['lr'] * np.sqrt(1. - beta2_pow) / (1. - beta1_pow) + lr = attr['lr'] * np.sqrt(1.0 - beta2_pow) / (1.0 - beta1_pow) param_out = param - lr * ( - moment1_out / - (np.sqrt(moment2_out) + epsilon * np.sqrt(1 - beta2_pow))) + moment1_out + / (np.sqrt(moment2_out) + epsilon * np.sqrt(1 - beta2_pow)) + ) # update hyper-parameter of optimizer self.param_attr[name]['beta1_pow'] = beta1_pow * beta1 diff --git a/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py b/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py index 60348330284..8bc2b796758 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py @@ -34,10 +34,9 @@ SEED = 2020 paddle.enable_static() -def static(train_data, - loss_in_switch=True, - use_cuda=False, - use_parallel_exe=False): +def static( + train_data, loss_in_switch=True, use_cuda=False, use_parallel_exe=False +): startup_program = Program() main_program = Program() startup_program.random_seed = SEED @@ -51,20 +50,26 @@ def static(train_data, size=FC_SIZE, act='relu', param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.99)), + initializer=fluid.initializer.Constant(value=0.99) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.5)), - name="hidden") + initializer=fluid.initializer.Constant(value=0.5) + ), + name="hidden", + ) prediction = layers.fc( hidden, size=CLASS_NUM, act='softmax', param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.2)), + initializer=fluid.initializer.Constant(value=1.2) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.8)), - name="prediction") + initializer=fluid.initializer.Constant(value=0.8) + ), + name="prediction", + ) return hidden, prediction def fn_1(opt, avg_loss=None, pred=None, label=None): @@ -76,8 +81,9 @@ def static(train_data, def fn_2(opt, avg_loss=None, pred=None, label=None): if avg_loss is None: - loss = layers.softmax_with_cross_entropy(logits=pred, - label=label) + loss = layers.softmax_with_cross_entropy( + logits=pred, label=label + ) avg_loss = paddle.mean(loss, name='mean_softmax_loss') opt.minimize(avg_loss) return avg_loss @@ -96,15 +102,19 @@ def static(train_data, if loss_in_switch: avg_loss = layers.case( [(mod_two, lambda: fn_1(adam, None, prediction, label))], - lambda: fn_2(sgd, None, prediction, label)) + lambda: fn_2(sgd, None, prediction, label), + ) else: loss_1 = layers.cross_entropy(input=prediction, label=label) avg_loss_1 = paddle.mean(loss_1) - loss_2 = layers.softmax_with_cross_entropy(logits=prediction, - label=label) + loss_2 = layers.softmax_with_cross_entropy( + logits=prediction, label=label + ) avg_loss_2 = paddle.mean(loss_2) - avg_loss = layers.case([(mod_two, lambda: fn_1(adam, avg_loss_1))], - lambda: fn_2(sgd, avg_loss_2)) + avg_loss = layers.case( + [(mod_two, lambda: fn_1(adam, avg_loss_1))], + lambda: fn_2(sgd, avg_loss_2), + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -116,7 +126,7 @@ def static(train_data, feed = { 'image': feed_image, 'label': feed_label, - 'id': np.array([epoch]).astype('int32') + 'id': np.array([epoch]).astype('int32'), } out = exe.run(main_program, feed=feed, fetch_list=fetch_list) out_hidden, out_pred, loss = out @@ -125,27 +135,31 @@ def static(train_data, class DygraphLayer(fluid.dygraph.Layer): - def __init__(self): super(DygraphLayer, self).__init__() self.fc_1 = fluid.dygraph.nn.Linear( INPUT_SIZE, FC_SIZE, act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) self.fc_2 = fluid.dygraph.nn.Linear( FC_SIZE, CLASS_NUM, act='softmax', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.2)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.2) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ), + ) def forward(self, inputs): hidden = self.fc_1(inputs) @@ -159,10 +173,12 @@ def dynamic(train_data, use_cuda=False, use_parallel_exe=False): fluid.default_startup_program().random_seed = SEED fluid.default_main_program().random_seed = SEED dy_layer = DygraphLayer() - adam = fluid.optimizer.Adam(learning_rate=LR, - parameter_list=dy_layer.parameters()) - sgd = fluid.optimizer.SGD(learning_rate=LR, - parameter_list=dy_layer.parameters()) + adam = fluid.optimizer.Adam( + learning_rate=LR, parameter_list=dy_layer.parameters() + ) + sgd = fluid.optimizer.SGD( + learning_rate=LR, parameter_list=dy_layer.parameters() + ) for epoch in range(EPOCH_NUM): image_data, label = train_data[epoch] @@ -177,7 +193,8 @@ def dynamic(train_data, use_cuda=False, use_parallel_exe=False): adam.minimize(loss) else: softmax_loss = layers.softmax_with_cross_entropy( - prediction, var_label) + prediction, var_label + ) loss = paddle.mean(softmax_loss) loss.backward() sgd.minimize(loss) @@ -192,16 +209,18 @@ class TestMultiTask(unittest.TestCase): Todo(liym27): add parallel GPU train. ''' - def random_input(self, - seed, - image_shape=[BATCH_SIZE, INPUT_SIZE], - label_shape=[BATCH_SIZE, 1]): + def random_input( + self, + seed, + image_shape=[BATCH_SIZE, INPUT_SIZE], + label_shape=[BATCH_SIZE, 1], + ): np.random.seed(seed) image_np = np.random.random(size=image_shape).astype('float32') np.random.seed(seed) - label_np = np.random.randint(low=0, - high=CLASS_NUM - 1, - size=label_shape).astype('int64') + label_np = np.random.randint( + low=0, high=CLASS_NUM - 1, size=label_shape + ).astype('int64') return image_np, label_np def init_train_data(self): @@ -214,15 +233,15 @@ class TestMultiTask(unittest.TestCase): use_cuda = core.is_compiled_with_cuda() hidden_2, pre_2, loss_2 = dynamic(self.train_data, use_cuda) for loss_in_switch in [True, False]: - hidden_1, pre_1, loss_1 = static(self.train_data, loss_in_switch, - use_cuda) + hidden_1, pre_1, loss_1 = static( + self.train_data, loss_in_switch, use_cuda + ) np.testing.assert_allclose(hidden_1, hidden_2, rtol=1e-05) np.testing.assert_allclose(pre_1, pre_2, rtol=1e-05) np.testing.assert_allclose(loss_1, loss_2, rtol=1e-05) class TestMultiOptimizersMultiCardsError(unittest.TestCase): - def test_error(self): startup_program = Program() main_program = Program() @@ -244,8 +263,10 @@ class TestMultiOptimizersMultiCardsError(unittest.TestCase): cond = layers.fill_constant([1], 'bool', True) - layers.case([(cond, lambda: fn_1(adam, avg_loss))], - lambda: fn_2(sgd, avg_loss)) + layers.case( + [(cond, lambda: fn_1(adam, avg_loss))], + lambda: fn_2(sgd, avg_loss), + ) cpu_place = fluid.CPUPlace() cuda_place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -263,17 +284,20 @@ class TestMultiOptimizersMultiCardsError(unittest.TestCase): # to use multi cards ** only on CPU ** not GPU to reduce CI time. os.environ['CPU_NUM'] = str(2) - pe_exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=main_program, - loss_name=avg_loss.name) + pe_exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=main_program, + loss_name=avg_loss.name, + ) num_devices = pe_exe.device_count def not_implemented_error(): - pe_exe.run(feed={ - 'X': - np.random.random(size=[64, 10]).astype('float32'), - }, - fetch_list=[avg_loss.name]) + pe_exe.run( + feed={ + 'X': np.random.random(size=[64, 10]).astype('float32'), + }, + fetch_list=[avg_loss.name], + ) if num_devices > 1: self.assertRaises(NotImplementedError, not_implemented_error) diff --git a/python/paddle/fluid/tests/unittests/test_outer.py b/python/paddle/fluid/tests/unittests/test_outer.py index a887615f0f3..5625618776c 100644 --- a/python/paddle/fluid/tests/unittests/test_outer.py +++ b/python/paddle/fluid/tests/unittests/test_outer.py @@ -22,27 +22,28 @@ from paddle.fluid.framework import _test_eager_guard class TestMultiplyApi(unittest.TestCase): - def _run_static_graph_case(self, x_data, y_data): with program_guard(Program(), Program()): paddle.enable_static() - x = paddle.static.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = paddle.static.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = paddle.static.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = paddle.static.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) res = paddle.outer(x, y) - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - outs = exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[res]) + outs = exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[res], + ) res = outs[0] return res @@ -94,17 +95,21 @@ class TestMultiplyApi(unittest.TestCase): # test dynamic computation graph: 2-d array Complex x_data = np.random.rand(20, 50).astype( - np.float64) + 1J * np.random.rand(20, 50).astype(np.float64) - y_data = np.random.rand(50).astype( - np.float64) + 1J * np.random.rand(50).astype(np.float64) + np.float64 + ) + 1j * np.random.rand(20, 50).astype(np.float64) + y_data = np.random.rand(50).astype(np.float64) + 1j * np.random.rand( + 50 + ).astype(np.float64) res = self._run_dynamic_graph_case(x_data, y_data) np.testing.assert_allclose(res, np.outer(x_data, y_data), rtol=1e-05) # test dynamic computation graph: 3-d array Complex x_data = np.random.rand(5, 10, 10).astype( - np.float64) + 1J * np.random.rand(5, 10, 10).astype(np.float64) - y_data = np.random.rand(2, 10).astype( - np.float64) + 1J * np.random.rand(2, 10).astype(np.float64) + np.float64 + ) + 1j * np.random.rand(5, 10, 10).astype(np.float64) + y_data = np.random.rand(2, 10).astype(np.float64) + 1j * np.random.rand( + 2, 10 + ).astype(np.float64) res = self._run_dynamic_graph_case(x_data, y_data) np.testing.assert_allclose(res, np.outer(x_data, y_data), rtol=1e-05) @@ -115,7 +120,6 @@ class TestMultiplyApi(unittest.TestCase): class TestMultiplyError(unittest.TestCase): - def func_test_errors(self): # test static computation graph: dtype can not be int8 paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_overlap_add_op.py b/python/paddle/fluid/tests/unittests/test_overlap_add_op.py index f3815ba4965..430fde53ace 100644 --- a/python/paddle/fluid/tests/unittests/test_overlap_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_overlap_add_op.py @@ -33,8 +33,9 @@ def overlap_add(x, hop_length, axis=-1): frame_length = x.shape[1] if axis == 0 else x.shape[-2] # Assure no gaps between frames. - assert 0 < hop_length <= frame_length, \ - f'hop_length should be in (0, frame_length({frame_length})], but got {hop_length}.' + assert ( + 0 < hop_length <= frame_length + ), f'hop_length should be in (0, frame_length({frame_length})], but got {hop_length}.' seq_length = (n_frames - 1) * hop_length + frame_length @@ -55,7 +56,7 @@ def overlap_add(x, hop_length, axis=-1): for i in range(x.shape[0]): for frame in range(x.shape[-1]): sample = frame * hop_length - y[i, sample:sample + frame_length] += x[i, :, frame] + y[i, sample : sample + frame_length] += x[i, :, frame] if axis == 0: y = y.transpose((1, 0)) @@ -70,7 +71,6 @@ def overlap_add(x, hop_length, axis=-1): class TestOverlapAddOp(OpTest): - def setUp(self): self.op_type = "overlap_add" self.python_api = paddle.signal.overlap_add @@ -101,7 +101,6 @@ class TestOverlapAddOp(OpTest): class TestCase1(TestOverlapAddOp): - def initTestCase(self): input_shape = (3, 50) input_type = 'float64' @@ -113,7 +112,6 @@ class TestCase1(TestOverlapAddOp): class TestCase2(TestOverlapAddOp): - def initTestCase(self): input_shape = (2, 40, 5) input_type = 'float64' @@ -125,7 +123,6 @@ class TestCase2(TestOverlapAddOp): class TestCase3(TestOverlapAddOp): - def initTestCase(self): input_shape = (5, 40, 2) input_type = 'float64' @@ -137,7 +134,6 @@ class TestCase3(TestOverlapAddOp): class TestCase4(TestOverlapAddOp): - def initTestCase(self): input_shape = (3, 5, 12, 8) input_type = 'float64' @@ -149,7 +145,6 @@ class TestCase4(TestOverlapAddOp): class TestCase5(TestOverlapAddOp): - def initTestCase(self): input_shape = (8, 12, 5, 3) input_type = 'float64' diff --git a/python/paddle/fluid/tests/unittests/test_pad2d_op.py b/python/paddle/fluid/tests/unittests/test_pad2d_op.py index 0f43ddbd8fc..aeca9becf84 100644 --- a/python/paddle/fluid/tests/unittests/test_pad2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pad2d_op.py @@ -20,7 +20,6 @@ from paddle.fluid import Program, program_guard class TestPad2dOp(OpTest): - def setUp(self): self.pad_value = 0.0 self.variable_paddings = False @@ -30,25 +29,37 @@ class TestPad2dOp(OpTest): self.attrs = {} if self.variable_paddings: self.attrs['paddings'] = [] - self.inputs['Paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.inputs['Paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) else: - self.attrs['paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.attrs['paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) self.attrs['pad_value'] = self.pad_value self.attrs['mode'] = self.mode self.attrs['data_format'] = self.data_format if self.data_format == "NCHW": - paddings = [(0, 0), (0, 0), (self.paddings[0], self.paddings[1]), - (self.paddings[2], self.paddings[3])] + paddings = [ + (0, 0), + (0, 0), + (self.paddings[0], self.paddings[1]), + (self.paddings[2], self.paddings[3]), + ] else: - paddings = [(0, 0), (self.paddings[0], self.paddings[1]), - (self.paddings[2], self.paddings[3]), (0, 0)] + paddings = [ + (0, 0), + (self.paddings[0], self.paddings[1]), + (self.paddings[2], self.paddings[3]), + (0, 0), + ] if self.mode == "constant": - out = np.pad(self.inputs['X'], - paddings, - mode=self.mode, - constant_values=self.pad_value) + out = np.pad( + self.inputs['X'], + paddings, + mode=self.mode, + constant_values=self.pad_value, + ) else: out = np.pad(self.inputs['X'], paddings, mode=self.mode) self.outputs = {'Out': out} @@ -68,7 +79,6 @@ class TestPad2dOp(OpTest): class TestCase1(TestPad2dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [0, 1, 2, 3] @@ -77,7 +87,6 @@ class TestCase1(TestPad2dOp): class TestCase2(TestPad2dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [0, 1, 2, 3] @@ -86,7 +95,6 @@ class TestCase2(TestPad2dOp): class TestCase3(TestPad2dOp): - def initTestCase(self): self.shape = (2, 4, 4, 4) self.paddings = [0, 1, 2, 3] @@ -95,7 +103,6 @@ class TestCase3(TestPad2dOp): class TestCase4(TestPad2dOp): - def initTestCase(self): self.shape = (2, 4, 4, 4) self.paddings = [0, 1, 2, 3] @@ -104,7 +111,6 @@ class TestCase4(TestPad2dOp): class TestCase5(TestPad2dOp): - def initTestCase(self): self.shape = (2, 4, 4, 4) self.paddings = [0, 1, 2, 3] @@ -114,7 +120,6 @@ class TestCase5(TestPad2dOp): class TestCase6(TestPad2dOp): - def initTestCase(self): self.shape = (2, 4, 4, 4) self.paddings = [0, 1, 2, 3] @@ -125,7 +130,6 @@ class TestCase6(TestPad2dOp): class TestCase7(TestPad2dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [0, 1, 2, 3] @@ -135,7 +139,6 @@ class TestCase7(TestPad2dOp): class TestPad2dOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((2, 2, 2, 2)).astype("float32") @@ -145,9 +148,9 @@ class TestPad2dOpError(unittest.TestCase): self.assertRaises(TypeError, test_Variable) - data = fluid.data(name='data', - shape=[None, 3, 20, 20], - dtype='float16') + data = fluid.data( + name='data', shape=[None, 3, 20, 20], dtype='float16' + ) fluid.layers.pad2d(input=data, paddings=[1, 1, 1, 1]) diff --git a/python/paddle/fluid/tests/unittests/test_pad3d_op.py b/python/paddle/fluid/tests/unittests/test_pad3d_op.py index c1c0aa7defb..ee4a9aed670 100644 --- a/python/paddle/fluid/tests/unittests/test_pad3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pad3d_op.py @@ -24,7 +24,6 @@ from paddle.fluid import Program, program_guard, Executor, default_main_program class TestPad3dOp(OpTest): - def setUp(self): paddle.enable_static() self.value = 0.0 @@ -35,11 +34,13 @@ class TestPad3dOp(OpTest): self.attrs = {} if self.variable_paddings: self.attrs['paddings'] = [] - self.inputs['Paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.inputs['Paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) else: - self.attrs['paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.attrs['paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) self.attrs['value'] = self.value self.attrs['mode'] = self.mode self.attrs['data_format'] = self.data_format @@ -60,10 +61,12 @@ class TestPad3dOp(OpTest): (0, 0), ] if self.mode == "constant": - out = np.pad(self.inputs['X'], - paddings, - mode=self.mode, - constant_values=self.value) + out = np.pad( + self.inputs['X'], + paddings, + mode=self.mode, + constant_values=self.value, + ) elif self.mode == "reflect": out = np.pad(self.inputs['X'], paddings, mode=self.mode) elif self.mode == "replicate": @@ -88,7 +91,6 @@ class TestPad3dOp(OpTest): class TestCase1(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 4, 5] @@ -99,7 +101,6 @@ class TestCase1(TestPad3dOp): class TestCase2(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [1, 1, 1, 1, 1, 1] @@ -110,7 +111,6 @@ class TestCase2(TestPad3dOp): class TestCase3(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 1, 0, 2, 3] @@ -120,7 +120,6 @@ class TestCase3(TestPad3dOp): class TestCase4(TestPad3dOp): - def initTestCase(self): self.shape = (4, 4, 4, 4, 4) self.paddings = [0, 1, 2, 1, 2, 3] @@ -130,7 +129,6 @@ class TestCase4(TestPad3dOp): class TestCase5(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 2, 1] @@ -140,7 +138,6 @@ class TestCase5(TestPad3dOp): class TestCase6(TestPad3dOp): - def initTestCase(self): self.shape = (4, 4, 4, 4, 4) self.paddings = [5, 4, 2, 1, 2, 3] @@ -150,7 +147,6 @@ class TestCase6(TestPad3dOp): class TestCase7(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 2, 1] @@ -160,7 +156,6 @@ class TestCase7(TestPad3dOp): class TestCase8(TestPad3dOp): - def initTestCase(self): self.shape = (4, 4, 4, 4, 4) self.paddings = [0, 1, 2, 1, 2, 3] @@ -170,7 +165,6 @@ class TestCase8(TestPad3dOp): class TestCase9(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 4, 5] @@ -181,7 +175,6 @@ class TestCase9(TestPad3dOp): class TestCase10(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 4, 5] @@ -192,7 +185,6 @@ class TestCase10(TestPad3dOp): class TestPadAPI(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if core.is_compiled_with_cuda(): @@ -207,15 +199,15 @@ class TestPadAPI(unittest.TestCase): value = 100 input_data = np.random.rand(*input_shape).astype(np.float32) x = paddle.fluid.data(name="x", shape=input_shape) - result = F.pad(x=x, - pad=pad, - value=value, - mode=mode, - data_format="NCDHW") + result = F.pad( + x=x, pad=pad, value=value, mode=mode, data_format="NCDHW" + ) exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result]) + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result], + ) np_out = self._get_numpy_out(input_data, pad, mode, value) np.testing.assert_allclose(fetches[0], np_out, rtol=1e-05) @@ -231,18 +223,18 @@ class TestPadAPI(unittest.TestCase): result1 = F.pad(x=x, pad=pad, mode=mode, data_format="NCDHW") result2 = F.pad(x=x, pad=pad, mode=mode, data_format="NDHWC") exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result1, result2]) - - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NDHWC") + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result1, result2], + ) + + np_out1 = self._get_numpy_out( + input_data, pad, mode, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, data_format="NDHWC" + ) np.testing.assert_allclose(fetches[0], np_out1, rtol=1e-05) np.testing.assert_allclose(fetches[1], np_out2, rtol=1e-05) @@ -257,18 +249,18 @@ class TestPadAPI(unittest.TestCase): result1 = F.pad(x=x, pad=pad, mode=mode, data_format="NCDHW") result2 = F.pad(x=x, pad=pad, mode=mode, data_format="NDHWC") exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result1, result2]) - - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NDHWC") + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result1, result2], + ) + + np_out1 = self._get_numpy_out( + input_data, pad, mode, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, data_format="NDHWC" + ) np.testing.assert_allclose(fetches[0], np_out1, rtol=1e-05) np.testing.assert_allclose(fetches[1], np_out2, rtol=1e-05) @@ -283,27 +275,24 @@ class TestPadAPI(unittest.TestCase): result1 = F.pad(x=x, pad=pad, mode=mode, data_format="NCDHW") result2 = F.pad(x=x, pad=pad, mode=mode, data_format="NDHWC") exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result1, result2]) - - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NDHWC") + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result1, result2], + ) + + np_out1 = self._get_numpy_out( + input_data, pad, mode, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, data_format="NDHWC" + ) np.testing.assert_allclose(fetches[0], np_out1, rtol=1e-05) np.testing.assert_allclose(fetches[1], np_out2, rtol=1e-05) - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0, - data_format="NCDHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0, data_format="NCDHW" + ): if mode == "constant" and len(pad) == len(input_data.shape) * 2: pad = np.reshape(pad, (-1, 2)).tolist() elif data_format == "NCDHW": @@ -375,38 +364,26 @@ class TestPadAPI(unittest.TestCase): mode = "constant" value = 100 input_data = np.random.rand(*input_shape).astype(np.float32) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NDHWC") - np_out3 = self._get_numpy_out(input_data, - pad_3, - mode, - value, - data_format="NCDHW") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NDHWC" + ) + np_out3 = self._get_numpy_out( + input_data, pad_3, mode, value, data_format="NCDHW" + ) tensor_data = paddle.to_tensor(input_data) - y1 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NCDHW") - y2 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NDHWC") - y3 = F.pad(tensor_data, - pad=pad_3, - mode=mode, - value=value, - data_format="NCDHW") + y1 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NCDHW" + ) + y2 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NDHWC" + ) + y3 = F.pad( + tensor_data, pad=pad_3, mode=mode, value=value, data_format="NCDHW" + ) np.testing.assert_allclose(y1.numpy(), np_out1, rtol=1e-05) np.testing.assert_allclose(y2.numpy(), np_out2, rtol=1e-05) @@ -420,40 +397,36 @@ class TestPadAPI(unittest.TestCase): mode = "constant" value = 100 input_data = np.random.rand(*input_shape).astype(np.float32) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NHWC") - np_out3 = self._get_numpy_out(input_data, - pad_3, - mode, - value, - data_format="NCHW") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NHWC" + ) + np_out3 = self._get_numpy_out( + input_data, pad_3, mode, value, data_format="NCHW" + ) tensor_data = paddle.to_tensor(input_data) tensor_pad = paddle.to_tensor(pad, dtype="int32") - y1 = F.pad(tensor_data, - pad=tensor_pad, - mode=mode, - value=value, - data_format="NCHW") - y2 = F.pad(tensor_data, - pad=tensor_pad, - mode=mode, - value=value, - data_format="NHWC") - y3 = F.pad(tensor_data, - pad=pad_3, - mode=mode, - value=value, - data_format="NCHW") + y1 = F.pad( + tensor_data, + pad=tensor_pad, + mode=mode, + value=value, + data_format="NCHW", + ) + y2 = F.pad( + tensor_data, + pad=tensor_pad, + mode=mode, + value=value, + data_format="NHWC", + ) + y3 = F.pad( + tensor_data, pad=pad_3, mode=mode, value=value, data_format="NCHW" + ) np.testing.assert_allclose(y1.numpy(), np_out1, rtol=1e-05) np.testing.assert_allclose(y2.numpy(), np_out2, rtol=1e-05) @@ -467,39 +440,35 @@ class TestPadAPI(unittest.TestCase): mode = "constant" value = 100 input_data = np.random.rand(*input_shape).astype(np.float32) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCL") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NLC") - np_out3 = self._get_numpy_out(input_data, - pad_3, - mode, - value, - data_format="NCL") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCL" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NLC" + ) + np_out3 = self._get_numpy_out( + input_data, pad_3, mode, value, data_format="NCL" + ) tensor_data = paddle.to_tensor(input_data) tensor_pad = paddle.to_tensor(pad, dtype="int32") - y1 = F.pad(tensor_data, - pad=tensor_pad, - mode=mode, - value=value, - data_format="NCL") - y2 = F.pad(tensor_data, - pad=tensor_pad, - mode=mode, - value=value, - data_format="NLC") - y3 = F.pad(tensor_data, - pad=pad_3, - mode=mode, - value=value, - data_format="NCL") + y1 = F.pad( + tensor_data, + pad=tensor_pad, + mode=mode, + value=value, + data_format="NCL", + ) + y2 = F.pad( + tensor_data, + pad=tensor_pad, + mode=mode, + value=value, + data_format="NLC", + ) + y3 = F.pad( + tensor_data, pad=pad_3, mode=mode, value=value, data_format="NCL" + ) np.testing.assert_allclose(y1.numpy(), np_out1, rtol=1e-05) np.testing.assert_allclose(y2.numpy(), np_out2, rtol=1e-05) @@ -507,13 +476,9 @@ class TestPadAPI(unittest.TestCase): class TestPad1dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCL"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCL" + ): if data_format == "NCL": pad = [ (0, 0), @@ -555,59 +520,52 @@ class TestPad1dAPI(unittest.TestCase): pad_reflection = nn.Pad1D(padding=pad, mode="reflect") pad_replication = nn.Pad1D(padding=pad, mode="replicate") pad_constant = nn.Pad1D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad1D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad1D( + padding=pad_int, mode="constant", value=value + ) pad_circular = nn.Pad1D(padding=pad, mode="circular") data = paddle.to_tensor(input_data) output = pad_reflection(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_replication(data) - np_out = self._get_numpy_out(input_data, - pad, - "replicate", - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "replicate", data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 2, - "constant", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 2, + "constant", + value=value, + data_format="NCL", + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_circular(data) - np_out = self._get_numpy_out(input_data, - pad, - "circular", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "circular", value=value, data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) class TestPad2dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCHW" + ): if data_format == "NCHW": pad = [ (0, 0), @@ -651,58 +609,52 @@ class TestPad2dAPI(unittest.TestCase): pad_reflection = nn.Pad2D(padding=pad, mode="reflect") pad_replication = nn.Pad2D(padding=pad, mode="replicate") pad_constant = nn.Pad2D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad2D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad2D( + padding=pad_int, mode="constant", value=value + ) pad_circular = nn.Pad2D(padding=pad, mode="circular") data = paddle.to_tensor(input_data) output = pad_reflection(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_replication(data) - np_out = self._get_numpy_out(input_data, - pad, - "replicate", - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "replicate", data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 4, - "constant", - value=value, - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 4, + "constant", + value=value, + data_format="NCHW", + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_circular(data) - np_out = self._get_numpy_out(input_data, - pad, - "circular", - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "circular", data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) class TestPad3dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCDHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCDHW" + ): if data_format == "NCDHW": pad = [ (0, 0), @@ -748,47 +700,45 @@ class TestPad3dAPI(unittest.TestCase): pad_reflection = nn.Pad3D(padding=pad, mode="reflect") pad_replication = nn.Pad3D(padding=pad, mode="replicate") pad_constant = nn.Pad3D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad3D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad3D( + padding=pad_int, mode="constant", value=value + ) pad_circular = nn.Pad3D(padding=pad, mode="circular") data = paddle.to_tensor(input_data) output = pad_reflection(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_replication(data) - np_out = self._get_numpy_out(input_data, - pad, - "replicate", - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "replicate", data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 6, - "constant", - value=value, - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 6, + "constant", + value=value, + data_format="NCDHW", + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_circular(data) - np_out = self._get_numpy_out(input_data, - pad, - "circular", - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "circular", data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) def test_pad_tensor(self): @@ -799,38 +749,34 @@ class TestPad3dAPI(unittest.TestCase): pad_tensor = paddle.to_tensor(pad) input_data = np.random.rand(*input_shape).astype(np.float32) - pad_reflection_ncdhw = nn.Pad3D(padding=pad_tensor, - mode="reflect", - data_format="NCDHW") - pad_reflection_ndhwc = nn.Pad3D(padding=pad_tensor, - mode="reflect", - data_format="NDHWC") + pad_reflection_ncdhw = nn.Pad3D( + padding=pad_tensor, mode="reflect", data_format="NCDHW" + ) + pad_reflection_ndhwc = nn.Pad3D( + padding=pad_tensor, mode="reflect", data_format="NDHWC" + ) data = paddle.to_tensor(input_data) output = pad_reflection_ncdhw(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_reflection_ndhwc(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NDHWC") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NDHWC" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) class TestPad3dOpError(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if core.is_compiled_with_cuda(): self.places.append(paddle.CUDAPlace(0)) def test_errors(self): - def test_variable(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) @@ -840,49 +786,53 @@ class TestPad3dOpError(unittest.TestCase): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[5, 6, 1, 1, 1, 1], - value=1, - mode='reflect', - data_format="NCDHW") + y = F.pad( + x, + pad=[5, 6, 1, 1, 1, 1], + value=1, + mode='reflect', + data_format="NCDHW", + ) def test_reflect_2(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[1, 1, 4, 3, 1, 1], - value=1, - mode='reflect', - data_format="NCDHW") + y = F.pad( + x, + pad=[1, 1, 4, 3, 1, 1], + value=1, + mode='reflect', + data_format="NCDHW", + ) def test_reflect_3(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[1, 1, 1, 1, 2, 3], - value=1, - mode='reflect', - data_format="NCDHW") + y = F.pad( + x, + pad=[1, 1, 1, 1, 2, 3], + value=1, + mode='reflect', + data_format="NCDHW", + ) def test_circular_1(): input_shape = (1, 2, 0, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[1, 1, 1, 1, 2, 3], - mode='circular', - data_format="NCDHW") + y = F.pad( + x, pad=[1, 1, 1, 1, 2, 3], mode='circular', data_format="NCDHW" + ) def test_replicate_1(): input_shape = (1, 2, 0, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[1, 1, 1, 1, 2, 3], - mode='replicate', - data_format="NCDHW") + y = F.pad( + x, pad=[1, 1, 1, 1, 2, 3], mode='replicate', data_format="NCDHW" + ) paddle.disable_static() for place in self.places: @@ -896,14 +846,16 @@ class TestPad3dOpError(unittest.TestCase): class TestPadDataformatError(unittest.TestCase): - def test_errors(self): - def test_ncl(): input_shape = (1, 2, 3, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) my_pad = nn.Pad1D(padding=pad, mode="replicate", data_format="NCL") data = paddle.to_tensor(data) result = my_pad(data) @@ -911,8 +863,12 @@ class TestPadDataformatError(unittest.TestCase): def test_nchw(): input_shape = (1, 2, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) my_pad = nn.Pad1D(padding=pad, mode="replicate", data_format="NCHW") data = paddle.to_tensor(data) result = my_pad(data) @@ -920,11 +876,15 @@ class TestPadDataformatError(unittest.TestCase): def test_ncdhw(): input_shape = (1, 2, 3, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 - my_pad = nn.Pad1D(padding=pad, - mode="replicate", - data_format="NCDHW") + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) + my_pad = nn.Pad1D( + padding=pad, mode="replicate", data_format="NCDHW" + ) data = paddle.to_tensor(data) result = my_pad(data) diff --git a/python/paddle/fluid/tests/unittests/test_pad_constant_like.py b/python/paddle/fluid/tests/unittests/test_pad_constant_like.py index 42fad15c97f..fda844eb09b 100644 --- a/python/paddle/fluid/tests/unittests/test_pad_constant_like.py +++ b/python/paddle/fluid/tests/unittests/test_pad_constant_like.py @@ -20,22 +20,22 @@ from paddle.fluid import Program, program_guard class TestPadConstantLikeOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = "pad_constant_like" self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'Y': np.random.random(self.y_shape).astype("float64") + 'Y': np.random.random(self.y_shape).astype("float64"), } self.attrs = {} self.attrs['pad_value'] = self.pad_value self.outputs = { - 'Out': - np.pad(self.inputs['Y'], - self.paddings, - mode='constant', - constant_values=self.pad_value) + 'Out': np.pad( + self.inputs['Y'], + self.paddings, + mode='constant', + constant_values=self.pad_value, + ) } def test_check_output(self): @@ -52,7 +52,6 @@ class TestPadConstantLikeOp(OpTest): class TestCase1(TestPadConstantLikeOp): - def initTestCase(self): self.x_shape = (4, 3, 4, 5) self.y_shape = (2, 3, 4, 5) @@ -61,7 +60,6 @@ class TestCase1(TestPadConstantLikeOp): class TestCase2(TestPadConstantLikeOp): - def initTestCase(self): self.x_shape = (4, 3, 4, 10) self.y_shape = (2, 3, 2, 10) @@ -70,38 +68,38 @@ class TestCase2(TestPadConstantLikeOp): class TestPadConstantLikeOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x_data = np.random.random((2, 2, 2, 2)).astype("float32") y_data = np.random.random((2, 2, 2, 2)).astype("float32") def test_Variable_x(): - var_y = fluid.data(name="data_y", - shape=[2, 2, 2, 2], - dtype="float32") + var_y = fluid.data( + name="data_y", shape=[2, 2, 2, 2], dtype="float32" + ) fluid.layers.pad_constant_like(x=x_data, y=var_y) self.assertRaises(TypeError, test_Variable_x) def test_Variable_y(): - var_x = fluid.data(name="data_x", - shape=[2, 2, 2, 2], - dtype="float32") + var_x = fluid.data( + name="data_x", shape=[2, 2, 2, 2], dtype="float32" + ) fluid.layers.pad_constant_like(x=var_x, y=y_data) self.assertRaises(TypeError, test_Variable_y) class TestOutDtype(unittest.TestCase): - def test_dtype(self): api_fn = fluid.layers.pad_constant_like - check_out_dtype(api_fn, - in_specs=[([2, 3, 2, 3], 'float64'), ([1, 3, 1, 3], )], - expect_dtypes=['float32', 'float64', 'int32', 'int64'], - target_index=1, - pad_value=0.) + check_out_dtype( + api_fn, + in_specs=[([2, 3, 2, 3], 'float64'), ([1, 3, 1, 3],)], + expect_dtypes=['float32', 'float64', 'int32', 'int64'], + target_index=1, + pad_value=0.0, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_pad_op.py b/python/paddle/fluid/tests/unittests/test_pad_op.py index fa31836ecf3..916f0399d9f 100644 --- a/python/paddle/fluid/tests/unittests/test_pad_op.py +++ b/python/paddle/fluid/tests/unittests/test_pad_op.py @@ -25,7 +25,6 @@ from test_attribute_var import UnittestBase class TestPadOp(OpTest): - def setUp(self): self.initTestCase() self.dtype = self.get_dtype() @@ -37,11 +36,12 @@ class TestPadOp(OpTest): self.attrs['paddings'] = np.array(self.paddings).flatten() self.attrs['pad_value'] = self.pad_value self.outputs = { - 'Out': - np.pad(self.inputs['X'], - self.paddings, - mode='constant', - constant_values=self.pad_value) + 'Out': np.pad( + self.inputs['X'], + self.paddings, + mode='constant', + constant_values=self.pad_value, + ) } def get_dtype(self): @@ -60,7 +60,6 @@ class TestPadOp(OpTest): class TestCase1(TestPadOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [(0, 1), (2, 3), (2, 1), (1, 1)] @@ -68,7 +67,6 @@ class TestCase1(TestPadOp): class TestCase2(TestPadOp): - def initTestCase(self): self.shape = (5, 5, 5) self.paddings = [(0, 0), (0, 0), (1, 2)] @@ -76,22 +74,20 @@ class TestCase2(TestPadOp): class TestCase3(TestPadOp): - def initTestCase(self): - self.shape = (100) + self.shape = 100 self.paddings = [(0, 1)] self.pad_value = 0.9 -#----------------Pad Fp16---------------- +# ----------------Pad Fp16---------------- def create_test_fp16(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestPadFp16(parent): - def get_dtype(self): return np.float16 @@ -110,7 +106,6 @@ create_test_fp16(TestCase3) class TestPadOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((2, 2)).astype("float32") @@ -125,7 +120,6 @@ class TestPadOpError(unittest.TestCase): class TestPaddingValueTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -148,15 +142,16 @@ class TestPaddingValueTensor(UnittestBase): exe = paddle.static.Executor() exe.run(starup_prog) res = exe.run(fetch_list=[feat, out]) - gt = np.pad(res[0], [1, 1], 'constant', constant_values=[1., 1.]) + gt = np.pad(res[0], [1, 1], 'constant', constant_values=[1.0, 1.0]) np.testing.assert_allclose(res[1], gt) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() - gt = np.pad(infer_outs[0], [1, 1], - 'constant', - constant_values=[1., 1.]) + gt = np.pad( + infer_outs[0], [1, 1], 'constant', constant_values=[1.0, 1.0] + ) np.testing.assert_allclose(infer_outs[1], gt) def path_prefix(self): @@ -167,22 +162,20 @@ class TestPaddingValueTensor(UnittestBase): def call_func(self, x): padding_value = paddle.assign([1.0]) - out = paddle.nn.functional.pad(x, - pad=[1, 1, 1, 1], - value=padding_value, - mode='constant') + out = paddle.nn.functional.pad( + x, pad=[1, 1, 1, 1], value=padding_value, mode='constant' + ) return out class TestPaddingValueTensor2(TestPaddingValueTensor): - def call_func(self, x): padding_value = paddle.assign([1.0]) # test for int value tmp = paddle.fluid.layers.pad(x, paddings=[1, 1, 1, 1], pad_value=1) - out = paddle.fluid.layers.pad(x, - paddings=[1, 1, 1, 1], - pad_value=padding_value) + out = paddle.fluid.layers.pad( + x, paddings=[1, 1, 1, 1], pad_value=padding_value + ) return out diff --git a/python/paddle/fluid/tests/unittests/test_paddle_imperative_double_grad.py b/python/paddle/fluid/tests/unittests/test_paddle_imperative_double_grad.py index 99b797fbf9e..dd958243fad 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_imperative_double_grad.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_imperative_double_grad.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph def _dygraph_guard_(func): - def __impl__(*args, **kwargs): if paddle.in_dynamic_mode(): return func(*args, **kwargs) @@ -42,26 +41,29 @@ def random_var(size, low=-1, high=1, dtype='float32'): class TestDygraphDoubleGrad(TestCase): - def setUp(self): self.sort_sum_gradient = False self.shape = [5, 10] - def grad(self, - outputs, - inputs, - grad_outputs=None, - no_grad_vars=None, - retain_graph=None, - create_graph=False, - allow_unused=False): - return paddle.grad(outputs=outputs, - inputs=inputs, - grad_outputs=grad_outputs, - no_grad_vars=no_grad_vars, - retain_graph=retain_graph, - create_graph=create_graph, - allow_unused=allow_unused) + def grad( + self, + outputs, + inputs, + grad_outputs=None, + no_grad_vars=None, + retain_graph=None, + create_graph=False, + allow_unused=False, + ): + return paddle.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=grad_outputs, + no_grad_vars=no_grad_vars, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=allow_unused, + ) @dygraph_guard def func_exception(self): @@ -83,12 +85,16 @@ class TestDygraphDoubleGrad(TestCase): self.grad([random_var(shape)], [1]) with self.assertRaises(AssertionError): - self.grad([random_var(shape), random_var(shape)], - [random_var(shape)], [random_var(shape)]) + self.grad( + [random_var(shape), random_var(shape)], + [random_var(shape)], + [random_var(shape)], + ) with self.assertRaises(AssertionError): - self.grad([random_var(shape)], [random_var(shape)], - no_grad_vars=[1]) + self.grad( + [random_var(shape)], [random_var(shape)], no_grad_vars=[1] + ) with self.assertRaises(AssertionError): self.grad([random_var(shape)], [random_var(shape)], no_grad_vars=1) @@ -105,31 +111,33 @@ class TestDygraphDoubleGrad(TestCase): y = x + 1 for create_graph in [False, True]: - dx, = self.grad([x], [x], - create_graph=create_graph, - retain_graph=True) + (dx,) = self.grad( + [x], [x], create_graph=create_graph, retain_graph=True + ) self.assertEqual(dx.shape, x.shape) self.assertTrue(np.all(dx.numpy() == 1)) self.assertNotEqual(dx.stop_gradient, create_graph) - dx_mul_2, = self.grad([y, x], [x], - create_graph=create_graph, - retain_graph=True) + (dx_mul_2,) = self.grad( + [y, x], [x], create_graph=create_graph, retain_graph=True + ) self.assertEqual(dx_mul_2.shape, x.shape) self.assertTrue(np.all(dx_mul_2.numpy() == 2)) self.assertNotEqual(dx_mul_2.stop_gradient, create_graph) - none_grad, = self.grad([x], [y], - create_graph=create_graph, - allow_unused=True) + (none_grad,) = self.grad( + [x], [y], create_graph=create_graph, allow_unused=True + ) self.assertTrue(none_grad is None) - grad_with_none_and_not_none, = self.grad([x, y], [y], - create_graph=create_graph) + (grad_with_none_and_not_none,) = self.grad( + [x, y], [y], create_graph=create_graph + ) self.assertTrue(grad_with_none_and_not_none.shape, x.shape) self.assertTrue(np.all(grad_with_none_and_not_none.numpy() == 1)) - self.assertNotEqual(grad_with_none_and_not_none.stop_gradient, - create_graph) + self.assertNotEqual( + grad_with_none_and_not_none.stop_gradient, create_graph + ) def test_simple_example(self): with _test_eager_guard(): @@ -144,11 +152,12 @@ class TestDygraphDoubleGrad(TestCase): half_numel = int(numel / 2) half_x_positive = np.random.uniform(low=1, high=2, size=[half_numel]) - half_x_negative = np.random.uniform(low=-2, - high=-1, - size=[numel - half_numel]) - x_np = np.array(list(half_x_positive) + - list(half_x_negative)).astype('float32') + half_x_negative = np.random.uniform( + low=-2, high=-1, size=[numel - half_numel] + ) + x_np = np.array(list(half_x_positive) + list(half_x_negative)).astype( + 'float32' + ) np.random.shuffle(x_np) x = fluid.dygraph.to_variable(x_np) @@ -163,8 +172,9 @@ class TestDygraphDoubleGrad(TestCase): relu_x_np = np.maximum(x_np, alpha * x_np).astype('float32') relu_x_grad_np = ((x_np > 0) + (x_np < 0) * alpha).astype('float32') dy_expected = (relu_x_np * relu_x_grad_np * 2).astype('float32') - dz_expected = (np.power(relu_x_np, 3) * relu_x_grad_np * - 4).astype('float32') + dz_expected = (np.power(relu_x_np, 3) * relu_x_grad_np * 4).astype( + 'float32' + ) random_grad_y = random_var(y.shape, low=1, high=2) random_grad_z = random_var(z.shape, low=1, high=2) @@ -177,31 +187,39 @@ class TestDygraphDoubleGrad(TestCase): for grad_y in [random_grad_y]: for grad_z in [random_grad_z]: for create_graph in [False, True]: - dx_actual, = self.grad(outputs=[y, z], - inputs=[x], - grad_outputs=[grad_y, grad_z], - create_graph=create_graph, - retain_graph=True) + (dx_actual,) = self.grad( + outputs=[y, z], + inputs=[x], + grad_outputs=[grad_y, grad_z], + create_graph=create_graph, + retain_graph=True, + ) - grad_y_np = ones_grad_y if grad_y is None else grad_y.numpy( + grad_y_np = ( + ones_grad_y if grad_y is None else grad_y.numpy() ) - grad_z_np = ones_grad_z if grad_z is None else grad_z.numpy( + grad_z_np = ( + ones_grad_z if grad_z is None else grad_z.numpy() ) - dx_expected = dy_expected * grad_y_np + dz_expected * grad_z_np - np.testing.assert_allclose(dx_actual.numpy(), - dx_expected, - rtol=1e-05) + dx_expected = ( + dy_expected * grad_y_np + dz_expected * grad_z_np + ) + np.testing.assert_allclose( + dx_actual.numpy(), dx_expected, rtol=1e-05 + ) if grad_y is not None: self.assertTrue(grad_y.stop_gradient) - np.testing.assert_array_equal(grad_y.numpy(), - original_random_grad_y) + np.testing.assert_array_equal( + grad_y.numpy(), original_random_grad_y + ) if grad_z is not None: self.assertTrue(grad_z.stop_gradient) - np.testing.assert_array_equal(grad_z.numpy(), - original_random_grad_z) + np.testing.assert_array_equal( + grad_z.numpy(), original_random_grad_z + ) def test_none_one_initial_gradient(self): with _test_eager_guard(): @@ -222,14 +240,15 @@ class TestDygraphDoubleGrad(TestCase): w_mean = fluid.layers.reduce_mean(w) del y, z, w - dx_actual, = self.grad([w_mean], [x], create_graph=True) + (dx_actual,) = self.grad([w_mean], [x], create_graph=True) del w_mean self.assertFalse(dx_actual.stop_gradient) # Theoritical result based on math calculation - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * (x_np > 0) * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) if not _in_legacy_dygraph(): @@ -240,12 +259,13 @@ class TestDygraphDoubleGrad(TestCase): x_grad_actual = x.gradient() x_grad_expected = ( - 2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 2 / float(numel))).astype('float32') - np.testing.assert_allclose(x_grad_actual, - x_grad_expected, - rtol=1e-05) + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 2 / float(numel)) + ).astype('float32') + np.testing.assert_allclose( + x_grad_actual, x_grad_expected, rtol=1e-05 + ) def test_example_with_gradient_accumulation_and_create_graph(self): with _test_eager_guard(): @@ -267,15 +287,20 @@ class TestDygraphDoubleGrad(TestCase): w_mean = fluid.layers.reduce_mean(w) del y1, z, w - dx_actual, = self.grad([w_mean], [x], - create_graph=True, - no_grad_vars=[y2]) + (dx_actual,) = self.grad( + [w_mean], [x], create_graph=True, no_grad_vars=[y2] + ) self.assertFalse(y2.stop_gradient) self.assertFalse(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + y2.numpy()) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 + / float(numel) + * (np.maximum(x_np, 0) + y2.numpy()) + * (x_np > 0) + * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) if not _in_legacy_dygraph(): @@ -286,12 +311,13 @@ class TestDygraphDoubleGrad(TestCase): x_grad_actual = x.gradient() x_grad_expected = ( - 2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 4 / float(numel))).astype('float32') - np.testing.assert_allclose(x_grad_actual, - x_grad_expected, - rtol=1e-05) + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 4 / float(numel)) + ).astype('float32') + np.testing.assert_allclose( + x_grad_actual, x_grad_expected, rtol=1e-05 + ) def test_example_with_gradient_accumulation_and_no_grad_vars(self): with _test_eager_guard(): @@ -312,13 +338,14 @@ class TestDygraphDoubleGrad(TestCase): w_mean = fluid.layers.reduce_mean(w) del y, z, w - dx_actual, = self.grad([w_mean], [x], create_graph=False) + (dx_actual,) = self.grad([w_mean], [x], create_graph=False) del w_mean self.assertTrue(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * (x_np > 0) * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) @@ -330,9 +357,9 @@ class TestDygraphDoubleGrad(TestCase): x_grad_actual = x.gradient() x_grad_expected = (2.0 * x_np / float(numel)).astype('float32') - np.testing.assert_allclose(x_grad_actual, - x_grad_expected, - rtol=1e-05) + np.testing.assert_allclose( + x_grad_actual, x_grad_expected, rtol=1e-05 + ) def test_example_with_gradient_accumulation_and_not_create_graph(self): with _test_eager_guard(): @@ -341,7 +368,6 @@ class TestDygraphDoubleGrad(TestCase): class TestDygraphDoubleGradSortGradient(TestDygraphDoubleGrad): - def setUp(self): self.sort_sum_gradient = True self.shape = [5, 10] diff --git a/python/paddle/fluid/tests/unittests/test_paddle_multiprocessing.py b/python/paddle/fluid/tests/unittests/test_paddle_multiprocessing.py index 8f4131c8d66..a6e3bf827d3 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_multiprocessing.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_multiprocessing.py @@ -18,7 +18,11 @@ import unittest import time import paddle import paddle.incubate.multiprocessing as mp -from paddle.fluid.framework import _enable_legacy_dygraph, _test_eager_guard, in_dygraph_mode +from paddle.fluid.framework import ( + _enable_legacy_dygraph, + _test_eager_guard, + in_dygraph_mode, +) REPEAT = 20 HAS_SHM_FILES = os.path.isdir('/dev/shm') @@ -47,14 +51,14 @@ def send_parambase(queue, event, device, dtype): tensor = paddle.nn.Layer().create_parameter( [5, 5], dtype=dtype, - default_initializer=paddle.nn.initializer.Constant(value=1.0)) + default_initializer=paddle.nn.initializer.Constant(value=1.0), + ) queue.put(tensor) queue.put(tensor) event.wait() class leak_checker(object): - def __init__(self, test_case): self.checked_pids = [os.getpid()] self.test_case = test_case @@ -98,7 +102,6 @@ class leak_checker(object): class TestMultiprocessingBase(unittest.TestCase): - def get_tensor(self, device="cpu"): self.device = device.lower() place = None @@ -108,7 +111,8 @@ class TestMultiprocessingBase(unittest.TestCase): def get_parameter(self): w = paddle.nn.Layer().create_parameter( [10, 10], - default_initializer=paddle.nn.initializer.Constant(value=0.0)) + default_initializer=paddle.nn.initializer.Constant(value=0.0), + ) return w def _test_empty(self, dtype="float32"): @@ -118,13 +122,9 @@ class TestMultiprocessingBase(unittest.TestCase): out = q.get(timeout=1) self.assertEqual(str(out), str(empty)) - def _test_sharing(self, - ctx=mp, - device='cpu', - dtype="float32", - repeat=1, - param=False): - + def _test_sharing( + self, ctx=mp, device='cpu', dtype="float32", repeat=1, param=False + ): def test_fill(): if param: x = self.get_parameter() @@ -159,7 +159,8 @@ class TestMultiprocessingBase(unittest.TestCase): process = ctx.Process( target=send_parambase if param else send_tensor, - args=(queue, event, device, dtype)) + args=(queue, event, device, dtype), + ) process.daemon = True lc.check_pid(process.pid) process.start() @@ -180,7 +181,6 @@ class TestMultiprocessingBase(unittest.TestCase): class TestMultiprocessingCpu(TestMultiprocessingBase): - def func_test_pass_tensor(self): if in_dygraph_mode(): return @@ -216,9 +216,10 @@ class TestMultiprocessingCpu(TestMultiprocessingBase): class TestMultiprocessingGpu(TestMultiprocessingBase): - - @unittest.skipIf(not paddle.fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not paddle.fluid.core.is_compiled_with_cuda(), + "core is not compiled with CUDA", + ) def func_test_pass_tensor(self): if in_dygraph_mode(): return diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py index f5aea5c3039..b4111697394 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py @@ -40,14 +40,17 @@ LARGE_PARAM = 2**26 def random_batch_reader(): - def _get_random_inputs_and_labels(): np.random.seed(SEED) image = np.random.random([BATCH_SIZE, IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, ( - BATCH_SIZE, - 1, - )).astype('int64') + label = np.random.randint( + 0, + CLASS_NUM - 1, + ( + BATCH_SIZE, + 1, + ), + ).astype('int64') return image, label def __reader__(): @@ -61,7 +64,6 @@ def random_batch_reader(): class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -71,7 +73,6 @@ class LinearNet(nn.Layer): class LayerWithLargeParameters(paddle.nn.Layer): - def __init__(self): super(LayerWithLargeParameters, self).__init__() self._l = paddle.nn.Linear(10, LARGE_PARAM) @@ -92,7 +93,6 @@ def train(layer, loader, loss_fn, opt): class TestSaveLoadLargeParameters(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -107,9 +107,11 @@ class TestSaveLoadLargeParameters(unittest.TestCase): layer = LayerWithLargeParameters() save_dict = layer.state_dict() - path = os.path.join(self.temp_dir.name, - "test_paddle_save_load_large_param_save", - "layer.pdparams") + path = os.path.join( + self.temp_dir.name, + "test_paddle_save_load_large_param_save", + "layer.pdparams", + ) protocol = 4 paddle.save(save_dict, path, protocol=protocol) dict_load = paddle.load(path, return_numpy=True) @@ -119,7 +121,6 @@ class TestSaveLoadLargeParameters(unittest.TestCase): class TestSaveLoadPickle(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -133,9 +134,11 @@ class TestSaveLoadPickle(unittest.TestCase): layer = LinearNet() save_dict = layer.state_dict() - path = os.path.join(self.temp_dir.name, - "test_paddle_save_load_pickle_protocol", - "layer.pdparams") + path = os.path.join( + self.temp_dir.name, + "test_paddle_save_load_pickle_protocol", + "layer.pdparams", + ) with self.assertRaises(ValueError): paddle.save(save_dict, path, 2.0) @@ -156,12 +159,12 @@ class TestSaveLoadPickle(unittest.TestCase): dict_load = paddle.load(path) # compare results before and after saving for key, value in save_dict.items(): - np.testing.assert_array_equal(dict_load[key].numpy(), - value.numpy()) + np.testing.assert_array_equal( + dict_load[key].numpy(), value.numpy() + ) class TestSaveLoadAny(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -209,9 +212,9 @@ class TestSaveLoadAny(unittest.TestCase): def test_replace_static_save_load(self): paddle.enable_static() with new_program_scope(): - x = paddle.static.data(name="static_x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="static_x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 10) z = paddle.static.nn.fc(z, 10, bias_attr=False) loss = fluid.layers.reduce_mean(z) @@ -226,19 +229,22 @@ class TestSaveLoadAny(unittest.TestCase): base_map = {} for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_map[var.name] = t - path = os.path.join(self.temp_dir.name, - "test_replace_static_save_load", "model") + path = os.path.join( + self.temp_dir.name, "test_replace_static_save_load", "model" + ) # paddle.save, legacy paddle.fluid.load self.replace_static_save(prog, path) self.set_zero(prog, place) paddle.fluid.io.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, np.array(base_t)) # legacy paddle.fluid.save, paddle.load @@ -247,8 +253,9 @@ class TestSaveLoadAny(unittest.TestCase): self.replace_static_load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) # test for return tensor @@ -258,7 +265,8 @@ class TestSaveLoadAny(unittest.TestCase): tensor = var.get_value(fluid.global_scope()) paddle.save( tensor, - os.path.join(self.temp_dir.name, path_vars, var.name)) + os.path.join(self.temp_dir.name, path_vars, var.name), + ) with self.assertRaises(TypeError): var.get_value('fluid.global_scope()') with self.assertRaises(ValueError): @@ -276,12 +284,14 @@ class TestSaveLoadAny(unittest.TestCase): self.set_zero(prog, place) for var in prog.list_vars(): if var.persistable: - tensor = paddle.load(os.path.join(self.temp_dir.name, - path_vars, var.name), - return_numpy=False) + tensor = paddle.load( + os.path.join(self.temp_dir.name, path_vars, var.name), + return_numpy=False, + ) var.set_value(tensor) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -289,17 +299,19 @@ class TestSaveLoadAny(unittest.TestCase): paddle.disable_static() class StepDecay(LRScheduler): - - def __init__(self, - learning_rate, - step_size, - gamma=0.1, - last_epoch=-1, - verbose=False): + def __init__( + self, + learning_rate, + step_size, + gamma=0.1, + last_epoch=-1, + verbose=False, + ): self.step_size = step_size self.gamma = gamma - super(StepDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(StepDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): i = self.last_epoch // self.step_size @@ -307,14 +319,16 @@ class TestSaveLoadAny(unittest.TestCase): layer = LinearNet() inps = paddle.randn([2, IMAGE_SIZE]) - adam = opt.Adam(learning_rate=StepDecay(0.1, 1), - parameters=layer.parameters()) + adam = opt.Adam( + learning_rate=StepDecay(0.1, 1), parameters=layer.parameters() + ) y = layer(inps) y.mean().backward() adam.step() state_dict = adam.state_dict() - path = os.path.join(self.temp_dir.name, - 'paddle_save_load_v2/model.pdparams') + path = os.path.join( + self.temp_dir.name, 'paddle_save_load_v2/model.pdparams' + ) with self.assertRaises(TypeError): paddle.save(state_dict, path, use_binary_format='False') # legacy paddle.save, paddle.load @@ -327,8 +341,9 @@ class TestSaveLoadAny(unittest.TestCase): if isinstance(v, dict): self.assertTrue(v == load_dict_tensor[k]) else: - np.testing.assert_array_equal(v.numpy(), - load_dict_tensor[k].numpy()) + np.testing.assert_array_equal( + v.numpy(), load_dict_tensor[k].numpy() + ) if not np.array_equal(v.numpy(), load_dict_np[k]): print(v.numpy()) print(load_dict_np[k]) @@ -338,8 +353,9 @@ class TestSaveLoadAny(unittest.TestCase): # enable dygraph mode paddle.disable_static() layer = LinearNet() - path = os.path.join(self.temp_dir.name, - 'paddle_save_load_v2/var_dygraph') + path = os.path.join( + self.temp_dir.name, 'paddle_save_load_v2/var_dygraph' + ) tensor = layer._linear.weight with self.assertRaises(ValueError): paddle.save(tensor, path, pickle_protocol='3') @@ -351,7 +367,9 @@ class TestSaveLoadAny(unittest.TestCase): self.assertTrue( isinstance( t_dygraph, - (paddle.fluid.core.VarBase, paddle.fluid.core.eager.Tensor))) + (paddle.fluid.core.VarBase, paddle.fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal(tensor.numpy(), np_dygraph) np.testing.assert_array_equal(tensor.numpy(), t_dygraph.numpy()) paddle.enable_static() @@ -366,14 +384,16 @@ class TestSaveLoadAny(unittest.TestCase): paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 128) loss = fluid.layers.reduce_mean(z) - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) prog = paddle.static.default_main_program() @@ -383,8 +403,9 @@ class TestSaveLoadAny(unittest.TestCase): break scope = fluid.global_scope() origin_tensor = np.array(tensor) - path = os.path.join(self.temp_dir.name, - 'test_single_pickle_var_static/var') + path = os.path.join( + self.temp_dir.name, 'test_single_pickle_var_static/var' + ) paddle.save(tensor, path) self.set_zero(prog, place, scope) # static load @@ -406,8 +427,10 @@ class TestSaveLoadAny(unittest.TestCase): def test_dygraph_save_static_load(self): inps = np.random.randn(1, IMAGE_SIZE).astype('float32') - path = os.path.join(self.temp_dir.name, - 'test_dygraph_save_static_load/dy-static.pdparams') + path = os.path.join( + self.temp_dir.name, + 'test_dygraph_save_static_load/dy-static.pdparams', + ) paddle.disable_static() with paddle.utils.unique_name.guard(): layer = LinearNet() @@ -416,14 +439,16 @@ class TestSaveLoadAny(unittest.TestCase): paddle.enable_static() with new_program_scope(): layer = LinearNet() - data = paddle.static.data(name='x_static_save', - shape=(None, IMAGE_SIZE), - dtype='float32') + data = paddle.static.data( + name='x_static_save', shape=(None, IMAGE_SIZE), dtype='float32' + ) y_static = layer(data) program = paddle.static.default_main_program() - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = paddle.static.Executor(paddle.CPUPlace()) exe.run(paddle.static.default_startup_program()) state_dict = paddle.load(path, keep_name_table=True) @@ -431,7 +456,8 @@ class TestSaveLoadAny(unittest.TestCase): state_dict_param = program.state_dict("param") for name, tensor in state_dict_dy.items(): np.testing.assert_array_equal( - tensor.numpy(), np.array(state_dict_param[tensor.name])) + tensor.numpy(), np.array(state_dict_param[tensor.name]) + ) def test_save_load_complex_object_dygraph_save(self): paddle.disable_static() @@ -440,24 +466,28 @@ class TestSaveLoadAny(unittest.TestCase): obj1 = [ paddle.randn([3, 4], dtype='float32'), np.random.randn(5, 6), - ('fake_weight', np.ones([7, 8], dtype='float32')) + ('fake_weight', np.ones([7, 8], dtype='float32')), ] obj2 = {'k1': obj1, 'k2': state_dict, 'epoch': 123} - obj3 = (paddle.randn([5, 4], dtype='float32'), - np.random.randn(3, 4).astype("float32"), { - "state_dict": state_dict, - "opt": state_dict - }) - obj4 = (np.random.randn(5, 6), (123, )) - - path1 = os.path.join(self.temp_dir.name, - "test_save_load_any_complex_object_dygraph/obj1") - path2 = os.path.join(self.temp_dir.name, - "test_save_load_any_complex_object_dygraph/obj2") - path3 = os.path.join(self.temp_dir.name, - "test_save_load_any_complex_object_dygraph/obj3") - path4 = os.path.join(self.temp_dir.name, - "test_save_load_any_complex_object_dygraph/obj4") + obj3 = ( + paddle.randn([5, 4], dtype='float32'), + np.random.randn(3, 4).astype("float32"), + {"state_dict": state_dict, "opt": state_dict}, + ) + obj4 = (np.random.randn(5, 6), (123,)) + + path1 = os.path.join( + self.temp_dir.name, "test_save_load_any_complex_object_dygraph/obj1" + ) + path2 = os.path.join( + self.temp_dir.name, "test_save_load_any_complex_object_dygraph/obj2" + ) + path3 = os.path.join( + self.temp_dir.name, "test_save_load_any_complex_object_dygraph/obj3" + ) + path4 = os.path.join( + self.temp_dir.name, "test_save_load_any_complex_object_dygraph/obj4" + ) paddle.save(obj1, path1) paddle.save(obj2, path2) paddle.save(obj3, path3) @@ -473,10 +503,12 @@ class TestSaveLoadAny(unittest.TestCase): np.testing.assert_array_equal(load_tensor1[2].numpy(), obj1[2][1]) for i in range(len(load_tensor1)): self.assertTrue( - type(load_tensor1[i]) == type(load_tensor2['k1'][i])) + type(load_tensor1[i]) == type(load_tensor2['k1'][i]) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(v.numpy(), - load_tensor2['k2'][k].numpy()) + np.testing.assert_array_equal( + v.numpy(), load_tensor2['k2'][k].numpy() + ) self.assertTrue(load_tensor2['epoch'] == 123) np.testing.assert_array_equal(load_tensor3[0].numpy(), obj3[0].numpy()) @@ -484,11 +516,13 @@ class TestSaveLoadAny(unittest.TestCase): for k, v in state_dict.items(): np.testing.assert_array_equal( - load_tensor3[2]['state_dict'][k].numpy(), v.numpy()) + load_tensor3[2]['state_dict'][k].numpy(), v.numpy() + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_tensor3[2]['opt'][k].numpy(), - v.numpy()) + np.testing.assert_array_equal( + load_tensor3[2]['opt'][k].numpy(), v.numpy() + ) np.testing.assert_array_equal(load_tensor4[0].numpy(), obj4[0]) @@ -510,8 +544,9 @@ class TestSaveLoadAny(unittest.TestCase): np.testing.assert_array_equal(load_array3[1], obj3[1]) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['state_dict'][k], - v.numpy()) + np.testing.assert_array_equal( + load_array3[2]['state_dict'][k], v.numpy() + ) for k, v in state_dict.items(): np.testing.assert_array_equal(load_array3[2]['opt'][k], v.numpy()) @@ -526,38 +561,50 @@ class TestSaveLoadAny(unittest.TestCase): load_tensor3 = paddle.load(path3, return_numpy=False) load_tensor4 = paddle.load(path4, return_numpy=False) - np.testing.assert_array_equal(np.array(load_tensor1[0]), - obj1[0].numpy()) + np.testing.assert_array_equal( + np.array(load_tensor1[0]), obj1[0].numpy() + ) np.testing.assert_array_equal(np.array(load_tensor1[1]), obj1[1]) np.testing.assert_array_equal(np.array(load_tensor1[2]), obj1[2][1]) for i in range(len(load_tensor1)): self.assertTrue( - type(load_tensor1[i]) == type(load_tensor2['k1'][i])) + type(load_tensor1[i]) == type(load_tensor2['k1'][i]) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(v.numpy(), - np.array(load_tensor2['k2'][k])) + np.testing.assert_array_equal( + v.numpy(), np.array(load_tensor2['k2'][k]) + ) self.assertTrue(load_tensor2['epoch'] == 123) - self.assertTrue(isinstance(load_tensor3[0], - paddle.fluid.core.LoDTensor)) - np.testing.assert_array_equal(np.array(load_tensor3[0]), - obj3[0].numpy()) + self.assertTrue( + isinstance(load_tensor3[0], paddle.fluid.core.LoDTensor) + ) + np.testing.assert_array_equal( + np.array(load_tensor3[0]), obj3[0].numpy() + ) np.testing.assert_array_equal(np.array(load_tensor3[1]), obj3[1]) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["state_dict"][k], - paddle.fluid.core.LoDTensor)) + isinstance( + load_tensor3[2]["state_dict"][k], + paddle.fluid.core.LoDTensor, + ) + ) np.testing.assert_array_equal( - np.array(load_tensor3[2]['state_dict'][k]), v.numpy()) + np.array(load_tensor3[2]['state_dict'][k]), v.numpy() + ) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["opt"][k], - paddle.fluid.core.LoDTensor)) - np.testing.assert_array_equal(np.array(load_tensor3[2]['opt'][k]), - v.numpy()) + isinstance( + load_tensor3[2]["opt"][k], paddle.fluid.core.LoDTensor + ) + ) + np.testing.assert_array_equal( + np.array(load_tensor3[2]['opt'][k]), v.numpy() + ) self.assertTrue(load_tensor4[0], paddle.fluid.core.LoDTensor) np.testing.assert_array_equal(np.array(load_tensor4[0]), obj4[0]) @@ -581,8 +628,9 @@ class TestSaveLoadAny(unittest.TestCase): np.testing.assert_array_equal(load_array3[1], obj3[1]) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['state_dict'][k], - v.numpy()) + np.testing.assert_array_equal( + load_array3[2]['state_dict'][k], v.numpy() + ) for k, v in state_dict.items(): np.testing.assert_array_equal(load_array3[2]['opt'][k], v.numpy()) @@ -593,15 +641,17 @@ class TestSaveLoadAny(unittest.TestCase): paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 10, bias_attr=False) z = paddle.static.nn.fc(z, 128, bias_attr=False) loss = fluid.layers.reduce_mean(z) - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) @@ -611,27 +661,32 @@ class TestSaveLoadAny(unittest.TestCase): obj1 = [ state_dict[keys[0]], np.random.randn(5, 6), - ('fake_weight', np.ones([7, 8], dtype='float32')) + ('fake_weight', np.ones([7, 8], dtype='float32')), ] obj2 = {'k1': obj1, 'k2': state_dict, 'epoch': 123} - obj3 = (state_dict[keys[0]], np.ndarray([3, 4], dtype="float32"), { - "state_dict": state_dict, - "opt": state_dict - }) - obj4 = (np.ndarray([3, 4], dtype="float32"), ) + obj3 = ( + state_dict[keys[0]], + np.ndarray([3, 4], dtype="float32"), + {"state_dict": state_dict, "opt": state_dict}, + ) + obj4 = (np.ndarray([3, 4], dtype="float32"),) path1 = os.path.join( self.temp_dir.name, - "test_save_load_any_complex_object_static/obj1") + "test_save_load_any_complex_object_static/obj1", + ) path2 = os.path.join( self.temp_dir.name, - "test_save_load_any_complex_object_static/obj2") + "test_save_load_any_complex_object_static/obj2", + ) path3 = os.path.join( self.temp_dir.name, - "test_save_load_any_complex_object_static/obj3") + "test_save_load_any_complex_object_static/obj3", + ) path4 = os.path.join( self.temp_dir.name, - "test_save_load_any_complex_object_static/obj4") + "test_save_load_any_complex_object_static/obj4", + ) paddle.save(obj1, path1) paddle.save(obj2, path2) paddle.save(obj3, path3) @@ -642,16 +697,19 @@ class TestSaveLoadAny(unittest.TestCase): load_tensor3 = paddle.load(path3, return_numpy=False) load_tensor4 = paddle.load(path4, return_numpy=False) - np.testing.assert_array_equal(np.array(load_tensor1[0]), - np.array(obj1[0])) + np.testing.assert_array_equal( + np.array(load_tensor1[0]), np.array(obj1[0]) + ) np.testing.assert_array_equal(np.array(load_tensor1[1]), obj1[1]) np.testing.assert_array_equal(np.array(load_tensor1[2]), obj1[2][1]) for i in range(len(load_tensor1)): self.assertTrue( - type(load_tensor1[i]) == type(load_tensor2['k1'][i])) + type(load_tensor1[i]) == type(load_tensor2['k1'][i]) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(np.array(v), - np.array(load_tensor2['k2'][k])) + np.testing.assert_array_equal( + np.array(v), np.array(load_tensor2['k2'][k]) + ) self.assertTrue(load_tensor2['epoch'] == 123) self.assertTrue(isinstance(load_tensor3[0], fluid.core.LoDTensor)) @@ -661,16 +719,21 @@ class TestSaveLoadAny(unittest.TestCase): for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["state_dict"][k], - fluid.core.LoDTensor)) + isinstance( + load_tensor3[2]["state_dict"][k], fluid.core.LoDTensor + ) + ) np.testing.assert_array_equal( - np.array(load_tensor3[2]['state_dict'][k]), np.array(v)) + np.array(load_tensor3[2]['state_dict'][k]), np.array(v) + ) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["opt"][k], fluid.core.LoDTensor)) + isinstance(load_tensor3[2]["opt"][k], fluid.core.LoDTensor) + ) np.testing.assert_array_equal( - np.array(load_tensor3[2]['opt'][k]), np.array(v)) + np.array(load_tensor3[2]['opt'][k]), np.array(v) + ) self.assertTrue(isinstance(load_tensor4[0], fluid.core.LoDTensor)) np.testing.assert_array_equal(np.array(load_tensor4[0]), obj4[0]) @@ -685,7 +748,8 @@ class TestSaveLoadAny(unittest.TestCase): np.testing.assert_array_equal(load_array1[2], obj1[2][1]) for i in range(len(load_array1)): self.assertTrue( - type(load_array1[i]) == type(load_array2['k1'][i])) + type(load_array1[i]) == type(load_array2['k1'][i]) + ) for k, v in state_dict.items(): np.testing.assert_array_equal(np.array(v), load_array2['k2'][k]) self.assertTrue(load_array2['epoch'] == 123) @@ -694,12 +758,14 @@ class TestSaveLoadAny(unittest.TestCase): np.testing.assert_array_equal(load_array3[1], obj3[1]) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['state_dict'][k], - np.array(v)) + np.testing.assert_array_equal( + load_array3[2]['state_dict'][k], np.array(v) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['opt'][k], - np.array(v)) + np.testing.assert_array_equal( + load_array3[2]['opt'][k], np.array(v) + ) np.testing.assert_array_equal(load_array4[0], obj4[0]) @@ -711,44 +777,64 @@ class TestSaveLoadAny(unittest.TestCase): load_tensor3 = paddle.load(path3, return_numpy=False) load_tensor4 = paddle.load(path4, return_numpy=False) - np.testing.assert_array_equal(np.array(load_tensor1[0]), - np.array(obj1[0])) + np.testing.assert_array_equal( + np.array(load_tensor1[0]), np.array(obj1[0]) + ) np.testing.assert_array_equal(np.array(load_tensor1[1]), obj1[1]) np.testing.assert_array_equal(load_tensor1[2].numpy(), obj1[2][1]) for i in range(len(load_tensor1)): self.assertTrue( - type(load_tensor1[i]) == type(load_tensor2['k1'][i])) + type(load_tensor1[i]) == type(load_tensor2['k1'][i]) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(np.array(v), - np.array(load_tensor2['k2'][k])) + np.testing.assert_array_equal( + np.array(v), np.array(load_tensor2['k2'][k]) + ) self.assertTrue(load_tensor2['epoch'] == 123) self.assertTrue( - isinstance(load_tensor3[0], - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + load_tensor3[0], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal(load_tensor3[0].numpy(), obj3[0]) self.assertTrue( - isinstance(load_tensor3[1], - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + load_tensor3[1], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal(load_tensor3[1].numpy(), obj3[1]) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["state_dict"][k], - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + load_tensor3[2]["state_dict"][k], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal( - load_tensor3[2]['state_dict'][k].numpy(), np.array(v)) + load_tensor3[2]['state_dict'][k].numpy(), np.array(v) + ) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["opt"][k], - (fluid.core.VarBase, fluid.core.eager.Tensor))) - np.testing.assert_array_equal(load_tensor3[2]['opt'][k].numpy(), - np.array(v)) + isinstance( + load_tensor3[2]["opt"][k], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) + np.testing.assert_array_equal( + load_tensor3[2]['opt'][k].numpy(), np.array(v) + ) self.assertTrue( - isinstance(load_tensor4[0], - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + load_tensor4[0], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal(load_tensor4[0].numpy(), obj4[0]) load_array1 = paddle.load(path1, return_numpy=True) @@ -761,7 +847,8 @@ class TestSaveLoadAny(unittest.TestCase): np.testing.assert_array_equal(load_array1[2], obj1[2][1]) for i in range(len(load_array1)): self.assertTrue( - type(load_array1[i]) == type(load_array2['k1'][i])) + type(load_array1[i]) == type(load_array2['k1'][i]) + ) for k, v in state_dict.items(): np.testing.assert_array_equal(np.array(v), load_array2['k2'][k]) self.assertTrue(load_array2['epoch'] == 123) @@ -770,12 +857,14 @@ class TestSaveLoadAny(unittest.TestCase): np.testing.assert_array_equal(load_array3[1], obj3[1]) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['state_dict'][k], - np.array(v)) + np.testing.assert_array_equal( + load_array3[2]['state_dict'][k], np.array(v) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['opt'][k], - np.array(v)) + np.testing.assert_array_equal( + load_array3[2]['opt'][k], np.array(v) + ) self.assertTrue(isinstance(load_array4[0], np.ndarray)) np.testing.assert_array_equal(load_array4[0], obj4[0]) @@ -783,8 +872,10 @@ class TestSaveLoadAny(unittest.TestCase): def test_varbase_binary_var(self): paddle.disable_static() varbase = paddle.randn([3, 2], dtype='float32') - path = os.path.join(self.temp_dir.name, - 'test_paddle_save_load_varbase_binary_var/varbase') + path = os.path.join( + self.temp_dir.name, + 'test_paddle_save_load_varbase_binary_var/varbase', + ) paddle.save(varbase, path, use_binary_format=True) load_array = paddle.load(path, return_numpy=True) load_tensor = paddle.load(path, return_numpy=False) @@ -797,7 +888,6 @@ class TestSaveLoadAny(unittest.TestCase): class TestSaveLoadToMemory(unittest.TestCase): - def test_dygraph_save_to_memory(self): paddle.disable_static() linear = LinearNet() @@ -826,15 +916,17 @@ class TestSaveLoadToMemory(unittest.TestCase): paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 10, bias_attr=False) z = paddle.static.nn.fc(z, 128, bias_attr=False) loss = fluid.layers.reduce_mean(z) - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) @@ -852,8 +944,10 @@ class TestSaveLoadToMemory(unittest.TestCase): byio2.seek(0) prog_load = paddle.load(byio2) - self.assertTrue(prog.desc.serialize_to_string() == - prog_load.desc.serialize_to_string()) + self.assertTrue( + prog.desc.serialize_to_string() + == prog_load.desc.serialize_to_string() + ) tensor_load = paddle.load(byio, return_numpy=True) np.testing.assert_array_equal(tensor_load, np.array(tensor)) @@ -864,7 +958,6 @@ class TestSaveLoadToMemory(unittest.TestCase): class TestSaveLoad(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -895,18 +988,23 @@ class TestSaveLoad(unittest.TestCase): def check_load_state_dict(self, orig_dict, load_dict): for var_name, value in orig_dict.items(): - load_value = load_dict[var_name].numpy() if hasattr( - load_dict[var_name], 'numpy') else np.array(load_dict[var_name]) + load_value = ( + load_dict[var_name].numpy() + if hasattr(load_dict[var_name], 'numpy') + else np.array(load_dict[var_name]) + ) np.testing.assert_array_equal(value.numpy(), load_value) def test_save_load(self): layer, opt = self.build_and_train_model() # save - layer_save_path = os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear.pdparams") - opt_save_path = os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear.pdopt") + layer_save_path = os.path.join( + self.temp_dir.name, "test_paddle_save_load.linear.pdparams" + ) + opt_save_path = os.path.join( + self.temp_dir.name, "test_paddle_save_load.linear.pdopt" + ) layer_state_dict = layer.state_dict() opt_state_dict = opt.state_dict() @@ -924,7 +1022,8 @@ class TestSaveLoad(unittest.TestCase): paddle.enable_static() static_save_path = os.path.join( self.temp_dir.name, - "static_mode_test/test_paddle_save_load.linear.pdparams") + "static_mode_test/test_paddle_save_load.linear.pdparams", + ) paddle.save(layer_state_dict, static_save_path) load_static_state_dict = paddle.load(static_save_path) self.check_load_state_dict(layer_state_dict, load_static_state_dict) @@ -937,33 +1036,36 @@ class TestSaveLoad(unittest.TestCase): with self.assertRaises(ValueError): paddle.save( layer_state_dict, - os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear.model/")) + os.path.join( + self.temp_dir.name, "test_paddle_save_load.linear.model/" + ), + ) # 3. test load path not exist error with self.assertRaises(ValueError): paddle.load( - os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear.params")) + os.path.join( + self.temp_dir.name, "test_paddle_save_load.linear.params" + ) + ) # 4. test load old save path error with self.assertRaises(ValueError): paddle.load( - os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear")) + os.path.join(self.temp_dir.name, "test_paddle_save_load.linear") + ) class TestSaveLoadProgram(unittest.TestCase): - def test_save_load_program(self): paddle.enable_static() temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): layer = LinearNet() - data = paddle.static.data(name='x_static_save', - shape=(None, IMAGE_SIZE), - dtype='float32') + data = paddle.static.data( + name='x_static_save', shape=(None, IMAGE_SIZE), dtype='float32' + ) y_static = layer(data) main_program = paddle.static.default_main_program() startup_program = paddle.static.default_startup_program() @@ -971,10 +1073,12 @@ class TestSaveLoadProgram(unittest.TestCase): origin_startup = startup_program.desc.serialize_to_string() path1 = os.path.join( temp_dir.name, - "test_paddle_save_load_program/main_program.pdmodel") + "test_paddle_save_load_program/main_program.pdmodel", + ) path2 = os.path.join( temp_dir.name, - "test_paddle_save_load_program/startup_program.pdmodel") + "test_paddle_save_load_program/startup_program.pdmodel", + ) paddle.save(main_program, path1) paddle.save(startup_program, path2) @@ -987,7 +1091,6 @@ class TestSaveLoadProgram(unittest.TestCase): class TestSaveLoadLayer(unittest.TestCase): - def test_save_load_layer(self): paddle.disable_static() temp_dir = tempfile.TemporaryDirectory() @@ -998,8 +1101,9 @@ class TestSaveLoadLayer(unittest.TestCase): layer2.eval() origin_layer = (layer1, layer2) origin = (layer1(inps), layer2(inps)) - path = os.path.join(temp_dir.name, - "test_save_load_layer_/layer.pdmodel") + path = os.path.join( + temp_dir.name, "test_save_load_layer_/layer.pdmodel" + ) with self.assertRaises(ValueError): paddle.save(origin_layer, path) temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py index 32ee64d3c70..5eca9492ee3 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py @@ -28,7 +28,6 @@ IMAGE_SIZE = 784 class TestSaveLoadBinaryFormat(unittest.TestCase): - def setUp(self): # enable static graph mode paddle.enable_static() @@ -49,18 +48,18 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): self.assertTrue(np.sum(np.abs(new_t)) == 0) def replace_save_vars(self, program, dirname): - def predicate(var): return var.persistable vars = filter(predicate, program.list_vars()) for var in vars: - paddle.save(var.get_value(), - os.path.join(dirname, var.name), - use_binary_format=True) + paddle.save( + var.get_value(), + os.path.join(dirname, var.name), + use_binary_format=True, + ) def replace_load_vars(self, program, dirname): - def predicate(var): return var.persistable @@ -74,60 +73,65 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 10, bias_attr=False) z = paddle.static.nn.fc(z, 128, bias_attr=False) loss = fluid.layers.reduce_mean(z) - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) prog = paddle.static.default_main_program() base_map = {} for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t # test for replace_save_vars/io.load_vars path_vars1 = os.path.join( - self.temp_dir.name, 'test_replace_save_load_vars_binary1/model') + self.temp_dir.name, 'test_replace_save_load_vars_binary1/model' + ) self.replace_save_vars(prog, path_vars1) # set var to zero self.set_zero(prog, place) var_list = list( - filter(lambda var: var.persistable, prog.list_vars())) - fluid.io.load_vars(exe, - path_vars1, - main_program=prog, - vars=var_list) + filter(lambda var: var.persistable, prog.list_vars()) + ) + fluid.io.load_vars( + exe, path_vars1, main_program=prog, vars=var_list + ) for var in prog.list_vars(): if var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) # test for io.save_vars/replace_load_vars path_vars2 = os.path.join( - self.temp_dir.name, - 'test_replace_save_load_vars_binary2/model/') - fluid.io.save_vars(exe, - path_vars2, - main_program=prog, - vars=var_list) + self.temp_dir.name, 'test_replace_save_load_vars_binary2/model/' + ) + fluid.io.save_vars( + exe, path_vars2, main_program=prog, vars=var_list + ) self.set_zero(prog, place) self.replace_load_vars(prog, path_vars2) for var in prog.list_vars(): if var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -143,22 +147,27 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): name='fc_vars', ) prog = fluid.default_main_program() - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = fluid.Executor(place) prog = paddle.static.default_main_program() exe.run(fluid.default_startup_program()) - dirname = os.path.join(self.temp_dir.name, - 'test_save_load_lod_tensor1/tensor_') + dirname = os.path.join( + self.temp_dir.name, 'test_save_load_lod_tensor1/tensor_' + ) for var in prog.list_vars(): - if var.persistable and list( - var.shape) == [IMAGE_SIZE, OUTPUT_NUM]: + if var.persistable and list(var.shape) == [ + IMAGE_SIZE, + OUTPUT_NUM, + ]: tensor = var.get_value() - paddle.save(tensor, - dirname + 'fc_vars.w_0', - use_binary_format=True) + paddle.save( + tensor, dirname + 'fc_vars.w_0', use_binary_format=True + ) break origin = np.array(var.get_value()) @@ -168,7 +177,8 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): loaded_tensor = paddle.load(dirname + 'fc_vars.w_0') self.assertTrue(isinstance(loaded_tensor, fluid.core.LoDTensor)) self.assertTrue( - list(loaded_tensor.shape()) == [IMAGE_SIZE, OUTPUT_NUM]) + list(loaded_tensor.shape()) == [IMAGE_SIZE, OUTPUT_NUM] + ) to_array = np.array(loaded_tensor) np.testing.assert_array_equal(origin, to_array) @@ -178,8 +188,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): # On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk. if 'Windows' != platform.system(): with self.assertRaises(ValueError): - path = os.path.join(self.temp_dir.name, - 'test_save_load_error/temp') + path = os.path.join( + self.temp_dir.name, 'test_save_load_error/temp' + ) with open(path, "w") as f: f.write('\0') paddle.load(path) @@ -193,14 +204,18 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): temp_lod, os.path.join( self.temp_dir.name, - 'test_save_load_error_not_exist_file/not_exist_file')) + 'test_save_load_error_not_exist_file/not_exist_file', + ), + ) with self.assertRaises(RuntimeError): fluid.core.load_lod_tensor( temp_lod, os.path.join( self.temp_dir.name, - 'test_save_load_error_not_exist_file/not_exist_file')) + 'test_save_load_error_not_exist_file/not_exist_file', + ), + ) # save to memory byio = BytesIO() @@ -218,14 +233,18 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): def test_save_load_selected_rows(self): paddle.enable_static() - place = fluid.CPUPlace() if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) height = 10 rows = [0, 4, 7] row_numel = 12 selected_rows = fluid.core.SelectedRows(rows, height) - path = os.path.join(self.temp_dir.name, - 'test_paddle_save_load_selected_rows/sr.pdsr') + path = os.path.join( + self.temp_dir.name, 'test_paddle_save_load_selected_rows/sr.pdsr' + ) with self.assertRaises(ValueError): paddle.save(selected_rows, path, use_binary_format=True) @@ -247,13 +266,17 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): selected_rows, os.path.join( self.temp_dir.name, - 'test_paddle_save_load_selected_rows_not_exist_file/temp')) + 'test_paddle_save_load_selected_rows_not_exist_file/temp', + ), + ) with self.assertRaises(RuntimeError): fluid.core.load_selected_rows( selected_rows, os.path.join( self.temp_dir.name, - 'test_paddle_save_load_selected_rows_not_exist_file/temp')) + 'test_paddle_save_load_selected_rows_not_exist_file/temp', + ), + ) # save to memory byio = BytesIO() @@ -265,8 +288,9 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): self.assertTrue(isinstance(selected_rows_mem, fluid.core.SelectedRows)) self.assertTrue(list(selected_rows_mem.rows()) == rows) self.assertTrue(selected_rows_mem.height() == height) - np.testing.assert_array_equal(np.array(selected_rows_mem.get_tensor()), - np_array) + np.testing.assert_array_equal( + np.array(selected_rows_mem.get_tensor()), np_array + ) with self.assertRaises(NotImplementedError): paddle.framework.io._save_selected_rows(selected_rows, 1) diff --git a/python/paddle/fluid/tests/unittests/test_pairwise_distance.py b/python/paddle/fluid/tests/unittests/test_pairwise_distance.py index 419084cf271..675196f57eb 100644 --- a/python/paddle/fluid/tests/unittests/test_pairwise_distance.py +++ b/python/paddle/fluid/tests/unittests/test_pairwise_distance.py @@ -27,121 +27,92 @@ def np_pairwise_distance(x, y, p=2.0, epsilon=1e-6, keepdim=False): return distance -def call_pairwise_distance_layer(x, y, p=2., epsilon=1e-6, keepdim='False'): - pairwise_distance = paddle.nn.PairwiseDistance(p=p, - epsilon=epsilon, - keepdim=keepdim) +def call_pairwise_distance_layer(x, y, p=2.0, epsilon=1e-6, keepdim='False'): + pairwise_distance = paddle.nn.PairwiseDistance( + p=p, epsilon=epsilon, keepdim=keepdim + ) distance = pairwise_distance(x=x, y=y) return distance -def call_pairwise_distance_functional(x, - y, - p=2., - epsilon=1e-6, - keepdim='False'): - distance = paddle.nn.functional.pairwise_distance(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) +def call_pairwise_distance_functional( + x, y, p=2.0, epsilon=1e-6, keepdim='False' +): + distance = paddle.nn.functional.pairwise_distance( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) return distance -def test_static(place, - x_np, - y_np, - p=2.0, - epsilon=1e-6, - keepdim=False, - functional=False): +def test_static( + place, x_np, y_np, p=2.0, epsilon=1e-6, keepdim=False, functional=False +): prog = paddle.static.Program() startup_prog = paddle.static.Program() - place = fluid.CUDAPlace( - 0) if paddle.fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.enable_static() with paddle.static.program_guard(prog, startup_prog): x = paddle.fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = paddle.fluid.data(name='y', shape=y_np.shape, dtype=x_np.dtype) if functional: - distance = call_pairwise_distance_functional(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + distance = call_pairwise_distance_functional( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) else: - distance = call_pairwise_distance_layer(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + distance = call_pairwise_distance_layer( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) exe = paddle.static.Executor(place) - static_ret = exe.run(prog, - feed={ - 'x': x_np, - 'y': y_np - }, - fetch_list=[distance]) + static_ret = exe.run( + prog, feed={'x': x_np, 'y': y_np}, fetch_list=[distance] + ) static_ret = static_ret[0] paddle.disable_static() return static_ret -def test_dygraph(place, - x_np, - y_np, - p=2.0, - epsilon=1e-6, - keepdim=False, - functional=False): +def test_dygraph( + place, x_np, y_np, p=2.0, epsilon=1e-6, keepdim=False, functional=False +): x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) if functional: - dy_distance = call_pairwise_distance_functional(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + dy_distance = call_pairwise_distance_functional( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) else: - dy_distance = call_pairwise_distance_layer(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + dy_distance = call_pairwise_distance_layer( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) dygraph_ret = dy_distance.numpy() return dygraph_ret -def test_legacy_dygraph(place, - x_np, - y_np, - p=2.0, - epsilon=1e-6, - keepdim=False, - functional=False): +def test_legacy_dygraph( + place, x_np, y_np, p=2.0, epsilon=1e-6, keepdim=False, functional=False +): paddle.fluid.framework._enable_legacy_dygraph() x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) if functional: - legacy_distance = call_pairwise_distance_functional(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + legacy_distance = call_pairwise_distance_functional( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) else: - legacy_distance = call_pairwise_distance_layer(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + legacy_distance = call_pairwise_distance_layer( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) legacy_ret = legacy_distance.numpy() paddle.fluid.framework._disable_legacy_dygraph() return legacy_ret class TestPairwiseDistance(unittest.TestCase): - def test_pairwise_distance(self): epsilon = 1e-6 all_shape = [[5], [100, 100]] @@ -159,81 +130,107 @@ class TestPairwiseDistance(unittest.TestCase): x_np = np.random.random(shape).astype(dtype) y_np = np.random.random(shape).astype(dtype) - static_ret = test_static(place, - x_np, - y_np, - p, - epsilon=epsilon, - keepdim=keepdim) - dygraph_ret = test_dygraph(place, - x_np, - y_np, - p, - epsilon=epsilon, - keepdim=keepdim) - legacy_ret = test_legacy_dygraph(place, - x_np, - y_np, - p, - epsilon=epsilon, - keepdim=keepdim) + static_ret = test_static( + place, + x_np, + y_np, + p, + epsilon=epsilon, + keepdim=keepdim, + ) + dygraph_ret = test_dygraph( + place, + x_np, + y_np, + p, + epsilon=epsilon, + keepdim=keepdim, + ) + legacy_ret = test_legacy_dygraph( + place, + x_np, + y_np, + p, + epsilon=epsilon, + keepdim=keepdim, + ) excepted_value = np_pairwise_distance( - x_np, y_np, p, epsilon=epsilon, keepdim=keepdim) - - self.assertEqual(static_ret.shape, - excepted_value.shape) - self.assertEqual(dygraph_ret.shape, - excepted_value.shape) - self.assertEqual(legacy_ret.shape, - excepted_value.shape) - - np.testing.assert_allclose(static_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(dygraph_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(legacy_ret, - excepted_value, - rtol=1e-05) - - static_functional_ret = test_static(place, - x_np, - y_np, - p, - epsilon=epsilon, - keepdim=keepdim) + x_np, y_np, p, epsilon=epsilon, keepdim=keepdim + ) + + self.assertEqual( + static_ret.shape, excepted_value.shape + ) + self.assertEqual( + dygraph_ret.shape, excepted_value.shape + ) + self.assertEqual( + legacy_ret.shape, excepted_value.shape + ) + + np.testing.assert_allclose( + static_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dygraph_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + legacy_ret, excepted_value, rtol=1e-05 + ) + + static_functional_ret = test_static( + place, + x_np, + y_np, + p, + epsilon=epsilon, + keepdim=keepdim, + ) dygraph_functional_ret = test_dygraph( place, x_np, y_np, p, epsilon=epsilon, - keepdim=keepdim) + keepdim=keepdim, + ) legacy_functional_ret = test_legacy_dygraph( place, x_np, y_np, p, epsilon=epsilon, - keepdim=keepdim) - - self.assertEqual(static_functional_ret.shape, - excepted_value.shape) - self.assertEqual(dygraph_functional_ret.shape, - excepted_value.shape) - self.assertEqual(legacy_functional_ret.shape, - excepted_value.shape) - - np.testing.assert_allclose(static_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(dygraph_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(legacy_functional_ret, - excepted_value, - rtol=1e-05) + keepdim=keepdim, + ) + + self.assertEqual( + static_functional_ret.shape, + excepted_value.shape, + ) + self.assertEqual( + dygraph_functional_ret.shape, + excepted_value.shape, + ) + self.assertEqual( + legacy_functional_ret.shape, + excepted_value.shape, + ) + + np.testing.assert_allclose( + static_functional_ret, + excepted_value, + rtol=1e-05, + ) + np.testing.assert_allclose( + dygraph_functional_ret, + excepted_value, + rtol=1e-05, + ) + np.testing.assert_allclose( + legacy_functional_ret, + excepted_value, + rtol=1e-05, + ) def test_pairwise_distance_broadcast_1(self): shape_x = [100, 100] @@ -243,25 +240,18 @@ class TestPairwiseDistance(unittest.TestCase): place = paddle.CPUPlace() x_np = np.random.random(shape_x).astype('float32') y_np = np.random.random(shape_y).astype('float32') - static_ret = test_static(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - dygraph_ret = test_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - legacy_ret = test_legacy_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - excepted_value = np_pairwise_distance(x_np, - y_np, - epsilon=epsilon, - keepdim=keepdim) + static_ret = test_static( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + dygraph_ret = test_dygraph( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + legacy_ret = test_legacy_dygraph( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + excepted_value = np_pairwise_distance( + x_np, y_np, epsilon=epsilon, keepdim=keepdim + ) self.assertEqual(static_ret.shape, excepted_value.shape) self.assertEqual(dygraph_ret.shape, excepted_value.shape) @@ -271,38 +261,44 @@ class TestPairwiseDistance(unittest.TestCase): np.testing.assert_allclose(dygraph_ret, excepted_value, rtol=1e-05) np.testing.assert_allclose(legacy_ret, excepted_value, rtol=1e-05) - static_functional_ret = test_static(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) - dygraph_functional_ret = test_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) - legacy_functional_ret = test_legacy_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) + static_functional_ret = test_static( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) + dygraph_functional_ret = test_dygraph( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) + legacy_functional_ret = test_legacy_dygraph( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) self.assertEqual(static_functional_ret.shape, excepted_value.shape) self.assertEqual(dygraph_functional_ret.shape, excepted_value.shape) self.assertEqual(legacy_functional_ret.shape, excepted_value.shape) - np.testing.assert_allclose(static_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(dygraph_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(legacy_functional_ret, - excepted_value, - rtol=1e-05) + np.testing.assert_allclose( + static_functional_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dygraph_functional_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + legacy_functional_ret, excepted_value, rtol=1e-05 + ) def test_pairwise_distance_broadcast_2(self): shape_x = [100, 100] @@ -312,25 +308,18 @@ class TestPairwiseDistance(unittest.TestCase): place = paddle.CPUPlace() x_np = np.random.random(shape_x).astype('float32') y_np = np.random.random(shape_y).astype('float32') - static_ret = test_static(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - dygraph_ret = test_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - legacy_ret = test_legacy_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - excepted_value = np_pairwise_distance(x_np, - y_np, - epsilon=epsilon, - keepdim=keepdim) + static_ret = test_static( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + dygraph_ret = test_dygraph( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + legacy_ret = test_legacy_dygraph( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + excepted_value = np_pairwise_distance( + x_np, y_np, epsilon=epsilon, keepdim=keepdim + ) self.assertEqual(static_ret.shape, excepted_value.shape) self.assertEqual(dygraph_ret.shape, excepted_value.shape) @@ -340,38 +329,44 @@ class TestPairwiseDistance(unittest.TestCase): np.testing.assert_allclose(dygraph_ret, excepted_value, rtol=1e-05) np.testing.assert_allclose(legacy_ret, excepted_value, rtol=1e-05) - static_functional_ret = test_static(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) - dygraph_functional_ret = test_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) - legacy_functional_ret = test_legacy_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) + static_functional_ret = test_static( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) + dygraph_functional_ret = test_dygraph( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) + legacy_functional_ret = test_legacy_dygraph( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) self.assertEqual(static_functional_ret.shape, excepted_value.shape) self.assertEqual(dygraph_functional_ret.shape, excepted_value.shape) self.assertEqual(legacy_functional_ret.shape, excepted_value.shape) - np.testing.assert_allclose(static_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(dygraph_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(legacy_functional_ret, - excepted_value, - rtol=1e-05) + np.testing.assert_allclose( + static_functional_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dygraph_functional_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + legacy_functional_ret, excepted_value, rtol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py index fb69986a348..a452f2b2b15 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py @@ -19,7 +19,12 @@ import copy import os import subprocess -from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc +from paddle.distributed.utils.launch_utils import ( + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, +) def get_cluster_from_args(selected_gpus): @@ -47,10 +52,9 @@ def get_gpus(selected_gpus): return selected_gpus -def start_local_trainers_cpu(trainer_endpoints, - training_script, - training_script_args, - log_dir=None): +def start_local_trainers_cpu( + trainer_endpoints, training_script, training_script_args, log_dir=None +): current_env = copy.copy(os.environ.copy()) current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -64,15 +68,16 @@ def start_local_trainers_cpu(trainer_endpoints, "PADDLE_TRAINER_ID": "%d" % rank_id, "PADDLE_CURRENT_ENDPOINT": "%s" % endpoint, "PADDLE_TRAINERS_NUM": "%d" % n_rank, - "PADDLE_TRAINER_ENDPOINTS": ",".join(trainer_endpoints) + "PADDLE_TRAINER_ENDPOINTS": ",".join(trainer_endpoints), } current_env.update(proc_env) print("trainer proc env:{}".format(current_env)) - assert os.getenv('WITH_COVERAGE', - 'OFF') == 'OFF', "Gloo don't support WITH_COVERAGE." + assert ( + os.getenv('WITH_COVERAGE', 'OFF') == 'OFF' + ), "Gloo don't support WITH_COVERAGE." cmd = "python -u " + training_script print("start trainer proc:{} env:{}".format(cmd, proc_env)) @@ -92,17 +97,19 @@ def start_local_trainers_cpu(trainer_endpoints, return procs -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - eager_mode=True, - log_dir=None): +def start_local_trainers( + cluster, + pod, + training_script, + training_script_args, + eager_mode=True, + log_dir=None, +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -113,7 +120,7 @@ def start_local_trainers(cluster, "PADDLE_TRAINER_ID": "%d" % t.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } if not eager_mode: @@ -146,10 +153,11 @@ def start_local_trainers(cluster, class TestMultipleGpus(unittest.TestCase): - def run_mnist_2gpu(self, target_file_name, eager_mode=True): - if not fluid.core.is_compiled_with_cuda( - ) or fluid.core.get_cuda_device_count() == 0: + if ( + not fluid.core.is_compiled_with_cuda() + or fluid.core.get_cuda_device_count() == 0 + ): return selected_gpus = get_gpus('0,1') @@ -158,11 +166,13 @@ class TestMultipleGpus(unittest.TestCase): cluster, pod = get_cluster_from_args(selected_gpus) - procs = start_local_trainers(cluster, - pod, - eager_mode=eager_mode, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + eager_mode=eager_mode, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_endpoints()) @@ -174,15 +184,17 @@ class TestMultipleGpus(unittest.TestCase): class TestMultipleWithGloo(unittest.TestCase): - def run_mnist_2cpu(self, target_file_name): cluster, pod = get_cluster_from_args( - [0, 1]) #tmp use. for getting trainer_nranks() + [0, 1] + ) # tmp use. for getting trainer_nranks() - procs = start_local_trainers_cpu(cluster.trainers_endpoints(), - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers_cpu( + cluster.trainers_endpoints(), + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_nranks()) @@ -194,22 +206,21 @@ class TestMultipleWithGloo(unittest.TestCase): class TestDataParallelGradientCheck(TestMultipleGpus): - def test_multiple_gpus_dynamic(self): - self.run_mnist_2gpu('parallel_dygraph_gradient_check.py', - eager_mode=False) + self.run_mnist_2gpu( + 'parallel_dygraph_gradient_check.py', eager_mode=False + ) class TestDataParallelWithPyLayer(TestMultipleGpus): - def test_parallel_dygraph_dataparallel_with_pylayer(self): self.run_mnist_2gpu('parallel_dygraph_dataparallel_with_pylayer.py') - self.run_mnist_2gpu('parallel_dygraph_dataparallel_with_pylayer.py', - eager_mode=False) + self.run_mnist_2gpu( + 'parallel_dygraph_dataparallel_with_pylayer.py', eager_mode=False + ) class TestGradientCheckInEagerMode(TestMultipleGpus): - def test_multiple_gpus_dynamic(self): self.run_mnist_2gpu('parallel_dygraph_gradient_check_in_eager_mode.py') diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel_cpuonly.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel_cpuonly.py index 8862a3310af..540c668a6f8 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel_cpuonly.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel_cpuonly.py @@ -18,7 +18,12 @@ import copy import os import subprocess -from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc +from paddle.distributed.utils.launch_utils import ( + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, +) def get_cluster_from_args(selected_gpus): @@ -46,16 +51,14 @@ def get_gpus(selected_gpus): return selected_gpus -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - log_dir=None): +def start_local_trainers( + cluster, pod, training_script, training_script_args, log_dir=None +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -69,8 +72,7 @@ def start_local_trainers(cluster, "MASTER_ADDR": "127.0.0.1", "MASTER_PORT": "6170", "NCCL_DEBUG": "INFO", - "PADDLE_DISTRI_BACKEND": - "gloo", # make init_parallel_env get 'gloo' argument. + "PADDLE_DISTRI_BACKEND": "gloo", # make init_parallel_env get 'gloo' argument. } current_env.update(proc_env) @@ -100,10 +102,9 @@ def start_local_trainers(cluster, class TestMultipleGpus(unittest.TestCase): - def run_mnist_2gpu(self, target_file_name): - #if not fluid.core.is_compiled_with_cuda( - #) or fluid.core.get_cuda_device_count() == 0: + # if not fluid.core.is_compiled_with_cuda( + # ) or fluid.core.get_cuda_device_count() == 0: # return selected_gpus = get_gpus('0,1') @@ -111,10 +112,12 @@ class TestMultipleGpus(unittest.TestCase): pod = None cluster, pod = get_cluster_from_args(selected_gpus) - procs = start_local_trainers(cluster, - pod, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_nranks()) @@ -126,13 +129,11 @@ class TestMultipleGpus(unittest.TestCase): class TestDataParallelGradientCheck(TestMultipleGpus): - def test_multiple_gpus_dynamic(self): self.run_mnist_2gpu('parallel_dygraph_gradient_check.py') class TestDataParallelGradientCheckInEagerMode(TestMultipleGpus): - def test_multiple_gpus_dynamic(self): self.run_mnist_2gpu('parallel_dygraph_gradient_check_in_eager_mode.py') diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_diff_length_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_diff_length_gloo.py index 278ea58de37..bc1c5bf4cfd 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_diff_length_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_diff_length_gloo.py @@ -21,7 +21,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphSparseEmdedding_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True @@ -29,10 +28,12 @@ class TestParallelDygraphSparseEmdedding_GLOO(TestDistBase): self._diff_batch = True def test_sparse_embedding(self): - self.check_with_place("parallel_dygraph_sparse_embedding.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_sparse_embedding.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_gloo.py index c9e4d669179..ef56153af7f 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_gloo.py @@ -21,31 +21,33 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphSparseEmdedding_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_sparse_embedding(self): - self.check_with_place("parallel_dygraph_sparse_embedding.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_sparse_embedding.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSparseEmdeddingFP64_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_sparse_embedding_fp64(self): - self.check_with_place("parallel_dygraph_sparse_embedding_fp64.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_sparse_embedding_fp64.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_over_height_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_over_height_gloo.py index 76f2d3ecaa8..c56f63893aa 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_over_height_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_over_height_gloo.py @@ -21,7 +21,6 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphSparseEmdeddingOverHeight_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True @@ -32,7 +31,8 @@ class TestParallelDygraphSparseEmdeddingOverHeight_GLOO(TestDistBase): "parallel_dygraph_sparse_embedding_over_height.py", delta=1e-7, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_transformer_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_transformer_gloo.py index 6089ef5d829..1af28e65cd7 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_transformer_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_transformer_gloo.py @@ -22,21 +22,21 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphTransformer_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_transformer(self): - self.check_with_place("parallel_dygraph_transformer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphTransformerAccGrad_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True @@ -46,10 +46,12 @@ class TestParallelDygraphTransformerAccGrad_GLOO(TestDistBase): def test_transformer(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_transformer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_unused_variables_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_unused_variables_gloo.py index 0cadd9a61b7..9278ff7280e 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_unused_variables_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_unused_variables_gloo.py @@ -21,45 +21,48 @@ flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphUnusedVar_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_net(self): - self.check_with_place("parallel_dygraph_unused_variables.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_unused_variables.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphNoVar_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_net(self): - self.check_with_place("parallel_dygraph_none_var.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_none_var.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSharedUnusedVariables_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_mnist(self): - self.check_with_place("parallel_dygraph_shared_unused_var.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_shared_unused_var.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py index 7ef548536a3..7f30c15735f 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py @@ -33,27 +33,42 @@ mix_hidden_lr = 1e-3 embedding_name = 'emb' -def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, - is_sparse, **ignored): +def db_lstm( + word, + predicate, + ctx_n2, + ctx_n1, + ctx_0, + ctx_p1, + ctx_p2, + mark, + is_sparse, + **ignored +): # 8 features - predicate_embedding = fluid.layers.embedding(input=predicate, - is_sparse=is_sparse, - size=[pred_dict_len, word_dim], - dtype='float32', - param_attr='vemb') - - mark_embedding = fluid.layers.embedding(input=mark, - is_sparse=is_sparse, - size=[mark_dict_len, mark_dim], - dtype='float32') + predicate_embedding = fluid.layers.embedding( + input=predicate, + is_sparse=is_sparse, + size=[pred_dict_len, word_dim], + dtype='float32', + param_attr='vemb', + ) + + mark_embedding = fluid.layers.embedding( + input=mark, + is_sparse=is_sparse, + size=[mark_dict_len, mark_dim], + dtype='float32', + ) word_input = [word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2] emb_layers = [ - fluid.layers.embedding(size=[word_dict_len, word_dim], - is_sparse=is_sparse, - input=x, - param_attr=fluid.ParamAttr(name=embedding_name, - trainable=False)) + fluid.layers.embedding( + size=[word_dict_len, word_dim], + is_sparse=is_sparse, + input=x, + param_attr=fluid.ParamAttr(name=embedding_name, trainable=False), + ) for x in word_input ] # TODO(zcd): if the parameter is not trainable, the @@ -71,92 +86,98 @@ def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, hidden_0 = fluid.layers.sums(input=hidden_0_layers) - lstm_0 = fluid.layers.dynamic_lstm(input=hidden_0, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid') + lstm_0 = fluid.layers.dynamic_lstm( + input=hidden_0, + size=hidden_dim, + candidate_activation='relu', + gate_activation='sigmoid', + cell_activation='sigmoid', + ) # stack L-LSTM and R-LSTM with direct edges input_tmp = [hidden_0, lstm_0] for i in range(1, depth): - mix_hidden = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=hidden_dim, act='tanh'), - fluid.layers.fc(input=input_tmp[1], size=hidden_dim, act='tanh') - ]) - - lstm = fluid.layers.dynamic_lstm(input=mix_hidden, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid', - is_reverse=((i % 2) == 1)) + mix_hidden = fluid.layers.sums( + input=[ + fluid.layers.fc( + input=input_tmp[0], size=hidden_dim, act='tanh' + ), + fluid.layers.fc( + input=input_tmp[1], size=hidden_dim, act='tanh' + ), + ] + ) + + lstm = fluid.layers.dynamic_lstm( + input=mix_hidden, + size=hidden_dim, + candidate_activation='relu', + gate_activation='sigmoid', + cell_activation='sigmoid', + is_reverse=((i % 2) == 1), + ) input_tmp = [mix_hidden, lstm] - feature_out = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=label_dict_len, act='tanh'), - fluid.layers.fc(input=input_tmp[1], size=label_dict_len, act='tanh') - ]) + feature_out = fluid.layers.sums( + input=[ + fluid.layers.fc( + input=input_tmp[0], size=label_dict_len, act='tanh' + ), + fluid.layers.fc( + input=input_tmp[1], size=label_dict_len, act='tanh' + ), + ] + ) return feature_out class TestCRFModel(unittest.TestCase): - - def check_network_convergence(self, - is_sparse, - build_strategy=None, - use_cuda=True): + def check_network_convergence( + self, is_sparse, build_strategy=None, use_cuda=True + ): os.environ['CPU_NUM'] = str(4) main = fluid.Program() startup = fluid.Program() scope = fluid.Scope() with fluid.scope_guard(scope): with fluid.program_guard(main, startup): - word = fluid.layers.data(name='word_data', - shape=[1], - dtype='int64', - lod_level=1) - predicate = fluid.layers.data(name='verb_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_n2 = fluid.layers.data(name='ctx_n2_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_n1 = fluid.layers.data(name='ctx_n1_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_0 = fluid.layers.data(name='ctx_0_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_p1 = fluid.layers.data(name='ctx_p1_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_p2 = fluid.layers.data(name='ctx_p2_data', - shape=[1], - dtype='int64', - lod_level=1) - mark = fluid.layers.data(name='mark_data', - shape=[1], - dtype='int64', - lod_level=1) + word = fluid.layers.data( + name='word_data', shape=[1], dtype='int64', lod_level=1 + ) + predicate = fluid.layers.data( + name='verb_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_n2 = fluid.layers.data( + name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_n1 = fluid.layers.data( + name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_0 = fluid.layers.data( + name='ctx_0_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_p1 = fluid.layers.data( + name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_p2 = fluid.layers.data( + name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1 + ) + mark = fluid.layers.data( + name='mark_data', shape=[1], dtype='int64', lod_level=1 + ) feature_out = db_lstm(**locals()) - target = fluid.layers.data(name='target', - shape=[1], - dtype='int64', - lod_level=1) + target = fluid.layers.data( + name='target', shape=[1], dtype='int64', lod_level=1 + ) crf_cost = fluid.layers.linear_chain_crf( input=feature_out, label=target, - param_attr=fluid.ParamAttr(name='crfw', learning_rate=1e-1)) + param_attr=fluid.ParamAttr(name='crfw', learning_rate=1e-1), + ) avg_cost = paddle.mean(crf_cost) sgd_optimizer = fluid.optimizer.SGD( @@ -164,41 +185,63 @@ class TestCRFModel(unittest.TestCase): learning_rate=0.01, decay_steps=100000, decay_rate=0.5, - staircase=True)) + staircase=True, + ) + ) sgd_optimizer.minimize(avg_cost) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.conll05.test(), buf_size=8192), - batch_size=8) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.conll05.test(), buf_size=8192 + ), + batch_size=8, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) train_cp = compiler.CompiledProgram(main).with_data_parallel( - loss_name=avg_cost.name, build_strategy=build_strategy) - - feeder = fluid.DataFeeder(feed_list=[ - word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, - mark, target - ], - place=fluid.CPUPlace()) + loss_name=avg_cost.name, build_strategy=build_strategy + ) + + feeder = fluid.DataFeeder( + feed_list=[ + word, + ctx_n2, + ctx_n1, + ctx_0, + ctx_p1, + ctx_p2, + predicate, + mark, + target, + ], + place=fluid.CPUPlace(), + ) data = train_data() for i in range(4): cur_batch = next(data) print( - exe.run(train_cp, - feed=feeder.feed(cur_batch), - fetch_list=[avg_cost.name])[0]) + exe.run( + train_cp, + feed=feeder.feed(cur_batch), + fetch_list=[avg_cost.name], + )[0] + ) def _new_build_strategy(self, use_reduce=False): build_strategy = fluid.BuildStrategy() if use_reduce: - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) else: - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.AllReduce + ) return build_strategy @@ -207,46 +250,54 @@ class TestCRFModel(unittest.TestCase): self.check_network_convergence( is_sparse=True, build_strategy=self._new_build_strategy(), - use_cuda=True) + use_cuda=True, + ) self.check_network_convergence( is_sparse=True, build_strategy=self._new_build_strategy(), - use_cuda=False) + use_cuda=False, + ) def test_update_dense_parameter_all_reduce(self): if core.is_compiled_with_cuda(): self.check_network_convergence( is_sparse=False, build_strategy=self._new_build_strategy(), - use_cuda=True) + use_cuda=True, + ) self.check_network_convergence( is_sparse=False, build_strategy=self._new_build_strategy(), - use_cuda=False) + use_cuda=False, + ) def test_update_sparse_parameter_reduce(self): if core.is_compiled_with_cuda(): self.check_network_convergence( is_sparse=True, build_strategy=self._new_build_strategy(use_reduce=True), - use_cuda=True) + use_cuda=True, + ) self.check_network_convergence( is_sparse=True, build_strategy=self._new_build_strategy(use_reduce=True), - use_cuda=False) + use_cuda=False, + ) def test_update_dense_parameter_reduce(self): if core.is_compiled_with_cuda(): self.check_network_convergence( is_sparse=False, build_strategy=self._new_build_strategy(use_reduce=True), - use_cuda=True) + use_cuda=True, + ) self.check_network_convergence( is_sparse=False, build_strategy=self._new_build_strategy(use_reduce=True), - use_cuda=False) + use_cuda=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_drop_scope.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_drop_scope.py index 6ff24c19f46..9c8d3429937 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_drop_scope.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_drop_scope.py @@ -20,7 +20,6 @@ import os class TestParallelExecutorDropExeScope(unittest.TestCase): - def check_drop_scope(self, use_cuda=True): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -42,14 +41,18 @@ class TestParallelExecutorDropExeScope(unittest.TestCase): exec_strateg = fluid.ExecutionStrategy() exec_strateg.num_iteration_per_drop_scope = 10 - train_exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=train_program, - loss_name=loss.name, - exec_strategy=exec_strateg) - test_exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=test_program, - share_vars_from=train_exe, - exec_strategy=exec_strateg) + train_exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=train_program, + loss_name=loss.name, + exec_strategy=exec_strateg, + ) + test_exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=test_program, + share_vars_from=train_exe, + exec_strategy=exec_strateg, + ) x = numpy.random.random(size=(10, 1)).astype('float32') train_exe.run(feed={"X": x}, fetch_list=[loss.name]) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_dry_run.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_dry_run.py index d2feddffc85..51b234c3719 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_dry_run.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_dry_run.py @@ -23,16 +23,18 @@ os.environ['CPU_NUM'] = str(4) class TestBase(unittest.TestCase): - - def main(self, - network_func, - iter=10, - iter_per_pe=10, - use_gpu=True, - use_experimental_executor=False): + def main( + self, + network_func, + iter=10, + iter_per_pe=10, + use_gpu=True, + use_experimental_executor=False, + ): if use_gpu and not fluid.core.is_compiled_with_cuda(): logging.warning( - "Paddle is not compiled with CUDA, skip GPU unittests") + "Paddle is not compiled with CUDA, skip GPU unittests" + ) return main_prog = fluid.Program() @@ -42,28 +44,34 @@ class TestBase(unittest.TestCase): with fluid.scope_guard(scope): loss = network_func() exe = fluid.Executor( - fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace()) + fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() + ) exe.run(startup_prog) exe_strategy = fluid.ExecutionStrategy() exe_strategy._dry_run = True - exe_strategy.use_experimental_executor = use_experimental_executor + exe_strategy.use_experimental_executor = ( + use_experimental_executor + ) train_cp = compiler.CompiledProgram( - main_prog).with_data_parallel(loss_name=loss.name, - exec_strategy=exe_strategy) + main_prog + ).with_data_parallel( + loss_name=loss.name, exec_strategy=exe_strategy + ) for _ in range(iter): for _ in range(iter_per_pe): exe.run(train_cp) class TestMNISTDryRun(TestBase): - def test_mnist_dry_run(self): for use_gpu in (False, True): for use_experimental_executor in (False, True): - self.main(network_func=TestMNISTDryRun.network_func, - use_gpu=use_gpu, - use_experimental_executor=use_experimental_executor) + self.main( + network_func=TestMNISTDryRun.network_func, + use_gpu=use_gpu, + use_experimental_executor=use_experimental_executor, + ) @staticmethod def network_func(): diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_feed_persistable_var.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_feed_persistable_var.py index 622095b85a7..4b5781921e2 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_feed_persistable_var.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_feed_persistable_var.py @@ -23,26 +23,27 @@ import os class TestFeedPersistableVar(unittest.TestCase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) batch_size = 4 - cls.img, cls.label = init_data(batch_size, - img_shape=[784], - label_range=9) + cls.img, cls.label = init_data( + batch_size, img_shape=[784], label_range=9 + ) cls.feed_dict = { 'image': cls.img, 'label': cls.label, - 'learning_rate': numpy.array([1.0]).astype("float32") + 'learning_rate': numpy.array([1.0]).astype("float32"), } def optimizer(self): - learning_rate = fluid.layers.create_global_var(name="learning_rate", - shape=[1], - value=1.0, - dtype='float32', - persistable=True) + learning_rate = fluid.layers.create_global_var( + name="learning_rate", + shape=[1], + value=1.0, + dtype='float32', + persistable=True, + ) optimizer = fluid.optimizer.SGD(learning_rate=learning_rate) return optimizer @@ -62,7 +63,8 @@ class TestFeedPersistableVar(unittest.TestCase): exe.run(program=startup) compiled_prog = fluid.compiler.CompiledProgram( - main).with_data_parallel(loss_name=loss.name) + main + ).with_data_parallel(loss_name=loss.name) exe.run(program=compiled_prog, feed=feed_dict) @@ -70,12 +72,14 @@ class TestFeedPersistableVar(unittest.TestCase): self.check_feed_persistable_var(self.feed_dict) self.check_feed_persistable_var(self.feed_dict, use_cuda=True) - self.feed_dict['learning_rate'] = numpy.array([1.0, - 1.0]).astype("float32") + self.feed_dict['learning_rate'] = numpy.array([1.0, 1.0]).astype( + "float32" + ) self.check_feed_persistable_var(self.feed_dict, use_cuda=True) - self.feed_dict['learning_rate'] = numpy.array([1.0, - 1.0]).astype("float32") + self.feed_dict['learning_rate'] = numpy.array([1.0, 1.0]).astype( + "float32" + ) run = partial(self.check_feed_persistable_var, self.feed_dict) self.assertRaises(RuntimeError, run) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py index 3695de024f4..950ff45e86f 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py @@ -37,24 +37,25 @@ def Lenet(data, class_dim): class TestFetchAndFeed(unittest.TestCase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - def parallel_exe(self, - use_cuda, - run_parallel_exe, - use_faster_executor=False, - num_threads=4, - seed=1): + def parallel_exe( + self, + use_cuda, + run_parallel_exe, + use_faster_executor=False, + num_threads=4, + seed=1, + ): main_program = fluid.Program() startup = fluid.Program() startup.random_seed = seed with fluid.program_guard(main_program, startup): - data = fluid.layers.data(name='image', - shape=[3, 224, 224], - dtype='float32') + data = fluid.layers.data( + name='image', shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = Lenet(data, class_dim=102) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -62,14 +63,15 @@ class TestFetchAndFeed(unittest.TestCase): opt = fluid.optimizer.Momentum( learning_rate=0.1, momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) opt.minimize(loss) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) - #FIXME force disable enable_inplace and memory_optimize to pass the unittest + # FIXME force disable enable_inplace and memory_optimize to pass the unittest build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = False build_strategy.memory_optimize = False @@ -79,56 +81,66 @@ class TestFetchAndFeed(unittest.TestCase): train_cp = compiler.CompiledProgram(main_program).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - exec_strategy=exec_strategy) + exec_strategy=exec_strategy, + ) run_parallel_exe(train_cp, exe, use_cuda, data, label, loss) - def run_parallel_exe_with_fetch(self, compiled_program, exe, use_cuda, data, - label, loss): - + def run_parallel_exe_with_fetch( + self, compiled_program, exe, use_cuda, data, label, loss + ): def get_data(batch_size=8): np.random.seed(5) while True: img = np.random.random(size=[batch_size, 3, 224, 224]).astype( - np.float32) + np.float32 + ) l = (np.random.random(size=[batch_size, 1]) * 10).astype( - np.int64) + np.int64 + ) yield img, l fetch_list = [] all_vars = compiled_program._program.global_block().vars for k, v in all_vars.items(): - if ('tmp' not in k) and ( - k[0] != '_' or v.persistable - ) and v.type == core.VarDesc.VarType.LOD_TENSOR: + if ( + ('tmp' not in k) + and (k[0] != '_' or v.persistable) + and v.type == core.VarDesc.VarType.LOD_TENSOR + ): fetch_list.append(k) for batch_id, img_label in enumerate(get_data()): img, l = img_label train_inputs = {data.name: img, label.name: l} - ret = exe.run(compiled_program, - fetch_list=fetch_list, - feed=train_inputs, - return_numpy=True) + ret = exe.run( + compiled_program, + fetch_list=fetch_list, + feed=train_inputs, + return_numpy=True, + ) for i in range(len(fetch_list)): - assert not math.isnan(np.sum(ret[i])) and \ - not math.isinf(np.sum(ret[i])) + assert not math.isnan(np.sum(ret[i])) and not math.isinf( + np.sum(ret[i]) + ) if batch_id == 2: break - def run_parallel_exe_with_feed(self, compiled_program, exe, use_cuda, data, - label, loss): - + def run_parallel_exe_with_feed( + self, compiled_program, exe, use_cuda, data, label, loss + ): def get_data(batch_size=8): np.random.seed(5) while True: train_data = [] for _ in range(batch_size): img = np.random.random(size=[1, 3, 224, 224]).astype( - np.float32) + np.float32 + ) label = (np.random.random(size=[1, 1]) * 10).astype( - np.int64) + np.int64 + ) train_data.append([img, label]) yield train_data @@ -137,37 +149,45 @@ class TestFetchAndFeed(unittest.TestCase): reader = feeder.decorate_reader(get_data, multi_devices=True) for batch_id, data in enumerate(reader()): - loss_np = exe.run(compiled_program, - feed=data, - fetch_list=[loss.name])[0] + loss_np = exe.run( + compiled_program, feed=data, fetch_list=[loss.name] + )[0] print(batch_id, loss_np) if batch_id == 2: break def check_executor(self, use_faster_executor=False, num_threads=4): if core.is_compiled_with_cuda(): - self.parallel_exe(use_cuda=True, - run_parallel_exe=self.run_parallel_exe_with_fetch, - use_faster_executor=use_faster_executor, - num_threads=num_threads) - self.parallel_exe(use_cuda=False, - run_parallel_exe=self.run_parallel_exe_with_fetch, - use_faster_executor=use_faster_executor, - num_threads=num_threads) + self.parallel_exe( + use_cuda=True, + run_parallel_exe=self.run_parallel_exe_with_fetch, + use_faster_executor=use_faster_executor, + num_threads=num_threads, + ) + self.parallel_exe( + use_cuda=False, + run_parallel_exe=self.run_parallel_exe_with_fetch, + use_faster_executor=use_faster_executor, + num_threads=num_threads, + ) def test_fetch(self): for use_faster_executor in {True, False}: - self.check_executor(use_faster_executor=use_faster_executor, - num_threads=4) - self.check_executor(use_faster_executor=use_faster_executor, - num_threads=1) + self.check_executor( + use_faster_executor=use_faster_executor, num_threads=4 + ) + self.check_executor( + use_faster_executor=use_faster_executor, num_threads=1 + ) def test_feed(self): if core.is_compiled_with_cuda(): - self.parallel_exe(use_cuda=True, - run_parallel_exe=self.run_parallel_exe_with_feed) - self.parallel_exe(use_cuda=False, - run_parallel_exe=self.run_parallel_exe_with_feed) + self.parallel_exe( + use_cuda=True, run_parallel_exe=self.run_parallel_exe_with_feed + ) + self.parallel_exe( + use_cuda=False, run_parallel_exe=self.run_parallel_exe_with_feed + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_isolated_var.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_isolated_var.py index 3d2146ab36b..9cfd502b3a3 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_isolated_var.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_isolated_var.py @@ -25,7 +25,6 @@ def enable_parallel_ssa_executor(enabled=True): class TestParallelExecutorFetchIsolatedVarBase(unittest.TestCase): - def build_network(self, is_training): x = fluid.data(name='x', shape=[-1, 10], dtype='float32') y = fluid.data(name='y', shape=[-1, 10], dtype='float32') @@ -55,19 +54,30 @@ class TestParallelExecutorFetchIsolatedVarBase(unittest.TestCase): for use_experimental_executor in [False, True]: for use_parallel_ssa_executor in [False, True]: func = lambda: self.run_impl( - use_gpu, dev_cnt, is_training, + use_gpu, + dev_cnt, + is_training, use_experimental_executor, - use_parallel_ssa_executor) + use_parallel_ssa_executor, + ) self.run_func_with_guard(func) - def run_impl(self, use_gpu, dev_cnt, is_training, use_experimental_executor, - use_parallel_ssa_executor): + def run_impl( + self, + use_gpu, + dev_cnt, + is_training, + use_experimental_executor, + use_parallel_ssa_executor, + ): paddle.enable_static() enable_parallel_ssa_executor(use_parallel_ssa_executor) if fluid.is_compiled_with_cuda(): - if fluid.core.globals( - )['FLAGS_enable_parallel_graph'] and not use_gpu: + if ( + fluid.core.globals()['FLAGS_enable_parallel_graph'] + and not use_gpu + ): return # windows has only 1 GPU if use_gpu and dev_cnt > 1 and os.name == "nt": @@ -85,22 +95,23 @@ class TestParallelExecutorFetchIsolatedVarBase(unittest.TestCase): exe.run(fluid.default_startup_program()) prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss_name, - exec_strategy=self.exec_strategy(use_experimental_executor), - places=places) + fluid.default_main_program() + ).with_data_parallel( + loss_name=loss_name, + exec_strategy=self.exec_strategy(use_experimental_executor), + places=places, + ) BATCH_SIZE = 8 * dev_cnt for _ in range(10): x_np = np.random.random(size=[BATCH_SIZE, 10]).astype('float32') y_np = np.random.random(size=[BATCH_SIZE, 10]).astype('float32') - _, y_np_fetch = exe.run(prog, - feed={ - 'x': x_np, - 'y': y_np - }, - fetch_list=[loss, isolated_var]) + _, y_np_fetch = exe.run( + prog, + feed={'x': x_np, 'y': y_np}, + fetch_list=[loss, isolated_var], + ) np.testing.assert_array_equal(y_np, y_np_fetch) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fix_op_run_order.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fix_op_run_order.py index 80da6b5ac61..4dcf4673a09 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fix_op_run_order.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fix_op_run_order.py @@ -21,7 +21,6 @@ from paddle.nn import CrossEntropyLoss class TestFixOpRunOrder(unittest.TestCase): - def setUp(self): paddle.enable_static() paddle.seed(1) @@ -30,8 +29,11 @@ class TestFixOpRunOrder(unittest.TestCase): fluid.set_flags({'FLAGS_cudnn_deterministic': 1}) def get_place(self): - return paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + return ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def get_feed(self): batch_size = 4 @@ -44,12 +46,12 @@ class TestFixOpRunOrder(unittest.TestCase): startup_prog = paddle.static.Program() scope = paddle.static.Scope() with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name="image", - shape=[None, 3, 224, 224], - dtype="float32") - label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + image = paddle.static.data( + name="image", shape=[None, 3, 224, 224], dtype="float32" + ) + label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) model = resnet18() pred = model(image) loss_fn = CrossEntropyLoss() @@ -64,7 +66,8 @@ class TestFixOpRunOrder(unittest.TestCase): main_prog = paddle.static.CompiledProgram(main_prog).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - places=[self.get_place()]) + places=[self.get_place()], + ) exe = paddle.static.Executor(self.get_place()) with paddle.static.scope_guard(scope): diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_inference_feed_partial_data.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_inference_feed_partial_data.py index 926201e6eaa..37ccb429b6e 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_inference_feed_partial_data.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_inference_feed_partial_data.py @@ -18,7 +18,6 @@ import unittest class TestInferencePartialFeed(unittest.TestCase): - def setUp(self): self.iterations = 10 self.size = 10 @@ -44,12 +43,15 @@ class TestInferencePartialFeed(unittest.TestCase): exe.run(startup_prog) prog = fluid.CompiledProgram(main_prog).with_data_parallel( - places=places) + places=places + ) gen_random = lambda shape: np.random.uniform( - low=-1.0, high=1.0, size=shape).astype('float32') + low=-1.0, high=1.0, size=shape + ).astype('float32') assert_result = lambda feed, result: np.testing.assert_array_equal( - np.maximum(0, feed), result) + np.maximum(0, feed), result + ) def assert_merged_unmerged(merged, unmerged): unmerged = np.concatenate(unmerged, axis=0) @@ -68,10 +70,16 @@ class TestInferencePartialFeed(unittest.TestCase): fetch_list = [relu_x, relu_y, relu_lr] relu_x_np, relu_y_np, relu_lr_np = exe.run( - prog, feed=feed, fetch_list=fetch_list, return_merged=True) + prog, feed=feed, fetch_list=fetch_list, return_merged=True + ) - relu_x_np_unmerged, relu_y_np_unmerged, relu_lr_np_unmerged = exe.run( - prog, feed=feed, fetch_list=fetch_list, return_merged=False) + ( + relu_x_np_unmerged, + relu_y_np_unmerged, + relu_lr_np_unmerged, + ) = exe.run( + prog, feed=feed, fetch_list=fetch_list, return_merged=False + ) assert_merged_unmerged(relu_x_np, relu_x_np_unmerged) assert_merged_unmerged(relu_y_np, relu_y_np_unmerged) @@ -99,24 +107,28 @@ class TestInferencePartialFeed(unittest.TestCase): y_np_list.append(y_np) lr_np_list.append(lr_np) - feed_list.append({ - x.name: x_np, - y.name: y_np, - lr.name: lr_np - }) + feed_list.append( + {x.name: x_np, y.name: y_np, lr.name: lr_np} + ) fetch_list = [relu_x, relu_y, relu_lr] relu_x_np, relu_y_np, relu_lr_np = exe.run( prog, feed=feed_list, fetch_list=fetch_list, - return_merged=True) - - relu_x_np_unmerged, relu_y_np_unmerged, relu_lr_np_unmerged = exe.run( + return_merged=True, + ) + + ( + relu_x_np_unmerged, + relu_y_np_unmerged, + relu_lr_np_unmerged, + ) = exe.run( prog, feed=feed_list, fetch_list=fetch_list, - return_merged=False) + return_merged=False, + ) assert_merged_unmerged(relu_x_np, relu_x_np_unmerged) assert_merged_unmerged(relu_y_np, relu_y_np_unmerged) @@ -144,20 +156,18 @@ class TestInferencePartialFeed(unittest.TestCase): for p in places: for has_persistable in [False, True]: for use_split in [False, True]: - self.run_network(p, - use_split=use_split, - has_persistable=has_persistable) + self.run_network( + p, use_split=use_split, has_persistable=has_persistable + ) class TestInferencePartialFeedUsingDataLoader(unittest.TestCase): - def setUp(self): self.epoch_num = 3 self.batch_num = 101 # a prime number self.batch_size = 32 def create_reader(self): - def __impl__(): for _ in range(self.batch_num): yield np.random.random([self.batch_size, 1]).astype('float32'), @@ -167,10 +177,9 @@ class TestInferencePartialFeedUsingDataLoader(unittest.TestCase): def run_network(self, iterable, use_cuda, drop_last): x = fluid.data(shape=[None, 1], name='x', dtype='float32') places = fluid.cuda_places() if use_cuda else fluid.cpu_places(4) - loader = fluid.io.DataLoader.from_generator(feed_list=[x], - capacity=16, - iterable=iterable, - drop_last=drop_last) + loader = fluid.io.DataLoader.from_generator( + feed_list=[x], capacity=16, iterable=iterable, drop_last=drop_last + ) y = fluid.layers.fc(x, size=10) loss = fluid.layers.reduce_mean(y) @@ -178,17 +187,18 @@ class TestInferencePartialFeedUsingDataLoader(unittest.TestCase): exe.run(fluid.default_startup_program()) prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - places=places, loss_name=loss.name) + fluid.default_main_program() + ).with_data_parallel(places=places, loss_name=loss.name) - loader.set_batch_generator(self.create_reader(), - places=places if iterable else None) + loader.set_batch_generator( + self.create_reader(), places=places if iterable else None + ) for _ in range(self.epoch_num): actual_batch_num = 0 if loader.iterable: for feed_data in loader(): - x_data, = exe.run(prog, feed=feed_data, fetch_list=[x]) + (x_data,) = exe.run(prog, feed=feed_data, fetch_list=[x]) self.assertEqual(x_data.shape[0] % self.batch_size, 0) self.assertTrue(x_data.shape[0] != 0) actual_batch_num += int(x_data.shape[0] / self.batch_size) @@ -196,11 +206,12 @@ class TestInferencePartialFeedUsingDataLoader(unittest.TestCase): loader.start() try: while True: - x_data, = exe.run(prog, fetch_list=[x]) + (x_data,) = exe.run(prog, fetch_list=[x]) self.assertEqual(x_data.shape[0] % self.batch_size, 0) self.assertTrue(x_data.shape[0] != 0) - actual_batch_num += int(x_data.shape[0] / - self.batch_size) + actual_batch_num += int( + x_data.shape[0] / self.batch_size + ) except fluid.core.EOFException: loader.reset() @@ -210,8 +221,9 @@ class TestInferencePartialFeedUsingDataLoader(unittest.TestCase): self.assertGreater(self.batch_num, actual_batch_num) def test_main(self): - use_cuda_list = [False, True - ] if fluid.is_compiled_with_cuda() else [False] + use_cuda_list = ( + [False, True] if fluid.is_compiled_with_cuda() else [False] + ) iterable_list = [False, True] drop_last_list = [False, True] for iterable in iterable_list: diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py index b4a69e89e4b..1341ccaad31 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py @@ -32,8 +32,10 @@ def simple_fc_net(use_feed): hidden, size=200, act='tanh', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) prediction = fluid.layers.fc(hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) loss = paddle.mean(loss) @@ -52,7 +54,9 @@ def fc_with_batchnorm(use_feed): size=200, act='tanh', bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) + initializer=fluid.initializer.Constant(value=1.0) + ), + ) hidden = fluid.layers.batch_norm(input=hidden) with fluid.name_scope("fc_layer"): @@ -71,16 +75,13 @@ def init_data(): class TestMNIST(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - def _compare_reduce_and_allreduce(self, - model, - use_device, - delta1=1e-6, - delta2=1e-4): + def _compare_reduce_and_allreduce( + self, model, use_device, delta1=1e-6, delta2=1e-4 + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return @@ -89,23 +90,23 @@ class TestMNIST(TestParallelExecutorBase): img, label = init_data() - all_reduce_first_loss, all_reduce_last_loss, _ = self.check_network_convergence( + ( + all_reduce_first_loss, + all_reduce_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_reduce=False) + use_reduce=False, + ) reduce_first_loss, reduce_last_loss, _ = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_reduce=True) + use_reduce=True, + ) for loss in zip(all_reduce_first_loss, reduce_first_loss): self.assertAlmostEqual(loss[0], loss[1], delta=delta1) @@ -122,13 +123,12 @@ class TestMNIST(TestParallelExecutorBase): img, label = init_data() - self.check_network_convergence(simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, - use_device=use_device, - use_reduce=use_reduce) + self.check_network_convergence( + simple_fc_net, + feed_dict={"image": img, "label": label}, + use_device=use_device, + use_reduce=use_reduce, + ) def test_simple_fc(self): # use_device @@ -140,10 +140,12 @@ class TestMNIST(TestParallelExecutorBase): # use_device, use_reduce # NOTE: the computation result of nccl_reduce is non-deterministic, # related issue: https://github.com/NVIDIA/nccl/issues/157 - self._compare_reduce_and_allreduce(simple_fc_net, DeviceType.CUDA, 1e-5, - 1e-2) - self._compare_reduce_and_allreduce(simple_fc_net, DeviceType.CPU, 1e-5, - 1e-2) + self._compare_reduce_and_allreduce( + simple_fc_net, DeviceType.CUDA, 1e-5, 1e-2 + ) + self._compare_reduce_and_allreduce( + simple_fc_net, DeviceType.CPU, 1e-5, 1e-2 + ) def check_simple_fc_parallel_accuracy(self, use_device): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): @@ -153,29 +155,29 @@ class TestMNIST(TestParallelExecutorBase): single_first_loss, single_last_loss, _ = self.check_network_convergence( method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_parallel_executor=False) - parallel_first_loss, parallel_last_loss, _ = self.check_network_convergence( + use_parallel_executor=False, + ) + ( + parallel_first_loss, + parallel_last_loss, + _, + ) = self.check_network_convergence( method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_parallel_executor=True) + use_parallel_executor=True, + ) self.assertAlmostEquals( np.mean(parallel_first_loss), single_first_loss, delta=1e-6, ) - self.assertAlmostEquals(np.mean(parallel_last_loss), - single_last_loss, - delta=1e-6) + self.assertAlmostEquals( + np.mean(parallel_last_loss), single_last_loss, delta=1e-6 + ) def test_simple_fc_parallel_accuracy(self): self.check_simple_fc_parallel_accuracy(DeviceType.CUDA) @@ -188,31 +190,32 @@ class TestMNIST(TestParallelExecutorBase): return img, label = init_data() - self.check_network_convergence(fc_with_batchnorm, - feed_dict={ - "image": img, - "label": label - }, - use_device=use_device, - use_fast_executor=use_fast_executor) + self.check_network_convergence( + fc_with_batchnorm, + feed_dict={"image": img, "label": label}, + use_device=use_device, + use_fast_executor=use_fast_executor, + ) def test_batchnorm_fc(self): for use_device in (DeviceType.CPU, DeviceType.CUDA): for use_fast_executor in (False, True): - self.check_batchnorm_fc_convergence(use_device, - use_fast_executor) + self.check_batchnorm_fc_convergence( + use_device, use_fast_executor + ) def test_batchnorm_fc_with_new_strategy(self): # NOTE: the computation result of nccl_reduce is non-deterministic, # related issue: https://github.com/NVIDIA/nccl/issues/157 - self._compare_reduce_and_allreduce(fc_with_batchnorm, DeviceType.CUDA, - 1e-5, 1e-2) - self._compare_reduce_and_allreduce(fc_with_batchnorm, DeviceType.CPU, - 1e-5, 1e-2) + self._compare_reduce_and_allreduce( + fc_with_batchnorm, DeviceType.CUDA, 1e-5, 1e-2 + ) + self._compare_reduce_and_allreduce( + fc_with_batchnorm, DeviceType.CPU, 1e-5, 1e-2 + ) class TestMNISTNoReduce(unittest.TestCase): - def run_program(self, device_type): if device_type == DeviceType.CUDA: if not paddle.is_compiled_with_cuda(): @@ -237,16 +240,18 @@ class TestMNISTNoReduce(unittest.TestCase): build_strategy = paddle.static.BuildStrategy() build_strategy.reduce_strategy = no_reduce main_multi_place = paddle.static.CompiledProgram( - main).with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - places=places) + main + ).with_data_parallel( + loss_name=loss.name, build_strategy=build_strategy, places=places + ) build_strategy = paddle.static.BuildStrategy() build_strategy.reduce_strategy = no_reduce main_single_place = paddle.static.CompiledProgram( - main.clone()).with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - places=places[0]) + main.clone() + ).with_data_parallel( + loss_name=loss.name, build_strategy=build_strategy, places=places[0] + ) image, label = init_data() feed = {'image': image, 'label': label} @@ -254,9 +259,9 @@ class TestMNISTNoReduce(unittest.TestCase): scope = paddle.static.Scope() with paddle.static.scope_guard(scope): exe.run(startup) - grads_multi_place = exe.run(main_multi_place, - feed=feed, - fetch_list=[grads]) + grads_multi_place = exe.run( + main_multi_place, feed=feed, fetch_list=[grads] + ) feeds = self.split_feed(feed, len(places)) grads_single_place = [list() for _ in range(len(grads))] @@ -266,8 +271,9 @@ class TestMNISTNoReduce(unittest.TestCase): grads_single_place[i].append(g) for i in range(len(grads)): - grads_single_place[i] = np.concatenate(grads_single_place[i], - axis=0) / len(places) + grads_single_place[i] = np.concatenate( + grads_single_place[i], axis=0 + ) / len(places) self.assertEqual(len(grads_multi_place), len(grads_single_place)) for g1, g2 in zip(grads_multi_place, grads_single_place): diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py index e312cdd125d..8813c962f8b 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py @@ -25,7 +25,6 @@ from simple_nets import simple_fc_net, init_data class TestMNIST(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -36,13 +35,12 @@ class TestMNIST(TestParallelExecutorBase): return img, label = init_data() - self.check_network_convergence(simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, - use_device=use_device, - use_reduce=use_reduce) + self.check_network_convergence( + simple_fc_net, + feed_dict={"image": img, "label": label}, + use_device=use_device, + use_reduce=use_reduce, + ) def test_simple_fc(self): # use_device @@ -55,29 +53,29 @@ class TestMNIST(TestParallelExecutorBase): img, label = init_data() single_first_loss, single_last_loss, _ = self.check_network_convergence( method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_parallel_executor=False) - parallel_first_loss, parallel_last_loss, _ = self.check_network_convergence( + use_parallel_executor=False, + ) + ( + parallel_first_loss, + parallel_last_loss, + _, + ) = self.check_network_convergence( method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_parallel_executor=True) + use_parallel_executor=True, + ) self.assertAlmostEquals( np.mean(parallel_first_loss), single_first_loss, delta=1e-6, ) - self.assertAlmostEquals(np.mean(parallel_last_loss), - single_last_loss, - delta=1e-6) + self.assertAlmostEquals( + np.mean(parallel_last_loss), single_last_loss, delta=1e-6 + ) def test_simple_fc_parallel_accuracy(self): self.check_simple_fc_parallel_accuracy(DeviceType.CUDA) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_profiler.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_profiler.py index 6ba451b0e68..4d8dc470fa9 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_profiler.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_profiler.py @@ -27,19 +27,20 @@ os.environ['NCCL_SHM_DISABLE'] = str(1) class TestPEProfiler(TestProfiler): - def test_cpu_profiler(self): exe = fluid.Executor(fluid.CPUPlace()) self.net_profiler(exe, 'CPU', "Default", use_parallel_executor=True) - @unittest.skipIf(not core.is_compiled_with_cuda(), - "profiler is enabled only with GPU") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "profiler is enabled only with GPU" + ) def test_cuda_profiler(self): exe = fluid.Executor(fluid.CUDAPlace(0)) self.net_profiler(exe, 'GPU', "OpDetail", use_parallel_executor=True) - @unittest.skipIf(not core.is_compiled_with_cuda(), - "profiler is enabled only with GPU") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "profiler is enabled only with GPU" + ) def test_all_profiler(self): exe = fluid.Executor(fluid.CUDAPlace(0)) self.net_profiler(exe, 'All', "AllOpDetail", use_parallel_executor=True) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py index 40a1d087cdc..3bb546eb917 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py @@ -22,8 +22,9 @@ import unittest paddle.enable_static() -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', - level=logging.INFO) +logging.basicConfig( + format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO +) logger = logging.getLogger("paddle_with_cinn") @@ -39,8 +40,9 @@ def set_cinn_flag(val): def reader(limit): for _ in range(limit): - yield np.random.random([1, 28]).astype('float32'), \ - np.random.randint(0, 2, size=[1]).astype('int64') + yield np.random.random([1, 28]).astype('float32'), np.random.randint( + 0, 2, size=[1] + ).astype('int64') def rand_data(img, label, loop_num=10): @@ -59,8 +61,12 @@ def build_program(main_program, startup_program): name="bias", shape=[1, 28], dtype="float32", - attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Assign( - np.random.rand(1, 28).astype(np.float32)))) + attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Assign( + np.random.rand(1, 28).astype(np.float32) + ) + ), + ) label = paddle.static.data(name="label", shape=[1], dtype='int64') hidden = paddle.add(img, param) @@ -83,15 +89,19 @@ def train(dot_save_dir, prefix, seed=1234): main_program = paddle.static.Program() img, label, loss = build_program(main_program, startup_program) - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) exe.run(startup_program) build_strategy = paddle.static.BuildStrategy() build_strategy.debug_graphviz_path = os.path.join(dot_save_dir, prefix) compiled_program = paddle.static.CompiledProgram( - main_program, build_strategy).with_data_parallel(loss_name=loss.name) + main_program, build_strategy + ).with_data_parallel(loss_name=loss.name) iters = 100 feed = rand_data(img.name, label.name, iters) @@ -104,7 +114,6 @@ def train(dot_save_dir, prefix, seed=1234): @unittest.skipIf(not set_cinn_flag(True), "Paddle is not compiled with CINN.") class TestParallelExecutorRunCinn(unittest.TestCase): - def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix="dots_") @@ -115,10 +124,9 @@ class TestParallelExecutorRunCinn(unittest.TestCase): cinn_losses = train(self.tmpdir, "paddle") set_cinn_flag(False) pd_losses = train(self.tmpdir, "cinn") - np.testing.assert_allclose(cinn_losses, - pd_losses, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + cinn_losses, pd_losses, rtol=1e-05, atol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_load_infer_program.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_load_infer_program.py index c0cd03b71fa..461461f7975 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_load_infer_program.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_load_infer_program.py @@ -19,12 +19,15 @@ from simple_nets import simple_fc_net, init_data class TestMNIST(unittest.TestCase): - @classmethod def setUpClass(cls): cls.save_dirname = "./" - cls.model_filename = "test_parallel_executor_run_load_infer_program_model" - cls.params_filename = "test_parallel_executor_run_load_infer_program_parameter" + cls.model_filename = ( + "test_parallel_executor_run_load_infer_program_model" + ) + cls.params_filename = ( + "test_parallel_executor_run_load_infer_program_parameter" + ) cls.place = fluid.CPUPlace() cls.exe = fluid.Executor(cls.place) img, label = init_data() @@ -35,22 +38,30 @@ class TestMNIST(unittest.TestCase): def test_simple_fc(self): exe_loss = self.run_with_executor() - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(self.save_dirname, - self.exe, - self.model_filename, - self.params_filename) - - train_exe = fluid.ParallelExecutor(use_cuda=False, - main_program=inference_program) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.save_dirname, + self.exe, + self.model_filename, + self.params_filename, + ) + + train_exe = fluid.ParallelExecutor( + use_cuda=False, main_program=inference_program + ) feed_vars = [ inference_program.global_block().var(var_name) for var_name in ["image", "label"] ] feeder = fluid.DataFeeder(place=self.place, feed_list=feed_vars) - pe_loss = train_exe.run(feed=feeder.feed(self.batch_data), - fetch_list=[fetch_targets[0].name]) + pe_loss = train_exe.run( + feed=feeder.feed(self.batch_data), + fetch_list=[fetch_targets[0].name], + ) assert exe_loss == pe_loss def run_with_executor(self): @@ -60,23 +71,25 @@ class TestMNIST(unittest.TestCase): loss = simple_fc_net() feed_vars = [ - main.global_block().var(var_name) - for var_name in ["image", "label"] + main.global_block().var(var_name) for var_name in ["image", "label"] ] feeder = fluid.DataFeeder(place=self.place, feed_list=feed_vars) self.exe.run(startup) - loss_data = self.exe.run(main, - feed=feeder.feed(self.batch_data), - fetch_list=[loss.name]) - - fluid.io.save_inference_model(self.save_dirname, ["image", "label"], - [loss], - self.exe, - model_filename=self.model_filename, - params_filename=self.params_filename, - main_program=main) + loss_data = self.exe.run( + main, feed=feeder.feed(self.batch_data), fetch_list=[loss.name] + ) + + fluid.io.save_inference_model( + self.save_dirname, + ["image", "label"], + [loss], + self.exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + main_program=main, + ) return loss_data diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_cpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_cpu.py index a3394833fd4..79f5fd4c3ca 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_cpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_cpu.py @@ -19,19 +19,22 @@ from functools import partial class TestResnetCPU(TestResnetBase): - def test_seresnext_with_learning_rate_decay(self): # NOTE(zcd): This test is compare the result of use parallel_executor # and executor, and the result of drop_out op and batch_norm op in # this two executor have diff, so the two ops should be removed # from the model. - check_func = partial(self.check_network_convergence, - optimizer=seresnext_net.optimizer, - use_parallel_executor=False) - self._compare_result_with_origin_model(check_func, - use_device=DeviceType.CPU, - compare_separately=False, - delta2=1e-3) + check_func = partial( + self.check_network_convergence, + optimizer=seresnext_net.optimizer, + use_parallel_executor=False, + ) + self._compare_result_with_origin_model( + check_func, + use_device=DeviceType.CPU, + compare_separately=False, + delta2=1e-3, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_gpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_gpu.py index e3f7d0a3462..6f7f0f507a4 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_gpu.py @@ -19,19 +19,22 @@ from functools import partial class TestResnetGPU(TestResnetBase): - def test_seresnext_with_learning_rate_decay(self): # NOTE(zcd): This test is compare the result of use parallel_executor # and executor, and the result of drop_out op and batch_norm op in # this two executor have diff, so the two ops should be removed # from the model. - check_func = partial(self.check_network_convergence, - optimizer=seresnext_net.optimizer, - use_parallel_executor=False) - self._compare_result_with_origin_model(check_func, - use_device=DeviceType.CUDA, - delta2=1e-5, - compare_separately=False) + check_func = partial( + self.check_network_convergence, + optimizer=seresnext_net.optimizer, + use_parallel_executor=False, + ) + self._compare_result_with_origin_model( + check_func, + use_device=DeviceType.CUDA, + delta2=1e-5, + compare_separately=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_cpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_cpu.py index 4ef6be1667c..46b7bb83147 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_cpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_cpu.py @@ -24,15 +24,17 @@ from functools import partial class TestResnetWithFuseAllReduceCPU(TestResnetBase): - def test_seresnext_with_fused_all_reduce(self): # NOTE(zcd): In order to make the program faster, # this unit test remove drop_out and batch_norm. - check_func = partial(self.check_network_convergence, - optimizer=seresnext_net.optimizer, - fuse_all_reduce_ops=True) - self._compare_result_with_origin_model(check_func, - use_device=DeviceType.CPU) + check_func = partial( + self.check_network_convergence, + optimizer=seresnext_net.optimizer, + fuse_all_reduce_ops=True, + ) + self._compare_result_with_origin_model( + check_func, use_device=DeviceType.CPU + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_gpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_gpu.py index 016c8f41cbe..ca349fc1100 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_gpu.py @@ -24,16 +24,17 @@ from functools import partial class TestResnetWithFuseAllReduceGPU(TestResnetBase): - def test_seresnext_with_fused_all_reduce(self): # NOTE(zcd): In order to make the program faster, # this unit test remove drop_out and batch_norm. - check_func = partial(self.check_network_convergence, - optimizer=seresnext_net.optimizer, - fuse_all_reduce_ops=True) - self._compare_result_with_origin_model(check_func, - use_device=DeviceType.CUDA, - delta2=1e-2) + check_func = partial( + self.check_network_convergence, + optimizer=seresnext_net.optimizer, + fuse_all_reduce_ops=True, + ) + self._compare_result_with_origin_model( + check_func, use_device=DeviceType.CUDA, delta2=1e-2 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py index d8d854fd315..abd650468c5 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py @@ -19,19 +19,23 @@ import paddle.fluid.core as core class TestResnetWithReduceBase(TestParallelExecutorBase): - def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return - all_reduce_first_loss, all_reduce_last_loss, _ = self.check_network_convergence( + ( + all_reduce_first_loss, + all_reduce_last_loss, + _, + ) = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), batch_size=seresnext_net.batch_size(use_device), use_device=use_device, use_reduce=False, - optimizer=seresnext_net.optimizer) + optimizer=seresnext_net.optimizer, + ) reduce_first_loss, reduce_last_loss, _ = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), @@ -39,7 +43,8 @@ class TestResnetWithReduceBase(TestParallelExecutorBase): batch_size=seresnext_net.batch_size(use_device), use_device=use_device, use_reduce=True, - optimizer=seresnext_net.optimizer) + optimizer=seresnext_net.optimizer, + ) for loss in zip(all_reduce_first_loss, reduce_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-5) @@ -49,7 +54,11 @@ class TestResnetWithReduceBase(TestParallelExecutorBase): if not use_device: return - all_reduce_first_loss_seq, all_reduce_last_loss_seq, _ = self.check_network_convergence( + ( + all_reduce_first_loss_seq, + all_reduce_last_loss_seq, + _, + ) = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), @@ -57,9 +66,14 @@ class TestResnetWithReduceBase(TestParallelExecutorBase): use_device=use_device, use_reduce=False, optimizer=seresnext_net.optimizer, - enable_sequential_execution=True) - - reduce_first_loss_seq, reduce_last_loss_seq, _ = self.check_network_convergence( + enable_sequential_execution=True, + ) + + ( + reduce_first_loss_seq, + reduce_last_loss_seq, + _, + ) = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), @@ -67,7 +81,8 @@ class TestResnetWithReduceBase(TestParallelExecutorBase): use_device=use_device, use_reduce=True, optimizer=seresnext_net.optimizer, - enable_sequential_execution=True) + enable_sequential_execution=True, + ) for loss in zip(all_reduce_first_loss, all_reduce_first_loss_seq): self.assertAlmostEquals(loss[0], loss[1], delta=1e-5) @@ -86,10 +101,10 @@ class TestResnetWithReduceBase(TestParallelExecutorBase): class TestResnetWithReduceCPU(TestResnetWithReduceBase): - def test_seresnext_with_reduce(self): - self._compare_reduce_and_allreduce(use_device=DeviceType.CPU, - delta2=1e-3) + self._compare_reduce_and_allreduce( + use_device=DeviceType.CPU, delta2=1e-3 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_gpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_gpu.py index 726f42e12a0..e4cce04e2a4 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_gpu.py @@ -13,14 +13,17 @@ # limitations under the License. import unittest -from test_parallel_executor_seresnext_with_reduce_cpu import TestResnetWithReduceBase, DeviceType +from test_parallel_executor_seresnext_with_reduce_cpu import ( + TestResnetWithReduceBase, + DeviceType, +) class TestResnetWithReduceGPU(TestResnetWithReduceBase): - def test_seresnext_with_reduce(self): - self._compare_reduce_and_allreduce(use_device=DeviceType.CUDA, - delta2=1e-2) + self._compare_reduce_and_allreduce( + use_device=DeviceType.CUDA, delta2=1e-2 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py index 8366f8f29cd..9391424692b 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py @@ -24,7 +24,6 @@ import math class ParallelExecutorTestingDuringTraining(unittest.TestCase): - def check_network_convergence(self, use_cuda, build_strategy=None): os.environ['CPU_NUM'] = str(4) main = fluid.Program() @@ -46,20 +45,22 @@ class ParallelExecutorTestingDuringTraining(unittest.TestCase): feed_dict = {'image': image, 'label': label} train_cp = compiler.CompiledProgram(main).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) test_cp = compiler.CompiledProgram(test_program).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - share_vars_from=train_cp) + share_vars_from=train_cp, + ) for i in range(5): exe.run(train_cp, feed=feed_dict, fetch_list=[loss.name]) - test_loss, = exe.run(test_cp, - feed=feed_dict, - fetch_list=[loss.name]) - train_loss, = exe.run(train_cp, - feed=feed_dict, - fetch_list=[loss.name]) + (test_loss,) = exe.run( + test_cp, feed=feed_dict, fetch_list=[loss.name] + ) + (train_loss,) = exe.run( + train_cp, feed=feed_dict, fetch_list=[loss.name] + ) avg_test_loss_val = np.array(test_loss).mean() if math.isnan(float(avg_test_loss_val)): @@ -69,32 +70,41 @@ class ParallelExecutorTestingDuringTraining(unittest.TestCase): if math.isnan(float(avg_train_loss_val)): sys.exit("got NaN loss, training failed.") - np.testing.assert_allclose(train_loss, - test_loss, - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + train_loss, test_loss, rtol=1e-05, atol=0.01 + ) def test_parallel_testing(self): build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.AllReduce + ) if core.is_compiled_with_cuda(): - self.check_network_convergence(use_cuda=True, - build_strategy=build_strategy) - self.check_network_convergence(use_cuda=False, - build_strategy=build_strategy) + self.check_network_convergence( + use_cuda=True, build_strategy=build_strategy + ) + self.check_network_convergence( + use_cuda=False, build_strategy=build_strategy + ) def test_parallel_testing_with_new_strategy_gpu(self): build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) if core.is_compiled_with_cuda(): - self.check_network_convergence(use_cuda=True, - build_strategy=build_strategy) + self.check_network_convergence( + use_cuda=True, build_strategy=build_strategy + ) def test_parallel_testing_with_new_strategy_cpu(self): build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce - self.check_network_convergence(use_cuda=False, - build_strategy=build_strategy) + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) + self.check_network_convergence( + use_cuda=False, build_strategy=build_strategy + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py index 684be9023b6..d447860862b 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py @@ -77,12 +77,14 @@ def prepare_batch_input(insts, src_pad_idx, trg_pad_idx, n_head): data to tensors and return a dict mapping names to tensors. """ - def __pad_batch_data(insts, - pad_idx, - is_target=False, - return_pos=True, - return_attn_bias=True, - return_max_len=True): + def __pad_batch_data( + insts, + pad_idx, + is_target=False, + return_pos=True, + return_attn_bias=True, + return_max_len=True, + ): """ Pad the instances to the max sequence length in batch, and generate the corresponding position data and attention bias. @@ -90,13 +92,19 @@ def prepare_batch_input(insts, src_pad_idx, trg_pad_idx, n_head): return_list = [] max_len = max(len(inst) for inst in insts) inst_data = np.array( - [inst + [pad_idx] * (max_len - len(inst)) for inst in insts]) + [inst + [pad_idx] * (max_len - len(inst)) for inst in insts] + ) return_list += [inst_data.astype("int64").reshape([-1, 1])] if return_pos: - inst_pos = np.array([[ - pos_i + 1 if w_i != pad_idx else 0 - for pos_i, w_i in enumerate(inst) - ] for inst in inst_data]) + inst_pos = np.array( + [ + [ + pos_i + 1 if w_i != pad_idx else 0 + for pos_i, w_i in enumerate(inst) + ] + for inst in inst_data + ] + ) return_list += [inst_pos.astype("int64").reshape([-1, 1])] if return_attn_bias: @@ -104,37 +112,55 @@ def prepare_batch_input(insts, src_pad_idx, trg_pad_idx, n_head): # This is used to avoid attention on paddings and subsequent # words. slf_attn_bias_data = np.ones( - (inst_data.shape[0], max_len, max_len)) + (inst_data.shape[0], max_len, max_len) + ) slf_attn_bias_data = np.triu(slf_attn_bias_data, 1).reshape( - [-1, 1, max_len, max_len]) - slf_attn_bias_data = np.tile(slf_attn_bias_data, - [1, n_head, 1, 1]) * [-1e9] + [-1, 1, max_len, max_len] + ) + slf_attn_bias_data = np.tile( + slf_attn_bias_data, [1, n_head, 1, 1] + ) * [-1e9] else: # This is used to avoid attention on paddings. - slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] * - (max_len - len(inst)) - for inst in insts]) + slf_attn_bias_data = np.array( + [ + [0] * len(inst) + [-1e9] * (max_len - len(inst)) + for inst in insts + ] + ) slf_attn_bias_data = np.tile( slf_attn_bias_data.reshape([-1, 1, 1, max_len]), - [1, n_head, max_len, 1]) + [1, n_head, max_len, 1], + ) return_list += [slf_attn_bias_data.astype("float32")] if return_max_len: return_list += [max_len] return return_list if len(return_list) > 1 else return_list[0] src_word, src_pos, src_slf_attn_bias, src_max_len = __pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, is_target=False) + [inst[0] for inst in insts], src_pad_idx, is_target=False + ) trg_word, trg_pos, trg_slf_attn_bias, trg_max_len = __pad_batch_data( - [inst[1] for inst in insts], trg_pad_idx, is_target=True) - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, trg_max_len, 1]).astype("float32") - lbl_word = __pad_batch_data([inst[2] for inst in insts], trg_pad_idx, False, - False, False, False) + [inst[1] for inst in insts], trg_pad_idx, is_target=True + ) + trg_src_attn_bias = np.tile( + src_slf_attn_bias[:, :, ::src_max_len, :], [1, 1, trg_max_len, 1] + ).astype("float32") + lbl_word = __pad_batch_data( + [inst[2] for inst in insts], trg_pad_idx, False, False, False, False + ) lbl_weight = (lbl_word != trg_pad_idx).astype("float32").reshape([-1, 1]) return [ - src_word, src_pos, trg_word, trg_pos, src_slf_attn_bias, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight + src_word, + src_pos, + trg_word, + trg_pos, + src_slf_attn_bias, + trg_slf_attn_bias, + trg_src_attn_bias, + lbl_word, + lbl_weight, ] @@ -145,12 +171,19 @@ def transformer(use_feed): assert not use_feed, "transfomer doesn't support feed yet" return transformer_model.transformer( ModelHyperParams.src_vocab_size + 1, - ModelHyperParams.trg_vocab_size + 1, ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, ModelHyperParams.n_head, - ModelHyperParams.d_key, ModelHyperParams.d_value, - ModelHyperParams.d_model, ModelHyperParams.d_inner_hid, - ModelHyperParams.dropout, ModelHyperParams.src_pad_idx, - ModelHyperParams.trg_pad_idx, ModelHyperParams.pos_pad_idx) + ModelHyperParams.trg_vocab_size + 1, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.dropout, + ModelHyperParams.src_pad_idx, + ModelHyperParams.trg_pad_idx, + ModelHyperParams.pos_pad_idx, + ) def get_feed_data_reader(): @@ -158,15 +191,21 @@ def get_feed_data_reader(): if feed_data_reader is not None: return feed_data_reader - reader = paddle.batch(wmt16.train(ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size), - batch_size=transformer_model.batch_size) + reader = paddle.batch( + wmt16.train( + ModelHyperParams.src_vocab_size, ModelHyperParams.trg_vocab_size + ), + batch_size=transformer_model.batch_size, + ) all_batch_tensors = [] for batch in reader(): tensors = [] - for tensor in prepare_batch_input(batch, ModelHyperParams.src_pad_idx, - ModelHyperParams.trg_pad_idx, - ModelHyperParams.n_head): + for tensor in prepare_batch_input( + batch, + ModelHyperParams.src_pad_idx, + ModelHyperParams.trg_pad_idx, + ModelHyperParams.n_head, + ): tensors.append(np.array(tensor)) all_batch_tensors.append(tensors) @@ -174,30 +213,36 @@ def get_feed_data_reader(): for t in all_batch_tensors: yield t - feed_data_reader = FeedDataReader(feed_list=transformer_model.build_inputs( - ModelHyperParams.max_length + 1, ModelHyperParams.n_head), - reader=__reader__) + feed_data_reader = FeedDataReader( + feed_list=transformer_model.build_inputs( + ModelHyperParams.max_length + 1, ModelHyperParams.n_head + ), + reader=__reader__, + ) return feed_data_reader class TestTransformer(TestParallelExecutorBase): - def test_main(self): if core.is_compiled_with_cuda(): self.check_network_convergence( transformer, use_device=DeviceType.CUDA, - feed_data_reader=get_feed_data_reader()) + feed_data_reader=get_feed_data_reader(), + ) self.check_network_convergence( transformer, use_device=DeviceType.CUDA, enable_sequential_execution=True, - feed_data_reader=get_feed_data_reader()) - self.check_network_convergence(transformer, - use_device=DeviceType.CPU, - iter=2, - feed_data_reader=get_feed_data_reader()) + feed_data_reader=get_feed_data_reader(), + ) + self.check_network_convergence( + transformer, + use_device=DeviceType.CPU, + iter=2, + feed_data_reader=get_feed_data_reader(), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parameter.py b/python/paddle/fluid/tests/unittests/test_parameter.py index e066b961271..d077fb98362 100644 --- a/python/paddle/fluid/tests/unittests/test_parameter.py +++ b/python/paddle/fluid/tests/unittests/test_parameter.py @@ -16,7 +16,12 @@ import unittest import copy import paddle from paddle.fluid.dygraph import guard -from paddle.fluid.framework import default_main_program, Variable, _test_eager_guard, ParamBase +from paddle.fluid.framework import ( + default_main_program, + Variable, + _test_eager_guard, + ParamBase, +) import paddle.fluid.core as core from paddle.fluid.executor import Executor import paddle.fluid.io as io @@ -28,15 +33,16 @@ main_program = default_main_program() class ParameterChecks(unittest.TestCase): - def test_parameter(self): shape = [784, 100] val = 1.0625 b = main_program.global_block() - param = b.create_parameter(name='fc.w', - shape=shape, - dtype='float32', - initializer=ConstantInitializer(val)) + param = b.create_parameter( + name='fc.w', + shape=shape, + dtype='float32', + initializer=ConstantInitializer(val), + ) self.assertIsNotNone(param) self.assertEqual('fc.w', param.name) self.assertEqual((784, 100), param.shape) @@ -65,8 +71,9 @@ class ParameterChecks(unittest.TestCase): np.testing.assert_array_equal(param_copy.numpy(), param.numpy()) self.assertEqual(param_copy.optimize_attr, param.optimize_attr) self.assertEqual(param_copy.regularizer, param.regularizer) - self.assertEqual(param_copy.do_model_average, - param.do_model_average) + self.assertEqual( + param_copy.do_model_average, param.do_model_average + ) self.assertEqual(param_copy.need_clip, param.need_clip) self.assertEqual(param_copy.is_distributed, param.is_distributed) @@ -84,30 +91,27 @@ class ParameterChecks(unittest.TestCase): def func_exception(self): b = main_program.global_block() with self.assertRaises(ValueError): - b.create_parameter(name='test', - shape=None, - dtype='float32', - initializer=None) + b.create_parameter( + name='test', shape=None, dtype='float32', initializer=None + ) with self.assertRaises(ValueError): - b.create_parameter(name='test', - shape=[1], - dtype=None, - initializer=None) + b.create_parameter( + name='test', shape=[1], dtype=None, initializer=None + ) with self.assertRaises(ValueError): - b.create_parameter(name='test', - shape=[], - dtype='float32', - initializer=None) + b.create_parameter( + name='test', shape=[], dtype='float32', initializer=None + ) with self.assertRaises(ValueError): - b.create_parameter(name='test', - shape=[-1], - dtype='float32', - initializer=None) + b.create_parameter( + name='test', shape=[-1], dtype='float32', initializer=None + ) def func_parambase_to_vector(self): with guard(): initializer = paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(3.)) + initializer=paddle.nn.initializer.Constant(3.0) + ) linear1 = paddle.nn.Linear(10, 15, initializer) vec = paddle.nn.utils.parameters_to_vector(linear1.parameters()) @@ -120,10 +124,12 @@ class ParameterChecks(unittest.TestCase): paddle.nn.utils.vector_to_parameters(vec, linear2.parameters()) self.assertEqual(linear2.weight.shape, [10, 15]) self.assertEqual(linear2.bias.shape, [15]) - np.testing.assert_array_equal(linear1.weight.numpy(), - linear2.weight.numpy()) - np.testing.assert_array_equal(linear1.bias.numpy(), - linear2.bias.numpy()) + np.testing.assert_array_equal( + linear1.weight.numpy(), linear2.weight.numpy() + ) + np.testing.assert_array_equal( + linear1.bias.numpy(), linear2.bias.numpy() + ) self.assertTrue(linear2.weight.is_leaf, True) self.assertTrue(linear2.bias.is_leaf, True) diff --git a/python/paddle/fluid/tests/unittests/test_partial_concat_op.py b/python/paddle/fluid/tests/unittests/test_partial_concat_op.py index 1e4c0076da6..8046d8fa1d7 100644 --- a/python/paddle/fluid/tests/unittests/test_partial_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_partial_concat_op.py @@ -19,33 +19,34 @@ import random def np_partial_concat(inputs, start, length): - assert (len(inputs[0].shape) == 2) + assert len(inputs[0].shape) == 2 size = inputs[0].shape[1] - assert (start >= -size and start < size) + assert start >= -size and start < size if start < 0: start += size if length < 0: length = size - start - assert (size >= start + length) + assert size >= start + length elems = [] for elem in inputs: - assert (elem.shape == inputs[0].shape) - elems.append(elem[:, start:start + length]) + assert elem.shape == inputs[0].shape + elems.append(elem[:, start : start + length]) res = np.concatenate(elems, axis=1) return np.concatenate(elems, axis=1) class TestPartialConcatOp(OpTest): - def setUp(self): self.op_type = "partial_concat" self.init_kernel_type() self.init_para() self.var_names = ['x' + str(num) for num in range(self.var_num)] - self.vars = [np.random.random((self.batch_size, self.column)).astype(self.dtype)\ - for num in range(self.var_num) ] + self.vars = [ + np.random.random((self.batch_size, self.column)).astype(self.dtype) + for num in range(self.var_num) + ] self.inputs = {'X': list(zip(self.var_names, self.vars))} self.attrs = {'start_index': self.start_index, 'length': self.length} y = np_partial_concat(self.vars[:], self.start_index, self.length) @@ -70,7 +71,6 @@ class TestPartialConcatOp(OpTest): class TestPartialConcatOp2(TestPartialConcatOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) @@ -80,7 +80,6 @@ class TestPartialConcatOp2(TestPartialConcatOp): class TestPartialConcatOp3(TestPartialConcatOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) @@ -90,7 +89,6 @@ class TestPartialConcatOp3(TestPartialConcatOp): class TestPartialConcatOp4(TestPartialConcatOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) diff --git a/python/paddle/fluid/tests/unittests/test_partial_sum_op.py b/python/paddle/fluid/tests/unittests/test_partial_sum_op.py index 5eb416cadc8..9a5304f7b42 100644 --- a/python/paddle/fluid/tests/unittests/test_partial_sum_op.py +++ b/python/paddle/fluid/tests/unittests/test_partial_sum_op.py @@ -19,7 +19,6 @@ import random class TestPartialSumOp(OpTest): - def setUp(self): self.op_type = "partial_sum" self.init_kernel_type() @@ -29,13 +28,15 @@ class TestPartialSumOp(OpTest): else: end_index = self.start_index + self.length self.var_names = ['x' + str(num) for num in range(self.var_num)] - self.vars = [np.random.random((self.batch_size, self.column)).astype(self.dtype)\ - for num in range(self.var_num) ] + self.vars = [ + np.random.random((self.batch_size, self.column)).astype(self.dtype) + for num in range(self.var_num) + ] self.inputs = {'X': list(zip(self.var_names, self.vars))} self.attrs = {'start_index': self.start_index, 'length': self.length} - y = self.vars[0][:, self.start_index:end_index] + y = self.vars[0][:, self.start_index : end_index] for i in range(1, self.var_num): - y = y + self.vars[i][:, self.start_index:end_index] + y = y + self.vars[i][:, self.start_index : end_index] self.outputs = {'Out': y} @@ -58,7 +59,6 @@ class TestPartialSumOp(OpTest): class TestPartialSumOp2(TestPartialSumOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) @@ -68,7 +68,6 @@ class TestPartialSumOp2(TestPartialSumOp): class TestPartialSumOp3(TestPartialSumOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) @@ -78,7 +77,6 @@ class TestPartialSumOp3(TestPartialSumOp): class TestPartialSumOp4(TestPartialSumOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) diff --git a/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py b/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py index e1abb2c2372..11e8aa2b7c3 100644 --- a/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py +++ b/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py @@ -18,13 +18,11 @@ from test_parallel_dygraph_dataparallel import TestMultipleGpus class TestPipelineParallel(TestMultipleGpus): - def test_pipeline_parallel(self): self.run_mnist_2gpu('hybrid_parallel_pp_alexnet.py') class TestModelParallelWithRecompute(TestMultipleGpus): - def test_model_parallel_with_recompute(self): self.run_mnist_2gpu("dygraph_recompute_hybrid.py") diff --git a/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py b/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py index 996cd826908..2b20ed0fc88 100644 --- a/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py +++ b/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py @@ -25,8 +25,14 @@ import paddle.fluid as fluid def pixel_shuffle_np(x, up_factor, data_format="NCHW"): if data_format == "NCHW": n, c, h, w = x.shape - new_shape = (n, c // (up_factor * up_factor), up_factor, up_factor, h, - w) + new_shape = ( + n, + c // (up_factor * up_factor), + up_factor, + up_factor, + h, + w, + ) # reshape to (num,output_channel,upscale_factor,upscale_factor,h,w) npresult = np.reshape(x, new_shape) # transpose to (num,output_channel,h,upscale_factor,w,upscale_factor) @@ -36,8 +42,14 @@ def pixel_shuffle_np(x, up_factor, data_format="NCHW"): return npresult else: n, h, w, c = x.shape - new_shape = (n, h, w, c // (up_factor * up_factor), up_factor, - up_factor) + new_shape = ( + n, + h, + w, + c // (up_factor * up_factor), + up_factor, + up_factor, + ) # reshape to (num,h,w,output_channel,upscale_factor,upscale_factor) npresult = np.reshape(x, new_shape) # transpose to (num,h,upscale_factor,w,upscale_factor,output_channel) @@ -48,7 +60,6 @@ def pixel_shuffle_np(x, up_factor, data_format="NCHW"): class TestPixelShuffleOp(OpTest): - def setUp(self): self.op_type = "pixel_shuffle" self.python_api = paddle.nn.functional.pixel_shuffle @@ -80,13 +91,11 @@ class TestPixelShuffleOp(OpTest): class TestChannelLast(TestPixelShuffleOp): - def init_data_format(self): self.format = "NHWC" class TestPixelShuffleAPI(unittest.TestCase): - def setUp(self): self.x_1_np = np.random.random([2, 9, 4, 4]).astype("float64") self.x_2_np = np.random.random([2, 4, 4, 9]).astype("float64") @@ -94,47 +103,53 @@ class TestPixelShuffleAPI(unittest.TestCase): self.out_2_np = pixel_shuffle_np(self.x_2_np, 3, "NHWC") def test_static_graph_functional(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 9, 4, 4], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 4, 4, 9], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64" + ) out_1 = F.pixel_shuffle(x_1, 3) out_2 = F.pixel_shuffle(x_2, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, self.out_1_np) assert np.allclose(res_2, self.out_2_np) # same test between layer and functional in this op. def test_static_graph_layer(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 9, 4, 4], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 4, 4, 9], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64" + ) # init instance ps_1 = paddle.nn.PixelShuffle(3) ps_2 = paddle.nn.PixelShuffle(3, "NHWC") @@ -144,15 +159,19 @@ class TestPixelShuffleAPI(unittest.TestCase): out_2_np = pixel_shuffle_np(self.x_2_np, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, out_1_np) assert np.allclose(res_2, out_2_np) @@ -170,23 +189,26 @@ class TestPixelShuffleAPI(unittest.TestCase): npresult = pixel_shuffle_np(x, up_factor, data_format) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) - pixel_shuffle = paddle.nn.PixelShuffle(up_factor, - data_format=data_format) + pixel_shuffle = paddle.nn.PixelShuffle( + up_factor, data_format=data_format + ) result = pixel_shuffle(paddle.to_tensor(x)) np.testing.assert_allclose(result.numpy(), npresult, rtol=1e-05) - result_functional = F.pixel_shuffle(paddle.to_tensor(x), 3, - data_format) - np.testing.assert_allclose(result_functional.numpy(), - npresult, - rtol=1e-05) + result_functional = F.pixel_shuffle( + paddle.to_tensor(x), 3, data_format + ) + np.testing.assert_allclose( + result_functional.numpy(), npresult, rtol=1e-05 + ) def test_dygraph1(self): self.run_dygraph(3, "NCHW") @@ -196,9 +218,7 @@ class TestPixelShuffleAPI(unittest.TestCase): class TestPixelShuffleError(unittest.TestCase): - def test_error_functional(self): - def error_upscale_factor(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 9, 4, 4]).astype("float64") @@ -214,7 +234,6 @@ class TestPixelShuffleError(unittest.TestCase): self.assertRaises(ValueError, error_data_format) def test_error_layer(self): - def error_upscale_factor_layer(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 9, 4, 4]).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py b/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py index 5ab545222d8..8f4de9314d3 100644 --- a/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py +++ b/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py @@ -27,23 +27,41 @@ def pixel_unshuffle_np(x, down_factor, data_format="NCHW"): if data_format == "NCHW": n, c, h, w = x.shape - new_shape = (n, c, h // down_factor, down_factor, w // down_factor, - down_factor) + new_shape = ( + n, + c, + h // down_factor, + down_factor, + w // down_factor, + down_factor, + ) npresult = np.reshape(x, new_shape) npresult = npresult.transpose(0, 1, 3, 5, 2, 4) oshape = [ - n, c * down_factor * down_factor, h // down_factor, w // down_factor + n, + c * down_factor * down_factor, + h // down_factor, + w // down_factor, ] npresult = np.reshape(npresult, oshape) return npresult else: n, h, w, c = x.shape - new_shape = (n, h // down_factor, down_factor, w // down_factor, - down_factor, c) + new_shape = ( + n, + h // down_factor, + down_factor, + w // down_factor, + down_factor, + c, + ) npresult = np.reshape(x, new_shape) npresult = npresult.transpose(0, 1, 3, 5, 2, 4) oshape = [ - n, h // down_factor, w // down_factor, c * down_factor * down_factor + n, + h // down_factor, + w // down_factor, + c * down_factor * down_factor, ] npresult = np.reshape(npresult, oshape) return npresult @@ -73,7 +91,7 @@ class TestPixelUnshuffleOp(OpTest): self.outputs = {"Out": npresult} self.attrs = { "downscale_factor": down_factor, - "data_format": self.format + "data_format": self.format, } def init_data_format(self): @@ -115,30 +133,35 @@ class TestPixelUnshuffleAPI(unittest.TestCase): def test_static_graph_functional(self): '''test_static_graph_functional''' - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 1, 12, 12], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 12, 12, 1], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 1, 12, 12], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 12, 12, 1], dtype="float64" + ) out_1 = F.pixel_unshuffle(x_1, 3) out_2 = F.pixel_unshuffle(x_2, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, self.out_1_np) assert np.allclose(res_2, self.out_2_np) @@ -147,17 +170,18 @@ class TestPixelUnshuffleAPI(unittest.TestCase): def test_static_graph_layer(self): '''test_static_graph_layer''' - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 1, 12, 12], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 12, 12, 1], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 1, 12, 12], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 12, 12, 1], dtype="float64" + ) # init instance ps_1 = paddle.nn.PixelUnshuffle(3) ps_2 = paddle.nn.PixelUnshuffle(3, "NHWC") @@ -167,15 +191,19 @@ class TestPixelUnshuffleAPI(unittest.TestCase): out_2_np = pixel_unshuffle_np(self.x_2_np, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, out_1_np) assert np.allclose(res_2, out_2_np) @@ -194,23 +222,26 @@ class TestPixelUnshuffleAPI(unittest.TestCase): npresult = pixel_unshuffle_np(x, down_factor, data_format) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) - pixel_unshuffle = paddle.nn.PixelUnshuffle(down_factor, - data_format=data_format) + pixel_unshuffle = paddle.nn.PixelUnshuffle( + down_factor, data_format=data_format + ) result = pixel_unshuffle(paddle.to_tensor(x)) np.testing.assert_allclose(result.numpy(), npresult, rtol=1e-05) - result_functional = F.pixel_unshuffle(paddle.to_tensor(x), 3, - data_format) - np.testing.assert_allclose(result_functional.numpy(), - npresult, - rtol=1e-05) + result_functional = F.pixel_unshuffle( + paddle.to_tensor(x), 3, data_format + ) + np.testing.assert_allclose( + result_functional.numpy(), npresult, rtol=1e-05 + ) pixel_unshuffle_str = 'downscale_factor={}'.format(down_factor) if data_format != 'NCHW': @@ -258,8 +289,9 @@ class TestPixelUnshuffleError(unittest.TestCase): def error_data_format(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 1, 12, 12]).astype("float64") - pixel_unshuffle = F.pixel_unshuffle(paddle.to_tensor(x), 3, - "WOW") + pixel_unshuffle = F.pixel_unshuffle( + paddle.to_tensor(x), 3, "WOW" + ) self.assertRaises(ValueError, error_data_format) diff --git a/python/paddle/fluid/tests/unittests/test_poisson_op.py b/python/paddle/fluid/tests/unittests/test_poisson_op.py index d6f878c9473..4e4a20c77cb 100644 --- a/python/paddle/fluid/tests/unittests/test_poisson_op.py +++ b/python/paddle/fluid/tests/unittests/test_poisson_op.py @@ -38,7 +38,6 @@ def output_hist(out, lam, a, b): class TestPoissonOp1(OpTest): - def setUp(self): self.op_type = "poisson" self.config() @@ -67,11 +66,11 @@ class TestPoissonOp1(OpTest): user_defined_grads=[np.zeros([2048, 1024], dtype=self.dtype)], user_defined_grad_outputs=[ np.random.rand(2048, 1024).astype(self.dtype) - ]) + ], + ) class TestPoissonOp2(TestPoissonOp1): - def config(self): self.lam = 5 self.a = 1 @@ -80,18 +79,20 @@ class TestPoissonOp2(TestPoissonOp1): class TestPoissonAPI(unittest.TestCase): - def test_static(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_np = np.random.rand(10, 10) x = paddle.static.data(name="x", shape=[10, 10], dtype='float64') y = paddle.poisson(x) exe = paddle.static.Executor() - y_np = exe.run(paddle.static.default_main_program(), - feed={"x": x_np}, - fetch_list=[y]) + y_np = exe.run( + paddle.static.default_main_program(), + feed={"x": x_np}, + fetch_list=[y], + ) self.assertTrue(np.min(y_np) >= 0) def test_dygraph(self): @@ -117,70 +118,250 @@ class TestPoissonAPI(unittest.TestCase): paddle.disable_static() paddle.set_device('gpu') paddle.seed(2021) - x = paddle.full([32, 3, 1024, 768], 10., dtype="float32") + x = paddle.full([32, 3, 1024, 768], 10.0, dtype="float32") y = paddle.poisson(x) y_np = y.numpy() expect = [ - 13., 13., 11., 8., 12., 6., 9., 15., 16., 6., 13., 12., 9., 15., - 17., 8., 11., 16., 11., 10. + 13.0, + 13.0, + 11.0, + 8.0, + 12.0, + 6.0, + 9.0, + 15.0, + 16.0, + 6.0, + 13.0, + 12.0, + 9.0, + 15.0, + 17.0, + 8.0, + 11.0, + 16.0, + 11.0, + 10.0, ] np.testing.assert_array_equal(y_np[0, 0, 0, 0:20], expect) expect = [ - 15., 7., 12., 8., 14., 10., 10., 11., 11., 11., 21., 6., 9., 13., - 13., 11., 6., 9., 12., 12. + 15.0, + 7.0, + 12.0, + 8.0, + 14.0, + 10.0, + 10.0, + 11.0, + 11.0, + 11.0, + 21.0, + 6.0, + 9.0, + 13.0, + 13.0, + 11.0, + 6.0, + 9.0, + 12.0, + 12.0, ] np.testing.assert_array_equal(y_np[8, 1, 300, 200:220], expect) expect = [ - 10., 15., 9., 6., 4., 13., 10., 10., 13., 12., 9., 7., 10., 14., 7., - 10., 8., 5., 10., 14. + 10.0, + 15.0, + 9.0, + 6.0, + 4.0, + 13.0, + 10.0, + 10.0, + 13.0, + 12.0, + 9.0, + 7.0, + 10.0, + 14.0, + 7.0, + 10.0, + 8.0, + 5.0, + 10.0, + 14.0, ] np.testing.assert_array_equal(y_np[16, 1, 600, 400:420], expect) expect = [ - 10., 9., 14., 12., 8., 9., 7., 8., 11., 10., 13., 8., 12., 9., 7., - 8., 11., 11., 12., 5. + 10.0, + 9.0, + 14.0, + 12.0, + 8.0, + 9.0, + 7.0, + 8.0, + 11.0, + 10.0, + 13.0, + 8.0, + 12.0, + 9.0, + 7.0, + 8.0, + 11.0, + 11.0, + 12.0, + 5.0, ] np.testing.assert_array_equal(y_np[24, 2, 900, 600:620], expect) expect = [ - 15., 5., 11., 13., 12., 12., 13., 16., 9., 9., 7., 9., 13., 11., - 15., 6., 11., 9., 10., 10. + 15.0, + 5.0, + 11.0, + 13.0, + 12.0, + 12.0, + 13.0, + 16.0, + 9.0, + 9.0, + 7.0, + 9.0, + 13.0, + 11.0, + 15.0, + 6.0, + 11.0, + 9.0, + 10.0, + 10.0, ] np.testing.assert_array_equal(y_np[31, 2, 1023, 748:768], expect) - x = paddle.full([16, 1024, 1024], 5., dtype="float32") + x = paddle.full([16, 1024, 1024], 5.0, dtype="float32") y = paddle.poisson(x) y_np = y.numpy() expect = [ - 4., 5., 2., 9., 8., 7., 4., 7., 4., 7., 6., 3., 10., 7., 5., 7., 2., - 5., 5., 6. + 4.0, + 5.0, + 2.0, + 9.0, + 8.0, + 7.0, + 4.0, + 7.0, + 4.0, + 7.0, + 6.0, + 3.0, + 10.0, + 7.0, + 5.0, + 7.0, + 2.0, + 5.0, + 5.0, + 6.0, ] np.testing.assert_array_equal(y_np[0, 0, 100:120], expect) expect = [ - 1., 4., 8., 11., 6., 5., 4., 4., 7., 4., 4., 7., 11., 6., 5., 3., - 4., 6., 3., 3. + 1.0, + 4.0, + 8.0, + 11.0, + 6.0, + 5.0, + 4.0, + 4.0, + 7.0, + 4.0, + 4.0, + 7.0, + 11.0, + 6.0, + 5.0, + 3.0, + 4.0, + 6.0, + 3.0, + 3.0, ] np.testing.assert_array_equal(y_np[4, 300, 300:320], expect) expect = [ - 7., 5., 4., 6., 8., 5., 6., 7., 7., 7., 3., 10., 5., 10., 4., 5., - 8., 7., 5., 7. + 7.0, + 5.0, + 4.0, + 6.0, + 8.0, + 5.0, + 6.0, + 7.0, + 7.0, + 7.0, + 3.0, + 10.0, + 5.0, + 10.0, + 4.0, + 5.0, + 8.0, + 7.0, + 5.0, + 7.0, ] np.testing.assert_array_equal(y_np[8, 600, 600:620], expect) expect = [ - 8., 6., 7., 4., 3., 0., 4., 6., 6., 4., 3., 10., 5., 1., 3., 8., 8., - 2., 1., 4. + 8.0, + 6.0, + 7.0, + 4.0, + 3.0, + 0.0, + 4.0, + 6.0, + 6.0, + 4.0, + 3.0, + 10.0, + 5.0, + 1.0, + 3.0, + 8.0, + 8.0, + 2.0, + 1.0, + 4.0, ] np.testing.assert_array_equal(y_np[12, 900, 900:920], expect) expect = [ - 2., 1., 14., 3., 6., 5., 2., 2., 6., 5., 7., 4., 8., 4., 8., 4., 5., - 7., 1., 7. + 2.0, + 1.0, + 14.0, + 3.0, + 6.0, + 5.0, + 2.0, + 2.0, + 6.0, + 5.0, + 7.0, + 4.0, + 8.0, + 4.0, + 8.0, + 4.0, + 5.0, + 7.0, + 1.0, + 7.0, ] np.testing.assert_array_equal(y_np[15, 1023, 1000:1020], expect) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py index 84f61f7bebc..b5deecc4700 100644 --- a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py +++ b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py @@ -24,23 +24,27 @@ def PolygonBoxRestore(input): geo_channels = shape[1] h = shape[2] w = shape[3] - h_indexes = np.array(list(range(h)) * w).reshape( - [w, h]).transpose()[np.newaxis, :] # [1, h, w] - w_indexes = np.array(list(range(w)) * h).reshape( - [h, w])[np.newaxis, :] # [1, h, w] - indexes = np.concatenate( - (w_indexes, h_indexes))[np.newaxis, :] # [1, 2, h, w] + h_indexes = ( + np.array(list(range(h)) * w).reshape([w, h]).transpose()[np.newaxis, :] + ) # [1, h, w] + w_indexes = np.array(list(range(w)) * h).reshape([h, w])[ + np.newaxis, : + ] # [1, h, w] + indexes = np.concatenate((w_indexes, h_indexes))[ + np.newaxis, : + ] # [1, 2, h, w] + indexes = indexes.repeat([geo_channels / 2], axis=0)[ + np.newaxis, : + ] # [1, geo_channels/2, 2, h, w] indexes = indexes.repeat( - [geo_channels / 2], - axis=0)[np.newaxis, :] # [1, geo_channels/2, 2, h, w] - indexes = indexes.repeat([batch_size], - axis=0) # [batch_size, geo_channels/2, 2, h, w] - return indexes.reshape( - input.shape) * 4 - input # [batch_size, geo_channels, h, w] + [batch_size], axis=0 + ) # [batch_size, geo_channels/2, 2, h, w] + return ( + indexes.reshape(input.shape) * 4 - input + ) # [batch_size, geo_channels, h, w] class TestPolygonBoxRestoreOp(OpTest): - def config(self): self.input_shape = (1, 8, 2, 2) @@ -57,25 +61,21 @@ class TestPolygonBoxRestoreOp(OpTest): class TestCase1(TestPolygonBoxRestoreOp): - def config(self): self.input_shape = (2, 10, 3, 2) class TestCase2(TestPolygonBoxRestoreOp): - def config(self): self.input_shape = (3, 12, 4, 5) class TestPolygonBoxInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): - input = fluid.data(name='input', - shape=[None, 3, 32, 32], - dtype='int64') + input = fluid.data( + name='input', shape=[None, 3, 32, 32], dtype='int64' + ) out = fluid.layers.polygon_box_transform(input) self.assertRaises(TypeError, test_invalid_input) diff --git a/python/paddle/fluid/tests/unittests/test_pool1d_api.py b/python/paddle/fluid/tests/unittests/test_pool1d_api.py index 19d20817342..a3aa60e0ebd 100644 --- a/python/paddle/fluid/tests/unittests/test_pool1d_api.py +++ b/python/paddle/fluid/tests/unittests/test_pool1d_api.py @@ -29,24 +29,28 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def max_pool1D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=False, - adaptive=False, - data_type=np.float64): +def max_pool1D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=False, + adaptive=False, + data_type=np.float64, +): N, C, L = x.shape if global_pool == 1: ksize = [L] if adaptive: L_out = ksize[0] else: - L_out = (L - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + L_out = ( + (L - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) out = np.zeros((N, C, L_out)) for i in range(L_out): @@ -62,24 +66,28 @@ def max_pool1D_forward_naive(x, return out -def avg_pool1D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=False, - adaptive=False, - data_type=np.float64): +def avg_pool1D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=False, + adaptive=False, + data_type=np.float64, +): N, C, L = x.shape if global_pool == 1: ksize = [L] if adaptive: L_out = ksize[0] else: - L_out = (L - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + L_out = ( + (L - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) out = np.zeros((N, C, L_out)) for i in range(L_out): @@ -91,19 +99,21 @@ def avg_pool1D_forward_naive(x, r_end = np.min((i * strides[0] + ksize[0] - paddings[0], L)) x_masked = x[:, :, r_start:r_end] - field_size = (r_end - r_start) \ - if (exclusive or adaptive) else (ksize[0]) + field_size = ( + (r_end - r_start) if (exclusive or adaptive) else (ksize[0]) + ) if data_type == np.int8 or data_type == np.uint8: - out[:, :, i] = (np.rint(np.sum(x_masked, axis=(2, 3)) / - field_size)).astype(data_type) + out[:, :, i] = ( + np.rint(np.sum(x_masked, axis=(2, 3)) / field_size) + ).astype(data_type) else: - out[:, :, - i] = (np.sum(x_masked, axis=(2)) / field_size).astype(data_type) + out[:, :, i] = (np.sum(x_masked, axis=(2)) / field_size).astype( + data_type + ) return out class TestPool1D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -116,16 +126,16 @@ class TestPool1D_API(unittest.TestCase): result = F.avg_pool1d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32]).astype("float32") - result_np = avg_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0], - ceil_mode=False) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0], ceil_mode=False + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_avg_dygraph_results(self, place): @@ -134,16 +144,15 @@ class TestPool1D_API(unittest.TestCase): input = fluid.dygraph.to_variable(input_np) result = F.avg_pool1d(input, kernel_size=2, stride=2, padding=[0]) - result_np = avg_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool1d_dg = paddle.nn.layer.AvgPool1D(kernel_size=2, - stride=None, - padding=0) + avg_pool1d_dg = paddle.nn.layer.AvgPool1D( + kernel_size=2, stride=None, padding=0 + ) result = avg_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -151,24 +160,19 @@ class TestPool1D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = F.avg_pool1d(input, - kernel_size=2, - stride=2, - padding=[1], - exclusive=True) - - result_np = avg_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[1], - exclusive=False) + result = F.avg_pool1d( + input, kernel_size=2, stride=2, padding=[1], exclusive=True + ) + + result_np = avg_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[1], exclusive=False + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool1d_dg = paddle.nn.AvgPool1D(kernel_size=2, - stride=None, - padding=1, - exclusive=True) + avg_pool1d_dg = paddle.nn.AvgPool1D( + kernel_size=2, stride=None, padding=1, exclusive=True + ) result = avg_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -179,15 +183,16 @@ class TestPool1D_API(unittest.TestCase): result = F.max_pool1d(input, kernel_size=2, stride=2, padding=[0]) input_np = np.random.random([2, 3, 32]).astype("float32") - result_np = max_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = max_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_max_dygraph_results(self, place): @@ -196,16 +201,15 @@ class TestPool1D_API(unittest.TestCase): input = fluid.dygraph.to_variable(input_np) result = F.max_pool1d(input, kernel_size=2, stride=2, padding=0) - result_np = max_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = max_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool1d_dg = paddle.nn.layer.MaxPool1D(kernel_size=2, - stride=None, - padding=0) + max_pool1d_dg = paddle.nn.layer.MaxPool1D( + kernel_size=2, stride=None, padding=0 + ) result = max_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -213,22 +217,19 @@ class TestPool1D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result, index = F.max_pool1d(input, - kernel_size=2, - stride=2, - padding=0, - return_mask=True) + result, index = F.max_pool1d( + input, kernel_size=2, stride=2, padding=0, return_mask=True + ) - result_np = max_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = max_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool1d_dg = paddle.nn.layer.MaxPool1D(kernel_size=2, - stride=None, - padding=0) + max_pool1d_dg = paddle.nn.layer.MaxPool1D( + kernel_size=2, stride=None, padding=0 + ) result = max_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -236,15 +237,13 @@ class TestPool1D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = F.max_pool1d(input, - kernel_size=2, - stride=2, - padding="SAME") + result = F.max_pool1d( + input, kernel_size=2, stride=2, padding="SAME" + ) - result_np = max_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = max_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -252,15 +251,13 @@ class TestPool1D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = F.avg_pool1d(input, - kernel_size=2, - stride=2, - padding="SAME") + result = F.avg_pool1d( + input, kernel_size=2, stride=2, padding="SAME" + ) - result_np = avg_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -281,129 +278,145 @@ class TestPool1D_API(unittest.TestCase): class TestPool2DError_API(unittest.TestCase): - def test_error_api(self): - def run1(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[2]] - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = F.max_pool1d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run1) def run2(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[2]] - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = F.max_pool1d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run2) def run3(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "padding" - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = F.max_pool1d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run3) def run4(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.max_pool1d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run4) def run5(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.max_pool1d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run5) def run6(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = F.avg_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.avg_pool1d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run6) def run7(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "paddle" - res_pd = F.avg_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.avg_pool1d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run7) def run_kernel_out_of_range(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = 0 - res_pd = F.avg_pool1d(input_pd, - kernel_size=-1, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.avg_pool1d( + input_pd, + kernel_size=-1, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run_kernel_out_of_range) def run_stride_out_of_range(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = 0 - res_pd = F.avg_pool1d(input_pd, - kernel_size=2, - stride=0, - padding=padding, - ceil_mode=True) + res_pd = F.avg_pool1d( + input_pd, + kernel_size=2, + stride=0, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run_stride_out_of_range) diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_api.py b/python/paddle/fluid/tests/unittests/test_pool2d_api.py index d20844d2558..7e100d4c905 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_api.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_api.py @@ -19,11 +19,14 @@ import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.framework import _test_eager_guard from paddle.nn.functional import avg_pool2d, max_pool2d -from test_pool2d_op import avg_pool2D_forward_naive, max_pool2D_forward_naive, pool2D_forward_naive +from test_pool2d_op import ( + avg_pool2D_forward_naive, + max_pool2D_forward_naive, + pool2D_forward_naive, +) class TestPool2D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -32,22 +35,26 @@ class TestPool2D_API(unittest.TestCase): def check_avg_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 32, 32], - dtype="float32") + input = fluid.data( + name="input", shape=[2, 3, 32, 32], dtype="float32" + ) result = avg_pool2d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32, 32]).astype("float32") - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='avg') + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='avg', + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_avg_dygraph_results(self, place): @@ -56,16 +63,18 @@ class TestPool2D_API(unittest.TestCase): input = fluid.dygraph.to_variable(input_np) result = avg_pool2d(input, kernel_size=2, stride=2, padding=0) - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='avg') + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='avg', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=0) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=0 + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -73,24 +82,23 @@ class TestPool2D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool2d(input, - kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) - - result_np = avg_pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[1, 1], - ceil_mode=False, - exclusive=False) + result = avg_pool2d( + input, kernel_size=2, stride=2, padding=1, ceil_mode=False + ) + + result_np = avg_pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[1, 1], + ceil_mode=False, + exclusive=False, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=1, ceil_mode=False + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -98,66 +106,69 @@ class TestPool2D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool2d(input, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) - - result_np = avg_pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - ceil_mode=True) + result = avg_pool2d( + input, kernel_size=2, stride=2, padding=0, ceil_mode=True + ) + + result_np = avg_pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + ceil_mode=True, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=0, ceil_mode=True + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) def check_max_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 32, 32], - dtype="float32") + input = fluid.data( + name="input", shape=[2, 3, 32, 32], dtype="float32" + ) result = max_pool2d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32, 32]).astype("float32") - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max') + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_max_dygraph_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=0, - return_mask=False) - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max') + result = max_pool2d( + input, kernel_size=2, stride=2, padding=0, return_mask=False + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=0) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -165,46 +176,51 @@ class TestPool2D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable( - np.transpose(input_np, [0, 2, 3, 1])) - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=0, - return_mask=False, - data_format="NHWC") - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max') - np.testing.assert_allclose(np.transpose(result.numpy(), - [0, 3, 1, 2]), - result_np, - rtol=1e-05) + np.transpose(input_np, [0, 2, 3, 1]) + ) + result = max_pool2d( + input, + kernel_size=2, + stride=2, + padding=0, + return_mask=False, + data_format="NHWC", + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + ) + np.testing.assert_allclose( + np.transpose(result.numpy(), [0, 3, 1, 2]), + result_np, + rtol=1e-05, + ) def check_max_dygraph_padding_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) - - result_np = max_pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[1, 1], - ceil_mode=False, - exclusive=False) + result = max_pool2d( + input, kernel_size=2, stride=2, padding=1, ceil_mode=False + ) + + result_np = max_pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[1, 1], + ceil_mode=False, + exclusive=False, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=1, ceil_mode=False + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -212,23 +228,22 @@ class TestPool2D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) - - result_np = max_pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - ceil_mode=True) + result = max_pool2d( + input, kernel_size=2, stride=2, padding=0, ceil_mode=True + ) + + result_np = max_pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + ceil_mode=True, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=0, ceil_mode=True + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -236,23 +251,27 @@ class TestPool2D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result, indices = max_pool2d(input, - kernel_size=2, - stride=None, - padding="SAME", - return_mask=True) - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max', - padding_algorithm="SAME") + result, indices = max_pool2d( + input, + kernel_size=2, + stride=None, + padding="SAME", + return_mask=True, + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + padding_algorithm="SAME", + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=0) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -260,22 +279,23 @@ class TestPool2D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool2d(input, - kernel_size=2, - stride=None, - padding="SAME") - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='avg', - padding_algorithm="SAME") + result = avg_pool2d( + input, kernel_size=2, stride=None, padding="SAME" + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='avg', + padding_algorithm="SAME", + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=0) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=0 + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -284,22 +304,26 @@ class TestPool2D_API(unittest.TestCase): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) padding = [[0, 0], [0, 0], [0, 0], [0, 0]] - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=padding, - return_mask=False) - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max') + result = max_pool2d( + input, + kernel_size=2, + stride=2, + padding=padding, + return_mask=False, + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=0) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -308,22 +332,26 @@ class TestPool2D_API(unittest.TestCase): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) padding = [[0, 0], [0, 0], [0, 0], [0, 0]] - result = avg_pool2d(input, - kernel_size=2, - stride=2, - padding=padding, - divisor_override=4) - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='avg') + result = avg_pool2d( + input, + kernel_size=2, + stride=2, + padding=padding, + divisor_override=4, + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='avg', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=0) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=0 + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -348,193 +376,227 @@ class TestPool2D_API(unittest.TestCase): class TestPool2DError_API(unittest.TestCase): - def test_error_api(self): - def run1(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0]] - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = max_pool2d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run1) def run2(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0]] - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NHWC') + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NHWC', + ) self.assertRaises(ValueError, run2) def run3(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "padding" - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NHWC') + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NHWC', + ) self.assertRaises(ValueError, run3) def run3_avg(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "padding" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NHWC', + ) self.assertRaises(ValueError, run3_avg) def run4(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True, - data_format='NHWC') + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + data_format='NHWC', + ) self.assertRaises(ValueError, run4) def run4_avg(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + data_format='NHWC', + ) self.assertRaises(ValueError, run4_avg) def run5(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "padding" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NHWC', + ) self.assertRaises(ValueError, run5) def run6(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + data_format='NHWC', + ) self.assertRaises(ValueError, run6) def run7(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=False, - data_format='NNNN') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=False, + data_format='NNNN', + ) self.assertRaises(ValueError, run7) def run8(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=False, - data_format='NNNN') + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=False, + data_format='NNNN', + ) self.assertRaises(ValueError, run8) def run9(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=False, - data_format='NHWC', - return_mask=True) + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=0, + ceil_mode=False, + data_format='NHWC', + return_mask=True, + ) self.assertRaises(ValueError, run9) def run_kernel_out_of_range(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool2d(input_pd, - kernel_size=[-1, 2], - stride=2, - padding=0, - ceil_mode=False, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=[-1, 2], + stride=2, + padding=0, + ceil_mode=False, + data_format='NHWC', + ) self.assertRaises(ValueError, run_kernel_out_of_range) def run_stride_out_of_range(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool2d(input_pd, - kernel_size=3, - stride=[0, 2], - padding=0, - ceil_mode=False, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=3, + stride=[0, 2], + padding=0, + ceil_mode=False, + data_format='NHWC', + ) self.assertRaises(ValueError, run_stride_out_of_range) diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_op.py b/python/paddle/fluid/tests/unittests/test_pool2d_op.py index 46a822a5138..7c44827262d 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_op.py @@ -29,15 +29,17 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def max_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_type=np.float64): +def max_pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_type=np.float64, +): if data_type == np.float64 and core.is_compiled_with_rocm(): data_type = np.float32 N, C, H, W = x.shape @@ -46,12 +48,16 @@ def max_pool2D_forward_naive(x, if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - - 1) // strides[1] + 1 if ceil_mode else ( - W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + H_out = ( + (H - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + 2 * paddings[1] + strides[1] - 1) // strides[1] + 1 + if ceil_mode + else (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + ) out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -71,15 +77,17 @@ def max_pool2D_forward_naive(x, return out -def avg_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_type=np.float64): +def avg_pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_type=np.float64, +): if data_type == np.float64 and core.is_compiled_with_rocm(): data_type = np.float32 N, C, H, W = x.shape @@ -88,12 +96,16 @@ def avg_pool2D_forward_naive(x, if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - - 1) // strides[1] + 1 if ceil_mode else ( - W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + H_out = ( + (H - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + 2 * paddings[1] + strides[1] - 1) // strides[1] + 1 + if ceil_mode + else (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + ) out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -115,39 +127,44 @@ def avg_pool2D_forward_naive(x, x_masked = x[:, :, r_start:r_end, c_start:c_end] - if (exclusive or adaptive): + if exclusive or adaptive: field_size = (r_end - r_start) * (c_end - c_start) if data_type == np.int8 or data_type == np.uint8: - out[:, :, i, - j] = (np.rint(np.sum(x_masked, axis=(2, 3)) / - field_size)).astype(data_type) + out[:, :, i, j] = ( + np.rint(np.sum(x_masked, axis=(2, 3)) / field_size) + ).astype(data_type) else: - out[:, :, i, j] = (np.sum(x_masked, axis=(2, 3)) / - field_size).astype(data_type) + out[:, :, i, j] = ( + np.sum(x_masked, axis=(2, 3)) / field_size + ).astype(data_type) return out -def pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCHW', - pool_type="max", - padding_algorithm="EXPLICIT"): +def pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCHW', + pool_type="max", + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -157,9 +174,10 @@ def pool2D_forward_naive(x, if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0] @@ -167,7 +185,8 @@ def pool2D_forward_naive(x, raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCHW": @@ -180,8 +199,11 @@ def pool2D_forward_naive(x, is_sys = True if len(paddings) == 2 else False N = x.shape[0] - C, H, W = [x.shape[1], x.shape[2], x.shape[3]] if data_format == 'NCHW' \ + C, H, W = ( + [x.shape[1], x.shape[2], x.shape[3]] + if data_format == 'NCHW' else [x.shape[3], x.shape[1], x.shape[2]] + ) if global_pool == 1: ksize = [H, W] @@ -195,13 +217,26 @@ def pool2D_forward_naive(x, if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 - W_out = (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 - - out = np.zeros((N, C, H_out, W_out)) if data_format=='NCHW' \ + H_out = ( + (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + ) + + out = ( + np.zeros((N, C, H_out, W_out)) + if data_format == 'NCHW' else np.zeros((N, H_out, W_out, C)) + ) for i in range(H_out): if adaptive: in_h_start = adaptive_start_index(i, H, ksize[0]) @@ -229,21 +264,22 @@ def pool2D_forward_naive(x, if data_format == 'NCHW': x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) - + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) -# if (exclusive or adaptive) else (ksize[0] * ksize[1]) + # if (exclusive or adaptive) else (ksize[0] * ksize[1]) out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size elif pool_type == 'max': out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) elif data_format == 'NHWC': x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size elif pool_type == 'max': out[:, i, j, :] = np.max(x_masked, axis=(1, 2)) @@ -251,7 +287,6 @@ def pool2D_forward_naive(x, class TestPool2D_Op_Mixin(object): - def setUp(self): self.op_type = "pool2d" self.use_cudnn = False @@ -271,12 +306,19 @@ class TestPool2D_Op_Mixin(object): self.init_shape() input = np.random.random(self.shape).astype(self.dtype) - output = pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, self.data_format, - self.pool_type, - self.padding_algorithm).astype(self.dtype) + output = pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} self.attrs = { @@ -304,7 +346,8 @@ class TestPool2D_Op_Mixin(object): if self.has_cudnn(): place = core.CUDAPlace(0) self.check_output_with_place( - place, atol=1e-5, check_dygraph=(self.use_mkldnn == False)) + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) else: self.check_output(check_dygraph=(self.use_mkldnn == False)) @@ -314,16 +357,20 @@ class TestPool2D_Op_Mixin(object): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.has_cudnn() and self.pool_type != "max": place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['X']), - 'Out', - max_relative_error=0.07, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + set(['X']), + 'Out', + max_relative_error=0.07, + check_dygraph=(self.use_mkldnn == False), + ) elif self.pool_type != "max": - self.check_grad(set(['X']), - 'Out', - max_relative_error=0.07, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + set(['X']), + 'Out', + max_relative_error=0.07, + check_dygraph=(self.use_mkldnn == False), + ) def init_data_format(self): self.data_format = "NCHW" @@ -367,7 +414,6 @@ class TestPool2D_Op(TestPool2D_Op_Mixin, OpTest): class TestCase1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -387,7 +433,6 @@ class TestCase1(TestPool2D_Op): class TestCase2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -407,35 +452,31 @@ class TestCase2(TestPool2D_Op): class TestCase3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive -#--------------------test pool2d cudnn-------------------- +# --------------------test pool2d cudnn-------------------- def create_test_cudnn_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -451,15 +492,14 @@ create_test_cudnn_class(TestCase3) create_test_cudnn_class(TestCase4) create_test_cudnn_class(TestCase5) -#--------------------test pool2d cudnn_fp16-------------------- +# --------------------test pool2d cudnn_fp16-------------------- def create_test_cudnn_fp16_class(parent, check_grad=True): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -472,19 +512,24 @@ def create_test_cudnn_fp16_class(parent, check_grad=True): self.check_output_with_place( place, atol=1e-3, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode place = core.CUDAPlace(0) - if core.is_float16_supported( - place) and self.pool_type != "max" and check_grad: + if ( + core.is_float16_supported(place) + and self.pool_type != "max" + and check_grad + ): self.check_grad_with_place( place, set(['X']), 'Out', max_relative_error=0.07, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNNFp16Op") TestCUDNNFp16Case.__name__ = cls_name @@ -492,11 +537,10 @@ def create_test_cudnn_fp16_class(parent, check_grad=True): def create_test_fp16_class(parent, check_grad=True): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = False self.dtype = np.float16 @@ -509,19 +553,24 @@ def create_test_fp16_class(parent, check_grad=True): self.check_output_with_place( place, atol=1e-3, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode place = core.CUDAPlace(0) - if core.is_float16_supported( - place) and self.pool_type != "max" and check_grad: + if ( + core.is_float16_supported(place) + and self.pool_type != "max" + and check_grad + ): self.check_grad_with_place( place, set(['X']), 'Out', max_relative_error=0.07, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16Op") TestFp16Case.__name__ = cls_name @@ -542,15 +591,14 @@ create_test_fp16_class(TestCase3) create_test_fp16_class(TestCase4) create_test_fp16_class(TestCase5) -#--------------------test pool2d use ceil mode-------------------- +# --------------------test pool2d use ceil mode-------------------- def create_test_cudnn_use_ceil_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestPool2DUseCeilCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -567,9 +615,7 @@ create_test_cudnn_use_ceil_class(TestCase1) def create_test_use_ceil_class(parent): - class TestPool2DUseCeilCase(parent): - def init_ceil_mode(self): self.ceil_mode = True @@ -583,13 +629,11 @@ create_test_use_ceil_class(TestCase2) class TestAvgInclude(TestCase2): - def init_exclusive(self): self.exclusive = False class TestCUDNNAvgInclude(TestCase2): - def init_kernel_type(self): self.use_cudnn = True @@ -598,13 +642,11 @@ class TestCUDNNAvgInclude(TestCase2): class TestAvgPoolAdaptive(TestCase1): - def init_adaptive(self): self.adaptive = True class TestAvgPoolAdaptiveAsyOutSize(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -617,11 +659,10 @@ class TestAvgPoolAdaptiveAsyOutSize(TestCase1): self.paddings = [0, 0, 0, 0] -#-------test pool2d with asymmetric padding----- +# -------test pool2d with asymmetric padding----- class TestPool2D_AsyPadding(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -632,7 +673,6 @@ class TestPool2D_AsyPadding(TestPool2D_Op): class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -643,7 +683,6 @@ class TestCase1_AsyPadding(TestCase1): class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -654,7 +693,6 @@ class TestCase2_AsyPadding(TestCase2): class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -665,7 +703,6 @@ class TestCase3_AsyPadding(TestCase3): class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -676,7 +713,6 @@ class TestCase4_AsyPadding(TestCase4): class TestCase5_AsyPadding((TestCase5)): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -708,7 +744,6 @@ create_test_use_ceil_class(TestCase2_AsyPadding) class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -722,7 +757,6 @@ class TestAvgInclude_AsyPadding(TestCase2): class TestCUDNNAvgInclude_AsyPadding(TestCase2): - def init_kernel_type(self): self.use_cudnn = True @@ -739,7 +773,6 @@ class TestCUDNNAvgInclude_AsyPadding(TestCase2): class TestAvgPoolAdaptive_AsyPadding(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -752,9 +785,8 @@ class TestAvgPoolAdaptive_AsyPadding(TestCase1): self.shape = [2, 3, 7, 7] -#----------- test channel_last -------------- +# ----------- test channel_last -------------- class TestPool2D_channel_last(TestPool2D_Op): - def init_data_format(self): self.data_format = "NHWC" @@ -763,7 +795,6 @@ class TestPool2D_channel_last(TestPool2D_Op): class TestCase1_channel_last(TestCase1): - def init_data_format(self): self.data_format = "NHWC" @@ -772,7 +803,6 @@ class TestCase1_channel_last(TestCase1): class TestCase2_channel_last(TestCase2): - def init_data_format(self): self.data_format = "NHWC" @@ -781,7 +811,6 @@ class TestCase2_channel_last(TestCase2): class TestCase3_channel_last(TestCase3): - def init_data_format(self): self.data_format = "NHWC" @@ -790,7 +819,6 @@ class TestCase3_channel_last(TestCase3): class TestCase4_channel_last(TestCase4): - def init_data_format(self): self.data_format = "NHWC" @@ -799,7 +827,6 @@ class TestCase4_channel_last(TestCase4): class TestCase5_channel_last(TestCase5): - def init_data_format(self): self.data_format = "NHWC" @@ -829,7 +856,6 @@ create_test_use_ceil_class(TestCase2_channel_last) class TestCase5_Max(TestCase2): - def init_pool_type(self): self.pool_type = "max" @@ -838,16 +864,14 @@ class TestCase5_Max(TestCase2): return if self.has_cudnn() and self.pool_type == "max": place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['X']), - 'Out', - max_relative_error=1.00) + self.check_grad_with_place( + place, set(['X']), 'Out', max_relative_error=1.00 + ) elif self.pool_type == "max": self.check_grad(set(['X']), 'Out', max_relative_error=1.00) class TestCase5_channel_last_Max(TestCase5_Max): - def init_data_format(self): self.data_format = "NHWC" @@ -860,13 +884,11 @@ create_test_cudnn_class(TestCase5_channel_last_Max) class TestAvgInclude_channel_last(TestCase2_channel_last): - def init_exclusive(self): self.exclusive = False class TestCUDNNAvgInclude_channel_last(TestCase2_channel_last): - def init_kernel_type(self): self.use_cudnn = True @@ -875,13 +897,11 @@ class TestCUDNNAvgInclude_channel_last(TestCase2_channel_last): class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): - def init_adaptive(self): self.adaptive = True class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -890,7 +910,6 @@ class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -899,7 +918,6 @@ class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -908,7 +926,6 @@ class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -917,7 +934,6 @@ class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -926,7 +942,6 @@ class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -942,8 +957,9 @@ create_test_cudnn_class(TestCase4_AsyPadding_channel_last) create_test_cudnn_class(TestCase5_AsyPadding_channel_last) create_test_cudnn_fp16_class(TestPool2D_AsyPadding_channel_last) -create_test_cudnn_fp16_class(TestCase1_AsyPadding_channel_last, - check_grad=False) +create_test_cudnn_fp16_class( + TestCase1_AsyPadding_channel_last, check_grad=False +) create_test_cudnn_fp16_class(TestCase2_AsyPadding_channel_last) create_test_cudnn_fp16_class(TestCase3_AsyPadding_channel_last) create_test_cudnn_fp16_class(TestCase4_AsyPadding_channel_last) @@ -957,7 +973,6 @@ create_test_use_ceil_class(TestCase2_AsyPadding_channel_last) class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -965,9 +980,9 @@ class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): self.shape = [2, 7, 7, 3] -class TestCUDNNAvgInclude_AsyPadding_channel_last(TestCUDNNAvgInclude_AsyPadding - ): - +class TestCUDNNAvgInclude_AsyPadding_channel_last( + TestCUDNNAvgInclude_AsyPadding +): def init_data_format(self): self.data_format = "NHWC" @@ -975,9 +990,9 @@ class TestCUDNNAvgInclude_AsyPadding_channel_last(TestCUDNNAvgInclude_AsyPadding self.shape = [2, 7, 7, 3] -class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding - ): - +class TestAvgPoolAdaptive_AsyPadding_channel_last( + TestAvgPoolAdaptive_AsyPadding +): def init_data_format(self): self.data_format = "NHWC" @@ -989,9 +1004,7 @@ class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.paddings = [0, 0] self.padding_algorithm = "SAME" @@ -1017,11 +1030,10 @@ create_test_padding_SAME_class(TestCase5_channel_last) def create_test_cudnn_padding_SAME_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingSMAECase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -1050,9 +1062,7 @@ create_test_cudnn_padding_SAME_class(TestCase5_channel_last) def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.paddings = [1, 1] self.padding_algorithm = "VALID" @@ -1078,11 +1088,10 @@ create_test_padding_VALID_class(TestCase5_channel_last) def create_test_cudnn_padding_VALID_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingVALIDCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -1111,7 +1120,6 @@ create_test_cudnn_padding_VALID_class(TestCase5_channel_last) class TestCase1_strides(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 2] @@ -1127,106 +1135,131 @@ create_test_cudnn_padding_SAME_class(TestCase1_strides) # ----- test API class TestPool2DAPI(unittest.TestCase): - def test_api(self): x_NHWC = np.random.random([2, 5, 5, 3]).astype("float32") x_NCHW = np.random.random([2, 3, 5, 5]).astype("float32") - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") - - input_NHWC_negetive = fluid.layers.data(name="input_NHWC_negetive", - shape=[2, -1, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW_negetive = fluid.layers.data(name="input_NCHW_negetive", - shape=[2, 3, -1, -1], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) + + input_NHWC_negetive = fluid.layers.data( + name="input_NHWC_negetive", + shape=[2, -1, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW_negetive = fluid.layers.data( + name="input_NCHW_negetive", + shape=[2, 3, -1, -1], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3] - out_1 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - use_cudnn=False, - data_format="NHWC") - - out_2 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - use_cudnn=False, - data_format="NHWC") - - out_3 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [0, 0], [1, 1], - [1, 1]], - use_cudnn=False, - data_format="NCHW") - - out_4 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[1, 2, 1, 0], - use_cudnn=False, - data_format="NCHW") + out_1 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + use_cudnn=False, + data_format="NHWC", + ) + + out_2 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + use_cudnn=False, + data_format="NHWC", + ) + + out_3 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [0, 0], [1, 1], [1, 1]], + use_cudnn=False, + data_format="NCHW", + ) + + out_4 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[1, 2, 1, 0], + use_cudnn=False, + data_format="NCHW", + ) # test VALID - out_5 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding="VALID", - use_cudnn=False, - data_format="NCHW") - - out_6 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - use_cudnn=False, - data_format="NHWC") + out_5 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding="VALID", + use_cudnn=False, + data_format="NCHW", + ) + + out_6 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + use_cudnn=False, + data_format="NHWC", + ) # test SAME - out_7 = fluid.layers.pool2d(input=input_NCHW, - pool_size=[4, 4], - pool_type="avg", - pool_padding="SAME", - use_cudnn=False, - data_format="NCHW") - - out_8 = fluid.layers.pool2d(input=input_NHWC, - pool_size=[4, 4], - pool_type="max", - pool_padding="SAME", - use_cudnn=False, - data_format="NHWC") + out_7 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=[4, 4], + pool_type="avg", + pool_padding="SAME", + use_cudnn=False, + data_format="NCHW", + ) + + out_8 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=[4, 4], + pool_type="max", + pool_padding="SAME", + use_cudnn=False, + data_format="NHWC", + ) # test negetive - out_9 = fluid.layers.pool2d(input=input_NHWC_negetive, - pool_size=ksize, - pool_type="avg", - pool_padding=[0, 0], - use_cudnn=False, - data_format="NHWC") + out_9 = fluid.layers.pool2d( + input=input_NHWC_negetive, + pool_size=ksize, + pool_type="avg", + pool_padding=[0, 0], + use_cudnn=False, + data_format="NHWC", + ) assert out_9.shape == (2, -1, 3, 3) - out_10 = fluid.layers.pool2d(input=input_NCHW_negetive, - pool_size=ksize, - pool_type="avg", - pool_padding=[0, 0], - use_cudnn=False, - data_format="NCHW") + out_10 = fluid.layers.pool2d( + input=input_NCHW_negetive, + pool_size=ksize, + pool_type="avg", + pool_padding=[0, 0], + use_cudnn=False, + data_format="NCHW", + ) assert out_10.shape == (2, 3, -1, -1) exe = fluid.Executor(place=fluid.CPUPlace()) @@ -1236,46 +1269,61 @@ class TestPool2DAPI(unittest.TestCase): "input_NHWC": x_NHWC, "input_NCHW": x_NCHW, "input_NHWC_negetive": x_NHWC, - "input_NCHW_negetive": x_NCHW + "input_NCHW_negetive": x_NCHW, }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.allclose( res_1, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1], - paddings=[1, 1], - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1], + paddings=[1, 1], + data_format="NHWC", + ), + ) assert np.allclose( res_2, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 1, 1, 1], - data_format="NHWC")) - assert np.allclose(res_3, - pool2D_forward_naive(x=x_NCHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 1, 1, 1], - data_format="NCHW"), - rtol=0.07, - atol=1e-05) - - assert np.allclose(res_4, - pool2D_forward_naive(x=x_NCHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 2, 1, 0], - data_format="NCHW"), - rtol=0.07, - atol=1e-05) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 1, 1, 1], + data_format="NHWC", + ), + ) + assert np.allclose( + res_3, + pool2D_forward_naive( + x=x_NCHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 1, 1, 1], + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) + + assert np.allclose( + res_4, + pool2D_forward_naive( + x=x_NCHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 2, 1, 0], + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) # VALID assert np.allclose( @@ -1287,192 +1335,234 @@ class TestPool2DAPI(unittest.TestCase): strides=[1, 1], paddings=[10, 20], # any ele is ok padding_algorithm="VALID", - data_format="NCHW"), + data_format="NCHW", + ), rtol=0.07, - atol=1e-05) + atol=1e-05, + ) assert np.allclose( res_6, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="VALID", - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="VALID", + data_format="NHWC", + ), + ) # SAME - assert np.allclose(res_7, - pool2D_forward_naive(x=x_NCHW, - ksize=[4, 4], - pool_type="avg", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NCHW"), - rtol=0.07, - atol=1e-05) + assert np.allclose( + res_7, + pool2D_forward_naive( + x=x_NCHW, + ksize=[4, 4], + pool_type="avg", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) assert np.allclose( res_8, - pool2D_forward_naive(x=x_NHWC, - ksize=[4, 4], - pool_type="max", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=[4, 4], + pool_type="max", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NHWC", + ), + ) class TestPool2DAPI_Error(unittest.TestCase): - def test_api(self): - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3] # cudnn type error def run_1(): - out_1 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - use_cudnn=[0], - data_format="NHWC") + out_1 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + use_cudnn=[0], + data_format="NHWC", + ) self.assertRaises(TypeError, run_1) # data_format value error def run_2(): - out_2 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - use_cudnn=False, - data_format="NHWCC") + out_2 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + use_cudnn=False, + data_format="NHWCC", + ) self.assertRaises(ValueError, run_2) # padding str value error def run_3(): - out_3 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALIDSAME", - use_cudnn=False, - data_format="NHWC") + out_3 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALIDSAME", + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_3) # padding str valid and ceil_mode value error def run_4(): - out_4 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - use_cudnn=False, - ceil_mode=True, - data_format="NHWC") + out_4 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + use_cudnn=False, + ceil_mode=True, + data_format="NHWC", + ) self.assertRaises(ValueError, run_4) # padding with 8 ele. value error def run_5(): - out_5 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[[1, 1], [0, 0], [0, 0], - [1, 1]], - use_cudnn=False, - data_format="NHWC") + out_5 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[[1, 1], [0, 0], [0, 0], [1, 1]], + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_5) class TestDygraphPool2DAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of Pool2D must be Variable. data1 = np.random.random((3, 32, 32, 5)).astype('float32') - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - global_pooling=False) + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False, + ) self.assertRaises(TypeError, pool2d, data1) # the input dtype of Pool2D must be uint8 or int8 or float16 or float32 or float64 # uint8 and int8 only can be set on mkldnn # float16 only can be set on GPU place - data2 = fluid.layers.data(name='x1', - shape=[3, 32, 32, 5], - dtype="int32") + data2 = fluid.layers.data( + name='x1', shape=[3, 32, 32, 5], dtype="int32" + ) self.assertRaises(TypeError, pool2d, data2) def test_data_format_error(self): with program_guard(Program(), Program()): # the data_format must be 'NCHW' or 'NHWC' data1 = np.random.random((3, 32, 32, 5)).astype('float32') - self.assertRaises(ValueError, - fluid.dygraph.Pool2D, - pool_size=2, - pool_type='max', - pool_stride=1, - global_pooling=False, - data_format='NWHC') + self.assertRaises( + ValueError, + fluid.dygraph.Pool2D, + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False, + data_format='NWHC', + ) class TestDygraphPool2DAPI(unittest.TestCase): - def test_nhwc(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='NHWC') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='NHWC', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2, rtol=1e-05) def test_lower_case(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='nhwc') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='nhwc', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2, rtol=1e-05) def test_upper_case(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='MAX', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='nhwc') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='MAX', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='nhwc', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_api.py b/python/paddle/fluid/tests/unittests/test_pool3d_api.py index 31426fe541d..29882a6c8bc 100644 --- a/python/paddle/fluid/tests/unittests/test_pool3d_api.py +++ b/python/paddle/fluid/tests/unittests/test_pool3d_api.py @@ -20,11 +20,14 @@ import paddle.fluid.core as core from paddle.fluid.framework import _test_eager_guard from paddle.nn.functional import avg_pool3d, max_pool3d from paddle.fluid.framework import _test_eager_guard -from test_pool3d_op import avg_pool3D_forward_naive, max_pool3D_forward_naive, pool3D_forward_naive +from test_pool3d_op import ( + avg_pool3D_forward_naive, + max_pool3D_forward_naive, + pool3D_forward_naive, +) class TestPool3D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -33,22 +36,26 @@ class TestPool3D_API(unittest.TestCase): def check_avg_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 32, 32, 32], - dtype="float32") + input = fluid.data( + name="input", shape=[2, 3, 32, 32, 32], dtype="float32" + ) result = avg_pool3d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='avg') + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='avg', + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_avg_dygraph_results(self, place): @@ -57,18 +64,20 @@ class TestPool3D_API(unittest.TestCase): input = fluid.dygraph.to_variable(input_np) result = avg_pool3d(input, kernel_size=2, stride=2, padding="SAME") - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='avg', - padding_algorithm="SAME") + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='avg', + padding_algorithm="SAME", + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool3d_dg = paddle.nn.layer.AvgPool3D(kernel_size=2, - stride=None, - padding="SAME") + avg_pool3d_dg = paddle.nn.layer.AvgPool3D( + kernel_size=2, stride=None, padding="SAME" + ) result = avg_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -76,27 +85,33 @@ class TestPool3D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool3d(input, - kernel_size=2, - stride=2, - padding=1, - ceil_mode=False, - exclusive=True) - - result_np = avg_pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[1, 1, 1], - ceil_mode=False, - exclusive=False) + result = avg_pool3d( + input, + kernel_size=2, + stride=2, + padding=1, + ceil_mode=False, + exclusive=True, + ) + + result_np = avg_pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[1, 1, 1], + ceil_mode=False, + exclusive=False, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool3d_dg = paddle.nn.layer.AvgPool3D(kernel_size=2, - stride=None, - padding=1, - ceil_mode=False, - exclusive=True) + avg_pool3d_dg = paddle.nn.layer.AvgPool3D( + kernel_size=2, + stride=None, + padding=1, + ceil_mode=False, + exclusive=True, + ) result = avg_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -104,45 +119,48 @@ class TestPool3D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool3d(input, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) - - result_np = avg_pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - ceil_mode=True) + result = avg_pool3d( + input, kernel_size=2, stride=2, padding=0, ceil_mode=True + ) + + result_np = avg_pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + ceil_mode=True, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool3d_dg = paddle.nn.layer.AvgPool3D(kernel_size=2, - stride=None, - padding=0, - ceil_mode=True) + avg_pool3d_dg = paddle.nn.layer.AvgPool3D( + kernel_size=2, stride=None, padding=0, ceil_mode=True + ) result = avg_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) def check_max_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 32, 32, 32], - dtype="float32") + input = fluid.data( + name="input", shape=[2, 3, 32, 32, 32], dtype="float32" + ) result = max_pool3d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max') + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_max_dygraph_results(self, place): @@ -151,16 +169,18 @@ class TestPool3D_API(unittest.TestCase): input = fluid.dygraph.to_variable(input_np) result = max_pool3d(input, kernel_size=2, stride=2, padding=0) - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max') + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=None, - padding=0) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=None, padding=0 + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -168,47 +188,52 @@ class TestPool3D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable( - np.transpose(input_np, [0, 2, 3, 4, 1])) - result = max_pool3d(input, - kernel_size=2, - stride=2, - padding=0, - data_format="NDHWC", - return_mask=False) - - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max') - - np.testing.assert_allclose(np.transpose(result.numpy(), - [0, 4, 1, 2, 3]), - result_np, - rtol=1e-05) + np.transpose(input_np, [0, 2, 3, 4, 1]) + ) + result = max_pool3d( + input, + kernel_size=2, + stride=2, + padding=0, + data_format="NDHWC", + return_mask=False, + ) + + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + ) + + np.testing.assert_allclose( + np.transpose(result.numpy(), [0, 4, 1, 2, 3]), + result_np, + rtol=1e-05, + ) def check_max_dygraph_ceilmode_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool3d(input, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) - - result_np = max_pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - ceil_mode=True) + result = max_pool3d( + input, kernel_size=2, stride=2, padding=0, ceil_mode=True + ) + + result_np = max_pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + ceil_mode=True, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=None, - padding=0, - ceil_mode=True) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=None, padding=0, ceil_mode=True + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -216,24 +241,23 @@ class TestPool3D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool3d(input, - kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) - - result_np = max_pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[1, 1, 1], - ceil_mode=False) + result = max_pool3d( + input, kernel_size=2, stride=2, padding=1, ceil_mode=False + ) + + result_np = max_pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[1, 1, 1], + ceil_mode=False, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=None, - padding=1, - ceil_mode=False) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=None, padding=1, ceil_mode=False + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -241,23 +265,27 @@ class TestPool3D_API(unittest.TestCase): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result, indices = max_pool3d(input, - kernel_size=2, - stride=None, - padding="SAME", - return_mask=True) - - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max', - padding_algorithm="SAME") + result, indices = max_pool3d( + input, + kernel_size=2, + stride=None, + padding="SAME", + return_mask=True, + ) + + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + padding_algorithm="SAME", + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=2, - padding=0) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -268,16 +296,18 @@ class TestPool3D_API(unittest.TestCase): padding = [[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]] result = max_pool3d(input, kernel_size=2, stride=2, padding=padding) - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max') + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=2, - padding=0) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -290,31 +320,37 @@ class TestPool3D_API(unittest.TestCase): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) padding = 0 - result = avg_pool3d(input, - kernel_size=2, - stride=2, - padding=padding, - divisor_override=8) - - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='avg') + result = avg_pool3d( + input, + kernel_size=2, + stride=2, + padding=padding, + divisor_override=8, + ) + + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='avg', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool3d_dg = paddle.nn.layer.AvgPool3D(kernel_size=2, - stride=2, - padding=0) + avg_pool3d_dg = paddle.nn.layer.AvgPool3D( + kernel_size=2, stride=2, padding=0 + ) result = avg_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) padding = [0, 0, 0, 0, 0, 0] - result = avg_pool3d(input, - kernel_size=2, - stride=2, - padding=padding, - divisor_override=8) + result = avg_pool3d( + input, + kernel_size=2, + stride=2, + padding=padding, + divisor_override=8, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) def test_pool3d(self): @@ -336,167 +372,198 @@ class TestPool3D_API(unittest.TestCase): class TestPool3DError_API(unittest.TestCase): - def test_error_api(self): - def run1(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0], [0, 0]] - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = avg_pool3d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run1) def run2(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0], [0, 0]] - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NCDHW') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NCDHW', + ) self.assertRaises(ValueError, run2) def run3(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0], [0, 0]] - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NDHWC') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NDHWC', + ) self.assertRaises(ValueError, run3) def run4(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=0, - data_format='NNNN') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=0, + data_format='NNNN', + ) self.assertRaises(ValueError, run4) def run5(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=0, - data_format='NNNN') + res_pd = max_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=0, + data_format='NNNN', + ) self.assertRaises(ValueError, run5) def run6(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding="padding", - data_format='NNNN') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding="padding", + data_format='NNNN', + ) self.assertRaises(ValueError, run6) def run7(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool3d(input_pd, - kernel_size=2, - stride=2, - padding="padding", - data_format='NNNN') + res_pd = max_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding="padding", + data_format='NNNN', + ) self.assertRaises(ValueError, run7) def run8(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding="VALID", - ceil_mode=True, - data_format='NNNN') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding="VALID", + ceil_mode=True, + data_format='NNNN', + ) self.assertRaises(ValueError, run8) def run9(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool3d(input_pd, - kernel_size=2, - stride=2, - padding="VALID", - ceil_mode=True, - data_format='NNNN') + res_pd = max_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding="VALID", + ceil_mode=True, + data_format='NNNN', + ) self.assertRaises(ValueError, run9) def run10(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=0, - data_format='NDHWC', - return_mask=True) + res_pd = max_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=0, + data_format='NDHWC', + return_mask=True, + ) self.assertRaises(ValueError, run10) def run_kernel_out_of_range(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=-1, - stride=2, - padding="VALID", - ceil_mode=True) + res_pd = avg_pool3d( + input_pd, + kernel_size=-1, + stride=2, + padding="VALID", + ceil_mode=True, + ) self.assertRaises(ValueError, run_kernel_out_of_range) def run_size_out_of_range(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=0, - padding="VALID", - ceil_mode=True) + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=0, + padding="VALID", + ceil_mode=True, + ) self.assertRaises(ValueError, run_size_out_of_range) diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_op.py b/python/paddle/fluid/tests/unittests/test_pool3d_op.py index 37fde90a457..a5bf5066956 100644 --- a/python/paddle/fluid/tests/unittests/test_pool3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool3d_op.py @@ -29,25 +29,29 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def pool3D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCDHW', - pool_type='max', - padding_algorithm="EXPLICIT"): +def pool3D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCDHW', + pool_type='max', + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -57,9 +61,10 @@ def pool3D_forward_naive(x, if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0, 0, 0] @@ -67,7 +72,8 @@ def pool3D_forward_naive(x, raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCDHW": @@ -80,8 +86,11 @@ def pool3D_forward_naive(x, is_sys = True if len(paddings) == 3 else False N = x.shape[0] - C,D, H, W = [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] \ - if data_format == 'NCDHW' else [x.shape[4], x.shape[1], x.shape[2],x.shape[3]] + C, D, H, W = ( + [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] + if data_format == 'NCDHW' + else [x.shape[4], x.shape[1], x.shape[2], x.shape[3]] + ) if global_pool == 1: ksize = [D, H, W] @@ -98,18 +107,35 @@ def pool3D_forward_naive(x, D_out, H_out, W_out = ksize else: - D_out = (D - ksize[0] + pad_d_forth+pad_d_back + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (D - ksize[0] + pad_d_forth+pad_d_back) // strides[0] + 1 - - H_out = (H - ksize[1] + pad_h_up + pad_h_down + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (H - ksize[1] + pad_h_up + pad_h_down) // strides[1] + 1 - - W_out = (W - ksize[2] + pad_w_left + pad_w_right + strides[2] - 1) // strides[2] + 1 \ - if ceil_mode else (W - ksize[2] + pad_w_left + pad_w_right) // strides[2] + 1 - - - out = np.zeros((N, C, D_out, H_out, W_out)) if data_format=='NCDHW' \ + D_out = ( + (D - ksize[0] + pad_d_forth + pad_d_back + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (D - ksize[0] + pad_d_forth + pad_d_back) // strides[0] + 1 + ) + + H_out = ( + (H - ksize[1] + pad_h_up + pad_h_down + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (H - ksize[1] + pad_h_up + pad_h_down) // strides[1] + 1 + ) + + W_out = ( + (W - ksize[2] + pad_w_left + pad_w_right + strides[2] - 1) + // strides[2] + + 1 + if ceil_mode + else (W - ksize[2] + pad_w_left + pad_w_right) // strides[2] + 1 + ) + + out = ( + np.zeros((N, C, D_out, H_out, W_out)) + if data_format == 'NCDHW' else np.zeros((N, D_out, H_out, W_out, C)) + ) for k in range(D_out): if adaptive: d_start = adaptive_start_index(k, D, ksize[0]) @@ -127,17 +153,29 @@ def pool3D_forward_naive(x, else: d_start = k * strides[0] - pad_d_forth - d_end = np.min((k * strides[0] + ksize[0] - pad_d_forth, - D + pad_d_back)) + d_end = np.min( + ( + k * strides[0] + ksize[0] - pad_d_forth, + D + pad_d_back, + ) + ) h_start = i * strides[1] - pad_h_up h_end = np.min( - (i * strides[1] + ksize[1] - pad_h_up, H + pad_h_down)) + (i * strides[1] + ksize[1] - pad_h_up, H + pad_h_down) + ) w_start = j * strides[2] - pad_w_left - w_end = np.min((j * strides[2] + ksize[2] - pad_w_left, - W + pad_w_right)) - - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) + w_end = np.min( + ( + j * strides[2] + ksize[2] - pad_w_left, + W + pad_w_right, + ) + ) + + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) w_start = np.max((w_start, 0)) d_start = np.max((d_start, 0)) h_start = np.max((h_start, 0)) @@ -145,78 +183,95 @@ def pool3D_forward_naive(x, d_end = np.min((d_end, D)) h_end = np.min((h_end, H)) if data_format == 'NCDHW': - x_masked = x[:, :, d_start:d_end, h_start:h_end, - w_start:w_end] + x_masked = x[ + :, :, d_start:d_end, h_start:h_end, w_start:w_end + ] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (d_end - d_start) * ( - h_end - h_start) * (w_end - w_start) - - out[:, :, k, i, - j] = np.sum(x_masked, axis=(2, 3, 4)) / field_size + if exclusive or adaptive: + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + + out[:, :, k, i, j] = ( + np.sum(x_masked, axis=(2, 3, 4)) / field_size + ) elif pool_type == 'max': out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) elif data_format == 'NDHWC': - x_masked = x[:, d_start:d_end, h_start:h_end, - w_start:w_end, :] + x_masked = x[ + :, d_start:d_end, h_start:h_end, w_start:w_end, : + ] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (d_end - d_start) * ( - h_end - h_start) * (w_end - w_start) - - out[:, k, i, j, :] = np.sum(x_masked, - axis=(1, 2, 3)) / field_size + if exclusive or adaptive: + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + + out[:, k, i, j, :] = ( + np.sum(x_masked, axis=(1, 2, 3)) / field_size + ) elif pool_type == 'max': out[:, k, i, j, :] = np.max(x_masked, axis=(1, 2, 3)) return out -def max_pool3D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False): - out = pool3D_forward_naive(x=x, - ksize=ksize, - strides=strides, - paddings=paddings, - global_pool=global_pool, - ceil_mode=ceil_mode, - exclusive=exclusive, - adaptive=adaptive, - data_format='NCDHW', - pool_type="max") +def max_pool3D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, +): + out = pool3D_forward_naive( + x=x, + ksize=ksize, + strides=strides, + paddings=paddings, + global_pool=global_pool, + ceil_mode=ceil_mode, + exclusive=exclusive, + adaptive=adaptive, + data_format='NCDHW', + pool_type="max", + ) return out -def avg_pool3D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False): - out = pool3D_forward_naive(x=x, - ksize=ksize, - strides=strides, - paddings=paddings, - global_pool=global_pool, - ceil_mode=ceil_mode, - exclusive=exclusive, - adaptive=adaptive, - data_format='NCDHW', - pool_type="avg") +def avg_pool3D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, +): + out = pool3D_forward_naive( + x=x, + ksize=ksize, + strides=strides, + paddings=paddings, + global_pool=global_pool, + ceil_mode=ceil_mode, + exclusive=exclusive, + adaptive=adaptive, + data_format='NCDHW', + pool_type="avg", + ) return out class TestPool3D_Op(OpTest): - def setUp(self): self.op_type = "pool3d" self.init_kernel_type() @@ -235,12 +290,19 @@ class TestPool3D_Op(OpTest): paddle.enable_static() input = np.random.random(self.shape).astype(self.dtype) - output = pool3D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, self.data_format, - self.pool_type, - self.padding_algorithm).astype(self.dtype) + output = pool3D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} @@ -276,10 +338,9 @@ class TestPool3D_Op(OpTest): if self.has_cudnn() and self.pool_type != "max": place = core.CUDAPlace(0) if core.is_compiled_with_rocm(): - self.check_grad_with_place(place, - set(['X']), - 'Out', - max_relative_error=1e-2) + self.check_grad_with_place( + place, set(['X']), 'Out', max_relative_error=1e-2 + ) else: self.check_grad_with_place(place, set(['X']), 'Out') elif self.pool_type != "max": @@ -322,7 +383,6 @@ class TestPool3D_Op(OpTest): class TestCase1(TestPool3D_Op): - def init_shape(self): self.shape = [1, 3, 7, 7, 7] @@ -341,7 +401,6 @@ class TestCase1(TestPool3D_Op): class TestCase2(TestPool3D_Op): - def init_shape(self): self.shape = [1, 3, 6, 7, 7] @@ -360,32 +419,28 @@ class TestCase2(TestPool3D_Op): class TestCase3(TestPool3D_Op): - def init_pool_type(self): self.pool_type = "max" class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" -#--------------------test pool3d cudnn-------------------- +# --------------------test pool3d cudnn-------------------- def create_test_cudnn_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -403,11 +458,10 @@ create_test_cudnn_class(TestCase5) def create_test_cudnn_fp16_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -427,11 +481,10 @@ def create_test_cudnn_fp16_class(parent): def create_test_fp16_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = False self.dtype = np.float16 @@ -464,11 +517,10 @@ create_test_fp16_class(TestCase5) # ---- test ceil mode ------ def create_test_cudnn_use_ceil_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestPool3DUseCeilCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -485,9 +537,7 @@ create_test_cudnn_use_ceil_class(TestCase1) def create_test_use_ceil_class(parent): - class TestPool3DUseCeilCase(parent): - def init_ceil_mode(self): self.ceil_mode = True @@ -501,15 +551,14 @@ create_test_use_ceil_class(TestCase2) class TestAvgInclude(TestCase2): - def init_exclusive(self): self.exclusive = False -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNAvgInclude(TestCase2): - def init_kernel_type(self): self.use_cudnn = True @@ -518,13 +567,11 @@ class TestCUDNNAvgInclude(TestCase2): class TestAvgPoolAdaptive(TestCase1): - def init_adaptive(self): self.adaptive = True class TestAvgPoolAdaptiveAsyOutSize(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -536,9 +583,8 @@ class TestAvgPoolAdaptiveAsyOutSize(TestCase1): self.strides = [1, 1, 1] -#-------test pool3d with asymmetric padding------ +# -------test pool3d with asymmetric padding------ class TestPool3D_Op_AsyPadding(TestPool3D_Op): - def init_test_case(self): self.ksize = [3, 4, 3] self.strides = [1, 1, 2] @@ -551,7 +597,6 @@ class TestPool3D_Op_AsyPadding(TestPool3D_Op): class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3, 4] self.strides = [1, 1, 2] @@ -564,7 +609,6 @@ class TestCase1_AsyPadding(TestCase1): class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -577,7 +621,6 @@ class TestCase2_AsyPadding(TestCase2): class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -590,7 +633,6 @@ class TestCase3_AsyPadding(TestCase3): class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -603,7 +645,6 @@ class TestCase4_AsyPadding(TestCase4): class TestCase5_AsyPadding(TestCase5): - def init_test_case(self): self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -637,7 +678,6 @@ create_test_use_ceil_class(TestCase2_AsyPadding) class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -645,10 +685,10 @@ class TestAvgInclude_AsyPadding(TestCase2): self.paddings = [2, 2, 1, 1, 0, 0] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNAvgInclude_AsyPadding(TestCase2): - def init_kernel_type(self): self.use_cudnn = True @@ -663,7 +703,6 @@ class TestCUDNNAvgInclude_AsyPadding(TestCase2): class TestAvgPoolAdaptive_AsyPadding(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -673,7 +712,6 @@ class TestAvgPoolAdaptive_AsyPadding(TestCase1): # ------------ test channel_last -------------- class TestPool3D_channel_last(TestPool3D_Op): - def init_data_format(self): self.data_format = "NDHWC" @@ -682,7 +720,6 @@ class TestPool3D_channel_last(TestPool3D_Op): class TestCase1_channel_last(TestCase1): - def init_data_format(self): self.data_format = "NDHWC" @@ -691,7 +728,6 @@ class TestCase1_channel_last(TestCase1): class TestCase2_channel_last(TestCase2): - def init_data_format(self): self.data_format = "NDHWC" @@ -700,7 +736,6 @@ class TestCase2_channel_last(TestCase2): class TestCase3_channel_last(TestCase3): - def init_data_format(self): self.data_format = "NDHWC" @@ -709,7 +744,6 @@ class TestCase3_channel_last(TestCase3): class TestCase4_channel_last(TestCase4): - def init_data_format(self): self.data_format = "NDHWC" @@ -718,7 +752,6 @@ class TestCase4_channel_last(TestCase4): class TestCase5_channel_last(TestCase5): - def init_data_format(self): self.data_format = "NDHWC" @@ -741,7 +774,6 @@ create_test_use_ceil_class(TestCase2_channel_last) class TestCase5_Max(TestCase2): - def init_pool_type(self): self.pool_type = "max" @@ -750,16 +782,14 @@ class TestCase5_Max(TestCase2): return if self.has_cudnn() and self.pool_type == "max": place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['X']), - 'Out', - max_relative_error=1.00) + self.check_grad_with_place( + place, set(['X']), 'Out', max_relative_error=1.00 + ) elif self.pool_type == "max": self.check_grad(set(['X']), 'Out', max_relative_error=1.00) class TestCase5_channel_last_Max(TestCase5_Max): - def init_data_format(self): self.data_format = "NDHWC" @@ -772,15 +802,14 @@ create_test_cudnn_class(TestCase5_channel_last_Max) class TestAvgInclude_channel_last(TestCase2_channel_last): - def init_exclusive(self): self.exclusive = False -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNAvgInclude_channel_last(TestCase2_channel_last): - def init_kernel_type(self): self.use_cudnn = True @@ -789,14 +818,12 @@ class TestCUDNNAvgInclude_channel_last(TestCase2_channel_last): class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): - def init_adaptive(self): self.adaptive = True # --- asy padding class TestPool3D_Op_AsyPadding_channel_last(TestPool3D_Op_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -805,7 +832,6 @@ class TestPool3D_Op_AsyPadding_channel_last(TestPool3D_Op_AsyPadding): class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -814,7 +840,6 @@ class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -823,7 +848,6 @@ class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -832,7 +856,6 @@ class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -841,7 +864,6 @@ class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -864,23 +886,23 @@ create_test_use_ceil_class(TestCase2_AsyPadding_channel_last) class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") -class TestCUDNNAvgInclude_AsyPadding_channel_last(TestCUDNNAvgInclude_AsyPadding - ): - +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) +class TestCUDNNAvgInclude_AsyPadding_channel_last( + TestCUDNNAvgInclude_AsyPadding +): def init_data_format(self): self.data_format = "NDHWC" -class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding - ): - +class TestAvgPoolAdaptive_AsyPadding_channel_last( + TestAvgPoolAdaptive_AsyPadding +): def init_data_format(self): self.data_format = "NDHWC" @@ -888,11 +910,9 @@ class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding self.shape = [1, 7, 7, 7, 3] -#test padding = SAME VALID +# test padding = SAME VALID def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.paddings = [0, 0, 0] self.padding_algorithm = "SAME" @@ -918,11 +938,10 @@ create_test_padding_SAME_class(TestCase5_channel_last) def create_test_cudnn_padding_SAME_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingSMAECase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -951,9 +970,7 @@ create_test_cudnn_padding_SAME_class(TestCase5_channel_last) def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.paddings = [1, 1, 1] self.padding_algorithm = "VALID" @@ -979,11 +996,10 @@ create_test_padding_VALID_class(TestCase5_channel_last) def create_test_cudnn_padding_VALID_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingVALIDCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -1011,236 +1027,289 @@ create_test_cudnn_padding_VALID_class(TestCase4_channel_last) create_test_cudnn_padding_VALID_class(TestCase5_channel_last) -#test API +# test API class TestPool3DAPI(unittest.TestCase): - def test_api(self): x_NDHWC = np.random.random([2, 5, 5, 5, 3]).astype("float32") x_NCDHW = np.random.random([2, 3, 5, 5, 5]).astype("float32") - input_NDHWC = fluid.layers.data(name="input_NDHWC", - shape=[2, 5, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NDHWC = fluid.layers.data( + name="input_NDHWC", + shape=[2, 5, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) - input_NCDHW = fluid.layers.data(name="input_NCDHW", - shape=[2, 3, 5, 5, 5], - append_batch_size=False, - dtype="float32") + input_NCDHW = fluid.layers.data( + name="input_NCDHW", + shape=[2, 3, 5, 5, 5], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3, 3] - out_1 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1, 1], - use_cudnn=False, - data_format="NDHWC") - - out_2 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [1, 1], [1, 1], - [1, 1], [0, 0]], - use_cudnn=False, - data_format="NDHWC") - - out_3 = fluid.layers.pool3d(input=input_NCDHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [0, 0], [1, 1], - [1, 1], [1, 1]], - use_cudnn=False, - data_format="NCDHW") - - out_4 = fluid.layers.pool3d(input=input_NCDHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[1, 2, 1, 0, 0, 1], - use_cudnn=False, - data_format="NCDHW") + out_1 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1, 1], + use_cudnn=False, + data_format="NDHWC", + ) + + out_2 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], + use_cudnn=False, + data_format="NDHWC", + ) + + out_3 = fluid.layers.pool3d( + input=input_NCDHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], + use_cudnn=False, + data_format="NCDHW", + ) + + out_4 = fluid.layers.pool3d( + input=input_NCDHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[1, 2, 1, 0, 0, 1], + use_cudnn=False, + data_format="NCDHW", + ) # test VALID - out_5 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="avg", - pool_padding="VALID", - use_cudnn=False, - data_format="NDHWC") - - out_6 = fluid.layers.pool3d(input=input_NCDHW, - pool_size=ksize, - pool_type="avg", - pool_padding="VALID", - use_cudnn=False, - data_format="NCDHW") + out_5 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="avg", + pool_padding="VALID", + use_cudnn=False, + data_format="NDHWC", + ) + + out_6 = fluid.layers.pool3d( + input=input_NCDHW, + pool_size=ksize, + pool_type="avg", + pool_padding="VALID", + use_cudnn=False, + data_format="NCDHW", + ) # test SAME - out_7 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_stride=[1, 1, 2], - pool_type="avg", - pool_padding="SAME", - use_cudnn=False, - data_format="NDHWC") - - out_8 = fluid.layers.pool3d(input=input_NCDHW, - pool_size=[4, 4, 4], - pool_type="avg", - pool_padding="SAME", - use_cudnn=False, - data_format="NCDHW") + out_7 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_stride=[1, 1, 2], + pool_type="avg", + pool_padding="SAME", + use_cudnn=False, + data_format="NDHWC", + ) + + out_8 = fluid.layers.pool3d( + input=input_NCDHW, + pool_size=[4, 4, 4], + pool_type="avg", + pool_padding="SAME", + use_cudnn=False, + data_format="NCDHW", + ) exe = fluid.Executor(place=fluid.CPUPlace()) [res_1, res_2, res_3, res_4, res_5, res_6, res_7, res_8] = exe.run( fluid.default_main_program(), - feed={ - "input_NDHWC": x_NDHWC, - "input_NCDHW": x_NCDHW - }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + feed={"input_NDHWC": x_NDHWC, "input_NCDHW": x_NCDHW}, + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.allclose( res_1, - pool3D_forward_naive(x=x_NDHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1, 1], - paddings=[1, 1, 1], - data_format="NDHWC")) + pool3D_forward_naive( + x=x_NDHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1, 1], + paddings=[1, 1, 1], + data_format="NDHWC", + ), + ) assert np.allclose( res_2, - pool3D_forward_naive(x=x_NDHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[1, 1, 1, 1, 1, 1], - data_format="NDHWC")) - assert np.allclose(res_3, - pool3D_forward_naive(x=x_NCDHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[1, 1, 1, 1, 1, 1], - data_format="NCDHW"), - rtol=0.07, - atol=1e-05) - - assert np.allclose(res_4, - pool3D_forward_naive(x=x_NCDHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[1, 2, 1, 0, 0, 1], - data_format="NCDHW"), - rtol=0.07, - atol=1e-05) + pool3D_forward_naive( + x=x_NDHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[1, 1, 1, 1, 1, 1], + data_format="NDHWC", + ), + ) + assert np.allclose( + res_3, + pool3D_forward_naive( + x=x_NCDHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[1, 1, 1, 1, 1, 1], + data_format="NCDHW", + ), + rtol=0.07, + atol=1e-05, + ) + + assert np.allclose( + res_4, + pool3D_forward_naive( + x=x_NCDHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[1, 2, 1, 0, 0, 1], + data_format="NCDHW", + ), + rtol=0.07, + atol=1e-05, + ) # VALID assert np.allclose( res_5, - pool3D_forward_naive(x=x_NDHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[10, 20], - padding_algorithm="VALID", - data_format="NDHWC")) - - assert np.allclose(res_6, - pool3D_forward_naive(x=x_NCDHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[10, 20], - padding_algorithm="VALID", - data_format="NCDHW"), - rtol=0.07, - atol=1e-05) + pool3D_forward_naive( + x=x_NDHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[10, 20], + padding_algorithm="VALID", + data_format="NDHWC", + ), + ) + + assert np.allclose( + res_6, + pool3D_forward_naive( + x=x_NCDHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[10, 20], + padding_algorithm="VALID", + data_format="NCDHW", + ), + rtol=0.07, + atol=1e-05, + ) # SAME assert np.allclose( res_7, - pool3D_forward_naive(x=x_NDHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 2], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NDHWC")) - - assert np.allclose(res_8, - pool3D_forward_naive(x=x_NCDHW, - ksize=[4, 4, 4], - pool_type="avg", - strides=[1, 1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NCDHW"), - rtol=0.07, - atol=1e-05) + pool3D_forward_naive( + x=x_NDHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 2], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NDHWC", + ), + ) + + assert np.allclose( + res_8, + pool3D_forward_naive( + x=x_NCDHW, + ksize=[4, 4, 4], + pool_type="avg", + strides=[1, 1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NCDHW", + ), + rtol=0.07, + atol=1e-05, + ) class TestPool3DAPI_Error(unittest.TestCase): - def test_api(self): - input_NDHWC = fluid.layers.data(name="input_NDHWC", - shape=[2, 5, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NDHWC = fluid.layers.data( + name="input_NDHWC", + shape=[2, 5, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3, 3] # cudnn type error def run_1(): - out_1 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1, 1], - use_cudnn=[0], - data_format="NDHWC") + out_1 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1, 1], + use_cudnn=[0], + data_format="NDHWC", + ) self.assertRaises(TypeError, run_1) # data_format value error def run_2(): - out_2 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1, 1], - use_cudnn=False, - data_format="NDHWCC") + out_2 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1, 1], + use_cudnn=False, + data_format="NDHWCC", + ) self.assertRaises(ValueError, run_2) # padding str value error def run_3(): - out_3 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALIDSAME", - use_cudnn=False, - data_format="NDHWC") + out_3 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALIDSAME", + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_3) # padding str valid and ceil_mode value error def run_4(): - out_4 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - use_cudnn=False, - ceil_mode=True, - data_format="NDHWC") + out_4 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + use_cudnn=False, + ceil_mode=True, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_4) # padding with 8 ele. value error def run_5(): - out_5 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[[1, 1], [0, 0], [0, 0], - [1, 1], [1, 1]], - use_cudnn=False, - data_format="NDHWC") + out_5 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[[1, 1], [0, 0], [0, 0], [1, 1], [1, 1]], + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_5) diff --git a/python/paddle/fluid/tests/unittests/test_pool_max_op.py b/python/paddle/fluid/tests/unittests/test_pool_max_op.py index 64204aa0ff2..f324adb6c8d 100644 --- a/python/paddle/fluid/tests/unittests/test_pool_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool_max_op.py @@ -25,12 +25,9 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def max_pool3D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=False, - adaptive=False): +def max_pool3D_forward_naive( + x, ksize, strides, paddings, global_pool=False, adaptive=False +): N, C, D, H, W = x.shape if global_pool: @@ -77,19 +74,19 @@ def max_pool3D_forward_naive(x, sub_deep = index[0][0] sub_row = index[1][0] sub_col = index[2][0] - index = ((d_start + sub_deep) * H + - (h_start + sub_row)) * W + w_start + sub_col + index = ( + ((d_start + sub_deep) * H + (h_start + sub_row)) * W + + w_start + + sub_col + ) mask[n, c, k, i, j] = index return out, mask -def max_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=False, - adaptive=False): +def max_pool2D_forward_naive( + x, ksize, strides, paddings, global_pool=False, adaptive=False +): N, C, H, W = x.shape if global_pool: @@ -132,17 +129,21 @@ def max_pool2D_forward_naive(x, class TestMaxPoolWithIndex_Op(OpTest): - def setUp(self): self.init_test_case() self.init_global() self.init_adaptive() input = np.random.random(self.shape).astype("float64") - input = np.round(input * 100., 2) - output, mask = self.pool_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.adaptive) + input = np.round(input * 100.0, 2) + output, mask = self.pool_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.adaptive, + ) output = output.astype("float64") mask = mask.astype("int32") @@ -179,13 +180,11 @@ class TestMaxPoolWithIndex_Op(OpTest): class TestCase1(TestMaxPoolWithIndex_Op): - def init_global(self): self.global_pool = True class TestCase2(TestMaxPoolWithIndex_Op): - def init_test_case(self): self.op_type = "max_pool3d_with_index" self.pool_forward_naive = max_pool3D_forward_naive @@ -199,14 +198,12 @@ class TestCase2(TestMaxPoolWithIndex_Op): class TestCase3(TestCase2): - def init_global(self): self.global_pool = False -#----------------max_pool2d_with_index---------------- +# ----------------max_pool2d_with_index---------------- class TestCase4(TestMaxPoolWithIndex_Op): - def init_test_case(self): self.op_type = "max_pool2d_with_index" self.pool_forward_naive = max_pool2D_forward_naive @@ -220,13 +217,11 @@ class TestCase4(TestMaxPoolWithIndex_Op): class TestCase5(TestCase4): - def init_global(self): self.global_pool = False class TestCase6(TestMaxPoolWithIndex_Op): - def init_test_case(self): self.op_type = "max_pool2d_with_index" self.pool_forward_naive = max_pool2D_forward_naive @@ -240,19 +235,16 @@ class TestCase6(TestMaxPoolWithIndex_Op): class TestCase7(TestCase6): - def init_global(self): self.global_pool = False class TestCastAdaptive2d(TestCase6): - def init_adaptive(self): self.adaptive = True class TestCastAdaptive3d(TestMaxPoolWithIndex_Op): - def init_adaptive(self): self.adaptive = True diff --git a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py index dcbd5928fc4..d666157e7d2 100644 --- a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py +++ b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py @@ -45,12 +45,14 @@ def py_pnpair_op(score, label, query, column=-1, weight=None): else: neg += w - return np.array(pos).astype('float32'), np.array(neg).astype( - 'float32'), np.array(neu).astype('float32') + return ( + np.array(pos).astype('float32'), + np.array(neg).astype('float32'), + np.array(neu).astype('float32'), + ) class TestPositiveNegativePairOp(OpTest): - def setUp(self): self.op_type = 'positive_negative_pair' batch_size = 20 @@ -58,7 +60,8 @@ class TestPositiveNegativePairOp(OpTest): score = np.random.normal(size=(batch_size, 1)).astype('float32') label = np.random.normal(size=(batch_size, 1)).astype('float32') query = np.array( - [np.random.randint(max_query_id) for i in range(batch_size)]) + [np.random.randint(max_query_id) for i in range(batch_size)] + ) query = np.reshape(query, newshape=(batch_size, 1)).astype('int64') pos, neg, neu = py_pnpair_op(score, label, query) @@ -67,7 +70,7 @@ class TestPositiveNegativePairOp(OpTest): self.outputs = { 'PositivePair': pos, 'NegativePair': neg, - 'NeutralPair': neu + 'NeutralPair': neu, } def test_check_output(self): @@ -75,7 +78,6 @@ class TestPositiveNegativePairOp(OpTest): class TestPositiveNegativePairOpAccumulateWeight(OpTest): - def setUp(self): self.op_type = 'positive_negative_pair' batch_size = 20 @@ -86,21 +88,23 @@ class TestPositiveNegativePairOpAccumulateWeight(OpTest): label = np.random.normal(size=(batch_size, 1)).astype('float32') weight = np.random.normal(size=(batch_size, 1)).astype('float32') query = np.array( - [np.random.randint(max_query_id) for i in range(batch_size)]) + [np.random.randint(max_query_id) for i in range(batch_size)] + ) query = np.reshape(query, newshape=(batch_size, 1)).astype('int64') - acc_pos = np.reshape(np.random.randint(max_random_num), - newshape=(1)).astype('float32') - acc_neg = np.reshape(np.random.randint(max_random_num), - newshape=(1)).astype('float32') - acc_neu = np.reshape(np.random.randint(max_random_num), - newshape=(1)).astype('float32') + acc_pos = np.reshape( + np.random.randint(max_random_num), newshape=(1) + ).astype('float32') + acc_neg = np.reshape( + np.random.randint(max_random_num), newshape=(1) + ).astype('float32') + acc_neu = np.reshape( + np.random.randint(max_random_num), newshape=(1) + ).astype('float32') column = np.random.randint(score_dim) - pos, neg, neu = py_pnpair_op(score, - label, - query, - column=column, - weight=weight) + pos, neg, neu = py_pnpair_op( + score, label, query, column=column, weight=weight + ) self.inputs = { 'Score': score, 'Label': label, @@ -108,13 +112,13 @@ class TestPositiveNegativePairOpAccumulateWeight(OpTest): 'AccumulatePositivePair': acc_pos, 'AccumulateNegativePair': acc_neg, 'AccumulateNeutralPair': acc_neu, - 'Weight': weight + 'Weight': weight, } self.attrs = {'column': column} self.outputs = { 'PositivePair': pos + acc_pos, 'NegativePair': neg + acc_neg, - 'NeutralPair': neu + acc_neu + 'NeutralPair': neu + acc_neu, } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_pow.py b/python/paddle/fluid/tests/unittests/test_pow.py index 9df1af1375c..79282f3460f 100755 --- a/python/paddle/fluid/tests/unittests/test_pow.py +++ b/python/paddle/fluid/tests/unittests/test_pow.py @@ -71,7 +71,7 @@ class TestPowerAPI(unittest.TestCase): """test_power.""" np.random.seed(7) # test 1-d float tensor ** float scalar - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float64) y = np.random.rand() * 10 res = _run_power(DYNAMIC, x, y) @@ -80,7 +80,7 @@ class TestPowerAPI(unittest.TestCase): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d float tensor ** int scalar - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float64) y = int(np.random.rand() * 10) res = _run_power(DYNAMIC, x, y) @@ -96,7 +96,7 @@ class TestPowerAPI(unittest.TestCase): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d float tensor ** 1-d float tensor - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float64) y = (np.random.rand(*dims) * 10).astype(np.float64) res = _run_power(DYNAMIC, x, y) @@ -105,7 +105,7 @@ class TestPowerAPI(unittest.TestCase): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d int tensor ** 1-d int tensor - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.int64) y = (np.random.rand(*dims) * 10).astype(np.int64) res = _run_power(DYNAMIC, x, y) @@ -114,7 +114,7 @@ class TestPowerAPI(unittest.TestCase): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d int tensor ** 1-d int tensor - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.int32) y = (np.random.rand(*dims) * 10).astype(np.int32) res = _run_power(DYNAMIC, x, y) @@ -123,7 +123,7 @@ class TestPowerAPI(unittest.TestCase): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d int tensor ** 1-d int tensor - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float32) y = (np.random.rand(*dims) * 10).astype(np.float32) res = _run_power(DYNAMIC, x, y) @@ -132,8 +132,11 @@ class TestPowerAPI(unittest.TestCase): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test broadcast - dims = (np.random.randint(1, 10), np.random.randint(5, 10), - np.random.randint(5, 10)) + dims = ( + np.random.randint(1, 10), + np.random.randint(5, 10), + np.random.randint(5, 10), + ) x = (np.random.rand(*dims) * 10).astype(np.float64) y = (np.random.rand(dims[-1]) * 10).astype(np.float64) res = _run_power(DYNAMIC, x, y) @@ -150,22 +153,28 @@ class TestPowerError(unittest.TestCase): np.random.seed(7) # test dynamic computation graph: inputs must be broadcastable - dims = (np.random.randint(1, 10), np.random.randint(5, 10), - np.random.randint(5, 10)) + dims = ( + np.random.randint(1, 10), + np.random.randint(5, 10), + np.random.randint(5, 10), + ) x = (np.random.rand(*dims) * 10).astype(np.float64) y = (np.random.rand(dims[-1] + 1) * 10).astype(np.float64) self.assertRaises(ValueError, _run_power, DYNAMIC, x, y) self.assertRaises(ValueError, _run_power, STATIC, x, y) # test dynamic computation graph: inputs must be broadcastable - dims = (np.random.randint(1, 10), np.random.randint(5, 10), - np.random.randint(5, 10)) + dims = ( + np.random.randint(1, 10), + np.random.randint(5, 10), + np.random.randint(5, 10), + ) x = (np.random.rand(*dims) * 10).astype(np.float64) y = (np.random.rand(dims[-1] + 1) * 10).astype(np.int8) self.assertRaises(TypeError, paddle.pow, x, y) # test 1-d float tensor ** int string - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float64) y = int(np.random.rand() * 10) self.assertRaises(TypeError, paddle.pow, x, str(y)) diff --git a/python/paddle/fluid/tests/unittests/test_pow2_decay_with_linear_warmup_op.py b/python/paddle/fluid/tests/unittests/test_pow2_decay_with_linear_warmup_op.py index 43f98cada42..69619ed02af 100644 --- a/python/paddle/fluid/tests/unittests/test_pow2_decay_with_linear_warmup_op.py +++ b/python/paddle/fluid/tests/unittests/test_pow2_decay_with_linear_warmup_op.py @@ -23,8 +23,9 @@ def gen_pow2_warmup_op_lr(warmup_steps, total_steps, base_lr, end_lr, place): main = paddle.static.Program() startup = paddle.static.Program() with paddle.static.program_guard(main, startup): - lr = pow2_decay_with_linear_warmup(warmup_steps, total_steps, base_lr, - end_lr) + lr = pow2_decay_with_linear_warmup( + warmup_steps, total_steps, base_lr, end_lr + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): exe.run(startup) @@ -34,18 +35,21 @@ def gen_pow2_warmup_op_lr(warmup_steps, total_steps, base_lr, end_lr, place): class Pow2Warmup(LinearWarmup): - def __init__(self, warmup_steps, total_steps, base_lr, end_lr): assert total_steps > warmup_steps - lr_sch = PolynomialDecay(learning_rate=base_lr, - decay_steps=total_steps - warmup_steps, - end_lr=end_lr, - power=2) + lr_sch = PolynomialDecay( + learning_rate=base_lr, + decay_steps=total_steps - warmup_steps, + end_lr=end_lr, + power=2, + ) - super(Pow2Warmup, self).__init__(learning_rate=lr_sch, - warmup_steps=warmup_steps, - start_lr=0.0, - end_lr=base_lr) + super(Pow2Warmup, self).__init__( + learning_rate=lr_sch, + warmup_steps=warmup_steps, + start_lr=0.0, + end_lr=base_lr, + ) def gen_pow2_warmup_py_lr(warmup_steps, total_steps, base_lr, end_lr, place): @@ -57,7 +61,6 @@ def gen_pow2_warmup_py_lr(warmup_steps, total_steps, base_lr, end_lr, place): class TestPow2WarmupLRScheduler(unittest.TestCase): - def setUp(self): paddle.enable_static() self.params = { diff --git a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py index 26e12b0deb0..8bc80de3429 100644 --- a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py +++ b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py @@ -85,16 +85,21 @@ def compute_metrics(states, cls_num): class TestPrecisionRecallOp_0(OpTest): - def setUp(self): self.op_type = "precision_recall" ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') - labels = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + idxs = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) + labels = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) states = get_states(idxs, labels, cls_num) metrics = compute_metrics(states, cls_num) @@ -105,7 +110,7 @@ class TestPrecisionRecallOp_0(OpTest): self.outputs = { 'BatchMetrics': metrics, 'AccumMetrics': metrics, - 'AccumStatesInfo': states + 'AccumStatesInfo': states, } def test_check_output(self): @@ -113,17 +118,22 @@ class TestPrecisionRecallOp_0(OpTest): class TestPrecisionRecallOp_1(OpTest): - def setUp(self): self.op_type = "precision_recall" ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + idxs = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - labels = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + labels = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) states = get_states(idxs, labels, cls_num, weights) metrics = compute_metrics(states, cls_num) @@ -134,13 +144,13 @@ class TestPrecisionRecallOp_1(OpTest): 'MaxProbs': max_probs, 'Indices': idxs, 'Labels': labels, - 'Weights': weights + 'Weights': weights, } self.outputs = { 'BatchMetrics': metrics, 'AccumMetrics': metrics, - 'AccumStatesInfo': states + 'AccumStatesInfo': states, } def test_check_output(self): @@ -148,17 +158,22 @@ class TestPrecisionRecallOp_1(OpTest): class TestPrecisionRecallOp_2(OpTest): - def setUp(self): self.op_type = "precision_recall" ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + idxs = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - labels = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + labels = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) states = np.random.randint(0, 30, (cls_num, 4)).astype('float32') accum_states = get_states(idxs, labels, cls_num, weights) @@ -173,13 +188,13 @@ class TestPrecisionRecallOp_2(OpTest): 'Indices': idxs, 'Labels': labels, 'Weights': weights, - 'StatesInfo': states + 'StatesInfo': states, } self.outputs = { 'BatchMetrics': batch_metrics, 'AccumMetrics': accum_metrics, - 'AccumStatesInfo': accum_states + 'AccumStatesInfo': accum_states, } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_prelu_op.py b/python/paddle/fluid/tests/unittests/test_prelu_op.py index bf6de7911da..c267286be9d 100644 --- a/python/paddle/fluid/tests/unittests/test_prelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_prelu_op.py @@ -38,11 +38,13 @@ def ref_prelu_nn(x, num_parameters, init): class TestFunctionalPReluAPI(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() - self.x_np = np.random.uniform(-1., 1., [1, 2, 3, 4]).astype('float32') + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + self.x_np = np.random.uniform(-1.0, 1.0, [1, 2, 3, 4]).astype('float32') self.weight_np_0 = np.random.randn(1).astype('float32') self.weight_np_1 = np.random.randn(self.x_np.shape[1]).astype('float32') @@ -52,11 +54,9 @@ class TestFunctionalPReluAPI(unittest.TestCase): weight = paddle.fluid.data('Alpha', weight_np.shape, 'float32') out = F.prelu(x, weight) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'X': self.x_np, - 'Alpha': weight_np - }, - fetch_list=[out]) + res = exe.run( + feed={'X': self.x_np, 'Alpha': weight_np}, fetch_list=[out] + ) out_ref = ref_prelu(self.x_np, weight_np) np.testing.assert_allclose(out_ref, res[0], rtol=1e-05) @@ -83,44 +83,46 @@ class TestFunctionalPReluAPI(unittest.TestCase): def test_error(self): with paddle.static.program_guard(paddle.static.Program()): - weight_fp32 = paddle.fluid.data(name='weight_fp32', - shape=[1], - dtype='float32') + weight_fp32 = paddle.fluid.data( + name='weight_fp32', shape=[1], dtype='float32' + ) # The input type must be Variable. self.assertRaises(TypeError, F.prelu, x=1, weight=weight_fp32) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) self.assertRaises(TypeError, F.prelu, x=x_int32, weight=weight_fp32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[2, 3], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[2, 3], dtype='float16' + ) F.prelu(x=x_fp16, weight=weight_fp32) class TestNNPReluAPI(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.x_np = np.ones([1, 2, 3, 4]).astype('float32') def test_static_api(self): startup_program = paddle.static.Program() train_program = paddle.static.Program() with paddle.static.program_guard(train_program, startup_program): - x = paddle.fluid.data(name='X', - shape=self.x_np.shape, - dtype='float32') + x = paddle.fluid.data( + name='X', shape=self.x_np.shape, dtype='float32' + ) m = paddle.nn.PReLU() out = m(x) exe = paddle.static.Executor(self.place) exe.run(startup_program) - res = exe.run(train_program, - feed={'X': self.x_np}, - fetch_list=[out]) + res = exe.run( + train_program, feed={'X': self.x_np}, fetch_list=[out] + ) out_ref = ref_prelu_nn(self.x_np, 1, 0.25) np.testing.assert_allclose(out_ref, res[0], rtol=1e-05) @@ -152,8 +154,11 @@ class TestNNPReluAPI(unittest.TestCase): np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-05) x = paddle.to_tensor(self.x_np) - m = paddle.nn.PReLU(weight_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.5))) + m = paddle.nn.PReLU( + weight_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(0.5) + ) + ) out = m(x) out_ref = ref_prelu_nn(self.x_np, 1, 0.5) np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-05) @@ -167,7 +172,6 @@ def prelu_api_wrapper(x, weight, data_format="NCHW"): class PReluTest(OpTest): - def setUp(self): self.init_dtype() self.init_input_shape() @@ -182,12 +186,9 @@ class PReluTest(OpTest): x_np[np.abs(x_np) < 0.005] = 0.02 if self.attrs == { - 'mode': "all", - "data_format": "NCHW" - } or self.attrs == { - 'mode': "all", - "data_format": "NHWC" - }: + 'mode': "all", + "data_format": "NCHW", + } or self.attrs == {'mode': "all", "data_format": "NHWC"}: alpha_np = np.random.uniform(-1, -0.5, (1)) elif self.attrs == {'mode': "channel", "data_format": "NCHW"}: alpha_np = np.random.uniform(-1, -0.5, [1, self.x_shape[1], 1, 1]) @@ -205,15 +206,17 @@ class PReluTest(OpTest): # since np operands could not be broadcast together with shapes (1,100,2,2,2,3) (1,100,1,1) reshaped_alpha = self.inputs['Alpha'] if self.attrs == {'mode': "channel", "data_format": "NCHW"}: - reshaped_alpha = np.reshape(self.inputs['Alpha'], - [1, self.x_shape[1]] + - [1] * len(self.x_shape[2:])) + reshaped_alpha = np.reshape( + self.inputs['Alpha'], + [1, self.x_shape[1]] + [1] * len(self.x_shape[2:]), + ) elif self.attrs == {'mode': "channel", "data_format": "NHWC"}: - reshaped_alpha = np.reshape(self.inputs['Alpha'], - [1] + [1] * len(self.x_shape[1:-1]) + - [self.x_shape[-1]]) - out_np = np.maximum(self.inputs['X'], 0.) - out_np = out_np + np.minimum(self.inputs['X'], 0.) * reshaped_alpha + reshaped_alpha = np.reshape( + self.inputs['Alpha'], + [1] + [1] * len(self.x_shape[1:-1]) + [self.x_shape[-1]], + ) + out_np = np.maximum(self.inputs['X'], 0.0) + out_np = out_np + np.minimum(self.inputs['X'], 0.0) * reshaped_alpha assert out_np is not self.inputs['X'] self.outputs = {'Out': out_np} @@ -234,11 +237,9 @@ class PReluTest(OpTest): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAll(PReluTest): - def init_input_shape(self): self.x_shape = [2, 3, 4, 5] @@ -247,11 +248,9 @@ class TestModeAll(PReluTest): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllNHWC(PReluTest): - def init_input_shape(self): self.x_shape = [2, 3, 4, 50] @@ -260,7 +259,6 @@ class TestModeAllNHWC(PReluTest): class TestModeElt(PReluTest): - def init_input_shape(self): self.x_shape = [3, 2, 5, 10] @@ -269,7 +267,6 @@ class TestModeElt(PReluTest): class TestModeEltNHWC(PReluTest): - def init_input_shape(self): self.x_shape = [3, 2, 5, 10] @@ -278,11 +275,9 @@ class TestModeEltNHWC(PReluTest): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllRank3(PReluTest): - def init_input_shape(self): self.x_shape = [1, 200, 3] @@ -291,11 +286,9 @@ class TestModeAllRank3(PReluTest): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllRank3NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [1, 200, 3] @@ -304,11 +297,9 @@ class TestModeAllRank3NHWC(PReluTest): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllRank6(PReluTest): - def init_input_shape(self): self.x_shape = [1, 2, 3, 4, 5, 6] @@ -317,11 +308,9 @@ class TestModeAllRank6(PReluTest): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllRank6NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [1, 2, 3, 4, 5, 6] @@ -330,7 +319,6 @@ class TestModeAllRank6NHWC(PReluTest): class TestModeChannelRank3(PReluTest): - def init_input_shape(self): self.x_shape = [1, 200, 3] @@ -339,7 +327,6 @@ class TestModeChannelRank3(PReluTest): class TestModeChannelRank3NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [1, 3, 100] @@ -348,7 +335,6 @@ class TestModeChannelRank3NHWC(PReluTest): class TestModeChannelRank6(PReluTest): - def init_input_shape(self): self.x_shape = [1, 100, 2, 2, 2, 2] @@ -357,7 +343,6 @@ class TestModeChannelRank6(PReluTest): class TestModeChannelRank6NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [1, 2, 2, 2, 2, 100] @@ -366,7 +351,6 @@ class TestModeChannelRank6NHWC(PReluTest): class TestModeElementRank3(PReluTest): - def init_input_shape(self): self.x_shape = [3, 10, 10] @@ -375,7 +359,6 @@ class TestModeElementRank3(PReluTest): class TestModeElementRank3NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [3, 10, 10] @@ -384,7 +367,6 @@ class TestModeElementRank3NHWC(PReluTest): class TestModeElementRank6(PReluTest): - def init_input_shape(self): self.x_shape = [3, 2, 2, 4, 5, 2] @@ -393,7 +375,6 @@ class TestModeElementRank6(PReluTest): class TestModeElementRank6NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [3, 2, 2, 4, 5, 2] @@ -401,15 +382,13 @@ class TestModeElementRank6NHWC(PReluTest): self.attrs = {'mode': "element", "data_format": "NHWC"} -def create_test_fp16_class(parent, - check_grad=True, - atol=1e-3, - max_relative_error=0.05): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +def create_test_fp16_class( + parent, check_grad=True, atol=1e-3, max_relative_error=0.05 +): + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestPReluFp16Case(parent): - def init_dtype(self): self.dtype = np.float16 @@ -417,18 +396,20 @@ def create_test_fp16_class(parent, if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_output_with_place(place, - atol=atol, - check_eager=self.eager_mode) + self.check_output_with_place( + place, atol=atol, check_eager=self.eager_mode + ) def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and check_grad: self.check_grad_with_place( - place, ['X', 'Alpha'], + place, + ['X', 'Alpha'], 'Out', max_relative_error=max_relative_error, - check_eager=self.eager_mode) + check_eager=self.eager_mode, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16Op") TestPReluFp16Case.__name__ = cls_name @@ -460,27 +441,26 @@ def prelu_t(x, mode, param_attr=None, name=None, data_format='NCHW'): shape=alpha_shape, dtype='float32', is_bias=False, - default_initializer=fluid.initializer.ConstantInitializer(0.25)) + default_initializer=fluid.initializer.ConstantInitializer(0.25), + ) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="prelu", - inputs={ - "X": x, - 'Alpha': alpha - }, - attrs={ - "mode": mode, - 'data_format': data_format - }, - outputs={"Out": out}) + helper.append_op( + type="prelu", + inputs={"X": x, 'Alpha': alpha}, + attrs={"mode": mode, 'data_format': data_format}, + outputs={"Out": out}, + ) return out # error message test if mode is not one of 'all', 'channel', 'element' class TestModeError(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.x_np = np.ones([1, 2, 3, 4]).astype('float32') def test_mode_error(self): @@ -490,7 +470,7 @@ class TestModeError(unittest.TestCase): try: y = prelu_t(x, 'any') except Exception as e: - assert (e.args[0].find('InvalidArgument') != -1) + assert e.args[0].find('InvalidArgument') != -1 def test_data_format_error1(self): main_program = Program() @@ -499,7 +479,7 @@ class TestModeError(unittest.TestCase): try: y = prelu_t(x, 'channel', data_format='N') except Exception as e: - assert (e.args[0].find('InvalidArgument') != -1) + assert e.args[0].find('InvalidArgument') != -1 def test_data_format_error2(self): main_program = Program() diff --git a/python/paddle/fluid/tests/unittests/test_print_op.py b/python/paddle/fluid/tests/unittests/test_print_op.py index 1dc6cca53cb..0ce1230f52e 100755 --- a/python/paddle/fluid/tests/unittests/test_print_op.py +++ b/python/paddle/fluid/tests/unittests/test_print_op.py @@ -28,7 +28,6 @@ paddle.enable_static() class TestPrintOpCPU(unittest.TestCase): - def setUp(self): self.place = paddle.CPUPlace() self.x_tensor = fluid.core.LoDTensor() @@ -48,17 +47,17 @@ class TestPrintOpCPU(unittest.TestCase): switch_main_program(Program()) printed = self.build_network(True, print_phase='forward') exe = paddle.static.Executor(self.place) - outs = exe.run(feed={'x': self.x_tensor}, - fetch_list=[printed], - return_numpy=False) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[printed], return_numpy=False + ) def test_backward(self): switch_main_program(Program()) loss = self.build_network(False, print_phase='backward') exe = paddle.static.Executor(self.place) - outs = exe.run(feed={'x': self.x_tensor}, - fetch_list=[loss], - return_numpy=False) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[loss], return_numpy=False + ) def test_all_parameters(self): x = layers.data('x', shape=[3], dtype='float32', lod_level=1) @@ -78,36 +77,36 @@ class TestPrintOpCPU(unittest.TestCase): loss = paddle.mean(x) paddle.static.append_backward(loss=loss) exe = paddle.static.Executor(self.place) - outs = exe.run(feed={'x': self.x_tensor}, - fetch_list=[loss], - return_numpy=False) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[loss], return_numpy=False + ) def test_no_summarize(self): switch_main_program(Program()) printed = self.build_network(True, summarize=-1, print_phase='forward') exe = paddle.static.Executor(self.place) - outs = exe.run(feed={'x': self.x_tensor}, - fetch_list=[printed], - return_numpy=False) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[printed], return_numpy=False + ) class TestPrintOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of Print_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.CPUPlace() + ) self.assertRaises(TypeError, paddle.static.Print, x1) # The input dtype of Print_op must be float32, float64, int32_t, int64_t or bool. x2 = paddle.static.data(name='x2', shape=[4], dtype="float16") self.assertRaises(TypeError, paddle.static.Print, x2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPrintOpGPU(TestPrintOpCPU): - def setUp(self): self.place = paddle.CUDAPlace(0) self.x_tensor = fluid.core.LoDTensor() @@ -117,7 +116,6 @@ class TestPrintOpGPU(TestPrintOpCPU): class TestPrintOpBackward(unittest.TestCase): - def check_backward(self, use_cuda): main = paddle.static.Program() startup = paddle.static.Program() @@ -135,7 +133,8 @@ class TestPrintOpBackward(unittest.TestCase): exe.run(startup) binary = paddle.static.CompiledProgram(main).with_data_parallel( - loss_name=loss.name) + loss_name=loss.name + ) img, label = init_data() feed_dict = {"image": img, "label": label} diff --git a/python/paddle/fluid/tests/unittests/test_prior_box_op.py b/python/paddle/fluid/tests/unittests/test_prior_box_op.py index 67d468ac626..80e567be34c 100644 --- a/python/paddle/fluid/tests/unittests/test_prior_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_prior_box_op.py @@ -19,19 +19,21 @@ from op_test import OpTest import paddle -def python_prior_box(input, - image, - min_sizes, - aspect_ratios=[1.], - variances=[0.1, 0.1, 0.2, 0.2], - max_sizes=None, - flip=False, - clip=False, - step_w=0, - step_h=0, - offset=0.5, - min_max_aspect_ratios_order=False, - name=None): +def python_prior_box( + input, + image, + min_sizes, + aspect_ratios=[1.0], + variances=[0.1, 0.1, 0.2, 0.2], + max_sizes=None, + flip=False, + clip=False, + step_w=0, + step_h=0, + offset=0.5, + min_max_aspect_ratios_order=False, + name=None, +): return paddle.fluid.layers.detection.prior_box( input, image, @@ -44,11 +46,11 @@ def python_prior_box(input, steps=[step_w, step_h], offset=offset, name=name, - min_max_aspect_ratios_order=min_max_aspect_ratios_order) + min_max_aspect_ratios_order=min_max_aspect_ratios_order, + ) class TestPriorBoxOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -107,8 +109,9 @@ class TestPriorBoxOp(OpTest): self.flip = True self.set_min_max_aspect_ratios_order() self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] - self.aspect_ratios = np.array(self.aspect_ratios, - dtype=np.float64).flatten() + self.aspect_ratios = np.array( + self.aspect_ratios, dtype=np.float64 + ).flatten() self.variances = [0.1, 0.1, 0.2, 0.2] self.variances = np.array(self.variances, dtype=np.float64).flatten() @@ -120,12 +123,12 @@ class TestPriorBoxOp(OpTest): def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype('float32') + (self.batch_size, self.image_channels, self.image_w, self.image_h) + ).astype('float32') self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype('float32') + (self.batch_size, self.input_channels, self.layer_w, self.layer_h) + ).astype('float32') def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -146,73 +149,78 @@ class TestPriorBoxOp(OpTest): ar = self.real_aspect_ratios[r] c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 else: - c_w = c_h = min_size / 2. - out_boxes[h, w, idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + c_w = c_h = min_size / 2.0 + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # rest of priors for r in range(len(self.real_aspect_ratios)): ar = self.real_aspect_ratios[r] - if abs(ar - 1.) < 1e-6: + if abs(ar - 1.0) < 1e-6: continue c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) # set the variance. - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype('float32') self.out_var = out_var.astype('float32') class TestPriorBoxOpWithoutMaxSize(TestPriorBoxOp): - def set_max_sizes(self): self.max_sizes = [] class TestPriorBoxOpWithSpecifiedOutOrder(TestPriorBoxOp): - def set_min_max_aspect_ratios_order(self): self.min_max_aspect_ratios_order = True diff --git a/python/paddle/fluid/tests/unittests/test_prod_op.py b/python/paddle/fluid/tests/unittests/test_prod_op.py index cd754f7dbaf..24605da11e8 100644 --- a/python/paddle/fluid/tests/unittests/test_prod_op.py +++ b/python/paddle/fluid/tests/unittests/test_prod_op.py @@ -19,7 +19,6 @@ from test_sum_op import TestReduceOPTensorAxisBase class TestProdOp(unittest.TestCase): - def setUp(self): self.input = np.random.random(size=(10, 10, 5)).astype(np.float32) @@ -27,54 +26,52 @@ class TestProdOp(unittest.TestCase): input = paddle.to_tensor(self.input) dy_result = paddle.prod(input) expected_result = np.prod(self.input) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=1) expected_result = np.prod(self.input, axis=1) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=-1) expected_result = np.prod(self.input, axis=-1) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=[0, 1]) expected_result = np.prod(self.input, axis=(0, 1)) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05, atol=1e-8 + ) dy_result = paddle.prod(input, axis=1, keepdim=True) expected_result = np.prod(self.input, axis=1, keepdims=True) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=1, dtype='int64') expected_result = np.prod(self.input, axis=1, dtype=np.int64) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=1, keepdim=True, dtype='int64') - expected_result = np.prod(self.input, - axis=1, - keepdims=True, - dtype=np.int64) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + expected_result = np.prod( + self.input, axis=1, keepdims=True, dtype=np.int64 + ) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) def run_static(self, use_gpu=False): - input = paddle.fluid.data(name='input', - shape=[10, 10, 5], - dtype='float32') + input = paddle.fluid.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) result0 = paddle.prod(input) result1 = paddle.prod(input, axis=1) result2 = paddle.prod(input, axis=-1) @@ -86,44 +83,49 @@ class TestProdOp(unittest.TestCase): place = paddle.CUDAPlace(0) if use_gpu else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) - static_result = exe.run(feed={"input": self.input}, - fetch_list=[ - result0, result1, result2, result3, result4, - result5, result6 - ]) + static_result = exe.run( + feed={"input": self.input}, + fetch_list=[ + result0, + result1, + result2, + result3, + result4, + result5, + result6, + ], + ) expected_result = np.prod(self.input) - np.testing.assert_allclose(static_result[0], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[0], expected_result, rtol=1e-05 + ) expected_result = np.prod(self.input, axis=1) - np.testing.assert_allclose(static_result[1], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[1], expected_result, rtol=1e-05 + ) expected_result = np.prod(self.input, axis=-1) - np.testing.assert_allclose(static_result[2], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[2], expected_result, rtol=1e-05 + ) expected_result = np.prod(self.input, axis=(0, 1)) - np.testing.assert_allclose(static_result[3], - expected_result, - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + static_result[3], expected_result, rtol=1e-05, atol=1e-8 + ) expected_result = np.prod(self.input, axis=1, keepdims=True) - np.testing.assert_allclose(static_result[4], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[4], expected_result, rtol=1e-05 + ) expected_result = np.prod(self.input, axis=1, dtype=np.int64) - np.testing.assert_allclose(static_result[5], - expected_result, - rtol=1e-05) - expected_result = np.prod(self.input, - axis=1, - keepdims=True, - dtype=np.int64) - np.testing.assert_allclose(static_result[6], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[5], expected_result, rtol=1e-05 + ) + expected_result = np.prod( + self.input, axis=1, keepdims=True, dtype=np.int64 + ) + np.testing.assert_allclose( + static_result[6], expected_result, rtol=1e-05 + ) def test_cpu(self): paddle.disable_static(place=paddle.CPUPlace()) @@ -146,14 +148,14 @@ class TestProdOp(unittest.TestCase): class TestProdOpError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[2, 2, 4], dtype='float32') - bool_x = paddle.fluid.data(name='bool_x', - shape=[2, 2, 4], - dtype='bool') + bool_x = paddle.fluid.data( + name='bool_x', shape=[2, 2, 4], dtype='bool' + ) # The argument x shoule be a Tensor self.assertRaises(TypeError, paddle.prod, [1]) @@ -168,7 +170,6 @@ class TestProdOpError(unittest.TestCase): class TestProdWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.prod self.np_api = np.prod @@ -178,7 +179,6 @@ class TestProdWithTensorAxis1(TestReduceOPTensorAxisBase): class TestProdWithTensorAxis2(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.prod self.np_api = np.prod @@ -187,7 +187,7 @@ class TestProdWithTensorAxis2(TestReduceOPTensorAxisBase): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] diff --git a/python/paddle/fluid/tests/unittests/test_profiler.py b/python/paddle/fluid/tests/unittests/test_profiler.py index 2b630ed3840..b7dd11f7b8a 100644 --- a/python/paddle/fluid/tests/unittests/test_profiler.py +++ b/python/paddle/fluid/tests/unittests/test_profiler.py @@ -26,7 +26,6 @@ import paddle.fluid.proto.profiler.profiler_pb2 as profiler_pb2 class TestProfiler(unittest.TestCase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -38,9 +37,9 @@ class TestProfiler(unittest.TestCase): image = fluid.layers.data(name='x', shape=[784], dtype='float32') hidden1 = fluid.layers.fc(input=image, size=64, act='relu') i = layers.zeros(shape=[1], dtype='int64') - counter = fluid.layers.zeros(shape=[1], - dtype='int64', - force_cpu=True) + counter = fluid.layers.zeros( + shape=[1], dtype='int64', force_cpu=True + ) until = layers.fill_constant([1], dtype='int64', value=10) data_arr = layers.array_write(hidden1, i) cond = fluid.layers.less_than(x=counter, y=until) @@ -58,9 +57,9 @@ class TestProfiler(unittest.TestCase): cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(cost) batch_size = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size + ) optimizer = fluid.optimizer.Momentum(learning_rate=0.001, momentum=0.9) opts = optimizer.minimize(avg_cost, startup_program=startup_program) @@ -71,8 +70,10 @@ class TestProfiler(unittest.TestCase): exec_strategy = fluid.ExecutionStrategy() exec_strategy.num_threads = 1 train_program = fluid.compiler.CompiledProgram( - main_program).with_data_parallel(loss_name=avg_cost.name, - exec_strategy=exec_strategy) + main_program + ).with_data_parallel( + loss_name=avg_cost.name, exec_strategy=exec_strategy + ) else: train_program = main_program return train_program, startup_program, avg_cost, batch_size, batch_acc @@ -84,41 +85,48 @@ class TestProfiler(unittest.TestCase): def check_profile_result(self, profile_path): data = open(profile_path, 'rb').read() - if (len(data) > 0): + if len(data) > 0: profile_pb = profiler_pb2.Profile() profile_pb.ParseFromString(data) self.assertGreater(len(profile_pb.events), 0) for event in profile_pb.events: if event.type == profiler_pb2.Event.GPUKernel: if not event.detail_info and not event.name.startswith( - "MEM"): + "MEM" + ): raise Exception( "Kernel %s missing event. Has this kernel been recorded by RecordEvent?" - % event.name) + % event.name + ) elif event.type == profiler_pb2.Event.CPU and ( - event.name.startswith("Driver API") - or event.name.startswith("Runtime API")): + event.name.startswith("Driver API") + or event.name.startswith("Runtime API") + ): print("Warning: unregister", event.name) def run_iter(self, exe, main_program, fetch_list): x = np.random.random((32, 784)).astype("float32") y = np.random.randint(0, 10, (32, 1)).astype("int64") - outs = exe.run(main_program, - feed={ - 'x': x, - 'y': y - }, - fetch_list=fetch_list) - - def net_profiler(self, - exe, - state, - tracer_option, - batch_range=None, - use_parallel_executor=False, - use_new_api=False): - main_program, startup_program, avg_cost, batch_size, batch_acc = self.build_program( - compile_program=use_parallel_executor) + outs = exe.run( + main_program, feed={'x': x, 'y': y}, fetch_list=fetch_list + ) + + def net_profiler( + self, + exe, + state, + tracer_option, + batch_range=None, + use_parallel_executor=False, + use_new_api=False, + ): + ( + main_program, + startup_program, + avg_cost, + batch_size, + batch_acc, + ) = self.build_program(compile_program=use_parallel_executor) exe.run(startup_program) profile_path = self.get_profile_path() @@ -127,22 +135,26 @@ class TestProfiler(unittest.TestCase): for iter in range(10): if iter == 2: profiler.reset_profiler() - self.run_iter(exe, main_program, - [avg_cost, batch_acc, batch_size]) + self.run_iter( + exe, main_program, [avg_cost, batch_acc, batch_size] + ) else: options = utils.ProfilerOptions( options={ 'state': state, 'sorted_key': 'total', 'tracer_level': tracer_option, - 'batch_range': - [0, 10] if batch_range is None else batch_range, - 'profile_path': profile_path - }) + 'batch_range': [0, 10] + if batch_range is None + else batch_range, + 'profile_path': profile_path, + } + ) with utils.Profiler(enabled=True, options=options) as prof: for iter in range(10): - self.run_iter(exe, main_program, - [avg_cost, batch_acc, batch_size]) + self.run_iter( + exe, main_program, [avg_cost, batch_acc, batch_size] + ) utils.get_profiler().record_step() if batch_range is None and iter == 2: utils.get_profiler().reset() @@ -153,37 +165,44 @@ class TestProfiler(unittest.TestCase): def test_cpu_profiler(self): exe = fluid.Executor(fluid.CPUPlace()) for use_new_api in [False, True]: - self.net_profiler(exe, - 'CPU', - "Default", - batch_range=[5, 10], - use_new_api=use_new_api) - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "profiler is enabled only with GPU") + self.net_profiler( + exe, + 'CPU', + "Default", + batch_range=[5, 10], + use_new_api=use_new_api, + ) + + @unittest.skipIf( + not core.is_compiled_with_cuda(), "profiler is enabled only with GPU" + ) def test_cuda_profiler(self): exe = fluid.Executor(fluid.CUDAPlace(0)) for use_new_api in [False, True]: - self.net_profiler(exe, - 'GPU', - "OpDetail", - batch_range=[0, 10], - use_new_api=use_new_api) - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "profiler is enabled only with GPU") + self.net_profiler( + exe, + 'GPU', + "OpDetail", + batch_range=[0, 10], + use_new_api=use_new_api, + ) + + @unittest.skipIf( + not core.is_compiled_with_cuda(), "profiler is enabled only with GPU" + ) def test_all_profiler(self): exe = fluid.Executor(fluid.CUDAPlace(0)) for use_new_api in [False, True]: - self.net_profiler(exe, - 'All', - "AllOpDetail", - batch_range=None, - use_new_api=use_new_api) + self.net_profiler( + exe, + 'All', + "AllOpDetail", + batch_range=None, + use_new_api=use_new_api, + ) class TestProfilerAPIError(unittest.TestCase): - def test_errors(self): options = utils.ProfilerOptions() self.assertTrue(options['profile_path'] is None) diff --git a/python/paddle/fluid/tests/unittests/test_profiler_statistic.py b/python/paddle/fluid/tests/unittests/test_profiler_statistic.py index e6e7a9c8b62..0483f4995d9 100644 --- a/python/paddle/fluid/tests/unittests/test_profiler_statistic.py +++ b/python/paddle/fluid/tests/unittests/test_profiler_statistic.py @@ -19,7 +19,6 @@ import paddle.profiler.profiler_statistic as profiler_statistic class HostPythonNode: - def __init__(self, name, type, start_ns, end_ns, process_id, thread_id): self.name = name self.type = type @@ -34,9 +33,9 @@ class HostPythonNode: class DevicePythonNode: - - def __init__(self, name, type, start_ns, end_ns, device_id, context_id, - stream_id): + def __init__( + self, name, type, start_ns, end_ns, device_id, context_id, stream_id + ): self.name = name self.type = type self.start_ns = start_ns @@ -47,8 +46,20 @@ class DevicePythonNode: class MemPythonNode: - def __init__(self, timestamp_ns, addr, type, process_id, thread_id, increase_bytes, place, current_allocated, \ - current_reserved, peak_allocated, peak_reserved): + def __init__( + self, + timestamp_ns, + addr, + type, + process_id, + thread_id, + increase_bytes, + place, + current_allocated, + current_reserved, + peak_allocated, + peak_reserved, + ): self.timestamp_ns = timestamp_ns self.addr = addr self.type = type @@ -63,155 +74,308 @@ class MemPythonNode: class TestProfilerStatistic(unittest.TestCase): - def test_statistic_case1(self): - root_node = HostPythonNode('Root Node', - profiler.TracerEventType.UserDefined, 0, - float('inf'), 1000, 1001) - profilerstep_node = HostPythonNode('ProfileStep#1', - profiler.TracerEventType.ProfileStep, - 0, 400, 1000, 1001) - dataloader_node = HostPythonNode('Dataloader', - profiler.TracerEventType.Dataloader, 5, - 15, 1000, 1001) - mobilenet_node = HostPythonNode('MobileNet', - profiler.TracerEventType.Forward, 20, - 50, 1000, 1001) - yolonet_node = HostPythonNode('Yolov3Net', - profiler.TracerEventType.Forward, 50, 110, - 1000, 1001) + root_node = HostPythonNode( + 'Root Node', + profiler.TracerEventType.UserDefined, + 0, + float('inf'), + 1000, + 1001, + ) + profilerstep_node = HostPythonNode( + 'ProfileStep#1', + profiler.TracerEventType.ProfileStep, + 0, + 400, + 1000, + 1001, + ) + dataloader_node = HostPythonNode( + 'Dataloader', profiler.TracerEventType.Dataloader, 5, 15, 1000, 1001 + ) + mobilenet_node = HostPythonNode( + 'MobileNet', profiler.TracerEventType.Forward, 20, 50, 1000, 1001 + ) + yolonet_node = HostPythonNode( + 'Yolov3Net', profiler.TracerEventType.Forward, 50, 110, 1000, 1001 + ) userdefined_node = HostPythonNode( - 'Communication Time', profiler.TracerEventType.PythonUserDefined, - 100, 110, 1000, 1001) + 'Communication Time', + profiler.TracerEventType.PythonUserDefined, + 100, + 110, + 1000, + 1001, + ) communication_node = HostPythonNode( - 'Communication', profiler.TracerEventType.Communication, 105, 110, - 1000, 1001) - backward_node = HostPythonNode('Gradient Backward', - profiler.TracerEventType.Backward, 120, - 200, 1000, 1001) + 'Communication', + profiler.TracerEventType.Communication, + 105, + 110, + 1000, + 1001, + ) + backward_node = HostPythonNode( + 'Gradient Backward', + profiler.TracerEventType.Backward, + 120, + 200, + 1000, + 1001, + ) optimization_node = HostPythonNode( - 'Optimization', profiler.TracerEventType.Optimization, 220, 300, - 1000, 1001) - conv2d_node = HostPythonNode('conv2d', - profiler.TracerEventType.Operator, 25, 40, - 1000, 1001) - sync_batch_norm_node = HostPythonNode('sync_batch_norm', - profiler.TracerEventType.Operator, - 60, 100, 1000, 1001) + 'Optimization', + profiler.TracerEventType.Optimization, + 220, + 300, + 1000, + 1001, + ) + conv2d_node = HostPythonNode( + 'conv2d', profiler.TracerEventType.Operator, 25, 40, 1000, 1001 + ) + sync_batch_norm_node = HostPythonNode( + 'sync_batch_norm', + profiler.TracerEventType.Operator, + 60, + 100, + 1000, + 1001, + ) conv2d_infer_shape = HostPythonNode( - 'conv2d::infer_shape', profiler.TracerEventType.OperatorInner, 25, - 30, 1000, 1001) - conv2d_compute = HostPythonNode('conv2d::compute', - profiler.TracerEventType.OperatorInner, - 30, 40, 1000, 1001) + 'conv2d::infer_shape', + profiler.TracerEventType.OperatorInner, + 25, + 30, + 1000, + 1001, + ) + conv2d_compute = HostPythonNode( + 'conv2d::compute', + profiler.TracerEventType.OperatorInner, + 30, + 40, + 1000, + 1001, + ) conv2d_compute.mem_node.append( - MemPythonNode(33, 0, profiler_statistic.TracerMemEventType.Allocate, - 1000, 1001, 20, 'place(gpu:0)', 200, 200, 800, 800)) + MemPythonNode( + 33, + 0, + profiler_statistic.TracerMemEventType.Allocate, + 1000, + 1001, + 20, + 'place(gpu:0)', + 200, + 200, + 800, + 800, + ) + ) conv2d_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 30, 35, - 1000, 1001) - conv2d_MemCpy = HostPythonNode('AsyncMemcpy', - profiler.TracerEventType.UserDefined, 35, - 40, 1000, 1001) - conv2d_cudaMemCpy = HostPythonNode('cudaMemcpy', - profiler.TracerEventType.CudaRuntime, - 35, 40, 1000, 1001) - conv2d_kernel = DevicePythonNode('conv2d_kernel', - profiler.TracerEventType.Kernel, 35, - 50, 0, 0, 0) - conv2d_memcpy = DevicePythonNode('conv2d_memcpy', - profiler.TracerEventType.Memcpy, 50, - 60, 0, 0, 0) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 30, + 35, + 1000, + 1001, + ) + conv2d_MemCpy = HostPythonNode( + 'AsyncMemcpy', + profiler.TracerEventType.UserDefined, + 35, + 40, + 1000, + 1001, + ) + conv2d_cudaMemCpy = HostPythonNode( + 'cudaMemcpy', + profiler.TracerEventType.CudaRuntime, + 35, + 40, + 1000, + 1001, + ) + conv2d_kernel = DevicePythonNode( + 'conv2d_kernel', profiler.TracerEventType.Kernel, 35, 50, 0, 0, 0 + ) + conv2d_memcpy = DevicePythonNode( + 'conv2d_memcpy', profiler.TracerEventType.Memcpy, 50, 60, 0, 0, 0 + ) sync_batch_norm_infer_shape = HostPythonNode( 'sync_batch_norm::infer_shape', - profiler.TracerEventType.OperatorInner, 60, 70, 1000, 1001) + profiler.TracerEventType.OperatorInner, + 60, + 70, + 1000, + 1001, + ) sync_batch_norm_compute = HostPythonNode( - 'sync_batch_norm::compute', profiler.TracerEventType.OperatorInner, - 80, 100, 1000, 1001) + 'sync_batch_norm::compute', + profiler.TracerEventType.OperatorInner, + 80, + 100, + 1000, + 1001, + ) sync_batch_norm_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 80, 90, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 80, + 90, + 1000, + 1001, + ) sync_batch_norm_MemCpy = HostPythonNode( - 'AsyncMemcpy', profiler.TracerEventType.UserDefined, 90, 100, 1000, - 1001) + 'AsyncMemcpy', + profiler.TracerEventType.UserDefined, + 90, + 100, + 1000, + 1001, + ) sync_batch_norm_cudaMemCpy = HostPythonNode( - 'cudaMemcpy', profiler.TracerEventType.CudaRuntime, 90, 100, 1000, - 1001) + 'cudaMemcpy', + profiler.TracerEventType.CudaRuntime, + 90, + 100, + 1000, + 1001, + ) sync_batch_norm_kernel = DevicePythonNode( - 'sync_batch_norm_kernel', profiler.TracerEventType.Kernel, 95, 155, - 0, 0, 0) + 'sync_batch_norm_kernel', + profiler.TracerEventType.Kernel, + 95, + 155, + 0, + 0, + 0, + ) sync_batch_norm_memcpy = DevicePythonNode( - 'sync_batch_norm_memcpy', profiler.TracerEventType.Memcpy, 150, 200, - 0, 0, 1) + 'sync_batch_norm_memcpy', + profiler.TracerEventType.Memcpy, + 150, + 200, + 0, + 0, + 1, + ) root_node.children_node.append(profilerstep_node) - profilerstep_node.children_node.extend([ - dataloader_node, mobilenet_node, yolonet_node, backward_node, - optimization_node - ]) + profilerstep_node.children_node.extend( + [ + dataloader_node, + mobilenet_node, + yolonet_node, + backward_node, + optimization_node, + ] + ) mobilenet_node.children_node.append(conv2d_node) yolonet_node.children_node.extend( - [sync_batch_norm_node, userdefined_node]) + [sync_batch_norm_node, userdefined_node] + ) userdefined_node.children_node.append(communication_node) conv2d_node.children_node.extend( - [conv2d_infer_shape, conv2d_compute, conv2d_MemCpy]) + [conv2d_infer_shape, conv2d_compute, conv2d_MemCpy] + ) conv2d_compute.runtime_node.append(conv2d_launchkernel) conv2d_MemCpy.runtime_node.append(conv2d_cudaMemCpy) conv2d_launchkernel.device_node.append(conv2d_kernel) conv2d_cudaMemCpy.device_node.append(conv2d_memcpy) - sync_batch_norm_node.children_node.extend([ - sync_batch_norm_infer_shape, sync_batch_norm_compute, - sync_batch_norm_MemCpy - ]) + sync_batch_norm_node.children_node.extend( + [ + sync_batch_norm_infer_shape, + sync_batch_norm_compute, + sync_batch_norm_MemCpy, + ] + ) sync_batch_norm_compute.runtime_node.append( - sync_batch_norm_launchkernel) + sync_batch_norm_launchkernel + ) sync_batch_norm_MemCpy.runtime_node.append(sync_batch_norm_cudaMemCpy) sync_batch_norm_launchkernel.device_node.append(sync_batch_norm_kernel) sync_batch_norm_cudaMemCpy.device_node.append(sync_batch_norm_memcpy) thread_tree = {'thread1001': root_node} extra_info = { 'Process Cpu Utilization': '1.02', - 'System Cpu Utilization': '0.68' + 'System Cpu Utilization': '0.68', } statistic_data = profiler.profiler_statistic.StatisticData( - thread_tree, extra_info) + thread_tree, extra_info + ) time_range_summary = statistic_data.time_range_summary event_summary = statistic_data.event_summary self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.ProfileStep), 400) + profiler.TracerEventType.ProfileStep + ), + 400, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Forward), 90) + profiler.TracerEventType.Forward + ), + 90, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Backward), 80) + profiler.TracerEventType.Backward + ), + 80, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Optimization), 80) + profiler.TracerEventType.Optimization + ), + 80, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Operator), 55) + profiler.TracerEventType.Operator + ), + 55, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.OperatorInner), 45) + profiler.TracerEventType.OperatorInner + ), + 45, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.CudaRuntime), 30) + profiler.TracerEventType.CudaRuntime + ), + 30, + ) self.assertEqual( time_range_summary.get_gpu_range_sum( - 0, profiler.TracerEventType.Kernel), 75) + 0, profiler.TracerEventType.Kernel + ), + 75, + ) self.assertEqual( time_range_summary.get_gpu_range_sum( - 0, profiler.TracerEventType.Memcpy), 60) + 0, profiler.TracerEventType.Memcpy + ), + 60, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.UserDefined), 15) + profiler.TracerEventType.UserDefined + ), + 15, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Communication), 5) + profiler.TracerEventType.Communication + ), + 5, + ) self.assertEqual(len(event_summary.items), 2) self.assertEqual(len(event_summary.userdefined_items), 1) self.assertEqual(len(event_summary.model_perspective_items), 5) @@ -219,171 +383,347 @@ class TestProfilerStatistic(unittest.TestCase): self.assertEqual(event_summary.items['conv2d'].cpu_time, 15) self.assertEqual(event_summary.items['conv2d'].general_gpu_time, 25) self.assertEqual( - event_summary.model_perspective_items['Forward'].cpu_time, 90) + event_summary.model_perspective_items['Forward'].cpu_time, 90 + ) self.assertEqual( event_summary.model_perspective_items['Forward'].general_gpu_time, - 135) + 135, + ) self.assertEqual( event_summary.model_perspective_items['Backward'].general_gpu_time, - 0) + 0, + ) self.assertEqual( - event_summary.memory_manipulation_items['AsyncMemcpy'].cpu_time, 15) + event_summary.memory_manipulation_items['AsyncMemcpy'].cpu_time, 15 + ) self.assertEqual( - event_summary.memory_manipulation_items['AsyncMemcpy']. - general_gpu_time, 60) + event_summary.memory_manipulation_items[ + 'AsyncMemcpy' + ].general_gpu_time, + 60, + ) self.assertEqual( - statistic_data.memory_summary.allocated_items['place(gpu:0)'] - ['conv2d'].allocation_count, 1) + statistic_data.memory_summary.allocated_items['place(gpu:0)'][ + 'conv2d' + ].allocation_count, + 1, + ) self.assertEqual( - statistic_data.memory_summary.allocated_items['place(gpu:0)'] - ['conv2d'].allocation_size, 20) + statistic_data.memory_summary.allocated_items['place(gpu:0)'][ + 'conv2d' + ].allocation_size, + 20, + ) self.assertEqual( - statistic_data.memory_summary.allocated_items['place(gpu:0)'] - ['conv2d'].increase_size, 20) + statistic_data.memory_summary.allocated_items['place(gpu:0)'][ + 'conv2d' + ].increase_size, + 20, + ) self.assertEqual( - statistic_data.memory_summary.allocated_items['place(gpu:0)'] - ['conv2d'].increase_size, 20) + statistic_data.memory_summary.allocated_items['place(gpu:0)'][ + 'conv2d' + ].increase_size, + 20, + ) self.assertEqual( - statistic_data.memory_summary. - peak_allocation_values['place(gpu:0)'], 800) + statistic_data.memory_summary.peak_allocation_values[ + 'place(gpu:0)' + ], + 800, + ) self.assertEqual( statistic_data.memory_summary.peak_reserved_values['place(gpu:0)'], - 800) + 800, + ) print( profiler.profiler_statistic._build_table( statistic_data, sorted_by=profiler.SortedKeys.CPUTotal, op_detail=True, thread_sep=False, - time_unit='ms')) + time_unit='ms', + ) + ) def test_statistic_case2(self): - root_node = HostPythonNode('Root Node', - profiler.TracerEventType.UserDefined, 0, - float('inf'), 1000, 1001) - profilerstep_node = HostPythonNode('ProfileStep#1', - profiler.TracerEventType.ProfileStep, - 0, 400, 1000, 1001) - - dataloader_node = HostPythonNode('Dataloader', - profiler.TracerEventType.Dataloader, 5, - 15, 1000, 1001) - - mobilenet_node = HostPythonNode('MobileNet', - profiler.TracerEventType.Forward, 20, - 50, 1000, 1001) - yolonet_node = HostPythonNode('Yolov3Net', - profiler.TracerEventType.Forward, 50, 110, - 1000, 1001) + root_node = HostPythonNode( + 'Root Node', + profiler.TracerEventType.UserDefined, + 0, + float('inf'), + 1000, + 1001, + ) + profilerstep_node = HostPythonNode( + 'ProfileStep#1', + profiler.TracerEventType.ProfileStep, + 0, + 400, + 1000, + 1001, + ) + + dataloader_node = HostPythonNode( + 'Dataloader', profiler.TracerEventType.Dataloader, 5, 15, 1000, 1001 + ) + + mobilenet_node = HostPythonNode( + 'MobileNet', profiler.TracerEventType.Forward, 20, 50, 1000, 1001 + ) + yolonet_node = HostPythonNode( + 'Yolov3Net', profiler.TracerEventType.Forward, 50, 110, 1000, 1001 + ) userdefined_node = HostPythonNode( - 'Communication Time', profiler.TracerEventType.PythonUserDefined, - 100, 110, 1000, 1001) + 'Communication Time', + profiler.TracerEventType.PythonUserDefined, + 100, + 110, + 1000, + 1001, + ) allreduce_launchkernel0 = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 102, 104, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 102, + 104, + 1000, + 1001, + ) nccl_allreduce_kernel0 = DevicePythonNode( - 'nccl_allreduce_kernel', profiler.TracerEventType.Kernel, 105, 120, - 0, 0, 2) + 'nccl_allreduce_kernel', + profiler.TracerEventType.Kernel, + 105, + 120, + 0, + 0, + 2, + ) communication_node = HostPythonNode( - 'Communication', profiler.TracerEventType.Communication, 105, 110, - 1000, 1001) + 'Communication', + profiler.TracerEventType.Communication, + 105, + 110, + 1000, + 1001, + ) - allreduce_op1 = HostPythonNode('allreduce_op1', - profiler.TracerEventType.Operator, 105, - 108, 1000, 1001) + allreduce_op1 = HostPythonNode( + 'allreduce_op1', + profiler.TracerEventType.Operator, + 105, + 108, + 1000, + 1001, + ) allreduce_op1_infershape = HostPythonNode( - 'allreduce_op1::infershape', profiler.TracerEventType.OperatorInner, - 105, 106, 1000, 1001) + 'allreduce_op1::infershape', + profiler.TracerEventType.OperatorInner, + 105, + 106, + 1000, + 1001, + ) allreduce_launchkernel1 = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 106, 107, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 106, + 107, + 1000, + 1001, + ) nccl_allreduce_kernel1 = DevicePythonNode( - 'nccl_allreduce_kernel', profiler.TracerEventType.Kernel, 130, 150, - 0, 0, 2) + 'nccl_allreduce_kernel', + profiler.TracerEventType.Kernel, + 130, + 150, + 0, + 0, + 2, + ) - backward_node = HostPythonNode('Gradient Backward', - profiler.TracerEventType.Backward, 120, - 200, 1000, 1001) + backward_node = HostPythonNode( + 'Gradient Backward', + profiler.TracerEventType.Backward, + 120, + 200, + 1000, + 1001, + ) optimization_node = HostPythonNode( - 'Optimization', profiler.TracerEventType.Optimization, 220, 300, - 1000, 1001) - conv2d_node = HostPythonNode('conv2d', - profiler.TracerEventType.Operator, 25, 40, - 1000, 1001) - sync_batch_norm_node = HostPythonNode('sync_batch_norm', - profiler.TracerEventType.Operator, - 60, 100, 1000, 1001) + 'Optimization', + profiler.TracerEventType.Optimization, + 220, + 300, + 1000, + 1001, + ) + conv2d_node = HostPythonNode( + 'conv2d', profiler.TracerEventType.Operator, 25, 40, 1000, 1001 + ) + sync_batch_norm_node = HostPythonNode( + 'sync_batch_norm', + profiler.TracerEventType.Operator, + 60, + 100, + 1000, + 1001, + ) conv2d_infer_shape = HostPythonNode( - 'conv2d::infer_shape', profiler.TracerEventType.OperatorInner, 25, - 30, 1000, 1001) - conv2d_compute = HostPythonNode('conv2d::compute', - profiler.TracerEventType.OperatorInner, - 30, 40, 1000, 1001) + 'conv2d::infer_shape', + profiler.TracerEventType.OperatorInner, + 25, + 30, + 1000, + 1001, + ) + conv2d_compute = HostPythonNode( + 'conv2d::compute', + profiler.TracerEventType.OperatorInner, + 30, + 40, + 1000, + 1001, + ) conv2d_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 30, 35, - 1000, 1001) - conv2d_MemCpy = HostPythonNode('AsyncMemcpy', - profiler.TracerEventType.UserDefined, 35, - 40, 1000, 1001) - conv2d_cudaMemCpy = HostPythonNode('cudaMemcpy', - profiler.TracerEventType.CudaRuntime, - 35, 40, 1000, 1001) - conv2d_kernel = DevicePythonNode('conv2d_kernel', - profiler.TracerEventType.Kernel, 35, - 50, 0, 0, 0) - conv2d_memcpy = DevicePythonNode('conv2d_memcpy', - profiler.TracerEventType.Memcpy, 50, - 60, 0, 0, 0) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 30, + 35, + 1000, + 1001, + ) + conv2d_MemCpy = HostPythonNode( + 'AsyncMemcpy', + profiler.TracerEventType.UserDefined, + 35, + 40, + 1000, + 1001, + ) + conv2d_cudaMemCpy = HostPythonNode( + 'cudaMemcpy', + profiler.TracerEventType.CudaRuntime, + 35, + 40, + 1000, + 1001, + ) + conv2d_kernel = DevicePythonNode( + 'conv2d_kernel', profiler.TracerEventType.Kernel, 35, 50, 0, 0, 0 + ) + conv2d_memcpy = DevicePythonNode( + 'conv2d_memcpy', profiler.TracerEventType.Memcpy, 50, 60, 0, 0, 0 + ) sync_batch_norm_infer_shape = HostPythonNode( 'sync_batch_norm::infer_shape', - profiler.TracerEventType.OperatorInner, 60, 70, 1000, 1001) + profiler.TracerEventType.OperatorInner, + 60, + 70, + 1000, + 1001, + ) sync_batch_norm_compute = HostPythonNode( - 'sync_batch_norm::compute', profiler.TracerEventType.OperatorInner, - 80, 100, 1000, 1001) + 'sync_batch_norm::compute', + profiler.TracerEventType.OperatorInner, + 80, + 100, + 1000, + 1001, + ) sync_batch_norm_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 80, 90, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 80, + 90, + 1000, + 1001, + ) sync_batch_norm_MemCpy = HostPythonNode( - 'AsyncMemcpy', profiler.TracerEventType.UserDefined, 90, 100, 1000, - 1001) + 'AsyncMemcpy', + profiler.TracerEventType.UserDefined, + 90, + 100, + 1000, + 1001, + ) sync_batch_norm_cudaMemCpy = HostPythonNode( - 'cudaMemcpy', profiler.TracerEventType.CudaRuntime, 90, 100, 1000, - 1001) + 'cudaMemcpy', + profiler.TracerEventType.CudaRuntime, + 90, + 100, + 1000, + 1001, + ) sync_batch_norm_kernel = DevicePythonNode( - 'sync_batch_norm_kernel', profiler.TracerEventType.Kernel, 95, 300, - 0, 0, 0) + 'sync_batch_norm_kernel', + profiler.TracerEventType.Kernel, + 95, + 300, + 0, + 0, + 0, + ) sync_batch_norm_memcpy = DevicePythonNode( - 'sync_batch_norm_memcpy', profiler.TracerEventType.Memcpy, 150, 200, - 0, 0, 1) + 'sync_batch_norm_memcpy', + profiler.TracerEventType.Memcpy, + 150, + 200, + 0, + 0, + 1, + ) - allreduce_node2 = HostPythonNode('allreduce', - profiler.TracerEventType.Operator, 230, - 250, 1000, 1001) + allreduce_node2 = HostPythonNode( + 'allreduce', profiler.TracerEventType.Operator, 230, 250, 1000, 1001 + ) allreduce_node2_infershape = HostPythonNode( 'allreduce_node2::infershape', - profiler.TracerEventType.OperatorInner, 231, 232, 1000, 1001) + profiler.TracerEventType.OperatorInner, + 231, + 232, + 1000, + 1001, + ) allreduce_launchkernel2 = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 235, 240, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 235, + 240, + 1000, + 1001, + ) nccl_allreduce_kernel2 = DevicePythonNode( - 'nccl_allreduce_kernel', profiler.TracerEventType.Kernel, 250, 280, - 0, 0, 2) + 'nccl_allreduce_kernel', + profiler.TracerEventType.Kernel, + 250, + 280, + 0, + 0, + 2, + ) root_node.children_node.append(profilerstep_node) - profilerstep_node.children_node.extend([ - dataloader_node, mobilenet_node, yolonet_node, backward_node, - optimization_node - ]) + profilerstep_node.children_node.extend( + [ + dataloader_node, + mobilenet_node, + yolonet_node, + backward_node, + optimization_node, + ] + ) mobilenet_node.children_node.append(conv2d_node) yolonet_node.children_node.extend( - [sync_batch_norm_node, userdefined_node]) + [sync_batch_norm_node, userdefined_node] + ) userdefined_node.children_node.append(communication_node) userdefined_node.runtime_node.append(allreduce_launchkernel0) allreduce_launchkernel0.device_node.append(nccl_allreduce_kernel0) @@ -392,17 +732,22 @@ class TestProfilerStatistic(unittest.TestCase): allreduce_op1.runtime_node.append(allreduce_launchkernel1) allreduce_launchkernel1.device_node.append(nccl_allreduce_kernel1) conv2d_node.children_node.extend( - [conv2d_infer_shape, conv2d_compute, conv2d_MemCpy]) + [conv2d_infer_shape, conv2d_compute, conv2d_MemCpy] + ) conv2d_compute.runtime_node.append(conv2d_launchkernel) conv2d_MemCpy.runtime_node.append(conv2d_cudaMemCpy) conv2d_launchkernel.device_node.append(conv2d_kernel) conv2d_cudaMemCpy.device_node.append(conv2d_memcpy) - sync_batch_norm_node.children_node.extend([ - sync_batch_norm_infer_shape, sync_batch_norm_compute, - sync_batch_norm_MemCpy - ]) + sync_batch_norm_node.children_node.extend( + [ + sync_batch_norm_infer_shape, + sync_batch_norm_compute, + sync_batch_norm_MemCpy, + ] + ) sync_batch_norm_compute.runtime_node.append( - sync_batch_norm_launchkernel) + sync_batch_norm_launchkernel + ) sync_batch_norm_MemCpy.runtime_node.append(sync_batch_norm_cudaMemCpy) sync_batch_norm_launchkernel.device_node.append(sync_batch_norm_kernel) sync_batch_norm_cudaMemCpy.device_node.append(sync_batch_norm_memcpy) @@ -413,62 +758,111 @@ class TestProfilerStatistic(unittest.TestCase): thread_tree = {'thread1001': root_node} extra_info = { 'Process Cpu Utilization': '1.02', - 'System Cpu Utilization': '0.68' + 'System Cpu Utilization': '0.68', } statistic_data = profiler.profiler_statistic.StatisticData( - thread_tree, extra_info) + thread_tree, extra_info + ) time_range_summary = statistic_data.time_range_summary event_summary = statistic_data.event_summary distributed_summary = statistic_data.distributed_summary self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.ProfileStep), 400) + profiler.TracerEventType.ProfileStep + ), + 400, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Forward), 90) + profiler.TracerEventType.Forward + ), + 90, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Backward), 80) + profiler.TracerEventType.Backward + ), + 80, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Optimization), 80) + profiler.TracerEventType.Optimization + ), + 80, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Operator), 78) + profiler.TracerEventType.Operator + ), + 78, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.OperatorInner), 47) + profiler.TracerEventType.OperatorInner + ), + 47, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.CudaRuntime), 38) + profiler.TracerEventType.CudaRuntime + ), + 38, + ) self.assertEqual( time_range_summary.get_gpu_range_sum( - 0, profiler.TracerEventType.Kernel), 220) + 0, profiler.TracerEventType.Kernel + ), + 220, + ) self.assertEqual( time_range_summary.get_gpu_range_sum( - 0, profiler.TracerEventType.Memcpy), 60) + 0, profiler.TracerEventType.Memcpy + ), + 60, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.UserDefined), 15) + profiler.TracerEventType.UserDefined + ), + 15, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Communication), 5) + profiler.TracerEventType.Communication + ), + 5, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.cpu_communication_range), 25) + distributed_summary.cpu_communication_range + ), + 25, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.gpu_communication_range), 65) + distributed_summary.gpu_communication_range + ), + 65, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.communication_range), 85) + distributed_summary.communication_range + ), + 85, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.computation_range), 220) + distributed_summary.computation_range + ), + 220, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.overlap_range), 85) + distributed_summary.overlap_range + ), + 85, + ) self.assertEqual(len(event_summary.items), 4) self.assertEqual(len(event_summary.userdefined_items), 1) self.assertEqual(len(event_summary.model_perspective_items), 5) @@ -476,76 +870,136 @@ class TestProfilerStatistic(unittest.TestCase): self.assertEqual(event_summary.items['conv2d'].cpu_time, 15) self.assertEqual(event_summary.items['conv2d'].general_gpu_time, 25) self.assertEqual( - event_summary.model_perspective_items['Forward'].cpu_time, 90) + event_summary.model_perspective_items['Forward'].cpu_time, 90 + ) self.assertEqual( event_summary.model_perspective_items['Forward'].general_gpu_time, - 315) + 315, + ) self.assertEqual( event_summary.model_perspective_items['Backward'].general_gpu_time, - 0) + 0, + ) self.assertEqual( - event_summary.memory_manipulation_items['AsyncMemcpy'].cpu_time, 15) + event_summary.memory_manipulation_items['AsyncMemcpy'].cpu_time, 15 + ) self.assertEqual( - event_summary.memory_manipulation_items['AsyncMemcpy']. - general_gpu_time, 60) + event_summary.memory_manipulation_items[ + 'AsyncMemcpy' + ].general_gpu_time, + 60, + ) print( profiler.profiler_statistic._build_table( statistic_data, sorted_by=profiler.SortedKeys.CPUTotal, op_detail=True, thread_sep=False, - time_unit='ms')) + time_unit='ms', + ) + ) def test_statistic_case3(self): # for coverage, test all time is 0 - root_node = HostPythonNode('Root Node', - profiler.TracerEventType.UserDefined, 0, - float('inf'), 1000, 1001) - profilerstep_node = HostPythonNode('ProfileStep#1', - profiler.TracerEventType.ProfileStep, - 0, 400, 1000, 1001) - dataloader_node = HostPythonNode('Dataloader', - profiler.TracerEventType.Dataloader, 5, - 15, 1000, 1001) - mobilenet_node = HostPythonNode('MobileNet', - profiler.TracerEventType.Forward, 20, - 50, 1000, 1001) - - backward_node = HostPythonNode('Gradient Backward', - profiler.TracerEventType.Backward, 120, - 200, 1000, 1001) + root_node = HostPythonNode( + 'Root Node', + profiler.TracerEventType.UserDefined, + 0, + float('inf'), + 1000, + 1001, + ) + profilerstep_node = HostPythonNode( + 'ProfileStep#1', + profiler.TracerEventType.ProfileStep, + 0, + 400, + 1000, + 1001, + ) + dataloader_node = HostPythonNode( + 'Dataloader', profiler.TracerEventType.Dataloader, 5, 15, 1000, 1001 + ) + mobilenet_node = HostPythonNode( + 'MobileNet', profiler.TracerEventType.Forward, 20, 50, 1000, 1001 + ) + + backward_node = HostPythonNode( + 'Gradient Backward', + profiler.TracerEventType.Backward, + 120, + 200, + 1000, + 1001, + ) optimization_node = HostPythonNode( - 'Optimization', profiler.TracerEventType.Optimization, 220, 300, - 1000, 1001) + 'Optimization', + profiler.TracerEventType.Optimization, + 220, + 300, + 1000, + 1001, + ) userdefined_node = HostPythonNode( - 'Communication Time', profiler.TracerEventType.PythonUserDefined, - 60, 70, 1000, 1001) + 'Communication Time', + profiler.TracerEventType.PythonUserDefined, + 60, + 70, + 1000, + 1001, + ) - conv2d_node = HostPythonNode('conv2d', - profiler.TracerEventType.Operator, 25, 25, - 1000, 1001) + conv2d_node = HostPythonNode( + 'conv2d', profiler.TracerEventType.Operator, 25, 25, 1000, 1001 + ) conv2d_infer_shape = HostPythonNode( - 'conv2d::infer_shape', profiler.TracerEventType.OperatorInner, 25, - 25, 1000, 1001) - conv2d_compute = HostPythonNode('conv2d::compute', - profiler.TracerEventType.OperatorInner, - 25, 25, 1000, 1001) + 'conv2d::infer_shape', + profiler.TracerEventType.OperatorInner, + 25, + 25, + 1000, + 1001, + ) + conv2d_compute = HostPythonNode( + 'conv2d::compute', + profiler.TracerEventType.OperatorInner, + 25, + 25, + 1000, + 1001, + ) conv2d_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 25, 25, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 25, + 25, + 1000, + 1001, + ) - conv2d_kernel = DevicePythonNode('conv2d_kernel', - profiler.TracerEventType.Kernel, 35, - 35, 0, 0, 0) + conv2d_kernel = DevicePythonNode( + 'conv2d_kernel', profiler.TracerEventType.Kernel, 35, 35, 0, 0, 0 + ) another_kernel = DevicePythonNode( 'void phi::funcs::VectorizedBroadcastKernel, phi::funcs::AddFunctor>()', - profiler.TracerEventType.Kernel, 35, 35, 0, 0, 0) + profiler.TracerEventType.Kernel, + 35, + 35, + 0, + 0, + 0, + ) root_node.children_node.append(profilerstep_node) - profilerstep_node.children_node.extend([ - dataloader_node, mobilenet_node, userdefined_node, backward_node, - optimization_node - ]) + profilerstep_node.children_node.extend( + [ + dataloader_node, + mobilenet_node, + userdefined_node, + backward_node, + optimization_node, + ] + ) mobilenet_node.children_node.append(conv2d_node) conv2d_node.children_node.extend([conv2d_infer_shape, conv2d_compute]) conv2d_compute.runtime_node.append(conv2d_launchkernel) @@ -554,30 +1008,41 @@ class TestProfilerStatistic(unittest.TestCase): thread_tree = {'thread1001': root_node} extra_info = { 'Process Cpu Utilization': '1.02', - 'System Cpu Utilization': '0.68' + 'System Cpu Utilization': '0.68', } statistic_data = profiler.profiler_statistic.StatisticData( - thread_tree, extra_info) + thread_tree, extra_info + ) time_range_summary = statistic_data.time_range_summary event_summary = statistic_data.event_summary self.assertEqual(event_summary.items['conv2d'].cpu_time, 0) self.assertEqual(event_summary.items['conv2d'].general_gpu_time, 0) self.assertEqual( - event_summary.userdefined_items['Communication Time']. - general_gpu_time, 0) + event_summary.userdefined_items[ + 'Communication Time' + ].general_gpu_time, + 0, + ) for sort_key in [ - profiler.SortedKeys.CPUTotal, profiler.SortedKeys.CPUMax, - profiler.SortedKeys.CPUMin, profiler.SortedKeys.CPUAvg, - profiler.SortedKeys.GPUTotal, profiler.SortedKeys.GPUMax, - profiler.SortedKeys.GPUMin, profiler.SortedKeys.GPUAvg + profiler.SortedKeys.CPUTotal, + profiler.SortedKeys.CPUMax, + profiler.SortedKeys.CPUMin, + profiler.SortedKeys.CPUAvg, + profiler.SortedKeys.GPUTotal, + profiler.SortedKeys.GPUMax, + profiler.SortedKeys.GPUMin, + profiler.SortedKeys.GPUAvg, ]: print( - profiler.profiler_statistic._build_table(statistic_data, - sorted_by=sort_key, - op_detail=True, - thread_sep=False, - time_unit='ms')) + profiler.profiler_statistic._build_table( + statistic_data, + sorted_by=sort_key, + op_detail=True, + thread_sep=False, + time_unit='ms', + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_program.py b/python/paddle/fluid/tests/unittests/test_program.py index 71c1e9239c0..a83574b1001 100644 --- a/python/paddle/fluid/tests/unittests/test_program.py +++ b/python/paddle/fluid/tests/unittests/test_program.py @@ -23,7 +23,6 @@ main_program = default_main_program() class TestProgram(unittest.TestCase): - def test_program(self): b = main_program.current_block() self.assertEqual(-1, b.parent_idx) @@ -55,20 +54,17 @@ class TestProgram(unittest.TestCase): def test_program_clone(self): prog = Program() - x = prog.global_block().create_var(name='X', - shape=[1000, 784], - dtype='float32') + x = prog.global_block().create_var( + name='X', shape=[1000, 784], dtype='float32' + ) - y = prog.global_block().create_var(name='Y', - shape=[784, 100], - dtype='float32') + y = prog.global_block().create_var( + name='Y', shape=[784, 100], dtype='float32' + ) out = prog.global_block().create_var(name='Out', dtype='float32') - prog.global_block().append_op(type="mul", - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + prog.global_block().append_op( + type="mul", inputs={'X': [x], 'Y': [y]}, outputs={'Out': [out]} + ) # FIXME(yuyang18): We manual compare the output string, since the order # of variable could be changed. @@ -78,20 +74,17 @@ class TestProgram(unittest.TestCase): def test_parse_program_from_string(self): prog = Program() - x = prog.global_block().create_var(name='X', - shape=[1000, 784], - dtype='float32') + x = prog.global_block().create_var( + name='X', shape=[1000, 784], dtype='float32' + ) - y = prog.global_block().create_var(name='Y', - shape=[784, 100], - dtype='float32') + y = prog.global_block().create_var( + name='Y', shape=[784, 100], dtype='float32' + ) out = prog.global_block().create_var(name='Out', dtype='float32') - prog.global_block().append_op(type="mul", - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + prog.global_block().append_op( + type="mul", inputs={'X': [x], 'Y': [y]}, outputs={'Out': [out]} + ) binary_str = prog.desc.serialize_to_string() prog_restored = Program.parse_from_string(binary_str) @@ -111,17 +104,19 @@ class TestProgram(unittest.TestCase): self.assertNotEqual(0, len(new_program.blocks[0].all_parameters())) def test_program_inference_optimize(self): - def net(): - reader = fluid.layers.py_reader(capacity=10, - shapes=[[-1, 10], [-1, 1]], - lod_levels=[0, 0], - dtypes=['float32', 'int64'], - use_double_buffer=True) + reader = fluid.layers.py_reader( + capacity=10, + shapes=[[-1, 10], [-1, 1]], + lod_levels=[0, 0], + dtypes=['float32', 'int64'], + use_double_buffer=True, + ) in_data, label = fluid.layers.read_file(reader) predict_label = fluid.layers.fc(in_data, size=2, act='softmax') loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -132,7 +127,8 @@ class TestProgram(unittest.TestCase): net() no_read_program = main_program._inference_optimize() keep_read_program = main_program._inference_optimize( - prune_read_op=False) + prune_read_op=False + ) no_read_ops = no_read_program.global_block().ops keep_read_ops = keep_read_program.global_block().ops self.assertEqual(len(keep_read_ops) - len(no_read_ops), 2) @@ -158,8 +154,9 @@ class TestProgram(unittest.TestCase): def test_prune_with_input_type_error(self): program = fluid.default_main_program() feed_var_names = [2, 3, 4] - self.assertRaises(ValueError, program._prune_with_input, feed_var_names, - []) + self.assertRaises( + ValueError, program._prune_with_input, feed_var_names, [] + ) def test_random_seed_error(self): program = fluid.default_main_program() @@ -169,21 +166,24 @@ class TestProgram(unittest.TestCase): def test_copy_info_from_error(self): program = fluid.default_main_program() self.assertRaises(TypeError, program._copy_param_info_from, "program") - self.assertRaises(TypeError, program._copy_dist_param_info_from, - "program") + self.assertRaises( + TypeError, program._copy_dist_param_info_from, "program" + ) def test_remove_training_info(self): - def net(): - reader = fluid.layers.py_reader(capacity=10, - shapes=[[-1, 10], [-1, 1]], - lod_levels=[0, 0], - dtypes=['float32', 'int64'], - use_double_buffer=True) + reader = fluid.layers.py_reader( + capacity=10, + shapes=[[-1, 10], [-1, 1]], + lod_levels=[0, 0], + dtypes=['float32', 'int64'], + use_double_buffer=True, + ) in_data, label = fluid.layers.read_file(reader) predict_label = fluid.layers.fc(in_data, size=2, act='softmax') loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -212,7 +212,6 @@ def build_program(): class TestProgramProto(unittest.TestCase): - def test_update_op(self): program = build_program() a = program.desc.serialize_to_string() @@ -241,7 +240,6 @@ class TestProgramProto(unittest.TestCase): class TestProgramHash(unittest.TestCase): - def build_program(self): main_program = paddle.static.Program() startuo_program = paddle.static.Program() @@ -264,8 +262,8 @@ class TestProgramHash(unittest.TestCase): program1, program2 = programs[0], programs[1] # why not write as below? # since the callstack attribute are not equal - #program1 = self.build_program() - #program2 = self.build_program() + # program1 = self.build_program() + # program2 = self.build_program() self.assertTrue(program1.desc.need_update()) self.assertTrue(program2.desc.need_update()) @@ -273,7 +271,8 @@ class TestProgramHash(unittest.TestCase): self.assertFalse(id(program1) == id(program2)) # print(program1, program2) self.assertTrue( - program1.desc.cached_hash_str() == program2.desc.cached_hash_str()) + program1.desc.cached_hash_str() == program2.desc.cached_hash_str() + ) self.assertFalse(program1.desc.need_update()) self.assertFalse(program2.desc.need_update()) @@ -283,8 +282,10 @@ class TestProgramHash(unittest.TestCase): program_clone = program.clone() self.assertFalse(id(program) == id(program_clone)) - self.assertTrue(program.desc.cached_hash_str() == - program_clone.desc.cached_hash_str()) + self.assertTrue( + program.desc.cached_hash_str() + == program_clone.desc.cached_hash_str() + ) def test_program_update(self): program = self.build_program() diff --git a/python/paddle/fluid/tests/unittests/test_program_code.py b/python/paddle/fluid/tests/unittests/test_program_code.py index 8bdb084a5f4..449f97c22f0 100644 --- a/python/paddle/fluid/tests/unittests/test_program_code.py +++ b/python/paddle/fluid/tests/unittests/test_program_code.py @@ -19,25 +19,22 @@ import paddle.fluid.layers as layers class TestProgramToReadableCode(unittest.TestCase): - def setUp(self): self.program = fluid.Program() self.block = self.program.current_block() - self.var = self.block.create_var(name="X", - shape=[-1, 23, 48], - dtype='float32') - self.param = self.block.create_parameter(name="W", - shape=[23, 48], - dtype='float32', - trainable=True) - self.op = self.block.append_op(type="abs", - inputs={"X": [self.var]}, - outputs={"Out": [self.var]}) + self.var = self.block.create_var( + name="X", shape=[-1, 23, 48], dtype='float32' + ) + self.param = self.block.create_parameter( + name="W", shape=[23, 48], dtype='float32', trainable=True + ) + self.op = self.block.append_op( + type="abs", inputs={"X": [self.var]}, outputs={"Out": [self.var]} + ) # add control flow op and sub block self.append_cond_op(self.program) def append_cond_op(self, program): - def true_func(): return layers.fill_constant(shape=[2, 3], dtype='int32', value=2) diff --git a/python/paddle/fluid/tests/unittests/test_program_prune_backward.py b/python/paddle/fluid/tests/unittests/test_program_prune_backward.py index 1fb423bc1f1..ad7617654d2 100755 --- a/python/paddle/fluid/tests/unittests/test_program_prune_backward.py +++ b/python/paddle/fluid/tests/unittests/test_program_prune_backward.py @@ -20,7 +20,11 @@ import paddle.fluid as fluid import paddle.fluid.core as core from simple_nets import init_data, simple_fc_net, fc_with_batchnorm import seresnext_net -from test_parallel_executor_transformer import transformer, get_feed_data_reader, DeviceType +from test_parallel_executor_transformer import ( + transformer, + get_feed_data_reader, + DeviceType, +) from fake_reader import fake_imdb_reader import paddle @@ -32,19 +36,19 @@ def lstm_net(use_feed): hid_dim2 = 96 class_dim = 2 emb_lr = 30.0 - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") emb = fluid.layers.embedding( input=data, size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) + param_attr=fluid.ParamAttr(learning_rate=emb_lr), + ) fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - lstm_h, c = fluid.layers.dynamic_lstm(input=fc0, - size=hid_dim * 4, - is_reverse=False) + lstm_h, c = fluid.layers.dynamic_lstm( + input=fc0, size=hid_dim * 4, is_reverse=False + ) lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') lstm_max_tanh = fluid.layers.tanh(lstm_max) fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') @@ -64,8 +68,10 @@ def simple_fc_net_with_accuracy(use_feed): hidden, size=200, act='relu', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) prediction = fluid.layers.fc(hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) loss = paddle.mean(loss) @@ -90,9 +96,11 @@ def cond_net(use_feed=None): return avg_loss two = fluid.layers.fill_constant([1], 'int32', 2) - pred = (two == 0) - avg_loss = fluid.layers.case([(pred, lambda: loss1(prediction, label))], - lambda: loss2(prediction, label)) + pred = two == 0 + avg_loss = fluid.layers.case( + [(pred, lambda: loss1(prediction, label))], + lambda: loss2(prediction, label), + ) return avg_loss @@ -118,15 +126,15 @@ def optimization_in_cond_net(with_optimize=False): sgd = fluid.optimizer.SGD(learning_rate=0.1) two = fluid.layers.fill_constant([1], 'int32', 2) - pred = (two == 0) + pred = two == 0 avg_loss = fluid.layers.case( [(pred, lambda: loss1(sgd, prediction, label, with_optimize))], - lambda: loss2(sgd, prediction, label, with_optimize)) + lambda: loss2(sgd, prediction, label, with_optimize), + ) return avg_loss class TestProgramPruneBackward(unittest.TestCase): - def program_compare(self, program_a, program_b): assert isinstance( program_a, fluid.framework.Program @@ -142,8 +150,9 @@ class TestProgramPruneBackward(unittest.TestCase): self.assertEqual(len(block_a.ops), len(block_b.ops)) self.assertEqual(len(block_a.vars), len(block_b.vars)) for op_idx in range(len(block_a.ops)): - self.assertEqual(block_a.ops[op_idx].type, - block_b.ops[op_idx].type) + self.assertEqual( + block_a.ops[op_idx].type, block_b.ops[op_idx].type + ) for var_key in list(block_a.vars.keys()): self.assertTrue(block_b.has_var(var_key)) @@ -165,113 +174,111 @@ class TestProgramPruneBackward(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - loss_data_prune, = exe.run(test_prog_prune, - feed=feed_dict, - fetch_list=[loss.name]) - loss_data_orig, = exe.run(test_prog_orig, - feed=feed_dict, - fetch_list=[loss.name]) + (loss_data_prune,) = exe.run( + test_prog_prune, feed=feed_dict, fetch_list=[loss.name] + ) + (loss_data_orig,) = exe.run( + test_prog_orig, feed=feed_dict, fetch_list=[loss.name] + ) self.assertEqual(loss_data_orig, loss_data_prune) def test_simple_fc_net(self): - def optimizer(): optimizer = fluid.optimizer.SGD( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): img, label = init_data() - self.check_prune_correctness(method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, - optimizer=optimizer) + self.check_prune_correctness( + method=simple_fc_net, + feed_dict={"image": img, "label": label}, + optimizer=optimizer, + ) def test_simple_fc_net_with_accuracy(self): - def optimizer(): optimizer = fluid.optimizer.SGD( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): img, label = init_data() - self.check_prune_correctness(method=simple_fc_net_with_accuracy, - feed_dict={ - "image": img, - "label": label - }, - optimizer=optimizer) + self.check_prune_correctness( + method=simple_fc_net_with_accuracy, + feed_dict={"image": img, "label": label}, + optimizer=optimizer, + ) def test_batchnorm_fc(self): - def optimizer(): optimizer = fluid.optimizer.SGD( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): img, label = init_data() - self.check_prune_correctness(method=fc_with_batchnorm, - feed_dict={ - "image": img, - "label": label - }, - optimizer=optimizer) + self.check_prune_correctness( + method=fc_with_batchnorm, + feed_dict={"image": img, "label": label}, + optimizer=optimizer, + ) def test_seresnet(self): with self.program_scope_guard(): self.check_prune_correctness( method=seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device=DeviceType.CPU), - optimizer=seresnext_net.optimizer) + optimizer=seresnext_net.optimizer, + ) def test_transformer(self): - def optimizer(): optimizer = fluid.optimizer.Adam( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): # the program argument is used to distinguish Program and CompiledProgram feed_dict = get_feed_data_reader().get_next( - fluid.Executor(core.CPUPlace()), fluid.default_main_program()) - self.check_prune_correctness(method=transformer, - feed_dict=feed_dict, - optimizer=optimizer) + fluid.Executor(core.CPUPlace()), fluid.default_main_program() + ) + self.check_prune_correctness( + method=transformer, feed_dict=feed_dict, optimizer=optimizer + ) def test_lstm(self): - def optimizer(): optimizer = fluid.optimizer.Adagrad( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): word_dict_size = 5147 reader = fake_imdb_reader(word_dict_size, 1) - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") - feeder = fluid.DataFeeder(feed_list=[data, label], - place=core.CPUPlace()) + feeder = fluid.DataFeeder( + feed_list=[data, label], place=core.CPUPlace() + ) feed_data = feeder.feed(reader()) - self.check_prune_correctness(method=lstm_net, - feed_dict=feed_data, - optimizer=optimizer) + self.check_prune_correctness( + method=lstm_net, feed_dict=feed_data, optimizer=optimizer + ) def test_cond(self): - def optimizer(): optimizer = fluid.optimizer.SGD(learning_rate=0.01) return optimizer @@ -280,9 +287,9 @@ class TestProgramPruneBackward(unittest.TestCase): x_in = np.random.random(size=(10, 4)).astype('float32') label_in = np.random.randint(1, size=(10, 1)).astype('int64') feed_dict = {'x': x_in, 'label': label_in} - self.check_prune_correctness(method=cond_net, - feed_dict=feed_dict, - optimizer=optimizer) + self.check_prune_correctness( + method=cond_net, feed_dict=feed_dict, optimizer=optimizer + ) def test_optimization_in_cond(self): x_in = np.random.random(size=(10, 4)).astype('float32') @@ -295,9 +302,9 @@ class TestProgramPruneBackward(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - loss_data_orig, = exe.run(test_prog_orig, - feed=feed_dict, - fetch_list=[loss.name]) + (loss_data_orig,) = exe.run( + test_prog_orig, feed=feed_dict, fetch_list=[loss.name] + ) with self.program_scope_guard(): loss = optimization_in_cond_net(True) @@ -307,9 +314,9 @@ class TestProgramPruneBackward(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - loss_data_prune, = exe.run(test_prog_prune, - feed=feed_dict, - fetch_list=[loss.name]) + (loss_data_prune,) = exe.run( + test_prog_prune, feed=feed_dict, fetch_list=[loss.name] + ) self.program_compare(test_prog_orig, test_prog_prune) self.assertEqual(loss_data_orig, loss_data_prune) diff --git a/python/paddle/fluid/tests/unittests/test_program_to_string.py b/python/paddle/fluid/tests/unittests/test_program_to_string.py index 8c010b90671..fda96d90467 100644 --- a/python/paddle/fluid/tests/unittests/test_program_to_string.py +++ b/python/paddle/fluid/tests/unittests/test_program_to_string.py @@ -17,17 +17,16 @@ import unittest class TestProgram(unittest.TestCase): - def test_program_to_string(self): prog = fluid.default_main_program() - a = fluid.layers.data(name="X", - shape=[2, 3], - dtype="float32", - append_batch_size=False) + a = fluid.layers.data( + name="X", shape=[2, 3], dtype="float32", append_batch_size=False + ) c = fluid.layers.fc(a, size=3) prog_string = prog.to_string(throw_on_error=True, with_details=False) - prog_string_with_details = prog.to_string(throw_on_error=False, - with_details=True) + prog_string_with_details = prog.to_string( + throw_on_error=False, with_details=True + ) assert prog_string is not None assert len(prog_string_with_details) > len(prog_string) diff --git a/python/paddle/fluid/tests/unittests/test_protobuf.py b/python/paddle/fluid/tests/unittests/test_protobuf.py index 39f32fb1a2d..c3f1fa80185 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf.py @@ -17,7 +17,6 @@ import unittest class TestFrameworkProto(unittest.TestCase): - def test_all(self): op_proto = framework_pb2.OpProto() ipt0 = op_proto.inputs.add() diff --git a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py index 6022bf02602..1099855eec7 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py @@ -18,7 +18,6 @@ from paddle.fluid.framework import Program class TestOpDesc(unittest.TestCase): - def test_op_desc(self): program_desc = core.ProgramDesc() self.assertIsNotNone(program_desc) @@ -79,7 +78,6 @@ class TestOpDesc(unittest.TestCase): class TestProgramDesc(unittest.TestCase): - def test_instance(self): program_desc = core.ProgramDesc() self.assertIsNotNone(program_desc) @@ -107,7 +105,6 @@ class TestProgramDesc(unittest.TestCase): class TestVarDesc(unittest.TestCase): - def test_shape(self): program_desc = core.ProgramDesc() block = program_desc.block(0) @@ -145,8 +142,9 @@ class TestVarDesc(unittest.TestCase): var = block.var(b'my_reader') var.set_type(core.VarDesc.VarType.READER) src_types = [ - core.VarDesc.VarType.INT32, core.VarDesc.VarType.FP64, - core.VarDesc.VarType.FP32 + core.VarDesc.VarType.INT32, + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, ] var.set_dtypes(src_types) self.assertEqual(src_types, var.dtypes()) @@ -164,7 +162,6 @@ class TestVarDesc(unittest.TestCase): class TestBlockDesc(unittest.TestCase): - def test_add_var(self): program_desc = core.ProgramDesc() self.assertIsNotNone(program_desc) diff --git a/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py index fb4dbf32630..3c268958506 100644 --- a/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py @@ -18,7 +18,6 @@ from op_test import OpTest class TestProximalAdagradOp(OpTest): - def setUp(self): self.op_type = "proximal_adagrad" w = np.random.random((102, 105)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py b/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py index bd78a5e8fd9..137594b9a08 100644 --- a/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py +++ b/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py @@ -18,7 +18,6 @@ from op_test import OpTest class TestProximalGDOp(OpTest): - def setUp(self): self.op_type = "proximal_gd" w = np.random.random((102, 105)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_prroi_pool_op.py b/python/paddle/fluid/tests/unittests/test_prroi_pool_op.py index a927c2a5e82..4a953b463be 100644 --- a/python/paddle/fluid/tests/unittests/test_prroi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_prroi_pool_op.py @@ -22,22 +22,24 @@ from paddle.fluid import Program, program_guard class TestPRROIPoolOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() self.prRoIPool = PyPrRoIPool() - self.outs = self.prRoIPool.compute(self.x, self.rois, - self.output_channels, - self.spatial_scale, - self.pooled_height, - self.pooled_width).astype('float32') + self.outs = self.prRoIPool.compute( + self.x, + self.rois, + self.output_channels, + self.spatial_scale, + self.pooled_height, + self.pooled_width, + ).astype('float32') self.inputs = {'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod)} self.attrs = { 'output_channels': self.output_channels, 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs} @@ -63,14 +65,18 @@ class TestPRROIPoolOp(OpTest): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.uniform( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.uniform( - 0, self.height // self.spatial_scale - self.pooled_height) - - x2 = np.random.uniform(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.uniform(y1 + self.pooled_height, - self.height // self.spatial_scale) + 0, self.height // self.spatial_scale - self.pooled_height + ) + + x2 = np.random.uniform( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.uniform( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) @@ -95,24 +101,24 @@ class TestPRROIPoolOp(OpTest): x = fluid.layers.data( name="X", shape=[self.channels, self.height, self.width], - dtype="float32") - rois = fluid.layers.data(name="ROIs", - shape=[4], - dtype="float32", - lod_level=1) + dtype="float32", + ) + rois = fluid.layers.data( + name="ROIs", shape=[4], dtype="float32", lod_level=1 + ) output = fluid.layers.prroi_pool(x, rois, 0.25, 2, 2) loss = paddle.mean(output) optimizer = fluid.optimizer.SGD(learning_rate=1e-3) optimizer.minimize(loss) input_x = fluid.create_lod_tensor(self.x, [], place) - input_rois = fluid.create_lod_tensor(self.rois[:, 1:5], - self.rois_lod, place) + input_rois = fluid.create_lod_tensor( + self.rois[:, 1:5], self.rois_lod, place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - exe.run(fluid.default_main_program(), { - 'X': input_x, - "ROIs": input_rois - }) + exe.run( + fluid.default_main_program(), {'X': input_x, "ROIs": input_rois} + ) def test_net(self): places = [fluid.CPUPlace()] @@ -123,47 +129,51 @@ class TestPRROIPoolOp(OpTest): def test_errors(self): with program_guard(Program(), Program()): - x = fluid.layers.data(name="x", - shape=[245, 30, 30], - dtype="float32") - rois = fluid.layers.data(name="rois", - shape=[4], - dtype="float32", - lod_level=1) + x = fluid.layers.data( + name="x", shape=[245, 30, 30], dtype="float32" + ) + rois = fluid.layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1 + ) # spatial_scale must be float type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 2, 7, - 7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 2, 7, 7 + ) # pooled_height must be int type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 0.25, - 0.7, 7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 0.25, 0.7, 7 + ) # pooled_width must be int type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 0.25, - 7, 0.7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 0.25, 7, 0.7 + ) class TestPRROIPoolOpTensorRoIs(OpTest): - def set_data(self): self.init_test_case() self.make_rois() self.prRoIPool = PyPrRoIPool() - self.outs = self.prRoIPool.compute(self.x, self.rois, - self.output_channels, - self.spatial_scale, - self.pooled_height, - self.pooled_width).astype('float32') + self.outs = self.prRoIPool.compute( + self.x, + self.rois, + self.output_channels, + self.spatial_scale, + self.pooled_height, + self.pooled_width, + ).astype('float32') self.rois_index = np.array(self.rois_lod).reshape([-1]).astype(np.int64) self.inputs = { 'X': self.x, 'ROIs': self.rois[:, 1:5], - 'BatchRoINums': self.rois_index + 'BatchRoINums': self.rois_index, } self.attrs = { 'output_channels': self.output_channels, 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs} @@ -189,14 +199,18 @@ class TestPRROIPoolOpTensorRoIs(OpTest): self.rois_lod.append(bno + 1) for i in range(bno + 1): x1 = np.random.uniform( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.uniform( - 0, self.height // self.spatial_scale - self.pooled_height) - - x2 = np.random.uniform(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.uniform(y1 + self.pooled_height, - self.height // self.spatial_scale) + 0, self.height // self.spatial_scale - self.pooled_height + ) + + x2 = np.random.uniform( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.uniform( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) @@ -221,27 +235,28 @@ class TestPRROIPoolOpTensorRoIs(OpTest): x = fluid.layers.data( name="X", shape=[self.channels, self.height, self.width], - dtype="float32") + dtype="float32", + ) rois = fluid.layers.data(name="ROIs", shape=[4], dtype="float32") - rois_index = fluid.layers.data(name='rois_idx', - shape=[], - dtype="int64") - output = fluid.layers.prroi_pool(x, - rois, - 0.25, - 2, - 2, - batch_roi_nums=rois_index) + rois_index = fluid.layers.data( + name='rois_idx', shape=[], dtype="int64" + ) + output = fluid.layers.prroi_pool( + x, rois, 0.25, 2, 2, batch_roi_nums=rois_index + ) loss = paddle.mean(output) optimizer = fluid.optimizer.SGD(learning_rate=1e-3) optimizer.minimize(loss) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - exe.run(fluid.default_main_program(), { - 'X': self.x, - "ROIs": self.rois[:, 1:5], - "rois_idx": self.rois_index - }) + exe.run( + fluid.default_main_program(), + { + 'X': self.x, + "ROIs": self.rois[:, 1:5], + "rois_idx": self.rois_index, + }, + ) def test_net(self): places = [fluid.CPUPlace()] @@ -252,42 +267,50 @@ class TestPRROIPoolOpTensorRoIs(OpTest): def test_errors(self): with program_guard(Program(), Program()): - x = fluid.layers.data(name="x", - shape=[245, 30, 30], - dtype="float32") - rois = fluid.layers.data(name="rois", - shape=[4], - dtype="float32", - lod_level=1) + x = fluid.layers.data( + name="x", shape=[245, 30, 30], dtype="float32" + ) + rois = fluid.layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1 + ) # spatial_scale must be float type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 2, 7, - 7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 2, 7, 7 + ) # pooled_height must be int type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 0.25, - 0.7, 7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 0.25, 0.7, 7 + ) # pooled_width must be int type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 0.25, - 7, 0.7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 0.25, 7, 0.7 + ) def test_bad_x(): - x = fluid.layers.data(name='data1', - shape=[2, 3, 16, 16], - dtype='int64', - append_batch_size=False) - label = fluid.layers.data(name='label1', - shape=[2, 4], - dtype='float32', - lod_level=1, - append_batch_size=False) + x = fluid.layers.data( + name='data1', + shape=[2, 3, 16, 16], + dtype='int64', + append_batch_size=False, + ) + label = fluid.layers.data( + name='label1', + shape=[2, 4], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) output = fluid.layers.prroi_pool(x, label, 0.25, 2, 2) self.assertRaises(TypeError, test_bad_x) def test_bad_y(): - x = fluid.layers.data(name='data2', - shape=[2, 3, 16, 16], - dtype='float32', - append_batch_size=False) + x = fluid.layers.data( + name='data2', + shape=[2, 3, 16, 16], + dtype='float32', + append_batch_size=False, + ) label = [[1, 2, 3, 4], [2, 3, 4, 5]] output = fluid.layers.prroi_pool(x, label, 0.25, 2, 2) @@ -296,5 +319,6 @@ class TestPRROIPoolOpTensorRoIs(OpTest): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_prune.py b/python/paddle/fluid/tests/unittests/test_prune.py index b6386e69674..5b7596d813d 100644 --- a/python/paddle/fluid/tests/unittests/test_prune.py +++ b/python/paddle/fluid/tests/unittests/test_prune.py @@ -23,7 +23,6 @@ import contextlib class TestPrune(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[2], dtype='float32') label = fluid.layers.data(name="label", shape=[1], dtype="int64") @@ -39,14 +38,24 @@ class TestPrune(unittest.TestCase): with fluid.program_guard(program, startup_program): (x, y, label, loss) = self.net() self.assertEqual(len(block.ops), 5) - self.assertEqual([op.type for op in block.ops], [ - "mul", "elementwise_add", "softmax", "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in block.ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) pruned_program = program._prune_with_input( - feeded_var_names=[y.name, label.name], targets=[loss]) + feeded_var_names=[y.name, label.name], targets=[loss] + ) self.assertEqual(len(pruned_program.global_block().ops), 2) - self.assertEqual([op.type for op in pruned_program.global_block().ops], - ["cross_entropy2", "reduce_mean"]) + self.assertEqual( + [op.type for op in pruned_program.global_block().ops], + ["cross_entropy2", "reduce_mean"], + ) def test_prune(self): program = framework.Program() @@ -55,16 +64,28 @@ class TestPrune(unittest.TestCase): with fluid.program_guard(program, startup_program): (x, y, label, loss) = self.net() self.assertEqual(len(block.ops), 5) - self.assertEqual([op.type for op in block.ops], [ - "mul", "elementwise_add", "softmax", "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in block.ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) pruned_program = program._prune(targets=[loss]) self.assertEqual(len(pruned_program.global_block().ops), 5) - self.assertEqual([op.type for op in pruned_program.global_block().ops], - [ - "mul", "elementwise_add", "softmax", - "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in pruned_program.global_block().ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) def test_prune_target_not_list(self): program = framework.Program() @@ -73,16 +94,28 @@ class TestPrune(unittest.TestCase): with fluid.program_guard(program, startup_program): (x, y, label, loss) = self.net() self.assertEqual(len(block.ops), 5) - self.assertEqual([op.type for op in block.ops], [ - "mul", "elementwise_add", "softmax", "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in block.ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) pruned_program = program._prune(targets=loss) self.assertEqual(len(pruned_program.global_block().ops), 5) - self.assertEqual([op.type for op in pruned_program.global_block().ops], - [ - "mul", "elementwise_add", "softmax", - "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in pruned_program.global_block().ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) def test_prune_target_none(self): program = framework.Program() @@ -91,15 +124,23 @@ class TestPrune(unittest.TestCase): with fluid.program_guard(program, startup_program): (x, y, label, loss) = self.net() self.assertEqual(len(block.ops), 5) - self.assertEqual([op.type for op in block.ops], [ - "mul", "elementwise_add", "softmax", "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in block.ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) try: pruned_program = program._prune(targets=None) except ValueError as e: self.assertIn( "All targets of Program._prune_with_input() can only be Variable or Operator", - str(e)) + str(e), + ) def mock(self, program, feed, fetch, optimize_ops): @@ -116,7 +157,6 @@ def _mock_guard(mock): class TestExecutorRunAutoPrune(unittest.TestCase): - def net1(self): x = fluid.layers.data(name='x', shape=[2], dtype='float32') label = fluid.layers.data(name="label", shape=[1], dtype="int64") @@ -124,11 +164,11 @@ class TestExecutorRunAutoPrune(unittest.TestCase): name="fc_weight", learning_rate=0.5, initializer=fluid.initializer.Constant(1.0), - trainable=True) - y = fluid.layers.fc(input=[x], - size=2, - act="softmax", - param_attr=w_param_attrs) + trainable=True, + ) + y = fluid.layers.fc( + input=[x], size=2, act="softmax", param_attr=w_param_attrs + ) loss1 = fluid.layers.cross_entropy(input=y, label=label) loss1 = paddle.mean(x=loss1) loss2 = fluid.layers.cross_entropy(input=y, label=label) @@ -145,25 +185,35 @@ class TestExecutorRunAutoPrune(unittest.TestCase): name="fc_weight1", learning_rate=0.5, initializer=fluid.initializer.Constant(1.0), - trainable=True) + trainable=True, + ) w2_param_attrs = fluid.ParamAttr( name="fc_weight2", learning_rate=0.5, initializer=fluid.initializer.Constant(1.0), - trainable=True) - y1 = fluid.layers.fc(input=[x1], - size=2, - act="softmax", - param_attr=w1_param_attrs) - y2 = fluid.layers.fc(input=[x2], - size=2, - act="softmax", - param_attr=w2_param_attrs) + trainable=True, + ) + y1 = fluid.layers.fc( + input=[x1], size=2, act="softmax", param_attr=w1_param_attrs + ) + y2 = fluid.layers.fc( + input=[x2], size=2, act="softmax", param_attr=w2_param_attrs + ) loss1 = fluid.layers.cross_entropy(input=y1, label=label) loss1 = paddle.mean(x=loss1) loss2 = fluid.layers.cross_entropy(input=y2, label=label) loss2 = paddle.mean(x=loss2) - return x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs, w2_param_attrs + return ( + x1, + x2, + y1, + y2, + label, + loss1, + loss2, + w1_param_attrs, + w2_param_attrs, + ) def test_not_prune(self): """ @@ -179,13 +229,12 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNotNone(scope.find_var(loss2.name)) @@ -202,22 +251,24 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) - self.assertIsNone(scope.find_var(loss2.name)) #loss2 is pruned + self.assertIsNone(scope.find_var(loss2.name)) # loss2 is pruned weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - np.testing.assert_array_equal(weight_init, - weight) # weight not changed + scope.find_var(w_param_attrs.name).get_tensor() + ) + np.testing.assert_array_equal( + weight_init, weight + ) # weight not changed def test_prune_fetches_with_optimizer(self): """ @@ -235,22 +286,24 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) - self.assertIsNone(scope.find_var(loss2.name)) #loss2 is pruned + self.assertIsNone(scope.find_var(loss2.name)) # loss2 is pruned weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - self.assertFalse(np.array_equal(weight_init, - weight)) # weight changed + scope.find_var(w_param_attrs.name).get_tensor() + ) + self.assertFalse( + np.array_equal(weight_init, weight) + ) # weight changed def test_prune_compiled_program(self): program = framework.Program() @@ -264,25 +317,29 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) compiled_prog = fluid.CompiledProgram( - program).with_data_parallel(loss_name=loss1.name, - places=fluid.CPUPlace()) + program + ).with_data_parallel( + loss_name=loss1.name, places=fluid.CPUPlace() + ) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(compiled_prog, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + compiled_prog, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - self.assertFalse(np.array_equal(weight_init, - weight)) # weight changed + scope.find_var(w_param_attrs.name).get_tensor() + ) + self.assertFalse( + np.array_equal(weight_init, weight) + ) # weight changed def test_prune_feed_without_optimizer(self): program = framework.Program() @@ -294,22 +351,24 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - y.name: x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + program, + feed={y.name: x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - np.testing.assert_array_equal(weight_init, - weight) # weight unchanged + scope.find_var(w_param_attrs.name).get_tensor() + ) + np.testing.assert_array_equal( + weight_init, weight + ) # weight unchanged def test_prune_feed_with_optimizer(self): program = framework.Program() @@ -324,15 +383,14 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - self.assertRaises(Exception, - exe.run, - program, - feed={ - y.name: x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + self.assertRaises( + Exception, + exe.run, + program, + feed={y.name: x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) @@ -358,16 +416,16 @@ class TestExecutorRunAutoPrune(unittest.TestCase): sgd_optimizer.minimize(loss1) exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') - label_np = np.random.randint(1, - size=(10, 1)).astype('int64') + label_np = np.random.randint(1, size=(10, 1)).astype( + 'int64' + ) for i in range(10): - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) if i == 0: self.assertEqual(exe.prune_called_times, 1) else: @@ -387,38 +445,54 @@ class TestExecutorRunAutoPrune(unittest.TestCase): scope = fluid.Scope() with fluid.scope_guard(scope): with fluid.program_guard(program, startup_program): - (x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs, - w2_param_attrs) = self.net2() + ( + x1, + x2, + y1, + y2, + label, + loss1, + loss2, + w1_param_attrs, + w2_param_attrs, + ) = self.net2() adam_optimizer1 = fluid.optimizer.AdamOptimizer( - learning_rate=0.5) + learning_rate=0.5 + ) train1 = adam_optimizer1.minimize(loss1) adam_optimizer2 = fluid.optimizer.AdamOptimizer( - learning_rate=0.5) + learning_rate=0.5 + ) train2 = adam_optimizer2.minimize(loss2) exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') - label_np = np.random.randint(1, - size=(10, 1)).astype('int64') + label_np = np.random.randint(1, size=(10, 1)).astype( + 'int64' + ) for i in range(10): if i % 2: - res = exe.run(program, - feed={ - 'x1': x_np, - 'x2': x_np, - 'label': label_np - }, - fetch_list=[loss1, loss2, train1], - use_prune=True) + res = exe.run( + program, + feed={ + 'x1': x_np, + 'x2': x_np, + 'label': label_np, + }, + fetch_list=[loss1, loss2, train1], + use_prune=True, + ) else: - res = exe.run(program, - feed={ - 'x1': x_np, - 'x2': x_np, - 'label': label_np - }, - fetch_list=[loss1, loss2, train2], - use_prune=True) + res = exe.run( + program, + feed={ + 'x1': x_np, + 'x2': x_np, + 'label': label_np, + }, + fetch_list=[loss1, loss2, train2], + use_prune=True, + ) if i == 0: self.assertEqual(exe.prune_called_times, 1) elif i == 1: @@ -448,19 +522,21 @@ class TestExecutorRunAutoPrune(unittest.TestCase): sgd_optimizer.minimize(loss1) exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') - label_np = np.random.randint(1, - size=(10, 1)).astype('int64') + label_np = np.random.randint(1, size=(10, 1)).astype( + 'int64' + ) compiled_prog = fluid.CompiledProgram( - program).with_data_parallel(loss_name=loss1.name, - places=fluid.CPUPlace()) + program + ).with_data_parallel( + loss_name=loss1.name, places=fluid.CPUPlace() + ) for i in range(10): - res = exe.run(compiled_prog, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + compiled_prog, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) if i == 0: self.assertEqual(exe.prune_called_times, 1) else: @@ -486,43 +562,43 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_without_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) scope = fluid.Scope() # use_prune with fluid.scope_guard(scope): exe.run(startup_program) - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name, train1], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name, train1], + use_prune=True, + ) weight_with_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) # expected scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) - exe.run(cloned_program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + exe.run( + cloned_program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_expected = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) np.testing.assert_array_equal(weight_with_prune, weight_expected) self.assertFalse(np.array_equal(weight_without_prune, weight_expected)) @@ -540,60 +616,72 @@ class TestExecutorRunAutoPrune(unittest.TestCase): # do not use_prune with fluid.scope_guard(scope): with fluid.program_guard(program, startup_program): - (x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs, - w2_param_attrs) = self.net2() + ( + x1, + x2, + y1, + y2, + label, + loss1, + loss2, + w1_param_attrs, + w2_param_attrs, + ) = self.net2() adam_optimizer1 = fluid.optimizer.AdamOptimizer( - learning_rate=0.5) + learning_rate=0.5 + ) train1 = adam_optimizer1.minimize(loss1) cloned_program = program.clone() adam_optimizer2 = fluid.optimizer.AdamOptimizer( - learning_rate=0.5) + learning_rate=0.5 + ) train2 = adam_optimizer2.minimize(loss2) exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') compiled_prog1 = fluid.CompiledProgram( - program).with_data_parallel(loss_name=loss1.name, - places=[fluid.CPUPlace()] * 2) + program + ).with_data_parallel( + loss_name=loss1.name, places=[fluid.CPUPlace()] * 2 + ) compiled_prog2 = fluid.CompiledProgram( - program).with_data_parallel(loss_name=loss2.name, - places=[fluid.CPUPlace()] * 2) + program + ).with_data_parallel( + loss_name=loss2.name, places=[fluid.CPUPlace()] * 2 + ) for i in range(10): if i % 2 == 1: - res = exe.run(compiled_prog1, - feed=[{ - 'x1': x_np[0:5, :], - 'label': label_np[0:5, :] - }, { - 'x1': x_np[5:, :], - 'label': label_np[5:, :] - }], - fetch_list=[loss1.name, train1], - use_prune=True) + res = exe.run( + compiled_prog1, + feed=[ + {'x1': x_np[0:5, :], 'label': label_np[0:5, :]}, + {'x1': x_np[5:, :], 'label': label_np[5:, :]}, + ], + fetch_list=[loss1.name, train1], + use_prune=True, + ) else: - res = exe.run(compiled_prog2, - feed={ - 'x2': x_np, - 'label': label_np - }, - fetch_list=[loss2.name, train2], - use_prune=True) + res = exe.run( + compiled_prog2, + feed={'x2': x_np, 'label': label_np}, + fetch_list=[loss2.name, train2], + use_prune=True, + ) weight1 = np.array( - scope.find_var(w1_param_attrs.name).get_tensor()) + scope.find_var(w1_param_attrs.name).get_tensor() + ) # expected scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) for i in range(10): if i % 2 == 1: - exe.run(cloned_program, - feed={ - 'x1': x_np, - 'x2': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + exe.run( + cloned_program, + feed={'x1': x_np, 'x2': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight2 = np.array(scope.find_var(w1_param_attrs.name).get_tensor()) np.testing.assert_allclose(weight1, weight2, rtol=1e-05) @@ -619,44 +707,44 @@ class TestExecutorRunAutoPrune(unittest.TestCase): x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_without_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) scope = fluid.Scope() # use_prune with fluid.scope_guard(scope): exe.run(startup_program) - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name, train1], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name, train1], + use_prune=True, + ) weight_with_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) # expected scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) - exe.run(cloned_program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + exe.run( + cloned_program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_expected = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) np.testing.assert_array_equal(weight_with_prune, weight_expected) self.assertFalse(np.array_equal(weight_without_prune, weight_expected)) @@ -671,8 +759,17 @@ class TestExecutorRunAutoPrune(unittest.TestCase): scope = fluid.Scope() with fluid.scope_guard(scope): with fluid.program_guard(program, startup_program): - (x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs, - w2_param_attrs) = self.net2() + ( + x1, + x2, + y1, + y2, + label, + loss1, + loss2, + w1_param_attrs, + w2_param_attrs, + ) = self.net2() loss1.persistable = True loss2.persistable = True sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.5) @@ -682,31 +779,36 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight1_init = np.array( - scope.find_var(w1_param_attrs.name).get_tensor()) + scope.find_var(w1_param_attrs.name).get_tensor() + ) weight2_init = np.array( - scope.find_var(w2_param_attrs.name).get_tensor()) + scope.find_var(w2_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x1': x_np, - 'label': label_np - }, - fetch_list=[loss1.name, train1], - use_prune=True) + res = exe.run( + program, + feed={'x1': x_np, 'label': label_np}, + fetch_list=[loss1.name, train1], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(w1_param_attrs.name)) self.assertIsNotNone(scope.find_var(w2_param_attrs.name)) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) weight1 = np.array( - scope.find_var(w1_param_attrs.name).get_tensor()) + scope.find_var(w1_param_attrs.name).get_tensor() + ) weight2 = np.array( - scope.find_var(w2_param_attrs.name).get_tensor()) - self.assertFalse(np.array_equal(weight1_init, - weight1)) # weight changed - np.testing.assert_array_equal(weight2_init, - weight2) # weight2 unchanged + scope.find_var(w2_param_attrs.name).get_tensor() + ) + self.assertFalse( + np.array_equal(weight1_init, weight1) + ) # weight changed + np.testing.assert_array_equal( + weight2_init, weight2 + ) # weight2 unchanged def test_prune_override_use_prune(self): ''' @@ -727,43 +829,43 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_without_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) scope = fluid.Scope() # use_prune with fluid.scope_guard(scope): exe.run(startup_program) - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name, train1]) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name, train1], + ) weight_with_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) # expected scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) - exe.run(cloned_program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + exe.run( + cloned_program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_expected = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) np.testing.assert_array_equal(weight_with_prune, weight_expected) self.assertFalse(np.array_equal(weight_without_prune, weight_expected)) @@ -779,23 +881,25 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - y.name: x_np, - 'label': label_np - }, - fetch_list=[y.name, loss1.name], - use_prune=True) + res = exe.run( + program, + feed={y.name: x_np, 'label': label_np}, + fetch_list=[y.name, loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) self.assertIsNone(scope.find_var(x.name)) weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - np.testing.assert_array_equal(weight_init, - weight) # weight unchanged + scope.find_var(w_param_attrs.name).get_tensor() + ) + np.testing.assert_array_equal( + weight_init, weight + ) # weight unchanged def test_prune_feed_var_in_fetchlist_2(self): # the variable to be fed is leaf @@ -808,22 +912,24 @@ class TestExecutorRunAutoPrune(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - x.name: x_np, - 'label': label_np - }, - fetch_list=[x.name, loss1.name], - use_prune=True) + res = exe.run( + program, + feed={x.name: x_np, 'label': label_np}, + fetch_list=[x.name, loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - np.testing.assert_array_equal(weight_init, - weight) # weight unchanged + scope.find_var(w_param_attrs.name).get_tensor() + ) + np.testing.assert_array_equal( + weight_init, weight + ) # weight unchanged if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_prune_gate_by_capacity_op.py b/python/paddle/fluid/tests/unittests/test_prune_gate_by_capacity_op.py index 4b73e05f502..0caab557347 100644 --- a/python/paddle/fluid/tests/unittests/test_prune_gate_by_capacity_op.py +++ b/python/paddle/fluid/tests/unittests/test_prune_gate_by_capacity_op.py @@ -21,7 +21,7 @@ from paddle.fluid.framework import _test_eager_guard def count(x, upper_num): - res = np.zeros((upper_num, )).astype(int) + res = np.zeros((upper_num,)).astype(int) for i in x.reshape(-1): if i >= 0 and i < len(res): res[i] += 1 @@ -64,20 +64,22 @@ def assert_allclose(output, expected, n_expert): assert np.allclose(c1, c2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPruneGateByCapacityAPI1(unittest.TestCase): - def init_test_case(self): - self.gate_idx = np.random.randint(0, self.n_expert, - size=(200, )).astype(self.dtype) + self.gate_idx = np.random.randint(0, self.n_expert, size=(200,)).astype( + self.dtype + ) expert_count = count(self.gate_idx, self.n_expert * self.n_worker) - capacity = np.random.randint(10, 200, size=(self.n_expert, )) - self.expert_count = limit_by_capacity(expert_count, capacity, - self.n_worker).astype(self.dtype) - self.out = prune_gate_by_capacity(self.gate_idx, self.expert_count, - self.n_expert, - self.n_worker).astype(self.dtype) + capacity = np.random.randint(10, 200, size=(self.n_expert,)) + self.expert_count = limit_by_capacity( + expert_count, capacity, self.n_worker + ).astype(self.dtype) + self.out = prune_gate_by_capacity( + self.gate_idx, self.expert_count, self.n_expert, self.n_worker + ).astype(self.dtype) self.place = paddle.CUDAPlace(0) def setUp(self): @@ -89,29 +91,35 @@ class TestPruneGateByCapacityAPI1(unittest.TestCase): def test_static_api(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - gate_idx_tensor = paddle.static.data('GateIdx', - shape=self.gate_idx.shape, - dtype="int64") + gate_idx_tensor = paddle.static.data( + 'GateIdx', shape=self.gate_idx.shape, dtype="int64" + ) expert_count_tensor = paddle.static.data( - 'ExpertCount', shape=self.expert_count.shape, dtype="int64") - out = utils._prune_gate_by_capacity(gate_idx_tensor, - expert_count_tensor, - self.n_expert, self.n_worker) + 'ExpertCount', shape=self.expert_count.shape, dtype="int64" + ) + out = utils._prune_gate_by_capacity( + gate_idx_tensor, + expert_count_tensor, + self.n_expert, + self.n_worker, + ) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'GateIdx': self.gate_idx, - 'ExpertCount': self.expert_count, - }, - fetch_list=out) + res = exe.run( + feed={ + 'GateIdx': self.gate_idx, + 'ExpertCount': self.expert_count, + }, + fetch_list=out, + ) assert_allclose(res[0], self.out, self.n_expert) def func_dygraph_api(self): paddle.disable_static(self.place) gate_idx_tensor = paddle.to_tensor(self.gate_idx) expert_count_tensor = paddle.to_tensor(self.expert_count) - out = utils._prune_gate_by_capacity(gate_idx_tensor, - expert_count_tensor, self.n_expert, - self.n_worker) + out = utils._prune_gate_by_capacity( + gate_idx_tensor, expert_count_tensor, self.n_expert, self.n_worker + ) assert_allclose(out.numpy(), self.out, self.n_expert) def test_dygraph_api(self): @@ -120,10 +128,10 @@ class TestPruneGateByCapacityAPI1(unittest.TestCase): self.func_dygraph_api() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPruneGateByCapacityAPI2(TestPruneGateByCapacityAPI1): - def setUp(self): self.n_expert = 12 self.n_worker = 1 diff --git a/python/paddle/fluid/tests/unittests/test_ps_dispatcher.py b/python/paddle/fluid/tests/unittests/test_ps_dispatcher.py index f290d164333..88d6c6f45ac 100644 --- a/python/paddle/fluid/tests/unittests/test_ps_dispatcher.py +++ b/python/paddle/fluid/tests/unittests/test_ps_dispatcher.py @@ -13,15 +13,20 @@ # limitations under the License. import unittest -from paddle.fluid.incubate.fleet.parameter_server.ir.ps_dispatcher import RoundRobin, HashName, PSDispatcher +from paddle.fluid.incubate.fleet.parameter_server.ir.ps_dispatcher import ( + RoundRobin, + HashName, + PSDispatcher, +) class TestPsDispatcher(unittest.TestCase): - def setUp(self): self.points = [ - "127.0.0.1:1001", "127.0.0.1:1002", "127.0.0.1:1003", - "127.0.0.1:1004" + "127.0.0.1:1001", + "127.0.0.1:1002", + "127.0.0.1:1003", + "127.0.0.1:1004", ] def test_base(self): @@ -33,9 +38,7 @@ class TestPsDispatcher(unittest.TestCase): base.dispatch([]) def test_hash(self): - class Var: - def __init__(self, index): self._name = "var_{}".format(index) @@ -53,9 +56,7 @@ class TestPsDispatcher(unittest.TestCase): self.assertEqual(len(eplist), 4) def test_round_rodin(self): - class Var: - def __init__(self, index): self._name = "var_{}".format(index) diff --git a/python/paddle/fluid/tests/unittests/test_psroi_pool_op.py b/python/paddle/fluid/tests/unittests/test_psroi_pool_op.py index d386c7145dd..0bb1adb4ad6 100644 --- a/python/paddle/fluid/tests/unittests/test_psroi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_psroi_pool_op.py @@ -19,8 +19,15 @@ import unittest from op_test import OpTest -def calc_psroi_pool(x, rois, rois_num_per_img, output_channels, spatial_scale, - pooled_height, pooled_width): +def calc_psroi_pool( + x, + rois, + rois_num_per_img, + output_channels, + spatial_scale, + pooled_height, + pooled_width, +): """ Psroi_pool implemented by Numpy. x: 4-D as (N, C, H, W), @@ -43,8 +50,8 @@ def calc_psroi_pool(x, rois, rois_num_per_img, output_channels, spatial_scale, batch_id += 1 roi_start_w = round(roi[0]) * spatial_scale roi_start_h = round(roi[1]) * spatial_scale - roi_end_w = (round(roi[2]) + 1.) * spatial_scale - roi_end_h = (round(roi[3]) + 1.) * spatial_scale + roi_end_w = (round(roi[2]) + 1.0) * spatial_scale + roi_end_h = (round(roi[3]) + 1.0) * spatial_scale roi_height = max(roi_end_h - roi_start_h, 0.1) roi_width = max(roi_end_w - roi_start_w, 0.1) @@ -58,13 +65,17 @@ def calc_psroi_pool(x, rois, rois_num_per_img, output_channels, spatial_scale, for ph in range(pooled_height): for pw in range(pooled_width): hstart = int( - math.floor(float(ph) * bin_size_h + roi_start_h)) + math.floor(float(ph) * bin_size_h + roi_start_h) + ) wstart = int( - math.floor(float(pw) * bin_size_w + roi_start_w)) + math.floor(float(pw) * bin_size_w + roi_start_w) + ) hend = int( - math.ceil(float(ph + 1) * bin_size_h + roi_start_h)) + math.ceil(float(ph + 1) * bin_size_h + roi_start_h) + ) wend = int( - math.ceil(float(pw + 1) * bin_size_w + roi_start_w)) + math.ceil(float(pw + 1) * bin_size_w + roi_start_w) + ) hstart = min(max(hstart, 0), x.shape[2]) hend = min(max(hend, 0), x.shape[2]) wstart = min(max(wstart, 0), x.shape[3]) @@ -72,37 +83,41 @@ def calc_psroi_pool(x, rois, rois_num_per_img, output_channels, spatial_scale, c_in = (c * pooled_height + ph) * pooled_width + pw is_empty = (hend <= hstart) or (wend <= wstart) - out_sum = 0. + out_sum = 0.0 for ih in range(hstart, hend): for iw in range(wstart, wend): out_sum += x_i[c_in, ih, iw] bin_area = (hend - hstart) * (wend - wstart) - out_data[i, c, ph, - pw] = 0. if is_empty else (out_sum / - float(bin_area)) + out_data[i, c, ph, pw] = ( + 0.0 if is_empty else (out_sum / float(bin_area)) + ) return out_data class TestPSROIPoolOp(OpTest): - def set_data(self): paddle.enable_static() self.init_test_case() self.make_rois() - self.outs = calc_psroi_pool(self.x, self.boxes, self.boxes_num, - self.output_channels, self.spatial_scale, - self.pooled_height, - self.pooled_width).astype('float64') + self.outs = calc_psroi_pool( + self.x, + self.boxes, + self.boxes_num, + self.output_channels, + self.spatial_scale, + self.pooled_height, + self.pooled_width, + ).astype('float64') self.inputs = { 'X': self.x, 'ROIs': (self.rois_with_batch_id[:, 1:5], self.rois_lod), - 'RoisNum': self.boxes_num + 'RoisNum': self.boxes_num, } self.attrs = { 'output_channels': self.output_channels, 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs} @@ -128,26 +143,32 @@ class TestPSROIPoolOp(OpTest): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.random_integers( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.random_integers(x1 + self.pooled_width, - self.width // self.spatial_scale) + x2 = np.random.random_integers( + x1 + self.pooled_width, self.width // self.spatial_scale + ) y2 = np.random.random_integers( - y1 + self.pooled_height, self.height // self.spatial_scale) + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) self.rois_with_batch_id = np.array(rois).astype('float64') self.boxes = self.rois_with_batch_id[:, 1:] - self.boxes_num = np.array([bno + 1 for bno in range(self.batch_size) - ]).astype('int32') + self.boxes_num = np.array( + [bno + 1 for bno in range(self.batch_size)] + ).astype('int32') def setUp(self): self.op_type = 'psroi_pool' self.python_api = lambda x, boxes, boxes_num, pooled_height, pooled_width, output_channels, spatial_scale: paddle.vision.ops.psroi_pool( - x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale) + x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale + ) self.set_data() def test_check_output(self): @@ -158,33 +179,38 @@ class TestPSROIPoolOp(OpTest): class TestPSROIPoolDynamicFunctionAPI(unittest.TestCase): - def setUp(self): self.x = np.random.random([2, 490, 28, 28]).astype(np.float32) - self.boxes = np.array([[1, 5, 8, 10], [4, 2, 6, 7], - [12, 12, 19, 21]]).astype(np.float32) + self.boxes = np.array( + [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]] + ).astype(np.float32) self.boxes_num = np.array([1, 2]).astype(np.int32) def test_output_size(self): - def test_output_size_is_int(): output_size = 7 - out = paddle.vision.ops.psroi_pool(paddle.to_tensor(self.x), - paddle.to_tensor(self.boxes), - paddle.to_tensor(self.boxes_num), - output_size).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 10, - 1.0, 7, 7) + out = paddle.vision.ops.psroi_pool( + paddle.to_tensor(self.x), + paddle.to_tensor(self.boxes), + paddle.to_tensor(self.boxes_num), + output_size, + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 10, 1.0, 7, 7 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) def test_output_size_is_tuple(): output_size = (7, 7) - out = paddle.vision.ops.psroi_pool(paddle.to_tensor(self.x), - paddle.to_tensor(self.boxes), - paddle.to_tensor(self.boxes_num), - output_size).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 10, - 1.0, 7, 7) + out = paddle.vision.ops.psroi_pool( + paddle.to_tensor(self.x), + paddle.to_tensor(self.boxes), + paddle.to_tensor(self.boxes_num), + output_size, + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 10, 1.0, 7, 7 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) def test_dytype_is_float64(): @@ -192,9 +218,12 @@ class TestPSROIPoolDynamicFunctionAPI(unittest.TestCase): out = paddle.vision.ops.psroi_pool( paddle.to_tensor(self.x, 'float64'), paddle.to_tensor(self.boxes, 'float64'), - paddle.to_tensor(self.boxes_num, 'int32'), output_size).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 10, - 1.0, 7, 7) + paddle.to_tensor(self.boxes_num, 'int32'), + output_size, + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 10, 1.0, 7, 7 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) places = ['cpu'] @@ -208,41 +237,48 @@ class TestPSROIPoolDynamicFunctionAPI(unittest.TestCase): class TestPSROIPoolDynamicClassAPI(unittest.TestCase): - def setUp(self): self.x = np.random.random([2, 128, 32, 32]).astype(np.float32) - self.boxes = np.array([[3, 5, 6, 13], [7, 4, 22, 18], [4, 5, 7, 10], - [5, 3, 25, 21]]).astype(np.float32) + self.boxes = np.array( + [[3, 5, 6, 13], [7, 4, 22, 18], [4, 5, 7, 10], [5, 3, 25, 21]] + ).astype(np.float32) self.boxes_num = np.array([2, 2]).astype(np.int32) def test_output_size(self): - def test_output_size_is_int(): psroi_module = paddle.vision.ops.PSRoIPool(8, 1.1) - out = psroi_module(paddle.to_tensor(self.x), - paddle.to_tensor(self.boxes), - paddle.to_tensor(self.boxes_num)).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 2, - 1.1, 8, 8) + out = psroi_module( + paddle.to_tensor(self.x), + paddle.to_tensor(self.boxes), + paddle.to_tensor(self.boxes_num), + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 2, 1.1, 8, 8 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) def test_output_size_is_tuple(): psroi_pool_module = paddle.vision.ops.PSRoIPool(8, 1.1) - out = psroi_pool_module(paddle.to_tensor(self.x), - paddle.to_tensor(self.boxes), - paddle.to_tensor(self.boxes_num)).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 2, - 1.1, 8, 8) + out = psroi_pool_module( + paddle.to_tensor(self.x), + paddle.to_tensor(self.boxes), + paddle.to_tensor(self.boxes_num), + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 2, 1.1, 8, 8 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) def test_dytype_is_float64(): psroi_pool_module = paddle.vision.ops.PSRoIPool(8, 1.1) - out = psroi_pool_module(paddle.to_tensor(self.x, 'float64'), - paddle.to_tensor(self.boxes, 'float64'), - paddle.to_tensor(self.boxes_num, - 'int32')).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 2, - 1.1, 8, 8) + out = psroi_pool_module( + paddle.to_tensor(self.x, 'float64'), + paddle.to_tensor(self.boxes, 'float64'), + paddle.to_tensor(self.boxes_num, 'int32'), + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 2, 1.1, 8, 8 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) paddle.disable_static() @@ -257,87 +293,89 @@ class TestPSROIPoolDynamicClassAPI(unittest.TestCase): class TestPSROIPoolBoxesNumError(unittest.TestCase): - def setUp(self): paddle.disable_static() self.x = paddle.uniform([2, 490, 28, 28], dtype='float32') self.boxes = paddle.to_tensor( - [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]], 'float32') + [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]], 'float32' + ) def test_errors(self): - def test_boxes_num_nums_error(): boxes_num = paddle.to_tensor([1, 5], 'int32') - out = paddle.vision.ops.psroi_pool(self.x, - self.boxes, - boxes_num, - output_size=7) + out = paddle.vision.ops.psroi_pool( + self.x, self.boxes, boxes_num, output_size=7 + ) self.assertRaises(ValueError, test_boxes_num_nums_error) def test_boxes_num_length_error(): boxes_num = paddle.to_tensor([1, 1, 1], 'int32') - out = paddle.vision.ops.psroi_pool(self.x, - self.boxes, - boxes_num, - output_size=7) + out = paddle.vision.ops.psroi_pool( + self.x, self.boxes, boxes_num, output_size=7 + ) self.assertRaises(ValueError, test_boxes_num_length_error) class TestPSROIPoolChannelError(unittest.TestCase): - def setUp(self): paddle.disable_static() self.x = paddle.uniform([2, 490, 28, 28], dtype='float32') self.boxes = paddle.to_tensor( - [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]], 'float32') + [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]], 'float32' + ) self.output_size = 4 def test_errors(self): - def test_channel_error(): boxes_num = paddle.to_tensor([2, 1], 'int32') - out = paddle.vision.ops.psroi_pool(self.x, self.boxes, boxes_num, - self.output_size) + out = paddle.vision.ops.psroi_pool( + self.x, self.boxes, boxes_num, self.output_size + ) self.assertRaises(ValueError, test_channel_error) class TestPSROIPoolStaticAPI(unittest.TestCase): - def setUp(self): paddle.enable_static() - self.x_placeholder = paddle.static.data(name='x', - shape=[2, 490, 28, 28]) + self.x_placeholder = paddle.static.data( + name='x', shape=[2, 490, 28, 28] + ) self.x = np.random.random([2, 490, 28, 28]).astype(np.float32) - self.boxes_placeholder = paddle.static.data(name='boxes', - shape=[3, 4], - lod_level=1) - self.boxes = np.array([[1, 5, 8, 10], [4, 2, 6, 7], - [12, 12, 19, 21]]).astype(np.float32) + self.boxes_placeholder = paddle.static.data( + name='boxes', shape=[3, 4], lod_level=1 + ) + self.boxes = np.array( + [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]] + ).astype(np.float32) self.boxes_num = np.array([1, 2]).astype(np.int32) def test_function_in_static(self): output_size = 7 - out = paddle.vision.ops.psroi_pool(self.x_placeholder, - self.boxes_placeholder, - self.boxes_num, output_size) - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 10, - 1.0, 7, 7) + out = paddle.vision.ops.psroi_pool( + self.x_placeholder, + self.boxes_placeholder, + self.boxes_num, + output_size, + ) + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 10, 1.0, 7, 7 + ) places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): places.append(paddle.CUDAPlace(0)) for place in places: exe = paddle.static.Executor(place) boxes_lod_data = paddle.fluid.create_lod_tensor( - self.boxes, [[1, 2]], place) - out_res, = exe.run(paddle.static.default_main_program(), - feed={ - 'x': self.x, - 'boxes': boxes_lod_data - }, - fetch_list=[out.name]) + self.boxes, [[1, 2]], place + ) + (out_res,) = exe.run( + paddle.static.default_main_program(), + feed={'x': self.x, 'boxes': boxes_lod_data}, + fetch_list=[out.name], + ) np.testing.assert_allclose(out_res, expect_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_pull_gpups_sparse_op.py b/python/paddle/fluid/tests/unittests/test_pull_gpups_sparse_op.py index 8bddcb6d50f..3b2eb69a549 100644 --- a/python/paddle/fluid/tests/unittests/test_pull_gpups_sparse_op.py +++ b/python/paddle/fluid/tests/unittests/test_pull_gpups_sparse_op.py @@ -30,15 +30,13 @@ class TestPullGpupsSparse(unittest.TestCase): slots = [] with fluid.program_guard(train_program, startup_program): - l = fluid.layers.data(name='input', - shape=[1], - dtype="int64", - lod_level=1) + l = fluid.layers.data( + name='input', shape=[1], dtype="int64", lod_level=1 + ) slots.append(l) - output = _pull_gpups_sparse(slots, - size=[11], - is_distributed=True, - is_sparse=True) + output = _pull_gpups_sparse( + slots, size=[11], is_distributed=True, is_sparse=True + ) cost = paddle.mean(output) sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) sgd_optimizer.minimize(cost, train_program) @@ -49,9 +47,9 @@ class TestPullGpupsSparse(unittest.TestCase): exe = fluid.Executor(place) exe.run(startup_program) img = np.array([1]).astype(np.int64) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_put_along_axis_op.py b/python/paddle/fluid/tests/unittests/test_put_along_axis_op.py index 43771731f9b..b5a1cb25e4d 100644 --- a/python/paddle/fluid/tests/unittests/test_put_along_axis_op.py +++ b/python/paddle/fluid/tests/unittests/test_put_along_axis_op.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestPutAlongAxisOp(OpTest): - def setUp(self): self.init_data() self.reduce_op = "assign" @@ -42,7 +41,7 @@ class TestPutAlongAxisOp(OpTest): self.inputs = { 'Input': self.xnp, 'Index': self.index_broadcast, - 'Value': self.value_broadcast + 'Value': self.value_broadcast, } self.attrs = {'Axis': self.axis, 'Reduce': self.reduce_op} self.outputs = {'Result': self.target} @@ -65,7 +64,6 @@ class TestPutAlongAxisOp(OpTest): class TestPutAlongAxisAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [1, 3] @@ -90,15 +88,18 @@ class TestPutAlongAxisAPI(unittest.TestCase): value = paddle.fluid.data('Value', self.value_shape) out = paddle.put_along_axis(x, index, value, self.axis) exe = paddle.static.Executor(self.place[0]) - res = exe.run(feed={ - 'X': self.x_feed, - 'Value': self.value_np, - 'Index': self.index_np - }, - fetch_list=[out]) - - np.put_along_axis(self.x_np, self.index_np, self.value_np, - self.axis) + res = exe.run( + feed={ + 'X': self.x_feed, + 'Value': self.value_np, + 'Index': self.index_np, + }, + fetch_list=[out], + ) + + np.put_along_axis( + self.x_np, self.index_np, self.value_np, self.axis + ) # numpy put_along_axis is an inplace opearion. out_ref = self.x_np @@ -109,25 +110,29 @@ class TestPutAlongAxisAPI(unittest.TestCase): run(place) def test_api_dygraph(self): - def run(place): paddle.disable_static(place) x_tensor = paddle.to_tensor(self.x_np) index_tensor = paddle.to_tensor(self.index_np) value_tensor = paddle.to_tensor(self.value_np) - out = paddle.put_along_axis(x_tensor, index_tensor, value_tensor, - self.axis) + out = paddle.put_along_axis( + x_tensor, index_tensor, value_tensor, self.axis + ) np.array( - np.put_along_axis(self.x_np, self.index_np, self.value_np, - self.axis)) + np.put_along_axis( + self.x_np, self.index_np, self.value_np, self.axis + ) + ) out_ref = self.x_np np.testing.assert_allclose(out.numpy(), out_ref, rtol=0.001) # for ci coverage, numpy put_along_axis did not support argument of 'reduce' - paddle.put_along_axis(x_tensor, index_tensor, value_tensor, - self.axis, 'mul') - paddle.put_along_axis(x_tensor, index_tensor, value_tensor, - self.axis, 'add') + paddle.put_along_axis( + x_tensor, index_tensor, value_tensor, self.axis, 'mul' + ) + paddle.put_along_axis( + x_tensor, index_tensor, value_tensor, self.axis, 'add' + ) paddle.enable_static() @@ -135,7 +140,6 @@ class TestPutAlongAxisAPI(unittest.TestCase): run(place) def test_inplace_dygraph(self): - def run(place): paddle.disable_static(place) x_tensor = paddle.to_tensor(self.x_np) @@ -145,8 +149,10 @@ class TestPutAlongAxisAPI(unittest.TestCase): x_tensor.put_along_axis_(index_tensor, value_tensor, self.axis) np.array( - np.put_along_axis(self.x_np, self.index_np, self.value_np, - self.axis)) + np.put_along_axis( + self.x_np, self.index_np, self.value_np, self.axis + ) + ) out_ref = self.x_np np.testing.assert_allclose(x_tensor.numpy(), out_ref, rtol=0.001) @@ -157,7 +163,6 @@ class TestPutAlongAxisAPI(unittest.TestCase): class TestPutAlongAxisAPICase2(TestPutAlongAxisAPI): - def setUp(self): np.random.seed(0) self.shape = [2, 2] @@ -174,13 +179,13 @@ class TestPutAlongAxisAPICase2(TestPutAlongAxisAPI): class TestPutAlongAxisAPICase3(TestPutAlongAxisAPI): - def setUp(self): np.random.seed(0) self.shape = [2, 2] self.index_shape = [4, 2] - self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, - 0]]).astype('int64') + self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, 0]]).astype( + 'int64' + ) self.x_np = np.random.random(self.shape).astype(np.float32) self.place = [paddle.CPUPlace()] self.axis = 0 diff --git a/python/paddle/fluid/tests/unittests/test_py_func_op.py b/python/paddle/fluid/tests/unittests/test_py_func_op.py index 91e41a28ee5..96121679567 100644 --- a/python/paddle/fluid/tests/unittests/test_py_func_op.py +++ b/python/paddle/fluid/tests/unittests/test_py_func_op.py @@ -64,8 +64,9 @@ def cross_entropy_grad(logits, labels, bwd_dout): N = logits.shape[1] dlogits = np.zeros([M, N]).astype(logits.dtype) for idx in range(M): - dlogits[idx][labels[idx] - [0]] = -bwd_dout[idx] / logits[idx][labels[idx][0]] + dlogits[idx][labels[idx][0]] = ( + -bwd_dout[idx] / logits[idx][labels[idx][0]] + ) return dlogits, None @@ -75,56 +76,85 @@ def simple_fc_net(img, label, use_py_func_op): hidden = fluid.layers.fc( hidden, size=200, - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) if not use_py_func_op: hidden = fluid.layers.tanh(hidden) else: - new_hidden = fluid.default_main_program().current_block( - ).create_var(name='hidden_{}'.format(idx), - dtype='float32', - shape=hidden.shape) - hidden = fluid.layers.py_func(func=tanh, - x=hidden, - out=new_hidden, - backward_func=tanh_grad, - skip_vars_in_backward_input=hidden) + new_hidden = ( + fluid.default_main_program() + .current_block() + .create_var( + name='hidden_{}'.format(idx), + dtype='float32', + shape=hidden.shape, + ) + ) + hidden = fluid.layers.py_func( + func=tanh, + x=hidden, + out=new_hidden, + backward_func=tanh_grad, + skip_vars_in_backward_input=hidden, + ) prediction = fluid.layers.fc(hidden, size=10, act='softmax') if not use_py_func_op: loss = fluid.layers.cross_entropy(input=prediction, label=label) else: - loss = fluid.default_main_program().current_block().create_var( - name='loss', dtype='float32', shape=[-1, 1]) - loss = fluid.layers.py_func(func=cross_entropy, - x=[prediction, label], - out=loss, - backward_func=cross_entropy_grad, - skip_vars_in_backward_input=loss) - - dummy_var = fluid.default_main_program().current_block().create_var( - name='test_tmp_var', dtype='float32', shape=[1]) - fluid.layers.py_func(func=dummy_func_with_no_input, - x=None, - out=dummy_var) + loss = ( + fluid.default_main_program() + .current_block() + .create_var(name='loss', dtype='float32', shape=[-1, 1]) + ) + loss = fluid.layers.py_func( + func=cross_entropy, + x=[prediction, label], + out=loss, + backward_func=cross_entropy_grad, + skip_vars_in_backward_input=loss, + ) + + dummy_var = ( + fluid.default_main_program() + .current_block() + .create_var(name='test_tmp_var', dtype='float32', shape=[1]) + ) + fluid.layers.py_func( + func=dummy_func_with_no_input, x=None, out=dummy_var + ) loss += dummy_var fluid.layers.py_func(func=dummy_func_with_no_output, x=loss, out=None) - loss_out = fluid.default_main_program().current_block().create_var( - dtype='float32', shape=[-1, 1]) - dummy_var_out = fluid.default_main_program().current_block().create_var( - dtype='float32', shape=[1]) - fluid.layers.py_func(func=dummy_func_with_multi_input_output, - x=(loss, dummy_var), - out=(loss_out, dummy_var_out)) - assert loss == loss_out and dummy_var == dummy_var_out, \ - "py_func failed with multi input and output" - - fluid.layers.py_func(func=dummy_func_with_multi_input_output, - x=[loss, dummy_var], - out=[loss_out, dummy_var_out]) - assert loss == loss_out and dummy_var == dummy_var_out, \ - "py_func failed with multi input and output" + loss_out = ( + fluid.default_main_program() + .current_block() + .create_var(dtype='float32', shape=[-1, 1]) + ) + dummy_var_out = ( + fluid.default_main_program() + .current_block() + .create_var(dtype='float32', shape=[1]) + ) + fluid.layers.py_func( + func=dummy_func_with_multi_input_output, + x=(loss, dummy_var), + out=(loss_out, dummy_var_out), + ) + assert ( + loss == loss_out and dummy_var == dummy_var_out + ), "py_func failed with multi input and output" + + fluid.layers.py_func( + func=dummy_func_with_multi_input_output, + x=[loss, dummy_var], + out=[loss_out, dummy_var_out], + ) + assert ( + loss == loss_out and dummy_var == dummy_var_out + ), "py_func failed with multi input and output" loss = paddle.mean(loss) return loss @@ -132,9 +162,9 @@ def simple_fc_net(img, label, use_py_func_op): def reader(): for _ in range(dev_cnt * 100): - yield np.random.random([784]), np.random.random_integers(size=[1], - low=0, - high=9) + yield np.random.random([784]), np.random.random_integers( + size=[1], low=0, high=9 + ) def test_main(use_cuda, use_py_func_op, use_parallel_executor): @@ -162,7 +192,8 @@ def test_main(use_cuda, use_py_func_op, use_parallel_executor): if use_parallel_executor: train_cp = compiler.CompiledProgram( - fluid.default_main_program()) + fluid.default_main_program() + ) train_cp = train_cp.with_data_parallel(loss_name=loss.name) fetch_list = [loss.name] else: @@ -171,15 +202,14 @@ def test_main(use_cuda, use_py_func_op, use_parallel_executor): ret = [] for epoch_id in range(2): for d in r(): - L, = exe.run(train_cp, - feed=feeder.feed(d), - fetch_list=fetch_list) + (L,) = exe.run( + train_cp, feed=feeder.feed(d), fetch_list=fetch_list + ) ret.append(L) return np.array(ret) class TestPyFuncOpUseExecutor(unittest.TestCase): - def setUp(self): self.use_parallel_executor = False @@ -187,8 +217,9 @@ class TestPyFuncOpUseExecutor(unittest.TestCase): for use_cuda in [True, False]: losses = [] for use_py_func_op in [True, False]: - L = test_main(use_cuda, use_py_func_op, - self.use_parallel_executor) + L = test_main( + use_cuda, use_py_func_op, self.use_parallel_executor + ) if L is not None: losses.append(L) @@ -198,7 +229,6 @@ class TestPyFuncOpUseExecutor(unittest.TestCase): class TestPyFuncOpUseParallelExecutor(TestPyFuncOpUseExecutor): - def setUp(self): self.use_parallel_executor = True diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_combination.py b/python/paddle/fluid/tests/unittests/test_py_reader_combination.py index 84ea5f5f7bb..1c399b89cec 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_combination.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_combination.py @@ -19,23 +19,20 @@ import numpy as np class TestPyReaderCombination(unittest.TestCase): - def setUp(self): self.n1 = 10 self.n2 = 20 self.batch_size = 2 def create_reader(self, batch_num): - def __impl__(): for _ in range(batch_num): - image = np.random.uniform(low=-1, high=1, - size=[batch_num, - 784]).astype('float32') - label = np.random.random_integers(low=0, - high=9, - size=[batch_num, - 1]).astype('int64') + image = np.random.uniform( + low=-1, high=1, size=[batch_num, 784] + ).astype('float32') + label = np.random.random_integers( + low=0, high=9, size=[batch_num, 1] + ).astype('int64') yield image, label return __impl__ @@ -57,17 +54,17 @@ class TestPyReaderCombination(unittest.TestCase): def main_impl(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - image = fluid.layers.data(name='image', - dtype='float32', - shape=[784]) + image = fluid.layers.data( + name='image', dtype='float32', shape=[784] + ) label = fluid.layers.data(name='label', dtype='int64', shape=[1]) - py_reader1 = fluid.io.PyReader(feed_list=[image, label], - capacity=16, - iterable=True) - py_reader2 = fluid.io.PyReader(feed_list=[image, label], - capacity=16, - iterable=True) + py_reader1 = fluid.io.PyReader( + feed_list=[image, label], capacity=16, iterable=True + ) + py_reader2 = fluid.io.PyReader( + feed_list=[image, label], capacity=16, iterable=True + ) reader1 = paddle.reader.cache(self.create_reader(self.n1)) reader2 = paddle.reader.cache(self.create_reader(self.n2)) @@ -77,8 +74,12 @@ class TestPyReaderCombination(unittest.TestCase): for _ in range(10): max_num = min(self.n1, self.n2) batch_num = 0 - for reader_np1, py_reader_dict1, reader_np2, py_reader_dict2 in zip( - reader1(), py_reader1(), reader2(), py_reader2()): + for ( + reader_np1, + py_reader_dict1, + reader_np2, + py_reader_dict2, + ) in zip(reader1(), py_reader1(), reader2(), py_reader2()): self.assertFeedVarEqual(reader_np1, py_reader_dict1) self.assertFeedVarEqual(reader_np2, py_reader_dict2) batch_num += 1 @@ -100,7 +101,6 @@ class TestPyReaderCombination(unittest.TestCase): class TestPyReaderCombination2(TestPyReaderCombination): - def setUp(self): self.n1 = 20 self.n2 = 10 @@ -108,7 +108,6 @@ class TestPyReaderCombination2(TestPyReaderCombination): class TestPyReaderCombination3(TestPyReaderCombination): - def setUp(self): self.n1 = 10 self.n2 = 10 diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_error_msg.py b/python/paddle/fluid/tests/unittests/test_py_reader_error_msg.py index 337cafbb124..40cdcdcc19f 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_error_msg.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_error_msg.py @@ -19,32 +19,38 @@ import paddle class TestPyReaderErrorMsg(unittest.TestCase): - def test_check_input_array(self): - fluid.reader.GeneratorLoader._check_input_array([ - np.random.randint(100, size=[2]), - np.random.randint(100, size=[2]), - np.random.randint(100, size=[2]) - ]) - self.assertRaises(TypeError, - fluid.reader.GeneratorLoader._check_input_array, [ - np.random.randint(100, size=[2]), - np.random.randint(100, size=[1]), - np.random.randint(100, size=[3]) - ]) + fluid.reader.GeneratorLoader._check_input_array( + [ + np.random.randint(100, size=[2]), + np.random.randint(100, size=[2]), + np.random.randint(100, size=[2]), + ] + ) + self.assertRaises( + TypeError, + fluid.reader.GeneratorLoader._check_input_array, + [ + np.random.randint(100, size=[2]), + np.random.randint(100, size=[1]), + np.random.randint(100, size=[3]), + ], + ) class TestDoubleBufferAPI(unittest.TestCase): - def test_double_buffer(self): paddle.enable_static() if fluid.core.is_compiled_with_cuda(): - reader = fluid.layers.py_reader(capacity=64, - shapes=[(-1, 1, 28, 28), (-1, 1)], - dtypes=['float32', 'int64'], - use_double_buffer=False) - reader = fluid.layers.double_buffer(reader, - place=fluid.core.CUDAPlace(0)) + reader = fluid.layers.py_reader( + capacity=64, + shapes=[(-1, 1, 28, 28), (-1, 1)], + dtypes=['float32', 'int64'], + use_double_buffer=False, + ) + reader = fluid.layers.double_buffer( + reader, place=fluid.core.CUDAPlace(0) + ) image, label = fluid.layers.read_file(reader) diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_lod_level_share.py b/python/paddle/fluid/tests/unittests/test_py_reader_lod_level_share.py index 4b5e2b9711e..ef55a226dea 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_lod_level_share.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_lod_level_share.py @@ -17,7 +17,6 @@ import unittest class TestLoDLevelShare(unittest.TestCase): - def setUp(self): self.use_double_buffer = False @@ -27,7 +26,8 @@ class TestLoDLevelShare(unittest.TestCase): shapes=([-1, 256], [-1, 512], [-1, 100]), dtypes=('float32', 'int64', 'double'), lod_levels=(1, 2, 0), - use_double_buffer=self.use_double_buffer) + use_double_buffer=self.use_double_buffer, + ) x, y, z = fluid.layers.read_file(reader) self.assertEqual(x.lod_level, 1) @@ -36,7 +36,6 @@ class TestLoDLevelShare(unittest.TestCase): class TestLoDLevelShare2(TestLoDLevelShare): - def setUp(self): self.use_double_buffer = True diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_pin_memory.py b/python/paddle/fluid/tests/unittests/test_py_reader_pin_memory.py index 596bc2ff839..4268d6fbcc1 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_pin_memory.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_pin_memory.py @@ -20,7 +20,6 @@ import numpy as np def user_reader(inputs): - def _reader(): for d in inputs: yield d @@ -29,7 +28,6 @@ def user_reader(inputs): def batch_feeder(batch_reader, pin_memory=False, img_dtype="float32"): - def _feeder(): for batch_data in batch_reader(): sample_batch = [] @@ -48,7 +46,6 @@ def batch_feeder(batch_reader, pin_memory=False, img_dtype="float32"): class TestPyReader(unittest.TestCase): - def setUp(self): self.capacity = 10 self.shapes = [(-1, 3, 2, 1), (-1, 1)] @@ -57,29 +54,37 @@ class TestPyReader(unittest.TestCase): def test_pin_memory_pyreader(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) executor = fluid.Executor(place) - data_file = fluid.layers.py_reader(capacity=self.capacity, - dtypes=self.dtypes, - lod_levels=self.lod_levels, - shapes=self.shapes) + data_file = fluid.layers.py_reader( + capacity=self.capacity, + dtypes=self.dtypes, + lod_levels=self.lod_levels, + shapes=self.shapes, + ) # feed_queue = data_file.queue read_out_data = fluid.layers.read_file(data_file) self.inputs = [] for _ in range(10): - sample = np.random.uniform(low=0, high=1, - size=[3, 2, 1]).astype("float32") + sample = np.random.uniform( + low=0, high=1, size=[3, 2, 1] + ).astype("float32") label = np.random.randint(low=0, high=10, dtype="int64") self.inputs.append((sample, label)) self.input_tensors = [] for d, l in batch_feeder( - paddle.batch(user_reader(self.inputs), batch_size=2), - pin_memory=True - if fluid.core.is_compiled_with_cuda() else False)(): + paddle.batch(user_reader(self.inputs), batch_size=2), + pin_memory=True + if fluid.core.is_compiled_with_cuda() + else False, + )(): ta = fluid.LoDTensorArray() ta.append(d) ta.append(l) @@ -95,10 +100,13 @@ class TestPyReader(unittest.TestCase): self.batched_inputs.append([feed_d, feed_l]) data_file.decorate_tensor_provider( - batch_feeder(paddle.batch(user_reader(self.inputs), - batch_size=2), - pin_memory=True - if fluid.core.is_compiled_with_cuda() else False)) + batch_feeder( + paddle.batch(user_reader(self.inputs), batch_size=2), + pin_memory=True + if fluid.core.is_compiled_with_cuda() + else False, + ) + ) executor.run(fluid.default_startup_program()) self.outputs = [] @@ -106,14 +114,16 @@ class TestPyReader(unittest.TestCase): data_file.start() for _ in self.input_tensors: self.outputs.append( - executor.run(fetch_list=list(read_out_data))) + executor.run(fetch_list=list(read_out_data)) + ) data_file.reset() self.validate() def validate(self): self.assertEqual(len(self.batched_inputs), len(self.outputs)) - for in_data_list, out_data_list in zip(self.batched_inputs, - self.outputs): + for in_data_list, out_data_list in zip( + self.batched_inputs, self.outputs + ): self.assertEqual(len(in_data_list), len(out_data_list)) in_data_list_np = [ np.array(in_lod_tensor) for in_lod_tensor in in_data_list diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py b/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py index cdca6695e94..4ccaaf02746 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py @@ -24,7 +24,6 @@ def feed_data(feed_queue, inputs): class TestPyReader(unittest.TestCase): - def setUp(self): self.capacity = 10 self.batch_size_min = 10 @@ -42,46 +41,55 @@ class TestPyReader(unittest.TestCase): def main(self, use_thread=False): with fluid.program_guard(fluid.Program(), fluid.Program()): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) executor = fluid.Executor(place) - data_file = fluid.layers.py_reader(capacity=self.capacity, - dtypes=self.dtypes, - lod_levels=self.lod_levels, - shapes=self.shapes) + data_file = fluid.layers.py_reader( + capacity=self.capacity, + dtypes=self.dtypes, + lod_levels=self.lod_levels, + shapes=self.shapes, + ) feed_queue = data_file.queue read_out_data = fluid.layers.read_file(data_file) self.inputs = [] for i in range(self.iterations): in_data = fluid.LoDTensorArray() - batch_size = np.random.random_integers(self.batch_size_min, - self.batch_size_max) + batch_size = np.random.random_integers( + self.batch_size_min, self.batch_size_max + ) for shape, dtype in zip(self.shapes, self.dtypes): - next_data = np.random.uniform(low=0, - high=1000, - size=(batch_size, ) + - shape[1:]).astype(dtype) + next_data = np.random.uniform( + low=0, high=1000, size=(batch_size,) + shape[1:] + ).astype(dtype) in_data.append( - fluid.executor._as_lodtensor(next_data, place)) + fluid.executor._as_lodtensor(next_data, place) + ) self.inputs.append(in_data) executor.run(fluid.default_startup_program()) self.outputs = [] if use_thread: - thread = Thread(target=feed_data, - args=(feed_queue, self.inputs)) + thread = Thread( + target=feed_data, args=(feed_queue, self.inputs) + ) thread.start() for in_data in self.inputs: self.outputs.append( - executor.run(fetch_list=list(read_out_data))) + executor.run(fetch_list=list(read_out_data)) + ) else: for in_data in self.inputs: feed_queue.push(in_data) self.outputs.append( - executor.run(fetch_list=list(read_out_data))) + executor.run(fetch_list=list(read_out_data)) + ) feed_queue.close() self.validate() diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_return_list.py b/python/paddle/fluid/tests/unittests/test_py_reader_return_list.py index c6e951997ea..bfc08245ee3 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_return_list.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_return_list.py @@ -19,39 +19,40 @@ import numpy as np class TestPyReader(unittest.TestCase): - def setUp(self): self.batch_size = 32 self.epoch_num = 2 self.sample_num = 10 def test_returnlist(self): - def reader_creator_random_image(height, width): - def reader(): for i in range(self.sample_num): - yield np.random.uniform(low=0, - high=255, - size=[height, width]), + yield np.random.uniform( + low=0, high=255, size=[height, width] + ), return reader for return_list in [True, False]: with fluid.program_guard(fluid.Program(), fluid.Program()): - image = fluid.layers.data(name='image', - shape=[784, 784], - dtype='float32') - reader = fluid.io.PyReader(feed_list=[image], - capacity=4, - iterable=True, - return_list=return_list) + image = fluid.layers.data( + name='image', shape=[784, 784], dtype='float32' + ) + reader = fluid.io.PyReader( + feed_list=[image], + capacity=4, + iterable=True, + return_list=return_list, + ) user_defined_reader = reader_creator_random_image(784, 784) reader.decorate_sample_list_generator( - paddle.batch(user_defined_reader, - batch_size=self.batch_size), - fluid.core.CPUPlace()) + paddle.batch( + user_defined_reader, batch_size=self.batch_size + ), + fluid.core.CPUPlace(), + ) # definition of network is omitted executor = fluid.Executor(fluid.core.CPUPlace()) executor.run(fluid.default_main_program()) @@ -69,7 +70,8 @@ class TestPyReader(unittest.TestCase): batch_py_reader.decorate_sample_generator( user_defined_reader, batch_size=self.batch_size, - places=fluid.core.CPUPlace()) + places=fluid.core.CPUPlace(), + ) for epoch in range(self.epoch_num): for _, data in enumerate(batch_py_reader()): diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py b/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py index 7f0cf633ed2..fa1297d46b6 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py @@ -23,18 +23,18 @@ os.environ['CPU_NUM'] = '1' def random_reader(sample_num): - def __impl__(): for _ in range(sample_num): - yield np.random.random( - size=[784]).astype('float32'), np.random.random_integers( - low=0, high=9, size=[1]).astype('int64') + yield np.random.random(size=[784]).astype( + 'float32' + ), np.random.random_integers(low=0, high=9, size=[1]).astype( + 'int64' + ) return paddle.reader.cache(__impl__) class TestCaseBase(unittest.TestCase): - def setUp(self): self.batch_size = 32 self.epoch_num = 2 @@ -54,22 +54,24 @@ class TestCaseBase(unittest.TestCase): def run_main(self, reader, use_sample_generator, iterable, drop_last): image = fluid.layers.data(name='image', dtype='float32', shape=[784]) label = fluid.layers.data(name='label', dtype='int64', shape=[1]) - py_reader = fluid.io.PyReader(feed_list=[image, label], - capacity=16, - iterable=iterable, - use_double_buffer=False) + py_reader = fluid.io.PyReader( + feed_list=[image, label], + capacity=16, + iterable=iterable, + use_double_buffer=False, + ) batch_reader = paddle.batch(reader, self.batch_size, drop_last) all_datas = self.generate_all_data(batch_reader) if not use_sample_generator: - py_reader.decorate_sample_list_generator(batch_reader, - places=fluid.cpu_places()) + py_reader.decorate_sample_list_generator( + batch_reader, places=fluid.cpu_places() + ) else: - py_reader.decorate_sample_generator(reader, - self.batch_size, - drop_last, - places=fluid.cpu_places()) + py_reader.decorate_sample_generator( + reader, self.batch_size, drop_last, places=fluid.cpu_places() + ) if drop_last: batch_num = int(self.sample_num / self.batch_size) @@ -111,12 +113,12 @@ class TestCaseBase(unittest.TestCase): for iterable in [False, True]: for drop_last in [False, True]: with fluid.program_guard(fluid.Program(), fluid.Program()): - self.run_main(reader, use_sample_generator, iterable, - drop_last) + self.run_main( + reader, use_sample_generator, iterable, drop_last + ) class TestCase1(TestCaseBase): - def setUp(self): self.batch_size = 32 self.epoch_num = 10 @@ -124,7 +126,6 @@ class TestCase1(TestCaseBase): class TestCase2(TestCaseBase): - def setUp(self): self.batch_size = 32 self.epoch_num = 2 @@ -132,7 +133,6 @@ class TestCase2(TestCaseBase): class TestCase3(TestCaseBase): - def setUp(self): self.batch_size = 32 self.epoch_num = 2 diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py b/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py index a0ab41e4fe2..dca3b061cbf 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py @@ -39,8 +39,11 @@ def as_tensor(np_array_or_tensor, place=None): def as_numpy(tensor_or_numpy): - return tensor_or_numpy if isinstance( - tensor_or_numpy, np.ndarray) else np.array(tensor_or_numpy) + return ( + tensor_or_numpy + if isinstance(tensor_or_numpy, np.ndarray) + else np.array(tensor_or_numpy) + ) def sample_list_to_tensor_array(sample_list): @@ -80,13 +83,15 @@ def feed_data(feed_queue, batch_reader): feed_queue.close() -def simple_fc_net(in_size, - class_num, - hidden_sizes, - batch_size, - queue_capacity, - use_double_buffer=False, - use_feed_list=True): +def simple_fc_net( + in_size, + class_num, + hidden_sizes, + batch_size, + queue_capacity, + use_double_buffer=False, + use_feed_list=True, +): in_data = fluid.layers.data(name="data", dtype='float32', shape=[in_size]) label = fluid.layers.data(name='label', dtype='int64', shape=[1]) if use_feed_list: @@ -94,14 +99,16 @@ def simple_fc_net(in_size, capacity=queue_capacity, use_double_buffer=use_double_buffer, feed_list=[in_data, label], - name=unique_name.generate('py_reader_name')) + name=unique_name.generate('py_reader_name'), + ) else: py_reader = fluid.layers.py_reader( capacity=queue_capacity, shapes=[in_data.shape, label.shape], dtypes=['float32', 'int64'], name=unique_name.generate('py_reader_name'), - use_double_buffer=use_double_buffer) + use_double_buffer=use_double_buffer, + ) in_data, label = fluid.layers.read_file(py_reader) @@ -113,12 +120,15 @@ def simple_fc_net(in_size, hidden, size=hidden_size, act='tanh', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) predict_label = fluid.layers.fc(hidden, size=class_num, act='softmax') loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -126,7 +136,6 @@ def simple_fc_net(in_size, class TestPyReaderUsingExecutor(unittest.TestCase): - def setUp(self): self.in_size = 1000 self.hidden_sizes = [50, 30, 20] @@ -136,39 +145,42 @@ class TestPyReaderUsingExecutor(unittest.TestCase): self.queue_capacity = 50 def test(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): for use_parallel_executor in [False, True]: for use_double_buffer in [False, True]: for use_feed_list in [False, True]: for use_decorate_paddle_reader in [False, True]: print('Test Parameters:'), - print({ - 'use_cuda': + print( + { + 'use_cuda': use_cuda, + 'use_parallel_executor': use_parallel_executor, + 'use_double_buffer': use_double_buffer, + 'use_feed_list': use_feed_list, + 'use_decorate_paddle_reader': use_decorate_paddle_reader, + } + ) + self.main( use_cuda, - 'use_parallel_executor': use_parallel_executor, - 'use_double_buffer': use_double_buffer, - 'use_feed_list': use_feed_list, - 'use_decorate_paddle_reader': - use_decorate_paddle_reader - }) - self.main(use_cuda, use_parallel_executor, - use_double_buffer, use_feed_list, - use_decorate_paddle_reader) + use_decorate_paddle_reader, + ) def tensor_reader(self, use_decorate_paddle_reader): - def reader(): - for sample_id in range(self.batch_size * self.iterations * - self.batch_size_times): + for sample_id in range( + self.batch_size * self.iterations * self.batch_size_times + ): in_data = np.random.uniform( - low=0, high=1, size=(self.in_size, )).astype('float32') - label = np.random.random_integers(low=0, - high=self.class_num - 1, - size=(1, )).astype('int64') + low=0, high=1, size=(self.in_size,) + ).astype('float32') + label = np.random.random_integers( + low=0, high=self.class_num - 1, size=(1,) + ).astype('int64') reshaped_in_data = np.reshape(in_data, [1, -1]) reshaped_label = np.reshape(label, [1, -1]) @@ -176,9 +188,11 @@ class TestPyReaderUsingExecutor(unittest.TestCase): self.inputs.append([reshaped_in_data, reshaped_label]) else: self.inputs[-1][0] = np.concatenate( - (self.inputs[-1][0], reshaped_in_data), axis=0) + (self.inputs[-1][0], reshaped_in_data), axis=0 + ) self.inputs[-1][1] = np.concatenate( - (self.inputs[-1][1], reshaped_label), axis=0) + (self.inputs[-1][1], reshaped_label), axis=0 + ) yield in_data, label @@ -187,12 +201,14 @@ class TestPyReaderUsingExecutor(unittest.TestCase): return reader - def main(self, - use_cuda=True, - use_parallel_executor=False, - use_double_buffer=False, - use_feed_list=False, - use_decorate_paddle_reader=False): + def main( + self, + use_cuda=True, + use_parallel_executor=False, + use_double_buffer=False, + use_feed_list=False, + use_decorate_paddle_reader=False, + ): assert not use_cuda or use_cuda and core.is_compiled_with_cuda() self.use_cuda = use_cuda @@ -205,14 +221,22 @@ class TestPyReaderUsingExecutor(unittest.TestCase): main_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - in_data, label, loss, optimizer, feed_queue, py_reader = simple_fc_net( + ( + in_data, + label, + loss, + optimizer, + feed_queue, + py_reader, + ) = simple_fc_net( in_size=self.in_size, class_num=self.class_num, hidden_sizes=self.hidden_sizes, batch_size=self.batch_size, queue_capacity=self.queue_capacity, use_double_buffer=self.use_double_buffer, - use_feed_list=self.use_feed_list) + use_feed_list=self.use_feed_list, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -222,12 +246,14 @@ class TestPyReaderUsingExecutor(unittest.TestCase): train_cp = main_program if use_parallel_executor: train_cp = compiler.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name) + main_program + ).with_data_parallel(loss_name=loss.name) if use_cuda: self.batch_size_times = core.get_cuda_device_count() else: self.batch_size_times = int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + os.environ.get('CPU_NUM', multiprocessing.cpu_count()) + ) else: self.batch_size_times = 1 @@ -244,15 +270,17 @@ class TestPyReaderUsingExecutor(unittest.TestCase): py_reader.decorate_sample_list_generator(batch_reader) py_reader.start() else: - thread = threading.Thread(target=feed_data, - args=(feed_queue, batch_reader)) + thread = threading.Thread( + target=feed_data, args=(feed_queue, batch_reader) + ) thread.daemon = True thread.start() try: while True: - fetches = exe.run(train_cp, - fetch_list=[in_data.name, label.name]) + fetches = exe.run( + train_cp, fetch_list=[in_data.name, label.name] + ) fetches = [as_numpy(fetch) for fetch in fetches] self.outputs.append(fetches) except fluid.core.EOFException: diff --git a/python/paddle/fluid/tests/unittests/test_pylayer_op.py b/python/paddle/fluid/tests/unittests/test_pylayer_op.py index 6a959dab07e..becab9796df 100644 --- a/python/paddle/fluid/tests/unittests/test_pylayer_op.py +++ b/python/paddle/fluid/tests/unittests/test_pylayer_op.py @@ -21,17 +21,13 @@ from paddle.fluid.framework import _test_eager_guard, in_dygraph_mode class FakeTensor(paddle.fluid.core.VarBase): - def __init__(self): pass class TestPyLayer(unittest.TestCase): - def func_test_simple_pylayer_multiple_output(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1, x2, func1, func2=paddle.square): ctx.func = func2 @@ -59,7 +55,8 @@ class TestPyLayer(unittest.TestCase): z2.mean().backward() self.assertTrue( - np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10) + np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10 + ) def test_simple_pylayer_multiple_output(self): with _test_eager_guard(): @@ -67,9 +64,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_simple_pylayer_multiple_output() def func_test_simple_pylayer_return_none_with_no_grad(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1, x2, func1, func2=paddle.square): ctx.func = func2 @@ -101,7 +96,8 @@ class TestPyLayer(unittest.TestCase): z2.mean().backward() self.assertTrue( - np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10) + np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10 + ) def test_simple_pylayer_return_none_with_no_grad(self): with _test_eager_guard(): @@ -109,9 +105,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_simple_pylayer_return_none_with_no_grad() def func_test_simple_pylayer_single_output(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1, func1, func2=paddle.square): ctx.func = func2 @@ -121,7 +115,7 @@ class TestPyLayer(unittest.TestCase): @staticmethod def backward(ctx, dy1): - y1, = ctx.saved_tensor() + (y1,) = ctx.saved_tensor() re1 = dy1 * (1 - ctx.func(y1)) return re1 @@ -135,7 +129,8 @@ class TestPyLayer(unittest.TestCase): z2.mean().backward() self.assertTrue( - np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10) + np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10 + ) def test_simple_pylayer_single_output(self): with _test_eager_guard(): @@ -143,9 +138,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_simple_pylayer_single_output() def func_test_pylayer_num_output_match(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward( ctx, @@ -172,9 +165,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_pylayer_num_output_match() def func_test_pylayer_dtype(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x, dtype): y = paddle.cast(x, dtype) @@ -185,10 +176,16 @@ class TestPyLayer(unittest.TestCase): return dy1 dtypes = [ - 'bool', 'float16', 'float32', 'float64', 'uint8', 'int32', 'int64' + 'bool', + 'float16', + 'float32', + 'float64', + 'uint8', + 'int32', + 'int64', ] for dtype in dtypes: - input1 = (paddle.randn([2, 3])) + input1 = paddle.randn([2, 3]) input1.stop_gradient = False self.assertTrue(input1.grad is None) @@ -203,9 +200,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_pylayer_dtype() def func_test_pylayer_Exception_forward(self): - class Layer_None1(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, *args): return None @@ -219,7 +214,6 @@ class TestPyLayer(unittest.TestCase): z = Layer_None1.apply(input1) class Layer_None2(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, *args): return [None, args[0]] @@ -233,7 +227,6 @@ class TestPyLayer(unittest.TestCase): z = Layer_None2.apply(input1) class Layer_one1(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, *args): return 1 @@ -248,7 +241,6 @@ class TestPyLayer(unittest.TestCase): z = Layer_one1.apply(input1) class Layer_one2(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, *args): return [1, 2, args[0]] @@ -262,7 +254,6 @@ class TestPyLayer(unittest.TestCase): z = Layer_one2.apply(input1) class Layer_no_fw(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def backward(ctx, *args): return args @@ -277,9 +268,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_pylayer_Exception_forward() def func_test_pylayer_nograd(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1, func1, func2=paddle.square, xx=None): ctx.func = func2 @@ -302,10 +291,9 @@ class TestPyLayer(unittest.TestCase): self.func_test_pylayer_nograd() def func_test_pylayer_Exception_bk(self): - class Layer_bk_none1( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x * 2 @@ -322,8 +310,8 @@ class TestPyLayer(unittest.TestCase): z.sum().backward() class Layer_bk_none2( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x1, x2): return x1 + x2 @@ -339,9 +327,9 @@ class TestPyLayer(unittest.TestCase): with self.assertRaises(ValueError): z.mean().backward() - class Layer_bk_one1(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class Layer_bk_one1( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x + x @@ -357,9 +345,9 @@ class TestPyLayer(unittest.TestCase): with self.assertRaises(ValueError): z.mean().backward() - class Layer_bk_one2(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class Layer_bk_one2( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x1, x2): return x1 * 2, x2 * 5 @@ -377,7 +365,6 @@ class TestPyLayer(unittest.TestCase): z.mean().backward() class Layer_no_bk(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x): return x * 2, x * 5 @@ -391,8 +378,8 @@ class TestPyLayer(unittest.TestCase): z.mean().backward() class Layer_bk_match( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x * 2, x * 5 @@ -414,10 +401,9 @@ class TestPyLayer(unittest.TestCase): self.func_test_pylayer_Exception_bk() def func_test_pylayer_bk_return_none(self): - class Layer_bk_none1( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x1, x2): return x1 + x2 @@ -436,8 +422,8 @@ class TestPyLayer(unittest.TestCase): z.mean().backward() class Layer_bk_none2( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x1, x2): return x1 * 2, x2 * 5 @@ -461,9 +447,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_pylayer_bk_return_none() def func_test_pylayer_inplace(self): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x): return x @@ -473,7 +457,6 @@ class TestPyLayer(unittest.TestCase): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -499,9 +482,9 @@ class TestPyLayer(unittest.TestCase): def test_pylayer_inplace_backward_error(self): with _test_eager_guard(): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class cus_tanh( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x @@ -511,7 +494,6 @@ class TestPyLayer(unittest.TestCase): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -527,17 +509,19 @@ class TestPyLayer(unittest.TestCase): layer = Layer() z = layer(data) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): z.backward() def test_pylayer_inplace_backward_success_1(self): with _test_eager_guard(): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class cus_tanh( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x @@ -547,7 +531,6 @@ class TestPyLayer(unittest.TestCase): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -569,9 +552,9 @@ class TestPyLayer(unittest.TestCase): def test_pylayer_inplace_backward_success_2(self): with _test_eager_guard(): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class cus_tanh( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x @@ -581,7 +564,6 @@ class TestPyLayer(unittest.TestCase): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -601,10 +583,9 @@ class TestPyLayer(unittest.TestCase): self.assertTrue(data.grad is not None) def func_test_pylayer_inplace_and_leaf_exception(self): - class cus_pylayer_op( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x @@ -614,7 +595,6 @@ class TestPyLayer(unittest.TestCase): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -636,9 +616,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_pylayer_inplace_and_leaf_exception() def func_test_backward_in_backward(self): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x): temp = x.detach() @@ -668,9 +646,7 @@ class TestPyLayer(unittest.TestCase): self.func_test_backward_in_backward() def func_test_return_to_tensor(self): - class Tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1): y1 = paddle.tanh(x1) @@ -680,7 +656,7 @@ class TestPyLayer(unittest.TestCase): @staticmethod def backward(ctx, dy1, dy2): - y1, = ctx.saved_tensor() + (y1,) = ctx.saved_tensor() re1 = dy1 * (1 - paddle.square(y1)) return dy1 @@ -700,7 +676,6 @@ class TestPyLayer(unittest.TestCase): with _test_eager_guard(): class Tanh(EagerPyLayer): - @staticmethod def forward(ctx, x): ctx.mark_not_inplace(x) @@ -719,7 +694,6 @@ class TestPyLayer(unittest.TestCase): with _test_eager_guard(): class Tanh(EagerPyLayer): - @staticmethod def forward(ctx, x): ctx.mark_not_inplace(x) @@ -739,7 +713,6 @@ class TestPyLayer(unittest.TestCase): with _test_eager_guard(): class Tanh(EagerPyLayer): - @staticmethod def forward(ctx, x): a = x + x @@ -760,7 +733,6 @@ class TestPyLayer(unittest.TestCase): with _test_eager_guard(): class Tanh(EagerPyLayer): - @staticmethod def forward(ctx, x): a = x + x @@ -782,11 +754,8 @@ class TestPyLayer(unittest.TestCase): class TestPyLayerReturnType(unittest.TestCase): - def test_forward_args_fake_tensor(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): y1 = FakeTensor() @@ -802,9 +771,7 @@ class TestPyLayerReturnType(unittest.TestCase): y1, y2 = Tanh.apply(input1) def test_forward_kwargs_fake_tensor(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): @@ -820,9 +787,7 @@ class TestPyLayerReturnType(unittest.TestCase): y = Tanh.apply(x1=input1) def test_forward_return_fake_tensor(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): @@ -838,9 +803,7 @@ class TestPyLayerReturnType(unittest.TestCase): y = Tanh.apply(x1=input1) def test_forward_return_fake_tensor_tuple(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): @@ -856,9 +819,7 @@ class TestPyLayerReturnType(unittest.TestCase): y = Tanh.apply(x1=input1) def test_backward_return_fake_tensor_tuple(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1, x2): return x1 + 1, x1 + 2 @@ -876,9 +837,7 @@ class TestPyLayerReturnType(unittest.TestCase): y.mean().backward() def test_backward_return_fake_tensor(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): return x1 + 1, x1 + 2 diff --git a/python/paddle/fluid/tests/unittests/test_pyramid_hash_op.py b/python/paddle/fluid/tests/unittests/test_pyramid_hash_op.py index 6f3f94253c8..d84730fa3bd 100644 --- a/python/paddle/fluid/tests/unittests/test_pyramid_hash_op.py +++ b/python/paddle/fluid/tests/unittests/test_pyramid_hash_op.py @@ -18,7 +18,6 @@ import paddle.fluid as fluid class TestPyramidHashOpApi(unittest.TestCase): - def test_api(self): num_voc = 128 embed_dim = 64 @@ -52,14 +51,14 @@ class TestPyramidHashOpApi(unittest.TestCase): place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.randint(0, num_voc, x_shape).astype('int32'), x_lod, - place) + np.random.randint(0, num_voc, x_shape).astype('int32'), x_lod, place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={'x': x_tensor}, - fetch_list=[hash_embd], - return_numpy=False) + ret = exe.run( + feed={'x': x_tensor}, fetch_list=[hash_embd], return_numpy=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_python_bf16_numpy_datatype.py b/python/paddle/fluid/tests/unittests/test_python_bf16_numpy_datatype.py index 3144e5da556..4a650dc07b9 100644 --- a/python/paddle/fluid/tests/unittests/test_python_bf16_numpy_datatype.py +++ b/python/paddle/fluid/tests/unittests/test_python_bf16_numpy_datatype.py @@ -18,7 +18,6 @@ import unittest class TestBF16DataType(unittest.TestCase): - def test_matmul(self): a_bf16 = np.random.random((6, 7)).astype(bfloat16) b_bf16 = np.random.random((7, 8)).astype(bfloat16) diff --git a/python/paddle/fluid/tests/unittests/test_qr_op.py b/python/paddle/fluid/tests/unittests/test_qr_op.py index 4d8bfb220fc..39360d2e721 100644 --- a/python/paddle/fluid/tests/unittests/test_qr_op.py +++ b/python/paddle/fluid/tests/unittests/test_qr_op.py @@ -22,7 +22,6 @@ from op_test import OpTest class TestQrOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = paddle.linalg.qr @@ -73,38 +72,36 @@ class TestQrOp(OpTest): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X'], ['Q', 'R'], - check_eager=True, - numeric_grad_delta=1e-5, - max_relative_error=1e-6) + self.check_grad( + ['X'], + ['Q', 'R'], + check_eager=True, + numeric_grad_delta=1e-5, + max_relative_error=1e-6, + ) class TestQrOpCase1(TestQrOp): - def get_shape(self): return (10, 12) class TestQrOpCase2(TestQrOp): - def get_shape(self): return (16, 15) class TestQrOpCase3(TestQrOp): - def get_shape(self): return (2, 12, 16) class TestQrOpCase4(TestQrOp): - def get_shape(self): return (3, 16, 15) class TestQrOpCase5(TestQrOp): - def get_mode(self): return "complete" @@ -113,7 +110,6 @@ class TestQrOpCase5(TestQrOp): class TestQrOpCase6(TestQrOp): - def get_mode(self): return "complete" @@ -122,7 +118,6 @@ class TestQrOpCase6(TestQrOp): class TestQrAPI(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() np.random.seed(7) @@ -180,12 +175,13 @@ class TestQrAPI(unittest.TestCase): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] modes = ["reduced", "complete", "r"] dtypes = ["float32", "float64"] for tensor_shape, mode, dtype in itertools.product( - tensor_shapes, modes, dtypes): + tensor_shapes, modes, dtypes + ): run_qr_dygraph(tensor_shape, mode, dtype) def test_static(self): @@ -227,33 +223,34 @@ class TestQrAPI(unittest.TestCase): tmp_q, tmp_r = np.linalg.qr(a[coord], mode=mode) np_q[coord] = tmp_q np_r[coord] = tmp_r - x = paddle.fluid.data(name="input", - shape=shape, - dtype=dtype) + x = paddle.fluid.data( + name="input", shape=shape, dtype=dtype + ) if mode == "r": r = paddle.linalg.qr(x, mode=mode) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[r]) - np.testing.assert_allclose(fetches[0], - np_r, - rtol=1e-05, - atol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[r], + ) + np.testing.assert_allclose( + fetches[0], np_r, rtol=1e-05, atol=1e-05 + ) else: q, r = paddle.linalg.qr(x, mode=mode) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[q, r]) - np.testing.assert_allclose(fetches[0], - np_q, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(fetches[1], - np_r, - rtol=1e-05, - atol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[q, r], + ) + np.testing.assert_allclose( + fetches[0], np_q, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + fetches[1], np_r, rtol=1e-05, atol=1e-05 + ) tensor_shapes = [ (3, 5), @@ -264,12 +261,13 @@ class TestQrAPI(unittest.TestCase): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] modes = ["reduced", "complete", "r"] dtypes = ["float32", "float64"] for tensor_shape, mode, dtype in itertools.product( - tensor_shapes, modes, dtypes): + tensor_shapes, modes, dtypes + ): run_qr_static(tensor_shape, mode, dtype) diff --git a/python/paddle/fluid/tests/unittests/test_quantile_and_nanquantile.py b/python/paddle/fluid/tests/unittests/test_quantile_and_nanquantile.py index 259e5556dee..3d7406c457e 100644 --- a/python/paddle/fluid/tests/unittests/test_quantile_and_nanquantile.py +++ b/python/paddle/fluid/tests/unittests/test_quantile_and_nanquantile.py @@ -16,15 +16,17 @@ import unittest import numpy as np import paddle -API_list = [(paddle.quantile, np.quantile), - (paddle.nanquantile, np.nanquantile)] +API_list = [ + (paddle.quantile, np.quantile), + (paddle.nanquantile, np.nanquantile), +] class TestQuantileAndNanquantile(unittest.TestCase): """ - This class is used for numerical precision testing. If there is - a corresponding numpy API, the precision comparison can be performed directly. - Otherwise, it needs to be verified by numpy implementated function. + This class is used for numerical precision testing. If there is + a corresponding numpy API, the precision comparison can be performed directly. + Otherwise, it needs to be verified by numpy implementated function. """ def setUp(self): @@ -99,10 +101,9 @@ class TestQuantileAndNanquantile(unittest.TestCase): x = paddle.to_tensor(input_data) paddle_res = paddle.quantile(x, q=0.35, axis=0) np_res = np.quantile(input_data, q=0.35, axis=0) - np.testing.assert_allclose(paddle_res.numpy(), - np_res, - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + paddle_res.numpy(), np_res, rtol=1e-05, equal_nan=True + ) # Test correctness when input filled with NaN. def test_nanquantile_all_NaN(self): @@ -111,15 +112,14 @@ class TestQuantileAndNanquantile(unittest.TestCase): x = paddle.to_tensor(input_data) paddle_res = paddle.nanquantile(x, q=0.35, axis=0) np_res = np.nanquantile(input_data, q=0.35, axis=0) - np.testing.assert_allclose(paddle_res.numpy(), - np_res, - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + paddle_res.numpy(), np_res, rtol=1e-05, equal_nan=True + ) class TestMuitlpleQ(unittest.TestCase): """ - This class is used to test multiple input of q. + This class is used to test multiple input of q. """ def setUp(self): @@ -139,21 +139,19 @@ class TestMuitlpleQ(unittest.TestCase): def test_quantile_multiple_axis_keepdim(self): x = paddle.to_tensor(self.input_data) - paddle_res = paddle.quantile(x, - q=[0.1, 0.2, 0.3], - axis=[1, 2], - keepdim=True) - np_res = np.quantile(self.input_data, - q=[0.1, 0.2, 0.3], - axis=[1, 2], - keepdims=True) + paddle_res = paddle.quantile( + x, q=[0.1, 0.2, 0.3], axis=[1, 2], keepdim=True + ) + np_res = np.quantile( + self.input_data, q=[0.1, 0.2, 0.3], axis=[1, 2], keepdims=True + ) np.testing.assert_allclose(paddle_res.numpy(), np_res, rtol=1e-05) class TestError(unittest.TestCase): """ - This class is used to test that exceptions are thrown correctly. - Validity of all parameter values and types should be considered. + This class is used to test that exceptions are thrown correctly. + Validity of all parameter values and types should be considered. """ def setUp(self): @@ -218,8 +216,8 @@ class TestError(unittest.TestCase): class TestQuantileRuntime(unittest.TestCase): """ - This class is used to test the API could run correctly with - different devices, different data types, and dygraph/static mode. + This class is used to test the API could run correctly with + different devices, different data types, and dygraph/static mode. """ def setUp(self): @@ -241,20 +239,22 @@ class TestQuantileRuntime(unittest.TestCase): x = paddle.to_tensor(np_input_data, dtype=dtype) paddle_res = func(x, q=0.5, axis=1) np_res = res_func(np_input_data, q=0.5, axis=1) - np.testing.assert_allclose(paddle_res.numpy(), - np_res, - rtol=1e-05) + np.testing.assert_allclose( + paddle_res.numpy(), np_res, rtol=1e-05 + ) def test_static(self): paddle.enable_static() for (func, res_func) in API_list: for device in self.devices: - x = paddle.static.data(name="x", - shape=self.input_data.shape, - dtype=paddle.float32) - x_fp64 = paddle.static.data(name="x_fp64", - shape=self.input_data.shape, - dtype=paddle.float64) + x = paddle.static.data( + name="x", shape=self.input_data.shape, dtype=paddle.float32 + ) + x_fp64 = paddle.static.data( + name="x_fp64", + shape=self.input_data.shape, + dtype=paddle.float64, + ) results = func(x, q=0.5, axis=1) np_input_data = self.input_data.astype('float32') @@ -264,16 +264,15 @@ class TestQuantileRuntime(unittest.TestCase): exe = paddle.static.Executor(device) paddle_res, paddle_res_fp64 = exe.run( paddle.static.default_main_program(), - feed={ - "x": np_input_data, - "x_fp64": np_input_data_fp64 - }, - fetch_list=[results, results_fp64]) + feed={"x": np_input_data, "x_fp64": np_input_data_fp64}, + fetch_list=[results, results_fp64], + ) np_res = res_func(np_input_data, q=0.5, axis=1) np_res_fp64 = res_func(np_input_data_fp64, q=0.5, axis=1) self.assertTrue( np.allclose(paddle_res, np_res) - and np.allclose(paddle_res_fp64, np_res_fp64)) + and np.allclose(paddle_res_fp64, np_res_fp64) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_query_op.py b/python/paddle/fluid/tests/unittests/test_query_op.py index 162c1f440e6..a97530febfa 100644 --- a/python/paddle/fluid/tests/unittests/test_query_op.py +++ b/python/paddle/fluid/tests/unittests/test_query_op.py @@ -18,7 +18,6 @@ from paddle.fluid import core class TestCudnnVersion(unittest.TestCase): - def test_no_cudnn(self): cudnn_version = paddle.get_cudnn_version() if not core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_queue.py b/python/paddle/fluid/tests/unittests/test_queue.py index 703c11e710c..bfc2f1e0e6f 100644 --- a/python/paddle/fluid/tests/unittests/test_queue.py +++ b/python/paddle/fluid/tests/unittests/test_queue.py @@ -21,7 +21,6 @@ import paddle.fluid.core as core class TestQueue(unittest.TestCase): - def test_eq(self): """ test queue_generator op, enqueue op and dequeue op. @@ -31,39 +30,51 @@ class TestQueue(unittest.TestCase): startup_program = fluid.Program() value = np.random.rand(1) with fluid.program_guard(main_program, startup_program): - data_in = layers.create_global_var(shape=[2, 3], - value=value, - dtype="float32", - persistable=True, - name='var_in') - data_out = layers.create_global_var(shape=[2, 3], - value=value - 1.0, - dtype="float32", - persistable=True, - name='var_out') + data_in = layers.create_global_var( + shape=[2, 3], + value=value, + dtype="float32", + persistable=True, + name='var_in', + ) + data_out = layers.create_global_var( + shape=[2, 3], + value=value - 1.0, + dtype="float32", + persistable=True, + name='var_out', + ) startup_block = startup_program.block(0) queue_name = 'blocking_queue' - startup_block.create_var(name=queue_name, - persistable=True, - type=core.VarDesc.VarType.RAW) - startup_block.append_op(type="queue_generator", - attrs={'names': [queue_name]}) + startup_block.create_var( + name=queue_name, persistable=True, type=core.VarDesc.VarType.RAW + ) + startup_block.append_op( + type="queue_generator", attrs={'names': [queue_name]} + ) block = main_program.block(0) - block.append_op(type='enqueue', - inputs={'X': data_in}, - attrs={'queue_name': queue_name}) - block.append_op(type='dequeue', - outputs={'Out': [data_out]}, - attrs={'queue_name': queue_name}) + block.append_op( + type='enqueue', + inputs={'X': data_in}, + attrs={'queue_name': queue_name}, + ) + block.append_op( + type='dequeue', + outputs={'Out': [data_out]}, + attrs={'queue_name': queue_name}, + ) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup_program) - ret, = exe.run(main_program, fetch_list=[data_out.name]) - np.testing.assert_allclose(np.asarray(ret), - np.full((2, 3), value, np.float32), - rtol=1e-05) + (ret,) = exe.run(main_program, fetch_list=[data_out.name]) + np.testing.assert_allclose( + np.asarray(ret), np.full((2, 3), value, np.float32), rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_rad2deg.py b/python/paddle/fluid/tests/unittests/test_rad2deg.py index 00cf6e9fad5..220cf96f04f 100644 --- a/python/paddle/fluid/tests/unittests/test_rad2deg.py +++ b/python/paddle/fluid/tests/unittests/test_rad2deg.py @@ -22,11 +22,11 @@ paddle.enable_static() class TestRad2degAPI(unittest.TestCase): - def setUp(self): self.x_dtype = 'float64' - self.x_np = np.array([3.142, -3.142, 6.283, -6.283, 1.570, - -1.570]).astype(np.float64) + self.x_np = np.array( + [3.142, -3.142, 6.283, -6.283, 1.570, -1.570] + ).astype(np.float64) self.x_shape = [6] self.out_np = np.rad2deg(self.x_np) @@ -37,12 +37,17 @@ class TestRad2degAPI(unittest.TestCase): x = fluid.data(name='input', dtype=self.x_dtype, shape=self.x_shape) out = paddle.rad2deg(x) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(fluid.default_main_program(), - feed={'input': self.x_np}, - fetch_list=[out]) + res = exe.run( + fluid.default_main_program(), + feed={'input': self.x_np}, + fetch_list=[out], + ) self.assertTrue((np.array(out[0]) == self.out_np).all()) def test_dygraph(self): @@ -55,7 +60,6 @@ class TestRad2degAPI(unittest.TestCase): class TestRad2degAPI2(TestRad2degAPI): - def setUp(self): self.x_np = np.pi / 2 self.x_shape = [1] diff --git a/python/paddle/fluid/tests/unittests/test_rand_op.py b/python/paddle/fluid/tests/unittests/test_rand_op.py index b889d79b078..35038054144 100644 --- a/python/paddle/fluid/tests/unittests/test_rand_op.py +++ b/python/paddle/fluid/tests/unittests/test_rand_op.py @@ -33,8 +33,9 @@ class TestRandOpError(unittest.TestCase): with program_guard(main_prog, start_prog): def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros((4, 784)), [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((4, 784)), [[1, 1, 1, 1]], fluid.CPUPlace() + ) rand(x1) self.assertRaises(TypeError, test_Variable) @@ -69,9 +70,9 @@ class TestRandOp(unittest.TestCase): var_shape = fluid.data(name='var_shape', shape=[2], dtype="int64") result_3 = rand(var_shape) - var_shape_int32 = fluid.data(name='var_shape_int32', - shape=[2], - dtype="int32") + var_shape_int32 = fluid.data( + name='var_shape_int32', shape=[2], dtype="int32" + ) result_4 = rand(var_shape_int32) exe.run(startup_program) @@ -80,11 +81,9 @@ class TestRandOp(unittest.TestCase): x2 = np.array([4, 3]).astype('int32') ret = exe.run( train_program, - feed={ - "var_shape": x1, - "var_shape_int32": x2 - }, - fetch_list=[result_1, result_1, result_2, result_3, result_4]) + feed={"var_shape": x1, "var_shape_int32": x2}, + fetch_list=[result_1, result_1, result_2, result_3, result_4], + ) def test_run(self): self.run_net(False) @@ -118,7 +117,6 @@ class TestRandOpForDygraph(unittest.TestCase): class TestRandDtype(unittest.TestCase): - def test_default_dtype(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_randint_like.py b/python/paddle/fluid/tests/unittests/test_randint_like.py index 972243f24b5..32a717d3815 100644 --- a/python/paddle/fluid/tests/unittests/test_randint_like.py +++ b/python/paddle/fluid/tests/unittests/test_randint_like.py @@ -20,7 +20,6 @@ from paddle.static import program_guard, Program # Test python API class TestRandintLikeAPI(unittest.TestCase): - def setUp(self): self.x_bool = np.zeros((10, 12)).astype("bool") self.x_int32 = np.zeros((10, 12)).astype("int32") @@ -30,16 +29,19 @@ class TestRandintLikeAPI(unittest.TestCase): self.x_float64 = np.zeros((10, 12)).astype("float64") self.dtype = ["bool", "int32", "int64", "float16", "float32", "float64"] - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() with program_guard(Program(), Program()): # results are from [-100, 100). - x_bool = paddle.fluid.data(name="x_bool", - shape=[10, 12], - dtype="bool") + x_bool = paddle.fluid.data( + name="x_bool", shape=[10, 12], dtype="bool" + ) exe = paddle.static.Executor(self.place) # x dtype is bool output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist1 = [ @@ -51,9 +53,9 @@ class TestRandintLikeAPI(unittest.TestCase): self.assertTrue(out.dtype, np.dtype(dtype)) self.assertTrue(((out >= -10) & (out <= 10)).all(), True) with program_guard(Program(), Program()): - x_int32 = paddle.fluid.data(name="x_int32", - shape=[10, 12], - dtype="int32") + x_int32 = paddle.fluid.data( + name="x_int32", shape=[10, 12], dtype="int32" + ) exe = paddle.static.Executor(self.place) # x dtype is int32 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist2 = [ @@ -66,9 +68,9 @@ class TestRandintLikeAPI(unittest.TestCase): self.assertTrue(((out >= -5) & (out <= 10)).all(), True) with program_guard(Program(), Program()): - x_int64 = paddle.fluid.data(name="x_int64", - shape=[10, 12], - dtype="int64") + x_int64 = paddle.fluid.data( + name="x_int64", shape=[10, 12], dtype="int64" + ) exe = paddle.static.Executor(self.place) # x dtype is int64 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist3 = [ @@ -81,49 +83,52 @@ class TestRandintLikeAPI(unittest.TestCase): self.assertTrue(((out >= -100) & (out <= 100)).all(), True) if paddle.is_compiled_with_cuda(): with program_guard(Program(), Program()): - x_float16 = paddle.fluid.data(name="x_float16", - shape=[10, 12], - dtype="float16") + x_float16 = paddle.fluid.data( + name="x_float16", shape=[10, 12], dtype="float16" + ) exe = paddle.static.Executor(self.place) # x dtype is float16 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist4 = [ paddle.randint_like(x_float16, low=-3, high=25, dtype=dtype) for dtype in self.dtype ] - outs4 = exe.run(feed={'x_float16': self.x_float16}, - fetch_list=outlist4) + outs4 = exe.run( + feed={'x_float16': self.x_float16}, fetch_list=outlist4 + ) for out, dtype in zip(outs4, self.dtype): self.assertTrue(out.dtype, np.dtype(dtype)) self.assertTrue(((out >= -3) & (out <= 25)).all(), True) with program_guard(Program(), Program()): - x_float32 = paddle.fluid.data(name="x_float32", - shape=[10, 12], - dtype="float32") + x_float32 = paddle.fluid.data( + name="x_float32", shape=[10, 12], dtype="float32" + ) exe = paddle.static.Executor(self.place) # x dtype is float32 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist5 = [ paddle.randint_like(x_float32, low=-25, high=25, dtype=dtype) for dtype in self.dtype ] - outs5 = exe.run(feed={'x_float32': self.x_float32}, - fetch_list=outlist5) + outs5 = exe.run( + feed={'x_float32': self.x_float32}, fetch_list=outlist5 + ) for out, dtype in zip(outs5, self.dtype): self.assertTrue(out.dtype, np.dtype(dtype)) self.assertTrue(((out >= -25) & (out <= 25)).all(), True) with program_guard(Program(), Program()): - x_float64 = paddle.fluid.data(name="x_float64", - shape=[10, 12], - dtype="float64") + x_float64 = paddle.fluid.data( + name="x_float64", shape=[10, 12], dtype="float64" + ) exe = paddle.static.Executor(self.place) # x dtype is float64 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist6 = [ paddle.randint_like(x_float64, low=-16, high=16, dtype=dtype) for dtype in self.dtype ] - outs6 = exe.run(feed={'x_float64': self.x_float64}, - fetch_list=outlist6) + outs6 = exe.run( + feed={'x_float64': self.x_float64}, fetch_list=outlist6 + ) for out, dtype in zip(outs6, self.dtype): self.assertTrue(out.dtype, dtype) self.assertTrue(((out >= -16) & (out <= 16)).all(), True) @@ -132,62 +137,63 @@ class TestRandintLikeAPI(unittest.TestCase): paddle.disable_static(self.place) # x dtype ["bool", "int32", "int64", "float32", "float64"] for x in [ - self.x_bool, self.x_int32, self.x_int64, self.x_float32, - self.x_float64 + self.x_bool, + self.x_int32, + self.x_int64, + self.x_float32, + self.x_float64, ]: x_inputs = paddle.to_tensor(x) # self.dtype ["bool", "int32", "int64", "float16", "float32", "float64"] for dtype in self.dtype: - out = paddle.randint_like(x_inputs, - low=-100, - high=100, - dtype=dtype) + out = paddle.randint_like( + x_inputs, low=-100, high=100, dtype=dtype + ) self.assertTrue(out.numpy().dtype, np.dtype(dtype)) self.assertTrue( - ((out.numpy() >= -100) & (out.numpy() <= 100)).all(), True) + ((out.numpy() >= -100) & (out.numpy() <= 100)).all(), True + ) # x dtype ["float16"] if paddle.is_compiled_with_cuda(): x_inputs = paddle.to_tensor(self.x_float16) # self.dtype ["bool", "int32", "int64", "float16", "float32", "float64"] for dtype in self.dtype: - out = paddle.randint_like(x_inputs, - low=-100, - high=100, - dtype=dtype) + out = paddle.randint_like( + x_inputs, low=-100, high=100, dtype=dtype + ) self.assertTrue(out.numpy().dtype, np.dtype(dtype)) self.assertTrue( - ((out.numpy() >= -100) & (out.numpy() <= 100)).all(), True) + ((out.numpy() >= -100) & (out.numpy() <= 100)).all(), True + ) paddle.enable_static() def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): - x_bool = paddle.fluid.data(name="x_bool", - shape=[10, 12], - dtype="bool") - x_int32 = paddle.fluid.data(name="x_int32", - shape=[10, 12], - dtype="int32") - x_int64 = paddle.fluid.data(name="x_int64", - shape=[10, 12], - dtype="int64") - x_float16 = paddle.fluid.data(name="x_float16", - shape=[10, 12], - dtype="float16") - x_float32 = paddle.fluid.data(name="x_float32", - shape=[10, 12], - dtype="float32") - x_float64 = paddle.fluid.data(name="x_float64", - shape=[10, 12], - dtype="float64") + x_bool = paddle.fluid.data( + name="x_bool", shape=[10, 12], dtype="bool" + ) + x_int32 = paddle.fluid.data( + name="x_int32", shape=[10, 12], dtype="int32" + ) + x_int64 = paddle.fluid.data( + name="x_int64", shape=[10, 12], dtype="int64" + ) + x_float16 = paddle.fluid.data( + name="x_float16", shape=[10, 12], dtype="float16" + ) + x_float32 = paddle.fluid.data( + name="x_float32", shape=[10, 12], dtype="float32" + ) + x_float64 = paddle.fluid.data( + name="x_float64", shape=[10, 12], dtype="float64" + ) # x dtype is bool # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_bool, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_bool, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high self.assertRaises(ValueError, paddle.randint_like, x_bool, high=-5) # if high is None, low must be greater than 0 @@ -195,11 +201,9 @@ class TestRandintLikeAPI(unittest.TestCase): # x dtype is int32 # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_int32, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_int32, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high self.assertRaises(ValueError, paddle.randint_like, x_int32, high=-5) # if high is None, low must be greater than 0 @@ -207,11 +211,9 @@ class TestRandintLikeAPI(unittest.TestCase): # x dtype is int64 # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_int64, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_int64, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high self.assertRaises(ValueError, paddle.randint_like, x_int64, high=-5) # if high is None, low must be greater than 0 @@ -220,57 +222,45 @@ class TestRandintLikeAPI(unittest.TestCase): # x dtype is float16 # low is 5 and high is 5, low must less then high if paddle.is_compiled_with_cuda(): - self.assertRaises(ValueError, - paddle.randint_like, - x_float16, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_float16, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float16, - high=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float16, high=-5 + ) # if high is None, low must be greater than 0 - self.assertRaises(ValueError, - paddle.randint_like, - x_float16, - low=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float16, low=-5 + ) # x dtype is float32 # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float32, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_float32, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float32, - high=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float32, high=-5 + ) # if high is None, low must be greater than 0 - self.assertRaises(ValueError, - paddle.randint_like, - x_float32, - low=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float32, low=-5 + ) # x dtype is float64 # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float64, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_float64, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float64, - high=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float64, high=-5 + ) # if high is None, low must be greater than 0 - self.assertRaises(ValueError, - paddle.randint_like, - x_float64, - low=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float64, low=-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_randint_op.py b/python/paddle/fluid/tests/unittests/test_randint_op.py index 549554733fe..505cd43923a 100644 --- a/python/paddle/fluid/tests/unittests/test_randint_op.py +++ b/python/paddle/fluid/tests/unittests/test_randint_op.py @@ -32,7 +32,6 @@ def output_hist(out): class TestRandintOp(OpTest): - def setUp(self): self.op_type = "randint" self.inputs = {} @@ -56,7 +55,6 @@ class TestRandintOp(OpTest): class TestRandintOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(TypeError, paddle.randint, 5, shape=np.array([2])) @@ -66,10 +64,9 @@ class TestRandintOpError(unittest.TestCase): self.assertRaises(TypeError, paddle.randint, 5, shape=['2']) shape_tensor = paddle.static.data('X', [1]) self.assertRaises(TypeError, paddle.randint, 5, shape=shape_tensor) - self.assertRaises(TypeError, - paddle.randint, - 5, - shape=[shape_tensor]) + self.assertRaises( + TypeError, paddle.randint, 5, shape=[shape_tensor] + ) def test_errors_eager(self): with _test_eager_guard(): @@ -77,14 +74,14 @@ class TestRandintOpError(unittest.TestCase): class TestRandintOp_attr_tensorlist(OpTest): - def setUp(self): self.op_type = "randint" self.new_shape = (10000, 784) shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype("int64") * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype("int64") * ele) + ) self.inputs = {'ShapeTensorList': shape_tensor} self.init_attrs() self.outputs = {"Out": np.zeros((10000, 784)).astype("int32")} @@ -106,7 +103,6 @@ class TestRandintOp_attr_tensorlist(OpTest): class TestRandint_attr_tensor(OpTest): - def setUp(self): self.op_type = "randint" self.inputs = {"ShapeTensor": np.array([10000, 784]).astype("int64")} @@ -131,43 +127,42 @@ class TestRandint_attr_tensor(OpTest): # Test python API class TestRandintAPI(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): # results are from [0, 5). out1 = paddle.randint(5) # shape is a list and dtype is 'int32' - out2 = paddle.randint(low=-100, - high=100, - shape=[64, 64], - dtype='int32') + out2 = paddle.randint( + low=-100, high=100, shape=[64, 64], dtype='int32' + ) # shape is a tuple and dtype is 'int64' - out3 = paddle.randint(low=-100, - high=100, - shape=(32, 32, 3), - dtype='int64') + out3 = paddle.randint( + low=-100, high=100, shape=(32, 32, 3), dtype='int64' + ) # shape is a tensorlist and dtype is 'float32' dim_1 = paddle.fluid.layers.fill_constant([1], "int64", 32) dim_2 = paddle.fluid.layers.fill_constant([1], "int32", 50) - out4 = paddle.randint(low=-100, - high=100, - shape=[dim_1, 5, dim_2], - dtype='int32') + out4 = paddle.randint( + low=-100, high=100, shape=[dim_1, 5, dim_2], dtype='int32' + ) # shape is a tensor and dtype is 'float64' - var_shape = paddle.static.data(name='var_shape', - shape=[2], - dtype="int64") - out5 = paddle.randint(low=1, - high=1000, - shape=var_shape, - dtype='int64') - - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + var_shape = paddle.static.data( + name='var_shape', shape=[2], dtype="int64" + ) + out5 = paddle.randint( + low=1, high=1000, shape=var_shape, dtype='int64' + ) + + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) outs = exe.run( feed={'var_shape': np.array([100, 100]).astype('int64')}, - fetch_list=[out1, out2, out3, out4, out5]) + fetch_list=[out1, out2, out3, out4, out5], + ) def test_api_eager(self): with _test_eager_guard(): @@ -175,7 +170,6 @@ class TestRandintAPI(unittest.TestCase): class TestRandintImperative(unittest.TestCase): - def test_api(self): paddle.disable_static() @@ -197,7 +191,6 @@ class TestRandintImperative(unittest.TestCase): class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -221,8 +214,9 @@ class TestRandomValue(unittest.TestCase): paddle.set_device('gpu') paddle.seed(100) - x = paddle.randint(-10000, 10000, [32, 3, 1024, 1024], - dtype='int32').numpy() + x = paddle.randint( + -10000, 10000, [32, 3, 1024, 1024], dtype='int32' + ).numpy() self.assertTrue(x.mean(), -0.7517569760481516) self.assertTrue(x.std(), 5773.696619107639) expect = [2535, 2109, 5916, -5011, -261] @@ -232,8 +226,9 @@ class TestRandomValue(unittest.TestCase): expect = [881, 1560, 1100, 9664, 1669] np.testing.assert_array_equal(x[30, 2, 1000, 1000:1005], expect) - x = paddle.randint(-10000, 10000, [32, 3, 1024, 1024], - dtype='int64').numpy() + x = paddle.randint( + -10000, 10000, [32, 3, 1024, 1024], dtype='int64' + ).numpy() self.assertTrue(x.mean(), -1.461287518342336) self.assertTrue(x.std(), 5773.023477548159) expect = [7213, -9597, 754, 8129, -1158] diff --git a/python/paddle/fluid/tests/unittests/test_randn_op.py b/python/paddle/fluid/tests/unittests/test_randn_op.py index 4590af0f10f..d136e3fd57e 100644 --- a/python/paddle/fluid/tests/unittests/test_randn_op.py +++ b/python/paddle/fluid/tests/unittests/test_randn_op.py @@ -20,7 +20,6 @@ from paddle.static import program_guard, Program class TestRandnOp(unittest.TestCase): - def test_api(self): shape = [1000, 784] train_program = Program() @@ -36,24 +35,31 @@ class TestRandnOp(unittest.TestCase): var_shape = paddle.static.data('X', [2], 'int32') x4 = paddle.randn(var_shape) - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - res = exe.run(train_program, - feed={'X': np.array(shape, dtype='int32')}, - fetch_list=[x1, x2, x3, x4]) + res = exe.run( + train_program, + feed={'X': np.array(shape, dtype='int32')}, + fetch_list=[x1, x2, x3, x4], + ) for out in res: - self.assertAlmostEqual(np.mean(out), .0, delta=0.1) - self.assertAlmostEqual(np.std(out), 1., delta=0.1) + self.assertAlmostEqual(np.mean(out), 0.0, delta=0.1) + self.assertAlmostEqual(np.std(out), 1.0, delta=0.1) class TestRandnOpForDygraph(unittest.TestCase): - def test_api(self): shape = [1000, 784] - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) paddle.disable_static(place) x1 = paddle.randn(shape, 'float32') x2 = paddle.randn(shape, 'float64') @@ -66,13 +72,12 @@ class TestRandnOpForDygraph(unittest.TestCase): x4 = paddle.randn(var_shape) for out in [x1, x2, x3, x4]: - self.assertAlmostEqual(np.mean(out.numpy()), .0, delta=0.1) - self.assertAlmostEqual(np.std(out.numpy()), 1., delta=0.1) + self.assertAlmostEqual(np.mean(out.numpy()), 0.0, delta=0.1) + self.assertAlmostEqual(np.std(out.numpy()), 1.0, delta=0.1) paddle.enable_static() class TestRandnOpError(unittest.TestCase): - def test_error(self): with program_guard(Program(), Program()): # The argument shape's type of randn_op should be list or tuple. diff --git a/python/paddle/fluid/tests/unittests/test_random_crop_op.py b/python/paddle/fluid/tests/unittests/test_random_crop_op.py index 580d348156c..ae474618733 100644 --- a/python/paddle/fluid/tests/unittests/test_random_crop_op.py +++ b/python/paddle/fluid/tests/unittests/test_random_crop_op.py @@ -19,15 +19,15 @@ import paddle.fluid as fluid class TestRandomCropOp(OpTest): - def setUp(self): - to_crop = np.array([[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]] * - 5).astype(np.int32) + to_crop = np.array( + [[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]] * 5 + ).astype(np.int32) self.possible_res = [ np.array([[1, 2, 3], [5, 6, 7]]).astype(np.int32), np.array([[2, 3, 4], [6, 7, 8]]).astype(np.int32), np.array([[5, 6, 7], [9, 10, 11]]).astype(np.int32), - np.array([[6, 7, 8], [10, 11, 12]]).astype(np.int32) + np.array([[6, 7, 8], [10, 11, 12]]).astype(np.int32), ] self.op_type = "random_crop" self.inputs = {'X': to_crop, 'Seed': np.array([10]).astype('int64')} @@ -45,7 +45,6 @@ class TestRandomCropOp(OpTest): class TestRandomCropOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): @@ -56,17 +55,17 @@ class TestRandomCropOpError(unittest.TestCase): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[None, 3, 256, 256], - dtype='float16') + x2 = fluid.layers.data( + name='x2', shape=[None, 3, 256, 256], dtype='float16' + ) fluid.layers.random_crop(x2) self.assertRaises(TypeError, test_x_dtype) def test_shape_type(): - x3 = fluid.layers.data(name='x3', - shape=[None, 3, 256, 256], - dtype='float32') + x3 = fluid.layers.data( + name='x3', shape=[None, 3, 256, 256], dtype='float32' + ) fluid.layers.random_crop(x3, shape=1) self.assertRaises(TypeError, test_shape_type) diff --git a/python/paddle/fluid/tests/unittests/test_random_routing_op.py b/python/paddle/fluid/tests/unittests/test_random_routing_op.py index 54313b4c10c..03cce6c9caa 100644 --- a/python/paddle/fluid/tests/unittests/test_random_routing_op.py +++ b/python/paddle/fluid/tests/unittests/test_random_routing_op.py @@ -32,22 +32,24 @@ def random_routing(topk_idx, topk_value, prob, topk=2): raise RuntimeError("only topk=2 is supported now") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNumberCountAPIFp32(unittest.TestCase): - def setUp(self): self.dtype = "float32" self.init() def init(self): self.upper_range = 8 - self.x = np.random.randint(-1, self.upper_range, - size=(200, 2)).astype('int64') - self.prob = np.random.random((self.x.shape[0], )).astype(self.dtype) + self.x = np.random.randint(-1, self.upper_range, size=(200, 2)).astype( + 'int64' + ) + self.prob = np.random.random((self.x.shape[0],)).astype(self.dtype) self.topk_value = np.random.random(self.x.shape).astype(self.dtype) - self.out = random_routing(self.x, self.topk_value, - self.prob).astype(self.dtype) + self.out = random_routing(self.x, self.topk_value, self.prob).astype( + self.dtype + ) self.place = paddle.CUDAPlace(0) def func_api_dygraph(self): @@ -64,10 +66,10 @@ class TestNumberCountAPIFp32(unittest.TestCase): self.func_api_dygraph() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNumberCountAPIFp16(TestNumberCountAPIFp32): - def setUp(self): self.dtype = "float16" self.init() diff --git a/python/paddle/fluid/tests/unittests/test_random_seed.py b/python/paddle/fluid/tests/unittests/test_random_seed.py index c49808f3ae5..97f3fa56dc5 100644 --- a/python/paddle/fluid/tests/unittests/test_random_seed.py +++ b/python/paddle/fluid/tests/unittests/test_random_seed.py @@ -36,23 +36,20 @@ class TestGeneratorSeed(unittest.TestCase): x = fluid.layers.uniform_random([10], dtype="float32", min=0.0, max=1.0) st1 = gen.get_state() - x1 = fluid.layers.uniform_random([10], - dtype="float32", - min=0.0, - max=1.0) + x1 = fluid.layers.uniform_random( + [10], dtype="float32", min=0.0, max=1.0 + ) gen.set_state(st1) print(gen.get_state()) - x2 = fluid.layers.uniform_random([10], - dtype="float32", - min=0.0, - max=1.0) + x2 = fluid.layers.uniform_random( + [10], dtype="float32", min=0.0, max=1.0 + ) paddle.seed(12312321111) - x3 = fluid.layers.uniform_random([10], - dtype="float32", - min=0.0, - max=1.0) + x3 = fluid.layers.uniform_random( + [10], dtype="float32", min=0.0, max=1.0 + ) x_np = x.numpy() x1_np = x1.numpy() @@ -78,14 +75,14 @@ class TestGeneratorSeed(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) - #gen.set_state(cur_state) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) + # gen.set_state(cur_state) gen.manual_seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -103,17 +100,15 @@ class TestGeneratorSeed(unittest.TestCase): gen = paddle.seed(111111111) st = gen.get_state() # x = np.arange(1,101).reshape(2,50).astype("float32") - x = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) + x = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) y = fluid.layers.dropout(x, 0.5) gen.manual_seed(111111111) - #gen.set_state(st) - x1 = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) + # gen.set_state(st) + x1 = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) y1 = fluid.layers.dropout(x1, 0.5) y_np = y.numpy() y1_np = y1.numpy() @@ -137,7 +132,7 @@ class TestGeneratorSeed(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) out1 = exe.run(train_program, feed={}, fetch_list=[y_1]) - #gen.set_state(cur_state) + # gen.set_state(cur_state) gen.manual_seed(123123143) out2 = exe.run(train_program, feed={}, fetch_list=[y_1]) out1_np = np.array(out1[0]) @@ -184,14 +179,14 @@ class TestGeneratorSeed(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) - #gen.set_state(cur_state) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) + # gen.set_state(cur_state) gen.manual_seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -243,14 +238,14 @@ class TestGeneratorSeed(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) - #gen.set_state(cur_state) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) + # gen.set_state(cur_state) gen.manual_seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -297,14 +292,14 @@ class TestGeneratorSeed(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) - #gen.set_state(cur_state) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) + # gen.set_state(cur_state) gen.manual_seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -356,14 +351,14 @@ class TestGeneratorSeed(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) paddle.seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -383,31 +378,27 @@ class TestGeneratorSeed(unittest.TestCase): fluid.enable_dygraph() gen.manual_seed(12312321111) - x = fluid.layers.uniform_random([10, 10], - dtype="float32", - min=0.0, - max=1.0) + x = fluid.layers.uniform_random( + [10, 10], dtype="float32", min=0.0, max=1.0 + ) y = fluid.layers.sampling_id(x) st1 = gen.get_state() - x1 = fluid.layers.uniform_random([10, 10], - dtype="float32", - min=0.0, - max=1.0) + x1 = fluid.layers.uniform_random( + [10, 10], dtype="float32", min=0.0, max=1.0 + ) y1 = fluid.layers.sampling_id(x) gen.set_state(st1) - x2 = fluid.layers.uniform_random([10, 10], - dtype="float32", - min=0.0, - max=1.0) + x2 = fluid.layers.uniform_random( + [10, 10], dtype="float32", min=0.0, max=1.0 + ) y2 = fluid.layers.sampling_id(x) gen.manual_seed(12312321111) - x3 = fluid.layers.uniform_random([10, 10], - dtype="float32", - min=0.0, - max=1.0) + x3 = fluid.layers.uniform_random( + [10, 10], dtype="float32", min=0.0, max=1.0 + ) y3 = fluid.layers.sampling_id(x) x_np = y.numpy() @@ -437,14 +428,14 @@ class TestGeneratorSeed(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) paddle.seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -472,26 +463,30 @@ class TestGeneratorSeed(unittest.TestCase): result_1 = fluid.layers.fc( input=x, size=10, - param_attr=fluid.initializer.TruncatedNormal(loc=0.0, - scale=2.0)) + param_attr=fluid.initializer.TruncatedNormal( + loc=0.0, scale=2.0 + ), + ) result_2 = fluid.layers.fc( input=x, size=10, - param_attr=fluid.initializer.TruncatedNormal(loc=0.0, - scale=2.0)) + param_attr=fluid.initializer.TruncatedNormal( + loc=0.0, scale=2.0 + ), + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) gen.manual_seed(123123143) with fluid.program_guard(train_program, startup_program): exe.run(startup_program) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) diff --git a/python/paddle/fluid/tests/unittests/test_randperm_op.py b/python/paddle/fluid/tests/unittests/test_randperm_op.py index bb1e493a65d..12543a727a2 100644 --- a/python/paddle/fluid/tests/unittests/test_randperm_op.py +++ b/python/paddle/fluid/tests/unittests/test_randperm_op.py @@ -22,31 +22,38 @@ from paddle.fluid.framework import _test_eager_guard def check_randperm_out(n, data_np): - assert isinstance(data_np, np.ndarray), \ - "The input data_np should be np.ndarray." + assert isinstance( + data_np, np.ndarray + ), "The input data_np should be np.ndarray." gt_sorted = np.arange(n) out_sorted = np.sort(data_np) return list(gt_sorted == out_sorted) def error_msg(data_np): - return "The sorted ground truth and sorted out should " + \ - "be equal, out = " + str(data_np) + return ( + "The sorted ground truth and sorted out should " + + "be equal, out = " + + str(data_np) + ) def convert_dtype(dtype_str): dtype_str_list = ["int32", "int64", "float32", "float64"] dtype_num_list = [ - core.VarDesc.VarType.INT32, core.VarDesc.VarType.INT64, - core.VarDesc.VarType.FP32, core.VarDesc.VarType.FP64 + core.VarDesc.VarType.INT32, + core.VarDesc.VarType.INT64, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP64, ] - assert dtype_str in dtype_str_list, dtype_str + \ - " should in " + str(dtype_str_list) + assert dtype_str in dtype_str_list, ( + dtype_str + " should in " + str(dtype_str_list) + ) return dtype_num_list[dtype_str_list.index(dtype_str)] class TestRandpermOp(OpTest): - """ Test randperm op.""" + """Test randperm op.""" def setUp(self): self.op_type = "randperm" @@ -70,8 +77,9 @@ class TestRandpermOp(OpTest): def verify_output(self, outs): out_np = np.array(outs[0]) - self.assertTrue(check_randperm_out(self.n, out_np), - msg=error_msg(out_np)) + self.assertTrue( + check_randperm_out(self.n, out_np), msg=error_msg(out_np) + ) def test_eager(self): with _test_eager_guard(): @@ -79,31 +87,26 @@ class TestRandpermOp(OpTest): class TestRandpermOpN(TestRandpermOp): - def init_attrs(self): self.n = 10000 class TestRandpermOpInt32(TestRandpermOp): - def init_attrs(self): self.dtype = "int32" class TestRandpermOpFloat32(TestRandpermOp): - def init_attrs(self): self.dtype = "float32" class TestRandpermOpFloat64(TestRandpermOp): - def init_attrs(self): self.dtype = "float64" class TestRandpermOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(ValueError, paddle.randperm, -3) @@ -111,11 +114,13 @@ class TestRandpermOpError(unittest.TestCase): class TestRandpermAPI(unittest.TestCase): - def test_out(self): n = 10 - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) with program_guard(Program(), Program()): x1 = paddle.randperm(n) x2 = paddle.randperm(n, 'float32') @@ -130,20 +135,19 @@ class TestRandpermAPI(unittest.TestCase): class TestRandpermImperative(unittest.TestCase): - def test_out(self): paddle.disable_static() n = 10 for dtype in ['int32', np.int64, 'float32', 'float64']: data_p = paddle.randperm(n, dtype) data_np = data_p.numpy() - self.assertTrue(check_randperm_out(n, data_np), - msg=error_msg(data_np)) + self.assertTrue( + check_randperm_out(n, data_np), msg=error_msg(data_np) + ) paddle.enable_static() class TestRandpermEager(unittest.TestCase): - def test_out(self): paddle.disable_static() n = 10 @@ -151,13 +155,13 @@ class TestRandpermEager(unittest.TestCase): for dtype in ['int32', np.int64, 'float32', 'float64']: data_p = paddle.randperm(n, dtype) data_np = data_p.numpy() - self.assertTrue(check_randperm_out(n, data_np), - msg=error_msg(data_np)) + self.assertTrue( + check_randperm_out(n, data_np), msg=error_msg(data_np) + ) paddle.enable_static() class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -170,25 +174,70 @@ class TestRandomValue(unittest.TestCase): x = paddle.randperm(30000, dtype='int32').numpy() expect = [ - 24562, 8409, 9379, 10328, 20503, 18059, 9681, 21883, 11783, 27413 + 24562, + 8409, + 9379, + 10328, + 20503, + 18059, + 9681, + 21883, + 11783, + 27413, ] np.testing.assert_array_equal(x[0:10], expect) expect = [ - 29477, 27100, 9643, 16637, 8605, 16892, 27767, 2724, 1612, 13096 + 29477, + 27100, + 9643, + 16637, + 8605, + 16892, + 27767, + 2724, + 1612, + 13096, ] np.testing.assert_array_equal(x[10000:10010], expect) expect = [ - 298, 4104, 16479, 22714, 28684, 7510, 14667, 9950, 15940, 28343 + 298, + 4104, + 16479, + 22714, + 28684, + 7510, + 14667, + 9950, + 15940, + 28343, ] np.testing.assert_array_equal(x[20000:20010], expect) x = paddle.randperm(30000, dtype='int64').numpy() expect = [ - 6587, 1909, 5525, 23001, 6488, 14981, 14355, 3083, 29561, 8171 + 6587, + 1909, + 5525, + 23001, + 6488, + 14981, + 14355, + 3083, + 29561, + 8171, ] np.testing.assert_array_equal(x[0:10], expect) expect = [ - 23460, 12394, 22501, 5427, 20185, 9100, 5127, 1651, 25806, 4818 + 23460, + 12394, + 22501, + 5427, + 20185, + 9100, + 5127, + 1651, + 25806, + 4818, ] np.testing.assert_array_equal(x[10000:10010], expect) expect = [5829, 4508, 16193, 24836, 8526, 242, 9984, 9243, 1977, 11839] @@ -196,35 +245,83 @@ class TestRandomValue(unittest.TestCase): x = paddle.randperm(30000, dtype='float32').numpy() expect = [ - 5154., 10537., 14362., 29843., 27185., 28399., 27561., 4144., - 22906., 10705. + 5154.0, + 10537.0, + 14362.0, + 29843.0, + 27185.0, + 28399.0, + 27561.0, + 4144.0, + 22906.0, + 10705.0, ] np.testing.assert_array_equal(x[0:10], expect) expect = [ - 1958., 18414., 20090., 21910., 22746., 27346., 22347., 3002., 4564., - 26991. + 1958.0, + 18414.0, + 20090.0, + 21910.0, + 22746.0, + 27346.0, + 22347.0, + 3002.0, + 4564.0, + 26991.0, ] np.testing.assert_array_equal(x[10000:10010], expect) expect = [ - 25580., 12606., 553., 16387., 29536., 4241., 20946., 16899., 16339., - 4662. + 25580.0, + 12606.0, + 553.0, + 16387.0, + 29536.0, + 4241.0, + 20946.0, + 16899.0, + 16339.0, + 4662.0, ] np.testing.assert_array_equal(x[20000:20010], expect) x = paddle.randperm(30000, dtype='float64').numpy() expect = [ - 19051., 2449., 21940., 11121., 282., 7330., 13747., 24321., 21147., - 9163. + 19051.0, + 2449.0, + 21940.0, + 11121.0, + 282.0, + 7330.0, + 13747.0, + 24321.0, + 21147.0, + 9163.0, ] np.testing.assert_array_equal(x[0:10], expect) expect = [ - 15483., 1315., 5723., 20954., 13251., 25539., 5074., 1823., 14945., - 17624. + 15483.0, + 1315.0, + 5723.0, + 20954.0, + 13251.0, + 25539.0, + 5074.0, + 1823.0, + 14945.0, + 17624.0, ] np.testing.assert_array_equal(x[10000:10010], expect) expect = [ - 10516., 2552., 29970., 5941., 986., 8007., 24805., 26753., 12202., - 21404. + 10516.0, + 2552.0, + 29970.0, + 5941.0, + 986.0, + 8007.0, + 24805.0, + 26753.0, + 12202.0, + 21404.0, ] np.testing.assert_array_equal(x[20000:20010], expect) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_range.py b/python/paddle/fluid/tests/unittests/test_range.py index cdf4e8cd5e5..cd6057e13b5 100644 --- a/python/paddle/fluid/tests/unittests/test_range.py +++ b/python/paddle/fluid/tests/unittests/test_range.py @@ -24,20 +24,19 @@ def arange_wrapper(start, end, step, dtype=None): class TestRangeOp(OpTest): - def setUp(self): self.op_type = "range" self.init_config() self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange(self.case[0], self.case[1], self.case[2]).astype( + self.dtype + ) } def init_config(self): @@ -50,7 +49,6 @@ class TestRangeOp(OpTest): class TestFloatRangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.float32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -58,7 +56,6 @@ class TestFloatRangeOpCase0(TestRangeOp): class TestInt32RangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -66,7 +63,6 @@ class TestInt32RangeOpCase0(TestRangeOp): class TestInt32RangeOpCase1(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -74,7 +70,6 @@ class TestInt32RangeOpCase1(TestRangeOp): class TestInt32RangeOpCase2(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) diff --git a/python/paddle/fluid/tests/unittests/test_rank_attention_op.py b/python/paddle/fluid/tests/unittests/test_rank_attention_op.py index 4974fd26d1c..679475a5884 100644 --- a/python/paddle/fluid/tests/unittests/test_rank_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_rank_attention_op.py @@ -60,7 +60,7 @@ def gen_param_help(input, rank_offset, param, max_rank): output_param_row = block_matrix_row * input_row output_param_col = param_col - output_param = np.zeros((output_param_row * output_param_col, )) + output_param = np.zeros((output_param_row * output_param_col,)) for idx in range(output_param_row * output_param_col): output_col_idx = idx % output_param_col @@ -75,7 +75,11 @@ def gen_param_help(input, rank_offset, param, max_rank): if lower < 0 or faster < 0: continue start = lower * max_rank + faster - ori_idx = start * param_col * input_col + k_offset * param_col + output_col_idx + ori_idx = ( + start * param_col * input_col + + k_offset * param_col + + output_col_idx + ) output_param[idx] = param[int(ori_idx / param_col), ori_idx % param_col] output_param = output_param.reshape([output_param_row, output_param_col]) @@ -87,20 +91,25 @@ def np_rank_attention(input, rank_offset, rank_para, max_rank, max_size): rank_offset_row, rank_offset_col = rank_offset.shape rank_para_row, rank_para_col = rank_para.shape - assert (input_row == rank_offset_row) - assert (max_rank == ((rank_offset_col - 1) / 2)) - assert (rank_para_row == max_rank * max_rank * input_col) + assert input_row == rank_offset_row + assert max_rank == ((rank_offset_col - 1) / 2) + assert rank_para_row == max_rank * max_rank * input_col - input_help, ins_rank = gen_input_help(input, rank_offset, max_rank, - max_size) + input_help, ins_rank = gen_input_help( + input, rank_offset, max_rank, max_size + ) param_help = gen_param_help(input, rank_offset, rank_para, max_rank) block_matrix_row = input_col * max_rank res = np.zeros((input_row, rank_para_col)) for ins in range(input_row): - res[ins, :] = \ - np.dot(input_help[ins, :], - param_help[int(block_matrix_row * ins):int(block_matrix_row * (ins+1)),:]) + res[ins, :] = np.dot( + input_help[ins, :], + param_help[ + int(block_matrix_row * ins) : int(block_matrix_row * (ins + 1)), + :, + ], + ) return res, input_help, param_help, ins_rank @@ -142,7 +151,6 @@ def gen_rank_offset(pv_nums, max_rank): class TestRankAttentionOpComplex(OpTest): - def config(self): self.pv_num = 100 self.x_feat = 10 @@ -156,22 +164,27 @@ class TestRankAttentionOpComplex(OpTest): ins_num, rank_offset = gen_rank_offset(self.pv_num, self.max_rank) input = np.random.random((ins_num, self.x_feat)).astype(self.dtype) rank_para_shape = [ - self.max_rank * self.max_rank * self.x_feat, self.y_feat + self.max_rank * self.max_rank * self.x_feat, + self.y_feat, ] rank_para = np.random.random(rank_para_shape).astype(self.dtype) np_out, np_input_help, np_param_help, np_ins_rank = np_rank_attention( - input, np.array(rank_offset), rank_para, self.max_rank, - self.pv_num * 7) + input, + np.array(rank_offset), + rank_para, + self.max_rank, + self.pv_num * 7, + ) self.inputs = { "X": input, "RankOffset": np.array(rank_offset).astype("int32"), - "RankParam": rank_para + "RankParam": rank_para, } self.attrs = {'MaxRank': self.max_rank, 'MaxSize': self.pv_num * 7} self.outputs = { "Out": np_out, "InputHelp": np_input_help, - "InsRank": np_ins_rank + "InsRank": np_ins_rank, } def test_check_output_gpu(self): @@ -184,7 +197,6 @@ class TestRankAttentionOpComplex(OpTest): class TestRankAttentionOpCpu(OpTest): - def config(self): self.pv_num = 100 self.x_feat = 10 @@ -198,22 +210,27 @@ class TestRankAttentionOpCpu(OpTest): ins_num, rank_offset = gen_rank_offset(self.pv_num, self.max_rank) input = np.random.random((ins_num, self.x_feat)).astype(self.dtype) rank_para_shape = [ - self.max_rank * self.max_rank * self.x_feat, self.y_feat + self.max_rank * self.max_rank * self.x_feat, + self.y_feat, ] rank_para = np.random.random(rank_para_shape).astype(self.dtype) np_out, np_input_help, np_param_help, np_ins_rank = np_rank_attention( - input, np.array(rank_offset), rank_para, self.max_rank, - self.pv_num * 7) + input, + np.array(rank_offset), + rank_para, + self.max_rank, + self.pv_num * 7, + ) self.inputs = { "X": input, "RankOffset": np.array(rank_offset).astype("int32"), - "RankParam": rank_para + "RankParam": rank_para, } self.attrs = {'MaxRank': self.max_rank, 'MaxSize': self.pv_num * 7} self.outputs = { "Out": np_out, "InputHelp": np_input_help, - "InsRank": np_ins_rank + "InsRank": np_ins_rank, } def test_check_output_cpu(self): diff --git a/python/paddle/fluid/tests/unittests/test_rank_loss_op.py b/python/paddle/fluid/tests/unittests/test_rank_loss_op.py index 5ef05e16c96..1ace41d2d24 100644 --- a/python/paddle/fluid/tests/unittests/test_rank_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_rank_loss_op.py @@ -20,7 +20,6 @@ from paddle.fluid import Program, program_guard class TestRankLossOp(OpTest): - def setUp(self): self.op_type = "rank_loss" shape = (100, 1) @@ -34,7 +33,7 @@ class TestRankLossOp(OpTest): self.inputs = { 'Label': label.reshape(label_shape), 'Left': left.reshape(left_shape), - 'Right': right.reshape(right_shape) + 'Right': right.reshape(right_shape), } self.outputs = {'Out': loss.reshape(label_shape)} @@ -56,42 +55,36 @@ class TestRankLossOp(OpTest): class TestRankLossOp1(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size), (batch_size, 1), (batch_size, 1) class TestRankLossOp2(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size, 1), (batch_size), (batch_size, 1) class TestRankLossOp3(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size, 1), (batch_size, 1), (batch_size) class TestRankLossOp4(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size), (batch_size), (batch_size, 1) class TestRankLossOp5(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size), (batch_size), (batch_size) class TestRankLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): label = fluid.data(name="label", shape=[16, 1], dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py index eb11bf975dd..fe86a667d1e 100644 --- a/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py @@ -22,7 +22,6 @@ import os class TestRawProgramOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" @@ -30,18 +29,19 @@ class TestRawProgramOptimizer(unittest.TestCase): def mlp(self, input_x, input_y, hid_dim=128, label_dim=2): fc_1 = paddle.static.nn.fc(x=input_x, size=hid_dim, activation='tanh') fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim, activation='tanh') - prediction = paddle.static.nn.fc(x=[fc_2], - size=label_dim, - activation='softmax') - cost = paddle.nn.functional.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.static.nn.fc( + x=[fc_2], size=label_dim, activation='softmax' + ) + cost = paddle.nn.functional.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) return avg_cost def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_gpu(self): @@ -53,16 +53,17 @@ class TestRawProgramOptimizer(unittest.TestCase): strategy.without_graph_optimization = True with fluid.program_guard(sharding_program, sharding_startup_program): with fluid.unique_name.guard(): - input_x = paddle.static.data(name="x", - shape=[None, 32], - dtype='float32') - input_y = paddle.static.data(name="y", - shape=[None, 1], - dtype='int64') + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32' + ) + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64' + ) cost = self.mlp(input_x=input_x, input_y=input_y) output_name = cost.name - optimizer = fleet.distributed_optimizer(fluid.optimizer.Adam(), - strategy) + optimizer = fleet.distributed_optimizer( + fluid.optimizer.Adam(), strategy + ) optimizer.minimize(cost) trainer_id = fleet.worker_index() diff --git a/python/paddle/fluid/tests/unittests/test_reader_reset.py b/python/paddle/fluid/tests/unittests/test_reader_reset.py index f3611059b2d..edd3c6c28b1 100644 --- a/python/paddle/fluid/tests/unittests/test_reader_reset.py +++ b/python/paddle/fluid/tests/unittests/test_reader_reset.py @@ -23,9 +23,7 @@ import unittest class TestReaderReset(unittest.TestCase): - def prepare_data(self): - def fake_data_generator(): for n in range(self.total_ins_num): yield np.ones(self.ins_shape) * n, n @@ -46,15 +44,16 @@ class TestReaderReset(unittest.TestCase): startup_prog = fluid.Program() with fluid.program_guard(main_prog, startup_prog): - image = fluid.layers.data(name='image', - shape=self.ins_shape, - dtype='float32') + image = fluid.layers.data( + name='image', shape=self.ins_shape, dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') data_reader_handle = fluid.io.PyReader( feed_list=[image, label], capacity=16, iterable=False, - use_double_buffer=with_double_buffer) + use_double_buffer=with_double_buffer, + ) fetch_list = [image.name, label.name] place = fluid.CUDAPlace(0) if self.use_cuda else fluid.CPUPlace() @@ -62,10 +61,12 @@ class TestReaderReset(unittest.TestCase): exe.run(startup_prog) data_reader_handle.decorate_sample_list_generator( - paddle.batch(self.prepare_data(), batch_size=self.batch_size)) + paddle.batch(self.prepare_data(), batch_size=self.batch_size) + ) train_cp = compiler.CompiledProgram(main_prog).with_data_parallel( - places=[place]) + places=[place] + ) batch_id = 0 pass_count = 0 @@ -73,13 +74,13 @@ class TestReaderReset(unittest.TestCase): data_reader_handle.start() try: while True: - data_val, label_val = exe.run(train_cp, - fetch_list=fetch_list, - return_numpy=True) + data_val, label_val = exe.run( + train_cp, fetch_list=fetch_list, return_numpy=True + ) ins_num = data_val.shape[0] - broadcasted_label = np.ones(( - ins_num, ) + tuple(self.ins_shape)) * label_val.reshape( - (ins_num, 1)) + broadcasted_label = np.ones( + (ins_num,) + tuple(self.ins_shape) + ) * label_val.reshape((ins_num, 1)) self.assertEqual(data_val.all(), broadcasted_label.all()) batch_id += 1 except fluid.core.EOFException: diff --git a/python/paddle/fluid/tests/unittests/test_real_imag_op.py b/python/paddle/fluid/tests/unittests/test_real_imag_op.py index 8b0a0ad635b..bcd8f3c561e 100644 --- a/python/paddle/fluid/tests/unittests/test_real_imag_op.py +++ b/python/paddle/fluid/tests/unittests/test_real_imag_op.py @@ -32,7 +32,6 @@ paddle_apis = { class TestRealOp(OpTest): - def setUp(self): # switch to static paddle.enable_static() @@ -46,31 +45,31 @@ class TestRealOp(OpTest): def init_input_output(self): self.inputs = { - 'X': - np.random.random( - (20, 5)).astype(self.dtype) + 1j * np.random.random( - (20, 5)).astype(self.dtype) + 'X': np.random.random((20, 5)).astype(self.dtype) + + 1j * np.random.random((20, 5)).astype(self.dtype) } self.outputs = {'Out': numpy_apis[self.op_type](self.inputs['X'])} def init_grad_input_output(self): self.grad_out = np.ones((20, 5), self.dtype) - self.grad_x = np.real( - self.grad_out) + 1j * np.zeros(self.grad_out.shape) + self.grad_x = np.real(self.grad_out) + 1j * np.zeros( + self.grad_out.shape + ) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestImagOp(TestRealOp): - def setUp(self): # switch to static paddle.enable_static() @@ -84,12 +83,12 @@ class TestImagOp(TestRealOp): def init_grad_input_output(self): self.grad_out = np.ones((20, 5), self.dtype) - self.grad_x = np.zeros( - self.grad_out.shape) + 1j * np.real(self.grad_out) + self.grad_x = np.zeros(self.grad_out.shape) + 1j * np.real( + self.grad_out + ) class TestRealAPI(unittest.TestCase): - def setUp(self): # switch to static paddle.enable_static() @@ -102,10 +101,10 @@ class TestRealAPI(unittest.TestCase): self._shape = [2, 20, 2, 3] def test_in_static_mode(self): - def init_input_output(dtype): input = np.random.random(self._shape).astype( - dtype) + 1j * np.random.random(self._shape).astype(dtype) + dtype + ) + 1j * np.random.random(self._shape).astype(dtype) return {'x': input}, numpy_apis[self.api](input) for dtype in self.dtypes: @@ -122,7 +121,8 @@ class TestRealAPI(unittest.TestCase): def test_in_dynamic_mode(self): for dtype in self.dtypes: input = np.random.random(self._shape).astype( - dtype) + 1j * np.random.random(self._shape).astype(dtype) + dtype + ) + 1j * np.random.random(self._shape).astype(dtype) np_res = numpy_apis[self.api](input) for place in self.places: # it is more convenient to use `guard` than `enable/disable_**` here @@ -130,8 +130,11 @@ class TestRealAPI(unittest.TestCase): input_t = paddle.to_tensor(input) res = paddle_apis[self.api](input_t).numpy() np.testing.assert_array_equal(np_res, res) - res_t = input_t.real().numpy( - ) if self.api == "real" else input_t.imag().numpy() + res_t = ( + input_t.real().numpy() + if self.api == "real" + else input_t.imag().numpy() + ) np.testing.assert_array_equal(np_res, res_t) def test_name_argument(self): @@ -156,7 +159,6 @@ class TestRealAPI(unittest.TestCase): class TestImagAPI(TestRealAPI): - def setUp(self): # switch to static paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_recurrent_op.py index 1153689bb94..37ebcb77003 100644 --- a/python/paddle/fluid/tests/unittests/test_recurrent_op.py +++ b/python/paddle/fluid/tests/unittests/test_recurrent_op.py @@ -28,7 +28,6 @@ np.random.seed(123) class PyRNNBase(object): - def __init__(self, input_shape, output_shape): self.x = np.ones(shape=input_shape).astype("float32") self.y = np.zeros(shape=output_shape).astype("float32") @@ -46,13 +45,13 @@ class PyRNNBase(object): class PySimpleRNN1(PyRNNBase): - def __init__(self, input_shape, output_shape): super(PySimpleRNN1, self).__init__(input_shape, output_shape) seq_len, batch_size, input_dim = input_shape - self.h_boot = np.random.normal(size=(batch_size, - input_dim)).astype("float32") + self.h_boot = np.random.normal(size=(batch_size, input_dim)).astype( + "float32" + ) self.scale = 1.0 / 2.0 men_dim = (seq_len, batch_size, input_dim) @@ -68,7 +67,6 @@ class PySimpleRNN1(PyRNNBase): class PySimpleRNN2(PyRNNBase): - def __init__(self, input_shape, output_shape): super(PySimpleRNN2, self).__init__(input_shape, output_shape) @@ -89,7 +87,7 @@ class PySimpleRNN2(PyRNNBase): hU = np.matmul(pre_mem, self.U).astype("float32") def py_sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) self.mems[step_id] = py_sigmoid(xW + hU) self.y[step_id] = self.mems[step_id] @@ -136,14 +134,16 @@ class RecurrentOpTest1(unittest.TestCase): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False rnn = layers.StaticRNN() @@ -151,8 +151,10 @@ class RecurrentOpTest1(unittest.TestCase): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) - h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), - scale=self.py_rnn.scale) + h = layers.scale( + x=layers.elementwise_add(x=h_pre, y=x_t), + scale=self.py_rnn.scale, + ) rnn.update_memory(h_pre, h) rnn.output(h) @@ -165,9 +167,9 @@ class RecurrentOpTest1(unittest.TestCase): for x in self.feed_data_field } exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=[self.output]) + out = exe.run( + self.main_program, feed=self.feed_map, fetch_list=[self.output] + ) return out[0] @@ -182,10 +184,12 @@ class RecurrentOpTest1(unittest.TestCase): ] exe = Executor(self.place) - return exe.run(self.main_program, - feed=self.feed_map, - fetch_list=fetch_list, - return_numpy=False) + return exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=fetch_list, + return_numpy=False, + ) def test_backward(self, rtol=0.01): self.check_forward() @@ -203,10 +207,18 @@ class RecurrentOpTest1(unittest.TestCase): ana_grad[idx], rtol=rtol, atol=1e-8, - err_msg='num_grad (' + name + ') has diff at ' + - str(self.place) + '\nExpect ' + str(num_grad[idx]) + '\n' + - 'But Got' + str(ana_grad[idx]) + ' in class ' + - self.__class__.__name__) + err_msg='num_grad (' + + name + + ') has diff at ' + + str(self.place) + + '\nExpect ' + + str(num_grad[idx]) + + '\n' + + 'But Got' + + str(ana_grad[idx]) + + ' in class ' + + self.__class__.__name__, + ) def check_forward(self): pd_output = self.forward() @@ -268,14 +280,16 @@ class RecurrentOpTest2(RecurrentOpTest1): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False rnn = layers.StaticRNN() @@ -288,15 +302,19 @@ class RecurrentOpTest2(RecurrentOpTest1): size=self.input_dim, param_attr=ParamAttr( name='W', - initializer=fluid.initializer.ConstantInitializer(1.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(1.0), + ), + bias_attr=False, + ) temp_r = layers.fc( input=h_pre, size=self.input_dim, param_attr=ParamAttr( name='U', - initializer=fluid.initializer.ConstantInitializer(0.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(0.0), + ), + bias_attr=False, + ) h = layers.sigmoid(x=layers.elementwise_add(x=temp_l, y=temp_r)) @@ -325,16 +343,18 @@ class RecurrentOpMultipleMemoryTest(RecurrentOpTest1): ''' class PySimpleRNN3(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(RecurrentOpMultipleMemoryTest.PySimpleRNN3, - self).__init__(input_shape, output_shape) + super(RecurrentOpMultipleMemoryTest.PySimpleRNN3, self).__init__( + input_shape, output_shape + ) seq_len, batch_size, input_dim = input_shape - self.h_boot1 = np.random.normal(size=(batch_size, - input_dim)).astype("float32") - self.h_boot2 = np.random.normal(size=(batch_size, - input_dim)).astype("float32") + self.h_boot1 = np.random.normal( + size=(batch_size, input_dim) + ).astype("float32") + self.h_boot2 = np.random.normal( + size=(batch_size, input_dim) + ).astype("float32") men_dim = (seq_len, batch_size, input_dim) self.mems1 = np.zeros(shape=men_dim).astype("float32") @@ -364,26 +384,33 @@ class RecurrentOpMultipleMemoryTest(RecurrentOpTest1): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpMultipleMemoryTest.PySimpleRNN3( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot1 = layers.data(shape=[self.batch_size, self.input_dim], - dtype='float32', - name='h_boot1', - append_batch_size=False) + h_boot1 = layers.data( + shape=[self.batch_size, self.input_dim], + dtype='float32', + name='h_boot1', + append_batch_size=False, + ) h_boot1.stop_gradient = False - h_boot2 = layers.data(shape=[self.batch_size, self.input_dim], - dtype='float32', - name='h_boot2', - append_batch_size=False) + h_boot2 = layers.data( + shape=[self.batch_size, self.input_dim], + dtype='float32', + name='h_boot2', + append_batch_size=False, + ) h_boot2.stop_gradient = False rnn = layers.StaticRNN() @@ -418,10 +445,10 @@ class RecurrentOpNoMemBootTest(RecurrentOpTest1): ''' class PySimpleRNN4(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(RecurrentOpNoMemBootTest.PySimpleRNN4, - self).__init__(input_shape, output_shape) + super(RecurrentOpNoMemBootTest.PySimpleRNN4, self).__init__( + input_shape, output_shape + ) men_dim = input_shape self.mems = np.zeros(shape=men_dim).astype("float32") @@ -446,16 +473,19 @@ class RecurrentOpNoMemBootTest(RecurrentOpTest1): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpNoMemBootTest.PySimpleRNN4( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False rnn = layers.StaticRNN() @@ -491,31 +521,28 @@ class RecurrentOpSubBlockTest(RecurrentOpTest1): ''' class PySimpleRNN5(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(RecurrentOpSubBlockTest.PySimpleRNN5, - self).__init__(input_shape, output_shape) + super(RecurrentOpSubBlockTest.PySimpleRNN5, self).__init__( + input_shape, output_shape + ) seq_len, batch_size, input_dim = input_shape - self.w1 = np.random.uniform(-0.1, 0.1, - size=(input_dim, - input_dim)).astype("float32") - self.w2 = np.random.uniform(-0.1, - 0.1, - size=(input_dim * 2, - input_dim)).astype("float32") - - self.emb = np.random.uniform(-0.1, - 0.1, - size=(seq_len, batch_size, - input_dim)).astype("float32") + self.w1 = np.random.uniform( + -0.1, 0.1, size=(input_dim, input_dim) + ).astype("float32") + self.w2 = np.random.uniform( + -0.1, 0.1, size=(input_dim * 2, input_dim) + ).astype("float32") + + self.emb = np.random.uniform( + -0.1, 0.1, size=(seq_len, batch_size, input_dim) + ).astype("float32") men_dim = (seq_len, batch_size, input_dim) self.mems = np.zeros(shape=men_dim).astype("float32") self.oy = np.matmul(self.emb, self.w1) def step(self, step_id, x): - def dot_attention(query, memory): attn = np.matmul(query, memory.transpose((0, 2, 1))) weight = softmax(attn) @@ -552,35 +579,43 @@ class RecurrentOpSubBlockTest(RecurrentOpTest1): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpSubBlockTest.PySimpleRNN5( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): rnn_out = self.create_rnn_op() self.output = paddle.mean(rnn_out) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False emb = layers.data( name='emb', shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', - append_batch_size=False) + append_batch_size=False, + ) emb.stop_gradient = False - w1 = layers.data(shape=[self.input_dim, self.input_dim], - dtype='float32', - name='w1', - append_batch_size=False) + w1 = layers.data( + shape=[self.input_dim, self.input_dim], + dtype='float32', + name='w1', + append_batch_size=False, + ) w1.stop_gradient = False - w2 = layers.data(shape=[self.input_dim * 2, self.input_dim], - dtype='float32', - name='w2', - append_batch_size=False) + w2 = layers.data( + shape=[self.input_dim * 2, self.input_dim], + dtype='float32', + name='w2', + append_batch_size=False, + ) w2.stop_gradient = False rnn = layers.StaticRNN() @@ -594,9 +629,11 @@ class RecurrentOpSubBlockTest(RecurrentOpTest1): y = layers.matmul(emb, w1) with rnn.step(): - pre_h = rnn.memory(shape=(self.sent_len, self.input_dim), - batch_ref=x, - init_value=0.0) + pre_h = rnn.memory( + shape=(self.sent_len, self.input_dim), + batch_ref=x, + init_value=0.0, + ) step_in = rnn.step_input(x) concat_in = layers.concat([step_in, pre_h], 1) new_h = layers.matmul(concat_in, w2) @@ -643,14 +680,16 @@ class RecurrentOpStopGradientTest(RecurrentOpTest1): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype="float32", - name="x", - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype="float32", + name="x", + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype="float32", - name="h_boot") + h_boot = layers.data( + shape=[self.input_dim], dtype="float32", name="h_boot" + ) h_boot.stop_gradient = True rnn = layers.StaticRNN() @@ -663,15 +702,19 @@ class RecurrentOpStopGradientTest(RecurrentOpTest1): size=self.input_dim, param_attr=ParamAttr( name="W", - initializer=fluid.initializer.ConstantInitializer(1.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(1.0), + ), + bias_attr=False, + ) temp_r = layers.fc( input=h_pre, size=self.input_dim, param_attr=ParamAttr( name="U", - initializer=fluid.initializer.ConstantInitializer(0.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(0.0), + ), + bias_attr=False, + ) h = layers.sigmoid(x=layers.elementwise_add(temp_l, temp_r)) diff --git a/python/paddle/fluid/tests/unittests/test_reduce_op.py b/python/paddle/fluid/tests/unittests/test_reduce_op.py index 8dba9dd1b56..8fa448a6927 100644 --- a/python/paddle/fluid/tests/unittests/test_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_reduce_op.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ class TestSumOp(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -39,7 +38,6 @@ class TestSumOp(OpTest): class TestSumOp_fp16(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -58,19 +56,18 @@ class TestSumOp_fp16(OpTest): def calc_gradient(self): x = self.inputs["X"] grad = np.ones(x.shape, dtype=x.dtype) - return grad, + return (grad,) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad( + ['X'], 'Out', user_defined_grads=self.gradient, check_eager=True + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSumOp_bf16(OpTest): - def setUp(self): np.random.seed(100) self.python_api = paddle.sum @@ -91,10 +88,13 @@ class TestSumOp_bf16(OpTest): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad_with_place( + place, + ['X'], + 'Out', + user_defined_grads=self.gradient, + check_eager=True, + ) def calc_gradient(self): x = self.x @@ -103,7 +103,6 @@ class TestSumOp_bf16(OpTest): class TestSumOp_fp16_withInt(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -124,17 +123,15 @@ class TestSumOp_fp16_withInt(OpTest): def calc_gradient(self): x = self.inputs["X"] grad = np.ones(x.shape, dtype=x.dtype) - return grad, + return (grad,) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad( + ['X'], 'Out', user_defined_grads=self.gradient, check_eager=True + ) class TestSumOp5D(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -152,7 +149,6 @@ class TestSumOp5D(OpTest): class TestSumOp6D(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -170,7 +166,6 @@ class TestSumOp6D(OpTest): class TestSumOp8D(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -189,7 +184,8 @@ class TestSumOp8D(OpTest): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestMaxOp(OpTest): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -208,7 +204,8 @@ class TestMaxOp(OpTest): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestMinOp(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -266,7 +263,6 @@ def raw_reduce_prod(x, dim=[0], keep_dim=False): class TestProdOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.python_api = raw_reduce_prod @@ -275,8 +271,9 @@ class TestProdOp(OpTest): self.outputs = {'Out': self.inputs['X'].prod(axis=0)} def init_data_type(self): - self.data_type = "float32" if core.is_compiled_with_rocm( - ) else "float64" + self.data_type = ( + "float32" if core.is_compiled_with_rocm() else "float64" + ) def test_check_output(self): self.check_output(check_eager=True) @@ -286,7 +283,6 @@ class TestProdOp(OpTest): class TestProd6DOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.python_api = raw_reduce_prod @@ -300,8 +296,9 @@ class TestProd6DOp(OpTest): } def init_data_type(self): - self.data_type = "float32" if core.is_compiled_with_rocm( - ) else "float64" + self.data_type = ( + "float32" if core.is_compiled_with_rocm() else "float64" + ) def test_check_output(self): self.check_output(check_eager=True) @@ -311,14 +308,14 @@ class TestProd6DOp(OpTest): class TestProd8DOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.python_api = raw_reduce_prod self.init_data_type() self.inputs = { - 'X': np.random.random( - (2, 5, 3, 2, 2, 3, 4, 2)).astype(self.data_type) + 'X': np.random.random((2, 5, 3, 2, 2, 3, 4, 2)).astype( + self.data_type + ) } self.attrs = {'dim': [2, 3, 4]} self.outputs = { @@ -326,8 +323,9 @@ class TestProd8DOp(OpTest): } def init_data_type(self): - self.data_type = "float32" if core.is_compiled_with_rocm( - ) else "float64" + self.data_type = ( + "float32" if core.is_compiled_with_rocm() else "float64" + ) def test_check_output(self): self.check_output(check_eager=True) @@ -337,7 +335,6 @@ class TestProd8DOp(OpTest): class TestAllOp(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all @@ -350,13 +347,13 @@ class TestAllOp(OpTest): class TestAll8DOp(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'reduce_all': True, 'dim': (2, 3, 4)} self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} @@ -366,12 +363,11 @@ class TestAll8DOp(OpTest): class TestAllOpWithDim(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} - self.attrs = {'dim': (1, )} + self.attrs = {'dim': (1,)} self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} def test_check_output(self): @@ -379,13 +375,13 @@ class TestAllOpWithDim(OpTest): class TestAll8DOpWithDim(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (1, 3, 4)} self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} @@ -395,7 +391,6 @@ class TestAll8DOpWithDim(OpTest): class TestAllOpWithKeepDim(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all @@ -410,18 +405,19 @@ class TestAllOpWithKeepDim(OpTest): class TestAll8DOpWithKeepDim(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } - self.attrs = {'dim': (5, ), 'keep_dim': True} + self.attrs = {'dim': (5,), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].all(axis=self.attrs['dim']), axis=5) + 'Out': np.expand_dims( + self.inputs['X'].all(axis=self.attrs['dim']), axis=5 + ) } def test_check_output(self): @@ -429,21 +425,19 @@ class TestAll8DOpWithKeepDim(OpTest): class TestAllOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of reduce_all_op must be Variable. input1 = 12 self.assertRaises(TypeError, fluid.layers.reduce_all, input1) # The input dtype of reduce_all_op must be bool. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.reduce_all, input2) class TestAnyOp(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any @@ -456,13 +450,13 @@ class TestAnyOp(OpTest): class TestAny8DOp(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'reduce_all': True, 'dim': (3, 5, 4)} self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -472,7 +466,6 @@ class TestAny8DOp(OpTest): class TestAnyOpWithDim(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any @@ -485,13 +478,13 @@ class TestAnyOpWithDim(OpTest): class TestAny8DOpWithDim(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (3, 6)} self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -501,15 +494,15 @@ class TestAny8DOpWithDim(OpTest): class TestAnyOpWithKeepDim(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} - self.attrs = {'dim': (1, ), 'keep_dim': True} + self.attrs = {'dim': (1,), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].any(axis=self.attrs['dim']), axis=1) + 'Out': np.expand_dims( + self.inputs['X'].any(axis=self.attrs['dim']), axis=1 + ) } def test_check_output(self): @@ -517,18 +510,19 @@ class TestAnyOpWithKeepDim(OpTest): class TestAny8DOpWithKeepDim(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } - self.attrs = {'dim': (1, ), 'keep_dim': True} + self.attrs = {'dim': (1,), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].any(axis=self.attrs['dim']), axis=1) + 'Out': np.expand_dims( + self.inputs['X'].any(axis=self.attrs['dim']), axis=1 + ) } def test_check_output(self): @@ -536,21 +530,19 @@ class TestAny8DOpWithKeepDim(OpTest): class TestAnyOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of reduce_any_op must be Variable. input1 = 12 self.assertRaises(TypeError, fluid.layers.reduce_any, input1) # The input dtype of reduce_any_op must be bool. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.reduce_any, input2) class Test1DReduce(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random(120).astype("float64")} @@ -564,7 +556,6 @@ class Test1DReduce(OpTest): class Test2DReduce0(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [0]} @@ -573,7 +564,6 @@ class Test2DReduce0(Test1DReduce): class Test2DReduce1(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [1]} @@ -584,7 +574,6 @@ class Test2DReduce1(Test1DReduce): class Test3DReduce0(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [1]} @@ -595,7 +584,6 @@ class Test3DReduce0(Test1DReduce): class Test3DReduce1(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [2]} @@ -606,7 +594,6 @@ class Test3DReduce1(Test1DReduce): class Test3DReduce2(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [-2]} @@ -617,7 +604,6 @@ class Test3DReduce2(Test1DReduce): class Test3DReduce3(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [1, 2]} @@ -628,7 +614,6 @@ class Test3DReduce3(Test1DReduce): class Test8DReduce0(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': (4, 2, 3)} @@ -641,20 +626,18 @@ class Test8DReduce0(Test1DReduce): class TestKeepDimReduce(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((5, 6, 10)).astype("float64")} self.attrs = {'dim': [1], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestKeepDim8DReduce(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.inputs = { @@ -662,15 +645,16 @@ class TestKeepDim8DReduce(Test1DReduce): } self.attrs = {'dim': (3, 4, 5), 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpMultiAxises(OpTest): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -689,7 +673,8 @@ class TestReduceMaxOpMultiAxises(OpTest): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpMultiAxises(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -707,14 +692,14 @@ class TestReduceMinOpMultiAxises(OpTest): class TestKeepDimReduceSumMultiAxises(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((5, 6, 10)).astype("float64")} self.attrs = {'dim': [-2, -1], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), keepdims=True) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=True + ) } def test_check_output(self): @@ -725,14 +710,14 @@ class TestKeepDimReduceSumMultiAxises(OpTest): class TestReduceSumWithDimOne(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((100, 1, 1)).astype("float64")} self.attrs = {'dim': [1, 2], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), keepdims=True) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=True + ) } def test_check_output(self): @@ -743,14 +728,14 @@ class TestReduceSumWithDimOne(OpTest): class TestReduceSumWithNumelOne(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((100, 1)).astype("float64")} self.attrs = {'dim': [1], 'keep_dim': False} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), keepdims=False) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=False + ) } def test_check_output(self): @@ -761,7 +746,6 @@ class TestReduceSumWithNumelOne(OpTest): class TestReduceAll(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((100, 1, 1)).astype("float64")} @@ -776,7 +760,6 @@ class TestReduceAll(OpTest): class Test1DReduceWithAxes1(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random(100).astype("float64")} @@ -791,18 +774,17 @@ class Test1DReduceWithAxes1(OpTest): class TestReduceWithDtype(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((6, 2, 10)).astype("float64")} self.outputs = {'Out': self.inputs['X'].sum().astype('float64')} self.attrs = {'reduce_all': True} - self.attrs.update({ - 'in_dtype': - int(convert_np_dtype_to_dtype_(np.float32)), - 'out_dtype': - int(convert_np_dtype_to_dtype_(np.float64)) - }) + self.attrs.update( + { + 'in_dtype': int(convert_np_dtype_to_dtype_(np.float32)), + 'out_dtype': int(convert_np_dtype_to_dtype_(np.float64)), + } + ) def test_check_output(self): self.check_output() @@ -812,42 +794,40 @@ class TestReduceWithDtype(OpTest): class TestReduceWithDtype1(TestReduceWithDtype): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((6, 2, 10)).astype("float64")} self.outputs = {'Out': self.inputs['X'].sum(axis=1)} self.attrs = {'dim': [1]} - self.attrs.update({ - 'in_dtype': - int(convert_np_dtype_to_dtype_(np.float32)), - 'out_dtype': - int(convert_np_dtype_to_dtype_(np.float64)) - }) + self.attrs.update( + { + 'in_dtype': int(convert_np_dtype_to_dtype_(np.float32)), + 'out_dtype': int(convert_np_dtype_to_dtype_(np.float64)), + } + ) class TestReduceWithDtype2(TestReduceWithDtype): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((6, 2, 10)).astype("float64")} self.outputs = {'Out': self.inputs['X'].sum(axis=1, keepdims=True)} self.attrs = {'dim': [1], 'keep_dim': True} - self.attrs.update({ - 'in_dtype': - int(convert_np_dtype_to_dtype_(np.float32)), - 'out_dtype': - int(convert_np_dtype_to_dtype_(np.float64)) - }) + self.attrs.update( + { + 'in_dtype': int(convert_np_dtype_to_dtype_(np.float32)), + 'out_dtype': int(convert_np_dtype_to_dtype_(np.float64)), + } + ) class TestReduceSumOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of reduce_sum_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.reduce_sum, x1) # The input dtype of reduce_sum_op must be float32 or float64 or int32 or int64. x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -855,13 +835,9 @@ class TestReduceSumOpError(unittest.TestCase): class API_TestSumOp(unittest.TestCase): - - def run_static(self, - shape, - x_dtype, - attr_axis, - attr_dtype=None, - np_axis=None): + def run_static( + self, shape, x_dtype, attr_axis, attr_dtype=None, np_axis=None + ): if np_axis is None: np_axis = attr_axis @@ -871,19 +847,21 @@ class API_TestSumOp(unittest.TestCase): for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.data("data", shape=shape, dtype=x_dtype) - result_sum = paddle.sum(x=data, - axis=attr_axis, - dtype=attr_dtype) + result_sum = paddle.sum( + x=data, axis=attr_axis, dtype=attr_dtype + ) exe = fluid.Executor(place) input_data = np.random.rand(*shape).astype(x_dtype) - res, = exe.run(feed={"data": input_data}, - fetch_list=[result_sum]) + (res,) = exe.run( + feed={"data": input_data}, fetch_list=[result_sum] + ) - np.testing.assert_allclose(res, - np.sum(input_data.astype(attr_dtype), - axis=np_axis), - rtol=1e-05) + np.testing.assert_allclose( + res, + np.sum(input_data.astype(attr_dtype), axis=np_axis), + rtol=1e-05, + ) def test_static(self): shape = [10, 10] @@ -914,10 +892,9 @@ class API_TestSumOp(unittest.TestCase): shape = [5, 5, 5] self.run_static(shape, "int32", (0, 1), attr_dtype="int32") - self.run_static(shape, - "int32", (), - attr_dtype="int32", - np_axis=(0, 1, 2)) + self.run_static( + shape, "int32", (), attr_dtype="int32", np_axis=(0, 1, 2) + ) def test_dygraph(self): np_x = np.random.random([2, 3, 4]).astype('int32') @@ -935,7 +912,6 @@ class API_TestSumOp(unittest.TestCase): class TestAllAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) paddle.enable_static() @@ -950,9 +926,11 @@ class TestAllAPI(unittest.TestCase): input_np = np.random.randint(0, 2, [4, 4]).astype("bool") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], np.all(input_np), rtol=1e-05) def test_static(self): @@ -991,7 +969,6 @@ class TestAllAPI(unittest.TestCase): class TestAnyAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) paddle.enable_static() @@ -1006,9 +983,11 @@ class TestAnyAPI(unittest.TestCase): input_np = np.random.randint(0, 2, [4, 4]).astype("bool") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], np.any(input_np), rtol=1e-05) def test_static(self): @@ -1048,5 +1027,6 @@ class TestAnyAPI(unittest.TestCase): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_reducescatter.py b/python/paddle/fluid/tests/unittests/test_reducescatter.py index 8320eb9ad59..ed46c272bca 100644 --- a/python/paddle/fluid/tests/unittests/test_reducescatter.py +++ b/python/paddle/fluid/tests/unittests/test_reducescatter.py @@ -21,13 +21,13 @@ paddle.enable_static() class TestReduceScatterOp(TestDistBase): - def _setup_config(self): pass def test_reducescatter(self): - self.check_with_place("collective_reducescatter_op.py", - "reduce_scatter") + self.check_with_place( + "collective_reducescatter_op.py", "reduce_scatter" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_reducescatter_api.py b/python/paddle/fluid/tests/unittests/test_reducescatter_api.py index decb90ada96..e5d6b261536 100644 --- a/python/paddle/fluid/tests/unittests/test_reducescatter_api.py +++ b/python/paddle/fluid/tests/unittests/test_reducescatter_api.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestReduceScatterAPI(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/test_registry.py b/python/paddle/fluid/tests/unittests/test_registry.py index be0ee288178..3fc6370b6c7 100644 --- a/python/paddle/fluid/tests/unittests/test_registry.py +++ b/python/paddle/fluid/tests/unittests/test_registry.py @@ -21,7 +21,6 @@ from decorator_helper import prog_scope class TestRegistry(unittest.TestCase): - @prog_scope() def test_registry_layer(self): x = fluid.layers.data(name='X', shape=[10, 10], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_regularizer.py b/python/paddle/fluid/tests/unittests/test_regularizer.py index 46cad0aaf75..a3f1697032b 100644 --- a/python/paddle/fluid/tests/unittests/test_regularizer.py +++ b/python/paddle/fluid/tests/unittests/test_regularizer.py @@ -26,7 +26,6 @@ from paddle.fluid.backward import append_backward class TestL2DecayRegularizer(unittest.TestCase): - def test_l2decay_regularizer(self): paddle.enable_static() program = framework.Program() @@ -36,32 +35,30 @@ class TestL2DecayRegularizer(unittest.TestCase): shape=[5, 10], lod_level=0, name="mul.x", - regularizer=regularizer.L2DecayRegularizer(0.5)) + regularizer=regularizer.L2DecayRegularizer(0.5), + ) self.assertTrue(mul_x.regularizer is not None) self.assertTrue( - isinstance(mul_x.regularizer, regularizer.L2DecayRegularizer)) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + isinstance(mul_x.regularizer, regularizer.L2DecayRegularizer) + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) count_ops = len(block.ops) @@ -74,7 +71,6 @@ class TestL2DecayRegularizer(unittest.TestCase): class TestL1DecayRegularizer(unittest.TestCase): - def test_l2decay_regularizer(self): paddle.enable_static() program = framework.Program() @@ -84,32 +80,30 @@ class TestL1DecayRegularizer(unittest.TestCase): shape=[5, 10], lod_level=0, name="mul.x", - regularizer=regularizer.L1DecayRegularizer(0.5)) + regularizer=regularizer.L1DecayRegularizer(0.5), + ) self.assertTrue(mul_x.regularizer is not None) self.assertTrue( - isinstance(mul_x.regularizer, regularizer.L1DecayRegularizer)) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + isinstance(mul_x.regularizer, regularizer.L1DecayRegularizer) + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) count_ops = len(block.ops) @@ -122,22 +116,24 @@ class TestL1DecayRegularizer(unittest.TestCase): self.assertEqual(block.ops[-3].type, 'sign') -def bow_net(data, - label, - dict_dim, - is_sparse=False, - emb_dim=8, - hid_dim=8, - hid_dim2=6, - class_dim=2): +def bow_net( + data, + label, + dict_dim, + is_sparse=False, + emb_dim=8, + hid_dim=8, + hid_dim2=6, + class_dim=2, +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - emb = fluid.layers.embedding(input=data, - is_sparse=is_sparse, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -149,11 +145,11 @@ def bow_net(data, class TestRegularizer(unittest.TestCase): - def setUp(self): self.word_len = 1500 - self.train_data = [[(random.sample(range(1000), 10), [0])] - for _ in range(2)] + self.train_data = [ + [(random.sample(range(1000), 10), [0])] for _ in range(2) + ] def get_places(self): places = [core.CPUPlace()] @@ -179,9 +175,9 @@ class TestRegularizer(unittest.TestCase): param_sum = [] for data in self.train_data: - out = exe.run(main_prog, - feed=feeder.feed(data), - fetch_list=param_list) + out = exe.run( + main_prog, feed=feeder.feed(data), fetch_list=param_list + ) p_sum = 0 for v in out: p_sum += np.sum(np.abs(v)) @@ -193,19 +189,19 @@ class TestRegularizer(unittest.TestCase): paddle.framework.random._manual_program_seed(1) main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() - with self.scope_prog_guard(main_prog=main_prog, - startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + with self.scope_prog_guard( + main_prog=main_prog, startup_prog=startup_prog + ): + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost = model(data, label, self.word_len) optimizer = fluid.optimizer.Adagrad( - learning_rate=0.1, - regularization=fluid.regularizer.L2Decay(1.0)) + learning_rate=0.1, regularization=fluid.regularizer.L2Decay(1.0) + ) optimizer.minimize(avg_cost) param_sum = self.run_program(place, [data, label]) return param_sum @@ -216,12 +212,12 @@ class TestRegularizer(unittest.TestCase): main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() - with self.scope_prog_guard(main_prog=main_prog, - startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + with self.scope_prog_guard( + main_prog=main_prog, startup_prog=startup_prog + ): + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost_l2 = model(data, label, self.word_len) @@ -231,7 +227,7 @@ class TestRegularizer(unittest.TestCase): for para in param_list: para_mul = fluid.layers.square(x=para) para_sum.append(fluid.layers.reduce_sum(input=para_mul)) - avg_cost_l2 += fluid.layers.sums(para_sum) * .5 + avg_cost_l2 += fluid.layers.sums(para_sum) * 0.5 optimizer = fluid.optimizer.Adagrad(learning_rate=0.1) optimizer.minimize(avg_cost_l2) @@ -252,9 +248,11 @@ class TestRegularizer(unittest.TestCase): assert len(dense_sparse_p_sum[0]) == len(dense_sparse_p_sum[1]) for i in range(len(dense_sparse_p_sum[0])): - assert np.isclose(a=dense_sparse_p_sum[0][i], - b=dense_sparse_p_sum[1][i], - rtol=5e-5) + assert np.isclose( + a=dense_sparse_p_sum[0][i], + b=dense_sparse_p_sum[1][i], + rtol=5e-5, + ) def test_repeated_regularization(self): l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1) @@ -268,43 +266,45 @@ class TestRegularizer(unittest.TestCase): sgd.minimize(loss) with fluid.dygraph.guard(): input = fluid.dygraph.to_variable( - np.random.randn(3, 2).astype('float32')) + np.random.randn(3, 2).astype('float32') + ) paddle.seed(1) paddle.framework.random._manual_program_seed(1) - linear1 = fluid.dygraph.Linear(2, - 2, - param_attr=fc_param_attr, - bias_attr=fc_param_attr) - linear2 = fluid.dygraph.Linear(2, - 2, - param_attr=fc_param_attr, - bias_attr=fc_param_attr) + linear1 = fluid.dygraph.Linear( + 2, 2, param_attr=fc_param_attr, bias_attr=fc_param_attr + ) + linear2 = fluid.dygraph.Linear( + 2, 2, param_attr=fc_param_attr, bias_attr=fc_param_attr + ) loss1 = linear1(input) loss1.backward() # set l2 regularizer in optimizer, but l1 in fluid.ParamAttr - fluid.optimizer.SGD(parameter_list=linear1.parameters(), - learning_rate=1e-2, - regularization=l2).minimize(loss1) + fluid.optimizer.SGD( + parameter_list=linear1.parameters(), + learning_rate=1e-2, + regularization=l2, + ).minimize(loss1) # only set l1 in fluid.ParamAttr loss2 = linear2(input) loss2.backward() - fluid.optimizer.SGD(parameter_list=linear2.parameters(), - learning_rate=1e-2).minimize(loss2) + fluid.optimizer.SGD( + parameter_list=linear2.parameters(), learning_rate=1e-2 + ).minimize(loss2) # they should both be applied by l1, and keep the same np.testing.assert_allclose( linear1.weight.numpy(), linear2.weight.numpy(), rtol=1e-05, - err_msg= - 'weight should use the regularization in fluid.ParamAttr!') + err_msg='weight should use the regularization in fluid.ParamAttr!', + ) np.testing.assert_allclose( linear1.bias.numpy(), linear2.bias.numpy(), rtol=1e-05, - err_msg='bias should use the regularization in fluid.ParamAttr!' + err_msg='bias should use the regularization in fluid.ParamAttr!', ) diff --git a/python/paddle/fluid/tests/unittests/test_regularizer_api.py b/python/paddle/fluid/tests/unittests/test_regularizer_api.py index b4200970dcd..930de8429f4 100644 --- a/python/paddle/fluid/tests/unittests/test_regularizer_api.py +++ b/python/paddle/fluid/tests/unittests/test_regularizer_api.py @@ -22,22 +22,24 @@ import paddle.fluid.core as core import paddle.fluid as fluid -def bow_net(data, - label, - dict_dim, - is_sparse=False, - emb_dim=8, - hid_dim=8, - hid_dim2=6, - class_dim=2): +def bow_net( + data, + label, + dict_dim, + is_sparse=False, + emb_dim=8, + hid_dim=8, + hid_dim2=6, + class_dim=2, +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - emb = fluid.layers.embedding(input=data, - is_sparse=is_sparse, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -50,11 +52,11 @@ def bow_net(data, class TestRegularizer(unittest.TestCase): - def setUp(self): self.word_len = 1500 - self.train_data = [[(random.sample(range(1000), 10), [0])] - for _ in range(2)] + self.train_data = [ + [(random.sample(range(1000), 10), [0])] for _ in range(2) + ] def get_places(self): places = [core.CPUPlace()] @@ -80,9 +82,9 @@ class TestRegularizer(unittest.TestCase): param_sum = [] for data in self.train_data: - out = exe.run(main_prog, - feed=feeder.feed(data), - fetch_list=param_list) + out = exe.run( + main_prog, feed=feeder.feed(data), fetch_list=param_list + ) p_sum = 0 for v in out: p_sum += np.sum(np.abs(v)) @@ -94,19 +96,20 @@ class TestRegularizer(unittest.TestCase): paddle.framework.random._manual_program_seed(1) main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() - with self.scope_prog_guard(main_prog=main_prog, - startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + with self.scope_prog_guard( + main_prog=main_prog, startup_prog=startup_prog + ): + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost = model(data, label, self.word_len) optimizer = fluid.optimizer.Adagrad( learning_rate=0.1, - regularization=paddle.regularizer.L2Decay(1.0)) + regularization=paddle.regularizer.L2Decay(1.0), + ) optimizer.minimize(avg_cost) param_sum = self.run_program(place, [data, label]) return param_sum @@ -117,12 +120,12 @@ class TestRegularizer(unittest.TestCase): main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() - with self.scope_prog_guard(main_prog=main_prog, - startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + with self.scope_prog_guard( + main_prog=main_prog, startup_prog=startup_prog + ): + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost_l2 = model(data, label, self.word_len) @@ -132,7 +135,7 @@ class TestRegularizer(unittest.TestCase): for para in param_list: para_mul = fluid.layers.square(x=para) para_sum.append(fluid.layers.reduce_sum(input=para_mul)) - avg_cost_l2 += fluid.layers.sums(para_sum) * .5 + avg_cost_l2 += fluid.layers.sums(para_sum) * 0.5 optimizer = fluid.optimizer.Adagrad(learning_rate=0.1) optimizer.minimize(avg_cost_l2) @@ -154,9 +157,11 @@ class TestRegularizer(unittest.TestCase): assert len(dense_sparse_p_sum[0]) == len(dense_sparse_p_sum[1]) for i in range(len(dense_sparse_p_sum[0])): - assert np.isclose(a=dense_sparse_p_sum[0][i], - b=dense_sparse_p_sum[1][i], - rtol=5e-5) + assert np.isclose( + a=dense_sparse_p_sum[0][i], + b=dense_sparse_p_sum[1][i], + rtol=5e-5, + ) def test_repeated_regularization(self): paddle.enable_static() @@ -171,43 +176,45 @@ class TestRegularizer(unittest.TestCase): sgd.minimize(loss) with fluid.dygraph.guard(): input = fluid.dygraph.to_variable( - np.random.randn(3, 2).astype('float32')) + np.random.randn(3, 2).astype('float32') + ) paddle.seed(1) paddle.framework.random._manual_program_seed(1) - linear1 = fluid.dygraph.Linear(2, - 2, - param_attr=fc_param_attr, - bias_attr=fc_param_attr) - linear2 = fluid.dygraph.Linear(2, - 2, - param_attr=fc_param_attr, - bias_attr=fc_param_attr) + linear1 = fluid.dygraph.Linear( + 2, 2, param_attr=fc_param_attr, bias_attr=fc_param_attr + ) + linear2 = fluid.dygraph.Linear( + 2, 2, param_attr=fc_param_attr, bias_attr=fc_param_attr + ) loss1 = linear1(input) loss1.backward() # set l2 regularizer in optimizer, but l1 in fluid.ParamAttr - fluid.optimizer.SGD(parameter_list=linear1.parameters(), - learning_rate=1e-2, - regularization=l2).minimize(loss1) + fluid.optimizer.SGD( + parameter_list=linear1.parameters(), + learning_rate=1e-2, + regularization=l2, + ).minimize(loss1) # only set l1 in fluid.ParamAttr loss2 = linear2(input) loss2.backward() - fluid.optimizer.SGD(parameter_list=linear2.parameters(), - learning_rate=1e-2).minimize(loss2) + fluid.optimizer.SGD( + parameter_list=linear2.parameters(), learning_rate=1e-2 + ).minimize(loss2) # they should both be applied by l1, and keep the same np.testing.assert_allclose( linear1.weight.numpy(), linear2.weight.numpy(), rtol=1e-05, - err_msg= - 'weight should use the regularization in fluid.ParamAttr!') + err_msg='weight should use the regularization in fluid.ParamAttr!', + ) np.testing.assert_allclose( linear1.bias.numpy(), linear2.bias.numpy(), rtol=1e-05, - err_msg='bias should use the regularization in fluid.ParamAttr!' + err_msg='bias should use the regularization in fluid.ParamAttr!', ) diff --git a/python/paddle/fluid/tests/unittests/test_renorm_op.py b/python/paddle/fluid/tests/unittests/test_renorm_op.py index f2754eb16bd..38362f039ea 100644 --- a/python/paddle/fluid/tests/unittests/test_renorm_op.py +++ b/python/paddle/fluid/tests/unittests/test_renorm_op.py @@ -22,10 +22,10 @@ paddle.set_device('cpu') class TestRenormAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[[2.0, 2, -2], [3, 0.3, 3]], - [[2, -8, 2], [3.1, 3.7, 3]]]) + self.data_x = np.array( + [[[2.0, 2, -2], [3, 0.3, 3]], [[2, -8, 2], [3.1, 3.7, 3]]] + ) self.p = 1.0 self.dim = 2 self.max_norm = 2.05 @@ -36,17 +36,25 @@ class TestRenormAPI(unittest.TestCase): # case 1: with program_guard(Program(), Program()): - #x = fluid.layers.data(name = 'x',shape=[-1, 2, 3]) + # x = fluid.layers.data(name = 'x',shape=[-1, 2, 3]) x = paddle.static.data(name="x", shape=[-1, 2, 3], dtype='float64') z = paddle.renorm(x, self.p, self.dim, self.max_norm) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={"x": self.data_x}, - fetch_list=[z], - return_numpy=False) - expected = np.array([[[0.40594056, 0.29285714, -0.41000000], - [0.60891086, 0.04392857, 0.61500001]], - [[0.40594056, -1.17142856, 0.41000000], - [0.62920785, 0.54178572, 0.61500001]]]) + (res,) = exe.run( + feed={"x": self.data_x}, fetch_list=[z], return_numpy=False + ) + expected = np.array( + [ + [ + [0.40594056, 0.29285714, -0.41000000], + [0.60891086, 0.04392857, 0.61500001], + ], + [ + [0.40594056, -1.17142856, 0.41000000], + [0.62920785, 0.54178572, 0.61500001], + ], + ] + ) np.testing.assert_allclose(expected, np.array(res), rtol=1e-05) def test_dygraph_api(self): @@ -56,20 +64,30 @@ class TestRenormAPI(unittest.TestCase): input = [[[2.0, 2, -2], [3, 0.3, 3]], [[2, -8, 2], [3.1, 3.7, 3]]] x = paddle.to_tensor(input, stop_gradient=False) y = paddle.renorm(x, 1.0, 2, 2.05) - expected = np.array([[[0.40594056, 0.29285714, -0.41000000], - [0.60891086, 0.04392857, 0.61500001]], - [[0.40594056, -1.17142856, 0.41000000], - [0.62920785, 0.54178572, 0.61500001]]]) + expected = np.array( + [ + [ + [0.40594056, 0.29285714, -0.41000000], + [0.60891086, 0.04392857, 0.61500001], + ], + [ + [0.40594056, -1.17142856, 0.41000000], + [0.62920785, 0.54178572, 0.61500001], + ], + ] + ) np.testing.assert_allclose(expected, np.array(y), rtol=1e-05) z = paddle.mean(y) z.backward(retain_graph=True) - expected_grad = np.array([[[0, 0.01394558, 0.02733333], - [0, 0.01394558, 0.00683333]], - [[0, 0.01045918, 0.00683333], - [0, 0.01394558, 0.00683333]]]) - np.testing.assert_allclose(expected_grad, - np.array(x.grad), - rtol=1e-05) + expected_grad = np.array( + [ + [[0, 0.01394558, 0.02733333], [0, 0.01394558, 0.00683333]], + [[0, 0.01045918, 0.00683333], [0, 0.01394558, 0.00683333]], + ] + ) + np.testing.assert_allclose( + expected_grad, np.array(x.grad), rtol=1e-05 + ) # #test exception: with fluid.dygraph.guard(): input = [[[2.0, 2, -2], [3, 0.3, 3]], [[2, -8, 2], [3.1, 3.7, 3]]] @@ -87,10 +105,18 @@ class TestRenormAPI(unittest.TestCase): exp = True self.assertTrue(exp) y = paddle.renorm(x, 1.0, -1, 2.05) - expected = np.array([[[0.40594056, 0.29285714, -0.41000000], - [0.60891086, 0.04392857, 0.61500001]], - [[0.40594056, -1.17142856, 0.41000000], - [0.62920785, 0.54178572, 0.61500001]]]) + expected = np.array( + [ + [ + [0.40594056, 0.29285714, -0.41000000], + [0.60891086, 0.04392857, 0.61500001], + ], + [ + [0.40594056, -1.17142856, 0.41000000], + [0.62920785, 0.54178572, 0.61500001], + ], + ] + ) np.testing.assert_allclose(expected, np.array(y), rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py index 1b7538f1bcc..dbf14c81948 100644 --- a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py @@ -40,14 +40,17 @@ class TestReorderLoDTensor(unittest.TestCase): @classmethod def set_program(cls): - dat = fluid.layers.data(name=cls.data_desc[0][0], - shape=cls.data_desc[0][1]) + dat = fluid.layers.data( + name=cls.data_desc[0][0], shape=cls.data_desc[0][1] + ) dat.stop_gradient = False - rank_dat = fluid.layers.data(name=cls.data_desc[1][0], - shape=cls.data_desc[1][1]) + rank_dat = fluid.layers.data( + name=cls.data_desc[1][0], shape=cls.data_desc[1][1] + ) table = lod_rank_table(rank_dat) - new_dat = fluid.layers.reorder_lod_tensor_by_rank(x=dat, - rank_table=table) + new_dat = fluid.layers.reorder_lod_tensor_by_rank( + x=dat, rank_table=table + ) loss = fluid.layers.reduce_sum(new_dat) fluid.backward.append_backward(loss=loss) cls.fetch_list = [new_dat, cls.data_desc[0][0] + '@GRAD'] @@ -61,10 +64,12 @@ class TestReorderLoDTensor(unittest.TestCase): for place in places: self.set_inputs(place) exe = fluid.Executor(place) - output, input_grad = exe.run(fluid.default_main_program(), - feed=self.inputs, - fetch_list=self.fetch_list, - return_numpy=False) + output, input_grad = exe.run( + fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=False, + ) outputs.append(output) input_grads.append(input_grad) self.actual_outputs = outputs @@ -81,11 +86,13 @@ class TestReorderLoDTensor(unittest.TestCase): lod_level_i = np.random.randint( low=1, high=5, - size=self.num_seq if i == 0 else sum(lod_level_i)).tolist() + size=self.num_seq if i == 0 else sum(lod_level_i), + ).tolist() data_lod.append(lod_level_i) data_value = np.random.random( - size=[sum(data_lod[-1]) if data_lod else self.num_seq] + - data_shape).astype('float32') + size=[sum(data_lod[-1]) if data_lod else self.num_seq] + + data_shape + ).astype('float32') self.data[data_name] = (data_value, data_lod) def set_inputs(self, place): @@ -104,8 +111,9 @@ class TestReorderLoDTensor(unittest.TestCase): rank_table = [] # list of (index, length) for i in range(len(ref_lod)): rank_table.append((i, ref_lod[i])) - rank_table = sorted(rank_table, - key=functools.cmp_to_key(lambda x, y: y[1] - x[1])) + rank_table = sorted( + rank_table, key=functools.cmp_to_key(lambda x, y: y[1] - x[1]) + ) # compute the input sequence info according to input_lod input_value, input_lod = self.data[self.data_desc[0][0]] @@ -120,8 +128,9 @@ class TestReorderLoDTensor(unittest.TestCase): for lod_level_i in offset_lod[level:]: sub_lod_i = [] for idx in range(start_idx, end_idx): - sub_lod_i.append(lod_level_i[idx + 1] - - lod_level_i[idx]) + sub_lod_i.append( + lod_level_i[idx + 1] - lod_level_i[idx] + ) sub_lod.append(sub_lod_i) start_idx = lod_level_i[start_idx] end_idx = lod_level_i[end_idx] @@ -137,8 +146,9 @@ class TestReorderLoDTensor(unittest.TestCase): input_seq_start = input_table[index][0] input_seq_len = input_table[index][1] input_seq_end = input_seq_start + input_seq_len - output_value[offset:offset + input_seq_len] = input_value[ - input_seq_start:input_seq_end] + output_value[offset : offset + input_seq_len] = input_value[ + input_seq_start:input_seq_end + ] offset += input_seq_len input_seq_sub_lod = input_table[index][2] @@ -155,22 +165,22 @@ class TestReorderLoDTensor(unittest.TestCase): # check output expect_output, expect_output_lod = self.reorder() for actual_output in self.actual_outputs: - np.testing.assert_allclose(np.array(actual_output), - expect_output, - rtol=1e-05, - atol=0.001) - self.assertEqual(expect_output_lod, - actual_output.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual_output), expect_output, rtol=1e-05, atol=0.001 + ) + self.assertEqual( + expect_output_lod, actual_output.recursive_sequence_lengths() + ) # check gradient expect_grad = np.ones_like(self.data[self.data_desc[0][0]][0]) expect_grad_lod = self.data[self.data_desc[0][0]][1] for actual_grad in self.actual_grads: - np.testing.assert_allclose(np.array(actual_grad), - expect_grad, - rtol=1e-05, - atol=0.001) - self.assertEqual(expect_grad_lod, - actual_grad.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual_grad), expect_grad, rtol=1e-05, atol=0.001 + ) + self.assertEqual( + expect_grad_lod, actual_grad.recursive_sequence_lengths() + ) def test_reorder_tensor(self): self.data_desc[0][-1] = 0 # input is tensor @@ -179,43 +189,43 @@ class TestReorderLoDTensor(unittest.TestCase): # check output expect_output, expect_output_lod = self.reorder() for actual_output in self.actual_outputs: - np.testing.assert_allclose(np.array(actual_output), - expect_output, - rtol=1e-05, - atol=0.001) - self.assertEqual(expect_output_lod, - actual_output.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual_output), expect_output, rtol=1e-05, atol=0.001 + ) + self.assertEqual( + expect_output_lod, actual_output.recursive_sequence_lengths() + ) # check gradient expect_grad = np.ones_like(self.data[self.data_desc[0][0]][0]) expect_grad_lod = self.data[self.data_desc[0][0]][1] for actual_grad in self.actual_grads: - np.testing.assert_allclose(np.array(actual_grad), - expect_grad, - rtol=1e-05, - atol=0.001) - self.assertEqual(expect_grad_lod, - actual_grad.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual_grad), expect_grad, rtol=1e-05, atol=0.001 + ) + self.assertEqual( + expect_grad_lod, actual_grad.recursive_sequence_lengths() + ) # compare outputs between LodTensors with explicit and implicit lod # use the same data but set the input lod explicitly input_lod = [[1] * len(self.data[self.data_desc[0][0]][0])] self.inputs[self.data_desc[0][0]].set_recursive_sequence_lengths( - input_lod) + input_lod + ) # preserve the output of LodTensor with implicit lod to compare expect_outputs = [ np.array(actual_output) for actual_output in self.actual_outputs ] self.run_program() - for actual_output, expect_output in zip(self.actual_outputs, - expect_outputs): - np.testing.assert_allclose(np.array(actual_output), - expect_output, - rtol=1e-05, - atol=0.001) + for actual_output, expect_output in zip( + self.actual_outputs, expect_outputs + ): + np.testing.assert_allclose( + np.array(actual_output), expect_output, rtol=1e-05, atol=0.001 + ) class TestReorderLoDTensorError(unittest.TestCase): - def test_errors(self): with program_guard(Program()): @@ -224,17 +234,19 @@ class TestReorderLoDTensorError(unittest.TestCase): x1 = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float64") table1 = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float64") new_dat = fluid.layers.reorder_lod_tensor_by_rank( - x=x1, rank_table=table1) + x=x1, rank_table=table1 + ) self.assertRaises(TypeError, test_Variable) def test_type(): x2 = fluid.layers.data(name='x1', shape=[4], dtype='float32') - table2 = fluid.layers.data(name='table2', - shape=[4], - dtype='int32') + table2 = fluid.layers.data( + name='table2', shape=[4], dtype='int32' + ) new_dat2 = fluid.layers.reorder_lod_tensor_by_rank( - x=x2, rank_table=table2) + x=x2, rank_table=table2 + ) self.assertRaises(TypeError, test_type) diff --git a/python/paddle/fluid/tests/unittests/test_repeat_interleave_op.py b/python/paddle/fluid/tests/unittests/test_repeat_interleave_op.py index fcc8647fbca..7332c36e1d2 100644 --- a/python/paddle/fluid/tests/unittests/test_repeat_interleave_op.py +++ b/python/paddle/fluid/tests/unittests/test_repeat_interleave_op.py @@ -21,20 +21,20 @@ from paddle.fluid import Program, program_guard class TestRepeatInterleaveOp(OpTest): - def setUp(self): self.op_type = "repeat_interleave" self.python_api = paddle.repeat_interleave self.init_dtype_type() index_np = np.random.randint( - low=0, high=3, size=self.index_size).astype(self.index_type) + low=0, high=3, size=self.index_size + ).astype(self.index_type) x_np = np.random.random(self.x_shape).astype(self.x_type) self.inputs = {'X': x_np, 'RepeatsTensor': index_np} self.attrs = {'dim': self.dim} - outer_loop = np.prod(self.x_shape[:self.dim]) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + outer_loop = np.prod(self.x_shape[: self.dim]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] for i in range(outer_loop): @@ -63,18 +63,17 @@ class TestRepeatInterleaveOp(OpTest): class TestRepeatInterleaveOp2(OpTest): - def setUp(self): self.op_type = "repeat_interleave" self.python_api = paddle.repeat_interleave self.init_dtype_type() index_np = 2 x_np = np.random.random(self.x_shape).astype(self.x_type) - self.inputs = {'X': x_np} #, 'RepeatsTensor': None} + self.inputs = {'X': x_np} # , 'RepeatsTensor': None} self.attrs = {'dim': self.dim, 'Repeats': index_np} - outer_loop = np.prod(self.x_shape[:self.dim]) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + outer_loop = np.prod(self.x_shape[: self.dim]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] for i in range(outer_loop): @@ -102,10 +101,14 @@ class TestRepeatInterleaveOp2(OpTest): class TestIndexSelectAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]]) + self.data_x = np.array( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] + ) self.data_index = np.array([0, 1, 2, 1]).astype('int32') def test_repeat_interleave_api(self): @@ -115,18 +118,19 @@ class TestIndexSelectAPI(unittest.TestCase): # case 1: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='repeats_', - shape=[4], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='repeats_', + shape=[4], + dtype='int32', + append_batch_size=False, + ) z = paddle.repeat_interleave(x, index, axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'repeats_': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': self.data_x, 'repeats_': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) expect_out = np.repeat(self.data_x, self.data_index, axis=1) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -134,18 +138,22 @@ class TestIndexSelectAPI(unittest.TestCase): repeats = np.array([1, 2, 1]).astype('int32') with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='repeats_', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='repeats_', + shape=[3], + dtype='int32', + append_batch_size=False, + ) z = paddle.repeat_interleave(x, index, axis=0) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'repeats_': repeats, - }, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={ + 'x': self.data_x, + 'repeats_': repeats, + }, + fetch_list=[z.name], + return_numpy=False, + ) expect_out = np.repeat(self.data_x, repeats, axis=0) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -154,9 +162,9 @@ class TestIndexSelectAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[-1, 4]) z = paddle.repeat_interleave(x, repeats, axis=0) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': self.data_x}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': self.data_x}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.repeat(self.data_x, repeats, axis=0) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_require_version.py b/python/paddle/fluid/tests/unittests/test_require_version.py index b5d9a1bfa54..92066e95392 100644 --- a/python/paddle/fluid/tests/unittests/test_require_version.py +++ b/python/paddle/fluid/tests/unittests/test_require_version.py @@ -20,21 +20,30 @@ import paddle class VersionTest(unittest.TestCase): - def test_check_output(self): warnings.warn( "paddle.__version__: %s, fluid_version.full_version: %s, fluid_version.major: %s, fluid_version.minor: %s, fluid_version.patch: %s, fluid_version.rc: %s." - % (paddle.__version__, fluid_version.full_version, - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc)) + % ( + paddle.__version__, + fluid_version.full_version, + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, + ) + ) ori_full_version = fluid_version.full_version ori_sep_version = [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ['1', '4', '1', '0'] fluid.require_version('1') @@ -52,21 +61,24 @@ class VersionTest(unittest.TestCase): # if installed version is 0.0.0.0, throw warning and skip the checking. [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ['0', '0', '0', '0'] fluid.require_version('0.0.0') fluid_version.full_version = ori_full_version [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ori_sep_version # Test Errors class TestErrors(unittest.TestCase): - def test_errors(self): # The type of params must be str. def test_input_type(): @@ -118,12 +130,16 @@ class TestErrors(unittest.TestCase): ori_full_version = fluid_version.full_version ori_sep_version = [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ['1', '4', '1', '0'] self.assertRaises(Exception, test_version) @@ -132,8 +148,10 @@ class TestErrors(unittest.TestCase): fluid_version.full_version = ori_full_version [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ori_sep_version diff --git a/python/paddle/fluid/tests/unittests/test_reset_grad_inplace_version.py b/python/paddle/fluid/tests/unittests/test_reset_grad_inplace_version.py index c28724b97ae..a13a0d2c9e8 100644 --- a/python/paddle/fluid/tests/unittests/test_reset_grad_inplace_version.py +++ b/python/paddle/fluid/tests/unittests/test_reset_grad_inplace_version.py @@ -22,7 +22,6 @@ paddle.set_device('cpu') # Test 1 def clear_grad_test_0(w, a): - @paddle.no_grad() def warp(*_): assert w.grad is not None @@ -33,7 +32,6 @@ def clear_grad_test_0(w, a): class TestInplaceAndClearGradient(unittest.TestCase): - def func_test(self): input_data = np.ones([1, 1]) w = paddle.to_tensor(input_data, 'float32', stop_gradient=False) @@ -43,8 +41,9 @@ class TestInplaceAndClearGradient(unittest.TestCase): for i in range(2): print(" Step: ", i) out0 = _legacy_C_ops.scale(w, 'scale', 0.1) - out = _legacy_C_ops.matmul_v2(out0, w, 'trans_x', False, 'trans_y', - False) + out = _legacy_C_ops.matmul_v2( + out0, w, 'trans_x', False, 'trans_y', False + ) out.backward() assert w.grad[0] == 0.15 @@ -56,14 +55,12 @@ class TestInplaceAndClearGradient(unittest.TestCase): # Test 2 class Counter: - def __init__(self): self.num_calls = 0 self.step = 0 def clear_grad_test_1(w, c): - @paddle.no_grad() def warp(*_): assert w.grad is not None @@ -77,7 +74,6 @@ def clear_grad_test_1(w, c): class TestInplaceClearGradAccumulation(unittest.TestCase): - def func_test(self): input_data = np.ones([1, 1]) w = paddle.to_tensor(input_data, 'float32', stop_gradient=False) @@ -88,8 +84,9 @@ class TestInplaceClearGradAccumulation(unittest.TestCase): for c.step in range(5): out0 = _legacy_C_ops.scale(w, 'scale', 0.1) - out = _legacy_C_ops.matmul_v2(out0, w, 'trans_x', False, 'trans_y', - False) + out = _legacy_C_ops.matmul_v2( + out0, w, 'trans_x', False, 'trans_y', False + ) out.backward() @@ -106,7 +103,6 @@ class TestInplaceClearGradAccumulation(unittest.TestCase): class TestInplaceClearGradAccumulationAlt(unittest.TestCase): - def func_test(self): input_data = np.ones([1, 1]) w = paddle.to_tensor(input_data, 'float32', stop_gradient=False) diff --git a/python/paddle/fluid/tests/unittests/test_reshape_op.py b/python/paddle/fluid/tests/unittests/test_reshape_op.py index b8cb8bfa88a..dad6e3fa328 100755 --- a/python/paddle/fluid/tests/unittests/test_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/test_reshape_op.py @@ -23,7 +23,6 @@ from paddle.static import Program, program_guard # situation 1: have shape( list, no tensor), no actual shape(Tensor) class TestReshapeOp(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" @@ -31,7 +30,7 @@ class TestReshapeOp(OpTest): self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -47,31 +46,27 @@ class TestReshapeOp(OpTest): class TestReshapeOp_ZeroDim1(OpTest): - def init_data(self): self.ori_shape = () - self.new_shape = (1) - self.infered_shape = (1) + self.new_shape = 1 + self.infered_shape = 1 class TestReshapeOp_ZeroDim2(OpTest): - def init_data(self): self.ori_shape = () - self.new_shape = (-1) - self.infered_shape = (1) + self.new_shape = -1 + self.infered_shape = 1 class TestReshapeOp_ZeroDim3(OpTest): - def init_data(self): - self.ori_shape = (1) + self.ori_shape = 1 self.new_shape = () self.infered_shape = () class TestReshapeBF16Op(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" @@ -81,11 +76,10 @@ class TestReshapeBF16Op(OpTest): self.inputs = {"X": convert_float_to_uint16(x)} self.attrs = {"shape": self.new_shape} self.outputs = { - "Out": - convert_float_to_uint16(out), - 'XShape': - convert_float_to_uint16( - np.random.random(self.ori_shape).astype("float32")) + "Out": convert_float_to_uint16(out), + 'XShape': convert_float_to_uint16( + np.random.random(self.ori_shape).astype("float32") + ), } def init_data(self): @@ -101,7 +95,6 @@ class TestReshapeBF16Op(OpTest): class TestReshapeOpDimInfer1(TestReshapeOp): - def init_data(self): self.ori_shape = (5, 25) self.new_shape = (5, -1, 5) @@ -109,7 +102,6 @@ class TestReshapeOpDimInfer1(TestReshapeOp): class TestReshapeOpDimInfer2(TestReshapeOp): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -118,19 +110,18 @@ class TestReshapeOpDimInfer2(TestReshapeOp): # situation 2: have shape(list, no tensor), have actual shape(Tensor) class TestReshapeOpWithInputShape(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - "Shape": np.array(self.actual_shape, dtype="int32") + "Shape": np.array(self.actual_shape, dtype="int32"), } self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.actual_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -147,24 +138,24 @@ class TestReshapeOpWithInputShape(OpTest): # Situation 3: have shape(list, have tensor), no actual shape(Tensor) class TestReshapeOp_attr_ShapeTensor(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - 'ShapeTensor': shape_tensor + 'ShapeTensor': shape_tensor, } self.attrs = {'shape': self.shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -181,7 +172,6 @@ class TestReshapeOp_attr_ShapeTensor(OpTest): class TestReshapeOpDimInfer1_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor): - def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 20) @@ -190,7 +180,6 @@ class TestReshapeOpDimInfer1_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor): class TestReshapeOpDimInfer2_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -200,19 +189,18 @@ class TestReshapeOpDimInfer2_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor): # Situation 4: have shape(Tensor), no actual shape(Tensor) class TestReshapeOp_attr_OnlyShape(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - "Shape": np.array(self.new_shape, dtype="int32") + "Shape": np.array(self.new_shape, dtype="int32"), } self.attrs = {} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -228,7 +216,6 @@ class TestReshapeOp_attr_OnlyShape(OpTest): class TestReshapeOpDimInfer1_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): - def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 10) @@ -237,7 +224,6 @@ class TestReshapeOpDimInfer1_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): class TestReshapeOpDimInfer2_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -247,7 +233,6 @@ class TestReshapeOpDimInfer2_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): # test int8 data type on CPU class TestReshapeInt8Op(OpTest): - def setUp(self): self.init_dtype() self.init_data() @@ -262,7 +247,7 @@ class TestReshapeInt8Op(OpTest): } self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype(np.float32) + 'XShape': np.random.random(self.ori_shape).astype(np.float32), } def init_dtype(self): @@ -274,9 +259,9 @@ class TestReshapeInt8Op(OpTest): self.infered_shape = (10, 2, 3, -1) def test_check_output(self): - self.check_output_with_place(fluid.core.CPUPlace(), - atol=1e-5, - no_check_set=['XShape']) + self.check_output_with_place( + fluid.core.CPUPlace(), atol=1e-5, no_check_set=['XShape'] + ) def test_check_grad(self): pass @@ -284,13 +269,11 @@ class TestReshapeInt8Op(OpTest): # test unt8 data type on CPU class TestReshapeUint8Op(TestReshapeInt8Op): - def init_dtype(self): self.dtype = np.uint8 class TestReshapeOpBool(TestReshapeOp): - def setUp(self): self.init_data() self.op_type = "reshape2" @@ -300,7 +283,7 @@ class TestReshapeOpBool(TestReshapeOp): self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def test_check_grad(self): @@ -309,7 +292,6 @@ class TestReshapeOpBool(TestReshapeOp): # Test python API class TestReshapeAPI(unittest.TestCase): - def _set_paddle_api(self): self.fill_constant = paddle.fluid.layers.fill_constant self.data = paddle.static.data @@ -339,9 +321,9 @@ class TestReshapeAPI(unittest.TestCase): out_1 = self.reshape(x, shape) # situation 2: have shape(list, no tensor), have actual shape(Tensor) - out_2 = fluid.layers.reshape(x, - shape=shape, - actual_shape=actual_shape) + out_2 = fluid.layers.reshape( + x, shape=shape, actual_shape=actual_shape + ) # Situation 3: have shape(list, have tensor), no actual shape(Tensor) out_3 = self.reshape(x, shape=[positive_five, 10]) @@ -352,11 +334,9 @@ class TestReshapeAPI(unittest.TestCase): exe = paddle.static.Executor(place=paddle.CPUPlace()) res_1, res_2, res_3, res_4 = exe.run( main_prog, - feed={ - "x": input, - "shape": np.array([2, 5, 5]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3, out_4]) + feed={"x": input, "shape": np.array([2, 5, 5]).astype("int32")}, + fetch_list=[out_1, out_2, out_3, out_4], + ) assert np.array_equal(res_1, input.reshape(shape)) assert np.array_equal(res_2, input.reshape(shape)) @@ -392,7 +372,6 @@ class TestReshapeAPI(unittest.TestCase): class TestStaticReshape_(TestReshapeAPI): - def _executed_api(self): self.reshape = paddle.reshape_ @@ -418,7 +397,6 @@ class TestStaticReshape_(TestReshapeAPI): # Test Input Error class TestReshapeOpError(unittest.TestCase): - def _set_paddle_api(self): self.data = paddle.static.data self.reshape = paddle.reshape @@ -431,8 +409,9 @@ class TestReshapeOpError(unittest.TestCase): with program_guard(Program(), Program()): # The x type of reshape_op must be Variable. def test_x_type(): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.CPUPlace() + ) self.reshape(x1, shape=[1]) self.assertRaises(TypeError, test_x_type) @@ -445,9 +424,9 @@ class TestReshapeOpError(unittest.TestCase): self.assertRaises(TypeError, test_x_dtype) def test_x_dtype_float16(): - x_float16 = self.data(name="x_float16", - shape=[2, 25], - dtype="float16") + x_float16 = self.data( + name="x_float16", shape=[2, 25], dtype="float16" + ) self.reshape(x_float16, shape=[2, 5, 5]) test_x_dtype_float16() @@ -494,7 +473,6 @@ class TestReshapeOpError(unittest.TestCase): class TestDygraphReshapeAPI(unittest.TestCase): - def setUp(self): self.executed_api() @@ -530,13 +508,11 @@ class TestDygraphReshapeAPI(unittest.TestCase): class TestDygraphReshapeInplaceAPI(TestDygraphReshapeAPI): - def executed_api(self): self.reshape = paddle.reshape_ class TestReshapeZeroTensor(unittest.TestCase): - def test_reshape_zero_tensor_success(self): zero_tensor = paddle.zeros([0, 2, 3]) # since we use "0" as the dimension copy semantically in reshape, @@ -551,7 +527,6 @@ class TestReshapeZeroTensor(unittest.TestCase): class TestReshapeAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -596,18 +571,18 @@ class TestReshapeAPI_ZeroDim(unittest.TestCase): # Test compile shape self.assertEqual(x.shape, ()) - self.assertEqual(out.shape, (1, )) + self.assertEqual(out.shape, (1,)) self.assertEqual(x_grad.shape, ()) - self.assertEqual(out_grad.shape, (1, )) + self.assertEqual(out_grad.shape, (1,)) exe = fluid.Executor() result = exe.run(main_prog, fetch_list=[x, out, x_grad, out_grad]) # Test runtime shape self.assertEqual(result[0].shape, ()) - self.assertEqual(result[1].shape, (1, )) + self.assertEqual(result[1].shape, (1,)) self.assertEqual(result[2].shape, ()) - self.assertEqual(result[3].shape, (1, )) + self.assertEqual(result[3].shape, (1,)) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py b/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py index 2a16242fc60..754bb0dcb59 100644 --- a/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py +++ b/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py @@ -19,8 +19,9 @@ import unittest paddle.enable_static() -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', - level=logging.INFO) +logging.basicConfig( + format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO +) logger = logging.getLogger(__name__) @@ -36,11 +37,14 @@ def set_cinn_flag(val): @unittest.skipIf(not set_cinn_flag(True), "Paddle is not compiled with CINN.") class TestResnet50Accuracy(unittest.TestCase): - def reader(self, limit): for _ in range(limit): - yield {'image': np.random.randint(0, 256, size=[32, 3, 224, 224]).astype('float32'), \ - 'label': np.random.randint(0, 1000, size=[32]).astype('int64')} + yield { + 'image': np.random.randint( + 0, 256, size=[32, 3, 224, 224] + ).astype('float32'), + 'label': np.random.randint(0, 1000, size=[32]).astype('int64'), + } def generate_random_data(self, loop_num=10): feed = [] @@ -51,9 +55,9 @@ class TestResnet50Accuracy(unittest.TestCase): def build_program(self, main_program, startup_program): with paddle.static.program_guard(main_program, startup_program): - image = paddle.static.data(name='image', - shape=[32, 3, 224, 224], - dtype='float32') + image = paddle.static.data( + name='image', shape=[32, 3, 224, 224], dtype='float32' + ) label = paddle.static.data(name='label', shape=[32], dtype='int64') # TODO: stop_gradient slower training speed, need fix @@ -62,8 +66,9 @@ class TestResnet50Accuracy(unittest.TestCase): model = paddle.vision.models.resnet50() prediction = model(image) - loss = paddle.nn.functional.cross_entropy(input=prediction, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=prediction, label=label + ) loss = paddle.mean(loss) adam = paddle.optimizer.Adam(learning_rate=0.001) adam.minimize(loss) @@ -83,23 +88,29 @@ class TestResnet50Accuracy(unittest.TestCase): exe = paddle.static.Executor(place) compiled_prog = paddle.static.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name) + main_program + ).with_data_parallel(loss_name=loss.name) loss_vals = [] scope = paddle.static.Scope() with paddle.static.scope_guard(scope): exe.run(startup_program) for step in range(iters): - loss_v = exe.run(compiled_prog, - feed=feed[step], - fetch_list=[loss], - return_numpy=True) + loss_v = exe.run( + compiled_prog, + feed=feed[step], + fetch_list=[loss], + return_numpy=True, + ) loss_vals.append(loss_v[0][0]) return loss_vals def test_check_resnet50_accuracy(self): - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) loop_num = 10 feed = self.generate_random_data(loop_num) diff --git a/python/paddle/fluid/tests/unittests/test_retain_graph.py b/python/paddle/fluid/tests/unittests/test_retain_graph.py index 71998f57e5f..06f0be1b879 100644 --- a/python/paddle/fluid/tests/unittests/test_retain_graph.py +++ b/python/paddle/fluid/tests/unittests/test_retain_graph.py @@ -24,7 +24,6 @@ paddle.seed(SEED) class Generator(fluid.dygraph.Layer): - def __init__(self): super(Generator, self).__init__() self.conv1 = paddle.nn.Conv2D(3, 3, 3, padding=1) @@ -36,7 +35,6 @@ class Generator(fluid.dygraph.Layer): class Discriminator(fluid.dygraph.Layer): - def __init__(self): super(Discriminator, self).__init__() self.convd = paddle.nn.Conv2D(6, 3, 1) @@ -47,15 +45,16 @@ class Discriminator(fluid.dygraph.Layer): class TestRetainGraph(unittest.TestCase): - - def cal_gradient_penalty(self, - netD, - real_data, - fake_data, - edge_data=None, - type='mixed', - constant=1.0, - lambda_gp=10.0): + def cal_gradient_penalty( + self, + netD, + real_data, + fake_data, + edge_data=None, + type='mixed', + constant=1.0, + lambda_gp=10.0, + ): if lambda_gp > 0.0: if type == 'real': interpolatesv = real_data @@ -63,10 +62,13 @@ class TestRetainGraph(unittest.TestCase): interpolatesv = fake_data elif type == 'mixed': alpha = paddle.rand((real_data.shape[0], 1)) - alpha = paddle.expand(alpha, [ - real_data.shape[0], - np.prod(real_data.shape) // real_data.shape[0] - ]) + alpha = paddle.expand( + alpha, + [ + real_data.shape[0], + np.prod(real_data.shape) // real_data.shape[0], + ], + ) alpha = paddle.reshape(alpha, real_data.shape) interpolatesv = alpha * real_data + ((1 - alpha) * fake_data) else: @@ -76,21 +78,26 @@ class TestRetainGraph(unittest.TestCase): fake_AB = paddle.concat((real_data.detach(), interpolatesv), 1) disc_interpolates = netD(fake_AB) - outs = paddle.fluid.layers.fill_constant(disc_interpolates.shape, - disc_interpolates.dtype, - 1.0) - gradients = paddle.grad(outputs=disc_interpolates, - inputs=fake_AB, - grad_outputs=outs, - create_graph=True, - retain_graph=True, - only_inputs=True) + outs = paddle.fluid.layers.fill_constant( + disc_interpolates.shape, disc_interpolates.dtype, 1.0 + ) + gradients = paddle.grad( + outputs=disc_interpolates, + inputs=fake_AB, + grad_outputs=outs, + create_graph=True, + retain_graph=True, + only_inputs=True, + ) gradients = paddle.reshape(gradients[0], [real_data.shape[0], -1]) - gradient_penalty = paddle.mean( - (paddle.norm(gradients + 1e-16, 2, 1) - constant)** - 2) * lambda_gp # added eps + gradient_penalty = ( + paddle.mean( + (paddle.norm(gradients + 1e-16, 2, 1) - constant) ** 2 + ) + * lambda_gp + ) # added eps return gradient_penalty, gradients else: return 0.0, None @@ -117,12 +124,12 @@ class TestRetainGraph(unittest.TestCase): G_pred_fake = d(fake_AB.detach()) false_target = paddle.fluid.layers.fill_constant( - G_pred_fake.shape, 'float32', 0.0) + G_pred_fake.shape, 'float32', 0.0 + ) - G_gradient_penalty, _ = self.cal_gradient_penalty(d, - realA, - fakeB, - lambda_gp=10.0) + G_gradient_penalty, _ = self.cal_gradient_penalty( + d, realA, fakeB, lambda_gp=10.0 + ) loss_d = gan_criterion(G_pred_fake, false_target) + G_gradient_penalty loss_d.backward(retain_graph=need_retain) @@ -131,10 +138,12 @@ class TestRetainGraph(unittest.TestCase): optim_g.clear_gradients() fake_AB = paddle.concat((realA, fakeB), 1) G_pred_fake = d(fake_AB) - true_target = paddle.fluid.layers.fill_constant(G_pred_fake.shape, - 'float32', 1.0) + true_target = paddle.fluid.layers.fill_constant( + G_pred_fake.shape, 'float32', 1.0 + ) loss_g = l1_criterion(fakeB, realB) + gan_criterion( - G_pred_fake, true_target) + G_pred_fake, true_target + ) loss_g.backward() optim_g.minimize(loss_g) diff --git a/python/paddle/fluid/tests/unittests/test_retinanet_detection_output.py b/python/paddle/fluid/tests/unittests/test_retinanet_detection_output.py index 61bcb875217..5d8527477af 100644 --- a/python/paddle/fluid/tests/unittests/test_retinanet_detection_output.py +++ b/python/paddle/fluid/tests/unittests/test_retinanet_detection_output.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License") -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -42,9 +42,9 @@ def multiclass_nms(prediction, class_num, keep_top_k, nms_threshold): for idx in indices: score_index.append((prediction[c][idx][4], c, idx)) - sorted_score_index = sorted(score_index, - key=lambda tup: tup[0], - reverse=True) + sorted_score_index = sorted( + score_index, key=lambda tup: tup[0], reverse=True + ) if keep_top_k > -1 and num_det > keep_top_k: sorted_score_index = sorted_score_index[:keep_top_k] num_det = keep_top_k @@ -59,9 +59,16 @@ def multiclass_nms(prediction, class_num, keep_top_k, nms_threshold): return nmsed_outs, num_det -def retinanet_detection_out(boxes_list, scores_list, anchors_list, im_info, - score_threshold, nms_threshold, nms_top_k, - keep_top_k): +def retinanet_detection_out( + boxes_list, + scores_list, + anchors_list, + im_info, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, +): class_num = scores_list[0].shape[-1] im_height, im_width, im_scale = im_info @@ -88,23 +95,37 @@ def retinanet_detection_out(boxes_list, scores_list, anchors_list, im_info, a = int(idx / class_num) c = int(idx % class_num) box_offset = a * 4 - anchor_box_width = anchors_per_level[ - box_offset + 2] - anchors_per_level[box_offset] + 1 - anchor_box_height = anchors_per_level[ - box_offset + 3] - anchors_per_level[box_offset + 1] + 1 - anchor_box_center_x = anchors_per_level[ - box_offset] + anchor_box_width / 2 - anchor_box_center_y = anchors_per_level[box_offset + - 1] + anchor_box_height / 2 - - target_box_center_x = bboxes_per_level[ - box_offset] * anchor_box_width + anchor_box_center_x - target_box_center_y = bboxes_per_level[ - box_offset + 1] * anchor_box_height + anchor_box_center_y - target_box_width = math.exp( - bboxes_per_level[box_offset + 2]) * anchor_box_width - target_box_height = math.exp( - bboxes_per_level[box_offset + 3]) * anchor_box_height + anchor_box_width = ( + anchors_per_level[box_offset + 2] + - anchors_per_level[box_offset] + + 1 + ) + anchor_box_height = ( + anchors_per_level[box_offset + 3] + - anchors_per_level[box_offset + 1] + + 1 + ) + anchor_box_center_x = ( + anchors_per_level[box_offset] + anchor_box_width / 2 + ) + anchor_box_center_y = ( + anchors_per_level[box_offset + 1] + anchor_box_height / 2 + ) + + target_box_center_x = ( + bboxes_per_level[box_offset] * anchor_box_width + + anchor_box_center_x + ) + target_box_center_y = ( + bboxes_per_level[box_offset + 1] * anchor_box_height + + anchor_box_center_y + ) + target_box_width = ( + math.exp(bboxes_per_level[box_offset + 2]) * anchor_box_width + ) + target_box_height = ( + math.exp(bboxes_per_level[box_offset + 3]) * anchor_box_height + ) pred_box_xmin = target_box_center_x - target_box_width / 2 pred_box_ymin = target_box_center_y - target_box_height / 2 @@ -117,33 +138,46 @@ def retinanet_detection_out(boxes_list, scores_list, anchors_list, im_info, pred_box_ymax = pred_box_ymax / im_scale pred_box_xmin = max( - min(pred_box_xmin, - np.round(im_width / im_scale) - 1), 0.) + min(pred_box_xmin, np.round(im_width / im_scale) - 1), 0.0 + ) pred_box_ymin = max( - min(pred_box_ymin, - np.round(im_height / im_scale) - 1), 0.) + min(pred_box_ymin, np.round(im_height / im_scale) - 1), 0.0 + ) pred_box_xmax = max( - min(pred_box_xmax, - np.round(im_width / im_scale) - 1), 0.) + min(pred_box_xmax, np.round(im_width / im_scale) - 1), 0.0 + ) pred_box_ymax = max( - min(pred_box_ymax, - np.round(im_height / im_scale) - 1), 0.) + min(pred_box_ymax, np.round(im_height / im_scale) - 1), 0.0 + ) if c not in prediction.keys(): prediction[c] = [] - prediction[c].append([ - pred_box_xmin, pred_box_ymin, pred_box_xmax, pred_box_ymax, - scores_per_level[idx] - ]) - - nmsed_outs, nmsed_num = multiclass_nms(prediction, class_num, keep_top_k, - nms_threshold) + prediction[c].append( + [ + pred_box_xmin, + pred_box_ymin, + pred_box_xmax, + pred_box_ymax, + scores_per_level[idx], + ] + ) + + nmsed_outs, nmsed_num = multiclass_nms( + prediction, class_num, keep_top_k, nms_threshold + ) return nmsed_outs, nmsed_num -def batched_retinanet_detection_out(boxes, scores, anchors, im_info, - score_threshold, nms_threshold, nms_top_k, - keep_top_k): +def batched_retinanet_detection_out( + boxes, + scores, + anchors, + im_info, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, +): batch_size = scores[0].shape[0] det_outs = [] lod = [] @@ -158,8 +192,15 @@ def batched_retinanet_detection_out(boxes, scores, anchors, im_info, scores_per_batch.append(scores[lvl][n]) nmsed_outs, nmsed_num = retinanet_detection_out( - boxes_per_batch, scores_per_batch, anchors, im_info[n], - score_threshold, nms_threshold, nms_top_k, keep_top_k) + boxes_per_batch, + scores_per_batch, + anchors, + im_info[n], + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) lod.append(nmsed_num) if nmsed_num == 0: continue @@ -169,7 +210,6 @@ def batched_retinanet_detection_out(boxes, scores, anchors, im_info, class TestRetinanetDetectionOutOp1(OpTest): - def set_argument(self): self.score_threshold = 0.05 self.min_level = 3 @@ -192,8 +232,8 @@ class TestRetinanetDetectionOutOp1(OpTest): self.layer_w = [] num_levels = self.max_level - self.min_level + 1 for i in range(num_levels): - self.layer_h.append(2**(num_levels - i)) - self.layer_w.append(2**(num_levels - i)) + self.layer_h.append(2 ** (num_levels - i)) + self.layer_w.append(2 ** (num_levels - i)) def init_test_input(self): anchor_num = len(self.aspect_ratios) * self.scales_per_octave @@ -206,62 +246,89 @@ class TestRetinanetDetectionOutOp1(OpTest): layer_h = self.layer_h[i] layer_w = self.layer_w[i] - input_feat = np.random.random((self.batch_size, self.input_channels, - layer_h, layer_w)).astype('float32') + input_feat = np.random.random( + (self.batch_size, self.input_channels, layer_h, layer_w) + ).astype('float32') score = np.random.random( - (self.batch_size, self.class_num * anchor_num, layer_h, - layer_w)).astype('float32') + (self.batch_size, self.class_num * anchor_num, layer_h, layer_w) + ).astype('float32') score = np.transpose(score, [0, 2, 3, 1]) score = score.reshape((self.batch_size, -1, self.class_num)) - box = np.random.random((self.batch_size, self.box_size * anchor_num, - layer_h, layer_w)).astype('float32') + box = np.random.random( + (self.batch_size, self.box_size * anchor_num, layer_h, layer_w) + ).astype('float32') box = np.transpose(box, [0, 2, 3, 1]) box = box.reshape((self.batch_size, -1, self.box_size)) anchor_sizes = [] for octave in range(self.scales_per_octave): anchor_sizes.append( - float(self.anchor_strides[i] * (2**octave)) / - float(self.scales_per_octave) * self.anchor_scale) + float(self.anchor_strides[i] * (2**octave)) + / float(self.scales_per_octave) + * self.anchor_scale + ) anchor, var = anchor_generator_in_python( input_feat=input_feat, anchor_sizes=anchor_sizes, aspect_ratios=self.aspect_ratios, variances=[1.0, 1.0, 1.0, 1.0], stride=[self.anchor_strides[i], self.anchor_strides[i]], - offset=0.5) + offset=0.5, + ) anchor = np.reshape(anchor, [-1, 4]) self.scores_list.append(score.astype('float32')) self.bboxes_list.append(box.astype('float32')) self.anchors_list.append(anchor.astype('float32')) - self.im_info = np.array([[256., 256., 1.5]]).astype( - 'float32') #im_height, im_width, scale + self.im_info = np.array([[256.0, 256.0, 1.5]]).astype( + 'float32' + ) # im_height, im_width, scale def setUp(self): self.set_argument() self.init_test_input() nmsed_outs, lod = batched_retinanet_detection_out( - self.bboxes_list, self.scores_list, self.anchors_list, self.im_info, - self.score_threshold, self.nms_threshold, self.nms_top_k, - self.keep_top_k) + self.bboxes_list, + self.scores_list, + self.anchors_list, + self.im_info, + self.score_threshold, + self.nms_threshold, + self.nms_top_k, + self.keep_top_k, + ) nmsed_outs = np.array(nmsed_outs).astype('float32') self.op_type = 'retinanet_detection_output' self.inputs = { - 'BBoxes': [('b0', self.bboxes_list[0]), ('b1', self.bboxes_list[1]), - ('b2', self.bboxes_list[2]), ('b3', self.bboxes_list[3]), - ('b4', self.bboxes_list[4])], - 'Scores': [('s0', self.scores_list[0]), ('s1', self.scores_list[1]), - ('s2', self.scores_list[2]), ('s3', self.scores_list[3]), - ('s4', self.scores_list[4])], - 'Anchors': [('a0', self.anchors_list[0]), - ('a1', self.anchors_list[1]), - ('a2', self.anchors_list[2]), - ('a3', self.anchors_list[3]), - ('a4', self.anchors_list[4])], - 'ImInfo': (self.im_info, [[ - 1, - ]]) + 'BBoxes': [ + ('b0', self.bboxes_list[0]), + ('b1', self.bboxes_list[1]), + ('b2', self.bboxes_list[2]), + ('b3', self.bboxes_list[3]), + ('b4', self.bboxes_list[4]), + ], + 'Scores': [ + ('s0', self.scores_list[0]), + ('s1', self.scores_list[1]), + ('s2', self.scores_list[2]), + ('s3', self.scores_list[3]), + ('s4', self.scores_list[4]), + ], + 'Anchors': [ + ('a0', self.anchors_list[0]), + ('a1', self.anchors_list[1]), + ('a2', self.anchors_list[2]), + ('a3', self.anchors_list[3]), + ('a4', self.anchors_list[4]), + ], + 'ImInfo': ( + self.im_info, + [ + [ + 1, + ] + ], + ), } self.outputs = {'Out': (nmsed_outs, [lod])} self.attrs = { @@ -269,7 +336,7 @@ class TestRetinanetDetectionOutOp1(OpTest): 'nms_top_k': self.nms_top_k, 'nms_threshold': self.nms_threshold, 'keep_top_k': self.keep_top_k, - 'nms_eta': 1., + 'nms_eta': 1.0, } def test_check_output(self): @@ -277,7 +344,6 @@ class TestRetinanetDetectionOutOp1(OpTest): class TestRetinanetDetectionOutOp2(OpTest): - def set_argument(self): self.score_threshold = 0.05 self.min_level = 3 @@ -302,7 +368,6 @@ class TestRetinanetDetectionOutOp2(OpTest): class TestRetinanetDetectionOutOpNo3(TestRetinanetDetectionOutOp1): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 @@ -327,12 +392,11 @@ class TestRetinanetDetectionOutOpNo3(TestRetinanetDetectionOutOp1): self.layer_w = [] num_levels = self.max_level - self.min_level + 1 for i in range(num_levels): - self.layer_h.append(2**(num_levels - i)) - self.layer_w.append(2**(num_levels - i)) + self.layer_h.append(2 ** (num_levels - i)) + self.layer_w.append(2 ** (num_levels - i)) class TestRetinanetDetectionOutOpNo4(TestRetinanetDetectionOutOp1): - def set_argument(self): self.score_threshold = 0.05 self.min_level = 2 @@ -355,33 +419,52 @@ class TestRetinanetDetectionOutOpNo4(TestRetinanetDetectionOutOp1): self.layer_w = [] num_levels = self.max_level - self.min_level + 1 for i in range(num_levels): - self.layer_h.append(2**(num_levels - i)) - self.layer_w.append(2**(num_levels - i)) + self.layer_h.append(2 ** (num_levels - i)) + self.layer_w.append(2 ** (num_levels - i)) def setUp(self): self.set_argument() self.init_test_input() nmsed_outs, lod = batched_retinanet_detection_out( - self.bboxes_list, self.scores_list, self.anchors_list, self.im_info, - self.score_threshold, self.nms_threshold, self.nms_top_k, - self.keep_top_k) + self.bboxes_list, + self.scores_list, + self.anchors_list, + self.im_info, + self.score_threshold, + self.nms_threshold, + self.nms_top_k, + self.keep_top_k, + ) nmsed_outs = np.array(nmsed_outs).astype('float32') self.op_type = 'retinanet_detection_output' self.inputs = { - 'BBoxes': [('b0', self.bboxes_list[0]), ('b1', self.bboxes_list[1]), - ('b2', self.bboxes_list[2]), - ('b3', self.bboxes_list[3])], - 'Scores': [('s0', self.scores_list[0]), ('s1', self.scores_list[1]), - ('s2', self.scores_list[2]), - ('s3', self.scores_list[3])], - 'Anchors': [('a0', self.anchors_list[0]), - ('a1', self.anchors_list[1]), - ('a2', self.anchors_list[2]), - ('a3', self.anchors_list[3])], - 'ImInfo': (self.im_info, [[ - 1, - ]]) + 'BBoxes': [ + ('b0', self.bboxes_list[0]), + ('b1', self.bboxes_list[1]), + ('b2', self.bboxes_list[2]), + ('b3', self.bboxes_list[3]), + ], + 'Scores': [ + ('s0', self.scores_list[0]), + ('s1', self.scores_list[1]), + ('s2', self.scores_list[2]), + ('s3', self.scores_list[3]), + ], + 'Anchors': [ + ('a0', self.anchors_list[0]), + ('a1', self.anchors_list[1]), + ('a2', self.anchors_list[2]), + ('a3', self.anchors_list[3]), + ], + 'ImInfo': ( + self.im_info, + [ + [ + 1, + ] + ], + ), } self.outputs = {'Out': (nmsed_outs, [lod])} self.attrs = { @@ -389,7 +472,7 @@ class TestRetinanetDetectionOutOpNo4(TestRetinanetDetectionOutOp1): 'nms_top_k': self.nms_top_k, 'nms_threshold': self.nms_threshold, 'keep_top_k': self.keep_top_k, - 'nms_eta': 1., + 'nms_eta': 1.0, } def test_check_output(self): @@ -397,7 +480,6 @@ class TestRetinanetDetectionOutOpNo4(TestRetinanetDetectionOutOp1): class TestRetinanetDetectionOutOpNo5(TestRetinanetDetectionOutOp1): - def set_argument(self): self.score_threshold = 0.05 self.min_level = 3 @@ -420,35 +502,34 @@ class TestRetinanetDetectionOutOpNo5(TestRetinanetDetectionOutOp1): self.layer_w = [] num_levels = self.max_level - self.min_level + 1 for i in range(num_levels): - self.layer_h.append(2**(num_levels - i)) - self.layer_w.append(2**(num_levels - i)) + self.layer_h.append(2 ** (num_levels - i)) + self.layer_w.append(2 ** (num_levels - i)) class TestRetinanetDetectionOutOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - bboxes_low1 = fluid.data(name='bboxes_low1', - shape=[1, 44, 4], - dtype='float32') - bboxes_high1 = fluid.data(name='bboxes_high1', - shape=[1, 11, 4], - dtype='float32') - scores_low1 = fluid.data(name='scores_low1', - shape=[1, 44, 10], - dtype='float32') - scores_high1 = fluid.data(name='scores_high1', - shape=[1, 11, 10], - dtype='float32') - anchors_low1 = fluid.data(name='anchors_low1', - shape=[44, 4], - dtype='float32') - anchors_high1 = fluid.data(name='anchors_high1', - shape=[11, 4], - dtype='float32') - im_info1 = fluid.data(name="im_info1", - shape=[1, 3], - dtype='float32') + bboxes_low1 = fluid.data( + name='bboxes_low1', shape=[1, 44, 4], dtype='float32' + ) + bboxes_high1 = fluid.data( + name='bboxes_high1', shape=[1, 11, 4], dtype='float32' + ) + scores_low1 = fluid.data( + name='scores_low1', shape=[1, 44, 10], dtype='float32' + ) + scores_high1 = fluid.data( + name='scores_high1', shape=[1, 11, 10], dtype='float32' + ) + anchors_low1 = fluid.data( + name='anchors_low1', shape=[44, 4], dtype='float32' + ) + anchors_high1 = fluid.data( + name='anchors_high1', shape=[11, 4], dtype='float32' + ) + im_info1 = fluid.data( + name="im_info1", shape=[1, 3], dtype='float32' + ) # The `bboxes` must be list, each element must be Variable and # its Tensor data type must be one of float32 and float64. @@ -457,19 +538,21 @@ class TestRetinanetDetectionOutOpError(unittest.TestCase): bboxes=bboxes_low1, scores=[scores_low1, scores_high1], anchors=[anchors_low1, anchors_high1], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_bboxes_type) def test_bboxes_tensor_dtype(): - bboxes_high2 = fluid.data(name='bboxes_high2', - shape=[1, 11, 4], - dtype='int32') + bboxes_high2 = fluid.data( + name='bboxes_high2', shape=[1, 11, 4], dtype='int32' + ) fluid.layers.retinanet_detection_output( bboxes=[bboxes_high2, 5], scores=[scores_low1, scores_high1], anchors=[anchors_low1, anchors_high1], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_bboxes_tensor_dtype) @@ -480,19 +563,21 @@ class TestRetinanetDetectionOutOpError(unittest.TestCase): bboxes=[bboxes_low1, bboxes_high1], scores=scores_low1, anchors=[anchors_low1, anchors_high1], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_scores_type) def test_scores_tensor_dtype(): - scores_high2 = fluid.data(name='scores_high2', - shape=[1, 11, 10], - dtype='int32') + scores_high2 = fluid.data( + name='scores_high2', shape=[1, 11, 10], dtype='int32' + ) fluid.layers.retinanet_detection_output( bboxes=[bboxes_low1, bboxes_high1], scores=[scores_high2, 5], anchors=[anchors_low1, anchors_high1], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_scores_tensor_dtype) @@ -503,19 +588,21 @@ class TestRetinanetDetectionOutOpError(unittest.TestCase): bboxes=[bboxes_low1, bboxes_high1], scores=[scores_low1, scores_high1], anchors=anchors_low1, - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_anchors_type) def test_anchors_tensor_dtype(): - anchors_high2 = fluid.data(name='anchors_high2', - shape=[11, 4], - dtype='int32') + anchors_high2 = fluid.data( + name='anchors_high2', shape=[11, 4], dtype='int32' + ) fluid.layers.retinanet_detection_output( bboxes=[bboxes_low1, bboxes_high1], scores=[scores_low1, scores_high1], anchors=[anchors_high2, 5], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_anchors_tensor_dtype) @@ -526,19 +613,21 @@ class TestRetinanetDetectionOutOpError(unittest.TestCase): bboxes=[bboxes_low1, bboxes_high1], scores=[scores_low1, scores_high1], anchors=[anchors_low1, anchors_high1], - im_info=[2, 3, 4]) + im_info=[2, 3, 4], + ) self.assertRaises(TypeError, test_iminfo_type) def test_iminfo_tensor_dtype(): - im_info2 = fluid.data(name='im_info2', - shape=[1, 3], - dtype='int32') + im_info2 = fluid.data( + name='im_info2', shape=[1, 3], dtype='int32' + ) fluid.layers.retinanet_detection_output( bboxes=[bboxes_low1, bboxes_high1], scores=[scores_low1, scores_high1], anchors=[anchors_low1, anchors_high1], - im_info=im_info2) + im_info=im_info2, + ) self.assertRaises(TypeError, test_iminfo_tensor_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_reverse_op.py b/python/paddle/fluid/tests/unittests/test_reverse_op.py index fc304d516f5..829ef2a883b 100644 --- a/python/paddle/fluid/tests/unittests/test_reverse_op.py +++ b/python/paddle/fluid/tests/unittests/test_reverse_op.py @@ -28,7 +28,6 @@ from test_attribute_var import UnittestBase class TestReverseOp(OpTest): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [0] @@ -52,63 +51,54 @@ class TestReverseOp(OpTest): class TestCase0(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [1] class TestCase0_neg(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [-1] class TestCase1(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [0, 1] class TestCase1_neg(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [0, -1] class TestCase2(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 4, 10)).astype('float64') self.axis = [0, 2] class TestCase2_neg(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 4, 10)).astype('float64') self.axis = [0, -2] class TestCase3(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 4, 10)).astype('float64') self.axis = [1, 2] class TestCase3_neg(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 4, 10)).astype('float64') self.axis = [-1, -2] class TestCase4(unittest.TestCase): - def test_error(self): place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -116,9 +106,9 @@ class TestCase4(unittest.TestCase): train_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(train_program, startup_program): - label = fluid.layers.data(name="label", - shape=[1, 1, 1, 1, 1, 1, 1, 1], - dtype="int64") + label = fluid.layers.data( + name="label", shape=[1, 1, 1, 1, 1, 1, 1, 1], dtype="int64" + ) rev = fluid.layers.reverse(label, axis=[-1, -2]) def _run_program(): @@ -129,11 +119,13 @@ class TestCase4(unittest.TestCase): class TestReverseLoDTensorArray(unittest.TestCase): - def setUp(self): self.shapes = [[5, 25], [5, 20], [5, 5]] - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.exe = fluid.Executor(self.place) def run_program(self, arr_len, axis=0): @@ -146,7 +138,8 @@ class TestReverseLoDTensorArray(unittest.TestCase): x.stop_gradient = False inputs.append(x) inputs_data.append( - np.random.random(self.shapes[i]).astype('float32')) + np.random.random(self.shapes[i]).astype('float32') + ) tensor_array = fluid.layers.create_array(dtype='float32') for i in range(arr_len): @@ -158,13 +151,18 @@ class TestReverseLoDTensorArray(unittest.TestCase): loss = fluid.layers.reduce_sum(output) fluid.backward.append_backward(loss) input_grads = list( - map(main_program.global_block().var, - [x.name + "@GRAD" for x in inputs])) + map( + main_program.global_block().var, + [x.name + "@GRAD" for x in inputs], + ) + ) feed_dict = dict(zip([x.name for x in inputs], inputs_data)) - res = self.exe.run(main_program, - feed=feed_dict, - fetch_list=input_grads + [output.name]) + res = self.exe.run( + main_program, + feed=feed_dict, + fetch_list=input_grads + [output.name], + ) return np.hstack(inputs_data[::-1]), res @@ -172,7 +170,7 @@ class TestReverseLoDTensorArray(unittest.TestCase): gt, res = self.run_program(arr_len=3) self.check_output(gt, res) # test with tuple type of axis - gt, res = self.run_program(arr_len=3, axis=(0, )) + gt, res = self.run_program(arr_len=3, axis=(0,)) self.check_output(gt, res) def test_case2(self): @@ -201,7 +199,6 @@ class TestReverseLoDTensorArray(unittest.TestCase): class TestReverseAxisTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -227,8 +224,9 @@ class TestReverseAxisTensor(UnittestBase): gt = res[0][::-1, :, ::-1] np.testing.assert_allclose(res[1], gt) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() gt = infer_outs[0][::-1, :, ::-1] @@ -248,7 +246,6 @@ class TestReverseAxisTensor(UnittestBase): class TestReverseAxisListTensor(TestReverseAxisTensor): - def path_prefix(self): return 'reverse_tensors' @@ -261,15 +258,18 @@ class TestReverseAxisListTensor(TestReverseAxisTensor): out = paddle.fluid.layers.reverse(x, axes) # check attrs - axis_attrs = paddle.static.default_main_program().block( - 0).ops[-1].all_attrs()["axis"] + axis_attrs = ( + paddle.static.default_main_program() + .block(0) + .ops[-1] + .all_attrs()["axis"] + ) self.assertTrue(axis_attrs[0].name, axes[0].name) self.assertTrue(axis_attrs[1].name, axes[1].name) return out class TestReverseDoubleGradCheck(unittest.TestCase): - def reverse_wrapper(self, x): return fluid.layers.reverse(x[0], [0, 1]) @@ -284,17 +284,13 @@ class TestReverseDoubleGradCheck(unittest.TestCase): out = fluid.layers.reverse(data, [0, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.reverse_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.reverse_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -306,7 +302,6 @@ class TestReverseDoubleGradCheck(unittest.TestCase): class TestReverseTripleGradCheck(unittest.TestCase): - def reverse_wrapper(self, x): return fluid.layers.reverse(x[0], [0, 1]) @@ -321,17 +316,13 @@ class TestReverseTripleGradCheck(unittest.TestCase): out = fluid.layers.reverse(data, [0, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.reverse_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.reverse_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py index 1948026b374..5845b421fb5 100644 --- a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py +++ b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py @@ -21,14 +21,19 @@ import paddle.fluid as fluid import paddle -def create_selected_rows_and_tensor(scope, place, height, row_num, - embedding_size): +def create_selected_rows_and_tensor( + scope, place, height, row_num, embedding_size +): sr = scope.var("@selected_rows@").get_selected_rows() tensor = scope.var("grad").get_tensor() - rows = np.random.random_integers(low=0, high=height - 1, size=[ - row_num, - ]).astype('int64') + rows = np.random.random_integers( + low=0, + high=height - 1, + size=[ + row_num, + ], + ).astype('int64') sr_val = np.random.random(size=[row_num, embedding_size]).astype('float32') sr.set_height(height) @@ -45,14 +50,9 @@ def create_selected_rows_and_tensor(scope, place, height, row_num, class TestBase(unittest.TestCase): - - def setup(self, - place, - is_sparse, - centered, - size, - row_num=None, - epsilon=1e-6): + def setup( + self, place, is_sparse, centered, size, row_num=None, epsilon=1e-6 + ): np.random.seed(5) # fix seed self.scope = fluid.global_scope() @@ -62,8 +62,9 @@ class TestBase(unittest.TestCase): self.param = np.random.random(size).astype("float32") self.mean_square_name = "mean_square" - self.mean_square = np.random.uniform(low=1, high=2, - size=size).astype("float32") + self.mean_square = np.random.uniform(low=1, high=2, size=size).astype( + "float32" + ) self.mean_grad_name = "mean_grad" self.mean_grad = np.random.random(size).astype("float32") @@ -77,31 +78,44 @@ class TestBase(unittest.TestCase): if self.is_sparse: self.grad_sr_name = "@selected_rows@" self.grad, self.grad_sr = create_selected_rows_and_tensor( - self.scope, place, size[0], row_num, size[1]) + self.scope, place, size[0], row_num, size[1] + ) else: self.grad = np.random.random(size).astype("float32") grad_tensor = self.scope.var(self.grad_name).get_tensor() grad_tensor.set(self.grad, place) self.moment_name = "moment" - self.moment = np.random.uniform(low=0, high=1, - size=size).astype("float32") + self.moment = np.random.uniform(low=0, high=1, size=size).astype( + "float32" + ) self.epsilon = epsilon self.decay = 0.9 self.momentum = 0.1 self.centered = centered - self.ms_out = self.decay * self.mean_square + ( - 1 - self.decay) * self.grad * self.grad + self.ms_out = ( + self.decay * self.mean_square + + (1 - self.decay) * self.grad * self.grad + ) if centered: - self.mg_out = self.decay * self.mean_grad + (1 - - self.decay) * self.grad - self.moment_out = self.momentum * self.moment + \ - self.learning_rate * self.grad / np.sqrt(self.ms_out - np.square(self.mg_out) + self.epsilon) + self.mg_out = ( + self.decay * self.mean_grad + (1 - self.decay) * self.grad + ) + self.moment_out = ( + self.momentum * self.moment + + self.learning_rate + * self.grad + / np.sqrt(self.ms_out - np.square(self.mg_out) + self.epsilon) + ) else: - self.moment_out = self.momentum * self.moment + \ - self.learning_rate * self.grad / np.sqrt(self.ms_out + self.epsilon) + self.moment_out = ( + self.momentum * self.moment + + self.learning_rate + * self.grad + / np.sqrt(self.ms_out + self.epsilon) + ) self.param_out = self.param - self.moment_out @@ -110,7 +124,8 @@ class TestBase(unittest.TestCase): self.param_tensor.set(self.param, place) self.mean_square_tensor = self.scope.var( - self.mean_square_name).get_tensor() + self.mean_square_name + ).get_tensor() self.mean_square_tensor.set(self.mean_square, place) lr = self.scope.var(self.lr_name).get_tensor() @@ -121,7 +136,8 @@ class TestBase(unittest.TestCase): if self.centered: self.mean_grad_tensor = self.scope.var( - self.mean_grad_name).get_tensor() + self.mean_grad_name + ).get_tensor() self.mean_grad_tensor.set(self.mean_grad, place) def check(self, actual_t, expect_t, place, out_name, atol=1e-5): @@ -130,19 +146,22 @@ class TestBase(unittest.TestCase): expect_t, rtol=1e-05, atol=atol, - err_msg='Output (' + out_name + ') has diff at ' + str(place) + - '\nExpect ' + str(expect_t) + '\n' + 'But Got' + str(actual_t)) + err_msg='Output (' + + out_name + + ') has diff at ' + + str(place) + + '\nExpect ' + + str(expect_t) + + '\n' + + 'But Got' + + str(actual_t), + ) class TestRmspropOp(TestBase): - - def check_with_place(self, - place, - is_sparse, - centered, - size, - row_num=None, - epsilon=1e-6): + def check_with_place( + self, place, is_sparse, centered, size, row_num=None, epsilon=1e-6 + ): self.setup(place, is_sparse, centered, size, row_num, epsilon) self.run_and_check() @@ -161,7 +180,7 @@ class TestRmspropOp(TestBase): 'epsilon': self.epsilon, 'decay': self.decay, 'momentum': self.momentum, - 'centered': self.centered + 'centered': self.centered, } if self.centered: @@ -173,25 +192,35 @@ class TestRmspropOp(TestBase): rmsprop_op.run(self.scope, self.place) - self.check(np.array(self.mean_square_tensor), - self.ms_out, - self.place, - self.mean_square_name, - atol=atol) - self.check(np.array(self.moment_tensor), - self.moment_out, - self.place, - self.moment_name, - atol=atol) - self.check(np.array(self.param_tensor), - self.param_out, - self.place, - self.param_name, - atol=atol) + self.check( + np.array(self.mean_square_tensor), + self.ms_out, + self.place, + self.mean_square_name, + atol=atol, + ) + self.check( + np.array(self.moment_tensor), + self.moment_out, + self.place, + self.moment_name, + atol=atol, + ) + self.check( + np.array(self.param_tensor), + self.param_out, + self.place, + self.param_name, + atol=atol, + ) if self.centered: - self.check(np.array(self.mean_grad_tensor), self.mg_out, self.place, - self.mean_grad_name) + self.check( + np.array(self.mean_grad_tensor), + self.mg_out, + self.place, + self.mean_grad_name, + ) def test_rmsprop(self): places = [core.CPUPlace()] @@ -202,37 +231,41 @@ class TestRmspropOp(TestBase): for place in places: for centered in [False, True]: with fluid.scope_guard(core.Scope()): - self.check_with_place(place, - is_sparse=False, - centered=centered, - size=size) + self.check_with_place( + place, is_sparse=False, centered=centered, size=size + ) with fluid.scope_guard(core.Scope()): - self.check_with_place(place, - is_sparse=True, - centered=centered, - row_num=512, - size=size) + self.check_with_place( + place, + is_sparse=True, + centered=centered, + row_num=512, + size=size, + ) with fluid.scope_guard(core.Scope()): - self.check_with_place(place, - is_sparse=True, - centered=centered, - row_num=60, - size=size) + self.check_with_place( + place, + is_sparse=True, + centered=centered, + row_num=60, + size=size, + ) class TestRMSPropV2(unittest.TestCase): - def test_rmsprop_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.RMSProp(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.RMSProp( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.01, + ) out = linear(a) out.backward() adam.step() @@ -253,8 +286,9 @@ class TestRMSPropV2(unittest.TestCase): rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -263,38 +297,40 @@ class TestRMSPropV2(unittest.TestCase): def test_raise_error(self): self.assertRaises(ValueError, paddle.optimizer.RMSProp, None) - self.assertRaises(ValueError, - paddle.optimizer.RMSProp, - learning_rate=0.1, - rho=None) - self.assertRaises(ValueError, - paddle.optimizer.RMSProp, - learning_rate=0.1, - epsilon=None) - self.assertRaises(ValueError, - paddle.optimizer.RMSProp, - learning_rate=0.1, - momentum=None) + self.assertRaises( + ValueError, paddle.optimizer.RMSProp, learning_rate=0.1, rho=None + ) + self.assertRaises( + ValueError, + paddle.optimizer.RMSProp, + learning_rate=0.1, + epsilon=None, + ) + self.assertRaises( + ValueError, + paddle.optimizer.RMSProp, + learning_rate=0.1, + momentum=None, + ) def test_rmsprop_op_invalid_input(self): paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.RMSProp(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.RMSProp( + 0.1, epsilon=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.RMSProp(0.1, - momentum=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.RMSProp( + 0.1, momentum=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.RMSProp(0.1, - rho=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.RMSProp( + 0.1, rho=-1, parameters=linear.parameters() + ) class TestRMSPropV2Group(TestRMSPropV2): - def test_rmsprop_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -302,17 +338,14 @@ class TestRMSPropV2Group(TestRMSPropV2): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.RMSProp(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001 - }], - weight_decay=0.01) + adam = paddle.optimizer.RMSProp( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + {'params': linear_2.parameters(), 'weight_decay': 0.001}, + ], + weight_decay=0.01, + ) out = linear_1(a) out = linear_2(out) out.backward() diff --git a/python/paddle/fluid/tests/unittests/test_rnn_cell_api.py b/python/paddle/fluid/tests/unittests/test_rnn_cell_api.py index 5cf1a79489e..1d04b431053 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_cell_api.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_cell_api.py @@ -33,62 +33,68 @@ import numpy as np class TestLSTMCellError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size, input_size, hidden_size = 4, 16, 16 - inputs = fluid.data(name='inputs', - shape=[None, input_size], - dtype='float32') - pre_hidden = fluid.data(name='pre_hidden', - shape=[None, hidden_size], - dtype='float32') - pre_cell = fluid.data(name='pre_cell', - shape=[None, hidden_size], - dtype='float32') + inputs = fluid.data( + name='inputs', shape=[None, input_size], dtype='float32' + ) + pre_hidden = fluid.data( + name='pre_hidden', shape=[None, hidden_size], dtype='float32' + ) + pre_cell = fluid.data( + name='pre_cell', shape=[None, hidden_size], dtype='float32' + ) cell = LSTMCell(hidden_size) def test_input_Variable(): - np_input = np.random.random( - (batch_size, input_size)).astype("float32") + np_input = np.random.random((batch_size, input_size)).astype( + "float32" + ) cell(np_input, [pre_hidden, pre_cell]) self.assertRaises(TypeError, test_input_Variable) def test_pre_hidden_Variable(): np_pre_hidden = np.random.random( - (batch_size, hidden_size)).astype("float32") + (batch_size, hidden_size) + ).astype("float32") cell(inputs, [np_pre_hidden, pre_cell]) self.assertRaises(TypeError, test_pre_hidden_Variable) def test_pre_cell_Variable(): - np_pre_cell = np.random.random( - (batch_size, input_size)).astype("float32") + np_pre_cell = np.random.random((batch_size, input_size)).astype( + "float32" + ) cell(inputs, [pre_hidden, np_pre_cell]) self.assertRaises(TypeError, test_pre_cell_Variable) def test_input_type(): - error_inputs = fluid.data(name='error_inputs', - shape=[None, input_size], - dtype='int32') + error_inputs = fluid.data( + name='error_inputs', shape=[None, input_size], dtype='int32' + ) cell(error_inputs, [pre_hidden, pre_cell]) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[None, hidden_size], - dtype='int32') + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[None, hidden_size], + dtype='int32', + ) cell(inputs, [error_pre_hidden, pre_cell]) self.assertRaises(TypeError, test_pre_hidden_type) def test_pre_cell_type(): - error_pre_cell = fluid.data(name='error_pre_cell', - shape=[None, hidden_size], - dtype='int32') + error_pre_cell = fluid.data( + name='error_pre_cell', + shape=[None, hidden_size], + dtype='int32', + ) cell(inputs, [pre_hidden, error_pre_cell]) self.assertRaises(TypeError, test_pre_cell_type) @@ -101,29 +107,35 @@ class TestLSTMCellError(unittest.TestCase): class TestLSTMCell(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.input_size = 16 self.hidden_size = 16 def test_run(self): - inputs = fluid.data(name='inputs', - shape=[None, self.input_size], - dtype='float32') - pre_hidden = fluid.data(name='pre_hidden', - shape=[None, self.hidden_size], - dtype='float32') - pre_cell = fluid.data(name='pre_cell', - shape=[None, self.hidden_size], - dtype='float32') + inputs = fluid.data( + name='inputs', shape=[None, self.input_size], dtype='float32' + ) + pre_hidden = fluid.data( + name='pre_hidden', shape=[None, self.hidden_size], dtype='float32' + ) + pre_cell = fluid.data( + name='pre_cell', shape=[None, self.hidden_size], dtype='float32' + ) cell = LSTMCell(self.hidden_size) lstm_hidden_new, lstm_states_new = cell(inputs, [pre_hidden, pre_cell]) lstm_unit = contrib.layers.rnn_impl.BasicLSTMUnit( - "basicLSTM", self.hidden_size, None, None, None, None, 1.0, - "float32") + "basicLSTM", + self.hidden_size, + None, + None, + None, + None, + 1.0, + "float32", + ) lstm_hidden, lstm_cell = lstm_unit(inputs, pre_hidden, pre_cell) if core.is_compiled_with_cuda(): @@ -134,76 +146,91 @@ class TestLSTMCell(unittest.TestCase): exe.run(framework.default_startup_program()) inputs_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_cell_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') - param_names = [[ - "LSTMCell/BasicLSTMUnit_0.w_0", "basicLSTM/BasicLSTMUnit_0.w_0" - ], ["LSTMCell/BasicLSTMUnit_0.b_0", "basicLSTM/BasicLSTMUnit_0.b_0"]] + param_names = [ + ["LSTMCell/BasicLSTMUnit_0.w_0", "basicLSTM/BasicLSTMUnit_0.w_0"], + ["LSTMCell/BasicLSTMUnit_0.b_0", "basicLSTM/BasicLSTMUnit_0.b_0"], + ] for names in param_names: - param = np.array(fluid.global_scope().find_var( - names[0]).get_tensor()) - param = np.random.uniform(-0.1, 0.1, - size=param.shape).astype('float32') + param = np.array( + fluid.global_scope().find_var(names[0]).get_tensor() + ) + param = np.random.uniform(-0.1, 0.1, size=param.shape).astype( + 'float32' + ) fluid.global_scope().find_var(names[0]).get_tensor().set( - param, place) + param, place + ) fluid.global_scope().find_var(names[1]).get_tensor().set( - param, place) - - out = exe.run(feed={ - 'inputs': inputs_np, - 'pre_hidden': pre_hidden_np, - 'pre_cell': pre_cell_np - }, - fetch_list=[lstm_hidden_new, lstm_hidden]) + param, place + ) + + out = exe.run( + feed={ + 'inputs': inputs_np, + 'pre_hidden': pre_hidden_np, + 'pre_cell': pre_cell_np, + }, + fetch_list=[lstm_hidden_new, lstm_hidden], + ) np.testing.assert_allclose(out[0], out[1], rtol=0.0001, atol=0) class TestGRUCellError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size, input_size, hidden_size = 4, 16, 16 - inputs = fluid.data(name='inputs', - shape=[None, input_size], - dtype='float32') - pre_hidden = layers.data(name='pre_hidden', - shape=[None, hidden_size], - append_batch_size=False, - dtype='float32') + inputs = fluid.data( + name='inputs', shape=[None, input_size], dtype='float32' + ) + pre_hidden = layers.data( + name='pre_hidden', + shape=[None, hidden_size], + append_batch_size=False, + dtype='float32', + ) cell = GRUCell(hidden_size) def test_input_Variable(): - np_input = np.random.random( - (batch_size, input_size)).astype("float32") + np_input = np.random.random((batch_size, input_size)).astype( + "float32" + ) cell(np_input, pre_hidden) self.assertRaises(TypeError, test_input_Variable) def test_pre_hidden_Variable(): np_pre_hidden = np.random.random( - (batch_size, hidden_size)).astype("float32") + (batch_size, hidden_size) + ).astype("float32") cell(inputs, np_pre_hidden) self.assertRaises(TypeError, test_pre_hidden_Variable) def test_input_type(): - error_inputs = fluid.data(name='error_inputs', - shape=[None, input_size], - dtype='int32') + error_inputs = fluid.data( + name='error_inputs', shape=[None, input_size], dtype='int32' + ) cell(error_inputs, pre_hidden) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[None, hidden_size], - dtype='int32') + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[None, hidden_size], + dtype='int32', + ) cell(inputs, error_pre_hidden) self.assertRaises(TypeError, test_pre_hidden_type) @@ -216,28 +243,28 @@ class TestGRUCellError(unittest.TestCase): class TestGRUCell(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.input_size = 16 self.hidden_size = 16 def test_run(self): - inputs = fluid.data(name='inputs', - shape=[None, self.input_size], - dtype='float32') - pre_hidden = layers.data(name='pre_hidden', - shape=[None, self.hidden_size], - append_batch_size=False, - dtype='float32') + inputs = fluid.data( + name='inputs', shape=[None, self.input_size], dtype='float32' + ) + pre_hidden = layers.data( + name='pre_hidden', + shape=[None, self.hidden_size], + append_batch_size=False, + dtype='float32', + ) cell = GRUCell(self.hidden_size) gru_hidden_new, _ = cell(inputs, pre_hidden) - gru_unit = contrib.layers.rnn_impl.BasicGRUUnit("basicGRU", - self.hidden_size, None, - None, None, None, - "float32") + gru_unit = contrib.layers.rnn_impl.BasicGRUUnit( + "basicGRU", self.hidden_size, None, None, None, None, "float32" + ) gru_hidden = gru_unit(inputs, pre_hidden) if core.is_compiled_with_cuda(): @@ -248,120 +275,143 @@ class TestGRUCell(unittest.TestCase): exe.run(framework.default_startup_program()) inputs_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') param_names = [ ["GRUCell/BasicGRUUnit_0.w_0", "basicGRU/BasicGRUUnit_0.w_0"], ["GRUCell/BasicGRUUnit_0.w_1", "basicGRU/BasicGRUUnit_0.w_1"], ["GRUCell/BasicGRUUnit_0.b_0", "basicGRU/BasicGRUUnit_0.b_0"], - ["GRUCell/BasicGRUUnit_0.b_1", "basicGRU/BasicGRUUnit_0.b_1"] + ["GRUCell/BasicGRUUnit_0.b_1", "basicGRU/BasicGRUUnit_0.b_1"], ] for names in param_names: - param = np.array(fluid.global_scope().find_var( - names[0]).get_tensor()) - param = np.random.uniform(-0.1, 0.1, - size=param.shape).astype('float32') + param = np.array( + fluid.global_scope().find_var(names[0]).get_tensor() + ) + param = np.random.uniform(-0.1, 0.1, size=param.shape).astype( + 'float32' + ) fluid.global_scope().find_var(names[0]).get_tensor().set( - param, place) + param, place + ) fluid.global_scope().find_var(names[1]).get_tensor().set( - param, place) + param, place + ) - out = exe.run(feed={ - 'inputs': inputs_np, - 'pre_hidden': pre_hidden_np - }, - fetch_list=[gru_hidden_new, gru_hidden]) + out = exe.run( + feed={'inputs': inputs_np, 'pre_hidden': pre_hidden_np}, + fetch_list=[gru_hidden_new, gru_hidden], + ) np.testing.assert_allclose(out[0], out[1], rtol=0.0001, atol=0) class TestRnnError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size = 4 input_size = 16 hidden_size = 16 seq_len = 4 - inputs = fluid.data(name='inputs', - shape=[None, input_size], - dtype='float32') - pre_hidden = layers.data(name='pre_hidden', - shape=[None, hidden_size], - append_batch_size=False, - dtype='float32') - inputs_basic_lstm = fluid.data(name='inputs_basic_lstm', - shape=[None, None, input_size], - dtype='float32') - sequence_length = fluid.data(name="sequence_length", - shape=[None], - dtype='int64') - - inputs_dynamic_rnn = layers.transpose(inputs_basic_lstm, - perm=[1, 0, 2]) + inputs = fluid.data( + name='inputs', shape=[None, input_size], dtype='float32' + ) + pre_hidden = layers.data( + name='pre_hidden', + shape=[None, hidden_size], + append_batch_size=False, + dtype='float32', + ) + inputs_basic_lstm = fluid.data( + name='inputs_basic_lstm', + shape=[None, None, input_size], + dtype='float32', + ) + sequence_length = fluid.data( + name="sequence_length", shape=[None], dtype='int64' + ) + + inputs_dynamic_rnn = layers.transpose( + inputs_basic_lstm, perm=[1, 0, 2] + ) cell = LSTMCell(hidden_size, name="LSTMCell_for_rnn") np_inputs_dynamic_rnn = np.random.random( - (seq_len, batch_size, input_size)).astype("float32") + (seq_len, batch_size, input_size) + ).astype("float32") def test_input_Variable(): - dynamic_rnn(cell=cell, - inputs=np_inputs_dynamic_rnn, - sequence_length=sequence_length, - is_reverse=False) + dynamic_rnn( + cell=cell, + inputs=np_inputs_dynamic_rnn, + sequence_length=sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_input_Variable) def test_input_list(): - dynamic_rnn(cell=cell, - inputs=[np_inputs_dynamic_rnn], - sequence_length=sequence_length, - is_reverse=False) + dynamic_rnn( + cell=cell, + inputs=[np_inputs_dynamic_rnn], + sequence_length=sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_input_list) def test_initial_states_type(): cell = GRUCell(hidden_size, name="GRUCell_for_rnn") error_initial_states = np.random.random( - (batch_size, hidden_size)).astype("float32") - dynamic_rnn(cell=cell, - inputs=inputs_dynamic_rnn, - initial_states=error_initial_states, - sequence_length=sequence_length, - is_reverse=False) + (batch_size, hidden_size) + ).astype("float32") + dynamic_rnn( + cell=cell, + inputs=inputs_dynamic_rnn, + initial_states=error_initial_states, + sequence_length=sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_initial_states_type) def test_initial_states_list(): error_initial_states = [ - np.random.random( - (batch_size, hidden_size)).astype("float32"), - np.random.random( - (batch_size, hidden_size)).astype("float32") + np.random.random((batch_size, hidden_size)).astype( + "float32" + ), + np.random.random((batch_size, hidden_size)).astype( + "float32" + ), ] - dynamic_rnn(cell=cell, - inputs=inputs_dynamic_rnn, - initial_states=error_initial_states, - sequence_length=sequence_length, - is_reverse=False) + dynamic_rnn( + cell=cell, + inputs=inputs_dynamic_rnn, + initial_states=error_initial_states, + sequence_length=sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_initial_states_type) def test_sequence_length_type(): - np_sequence_length = np.random.random( - (batch_size)).astype("float32") - dynamic_rnn(cell=cell, - inputs=inputs_dynamic_rnn, - sequence_length=np_sequence_length, - is_reverse=False) + np_sequence_length = np.random.random((batch_size)).astype( + "float32" + ) + dynamic_rnn( + cell=cell, + inputs=inputs_dynamic_rnn, + sequence_length=np_sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_sequence_length_type) class TestRnn(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.input_size = 16 @@ -369,23 +419,36 @@ class TestRnn(unittest.TestCase): self.seq_len = 4 def test_run(self): - inputs_basic_lstm = fluid.data(name='inputs_basic_lstm', - shape=[None, None, self.input_size], - dtype='float32') - sequence_length = fluid.data(name="sequence_length", - shape=[None], - dtype='int64') + inputs_basic_lstm = fluid.data( + name='inputs_basic_lstm', + shape=[None, None, self.input_size], + dtype='float32', + ) + sequence_length = fluid.data( + name="sequence_length", shape=[None], dtype='int64' + ) inputs_dynamic_rnn = layers.transpose(inputs_basic_lstm, perm=[1, 0, 2]) cell = LSTMCell(self.hidden_size, name="LSTMCell_for_rnn") - output, final_state = dynamic_rnn(cell=cell, - inputs=inputs_dynamic_rnn, - sequence_length=sequence_length, - is_reverse=False) + output, final_state = dynamic_rnn( + cell=cell, + inputs=inputs_dynamic_rnn, + sequence_length=sequence_length, + is_reverse=False, + ) output_new = layers.transpose(output, perm=[1, 0, 2]) - rnn_out, last_hidden, last_cell = basic_lstm(inputs_basic_lstm, None, None, self.hidden_size, num_layers=1, \ - batch_first = False, bidirectional=False, sequence_length=sequence_length, forget_bias = 1.0) + rnn_out, last_hidden, last_cell = basic_lstm( + inputs_basic_lstm, + None, + None, + self.hidden_size, + num_layers=1, + batch_first=False, + bidirectional=False, + sequence_length=sequence_length, + forget_bias=1.0, + ) if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) @@ -395,45 +458,57 @@ class TestRnn(unittest.TestCase): exe.run(framework.default_startup_program()) inputs_basic_lstm_np = np.random.uniform( - -0.1, 0.1, - (self.seq_len, self.batch_size, self.input_size)).astype('float32') - sequence_length_np = np.ones(self.batch_size, - dtype='int64') * self.seq_len + -0.1, 0.1, (self.seq_len, self.batch_size, self.input_size) + ).astype('float32') + sequence_length_np = ( + np.ones(self.batch_size, dtype='int64') * self.seq_len + ) inputs_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_cell_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') - param_names = [[ - "LSTMCell_for_rnn/BasicLSTMUnit_0.w_0", - "basic_lstm_layers_0/BasicLSTMUnit_0.w_0" - ], - [ - "LSTMCell_for_rnn/BasicLSTMUnit_0.b_0", - "basic_lstm_layers_0/BasicLSTMUnit_0.b_0" - ]] + param_names = [ + [ + "LSTMCell_for_rnn/BasicLSTMUnit_0.w_0", + "basic_lstm_layers_0/BasicLSTMUnit_0.w_0", + ], + [ + "LSTMCell_for_rnn/BasicLSTMUnit_0.b_0", + "basic_lstm_layers_0/BasicLSTMUnit_0.b_0", + ], + ] for names in param_names: - param = np.array(fluid.global_scope().find_var( - names[0]).get_tensor()) - param = np.random.uniform(-0.1, 0.1, - size=param.shape).astype('float32') + param = np.array( + fluid.global_scope().find_var(names[0]).get_tensor() + ) + param = np.random.uniform(-0.1, 0.1, size=param.shape).astype( + 'float32' + ) fluid.global_scope().find_var(names[0]).get_tensor().set( - param, place) + param, place + ) fluid.global_scope().find_var(names[1]).get_tensor().set( - param, place) - - out = exe.run(feed={ - 'inputs_basic_lstm': inputs_basic_lstm_np, - 'sequence_length': sequence_length_np, - 'inputs': inputs_np, - 'pre_hidden': pre_hidden_np, - 'pre_cell': pre_cell_np - }, - fetch_list=[output_new, rnn_out]) + param, place + ) + + out = exe.run( + feed={ + 'inputs_basic_lstm': inputs_basic_lstm_np, + 'sequence_length': sequence_length_np, + 'inputs': inputs_np, + 'pre_hidden': pre_hidden_np, + 'pre_cell': pre_cell_np, + }, + fetch_list=[output_new, rnn_out], + ) np.testing.assert_allclose(out[0], out[1], rtol=0.0001) @@ -462,7 +537,7 @@ class EncoderCell(RNNCell): self, num_layers, hidden_size, - dropout_prob=0., + dropout_prob=0.0, init_scale=0.1, ): self.num_layers = num_layers @@ -477,10 +552,14 @@ class EncoderCell(RNNCell): new_states = [] for i in range(self.num_layers): out, new_state = self.lstm_cells[i](step_input, states[i]) - step_input = layers.dropout( - out, - self.dropout_prob, - ) if self.dropout_prob else out + step_input = ( + layers.dropout( + out, + self.dropout_prob, + ) + if self.dropout_prob + else out + ) new_states.append(new_state) return step_input, new_states @@ -492,7 +571,7 @@ class EncoderCell(RNNCell): class DecoderCell(RNNCell): """Decoder Cell""" - def __init__(self, num_layers, hidden_size, dropout_prob=0.): + def __init__(self, num_layers, hidden_size, dropout_prob=0.0): self.num_layers = num_layers self.hidden_size = hidden_size self.dropout_prob = dropout_prob @@ -504,26 +583,31 @@ class DecoderCell(RNNCell): new_lstm_states = [] for i in range(self.num_layers): out, new_lstm_state = self.lstm_cells[i](step_input, states[i]) - step_input = layers.dropout( - out, - self.dropout_prob, - ) if self.dropout_prob else out + step_input = ( + layers.dropout( + out, + self.dropout_prob, + ) + if self.dropout_prob + else out + ) new_lstm_states.append(new_lstm_state) return step_input, new_lstm_states -def def_seq2seq_model(num_layers, hidden_size, dropout_prob, src_vocab_size, - trg_vocab_size): +def def_seq2seq_model( + num_layers, hidden_size, dropout_prob, src_vocab_size, trg_vocab_size +): "vanilla seq2seq model" # data source = fluid.data(name="src", shape=[None, None], dtype="int64") - source_length = fluid.data(name="src_sequence_length", - shape=[None], - dtype="int64") + source_length = fluid.data( + name="src_sequence_length", shape=[None], dtype="int64" + ) target = fluid.data(name="trg", shape=[None, None], dtype="int64") - target_length = fluid.data(name="trg_sequence_length", - shape=[None], - dtype="int64") + target_length = fluid.data( + name="trg_sequence_length", shape=[None], dtype="int64" + ) label = fluid.data(name="label", shape=[None, None, 1], dtype="int64") # embedding @@ -532,29 +616,31 @@ def def_seq2seq_model(num_layers, hidden_size, dropout_prob, src_vocab_size, # encoder enc_cell = EncoderCell(num_layers, hidden_size, dropout_prob) - enc_output, enc_final_state = dynamic_rnn(cell=enc_cell, - inputs=src_emb, - sequence_length=source_length) + enc_output, enc_final_state = dynamic_rnn( + cell=enc_cell, inputs=src_emb, sequence_length=source_length + ) # decoder dec_cell = DecoderCell(num_layers, hidden_size, dropout_prob) - dec_output, dec_final_state = dynamic_rnn(cell=dec_cell, - inputs=tar_emb, - initial_states=enc_final_state) - logits = layers.fc(dec_output, - size=trg_vocab_size, - num_flatten_dims=len(dec_output.shape) - 1, - bias_attr=False) + dec_output, dec_final_state = dynamic_rnn( + cell=dec_cell, inputs=tar_emb, initial_states=enc_final_state + ) + logits = layers.fc( + dec_output, + size=trg_vocab_size, + num_flatten_dims=len(dec_output.shape) - 1, + bias_attr=False, + ) # loss - loss = layers.softmax_with_cross_entropy(logits=logits, - label=label, - soft_label=False) + loss = layers.softmax_with_cross_entropy( + logits=logits, label=label, soft_label=False + ) loss = layers.unsqueeze(loss, axes=[2]) max_tar_seq_len = layers.shape(target)[1] - tar_mask = layers.sequence_mask(target_length, - maxlen=max_tar_seq_len, - dtype="float32") + tar_mask = layers.sequence_mask( + target_length, maxlen=max_tar_seq_len, dtype="float32" + ) loss = loss * tar_mask loss = layers.reduce_mean(loss, dim=[0]) loss = layers.reduce_sum(loss) @@ -577,7 +663,7 @@ class TestSeq2SeqModel(unittest.TestCase): "hidden_size": 128, "dropout_prob": 0.1, "src_vocab_size": 100, - "trg_vocab_size": 100 + "trg_vocab_size": 100, } self.iter_num = iter_num = 2 @@ -585,28 +671,34 @@ class TestSeq2SeqModel(unittest.TestCase): src_seq_len = 10 trg_seq_len = 12 self.data = { - "src": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, src_seq_len)).astype("int64"), - "src_sequence_length": - np.random.randint(1, src_seq_len, - (iter_num * batch_size, )).astype("int64"), - "trg": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, trg_seq_len)).astype("int64"), - "trg_sequence_length": - np.random.randint(1, trg_seq_len, - (iter_num * batch_size, )).astype("int64"), - "label": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, trg_seq_len, 1)).astype("int64"), + "src": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, src_seq_len), + ).astype("int64"), + "src_sequence_length": np.random.randint( + 1, src_seq_len, (iter_num * batch_size,) + ).astype("int64"), + "trg": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, trg_seq_len), + ).astype("int64"), + "trg_sequence_length": np.random.randint( + 1, trg_seq_len, (iter_num * batch_size,) + ).astype("int64"), + "label": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, trg_seq_len, 1), + ).astype("int64"), } - place = core.CUDAPlace( - 0) if core.is_compiled_with_cuda() else core.CPUPlace() + place = ( + core.CUDAPlace(0) + if core.is_compiled_with_cuda() + else core.CPUPlace() + ) self.exe = Executor(place) def test_seq2seq_model(self): @@ -616,29 +708,38 @@ class TestSeq2SeqModel(unittest.TestCase): cost = def_seq2seq_model(**self.model_hparams) self.exe.run(startup_program) for iter_idx in range(self.iter_num): - cost_val = self.exe.run(feed={ - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * - self.batch_size:(iter_idx + - 1) * - self.batch_size], - "trg": - self.data["trg"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "trg_sequence_length": - self.data["trg_sequence_length"][iter_idx * - self.batch_size:(iter_idx + - 1) * - self.batch_size], - "label": - self.data["label"][iter_idx * - self.batch_size:(iter_idx + 1) * - self.batch_size] - }, - fetch_list=[cost])[0] + cost_val = self.exe.run( + feed={ + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + "trg": self.data["trg"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "trg_sequence_length": self.data["trg_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + "label": self.data["label"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + }, + fetch_list=[cost], + )[0] print("iter_idx: %d, cost: %f" % (iter_idx, cost_val)) diff --git a/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py b/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py index a6f7b6ea8c2..00bf9735fa5 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py @@ -34,8 +34,7 @@ paddle.enable_static() class EncoderCell(layers.RNNCell): - - def __init__(self, num_layers, hidden_size, dropout_prob=0.): + def __init__(self, num_layers, hidden_size, dropout_prob=0.0): self.num_layers = num_layers self.hidden_size = hidden_size self.dropout_prob = dropout_prob @@ -47,8 +46,11 @@ class EncoderCell(layers.RNNCell): new_states = [] for i in range(self.num_layers): out, new_state = self.lstm_cells[i](step_input, states[i]) - step_input = layers.dropout( - out, self.dropout_prob) if self.dropout_prob > 0 else out + step_input = ( + layers.dropout(out, self.dropout_prob) + if self.dropout_prob > 0 + else out + ) new_states.append(new_state) return step_input, new_states @@ -58,8 +60,7 @@ class EncoderCell(layers.RNNCell): class DecoderCell(layers.RNNCell): - - def __init__(self, num_layers, hidden_size, dropout_prob=0.): + def __init__(self, num_layers, hidden_size, dropout_prob=0.0): self.num_layers = num_layers self.hidden_size = hidden_size self.dropout_prob = dropout_prob @@ -68,42 +69,44 @@ class DecoderCell(layers.RNNCell): ] def attention(self, hidden, encoder_output, encoder_padding_mask): - query = layers.fc(hidden, - size=encoder_output.shape[-1], - bias_attr=False) - attn_scores = layers.matmul(layers.unsqueeze(query, [1]), - encoder_output, - transpose_y=True) + query = layers.fc( + hidden, size=encoder_output.shape[-1], bias_attr=False + ) + attn_scores = layers.matmul( + layers.unsqueeze(query, [1]), encoder_output, transpose_y=True + ) if encoder_padding_mask is not None: - attn_scores = layers.elementwise_add(attn_scores, - encoder_padding_mask) + attn_scores = layers.elementwise_add( + attn_scores, encoder_padding_mask + ) attn_scores = layers.softmax(attn_scores) - attn_out = layers.squeeze(layers.matmul(attn_scores, encoder_output), - [1]) + attn_out = layers.squeeze( + layers.matmul(attn_scores, encoder_output), [1] + ) attn_out = layers.concat([attn_out, hidden], 1) attn_out = layers.fc(attn_out, size=self.hidden_size, bias_attr=False) return attn_out - def call(self, - step_input, - states, - encoder_output, - encoder_padding_mask=None): + def call( + self, step_input, states, encoder_output, encoder_padding_mask=None + ): lstm_states, input_feed = states new_lstm_states = [] step_input = layers.concat([step_input, input_feed], 1) for i in range(self.num_layers): out, new_lstm_state = self.lstm_cells[i](step_input, lstm_states[i]) - step_input = layers.dropout( - out, self.dropout_prob) if self.dropout_prob > 0 else out + step_input = ( + layers.dropout(out, self.dropout_prob) + if self.dropout_prob > 0 + else out + ) new_lstm_states.append(new_lstm_state) out = self.attention(step_input, encoder_output, encoder_padding_mask) return out, [new_lstm_states, out] class Encoder(object): - - def __init__(self, num_layers, hidden_size, dropout_prob=0.): + def __init__(self, num_layers, hidden_size, dropout_prob=0.0): self.encoder_cell = EncoderCell(num_layers, hidden_size, dropout_prob) def __call__(self, src_emb, src_sequence_length): @@ -111,25 +114,35 @@ class Encoder(object): cell=self.encoder_cell, inputs=src_emb, sequence_length=src_sequence_length, - is_reverse=False) + is_reverse=False, + ) return encoder_output, encoder_final_state class Decoder(object): - - def __init__(self, - num_layers, - hidden_size, - dropout_prob, - decoding_strategy="infer_sample", - max_decoding_length=20): + def __init__( + self, + num_layers, + hidden_size, + dropout_prob, + decoding_strategy="infer_sample", + max_decoding_length=20, + ): self.decoder_cell = DecoderCell(num_layers, hidden_size, dropout_prob) self.decoding_strategy = decoding_strategy - self.max_decoding_length = None if ( - self.decoding_strategy == "train_greedy") else max_decoding_length - - def __call__(self, decoder_initial_states, encoder_output, - encoder_padding_mask, **kwargs): + self.max_decoding_length = ( + None + if (self.decoding_strategy == "train_greedy") + else max_decoding_length + ) + + def __call__( + self, + decoder_initial_states, + encoder_output, + encoder_padding_mask, + **kwargs + ): output_layer = kwargs.pop("output_layer", None) if self.decoding_strategy == "train_greedy": # for teach-forcing MLE pre-training @@ -141,113 +154,150 @@ class Decoder(object): if self.decoding_strategy == "beam_search": beam_size = kwargs.get("beam_size", 4) - encoder_output = layers.BeamSearchDecoder.tile_beam_merge_with_batch( - encoder_output, beam_size) - encoder_padding_mask = layers.BeamSearchDecoder.tile_beam_merge_with_batch( - encoder_padding_mask, beam_size) - decoder = layers.BeamSearchDecoder(cell=self.decoder_cell, - output_fn=output_layer, - **kwargs) + encoder_output = ( + layers.BeamSearchDecoder.tile_beam_merge_with_batch( + encoder_output, beam_size + ) + ) + encoder_padding_mask = ( + layers.BeamSearchDecoder.tile_beam_merge_with_batch( + encoder_padding_mask, beam_size + ) + ) + decoder = layers.BeamSearchDecoder( + cell=self.decoder_cell, output_fn=output_layer, **kwargs + ) else: - decoder = layers.BasicDecoder(self.decoder_cell, - helper, - output_fn=output_layer) - - (decoder_output, decoder_final_state, - dec_seq_lengths) = layers.dynamic_decode( - decoder, - inits=decoder_initial_states, - max_step_num=self.max_decoding_length, - encoder_output=encoder_output, - encoder_padding_mask=encoder_padding_mask, - impute_finished=False # for test coverage - if self.decoding_strategy == "beam_search" else True, - is_test=True if self.decoding_strategy == "beam_search" else False, - return_length=True) + decoder = layers.BasicDecoder( + self.decoder_cell, helper, output_fn=output_layer + ) + + ( + decoder_output, + decoder_final_state, + dec_seq_lengths, + ) = layers.dynamic_decode( + decoder, + inits=decoder_initial_states, + max_step_num=self.max_decoding_length, + encoder_output=encoder_output, + encoder_padding_mask=encoder_padding_mask, + impute_finished=False # for test coverage + if self.decoding_strategy == "beam_search" + else True, + is_test=True if self.decoding_strategy == "beam_search" else False, + return_length=True, + ) return decoder_output, decoder_final_state, dec_seq_lengths class Seq2SeqModel(object): """Seq2Seq model: RNN encoder-decoder with attention""" - def __init__(self, - num_layers, - hidden_size, - dropout_prob, - src_vocab_size, - trg_vocab_size, - start_token, - end_token, - decoding_strategy="infer_sample", - max_decoding_length=20, - beam_size=4): + def __init__( + self, + num_layers, + hidden_size, + dropout_prob, + src_vocab_size, + trg_vocab_size, + start_token, + end_token, + decoding_strategy="infer_sample", + max_decoding_length=20, + beam_size=4, + ): self.start_token, self.end_token = start_token, end_token - self.max_decoding_length, self.beam_size = max_decoding_length, beam_size + self.max_decoding_length, self.beam_size = ( + max_decoding_length, + beam_size, + ) self.src_embeder = paddle.nn.Embedding( src_vocab_size, hidden_size, - weight_attr=fluid.ParamAttr(name="source_embedding")) + weight_attr=fluid.ParamAttr(name="source_embedding"), + ) self.trg_embeder = paddle.nn.Embedding( trg_vocab_size, hidden_size, - weight_attr=fluid.ParamAttr(name="target_embedding")) + weight_attr=fluid.ParamAttr(name="target_embedding"), + ) self.encoder = Encoder(num_layers, hidden_size, dropout_prob) - self.decoder = Decoder(num_layers, hidden_size, dropout_prob, - decoding_strategy, max_decoding_length) - self.output_layer = lambda x: layers.fc(x, - size=trg_vocab_size, - num_flatten_dims=len(x.shape) - - 1, - param_attr=fluid.ParamAttr(), - bias_attr=False) + self.decoder = Decoder( + num_layers, + hidden_size, + dropout_prob, + decoding_strategy, + max_decoding_length, + ) + self.output_layer = lambda x: layers.fc( + x, + size=trg_vocab_size, + num_flatten_dims=len(x.shape) - 1, + param_attr=fluid.ParamAttr(), + bias_attr=False, + ) def __call__(self, src, src_length, trg=None, trg_length=None): # encoder encoder_output, encoder_final_state = self.encoder( - self.src_embeder(src), src_length) + self.src_embeder(src), src_length + ) decoder_initial_states = [ encoder_final_state, self.decoder.decoder_cell.get_initial_states( - batch_ref=encoder_output, shape=[encoder_output.shape[-1]]) + batch_ref=encoder_output, shape=[encoder_output.shape[-1]] + ), ] - src_mask = layers.sequence_mask(src_length, - maxlen=layers.shape(src)[1], - dtype="float32") + src_mask = layers.sequence_mask( + src_length, maxlen=layers.shape(src)[1], dtype="float32" + ) encoder_padding_mask = (src_mask - 1.0) * 1e9 encoder_padding_mask = layers.unsqueeze(encoder_padding_mask, [1]) # decoder - decoder_kwargs = { - "inputs": self.trg_embeder(trg), - "sequence_length": trg_length, - } if self.decoder.decoding_strategy == "train_greedy" else ( + decoder_kwargs = ( { - "embedding_fn": self.trg_embeder, - "beam_size": self.beam_size, - "start_token": self.start_token, - "end_token": self.end_token - } if self.decoder.decoding_strategy == "beam_search" else { - "embedding_fn": - self.trg_embeder, - "start_tokens": - layers.fill_constant_batch_size_like(input=encoder_output, - shape=[-1], - dtype=src.dtype, - value=self.start_token), - "end_token": - self.end_token - }) + "inputs": self.trg_embeder(trg), + "sequence_length": trg_length, + } + if self.decoder.decoding_strategy == "train_greedy" + else ( + { + "embedding_fn": self.trg_embeder, + "beam_size": self.beam_size, + "start_token": self.start_token, + "end_token": self.end_token, + } + if self.decoder.decoding_strategy == "beam_search" + else { + "embedding_fn": self.trg_embeder, + "start_tokens": layers.fill_constant_batch_size_like( + input=encoder_output, + shape=[-1], + dtype=src.dtype, + value=self.start_token, + ), + "end_token": self.end_token, + } + ) + ) decoder_kwargs["output_layer"] = self.output_layer - (decoder_output, decoder_final_state, - dec_seq_lengths) = self.decoder(decoder_initial_states, encoder_output, - encoder_padding_mask, **decoder_kwargs) + (decoder_output, decoder_final_state, dec_seq_lengths) = self.decoder( + decoder_initial_states, + encoder_output, + encoder_padding_mask, + **decoder_kwargs + ) if self.decoder.decoding_strategy == "beam_search": # for inference return decoder_output - logits, samples, sample_length = (decoder_output.cell_outputs, - decoder_output.sample_ids, - dec_seq_lengths) + logits, samples, sample_length = ( + decoder_output.cell_outputs, + decoder_output.sample_ids, + dec_seq_lengths, + ) probs = layers.softmax(logits) return probs, samples, sample_length @@ -262,13 +312,16 @@ class PolicyGradient(object): """ update policy model self.model with policy gradient algorithm """ - self.reward = fluid.layers.py_func(func=reward_func, - x=[action, length], - out=reward) + self.reward = fluid.layers.py_func( + func=reward_func, x=[action, length], out=reward + ) neg_log_prob = layers.cross_entropy(act_prob, action) cost = neg_log_prob * reward - cost = (layers.reduce_sum(cost) / layers.reduce_sum(length) - ) if length is not None else layers.reduce_mean(cost) + cost = ( + (layers.reduce_sum(cost) / layers.reduce_sum(length)) + if length is not None + else layers.reduce_mean(cost) + ) optimizer = fluid.optimizer.Adam(self.lr) optimizer.minimize(cost) return cost @@ -277,24 +330,26 @@ class PolicyGradient(object): def reward_func(samples, sample_length): """toy reward""" - def discount_reward(reward, sequence_length, discount=1.): + def discount_reward(reward, sequence_length, discount=1.0): return discount_reward_1d(reward, sequence_length, discount) - def discount_reward_1d(reward, sequence_length, discount=1., dtype=None): + def discount_reward_1d(reward, sequence_length, discount=1.0, dtype=None): if sequence_length is None: raise ValueError( - 'sequence_length must not be `None` for 1D reward.') + 'sequence_length must not be `None` for 1D reward.' + ) reward = np.array(reward) sequence_length = np.array(sequence_length) batch_size = reward.shape[0] max_seq_length = np.max(sequence_length) dtype = dtype or reward.dtype - if discount == 1.: + if discount == 1.0: dmat = np.ones([batch_size, max_seq_length], dtype=dtype) else: steps = np.tile(np.arange(max_seq_length), [batch_size, 1]) - mask = np.asarray(steps < (sequence_length - 1)[:, None], - dtype=dtype) + mask = np.asarray( + steps < (sequence_length - 1)[:, None], dtype=dtype + ) # Make each row = [discount, ..., discount, 1, ..., 1] dmat = mask * discount + (1 - mask) dmat = np.cumprod(dmat[:, ::-1], axis=1)[:, ::-1] @@ -329,9 +384,15 @@ def reward_func(samples, sample_length): # repeat punishment to trapped into local minima getting all same words # beam search to get more than one sample may also can avoid this for i in range(reward.shape[0]): - reward[i] += -10 if sample_length[i] > 1 and np.all( - samples[i][:sample_length[i] - 1] == samples[i][0]) else 0 - return discount_reward(reward, sample_length, discount=1.).astype("float32") + reward[i] += ( + -10 + if sample_length[i] > 1 + and np.all(samples[i][: sample_length[i] - 1] == samples[i][0]) + else 0 + ) + return discount_reward(reward, sample_length, discount=1.0).astype( + "float32" + ) class MLE(object): @@ -353,20 +414,23 @@ class MLE(object): class SeqPGAgent(object): - - def __init__(self, - model_cls, - alg_cls=PolicyGradient, - model_hparams={}, - alg_hparams={}, - executor=None, - main_program=None, - startup_program=None, - seed=None): - self.main_program = fluid.Program( - ) if main_program is None else main_program - self.startup_program = fluid.Program( - ) if startup_program is None else startup_program + def __init__( + self, + model_cls, + alg_cls=PolicyGradient, + model_hparams={}, + alg_hparams={}, + executor=None, + main_program=None, + startup_program=None, + seed=None, + ): + self.main_program = ( + fluid.Program() if main_program is None else main_program + ) + self.startup_program = ( + fluid.Program() if startup_program is None else startup_program + ) if seed is not None: self.main_program.random_seed = seed self.startup_program.random_seed = seed @@ -376,51 +440,55 @@ class SeqPGAgent(object): def build_program(self, model_cls, alg_cls, model_hparams, alg_hparams): with fluid.program_guard(self.main_program, self.startup_program): source = fluid.data(name="src", shape=[None, None], dtype="int64") - source_length = fluid.data(name="src_sequence_length", - shape=[None], - dtype="int64") + source_length = fluid.data( + name="src_sequence_length", shape=[None], dtype="int64" + ) # only for teacher-forcing MLE training target = fluid.data(name="trg", shape=[None, None], dtype="int64") - target_length = fluid.data(name="trg_sequence_length", - shape=[None], - dtype="int64") - label = fluid.data(name="label", - shape=[None, None, 1], - dtype="int64") + target_length = fluid.data( + name="trg_sequence_length", shape=[None], dtype="int64" + ) + label = fluid.data( + name="label", shape=[None, None, 1], dtype="int64" + ) self.model = model_cls(**model_hparams) self.alg = alg_cls(**alg_hparams) self.probs, self.samples, self.sample_length = self.model( - source, source_length, target, target_length) + source, source_length, target, target_length + ) self.samples.stop_gradient = True self.reward = fluid.data( name="reward", shape=[None, None], # batch_size, seq_len - dtype=self.probs.dtype) + dtype=self.probs.dtype, + ) self.samples.stop_gradient = False - self.cost = self.alg.learn(self.probs, self.samples, self.reward, - self.sample_length) + self.cost = self.alg.learn( + self.probs, self.samples, self.reward, self.sample_length + ) # to define the same parameters between different programs self.pred_program = self.main_program._prune_with_input( [source.name, source_length.name], - [self.probs, self.samples, self.sample_length]) + [self.probs, self.samples, self.sample_length], + ) def predict(self, feed_dict): samples, sample_length = self.executor.run( self.pred_program, feed=feed_dict, - fetch_list=[self.samples, self.sample_length]) + fetch_list=[self.samples, self.sample_length], + ) return samples, sample_length def learn(self, feed_dict, fetch_list): - results = self.executor.run(self.main_program, - feed=feed_dict, - fetch_list=fetch_list) + results = self.executor.run( + self.main_program, feed=feed_dict, fetch_list=fetch_list + ) return results class TestDynamicDecode(unittest.TestCase): - def setUp(self): np.random.seed(123) self.model_hparams = { @@ -432,7 +500,7 @@ class TestDynamicDecode(unittest.TestCase): "start_token": 0, "end_token": 1, "decoding_strategy": "infer_greedy", - "max_decoding_length": 10 + "max_decoding_length": 10, } self.iter_num = iter_num = 2 @@ -440,121 +508,159 @@ class TestDynamicDecode(unittest.TestCase): src_seq_len = 10 trg_seq_len = 12 self.data = { - "src": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, src_seq_len)).astype("int64"), - "src_sequence_length": - np.random.randint(1, src_seq_len, - (iter_num * batch_size, )).astype("int64"), - "trg": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, trg_seq_len)).astype("int64"), - "trg_sequence_length": - np.random.randint(1, trg_seq_len, - (iter_num * batch_size, )).astype("int64"), - "label": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, trg_seq_len, 1)).astype("int64"), + "src": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, src_seq_len), + ).astype("int64"), + "src_sequence_length": np.random.randint( + 1, src_seq_len, (iter_num * batch_size,) + ).astype("int64"), + "trg": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, trg_seq_len), + ).astype("int64"), + "trg_sequence_length": np.random.randint( + 1, trg_seq_len, (iter_num * batch_size,) + ).astype("int64"), + "label": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, trg_seq_len, 1), + ).astype("int64"), } - place = core.CUDAPlace( - 0) if core.is_compiled_with_cuda() else core.CPUPlace() + place = ( + core.CUDAPlace(0) + if core.is_compiled_with_cuda() + else core.CPUPlace() + ) self.exe = Executor(place) def test_mle_train(self): paddle.enable_static() self.model_hparams["decoding_strategy"] = "train_greedy" - agent = SeqPGAgent(model_cls=Seq2SeqModel, - alg_cls=MLE, - model_hparams=self.model_hparams, - alg_hparams={"lr": 0.001}, - executor=self.exe, - main_program=fluid.Program(), - startup_program=fluid.Program(), - seed=123) + agent = SeqPGAgent( + model_cls=Seq2SeqModel, + alg_cls=MLE, + model_hparams=self.model_hparams, + alg_hparams={"lr": 0.001}, + executor=self.exe, + main_program=fluid.Program(), + startup_program=fluid.Program(), + seed=123, + ) self.exe.run(agent.startup_program) for iter_idx in range(self.iter_num): reward, cost = agent.learn( { - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * self.batch_size: - (iter_idx + 1) * - self.batch_size], - "trg": - self.data["trg"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "trg_sequence_length": - self.data["trg_sequence_length"][iter_idx * self.batch_size: - (iter_idx + 1) * - self.batch_size], - "label": - self.data["label"][iter_idx * - self.batch_size:(iter_idx + 1) * - self.batch_size] + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + "trg": self.data["trg"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "trg_sequence_length": self.data["trg_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + "label": self.data["label"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], }, - fetch_list=[agent.cost, agent.cost]) - print("iter_idx: %d, reward: %f, cost: %f" % - (iter_idx, reward.mean(), cost)) + fetch_list=[agent.cost, agent.cost], + ) + print( + "iter_idx: %d, reward: %f, cost: %f" + % (iter_idx, reward.mean(), cost) + ) def test_greedy_train(self): paddle.enable_static() self.model_hparams["decoding_strategy"] = "infer_greedy" - agent = SeqPGAgent(model_cls=Seq2SeqModel, - alg_cls=PolicyGradient, - model_hparams=self.model_hparams, - alg_hparams={"lr": 0.001}, - executor=self.exe, - main_program=fluid.Program(), - startup_program=fluid.Program(), - seed=123) + agent = SeqPGAgent( + model_cls=Seq2SeqModel, + alg_cls=PolicyGradient, + model_hparams=self.model_hparams, + alg_hparams={"lr": 0.001}, + executor=self.exe, + main_program=fluid.Program(), + startup_program=fluid.Program(), + seed=123, + ) self.exe.run(agent.startup_program) for iter_idx in range(self.iter_num): reward, cost = agent.learn( { - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * self.batch_size: - (iter_idx + 1) * - self.batch_size] + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], }, - fetch_list=[agent.reward, agent.cost]) - print("iter_idx: %d, reward: %f, cost: %f" % - (iter_idx, reward.mean(), cost)) + fetch_list=[agent.reward, agent.cost], + ) + print( + "iter_idx: %d, reward: %f, cost: %f" + % (iter_idx, reward.mean(), cost) + ) def test_sample_train(self): paddle.enable_static() self.model_hparams["decoding_strategy"] = "infer_sample" - agent = SeqPGAgent(model_cls=Seq2SeqModel, - alg_cls=PolicyGradient, - model_hparams=self.model_hparams, - alg_hparams={"lr": 0.001}, - executor=self.exe, - main_program=fluid.Program(), - startup_program=fluid.Program(), - seed=123) + agent = SeqPGAgent( + model_cls=Seq2SeqModel, + alg_cls=PolicyGradient, + model_hparams=self.model_hparams, + alg_hparams={"lr": 0.001}, + executor=self.exe, + main_program=fluid.Program(), + startup_program=fluid.Program(), + seed=123, + ) self.exe.run(agent.startup_program) for iter_idx in range(self.iter_num): reward, cost = agent.learn( { - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * self.batch_size: - (iter_idx + 1) * - self.batch_size] + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], }, - fetch_list=[agent.reward, agent.cost]) - print("iter_idx: %d, reward: %f, cost: %f" % - (iter_idx, reward.mean(), cost)) + fetch_list=[agent.reward, agent.cost], + ) + print( + "iter_idx: %d, reward: %f, cost: %f" + % (iter_idx, reward.mean(), cost) + ) def test_beam_search_infer(self): paddle.set_default_dtype("float32") @@ -564,9 +670,9 @@ class TestDynamicDecode(unittest.TestCase): startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): source = fluid.data(name="src", shape=[None, None], dtype="int64") - source_length = fluid.data(name="src_sequence_length", - shape=[None], - dtype="int64") + source_length = fluid.data( + name="src_sequence_length", shape=[None], dtype="int64" + ) model = Seq2SeqModel(**self.model_hparams) output = model(source, source_length) @@ -575,16 +681,20 @@ class TestDynamicDecode(unittest.TestCase): trans_ids = self.exe.run( program=main_program, feed={ - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * - self.batch_size:(iter_idx + - 1) * - self.batch_size] + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], }, - fetch_list=[output])[0] + fetch_list=[output], + )[0] def func_dynamic_basic_decoder(self): paddle.disable_static() @@ -601,7 +711,6 @@ class TestDynamicDecode(unittest.TestCase): class ModuleApiTest(unittest.TestCase): - @classmethod def setUpClass(cls): cls._np_rand_state = np.random.get_state() @@ -611,10 +720,13 @@ class ModuleApiTest(unittest.TestCase): random.seed(cls._random_seed) cls.model_cls = type( - cls.__name__ + "Model", (Layer, ), { + cls.__name__ + "Model", + (Layer,), + { "__init__": cls.model_init_wrapper(cls.model_init), - "forward": cls.model_forward - }) + "forward": cls.model_forward, + }, + ) @classmethod def tearDownClass(cls): @@ -623,7 +735,6 @@ class ModuleApiTest(unittest.TestCase): @staticmethod def model_init_wrapper(func): - def __impl__(self, *args, **kwargs): Layer.__init__(self) func(self, *args, **kwargs) @@ -633,7 +744,8 @@ class ModuleApiTest(unittest.TestCase): @staticmethod def model_init(model, *args, **kwargs): raise NotImplementedError( - "model_init acts as `Model.__init__`, thus must implement it") + "model_init acts as `Model.__init__`, thus must implement it" + ) @staticmethod def model_forward(model, *args, **kwargs): @@ -642,7 +754,8 @@ class ModuleApiTest(unittest.TestCase): def make_inputs(self): # TODO(guosheng): add default from `self.inputs` raise NotImplementedError( - "model_inputs makes inputs for model, thus must implement it") + "model_inputs makes inputs for model, thus must implement it" + ) def setUp(self): """ @@ -667,8 +780,11 @@ class ModuleApiTest(unittest.TestCase): paddle.framework.random._manual_program_seed(self._random_seed) scope = fluid.core.Scope() with fluid.scope_guard(scope): - layer = self.model_cls(**self.attrs) if isinstance( - self.attrs, dict) else self.model_cls(*self.attrs) + layer = ( + self.model_cls(**self.attrs) + if isinstance(self.attrs, dict) + else self.model_cls(*self.attrs) + ) model = Model(layer, inputs=self.make_inputs()) model.prepare() if self.param_states: @@ -683,10 +799,9 @@ class ModuleApiTest(unittest.TestCase): np.testing.assert_allclose(actual_t, expect_t, rtol=1e-05, atol=0) if expect_output: for actual_t, expect_t in zip(dygraph_output, expect_output): - np.testing.assert_allclose(actual_t, - expect_t, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + actual_t, expect_t, rtol=1e-05, atol=0 + ) def check_output(self): devices = ["CPU", "GPU"] if fluid.is_compiled_with_cuda() else ["CPU"] @@ -696,13 +811,12 @@ class ModuleApiTest(unittest.TestCase): class TestBeamSearch(ModuleApiTest): - def setUp(self): paddle.set_default_dtype("float64") shape = (8, 32) self.inputs = [ np.random.random(shape).astype("float64"), - np.random.random(shape).astype("float64") + np.random.random(shape).astype("float64"), ] self.outputs = None self.attrs = { @@ -713,33 +827,40 @@ class TestBeamSearch(ModuleApiTest): self.param_states = {} @staticmethod - def model_init(self, - vocab_size, - embed_dim, - hidden_size, - bos_id=0, - eos_id=1, - beam_size=4, - max_step_num=20): - embedder = paddle.fluid.dygraph.Embedding(size=[vocab_size, embed_dim], - dtype="float64") + def model_init( + self, + vocab_size, + embed_dim, + hidden_size, + bos_id=0, + eos_id=1, + beam_size=4, + max_step_num=20, + ): + embedder = paddle.fluid.dygraph.Embedding( + size=[vocab_size, embed_dim], dtype="float64" + ) output_layer = nn.Linear(hidden_size, vocab_size) cell = nn.LSTMCell(embed_dim, hidden_size) self.max_step_num = max_step_num - self.beam_search_decoder = BeamSearchDecoder(cell, - start_token=bos_id, - end_token=eos_id, - beam_size=beam_size, - embedding_fn=embedder, - output_fn=output_layer) + self.beam_search_decoder = BeamSearchDecoder( + cell, + start_token=bos_id, + end_token=eos_id, + beam_size=beam_size, + embedding_fn=embedder, + output_fn=output_layer, + ) @staticmethod def model_forward(model, init_hidden, init_cell): - return dynamic_decode(model.beam_search_decoder, - [init_hidden, init_cell], - max_step_num=model.max_step_num, - impute_finished=True, - is_test=True)[0] + return dynamic_decode( + model.beam_search_decoder, + [init_hidden, init_cell], + max_step_num=model.max_step_num, + impute_finished=True, + is_test=True, + )[0] def make_inputs(self): inputs = [ diff --git a/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py b/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py index aa5b852ff88..f2014c29f91 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py @@ -21,59 +21,61 @@ import paddle.fluid.core as core class RNNMemoryHelperOpTest(unittest.TestCase): - def setUp(self): self.program = Program() self.place = core.CPUPlace() - self.X = self.program.global_block().create_var(name='X', - shape=[2, 3], - dtype='float32') - self.Out = self.program.global_block().create_var(name='Out', - shape=[2, 3], - dtype='float32') - self.program.global_block().append_op(type='rnn_memory_helper', - inputs={"X": self.X}, - outputs={"Out": self.Out}, - attrs={}) + self.X = self.program.global_block().create_var( + name='X', shape=[2, 3], dtype='float32' + ) + self.Out = self.program.global_block().create_var( + name='Out', shape=[2, 3], dtype='float32' + ) + self.program.global_block().append_op( + type='rnn_memory_helper', + inputs={"X": self.X}, + outputs={"Out": self.Out}, + attrs={}, + ) def test_forward(self): x_np = np.random.normal(size=(2, 3)).astype("float32") self.feed_map = {'X': x_np} self.fetch_list = [self.Out] exe = Executor(self.place) - out = exe.run(self.program, - feed=self.feed_map, - fetch_list=self.fetch_list) + out = exe.run( + self.program, feed=self.feed_map, fetch_list=self.fetch_list + ) np.testing.assert_allclose(out[0], x_np, rtol=1e-05) class RNNMemoryHelperGradOpTest(unittest.TestCase): - def setUp(self): self.program = Program() self.place = core.CPUPlace() self.input_names = ['X', 'Out', 'Out@GRAD'] self.input_vars = { - name: self.program.global_block().create_var(name=name, - shape=[2, 3], - dtype='float32') + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32' + ) for name in self.input_names } self.output_names = ['X@GRAD'] self.output_vars = { - name: self.program.global_block().create_var(name=name, - shape=[2, 3], - dtype='float32') + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32' + ) for name in self.output_names } - self.program.global_block().append_op(type='rnn_memory_helper_grad', - inputs=self.input_vars, - outputs=self.output_vars, - attrs={}) + self.program.global_block().append_op( + type='rnn_memory_helper_grad', + inputs=self.input_vars, + outputs=self.output_vars, + attrs={}, + ) def test_backward(self): self.feed_map = { @@ -83,14 +85,13 @@ class RNNMemoryHelperGradOpTest(unittest.TestCase): self.fetch_list = [self.output_vars['X@GRAD']] exe = Executor(self.place) - out = exe.run(self.program, - feed=self.feed_map, - fetch_list=self.fetch_list) + out = exe.run( + self.program, feed=self.feed_map, fetch_list=self.fetch_list + ) np.isclose(out[0], self.feed_map['Out@GRAD'], rtol=1e-5) class RNNMemoryHelperGradOpWithoutInputTest(unittest.TestCase): - def setUp(self): self.program = Program() self.fake_program = Program() @@ -98,27 +99,31 @@ class RNNMemoryHelperGradOpWithoutInputTest(unittest.TestCase): self.input_names = ['X', 'Out'] self.input_vars = { - name: self.program.global_block().create_var(name=name, - shape=[2, 3], - dtype='float32') + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32' + ) for name in self.input_names } - self.input_vars["Out@GRAD"] = \ - self.fake_program.global_block().create_var( - name="Out@GRAD", shape=[2, 3], dtype='float32') + self.input_vars[ + "Out@GRAD" + ] = self.fake_program.global_block().create_var( + name="Out@GRAD", shape=[2, 3], dtype='float32' + ) self.output_names = ['X@GRAD'] self.output_vars = { - name: self.program.global_block().create_var(name=name, - shape=[2, 3], - dtype='float32') + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32' + ) for name in self.output_names } - self.program.global_block().append_op(type='rnn_memory_helper_grad', - inputs=self.input_vars, - outputs=self.output_vars, - attrs={}) + self.program.global_block().append_op( + type='rnn_memory_helper_grad', + inputs=self.input_vars, + outputs=self.output_vars, + attrs={}, + ) def test_backward(self): self.feed_map = { @@ -128,12 +133,12 @@ class RNNMemoryHelperGradOpWithoutInputTest(unittest.TestCase): self.fetch_list = [self.output_vars['X@GRAD']] exe = Executor(self.place) - out = exe.run(self.program, - feed=self.feed_map, - fetch_list=self.fetch_list) - np.testing.assert_allclose(out[0], - np.zeros(shape=(2, 3)).astype('float32'), - rtol=1e-05) + out = exe.run( + self.program, feed=self.feed_map, fetch_list=self.fetch_list + ) + np.testing.assert_allclose( + out[0], np.zeros(shape=(2, 3)).astype('float32'), rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_rnn_op.py b/python/paddle/fluid/tests/unittests/test_rnn_op.py index ecf79cabab5..5dd255562e6 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_op.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestRNNOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(self.num_layers): @@ -45,8 +44,11 @@ class TestRNNOp(OpTest): def setUp(self): self.op_type = "rnn" self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.sequence_length = None if core.is_compiled_with_rocm( - ) else np.array([12, 11, 10, 9, 8], dtype=np.int32) + self.sequence_length = ( + None + if core.is_compiled_with_rocm() + else np.array([12, 11, 10, 9, 8], dtype=np.int32) + ) self.num_layers = 1 self.is_bidirec = False self.mode = "LSTM" @@ -61,27 +63,29 @@ class TestRNNOp(OpTest): input_size = 3 hidden_size = 2 - input = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_length, batch_size, - input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, high=0.1, size=(seq_length, batch_size, input_size) + ).astype(self.dtype) if self.sequence_length is not None: input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 - rnn1 = LSTM(input_size, - hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - dtype=self.dtype) + rnn1 = LSTM( + input_size, + hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) - output, (last_hidden, - last_cell) = rnn1(input, sequence_length=self.sequence_length) + output, (last_hidden, last_cell) = rnn1( + input, sequence_length=self.sequence_length + ) if core.is_compiled_with_rocm(): @@ -91,17 +95,19 @@ class TestRNNOp(OpTest): self._get_places = rocm_rnn_get_place - init_h = np.zeros((self.num_layers * self.direction_num, batch_size, - hidden_size)).astype(self.dtype) - init_c = np.zeros((self.num_layers * self.direction_num, batch_size, - hidden_size)).astype(self.dtype) + init_h = np.zeros( + (self.num_layers * self.direction_num, batch_size, hidden_size) + ).astype(self.dtype) + init_c = np.zeros( + (self.num_layers * self.direction_num, batch_size, hidden_size) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") self.inputs = { 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h), ('init_c', init_c)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -116,13 +122,13 @@ class TestRNNOp(OpTest): 'hidden_size': hidden_size, 'num_layers': self.num_layers, 'mode': self.mode, - 'is_test': self.is_test + 'is_test': self.is_test, } self.outputs = { 'Out': output, "State": [('last_hidden', last_hidden), ('last_cell', last_cell)], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def test_output(self): @@ -136,56 +142,56 @@ class TestRNNOp(OpTest): var_name_list = self.get_weight_names() grad_check_list = ['Input', 'init_h', 'init_c'] grad_check_list.extend(var_name_list) - self.check_grad(set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad( + set(grad_check_list), ['Out', 'last_hidden', 'last_cell'] + ) def test_grad_only_input(self): if not self.is_test: var_name_list = self.get_weight_names() grad_check_list = ['Input'] grad_check_list.extend(var_name_list) - self.check_grad(set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad( + set(grad_check_list), ['Out', 'last_hidden', 'last_cell'] + ) def test_grad_only_h(self): if not self.is_test: var_name_list = self.get_weight_names() grad_check_list = ['init_h'] grad_check_list.extend(var_name_list) - self.check_grad(set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad( + set(grad_check_list), ['Out', 'last_hidden', 'last_cell'] + ) def test_grad_only_c(self): if not self.is_test: var_name_list = self.get_weight_names() grad_check_list = ['init_c'] grad_check_list.extend(var_name_list) - self.check_grad(set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad( + set(grad_check_list), ['Out', 'last_hidden', 'last_cell'] + ) class TestRNNOp1(TestRNNOp): - def set_attrs(self): self.sequence_length = None class TestRNNOp2(TestRNNOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True class TestRNNOp3(TestRNNOp): - def set_attrs(self): self.is_test = True self.sequence_length = None class TestRNNOp4(TestRNNOp): - def set_attrs(self): self.is_test = True self.sequence_length = None @@ -193,20 +199,17 @@ class TestRNNOp4(TestRNNOp): class TestRNNOp5(TestRNNOp): - def set_attrs(self): self.num_layers = 2 class TestRNNOp6(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True class TestRNNOp7(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True @@ -214,7 +217,6 @@ class TestRNNOp7(TestRNNOp): class TestRNNOp8(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True @@ -222,7 +224,6 @@ class TestRNNOp8(TestRNNOp): class TestRNNOp9(TestRNNOp): - def set_attrs(self): self.num_layers = 3 diff --git a/python/paddle/fluid/tests/unittests/test_roi_align_op.py b/python/paddle/fluid/tests/unittests/test_roi_align_op.py index d9111adceb6..c26b6691f66 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_align_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_align_op.py @@ -20,7 +20,6 @@ from op_test import OpTest class TestROIAlignOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -29,7 +28,7 @@ class TestROIAlignOp(OpTest): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': self.boxes_num + 'RoisNum': self.boxes_num, } self.attrs = { 'spatial_scale': self.spatial_scale, @@ -58,25 +57,45 @@ class TestROIAlignOp(OpTest): self.x = np.random.random(self.x_dim).astype('float64') - def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, - bin_size_h, bin_size_w): + def pre_calc( + self, + x_i, + roi_xmin, + roi_ymin, + roi_bin_grid_h, + roi_bin_grid_w, + bin_size_h, + bin_size_w, + ): count = roi_bin_grid_h * roi_bin_grid_w bilinear_pos = np.zeros( [self.channels, self.pooled_height, self.pooled_width, count, 4], - np.float64) - bilinear_w = np.zeros([self.pooled_height, self.pooled_width, count, 4], - np.float64) + np.float64, + ) + bilinear_w = np.zeros( + [self.pooled_height, self.pooled_width, count, 4], np.float64 + ) for ph in range(self.pooled_width): for pw in range(self.pooled_height): c = 0 for iy in range(roi_bin_grid_h): - y = roi_ymin + ph * bin_size_h + (iy + 0.5) * \ - bin_size_h / roi_bin_grid_h + y = ( + roi_ymin + + ph * bin_size_h + + (iy + 0.5) * bin_size_h / roi_bin_grid_h + ) for ix in range(roi_bin_grid_w): - x = roi_xmin + pw * bin_size_w + (ix + 0.5) * \ - bin_size_w / roi_bin_grid_w - if y < -1.0 or y > self.height or \ - x < -1.0 or x > self.width: + x = ( + roi_xmin + + pw * bin_size_w + + (ix + 0.5) * bin_size_w / roi_bin_grid_w + ) + if ( + y < -1.0 + or y > self.height + or x < -1.0 + or x > self.width + ): continue if y <= 0: y = 0 @@ -97,14 +116,18 @@ class TestROIAlignOp(OpTest): hy = 1 - ly hx = 1 - lx for ch in range(self.channels): - bilinear_pos[ch, ph, pw, c, 0] = x_i[ch, y_low, - x_low] - bilinear_pos[ch, ph, pw, c, 1] = x_i[ch, y_low, - x_high] - bilinear_pos[ch, ph, pw, c, 2] = x_i[ch, y_high, - x_low] - bilinear_pos[ch, ph, pw, c, 3] = x_i[ch, y_high, - x_high] + bilinear_pos[ch, ph, pw, c, 0] = x_i[ + ch, y_low, x_low + ] + bilinear_pos[ch, ph, pw, c, 1] = x_i[ + ch, y_low, x_high + ] + bilinear_pos[ch, ph, pw, c, 2] = x_i[ + ch, y_high, x_low + ] + bilinear_pos[ch, ph, pw, c, 3] = x_i[ + ch, y_high, x_high + ] bilinear_w[ph, pw, c, 0] = hy * hx bilinear_w[ph, pw, c, 1] = hy * lx bilinear_w[ph, pw, c, 2] = ly * hx @@ -114,10 +137,15 @@ class TestROIAlignOp(OpTest): def calc_roi_align(self): self.out_data = np.zeros( - (self.rois_num, self.channels, self.pooled_height, - self.pooled_width)).astype('float64') - - offset = 0.5 if self.aligned else 0. + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ).astype('float64') + + offset = 0.5 if self.aligned else 0.0 for i in range(self.rois_num): roi = self.rois[i] roi_batch_id = int(roi[0]) @@ -135,16 +163,27 @@ class TestROIAlignOp(OpTest): bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) - roi_bin_grid_h = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_height / self.pooled_height) - roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_width / self.pooled_width) + roi_bin_grid_h = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_height / self.pooled_height) + ) + roi_bin_grid_w = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_width / self.pooled_width) + ) count = max(int(roi_bin_grid_h * roi_bin_grid_w), 1) pre_size = count * self.pooled_width * self.pooled_height - bilinear_pos, bilinear_w = self.pre_calc(x_i, roi_xmin, roi_ymin, - int(roi_bin_grid_h), - int(roi_bin_grid_w), - bin_size_h, bin_size_w) + bilinear_pos, bilinear_w = self.pre_calc( + x_i, + roi_xmin, + roi_ymin, + int(roi_bin_grid_h), + int(roi_bin_grid_w), + bin_size_h, + bin_size_w, + ) for ch in range(self.channels): align_per_bin = (bilinear_pos[ch] * bilinear_w).sum(axis=-1) output_val = align_per_bin.mean(axis=-1) @@ -157,27 +196,38 @@ class TestROIAlignOp(OpTest): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.random_integers( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.random_integers(x1 + self.pooled_width, - self.width // self.spatial_scale) + x2 = np.random.random_integers( + x1 + self.pooled_width, self.width // self.spatial_scale + ) y2 = np.random.random_integers( - y1 + self.pooled_height, self.height // self.spatial_scale) + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) self.rois = np.array(rois).astype("float64") - self.boxes_num = np.array([bno + 1 for bno in range(self.batch_size) - ]).astype('int32') + self.boxes_num = np.array( + [bno + 1 for bno in range(self.batch_size)] + ).astype('int32') def setUp(self): self.op_type = "roi_align" self.python_api = lambda x, boxes, boxes_num, pooled_height, pooled_width, spatial_scale, sampling_ratio, aligned: paddle.vision.ops.roi_align( - x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale, - sampling_ratio, aligned) + x, + boxes, + boxes_num, + (pooled_height, pooled_width), + spatial_scale, + sampling_ratio, + aligned, + ) self.set_data() def test_check_output(self): @@ -188,7 +238,6 @@ class TestROIAlignOp(OpTest): class TestROIAlignInLodOp(TestROIAlignOp): - def set_data(self): self.init_test_case() self.make_rois() @@ -199,7 +248,7 @@ class TestROIAlignInLodOp(TestROIAlignOp): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } self.attrs = { @@ -207,14 +256,13 @@ class TestROIAlignInLodOp(TestROIAlignOp): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.aligned + 'aligned': self.aligned, } self.outputs = {'Out': self.out_data} class TestROIAlignOpWithAligned(TestROIAlignOp): - def init_test_case(self): self.batch_size = 3 self.channels = 3 diff --git a/python/paddle/fluid/tests/unittests/test_roi_perspective_transform_op.py b/python/paddle/fluid/tests/unittests/test_roi_perspective_transform_op.py index ff680559043..d03bdcd5920 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_perspective_transform_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_perspective_transform_op.py @@ -40,13 +40,20 @@ def in_quad(x, y, roi_x, roi_y): xe = roi_x[(i + 1) % 4] ye = roi_y[(i + 1) % 4] if abs(ys - ye) < 1e-4: - if abs(y - ys) < 1e-4 and abs(y - ye) < 1e-4 and gt_e( - x, min(xs, xe)) and lt_e(x, max(xs, xe)): + if ( + abs(y - ys) < 1e-4 + and abs(y - ye) < 1e-4 + and gt_e(x, min(xs, xe)) + and lt_e(x, max(xs, xe)) + ): return True else: intersec_x = (y - ys) * (xe - xs) / (ye - ys) + xs - if abs(intersec_x - x) < 1e-4 and gt_e(y, min(ys, ye)) and lt_e( - y, max(ys, ye)): + if ( + abs(intersec_x - x) < 1e-4 + and gt_e(y, min(ys, ye)) + and lt_e(y, max(ys, ye)) + ): return True n_cross = 0 for i in range(4): @@ -63,7 +70,7 @@ def in_quad(x, y, roi_x, roi_y): return True if gt(intersec_x, x): n_cross += 1 - return (n_cross % 2 == 1) + return n_cross % 2 == 1 def get_transform_matrix(transformed_width, transformed_height, roi_x, roi_y): @@ -84,8 +91,9 @@ def get_transform_matrix(transformed_width, transformed_height, roi_x, roi_y): estimated_width = (len1 + len3) / 2.0 normalized_height = max(2, transformed_height) - normalized_width = round(estimated_width * - (normalized_height - 1) / estimated_height) + 1 + normalized_width = ( + round(estimated_width * (normalized_height - 1) / estimated_height) + 1 + ) normalized_width = max(2, min(normalized_width, transformed_width)) dx1 = x1 - x2 @@ -95,22 +103,32 @@ def get_transform_matrix(transformed_width, transformed_height, roi_x, roi_y): dy2 = y3 - y2 dy3 = y0 - y1 + y2 - y3 matrix = np.zeros([9]) - matrix[6] = (dx3 * dy2 - dx2 * dy3) / (dx1 * dy2 - dx2 * dy1 + - 1e-5) / (normalized_width - 1) - matrix[7] = (dx1 * dy3 - dx3 * dy1) / (dx1 * dy2 - dx2 * dy1 + - 1e-5) / (normalized_height - 1) + matrix[6] = ( + (dx3 * dy2 - dx2 * dy3) + / (dx1 * dy2 - dx2 * dy1 + 1e-5) + / (normalized_width - 1) + ) + matrix[7] = ( + (dx1 * dy3 - dx3 * dy1) + / (dx1 * dy2 - dx2 * dy1 + 1e-5) + / (normalized_height - 1) + ) matrix[8] = 1 - matrix[3] = (y1 - y0 + matrix[6] * - (normalized_width - 1) * y1) / (normalized_width - 1) - matrix[4] = (y3 - y0 + matrix[7] * - (normalized_height - 1) * y3) / (normalized_height - 1) + matrix[3] = (y1 - y0 + matrix[6] * (normalized_width - 1) * y1) / ( + normalized_width - 1 + ) + matrix[4] = (y3 - y0 + matrix[7] * (normalized_height - 1) * y3) / ( + normalized_height - 1 + ) matrix[5] = y0 - matrix[0] = (x1 - x0 + matrix[6] * - (normalized_width - 1) * x1) / (normalized_width - 1) - matrix[1] = (x3 - x0 + matrix[7] * - (normalized_height - 1) * x3) / (normalized_height - 1) + matrix[0] = (x1 - x0 + matrix[6] * (normalized_width - 1) * x1) / ( + normalized_width - 1 + ) + matrix[1] = (x3 - x0 + matrix[7] * (normalized_height - 1) * x3) / ( + normalized_height - 1 + ) matrix[2] = x0 return matrix @@ -131,8 +149,12 @@ def bilinear_interpolate(in_data, in_n, in_c, in_w, in_h): height = in_data.shape[2] width = in_data.shape[3] - if gt_e(-0.5, in_w) or gt_e(in_w, width - 0.5) or gt_e(-0.5, in_h) or gt_e( - in_h, height - 0.5): + if ( + gt_e(-0.5, in_w) + or gt_e(in_w, width - 0.5) + or gt_e(-0.5, in_h) + or gt_e(in_h, height - 0.5) + ): return 0.0 if gt_e(0, in_w): @@ -180,8 +202,14 @@ def lod_convert(lod): return ret -def roi_transform(in_data, rois, rois_lod, transformed_height, - transformed_width, spatial_scale): +def roi_transform( + in_data, + rois, + rois_lod, + transformed_height, + transformed_width, + spatial_scale, +): channels = in_data.shape[1] in_height = in_data.shape[2] in_width = in_data.shape[3] @@ -194,8 +222,9 @@ def roi_transform(in_data, rois, rois_lod, transformed_height, roi2image[j] = i out = np.zeros([rois_num, channels, transformed_height, transformed_width]) - mask = np.zeros([rois_num, 1, transformed_height, - transformed_width]).astype('int') + mask = np.zeros( + [rois_num, 1, transformed_height, transformed_width] + ).astype('int') matrix = np.zeros([rois_num, 9], dtype=in_data.dtype) for n in range(rois_num): roi_x = [] @@ -204,20 +233,26 @@ def roi_transform(in_data, rois, rois_lod, transformed_height, roi_x.append(rois[n][2 * k] * spatial_scale) roi_y.append(rois[n][2 * k + 1] * spatial_scale) image_id = roi2image[n] - transform_matrix = get_transform_matrix(transformed_width, - transformed_height, roi_x, - roi_y) + transform_matrix = get_transform_matrix( + transformed_width, transformed_height, roi_x, roi_y + ) matrix[n] = transform_matrix for c in range(channels): for out_h in range(transformed_height): for out_w in range(transformed_width): - in_w, in_h = get_source_coords(transform_matrix, out_w, - out_h) - if in_quad(in_w, in_h, roi_x, roi_y) and gt( - in_w, -0.5) and gt(in_width - 0.5, in_w) and gt( - in_h, -0.5) and gt(in_height - 0.5, in_h): + in_w, in_h = get_source_coords( + transform_matrix, out_w, out_h + ) + if ( + in_quad(in_w, in_h, roi_x, roi_y) + and gt(in_w, -0.5) + and gt(in_width - 0.5, in_w) + and gt(in_h, -0.5) + and gt(in_height - 0.5, in_h) + ): out[n][c][out_h][out_w] = bilinear_interpolate( - in_data, image_id, c, in_w, in_h) + in_data, image_id, c, in_w, in_h + ) mask[n][0][out_h][out_w] = 1 else: out[n][c][out_h][out_w] = 0.0 @@ -226,7 +261,6 @@ def roi_transform(in_data, rois, rois_lod, transformed_height, class TestROIPoolOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -236,17 +270,20 @@ class TestROIPoolOp(OpTest): self.attrs = { 'spatial_scale': self.spatial_scale, 'transformed_height': self.transformed_height, - 'transformed_width': self.transformed_width + 'transformed_width': self.transformed_width, } - out, mask, transform_matrix = roi_transform(self.x, self.rois, - self.rois_lod, - self.transformed_height, - self.transformed_width, - self.spatial_scale) + out, mask, transform_matrix = roi_transform( + self.x, + self.rois, + self.rois_lod, + self.transformed_height, + self.transformed_width, + self.spatial_scale, + ) self.outputs = { 'Out': out, 'Mask': mask, - 'TransformMatrix': transform_matrix + 'TransformMatrix': transform_matrix, } def init_test_case(self): @@ -271,28 +308,38 @@ class TestROIPoolOp(OpTest): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.randint( - 0, - self.width // self.spatial_scale - self.transformed_width) + 0, self.width // self.spatial_scale - self.transformed_width + ) y1 = np.random.randint( 0, - self.height // self.spatial_scale - self.transformed_height) + self.height // self.spatial_scale - self.transformed_height, + ) - x2 = np.random.randint(x1 + self.transformed_width, - self.width // self.spatial_scale) + x2 = np.random.randint( + x1 + self.transformed_width, + self.width // self.spatial_scale, + ) y2 = np.random.randint( 0, - self.height // self.spatial_scale - self.transformed_height) - - x3 = np.random.randint(x1 + self.transformed_width, - self.width // self.spatial_scale) - y3 = np.random.randint(y1 + self.transformed_height, - self.height // self.spatial_scale) + self.height // self.spatial_scale - self.transformed_height, + ) + + x3 = np.random.randint( + x1 + self.transformed_width, + self.width // self.spatial_scale, + ) + y3 = np.random.randint( + y1 + self.transformed_height, + self.height // self.spatial_scale, + ) x4 = np.random.randint( - 0, - self.width // self.spatial_scale - self.transformed_width) - y4 = np.random.randint(y1 + self.transformed_height, - self.height // self.spatial_scale) + 0, self.width // self.spatial_scale - self.transformed_width + ) + y4 = np.random.randint( + y1 + self.transformed_height, + self.height // self.spatial_scale, + ) roi = [x1, y1, x2, y2, x3, y3, x4, y4] rois.append(roi) @@ -308,52 +355,70 @@ class TestROIPoolOp(OpTest): def test_check_grad(self): self.outputs['Out2InIdx'] = np.zeros( - [np.product(self.outputs['Out'].shape), 4]).astype("int32") + [np.product(self.outputs['Out'].shape), 4] + ).astype("int32") self.outputs['Out2InWeights'] = np.zeros( - [np.product(self.outputs['Out'].shape), 4]).astype("float32") + [np.product(self.outputs['Out'].shape), 4] + ).astype("float32") self.check_grad(['X'], 'Out') def test_errors(self): x = fluid.data(name='x', shape=[100, 256, 28, 28], dtype='float32') - rois = fluid.data(name='rois', - shape=[None, 8], - lod_level=1, - dtype='float32') - - x_int = fluid.data(name='x_int', - shape=[100, 256, 28, 28], - dtype='int32') - rois_int = fluid.data(name='rois_int', - shape=[None, 8], - lod_level=1, - dtype='int32') + rois = fluid.data( + name='rois', shape=[None, 8], lod_level=1, dtype='float32' + ) + + x_int = fluid.data( + name='x_int', shape=[100, 256, 28, 28], dtype='int32' + ) + rois_int = fluid.data( + name='rois_int', shape=[None, 8], lod_level=1, dtype='int32' + ) x_tmp = [1, 2] rois_tmp = [1, 2] # type of intput and rois must be variable - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, - x_tmp, rois, 7, 7) - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois_tmp, 7, 7) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x_tmp, rois, 7, 7 + ) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x, rois_tmp, 7, 7 + ) # dtype of intput and rois must be float32 - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, - x_int, rois, 7, 7) - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois_int, 7, 7) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x_int, rois, 7, 7 + ) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x, rois_int, 7, 7 + ) height = 7.5 width = 7.5 # type of transformed_height and transformed_width must be int - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois, height, 7) - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois, 7, width) + self.assertRaises( + TypeError, + fluid.layers.roi_perspective_transform, + x, + rois, + height, + 7, + ) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x, rois, 7, width + ) scale = int(2) # type of spatial_scale must be float - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois, 7, 7, scale) + self.assertRaises( + TypeError, + fluid.layers.roi_perspective_transform, + x, + rois, + 7, + 7, + scale, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py index 16ff2f57fb8..2fcd3eda287 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py @@ -33,7 +33,6 @@ def _round(x): class TestROIPoolOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -42,13 +41,13 @@ class TestROIPoolOp(OpTest): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': self.boxes_num + 'RoisNum': self.boxes_num, } self.attrs = { 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs, 'Argmax': self.argmaxes} @@ -69,10 +68,22 @@ class TestROIPoolOp(OpTest): self.x = np.random.random(self.x_dim).astype('float64') def calc_roi_pool(self): - out_data = np.zeros((self.rois_num, self.channels, self.pooled_height, - self.pooled_width)) - argmax_data = np.zeros((self.rois_num, self.channels, - self.pooled_height, self.pooled_width)) + out_data = np.zeros( + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ) + argmax_data = np.zeros( + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ) for i in range(self.rois_num): roi = self.rois[i] @@ -115,8 +126,9 @@ class TestROIPoolOp(OpTest): for w in range(wstart, wend): if x_i[c, h, w] > out_data[i, c, ph, pw]: out_data[i, c, ph, pw] = x_i[c, h, w] - argmax_data[i, c, ph, - pw] = h * self.width + w + argmax_data[i, c, ph, pw] = ( + h * self.width + w + ) self.outs = out_data.astype('float64') self.argmaxes = argmax_data.astype('int64') @@ -128,26 +140,32 @@ class TestROIPoolOp(OpTest): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.randint( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.randint( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.randint(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.randint(y1 + self.pooled_height, - self.height // self.spatial_scale) + x2 = np.random.randint( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.randint( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) self.rois = np.array(rois).astype("float64") - self.boxes_num = np.array([bno + 1 for bno in range(self.batch_size) - ]).astype('int32') + self.boxes_num = np.array( + [bno + 1 for bno in range(self.batch_size)] + ).astype('int32') def setUp(self): self.op_type = "roi_pool" self.python_api = lambda x, boxes, boxes_num, pooled_height, pooled_width, spatial_scale: paddle.vision.ops.roi_pool( - x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale) + x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale + ) self.python_out_sig = ["Out"] self.set_data() @@ -159,27 +177,27 @@ class TestROIPoolOp(OpTest): class BadInputTestRoiPool(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - x = fluid.layers.data(name='data1', - shape=[2, 1, 4, 4], - dtype='int64') - label = fluid.layers.data(name='label', - shape=[2, 4], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='data1', shape=[2, 1, 4, 4], dtype='int64' + ) + label = fluid.layers.data( + name='label', shape=[2, 4], dtype='float32', lod_level=1 + ) output = fluid.layers.roi_pool(x, label, 1, 1, 1.0) self.assertRaises(TypeError, test_bad_x) def test_bad_y(): - x = fluid.layers.data(name='data2', - shape=[2, 1, 4, 4], - dtype='float32', - append_batch_size=False) + x = fluid.layers.data( + name='data2', + shape=[2, 1, 4, 4], + dtype='float32', + append_batch_size=False, + ) label = [[1, 2, 3, 4], [2, 3, 4, 5]] output = fluid.layers.roi_pool(x, label, 1, 1, 1.0) @@ -187,7 +205,6 @@ class BadInputTestRoiPool(unittest.TestCase): class TestROIPoolInLodOp(TestROIPoolOp): - def set_data(self): self.init_test_case() self.make_rois() @@ -198,13 +215,13 @@ class TestROIPoolInLodOp(TestROIPoolOp): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } self.attrs = { 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs, 'Argmax': self.argmaxes} diff --git a/python/paddle/fluid/tests/unittests/test_roll_op.py b/python/paddle/fluid/tests/unittests/test_roll_op.py index 43e56efd642..bf11b0d86f6 100644 --- a/python/paddle/fluid/tests/unittests/test_roll_op.py +++ b/python/paddle/fluid/tests/unittests/test_roll_op.py @@ -21,7 +21,6 @@ from paddle.fluid import Program, program_guard class TestRollOp(OpTest): - def setUp(self): self.python_api = paddle.roll self.op_type = "roll" @@ -29,8 +28,9 @@ class TestRollOp(OpTest): self.inputs = {'X': np.random.random(self.x_shape).astype(self.dtype)} self.attrs = {'shifts': self.shifts, 'axis': self.axis} self.outputs = { - 'Out': - np.roll(self.inputs['X'], self.attrs['shifts'], self.attrs['axis']) + 'Out': np.roll( + self.inputs['X'], self.attrs['shifts'], self.attrs['axis'] + ) } def init_dtype_type(self): @@ -47,7 +47,6 @@ class TestRollOp(OpTest): class TestRollOpCase2(TestRollOp): - def init_dtype_type(self): self.dtype = np.float32 self.x_shape = (100, 10, 5) @@ -56,10 +55,10 @@ class TestRollOpCase2(TestRollOp): class TestRollAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], - [7.0, 8.0, 9.0]]) + self.data_x = np.array( + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]] + ) def test_roll_op_api(self): self.input_data() @@ -70,11 +69,12 @@ class TestRollAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[-1, 3]) z = paddle.roll(x, shifts=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': self.data_x}, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[9.0, 1.0, 2.0], [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0]]) + (res,) = exe.run( + feed={'x': self.data_x}, fetch_list=[z.name], return_numpy=False + ) + expect_out = np.array( + [[9.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 2: @@ -82,11 +82,12 @@ class TestRollAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[-1, 3]) z = paddle.roll(x, shifts=1, axis=0) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': self.data_x}, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[7.0, 8.0, 9.0], [1.0, 2.0, 3.0], - [4.0, 5.0, 6.0]]) + (res,) = exe.run( + feed={'x': self.data_x}, fetch_list=[z.name], return_numpy=False + ) + expect_out = np.array( + [[7.0, 8.0, 9.0], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) def test_dygraph_api(self): @@ -96,8 +97,9 @@ class TestRollAPI(unittest.TestCase): x = fluid.dygraph.to_variable(self.data_x) z = paddle.roll(x, shifts=1) np_z = z.numpy() - expect_out = np.array([[9.0, 1.0, 2.0], [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0]]) + expect_out = np.array( + [[9.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) # case 2: @@ -105,8 +107,9 @@ class TestRollAPI(unittest.TestCase): x = fluid.dygraph.to_variable(self.data_x) z = paddle.roll(x, shifts=1, axis=0) np_z = z.numpy() - expect_out = np.array([[7.0, 8.0, 9.0], [1.0, 2.0, 3.0], - [4.0, 5.0, 6.0]]) + expect_out = np.array( + [[7.0, 8.0, 9.0], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) def test_roll_op_false(self): @@ -117,9 +120,11 @@ class TestRollAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[-1, 3]) z = paddle.roll(x, shifts=1, axis=10) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': self.data_x}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': self.data_x}, + fetch_list=[z.name], + return_numpy=False, + ) self.assertRaises(ValueError, test_axis_out_range) diff --git a/python/paddle/fluid/tests/unittests/test_rot90_op.py b/python/paddle/fluid/tests/unittests/test_rot90_op.py index 6a1cbcaccda..1cf3a21e9d0 100644 --- a/python/paddle/fluid/tests/unittests/test_rot90_op.py +++ b/python/paddle/fluid/tests/unittests/test_rot90_op.py @@ -37,15 +37,17 @@ class TestRot90_API(unittest.TestCase): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[4, 1], [5, 2], [6, 3]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_k_0(self): paddle.enable_static() @@ -62,15 +64,17 @@ class TestRot90_API(unittest.TestCase): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_k_2(self): paddle.enable_static() @@ -87,15 +91,17 @@ class TestRot90_API(unittest.TestCase): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[6, 5, 4], [3, 2, 1]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_k_3(self): paddle.enable_static() @@ -112,15 +118,17 @@ class TestRot90_API(unittest.TestCase): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[4, 1], [5, 2], [6, 3]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_neg_k_1(self): paddle.enable_static() @@ -137,15 +145,17 @@ class TestRot90_API(unittest.TestCase): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[4, 1], [5, 2], [6, 3]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_neg_k_2(self): paddle.enable_static() @@ -162,15 +172,17 @@ class TestRot90_API(unittest.TestCase): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[6, 5, 4], [3, 2, 1]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_neg_k_3(self): paddle.enable_static() @@ -187,15 +199,17 @@ class TestRot90_API(unittest.TestCase): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[3, 6], [2, 5], [1, 4]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_neg_k_4(self): paddle.enable_static() @@ -212,15 +226,17 @@ class TestRot90_API(unittest.TestCase): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_error_api(self): paddle.enable_static() @@ -269,7 +285,8 @@ class TestRot90_API(unittest.TestCase): self.assertTrue( (ret.numpy() == out_ref).all(), - msg='rot90 output is wrong, out =' + str(ret.numpy())) + msg='rot90 output is wrong, out =' + str(ret.numpy()), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_row_conv_op.py b/python/paddle/fluid/tests/unittests/test_row_conv_op.py index 6350b257247..6f4df64ce50 100644 --- a/python/paddle/fluid/tests/unittests/test_row_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_row_conv_op.py @@ -43,7 +43,6 @@ def row_conv_forward(x, lod, wt): class TestRowConvOp1(OpTest): - def setUp(self): self.op_type = "row_conv" @@ -66,20 +65,17 @@ class TestRowConvOp1(OpTest): self.check_grad(['X', 'Filter'], 'Out', check_dygraph=False) def test_check_grad_ignore_x(self): - self.check_grad(['Filter'], - 'Out', - no_grad_set=set('X'), - check_dygraph=False) + self.check_grad( + ['Filter'], 'Out', no_grad_set=set('X'), check_dygraph=False + ) def test_check_grad_ignore_wt(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Filter'), - check_dygraph=False) + self.check_grad( + ['X'], 'Out', no_grad_set=set('Filter'), check_dygraph=False + ) class TestRowConvOp2(OpTest): - def setUp(self): self.op_type = "row_conv" @@ -98,28 +94,31 @@ class TestRowConvOp2(OpTest): def test_check_output(self): self.check_output(check_dygraph=False) - #max_relative_error is increased from 0.05 to 0.06 as for higher - #dimensional input, the dX on CPU for some values has max_rel_error - #slightly more than 0.05 + # max_relative_error is increased from 0.05 to 0.06 as for higher + # dimensional input, the dX on CPU for some values has max_rel_error + # slightly more than 0.05 def test_check_grad_normal(self): - self.check_grad(['X', 'Filter'], - 'Out', - max_relative_error=0.06, - check_dygraph=False) + self.check_grad( + ['X', 'Filter'], 'Out', max_relative_error=0.06, check_dygraph=False + ) def test_check_grad_ignore_x(self): - self.check_grad(['Filter'], - 'Out', - max_relative_error=0.06, - no_grad_set=set('X'), - check_dygraph=False) + self.check_grad( + ['Filter'], + 'Out', + max_relative_error=0.06, + no_grad_set=set('X'), + check_dygraph=False, + ) def test_check_grad_ignore_wt(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.06, - no_grad_set=set('Filter'), - check_dygraph=False) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.06, + no_grad_set=set('Filter'), + check_dygraph=False, + ) def row_conv_foward_Tensor(x, wt): @@ -128,8 +127,8 @@ def row_conv_foward_Tensor(x, wt): timesteps = x.shape[1] context_length = wt.shape[0] for i in range(num_sequence): - cur_in = x[i:i + 1, :][0] - cur_out = out[i:i + 1, :][0] + cur_in = x[i : i + 1, :][0] + cur_out = out[i : i + 1, :][0] for j in range(timesteps): for k in range(context_length): if j + k >= timesteps: @@ -139,7 +138,6 @@ def row_conv_foward_Tensor(x, wt): class TestRowOpWithTensorInput(OpTest): - def setUp(self): self.op_type = "row_conv" length = [1, 2, 3] @@ -159,23 +157,20 @@ class TestRowOpWithTensorInput(OpTest): self.check_output(check_dygraph=False) def test_check_grad_ignore_x(self): - self.check_grad(['Filter'], - 'Out', - no_grad_set=set('X'), - check_dygraph=False) + self.check_grad( + ['Filter'], 'Out', no_grad_set=set('X'), check_dygraph=False + ) def test_check_grad_normal(self): self.check_grad(['X', 'Filter'], 'Out', check_dygraph=False) def test_check_grad_ignore_wt(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Filter'), - check_dygraph=False) + self.check_grad( + ['X'], 'Out', no_grad_set=set('Filter'), check_dygraph=False + ) class TestRowConvLayer(unittest.TestCase): - def setUp(self): self.B = 2 self.T = 6 @@ -183,8 +178,9 @@ class TestRowConvLayer(unittest.TestCase): self.context_length = 6 self.x = np.random.random((self.B, self.T, self.C)).astype("float32") - self.w = np.random.random( - (self.context_length, self.C)).astype("float32") + self.w = np.random.random((self.context_length, self.C)).astype( + "float32" + ) self.out = row_conv_foward_Tensor(self.x, self.w) def check_identity(self): @@ -196,11 +192,12 @@ class TestRowConvLayer(unittest.TestCase): out = fluid.layers.row_conv( x, self.context_length, - param_attr=fluid.initializer.NumpyArrayInitializer(self.w)) + param_attr=fluid.initializer.NumpyArrayInitializer(self.w), + ) place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(start) - out_np, = exe.run(main, feed={'x': self.x}, fetch_list=[out]) + (out_np,) = exe.run(main, feed={'x': self.x}, fetch_list=[out]) np.testing.assert_allclose(out_np, self.out) diff --git a/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py index de3e8e21136..ce37fedaf59 100644 --- a/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py @@ -22,33 +22,37 @@ from test_generate_proposal_labels_op import _generate_groundtruth from test_generate_proposal_labels_op import _bbox_overlaps, _box_to_delta -def rpn_target_assign(anchor_by_gt_overlap, - rpn_batch_size_per_im, - rpn_positive_overlap, - rpn_negative_overlap, - rpn_fg_fraction, - use_random=True): +def rpn_target_assign( + anchor_by_gt_overlap, + rpn_batch_size_per_im, + rpn_positive_overlap, + rpn_negative_overlap, + rpn_fg_fraction, + use_random=True, +): anchor_to_gt_argmax = anchor_by_gt_overlap.argmax(axis=1) anchor_to_gt_max = anchor_by_gt_overlap[ - np.arange(anchor_by_gt_overlap.shape[0]), anchor_to_gt_argmax] + np.arange(anchor_by_gt_overlap.shape[0]), anchor_to_gt_argmax + ] gt_to_anchor_argmax = anchor_by_gt_overlap.argmax(axis=0) gt_to_anchor_max = anchor_by_gt_overlap[ - gt_to_anchor_argmax, - np.arange(anchor_by_gt_overlap.shape[1])] + gt_to_anchor_argmax, np.arange(anchor_by_gt_overlap.shape[1]) + ] anchors_with_max_overlap = np.where( - anchor_by_gt_overlap == gt_to_anchor_max)[0] + anchor_by_gt_overlap == gt_to_anchor_max + )[0] - labels = np.ones((anchor_by_gt_overlap.shape[0], ), dtype=np.int32) * -1 + labels = np.ones((anchor_by_gt_overlap.shape[0],), dtype=np.int32) * -1 labels[anchors_with_max_overlap] = 1 labels[anchor_to_gt_max >= rpn_positive_overlap] = 1 num_fg = int(rpn_fg_fraction * rpn_batch_size_per_im) fg_inds = np.where(labels == 1)[0] if len(fg_inds) > num_fg and use_random: - disable_inds = np.random.choice(fg_inds, - size=(len(fg_inds) - num_fg), - replace=False) + disable_inds = np.random.choice( + fg_inds, size=(len(fg_inds) - num_fg), replace=False + ) else: disable_inds = fg_inds[num_fg:] @@ -87,26 +91,30 @@ def rpn_target_assign(anchor_by_gt_overlap, def get_anchor(n, c, h, w): input_feat = np.random.random((n, c, h, w)).astype('float32') - anchors, _ = anchor_generator_in_python(input_feat=input_feat, - anchor_sizes=[32., 64.], - aspect_ratios=[0.5, 1.0], - variances=[1.0, 1.0, 1.0, 1.0], - stride=[16.0, 16.0], - offset=0.5) + anchors, _ = anchor_generator_in_python( + input_feat=input_feat, + anchor_sizes=[32.0, 64.0], + aspect_ratios=[0.5, 1.0], + variances=[1.0, 1.0, 1.0, 1.0], + stride=[16.0, 16.0], + offset=0.5, + ) return anchors -def rpn_target_assign_in_python(all_anchors, - gt_boxes, - is_crowd, - im_info, - lod, - rpn_straddle_thresh, - rpn_batch_size_per_im, - rpn_positive_overlap, - rpn_negative_overlap, - rpn_fg_fraction, - use_random=True): +def rpn_target_assign_in_python( + all_anchors, + gt_boxes, + is_crowd, + im_info, + lod, + rpn_straddle_thresh, + rpn_batch_size_per_im, + rpn_positive_overlap, + rpn_negative_overlap, + rpn_fg_fraction, + use_random=True, +): anchor_num = all_anchors.shape[0] batch_size = len(lod) - 1 for i in range(batch_size): @@ -119,7 +127,8 @@ def rpn_target_assign_in_python(all_anchors, (all_anchors[:, 0] >= -rpn_straddle_thresh) & (all_anchors[:, 1] >= -rpn_straddle_thresh) & (all_anchors[:, 2] < im_width + rpn_straddle_thresh) - & (all_anchors[:, 3] < im_height + rpn_straddle_thresh))[0] + & (all_anchors[:, 3] < im_height + rpn_straddle_thresh) + )[0] # keep only inside anchors inside_anchors = all_anchors[inds_inside, :] else: @@ -134,19 +143,29 @@ def rpn_target_assign_in_python(all_anchors, gt_boxes_slice = gt_boxes_slice[not_crowd_inds] iou = _bbox_overlaps(inside_anchors, gt_boxes_slice) - loc_inds, score_inds, labels, gt_inds, bbox_inside_weight = \ - rpn_target_assign(iou, rpn_batch_size_per_im, - rpn_positive_overlap, - rpn_negative_overlap, - rpn_fg_fraction, - use_random) + ( + loc_inds, + score_inds, + labels, + gt_inds, + bbox_inside_weight, + ) = rpn_target_assign( + iou, + rpn_batch_size_per_im, + rpn_positive_overlap, + rpn_negative_overlap, + rpn_fg_fraction, + use_random, + ) # unmap to all anchor loc_inds = inds_inside[loc_inds] score_inds = inds_inside[score_inds] sampled_gt = gt_boxes_slice[gt_inds] sampled_anchor = all_anchors[loc_inds] - box_deltas = _box_to_delta(sampled_anchor, sampled_gt, [1., 1., 1., 1.]) + box_deltas = _box_to_delta( + sampled_anchor, sampled_gt, [1.0, 1.0, 1.0, 1.0] + ) if i == 0: loc_indexes = loc_inds @@ -156,31 +175,43 @@ def rpn_target_assign_in_python(all_anchors, bbox_inside_weights = bbox_inside_weight else: loc_indexes = np.concatenate( - [loc_indexes, loc_inds + i * anchor_num]) + [loc_indexes, loc_inds + i * anchor_num] + ) score_indexes = np.concatenate( - [score_indexes, score_inds + i * anchor_num]) + [score_indexes, score_inds + i * anchor_num] + ) tgt_labels = np.concatenate([tgt_labels, labels]) tgt_bboxes = np.vstack([tgt_bboxes, box_deltas]) - bbox_inside_weights = np.vstack([bbox_inside_weights, \ - bbox_inside_weight]) - - return loc_indexes, score_indexes, tgt_bboxes, tgt_labels, bbox_inside_weights - - -def retinanet_target_assign(anchor_by_gt_overlap, gt_labels, positive_overlap, - negative_overlap): + bbox_inside_weights = np.vstack( + [bbox_inside_weights, bbox_inside_weight] + ) + + return ( + loc_indexes, + score_indexes, + tgt_bboxes, + tgt_labels, + bbox_inside_weights, + ) + + +def retinanet_target_assign( + anchor_by_gt_overlap, gt_labels, positive_overlap, negative_overlap +): anchor_to_gt_argmax = anchor_by_gt_overlap.argmax(axis=1) anchor_to_gt_max = anchor_by_gt_overlap[ - np.arange(anchor_by_gt_overlap.shape[0]), anchor_to_gt_argmax] + np.arange(anchor_by_gt_overlap.shape[0]), anchor_to_gt_argmax + ] gt_to_anchor_argmax = anchor_by_gt_overlap.argmax(axis=0) gt_to_anchor_max = anchor_by_gt_overlap[ - gt_to_anchor_argmax, - np.arange(anchor_by_gt_overlap.shape[1])] + gt_to_anchor_argmax, np.arange(anchor_by_gt_overlap.shape[1]) + ] anchors_with_max_overlap = np.where( - anchor_by_gt_overlap == gt_to_anchor_max)[0] + anchor_by_gt_overlap == gt_to_anchor_max + )[0] - labels = np.ones((anchor_by_gt_overlap.shape[0], ), dtype=np.int32) * -1 + labels = np.ones((anchor_by_gt_overlap.shape[0],), dtype=np.int32) * -1 labels[anchors_with_max_overlap] = 1 labels[anchor_to_gt_max >= positive_overlap] = 1 @@ -209,15 +240,22 @@ def retinanet_target_assign(anchor_by_gt_overlap, gt_labels, positive_overlap, gt_inds = anchor_to_gt_argmax[loc_index] label_inds = anchor_to_gt_argmax[score_index_tmp] - labels[0:len(fg_inds)] = np.squeeze(gt_labels[label_inds]) + labels[0 : len(fg_inds)] = np.squeeze(gt_labels[label_inds]) fg_num = len(fg_fake_inds) + len(fg_inds) + 1 assert not np.any(labels == -1), "Wrong labels with -1" return loc_index, score_index, labels, gt_inds, bbox_inside_weight, fg_num -def retinanet_target_assign_in_python(all_anchors, gt_boxes, gt_labels, - is_crowd, im_info, lod, positive_overlap, - negative_overlap): +def retinanet_target_assign_in_python( + all_anchors, + gt_boxes, + gt_labels, + is_crowd, + im_info, + lod, + positive_overlap, + negative_overlap, +): anchor_num = all_anchors.shape[0] batch_size = len(lod) - 1 for i in range(batch_size): @@ -235,16 +273,25 @@ def retinanet_target_assign_in_python(all_anchors, gt_boxes, gt_labels, gt_labels_slice = gt_labels_slice[not_crowd_inds] iou = _bbox_overlaps(inside_anchors, gt_boxes_slice) - loc_inds, score_inds, labels, gt_inds, bbox_inside_weight, fg_num = \ - retinanet_target_assign(iou, gt_labels_slice, - positive_overlap, negative_overlap) + ( + loc_inds, + score_inds, + labels, + gt_inds, + bbox_inside_weight, + fg_num, + ) = retinanet_target_assign( + iou, gt_labels_slice, positive_overlap, negative_overlap + ) # unmap to all anchor loc_inds = inds_inside[loc_inds] score_inds = inds_inside[score_inds] sampled_gt = gt_boxes_slice[gt_inds] sampled_anchor = all_anchors[loc_inds] - box_deltas = _box_to_delta(sampled_anchor, sampled_gt, [1., 1., 1., 1.]) + box_deltas = _box_to_delta( + sampled_anchor, sampled_gt, [1.0, 1.0, 1.0, 1.0] + ) if i == 0: loc_indexes = loc_inds @@ -255,20 +302,29 @@ def retinanet_target_assign_in_python(all_anchors, gt_boxes, gt_labels, fg_nums = [[fg_num]] else: loc_indexes = np.concatenate( - [loc_indexes, loc_inds + i * anchor_num]) + [loc_indexes, loc_inds + i * anchor_num] + ) score_indexes = np.concatenate( - [score_indexes, score_inds + i * anchor_num]) + [score_indexes, score_inds + i * anchor_num] + ) tgt_labels = np.concatenate([tgt_labels, labels]) tgt_bboxes = np.vstack([tgt_bboxes, box_deltas]) - bbox_inside_weights = np.vstack([bbox_inside_weights, \ - bbox_inside_weight]) + bbox_inside_weights = np.vstack( + [bbox_inside_weights, bbox_inside_weight] + ) fg_nums = np.concatenate([fg_nums, [[fg_num]]]) - return loc_indexes, score_indexes, tgt_bboxes, tgt_labels, bbox_inside_weights, fg_nums + return ( + loc_indexes, + score_indexes, + tgt_bboxes, + tgt_labels, + bbox_inside_weights, + fg_nums, + ) class TestRpnTargetAssignOp(OpTest): - def setUp(self): n, c, h, w = 2, 4, 14, 14 all_anchors = get_anchor(n, c, h, w) @@ -277,16 +333,16 @@ class TestRpnTargetAssignOp(OpTest): anchor_num = all_anchors.shape[0] images_shape = [[64, 64], [64, 64]] - #images_shape = [[64, 64]] + # images_shape = [[64, 64]] groundtruth, lod = _generate_groundtruth(images_shape, 3, 4) lod = [0, 4, 8] - #lod = [0, 4] + # lod = [0, 4] im_info = np.ones((len(images_shape), 3)).astype(np.float32) for i in range(len(images_shape)): im_info[i, 0] = images_shape[i][0] im_info[i, 1] = images_shape[i][1] - im_info[i, 2] = 0.8 #scale + im_info[i, 2] = 0.8 # scale gt_boxes = np.vstack([v['boxes'] for v in groundtruth]) is_crowd = np.hstack([v['is_crowd'] for v in groundtruth]) @@ -300,12 +356,25 @@ class TestRpnTargetAssignOp(OpTest): rpn_fg_fraction = 0.5 use_random = False - loc_index, score_index, tgt_bbox, labels, bbox_inside_weights = \ - rpn_target_assign_in_python(all_anchors, gt_boxes, is_crowd, - im_info, lod, rpn_straddle_thresh, - rpn_batch_size_per_im, rpn_positive_overlap, - rpn_negative_overlap, - rpn_fg_fraction, use_random) + ( + loc_index, + score_index, + tgt_bbox, + labels, + bbox_inside_weights, + ) = rpn_target_assign_in_python( + all_anchors, + gt_boxes, + is_crowd, + im_info, + lod, + rpn_straddle_thresh, + rpn_batch_size_per_im, + rpn_positive_overlap, + rpn_negative_overlap, + rpn_fg_fraction, + use_random, + ) labels = labels[:, np.newaxis] self.op_type = "rpn_target_assign" @@ -313,7 +382,7 @@ class TestRpnTargetAssignOp(OpTest): 'Anchor': all_anchors, 'GtBoxes': (gt_boxes, [[4, 4]]), 'IsCrowd': (is_crowd, [[4, 4]]), - 'ImInfo': (im_info, [[1, 1]]) + 'ImInfo': (im_info, [[1, 1]]), } self.attrs = { 'rpn_batch_size_per_im': rpn_batch_size_per_im, @@ -321,14 +390,14 @@ class TestRpnTargetAssignOp(OpTest): 'rpn_positive_overlap': rpn_positive_overlap, 'rpn_negative_overlap': rpn_negative_overlap, 'rpn_fg_fraction': rpn_fg_fraction, - 'use_random': use_random + 'use_random': use_random, } self.outputs = { 'LocationIndex': loc_index.astype('int32'), 'ScoreIndex': score_index.astype('int32'), 'TargetBBox': tgt_bbox.astype('float32'), 'TargetLabel': labels.astype('int32'), - 'BBoxInsideWeight': bbox_inside_weights.astype('float32') + 'BBoxInsideWeight': bbox_inside_weights.astype('float32'), } def test_check_output(self): @@ -336,7 +405,6 @@ class TestRpnTargetAssignOp(OpTest): class TestRetinanetTargetAssignOp(OpTest): - def setUp(self): n, c, h, w = 2, 4, 14, 14 all_anchors = get_anchor(n, c, h, w) @@ -352,13 +420,15 @@ class TestRetinanetTargetAssignOp(OpTest): for i in range(len(images_shape)): im_info[i, 0] = images_shape[i][0] im_info[i, 1] = images_shape[i][1] - im_info[i, 2] = 0.8 #scale + im_info[i, 2] = 0.8 # scale gt_boxes = np.vstack([v['boxes'] for v in groundtruth]) is_crowd = np.hstack([v['is_crowd'] for v in groundtruth]) - gt_labels = np.vstack([ - v['gt_classes'].reshape(len(v['gt_classes']), 1) - for v in groundtruth - ]) + gt_labels = np.vstack( + [ + v['gt_classes'].reshape(len(v['gt_classes']), 1) + for v in groundtruth + ] + ) gt_labels = gt_labels.reshape(len(gt_labels), 1) all_anchors = all_anchors.astype('float32') gt_boxes = gt_boxes.astype('float32') @@ -367,9 +437,23 @@ class TestRetinanetTargetAssignOp(OpTest): positive_overlap = 0.5 negative_overlap = 0.4 - loc_index, score_index, tgt_bbox, labels, bbox_inside_weights, fg_num = \ - retinanet_target_assign_in_python(all_anchors, gt_boxes, gt_labels, is_crowd, - im_info, lod, positive_overlap, negative_overlap) + ( + loc_index, + score_index, + tgt_bbox, + labels, + bbox_inside_weights, + fg_num, + ) = retinanet_target_assign_in_python( + all_anchors, + gt_boxes, + gt_labels, + is_crowd, + im_info, + lod, + positive_overlap, + negative_overlap, + ) labels = labels[:, np.newaxis] self.op_type = "retinanet_target_assign" self.inputs = { @@ -377,11 +461,11 @@ class TestRetinanetTargetAssignOp(OpTest): 'GtBoxes': (gt_boxes, [[4, 4]]), 'GtLabels': (gt_labels, [[4, 4]]), 'IsCrowd': (is_crowd, [[4, 4]]), - 'ImInfo': (im_info, [[1, 1]]) + 'ImInfo': (im_info, [[1, 1]]), } self.attrs = { 'positive_overlap': positive_overlap, - 'negative_overlap': negative_overlap + 'negative_overlap': negative_overlap, } self.outputs = { 'LocationIndex': loc_index.astype('int32'), @@ -389,7 +473,7 @@ class TestRetinanetTargetAssignOp(OpTest): 'TargetBBox': tgt_bbox.astype('float32'), 'TargetLabel': labels.astype('int32'), 'BBoxInsideWeight': bbox_inside_weights.astype('float32'), - 'ForegroundNumber': fg_num.astype('int32') + 'ForegroundNumber': fg_num.astype('int32'), } def test_check_output(self): @@ -397,181 +481,420 @@ class TestRetinanetTargetAssignOp(OpTest): class TestRetinanetTargetAssignOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - bbox_pred1 = fluid.data(name='bbox_pred1', - shape=[1, 100, 4], - dtype='float32') - cls_logits1 = fluid.data(name='cls_logits1', - shape=[1, 100, 10], - dtype='float32') - anchor_box1 = fluid.data(name='anchor_box1', - shape=[100, 4], - dtype='float32') - anchor_var1 = fluid.data(name='anchor_var1', - shape=[100, 4], - dtype='float32') - gt_boxes1 = fluid.data(name='gt_boxes1', - shape=[10, 4], - dtype='float32') - gt_labels1 = fluid.data(name='gt_labels1', - shape=[10, 1], - dtype='int32') + bbox_pred1 = fluid.data( + name='bbox_pred1', shape=[1, 100, 4], dtype='float32' + ) + cls_logits1 = fluid.data( + name='cls_logits1', shape=[1, 100, 10], dtype='float32' + ) + anchor_box1 = fluid.data( + name='anchor_box1', shape=[100, 4], dtype='float32' + ) + anchor_var1 = fluid.data( + name='anchor_var1', shape=[100, 4], dtype='float32' + ) + gt_boxes1 = fluid.data( + name='gt_boxes1', shape=[10, 4], dtype='float32' + ) + gt_labels1 = fluid.data( + name='gt_labels1', shape=[10, 1], dtype='int32' + ) is_crowd1 = fluid.data(name='is_crowd1', shape=[1], dtype='float32') - im_info1 = fluid.data(name='im_info1', - shape=[1, 3], - dtype='float32') + im_info1 = fluid.data( + name='im_info1', shape=[1, 3], dtype='float32' + ) # The `bbox_pred` must be Variable and the data type of `bbox_pred` Tensor # one of float32 and float64. def test_bbox_pred_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign([1], cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + [1], + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_bbox_pred_type) def test_bbox_pred_tensor_dtype(): - bbox_pred2 = fluid.data(name='bbox_pred2', - shape=[1, 100, 4], - dtype='intt32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred2, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + bbox_pred2 = fluid.data( + name='bbox_pred2', shape=[1, 100, 4], dtype='intt32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred2, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_bbox_pred_tensor_dtype) # The `cls_logits` must be Variable and the data type of `cls_logits` Tensor # one of float32 and float64. def test_cls_logits_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, 2, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + 2, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_cls_logits_type) def test_cls_logits_tensor_dtype(): - cls_logits2 = fluid.data(name='cls_logits2', - shape=[1, 100, 10], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits2, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + cls_logits2 = fluid.data( + name='cls_logits2', shape=[1, 100, 10], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits2, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_cls_logits_tensor_dtype) # The `anchor_box` must be Variable and the data type of `anchor_box` Tensor # one of float32 and float64. def test_anchor_box_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, [5], - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + [5], + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_anchor_box_type) def test_anchor_box_tensor_dtype(): - anchor_box2 = fluid.data(name='anchor_box2', - shape=[100, 4], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box2, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + anchor_box2 = fluid.data( + name='anchor_box2', shape=[100, 4], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box2, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_anchor_box_tensor_dtype) # The `anchor_var` must be Variable and the data type of `anchor_var` Tensor # one of float32 and float64. def test_anchor_var_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - 5, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + 5, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_anchor_var_type) def test_anchor_var_tensor_dtype(): - anchor_var2 = fluid.data(name='anchor_var2', - shape=[100, 4], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var2, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + anchor_var2 = fluid.data( + name='anchor_var2', shape=[100, 4], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var2, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_anchor_var_tensor_dtype) # The `gt_boxes` must be Variable and the data type of `gt_boxes` Tensor # one of float32 and float64. def test_gt_boxes_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, [4], gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + [4], + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_gt_boxes_type) def test_gt_boxes_tensor_dtype(): - gt_boxes2 = fluid.data(name='gt_boxes2', - shape=[10, 4], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes2, gt_labels1, is_crowd1, im_info1, 10) + gt_boxes2 = fluid.data( + name='gt_boxes2', shape=[10, 4], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes2, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_gt_boxes_tensor_dtype) # The `gt_label` must be Variable and the data type of `gt_label` Tensor # int32. def test_gt_label_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, 9, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + 9, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_gt_label_type) def test_gt_label_tensor_dtype(): - gt_labels2 = fluid.data(name='label2', - shape=[10, 1], - dtype='float32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels2, is_crowd1, im_info1, 10) + gt_labels2 = fluid.data( + name='label2', shape=[10, 1], dtype='float32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels2, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_gt_label_tensor_dtype) # The `is_crowd` must be Variable and the data type of `is_crowd` Tensor # int32. def test_is_crowd_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, [10], im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + [10], + im_info1, + 10, + ) self.assertRaises(TypeError, test_is_crowd_type) def test_is_crowd_tensor_dtype(): - is_crowd2 = fluid.data(name='is_crowd2', - shape=[10, 1], - dtype='float32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd2, im_info1, 10) + is_crowd2 = fluid.data( + name='is_crowd2', shape=[10, 1], dtype='float32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd2, + im_info1, + 10, + ) self.assertRaises(TypeError, test_is_crowd_tensor_dtype) # The `im_info` must be Variable and the data type of `im_info` Tensor # must be one of float32 and float64. def test_im_info_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, 1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + 1, + 10, + ) self.assertRaises(TypeError, test_im_info_type) def test_im_info_tensor_dtype(): - im_info2 = fluid.data(name='im_info2', - shape=[1, 3], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info2, 10) + im_info2 = fluid.data( + name='im_info2', shape=[1, 3], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info2, + 10, + ) self.assertRaises(TypeError, test_im_info_tensor_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_rrelu_op.py b/python/paddle/fluid/tests/unittests/test_rrelu_op.py index 6701802b443..9da00696fa3 100644 --- a/python/paddle/fluid/tests/unittests/test_rrelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_rrelu_op.py @@ -42,9 +42,8 @@ def check_output(input, output, lower, upper): class TestFunctionalRReluAPI(unittest.TestCase): - def setUp(self): - self.x_np = np.random.uniform(-1., 1., [1, 2, 3, 4]).astype('float64') + self.x_np = np.random.uniform(-1.0, 1.0, [1, 2, 3, 4]).astype('float64') self.lower_0 = 0.05 self.lower_1 = 0.1 self.upper_0 = 0.25 @@ -52,36 +51,39 @@ class TestFunctionalRReluAPI(unittest.TestCase): self.places = [ fluid.CUDAPlace(0) - if core.is_compiled_with_cuda() else fluid.CPUPlace() + if core.is_compiled_with_cuda() + else fluid.CPUPlace() ] def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 4, 5], - dtype="float32") - res1 = F.rrelu(x=input, - lower=self.lower_0, - upper=self.upper_0, - training=False) - res2 = F.rrelu(x=input, - lower=self.lower_1, - upper=self.upper_1, - training=False) - in_np = np.random.uniform(-1., 1., [2, 3, 4, 5]).astype("float32") + input = fluid.data( + name="input", shape=[2, 3, 4, 5], dtype="float32" + ) + res1 = F.rrelu( + x=input, lower=self.lower_0, upper=self.upper_0, training=False + ) + res2 = F.rrelu( + x=input, lower=self.lower_1, upper=self.upper_1, training=False + ) + in_np = np.random.uniform(-1.0, 1.0, [2, 3, 4, 5]).astype("float32") res_np1 = ref_rrelu(in_np, self.lower_0, self.upper_0) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res1]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res1], + ) np.testing.assert_allclose(fetches[0], res_np1, rtol=1e-05) res_np2 = ref_rrelu(in_np, self.lower_1, self.upper_1) - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res2]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res2], + ) np.testing.assert_allclose(fetches[0], res_np2, rtol=1e-05) def test_static(self): @@ -93,48 +95,55 @@ class TestFunctionalRReluAPI(unittest.TestCase): for place in self.places: paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=self.x_np.shape, - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=self.x_np.shape, - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=self.x_np.shape, dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=self.x_np.shape, dtype="float64" + ) out_1 = F.rrelu(x_1, self.lower_0, self.upper_0, training=False) out_2 = F.rrelu(x_2, self.lower_1, self.upper_1, training=False) out_3 = F.rrelu(x_2, self.lower_1, self.upper_1, training=True) exe = paddle.static.Executor(place=place) - res_1, = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=out_1, - use_prune=True) - res_2, = exe.run(fluid.default_main_program(), - feed={"x2": self.x_np}, - fetch_list=out_2, - use_prune=True) - res_3, = exe.run(fluid.default_main_program(), - feed={"x2": self.x_np}, - fetch_list=out_3, - use_prune=True) + (res_1,) = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=out_1, + use_prune=True, + ) + (res_2,) = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_np}, + fetch_list=out_2, + use_prune=True, + ) + (res_3,) = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_np}, + fetch_list=out_3, + use_prune=True, + ) out_ref_1 = ref_rrelu(self.x_np, self.lower_0, self.upper_0) out_ref_2 = ref_rrelu(self.x_np, self.lower_1, self.upper_1) np.testing.assert_allclose(out_ref_1, res_1, rtol=1e-05) np.testing.assert_allclose(out_ref_2, res_2, rtol=1e-05) self.assertTrue( - check_output(self.x_np, res_3[0], self.lower_1, self.upper_1)) + check_output(self.x_np, res_3[0], self.lower_1, self.upper_1) + ) def test_static_graph_layer(self): '''test_static_graph_layer''' for place in self.places: paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=self.x_np.shape, - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=self.x_np.shape, - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=self.x_np.shape, dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=self.x_np.shape, dtype="float64" + ) # init instance rrelu_1 = paddle.nn.RReLU(self.lower_0, self.upper_0) rrelu_2 = paddle.nn.RReLU(self.lower_1, self.upper_1) @@ -142,19 +151,25 @@ class TestFunctionalRReluAPI(unittest.TestCase): out_2 = rrelu_2(x_2) exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=out_1, - use_prune=True) - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=out_1, + use_prune=True, + ) + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_np}, + fetch_list=out_2, + use_prune=True, + ) self.assertTrue( - check_output(self.x_np, res_1[0], self.lower_0, self.upper_0)) + check_output(self.x_np, res_1[0], self.lower_0, self.upper_0) + ) self.assertTrue( - check_output(self.x_np, res_2[0], self.lower_1, self.upper_1)) + check_output(self.x_np, res_2[0], self.lower_1, self.upper_1) + ) def dygraph_check(self, lower, upper): for place in self.places: @@ -179,8 +194,10 @@ class TestFunctionalRReluAPI(unittest.TestCase): rrelu = paddle.nn.RReLU(self.lower_0, self.upper_0) result = rrelu(paddle.to_tensor(self.x_np)) self.assertTrue( - check_output(self.x_np, result.numpy(), self.lower_0, - self.upper_0)) + check_output( + self.x_np, result.numpy(), self.lower_0, self.upper_0 + ) + ) paddle.enable_static() def test_dygraph(self): @@ -190,67 +207,64 @@ class TestFunctionalRReluAPI(unittest.TestCase): rrelu = paddle.nn.RReLU(self.lower_0, self.upper_0) out_np = rrelu(paddle.to_tensor(self.x_np)) self.assertTrue( - check_output(self.x_np, out_np.numpy(), self.lower_0, - self.upper_0)) + check_output( + self.x_np, out_np.numpy(), self.lower_0, self.upper_0 + ) + ) paddle.enable_static() def test_error_functional(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): # The input type must be Variable. - self.assertRaises(TypeError, - F.rrelu, - x=1, - lower=self.lower_0, - upper=self.upper_0) + self.assertRaises( + TypeError, F.rrelu, x=1, lower=self.lower_0, upper=self.upper_0 + ) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') - self.assertRaises(TypeError, - F.rrelu, - x=x_int32, - lower=self.lower_0, - upper=self.upper_0) - x_bool = paddle.fluid.data(name='x_bool', - shape=[2, 3], - dtype='int32') - self.assertRaises(TypeError, - F.rrelu, - x=x_bool, - lower=self.lower_0, - upper=self.upper_0) + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) + self.assertRaises( + TypeError, + F.rrelu, + x=x_int32, + lower=self.lower_0, + upper=self.upper_0, + ) + x_bool = paddle.fluid.data( + name='x_bool', shape=[2, 3], dtype='int32' + ) + self.assertRaises( + TypeError, + F.rrelu, + x=x_bool, + lower=self.lower_0, + upper=self.upper_0, + ) # lower and upper must be float - x_fp32 = paddle.fluid.data(name='x_fp32', - shape=[2, 3], - dtype='float32') + x_fp32 = paddle.fluid.data( + name='x_fp32', shape=[2, 3], dtype='float32' + ) self.assertRaises(TypeError, F.rrelu, x=x_fp32, lower=0, upper=0.5) self.assertRaises(TypeError, F.rrelu, x=x_fp32, lower=0.5, upper=1) # lower and upper must be in (0, 1) - self.assertRaises(ValueError, - F.rrelu, - x=x_fp32, - lower=-1., - upper=0.5) - self.assertRaises(ValueError, - F.rrelu, - x=x_fp32, - lower=0.5, - upper=2.) + self.assertRaises( + ValueError, F.rrelu, x=x_fp32, lower=-1.0, upper=0.5 + ) + self.assertRaises( + ValueError, F.rrelu, x=x_fp32, lower=0.5, upper=2.0 + ) # upper should not be less than lower - self.assertRaises(ValueError, - F.rrelu, - x=x_fp32, - lower=0.5, - upper=0.2) + self.assertRaises( + ValueError, F.rrelu, x=x_fp32, lower=0.5, upper=0.2 + ) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[2, 3], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[2, 3], dtype='float16' + ) F.rrelu(x=x_fp16, lower=self.lower_0, upper=self.upper_0) def test_error_layer(self): - def error_int_dtype(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 3]).astype("float64") @@ -296,7 +310,6 @@ class TestFunctionalRReluAPI(unittest.TestCase): class RReluTest(OpTest): - def setUp(self): self.op_type = "rrelu" self.lower = 0.1 @@ -318,7 +331,7 @@ class RReluTest(OpTest): self.attrs = { 'lower': self.lower, "upper": self.upper, - "is_test": self.is_test + "is_test": self.is_test, } def test_check_output(self): @@ -329,7 +342,6 @@ class RReluTest(OpTest): class RReluTrainingTest(OpTest): - def setUp(self): self.op_type = "rrelu" self.lower = 0.3 @@ -339,7 +351,6 @@ class RReluTrainingTest(OpTest): class RReluTrainingTest(OpTest): - def setUp(self): self.op_type = "rrelu" self.lower = 0.3 diff --git a/python/paddle/fluid/tests/unittests/test_run.py b/python/paddle/fluid/tests/unittests/test_run.py index 15af313b44c..514384c0192 100644 --- a/python/paddle/fluid/tests/unittests/test_run.py +++ b/python/paddle/fluid/tests/unittests/test_run.py @@ -50,13 +50,13 @@ def write_file(name, ct): def get_files(pth, prefix): return [ - f for f in listdir(pth) + f + for f in listdir(pth) if isfile(join(pth, f)) and not f.endswith('gpu.log') ] class Collective_Test(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.path = os.path.join(self.temp_dir.name, pyname) @@ -87,7 +87,8 @@ class Collective_Test(unittest.TestCase): def test_collective_2(self): log_dir = tempfile.TemporaryDirectory() args = "--job_id test2 --devices 0,1,2 --log_dir {}".format( - log_dir.name) + log_dir.name + ) p = self.pdrun(args) p.wait() self.assertTrue(p.poll() == 0) @@ -116,7 +117,6 @@ class Collective_Test(unittest.TestCase): class PS_Test(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.path = os.path.join(self.temp_dir.name, pyname) @@ -143,8 +143,11 @@ class PS_Test(unittest.TestCase): def test_ps_2(self): log_dir = tempfile.TemporaryDirectory() - args = "--job_id ps2 --server_num=2 --trainer_num=2 --log_dir {}".format( - log_dir.name) + args = ( + "--job_id ps2 --server_num=2 --trainer_num=2 --log_dir {}".format( + log_dir.name + ) + ) p = self.pdrun(args) p.wait() self.assertTrue(p.poll() == 0) @@ -174,7 +177,8 @@ class PS_Test(unittest.TestCase): def test_ps_4(self): log_dir = tempfile.TemporaryDirectory() args = "--job_id ps4 --log_dir {} --servers 127.0.0.1:8900,127.0.0.1:8901 --trainers 127.0.0.1:8902,127.0.0.1:8903".format( - log_dir.name) + log_dir.name + ) p1 = self.pdrun(args) p1.wait() self.assertTrue(p1.poll() == 0) diff --git a/python/paddle/fluid/tests/unittests/test_run_fluid_by_module_or_command_line.py b/python/paddle/fluid/tests/unittests/test_run_fluid_by_module_or_command_line.py index d59c9db637f..df626dc6dde 100644 --- a/python/paddle/fluid/tests/unittests/test_run_fluid_by_module_or_command_line.py +++ b/python/paddle/fluid/tests/unittests/test_run_fluid_by_module_or_command_line.py @@ -18,7 +18,6 @@ import sys class TestRunFluidByModule(unittest.TestCase): - def test_module(self): print(sys.executable) res = os.system(sys.executable + ' -m "paddle.fluid.reader"') @@ -26,7 +25,6 @@ class TestRunFluidByModule(unittest.TestCase): class TestRunFluidByCommand(unittest.TestCase): - def test_command(self): res = os.system(sys.executable + ' -c "import paddle.fluid"') self.assertEqual(res, 0) # 0 means status OK diff --git a/python/paddle/fluid/tests/unittests/test_run_program_op.py b/python/paddle/fluid/tests/unittests/test_run_program_op.py index 51538266b5d..c2f871bd576 100644 --- a/python/paddle/fluid/tests/unittests/test_run_program_op.py +++ b/python/paddle/fluid/tests/unittests/test_run_program_op.py @@ -22,7 +22,10 @@ import paddle.fluid as fluid from paddle.fluid import core, framework from paddle.fluid.layers.utils import _hash_with_id from paddle.fluid.framework import _in_eager_mode_ -from paddle.fluid.executor import _is_enable_standalone_executor, _is_dy2st_enable_standalone_executor +from paddle.fluid.executor import ( + _is_enable_standalone_executor, + _is_dy2st_enable_standalone_executor, +) from paddle.fluid.dygraph.base import switch_to_static_graph paddle.enable_static() @@ -43,9 +46,11 @@ def program_scope_guard(): def _add_build_strategy_for(input_program, start_op_index, end_op_index): compiled_program = paddle.static.CompiledProgram( core.Graph(input_program.desc, start_op_index, end_op_index), - build_strategy=paddle.static.BuildStrategy()) - compiled_program._compile(core.Scope(), - paddle.framework._current_expected_place()) + build_strategy=paddle.static.BuildStrategy(), + ) + compiled_program._compile( + core.Scope(), paddle.framework._current_expected_place() + ) ir_graph = paddle.fluid.framework.IrGraph(compiled_program._graph) builded_program = ir_graph.to_program() return builded_program @@ -70,10 +75,10 @@ def _build_program_by_desc(program_desc): # when create Operator, so here compare gradients with static graph # NOTE: Here rewrite a simple unittest framework for RunProgramOp class RunProgramOpTest(unittest.TestCase): - def build_model(self): raise NotImplementedError( - "RunProgramOp test should implement build_model") + "RunProgramOp test should implement build_model" + ) def check_output(self): places = [fluid.CPUPlace()] @@ -108,9 +113,9 @@ class RunProgramOpTest(unittest.TestCase): else: fetch_list = self.get_param_grad_names() - outs = exe.run(main_program, - feed=self.inputs['X'], - fetch_list=fetch_list) + outs = exe.run( + main_program, feed=self.inputs['X'], fetch_list=fetch_list + ) return outs def get_program_desc(self): @@ -118,21 +123,28 @@ class RunProgramOpTest(unittest.TestCase): fwd_op_num = self.build_model() return fluid.default_main_program().desc, fwd_op_num - def get_forward_backward_program_desc(self, whole_program_desc, - forward_op_num, output_num): + def get_forward_backward_program_desc( + self, whole_program_desc, forward_op_num, output_num + ): program = _build_program_by_desc(whole_program_desc) forward_program = _add_build_strategy_for(program, 0, forward_op_num) backward_program = _add_build_strategy_for( - program, forward_op_num + 2 * output_num, - program.desc.block(0).op_size()) + program, + forward_op_num + 2 * output_num, + program.desc.block(0).op_size(), + ) return forward_program.desc, backward_program.desc def prepare_attrs(self): return [ 'global_block', - self.program_desc.block(0), 'start_op_index', 0, 'end_op_index', - self.fwd_op_num, 'program_id', - _hash_with_id(self.program_desc, self) + self.program_desc.block(0), + 'start_op_index', + 0, + 'end_op_index', + self.fwd_op_num, + 'program_id', + _hash_with_id(self.program_desc, self), ] def get_param_grad_names(self): @@ -147,10 +159,9 @@ class RunProgramOpTest(unittest.TestCase): # Step 2. compare output for expect_v, actual_v in zip(self.expect_outs, actual_outs): - np.testing.assert_allclose(expect_v, - actual_v.numpy(), - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + expect_v, actual_v.numpy(), rtol=1e-05, atol=1e-05 + ) def check_grad_with_place(self, place): # Step 1. calc grads @@ -159,24 +170,20 @@ class RunProgramOpTest(unittest.TestCase): # Step 2. compare grads for expect_v, actual_v in zip(self.expect_grads, actual_grads): np.testing.assert_array_almost_equal(expect_v, actual_v) - np.testing.assert_allclose(expect_v, - actual_v, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + expect_v, actual_v, rtol=1e-05, atol=1e-05 + ) def prepare_dygraph_input(self, place, return_param_list=False): - def create_var_base(is_input, name, np_value, stop_gradient): if _in_eager_mode_: - var = core.eager.Tensor(value=np_value, - name=name, - place=place, - zero_copy=True) + var = core.eager.Tensor( + value=np_value, name=name, place=place, zero_copy=True + ) else: - var = core.VarBase(value=np_value, - name=name, - place=place, - zero_copy=True) + var = core.VarBase( + value=np_value, name=name, place=place, zero_copy=True + ) var.stop_gradient = stop_gradient return var @@ -199,7 +206,6 @@ class RunProgramOpTest(unittest.TestCase): return inputs def prepare_dygraph_output(self): - def create_var_base(is_input, name): var = framework._varbase_creator(dtype=None, shape=None, name=name) var.stop_gradient = False @@ -217,7 +223,8 @@ class RunProgramOpTest(unittest.TestCase): outputs['OutScope'] = framework._varbase_creator( type=core.VarDesc.VarType.STEP_SCOPES, name="program_out_scope", - persistable=True) + persistable=True, + ) inner_scope = core.Scope() outputs['OutScope'].value().set_scope(inner_scope) @@ -232,20 +239,37 @@ class RunProgramOpTest(unittest.TestCase): inputs = self.prepare_dygraph_input(place) outputs = self.prepare_dygraph_output() - forward_program_desc, backward_program_desc = self.get_forward_backward_program_desc( - self.program_desc, self.fwd_op_num, len(outputs['Out'])) - - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + ( + forward_program_desc, + backward_program_desc, + ) = self.get_forward_backward_program_desc( + self.program_desc, self.fwd_op_num, len(outputs['Out']) + ) + + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) self.attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: self.attrs.extend( - ('forward_global_block', forward_program_desc.block(0), - 'backward_global_block', backward_program_desc.block(0))) - - _legacy_C_ops.run_program(inputs['X'], inputs['Params'], - outputs['Out'], outputs['OutScope'], - outputs['DOut'], None, *self.attrs) + ( + 'forward_global_block', + forward_program_desc.block(0), + 'backward_global_block', + backward_program_desc.block(0), + ) + ) + + _legacy_C_ops.run_program( + inputs['X'], + inputs['Params'], + outputs['Out'], + outputs['OutScope'], + outputs['DOut'], + None, + *self.attrs + ) return outputs['Out'] @@ -258,20 +282,37 @@ class RunProgramOpTest(unittest.TestCase): inputs, input_param_list = self.prepare_dygraph_input(place, True) outputs = self.prepare_dygraph_output() - forward_program_desc, backward_program_desc = self.get_forward_backward_program_desc( - self.program_desc, self.fwd_op_num, len(outputs['Out'])) - - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + ( + forward_program_desc, + backward_program_desc, + ) = self.get_forward_backward_program_desc( + self.program_desc, self.fwd_op_num, len(outputs['Out']) + ) + + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) self.attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: self.attrs.extend( - ('forward_global_block', forward_program_desc.block(0), - 'backward_global_block', backward_program_desc.block(0))) - - _legacy_C_ops.run_program(inputs['X'], inputs['Params'], - outputs['Out'], outputs['OutScope'], - outputs['DOut'], None, *self.attrs) + ( + 'forward_global_block', + forward_program_desc.block(0), + 'backward_global_block', + backward_program_desc.block(0), + ) + ) + + _legacy_C_ops.run_program( + inputs['X'], + inputs['Params'], + outputs['Out'], + outputs['OutScope'], + outputs['DOut'], + None, + *self.attrs + ) for param in input_param_list: var_type = self._get_grad_vartype(param.name) @@ -302,27 +343,29 @@ class RunProgramOpTest(unittest.TestCase): class TestRunProgramOpWithFC(RunProgramOpTest): - def setUp(self): self.op_type = "run_program" self.dtype = np.float32 self.input_names = { 'X': ['img'], - 'Params': ['weight_param', 'bias_param'] + 'Params': ['weight_param', 'bias_param'], } self.output_names = {'Out': ['fc_0.tmp_2']} self.inputs = { 'X': { - self.input_names['X'][0]: - np.random.random((32, 1, 28, 28)).astype(self.dtype) + self.input_names['X'][0]: np.random.random( + (32, 1, 28, 28) + ).astype(self.dtype) }, 'Params': { - self.input_names['Params'][0]: - np.random.random((784, 10)).astype(self.dtype), - self.input_names['Params'][1]: - np.random.random((32, 10)).astype(self.dtype) - } + self.input_names['Params'][0]: np.random.random( + (784, 10) + ).astype(self.dtype), + self.input_names['Params'][1]: np.random.random( + (32, 10) + ).astype(self.dtype), + }, } def test_check_output(self): @@ -333,26 +376,34 @@ class TestRunProgramOpWithFC(RunProgramOpTest): def build_model(self): # 1. simple model - img = fluid.data(name=self.input_names['X'][0], - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name=self.input_names['X'][0], + shape=[None, 1, 28, 28], + dtype='float32', + ) weight_attr = fluid.ParamAttr( name=self.input_names['Params'][0], learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][0]]), - trainable=True) + self.inputs['Params'][self.input_names['Params'][0]] + ), + trainable=True, + ) bias_attr = fluid.ParamAttr( name=self.input_names['Params'][1], learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][1]]), - trainable=True) - pred = fluid.layers.fc(input=img, - size=10, - param_attr=weight_attr, - bias_attr=bias_attr, - act='relu') + self.inputs['Params'][self.input_names['Params'][1]] + ), + trainable=True, + ) + pred = fluid.layers.fc( + input=img, + size=10, + param_attr=weight_attr, + bias_attr=bias_attr, + act='relu', + ) # 2. get forward op num fwd_op_num = fluid.default_main_program().global_block().desc.op_size() # 3. append backward @@ -362,7 +413,6 @@ class TestRunProgramOpWithFC(RunProgramOpTest): class TestRunProgramOpWithEmbedding(RunProgramOpTest): - def setUp(self): self.op_type = "run_program" self.dtype = np.float32 @@ -370,12 +420,10 @@ class TestRunProgramOpWithEmbedding(RunProgramOpTest): self.output_names = {'Out': ['reduce_sum_0.tmp_0']} self.inputs = { - 'X': { - 'x': np.array([[1, 3, 0, 4, 7]]).astype("int64") - }, + 'X': {'x': np.array([[1, 3, 0, 4, 7]]).astype("int64")}, 'Params': { 'emb_weight': np.random.random(size=(10, 16)).astype("float32") - } + }, } def test_check_output(self): @@ -393,9 +441,9 @@ class TestRunProgramOpWithEmbedding(RunProgramOpTest): def build_model(self): # 1. simple model - x = fluid.layers.data(name=self.input_names['X'][0], - shape=[5], - dtype='int64') + x = fluid.layers.data( + name=self.input_names['X'][0], shape=[5], dtype='int64' + ) emb = fluid.input.embedding( input=x, size=[10, 16], @@ -403,8 +451,11 @@ class TestRunProgramOpWithEmbedding(RunProgramOpTest): name="emb_weight", learning_rate=10, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][0]])), - is_sparse=True) + self.inputs['Params'][self.input_names['Params'][0]] + ), + ), + is_sparse=True, + ) y = fluid.layers.reduce_sum(emb, dim=-1) # 2. get forward op num fwd_op_num = fluid.default_main_program().global_block().desc.op_size() @@ -415,7 +466,6 @@ class TestRunProgramOpWithEmbedding(RunProgramOpTest): class Net(paddle.nn.Layer): - def __init__(self): super(Net, self).__init__() self.fc1 = paddle.nn.Linear(10, 10) @@ -429,7 +479,6 @@ class Net(paddle.nn.Layer): class TestParametersWithStopGradient(unittest.TestCase): - def setUp(self): self.seed = 2021 self.iter = 5 diff --git a/python/paddle/fluid/tests/unittests/test_runtime_and_compiletime_exception.py b/python/paddle/fluid/tests/unittests/test_runtime_and_compiletime_exception.py index 36ccb8e0322..d498035327b 100644 --- a/python/paddle/fluid/tests/unittests/test_runtime_and_compiletime_exception.py +++ b/python/paddle/fluid/tests/unittests/test_runtime_and_compiletime_exception.py @@ -19,7 +19,6 @@ import paddle.fluid as fluid class TestRunTimeException(unittest.TestCase): - def test_run_time_exception(self): place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -38,7 +37,6 @@ class TestRunTimeException(unittest.TestCase): class TestCompileTimeException(unittest.TestCase): - def test_compile_time_exception(self): self.assertRaises(ValueError, self.build_model) @@ -46,10 +44,9 @@ class TestCompileTimeException(unittest.TestCase): train_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(train_program, startup_program): - label = fluid.layers.data(name="label", - shape=[1], - dtype="int64", - append_batch_size=False) + label = fluid.layers.data( + name="label", shape=[1], dtype="int64", append_batch_size=False + ) fluid.layers.one_hot(input=label, depth=100) diff --git a/python/paddle/fluid/tests/unittests/test_sample_logits_op.py b/python/paddle/fluid/tests/unittests/test_sample_logits_op.py index 749a32978be..8676d8ff6e5 100644 --- a/python/paddle/fluid/tests/unittests/test_sample_logits_op.py +++ b/python/paddle/fluid/tests/unittests/test_sample_logits_op.py @@ -19,7 +19,6 @@ from op_test import OpTest class TestSampleLogitsOp(OpTest): - def setUp(self): self.op_type = "sample_logits" self.dtype = np.float64 @@ -46,33 +45,49 @@ class TestSampleLogitsOp(OpTest): self.inputs = {"Logits": self.Logits, "Labels": self.Labels} self.fetch_list = [ - 'Samples', 'Probabilities', 'SampledLogits', 'SampledLabels' + 'Samples', + 'Probabilities', + 'SampledLogits', + 'SampledLabels', ] self.outputs = collections.OrderedDict( - (('Samples', Samples), ('Probabilities', Probabilities), - ('LogitsDim', LogitsDim), ('LabelsDim', LabelsDim), - ('SampledLogits', SampledLogits), ('SampledLabels', - SampledLabels))) + ( + ('Samples', Samples), + ('Probabilities', Probabilities), + ('LogitsDim', LogitsDim), + ('LabelsDim', LabelsDim), + ('SampledLogits', SampledLogits), + ('SampledLabels', SampledLabels), + ) + ) self.attrs = {'num_samples': self.S} def test_check_output(self): places = self._get_places() for p in places: - (Samples, Probabilities, SampledLogits, - SampledLabels) = [np.array(o) for o in self.calc_output(p)] - - assert Samples.dtype == np.int64, \ - "Samples dtype is {}, not int64".format(Samples.dtype) - assert Probabilities.dtype == np.float64, \ - "Probabilities dtype is {}, not float64".format( - Probabilities.dtype) - assert SampledLogits.dtype == np.float64, \ - "SampledLogits dtype is {}, not float64".format( - SampledLogits.dtype) - assert SampledLabels.dtype == np.int64, \ - "SampledLabels dtype is {}, not int64".format( - SampledLabels.dtype) + (Samples, Probabilities, SampledLogits, SampledLabels) = [ + np.array(o) for o in self.calc_output(p) + ] + + assert ( + Samples.dtype == np.int64 + ), "Samples dtype is {}, not int64".format(Samples.dtype) + assert ( + Probabilities.dtype == np.float64 + ), "Probabilities dtype is {}, not float64".format( + Probabilities.dtype + ) + assert ( + SampledLogits.dtype == np.float64 + ), "SampledLogits dtype is {}, not float64".format( + SampledLogits.dtype + ) + assert ( + SampledLabels.dtype == np.int64 + ), "SampledLabels dtype is {}, not int64".format( + SampledLabels.dtype + ) assert Samples.shape == (self.bs, self.NT + self.S) assert Probabilities.shape == (self.bs, self.NT + self.S) @@ -80,27 +95,26 @@ class TestSampleLogitsOp(OpTest): assert SampledLabels.shape == (self.bs, self.NT) assert (SampledLabels == self.Labels).all() - sampled_logits = self.Logits[:, Samples[0][:self.NT]] - sampled_logits -= np.log(Probabilities[:, :self.NT]) - np.testing.assert_almost_equal(sampled_logits, - SampledLogits[:, :self.NT]) + sampled_logits = self.Logits[:, Samples[0][: self.NT]] + sampled_logits -= np.log(Probabilities[:, : self.NT]) + np.testing.assert_almost_equal( + sampled_logits, SampledLogits[:, : self.NT] + ) def test_check_grad(self): self._check_grad_helper() for p in self._get_places(): grads = self._get_gradient(['Logits'], p, ['SampledLogits'], []) - np.testing.assert_almost_equal(grads[0].sum(), np.array([1.])) + np.testing.assert_almost_equal(grads[0].sum(), np.array([1.0])) class TestSampleLogitsOpNoUniq(TestSampleLogitsOp): - def setUp(self): super(TestSampleLogitsOpNoUniq, self).setUp() self.attrs = {'num_samples': self.S, 'uniq': False} class TestSampleLogitsOpWithAccidentalHits(TestSampleLogitsOp): - def setUp(self): super(TestSampleLogitsOpWithAccidentalHits, self).setUp() self.attrs = {'num_samples': self.S, 'remove_accidental_hits': False} diff --git a/python/paddle/fluid/tests/unittests/test_sampling_id_op.py b/python/paddle/fluid/tests/unittests/test_sampling_id_op.py index 3481db22907..2e73a90abdb 100644 --- a/python/paddle/fluid/tests/unittests/test_sampling_id_op.py +++ b/python/paddle/fluid/tests/unittests/test_sampling_id_op.py @@ -20,7 +20,6 @@ import paddle class TestSamplingIdShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() x = fluid.layers.data(name='x', shape=[3], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_save_inference_model_conditional_op.py b/python/paddle/fluid/tests/unittests/test_save_inference_model_conditional_op.py index 240c0c8e129..399be5ee60d 100644 --- a/python/paddle/fluid/tests/unittests/test_save_inference_model_conditional_op.py +++ b/python/paddle/fluid/tests/unittests/test_save_inference_model_conditional_op.py @@ -28,14 +28,13 @@ def getModelOp(model_path): result = set() for i in range(0, size): - #print(main_block.op(i).type()) + # print(main_block.op(i).type()) result.add(main_block.op(i).type()) return result class WhileNet(paddle.nn.Layer): - def __init__(self): super(WhileNet, self).__init__() @@ -53,7 +52,6 @@ class WhileNet(paddle.nn.Layer): class ForNet(paddle.nn.Layer): - def __init__(self): super(ForNet, self).__init__() @@ -67,7 +65,6 @@ class ForNet(paddle.nn.Layer): class IfElseNet(paddle.nn.Layer): - def __init__(self): super(IfElseNet, self).__init__() @@ -81,68 +78,91 @@ class IfElseNet(paddle.nn.Layer): class TestConditionalOp(unittest.TestCase): - def test_while_op(self): paddle.disable_static() net = WhileNet() - net = paddle.jit.to_static(net, - input_spec=[ - paddle.static.InputSpec( - shape=[1, 3, 8, 8], dtype='float32') - ]) + net = paddle.jit.to_static( + net, + input_spec=[ + paddle.static.InputSpec(shape=[1, 3, 8, 8], dtype='float32') + ], + ) root_path = tempfile.TemporaryDirectory() model_file = os.path.join(root_path.name, "while_net") paddle.jit.save(net, model_file) - right_pdmodel = set([ - "uniform_random", "shape", "slice", "not_equal", "while", - "elementwise_add" - ]) + right_pdmodel = set( + [ + "uniform_random", + "shape", + "slice", + "not_equal", + "while", + "elementwise_add", + ] + ) paddle.enable_static() pdmodel = getModelOp(model_file + ".pdmodel") self.assertTrue( len(right_pdmodel.difference(pdmodel)) == 0, - "The while op is pruned by mistake.") + "The while op is pruned by mistake.", + ) root_path.cleanup() def test_for_op(self): paddle.disable_static() net = ForNet() net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec(shape=[1], dtype='int32')]) + net, input_spec=[paddle.static.InputSpec(shape=[1], dtype='int32')] + ) root_path = tempfile.TemporaryDirectory() model_file = os.path.join(root_path.name, "for_net") paddle.jit.save(net, model_file) - right_pdmodel = set([ - "randint", "fill_constant", "cast", "less_than", "while", - "elementwise_add" - ]) + right_pdmodel = set( + [ + "randint", + "fill_constant", + "cast", + "less_than", + "while", + "elementwise_add", + ] + ) paddle.enable_static() pdmodel = getModelOp(model_file + ".pdmodel") self.assertTrue( len(right_pdmodel.difference(pdmodel)) == 0, - "The for op is pruned by mistake.") + "The for op is pruned by mistake.", + ) root_path.cleanup() def test_if_op(self): paddle.disable_static() net = IfElseNet() net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec(shape=[1], dtype='int32')]) + net, input_spec=[paddle.static.InputSpec(shape=[1], dtype='int32')] + ) root_path = tempfile.TemporaryDirectory() model_file = os.path.join(root_path.name, "if_net") paddle.jit.save(net, model_file) - right_pdmodel = set([ - "assign_value", "greater_than", "cast", "conditional_block", - "logical_not", "select_input" - ]) + right_pdmodel = set( + [ + "assign_value", + "greater_than", + "cast", + "conditional_block", + "logical_not", + "select_input", + ] + ) paddle.enable_static() pdmodel = getModelOp(model_file + ".pdmodel") self.assertTrue( len(right_pdmodel.difference(pdmodel)) == 0, - "The if op is pruned by mistake.") + "The if op is pruned by mistake.", + ) root_path.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_save_model_without_var.py b/python/paddle/fluid/tests/unittests/test_save_model_without_var.py index e8cad315143..ac02edc42e7 100644 --- a/python/paddle/fluid/tests/unittests/test_save_model_without_var.py +++ b/python/paddle/fluid/tests/unittests/test_save_model_without_var.py @@ -18,12 +18,10 @@ import paddle.fluid as fluid class TestSaveModelWithoutVar(unittest.TestCase): - def test_no_var_save(self): - data = fluid.layers.data(name='data', - shape=[-1, 1], - dtype='float32', - append_batch_size=False) + data = fluid.layers.data( + name='data', shape=[-1, 1], dtype='float32', append_batch_size=False + ) data_plus = data + 1 if fluid.core.is_compiled_with_cuda(): @@ -37,12 +35,14 @@ class TestSaveModelWithoutVar(unittest.TestCase): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - fluid.io.save_inference_model(dirname='test', - feeded_var_names=['data'], - target_vars=[data_plus], - executor=exe, - model_filename='model', - params_filename='params') + fluid.io.save_inference_model( + dirname='test', + feeded_var_names=['data'], + target_vars=[data_plus], + executor=exe, + model_filename='model', + params_filename='params', + ) expected_warn = "no variable in your model, please ensure there are any variables in your model to save" self.assertTrue(len(w) > 0) self.assertTrue(expected_warn == str(w[-1].message)) diff --git a/python/paddle/fluid/tests/unittests/test_saved_tensors_hooks.py b/python/paddle/fluid/tests/unittests/test_saved_tensors_hooks.py index 02b9d146fe3..c945eee068a 100644 --- a/python/paddle/fluid/tests/unittests/test_saved_tensors_hooks.py +++ b/python/paddle/fluid/tests/unittests/test_saved_tensors_hooks.py @@ -18,9 +18,7 @@ from paddle.autograd import PyLayer class TestSavedTensorsHooks(unittest.TestCase): - def test_save_for_multiply(self): - def pack_hook(x): return x.numpy() @@ -46,9 +44,7 @@ class TestSavedTensorsHooks(unittest.TestCase): self.assertTrue(paddle.equal_all(bb.grad, b.grad)) def test_save_for_pylayer(self): - class cus_multiply(PyLayer): - @staticmethod def forward(ctx, a, b): y = paddle.multiply(a, b) diff --git a/python/paddle/fluid/tests/unittests/test_scale_op.py b/python/paddle/fluid/tests/unittests/test_scale_op.py index 6eb4d90d610..4fec8990537 100644 --- a/python/paddle/fluid/tests/unittests/test_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_scale_op.py @@ -26,7 +26,6 @@ import paddle.fluid.layers as layers class TestScaleOp(OpTest): - def setUp(self): self.op_type = "scale" self.python_api = paddle.scale @@ -49,7 +48,6 @@ class TestScaleOp(OpTest): class TestScaleOpScaleVariable(OpTest): - def setUp(self): self.op_type = "scale" self.python_api = paddle.scale @@ -58,7 +56,7 @@ class TestScaleOpScaleVariable(OpTest): self.scale = -2.3 self.inputs = { 'X': np.random.random((10, 10)).astype(self.dtype), - 'ScaleTensor': np.array([self.scale]).astype('float64') + 'ScaleTensor': np.array([self.scale]).astype('float64'), } self.attrs = {} self.outputs = {'Out': self.inputs['X'] * self.dtype(self.scale)} @@ -74,7 +72,6 @@ class TestScaleOpScaleVariable(OpTest): class TestScaleOpSelectedRows(unittest.TestCase): - def init_dtype_type(self): pass @@ -93,8 +90,9 @@ class TestScaleOpSelectedRows(unittest.TestCase): in_selected_rows = scope.var(in_name).get_selected_rows() in_selected_rows.set_height(in_height) in_selected_rows.set_rows(in_rows) - in_array = np.random.random( - (len(in_rows), in_row_numel)).astype(self.dtype) + in_array = np.random.random((len(in_rows), in_row_numel)).astype( + self.dtype + ) in_tensor = in_selected_rows.get_tensor() in_tensor.set(in_array, place) @@ -133,9 +131,7 @@ class TestScaleOpSelectedRows(unittest.TestCase): class TestScaleRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.scale([10]) @@ -143,10 +139,10 @@ class TestScaleRaiseError(unittest.TestCase): # Add FP16 test -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScaleFp16Op(TestScaleOp): - def init_dtype_type(self): self.dtype = np.float16 @@ -158,14 +154,12 @@ class TestScaleFp16Op(TestScaleOp): def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ["X"], - "Out", - max_relative_error=0.05, - check_eager=True) + self.check_grad_with_place( + place, ["X"], "Out", max_relative_error=0.05, check_eager=True + ) class TestScaleBF16Op(OpTest): - def setUp(self): self.op_type = "scale" self.python_api = paddle.scale @@ -183,10 +177,10 @@ class TestScaleBF16Op(OpTest): self.check_grad(['X'], 'Out', numeric_grad_delta=0.8, check_eager=True) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScaleFp16OpSelectedRows(TestScaleOpSelectedRows): - def init_dtype_type(self): self.dtype = np.float16 @@ -202,7 +196,6 @@ class TestScaleFp16OpSelectedRows(TestScaleOpSelectedRows): class TestScaleApiStatic(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -220,13 +213,11 @@ class TestScaleApiStatic(unittest.TestCase): class TestScaleInplaceApiStatic(TestScaleApiStatic): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) class TestScaleApiDygraph(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -240,13 +231,11 @@ class TestScaleApiDygraph(unittest.TestCase): class TestScaleInplaceApiDygraph(TestScaleApiDygraph): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) class TestScaleDoubleGradCheck(unittest.TestCase): - def scale_wrapper(self, x): return paddle.scale(x[0], scale=2.0) @@ -261,17 +250,13 @@ class TestScaleDoubleGradCheck(unittest.TestCase): out = paddle.scale(data, 2.0) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.scale_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.scale_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -283,7 +268,6 @@ class TestScaleDoubleGradCheck(unittest.TestCase): class TestScaleTripleGradCheck(unittest.TestCase): - def scale_wrapper(self, x): return paddle.scale(x[0], scale=2.0) @@ -298,17 +282,13 @@ class TestScaleTripleGradCheck(unittest.TestCase): out = paddle.scale(data, 2.0) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.scale_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.scale_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_scaled_dot_product_attention.py b/python/paddle/fluid/tests/unittests/test_scaled_dot_product_attention.py index 031b45d3dd9..7f702482440 100644 --- a/python/paddle/fluid/tests/unittests/test_scaled_dot_product_attention.py +++ b/python/paddle/fluid/tests/unittests/test_scaled_dot_product_attention.py @@ -19,83 +19,89 @@ from paddle.fluid import Program, program_guard class TestScaledDotProductAttentionError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - queries = fluid.data(name="queries", - shape=[3, 5, 9], - dtype="float32") + queries = fluid.data( + name="queries", shape=[3, 5, 9], dtype="float32" + ) keys = fluid.data(name="keys", shape=[3, 6, 9], dtype="float32") - values = fluid.data(name="values", - shape=[3, 6, 10], - dtype="float32") + values = fluid.data( + name="values", shape=[3, 6, 10], dtype="float32" + ) def test_queries_Variable(): queries_data = np.random.rand(3, 5, 9).astype("float32") - fluid.nets.scaled_dot_product_attention(queries_data, keys, - values) + fluid.nets.scaled_dot_product_attention( + queries_data, keys, values + ) self.assertRaises(TypeError, test_queries_Variable) def test_keys_Variable(): keys_data = np.random.rand(3, 6, 9).astype("float32") - fluid.nets.scaled_dot_product_attention(queries, keys_data, - values) + fluid.nets.scaled_dot_product_attention( + queries, keys_data, values + ) self.assertRaises(TypeError, test_keys_Variable) def test_values_Variable(): values_data = np.random.rand(3, 6, 10).astype("float32") - fluid.nets.scaled_dot_product_attention(queries, keys, - values_data) + fluid.nets.scaled_dot_product_attention( + queries, keys, values_data + ) self.assertRaises(TypeError, test_values_Variable) def test_diff_dtype(): - keys_error = fluid.data(name="keys_error", - shape=[3, 6, 9], - dtype="float64") - values_error = fluid.data(name="values_error", - shape=[3, 6, 10], - dtype="float64") - fluid.nets.scaled_dot_product_attention(queries, keys_error, - values_error) + keys_error = fluid.data( + name="keys_error", shape=[3, 6, 9], dtype="float64" + ) + values_error = fluid.data( + name="values_error", shape=[3, 6, 10], dtype="float64" + ) + fluid.nets.scaled_dot_product_attention( + queries, keys_error, values_error + ) self.assertRaises(TypeError, test_diff_dtype) def test_diff_dim(): - keys_error_dim = fluid.data(name="keys_error_dim", - shape=[3, 6], - dtype="float32") - values_error_dim = fluid.data(name="values_error_dim", - shape=[3], - dtype="float32") - fluid.nets.scaled_dot_product_attention(queries, keys_error_dim, - values_error_dim) + keys_error_dim = fluid.data( + name="keys_error_dim", shape=[3, 6], dtype="float32" + ) + values_error_dim = fluid.data( + name="values_error_dim", shape=[3], dtype="float32" + ) + fluid.nets.scaled_dot_product_attention( + queries, keys_error_dim, values_error_dim + ) self.assertRaises(ValueError, test_diff_dim) def test_diff_hidden_size(): - queries_error_hs = fluid.data(name="queries_error_hs", - shape=[3, 5, 9], - dtype="float32") - keys_error_hs = fluid.data(name="keys_error_hs", - shape=[3, 6, 10], - dtype="float32") - fluid.nets.scaled_dot_product_attention(queries_error_hs, - keys_error_hs, values) + queries_error_hs = fluid.data( + name="queries_error_hs", shape=[3, 5, 9], dtype="float32" + ) + keys_error_hs = fluid.data( + name="keys_error_hs", shape=[3, 6, 10], dtype="float32" + ) + fluid.nets.scaled_dot_product_attention( + queries_error_hs, keys_error_hs, values + ) self.assertRaises(ValueError, test_diff_hidden_size) def test_diff_max_len(): - keys_error_len = fluid.data(name="keys_error_len", - shape=[3, 7, 9], - dtype="float32") - values_error_len = fluid.data(name="values_error_len", - shape=[3, 6, 10], - dtype="float32") - fluid.nets.scaled_dot_product_attention(queries, keys_error_len, - values_error_len) + keys_error_len = fluid.data( + name="keys_error_len", shape=[3, 7, 9], dtype="float32" + ) + values_error_len = fluid.data( + name="values_error_len", shape=[3, 6, 10], dtype="float32" + ) + fluid.nets.scaled_dot_product_attention( + queries, keys_error_len, values_error_len + ) self.assertRaises(ValueError, test_diff_max_len) diff --git a/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py b/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py index b9b0deb3b36..e9edd089f5d 100644 --- a/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py +++ b/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py @@ -115,8 +115,9 @@ class TestScatterNdAddWithHighRankSame(OpTest): self.python_api = paddle.scatter_nd_add shape = (3, 2, 2, 1, 10) ref_np = np.random.rand(*shape).astype("float64") - index_np = np.vstack([np.random.randint(0, s, size=100) - for s in shape]).T.astype("int32") + index_np = np.vstack( + [np.random.randint(0, s, size=100) for s in shape] + ).T.astype("int32") update_shape = judge_update_shape(ref_np, index_np) updates_np = np.random.rand(*update_shape).astype("float64") expect_np = numpy_scatter_nd_add(ref_np.copy(), index_np, updates_np) @@ -157,71 +158,89 @@ class TestScatterNdAddWithHighRankDiff(OpTest): self.check_grad(['X', 'Updates'], 'Out', check_eager=True) -#Test Python API +# Test Python API class TestScatterNdOpAPI(unittest.TestCase): """ test scatter_nd_add api and scatter_nd api """ def testcase1(self): - ref1 = fluid.layers.data(name='ref1', - shape=[10, 9, 8, 1, 3], - dtype='float32', - append_batch_size=False) - index1 = fluid.layers.data(name='index1', - shape=[5, 5, 8, 5], - dtype='int32', - append_batch_size=False) - updates1 = fluid.layers.data(name='update1', - shape=[5, 5, 8], - dtype='float32', - append_batch_size=False) + ref1 = fluid.layers.data( + name='ref1', + shape=[10, 9, 8, 1, 3], + dtype='float32', + append_batch_size=False, + ) + index1 = fluid.layers.data( + name='index1', + shape=[5, 5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates1 = fluid.layers.data( + name='update1', + shape=[5, 5, 8], + dtype='float32', + append_batch_size=False, + ) output1 = fluid.layers.scatter_nd_add(ref1, index1, updates1) def testcase2(self): - ref2 = fluid.layers.data(name='ref2', - shape=[10, 9, 8, 1, 3], - dtype='double', - append_batch_size=False) - index2 = fluid.layers.data(name='index2', - shape=[5, 8, 5], - dtype='int32', - append_batch_size=False) - updates2 = fluid.layers.data(name='update2', - shape=[5, 8], - dtype='double', - append_batch_size=False) - output2 = fluid.layers.scatter_nd_add(ref2, - index2, - updates2, - name="scatter_nd_add") + ref2 = fluid.layers.data( + name='ref2', + shape=[10, 9, 8, 1, 3], + dtype='double', + append_batch_size=False, + ) + index2 = fluid.layers.data( + name='index2', + shape=[5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates2 = fluid.layers.data( + name='update2', + shape=[5, 8], + dtype='double', + append_batch_size=False, + ) + output2 = fluid.layers.scatter_nd_add( + ref2, index2, updates2, name="scatter_nd_add" + ) def testcase3(self): shape3 = [10, 9, 8, 1, 3] - index3 = fluid.layers.data(name='index3', - shape=[5, 5, 8, 5], - dtype='int32', - append_batch_size=False) - updates3 = fluid.layers.data(name='update3', - shape=[5, 5, 8], - dtype='float32', - append_batch_size=False) + index3 = fluid.layers.data( + name='index3', + shape=[5, 5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates3 = fluid.layers.data( + name='update3', + shape=[5, 5, 8], + dtype='float32', + append_batch_size=False, + ) output3 = fluid.layers.scatter_nd(index3, updates3, shape3) def testcase4(self): shape4 = [10, 9, 8, 1, 3] - index4 = fluid.layers.data(name='index4', - shape=[5, 5, 8, 5], - dtype='int32', - append_batch_size=False) - updates4 = fluid.layers.data(name='update4', - shape=[5, 5, 8], - dtype='double', - append_batch_size=False) - output4 = fluid.layers.scatter_nd(index4, - updates4, - shape4, - name='scatter_nd') + index4 = fluid.layers.data( + name='index4', + shape=[5, 5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates4 = fluid.layers.data( + name='update4', + shape=[5, 5, 8], + dtype='double', + append_batch_size=False, + ) + output4 = fluid.layers.scatter_nd( + index4, updates4, shape4, name='scatter_nd' + ) def testcase5(self): if not fluid.core.is_compiled_with_cuda(): @@ -235,27 +254,32 @@ class TestScatterNdOpAPI(unittest.TestCase): with fluid.dygraph.guard(): device = paddle.get_device() paddle.set_device('gpu') - gpu_value = paddle.scatter_nd_add(paddle.to_tensor(x), - paddle.to_tensor(index), - paddle.to_tensor(val)) + gpu_value = paddle.scatter_nd_add( + paddle.to_tensor(x), + paddle.to_tensor(index), + paddle.to_tensor(val), + ) paddle.set_device('cpu') - cpu_value = paddle.scatter_nd_add(paddle.to_tensor(x), - paddle.to_tensor(index), - paddle.to_tensor(val)) + cpu_value = paddle.scatter_nd_add( + paddle.to_tensor(x), + paddle.to_tensor(index), + paddle.to_tensor(val), + ) np.testing.assert_array_equal(gpu_value.numpy(), cpu_value.numpy()) paddle.set_device(device) @switch_to_static_graph def test_static_graph(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_t = paddle.static.data(name="x", dtype=x.dtype, shape=x.shape) - index_t = paddle.static.data(name="index", - dtype=index.dtype, - shape=index.shape) - val_t = paddle.static.data(name="val", - dtype=val.dtype, - shape=val.shape) + index_t = paddle.static.data( + name="index", dtype=index.dtype, shape=index.shape + ) + val_t = paddle.static.data( + name="val", dtype=val.dtype, shape=val.shape + ) out_t = paddle.scatter_nd_add(x_t, index_t, val_t) feed = {x_t.name: x, index_t.name: index, val_t.name: val} fetch = [out_t] @@ -269,26 +293,23 @@ class TestScatterNdOpAPI(unittest.TestCase): test_static_graph() -#Test Raise Error +# Test Raise Error class TestScatterNdOpRaise(unittest.TestCase): - def test_check_raise(self): - def check_raise_is_test(): try: - ref5 = fluid.layers.data(name='ref5', - shape=[3, 4, 5], - dtype='float32') - index5 = fluid.layers.data(name='index5', - shape=[2, 10], - dtype='int32') - updates5 = fluid.layers.data(name='updates5', - shape=[2, 10], - dtype='float32') + ref5 = fluid.layers.data( + name='ref5', shape=[3, 4, 5], dtype='float32' + ) + index5 = fluid.layers.data( + name='index5', shape=[2, 10], dtype='int32' + ) + updates5 = fluid.layers.data( + name='updates5', shape=[2, 10], dtype='float32' + ) output5 = fluid.layers.scatter_nd_add(ref5, index5, updates5) except Exception as e: - t = \ - "The last dimension of Input(Index)'s shape should be no greater " + t = "The last dimension of Input(Index)'s shape should be no greater " if t in str(e): raise IndexError @@ -296,35 +317,39 @@ class TestScatterNdOpRaise(unittest.TestCase): def test_check_raise2(self): with self.assertRaises(ValueError): - ref6 = fluid.layers.data(name='ref6', - shape=[10, 9, 8, 1, 3], - dtype='double', - append_batch_size=False) - index6 = fluid.layers.data(name='index6', - shape=[5, 8, 5], - dtype='int32', - append_batch_size=False) - updates6 = fluid.layers.data(name='update6', - shape=[5, 8], - dtype='float32', - append_batch_size=False) + ref6 = fluid.layers.data( + name='ref6', + shape=[10, 9, 8, 1, 3], + dtype='double', + append_batch_size=False, + ) + index6 = fluid.layers.data( + name='index6', + shape=[5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates6 = fluid.layers.data( + name='update6', + shape=[5, 8], + dtype='float32', + append_batch_size=False, + ) output6 = fluid.layers.scatter_nd_add(ref6, index6, updates6) def test_check_raise3(self): - def check_raise_is_test(): try: shape = [3, 4, 5] - index7 = fluid.layers.data(name='index7', - shape=[2, 1], - dtype='int32') - updates7 = fluid.layers.data(name='updates7', - shape=[2, 4, 5, 20], - dtype='float32') + index7 = fluid.layers.data( + name='index7', shape=[2, 1], dtype='int32' + ) + updates7 = fluid.layers.data( + name='updates7', shape=[2, 4, 5, 20], dtype='float32' + ) output7 = fluid.layers.scatter_nd(index7, updates7, shape) except Exception as e: - t = \ - "Updates has wrong shape" + t = "Updates has wrong shape" if t in str(e): raise ValueError @@ -332,7 +357,6 @@ class TestScatterNdOpRaise(unittest.TestCase): class TestDygraph(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(fluid.CPUPlace()): index_data = np.array([[1, 1], [0, 1], [1, 3]]).astype(np.int64) diff --git a/python/paddle/fluid/tests/unittests/test_scatter_op.py b/python/paddle/fluid/tests/unittests/test_scatter_op.py index 05aba63fbfe..479498bfbb4 100644 --- a/python/paddle/fluid/tests/unittests/test_scatter_op.py +++ b/python/paddle/fluid/tests/unittests/test_scatter_op.py @@ -23,7 +23,6 @@ from paddle.fluid.dygraph.base import switch_to_static_graph class TestScatterOp(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -43,7 +42,6 @@ class TestScatterOp(OpTest): class TestScatterOp0(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -64,7 +62,6 @@ class TestScatterOp0(OpTest): class TestScatterOp1(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -87,10 +84,10 @@ class TestScatterOp1(OpTest): self.check_grad(["X", "Updates"], "Out", check_eager=False) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScatterOp2(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -110,15 +107,15 @@ class TestScatterOp2(OpTest): def test_check_grad(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X', 'Updates'], - 'Out', - check_eager=False) + self.check_grad_with_place( + place, ['X', 'Updates'], 'Out', check_eager=False + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScatterOp3(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -142,13 +139,12 @@ class TestScatterOp3(OpTest): def test_check_grad(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X', 'Updates'], - 'Out', - check_eager=False) + self.check_grad_with_place( + place, ['X', 'Updates'], 'Out', check_eager=False + ) class TestScatterOp4(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -167,10 +163,10 @@ class TestScatterOp4(OpTest): self.check_grad(['X', 'Updates'], 'Out', check_eager=False) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScatterOp5(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -190,13 +186,12 @@ class TestScatterOp5(OpTest): def test_check_grad(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X', 'Updates'], - 'Out', - check_eager=False) + self.check_grad_with_place( + place, ['X', 'Updates'], 'Out', check_eager=False + ) class TestScatterAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -215,19 +210,26 @@ class TestScatterAPI(unittest.TestCase): input_data = np.array([[1, 1], [2, 2], [3, 3]]).astype(np.float64) index_data = np.array([2, 1, 0, 1]).astype(np.int64) - updates_data = np.array([[1, 1], [2, 2], [3, 3], - [4, 4]]).astype(np.float64) + updates_data = np.array([[1, 1], [2, 2], [3, 3], [4, 4]]).astype( + np.float64 + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input": input_data, - "index": index_data, - "updates": updates_data - }, - fetch_list=[result]) - self.assertEqual((fetches[0] == \ - np.array([[3., 3.],[6., 6.],[1., 1.]])).all(), True) + fetches = exe.run( + fluid.default_main_program(), + feed={ + "input": input_data, + "index": index_data, + "updates": updates_data, + }, + fetch_list=[result], + ) + self.assertEqual( + ( + fetches[0] == np.array([[3.0, 3.0], [6.0, 6.0], [1.0, 1.0]]) + ).all(), + True, + ) def test_static(self): for place in self.places: @@ -238,16 +240,22 @@ class TestScatterAPI(unittest.TestCase): with fluid.dygraph.guard(place): x_data = np.array([[1, 1], [2, 2], [3, 3]]).astype(np.float64) index_data = np.array([2, 1, 0, 1]).astype(np.int64) - updates_data = np.array([[1, 1], [2, 2], [3, 3], - [4, 4]]).astype(np.float64) + updates_data = np.array( + [[1, 1], [2, 2], [3, 3], [4, 4]] + ).astype(np.float64) x = fluid.dygraph.to_variable(x_data) index = fluid.dygraph.to_variable(index_data) updates = fluid.dygraph.to_variable(updates_data) output1 = self.scatter(x, index, updates, overwrite=False) - self.assertEqual((output1.numpy() == \ - np.array([[3., 3.],[6., 6.],[1., 1.]])).all(), True) + self.assertEqual( + ( + output1.numpy() + == np.array([[3.0, 3.0], [6.0, 6.0], [1.0, 1.0]]) + ).all(), + True, + ) def test_large_data(self): if os.name == "nt" or not paddle.is_compiled_with_cuda(): @@ -259,27 +267,30 @@ class TestScatterAPI(unittest.TestCase): def test_dygraph(): with fluid.dygraph.guard(): - gpu_out = paddle.scatter(paddle.to_tensor(x), - paddle.to_tensor(index), - paddle.to_tensor(updates)) + gpu_out = paddle.scatter( + paddle.to_tensor(x), + paddle.to_tensor(index), + paddle.to_tensor(updates), + ) return gpu_out.numpy() @switch_to_static_graph def test_static_graph(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_t = paddle.static.data(name="x", dtype=x.dtype, shape=x.shape) - index_t = paddle.static.data(name="index", - dtype=index.dtype, - shape=index.shape) - updates_t = paddle.static.data(name="updates", - dtype=updates.dtype, - shape=updates.shape) + index_t = paddle.static.data( + name="index", dtype=index.dtype, shape=index.shape + ) + updates_t = paddle.static.data( + name="updates", dtype=updates.dtype, shape=updates.shape + ) out_t = paddle.scatter(x_t, index_t, updates_t) feed = { x_t.name: x, index_t.name: index, - updates_t.name: updates + updates_t.name: updates, } fetch = [out_t] @@ -290,10 +301,10 @@ class TestScatterAPI(unittest.TestCase): np.testing.assert_array_equal(test_dygraph(), test_static_graph()) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScatterOpFp16(OpTest): - def setUp(self): self.__class__.op_type = "scatter" self.python_api = paddle.scatter @@ -313,8 +324,9 @@ class TestScatterOpFp16(OpTest): self.ref_dx[self.index_np] = zero_np def compute_ref_grad_updates(self): - ref_grad_updates = paddle.gather(paddle.to_tensor(self.dout_np), - paddle.to_tensor(self.index_np)) + ref_grad_updates = paddle.gather( + paddle.to_tensor(self.dout_np), paddle.to_tensor(self.index_np) + ) return ref_grad_updates def test_scatter_fp16(self): @@ -323,21 +335,22 @@ class TestScatterOpFp16(OpTest): index_tensor = paddle.to_tensor(self.index_np) updates_tensor = paddle.to_tensor(self.updates_np, stop_gradient=False) out_tensor = paddle.scatter(x_tensor, index_tensor, updates_tensor) - paddle.autograd.backward([out_tensor], [paddle.to_tensor(self.dout_np)], - retain_graph=True) + paddle.autograd.backward( + [out_tensor], [paddle.to_tensor(self.dout_np)], retain_graph=True + ) ref_grad_updates = self.compute_ref_grad_updates() - np.testing.assert_allclose(ref_grad_updates.numpy(), - updates_tensor.grad.numpy(), - rtol=1e-5, - atol=1e-5) - np.testing.assert_allclose(self.ref_dx, - x_tensor.grad.numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + ref_grad_updates.numpy(), + updates_tensor.grad.numpy(), + rtol=1e-5, + atol=1e-5, + ) + np.testing.assert_allclose( + self.ref_dx, x_tensor.grad.numpy(), rtol=1e-5, atol=1e-5 + ) class TestScatterInplaceAPI(TestScatterAPI): - def executed_api(self): self.scatter = paddle.scatter_ diff --git a/python/paddle/fluid/tests/unittests/test_scope.py b/python/paddle/fluid/tests/unittests/test_scope.py index ddd26ad1f3b..6eaab539fc3 100644 --- a/python/paddle/fluid/tests/unittests/test_scope.py +++ b/python/paddle/fluid/tests/unittests/test_scope.py @@ -17,7 +17,6 @@ import unittest class TestScope(unittest.TestCase): - def test_create_destroy(self): paddle_c = paddle.fluid.core scope = paddle_c.Scope() @@ -53,7 +52,8 @@ class TestScope(unittest.TestCase): # Delete the scope. scope._remove_from_pool() with self.assertRaisesRegexp( - Exception, "Deleting a nonexistent scope is not allowed*"): + Exception, "Deleting a nonexistent scope is not allowed*" + ): # It is not allowed to delete a nonexistent scope. scope._remove_from_pool() diff --git a/python/paddle/fluid/tests/unittests/test_searchsorted_op.py b/python/paddle/fluid/tests/unittests/test_searchsorted_op.py index e63107026e6..d185fbaa110 100644 --- a/python/paddle/fluid/tests/unittests/test_searchsorted_op.py +++ b/python/paddle/fluid/tests/unittests/test_searchsorted_op.py @@ -23,7 +23,6 @@ from op_test import OpTest class TestSearchSorted(OpTest): - def setUp(self): self.python_api = paddle.searchsorted self.op_type = "searchsorted" @@ -31,13 +30,14 @@ class TestSearchSorted(OpTest): self.inputs = { 'SortedSequence': self.sorted_sequence, - 'Values': self.values + 'Values': self.values, } self.attrs = {"out_int32": False, "right": False} self.attrs["right"] = True if self.side == 'right' else False self.outputs = { - 'Out': - np.searchsorted(self.sorted_sequence, self.values, side=self.side) + 'Out': np.searchsorted( + self.sorted_sequence, self.values, side=self.side + ) } def test_check_output(self): @@ -50,7 +50,6 @@ class TestSearchSorted(OpTest): class TestSearchSortedOp1(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("int32") self.values = np.array([[3, 6, 9], [3, 6, 9]]).astype("int32") @@ -58,7 +57,6 @@ class TestSearchSortedOp1(TestSearchSorted): class TestSearchSortedOp2(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("int64") self.values = np.array([[3, 6, 9], [3, 6, 9]]).astype("int64") @@ -66,34 +64,33 @@ class TestSearchSortedOp2(TestSearchSorted): class TestSearchSortedOp3(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("float64") - self.values = np.array([[np.nan, np.nan, np.nan], - [3, 6, 9]]).astype("float64") + self.values = np.array([[np.nan, np.nan, np.nan], [3, 6, 9]]).astype( + "float64" + ) self.side = "left" class TestSearchSortedOp4(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("float64") - self.values = np.array([[np.inf, np.inf, np.inf], - [3, 6, 9]]).astype("float64") + self.values = np.array([[np.inf, np.inf, np.inf], [3, 6, 9]]).astype( + "float64" + ) self.side = "right" class TestSearchSortedOp5(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("float64") - self.values = np.array([[np.inf, np.inf, np.inf], - [np.nan, np.nan, np.nan]]).astype("float64") + self.values = np.array( + [[np.inf, np.inf, np.inf], [np.nan, np.nan, np.nan]] + ).astype("float64") self.side = "right" class TestSearchSortedAPI(unittest.TestCase): - def init_test_case(self): self.sorted_sequence = np.array([2, 4, 6, 8, 10]).astype("float64") self.values = np.array([[3, 6, 9], [3, 6, 9]]).astype("float64") @@ -112,17 +109,20 @@ class TestSearchSortedAPI(unittest.TestCase): sorted_sequence = paddle.static.data( 'SortedSequence', shape=self.sorted_sequence.shape, - dtype="float64") - values = paddle.static.data('Values', - shape=self.values.shape, - dtype="float64") + dtype="float64", + ) + values = paddle.static.data( + 'Values', shape=self.values.shape, dtype="float64" + ) out = paddle.searchsorted(sorted_sequence, values) exe = paddle.static.Executor(place) - res, = exe.run(feed={ - 'SortedSequence': self.sorted_sequence, - 'Values': self.values - }, - fetch_list=out) + (res,) = exe.run( + feed={ + 'SortedSequence': self.sorted_sequence, + 'Values': self.values, + }, + fetch_list=out, + ) out_ref = np.searchsorted(self.sorted_sequence, self.values) np.testing.assert_allclose(out_ref, res, rtol=1e-05) @@ -130,16 +130,15 @@ class TestSearchSortedAPI(unittest.TestCase): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) sorted_sequence = paddle.to_tensor(self.sorted_sequence) values = paddle.to_tensor(self.values) out = paddle.searchsorted(sorted_sequence, values, right=True) - out_ref = np.searchsorted(self.sorted_sequence, - self.values, - side='right') + out_ref = np.searchsorted( + self.sorted_sequence, self.values, side='right' + ) np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-05) paddle.enable_static() @@ -155,60 +154,61 @@ class TestSearchSortedAPI(unittest.TestCase): class TestSearchSortedError(unittest.TestCase): - def test_error_api(self): paddle.enable_static() def test_searchsorted_dims_matched_before_lastdim_error1(): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 2, 3], - dtype="float64") - values = paddle.static.data('Values', - shape=[2, 5], - dtype="float64") + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 2, 3], dtype="float64" + ) + values = paddle.static.data( + 'Values', shape=[2, 5], dtype="float64" + ) out = paddle.searchsorted(sorted_sequence, values) - self.assertRaises(RuntimeError, - test_searchsorted_dims_matched_before_lastdim_error1) + self.assertRaises( + RuntimeError, test_searchsorted_dims_matched_before_lastdim_error1 + ) def test_searchsorted_dims_matched_before_lastdim_error2(): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 2, 3], - dtype="float64") - values = paddle.static.data('Values', - shape=[2, 3, 5], - dtype="float64") + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 2, 3], dtype="float64" + ) + values = paddle.static.data( + 'Values', shape=[2, 3, 5], dtype="float64" + ) out = paddle.searchsorted(sorted_sequence, values) - self.assertRaises(RuntimeError, - test_searchsorted_dims_matched_before_lastdim_error2) + self.assertRaises( + RuntimeError, test_searchsorted_dims_matched_before_lastdim_error2 + ) def test_searchsorted_sortedsequence_size_error(): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 2, - pow(2, 34)], - dtype="float64") - values = paddle.static.data('Values', - shape=[2, 2, 5], - dtype="float64") - out = paddle.searchsorted(sorted_sequence, - values, - out_int32=True) - - self.assertRaises(RuntimeError, - test_searchsorted_sortedsequence_size_error) + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 2, pow(2, 34)], dtype="float64" + ) + values = paddle.static.data( + 'Values', shape=[2, 2, 5], dtype="float64" + ) + out = paddle.searchsorted( + sorted_sequence, values, out_int32=True + ) + + self.assertRaises( + RuntimeError, test_searchsorted_sortedsequence_size_error + ) def test_sortedsequence_values_type_error(): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 3], - dtype="int16") - values = paddle.static.data('Values', - shape=[2, 5], - dtype="int16") + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 3], dtype="int16" + ) + values = paddle.static.data( + 'Values', shape=[2, 5], dtype="int16" + ) out = paddle.searchsorted(sorted_sequence, values) self.assertRaises(TypeError, test_sortedsequence_values_type_error) diff --git a/python/paddle/fluid/tests/unittests/test_seed_op.py b/python/paddle/fluid/tests/unittests/test_seed_op.py index 8a1933fbe6e..14aa2c4f4dd 100644 --- a/python/paddle/fluid/tests/unittests/test_seed_op.py +++ b/python/paddle/fluid/tests/unittests/test_seed_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestSeedOpFixSeed(OpTest): - def setUp(self): self.op_type = "seed" self.inputs = {} @@ -34,7 +33,6 @@ class TestSeedOpFixSeed(OpTest): class TestSeedOpDiffSeed(OpTest): - def setUp(self): self.op_type = "seed" self.inputs = {} @@ -46,7 +44,6 @@ class TestSeedOpDiffSeed(OpTest): class TestDropoutWithRandomSeedGenerator(unittest.TestCase): - def setUp(self): paddle.framework.random.set_random_seed_generator('seed0', 123) paddle.framework.random.set_random_seed_generator('seed1', 123) @@ -58,14 +55,16 @@ class TestDropoutWithRandomSeedGenerator(unittest.TestCase): def check_static_result(self, place): import paddle.distributed.fleet.meta_parallel.parallel_layers.random as random + with static.program_guard(static.Program(), static.Program()): res1 = random.determinate_seed('seed0') exe = static.Executor(place) res_list = [res1] for i in range(2): - out1, = exe.run(static.default_main_program(), - fetch_list=res_list) + (out1,) = exe.run( + static.default_main_program(), fetch_list=res_list + ) self.assertEqual(out1, np.cast['int32'](self.rng1.random())) def test_static(self): diff --git a/python/paddle/fluid/tests/unittests/test_segment_ops.py b/python/paddle/fluid/tests/unittests/test_segment_ops.py index 71672723e82..1c6fe88ca77 100644 --- a/python/paddle/fluid/tests/unittests/test_segment_ops.py +++ b/python/paddle/fluid/tests/unittests/test_segment_ops.py @@ -83,7 +83,6 @@ def segment_pool_split(X, SegmentIds, pooltype): class TestSegmentOps(OpTest): - def set_data(self): x = np.random.uniform(-1, 1, self.shape).astype(self.dtype) segment_ids = self.set_segment(len(x), len(x) // 5 + 1) @@ -112,7 +111,7 @@ class TestSegmentOps(OpTest): result = self.compute(x, segment_ids) self.inputs = { 'X': x.astype(self.dtype), - 'SegmentIds': segment_ids.astype(np.int64) + 'SegmentIds': segment_ids.astype(np.int64), } self.outputs = {'Out': result.astype(self.dtype)} @@ -124,7 +123,6 @@ class TestSegmentOps(OpTest): class TestSegmentSum2(TestSegmentOps): - def prepare(self): super(TestSegmentSum2, self).prepare() self.shape = [40, 20] @@ -136,13 +134,12 @@ class TestSegmentSum2(TestSegmentOps): result = self.compute(x, segment_ids) self.inputs = { 'X': x.astype(self.dtype), - 'SegmentIds': segment_ids.astype(np.int32) + 'SegmentIds': segment_ids.astype(np.int32), } self.outputs = {'Out': result.astype(self.dtype)} class TestSegmentMax(TestSegmentOps): - def compute(self, x, segment_ids): return compute_segment_min_max(x, segment_ids, pooltype="MAX") @@ -157,7 +154,7 @@ class TestSegmentMax(TestSegmentOps): result, self.gradient = self.compute(x, segment_ids) self.inputs = { 'X': x.astype(self.dtype), - 'SegmentIds': segment_ids.astype(np.int32) + 'SegmentIds': segment_ids.astype(np.int32), } self.outputs = {'Out': result.astype(self.dtype)} @@ -166,14 +163,12 @@ class TestSegmentMax(TestSegmentOps): class TestSegmentMax2(TestSegmentMax): - def prepare(self): super(TestSegmentMax2, self).prepare() self.dtype = np.float32 class TestSegmentMin(TestSegmentMax): - def compute(self, x, segment_ids): return compute_segment_min_max(x, segment_ids, pooltype="MIN") @@ -183,14 +178,12 @@ class TestSegmentMin(TestSegmentMax): class TestSegmentMin2(TestSegmentMin): - def prepare(self): super(TestSegmentMin2, self).prepare() self.dtype = np.float32 class TestSegmentMean(TestSegmentOps): - def compute(self, x, segment_ids): return compute_segment_mean(x, segment_ids) @@ -205,16 +198,14 @@ class TestSegmentMean(TestSegmentOps): result = self.compute(x, segment_ids) self.inputs = {'X': x, 'SegmentIds': segment_ids} self.outputs = { - 'Out': - result, - 'SummedIds': - compute_segment_sum( - np.ones([len(x), 1]).astype(self.dtype), segment_ids) + 'Out': result, + 'SummedIds': compute_segment_sum( + np.ones([len(x), 1]).astype(self.dtype), segment_ids + ), } class TestSegmentMean2(TestSegmentMean): - def prepare(self): super(TestSegmentMean2, self).prepare() self.dtype = np.float32 @@ -223,7 +214,6 @@ class TestSegmentMean2(TestSegmentMean): class API_SegmentOpsTest(unittest.TestCase): - def test_static(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[3, 3], dtype="float32") @@ -243,11 +233,10 @@ class API_SegmentOpsTest(unittest.TestCase): np_max = np.array([[3, 2, 3], [4, 5, 6]], dtype="float32") np_min = np.array([[1, 2, 1], [4, 5, 6]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'y': data2 - }, - fetch_list=[res_sum, res_mean, res_max, res_min]) + ret = exe.run( + feed={'x': data1, 'y': data2}, + fetch_list=[res_sum, res_mean, res_max, res_min], + ) for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): np.testing.assert_allclose(np_res, ret_res, rtol=1e-05, atol=1e-06) @@ -255,8 +244,9 @@ class API_SegmentOpsTest(unittest.TestCase): def test_dygraph(self): device = paddle.CPUPlace() with paddle.fluid.dygraph.guard(device): - x = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], - dtype='float32') + x = paddle.to_tensor( + [[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float32' + ) y = paddle.to_tensor([0, 0, 1], dtype="int32") res_sum = paddle.incubate.segment_sum(x, y) res_mean = paddle.incubate.segment_mean(x, y) @@ -271,14 +261,12 @@ class API_SegmentOpsTest(unittest.TestCase): ret = [res_sum, res_mean, res_max, res_min] for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): - np.testing.assert_allclose(np_res, - ret_res.numpy(), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res, ret_res.numpy(), rtol=1e-05, atol=1e-06 + ) class API_GeometricSegmentOpsTest(unittest.TestCase): - def test_static(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[3, 3], dtype="float32") @@ -298,11 +286,10 @@ class API_GeometricSegmentOpsTest(unittest.TestCase): np_max = np.array([[3, 2, 3], [4, 5, 6]], dtype="float32") np_min = np.array([[1, 2, 1], [4, 5, 6]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'y': data2 - }, - fetch_list=[res_sum, res_mean, res_max, res_min]) + ret = exe.run( + feed={'x': data1, 'y': data2}, + fetch_list=[res_sum, res_mean, res_max, res_min], + ) for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): np.testing.assert_allclose(np_res, ret_res, rtol=1e-05, atol=1e-06) @@ -310,8 +297,9 @@ class API_GeometricSegmentOpsTest(unittest.TestCase): def test_dygraph(self): device = paddle.CPUPlace() with paddle.fluid.dygraph.guard(device): - x = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], - dtype='float32') + x = paddle.to_tensor( + [[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float32' + ) y = paddle.to_tensor([0, 0, 1], dtype="int32") res_sum = paddle.geometric.segment_sum(x, y) res_mean = paddle.geometric.segment_mean(x, y) @@ -326,16 +314,16 @@ class API_GeometricSegmentOpsTest(unittest.TestCase): ret = [res_sum, res_mean, res_max, res_min] for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): - np.testing.assert_allclose(np_res, - ret_res.numpy(), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res, ret_res.numpy(), rtol=1e-05, atol=1e-06 + ) def test_dygraph_cpu_float16(self): device = paddle.CPUPlace() with paddle.fluid.dygraph.guard(device): - x = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], - dtype='float16') + x = paddle.to_tensor( + [[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float16' + ) y = paddle.to_tensor([0, 0, 1], dtype="int32") res_sum = paddle.geometric.segment_sum(x, y) res_mean = paddle.geometric.segment_mean(x, y) @@ -349,17 +337,17 @@ class API_GeometricSegmentOpsTest(unittest.TestCase): ret = [res_sum, res_mean, res_max, res_min] for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): - np.testing.assert_allclose(np_res, - ret_res.numpy(), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res, ret_res.numpy(), rtol=1e-05, atol=1e-06 + ) def test_dygraph_cuda_float16(self): if core.is_compiled_with_cuda(): device = paddle.CUDAPlace(0) with paddle.fluid.dygraph.guard(device): - x = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], - dtype='float16') + x = paddle.to_tensor( + [[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float16' + ) y = paddle.to_tensor([0, 0, 1], dtype="int32") res_sum = paddle.geometric.segment_sum(x, y) res_mean = paddle.geometric.segment_mean(x, y) @@ -374,10 +362,9 @@ class API_GeometricSegmentOpsTest(unittest.TestCase): ret = [res_sum, res_mean, res_max, res_min] for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): - np.testing.assert_allclose(np_res, - ret_res.numpy(), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res, ret_res.numpy(), rtol=1e-05, atol=1e-06 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_select_input_output_op.py b/python/paddle/fluid/tests/unittests/test_select_input_output_op.py index 119bcc0ee5a..eeab0ca874d 100644 --- a/python/paddle/fluid/tests/unittests/test_select_input_output_op.py +++ b/python/paddle/fluid/tests/unittests/test_select_input_output_op.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestSplitMergeSelectedVarOps(unittest.TestCase): - def test_forward_backward_list_output(self): for branch_num in range(2, 10): program = Program() @@ -41,7 +40,8 @@ class TestSplitMergeSelectedVarOps(unittest.TestCase): out = program.current_block().create_var( dtype='float32', shape=[2], - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) outputs.append(out) select_output(x, outputs, mask) @@ -49,30 +49,31 @@ class TestSplitMergeSelectedVarOps(unittest.TestCase): mean = paddle.mean(y) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = Executor(place) feed_x = np.asarray([1.3, -1.4]).astype(np.float32) for i in range(branch_num): feed_mask = np.asarray([i]).astype(np.int32) - ret = exe.run(program, - feed={ - 'x': feed_x, - 'mask': feed_mask - }, - fetch_list=[y.name, x.grad_name]) + ret = exe.run( + program, + feed={'x': feed_x, 'mask': feed_mask}, + fetch_list=[y.name, x.grad_name], + ) x_grad = np.asarray([0.5, 0.5]).astype(np.float32) - np.testing.assert_allclose(np.asarray(ret[0]), - feed_x, - rtol=1e-05) - np.testing.assert_allclose(np.asarray(ret[1]), - x_grad, - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(ret[0]), feed_x, rtol=1e-05 + ) + np.testing.assert_allclose( + np.asarray(ret[1]), x_grad, rtol=1e-05 + ) class TestSelectInputOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): mask = layers.data(name='mask', shape=[1], dtype='int32') @@ -99,17 +100,16 @@ class TestSelectInputOpError(unittest.TestCase): class TestSelectOutput_Error(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): in1 = layers.data(name='in1', shape=[1], dtype='int32') - mask_int32 = layers.data(name='mask_int32', - shape=[1], - dtype='int32') - mask_float32 = layers.data(name='mask_float32', - shape=[1], - dtype='float32') + mask_int32 = layers.data( + name='mask_int32', shape=[1], dtype='int32' + ) + mask_float32 = layers.data( + name='mask_float32', shape=[1], dtype='float32' + ) out1 = layers.data(name='out1', shape=[1], dtype='int32') # 1. The type of input in select_output must Variable. diff --git a/python/paddle/fluid/tests/unittests/test_selected_rows.py b/python/paddle/fluid/tests/unittests/test_selected_rows.py index 4eac57f834b..a66dc40dfb1 100644 --- a/python/paddle/fluid/tests/unittests/test_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_selected_rows.py @@ -18,7 +18,6 @@ import numpy as np class TestSelectedRows(unittest.TestCase): - def test_selected_rows(self): place = core.CPUPlace() height = 10 @@ -40,13 +39,16 @@ class TestSelectedRows(unittest.TestCase): self.assertEqual(10, selected_rows.height()) # compare tensor - self.assertAlmostEqual(2.0, - selected_rows.get_tensor()._get_float_element(0)) - self.assertAlmostEqual(1.0, - selected_rows.get_tensor()._get_float_element(1)) + self.assertAlmostEqual( + 2.0, selected_rows.get_tensor()._get_float_element(0) + ) + self.assertAlmostEqual( + 1.0, selected_rows.get_tensor()._get_float_element(1) + ) self.assertAlmostEqual( 4.0, - selected_rows.get_tensor()._get_float_element(2 * row_numel + 8)) + selected_rows.get_tensor()._get_float_element(2 * row_numel + 8), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_selu_op.py b/python/paddle/fluid/tests/unittests/test_selu_op.py index 5de31d26b2b..1390ddb5c96 100644 --- a/python/paddle/fluid/tests/unittests/test_selu_op.py +++ b/python/paddle/fluid/tests/unittests/test_selu_op.py @@ -21,9 +21,11 @@ import paddle.fluid as fluid import paddle.nn.functional as F -def ref_selu(x, - scale=1.0507009873554804934193349852946, - alpha=1.6732632423543772848170429916717): +def ref_selu( + x, + scale=1.0507009873554804934193349852946, + alpha=1.6732632423543772848170429916717, +): out = np.copy(x) out_flat = out.flatten() for i in range(out_flat.size): @@ -35,7 +37,6 @@ def ref_selu(x, class SeluTest(OpTest): - def setUp(self): self.op_type = "selu" self.python_api = paddle.nn.functional.selu @@ -85,8 +86,11 @@ class TestSeluAPI(unittest.TestCase): # Since zero point in selu is not differentiable, avoid randomize # zero. self.x_np[np.abs(self.x_np) < 0.005] = 0.02 - self.place=paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -125,21 +129,21 @@ class TestSeluAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, F.selu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.selu, x_int32) # The scale must be greater than 1.0 - x_fp32 = paddle.fluid.data(name='x_fp32', - shape=[12, 10], - dtype='float32') + x_fp32 = paddle.fluid.data( + name='x_fp32', shape=[12, 10], dtype='float32' + ) self.assertRaises(ValueError, F.selu, x_fp32, -1.0) # The alpha must be no less than 0 self.assertRaises(ValueError, F.selu, x_fp32, 1.6, -1.0) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.selu(x_fp16) diff --git a/python/paddle/fluid/tests/unittests/test_set_bool_attr.py b/python/paddle/fluid/tests/unittests/test_set_bool_attr.py index a089563b6ec..2c233141d09 100644 --- a/python/paddle/fluid/tests/unittests/test_set_bool_attr.py +++ b/python/paddle/fluid/tests/unittests/test_set_bool_attr.py @@ -17,18 +17,19 @@ import unittest class TestAttrSet(unittest.TestCase): - def test_set_bool_attr(self): x = fluid.layers.data(name='x', shape=[3, 7, 3, 7], dtype='float32') param_attr = fluid.ParamAttr( name='batch_norm_w', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) bias_attr = fluid.ParamAttr( name='batch_norm_b', - initializer=fluid.initializer.Constant(value=0.0)) - bn = fluid.layers.batch_norm(input=x, - param_attr=param_attr, - bias_attr=bias_attr) + initializer=fluid.initializer.Constant(value=0.0), + ) + bn = fluid.layers.batch_norm( + input=x, param_attr=param_attr, bias_attr=bias_attr + ) block = fluid.default_main_program().desc.block(0) op = block.op(0) before_type = op.attr_type('is_test') diff --git a/python/paddle/fluid/tests/unittests/test_set_value_op.py b/python/paddle/fluid/tests/unittests/test_set_value_op.py index 7f6b9831aae..92dc8b8fadc 100644 --- a/python/paddle/fluid/tests/unittests/test_set_value_op.py +++ b/python/paddle/fluid/tests/unittests/test_set_value_op.py @@ -23,8 +23,8 @@ from paddle.fluid.layer_helper import LayerHelper from functools import reduce from paddle.fluid.framework import _test_eager_guard -class TestSetValueBase(unittest.TestCase): +class TestSetValueBase(unittest.TestCase): def setUp(self): paddle.enable_static() self.set_dtype() @@ -50,7 +50,6 @@ class TestSetValueBase(unittest.TestCase): class TestSetValueApi(TestSetValueBase): - def _run_static(self): paddle.enable_static() with paddle.static.program_guard(self.program): @@ -75,11 +74,17 @@ class TestSetValueApi(TestSetValueBase): dynamic_out = self._run_dynamic() self._get_answer() - error_msg = "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" - self.assertTrue((self.data == static_out).all(), - msg=error_msg.format("static", self.data, static_out)) - self.assertTrue((self.data == dynamic_out).all(), - msg=error_msg.format("dynamic", self.data, dynamic_out)) + error_msg = ( + "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" + ) + self.assertTrue( + (self.data == static_out).all(), + msg=error_msg.format("static", self.data, static_out), + ) + self.assertTrue( + (self.data == dynamic_out).all(), + msg=error_msg.format("dynamic", self.data, dynamic_out), + ) def test_api(self): with _test_eager_guard(): @@ -90,7 +95,6 @@ class TestSetValueApi(TestSetValueBase): # 1. Test different type of item: int, Python slice, Paddle Tensor # 1.1 item is int class TestSetValueItemInt(TestSetValueApi): - def _call_setitem(self, x): x[0] = self.value @@ -101,7 +105,6 @@ class TestSetValueItemInt(TestSetValueApi): # 1.2 item is slice # 1.2.1 step is 1 class TestSetValueItemSlice(TestSetValueApi): - def _call_setitem(self, x): x[0:2] = self.value @@ -110,7 +113,6 @@ class TestSetValueItemSlice(TestSetValueApi): class TestSetValueItemSlice2(TestSetValueApi): - def _call_setitem(self, x): x[0:-1] = self.value @@ -119,7 +121,6 @@ class TestSetValueItemSlice2(TestSetValueApi): class TestSetValueItemSlice3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2] = self.value @@ -128,7 +129,6 @@ class TestSetValueItemSlice3(TestSetValueApi): class TestSetValueItemSlice4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2, :] = self.value @@ -137,7 +137,6 @@ class TestSetValueItemSlice4(TestSetValueApi): class TestSetValueItemSlice5(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:1, :] = self.value @@ -146,9 +145,7 @@ class TestSetValueItemSlice5(TestSetValueApi): class TestSetValueItemSliceInWhile(TestSetValueApi): - def _call_setitem(self, x): - def cond(i, x): return i < 1 @@ -157,7 +154,7 @@ class TestSetValueItemSliceInWhile(TestSetValueApi): i = i + 1 return i, x - i = paddle.zeros(shape=(1, ), dtype='int32') + i = paddle.zeros(shape=(1,), dtype='int32') i, x = paddle.fluid.layers.while_loop(cond, body, [i, x]) def _get_answer(self): @@ -166,7 +163,6 @@ class TestSetValueItemSliceInWhile(TestSetValueApi): # 1.2.2 step > 1 class TestSetValueItemSliceStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 5, 5] @@ -178,7 +174,6 @@ class TestSetValueItemSliceStep(TestSetValueApi): class TestSetValueItemSliceStep2(TestSetValueApi): - def set_shape(self): self.shape = [7, 5, 5] @@ -190,7 +185,6 @@ class TestSetValueItemSliceStep2(TestSetValueApi): class TestSetValueItemSliceStep3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2, ::2] = self.value @@ -199,7 +193,6 @@ class TestSetValueItemSliceStep3(TestSetValueApi): class TestSetValueItemSliceStep4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2:2, :] = self.value @@ -209,7 +202,6 @@ class TestSetValueItemSliceStep4(TestSetValueApi): # 1.2.3 step < 0 class TestSetValueItemSliceNegetiveStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 2] @@ -224,7 +216,6 @@ class TestSetValueItemSliceNegetiveStep(TestSetValueApi): class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): - def set_shape(self): self.shape = [5] @@ -239,7 +230,6 @@ class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): - def set_shape(self): self.shape = [3] @@ -254,7 +244,6 @@ class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -269,7 +258,6 @@ class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): class TestSetValueItemEllipsis1(TestSetValueApi): - def _call_setitem(self, x): x[0:, ..., 1:] = self.value @@ -278,7 +266,6 @@ class TestSetValueItemEllipsis1(TestSetValueApi): class TestSetValueItemEllipsis2(TestSetValueApi): - def _call_setitem(self, x): x[0:, ...] = self.value @@ -287,7 +274,6 @@ class TestSetValueItemEllipsis2(TestSetValueApi): class TestSetValueItemEllipsis3(TestSetValueApi): - def _call_setitem(self, x): x[..., 1:] = self.value @@ -296,7 +282,6 @@ class TestSetValueItemEllipsis3(TestSetValueApi): class TestSetValueItemEllipsis4(TestSetValueApi): - def _call_setitem(self, x): x[...] = self.value @@ -306,7 +291,6 @@ class TestSetValueItemEllipsis4(TestSetValueApi): # 1.4 item is Paddle Tensor class TestSetValueItemTensor(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") x[zero] = self.value @@ -316,7 +300,6 @@ class TestSetValueItemTensor(TestSetValueApi): class TestSetValueItemTensor2(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -327,7 +310,6 @@ class TestSetValueItemTensor2(TestSetValueApi): class TestSetValueItemTensor3(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -338,7 +320,6 @@ class TestSetValueItemTensor3(TestSetValueApi): class TestSetValueItemTensor4(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -349,7 +330,6 @@ class TestSetValueItemTensor4(TestSetValueApi): class TestSetValueItemTensor5(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -360,7 +340,6 @@ class TestSetValueItemTensor5(TestSetValueApi): class TestSetValueItemTensor6(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -375,7 +354,6 @@ class TestSetValueItemTensor6(TestSetValueApi): # 1.5 item is None class TestSetValueItemNone1(TestSetValueApi): - def _call_setitem(self, x): x[None] = self.value @@ -384,7 +362,6 @@ class TestSetValueItemNone1(TestSetValueApi): class TestSetValueItemNone2(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 1] = self.value @@ -393,7 +370,6 @@ class TestSetValueItemNone2(TestSetValueApi): class TestSetValueItemNone3(TestSetValueApi): - def _call_setitem(self, x): x[:, None, None, 1] = self.value @@ -402,7 +378,6 @@ class TestSetValueItemNone3(TestSetValueApi): class TestSetValueItemNone4(TestSetValueApi): - def _call_setitem(self, x): x[0, 0, None, 1] = self.value @@ -411,7 +386,6 @@ class TestSetValueItemNone4(TestSetValueApi): class TestSetValueItemNone5(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 0, None, 1] = self.value @@ -420,7 +394,6 @@ class TestSetValueItemNone5(TestSetValueApi): class TestSetValueItemNone6(TestSetValueApi): - def _call_setitem(self, x): x[None, 0, 0, None, 0] = self.value @@ -429,7 +402,6 @@ class TestSetValueItemNone6(TestSetValueApi): class TestSetValueItemNone7(TestSetValueApi): - def _call_setitem(self, x): x[:, None, 1] = np.zeros(self.shape)[:, None, 0] @@ -438,7 +410,6 @@ class TestSetValueItemNone7(TestSetValueApi): class TestSetValueItemNone8(TestSetValueApi): - def _call_setitem(self, x): x[:, 1, None] = np.zeros(self.shape)[:, 0, None] @@ -447,7 +418,6 @@ class TestSetValueItemNone8(TestSetValueApi): class TestSetValueItemNone9(TestSetValueApi): - def _call_setitem(self, x): x[None, :, 1, ..., None] = np.zeros(self.shape)[0, 0, :, None] @@ -456,7 +426,6 @@ class TestSetValueItemNone9(TestSetValueApi): class TestSetValueItemNone10(TestSetValueApi): - def _call_setitem(self, x): x[..., None, :, None] = np.zeros(self.shape)[..., None, :, None] @@ -466,7 +435,6 @@ class TestSetValueItemNone10(TestSetValueApi): # 1.5 item is list or Tensor of bol class TestSetValueItemBool1(TestSetValueApi): - def _call_setitem(self, x): x[[True, False]] = self.value @@ -475,7 +443,6 @@ class TestSetValueItemBool1(TestSetValueApi): class TestSetValueItemBool2(TestSetValueApi): - def _call_setitem(self, x): x[[False, False]] = self.value @@ -484,7 +451,6 @@ class TestSetValueItemBool2(TestSetValueApi): class TestSetValueItemBool3(TestSetValueApi): - def _call_setitem(self, x): x[[False, True]] = np.zeros(self.shape[2]) @@ -493,7 +459,6 @@ class TestSetValueItemBool3(TestSetValueApi): class TestSetValueItemBool4(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign(np.array([False, True])) x[idx] = np.zeros(self.shape[2]) @@ -503,19 +468,19 @@ class TestSetValueItemBool4(TestSetValueApi): class TestSetValueItemBool5(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign( - np.array([[False, True, False], [True, True, False]])) + np.array([[False, True, False], [True, True, False]]) + ) x[idx] = self.value def _get_answer(self): - self.data[np.array([[False, True, False], [True, True, - False]])] = self.value + self.data[ + np.array([[False, True, False], [True, True, False]]) + ] = self.value class TestSetValueItemBool6(TestSetValueApi): - def _call_setitem(self, x): x[0, ...] = 0 x[x > 0] = self.value @@ -530,9 +495,7 @@ class TestSetValueItemBool6(TestSetValueApi): def create_test_value_int32(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -552,9 +515,7 @@ create_test_value_int32(TestSetValueItemSlice4) def create_test_value_int64(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -574,9 +535,7 @@ create_test_value_int64(TestSetValueItemSlice4) def create_test_value_fp16(parent): - class TestValueInt(parent): - def set_value(self): self.value = 3.7 @@ -596,9 +555,7 @@ create_test_value_fp16(TestSetValueItemSlice4) def create_test_value_fp32(parent): - class TestValueInt(parent): - def set_value(self): self.value = 3.3 @@ -618,9 +575,7 @@ create_test_value_fp32(TestSetValueItemSlice4) def create_test_value_fp64(parent): - class TestValueInt(parent): - def set_value(self): self.value = 2.0**127 # float32:[-2^128, 2^128) @@ -640,9 +595,7 @@ create_test_value_fp64(TestSetValueItemSlice4) def create_test_value_bool(parent): - class TestValueInt(parent): - def set_value(self): self.value = 0 @@ -663,9 +616,7 @@ create_test_value_bool(TestSetValueItemSlice4) # 2.2 value is numpy.array (int32, int64, float32, float64, bool) def create_test_value_numpy_int32(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([5]) @@ -685,9 +636,7 @@ create_test_value_numpy_int32(TestSetValueItemSlice4) def create_test_value_numpy_int64(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([1]) @@ -707,9 +656,7 @@ create_test_value_numpy_int64(TestSetValueItemSlice4) def create_test_value_numpy_fp32(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([1]) @@ -729,9 +676,7 @@ create_test_value_numpy_fp32(TestSetValueItemSlice4) def create_test_value_numpy_fp64(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([2**127]).astype("float64") @@ -751,9 +696,7 @@ create_test_value_numpy_fp64(TestSetValueItemSlice4) def create_test_value_numpy_bool(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([0]) @@ -774,9 +717,7 @@ create_test_value_numpy_bool(TestSetValueItemSlice4) # 2.3 value is a Paddle Tensor (int32, int64, float32, float64, bool) def create_test_value_tensor_int32(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "int32" @@ -800,9 +741,7 @@ create_test_value_tensor_int32(TestSetValueItemSlice4) def create_test_value_tensor_int64(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "int64" @@ -826,9 +765,7 @@ create_test_value_tensor_int64(TestSetValueItemSlice4) def create_test_value_tensor_fp32(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "float32" @@ -852,9 +789,7 @@ create_test_value_tensor_fp32(TestSetValueItemSlice4) def create_test_value_tensor_fp64(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "float64" @@ -878,9 +813,7 @@ create_test_value_tensor_fp64(TestSetValueItemSlice4) def create_test_value_tensor_bool(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "bool" @@ -905,7 +838,6 @@ create_test_value_tensor_bool(TestSetValueItemSlice4) # 3. Test different shape of value class TestSetValueValueShape1(TestSetValueApi): - def set_value(self): self.value = np.array([3, 4, 5, 6]) # shape is (4,) @@ -917,7 +849,6 @@ class TestSetValueValueShape1(TestSetValueApi): class TestSetValueValueShape2(TestSetValueApi): - def set_value(self): self.value = np.array([[3, 4, 5, 6]]) # shape is (1,4) @@ -929,10 +860,10 @@ class TestSetValueValueShape2(TestSetValueApi): class TestSetValueValueShape3(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, 3]]) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ) # shape is (3,4) def _call_setitem(self, x): x[0] = self.value @@ -942,11 +873,12 @@ class TestSetValueValueShape3(TestSetValueApi): class TestSetValueValueShape4(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, - 3]]).astype(self.dtype) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ).astype( + self.dtype + ) # shape is (3,4) def _call_setitem(self, x): x[0] = paddle.assign(self.value) # x is Paddle.Tensor @@ -956,7 +888,6 @@ class TestSetValueValueShape4(TestSetValueApi): class TestSetValueValueShape5(TestSetValueApi): - def set_value(self): self.value = np.array([3, 3, 3]).astype(self.dtype) @@ -972,11 +903,10 @@ class TestSetValueValueShape5(TestSetValueApi): # 4. Test error class TestError(TestSetValueBase): - def _value_type_error(self): with self.assertRaisesRegexp( - TypeError, - "Only support to assign an integer, float, numpy.ndarray or paddle.Tensor" + TypeError, + "Only support to assign an integer, float, numpy.ndarray or paddle.Tensor", ): x = paddle.ones(shape=self.shape, dtype=self.dtype) value = [1] @@ -984,8 +914,8 @@ class TestError(TestSetValueBase): def _dtype_error(self): with self.assertRaisesRegexp( - TypeError, - "When assign a numpy.ndarray, integer or float to a paddle.Tensor, " + TypeError, + "When assign a numpy.ndarray, integer or float to a paddle.Tensor, ", ): y = paddle.ones(shape=self.shape, dtype="float16") y[0] = 1 @@ -997,7 +927,8 @@ class TestError(TestSetValueBase): def _ellipsis_error(self): with self.assertRaisesRegexp( - IndexError, "An index can only have a single ellipsis"): + IndexError, "An index can only have a single ellipsis" + ): x = paddle.ones(shape=self.shape, dtype=self.dtype) x[..., ...] = self.value with self.assertRaisesRegexp(ValueError, "the start or end is None"): @@ -1044,7 +975,6 @@ class TestError(TestSetValueBase): class Model(paddle.nn.Layer): - def __init__(self): super(Model, self).__init__() self.conv = paddle.nn.Conv2D(12, 12, 3) @@ -1060,7 +990,6 @@ class Model(paddle.nn.Layer): class TestBackward(unittest.TestCase): - def test_static(self): paddle.enable_static() main_program = paddle.static.Program() @@ -1074,9 +1003,9 @@ class TestBackward(unittest.TestCase): x = paddle.static.data(name="x", shape=[4, 4], dtype='float32') y = paddle.static.data(name="y", shape=[4, 4], dtype='float32') - label = paddle.static.data(name="label", - shape=[4, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[4, 1], dtype='int64' + ) z = paddle.add(x, y) var = y[0, :] @@ -1084,8 +1013,9 @@ class TestBackward(unittest.TestCase): prediction = paddle.static.nn.fc(x=z, size=2, activation='softmax') - cost = paddle.nn.functional.cross_entropy(input=prediction, - label=label) + cost = paddle.nn.functional.cross_entropy( + input=prediction, label=label + ) loss = paddle.mean(cost) sgd = paddle.optimizer.SGD(learning_rate=0.01) sgd.minimize(loss) @@ -1095,12 +1025,9 @@ class TestBackward(unittest.TestCase): var_grad, z_grad = exe.run( main_program, - feed={ - "x": x_np, - "y": y_np, - "label": label_np - }, - fetch_list=[var.name + "@GRAD", z.name + "@GRAD"]) + feed={"x": x_np, "y": y_np, "label": label_np}, + fetch_list=[var.name + "@GRAD", z.name + "@GRAD"], + ) self.assertTrue((var_grad == z_grad[0, :]).all()) paddle.disable_static() @@ -1124,7 +1051,6 @@ class TestBackward(unittest.TestCase): class TestGradientTruncated(unittest.TestCase): - def func_test_consistent_with_competitor(self): paddle.disable_static() @@ -1135,8 +1061,9 @@ class TestGradientTruncated(unittest.TestCase): return y.sum() # case 1 - array = np.arange(1, 1 + 2 * 3 * 4, - dtype="float32").reshape([1, 2, 1, 3, 1, 4]) + array = np.arange(1, 1 + 2 * 3 * 4, dtype="float32").reshape( + [1, 2, 1, 3, 1, 4] + ) value = np.arange(100, 104, dtype="float32").reshape(1, 4) inps = paddle.to_tensor(array, stop_gradient=False) @@ -1145,22 +1072,41 @@ class TestGradientTruncated(unittest.TestCase): loss = set_value(inps, value) loss.backward() - value_grad = np.array([[600., 606., 612., 618.]]) - input_grad = np.array([[[[[[4., 32., 108., 256.]], - [[500., 864., 1372., 2048.]], - [[2916., 4000., 5324., 6912.]]]], - [[[[0., 0., 0., 0.]], [[0., 0., 0., 0.]], - [[0., 0., 0., 0.]]]]]]) + value_grad = np.array([[600.0, 606.0, 612.0, 618.0]]) + input_grad = np.array( + [ + [ + [ + [ + [[4.0, 32.0, 108.0, 256.0]], + [[500.0, 864.0, 1372.0, 2048.0]], + [[2916.0, 4000.0, 5324.0, 6912.0]], + ] + ], + [ + [ + [[0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, 0.0]], + ] + ], + ] + ] + ) np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps.grad.numpy() + ), + ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value.grad.numpy() + ), + ) # case 2 array = np.arange(1, 2 * 3 * 4 + 1, dtype="float32").reshape([4, 2, 3]) @@ -1172,23 +1118,29 @@ class TestGradientTruncated(unittest.TestCase): loss = set_value(inps2, value2) loss.backward() - value_grad2 = np.array([600.]) - input_grad2 = np.array([[[4., 32., 108.], [0., 0., 0.]], - [[1372., 2048., 2916.], [4000., 5324., 6912.]], - [[8788., 10976., 13500.], - [16384., 19652., 23328.]], - [[27436., 32000., 37044.], - [42592., 48668., 55296.]]]) + value_grad2 = np.array([600.0]) + input_grad2 = np.array( + [ + [[4.0, 32.0, 108.0], [0.0, 0.0, 0.0]], + [[1372.0, 2048.0, 2916.0], [4000.0, 5324.0, 6912.0]], + [[8788.0, 10976.0, 13500.0], [16384.0, 19652.0, 23328.0]], + [[27436.0, 32000.0, 37044.0], [42592.0, 48668.0, 55296.0]], + ] + ) np.testing.assert_array_equal( inps2.grad.numpy(), input_grad2, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps2.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps2.grad.numpy() + ), + ) np.testing.assert_array_equal( value2.grad.numpy(), value_grad2, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value2.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value2.grad.numpy() + ), + ) # case 3 def set_value3(t, value): @@ -1197,8 +1149,9 @@ class TestGradientTruncated(unittest.TestCase): y = a * a return y.sum() - array = np.arange(1, 1 + 2 * 3 * 4, - dtype="float32").reshape([4, 3, 1, 1, 2, 1]) + array = np.arange(1, 1 + 2 * 3 * 4, dtype="float32").reshape( + [4, 3, 1, 1, 2, 1] + ) value = np.arange(100, 100 + 2, dtype="float32").reshape(1, 2, 1) inps = paddle.to_tensor(array, stop_gradient=False) @@ -1207,37 +1160,52 @@ class TestGradientTruncated(unittest.TestCase): loss = set_value3(inps, value) loss.backward() - value_grad = np.array([[[600.], [606.]]]) - input_grad = np.array([[[[[[0.], [0.]]]], [[[[0.], [0.]]]], - [[[[0.], [0.]]]]], - [[[[[1372.], [2048.]]]], [[[[2916.], [4000.]]]], - [[[[5324.], [6912.]]]]], - [[[[[8788.], [10976.]]]], [[[[13500.], - [16384.]]]], - [[[[19652.], [23328.]]]]], - [[[[[27436.], [32000.]]]], - [[[[37044.], [42592.]]]], - [[[[48668.], [55296.]]]]]]) + value_grad = np.array([[[600.0], [606.0]]]) + input_grad = np.array( + [ + [[[[[0.0], [0.0]]]], [[[[0.0], [0.0]]]], [[[[0.0], [0.0]]]]], + [ + [[[[1372.0], [2048.0]]]], + [[[[2916.0], [4000.0]]]], + [[[[5324.0], [6912.0]]]], + ], + [ + [[[[8788.0], [10976.0]]]], + [[[[13500.0], [16384.0]]]], + [[[[19652.0], [23328.0]]]], + ], + [ + [[[[27436.0], [32000.0]]]], + [[[[37044.0], [42592.0]]]], + [[[[48668.0], [55296.0]]]], + ], + ] + ) np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps.grad.numpy() + ), + ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value.grad.numpy() + ), + ) - #case 4: step >0 + # case 4: step >0 def set_value4(t, value): a = t * t a[0, :, 0, ::3] = value y = a * a return y.sum() - array = np.arange(1, 1 + 2 * 3 * 4, - dtype="float32").reshape([2, 3, 1, 4, 1]) + array = np.arange(1, 1 + 2 * 3 * 4, dtype="float32").reshape( + [2, 3, 1, 4, 1] + ) value = np.arange(100, 100 + 2, dtype="float32").reshape(1, 2, 1) inps = paddle.to_tensor(array, stop_gradient=False) @@ -1246,23 +1214,35 @@ class TestGradientTruncated(unittest.TestCase): loss = set_value4(inps, value) loss.backward() - value_grad = np.array([[[600.], [606.]]]) - input_grad = np.array([[[[[0.], [32.], [108.], [0.]]], - [[[0.], [864.], [1372.], [0.]]], - [[[0.], [4000.], [5324.], [0.]]]], - [[[[8788.], [10976.], [13500.], [16384.]]], - [[[19652.], [23328.], [27436.], [32000.]]], - [[[37044.], [42592.], [48668.], [55296.]]]]]) + value_grad = np.array([[[600.0], [606.0]]]) + input_grad = np.array( + [ + [ + [[[0.0], [32.0], [108.0], [0.0]]], + [[[0.0], [864.0], [1372.0], [0.0]]], + [[[0.0], [4000.0], [5324.0], [0.0]]], + ], + [ + [[[8788.0], [10976.0], [13500.0], [16384.0]]], + [[[19652.0], [23328.0], [27436.0], [32000.0]]], + [[[37044.0], [42592.0], [48668.0], [55296.0]]], + ], + ] + ) np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps.grad.numpy() + ), + ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value.grad.numpy() + ), + ) # case 5:a[0].shape==value.shape def set_value5(t, value): @@ -1280,24 +1260,41 @@ class TestGradientTruncated(unittest.TestCase): loss = set_value5(inps, value) loss.backward() - value_grad = np.array([[200., 202., 204., - 206.], [208., 210., 212., 214.], - [216., 218., 220., 222.]]) - input_grad = np.array([[[0., 0., 0., 0.], [0., 0., 0., 0.], - [0., 0., 0., 0.]], - [[8788., 10976., 13500., 16384.], - [19652., 23328., 27436., 32000.], - [37044., 42592., 48668., 55296.]]]) + value_grad = np.array( + [ + [200.0, 202.0, 204.0, 206.0], + [208.0, 210.0, 212.0, 214.0], + [216.0, 218.0, 220.0, 222.0], + ] + ) + input_grad = np.array( + [ + [ + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + ], + [ + [8788.0, 10976.0, 13500.0, 16384.0], + [19652.0, 23328.0, 27436.0, 32000.0], + [37044.0, 42592.0, 48668.0, 55296.0], + ], + ] + ) np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps.grad.numpy() + ), + ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value.grad.numpy() + ), + ) # case 6: pass stop_gradient from value to x x = paddle.zeros([8, 8], dtype='float32') @@ -1319,7 +1316,7 @@ class TestGradientTruncated(unittest.TestCase): def test_static_graph(self): paddle.enable_static() - to_string = lambda x, i, : x + '_' + str(i) + to_string = lambda x, i,: x + '_' + str(i) numel = lambda input_shape: reduce(lambda x, y: x * y, input_shape) def op1(x): @@ -1327,18 +1324,15 @@ class TestGradientTruncated(unittest.TestCase): # test stop_gradient value.stop_gradient = True x.stop_gradient = False - start = paddle.fluid.layers.fill_constant([1], - "int32", - 5, - force_cpu=True) - end = paddle.fluid.layers.fill_constant([1], - "int32", - 0, - force_cpu=True) - step = paddle.fluid.layers.fill_constant([1], - "int32", - -2, - force_cpu=True) + start = paddle.fluid.layers.fill_constant( + [1], "int32", 5, force_cpu=True + ) + end = paddle.fluid.layers.fill_constant( + [1], "int32", 0, force_cpu=True + ) + step = paddle.fluid.layers.fill_constant( + [1], "int32", -2, force_cpu=True + ) inputs = { 'Input': x, @@ -1351,16 +1345,18 @@ class TestGradientTruncated(unittest.TestCase): ], 'StepsTensorList': [ step, - ] + ], } helper = LayerHelper("set_value") y = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="set_value", - inputs=inputs, - outputs={'Out': y}, - attrs={'axes': [0]}) + helper.append_op( + type="set_value", + inputs=inputs, + outputs={'Out': y}, + attrs={'axes': [0]}, + ) return y, value @@ -1376,17 +1372,16 @@ class TestGradientTruncated(unittest.TestCase): 'steps': [-4], 'decrease_axes': [], 'none_axes': [], - 'dtype': paddle.float32 + 'dtype': paddle.float32, } inputs = {'Input': x, 'ValueTensor': value} helper = LayerHelper("set_value") y = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="set_value", - inputs=inputs, - outputs={'Out': y}, - attrs=attrs) + helper.append_op( + type="set_value", inputs=inputs, outputs={'Out': y}, attrs=attrs + ) return y, value @@ -1394,18 +1389,15 @@ class TestGradientTruncated(unittest.TestCase): value = paddle.fluid.layers.fill_constant([1], "float32", 1) x.stop_gradient = True value.stop_gradient = False - start = paddle.fluid.layers.fill_constant([1], - "int32", - 0, - force_cpu=True) - end = paddle.fluid.layers.fill_constant([1], - "int32", - 5, - force_cpu=True) - step = paddle.fluid.layers.fill_constant([1], - "int32", - 3, - force_cpu=True) + start = paddle.fluid.layers.fill_constant( + [1], "int32", 0, force_cpu=True + ) + end = paddle.fluid.layers.fill_constant( + [1], "int32", 5, force_cpu=True + ) + step = paddle.fluid.layers.fill_constant( + [1], "int32", 3, force_cpu=True + ) inputs = { 'Input': x, @@ -1418,24 +1410,26 @@ class TestGradientTruncated(unittest.TestCase): ], 'StepsTensorList': [ step, - ] + ], } helper = LayerHelper("set_value") y = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="set_value", - inputs=inputs, - outputs={'Out': y}, - attrs={'axes': [0]}) + helper.append_op( + type="set_value", + inputs=inputs, + outputs={'Out': y}, + attrs={'axes': [0]}, + ) return y, value def set_value(array, i, op): name_x = to_string('x', i) - x = paddle.static.data(name=name_x, - shape=array.shape, - dtype='float32') + x = paddle.static.data( + name=name_x, shape=array.shape, dtype='float32' + ) # set_value_op in __get/setitem__ is an inplace operation. # When `input.stop_gradient = True` and `value.stop_gradient = False`, @@ -1445,9 +1439,11 @@ class TestGradientTruncated(unittest.TestCase): loss = paddle.fluid.layers.reduce_sum(y2) sgd = paddle.optimizer.Adam() sgd.minimize(loss) - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) @@ -1462,8 +1458,9 @@ class TestGradientTruncated(unittest.TestCase): input_shape = [7, 6, 5, 4, 3, 2] - array = np.arange(0, numel(input_shape), - dtype="float32").reshape(input_shape) + array = np.arange(0, numel(input_shape), dtype="float32").reshape( + input_shape + ) for i in range(len(input_shape)): program = paddle.static.Program() @@ -1487,7 +1484,6 @@ class TestGradientTruncated(unittest.TestCase): class TestSetValueInplace(unittest.TestCase): - def test_inplace(self): paddle.disable_static() with paddle.fluid.dygraph.guard(): @@ -1506,7 +1502,6 @@ class TestSetValueInplace(unittest.TestCase): class TestSetValueInplaceLeafVar(unittest.TestCase): - def test_inplace_var_become_leaf_var(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op.py b/python/paddle/fluid/tests/unittests/test_sgd_op.py index 2643dfe272c..34420ce5a9c 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op.py @@ -25,7 +25,6 @@ paddle.enable_static() class TestSGDOp(OpTest): - def setUp(self): self.op_type = "sgd" self.conf() @@ -45,14 +44,12 @@ class TestSGDOp(OpTest): class TestSGDOpCase8X(TestSGDOp): - def conf(self): self.h = 10 self.w = 64 class TestSparseSGDOp(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() @@ -82,11 +79,13 @@ class TestSparseSGDOp(unittest.TestCase): lr.set(lr_array, place) # create and run sgd operator - sgd_op = Operator("sgd", - Param='Param', - Grad='Grad', - ParamOut='Param', - LearningRate='LearningRate') + sgd_op = Operator( + "sgd", + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate', + ) sgd_op.run(scope, place) # get and compare result @@ -119,13 +118,11 @@ class TestSparseSGDOp(unittest.TestCase): class TestSparseSGDOpCase8X(TestSparseSGDOp): - def conf(self): self.row_numel = 16 class TestSGDOpOptimizeSelectedRows(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() @@ -170,15 +167,18 @@ class TestSGDOpOptimizeSelectedRows(unittest.TestCase): # optimize with Python w_after_optimize = np.copy(w_before_optimize) for index, id in enumerate(grad_rows): - w_after_optimize[ - id] = w_before_optimize[id] - lr_value * grad_array[index] + w_after_optimize[id] = ( + w_before_optimize[id] - lr_value * grad_array[index] + ) # create and run sgd operator - sgd_op = Operator("sgd", - Param='Param', - Grad='Grad', - ParamOut='Param', - LearningRate='LearningRate') + sgd_op = Operator( + "sgd", + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate', + ) sgd_op.run(scope, place) # get and compare result @@ -193,13 +193,12 @@ class TestSGDOpOptimizeSelectedRows(unittest.TestCase): class TestSGDOpWithLargeInput(unittest.TestCase): - def runTest(self): paddle.enable_static() data = fluid.layers.fill_constant(shape=[1], value=128, dtype='int64') - label = fluid.layers.fill_constant(shape=[1, 150], - value=0.5, - dtype='float32') + label = fluid.layers.fill_constant( + shape=[1, 150], value=0.5, dtype='float32' + ) emb = fluid.embedding(input=data, size=(10000000, 150), dtype='float32') out = fluid.layers.l2_normalize(x=emb, axis=-1) @@ -212,21 +211,23 @@ class TestSGDOpWithLargeInput(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) compiled_prog = fluid.compiler.CompiledProgram( - fluid.default_main_program()) + fluid.default_main_program() + ) result = exe.run(compiled_prog, fetch_list=[avg_cost]) class TestSGDV2(unittest.TestCase): - def test_sgd_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.SGD(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.SGD( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.01, + ) out = linear(a) out.backward() adam.step() @@ -239,33 +240,31 @@ class TestSGDV2(unittest.TestCase): init_program = paddle.static.Program() program = paddle.static.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr=optimizer_attr) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr=optimizer_attr, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) sgd_optimizer = paddle.optimizer.SGD(learning_rate=0.01) opts, _ = sgd_optimizer.minimize(mean_out, init_program) return opts @@ -288,15 +287,18 @@ class TestSGDV2(unittest.TestCase): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.SGD(learning_rate=0.01, - parameters=[{ - 'params': linear_1.parameters() - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001, - 'learning_rate': 0.1 - }], - weight_decay=0.01) + adam = paddle.optimizer.SGD( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + }, + ], + weight_decay=0.01, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -310,15 +312,15 @@ class TestSGDV2(unittest.TestCase): class TestSGDMultiPrecision2_0(unittest.TestCase): - def dygraph_sgd_mp(self, mp): paddle.disable_static() paddle.seed(10) paddle.set_device('gpu') input = paddle.randn((2, 2)) model = paddle.nn.Linear(2, 2) - optimizer = paddle.optimizer.SGD(parameters=model.parameters(), - multi_precision=mp) + optimizer = paddle.optimizer.SGD( + parameters=model.parameters(), multi_precision=mp + ) if mp == True: model = paddle.amp.decorate(models=model, level='O2') scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -355,16 +357,17 @@ class TestSGDMultiPrecision2_0(unittest.TestCase): init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if mp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -377,9 +380,9 @@ class TestSGDMultiPrecision2_0(unittest.TestCase): x = np.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -389,28 +392,32 @@ class TestSGDMultiPrecision2_0(unittest.TestCase): "Test dygraph mode" output1_dy, params1_dy = self.dygraph_sgd_mp(mp=True) output2_dy, params2_dy = self.dygraph_sgd_mp(mp=False) - np.testing.assert_allclose(output1_dy.astype('float32').numpy(), - output2_dy.astype('float32').numpy(), - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + output1_dy.astype('float32').numpy(), + output2_dy.astype('float32').numpy(), + rtol=1e-05, + atol=0.1, + ) for idx in range(len(params1_dy)): np.testing.assert_allclose( params1_dy[idx].astype('float32').numpy(), params2_dy[idx].astype('float32').numpy(), rtol=1e-05, - atol=0.1) + atol=0.1, + ) "Test static mode" output1_st = self.static_sgd_mp(mp=True) output2_st = self.static_sgd_mp(mp=False) for idx in range(len(output1_st)): - np.testing.assert_allclose(output1_st[idx].astype('float32'), - output2_st[idx].astype('float32'), - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + output1_st[idx].astype('float32'), + output2_st[idx].astype('float32'), + rtol=1e-05, + atol=0.1, + ) class TestSGDMultiPrecision1_0(unittest.TestCase): - def dygraph_sgd_mp(self, mp): paddle.disable_static() paddle.seed(10) @@ -420,7 +427,8 @@ class TestSGDMultiPrecision1_0(unittest.TestCase): optimizer = paddle.fluid.optimizer.SGD( learning_rate=0.001, parameter_list=model.parameters(), - multi_precision=mp) + multi_precision=mp, + ) if mp == True: model = paddle.amp.decorate(models=model, level='O2') scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -449,8 +457,9 @@ class TestSGDMultiPrecision1_0(unittest.TestCase): exe = paddle.static.Executor('gpu') train_program = paddle.static.Program() startup_program = paddle.static.Program() - optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.001, - multi_precision=mp) + optimizer = paddle.fluid.optimizer.SGD( + learning_rate=0.001, multi_precision=mp + ) if mp: optimizer = paddle.static.amp.decorate( @@ -458,16 +467,17 @@ class TestSGDMultiPrecision1_0(unittest.TestCase): init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if mp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -480,9 +490,9 @@ class TestSGDMultiPrecision1_0(unittest.TestCase): x = np.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -492,24 +502,29 @@ class TestSGDMultiPrecision1_0(unittest.TestCase): "Test dygraph mode" output1_dy, params1_dy = self.dygraph_sgd_mp(mp=True) output2_dy, params2_dy = self.dygraph_sgd_mp(mp=False) - np.testing.assert_allclose(output1_dy.astype('float32').numpy(), - output2_dy.astype('float32').numpy(), - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + output1_dy.astype('float32').numpy(), + output2_dy.astype('float32').numpy(), + rtol=1e-05, + atol=0.1, + ) for idx in range(len(params1_dy)): np.testing.assert_allclose( params1_dy[idx].astype('float32').numpy(), params2_dy[idx].astype('float32').numpy(), rtol=1e-05, - atol=0.1) + atol=0.1, + ) "Test static mode" output1_st = self.static_sgd_mp(mp=True) output2_st = self.static_sgd_mp(mp=False) for idx in range(len(output1_st)): - np.testing.assert_allclose(output1_st[idx].astype('float32'), - output2_st[idx].astype('float32'), - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + output1_st[idx].astype('float32'), + output2_st[idx].astype('float32'), + rtol=1e-05, + atol=0.1, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py b/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py index 3eb75a9b140..ae7e87d1b3f 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py @@ -17,18 +17,21 @@ import numpy as np import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator -from paddle.fluid.tests.unittests.op_test import (convert_float_to_uint16, - convert_uint16_to_float, - OpTest, OpTestTool) +from paddle.fluid.tests.unittests.op_test import ( + convert_float_to_uint16, + convert_uint16_to_float, + OpTest, + OpTestTool, +) import paddle import paddle.static.amp as amp import struct -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSGDOpBF16(OpTest): - def setUp(self): self.op_type = 'sgd' self.dtype = np.uint16 @@ -53,17 +56,16 @@ class TestSGDOpBF16(OpTest): self.check_output_with_place(core.CPUPlace(), check_dygraph=False) -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSGDOpBF16Case2(TestSGDOpBF16): - def conf(self): self.h = 10 self.w = 64 class TestSparseSGDOpBF16(unittest.TestCase): - @classmethod def setUpClass(cls): np.random.seed(12345) @@ -121,10 +123,10 @@ class TestSparseSGDOpBF16(unittest.TestCase): return lr_tensor, lr_value -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSparseGradSGDOpBF16(TestSparseSGDOpBF16): - def setUp(self): self.setup_params() @@ -136,32 +138,35 @@ class TestSparseGradSGDOpBF16(TestSparseSGDOpBF16): def test_sparse_grad_sgd(self): scope = core.Scope() place = core.CPUPlace() - _, grad_array = self.create_sparse_grad_var(scope, place, - self.grad_height, - self.grad_rows, - self.grad_row_numel) + _, grad_array = self.create_sparse_grad_var( + scope, place, self.grad_height, self.grad_rows, self.grad_row_numel + ) param_tensor, param_array = self.create_dense_param_var( - scope, place, self.grad_height, self.grad_row_numel) + scope, place, self.grad_height, self.grad_row_numel + ) _, lr_value = self.create_dense_lr_var(scope, place) - sgd_op = Operator('sgd', - Param='Param', - Grad='Grad', - ParamOut='Param', - LearningRate='LearningRate', - use_mkldnn=True) + sgd_op = Operator( + 'sgd', + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate', + use_mkldnn=True, + ) sgd_op.run(scope, place) - reference = self.ref_optimize(param_array, self.grad_rows, grad_array, - lr_value) + reference = self.ref_optimize( + param_array, self.grad_rows, grad_array, lr_value + ) output = np.array(param_tensor) self.check_output(output, reference, atol=5e-3, rtol=1e-1) -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSparseGradSGDOpBF16Case2(TestSparseGradSGDOpBF16): - def setup_params(self): self.grad_height = 14 self.grad_rows = [1, 4, 12, 7, 8] @@ -169,17 +174,16 @@ class TestSparseGradSGDOpBF16Case2(TestSparseGradSGDOpBF16): class TestSparseGradSGDOpBF16Case3(TestSparseGradSGDOpBF16): - def setup_params(self): self.grad_height = 10 self.grad_rows = [0, 4, 7] self.grad_row_numel = 120 -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSparseGradParamSGDOpBF16(TestSparseSGDOpBF16): - def setUp(self): self.setup_params() @@ -192,31 +196,32 @@ class TestSparseGradParamSGDOpBF16(TestSparseSGDOpBF16): def test_sparse_param_grad_sgd(self): scope = core.Scope() place = core.CPUPlace() - _, grad_array = self.create_sparse_grad_var(scope, place, - self.grad_height, - self.grad_rows, - self.grad_row_numel) + _, grad_array = self.create_sparse_grad_var( + scope, place, self.grad_height, self.grad_rows, self.grad_row_numel + ) param_tensor, param_array = self.create_sparse_param_var( - scope, place, self.grad_height, self.param_rows, - self.grad_row_numel) + scope, place, self.grad_height, self.param_rows, self.grad_row_numel + ) _, lr_value = self.create_dense_lr_var(scope, place) - sgd_op = Operator('sgd', - Param='Param', - Grad='Grad', - ParamOut='Param', - LearningRate='LearningRate', - use_mkldnn=True) + sgd_op = Operator( + 'sgd', + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate', + use_mkldnn=True, + ) sgd_op.run(scope, place) - reference = self.ref_optimize(param_array, self.grad_rows, grad_array, - lr_value) + reference = self.ref_optimize( + param_array, self.grad_rows, grad_array, lr_value + ) output = np.array(param_tensor) self.check_output(output, reference, atol=5e-3, rtol=1e-1) class TestSparseGradParamSGDOpBF16Case2(TestSparseGradParamSGDOpBF16): - def setup_params(self): self.grad_height = 14 self.grad_rows = [1, 4, 12, 7, 8] @@ -226,7 +231,6 @@ class TestSparseGradParamSGDOpBF16Case2(TestSparseGradParamSGDOpBF16): @OpTestTool.skip_if_not_cpu_bf16() class TestSGDOpBF16API(unittest.TestCase): - @classmethod def setUpClass(cls): np.random.seed(12345) @@ -259,18 +263,20 @@ class TestSGDOpBF16API(unittest.TestCase): return self._fp322bf16(self._bf162fp32(lhs) * self._bf162fp32(rhs)) def _reference(self, data, emb_weight, bf16=False): - emb_out_shape = np.array([self.ids_shape[0], self.w_shape[1]], - dtype=np.int64) - mean_grad_value = np.float32(1.0) / np.prod(emb_out_shape, - dtype=np.float32) + emb_out_shape = np.array( + [self.ids_shape[0], self.w_shape[1]], dtype=np.int64 + ) + mean_grad_value = np.float32(1.0) / np.prod( + emb_out_shape, dtype=np.float32 + ) if bf16: - mean_grad = np.full(emb_out_shape, - self._fp322bf16(mean_grad_value), - dtype=np.uint16) + mean_grad = np.full( + emb_out_shape, self._fp322bf16(mean_grad_value), dtype=np.uint16 + ) else: - mean_grad = np.full(emb_out_shape, - mean_grad_value, - dtype=np.float32) + mean_grad = np.full( + emb_out_shape, mean_grad_value, dtype=np.float32 + ) # add_grad = 1 * mean_grad out_dtype = np.uint16 if bf16 else np.float32 lookup_table_grad = np.zeros(self.w_shape, dtype=out_dtype) @@ -281,7 +287,8 @@ class TestSGDOpBF16API(unittest.TestCase): idxv = idx[0] for j in range(self.w_shape[1]): lookup_table_grad[idxv, j] = self._add_bf16( - lookup_table_grad[idxv, j], mean_grad[i, j]) + lookup_table_grad[idxv, j], mean_grad[i, j] + ) ref_grad = np.ndarray(shape=emb_weight.shape, dtype=np.uint16) lr_bf16 = self._fp322bf16(self.learning_rate) @@ -289,29 +296,26 @@ class TestSGDOpBF16API(unittest.TestCase): for i, row in enumerate(emb_weight): for j, val in enumerate(row): ref_grad[i, j] = self._sub_bf16( - val, self._mul_bf16(lr_bf16, lookup_table_grad[i, j])) + val, self._mul_bf16(lr_bf16, lookup_table_grad[i, j]) + ) else: for i, idx in enumerate(data): lookup_table_grad[idx, :] += mean_grad[i] ref_grad = emb_weight - self.learning_rate * lookup_table_grad return ref_grad - def _check_output(self, - actual, - reference, - bf16=False, - atol=0, - rtol=0.15e-2): + def _check_output( + self, actual, reference, bf16=False, atol=0, rtol=0.15e-2 + ): output = actual if bf16 else convert_uint16_to_float(actual) if bf16: np.testing.assert_allclose(output, reference, atol=atol, rtol=rtol) else: try: print('Compare with FP32 values:') - np.testing.assert_allclose(output, - reference, - atol=atol, - rtol=rtol) + np.testing.assert_allclose( + output, reference, atol=atol, rtol=rtol + ) except AssertionError as e: print(e) @@ -329,56 +333,61 @@ class TestSGDOpBF16API(unittest.TestCase): main = fluid.Program() with fluid.program_guard(main): x = fluid.layers.data(name='X', shape=self.ids_shape, dtype='int64') - label = fluid.layers.data(name='Y', - shape=self.y_shape, - dtype='uint16') - emb = fluid.layers.embedding(input=x, - size=self.w_shape, - param_attr=fluid.ParamAttr( - name="emb_weight", - initializer=self.initializer), - is_sparse=False, - dtype="uint16") # bfloat16 + label = fluid.layers.data( + name='Y', shape=self.y_shape, dtype='uint16' + ) + emb = fluid.layers.embedding( + input=x, + size=self.w_shape, + param_attr=fluid.ParamAttr( + name="emb_weight", initializer=self.initializer + ), + is_sparse=False, + dtype="uint16", + ) # bfloat16 cost = fluid.layers.elementwise_add(emb, label) avg_cost = paddle.mean(cost) sgd_optimizer = paddle.optimizer.SGD( - learning_rate=self.learning_rate) + learning_rate=self.learning_rate + ) sgd_optimizer = amp.bf16.decorate_bf16( sgd_optimizer, amp_lists=amp.bf16.AutoMixedPrecisionListsBF16( custom_bf16_list={ 'lookup_table', - }), + } + ), use_bf16_guard=False, - use_pure_bf16=True) + use_pure_bf16=True, + ) sgd_optimizer.minimize( - avg_cost, startup_program=fluid.default_startup_program()) + avg_cost, startup_program=fluid.default_startup_program() + ) train_reader = paddle.batch(self._data_reader, batch_size=1) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) test_prog = main.clone(for_test=True) - sgd_optimizer.amp_init(place, - test_program=test_prog, - use_bf16_test=True) + sgd_optimizer.amp_init( + place, test_program=test_prog, use_bf16_test=True + ) ref_emb = np.full(self.w_shape, self.value, dtype=np.float32) - ref_emb_bf16 = np.full(self.w_shape, - self._fp322bf16(self.value), - dtype=np.uint16) + ref_emb_bf16 = np.full( + self.w_shape, self._fp322bf16(self.value), dtype=np.uint16 + ) emb_weight = [] for sample in train_reader(): data = sample[0][0] label = sample[0][1] y_bf16 = convert_float_to_uint16(label) - emb_weight = exe.run(main, - feed={ - 'X': data, - 'Y': y_bf16 - }, - fetch_list=['emb_weight']) + emb_weight = exe.run( + main, + feed={'X': data, 'Y': y_bf16}, + fetch_list=['emb_weight'], + ) ref_emb = self._reference(data, ref_emb) ref_emb_bf16 = self._reference(data, ref_emb_bf16, True) diff --git a/python/paddle/fluid/tests/unittests/test_sgn.py b/python/paddle/fluid/tests/unittests/test_sgn.py index dacfe7aa1d2..75d5d1b7847 100644 --- a/python/paddle/fluid/tests/unittests/test_sgn.py +++ b/python/paddle/fluid/tests/unittests/test_sgn.py @@ -29,40 +29,49 @@ def np_sgn(x: np.ndarray): class TestSgnError(unittest.TestCase): - def test_errors(self): # The input dtype of sgn must be float16, float32, float64,complex64,complex128. input2 = paddle.to_tensor( - np.random.randint(-10, 10, size=[12, 20]).astype('int32')) + np.random.randint(-10, 10, size=[12, 20]).astype('int32') + ) input3 = paddle.to_tensor( - np.random.randint(-10, 10, size=[12, 20]).astype('int64')) + np.random.randint(-10, 10, size=[12, 20]).astype('int64') + ) self.assertRaises(TypeError, paddle.sgn, input2) self.assertRaises(TypeError, paddle.sgn, input3) class TestSignAPI(unittest.TestCase): - def setUp(self) -> None: self.support_dtypes = [ - 'float16', 'float32', 'float64', 'complex64', 'complex128' + 'float16', + 'float32', + 'float64', + 'complex64', + 'complex128', ] if paddle.device.get_device() == 'cpu': self.support_dtypes = [ - 'float32', 'float64', 'complex64', 'complex128' + 'float32', + 'float64', + 'complex64', + 'complex128', ] def test_dtype(self): for dtype in self.support_dtypes: x = paddle.to_tensor( - np.random.randint(-10, 10, size=[12, 20, 2]).astype(dtype)) + np.random.randint(-10, 10, size=[12, 20, 2]).astype(dtype) + ) paddle.sgn(x) def test_complex(self): for dtype in ['complex64', 'complex128']: - np_x = np.array([[3 + 4j, 7 - 24j, 0, 1 + 2j], [6 + 8j, 3, 0, -2]], - dtype=dtype) + np_x = np.array( + [[3 + 4j, 7 - 24j, 0, 1 + 2j], [6 + 8j, 3, 0, -2]], dtype=dtype + ) x = paddle.to_tensor(np_x) z = paddle.sgn(x) np_z = z.numpy() diff --git a/python/paddle/fluid/tests/unittests/test_shape_op.py b/python/paddle/fluid/tests/unittests/test_shape_op.py index 8bee609623e..f4de948d229 100644 --- a/python/paddle/fluid/tests/unittests/test_shape_op.py +++ b/python/paddle/fluid/tests/unittests/test_shape_op.py @@ -21,7 +21,6 @@ from paddle.fluid.op import Operator class TestShapeOp(OpTest): - def setUp(self): self.op_type = "shape" self.python_api = paddle.shape @@ -39,19 +38,16 @@ class TestShapeOp(OpTest): class case1(TestShapeOp): - def config(self): self.shape = [2] class case2(TestShapeOp): - def config(self): self.shape = [1, 2, 3] class TestShapeWithSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_shard_index_op.py b/python/paddle/fluid/tests/unittests/test_shard_index_op.py index 5c51b3cf58e..8c13e2b44c3 100644 --- a/python/paddle/fluid/tests/unittests/test_shard_index_op.py +++ b/python/paddle/fluid/tests/unittests/test_shard_index_op.py @@ -37,13 +37,12 @@ def common_setup(self, index_num, nshards, shard_id, ignore_value): 'index_num': index_num, 'nshards': nshards, 'shard_id': shard_id, - 'ignore_value': ignore_value + 'ignore_value': ignore_value, } self.outputs = {'Out': (out, x_lod)} class TestShardIndexShardId0Op(OpTest): - def setUp(self): common_setup(self, 20, 2, 0, -1) @@ -52,7 +51,6 @@ class TestShardIndexShardId0Op(OpTest): class TestShardIndexShardId1Op(OpTest): - def setUp(self): common_setup(self, 20, 2, 1, -1) @@ -61,7 +59,6 @@ class TestShardIndexShardId1Op(OpTest): class TestShardIndexIgnoreValueOp(OpTest): - def setUp(self): common_setup(self, 20, 2, 0, -2) @@ -70,7 +67,6 @@ class TestShardIndexIgnoreValueOp(OpTest): class TestShardIndexNotEvenlyDividedOp(OpTest): - def setUp(self): common_setup(self, 15, 2, 1, -1) diff --git a/python/paddle/fluid/tests/unittests/test_share_data_op.py b/python/paddle/fluid/tests/unittests/test_share_data_op.py index 56ac3af6d85..b6f2d1ecbec 100644 --- a/python/paddle/fluid/tests/unittests/test_share_data_op.py +++ b/python/paddle/fluid/tests/unittests/test_share_data_op.py @@ -20,7 +20,6 @@ from paddle.fluid.op import Operator class TestShareDataOp(OpTest): - def setUp(self): self.op_type = "share_data" input = np.random.rand(2, 3, 5).astype("float32") @@ -32,7 +31,6 @@ class TestShareDataOp(OpTest): class TestShareDataOpOnDifferentPlaces(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py index 9e765859127..c30c2095c9b 100644 --- a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py +++ b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py @@ -27,16 +27,14 @@ from paddle.fluid.layers.control_flow import lod_rank_table class TestShrinkRNNMemoryBase(unittest.TestCase): - def setUp(self): self.main_program = Program() switch_main_program(self.main_program) x = layers.data('x', shape=[100], dtype='float32') x.stop_gradient = False - rank_table_tensor = layers.data('rank_table_tensor', - shape=[1], - dtype='float32', - lod_level=1) + rank_table_tensor = layers.data( + 'rank_table_tensor', shape=[1], dtype='float32', lod_level=1 + ) table = lod_rank_table(x=rank_table_tensor) i = layers.zeros(dtype='int64', shape=[1]) self.mem1 = shrink_memory(x=x, i=i, table=table) @@ -58,7 +56,6 @@ class TestShrinkRNNMemoryBase(unittest.TestCase): class TestShrinkRNNMemoryReferLoD(TestShrinkRNNMemoryBase): - def test_refer_lod(self): cpu = core.CPUPlace() x_tensor = core.LoDTensor() @@ -69,16 +66,15 @@ class TestShrinkRNNMemoryReferLoD(TestShrinkRNNMemoryBase): rank_table_tensor = core.LoDTensor() rank_table_tensor.set_recursive_sequence_lengths([[1, 2, 3]]) rank_table_tensor.set( - np.random.random(size=(6, 1)).astype('float32'), cpu) + np.random.random(size=(6, 1)).astype('float32'), cpu + ) exe = Executor(cpu) outs = exe.run( - feed={ - 'x': x_tensor, - 'rank_table_tensor': rank_table_tensor - }, + feed={'x': x_tensor, 'rank_table_tensor': rank_table_tensor}, fetch_list=[self.mem1, self.mem2, self.mem3, self.x_grad], - return_numpy=False) + return_numpy=False, + ) np.testing.assert_allclose(tensor_np[0:6], outs[0], rtol=1e-05) np.testing.assert_allclose(tensor_np[0:5], outs[1], rtol=1e-05) np.testing.assert_allclose(tensor_np[0:2], outs[2], rtol=1e-05) @@ -86,7 +82,6 @@ class TestShrinkRNNMemoryReferLoD(TestShrinkRNNMemoryBase): class TestShrinkRNNMemoryNoLoD(TestShrinkRNNMemoryBase): - def test_no_lod(self): cpu = core.CPUPlace() x_tensor = core.LoDTensor() @@ -96,16 +91,15 @@ class TestShrinkRNNMemoryNoLoD(TestShrinkRNNMemoryBase): rank_table_tensor = core.LoDTensor() rank_table_tensor.set_recursive_sequence_lengths([[1, 2, 3]]) rank_table_tensor.set( - np.random.random(size=(6, 1)).astype('float32'), cpu) + np.random.random(size=(6, 1)).astype('float32'), cpu + ) exe = Executor(cpu) outs = exe.run( - feed={ - 'x': x_tensor, - 'rank_table_tensor': rank_table_tensor - }, + feed={'x': x_tensor, 'rank_table_tensor': rank_table_tensor}, fetch_list=[self.mem1, self.mem2, self.mem3, self.x_grad], - return_numpy=False) + return_numpy=False, + ) np.testing.assert_allclose(tensor_np[0:3], outs[0], rtol=1e-05) np.testing.assert_allclose(tensor_np[0:2], outs[1], rtol=1e-05) np.testing.assert_allclose(tensor_np[0:1], outs[2], rtol=1e-05) @@ -113,7 +107,6 @@ class TestShrinkRNNMemoryNoLoD(TestShrinkRNNMemoryBase): class TestShrinkRNNMemoryOpError(unittest.TestCase): - def test_erroes(self): with program_guard(Program(), Program()): x = layers.zeros(dtype='int64', shape=[3, 100]) @@ -121,8 +114,8 @@ class TestShrinkRNNMemoryOpError(unittest.TestCase): rank_table_tensor = core.LoDTensor() rank_table_tensor.set_recursive_sequence_lengths([[1, 2, 3]]) rank_table_tensor.set( - np.random.random(size=(6, 1)).astype('float32'), - core.CPUPlace()) + np.random.random(size=(6, 1)).astype('float32'), core.CPUPlace() + ) rank_table = np.random.random(size=(6, 1)).astype('float32') # The type of x in shrink_rnn_memory must be Variable. diff --git a/python/paddle/fluid/tests/unittests/test_shuffle_batch_op.py b/python/paddle/fluid/tests/unittests/test_shuffle_batch_op.py index 7752cfa3a65..95fc233239a 100644 --- a/python/paddle/fluid/tests/unittests/test_shuffle_batch_op.py +++ b/python/paddle/fluid/tests/unittests/test_shuffle_batch_op.py @@ -21,7 +21,6 @@ import os class TestShuffleBatchOpBase(OpTest): - def gen_random_array(self, shape, low=0, high=1): rnd = (high - low) * np.random.random(shape) + low return rnd.astype(self.dtype) @@ -40,8 +39,9 @@ class TestShuffleBatchOpBase(OpTest): self.dtype = np.float64 self.shape = self.get_shape() x = self.gen_random_array(self.shape) - seed = np.random.random_integers(low=10, high=100, - size=(1, )).astype('int64') + seed = np.random.random_integers(low=10, high=100, size=(1,)).astype( + 'int64' + ) self.inputs = {'X': x, 'Seed': seed} self.outputs = { 'Out': np.array([]).astype(x.dtype), @@ -78,7 +78,6 @@ class TestShuffleBatchOpBase(OpTest): class TestShuffleBatchOp2(TestShuffleBatchOpBase): - def get_shape(self): return (4, 30) diff --git a/python/paddle/fluid/tests/unittests/test_shuffle_channel_op.py b/python/paddle/fluid/tests/unittests/test_shuffle_channel_op.py index 594df3f6c09..01bdd48ff24 100644 --- a/python/paddle/fluid/tests/unittests/test_shuffle_channel_op.py +++ b/python/paddle/fluid/tests/unittests/test_shuffle_channel_op.py @@ -18,7 +18,6 @@ from op_test import OpTest class TestShuffleChannelOp(OpTest): - def setUp(self): self.op_type = "shuffle_channel" self.batch_size = 10 @@ -27,13 +26,14 @@ class TestShuffleChannelOp(OpTest): self.layer_w = 4 self.group = 4 self.x = np.random.random( - (self.batch_size, self.input_channels, self.layer_h, - self.layer_w)).astype('float32') + (self.batch_size, self.input_channels, self.layer_h, self.layer_w) + ).astype('float32') self.inputs = {'X': self.x} self.attrs = {'group': self.group} n, c, h, w = self.x.shape - input_reshaped = np.reshape(self.x, - (-1, self.group, c // self.group, h, w)) + input_reshaped = np.reshape( + self.x, (-1, self.group, c // self.group, h, w) + ) input_transposed = np.transpose(input_reshaped, (0, 2, 1, 3, 4)) self.outputs = {'Out': np.reshape(input_transposed, (-1, c, h, w))} diff --git a/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py b/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py index f6ea5018ea9..1d09aefd2d8 100644 --- a/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py +++ b/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py @@ -24,12 +24,12 @@ import paddle def test_fluid_sigmoid(x, label, normalize=False, ignore_index=-100): return paddle.fluid.layers.sigmoid_cross_entropy_with_logits( - x, label, int(ignore_index), normalize=normalize) + x, label, int(ignore_index), normalize=normalize + ) class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -37,12 +37,14 @@ class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.randint(0, 2, (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.randint(0, 2, (batch_size, num_classes)).astype( + "float64" + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -61,8 +63,7 @@ class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): class TestSigmoidCrossEntropyWithLogitsOp2(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -71,13 +72,14 @@ class TestSigmoidCrossEntropyWithLogitsOp2(OpTest): num_classes = 20 ignore_index = -1 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.randint(-1, 2, - (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.randint(-1, 2, (batch_size, num_classes)).astype( + "float64" + ), } self.attrs = { 'ignore_index': ignore_index, @@ -100,8 +102,7 @@ class TestSigmoidCrossEntropyWithLogitsOp2(OpTest): class TestSigmoidCrossEntropyWithLogitsOp3(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -109,12 +110,14 @@ class TestSigmoidCrossEntropyWithLogitsOp3(OpTest): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.uniform(0, 1, (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -133,7 +136,6 @@ class TestSigmoidCrossEntropyWithLogitsOp3(OpTest): class TestSigmoidCrossEntropyWithNorm(OpTest): - def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" self.python_api = test_fluid_sigmoid @@ -141,13 +143,14 @@ class TestSigmoidCrossEntropyWithNorm(OpTest): num_classes = 20 ignore_index = -1 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.randint(-1, 2, - (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.randint(-1, 2, (batch_size, num_classes)).astype( + "float64" + ), } self.attrs = {'ignore_index': ignore_index, 'normalize': True} sigmoid_X = expit(self.inputs['X']) @@ -157,7 +160,8 @@ class TestSigmoidCrossEntropyWithNorm(OpTest): out[np.where(self.inputs['Label'] == ignore_index)] = 0 if self.attrs['normalize']: out = out / float( - np.where(self.inputs['Label'] != ignore_index)[0].size) + np.where(self.inputs['Label'] != ignore_index)[0].size + ) self.outputs = {'Out': out} def test_check_output(self): @@ -168,8 +172,7 @@ class TestSigmoidCrossEntropyWithNorm(OpTest): class TestSigmoidCrossEntropyWithLogitsOp5(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -177,13 +180,14 @@ class TestSigmoidCrossEntropyWithLogitsOp5(OpTest): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform( - 0, 1, tuple(batch_size + [num_classes])).astype("float64")), - 'Label': - np.random.uniform(0, 1, tuple(batch_size + - [num_classes])).astype("float64") + 0, 1, tuple(batch_size + [num_classes]) + ).astype("float64") + ), + 'Label': np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype("float64"), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -202,7 +206,6 @@ class TestSigmoidCrossEntropyWithLogitsOp5(OpTest): class TestSigmoidCrossEntropyWithNorm2(OpTest): - def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" self.python_api = test_fluid_sigmoid @@ -210,13 +213,14 @@ class TestSigmoidCrossEntropyWithNorm2(OpTest): num_classes = 20 ignore_index = -1 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform( - 0, 1, tuple(batch_size + [num_classes])).astype("float64")), - 'Label': - np.random.randint(-1, 2, tuple(batch_size + - [num_classes])).astype("float64") + 0, 1, tuple(batch_size + [num_classes]) + ).astype("float64") + ), + 'Label': np.random.randint( + -1, 2, tuple(batch_size + [num_classes]) + ).astype("float64"), } self.attrs = {'ignore_index': ignore_index, 'normalize': True} sigmoid_X = expit(self.inputs['X']) @@ -226,7 +230,8 @@ class TestSigmoidCrossEntropyWithNorm2(OpTest): out[np.where(self.inputs['Label'] == ignore_index)] = 0 if self.attrs['normalize']: out = out / float( - np.where(self.inputs['Label'] != ignore_index)[0].size) + np.where(self.inputs['Label'] != ignore_index)[0].size + ) self.outputs = {'Out': out} def test_check_output(self): @@ -236,8 +241,7 @@ class TestSigmoidCrossEntropyWithNorm2(OpTest): self.check_grad(['X'], 'Out', check_eager=True) class TestSigmoidCrossEntropyWithLogitsOp6(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -245,14 +249,14 @@ class TestSigmoidCrossEntropyWithNorm2(OpTest): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + - [num_classes])).astype("float64")), - 'Label': - np.random.randint(0, 2, tuple(batch_size + - [num_classes])).astype("float64") + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype("float64") + ), + 'Label': np.random.randint( + 0, 2, tuple(batch_size + [num_classes]) + ).astype("float64"), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -270,18 +274,21 @@ class TestSigmoidCrossEntropyWithNorm2(OpTest): self.check_grad(['X'], 'Out', check_eager=True) class TestSigmoidCrossEntropyWithLogitsOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of sigmoid_cross_entropy_with_logits must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, - 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) - lab1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), + [[1, 1, 1, 1]], + fluid.CPUPlace(), + ) + lab1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), + [[1, 1, 1, 1]], + fluid.CPUPlace(), + ) fluid.layers.sigmoid_cross_entropy_with_logits(x1, lab1) self.assertRaises(TypeError, test_Variable) @@ -289,12 +296,12 @@ class TestSigmoidCrossEntropyWithNorm2(OpTest): def test_dtype(): # the input dtype of sigmoid_cross_entropy_with_logits must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") - lab2 = fluid.layers.data(name='lab2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) + lab2 = fluid.layers.data( + name='lab2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.sigmoid_cross_entropy_with_logits(x2, lab2) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss.py b/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss.py index a3b614cb51f..e36ba383c07 100644 --- a/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss.py +++ b/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss.py @@ -19,84 +19,83 @@ import unittest from paddle.fluid.framework import _test_eager_guard -def call_sfl_functional(logit, - label, - normalizer, - alpha=0.25, - gamma=2.0, - reduction='sum'): - res = paddle.nn.functional.sigmoid_focal_loss(logit, - label, - normalizer, - alpha=alpha, - gamma=gamma, - reduction=reduction) +def call_sfl_functional( + logit, label, normalizer, alpha=0.25, gamma=2.0, reduction='sum' +): + res = paddle.nn.functional.sigmoid_focal_loss( + logit, label, normalizer, alpha=alpha, gamma=gamma, reduction=reduction + ) return res -def test_static(place, - logit_np, - label_np, - normalizer_np, - alpha=0.25, - gamma=2.0, - reduction='sum'): +def test_static( + place, + logit_np, + label_np, + normalizer_np, + alpha=0.25, + gamma=2.0, + reduction='sum', +): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - logit = paddle.fluid.data(name='logit', - shape=logit_np.shape, - dtype='float64') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float64') + logit = paddle.fluid.data( + name='logit', shape=logit_np.shape, dtype='float64' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float64' + ) feed_dict = {"logit": logit_np, "label": label_np} normalizer = None if normalizer_np is not None: - normalizer = paddle.fluid.data(name='normalizer', - shape=normalizer_np.shape, - dtype='float64') + normalizer = paddle.fluid.data( + name='normalizer', shape=normalizer_np.shape, dtype='float64' + ) feed_dict["normalizer"] = normalizer_np - res = call_sfl_functional(logit, label, normalizer, alpha, gamma, - reduction) + res = call_sfl_functional( + logit, label, normalizer, alpha, gamma, reduction + ) exe = paddle.static.Executor(place) static_result = exe.run(prog, feed=feed_dict, fetch_list=[res]) return static_result -def test_dygraph(place, - logit_np, - label_np, - normalizer_np, - alpha=0.25, - gamma=2.0, - reduction='sum'): +def test_dygraph( + place, + logit_np, + label_np, + normalizer_np, + alpha=0.25, + gamma=2.0, + reduction='sum', +): paddle.disable_static() logit = paddle.to_tensor(logit_np) label = paddle.to_tensor(label_np) normalizer = None if normalizer_np is not None: normalizer = paddle.to_tensor(normalizer_np) - dy_res = call_sfl_functional(logit, label, normalizer, alpha, gamma, - reduction) + dy_res = call_sfl_functional( + logit, label, normalizer, alpha, gamma, reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result -def calc_sigmoid_focal_loss(logit_np, - label_np, - normalizer_np, - alpha=0.25, - gamma=2.0, - reduction='sum'): +def calc_sigmoid_focal_loss( + logit_np, label_np, normalizer_np, alpha=0.25, gamma=2.0, reduction='sum' +): - loss = np.maximum( - logit_np, - 0) - logit_np * label_np + np.log(1 + np.exp(-np.abs(logit_np))) + loss = ( + np.maximum(logit_np, 0) + - logit_np * label_np + + np.log(1 + np.exp(-np.abs(logit_np))) + ) pred = 1 / (1 + np.exp(-logit_np)) p_t = pred * label_np + (1 - pred) * (1 - label_np) @@ -106,7 +105,7 @@ def calc_sigmoid_focal_loss(logit_np, loss = alpha_t * loss if gamma is not None: - loss = loss * ((1 - p_t)**gamma) + loss = loss * ((1 - p_t) ** gamma) if normalizer_np is not None: loss = loss / normalizer_np @@ -120,62 +119,89 @@ def calc_sigmoid_focal_loss(logit_np, class TestSigmoidFocalLoss(unittest.TestCase): - def test_SigmoidFocalLoss(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float64) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float64) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float64 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float64 + ) normalizer_nps = [ - np.asarray([np.sum(label_np > 0)], dtype=label_np.dtype), None + np.asarray([np.sum(label_np > 0)], dtype=label_np.dtype), + None, ] places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) reductions = ['sum', 'mean', 'none'] alphas = [0.25, 0.5] - gammas = [3, 0.] + gammas = [3, 0.0] for place in places: for reduction in reductions: for alpha in alphas: for gamma in gammas: for normalizer_np in normalizer_nps: - static_result, = test_static( - place, logit_np, label_np, normalizer_np, alpha, - gamma, reduction) - dy_result = test_dygraph(place, logit_np, label_np, - normalizer_np, alpha, - gamma, reduction) + (static_result,) = test_static( + place, + logit_np, + label_np, + normalizer_np, + alpha, + gamma, + reduction, + ) + dy_result = test_dygraph( + place, + logit_np, + label_np, + normalizer_np, + alpha, + gamma, + reduction, + ) with _test_eager_guard(): eager_result = test_dygraph( - place, logit_np, label_np, normalizer_np, - alpha, gamma, reduction) + place, + logit_np, + label_np, + normalizer_np, + alpha, + gamma, + reduction, + ) expected = calc_sigmoid_focal_loss( - logit_np, label_np, normalizer_np, alpha, gamma, - reduction) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-05) - np.testing.assert_allclose(dy_result, - expected, - rtol=1e-05) - np.testing.assert_allclose(eager_result, - expected, - rtol=1e-05) + logit_np, + label_np, + normalizer_np, + alpha, + gamma, + reduction, + ) + np.testing.assert_allclose( + static_result, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-05 + ) + np.testing.assert_allclose( + dy_result, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + eager_result, expected, rtol=1e-05 + ) def test_SigmoidFocalLoss_error(self): paddle.disable_static() logit = paddle.to_tensor([[0.97], [0.91], [0.03]], dtype='float32') label = paddle.to_tensor([[1.0], [1.0], [0.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.sigmoid_focal_loss, - logit=logit, - label=label, - normalizer=None, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.sigmoid_focal_loss, + logit=logit, + label=label, + normalizer=None, + reduction="unsupport reduction", + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss_op.py b/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss_op.py index 91b2981acd5..4b9831f8e8d 100644 --- a/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss_op.py @@ -22,8 +22,9 @@ from paddle.fluid import core from paddle.fluid import Program, program_guard -def sigmoid_focal_loss_forward(x_data, label_data, fg_num_data, gamma, alpha, - num_classes): +def sigmoid_focal_loss_forward( + x_data, label_data, fg_num_data, gamma, alpha, num_classes +): x_data_t = copy.deepcopy(x_data) out_data = copy.deepcopy(x_data) x_width = len(x_data) @@ -41,12 +42,13 @@ def sigmoid_focal_loss_forward(x_data, label_data, fg_num_data, gamma, alpha, z_neg = (1.0 - alpha) / fg_num z_pos = alpha / fg_num - p = 1. / (1. + math.exp(-x)) + p = 1.0 / (1.0 + math.exp(-x)) FLT_MIN = 1.175494351e-38 - term_pos = math.pow((1. - p), gamma) * math.log(max(FLT_MIN, p)) - term_neg = math.pow(p, gamma) * (-1. * x * (x >= 0) - - math.log(1. + math.exp(x - 2. * x * - (x >= 0)))) + term_pos = math.pow((1.0 - p), gamma) * math.log(max(FLT_MIN, p)) + term_neg = math.pow(p, gamma) * ( + -1.0 * x * (x >= 0) + - math.log(1.0 + math.exp(x - 2.0 * x * (x >= 0))) + ) out_data[idx] = 0.0 out_data[idx] += -c_pos * term_pos * z_pos out_data[idx] += -c_neg * term_neg * z_neg @@ -56,7 +58,6 @@ def sigmoid_focal_loss_forward(x_data, label_data, fg_num_data, gamma, alpha, class TestSigmoidFocalLossOp1(OpTest): - def set_argument(self): self.num_anchors = 10 self.num_classes = 10 @@ -68,8 +69,9 @@ class TestSigmoidFocalLossOp1(OpTest): dims = (self.num_anchors, self.num_classes) X = np.random.standard_normal(dims).astype("float64") - L = np.random.randint(0, self.num_classes + 1, - (dims[0], 1)).astype("int32") + L = np.random.randint(0, self.num_classes + 1, (dims[0], 1)).astype( + "int32" + ) F = np.zeros(1) F[0] = len(np.where(L > 0)[0]) F = F.astype("int32") @@ -84,10 +86,14 @@ class TestSigmoidFocalLossOp1(OpTest): 'gamma': self.gamma, 'alpha': self.alpha, } - loss = sigmoid_focal_loss_forward(self.inputs['X'], - self.inputs['Label'], - self.inputs['FgNum'], self.gamma, - self.alpha, self.num_classes) + loss = sigmoid_focal_loss_forward( + self.inputs['X'], + self.inputs['Label'], + self.inputs['FgNum'], + self.gamma, + self.alpha, + self.num_classes, + ) self.outputs = {'Out': loss.astype('float64')} def test_check_output(self): @@ -97,23 +103,22 @@ class TestSigmoidFocalLossOp1(OpTest): self.check_grad(['X'], 'Out') -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSigmoidFocalLossOp2(TestSigmoidFocalLossOp1): - def test_check_output(self): place = core.CUDAPlace(0) self.check_output_with_place(place, atol=2e-3) def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.002) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=0.002 + ) class TestSigmoidFocalLossOp3(TestSigmoidFocalLossOp1): - def set_argument(self): self.num_anchors = 200 self.num_classes = 10 @@ -121,50 +126,45 @@ class TestSigmoidFocalLossOp3(TestSigmoidFocalLossOp1): self.alpha = 0.5 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSigmoidFocalLossOp4(TestSigmoidFocalLossOp3): - def test_check_output(self): place = core.CUDAPlace(0) self.check_output_with_place(place, atol=2e-3) def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.002) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=0.002 + ) class TestSigmoidFocalLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - label1 = fluid.layers.fill_constant(shape=[10, 1], - dtype="int32", - value=1) - fg_num1 = fluid.layers.fill_constant(shape=[1], - dtype="int32", - value=5) + label1 = fluid.layers.fill_constant( + shape=[10, 1], dtype="int32", value=1 + ) + fg_num1 = fluid.layers.fill_constant( + shape=[1], dtype="int32", value=5 + ) # The `x` must be Variable and the data type of `x` Tensor must be one of float32 and float64. def test_x_type(): x1 = [2] - fluid.layers.sigmoid_focal_loss(x=x1, - label=label1, - fg_num=fg_num1, - gamma=2., - alpha=0.25) + fluid.layers.sigmoid_focal_loss( + x=x1, label=label1, fg_num=fg_num1, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_x_type) def test_x_tensor_dtype(): x2 = fluid.layers.data(name='x2', shape=[10, 10], dtype="int16") - fluid.layers.sigmoid_focal_loss(x=x2, - label=label1, - fg_num=fg_num1, - gamma=2., - alpha=0.25) + fluid.layers.sigmoid_focal_loss( + x=x2, label=label1, fg_num=fg_num1, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_x_tensor_dtype) @@ -173,46 +173,38 @@ class TestSigmoidFocalLossOpError(unittest.TestCase): # The `label` must be Variable and the data type of `label` Tensor must be int32. def test_label_type(): label2 = [2] - fluid.layers.sigmoid_focal_loss(x=x3, - label=label2, - fg_num=fg_num1, - gamma=2., - alpha=0.25) + fluid.layers.sigmoid_focal_loss( + x=x3, label=label2, fg_num=fg_num1, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_label_type) def test_label_tensor_dtype(): - label3 = fluid.layers.fill_constant(shape=[10, 1], - dtype="float32", - value=1.) - fluid.layers.sigmoid_focal_loss(x=x3, - label=label3, - fg_num=fg_num1, - gamma=2., - alpha=0.25) + label3 = fluid.layers.fill_constant( + shape=[10, 1], dtype="float32", value=1.0 + ) + fluid.layers.sigmoid_focal_loss( + x=x3, label=label3, fg_num=fg_num1, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_label_tensor_dtype) # The `fg_num` must be Variable and the data type of `fg_num` Tensor must be int32. def test_fgnum_type(): fg_num2 = [2] - fluid.layers.sigmoid_focal_loss(x=x3, - label=label1, - fg_num=fg_num2, - gamma=2., - alpha=0.25) + fluid.layers.sigmoid_focal_loss( + x=x3, label=label1, fg_num=fg_num2, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_fgnum_type) def test_fgnum_tensor_dtype(): - fg_num3 = fluid.layers.fill_constant(shape=[1], - dtype="float32", - value=5.) - fluid.layers.sigmoid_focal_loss(x=x3, - label=label1, - fg_num=fg_num3, - gamma=2., - alpha=0.25) + fg_num3 = fluid.layers.fill_constant( + shape=[1], dtype="float32", value=5.0 + ) + fluid.layers.sigmoid_focal_loss( + x=x3, label=label1, fg_num=fg_num3, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_fgnum_tensor_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_sign_op.py b/python/paddle/fluid/tests/unittests/test_sign_op.py index 1a55cfd3fd9..2db25d0eaee 100644 --- a/python/paddle/fluid/tests/unittests/test_sign_op.py +++ b/python/paddle/fluid/tests/unittests/test_sign_op.py @@ -25,7 +25,6 @@ import paddle.fluid.layers as layers class TestSignOp(OpTest): - def setUp(self): self.op_type = "sign" self.inputs = { @@ -41,32 +40,30 @@ class TestSignOp(OpTest): class TestSignOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of sign_op must be Variable or numpy.ndarray. input1 = 12 self.assertRaises(TypeError, fluid.layers.sign, input1) # The input dtype of sign_op must be float16, float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") - input3 = fluid.layers.data(name='input3', - shape=[12, 10], - dtype="int64") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) + input3 = fluid.layers.data( + name='input3', shape=[12, 10], dtype="int64" + ) self.assertRaises(TypeError, fluid.layers.sign, input2) self.assertRaises(TypeError, fluid.layers.sign, input3) - input4 = fluid.layers.data(name='input4', - shape=[4], - dtype="float16") + input4 = fluid.layers.data( + name='input4', shape=[4], dtype="float16" + ) fluid.layers.sign(input4) class TestSignAPI(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): - np_x = np.array([-1., 0., -0., 1.2, 1.5], dtype='float64') + np_x = np.array([-1.0, 0.0, -0.0, 1.2, 1.5], dtype='float64') x = paddle.to_tensor(np_x) z = paddle.sign(x) np_z = z.numpy() @@ -79,22 +76,21 @@ class TestSignAPI(unittest.TestCase): input1 = 12 self.assertRaises(TypeError, paddle.tensor.math.sign, input1) # The input dtype of sign_op must be float16, float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") - input3 = fluid.layers.data(name='input3', - shape=[12, 10], - dtype="int64") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) + input3 = fluid.layers.data( + name='input3', shape=[12, 10], dtype="int64" + ) self.assertRaises(TypeError, paddle.tensor.math.sign, input2) self.assertRaises(TypeError, paddle.tensor.math.sign, input3) - input4 = fluid.layers.data(name='input4', - shape=[4], - dtype="float16") + input4 = fluid.layers.data( + name='input4', shape=[4], dtype="float16" + ) paddle.sign(input4) class TestSignDoubleGradCheck(unittest.TestCase): - def sign_wrapper(self, x): return paddle.sign(x[0]) @@ -109,17 +105,13 @@ class TestSignDoubleGradCheck(unittest.TestCase): out = paddle.sign(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.sign_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.sign_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -131,7 +123,6 @@ class TestSignDoubleGradCheck(unittest.TestCase): class TestSignTripleGradCheck(unittest.TestCase): - def sign_wrapper(self, x): return paddle.sign(x[0]) @@ -146,17 +137,13 @@ class TestSignTripleGradCheck(unittest.TestCase): out = paddle.sign(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.sign_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.sign_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_signal.py b/python/paddle/fluid/tests/unittests/test_signal.py index d07e054bd48..5e363dd3951 100644 --- a/python/paddle/fluid/tests/unittests/test_signal.py +++ b/python/paddle/fluid/tests/unittests/test_signal.py @@ -57,7 +57,8 @@ def tiny(x): # Only floating types generate a tiny if np.issubdtype(x.dtype, np.floating) or np.issubdtype( - x.dtype, np.complexfloating): + x.dtype, np.complexfloating + ): dtype = x.dtype else: dtype = np.float32 @@ -71,8 +72,9 @@ def normalize(S, norm=np.inf, axis=0, threshold=None, fill=None): threshold = tiny(S) elif threshold <= 0: - raise Exception("threshold={} must be strictly " - "positive".format(threshold)) + raise Exception( + "threshold={} must be strictly " "positive".format(threshold) + ) if fill not in [None, False, True]: raise Exception("fill={} must be None or boolean".format(fill)) @@ -99,12 +101,12 @@ def normalize(S, norm=np.inf, axis=0, threshold=None, fill=None): length = np.sum(mag > 0, axis=axis, keepdims=True, dtype=mag.dtype) elif np.issubdtype(type(norm), np.number) and norm > 0: - length = np.sum(mag**norm, axis=axis, keepdims=True)**(1.0 / norm) + length = np.sum(mag**norm, axis=axis, keepdims=True) ** (1.0 / norm) if axis is None: - fill_norm = mag.size**(-1.0 / norm) + fill_norm = mag.size ** (-1.0 / norm) else: - fill_norm = mag.shape[axis]**(-1.0 / norm) + fill_norm = mag.shape[axis] ** (-1.0 / norm) elif norm is None: return S @@ -144,8 +146,9 @@ def __window_ss_fill(x, win_sq, n_frames, hop_length): # pragma: no cover n_fft = len(win_sq) for i in range(n_frames): sample = i * hop_length - x[sample:min(n, sample + - n_fft)] += win_sq[:max(0, min(n_fft, n - sample))] + x[sample : min(n, sample + n_fft)] += win_sq[ + : max(0, min(n_fft, n - sample)) + ] def window_sumsquare( @@ -165,7 +168,7 @@ def window_sumsquare( # Compute the squared window at the desired length win_sq = get_window(window, win_length) - win_sq = normalize(win_sq, norm=norm)**2 + win_sq = normalize(win_sq, norm=norm) ** 2 win_sq = pad_center(win_sq, n_fft) # Fill the envelope @@ -208,26 +211,35 @@ def dtype_r2c(d, default=np.complex64): def frame(x, frame_length, hop_length, axis=-1): if not isinstance(x, np.ndarray): - raise Exception("Input must be of type numpy.ndarray, " - "given type(x)={}".format(type(x))) + raise Exception( + "Input must be of type numpy.ndarray, " + "given type(x)={}".format(type(x)) + ) if x.shape[axis] < frame_length: - raise Exception("Input is too short (n={:d})" - " for frame_length={:d}".format(x.shape[axis], - frame_length)) + raise Exception( + "Input is too short (n={:d})" + " for frame_length={:d}".format(x.shape[axis], frame_length) + ) if hop_length < 1: raise Exception("Invalid hop_length: {:d}".format(hop_length)) if axis == -1 and not x.flags["F_CONTIGUOUS"]: - print("librosa.util.frame called with axis={} " - "on a non-contiguous input. This will result in a copy.".format( - axis)) + print( + "librosa.util.frame called with axis={} " + "on a non-contiguous input. This will result in a copy.".format( + axis + ) + ) x = np.asfortranarray(x) elif axis == 0 and not x.flags["C_CONTIGUOUS"]: - print("librosa.util.frame called with axis={} " - "on a non-contiguous input. This will result in a copy.".format( - axis)) + print( + "librosa.util.frame called with axis={} " + "on a non-contiguous input. This will result in a copy.".format( + axis + ) + ) x = np.ascontiguousarray(x) n_frames = 1 + (x.shape[axis] - frame_length) // hop_length @@ -260,8 +272,11 @@ def pad_center(data, size, axis=-1, **kwargs): lengths[axis] = (lpad, int(size - n - lpad)) if lpad < 0: - raise Exception(("Target size ({:d}) must be " - "at least input size ({:d})").format(size, n)) + raise Exception( + ("Target size ({:d}) must be " "at least input size ({:d})").format( + size, n + ) + ) return np.pad(data, lengths, **kwargs) @@ -279,8 +294,9 @@ def get_window(window, Nx, fftbins=True): if len(window) == Nx: return np.asarray(window) - raise Exception("Window size mismatch: " - "{:d} != {:d}".format(len(window), Nx)) + raise Exception( + "Window size mismatch: " "{:d} != {:d}".format(len(window), Nx) + ) else: raise Exception("Invalid window specification: {}".format(window)) @@ -294,16 +310,18 @@ def __overlap_add(y, ytmp, hop_length): n_fft = ytmp.shape[0] for frame in range(ytmp.shape[1]): sample = frame * hop_length - y[sample:(sample + n_fft)] += ytmp[:, frame] + y[sample : (sample + n_fft)] += ytmp[:, frame] -def stft(x, - n_fft=2048, - hop_length=None, - win_length=None, - window="hann", - center=True, - pad_mode="reflect"): +def stft( + x, + n_fft=2048, + hop_length=None, + win_length=None, + window="hann", + center=True, + pad_mode="reflect", +): y = x input_rank = len(y.shape) if input_rank == 2: @@ -330,15 +348,20 @@ def stft(x, # Pad the time series so that frames are centered if center: if n_fft > y.shape[-1]: - print("n_fft={} is too small for input signal of length={}".format( - n_fft, y.shape[-1])) + print( + "n_fft={} is too small for input signal of length={}".format( + n_fft, y.shape[-1] + ) + ) y = np.pad(y, int(n_fft // 2), mode=pad_mode) elif n_fft > y.shape[-1]: raise Exception( "n_fft={} is too large for input signal of length={}".format( - n_fft, y.shape[-1])) + n_fft, y.shape[-1] + ) + ) # Window the time series. y_frames = frame(y, frame_length=n_fft, hop_length=hop_length) @@ -347,9 +370,9 @@ def stft(x, dtype = dtype_r2c(y.dtype) # Pre-allocate the STFT matrix - stft_matrix = np.empty((int(1 + n_fft // 2), y_frames.shape[1]), - dtype=dtype, - order="F") + stft_matrix = np.empty( + (int(1 + n_fft // 2), y_frames.shape[1]), dtype=dtype, order="F" + ) # how many columns can we fit within MAX_MEM_BLOCK? n_columns = MAX_MEM_BLOCK // (stft_matrix.shape[0] * stft_matrix.itemsize) @@ -358,9 +381,9 @@ def stft(x, for bl_s in range(0, stft_matrix.shape[1], n_columns): bl_t = min(bl_s + n_columns, stft_matrix.shape[1]) - stft_matrix[:, - bl_s:bl_t] = fft.rfft(fft_window * y_frames[:, bl_s:bl_t], - axis=0) + stft_matrix[:, bl_s:bl_t] = fft.rfft( + fft_window * y_frames[:, bl_s:bl_t], axis=0 + ) if input_rank == 2: stft_matrix = np.expand_dims(stft_matrix, 0) @@ -405,8 +428,9 @@ def istft( padded_length = length + int(n_fft) else: padded_length = length - n_frames = min(stft_matrix.shape[1], - int(np.ceil(padded_length / hop_length))) + n_frames = min( + stft_matrix.shape[1], int(np.ceil(padded_length / hop_length)) + ) else: n_frames = stft_matrix.shape[1] @@ -428,7 +452,7 @@ def istft( ytmp = ifft_window * fft.irfft(stft_matrix[:, bl_s:bl_t], axis=0) # Overlap-add the istft block starting at the i'th frame - __overlap_add(y[frame * hop_length:], ytmp, hop_length) + __overlap_add(y[frame * hop_length :], ytmp, hop_length) frame += bl_t - bl_s @@ -449,7 +473,7 @@ def istft( # If we don't need to control length, just do the usual center trimming # to eliminate padded data if center: - y = y[int(n_fft // 2):-int(n_fft // 2)] + y = y[int(n_fft // 2) : -int(n_fft // 2)] else: if center: # If we're centering, crop off the first n_fft//2 samples @@ -506,8 +530,9 @@ def overlap_add_for_api_test(x, hop_length, axis=-1): frame_length = x.shape[1] if axis == 0 else x.shape[-2] # Assure no gaps between frames. - assert 0 < hop_length <= frame_length, \ - f'hop_length should be in (0, frame_length({frame_length})], but got {hop_length}.' + assert ( + 0 < hop_length <= frame_length + ), f'hop_length should be in (0, frame_length({frame_length})], but got {hop_length}.' seq_length = (n_frames - 1) * hop_length + frame_length @@ -528,7 +553,7 @@ def overlap_add_for_api_test(x, hop_length, axis=-1): for i in range(x.shape[0]): for frame in range(x.shape[-1]): sample = frame * hop_length - y[i, sample:sample + frame_length] += x[i, :, frame] + y[i, sample : sample + frame_length] += x[i, :, frame] if axis == 0: y = y.transpose((1, 0)) @@ -543,12 +568,10 @@ def overlap_add_for_api_test(x, hop_length, axis=-1): def place(devices, key='place'): - def decorate(cls): module = sys.modules[cls.__module__].__dict__ raw_classes = { - k: v - for k, v in module.items() if k.startswith(cls.__name__) + k: v for k, v in module.items() if k.startswith(cls.__name__) } for raw_name, raw_cls in raw_classes.items(): @@ -556,7 +579,7 @@ def place(devices, key='place'): test_cls = dict(raw_cls.__dict__) test_cls.update({key: d}) new_name = raw_name + '.' + d.__class__.__name__ - module[new_name] = type(new_name, (raw_cls, ), test_cls) + module[new_name] = type(new_name, (raw_cls,), test_cls) del module[raw_name] return cls @@ -586,20 +609,23 @@ def tearDownModule(): pass -def rand_x(dims=1, - dtype='float64', - min_dim_len=1, - max_dim_len=10, - shape=None, - complex=False): +def rand_x( + dims=1, + dtype='float64', + min_dim_len=1, + max_dim_len=10, + shape=None, + complex=False, +): if shape is None: shape = [ np.random.randint(min_dim_len, max_dim_len) for i in range(dims) ] if complex: - return np.random.randn( - *shape).astype(dtype) + 1.j * np.random.randn(*shape).astype(dtype) + return np.random.randn(*shape).astype(dtype) + 1.0j * np.random.randn( + *shape + ).astype(dtype) else: return np.random.randn(*shape).astype(dtype) @@ -608,8 +634,11 @@ def parameterize(attrs, input_values=None): if isinstance(attrs, str): attrs = [attrs] - input_dicts = (attrs if input_values is None else - [dict(zip(attrs, vals)) for vals in input_values]) + input_dicts = ( + attrs + if input_values is None + else [dict(zip(attrs, vals)) for vals in input_values] + ) def decorator(base_class): test_class_module = sys.modules[base_class.__module__].__dict__ @@ -619,8 +648,7 @@ def parameterize(attrs, input_values=None): name = class_name(base_class, idx, input_dict) - test_class_module[name] = type(name, (base_class, ), - test_class_dict) + test_class_module[name] = type(name, (base_class,), test_class_dict) for method_name in list(base_class.__dict__): if method_name.startswith("test"): @@ -632,7 +660,8 @@ def parameterize(attrs, input_values=None): def class_name(cls, num, params_dict): suffix = to_safe_name( - next((v for v in params_dict.values() if isinstance(v, str)), "")) + next((v for v in params_dict.values() if isinstance(v, str)), "") + ) if TEST_CASE_NAME in params_dict: suffix = to_safe_name(params_dict["test_case"]) return "{}_{}{}".format(cls.__name__, num, suffix and "_" + suffix) @@ -652,19 +681,22 @@ def to_safe_name(s): ('test_2d_input2', rand_x(2, np.float64, shape=[8, 150]), 50, 15, -1), ('test_3d_input1', rand_x(3, np.float64, shape=[150, 4, 2]), 50, 15, 0), ('test_3d_input2', rand_x(3, np.float64, shape=[4, 2, 150]), 50, 15, -1), - ]) # yapf: disable + ]) # fmt: skip class TestFrame(unittest.TestCase): - def test_frame(self): - np.testing.assert_allclose(frame_for_api_test(self.x, self.frame_length, - self.hop_length, - self.axis), - paddle.signal.frame(paddle.to_tensor(self.x), - self.frame_length, - self.hop_length, - self.axis), - rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + np.testing.assert_allclose( + frame_for_api_test( + self.x, self.frame_length, self.hop_length, self.axis + ), + paddle.signal.frame( + paddle.to_tensor(self.x), + self.frame_length, + self.hop_length, + self.axis, + ), + rtol=rtol.get(str(self.x.dtype)), + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -677,29 +709,33 @@ class TestFrame(unittest.TestCase): ('test_2d_input2', rand_x(2, np.float64, shape=[8, 150]), 50, 15, -1), ('test_3d_input1', rand_x(3, np.float64, shape=[150, 4, 2]), 50, 15, 0), ('test_3d_input2', rand_x(3, np.float64, shape=[4, 2, 150]), 50, 15, -1), - ]) # yapf: disable + ]) # fmt: skip class TestFrameStatic(unittest.TestCase): - def test_frame_static(self): paddle.enable_static() mp, sp = paddle.static.Program(), paddle.static.Program() with paddle.static.program_guard(mp, sp): - input = paddle.static.data('input', - self.x.shape, - dtype=self.x.dtype) - output = paddle.signal.frame(input, self.frame_length, - self.hop_length, self.axis), + input = paddle.static.data( + 'input', self.x.shape, dtype=self.x.dtype + ) + output = ( + paddle.signal.frame( + input, self.frame_length, self.hop_length, self.axis + ), + ) exe = paddle.static.Executor(self.place) exe.run(sp) [output] = exe.run(mp, feed={'input': self.x}, fetch_list=[output]) paddle.disable_static() - np.testing.assert_allclose(frame_for_api_test(self.x, self.frame_length, - self.hop_length, - self.axis), - output, - rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + np.testing.assert_allclose( + frame_for_api_test( + self.x, self.frame_length, self.hop_length, self.axis + ), + output, + rtol=rtol.get(str(self.x.dtype)), + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -710,13 +746,16 @@ class TestFrameStatic(unittest.TestCase): ('test_hop_length', rand_x(1, np.float64, shape=[150]), 50, 0, -1, ValueError), ('test_frame_length1', rand_x(2, np.float64, shape=[150, 8]), 0, 15, 0, ValueError), ('test_frame_length2', rand_x(2, np.float64, shape=[150, 8]), 151, 15, 0, ValueError), - ]) # yapf: disable + ]) # fmt: skip class TestFrameException(unittest.TestCase): - def test_frame(self): with self.assertRaises(self.expect_exception): - paddle.signal.frame(paddle.to_tensor(self.x), self.frame_length, - self.hop_length, self.axis) + paddle.signal.frame( + paddle.to_tensor(self.x), + self.frame_length, + self.hop_length, + self.axis, + ) @place(DEVICES) @@ -729,16 +768,17 @@ class TestFrameException(unittest.TestCase): ('test_3d_input2', rand_x(3, np.float64, shape=[2, 40, 5]), 10, -1), ('test_4d_input1', rand_x(4, np.float64, shape=[8, 12, 5, 3]), 5, 0), ('test_4d_input2', rand_x(4, np.float64, shape=[3, 5, 12, 8]), 5, -1), - ]) # yapf: disable + ]) # fmt: skip class TestOverlapAdd(unittest.TestCase): - def test_overlap_add(self): np.testing.assert_allclose( overlap_add_for_api_test(self.x, self.hop_length, self.axis), - paddle.signal.overlap_add(paddle.to_tensor(self.x), self.hop_length, - self.axis), + paddle.signal.overlap_add( + paddle.to_tensor(self.x), self.hop_length, self.axis + ), rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -751,28 +791,29 @@ class TestOverlapAdd(unittest.TestCase): ('test_3d_input2', rand_x(3, np.float64, shape=[2, 40, 5]), 10, -1), ('test_4d_input1', rand_x(4, np.float64, shape=[8, 12, 5, 3]), 5, 0), ('test_4d_input2', rand_x(4, np.float64, shape=[3, 5, 12, 8]), 5, -1), - ]) # yapf: disable + ]) # fmt: skip class TestOverlapAddStatic(unittest.TestCase): - def test_overlap_add_static(self): paddle.enable_static() mp, sp = paddle.static.Program(), paddle.static.Program() with paddle.static.program_guard(mp, sp): - input = paddle.static.data('input', - self.x.shape, - dtype=self.x.dtype) - output = paddle.signal.overlap_add(input, self.hop_length, - self.axis), + input = paddle.static.data( + 'input', self.x.shape, dtype=self.x.dtype + ) + output = ( + paddle.signal.overlap_add(input, self.hop_length, self.axis), + ) exe = paddle.static.Executor(self.place) exe.run(sp) [output] = exe.run(mp, feed={'input': self.x}, fetch_list=[output]) paddle.disable_static() - np.testing.assert_allclose(overlap_add_for_api_test( - self.x, self.hop_length, self.axis), - output, - rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + np.testing.assert_allclose( + overlap_add_for_api_test(self.x, self.hop_length, self.axis), + output, + rtol=rtol.get(str(self.x.dtype)), + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -781,13 +822,13 @@ class TestOverlapAddStatic(unittest.TestCase): [ ('test_axis', rand_x(2, np.float64, shape=[3, 50]), 4, 2, ValueError), ('test_hop_length', rand_x(2, np.float64, shape=[50, 3]), -1, -1, ValueError), - ]) # yapf: disable + ]) # fmt: skip class TestOverlapAddException(unittest.TestCase): - def test_overlap_add(self): with self.assertRaises(self.expect_exception): - paddle.signal.overlap_add(paddle.to_tensor(self.x), self.hop_length, - self.axis) + paddle.signal.overlap_add( + paddle.to_tensor(self.x), self.hop_length, self.axis + ) # ================= STFT @@ -836,9 +877,8 @@ class TestOverlapAddException(unittest.TestCase): 512, None, None, None, True, 'reflect', False, True), ('test_center', rand_x(2, np.float64, shape=[1, 160000]), 512, None, None, None, False, 'reflect', False, True), - ])# yapf: disable + ]) # fmt: skip class TestStft(unittest.TestCase): - def test_stft(self): if self.window is None: win_p = None @@ -848,14 +888,29 @@ class TestStft(unittest.TestCase): win_l = self.window np.testing.assert_allclose( - stft(self.x, self.n_fft, self.hop_length, self.win_length, win_l, - self.center, self.pad_mode), - paddle.signal.stft(paddle.to_tensor(self.x), self.n_fft, - self.hop_length, self.win_length, win_p, - self.center, self.pad_mode, self.normalized, - self.onesided), + stft( + self.x, + self.n_fft, + self.hop_length, + self.win_length, + win_l, + self.center, + self.pad_mode, + ), + paddle.signal.stft( + paddle.to_tensor(self.x), + self.n_fft, + self.hop_length, + self.win_length, + win_p, + self.center, + self.pad_mode, + self.normalized, + self.onesided, + ), rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -878,9 +933,8 @@ class TestStft(unittest.TestCase): 512, None, None, None, True, 'nonsense', False, True, AssertionError), ('test_complex_onesided', rand_x(1, np.float64, shape=[16000], complex=True), 512, None, None, None, False, 'reflect', False, True, AssertionError), - ]) # yapf: disable + ]) # fmt: skip class TestStftException(unittest.TestCase): - def test_stft(self): if self.window is None: win_p = None @@ -888,10 +942,17 @@ class TestStftException(unittest.TestCase): win_p = paddle.to_tensor(self.window) with self.assertRaises(self.expect_exception): - paddle.signal.stft(paddle.to_tensor(self.x), self.n_fft, - self.hop_length, self.win_length, win_p, - self.center, self.pad_mode, self.normalized, - self.onesided), + paddle.signal.stft( + paddle.to_tensor(self.x), + self.n_fft, + self.hop_length, + self.win_length, + win_p, + self.center, + self.pad_mode, + self.normalized, + self.onesided, + ), @place(DEVICES) @@ -912,9 +973,8 @@ class TestStftException(unittest.TestCase): 512, None, None, None, False, False, True, None, False), ('test_length', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, None, None, None, False, False, True, 1888, False), - ]) # yapf: disable + ]) # fmt: skip class TestIstft(unittest.TestCase): - def test_istft(self): if self.window is None: win_p = None @@ -924,14 +984,29 @@ class TestIstft(unittest.TestCase): win_l = self.window np.testing.assert_allclose( - istft(self.x, self.hop_length, self.win_length, win_l, self.center, - self.length), - paddle.signal.istft(paddle.to_tensor(self.x), self.n_fft, - self.hop_length, self.win_length, win_p, - self.center, self.normalized, self.onesided, - self.length, self.return_complex), + istft( + self.x, + self.hop_length, + self.win_length, + win_l, + self.center, + self.length, + ), + paddle.signal.istft( + paddle.to_tensor(self.x), + self.n_fft, + self.hop_length, + self.win_length, + win_p, + self.center, + self.normalized, + self.onesided, + self.length, + self.return_complex, + ), rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -962,9 +1037,8 @@ class TestIstft(unittest.TestCase): 512, None, None, rand_x(1, np.float64, shape=[512], complex=True), True, False, True, None, False, AssertionError), ('test_NOLA', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, 512, None, get_window('hann', 512), True, False, True, None, False, ValueError), - ]) # yapf: disable + ]) # fmt: skip class TestIstftException(unittest.TestCase): - def test_istft(self): if self.window is None: win_p = None @@ -972,10 +1046,18 @@ class TestIstftException(unittest.TestCase): win_p = paddle.to_tensor(self.window) with self.assertRaises(self.expect_exception): - paddle.signal.istft(paddle.to_tensor(self.x), self.n_fft, - self.hop_length, self.win_length, win_p, - self.center, self.normalized, self.onesided, - self.length, self.return_complex), + paddle.signal.istft( + paddle.to_tensor(self.x), + self.n_fft, + self.hop_length, + self.win_length, + win_p, + self.center, + self.normalized, + self.onesided, + self.length, + self.return_complex, + ), if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_similarity_focus_op.py b/python/paddle/fluid/tests/unittests/test_similarity_focus_op.py index d7d13bdd39e..15a050211a1 100755 --- a/python/paddle/fluid/tests/unittests/test_similarity_focus_op.py +++ b/python/paddle/fluid/tests/unittests/test_similarity_focus_op.py @@ -20,17 +20,25 @@ from paddle.fluid import Program, program_guard class TestSimilarityFocusOp(OpTest): - def setUp(self): self.op_type = "similarity_focus" batch_size = 2 x_dim, y_dim, z_dim = 3, 2, 2 self.inputs = { - 'X': - np.array([[[[0.8, 0.1], [0.4, 0.5]], [[0.9, 0.7], [0.9, 0.9]], - [[0.8, 0.9], [0.1, 0.2]]], - [[[0.2, 0.5], [0.3, 0.4]], [[0.9, 0.7], [0.8, 0.4]], - [[0.0, 0.2], [0.4, 0.7]]]]), + 'X': np.array( + [ + [ + [[0.8, 0.1], [0.4, 0.5]], + [[0.9, 0.7], [0.9, 0.9]], + [[0.8, 0.9], [0.1, 0.2]], + ], + [ + [[0.2, 0.5], [0.3, 0.4]], + [[0.9, 0.7], [0.8, 0.4]], + [[0.0, 0.2], [0.4, 0.7]], + ], + ] + ), } self.attrs = { 'axis': 1, @@ -41,8 +49,9 @@ class TestSimilarityFocusOp(OpTest): for batch in range(batch_size): res = np.zeros((1, y_dim, z_dim)).astype("float32").reshape(-1) for index in self.attrs['indexes']: - channel = self.inputs['X'][batch, - index, :, :].reshape(-1).copy() + channel = ( + self.inputs['X'][batch, index, :, :].reshape(-1).copy() + ) tag1 = [0 for i in range(y_dim)] tag2 = [0 for i in range(z_dim)] cnt = 0 @@ -71,15 +80,14 @@ class TestSimilarityFocusOp(OpTest): class TestSimilarityFocusOp_axis1(OpTest): - def setUp(self): self.op_type = "similarity_focus" batch_size = 3 x_dim, y_dim, z_dim = 4, 5, 6 self.inputs = { - 'X': - np.random.random( - (batch_size, x_dim, y_dim, z_dim)).astype("float32"), + 'X': np.random.random((batch_size, x_dim, y_dim, z_dim)).astype( + "float32" + ), } self.attrs = { 'axis': 1, @@ -90,8 +98,9 @@ class TestSimilarityFocusOp_axis1(OpTest): for batch in range(batch_size): res = np.zeros((1, y_dim, z_dim)).astype("float32").reshape(-1) for index in self.attrs['indexes']: - channel = self.inputs['X'][batch, - index, :, :].reshape(-1).copy() + channel = ( + self.inputs['X'][batch, index, :, :].reshape(-1).copy() + ) tag1 = [0 for i in range(y_dim)] tag2 = [0 for i in range(z_dim)] cnt = 0 @@ -121,15 +130,14 @@ class TestSimilarityFocusOp_axis1(OpTest): class TestSimilarityFocusOp_axis2(OpTest): - def setUp(self): self.op_type = "similarity_focus" batch_size = 6 x_dim, y_dim, z_dim = 7, 8, 9 self.inputs = { - 'X': - np.random.random( - (batch_size, x_dim, y_dim, z_dim)).astype("float32"), + 'X': np.random.random((batch_size, x_dim, y_dim, z_dim)).astype( + "float32" + ), } self.attrs = { 'axis': 2, @@ -140,8 +148,9 @@ class TestSimilarityFocusOp_axis2(OpTest): for batch in range(batch_size): res = np.zeros((x_dim, 1, z_dim)).astype("float32").reshape(-1) for index in self.attrs['indexes']: - channel = self.inputs['X'][batch, :, - index, :].reshape(-1).copy() + channel = ( + self.inputs['X'][batch, :, index, :].reshape(-1).copy() + ) tag1 = [0 for i in range(x_dim)] tag2 = [0 for i in range(z_dim)] cnt = 0 @@ -171,15 +180,14 @@ class TestSimilarityFocusOp_axis2(OpTest): class TestSimilarityFocusOp_axis3(OpTest): - def setUp(self): self.op_type = "similarity_focus" batch_size = 64 x_dim, y_dim, z_dim = 48, 48, 13 self.inputs = { - 'X': - np.random.random( - (batch_size, x_dim, y_dim, z_dim)).astype("float32"), + 'X': np.random.random((batch_size, x_dim, y_dim, z_dim)).astype( + "float32" + ), } self.attrs = { 'axis': 3, @@ -190,8 +198,9 @@ class TestSimilarityFocusOp_axis3(OpTest): for batch in range(batch_size): res = np.zeros((x_dim, y_dim, 1)).astype("float32").reshape(-1) for index in self.attrs['indexes']: - channel = self.inputs['X'][batch, :, :, - index].reshape(-1).copy() + channel = ( + self.inputs['X'][batch, :, :, index].reshape(-1).copy() + ) tag1 = [0 for i in range(x_dim)] tag2 = [0 for i in range(y_dim)] cnt = 0 @@ -221,32 +230,31 @@ class TestSimilarityFocusOp_axis3(OpTest): class TestSimilarityFocusOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): data = fluid.data(name='data', shape=[16, 3, 2, 2], dtype='float32') def test_input_Variable(): input = np.random.rand(16, 3, 2, 2).astype("float32") - out = fluid.layers.similarity_focus(input=input, - axis=1, - indexes=[0]) + out = fluid.layers.similarity_focus( + input=input, axis=1, indexes=[0] + ) self.assertRaises(TypeError, test_input_Variable) def test_axis_Int(): axis = 1.0 - out = fluid.layers.similarity_focus(input=data, - axis=axis, - indexes=[0]) + out = fluid.layers.similarity_focus( + input=data, axis=axis, indexes=[0] + ) self.assertRaises(TypeError, test_axis_Int) def test_indexes_List(): indexes = 0 - out = fluid.layers.similarity_focus(input=data, - axis=1, - indexes=indexes) + out = fluid.layers.similarity_focus( + input=data, axis=1, indexes=indexes + ) self.assertRaises(TypeError, test_indexes_List) diff --git a/python/paddle/fluid/tests/unittests/test_simple_rnn_op.py b/python/paddle/fluid/tests/unittests/test_simple_rnn_op.py index 4a0f02204c7..fa129cda066 100644 --- a/python/paddle/fluid/tests/unittests/test_simple_rnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_simple_rnn_op.py @@ -31,7 +31,6 @@ paddle.enable_static() class TestSimpleRNNOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(self.num_layers): @@ -45,13 +44,16 @@ class TestSimpleRNNOp(OpTest): def setUp(self): self.op_type = "rnn" self.dtype = "float32" if core.is_compiled_with_rocm() else "float64" - self.sequence_length = None if core.is_compiled_with_rocm( - ) else np.array([12, 11, 10, 9, 8], dtype=np.int32) + self.sequence_length = ( + None + if core.is_compiled_with_rocm() + else np.array([12, 11, 10, 9, 8], dtype=np.int32) + ) self.num_layers = 1 self.is_bidirec = False self.is_test = False self.mode = "RNN_TANH" - self.dropout = 0. + self.dropout = 0.0 self.set_attrs() self.direction_num = 2 if self.is_bidirec else 1 @@ -61,31 +63,33 @@ class TestSimpleRNNOp(OpTest): input_size = 3 hidden_size = 2 - input = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_length, batch_size, - input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, high=0.1, size=(seq_length, batch_size, input_size) + ).astype(self.dtype) if self.sequence_length is not None: input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 - rnn1 = SimpleRNN(input_size, - hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - nonlinearity=self.mode, - dtype=self.dtype) + rnn1 = SimpleRNN( + input_size, + hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + nonlinearity=self.mode, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) output, last_hidden = rnn1(input, sequence_length=self.sequence_length) - init_h = np.zeros((self.num_layers * self.direction_num, batch_size, - hidden_size)).astype(self.dtype) + init_h = np.zeros( + (self.num_layers * self.direction_num, batch_size, hidden_size) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") @@ -93,13 +97,13 @@ class TestSimpleRNNOp(OpTest): 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { 'Input': input, 'WeightList': flat_w, - 'PreState': [('init_h', init_h)] + 'PreState': [('init_h', init_h)], } self.attrs = { 'dropout_prob': self.dropout, @@ -108,13 +112,13 @@ class TestSimpleRNNOp(OpTest): 'hidden_size': hidden_size, 'num_layers': self.num_layers, 'is_test': self.is_test, - 'mode': self.mode + 'mode': self.mode, } self.outputs = { 'Out': output, 'State': [('last_hidden', last_hidden)], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def set_attrs(self): @@ -132,27 +136,23 @@ class TestSimpleRNNOp(OpTest): class TestSimpleRNNOp1(TestSimpleRNNOp): - def set_attrs(self): self.sequence_length = None class TestSimpleRNNOp2(TestSimpleRNNOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True class TestSimpleRNNOp3(TestSimpleRNNOp): - def set_attrs(self): self.sequence_length = None self.is_test = True class TestSimpleRNNOp4(TestSimpleRNNOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True @@ -160,7 +160,6 @@ class TestSimpleRNNOp4(TestSimpleRNNOp): class TestSimpleRNNOp5(TestSimpleRNNOp): - def set_attrs(self): self.mode = "RNN_RELU" diff --git a/python/paddle/fluid/tests/unittests/test_size_op.py b/python/paddle/fluid/tests/unittests/test_size_op.py index bb64e3e66b2..6f9898ade4f 100644 --- a/python/paddle/fluid/tests/unittests/test_size_op.py +++ b/python/paddle/fluid/tests/unittests/test_size_op.py @@ -20,7 +20,6 @@ from op_test import OpTest class TestSizeOp(OpTest): - def setUp(self): self.op_type = "size" self.shape = [] @@ -37,31 +36,26 @@ class TestSizeOp(OpTest): class TestRank1Tensor(TestSizeOp): - def config(self): self.shape = [2] class TestRank2Tensor(TestSizeOp): - def config(self): self.shape = [2, 3] class TestRank3Tensor(TestSizeOp): - def config(self): self.shape = [2, 3, 100] class TestLargeTensor(TestSizeOp): - def config(self): self.shape = [2**10] class TestSizeAPI(unittest.TestCase): - def test_size_static(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -75,17 +69,19 @@ class TestSizeAPI(unittest.TestCase): out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) exe = paddle.static.Executor(place=paddle.CPUPlace()) - res_1, res_2 = exe.run(feed={ - "x_1": input_1, - "x_2": input_2, - }, - fetch_list=[out_1, out_2]) - assert (np.array_equal(res_1, - np.array([np.size(input_1) - ]).astype("int64"))) - assert (np.array_equal(res_2, - np.array([np.size(input_2) - ]).astype("int64"))) + res_1, res_2 = exe.run( + feed={ + "x_1": input_1, + "x_2": input_2, + }, + fetch_list=[out_1, out_2], + ) + assert np.array_equal( + res_1, np.array([np.size(input_1)]).astype("int64") + ) + assert np.array_equal( + res_2, np.array([np.size(input_2)]).astype("int64") + ) def test_size_imperative(self): paddle.disable_static(paddle.CPUPlace()) @@ -95,8 +91,8 @@ class TestSizeAPI(unittest.TestCase): x_2 = paddle.to_tensor(input_2) out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) - assert (np.array_equal(out_1.numpy().item(0), np.size(input_1))) - assert (np.array_equal(out_2.numpy().item(0), np.size(input_2))) + assert np.array_equal(out_1.numpy().item(0), np.size(input_1)) + assert np.array_equal(out_2.numpy().item(0), np.size(input_2)) paddle.enable_static() def test_error(self): diff --git a/python/paddle/fluid/tests/unittests/test_slice_op.py b/python/paddle/fluid/tests/unittests/test_slice_op.py index bf8b9e92209..59cd41ae9de 100644 --- a/python/paddle/fluid/tests/unittests/test_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_slice_op.py @@ -30,7 +30,6 @@ paddle.enable_static() # Situation 1: starts(list, no tensor), ends(list, no tensor) # 1.1 without attr(decrease) class TestSliceOp(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -40,7 +39,7 @@ class TestSliceOp(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -59,7 +58,6 @@ class TestSliceOp(OpTest): class TestCase1(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-3, 0, 2] @@ -70,7 +68,6 @@ class TestCase1(TestSliceOp): class TestCase2(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-3, 0, 2] @@ -81,7 +78,6 @@ class TestCase2(TestSliceOp): class TestSliceZerosShapeTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -92,7 +88,7 @@ class TestSliceZerosShapeTensor(OpTest): 'starts': self.starts, 'ends': self.ends, 'infer_flags': self.infer_flags, - 'use_mkldnn': True + 'use_mkldnn': True, } def config(self): @@ -109,7 +105,6 @@ class TestSliceZerosShapeTensor(OpTest): # 1.2 with attr(decrease) class TestSliceOp_decs_dim(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -140,7 +135,6 @@ class TestSliceOp_decs_dim(OpTest): class TestSliceOp_decs_dim_2(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [1, 0, 2] @@ -152,7 +146,6 @@ class TestSliceOp_decs_dim_2(TestSliceOp_decs_dim): class TestSliceOp_decs_dim_3(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-1, 0, 2] @@ -164,7 +157,6 @@ class TestSliceOp_decs_dim_3(TestSliceOp_decs_dim): class TestSliceOp_decs_dim_4(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype("float64") self.starts = [0, 1, 2, 3] @@ -176,7 +168,6 @@ class TestSliceOp_decs_dim_4(TestSliceOp_decs_dim): class TestSliceOp_decs_dim_5(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-1] @@ -188,7 +179,6 @@ class TestSliceOp_decs_dim_5(TestSliceOp_decs_dim): class TestSliceOp_decs_dim_6(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [0, 1, 2, 3] @@ -202,15 +192,15 @@ class TestSliceOp_decs_dim_6(TestSliceOp_decs_dim): # Situation 2: starts(list, have tensor), ends(list, no tensor) # without attr(decrease) class TestSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int64') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int64') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.out} @@ -218,7 +208,7 @@ class TestSliceOp_starts_ListTensor(OpTest): 'axes': self.axes, 'starts': self.starts_infer, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -241,15 +231,15 @@ class TestSliceOp_starts_ListTensor(OpTest): # Situation 2: starts(list, have tensor), ends(list, no tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} @@ -281,8 +271,8 @@ class TestSliceOp_decs_dim_starts_ListTensor(OpTest): class TestSliceOp_decs_dim_5_starts_ListTensor( - TestSliceOp_decs_dim_starts_ListTensor): - + TestSliceOp_decs_dim_starts_ListTensor +): def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-1] @@ -298,13 +288,12 @@ class TestSliceOp_decs_dim_5_starts_ListTensor( # Situation 3: starts(tensor), ends(list, no tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { @@ -334,7 +323,6 @@ class TestSliceOp_decs_dim_starts_OneTensor(OpTest): # Situation 4: starts(tensor), ends(tensor) # without attr(decrease) class TestSliceOp_starts_OneTensor_ends_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -342,14 +330,14 @@ class TestSliceOp_starts_OneTensor_ends_OneTensor(OpTest): self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int64"), - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, #'starts': self.starts, #'ends': self.ends_infer, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -370,14 +358,13 @@ class TestSliceOp_starts_OneTensor_ends_OneTensor(OpTest): # Situation 5: starts(tensor), ends(tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_and_ends_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { @@ -407,27 +394,27 @@ class TestSliceOp_decs_dim_starts_and_ends_OneTensor(OpTest): # Situation 6: starts(tensor), ends(list, have tensor) # without attr(decrease) class TestSliceOp_starts_OneTensor_ends_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("y" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("y" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - 'EndsTensorList': ends_tensor + 'EndsTensorList': ends_tensor, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, #'starts': self.starts, 'ends': self.ends_infer, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -448,10 +435,10 @@ class TestSliceOp_starts_OneTensor_ends_ListTensor(OpTest): # Test CUDA float16 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -461,7 +448,7 @@ class TestFP16(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -481,15 +468,15 @@ class TestFP16(OpTest): def test_check_grad_normal(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + place, ['Input'], 'Out', max_relative_error=0.006 + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16_2(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -499,7 +486,7 @@ class TestFP16_2(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -519,14 +506,16 @@ class TestFP16_2(OpTest): def test_check_grad_normal(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['Input'], - 'Out', - max_relative_error=0.006, - numeric_grad_delta=0.5) + self.check_grad_with_place( + place, + ['Input'], + 'Out', + max_relative_error=0.006, + numeric_grad_delta=0.5, + ) class TestBF16(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -536,7 +525,7 @@ class TestBF16(OpTest): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -557,38 +546,36 @@ class TestBF16(OpTest): # Test python API class TestSliceAPI(unittest.TestCase): - def test_1(self): input = np.random.random([3, 4, 5, 6]).astype("float64") minus_1 = fluid.layers.fill_constant([1], "int32", -1) minus_3 = fluid.layers.fill_constant([1], "int64", -3) - starts = fluid.layers.data(name='starts', - shape=[1, 3], - append_batch_size=False) - ends = fluid.layers.data(name='ends', - shape=[3], - append_batch_size=False) - - x = fluid.layers.data(name="x", - shape=[3, 4, 5, 6], - append_batch_size=False, - dtype="float64") + starts = fluid.layers.data( + name='starts', shape=[1, 3], append_batch_size=False + ) + ends = fluid.layers.data( + name='ends', shape=[3], append_batch_size=False + ) + + x = fluid.layers.data( + name="x", + shape=[3, 4, 5, 6], + append_batch_size=False, + dtype="float64", + ) # value_int64 is greater than 2147483647 which is the max of int32 value_int64 = fluid.layers.fill_constant([1], "int64", 2147483648) - out_1 = paddle.slice(x, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[value_int64, 100, -1]) - out_2 = paddle.slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, -1]) - out_3 = paddle.slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, minus_1]) + out_1 = paddle.slice( + x, axes=[0, 1, 2], starts=[-3, 0, 2], ends=[value_int64, 100, -1] + ) + out_2 = paddle.slice( + x, axes=[0, 1, 3], starts=[minus_3, 0, 2], ends=[3, 100, -1] + ) + out_3 = paddle.slice( + x, axes=[0, 1, 3], starts=[minus_3, 0, 2], ends=[3, 100, minus_1] + ) out_4 = paddle.slice(x, axes=[0, 1, 2], starts=starts, ends=ends) out_5 = x[-3:3, 0:100, 2:-1] @@ -601,9 +588,10 @@ class TestSliceAPI(unittest.TestCase): feed={ "x": input, 'starts': np.array([-3, 0, 2]).astype("int32"), - 'ends': np.array([3, 100, -1]).astype("int32") + 'ends': np.array([3, 100, -1]).astype("int32"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, input[-3:3, 0:100, 2:-1, :]) assert np.array_equal(res_2, input[-3:3, 0:100, :, 2:-1]) @@ -615,17 +603,18 @@ class TestSliceAPI(unittest.TestCase): class TestSliceApiWithTensor(unittest.TestCase): - def test_starts_ends_is_tensor(self): with paddle.fluid.dygraph.guard(): a = paddle.rand(shape=[4, 5, 6], dtype='float32') axes = [0, 1, 2] starts = [-3, 0, 2] ends = [3, 2, 4] - a_1 = paddle.slice(a, - axes=axes, - starts=paddle.to_tensor(starts, dtype='int32'), - ends=paddle.to_tensor(ends, dtype='int32')) + a_1 = paddle.slice( + a, + axes=axes, + starts=paddle.to_tensor(starts, dtype='int32'), + ends=paddle.to_tensor(ends, dtype='int32'), + ) a_2 = paddle.slice(a, axes=axes, starts=starts, ends=ends) np.testing.assert_array_equal(a_1.numpy(), a_2.numpy()) @@ -648,7 +637,6 @@ class TestSliceApiWithTensor(unittest.TestCase): class TestSliceApiEager(unittest.TestCase): - def test_slice_api(self): with paddle.fluid.dygraph.guard(): with _test_eager_guard(): @@ -659,23 +647,24 @@ class TestSliceApiEager(unittest.TestCase): ends = [3, 2, 4] a_1 = paddle.slice(a, axes=axes, starts=starts, ends=ends) - a_2 = paddle.slice(a, - axes=axes, - starts=paddle.to_tensor(starts), - ends=paddle.to_tensor(ends)) + a_2 = paddle.slice( + a, + axes=axes, + starts=paddle.to_tensor(starts), + ends=paddle.to_tensor(ends), + ) np.testing.assert_array_equal(a_1.numpy(), a_2.numpy()) a_1.backward() grad_truth = paddle.zeros_like(a) grad_truth[-3:3, 0:2, 2:4] = 1 np.testing.assert_array_equal(grad_truth, a.gradient()) - np.testing.assert_allclose(a_1.numpy(), - a[-3:3, 0:2, 2:4], - rtol=1e-05) + np.testing.assert_allclose( + a_1.numpy(), a[-3:3, 0:2, 2:4], rtol=1e-05 + ) class TestSliceApiWithLoDTensorArray(unittest.TestCase): - def setUp(self): self.shape = (3, 4) self.data = np.random.random(size=self.shape).astype('float32') @@ -684,8 +673,11 @@ class TestSliceApiWithLoDTensorArray(unittest.TestCase): self.end = 2 self.axis = 1 - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.exe = fluid.Executor(self.place) def set_program_and_run(self, main_program, case_num): @@ -693,7 +685,7 @@ class TestSliceApiWithLoDTensorArray(unittest.TestCase): x = [ fluid.data(name='x0', shape=self.shape, dtype="float32"), fluid.data(name='x1', shape=self.shape, dtype="float32"), - fluid.data(name='x2', shape=self.shape, dtype="float32") + fluid.data(name='x2', shape=self.shape, dtype="float32"), ] for each_x in x: @@ -708,31 +700,35 @@ class TestSliceApiWithLoDTensorArray(unittest.TestCase): self.sliced_arr = output = arr[0] elif case_num == 2: - end = fluid.layers.array_length( - arr) - 1 # dtype of end is int64 - self.sliced_arr = slice_arr = arr[self.start:end] - output, _ = fluid.layers.tensor_array_to_tensor(slice_arr, - axis=self.axis, - use_stack=True) + end = ( + fluid.layers.array_length(arr) - 1 + ) # dtype of end is int64 + self.sliced_arr = slice_arr = arr[self.start : end] + output, _ = fluid.layers.tensor_array_to_tensor( + slice_arr, axis=self.axis, use_stack=True + ) elif case_num == 3: - value_int64 = fluid.layers.fill_constant([1], "int64", - 2147483648) - self.sliced_arr = slice_arr = arr[self.start:value_int64] - output, _ = fluid.layers.tensor_array_to_tensor(slice_arr, - axis=self.axis, - use_stack=True) + value_int64 = fluid.layers.fill_constant( + [1], "int64", 2147483648 + ) + self.sliced_arr = slice_arr = arr[self.start : value_int64] + output, _ = fluid.layers.tensor_array_to_tensor( + slice_arr, axis=self.axis, use_stack=True + ) loss = fluid.layers.reduce_sum(output) fluid.backward.append_backward(loss) g_vars = list( - map(main_program.global_block().var, - [each_x.name + "@GRAD" for each_x in x])) - self.out, self.g_x0, self.g_x1, self.g_x2 = \ - self.exe.run(main_program, - feed = {'x0': self.data, - 'x1': self.data, - 'x2': self.data}, - fetch_list=[output] + g_vars) + map( + main_program.global_block().var, + [each_x.name + "@GRAD" for each_x in x], + ) + ) + self.out, self.g_x0, self.g_x1, self.g_x2 = self.exe.run( + main_program, + feed={'x0': self.data, 'x1': self.data, 'x2': self.data}, + fetch_list=[output] + g_vars, + ) def test_case_1(self): main_program = fluid.Program() @@ -750,10 +746,12 @@ class TestSliceApiWithLoDTensorArray(unittest.TestCase): self.set_program_and_run(main_program, 2) self.assertTrue( - self.sliced_arr.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY) + self.sliced_arr.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ) self.assertEqual(self.sliced_arr.shape, self.shape) np.testing.assert_array_equal( - self.out, np.stack([self.data, self.data], axis=self.axis)) + self.out, np.stack([self.data, self.data], axis=self.axis) + ) np.testing.assert_array_equal(self.g_x0, np.ones_like(self.data)) np.testing.assert_array_equal(self.g_x1, np.ones_like(self.data)) np.testing.assert_array_equal(self.g_x2, np.zeros_like(self.data)) @@ -763,35 +761,35 @@ class TestSliceApiWithLoDTensorArray(unittest.TestCase): self.set_program_and_run(main_program, 3) self.assertTrue( - self.sliced_arr.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY) + self.sliced_arr.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ) self.assertEqual(self.sliced_arr.shape, self.shape) np.testing.assert_array_equal( - self.out, np.stack([self.data, self.data, self.data], - axis=self.axis)) + self.out, + np.stack([self.data, self.data, self.data], axis=self.axis), + ) np.testing.assert_array_equal(self.g_x0, np.ones_like(self.data)) np.testing.assert_array_equal(self.g_x1, np.ones_like(self.data)) np.testing.assert_array_equal(self.g_x2, np.ones_like(self.data)) class TestImperativeVarBaseGetItem(unittest.TestCase): - def test_getitem_with_long(self): with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') var = fluid.dygraph.to_variable(data) - sliced = var[:, 10:, :var.shape[1]] # var.shape[1] is 80L here + sliced = var[:, 10:, : var.shape[1]] # var.shape[1] is 80L here self.assertEqual(sliced.shape, [2, 70, 80]) - sliced = var[:, var.shape[0]:, var.shape[0]:var.shape[1]] + sliced = var[:, var.shape[0] :, var.shape[0] : var.shape[1]] self.assertEqual(sliced.shape, [2, 78, 78]) def test_getitem_with_float(self): - def test_float_in_slice_item(): with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') var = fluid.dygraph.to_variable(data) - sliced = var[:, 1.1:, :var.shape[1]] + sliced = var[:, 1.1:, : var.shape[1]] self.assertRaises(Exception, test_float_in_slice_item) @@ -805,7 +803,6 @@ class TestImperativeVarBaseGetItem(unittest.TestCase): class TestInferShape(unittest.TestCase): - def test(self): x = paddle.ones(shape=[3, 4, 5]) x.desc.set_shape([3, -1, 5]) @@ -820,13 +817,18 @@ class TestInferShape(unittest.TestCase): x_arr = np.arange(0, 24, dtype=np.float32).reshape([2, 3, 4]) x = paddle.to_tensor(x_arr) - pp_slice = paddle.slice(x, [ - 100, - ], [0], [1]) + pp_slice = paddle.slice( + x, + [ + 100, + ], + [0], + [1], + ) np_slice = x_arr[:, :, 0:1] np.testing.assert_array_equal(pp_slice, np_slice) - pp_slice = paddle.slice(x, (-100, ), [0], [1]) + pp_slice = paddle.slice(x, (-100,), [0], [1]) np_slice = x_arr[0:1] np.testing.assert_array_equal(pp_slice, np_slice) @@ -834,9 +836,11 @@ class TestInferShape(unittest.TestCase): x = paddle.to_tensor(np.reshape(x_arr, (0, 0, 0))) starts = paddle.to_tensor( - np.reshape(np.array([], dtype=np.int32), (0, ))) + np.reshape(np.array([], dtype=np.int32), (0,)) + ) ends = paddle.to_tensor( - np.reshape(np.array([], dtype=np.int32), (0, ))) + np.reshape(np.array([], dtype=np.int32), (0,)) + ) with self.assertRaises(ValueError): paddle.slice(x, [-1000000], starts, ends) @@ -851,30 +855,30 @@ class TestInferShape(unittest.TestCase): paddle.slice(x, 0, starts, ends) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestImperativeCUDAPinnedInput(unittest.TestCase): - def test_input_cuda_pinned_var(self): _enable_legacy_dygraph() with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') - var = core.VarBase(value=data, - name='', - persistable=False, - place=fluid.CUDAPinnedPlace(), - zero_copy=False) - sliced = var[:, 10:, :var.shape[1]] + var = core.VarBase( + value=data, + name='', + persistable=False, + place=fluid.CUDAPinnedPlace(), + zero_copy=False, + ) + sliced = var[:, 10:, : var.shape[1]] self.assertEqual(sliced.shape, [2, 70, 80]) class TestSliceDoubleGradCheck(unittest.TestCase): - def slice_wrapper(self, x): - return paddle.slice(x[0], - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 2, 4]) + return paddle.slice( + x[0], axes=[0, 1, 2], starts=[-3, 0, 2], ends=[3, 2, 4] + ) @prog_scope() def func(self, place): @@ -884,23 +888,18 @@ class TestSliceDoubleGradCheck(unittest.TestCase): data = layers.data('data', [4, 5, 6], False, dtype) data.persistable = True - out = paddle.slice(data, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 2, 4]) + out = paddle.slice( + data, axes=[0, 1, 2], starts=[-3, 0, 2], ends=[3, 2, 4] + ) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.slice_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.slice_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -912,12 +911,10 @@ class TestSliceDoubleGradCheck(unittest.TestCase): class TestSliceTripleGradCheck(unittest.TestCase): - def slice_wrapper(self, x): - return paddle.slice(x[0], - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 2, 4]) + return paddle.slice( + x[0], axes=[0, 1, 2], starts=[-3, 0, 2], ends=[3, 2, 4] + ) @prog_scope() def func(self, place): @@ -927,23 +924,18 @@ class TestSliceTripleGradCheck(unittest.TestCase): data = layers.data('data', [4, 5, 6], False, dtype) data.persistable = True - out = paddle.slice(data, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 2, 4]) + out = paddle.slice( + data, axes=[0, 1, 2], starts=[-3, 0, 2], ends=[3, 2, 4] + ) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.slice_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.slice_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_slice_var.py b/python/paddle/fluid/tests/unittests/test_slice_var.py index a48b6de5566..b97e5243c32 100644 --- a/python/paddle/fluid/tests/unittests/test_slice_var.py +++ b/python/paddle/fluid/tests/unittests/test_slice_var.py @@ -19,15 +19,15 @@ import random class TestSliceVar(unittest.TestCase): - def check_slice_output(self, shapes, expected_sizes, min_size): var_list = [] program = fluid.Program() for shape in shapes: - var = program.global_block().create_var(name=str( - random.randint(10000, 99999)), - persistable=True, - shape=shape) + var = program.global_block().create_var( + name=str(random.randint(10000, 99999)), + persistable=True, + shape=shape, + ) var_list.append(var) blocks = slice_variable(var_list, 10, min_size) all_sizes = [] @@ -40,20 +40,33 @@ class TestSliceVar(unittest.TestCase): def test_1k(self): shapes = [[3, 5], [1024], [28, 784], [8, 1020], [800, 10]] - expected_sizes = [[15], [1024], - [ - 2352, 2352, 2352, 2352, 2352, 2352, 2352, 2352, - 2352, 784 - ], [2040, 2040, 2040, 2040], - [1150, 1150, 1150, 1150, 1150, 1150, 1100]] + expected_sizes = [ + [15], + [1024], + [2352, 2352, 2352, 2352, 2352, 2352, 2352, 2352, 2352, 784], + [2040, 2040, 2040, 2040], + [1150, 1150, 1150, 1150, 1150, 1150, 1100], + ] self.check_slice_output(shapes, expected_sizes, 1024) def test_check_output_8k(self): - shapes = [[3, 5], [1024], [28, 784], [8, 1020], [800, 10], - [6, 33, 33, 33]] - expected_sizes = [[15], [1024], [10976, 10976], [8160], [8000], - [35937, 35937, 35937, 35937, 35937, 35937]] + shapes = [ + [3, 5], + [1024], + [28, 784], + [8, 1020], + [800, 10], + [6, 33, 33, 33], + ] + expected_sizes = [ + [15], + [1024], + [10976, 10976], + [8160], + [8000], + [35937, 35937, 35937, 35937, 35937, 35937], + ] self.check_slice_output(shapes, expected_sizes, 8192) diff --git a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss.py b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss.py index f9df4160ff6..4e078e506c7 100644 --- a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss.py +++ b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss.py @@ -38,7 +38,6 @@ def smooth_l1_loss_np(input, label, reduction='mean', delta=1.0): class SmoothL1Loss(unittest.TestCase): - def setUp(self): np.random.seed(123) @@ -47,8 +46,11 @@ class SmoothL1Loss(unittest.TestCase): label_np = np.random.random([100, 200]).astype(np.float32) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype='float32') label = fluid.data(name='label', shape=[100, 200], dtype='float32') @@ -56,17 +58,21 @@ class SmoothL1Loss(unittest.TestCase): ret = smooth_l1_loss(input, label) exe = fluid.Executor(place) - static_ret, = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + (static_ret,) = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss() - dy_ret = smooth_l1_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = smooth_l1_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = smooth_l1_loss_np(input_np, label_np, reduction='mean') @@ -79,8 +85,11 @@ class SmoothL1Loss(unittest.TestCase): label_np = np.random.random([100, 200]).astype(np.float32) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype='float32') label = fluid.data(name='label', shape=[100, 200], dtype='float32') @@ -88,17 +97,21 @@ class SmoothL1Loss(unittest.TestCase): ret = smooth_l1_loss(input, label) exe = fluid.Executor(place) - static_ret, = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + (static_ret,) = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='sum') - dy_ret = smooth_l1_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = smooth_l1_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = smooth_l1_loss_np(input_np, label_np, reduction='sum') @@ -111,8 +124,11 @@ class SmoothL1Loss(unittest.TestCase): label_np = np.random.random([100, 200]).astype(np.float32) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype='float32') label = fluid.data(name='label', shape=[100, 200], dtype='float32') @@ -120,17 +136,21 @@ class SmoothL1Loss(unittest.TestCase): ret = smooth_l1_loss(input, label) exe = fluid.Executor(place) - static_ret, = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + (static_ret,) = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='none') - dy_ret = smooth_l1_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = smooth_l1_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = smooth_l1_loss_np(input_np, label_np, reduction='none') @@ -144,8 +164,11 @@ class SmoothL1Loss(unittest.TestCase): delta = np.random.rand() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype='float32') label = fluid.data(name='label', shape=[100, 200], dtype='float32') @@ -153,17 +176,21 @@ class SmoothL1Loss(unittest.TestCase): ret = smooth_l1_loss(input, label) exe = fluid.Executor(place) - static_ret, = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + (static_ret,) = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(delta=delta) - dy_ret = smooth_l1_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = smooth_l1_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = smooth_l1_loss_np(input_np, label_np, delta=delta) diff --git a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py index ffabe17915a..4b31e7ead6b 100644 --- a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py @@ -27,13 +27,12 @@ def smooth_l1_loss_forward(val, sigma2): class TestSmoothL1LossOp1(OpTest): - def setUp(self): self.op_type = "smooth_l1_loss" dims = (5, 20) self.inputs = { 'X': np.random.random(dims).astype("float32"), - 'Y': np.random.random(dims).astype("float32") + 'Y': np.random.random(dims).astype("float32"), } sigma = 3.0 self.attrs = {'sigma': sigma} @@ -43,35 +42,37 @@ class TestSmoothL1LossOp1(OpTest): loss = loss.reshape((dims[0], 1)) self.outputs = { 'Diff': diff.astype('float32'), - 'Out': loss.astype('float32') + 'Out': loss.astype('float32'), } def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.02, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.02, check_eager=True + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.03, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.03, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set('Y'), + check_eager=True, + ) class TestSmoothL1LossOp2(OpTest): - def setUp(self): self.op_type = "smooth_l1_loss" dims = (5, 20) @@ -79,7 +80,7 @@ class TestSmoothL1LossOp2(OpTest): 'X': np.random.random(dims).astype("float32"), 'Y': np.random.random(dims).astype("float32"), 'InsideWeight': np.random.random(dims).astype("float32"), - 'OutsideWeight': np.random.random(dims).astype("float32") + 'OutsideWeight': np.random.random(dims).astype("float32"), } sigma = 3.0 self.attrs = {'sigma': sigma} @@ -91,42 +92,46 @@ class TestSmoothL1LossOp2(OpTest): loss = loss.sum(1).reshape((dims[0], 1)) self.outputs = { 'Diff': diff.astype('float32'), - 'Out': loss.astype('float32') + 'Out': loss.astype('float32'), } def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.03, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.03, check_eager=True + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.03, - no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight']), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight']), + check_eager=True, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.03, - no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight']), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight']), + check_eager=True, + ) class TestSmoothL1LossOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.smooth_l1, x1, y1) # The input dtype of accuracy_op must be float32 or float64. x2 = fluid.layers.data(name='x2', shape=[4], dtype="int32") diff --git a/python/paddle/fluid/tests/unittests/test_soft_margin_loss.py b/python/paddle/fluid/tests/unittests/test_soft_margin_loss.py index b69a2a5c187..98918fb4b0b 100644 --- a/python/paddle/fluid/tests/unittests/test_soft_margin_loss.py +++ b/python/paddle/fluid/tests/unittests/test_soft_margin_loss.py @@ -27,21 +27,18 @@ def test_static_layer( prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype=input_np.dtype) - label = paddle.static.data(name='label', - shape=label_np.shape, - dtype=label_np.dtype) + input = paddle.static.data( + name='input', shape=input_np.shape, dtype=input_np.dtype + ) + label = paddle.static.data( + name='label', shape=label_np.shape, dtype=label_np.dtype + ) sm_loss = paddle.nn.loss.SoftMarginLoss(reduction=reduction) res = sm_loss(input, label) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, feed={"input": input_np, "label": label_np}, fetch_list=[res] + ) return static_result @@ -55,23 +52,20 @@ def test_static_functional( prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype=input_np.dtype) - label = paddle.static.data(name='label', - shape=label_np.shape, - dtype=label_np.dtype) - - res = paddle.nn.functional.soft_margin_loss(input, - label, - reduction=reduction) + input = paddle.static.data( + name='input', shape=input_np.shape, dtype=input_np.dtype + ) + label = paddle.static.data( + name='label', shape=label_np.shape, dtype=label_np.dtype + ) + + res = paddle.nn.functional.soft_margin_loss( + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, feed={"input": input_np, "label": label_np}, fetch_list=[res] + ) return static_result @@ -99,9 +93,9 @@ def test_dygraph_functional( input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) - dy_res = paddle.nn.functional.soft_margin_loss(input, - label, - reduction=reduction) + dy_res = paddle.nn.functional.soft_margin_loss( + input, label, reduction=reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -126,7 +120,6 @@ def calc_softmarginloss( class TestSoftMarginLoss(unittest.TestCase): - def test_SoftMarginLoss(self): input_np = np.random.uniform(0.1, 0.8, size=(5, 5)).astype(np.float64) types = [np.int32, np.int64, np.float32, np.float64] @@ -137,48 +130,58 @@ class TestSoftMarginLoss(unittest.TestCase): for place in places: for reduction in reductions: for _type in types: - label_np = np.random.randint(0, 2, - size=(5, 5)).astype(_type) + label_np = np.random.randint(0, 2, size=(5, 5)).astype( + _type + ) label_np[label_np == 0] = -1 - static_result = test_static_layer(place, input_np, label_np, - reduction) - dy_result = test_dygraph_layer(place, input_np, label_np, - reduction) - expected = calc_softmarginloss(input_np, label_np, - reduction) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-05) + static_result = test_static_layer( + place, input_np, label_np, reduction + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction + ) + expected = calc_softmarginloss( + input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_result, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-05 + ) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) static_functional = test_static_functional( - place, input_np, label_np, reduction) + place, input_np, label_np, reduction + ) dy_functional = test_dygraph_functional( - place, input_np, label_np, reduction) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-05) + place, input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-05 + ) def test_SoftMarginLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.loss.SoftMarginLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.loss.SoftMarginLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[-1.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.soft_margin_loss, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.soft_margin_loss, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_softmax2d.py b/python/paddle/fluid/tests/unittests/test_softmax2d.py index c5e45dec228..8297ca9c908 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax2d.py +++ b/python/paddle/fluid/tests/unittests/test_softmax2d.py @@ -20,13 +20,15 @@ from test_softmax_op import ref_softmax class TestSoftmax2DAPI(unittest.TestCase): - def setUp(self): self.shape = [2, 6, 5, 4] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') self.axis = -3 - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -35,7 +37,7 @@ class TestSoftmax2DAPI(unittest.TestCase): m = paddle.nn.Softmax2D() out = m(x) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={'X': self.x_np}, fetch_list=[out]) + (res,) = exe.run(feed={'X': self.x_np}, fetch_list=[out]) out_ref = ref_softmax(self.x_np, self.axis) np.testing.assert_allclose(out_ref, res, rtol=1e-05) @@ -50,27 +52,30 @@ class TestSoftmax2DAPI(unittest.TestCase): class TestSoftmax2DShape(TestSoftmax2DAPI): - def setUp(self): self.shape = [2, 6, 4] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') self.axis = -3 - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSoftmax2DFloat32(TestSoftmax2DAPI): - def setUp(self): self.shape = [2, 3, 4] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float32') self.axis = -3 - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSoftmax2DCPU(TestSoftmax2DAPI): - def setUp(self): self.shape = [2, 6, 4] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') @@ -79,10 +84,12 @@ class TestSoftmax2DCPU(TestSoftmax2DAPI): class TestSoftmax2DRepr(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_extra_repr(self): paddle.disable_static(self.place) @@ -92,10 +99,12 @@ class TestSoftmax2DRepr(unittest.TestCase): class TestSoftmax2DError(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_error(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_op.py b/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_op.py index 2bdede66f4a..c36b8362c84 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_op.py @@ -35,10 +35,10 @@ def _get_softmax(x, mask, fp16=True): return rst -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxMaskFuseOp(OpTest): - def setUp(self): self.op_type = "fused_softmax_mask" x = np.random.random((1, 1, 8, 32)) @@ -61,10 +61,10 @@ class TestSoftmaxMaskFuseOp(OpTest): pass -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxMaskFuseOp0(OpTest): - def setUp(self): self.op_type = "fused_softmax_mask" x = np.random.random((1, 1, 8, 32)).astype("float16") @@ -81,16 +81,16 @@ class TestSoftmaxMaskFuseOp0(OpTest): self.check_grad_with_place(core.CUDAPlace(0), ["X"], "Out") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestDropoutBiasFuseOp3(unittest.TestCase): - def test_static_result(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_x = fluid.data(name="x", shape=[1, 1, 8, 32], dtype="float32") - input_mask = fluid.data(name="mask", - shape=[1, 1, 8, 32], - dtype="float32") + input_mask = fluid.data( + name="mask", shape=[1, 1, 8, 32], dtype="float32" + ) rst = incubate.softmax_mask_fuse(input_x, input_mask) x_in_np = np.random.random((1, 1, 8, 32)).astype("float32") @@ -99,12 +99,11 @@ class TestDropoutBiasFuseOp3(unittest.TestCase): rst_np = _get_softmax(x_in_np, mask_in_np, False) exe = fluid.Executor(fluid.CUDAPlace(0)) - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": x_in_np, - "mask": mask_in_np - }, - fetch_list=[rst]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": x_in_np, "mask": mask_in_np}, + fetch_list=[rst], + ) np.testing.assert_allclose(fetches[0], rst_np, rtol=1e-05) def test_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_upper_triangle_op.py b/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_upper_triangle_op.py index 62a2e729051..84032336402 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_upper_triangle_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_upper_triangle_op.py @@ -36,10 +36,10 @@ def _get_softmax_upper(x, fp16=True): return rst -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxMaskFuseOp(OpTest): - def setUp(self): self.op_type = "fused_softmax_mask_upper_triangle" x = np.random.random((1, 4, 32, 32)).astype("float16") @@ -54,10 +54,10 @@ class TestSoftmaxMaskFuseOp(OpTest): self.check_grad_with_place(core.CUDAPlace(0), ["X"], "Out") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxMaskFuseOp1(OpTest): - def setUp(self): self.op_type = "fused_softmax_mask_upper_triangle" x = np.random.random((1, 4, 32, 32)) @@ -78,8 +78,9 @@ class TestSoftmaxMaskFuseOp1(OpTest): pass -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestDropoutBiasFuseOp2(unittest.TestCase): # test the python side API for softmax_mask_fuse op def setUp(self): @@ -89,18 +90,20 @@ class TestDropoutBiasFuseOp2(unittest.TestCase): def test_static(self): for dtype in self.dtypes: with fluid.program_guard(fluid.Program(), fluid.Program()): - input_x = fluid.data(name="x", - shape=[1, 4, 32, 32], - dtype=dtype) + input_x = fluid.data( + name="x", shape=[1, 4, 32, 32], dtype=dtype + ) rst = incubate.softmax_mask_fuse_upper_triangle(input_x) x_in_np = np.random.random((1, 4, 32, 32)).astype(dtype) rst_np = _get_softmax_upper(x_in_np, dtype == 'float16') exe = fluid.Executor(fluid.CUDAPlace(0)) - fetches = exe.run(fluid.default_main_program(), - feed={"x": x_in_np}, - fetch_list=[rst]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": x_in_np}, + fetch_list=[rst], + ) np.testing.assert_allclose(fetches[0], rst_np, rtol=1e-05) def test_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_softmax_op.py b/python/paddle/fluid/tests/unittests/test_softmax_op.py index 13a43e4d3e9..75324475f46 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_op.py @@ -26,7 +26,7 @@ def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -41,7 +41,6 @@ def ref_softmax(x, axis=None, dtype=None): class TestSoftmaxOp(OpTest): - def get_x_shape(self): return [10, 10] @@ -67,7 +66,7 @@ class TestSoftmaxOp(OpTest): self.attrs = { 'axis': self.axis, 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def init_kernel_type(self): @@ -78,7 +77,8 @@ class TestSoftmaxOp(OpTest): if self.use_cudnn: place = core.CUDAPlace(0) self.check_output_with_place( - place, atol=1e-5, check_dygraph=(self.use_mkldnn == False)) + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) else: self.check_output(check_dygraph=(self.use_mkldnn == False)) @@ -88,25 +88,27 @@ class TestSoftmaxOp(OpTest): place = core.CUDAPlace(0) if core.is_float16_supported(place): self.check_grad_with_place( - place, ["X"], + place, + ["X"], "Out", max_relative_error=0.01, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) else: - self.check_grad(["X"], - "Out", - max_relative_error=0.01, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + ["X"], + "Out", + max_relative_error=0.01, + check_dygraph=(self.use_mkldnn == False), + ) class TestSoftmaxOp2(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] class TestSoftmaxOp3(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -115,7 +117,6 @@ class TestSoftmaxOp3(TestSoftmaxOp): class TestSoftmaxOp4(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -124,7 +125,6 @@ class TestSoftmaxOp4(TestSoftmaxOp): class TestSoftmaxOp5(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -133,7 +133,6 @@ class TestSoftmaxOp5(TestSoftmaxOp): class TestSoftmaxOp6(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -141,26 +140,26 @@ class TestSoftmaxOp6(TestSoftmaxOp): return 3 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp(TestSoftmaxOp): - def init_kernel_type(self): self.use_cudnn = True -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp2(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp3(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -168,10 +167,10 @@ class TestSoftmaxCUDNNOp3(TestSoftmaxCUDNNOp): return 0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp4(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -179,10 +178,10 @@ class TestSoftmaxCUDNNOp4(TestSoftmaxCUDNNOp): return 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp5(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -190,10 +189,10 @@ class TestSoftmaxCUDNNOp5(TestSoftmaxCUDNNOp): return 2 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp6(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -201,18 +200,18 @@ class TestSoftmaxCUDNNOp6(TestSoftmaxCUDNNOp): return 3 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp7(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp8(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -220,10 +219,10 @@ class TestSoftmaxCUDNNOp8(TestSoftmaxCUDNNOp): return 0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp9(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -231,10 +230,10 @@ class TestSoftmaxCUDNNOp9(TestSoftmaxCUDNNOp): return 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp10(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -242,10 +241,10 @@ class TestSoftmaxCUDNNOp10(TestSoftmaxCUDNNOp): return 2 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp11(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -253,10 +252,10 @@ class TestSoftmaxCUDNNOp11(TestSoftmaxCUDNNOp): return 3 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp12(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -264,10 +263,10 @@ class TestSoftmaxCUDNNOp12(TestSoftmaxCUDNNOp): return 4 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxFP16Op(TestSoftmaxOp): - def init_kernel_type(self): self.dtype = np.float16 @@ -282,18 +281,18 @@ class TestSoftmaxFP16Op(TestSoftmaxOp): pass -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxFP16Op2(TestSoftmaxFP16Op): - def get_x_shape(self): return [2, 3, 4, 10] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxFP16CUDNNOp(TestSoftmaxOp): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -305,18 +304,18 @@ class TestSoftmaxFP16CUDNNOp(TestSoftmaxOp): self.check_output_with_place(place, atol=1e-3) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxFP16CUDNNOp2(TestSoftmaxFP16CUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxBF16Op(OpTest): - def setUp(self): self.op_type = "softmax" self.use_cudnn = self.init_cudnn() @@ -336,7 +335,7 @@ class TestSoftmaxBF16Op(OpTest): self.attrs = { 'axis': self.axis, 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def init_cudnn(self): @@ -344,34 +343,40 @@ class TestSoftmaxBF16Op(OpTest): def test_check_output(self): place = core.CUDAPlace(0) - self.check_output_with_place(place, - check_dygraph=(self.use_mkldnn == False)) + self.check_output_with_place( + place, check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ["X"], - "Out", - numeric_grad_delta=0.05, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ["X"], + "Out", + numeric_grad_delta=0.05, + check_dygraph=(self.use_mkldnn == False), + ) @unittest.skipIf( - not core.is_compiled_with_cuda() or core.cudnn_version() < 8100 + not core.is_compiled_with_cuda() + or core.cudnn_version() < 8100 or paddle.device.cuda.get_device_capability()[0] < 8, - "only support compiled with CUDA and cudnn version need larger than 8.1.0 and device's compute capability is at least 8.0" + "only support compiled with CUDA and cudnn version need larger than 8.1.0 and device's compute capability is at least 8.0", ) class TestSoftmaxBF16CUDNNOp(TestSoftmaxBF16Op): - def init_cudnn(self): return True class TestSoftmaxAPI(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() - self.x_np = np.random.uniform(-1., 1., [2, 3, 4, 5]).astype('float32') + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + self.x_np = np.random.uniform(-1.0, 1.0, [2, 3, 4, 5]).astype('float32') self.out_ref = np.apply_along_axis(stable_softmax, -1, self.x_np) self.executed_api() @@ -426,19 +431,18 @@ class TestSoftmaxAPI(unittest.TestCase): # The input type must be Variable. self.assertRaises(TypeError, self.softmax, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) self.assertRaises(TypeError, self.softmax, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[2, 3], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[2, 3], dtype='float16' + ) self.softmax(x_fp16) class TestSoftmaxInplaceAPI(TestSoftmaxAPI): - def executed_api(self): self.softmax = F.softmax_ diff --git a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py index 6b4a733853b..fb0e46b6740 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py @@ -28,7 +28,7 @@ def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): axis %= len(shape) n = int(np.prod(shape[:axis])) axis_dim = shape[axis] - remain = int(np.prod(shape[axis + 1:])) + remain = int(np.prod(shape[axis + 1 :])) softmax_reshape = softmax.reshape((n, axis_dim, remain)) label_reshape = label.reshape((n, 1, remain)) result = np.zeros_like(label_reshape, dtype=softmax.dtype) @@ -40,13 +40,15 @@ def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): return result.reshape(label.shape) -def python_api(logits, - label, - soft_label=False, - use_softmax=True, - numeric_stable_mode=True, - ignore_index=-100, - axis=-1): +def python_api( + logits, + label, + soft_label=False, + use_softmax=True, + numeric_stable_mode=True, + ignore_index=-100, + axis=-1, +): # here only can test paddle.nn.functional.softmax_with_cross_entropy, # the paddle.nn.functional.cross_entropy contains other math ops return paddle.nn.functional.softmax_with_cross_entropy( @@ -56,23 +58,31 @@ def python_api(logits, ignore_index=ignore_index, numeric_stable_mode=numeric_stable_mode, return_softmax=use_softmax, - axis=axis) - - -def python_core_api_without_softmax(logits, - label, - soft_label=False, - use_softmax=False, - numeric_stable_mode=True, - ignore_index=-100, - axis=-1): + axis=axis, + ) + + +def python_core_api_without_softmax( + logits, + label, + soft_label=False, + use_softmax=False, + numeric_stable_mode=True, + ignore_index=-100, + axis=-1, +): # the API paddle.nn.functional.softmax_with_cross_entropy cannot # set use_softmax=False, so add a core api manually assert use_softmax is False - _, loss = paddle._C_ops.cross_entropy_with_softmax(logits, label, - soft_label, use_softmax, - numeric_stable_mode, - ignore_index, axis) + _, loss = paddle._C_ops.cross_entropy_with_softmax( + logits, + label, + soft_label, + use_softmax, + numeric_stable_mode, + ignore_index, + axis, + ) return loss @@ -101,8 +111,10 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): self.initParams() logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, logits) if self.soft_label: @@ -111,13 +123,13 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): else: axis_dim = self.shape[self.axis] self.shape[self.axis] = 1 - labels = np.random.randint(0, - axis_dim, - self.shape, - dtype=self.hard_label_dtype()) + labels = np.random.randint( + 0, axis_dim, self.shape, dtype=self.hard_label_dtype() + ) - loss = cross_entropy(softmax, labels, self.soft_label, self.axis, - self.ignore_index) + loss = cross_entropy( + softmax, labels, self.soft_label, self.axis, self.ignore_index + ) if self.use_softmax == False: self.inputs = {"Logits": softmax, "Label": labels} @@ -126,7 +138,7 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): self.outputs = { "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -146,48 +158,48 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): def test_check_grad(self): if core.is_compiled_with_rocm(): if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - max_relative_error=5e-1, - check_eager=True) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=5e-1, + check_eager=True, + ) # HIP will have accuracy fail when using float32 in CPU place self.check_grad(["Logits"], "Loss", max_relative_error=5e-1) else: if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - numeric_grad_delta=0.001, - check_eager=True) + self.check_grad( + ["Logits"], + "Loss", + numeric_grad_delta=0.001, + check_eager=True, + ) self.check_grad(["Logits"], "Loss", numeric_grad_delta=0.001) class TestSoftmaxWithCrossEntropyOpInt32(TestSoftmaxWithCrossEntropyOp): - def hard_label_dtype(self): return "int32" class TestSoftmaxWithCrossEntropyOpInt16(TestSoftmaxWithCrossEntropyOp): - def hard_label_dtype(self): return "int16" class TestSoftmaxWithCrossEntropyOpInt8(TestSoftmaxWithCrossEntropyOp): - def hard_label_dtype(self): return "int8" class TestSoftmaxWithCrossEntropyOpUInt8(TestSoftmaxWithCrossEntropyOp): - def hard_label_dtype(self): return "uint8" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_1D( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -198,12 +210,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_1D( self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_1D( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -214,15 +226,15 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_1D( self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" ############################################################################## -#NotWithSoftmax_SoftLabel_2D start +# NotWithSoftmax_SoftLabel_2D start ############################################################################## class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -233,12 +245,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D( self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis2( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -249,12 +261,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis2( self.axis = 1 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis3( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -265,12 +277,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis3( self.axis = 2 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis4( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -281,21 +293,21 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis4( self.axis = 3 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" ############################################################################## -#NotWithSoftmax_SoftLabel_2D end +# NotWithSoftmax_SoftLabel_2D end ############################################################################## ############################################################################## -#NotWithSoftmax_HardLabel_2D start +# NotWithSoftmax_HardLabel_2D start ############################################################################## class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -306,12 +318,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D( self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis2( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -322,12 +334,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis2( self.axis = 1 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis3( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -338,12 +350,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis3( self.axis = 2 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis4( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -354,21 +366,21 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis4( self.axis = 3 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" ############################################################################## -#NotWithSoftmax_HardLabel_2D end +# NotWithSoftmax_HardLabel_2D end ############################################################################## ############################################################################## -#NotWithSoftmax_HardLabel_2D_Ignore start +# NotWithSoftmax_HardLabel_2D_Ignore start ############################################################################## class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_Ignore( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -379,12 +391,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_Ignore( self.axis = -1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_Ignore_Axis( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -395,12 +407,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_Ignore_Axis( self.axis = 1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Ignore( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -411,12 +423,12 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Ignore( self.axis = -1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Ignore_Axis3( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -427,16 +439,15 @@ class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Ignore_Axis3( self.axis = 2 self.ignore_index = 2 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" ############################################################################## -#NotWithSoftmax_HardLabel_2D_Ignore end +# NotWithSoftmax_HardLabel_2D_Ignore end ############################################################################## class TestSoftmaxWithCrossEntropyOpNoCudnn(TestSoftmaxWithCrossEntropyOp): - def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -450,10 +461,10 @@ class TestSoftmaxWithCrossEntropyOpNoCudnn(TestSoftmaxWithCrossEntropyOp): self.use_softmax = True -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxWithCrossEntropyOpFp16(TestSoftmaxWithCrossEntropyOp): - def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -472,8 +483,10 @@ class TestSoftmaxWithCrossEntropyOpFp16(TestSoftmaxWithCrossEntropyOp): # NOTE: numpy float16 have very low accuracy, use float32 for numpy check. date_type = np.float32 if core.is_compiled_with_rocm() else np.float64 logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(date_type)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(date_type), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, logits) axis_dim = self.shape[self.axis] @@ -485,7 +498,7 @@ class TestSoftmaxWithCrossEntropyOpFp16(TestSoftmaxWithCrossEntropyOp): self.inputs = {"Logits": logits.astype(self.dtype), "Label": labels} self.outputs = { "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -501,16 +514,15 @@ class TestSoftmaxWithCrossEntropyOpFp16(TestSoftmaxWithCrossEntropyOp): def test_check_grad(self): if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.1, - check_eager=True) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_eager=True + ) self.check_grad(["Logits"], "Loss", max_relative_error=0.1) -class TestSoftmaxWithCrossEntropyOpNoCudnnFp16(TestSoftmaxWithCrossEntropyOpFp16 - ): - +class TestSoftmaxWithCrossEntropyOpNoCudnnFp16( + TestSoftmaxWithCrossEntropyOpFp16 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -524,10 +536,9 @@ class TestSoftmaxWithCrossEntropyOpNoCudnnFp16(TestSoftmaxWithCrossEntropyOpFp16 def test_check_grad(self): if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.1, - check_eager=True) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_eager=True + ) self.check_grad(["Logits"], "Loss", max_relative_error=0.1) @@ -557,10 +568,9 @@ class TestSoftmaxWithCrossEntropyOp2(TestSoftmaxWithCrossEntropyOp): if core.is_compiled_with_rocm(): # HIP will have accuracy fail when using float32 in CPU place if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.1, - check_eager=True) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_eager=True + ) self.check_grad(["Logits"], "Loss", max_relative_error=0.1) else: if self.python_api is not None: @@ -587,7 +597,6 @@ class TestSoftmaxWithCrossEntropyOp3(TestSoftmaxWithCrossEntropyOp): class TestSoftmaxWithCrossEntropyOp3NoCudnn(TestSoftmaxWithCrossEntropyOp3): - def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -677,8 +686,9 @@ class TestSoftmaxWithCrossEntropyOpAxis4(TestSoftmaxWithCrossEntropyOp): self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpAxisDimEqualOne(TestSoftmaxWithCrossEntropyOp - ): +class TestSoftmaxWithCrossEntropyOpAxisDimEqualOne( + TestSoftmaxWithCrossEntropyOp +): """ Test softmax with cross entropy operator with discreate one-hot labels. Given axis != -1 @@ -698,8 +708,8 @@ class TestSoftmaxWithCrossEntropyOpAxisDimEqualOne(TestSoftmaxWithCrossEntropyOp class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis1( - TestSoftmaxWithCrossEntropyOpNoCudnnFp16): - + TestSoftmaxWithCrossEntropyOpNoCudnnFp16 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -714,8 +724,8 @@ class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis1( class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis2( - TestSoftmaxWithCrossEntropyOpNoCudnnFp16): - + TestSoftmaxWithCrossEntropyOpNoCudnnFp16 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -730,8 +740,8 @@ class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis2( class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis3( - TestSoftmaxWithCrossEntropyOpNoCudnnFp16): - + TestSoftmaxWithCrossEntropyOpNoCudnnFp16 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -745,9 +755,9 @@ class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis3( self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpSoftLabelAxis1(TestSoftmaxWithCrossEntropyOp2 - ): - +class TestSoftmaxWithCrossEntropyOpSoftLabelAxis1( + TestSoftmaxWithCrossEntropyOp2 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -761,9 +771,9 @@ class TestSoftmaxWithCrossEntropyOpSoftLabelAxis1(TestSoftmaxWithCrossEntropyOp2 self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpSoftLabelAxis2(TestSoftmaxWithCrossEntropyOp2 - ): - +class TestSoftmaxWithCrossEntropyOpSoftLabelAxis2( + TestSoftmaxWithCrossEntropyOp2 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -777,9 +787,9 @@ class TestSoftmaxWithCrossEntropyOpSoftLabelAxis2(TestSoftmaxWithCrossEntropyOp2 self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpSoftLabelAxis3(TestSoftmaxWithCrossEntropyOp2 - ): - +class TestSoftmaxWithCrossEntropyOpSoftLabelAxis3( + TestSoftmaxWithCrossEntropyOp2 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -793,9 +803,9 @@ class TestSoftmaxWithCrossEntropyOpSoftLabelAxis3(TestSoftmaxWithCrossEntropyOp2 self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpSoftLabelAxis4(TestSoftmaxWithCrossEntropyOp2 - ): - +class TestSoftmaxWithCrossEntropyOpSoftLabelAxis4( + TestSoftmaxWithCrossEntropyOp2 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -810,8 +820,8 @@ class TestSoftmaxWithCrossEntropyOpSoftLabelAxis4(TestSoftmaxWithCrossEntropyOp2 class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis1( - TestSoftmaxWithCrossEntropyOp3): - + TestSoftmaxWithCrossEntropyOp3 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -826,8 +836,8 @@ class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis1( class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis2( - TestSoftmaxWithCrossEntropyOp3): - + TestSoftmaxWithCrossEntropyOp3 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -842,8 +852,8 @@ class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis2( class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis3( - TestSoftmaxWithCrossEntropyOp3): - + TestSoftmaxWithCrossEntropyOp3 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -858,8 +868,8 @@ class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis3( class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis4( - TestSoftmaxWithCrossEntropyOp3): - + TestSoftmaxWithCrossEntropyOp3 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api diff --git a/python/paddle/fluid/tests/unittests/test_solve_op.py b/python/paddle/fluid/tests/unittests/test_solve_op.py index b07c3d5d080..4fcea5e67d5 100644 --- a/python/paddle/fluid/tests/unittests/test_solve_op.py +++ b/python/paddle/fluid/tests/unittests/test_solve_op.py @@ -27,7 +27,6 @@ from paddle.fluid.framework import _test_eager_guard # 2D normal case class TestSolveOp(OpTest): - def config(self): self.python_api = paddle.linalg.solve self.input_x_matrix_shape = [15, 15] @@ -42,7 +41,7 @@ class TestSolveOp(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random(self.input_x_matrix_shape).astype(self.dtype), - 'Y': np.random.random(self.input_y_matrix_shape).astype(self.dtype) + 'Y': np.random.random(self.input_y_matrix_shape).astype(self.dtype), } self.outputs = { 'Out': np.linalg.solve(self.inputs['X'], self.inputs['Y']) @@ -57,7 +56,6 @@ class TestSolveOp(OpTest): # x broadcast + 3D batch case class TestSolveOpBatched_case0(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -65,7 +63,7 @@ class TestSolveOpBatched_case0(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((11, 11)).astype(self.dtype), - 'Y': np.random.random((2, 11, 7)).astype(self.dtype) + 'Y': np.random.random((2, 11, 7)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -74,15 +72,13 @@ class TestSolveOpBatched_case0(OpTest): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=1e-1, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=1e-1, check_eager=True + ) # 3D batch + y vector case class TestSolveOpBatched_case1(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -90,7 +86,7 @@ class TestSolveOpBatched_case1(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((20, 6, 6)).astype(self.dtype), - 'Y': np.random.random((20, 6)).astype(self.dtype) + 'Y': np.random.random((20, 6)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -99,15 +95,13 @@ class TestSolveOpBatched_case1(OpTest): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.04, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.04, check_eager=True + ) # 3D batch + y broadcast case class TestSolveOpBatched_case2(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -115,7 +109,7 @@ class TestSolveOpBatched_case2(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 10, 10)).astype(self.dtype), - 'Y': np.random.random((1, 10, 10)).astype(self.dtype) + 'Y': np.random.random((1, 10, 10)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -124,15 +118,13 @@ class TestSolveOpBatched_case2(OpTest): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.02, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.02, check_eager=True + ) # x broadcast + 3D batch case class TestSolveOpBatched_case3(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -140,7 +132,7 @@ class TestSolveOpBatched_case3(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((1, 10, 10)).astype(self.dtype), - 'Y': np.random.random((2, 10, 10)).astype(self.dtype) + 'Y': np.random.random((2, 10, 10)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -149,15 +141,13 @@ class TestSolveOpBatched_case3(OpTest): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.02, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.02, check_eager=True + ) # 3D normal batch case class TestSolveOpBatched_case4(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -165,7 +155,7 @@ class TestSolveOpBatched_case4(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((3, 6, 6)).astype(self.dtype), - 'Y': np.random.random((3, 6, 7)).astype(self.dtype) + 'Y': np.random.random((3, 6, 7)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -179,7 +169,6 @@ class TestSolveOpBatched_case4(OpTest): # 4D normal batch case class TestSolveOpBatched_case5(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -187,7 +176,7 @@ class TestSolveOpBatched_case5(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 2, 6, 6)).astype(self.dtype), - 'Y': np.random.random((2, 2, 6, 6)).astype(self.dtype) + 'Y': np.random.random((2, 2, 6, 6)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -201,7 +190,6 @@ class TestSolveOpBatched_case5(OpTest): # 4D batch + y broadcast case class TestSolveOpBatched_case6(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -209,7 +197,7 @@ class TestSolveOpBatched_case6(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 2, 6, 6)).astype(self.dtype), - 'Y': np.random.random((1, 2, 6, 9)).astype(self.dtype) + 'Y': np.random.random((1, 2, 6, 9)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -223,7 +211,6 @@ class TestSolveOpBatched_case6(OpTest): # 5D normal batch case class TestSolveOpBatched_case7(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -231,7 +218,7 @@ class TestSolveOpBatched_case7(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 2, 2, 4, 4)).astype(self.dtype), - 'Y': np.random.random((2, 2, 2, 4, 4)).astype(self.dtype) + 'Y': np.random.random((2, 2, 2, 4, 4)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -240,15 +227,13 @@ class TestSolveOpBatched_case7(OpTest): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.04, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.04, check_eager=True + ) # 5D batch + y broadcast case class TestSolveOpBatched_case8(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -256,7 +241,7 @@ class TestSolveOpBatched_case8(OpTest): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 2, 2, 4, 4)).astype(self.dtype), - 'Y': np.random.random((1, 2, 2, 4, 7)).astype(self.dtype) + 'Y': np.random.random((1, 2, 2, 4, 7)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -265,21 +250,21 @@ class TestSolveOpBatched_case8(OpTest): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.04, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.04, check_eager=True + ) class TestSolveOpError(unittest.TestCase): - def func_errors(self): with program_guard(Program(), Program()): # The input type of solve_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.linalg.solve, x1, y1) # The data type of input must be float32 or float64. @@ -317,7 +302,6 @@ class TestSolveOpError(unittest.TestCase): # 2D + vector case, FP64 class TestSolveOpAPI_1(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -327,12 +311,12 @@ class TestSolveOpAPI_1(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - paddle_input_x = fluid.data(name="input_x", - shape=[3, 3], - dtype=self.dtype) - paddle_input_y = fluid.data(name="input_y", - shape=[3], - dtype=self.dtype) + paddle_input_x = fluid.data( + name="input_x", shape=[3, 3], dtype=self.dtype + ) + paddle_input_y = fluid.data( + name="input_y", shape=[3], dtype=self.dtype + ) paddle_result = paddle.linalg.solve(paddle_input_x, paddle_input_y) np_input_x = np.random.random([3, 3]).astype(self.dtype) @@ -341,22 +325,20 @@ class TestSolveOpAPI_1(unittest.TestCase): np_result = np.linalg.solve(np_input_x, np_input_y) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": np_input_x, - "input_y": np_input_y - }, - fetch_list=[paddle_result]) - np.testing.assert_allclose(fetches[0], - np.linalg.solve(np_input_x, np_input_y), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": np_input_x, "input_y": np_input_y}, + fetch_list=[paddle_result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.solve(np_input_x, np_input_y), rtol=1e-05 + ) def test_static(self): for place in self.place: self.check_static_result(place=place) def func_dygraph(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -368,9 +350,9 @@ class TestSolveOpAPI_1(unittest.TestCase): numpy_output = np.linalg.solve(input_x_np, input_y_np) paddle_output = paddle.linalg.solve(tensor_input_x, tensor_input_y) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -385,7 +367,6 @@ class TestSolveOpAPI_1(unittest.TestCase): # 2D normal case, FP64 class TestSolveOpAPI_2(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -396,12 +377,12 @@ class TestSolveOpAPI_2(unittest.TestCase): def check_static_result(self, place): paddle.enable_static() with fluid.program_guard(fluid.Program(), fluid.Program()): - paddle_input_x = fluid.data(name="input_x", - shape=[10, 10], - dtype=self.dtype) - paddle_input_y = fluid.data(name="input_y", - shape=[10, 4], - dtype=self.dtype) + paddle_input_x = fluid.data( + name="input_x", shape=[10, 10], dtype=self.dtype + ) + paddle_input_y = fluid.data( + name="input_y", shape=[10, 4], dtype=self.dtype + ) paddle_result = paddle.linalg.solve(paddle_input_x, paddle_input_y) np_input_x = np.random.random([10, 10]).astype(self.dtype) @@ -410,22 +391,20 @@ class TestSolveOpAPI_2(unittest.TestCase): np_result = np.linalg.solve(np_input_x, np_input_y) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": np_input_x, - "input_y": np_input_y - }, - fetch_list=[paddle_result]) - np.testing.assert_allclose(fetches[0], - np.linalg.solve(np_input_x, np_input_y), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": np_input_x, "input_y": np_input_y}, + fetch_list=[paddle_result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.solve(np_input_x, np_input_y), rtol=1e-05 + ) def test_static(self): for place in self.place: self.check_static_result(place=place) def func_dygraph(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -436,9 +415,9 @@ class TestSolveOpAPI_2(unittest.TestCase): numpy_output = np.linalg.solve(input_x_np, input_y_np) paddle_output = paddle.linalg.solve(tensor_input_x, tensor_input_y) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -453,7 +432,6 @@ class TestSolveOpAPI_2(unittest.TestCase): # 2D normal case, FP32 class TestSolveOpAPI_3(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -464,12 +442,12 @@ class TestSolveOpAPI_3(unittest.TestCase): def check_static_result(self, place): paddle.enable_static() with fluid.program_guard(fluid.Program(), fluid.Program()): - paddle_input_x = fluid.data(name="input_x", - shape=[10, 10], - dtype=self.dtype) - paddle_input_y = fluid.data(name="input_y", - shape=[10, 4], - dtype=self.dtype) + paddle_input_x = fluid.data( + name="input_x", shape=[10, 10], dtype=self.dtype + ) + paddle_input_y = fluid.data( + name="input_y", shape=[10, 4], dtype=self.dtype + ) paddle_result = paddle.linalg.solve(paddle_input_x, paddle_input_y) np_input_x = np.random.random([10, 10]).astype(self.dtype) @@ -478,22 +456,20 @@ class TestSolveOpAPI_3(unittest.TestCase): np_result = np.linalg.solve(np_input_x, np_input_y) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": np_input_x, - "input_y": np_input_y - }, - fetch_list=[paddle_result]) - np.testing.assert_allclose(fetches[0], - np.linalg.solve(np_input_x, np_input_y), - rtol=0.0001) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": np_input_x, "input_y": np_input_y}, + fetch_list=[paddle_result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.solve(np_input_x, np_input_y), rtol=0.0001 + ) def test_static(self): for place in self.place: self.check_static_result(place=place) def func_dygraph(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -505,9 +481,9 @@ class TestSolveOpAPI_3(unittest.TestCase): numpy_output = np.linalg.solve(input_x_np, input_y_np) paddle_output = paddle.linalg.solve(tensor_input_x, tensor_input_y) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=0.0001) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=0.0001 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -522,7 +498,6 @@ class TestSolveOpAPI_3(unittest.TestCase): # 3D + y broadcast case, FP64 class TestSolveOpAPI_4(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -532,12 +507,12 @@ class TestSolveOpAPI_4(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - paddle_input_x = fluid.data(name="input_x", - shape=[2, 3, 3], - dtype=self.dtype) - paddle_input_y = fluid.data(name="input_y", - shape=[1, 3, 3], - dtype=self.dtype) + paddle_input_x = fluid.data( + name="input_x", shape=[2, 3, 3], dtype=self.dtype + ) + paddle_input_y = fluid.data( + name="input_y", shape=[1, 3, 3], dtype=self.dtype + ) paddle_result = paddle.linalg.solve(paddle_input_x, paddle_input_y) np_input_x = np.random.random([2, 3, 3]).astype(self.dtype) @@ -546,22 +521,20 @@ class TestSolveOpAPI_4(unittest.TestCase): np_result = np.linalg.solve(np_input_x, np_input_y) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": np_input_x, - "input_y": np_input_y - }, - fetch_list=[paddle_result]) - np.testing.assert_allclose(fetches[0], - np.linalg.solve(np_input_x, np_input_y), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": np_input_x, "input_y": np_input_y}, + fetch_list=[paddle_result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.solve(np_input_x, np_input_y), rtol=1e-05 + ) def test_static(self): for place in self.place: self.check_static_result(place=place) def func_dygraph(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -573,9 +546,9 @@ class TestSolveOpAPI_4(unittest.TestCase): numpy_output = np.linalg.solve(input_x_np, input_y_np) paddle_output = paddle.linalg.solve(tensor_input_x, tensor_input_y) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -608,12 +581,11 @@ class TestSolveOpSingularAPI(unittest.TestCase): exe = fluid.Executor(place) try: - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": input_x_np, - "y": input_y_np - }, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": input_x_np, "y": input_y_np}, + fetch_list=[result], + ) except RuntimeError as ex: print("The mat is singular") except ValueError as ex: diff --git a/python/paddle/fluid/tests/unittests/test_sort_op.py b/python/paddle/fluid/tests/unittests/test_sort_op.py index d28af8f684b..387f5474dd4 100644 --- a/python/paddle/fluid/tests/unittests/test_sort_op.py +++ b/python/paddle/fluid/tests/unittests/test_sort_op.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestSortOnCPU(unittest.TestCase): - def setUp(self): self.place = core.CPUPlace() @@ -30,10 +29,14 @@ class TestSortOnCPU(unittest.TestCase): input = fluid.data(name="input", shape=[2, 3, 4], dtype="float32") output = paddle.sort(x=input) exe = fluid.Executor(self.place) - data = np.array([[[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]], - [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]]], - dtype='float32') - result, = exe.run(feed={'input': data}, fetch_list=[output]) + data = np.array( + [ + [[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]], + [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]], + ], + dtype='float32', + ) + (result,) = exe.run(feed={'input': data}, fetch_list=[output]) np_result = np.sort(result) self.assertEqual((result == np_result).all(), True) @@ -42,16 +45,19 @@ class TestSortOnCPU(unittest.TestCase): input = fluid.data(name="input", shape=[2, 3, 4], dtype="float32") output = paddle.sort(x=input, axis=1) exe = fluid.Executor(self.place) - data = np.array([[[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]], - [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]]], - dtype='float32') - result, = exe.run(feed={'input': data}, fetch_list=[output]) + data = np.array( + [ + [[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]], + [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]], + ], + dtype='float32', + ) + (result,) = exe.run(feed={'input': data}, fetch_list=[output]) np_result = np.sort(result, axis=1) self.assertEqual((result == np_result).all(), True) class TestSortOnGPU(TestSortOnCPU): - def init_place(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -60,7 +66,6 @@ class TestSortOnGPU(TestSortOnCPU): class TestSortDygraph(unittest.TestCase): - def setUp(self): self.input_data = np.random.rand(10, 10) if core.is_compiled_with_cuda(): @@ -84,8 +89,9 @@ class TestSortDygraph(unittest.TestCase): paddle.disable_static(self.place) var_x = paddle.to_tensor(self.input_data) out = paddle.sort(var_x, axis=-1) - self.assertEqual((np.sort(self.input_data, - axis=-1) == out.numpy()).all(), True) + self.assertEqual( + (np.sort(self.input_data, axis=-1) == out.numpy()).all(), True + ) paddle.enable_static() def test_api_1(self): diff --git a/python/paddle/fluid/tests/unittests/test_space_to_depth_op.py b/python/paddle/fluid/tests/unittests/test_space_to_depth_op.py index b80f511716c..9c43836e294 100644 --- a/python/paddle/fluid/tests/unittests/test_space_to_depth_op.py +++ b/python/paddle/fluid/tests/unittests/test_space_to_depth_op.py @@ -19,7 +19,6 @@ from op_test import OpTest class TestSpaceToDepthOp(OpTest): - @staticmethod def helper(in_, width, height, channel, batch, blocksize, forward, out_): channel_out = channel // (blocksize * blocksize) @@ -33,8 +32,9 @@ class TestSpaceToDepthOp(OpTest): width2 = i * blocksize + offset % blocksize height2 = j * blocksize + offset // blocksize out_index = width2 + width * blocksize * ( - height2 + height * blocksize * - (channel2 + channel_out * b)) + height2 + + height * blocksize * (channel2 + channel_out * b) + ) if forward: out_[out_index] = in_[in_index] else: @@ -45,9 +45,16 @@ class TestSpaceToDepthOp(OpTest): self.op_type = "space_to_depth" self.inputs = {"X": self.x} - self.helper(self.x_1d, self.x.shape[3], self.x.shape[2], - self.x.shape[1], self.x.shape[0], self.blocksize, - self.forward, self.out_1d) + self.helper( + self.x_1d, + self.x.shape[3], + self.x.shape[2], + self.x.shape[1], + self.x.shape[0], + self.blocksize, + self.forward, + self.out_1d, + ) self.out = np.reshape(self.out_1d, self.infered_shape) self.attrs = {"blocksize": self.blocksize} self.outputs = {"Out": self.out} @@ -65,18 +72,23 @@ class TestSpaceToDepthOp(OpTest): self.forward = 1 def test_check_output(self): - place = fluid.core.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.core.CPUPlace() + place = ( + fluid.core.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.core.CPUPlace() + ) self.check_output_with_place(place, 1e-5, None, False) def test_check_grad(self): - place = fluid.core.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.core.CPUPlace() + place = ( + fluid.core.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.core.CPUPlace() + ) self.check_grad_with_place(place, ['X'], 'Out') class TestSpaceToDepthOpBasic(TestSpaceToDepthOp): - def init_data(self): self.ori_shape = (32, 8, 6, 6) self.infered_shape = (32, 32, 3, 3) @@ -91,7 +103,6 @@ class TestSpaceToDepthOpBasic(TestSpaceToDepthOp): class TestSpaceToDepthOpDoubleBasic(TestSpaceToDepthOp): - def init_data(self): self.ori_shape = (32, 8, 6, 6) self.infered_shape = (32, 32, 3, 3) @@ -106,7 +117,6 @@ class TestSpaceToDepthOpDoubleBasic(TestSpaceToDepthOp): class TestSpaceToDepthOpWithStride3(TestSpaceToDepthOp): - def init_data(self): self.ori_shape = (32, 9, 6, 6) self.infered_shape = (32, 81, 2, 2) @@ -121,7 +131,6 @@ class TestSpaceToDepthOpWithStride3(TestSpaceToDepthOp): class TestSpaceToDepthOpWithNotSquare(TestSpaceToDepthOp): - def init_data(self): self.ori_shape = (32, 9, 9, 6) self.infered_shape = (32, 81, 3, 2) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_addmm_op.py b/python/paddle/fluid/tests/unittests/test_sparse_addmm_op.py index d93dba5c770..ca3dbe4a19a 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_addmm_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_addmm_op.py @@ -51,7 +51,7 @@ class TestAddmm(unittest.TestCase): dense_x.stop_gradient = False dense_y = origin_y.detach() dense_y.stop_gradient = False - dense_out = 2. * paddle.matmul(dense_x, dense_y) + 3. * dense_input + dense_out = 2.0 * paddle.matmul(dense_x, dense_y) + 3.0 * dense_input sp_input = dense_input.detach() sp_input.stop_gradient = False @@ -64,30 +64,36 @@ class TestAddmm(unittest.TestCase): sp_y.stop_gradient = False sp_out = paddle.sparse.addmm(sp_input, sp_x, sp_y, 3.0, 2.0) - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) if get_cuda_version() >= 11030: dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_input.grad.numpy(), - dense_input.grad.numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_y.grad.numpy(), - dense_y.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_input.grad.numpy(), dense_input.grad.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_y.grad.numpy(), dense_y.grad.numpy(), rtol=1e-05 + ) - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11000, "only support cuda>=11.0") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11000, + "only support cuda>=11.0", + ) def test_addmm_2d(self): self.check_result([16, 10], [16, 12], [12, 10], 'coo') self.check_result([16, 10], [16, 12], [12, 10], 'csr') - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11070, "only support cuda>=11.7") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11070, + "only support cuda>=11.7", + ) def test_addmm_3d(self): self.check_result([8, 16, 10], [8, 16, 12], [8, 12, 10], 'coo') self.check_result([8, 16, 10], [8, 16, 12], [8, 12, 10], 'csr') diff --git a/python/paddle/fluid/tests/unittests/test_sparse_attention_op.py b/python/paddle/fluid/tests/unittests/test_sparse_attention_op.py index 6e077502c01..3b29d335da4 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_attention_op.py @@ -91,14 +91,9 @@ def get_csr_value(mat, layout, nnz): return value -def ref_sparse_attention(q, - k, - v, - offset, - columns, - kp_mask=None, - attn_mask=None, - bsz=None): +def ref_sparse_attention( + q, k, v, offset, columns, kp_mask=None, attn_mask=None, bsz=None +): row, col, nnz = q.shape[0], q.shape[1], columns.shape[0] mat = np.zeros((row, row)) for cur_row in range(row): @@ -109,7 +104,7 @@ def ref_sparse_attention(q, mat[cur_row][cur_col] = 1 a = np.dot(q, k.T) * mat a_value = get_csr_value(a, mat, nnz) - scaling = float(col)**-0.5 + scaling = float(col) ** -0.5 a = scaling * a for i in range(row): for j in range(row): @@ -125,13 +120,9 @@ def ref_sparse_attention(q, return result, a_value, b_value -def ref_batch_sparse_attention(q, - k, - v, - offset, - columns, - kp_mask=None, - attn_mask=None): +def ref_batch_sparse_attention( + q, k, v, offset, columns, kp_mask=None, attn_mask=None +): batch_size, num_heads, row, col = q.shape nnz = columns.shape[2] result = np.zeros((batch_size, num_heads, row, col)) @@ -139,11 +130,16 @@ def ref_batch_sparse_attention(q, result_softmax = np.zeros((batch_size, num_heads, nnz)) for i in range(batch_size): for j in range(num_heads): - cur_q, cur_k, cur_v, = q[i][j], k[i][j], v[i][j] + cur_q, cur_k, cur_v, = ( + q[i][j], + k[i][j], + v[i][j], + ) cur_offset, cur_columns = offset[i][j], columns[i][j] if kp_mask is None and attn_mask is None: cur_result, cur_sdd, cur_softmax = ref_sparse_attention( - cur_q, cur_k, cur_v, cur_offset, cur_columns) + cur_q, cur_k, cur_v, cur_offset, cur_columns + ) else: cur_result, cur_sdd, cur_softmax = ref_sparse_attention( cur_q, @@ -153,7 +149,8 @@ def ref_batch_sparse_attention(q, cur_columns, kp_mask=kp_mask, attn_mask=attn_mask, - bsz=i) + bsz=i, + ) result[i][j] = cur_result result_sdd[i][j], result_softmax[i][j] = cur_sdd, cur_softmax return result, result_sdd, result_softmax @@ -191,10 +188,9 @@ def init_csr_format(batch_size, num_heads, rows, blocksize): @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11030, - "core is not compiled with CUDA and cuda version need larger than or equal to 11.3" + "core is not compiled with CUDA and cuda version need larger than or equal to 11.3", ) class TestSparseAttentionOp(OpTest): - def config(self): self.shape = (1, 1, 16, 16) self.blocksize = 4 @@ -210,8 +206,9 @@ class TestSparseAttentionOp(OpTest): self.k = np.random.random(self.shape).astype(self.dtype) self.v = np.random.random(self.shape).astype(self.dtype) # init CSR tensor - offset, columns = init_csr_format(self.shape[0], self.shape[1], - self.shape[2], self.blocksize) + offset, columns = init_csr_format( + self.shape[0], self.shape[1], self.shape[2], self.blocksize + ) self.offset = offset.astype('int32') self.columns = columns.astype('int32') # init mask tensor @@ -232,10 +229,12 @@ class TestSparseAttentionOp(OpTest): self.offset, self.columns, kp_mask=self.key_padding_mask, - attn_mask=self.attn_mask) + attn_mask=self.attn_mask, + ) else: result, result_sdd, result_softmax = ref_batch_sparse_attention( - self.q, self.k, self.v, self.offset, self.columns) + self.q, self.k, self.v, self.offset, self.columns + ) if self.use_mask == True: self.inputs = { @@ -258,7 +257,7 @@ class TestSparseAttentionOp(OpTest): self.outputs = { 'Out': result.astype(self.dtype), 'SparseDotSdd': result_sdd.astype(self.dtype), - 'Softmax': result_softmax.astype(self.dtype) + 'Softmax': result_softmax.astype(self.dtype), } def test_check_output(self): @@ -271,7 +270,6 @@ class TestSparseAttentionOp(OpTest): class TestSparseAttentionOpFp32Test(TestSparseAttentionOp): - def config(self): self.shape = (1, 1, 8, 16) self.blocksize = 2 @@ -280,7 +278,6 @@ class TestSparseAttentionOpFp32Test(TestSparseAttentionOp): class TestSparseAttentionOpShapeTest(TestSparseAttentionOp): - def config(self): self.shape = (2, 2, 32, 8) self.blocksize = 8 @@ -290,10 +287,9 @@ class TestSparseAttentionOpShapeTest(TestSparseAttentionOp): @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11030, - "core is not compiled with CUDA and cuda version need larger than or equal to 11.3" + "core is not compiled with CUDA and cuda version need larger than or equal to 11.3", ) class TestSparseAttentionAPI(unittest.TestCase): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (1, 1, 8, 4) @@ -308,54 +304,62 @@ class TestSparseAttentionAPI(unittest.TestCase): K = paddle.static.data(name="K", shape=self.shape, dtype=self.dtype) V = paddle.static.data(name="V", shape=self.shape, dtype=self.dtype) - batch_size, num_heads, rows = self.shape[0], self.shape[ - 1], self.shape[2] + batch_size, num_heads, rows = ( + self.shape[0], + self.shape[1], + self.shape[2], + ) block_num = rows / self.blocksize block_last = rows % self.blocksize - sparse_nnz_num = block_num * self.blocksize * self.blocksize + block_last * block_last + sparse_nnz_num = ( + block_num * self.blocksize * self.blocksize + + block_last * block_last + ) offset_shape = (batch_size, num_heads, rows + 1) columns_shape = (batch_size, num_heads, int(sparse_nnz_num)) - offset = paddle.static.data(name="Offset", - shape=offset_shape, - dtype="int32") - columns = paddle.static.data(name="Columns", - shape=columns_shape, - dtype="int32") + offset = paddle.static.data( + name="Offset", shape=offset_shape, dtype="int32" + ) + columns = paddle.static.data( + name="Columns", shape=columns_shape, dtype="int32" + ) key_padding_mask_shape = (self.shape[0], self.shape[2]) attn_mask_shape = (self.shape[2], self.shape[2]) if self.use_mask == True: key_padding_mask = paddle.static.data( name="KeyPaddingMask", shape=key_padding_mask_shape, - dtype=self.dtype) - attn_mask = paddle.static.data(name="AttnMask", - shape=attn_mask_shape, - dtype=self.dtype) - Out = F.sparse_attention(Q, - K, - V, - offset, - columns, - key_padding_mask=key_padding_mask, - attn_mask=attn_mask) + dtype=self.dtype, + ) + attn_mask = paddle.static.data( + name="AttnMask", shape=attn_mask_shape, dtype=self.dtype + ) + Out = F.sparse_attention( + Q, + K, + V, + offset, + columns, + key_padding_mask=key_padding_mask, + attn_mask=attn_mask, + ) else: Out = F.sparse_attention(Q, K, V, offset, columns) Q_np = np.random.random(self.shape).astype(self.dtype) K_np = np.random.random(self.shape).astype(self.dtype) V_np = np.random.random(self.shape).astype(self.dtype) - offset_np, columns_np = init_csr_format(self.shape[0], - self.shape[1], - self.shape[2], - self.blocksize) + offset_np, columns_np = init_csr_format( + self.shape[0], self.shape[1], self.shape[2], self.blocksize + ) offset_np = offset_np.astype('int32') columns_np = columns_np.astype('int32') # init mask tensor - key_padding_mask_np = np.random.randint(0, - 2, - size=key_padding_mask_shape) + key_padding_mask_np = np.random.randint( + 0, 2, size=key_padding_mask_shape + ) attn_mask_np = np.random.randint(0, 2, size=attn_mask_shape) key_padding_mask_np = init_mask(key_padding_mask_np) attn_mask_np = init_mask(attn_mask_np) @@ -364,16 +368,18 @@ class TestSparseAttentionAPI(unittest.TestCase): exe = fluid.Executor(self.place) if self.use_mask == True: - fetches_result = exe.run(feed={ - "Q": Q_np, - "K": K_np, - "V": V_np, - "Offset": offset_np, - "Columns": columns_np, - 'KeyPaddingMask': key_padding_mask_np, - 'AttnMask': attn_mask_np - }, - fetch_list=[Out]) + fetches_result = exe.run( + feed={ + "Q": Q_np, + "K": K_np, + "V": V_np, + "Offset": offset_np, + "Columns": columns_np, + 'KeyPaddingMask': key_padding_mask_np, + 'AttnMask': attn_mask_np, + }, + fetch_list=[Out], + ) expected_result, __, __ = ref_batch_sparse_attention( Q_np, K_np, @@ -381,28 +387,32 @@ class TestSparseAttentionAPI(unittest.TestCase): offset_np, columns_np, kp_mask=key_padding_mask_np, - attn_mask=attn_mask_np) + attn_mask=attn_mask_np, + ) else: - fetches_result = exe.run(feed={ - "Q": Q_np, - "K": K_np, - "V": V_np, - "Offset": offset_np, - "Columns": columns_np - }, - fetch_list=[Out]) + fetches_result = exe.run( + feed={ + "Q": Q_np, + "K": K_np, + "V": V_np, + "Offset": offset_np, + "Columns": columns_np, + }, + fetch_list=[Out], + ) expected_result, __, __ = ref_batch_sparse_attention( - Q_np, K_np, V_np, offset_np, columns_np) + Q_np, K_np, V_np, offset_np, columns_np + ) - np.testing.assert_allclose(fetches_result, - expected_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + fetches_result, expected_result, rtol=1e-05, atol=1e-05 + ) def test_dygraph(self): paddle.disable_static() - offset, columns = init_csr_format(self.shape[0], self.shape[1], - self.shape[2], self.blocksize) + offset, columns = init_csr_format( + self.shape[0], self.shape[1], self.shape[2], self.blocksize + ) offset = offset.astype('int32') columns = columns.astype('int32') query = np.random.random(self.shape).astype(self.dtype) @@ -427,13 +437,15 @@ class TestSparseAttentionAPI(unittest.TestCase): paddle_attn_mask = paddle.to_tensor(attn_mask, place=self.place) if self.use_mask == True: - paddle_result = F.sparse_attention(paddle_query, - paddle_key, - paddle_value, - paddle_offset, - paddle_colunmns, - key_padding_mask=paddle_kp_mask, - attn_mask=paddle_attn_mask) + paddle_result = F.sparse_attention( + paddle_query, + paddle_key, + paddle_value, + paddle_offset, + paddle_colunmns, + key_padding_mask=paddle_kp_mask, + attn_mask=paddle_attn_mask, + ) numpy_result, __, __ = ref_batch_sparse_attention( query, @@ -442,25 +454,29 @@ class TestSparseAttentionAPI(unittest.TestCase): offset, columns, kp_mask=key_padding_mask, - attn_mask=attn_mask) + attn_mask=attn_mask, + ) numpy_result = numpy_result.astype(self.dtype) else: - paddle_result = F.sparse_attention(paddle_query, paddle_key, - paddle_value, paddle_offset, - paddle_colunmns) + paddle_result = F.sparse_attention( + paddle_query, + paddle_key, + paddle_value, + paddle_offset, + paddle_colunmns, + ) numpy_result, __, __ = ref_batch_sparse_attention( - query, key, value, offset, columns) + query, key, value, offset, columns + ) numpy_result = numpy_result.astype(self.dtype) - np.testing.assert_allclose(paddle_result.numpy(), - numpy_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + paddle_result.numpy(), numpy_result, rtol=1e-05, atol=1e-05 + ) class TestSparseAttentionAPITestFloat(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (2, 2, 8, 4) @@ -470,7 +486,6 @@ class TestSparseAttentionAPITestFloat(TestSparseAttentionAPI): class TestSparseAttentionAPITestShape1(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (2, 2, 64, 32) @@ -480,7 +495,6 @@ class TestSparseAttentionAPITestShape1(TestSparseAttentionAPI): class TestSparseAttentionAPITestShape2(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (2, 1, 64, 32) @@ -490,7 +504,6 @@ class TestSparseAttentionAPITestShape2(TestSparseAttentionAPI): class TestSparseAttentionAPITestShape3(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (4, 4, 128, 32) @@ -500,7 +513,6 @@ class TestSparseAttentionAPITestShape3(TestSparseAttentionAPI): class TestSparseAttentionAPITestShape4(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (3, 3, 35, 15) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py b/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py index dfd7d23f8a4..0b9c88aad9d 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py @@ -21,13 +21,12 @@ import paddle.sparse as sparse class TestSparseConv(unittest.TestCase): - def test_conv3d(self): with _test_eager_guard(): kernel = [[[[[1], [1], [1]], [[1], [1], [1]], [[1], [1], [1]]]]] - dense_kernel = paddle.to_tensor(kernel, - dtype='float32', - stop_gradient=False) + dense_kernel = paddle.to_tensor( + kernel, dtype='float32', stop_gradient=False + ) dense_kernel = paddle.reshape(dense_kernel, [1, 3, 3, 1, 1]) paddings = [0, 0, 0] strides = [1, 1, 1] @@ -40,17 +39,19 @@ class TestSparseConv(unittest.TestCase): values = paddle.to_tensor(values, dtype='float32') dense_shape = [1, 1, 3, 4, 1] correct_out_values = [[5], [11]] - sparse_input = core.eager.sparse_coo_tensor(indices, values, - dense_shape, False) - out = paddle.sparse.nn.functional.conv3d(sparse_input, - dense_kernel, - bias=paddle.to_tensor( - bias, dtype='float32'), - stride=strides, - padding=paddings, - dilation=dilations, - groups=1, - data_format="NDHWC") + sparse_input = core.eager.sparse_coo_tensor( + indices, values, dense_shape, False + ) + out = paddle.sparse.nn.functional.conv3d( + sparse_input, + dense_kernel, + bias=paddle.to_tensor(bias, dtype='float32'), + stride=strides, + padding=paddings, + dilation=dilations, + groups=1, + data_format="NDHWC", + ) out.backward(out) out = paddle.sparse.coalesce(out) assert np.array_equal(correct_out_values, out.values().numpy()) @@ -62,41 +63,41 @@ class TestSparseConv(unittest.TestCase): indices = paddle.to_tensor(indices, dtype='int32') values = paddle.to_tensor(values, dtype='float32') dense_shape = [1, 1, 3, 4, 1] - sparse_x = paddle.sparse.sparse_coo_tensor(indices, - values, - dense_shape, - stop_gradient=True) + sparse_x = paddle.sparse.sparse_coo_tensor( + indices, values, dense_shape, stop_gradient=True + ) weight = paddle.randn((1, 3, 3, 1, 1), dtype='float32') - y = paddle.sparse.nn.functional.subm_conv3d(sparse_x, - weight, - key='subm_conv') - assert np.array_equal(sparse_x.indices().numpy(), - y.indices().numpy()) + y = paddle.sparse.nn.functional.subm_conv3d( + sparse_x, weight, key='subm_conv' + ) + assert np.array_equal( + sparse_x.indices().numpy(), y.indices().numpy() + ) def test_Conv3D(self): with _test_eager_guard(): - #(4, non_zero_num), 4-D:(N, D, H, W) + # (4, non_zero_num), 4-D:(N, D, H, W) indices = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 1, 2], [1, 3, 2, 3]] - #(non_zero_num, C) + # (non_zero_num, C) values = [[1], [2], [3], [4]] indices = paddle.to_tensor(indices, dtype='int32') values = paddle.to_tensor(values, dtype='float32') dense_shape = [1, 1, 3, 4, 1] correct_out_values = [[4], [10]] sparse_input = paddle.sparse.sparse_coo_tensor( - indices, values, dense_shape, False) + indices, values, dense_shape, False + ) - sparse_conv3d = paddle.sparse.nn.Conv3D(1, - 1, (1, 3, 3), - data_format='NDHWC') + sparse_conv3d = paddle.sparse.nn.Conv3D( + 1, 1, (1, 3, 3), data_format='NDHWC' + ) sparse_out = sparse_conv3d(sparse_input) - #test errors + # test errors with self.assertRaises(ValueError): - #Currently, only support data_format='NDHWC' - conv3d = paddle.sparse.nn.SubmConv3D(1, - 1, (1, 3, 3), - data_format='NCDHW', - key='subm_conv') + # Currently, only support data_format='NDHWC' + conv3d = paddle.sparse.nn.SubmConv3D( + 1, 1, (1, 3, 3), data_format='NCDHW', key='subm_conv' + ) def test_SubmConv3D(self): with _test_eager_guard(): @@ -107,12 +108,12 @@ class TestSparseConv(unittest.TestCase): dense_shape = [1, 1, 3, 4, 1] correct_out_values = [[4], [10]] sparse_input = paddle.sparse.sparse_coo_tensor( - indices, values, dense_shape, False) + indices, values, dense_shape, False + ) - subm_conv3d = paddle.sparse.nn.SubmConv3D(1, - 1, (1, 3, 3), - data_format='NDHWC', - key='subm_conv') + subm_conv3d = paddle.sparse.nn.SubmConv3D( + 1, 1, (1, 3, 3), data_format='NDHWC', key='subm_conv' + ) # test extra_repr print(subm_conv3d.extra_repr()) @@ -120,13 +121,12 @@ class TestSparseConv(unittest.TestCase): # the output shape of subm_conv is same as input shape assert np.array_equal(indices, sparse_out.indices().numpy()) - #test errors + # test errors with self.assertRaises(ValueError): - #Currently, only support data_format='NDHWC' - conv3d = paddle.sparse.nn.SubmConv3D(1, - 1, (1, 3, 3), - data_format='NCDHW', - key='subm_conv') + # Currently, only support data_format='NDHWC' + conv3d = paddle.sparse.nn.SubmConv3D( + 1, 1, (1, 3, 3), data_format='NCDHW', key='subm_conv' + ) def test_Conv3D_bias(self): with _test_eager_guard(): @@ -138,8 +138,8 @@ class TestSparseConv(unittest.TestCase): sp_conv3d = paddle.sparse.nn.Conv3D(3, 2, 3, data_format='NDHWC') sp_conv3d.weight.set_value( - paddle.to_tensor(conv3d.weight.numpy().transpose(2, 3, 4, 1, - 0))) + paddle.to_tensor(conv3d.weight.numpy().transpose(2, 3, 4, 1, 0)) + ) sp_conv3d.bias.set_value(paddle.to_tensor(conv3d.bias.numpy())) x.stop_gradient = False @@ -152,50 +152,53 @@ class TestSparseConv(unittest.TestCase): dense_out = sp_out.to_dense() sp_loss = dense_out.mean() sp_loss.backward() - assert np.allclose(out.numpy(), - dense_out.numpy(), - atol=1e-3, - rtol=1e-3) - assert np.allclose(conv3d.weight.grad.numpy().transpose( - 2, 3, 4, 1, 0), - sp_conv3d.weight.grad.numpy(), - atol=1e-3, - rtol=1e-3) - assert np.allclose(conv3d.bias.grad.numpy(), - sp_conv3d.bias.grad.numpy(), - atol=1e-5, - rtol=1e-5) + assert np.allclose( + out.numpy(), dense_out.numpy(), atol=1e-3, rtol=1e-3 + ) + assert np.allclose( + conv3d.weight.grad.numpy().transpose(2, 3, 4, 1, 0), + sp_conv3d.weight.grad.numpy(), + atol=1e-3, + rtol=1e-3, + ) + assert np.allclose( + conv3d.bias.grad.numpy(), + sp_conv3d.bias.grad.numpy(), + atol=1e-5, + rtol=1e-5, + ) class TestStatic(unittest.TestCase): - def test(self): paddle.enable_static() - indices = paddle.static.data(name='indices', - shape=[4, 4], - dtype='int32') - values = paddle.static.data(name='values', - shape=[4, 1], - dtype='float32') + indices = paddle.static.data( + name='indices', shape=[4, 4], dtype='int32' + ) + values = paddle.static.data( + name='values', shape=[4, 1], dtype='float32' + ) dense_shape = [1, 1, 3, 4, 1] sp_x = sparse.sparse_coo_tensor(indices, values, dense_shape) weight_shape = [1, 3, 3, 1, 1] - weight = paddle.static.data(name='weight', - shape=weight_shape, - dtype='float32') + weight = paddle.static.data( + name='weight', shape=weight_shape, dtype='float32' + ) bias_shape = [1] - bias = paddle.static.data(name='bias', - shape=bias_shape, - dtype='float32') - out = sparse.nn.functional.conv3d(sp_x, - weight, - bias, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NDHWC") + bias = paddle.static.data( + name='bias', shape=bias_shape, dtype='float32' + ) + out = sparse.nn.functional.conv3d( + sp_x, + weight, + bias, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NDHWC", + ) sp_out = sparse.nn.functional.relu(out) out_indices = sp_out.indices() out_values = sp_out.values() @@ -205,19 +208,22 @@ class TestStatic(unittest.TestCase): indices_data = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 1, 2], [1, 3, 2, 3]] values_data = [[1.0], [2.0], [3.0], [4.0]] - weight_data = np.array([[[[[1], [1], [1]], [[1], [1], [1]], - [[1], [1], [1]]]]]).astype('float32') + weight_data = np.array( + [[[[[1], [1], [1]], [[1], [1], [1]], [[1], [1], [1]]]]] + ).astype('float32') weight_data = weight_data.reshape(weight_shape) bias_data = np.array([1]).astype('float32') - fetch = exe.run(feed={ - 'indices': indices_data, - 'values': values_data, - 'weight': weight_data, - 'bias': bias_data - }, - fetch_list=[out, out_indices, out_values], - return_numpy=True) + fetch = exe.run( + feed={ + 'indices': indices_data, + 'values': values_data, + 'weight': weight_data, + 'bias': bias_data, + }, + fetch_list=[out, out_indices, out_values], + return_numpy=True, + ) correct_out = np.array([[[[[5.0], [11.0]]]]]).astype('float64') correct_out_values = [[5.0], [11.0]] assert np.array_equal(correct_out, fetch[0]) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_copy_op.py b/python/paddle/fluid/tests/unittests/test_sparse_copy_op.py index 587e4b6d68a..c31dc4e159f 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_copy_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_copy_op.py @@ -19,7 +19,6 @@ from paddle.fluid.framework import _test_eager_guard class TestSparseCopy(unittest.TestCase): - def test_copy_sparse_coo(self): with _test_eager_guard(): np_x = [[0, 1.0, 0], [2.0, 0, 0], [0, 3.0, 0]] diff --git a/python/paddle/fluid/tests/unittests/test_sparse_elementwise_op.py b/python/paddle/fluid/tests/unittests/test_sparse_elementwise_op.py index 172bdaee2ab..f762584801e 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_elementwise_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_elementwise_op.py @@ -68,37 +68,45 @@ class TestSparseElementWiseAPI(unittest.TestCase): expect_res = op(dense_x, dense_y) expect_res.backward(expect_res) - np.testing.assert_allclose(expect_res.numpy(), - actual_res.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + expect_res.numpy(), + actual_res.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) if not (op == __truediv__ and dtype in ['int32', 'int64']): - np.testing.assert_allclose(dense_x.grad.numpy(), - csr_x.grad.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) - np.testing.assert_allclose(dense_y.grad.numpy(), - csr_y.grad.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + dense_x.grad.numpy(), + csr_x.grad.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) + np.testing.assert_allclose( + dense_y.grad.numpy(), + csr_y.grad.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) def func_test_coo(self, op): for sparse_dim in range(len(self.coo_shape) - 1, len(self.coo_shape)): for dtype in self.support_dtypes: - x = np.random.randint(-255, 255, - size=self.coo_shape).astype(dtype) - y = np.random.randint(-255, 255, - size=self.coo_shape).astype(dtype) + x = np.random.randint(-255, 255, size=self.coo_shape).astype( + dtype + ) + y = np.random.randint(-255, 255, size=self.coo_shape).astype( + dtype + ) dense_x = paddle.to_tensor(x, dtype=dtype, stop_gradient=False) dense_y = paddle.to_tensor(y, dtype=dtype, stop_gradient=False) - s_dense_x = paddle.to_tensor(x, - dtype=dtype, - stop_gradient=False) - s_dense_y = paddle.to_tensor(y, - dtype=dtype, - stop_gradient=False) + s_dense_x = paddle.to_tensor( + x, dtype=dtype, stop_gradient=False + ) + s_dense_y = paddle.to_tensor( + y, dtype=dtype, stop_gradient=False + ) coo_x = s_dense_x.to_sparse_coo(sparse_dim) coo_y = s_dense_y.to_sparse_coo(sparse_dim) @@ -108,18 +116,24 @@ class TestSparseElementWiseAPI(unittest.TestCase): expect_res = op(dense_x, dense_y) expect_res.backward(expect_res) - np.testing.assert_allclose(expect_res.numpy(), - actual_res.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) - np.testing.assert_allclose(dense_x.grad.numpy(), - coo_x.grad.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) - np.testing.assert_allclose(dense_y.grad.numpy(), - coo_y.grad.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + expect_res.numpy(), + actual_res.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) + np.testing.assert_allclose( + dense_x.grad.numpy(), + coo_x.grad.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) + np.testing.assert_allclose( + dense_y.grad.numpy(), + coo_y.grad.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) def test_support_dtypes_csr(self): paddle.device.set_device('cpu') @@ -139,38 +153,37 @@ class TestSparseElementWiseAPI(unittest.TestCase): values2_data = [[1.0], [2.0]] shape = [2, 4, 2] - sp_a = sparse.sparse_coo_tensor(indices_data, - values1_data, - shape, - stop_gradient=False) - sp_b = sparse.sparse_coo_tensor(indices_data, - values2_data, - shape, - stop_gradient=False) + sp_a = sparse.sparse_coo_tensor( + indices_data, values1_data, shape, stop_gradient=False + ) + sp_b = sparse.sparse_coo_tensor( + indices_data, values2_data, shape, stop_gradient=False + ) values1 = paddle.to_tensor(values1_data, stop_gradient=False) values2 = paddle.to_tensor(values2_data, stop_gradient=False) - #c.values() = a.values() + b.values() + # c.values() = a.values() + b.values() sp_c = sparse.add(sp_a, sp_b) sp_c.backward() ref_c = values1 + values2 ref_c.backward() np.testing.assert_allclose(sp_c.values().numpy(), ref_c.numpy()) - np.testing.assert_allclose(sp_a.grad.values().numpy(), - values1.grad.numpy()) - np.testing.assert_allclose(sp_b.grad.values().numpy(), - values2.grad.numpy()) + np.testing.assert_allclose( + sp_a.grad.values().numpy(), values1.grad.numpy() + ) + np.testing.assert_allclose( + sp_b.grad.values().numpy(), values2.grad.numpy() + ) def test_add_bias(self): indices_data = [[0, 1], [0, 3]] values_data = [[1.0, 1.0], [2.0, 2.0]] shape = [2, 4, 2] - sp_a = sparse.sparse_coo_tensor(indices_data, - values_data, - shape, - stop_gradient=False) + sp_a = sparse.sparse_coo_tensor( + indices_data, values_data, shape, stop_gradient=False + ) bias_values = [1.0, 2.0] @@ -178,14 +191,15 @@ class TestSparseElementWiseAPI(unittest.TestCase): values2 = paddle.to_tensor(bias_values, stop_gradient=False) values3 = paddle.to_tensor(bias_values, stop_gradient=False) - #c.values() = a.values() + b + # c.values() = a.values() + b sp_c = sparse.add(sp_a, values2) sp_c.backward() ref_c = values1 + values3 ref_c.backward() np.testing.assert_allclose(sp_c.values().numpy(), ref_c.numpy()) - np.testing.assert_allclose(sp_a.grad.values().numpy(), - values1.grad.numpy()) + np.testing.assert_allclose( + sp_a.grad.values().numpy(), values1.grad.numpy() + ) np.testing.assert_allclose(values2.grad.numpy(), values3.grad.numpy()) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_fused_attention_op.py b/python/paddle/fluid/tests/unittests/test_sparse_fused_attention_op.py index 50e9218a27d..dfc8806fd02 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_fused_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_fused_attention_op.py @@ -37,10 +37,9 @@ def get_cuda_version(): @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11070, - "core is not compiled with CUDA and cuda version need larger than or equal to 11.7" + "core is not compiled with CUDA and cuda version need larger than or equal to 11.7", ) class TestSparseAttentionAPI1(unittest.TestCase): - def setUp(self): self.batch_size = 16 self.num_heads = 16 @@ -52,7 +51,10 @@ class TestSparseAttentionAPI1(unittest.TestCase): def test_dygraph(self): with _test_eager_guard(): self.shape = [ - self.batch_size, self.num_heads, self.seq_len, self.head_dim + self.batch_size, + self.num_heads, + self.seq_len, + self.head_dim, ] query = paddle.rand(self.shape, self.dtype) key = paddle.rand(self.shape, self.dtype) @@ -62,13 +64,16 @@ class TestSparseAttentionAPI1(unittest.TestCase): key.stop_gradient = False value.stop_gradient = False - mask = paddle.nn.functional.dropout(paddle.ones( - [self.seq_len, self.seq_len]), - mode='downscale_in_infer') + mask = paddle.nn.functional.dropout( + paddle.ones([self.seq_len, self.seq_len]), + mode='downscale_in_infer', + ) mask = mask.expand( - [self.batch_size, self.num_heads, self.seq_len, self.seq_len]) - sp_mask = mask.reshape([-1, self.seq_len, - self.seq_len]).to_sparse_csr() + [self.batch_size, self.num_heads, self.seq_len, self.seq_len] + ) + sp_mask = mask.reshape( + [-1, self.seq_len, self.seq_len] + ).to_sparse_csr() query_sp = copy.deepcopy(query) key_sp = copy.deepcopy(key) @@ -80,49 +85,57 @@ class TestSparseAttentionAPI1(unittest.TestCase): if self.use_mask: kp_mask = paddle.randint( - 0, 2, [self.batch_size, self.seq_len]).astype(self.dtype) + 0, 2, [self.batch_size, self.seq_len] + ).astype(self.dtype) attn_mask = paddle.randint( - 0, 2, [self.seq_len, self.seq_len]).astype(self.dtype) + 0, 2, [self.seq_len, self.seq_len] + ).astype(self.dtype) sdd = paddle.matmul(query, key, False, True) / math.sqrt( - float(self.head_dim)) - sdd = sdd + ( - (mask * kp_mask.unsqueeze([1, 2]) * attn_mask) - 1.0) * 1e9 + float(self.head_dim) + ) + sdd = ( + sdd + + ((mask * kp_mask.unsqueeze([1, 2]) * attn_mask) - 1.0) + * 1e9 + ) softmax = paddle.nn.functional.softmax(sdd) output = paddle.matmul(softmax, value) output.backward() output_sp = paddle.sparse.nn.functional.attention( - query_sp, key_sp, value_sp, sp_mask, kp_mask, attn_mask) + query_sp, key_sp, value_sp, sp_mask, kp_mask, attn_mask + ) output_sp.backward() else: sdd = paddle.matmul(query, key, False, True) / math.sqrt( - float(self.head_dim)) + float(self.head_dim) + ) sdd = sdd + (mask - 1.0) * 1e9 softmax = paddle.nn.functional.softmax(sdd) output = paddle.matmul(softmax, value) output.backward() output_sp = paddle.sparse.nn.functional.attention( - query_sp, key_sp, value_sp, sp_mask) + query_sp, key_sp, value_sp, sp_mask + ) output_sp.backward() - np.testing.assert_allclose(output_sp.numpy(), - output.numpy(), - rtol=1e-05) - np.testing.assert_allclose(query_sp.grad.numpy(), - query.grad.numpy(), - rtol=1e-05) - np.testing.assert_allclose(key_sp.grad.numpy(), - key.grad.numpy(), - rtol=1e-05) - np.testing.assert_allclose(value_sp.grad.numpy(), - value.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + output_sp.numpy(), output.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + query_sp.grad.numpy(), query.grad.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + key_sp.grad.numpy(), key.grad.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + value_sp.grad.numpy(), value.grad.numpy(), rtol=1e-05 + ) class TestSparseAttentionAPI2(TestSparseAttentionAPI1): - def setUp(self): self.batch_size = 16 self.num_heads = 16 @@ -133,7 +146,6 @@ class TestSparseAttentionAPI2(TestSparseAttentionAPI1): class TestSparseAttentionAPI3(TestSparseAttentionAPI1): - def setUp(self): self.batch_size = 16 self.num_heads = 16 @@ -144,7 +156,6 @@ class TestSparseAttentionAPI3(TestSparseAttentionAPI1): class TestSparseAttentionAPI4(TestSparseAttentionAPI1): - def setUp(self): self.batch_size = 16 self.num_heads = 16 @@ -155,7 +166,6 @@ class TestSparseAttentionAPI4(TestSparseAttentionAPI1): class TestSparseAttentionAPI5(TestSparseAttentionAPI1): - def setUp(self): self.batch_size = 16 self.num_heads = 16 diff --git a/python/paddle/fluid/tests/unittests/test_sparse_is_same_shape.py b/python/paddle/fluid/tests/unittests/test_sparse_is_same_shape.py index 8715b55d522..f1c2ded049d 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_is_same_shape.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_is_same_shape.py @@ -28,7 +28,7 @@ class TestSparseIsSameShapeAPI(unittest.TestCase): self.tensors = [ paddle.rand(self.shapes[0]), paddle.rand(self.shapes[0]), - paddle.rand(self.shapes[1]) + paddle.rand(self.shapes[1]), ] self.sparse_dim = 2 @@ -39,85 +39,136 @@ class TestSparseIsSameShapeAPI(unittest.TestCase): def test_dense_csr(self): self.assertTrue( - is_same_shape(self.tensors[0], self.tensors[1].to_sparse_csr())) + is_same_shape(self.tensors[0], self.tensors[1].to_sparse_csr()) + ) self.assertFalse( - is_same_shape(self.tensors[0], self.tensors[2].to_sparse_csr())) + is_same_shape(self.tensors[0], self.tensors[2].to_sparse_csr()) + ) self.assertFalse( - is_same_shape(self.tensors[1], self.tensors[2].to_sparse_csr())) + is_same_shape(self.tensors[1], self.tensors[2].to_sparse_csr()) + ) def test_dense_coo(self): self.assertTrue( - is_same_shape(self.tensors[0], - self.tensors[1].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[0], self.tensors[1].to_sparse_coo(self.sparse_dim) + ) + ) self.assertFalse( - is_same_shape(self.tensors[0], - self.tensors[2].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[0], self.tensors[2].to_sparse_coo(self.sparse_dim) + ) + ) self.assertFalse( - is_same_shape(self.tensors[1], - self.tensors[2].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[1], self.tensors[2].to_sparse_coo(self.sparse_dim) + ) + ) def test_csr_dense(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_csr(), self.tensors[1])) + is_same_shape(self.tensors[0].to_sparse_csr(), self.tensors[1]) + ) self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_csr(), self.tensors[2])) + is_same_shape(self.tensors[0].to_sparse_csr(), self.tensors[2]) + ) self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_csr(), self.tensors[2])) + is_same_shape(self.tensors[1].to_sparse_csr(), self.tensors[2]) + ) def test_csr_csr(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_csr(), - self.tensors[1].to_sparse_csr())) + is_same_shape( + self.tensors[0].to_sparse_csr(), self.tensors[1].to_sparse_csr() + ) + ) self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_csr(), - self.tensors[2].to_sparse_csr())) + is_same_shape( + self.tensors[0].to_sparse_csr(), self.tensors[2].to_sparse_csr() + ) + ) self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_csr(), - self.tensors[2].to_sparse_csr())) + is_same_shape( + self.tensors[1].to_sparse_csr(), self.tensors[2].to_sparse_csr() + ) + ) def test_csr_coo(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_csr(), - self.tensors[1].to_sparse_coo(self.sparse_dim))) - self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_csr(), - self.tensors[2].to_sparse_coo(self.sparse_dim))) - self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_csr(), - self.tensors[2].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[0].to_sparse_csr(), + self.tensors[1].to_sparse_coo(self.sparse_dim), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[0].to_sparse_csr(), + self.tensors[2].to_sparse_coo(self.sparse_dim), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[1].to_sparse_csr(), + self.tensors[2].to_sparse_coo(self.sparse_dim), + ) + ) def test_coo_dense(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[1])) + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), self.tensors[1] + ) + ) self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[2])) + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), self.tensors[2] + ) + ) self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_coo(self.sparse_dim), - self.tensors[2])) + is_same_shape( + self.tensors[1].to_sparse_coo(self.sparse_dim), self.tensors[2] + ) + ) def test_coo_csr(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[1].to_sparse_csr())) - self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[2].to_sparse_csr())) - self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_coo(self.sparse_dim), - self.tensors[2].to_sparse_csr())) + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), + self.tensors[1].to_sparse_csr(), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), + self.tensors[2].to_sparse_csr(), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[1].to_sparse_coo(self.sparse_dim), + self.tensors[2].to_sparse_csr(), + ) + ) def test_coo_coo(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[1].to_sparse_coo(self.sparse_dim))) - self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[2].to_sparse_coo(self.sparse_dim))) - self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_coo(self.sparse_dim), - self.tensors[2].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), + self.tensors[1].to_sparse_coo(self.sparse_dim), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), + self.tensors[2].to_sparse_coo(self.sparse_dim), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[1].to_sparse_coo(self.sparse_dim), + self.tensors[2].to_sparse_coo(self.sparse_dim), + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_matmul_op.py b/python/paddle/fluid/tests/unittests/test_sparse_matmul_op.py index 34a270b0fe5..368e9cbbd2f 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_matmul_op.py @@ -59,27 +59,33 @@ class TestMatmul(unittest.TestCase): sp_y.stop_gradient = False sp_out = paddle.sparse.matmul(sp_x, sp_y) - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) if get_cuda_version() >= 11030: dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_y.grad.numpy(), - dense_y.grad.numpy(), - rtol=1e-05) - - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11000, "only support cuda>=11.0") + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_y.grad.numpy(), dense_y.grad.numpy(), rtol=1e-05 + ) + + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11000, + "only support cuda>=11.0", + ) def test_matmul_2d(self): self.check_result([16, 12], [12, 10], 'coo') self.check_result([16, 12], [12, 10], 'csr') - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11070, "only support cuda>=11.7") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11070, + "only support cuda>=11.7", + ) def test_matmul_3d(self): self.check_result([8, 16, 12], [8, 12, 10], 'coo') self.check_result([8, 16, 12], [8, 12, 10], 'csr') @@ -87,9 +93,10 @@ class TestMatmul(unittest.TestCase): class TestMaskedMatmul(unittest.TestCase): # x: dense, y: dense, out: sparse_`csr - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11030, - "only support on cuda>=11.3") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11030, + "only support on cuda>=11.3", + ) def test_masked_matmul_2d(self): np_mask = np.random.rand(10, 6) < 0.2 @@ -108,24 +115,25 @@ class TestMaskedMatmul(unittest.TestCase): mask = paddle.to_tensor(np.ones([10, 6]) * np_mask).to_sparse_csr() out = paddle.sparse.masked_matmul(x, y, mask) - np.testing.assert_allclose(np_out.indptr, - out.crows().numpy(), - rtol=1e-05) - np.testing.assert_allclose(np_out.indices, - out.cols().numpy(), - rtol=1e-05) - np.testing.assert_allclose(np_out.data, - out.values().numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np_out.indptr, out.crows().numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + np_out.indices, out.cols().numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + np_out.data, out.values().numpy(), rtol=1e-05 + ) out.backward() np.testing.assert_allclose(out.is_sparse_csr(), True, rtol=1e-05) np.testing.assert_allclose(np_x_grad, x.grad.numpy(), rtol=1e-05) np.testing.assert_allclose(np_y_grad, y.grad.numpy(), rtol=1e-05) - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11070, - "only support on cuda>=11.7") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11070, + "only support on cuda>=11.7", + ) def test_masked_matmul_3d(self): paddle.set_default_dtype('float32') origin_x = paddle.rand([16, 16, 12]) @@ -147,15 +155,17 @@ class TestMaskedMatmul(unittest.TestCase): sp_out = paddle.sparse.matmul(sp_x, sp_y) sp_out.backward() - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_y.grad.numpy(), - dense_y.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_y.grad.numpy(), dense_y.grad.numpy(), rtol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_model.py b/python/paddle/fluid/tests/unittests/test_sparse_model.py index aa3d97919c4..52f37e60dfb 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_model.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_model.py @@ -20,15 +20,16 @@ from paddle.fluid.framework import _test_eager_guard class TestGradientAdd(unittest.TestCase): - def sparse(self, sp_x): indentity = sp_x out = nn.functional.relu(sp_x) values = out.values() + indentity.values() - out = paddle.sparse.sparse_coo_tensor(out.indices(), - values, - shape=out.shape, - stop_gradient=out.stop_gradient) + out = paddle.sparse.sparse_coo_tensor( + out.indices(), + values, + shape=out.shape, + stop_gradient=out.stop_gradient, + ) return out def dense(self, x): diff --git a/python/paddle/fluid/tests/unittests/test_sparse_momentum_op.py b/python/paddle/fluid/tests/unittests/test_sparse_momentum_op.py index bcc555b9a3e..fefb3125524 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_momentum_op.py @@ -17,16 +17,18 @@ import numpy as np from op_test import OpTest -def calculate_sparse_momentum_by_numpy(param, - grad, - mu, - velocity, - use_nesterov, - learning_rate, - index, - axis, - regularization_method=None, - regularization_coeff=1.0): +def calculate_sparse_momentum_by_numpy( + param, + grad, + mu, + velocity, + use_nesterov, + learning_rate, + index, + axis, + regularization_method=None, + regularization_coeff=1.0, +): sub_grad = grad.copy() grad = np.zeros_like(param) if axis == 0: @@ -48,8 +50,9 @@ def calculate_sparse_momentum_by_numpy(param, else: velocity_out = mu * velocity + grad if use_nesterov: - param_out = param - grad * learning_rate - \ - velocity_out * mu * learning_rate + param_out = ( + param - grad * learning_rate - velocity_out * mu * learning_rate + ) else: param_out = param - learning_rate * velocity_out @@ -57,7 +60,6 @@ def calculate_sparse_momentum_by_numpy(param, class TestSparseMomentumOp(OpTest): - def setUp(self): self.op_type = "sparse_momentum" self.dtype = np.float32 @@ -75,24 +77,31 @@ class TestSparseMomentumOp(OpTest): if self.multi_precision: assert self.dtype == np.float16 - param = np.random.random( - (self.batch_size, self.num_classes)).astype(self.dtype) - grad = np.random.random( - (self.batch_size, self.num_classes)).astype(self.dtype) + param = np.random.random((self.batch_size, self.num_classes)).astype( + self.dtype + ) + grad = np.random.random((self.batch_size, self.num_classes)).astype( + self.dtype + ) if self.axis == 0: - index = np.random.randint(0, - self.batch_size, - size=(self.batch_size // 2, ), - dtype=self.index_dtype) + index = np.random.randint( + 0, + self.batch_size, + size=(self.batch_size // 2,), + dtype=self.index_dtype, + ) grad = grad[index] else: - index = np.random.randint(0, - self.num_classes, - size=(self.num_classes // 2, ), - dtype=self.index_dtype) + index = np.random.randint( + 0, + self.num_classes, + size=(self.num_classes // 2,), + dtype=self.index_dtype, + ) grad = grad[:, index] - velocity = np.random.random( - (self.batch_size, self.num_classes)).astype(self.dtype) + velocity = np.random.random((self.batch_size, self.num_classes)).astype( + self.dtype + ) learning_rate = np.array([0.001]).astype(self.dtype) mu = 0.9 @@ -109,7 +118,8 @@ class TestSparseMomentumOp(OpTest): regularization_method=regularization_method, regularization_coeff=regularization_coeff, index=index, - axis=self.axis) + axis=self.axis, + ) self.attrs = { 'mu': mu, @@ -121,33 +131,35 @@ class TestSparseMomentumOp(OpTest): } self.inputs = { - 'Param': - param.astype("float16") if self.multi_precision else param, - 'Velocity': - velocity.astype("float32") if self.multi_precision else velocity, - 'LearningRate': - learning_rate.astype("float32") - if self.multi_precision else learning_rate, - 'Grad': - grad.astype("float16") if self.multi_precision else grad, - 'Index': - index, - 'Axis': - np.array(self.axis).astype(np.int32), + 'Param': param.astype("float16") if self.multi_precision else param, + 'Velocity': velocity.astype("float32") + if self.multi_precision + else velocity, + 'LearningRate': learning_rate.astype("float32") + if self.multi_precision + else learning_rate, + 'Grad': grad.astype("float16") if self.multi_precision else grad, + 'Index': index, + 'Axis': np.array(self.axis).astype(np.int32), } self.outputs = { - 'ParamOut': - param_out.astype("float16") if self.multi_precision else param_out, - 'VelocityOut': - velocity_out.astype("float32") - if self.multi_precision else velocity_out, + 'ParamOut': param_out.astype("float16") + if self.multi_precision + else param_out, + 'VelocityOut': velocity_out.astype("float32") + if self.multi_precision + else velocity_out, } if self.multi_precision: - self.inputs['MasterParam'] = param.astype( - "float32") if self.multi_precision else param - self.outputs['MasterParamOut'] = param_out.astype( - "float32") if self.multi_precision else param_out + self.inputs['MasterParam'] = ( + param.astype("float32") if self.multi_precision else param + ) + self.outputs['MasterParamOut'] = ( + param_out.astype("float32") + if self.multi_precision + else param_out + ) def init_dtype(self): pass @@ -162,45 +174,40 @@ class TestSparseMomentumOp(OpTest): pass def test_check_output(self): - self.check_output(atol=5e-3 if self.multi_precision else 1e-5, - check_eager=True) + self.check_output( + atol=5e-3 if self.multi_precision else 1e-5, check_eager=True + ) class TestSparseMomentumOpDtype1(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float32 self.index_dtype = np.int64 class TestSparseMomentumOpDtype2(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float64 self.index_dtype = np.int32 class TestSparseMomentumOpDtype3(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float64 self.index_dtype = np.int64 class TestSparseMomentumOpAxis(TestSparseMomentumOp): - def init_axis(self): self.axis = 1 class TestSparseMomentumOpNesterov(TestSparseMomentumOp): - def init_use_nesterov(self): self.use_nesterov = True class TestSparseMomentumOpMultiPrecision(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float16 self.index_dtype = np.int32 @@ -213,7 +220,6 @@ class TestSparseMomentumOpMultiPrecision(TestSparseMomentumOp): class TestSparseMomentumOpMultiPrecision1(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float16 self.index_dtype = np.int64 @@ -226,7 +232,6 @@ class TestSparseMomentumOpMultiPrecision1(TestSparseMomentumOp): class TestSparseMomentumOpMultiPrecision2(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float16 self.index_dtype = np.int32 @@ -239,7 +244,6 @@ class TestSparseMomentumOpMultiPrecision2(TestSparseMomentumOp): class TestSparseMomentumOpMultiPrecision3(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float16 self.index_dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/test_sparse_mv_op.py b/python/paddle/fluid/tests/unittests/test_sparse_mv_op.py index e532063ad8f..ae04ddc7a48 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_mv_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_mv_op.py @@ -37,7 +37,8 @@ def get_cuda_version(): @unittest.skipIf( not paddle.is_compiled_with_cuda() or get_cuda_version() < 11000, - "paddle is not compiled with CUDA and cuda version need to >= 11.0") + "paddle is not compiled with CUDA and cuda version need to >= 11.0", +) class TestCsrMv(unittest.TestCase): # x: csr-matrix, y: dense-vec, out: dense-vec def test_mv(self): @@ -62,20 +63,23 @@ class TestCsrMv(unittest.TestCase): sp_out = paddle.sparse.mv(sp_x, sp_vec) sp_out.backward() - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_vec.grad.numpy(), - dense_vec.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_vec.grad.numpy(), dense_vec.grad.numpy(), rtol=1e-05 + ) @unittest.skipIf( not paddle.is_compiled_with_cuda() or get_cuda_version() < 11000, - "paddle is not compiled with CUDA and cuda version need to >= 11.0") + "paddle is not compiled with CUDA and cuda version need to >= 11.0", +) class TestCooMv(unittest.TestCase): # x: csr-matrix, y: dense-vec, out: dense-vec def test_mv(self): @@ -100,15 +104,17 @@ class TestCooMv(unittest.TestCase): sp_out = paddle.sparse.mv(sp_x, sp_vec) sp_out.backward() - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_vec.grad.numpy(), - dense_vec.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_vec.grad.numpy(), dense_vec.grad.numpy(), rtol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py b/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py index 503f171bdc3..4f48b08e9f8 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py @@ -22,13 +22,12 @@ import copy class TestSparseBatchNorm(unittest.TestCase): - def test(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) paddle.seed(0) channels = 4 shape = [2, 3, 6, 6, channels] - #there is no zero in dense_x + # there is no zero in dense_x dense_x = paddle.randn(shape) dense_x.stop_gradient = False @@ -48,17 +47,21 @@ class TestSparseBatchNorm(unittest.TestCase): sparse_y = sparse_batch_norm(sparse_x) # compare the result with dense batch_norm - assert np.allclose(dense_y.flatten().numpy(), - sparse_y.values().flatten().numpy(), - atol=1e-5, - rtol=1e-5) + assert np.allclose( + dense_y.flatten().numpy(), + sparse_y.values().flatten().numpy(), + atol=1e-5, + rtol=1e-5, + ) # test backward sparse_y.backward(sparse_y) - assert np.allclose(dense_x.grad.flatten().numpy(), - sparse_x.grad.values().flatten().numpy(), - atol=1e-5, - rtol=1e-5) + assert np.allclose( + dense_x.grad.flatten().numpy(), + sparse_x.grad.values().flatten().numpy(), + atol=1e-5, + rtol=1e-5, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_error_layout(self): @@ -66,8 +69,9 @@ class TestSparseBatchNorm(unittest.TestCase): shape = [2, 3, 6, 6, 3] x = paddle.randn(shape) sparse_x = x.to_sparse_coo(4) - sparse_batch_norm = paddle.sparse.nn.BatchNorm(3, - data_format='NCDHW') + sparse_batch_norm = paddle.sparse.nn.BatchNorm( + 3, data_format='NCDHW' + ) sparse_batch_norm(sparse_x) def test2(self): @@ -86,10 +90,10 @@ class TestSparseBatchNorm(unittest.TestCase): class TestSyncBatchNorm(unittest.TestCase): - def test_sync_batch_norm(self): - x = np.array([[[[0.3, 0.4], [0.3, 0.07]], - [[0.83, 0.37], [0.18, 0.93]]]]).astype('float32') + x = np.array( + [[[[0.3, 0.4], [0.3, 0.07]], [[0.83, 0.37], [0.18, 0.93]]]] + ).astype('float32') x = paddle.to_tensor(x) sparse_x = x.to_sparse_coo(len(x.shape) - 1) @@ -100,18 +104,24 @@ class TestSyncBatchNorm(unittest.TestCase): dense_sync_bn = paddle.nn.SyncBatchNorm(2) x = x.reshape((-1, x.shape[-1])) dense_hidden = dense_sync_bn(x) - assert np.allclose(sparse_hidden.values().numpy(), - dense_hidden.numpy()) + assert np.allclose( + sparse_hidden.values().numpy(), dense_hidden.numpy() + ) def test_convert(self): - base_model = paddle.nn.Sequential(nn.Conv3D(3, 5, 3), nn.BatchNorm(5), - nn.BatchNorm(5)) + base_model = paddle.nn.Sequential( + nn.Conv3D(3, 5, 3), nn.BatchNorm(5), nn.BatchNorm(5) + ) model = paddle.nn.Sequential( - nn.Conv3D(3, 5, 3), nn.BatchNorm(5), - nn.BatchNorm(5, - weight_attr=fluid.ParamAttr(name='bn.scale'), - bias_attr=fluid.ParamAttr(name='bn.bias'))) + nn.Conv3D(3, 5, 3), + nn.BatchNorm(5), + nn.BatchNorm( + 5, + weight_attr=fluid.ParamAttr(name='bn.scale'), + bias_attr=fluid.ParamAttr(name='bn.bias'), + ), + ) model = nn.SyncBatchNorm.convert_sync_batchnorm(model) for idx, sublayer in enumerate(base_model.sublayers()): if isinstance(sublayer, nn.BatchNorm): @@ -119,15 +129,14 @@ class TestSyncBatchNorm(unittest.TestCase): class TestStatic(unittest.TestCase): - def test(self): paddle.enable_static() - indices = paddle.static.data(name='indices', - shape=[4, 4], - dtype='int32') - values = paddle.static.data(name='values', - shape=[4, 1], - dtype='float32') + indices = paddle.static.data( + name='indices', shape=[4, 4], dtype='int32' + ) + values = paddle.static.data( + name='values', shape=[4, 1], dtype='float32' + ) channels = 1 dense_shape = [1, 1, 3, 4, channels] sp_x = sparse.sparse_coo_tensor(indices, values, dense_shape) @@ -144,20 +153,29 @@ class TestStatic(unittest.TestCase): mean_data = np.array([1.0]).astype('float32') variance_data = np.array([2.0]).astype('float32') - fetch = exe.run(feed={ - 'indices': indices_data, - 'values': values_data, - 'batch_norm_0.b_0': bias_data, - 'batch_norm_0.w_0': weight_data, - 'batch_norm_0.w_1': mean_data, - 'batch_norm_0.w_2': variance_data - }, - fetch_list=[out], - return_numpy=True) - correct_out = np.array([[[[[0.0], [-1.6832708], [0.0], [0.1055764]], - [[0.0], [0.0], [1.8944236], [0.0]], - [[0.0], [0.0], [0.0], - [3.683271]]]]]).astype('float32') + fetch = exe.run( + feed={ + 'indices': indices_data, + 'values': values_data, + 'batch_norm_0.b_0': bias_data, + 'batch_norm_0.w_0': weight_data, + 'batch_norm_0.w_1': mean_data, + 'batch_norm_0.w_2': variance_data, + }, + fetch_list=[out], + return_numpy=True, + ) + correct_out = np.array( + [ + [ + [ + [[0.0], [-1.6832708], [0.0], [0.1055764]], + [[0.0], [0.0], [1.8944236], [0.0]], + [[0.0], [0.0], [0.0], [3.683271]], + ] + ] + ] + ).astype('float32') np.testing.assert_allclose(correct_out, fetch[0], rtol=1e-5) paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py b/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py index 998b5be9bb1..5425ebb6ea9 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py @@ -20,7 +20,6 @@ import copy class TestMaxPool3DFunc(unittest.TestCase): - def setInput(self): paddle.seed(0) self.dense_x = paddle.randn((1, 4, 4, 4, 4)) @@ -49,31 +48,32 @@ class TestMaxPool3DFunc(unittest.TestCase): sparse_x, self.kernel_sizes, stride=self.strides, - padding=self.paddings) + padding=self.paddings, + ) out = sparse_out.to_dense() out.backward(out) dense_x = copy.deepcopy(self.dense_x) - dense_out = paddle.nn.functional.max_pool3d(dense_x, - self.kernel_sizes, - stride=self.strides, - padding=self.paddings, - data_format='NDHWC') + dense_out = paddle.nn.functional.max_pool3d( + dense_x, + self.kernel_sizes, + stride=self.strides, + padding=self.paddings, + data_format='NDHWC', + ) dense_out.backward(dense_out) - #compare with dense + # compare with dense assert np.allclose(dense_out.numpy(), out.numpy()) assert np.allclose(dense_x.grad.numpy(), self.dense_x.grad.numpy()) class TestStride(TestMaxPool3DFunc): - def setStride(self): self.strides = 1 class TestPadding(TestMaxPool3DFunc): - def setPadding(self): self.paddings = 1 @@ -82,7 +82,6 @@ class TestPadding(TestMaxPool3DFunc): class TestKernelSize(TestMaxPool3DFunc): - def setKernelSize(self): self.kernel_sizes = [5, 5, 5] @@ -92,7 +91,6 @@ class TestKernelSize(TestMaxPool3DFunc): class TestInput(TestMaxPool3DFunc): - def setInput(self): paddle.seed(0) self.dense_x = paddle.randn((2, 6, 7, 9, 3)) @@ -101,19 +99,19 @@ class TestInput(TestMaxPool3DFunc): class TestMaxPool3DAPI(unittest.TestCase): - def test(self): with _test_eager_guard(): dense_x = paddle.randn((2, 3, 6, 6, 3)) sparse_x = dense_x.to_sparse_coo(4) - max_pool3d = paddle.sparse.nn.MaxPool3D(kernel_size=3, - data_format='NDHWC') + max_pool3d = paddle.sparse.nn.MaxPool3D( + kernel_size=3, data_format='NDHWC' + ) out = max_pool3d(sparse_x) out = out.to_dense() - dense_out = paddle.nn.functional.max_pool3d(dense_x, - 3, - data_format='NDHWC') + dense_out = paddle.nn.functional.max_pool3d( + dense_x, 3, data_format='NDHWC' + ) assert np.allclose(dense_out.numpy(), out.numpy()) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py b/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py index 01619f76dc9..fd748c4dcb2 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py @@ -42,25 +42,27 @@ class TestReshape(unittest.TestCase): dense_out = paddle.reshape(dense_x, new_shape) if format == "coo": - sp_x = paddle.to_tensor(np_x, - place=paddle.CPUPlace()).to_sparse_coo( - len(x_shape)) + sp_x = paddle.to_tensor( + np_x, place=paddle.CPUPlace() + ).to_sparse_coo(len(x_shape)) else: - sp_x = paddle.to_tensor(np_x, - place=paddle.CPUPlace()).to_sparse_csr() + sp_x = paddle.to_tensor( + np_x, place=paddle.CPUPlace() + ).to_sparse_csr() sp_x.stop_gradient = False sp_out = paddle.sparse.reshape(sp_x, new_shape) - np.testing.assert_allclose(sp_out.to_dense().numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05 + ) dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - dense_x.grad.numpy() * - np_x.astype('bool').astype('int'), - rtol=1e-05) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + dense_x.grad.numpy() * np_x.astype('bool').astype('int'), + rtol=1e-05, + ) # check gpu kernel if paddle.device.is_compiled_with_cuda(): @@ -70,28 +72,35 @@ class TestReshape(unittest.TestCase): if format == "coo": sp_x = paddle.to_tensor( - np_x, place=paddle.CUDAPlace(0)).to_sparse_coo(len(x_shape)) + np_x, place=paddle.CUDAPlace(0) + ).to_sparse_coo(len(x_shape)) else: sp_x = paddle.to_tensor( - np_x, place=paddle.CUDAPlace(0)).to_sparse_csr() + np_x, place=paddle.CUDAPlace(0) + ).to_sparse_csr() sp_x.stop_gradient = False sp_out = paddle.sparse.reshape(sp_x, new_shape) - np.testing.assert_allclose(sp_out.to_dense().numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05 + ) dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - dense_x.grad.numpy() * - np_x.astype('bool').astype('int'), - rtol=1e-05) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + dense_x.grad.numpy() * np_x.astype('bool').astype('int'), + rtol=1e-05, + ) def test_reshape_2d(self): - self.check_result([2, 5], [ - 10, - ], 'coo') + self.check_result( + [2, 5], + [ + 10, + ], + 'coo', + ) self.check_result([12, 5], [15, 4], 'coo') self.check_result([10, 5], [2, 25], 'csr') diff --git a/python/paddle/fluid/tests/unittests/test_sparse_softmax_op.py b/python/paddle/fluid/tests/unittests/test_sparse_softmax_op.py index 251f6cce1ce..ef1f672047f 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_softmax_op.py @@ -23,7 +23,6 @@ np.random.seed(2022) class TestCsrSoftmax(unittest.TestCase): - def test_softmax2d(self): with _test_eager_guard(): mask = np.random.rand(16, 128) < 0.5 @@ -46,12 +45,12 @@ class TestCsrSoftmax(unittest.TestCase): csr = paddle.to_tensor(np_x, stop_gradient=False).to_sparse_csr() m = paddle.sparse.nn.Softmax() out = m(csr) - np.testing.assert_allclose(out.crows().numpy(), - np_csr.indptr, - rtol=1e-05) - np.testing.assert_allclose(out.cols().numpy(), - np_csr.indices, - rtol=1e-05) + np.testing.assert_allclose( + out.crows().numpy(), np_csr.indptr, rtol=1e-05 + ) + np.testing.assert_allclose( + out.cols().numpy(), np_csr.indices, rtol=1e-05 + ) np.testing.assert_allclose(out.values().numpy(), np_out, rtol=1e-05) # dx = (dout - sum(dout * out)) * out, dout=rand_x @@ -67,15 +66,15 @@ class TestCsrSoftmax(unittest.TestCase): sum = np.sum(dout * out, keepdims=True) dx = np.concatenate([dx, (dout - sum) * out]) - np.testing.assert_allclose(csr.grad.crows().numpy(), - np_csr.indptr, - rtol=1e-05) - np.testing.assert_allclose(csr.grad.cols().numpy(), - np_csr.indices, - rtol=1e-05) - np.testing.assert_allclose(csr.grad.values().numpy(), - dx, - rtol=1e-05) + np.testing.assert_allclose( + csr.grad.crows().numpy(), np_csr.indptr, rtol=1e-05 + ) + np.testing.assert_allclose( + csr.grad.cols().numpy(), np_csr.indices, rtol=1e-05 + ) + np.testing.assert_allclose( + csr.grad.values().numpy(), dx, rtol=1e-05 + ) def test_softmax3d(self): with _test_eager_guard(): @@ -88,7 +87,9 @@ class TestCsrSoftmax(unittest.TestCase): for i in range(batchNum): np_csr = sp.csr_matrix(np_x[i, :, :]) row_number = np_csr.shape[0] - for j in range(row_number, ): + for j in range( + row_number, + ): start = np_csr.indptr[j] end = np_csr.indptr[j + 1] if start == end: @@ -118,15 +119,15 @@ class TestCsrSoftmax(unittest.TestCase): if start == end: continue dout = np_csr.data[start:end] - out = np_out[batch_offset + start:batch_offset + end] + out = np_out[batch_offset + start : batch_offset + end] sum = np.sum(dout * out, keepdims=True) dx = np.concatenate([dx, (dout - sum) * out]) batch_offset += np_csr.nnz - np.testing.assert_allclose(csr.grad.values().numpy(), - dx, - rtol=1e-05) + np.testing.assert_allclose( + csr.grad.values().numpy(), dx, rtol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py b/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py index 12d1fd9a8b7..06f221bfe6c 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py @@ -37,14 +37,16 @@ class TestTranspose(unittest.TestCase): sp_x.stop_gradient = False sp_out = paddle.sparse.transpose(sp_x, dims) - np.testing.assert_allclose(sp_out.to_dense().numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05 + ) dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) def test_transpose_2d(self): self.check_result([2, 5], [0, 1], 'coo') @@ -69,8 +71,9 @@ class TestTranspose(unittest.TestCase): def test_transpose_nd(self): self.check_result([8, 3, 4, 4, 5, 3], [5, 3, 4, 1, 0, 2], 'coo') # Randint now only supports access to dimension 0 to 9. - self.check_result([2, 3, 4, 2, 3, 4, 2, 3, 4], - [2, 3, 4, 5, 6, 7, 8, 0, 1], 'coo') + self.check_result( + [2, 3, 4, 2, 3, 4, 2, 3, 4], [2, 3, 4, 5, 6, 7, 8, 0, 1], 'coo' + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py b/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py index fc39ab973c0..dd50608050e 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py @@ -19,7 +19,6 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ class TestSparseUnary(unittest.TestCase): - def to_sparse(self, x, format): if format == 'coo': return x.detach().to_sparse_coo(sparse_dim=x.ndim) @@ -63,18 +62,18 @@ class TestSparseUnary(unittest.TestCase): dense_out.backward() # compare forward - np.testing.assert_allclose(sp_out.to_dense().numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05 + ) # compare backward if dense_func == paddle.sqrt: - expect_grad = np.nan_to_num(dense_x.grad.numpy(), 0., 0., 0.) + expect_grad = np.nan_to_num(dense_x.grad.numpy(), 0.0, 0.0, 0.0) else: expect_grad = (dense_x.grad * mask).numpy() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - expect_grad, - rtol=1e-05) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), expect_grad, rtol=1e-05 + ) def compare_with_dense(self, dense_func, sparse_func): self.check_result(dense_func, sparse_func, 'coo') @@ -84,8 +83,9 @@ class TestSparseUnary(unittest.TestCase): self.check_result(dense_func, sparse_func, 'coo', attr1) self.check_result(dense_func, sparse_func, 'csr', attr1) - def compare_with_dense_two_attr(self, dense_func, sparse_func, attr1, - attr2): + def compare_with_dense_two_attr( + self, dense_func, sparse_func, attr1, attr2 + ): self.check_result(dense_func, sparse_func, 'coo', attr1, attr2) self.check_result(dense_func, sparse_func, 'csr', attr1, attr2) @@ -129,8 +129,9 @@ class TestSparseUnary(unittest.TestCase): self.compare_with_dense(paddle.nn.ReLU6(), paddle.sparse.nn.ReLU6()) def test_sparse_leaky_relu(self): - self.compare_with_dense(paddle.nn.LeakyReLU(0.1), - paddle.sparse.nn.LeakyReLU(0.1)) + self.compare_with_dense( + paddle.nn.LeakyReLU(0.1), paddle.sparse.nn.LeakyReLU(0.1) + ) def test_sparse_abs(self): self.compare_with_dense(paddle.abs, paddle.sparse.abs) @@ -151,18 +152,22 @@ class TestSparseUnary(unittest.TestCase): self.compare_with_dense_one_attr(paddle.pow, paddle.sparse.pow, 3) def test_sparse_mul_scalar(self): - self.compare_with_dense_one_attr(paddle.Tensor.__mul__, - paddle.sparse.multiply, 3) + self.compare_with_dense_one_attr( + paddle.Tensor.__mul__, paddle.sparse.multiply, 3 + ) def test_sparse_div_scalar(self): - self.compare_with_dense_one_attr(paddle.Tensor.__div__, - paddle.sparse.divide, 2) + self.compare_with_dense_one_attr( + paddle.Tensor.__div__, paddle.sparse.divide, 2 + ) def test_sparse_cast(self): - self.compare_with_dense_two_attr(paddle.cast, paddle.sparse.cast, - 'int32', 'float32') - self.compare_with_dense_two_attr(paddle.cast, paddle.sparse.cast, - 'int32', 'float64') + self.compare_with_dense_two_attr( + paddle.cast, paddle.sparse.cast, 'int32', 'float32' + ) + self.compare_with_dense_two_attr( + paddle.cast, paddle.sparse.cast, 'int32', 'float64' + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py b/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py index 700c0795a0d..8659491da2e 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py @@ -23,7 +23,6 @@ devices = ['cpu', 'gpu'] class TestSparseCreate(unittest.TestCase): - def test_create_coo_by_tensor(self): with _test_eager_guard(): indices = [[0, 0, 1, 2, 2], [1, 3, 2, 0, 1]] @@ -31,10 +30,9 @@ class TestSparseCreate(unittest.TestCase): dense_shape = [3, 4] dense_indices = paddle.to_tensor(indices) dense_elements = paddle.to_tensor(values, dtype='float32') - coo = paddle.sparse.sparse_coo_tensor(dense_indices, - dense_elements, - dense_shape, - stop_gradient=False) + coo = paddle.sparse.sparse_coo_tensor( + dense_indices, dense_elements, dense_shape, stop_gradient=False + ) # test the to_string.py assert np.array_equal(indices, coo.indices().numpy()) assert np.array_equal(values, coo.values().numpy()) @@ -59,11 +57,13 @@ class TestSparseCreate(unittest.TestCase): dense_cols = paddle.to_tensor(cols) dense_elements = paddle.to_tensor(values, dtype='float32') stop_gradient = False - csr = paddle.sparse.sparse_csr_tensor(dense_crows, - dense_cols, - dense_elements, - dense_shape, - stop_gradient=stop_gradient) + csr = paddle.sparse.sparse_csr_tensor( + dense_crows, + dense_cols, + dense_elements, + dense_shape, + stop_gradient=stop_gradient, + ) def test_create_csr_by_np(self): with _test_eager_guard(): @@ -71,8 +71,9 @@ class TestSparseCreate(unittest.TestCase): cols = [1, 3, 2, 0, 1] values = [1, 2, 3, 4, 5] dense_shape = [3, 4] - csr = paddle.sparse.sparse_csr_tensor(crows, cols, values, - dense_shape) + csr = paddle.sparse.sparse_csr_tensor( + crows, cols, values, dense_shape + ) # test the to_string.py assert np.array_equal(5, csr.nnz()) assert np.array_equal(crows, csr.crows().numpy()) @@ -85,10 +86,9 @@ class TestSparseCreate(unittest.TestCase): indices = [[0, 1], [0, 1]] values = [1.0, 2.0] dense_shape = [2, 2] - coo = paddle.sparse.sparse_coo_tensor(indices, - values, - dense_shape, - place=place) + coo = paddle.sparse.sparse_coo_tensor( + indices, values, dense_shape, place=place + ) assert coo.place.is_cpu_place() assert coo.values().place.is_cpu_place() assert coo.indices().place.is_cpu_place() @@ -96,10 +96,9 @@ class TestSparseCreate(unittest.TestCase): crows = [0, 2, 3, 5] cols = [1, 3, 2, 0, 1] values = [1.0, 2.0, 3.0, 4.0, 5.0] - csr = paddle.sparse.sparse_csr_tensor(crows, - cols, - values, [3, 5], - place=place) + csr = paddle.sparse.sparse_csr_tensor( + crows, cols, values, [3, 5], place=place + ) assert csr.place.is_cpu_place() assert csr.crows().place.is_cpu_place() assert csr.cols().place.is_cpu_place() @@ -112,19 +111,17 @@ class TestSparseCreate(unittest.TestCase): dense_shape = [2, 2] indices = paddle.to_tensor(indices, dtype='int32') values = paddle.to_tensor(values, dtype='float32') - coo = paddle.sparse.sparse_coo_tensor(indices, - values, - dense_shape, - dtype='float64') + coo = paddle.sparse.sparse_coo_tensor( + indices, values, dense_shape, dtype='float64' + ) assert coo.dtype == paddle.float64 crows = [0, 2, 3, 5] cols = [1, 3, 2, 0, 1] values = [1.0, 2.0, 3.0, 4.0, 5.0] - csr = paddle.sparse.sparse_csr_tensor(crows, - cols, - values, [3, 5], - dtype='float16') + csr = paddle.sparse.sparse_csr_tensor( + crows, cols, values, [3, 5], dtype='float16' + ) assert csr.dtype == paddle.float16 def test_create_coo_no_shape(self): @@ -138,7 +135,6 @@ class TestSparseCreate(unittest.TestCase): class TestSparseConvert(unittest.TestCase): - def test_to_sparse_coo(self): with _test_eager_guard(): x = [[0, 1, 0, 2], [0, 0, 3, 0], [4, 5, 0, 0]] @@ -148,17 +144,19 @@ class TestSparseConvert(unittest.TestCase): out = dense_x.to_sparse_coo(2) assert np.array_equal(out.indices().numpy(), indices) assert np.array_equal(out.values().numpy(), values) - #test to_sparse_coo_grad backward + # test to_sparse_coo_grad backward out_grad_indices = [[0, 1], [0, 1]] out_grad_values = [2.0, 3.0] out_grad = paddle.sparse.sparse_coo_tensor( paddle.to_tensor(out_grad_indices), paddle.to_tensor(out_grad_values), shape=out.shape, - stop_gradient=True) + stop_gradient=True, + ) out.backward(out_grad) - assert np.array_equal(dense_x.grad.numpy(), - out_grad.to_dense().numpy()) + assert np.array_equal( + dense_x.grad.numpy(), out_grad.to_dense().numpy() + ) def test_coo_to_dense(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -171,27 +169,34 @@ class TestSparseConvert(unittest.TestCase): paddle.to_tensor(indices, dtype=indices_dtype), paddle.to_tensor(values), shape=[3, 4], - stop_gradient=False) + stop_gradient=False, + ) dense_tensor = sparse_x.to_dense() - #test to_dense_grad backward - out_grad = [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]] + # test to_dense_grad backward + out_grad = [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] dense_tensor.backward(paddle.to_tensor(out_grad)) - #mask the out_grad by sparse_x.indices() + # mask the out_grad by sparse_x.indices() correct_x_grad = [2.0, 4.0, 7.0, 9.0, 10.0] - assert np.array_equal(correct_x_grad, - sparse_x.grad.values().numpy()) + assert np.array_equal( + correct_x_grad, sparse_x.grad.values().numpy() + ) paddle.device.set_device("cpu") sparse_x_cpu = paddle.sparse.sparse_coo_tensor( paddle.to_tensor(indices, dtype=indices_dtype), paddle.to_tensor(values), shape=[3, 4], - stop_gradient=False) + stop_gradient=False, + ) dense_tensor_cpu = sparse_x_cpu.to_dense() dense_tensor_cpu.backward(paddle.to_tensor(out_grad)) - assert np.array_equal(correct_x_grad, - sparse_x_cpu.grad.values().numpy()) + assert np.array_equal( + correct_x_grad, sparse_x_cpu.grad.values().numpy() + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_to_sparse_csr(self): @@ -218,23 +223,35 @@ class TestSparseConvert(unittest.TestCase): paddle.to_tensor(indices), paddle.to_tensor(values), shape=[3, 4], - stop_gradient=False) + stop_gradient=False, + ) values_tensor = sparse_x.values() out_grad = [2.0, 3.0, 5.0, 8.0, 9.0] # test coo_values_grad values_tensor.backward(paddle.to_tensor(out_grad)) assert np.array_equal(out_grad, sparse_x.grad.values().numpy()) indices = [[0, 0, 1, 2, 2], [1, 3, 2, 0, 1]] - values = [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0], - [5.0, 5.0]] + values = [ + [1.0, 1.0], + [2.0, 2.0], + [3.0, 3.0], + [4.0, 4.0], + [5.0, 5.0], + ] sparse_x = paddle.sparse.sparse_coo_tensor( paddle.to_tensor(indices), paddle.to_tensor(values), shape=[3, 4, 2], - stop_gradient=False) + stop_gradient=False, + ) values_tensor = sparse_x.values() - out_grad = [[2.0, 2.0], [3.0, 3.0], [5.0, 5.0], [8.0, 8.0], - [9.0, 9.0]] + out_grad = [ + [2.0, 2.0], + [3.0, 3.0], + [5.0, 5.0], + [8.0, 8.0], + [9.0, 9.0], + ] # test coo_values_grad values_tensor.backward(paddle.to_tensor(out_grad)) assert np.array_equal(out_grad, sparse_x.grad.values().numpy()) @@ -243,51 +260,59 @@ class TestSparseConvert(unittest.TestCase): def test_sparse_coo_tensor_grad(self): with _test_eager_guard(): for device in devices: - if device == 'cpu' or (device == 'gpu' - and paddle.is_compiled_with_cuda()): + if device == 'cpu' or ( + device == 'gpu' and paddle.is_compiled_with_cuda() + ): paddle.device.set_device(device) indices = [[0, 1], [0, 1]] values = [1, 2] indices = paddle.to_tensor(indices, dtype='int32') - values = paddle.to_tensor(values, - dtype='float32', - stop_gradient=False) + values = paddle.to_tensor( + values, dtype='float32', stop_gradient=False + ) sparse_x = paddle.sparse.sparse_coo_tensor( - indices, values, shape=[2, 2], stop_gradient=False) + indices, values, shape=[2, 2], stop_gradient=False + ) grad_indices = [[0, 1], [1, 1]] grad_values = [2, 3] grad_indices = paddle.to_tensor(grad_indices, dtype='int32') grad_values = paddle.to_tensor(grad_values, dtype='float32') sparse_out_grad = paddle.sparse.sparse_coo_tensor( - grad_indices, grad_values, shape=[2, 2]) + grad_indices, grad_values, shape=[2, 2] + ) sparse_x.backward(sparse_out_grad) correct_values_grad = [0, 3] - assert np.array_equal(correct_values_grad, - values.grad.numpy()) + assert np.array_equal( + correct_values_grad, values.grad.numpy() + ) # test the non-zero values is a vector values = [[1, 1], [2, 2]] - values = paddle.to_tensor(values, - dtype='float32', - stop_gradient=False) + values = paddle.to_tensor( + values, dtype='float32', stop_gradient=False + ) sparse_x = paddle.sparse.sparse_coo_tensor( - indices, values, shape=[2, 2, 2], stop_gradient=False) + indices, values, shape=[2, 2, 2], stop_gradient=False + ) grad_values = [[2, 2], [3, 3]] grad_values = paddle.to_tensor(grad_values, dtype='float32') sparse_out_grad = paddle.sparse.sparse_coo_tensor( - grad_indices, grad_values, shape=[2, 2, 2]) + grad_indices, grad_values, shape=[2, 2, 2] + ) sparse_x.backward(sparse_out_grad) correct_values_grad = [[0, 0], [3, 3]] - assert np.array_equal(correct_values_grad, - values.grad.numpy()) + assert np.array_equal( + correct_values_grad, values.grad.numpy() + ) def test_sparse_coo_tensor_sorted(self): with _test_eager_guard(): for device in devices: - if device == 'cpu' or (device == 'gpu' - and paddle.is_compiled_with_cuda()): + if device == 'cpu' or ( + device == 'gpu' and paddle.is_compiled_with_cuda() + ): paddle.device.set_device(device) - #test unsorted and duplicate indices + # test unsorted and duplicate indices indices = [[1, 0, 0], [0, 1, 1]] values = [1.0, 2.0, 3.0] indices = paddle.to_tensor(indices, dtype='int32') @@ -296,10 +321,12 @@ class TestSparseConvert(unittest.TestCase): sparse_x = paddle.sparse.coalesce(sparse_x) indices_sorted = [[0, 1], [1, 0]] values_sorted = [5.0, 1.0] - assert np.array_equal(indices_sorted, - sparse_x.indices().numpy()) - assert np.array_equal(values_sorted, - sparse_x.values().numpy()) + assert np.array_equal( + indices_sorted, sparse_x.indices().numpy() + ) + assert np.array_equal( + values_sorted, sparse_x.values().numpy() + ) # test the non-zero values is a vector values = [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]] @@ -307,10 +334,12 @@ class TestSparseConvert(unittest.TestCase): sparse_x = paddle.sparse.sparse_coo_tensor(indices, values) sparse_x = paddle.sparse.coalesce(sparse_x) values_sorted = [[5.0, 5.0], [1.0, 1.0]] - assert np.array_equal(indices_sorted, - sparse_x.indices().numpy()) - assert np.array_equal(values_sorted, - sparse_x.values().numpy()) + assert np.array_equal( + indices_sorted, sparse_x.indices().numpy() + ) + assert np.array_equal( + values_sorted, sparse_x.values().numpy() + ) def test_batch_csr(self): with _test_eager_guard(): @@ -326,7 +355,7 @@ class TestSparseConvert(unittest.TestCase): dense_x = paddle.nn.functional.dropout(dense_x, p=0.5) verify(dense_x) - #test batchs=1 + # test batchs=1 shape[0] = 1 dense_x = paddle.randn(shape) dense_x = paddle.nn.functional.dropout(dense_x, p=0.5) @@ -335,23 +364,22 @@ class TestSparseConvert(unittest.TestCase): shape = np.random.randint(low=3, high=10, size=3) shape = list(shape) dense_x = paddle.randn(shape) - #set the 0th batch to zero + # set the 0th batch to zero dense_x[0] = 0 verify(dense_x) dense_x = paddle.randn(shape) - #set the 1th batch to zero + # set the 1th batch to zero dense_x[1] = 0 verify(dense_x) dense_x = paddle.randn(shape) - #set the 2th batch to zero + # set the 2th batch to zero dense_x[2] = 0 verify(dense_x) class TestCooError(unittest.TestCase): - def test_small_shape(self): with _test_eager_guard(): with self.assertRaises(ValueError): @@ -359,9 +387,9 @@ class TestCooError(unittest.TestCase): values = [1, 2] # 1. the shape too small dense_shape = [2, 2] - sparse_x = paddle.sparse.sparse_coo_tensor(indices, - values, - shape=dense_shape) + sparse_x = paddle.sparse.sparse_coo_tensor( + indices, values, shape=dense_shape + ) def test_same_nnz(self): with _test_eager_guard(): @@ -377,9 +405,9 @@ class TestCooError(unittest.TestCase): indices = [[1, 2], [1, 0]] values = [1, 2, 3] shape = [2, 3, 4] - sparse_x = paddle.sparse.sparse_coo_tensor(indices, - values, - shape=shape) + sparse_x = paddle.sparse.sparse_coo_tensor( + indices, values, shape=shape + ) def test_indices_dtype(self): with _test_eager_guard(): @@ -390,7 +418,6 @@ class TestCooError(unittest.TestCase): class TestCsrError(unittest.TestCase): - def test_dimension1(self): with _test_eager_guard(): with self.assertRaises(ValueError): @@ -399,7 +426,8 @@ class TestCsrError(unittest.TestCase): values = [1, 2, 3] shape = [3] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_dimension2(self): with _test_eager_guard(): @@ -409,7 +437,8 @@ class TestCsrError(unittest.TestCase): values = [1, 2, 3] shape = [3, 3, 3, 3] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_same_shape1(self): with _test_eager_guard(): @@ -419,7 +448,8 @@ class TestCsrError(unittest.TestCase): values = [1, 2, 3] shape = [3, 4] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_same_shape2(self): with _test_eager_guard(): @@ -429,7 +459,8 @@ class TestCsrError(unittest.TestCase): values = [1, 2, 3, 4] shape = [3, 4] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_same_shape3(self): with _test_eager_guard(): @@ -439,7 +470,8 @@ class TestCsrError(unittest.TestCase): values = [1, 2, 3, 4, 0, 1, 2] shape = [2, 3, 4] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_crows_first_value(self): with _test_eager_guard(): @@ -449,7 +481,8 @@ class TestCsrError(unittest.TestCase): values = [1, 2, 3] shape = [3, 4] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_dtype(self): with _test_eager_guard(): @@ -459,7 +492,8 @@ class TestCsrError(unittest.TestCase): values = [1, 2, 3] shape = [3] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py b/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py index 1b230df9128..e05b1cdaaf3 100644 --- a/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py +++ b/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py @@ -17,7 +17,11 @@ import unittest import paddle import paddle.distributed as dist -from paddle.distributed.spawn import _get_subprocess_env_list, _options_valid_check, _get_default_nprocs +from paddle.distributed.spawn import ( + _get_subprocess_env_list, + _options_valid_check, + _get_default_nprocs, +) from paddle.fluid import core from paddle.fluid.dygraph import parallel_helper @@ -28,10 +32,10 @@ import multiprocessing # executed in the python3 sub-process. -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestInitParallelEnv(unittest.TestCase): - def test_check_env_failed(self): os.environ['FLAGS_selected_gpus'] = '0' os.environ['PADDLE_TRAINER_ID'] = '0' @@ -51,10 +55,10 @@ class TestInitParallelEnv(unittest.TestCase): self.assertFalse(parallel_helper._is_parallel_ctx_initialized()) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSpawnAssistMethod(unittest.TestCase): - def test_nprocs_greater_than_device_num_error(self): with self.assertRaises(RuntimeError): _get_subprocess_env_list(nprocs=100, options=dict()) diff --git a/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py b/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py index c310819bf18..5cff9628d8d 100644 --- a/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py @@ -47,15 +47,15 @@ def spectral_norm(weight, u, v, dim, power_iters, eps): @skip_check_grad_ci( reason="Spectral norm do not check grad when power_iters > 0 " "because grad is not calculated in power iterations, " - "which cannot be checked by python grad unittests") + "which cannot be checked by python grad unittests" +) class TestSpectralNormOpNoGrad(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'spectral_norm' weight = np.random.random(self.weight_shape).astype('float64') - u = np.random.normal(0., 1., self.u_shape).astype('float64') - v = np.random.normal(0., 1., self.v_shape).astype('float64') + u = np.random.normal(0.0, 1.0, self.u_shape).astype('float64') + v = np.random.normal(0.0, 1.0, self.v_shape).astype('float64') self.attrs = { "dim": self.dim, @@ -69,8 +69,9 @@ class TestSpectralNormOpNoGrad(OpTest): "V": v, } - output = spectral_norm(weight, u, v, self.dim, self.power_iters, - self.eps) + output = spectral_norm( + weight, u, v, self.dim, self.power_iters, self.eps + ) self.outputs = {"Out": output} def test_check_output(self): @@ -78,8 +79,8 @@ class TestSpectralNormOpNoGrad(OpTest): def initTestCase(self): self.weight_shape = (10, 12) - self.u_shape = (10, ) - self.v_shape = (12, ) + self.u_shape = (10,) + self.v_shape = (12,) self.dim = 0 self.power_iters = 5 self.eps = 1e-12 @@ -88,20 +89,19 @@ class TestSpectralNormOpNoGrad(OpTest): @skip_check_grad_ci( reason="Spectral norm do not check grad when power_iters > 0 " "because grad is not calculated in power iterations, " - "which cannot be checked by python grad unittests") + "which cannot be checked by python grad unittests" +) class TestSpectralNormOpNoGrad2(TestSpectralNormOpNoGrad): - def initTestCase(self): self.weight_shape = (2, 3, 3, 3) - self.u_shape = (3, ) - self.v_shape = (18, ) + self.u_shape = (3,) + self.v_shape = (18,) self.dim = 1 self.power_iters = 10 self.eps = 1e-12 class TestSpectralNormOp(TestSpectralNormOpNoGrad): - def test_check_grad_ignore_uv(self): self.check_grad( ['Weight'], @@ -111,26 +111,24 @@ class TestSpectralNormOp(TestSpectralNormOpNoGrad): def initTestCase(self): self.weight_shape = (10, 12) - self.u_shape = (10, ) - self.v_shape = (12, ) + self.u_shape = (10,) + self.v_shape = (12,) self.dim = 0 self.power_iters = 0 self.eps = 1e-12 class TestSpectralNormOp2(TestSpectralNormOp): - def initTestCase(self): self.weight_shape = (2, 6, 3, 3) - self.u_shape = (6, ) - self.v_shape = (18, ) + self.u_shape = (6,) + self.v_shape = (18,) self.dim = 1 self.power_iters = 0 self.eps = 1e-12 class TestSpectralNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -150,13 +148,12 @@ class TestSpectralNormOpError(unittest.TestCase): class TestDygraphSpectralNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): shape = (2, 4, 3, 3) - spectralNorm = fluid.dygraph.nn.SpectralNorm(shape, - dim=1, - power_iters=2) + spectralNorm = fluid.dygraph.nn.SpectralNorm( + shape, dim=1, power_iters=2 + ) def test_Variable(): weight_1 = np.random.random((2, 4)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py index 0afd3ec56bb..fefa11be9aa 100644 --- a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py @@ -27,7 +27,6 @@ from paddle.fluid.layer_helper import LayerHelper class TestCPULoDTensorArrayOps(unittest.TestCase): - def place(self): return core.CPUPlace() @@ -52,11 +51,13 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): expect_false = core.LoDTensor() expect_false.set(expect_false_tensor, self.place()) - self.main(tensor=tensor, - mask=mask, - expect_true=expect_true, - expect_false=expect_false, - expect_out=tensor) + self.main( + tensor=tensor, + mask=mask, + expect_true=expect_true, + expect_false=expect_false, + expect_out=tensor, + ) def split_and_merge_lod_tensor_level_0(self, use_merge_lod_infer=False): tensor = core.LoDTensor() @@ -83,12 +84,14 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): expect_false.set(expect_false_tensor, self.place()) expect_false.set_recursive_sequence_lengths(expect_false_lod) - self.main(tensor=tensor, - mask=mask, - expect_true=expect_true, - expect_false=expect_false, - expect_out=tensor, - use_merge_lod_infer=use_merge_lod_infer) + self.main( + tensor=tensor, + mask=mask, + expect_true=expect_true, + expect_false=expect_false, + expect_out=tensor, + use_merge_lod_infer=use_merge_lod_infer, + ) def test_split_and_merge_lod_tensor_1(self): self.split_and_merge_lod_tensor_level_0() @@ -96,14 +99,16 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): def test_split_and_merge_lod_tensor_2(self): self.split_and_merge_lod_tensor_level_0(True) - def main(self, - tensor, - mask, - expect_true, - expect_false, - expect_out, - level=0, - use_merge_lod_infer=False): + def main( + self, + tensor, + mask, + expect_true, + expect_false, + expect_out, + level=0, + use_merge_lod_infer=False, + ): place = self.place() program = Program() with program_guard(program): @@ -122,38 +127,42 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): 'Mask': mask, 'InTrue': out_true, 'InFalse': out_false, - 'level': level + 'level': level, } helper = LayerHelper('merge_lod_tensor_infer') out = helper.create_variable_for_type_inference( - dtype=out_true.dtype) - helper.append_op(type='merge_lod_tensor_infer', - inputs={ - 'X': x, - 'Mask': y, - 'InTrue': out_true, - 'InFalse': out_false - }, - outputs={'Out': out}, - attrs={'level': level}) + dtype=out_true.dtype + ) + helper.append_op( + type='merge_lod_tensor_infer', + inputs={ + 'X': x, + 'Mask': y, + 'InTrue': out_true, + 'InFalse': out_false, + }, + outputs={'Out': out}, + attrs={'level': level}, + ) out.persistable = True else: - out = merge_lod_tensor(in_true=out_true, - in_false=out_false, - mask=y, - x=x, - level=level) + out = merge_lod_tensor( + in_true=out_true, + in_false=out_false, + mask=y, + x=x, + level=level, + ) out.persistable = True exe = Executor(place) scope = core.Scope() - exe.run(program, - feed={ - 'x': tensor, - 'y': mask - }, - scope=scope, - return_numpy=False) + exe.run( + program, + feed={'x': tensor, 'y': mask}, + scope=scope, + return_numpy=False, + ) var_true = scope.find_var(out_true.name).get_tensor() @@ -166,36 +175,33 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): self.check_tensor_same(var_out, expect_out) def check_tensor_same(self, actual, expect): - np.testing.assert_allclose(np.array(actual), - np.array(expect), - rtol=1e-05) - self.assertEqual(actual.recursive_sequence_lengths(), - expect.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual), np.array(expect), rtol=1e-05 + ) + self.assertEqual( + actual.recursive_sequence_lengths(), + expect.recursive_sequence_lengths(), + ) class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase): - def test_grad(self): place = core.CPUPlace() program = Program() with program_guard(program): - x = layers.data(name='x', - shape=[1], - dtype='float32', - stop_gradient=False) - y = layers.data(name='y', - shape=[1], - dtype='bool', - stop_gradient=False) + x = layers.data( + name='x', shape=[1], dtype='float32', stop_gradient=False + ) + y = layers.data( + name='y', shape=[1], dtype='bool', stop_gradient=False + ) level = 0 out_true, out_false = split_lod_tensor(input=x, mask=y, level=level) - out = merge_lod_tensor(in_true=out_true, - in_false=out_false, - mask=y, - x=x, - level=level) + out = merge_lod_tensor( + in_true=out_true, in_false=out_false, mask=y, x=x, level=level + ) mean = paddle.mean(out) append_backward(mean) @@ -215,16 +221,17 @@ class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase): g_vars = program.global_block().var(x.name + "@GRAD") g_out = [ - item.sum() for item in map( + item.sum() + for item in map( np.array, - exe.run(program, - feed={ - 'x': tensor, - 'y': mask - }, - fetch_list=[g_vars], - scope=scope, - return_numpy=False)) + exe.run( + program, + feed={'x': tensor, 'y': mask}, + fetch_list=[g_vars], + scope=scope, + return_numpy=False, + ), + ) ] g_out_sum = np.array(g_out).sum() @@ -233,78 +240,78 @@ class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase): class TestMergeLodTensorOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - input_data = layers.data(name='x', - shape=[1], - dtype='float32', - stop_gradient=False) - y = layers.data(name='y', - shape=[1], - dtype='bool', - stop_gradient=False) - x_true = layers.data(name='x_true', - shape=[1], - dtype='float32', - stop_gradient=False) - x_false = layers.data(name='x_false', - shape=[1], - dtype='float32', - stop_gradient=False) + input_data = layers.data( + name='x', shape=[1], dtype='float32', stop_gradient=False + ) + y = layers.data( + name='y', shape=[1], dtype='bool', stop_gradient=False + ) + x_true = layers.data( + name='x_true', shape=[1], dtype='float32', stop_gradient=False + ) + x_false = layers.data( + name='x_false', shape=[1], dtype='float32', stop_gradient=False + ) level = 0 def test_x(): - out = merge_lod_tensor(int_true=x_true, - in_false=x_false, - x=set(), - mask=y, - level=level) + out = merge_lod_tensor( + int_true=x_true, + in_false=x_false, + x=set(), + mask=y, + level=level, + ) self.assertRaises(TypeError, test_x) def test_mask(): - out = merge_lod_tensor(int_true=x_true, - in_false=x_false, - x=input_data, - mask=set(), - level=level) + out = merge_lod_tensor( + int_true=x_true, + in_false=x_false, + x=input_data, + mask=set(), + level=level, + ) self.assertRaises(TypeError, test_mask) def test_xtrue(): - out = merge_lod_tensor(int_true=set(), - in_false=x_false, - x=input_data, - mask=y, - level=level) + out = merge_lod_tensor( + int_true=set(), + in_false=x_false, + x=input_data, + mask=y, + level=level, + ) self.assertRaises(TypeError, test_xtrue) def test_xfalse(): - out = merge_lod_tensor(int_true=x_true, - in_false=set(), - x=input_data, - mask=y, - level=level) + out = merge_lod_tensor( + int_true=x_true, + in_false=set(), + x=input_data, + mask=y, + level=level, + ) self.assertRaises(TypeError, test_xfalse) class TestSplitLodTensorWithError(unittest.TestCase): - def test_error(self): main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): - x = layers.data(name='x', - shape=[1], - dtype='float32', - stop_gradient=False) - y = layers.data(name='y', - shape=[1], - dtype='bool', - stop_gradient=False) + x = layers.data( + name='x', shape=[1], dtype='float32', stop_gradient=False + ) + y = layers.data( + name='y', shape=[1], dtype='bool', stop_gradient=False + ) level = 0 with self.assertRaises(TypeError): diff --git a/python/paddle/fluid/tests/unittests/test_split_op.py b/python/paddle/fluid/tests/unittests/test_split_op.py index 14ba9eb45a0..e087cc8b1bb 100644 --- a/python/paddle/fluid/tests/unittests/test_split_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_op.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class TestSplitOp(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -31,14 +30,19 @@ class TestSplitOp(OpTest): x = np.random.random((4, 5, 6)).astype(np.float32) out = np.split(x, [2, 3], axis) self.inputs = {'X': convert_float_to_uint16(x)} - self.outputs = {'Out': [('out%d' % i, convert_float_to_uint16(out[i])) \ - for i in range(len(out))]} + self.outputs = { + 'Out': [ + ('out%d' % i, convert_float_to_uint16(out[i])) + for i in range(len(out)) + ] + } else: x = np.random.random((4, 5, 6)).astype(self.dtype) out = np.split(x, [2, 3], axis) self.inputs = {'X': x} - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = { + 'Out': [('out%d' % i, out[i]) for i in range(len(out))] + } self.attrs = {'axis': axis, 'sections': [2, 1, 2]} def get_dtype(self): @@ -56,7 +60,6 @@ class TestSplitOp(OpTest): # test with attr(num) class TestSplitOp_2(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -65,12 +68,11 @@ class TestSplitOp_2(OpTest): self.attrs = { 'axis': self.axis, 'sections': self.sections, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} def init_data(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) @@ -94,20 +96,18 @@ class TestSplitOp_2(OpTest): # attr(axis) is Tensor class TestSplitOp_AxisTensor(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() self.init_data() self.inputs = { 'X': self.x, - 'AxisTensor': np.array([self.axis]).astype("int32") + 'AxisTensor': np.array([self.axis]).astype("int32"), } self.attrs = {'sections': self.sections, 'num': self.num} out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} def init_data(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) @@ -131,7 +131,6 @@ class TestSplitOp_AxisTensor(OpTest): # attr(sections) is list containing Tensor class TestSplitOp_SectionsTensor(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -140,20 +139,20 @@ class TestSplitOp_SectionsTensor(OpTest): sections_tensor = [] for index, ele in enumerate(self.sections): - sections_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + sections_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SectionsTensorList'] = sections_tensor self.attrs = { 'axis': self.axis, 'sections': self.sections_infer, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} def init_data(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) @@ -177,7 +176,6 @@ class TestSplitOp_SectionsTensor(OpTest): class TestSplitOp_unk_section(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -186,12 +184,11 @@ class TestSplitOp_unk_section(OpTest): self.attrs = { 'axis': self.axis, 'sections': self.sections, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} def init_data(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) @@ -214,20 +211,18 @@ class TestSplitOp_unk_section(OpTest): class TestSplitByrefOp(OpTest): - def _set_op_type(self): self.op_type = "split_byref" -#----------------Split Fp16---------------- +# ----------------Split Fp16---------------- def create_test_fp16(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestSplitFp16(parent): - def get_dtype(self): return np.float16 @@ -241,15 +236,14 @@ def create_test_fp16(parent): create_test_fp16(TestSplitOp) -#----------------Split Bf16---------------- +# ----------------Split Bf16---------------- def create_test_bf16(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestSplitBf16(parent): - def get_dtype(self): return np.uint16 @@ -269,7 +263,6 @@ create_test_bf16(TestSplitOp) class TestSplitAPI(unittest.TestCase): - def test_api(self): input_1 = np.random.random([4, 5, 6]).astype("int32") positive_1_int32 = fluid.layers.fill_constant([1], "int32", 1) @@ -281,21 +274,20 @@ class TestSplitAPI(unittest.TestCase): out_0, out_1, out_2 = fluid.layers.split( input=x_1, num_or_sections=[positive_2_int64, positive_1_int32, -1], - dim=positive_1_int64) + dim=positive_1_int64, + ) - out_3, out_4, out_5 = fluid.layers.split(input=x_1, - num_or_sections=[2, 1, 2], - dim=positive_1_int32) + out_3, out_4, out_5 = fluid.layers.split( + input=x_1, num_or_sections=[2, 1, 2], dim=positive_1_int32 + ) fluid.layers.split(input=x_2, num_or_sections=2, dim=2) exe = fluid.Executor(place=fluid.CPUPlace()) - [res_0, res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={ - "x_1": input_1, - "x_2": input_1 - }, - fetch_list=[out_0, out_1, out_2, out_3, out_4, out_5]) + [res_0, res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x_1": input_1, "x_2": input_1}, + fetch_list=[out_0, out_1, out_2, out_3, out_4, out_5], + ) out = np.split(input_1, [2, 3], 1) assert np.array_equal(res_0, out[0]) @@ -307,7 +299,6 @@ class TestSplitAPI(unittest.TestCase): class TestSplitOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The type of axis in split_op should be int or Variable. @@ -346,7 +337,6 @@ class TestSplitOpError(unittest.TestCase): class API_TestSplit(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[4, 6, 6], dtype='float64') @@ -356,11 +346,9 @@ class API_TestSplit(unittest.TestCase): exe = fluid.Executor(place) input1 = np.random.random([4, 6, 6]).astype('float64') input2 = np.array([2]).astype('int32') - r0, r1, r2, = exe.run(feed={ - "data1": input1, - "data2": input2 - }, - fetch_list=[x0, x1, x2]) + r0, r1, r2, = exe.run( + feed={"data1": input1, "data2": input2}, fetch_list=[x0, x1, x2] + ) ex_x0, ex_x1, ex_x2 = np.split(input1, 3, axis=2) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) @@ -368,7 +356,6 @@ class API_TestSplit(unittest.TestCase): class API_TestSplit2(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[4, 6, 6], dtype='float64') @@ -376,8 +363,11 @@ class API_TestSplit2(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) input1 = np.random.random([4, 6, 6]).astype('float64') - r0, r1, r2, = exe.run(feed={"data1": input1}, - fetch_list=[x0, x1, x2]) + ( + r0, + r1, + r2, + ) = exe.run(feed={"data1": input1}, fetch_list=[x0, x1, x2]) ex_x0, ex_x1, ex_x2 = np.split(input1, 3, axis=2) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) @@ -385,7 +375,6 @@ class API_TestSplit2(unittest.TestCase): class API_TestSplit3(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float64') @@ -394,13 +383,12 @@ class API_TestSplit3(unittest.TestCase): exe = fluid.Executor(place) input1 = np.random.random([1, 10]).astype('float64') r0, r1 = exe.run(feed={"data": input1}, fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (3, ), axis=1) + ex_x0, ex_x1 = np.split(input1, (3,), axis=1) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) class API_TestSplit4(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float64') @@ -410,21 +398,19 @@ class API_TestSplit4(unittest.TestCase): exe = fluid.Executor(place) input1 = np.random.random([1, 10]).astype('float64') input2 = np.array([7]).astype('int32') - r0, r1 = exe.run(feed={ - "data": input1, - "index": input2 - }, - fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (3, ), axis=1) + r0, r1 = exe.run( + feed={"data": input1, "index": input2}, fetch_list=[x0, x1] + ) + ex_x0, ex_x1 = np.split(input1, (3,), axis=1) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) class API_TestSplit5(unittest.TestCase): - def test_out(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() with fluid.program_guard(fluid.Program(), fluid.Program()): input_1 = np.random.random([5, 4]).astype("int32") @@ -441,7 +427,6 @@ class API_TestSplit5(unittest.TestCase): class API_TestSplit6(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float64') @@ -450,13 +435,12 @@ class API_TestSplit6(unittest.TestCase): exe = fluid.Executor(place) input1 = np.random.random([2, 10]).astype('float64') r0, r1 = exe.run(feed={"data": input1}, fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (1, ), axis=0) + ex_x0, ex_x1 = np.split(input1, (1,), axis=0) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) class API_TestDygraphFluidSplit(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(): input_1 = np.random.random([4, 6, 6]).astype("int32") @@ -479,9 +463,9 @@ class API_TestDygraphFluidSplit(unittest.TestCase): loss.backward() manul_grad = np.zeros_like(input_1) manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), - manul_grad, - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), manul_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) @@ -512,9 +496,9 @@ class API_TestDygraphFluidSplit(unittest.TestCase): loss.backward() manul_grad = np.zeros_like(input_1) manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), - manul_grad, - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), manul_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) @@ -525,7 +509,6 @@ class API_TestDygraphFluidSplit(unittest.TestCase): class API_TestDygraphSplit(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(): input_1 = np.random.random([4, 6, 6]).astype("int32") @@ -549,9 +532,9 @@ class API_TestDygraphSplit(unittest.TestCase): loss.backward() manul_grad = np.zeros_like(input_1) manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), - manul_grad, - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), manul_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) @@ -599,9 +582,9 @@ class API_TestDygraphSplit(unittest.TestCase): # input is a variable which shape is [4, 6, 6] input = paddle.to_tensor(input_1) num1 = paddle.full(shape=[1], fill_value=2, dtype='int32') - x0, x1, x2 = paddle.split(input, - num_or_sections=[num1, 2, 2], - axis=1) + x0, x1, x2 = paddle.split( + input, num_or_sections=[num1, 2, 2], axis=1 + ) x0_out = x0.numpy() x1_out = x1.numpy() x2_out = x2.numpy() @@ -616,9 +599,9 @@ class API_TestDygraphSplit(unittest.TestCase): # input is a variable which shape is [4, 6, 6] input = paddle.to_tensor(input_1) num1 = paddle.full(shape=[1], fill_value=1, dtype='int32') - x0, x1, x2 = paddle.split(input, - num_or_sections=[2, 2, 2], - axis=num1) + x0, x1, x2 = paddle.split( + input, num_or_sections=[2, 2, 2], axis=num1 + ) x0_out = x0.numpy() x1_out = x1.numpy() x2_out = x2.numpy() @@ -644,7 +627,6 @@ class API_TestDygraphSplit(unittest.TestCase): class API_TestEmptySplit(unittest.TestCase): - def test_axis_input_empty_section(self): with fluid.dygraph.guard(): input_1 = np.random.random([8, 6, 6]).astype("float32") @@ -654,10 +636,13 @@ class API_TestEmptySplit(unittest.TestCase): x0_out = x0.numpy() x1_out = x1.numpy() x2_out = x2.numpy() - ex_x0, ex_x1, ex_x2 = np.split(input_1, [ - 5, - 5, - ]) + ex_x0, ex_x1, ex_x2 = np.split( + input_1, + [ + 5, + 5, + ], + ) np.testing.assert_allclose(ex_x0, x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, x2_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_split_program.py b/python/paddle/fluid/tests/unittests/test_split_program.py index 761c5ed5587..bab36fd88ce 100644 --- a/python/paddle/fluid/tests/unittests/test_split_program.py +++ b/python/paddle/fluid/tests/unittests/test_split_program.py @@ -21,7 +21,6 @@ import numpy as np class TestSplitProgram(unittest.TestCase): - def setUp(self): paddle.enable_static() if paddle.is_compiled_with_cuda(): @@ -31,12 +30,12 @@ class TestSplitProgram(unittest.TestCase): main = paddle.static.Program() startup = paddle.static.Program() with paddle.static.program_guard(main, startup): - image = paddle.static.data(shape=[batch_size, 3, 224, 224], - dtype='float32', - name='image') - label = paddle.static.data(shape=[batch_size, 1], - dtype='int64', - name='label') + image = paddle.static.data( + shape=[batch_size, 3, 224, 224], dtype='float32', name='image' + ) + label = paddle.static.data( + shape=[batch_size, 1], dtype='int64', name='label' + ) model = resnet(pretrained=False) loss_fn = nn.loss.CrossEntropyLoss() @@ -65,10 +64,11 @@ class TestSplitProgram(unittest.TestCase): self.assertEqual(len(vars_actual), len(vars_expected)) for actual, expected in zip(vars_actual, vars_expected): self.assertEqual(actual.shape, expected.shape) - np.testing.assert_array_equal(actual, - expected, - err_msg='{}\n{}\n'.format( - actual, expected)) + np.testing.assert_array_equal( + actual, + expected, + err_msg='{}\n{}\n'.format(actual, expected), + ) def get_places(self): places = [paddle.CPUPlace()] @@ -93,27 +93,26 @@ class TestSplitProgram(unittest.TestCase): exe = paddle.static.Executor(place) image_np = np.random.random(size=image.shape).astype('float32') - label_np = np.random.randint(low=0, - high=1000, - dtype='int64', - size=label.shape) + label_np = np.random.randint( + low=0, high=1000, dtype='int64', size=label.shape + ) scope = paddle.static.Scope() if not use_split: with paddle.static.scope_guard(scope): exe.run(startup_prog) for _ in range(batch_num): - exe.run(main_prog, - feed={ - image.name: image_np, - label.name: label_np - }) + exe.run( + main_prog, + feed={image.name: image_np, label.name: label_np}, + ) return self.get_var_values(scope, startup_vars) op_num = len(main_prog.global_block().ops) split_op_indices = [int(op_num / 3.0), int(op_num * 3 / 4.0)] programs, input_vars, output_vars = split_program( - main_prog, split_op_indices) + main_prog, split_op_indices + ) op_nums = [0] + split_op_indices + [op_num] op_nums = [op_nums[i + 1] - op_nums[i] for i in range(len(op_nums) - 1)] num_split = len(split_op_indices) + 1 @@ -137,15 +136,19 @@ class TestSplitProgram(unittest.TestCase): if tmp_vars[in_name] is not None: feed_dict[in_name] = tmp_vars[in_name] - output_var_values = exe.run(program, - feed=feed_dict, - fetch_list=output_vars[i], - return_numpy=False) - for out_name, out_value in zip(output_vars[i], - output_var_values): + output_var_values = exe.run( + program, + feed=feed_dict, + fetch_list=output_vars[i], + return_numpy=False, + ) + for out_name, out_value in zip( + output_vars[i], output_var_values + ): if not out_value._is_initialized(): tmp_vars[out_name] = np.ndarray( - out_value._get_dims()).astype('float32') + out_value._get_dims() + ).astype('float32') else: tmp_vars[out_name] = np.array(out_value) diff --git a/python/paddle/fluid/tests/unittests/test_splits_api.py b/python/paddle/fluid/tests/unittests/test_splits_api.py index 3613c7be3b6..491fed74f77 100644 --- a/python/paddle/fluid/tests/unittests/test_splits_api.py +++ b/python/paddle/fluid/tests/unittests/test_splits_api.py @@ -35,7 +35,6 @@ test_list = [ class TestSplitsAPI(unittest.TestCase): - def setUp(self): self.rtol = 1e-5 self.atol = 1e-8 @@ -45,8 +44,11 @@ class TestSplitsAPI(unittest.TestCase): self.shape = [4, 5, 2] self.num_or_sections = 2 self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -67,68 +69,79 @@ class TestSplitsAPI(unittest.TestCase): out = func(x, self.num_or_sections) out_ref = func_ref(func_type, self.x_np, self.num_or_sections) for n, p in zip(out_ref, out): - np.testing.assert_allclose(n, - p.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + n, p.numpy(), rtol=self.rtol, atol=self.atol + ) paddle.enable_static() class TestSplitsSections(TestSplitsAPI): """ - Test num_or_sections which is a list and date type is float64. + Test num_or_sections which is a list and date type is float64. """ def set_input(self): self.shape = [6, 2, 4] self.num_or_sections = [2, 1, 3] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSplitsFloat32(TestSplitsAPI): """ - Test num_or_sections which is an integer and data type is float32. + Test num_or_sections which is an integer and data type is float32. """ def set_input(self): self.shape = [2, 3, 4] self.num_or_sections = 2 self.x_np = np.random.uniform(-1, 1, self.shape).astype('float32') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSplitsInt32(TestSplitsAPI): """ - Test data type int32. + Test data type int32. """ def set_input(self): self.shape = [5, 1, 2] self.num_or_sections = 5 self.x_np = np.random.uniform(-1, 1, self.shape).astype('int32') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSplitsInt64(TestSplitsAPI): """ - Test data type int64. + Test data type int64. """ def set_input(self): self.shape = [4, 3, 2] self.num_or_sections = 2 self.x_np = np.random.uniform(-1, 1, self.shape).astype('int64') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSplitsCPU(TestSplitsAPI): """ - Test cpu place and num_or_sections which is a tuple. + Test cpu place and num_or_sections which is a tuple. """ def set_input(self): @@ -140,13 +153,16 @@ class TestSplitsCPU(TestSplitsAPI): class TestSplitsError(unittest.TestCase): """ - Test the situation that input shape less than 2. + Test the situation that input shape less than 2. """ def setUp(self): self.num_or_sections = 1 - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_error(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_spp_op.py b/python/paddle/fluid/tests/unittests/test_spp_op.py index 460abd59c42..776cde9b553 100644 --- a/python/paddle/fluid/tests/unittests/test_spp_op.py +++ b/python/paddle/fluid/tests/unittests/test_spp_op.py @@ -20,7 +20,6 @@ from test_pool2d_op import avg_pool2D_forward_naive class TestSppOp(OpTest): - def setUp(self): self.op_type = "spp" self.init_test_case() @@ -34,19 +33,25 @@ class TestSppOp(OpTest): bins = np.power(2, i) kernel_size = [0, 0] padding = [0, 0] - kernel_size[0] = np.ceil(hsize / - bins.astype("double")).astype("int32") - padding[0] = ((kernel_size[0] * bins - hsize + 1) / - 2).astype("int32") + kernel_size[0] = np.ceil(hsize / bins.astype("double")).astype( + "int32" + ) + padding[0] = ((kernel_size[0] * bins - hsize + 1) / 2).astype( + "int32" + ) - kernel_size[1] = np.ceil(wsize / - bins.astype("double")).astype("int32") - padding[1] = ((kernel_size[1] * bins - wsize + 1) / - 2).astype("int32") - out_level = self.pool2D_forward_naive(input, kernel_size, - kernel_size, padding) + kernel_size[1] = np.ceil(wsize / bins.astype("double")).astype( + "int32" + ) + padding[1] = ((kernel_size[1] * bins - wsize + 1) / 2).astype( + "int32" + ) + out_level = self.pool2D_forward_naive( + input, kernel_size, kernel_size, padding + ) out_level_flatten.append( - out_level.reshape(nsize, bins * bins * csize)) + out_level.reshape(nsize, bins * bins * csize) + ) if i == 0: output = out_level_flatten[i] else: @@ -57,7 +62,7 @@ class TestSppOp(OpTest): } self.attrs = { 'pyramid_height': self.pyramid_height, - 'pooling_type': self.pool_type + 'pooling_type': self.pool_type, } self.outputs = {'Out': output.astype('float64')} @@ -75,7 +80,6 @@ class TestSppOp(OpTest): class TestCase2(TestSppOp): - def init_test_case(self): self.shape = [3, 2, 16, 16] self.pyramid_height = 3 diff --git a/python/paddle/fluid/tests/unittests/test_square_error_cost.py b/python/paddle/fluid/tests/unittests/test_square_error_cost.py index 89001fe78b0..498eee8051e 100644 --- a/python/paddle/fluid/tests/unittests/test_square_error_cost.py +++ b/python/paddle/fluid/tests/unittests/test_square_error_cost.py @@ -21,7 +21,6 @@ from paddle.fluid.executor import Executor class TestSquareErrorCost(unittest.TestCase): - def test_square_error_cost(self): input_val = np.random.uniform(0.1, 0.5, (2, 3)).astype("float32") label_val = np.random.uniform(0.1, 0.5, (2, 3)).astype("float32") @@ -33,25 +32,23 @@ class TestSquareErrorCost(unittest.TestCase): label_var = layers.create_tensor(dtype="float32", name="label") output = layers.square_error_cost(input=input_var, label=label_var) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) - result, = exe.run(fluid.default_main_program(), - feed={ - "input": input_val, - "label": label_val - }, - fetch_list=[output]) + (result,) = exe.run( + fluid.default_main_program(), + feed={"input": input_val, "label": label_val}, + fetch_list=[output], + ) np.testing.assert_allclose(np_result, result, rtol=1e-05) class TestSquareErrorInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): input = [256, 3] label = fluid.data(name='label1', shape=[None, 3], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py b/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py index 6804d8a771e..21fdb592402 100644 --- a/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py @@ -18,18 +18,17 @@ from op_test import OpTest class TestSquaredL2DistanceOp_f0(OpTest): - def setUp(self): self.op_type = "squared_l2_distance" self.inputs = { 'X': np.random.uniform(0.1, 0.6, (5, 20)).astype("float32"), - 'Y': np.random.uniform(0.1, 0.6, (5, 20)).astype("float32") + 'Y': np.random.uniform(0.1, 0.6, (5, 20)).astype("float32"), } sub_res = self.inputs['X'] - self.inputs['Y'] output = sub_res * sub_res self.outputs = { 'sub_result': sub_res, - 'Out': np.expand_dims(output.sum(1), 1) + 'Out': np.expand_dims(output.sum(1), 1), } def test_check_output(self): @@ -40,18 +39,17 @@ class TestSquaredL2DistanceOp_f0(OpTest): class TestSquaredL2DistanceOp_f1(OpTest): - def setUp(self): self.op_type = "squared_l2_distance" self.inputs = { 'X': np.random.uniform(0.1, 0.6, (2, 3)).astype("float32"), - 'Y': np.random.uniform(0.1, 0.6, (1, 3)).astype("float32") + 'Y': np.random.uniform(0.1, 0.6, (1, 3)).astype("float32"), } sub_res = self.inputs['X'] - self.inputs['Y'] output = sub_res * sub_res self.outputs = { 'sub_result': sub_res, - 'Out': np.expand_dims(output.sum(1), 1) + 'Out': np.expand_dims(output.sum(1), 1), } def test_check_output(self): @@ -62,19 +60,18 @@ class TestSquaredL2DistanceOp_f1(OpTest): class TestSquaredL2DistanceOp_f2(OpTest): - def setUp(self): self.op_type = "squared_l2_distance" self.inputs = { 'X': np.random.uniform(0.1, 0.6, (2, 3, 4)).astype("float32"), - 'Y': np.random.uniform(0.1, 0.6, (1, 3, 4)).astype("float32") + 'Y': np.random.uniform(0.1, 0.6, (1, 3, 4)).astype("float32"), } sub_res = self.inputs['X'] - self.inputs['Y'] sub_res = sub_res.reshape((2, 3 * 4)) output = sub_res * sub_res self.outputs = { 'sub_result': sub_res, - 'Out': np.expand_dims(output.sum(1), 1) + 'Out': np.expand_dims(output.sum(1), 1), } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py b/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py index bc4dcfd1e28..8e09d83975a 100644 --- a/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py @@ -29,8 +29,7 @@ def test_squared_l2_norm(x): class TestL2LossOp(OpTest): - """Test squared_l2_norm - """ + """Test squared_l2_norm""" def setUp(self): self.python_api = test_squared_l2_norm @@ -46,14 +45,15 @@ class TestL2LossOp(OpTest): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - max_relative_error=self.max_relative_error, - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=self.max_relative_error, + check_eager=True, + ) class TestL2LossDeterministic(unittest.TestCase): - def check_place(self, place): with paddle.fluid.dygraph.guard(place): x_np = np.random.rand(5, 11, 13).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_squeeze2_op.py b/python/paddle/fluid/tests/unittests/test_squeeze2_op.py index 798afe70324..c64b58efea1 100755 --- a/python/paddle/fluid/tests/unittests/test_squeeze2_op.py +++ b/python/paddle/fluid/tests/unittests/test_squeeze2_op.py @@ -27,7 +27,6 @@ paddle.enable_static() # Correct: General. class TestSqueezeOp(OpTest): - def setUp(self): self.op_type = "squeeze2" self.python_api = paddle.squeeze @@ -39,7 +38,7 @@ class TestSqueezeOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float64") + "XShape": np.random.random(self.ori_shape).astype("float64"), } def test_check_output(self): @@ -59,7 +58,6 @@ class TestSqueezeOp(OpTest): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -68,7 +66,6 @@ class TestSqueezeOp1(TestSqueezeOp): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -77,7 +74,6 @@ class TestSqueezeOp2(TestSqueezeOp): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -85,7 +81,6 @@ class TestSqueezeOp3(TestSqueezeOp): class TestSqueeze2AxesTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, 'squeeze_tensor') @@ -122,7 +117,6 @@ class TestSqueeze2AxesTensor(UnittestBase): class TestSqueeze2AxesTensorList(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, 'squeeze_tensor') @@ -139,7 +133,7 @@ class TestSqueeze2AxesTensorList(UnittestBase): # axes is a list[Variable] axes = [ paddle.full([1], 0, dtype='int32'), - paddle.full([1], 2, dtype='int32') + paddle.full([1], 2, dtype='int32'), ] out = paddle.squeeze(feat, axes) out2 = paddle.fluid.layers.squeeze(feat, axes) diff --git a/python/paddle/fluid/tests/unittests/test_squeeze_op.py b/python/paddle/fluid/tests/unittests/test_squeeze_op.py index 53b5cb27253..e4ea13844b7 100755 --- a/python/paddle/fluid/tests/unittests/test_squeeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_squeeze_op.py @@ -30,7 +30,6 @@ paddle.enable_static() # Correct: General. class TestSqueezeOp(OpTest): - def setUp(self): self.op_type = "squeeze" self.init_test_case() @@ -56,7 +55,6 @@ class TestSqueezeOp(OpTest): class TestSqueezeBF16Op(OpTest): - def setUp(self): self.op_type = "squeeze" self.dtype = np.uint16 @@ -84,7 +82,6 @@ class TestSqueezeBF16Op(OpTest): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 3, 1, 40) self.axes = (0, -2) @@ -93,7 +90,6 @@ class TestSqueezeOp1(TestSqueezeOp): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -102,7 +98,6 @@ class TestSqueezeOp2(TestSqueezeOp): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -111,7 +106,6 @@ class TestSqueezeOp3(TestSqueezeOp): # Correct: The demension of axis is not of size 1 remains unchanged. class TestSqueezeOp4(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, 2) @@ -119,13 +113,13 @@ class TestSqueezeOp4(TestSqueezeOp): class TestSqueezeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input type of softmax_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.CPUPlace() + ) self.assertRaises(TypeError, paddle.squeeze, x1) # The input axes of squeeze must be list. x2 = paddle.static.data(name='x2', shape=[4], dtype="int32") @@ -136,7 +130,6 @@ class TestSqueezeOpError(unittest.TestCase): class API_TestSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -145,29 +138,29 @@ class API_TestSqueeze(unittest.TestCase): def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - data1 = paddle.static.data('data1', - shape=[-1, 1, 10], - dtype='float64') + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + data1 = paddle.static.data( + 'data1', shape=[-1, 1, 10], dtype='float64' + ) result_squeeze = self.squeeze(data1, axis=[1]) place = paddle.CPUPlace() exe = paddle.static.Executor(place) input1 = np.random.random([5, 1, 10]).astype('float64') - result, = exe.run(feed={"data1": input1}, - fetch_list=[result_squeeze]) + (result,) = exe.run( + feed={"data1": input1}, fetch_list=[result_squeeze] + ) expected_result = np.squeeze(input1, axis=1) np.testing.assert_allclose(expected_result, result, rtol=1e-05) class API_TestStaticSqueeze_(API_TestSqueeze): - def executed_api(self): self.squeeze = paddle.squeeze_ class API_TestDygraphSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -221,13 +214,11 @@ class API_TestDygraphSqueeze(unittest.TestCase): class API_TestDygraphSqueezeInplace(API_TestDygraphSqueeze): - def executed_api(self): self.squeeze = paddle.squeeze_ class TestSqueezeDoubleGradCheck(unittest.TestCase): - def squeeze_wrapper(self, x): return paddle.squeeze(x[0]) @@ -242,17 +233,13 @@ class TestSqueezeDoubleGradCheck(unittest.TestCase): out = paddle.squeeze(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.squeeze_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.squeeze_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -264,7 +251,6 @@ class TestSqueezeDoubleGradCheck(unittest.TestCase): class TestSqueezeTripleGradCheck(unittest.TestCase): - def squeeze_wrapper(self, x): return paddle.squeeze(x[0]) @@ -279,17 +265,13 @@ class TestSqueezeTripleGradCheck(unittest.TestCase): out = paddle.squeeze(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.squeeze_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.squeeze_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_stack_op.py b/python/paddle/fluid/tests/unittests/test_stack_op.py index 0677211aed9..0f71ebeac5b 100644 --- a/python/paddle/fluid/tests/unittests/test_stack_op.py +++ b/python/paddle/fluid/tests/unittests/test_stack_op.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestStackOpBase(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -47,7 +46,8 @@ class TestStackOpBase(OpTest): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(self.dtype)) + np.random.random(size=self.input_dim).astype(self.dtype) + ) tmp = [] x_names = self.get_x_names() @@ -66,49 +66,41 @@ class TestStackOpBase(OpTest): class TestStackOp1(TestStackOpBase): - def initParameters(self): self.num_inputs = 8 class TestStackOp2(TestStackOpBase): - def initParameters(self): self.num_inputs = 10 class TestStackOp3(TestStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestStackOpBase): - def initParameters(self): self.axis = -4 class TestStackOp5(TestStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestStackOpBase): - def initParameters(self): self.axis = 3 class TestStackOp_ZeroDim(TestStackOpBase): - def initParameters(self): self.input_dim = () class TestStackBF16Op(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -132,7 +124,8 @@ class TestStackBF16Op(OpTest): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(np.float32)) + np.random.random(size=self.input_dim).astype(np.float32) + ) out = np.stack(self.x, axis=self.axis) @@ -162,8 +155,11 @@ class TestStackAPIWithLoDTensorArray(unittest.TestCase): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = fluid.CUDAPlace(0) \ - if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_program() def set_program(self): @@ -183,7 +179,8 @@ class TestStackAPIWithLoDTensorArray(unittest.TestCase): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_array_equal( - res[0], np.stack([self.x] * self.iter_num, axis=self.axis)) + res[0], np.stack([self.x] * self.iter_num, axis=self.axis) + ) class TestTensorStackAPIWithLoDTensorArray(unittest.TestCase): @@ -196,8 +193,11 @@ class TestTensorStackAPIWithLoDTensorArray(unittest.TestCase): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = fluid.CUDAPlace(0) \ - if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_program() def set_program(self): @@ -217,11 +217,11 @@ class TestTensorStackAPIWithLoDTensorArray(unittest.TestCase): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_array_equal( - res[0], np.stack([self.x] * self.iter_num, axis=self.axis)) + res[0], np.stack([self.x] * self.iter_num, axis=self.axis) + ) class API_test(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[1, 2], dtype='float64') @@ -233,12 +233,10 @@ class API_test(unittest.TestCase): input1 = np.random.random([1, 2]).astype('float64') input2 = np.random.random([1, 2]).astype('float64') input3 = np.random.random([1, 2]).astype('float64') - result, = exe.run(feed={ - "data1": input1, - "data2": input2, - "data3": input3 - }, - fetch_list=[result_stack]) + (result,) = exe.run( + feed={"data1": input1, "data2": input2, "data3": input3}, + fetch_list=[result_stack], + ) expected_result = np.stack([input1, input2, input3], axis=0) np.testing.assert_allclose(expected_result, result, rtol=1e-05) @@ -249,7 +247,6 @@ class API_test(unittest.TestCase): class API_DygraphTest(unittest.TestCase): - def test_out(self): data1 = np.array([[1.0, 2.0]]) data2 = np.array([[3.0, 4.0]]) @@ -277,7 +274,6 @@ class API_DygraphTest(unittest.TestCase): class TestStackOpWithNegativeShape(unittest.TestCase): - def test_out(self): main_prg, startup_prg = Program(), Program() with program_guard(main_prg, startup_prg): @@ -286,23 +282,28 @@ class TestStackOpWithNegativeShape(unittest.TestCase): k = paddle.stack([b, e], axis=0) exe = paddle.static.Executor() exe.run(startup_prg) - out = exe.run(main_prg, - feed={ - 'b': np.ones([ - 3, - ]).astype("int64"), - 'e': np.zeros([ - 3, - ]).astype("int64") - }, - fetch_list=[k]) - np.testing.assert_allclose(out[0], - np.array([[1, 1, 1], [0, 0, 0]]), - rtol=1e-05) + out = exe.run( + main_prg, + feed={ + 'b': np.ones( + [ + 3, + ] + ).astype("int64"), + 'e': np.zeros( + [ + 3, + ] + ).astype("int64"), + }, + fetch_list=[k], + ) + np.testing.assert_allclose( + out[0], np.array([[1, 1, 1], [0, 0, 0]]), rtol=1e-05 + ) class TestStackAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_attention.py b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_attention.py index 85b9edd2e33..24ee5a36775 100644 --- a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_attention.py +++ b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_attention.py @@ -23,7 +23,6 @@ flag_name = os.path.splitext(__file__)[0] class TestStaticModelParallel(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,11 +32,14 @@ class TestStaticModelParallel(TestDistBase): def test_dist_static_model_parallel_fused_feedforward(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_fused_attention.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_fused_attention.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_feedforward.py b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_feedforward.py index d2779eea749..c6153f2be0b 100644 --- a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_feedforward.py +++ b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_feedforward.py @@ -23,7 +23,6 @@ flag_name = os.path.splitext(__file__)[0] class TestStaticModelParallel(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,11 +32,14 @@ class TestStaticModelParallel(TestDistBase): def test_dist_static_model_parallel_fused_feedforward(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_fused_feedforward.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_fused_feedforward.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py index 211920cfebc..d34ced7b4f4 100644 --- a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py @@ -23,7 +23,6 @@ flag_name = os.path.splitext(__file__)[0] class TestStaticModelParallel(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,12 +32,14 @@ class TestStaticModelParallel(TestDistBase): def test_dist_static_model_parallel_fused_multi_transformer(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place( "static_model_parallel_fused_multi_transformer.py", delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index 562477a0552..a0cd268f484 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -31,14 +31,15 @@ paddle.enable_static() class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - name_scope, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, + name_scope, + hidden_size, + num_steps, + num_layers=2, + init_scale=0.1, + dropout=None, + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -58,19 +59,26 @@ class SimpleLSTMRNN(fluid.Layer): weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -78,29 +86,29 @@ class SimpleLSTMRNN(fluid.Layer): self.hidden_array = [] for i in range(self._num_layers): - pre_hidden = fluid.layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = fluid.layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = fluid.layers.reshape(pre_hidden, - shape=[-1, self._hidden_size]) - pre_cell = fluid.layers.reshape(pre_cell, - shape=[-1, self._hidden_size]) + pre_hidden = fluid.layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = fluid.layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = fluid.layers.reshape( + pre_hidden, shape=[-1, self._hidden_size] + ) + pre_cell = fluid.layers.reshape( + pre_cell, shape=[-1, self._hidden_size] + ) self.hidden_array.append(pre_hidden) self.cell_array.append(pre_cell) res = [] for index in range(self._num_steps): - self._input = fluid.layers.slice(input_embedding, - axes=[1], - starts=[index], - ends=[index + 1]) - self._input = fluid.layers.reshape(self._input, - shape=[-1, self._hidden_size]) + self._input = fluid.layers.slice( + input_embedding, axes=[1], starts=[index], ends=[index + 1] + ) + self._input = fluid.layers.reshape( + self._input, shape=[-1, self._hidden_size] + ) for k in range(self._num_layers): pre_hidden = self.hidden_array[k] pre_cell = self.cell_array[k] @@ -111,11 +119,12 @@ class SimpleLSTMRNN(fluid.Layer): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) self.hidden_array[k] = m self.cell_array[k] = c @@ -125,33 +134,39 @@ class SimpleLSTMRNN(fluid.Layer): self._input = fluid.layers.dropout( self._input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append( - fluid.layers.reshape(self._input, - shape=[1, -1, self._hidden_size])) + fluid.layers.reshape( + self._input, shape=[1, -1, self._hidden_size] + ) + ) real_res = fluid.layers.concat(res, 0) real_res = fluid.layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = fluid.layers.concat(self.hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(self.cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - name_scope, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + name_scope, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -159,61 +174,77 @@ class PtbModel(fluid.Layer): self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(self.full_name(), - hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + self.full_name(), + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = paddle.nn.Embedding( num_embeddings=vocab_size, embedding_dim=hidden_size, weight_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) # NPU 'tok_k' kernel only support `int32` dtype, so cast `input` from `int64` to `int32`. input = fluid.layers.cast(input, "int32") x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.drop_out, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) rnn_out = fluid.layers.reshape( - rnn_out, shape=[-1, self.num_steps, self.hidden_size]) + rnn_out, shape=[-1, self.num_steps, self.hidden_size] + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -222,10 +253,12 @@ class PtbModel(fluid.Layer): class TestSaveLoadBase(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -241,29 +274,32 @@ class TestSaveLoadBase(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -279,18 +315,22 @@ class TestSaveLoadBase(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -300,8 +340,9 @@ class TestSaveLoadBase(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -314,28 +355,35 @@ class TestSaveLoadBase(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, - os.path.join(temp_dir.name, "test_1.pdparams"), exe) + fluid.load( + main_program, + os.path.join(temp_dir.name, "test_1.pdparams"), + exe, + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) temp_dir.cleanup() class TestSaveLoadPartial(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -351,36 +399,41 @@ class TestSaveLoadPartial(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_program = fluid.default_main_program().clone(for_test=True) - add_1 = fluid.layers.fc(static_last_hidden, - size=hidden_size, - num_flatten_dims=2, - bias_attr=False) + add_1 = fluid.layers.fc( + static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False, + ) sgd.minimize(static_loss) static_param_updated = dict() @@ -397,18 +450,22 @@ class TestSaveLoadPartial(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -418,8 +475,9 @@ class TestSaveLoadPartial(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -432,30 +490,38 @@ class TestSaveLoadPartial(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(test_program, os.path.join(temp_dir.name, - "test_1.pdopt"), None) + fluid.load( + test_program, os.path.join(temp_dir.name, "test_1.pdopt"), None + ) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) - fluid.load(test_program, - os.path.join(temp_dir.name, "test_1.pdmodel"), None) + fluid.load( + test_program, + os.path.join(temp_dir.name, "test_1.pdmodel"), + None, + ) temp_dir.cleanup() class TestSaveLoadSetStateDict(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -471,29 +537,32 @@ class TestSaveLoadSetStateDict(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -509,18 +578,22 @@ class TestSaveLoadSetStateDict(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -530,8 +603,9 @@ class TestSaveLoadSetStateDict(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -544,8 +618,9 @@ class TestSaveLoadSetStateDict(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -553,18 +628,21 @@ class TestSaveLoadSetStateDict(unittest.TestCase): for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) temp_dir.cleanup() class TestProgramStatePartial(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -580,36 +658,41 @@ class TestProgramStatePartial(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_program = fluid.default_main_program().clone(for_test=True) - add_1 = fluid.layers.fc(static_last_hidden, - size=hidden_size, - num_flatten_dims=2, - bias_attr=False) + add_1 = fluid.layers.fc( + static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False, + ) sgd.minimize(static_loss) static_param_updated = dict() @@ -626,18 +709,22 @@ class TestProgramStatePartial(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -647,8 +734,9 @@ class TestProgramStatePartial(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -661,30 +749,36 @@ class TestProgramStatePartial(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - #fluid.load(test_program, "./test_1", None ) + # fluid.load(test_program, "./test_1", None ) program_state = fluid.load_program_state( - os.path.join(temp_dir.name, 'test_1')) + os.path.join(temp_dir.name, 'test_1') + ) program_state_1 = fluid.load_program_state( - os.path.join(temp_dir.name, 'test_1.pdparams')) + os.path.join(temp_dir.name, 'test_1.pdparams') + ) program_state_2 = fluid.load_program_state( - os.path.join(temp_dir.name, 'test_1.pdopt')) + os.path.join(temp_dir.name, 'test_1.pdopt') + ) program_state_3 = fluid.load_program_state( - os.path.join(temp_dir.name, 'test_1.pdmodel')) + os.path.join(temp_dir.name, 'test_1.pdmodel') + ) fluid.set_program_state(test_program, program_state) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -694,8 +788,9 @@ class TestProgramStatePartial(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -703,8 +798,9 @@ class TestProgramStatePartial(unittest.TestCase): for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -714,8 +810,9 @@ class TestProgramStatePartial(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -723,8 +820,9 @@ class TestProgramStatePartial(unittest.TestCase): for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -734,8 +832,9 @@ class TestProgramStatePartial(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -743,18 +842,21 @@ class TestProgramStatePartial(unittest.TestCase): for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) temp_dir.cleanup() class TestVariableInit(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_variable_init(self): @@ -767,8 +869,10 @@ class TestVariableInit(unittest.TestCase): exe.run(fluid.default_startup_program()) temp_dir = tempfile.TemporaryDirectory() - fluid.save(fluid.default_main_program(), - os.path.join(temp_dir.name, "test_path")) + fluid.save( + fluid.default_main_program(), + os.path.join(temp_dir.name, "test_path"), + ) def set_var(var, ndarray): t = var.get_tensor() @@ -789,35 +893,43 @@ class TestVariableInit(unittest.TestCase): place = self.set_place() exe = fluid.Executor(place) - parameter_list = list(filter(fluid.io.is_parameter, - program.list_vars())) + parameter_list = list( + filter(fluid.io.is_parameter, program.list_vars()) + ) - fluid.core._create_loaded_parameter(parameter_list, new_scope, - exe._default_executor) + fluid.core._create_loaded_parameter( + parameter_list, new_scope, exe._default_executor + ) parameter_file_name = os.path.join(temp_dir.name, "test_path.pdparams") with open(parameter_file_name, 'rb') as f: load_dict = pickle.load(f) for v in parameter_list: - assert v.name in load_dict, \ - "Can not find [{}] in model file [{}]".format( - v.name, parameter_file_name) + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, parameter_file_name + ) new_v = new_scope.find_var(v.name) set_var(new_v, load_dict[v.name]) opt_list = list( - filter(fluid.io.is_belong_to_optimizer, program.list_vars())) + filter(fluid.io.is_belong_to_optimizer, program.list_vars()) + ) - fluid.core._create_loaded_parameter(opt_list, new_scope, - exe._default_executor) + fluid.core._create_loaded_parameter( + opt_list, new_scope, exe._default_executor + ) opt_file_name = os.path.join(temp_dir.name, "test_path.pdopt") with open(opt_file_name, 'rb') as f: load_dict = pickle.load(f) for v in opt_list: - assert v.name in load_dict, \ - "Can not find [{}] in model file [{}]".format( - v.name, opt_file_name) + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, opt_file_name + ) new_v = new_scope.find_var(v.name) set_var(new_v, load_dict[v.name]) @@ -825,8 +937,9 @@ class TestVariableInit(unittest.TestCase): base_map = {} for var in program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update base_map[var.name] = t @@ -840,7 +953,6 @@ class TestVariableInit(unittest.TestCase): class TestLoadFromOldInterface(unittest.TestCase): - def setUp(self): if os.path.exists("test_path.pdparams"): os.remove("test_path.pdparams") @@ -851,8 +963,11 @@ class TestLoadFromOldInterface(unittest.TestCase): self.temp_dir = tempfile.TemporaryDirectory() def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def tearDown(self): self.temp_dir.cleanup() @@ -870,29 +985,32 @@ class TestLoadFromOldInterface(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_clone_program = fluid.default_main_program().clone() sgd.minimize(static_loss) @@ -910,18 +1028,22 @@ class TestLoadFromOldInterface(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -931,16 +1053,17 @@ class TestLoadFromOldInterface(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - #fluid.save(main_program, "./test_1") + # fluid.save(main_program, "./test_1") fluid.io.save_persistables( - exe, os.path.join(self.temp_dir.name, "test_path"), - main_program) + exe, os.path.join(self.temp_dir.name, "test_path"), main_program + ) # set var to zero for var in main_program.list_vars(): @@ -948,18 +1071,21 @@ class TestLoadFromOldInterface(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, - os.path.join(self.temp_dir.name, "test_path"), exe) + fluid.load( + main_program, os.path.join(self.temp_dir.name, "test_path"), exe + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -971,13 +1097,19 @@ class TestLoadFromOldInterface(unittest.TestCase): var.desc.set_shape(new_shape) with self.assertRaises(RuntimeError): - fluid.load(main_program, - os.path.join(self.temp_dir.name, "test_path"), exe) + fluid.load( + main_program, + os.path.join(self.temp_dir.name, "test_path"), + exe, + ) # check unused parameter - fluid.load(test_clone_program, - os.path.join(self.temp_dir.name, "test_path"), exe) + fluid.load( + test_clone_program, + os.path.join(self.temp_dir.name, "test_path"), + exe, + ) def test_load_from_old_interface_var_list(self): seed = 90 @@ -992,29 +1124,32 @@ class TestLoadFromOldInterface(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_clone_program = fluid.default_main_program().clone() sgd.minimize(static_loss) @@ -1032,18 +1167,22 @@ class TestLoadFromOldInterface(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -1053,17 +1192,19 @@ class TestLoadFromOldInterface(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - #fluid.save(main_program, "./test_1") + # fluid.save(main_program, "./test_1") fluid.io.save_persistables( exe, os.path.join(self.temp_dir.name, "test_static_load_var_list"), - main_program) + main_program, + ) # set var to zero var_list = [] @@ -1074,34 +1215,40 @@ class TestLoadFromOldInterface(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) fluid.load( main_program, os.path.join(self.temp_dir.name, "test_static_load_var_list"), - exe, var_list) + exe, + var_list, + ) var_list_names = [var.name for var in var_list] for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) if var.name in var_list_names: # loaded vars base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) else: - #not loaded vars + # not loaded vars self.assertTrue(np.sum(np.abs(new_t)) == 0) class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_load_from_old_interface(self): seed = 90 @@ -1117,29 +1264,32 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -1155,18 +1305,22 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -1176,17 +1330,17 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t save_dir = os.path.join(temp_dir.name, "test_path") - #fluid.save(main_program, "./test_1") - fluid.io.save_persistables(exe, - save_dir, - main_program, - filename="model_single") + # fluid.save(main_program, "./test_1") + fluid.io.save_persistables( + exe, save_dir, main_program, filename="model_single" + ) # set var to zero for var in main_program.list_vars(): @@ -1194,19 +1348,25 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) file_model_path = os.path.join(save_dir, "model_single") - fluid.load(main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program)) + fluid.load( + main_program, + file_model_path, + exe, + fluid.io.get_program_persistable_vars(main_program), + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -1221,21 +1381,32 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): var.desc.set_shape(new_shape) with self.assertRaises(RuntimeError): - fluid.load(main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program)) - - fluid.io.save_params(exe, - "test_path", - main_program, - filename="model_single") + fluid.load( + main_program, + file_model_path, + exe, + fluid.io.get_program_persistable_vars(main_program), + ) + + fluid.io.save_params( + exe, "test_path", main_program, filename="model_single" + ) with self.assertRaises(RuntimeError): - fluid.load(main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program)) + fluid.load( + main_program, + file_model_path, + exe, + fluid.io.get_program_persistable_vars(main_program), + ) # check when executor is None with self.assertRaises(ValueError): - fluid.load(main_program, file_model_path, None, - fluid.io.get_program_persistable_vars(main_program)) + fluid.load( + main_program, + file_model_path, + None, + fluid.io.get_program_persistable_vars(main_program), + ) # check when var list is None with self.assertRaises(ValueError): @@ -1243,17 +1414,20 @@ class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): # check save params, load var_list = get_program_persistable_vars with self.assertRaises(RuntimeError): - temp_var = framework.Variable(main_program.global_block(), - shape=[1], - name="test_temp_var") + temp_var = framework.Variable( + main_program.global_block(), shape=[1], name="test_temp_var" + ) all_var_list = list(main_program.list_vars()) - fluid.load(main_program, file_model_path, exe, - all_var_list + [temp_var]) + fluid.load( + main_program, + file_model_path, + exe, + all_var_list + [temp_var], + ) temp_dir.cleanup() class TestProgramStateOldSave(unittest.TestCase): - def setUp(self): self.test_dygraph = True self.temp_dir = tempfile.TemporaryDirectory() @@ -1262,8 +1436,11 @@ class TestProgramStateOldSave(unittest.TestCase): self.temp_dir.cleanup() def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -1278,36 +1455,41 @@ class TestProgramStateOldSave(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_program = fluid.default_main_program().clone(for_test=True) - add_1 = fluid.layers.fc(static_last_hidden, - size=hidden_size, - num_flatten_dims=2, - bias_attr=False) + add_1 = fluid.layers.fc( + static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False, + ) sgd.minimize(static_loss) static_param_updated = dict() @@ -1324,18 +1506,22 @@ class TestProgramStateOldSave(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -1345,8 +1531,9 @@ class TestProgramStateOldSave(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -1359,8 +1546,9 @@ class TestProgramStateOldSave(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -1386,7 +1574,8 @@ class TestProgramStateOldSave(unittest.TestCase): # case 3: load with var_list program_state = fluid.load_program_state( - save_dir, main_program.all_parameters()) + save_dir, main_program.all_parameters() + ) fluid.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) @@ -1402,23 +1591,27 @@ class TestProgramStateOldSave(unittest.TestCase): os.symlink(target, link_name) except AttributeError: import ctypes + kernel_dll = ctypes.windll.LoadLibrary("kernel32.dll") kernel_dll.CreateSymbolicLinkA(target, link_name, 0) def check_in_static(self, main_program, base_map): for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) class TestProgramStateOldSaveSingleModel(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -1434,36 +1627,41 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_program = fluid.default_main_program().clone(for_test=True) - add_1 = fluid.layers.fc(static_last_hidden, - size=hidden_size, - num_flatten_dims=2, - bias_attr=False) + add_1 = fluid.layers.fc( + static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False, + ) sgd.minimize(static_loss) static_param_updated = dict() @@ -1480,18 +1678,22 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -1501,17 +1703,17 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t save_dir = os.path.join(temp_dir.name, "test_program_2") - fluid.io.save_persistables(exe, - save_dir, - main_program, - filename="model_1") + fluid.io.save_persistables( + exe, save_dir, main_program, filename="model_1" + ) # set var to zero for var in main_program.list_vars(): @@ -1519,21 +1721,24 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - #fluid.load(test_program, "./test_1", None ) + # fluid.load(test_program, "./test_1", None ) program_state = fluid.load_program_state( os.path.join(save_dir, "model_1"), - var_list=fluid.io.get_program_persistable_vars(main_program)) + var_list=fluid.io.get_program_persistable_vars(main_program), + ) fluid.set_program_state(main_program, program_state) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -1541,30 +1746,34 @@ class TestProgramStateOldSaveSingleModel(unittest.TestCase): fluid.load_program_state(os.path.join(save_dir, "model_1")) with self.assertRaises(TypeError): - fluid.load_program_state(os.path.join(save_dir, "model_1"), - var_list=["str"]) + fluid.load_program_state( + os.path.join(save_dir, "model_1"), var_list=["str"] + ) with self.assertRaises(RuntimeError): fluid.load_program_state( os.path.join(save_dir, "model_1"), var_list=[ main_program.global_block().create_var( - name="fake_var_name", persistable=True) - ]) + name="fake_var_name", persistable=True + ) + ], + ) temp_dir.cleanup() class TestStaticSaveLoadPickle(unittest.TestCase): - def test_pickle_protocol(self): # enable static mode paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="static_save_load_large_x", - shape=[None, 10], - dtype='float32') + x = paddle.static.data( + name="static_save_load_large_x", + shape=[None, 10], + dtype='float32', + ) z = paddle.static.nn.fc(x, 10, bias_attr=False) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -1574,15 +1783,17 @@ class TestStaticSaveLoadPickle(unittest.TestCase): base_map = {} for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t temp_dir = tempfile.TemporaryDirectory() - path = os.path.join(temp_dir.name, "test_static_save_load_pickle", - "pickle_protocol") + path = os.path.join( + temp_dir.name, "test_static_save_load_pickle", "pickle_protocol" + ) with self.assertRaises(ValueError): paddle.fluid.save(prog, path, 2.0) @@ -1603,26 +1814,28 @@ class TestStaticSaveLoadPickle(unittest.TestCase): # set var to zero for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - ten = fluid.global_scope().find_var( - var.name).get_tensor() + ten = ( + fluid.global_scope().find_var(var.name).get_tensor() + ) ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.fluid.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) class TestSaveLoadInferenceModel(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_path = os.path.join(self.temp_dir.name, 'no_params') @@ -1641,8 +1854,11 @@ class TestSaveLoadInferenceModel(unittest.TestCase): paddle.static.save_inference_model(self.model_path, [x], [y], exe) - [inference_program, feed_target_names, fetch_targets - ] = (paddle.static.load_inference_model(self.model_path, exe)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(self.model_path, exe) self.assertEqual(feed_target_names, ['x']) self.assertEqual(fetch_targets[0].shape, (10, 10)) diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py index 3e3f41e58d2..fe578907b86 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py @@ -25,10 +25,10 @@ import tempfile import os -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestSaveLoadBF16(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -51,36 +51,41 @@ class TestSaveLoadBF16(unittest.TestCase): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd = paddle.static.amp.bf16.decorate_bf16( sgd, amp_lists=paddle.static.amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'transpose2', 'concat'}), + custom_fp32_list={'transpose2', 'concat'} + ), use_bf16_guard=False, - use_pure_bf16=True) + use_pure_bf16=True, + ) sgd.minimize(static_loss, framework.default_startup_program()) out = exe.run(framework.default_startup_program()) @@ -90,30 +95,35 @@ class TestSaveLoadBF16(unittest.TestCase): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) - #TODO investigate initializing model with "float32" instead of "uint16" as it was before + # TODO investigate initializing model with "float32" instead of "uint16" as it was before # slice_op PR(datatypes in model graph are different than datatypes during runtime because of that) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='uint16') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='uint16') + (num_layers, batch_size, hidden_size), dtype='uint16' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='uint16' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) # get value before save main_program = framework.default_main_program() base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -126,18 +136,23 @@ class TestSaveLoadBF16(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, - os.path.join(self.temp_dir.name, "test_1.pdparams"), exe) + fluid.load( + main_program, + os.path.join(self.temp_dir.name, "test_1.pdparams"), + exe, + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py index 61568b50c1b..ddae3373b5a 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py @@ -26,15 +26,16 @@ LARGE_PARAM = 2**26 class TestStaticSaveLoadLargeParameters(unittest.TestCase): - def test_large_parameters_static_save(self): # enable static mode paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="static_save_load_large_x", - shape=[None, 10], - dtype='float32') + x = paddle.static.data( + name="static_save_load_large_x", + shape=[None, 10], + dtype='float32', + ) z = paddle.static.nn.fc(x, LARGE_PARAM, bias_attr=False) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -44,14 +45,16 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): base_map = {} for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t temp_dir = tempfile.TemporaryDirectory() - path = os.path.join(temp_dir.name, - "test_static_save_load_large_param") + path = os.path.join( + temp_dir.name, "test_static_save_load_large_param" + ) path = os.path.join(path, "static_save") protocol = 4 paddle.fluid.save(prog, path, pickle_protocol=protocol) @@ -61,16 +64,18 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.fluid.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -80,16 +85,18 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) self.assertTrue(np.sum(np.abs(new_t)) == 0) program_state = fluid.load_program_state(path) fluid.set_program_state(prog, program_state) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_static_shape_inferrence_for_shape_tensor.py b/python/paddle/fluid/tests/unittests/test_static_shape_inferrence_for_shape_tensor.py index 0e22905e81d..c6fd490cb60 100644 --- a/python/paddle/fluid/tests/unittests/test_static_shape_inferrence_for_shape_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_static_shape_inferrence_for_shape_tensor.py @@ -17,12 +17,11 @@ import unittest class StaticShapeInferrenceTest(unittest.TestCase): - def test_static_graph(self): paddle.enable_static() - data = paddle.fluid.layers.data(name="x", - shape=[-1, 2], - dtype='float32') + data = paddle.fluid.layers.data( + name="x", shape=[-1, 2], dtype='float32' + ) shape = paddle.fluid.layers.shape(data) # shape should be [-1, 2] x = paddle.fluid.layers.uniform_random(shape) self.assertEqual(x.shape, data.shape) diff --git a/python/paddle/fluid/tests/unittests/test_std_layer.py b/python/paddle/fluid/tests/unittests/test_std_layer.py index a724e320c95..cc3e7740f45 100644 --- a/python/paddle/fluid/tests/unittests/test_std_layer.py +++ b/python/paddle/fluid/tests/unittests/test_std_layer.py @@ -20,14 +20,13 @@ import paddle def ref_std(x, axis=None, unbiased=True, keepdim=False): ddof = 1 if unbiased else 0 if isinstance(axis, int): - axis = (axis, ) + axis = (axis,) if axis is not None: axis = tuple(axis) return np.std(x, axis=axis, ddof=ddof, keepdims=keepdim) class TestStdAPI(unittest.TestCase): - def setUp(self): self.dtype = 'float64' self.shape = [1, 3, 4, 10] @@ -36,9 +35,11 @@ class TestStdAPI(unittest.TestCase): self.unbiased = True self.set_attrs() self.x = np.random.uniform(-1, 1, self.shape).astype(self.dtype) - self.place=paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def set_attrs(self): pass @@ -68,43 +69,36 @@ class TestStdAPI(unittest.TestCase): class TestStdAPI_dtype(TestStdAPI): - def set_attrs(self): self.dtype = 'float32' class TestStdAPI_axis_int(TestStdAPI): - def set_attrs(self): self.axis = 2 class TestStdAPI_axis_list(TestStdAPI): - def set_attrs(self): self.axis = [1, 2] class TestStdAPI_axis_tuple(TestStdAPI): - def set_attrs(self): self.axis = (1, 3) class TestStdAPI_keepdim(TestStdAPI): - def set_attrs(self): self.keepdim = False class TestStdAPI_unbiased(TestStdAPI): - def set_attrs(self): self.unbiased = False class TestStdAPI_alias(unittest.TestCase): - def test_alias(self): paddle.disable_static() x = paddle.to_tensor(np.array([10, 12], 'float32')) @@ -117,7 +111,6 @@ class TestStdAPI_alias(unittest.TestCase): class TestStdError(unittest.TestCase): - def test_error(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('X', [2, 3, 4], 'int32') diff --git a/python/paddle/fluid/tests/unittests/test_stft_op.py b/python/paddle/fluid/tests/unittests/test_stft_op.py index 8110f1d805f..91206c3da12 100644 --- a/python/paddle/fluid/tests/unittests/test_stft_op.py +++ b/python/paddle/fluid/tests/unittests/test_stft_op.py @@ -45,14 +45,14 @@ def frame_from_librosa(x, frame_length, hop_length, axis=-1): def stft_np(x, window, n_fft, hop_length, **kwargs): frames = frame_from_librosa(x, n_fft, hop_length) - frames = np.multiply(frames.transpose([0, 2, 1]), - window).transpose([0, 2, 1]) + frames = np.multiply(frames.transpose([0, 2, 1]), window).transpose( + [0, 2, 1] + ) res = np.fft.rfft(frames, axis=1) return res class TestStftOp(OpTest): - def setUp(self): self.op_type = "stft" self.shape, self.type, self.attrs = self.initTestCase() @@ -61,10 +61,9 @@ class TestStftOp(OpTest): 'Window': np.hamming(self.attrs['n_fft']).astype(self.type), } self.outputs = { - 'Out': - stft_np(x=self.inputs['X'], - window=self.inputs['Window'], - **self.attrs) + 'Out': stft_np( + x=self.inputs['X'], window=self.inputs['Window'], **self.attrs + ) } def initTestCase(self): diff --git a/python/paddle/fluid/tests/unittests/test_strided_slice_op.py b/python/paddle/fluid/tests/unittests/test_strided_slice_op.py index 05f4e4a9941..7ce7db1ba78 100644 --- a/python/paddle/fluid/tests/unittests/test_strided_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_strided_slice_op.py @@ -37,32 +37,51 @@ def strided_slice_native_forward(input, axes, starts, ends, strides): stride[axes[i]] = strides[i] result = { - 1: lambda input, start, end, stride: input[start[0]:end[0]:stride[0]], - 2: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1]], - 3: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2]], - 4: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3]], - 5: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3], start[4]:end[4]:stride[4]], - 6: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3], \ - start[4]:end[4]:stride[4], start[5]:end[5]:stride[5]] + 1: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0] + ], + 2: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], start[1] : end[1] : stride[1] + ], + 3: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + ], + 4: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + ], + 5: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + start[4] : end[4] : stride[4], + ], + 6: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + start[4] : end[4] : stride[4], + start[5] : end[5] : stride[5], + ], }[dim](input, start, end, stride) return result class TestStrideSliceOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'strided_slice' self.python_api = paddle.strided_slice - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.inputs = {'Input': self.input} self.outputs = {'Out': self.output} @@ -71,7 +90,7 @@ class TestStrideSliceOp(OpTest): 'starts': self.starts, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def test_check_output(self): @@ -90,7 +109,6 @@ class TestStrideSliceOp(OpTest): class TestStrideSliceOp1(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -101,7 +119,6 @@ class TestStrideSliceOp1(TestStrideSliceOp): class TestStrideSliceOp2(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -112,7 +129,6 @@ class TestStrideSliceOp2(TestStrideSliceOp): class TestStrideSliceOp3(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -123,7 +139,6 @@ class TestStrideSliceOp3(TestStrideSliceOp): class TestStrideSliceOp4(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 4, 10) self.axes = [0, 1, 2] @@ -134,7 +149,6 @@ class TestStrideSliceOp4(TestStrideSliceOp): class TestStrideSliceOp5(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -145,7 +159,6 @@ class TestStrideSliceOp5(TestStrideSliceOp): class TestStrideSliceOp6(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -156,7 +169,6 @@ class TestStrideSliceOp6(TestStrideSliceOp): class TestStrideSliceOp7(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -167,7 +179,6 @@ class TestStrideSliceOp7(TestStrideSliceOp): class TestStrideSliceOp8(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1) self.axes = [1] @@ -178,7 +189,6 @@ class TestStrideSliceOp8(TestStrideSliceOp): class TestStrideSliceOp9(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1) self.axes = [1] @@ -189,7 +199,6 @@ class TestStrideSliceOp9(TestStrideSliceOp): class TestStrideSliceOp10(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(10, 10) self.axes = [0, 1] @@ -200,7 +209,6 @@ class TestStrideSliceOp10(TestStrideSliceOp): class TestStrideSliceOp11(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4) self.axes = [0, 1, 2, 3] @@ -211,7 +219,6 @@ class TestStrideSliceOp11(TestStrideSliceOp): class TestStrideSliceOp12(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5) self.axes = [0, 1, 2, 3, 4] @@ -222,7 +229,6 @@ class TestStrideSliceOp12(TestStrideSliceOp): class TestStrideSliceOp13(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8) self.axes = [0, 1, 2, 3, 4, 5] @@ -233,7 +239,6 @@ class TestStrideSliceOp13(TestStrideSliceOp): class TestStrideSliceOp14(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(4, 4, 4, 4) self.axes = [1, 2, 3] @@ -244,13 +249,11 @@ class TestStrideSliceOp14(TestStrideSliceOp): class TestStrideSliceOpBool(TestStrideSliceOp): - def test_check_grad(self): pass class TestStrideSliceOpBool1D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(100).astype("bool") self.axes = [0] @@ -261,7 +264,6 @@ class TestStrideSliceOpBool1D(TestStrideSliceOpBool): class TestStrideSliceOpBool2D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(10, 10).astype("bool") self.axes = [0, 1] @@ -272,7 +274,6 @@ class TestStrideSliceOpBool2D(TestStrideSliceOpBool): class TestStrideSliceOpBool3D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 4, 10).astype("bool") self.axes = [0, 1, 2] @@ -283,7 +284,6 @@ class TestStrideSliceOpBool3D(TestStrideSliceOpBool): class TestStrideSliceOpBool4D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4).astype("bool") self.axes = [0, 1, 2, 3] @@ -294,7 +294,6 @@ class TestStrideSliceOpBool4D(TestStrideSliceOpBool): class TestStrideSliceOpBool5D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5).astype("bool") self.axes = [0, 1, 2, 3, 4] @@ -305,7 +304,6 @@ class TestStrideSliceOpBool5D(TestStrideSliceOpBool): class TestStrideSliceOpBool6D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype("bool") self.axes = [0, 1, 2, 3, 4, 5] @@ -316,15 +314,15 @@ class TestStrideSliceOpBool6D(TestStrideSliceOpBool): class TestStridedSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.output} @@ -333,7 +331,7 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): 'starts': self.starts_infer, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -343,9 +341,9 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.starts_infer = [1, 10, 2] @@ -357,15 +355,15 @@ class TestStridedSliceOp_starts_ListTensor(OpTest): class TestStridedSliceOp_ends_ListTensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'EndsTensorList': ends_tensor} self.outputs = {'Out': self.output} @@ -374,7 +372,7 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): 'starts': self.starts, 'ends': self.ends_infer, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -384,9 +382,9 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 2] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.ends_infer = [3, 1, 4] @@ -398,13 +396,12 @@ class TestStridedSliceOp_ends_ListTensor(OpTest): class TestStridedSliceOp_starts_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -422,9 +419,9 @@ class TestStridedSliceOp_starts_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output() @@ -434,13 +431,12 @@ class TestStridedSliceOp_starts_Tensor(OpTest): class TestStridedSliceOp_ends_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() self.inputs = { 'Input': self.input, - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -458,9 +454,9 @@ class TestStridedSliceOp_ends_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output() @@ -470,19 +466,19 @@ class TestStridedSliceOp_ends_Tensor(OpTest): class TestStridedSliceOp_listTensor_Tensor(OpTest): - def setUp(self): self.config() ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.op_type = "strided_slice" self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensorList": ends_tensor + "EndsTensorList": ends_tensor, } self.outputs = {'Out': self.output} self.attrs = { @@ -500,9 +496,9 @@ class TestStridedSliceOp_listTensor_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output() @@ -512,13 +508,12 @@ class TestStridedSliceOp_listTensor_Tensor(OpTest): class TestStridedSliceOp_strides_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() self.inputs = { 'Input': self.input, - "StridesTensor": np.array(self.strides, dtype="int32") + "StridesTensor": np.array(self.strides, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -536,9 +531,9 @@ class TestStridedSliceOp_strides_Tensor(OpTest): self.axes = [0, 1, 2] self.strides = [1, -1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output() @@ -549,48 +544,50 @@ class TestStridedSliceOp_strides_Tensor(OpTest): # Test python API class TestStridedSliceAPI(unittest.TestCase): - def test_1(self): input = np.random.random([3, 4, 5, 6]).astype("float64") minus_1 = fluid.layers.fill_constant([1], "int32", -1) minus_3 = fluid.layers.fill_constant([1], "int32", -3) - starts = fluid.layers.data(name='starts', - shape=[3], - dtype='int32', - append_batch_size=False) - ends = fluid.layers.data(name='ends', - shape=[3], - dtype='int32', - append_batch_size=False) - strides = fluid.layers.data(name='strides', - shape=[3], - dtype='int32', - append_batch_size=False) - - x = fluid.layers.data(name="x", - shape=[3, 4, 5, 6], - append_batch_size=False, - dtype="float64") - out_1 = paddle.strided_slice(x, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_2 = paddle.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_3 = paddle.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, minus_1], - strides=[1, 1, 1]) - out_4 = paddle.strided_slice(x, - axes=[0, 1, 2], - starts=starts, - ends=ends, - strides=strides) + starts = fluid.layers.data( + name='starts', shape=[3], dtype='int32', append_batch_size=False + ) + ends = fluid.layers.data( + name='ends', shape=[3], dtype='int32', append_batch_size=False + ) + strides = fluid.layers.data( + name='strides', shape=[3], dtype='int32', append_batch_size=False + ) + + x = fluid.layers.data( + name="x", + shape=[3, 4, 5, 6], + append_batch_size=False, + dtype="float64", + ) + out_1 = paddle.strided_slice( + x, + axes=[0, 1, 2], + starts=[-3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_2 = paddle.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_3 = paddle.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, minus_1], + strides=[1, 1, 1], + ) + out_4 = paddle.strided_slice( + x, axes=[0, 1, 2], starts=starts, ends=ends, strides=strides + ) out_5 = x[-3:3, 0:100:2, -1:2:-1] out_6 = x[minus_3:3:1, 0:100:2, :, minus_1:2:minus_1] @@ -603,9 +600,10 @@ class TestStridedSliceAPI(unittest.TestCase): "x": input, 'starts': np.array([-3, 0, 2]).astype("int32"), 'ends': np.array([3, 2147483648, -1]).astype("int64"), - 'strides': np.array([1, 1, 1]).astype("int32") + 'strides': np.array([1, 1, 1]).astype("int32"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, input[-3:3, 0:100, 2:-1, :]) assert np.array_equal(res_2, input[-3:3, 0:100, :, 2:-1]) assert np.array_equal(res_3, input[-3:3, 0:100, :, 2:-1]) @@ -620,19 +618,20 @@ class TestStridedSliceAPI(unittest.TestCase): starts = [-3, 0, 2] ends = [3, 2, 4] strides_1 = [1, 1, 1] - sliced_1 = paddle.strided_slice(x, - axes=axes, - starts=starts, - ends=ends, - strides=strides_1) + sliced_1 = paddle.strided_slice( + x, axes=axes, starts=starts, ends=ends, strides=strides_1 + ) assert sliced_1.shape == (3, 2, 2, 2) - @unittest.skipIf(not paddle.is_compiled_with_cuda(), - "Cannot use CUDAPinnedPlace in CPU only version") + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), + "Cannot use CUDAPinnedPlace in CPU only version", + ) def test_cuda_pinned_place(self): with paddle.fluid.dygraph.guard(): - x = paddle.to_tensor(np.random.randn(2, 10), - place=paddle.CUDAPinnedPlace()) + x = paddle.to_tensor( + np.random.randn(2, 10), place=paddle.CUDAPinnedPlace() + ) self.assertTrue(x.place.is_cuda_pinned_place()) y = x[:, ::2] self.assertFalse(x.place.is_cuda_pinned_place()) @@ -640,15 +639,17 @@ class TestStridedSliceAPI(unittest.TestCase): class ArrayLayer(paddle.nn.Layer): - def __init__(self, input_size=224, output_size=10, array_size=1): super(ArrayLayer, self).__init__() self.input_size = input_size self.output_size = output_size self.array_size = array_size for i in range(self.array_size): - setattr(self, self.create_name(i), - paddle.nn.Linear(input_size, output_size)) + setattr( + self, + self.create_name(i), + paddle.nn.Linear(input_size, output_size), + ) def create_name(self, index): return 'linear_' + str(index) @@ -704,7 +705,6 @@ class ArrayLayer(paddle.nn.Layer): class TestStridedSliceTensorArray(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -718,15 +718,17 @@ class TestStridedSliceTensorArray(unittest.TestCase): def is_grads_equal(self, g1, g2): for i, g in enumerate(g1): - self.assertTrue(self.grad_equal(g, g2[i]), - msg="gradient_1:\n{} \ngradient_2:\n{}".format( - g, g2)) + self.assertTrue( + self.grad_equal(g, g2[i]), + msg="gradient_1:\n{} \ngradient_2:\n{}".format(g, g2), + ) def is_grads_equal_zeros(self, grads): for g in grads: self.assertTrue( self.grad_equal(np.zeros_like(g), g), - msg="The gradient should be zeros, but received \n{}".format(g)) + msg="The gradient should be zeros, but received \n{}".format(g), + ) def create_case(self, net): inps1 = paddle.randn([1, net.input_size], dtype='float32') @@ -751,31 +753,33 @@ class TestStridedSliceTensorArray(unittest.TestCase): np.testing.assert_array_equal( s1, s2, - err_msg='dygraph graph result:\n{} \nstatic dygraph result:\n{}'. - format(l1.numpy(), l2.numpy())) + err_msg='dygraph graph result:\n{} \nstatic dygraph result:\n{}'.format( + l1.numpy(), l2.numpy() + ), + ) def test_strided_slice_tensor_array_cuda_pinned_place(self): if paddle.device.is_compiled_with_cuda(): with paddle.fluid.dygraph.guard(): class Simple(paddle.nn.Layer): - def __init__(self): super(Simple, self).__init__() def forward(self, inps): tensor_array = None for i, tensor in enumerate(inps): - index = paddle.full(shape=[1], - dtype='int64', - fill_value=i) + index = paddle.full( + shape=[1], dtype='int64', fill_value=i + ) if tensor_array is None: tensor_array = paddle.tensor.array_write( - tensor, i=index) + tensor, i=index + ) else: - paddle.tensor.array_write(tensor, - i=index, - array=tensor_array) + paddle.tensor.array_write( + tensor, i=index, array=tensor_array + ) array1 = paddle.concat(tensor_array) array2 = paddle.concat(tensor_array[::-1]) @@ -784,12 +788,16 @@ class TestStridedSliceTensorArray(unittest.TestCase): net = Simple() func = paddle.jit.to_static(net.forward) - inps1 = paddle.to_tensor(np.random.randn(2, 10), - place=paddle.CUDAPinnedPlace(), - stop_gradient=False) - inps2 = paddle.to_tensor(np.random.randn(2, 10), - place=paddle.CUDAPinnedPlace(), - stop_gradient=False) + inps1 = paddle.to_tensor( + np.random.randn(2, 10), + place=paddle.CUDAPinnedPlace(), + stop_gradient=False, + ) + inps2 = paddle.to_tensor( + np.random.randn(2, 10), + place=paddle.CUDAPinnedPlace(), + stop_gradient=False, + ) self.assertTrue(inps1.place.is_cuda_pinned_place()) self.assertTrue(inps2.place.is_cuda_pinned_place()) @@ -799,201 +807,173 @@ class TestStridedSliceTensorArray(unittest.TestCase): self.assertFalse(result.place.is_cuda_pinned_place()) def test_strided_slice_tensor_array(self): - class Net01(ArrayLayer): - def array_slice(self, tensors): return tensors[::-1] self.create_case(Net01(array_size=10)) class Net02(ArrayLayer): - def array_slice(self, tensors): return tensors[::-2] self.create_case(Net02(input_size=112, array_size=11)) class Net03(ArrayLayer): - def array_slice(self, tensors): return tensors[::-3] self.create_case(Net03(input_size=112, array_size=9)) class Net04(ArrayLayer): - def array_slice(self, tensors): return tensors[1::-4] self.create_case(Net04(input_size=112, array_size=9)) class Net05(ArrayLayer): - def array_slice(self, tensors): return tensors[:7:-4] self.create_case(Net05(input_size=112, array_size=9)) class Net06(ArrayLayer): - def array_slice(self, tensors): return tensors[8:0:-4] self.create_case(Net06(input_size=112, array_size=9)) class Net07(ArrayLayer): - def array_slice(self, tensors): return tensors[8:1:-4] self.create_case(Net07(input_size=112, array_size=9)) class Net08(ArrayLayer): - def array_slice(self, tensors): return tensors[::2] self.create_case(Net08(input_size=112, array_size=11)) class Net09(ArrayLayer): - def array_slice(self, tensors): return tensors[::3] self.create_case(Net09(input_size=112, array_size=9)) class Net10(ArrayLayer): - def array_slice(self, tensors): return tensors[1::4] self.create_case(Net10(input_size=112, array_size=9)) class Net11(ArrayLayer): - def array_slice(self, tensors): return tensors[:8:4] self.create_case(Net11(input_size=112, array_size=9)) class Net12(ArrayLayer): - def array_slice(self, tensors): return tensors[1:8:4] self.create_case(Net12(input_size=112, array_size=9)) class Net13(ArrayLayer): - def array_slice(self, tensors): return tensors[8:10:4] self.create_case(Net13(input_size=112, array_size=13)) class Net14(ArrayLayer): - def array_slice(self, tensors): return tensors[3:10:4] self.create_case(Net14(input_size=112, array_size=13)) class Net15(ArrayLayer): - def array_slice(self, tensors): return tensors[2:10:4] self.create_case(Net15(input_size=112, array_size=13)) class Net16(ArrayLayer): - def array_slice(self, tensors): return tensors[3:10:3] self.create_case(Net16(input_size=112, array_size=13)) class Net17(ArrayLayer): - def array_slice(self, tensors): return tensors[3:15:3] self.create_case(Net17(input_size=112, array_size=13)) class Net18(ArrayLayer): - def array_slice(self, tensors): return tensors[0:15:3] self.create_case(Net18(input_size=112, array_size=13)) class Net19(ArrayLayer): - def array_slice(self, tensors): return tensors[-1:-5:-3] self.create_case(Net19(input_size=112, array_size=13)) class Net20(ArrayLayer): - def array_slice(self, tensors): return tensors[-1:-6:-3] self.create_case(Net20(input_size=112, array_size=13)) class Net21(ArrayLayer): - def array_slice(self, tensors): return tensors[-3:-6:-3] self.create_case(Net21(input_size=112, array_size=13)) class Net22(ArrayLayer): - def array_slice(self, tensors): return tensors[-5:-1:3] self.create_case(Net22(input_size=112, array_size=13)) class Net23(ArrayLayer): - def array_slice(self, tensors): return tensors[-6:-1:3] self.create_case(Net23(input_size=112, array_size=13)) class Net24(ArrayLayer): - def array_slice(self, tensors): return tensors[-6:-3:3] self.create_case(Net24(input_size=112, array_size=13)) class Net25(ArrayLayer): - def array_slice(self, tensors): return tensors[0::3] self.create_case(Net25(input_size=112, array_size=13)) class Net26(ArrayLayer): - def array_slice(self, tensors): return tensors[-60:20:3] self.create_case(Net26(input_size=112, array_size=13)) class Net27(ArrayLayer): - def array_slice(self, tensors): return tensors[-3:-60:-3] self.create_case(Net27(input_size=112, array_size=13)) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestStridedSliceFloat16(unittest.TestCase): - def init_test_case(self): self.op_type = 'strided_slice' self.input_shape = [3, 3, 3, 6, 7, 8] @@ -1008,8 +988,9 @@ class TestStridedSliceFloat16(unittest.TestCase): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - output = strided_slice_native_forward(x, self.axes, self.starts, - self.ends, self.strides) + output = strided_slice_native_forward( + x, self.axes, self.starts, self.ends, self.strides + ) x_grad = paddle.grad(output, x) output_np = output[0].numpy().astype('float32') x_grad_np = x_grad[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_subtract_op.py b/python/paddle/fluid/tests/unittests/test_subtract_op.py index 5eda6e367d4..ce58725d050 100644 --- a/python/paddle/fluid/tests/unittests/test_subtract_op.py +++ b/python/paddle/fluid/tests/unittests/test_subtract_op.py @@ -19,7 +19,6 @@ import paddle.fluid.core as core class ApiSubtractTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -40,56 +39,56 @@ class ApiSubtractTest(unittest.TestCase): def test_static_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_max = paddle.subtract(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_max = paddle.subtract(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.subtract(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.subtract(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): diff --git a/python/paddle/fluid/tests/unittests/test_sum_op.py b/python/paddle/fluid/tests/unittests/test_sum_op.py index 7d99391e5b1..6c8cc00ea8f 100644 --- a/python/paddle/fluid/tests/unittests/test_sum_op.py +++ b/python/paddle/fluid/tests/unittests/test_sum_op.py @@ -22,9 +22,11 @@ from paddle import enable_static import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator -from paddle.fluid.tests.unittests.op_test import (OpTest, - convert_float_to_uint16, - convert_uint16_to_float) +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + convert_uint16_to_float, +) from paddle.fluid.framework import _test_eager_guard import paddle.inference as paddle_infer import gradient_checker @@ -33,7 +35,6 @@ import paddle.fluid.layers as layers class TestSumOp(OpTest): - def setUp(self): self.op_type = "sum" self.init_kernel_type() @@ -58,7 +59,6 @@ class TestSumOp(OpTest): class TestSelectedRowsSumOp(unittest.TestCase): - def setUp(self): self.height = 10 self.row_numel = 12 @@ -67,14 +67,18 @@ class TestSelectedRowsSumOp(unittest.TestCase): self.init_kernel_type() def check_with_place(self, place, inplace): - self.check_input_and_optput(core.Scope(), place, inplace, True, True, - True) - self.check_input_and_optput(core.Scope(), place, inplace, False, True, - True) - self.check_input_and_optput(core.Scope(), place, inplace, False, False, - True) - self.check_input_and_optput(core.Scope(), place, inplace, False, False, - False) + self.check_input_and_optput( + core.Scope(), place, inplace, True, True, True + ) + self.check_input_and_optput( + core.Scope(), place, inplace, False, True, True + ) + self.check_input_and_optput( + core.Scope(), place, inplace, False, False, True + ) + self.check_input_and_optput( + core.Scope(), place, inplace, False, False, False + ) def init_kernel_type(self): pass @@ -85,13 +89,15 @@ class TestSelectedRowsSumOp(unittest.TestCase): array[i] *= rows[i] return array - def check_input_and_optput(self, - scope, - place, - inplace, - w1_has_data=False, - w2_has_data=False, - w3_has_data=False): + def check_input_and_optput( + self, + scope, + place, + inplace, + w1_has_data=False, + w2_has_data=False, + w3_has_data=False, + ): self.create_selected_rows(scope, place, "W1", w1_has_data) self.create_selected_rows(scope, place, "W2", w2_has_data) @@ -117,7 +123,8 @@ class TestSelectedRowsSumOp(unittest.TestCase): self.assertEqual(len(out.rows()), 7) np.testing.assert_array_equal( np.array(out.get_tensor()), - self._get_array(self.rows, self.row_numel) * has_data_w_num) + self._get_array(self.rows, self.row_numel) * has_data_w_num, + ) else: self.assertEqual(len(out.rows()), 0) @@ -148,15 +155,14 @@ class TestSelectedRowsSumOp(unittest.TestCase): class TestSelectedRowsSumOpInt(TestSelectedRowsSumOp): - def init_kernel_type(self): self.dtype = np.int32 -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSelectedRowsSumBF16Op(TestSelectedRowsSumOp): - def setUp(self): self.height = 10 self.row_numel = 12 @@ -164,8 +170,9 @@ class TestSelectedRowsSumBF16Op(TestSelectedRowsSumOp): self.dtype = np.uint16 self.init_kernel_type() np.random.seed(12345) - self.data = np.random.random( - (len(self.rows), self.row_numel)).astype(np.float32) + self.data = np.random.random((len(self.rows), self.row_numel)).astype( + np.float32 + ) def _get_array(self, rows, row_numel): if len(rows) > 0: @@ -173,13 +180,15 @@ class TestSelectedRowsSumBF16Op(TestSelectedRowsSumOp): else: return np.ndarray((0, row_numel), dtype=self.dtype) - def check_input_and_optput(self, - scope, - place, - inplace, - w1_has_data=False, - w2_has_data=False, - w3_has_data=False): + def check_input_and_optput( + self, + scope, + place, + inplace, + w1_has_data=False, + w2_has_data=False, + w3_has_data=False, + ): self.create_selected_rows(scope, place, "W1", w1_has_data) self.create_selected_rows(scope, place, "W2", w2_has_data) @@ -205,8 +214,12 @@ class TestSelectedRowsSumBF16Op(TestSelectedRowsSumOp): self.assertEqual(len(out.rows()), 7) out_bf16 = np.array(out.get_tensor()) out_fp32 = convert_uint16_to_float(out_bf16) - ref_fp32 = convert_uint16_to_float( - self._get_array(self.rows, self.row_numel)) * has_data_w_num + ref_fp32 = ( + convert_uint16_to_float( + self._get_array(self.rows, self.row_numel) + ) + * has_data_w_num + ) np.testing.assert_allclose(out_fp32, ref_fp32, atol=0, rtol=0.95e-2) else: self.assertEqual(len(out.rows()), 0) @@ -217,13 +230,11 @@ class TestSelectedRowsSumBF16Op(TestSelectedRowsSumOp): class TestSelectedRowsSumBF16OpBigRow(TestSelectedRowsSumBF16Op): - def init_kernel_type(self): self.row_numel = 102 class TestLoDTensorAndSelectedRowsOp(TestSelectedRowsSumOp): - def setUp(self): self.height = 10 self.row_numel = 12 @@ -255,23 +266,25 @@ class TestLoDTensorAndSelectedRowsOp(TestSelectedRowsSumOp): self.assertEqual(out_t.shape[0], self.height) np.testing.assert_array_equal( out_t, - self._get_array([i for i in range(self.height)], self.row_numel) * - np.tile(np.array(result).reshape(self.height, 1), self.row_numel)) + self._get_array([i for i in range(self.height)], self.row_numel) + * np.tile(np.array(result).reshape(self.height, 1), self.row_numel), + ) def create_lod_tensor(self, scope, place, var_name): var = scope.var(var_name) w_tensor = var.get_tensor() - w_array = self._get_array([i for i in range(self.height)], - self.row_numel) + w_array = self._get_array( + [i for i in range(self.height)], self.row_numel + ) w_tensor.set(w_array, place) return var -#----------- test fp16 ----------- -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +# ----------- test fp16 ----------- +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16SumOp(TestSumOp): - def init_kernel_type(self): self.dtype = np.float16 @@ -289,11 +302,10 @@ class TestFP16SumOp(TestSumOp): def create_test_sum_fp16_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestSumFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -308,9 +320,8 @@ def create_test_sum_fp16_class(parent): globals()[cls_name] = TestSumFp16Case -#----------- test bf16 ----------- +# ----------- test bf16 ----------- class TestSumBF16Op(OpTest): - def setUp(self): self.op_type = "sum" self.init_kernel_type() @@ -319,9 +330,11 @@ class TestSumBF16Op(OpTest): x2 = np.random.random((3, 40)).astype(np.float32) y = x0 + x1 + x2 self.inputs = { - "X": [("x0", convert_float_to_uint16(x0)), - ("x1", convert_float_to_uint16(x1)), - ("x2", convert_float_to_uint16(x2))] + "X": [ + ("x0", convert_float_to_uint16(x0)), + ("x1", convert_float_to_uint16(x1)), + ("x2", convert_float_to_uint16(x2)), + ] } self.outputs = {'Out': convert_float_to_uint16(y)} @@ -336,15 +349,14 @@ class TestSumBF16Op(OpTest): class API_Test_Add_n(unittest.TestCase): - def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - input0 = fluid.layers.fill_constant(shape=[2, 3], - dtype='int64', - value=5) - input1 = fluid.layers.fill_constant(shape=[2, 3], - dtype='int64', - value=3) + input0 = fluid.layers.fill_constant( + shape=[2, 3], dtype='int64', value=5 + ) + input1 = fluid.layers.fill_constant( + shape=[2, 3], dtype='int64', value=3 + ) expected_result = np.empty((2, 3)) expected_result.fill(8) sum_value = paddle.add_n([input0, input1]) @@ -371,16 +383,19 @@ class API_Test_Add_n(unittest.TestCase): expected_result = np.empty((2, 3)) expected_result.fill(2) sum_value = paddle.add_n([input0, input1]) - self.assertEqual((sum_value.numpy() == expected_result).all(), - True) + self.assertEqual( + (sum_value.numpy() == expected_result).all(), True + ) expected_grad_result = np.empty((2, 3)) expected_grad_result.fill(1) sum_value.backward() self.assertEqual( - (input0.grad.numpy() == expected_grad_result).all(), True) + (input0.grad.numpy() == expected_grad_result).all(), True + ) self.assertEqual( - (input1.grad.numpy() == expected_grad_result).all(), True) + (input1.grad.numpy() == expected_grad_result).all(), True + ) def test_add_n_and_add_and_grad(self): with fluid.dygraph.guard(): @@ -397,7 +412,7 @@ class API_Test_Add_n(unittest.TestCase): dx, dy = paddle.grad([out], [x, y], create_graph=True) - expected_out = np.array([[10., 12., 14.], [16., 18., 20.]]) + expected_out = np.array([[10.0, 12.0, 14.0], [16.0, 18.0, 20.0]]) expected_dx = np.array([[1, 1, 1], [1, 1, 1]]) expected_dy = np.array([[1, 1, 1], [1, 1, 1]]) @@ -407,9 +422,7 @@ class API_Test_Add_n(unittest.TestCase): class TestRaiseSumError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.sum([11, 22]) @@ -430,9 +443,7 @@ class TestRaiseSumError(unittest.TestCase): class TestRaiseSumsError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.sums([11, 22]) @@ -468,9 +479,7 @@ class TestRaiseSumsError(unittest.TestCase): class TestSumOpError(unittest.TestCase): - def test_errors(self): - def test_empty_list_input(): with fluid.dygraph.guard(): fluid._legacy_C_ops.sum([]) @@ -488,14 +497,16 @@ create_test_sum_fp16_class(TestLoDTensorAndSelectedRowsOp) class TestReduceOPTensorAxisBase(unittest.TestCase): - def setUp(self): paddle.disable_static() paddle.seed(2022) self.temp_dir = tempfile.TemporaryDirectory() self.save_path = os.path.join(self.temp_dir.name, 'reduce_tensor_axis') - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.keepdim = False self.init_data() @@ -514,7 +525,8 @@ class TestReduceOPTensorAxisBase(unittest.TestCase): pd_out = self.pd_api(self.x, self.tensor_axis) np_out = self.np_api(self.x.numpy(), tuple(self.np_axis)) np.testing.assert_allclose( - pd_out.numpy() if pd_out.size > 1 else pd_out.item(), np_out) + pd_out.numpy() if pd_out.size > 1 else pd_out.item(), np_out + ) pd_out.backward() self.assertEqual(self.x.gradient().shape, tuple(self.x.shape)) @@ -524,9 +536,9 @@ class TestReduceOPTensorAxisBase(unittest.TestCase): starup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, starup_prog): # run static - x = paddle.static.data(shape=self.x.shape, - name='x', - dtype='float32') + x = paddle.static.data( + shape=self.x.shape, name='x', dtype='float32' + ) if isinstance(self.tensor_axis, paddle.Tensor): axis = paddle.assign(self.np_axis) else: @@ -541,17 +553,19 @@ class TestReduceOPTensorAxisBase(unittest.TestCase): linear_out = linear(x) out = self.pd_api(linear_out, axis, keepdim=self.keepdim) - sgd = paddle.optimizer.SGD(learning_rate=0.) + sgd = paddle.optimizer.SGD(learning_rate=0.0) sgd.minimize(paddle.mean(out)) exe = paddle.static.Executor(self.place) exe.run(starup_prog) - static_out = exe.run(feed={'x': self.x.numpy().astype('float32')}, - fetch_list=[out]) + static_out = exe.run( + feed={'x': self.x.numpy().astype('float32')}, fetch_list=[out] + ) # run infer paddle.static.save_inference_model(self.save_path, [x], [out], exe) - config = paddle_infer.Config(self.save_path + '.pdmodel', - self.save_path + '.pdiparams') + config = paddle_infer.Config( + self.save_path + '.pdmodel', self.save_path + '.pdiparams' + ) if paddle.is_compiled_with_cuda(): config.enable_use_gpu(100, 0) else: @@ -570,7 +584,6 @@ class TestReduceOPTensorAxisBase(unittest.TestCase): class TestSumWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.sum self.np_api = np.sum @@ -579,12 +592,11 @@ class TestSumWithTensorAxis1(TestReduceOPTensorAxisBase): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] class TestAddNDoubleGradCheck(unittest.TestCase): - def add_n_wrapper(self, x): return paddle.add_n(x) @@ -602,17 +614,21 @@ class TestAddNDoubleGradCheck(unittest.TestCase): data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) data2_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) - gradient_checker.double_grad_check([data1, data2], - out, - x_init=[data1_arr, data2_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) gradient_checker.double_grad_check_for_dygraph( - self.add_n_wrapper, [data1, data2], + self.add_n_wrapper, + [data1, data2], out, x_init=[data1_arr, data2_arr], - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -624,7 +640,6 @@ class TestAddNDoubleGradCheck(unittest.TestCase): class TestAddNTripleGradCheck(unittest.TestCase): - def add_n_wrapper(self, x): return paddle.add_n(x) @@ -642,17 +657,21 @@ class TestAddNTripleGradCheck(unittest.TestCase): data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) data2_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) - gradient_checker.triple_grad_check([data1, data2], - out, - x_init=[data1_arr, data2_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) gradient_checker.triple_grad_check_for_dygraph( - self.add_n_wrapper, [data1, data2], + self.add_n_wrapper, + [data1, data2], out, x_init=[data1_arr, data2_arr], - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -664,7 +683,6 @@ class TestAddNTripleGradCheck(unittest.TestCase): class TestSumDoubleGradCheck(unittest.TestCase): - def sum_wrapper(self, x): return paddle.sum(x[0], axis=1, keepdim=True) @@ -679,16 +697,13 @@ class TestSumDoubleGradCheck(unittest.TestCase): out = paddle.sum(data, axis=1, keepdim=True) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.sum_wrapper, [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.sum_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -700,7 +715,6 @@ class TestSumDoubleGradCheck(unittest.TestCase): class TestSumTripleGradCheck(unittest.TestCase): - def sum_wrapper(self, x): return paddle.sum(x[0], axis=1, keepdim=True) @@ -715,16 +729,13 @@ class TestSumTripleGradCheck(unittest.TestCase): out = paddle.sum(data, axis=1, keepdim=True) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.sum_wrapper, [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.sum_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_svd_op.py b/python/paddle/fluid/tests/unittests/test_svd_op.py index aca403dac07..146b18ae20c 100644 --- a/python/paddle/fluid/tests/unittests/test_svd_op.py +++ b/python/paddle/fluid/tests/unittests/test_svd_op.py @@ -21,25 +21,23 @@ from op_test import OpTest, skip_check_grad_ci class TestSvdOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = paddle.linalg.svd self.generate_input() self.generate_output() self.op_type = "svd" - assert (hasattr(self, "_output_data")) + assert hasattr(self, "_output_data") self.inputs = {"X": self._input_data} self.attrs = {'full_matrices': self.get_full_matrices_option()} self.outputs = { "U": self._output_data[0], "S": self._output_data[1], - "VH": self._output_data[2] + "VH": self._output_data[2], } def generate_input(self): - """ return a input_data and input_shape - """ + """return a input_data and input_shape""" self._input_shape = (100, 1) self._input_data = np.random.random(self._input_shape).astype("float64") @@ -47,17 +45,17 @@ class TestSvdOp(OpTest): return False def generate_output(self): - assert (hasattr(self, "_input_data")) + assert hasattr(self, "_input_data") self._output_data = np.linalg.svd(self._input_data) def test_check_output(self): self.check_output(no_check_set=['U', 'VH'], check_eager=True) def test_svd_forward(self): - """ u matmul diag(s) matmul vt must become X - """ + """u matmul diag(s) matmul vt must become X""" single_input = self._input_data.reshape( - [-1, self._input_shape[-2], self._input_shape[-1]])[0] + [-1, self._input_shape[-2], self._input_shape[-1]] + )[0] paddle.disable_static() dy_x = paddle.to_tensor(single_input) dy_u, dy_s, dy_vt = paddle.linalg.svd(dy_x) @@ -71,19 +69,19 @@ class TestSvdOp(OpTest): paddle.enable_static() def check_S_grad(self): - self.check_grad(['X'], ['S'], - numeric_grad_delta=0.001, - check_eager=True) + self.check_grad( + ['X'], ['S'], numeric_grad_delta=0.001, check_eager=True + ) def check_U_grad(self): - self.check_grad(['X'], ['U'], - numeric_grad_delta=0.001, - check_eager=True) + self.check_grad( + ['X'], ['U'], numeric_grad_delta=0.001, check_eager=True + ) def check_V_grad(self): - self.check_grad(['X'], ['VH'], - numeric_grad_delta=0.001, - check_eager=True) + self.check_grad( + ['X'], ['VH'], numeric_grad_delta=0.001, check_eager=True + ) def test_check_grad(self): """ @@ -101,123 +99,146 @@ class TestSvdCheckGrad2(TestSvdOp): no_need_check_grad = True def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (5, 5) - self._input_data = np.vander([2, 3, 4, 5, 6]).astype("float64").reshape( - self._input_shape) + self._input_data = ( + np.vander([2, 3, 4, 5, 6]) + .astype("float64") + .reshape(self._input_shape) + ) class TestSvdNormalMatrixSmall(TestSvdCheckGrad2): - def generate_input(self): - """ small matrix SVD. - """ + """small matrix SVD.""" self._input_shape = (1, 1) self._input_data = np.random.random(self._input_shape).astype("float64") class TestSvdNormalMatrix6x3(TestSvdCheckGrad2): - def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (6, 3) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") class TestSvdNormalMatrix3x6(TestSvdCheckGrad2): - def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (3, 6) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") self._input_data = self._input_data.transpose((-1, -2)) class TestSvdNormalMatrix6x3Batched(TestSvdOp): - def generate_input(self): self._input_shape = (10, 6, 3) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") self._input_data = np.stack([self._input_data] * 10, axis=0) def test_svd_forward(self): - """ test_svd_forward not support batched input, so disable this test. - """ + """test_svd_forward not support batched input, so disable this test.""" pass class TestSvdNormalMatrix3x6Batched(TestSvdOp): - def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (10, 3, 6) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") self._input_data = self._input_data.transpose((-1, -2)) self._input_data = np.stack([self._input_data] * 10, axis=0) def test_svd_forward(self): - """ test_svd_forward not support batched input, so disable this test. - """ + """test_svd_forward not support batched input, so disable this test.""" pass class TestSvdNormalMatrix3x3x3x6Batched(TestSvdOp): - def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (3, 3, 3, 6) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") self._input_data = self._input_data.transpose((-1, -2)) self._input_data = np.stack( - [self._input_data, self._input_data, self._input_data], axis=0) + [self._input_data, self._input_data, self._input_data], axis=0 + ) self._input_data = np.stack( - [self._input_data, self._input_data, self._input_data], axis=0) + [self._input_data, self._input_data, self._input_data], axis=0 + ) def test_svd_forward(self): - """ test_svd_forward not support batched input, so disable this test. - """ + """test_svd_forward not support batched input, so disable this test.""" pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestSvdNormalMatrixBig(TestSvdOp): - def generate_input(self): - """ big matrix SVD. - - """ + """big matrix SVD.""" self._input_shape = (2, 200, 300) self._input_data = np.random.random(self._input_shape).astype("float64") def test_svd_forward(self): - """ test_svd_forward not support batched input, so disable this test. - """ + """test_svd_forward not support batched input, so disable this test.""" pass def test_check_grad(self): @@ -225,16 +246,13 @@ class TestSvdNormalMatrixBig(TestSvdOp): class TestSvdNormalMatrixBig2(TestSvdOp): - def generate_input(self): - """ big matrix SVD. - """ + """big matrix SVD.""" self._input_shape = (1, 100) self._input_data = np.random.random(self._input_shape).astype("float64") class TestSvdNormalMatrixFullMatrices(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -246,21 +264,19 @@ class TestSvdNormalMatrixFullMatrices(unittest.TestCase): mat = np.random.random(mat_shape).astype("float64") x = paddle.to_tensor(mat) u, s, vh = paddle.linalg.svd(x, full_matrices=True) - assert (u.shape == [2, 2]) - assert (vh.shape == [3, 3]) + assert u.shape == [2, 2] + assert vh.shape == [3, 3] x_recover = u.matmul(paddle.diag(s)).matmul(vh[0:2]) - if ((paddle.abs(x_recover - x) > 1e-4).any()): + if (paddle.abs(x_recover - x) > 1e-4).any(): raise RuntimeError("mat can't be recovered\n") class TestSvdFullMatriceGrad(TestSvdNormalMatrix6x3): - def get_full_matrices_option(self): return True def test_svd_forward(self): - """ test_svd_forward not support full matrices, so disable this test. - """ + """test_svd_forward not support full matrices, so disable this test.""" pass def test_check_grad(self): @@ -268,12 +284,11 @@ class TestSvdFullMatriceGrad(TestSvdNormalMatrix6x3): remember the input matrix must be the full rank matrix, otherwise the gradient will stochatic because the u / v 's (n-k) freedom vectors """ self.check_S_grad() - #self.check_U_grad() // don't check U grad, because U have freedom vector + # self.check_U_grad() // don't check U grad, because U have freedom vector self.check_V_grad() class TestSvdAPI(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() a = np.random.rand(5, 5) @@ -290,15 +305,17 @@ class TestSvdAPI(unittest.TestCase): for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): a = np.random.rand(5, 5) - x = paddle.fluid.data(name="input", - shape=[5, 5], - dtype='float64') + x = paddle.fluid.data( + name="input", shape=[5, 5], dtype='float64' + ) u, s, vh = paddle.linalg.svd(x) exe = fluid.Executor(place) gt_u, gt_s, gt_vh = np.linalg.svd(a, full_matrices=False) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[s]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[s], + ) np.testing.assert_allclose(fetches[0], gt_s, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_switch.py b/python/paddle/fluid/tests/unittests/test_switch.py index cdc2162a599..31fe519593f 100644 --- a/python/paddle/fluid/tests/unittests/test_switch.py +++ b/python/paddle/fluid/tests/unittests/test_switch.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import default_startup_program class TestSwitch(unittest.TestCase): - def check_switch(self, value): x = layers.fill_constant(shape=[1], dtype='float32', value=value) zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) @@ -30,10 +29,9 @@ class TestSwitch(unittest.TestCase): two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0) three_var = layers.fill_constant(shape=[1], dtype='float32', value=3.0) - result = layers.create_global_var(shape=[1], - value=-1.0, - dtype='float32', - persistable=True) + result = layers.create_global_var( + shape=[1], value=-1.0, dtype='float32', persistable=True + ) with layers.Switch() as switch: with switch.case(layers.less_than(x, zero_var)): @@ -63,20 +61,18 @@ class TestSwitch(unittest.TestCase): class TestSwitchCaseError(unittest.TestCase): - def test_error(self): main_program = framework.Program() startup_program = framework.Program() with framework.program_guard(main_program, startup_program): cond = layers.fill_constant(shape=[1], dtype='float32', value=0.0) - zero_var = layers.fill_constant(shape=[1], - dtype='float32', - value=0.0) - - result = layers.create_global_var(shape=[1], - value=-1.0, - dtype='float32', - persistable=True) + zero_var = layers.fill_constant( + shape=[1], dtype='float32', value=0.0 + ) + + result = layers.create_global_var( + shape=[1], value=-1.0, dtype='float32', persistable=True + ) # 1. The type of 'condition' in case must be Variable. def test_condition_type(): diff --git a/python/paddle/fluid/tests/unittests/test_switch_autotune.py b/python/paddle/fluid/tests/unittests/test_switch_autotune.py index 93dd06ec291..52b39fcf617 100644 --- a/python/paddle/fluid/tests/unittests/test_switch_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_switch_autotune.py @@ -22,7 +22,6 @@ import os class SimpleNet(paddle.nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.conv = paddle.nn.Conv2D(1, 2, (3, 3)) @@ -51,7 +50,6 @@ def static_program(net, data): class TestAutoTune(unittest.TestCase): - def set_flags(self, enable_autotune): if paddle.is_compiled_with_cuda(): if enable_autotune: @@ -67,7 +65,7 @@ class TestAutoTune(unittest.TestCase): expected_res = { "step_id": step_id, "cache_size": 0, - "cache_hit_rate": 0 + "cache_hit_rate": 0, } if paddle.is_compiled_with_cuda(): # Total 3 * num_iters cache accesses, only iter 2 hits the cache. @@ -77,9 +75,8 @@ class TestAutoTune(unittest.TestCase): def test_autotune(self): paddle.incubate.autotune.set_config( - config={"kernel": { - "enable": False - }}) + config={"kernel": {"enable": False}} + ) self.assertEqual(self.get_flags("FLAGS_use_autotune"), False) paddle.incubate.autotune.set_config(config={"kernel": {"enable": True}}) @@ -96,21 +93,17 @@ class TestAutoTune(unittest.TestCase): class TestDygraphAutoTuneStatus(TestAutoTune): - def run_program(self, enable_autotune): self.set_flags(enable_autotune) if enable_autotune: paddle.incubate.autotune.set_config( - config={"kernel": { - "enable": True, - "tuning_range": [1, 2] - }}) + config={"kernel": {"enable": True, "tuning_range": [1, 2]}} + ) else: paddle.incubate.autotune.set_config( - config={"kernel": { - "enable": False - }}) - x_var = paddle.uniform((1, 1, 8, 8), dtype='float32', min=-1., max=1.) + config={"kernel": {"enable": False}} + ) + x_var = paddle.uniform((1, 1, 8, 8), dtype='float32', min=-1.0, max=1.0) net = SimpleNet() for i in range(3): train_dygraph(net, x_var) @@ -135,7 +128,6 @@ class TestDygraphAutoTuneStatus(TestAutoTune): class TestStaticAutoTuneStatus(TestAutoTune): - def run_program(self, enable_autotune): paddle.enable_static() @@ -143,13 +135,16 @@ class TestStaticAutoTuneStatus(TestAutoTune): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data = paddle.static.data(name='X', - shape=data_shape, - dtype='float32') + data = paddle.static.data( + name='X', shape=data_shape, dtype='float32' + ) net = SimpleNet() loss = static_program(net, data) - place = paddle.CUDAPlace(0) if paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) exe.run(startup_program) x = np.random.random(size=data_shape).astype('float32') @@ -164,10 +159,8 @@ class TestStaticAutoTuneStatus(TestAutoTune): os.remove(tfile.name) else: paddle.incubate.autotune.set_config( - config={"kernel": { - "enable": False, - "tuning_range": [1, 2] - }}) + config={"kernel": {"enable": False, "tuning_range": [1, 2]}} + ) for i in range(3): exe.run(program=main_program, feed={'X': x}, fetch_list=[loss]) @@ -190,7 +183,6 @@ class TestStaticAutoTuneStatus(TestAutoTune): class TestAutoTuneAPI(unittest.TestCase): - def test_set_config_warnings(self): with warnings.catch_warnings(record=True) as w: config = {"kernel": {"enable": 1, "tuning_range": 1}} @@ -204,7 +196,8 @@ class TestAutoTuneAPI(unittest.TestCase): def test_set_config_attr(self): paddle.incubate.autotune.set_config(config=None) self.assertEqual( - paddle.get_flags("FLAGS_use_autotune")["FLAGS_use_autotune"], True) + paddle.get_flags("FLAGS_use_autotune")["FLAGS_use_autotune"], True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_switch_case.py b/python/paddle/fluid/tests/unittests/test_switch_case.py index 3c529d789be..aad70fe789b 100644 --- a/python/paddle/fluid/tests/unittests/test_switch_case.py +++ b/python/paddle/fluid/tests/unittests/test_switch_case.py @@ -23,9 +23,7 @@ from functools import partial class TestAPISwitchCase(unittest.TestCase): - def test_return_single_var(self): - def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) @@ -43,86 +41,90 @@ class TestAPISwitchCase(unittest.TestCase): index_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) # call fn_1 - out_0 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2, - 3: fn_3 - }) + out_0 = layers.switch_case( + branch_index=index_1, branch_fns={1: fn_1, 2: fn_2, 3: fn_3} + ) # call fn_2 : branch_fns={0: fn_1, 1:fn_2, 2:fn_3} - out_1 = layers.switch_case(branch_index=index_1, - branch_fns=(fn_1, fn_2, fn_3)) + out_1 = layers.switch_case( + branch_index=index_1, branch_fns=(fn_1, fn_2, fn_3) + ) # call default fn_3 - out_2 = layers.switch_case(branch_index=index_5, - branch_fns=((1, fn_1), (2, fn_2)), - default=fn_3) + out_2 = layers.switch_case( + branch_index=index_5, + branch_fns=((1, fn_1), (2, fn_2)), + default=fn_3, + ) # no default, call fn_2 - out_3 = layers.switch_case(branch_index=index_2, - branch_fns=[(1, fn_1), (2, fn_2)]) + out_3 = layers.switch_case( + branch_index=index_2, branch_fns=[(1, fn_1), (2, fn_2)] + ) # no default, call fn_2 but branch_index is 5 - out_4 = layers.switch_case(branch_index=index_5, - branch_fns=[(1, fn_1), (3, fn_2), - (2, fn_3)]) - - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + out_4 = layers.switch_case( + branch_index=index_5, + branch_fns=[(1, fn_1), (3, fn_2), (2, fn_3)], + ) + + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - fetch_list=[out_0, out_1, out_2, out_3, out_4]) + res = exe.run( + main_program, fetch_list=[out_0, out_1, out_2, out_3, out_4] + ) np.testing.assert_allclose( res[0], 1, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 1)) + err_msg='result is {} but answer is {}'.format(res[0], 1), + ) np.testing.assert_allclose( res[1], 2, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 2)) + err_msg='result is {} but answer is {}'.format(res[0], 2), + ) np.testing.assert_allclose( res[2], 3, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 3)) + err_msg='result is {} but answer is {}'.format(res[0], 3), + ) np.testing.assert_allclose( res[3], 2, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 2)) + err_msg='result is {} but answer is {}'.format(res[0], 2), + ) np.testing.assert_allclose( res[4], 2, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 2)) + err_msg='result is {} but answer is {}'.format(res[0], 2), + ) def test_return_var_tuple(self): - def fn_1(): - return layers.fill_constant(shape=[1, 2], dtype='int32', - value=1), layers.fill_constant( - shape=[2, 3], - dtype='float32', - value=2) + return layers.fill_constant( + shape=[1, 2], dtype='int32', value=1 + ), layers.fill_constant(shape=[2, 3], dtype='float32', value=2) def fn_2(): - return layers.fill_constant(shape=[3, 4], dtype='int32', - value=3), layers.fill_constant( - shape=[4, 5], - dtype='float32', - value=4) + return layers.fill_constant( + shape=[3, 4], dtype='int32', value=3 + ), layers.fill_constant(shape=[4, 5], dtype='float32', value=4) def fn_3(): - return layers.fill_constant(shape=[5], dtype='int32', - value=5), layers.fill_constant( - shape=[5, 6], - dtype='float32', - value=6) + return layers.fill_constant( + shape=[5], dtype='int32', value=5 + ), layers.fill_constant(shape=[5, 6], dtype='float32', value=6) main_program = Program() startup_program = Program() @@ -131,66 +133,72 @@ class TestAPISwitchCase(unittest.TestCase): out = layers.switch_case(index_1, ((1, fn_1), (2, fn_2)), fn_3) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=out) - np.testing.assert_allclose(np.asarray(ret[0]), - np.full((1, 2), 1, np.int32), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(ret[1]), - np.full((2, 3), 2, np.float32), - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(ret[0]), np.full((1, 2), 1, np.int32), rtol=1e-05 + ) + np.testing.assert_allclose( + np.asarray(ret[1]), np.full((2, 3), 2, np.float32), rtol=1e-05 + ) class TestAPISwitchCase_Nested(unittest.TestCase): - def test_nested_switch_case(self): - def fn_1(x=1): - out = layers.switch_case(branch_index=layers.fill_constant( - shape=[1], dtype='int32', value=x), - branch_fns={ - 1: - partial(layers.fill_constant, - shape=[1], - dtype='int32', - value=1), - x: - partial(layers.fill_constant, - shape=[2], - dtype='int32', - value=x) - }) + out = layers.switch_case( + branch_index=layers.fill_constant( + shape=[1], dtype='int32', value=x + ), + branch_fns={ + 1: partial( + layers.fill_constant, shape=[1], dtype='int32', value=1 + ), + x: partial( + layers.fill_constant, shape=[2], dtype='int32', value=x + ), + }, + ) return out def fn_2(x=2): - out = layers.switch_case(branch_index=layers.fill_constant( - shape=[1], dtype='int32', value=2), - branch_fns={ - 1: - partial(layers.fill_constant, - shape=[4, 3], - dtype='int32', - value=1), - 2: - partial(fn_1, x=x) - }) + out = layers.switch_case( + branch_index=layers.fill_constant( + shape=[1], dtype='int32', value=2 + ), + branch_fns={ + 1: partial( + layers.fill_constant, + shape=[4, 3], + dtype='int32', + value=1, + ), + 2: partial(fn_1, x=x), + }, + ) return out def fn_3(): - out = layers.switch_case(branch_index=layers.fill_constant( - shape=[1], dtype='int32', value=3), - branch_fns={ - 1: - partial(layers.fill_constant, - shape=[4, 3], - dtype='int32', - value=1), - 3: - partial(fn_2, x=3) - }) + out = layers.switch_case( + branch_index=layers.fill_constant( + shape=[1], dtype='int32', value=3 + ), + branch_fns={ + 1: partial( + layers.fill_constant, + shape=[4, 3], + dtype='int32', + value=1, + ), + 3: partial(fn_2, x=3), + }, + ) return out main_program = Program() @@ -200,56 +208,53 @@ class TestAPISwitchCase_Nested(unittest.TestCase): index_2 = layers.fill_constant(shape=[1], dtype='int32', value=2) index_3 = layers.fill_constant(shape=[1], dtype='int64', value=3) - out_1 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2, - 3: fn_3 - }) - out_2 = layers.switch_case(branch_index=index_2, - branch_fns={ - 1: fn_1, - 2: fn_2, - 3: fn_3 - }) - - out_3 = layers.switch_case(branch_index=index_3, - branch_fns={ - 1: fn_1, - 2: fn_2, - 3: fn_3 - }) - - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + out_1 = layers.switch_case( + branch_index=index_1, branch_fns={1: fn_1, 2: fn_2, 3: fn_3} + ) + out_2 = layers.switch_case( + branch_index=index_2, branch_fns={1: fn_1, 2: fn_2, 3: fn_3} + ) + + out_3 = layers.switch_case( + branch_index=index_3, branch_fns={1: fn_1, 2: fn_2, 3: fn_3} + ) + + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - feed={"index_1": np.array([1], dtype="uint8")}, - fetch_list=[out_1, out_2, out_3]) + res = exe.run( + main_program, + feed={"index_1": np.array([1], dtype="uint8")}, + fetch_list=[out_1, out_2, out_3], + ) np.testing.assert_allclose( res[0], 1, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 1)) + err_msg='result is {} but answer is {}'.format(res[0], 1), + ) np.testing.assert_allclose( res[1], 2, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[1], 2)) + err_msg='result is {} but answer is {}'.format(res[1], 2), + ) np.testing.assert_allclose( res[2], 3, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[2], 3)) + err_msg='result is {} but answer is {}'.format(res[2], 3), + ) # test TypeError and ValueError of api switch_case class TestAPISwitchCase_Error(unittest.TestCase): - def test_error(self): - def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) @@ -262,82 +267,90 @@ class TestAPISwitchCase_Error(unittest.TestCase): main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): - key_float32 = layers.fill_constant(shape=[1], - dtype='float32', - value=0.23) - key_int32 = layers.fill_constant(shape=[1], - dtype='int32', - value=0.23) + key_float32 = layers.fill_constant( + shape=[1], dtype='float32', value=0.23 + ) + key_int32 = layers.fill_constant( + shape=[1], dtype='int32', value=0.23 + ) # The type of 'branch_index' in Op(switch_case) must be Variable def type_error_branch_index(): - layers.switch_case(branch_index=1, - branch_fns=[(1, fn_1)], - default=fn_3) + layers.switch_case( + branch_index=1, branch_fns=[(1, fn_1)], default=fn_3 + ) self.assertRaises(TypeError, type_error_branch_index) # The data type of 'branch_index' in Op(switch_case) must be int32, int64 or uint8 def dtype_error_branch_index(): - layers.switch_case(branch_index=key_float32, - branch_fns=[(1, fn_1)], - default=fn_3) + layers.switch_case( + branch_index=key_float32, + branch_fns=[(1, fn_1)], + default=fn_3, + ) self.assertRaises(TypeError, dtype_error_branch_index) # The type of 'branch_fns' in Op(switch_case) must be list, tuple or dict def type_error_branch_fns(): - layers.switch_case(branch_index=key_int32, - branch_fns=1, - default=fn_3) + layers.switch_case( + branch_index=key_int32, branch_fns=1, default=fn_3 + ) self.assertRaises(TypeError, type_error_branch_fns) # The elements' type of 'branch_fns' in Op(switch_case) must be tuple def type_error_index_fn_pair_1(): - layers.switch_case(branch_index=key_int32, - branch_fns=[1], - default=fn_3) + layers.switch_case( + branch_index=key_int32, branch_fns=[1], default=fn_3 + ) self.assertRaises(TypeError, type_error_index_fn_pair_1) # The tuple's size of 'branch_fns' in Op(switch_case) must be 2 def type_error_index_fn_pair_2(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(1, 2, 3)], - default=fn_3) + layers.switch_case( + branch_index=key_int32, branch_fns=[(1, 2, 3)], default=fn_3 + ) self.assertRaises(TypeError, type_error_index_fn_pair_2) # The key's type of 'branch_fns' in Op(switch_case) must be int def type_error_key(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(2.3, 2)], - default=fn_3) + layers.switch_case( + branch_index=key_int32, branch_fns=[(2.3, 2)], default=fn_3 + ) self.assertRaises(TypeError, type_error_key) # The key in 'branch_fns' must be unique def value_error_key(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(2, fn_1), (2, fn_2)], - default=fn_3) + layers.switch_case( + branch_index=key_int32, + branch_fns=[(2, fn_1), (2, fn_2)], + default=fn_3, + ) self.assertRaises(ValueError, value_error_key) # The type of function in 'branch_fns' must be callable def type_error_fn(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(1, 1), (2, fn_2)], - default=fn_3) + layers.switch_case( + branch_index=key_int32, + branch_fns=[(1, 1), (2, fn_2)], + default=fn_3, + ) self.assertRaises(TypeError, type_error_fn) # The default in Op(case) must be callable def type_error_default(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(1, fn_1), (2, fn_2)], - default=1) + layers.switch_case( + branch_index=key_int32, + branch_fns=[(1, fn_1), (2, fn_2)], + default=1, + ) self.assertRaises(TypeError, type_error_default) diff --git a/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py index 055187d8aab..e2a53465d44 100644 --- a/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py @@ -47,7 +47,7 @@ class TestSyncBatchNormOpTraining(unittest.TestCase): def setUp(self): """Setup.""" - #self.dtype = np.float32 + # self.dtype = np.float32 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.N = 8 self.C = 16 @@ -56,12 +56,9 @@ class TestSyncBatchNormOpTraining(unittest.TestCase): self.dshape = [self.N, self.C, self.H, self.W] self.atol = 1e-3 - def _build_program(self, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def _build_program( + self, place, layout, seed, sync_bn=False, only_forward=False + ): """Build program.""" main = fluid.Program() startup = fluid.Program() @@ -70,17 +67,20 @@ class TestSyncBatchNormOpTraining(unittest.TestCase): use_cudnn = self.dtype == np.float16 with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - data = fluid.layers.data(name='input', - shape=self.dshape, - dtype=self.dtype, - append_batch_size=False) + data = fluid.layers.data( + name='input', + shape=self.dshape, + dtype=self.dtype, + append_batch_size=False, + ) conv = fluid.layers.conv2d( input=data, num_filters=32, filter_size=1, param_attr=fluid.ParamAttr(name='conv2d_weight'), bias_attr=False, - use_cudnn=use_cudnn) + use_cudnn=use_cudnn, + ) bn = fluid.layers.batch_norm( conv, param_attr=fluid.ParamAttr(name='bn_scale'), @@ -88,7 +88,8 @@ class TestSyncBatchNormOpTraining(unittest.TestCase): moving_mean_name='bn_moving_mean', moving_variance_name='bn_moving_variance', data_layout=layout, - is_test=only_forward) + is_test=only_forward, + ) if core.is_compiled_with_rocm(): bn = fluid.layers.cast(bn, 'float32') else: @@ -108,42 +109,59 @@ class TestSyncBatchNormOpTraining(unittest.TestCase): seed = 10 os.environ['FLAGS_cudnn_deterministic'] = "1" scope = core.Scope() - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 - data = create_or_get_tensor(scope, "input", - OpTest.np_dtype_to_fluid_dtype(data), place) + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 + data = create_or_get_tensor( + scope, "input", OpTest.np_dtype_to_fluid_dtype(data), place + ) # Single-GPU, N = 32 per GPU - main, startup, outs = self._build_program(place, layout, seed, False, - only_forward) + main, startup, outs = self._build_program( + place, layout, seed, False, only_forward + ) exe = fluid.Executor(place) exe.run(startup) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - bn_fetches = exe.run(program=main, - feed={'input': data}, - fetch_list=fetch_names) + bn_fetches = exe.run( + program=main, feed={'input': data}, fetch_list=fetch_names + ) ##################################################################### # Multi-GPUs, self.N / core.get_cuda_device_count() per GPU assert core.get_cuda_device_count() > 1 - main, startup, outs = self._build_program(place, layout, seed, True, - only_forward) + main, startup, outs = self._build_program( + place, layout, seed, True, only_forward + ) exe = fluid.Executor(place) exe.run(startup) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others for nm in fetch_names: @@ -155,24 +173,31 @@ class TestSyncBatchNormOpTraining(unittest.TestCase): build_strategy.memory_optimize = False comp_prog = compiler.CompiledProgram(main).with_data_parallel( outs[0].name if not only_forward else None, - build_strategy=build_strategy) - sync_bn_fetches = exe.run(program=comp_prog, - feed={'input': data}, - fetch_list=fetch_names) + build_strategy=build_strategy, + ) + sync_bn_fetches = exe.run( + program=comp_prog, feed={'input': data}, fetch_list=fetch_names + ) for i in range(1, len(sync_bn_fetches)): bn_val = bn_fetches[i] sync_bn_val = sync_bn_fetches[i] if sync_bn_val.shape != bn_val.shape: - sync_bn_val = sync_bn_val[:bn_val.shape[0]] - np.testing.assert_allclose(bn_val, - sync_bn_val, - rtol=1e-05, - atol=self.atol, - err_msg='Output (' + fetch_names[i] + - ') has diff. \n' + '\nBN ' + - str(bn_val) + '\n' + 'Sync BN ' + - str(sync_bn_val)) + sync_bn_val = sync_bn_val[: bn_val.shape[0]] + np.testing.assert_allclose( + bn_val, + sync_bn_val, + rtol=1e-05, + atol=self.atol, + err_msg='Output (' + + fetch_names[i] + + ') has diff. \n' + + '\nBN ' + + str(bn_val) + + '\n' + + 'Sync BN ' + + str(sync_bn_val), + ) def test_train(self): """Test training.""" @@ -210,15 +235,15 @@ class TestFP16SyncBatchNormOpTraining(TestSyncBatchNormOpTraining): class TestDygraphSyncBatchNormAPIError(unittest.TestCase): - def test_errors(self): if not core.is_compiled_with_cuda(): return with program_guard(Program(), Program()): my_sync_batch_norm = paddle.nn.SyncBatchNorm(10) - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CUDAPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CUDAPlace(0) + ) self.assertRaises(TypeError, my_sync_batch_norm, x1) # the input dtype of SyncBatchNorm must be float16 or float32 or float64 @@ -228,36 +253,39 @@ class TestDygraphSyncBatchNormAPIError(unittest.TestCase): class TestConvertSyncBatchNorm(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_cuda(): return with program_guard(Program(), Program()): - compare_model = paddle.nn.Sequential(paddle.nn.Conv2D(3, 5, 3), - paddle.nn.BatchNorm2D(5), - paddle.nn.BatchNorm2D(5)) + compare_model = paddle.nn.Sequential( + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), + paddle.nn.BatchNorm2D(5), + ) model = paddle.nn.Sequential( - paddle.nn.Conv2D(3, 5, 3), paddle.nn.BatchNorm2D(5), + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), paddle.nn.BatchNorm2D( 5, weight_attr=fluid.ParamAttr(name='bn.scale'), - bias_attr=fluid.ParamAttr(name='bn.bias'))) + bias_attr=fluid.ParamAttr(name='bn.bias'), + ), + ) model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model) for idx, sublayer in enumerate(compare_model.sublayers()): if isinstance(sublayer, paddle.nn.BatchNorm2D): self.assertEqual( - isinstance(model[idx], paddle.nn.SyncBatchNorm), True) + isinstance(model[idx], paddle.nn.SyncBatchNorm), True + ) class TestConvertSyncBatchNormCast1(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_cuda(): return class Net(nn.Layer): - def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2D(3, 5, 3) @@ -282,7 +310,6 @@ class TestConvertSyncBatchNormCast1(unittest.TestCase): class TestConvertSyncBatchNormCase2(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_cuda(): return @@ -290,16 +317,19 @@ class TestConvertSyncBatchNormCase2(unittest.TestCase): with fluid.dygraph.guard(fluid.CUDAPlace(0)): class SyBNNet(paddle.nn.Layer): - def __init__(self, in_ch=3, out_ch=3, dirate=1): super(SyBNNet, self).__init__() self.bn_s1 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( paddle.nn.BatchNorm3D( out_ch, weight_attr=paddle.ParamAttr( - regularizer=paddle.regularizer.L2Decay(0.)))) + regularizer=paddle.regularizer.L2Decay(0.0) + ), + ) + ) self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( - paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC')) + paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC') + ) def forward(self, x): x = self.bn_s1(x) @@ -307,15 +337,17 @@ class TestConvertSyncBatchNormCase2(unittest.TestCase): return out class BNNet(paddle.nn.Layer): - def __init__(self, in_ch=3, out_ch=3, dirate=1): super(BNNet, self).__init__() self.bn_s1 = paddle.nn.BatchNorm3D( out_ch, weight_attr=paddle.ParamAttr( - regularizer=paddle.regularizer.L2Decay(0.))) + regularizer=paddle.regularizer.L2Decay(0.0) + ), + ) self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( - paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC')) + paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC') + ) def forward(self, x): x = self.bn_s1(x) @@ -333,12 +365,16 @@ class TestConvertSyncBatchNormCase2(unittest.TestCase): bn_out.numpy(), sybn_out.numpy(), rtol=1e-05, - err_msg='Output has diff. \n' + '\nBN ' + - str(bn_out.numpy()) + '\n' + 'Sync BN ' + str(sybn_out.numpy())) + err_msg='Output has diff. \n' + + '\nBN ' + + str(bn_out.numpy()) + + '\n' + + 'Sync BN ' + + str(sybn_out.numpy()), + ) class TestDygraphSyncBatchNormDataFormatError(unittest.TestCase): - def test_errors(self): if not core.is_compiled_with_cuda(): return diff --git a/python/paddle/fluid/tests/unittests/test_take.py b/python/paddle/fluid/tests/unittests/test_take.py index 27ccee81d76..f713d777c16 100644 --- a/python/paddle/fluid/tests/unittests/test_take.py +++ b/python/paddle/fluid/tests/unittests/test_take.py @@ -21,7 +21,6 @@ from paddle.fluid import Program, program_guard class TestTakeAPI(unittest.TestCase): - def set_mode(self): self.mode = 'raise' @@ -32,41 +31,46 @@ class TestTakeAPI(unittest.TestCase): def set_input(self): self.input_shape = [3, 4] self.index_shape = [2, 3] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-4, 2).reshape(self.index_shape).astype( - self.index_dtype) + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-4, 2).reshape(self.index_shape).astype(self.index_dtype) + ) def setUp(self): self.set_mode() self.set_dtype() self.set_input() - self.place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def test_static_graph(self): paddle.enable_static() startup_program = Program() train_program = Program() with program_guard(startup_program, train_program): - x = fluid.data(name='input', - dtype=self.input_dtype, - shape=self.input_shape) - index = fluid.data(name='index', - dtype=self.index_dtype, - shape=self.index_shape) + x = fluid.data( + name='input', dtype=self.input_dtype, shape=self.input_shape + ) + index = fluid.data( + name='index', dtype=self.index_dtype, shape=self.index_shape + ) out = paddle.take(x, index, mode=self.mode) exe = fluid.Executor(self.place) - st_result = exe.run(fluid.default_main_program(), - feed={ - 'input': self.input_np, - 'index': self.index_np - }, - fetch_list=out) + st_result = exe.run( + fluid.default_main_program(), + feed={'input': self.input_np, 'index': self.index_np}, + fetch_list=out, + ) np.testing.assert_allclose( st_result[0], - np.take(self.input_np, self.index_np, mode=self.mode)) + np.take(self.input_np, self.index_np, mode=self.mode), + ) def test_dygraph(self): paddle.disable_static(self.place) @@ -75,7 +79,8 @@ class TestTakeAPI(unittest.TestCase): dy_result = paddle.take(x, index, mode=self.mode) np.testing.assert_allclose( np.take(self.input_np, self.index_np, mode=self.mode), - dy_result.numpy()) + dy_result.numpy(), + ) class TestTakeInt32(TestTakeAPI): @@ -109,11 +114,12 @@ class TestTakeTypeError(TestTakeAPI): """Argument 'index' must be Tensor""" paddle.enable_static() with program_guard(Program()): - x = fluid.data(name='input', - dtype=self.input_dtype, - shape=self.input_shape) - self.assertRaises(TypeError, paddle.take, x, self.index_np, - self.mode) + x = fluid.data( + name='input', dtype=self.input_dtype, shape=self.input_shape + ) + self.assertRaises( + TypeError, paddle.take, x, self.index_np, self.mode + ) def test_dygraph_type_error(self): paddle.disable_static(self.place) @@ -124,12 +130,12 @@ class TestTakeTypeError(TestTakeAPI): """Data type of argument 'index' must be in [paddle.int32, paddle.int64]""" paddle.enable_static() with program_guard(Program()): - x = fluid.data(name='input', - dtype='float64', - shape=self.input_shape) - index = fluid.data(name='index', - dtype='float32', - shape=self.index_shape) + x = fluid.data( + name='input', dtype='float64', shape=self.input_shape + ) + index = fluid.data( + name='index', dtype='float32', shape=self.index_shape + ) self.assertRaises(TypeError, paddle.take, x, index, self.mode) def test_dygraph_dtype_error(self): @@ -152,29 +158,36 @@ class TestTakeModeRaisePos(unittest.TestCase): def set_input(self): self.input_shape = [3, 4] self.index_shape = [5, 6] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-10, 20).reshape(self.index_shape).astype( - self.index_dtype) # positive indices are out of range + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-10, 20) + .reshape(self.index_shape) + .astype(self.index_dtype) + ) # positive indices are out of range def setUp(self): self.set_mode() self.set_dtype() self.set_input() - self.place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def test_static_index_error(self): """When the index is out of range, an error is reported directly through `paddle.index_select`""" paddle.enable_static() with program_guard(Program()): - x = fluid.data(name='input', - dtype=self.input_dtype, - shape=self.input_shape) - index = fluid.data(name='index', - dtype=self.index_dtype, - shape=self.index_shape) + x = fluid.data( + name='input', dtype=self.input_dtype, shape=self.input_shape + ) + index = fluid.data( + name='index', dtype=self.index_dtype, shape=self.index_shape + ) self.assertRaises(ValueError, paddle.index_select, x, index) def test_dygraph_index_error(self): @@ -197,17 +210,24 @@ class TestTakeModeRaiseNeg(TestTakeModeRaisePos): def set_input(self): self.input_shape = [3, 4] self.index_shape = [5, 6] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-20, 10).reshape(self.index_shape).astype( - self.index_dtype) # negative indices are out of range + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-20, 10) + .reshape(self.index_shape) + .astype(self.index_dtype) + ) # negative indices are out of range def setUp(self): self.set_mode() self.set_dtype() self.set_input() - self.place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) class TestTakeModeWrap(TestTakeAPI): @@ -219,10 +239,14 @@ class TestTakeModeWrap(TestTakeAPI): def set_input(self): self.input_shape = [3, 4] self.index_shape = [5, 8] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-20, 20).reshape(self.index_shape).astype( - self.index_dtype) # Both ends of the index are out of bounds + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-20, 20) + .reshape(self.index_shape) + .astype(self.index_dtype) + ) # Both ends of the index are out of bounds class TestTakeModeClip(TestTakeAPI): @@ -234,10 +258,14 @@ class TestTakeModeClip(TestTakeAPI): def set_input(self): self.input_shape = [3, 4] self.index_shape = [5, 8] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-20, 20).reshape(self.index_shape).astype( - self.index_dtype) # Both ends of the index are out of bounds + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-20, 20) + .reshape(self.index_shape) + .astype(self.index_dtype) + ) # Both ends of the index are out of bounds if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_take_along_axis_op.py b/python/paddle/fluid/tests/unittests/test_take_along_axis_op.py index 947ea1fb6f2..a7a65aec326 100644 --- a/python/paddle/fluid/tests/unittests/test_take_along_axis_op.py +++ b/python/paddle/fluid/tests/unittests/test_take_along_axis_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestTakeAlongAxisOp(OpTest): - def setUp(self): self.init_data() self.op_type = "take_along_axis" @@ -49,14 +48,14 @@ class TestTakeAlongAxisOp(OpTest): self.x_type = "float64" self.x_shape = (5, 5, 5) self.index_type = "int32" - self.index = np.array([[[1]], [[1]], [[2]], [[4]], - [[3]]]).astype(self.index_type) + self.index = np.array([[[1]], [[1]], [[2]], [[4]], [[3]]]).astype( + self.index_type + ) self.axis = 2 self.axis_type = "int64" class TestCase1(TestTakeAlongAxisOp): - def init_data(self): self.x_type = "float64" self.x_shape = (5, 5, 5) @@ -67,7 +66,6 @@ class TestCase1(TestTakeAlongAxisOp): class TestTakeAlongAxisAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [3, 3] @@ -86,13 +84,12 @@ class TestTakeAlongAxisAPI(unittest.TestCase): index = paddle.fluid.data('Index', self.index_shape, "int64") out = paddle.take_along_axis(x, index, self.axis) exe = paddle.static.Executor(self.place[0]) - res = exe.run(feed={ - 'X': self.x_np, - 'Index': self.index_np - }, - fetch_list=[out]) + res = exe.run( + feed={'X': self.x_np, 'Index': self.index_np}, fetch_list=[out] + ) out_ref = np.array( - np.take_along_axis(self.x_np, self.index_np, self.axis)) + np.take_along_axis(self.x_np, self.index_np, self.axis) + ) for out in res: np.testing.assert_allclose(out, out_ref, rtol=0.001) @@ -102,19 +99,20 @@ class TestTakeAlongAxisAPI(unittest.TestCase): self.index = paddle.to_tensor(self.index_np) out = paddle.take_along_axis(x_tensor, self.index, self.axis) out_ref = np.array( - np.take_along_axis(self.x_np, self.index_np, self.axis)) + np.take_along_axis(self.x_np, self.index_np, self.axis) + ) np.testing.assert_allclose(out.numpy(), out_ref, rtol=0.001) paddle.enable_static() class TestTakeAlongAxisAPICase1(TestTakeAlongAxisAPI): - def setUp(self): np.random.seed(0) self.shape = [2, 2] self.index_shape = [4, 2] - self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, - 0]]).astype('int64') + self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, 0]]).astype( + 'int64' + ) self.x_np = np.random.random(self.shape).astype(np.float32) self.place = [paddle.CPUPlace()] self.axis = 0 diff --git a/python/paddle/fluid/tests/unittests/test_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_target_assign_op.py index d51444da406..103f28a2024 100644 --- a/python/paddle/fluid/tests/unittests/test_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_target_assign_op.py @@ -36,16 +36,23 @@ def gen_match_and_neg_indices(num_prior, gt_lod, neg_lod): ret_ids = set([i for i in range(num_prior)]) - set(ids) l = neg_lod[n] neg_ids = random.sample(ret_ids, l) - neg_indices[offset:offset + - neg_lod[n], :] = np.array(neg_ids).astype('int32').reshape( - l, 1) + neg_indices[offset : offset + neg_lod[n], :] = ( + np.array(neg_ids).astype('int32').reshape(l, 1) + ) offset += neg_lod[n] return match_indices, neg_indices -def target_assign(encoded_box, gt_label, match_indices, neg_indices, gt_lod, - neg_lod, mismatch_value): +def target_assign( + encoded_box, + gt_label, + match_indices, + neg_indices, + gt_lod, + neg_lod, + mismatch_value, +): batch_size, num_prior = match_indices.shape # init target bbox @@ -75,7 +82,7 @@ def target_assign(encoded_box, gt_label, match_indices, neg_indices, gt_lod, trg_label_wt[i][col_ids] = 1.0 # set target label weight to 1.0 for the negative samples if neg_indices is not None: - neg_ids = neg_indices[neg_offset:neg_offset + neg_lod[i]] + neg_ids = neg_indices[neg_offset : neg_offset + neg_lod[i]] trg_label_wt[i][neg_ids] = 1.0 # update offset gt_offset += gt_lod[i] @@ -85,7 +92,6 @@ def target_assign(encoded_box, gt_label, match_indices, neg_indices, gt_lod, class TestTargetAssginFloatType(OpTest): - def setUp(self): self.op_type = "target_assign" num_prior = 120 @@ -97,15 +103,23 @@ class TestTargetAssginFloatType(OpTest): num_gt = sum(gt_lod) encoded_box = np.random.random((num_gt, num_prior, 4)).astype('float32') - gt_label = np.random.randint(num_class, - size=(num_gt, 1)).astype('int32') + gt_label = np.random.randint(num_class, size=(num_gt, 1)).astype( + 'int32' + ) match_indices, neg_indices = gen_match_and_neg_indices( - num_prior, gt_lod, neg_lod) - - out, out_wt, _, _ = target_assign(encoded_box, gt_label, match_indices, - neg_indices, gt_lod, neg_lod, - mismatch_value) + num_prior, gt_lod, neg_lod + ) + + out, out_wt, _, _ = target_assign( + encoded_box, + gt_label, + match_indices, + neg_indices, + gt_lod, + neg_lod, + mismatch_value, + ) # assign regression targets x = encoded_box @@ -124,7 +138,6 @@ class TestTargetAssginFloatType(OpTest): class TestTargetAssginIntType(OpTest): - def setUp(self): self.op_type = "target_assign" num_prior = 120 @@ -136,15 +149,23 @@ class TestTargetAssginIntType(OpTest): num_gt = sum(gt_lod) encoded_box = np.random.random((num_gt, num_prior, 4)).astype('float32') - gt_label = np.random.randint(num_class, - size=(num_gt, 1)).astype('int32') + gt_label = np.random.randint(num_class, size=(num_gt, 1)).astype( + 'int32' + ) match_indices, neg_indices = gen_match_and_neg_indices( - num_prior, gt_lod, neg_lod) - - _, _, out, out_wt, = target_assign(encoded_box, gt_label, match_indices, - neg_indices, gt_lod, neg_lod, - mismatch_value) + num_prior, gt_lod, neg_lod + ) + + _, _, out, out_wt, = target_assign( + encoded_box, + gt_label, + match_indices, + neg_indices, + gt_lod, + neg_lod, + mismatch_value, + ) # assign cassification argets x = np.reshape(gt_label, (num_gt, 1, 1)) diff --git a/python/paddle/fluid/tests/unittests/test_tdm_child_op.py b/python/paddle/fluid/tests/unittests/test_tdm_child_op.py index 5c17b737eef..95c49c904db 100644 --- a/python/paddle/fluid/tests/unittests/test_tdm_child_op.py +++ b/python/paddle/fluid/tests/unittests/test_tdm_child_op.py @@ -52,15 +52,15 @@ def create_tdm_tree(): class TestTDMChildOp(OpTest): - def setUp(self): self.__class__.op_type = "tdm_child" self.config() tree_info = create_tdm_tree() tree_info_np = np.array(tree_info).astype(self.info_type) - x_np = np.random.randint(low=0, high=26, - size=self.x_shape).astype(self.x_type) + x_np = np.random.randint(low=0, high=26, size=self.x_shape).astype( + self.x_type + ) children_res = [] leaf_mask_res = [] for batch in x_np: @@ -100,9 +100,8 @@ class TestTDMChildOp(OpTest): class TestCase1(TestTDMChildOp): - def config(self): - """check int int64_t """ + """check int int64_t""" self.x_shape = (10, 20) self.child_shape = (10, 20, 2) self.x_type = 'int32' @@ -110,9 +109,8 @@ class TestCase1(TestTDMChildOp): class TestCase2(TestTDMChildOp): - def config(self): - """check int64_t int64_t """ + """check int64_t int64_t""" self.x_shape = (10, 20) self.child_shape = (10, 20, 2) self.x_type = 'int64' @@ -120,9 +118,8 @@ class TestCase2(TestTDMChildOp): class TestCase3(TestTDMChildOp): - def config(self): - """check int64 int32 """ + """check int64 int32""" self.x_shape = (10, 20) self.child_shape = (10, 20, 2) self.x_type = 'int64' @@ -130,9 +127,8 @@ class TestCase3(TestTDMChildOp): class TestCase4(TestTDMChildOp): - def config(self): - """check large shape """ + """check large shape""" self.x_shape = (100, 20) self.child_shape = (100, 20, 2) self.x_type = 'int32' @@ -140,7 +136,6 @@ class TestCase4(TestTDMChildOp): class TestTDMChildShape(unittest.TestCase): - def test_shape(self): x = fluid.layers.data(name='x', shape=[1], dtype='int32', lod_level=1) tdm_tree_info = create_tdm_tree() @@ -150,17 +145,21 @@ class TestTDMChildShape(unittest.TestCase): x=x, node_nums=26, child_nums=2, - param_attr=fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(tree_info_np))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + tree_info_np + ) + ), + ) place = fluid.CPUPlace() exe = fluid.Executor(place=place) exe.run(fluid.default_startup_program()) feed = { - 'x': - np.array([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], - [12]]).astype('int32') + 'x': np.array( + [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12]] + ).astype('int32') } exe.run(feed=feed) diff --git a/python/paddle/fluid/tests/unittests/test_tdm_sampler_op.py b/python/paddle/fluid/tests/unittests/test_tdm_sampler_op.py index 1d576038e70..9c9ce120780 100644 --- a/python/paddle/fluid/tests/unittests/test_tdm_sampler_op.py +++ b/python/paddle/fluid/tests/unittests/test_tdm_sampler_op.py @@ -21,27 +21,41 @@ import paddle.fluid as fluid def create_tdm_travel(): - tree_travel = [[1, 3, 7, 14], [1, 3, 7, 15], [1, 3, 8, 16], [1, 3, 8, 17], - [1, 4, 9, 18], [1, 4, 9, 19], [1, 4, 10, 20], [1, 4, 10, 21], - [2, 5, 11, 22], [2, 5, 11, 23], [2, 5, 12, 24], - [2, 5, 12, 25], [2, 6, 13, 0]] + tree_travel = [ + [1, 3, 7, 14], + [1, 3, 7, 15], + [1, 3, 8, 16], + [1, 3, 8, 17], + [1, 4, 9, 18], + [1, 4, 9, 19], + [1, 4, 10, 20], + [1, 4, 10, 21], + [2, 5, 11, 22], + [2, 5, 11, 23], + [2, 5, 12, 24], + [2, 5, 12, 25], + [2, 6, 13, 0], + ] return tree_travel def create_tdm_layer(): - tree_layer = [[1, 2], [3, 4, 5, 6], [7, 8, 9, 10, 11, 12, 13], - [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]] + tree_layer = [ + [1, 2], + [3, 4, 5, 6], + [7, 8, 9, 10, 11, 12, 13], + [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25], + ] return tree_layer type_dict = { "int32": int(core.VarDesc.VarType.INT32), - "int64": int(core.VarDesc.VarType.INT64) + "int64": int(core.VarDesc.VarType.INT64), } class TestTDMSamplerOp(OpTest): - def setUp(self): self.__class__.op_type = "tdm_sampler" self.config() @@ -50,8 +64,9 @@ class TestTDMSamplerOp(OpTest): self.tree_layer = create_tdm_layer() output_0 = self.x_shape[0] - output_1 = len(self.neg_samples_num_list) + \ - np.sum(self.neg_samples_num_list) + output_1 = len(self.neg_samples_num_list) + np.sum( + self.neg_samples_num_list + ) self.output_shape = (output_0, output_1) self.layer_sample_nums = [1 + i for i in self.neg_samples_num_list] @@ -68,8 +83,9 @@ class TestTDMSamplerOp(OpTest): layer_np = np.array(tree_layer_flat).astype(self.tree_dtype) layer_np = layer_np.reshape([-1, 1]) - self.x_np = np.random.randint(low=0, high=13, - size=self.x_shape).astype(self.x_type) + self.x_np = np.random.randint(low=0, high=13, size=self.x_shape).astype( + self.x_type + ) out = np.random.random(self.output_shape).astype(self.out_dtype) label = np.random.random(self.output_shape).astype(self.out_dtype) @@ -80,7 +96,7 @@ class TestTDMSamplerOp(OpTest): 'output_positive': True, 'layer_offset_lod': tree_layer_offset_lod, 'seed': 0, - 'dtype': type_dict[self.out_dtype] + 'dtype': type_dict[self.out_dtype], } self.inputs = {'X': self.x_np, 'Travel': travel_np, 'Layer': layer_np} self.outputs = {'Out': out, 'Labels': label, 'Mask': mask} @@ -128,7 +144,8 @@ class TestTDMSamplerOp(OpTest): positive_travel.append(sampling_res_list[0]) label_sampling_res = label_res[batch_ids][ - start_offset:end_offset] + start_offset:end_offset + ] mask_sampling_res = mask_res[batch_ids][start_offset:end_offset] # check unique @@ -136,8 +153,12 @@ class TestTDMSamplerOp(OpTest): assert len(set(sampling_res_list)) == len( sampling_res_list ), "len(set(sampling_res_list)): {}, len(sampling_res_list): {} , sample_res: {}, label_res:{}, mask_res: {}".format( - len(set(sampling_res_list)), len(sampling_res_list), - sampling_res, label_sampling_res, mask_sampling_res) + len(set(sampling_res_list)), + len(sampling_res_list), + sampling_res, + label_sampling_res, + mask_sampling_res, + ) # check legal layer_node = self.tree_layer[layer_idx] layer_node.append(0) @@ -145,8 +166,12 @@ class TestTDMSamplerOp(OpTest): assert ( sample in layer_node ), "sample: {}, layer_node: {} , sample_res: {}, label_res: {}, mask_res:{}".format( - sample, layer_node, sampling_res, label_sampling_res, - mask_sampling_res) + sample, + layer_node, + sampling_res, + label_sampling_res, + mask_sampling_res, + ) # check label label_flag = 1 @@ -158,15 +183,16 @@ class TestTDMSamplerOp(OpTest): assert not np.sum( mask_sampling_res[padding_index] ), "np.sum(mask_sampling_res[padding_index]): {} ".format( - np.sum(mask_sampling_res[padding_index])) + np.sum(mask_sampling_res[padding_index]) + ) start_offset = end_offset # check travel legal - assert self.tree_travel[int( - self.x_np[batch_ids])] == positive_travel + assert ( + self.tree_travel[int(self.x_np[batch_ids])] == positive_travel + ) class TestCase1(TestTDMSamplerOp): - def config(self): """test input int64""" self.neg_samples_num_list = [0, 0, 0, 0] @@ -177,7 +203,6 @@ class TestCase1(TestTDMSamplerOp): class TestCase2(TestTDMSamplerOp): - def config(self): """test dtype int64""" self.neg_samples_num_list = [0, 0, 0, 0] @@ -188,7 +213,6 @@ class TestCase2(TestTDMSamplerOp): class TestCase3(TestTDMSamplerOp): - def config(self): """test all dtype int64""" self.neg_samples_num_list = [0, 0, 0, 0] @@ -199,7 +223,6 @@ class TestCase3(TestTDMSamplerOp): class TestCase4(TestTDMSamplerOp): - def config(self): """test one neg""" self.neg_samples_num_list = [1, 1, 1, 1] @@ -210,7 +233,6 @@ class TestCase4(TestTDMSamplerOp): class TestCase5(TestTDMSamplerOp): - def config(self): """test normal neg""" self.neg_samples_num_list = [1, 2, 3, 4] @@ -221,7 +243,6 @@ class TestCase5(TestTDMSamplerOp): class TestCase6(TestTDMSamplerOp): - def config(self): """test huge batchsize""" self.neg_samples_num_list = [1, 2, 3, 4] @@ -232,7 +253,6 @@ class TestCase6(TestTDMSamplerOp): class TestCase7(TestTDMSamplerOp): - def config(self): """test full neg""" self.neg_samples_num_list = [1, 3, 6, 11] @@ -243,7 +263,6 @@ class TestCase7(TestTDMSamplerOp): class TestTDMSamplerShape(unittest.TestCase): - def test_shape(self): x = fluid.layers.data(name='x', shape=[1], dtype='int32', lod_level=1) tdm_tree_travel = create_tdm_travel() @@ -267,23 +286,41 @@ class TestTDMSamplerShape(unittest.TestCase): leaf_node_num, tree_travel_attr=fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - travel_array)), - tree_layer_attr=fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(layer_array)), + travel_array + ) + ), + tree_layer_attr=fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer(layer_array) + ), output_positive=True, output_list=True, seed=0, tree_dtype='int32', - dtype='int32') + dtype='int32', + ) place = fluid.CPUPlace() exe = fluid.Executor(place=place) exe.run(fluid.default_startup_program()) feed = { - 'x': - np.array([[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], - [11], [12]]).astype('int32') + 'x': np.array( + [ + [0], + [1], + [2], + [3], + [4], + [5], + [6], + [7], + [8], + [9], + [10], + [11], + [12], + ] + ).astype('int32') } exe.run(feed=feed) diff --git a/python/paddle/fluid/tests/unittests/test_teacher_student_sigmoid_loss_op.py b/python/paddle/fluid/tests/unittests/test_teacher_student_sigmoid_loss_op.py index 228ecbb06ea..88ea8f647c4 100644 --- a/python/paddle/fluid/tests/unittests/test_teacher_student_sigmoid_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_teacher_student_sigmoid_loss_op.py @@ -23,7 +23,7 @@ import paddle.fluid as fluid class TestTeacherStudentSigmoidLossOp(OpTest): """ - Test teacher_student_sigmoid_loss with discrete one-hot labels. + Test teacher_student_sigmoid_loss with discrete one-hot labels. """ def setUp(self): @@ -31,12 +31,14 @@ class TestTeacherStudentSigmoidLossOp(OpTest): batch_size = 100 num_classes = 1 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.uniform(0, 2, (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.uniform(0, 2, (batch_size, num_classes)).astype( + "float64" + ), } outs = [] for index, label in enumerate(self.inputs["Label"]): @@ -46,11 +48,22 @@ class TestTeacherStudentSigmoidLossOp(OpTest): elif label < 0.0: outs.append(max(x, 0.0) - x + log(1.0 + exp(-abs(x)))) elif label < 1.0: - outs.append(max(x, 0.0) + log(1.0 + exp(-abs(x))) + \ - max(x, 0.0) - x * label + log(1.0 + exp(-abs(x)))) + outs.append( + max(x, 0.0) + + log(1.0 + exp(-abs(x))) + + max(x, 0.0) + - x * label + + log(1.0 + exp(-abs(x))) + ) else: - outs.append(max(x, 0.0) - x + log(1.0 + exp(-abs(x))) + \ - max(x, 0.0) - x * (label - 1.0) + log(1.0 + exp(-abs(x)))) + outs.append( + max(x, 0.0) + - x + + log(1.0 + exp(-abs(x))) + + max(x, 0.0) + - x * (label - 1.0) + + log(1.0 + exp(-abs(x))) + ) self.outputs = {'Y': np.array(outs)} def test_check_output(self): @@ -61,9 +74,7 @@ class TestTeacherStudentSigmoidLossOp(OpTest): class TestTeacherStudentSigmoidLossInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): input = [512, 1] label = fluid.data(name='label', shape=[None, 1], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_temporal_shift_op.py b/python/paddle/fluid/tests/unittests/test_temporal_shift_op.py index ea6d9abc372..13dc115f6bc 100644 --- a/python/paddle/fluid/tests/unittests/test_temporal_shift_op.py +++ b/python/paddle/fluid/tests/unittests/test_temporal_shift_op.py @@ -25,13 +25,14 @@ def temporal_shift(x, seg_num, shift_ratio, data_format): x = np.transpose(x, (0, 3, 1, 2)) shape = x.shape reshape_x = x.reshape((-1, seg_num, shape[1], shape[2], shape[3])) - pad_x = np.pad(reshape_x, ((0, 0), (1, 1), (0, 0), (0, 0), (0, 0)), - 'constant') + pad_x = np.pad( + reshape_x, ((0, 0), (1, 1), (0, 0), (0, 0), (0, 0)), 'constant' + ) c1 = int(shape[1] * shift_ratio) c2 = int(shape[1] * 2 * shift_ratio) slice1 = pad_x[:, :seg_num, :c1, :, :] - slice2 = pad_x[:, 2:seg_num + 2, c1:c2, :, :] - slice3 = pad_x[:, 1:seg_num + 1, c2:, :, :] + slice2 = pad_x[:, 2 : seg_num + 2, c1:c2, :, :] + slice3 = pad_x[:, 1 : seg_num + 1, c2:, :, :] concat_x = np.concatenate([slice1, slice2, slice3], axis=2) out = concat_x.reshape(shape) if data_format == "NHWC": @@ -40,7 +41,6 @@ def temporal_shift(x, seg_num, shift_ratio, data_format): class TestTemporalShift(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'temporal_shift' @@ -50,15 +50,16 @@ class TestTemporalShift(OpTest): self.attrs = { "seg_num": self.seg_num, "shift_ratio": self.shift_ratio, - "data_format": self.data_format + "data_format": self.data_format, } self.inputs = { "X": x, } - output = temporal_shift(x, self.seg_num, self.shift_ratio, - self.data_format) + output = temporal_shift( + x, self.seg_num, self.shift_ratio, self.data_format + ) self.outputs = {"Out": output} self.python_out_sig = ["Out"] @@ -77,7 +78,6 @@ class TestTemporalShift(OpTest): class TestTemporalShift2(TestTemporalShift): - def initTestCase(self): self.x_shape = (4, 9, 7, 7) self.seg_num = 2 @@ -86,7 +86,6 @@ class TestTemporalShift2(TestTemporalShift): class TestTemporalShift3(TestTemporalShift): - def initTestCase(self): self.x_shape = (3, 10, 5, 5) self.seg_num = 1 @@ -95,7 +94,6 @@ class TestTemporalShift3(TestTemporalShift): class TestTemporalShift4(TestTemporalShift): - def initTestCase(self): self.x_shape = (6, 5, 5, 4) self.seg_num = 3 @@ -103,10 +101,10 @@ class TestTemporalShift4(TestTemporalShift): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestTemporalShiftFP16(TestTemporalShift): - def initTestCase(self): self.x_shape = (3, 10, 5, 5) self.seg_num = 1 @@ -126,32 +124,29 @@ class TestTemporalShiftFP16(TestTemporalShift): class TestTemporalShiftAPI(unittest.TestCase): - def test_api(self): input = paddle.randn([6, 4, 2, 2]) - out = paddle.fluid.layers.temporal_shift(x=input, - seg_num=2, - shift_ratio=0.2) + out = paddle.fluid.layers.temporal_shift( + x=input, seg_num=2, shift_ratio=0.2 + ) - out_from_function = paddle.nn.functional.temporal_shift(x=input, - seg_num=2, - shift_ratio=0.2) + out_from_function = paddle.nn.functional.temporal_shift( + x=input, seg_num=2, shift_ratio=0.2 + ) # dygraph with paddle.fluid.dygraph.guard(): input = paddle.randn([6, 4, 2, 2]) - out = paddle.nn.functional.temporal_shift(x=input, - seg_num=2, - shift_ratio=0.2) + out = paddle.nn.functional.temporal_shift( + x=input, seg_num=2, shift_ratio=0.2 + ) def test_error(self): - def attr_data_format(): input = paddle.randn([6, 4, 2, 2]) - out = paddle.nn.functional.temporal_shift(x=input, - seg_num=2, - shift_ratio=0.2, - data_format="HWC") + out = paddle.nn.functional.temporal_shift( + x=input, seg_num=2, shift_ratio=0.2, data_format="HWC" + ) self.assertRaises(ValueError, attr_data_format) diff --git a/python/paddle/fluid/tests/unittests/test_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor.py index 2226805b125..ef8e60df9d1 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor.py @@ -20,7 +20,6 @@ import numbers class TestTensorPtr(unittest.TestCase): - def test_tensor_ptr(self): t = core.Tensor() np_arr = np.zeros([2, 3]) @@ -29,11 +28,17 @@ class TestTensorPtr(unittest.TestCase): class TestTensor(unittest.TestCase): - def setUp(self): self.support_dtypes = [ - 'bool', 'uint8', 'int8', 'int16', 'int32', 'int64', 'float16', - 'float32', 'float64' + 'bool', + 'uint8', + 'int8', + 'int16', + 'int32', + 'int64', + 'float16', + 'float32', + 'float64', ] def test_int_tensor(self): @@ -79,10 +84,9 @@ class TestTensor(unittest.TestCase): scope = core.Scope() var = scope.var("int8_tensor") cpu_tensor = var.get_tensor() - tensor_array = np.random.randint(-127, - high=128, - size=[100, 200], - dtype=np.int8) + tensor_array = np.random.randint( + -127, high=128, size=[100, 200], dtype=np.int8 + ) place = core.CPUPlace() cpu_tensor.set(tensor_array, place) cpu_tensor_array_2 = np.array(cpu_tensor) @@ -90,15 +94,15 @@ class TestTensor(unittest.TestCase): if core.is_compiled_with_cuda(): cuda_tensor = var.get_tensor() - tensor_array = np.random.randint(-127, - high=128, - size=[100, 200], - dtype=np.int8) + tensor_array = np.random.randint( + -127, high=128, size=[100, 200], dtype=np.int8 + ) place = core.CUDAPlace(0) cuda_tensor.set(tensor_array, place) cuda_tensor_array_2 = np.array(cuda_tensor) - self.assertAlmostEqual(cuda_tensor_array_2.all(), - tensor_array.all()) + self.assertAlmostEqual( + cuda_tensor_array_2.all(), tensor_array.all() + ) def test_int_lod_tensor(self): place = core.CPUPlace() @@ -207,10 +211,13 @@ class TestTensor(unittest.TestCase): shape = [3, 3, 3] tensor._set_dims(shape) - tensor_array = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], [16, 17, 18]], - [[19, 20, 21], [22, 23, 24], - [25, 26, 27]]]).astype(dtype) + tensor_array = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]], + ] + ).astype(dtype) tensor.set(tensor_array, place) n1 = tensor[1] @@ -283,21 +290,24 @@ class TestTensor(unittest.TestCase): tensor = var.get_tensor() dtype = core.VarDesc.VarType.FP32 self.assertTrue( - isinstance(tensor._mutable_data(place, dtype), numbers.Integral)) + isinstance(tensor._mutable_data(place, dtype), numbers.Integral) + ) if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) self.assertTrue( - isinstance(tensor._mutable_data(place, dtype), - numbers.Integral)) + isinstance(tensor._mutable_data(place, dtype), numbers.Integral) + ) place = core.CUDAPinnedPlace() self.assertTrue( - isinstance(tensor._mutable_data(place, dtype), - numbers.Integral)) + isinstance(tensor._mutable_data(place, dtype), numbers.Integral) + ) places = fluid.cuda_pinned_places() self.assertTrue( - isinstance(tensor._mutable_data(places[0], dtype), - numbers.Integral)) + isinstance( + tensor._mutable_data(places[0], dtype), numbers.Integral + ) + ) def test_tensor_set_fp16(self): array = np.random.random((300, 500)).astype("float16") diff --git a/python/paddle/fluid/tests/unittests/test_tensor_array_to_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor_array_to_tensor.py index 0291052a7e0..21f90556f7a 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_array_to_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_array_to_tensor.py @@ -50,8 +50,10 @@ class TestLoDTensorArrayConcat(unittest.TestCase): program = fluid.Program() block = program.global_block() - input_arr = block.create_var(name="tmp_lod_tensor_array", - type=core.VarDesc.VarType.LOD_TENSOR_ARRAY) + input_arr = block.create_var( + name="tmp_lod_tensor_array", + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ) input_arr.persistable = True input_arr_var = scope.var('tmp_lod_tensor_array') input_tensor_array = input_arr_var.get_lod_tensor_array() @@ -75,28 +77,30 @@ class TestLoDTensorArrayConcat(unittest.TestCase): y_out_index = block.create_var(name="OutIndex") y_out_index.persistable = True - y_grad_arr = block.create_var(name='Out@GRAD', - dtype='float32', - shape=[11]) + y_grad_arr = block.create_var( + name='Out@GRAD', dtype='float32', shape=[11] + ) y_grad_arr.persistable = True y_grad = scope.var('Out@GRAD') y_grad_tensor = y_grad.get_tensor() y_grad_tensor.set(random_grad, cpu) - op = block.append_op(type=self.op_type, - inputs={"X": input_arr}, - outputs={ - "Out": y_out, - "OutIndex": y_out_index - }, - attrs=self.attrs) - - out_grad = block.create_var(name="tmp_lod_tensor_array@GRAD", - type=core.VarDesc.VarType.LOD_TENSOR_ARRAY) + op = block.append_op( + type=self.op_type, + inputs={"X": input_arr}, + outputs={"Out": y_out, "OutIndex": y_out_index}, + attrs=self.attrs, + ) + + out_grad = block.create_var( + name="tmp_lod_tensor_array@GRAD", + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ) out_grad.persistable = True grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - op.desc, set(), []) + op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -115,13 +119,14 @@ class TestLoDTensorArrayConcat(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace()) out = exe.run(program, fetch_list=fetch_list, scope=scope) - #print ("index: ", np.array(out[1])) + # print ("index: ", np.array(out[1])) # test forward tensor_res = np.array(out[0]) tensor_res_out_idx = np.array(out[1]) - tensor_gt = np.array([0] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - dtype='float32') + tensor_gt = np.array( + [0] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype='float32' + ) self.assertEqual(len(tensor_res), len(tensor_gt)) self.assertEqual(len(tensor_res_out_idx), 10) @@ -144,13 +149,16 @@ class TestLoDTensorArrayConcat(unittest.TestCase): for i in range(len(grad_tensor_array)): if i == 0: self.assertEqual( - np.array(grad_tensor_array[i])[0], np.array(random_grad[i])) + np.array(grad_tensor_array[i])[0], np.array(random_grad[i]) + ) self.assertEqual( np.array(grad_tensor_array[i])[1], - np.array(random_grad[i + 1])) + np.array(random_grad[i + 1]), + ) if i == 1: - self.assertEqual(np.array(grad_tensor_array[i]), - np.array(random_grad[i + 1])) + self.assertEqual( + np.array(grad_tensor_array[i]), np.array(random_grad[i + 1]) + ) class TestLoDTensorArrayStack(unittest.TestCase): @@ -162,12 +170,14 @@ class TestLoDTensorArrayStack(unittest.TestCase): self.inputs = [ np.random.rand(2, 3, 4).astype("float32"), np.random.rand(2, 3, 4).astype("float32"), - np.random.rand(2, 3, 4).astype("float32") + np.random.rand(2, 3, 4).astype("float32"), ] self.outputs = [ np.stack(self.inputs, axis=self.attrs["axis"]), - np.array([x.shape[self.attrs["axis"]] for x in self.inputs], - dtype="int32") + np.array( + [x.shape[self.attrs["axis"]] for x in self.inputs], + dtype="int32", + ), ] self.input_grads = [np.ones_like(x) for x in self.inputs] self.set_program() @@ -184,7 +194,8 @@ class TestLoDTensorArrayStack(unittest.TestCase): x = fluid.layers.assign(x) fluid.layers.array_write(x, idx + i, array) output, output_index = fluid.layers.tensor_array_to_tensor( - input=array, **self.attrs) + input=array, **self.attrs + ) loss = fluid.layers.reduce_sum(output) fluid.backward.append_backward(loss) self.output_vars = [output, output_index] @@ -192,10 +203,11 @@ class TestLoDTensorArrayStack(unittest.TestCase): def run_check(self, executor, scope): executor.run(self.program, scope=scope) for i, output in enumerate(self.outputs): - np.allclose(np.array( - scope.var(self.output_vars[i].name).get_tensor()), - output, - atol=0) + np.allclose( + np.array(scope.var(self.output_vars[i].name).get_tensor()), + output, + atol=0, + ) tensor_array_grad = scope.var(self.array.name).get_lod_tensor_array() for i, input_grad in enumerate(self.input_grads): np.allclose(np.array(tensor_array_grad[i]), input_grad, atol=0) @@ -215,7 +227,6 @@ class TestLoDTensorArrayStack(unittest.TestCase): class TestTensorArrayToTensorAPI(unittest.TestCase): - def _test_case(self, inp1, inp2): x0 = fluid.layers.assign(inp1) x0.stop_gradient = False @@ -226,10 +237,20 @@ class TestTensorArrayToTensorAPI(unittest.TestCase): fluid.layers.array_write(x0, i, array) fluid.layers.array_write(x1, i + 1, array) output_stack, output_index_stack = fluid.layers.tensor_array_to_tensor( - input=array, axis=1, use_stack=True) - output_concat, output_index_concat = fluid.layers.tensor_array_to_tensor( - input=array, axis=1, use_stack=False) - return output_stack, output_index_stack, output_concat, output_index_concat + input=array, axis=1, use_stack=True + ) + ( + output_concat, + output_index_concat, + ) = fluid.layers.tensor_array_to_tensor( + input=array, axis=1, use_stack=False + ) + return ( + output_stack, + output_index_stack, + output_concat, + output_index_concat, + ) def test_case(self): inp0 = np.random.rand(2, 3, 4).astype("float32") @@ -269,7 +290,8 @@ class TestTensorArrayToTensorAPI(unittest.TestCase): self.assertTrue(fluid.layers.array_length(array), 10) last = fluid.layers.fill_constant(shape=[1], dtype='int64', value=9) np.testing.assert_array_equal( - fluid.layers.array_read(array, last).numpy(), inp0) + fluid.layers.array_read(array, last).numpy(), inp0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_tensor_copy_from.py b/python/paddle/fluid/tests/unittests/test_tensor_copy_from.py index 6c38d234990..3f2b00e8e73 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_copy_from.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_copy_from.py @@ -19,7 +19,6 @@ from paddle.fluid.core import LoDTensor as Tensor class TestTensorCopyFrom(unittest.TestCase): - def test_main(self): place = paddle.CPUPlace() np_value = np.random.random(size=[10, 30]).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_tensor_fill_.py b/python/paddle/fluid/tests/unittests/test_tensor_fill_.py index e52c761fd4e..7998cbcc19e 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_fill_.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_fill_.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class TensorFill_Test(unittest.TestCase): - def setUp(self): self.shape = [32, 32] @@ -36,15 +35,16 @@ class TensorFill_Test(unittest.TestCase): paddle.set_device('cpu') else: paddle.set_device('gpu') - np_arr = np.reshape(np.array(range(np.prod(self.shape))), - self.shape) + np_arr = np.reshape( + np.array(range(np.prod(self.shape))), self.shape + ) for dtype in typelist: - var = 1. + var = 1.0 tensor = paddle.to_tensor(np_arr, place=p, dtype=dtype) target = tensor.numpy() target[...] = var - tensor.fill_(var) #var type is basic type in typelist + tensor.fill_(var) # var type is basic type in typelist self.assertEqual((tensor.numpy() == target).all(), True) def test_tensor_fill_true(self): @@ -64,8 +64,9 @@ class TensorFill_Test(unittest.TestCase): paddle.set_device('cpu') else: paddle.set_device('gpu') - np_arr = np.reshape(np.array(range(np.prod(self.shape))), - self.shape) + np_arr = np.reshape( + np.array(range(np.prod(self.shape))), self.shape + ) for dtype in typelist: var = int(1) tensor = paddle.to_tensor(np_arr, place=p, dtype=dtype) @@ -85,7 +86,6 @@ class TensorFill_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_test_errors(self): - def test_list(): x = paddle.to_tensor([2, 3, 4]) x.fill_([1]) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_.py b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_.py index 3eaf01642a6..b3c68f93204 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_.py @@ -20,12 +20,13 @@ from paddle.fluid.framework import _test_eager_guard class TensorFillDiagonal_Test(unittest.TestCase): - def func_dim2_normal(self): - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, - 1]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, - 0]]).astype('float32') + expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1]]).astype( + 'float32' + ) + expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0]]).astype( + 'float32' + ) typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -46,10 +47,12 @@ class TensorFillDiagonal_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_normal(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -59,10 +62,12 @@ class TensorFillDiagonal_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_offset(self): - expected_np = np.array([[2, 2, 1], [2, 2, 2], [2, 2, - 2]]).astype('float32') - expected_grad = np.array([[1, 1, 0], [1, 1, 1], [1, 1, - 1]]).astype('float32') + expected_np = np.array([[2, 2, 1], [2, 2, 2], [2, 2, 2]]).astype( + 'float32' + ) + expected_grad = np.array([[1, 1, 0], [1, 1, 1], [1, 1, 1]]).astype( + 'float32' + ) typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -83,10 +88,12 @@ class TensorFillDiagonal_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_offset(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -96,8 +103,9 @@ class TensorFillDiagonal_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_bool(self): - expected_np = np.array([[False, True, True], [True, False, True], - [True, True, False]]) + expected_np = np.array( + [[False, True, True], [True, False, True], [True, True, False]] + ) typelist = ['bool'] places = [fluid.CPUPlace()] @@ -122,12 +130,28 @@ class TensorFillDiagonal_Test(unittest.TestCase): self.func_bool() def func_dim2_unnormal_wrap(self): - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1], [2, 2, 2], - [1, 2, 2], [2, 1, 2], [2, 2, - 1]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0], [1, 1, 1], - [0, 1, 1], [1, 0, 1], [1, 1, - 0]]).astype('float32') + expected_np = np.array( + [ + [1, 2, 2], + [2, 1, 2], + [2, 2, 1], + [2, 2, 2], + [1, 2, 2], + [2, 1, 2], + [2, 2, 1], + ] + ).astype('float32') + expected_grad = np.array( + [ + [0, 1, 1], + [1, 0, 1], + [1, 1, 0], + [1, 1, 1], + [0, 1, 1], + [1, 0, 1], + [1, 1, 0], + ] + ).astype('float32') typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -148,10 +172,12 @@ class TensorFillDiagonal_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_unnormal_wrap(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -161,12 +187,28 @@ class TensorFillDiagonal_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim2_unnormal_unwrap(self): - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1], [2, 2, 2], - [2, 2, 2], [2, 2, 2], [2, 2, - 2]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0], [1, 1, 1], - [1, 1, 1], [1, 1, 1], [1, 1, - 1]]).astype('float32') + expected_np = np.array( + [ + [1, 2, 2], + [2, 1, 2], + [2, 2, 1], + [2, 2, 2], + [2, 2, 2], + [2, 2, 2], + [2, 2, 2], + ] + ).astype('float32') + expected_grad = np.array( + [ + [0, 1, 1], + [1, 0, 1], + [1, 1, 0], + [1, 1, 1], + [1, 1, 1], + [1, 1, 1], + [1, 1, 1], + ] + ).astype('float32') typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -187,10 +229,12 @@ class TensorFillDiagonal_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_unnormal_unwrap(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -200,14 +244,20 @@ class TensorFillDiagonal_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim_larger2_normal(self): - expected_np = np.array([[[1, 2, 2], [2, 2, 2], [2, 2, 2]], - [[2, 2, 2], [2, 1, 2], [2, 2, 2]], - [[2, 2, 2], [2, 2, 2], [2, 2, - 1]]]).astype('float32') - expected_grad = np.array([[[0, 1, 1], [1, 1, 1], [1, 1, 1]], - [[1, 1, 1], [1, 0, 1], [1, 1, 1]], - [[1, 1, 1], [1, 1, 1], - [1, 1, 0]]]).astype('float32') + expected_np = np.array( + [ + [[1, 2, 2], [2, 2, 2], [2, 2, 2]], + [[2, 2, 2], [2, 1, 2], [2, 2, 2]], + [[2, 2, 2], [2, 2, 2], [2, 2, 1]], + ] + ).astype('float32') + expected_grad = np.array( + [ + [[0, 1, 1], [1, 1, 1], [1, 1, 1]], + [[1, 1, 1], [1, 0, 1], [1, 1, 1]], + [[1, 1, 1], [1, 1, 1], [1, 1, 0]], + ] + ).astype('float32') typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -228,10 +278,12 @@ class TensorFillDiagonal_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim_larger2_normal(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor.py index 3aa2ac96123..8dd9b327f78 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _enable_legacy_dygraph class TensorFillDiagTensor_Test(unittest.TestCase): - def setUp(self): self.typelist = ['float32', 'float64', 'int32', 'int64'] self.places = [fluid.CPUPlace()] @@ -30,10 +29,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): def test_dim2(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1], - [2, 2, 2]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0], - [1, 1, 1]]).astype('float32') + expected_np = np.array( + [[1, 2, 2], [2, 1, 2], [2, 2, 1], [2, 2, 2]] + ).astype('float32') + expected_grad = np.array( + [[0, 1, 1], [1, 0, 1], [1, 1, 0], [1, 1, 1]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -41,8 +42,8 @@ class TensorFillDiagTensor_Test(unittest.TestCase): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((3, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((3,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -51,18 +52,22 @@ class TensorFillDiagTensor_Test(unittest.TestCase): loss.backward() self.assertEqual( - (ny.numpy().astype('float32') == expected_np).all(), True) + (ny.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_dim2_offset_1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - expected_np = np.array([[2, 2, 2], [1, 2, 2], [2, 1, 2], - [2, 2, 1]]).astype('float32') - expected_grad = np.array([[1, 1, 1], [0, 1, 1], [1, 0, 1], - [1, 1, 0]]).astype('float32') + expected_np = np.array( + [[2, 2, 2], [1, 2, 2], [2, 1, 2], [2, 2, 1]] + ).astype('float32') + expected_grad = np.array( + [[1, 1, 1], [0, 1, 1], [1, 0, 1], [1, 1, 0]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -70,8 +75,8 @@ class TensorFillDiagTensor_Test(unittest.TestCase): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((3, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((3,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -80,18 +85,22 @@ class TensorFillDiagTensor_Test(unittest.TestCase): loss.backward() self.assertEqual( - (ny.numpy().astype('float32') == expected_np).all(), True) + (ny.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_dim2_offset1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - expected_np = np.array([[2, 1, 2], [2, 2, 1], [2, 2, 2], - [2, 2, 2]]).astype('float32') - expected_grad = np.array([[1, 0, 1], [1, 1, 0], [1, 1, 1], - [1, 1, 1]]).astype('float32') + expected_np = np.array( + [[2, 1, 2], [2, 2, 1], [2, 2, 2], [2, 2, 2]] + ).astype('float32') + expected_grad = np.array( + [[1, 0, 1], [1, 1, 0], [1, 1, 1], [1, 1, 1]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -99,8 +108,8 @@ class TensorFillDiagTensor_Test(unittest.TestCase): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((2, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((2,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -109,30 +118,48 @@ class TensorFillDiagTensor_Test(unittest.TestCase): loss.backward() self.assertEqual( - (ny.numpy().astype('float32') == expected_np).all(), True) + (ny.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_dim4(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - expected_np = np.array([[[[0, 3], [2, 2], [2, 2]], - [[2, 2], [1, 4], [2, 2]], - [[2, 2], [2, 2], [2, 5]], - [[2, 2], [2, 2], [2, 2]]], - [[[6, 9], [2, 2], [2, 2]], - [[2, 2], [7, 10], [2, 2]], - [[2, 2], [2, 2], [8, 11]], - [[2, 2], [2, 2], [2, 2]]]]).astype('float32') - expected_grad = np.array([[[[0, 0], [1, 1], [1, 1]], - [[1, 1], [0, 0], [1, 1]], - [[1, 1], [1, 1], [0, 0]], - [[1, 1], [1, 1], [1, 1]]], - [[[0, 0], [1, 1], [1, 1]], - [[1, 1], [0, 0], [1, 1]], - [[1, 1], [1, 1], [0, 0]], - [[1, 1], [1, 1], [1, 1]]]]).astype('float32') + expected_np = np.array( + [ + [ + [[0, 3], [2, 2], [2, 2]], + [[2, 2], [1, 4], [2, 2]], + [[2, 2], [2, 2], [2, 5]], + [[2, 2], [2, 2], [2, 2]], + ], + [ + [[6, 9], [2, 2], [2, 2]], + [[2, 2], [7, 10], [2, 2]], + [[2, 2], [2, 2], [8, 11]], + [[2, 2], [2, 2], [2, 2]], + ], + ] + ).astype('float32') + expected_grad = np.array( + [ + [ + [[0, 0], [1, 1], [1, 1]], + [[1, 1], [0, 0], [1, 1]], + [[1, 1], [1, 1], [0, 0]], + [[1, 1], [1, 1], [1, 1]], + ], + [ + [[0, 0], [1, 1], [1, 1]], + [[1, 1], [0, 0], [1, 1]], + [[1, 1], [1, 1], [0, 0]], + [[1, 1], [1, 1], [1, 1]], + ], + ] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -140,9 +167,10 @@ class TensorFillDiagTensor_Test(unittest.TestCase): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.to_tensor(np.arange(12).reshape(2, 2, 3), - dtype=dtype) - var = (np.random.random() + 1) + v = paddle.to_tensor( + np.arange(12).reshape(2, 2, 3), dtype=dtype + ) + var = np.random.random() + 1 x = paddle.ones((2, 4, 3, 2), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -151,10 +179,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): loss.backward() self.assertEqual( - (ny.numpy().astype('float32') == expected_np).all(), True) + (ny.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_largedim(self): @@ -165,7 +195,8 @@ class TensorFillDiagTensor_Test(unittest.TestCase): paddle.set_device('gpu') for dtype in self.typelist: v = paddle.arange(bsdim * fsdim, dtype=dtype).reshape( - (bsdim, fsdim)) + (bsdim, fsdim) + ) y = paddle.ones((bsdim, fsdim, fsdim), dtype=dtype) y.stop_gradient = False y = y * 2 diff --git a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor_.py b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor_.py index eb223dc6317..17be54720c9 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor_.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor_.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TensorFillDiagTensor_Test(unittest.TestCase): - def setUp(self): self.typelist = ['float32', 'float64', 'int32', 'int64'] self.places = [fluid.CPUPlace()] @@ -29,10 +28,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): self.places.append(fluid.CUDAPlace(0)) def func_dim2(self): - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1], - [2, 2, 2]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0], - [1, 1, 1]]).astype('float32') + expected_np = np.array( + [[1, 2, 2], [2, 1, 2], [2, 2, 1], [2, 2, 2]] + ).astype('float32') + expected_grad = np.array( + [[0, 1, 1], [1, 0, 1], [1, 1, 0], [1, 1, 1]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -40,8 +41,8 @@ class TensorFillDiagTensor_Test(unittest.TestCase): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((3, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((3,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -50,10 +51,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -63,10 +66,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim2_offset_1(self): - expected_np = np.array([[2, 2, 2], [1, 2, 2], [2, 1, 2], - [2, 2, 1]]).astype('float32') - expected_grad = np.array([[1, 1, 1], [0, 1, 1], [1, 0, 1], - [1, 1, 0]]).astype('float32') + expected_np = np.array( + [[2, 2, 2], [1, 2, 2], [2, 1, 2], [2, 2, 1]] + ).astype('float32') + expected_grad = np.array( + [[1, 1, 1], [0, 1, 1], [1, 0, 1], [1, 1, 0]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -74,8 +79,8 @@ class TensorFillDiagTensor_Test(unittest.TestCase): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((3, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((3,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -84,10 +89,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_offset_1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -97,10 +104,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim2_offset1(self): - expected_np = np.array([[2, 1, 2], [2, 2, 1], [2, 2, 2], - [2, 2, 2]]).astype('float32') - expected_grad = np.array([[1, 0, 1], [1, 1, 0], [1, 1, 1], - [1, 1, 1]]).astype('float32') + expected_np = np.array( + [[2, 1, 2], [2, 2, 1], [2, 2, 2], [2, 2, 2]] + ).astype('float32') + expected_grad = np.array( + [[1, 0, 1], [1, 1, 0], [1, 1, 1], [1, 1, 1]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -108,8 +117,8 @@ class TensorFillDiagTensor_Test(unittest.TestCase): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((2, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((2,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -118,10 +127,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_offset1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -131,22 +142,38 @@ class TensorFillDiagTensor_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim4(self): - expected_np = np.array([[[[0, 3], [2, 2], [2, 2]], - [[2, 2], [1, 4], [2, 2]], - [[2, 2], [2, 2], [2, 5]], - [[2, 2], [2, 2], [2, 2]]], - [[[6, 9], [2, 2], [2, 2]], - [[2, 2], [7, 10], [2, 2]], - [[2, 2], [2, 2], [8, 11]], - [[2, 2], [2, 2], [2, 2]]]]).astype('float32') - expected_grad = np.array([[[[0, 0], [1, 1], [1, 1]], - [[1, 1], [0, 0], [1, 1]], - [[1, 1], [1, 1], [0, 0]], - [[1, 1], [1, 1], [1, 1]]], - [[[0, 0], [1, 1], [1, 1]], - [[1, 1], [0, 0], [1, 1]], - [[1, 1], [1, 1], [0, 0]], - [[1, 1], [1, 1], [1, 1]]]]).astype('float32') + expected_np = np.array( + [ + [ + [[0, 3], [2, 2], [2, 2]], + [[2, 2], [1, 4], [2, 2]], + [[2, 2], [2, 2], [2, 5]], + [[2, 2], [2, 2], [2, 2]], + ], + [ + [[6, 9], [2, 2], [2, 2]], + [[2, 2], [7, 10], [2, 2]], + [[2, 2], [2, 2], [8, 11]], + [[2, 2], [2, 2], [2, 2]], + ], + ] + ).astype('float32') + expected_grad = np.array( + [ + [ + [[0, 0], [1, 1], [1, 1]], + [[1, 1], [0, 0], [1, 1]], + [[1, 1], [1, 1], [0, 0]], + [[1, 1], [1, 1], [1, 1]], + ], + [ + [[0, 0], [1, 1], [1, 1]], + [[1, 1], [0, 0], [1, 1]], + [[1, 1], [1, 1], [0, 0]], + [[1, 1], [1, 1], [1, 1]], + ], + ] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -154,9 +181,10 @@ class TensorFillDiagTensor_Test(unittest.TestCase): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.to_tensor(np.arange(12).reshape(2, 2, 3), - dtype=dtype) - var = (np.random.random() + 1) + v = paddle.to_tensor( + np.arange(12).reshape(2, 2, 3), dtype=dtype + ) + var = np.random.random() + 1 x = paddle.ones((2, 4, 3, 2), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -165,10 +193,12 @@ class TensorFillDiagTensor_Test(unittest.TestCase): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_func_dim4(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -178,14 +208,15 @@ class TensorFillDiagTensor_Test(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_largedim(self): - #large dim only test on gpu because the cpu version is too slow for ci test, and the memory is limited + # large dim only test on gpu because the cpu version is too slow for ci test, and the memory is limited if len(self.places) > 1: bsdim = 1024 fsdim = 128 paddle.set_device('gpu') for dtype in self.typelist: v = paddle.arange(bsdim * fsdim, dtype=dtype).reshape( - (bsdim, fsdim)) + (bsdim, fsdim) + ) y = paddle.ones((bsdim, fsdim, fsdim), dtype=dtype) y.stop_gradient = False y = y * 2 diff --git a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py index 0a249d00283..8e7de658969 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py @@ -23,7 +23,6 @@ import paddle.fluid.core as core class SimpleNet(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleNet, self).__init__() self.linear1 = nn.Linear(in_size, in_size) @@ -42,7 +41,6 @@ class SimpleNet(nn.Layer): class SimpleNetForStatic(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleNetForStatic, self).__init__() self.linear1 = nn.Linear(in_size, in_size) @@ -58,7 +56,6 @@ class SimpleNetForStatic(nn.Layer): class TestTensorRegisterHook(unittest.TestCase): - def setUp(self): self.seed = 2021 self.in_size = 10 @@ -69,13 +66,12 @@ class TestTensorRegisterHook(unittest.TestCase): self.devices.append("gpu") def func_hook_for_interior_var(self): - def run_double_hook_for_interior_var(double_hook, removed=False): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 3.]) - y = paddle.to_tensor([4., 5., 6., 7.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 3.0]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) x.stop_gradient = False y.stop_gradient = False @@ -83,7 +79,7 @@ class TestTensorRegisterHook(unittest.TestCase): w.stop_gradient = False helper = w.register_hook(double_hook) - z = paddle.to_tensor([1., 2., 3., 4.]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) z.stop_gradient = False o = z.matmul(w) @@ -100,18 +96,18 @@ class TestTensorRegisterHook(unittest.TestCase): np.testing.assert_array_equal(w.grad.numpy(), z.numpy()) # x.grad and y.grad are changed if run hook np.testing.assert_array_equal( - x.grad.numpy(), - z.numpy() * 2 if not removed else z.numpy()) + x.grad.numpy(), z.numpy() * 2 if not removed else z.numpy() + ) np.testing.assert_array_equal( - y.grad.numpy(), - z.numpy() * 2 if not removed else z.numpy()) + y.grad.numpy(), z.numpy() * 2 if not removed else z.numpy() + ) def run_print_hook_for_interior_var(print_hook, removed=False): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 3.]) - y = paddle.to_tensor([4., 5., 6., 7.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 3.0]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) x.stop_gradient = False y.stop_gradient = False @@ -119,7 +115,7 @@ class TestTensorRegisterHook(unittest.TestCase): w.stop_gradient = False helper = w.register_hook(print_hook) - z = paddle.to_tensor([1., 2., 3., 4.]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) z.stop_gradient = False o = z.matmul(w) @@ -167,13 +163,12 @@ class TestTensorRegisterHook(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_hook_for_leaf_var(self): - def run_double_hook_for_leaf_var(double_hook, removed=False): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 3.]) - y = paddle.to_tensor([4., 5., 6., 7.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 3.0]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) x.stop_gradient = False y.stop_gradient = False helper = y.register_hook(double_hook) @@ -181,7 +176,7 @@ class TestTensorRegisterHook(unittest.TestCase): w = x + y w.stop_gradient = False - z = paddle.to_tensor([1., 2., 3., 4.]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) z.stop_gradient = False o = z.matmul(w) @@ -198,8 +193,8 @@ class TestTensorRegisterHook(unittest.TestCase): np.testing.assert_array_equal(x.grad.numpy(), z.numpy()) # y.grad are changed if run hook np.testing.assert_array_equal( - y.grad.numpy(), - z.numpy() * 2 if not removed else z.numpy()) + y.grad.numpy(), z.numpy() * 2 if not removed else z.numpy() + ) # register hook run_double_hook_for_leaf_var(lambda grad: grad * 2) @@ -214,14 +209,14 @@ class TestTensorRegisterHook(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_hook_for_accumulated_grad_interior_var(self): - def run_double_hook_for_accumulated_grad_interior_var( - double_hook, removed=False): + double_hook, removed=False + ): for device in self.devices: paddle.set_device(device) - a = paddle.to_tensor([0., 1., 1., 2.]) - b = paddle.to_tensor([0., 0., 1., 2.]) + a = paddle.to_tensor([0.0, 1.0, 1.0, 2.0]) + b = paddle.to_tensor([0.0, 0.0, 1.0, 2.0]) a.stop_gradient = False b.stop_gradient = False @@ -232,8 +227,8 @@ class TestTensorRegisterHook(unittest.TestCase): helper2 = x.register_hook(double_hook) - y = paddle.to_tensor([4., 5., 6., 7.]) - z = paddle.to_tensor([1., 2., 3., 4.]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) y.stop_gradient = False z.stop_gradient = False @@ -251,21 +246,24 @@ class TestTensorRegisterHook(unittest.TestCase): o.backward() - base_grad = np.array([5., 9., 13., 19.]) + base_grad = np.array([5.0, 9.0, 13.0, 19.0]) # x.grad is not changed np.testing.assert_array_equal(x.grad.numpy(), base_grad) # b.grad is changed by x.hook np.testing.assert_array_equal( - b.grad.numpy(), base_grad * 2 if not removed else base_grad) + b.grad.numpy(), base_grad * 2 if not removed else base_grad + ) # a.grad is changed by x.hook and a.hook np.testing.assert_array_equal( - a.grad.numpy(), base_grad * 4 if not removed else base_grad) + a.grad.numpy(), base_grad * 4 if not removed else base_grad + ) # register hook run_double_hook_for_accumulated_grad_interior_var(lambda grad: grad * 2) # register hook and removed - run_double_hook_for_accumulated_grad_interior_var(lambda grad: grad * 2, - removed=True) + run_double_hook_for_accumulated_grad_interior_var( + lambda grad: grad * 2, removed=True + ) def test_hook_for_accumulated_grad_interior_var(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -275,19 +273,19 @@ class TestTensorRegisterHook(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_hook_for_accumulated_grad_leaf_var(self): - def run_double_hook_for_accumulated_grad_leaf_var( - double_hook, removed=False): + double_hook, removed=False + ): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 4.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 4.0]) x.stop_gradient = False helper = x.register_hook(double_hook) - y = paddle.to_tensor([4., 5., 6., 7.]) - z = paddle.to_tensor([1., 2., 3., 4.]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) y.stop_gradient = False z.stop_gradient = False @@ -304,16 +302,18 @@ class TestTensorRegisterHook(unittest.TestCase): o.backward() - base_grad = np.array([5., 9., 13., 19.]) + base_grad = np.array([5.0, 9.0, 13.0, 19.0]) # x.grad is changed by x.hook np.testing.assert_array_equal( - x.grad.numpy(), base_grad * 2 if not removed else base_grad) + x.grad.numpy(), base_grad * 2 if not removed else base_grad + ) # register hook run_double_hook_for_accumulated_grad_leaf_var(lambda grad: grad * 2) # register hook and removed - run_double_hook_for_accumulated_grad_leaf_var(lambda grad: grad * 2, - removed=True) + run_double_hook_for_accumulated_grad_leaf_var( + lambda grad: grad * 2, removed=True + ) def test_hook_for_accumulated_grad_leaf_var(self): with _test_eager_guard(): @@ -321,12 +321,9 @@ class TestTensorRegisterHook(unittest.TestCase): self.func_hook_for_accumulated_grad_leaf_var() def func_hook_in_model(self): - - def run_double_hook_in_model(data, - label, - hook=None, - register=False, - remove=False): + def run_double_hook_in_model( + data, label, hook=None, register=False, remove=False + ): for device in self.devices: paddle.seed(self.seed) paddle.set_device(device) @@ -341,22 +338,35 @@ class TestTensorRegisterHook(unittest.TestCase): loss = loss_fn(out, label) loss.backward() - return (ret1.grad.numpy(), net.linear1.weight.grad.numpy(), - net.linear1.bias.grad.numpy()) + return ( + ret1.grad.numpy(), + net.linear1.weight.grad.numpy(), + net.linear1.bias.grad.numpy(), + ) - data = np.random.uniform( - size=[self.batch_size, self.in_size]).astype('float32') + data = np.random.uniform(size=[self.batch_size, self.in_size]).astype( + 'float32' + ) label = np.random.uniform(size=[self.batch_size, 1]).astype('float32') # get original value ret1_grad, linear1_w_grad, linear1_b_grad = run_double_hook_in_model( - data, label) + data, label + ) # get value changed by hook - ret1_grad_hook, linear1_w_grad_hook, linear1_b_grad_hook = run_double_hook_in_model( - data, label, lambda grad: grad * 2, True) + ( + ret1_grad_hook, + linear1_w_grad_hook, + linear1_b_grad_hook, + ) = run_double_hook_in_model(data, label, lambda grad: grad * 2, True) # get value after removing hook - ret1_grad_rm, linear1_w_grad_rm, linear1_b_grad_rm = run_double_hook_in_model( - data, label, lambda grad: grad * 2, True, True) + ( + ret1_grad_rm, + linear1_w_grad_rm, + linear1_b_grad_rm, + ) = run_double_hook_in_model( + data, label, lambda grad: grad * 2, True, True + ) # compare original value and with hook np.testing.assert_array_equal(ret1_grad, ret1_grad_hook) @@ -376,16 +386,13 @@ class TestTensorRegisterHook(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_multiple_hooks_for_interior_var(self): - - def run_multiple_hooks_for_interior_var(device, - hooks, - remove1=False, - remove2=False, - remove3=False): + def run_multiple_hooks_for_interior_var( + device, hooks, remove1=False, remove2=False, remove3=False + ): paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 3.]) - y = paddle.to_tensor([4., 5., 6., 7.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 3.0]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) x.stop_gradient = False y.stop_gradient = False @@ -397,7 +404,7 @@ class TestTensorRegisterHook(unittest.TestCase): helper = w.register_hook(hook) helpers.append(helper) - z = paddle.to_tensor([1., 2., 3., 4.]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) z.stop_gradient = False o = z.matmul(w) @@ -420,35 +427,40 @@ class TestTensorRegisterHook(unittest.TestCase): for device in self.devices: z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks) + device, hooks + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z * 8) np.testing.assert_array_equal(y_grad, z * 8) z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks, remove1=True) + device, hooks, remove1=True + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z * 4) np.testing.assert_array_equal(y_grad, z * 4) z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks, remove2=True) + device, hooks, remove2=True + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z * 4) np.testing.assert_array_equal(y_grad, z * 4) z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks, remove3=True) + device, hooks, remove3=True + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z * 4) np.testing.assert_array_equal(y_grad, z * 4) z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks, remove1=True, remove2=True, remove3=True) + device, hooks, remove1=True, remove2=True, remove3=True + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z) @@ -462,7 +474,6 @@ class TestTensorRegisterHook(unittest.TestCase): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_hook_in_double_grad(self): - def double_print_hook(grad): grad = grad * 2 print(grad) @@ -478,10 +489,9 @@ class TestTensorRegisterHook(unittest.TestCase): y = x * x # Since y = x * x, dx = 2 * x - dx = paddle.grad(outputs=[y], - inputs=[x], - create_graph=True, - retain_graph=True)[0] + dx = paddle.grad( + outputs=[y], inputs=[x], create_graph=True, retain_graph=True + )[0] z = y + dx self.assertTrue(x.grad is None) @@ -508,7 +518,7 @@ class TestTensorRegisterHook(unittest.TestCase): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([1., 2., 3., 4.]) + x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) x.stop_gradient = False h = x.register_hook(lambda grad: grad * 2) @@ -524,7 +534,7 @@ class TestTensorRegisterHook(unittest.TestCase): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([1., 2., 3., 4.]) + x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) with self.assertRaises(RuntimeError): x.register_hook(lambda grad: grad * 2) @@ -541,9 +551,9 @@ class TestTensorRegisterHook(unittest.TestCase): main_program = paddle.static.Program() with paddle.static.scope_guard(paddle.static.Scope()): with paddle.static.program_guard(main_program, startup_program): - x = paddle.static.data(name='x', - shape=[None, self.in_size], - dtype='float32') + x = paddle.static.data( + name='x', shape=[None, self.in_size], dtype='float32' + ) net = SimpleNetForStatic(self.in_size, self.out_size) with self.assertRaises(AssertionError): @@ -554,10 +564,12 @@ class TestTensorRegisterHook(unittest.TestCase): def func_register_hook_in_dy2static_mode(self): net = SimpleNetForStatic(self.in_size, self.out_size) jit_net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec([None, self.in_size])]) + net, input_spec=[paddle.static.InputSpec([None, self.in_size])] + ) - data = np.random.uniform( - size=[self.batch_size, self.in_size]).astype('float32') + data = np.random.uniform(size=[self.batch_size, self.in_size]).astype( + 'float32' + ) data_t = paddle.to_tensor(data) with self.assertRaises(AssertionError): @@ -581,7 +593,6 @@ def global_void_hook(): class TestTensorRegisterBackwardHook(unittest.TestCase): - def setUp(self): self.devices = ["cpu"] if paddle.is_compiled_with_cuda(): @@ -591,7 +602,7 @@ class TestTensorRegisterBackwardHook(unittest.TestCase): global HOOK_INIT_VALUE global HOOK_IS_CALLED for device in self.devices: - x = paddle.to_tensor(5., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) x._register_backward_hook(global_void_hook) for i in range(5): y = paddle.pow(x, 4.0) @@ -610,7 +621,7 @@ class TestTensorRegisterBackwardHook(unittest.TestCase): self.func_register_backward_hook() def func_register_backward_hook_for_interior_var(self): - x = paddle.to_tensor(5., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) y = paddle.pow(x, 4.0) with self.assertRaises(ValueError): @@ -622,7 +633,7 @@ class TestTensorRegisterBackwardHook(unittest.TestCase): self.func_register_backward_hook_for_interior_var() def func_register_backward_hook_for_var_without_gradient(self): - x = paddle.to_tensor(5.) + x = paddle.to_tensor(5.0) y = paddle.pow(x, 4.0) with self.assertRaises(ValueError): @@ -635,7 +646,6 @@ class TestTensorRegisterBackwardHook(unittest.TestCase): class TestRegsiterBackwardFinalHook(unittest.TestCase): - def setUp(self): self.devices = ["cpu"] if paddle.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py index 594731fb08f..0ab01f41551 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py @@ -28,7 +28,6 @@ from paddle.fluid.framework import _test_eager_guard class TestTensorScalarTypePromotionDynamic(unittest.TestCase): - def check_operation(self, a, b, c, op): if op == '+': c_rlt = a + b diff --git a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_static.py b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_static.py index 127a36d7070..7949d2457e9 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_static.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_static.py @@ -29,7 +29,6 @@ from paddle.static import Program class TestTensorScalarTypePromotionStatic(unittest.TestCase): - def setUp(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_tensor_to_list.py b/python/paddle/fluid/tests/unittests/test_tensor_to_list.py index 899be8c996c..709ff84179f 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_to_list.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_to_list.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class TensorToListTest(unittest.TestCase): - def setUp(self): self.shape = [11, 25, 32, 43] @@ -31,8 +30,9 @@ class TensorToListTest(unittest.TestCase): places.append(fluid.CUDAPinnedPlace()) for p in places: - np_arr = np.reshape(np.array(range(np.prod(self.shape))), - self.shape) + np_arr = np.reshape( + np.array(range(np.prod(self.shape))), self.shape + ) expectlist = np_arr.tolist() t = paddle.to_tensor(np_arr, place=p) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_to_numpy.py b/python/paddle/fluid/tests/unittests/test_tensor_to_numpy.py index 1bd2f104854..30843455ca5 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_to_numpy.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_to_numpy.py @@ -18,13 +18,18 @@ import numpy as np class TensorToNumpyTest(unittest.TestCase): - def setUp(self): self.shape = [11, 25, 32, 43] def test_main(self): dtypes = [ - 'float32', 'float64', 'int32', 'int64', 'uint8', 'int8', 'bool' + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'int8', + 'bool', ] places = [fluid.CPUPlace()] @@ -36,7 +41,8 @@ class TensorToNumpyTest(unittest.TestCase): for dtype in dtypes: np_arr = np.reshape( np.array(range(np.prod(self.shape))).astype(dtype), - self.shape) + self.shape, + ) t = fluid.LoDTensor() t.set(np_arr, p) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_type_promotion.py b/python/paddle/fluid/tests/unittests/test_tensor_type_promotion.py index 4e31fec3543..f6a1f7e5def 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_type_promotion.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_type_promotion.py @@ -19,7 +19,6 @@ from paddle.fluid.framework import _test_eager_guard class TestTensorTypePromotion(unittest.TestCase): - def setUp(self): self.x = paddle.to_tensor([2, 3]) self.y = paddle.to_tensor([1.0, 2.0]) @@ -29,32 +28,36 @@ class TestTensorTypePromotion(unittest.TestCase): warnings.simplefilter("always") self.x + self.y self.assertTrue( - "The dtype of left and right variables are not the same" in str( - context[-1].message)) + "The dtype of left and right variables are not the same" + in str(context[-1].message) + ) def sub_operator(self): with warnings.catch_warnings(record=True) as context: warnings.simplefilter("always") self.x - self.y self.assertTrue( - "The dtype of left and right variables are not the same" in str( - context[-1].message)) + "The dtype of left and right variables are not the same" + in str(context[-1].message) + ) def mul_operator(self): with warnings.catch_warnings(record=True) as context: warnings.simplefilter("always") self.x * self.y self.assertTrue( - "The dtype of left and right variables are not the same" in str( - context[-1].message)) + "The dtype of left and right variables are not the same" + in str(context[-1].message) + ) def div_operator(self): with warnings.catch_warnings(record=True) as context: warnings.simplefilter("always") self.x / self.y self.assertTrue( - "The dtype of left and right variables are not the same" in str( - context[-1].message)) + "The dtype of left and right variables are not the same" + in str(context[-1].message) + ) def test_operator(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_tensor_uva.py b/python/paddle/fluid/tests/unittests/test_tensor_uva.py index 1261cf41b79..8fe03148f49 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_uva.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_uva.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph class TestTensorCopyFrom(unittest.TestCase): - def func_main(self): if paddle.fluid.core.is_compiled_with_cuda(): place = paddle.CPUPlace() @@ -36,12 +35,17 @@ class TestTensorCopyFrom(unittest.TestCase): class TestUVATensorFromNumpy(unittest.TestCase): - def func_uva_tensor_creation(self): if paddle.fluid.core.is_compiled_with_cuda(): dtype_list = [ - "int32", "int64", "float32", "float64", "float16", "int8", - "int16", "bool" + "int32", + "int64", + "float32", + "float64", + "float16", + "int8", + "int16", + "bool", ] for dtype in dtype_list: data = np.random.randint(10, size=[4, 5]).astype(dtype) @@ -64,9 +68,9 @@ class TestUVATensorFromNumpy(unittest.TestCase): slice_a = a[:, 5] tensor1 = paddle.to_tensor(slice_a) tensor2 = core.eager.to_uva_tensor(slice_a) - np.testing.assert_allclose(tensor1.numpy(), - tensor2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + tensor1.numpy(), tensor2.numpy(), rtol=1e-05 + ) def test_uva_tensor_creation(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_tensor_zero_.py b/python/paddle/fluid/tests/unittests/test_tensor_zero_.py index b8d55028ed2..2870ad8e75a 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_zero_.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_zero_.py @@ -20,7 +20,6 @@ from paddle.fluid.framework import _test_eager_guard class TensorFill_Test(unittest.TestCase): - def setUp(self): self.shape = [32, 32] @@ -32,8 +31,9 @@ class TensorFill_Test(unittest.TestCase): places.append(fluid.CUDAPinnedPlace()) for p in places: - np_arr = np.reshape(np.array(range(np.prod(self.shape))), - self.shape) + np_arr = np.reshape( + np.array(range(np.prod(self.shape))), self.shape + ) for dtype in typelist: tensor = paddle.to_tensor(np_arr, place=p, dtype=dtype) target = tensor.numpy() diff --git a/python/paddle/fluid/tests/unittests/test_tensordot.py b/python/paddle/fluid/tests/unittests/test_tensordot.py index c0fb398d830..8709772c18e 100644 --- a/python/paddle/fluid/tests/unittests/test_tensordot.py +++ b/python/paddle/fluid/tests/unittests/test_tensordot.py @@ -26,7 +26,7 @@ def tensordot_np(x, y, axes): # np.tensordot does not support empty axes if not axes: axes = 0 - if (isinstance(axes, (tuple, list))): + if isinstance(axes, (tuple, list)): if all(np.issubdtype(type(i), np.integer) for i in axes): axes = [axes, axes] else: @@ -43,7 +43,7 @@ def tensordot_np(x, y, axes): axes = [axes_x, axes_y] # np.tensordot does not support broadcast - if (isinstance(axes, (tuple, list))): + if isinstance(axes, (tuple, list)): axes_x, axes_y = axes else: axes_x = list(range(x.ndim - axes, x.ndim)) @@ -65,7 +65,6 @@ def tensordot_np(x, y, axes): class TestTensordotAPI(unittest.TestCase): - def setUp(self): self.set_place() self.set_dtype() @@ -90,60 +89,108 @@ class TestTensordotAPI(unittest.TestCase): self.y = np.random.random(self.y_shape).astype(self.dtype) def set_test_axes(self): - self.all_axes = [[[3, 2], [3]], [[2, 1, 0], [2, 1]], - [[1, 2, 0], [1, 3, 2]], [3, 0], [[], [0, 3, 1]], - [[2, 1, 0, 3], [2, 0, 1, 3]], [[3, 1, 2], [1, 3, 2, - 0]], - [[2, 1], [0, 2]], [[2, 0, 1, 3], [2]], - [[1, 2, 0, 3], [0, 2, 1]], [[2, 1, 3, 0], [1, 2, 3]], - [[2, 0, 1, 3], [3, 1, 0, 2]], [[0, 3], [0, 3, 2, 1]], - [[1, 3, 2, 0], [2, 1, 0, 3]], - [[1, 3, 2, 0], [1, 3, 2, 0]], [[1, 0, 2], [0, 1]], - [[2, 3, 0], [3, 1]], [[1, 3, 2, 0], [3, 0, 1, 2]], - [[3, 2, 1], [2, 0, 1]], [[0], []], - [[2, 3, 0], [1, 2, 0]], [[3, 0, 2, 1], [2, 1, 0, 3]], - [[3, 1, 2], [2, 3, 1]], [[1, 0, 2, 3], []], - [[1, 2], [1, 2, 3]], [[2, 0, 1, 3], [2, 0, 1]], - [[3, 1, 2], [1, 3, 2]], [[3, 1, 2, 0], [1, 2, 3, 0]], - [[0, 2, 3], [0, 1, 2]], [[3, 2, 0], [2, 0, 3, 1]], - [[2, 1, 0, 3], [3, 1, 2, 0]], - [[1, 2, 3, 0], [1, 3, 0, 2]], [[3, 0], [2, 1]], - [[0, 1, 3, 2], [0, 2, 1, 3]], [[1, 0], [2, 1, 3]], - [[1, 0, 3, 2], [2, 3, 0, 1]], [[1, 2], [3]], - [[1, 2, 3, 0], [3, 2, 1, 0]], - [[0, 3, 2, 1], [2, 1, 3, 0]], [0], - [[0, 2, 3], [3, 2, 0, 1]], [[1, 2, 3, 0], [3, 2, 1, - 0]], - [[3, 1], [3]], [[3, 2, 0, 1], [3, 2, 0]], - [[2, 3, 0, 1], [0, 3, 2]], [[1], [1, 3]], - [[1, 2], [2, 1, 0]], [[3, 1, 2], [3, 1, 0]], - [[1, 3], [3, 1, 2]], [[2, 0, 1, 3], [3, 1, 0, 2]], - [[1, 3, 0], [1, 3]], [[2, 3, 1], [1, 0, 2]], - [[1, 2, 0, 3], [0, 2, 1, 3]], [[2], [0, 1, 3]], - [[1], [1, 2]], [[1, 0, 2, 3], [3, 0, 1, 2]], - [[0, 1, 3, 2], [1, 3, 0, 2]], [[3, 0, 2, 1], [0, 2, - 3]], - [[1, 2, 0], [1, 2, 3]], [[1, 0, 3], [2, 3, 0]], - [[2, 3, 0], [3, 1, 0]], [[1, 3], [1, 0]], - [[2, 1, 0, 3], [2, 0, 3, 1]], [[3, 2, 0], [2, 1, 0]], - [[0, 1, 3], [0, 3, 1]], [[3, 1, 0], [3, 2, 1]], - [[3, 2], [3, 1]], [[3], [2, 1, 0]], [[1, 2, 3, 0], []], - [[1, 3, 2, 0], [3, 1, 2]], [[1], [0, 2]], - [[3, 2, 0], [3, 2, 0]], [[3], []], [[1, 0, 3], [2, 1]], - [[3, 1, 0, 2], [2, 3, 1, 0]], [[0, 1], [0, 3, 2]], - [[0, 2, 3], [0, 2, 1]], [[1, 3, 0], [3, 0, 2]], - [[3, 1, 2], [1, 2, 3]], [[3, 1, 2], [3, 1, 0]], - [[0, 3, 1, 2], [3, 2, 1, 0]], [[0, 3], [3, 2, 1]], - [[2, 3], [1, 3, 0]], [[0, 3, 2], [2, 0, 3, 1]], - [[2, 3], [1, 3]], [[3, 1, 2, 0], [2, 3, 1, 0]], - [[1, 0, 3, 2], [3, 0, 1, 2]], - [[3, 2, 1, 0], [0, 1, 3, 2]], [[3, 1, 2], [3]], - [[0, 1, 3, 2], [2, 3, 0, 1]], - [[1, 2, 3, 0], [1, 3, 0, 2]], [3, 1, 2], - [[3, 1, 2], [0, 3, 2]], [[2, 3, 0], [1, 2, 0]], - [[2, 0, 3], [2, 0]], [[3, 1, 0, 2], [3, 1, 0, 2]], - [[0, 1, 2], [2, 0, 1]], [[1, 0, 3], [2, 3, 0]], - [[2, 0, 1], [0, 1, 3]], [[2, 1], [0, 1, 3]]] + self.all_axes = [ + [[3, 2], [3]], + [[2, 1, 0], [2, 1]], + [[1, 2, 0], [1, 3, 2]], + [3, 0], + [[], [0, 3, 1]], + [[2, 1, 0, 3], [2, 0, 1, 3]], + [[3, 1, 2], [1, 3, 2, 0]], + [[2, 1], [0, 2]], + [[2, 0, 1, 3], [2]], + [[1, 2, 0, 3], [0, 2, 1]], + [[2, 1, 3, 0], [1, 2, 3]], + [[2, 0, 1, 3], [3, 1, 0, 2]], + [[0, 3], [0, 3, 2, 1]], + [[1, 3, 2, 0], [2, 1, 0, 3]], + [[1, 3, 2, 0], [1, 3, 2, 0]], + [[1, 0, 2], [0, 1]], + [[2, 3, 0], [3, 1]], + [[1, 3, 2, 0], [3, 0, 1, 2]], + [[3, 2, 1], [2, 0, 1]], + [[0], []], + [[2, 3, 0], [1, 2, 0]], + [[3, 0, 2, 1], [2, 1, 0, 3]], + [[3, 1, 2], [2, 3, 1]], + [[1, 0, 2, 3], []], + [[1, 2], [1, 2, 3]], + [[2, 0, 1, 3], [2, 0, 1]], + [[3, 1, 2], [1, 3, 2]], + [[3, 1, 2, 0], [1, 2, 3, 0]], + [[0, 2, 3], [0, 1, 2]], + [[3, 2, 0], [2, 0, 3, 1]], + [[2, 1, 0, 3], [3, 1, 2, 0]], + [[1, 2, 3, 0], [1, 3, 0, 2]], + [[3, 0], [2, 1]], + [[0, 1, 3, 2], [0, 2, 1, 3]], + [[1, 0], [2, 1, 3]], + [[1, 0, 3, 2], [2, 3, 0, 1]], + [[1, 2], [3]], + [[1, 2, 3, 0], [3, 2, 1, 0]], + [[0, 3, 2, 1], [2, 1, 3, 0]], + [0], + [[0, 2, 3], [3, 2, 0, 1]], + [[1, 2, 3, 0], [3, 2, 1, 0]], + [[3, 1], [3]], + [[3, 2, 0, 1], [3, 2, 0]], + [[2, 3, 0, 1], [0, 3, 2]], + [[1], [1, 3]], + [[1, 2], [2, 1, 0]], + [[3, 1, 2], [3, 1, 0]], + [[1, 3], [3, 1, 2]], + [[2, 0, 1, 3], [3, 1, 0, 2]], + [[1, 3, 0], [1, 3]], + [[2, 3, 1], [1, 0, 2]], + [[1, 2, 0, 3], [0, 2, 1, 3]], + [[2], [0, 1, 3]], + [[1], [1, 2]], + [[1, 0, 2, 3], [3, 0, 1, 2]], + [[0, 1, 3, 2], [1, 3, 0, 2]], + [[3, 0, 2, 1], [0, 2, 3]], + [[1, 2, 0], [1, 2, 3]], + [[1, 0, 3], [2, 3, 0]], + [[2, 3, 0], [3, 1, 0]], + [[1, 3], [1, 0]], + [[2, 1, 0, 3], [2, 0, 3, 1]], + [[3, 2, 0], [2, 1, 0]], + [[0, 1, 3], [0, 3, 1]], + [[3, 1, 0], [3, 2, 1]], + [[3, 2], [3, 1]], + [[3], [2, 1, 0]], + [[1, 2, 3, 0], []], + [[1, 3, 2, 0], [3, 1, 2]], + [[1], [0, 2]], + [[3, 2, 0], [3, 2, 0]], + [[3], []], + [[1, 0, 3], [2, 1]], + [[3, 1, 0, 2], [2, 3, 1, 0]], + [[0, 1], [0, 3, 2]], + [[0, 2, 3], [0, 2, 1]], + [[1, 3, 0], [3, 0, 2]], + [[3, 1, 2], [1, 2, 3]], + [[3, 1, 2], [3, 1, 0]], + [[0, 3, 1, 2], [3, 2, 1, 0]], + [[0, 3], [3, 2, 1]], + [[2, 3], [1, 3, 0]], + [[0, 3, 2], [2, 0, 3, 1]], + [[2, 3], [1, 3]], + [[3, 1, 2, 0], [2, 3, 1, 0]], + [[1, 0, 3, 2], [3, 0, 1, 2]], + [[3, 2, 1, 0], [0, 1, 3, 2]], + [[3, 1, 2], [3]], + [[0, 1, 3, 2], [2, 3, 0, 1]], + [[1, 2, 3, 0], [1, 3, 0, 2]], + [3, 1, 2], + [[3, 1, 2], [0, 3, 2]], + [[2, 3, 0], [1, 2, 0]], + [[2, 0, 3], [2, 0]], + [[3, 1, 0, 2], [3, 1, 0, 2]], + [[0, 1, 2], [2, 0, 1]], + [[1, 0, 3], [2, 3, 0]], + [[2, 0, 1], [0, 1, 3]], + [[2, 1], [0, 1, 3]], + ] def test_dygraph(self): paddle.disable_static() @@ -159,88 +206,89 @@ class TestTensordotAPI(unittest.TestCase): paddle.enable_static() for axes in self.all_axes: for place in self.places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - x = paddle.static.data(name='x', - shape=self.x_shape, - dtype=self.dtype) - y = paddle.static.data(name='y', - shape=self.y_shape, - dtype=self.dtype) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + x = paddle.static.data( + name='x', shape=self.x_shape, dtype=self.dtype + ) + y = paddle.static.data( + name='y', shape=self.y_shape, dtype=self.dtype + ) z = paddle.tensordot(x, y, axes) exe = paddle.static.Executor(place) - paddle_res = exe.run(feed={ - 'x': self.x, - 'y': self.y - }, - fetch_list=[z]) + paddle_res = exe.run( + feed={'x': self.x, 'y': self.y}, fetch_list=[z] + ) np_res = tensordot_np(self.x, self.y, axes) np.testing.assert_allclose(paddle_res[0], np_res, rtol=1e-6) class TestTensordotAPIFloat64(TestTensordotAPI): - def set_dtype(self): self.dtype = np.float64 class TestTensordotAPIBroadcastCase1(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [1, 1, 1, 5] self.y_shape = [1, 5, 1, 1] class TestTensordotAPIBroadcastCase2(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [1, 5, 5, 5] self.y_shape = [1, 1, 1, 5] class TestTensordotAPIBroadcastCase3(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [5, 5, 5, 1] self.y_shape = [5, 5, 1, 5] class TestTensordotAPIBroadcastCase4(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [5, 5, 5, 1] self.y_shape = [1, 1, 1, 1] class TestTensordotAPIBroadcastCase5(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [1, 1, 5, 5] self.y_shape = [5, 5, 1, 5] class TestTensordotAPIAxesType(TestTensordotAPI): - def set_input_shape(self): self.x_shape = [3, 4, 4] self.y_shape = [4, 4, 5] def set_test_axes(self): self.all_axes = [ - 0, 1, 2, (1, ), [1], ((1, ), ), ([1], ), ((2, 1), (0, )), - ((1, 2), (0, 1)), ([1, 2], [0, 1]), ([1, 2], [0, 1]), - [[1, 2], [0, 1]] + 0, + 1, + 2, + (1,), + [1], + ((1,),), + ([1],), + ((2, 1), (0,)), + ((1, 2), (0, 1)), + ([1, 2], [0, 1]), + ([1, 2], [0, 1]), + [[1, 2], [0, 1]], ] def test_tensor_axes(self): # The 'axes' with type 'Tensor' in tensordot is not available in static mode paddle.disable_static() tensor_axes = [ - paddle.to_tensor([1]), (paddle.to_tensor([1])), + paddle.to_tensor([1]), + (paddle.to_tensor([1])), (paddle.to_tensor([1, 2]), paddle.to_tensor([0, 1])), - [paddle.to_tensor([1, 2]), - paddle.to_tensor([0, 1])], - paddle.to_tensor([[1, 2], [0, 1]]) + [paddle.to_tensor([1, 2]), paddle.to_tensor([0, 1])], + paddle.to_tensor([[1, 2], [0, 1]]), ] for place in self.places: @@ -252,8 +300,15 @@ class TestTensordotAPIAxesType(TestTensordotAPI): np.testing.assert_allclose(paddle_res, np_res, rtol=1e-6) def test_error(self): - self.all_axes = [[[[0], [1]]], 0.1, -1, 100, [[1, 2], [0, 0]], - [[1, 2], [0, -1]], [0, 1, 2, 3]] + self.all_axes = [ + [[[0], [1]]], + 0.1, + -1, + 100, + [[1, 2], [0, 0]], + [[1, 2], [0, -1]], + [0, 1, 2, 3], + ] paddle.disable_static() x = paddle.to_tensor(self.x) y = paddle.to_tensor(self.y) @@ -263,7 +318,6 @@ class TestTensordotAPIAxesType(TestTensordotAPI): class TestTensordotAPIAxesTypeFloat64(TestTensordotAPIAxesType): - def set_dtype(self): self.dtype = np.float64 diff --git a/python/paddle/fluid/tests/unittests/test_tf32_cublas.py b/python/paddle/fluid/tests/unittests/test_tf32_cublas.py index 91dbb9938bb..ce97a7ad173 100644 --- a/python/paddle/fluid/tests/unittests/test_tf32_cublas.py +++ b/python/paddle/fluid/tests/unittests/test_tf32_cublas.py @@ -20,7 +20,6 @@ import paddle.fluid.core as core class TestTF32Switch(unittest.TestCase): - def test_on_off(self): if core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) @@ -36,7 +35,6 @@ class TestTF32Switch(unittest.TestCase): class TestTF32OnMatmul(unittest.TestCase): - def test_dygraph_without_out(self): if core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) diff --git a/python/paddle/fluid/tests/unittests/test_tf32_cudnn.py b/python/paddle/fluid/tests/unittests/test_tf32_cudnn.py index e50bc281333..cb57c93b71c 100644 --- a/python/paddle/fluid/tests/unittests/test_tf32_cudnn.py +++ b/python/paddle/fluid/tests/unittests/test_tf32_cudnn.py @@ -17,7 +17,6 @@ import paddle.fluid.core as core class TestTF32Switch(unittest.TestCase): - def test_on_off(self): if core.is_compiled_with_cuda(): self.assertTrue(core.get_cudnn_switch()) # default diff --git a/python/paddle/fluid/tests/unittests/test_tile_op.py b/python/paddle/fluid/tests/unittests/test_tile_op.py index 362464a1ba3..b6ca4b5711a 100644 --- a/python/paddle/fluid/tests/unittests/test_tile_op.py +++ b/python/paddle/fluid/tests/unittests/test_tile_op.py @@ -23,9 +23,8 @@ from decorator_helper import prog_scope import paddle.fluid.layers as layers -#Situation 1: repeat_times is a list (without tensor) +# Situation 1: repeat_times is a list (without tensor) class TestTileOpRank1(OpTest): - def setUp(self): self.op_type = "tile" self.init_data() @@ -47,21 +46,18 @@ class TestTileOpRank1(OpTest): class TestTileOpRank_ZeroDim1(TestTileOpRank1): - def init_data(self): self.ori_shape = [] self.repeat_times = [] class TestTileOpRank_ZeroDim2(TestTileOpRank1): - def init_data(self): self.ori_shape = [] self.repeat_times = [2] class TestTileOpRank_ZeroDim3(TestTileOpRank1): - def init_data(self): self.ori_shape = [] self.repeat_times = [2, 3] @@ -69,42 +65,36 @@ class TestTileOpRank_ZeroDim3(TestTileOpRank1): # with dimension expanding class TestTileOpRank2Expanding(TestTileOpRank1): - def init_data(self): self.ori_shape = [120] self.repeat_times = [2, 2] class TestTileOpRank2(TestTileOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] class TestTileOpRank3_Corner(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (1, 1, 1) class TestTileOpRank3_Corner2(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (2, 2) class TestTileOpRank3(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.repeat_times = (2, 1, 4) class TestTileOpRank4(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.repeat_times = (3, 2, 1, 2) @@ -112,14 +102,14 @@ class TestTileOpRank4(TestTileOpRank1): # Situation 2: repeat_times is a list (with tensor) class TestTileOpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "tile" self.init_data() repeat_times_tensor = [] for index, ele in enumerate(self.repeat_times): - repeat_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + repeat_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float64"), @@ -142,7 +132,6 @@ class TestTileOpRank1_tensor_attr(OpTest): class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [1, 1] @@ -150,7 +139,6 @@ class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -159,7 +147,6 @@ class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): # Situation 3: repeat_times is a tensor class TestTileOpRank1_tensor(OpTest): - def setUp(self): self.op_type = "tile" self.init_data() @@ -184,7 +171,6 @@ class TestTileOpRank1_tensor(OpTest): class TestTileOpRank2_tensor(TestTileOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -192,7 +178,6 @@ class TestTileOpRank2_tensor(TestTileOpRank1_tensor): # Situation 4: input x is Integer class TestTileOpInteger(OpTest): - def setUp(self): self.op_type = "tile" self.inputs = { @@ -208,7 +193,6 @@ class TestTileOpInteger(OpTest): # Situation 5: input x is Bool class TestTileOpBoolean(OpTest): - def setUp(self): self.op_type = "tile" self.inputs = {'X': np.random.randint(2, size=(2, 4, 5)).astype("bool")} @@ -222,7 +206,6 @@ class TestTileOpBoolean(OpTest): # Situation 56: input x is Integer class TestTileOpInt64_t(OpTest): - def setUp(self): self.op_type = "tile" self.inputs = { @@ -237,11 +220,11 @@ class TestTileOpInt64_t(OpTest): class TestTileError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) repeat_times = [2, 2] self.assertRaises(TypeError, paddle.tile, x1, repeat_times) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -252,7 +235,6 @@ class TestTileError(unittest.TestCase): class TestTileAPIStatic(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): repeat_times = [2, 2] @@ -264,7 +246,6 @@ class TestTileAPIStatic(unittest.TestCase): # Test python API class TestTileAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): np_x = np.random.random([12, 14]).astype("float32") @@ -286,7 +267,6 @@ class TestTileAPI(unittest.TestCase): class TestTileDoubleGradCheck(unittest.TestCase): - def tile_wrapper(self, x): return paddle.tile(x[0], [2, 1]) @@ -301,17 +281,13 @@ class TestTileDoubleGradCheck(unittest.TestCase): out = paddle.tile(data, [2, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.tile_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.tile_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -323,7 +299,6 @@ class TestTileDoubleGradCheck(unittest.TestCase): class TestTileTripleGradCheck(unittest.TestCase): - def tile_wrapper(self, x): return paddle.tile(x[0], [2, 1]) @@ -338,17 +313,13 @@ class TestTileTripleGradCheck(unittest.TestCase): out = paddle.tile(data, [2, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.tile_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.tile_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -360,7 +331,6 @@ class TestTileTripleGradCheck(unittest.TestCase): class TestTileAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_top_k_op.py b/python/paddle/fluid/tests/unittests/test_top_k_op.py index a4e54af05bd..d722d3f622f 100644 --- a/python/paddle/fluid/tests/unittests/test_top_k_op.py +++ b/python/paddle/fluid/tests/unittests/test_top_k_op.py @@ -19,7 +19,6 @@ import paddle class TestTopkOp(OpTest): - def setUp(self): self.variable_k = False self.set_args() diff --git a/python/paddle/fluid/tests/unittests/test_top_k_v2_op.py b/python/paddle/fluid/tests/unittests/test_top_k_v2_op.py index c6254801b6e..60d2502a56b 100644 --- a/python/paddle/fluid/tests/unittests/test_top_k_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_top_k_v2_op.py @@ -36,7 +36,6 @@ def numpy_topk(x, k=1, axis=-1, largest=True): class TestTopkOp(OpTest): - def init_args(self): self.k = 3 self.axis = 1 @@ -50,10 +49,9 @@ class TestTopkOp(OpTest): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -64,7 +62,6 @@ class TestTopkOp(OpTest): class TestTopkOp1(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 0 @@ -72,7 +69,6 @@ class TestTopkOp1(TestTopkOp): class TestTopkOp2(TestTopkOp): - def init_args(self): self.k = 4 self.axis = 0 @@ -80,7 +76,6 @@ class TestTopkOp2(TestTopkOp): class TestTopkOp3(OpTest): - def init_args(self): self.k = 6 self.axis = 1 @@ -94,15 +89,13 @@ class TestTopkOp3(OpTest): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp4(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -116,15 +109,13 @@ class TestTopkOp4(TestTopkOp): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp5(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -138,15 +129,13 @@ class TestTopkOp5(TestTopkOp): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp6(OpTest): - def init_args(self): self.k = 100 self.axis = 1 @@ -160,15 +149,13 @@ class TestTopkOp6(OpTest): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopKAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.input_data = np.random.rand(6, 7, 8) @@ -181,92 +168,89 @@ class TestTopKAPI(unittest.TestCase): # test case for basic test case 1 paddle_result = paddle.topk(input_tensor, k=2) numpy_result = numpy_topk(self.input_data, k=2) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 2 with axis paddle_result = paddle.topk(input_tensor, k=2, axis=1) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 3 with tensor K k_tensor = paddle.to_tensor(np.array([2])) paddle_result = paddle.topk(input_tensor, k=k_tensor, axis=1) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 4 with tensor largest k_tensor = paddle.to_tensor(np.array([2])) - paddle_result = paddle.topk(input_tensor, - k=2, - axis=1, - largest=False) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=1, - largest=False) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + paddle_result = paddle.topk( + input_tensor, k=2, axis=1, largest=False + ) + numpy_result = numpy_topk( + self.input_data, k=2, axis=1, largest=False + ) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 5 with axis -1 k_tensor = paddle.to_tensor(np.array([2])) - paddle_result = paddle.topk(input_tensor, - k=2, - axis=-1, - largest=False) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=-1, - largest=False) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + paddle_result = paddle.topk( + input_tensor, k=2, axis=-1, largest=False + ) + numpy_result = numpy_topk( + self.input_data, k=2, axis=-1, largest=False + ) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 6 for the partial sort paddle_result = paddle.topk(large_input_tensor, k=1, axis=-1) numpy_result = numpy_topk(self.large_input_data, k=1, axis=-1) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 7 for the unsorted paddle_result = paddle.topk(input_tensor, k=2, axis=1, sorted=False) - sort_paddle = numpy_topk(np.array(paddle_result[0].numpy()), - axis=1, - k=2) + sort_paddle = numpy_topk( + np.array(paddle_result[0].numpy()), axis=1, k=2 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(sort_paddle[0], - numpy_result[0], - rtol=1e-05) + np.testing.assert_allclose( + sort_paddle[0], numpy_result[0], rtol=1e-05 + ) def run_static(self, place): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[6, 7, 8], - dtype="float64") - large_input_tensor = paddle.static.data(name="large_x", - shape=[2, 1030], - dtype="float64") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[6, 7, 8], dtype="float64" + ) + large_input_tensor = paddle.static.data( + name="large_x", shape=[2, 1030], dtype="float64" + ) k_tensor = paddle.static.data(name="k", shape=[1], dtype="int32") result1 = paddle.topk(input_tensor, k=2) result2 = paddle.topk(input_tensor, k=2, axis=-1) @@ -280,71 +264,80 @@ class TestTopKAPI(unittest.TestCase): exe = paddle.static.Executor(place) input_data = np.random.rand(10, 20).astype("float64") large_input_data = np.random.rand(2, 100).astype("float64") - paddle_result = exe.run(feed={ - "x": self.input_data, - "large_x": self.large_input_data, - "k": np.array([2]).astype("int32") - }, - fetch_list=[ - result1[0], result1[1], result2[0], - result2[1], result3[0], result3[1], - result4[0], result4[1], result5[0], - result5[1], result6[0], result6[1], - result7[0], result7[1] - ]) + paddle_result = exe.run( + feed={ + "x": self.input_data, + "large_x": self.large_input_data, + "k": np.array([2]).astype("int32"), + }, + fetch_list=[ + result1[0], + result1[1], + result2[0], + result2[1], + result3[0], + result3[1], + result4[0], + result4[1], + result5[0], + result5[1], + result6[0], + result6[1], + result7[0], + result7[1], + ], + ) numpy_result = numpy_topk(self.input_data, k=2) - np.testing.assert_allclose(paddle_result[0], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1], - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1], numpy_result[1], rtol=1e-05 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=-1) - np.testing.assert_allclose(paddle_result[2], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[3], - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[2], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[3], numpy_result[1], rtol=1e-05 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(paddle_result[4], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[5], - numpy_result[1], - rtol=1e-05) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=1, - largest=False) - np.testing.assert_allclose(paddle_result[6], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[7], - numpy_result[1], - rtol=1e-05) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=-1, - largest=False) - np.testing.assert_allclose(paddle_result[8], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[9], - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[4], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[5], numpy_result[1], rtol=1e-05 + ) + numpy_result = numpy_topk( + self.input_data, k=2, axis=1, largest=False + ) + np.testing.assert_allclose( + paddle_result[6], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[7], numpy_result[1], rtol=1e-05 + ) + numpy_result = numpy_topk( + self.input_data, k=2, axis=-1, largest=False + ) + np.testing.assert_allclose( + paddle_result[8], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[9], numpy_result[1], rtol=1e-05 + ) numpy_result = numpy_topk(self.large_input_data, k=1, axis=-1) - np.testing.assert_allclose(paddle_result[10], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[11], - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[10], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[11], numpy_result[1], rtol=1e-05 + ) sort_paddle = numpy_topk(paddle_result[12], axis=1, k=2) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(sort_paddle[0], - numpy_result[0], - rtol=1e-05) + np.testing.assert_allclose( + sort_paddle[0], numpy_result[0], rtol=1e-05 + ) def test_cases(self): places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_trace_op.py b/python/paddle/fluid/tests/unittests/test_trace_op.py index 83aca5654c2..8bceee2fdfd 100644 --- a/python/paddle/fluid/tests/unittests/test_trace_op.py +++ b/python/paddle/fluid/tests/unittests/test_trace_op.py @@ -22,7 +22,6 @@ import paddle class TestTraceOp(OpTest): - def setUp(self): self.op_type = "trace" self.python_api = paddle.trace @@ -43,31 +42,32 @@ class TestTraceOp(OpTest): class TestTraceOpCase1(TestTraceOp): - def init_config(self): self.case = np.random.randn(2, 20, 2, 3).astype('float32') self.inputs = {'Input': self.case} self.attrs = {'offset': 1, 'axis1': 0, 'axis2': 2} - self.target = np.trace(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.trace( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) class TestTraceOpCase2(TestTraceOp): - def init_config(self): self.case = np.random.randn(2, 20, 2, 3).astype('float32') self.inputs = {'Input': self.case} self.attrs = {'offset': -5, 'axis1': 1, 'axis2': -1} - self.target = np.trace(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.trace( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) class TestTraceAPICase(unittest.TestCase): - def test_case1(self): case = np.random.randn(2, 20, 2, 3).astype('float32') data1 = fluid.data(name='data1', shape=[2, 20, 2, 3], dtype='float32') @@ -76,10 +76,12 @@ class TestTraceAPICase(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) - results = exe.run(fluid.default_main_program(), - feed={"data1": case}, - fetch_list=[out1, out2], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": case}, + fetch_list=[out1, out2], + return_numpy=True, + ) target1 = np.trace(case) target2 = np.trace(case, offset=-5, axis1=1, axis2=-1) np.testing.assert_allclose(results[0], target1, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py b/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py index 59b652b1d3d..b160debe41c 100644 --- a/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py +++ b/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py @@ -24,12 +24,12 @@ import paddle.nn as nn class SimpleFCLayer(nn.Layer): - def __init__(self, feature_size, batch_size, fc_size): super(SimpleFCLayer, self).__init__() self._linear = nn.Linear(feature_size, fc_size) self._offset = paddle.to_tensor( - np.random.random((batch_size, fc_size)).astype('float32')) + np.random.random((batch_size, fc_size)).astype('float32') + ) def forward(self, x): fc = self._linear(x) @@ -37,7 +37,6 @@ class SimpleFCLayer(nn.Layer): class LinearNetWithNone(nn.Layer): - def __init__(self, feature_size, fc_size): super(LinearNetWithNone, self).__init__() self._linear = nn.Linear(feature_size, fc_size) @@ -49,7 +48,6 @@ class LinearNetWithNone(nn.Layer): class TestTracedLayerErrMsg(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.feature_size = 3 @@ -66,116 +64,163 @@ class TestTracedLayerErrMsg(unittest.TestCase): return with fluid.dygraph.guard(): in_x = fluid.dygraph.to_variable( - np.random.random( - (self.batch_size, self.feature_size)).astype('float32')) + np.random.random((self.batch_size, self.feature_size)).astype( + 'float32' + ) + ) with self.assertRaises(AssertionError) as e: dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - None, [in_x]) + None, [in_x] + ) self.assertEqual( - "The type of 'layer' in fluid.dygraph.jit.TracedLayer.trace must be fluid.dygraph.Layer, but received <{} 'NoneType'>." - .format(self.type_str), str(e.exception)) + "The type of 'layer' in fluid.dygraph.jit.TracedLayer.trace must be fluid.dygraph.Layer, but received <{} 'NoneType'>.".format( + self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, 3) + self.layer, 3 + ) self.assertEqual( - "The type of 'each element of inputs' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received <{} 'int'>." - .format(self.type_str), str(e.exception)) + "The type of 'each element of inputs' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received <{} 'int'>.".format( + self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, [True, 1]) + self.layer, [True, 1] + ) self.assertEqual( - "The type of 'each element of inputs' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received <{} 'bool'>." - .format(self.type_str), str(e.exception)) + "The type of 'each element of inputs' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received <{} 'bool'>.".format( + self.type_str + ), + str(e.exception), + ) dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, [in_x]) + self.layer, [in_x] + ) def test_set_strategy_err(self): if fluid.framework.in_dygraph_mode(): return with fluid.dygraph.guard(): in_x = fluid.dygraph.to_variable( - np.random.random( - (self.batch_size, self.feature_size)).astype('float32')) + np.random.random((self.batch_size, self.feature_size)).astype( + 'float32' + ) + ) dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, [in_x]) + self.layer, [in_x] + ) with self.assertRaises(AssertionError) as e: traced_layer.set_strategy(1, fluid.ExecutionStrategy()) self.assertEqual( - "The type of 'build_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.BuildStrategy, but received <{} 'int'>." - .format(self.type_str), str(e.exception)) + "The type of 'build_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.BuildStrategy, but received <{} 'int'>.".format( + self.type_str + ), + str(e.exception), + ) with self.assertRaises(AssertionError) as e: traced_layer.set_strategy(fluid.BuildStrategy(), False) self.assertEqual( - "The type of 'exec_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.ExecutionStrategy, but received <{} 'bool'>." - .format(self.type_str), str(e.exception)) + "The type of 'exec_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.ExecutionStrategy, but received <{} 'bool'>.".format( + self.type_str + ), + str(e.exception), + ) traced_layer.set_strategy(build_strategy=fluid.BuildStrategy()) traced_layer.set_strategy(exec_strategy=fluid.ExecutionStrategy()) - traced_layer.set_strategy(fluid.BuildStrategy(), - fluid.ExecutionStrategy()) + traced_layer.set_strategy( + fluid.BuildStrategy(), fluid.ExecutionStrategy() + ) def test_save_inference_model_err(self): if fluid.framework.in_dygraph_mode(): return with fluid.dygraph.guard(): in_x = fluid.dygraph.to_variable( - np.random.random( - (self.batch_size, self.feature_size)).astype('float32')) + np.random.random((self.batch_size, self.feature_size)).astype( + 'float32' + ) + ) dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, [in_x]) + self.layer, [in_x] + ) path = os.path.join(self.temp_dir.name, './traced_layer_err_msg') with self.assertRaises(TypeError) as e: traced_layer.save_inference_model([0]) self.assertEqual( - "The type of 'path' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'str'>, but received <{} 'list'>. " - .format(self.type_str, self.type_str), str(e.exception)) + "The type of 'path' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'str'>, but received <{} 'list'>. ".format( + self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: traced_layer.save_inference_model(path, [0], [None]) self.assertEqual( - "The type of 'each element of fetch' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. " - .format(self.type_str, self.type_str), str(e.exception)) + "The type of 'each element of fetch' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. ".format( + self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: traced_layer.save_inference_model(path, [0], False) self.assertEqual( - "The type of 'fetch' in fluid.dygraph.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. " - .format(self.type_str, self.type_str, self.type_str), - str(e.exception)) + "The type of 'fetch' in fluid.dygraph.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. ".format( + self.type_str, self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: traced_layer.save_inference_model(path, [None], [0]) self.assertEqual( - "The type of 'each element of feed' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. " - .format(self.type_str, self.type_str), str(e.exception)) + "The type of 'each element of feed' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. ".format( + self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: traced_layer.save_inference_model(path, True, [0]) self.assertEqual( - "The type of 'feed' in fluid.dygraph.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. " - .format(self.type_str, self.type_str, self.type_str), - str(e.exception)) + "The type of 'feed' in fluid.dygraph.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. ".format( + self.type_str, self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(ValueError) as e: traced_layer.save_inference_model("") self.assertEqual( "The input path MUST be format of dirname/file_prefix [dirname\\file_prefix in Windows system], " - "but received file_prefix is empty string.", str(e.exception)) + "but received file_prefix is empty string.", + str(e.exception), + ) traced_layer.save_inference_model(path) def _train_simple_net(self): layer = None with fluid.dygraph.guard(): - layer = SimpleFCLayer(self.feature_size, self.batch_size, - self.fc_size) - optimizer = fluid.optimizer.SGD(learning_rate=1e-3, - parameter_list=layer.parameters()) + layer = SimpleFCLayer( + self.feature_size, self.batch_size, self.fc_size + ) + optimizer = fluid.optimizer.SGD( + learning_rate=1e-3, parameter_list=layer.parameters() + ) for i in range(5): in_x = fluid.dygraph.to_variable( np.random.random( - (self.batch_size, self.feature_size)).astype('float32')) + (self.batch_size, self.feature_size) + ).astype('float32') + ) dygraph_out = layer(in_x) loss = fluid.layers.reduce_mean(dygraph_out) loss.backward() @@ -184,7 +229,6 @@ class TestTracedLayerErrMsg(unittest.TestCase): class TestOutVarWithNoneErrMsg(unittest.TestCase): - def test_linear_net_with_none(self): if fluid.framework.in_dygraph_mode(): return @@ -192,7 +236,8 @@ class TestOutVarWithNoneErrMsg(unittest.TestCase): in_x = paddle.to_tensor(np.random.random((4, 100)).astype('float32')) with self.assertRaises(TypeError): dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - model, [in_x]) + model, [in_x] + ) class TestTracedLayerSaveInferenceModel(unittest.TestCase): @@ -202,6 +247,7 @@ class TestTracedLayerSaveInferenceModel(unittest.TestCase): self.temp_dir = tempfile.TemporaryDirectory() self.save_path = os.path.join(self.temp_dir.name, "./nonexist_dir/fc") import shutil + if os.path.exists(os.path.dirname(self.save_path)): shutil.rmtree(os.path.dirname(self.save_path)) @@ -215,7 +261,8 @@ class TestTracedLayerSaveInferenceModel(unittest.TestCase): input_var = paddle.to_tensor(np.random.random([4, 3]).astype('float32')) with fluid.dygraph.guard(): dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - fc_layer, inputs=[input_var]) + fc_layer, inputs=[input_var] + ) self.assertFalse(os.path.exists(os.path.dirname(self.save_path))) traced_layer.save_inference_model(self.save_path) self.assertTrue(os.path.exists(os.path.dirname(self.save_path))) diff --git a/python/paddle/fluid/tests/unittests/test_trainable.py b/python/paddle/fluid/tests/unittests/test_trainable.py index d9d5121ff9b..bbd7ae55d30 100644 --- a/python/paddle/fluid/tests/unittests/test_trainable.py +++ b/python/paddle/fluid/tests/unittests/test_trainable.py @@ -22,21 +22,18 @@ from simple_nets import init_data def test_trainable(): x = fluid.layers.data(name='image', shape=[784], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') - feature = fluid.layers.fc(input=x, - size=10, - param_attr=fluid.ParamAttr(trainable=False)) + feature = fluid.layers.fc( + input=x, size=10, param_attr=fluid.ParamAttr(trainable=False) + ) loss = fluid.layers.cross_entropy(input=feature, label=label) loss = paddle.mean(loss) return loss class TestTrainable(unittest.TestCase): - - def check_trainable(self, - model, - feed_dict, - op_count, - optimizer=fluid.optimizer.Adam()): + def check_trainable( + self, model, feed_dict, op_count, optimizer=fluid.optimizer.Adam() + ): place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -64,22 +61,17 @@ class TestTrainable(unittest.TestCase): feed_dict = {'image': img, 'label': label} # Note that, because the Weight of FC is not trainable and the x is stop_gradient, # so the 'mul_grad' should not be appended. - self.check_trainable(test_trainable, - feed_dict, - op_count={ - 'adam': 1, - 'scale': 0, - 'mul_grad': 0 - }) self.check_trainable( test_trainable, feed_dict, - op_count={ - 'adamax': 1, - 'scale': 1, - 'mul_grad': 0 - }, - optimizer=fluid.optimizer.Adamax(learning_rate=0.2)) + op_count={'adam': 1, 'scale': 0, 'mul_grad': 0}, + ) + self.check_trainable( + test_trainable, + feed_dict, + op_count={'adamax': 1, 'scale': 1, 'mul_grad': 0}, + optimizer=fluid.optimizer.Adamax(learning_rate=0.2), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_trainer_desc.py b/python/paddle/fluid/tests/unittests/test_trainer_desc.py index 390a650fd13..bc584439fdf 100644 --- a/python/paddle/fluid/tests/unittests/test_trainer_desc.py +++ b/python/paddle/fluid/tests/unittests/test_trainer_desc.py @@ -21,7 +21,7 @@ import unittest class TestTrainerDesc(unittest.TestCase): - """ TestCases for TrainerDesc. """ + """TestCases for TrainerDesc.""" def test_config(self): """ diff --git a/python/paddle/fluid/tests/unittests/test_transfer_dtype_op.py b/python/paddle/fluid/tests/unittests/test_transfer_dtype_op.py index 9d3edc0e062..364e6ff167c 100644 --- a/python/paddle/fluid/tests/unittests/test_transfer_dtype_op.py +++ b/python/paddle/fluid/tests/unittests/test_transfer_dtype_op.py @@ -21,14 +21,13 @@ from op_test import OpTest, convert_uint16_to_float, convert_float_to_uint16 class TestTransferDtypeOpFp32ToFp64(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float64')} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.FP64), - 'in_dtype': int(core.VarDesc.VarType.FP32) + 'in_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'transfer_dtype' @@ -37,14 +36,13 @@ class TestTransferDtypeOpFp32ToFp64(OpTest): class TestTransferDtypeOpFp16ToFp32(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float16')} self.outputs = {'Out': ipt.astype('float32')} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.FP32), - 'in_dtype': int(core.VarDesc.VarType.FP16) + 'in_dtype': int(core.VarDesc.VarType.FP16), } self.op_type = 'transfer_dtype' @@ -53,14 +51,13 @@ class TestTransferDtypeOpFp16ToFp32(OpTest): class TestTransferDtypeOpFp32ToFp16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float16')} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.FP16), - 'in_dtype': int(core.VarDesc.VarType.FP32) + 'in_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'transfer_dtype' @@ -69,14 +66,13 @@ class TestTransferDtypeOpFp32ToFp16(OpTest): class TestTransferDtypeOpBf16ToFp32(OpTest): - def setUp(self): ipt = np.array(np.random.randint(10, size=[10, 10])).astype('uint16') self.inputs = {'X': ipt} self.outputs = {'Out': convert_uint16_to_float(ipt)} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.FP32), - 'in_dtype': int(core.VarDesc.VarType.BF16) + 'in_dtype': int(core.VarDesc.VarType.BF16), } self.op_type = 'transfer_dtype' @@ -85,14 +81,13 @@ class TestTransferDtypeOpBf16ToFp32(OpTest): class TestTransferDtypeFp32ToBf16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]).astype('float32') self.inputs = {'X': ipt} self.outputs = {'Out': convert_float_to_uint16(ipt)} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.BF16), - 'in_dtype': int(core.VarDesc.VarType.FP32) + 'in_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'transfer_dtype' diff --git a/python/paddle/fluid/tests/unittests/test_transfer_layout_op.py b/python/paddle/fluid/tests/unittests/test_transfer_layout_op.py index a24fb767d10..48c2897b005 100644 --- a/python/paddle/fluid/tests/unittests/test_transfer_layout_op.py +++ b/python/paddle/fluid/tests/unittests/test_transfer_layout_op.py @@ -25,15 +25,11 @@ from op_test import OpTest # default kNCHW class TestTransferLayoutOpkNCHWTokNHWC(OpTest): - def setUp(self): ipt = np.random.random(size=[2, 3, 10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.transpose([0, 2, 3, 1])} - self.attrs = { - 'src_layout': 0, - 'dst_layout': 1 # kNHWC - } + self.attrs = {'src_layout': 0, 'dst_layout': 1} # kNHWC self.op_type = 'transfer_layout' def test_check_output(self): @@ -45,20 +41,17 @@ def softmax_with_data_format(x, data_format, axis=-1, dtype=None, name=None): outs_cast = x outs_softmax = helper.create_variable_for_type_inference(outs_cast.dtype) - helper.append_op(type='softmax', - inputs={'X': outs_cast}, - outputs={'Out': outs_softmax}, - attrs={ - 'axis': axis, - 'use_cudnn': True, - 'data_format': data_format - }) + helper.append_op( + type='softmax', + inputs={'X': outs_cast}, + outputs={'Out': outs_softmax}, + attrs={'axis': axis, 'use_cudnn': True, 'data_format': data_format}, + ) return outs_softmax class TestTransferLayoutOpGpu(unittest.TestCase): - def test_layout_transfer(self): if not core.is_compiled_with_cuda(): return @@ -69,19 +62,24 @@ class TestTransferLayoutOpGpu(unittest.TestCase): startup_program = Program() n, c, h, w = 2, 3, 4, 5 with program_guard(main_program, startup_program): - x = paddle.static.data(shape=[n, c, h, w], - dtype='float32', - name='x') + x = paddle.static.data( + shape=[n, c, h, w], dtype='float32', name='x' + ) y = softmax_with_data_format(x, data_format='NCHW') z = softmax_with_data_format(y, data_format='NHWC') - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup_program) - ret = exe.run(main_program, - feed={'x': np.full((n, c, h, w), 1, np.float32)}, - fetch_list=[z.name]) + ret = exe.run( + main_program, + feed={'x': np.full((n, c, h, w), 1, np.float32)}, + fetch_list=[z.name], + ) assert len(ret) == 1 assert ret[0].shape == (n, h, w, c) diff --git a/python/paddle/fluid/tests/unittests/test_transformer_api.py b/python/paddle/fluid/tests/unittests/test_transformer_api.py index 6b254ac3115..3f2e2185f9e 100644 --- a/python/paddle/fluid/tests/unittests/test_transformer_api.py +++ b/python/paddle/fluid/tests/unittests/test_transformer_api.py @@ -15,7 +15,14 @@ import numpy as np import paddle import paddle.fluid as fluid -from paddle.nn.layer.transformer import MultiHeadAttention, TransformerEncoderLayer, TransformerDecoderLayer, TransformerEncoder, TransformerDecoder, Transformer +from paddle.nn.layer.transformer import ( + MultiHeadAttention, + TransformerEncoderLayer, + TransformerDecoderLayer, + TransformerEncoder, + TransformerDecoder, + Transformer, +) import unittest @@ -33,34 +40,67 @@ def generate_basic_params(mode="attn", self_attention=True): kdim, vdim = [np.random.randint(5, 20) for _ in range(2)] key_length = np.random.randint(2, 10) value_length = key_length - return batch_size, query_length, key_length, value_length, embed_dim, kdim, vdim, num_heads, attn_dropout + return ( + batch_size, + query_length, + key_length, + value_length, + embed_dim, + kdim, + vdim, + num_heads, + attn_dropout, + ) else: dropout, act_dropout = 0.0, 0.0 dim_feedforward = np.random.randint(128, 1024) sequence_length = np.random.randint(2, 10) if mode == "encoder_layer": - return batch_size, embed_dim, num_heads, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length + return ( + batch_size, + embed_dim, + num_heads, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) elif mode == "decoder_layer": target_length = np.random.randint(2, 10) - return batch_size, embed_dim, num_heads, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length, target_length - - -def generate_query_key_value_cache(self_attention, - batch_size, - num_heads, - query_length, - embed_dim, - attn_mask_type, - key_length=None, - value_length=None, - kdim=None, - vdim=None, - cache=None): - query = np.random.rand(batch_size, query_length, - embed_dim).astype("float32") - attn_mask = np.ones((batch_size, num_heads, query_length, key_length), - dtype=attn_mask_type) + return ( + batch_size, + embed_dim, + num_heads, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + target_length, + ) + + +def generate_query_key_value_cache( + self_attention, + batch_size, + num_heads, + query_length, + embed_dim, + attn_mask_type, + key_length=None, + value_length=None, + kdim=None, + vdim=None, + cache=None, +): + query = np.random.rand(batch_size, query_length, embed_dim).astype( + "float32" + ) + attn_mask = np.ones( + (batch_size, num_heads, query_length, key_length), dtype=attn_mask_type + ) if attn_mask_type == 'int64': attn_mask = np.tril(attn_mask) elif attn_mask_type == 'float64': @@ -77,18 +117,19 @@ def generate_query_key_value_cache(self_attention, cache_dict = {} if cache: if not self_attention: - cache_dict["static_k"] = np.random.rand(batch_size, num_heads, - key_length, - head_dim).astype("float32") - cache_dict["static_v"] = np.random.rand(batch_size, num_heads, - value_length, - head_dim).astype("float32") + cache_dict["static_k"] = np.random.rand( + batch_size, num_heads, key_length, head_dim + ).astype("float32") + cache_dict["static_v"] = np.random.rand( + batch_size, num_heads, value_length, head_dim + ).astype("float32") else: - cache_dict["k"] = np.random.rand(batch_size, num_heads, key_length, - head_dim).astype("float32") - cache_dict["v"] = np.random.rand(batch_size, num_heads, - value_length, - head_dim).astype("float32") + cache_dict["k"] = np.random.rand( + batch_size, num_heads, key_length, head_dim + ).astype("float32") + cache_dict["v"] = np.random.rand( + batch_size, num_heads, value_length, head_dim + ).astype("float32") else: cache_dict = None return query, key, value, attn_mask, cache_dict @@ -113,8 +154,9 @@ def softmax(x): def batch_matmul(x, y): assert x.shape[0] == y.shape[0] assert x.shape[1] == y.shape[1] - retval = np.zeros((x.shape[0], x.shape[1], x.shape[2], y.shape[3]), - dtype=np.float64) + retval = np.zeros( + (x.shape[0], x.shape[1], x.shape[2], y.shape[3]), dtype=np.float64 + ) for i in range(x.shape[0]): for j in range(x.shape[1]): retval[i, j, :, :] = np.matmul(x[i, j, :, :], y[i, j, :, :]) @@ -133,8 +175,13 @@ def scaled_dot_product_attention(q, k, v, d_key, attn_mask, multi_head_attn): weight = softmax(qkt) attn_heads = batch_matmul(weight, v) attn_heads = attn_heads.transpose((0, 2, 1, 3)) - attn_heads = attn_heads.reshape((attn_heads.shape[0], attn_heads.shape[1], - attn_heads.shape[2] * attn_heads.shape[3])) + attn_heads = attn_heads.reshape( + ( + attn_heads.shape[0], + attn_heads.shape[1], + attn_heads.shape[2] * attn_heads.shape[3], + ) + ) return attn_heads @@ -152,8 +199,16 @@ def cal_qkv(key, value, num_heads, embed_dim, multi_head_attn): return k, v -def prepare_qkv(query, key, value, num_heads, embed_dim, self_attention, - multi_head_attn, cache_dict): +def prepare_qkv( + query, + key, + value, + num_heads, + embed_dim, + self_attention, + multi_head_attn, + cache_dict, +): q_weight = multi_head_attn.q_proj.weight.numpy() q = fc(query, q_weight) q = q.reshape((q.shape[0], q.shape[1], num_heads, embed_dim // num_heads)) @@ -196,7 +251,7 @@ def layer_norm(x, normalized_shape, norm, epsilon=1e-05, act=None): x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model - x1_up = (x - mu) + x1_up = x - mu x1_down_1 = sigma_squar + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) @@ -223,9 +278,7 @@ def ffn(src, encoder_layer, ffn_fc1_act="relu"): class TestTransformer(unittest.TestCase): - def test_multi_head_attention(self): - def multihead_attention_test_helper(self_attention, cache): paddle.seed(2020) paddle.framework.random._manual_program_seed(2020) @@ -233,59 +286,111 @@ class TestTransformer(unittest.TestCase): with fluid.dygraph.guard(fluid.CPUPlace()): # generate params for multi_head_attention - batch_size, query_length, key_length, value_length, embed_dim, kdim, vdim, num_heads, attn_dropout = generate_basic_params( - "attn", self_attention) + ( + batch_size, + query_length, + key_length, + value_length, + embed_dim, + kdim, + vdim, + num_heads, + attn_dropout, + ) = generate_basic_params("attn", self_attention) for attn_mask_type in ['int64', 'float64']: - query, key, value, attn_mask, cache_dict = generate_query_key_value_cache( - self_attention, batch_size, num_heads, query_length, - embed_dim, attn_mask_type, key_length, value_length, - kdim, vdim, cache) + ( + query, + key, + value, + attn_mask, + cache_dict, + ) = generate_query_key_value_cache( + self_attention, + batch_size, + num_heads, + query_length, + embed_dim, + attn_mask_type, + key_length, + value_length, + kdim, + vdim, + cache, + ) if cache and self_attention: - attn_mask = np.concatenate((attn_mask, attn_mask), - axis=3) + attn_mask = np.concatenate( + (attn_mask, attn_mask), axis=3 + ) need_weight, param_attr, bias_attr = False, None, None # call paddle's function multi_head_attn = MultiHeadAttention( - embed_dim, num_heads, attn_dropout, kdim, vdim, - need_weight, param_attr, bias_attr) + embed_dim, + num_heads, + attn_dropout, + kdim, + vdim, + need_weight, + param_attr, + bias_attr, + ) # construct cache object cache_obj = None if cache_dict: if 'k' and 'v' in cache_dict: cache_obj = multi_head_attn.Cache( paddle.to_tensor(cache_dict['k']), - paddle.to_tensor(cache_dict['v'])) + paddle.to_tensor(cache_dict['v']), + ) elif 'static_k' and 'static_v' in cache_dict: cache_obj = multi_head_attn.StaticCache( paddle.to_tensor(cache_dict['static_k']), - paddle.to_tensor(cache_dict['static_v'])) + paddle.to_tensor(cache_dict['static_v']), + ) if attn_mask is not None: attn_output = multi_head_attn( - paddle.to_tensor(query), paddle.to_tensor(key), + paddle.to_tensor(query), + paddle.to_tensor(key), paddle.to_tensor(value), - paddle.to_tensor(attn_mask), cache_obj) + paddle.to_tensor(attn_mask), + cache_obj, + ) else: - attn_output = multi_head_attn(paddle.to_tensor(query), - paddle.to_tensor(key), - paddle.to_tensor(value), - attn_mask, cache_obj) + attn_output = multi_head_attn( + paddle.to_tensor(query), + paddle.to_tensor(key), + paddle.to_tensor(value), + attn_mask, + cache_obj, + ) attn_output = attn_output[0] if cache_dict else attn_output # implementation by numpy # compute q, k, v - q, k, v, _ = prepare_qkv(query, key, value, num_heads, - embed_dim, self_attention, - multi_head_attn, cache_dict) + q, k, v, _ = prepare_qkv( + query, + key, + value, + num_heads, + embed_dim, + self_attention, + multi_head_attn, + cache_dict, + ) # scale dot product attention attn_heads = scaled_dot_product_attention( - q, k, v, embed_dim // num_heads, attn_mask, - multi_head_attn) + q, + k, + v, + embed_dim // num_heads, + attn_mask, + multi_head_attn, + ) out_proj_weight = multi_head_attn.out_proj.weight.numpy() reference = fc(attn_heads, out_proj_weight) - np.testing.assert_allclose(attn_output.numpy(), - reference, - atol=1e-6) + np.testing.assert_allclose( + attn_output.numpy(), reference, atol=1e-6 + ) multihead_attention_test_helper(True, True) multihead_attention_test_helper(True, False) @@ -300,34 +405,51 @@ class TestTransformer(unittest.TestCase): ffn_fc1_act = "relu" # 1.generate basic params - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length = generate_basic_params( - mode="encoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) = generate_basic_params(mode="encoder_layer") # 2.generate input for encoder - src = np.random.rand(batch_size, sequence_length, - d_model).astype("float32") + src = np.random.rand(batch_size, sequence_length, d_model).astype( + "float32" + ) residual = src - src_mask = np.zeros((batch_size, n_head, sequence_length, - sequence_length)).astype("float32") + src_mask = np.zeros( + (batch_size, n_head, sequence_length, sequence_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf # paddle - encoder_layer = TransformerEncoderLayer(d_model, n_head, - dim_feedforward, dropout, - ffn_fc1_act, attn_dropout, - act_dropout) + encoder_layer = TransformerEncoderLayer( + d_model, + n_head, + dim_feedforward, + dropout, + ffn_fc1_act, + attn_dropout, + act_dropout, + ) encoder_output = encoder_layer( - paddle.to_tensor(src), - paddle.to_tensor(src_mask)) # paddle.to_tensor(src_mask)) + paddle.to_tensor(src), paddle.to_tensor(src_mask) + ) # paddle.to_tensor(src_mask)) # 4.numpy: # paddle self attention - self_attn = MultiHeadAttention(d_model, - n_head, - dropout=attn_dropout) - attn_output = self_attn(paddle.to_tensor(src), - paddle.to_tensor(src), - paddle.to_tensor(src), - paddle.to_tensor(src_mask)).numpy() + self_attn = MultiHeadAttention( + d_model, n_head, dropout=attn_dropout + ) + attn_output = self_attn( + paddle.to_tensor(src), + paddle.to_tensor(src), + paddle.to_tensor(src), + paddle.to_tensor(src_mask), + ).numpy() src = attn_output + residual src_norm = layer_norm(src, d_model, encoder_layer.norm1) @@ -337,10 +459,9 @@ class TestTransformer(unittest.TestCase): src = residual + ffn_output src = layer_norm(src, d_model, encoder_layer.norm2) - np.testing.assert_allclose(encoder_output.numpy(), - src, - rtol=1e-5, - atol=1e-6) + np.testing.assert_allclose( + encoder_output.numpy(), src, rtol=1e-5, atol=1e-6 + ) def test_transformer_encoder_layer_attr_1(self): with fluid.dygraph.guard(fluid.CPUPlace()): @@ -349,44 +470,67 @@ class TestTransformer(unittest.TestCase): ffn_fc1_act = "relu" # 1.generate basic params - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length = generate_basic_params( - mode="encoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) = generate_basic_params(mode="encoder_layer") # 2.generate input for encoder - src = np.random.rand(batch_size, sequence_length, - d_model).astype("float32") - src_mask = np.zeros((batch_size, n_head, sequence_length, - sequence_length)).astype("float32") + src = np.random.rand(batch_size, sequence_length, d_model).astype( + "float32" + ) + src_mask = np.zeros( + (batch_size, n_head, sequence_length, sequence_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf for cache in [True, False]: # paddle - encoder_layer = TransformerEncoderLayer(d_model, n_head, - dim_feedforward, - dropout, ffn_fc1_act, - attn_dropout, - act_dropout) + encoder_layer = TransformerEncoderLayer( + d_model, + n_head, + dim_feedforward, + dropout, + ffn_fc1_act, + attn_dropout, + act_dropout, + ) cache_objs = None if cache: cache_objs = encoder_layer.gen_cache(paddle.to_tensor(src)) - encoder_output = encoder_layer(paddle.to_tensor(src), - paddle.to_tensor(src_mask), - cache_objs) - encoder_output = encoder_output[0].numpy( - ) if cache else encoder_output.numpy() + encoder_output = encoder_layer( + paddle.to_tensor(src), + paddle.to_tensor(src_mask), + cache_objs, + ) + encoder_output = ( + encoder_output[0].numpy() + if cache + else encoder_output.numpy() + ) # 4.numpy: residual = src # paddle self attention - self_attn = MultiHeadAttention(d_model, - n_head, - dropout=attn_dropout) - attn_output = self_attn(paddle.to_tensor(src), - paddle.to_tensor(src), - paddle.to_tensor(src), - paddle.to_tensor(src_mask), cache_objs) - attn_output = attn_output[0].numpy( - ) if cache else attn_output.numpy() + self_attn = MultiHeadAttention( + d_model, n_head, dropout=attn_dropout + ) + attn_output = self_attn( + paddle.to_tensor(src), + paddle.to_tensor(src), + paddle.to_tensor(src), + paddle.to_tensor(src_mask), + cache_objs, + ) + attn_output = ( + attn_output[0].numpy() if cache else attn_output.numpy() + ) src = attn_output + residual src_norm = layer_norm(src, d_model, encoder_layer.norm1) @@ -396,62 +540,92 @@ class TestTransformer(unittest.TestCase): src = residual + ffn_output src = layer_norm(src, d_model, encoder_layer.norm2) - np.testing.assert_allclose(encoder_output, - src, - rtol=1e-5, - atol=1e-6) + np.testing.assert_allclose( + encoder_output, src, rtol=1e-5, atol=1e-6 + ) def test_transformer_decoder_layer(self): with fluid.dygraph.guard(fluid.CPUPlace()): paddle.framework.seed(2020) activation = "relu" normalize_before = False - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, source_length, target_length = generate_basic_params( - mode="decoder_layer") - tgt = np.random.rand(batch_size, target_length, - d_model).astype("float32") - memory = np.random.rand(batch_size, source_length, - d_model).astype("float32") - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") + tgt = np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + memory = np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 for cache in [True, False]: - self_attn = MultiHeadAttention(d_model, - n_head, - dropout=attn_dropout) - cross_attn = MultiHeadAttention(d_model, - n_head, - dropout=attn_dropout) + self_attn = MultiHeadAttention( + d_model, n_head, dropout=attn_dropout + ) + cross_attn = MultiHeadAttention( + d_model, n_head, dropout=attn_dropout + ) # paddle decoderlayer: decoder_layer = TransformerDecoderLayer( - d_model, n_head, dim_feedforward, dropout, activation, - attn_dropout, act_dropout, normalize_before) + d_model, + n_head, + dim_feedforward, + dropout, + activation, + attn_dropout, + act_dropout, + normalize_before, + ) cache_objs = None if cache: cache_objs = decoder_layer.gen_cache( - paddle.to_tensor(memory)) - - decoder_output = decoder_layer(paddle.to_tensor(tgt), - paddle.to_tensor(memory), - paddle.to_tensor(tgt_mask), - paddle.to_tensor(memory_mask), - cache_objs) - - decoder_output = decoder_output[0].numpy( - ) if cache else decoder_output.numpy() + paddle.to_tensor(memory) + ) + + decoder_output = decoder_layer( + paddle.to_tensor(tgt), + paddle.to_tensor(memory), + paddle.to_tensor(tgt_mask), + paddle.to_tensor(memory_mask), + cache_objs, + ) + + decoder_output = ( + decoder_output[0].numpy() + if cache + else decoder_output.numpy() + ) # numpy: residual = tgt # self-attn - self_attn_cache = cache_objs[ - 0] if cache_objs is not None else None - tgt = self_attn(paddle.to_tensor(tgt), paddle.to_tensor(tgt), - paddle.to_tensor(tgt), - paddle.to_tensor(tgt_mask), self_attn_cache) + self_attn_cache = ( + cache_objs[0] if cache_objs is not None else None + ) + tgt = self_attn( + paddle.to_tensor(tgt), + paddle.to_tensor(tgt), + paddle.to_tensor(tgt), + paddle.to_tensor(tgt_mask), + self_attn_cache, + ) tgt = tgt[0].numpy() if cache else tgt.numpy() @@ -460,13 +634,16 @@ class TestTransformer(unittest.TestCase): tgt_norm = layer_norm(tgt, d_model, decoder_layer.norm1) residual = tgt_norm # cross-attn - cross_attn_cache = cache_objs[ - 1] if cache_objs is not None else None - tgt = cross_attn(paddle.to_tensor(tgt_norm), - paddle.to_tensor(memory), - paddle.to_tensor(memory), - paddle.to_tensor(memory_mask), - cross_attn_cache) + cross_attn_cache = ( + cache_objs[1] if cache_objs is not None else None + ) + tgt = cross_attn( + paddle.to_tensor(tgt_norm), + paddle.to_tensor(memory), + paddle.to_tensor(memory), + paddle.to_tensor(memory_mask), + cross_attn_cache, + ) tgt = tgt[0].numpy() if cache else tgt.numpy() # postprocess @@ -479,46 +656,67 @@ class TestTransformer(unittest.TestCase): tgt = residual + ffn_output tgt_norm = layer_norm(tgt, d_model, decoder_layer.norm3) - np.testing.assert_allclose(decoder_output, - tgt_norm, - rtol=1e-5, - atol=1e-6) + np.testing.assert_allclose( + decoder_output, tgt_norm, rtol=1e-5, atol=1e-6 + ) def test_encoder(self): - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length = generate_basic_params( - mode="encoder_layer") - - src = np.random.rand(batch_size, sequence_length, - d_model).astype("float32") - - src_mask = np.zeros((batch_size, n_head, sequence_length, - sequence_length)).astype("float32") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) = generate_basic_params(mode="encoder_layer") + + src = np.random.rand(batch_size, sequence_length, d_model).astype( + "float32" + ) + + src_mask = np.zeros( + (batch_size, n_head, sequence_length, sequence_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf with fluid.dygraph.guard(fluid.CPUPlace()): - encoder_layer = TransformerEncoderLayer(d_model, n_head, - dim_feedforward, dropout) + encoder_layer = TransformerEncoderLayer( + d_model, n_head, dim_feedforward, dropout + ) num_layers = 6 encoder = TransformerEncoder(encoder_layer, num_layers) # src, src_mask - enc_output = encoder(paddle.to_tensor(src), - paddle.to_tensor(src_mask)) + enc_output = encoder( + paddle.to_tensor(src), paddle.to_tensor(src_mask) + ) def test_encoder_attr_1(self): - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length = generate_basic_params( - mode="encoder_layer") - - src = np.random.rand(batch_size, sequence_length, - d_model).astype("float32") - - src_mask = np.zeros((batch_size, n_head, sequence_length, - sequence_length)).astype("float32") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) = generate_basic_params(mode="encoder_layer") + + src = np.random.rand(batch_size, sequence_length, d_model).astype( + "float32" + ) + + src_mask = np.zeros( + (batch_size, n_head, sequence_length, sequence_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf with fluid.dygraph.guard(fluid.CPUPlace()): for cache in [True, False]: # paddle - encoder_layer = TransformerEncoderLayer(d_model, n_head, - dim_feedforward, - dropout) + encoder_layer = TransformerEncoderLayer( + d_model, n_head, dim_feedforward, dropout + ) num_layers = 6 encoder = TransformerEncoder(encoder_layer, num_layers) cache_objs = None @@ -526,200 +724,320 @@ class TestTransformer(unittest.TestCase): cache_objs = encoder.gen_cache(paddle.to_tensor(src)) # src, src_mask - enc_output = encoder(paddle.to_tensor(src), - paddle.to_tensor(src_mask), cache_objs) + enc_output = encoder( + paddle.to_tensor(src), + paddle.to_tensor(src_mask), + cache_objs, + ) def test_decoder(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") - tgt = np.random.rand(batch_size, target_length, - d_model).astype("float32") - memory = np.random.rand(batch_size, source_length, - d_model).astype("float32") - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") + tgt = np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + memory = np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 with fluid.dygraph.guard(fluid.CPUPlace()): - decoder_layer = TransformerDecoderLayer(d_model, n_head, - dim_feedforward, dropout) + decoder_layer = TransformerDecoderLayer( + d_model, n_head, dim_feedforward, dropout + ) num_layers = 6 decoder = TransformerDecoder(decoder_layer, num_layers) - output = decoder(paddle.to_tensor(tgt), paddle.to_tensor(memory), - paddle.to_tensor(tgt_mask), - paddle.to_tensor(memory_mask)) + output = decoder( + paddle.to_tensor(tgt), + paddle.to_tensor(memory), + paddle.to_tensor(tgt_mask), + paddle.to_tensor(memory_mask), + ) def test_transformer(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_transformer_attr_1(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout, - weight_attr=[None], - bias_attr=[False]) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + weight_attr=[None], + bias_attr=[False], + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_transformer_attr_2(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout, - weight_attr=[None, None], - bias_attr=[False, False]) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + weight_attr=[None, None], + bias_attr=[False, False], + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_transformer_attr_3(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout, - weight_attr=[None, None, None], - bias_attr=[False, False, True]) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + weight_attr=[None, None, None], + bias_attr=[False, False, True], + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_transformer_attr_boolean(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout, - bias_attr=False) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + bias_attr=False, + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_generate_square_subsequent_mask(self): length = 5 d_model, n_head, dim_feedforward = 8, 4, 64 - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward) + transformer = Transformer( + d_model, n_head, dim_feedforward=dim_feedforward + ) mask = transformer.generate_square_subsequent_mask(length) diff --git a/python/paddle/fluid/tests/unittests/test_translated_layer.py b/python/paddle/fluid/tests/unittests/test_translated_layer.py index ef75af2fb61..2c249b0e558 100644 --- a/python/paddle/fluid/tests/unittests/test_translated_layer.py +++ b/python/paddle/fluid/tests/unittests/test_translated_layer.py @@ -31,14 +31,13 @@ CLASS_NUM = 10 # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): np.random.seed(SEED) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -46,17 +45,18 @@ class RandomDataset(paddle.io.Dataset): class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) self._dropout = paddle.nn.Dropout(p=0.5) - @paddle.jit.to_static(input_spec=[ - paddle.static.InputSpec(shape=[None, IMAGE_SIZE], - dtype='float32', - name='x') - ]) + @paddle.jit.to_static( + input_spec=[ + paddle.static.InputSpec( + shape=[None, IMAGE_SIZE], dtype='float32', name='x' + ) + ] + ) def forward(self, x): return self._linear(x) @@ -69,13 +69,15 @@ def train(layer, loader, loss_fn, opt): loss.backward() opt.step() opt.clear_grad() - print("Epoch {} batch {}: loss = {}".format(epoch_id, batch_id, - np.mean(loss.numpy()))) + print( + "Epoch {} batch {}: loss = {}".format( + epoch_id, batch_id, np.mean(loss.numpy()) + ) + ) return loss class TestTranslatedLayer(unittest.TestCase): - def tearDown(self): self.temp_dir.cleanup() @@ -91,17 +93,20 @@ class TestTranslatedLayer(unittest.TestCase): # create network self.layer = LinearNet() self.loss_fn = nn.CrossEntropyLoss() - self.sgd = opt.SGD(learning_rate=0.001, - parameters=self.layer.parameters()) + self.sgd = opt.SGD( + learning_rate=0.001, parameters=self.layer.parameters() + ) # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - self.loader = paddle.io.DataLoader(dataset, - places=place, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=0) + self.loader = paddle.io.DataLoader( + dataset, + places=place, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=0, + ) self.temp_dir = tempfile.TemporaryDirectory() @@ -109,8 +114,9 @@ class TestTranslatedLayer(unittest.TestCase): train(self.layer, self.loader, self.loss_fn, self.sgd) # save - self.model_path = os.path.join(self.temp_dir.name, - './linear.example.model') + self.model_path = os.path.join( + self.temp_dir.name, './linear.example.model' + ) paddle.jit.save(self.layer, self.model_path) def test_inference_and_fine_tuning(self): @@ -142,15 +148,18 @@ class TestTranslatedLayer(unittest.TestCase): # fine-tuning translated_layer.train() - sgd = opt.SGD(learning_rate=0.001, - parameters=translated_layer.parameters()) + sgd = opt.SGD( + learning_rate=0.001, parameters=translated_layer.parameters() + ) loss = train(translated_layer, self.loader, self.loss_fn, sgd) np.testing.assert_array_equal( orig_loss.numpy(), loss.numpy(), err_msg='original loss:\n{}\nnew loss:\n{}\n'.format( - orig_loss.numpy(), loss.numpy())) + orig_loss.numpy(), loss.numpy() + ), + ) def test_get_program(self): # load @@ -171,9 +180,9 @@ class TestTranslatedLayer(unittest.TestCase): translated_layer = paddle.jit.load(self.model_path) expect_spec = [ - paddle.static.InputSpec(shape=[None, IMAGE_SIZE], - dtype='float32', - name='x') + paddle.static.InputSpec( + shape=[None, IMAGE_SIZE], dtype='float32', name='x' + ) ] actual_spec = translated_layer._input_spec() @@ -185,9 +194,11 @@ class TestTranslatedLayer(unittest.TestCase): translated_layer = paddle.jit.load(self.model_path) expect_spec = [ - paddle.static.InputSpec(shape=[None, CLASS_NUM], - dtype='float32', - name='translated_layer/scale_0.tmp_1') + paddle.static.InputSpec( + shape=[None, CLASS_NUM], + dtype='float32', + name='translated_layer/scale_0.tmp_1', + ) ] actual_spec = translated_layer._output_spec() diff --git a/python/paddle/fluid/tests/unittests/test_transpose_op.py b/python/paddle/fluid/tests/unittests/test_transpose_op.py index 390208c0cdd..67c2b8772c5 100644 --- a/python/paddle/fluid/tests/unittests/test_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_transpose_op.py @@ -27,7 +27,6 @@ paddle.enable_static() class TestTransposeOp(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -39,7 +38,7 @@ class TestTransposeOp(OpTest): } self.outputs = { 'XShape': np.random.random(self.shape).astype("float64"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -58,84 +57,72 @@ class TestTransposeOp(OpTest): class TestCase0(TestTransposeOp): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1(TestTransposeOp): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase2(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) class TestCase5(TestTransposeOp): - def initTestCase(self): self.shape = (2, 16, 96) self.axis = (0, 2, 1) class TestCase6(TestTransposeOp): - def initTestCase(self): self.shape = (2, 10, 12, 16) self.axis = (3, 1, 2, 0) class TestCase7(TestTransposeOp): - def initTestCase(self): self.shape = (2, 10, 2, 16) self.axis = (0, 1, 3, 2) class TestCase8(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (0, 1, 3, 2, 4, 5, 6, 7) class TestCase9(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) class TestCase_ZeroDim(TestTransposeOp): - def initTestCase(self): self.shape = () self.axis = () class TestAutoTuneTransposeOp(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -147,7 +134,7 @@ class TestAutoTuneTransposeOp(OpTest): } self.outputs = { 'XShape': np.random.random(self.shape).astype("float64"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def initTestCase(self): @@ -170,7 +157,6 @@ class TestAutoTuneTransposeOp(OpTest): class TestTransposeBF16Op(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -183,11 +169,10 @@ class TestTransposeBF16Op(OpTest): 'use_mkldnn': self.use_mkldnn, } self.outputs = { - 'XShape': - convert_float_to_uint16( - np.random.random(self.shape).astype("float32")), - 'Out': - self.inputs['X'].transpose(self.axis) + 'XShape': convert_float_to_uint16( + np.random.random(self.shape).astype("float32") + ), + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -206,109 +191,99 @@ class TestTransposeBF16Op(OpTest): class TestTransposeOpBool(TestTransposeOp): - def test_check_grad(self): pass class TestTransposeOpBool1D(TestTransposeOpBool): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool2D(TestTransposeOpBool): - def initTestCase(self): self.shape = (3, 40) self.axis = (1, 0) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool3D(TestTransposeOpBool): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool4D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool5D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool6D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool7D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3) self.axis = (0, 1, 3, 2, 4, 5, 6) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool8D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -322,9 +297,9 @@ class TestTransposeOpError(unittest.TestCase): def test_x_dtype_check(): # the Input(x)'s dtype must be one of [bool, float16, float32, float64, int32, int64] - x1 = fluid.layers.data(name='x1', - shape=[10, 5, 3], - dtype='int8') + x1 = fluid.layers.data( + name='x1', shape=[10, 5, 3], dtype='int8' + ) fluid.layers.transpose(x1, perm=[1, 0, 2]) self.assertRaises(TypeError, test_x_dtype_check) @@ -350,7 +325,6 @@ class TestTransposeOpError(unittest.TestCase): class TestTransposeApi(unittest.TestCase): - def test_static_out(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -360,8 +334,9 @@ class TestTransposeApi(unittest.TestCase): place = paddle.CPUPlace() exe = paddle.static.Executor(place) x_np = np.random.random([2, 3, 4]).astype("float32") - result1, result2 = exe.run(feed={"x": x_np}, - fetch_list=[x_trans1, x_trans2]) + result1, result2 = exe.run( + feed={"x": x_np}, fetch_list=[x_trans1, x_trans2] + ) expected_result1 = np.transpose(x_np, [1, 0, 2]) expected_result2 = np.transpose(x_np, (2, 1, 0)) @@ -387,7 +362,6 @@ class TestTransposeApi(unittest.TestCase): class TestTAPI(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program()): data = fluid.data(shape=[10], dtype="float64", name="data") @@ -395,7 +369,7 @@ class TestTAPI(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) data_np = np.random.random([10]).astype("float64") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -405,7 +379,7 @@ class TestTAPI(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) data_np = np.random.random([10, 5]).astype("float64") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -415,7 +389,7 @@ class TestTAPI(unittest.TestCase): place = fluid.CPUPlace() exe = fluid.Executor(place) data_np = np.random.random([1, 5]).astype("float64") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -454,7 +428,6 @@ class TestTAPI(unittest.TestCase): class TestMoveAxis(unittest.TestCase): - def test_moveaxis1(self): x_np = np.random.randn(2, 3, 4, 5, 7) expected = np.moveaxis(x_np, [0, 4, 3, 2], [1, 3, 2, 0]) @@ -497,8 +470,9 @@ class TestMoveAxis(unittest.TestCase): def test_moveaxis3(self): paddle.disable_static() - x = paddle.to_tensor([[1 + 1j, -1 - 1j], [1 + 1j, -1 - 1j], - [1 + 1j, -1 - 1j]]) + x = paddle.to_tensor( + [[1 + 1j, -1 - 1j], [1 + 1j, -1 - 1j], [1 + 1j, -1 - 1j]] + ) out = x.moveaxis(0, 1) self.assertEqual(out.shape, [2, 3]) paddle.enable_static() @@ -535,7 +509,6 @@ class TestMoveAxis(unittest.TestCase): class TestTransposeDoubleGradCheck(unittest.TestCase): - def transpose_wrapper(self, x): return paddle.transpose(x[0], [1, 0, 2]) @@ -550,17 +523,13 @@ class TestTransposeDoubleGradCheck(unittest.TestCase): out = paddle.transpose(data, [1, 0, 2]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.transpose_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.transpose_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -572,7 +541,6 @@ class TestTransposeDoubleGradCheck(unittest.TestCase): class TestTransposeTripleGradCheck(unittest.TestCase): - def transpose_wrapper(self, x): return paddle.transpose(x[0], [1, 0, 2]) @@ -587,17 +555,13 @@ class TestTransposeTripleGradCheck(unittest.TestCase): out = paddle.transpose(data, [1, 0, 2]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.transpose_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.transpose_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -609,7 +573,6 @@ class TestTransposeTripleGradCheck(unittest.TestCase): class TestTransposeAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_tree_conv_op.py b/python/paddle/fluid/tests/unittests/test_tree_conv_op.py index 114d713b092..d4582c3ad2e 100644 --- a/python/paddle/fluid/tests/unittests/test_tree_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_tree_conv_op.py @@ -51,7 +51,6 @@ def collect_node_patch(og, max_depth): class TestTreeConvOp(OpTest): - def setUp(self): self.n = 17 self.fea_size = 3 @@ -60,22 +59,51 @@ class TestTreeConvOp(OpTest): self.batch_size = 2 self.num_filters = 1 adj_array = [ - 1, 2, 1, 3, 1, 4, 1, 5, 2, 6, 2, 7, 2, 8, 4, 9, 4, 10, 5, 11, 6, 12, - 6, 13, 9, 14, 9, 15, 9, 16, 9, 17 + 1, + 2, + 1, + 3, + 1, + 4, + 1, + 5, + 2, + 6, + 2, + 7, + 2, + 8, + 4, + 9, + 4, + 10, + 5, + 11, + 6, + 12, + 6, + 13, + 9, + 14, + 9, + 15, + 9, + 16, + 9, + 17, ] adj = np.array(adj_array).reshape((1, self.n - 1, 2)).astype('int32') adj = np.tile(adj, (self.batch_size, 1, 1)) self.op_type = 'tree_conv' vectors = np.random.random( - (self.batch_size, self.n, self.fea_size)).astype('float64') + (self.batch_size, self.n, self.fea_size) + ).astype('float64') self.inputs = { - 'EdgeSet': - adj, - 'NodesVector': - vectors, - 'Filter': - np.random.random((self.fea_size, 3, self.output_size, - self.num_filters)).astype('float64') + 'EdgeSet': adj, + 'NodesVector': vectors, + 'Filter': np.random.random( + (self.fea_size, 3, self.output_size, self.num_filters) + ).astype('float64'), } self.attrs = {'max_depth': self.max_depth} vectors = [] @@ -90,9 +118,9 @@ class TestTreeConvOp(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['NodesVector', 'Filter'], - 'Out', - max_relative_error=0.5) + self.check_grad( + ['NodesVector', 'Filter'], 'Out', max_relative_error=0.5 + ) def get_output_naive(self, batch_id): og = [[] for i in range(1, self.n + 2)] @@ -107,71 +135,88 @@ class TestTreeConvOp(OpTest): result = np.zeros((1, W.shape[2], W.shape[3])) for v in patch: eta_t = float(v[4] - v[3]) / float(v[4]) - eta_l = (1.0 - eta_t) * (0.5 if v[2] == 1 else - float(v[1] - 1.0) / float(v[2] - 1.0)) + eta_l = (1.0 - eta_t) * ( + 0.5 if v[2] == 1 else float(v[1] - 1.0) / float(v[2] - 1.0) + ) eta_r = (1.0 - eta_t) * (1.0 - eta_l) x = self.inputs['NodesVector'][batch_id][v[0] - 1] - eta = np.array([eta_l, eta_r, eta_t]).reshape( - (3, 1)).astype('float64') + eta = ( + np.array([eta_l, eta_r, eta_t]) + .reshape((3, 1)) + .astype('float64') + ) Wconvi = np.tensordot(eta, W, axes=([0], [0])) x = np.array(x).reshape((1, 1, self.fea_size)) res = np.tensordot(x, Wconvi, axes=2) result = result + res vec.append(result) vec = np.concatenate(vec, axis=0) - vec = np.concatenate([ - vec, - np.zeros((self.n - vec.shape[0], W.shape[2], W.shape[3]), - dtype='float64') - ], - axis=0) + vec = np.concatenate( + [ + vec, + np.zeros( + (self.n - vec.shape[0], W.shape[2], W.shape[3]), + dtype='float64', + ), + ], + axis=0, + ) return vec class TestTreeConv_OpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): nodes_vector_1 = np.random.random((10, 5)).astype("float32") - edge_set_1 = fluid.layers.data(name='edge_set_1', - shape=[10, 2], - dtype='float32') + edge_set_1 = fluid.layers.data( + name='edge_set_1', shape=[10, 2], dtype='float32' + ) # the nodes_vector of tree_conv must be Variable. - self.assertRaises(TypeError, fluid.contrib.layers.tree_conv, - nodes_vector_1, edge_set_1, 3) - - nodes_vector_2 = fluid.layers.data(name='vectors2', - shape=[10, 5], - dtype='float32') + self.assertRaises( + TypeError, + fluid.contrib.layers.tree_conv, + nodes_vector_1, + edge_set_1, + 3, + ) + + nodes_vector_2 = fluid.layers.data( + name='vectors2', shape=[10, 5], dtype='float32' + ) edge_set_2 = np.random.random((10, 2)).astype("float32") # the edge_set of tree_conv must be Variable. - self.assertRaises(TypeError, fluid.contrib.layers.tree_conv, - nodes_vector_2, edge_set_2, 3) + self.assertRaises( + TypeError, + fluid.contrib.layers.tree_conv, + nodes_vector_2, + edge_set_2, + 3, + ) class TestDygraphTreeConv_OpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - TreeConv = fluid.dygraph.nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2) + TreeConv = fluid.dygraph.nn.TreeConv( + feature_size=5, output_size=6, num_filters=1, max_depth=2 + ) nodes_vector_1 = np.random.random((10, 5)).astype("float32") - edge_set_1 = fluid.layers.data(name='edge_set_1', - shape=[10, 2], - dtype='float32') + edge_set_1 = fluid.layers.data( + name='edge_set_1', shape=[10, 2], dtype='float32' + ) # the nodes_vector of TreeConv must be Variable. - self.assertRaises(TypeError, TreeConv, nodes_vector_1, edge_set_1, - 3) + self.assertRaises( + TypeError, TreeConv, nodes_vector_1, edge_set_1, 3 + ) - nodes_vector_2 = fluid.layers.data(name='vectors2', - shape=[10, 5], - dtype='float32') + nodes_vector_2 = fluid.layers.data( + name='vectors2', shape=[10, 5], dtype='float32' + ) edge_set_2 = np.random.random((10, 2)).astype("float32") # the edge_set of TreeConv must be Variable. - self.assertRaises(TypeError, TreeConv, nodes_vector_2, edge_set_2, - 3) + self.assertRaises( + TypeError, TreeConv, nodes_vector_2, edge_set_2, 3 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_triangular_solve_op.py b/python/paddle/fluid/tests/unittests/test_triangular_solve_op.py index fabef08b189..c71619e3137 100644 --- a/python/paddle/fluid/tests/unittests/test_triangular_solve_op.py +++ b/python/paddle/fluid/tests/unittests/test_triangular_solve_op.py @@ -41,8 +41,9 @@ class TestTriangularSolveOp(OpTest): self.dtype = "float64" def set_output(self): - self.output = np.linalg.solve(np.triu(self.inputs['X']), - self.inputs['Y']) + self.output = np.linalg.solve( + np.triu(self.inputs['X']), self.inputs['Y'] + ) def setUp(self): self.op_type = "triangular_solve" @@ -51,7 +52,7 @@ class TestTriangularSolveOp(OpTest): self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.attrs = { 'upper': self.upper, @@ -144,24 +145,26 @@ class TestTriangularSolveOp5(TestTriangularSolveOp): def set_output(self): x = np.triu(self.inputs['X']) - np.fill_diagonal(x, 1.) + np.fill_diagonal(x, 1.0) y = self.inputs['Y'] self.output = np.linalg.solve(x, y) def test_check_grad_normal(self): x = np.triu(self.inputs['X']) - np.fill_diagonal(x, 1.) + np.fill_diagonal(x, 1.0) grad_out = np.ones([10, 10]).astype('float64') grad_y = np.linalg.solve(x.transpose(1, 0), grad_out) grad_x = -np.matmul(grad_y, self.output.transpose(1, 0)) grad_x = np.triu(grad_x) - np.fill_diagonal(grad_x, 0.) + np.fill_diagonal(grad_x, 0.0) - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[grad_x, grad_y], - user_defined_grad_outputs=[grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[grad_x, grad_y], + user_defined_grad_outputs=[grad_out], + ) # 4D(broadcast) + 4D(broadcast) @@ -245,7 +248,6 @@ class TestTriangularSolveOp9(TestTriangularSolveOp): class TestTriangularSolveAPI(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -264,12 +266,11 @@ class TestTriangularSolveAPI(unittest.TestCase): z_np = np.linalg.solve(np.triu(x_np), y_np) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[z]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": y_np}, + fetch_list=[z], + ) np.testing.assert_allclose(fetches[0], z_np, rtol=1e-05) def test_static(self): @@ -277,7 +278,6 @@ class TestTriangularSolveAPI(unittest.TestCase): self.check_static_result(place=place) def test_dygraph(self): - def run(place): paddle.disable_static(place) x_np = np.random.random([3, 3]).astype(self.dtype) @@ -297,14 +297,15 @@ class TestTriangularSolveAPI(unittest.TestCase): class TestTriangularSolveOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of solve_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.linalg.triangular_solve, x1, y1) # The data type of input must be float32 or float64. @@ -323,20 +324,23 @@ class TestTriangularSolveOpError(unittest.TestCase): # The number of dimensions of input'X must be >= 2. x5 = fluid.data(name="x5", shape=[30], dtype="float64") y5 = fluid.data(name="y5", shape=[30, 30], dtype="float64") - self.assertRaises(ValueError, paddle.linalg.triangular_solve, x5, - y5) + self.assertRaises( + ValueError, paddle.linalg.triangular_solve, x5, y5 + ) # The number of dimensions of input'Y must be >= 2. x6 = fluid.data(name="x6", shape=[30, 30], dtype="float64") y6 = fluid.data(name="y6", shape=[30], dtype="float64") - self.assertRaises(ValueError, paddle.linalg.triangular_solve, x6, - y6) + self.assertRaises( + ValueError, paddle.linalg.triangular_solve, x6, y6 + ) # The inner-most 2 dimensions of input'X should be equal to each other x7 = fluid.data(name="x7", shape=[2, 3, 4], dtype="float64") y7 = fluid.data(name="y7", shape=[2, 4, 3], dtype="float64") - self.assertRaises(ValueError, paddle.linalg.triangular_solve, x7, - y7) + self.assertRaises( + ValueError, paddle.linalg.triangular_solve, x7, y7 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_tril_indices_op.py b/python/paddle/fluid/tests/unittests/test_tril_indices_op.py index fae0221240c..0d18b32304e 100644 --- a/python/paddle/fluid/tests/unittests/test_tril_indices_op.py +++ b/python/paddle/fluid/tests/unittests/test_tril_indices_op.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestTrilIndicesOp(OpTest): - def setUp(self): self.op_type = "tril_indices" self.inputs = {} @@ -34,13 +33,13 @@ class TestTrilIndicesOp(OpTest): def init_config(self): self.attrs = {'rows': 4, 'cols': 4, 'offset': -1} - self.target = np.tril_indices(self.attrs['rows'], self.attrs['offset'], - self.attrs['cols']) + self.target = np.tril_indices( + self.attrs['rows'], self.attrs['offset'], self.attrs['cols'] + ) self.target = np.array(self.target) class TestTrilIndicesOpCase1(TestTrilIndicesOp): - def init_config(self): self.attrs = {'rows': 0, 'cols': 0, 'offset': 0} self.target = np.tril_indices(0, 0, 0) @@ -48,37 +47,40 @@ class TestTrilIndicesOpCase1(TestTrilIndicesOp): class TestTrilIndicesOpCase2(TestTrilIndicesOp): - def init_config(self): self.attrs = {'rows': 4, 'cols': 4, 'offset': 2} - self.target = np.tril_indices(self.attrs['rows'], self.attrs['offset'], - self.attrs['cols']) + self.target = np.tril_indices( + self.attrs['rows'], self.attrs['offset'], self.attrs['cols'] + ) self.target = np.array(self.target) class TestTrilIndicesAPICaseStatic(unittest.TestCase): - def test_static(self): - places = [ - paddle.CPUPlace(), paddle.fluid.CUDAPlace(0) - ] if fluid.core.is_compiled_with_cuda() else [paddle.CPUPlace()] + places = ( + [paddle.CPUPlace(), paddle.fluid.CUDAPlace(0)] + if fluid.core.is_compiled_with_cuda() + else [paddle.CPUPlace()] + ) paddle.enable_static() for place in places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data1 = paddle.tril_indices(4, 4, -1) exe1 = paddle.static.Executor(place) - result1, = exe1.run(feed={}, fetch_list=[data1]) + (result1,) = exe1.run(feed={}, fetch_list=[data1]) expected_result1 = np.tril_indices(4, -1, 4) np.testing.assert_allclose(result1, expected_result1, rtol=1e-05) class TestTrilIndicesAPICaseDygraph(unittest.TestCase): - def test_dygraph(self): - places = [ - paddle.CPUPlace(), paddle.fluid.CUDAPlace(0) - ] if fluid.core.is_compiled_with_cuda() else [paddle.CPUPlace()] + places = ( + [paddle.CPUPlace(), paddle.fluid.CUDAPlace(0)] + if fluid.core.is_compiled_with_cuda() + else [paddle.CPUPlace()] + ) for place in places: with fluid.dygraph.base.guard(place=place): out1 = paddle.tril_indices(4, 4, 2) @@ -91,9 +93,7 @@ class TestTrilIndicesAPICaseDygraph(unittest.TestCase): class TestTrilIndicesAPICaseError(unittest.TestCase): - def test_case_error(self): - def test_num_rows_type_check(): out1 = paddle.tril_indices(1.0, 1, 2) @@ -111,14 +111,14 @@ class TestTrilIndicesAPICaseError(unittest.TestCase): class TestTrilIndicesAPICaseDefault(unittest.TestCase): - def test_default_CPU(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.tril_indices(4, None, 2) exe = paddle.static.Executor(paddle.CPUPlace()) - result, = exe.run(feed={}, fetch_list=[data]) + (result,) = exe.run(feed={}, fetch_list=[data]) expected_result = np.tril_indices(4, 2) np.testing.assert_allclose(result, expected_result, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_tril_triu_op.py b/python/paddle/fluid/tests/unittests/test_tril_triu_op.py index 07cb0aca214..a17a1f24928 100644 --- a/python/paddle/fluid/tests/unittests/test_tril_triu_op.py +++ b/python/paddle/fluid/tests/unittests/test_tril_triu_op.py @@ -21,12 +21,13 @@ from paddle.fluid.framework import Program, program_guard class TrilTriuOpDefaultTest(OpTest): - """ the base class of other op testcases - """ + """the base class of other op testcases""" def setUp(self): self.initTestCase() - self.python_api = paddle.tril if self.real_op_type == 'tril' else paddle.triu + self.python_api = ( + paddle.tril if self.real_op_type == 'tril' else paddle.triu + ) self.real_np_op = getattr(np, self.real_op_type) self.op_type = "tril_triu" @@ -36,9 +37,9 @@ class TrilTriuOpDefaultTest(OpTest): 'lower': True if self.real_op_type == 'tril' else False, } self.outputs = { - 'Out': - self.real_np_op(self.X, self.diagonal) - if self.diagonal else self.real_np_op(self.X) + 'Out': self.real_np_op(self.X, self.diagonal) + if self.diagonal + else self.real_np_op(self.X) } def test_check_output(self): @@ -59,27 +60,29 @@ def case_generator(op_type, Xshape, diagonal, expected): If arg`expercted` is 'success', it will register an Optest case and expect to pass. Otherwise, it will register an API case and check the expect failure. """ - cls_name = "{0}_{1}_shape_{2}_diag_{3}".format(expected, op_type, Xshape, - diagonal) + cls_name = "{0}_{1}_shape_{2}_diag_{3}".format( + expected, op_type, Xshape, diagonal + ) errmsg = { - "diagonal: TypeError": - "diagonal in {} must be a python Int".format(op_type), - "input: ValueError": - "x shape in {} must be at least 2-D".format(op_type), + "diagonal: TypeError": "diagonal in {} must be a python Int".format( + op_type + ), + "input: ValueError": "x shape in {} must be at least 2-D".format( + op_type + ), } class FailureCase(unittest.TestCase): - def test_failure(self): paddle.enable_static() data = fluid.data(shape=Xshape, dtype='float64', name=cls_name) - with self.assertRaisesRegexp(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegexp( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal=diagonal) class SuccessCase(TrilTriuOpDefaultTest): - def initTestCase(self): paddle.enable_static() @@ -104,15 +107,13 @@ cases = { (20, 20): [ '2020', [20], - { - 20: 20 - }, + {20: 20}, (20, 20), 20.20, ], # str, list, dict, tuple, float }, 'input: ValueError': { - (2020, ): [None], + (2020,): [None], }, } for _op_type in ['tril', 'triu']: @@ -121,12 +122,15 @@ for _op_type in ['tril', 'triu']: list( map( lambda _diagonal: case_generator( - _op_type, _Xshape, _diagonal, _expected), _diaglist)) + _op_type, _Xshape, _diagonal, _expected + ), + _diaglist, + ) + ) class TestTrilTriuOpAPI(unittest.TestCase): - """ test case by using API and has -1 dimension - """ + """test case by using API and has -1 dimension""" def test_api(self): paddle.enable_static() @@ -140,8 +144,11 @@ class TestTrilTriuOpAPI(unittest.TestCase): x = fluid.data(shape=[1, 9, -1, 4], dtype=dtype, name='x') tril_out, triu_out = tensor.tril(x), tensor.triu(x) - place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( - ) else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) tril_out, triu_out = exe.run( fluid.default_main_program(), @@ -159,8 +166,10 @@ class TestTrilTriuOpAPI(unittest.TestCase): with fluid.dygraph.guard(): data = np.random.random([1, 9, 9, 4]).astype(dtype) x = fluid.dygraph.to_variable(data) - tril_out, triu_out = tensor.tril(x).numpy(), tensor.triu( - x).numpy() + tril_out, triu_out = ( + tensor.tril(x).numpy(), + tensor.triu(x).numpy(), + ) np.testing.assert_allclose(tril_out, np.tril(data), rtol=1e-05) np.testing.assert_allclose(triu_out, np.triu(data), rtol=1e-05) @@ -176,12 +185,17 @@ class TestTrilTriuOpAPI(unittest.TestCase): x = fluid.data(shape=[1, 9, -1, 4], dtype=dtype, name='x') triu_out = fluid.layers.triu(x) - place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( - ) else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - triu_out = exe.run(fluid.default_main_program(), - feed={"x": data}, - fetch_list=[triu_out]) + triu_out = exe.run( + fluid.default_main_program(), + feed={"x": data}, + fetch_list=[triu_out], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_trilinear_interp_op.py b/python/paddle/fluid/tests/unittests/test_trilinear_interp_op.py index 2548c0aa2db..901d2584661 100755 --- a/python/paddle/fluid/tests/unittests/test_trilinear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_trilinear_interp_op.py @@ -20,15 +20,17 @@ import paddle.fluid as fluid from paddle.nn.functional import interpolate -def trilinear_interp_np(input, - out_d, - out_h, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCDHW'): +def trilinear_interp_np( + input, + out_d, + out_h, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCDHW', +): """trilinear interpolation implement in shape [N, C, D, H, W]""" if data_layout == "NDHWC": input = np.transpose(input, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -44,17 +46,17 @@ def trilinear_interp_np(input, ratio_d = ratio_h = ratio_w = 0.0 if out_d > 1: - if (align_corners): + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: ratio_d = 1.0 * in_d / out_d if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -62,14 +64,14 @@ def trilinear_interp_np(input, out = np.zeros((batch_size, channel, out_d, out_h, out_w)) for i in range(out_d): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: d = int(ratio_d * (i + 0.5) - 0.5) else: d = int(ratio_d * i) d = max(0, d) did = 1 if d < in_d - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_d = max(ratio_d * (i + 0.5) - 0.5, 0) d1lambda = idx_src_d - d else: @@ -77,14 +79,14 @@ def trilinear_interp_np(input, d2lambda = 1.0 - d1lambda for j in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (j + 0.5) - 0.5) else: h = int(ratio_h * j) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (j + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: @@ -92,30 +94,42 @@ def trilinear_interp_np(input, h2lambda = 1.0 - h1lambda for k in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (k + 0.5) - 0.5) else: w = int(ratio_w * k) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (k + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * k - w w2lambda = 1.0 - w1lambda - out[:, :, i, j, k] = \ - d2lambda * \ - (h2lambda * (w2lambda * input[:, :, d, h, w] + \ - w1lambda * input[:, :, d, h, w+wid]) + \ - h1lambda * (w2lambda * input[:, :, d, h+hid, w] + \ - w1lambda * input[:, :, d, h+hid, w+wid])) + \ - d1lambda * \ - (h2lambda * (w2lambda * input[:, :, d+did, h, w] + \ - w1lambda * input[:, :, d+did, h, w+wid]) + \ - h1lambda * (w2lambda * input[:, :, d+did, h+hid, w] + \ - w1lambda * input[:, :, d+did, h+hid, w+wid])) + out[:, :, i, j, k] = d2lambda * ( + h2lambda + * ( + w2lambda * input[:, :, d, h, w] + + w1lambda * input[:, :, d, h, w + wid] + ) + + h1lambda + * ( + w2lambda * input[:, :, d, h + hid, w] + + w1lambda * input[:, :, d, h + hid, w + wid] + ) + ) + d1lambda * ( + h2lambda + * ( + w2lambda * input[:, :, d + did, h, w] + + w1lambda * input[:, :, d + did, h, w + wid] + ) + + h1lambda + * ( + w2lambda * input[:, :, d + did, h + hid, w] + + w1lambda * input[:, :, d + did, h + hid, w + wid] + ) + ) if data_layout == "NDHWC": out = np.transpose(out, (0, 2, 3, 4, 1)) # NCDHW => NDHWC @@ -123,7 +137,6 @@ def trilinear_interp_np(input, class TestTrilinearInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -153,10 +166,17 @@ class TestTrilinearInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -177,7 +197,7 @@ class TestTrilinearInterpOp(OpTest): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': data_layout + 'data_layout': data_layout, } self.outputs = {'Out': output_np} @@ -185,10 +205,9 @@ class TestTrilinearInterpOp(OpTest): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -196,142 +215,132 @@ class TestTrilinearInterpOp(OpTest): self.out_d = 2 self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase1(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 1, 7, 8, 9] self.out_d = 1 self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase2(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 9, 6, 8] self.out_d = 12 self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase3(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 16, 8, 4] self.out_d = 32 self.out_h = 16 self.out_w = 8 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase4(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [4, 1, 7, 8, 9] self.out_d = 1 self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase5(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 3, 9, 6, 8] self.out_d = 12 self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase6(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] self.out_d = 8 self.out_h = 32 self.out_w = 16 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([17, 9, 5]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpSame(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] self.out_d = 16 self.out_h = 8 self.out_w = 4 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpSameHW(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] self.out_d = 8 self.out_h = 8 self.out_w = 4 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpActualShape(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 16, 8, 4] self.out_d = 64 self.out_h = 32 self.out_w = 16 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([33, 19, 7]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpDatalayout(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 4, 4, 4, 3] self.out_d = 2 self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -339,15 +348,15 @@ class TestTrilinearInterpDatalayout(TestTrilinearInterpOp): class TestTrilinearInterpOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "trilinear_interp" self.check_eager = True - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale > 0: out_d = int(self.input_shape[2] * self.scale) @@ -358,9 +367,16 @@ class TestTrilinearInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -373,14 +389,14 @@ class TestTrilinearInterpOpUint8(OpTest): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=self.check_eager) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -388,87 +404,79 @@ class TestTrilinearInterpOpUint8(OpTest): self.out_d = 13 self.out_h = 10 self.out_w = 9 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase1Uint8(TestTrilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 16, 8, 4] self.out_d = 13 self.out_h = 7 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase2Uint8(TestTrilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [4, 1, 7, 8, 9] self.out_d = 3 self.out_h = 5 self.out_w = 13 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 15, 21]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpOtherMethod1(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestTrilinearInterpWithMethod2(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestTrilinearInterpWithMethod3(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestTrilinearInterpScale1(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] self.out_d = 82 self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpScale2(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] self.out_d = 60 self.out_h = 40 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpScale3(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] @@ -481,7 +489,6 @@ class TestTrilinearInterpScale3(TestTrilinearInterpOp): class TestTrilinearInterpZero(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 11] @@ -494,7 +501,6 @@ class TestTrilinearInterpZero(TestTrilinearInterpOp): class TestTrilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -506,7 +512,7 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): self.attrs = { 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } input_np = np.random.random(self.input_shape).astype("float32") @@ -530,27 +536,34 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.check_eager = False self.attrs['out_d'] = self.out_d self.attrs['out_h'] = self.out_h self.attrs['out_w'] = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.outputs = {'Out': output_np} def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -558,7 +571,7 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): self.out_d = 2 self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [2, 3, 3] self.align_corners = True self.align_mode = 1 @@ -566,7 +579,6 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): # out_size is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case1(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 9, 6, 8] @@ -581,14 +593,13 @@ class TestTrilinearInterp_attr_tensor_Case1(TestTrilinearInterpOp_attr_tensor): # scale is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case2(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 8, 8, 4] self.out_d = 16 self.out_h = 12 self.out_w = 4 - self.scale = 0. + self.scale = 0.0 self.out_size = [16, 4, 10] self.align_corners = True self.align_mode = 1 @@ -597,7 +608,6 @@ class TestTrilinearInterp_attr_tensor_Case2(TestTrilinearInterpOp_attr_tensor): # scale is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case3(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 8, 8, 4] @@ -612,7 +622,6 @@ class TestTrilinearInterp_attr_tensor_Case3(TestTrilinearInterpOp_attr_tensor): class TestTrilinearInterpAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 9, 4], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 9, 4, 3], dtype="float32") @@ -620,31 +629,28 @@ class TestTrilinearInterpAPI(unittest.TestCase): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[3], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[3], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) - out1 = fluid.layers.resize_trilinear(y, - out_shape=[12, 18, 8], - data_format='NDHWC') + out1 = fluid.layers.resize_trilinear( + y, out_shape=[12, 18, 8], data_format='NDHWC' + ) out2 = fluid.layers.resize_trilinear(x, out_shape=[12, dim, 8]) out3 = fluid.layers.resize_trilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_trilinear(x, - out_shape=[4, 4, 8], - actual_shape=actual_size) + out4 = fluid.layers.resize_trilinear( + x, out_shape=[4, 4, 8], actual_shape=actual_size + ) out5 = fluid.layers.resize_trilinear(x, scale=scale_tensor) - out6 = interpolate(x, - scale_factor=scale_tensor, - mode='trilinear', - data_format="NCDHW") - out7 = interpolate(x, - size=[4, 4, 8], - mode='trilinear', - data_format="NCDHW") - out8 = interpolate(x, - size=shape_tensor, - mode='trilinear', - data_format="NCDHW") + out6 = interpolate( + x, scale_factor=scale_tensor, mode='trilinear', data_format="NCDHW" + ) + out7 = interpolate( + x, size=[4, 4, 8], mode='trilinear', data_format="NCDHW" + ) + out8 = interpolate( + x, size=shape_tensor, mode='trilinear', data_format="NCDHW" + ) x_data = np.random.random((2, 3, 6, 9, 4)).astype("float32") dim_data = np.array([18]).astype("int32") @@ -658,40 +664,39 @@ class TestTrilinearInterpAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 4, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = trilinear_interp_np(x_data, - out_d=12, - out_h=18, - out_w=8, - align_mode=1) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 4, 1)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 4, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = trilinear_interp_np( + x_data, out_d=12, out_h=18, out_w=8, align_mode=1 + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 4, 1)), rtol=1e-05 + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res, rtol=1e-05) class TestTrilinearInterpOpException(unittest.TestCase): - def test_exception(self): input = fluid.data(name="input", shape=[2, 3, 6, 9, 4], dtype="float32") def attr_data_format(): # for 5-D input, data_format only can be NCDHW or NDHWC - out = fluid.layers.resize_trilinear(input, - out_shape=[4, 8, 4], - data_format='NHWC') + out = fluid.layers.resize_trilinear( + input, out_shape=[4, 8, 4], data_format='NHWC' + ) self.assertRaises(ValueError, attr_data_format) diff --git a/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py index 6b5391aa195..67ab8805dc1 100755 --- a/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py @@ -24,18 +24,20 @@ import paddle np.random.seed(123) -def trilinear_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='NCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='trilinear', - align_corners=True, - align_mode=0): +def trilinear_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='NCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='trilinear', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -45,26 +47,39 @@ def trilinear_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.trilinear_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, - scale, interp_method, align_corners, - align_mode) - - -def trilinear_interp_np(input, - out_d, - out_h, - out_w, - scale_d=0, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCDHW'): + return paddle._C_ops.trilinear_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) + + +def trilinear_interp_np( + input, + out_d, + out_h, + out_w, + scale_d=0, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCDHW', +): """trilinear interpolation implement in shape [N, C, D, H, W]""" if data_layout == "NDHWC": input = np.transpose(input, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -80,7 +95,7 @@ def trilinear_interp_np(input, ratio_d = ratio_h = ratio_w = 0.0 if out_d > 1: - if (align_corners): + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: if scale_d > 0: @@ -88,7 +103,7 @@ def trilinear_interp_np(input, else: ratio_d = 1.0 * in_d / out_d if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -96,7 +111,7 @@ def trilinear_interp_np(input, else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -107,14 +122,14 @@ def trilinear_interp_np(input, out = np.zeros((batch_size, channel, out_d, out_h, out_w)) for i in range(out_d): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: d = int(ratio_d * (i + 0.5) - 0.5) else: d = int(ratio_d * i) d = max(0, d) did = 1 if d < in_d - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_d = max(ratio_d * (i + 0.5) - 0.5, 0) d1lambda = idx_src_d - d else: @@ -122,14 +137,14 @@ def trilinear_interp_np(input, d2lambda = 1.0 - d1lambda for j in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (j + 0.5) - 0.5) else: h = int(ratio_h * j) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (j + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: @@ -137,30 +152,42 @@ def trilinear_interp_np(input, h2lambda = 1.0 - h1lambda for k in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (k + 0.5) - 0.5) else: w = int(ratio_w * k) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (k + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * k - w w2lambda = 1.0 - w1lambda - out[:, :, i, j, k] = \ - d2lambda * \ - (h2lambda * (w2lambda * input[:, :, d, h, w] + - w1lambda * input[:, :, d, h, w+wid]) + - h1lambda * (w2lambda * input[:, :, d, h+hid, w] + - w1lambda * input[:, :, d, h+hid, w+wid])) + \ - d1lambda * \ - (h2lambda * (w2lambda * input[:, :, d+did, h, w] + - w1lambda * input[:, :, d+did, h, w+wid]) + - h1lambda * (w2lambda * input[:, :, d+did, h+hid, w] + - w1lambda * input[:, :, d+did, h+hid, w+wid])) + out[:, :, i, j, k] = d2lambda * ( + h2lambda + * ( + w2lambda * input[:, :, d, h, w] + + w1lambda * input[:, :, d, h, w + wid] + ) + + h1lambda + * ( + w2lambda * input[:, :, d, h + hid, w] + + w1lambda * input[:, :, d, h + hid, w + wid] + ) + ) + d1lambda * ( + h2lambda + * ( + w2lambda * input[:, :, d + did, h, w] + + w1lambda * input[:, :, d + did, h, w + wid] + ) + + h1lambda + * ( + w2lambda * input[:, :, d + did, h + hid, w] + + w1lambda * input[:, :, d + did, h + hid, w + wid] + ) + ) if data_layout == "NDHWC": out = np.transpose(out, (0, 2, 3, 4, 1)) # NCDHW => NDHWC @@ -168,7 +195,6 @@ def trilinear_interp_np(input, class TestTrilinearInterpOp(OpTest): - def setUp(self): self.python_api = trilinear_interp_test self.out_size = None @@ -211,10 +237,20 @@ class TestTrilinearInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, scale_d, - scale_h, scale_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + scale_d, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -234,7 +270,7 @@ class TestTrilinearInterpOp(OpTest): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': data_layout + 'data_layout': data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -249,10 +285,9 @@ class TestTrilinearInterpOp(OpTest): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -267,7 +302,6 @@ class TestTrilinearInterpOp(OpTest): class TestTrilinearInterpCase1(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 1, 7, 8, 9] @@ -280,7 +314,6 @@ class TestTrilinearInterpCase1(TestTrilinearInterpOp): class TestTrilinearInterpCase2(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 9, 6, 8] @@ -293,7 +326,6 @@ class TestTrilinearInterpCase2(TestTrilinearInterpOp): class TestTrilinearInterpCase3(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 16, 8, 4] @@ -306,7 +338,6 @@ class TestTrilinearInterpCase3(TestTrilinearInterpOp): class TestTrilinearInterpCase4(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [4, 1, 7, 8, 9] @@ -320,7 +351,6 @@ class TestTrilinearInterpCase4(TestTrilinearInterpOp): class TestTrilinearInterpCase5(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 3, 9, 6, 8] @@ -334,7 +364,6 @@ class TestTrilinearInterpCase5(TestTrilinearInterpOp): class TestTrilinearInterpCase6(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] @@ -348,7 +377,6 @@ class TestTrilinearInterpCase6(TestTrilinearInterpOp): class TestTrilinearInterpSame(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] @@ -361,7 +389,6 @@ class TestTrilinearInterpSame(TestTrilinearInterpOp): class TestTrilinearInterpSameHW(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] @@ -374,7 +401,6 @@ class TestTrilinearInterpSameHW(TestTrilinearInterpOp): class TestTrilinearInterpActualShape(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 16, 8, 4] @@ -388,7 +414,6 @@ class TestTrilinearInterpActualShape(TestTrilinearInterpOp): class TestTrilinearInterpDatalayout(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 4, 4, 4, 3] @@ -403,7 +428,6 @@ class TestTrilinearInterpDatalayout(TestTrilinearInterpOp): class TestTrilinearInterpOpUint8(OpTest): - def setUp(self): self.python_api = trilinear_interp_test self.out_size = None @@ -411,8 +435,9 @@ class TestTrilinearInterpOpUint8(OpTest): self.init_test_case() self.op_type = "trilinear_interp_v2" self.check_eager = True - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -432,9 +457,19 @@ class TestTrilinearInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, 0, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + 0, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -446,7 +481,7 @@ class TestTrilinearInterpOpUint8(OpTest): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -458,9 +493,9 @@ class TestTrilinearInterpOpUint8(OpTest): self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=self.check_eager) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -474,7 +509,6 @@ class TestTrilinearInterpOpUint8(OpTest): class TestTrilinearInterpCase1Uint8(TestTrilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 16, 8, 4] @@ -487,7 +521,6 @@ class TestTrilinearInterpCase1Uint8(TestTrilinearInterpOpUint8): class TestTrilinearInterpCase2Uint8(TestTrilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [4, 1, 7, 8, 9] @@ -501,54 +534,48 @@ class TestTrilinearInterpCase2Uint8(TestTrilinearInterpOpUint8): class TestTrilinearInterpOtherMethod1(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestTrilinearInterpWithMethod2(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestTrilinearInterpWithMethod3(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestTrilinearInterpScale1(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] self.out_d = 19 self.out_h = 15 self.out_w = 8 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpScale2(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] self.out_d = 30 self.out_h = 20 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpScale3(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] @@ -561,7 +588,6 @@ class TestTrilinearInterpScale3(TestTrilinearInterpOp): class TestTrilinearInterpZero(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 11] @@ -574,7 +600,6 @@ class TestTrilinearInterpZero(TestTrilinearInterpOp): class TestTrilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.python_api = trilinear_interp_test self.out_size = None @@ -587,7 +612,7 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): self.attrs = { 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } input_np = np.random.random(self.input_shape).astype("float32") @@ -619,8 +644,9 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.check_eager = False @@ -634,19 +660,28 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, 0, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + 0, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.outputs = {'Out': output_np} def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -662,7 +697,6 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): # out_size is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case1(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 9, 6, 8] @@ -677,7 +711,6 @@ class TestTrilinearInterp_attr_tensor_Case1(TestTrilinearInterpOp_attr_tensor): # scale is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case2(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 8, 8, 4] @@ -693,7 +726,6 @@ class TestTrilinearInterp_attr_tensor_Case2(TestTrilinearInterpOp_attr_tensor): # scale is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case3(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 8, 8, 4] @@ -708,7 +740,6 @@ class TestTrilinearInterp_attr_tensor_Case3(TestTrilinearInterpOp_attr_tensor): class TestTrilinearInterpAPI(unittest.TestCase): - def test_imperative_case(self): with _test_eager_guard(): self.func_case() @@ -721,31 +752,28 @@ class TestTrilinearInterpAPI(unittest.TestCase): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[3], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[3], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) - out1 = fluid.layers.resize_trilinear(y, - out_shape=[12, 18, 8], - data_format='NDHWC') + out1 = fluid.layers.resize_trilinear( + y, out_shape=[12, 18, 8], data_format='NDHWC' + ) out2 = fluid.layers.resize_trilinear(x, out_shape=[12, dim, 8]) out3 = fluid.layers.resize_trilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_trilinear(x, - out_shape=[4, 4, 8], - actual_shape=actual_size) + out4 = fluid.layers.resize_trilinear( + x, out_shape=[4, 4, 8], actual_shape=actual_size + ) out5 = fluid.layers.resize_trilinear(x, scale=scale_tensor) - out6 = interpolate(x, - scale_factor=scale_tensor, - mode='trilinear', - data_format="NCDHW") - out7 = interpolate(x, - size=[4, 4, 8], - mode='trilinear', - data_format="NCDHW") - out8 = interpolate(x, - size=shape_tensor, - mode='trilinear', - data_format="NCDHW") + out6 = interpolate( + x, scale_factor=scale_tensor, mode='trilinear', data_format="NCDHW" + ) + out7 = interpolate( + x, size=[4, 4, 8], mode='trilinear', data_format="NCDHW" + ) + out8 = interpolate( + x, size=shape_tensor, mode='trilinear', data_format="NCDHW" + ) x_data = np.random.random((2, 3, 6, 9, 4)).astype("float32") dim_data = np.array([18]).astype("int32") @@ -759,26 +787,26 @@ class TestTrilinearInterpAPI(unittest.TestCase): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 4, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = trilinear_interp_np(x_data, - out_d=12, - out_h=18, - out_w=8, - align_mode=1) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 4, 1)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 4, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = trilinear_interp_np( + x_data, out_d=12, out_h=18, out_w=8, align_mode=1 + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 4, 1)), rtol=1e-05 + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res, rtol=1e-05) @@ -787,30 +815,28 @@ class TestTrilinearInterpAPI(unittest.TestCase): # Preceding PR link: https://github.com/PaddlePaddle/Paddle/pull/26520/files#diff-ee0c2b73d08659e90a8f3ac48451a6588d35e1613742f864f9aad4394e12c290 with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) - out9 = interpolate(x, - size=[12, 18, 8], - mode='trilinear', - data_format="NCDHW") + out9 = interpolate( + x, size=[12, 18, 8], mode='trilinear', data_format="NCDHW" + ) class TestTrilinearInterpOpException(unittest.TestCase): - def test_exception(self): input = fluid.data(name="input", shape=[2, 3, 6, 9, 4], dtype="float32") def attr_data_format(): # for 5-D input, data_format only can be NCDHW or NDHWC - out = fluid.layers.resize_trilinear(input, - out_shape=[4, 8, 4], - data_format='NHWC') + out = fluid.layers.resize_trilinear( + input, out_shape=[4, 8, 4], data_format='NHWC' + ) self.assertRaises(ValueError, attr_data_format) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestTrilinearInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 4, 4, 4] @@ -824,12 +850,14 @@ class TestTrilinearInterpOpForFloat16(unittest.TestCase): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - size=self.out_size.tolist(), - mode=self.interp_method, - align_corners=self.align_corners, - align_mode=self.align_mode, - data_format=self.data_layout) + y = interpolate( + x, + size=self.out_size.tolist(), + mode=self.interp_method, + align_corners=self.align_corners, + align_mode=self.align_mode, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') @@ -848,10 +876,10 @@ class TestTrilinearInterpOpForFloat16(unittest.TestCase): np.testing.assert_allclose(x_g_np_1, x_g_np_2, rtol=1e-05) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestTrilinearInterpDatalayoutForFloat16(TestTrilinearInterpOpForFloat16): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 4, 4, 4, 3] diff --git a/python/paddle/fluid/tests/unittests/test_triplet_margin_loss.py b/python/paddle/fluid/tests/unittests/test_triplet_margin_loss.py index bf6faeca556..59f63a699d5 100644 --- a/python/paddle/fluid/tests/unittests/test_triplet_margin_loss.py +++ b/python/paddle/fluid/tests/unittests/test_triplet_margin_loss.py @@ -27,11 +27,9 @@ def call_TripletMarginLoss_layer( eps=1e-6, reduction='mean', ): - triplet_margin_loss = paddle.nn.TripletMarginLoss(p=p, - epsilon=eps, - margin=margin, - swap=swap, - reduction=reduction) + triplet_margin_loss = paddle.nn.TripletMarginLoss( + p=p, epsilon=eps, margin=margin, swap=swap, reduction=reduction + ) res = triplet_margin_loss( input=input, positive=positive, @@ -50,102 +48,116 @@ def call_TripletMarginLoss_functional( eps=1e-6, reduction='mean', ): - res = paddle.nn.functional.triplet_margin_loss(input=input, - positive=positive, - negative=negative, - p=p, - epsilon=eps, - margin=margin, - swap=swap, - reduction=reduction) + res = paddle.nn.functional.triplet_margin_loss( + input=input, + positive=positive, + negative=negative, + p=p, + epsilon=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) return res -def test_static(place, - input_np, - positive_np, - negative_np, - p=2, - margin=0.3, - swap=False, - eps=1e-6, - reduction='mean', - functional=False): +def test_static( + place, + input_np, + positive_np, + negative_np, + p=2, + margin=0.3, + swap=False, + eps=1e-6, + reduction='mean', + functional=False, +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype='float64') - positive = paddle.static.data(name='positive', - shape=positive_np.shape, - dtype='float64') - negative = paddle.static.data(name='negative', - shape=negative_np.shape, - dtype='float64') + input = paddle.static.data( + name='input', shape=input_np.shape, dtype='float64' + ) + positive = paddle.static.data( + name='positive', shape=positive_np.shape, dtype='float64' + ) + negative = paddle.static.data( + name='negative', shape=negative_np.shape, dtype='float64' + ) feed_dict = { "input": input_np, "positive": positive_np, - "negative": negative_np + "negative": negative_np, } if functional: - res = call_TripletMarginLoss_functional(input=input, - positive=positive, - negative=negative, - p=p, - eps=eps, - margin=margin, - swap=swap, - reduction=reduction) + res = call_TripletMarginLoss_functional( + input=input, + positive=positive, + negative=negative, + p=p, + eps=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) else: - res = call_TripletMarginLoss_layer(input=input, - positive=positive, - negative=negative, - p=p, - eps=eps, - margin=margin, - swap=swap, - reduction=reduction) + res = call_TripletMarginLoss_layer( + input=input, + positive=positive, + negative=negative, + p=p, + eps=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) exe = paddle.static.Executor(place) static_result = exe.run(prog, feed=feed_dict, fetch_list=[res])[0] return static_result -def test_dygraph(place, - input, - positive, - negative, - p=2, - margin=0.3, - swap=False, - eps=1e-6, - reduction='mean', - functional=False): +def test_dygraph( + place, + input, + positive, + negative, + p=2, + margin=0.3, + swap=False, + eps=1e-6, + reduction='mean', + functional=False, +): paddle.disable_static() input = paddle.to_tensor(input) positive = paddle.to_tensor(positive) negative = paddle.to_tensor(negative) if functional: - dy_res = call_TripletMarginLoss_functional(input=input, - positive=positive, - negative=negative, - p=p, - eps=eps, - margin=margin, - swap=swap, - reduction=reduction) + dy_res = call_TripletMarginLoss_functional( + input=input, + positive=positive, + negative=negative, + p=p, + eps=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) else: - dy_res = call_TripletMarginLoss_layer(input=input, - positive=positive, - negative=negative, - p=p, - eps=eps, - margin=margin, - swap=swap, - reduction=reduction) + dy_res = call_TripletMarginLoss_layer( + input=input, + positive=positive, + negative=negative, + p=p, + eps=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -179,7 +191,6 @@ def calc_triplet_margin_loss( class TestTripletMarginLoss(unittest.TestCase): - def test_TripletMarginLoss(self): shape = (2, 2) input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) @@ -192,10 +203,12 @@ class TestTripletMarginLoss(unittest.TestCase): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - expected = calc_triplet_margin_loss(input=input, - positive=positive, - negative=negative, - reduction=reduction) + expected = calc_triplet_margin_loss( + input=input, + positive=positive, + negative=negative, + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -212,57 +225,59 @@ class TestTripletMarginLoss(unittest.TestCase): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_result, - expected, - rtol=1e-5, - atol=1e-8) - static_functional = test_static(place=place, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_result, expected, rtol=1e-5, atol=1e-8 + ) + static_functional = test_static( + place=place, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.loss.TripletMarginLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.loss.TripletMarginLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') positive = paddle.to_tensor([[0.0, 1.0]], dtype='float32') negative = paddle.to_tensor([[0.2, 0.1]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.triplet_margin_loss, - input=input, - positive=positive, - negative=negative, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.triplet_margin_loss, + input=input, + positive=positive, + negative=negative, + reduction="unsupport reduction", + ) paddle.enable_static() def test_TripletMarginLoss_dimension(self): @@ -295,11 +310,13 @@ class TestTripletMarginLoss(unittest.TestCase): input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) positive = np.random.uniform(0, 2, size=shape).astype(np.float64) negative = np.random.uniform(0, 2, size=shape).astype(np.float64) - expected = calc_triplet_margin_loss(input=input, - swap=True, - positive=positive, - negative=negative, - reduction=reduction) + expected = calc_triplet_margin_loss( + input=input, + swap=True, + positive=positive, + negative=negative, + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -318,41 +335,40 @@ class TestTripletMarginLoss(unittest.TestCase): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) np.testing.assert_allclose(dy_result, expected, rtol=1e-5, atol=1e-8) - static_functional = test_static(place=place, - swap=True, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - swap=True, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + static_functional = test_static( + place=place, + swap=True, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + swap=True, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginLoss_margin(self): paddle.disable_static() @@ -379,11 +395,13 @@ class TestTripletMarginLoss(unittest.TestCase): input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) positive = np.random.uniform(0, 2, size=shape).astype(np.float64) negative = np.random.uniform(0, 2, size=shape).astype(np.float64) - expected = calc_triplet_margin_loss(input=input, - p=p, - positive=positive, - negative=negative, - reduction=reduction) + expected = calc_triplet_margin_loss( + input=input, + p=p, + positive=positive, + negative=negative, + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -402,41 +420,40 @@ class TestTripletMarginLoss(unittest.TestCase): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) np.testing.assert_allclose(dy_result, expected, rtol=1e-5, atol=1e-8) - static_functional = test_static(place=place, - p=p, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - p=p, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + static_functional = test_static( + place=place, + p=p, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + p=p, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_triplet_margin_with_distance_loss.py b/python/paddle/fluid/tests/unittests/test_triplet_margin_with_distance_loss.py index 8b7f79b77ab..c8d4da36a1c 100644 --- a/python/paddle/fluid/tests/unittests/test_triplet_margin_with_distance_loss.py +++ b/python/paddle/fluid/tests/unittests/test_triplet_margin_with_distance_loss.py @@ -30,7 +30,8 @@ def call_TripletMarginDistanceLoss_layer( distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) res = triplet_margin_with_distance_loss( input=input, positive=positive, @@ -55,35 +56,38 @@ def call_TripletMaginDistanceLoss_functional( distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) return res -def test_static(place, - input_np, - positive_np, - negative_np, - distance_function=None, - margin=0.3, - swap=False, - reduction='mean', - functional=False): +def test_static( + place, + input_np, + positive_np, + negative_np, + distance_function=None, + margin=0.3, + swap=False, + reduction='mean', + functional=False, +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype='float64') - positive = paddle.static.data(name='positive', - shape=positive_np.shape, - dtype='float64') - negative = paddle.static.data(name='negative', - shape=negative_np.shape, - dtype='float64') + input = paddle.static.data( + name='input', shape=input_np.shape, dtype='float64' + ) + positive = paddle.static.data( + name='positive', shape=positive_np.shape, dtype='float64' + ) + negative = paddle.static.data( + name='negative', shape=negative_np.shape, dtype='float64' + ) feed_dict = { "input": input_np, "positive": positive_np, - "negative": negative_np + "negative": negative_np, } if functional: @@ -94,7 +98,8 @@ def test_static(place, distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) else: res = call_TripletMarginDistanceLoss_layer( input=input, @@ -103,7 +108,8 @@ def test_static(place, distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) exe = paddle.static.Executor(place) static_result = exe.run(prog, feed=feed_dict, fetch_list=[res])[0] @@ -111,15 +117,17 @@ def test_static(place, return static_result -def test_dygraph(place, - input, - positive, - negative, - distance_function=None, - margin=0.3, - swap=False, - reduction='mean', - functional=False): +def test_dygraph( + place, + input, + positive, + negative, + distance_function=None, + margin=0.3, + swap=False, + reduction='mean', + functional=False, +): paddle.disable_static() input = paddle.to_tensor(input) positive = paddle.to_tensor(positive) @@ -133,7 +141,8 @@ def test_dygraph(place, distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) else: dy_res = call_TripletMarginDistanceLoss_layer( input=input, @@ -142,7 +151,8 @@ def test_dygraph(place, distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -177,7 +187,6 @@ def calc_triplet_margin_distance_loss( class TestTripletMarginWithDistanceLoss(unittest.TestCase): - def test_TripletMarginDistanceLoss(self): shape = (5, 5) input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) @@ -194,7 +203,8 @@ class TestTripletMarginWithDistanceLoss(unittest.TestCase): input=input, positive=positive, negative=negative, - reduction=reduction) + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -211,48 +221,48 @@ class TestTripletMarginWithDistanceLoss(unittest.TestCase): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_result, - expected, - rtol=1e-5, - atol=1e-8) - static_functional = test_static(place=place, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_result, expected, rtol=1e-5, atol=1e-8 + ) + static_functional = test_static( + place=place, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginDistanceLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.TripletMarginWithDistanceLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.TripletMarginWithDistanceLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') positive = paddle.to_tensor([[0.0, 1.0]], dtype='float32') negative = paddle.to_tensor([[0.2, 0.1]], dtype='float32') @@ -262,11 +272,11 @@ class TestTripletMarginWithDistanceLoss(unittest.TestCase): input=input, positive=positive, negative=negative, - reduction="unsupport reduction") + reduction="unsupport reduction", + ) paddle.enable_static() def test_TripletMarginDistanceLoss_distance_function(self): - def distance_function_1(x1, x2): return 1.0 - paddle.nn.functional.cosine_similarity(x1, x2) @@ -299,28 +309,30 @@ class TestTripletMarginWithDistanceLoss(unittest.TestCase): distance_function=distance_function, reduction=reduction, ) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) - static_functional = test_static(place=place, - input_np=input, - positive_np=positive, - negative_np=negative, - distance_function=distance_function, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - positive=positive, - negative=negative, - distance_function=distance_function, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) + static_functional = test_static( + place=place, + input_np=input, + positive_np=positive, + negative_np=negative, + distance_function=distance_function, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + positive=positive, + negative=negative, + distance_function=distance_function, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginWithDistanceLoss_distance_funtion_error(self): paddle.disable_static() @@ -357,7 +369,8 @@ class TestTripletMarginWithDistanceLoss(unittest.TestCase): positive=positive, negative=negative, ) - triplet_margin_with_distance_loss = paddle.nn.loss.TripletMarginWithDistanceLoss( + triplet_margin_with_distance_loss = ( + paddle.nn.loss.TripletMarginWithDistanceLoss() ) self.assertRaises( ValueError, @@ -375,11 +388,13 @@ class TestTripletMarginWithDistanceLoss(unittest.TestCase): input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) positive = np.random.uniform(0, 2, size=shape).astype(np.float64) negative = np.random.uniform(0, 2, size=shape).astype(np.float64) - expected = calc_triplet_margin_distance_loss(input=input, - swap=True, - positive=positive, - negative=negative, - reduction=reduction) + expected = calc_triplet_margin_distance_loss( + input=input, + swap=True, + positive=positive, + negative=negative, + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -398,41 +413,40 @@ class TestTripletMarginWithDistanceLoss(unittest.TestCase): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) np.testing.assert_allclose(dy_result, expected, rtol=1e-5, atol=1e-8) - static_functional = test_static(place=place, - swap=True, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - swap=True, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + static_functional = test_static( + place=place, + swap=True, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + swap=True, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginWithDistanceLoss_margin(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_triu_indices_op.py b/python/paddle/fluid/tests/unittests/test_triu_indices_op.py index 6863517874e..1ad6b7bbd45 100644 --- a/python/paddle/fluid/tests/unittests/test_triu_indices_op.py +++ b/python/paddle/fluid/tests/unittests/test_triu_indices_op.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestTriuIndicesOp(OpTest): - def setUp(self): self.op_type = "triu_indices" self.inputs = {} @@ -34,13 +33,13 @@ class TestTriuIndicesOp(OpTest): def init_config(self): self.attrs = {'row': 4, 'col': 4, 'offset': -1} - self.target = np.triu_indices(self.attrs['row'], self.attrs['offset'], - self.attrs['col']) + self.target = np.triu_indices( + self.attrs['row'], self.attrs['offset'], self.attrs['col'] + ) self.target = np.array(self.target) class TestTriuIndicesOpCase1(TestTriuIndicesOp): - def init_config(self): self.attrs = {'row': 0, 'col': 0, 'offset': 0} self.target = np.triu_indices(0, 0, 0) @@ -48,23 +47,23 @@ class TestTriuIndicesOpCase1(TestTriuIndicesOp): class TestTriuIndicesOpCase2(TestTriuIndicesOp): - def init_config(self): self.attrs = {'row': 4, 'col': 4, 'offset': 2} - self.target = np.triu_indices(self.attrs['row'], self.attrs['offset'], - self.attrs['col']) + self.target = np.triu_indices( + self.attrs['row'], self.attrs['offset'], self.attrs['col'] + ) self.target = np.array(self.target) class TestTriuIndicesAPICaseStatic(unittest.TestCase): - def test_static(self): if fluid.core.is_compiled_with_cuda(): place = paddle.fluid.CUDAPlace(0) else: place = paddle.CPUPlace() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.triu_indices(4, 4, -1) exe = paddle.static.Executor(place) result = exe.run(feed={}, fetch_list=[data]) @@ -73,7 +72,6 @@ class TestTriuIndicesAPICaseStatic(unittest.TestCase): class TestTriuIndicesAPICaseDygraph(unittest.TestCase): - def test_dygraph(self): if fluid.core.is_compiled_with_cuda(): place = paddle.fluid.CUDAPlace(0) @@ -90,9 +88,7 @@ class TestTriuIndicesAPICaseDygraph(unittest.TestCase): class TestTriuIndicesAPICaseError(unittest.TestCase): - def test_case_error(self): - def test_num_rows_type_check(): out1 = paddle.triu_indices(1.0, 1, 2) @@ -110,11 +106,11 @@ class TestTriuIndicesAPICaseError(unittest.TestCase): class TestTriuIndicesAPICaseDefault(unittest.TestCase): - def test_default_CPU(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.triu_indices(4, None, 2) exe = paddle.static.Executor(paddle.CPUPlace()) result = exe.run(feed={}, fetch_list=[data]) diff --git a/python/paddle/fluid/tests/unittests/test_trunc_op.py b/python/paddle/fluid/tests/unittests/test_trunc_op.py index 0a410e53805..f8e23bd8074 100644 --- a/python/paddle/fluid/tests/unittests/test_trunc_op.py +++ b/python/paddle/fluid/tests/unittests/test_trunc_op.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestTruncOp(OpTest): - def setUp(self): self.op_type = "trunc" self.python_api = paddle.trunc @@ -42,7 +41,6 @@ class TestTruncOp(OpTest): class TestFloatTruncOp(TestTruncOp): - def init_dtype_type(self): self.dtype = np.float32 self.__class__.exist_fp64_check_grad = True @@ -52,7 +50,6 @@ class TestFloatTruncOp(TestTruncOp): class TestIntTruncOp(TestTruncOp): - def init_dtype_type(self): self.dtype = np.int32 self.__class__.exist_fp64_check_grad = True @@ -62,7 +59,6 @@ class TestIntTruncOp(TestTruncOp): class TestTruncAPI(unittest.TestCase): - def setUp(self): self.shape = [20, 20] self.x = np.random.random((20, 20)).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_truncated_gaussian_random_op.py b/python/paddle/fluid/tests/unittests/test_truncated_gaussian_random_op.py index 780caa74333..70e79b4898a 100644 --- a/python/paddle/fluid/tests/unittests/test_truncated_gaussian_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_truncated_gaussian_random_op.py @@ -23,14 +23,13 @@ from paddle.fluid.framework import _test_eager_guard class TestTrunctedGaussianRandomOp(unittest.TestCase): - def setUp(self): self.op_type = "truncated_gaussian_random" self.inputs = {} self.attrs = { "shape": [10000], - "mean": .0, - "std": 1., + "mean": 0.0, + "std": 1.0, "seed": 10, } self.outputs = ["Out"] @@ -49,9 +48,9 @@ class TestTrunctedGaussianRandomOp(unittest.TestCase): program = fluid.Program() block = program.global_block() vout = block.create_var(name="Out") - op = block.append_op(type=self.op_type, - outputs={"Out": vout}, - attrs=self.attrs) + op = block.append_op( + type=self.op_type, outputs={"Out": vout}, attrs=self.attrs + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) @@ -63,7 +62,7 @@ class TestTrunctedGaussianRandomOp(unittest.TestCase): exe = Executor(place) outs = exe.run(program, fetch_list=fetch_list) tensor = outs[0] - self.assertAlmostEqual(numpy.mean(tensor), .0, delta=0.1) + self.assertAlmostEqual(numpy.mean(tensor), 0.0, delta=0.1) self.assertAlmostEqual(numpy.var(tensor), 0.773, delta=0.1) # TruncatedNormal.__call__ has no return value, so here call _C_ops api @@ -72,9 +71,14 @@ class TestTrunctedGaussianRandomOp(unittest.TestCase): with fluid.dygraph.guard(place): with _test_eager_guard(): out = paddle._C_ops.truncated_gaussian_random( - self.attrs["shape"], self.attrs["mean"], self.attrs["std"], - self.attrs["seed"], core.VarDesc.VarType.FP32, place) - self.assertAlmostEqual(numpy.mean(out.numpy()), .0, delta=0.1) + self.attrs["shape"], + self.attrs["mean"], + self.attrs["std"], + self.attrs["seed"], + core.VarDesc.VarType.FP32, + place, + ) + self.assertAlmostEqual(numpy.mean(out.numpy()), 0.0, delta=0.1) self.assertAlmostEqual(numpy.var(out.numpy()), 0.773, delta=0.1) diff --git a/python/paddle/fluid/tests/unittests/test_unbind_op.py b/python/paddle/fluid/tests/unittests/test_unbind_op.py index e030792ce60..6f719ae7d4d 100644 --- a/python/paddle/fluid/tests/unittests/test_unbind_op.py +++ b/python/paddle/fluid/tests/unittests/test_unbind_op.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import _test_eager_guard class TestUnbind(unittest.TestCase): - def test_unbind(self): x_1 = fluid.data(shape=[2, 3], dtype='float32', name='x_1') @@ -32,12 +31,11 @@ class TestUnbind(unittest.TestCase): axis = fluid.data(shape=[1], dtype='int32', name='axis') exe = fluid.Executor(place=fluid.CPUPlace()) - [res_1, res_2] = exe.run(fluid.default_main_program(), - feed={ - "x_1": input_1, - "axis": 0 - }, - fetch_list=[out_0, out_1]) + [res_1, res_2] = exe.run( + fluid.default_main_program(), + feed={"x_1": input_1, "axis": 0}, + fetch_list=[out_0, out_1], + ) assert np.array_equal(res_1, input_1[0, 0:100]) assert np.array_equal(res_2, input_1[1, 0:100]) @@ -63,7 +61,6 @@ class TestUnbind(unittest.TestCase): class TestLayersUnbind(unittest.TestCase): - def test_layers_unbind(self): x_1 = fluid.data(shape=[2, 3], dtype='float32', name='x_1') @@ -72,19 +69,17 @@ class TestLayersUnbind(unittest.TestCase): axis = fluid.data(shape=[1], dtype='int32', name='axis') exe = fluid.Executor(place=fluid.CPUPlace()) - [res_1, res_2] = exe.run(fluid.default_main_program(), - feed={ - "x_1": input_1, - "axis": 0 - }, - fetch_list=[out_0, out_1]) + [res_1, res_2] = exe.run( + fluid.default_main_program(), + feed={"x_1": input_1, "axis": 0}, + fetch_list=[out_0, out_1], + ) assert np.array_equal(res_1, input_1[0, 0:100]) assert np.array_equal(res_2, input_1[1, 0:100]) class TestUnbindOp(OpTest): - def initParameters(self): pass @@ -106,8 +101,9 @@ class TestUnbindOp(OpTest): self.inputs = {'X': x} self.attrs = {'axis': self.axis} self.setAxis() - self.outputs = {'Out': [('out%d' % i, self.out[i]) \ - for i in range(len(self.out))]} + self.outputs = { + 'Out': [('out%d' % i, self.out[i]) for i in range(len(self.out))] + } def get_dtype(self): return "float64" @@ -123,7 +119,6 @@ class TestUnbindOp(OpTest): class TestUnbindOp1(TestUnbindOp): - def initParameters(self): self.axis = 1 self.num = 2 @@ -137,7 +132,6 @@ class TestUnbindOp1(TestUnbindOp): class TestUnbindOp2(TestUnbindOp): - def initParameters(self): self.axis = 2 self.num = 2 @@ -151,7 +145,6 @@ class TestUnbindOp2(TestUnbindOp): class TestUnbindOp3(TestUnbindOp): - def initParameters(self): self.axis = 2 self.num = 2 @@ -168,7 +161,6 @@ class TestUnbindOp3(TestUnbindOp): class TestUnbindOp4(TestUnbindOp): - def initParameters(self): self.axis = 1 self.num = 2 @@ -185,7 +177,6 @@ class TestUnbindOp4(TestUnbindOp): class TestUnbindBF16Op(OpTest): - def setUp(self): self._set_op_type() self.python_api = paddle.unbind @@ -196,8 +187,12 @@ class TestUnbindBF16Op(OpTest): self.out = np.split(x, self.num, self.axis) self.inputs = {'X': convert_float_to_uint16(x)} self.attrs = {'axis': self.axis} - self.outputs = {'Out': [('out%d' % i, convert_float_to_uint16(self.out[i])) \ - for i in range(len(self.out))]} + self.outputs = { + 'Out': [ + ('out%d' % i, convert_float_to_uint16(self.out[i])) + for i in range(len(self.out)) + ] + } def get_dtype(self): return np.uint16 @@ -213,7 +208,6 @@ class TestUnbindBF16Op(OpTest): class TestUnbindAxisError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = fluid.data(shape=[2, 3], dtype='float32', name='x') diff --git a/python/paddle/fluid/tests/unittests/test_unfold_op.py b/python/paddle/fluid/tests/unittests/test_unfold_op.py index ea38ae2be1c..0bf5f3d2932 100644 --- a/python/paddle/fluid/tests/unittests/test_unfold_op.py +++ b/python/paddle/fluid/tests/unittests/test_unfold_op.py @@ -35,23 +35,45 @@ class TestUnfoldOp(OpTest): self.paddings = [1, 1, 1, 1] self.dilations = [1, 1] input_shape = [ - self.batch_size, self.input_channels, self.input_height, - self.input_width + self.batch_size, + self.input_channels, + self.input_height, + self.input_width, ] self.x = np.random.rand(*input_shape).astype(np.float64) def calc_unfold(self): output_shape = [0] * 3 output_shape[0] = self.batch_size - output_shape[1] = self.input_channels * self.kernel_sizes[ - 0] * self.kernel_sizes[1] + output_shape[1] = ( + self.input_channels * self.kernel_sizes[0] * self.kernel_sizes[1] + ) dkernel_h = self.dilations[0] * (self.kernel_sizes[0] - 1) + 1 dkernel_w = self.dilations[1] * (self.kernel_sizes[1] - 1) + 1 - out_height = int((self.input_height + self.paddings[0] + - self.paddings[2] - dkernel_h) / self.strides[0]) + 1 - out_width = int( - (self.input_width + self.paddings[1] + self.paddings[3] - dkernel_w) - / self.strides[1]) + 1 + out_height = ( + int( + ( + self.input_height + + self.paddings[0] + + self.paddings[2] + - dkernel_h + ) + / self.strides[0] + ) + + 1 + ) + out_width = ( + int( + ( + self.input_width + + self.paddings[1] + + self.paddings[3] + - dkernel_w + ) + / self.strides[1] + ) + + 1 + ) output_shape[2] = out_height * out_width output = np.zeros(output_shape).astype(np.float64) ############ calculate output ############## @@ -61,16 +83,25 @@ class TestUnfoldOp(OpTest): h_out = int(k / out_width) w_out = k % out_width w_offset = j % self.kernel_sizes[1] - h_offset = int( - j / self.kernel_sizes[1]) % self.kernel_sizes[0] - c_in = int(j / - (self.kernel_sizes[0] * self.kernel_sizes[1])) - h_in = h_offset * self.dilations[0] + h_out * self.strides[ - 0] - self.paddings[0] - w_in = w_offset * self.dilations[1] + w_out * self.strides[ - 1] - self.paddings[1] - if (h_in>=0 and h_in=0 and w_in= 0 and h_in < self.input_height) and ( + w_in >= 0 and w_in < self.input_width + ): output[i, j, k] = self.x[i, c_in, h_in, w_in] self.outputs = output @@ -84,7 +115,7 @@ class TestUnfoldOp(OpTest): 'kernel_sizes': self.kernel_sizes, 'paddings': self.paddings, 'dilations': self.dilations, - 'strides': self.strides + 'strides': self.strides, } self.outputs = {'Y': self.outputs} @@ -118,9 +149,9 @@ class TestUnfoldAPI(TestUnfoldOp): m = paddle.nn.Unfold(**self.attrs) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - self.outputs['Y'], - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), self.outputs['Y'], rtol=1e-05 + ) def test_info(self): str(paddle.nn.Unfold(**self.attrs)) diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py index 77d9903ceff..0977ec69ac3 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py @@ -19,11 +19,13 @@ import paddle import paddle.fluid.core as core from paddle.fluid.op import Operator import paddle.fluid as fluid -from paddle.fluid.tests.unittests.test_uniform_random_op import output_hist, output_hist_diag +from paddle.fluid.tests.unittests.test_uniform_random_op import ( + output_hist, + output_hist_diag, +) class TestUniformRandomOpBF16(OpTest): - def setUp(self): self.op_type = "uniform_random" self.dtype = "uint16" @@ -37,7 +39,7 @@ class TestUniformRandomOpBF16(OpTest): "min": -5.0, "max": 10.0, "seed": 10, - 'dtype': int(core.VarDesc.VarType.BF16) + 'dtype': int(core.VarDesc.VarType.BF16), } self.output_hist = output_hist @@ -58,15 +60,15 @@ class TestUniformRandomOpBF16(OpTest): class TestUniformRandomOpBF16AttrTensorList(TestUniformRandomOpBF16): - def setUp(self): self.op_type = "uniform_random" self.new_shape = (1000, 784) self.dtype = "uint16" shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype("int64") * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype("int64") * ele) + ) self.inputs = {'ShapeTensorList': shape_tensor} self.init_attrs() self.outputs = {"Out": np.zeros((1000, 784)).astype("uint16")} @@ -76,14 +78,14 @@ class TestUniformRandomOpBF16AttrTensorList(TestUniformRandomOpBF16): "min": -5.0, "max": 10.0, "seed": 10, - 'dtype': int(core.VarDesc.VarType.BF16) + 'dtype': int(core.VarDesc.VarType.BF16), } self.output_hist = output_hist class TestUniformRandomOpBF16AttrTensorInt32( - TestUniformRandomOpBF16AttrTensorList): - + TestUniformRandomOpBF16AttrTensorList +): def setUp(self): self.op_type = "uniform_random" self.dtype = "uint16" @@ -93,7 +95,6 @@ class TestUniformRandomOpBF16AttrTensorInt32( class TestUniformRandomOpBF16WithDiagInit(TestUniformRandomOpBF16): - def init_attrs(self): self.attrs = { "shape": [1000, 784], @@ -103,13 +104,12 @@ class TestUniformRandomOpBF16WithDiagInit(TestUniformRandomOpBF16): "diag_num": 784, "diag_step": 784, "diag_val": 1.0, - 'dtype': int(core.VarDesc.VarType.BF16) + 'dtype': int(core.VarDesc.VarType.BF16), } self.output_hist = output_hist_diag class TestUniformRandomOpBF16SelectedRows(unittest.TestCase): - def test_check_output(self): self.check_with_place(core.CPUPlace()) @@ -117,13 +117,15 @@ class TestUniformRandomOpBF16SelectedRows(unittest.TestCase): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10, - dtype=int(core.VarDesc.VarType.BF16)) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16), + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) result = convert_uint16_to_float(np.array(out.get_tensor())) @@ -132,22 +134,24 @@ class TestUniformRandomOpBF16SelectedRows(unittest.TestCase): class TestUniformRandomOpBF16SelectedRowsWithDiagInit( - TestUniformRandomOpBF16SelectedRows): - + TestUniformRandomOpBF16SelectedRows +): def check_with_place(self, place): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[500, 784], - min=-5.0, - max=10.0, - seed=10, - diag_num=500, - diag_step=784, - diag_val=1.0, - dtype=int(core.VarDesc.VarType.BF16)) + op = Operator( + "uniform_random", + Out="X", + shape=[500, 784], + min=-5.0, + max=10.0, + seed=10, + diag_num=500, + diag_step=784, + diag_val=1.0, + dtype=int(core.VarDesc.VarType.BF16), + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [500, 784]) result = convert_uint16_to_float(np.array(out.get_tensor())) @@ -156,14 +160,14 @@ class TestUniformRandomOpBF16SelectedRowsWithDiagInit( class TestUniformRandomOpBF16AttrTensorAPI(unittest.TestCase): - def test_attr_tensor_API(self): startup_program = fluid.Program() train_program = fluid.Program() with fluid.program_guard(train_program, startup_program): dim_tensor = fluid.layers.fill_constant([1], "int64", 3) - ret = fluid.layers.nn.uniform_random([1, dim_tensor, 2], - dtype=np.uint16) + ret = fluid.layers.nn.uniform_random( + [1, dim_tensor, 2], dtype=np.uint16 + ) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -173,7 +177,6 @@ class TestUniformRandomOpBF16AttrTensorAPI(unittest.TestCase): class TestUniformRandomOpAPISeed(unittest.TestCase): - def test_attr_tensor_API(self): _seed = 10 gen = paddle.seed(_seed) @@ -183,14 +186,12 @@ class TestUniformRandomOpAPISeed(unittest.TestCase): _min = 5 _max = 10 - ret = fluid.layers.nn.uniform_random([2, 3, 2], - min=_min, - max=_max, - seed=_seed) - ret_2 = fluid.layers.nn.uniform_random([2, 3, 2], - min=_min, - max=_max, - seed=_seed) + ret = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed + ) + ret_2 = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed + ) res = fluid.layers.equal(ret, ret_2) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -204,7 +205,6 @@ class TestUniformRandomOpAPISeed(unittest.TestCase): class TestUniformRandomOpBF16SelectedRowsShapeTensor(unittest.TestCase): - def test_check_output(self): place = core.CPUPlace() scope = core.Scope() @@ -212,13 +212,15 @@ class TestUniformRandomOpBF16SelectedRowsShapeTensor(unittest.TestCase): shape_tensor = scope.var("Shape").get_tensor() shape_tensor.set(np.array([1000, 784]).astype("int64"), place) paddle.seed(10) - op = Operator("uniform_random", - ShapeTensor="Shape", - Out="X", - min=-5.0, - max=10.0, - seed=10, - dtype=int(core.VarDesc.VarType.BF16)) + op = Operator( + "uniform_random", + ShapeTensor="Shape", + Out="X", + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16), + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) result = convert_uint16_to_float(np.array(out.get_tensor())) @@ -227,8 +229,8 @@ class TestUniformRandomOpBF16SelectedRowsShapeTensor(unittest.TestCase): class TestUniformRandomOpBF16SelectedRowsShapeTensorList( - TestUniformRandomOpBF16SelectedRowsShapeTensor): - + TestUniformRandomOpBF16SelectedRowsShapeTensor +): def test_check_output(self): place = core.CPUPlace() scope = core.Scope() @@ -238,13 +240,15 @@ class TestUniformRandomOpBF16SelectedRowsShapeTensorList( shape_2 = scope.var("shape2").get_tensor() shape_2.set(np.array([784]).astype("int64"), place) paddle.seed(10) - op = Operator("uniform_random", - ShapeTensorList=["shape1", "shape2"], - Out="X", - min=-5.0, - max=10.0, - seed=10, - dtype=int(core.VarDesc.VarType.BF16)) + op = Operator( + "uniform_random", + ShapeTensorList=["shape1", "shape2"], + Out="X", + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16), + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) result = convert_uint16_to_float(np.array(out.get_tensor())) @@ -253,14 +257,14 @@ class TestUniformRandomOpBF16SelectedRowsShapeTensorList( class TestUniformRandomBatchSizeLikeOpBF16API(unittest.TestCase): - def test_attr_tensorlist_int32_API(self): startup_program = fluid.Program() train_program = fluid.Program() with fluid.program_guard(train_program, startup_program): input = fluid.data(name="input", shape=[1, 3], dtype='uint16') out_1 = fluid.layers.uniform_random_batch_size_like( - input, [2, 4], dtype=np.uint16) # out_1.shape=[1, 4] + input, [2, 4], dtype=np.uint16 + ) # out_1.shape=[1, 4] place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -271,5 +275,6 @@ class TestUniformRandomBatchSizeLikeOpBF16API(unittest.TestCase): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py index 2b3881535c9..ea30b0da0ab 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py @@ -16,16 +16,17 @@ import unittest import paddle import paddle.fluid as fluid import numpy as np -from paddle.fluid.framework import _enable_legacy_dygraph, _disable_legacy_dygraph +from paddle.fluid.framework import ( + _enable_legacy_dygraph, + _disable_legacy_dygraph, +) class TestUniformRandomInplaceOpDtype(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) def test_uniform_random_inplace_op_dtype(self): - def test_fp32(): tensor_fp32 = paddle.ones(self.shape, dtype=paddle.float32) tensor_fp32.uniform_() @@ -46,7 +47,6 @@ class TestUniformRandomInplaceOpDtype(unittest.TestCase): class TestUniformRandomInplaceOpIsInplace(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) @@ -57,7 +57,6 @@ class TestUniformRandomInplaceOpIsInplace(unittest.TestCase): class TestUniformRandomInplaceOpSeedIsZero(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) self.seed = 0 @@ -72,7 +71,6 @@ class TestUniformRandomInplaceOpSeedIsZero(unittest.TestCase): class TestUniformRandomInplaceOpSeedIsNotZero(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) self.seed = 10 @@ -87,7 +85,6 @@ class TestUniformRandomInplaceOpSeedIsNotZero(unittest.TestCase): class TestUniformRandomInplaceOpWithinRange(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) self.min = -2 @@ -98,12 +95,12 @@ class TestUniformRandomInplaceOpWithinRange(unittest.TestCase): tensor = paddle.ones(self.shape) tensor.uniform_(min=self.min, max=self.max, seed=self.seed) tensor_data = tensor.numpy() - self.assertTrue((tensor_data > self.min).all() - and (tensor_data < self.max).all()) + self.assertTrue( + (tensor_data > self.min).all() and (tensor_data < self.max).all() + ) class TestUniformRandomInplaceOpShape(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) @@ -116,7 +113,6 @@ class TestUniformRandomInplaceOpShape(unittest.TestCase): class TestUniformRandomInplaceOpDistribution(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) self.min = -3 @@ -130,17 +126,15 @@ class TestUniformRandomInplaceOpDistribution(unittest.TestCase): hist, _ = np.histogram(tensor.numpy()[0], bins=self.bins) prob = hist / float(self.shape[0]) - prob_expect = np.ones((self.bins, )) / float(self.bins) + prob_expect = np.ones((self.bins,)) / float(self.bins) np.testing.assert_allclose(prob, prob_expect, rtol=0, atol=0.01) class TestUniformRandomInplaceOpError(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) def test_uniform_random_inplace_op_error(self): - def test_attr_error(): tensor = paddle.ones(self.shape) tensor.uniform_(shape=self.shape, min=-2, max=2) @@ -149,7 +143,6 @@ class TestUniformRandomInplaceOpError(unittest.TestCase): class TestUniformRandomInplaceOpEmptyTensor(unittest.TestCase): - def test_uniform_random_inplace_op_empty_tensor(self): places = ['cpu'] if fluid.core.is_compiled_with_cuda(): @@ -166,7 +159,6 @@ class TestUniformRandomInplaceOpEmptyTensor(unittest.TestCase): class TestUniformRandomInplaceGrad(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_op.py index 285e4f4f20b..efcbf075bf3 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_op.py @@ -49,15 +49,15 @@ def output_hist_diag(out): class TestUniformRandomOp_attr_tensorlist(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform self.new_shape = (1000, 784) shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype("int64") * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype("int64") * ele) + ) self.inputs = {'ShapeTensorList': shape_tensor} self.init_attrs() self.outputs = {"Out": np.zeros((1000, 784)).astype("float32")} @@ -75,22 +75,21 @@ class TestUniformRandomOp_attr_tensorlist(OpTest): class TestMaxMinAreInt(TestUniformRandomOp_attr_tensorlist): - def init_attrs(self): self.attrs = {"min": -5, "max": 10, "seed": 10} self.output_hist = output_hist class TestUniformRandomOp_attr_tensorlist_int32(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform self.new_shape = (1000, 784) shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype("int32") * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype("int32") * ele) + ) self.inputs = {'ShapeTensorList': shape_tensor} self.init_attrs() self.outputs = {"Out": np.zeros((1000, 784)).astype("float32")} @@ -108,7 +107,6 @@ class TestUniformRandomOp_attr_tensorlist_int32(OpTest): class TestUniformRandomOp_attr_tensor(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform @@ -129,7 +127,6 @@ class TestUniformRandomOp_attr_tensor(OpTest): class TestUniformRandomOp_attr_tensor_int32(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform @@ -150,7 +147,6 @@ class TestUniformRandomOp_attr_tensor_int32(OpTest): class TestUniformRandomOp(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform @@ -163,7 +159,7 @@ class TestUniformRandomOp(OpTest): "shape": [1000, 784], "min": -5.0, "max": 10.0, - "seed": 10 + "seed": 10, } self.output_hist = output_hist @@ -178,9 +174,13 @@ class TestUniformRandomOp(OpTest): places = self._get_places() for place in places: with fluid.dygraph.base.guard(place=place): - out = self.python_api(self.attrs['shape'], 'float32', - self.attrs['min'], self.attrs['max'], - self.attrs['seed']) + out = self.python_api( + self.attrs['shape'], + 'float32', + self.attrs['min'], + self.attrs['max'], + self.attrs['seed'], + ) def test_check_api_eager(self): with _test_eager_guard(): @@ -189,15 +189,15 @@ class TestUniformRandomOp(OpTest): class TestUniformRandomOpError(unittest.TestCase): - def test_errors(self): main_prog = Program() start_prog = Program() with program_guard(main_prog, start_prog): def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros((4, 784)), [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((4, 784)), [[1, 1, 1, 1]], fluid.CPUPlace() + ) fluid.layers.uniform_random(x1) self.assertRaises(TypeError, test_Variable) @@ -209,9 +209,9 @@ class TestUniformRandomOpError(unittest.TestCase): self.assertRaises(TypeError, test_Variable2) def test_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[4, 784], - dtype='float32') + x2 = fluid.layers.data( + name='x2', shape=[4, 784], dtype='float32' + ) fluid.layers.uniform_random(x2, 'int32') self.assertRaises(TypeError, test_dtype) @@ -224,7 +224,6 @@ class TestUniformRandomOpError(unittest.TestCase): class TestUniformRandomOpWithDiagInit(TestUniformRandomOp): - def init_attrs(self): self.attrs = { "shape": [1000, 784], @@ -233,13 +232,12 @@ class TestUniformRandomOpWithDiagInit(TestUniformRandomOp): "seed": 10, "diag_num": 784, "diag_step": 784, - "diag_val": 1.0 + "diag_val": 1.0, } self.output_hist = output_hist_diag class TestUniformRandomOpSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): @@ -254,12 +252,14 @@ class TestUniformRandomOpSelectedRows(unittest.TestCase): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) @@ -267,21 +267,23 @@ class TestUniformRandomOpSelectedRows(unittest.TestCase): class TestUniformRandomOpSelectedRowsWithDiagInit( - TestUniformRandomOpSelectedRows): - + TestUniformRandomOpSelectedRows +): def check_with_place(self, place): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[500, 784], - min=-5.0, - max=10.0, - seed=10, - diag_num=500, - diag_step=784, - diag_val=1.0) + op = Operator( + "uniform_random", + Out="X", + shape=[500, 784], + min=-5.0, + max=10.0, + seed=10, + diag_num=500, + diag_step=784, + diag_val=1.0, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [500, 784]) hist, prob = output_hist_diag(np.array(out.get_tensor())) @@ -289,29 +291,32 @@ class TestUniformRandomOpSelectedRowsWithDiagInit( class TestUniformRandomOpApi(unittest.TestCase): - def test_api(self): paddle.seed(10) x = fluid.layers.data('x', shape=[16], dtype='float32', lod_level=1) - y = fluid.layers.fc(x, - size=16, - param_attr=fluid.initializer.Uniform(low=-0.5, - high=0.5, - seed=10, - diag_num=16, - diag_step=16, - diag_val=1.0)) + y = fluid.layers.fc( + x, + size=16, + param_attr=fluid.initializer.Uniform( + low=-0.5, + high=0.5, + seed=10, + diag_num=16, + diag_step=16, + diag_val=1.0, + ), + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(3, 16).astype("float32"), [[1, 2]], place) + np.random.rand(3, 16).astype("float32"), [[1, 2]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) ret = exe.run(feed={'x': x_tensor}, fetch_list=[y], return_numpy=False) class TestUniformRandomOp_attr_tensor_API(unittest.TestCase): - def test_attr_tensor_API(self): startup_program = fluid.Program() train_program = fluid.Program() @@ -356,13 +361,12 @@ class TestUniformRandomOp_attr_tensor_API(unittest.TestCase): exe = fluid.Executor(place) Shape = np.array([2, 3]).astype('int32') exe.run(startup_program) - outs = exe.run(train_program, - feed={'shape_tensor': Shape}, - fetch_list=[ret]) + outs = exe.run( + train_program, feed={'shape_tensor': Shape}, fetch_list=[ret] + ) class TestUniformRandomOp_API_seed(unittest.TestCase): - def test_attr_tensor_API(self): _seed = 10 gen = paddle.seed(_seed) @@ -372,14 +376,12 @@ class TestUniformRandomOp_API_seed(unittest.TestCase): _min = 5 _max = 10 - ret = fluid.layers.nn.uniform_random([2, 3, 2], - min=_min, - max=_max, - seed=_seed) - ret_2 = fluid.layers.nn.uniform_random([2, 3, 2], - min=_min, - max=_max, - seed=_seed) + ret = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed + ) + ret_2 = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed + ) res = fluid.layers.equal(ret, ret_2) place = fluid.CPUPlace() if fluid.core.is_compiled_with_cuda(): @@ -395,7 +397,6 @@ class TestUniformRandomOp_API_seed(unittest.TestCase): class TestUniformRandomOpSelectedRowsShapeTensor(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): @@ -412,12 +413,14 @@ class TestUniformRandomOpSelectedRowsShapeTensor(unittest.TestCase): shape_tensor = scope.var("Shape").get_tensor() shape_tensor.set(np.array([1000, 784]).astype("int64"), place) paddle.seed(10) - op = Operator("uniform_random", - ShapeTensor="Shape", - Out="X", - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + ShapeTensor="Shape", + Out="X", + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) @@ -425,7 +428,6 @@ class TestUniformRandomOpSelectedRowsShapeTensor(unittest.TestCase): class TestUniformRandomOpSelectedRowsShapeTensorList(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): @@ -444,12 +446,14 @@ class TestUniformRandomOpSelectedRowsShapeTensorList(unittest.TestCase): shape_2 = scope.var("shape2").get_tensor() shape_2.set(np.array([784]).astype("int64"), place) paddle.seed(10) - op = Operator("uniform_random", - ShapeTensorList=["shape1", "shape2"], - Out="X", - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + ShapeTensorList=["shape1", "shape2"], + Out="X", + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) @@ -457,51 +461,48 @@ class TestUniformRandomOpSelectedRowsShapeTensorList(unittest.TestCase): class TestUniformRandomDygraphMode(unittest.TestCase): - def test_check_output(self): with fluid.dygraph.guard(): - x = fluid.layers.uniform_random([10], - dtype="float32", - min=0.0, - max=1.0) + x = fluid.layers.uniform_random( + [10], dtype="float32", min=0.0, max=1.0 + ) x_np = x.numpy() for i in range(10): self.assertTrue((x_np[i] > 0 and x_np[i] < 1.0)) class TestUniformRandomBatchSizeLikeOpError(unittest.TestCase): - def test_errors(self): main_prog = Program() start_prog = Program() with program_guard(main_prog, start_prog): def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros( - (100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace() + ) fluid.layers.uniform_random_batch_size_like(x1) self.assertRaises(TypeError, test_Variable) def test_shape(): - x1 = fluid.layers.data(name='x2', - shape=[100, 784], - dtype='float32') + x1 = fluid.layers.data( + name='x2', shape=[100, 784], dtype='float32' + ) fluid.layers.uniform_random_batch_size_like(x1, shape="shape") self.assertRaises(TypeError, test_shape) def test_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[100, 784], - dtype='float32') + x2 = fluid.layers.data( + name='x2', shape=[100, 784], dtype='float32' + ) fluid.layers.uniform_random_batch_size_like(x2, 'int32') self.assertRaises(TypeError, test_dtype) class TestUniformAlias(unittest.TestCase): - def test_alias(self): paddle.uniform([2, 3], min=-5.0, max=5.0) paddle.tensor.uniform([2, 3], min=-5.0, max=5.0) @@ -514,15 +515,15 @@ class TestUniformAlias(unittest.TestCase): class TestUniformOpError(unittest.TestCase): - def test_errors(self): main_prog = Program() start_prog = Program() with program_guard(main_prog, start_prog): def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros( - (100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace() + ) paddle.tensor.random.uniform(x1) self.assertRaises(TypeError, test_Variable) @@ -534,36 +535,34 @@ class TestUniformOpError(unittest.TestCase): self.assertRaises(TypeError, test_Variable2) def test_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[100, 784], - dtype='float32') + x2 = fluid.layers.data( + name='x2', shape=[100, 784], dtype='float32' + ) paddle.tensor.random.uniform(x2, 'int32') self.assertRaises(TypeError, test_dtype) def test_out_dtype(): - out = paddle.tensor.random.uniform(shape=[3, 4], - dtype='float64') + out = paddle.tensor.random.uniform( + shape=[3, 4], dtype='float64' + ) self.assertEqual(out.dtype, fluid.core.VarDesc.VarType.FP64) test_out_dtype() class TestUniformDygraphMode(unittest.TestCase): - def test_check_output(self): with fluid.dygraph.guard(): - x = paddle.tensor.random.uniform([10], - dtype="float32", - min=0.0, - max=1.0) + x = paddle.tensor.random.uniform( + [10], dtype="float32", min=0.0, max=1.0 + ) x_np = x.numpy() for i in range(10): self.assertTrue((x_np[i] > 0 and x_np[i] < 1.0)) class TestUniformDtype(unittest.TestCase): - def test_default_dtype(self): paddle.disable_static() @@ -599,7 +598,6 @@ class TestUniformDtype(unittest.TestCase): class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -618,37 +616,64 @@ class TestRandomValue(unittest.TestCase): expect_mean = 0.50000454338820143895816272561205551028251647949218750 expect_std = 0.28867379167297479991560749112977646291255950927734375 expect = [ - 0.55298901, 0.65184678, 0.49375412, 0.57943639, 0.16459608, - 0.67181056, 0.03021481, 0.0238559, 0.07742096, 0.55972187 + 0.55298901, + 0.65184678, + 0.49375412, + 0.57943639, + 0.16459608, + 0.67181056, + 0.03021481, + 0.0238559, + 0.07742096, + 0.55972187, ] out = paddle.rand([32, 3, 1024, 1024], dtype='float64').numpy() self.assertEqual(np.mean(out), expect_mean) self.assertEqual(np.std(out), expect_std) - np.testing.assert_allclose(out[2, 1, 512, 1000:1010], - expect, - rtol=1e-05) + np.testing.assert_allclose( + out[2, 1, 512, 1000:1010], expect, rtol=1e-05 + ) expect_mean = 0.50002604722976684570312500 expect_std = 0.2886914908885955810546875 expect = [ - 0.45320973, 0.17582087, 0.725341, 0.30849215, 0.622257, 0.46352342, - 0.97228295, 0.12771158, 0.286525, 0.9810645 + 0.45320973, + 0.17582087, + 0.725341, + 0.30849215, + 0.622257, + 0.46352342, + 0.97228295, + 0.12771158, + 0.286525, + 0.9810645, ] out = paddle.rand([32, 3, 1024, 1024], dtype='float32').numpy() self.assertEqual(np.mean(out), expect_mean) self.assertEqual(np.std(out), expect_std) - np.testing.assert_allclose(out[2, 1, 512, 1000:1010], - expect, - rtol=1e-05) + np.testing.assert_allclose( + out[2, 1, 512, 1000:1010], expect, rtol=1e-05 + ) expect_mean = 25.11843109130859375 expect_std = 43.370647430419921875 expect = [ - 30.089634, 77.05225, 3.1201615, 68.34072, 59.266724, -25.33281, - 12.973292, 27.41127, -17.412298, 27.931019 + 30.089634, + 77.05225, + 3.1201615, + 68.34072, + 59.266724, + -25.33281, + 12.973292, + 27.41127, + -17.412298, + 27.931019, ] - out = paddle.empty([16, 16, 16, 16], - dtype='float32').uniform_(-50, 100).numpy() + out = ( + paddle.empty([16, 16, 16, 16], dtype='float32') + .uniform_(-50, 100) + .numpy() + ) self.assertEqual(np.mean(out), expect_mean) self.assertEqual(np.std(out), expect_std) np.testing.assert_allclose(out[10, 10, 10, 0:10], expect, rtol=1e-05) @@ -657,7 +682,6 @@ class TestRandomValue(unittest.TestCase): class TestUniformMinMaxTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -673,9 +697,9 @@ class TestUniformMinMaxTensor(UnittestBase): min_v = paddle.to_tensor([0.1]) max_v = paddle.to_tensor([0.9]) y = paddle.uniform([2, 3, 10], min=min_v, max=max_v) - z = paddle.fluid.layers.uniform_random([2, 3, 10], - min=min_v, - max=max_v) + z = paddle.fluid.layers.uniform_random( + [2, 3, 10], min=min_v, max=max_v + ) out = feat + y + z diff --git a/python/paddle/fluid/tests/unittests/test_unique.py b/python/paddle/fluid/tests/unittests/test_unique.py index a62f5e2808d..6e739bb3859 100644 --- a/python/paddle/fluid/tests/unittests/test_unique.py +++ b/python/paddle/fluid/tests/unittests/test_unique.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class TestUniqueOp(OpTest): - def setUp(self): self.op_type = "unique" self.init_config() @@ -37,12 +36,11 @@ class TestUniqueOp(OpTest): self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)} self.outputs = { 'Out': np.array([2, 3, 1, 5], dtype='int64'), - 'Index': np.array([0, 1, 1, 2, 3, 1], dtype='int32') + 'Index': np.array([0, 1, 1, 2, 3, 1], dtype='int32'), } class TestOne(TestUniqueOp): - def init_config(self): self.inputs = { 'X': np.array([2], dtype='int64'), @@ -50,31 +48,29 @@ class TestOne(TestUniqueOp): self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)} self.outputs = { 'Out': np.array([2], dtype='int64'), - 'Index': np.array([0], dtype='int32') + 'Index': np.array([0], dtype='int32'), } class TestRandom(TestUniqueOp): - def init_config(self): - self.inputs = {'X': np.random.randint(0, 100, (150, ), dtype='int64')} + self.inputs = {'X': np.random.randint(0, 100, (150,), dtype='int64')} self.attrs = {'dtype': int(core.VarDesc.VarType.INT64)} - np_unique, np_index, reverse_index = np.unique(self.inputs['X'], True, - True) + np_unique, np_index, reverse_index = np.unique( + self.inputs['X'], True, True + ) np_tuple = [(np_unique[i], np_index[i]) for i in range(len(np_unique))] np_tuple.sort(key=lambda x: x[1]) target_out = np.array([i[0] for i in np_tuple], dtype='int64') target_index = np.array( - [list(target_out).index(i) for i in self.inputs['X']], - dtype='int64') + [list(target_out).index(i) for i in self.inputs['X']], dtype='int64' + ) self.outputs = {'Out': target_out, 'Index': target_index} class TestUniqueRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.unique([10]) @@ -87,10 +83,10 @@ class TestUniqueRaiseError(unittest.TestCase): self.assertRaises(TypeError, test_dtype) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestOneGPU(TestUniqueOp): - def init_config(self): self.inputs = { 'X': np.array([2], dtype='int64'), @@ -98,7 +94,7 @@ class TestOneGPU(TestUniqueOp): self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)} self.outputs = { 'Out': np.array([2], dtype='int64'), - 'Index': np.array([0], dtype='int32') + 'Index': np.array([0], dtype='int32'), } def test_check_output(self): @@ -107,21 +103,22 @@ class TestOneGPU(TestUniqueOp): self.check_output_with_place(place, atol=1e-5) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestRandomGPU(TestUniqueOp): - def init_config(self): - self.inputs = {'X': np.random.randint(0, 100, (150, ), dtype='int64')} + self.inputs = {'X': np.random.randint(0, 100, (150,), dtype='int64')} self.attrs = {'dtype': int(core.VarDesc.VarType.INT64)} - np_unique, np_index, reverse_index = np.unique(self.inputs['X'], True, - True) + np_unique, np_index, reverse_index = np.unique( + self.inputs['X'], True, True + ) np_tuple = [(np_unique[i], np_index[i]) for i in range(len(np_unique))] np_tuple.sort(key=lambda x: x[1]) target_out = np.array([i[0] for i in np_tuple], dtype='int64') target_index = np.array( - [list(target_out).index(i) for i in self.inputs['X']], - dtype='int64') + [list(target_out).index(i) for i in self.inputs['X']], dtype='int64' + ) self.outputs = {'Out': target_out, 'Index': target_index} @@ -132,21 +129,22 @@ class TestRandomGPU(TestUniqueOp): class TestSortedUniqueOp(TestUniqueOp): - def init_config(self): self.inputs = {'X': np.array([2, 3, 3, 1, 5, 3], dtype='int64')} - unique, indices, inverse, count = np.unique(self.inputs['X'], - return_index=True, - return_inverse=True, - return_counts=True, - axis=None) + unique, indices, inverse, count = np.unique( + self.inputs['X'], + return_index=True, + return_inverse=True, + return_counts=True, + axis=None, + ) self.attrs = { 'dtype': int(core.VarDesc.VarType.INT32), "return_index": True, "return_inverse": True, "return_counts": True, "axis": None, - "is_sorted": True + "is_sorted": True, } self.outputs = { 'Out': unique, @@ -157,21 +155,22 @@ class TestSortedUniqueOp(TestUniqueOp): class TestUniqueOpAxisNone(TestUniqueOp): - def init_config(self): self.inputs = {'X': np.random.random((4, 7, 10)).astype('float64')} - unique, indices, inverse, counts = np.unique(self.inputs['X'], - return_index=True, - return_inverse=True, - return_counts=True, - axis=None) + unique, indices, inverse, counts = np.unique( + self.inputs['X'], + return_index=True, + return_inverse=True, + return_counts=True, + axis=None, + ) self.attrs = { 'dtype': int(core.VarDesc.VarType.INT32), "return_index": True, "return_inverse": True, "return_counts": True, "axis": None, - "is_sorted": True + "is_sorted": True, } self.outputs = { 'Out': unique, @@ -182,21 +181,22 @@ class TestUniqueOpAxisNone(TestUniqueOp): class TestUniqueOpAxis1(TestUniqueOp): - def init_config(self): self.inputs = {'X': np.random.random((3, 8, 8)).astype('float64')} - unique, indices, inverse, counts = np.unique(self.inputs['X'], - return_index=True, - return_inverse=True, - return_counts=True, - axis=1) + unique, indices, inverse, counts = np.unique( + self.inputs['X'], + return_index=True, + return_inverse=True, + return_counts=True, + axis=1, + ) self.attrs = { 'dtype': int(core.VarDesc.VarType.INT32), "return_index": True, "return_inverse": True, "return_counts": True, "axis": [1], - "is_sorted": True + "is_sorted": True, } self.outputs = { 'Out': unique, @@ -207,7 +207,6 @@ class TestUniqueOpAxis1(TestUniqueOp): class TestUniqueAPI(unittest.TestCase): - def test_dygraph_api_out(self): paddle.disable_static() x_data = x_data = np.random.randint(0, 10, (120)) @@ -221,16 +220,20 @@ class TestUniqueAPI(unittest.TestCase): paddle.disable_static() x_data = np.random.random((3, 5, 5)).astype("float32") x = paddle.to_tensor(x_data) - out, index, inverse, counts = paddle.unique(x, - return_index=True, - return_inverse=True, - return_counts=True, - axis=0) - np_out, np_index, np_inverse, np_counts = np.unique(x_data, - return_index=True, - return_inverse=True, - return_counts=True, - axis=0) + out, index, inverse, counts = paddle.unique( + x, + return_index=True, + return_inverse=True, + return_counts=True, + axis=0, + ) + np_out, np_index, np_inverse, np_counts = np.unique( + x_data, + return_index=True, + return_inverse=True, + return_counts=True, + axis=0, + ) self.assertTrue((out.numpy() == np_out).all(), True) self.assertTrue((index.numpy() == np_index).all(), True) self.assertTrue((inverse.numpy() == np_inverse).all(), True) @@ -241,13 +244,16 @@ class TestUniqueAPI(unittest.TestCase): paddle.disable_static() x_data = x_data = np.random.randint(0, 10, (120)) x = paddle.to_tensor(x_data) - out, indices, inverse, counts = paddle.unique(x, - return_index=True, - return_inverse=True, - return_counts=True, - dtype="int32") + out, indices, inverse, counts = paddle.unique( + x, + return_index=True, + return_inverse=True, + return_counts=True, + dtype="int32", + ) expected_out, np_indices, np_inverse, np_counts = np.unique( - x_data, return_index=True, return_inverse=True, return_counts=True) + x_data, return_index=True, return_inverse=True, return_counts=True + ) self.assertTrue((out.numpy() == expected_out).all(), True) self.assertTrue((indices.numpy() == np_indices).all(), True) self.assertTrue((inverse.numpy() == np_inverse).all(), True) @@ -261,34 +267,33 @@ class TestUniqueAPI(unittest.TestCase): self.test_dygraph_attr_dtype() def test_static_graph(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[3, 2], dtype='float64') - unique, inverse, counts = paddle.unique(x, - return_inverse=True, - return_counts=True, - axis=0) + unique, inverse, counts = paddle.unique( + x, return_inverse=True, return_counts=True, axis=0 + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) x_np = np.array([[1, 2], [3, 4], [1, 2]]).astype('float64') - result = exe.run(feed={"x": x_np}, - fetch_list=[unique, inverse, counts]) - np_unique, np_inverse, np_counts = np.unique(x_np, - return_inverse=True, - return_counts=True, - axis=0) + result = exe.run( + feed={"x": x_np}, fetch_list=[unique, inverse, counts] + ) + np_unique, np_inverse, np_counts = np.unique( + x_np, return_inverse=True, return_counts=True, axis=0 + ) np.testing.assert_allclose(result[0], np_unique, rtol=1e-05) np.testing.assert_allclose(result[1], np_inverse, rtol=1e-05) np.testing.assert_allclose(result[2], np_counts, rtol=1e-05) class TestUniqueError(unittest.TestCase): - def test_input_dtype(self): - def test_x_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float16') result = paddle.unique(x) diff --git a/python/paddle/fluid/tests/unittests/test_unique_consecutive_op.py b/python/paddle/fluid/tests/unittests/test_unique_consecutive_op.py index 91a8c6d6a4b..b8a3096dbd6 100644 --- a/python/paddle/fluid/tests/unittests/test_unique_consecutive_op.py +++ b/python/paddle/fluid/tests/unittests/test_unique_consecutive_op.py @@ -50,7 +50,7 @@ def reference_unique_consecutive(X, return_inverse=False, return_counts=False): last += 1 inverse_vec[last] = i if return_counts: - counts_vec = counts_vec[:len(X)] + counts_vec = counts_vec[: len(X)] if return_inverse and return_counts: return X, np.array(inverse_vec), np.array(counts_vec) elif return_counts: @@ -79,8 +79,9 @@ class TestUniqueConsecutiveOp(OpTest): self.config() self.op_type = "unique_consecutive" x = np.random.randint(self.x_range, size=self.x_size).astype(self.dtype) - result = reference_unique_consecutive(x, self.return_inverse, - self.return_counts) + result = reference_unique_consecutive( + x, self.return_inverse, self.return_counts + ) out = reference_unique_consecutive(x) out = np.array(out).astype(self.dtype) self.inputs = { @@ -111,8 +112,9 @@ class TestUniqueConsecutiveOp2(TestUniqueConsecutiveOp): self.config() self.op_type = "unique_consecutive" x = np.random.randint(self.x_range, size=self.x_size).astype(self.dtype) - result, inverse = reference_unique_consecutive(x, self.return_inverse, - self.return_counts) + result, inverse = reference_unique_consecutive( + x, self.return_inverse, self.return_counts + ) result = np.array(result).astype(self.dtype) inverse = inverse.astype(self.dtype) self.inputs = { @@ -120,7 +122,7 @@ class TestUniqueConsecutiveOp2(TestUniqueConsecutiveOp): } self.attrs = { 'return_inverse': self.return_inverse, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.python_out_sig = ["Out"] self.outputs = {'Out': result, 'Index': inverse} @@ -141,8 +143,9 @@ class TestUniqueConsecutiveOp3(TestUniqueConsecutiveOp): self.config() self.op_type = "unique_consecutive" x = np.random.randint(self.x_range, size=self.x_size).astype(self.dtype) - result, counts = reference_unique_consecutive(x, self.return_inverse, - self.return_counts) + result, counts = reference_unique_consecutive( + x, self.return_inverse, self.return_counts + ) result = np.array(result).astype(self.dtype) counts = counts.astype(self.dtype) self.inputs = { @@ -150,7 +153,7 @@ class TestUniqueConsecutiveOp3(TestUniqueConsecutiveOp): } self.attrs = { 'return_counts': self.return_counts, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.python_out_sig = ["Out"] self.outputs = {'Out': result, 'Counts': counts} @@ -172,7 +175,8 @@ class TestUniqueConsecutiveOp4(TestUniqueConsecutiveOp): self.op_type = "unique_consecutive" x = np.random.randint(self.x_range, size=self.x_size).astype(self.dtype) result, inverse, counts = reference_unique_consecutive( - x, self.return_inverse, self.return_counts) + x, self.return_inverse, self.return_counts + ) result = np.array(result).astype(self.dtype) inverse = inverse.astype(self.dtype) counts = counts.astype(self.dtype) @@ -182,14 +186,13 @@ class TestUniqueConsecutiveOp4(TestUniqueConsecutiveOp): self.attrs = { 'return_inverse': self.return_inverse, 'return_counts': self.return_counts, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.python_out_sig = ["Out"] self.outputs = {'Out': result, 'Index': inverse, 'Counts': counts} class TestUniqueConsecutiveAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -198,15 +201,21 @@ class TestUniqueConsecutiveAPI(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): paddle.enable_static() - input_x = fluid.data(name="input_x", shape=[ - 100, - ], dtype="float32") + input_x = fluid.data( + name="input_x", + shape=[ + 100, + ], + dtype="float32", + ) result = paddle.unique_consecutive(input_x) x_np = np.random.randint(20, size=100).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input_x": x_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: @@ -221,7 +230,6 @@ class TestUniqueConsecutiveAPI(unittest.TestCase): class TestUniqueConsecutiveCase2API(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -230,16 +238,23 @@ class TestUniqueConsecutiveCase2API(unittest.TestCase): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): paddle.enable_static() - input_x = fluid.data(name="input_x", shape=[ - 100, - ], dtype="float32") + input_x = fluid.data( + name="input_x", + shape=[ + 100, + ], + dtype="float32", + ) result, inverse, counts = paddle.unique_consecutive( - input_x, return_inverse=True, return_counts=True) + input_x, return_inverse=True, return_counts=True + ) x_np = np.random.randint(20, size=100).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input_x": x_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: @@ -251,7 +266,8 @@ class TestUniqueConsecutiveCase2API(unittest.TestCase): input_x = np.random.randint(20, size=100).astype("float64") x = paddle.to_tensor(input_x) result, inverse, counts = paddle.unique_consecutive( - x, return_inverse=True, return_counts=True) + x, return_inverse=True, return_counts=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_unique_name.py b/python/paddle/fluid/tests/unittests/test_unique_name.py index 4393089bab8..a423247370d 100644 --- a/python/paddle/fluid/tests/unittests/test_unique_name.py +++ b/python/paddle/fluid/tests/unittests/test_unique_name.py @@ -17,7 +17,6 @@ import paddle.fluid as fluid class TestUniqueName(unittest.TestCase): - def test_guard(self): with fluid.unique_name.guard(): name_1 = fluid.unique_name.generate('') @@ -45,7 +44,6 @@ class TestUniqueName(unittest.TestCase): class TestImperativeUniqueName(unittest.TestCase): - def test_name_generator(self): with fluid.dygraph.guard(): tracer = fluid.framework._dygraph_tracer() diff --git a/python/paddle/fluid/tests/unittests/test_unique_with_counts.py b/python/paddle/fluid/tests/unittests/test_unique_with_counts.py index 30fc203d933..6a5b58454b5 100644 --- a/python/paddle/fluid/tests/unittests/test_unique_with_counts.py +++ b/python/paddle/fluid/tests/unittests/test_unique_with_counts.py @@ -20,7 +20,6 @@ import paddle.fluid.core as core class TestUniqueWithCountsOp(OpTest): - def setUp(self): self.op_type = "unique_with_counts" self.init_config() @@ -36,12 +35,11 @@ class TestUniqueWithCountsOp(OpTest): self.outputs = { 'Out': np.array([2, 3, 1, 5], dtype='int64'), 'Index': np.array([0, 1, 1, 2, 3, 1], dtype='int32'), - 'Count': np.array([1, 3, 1, 1], dtype='int32') + 'Count': np.array([1, 3, 1, 1], dtype='int32'), } class TestOne(TestUniqueWithCountsOp): - def init_config(self): self.inputs = { 'X': np.array([2], dtype='int64'), @@ -50,24 +48,24 @@ class TestOne(TestUniqueWithCountsOp): self.outputs = { 'Out': np.array([2], dtype='int64'), 'Index': np.array([0], dtype='int32'), - 'Count': np.array([1], dtype='int32') + 'Count': np.array([1], dtype='int32'), } class TestRandom(TestUniqueWithCountsOp): - def init_config(self): - input_data = np.random.randint(0, 100, (2000, ), dtype='int64') + input_data = np.random.randint(0, 100, (2000,), dtype='int64') self.inputs = {'X': input_data} self.attrs = {'dtype': int(core.VarDesc.VarType.INT64)} - np_unique, np_index, reverse_index = np.unique(self.inputs['X'], True, - True) + np_unique, np_index, reverse_index = np.unique( + self.inputs['X'], True, True + ) np_tuple = [(np_unique[i], np_index[i]) for i in range(len(np_unique))] np_tuple.sort(key=lambda x: x[1]) target_out = np.array([i[0] for i in np_tuple], dtype='int64') target_index = np.array( - [list(target_out).index(i) for i in self.inputs['X']], - dtype='int64') + [list(target_out).index(i) for i in self.inputs['X']], dtype='int64' + ) count = [0 for i in range(len(np_unique))] for i in range(target_index.shape[0]): count[target_index[i]] += 1 @@ -75,14 +73,12 @@ class TestRandom(TestUniqueWithCountsOp): self.outputs = { 'Out': target_out, 'Index': target_index, - 'Count': target_count + 'Count': target_count, } class TestUniqueWithCountsRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.unique_with_counts([10]) @@ -95,10 +91,10 @@ class TestUniqueWithCountsRaiseError(unittest.TestCase): self.assertRaises(TypeError, test_dtype) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestOneGPU(TestUniqueWithCountsOp): - def init_config(self): self.inputs = { 'X': np.array([2], dtype='int64'), @@ -107,7 +103,7 @@ class TestOneGPU(TestUniqueWithCountsOp): self.outputs = { 'Out': np.array([2], dtype='int64'), 'Index': np.array([0], dtype='int32'), - 'Count': np.array([1], dtype='int32') + 'Count': np.array([1], dtype='int32'), } def test_check_output(self): @@ -116,22 +112,23 @@ class TestOneGPU(TestUniqueWithCountsOp): self.check_output_with_place(place, atol=1e-5) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestRandomGPU(TestUniqueWithCountsOp): - def init_config(self): - input_data = np.random.randint(0, 100, (2000, ), dtype='int64') + input_data = np.random.randint(0, 100, (2000,), dtype='int64') self.inputs = {'X': input_data} self.attrs = {'dtype': int(core.VarDesc.VarType.INT64)} - np_unique, np_index, reverse_index = np.unique(self.inputs['X'], True, - True) + np_unique, np_index, reverse_index = np.unique( + self.inputs['X'], True, True + ) np_tuple = [(np_unique[i], np_index[i]) for i in range(len(np_unique))] np_tuple.sort(key=lambda x: x[1]) target_out = np.array([i[0] for i in np_tuple], dtype='int64') target_index = np.array( - [list(target_out).index(i) for i in self.inputs['X']], - dtype='int64') + [list(target_out).index(i) for i in self.inputs['X']], dtype='int64' + ) count = [0 for i in range(len(np_unique))] for i in range(target_index.shape[0]): count[target_index[i]] += 1 @@ -139,7 +136,7 @@ class TestRandomGPU(TestUniqueWithCountsOp): self.outputs = { 'Out': target_out, 'Index': target_index, - 'Count': target_count + 'Count': target_count, } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_unpool1d_op.py b/python/paddle/fluid/tests/unittests/test_unpool1d_op.py index f15d295ce9e..787d9367f58 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool1d_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool1d_op.py @@ -25,8 +25,11 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): input_size = x.shape default_size = [] for d in range(len(kernel_size)): - default_size.append((input_size[-len(kernel_size) + d] - 1) * - stride[d] + kernel_size[d] - 2 * padding[d]) + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) if output_size is None: ret = default_size else: @@ -34,11 +37,13 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): return ret -def unpool1dmax_forward_naive(input, indices, ksize, strides, paddings, - output_size): +def unpool1dmax_forward_naive( + input, indices, ksize, strides, paddings, output_size +): s0, s1, s2 = input.shape - output_size = _unpool_output_size(input, ksize, strides, paddings, - output_size) + output_size = _unpool_output_size( + input, ksize, strides, paddings, output_size + ) out_lsize = output_size[0] out = np.zeros((s0, s1, out_lsize)) for nidx in range(s0): @@ -52,7 +57,6 @@ def unpool1dmax_forward_naive(input, indices, ksize, strides, paddings, class TestUnpool1DOpAPI_dygraph(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -61,25 +65,23 @@ class TestUnpool1DOpAPI_dygraph(unittest.TestCase): paddle.disable_static() input_data = np.random.rand(1, 3, 16) input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool1d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool1d(output, - indices, - kernel_size=2, - stride=2) + output, indices = F.max_pool1d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool1d( + output, indices, kernel_size=2, stride=2 + ) expected_output_unpool = unpool1dmax_forward_naive( - output.numpy(), indices.numpy(), [2], [2], [0], [16]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), indices.numpy(), [2], [2], [0], [16] + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool1DOpAPI_dygraph2(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -88,25 +90,23 @@ class TestUnpool1DOpAPI_dygraph2(unittest.TestCase): paddle.disable_static() input_data = np.random.rand(1, 3, 16) input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool1d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool1d(output, - indices, - kernel_size=2, - stride=None) + output, indices = F.max_pool1d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool1d( + output, indices, kernel_size=2, stride=None + ) expected_output_unpool = unpool1dmax_forward_naive( - output.numpy(), indices.numpy(), [2], [2], [0], [16]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), indices.numpy(), [2], [2], [0], [16] + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool1DOpAPI_dygraph3(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -115,61 +115,66 @@ class TestUnpool1DOpAPI_dygraph3(unittest.TestCase): paddle.disable_static() input_data = np.random.rand(1, 3, 16) input_x = paddle.to_tensor(input_data) - Pool1d = paddle.nn.MaxPool1D(kernel_size=2, - stride=2, - return_mask=True) + Pool1d = paddle.nn.MaxPool1D( + kernel_size=2, stride=2, return_mask=True + ) UnPool1d = paddle.nn.MaxUnPool1D(kernel_size=2, stride=2) output, indices = Pool1d(input_x) output_unpool = UnPool1d(output, indices) expected_output_unpool = unpool1dmax_forward_naive( - output.numpy(), indices.numpy(), [2], [2], [0], [16]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), indices.numpy(), [2], [2], [0], [16] + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool1DOpAPI_static(unittest.TestCase): - def test_case(self): paddle.enable_static() places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): places.append(paddle.CUDAPlace(0)) for place in places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - - input_data = np.array([[[1, 2, 3, 4], [5, 6, 7, 8], - [9, 10, 11, 12]]]).astype("float32") - x = paddle.fluid.data(name='x', - shape=[1, 3, 4], - dtype='float32') - output, indices = F.max_pool1d(x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool1d(output, - indices, - kernel_size=2, - stride=None) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + + input_data = np.array( + [[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]] + ).astype("float32") + x = paddle.fluid.data( + name='x', shape=[1, 3, 4], dtype='float32' + ) + output, indices = F.max_pool1d( + x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool1d( + output, indices, kernel_size=2, stride=None + ) exe = paddle.fluid.Executor(place) - fetches = exe.run(paddle.fluid.default_main_program(), - feed={"x": input_data}, - fetch_list=[output_unpool], - return_numpy=True) - pool1d_out_np = np.array([[[2., 4.], [6., 8.], - [10., 12.]]]).astype("float32") - indices_np = np.array([[[1, 3], [1, 3], [1, - 3]]]).astype("int32") + fetches = exe.run( + paddle.fluid.default_main_program(), + feed={"x": input_data}, + fetch_list=[output_unpool], + return_numpy=True, + ) + pool1d_out_np = np.array( + [[[2.0, 4.0], [6.0, 8.0], [10.0, 12.0]]] + ).astype("float32") + indices_np = np.array([[[1, 3], [1, 3], [1, 3]]]).astype( + "int32" + ) expected_output_unpool = unpool1dmax_forward_naive( - pool1d_out_np, indices_np, [2], [2], [0], [4]) - np.testing.assert_allclose(fetches[0], - expected_output_unpool, - rtol=1e-05) + pool1d_out_np, indices_np, [2], [2], [0], [4] + ) + np.testing.assert_allclose( + fetches[0], expected_output_unpool, rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_unpool3d_op.py b/python/paddle/fluid/tests/unittests/test_unpool3d_op.py index 5c3411d6899..3db2d1514ca 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool3d_op.py @@ -26,8 +26,11 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): input_size = x.shape default_size = [] for d in range(len(kernel_size)): - default_size.append((input_size[-len(kernel_size) + d] - 1) * - stride[d] + kernel_size[d] - 2 * padding[d]) + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) if output_size is None: ret = default_size else: @@ -35,11 +38,13 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): return ret -def unpool3dmax_forward_naive(input, indices, ksize, strides, paddings, - output_size): +def unpool3dmax_forward_naive( + input, indices, ksize, strides, paddings, output_size +): s0, s1, s2, s3, s4 = input.shape - output_size = _unpool_output_size(input, ksize, strides, paddings, - output_size) + output_size = _unpool_output_size( + input, ksize, strides, paddings, output_size + ) out_dsize = output_size[0] out_hsize = output_size[1] out_wsize = output_size[2] @@ -51,49 +56,58 @@ def unpool3dmax_forward_naive(input, indices, ksize, strides, paddings, for w in range(s4): index = indices[nidx, cidx, d, h, w] didx = index // (out_wsize * out_hsize) - hidx = (index - - didx * out_hsize * out_wsize) // out_wsize - widx = (index - - didx * out_hsize * out_wsize) % out_wsize - out[nidx, cidx, didx, hidx, widx] = \ - input[nidx, cidx, d, h, w] + hidx = ( + index - didx * out_hsize * out_wsize + ) // out_wsize + widx = ( + index - didx * out_hsize * out_wsize + ) % out_wsize + out[nidx, cidx, didx, hidx, widx] = input[ + nidx, cidx, d, h, w + ] return out -def max_unpool3d_wrapper(x, - indices, - kernel_size, - stride=None, - padding=0, - output_size=None, - data_format="NCDHW", - name=None): - out = paddle.nn.functional.max_unpool3d(x, - indices, - kernel_size, - stride=stride, - padding=padding, - data_format=data_format, - output_size=output_size, - name=name) +def max_unpool3d_wrapper( + x, + indices, + kernel_size, + stride=None, + padding=0, + output_size=None, + data_format="NCDHW", + name=None, +): + out = paddle.nn.functional.max_unpool3d( + x, + indices, + kernel_size, + stride=stride, + padding=padding, + data_format=data_format, + output_size=output_size, + name=name, + ) return out class TestUnpool3DOp(OpTest): - def setUp(self): self.op_type = "unpool3d" self.python_api = max_unpool3d_wrapper self.init_test_case() inputs = np.random.randint(0, 100, self.shape) nsize, csize, dsize, hsize, wsize = inputs.shape - self.output_size = _unpool_output_size(inputs, self.ksize, self.strides, - self.paddings, self.output_size) + self.output_size = _unpool_output_size( + inputs, self.ksize, self.strides, self.paddings, self.output_size + ) indices = np.random.permutation( np.arange( - 0, self.output_size[0] * self.output_size[1] * - self.output_size[2]))[:dsize * hsize * wsize] + 0, + self.output_size[0] * self.output_size[1] * self.output_size[2], + ) + )[: dsize * hsize * wsize] indices = np.reshape(indices, [dsize, hsize, wsize]) idx_list = [] for n in range(nsize): @@ -103,12 +117,18 @@ class TestUnpool3DOp(OpTest): idx_list.append(c_list) indices = np.array(idx_list) - output = self.unpool3d_forward_naive(inputs, indices, self.ksize, \ - self.strides, self.paddings, self.output_size).astype("float64") + output = self.unpool3d_forward_naive( + inputs, + indices, + self.ksize, + self.strides, + self.paddings, + self.output_size, + ).astype("float64") self.inputs = { 'X': inputs.astype('float64'), - 'Indices': indices.astype('int32') + 'Indices': indices.astype('int32'), } self.attrs = { 'strides': self.strides, @@ -136,7 +156,6 @@ class TestUnpool3DOp(OpTest): class TestUnpool3DOpcase1(TestUnpool3DOp): - def init_test_case(self): self.unpool3d_forward_naive = unpool3dmax_forward_naive self.unpooling_type = "max" @@ -148,7 +167,6 @@ class TestUnpool3DOpcase1(TestUnpool3DOp): class TestUnpool3DOpOutput(TestUnpool3DOp): - def init_test_case(self): self.unpool3d_forward_naive = unpool3dmax_forward_naive self.unpooling_type = "max" @@ -160,50 +178,51 @@ class TestUnpool3DOpOutput(TestUnpool3DOp): class TestUnpool3DOpException(unittest.TestCase): - def test_exception(self): - def indices_size_error(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(0, 36), - shape=[1, 1, 3, 3, 4]) + indices = paddle.reshape( + paddle.arange(0, 36), shape=[1, 1, 3, 3, 4] + ) MaxUnPool3D = F.maxunpool3d(data, indices, kernel_size=2, stride=2) def indices_value_error(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(4, 40), - shape=[1, 1, 3, 3, 3]) + indices = paddle.reshape( + paddle.arange(4, 40), shape=[1, 1, 3, 3, 3] + ) MaxUnPool3D = F.maxunpool3d(data, indices, kernel_size=2, stride=2) def data_format_error(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(0, 27), - shape=[1, 1, 3, 3, 3]) - MaxUnPool3D = F.maxunpool3d(data, - indices, - kernel_size=2, - stride=2, - data_format="NDHWC") + indices = paddle.reshape( + paddle.arange(0, 27), shape=[1, 1, 3, 3, 3] + ) + MaxUnPool3D = F.maxunpool3d( + data, indices, kernel_size=2, stride=2, data_format="NDHWC" + ) def data_outputsize_error(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(0, 27), - shape=[1, 1, 3, 3, 3]) - MaxUnPool3D = F.maxunpool3d(data, - indices, - kernel_size=2, - stride=2, - output_size=[2, 2, 3, 4, 5]) + indices = paddle.reshape( + paddle.arange(0, 27), shape=[1, 1, 3, 3, 3] + ) + MaxUnPool3D = F.maxunpool3d( + data, + indices, + kernel_size=2, + stride=2, + output_size=[2, 2, 3, 4, 5], + ) def data_outputsize_error2(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(0, 27), - shape=[1, 1, 3, 3, 3]) - MaxUnPool3D = F.maxunpool3d(data, - indices, - kernel_size=2, - stride=2, - output_size=[10, 10, 10]) + indices = paddle.reshape( + paddle.arange(0, 27), shape=[1, 1, 3, 3, 3] + ) + MaxUnPool3D = F.maxunpool3d( + data, indices, kernel_size=2, stride=2, output_size=[10, 10, 10] + ) self.assertRaises(ValueError, indices_size_error) self.assertRaises(ValueError, indices_value_error) @@ -213,7 +232,6 @@ class TestUnpool3DOpException(unittest.TestCase): class TestUnpool3DOpAPI_dygraph(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -222,26 +240,28 @@ class TestUnpool3DOpAPI_dygraph(unittest.TestCase): paddle.disable_static() input_data = np.random.rand(1, 3, 4, 4, 6) input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool3d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool3d(output, - indices, - kernel_size=2, - stride=2) + output, indices = F.max_pool3d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool3d( + output, indices, kernel_size=2, stride=2 + ) expected_output_unpool = unpool3dmax_forward_naive( - output.numpy(), indices.numpy(), [2, 2, 2], [2, 2, 2], - [0, 0, 0], [4, 4, 6]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), + indices.numpy(), + [2, 2, 2], + [2, 2, 2], + [0, 0, 0], + [4, 4, 6], + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool3DOpAPI_dygraph2(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -250,26 +270,28 @@ class TestUnpool3DOpAPI_dygraph2(unittest.TestCase): paddle.disable_static() input_data = np.random.rand(1, 3, 4, 4, 6) input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool3d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool3d(output, - indices, - kernel_size=2, - stride=None) + output, indices = F.max_pool3d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool3d( + output, indices, kernel_size=2, stride=None + ) expected_output_unpool = unpool3dmax_forward_naive( - output.numpy(), indices.numpy(), [2, 2, 2], [2, 2, 2], - [0, 0, 0], [4, 4, 6]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), + indices.numpy(), + [2, 2, 2], + [2, 2, 2], + [0, 0, 0], + [4, 4, 6], + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool3DOpAPI_dygraph3(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -278,63 +300,91 @@ class TestUnpool3DOpAPI_dygraph3(unittest.TestCase): paddle.disable_static() input_data = np.random.rand(1, 3, 4, 4, 6) input_x = paddle.to_tensor(input_data) - Pool3d = paddle.nn.MaxPool3D(kernel_size=2, - stride=2, - return_mask=True) + Pool3d = paddle.nn.MaxPool3D( + kernel_size=2, stride=2, return_mask=True + ) UnPool3d = paddle.nn.MaxUnPool3D(kernel_size=2, stride=2) output, indices = Pool3d(input_x) output_unpool = UnPool3d(output, indices) expected_output_unpool = unpool3dmax_forward_naive( - output.numpy(), indices.numpy(), [2, 2, 2], [2, 2, 2], - [0, 0, 0], [4, 4, 6]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), + indices.numpy(), + [2, 2, 2], + [2, 2, 2], + [0, 0, 0], + [4, 4, 6], + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool3DOpAPI_static(unittest.TestCase): - def test_case(self): paddle.enable_static() places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): places.append(paddle.CUDAPlace(0)) for place in places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - - input_data = np.array([[[[[1, 2, 3, 4], [5, 6, 7, 8], \ - [9, 10, 11, 12], [13, 14, 15, 16]], [[1, 2, 3, 4], [5, 6, 7, 8], \ - [9, 10, 11, 12], [13, 14, 15, 16]]]]]).astype("float32") - x = paddle.fluid.data(name='x', - shape=[1, 1, 2, 4, 4], - dtype='float32') - output, indices = F.max_pool3d(x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool3d(output, - indices, - kernel_size=2, - stride=None) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + + input_data = np.array( + [ + [ + [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ], + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ], + ] + ] + ] + ).astype("float32") + x = paddle.fluid.data( + name='x', shape=[1, 1, 2, 4, 4], dtype='float32' + ) + output, indices = F.max_pool3d( + x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool3d( + output, indices, kernel_size=2, stride=None + ) exe = paddle.fluid.Executor(place) - fetches = exe.run(paddle.fluid.default_main_program(), - feed={"x": input_data}, - fetch_list=[output_unpool], - return_numpy=True) - pool3d_out_np = np.array([[[[[6., 8.], - [14., 16.]]]]]).astype("float32") + fetches = exe.run( + paddle.fluid.default_main_program(), + feed={"x": input_data}, + fetch_list=[output_unpool], + return_numpy=True, + ) + pool3d_out_np = np.array( + [[[[[6.0, 8.0], [14.0, 16.0]]]]] + ).astype("float32") indices_np = np.array([[[[[5, 7], [13, 15]]]]]).astype("int32") expected_output_unpool = unpool3dmax_forward_naive( - pool3d_out_np, indices_np, [2, 2, 2], [2, 2, 2], [0, 0, 0], - [2, 4, 4]) - np.testing.assert_allclose(fetches[0], - expected_output_unpool, - rtol=1e-05) + pool3d_out_np, + indices_np, + [2, 2, 2], + [2, 2, 2], + [0, 0, 0], + [2, 4, 4], + ) + np.testing.assert_allclose( + fetches[0], expected_output_unpool, rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_unpool_op.py b/python/paddle/fluid/tests/unittests/test_unpool_op.py index 7c59d628e81..00c48e0872d 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool_op.py @@ -27,8 +27,11 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): input_size = x.shape default_size = [] for d in range(len(kernel_size)): - default_size.append((input_size[-len(kernel_size) + d] - 1) * - stride[d] + kernel_size[d] - 2 * padding[d]) + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) if output_size is None: ret = default_size else: @@ -36,11 +39,13 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): return ret -def unpool2dmax_forward_naive(input, indices, ksize, strides, paddings, - output_size): +def unpool2dmax_forward_naive( + input, indices, ksize, strides, paddings, output_size +): s0, s1, s2, s3 = input.shape - output_size = _unpool_output_size(input, ksize, strides, paddings, - output_size) + output_size = _unpool_output_size( + input, ksize, strides, paddings, output_size + ) out_hsize = output_size[0] out_wsize = output_size[1] out = np.zeros((s0, s1, out_hsize, out_wsize)) @@ -51,44 +56,47 @@ def unpool2dmax_forward_naive(input, indices, ksize, strides, paddings, index = indices[nidx, cidx, h, w] hidx = (index - index % out_wsize) // out_wsize widx = index % out_wsize - out[nidx, cidx, hidx, widx] = \ - input[nidx, cidx, h, w] + out[nidx, cidx, hidx, widx] = input[nidx, cidx, h, w] return out -def max_unpool2d_wrapper(x, - indices, - kernel_size, - stride=None, - padding=0, - output_size=None, - data_format="NCHW", - name=None): - out = paddle.nn.functional.max_unpool2d(x, - indices, - kernel_size, - stride=stride, - padding=padding, - data_format=data_format, - output_size=output_size, - name=name) +def max_unpool2d_wrapper( + x, + indices, + kernel_size, + stride=None, + padding=0, + output_size=None, + data_format="NCHW", + name=None, +): + out = paddle.nn.functional.max_unpool2d( + x, + indices, + kernel_size, + stride=stride, + padding=padding, + data_format=data_format, + output_size=output_size, + name=name, + ) return out class TestUnpoolOp(OpTest): - def setUp(self): self.op_type = "unpool" self.python_api = max_unpool2d_wrapper self.init_test_case() input = np.random.randint(0, 100, self.shape) nsize, csize, hsize, wsize = input.shape - self.output_size = _unpool_output_size(input, self.ksize, self.strides, - self.paddings, self.output_size) + self.output_size = _unpool_output_size( + input, self.ksize, self.strides, self.paddings, self.output_size + ) indices = np.random.permutation( - np.arange(0, self.output_size[0] * self.output_size[1]))[:hsize * - wsize] + np.arange(0, self.output_size[0] * self.output_size[1]) + )[: hsize * wsize] indices = np.reshape(indices, [hsize, wsize]) idx_list = [] for n in range(nsize): @@ -98,12 +106,18 @@ class TestUnpoolOp(OpTest): idx_list.append(c_list) indices = np.array(idx_list) - output = self.unpool2d_forward_naive(input, indices, self.ksize, \ - self.strides, self.paddings, self.output_size).astype("float64") + output = self.unpool2d_forward_naive( + input, + indices, + self.ksize, + self.strides, + self.paddings, + self.output_size, + ).astype("float64") self.inputs = { 'X': input.astype('float64'), - 'Indices': indices.astype('int32') + 'Indices': indices.astype('int32'), } self.attrs = { 'strides': self.strides, @@ -131,7 +145,6 @@ class TestUnpoolOp(OpTest): class TestUnpoolOpcase1(TestUnpoolOp): - def init_test_case(self): self.unpool2d_forward_naive = unpool2dmax_forward_naive self.unpooling_type = "max" @@ -143,7 +156,6 @@ class TestUnpoolOpcase1(TestUnpoolOp): class TestUnpoolOpOutputsize(TestUnpoolOp): - def init_test_case(self): self.unpool2d_forward_naive = unpool2dmax_forward_naive self.unpooling_type = "max" @@ -155,7 +167,6 @@ class TestUnpoolOpOutputsize(TestUnpoolOp): class TestUnpoolOpOutput(TestUnpoolOp): - def init_test_case(self): self.unpool2d_forward_naive = unpool2dmax_forward_naive self.unpooling_type = "max" @@ -167,7 +178,6 @@ class TestUnpoolOpOutput(TestUnpoolOp): class TestUnpoolOpException(unittest.TestCase): - def test_exception(self): import paddle.nn.functional as F import paddle @@ -185,29 +195,23 @@ class TestUnpoolOpException(unittest.TestCase): def data_format_error(): data = paddle.randint(shape=[1, 1, 3, 3]) indices = paddle.reshape(paddle.arange(4, 40), shape[1, 1, 3, 4]) - MaxPool2D = F.maxunpool2d(data, - indices, - kernel_size=2, - stride=2, - data_format="NHWC") + MaxPool2D = F.maxunpool2d( + data, indices, kernel_size=2, stride=2, data_format="NHWC" + ) def data_outputsize_error(): data = paddle.randint(shape=[1, 1, 3, 3]) indices = paddle.reshape(paddle.arange(4, 40), shape[1, 1, 3, 4]) - MaxPool2D = F.maxunpool2d(data, - indices, - kernel_size=2, - stride=2, - output_size=[5, 6, 7, 8]) + MaxPool2D = F.maxunpool2d( + data, indices, kernel_size=2, stride=2, output_size=[5, 6, 7, 8] + ) def data_outputsize_error2(): data = paddle.randint(shape=[1, 1, 3, 3]) indices = paddle.reshape(paddle.arange(4, 40), shape[1, 1, 3, 4]) - MaxPool2D = F.maxunpool2d(data, - indices, - kernel_size=2, - stride=2, - output_size=[100, 100]) + MaxPool2D = F.maxunpool2d( + data, indices, kernel_size=2, stride=2, output_size=[100, 100] + ) self.assertRaises(ValueError, indices_size_error) self.assertRaises(ValueError, indices_value_error) @@ -217,7 +221,6 @@ class TestUnpoolOpException(unittest.TestCase): class TestUnpoolOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle import paddle.nn.functional as F @@ -230,28 +233,34 @@ class TestUnpoolOpAPI_dy(unittest.TestCase): else: place = core.CPUPlace() with fluid.dygraph.guard(place): - input_data = np.array([[[[1, 2, 3, 4], [5, 6, 7, - 8], [9, 10, 11, 12], - [13, 14, 15, 16]]]]).astype("float32") + input_data = np.array( + [ + [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ] + ] + ] + ).astype("float32") input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool2d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - out_pp = F.max_unpool2d(output, - indices, - kernel_size=2, - stride=2, - output_size=(5, 5)) + output, indices = F.max_pool2d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + out_pp = F.max_unpool2d( + output, indices, kernel_size=2, stride=2, output_size=(5, 5) + ) output_np = output.numpy() indices_np = indices.numpy() - expect_res =unpool2dmax_forward_naive(output_np, indices_np, [2,2], \ - [2,2], [0,0], [5,5]).astype("float64") + expect_res = unpool2dmax_forward_naive( + output_np, indices_np, [2, 2], [2, 2], [0, 0], [5, 5] + ).astype("float64") np.testing.assert_allclose(out_pp.numpy(), expect_res, rtol=1e-05) class TestUnpoolOpAPI_dy2(unittest.TestCase): - def test_case(self): import paddle import paddle.nn.functional as F @@ -264,28 +273,34 @@ class TestUnpoolOpAPI_dy2(unittest.TestCase): else: place = core.CPUPlace() with fluid.dygraph.guard(place): - input_data = np.array([[[[1, 2, 3, 4], [5, 6, 7, - 8], [9, 10, 11, 12], - [13, 14, 15, 16]]]]).astype("float32") + input_data = np.array( + [ + [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ] + ] + ] + ).astype("float32") input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool2d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - out_pp = F.max_unpool2d(output, - indices, - kernel_size=2, - stride=None, - output_size=(5, 5)) + output, indices = F.max_pool2d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + out_pp = F.max_unpool2d( + output, indices, kernel_size=2, stride=None, output_size=(5, 5) + ) output_np = output.numpy() indices_np = indices.numpy() - expect_res =unpool2dmax_forward_naive(output_np, indices_np, [2,2], \ - [2,2], [0,0], [5,5]).astype("float64") + expect_res = unpool2dmax_forward_naive( + output_np, indices_np, [2, 2], [2, 2], [0, 0], [5, 5] + ).astype("float64") np.testing.assert_allclose(out_pp.numpy(), expect_res, rtol=1e-05) class TestUnpoolOpAPI_dy3(unittest.TestCase): - def test_case(self): import paddle import paddle.fluid.core as core @@ -297,46 +312,54 @@ class TestUnpoolOpAPI_dy3(unittest.TestCase): else: place = core.CPUPlace() with fluid.dygraph.guard(place): - input_data = np.array([[[[1, 2, 3, 4], [5, 6, 7, - 8], [9, 10, 11, 12], - [13, 14, 15, 16]]]]).astype("float32") + input_data = np.array( + [ + [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ] + ] + ] + ).astype("float32") input_x = paddle.to_tensor(input_data) - Pool2d = paddle.nn.MaxPool2D(kernel_size=2, - stride=2, - return_mask=True) + Pool2d = paddle.nn.MaxPool2D( + kernel_size=2, stride=2, return_mask=True + ) UnPool = paddle.nn.MaxUnPool2D(kernel_size=2, stride=2) output, indices = Pool2d(input_x) out_pp = UnPool(output, indices) output_np = output.numpy() indices_np = indices.numpy() - expect_res =unpool2dmax_forward_naive(output_np, indices_np, [2,2], \ - [2,2], [0,0], [4,4]).astype("float64") + expect_res = unpool2dmax_forward_naive( + output_np, indices_np, [2, 2], [2, 2], [0, 0], [4, 4] + ).astype("float64") np.testing.assert_allclose(out_pp.numpy(), expect_res, rtol=1e-05) class TestUnpoolOpAPI_st(unittest.TestCase): - def test_case(self): import paddle import paddle.nn.functional as F import paddle.fluid.core as core import paddle.fluid as fluid + paddle.enable_static() - input_data = np.array([[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], - [13, 14, 15, 16]]]]).astype("float32") + input_data = np.array( + [[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]]] + ).astype("float32") x = fluid.data(name="x", shape=[1, 1, 4, 4], dtype="float32") - output, indices = F.max_pool2d(x, - kernel_size=2, - stride=2, - return_mask=True) - unpool_out = F.max_unpool2d(output, - indices, - kernel_size=2, - stride=None, - output_size=(5, 5)) + output, indices = F.max_pool2d( + x, kernel_size=2, stride=2, return_mask=True + ) + unpool_out = F.max_unpool2d( + output, indices, kernel_size=2, stride=None, output_size=(5, 5) + ) if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -344,20 +367,22 @@ class TestUnpoolOpAPI_st(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(paddle.fluid.default_main_program(),\ - feed={"x":input_data}, - fetch_list=[unpool_out], - return_numpy=True) + results = exe.run( + paddle.fluid.default_main_program(), + feed={"x": input_data}, + fetch_list=[unpool_out], + return_numpy=True, + ) - pool_out_np = np.array([[[[6., 8.], [14., 16.]]]]).astype("float32") + pool_out_np = np.array([[[[6.0, 8.0], [14.0, 16.0]]]]).astype("float32") indices_np = np.array([[[[5, 7], [13, 15]]]]).astype("int32") - expect_res =unpool2dmax_forward_naive(pool_out_np, indices_np, [2,2], \ - [2,2], [0,0], [5,5]).astype("float64") + expect_res = unpool2dmax_forward_naive( + pool_out_np, indices_np, [2, 2], [2, 2], [0, 0], [5, 5] + ).astype("float64") np.testing.assert_allclose(results[0], expect_res, rtol=1e-05) class TestOutputSizeTensor(UnittestBase): - def init_info(self): self.shapes = [[1, 3, 6, 6]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -394,23 +419,18 @@ class TestOutputSizeTensor(UnittestBase): def call_func(self, x): output_size = [paddle.assign([7]), paddle.assign([7])] - pool_out, indices = F.max_pool2d(x, - kernel_size=2, - stride=2, - padding=0, - return_mask=True) + pool_out, indices = F.max_pool2d( + x, kernel_size=2, stride=2, padding=0, return_mask=True + ) # pool_out shape: [1, 1, 6, 6], indices shape: [1, 1, 6, 6] - unpool_out = F.max_unpool2d(pool_out, - indices, - kernel_size=2, - padding=0, - output_size=output_size) + unpool_out = F.max_unpool2d( + pool_out, indices, kernel_size=2, padding=0, output_size=output_size + ) # unpool_out shape: [1, 1, 7, 7] return unpool_out class TestZOutputSizeTensor2(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -419,22 +439,17 @@ class TestZOutputSizeTensor2(unittest.TestCase): def test_dygraph(self): x = paddle.randn([1, 3, 6, 6]) - pool_out, indices = F.max_pool2d(x, - kernel_size=2, - stride=2, - padding=0, - return_mask=True) + pool_out, indices = F.max_pool2d( + x, kernel_size=2, stride=2, padding=0, return_mask=True + ) output_size = [paddle.assign([7]), paddle.assign([7])] - unpool_out = F.max_unpool2d(pool_out, - indices, - kernel_size=2, - padding=0, - output_size=output_size) + unpool_out = F.max_unpool2d( + pool_out, indices, kernel_size=2, padding=0, output_size=output_size + ) np.testing.assert_array_equal(unpool_out.shape, [1, 3, 7, 7]) class TestZOutputSizeTensor3(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -443,22 +458,18 @@ class TestZOutputSizeTensor3(unittest.TestCase): def test_dygraph(self): x = paddle.randn([1, 3, 6, 6]) - pool_out, indices = F.max_pool2d(x, - kernel_size=2, - stride=2, - padding=0, - return_mask=True) + pool_out, indices = F.max_pool2d( + x, kernel_size=2, stride=2, padding=0, return_mask=True + ) output_size = [ paddle.assign([1]), paddle.assign([1]), paddle.assign([7]), - paddle.assign([7]) + paddle.assign([7]), ] - unpool_out = F.max_unpool2d(pool_out, - indices, - kernel_size=2, - padding=0, - output_size=output_size) + unpool_out = F.max_unpool2d( + pool_out, indices, kernel_size=2, padding=0, output_size=output_size + ) np.testing.assert_array_equal(unpool_out.shape, [1, 3, 7, 7]) diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze2_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze2_op.py index d585c8a3d0f..8de9185162a 100755 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze2_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze2_op.py @@ -25,7 +25,6 @@ paddle.enable_static() # Correct: General. class TestUnsqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze2" @@ -35,7 +34,7 @@ class TestUnsqueezeOp(OpTest): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float64") + "XShape": np.random.random(self.ori_shape).astype("float64"), } def test_check_output(self): @@ -55,16 +54,14 @@ class TestUnsqueezeOp(OpTest): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -73,7 +70,6 @@ class TestUnsqueezeOp2(TestUnsqueezeOp): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -82,7 +78,6 @@ class TestUnsqueezeOp3(TestUnsqueezeOp): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -90,15 +85,13 @@ class TestUnsqueezeOp4(TestUnsqueezeOp): class TestUnsqueezeOp_ZeroDim1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () - self.axes = (-1, ) - self.new_shape = (1) + self.axes = (-1,) + self.new_shape = 1 class TestUnsqueezeOp_ZeroDim2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () self.axes = (-1, 1) @@ -106,7 +99,6 @@ class TestUnsqueezeOp_ZeroDim2(TestUnsqueezeOp): class TestUnsqueezeOp_ZeroDim3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () self.axes = (0, 1, 2) @@ -115,7 +107,6 @@ class TestUnsqueezeOp_ZeroDim3(TestUnsqueezeOp): # axes is a list(with tensor) class TestUnsqueezeOp_AxesTensorList(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze2" @@ -124,17 +115,18 @@ class TestUnsqueezeOp_AxesTensorList(OpTest): axes_tensor_list = [] for index, ele in enumerate(self.axes): - axes_tensor_list.append(("axes" + str(index), np.ones( - (1)).astype('int32') * ele)) + axes_tensor_list.append( + ("axes" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype("float64"), - "AxesTensorList": axes_tensor_list + "AxesTensorList": axes_tensor_list, } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float64") + "XShape": np.random.random(self.ori_shape).astype("float64"), } def test_check_output(self): @@ -153,15 +145,13 @@ class TestUnsqueezeOp_AxesTensorList(OpTest): class TestUnsqueezeOp1_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueezeOp2_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -169,7 +159,6 @@ class TestUnsqueezeOp2_AxesTensorList(TestUnsqueezeOp_AxesTensorList): class TestUnsqueezeOp3_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -177,7 +166,6 @@ class TestUnsqueezeOp3_AxesTensorList(TestUnsqueezeOp_AxesTensorList): class TestUnsqueezeOp4_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -186,7 +174,6 @@ class TestUnsqueezeOp4_AxesTensorList(TestUnsqueezeOp_AxesTensorList): # axes is a Tensor class TestUnsqueezeOp_AxesTensor(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze2" @@ -195,12 +182,12 @@ class TestUnsqueezeOp_AxesTensor(OpTest): self.inputs = { "X": np.random.random(self.ori_shape).astype("float64"), - "AxesTensor": np.array(self.axes).astype("int32") + "AxesTensor": np.array(self.axes).astype("int32"), } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float64") + "XShape": np.random.random(self.ori_shape).astype("float64"), } def test_check_output(self): @@ -219,15 +206,13 @@ class TestUnsqueezeOp_AxesTensor(OpTest): class TestUnsqueezeOp1_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueezeOp2_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -235,7 +220,6 @@ class TestUnsqueezeOp2_AxesTensor(TestUnsqueezeOp_AxesTensor): class TestUnsqueezeOp3_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -243,7 +227,6 @@ class TestUnsqueezeOp3_AxesTensor(TestUnsqueezeOp_AxesTensor): class TestUnsqueezeOp4_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -252,7 +235,6 @@ class TestUnsqueezeOp4_AxesTensor(TestUnsqueezeOp_AxesTensor): # test api class TestUnsqueezeAPI(unittest.TestCase): - def setUp(self): self.executed_api() @@ -264,12 +246,12 @@ class TestUnsqueezeAPI(unittest.TestCase): x = paddle.static.data(name='x', shape=[3, 2, 5], dtype="float64") positive_3_int32 = fluid.layers.fill_constant([1], "int32", 3) positive_1_int64 = fluid.layers.fill_constant([1], "int64", 1) - axes_tensor_int32 = paddle.static.data(name='axes_tensor_int32', - shape=[3], - dtype="int32") - axes_tensor_int64 = paddle.static.data(name='axes_tensor_int64', - shape=[3], - dtype="int64") + axes_tensor_int32 = paddle.static.data( + name='axes_tensor_int32', shape=[3], dtype="int32" + ) + axes_tensor_int64 = paddle.static.data( + name='axes_tensor_int64', shape=[3], dtype="int64" + ) out_1 = self.unsqueeze(x, axis=[3, 1, 1]) out_2 = self.unsqueeze(x, axis=[positive_3_int32, positive_1_int64, 1]) @@ -283,9 +265,10 @@ class TestUnsqueezeAPI(unittest.TestCase): feed={ "x": input, "axes_tensor_int32": np.array([3, 1, 1]).astype("int32"), - "axes_tensor_int64": np.array([3, 1, 1]).astype("int64") + "axes_tensor_int64": np.array([3, 1, 1]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.array_equal(res_1, input.reshape([3, 1, 1, 2, 5, 1])) assert np.array_equal(res_2, input.reshape([3, 1, 1, 2, 5, 1])) @@ -294,7 +277,6 @@ class TestUnsqueezeAPI(unittest.TestCase): assert np.array_equal(res_5, input.reshape([3, 1, 1, 2, 5, 1])) def test_error(self): - def test_axes_type(): x2 = paddle.static.data(name="x2", shape=[2, 25], dtype="int32") self.unsqueeze(x2, axis=2.1) @@ -303,13 +285,11 @@ class TestUnsqueezeAPI(unittest.TestCase): class TestUnsqueezeInplaceAPI(TestUnsqueezeAPI): - def executed_api(self): self.unsqueeze = paddle.unsqueeze_ class TestUnsqueezeAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py index c1261d04857..b73c2a3906f 100755 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py @@ -29,7 +29,6 @@ paddle.enable_static() # Correct: General. class TestUnsqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze" @@ -53,7 +52,6 @@ class TestUnsqueezeOp(OpTest): class TestUnsqueezeBF16Op(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze" @@ -81,16 +79,14 @@ class TestUnsqueezeBF16Op(OpTest): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -99,7 +95,6 @@ class TestUnsqueezeOp2(TestUnsqueezeOp): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -108,7 +103,6 @@ class TestUnsqueezeOp3(TestUnsqueezeOp): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -116,15 +110,13 @@ class TestUnsqueezeOp4(TestUnsqueezeOp): class TestUnsqueezeOp_ZeroDim1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () - self.axes = (-1, ) - self.new_shape = (1) + self.axes = (-1,) + self.new_shape = 1 class TestUnsqueezeOp_ZeroDim2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () self.axes = (-1, 1) @@ -132,7 +124,6 @@ class TestUnsqueezeOp_ZeroDim2(TestUnsqueezeOp): class TestUnsqueezeOp_ZeroDim3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () self.axes = (0, 1, 2) @@ -140,44 +131,45 @@ class TestUnsqueezeOp_ZeroDim3(TestUnsqueezeOp): class API_TestUnsqueeze(unittest.TestCase): - def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data1 = paddle.static.data('data1', shape=[-1, 10], dtype='float64') result_squeeze = paddle.unsqueeze(data1, axis=[1]) place = paddle.CPUPlace() exe = paddle.static.Executor(place) input1 = np.random.random([5, 1, 10]).astype('float64') input = np.squeeze(input1, axis=1) - result, = exe.run(feed={"data1": input}, - fetch_list=[result_squeeze]) + (result,) = exe.run( + feed={"data1": input}, fetch_list=[result_squeeze] + ) np.testing.assert_allclose(input1, result, rtol=1e-05) class TestUnsqueezeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): # The type of axis in split_op should be int or Variable. def test_axes_type(): - x6 = paddle.static.data(shape=[-1, 10], - dtype='float16', - name='x3') + x6 = paddle.static.data( + shape=[-1, 10], dtype='float16', name='x3' + ) paddle.unsqueeze(x6, axis=3.2) self.assertRaises(TypeError, test_axes_type) class API_TestUnsqueeze2(unittest.TestCase): - def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data1 = paddle.static.data('data1', shape=[-1, 10], dtype='float64') data2 = paddle.static.data('data2', shape=[1], dtype='int32') result_squeeze = paddle.unsqueeze(data1, axis=data2) @@ -186,20 +178,19 @@ class API_TestUnsqueeze2(unittest.TestCase): input1 = np.random.random([5, 1, 10]).astype('float64') input2 = np.array([1]).astype('int32') input = np.squeeze(input1, axis=1) - result1, = exe.run(feed={ - "data1": input, - "data2": input2 - }, - fetch_list=[result_squeeze]) + (result1,) = exe.run( + feed={"data1": input, "data2": input2}, + fetch_list=[result_squeeze], + ) np.testing.assert_allclose(input1, result1, rtol=1e-05) class API_TestUnsqueeze3(unittest.TestCase): - def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data1 = paddle.static.data('data1', shape=[-1, 10], dtype='float64') data2 = paddle.static.data('data2', shape=[1], dtype='int32') result_squeeze = paddle.unsqueeze(data1, axis=[data2, 3]) @@ -208,17 +199,15 @@ class API_TestUnsqueeze3(unittest.TestCase): input1 = np.random.random([5, 1, 10, 1]).astype('float64') input2 = np.array([1]).astype('int32') input = np.squeeze(input1) - result1, = exe.run(feed={ - "data1": input, - "data2": input2 - }, - fetch_list=[result_squeeze]) + (result1,) = exe.run( + feed={"data1": input, "data2": input2}, + fetch_list=[result_squeeze], + ) np.testing.assert_array_equal(input1, result1) self.assertEqual(input1.shape, result1.shape) class API_TestDyUnsqueeze(unittest.TestCase): - def test_out(self): paddle.disable_static() input_1 = np.random.random([5, 1, 10]).astype("int32") @@ -231,7 +220,6 @@ class API_TestDyUnsqueeze(unittest.TestCase): class API_TestDyUnsqueeze2(unittest.TestCase): - def test_out(self): paddle.disable_static() input1 = np.random.random([5, 10]).astype("int32") @@ -244,7 +232,6 @@ class API_TestDyUnsqueeze2(unittest.TestCase): class API_TestDyUnsqueezeAxisTensor(unittest.TestCase): - def test_out(self): paddle.disable_static() input1 = np.random.random([5, 10]).astype("int32") @@ -258,7 +245,6 @@ class API_TestDyUnsqueezeAxisTensor(unittest.TestCase): class API_TestDyUnsqueezeAxisTensorList(unittest.TestCase): - def test_out(self): paddle.disable_static() input1 = np.random.random([5, 10]).astype("int32") @@ -268,15 +254,14 @@ class API_TestDyUnsqueezeAxisTensorList(unittest.TestCase): input = paddle.to_tensor(input1) output = paddle.unsqueeze( paddle.to_tensor(input1), - axis=[paddle.to_tensor([1]), - paddle.to_tensor([2])]) + axis=[paddle.to_tensor([1]), paddle.to_tensor([2])], + ) out_np = output.numpy() np.testing.assert_array_equal(out1, out_np) self.assertEqual(out1.shape, out_np.shape) class API_TestDygraphUnSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -330,13 +315,11 @@ class API_TestDygraphUnSqueeze(unittest.TestCase): class API_TestDygraphUnSqueezeInplace(API_TestDygraphUnSqueeze): - def executed_api(self): self.unsqueeze = paddle.unsqueeze_ class TestUnsqueezeDoubleGradCheck(unittest.TestCase): - def unsqueeze_wrapper(self, x): return paddle.unsqueeze(x[0], [0, 2]) @@ -351,17 +334,13 @@ class TestUnsqueezeDoubleGradCheck(unittest.TestCase): out = paddle.unsqueeze(data, [0, 2]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.unsqueeze_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.unsqueeze_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -373,7 +352,6 @@ class TestUnsqueezeDoubleGradCheck(unittest.TestCase): class TestUnsqueezeTripleGradCheck(unittest.TestCase): - def unsqueeze_wrapper(self, x): return paddle.unsqueeze(x[0], [0, 2]) @@ -388,17 +366,13 @@ class TestUnsqueezeTripleGradCheck(unittest.TestCase): out = paddle.unsqueeze(data, [0, 2]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.unsqueeze_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.unsqueeze_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_unstack_op.py b/python/paddle/fluid/tests/unittests/test_unstack_op.py index bb28bdeba79..87095d8eeac 100755 --- a/python/paddle/fluid/tests/unittests/test_unstack_op.py +++ b/python/paddle/fluid/tests/unittests/test_unstack_op.py @@ -19,7 +19,6 @@ import paddle class TestUnStackOpBase(OpTest): - def initDefaultParameters(self): self.input_dim = (5, 6, 7) self.axis = 0 @@ -64,25 +63,21 @@ class TestUnStackOpBase(OpTest): class TestStackOp3(TestUnStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestUnStackOpBase): - def initParameters(self): self.axis = -3 class TestStackOp5(TestUnStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestUnStackOpBase): - def initParameters(self): self.axis = 2 diff --git a/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py b/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py index 90328249501..7f1e6c8614f 100644 --- a/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py +++ b/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py @@ -19,31 +19,43 @@ import paddle.fluid as fluid import paddle.fluid.contrib.mixed_precision.amp_nn as amp_nn -def update_loss_scaling_wrapper(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - stop_update=False): - amp_nn.update_loss_scaling([x], found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps, - incr_every_n_steps, decr_every_n_nan_or_inf, - incr_ratio, decr_ratio, stop_update) +def update_loss_scaling_wrapper( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + stop_update=False, +): + amp_nn.update_loss_scaling( + [x], + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + stop_update, + ) return x, prev_loss_scaling, num_good_steps, num_bad_steps class TestUpdateLossScalingOp(OpTest): - def setUp(self): self.op_type = "update_loss_scaling" self.init() self.python_api = update_loss_scaling_wrapper self.python_out_sig = [ - "out0", "LossScaling", "OutGoodSteps", "OutBadSteps" + "out0", + "LossScaling", + "OutGoodSteps", + "OutBadSteps", ] found_inf = np.array([False], dtype=np.bool_) x = np.random.random((1024, 1024)).astype(self.dtype) @@ -53,14 +65,14 @@ class TestUpdateLossScalingOp(OpTest): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', x)], 'LossScaling': self.prev_loss_scaling * self.incr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def init(self): @@ -84,13 +96,15 @@ class TestUpdateLossScalingOp(OpTest): class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): - def setUp(self): self.op_type = "update_loss_scaling" self.init() self.python_api = update_loss_scaling_wrapper self.python_out_sig = [ - "out0", "LossScaling", "OutGoodSteps", "OutBadSteps" + "out0", + "LossScaling", + "OutGoodSteps", + "OutBadSteps", ] found_inf = np.array([True], dtype=np.bool_) x = np.random.random((1024, 1024)).astype(self.dtype) @@ -104,14 +118,14 @@ class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, 'InBadSteps': self.num_bad_steps, - 'StopUpdate': self.stop_update + 'StopUpdate': self.stop_update, } self.outputs = { 'Out': [('out0', np.zeros_like(x))], 'LossScaling': self.prev_loss_scaling * self.decr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def test_check_output(self): @@ -119,21 +133,20 @@ class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): class TestUpdateLossScalingLayer(unittest.TestCase): - def loss_scaling_check(self, use_cuda=True, scope=fluid.Scope()): a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -147,33 +160,41 @@ class TestUpdateLossScalingLayer(unittest.TestCase): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], a_v) assert np.array_equal(result_v[1], b_v) assert np.array_equal(result_v[0], result_v[2]) @@ -188,15 +209,15 @@ class TestUpdateLossScalingLayer(unittest.TestCase): b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -213,33 +234,41 @@ class TestUpdateLossScalingLayer(unittest.TestCase): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], np.zeros_like(a_v)) assert np.array_equal(result_v[1], np.zeros_like(b_v)) assert np.array_equal(result_v[2], np.zeros_like(a_v)) diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py index 7804aec60c9..d1b820bd74c 100644 --- a/python/paddle/fluid/tests/unittests/test_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_var_base.py @@ -23,14 +23,12 @@ from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph class TestVarBase(unittest.TestCase): - def setUp(self): self.shape = [512, 1234] self.dtype = np.float32 self.array = np.random.uniform(0.1, 1, self.shape).astype(self.dtype) def func_test_to_tensor(self): - def check_with_place(place): with fluid.dygraph.guard(): paddle.set_default_dtype('float32') @@ -43,11 +41,14 @@ class TestVarBase(unittest.TestCase): self.assertEqual(str(x.place), str(y.place)) # set_default_dtype should not take effect on numpy - x = paddle.to_tensor(np.array([1.2]).astype('float16'), - place=place, - stop_gradient=False) - np.testing.assert_array_equal(x.numpy(), - np.array([1.2], 'float16')) + x = paddle.to_tensor( + np.array([1.2]).astype('float16'), + place=place, + stop_gradient=False, + ) + np.testing.assert_array_equal( + x.numpy(), np.array([1.2], 'float16') + ) self.assertEqual(x.dtype, core.VarDesc.VarType.FP16) # set_default_dtype take effect on int @@ -56,17 +57,20 @@ class TestVarBase(unittest.TestCase): # set_default_dtype take effect on float x = paddle.to_tensor(1.2, place=place, stop_gradient=False) - np.testing.assert_array_equal(x.numpy(), - np.array([1.2]).astype('float32')) + np.testing.assert_array_equal( + x.numpy(), np.array([1.2]).astype('float32') + ) self.assertEqual(x.dtype, core.VarDesc.VarType.FP32) clone_x = x.clone() - np.testing.assert_array_equal(clone_x.numpy(), - np.array([1.2]).astype('float32')) + np.testing.assert_array_equal( + clone_x.numpy(), np.array([1.2]).astype('float32') + ) self.assertEqual(clone_x.dtype, core.VarDesc.VarType.FP32) y = clone_x**2 y.backward() - np.testing.assert_array_equal(x.grad.numpy(), - np.array([2.4]).astype('float32')) + np.testing.assert_array_equal( + x.grad.numpy(), np.array([2.4]).astype('float32') + ) y = x.cpu() self.assertEqual(y.place.__repr__(), "Place(cpu)") if core.is_compiled_with_cuda(): @@ -104,24 +108,21 @@ class TestVarBase(unittest.TestCase): np.testing.assert_array_equal(x.numpy(), [1 + 2j]) self.assertEqual(x.dtype, core.VarDesc.VarType.COMPLEX128) - x = paddle.to_tensor(1, - dtype='float32', - place=place, - stop_gradient=False) + x = paddle.to_tensor( + 1, dtype='float32', place=place, stop_gradient=False + ) np.testing.assert_array_equal(x.numpy(), [1.0]) self.assertEqual(x.dtype, core.VarDesc.VarType.FP32) self.assertEqual(x.shape, [1]) self.assertEqual(x.stop_gradient, False) self.assertEqual(x.type, core.VarDesc.VarType.LOD_TENSOR) - x = paddle.to_tensor((1, 2), - dtype='float32', - place=place, - stop_gradient=False) - x = paddle.to_tensor([1, 2], - dtype='float32', - place=place, - stop_gradient=False) + x = paddle.to_tensor( + (1, 2), dtype='float32', place=place, stop_gradient=False + ) + x = paddle.to_tensor( + [1, 2], dtype='float32', place=place, stop_gradient=False + ) np.testing.assert_array_equal(x.numpy(), [1.0, 2.0]) self.assertEqual(x.dtype, core.VarDesc.VarType.FP32) self.assertEqual(x.grad, None) @@ -129,10 +130,12 @@ class TestVarBase(unittest.TestCase): self.assertEqual(x.stop_gradient, False) self.assertEqual(x.type, core.VarDesc.VarType.LOD_TENSOR) - x = paddle.to_tensor(self.array, - dtype='float32', - place=place, - stop_gradient=False) + x = paddle.to_tensor( + self.array, + dtype='float32', + place=place, + stop_gradient=False, + ) np.testing.assert_array_equal(x.numpy(), self.array) self.assertEqual(x.dtype, core.VarDesc.VarType.FP32) self.assertEqual(x.shape, self.shape) @@ -149,9 +152,9 @@ class TestVarBase(unittest.TestCase): z = x + y np.testing.assert_array_equal(z.numpy(), 2 * self.array) - x = paddle.to_tensor([1 + 2j, 1 - 2j], - dtype='complex64', - place=place) + x = paddle.to_tensor( + [1 + 2j, 1 - 2j], dtype='complex64', place=place + ) y = paddle.to_tensor(x) np.testing.assert_array_equal(x.numpy(), [1 + 2j, 1 - 2j]) self.assertEqual(y.dtype, core.VarDesc.VarType.COMPLEX64) @@ -172,8 +175,9 @@ class TestVarBase(unittest.TestCase): self.assertTrue(isinstance(x.item(5), float)) self.assertTrue(isinstance(x.item(1, 0, 1), float)) self.assertEqual(x.item(5), x.item(1, 0, 1)) - np.testing.assert_array_equal(x.item(1, 0, 1), - x.numpy().item(1, 0, 1)) + np.testing.assert_array_equal( + x.item(1, 0, 1), x.numpy().item(1, 0, 1) + ) x = paddle.to_tensor([[1.111111, 2.222222, 3.333333]]) self.assertEqual(x.item(0, 2), x.item(2)) @@ -404,16 +408,18 @@ class TestVarBase(unittest.TestCase): y = x + 1 self.assertTrue(y.is_leaf) - x = paddle.to_tensor(np.random.uniform(-1, 1, size=[10, 10]), - stop_gradient=False) + x = paddle.to_tensor( + np.random.uniform(-1, 1, size=[10, 10]), stop_gradient=False + ) self.assertTrue(x.is_leaf) y = x + 1 self.assertFalse(y.is_leaf) linear = paddle.nn.Linear(10, 10) - input = paddle.to_tensor(np.random.uniform( - -1, 1, size=[10, 10]).astype('float32'), - stop_gradient=False) + input = paddle.to_tensor( + np.random.uniform(-1, 1, size=[10, 10]).astype('float32'), + stop_gradient=False, + ) self.assertTrue(input.is_leaf) out = linear(input) @@ -432,8 +438,9 @@ class TestVarBase(unittest.TestCase): detach_x = x.detach() self.assertTrue(detach_x.stop_gradient, True) - cmp_float = np.allclose if core.is_compiled_with_rocm( - ) else np.array_equal + cmp_float = ( + np.allclose if core.is_compiled_with_rocm() else np.array_equal + ) detach_x[:] = 10.0 self.assertTrue(cmp_float(x.numpy(), [10.0])) @@ -442,7 +449,9 @@ class TestVarBase(unittest.TestCase): self.assertTrue(cmp_float(x.grad.numpy(), [20.0])) self.assertEqual(detach_x.grad, None) - detach_x.stop_gradient = False # Set stop_gradient to be False, supported auto-grad + detach_x.stop_gradient = ( + False # Set stop_gradient to be False, supported auto-grad + ) z = 3 * detach_x**2 z.backward() self.assertTrue(cmp_float(x.grad.numpy(), [20.0])) @@ -492,14 +501,15 @@ class TestVarBase(unittest.TestCase): else: empty_var = core.eager.Tensor() empty_var_copy = copy.deepcopy(empty_var) - self.assertEqual(empty_var.stop_gradient, - empty_var_copy.stop_gradient) + self.assertEqual( + empty_var.stop_gradient, empty_var_copy.stop_gradient + ) self.assertEqual(empty_var.persistable, empty_var_copy.persistable) self.assertEqual(empty_var.type, empty_var_copy.type) self.assertEqual(empty_var.dtype, empty_var_copy.dtype) - x = paddle.to_tensor([2.], stop_gradient=False) - y = paddle.to_tensor([3.], stop_gradient=False) + x = paddle.to_tensor([2.0], stop_gradient=False) + y = paddle.to_tensor([3.0], stop_gradient=False) z = x * y memo = {} x_copy = copy.deepcopy(x, memo) @@ -516,7 +526,7 @@ class TestVarBase(unittest.TestCase): np.testing.assert_array_equal(x.numpy(), [2.0]) with self.assertRaises(ValueError): - x_copy[:] = 5. + x_copy[:] = 5.0 with self.assertRaises(RuntimeError): copy.deepcopy(z) @@ -528,17 +538,26 @@ class TestVarBase(unittest.TestCase): # test copy selected rows if _in_legacy_dygraph(): - x = core.VarBase(core.VarDesc.VarType.FP32, [3, 100], - "selected_rows", - core.VarDesc.VarType.SELECTED_ROWS, True) + x = core.VarBase( + core.VarDesc.VarType.FP32, + [3, 100], + "selected_rows", + core.VarDesc.VarType.SELECTED_ROWS, + True, + ) else: - x = core.eager.Tensor(core.VarDesc.VarType.FP32, [3, 100], - "selected_rows", - core.VarDesc.VarType.SELECTED_ROWS, True) + x = core.eager.Tensor( + core.VarDesc.VarType.FP32, + [3, 100], + "selected_rows", + core.VarDesc.VarType.SELECTED_ROWS, + True, + ) selected_rows = x.value().get_selected_rows() - selected_rows.get_tensor().set(np.random.rand(3, 100), - core.CPUPlace()) + selected_rows.get_tensor().set( + np.random.rand(3, 100), core.CPUPlace() + ) selected_rows.set_height(10) selected_rows.set_rows([3, 5, 7]) x_copy = copy.deepcopy(x) @@ -549,12 +568,14 @@ class TestVarBase(unittest.TestCase): self.assertEqual(x_copy.dtype, x.dtype) copy_selected_rows = x_copy.value().get_selected_rows() - self.assertEqual(copy_selected_rows.height(), - selected_rows.height()) + self.assertEqual( + copy_selected_rows.height(), selected_rows.height() + ) self.assertEqual(copy_selected_rows.rows(), selected_rows.rows()) np.testing.assert_array_equal( np.array(copy_selected_rows.get_tensor()), - np.array(selected_rows.get_tensor())) + np.array(selected_rows.get_tensor()), + ) def test_deep_copy(self): with _test_eager_guard(): @@ -658,8 +679,9 @@ class TestVarBase(unittest.TestCase): def func_test_block(self): with fluid.dygraph.guard(): var = fluid.dygraph.to_variable(self.array) - self.assertEqual(var.block, - fluid.default_main_program().global_block()) + self.assertEqual( + var.block, fluid.default_main_program().global_block() + ) def test_block(self): with _test_eager_guard(): @@ -668,7 +690,8 @@ class TestVarBase(unittest.TestCase): def _test_slice(self): w = fluid.dygraph.to_variable( - np.random.random((784, 100, 100)).astype('float64')) + np.random.random((784, 100, 100)).astype('float64') + ) for i in range(3): nw = w[i] @@ -691,10 +714,13 @@ class TestVarBase(unittest.TestCase): nw = w[:, :, :-1] self.assertEqual((784, 100, 99), tuple(nw.shape)) - tensor_array = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], [16, 17, 18]], - [[19, 20, 21], [22, 23, 24], - [25, 26, 27]]]).astype('float32') + tensor_array = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]], + ] + ).astype('float32') var = fluid.dygraph.to_variable(tensor_array) var1 = var[0, 1, 1] var2 = var[1:] @@ -717,8 +743,25 @@ class TestVarBase(unittest.TestCase): var18 = var[:, 1:1:2] vars = [ - var, var1, var2, var3, var4, var5, var6, var7, var8, var9, var10, - var11, var12, var13, var14, var15, var16, var17, var18 + var, + var1, + var2, + var3, + var4, + var5, + var6, + var7, + var8, + var9, + var10, + var11, + var12, + var13, + var14, + var15, + var16, + var17, + var18, ] local_out = [var.numpy() for var in vars] @@ -728,32 +771,41 @@ class TestVarBase(unittest.TestCase): np.testing.assert_array_equal(local_out[4], tensor_array[::-1]) np.testing.assert_array_equal(local_out[5], tensor_array[1, 1:, 1:]) np.testing.assert_array_equal( - local_out[6], - tensor_array.reshape((3, -1, 3))[:, :, -1]) + local_out[6], tensor_array.reshape((3, -1, 3))[:, :, -1] + ) np.testing.assert_array_equal(local_out[7], tensor_array[:, :, :-1]) np.testing.assert_array_equal(local_out[8], tensor_array[:1, :1, :1]) np.testing.assert_array_equal(local_out[9], tensor_array[:-1, :-1, :-1]) - np.testing.assert_array_equal(local_out[10], - tensor_array[::-1, :1, :-1]) - np.testing.assert_array_equal(local_out[11], tensor_array[:-1, ::-1, - -1:]) - np.testing.assert_array_equal(local_out[12], tensor_array[1:2, - 2:, ::-1]) - np.testing.assert_array_equal(local_out[13], tensor_array[2:10, 2:, - -2:-1]) - np.testing.assert_array_equal(local_out[14], tensor_array[1:-1, - 0:2, ::-1]) - np.testing.assert_array_equal(local_out[15], - tensor_array[::-1, ::-1, ::-1]) + np.testing.assert_array_equal( + local_out[10], tensor_array[::-1, :1, :-1] + ) + np.testing.assert_array_equal( + local_out[11], tensor_array[:-1, ::-1, -1:] + ) + np.testing.assert_array_equal( + local_out[12], tensor_array[1:2, 2:, ::-1] + ) + np.testing.assert_array_equal( + local_out[13], tensor_array[2:10, 2:, -2:-1] + ) + np.testing.assert_array_equal( + local_out[14], tensor_array[1:-1, 0:2, ::-1] + ) + np.testing.assert_array_equal( + local_out[15], tensor_array[::-1, ::-1, ::-1] + ) np.testing.assert_array_equal(local_out[16], tensor_array[-4:4]) np.testing.assert_array_equal(local_out[17], tensor_array[:, 0, 0:0]) np.testing.assert_array_equal(local_out[18], tensor_array[:, 1:1:2]) def _test_slice_for_tensor_attr(self): - tensor_array = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], [16, 17, 18]], - [[19, 20, 21], [22, 23, 24], - [25, 26, 27]]]).astype('float32') + tensor_array = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]], + ] + ).astype('float32') var = paddle.to_tensor(tensor_array) @@ -782,8 +834,23 @@ class TestVarBase(unittest.TestCase): var16 = var[-4:4] vars = [ - var, var1, var2, var3, var4, var5, var6, var7, var8, var9, var10, - var11, var12, var13, var14, var15, var16 + var, + var1, + var2, + var3, + var4, + var5, + var6, + var7, + var8, + var9, + var10, + var11, + var12, + var13, + var14, + var15, + var16, ] local_out = [var.numpy() for var in vars] @@ -793,23 +860,29 @@ class TestVarBase(unittest.TestCase): np.testing.assert_array_equal(local_out[4], tensor_array[::-1]) np.testing.assert_array_equal(local_out[5], tensor_array[1, 1:, 1:]) np.testing.assert_array_equal( - local_out[6], - tensor_array.reshape((3, -1, 3))[:, :, -1]) + local_out[6], tensor_array.reshape((3, -1, 3))[:, :, -1] + ) np.testing.assert_array_equal(local_out[7], tensor_array[:, :, :-1]) np.testing.assert_array_equal(local_out[8], tensor_array[:1, :1, :1]) np.testing.assert_array_equal(local_out[9], tensor_array[:-1, :-1, :-1]) - np.testing.assert_array_equal(local_out[10], - tensor_array[::-1, :1, :-1]) - np.testing.assert_array_equal(local_out[11], tensor_array[:-1, ::-1, - -1:]) - np.testing.assert_array_equal(local_out[12], tensor_array[1:2, - 2:, ::-1]) - np.testing.assert_array_equal(local_out[13], tensor_array[2:10, 2:, - -2:-1]) - np.testing.assert_array_equal(local_out[14], tensor_array[1:-1, - 0:2, ::-1]) - np.testing.assert_array_equal(local_out[15], - tensor_array[::-1, ::-1, ::-1]) + np.testing.assert_array_equal( + local_out[10], tensor_array[::-1, :1, :-1] + ) + np.testing.assert_array_equal( + local_out[11], tensor_array[:-1, ::-1, -1:] + ) + np.testing.assert_array_equal( + local_out[12], tensor_array[1:2, 2:, ::-1] + ) + np.testing.assert_array_equal( + local_out[13], tensor_array[2:10, 2:, -2:-1] + ) + np.testing.assert_array_equal( + local_out[14], tensor_array[1:-1, 0:2, ::-1] + ) + np.testing.assert_array_equal( + local_out[15], tensor_array[::-1, ::-1, ::-1] + ) np.testing.assert_array_equal(local_out[16], tensor_array[-4:4]) def _test_for_getitem_ellipsis_index(self): @@ -851,8 +924,9 @@ class TestVarBase(unittest.TestCase): # test 1 dim tensor var_one_dim = paddle.to_tensor([1, 2, 3, 4]) - np.testing.assert_array_equal(var_one_dim[..., 0].numpy(), - np.array([1])) + np.testing.assert_array_equal( + var_one_dim[..., 0].numpy(), np.array([1]) + ) def _test_none_index(self): shape = (8, 64, 5, 256) @@ -883,8 +957,9 @@ class TestVarBase(unittest.TestCase): np.testing.assert_array_equal(var[6], np_value[None, 2, None, 1]) np.testing.assert_array_equal(var[7], np_value[None]) np.testing.assert_array_equal(var[8], np_value[0, 0, None, 0, 0, None]) - np.testing.assert_array_equal(var[9], np_value[None, None, 0, ..., - None]) + np.testing.assert_array_equal( + var[9], np_value[None, None, 0, ..., None] + ) np.testing.assert_array_equal(var[10], np_value[..., None, :, None]) # TODO(zyfncg) there is a bug of dimensions when slice step > 1 and @@ -896,18 +971,25 @@ class TestVarBase(unittest.TestCase): shape = (4, 2, 5, 64) np_value = np.random.random(shape).astype('float32') var_tensor = paddle.to_tensor(np_value) - index = [[True, True, True, True], [True, False, True, True], - [True, False, False, True], [False, 0, 1, True, True], - [False, False, False, False]] - index2d = np.array([[True, True], [False, False], [True, False], - [True, True]]) + index = [ + [True, True, True, True], + [True, False, True, True], + [True, False, False, True], + [False, 0, 1, True, True], + [False, False, False, False], + ] + index2d = np.array( + [[True, True], [False, False], [True, False], [True, True]] + ) tensor_index = paddle.to_tensor(index2d) var = [ - var_tensor[index[0]].numpy(), var_tensor[index[1]].numpy(), - var_tensor[index[2]].numpy(), var_tensor[index[3]].numpy(), + var_tensor[index[0]].numpy(), + var_tensor[index[1]].numpy(), + var_tensor[index[2]].numpy(), + var_tensor[index[3]].numpy(), var_tensor[paddle.to_tensor(index[0])].numpy(), var_tensor[tensor_index].numpy(), - var_tensor[paddle.to_tensor(index[4])].numpy() + var_tensor[paddle.to_tensor(index[4])].numpy(), ] np.testing.assert_array_equal(var[0], np_value[index[0]]) np.testing.assert_array_equal(var[1], np_value[index[1]]) @@ -916,10 +998,12 @@ class TestVarBase(unittest.TestCase): np.testing.assert_array_equal(var[4], np_value[index[0]]) np.testing.assert_array_equal(var[5], np_value[index2d]) np.testing.assert_array_equal(var[6], np_value[index[4]]) - np.testing.assert_array_equal(var_tensor[var_tensor > 0.67], - np_value[np_value > 0.67]) - np.testing.assert_array_equal(var_tensor[var_tensor < 0.55], - np_value[np_value < 0.55]) + np.testing.assert_array_equal( + var_tensor[var_tensor > 0.67], np_value[np_value > 0.67] + ) + np.testing.assert_array_equal( + var_tensor[var_tensor < 0.55], np_value[np_value < 0.55] + ) with self.assertRaises(ValueError): var_tensor[[False, False, False, False]] @@ -953,13 +1037,16 @@ class TestVarBase(unittest.TestCase): t = paddle.to_tensor(array) np.testing.assert_array_equal(t[np.longlong(0)].numpy(), array[0]) np.testing.assert_array_equal( - t[np.longlong(0):np.longlong(4):np.longlong(2)].numpy(), - array[0:4:2]) + t[np.longlong(0) : np.longlong(4) : np.longlong(2)].numpy(), + array[0:4:2], + ) np.testing.assert_array_equal(t[np.int64(0)].numpy(), array[0]) np.testing.assert_array_equal( - t[np.int32(1):np.int32(4):np.int32(2)].numpy(), array[1:4:2]) + t[np.int32(1) : np.int32(4) : np.int32(2)].numpy(), array[1:4:2] + ) np.testing.assert_array_equal( - t[np.int16(0):np.int16(4):np.int16(2)].numpy(), array[0:4:2]) + t[np.int16(0) : np.int16(4) : np.int16(2)].numpy(), array[0:4:2] + ) def _test_list_index(self): # case1: @@ -971,13 +1058,18 @@ class TestVarBase(unittest.TestCase): np.testing.assert_array_equal(x[py_idx].numpy(), array[py_idx]) # case2: tensor_x = paddle.to_tensor( - np.zeros(12).reshape(2, 6).astype(np.float32)) + np.zeros(12).reshape(2, 6).astype(np.float32) + ) tensor_y1 = paddle.zeros([1], dtype='int32') + 2 tensor_y2 = paddle.zeros([1], dtype='int32') + 5 tensor_x[:, tensor_y1:tensor_y2] = 42 res = tensor_x.numpy() - exp = np.array([[0., 0., 42., 42., 42., 0.], - [0., 0., 42., 42., 42., 0.]]) + exp = np.array( + [ + [0.0, 0.0, 42.0, 42.0, 42.0, 0.0], + [0.0, 0.0, 42.0, 42.0, 42.0, 0.0], + ] + ) np.testing.assert_array_equal(res, exp) # case3: @@ -1019,8 +1111,9 @@ class TestVarBase(unittest.TestCase): def func_test_var_base_to_np(self): with fluid.dygraph.guard(): var = fluid.dygraph.to_variable(self.array) - np.testing.assert_array_equal(var.numpy(), - fluid.framework._var_base_to_np(var)) + np.testing.assert_array_equal( + var.numpy(), fluid.framework._var_base_to_np(var) + ) def test_var_base_to_np(self): with _test_eager_guard(): @@ -1031,8 +1124,9 @@ class TestVarBase(unittest.TestCase): with fluid.dygraph.guard(): var = fluid.dygraph.to_variable(self.array) np.testing.assert_array_equal(var.numpy(), np.array(var)) - np.testing.assert_array_equal(var.numpy(), - np.array(var, dtype=np.float32)) + np.testing.assert_array_equal( + var.numpy(), np.array(var, dtype=np.float32) + ) def test_var_base_as_np(self): with _test_eager_guard(): @@ -1081,7 +1175,9 @@ class TestVarBase(unittest.TestCase): param_attr=fluid.ParamAttr( learning_rate=0.001, do_model_average=True, - regularizer=fluid.regularizer.L1Decay())) + regularizer=fluid.regularizer.L1Decay(), + ), + ) weight = fc.parameters()[0] static_param = weight._to_static_var() self._assert_to_static(weight, static_param, True) @@ -1097,14 +1193,18 @@ class TestVarBase(unittest.TestCase): self.assertTrue(static_var.persistable, True) if isinstance(var_base, fluid.framework.ParamBase): for attr in ['trainable', 'is_distributed', 'do_model_average']: - self.assertEqual(getattr(var_base, attr), - getattr(static_var, attr)) + self.assertEqual( + getattr(var_base, attr), getattr(static_var, attr) + ) - self.assertEqual(static_var.optimize_attr['learning_rate'], - 0.001) + self.assertEqual( + static_var.optimize_attr['learning_rate'], 0.001 + ) self.assertTrue( - isinstance(static_var.regularizer, - fluid.regularizer.L1Decay)) + isinstance( + static_var.regularizer, fluid.regularizer.L1Decay + ) + ) else: self.assertTrue(isinstance(static_var, fluid.framework.Variable)) @@ -1205,10 +1305,9 @@ class TestVarBase(unittest.TestCase): paddle.disable_static(paddle.CPUPlace()) paddle.seed(2021) x = paddle.rand([128]) - paddle.set_printoptions(precision=4, - threshold=1000, - edgeitems=3, - linewidth=80) + paddle.set_printoptions( + precision=4, threshold=1000, edgeitems=3, linewidth=80 + ) a_str = str(x) expected = '''Tensor(shape=[128], dtype=float32, place=Place(cpu), stop_gradient=True, @@ -1301,7 +1400,6 @@ class TestVarBase(unittest.TestCase): class TestVarBaseSetitem(unittest.TestCase): - def func_setUp(self): self.set_dtype() self.tensor_x = paddle.to_tensor(np.ones((4, 2, 3)).astype(self.dtype)) @@ -1373,13 +1471,11 @@ class TestVarBaseSetitem(unittest.TestCase): class TestVarBaseSetitemInt64(TestVarBaseSetitem): - def set_dtype(self): self.dtype = "int64" class TestVarBaseSetitemFp32(TestVarBaseSetitem): - def set_dtype(self): self.dtype = "float32" @@ -1396,13 +1492,11 @@ class TestVarBaseSetitemFp32(TestVarBaseSetitem): class TestVarBaseSetitemFp64(TestVarBaseSetitem): - def set_dtype(self): self.dtype = "float64" class TestVarBaseSetitemBoolIndex(unittest.TestCase): - def func_setUp(self): paddle.disable_static() self.set_dtype() @@ -1481,7 +1575,6 @@ class TestVarBaseSetitemBoolIndex(unittest.TestCase): class TestVarBaseSetitemBoolScalarIndex(unittest.TestCase): - def set_input(self): self.tensor_x = paddle.to_tensor(np.ones((1, 2, 3)).astype(self.dtype)) self.np_value = np.random.random((2, 3)).astype(self.dtype) @@ -1507,7 +1600,6 @@ class TestVarBaseSetitemBoolScalarIndex(unittest.TestCase): class TestVarBaseInplaceVersion(unittest.TestCase): - def func_test_setitem(self): paddle.disable_static() @@ -1543,7 +1635,6 @@ class TestVarBaseInplaceVersion(unittest.TestCase): class TestVarBaseSlice(unittest.TestCase): - def func_test_slice(self): paddle.disable_static() np_x = np.random.random((3, 8, 8)) @@ -1559,7 +1650,6 @@ class TestVarBaseSlice(unittest.TestCase): class TestVarBaseClear(unittest.TestCase): - def func_test_clear(self): paddle.disable_static() np_x = np.random.random((3, 8, 8)) @@ -1574,7 +1664,6 @@ class TestVarBaseClear(unittest.TestCase): class TestVarBaseOffset(unittest.TestCase): - def func_offset(self): paddle.disable_static() np_x = np.random.random((3, 8, 8)) @@ -1591,7 +1680,6 @@ class TestVarBaseOffset(unittest.TestCase): class TestVarBaseShareBufferTo(unittest.TestCase): - def func_test_share_buffer_To(self): paddle.disable_static() np_src = np.random.random((3, 8, 8)) @@ -1611,7 +1699,6 @@ class TestVarBaseShareBufferTo(unittest.TestCase): class TestVarBaseTo(unittest.TestCase): - def func_setUp(self): paddle.disable_static() self.np_x = np.random.random((3, 8, 8)) @@ -1638,14 +1725,16 @@ class TestVarBaseTo(unittest.TestCase): x_gpu1 = self.x._to(device='gpu:0', dtype="float64") self.assertTrue(x_gpu1.place.is_gpu_place()) self.assertEqual(x_gpu1.place.gpu_device_id(), 0) - self.assertEqual(x_gpu1.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + x_gpu1.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) x_gpu2 = self.x._to(device='gpu:0', dtype="float16") self.assertTrue(x_gpu2.place.is_gpu_place()) self.assertEqual(x_gpu2.place.gpu_device_id(), 0) - self.assertEqual(x_gpu2.dtype, - paddle.fluid.core.VarDesc.VarType.FP16) + self.assertEqual( + x_gpu2.dtype, paddle.fluid.core.VarDesc.VarType.FP16 + ) x_cpu = self.x._to(device=paddle.CPUPlace()) self.assertTrue(x_cpu.place.is_cpu_place()) @@ -1673,7 +1762,6 @@ class TestVarBaseTo(unittest.TestCase): class TestVarBaseInitVarBaseFromTensorWithDevice(unittest.TestCase): - def func_test_varbase_init(self): paddle.disable_static() t = fluid.Tensor() @@ -1703,7 +1791,6 @@ class TestVarBaseInitVarBaseFromTensorWithDevice(unittest.TestCase): class TestVarBaseNumel(unittest.TestCase): - def func_test_numel_normal(self): paddle.disable_static() np_x = np.random.random((3, 8, 8)) @@ -1733,7 +1820,6 @@ class TestVarBaseNumel(unittest.TestCase): class TestVarBaseCopyGradientFrom(unittest.TestCase): - def func_test_copy_gradient_from(self): paddle.disable_static() np_x = np.random.random((2, 2)) @@ -1752,7 +1838,6 @@ class TestVarBaseCopyGradientFrom(unittest.TestCase): class TestEagerTensorGradNameValue(unittest.TestCase): - def test_eager_tensor_grad_name_value(self): with _test_eager_guard(): a_np = np.array([2, 3]).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_var_conv_2d.py b/python/paddle/fluid/tests/unittests/test_var_conv_2d.py index 9836ea72aab..ef313a13944 100644 --- a/python/paddle/fluid/tests/unittests/test_var_conv_2d.py +++ b/python/paddle/fluid/tests/unittests/test_var_conv_2d.py @@ -18,7 +18,6 @@ from op_test import OpTest, skip_check_grad_ci class TestVarConv2DOp(OpTest): - def setUp(self): self.init_op_type() self.set_data() @@ -34,11 +33,13 @@ class TestVarConv2DOp(OpTest): stride = [1, 1] row = [2, 4] col = [3, 2] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) - def init_data(self, input_channel, output_channel, filter_size, stride, row, - col): + def init_data( + self, input_channel, output_channel, filter_size, stride, row, col + ): feature = [row[i] * col[i] for i in range(len(row))] numel = sum(feature) * input_channel @@ -46,14 +47,16 @@ class TestVarConv2DOp(OpTest): x_lod = [[x * input_channel for x in feature]] row_data = np.random.random((sum(row), 10)).astype('float32') col_data = np.random.random((sum(col), 10)).astype('float32') - w_shape = (output_channel, - input_channel * filter_size[0] * filter_size[1]) + w_shape = ( + output_channel, + input_channel * filter_size[0] * filter_size[1], + ) w_data = np.random.random(w_shape).astype('float32') self.inputs = { 'X': (x_data, x_lod), 'ROW': (row_data, [row]), 'COLUMN': (col_data, [col]), - 'W': w_data + 'W': w_data, } self.attrs = { 'InputChannel': input_channel, @@ -95,8 +98,9 @@ class TestVarConv2DOp(OpTest): if top_im_size == 0: out_tmp = np.zeros((out_ch * top_im_size, 1)).astype('float32') else: - col_batch_data = col_res_data[col_data_offset:col_data_offset + - col_res_lod[0][idx]] + col_batch_data = col_res_data[ + col_data_offset : col_data_offset + col_res_lod[0][idx] + ] gemm_shape = (in_ch * kernel_h * kernel_w, top_im_size) col_batch_data = col_batch_data.reshape(gemm_shape) out_tmp = np.dot(w_data, col_batch_data).reshape(-1, 1) @@ -106,7 +110,7 @@ class TestVarConv2DOp(OpTest): self.outputs = { 'Out': (out_data.astype('float32'), out_lod), - 'Col': (col_res_data, col_res_lod) + 'Col': (col_res_data, col_res_lod), } def Im2Col(self): @@ -159,11 +163,23 @@ class TestVarConv2DOp(OpTest): for kx in range(kernel_w): im_y = y + ky - half_kernel_h im_x = x + kx - half_kernel_w - if im_x >= 0 and im_x < width and im_y >= 0 and im_y < height: - col_res[t_offset + - (row_offset + ky * kernel_w + kx) * top_x + - col_offset] = \ - x_data[b_offset + im_offset + im_y * width + im_x] + if ( + im_x >= 0 + and im_x < width + and im_y >= 0 + and im_y < height + ): + col_res[ + t_offset + + (row_offset + ky * kernel_w + kx) + * top_x + + col_offset + ] = x_data[ + b_offset + + im_offset + + im_y * width + + im_x + ] t_offset += col_res_lod[0][idx] b_offset += x_lod[0][idx] @@ -174,14 +190,12 @@ class TestVarConv2DOp(OpTest): self.check_output(check_dygraph=False) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - check_dygraph=False) + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, check_dygraph=False + ) class TestVarConv2DOpCase1(TestVarConv2DOp): - def set_data(self): # set in_ch 1 input_channel = 1 @@ -190,12 +204,12 @@ class TestVarConv2DOpCase1(TestVarConv2DOp): stride = [1, 1] row = [1, 10] col = [40, 6] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase2(TestVarConv2DOp): - def set_data(self): # set out_ch 1 input_channel = 2 @@ -204,12 +218,12 @@ class TestVarConv2DOpCase2(TestVarConv2DOp): stride = [2, 2] row = [6, 7] col = [8, 2] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase3(TestVarConv2DOp): - def set_data(self): # set batch 1 input_channel = 2 @@ -218,12 +232,12 @@ class TestVarConv2DOpCase3(TestVarConv2DOp): stride = [2, 2] row = [14] col = [4] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase4(TestVarConv2DOp): - def set_data(self): # set filter size very large input_channel = 3 @@ -232,12 +246,12 @@ class TestVarConv2DOpCase4(TestVarConv2DOp): stride = [2, 2] row = [4, 7] col = [5, 2] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase5(TestVarConv2DOp): - def set_data(self): # set input very small input_channel = 50 @@ -246,16 +260,15 @@ class TestVarConv2DOpCase5(TestVarConv2DOp): stride = [1, 1] row = [1, 1] col = [1, 1] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) @skip_check_grad_ci( - reason= - "[skip shape check] Use shape of input_channel, row and col all is 1 to test special LoDTensor." + reason="[skip shape check] Use shape of input_channel, row and col all is 1 to test special LoDTensor." ) class TestVarConv2DOpCase6(TestVarConv2DOp): - def set_data(self): input_channel = 1 output_channel = 3 @@ -263,12 +276,12 @@ class TestVarConv2DOpCase6(TestVarConv2DOp): stride = [1, 1] row = [1, 1] col = [1, 1] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase7(TestVarConv2DOp): - def set_data(self): input_channel = 2 output_channel = 3 @@ -276,43 +289,46 @@ class TestVarConv2DOpCase7(TestVarConv2DOp): stride = [1, 1] row = [5, 4] col = [6, 7] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DApi(unittest.TestCase): - def test_api(self): import paddle.fluid as fluid x = fluid.layers.data(name='x', shape=[1], lod_level=1) row = fluid.layers.data(name='row', shape=[6], lod_level=1) col = fluid.layers.data(name='col', shape=[6], lod_level=1) - out = fluid.contrib.var_conv_2d(input=x, - row=row, - col=col, - input_channel=3, - output_channel=5, - filter_size=[3, 3], - stride=1) + out = fluid.contrib.var_conv_2d( + input=x, + row=row, + col=col, + input_channel=3, + output_channel=5, + filter_size=[3, 3], + stride=1, + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(116, 1).astype('float32'), [[60, 56]], place) + np.random.rand(116, 1).astype('float32'), [[60, 56]], place + ) row_tensor = fluid.create_lod_tensor( - np.random.rand(9, 6).astype('float32'), [[5, 4]], place) + np.random.rand(9, 6).astype('float32'), [[5, 4]], place + ) col_tensor = fluid.create_lod_tensor( - np.random.rand(13, 6).astype('float32'), [[6, 7]], place) + np.random.rand(13, 6).astype('float32'), [[6, 7]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_tensor, - 'row': row_tensor, - 'col': col_tensor - }, - fetch_list=[out], - return_numpy=False) + ret = exe.run( + feed={'x': x_tensor, 'row': row_tensor, 'col': col_tensor}, + fetch_list=[out], + return_numpy=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_var_info.py b/python/paddle/fluid/tests/unittests/test_var_info.py index 7af77ae96b5..4bb6648488e 100644 --- a/python/paddle/fluid/tests/unittests/test_var_info.py +++ b/python/paddle/fluid/tests/unittests/test_var_info.py @@ -22,10 +22,10 @@ import unittest class TestVarInfo(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def test_var_info(self): - """ Testcase for get and set info for variable. """ + """Testcase for get and set info for variable.""" value = np.random.randn(1) var = fluid.layers.create_global_var([1], value, "float32") var._set_info("name", "test") diff --git a/python/paddle/fluid/tests/unittests/test_variable.py b/python/paddle/fluid/tests/unittests/test_variable.py index 8ab9d14c924..bf9ee54afc7 100644 --- a/python/paddle/fluid/tests/unittests/test_variable.py +++ b/python/paddle/fluid/tests/unittests/test_variable.py @@ -16,7 +16,11 @@ import unittest from functools import reduce import paddle -from paddle.fluid.framework import Program, convert_np_dtype_to_dtype_, default_main_program +from paddle.fluid.framework import ( + Program, + convert_np_dtype_to_dtype_, + default_main_program, +) import paddle import paddle.fluid as fluid import paddle.fluid.core as core @@ -26,7 +30,6 @@ paddle.enable_static() class TestVariable(unittest.TestCase): - def setUp(self): np.random.seed(2022) @@ -45,10 +48,9 @@ class TestVariable(unittest.TestCase): def test_var(self): b = default_main_program().current_block() - w = b.create_var(dtype="float64", - shape=[784, 100], - lod_level=0, - name="fc.w") + w = b.create_var( + dtype="float64", shape=[784, 100], lod_level=0, name="fc.w" + ) self.assertNotEqual(str(w), "") self.assertEqual(core.VarDesc.VarType.FP64, w.dtype) self.assertEqual((784, 100), w.shape) @@ -63,12 +65,15 @@ class TestVariable(unittest.TestCase): self.assertEqual("fc.w@GRAD", w.grad_name) self.assertEqual(0, w.lod_level) - self.assertRaises(ValueError, - lambda: b.create_var(name="fc.w", shape=(24, 100))) + self.assertRaises( + ValueError, lambda: b.create_var(name="fc.w", shape=(24, 100)) + ) - w = b.create_var(dtype=paddle.fluid.core.VarDesc.VarType.STRINGS, - shape=[1], - name="str_var") + w = b.create_var( + dtype=paddle.fluid.core.VarDesc.VarType.STRINGS, + shape=[1], + name="str_var", + ) self.assertEqual(None, w.lod_level) def test_element_size(self): @@ -103,8 +108,9 @@ class TestVariable(unittest.TestCase): def test_step_scopes(self): prog = Program() b = prog.current_block() - var = b.create_var(name='step_scopes', - type=core.VarDesc.VarType.STEP_SCOPES) + var = b.create_var( + name='step_scopes', type=core.VarDesc.VarType.STEP_SCOPES + ) self.assertEqual(core.VarDesc.VarType.STEP_SCOPES, var.type) def _test_slice(self, place): @@ -137,10 +143,13 @@ class TestVariable(unittest.TestCase): main = fluid.Program() with fluid.program_guard(main): exe = fluid.Executor(place) - tensor_array = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], [16, 17, 18]], - [[19, 20, 21], [22, 23, 24], - [25, 26, 27]]]).astype('float32') + tensor_array = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]], + ] + ).astype('float32') var = fluid.layers.assign(tensor_array) var1 = var[0, 1, 1] var2 = var[1:] @@ -167,13 +176,28 @@ class TestVariable(unittest.TestCase): data.append((np.random.randint(10, size=[13]).astype('float32'))) exe.run(fluid.default_startup_program()) - local_out = exe.run(main, - feed=feeder.feed([data]), - fetch_list=[ - var, var1, var2, var3, var4, var5, var6, - var7, var8, var9, var10, var11, var12, - var13, var14, var15 - ]) + local_out = exe.run( + main, + feed=feeder.feed([data]), + fetch_list=[ + var, + var1, + var2, + var3, + var4, + var5, + var6, + var7, + var8, + var9, + var10, + var11, + var12, + var13, + var14, + var15, + ], + ) np.testing.assert_array_equal(local_out[1], tensor_array[0, 1, 1:2]) np.testing.assert_array_equal(local_out[2], tensor_array[1:]) @@ -181,25 +205,33 @@ class TestVariable(unittest.TestCase): np.testing.assert_array_equal(local_out[4], tensor_array[::-1]) np.testing.assert_array_equal(local_out[5], tensor_array[1, 1:, 1:]) np.testing.assert_array_equal( - local_out[6], - tensor_array.reshape((3, -1, 3))[:, :, -1]) + local_out[6], tensor_array.reshape((3, -1, 3))[:, :, -1] + ) np.testing.assert_array_equal(local_out[7], tensor_array[:, :, :-1]) - np.testing.assert_array_equal(local_out[8], - tensor_array[:1, :1, :1]) - np.testing.assert_array_equal(local_out[9], - tensor_array[:-1, :-1, :-1]) - np.testing.assert_array_equal(local_out[10], - tensor_array[::-1, :1, :-1]) - np.testing.assert_array_equal(local_out[11], tensor_array[:-1, ::-1, - -1:]) - np.testing.assert_array_equal(local_out[12], tensor_array[1:2, - 2:, ::-1]) - np.testing.assert_array_equal(local_out[13], tensor_array[2:10, 2:, - -2:-1]) - np.testing.assert_array_equal(local_out[14], - tensor_array[1:-1, 0:2, ::-1]) - np.testing.assert_array_equal(local_out[15], - tensor_array[::-1, ::-1, ::-1]) + np.testing.assert_array_equal( + local_out[8], tensor_array[:1, :1, :1] + ) + np.testing.assert_array_equal( + local_out[9], tensor_array[:-1, :-1, :-1] + ) + np.testing.assert_array_equal( + local_out[10], tensor_array[::-1, :1, :-1] + ) + np.testing.assert_array_equal( + local_out[11], tensor_array[:-1, ::-1, -1:] + ) + np.testing.assert_array_equal( + local_out[12], tensor_array[1:2, 2:, ::-1] + ) + np.testing.assert_array_equal( + local_out[13], tensor_array[2:10, 2:, -2:-1] + ) + np.testing.assert_array_equal( + local_out[14], tensor_array[1:-1, 0:2, ::-1] + ) + np.testing.assert_array_equal( + local_out[15], tensor_array[::-1, ::-1, ::-1] + ) def _test_slice_index_tensor(self, place): data = np.random.rand(2, 3).astype("float32") @@ -270,13 +302,18 @@ class TestVariable(unittest.TestCase): out7 = y[..., 0] exe = paddle.static.Executor(place) - result = exe.run(prog, - fetch_list=[out1, out2, out3, out4, out5, out6, out7]) + result = exe.run( + prog, fetch_list=[out1, out2, out3, out4, out5, out6, out7] + ) expected = [ - data[0:, ..., 1:], data[0:, ...], data[..., 1:], data[...], - data[[1, 0], [0, 0]], data[([1, 0], [0, 0])], - np.array([1]) + data[0:, ..., 1:], + data[0:, ...], + data[..., 1:], + data[...], + data[[1, 0], [0, 0]], + data[([1, 0], [0, 0])], + np.array([1]), ] self.assertTrue((result[0] == expected[0]).all()) @@ -314,11 +351,18 @@ class TestVariable(unittest.TestCase): exe = paddle.static.Executor(place) result = exe.run( - prog, fetch_list=[out0, out1, out2, out3, out4, out5, out6, out7]) + prog, fetch_list=[out0, out1, out2, out3, out4, out5, out6, out7] + ) expected = [ - data[idx0], data[idx1], data[idx2], data[idx3], data[idx4], - data[np_idx], data[data < 0.36], data[data > 0.6] + data[idx0], + data[idx1], + data[idx2], + data[idx3], + data[idx4], + data[np_idx], + data[data < 0.36], + data[data > 0.6], ] self.assertTrue((result[0] == expected[0]).all()) @@ -413,11 +457,13 @@ class TestVariable(unittest.TestCase): def test_create_selected_rows(self): b = default_main_program().current_block() - var = b.create_var(name="var", - shape=[1, 1], - dtype="float32", - type=fluid.core.VarDesc.VarType.SELECTED_ROWS, - persistable=True) + var = b.create_var( + name="var", + shape=[1, 1], + dtype="float32", + type=fluid.core.VarDesc.VarType.SELECTED_ROWS, + persistable=True, + ) def _test(): var.lod_level() @@ -452,17 +498,17 @@ class TestVariable(unittest.TestCase): scope = fluid.core.Scope() with paddle.static.scope_guard(scope): with paddle.static.program_guard(main, startup): - x = paddle.static.data(name='x', - shape=[3, 2, 1], - dtype='float32') + x = paddle.static.data( + name='x', shape=[3, 2, 1], dtype='float32' + ) x.persistable = True feed_data = np.ones(shape=[3, 2, 1], dtype=np.float32) detach_x = x.detach() exe = paddle.static.Executor(paddle.CPUPlace()) exe.run(startup) - result = exe.run(main, - feed={'x': feed_data}, - fetch_list=[x, detach_x]) + result = exe.run( + main, feed={'x': feed_data}, fetch_list=[x, detach_x] + ) self.assertTrue((result[1] == feed_data).all()) self.assertTrue((result[0] == result[1]).all()) @@ -472,9 +518,9 @@ class TestVariable(unittest.TestCase): self.assertTrue((result[1] == modified_value).all()) self.assertTrue((result[0] == result[1]).all()) - modified_value = np.random.uniform(-1, 1, - size=[3, 2, - 1]).astype('float32') + modified_value = np.random.uniform( + -1, 1, size=[3, 2, 1] + ).astype('float32') x.set_value(modified_value, scope) result = exe.run(main, fetch_list=[x, detach_x]) self.assertTrue((result[1] == modified_value).all()) @@ -482,7 +528,6 @@ class TestVariable(unittest.TestCase): class TestVariableSlice(unittest.TestCase): - def setUp(self): np.random.seed(2022) @@ -502,8 +547,11 @@ class TestVariableSlice(unittest.TestCase): result = exe.run(prog, fetch_list=outs) expected = [ - data[0:, None, 1:], data[0:, None], data[None, 1:], data[None], - data[..., None, :, None] + data[0:, None, 1:], + data[0:, None], + data[None, 1:], + data[None], + data[..., None, :, None], ] for i in range(len(outs)): self.assertEqual(outs[i].shape, expected[i].shape) @@ -525,8 +573,12 @@ class TestVariableSlice(unittest.TestCase): exe = paddle.static.Executor(place) result = exe.run(prog, fetch_list=outs) expected = [ - data[0, 1:, None], data[0, None], data[None, 1], data[None], - data[0, 0, 0, None], data[None, 0, 0, 0, None] + data[0, 1:, None], + data[0, None], + data[None, 1], + data[None], + data[0, 0, 0, None], + data[None, 0, 0, 0, None], ] for i in range(len(outs)): @@ -544,7 +596,6 @@ class TestVariableSlice(unittest.TestCase): class TestListIndex(unittest.TestCase): - def setUp(self): np.random.seed(2022) @@ -555,8 +606,9 @@ class TestListIndex(unittest.TestCase): paddle.enable_static() inps_shape = [3, 4, 5, 2] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [3, 3, 2, 1] index = np.arange(self.numel(index_shape)).reshape(index_shape) @@ -567,15 +619,17 @@ class TestListIndex(unittest.TestCase): index_mod = (index % (array.shape[0])).tolist() with paddle.static.program_guard(program): - x = paddle.static.data(name='x', - shape=array.shape, - dtype='float32') + x = paddle.static.data( + name='x', shape=array.shape, dtype='float32' + ) y = x[index_mod] - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) @@ -584,9 +638,9 @@ class TestListIndex(unittest.TestCase): fetch_list = [y.name] getitem_np = array[index_mod] - getitem_pp = exe.run(prog, - feed={x.name: array}, - fetch_list=fetch_list) + getitem_pp = exe.run( + prog, feed={x.name: array}, fetch_list=fetch_list + ) np.testing.assert_array_equal(getitem_np, getitem_pp[0]) array = array[0] @@ -622,16 +676,21 @@ class TestListIndex(unittest.TestCase): def test_static_graph_list_index_muti_dim(self): paddle.enable_static() inps_shape = [3, 4, 5] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 2] index1 = np.arange(self.numel(index_shape)).reshape(index_shape) index2 = np.arange(self.numel(index_shape)).reshape(index_shape) + 2 value_shape = [3, 2, 2, 3] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) index_mod1 = (index1 % (min(array.shape))).tolist() index_mod2 = (index2 % (min(array.shape))).tolist() @@ -641,21 +700,23 @@ class TestListIndex(unittest.TestCase): x = paddle.static.data(name='x', shape=array.shape, dtype='float32') - value = paddle.static.data(name='value', - shape=value_np.shape, - dtype='float32') - index1 = paddle.static.data(name='index1', - shape=index1.shape, - dtype='int32') - index2 = paddle.static.data(name='index2', - shape=index2.shape, - dtype='int32') + value = paddle.static.data( + name='value', shape=value_np.shape, dtype='float32' + ) + index1 = paddle.static.data( + name='index1', shape=index1.shape, dtype='int32' + ) + index2 = paddle.static.data( + name='index2', shape=index2.shape, dtype='int32' + ) y = x[index1, index2] - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) @@ -666,32 +727,40 @@ class TestListIndex(unittest.TestCase): y2 = array2[index_mod1, index_mod2] - getitem_pp = exe.run(prog, - feed={ - x.name: array, - index1.name: index_mod1, - index2.name: index_mod2 - }, - fetch_list=fetch_list) + getitem_pp = exe.run( + prog, + feed={ + x.name: array, + index1.name: index_mod1, + index2.name: index_mod2, + }, + fetch_list=fetch_list, + ) np.testing.assert_array_equal( y2, getitem_pp[0], - err_msg='\n numpy:{},\n paddle:{}'.format(y2, getitem_pp[0])) + err_msg='\n numpy:{},\n paddle:{}'.format(y2, getitem_pp[0]), + ) def test_dygraph_list_index_muti_dim(self): paddle.disable_static() inps_shape = [3, 4, 5] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 2] index1 = np.arange(self.numel(index_shape)).reshape(index_shape) index2 = np.arange(self.numel(index_shape)).reshape(index_shape) + 2 value_shape = [3, 2, 2, 3] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) index_mod1 = (index1 % (min(array.shape))).tolist() index_mod2 = (index2 % (min(array.shape))).tolist() @@ -721,16 +790,15 @@ class TestListIndex(unittest.TestCase): value_np = array2[index] except: with self.assertRaises(ValueError): - getitem_pp = exe.run(prog, - feed={x.name: array}, - fetch_list=fetch_list) + getitem_pp = exe.run( + prog, feed={x.name: array}, fetch_list=fetch_list + ) return getitem_pp = exe.run(prog, feed={x.name: array}, fetch_list=fetch_list) - np.testing.assert_allclose(value_np, - getitem_pp[0], - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + value_np, getitem_pp[0], rtol=1e-5, atol=1e-8 + ) def test_static_graph_getitem_bool_index(self): paddle.enable_static() @@ -762,9 +830,9 @@ class TestListIndex(unittest.TestCase): def run_setitem_list_index(self, array, index, value_np): x = paddle.static.data(name='x', shape=array.shape, dtype='float32') - value = paddle.static.data(name='value', - shape=value_np.shape, - dtype='float32') + value = paddle.static.data( + name='value', shape=value_np.shape, dtype='float32' + ) x[index] = value y = x @@ -781,19 +849,17 @@ class TestListIndex(unittest.TestCase): array2[index] = value_np except: with self.assertRaises(ValueError): - setitem_pp = exe.run(prog, - feed={ - x.name: array, - value.name: value_np - }, - fetch_list=fetch_list) + setitem_pp = exe.run( + prog, + feed={x.name: array, value.name: value_np}, + fetch_list=fetch_list, + ) return - setitem_pp = exe.run(prog, - feed={ - x.name: array, - value.name: value_np - }, - fetch_list=fetch_list) + setitem_pp = exe.run( + prog, + feed={x.name: array, value.name: value_np}, + fetch_list=fetch_list, + ) np.testing.assert_allclose(array2, setitem_pp[0], rtol=1e-5, atol=1e-8) @@ -801,15 +867,20 @@ class TestListIndex(unittest.TestCase): paddle.enable_static() # case 1: inps_shape = [3, 4, 5, 2, 3] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [3, 3, 1, 2] index = np.arange(self.numel(index_shape)).reshape(index_shape) value_shape = inps_shape[3:] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) for _ in range(3): program = paddle.static.Program() @@ -824,15 +895,20 @@ class TestListIndex(unittest.TestCase): # case 2: inps_shape = [3, 4, 5, 4, 3] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [4, 3, 2, 2] index = np.arange(self.numel(index_shape)).reshape(index_shape) value_shape = [3] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) for _ in range(4): program = paddle.static.Program() @@ -846,15 +922,20 @@ class TestListIndex(unittest.TestCase): # case 3: inps_shape = [3, 4, 5, 3, 3] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [4, 3, 2, 2] index = np.arange(self.numel(index_shape)).reshape(index_shape) value_shape = [3, 2, 2, 3] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) index_mod = (index % (min(array.shape))).tolist() self.run_setitem_list_index(array, index_mod, value_np) @@ -897,18 +978,28 @@ class TestListIndex(unittest.TestCase): def test_static_graph_tensor_index_setitem_muti_dim(self): paddle.enable_static() inps_shape = [3, 4, 5, 4] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 3, 4] - index1 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) - index2 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) + 2 + index1 = np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + index2 = ( + np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + + 2 + ) value_shape = [4] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) for _ in range(3): index_mod1 = index1 % (min(array.shape)) @@ -922,29 +1013,31 @@ class TestListIndex(unittest.TestCase): program = paddle.static.Program() with paddle.static.program_guard(program): - x1 = paddle.static.data(name='x1', - shape=array.shape, - dtype='float32') - x2 = paddle.static.data(name='x2', - shape=array.shape, - dtype='float32') - - value = paddle.static.data(name='value', - shape=value_np.shape, - dtype='float32') - index_1 = paddle.static.data(name='index_1', - shape=index1.shape, - dtype='int32') - index_2 = paddle.static.data(name='index_2', - shape=index2.shape, - dtype='int32') + x1 = paddle.static.data( + name='x1', shape=array.shape, dtype='float32' + ) + x2 = paddle.static.data( + name='x2', shape=array.shape, dtype='float32' + ) + + value = paddle.static.data( + name='value', shape=value_np.shape, dtype='float32' + ) + index_1 = paddle.static.data( + name='index_1', shape=index1.shape, dtype='int32' + ) + index_2 = paddle.static.data( + name='index_2', shape=index2.shape, dtype='int32' + ) x1[index_1, index_2] = value x2[index_1] = value - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) @@ -952,25 +1045,31 @@ class TestListIndex(unittest.TestCase): exe.run(paddle.static.default_startup_program()) fetch_list = [x1.name, x2.name] - setitem_pp = exe.run(prog, - feed={ - x1.name: array, - x2.name: array, - value.name: value_np, - index_1.name: index_mod1, - index_2.name: index_mod2 - }, - fetch_list=fetch_list) + setitem_pp = exe.run( + prog, + feed={ + x1.name: array, + x2.name: array, + value.name: value_np, + index_1.name: index_mod1, + index_2.name: index_mod2, + }, + fetch_list=fetch_list, + ) np.testing.assert_array_equal( array2, setitem_pp[0], err_msg='\n numpy:{},\n paddle:{}'.format( - array2, setitem_pp[0])) + array2, setitem_pp[0] + ), + ) np.testing.assert_array_equal( array3, setitem_pp[1], err_msg='\n numpy:{},\n paddle:{}'.format( - array3, setitem_pp[1])) + array3, setitem_pp[1] + ), + ) array = array[0] index1 = index1[0] index2 = index2[0] @@ -978,14 +1077,20 @@ class TestListIndex(unittest.TestCase): def test_static_graph_array_index_muti_dim(self): paddle.enable_static() inps_shape = [3, 4, 5, 4] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 3, 4] - index1 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) - index2 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) + 2 + index1 = np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + index2 = ( + np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + + 2 + ) for _ in range(3): index_mod1 = index1 % (min(array.shape)) @@ -1001,53 +1106,62 @@ class TestListIndex(unittest.TestCase): program = paddle.static.Program() with paddle.static.program_guard(program): - x1 = paddle.static.data(name='x1', - shape=array.shape, - dtype='float32') - x2 = paddle.static.data(name='x2', - shape=array.shape, - dtype='float32') + x1 = paddle.static.data( + name='x1', shape=array.shape, dtype='float32' + ) + x2 = paddle.static.data( + name='x2', shape=array.shape, dtype='float32' + ) x1[index_mod1, index_mod2] = 1 x2[index_mod1] = 2.5 y1 = x1[index_mod2, index_mod1] y2 = x2[index_mod2] - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) fetch_list = [x1.name, x2.name, y1.name, y2.name] - setitem_pp = exe.run(prog, - feed={ - x1.name: array, - x2.name: array - }, - fetch_list=fetch_list) + setitem_pp = exe.run( + prog, + feed={x1.name: array, x2.name: array}, + fetch_list=fetch_list, + ) np.testing.assert_array_equal( array2, setitem_pp[0], err_msg='\n numpy:{},\n paddle:{}'.format( - array2, setitem_pp[0])) + array2, setitem_pp[0] + ), + ) np.testing.assert_array_equal( array3, setitem_pp[1], err_msg='\n numpy:{},\n paddle:{}'.format( - array3, setitem_pp[1])) + array3, setitem_pp[1] + ), + ) np.testing.assert_array_equal( y_np1, setitem_pp[2], err_msg='\n numpy:{},\n paddle:{}'.format( - y_np1, setitem_pp[2])) + y_np1, setitem_pp[2] + ), + ) np.testing.assert_array_equal( y_np2, setitem_pp[3], err_msg='\n numpy:{},\n paddle:{}'.format( - y_np2, setitem_pp[3])) + y_np2, setitem_pp[3] + ), + ) array = array[0] index1 = index1[0] index2 = index2[0] @@ -1055,13 +1169,19 @@ class TestListIndex(unittest.TestCase): def test_dygraph_array_index_muti_dim(self): paddle.disable_static() inps_shape = [3, 4, 5, 4] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 3, 4] - index1 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) - index2 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) + 2 + index1 = np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + index2 = ( + np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + + 2 + ) for _ in range(3): @@ -1079,7 +1199,8 @@ class TestListIndex(unittest.TestCase): np.testing.assert_array_equal( y_t1.numpy(), y_np1, - err_msg='\n numpy:{},\n paddle:{}'.format(y_np1, y_t1.numpy())) + err_msg='\n numpy:{},\n paddle:{}'.format(y_np1, y_t1.numpy()), + ) # 1 dim getitem array2 = array.copy() y_np2 = array2[index_mod2] @@ -1089,7 +1210,8 @@ class TestListIndex(unittest.TestCase): np.testing.assert_array_equal( y_t2.numpy(), y_np2, - err_msg='\n numpy:{},\n paddle:{}'.format(y_np2, y_t2.numpy())) + err_msg='\n numpy:{},\n paddle:{}'.format(y_np2, y_t2.numpy()), + ) # 2 dim setitem array1 = array.copy() @@ -1099,7 +1221,9 @@ class TestListIndex(unittest.TestCase): tensor1.numpy(), array1, err_msg='\n numpy:{},\n paddle:{}'.format( - array1, tensor1.numpy())) + array1, tensor1.numpy() + ), + ) # 1 dim setitem array2 = array.copy() @@ -1111,7 +1235,9 @@ class TestListIndex(unittest.TestCase): tensor2.numpy(), array2, err_msg='\n numpy:{},\n paddle:{}'.format( - array2, tensor2.numpy())) + array2, tensor2.numpy() + ), + ) array = array[0] index1 = index1[0] diff --git a/python/paddle/fluid/tests/unittests/test_variance_layer.py b/python/paddle/fluid/tests/unittests/test_variance_layer.py index ca0ad7e9fd9..fc97a20533b 100644 --- a/python/paddle/fluid/tests/unittests/test_variance_layer.py +++ b/python/paddle/fluid/tests/unittests/test_variance_layer.py @@ -20,14 +20,13 @@ import paddle def ref_var(x, axis=None, unbiased=True, keepdim=False): ddof = 1 if unbiased else 0 if isinstance(axis, int): - axis = (axis, ) + axis = (axis,) if axis is not None: axis = tuple(axis) return np.var(x, axis=axis, ddof=ddof, keepdims=keepdim) class TestVarAPI(unittest.TestCase): - def setUp(self): self.dtype = 'float64' self.shape = [1, 3, 4, 10] @@ -36,9 +35,11 @@ class TestVarAPI(unittest.TestCase): self.unbiased = True self.set_attrs() self.x = np.random.uniform(-1, 1, self.shape).astype(self.dtype) - self.place=paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def set_attrs(self): pass @@ -68,43 +69,36 @@ class TestVarAPI(unittest.TestCase): class TestVarAPI_dtype(TestVarAPI): - def set_attrs(self): self.dtype = 'float32' class TestVarAPI_axis_int(TestVarAPI): - def set_attrs(self): self.axis = 2 class TestVarAPI_axis_list(TestVarAPI): - def set_attrs(self): self.axis = [1, 2] class TestVarAPI_axis_tuple(TestVarAPI): - def set_attrs(self): self.axis = (1, 3) class TestVarAPI_keepdim(TestVarAPI): - def set_attrs(self): self.keepdim = False class TestVarAPI_unbiased(TestVarAPI): - def set_attrs(self): self.unbiased = False class TestVarAPI_alias(unittest.TestCase): - def test_alias(self): paddle.disable_static() x = paddle.to_tensor(np.array([10, 12], 'float32')) @@ -117,7 +111,6 @@ class TestVarAPI_alias(unittest.TestCase): class TestVarError(unittest.TestCase): - def test_error(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('X', [2, 3, 4], 'int32') diff --git a/python/paddle/fluid/tests/unittests/test_version.py b/python/paddle/fluid/tests/unittests/test_version.py index 0fda61f7bbe..507ec953764 100644 --- a/python/paddle/fluid/tests/unittests/test_version.py +++ b/python/paddle/fluid/tests/unittests/test_version.py @@ -19,7 +19,6 @@ import paddle.version as fluid_version class VersionTest(unittest.TestCase): - def setUp(self): self._major_regex = "[0-9]+" self._minor_regex = "[0-9]+" @@ -46,4 +45,5 @@ class VersionTest(unittest.TestCase): self.assertTrue(re.match(self._patch_regex, fluid_version.patch)) self.assertTrue(re.match(self._rc_regex, fluid_version.rc)) self.assertTrue( - re.match(self._version_regex, fluid_version.full_version)) + re.match(self._version_regex, fluid_version.full_version) + ) diff --git a/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py b/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py index 1f4725568a8..ab2c091cc0c 100644 --- a/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py +++ b/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py @@ -26,7 +26,6 @@ from paddle.fluid.framework import _test_eager_guard # reuse the input varbase's allocation. # View APIs include: `squeeze`, `unsqueeze`, `reshape`, `flatten`, `detach` class TestDygraphViewReuseAllocation(unittest.TestCase): - def setUp(self): self.init_shape() @@ -40,7 +39,7 @@ class TestDygraphViewReuseAllocation(unittest.TestCase): def func_test_view_api(self): var = paddle.rand(self.input_shape) view_var = self.view_api_processing(var) - view_var[0] = 2. + view_var[0] = 2.0 self.assertEqual(var.shape, self.input_shape) self.assertEqual(view_var.shape, self.output_shape) @@ -59,14 +58,14 @@ class TestDygraphViewReuseAllocation(unittest.TestCase): view_var = self.view_api_processing(var) self.assertEqual(view_var.inplace_version, 0) - var[0] = 2. + var[0] = 2.0 self.assertEqual(var.inplace_version, 1) self.assertEqual(view_var.inplace_version, 1) view_var_2 = self.view_api_processing(var) self.assertEqual(view_var_2.inplace_version, 1) - var[0] = 3. + var[0] = 3.0 self.assertEqual(view_var.inplace_version, 2) self.assertEqual(view_var_2.inplace_version, 2) @@ -87,13 +86,15 @@ class TestDygraphViewReuseAllocation(unittest.TestCase): # Here, the gradient computation will use the value of var_b var_c = var_b**2 view_var_b = self.view_api_processing(var_b) - view_var_b[0] = 2. # var_b is modified inplace + view_var_b[0] = 2.0 # var_b is modified inplace loss = paddle.nn.functional.relu(var_c) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): loss.backward() def test_backward_error(self): @@ -103,7 +104,6 @@ class TestDygraphViewReuseAllocation(unittest.TestCase): class TestUnsqueezeDygraphViewReuseAllocation(TestDygraphViewReuseAllocation): - def init_shape(self): self.input_shape = [2, 3] self.output_shape = [2, 3, 1] @@ -113,7 +113,6 @@ class TestUnsqueezeDygraphViewReuseAllocation(TestDygraphViewReuseAllocation): class TestReshapeDygraphViewReuseAllocation(TestDygraphViewReuseAllocation): - def init_shape(self): self.input_shape = [3, 4] self.output_shape = [2, 2, 3] @@ -123,7 +122,6 @@ class TestReshapeDygraphViewReuseAllocation(TestDygraphViewReuseAllocation): class TestFlattenDygraphViewReuseAllocation(TestDygraphViewReuseAllocation): - def init_shape(self): self.input_shape = [3, 4] self.output_shape = [12] diff --git a/python/paddle/fluid/tests/unittests/test_viterbi_decode_op.py b/python/paddle/fluid/tests/unittests/test_viterbi_decode_op.py index 7b789577771..624f2f5e616 100644 --- a/python/paddle/fluid/tests/unittests/test_viterbi_decode_op.py +++ b/python/paddle/fluid/tests/unittests/test_viterbi_decode_op.py @@ -19,7 +19,6 @@ paddle.enable_static() class Decoder(object): - def __init__(self, transitions, use_tag=True): self.transitions = transitions self.use_tag = use_tag @@ -33,8 +32,11 @@ class Decoder(object): left_length = np.array(length) max_seq_len = np.amax(left_length) left_length = np.expand_dims(left_length, 1) - alpha = np.full((bs, n_label), -1e4, dtype='float32') if self.use_tag \ + alpha = ( + np.full((bs, n_label), -1e4, dtype='float32') + if self.use_tag else np.zeros((bs, n_label), dtype='float32') + ) alpha[:, -1] = 0 for i, logit in enumerate(inputs_t[:max_seq_len]): if i == 0 and not self.use_tag: @@ -46,7 +48,7 @@ class Decoder(object): max_res = np.amax(alpha_trn_sum, 1), np.argmax(alpha_trn_sum, 1) historys = historys + [max_res[1]] if i >= 1 else [] alpha_nxt = max_res[0] + logit - mask = (left_length > 0) + mask = left_length > 0 alpha = mask * alpha_nxt + (1 - mask) * alpha if self.use_tag: alpha += (left_length == 1) * trans_exp[:, self.stop_idx] @@ -60,7 +62,7 @@ class Decoder(object): left_length = left_length + 1 gather_idx = batch_offset + last_ids last_ids_update = np.take(hist, gather_idx) * (left_length > 0) - mask = (left_length == 0) + mask = left_length == 0 last_ids_update = last_ids_update * (1 - mask) + last_ids * mask batch_path.insert(0, last_ids_update) last_ids = last_ids_update + (left_length < 0) * last_ids @@ -69,7 +71,6 @@ class Decoder(object): class TestViterbiOp(OpTest): - def set_attr(self): self.dtype = "float32" if core.is_compiled_with_rocm() else "float64" self.use_tag = True @@ -88,7 +89,7 @@ class TestViterbiOp(OpTest): self.inputs = { 'Input': self.input, 'Transition': self.trans, - 'Length': self.length + 'Length': self.length, } self.attrs = { 'include_bos_eos_tag': self.use_tag, @@ -100,12 +101,14 @@ class TestViterbiOp(OpTest): class TestViterbiAPI(unittest.TestCase): - def set_attr(self): self.use_tag = True self.bz, self.len, self.ntags = 4, 8, 10 - self.places = [fluid.CPUPlace(), fluid.CUDAPlace(0)] \ - if core.is_compiled_with_cuda() else [fluid.CPUPlace()] + self.places = ( + [fluid.CPUPlace(), fluid.CUDAPlace(0)] + if core.is_compiled_with_cuda() + else [fluid.CPUPlace()] + ) def setUp(self): self.set_attr() @@ -119,12 +122,12 @@ class TestViterbiAPI(unittest.TestCase): def check_static_result(self, place): bz, length, ntags = self.bz, self.len, self.ntags with fluid.program_guard(fluid.Program(), fluid.Program()): - Input = fluid.data(name="Input", - shape=[bz, length, ntags], - dtype="float32") - Transition = fluid.data(name="Transition", - shape=[ntags, ntags], - dtype="float32") + Input = fluid.data( + name="Input", shape=[bz, length, ntags], dtype="float32" + ) + Transition = fluid.data( + name="Transition", shape=[ntags, ntags], dtype="float32" + ) Length = fluid.data(name="Length", shape=[bz], dtype="int64") decoder = paddle.text.ViterbiDecoder(Transition, self.use_tag) score, path = decoder(Input, Length) @@ -132,7 +135,7 @@ class TestViterbiAPI(unittest.TestCase): feed_list = { "Input": self.input, "Transition": self.transitions, - "Length": self.length + "Length": self.length, } fetches = exe.run(feed=feed_list, fetch_list=[score, path]) np.testing.assert_allclose(fetches[0], self.scores, rtol=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_warpctc_op.py b/python/paddle/fluid/tests/unittests/test_warpctc_op.py index 81dce5b1a86..91bebab2f6c 100644 --- a/python/paddle/fluid/tests/unittests/test_warpctc_op.py +++ b/python/paddle/fluid/tests/unittests/test_warpctc_op.py @@ -29,9 +29,17 @@ CUDA_BLOCK_SIZE = 32 class CTCForward(object): - - def __init__(self, softmax, softmax_lod, labels, labels_lod, num_classes, - batch_size, blank, norm_by_times): + def __init__( + self, + softmax, + softmax_lod, + labels, + labels_lod, + num_classes, + batch_size, + blank, + norm_by_times, + ): self.softmax = softmax self.softmax_lod = softmax_lod self.labels = labels @@ -114,15 +122,17 @@ class CTCForward(object): # calculate the forward and backward variables, # reference Chapter 7.3 of "Alex Grave, Supervised Sequence # Labelling with Recurrent Neural Networks" - log_acts = np.zeros([total_times, self.num_classes], - dtype=softmax_a_sequence.dtype) + log_acts = np.zeros( + [total_times, self.num_classes], dtype=softmax_a_sequence.dtype + ) for i in range(total_times): for j in range(self.num_classes): log_acts[i, j] = self.safe_log(softmax_a_sequence[i, j]) # calculate the forward variables - forward_vars = np.zeros([total_times, total_segments], - dtype=softmax_a_sequence.dtype) + forward_vars = np.zeros( + [total_times, total_segments], dtype=softmax_a_sequence.dtype + ) for i in range(total_times): for j in range(total_segments): forward_vars[i, j] = self.LOG_ZERO @@ -142,10 +152,13 @@ class CTCForward(object): if j & 1 == 1: label_idx = j // 2 label_val = labels_a_sequence[label_idx, 0] - fv = self.log_add(forward_vars[i - 1, j], - forward_vars[i - 1, j - 1]) - if j > 1 and label_val != labels_a_sequence[label_idx - 1, - 0]: + fv = self.log_add( + forward_vars[i - 1, j], forward_vars[i - 1, j - 1] + ) + if ( + j > 1 + and label_val != labels_a_sequence[label_idx - 1, 0] + ): fv = self.log_add(fv, forward_vars[i - 1, j - 2]) fv = self.log_mul(fv, log_acts[i, label_val]) else: @@ -159,7 +172,8 @@ class CTCForward(object): log_prob = forward_vars[total_times - 1, total_segments - 1] if total_segments > 1: log_prob = self.log_add( - log_prob, forward_vars[total_times - 1, total_segments - 2]) + log_prob, forward_vars[total_times - 1, total_segments - 2] + ) return -log_prob @@ -174,33 +188,38 @@ class CTCForward(object): labels_end_i = labels_offset + self.labels_lod[self.level][i] softmax_a_sequence = self.softmax[ - softmax_start_i:softmax_end_i, :] + softmax_start_i:softmax_end_i, : + ] labels_a_sequence = self.labels[labels_start_i:labels_end_i, :] - self.loss[i] = self.forward_a_sequence(softmax_a_sequence, - labels_a_sequence) + self.loss[i] = self.forward_a_sequence( + softmax_a_sequence, labels_a_sequence + ) softmax_offset += self.softmax_lod[self.level][i] labels_offset += self.labels_lod[self.level][i] else: - softmax_a_sequence = self.softmax[:self.softmax_lod[i], i, :] - labels_a_sequence = self.labels[:self.labels_lod[i], :] - self.loss[i] = self.forward_a_sequence(softmax_a_sequence, - labels_a_sequence) + softmax_a_sequence = self.softmax[: self.softmax_lod[i], i, :] + labels_a_sequence = self.labels[: self.labels_lod[i], :] + self.loss[i] = self.forward_a_sequence( + softmax_a_sequence, labels_a_sequence + ) return self.loss -def python_api(logits, - label, - logits_length=None, - labels_length=None, - blank=0, - norm_by_times=False): - return paddle.fluid.layers.warpctc(logits, label, blank, norm_by_times, - logits_length, labels_length) +def python_api( + logits, + label, + logits_length=None, + labels_length=None, + blank=0, + norm_by_times=False, +): + return paddle.fluid.layers.warpctc( + logits, label, blank, norm_by_times, logits_length, labels_length + ) class TestWarpCTCOp(OpTest): - def config(self): self.batch_size = 4 self.num_classes = 12 @@ -214,31 +233,39 @@ class TestWarpCTCOp(OpTest): self.config() logits = np.random.uniform( - 0.1, 1.0, - [sum(self.logits_lod[0]), self.num_classes]).astype("float32") + 0.1, 1.0, [sum(self.logits_lod[0]), self.num_classes] + ).astype("float32") softmax = np.apply_along_axis(stable_softmax, 1, logits) # labels should not be blank - labels = np.random.randint(0, - self.num_classes - 1, - [sum(self.labels_lod[0]), 1], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_lod, labels, self.labels_lod, - self.num_classes, self.batch_size, self.blank, - self.norm_by_times) + labels = np.random.randint( + 0, self.num_classes - 1, [sum(self.labels_lod[0]), 1], dtype="int32" + ) + + ctc = CTCForward( + softmax, + self.logits_lod, + labels, + self.labels_lod, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss = ctc.forward() max_sequence_length = 0 for i in range(self.batch_size): - max_sequence_length = max(max_sequence_length, - self.logits_lod[0][i]) + max_sequence_length = max( + max_sequence_length, self.logits_lod[0][i] + ) self.gradient = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) self.inputs = { "Logits": (logits, self.logits_lod), - "Label": (labels, self.labels_lod) + "Label": (labels, self.labels_lod), } self.outputs = {"Loss": loss} self.attrs = { @@ -252,19 +279,22 @@ class TestWarpCTCOp(OpTest): def test_check_grad(self): self.outputs['WarpCTCGrad'] = self.gradient if core.is_compiled_with_rocm(): - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.009, - check_dygraph=False) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=0.009, + check_dygraph=False, + ) else: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.007, - check_dygraph=False) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=0.007, + check_dygraph=False, + ) class TestWarpCTCOpCase1(TestWarpCTCOp): - def config(self): self.batch_size = 4 self.num_classes = CUDA_BLOCK_SIZE + 2 @@ -275,7 +305,6 @@ class TestWarpCTCOpCase1(TestWarpCTCOp): class TestWarpCTCOpWithPadding(OpTest): - def config(self): self.batch_size = 4 self.num_classes = 8 @@ -293,28 +322,36 @@ class TestWarpCTCOpWithPadding(OpTest): self.config() logits = np.random.uniform( - 0.1, 1.0, - [sum(self.logits_length), self.num_classes]).astype("float32") + 0.1, 1.0, [sum(self.logits_length), self.num_classes] + ).astype("float32") softmax = np.apply_along_axis(stable_softmax, 1, logits) # labels should not be blank - labels = np.random.randint(0, - self.num_classes - 1, - [sum(self.labels_length), 1], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_lod, labels, self.labels_lod, - self.num_classes, self.batch_size, self.blank, - self.norm_by_times) + labels = np.random.randint( + 0, self.num_classes - 1, [sum(self.labels_length), 1], dtype="int32" + ) + + ctc = CTCForward( + softmax, + self.logits_lod, + labels, + self.labels_lod, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss = ctc.forward() max_sequence_length = 0 for i in range(self.batch_size): - max_sequence_length = max(max_sequence_length, - self.logits_length[i]) + max_sequence_length = max( + max_sequence_length, self.logits_length[i] + ) # reshape logits to T*N*S new_logits = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) cur = 0 for batch_id in range(self.batch_size): @@ -326,10 +363,12 @@ class TestWarpCTCOpWithPadding(OpTest): # reshape labels to N*S max_target_seq_length = 0 for i in range(self.batch_size): - max_target_seq_length = max(max_target_seq_length, - self.labels_length[i]) - new_labels = np.zeros([self.batch_size, max_target_seq_length], - dtype="int32") + max_target_seq_length = max( + max_target_seq_length, self.labels_length[i] + ) + new_labels = np.zeros( + [self.batch_size, max_target_seq_length], dtype="int32" + ) cur = 0 for batch_id in range(self.batch_size): @@ -339,13 +378,14 @@ class TestWarpCTCOpWithPadding(OpTest): self.gradient = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) self.inputs = { "Logits": new_logits, "Label": new_labels, "LogitsLength": self.logits_length, - "LabelLength": self.labels_length + "LabelLength": self.labels_length, } self.outputs = {"Loss": loss} self.attrs = { @@ -359,19 +399,22 @@ class TestWarpCTCOpWithPadding(OpTest): def test_check_grad(self): self.outputs['WarpCTCGrad'] = self.gradient if core.is_compiled_with_rocm(): - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.009, - check_dygraph=False) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=0.009, + check_dygraph=False, + ) else: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.007, - check_dygraph=False) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=0.007, + check_dygraph=False, + ) class TestWarpCTCOpWithPaddingCase1(TestWarpCTCOpWithPadding): - def config(self): self.batch_size = 4 self.num_classes = CUDA_BLOCK_SIZE + 2 @@ -384,7 +427,6 @@ class TestWarpCTCOpWithPaddingCase1(TestWarpCTCOpWithPadding): class TestWarpCTCOpFp64(OpTest): - def config(self): self.batch_size = 4 self.num_classes = 8 @@ -402,28 +444,36 @@ class TestWarpCTCOpFp64(OpTest): self.config() logits = np.random.uniform( - 0.1, 1.0, - [sum(self.logits_length), self.num_classes]).astype("float64") + 0.1, 1.0, [sum(self.logits_length), self.num_classes] + ).astype("float64") softmax = np.apply_along_axis(stable_softmax, 1, logits) # labels should not be blank - labels = np.random.randint(0, - self.num_classes - 1, - [sum(self.labels_length), 1], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_lod, labels, self.labels_lod, - self.num_classes, self.batch_size, self.blank, - self.norm_by_times) + labels = np.random.randint( + 0, self.num_classes - 1, [sum(self.labels_length), 1], dtype="int32" + ) + + ctc = CTCForward( + softmax, + self.logits_lod, + labels, + self.labels_lod, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss = ctc.forward() max_sequence_length = 0 for i in range(self.batch_size): - max_sequence_length = max(max_sequence_length, - self.logits_length[i]) + max_sequence_length = max( + max_sequence_length, self.logits_length[i] + ) # reshape logits to T*N*S new_logits = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) cur = 0 for batch_id in range(self.batch_size): @@ -435,10 +485,12 @@ class TestWarpCTCOpFp64(OpTest): # reshape labels to N*S max_target_seq_length = 0 for i in range(self.batch_size): - max_target_seq_length = max(max_target_seq_length, - self.labels_length[i]) - new_labels = np.zeros([self.batch_size, max_target_seq_length], - dtype="int32") + max_target_seq_length = max( + max_target_seq_length, self.labels_length[i] + ) + new_labels = np.zeros( + [self.batch_size, max_target_seq_length], dtype="int32" + ) cur = 0 for batch_id in range(self.batch_size): @@ -448,13 +500,14 @@ class TestWarpCTCOpFp64(OpTest): self.gradient = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) self.inputs = { "Logits": new_logits, "Label": new_labels, "LogitsLength": self.logits_length, - "LabelLength": self.labels_length + "LabelLength": self.labels_length, } self.outputs = {"Loss": loss} self.attrs = { @@ -471,58 +524,64 @@ class TestWarpCTCOpFp64(OpTest): class TestWarpCTCOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - logits = fluid.data(name='logits', - shape=[5, 16, 6], - dtype='float32') - logits_length = fluid.data(name='logits_length', - shape=[None], - dtype='int64') + logits = fluid.data( + name='logits', shape=[5, 16, 6], dtype='float32' + ) + logits_length = fluid.data( + name='logits_length', shape=[None], dtype='int64' + ) label = fluid.data(name='label', shape=[16, 3], dtype='int32') - label_length = fluid.data(name='labels_length', - shape=[None], - dtype='int64') + label_length = fluid.data( + name='labels_length', shape=[None], dtype='int64' + ) def test_logits_Variable(): logits_data = np.random.rand(5, 16, 6).astype(logits.dtype) - fluid.layers.warpctc(input=logits_data, - label=label, - input_length=logits_length, - label_length=label_length) + fluid.layers.warpctc( + input=logits_data, + label=label, + input_length=logits_length, + label_length=label_length, + ) self.assertRaises(TypeError, test_logits_Variable) def test_label_Variable(): label_data = np.random.randint(0, 5, [5, 1]).astype("int32") - fluid.layers.warpctc(input=logits, - label=label_data, - input_length=logits_length, - label_length=label_length) + fluid.layers.warpctc( + input=logits, + label=label_data, + input_length=logits_length, + label_length=label_length, + ) self.assertRaises(TypeError, test_label_Variable) def test_logits_len_Variable(): logits_length_data = np.array([5] * 16).astype("int64") - fluid.layers.warpctc(input=logits, - label=label, - input_length=logits_length_data, - label_length=label_length) + fluid.layers.warpctc( + input=logits, + label=label, + input_length=logits_length_data, + label_length=label_length, + ) self.assertRaises(TypeError, test_logits_len_Variable) def test_label_len_Variable(): label_length_data = np.array([3] * 16).astype("int64") - fluid.layers.warpctc(input=logits, - label=label, - input_length=logits_length, - label_length=label_length_data) + fluid.layers.warpctc( + input=logits, + label=label, + input_length=logits_length, + label_length=label_length_data, + ) self.assertRaises(TypeError, test_label_len_Variable) def test_dygraph_errors(self): - def test_dygraph_with_lod(): logits = np.random.uniform(0.1, 1.0, [20, 15]).astype("float32") @@ -539,7 +598,6 @@ class TestWarpCTCOpError(unittest.TestCase): class TestCTCLossAPICase(unittest.TestCase): - def test_functinal_api(self): self.batch_size = 4 self.num_classes = CUDA_BLOCK_SIZE + 2 @@ -549,20 +607,29 @@ class TestCTCLossAPICase(unittest.TestCase): self.norm_by_times = False logits = np.random.uniform( - 0.1, 1.0, - [max(self.logits_length), self.batch_size, self.num_classes - ]).astype("float32") + 0.1, + 1.0, + [max(self.logits_length), self.batch_size, self.num_classes], + ).astype("float32") softmax = np.apply_along_axis(stable_softmax, -1, logits) # labels should not be blank labels = np.random.randint( 0, self.num_classes - 1, [self.batch_size, max(self.labels_length)], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_length, labels, - self.labels_length, self.num_classes, self.batch_size, - self.blank, self.norm_by_times) + dtype="int32", + ) + + ctc = CTCForward( + softmax, + self.logits_length, + labels, + self.labels_length, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss_np = ctc.forward() paddle.disable_static() @@ -570,30 +637,33 @@ class TestCTCLossAPICase(unittest.TestCase): labels = paddle.to_tensor(labels) logits_length = paddle.to_tensor(self.logits_length) labels_length = paddle.to_tensor(self.labels_length) - loss_pd_mean = F.ctc_loss(softmax, - labels, - logits_length, - labels_length, - blank=self.blank, - reduction='mean') + loss_pd_mean = F.ctc_loss( + softmax, + labels, + logits_length, + labels_length, + blank=self.blank, + reduction='mean', + ) loss_pd_mean = loss_pd_mean.numpy() - loss_pd_sum = F.ctc_loss(softmax, - labels, - logits_length, - labels_length, - blank=self.blank, - reduction='sum') + loss_pd_sum = F.ctc_loss( + softmax, + labels, + logits_length, + labels_length, + blank=self.blank, + reduction='sum', + ) loss_pd_sum = loss_pd_sum.numpy() paddle.enable_static() loss_np = np.squeeze(loss_np, axis=-1) loss_np_mean = (loss_np / labels_length.numpy()).mean() loss_np_sum = loss_np.sum() - np.testing.assert_allclose(loss_pd_mean, - loss_np_mean, - rtol=1e-05, - atol=1) + np.testing.assert_allclose( + loss_pd_mean, loss_np_mean, rtol=1e-05, atol=1 + ) np.testing.assert_allclose(loss_pd_sum, loss_np_sum, rtol=1e-05, atol=1) def test_class_api(self): @@ -605,20 +675,29 @@ class TestCTCLossAPICase(unittest.TestCase): self.norm_by_times = False logits = np.random.uniform( - 0.1, 1.0, - [max(self.logits_length), self.batch_size, self.num_classes - ]).astype("float32") + 0.1, + 1.0, + [max(self.logits_length), self.batch_size, self.num_classes], + ).astype("float32") softmax = np.apply_along_axis(stable_softmax, -1, logits) # labels should not be blank labels = np.random.randint( 1, self.num_classes, [self.batch_size, max(self.labels_length)], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_length, labels, - self.labels_length, self.num_classes, self.batch_size, - self.blank, self.norm_by_times) + dtype="int32", + ) + + ctc = CTCForward( + softmax, + self.logits_length, + labels, + self.labels_length, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss_np = ctc.forward() paddle.disable_static() @@ -627,9 +706,9 @@ class TestCTCLossAPICase(unittest.TestCase): logits_length = paddle.to_tensor(self.logits_length) labels_length = paddle.to_tensor(self.labels_length) - loss_pd = paddle.nn.CTCLoss(self.blank, - 'none')(softmax, labels, logits_length, - labels_length) + loss_pd = paddle.nn.CTCLoss(self.blank, 'none')( + softmax, labels, logits_length, labels_length + ) loss_pd = loss_pd.numpy() paddle.enable_static() loss_np = np.squeeze(loss_np, axis=-1) diff --git a/python/paddle/fluid/tests/unittests/test_weight_decay.py b/python/paddle/fluid/tests/unittests/test_weight_decay.py index 1b099ab25a3..5133a203755 100644 --- a/python/paddle/fluid/tests/unittests/test_weight_decay.py +++ b/python/paddle/fluid/tests/unittests/test_weight_decay.py @@ -40,22 +40,24 @@ def prog_scope_guard(main_prog, startup_prog): yield -def bow_net(data, - label, - dict_dim, - is_sparse=False, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): +def bow_net( + data, + label, + dict_dim, + is_sparse=False, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - emb = fluid.layers.embedding(input=data, - is_sparse=is_sparse, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -68,13 +70,13 @@ def bow_net(data, class TestWeightDecay(unittest.TestCase): - def setUp(self): self.word_dict = paddle.dataset.imdb.word_dict() - reader = paddle.batch(paddle.dataset.imdb.train(self.word_dict), - batch_size=4)() + reader = paddle.batch( + paddle.dataset.imdb.train(self.word_dict), batch_size=4 + )() self.train_data = [next(reader) for _ in range(5)] - self.learning_rate = .5 + self.learning_rate = 0.5 def run_executor(self, place, feed_list, loss): exe = fluid.Executor(place) @@ -83,22 +85,24 @@ class TestWeightDecay(unittest.TestCase): main_prog = fluid.default_main_program() loss_set = [] for data in self.train_data: - out = exe.run(main_prog, - feed=feeder.feed(data), - fetch_list=[loss.name]) + out = exe.run( + main_prog, feed=feeder.feed(data), fetch_list=[loss.name] + ) print("loss %s" % (np.average(out))) loss_set.append(np.average(out)) return loss_set - def run_parallel_exe(self, - place, - feed_list, - loss, - use_reduce=False, - use_fast_executor=False, - use_ir_memory_optimize=False): + def run_parallel_exe( + self, + place, + feed_list, + loss, + use_reduce=False, + use_fast_executor=False, + use_ir_memory_optimize=False, + ): exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=feed_list, place=place) exe.run(fluid.default_startup_program()) @@ -108,57 +112,63 @@ class TestWeightDecay(unittest.TestCase): exec_strategy.use_experimental_executor = True build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce \ - if use_reduce else fluid.BuildStrategy.ReduceStrategy.AllReduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + if use_reduce + else fluid.BuildStrategy.ReduceStrategy.AllReduce + ) build_strategy.memory_optimize = use_ir_memory_optimize train_cp = compiler.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name, - exec_strategy=exec_strategy, - build_strategy=build_strategy) + fluid.default_main_program() + ).with_data_parallel( + loss_name=loss.name, + exec_strategy=exec_strategy, + build_strategy=build_strategy, + ) loss_set = [] for data in self.train_data: - out = exe.run(train_cp, - feed=feeder.feed(data), - fetch_list=[loss.name]) + out = exe.run( + train_cp, feed=feeder.feed(data), fetch_list=[loss.name] + ) loss_set.append(np.average(out)) return loss_set - def check_weight_decay(self, - place, - model, - use_parallel_exe=False, - use_reduce=False): + def check_weight_decay( + self, place, model, use_parallel_exe=False, use_reduce=False + ): main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() startup_prog.random_seed = 1 with prog_scope_guard(main_prog=main_prog, startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost = model(data, label, len(self.word_dict)) - param_list = [(var, var * self.learning_rate) - for var in main_prog.block(0).all_parameters()] + param_list = [ + (var, var * self.learning_rate) + for var in main_prog.block(0).all_parameters() + ] optimizer = fluid.optimizer.Adagrad( - learning_rate=self.learning_rate) + learning_rate=self.learning_rate + ) optimizer.minimize(avg_cost) for params in param_list: - updated_p = fluid.layers.elementwise_sub(x=params[0], - y=params[1]) + updated_p = fluid.layers.elementwise_sub( + x=params[0], y=params[1] + ) fluid.layers.assign(input=updated_p, output=params[0]) if use_parallel_exe: - loss = self.run_parallel_exe(place, [data, label], - loss=avg_cost, - use_reduce=use_reduce) + loss = self.run_parallel_exe( + place, [data, label], loss=avg_cost, use_reduce=use_reduce + ) else: loss = self.run_executor(place, [data, label], loss=avg_cost) @@ -170,16 +180,21 @@ class TestWeightDecay(unittest.TestCase): loss = self.check_weight_decay(place, model, use_parallel_exe=False) # TODO(zcd): should test use_reduce=True - loss2 = self.check_weight_decay(place, - model, - use_parallel_exe=True, - use_reduce=False) + loss2 = self.check_weight_decay( + place, model, use_parallel_exe=True, use_reduce=False + ) for i in range(len(loss)): self.assertTrue( np.isclose(a=loss[i], b=loss2[i], rtol=5e-5), - "Expect " + str(loss[i]) + "\n" + "But Got" + - str(loss2[i]) + " in class " + self.__class__.__name__) + "Expect " + + str(loss[i]) + + "\n" + + "But Got" + + str(loss2[i]) + + " in class " + + self.__class__.__name__, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_weight_normalization.py b/python/paddle/fluid/tests/unittests/test_weight_normalization.py index d7a0fa3cd18..9d77dadf8dc 100644 --- a/python/paddle/fluid/tests/unittests/test_weight_normalization.py +++ b/python/paddle/fluid/tests/unittests/test_weight_normalization.py @@ -24,7 +24,7 @@ from paddle.fluid.param_attr import WeightNormParamAttr class TestWeightNormalization(unittest.TestCase): batch_size = 3 hidden_size = 5 - data_desc = (['x', [10], 0], ) + data_desc = (['x', [10], 0],) @classmethod def setUpClass(cls): @@ -32,21 +32,26 @@ class TestWeightNormalization(unittest.TestCase): @classmethod def set_program(cls): - data = fluid.layers.data(name=cls.data_desc[0][0], - shape=cls.data_desc[0][1]) - out = fluid.layers.fc(input=data, - size=cls.hidden_size, - param_attr=WeightNormParamAttr( - dim=None, - name='weight_norm_param', - initializer=ConstantInitializer(1.0)), - bias_attr=False, - act=None) + data = fluid.layers.data( + name=cls.data_desc[0][0], shape=cls.data_desc[0][1] + ) + out = fluid.layers.fc( + input=data, + size=cls.hidden_size, + param_attr=WeightNormParamAttr( + dim=None, + name='weight_norm_param', + initializer=ConstantInitializer(1.0), + ), + bias_attr=False, + act=None, + ) loss = fluid.layers.reduce_sum(out) fluid.backward.append_backward(loss=loss) cls.fetch_list = [ - 'weight_norm_param_g', 'weight_norm_param_v', - 'weight_norm_param_g@GRAD' + 'weight_norm_param_g', + 'weight_norm_param_v', + 'weight_norm_param_g@GRAD', ] def run_program(self): @@ -58,10 +63,12 @@ class TestWeightNormalization(unittest.TestCase): self.set_inputs(place) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - output = exe.run(fluid.default_main_program(), - feed=self.inputs, - fetch_list=self.fetch_list, - return_numpy=False) + output = exe.run( + fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=False, + ) outputs.append(output) self.actual_outputs = outputs @@ -73,14 +80,16 @@ class TestWeightNormalization(unittest.TestCase): data_lod_level = desc[2] data_lod = [] for i in range(data_lod_level): - lod_level_i = np.random.randint(low=1, - high=5, - size=self.batch_size if i == 0 - else sum(lod_level_i)).tolist() + lod_level_i = np.random.randint( + low=1, + high=5, + size=self.batch_size if i == 0 else sum(lod_level_i), + ).tolist() data_lod.append(lod_level_i) data_value = np.random.random( - size=[sum(data_lod[-1]) if data_lod else self.batch_size] + - data_shape).astype('float32') + size=[sum(data_lod[-1]) if data_lod else self.batch_size] + + data_shape + ).astype('float32') self.data[data_name] = (data_value, data_lod) def set_inputs(self, place): @@ -94,14 +103,16 @@ class TestWeightNormalization(unittest.TestCase): def weight_normalize(self): v = np.ones( - (self.data[self.data_desc[0][0]][0].shape[-1], self.hidden_size)) + (self.data[self.data_desc[0][0]][0].shape[-1], self.hidden_size) + ) g = np.linalg.norm(v, axis=None, keepdims=True) w = g * v / np.linalg.norm(v, axis=None, keepdims=True) x = self.data[self.data_desc[0][0]][0] out = np.dot(x, w) - g_grad = (np.dot(x.T, np.ones_like(out)) * - (v / np.linalg.norm(v, axis=None, keepdims=True))).sum( - axis=None, keepdims=True) + g_grad = ( + np.dot(x.T, np.ones_like(out)) + * (v / np.linalg.norm(v, axis=None, keepdims=True)) + ).sum(axis=None, keepdims=True) return g, v, g_grad def test_weight_normalization(self): @@ -110,10 +121,9 @@ class TestWeightNormalization(unittest.TestCase): expect_output = self.weight_normalize() for actual_output in self.actual_outputs: [ - np.testing.assert_allclose(np.array(actual), - expect, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + np.array(actual), expect, rtol=1e-05, atol=0.001 + ) for expect, actual in zip(expect_output, actual_output) ] diff --git a/python/paddle/fluid/tests/unittests/test_where_index.py b/python/paddle/fluid/tests/unittests/test_where_index.py index ad561c4a512..96b26dd6cae 100644 --- a/python/paddle/fluid/tests/unittests/test_where_index.py +++ b/python/paddle/fluid/tests/unittests/test_where_index.py @@ -22,7 +22,6 @@ from paddle.fluid import Program, program_guard class TestWhereIndexOp(OpTest): - def setUp(self): self.op_type = "where_index" self.init_config() @@ -39,7 +38,6 @@ class TestWhereIndexOp(OpTest): class TestAllFalse(unittest.TestCase): - def setUp(self): self.op_type = "where_index" self.init_config() @@ -71,7 +69,6 @@ class TestAllFalse(unittest.TestCase): class TestRank2(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([[True, False], [False, True]]), @@ -81,24 +78,26 @@ class TestRank2(TestWhereIndexOp): class TestRank3(TestWhereIndexOp): - def init_config(self): self.inputs = { - 'Condition': - np.array([[[True, False], [False, True]], - [[False, True], [True, False]], - [[False, False], [False, True]]]), + 'Condition': np.array( + [ + [[True, False], [False, True]], + [[False, True], [True, False]], + [[False, False], [False, True]], + ] + ), } self.outputs = { - 'Out': - np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], - dtype='int64') + 'Out': np.array( + [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], + dtype='int64', + ) } class TestWhereOpError(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): cond = fluid.layers.data(name='cond', shape=[4], dtype='bool') @@ -111,9 +110,7 @@ class TestWhereOpError(unittest.TestCase): class TestWhereRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.where([10]) diff --git a/python/paddle/fluid/tests/unittests/test_where_op.py b/python/paddle/fluid/tests/unittests/test_where_op.py index fafa4e52791..9ae7d9a4833 100644 --- a/python/paddle/fluid/tests/unittests/test_where_op.py +++ b/python/paddle/fluid/tests/unittests/test_where_op.py @@ -23,7 +23,6 @@ from paddle.fluid.framework import _test_eager_guard class TestWhereOp(OpTest): - def setUp(self): self.op_type = 'where' self.python_api = paddle.where @@ -44,7 +43,6 @@ class TestWhereOp(OpTest): class TestWhereOp2(TestWhereOp): - def init_config(self): self.x = np.random.uniform((-5), 5, (60, 2)).astype('float64') self.y = np.random.uniform((-5), 5, (60, 2)).astype('float64') @@ -52,7 +50,6 @@ class TestWhereOp2(TestWhereOp): class TestWhereOp3(TestWhereOp): - def init_config(self): self.x = np.random.uniform((-3), 5, (20, 2, 4)).astype('float64') self.y = np.random.uniform((-3), 5, (20, 2, 4)).astype('float64') @@ -60,7 +57,6 @@ class TestWhereOp3(TestWhereOp): class TestWhereAPI(unittest.TestCase): - def setUp(self): self.init_data() @@ -81,48 +77,51 @@ class TestWhereAPI(unittest.TestCase): for x_stop_gradient in [False, True]: for y_stop_gradient in [False, True]: with fluid.program_guard(Program(), Program()): - cond = fluid.layers.data(name='cond', - shape=self.shape, - dtype='bool') - x = fluid.layers.data(name='x', - shape=self.shape, - dtype='float32') - y = fluid.layers.data(name='y', - shape=self.shape, - dtype='float32') + cond = fluid.layers.data( + name='cond', shape=self.shape, dtype='bool' + ) + x = fluid.layers.data( + name='x', shape=self.shape, dtype='float32' + ) + y = fluid.layers.data( + name='y', shape=self.shape, dtype='float32' + ) x.stop_gradient = x_stop_gradient y.stop_gradient = y_stop_gradient result = paddle.where(cond, x, y) append_backward(paddle.mean(result)) for use_cuda in [False, True]: - if (use_cuda - and (not fluid.core.is_compiled_with_cuda())): + if use_cuda and ( + not fluid.core.is_compiled_with_cuda() + ): break - place = (fluid.CUDAPlace(0) - if use_cuda else fluid.CPUPlace()) + place = ( + fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() + ) exe = fluid.Executor(place) fetch_list = [result, result.grad_name] - if (x_stop_gradient is False): + if x_stop_gradient is False: fetch_list.append(x.grad_name) - if (y_stop_gradient is False): + if y_stop_gradient is False: fetch_list.append(y.grad_name) - out = exe.run(fluid.default_main_program(), - feed={ - 'cond': self.cond, - 'x': self.x, - 'y': self.y - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={'cond': self.cond, 'x': self.x, 'y': self.y}, + fetch_list=fetch_list, + ) assert np.array_equal(out[0], self.out) - if (x_stop_gradient is False): - assert np.array_equal(out[2], - self.ref_x_backward(out[1])) - if (y.stop_gradient is False): + if x_stop_gradient is False: + assert np.array_equal( + out[2], self.ref_x_backward(out[1]) + ) + if y.stop_gradient is False: assert np.array_equal( - out[3], self.ref_y_backward(out[1])) - elif (y.stop_gradient is False): - assert np.array_equal(out[2], - self.ref_y_backward(out[1])) + out[3], self.ref_y_backward(out[1]) + ) + elif y.stop_gradient is False: + assert np.array_equal( + out[2], self.ref_y_backward(out[1]) + ) def test_api_broadcast(self, use_cuda=False): main_program = Program() @@ -130,20 +129,20 @@ class TestWhereAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32') y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32') x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype('float32') - y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, - 1.0]]).astype('float32') + y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]).astype( + 'float32' + ) result = paddle.where((x > 1), x=x, y=y) for use_cuda in [False, True]: - if (use_cuda and (not fluid.core.is_compiled_with_cuda())): + if use_cuda and (not fluid.core.is_compiled_with_cuda()): return - place = (fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()) + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'x': x_i, 'y': y_i}, + fetch_list=[result], + ) assert np.array_equal(out[0], np.where((x_i > 1), x_i, y_i)) def test_scalar(self): @@ -151,21 +150,23 @@ class TestWhereAPI(unittest.TestCase): main_program = Program() with fluid.program_guard(main_program): cond_shape = [2, 4] - cond = fluid.layers.data(name='cond', - shape=cond_shape, - dtype='bool') + cond = fluid.layers.data( + name='cond', shape=cond_shape, dtype='bool' + ) x_data = 1.0 y_data = 2.0 cond_data = np.array([False, False, True, True]).astype('bool') result = paddle.where(condition=cond, x=x_data, y=y_data) for use_cuda in [False, True]: - if (use_cuda and (not fluid.core.is_compiled_with_cuda())): + if use_cuda and (not fluid.core.is_compiled_with_cuda()): return - place = (fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()) + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={'cond': cond_data}, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'cond': cond_data}, + fetch_list=[result], + ) expect = np.where(cond_data, x_data, y_data) assert np.array_equal(out[0], expect) @@ -173,28 +174,26 @@ class TestWhereAPI(unittest.TestCase): paddle.enable_static() main_program = Program() with fluid.program_guard(main_program): - cond = fluid.layers.data(name='cond', - shape=cond_shape, - dtype='bool') + cond = fluid.layers.data( + name='cond', shape=cond_shape, dtype='bool' + ) x = fluid.layers.data(name='x', shape=x_shape, dtype='float32') y = fluid.layers.data(name='y', shape=y_shape, dtype='float32') cond_data_tmp = np.random.random(size=cond_shape).astype('float32') - cond_data = (cond_data_tmp < 0.3) + cond_data = cond_data_tmp < 0.3 x_data = np.random.random(size=x_shape).astype('float32') y_data = np.random.random(size=y_shape).astype('float32') result = paddle.where(condition=cond, x=x, y=y) for use_cuda in [False, True]: - if (use_cuda and (not fluid.core.is_compiled_with_cuda())): + if use_cuda and (not fluid.core.is_compiled_with_cuda()): return - place = (fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()) + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'cond': cond_data, - 'x': x_data, - 'y': y_data - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'cond': cond_data, 'x': x_data, 'y': y_data}, + fetch_list=[result], + ) expect = np.where(cond_data, x_data, y_data) assert np.array_equal(out[0], expect) @@ -248,7 +247,6 @@ class TestWhereAPI(unittest.TestCase): class TestWhereDygraphAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype('float64') @@ -272,7 +270,7 @@ class TestWhereDygraphAPI(unittest.TestCase): def __test_where_with_broadcast_dygraph(self, cond_shape, a_shape, b_shape): with fluid.dygraph.guard(): cond_tmp = paddle.rand(cond_shape) - cond = (cond_tmp < 0.3) + cond = cond_tmp < 0.3 a = paddle.rand(a_shape) b = paddle.rand(b_shape) result = paddle.where(cond, a, b) @@ -337,9 +335,9 @@ class TestWhereDygraphAPI(unittest.TestCase): self.assertEqual(len(y), 2) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(fluid.CPUPlace()) - (res, ) = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0, 0], [1, 1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) data = np.array([True, True, False]) @@ -350,9 +348,9 @@ class TestWhereDygraphAPI(unittest.TestCase): self.assertEqual(len(y), 1) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(fluid.CPUPlace()) - (res, ) = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0], [1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -370,7 +368,6 @@ class TestWhereDygraphAPI(unittest.TestCase): class TestWhereOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype('float64') @@ -394,7 +391,7 @@ class TestWhereOpError(unittest.TestCase): with fluid.dygraph.guard(): cond_shape = [2, 2, 4] cond_tmp = paddle.rand(cond_shape) - cond = (cond_tmp < 0.3) + cond = cond_tmp < 0.3 a = paddle.rand(cond_shape) self.assertRaises(ValueError, paddle.where, cond, a) diff --git a/python/paddle/fluid/tests/unittests/test_while_loop_op.py b/python/paddle/fluid/tests/unittests/test_while_loop_op.py index 7d5fc61f28b..3c91b8c1e23 100644 --- a/python/paddle/fluid/tests/unittests/test_while_loop_op.py +++ b/python/paddle/fluid/tests/unittests/test_while_loop_op.py @@ -26,9 +26,7 @@ paddle.enable_static() class TestApiWhileLoop(unittest.TestCase): - def test_var_tuple(self): - def cond(i): return layers.less_than(i, ten) @@ -41,18 +39,20 @@ class TestApiWhileLoop(unittest.TestCase): i = layers.fill_constant(shape=[1], dtype='int64', value=0) one = layers.fill_constant(shape=[1], dtype='int64', value=1) ten = layers.fill_constant(shape=[1], dtype='int64', value=10) - out = layers.while_loop(cond, body, (i, )) + out = layers.while_loop(cond, body, (i,)) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) res = exe.run(main_program, fetch_list=out) - np.testing.assert_allclose(np.asarray(res[0]), - np.full(1, 10, np.int64), - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(res[0]), np.full(1, 10, np.int64), rtol=1e-05 + ) def test_var_list(self): - def cond(i, mem): return layers.less_than(i, ten) @@ -73,8 +73,11 @@ class TestApiWhileLoop(unittest.TestCase): data = np.random.rand(10).astype('float32') data_one = np.ones(10).astype('float32') - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) res = exe.run(main_program, feed={'mem': data}, fetch_list=out) for i in range(10): @@ -82,7 +85,6 @@ class TestApiWhileLoop(unittest.TestCase): np.testing.assert_allclose(np.asarray(res[1]), data, rtol=1e-05) def test_var_dict(self): - def cond(i, ten, test_dict, test_list, test_list_dict): return layers.less_than(i, ten) @@ -94,7 +96,8 @@ class TestApiWhileLoop(unittest.TestCase): test_list_dict[0]["test_key"] += 1 test_list_dict[0]["test_key"] = fluid.layers.relu( - test_list_dict[0]["test_key"]) + test_list_dict[0]["test_key"] + ) i = layers.increment(i) return [i, ten, test_dict, test_list, test_list_dict] @@ -110,47 +113,54 @@ class TestApiWhileLoop(unittest.TestCase): test_list = [ layers.fill_constant(shape=[1, 2], dtype='int64', value=0) ] - test_list_dict = [{ - "test_key": - layers.fill_constant(shape=[1], dtype='float32', value=0) - }] + test_list_dict = [ + { + "test_key": layers.fill_constant( + shape=[1], dtype='float32', value=0 + ) + } + ] i, ten, test_dict, test_list, test_list_dict = layers.while_loop( - cond, body, [i, ten, test_dict, test_list, test_list_dict]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + cond, body, [i, ten, test_dict, test_list, test_list_dict] + ) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - fetch_list=[ - test_dict["test_key"], test_list[0], - test_list_dict[0]["test_key"] - ]) - np.testing.assert_allclose(np.asarray(res[0]), - np.full(shape=1, - fill_value=10, - dtype=np.int64), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(res[1]), - np.full(shape=(2, 1), - fill_value=10, - dtype=np.int64), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(res[2]), - np.full(shape=1, - fill_value=10, - dtype=np.float32), - rtol=1e-05) + res = exe.run( + main_program, + fetch_list=[ + test_dict["test_key"], + test_list[0], + test_list_dict[0]["test_key"], + ], + ) + np.testing.assert_allclose( + np.asarray(res[0]), + np.full(shape=1, fill_value=10, dtype=np.int64), + rtol=1e-05, + ) + np.testing.assert_allclose( + np.asarray(res[1]), + np.full(shape=(2, 1), fill_value=10, dtype=np.int64), + rtol=1e-05, + ) + np.testing.assert_allclose( + np.asarray(res[2]), + np.full(shape=1, fill_value=10, dtype=np.float32), + rtol=1e-05, + ) class TestApiWhileLoop_Nested(unittest.TestCase): - def test_nested_net(self): - def external_cond(i, j, init, sums): return layers.less_than(i, loop_len1) def external_body(i, j, init, sums): - def internal_cond(j, init, sums): return layers.less_than(j, loop_len2) @@ -160,8 +170,9 @@ class TestApiWhileLoop_Nested(unittest.TestCase): j = layers.increment(j) return [j, init, sums] - result = layers.while_loop(internal_cond, internal_body, - [j, init, sums]) + result = layers.while_loop( + internal_cond, internal_body, [j, init, sums] + ) j = result[0] init = result[1] sums = result[2] @@ -180,21 +191,22 @@ class TestApiWhileLoop_Nested(unittest.TestCase): loop_len2 = layers.fill_constant(shape=[1], dtype='int64', value=3) ones = layers.fill_constant(shape=[3, 3], dtype='float32', value=1) - out = layers.while_loop(external_cond, external_body, - [i, j, init, sums]) + out = layers.while_loop( + external_cond, external_body, [i, j, init, sums] + ) data = np.random.rand(3, 3).astype('float32') data_sums = np.zeros([3, 3]).astype('float32') - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - feed={ - 'init': data, - 'sums': data_sums - }, - fetch_list=out) + res = exe.run( + main_program, feed={'init': data, 'sums': data_sums}, fetch_list=out + ) for i in range(3): data = np.add(data, 1) data_sums = np.add(data, data_sums) @@ -204,9 +216,7 @@ class TestApiWhileLoop_Nested(unittest.TestCase): class TestApiWhileLoop_Backward(unittest.TestCase): - def test_while_loop_backward(self): - def cond(i, x): return layers.less_than(i, eleven) @@ -229,8 +239,11 @@ class TestApiWhileLoop_Backward(unittest.TestCase): mean = paddle.mean(out[1]) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feed_i = np.ones(1).astype('float32') @@ -238,17 +251,15 @@ class TestApiWhileLoop_Backward(unittest.TestCase): data = np.asarray([100]).astype('float32') i_grad = np.asarray([110]).astype('float32') - res = exe.run(main_program, - feed={ - 'i': feed_i, - 'x': feed_x - }, - fetch_list=[mean.name, i.grad_name]) + res = exe.run( + main_program, + feed={'i': feed_i, 'x': feed_x}, + fetch_list=[mean.name, i.grad_name], + ) np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) def test_while_loop_backward2(self): - def cond(i, x): return i < 3 @@ -269,8 +280,11 @@ class TestApiWhileLoop_Backward(unittest.TestCase): mean = paddle.mean(out[1]) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feed_i = np.ones(1).astype('float32') @@ -279,26 +293,22 @@ class TestApiWhileLoop_Backward(unittest.TestCase): i_grad = np.asarray([3]).astype('float32') x_grad = np.asarray([2]).astype('float32') - res = exe.run(main_program, - feed={ - 'i': feed_i, - 'x': feed_x - }, - fetch_list=[mean.name, i.grad_name, x.grad_name]) + res = exe.run( + main_program, + feed={'i': feed_i, 'x': feed_x}, + fetch_list=[mean.name, i.grad_name, x.grad_name], + ) np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[2]), x_grad, rtol=1e-05) class TestApiWhileLoop_NestedWithBackwardAndLoDTensorArray(unittest.TestCase): - def test_nested_net_with_backward_and_lodtensor(self): - def external_cond(i, j, x, mem_array): return layers.less_than(i, array_len) def external_body(i, j, x, mem_array): - def internal_cond(j, x, mem_array): return layers.less_than(j, array_len2) @@ -317,8 +327,9 @@ class TestApiWhileLoop_NestedWithBackwardAndLoDTensorArray(unittest.TestCase): outer_sum_1 = layers.elementwise_add(x=x, y=outer_sum_0) i = layers.increment(x=i, in_place=True) layers.array_write(outer_sum_1, i=i, array=mem_array) - j, x, mem_array = layers.while_loop(internal_cond, internal_body, - [j, x, mem_array]) + j, x, mem_array = layers.while_loop( + internal_cond, internal_body, [j, x, mem_array] + ) return [i, j, x, mem_array] main_program = Program() @@ -345,15 +356,19 @@ class TestApiWhileLoop_NestedWithBackwardAndLoDTensorArray(unittest.TestCase): j.stop_gradient = True array_len2 = layers.fill_constant(shape=[1], dtype='int64', value=3) - out = layers.while_loop(external_cond, external_body, - [i, j, x, mem_array]) + out = layers.while_loop( + external_cond, external_body, [i, j, x, mem_array] + ) sum_result = layers.array_read(array=mem_array, i=j) mean = paddle.mean(sum_result) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) d = [] @@ -362,27 +377,21 @@ class TestApiWhileLoop_NestedWithBackwardAndLoDTensorArray(unittest.TestCase): feed_x = np.ones(10).astype('float32') data_sum = d[0] + d[1] + d[2] + 3 * feed_x x_grad = [0.3] * 10 - res = exe.run(main_program, - feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2], - 'x': feed_x - }, - fetch_list=[sum_result.name, x.grad_name]) + res = exe.run( + main_program, + feed={'d0': d[0], 'd1': d[1], 'd2': d[2], 'x': feed_x}, + fetch_list=[sum_result.name, x.grad_name], + ) np.testing.assert_allclose(res[0], data_sum, rtol=1e-05) np.testing.assert_allclose(res[1], x_grad, rtol=1e-05) class TestApiWhileLoopWithSwitchCase(unittest.TestCase): - def test_with_switch_case(self): - def cond(i): return layers.less_than(i, ten) def body(i): - def fn_add_three(): data_add_three = layers.elementwise_add(x=i, y=three) return data_add_three @@ -395,12 +404,11 @@ class TestApiWhileLoopWithSwitchCase(unittest.TestCase): data_add_one = layers.elementwise_add(x=i, y=one) return data_add_one - return layers.switch_case(branch_index=i, - branch_fns={ - 2: fn_add_three, - 5: fn_square - }, - default=fn_add_one) + return layers.switch_case( + branch_index=i, + branch_fns={2: fn_add_three, 5: fn_square}, + default=fn_add_one, + ) main_program = Program() startup_program = Program() @@ -411,8 +419,11 @@ class TestApiWhileLoopWithSwitchCase(unittest.TestCase): one = layers.fill_constant(shape=[1], dtype='int64', value=1) out = layers.while_loop(cond, body, [i]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) res = exe.run(main_program, fetch_list=out) @@ -421,9 +432,7 @@ class TestApiWhileLoopWithSwitchCase(unittest.TestCase): class TestApiWhileLoop_Error(unittest.TestCase): - def test_error(self): - def cond_returns_constant(i): return 1 @@ -453,9 +462,9 @@ class TestApiWhileLoop_Error(unittest.TestCase): return i > 0 def body_returns_with_mutable_dict(i, test_dict): - test_dict['new_key'] = layers.fill_constant(shape=[1], - dtype='int64', - value=1) + test_dict['new_key'] = layers.fill_constant( + shape=[1], dtype='int64', value=1 + ) return layers.increment(i), test_dict def cond_returns_with_mutable_list(i, test_list): @@ -463,7 +472,8 @@ class TestApiWhileLoop_Error(unittest.TestCase): def body_returns_with_mutable_list(i, test_list): test_list.append( - layers.fill_constant(shape=[1], dtype='int64', value=1)) + layers.fill_constant(shape=[1], dtype='int64', value=1) + ) return layers.increment(i), test_list main_program = Program() @@ -483,8 +493,9 @@ class TestApiWhileLoop_Error(unittest.TestCase): # The type of `body` in Op(while_loop) must be callable def type_error_body(): - out = layers.while_loop(cond_returns_bool_tensor, data, - [data_1d]) + out = layers.while_loop( + cond_returns_bool_tensor, data, [data_1d] + ) self.assertRaises(TypeError, type_error_body) @@ -508,8 +519,9 @@ class TestApiWhileLoop_Error(unittest.TestCase): # The type of `cond` returns in Op(while_loop) must be a bollean variable def type_error_cond_returns_not_boolean(): - out = layers.while_loop(cond_returns_not_bool_tensor, body, - [data_1d]) + out = layers.while_loop( + cond_returns_not_bool_tensor, body, [data_1d] + ) self.assertRaises(TypeError, type_error_cond_returns_not_boolean) @@ -521,47 +533,54 @@ class TestApiWhileLoop_Error(unittest.TestCase): # The length of `body` returns in Op(while_loop) must be same as `loop_vars` def value_error_body_returns_error_length(): - out = layers.while_loop(cond_returns_bool_tensor, - body_returns_error_length, [data]) + out = layers.while_loop( + cond_returns_bool_tensor, body_returns_error_length, [data] + ) self.assertRaises(ValueError, value_error_body_returns_error_length) # The type of `body` returns in Op(while_loop) must be same as `loop_vars` def value_error_body_returns_error_type(): - out = layers.while_loop(cond_receives_two_args, - body_returns_error_type, [data, ten]) + out = layers.while_loop( + cond_receives_two_args, body_returns_error_type, [data, ten] + ) self.assertRaises(ValueError, value_error_body_returns_error_type) # The length of `output_vars` with mutable value should keep same with `loop_vars` def value_error_body_returns_with_mutable_dict(): test_dict = { - "int_constant": - layers.fill_constant(shape=[2, 2], dtype='int64', value=1) + "int_constant": layers.fill_constant( + shape=[2, 2], dtype='int64', value=1 + ) } - out = layers.while_loop(cond_returns_with_mutable_dict, - body_returns_with_mutable_dict, - [data, test_dict]) + out = layers.while_loop( + cond_returns_with_mutable_dict, + body_returns_with_mutable_dict, + [data, test_dict], + ) - self.assertRaises(ValueError, - value_error_body_returns_with_mutable_dict) + self.assertRaises( + ValueError, value_error_body_returns_with_mutable_dict + ) def value_error_body_returns_with_mutable_list(): test_list = [ layers.fill_constant(shape=[2, 2], dtype='int64', value=1) ] - out = layers.while_loop(cond_returns_with_mutable_list, - body_returns_with_mutable_list, - [data, test_list]) + out = layers.while_loop( + cond_returns_with_mutable_list, + body_returns_with_mutable_list, + [data, test_list], + ) - self.assertRaises(ValueError, - value_error_body_returns_with_mutable_list) + self.assertRaises( + ValueError, value_error_body_returns_with_mutable_list + ) class TestApiWhileLoopSliceInBody(unittest.TestCase): - def test_var_slice(self): - def cond(z, i): return i + 1 <= x_shape[0] @@ -579,8 +598,11 @@ class TestApiWhileLoopSliceInBody(unittest.TestCase): i = fluid.layers.fill_constant([1], 'int32', 0) z, _ = fluid.layers.while_loop(cond, body, [z, i]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) np_x = np.array([1, 2, 3, 4, 5], dtype='int32') diff --git a/python/paddle/fluid/tests/unittests/test_while_op.py b/python/paddle/fluid/tests/unittests/test_while_op.py index 648e93dede8..341e6a8ef56 100644 --- a/python/paddle/fluid/tests/unittests/test_while_op.py +++ b/python/paddle/fluid/tests/unittests/test_while_op.py @@ -25,20 +25,16 @@ paddle.enable_static() class TestWhileOp(unittest.TestCase): - def simple_net(self): - d0 = layers.data("d0", - shape=[10], - append_batch_size=False, - dtype='float32') - d1 = layers.data("d1", - shape=[10], - append_batch_size=False, - dtype='float32') - d2 = layers.data("d2", - shape=[10], - append_batch_size=False, - dtype='float32') + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32' + ) + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32' + ) + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32' + ) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = True init = layers.zeros(shape=[10], dtype='float32') @@ -96,12 +92,10 @@ class TestWhileOp(unittest.TestCase): for i in range(3): d.append(numpy.random.random(size=[10]).astype('float32')) - outs = exe.run(feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2] - }, - fetch_list=[sum_result]) + outs = exe.run( + feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, + fetch_list=[sum_result], + ) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) def test_simple_net_forward(self): @@ -133,7 +127,6 @@ class TestWhileOp(unittest.TestCase): class BadInputTest(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): @@ -145,9 +138,7 @@ class BadInputTest(unittest.TestCase): class TestIgnoreVarNameInWhile(unittest.TestCase): - def test_ignore_var(self): - def cond(i, ten, temp, y): return i < ten @@ -164,8 +155,9 @@ class TestIgnoreVarNameInWhile(unittest.TestCase): i = layers.fill_constant(shape=[1], value=0, dtype='int32') num = layers.fill_constant(shape=[1], value=5, dtype='int32') - i, ten, shuffle_temp, y = layers.while_loop(cond, body_func, - [i, num, temp, y]) + i, ten, shuffle_temp, y = layers.while_loop( + cond, body_func, [i, num, temp, y] + ) output = shuffle_temp @@ -177,18 +169,16 @@ class TestIgnoreVarNameInWhile(unittest.TestCase): input_y = numpy.array([[10], [12], [33]]) input_y = input_y.reshape(3, 1, 1) - res, = exe.run(fluid.default_main_program(), - feed={ - 'x': input_x, - 'y': input_y - }, - fetch_list=[output]) + (res,) = exe.run( + fluid.default_main_program(), + feed={'x': input_x, 'y': input_y}, + fetch_list=[output], + ) self.assertListEqual(list(res.shape), [3, 1, 5]) class TestOutputsMustExistsInputs(unittest.TestCase): - def test_outputs_exists_inputs(self): """ We guarantee that the output tensor must be in the input tensor, so that the output and input can correspond to each other, but the input can be greater than the number of outputs. It's required in paddle2onnx. @@ -220,11 +210,14 @@ class TestOutputsMustExistsInputs(unittest.TestCase): for op in main_program.block(0).ops: if op.type == "while": for out_name in op.output("Out"): - if out_name in op.input("Condition"): continue + if out_name in op.input("Condition"): + continue self.assertTrue( out_name in op.input("X"), - "In while op, the variable in output(`Out`) must exists in inputs(`X`), but the variable with name `{}` not meet the precondition." - .format(out_name)) + "In while op, the variable in output(`Out`) must exists in inputs(`X`), but the variable with name `{}` not meet the precondition.".format( + out_name + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_yolo_box_op.py b/python/paddle/fluid/tests/unittests/test_yolo_box_op.py index 0a4897b821d..8584332f02c 100644 --- a/python/paddle/fluid/tests/unittests/test_yolo_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_yolo_box_op.py @@ -20,7 +20,7 @@ from paddle.fluid.framework import _test_eager_guard def sigmoid(x): - return (1.0 / (1.0 + np.exp(((-1.0) * x)))) + return 1.0 / (1.0 + np.exp(((-1.0) * x))) def YoloBox(x, img_size, attrs): @@ -34,9 +34,9 @@ def YoloBox(x, img_size, attrs): scale_x_y = attrs['scale_x_y'] iou_aware = attrs['iou_aware'] iou_aware_factor = attrs['iou_aware_factor'] - bias_x_y = ((-0.5) * (scale_x_y - 1.0)) - input_h = (downsample * h) - input_w = (downsample * w) + bias_x_y = (-0.5) * (scale_x_y - 1.0) + input_h = downsample * h + input_w = downsample * w if iou_aware: ioup = x[:, :an_num, :, :] ioup = np.expand_dims(ioup, axis=(-1)) @@ -45,49 +45,54 @@ def YoloBox(x, img_size, attrs): pred_box = x[:, :, :, :, :4].copy() grid_x = np.tile(np.arange(w).reshape((1, w)), (h, 1)) grid_y = np.tile(np.arange(h).reshape((h, 1)), (1, w)) - pred_box[:, :, :, :, 0] = (( - (grid_x + (sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y)) + bias_x_y) / - w) - pred_box[:, :, :, :, 1] = (( - (grid_y + (sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y)) + bias_x_y) / - h) - anchors = [(anchors[i], anchors[(i + 1)]) - for i in range(0, len(anchors), 2)] - anchors_s = np.array([((an_w / input_w), (an_h / input_h)) - for (an_w, an_h) in anchors]) + pred_box[:, :, :, :, 0] = ( + (grid_x + (sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y)) + bias_x_y + ) / w + pred_box[:, :, :, :, 1] = ( + (grid_y + (sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y)) + bias_x_y + ) / h + anchors = [ + (anchors[i], anchors[(i + 1)]) for i in range(0, len(anchors), 2) + ] + anchors_s = np.array( + [((an_w / input_w), (an_h / input_h)) for (an_w, an_h) in anchors] + ) anchor_w = anchors_s[:, 0:1].reshape((1, an_num, 1, 1)) anchor_h = anchors_s[:, 1:2].reshape((1, an_num, 1, 1)) - pred_box[:, :, :, :, 2] = (np.exp(pred_box[:, :, :, :, 2]) * anchor_w) - pred_box[:, :, :, :, 3] = (np.exp(pred_box[:, :, :, :, 3]) * anchor_h) + pred_box[:, :, :, :, 2] = np.exp(pred_box[:, :, :, :, 2]) * anchor_w + pred_box[:, :, :, :, 3] = np.exp(pred_box[:, :, :, :, 3]) * anchor_h if iou_aware: - pred_conf = ((sigmoid(x[:, :, :, :, 4:5])**(1 - iou_aware_factor)) * - (sigmoid(ioup)**iou_aware_factor)) + pred_conf = (sigmoid(x[:, :, :, :, 4:5]) ** (1 - iou_aware_factor)) * ( + sigmoid(ioup) ** iou_aware_factor + ) else: pred_conf = sigmoid(x[:, :, :, :, 4:5]) pred_conf[(pred_conf < conf_thresh)] = 0.0 - pred_score = (sigmoid(x[:, :, :, :, 5:]) * pred_conf) - pred_box = (pred_box * (pred_conf > 0.0).astype('float32')) + pred_score = sigmoid(x[:, :, :, :, 5:]) * pred_conf + pred_box = pred_box * (pred_conf > 0.0).astype('float32') pred_box = pred_box.reshape((n, (-1), 4)) - (pred_box[:, :, :2], - pred_box[:, :, 2:4]) = ((pred_box[:, :, :2] - (pred_box[:, :, 2:4] / 2.0)), - (pred_box[:, :, :2] + (pred_box[:, :, 2:4] / 2.0))) - pred_box[:, :, 0] = (pred_box[:, :, 0] * img_size[:, 1][:, np.newaxis]) - pred_box[:, :, 1] = (pred_box[:, :, 1] * img_size[:, 0][:, np.newaxis]) - pred_box[:, :, 2] = (pred_box[:, :, 2] * img_size[:, 1][:, np.newaxis]) - pred_box[:, :, 3] = (pred_box[:, :, 3] * img_size[:, 0][:, np.newaxis]) + (pred_box[:, :, :2], pred_box[:, :, 2:4]) = ( + (pred_box[:, :, :2] - (pred_box[:, :, 2:4] / 2.0)), + (pred_box[:, :, :2] + (pred_box[:, :, 2:4] / 2.0)), + ) + pred_box[:, :, 0] = pred_box[:, :, 0] * img_size[:, 1][:, np.newaxis] + pred_box[:, :, 1] = pred_box[:, :, 1] * img_size[:, 0][:, np.newaxis] + pred_box[:, :, 2] = pred_box[:, :, 2] * img_size[:, 1][:, np.newaxis] + pred_box[:, :, 3] = pred_box[:, :, 3] * img_size[:, 0][:, np.newaxis] if clip_bbox: for i in range(len(pred_box)): pred_box[i, :, 0] = np.clip(pred_box[i, :, 0], 0, np.inf) pred_box[i, :, 1] = np.clip(pred_box[i, :, 1], 0, np.inf) - pred_box[i, :, 2] = np.clip(pred_box[i, :, 2], (-np.inf), - (img_size[(i, 1)] - 1)) - pred_box[i, :, 3] = np.clip(pred_box[i, :, 3], (-np.inf), - (img_size[(i, 0)] - 1)) + pred_box[i, :, 2] = np.clip( + pred_box[i, :, 2], (-np.inf), (img_size[(i, 1)] - 1) + ) + pred_box[i, :, 3] = np.clip( + pred_box[i, :, 3], (-np.inf), (img_size[(i, 0)] - 1) + ) return (pred_box, pred_score.reshape((n, (-1), class_num))) class TestYoloBoxOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'yolo_box' @@ -102,7 +107,7 @@ class TestYoloBoxOp(OpTest): 'clip_bbox': self.clip_bbox, 'scale_x_y': self.scale_x_y, 'iou_aware': self.iou_aware, - 'iou_aware_factor': self.iou_aware_factor + 'iou_aware_factor': self.iou_aware_factor, } self.inputs = {'X': x, 'ImgSize': img_size} (boxes, scores) = YoloBox(x, img_size, self.attrs) @@ -119,8 +124,12 @@ class TestYoloBoxOp(OpTest): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = False @@ -128,7 +137,6 @@ class TestYoloBoxOp(OpTest): class TestYoloBoxOpNoClipBbox(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -137,8 +145,12 @@ class TestYoloBoxOpNoClipBbox(TestYoloBoxOp): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = False - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = False @@ -146,7 +158,6 @@ class TestYoloBoxOpNoClipBbox(TestYoloBoxOp): class TestYoloBoxOpScaleXY(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -155,8 +166,12 @@ class TestYoloBoxOpScaleXY(TestYoloBoxOp): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.2 self.iou_aware = False @@ -164,7 +179,6 @@ class TestYoloBoxOpScaleXY(TestYoloBoxOp): class TestYoloBoxOpIoUAware(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -173,8 +187,12 @@ class TestYoloBoxOpIoUAware(TestYoloBoxOp): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (6 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (6 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = True @@ -182,34 +200,37 @@ class TestYoloBoxOpIoUAware(TestYoloBoxOp): class TestYoloBoxDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() img_size = np.ones((2, 2)).astype('int32') img_size = paddle.to_tensor(img_size) x1 = np.random.random([2, 14, 8, 8]).astype('float32') x1 = paddle.to_tensor(x1) - (boxes, scores) = paddle.vision.ops.yolo_box(x1, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x1, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + ) + assert (boxes is not None) and (scores is not None) x2 = np.random.random([2, 16, 8, 8]).astype('float32') x2 = paddle.to_tensor(x2) - (boxes, scores) = paddle.vision.ops.yolo_box(x2, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0, - iou_aware=True, - iou_aware_factor=0.5) + (boxes, scores) = paddle.vision.ops.yolo_box( + x2, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + iou_aware=True, + iou_aware_factor=0.5, + ) paddle.enable_static() def test_eager(self): @@ -218,35 +239,37 @@ class TestYoloBoxDygraph(unittest.TestCase): class TestYoloBoxStatic(unittest.TestCase): - def test_static(self): x1 = paddle.static.data('x1', [2, 14, 8, 8], 'float32') img_size = paddle.static.data('img_size', [2, 2], 'int32') - (boxes, scores) = paddle.vision.ops.yolo_box(x1, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x1, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + ) + assert (boxes is not None) and (scores is not None) x2 = paddle.static.data('x2', [2, 16, 8, 8], 'float32') - (boxes, scores) = paddle.vision.ops.yolo_box(x2, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0, - iou_aware=True, - iou_aware_factor=0.5) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x2, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + iou_aware=True, + iou_aware_factor=0.5, + ) + assert (boxes is not None) and (scores is not None) class TestYoloBoxOpHW(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) diff --git a/python/paddle/fluid/tests/unittests/test_yolov3_loss_op.py b/python/paddle/fluid/tests/unittests/test_yolov3_loss_op.py index c2c747836ee..9105f27e96d 100644 --- a/python/paddle/fluid/tests/unittests/test_yolov3_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_yolov3_loss_op.py @@ -51,11 +51,12 @@ def batch_xywh_box_iou(box1, box2): left = np.maximum(b1_left[:, :, np.newaxis], b2_left[:, np.newaxis, :]) right = np.minimum(b1_right[:, :, np.newaxis], b2_right[:, np.newaxis, :]) top = np.maximum(b1_top[:, :, np.newaxis], b2_top[:, np.newaxis, :]) - bottom = np.minimum(b1_bottom[:, :, np.newaxis], b2_bottom[:, - np.newaxis, :]) + bottom = np.minimum( + b1_bottom[:, :, np.newaxis], b2_bottom[:, np.newaxis, :] + ) - inter_w = np.clip(right - left, 0., 1.) - inter_h = np.clip(bottom - top, 0., 1.) + inter_w = np.clip(right - left, 0.0, 1.0) + inter_h = np.clip(bottom - top, 0.0, 1.0) inter_area = inter_w * inter_h b1_area = (b1_right - b1_left) * (b1_bottom - b1_top) @@ -77,7 +78,7 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): downsample_ratio = attrs['downsample_ratio'] use_label_smooth = attrs['use_label_smooth'] scale_x_y = attrs['scale_x_y'] - bias_x_y = -0.5 * (scale_x_y - 1.) + bias_x_y = -0.5 * (scale_x_y - 1.0) input_size = downsample_ratio * h x = x.reshape((n, mask_num, 5 + class_num, h, w)).transpose((0, 1, 3, 4, 2)) loss = np.zeros((n)).astype('float64') @@ -89,18 +90,19 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): pred_box = x[:, :, :, :, :4].copy() grid_x = np.tile(np.arange(w).reshape((1, w)), (h, 1)) grid_y = np.tile(np.arange(h).reshape((h, 1)), (1, w)) - pred_box[:, :, :, :, - 0] = (grid_x + sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y + - bias_x_y) / w - pred_box[:, :, :, :, - 1] = (grid_y + sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y + - bias_x_y) / h + pred_box[:, :, :, :, 0] = ( + grid_x + sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y + bias_x_y + ) / w + pred_box[:, :, :, :, 1] = ( + grid_y + sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y + bias_x_y + ) / h mask_anchors = [] for m in anchor_mask: mask_anchors.append((anchors[2 * m], anchors[2 * m + 1])) - anchors_s = np.array([(an_w / input_size, an_h / input_size) - for an_w, an_h in mask_anchors]) + anchors_s = np.array( + [(an_w / input_size, an_h / input_size) for an_w, an_h in mask_anchors] + ) anchor_w = anchors_s[:, 0:1].reshape((1, mask_num, 1, 1)) anchor_h = anchors_s[:, 1:2].reshape((1, mask_num, 1, 1)) pred_box[:, :, :, :, 2] = np.exp(pred_box[:, :, :, :, 2]) * anchor_w @@ -111,18 +113,21 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): objness = np.zeros(pred_box.shape[:2]).astype('float64') ious = batch_xywh_box_iou(pred_box, gtbox) ious_max = np.max(ious, axis=-1) - objness = np.where(ious_max > ignore_thresh, -np.ones_like(objness), - objness) + objness = np.where( + ious_max > ignore_thresh, -np.ones_like(objness), objness + ) gtbox_shift = gtbox.copy() gtbox_shift[:, :, 0] = 0 gtbox_shift[:, :, 1] = 0 anchors = [(anchors[2 * i], anchors[2 * i + 1]) for i in range(0, an_num)] - anchors_s = np.array([(an_w / input_size, an_h / input_size) - for an_w, an_h in anchors]) - anchor_boxes = np.concatenate([np.zeros_like(anchors_s), anchors_s], - axis=-1) + anchors_s = np.array( + [(an_w / input_size, an_h / input_size) for an_w, an_h in anchors] + ) + anchor_boxes = np.concatenate( + [np.zeros_like(anchors_s), anchors_s], axis=-1 + ) anchor_boxes = np.tile(anchor_boxes[np.newaxis, :, :], (n, 1, 1)) ious = batch_xywh_box_iou(gtbox_shift, anchor_boxes) iou_matches = np.argmax(ious, axis=-1) @@ -153,9 +158,13 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): objness[i, an_idx * h * w + gj * w + gi] = gtscore[i, j] for label_idx in range(class_num): - loss[i] += sce( - x[i, an_idx, gj, gi, 5 + label_idx], label_pos if label_idx - == gtlabel[i, j] else label_neg) * gtscore[i, j] + loss[i] += ( + sce( + x[i, an_idx, gj, gi, 5 + label_idx], + label_pos if label_idx == gtlabel[i, j] else label_neg, + ) + * gtscore[i, j] + ) for j in range(mask_num * h * w): if objness[i, j] > 0: @@ -163,40 +172,62 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): elif objness[i, j] == 0: loss[i] += sce(pred_obj[i, j], 0.0) - return (loss, objness.reshape((n, mask_num, h, w)).astype('float64'), \ - gt_matches.astype('int32')) - - -def yolo_loss_wrapper(x, - gt_box, - gt_label, - gt_score=None, - anchors=[ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, - 90, 156, 198, 373, 326 - ], - anchor_mask=[0, 1, 2], - class_num=5, - ignore_thresh=0.7, - downsample_ratio=32, - use_label_smooth=True, - scale_x_y=1.): - loss = paddle.vision.ops.yolo_loss(x, - gt_box=gt_box, - gt_label=gt_label, - anchors=anchors, - anchor_mask=anchor_mask, - class_num=class_num, - ignore_thresh=ignore_thresh, - downsample_ratio=downsample_ratio, - gt_score=gt_score, - use_label_smooth=use_label_smooth, - scale_x_y=scale_x_y) + return ( + loss, + objness.reshape((n, mask_num, h, w)).astype('float64'), + gt_matches.astype('int32'), + ) + + +def yolo_loss_wrapper( + x, + gt_box, + gt_label, + gt_score=None, + anchors=[ + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, + ], + anchor_mask=[0, 1, 2], + class_num=5, + ignore_thresh=0.7, + downsample_ratio=32, + use_label_smooth=True, + scale_x_y=1.0, +): + loss = paddle.vision.ops.yolo_loss( + x, + gt_box=gt_box, + gt_label=gt_label, + anchors=anchors, + anchor_mask=anchor_mask, + class_num=class_num, + ignore_thresh=ignore_thresh, + downsample_ratio=downsample_ratio, + gt_score=gt_score, + use_label_smooth=use_label_smooth, + scale_x_y=scale_x_y, + ) return loss class TestYolov3LossOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'yolov3_loss' @@ -230,12 +261,13 @@ class TestYolov3LossOp(OpTest): gtscore = np.random.random(self.gtbox_shape[:2]).astype('float64') self.inputs['GTScore'] = gtscore - loss, objness, gt_matches = YOLOv3Loss(x, gtbox, gtlabel, gtscore, - self.attrs) + loss, objness, gt_matches = YOLOv3Loss( + x, gtbox, gtlabel, gtscore, self.attrs + ) self.outputs = { 'Loss': loss, 'ObjectnessMask': objness, - "GTMatchMask": gt_matches + "GTMatchMask": gt_matches, } def test_check_output(self): @@ -244,15 +276,30 @@ class TestYolov3LossOp(OpTest): def test_check_grad_ignore_gtbox(self): place = core.CPUPlace() - self.check_grad_with_place(place, ['X'], - 'Loss', - max_relative_error=0.2, - check_eager=True) + self.check_grad_with_place( + place, ['X'], 'Loss', max_relative_error=0.2, check_eager=True + ) def initTestCase(self): self.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, - 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] self.anchor_mask = [0, 1, 2] self.class_num = 5 @@ -262,15 +309,30 @@ class TestYolov3LossOp(OpTest): self.gtbox_shape = (3, 5, 4) self.gtscore = True self.use_label_smooth = True - self.scale_x_y = 1. + self.scale_x_y = 1.0 class TestYolov3LossWithoutLabelSmooth(TestYolov3LossOp): - def initTestCase(self): self.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, - 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] self.anchor_mask = [0, 1, 2] self.class_num = 5 @@ -280,15 +342,30 @@ class TestYolov3LossWithoutLabelSmooth(TestYolov3LossOp): self.gtbox_shape = (3, 5, 4) self.gtscore = True self.use_label_smooth = False - self.scale_x_y = 1. + self.scale_x_y = 1.0 class TestYolov3LossNoGTScore(TestYolov3LossOp): - def initTestCase(self): self.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, - 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] self.anchor_mask = [0, 1, 2] self.class_num = 5 @@ -298,15 +375,30 @@ class TestYolov3LossNoGTScore(TestYolov3LossOp): self.gtbox_shape = (3, 5, 4) self.gtscore = False self.use_label_smooth = True - self.scale_x_y = 1. + self.scale_x_y = 1.0 class TestYolov3LossWithScaleXY(TestYolov3LossOp): - def initTestCase(self): self.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, - 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] self.anchor_mask = [0, 1, 2] self.class_num = 5 @@ -320,7 +412,6 @@ class TestYolov3LossWithScaleXY(TestYolov3LossOp): class TestYolov3LossDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() x = np.random.random([2, 14, 8, 8]).astype('float32') @@ -331,52 +422,57 @@ class TestYolov3LossDygraph(unittest.TestCase): gt_box = paddle.to_tensor(gt_box) gt_label = paddle.to_tensor(gt_label) - loss = paddle.vision.ops.yolo_loss(x, - gt_box=gt_box, - gt_label=gt_label, - anchors=[10, 13, 16, 30], - anchor_mask=[0, 1], - class_num=2, - ignore_thresh=0.7, - downsample_ratio=8, - use_label_smooth=True, - scale_x_y=1.) + loss = paddle.vision.ops.yolo_loss( + x, + gt_box=gt_box, + gt_label=gt_label, + anchors=[10, 13, 16, 30], + anchor_mask=[0, 1], + class_num=2, + ignore_thresh=0.7, + downsample_ratio=8, + use_label_smooth=True, + scale_x_y=1.0, + ) assert loss is not None assert loss.shape == [2] paddle.enable_static() class TestYolov3LossStatic(unittest.TestCase): - def test_static(self): x = paddle.static.data('x', [2, 14, 8, 8], 'float32') gt_box = paddle.static.data('gt_box', [2, 10, 4], 'float32') gt_label = paddle.static.data('gt_label', [2, 10], 'int32') gt_score = paddle.static.data('gt_score', [2, 10], 'float32') - loss = paddle.vision.ops.yolo_loss(x, - gt_box=gt_box, - gt_label=gt_label, - anchors=[10, 13, 16, 30], - anchor_mask=[0, 1], - class_num=2, - ignore_thresh=0.7, - downsample_ratio=8, - gt_score=gt_score, - use_label_smooth=True, - scale_x_y=1.) + loss = paddle.vision.ops.yolo_loss( + x, + gt_box=gt_box, + gt_label=gt_label, + anchors=[10, 13, 16, 30], + anchor_mask=[0, 1], + class_num=2, + ignore_thresh=0.7, + downsample_ratio=8, + gt_score=gt_score, + use_label_smooth=True, + scale_x_y=1.0, + ) assert loss is not None - loss = paddle.vision.ops.yolo_loss(x, - gt_box=gt_box, - gt_label=gt_label, - anchors=[10, 13, 16, 30], - anchor_mask=[0, 1], - class_num=2, - ignore_thresh=0.7, - downsample_ratio=8, - use_label_smooth=True, - scale_x_y=1.) + loss = paddle.vision.ops.yolo_loss( + x, + gt_box=gt_box, + gt_label=gt_label, + anchors=[10, 13, 16, 30], + anchor_mask=[0, 1], + class_num=2, + ignore_thresh=0.7, + downsample_ratio=8, + use_label_smooth=True, + scale_x_y=1.0, + ) assert loss is not None diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_shape.py b/python/paddle/fluid/tests/unittests/test_zero_dim_shape.py index 089a076ad80..df4fa96d4a3 100644 --- a/python/paddle/fluid/tests/unittests/test_zero_dim_shape.py +++ b/python/paddle/fluid/tests/unittests/test_zero_dim_shape.py @@ -68,7 +68,6 @@ unary_api_list = [ # Use to test zero-dim in the whole API class TestUnaryAPI(unittest.TestCase): - def test_dygraph_unary(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -108,13 +107,14 @@ class TestUnaryAPI(unittest.TestCase): self.assertEqual(out.shape, ()) exe = fluid.Executor() - result = exe.run(main_prog, - fetch_list=[x, out, x_grad, out_grad]) + result = exe.run( + main_prog, fetch_list=[x, out, x_grad, out_grad] + ) # Test runtime shape self.assertEqual(result[0].shape, ()) self.assertEqual(result[1].shape, ()) - self.assertEqual(result[3].shape, (1, )) + self.assertEqual(result[3].shape, (1,)) # 0D will be stacked when 1+ place, due to it cannot be concated # for 1 place: [ x-place1 ] @@ -126,28 +126,34 @@ class TestUnaryAPI(unittest.TestCase): else: places = [paddle.CPUPlace()] * 4 device_num = 4 - expect_shape = (device_num, ) + expect_shape = (device_num,) compiled_program = fluid.CompiledProgram( - main_prog).with_data_parallel(out.name, places=places) - result = exe.run(compiled_program, - fetch_list=[x, out, x_grad, out_grad], - return_merged=True) + main_prog + ).with_data_parallel(out.name, places=places) + result = exe.run( + compiled_program, + fetch_list=[x, out, x_grad, out_grad], + return_merged=True, + ) # Test runtime parallel shape self.assertEqual(result[0].shape, expect_shape) self.assertEqual(result[1].shape, expect_shape) - self.assertEqual(result[3].shape, (device_num, )) + self.assertEqual(result[3].shape, (device_num,)) compiled_program = fluid.CompiledProgram( - main_prog).with_data_parallel(out.name, places=places) - result = exe.run(compiled_program, - fetch_list=[x, out, x_grad, out_grad], - return_merged=False) + main_prog + ).with_data_parallel(out.name, places=places) + result = exe.run( + compiled_program, + fetch_list=[x, out, x_grad, out_grad], + return_merged=False, + ) # [[x-place1, x-place2, ...], [], [], ...] - self.assertEqual(np.array(result[0]).shape, (device_num, )) - self.assertEqual(np.array(result[1]).shape, (device_num, )) + self.assertEqual(np.array(result[0]).shape, (device_num,)) + self.assertEqual(np.array(result[1]).shape, (device_num,)) self.assertEqual(np.array(result[3]).shape, (device_num, 1)) paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_zeropad2d.py b/python/paddle/fluid/tests/unittests/test_zeropad2d.py index f15f3d277d5..99a6b9143a9 100644 --- a/python/paddle/fluid/tests/unittests/test_zeropad2d.py +++ b/python/paddle/fluid/tests/unittests/test_zeropad2d.py @@ -85,7 +85,8 @@ class TestZeroPad2dAPI(unittest.TestCase): pad = [1, 2, 3, 4] x = np.random.randint(-255, 255, size=self.shape) expect_res = np.pad( - x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]]) + x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]] + ) x_tensor = to_tensor(x) ret_res = zeropad2d(x_tensor, pad).numpy() @@ -103,7 +104,8 @@ class TestZeroPad2dAPI(unittest.TestCase): pad = (1, 2, 3, 4) x = np.random.randint(-255, 255, size=self.shape) expect_res = np.pad( - x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]]) + x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]] + ) x_tensor = to_tensor(x) ret_res = zeropad2d(x_tensor, pad).numpy() @@ -121,7 +123,8 @@ class TestZeroPad2dAPI(unittest.TestCase): pad = [1, 2, 3, 4] x = np.random.randint(-255, 255, size=self.shape) expect_res = np.pad( - x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]]) + x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]] + ) x_tensor = to_tensor(x) pad_tensor = to_tensor(pad, dtype='int32') @@ -144,15 +147,22 @@ class TestZeroPad2DLayer(unittest.TestCase): self.pad = [2, 2, 4, 1] self.padLayer = ZeroPad2D(padding=self.pad) self.x = np.random.randint(-255, 255, size=self.shape) - self.expect_res = np.pad(self.x, - [[0, 0], [0, 0], [self.pad[2], self.pad[3]], - [self.pad[0], self.pad[1]]]) + self.expect_res = np.pad( + self.x, + [ + [0, 0], + [0, 0], + [self.pad[2], self.pad[3]], + [self.pad[0], self.pad[1]], + ], + ) def func_layer(self): - np.testing.assert_allclose(zeropad2d(to_tensor(self.x), - self.pad).numpy(), - self.padLayer(to_tensor(self.x)), - rtol=1e-05) + np.testing.assert_allclose( + zeropad2d(to_tensor(self.x), self.pad).numpy(), + self.padLayer(to_tensor(self.x)), + rtol=1e-05, + ) def test_layer(self): with paddle.fluid.framework._test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_zeros_like_op.py b/python/paddle/fluid/tests/unittests/test_zeros_like_op.py index 0a86123ea2c..5712fc4df6e 100644 --- a/python/paddle/fluid/tests/unittests/test_zeros_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_zeros_like_op.py @@ -24,7 +24,6 @@ from paddle.fluid.framework import convert_np_dtype_to_dtype_ class TestZerosLikeAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = paddle.fluid.data('x', [3, 4]) @@ -36,7 +35,6 @@ class TestZerosLikeAPIError(unittest.TestCase): class TestZerosLikeAPI(unittest.TestCase): - def test_api(self): shape = [3, 4] startup_program = Program() @@ -48,14 +46,20 @@ class TestZerosLikeAPI(unittest.TestCase): out3 = zeros_like(x, 'float64') out4 = zeros_like(x, 'int32') out5 = zeros_like(x, 'int64') - place = (fluid.CUDAPlace(0) - if core.is_compiled_with_cuda() else fluid.CPUPlace()) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - outs = exe.run(train_program, - feed={'X': np.ones(shape).astype('float32')}, - fetch_list=[out1, out2, out3, out4, out5]) + outs = exe.run( + train_program, + feed={'X': np.ones(shape).astype('float32')}, + fetch_list=[out1, out2, out3, out4, out5], + ) for (i, dtype) in enumerate( - [np.float32, np.bool_, np.float64, np.int32, np.int64]): + [np.float32, np.bool_, np.float64, np.int32, np.int64] + ): self.assertEqual(outs[i].dtype, dtype) self.assertEqual((outs[i] == np.zeros(shape, dtype)).all(), True) @@ -65,17 +69,20 @@ class TestZerosLikeAPI(unittest.TestCase): class TestZerosLikeImpeartive(unittest.TestCase): - def test_out(self): shape = [3, 4] - place = (fluid.CUDAPlace(0) - if core.is_compiled_with_cuda() else fluid.CPUPlace()) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) x = paddle.to_tensor(np.ones(shape)) for dtype in [np.bool_, np.float32, np.float64, np.int32, np.int64]: out = zeros_like(x, dtype) - self.assertEqual((out.numpy() == np.zeros(shape, dtype)).all(), - True) + self.assertEqual( + (out.numpy() == np.zeros(shape, dtype)).all(), True + ) out = paddle.tensor.zeros_like(x) self.assertEqual((out.numpy() == np.zeros(shape, dtype)).all(), True) out = paddle.tensor.creation.zeros_like(x) @@ -88,20 +95,23 @@ class TestZerosLikeImpeartive(unittest.TestCase): class TestZerosAPI(unittest.TestCase): - def test_api(self): shape = [3, 4] - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) for dtype in [np.float32, np.float64, np.int32, np.int64]: out = _C_ops.zeros(shape, convert_np_dtype_to_dtype_(dtype), place) - self.assertEqual((out.numpy() == np.zeros(shape, dtype)).all(), - True) + self.assertEqual( + (out.numpy() == np.zeros(shape, dtype)).all(), True + ) paddle.enable_static() -if (__name__ == '__main__'): +if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_zeros_op.py b/python/paddle/fluid/tests/unittests/test_zeros_op.py index d91230a48f2..59a87dba56d 100644 --- a/python/paddle/fluid/tests/unittests/test_zeros_op.py +++ b/python/paddle/fluid/tests/unittests/test_zeros_op.py @@ -21,7 +21,6 @@ from paddle.fluid.framework import _test_eager_guard class TestZerosOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): shape = [4] @@ -34,27 +33,26 @@ class TestZerosOpError(unittest.TestCase): class ApiZerosTest(unittest.TestCase): - def test_out(self): with program_guard(Program()): zeros = paddle.zeros(shape=[10], dtype='float64') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - (result, ) = exe.run(fetch_list=[zeros]) + (result,) = exe.run(fetch_list=[zeros]) expected_result = np.zeros(10, dtype='float64') self.assertEqual((result == expected_result).all(), True) with paddle.static.program_guard(Program()): zeros = paddle.zeros(shape=[10], dtype='int64') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - (result, ) = exe.run(fetch_list=[zeros]) + (result,) = exe.run(fetch_list=[zeros]) expected_result = np.zeros(10, dtype='int64') self.assertEqual((result == expected_result).all(), True) with program_guard(Program()): zeros = paddle.zeros(shape=[10], dtype='int64') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - (result, ) = exe.run(fetch_list=[zeros]) + (result,) = exe.run(fetch_list=[zeros]) expected_result = np.zeros(10, dtype='int64') self.assertEqual((result == expected_result).all(), True) with program_guard(Program()): @@ -70,7 +68,7 @@ class ApiZerosTest(unittest.TestCase): zeros = fluid.layers.zeros(shape=[10], dtype='int64') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - (result, ) = exe.run(fetch_list=[zeros]) + (result,) = exe.run(fetch_list=[zeros]) expected_result = np.zeros(10, dtype='int64') self.assertEqual((result == expected_result).all(), True) @@ -81,9 +79,7 @@ class ApiZerosTest(unittest.TestCase): class ApiZerosError(unittest.TestCase): - def test_errors(self): - def test_error1(): with paddle.static.program_guard(fluid.Program()): ones = fluid.layers.zeros(shape=10, dtype='int64') @@ -111,5 +107,5 @@ class ApiZerosError(unittest.TestCase): self.test_shape_errors() -if (__name__ == '__main__'): +if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/testsuite.py b/python/paddle/fluid/tests/unittests/testsuite.py index 619d8232811..a13b73f501b 100644 --- a/python/paddle/fluid/tests/unittests/testsuite.py +++ b/python/paddle/fluid/tests/unittests/testsuite.py @@ -70,7 +70,6 @@ def create_op(scope, op_type, inputs, outputs, attrs, cache_list=None): def set_input(scope, op, inputs, place): - def __set_input__(var_name, var): if isinstance(var, tuple) or isinstance(var, np.ndarray): tensor = scope.find_var(var_name).get_tensor() @@ -119,10 +118,9 @@ def append_input_output(block, op_proto, np_list, is_input, dtype): if is_input: shape = list(np_value.shape) lod_level = 0 - return block.create_var(dtype=dtype, - shape=shape, - lod_level=lod_level, - name=name) + return block.create_var( + dtype=dtype, shape=shape, lod_level=lod_level, name=name + ) var_dict = {} for var_proto in proto_list: @@ -130,15 +128,18 @@ def append_input_output(block, op_proto, np_list, is_input, dtype): if (var_name not in np_list) and var_proto.dispensable: continue if is_input: - assert (var_name in np_list) or (var_proto.dispensable), \ - "Missing {} as input".format(var_name) + assert (var_name in np_list) or ( + var_proto.dispensable + ), "Missing {} as input".format(var_name) if var_proto.duplicable: - assert isinstance(np_list[var_name], list), \ - "Duplicable {} should be set as list".format(var_name) + assert isinstance( + np_list[var_name], list + ), "Duplicable {} should be set as list".format(var_name) var_list = [] for (name, np_value) in np_list[var_name]: var_list.append( - create_var(block, name, {name: np_value}, var_proto)) + create_var(block, name, {name: np_value}, var_proto) + ) var_dict[var_name] = var_list else: var_dict[var_name] = create_var(block, var_name, np_list, var_proto) @@ -151,34 +152,38 @@ def append_loss_ops(block, output_names): if len(mean_inputs) == 1: loss = block.create_var(dtype=mean_inputs[0].dtype, shape=[1]) - op = block.append_op(inputs={"X": mean_inputs}, - outputs={"Out": loss}, - type='mean') + op = block.append_op( + inputs={"X": mean_inputs}, outputs={"Out": loss}, type='mean' + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) else: avg_sum = [] for cur_loss in mean_inputs: cur_avg_loss = block.create_var(dtype=cur_loss.dtype, shape=[1]) - op = block.append_op(inputs={"X": [cur_loss]}, - outputs={"Out": [cur_avg_loss]}, - type="mean") + op = block.append_op( + inputs={"X": [cur_loss]}, + outputs={"Out": [cur_avg_loss]}, + type="mean", + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) avg_sum.append(cur_avg_loss) loss_sum = block.create_var(dtype=avg_sum[0].dtype, shape=[1]) - op_sum = block.append_op(inputs={"X": avg_sum}, - outputs={"Out": loss_sum}, - type='sum') + op_sum = block.append_op( + inputs={"X": avg_sum}, outputs={"Out": loss_sum}, type='sum' + ) op_sum.desc.infer_var_type(block.desc) op_sum.desc.infer_shape(block.desc) loss = block.create_var(dtype=loss_sum.dtype, shape=[1]) - op_loss = block.append_op(inputs={"X": loss_sum}, - outputs={"Out": loss}, - type='scale', - attrs={'scale': 1.0 / float(len(avg_sum))}) + op_loss = block.append_op( + inputs={"X": loss_sum}, + outputs={"Out": loss}, + type='scale', + attrs={'scale': 1.0 / float(len(avg_sum))}, + ) op_loss.desc.infer_var_type(block.desc) op_loss.desc.infer_shape(block.desc) return loss diff --git a/python/paddle/fluid/tests/unittests/tokenizer/bert_tokenizer.py b/python/paddle/fluid/tests/unittests/tokenizer/bert_tokenizer.py index 24f2752854f..cd3546ebe3c 100755 --- a/python/paddle/fluid/tests/unittests/tokenizer/bert_tokenizer.py +++ b/python/paddle/fluid/tests/unittests/tokenizer/bert_tokenizer.py @@ -17,7 +17,13 @@ import os import unicodedata from tokenizer_utils import PretrainedTokenizer -from tokenizer_utils import convert_to_unicode, whitespace_tokenize, _is_whitespace, _is_control, _is_punctuation +from tokenizer_utils import ( + convert_to_unicode, + whitespace_tokenize, + _is_whitespace, + _is_control, + _is_punctuation, +) class BasicTokenizer(object): @@ -129,14 +135,16 @@ class BasicTokenizer(object): # as is Japanese Hiragana and Katakana. Those alphabets are used to write # space-separated words, so they are not treated specially and handled # like the all of the other languages. - if ((cp >= 0x4E00 and cp <= 0x9FFF) or # - (cp >= 0x3400 and cp <= 0x4DBF) or # - (cp >= 0x20000 and cp <= 0x2A6DF) or # - (cp >= 0x2A700 and cp <= 0x2B73F) or # - (cp >= 0x2B740 and cp <= 0x2B81F) or # - (cp >= 0x2B820 and cp <= 0x2CEAF) or - (cp >= 0xF900 and cp <= 0xFAFF) or # - (cp >= 0x2F800 and cp <= 0x2FA1F)): # + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # return True return False @@ -148,7 +156,7 @@ class BasicTokenizer(object): output = [] for char in text: cp = ord(char) - if cp == 0 or cp == 0xfffd or _is_control(char): + if cp == 0 or cp == 0xFFFD or _is_control(char): continue if _is_whitespace(char): output.append(" ") @@ -273,101 +281,71 @@ class BertTokenizer(PretrainedTokenizer): {'input_ids': [101, 2002, 2001, 1037, 13997, 11510, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0]} ''' """ + resource_files_names = {"vocab_file": "vocab.txt"} # for save_pretrained pretrained_resource_files_map = { "vocab_file": { - "bert-base-uncased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-uncased-vocab.txt", - "bert-large-uncased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-large-uncased-vocab.txt", - "bert-base-cased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-cased-vocab.txt", - "bert-large-cased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-large-cased-vocab.txt", - "bert-base-multilingual-uncased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-multilingual-uncased-vocab.txt", - "bert-base-multilingual-cased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-multilingual-cased-vocab.txt", - "bert-base-chinese": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", - "bert-wwm-chinese": - "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-wwm-chinese-vocab.txt", - "bert-wwm-ext-chinese": - "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-wwm-ext-chinese-vocab.txt", - "macbert-large-chinese": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", - "macbert-base-chinese": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", - "simbert-base-chinese": - "https://paddlenlp.bj.bcebos.com/models/transformers/simbert/vocab.txt", + "bert-base-uncased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-uncased-vocab.txt", + "bert-large-uncased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-large-uncased-vocab.txt", + "bert-base-cased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-cased-vocab.txt", + "bert-large-cased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-large-cased-vocab.txt", + "bert-base-multilingual-uncased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-multilingual-uncased-vocab.txt", + "bert-base-multilingual-cased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-multilingual-cased-vocab.txt", + "bert-base-chinese": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", + "bert-wwm-chinese": "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-wwm-chinese-vocab.txt", + "bert-wwm-ext-chinese": "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-wwm-ext-chinese-vocab.txt", + "macbert-large-chinese": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", + "macbert-base-chinese": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", + "simbert-base-chinese": "https://paddlenlp.bj.bcebos.com/models/transformers/simbert/vocab.txt", } } pretrained_init_configuration = { - "bert-base-uncased": { - "do_lower_case": True - }, - "bert-large-uncased": { - "do_lower_case": True - }, - "bert-base-cased": { - "do_lower_case": False - }, - "bert-large-cased": { - "do_lower_case": False - }, - "bert-base-multilingual-uncased": { - "do_lower_case": True - }, - "bert-base-multilingual-cased": { - "do_lower_case": False - }, - "bert-base-chinese": { - "do_lower_case": False - }, - "bert-wwm-chinese": { - "do_lower_case": False - }, - "bert-wwm-ext-chinese": { - "do_lower_case": False - }, - "macbert-large-chinese": { - "do_lower_case": False - }, - "macbert-base-chinese": { - "do_lower_case": False - }, - "simbert-base-chinese": { - "do_lower_case": True - }, + "bert-base-uncased": {"do_lower_case": True}, + "bert-large-uncased": {"do_lower_case": True}, + "bert-base-cased": {"do_lower_case": False}, + "bert-large-cased": {"do_lower_case": False}, + "bert-base-multilingual-uncased": {"do_lower_case": True}, + "bert-base-multilingual-cased": {"do_lower_case": False}, + "bert-base-chinese": {"do_lower_case": False}, + "bert-wwm-chinese": {"do_lower_case": False}, + "bert-wwm-ext-chinese": {"do_lower_case": False}, + "macbert-large-chinese": {"do_lower_case": False}, + "macbert-base-chinese": {"do_lower_case": False}, + "simbert-base-chinese": {"do_lower_case": True}, } padding_side = 'right' - def __init__(self, - vocab_file, - do_lower_case=True, - unk_token="[UNK]", - sep_token="[SEP]", - pad_token="[PAD]", - cls_token="[CLS]", - mask_token="[MASK]"): + def __init__( + self, + vocab_file, + do_lower_case=True, + unk_token="[UNK]", + sep_token="[SEP]", + pad_token="[PAD]", + cls_token="[CLS]", + mask_token="[MASK]", + ): if not os.path.isfile(vocab_file): raise ValueError( "Can't find a vocabulary file at path '{}'. To load the " "vocabulary from a pretrained model please use " - "`tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`" - .format(vocab_file)) + "`tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format( + vocab_file + ) + ) self.vocab = self.load_vocabulary(vocab_file, unk_token=unk_token) self.do_lower_case = do_lower_case self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) - self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, - unk_token=unk_token) + self.wordpiece_tokenizer = WordpieceTokenizer( + vocab=self.vocab, unk_token=unk_token + ) self.special_tokens_map = { 'unk_token': unk_token, 'sep_token': sep_token, 'pad_token': pad_token, 'cls_token': cls_token, - 'mask_token': mask_token + 'mask_token': mask_token, } @property @@ -430,7 +408,9 @@ class BertTokenizer(PretrainedTokenizer): token_ids_1 = [] return len( self.build_inputs_with_special_tokens( - token_ids_0, token_ids_1 if pair else None)) + token_ids_0, token_ids_1 if pair else None + ) + ) def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): """ @@ -454,9 +434,9 @@ class BertTokenizer(PretrainedTokenizer): _sep = [self.sep_token_id] return _cls + token_ids_0 + _sep + token_ids_1 + _sep - def create_token_type_ids_from_sequences(self, - token_ids_0, - token_ids_1=None): + def create_token_type_ids_from_sequences( + self, token_ids_0, token_ids_1=None + ): """ Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format: @@ -476,13 +456,13 @@ class BertTokenizer(PretrainedTokenizer): _cls = [self.cls_token_id] if token_ids_1 is None: return len(_cls + token_ids_0 + _sep) * [0] - return len(_cls + token_ids_0 + _sep) * [0] + len(token_ids_1 + - _sep) * [1] + return len(_cls + token_ids_0 + _sep) * [0] + len( + token_ids_1 + _sep + ) * [1] - def get_special_tokens_mask(self, - token_ids_0, - token_ids_1=None, - already_has_special_tokens=False): + def get_special_tokens_mask( + self, token_ids_0, token_ids_1=None, already_has_special_tokens=False + ): """ Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer ``encode`` methods. @@ -506,10 +486,18 @@ class BertTokenizer(PretrainedTokenizer): return list( map( lambda x: 1 - if x in [self.sep_token_id, self.cls_token_id] else 0, - token_ids_0)) + if x in [self.sep_token_id, self.cls_token_id] + else 0, + token_ids_0, + ) + ) if token_ids_1 is not None: - return [1] + ([0] * len(token_ids_0)) + [1] + ( - [0] * len(token_ids_1)) + [1] + return ( + [1] + + ([0] * len(token_ids_0)) + + [1] + + ([0] * len(token_ids_1)) + + [1] + ) return [1] + ([0] * len(token_ids_0)) + [1] diff --git a/python/paddle/fluid/tests/unittests/tokenizer/tokenizer_utils.py b/python/paddle/fluid/tests/unittests/tokenizer/tokenizer_utils.py index 0524854834b..9e60b29ffb7 100644 --- a/python/paddle/fluid/tests/unittests/tokenizer/tokenizer_utils.py +++ b/python/paddle/fluid/tests/unittests/tokenizer/tokenizer_utils.py @@ -89,8 +89,12 @@ def _is_punctuation(char): # Characters such as "^", "$", and "`" are not in the Unicode # Punctuation class but we treat them as punctuation anyways, for # consistency. - if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) - or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + if ( + (cp >= 33 and cp <= 47) + or (cp >= 58 and cp <= 64) + or (cp >= 91 and cp <= 96) + or (cp >= 123 and cp <= 126) + ): return True cat = unicodedata.category(char) if cat.startswith("P"): @@ -108,14 +112,16 @@ def is_chinese_char(cp): # as is Japanese Hiragana and Katakana. Those alphabets are used to write # space-separated words, so they are not treated specially and handled # like the all of the other languages. - if ((cp >= 0x4E00 and cp <= 0x9FFF) or # - (cp >= 0x3400 and cp <= 0x4DBF) or # - (cp >= 0x20000 and cp <= 0x2A6DF) or # - (cp >= 0x2A700 and cp <= 0x2B73F) or # - (cp >= 0x2B740 and cp <= 0x2B81F) or # - (cp >= 0x2B820 and cp <= 0x2CEAF) or - (cp >= 0xF900 and cp <= 0xFAFF) or # - (cp >= 0x2F800 and cp <= 0x2FA1F)): # + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # return True return False @@ -172,6 +178,7 @@ class PretrainedTokenizer(object): by which subclasses can track arguments for initialization automatically and expose special tokens initialization used as attributes. """ + tokenizer_config_file = "tokenizer_config.json" pretrained_init_configuration = {} resource_files_names = {} # keys are arguments of __init__ @@ -179,20 +186,22 @@ class PretrainedTokenizer(object): padding_side = 'right' pad_token_type_id = 0 - def __call__(self, - text, - text_pair=None, - max_seq_len: Optional[int] = None, - stride=0, - is_split_into_words=False, - pad_to_max_seq_len=False, - truncation_strategy="longest_first", - return_position_ids=False, - return_token_type_ids=True, - return_attention_mask=False, - return_length=False, - return_overflowing_tokens=False, - return_special_tokens_mask=False): + def __call__( + self, + text, + text_pair=None, + max_seq_len: Optional[int] = None, + stride=0, + is_split_into_words=False, + pad_to_max_seq_len=False, + truncation_strategy="longest_first", + return_position_ids=False, + return_token_type_ids=True, + return_attention_mask=False, + return_length=False, + return_overflowing_tokens=False, + return_special_tokens_mask=False, + ): """ Performs tokenization and uses the tokenized tokens to prepare model inputs. It supports sequence or sequence pair as input, and batch input @@ -287,32 +296,60 @@ class PretrainedTokenizer(object): """ # Input type checking for clearer error assert isinstance(text, str) or ( - isinstance(text, (list, tuple)) and - (len(text) == 0 or - (isinstance(text[0], str) or - (isinstance(text[0], (list, tuple)) and - (len(text[0]) == 0 or isinstance(text[0][0], str))))) - ), ("text input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " - "or `List[List[str]]` (batch of pretokenized examples).") + isinstance(text, (list, tuple)) + and ( + len(text) == 0 + or ( + isinstance(text[0], str) + or ( + isinstance(text[0], (list, tuple)) + and (len(text[0]) == 0 or isinstance(text[0][0], str)) + ) + ) + ) + ), ( + "text input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " + "or `List[List[str]]` (batch of pretokenized examples)." + ) assert ( - text_pair is None or isinstance(text_pair, str) or - (isinstance(text_pair, (list, tuple)) and - (len(text_pair) == 0 or - (isinstance(text_pair[0], str) or - (isinstance(text_pair[0], (list, tuple)) and - (len(text_pair[0]) == 0 or isinstance(text_pair[0][0], str)))))) - ), ("text_pair input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " - "or `List[List[str]]` (batch of pretokenized examples).") + text_pair is None + or isinstance(text_pair, str) + or ( + isinstance(text_pair, (list, tuple)) + and ( + len(text_pair) == 0 + or ( + isinstance(text_pair[0], str) + or ( + isinstance(text_pair[0], (list, tuple)) + and ( + len(text_pair[0]) == 0 + or isinstance(text_pair[0][0], str) + ) + ) + ) + ) + ) + ), ( + "text_pair input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " + "or `List[List[str]]` (batch of pretokenized examples)." + ) is_batched = bool( (not is_split_into_words and isinstance(text, (list, tuple))) - or (is_split_into_words and isinstance(text, (list, tuple)) and text - and isinstance(text[0], (list, tuple)))) + or ( + is_split_into_words + and isinstance(text, (list, tuple)) + and text + and isinstance(text[0], (list, tuple)) + ) + ) if is_batched: - batch_text_or_text_pairs = list(zip( - text, text_pair)) if text_pair is not None else text + batch_text_or_text_pairs = ( + list(zip(text, text_pair)) if text_pair is not None else text + ) return self.batch_encode( batch_text_or_text_pairs=batch_text_or_text_pairs, max_seq_len=max_seq_len, @@ -325,7 +362,8 @@ class PretrainedTokenizer(object): return_attention_mask=return_attention_mask, return_length=return_length, return_overflowing_tokens=return_overflowing_tokens, - return_special_tokens_mask=return_special_tokens_mask) + return_special_tokens_mask=return_special_tokens_mask, + ) else: return self.encode( text=text, @@ -338,7 +376,8 @@ class PretrainedTokenizer(object): return_attention_mask=return_attention_mask, return_length=return_length, return_overflowing_tokens=return_overflowing_tokens, - return_special_tokens_mask=return_special_tokens_mask) + return_special_tokens_mask=return_special_tokens_mask, + ) @property def all_special_tokens(self): @@ -349,8 +388,11 @@ class PretrainedTokenizer(object): all_toks = [] set_attr = self.special_tokens_map for attr_value in set_attr.values(): - all_toks = all_toks + (list(attr_value) if isinstance( - attr_value, (list, tuple)) else [attr_value]) + all_toks = all_toks + ( + list(attr_value) + if isinstance(attr_value, (list, tuple)) + else [attr_value] + ) all_toks = list(set(all_toks)) return all_toks @@ -426,11 +468,13 @@ class PretrainedTokenizer(object): # From local dir path elif os.path.isdir(pretrained_model_name_or_path): for file_id, file_name in cls.resource_files_names.items(): - full_file_name = os.path.join(pretrained_model_name_or_path, - file_name) + full_file_name = os.path.join( + pretrained_model_name_or_path, file_name + ) vocab_files[file_id] = full_file_name vocab_files["tokenizer_config_file"] = os.path.join( - pretrained_model_name_or_path, cls.tokenizer_config_file) + pretrained_model_name_or_path, cls.tokenizer_config_file + ) default_root = os.path.join(DATA_HOME, pretrained_model_name_or_path) resolved_vocab_files = {} @@ -443,11 +487,13 @@ class PretrainedTokenizer(object): print("Already cached %s" % path) resolved_vocab_files[file_id] = path else: - print("Downloading %s and saved to %s" % - (file_path, default_root)) + print( + "Downloading %s and saved to %s" % (file_path, default_root) + ) try: resolved_vocab_files[file_id] = get_path_from_url( - file_path, default_root) + file_path, default_root + ) except RuntimeError as err: print(err) raise RuntimeError( @@ -461,7 +507,8 @@ class PretrainedTokenizer(object): # Prepare tokenizer initialization kwargs # Did we saved some inputs and kwargs to reload ? tokenizer_config_file = resolved_vocab_files.pop( - "tokenizer_config_file", None) + "tokenizer_config_file", None + ) if tokenizer_config_file is not None: with open(tokenizer_config_file, 'r', encoding="utf-8") as f: init_kwargs = json.load(f) @@ -489,8 +536,9 @@ class PretrainedTokenizer(object): # does include a vocab file path in it. However, if the vocab file # path included in json does not exist, such as was deleted, to make # it still work, use the vocab file under this dir. - elif not os.path.isfile( - init_kwargs[args_name]) and os.path.isfile(file_path): + elif not os.path.isfile(init_kwargs[args_name]) and os.path.isfile( + file_path + ): init_kwargs[args_name] = file_path # TODO(guosheng): avoid reduplication of position args and key word args tokenizer = cls(*init_args, **init_kwargs) @@ -519,11 +567,13 @@ class PretrainedTokenizer(object): assert not os.path.isfile( save_directory ), "Saving directory ({}) should be a directory, not a file".format( - save_directory) + save_directory + ) os.makedirs(save_directory, exist_ok=True) - tokenizer_config_file = os.path.join(save_directory, - self.tokenizer_config_file) + tokenizer_config_file = os.path.join( + save_directory, self.tokenizer_config_file + ) # init_config is set in metaclass created `__init__`, tokenizer_config = self.init_config with open(tokenizer_config_file, "w", encoding="utf-8") as f: @@ -545,12 +595,14 @@ class PretrainedTokenizer(object): copyfile(src_path, dst_path) @staticmethod - def load_vocabulary(filepath, - unk_token=None, - pad_token=None, - bos_token=None, - eos_token=None, - **kwargs): + def load_vocabulary( + filepath, + unk_token=None, + pad_token=None, + bos_token=None, + eos_token=None, + **kwargs, + ): """ Instantiate an instance of `Vocab` from a file reserving all tokens by using `Vocab.from_dict`. The file contains a token per line, and the @@ -581,15 +633,20 @@ class PretrainedTokenizer(object): return self.special_tokens_map[name] elif name.endswith('_token_id'): return self.vocab[self.special_tokens_map[name[:-3]]] - raise AttributeError("'{}' object has no attribute '{}'".format( - type(self).__name__, name)) - - def truncate_sequences(self, - ids, - pair_ids=None, - num_tokens_to_remove=0, - truncation_strategy='longest_first', - stride=0): + raise AttributeError( + "'{}' object has no attribute '{}'".format( + type(self).__name__, name + ) + ) + + def truncate_sequences( + self, + ids, + pair_ids=None, + num_tokens_to_remove=0, + truncation_strategy='longest_first', + stride=0, + ): """ Truncates a sequence pair in place to the maximum length. Args: @@ -662,9 +719,9 @@ class PretrainedTokenizer(object): return token_ids_0 + token_ids_1 - def build_offset_mapping_with_special_tokens(self, - offset_mapping_0, - offset_mapping_1=None): + def build_offset_mapping_with_special_tokens( + self, offset_mapping_0, offset_mapping_1=None + ): """ Build offset map from a pair of offset map by concatenating and adding offsets of special tokens. Should be overridden in a subclass if the model has a special way of building those. @@ -681,10 +738,9 @@ class PretrainedTokenizer(object): return offset_mapping_0 + offset_mapping_1 - def get_special_tokens_mask(self, - token_ids_0, - token_ids_1=None, - already_has_special_tokens=False): + def get_special_tokens_mask( + self, token_ids_0, token_ids_1=None, already_has_special_tokens=False + ): """ Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer ``encode`` methods. @@ -698,11 +754,12 @@ class PretrainedTokenizer(object): 1 for a special token, 0 for a sequence token. """ return [0] * ( - (len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0)) + (len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0) + ) - def create_token_type_ids_from_sequences(self, - token_ids_0, - token_ids_1=None): + def create_token_type_ids_from_sequences( + self, token_ids_0, token_ids_1=None + ): """ Create a mask from the two sequences passed to be used in a sequence-pair classification task. Should be overridden in a subclass if the model has a special way of building those. @@ -733,20 +790,24 @@ class PretrainedTokenizer(object): token_ids_1 = [] return len( self.build_inputs_with_special_tokens( - token_ids_0, token_ids_1 if pair else None)) - - def encode(self, - text, - text_pair=None, - max_seq_len=512, - pad_to_max_seq_len=False, - truncation_strategy="longest_first", - return_position_ids=False, - return_token_type_ids=True, - return_attention_mask=False, - return_length=False, - return_overflowing_tokens=False, - return_special_tokens_mask=False): + token_ids_0, token_ids_1 if pair else None + ) + ) + + def encode( + self, + text, + text_pair=None, + max_seq_len=512, + pad_to_max_seq_len=False, + truncation_strategy="longest_first", + return_position_ids=False, + return_token_type_ids=True, + return_attention_mask=False, + return_length=False, + return_overflowing_tokens=False, + return_special_tokens_mask=False, + ): """ Performs tokenization and uses the tokenized tokens to prepare model inputs. It supports sequence or sequence pair as input, and batch input @@ -834,13 +895,17 @@ class PretrainedTokenizer(object): if isinstance(text, str): tokens = self._tokenize(text) return self.convert_tokens_to_ids(tokens) - elif isinstance(text, - (list, tuple)) and len(text) > 0 and isinstance( - text[0], str): + elif ( + isinstance(text, (list, tuple)) + and len(text) > 0 + and isinstance(text[0], str) + ): return self.convert_tokens_to_ids(text) - elif isinstance(text, - (list, tuple)) and len(text) > 0 and isinstance( - text[0], int): + elif ( + isinstance(text, (list, tuple)) + and len(text) > 0 + and isinstance(text[0], int) + ): return text else: raise ValueError( @@ -857,8 +922,9 @@ class PretrainedTokenizer(object): encoded_inputs = {} # Truncation: Handle max sequence length - total_len = len_ids + len_pair_ids + (self.num_special_tokens_to_add( - pair=pair)) + total_len = ( + len_ids + len_pair_ids + (self.num_special_tokens_to_add(pair=pair)) + ) if max_seq_len and total_len > max_seq_len: ids, pair_ids, overflowing_tokens = self.truncate_sequences( @@ -875,7 +941,8 @@ class PretrainedTokenizer(object): sequence = self.build_inputs_with_special_tokens(ids, pair_ids) token_type_ids = self.create_token_type_ids_from_sequences( - ids, pair_ids) + ids, pair_ids + ) # Build output dictionnary encoded_inputs["input_ids"] = sequence @@ -883,44 +950,53 @@ class PretrainedTokenizer(object): encoded_inputs["token_type_ids"] = token_type_ids if return_special_tokens_mask: encoded_inputs[ - "special_tokens_mask"] = self.get_special_tokens_mask( - ids, pair_ids) + "special_tokens_mask" + ] = self.get_special_tokens_mask(ids, pair_ids) if return_length: encoded_inputs["seq_len"] = len(encoded_inputs["input_ids"]) # Check lengths - assert max_seq_len is None or len( - encoded_inputs["input_ids"]) <= max_seq_len + assert ( + max_seq_len is None + or len(encoded_inputs["input_ids"]) <= max_seq_len + ) # Padding - needs_to_be_padded = pad_to_max_seq_len and \ - max_seq_len and len(encoded_inputs["input_ids"]) < max_seq_len + needs_to_be_padded = ( + pad_to_max_seq_len + and max_seq_len + and len(encoded_inputs["input_ids"]) < max_seq_len + ) if needs_to_be_padded: difference = max_seq_len - len(encoded_inputs["input_ids"]) if self.padding_side == 'right': if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len( - encoded_inputs["input_ids"]) + [0] * difference + encoded_inputs["input_ids"] + ) + [0] * difference if return_token_type_ids: encoded_inputs["token_type_ids"] = ( - encoded_inputs["token_type_ids"] + - [self.pad_token_type_id] * difference) + encoded_inputs["token_type_ids"] + + [self.pad_token_type_id] * difference + ) if return_special_tokens_mask: - encoded_inputs["special_tokens_mask"] = encoded_inputs[ - "special_tokens_mask"] + [1] * difference - encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [ - self.pad_token_id - ] * difference + encoded_inputs["special_tokens_mask"] = ( + encoded_inputs["special_tokens_mask"] + [1] * difference + ) + encoded_inputs["input_ids"] = ( + encoded_inputs["input_ids"] + + [self.pad_token_id] * difference + ) elif self.padding_side == 'left': if return_attention_mask: encoded_inputs["attention_mask"] = [0] * difference + [ 1 ] * len(encoded_inputs["input_ids"]) if return_token_type_ids: - encoded_inputs["token_type_ids"] = ( - [self.pad_token_type_id] * difference + - encoded_inputs["token_type_ids"]) + encoded_inputs["token_type_ids"] = [ + self.pad_token_type_id + ] * difference + encoded_inputs["token_type_ids"] if return_special_tokens_mask: encoded_inputs["special_tokens_mask"] = [ 1 @@ -931,27 +1007,31 @@ class PretrainedTokenizer(object): else: if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) if return_position_ids: encoded_inputs["position_ids"] = list( - range(len(encoded_inputs["input_ids"]))) + range(len(encoded_inputs["input_ids"])) + ) return encoded_inputs - def batch_encode(self, - batch_text_or_text_pairs, - max_seq_len=512, - pad_to_max_seq_len=False, - stride=0, - is_split_into_words=False, - truncation_strategy="longest_first", - return_position_ids=False, - return_token_type_ids=True, - return_attention_mask=False, - return_length=False, - return_overflowing_tokens=False, - return_special_tokens_mask=False): + def batch_encode( + self, + batch_text_or_text_pairs, + max_seq_len=512, + pad_to_max_seq_len=False, + stride=0, + is_split_into_words=False, + truncation_strategy="longest_first", + return_position_ids=False, + return_token_type_ids=True, + return_attention_mask=False, + return_length=False, + return_overflowing_tokens=False, + return_special_tokens_mask=False, + ): """ Performs tokenization and uses the tokenized tokens to prepare model inputs. It supports batch inputs of sequence or sequence pair. @@ -1043,13 +1123,17 @@ class PretrainedTokenizer(object): if isinstance(text, str): tokens = self._tokenize(text) return self.convert_tokens_to_ids(tokens) - elif isinstance(text, - (list, tuple)) and len(text) > 0 and isinstance( - text[0], str): + elif ( + isinstance(text, (list, tuple)) + and len(text) > 0 + and isinstance(text[0], str) + ): return self.convert_tokens_to_ids(text) - elif isinstance(text, - (list, tuple)) and len(text) > 0 and isinstance( - text[0], int): + elif ( + isinstance(text, (list, tuple)) + and len(text) > 0 + and isinstance(text[0], int) + ): return text else: raise ValueError( @@ -1058,23 +1142,29 @@ class PretrainedTokenizer(object): batch_encode_inputs = [] for example_id, tokens_or_pair_tokens in enumerate( - batch_text_or_text_pairs): + batch_text_or_text_pairs + ): if not isinstance(tokens_or_pair_tokens, (list, tuple)): text, text_pair = tokens_or_pair_tokens, None elif is_split_into_words and not isinstance( - tokens_or_pair_tokens[0], (list, tuple)): + tokens_or_pair_tokens[0], (list, tuple) + ): text, text_pair = tokens_or_pair_tokens, None else: text, text_pair = tokens_or_pair_tokens first_ids = get_input_ids(text) - second_ids = get_input_ids( - text_pair) if text_pair is not None else None + second_ids = ( + get_input_ids(text_pair) if text_pair is not None else None + ) if stride > 0 and second_ids is not None: - max_len_for_pair = max_seq_len - len( - first_ids) - self.num_special_tokens_to_add(pair=True) + max_len_for_pair = ( + max_seq_len + - len(first_ids) + - self.num_special_tokens_to_add(pair=True) + ) token_offset_mapping = self.get_offset_mapping(text) token_pair_offset_mapping = self.get_offset_mapping(text_pair) @@ -1087,18 +1177,24 @@ class PretrainedTokenizer(object): length = max_len_for_pair ids = first_ids - pair_ids = second_ids[offset:offset + length] + pair_ids = second_ids[offset : offset + length] mapping = token_offset_mapping - pair_mapping = token_pair_offset_mapping[offset:offset + - length] - - offset_mapping = self.build_offset_mapping_with_special_tokens( - mapping, pair_mapping) + pair_mapping = token_pair_offset_mapping[ + offset : offset + length + ] + + offset_mapping = ( + self.build_offset_mapping_with_special_tokens( + mapping, pair_mapping + ) + ) sequence = self.build_inputs_with_special_tokens( - ids, pair_ids) + ids, pair_ids + ) token_type_ids = self.create_token_type_ids_from_sequences( - ids, pair_ids) + ids, pair_ids + ) # Build output dictionnary encoded_inputs["input_ids"] = sequence @@ -1106,25 +1202,32 @@ class PretrainedTokenizer(object): encoded_inputs["token_type_ids"] = token_type_ids if return_special_tokens_mask: encoded_inputs[ - "special_tokens_mask"] = self.get_special_tokens_mask( - ids, pair_ids) + "special_tokens_mask" + ] = self.get_special_tokens_mask(ids, pair_ids) if return_length: encoded_inputs["seq_len"] = len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) # Check lengths - assert max_seq_len is None or len( - encoded_inputs["input_ids"]) <= max_seq_len + assert ( + max_seq_len is None + or len(encoded_inputs["input_ids"]) <= max_seq_len + ) # Padding - needs_to_be_padded = pad_to_max_seq_len and \ - max_seq_len and len(encoded_inputs["input_ids"]) < max_seq_len + needs_to_be_padded = ( + pad_to_max_seq_len + and max_seq_len + and len(encoded_inputs["input_ids"]) < max_seq_len + ) encoded_inputs['offset_mapping'] = offset_mapping if needs_to_be_padded: difference = max_seq_len - len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) if self.padding_side == 'right': if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len( @@ -1133,33 +1236,42 @@ class PretrainedTokenizer(object): if return_token_type_ids: # 0 for padding token mask encoded_inputs["token_type_ids"] = ( - encoded_inputs["token_type_ids"] + - [self.pad_token_type_id] * difference) + encoded_inputs["token_type_ids"] + + [self.pad_token_type_id] * difference + ) if return_special_tokens_mask: - encoded_inputs[ - "special_tokens_mask"] = encoded_inputs[ - "special_tokens_mask"] + [1 - ] * difference - encoded_inputs["input_ids"] = encoded_inputs[ - "input_ids"] + [self.pad_token_id] * difference - encoded_inputs['offset_mapping'] = encoded_inputs[ - 'offset_mapping'] + [(0, 0)] * difference + encoded_inputs["special_tokens_mask"] = ( + encoded_inputs["special_tokens_mask"] + + [1] * difference + ) + encoded_inputs["input_ids"] = ( + encoded_inputs["input_ids"] + + [self.pad_token_id] * difference + ) + encoded_inputs['offset_mapping'] = ( + encoded_inputs['offset_mapping'] + + [(0, 0)] * difference + ) elif self.padding_side == 'left': if return_attention_mask: encoded_inputs["attention_mask"] = [ 0 ] * difference + [1] * len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) if return_token_type_ids: # 0 for padding token mask - encoded_inputs["token_type_ids"] = ( - [self.pad_token_type_id] * difference + - encoded_inputs["token_type_ids"]) + encoded_inputs["token_type_ids"] = [ + self.pad_token_type_id + ] * difference + encoded_inputs[ + "token_type_ids" + ] if return_special_tokens_mask: encoded_inputs["special_tokens_mask"] = [ 1 ] * difference + encoded_inputs[ - "special_tokens_mask"] + "special_tokens_mask" + ] encoded_inputs["input_ids"] = [ self.pad_token_id ] * difference + encoded_inputs["input_ids"] @@ -1169,11 +1281,13 @@ class PretrainedTokenizer(object): else: if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) if return_position_ids: encoded_inputs["position_ids"] = list( - range(len(encoded_inputs["input_ids"]))) + range(len(encoded_inputs["input_ids"])) + ) encoded_inputs['overflow_to_sample'] = example_id batch_encode_inputs.append(encoded_inputs) @@ -1194,7 +1308,9 @@ class PretrainedTokenizer(object): return_attention_mask=return_attention_mask, return_length=return_length, return_overflowing_tokens=return_overflowing_tokens, - return_special_tokens_mask=return_special_tokens_mask)) + return_special_tokens_mask=return_special_tokens_mask, + ) + ) return batch_encode_inputs @@ -1213,7 +1329,8 @@ class PretrainedTokenizer(object): for token in self.basic_tokenizer.tokenize(text): for sub_token in self.wordpiece_tokenizer.tokenize(token): split_tokens.append( - sub_token if sub_token != self.unk_token else token) + sub_token if sub_token != self.unk_token else token + ) normalized_text, char_mapping = '', [] @@ -1223,10 +1340,13 @@ class PretrainedTokenizer(object): ch = unicodedata.normalize('NFD', ch) ch = ''.join([c for c in ch if unicodedata.category(c) != 'Mn']) - ch = ''.join([ - c for c in ch - if not (ord(c) == 0 or ord(c) == 0xfffd or _is_control(c)) - ]) + ch = ''.join( + [ + c + for c in ch + if not (ord(c) == 0 or ord(c) == 0xFFFD or _is_control(c)) + ] + ) normalized_text += ch char_mapping.extend([i] * len(ch)) @@ -1241,7 +1361,8 @@ class PretrainedTokenizer(object): end = start + len(token) token_mapping.append( - (char_mapping[start], char_mapping[end - 1] + 1)) + (char_mapping[start], char_mapping[end - 1] + 1) + ) offset = end return token_mapping diff --git a/python/paddle/fluid/tests/unittests/transformer_model.py b/python/paddle/fluid/tests/unittests/transformer_model.py index 7fd02339ec9..2baf56caaf3 100644 --- a/python/paddle/fluid/tests/unittests/transformer_model.py +++ b/python/paddle/fluid/tests/unittests/transformer_model.py @@ -30,24 +30,33 @@ def position_encoding_init(n_position, d_pos_vec): """ Generate the initial values for the sinusoid position encoding table. """ - position_enc = np.array([[ - pos / np.power(10000, 2 * (j // 2) / d_pos_vec) - for j in range(d_pos_vec) - ] if pos != 0 else np.zeros(d_pos_vec) for pos in range(n_position)]) + position_enc = np.array( + [ + [ + pos / np.power(10000, 2 * (j // 2) / d_pos_vec) + for j in range(d_pos_vec) + ] + if pos != 0 + else np.zeros(d_pos_vec) + for pos in range(n_position) + ] + ) position_enc[1:, 0::2] = np.sin(position_enc[1:, 0::2]) # dim 2i position_enc[1:, 1::2] = np.cos(position_enc[1:, 1::2]) # dim 2i+1 return position_enc.astype("float32") -def multi_head_attention(queries, - keys, - values, - attn_bias, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0.): +def multi_head_attention( + queries, + keys, + values, + attn_bias, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, +): """ Multi-Head Attention. Note that attn_bias is added to the logit before computing softmax activiation to mask certain selected positions so that @@ -55,36 +64,42 @@ def multi_head_attention(queries, """ if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): raise ValueError( - "Inputs: queries, keys and values should all be 3-D tensors.") + "Inputs: queries, keys and values should all be 3-D tensors." + ) def __compute_qkv(queries, keys, values, n_head, d_key, d_value): """ Add linear projection to queries, keys, and values. """ - q = layers.fc(input=queries, - size=d_key * n_head, - param_attr=fluid.initializer.Xavier( - uniform=False, - fan_in=d_model * d_key, - fan_out=n_head * d_key), - bias_attr=False, - num_flatten_dims=2) - k = layers.fc(input=keys, - size=d_key * n_head, - param_attr=fluid.initializer.Xavier( - uniform=False, - fan_in=d_model * d_key, - fan_out=n_head * d_key), - bias_attr=False, - num_flatten_dims=2) - v = layers.fc(input=values, - size=d_value * n_head, - param_attr=fluid.initializer.Xavier( - uniform=False, - fan_in=d_model * d_value, - fan_out=n_head * d_value), - bias_attr=False, - num_flatten_dims=2) + q = layers.fc( + input=queries, + size=d_key * n_head, + param_attr=fluid.initializer.Xavier( + uniform=False, fan_in=d_model * d_key, fan_out=n_head * d_key + ), + bias_attr=False, + num_flatten_dims=2, + ) + k = layers.fc( + input=keys, + size=d_key * n_head, + param_attr=fluid.initializer.Xavier( + uniform=False, fan_in=d_model * d_key, fan_out=n_head * d_key + ), + bias_attr=False, + num_flatten_dims=2, + ) + v = layers.fc( + input=values, + size=d_value * n_head, + param_attr=fluid.initializer.Xavier( + uniform=False, + fan_in=d_model * d_value, + fan_out=n_head * d_value, + ), + bias_attr=False, + num_flatten_dims=2, + ) return q, k, v def __split_heads(x, n_head): @@ -100,7 +115,8 @@ def multi_head_attention(queries, hidden_size = x.shape[-1] # FIXME(guosheng): Decouple the program desc with batch_size. reshaped = layers.reshape( - x=x, shape=[batch_size, -1, n_head, hidden_size // n_head]) + x=x, shape=[batch_size, -1, n_head, hidden_size // n_head] + ) # permute the dimensions into: # [batch_size, n_head, max_sequence_len, hidden_size_per_head] @@ -111,7 +127,8 @@ def multi_head_attention(queries, Transpose and then reshape the last two dimensions of input tensor x so that it becomes one dimension, which is reverse to __split_heads. """ - if len(x.shape) == 3: return x + if len(x.shape) == 3: + return x if len(x.shape) != 4: raise ValueError("Input(x) should be a 4-D Tensor.") @@ -120,8 +137,9 @@ def multi_head_attention(queries, return layers.reshape( x=trans_x, shape=list( - map(int, - [batch_size, -1, trans_x.shape[2] * trans_x.shape[3]]))) + map(int, [batch_size, -1, trans_x.shape[2] * trans_x.shape[3]]) + ), + ) def scaled_dot_product_attention(q, k, v, attn_bias, d_model, dropout_rate): """ @@ -146,9 +164,9 @@ def multi_head_attention(queries, product = layers.matmul(x=scaled_q, y=k, transpose_y=True) weights = __softmax(layers.elementwise_add(x=product, y=attn_bias)) if dropout_rate: - weights = layers.dropout(weights, - dropout_prob=dropout_rate, - is_test=False) + weights = layers.dropout( + weights, dropout_prob=dropout_rate, is_test=False + ) out = layers.matmul(weights, v) return out @@ -158,17 +176,20 @@ def multi_head_attention(queries, k = __split_heads(k, n_head) v = __split_heads(v, n_head) - ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_model, - dropout_rate) + ctx_multiheads = scaled_dot_product_attention( + q, k, v, attn_bias, d_model, dropout_rate + ) out = __combine_heads(ctx_multiheads) # Project back to the model size. - proj_out = layers.fc(input=out, - size=d_model, - param_attr=fluid.initializer.Xavier(uniform=False), - bias_attr=False, - num_flatten_dims=2) + proj_out = layers.fc( + input=out, + size=d_model, + param_attr=fluid.initializer.Xavier(uniform=False), + bias_attr=False, + num_flatten_dims=2, + ) return proj_out @@ -178,21 +199,27 @@ def positionwise_feed_forward(x, d_inner_hid, d_hid): This module consists of two linear transformations with a ReLU activation in between, which is applied to each position separately and identically. """ - hidden = layers.fc(input=x, - size=d_inner_hid, - num_flatten_dims=2, - param_attr=fluid.initializer.Uniform(low=-(d_hid**-0.5), - high=(d_hid**-0.5)), - act="relu") - out = layers.fc(input=hidden, - size=d_hid, - num_flatten_dims=2, - param_attr=fluid.initializer.Uniform( - low=-(d_inner_hid**-0.5), high=(d_inner_hid**-0.5))) + hidden = layers.fc( + input=x, + size=d_inner_hid, + num_flatten_dims=2, + param_attr=fluid.initializer.Uniform( + low=-(d_hid**-0.5), high=(d_hid**-0.5) + ), + act="relu", + ) + out = layers.fc( + input=hidden, + size=d_hid, + num_flatten_dims=2, + param_attr=fluid.initializer.Uniform( + low=-(d_inner_hid**-0.5), high=(d_inner_hid**-0.5) + ), + ) return out -def pre_post_process_layer(prev_out, out, process_cmd, dropout=0.): +def pre_post_process_layer(prev_out, out, process_cmd, dropout=0.0): """ Add residual connection, layer normalization and droput to the out tensor optionally according to the value of process_cmd. @@ -204,10 +231,12 @@ def pre_post_process_layer(prev_out, out, process_cmd, dropout=0.): if cmd == "a": # add residual connection out = out + prev_out if prev_out else out elif cmd == "n": # add layer normalization - out = layers.layer_norm(out, - begin_norm_axis=len(out.shape) - 1, - param_attr=fluid.initializer.Constant(1.), - bias_attr=fluid.initializer.Constant(0.)) + out = layers.layer_norm( + out, + begin_norm_axis=len(out.shape) - 1, + param_attr=fluid.initializer.Constant(1.0), + bias_attr=fluid.initializer.Constant(0.0), + ) elif cmd == "d": # add dropout if dropout: out = layers.dropout(out, dropout_prob=dropout, is_test=False) @@ -218,53 +247,65 @@ pre_process_layer = partial(pre_post_process_layer, None) post_process_layer = pre_post_process_layer -def prepare_encoder(src_word, - src_pos, - src_vocab_size, - src_emb_dim, - src_pad_idx, - src_max_len, - dropout=0., - pos_pad_idx=0, - pos_enc_param_name=None): +def prepare_encoder( + src_word, + src_pos, + src_vocab_size, + src_emb_dim, + src_pad_idx, + src_max_len, + dropout=0.0, + pos_pad_idx=0, + pos_enc_param_name=None, +): """Add word embeddings and position encodings. The output tensor has a shape of: [batch_size, max_src_length_in_batch, d_model]. This module is used at the bottom of the encoder stacks. """ - src_word_emb = layers.embedding(src_word, - size=[src_vocab_size, src_emb_dim], - padding_idx=src_pad_idx, - param_attr=fluid.initializer.Normal(0., 1.)) + src_word_emb = layers.embedding( + src_word, + size=[src_vocab_size, src_emb_dim], + padding_idx=src_pad_idx, + param_attr=fluid.initializer.Normal(0.0, 1.0), + ) src_pos_enc = layers.embedding( src_pos, size=[src_max_len, src_emb_dim], padding_idx=pos_pad_idx, - param_attr=fluid.ParamAttr(name=pos_enc_param_name, trainable=False)) + param_attr=fluid.ParamAttr(name=pos_enc_param_name, trainable=False), + ) src_pos_enc.stop_gradient = True enc_input = src_word_emb + src_pos_enc # FIXME(guosheng): Decouple the program desc with batch_size. enc_input = layers.reshape(x=enc_input, shape=[batch_size, -1, src_emb_dim]) - return layers.dropout(enc_input, dropout_prob=dropout, - is_test=False) if dropout else enc_input + return ( + layers.dropout(enc_input, dropout_prob=dropout, is_test=False) + if dropout + else enc_input + ) -prepare_encoder = partial(prepare_encoder, - pos_enc_param_name=pos_enc_param_names[0]) -prepare_decoder = partial(prepare_encoder, - pos_enc_param_name=pos_enc_param_names[1]) +prepare_encoder = partial( + prepare_encoder, pos_enc_param_name=pos_enc_param_names[0] +) +prepare_decoder = partial( + prepare_encoder, pos_enc_param_name=pos_enc_param_names[1] +) -def encoder_layer(enc_input, - attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0.): +def encoder_layer( + enc_input, + attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): """The encoder layers that can be stacked to form a deep encoder. This module consits of a multi-head (self) attention followed by @@ -272,46 +313,67 @@ def encoder_layer(enc_input, with the post_process_layer to add residual connection, layer normalization and droput. """ - attn_output = multi_head_attention(enc_input, enc_input, enc_input, - attn_bias, d_key, d_value, d_model, - n_head, dropout_rate) - attn_output = post_process_layer(enc_input, attn_output, "dan", - dropout_rate) + attn_output = multi_head_attention( + enc_input, + enc_input, + enc_input, + attn_bias, + d_key, + d_value, + d_model, + n_head, + dropout_rate, + ) + attn_output = post_process_layer( + enc_input, attn_output, "dan", dropout_rate + ) ffd_output = positionwise_feed_forward(attn_output, d_inner_hid, d_model) return post_process_layer(attn_output, ffd_output, "dan", dropout_rate) -def encoder(enc_input, +def encoder( + enc_input, + attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): + """ + The encoder is composed of a stack of identical layers returned by calling + encoder_layer. + """ + for i in range(n_layer): + enc_output = encoder_layer( + enc_input, attn_bias, - n_layer, n_head, d_key, d_value, d_model, d_inner_hid, - dropout_rate=0.): - """ - The encoder is composed of a stack of identical layers returned by calling - encoder_layer. - """ - for i in range(n_layer): - enc_output = encoder_layer(enc_input, attn_bias, n_head, d_key, d_value, - d_model, d_inner_hid, dropout_rate) + dropout_rate, + ) enc_input = enc_output return enc_output -def decoder_layer(dec_input, - enc_output, - slf_attn_bias, - dec_enc_attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0.): - """ The layer to be stacked in decoder part. +def decoder_layer( + dec_input, + enc_output, + slf_attn_bias, + dec_enc_attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): + """The layer to be stacked in decoder part. The structure of this module is similar to that in the encoder part except a multi-head attention is added to implement encoder-decoder attention. @@ -364,17 +426,19 @@ def decoder_layer(dec_input, return dec_output -def decoder(dec_input, - enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0.): +def decoder( + dec_input, + enc_output, + dec_slf_attn_bias, + dec_enc_attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): """ The decoder is composed of a stack of identical decoder_layer layers. """ @@ -435,10 +499,10 @@ def build_inputs(max_length, n_head): all_inputs = [] for name, shape, dtype in zip(names, shapes, dtypes): all_inputs.append( - fluid.layers.data(name=name, - shape=shape, - dtype=dtype, - append_batch_size=False)) + fluid.layers.data( + name=name, shape=shape, dtype=dtype, append_batch_size=False + ) + ) return all_inputs @@ -458,8 +522,17 @@ def transformer( pos_pad_idx, ): - src_word, src_pos, trg_word, trg_pos, src_slf_attn_bias, trg_slf_attn_bias, trg_src_attn_bias, gold, weights = build_inputs( - max_length, n_head) + ( + src_word, + src_pos, + trg_word, + trg_pos, + src_slf_attn_bias, + trg_slf_attn_bias, + trg_src_attn_bias, + gold, + weights, + ) = build_inputs(max_length, n_head) enc_input = prepare_encoder( src_word, @@ -507,14 +580,17 @@ def transformer( # TODO(guosheng): Share the weight matrix between the embedding layers and # the pre-softmax linear transformation. - predict = layers.reshape(x=layers.fc( - input=dec_output, - size=trg_vocab_size, - param_attr=fluid.initializer.Xavier(uniform=False), - bias_attr=False, - num_flatten_dims=2), - shape=[-1, trg_vocab_size], - act="softmax") + predict = layers.reshape( + x=layers.fc( + input=dec_output, + size=trg_vocab_size, + param_attr=fluid.initializer.Xavier(uniform=False), + bias_attr=False, + num_flatten_dims=2, + ), + shape=[-1, trg_vocab_size], + act="softmax", + ) cost = layers.cross_entropy(input=predict, label=gold) weighted_cost = cost * weights diff --git a/python/paddle/fluid/tests/unittests/utils.py b/python/paddle/fluid/tests/unittests/utils.py index 346d4002583..03993c2355e 100644 --- a/python/paddle/fluid/tests/unittests/utils.py +++ b/python/paddle/fluid/tests/unittests/utils.py @@ -83,7 +83,6 @@ def _is_equal_program(prog1, prog2): def load_dygraph_vars_to_scope(model_path, scope, place): - def load_dict_to_scope(scope, dictionary): if scope is None: scope = fluid.global_scope() @@ -103,7 +102,6 @@ def load_dygraph_vars_to_scope(model_path, scope, place): class DyGraphProgramDescTracerTestHelper(object): - def __init__(self, unittest_obj): self.unittest_obj = unittest_obj diff --git a/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py b/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py index a3a3eca128c..730d7791621 100644 --- a/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py @@ -13,8 +13,21 @@ # limitations under the License. NEED_TO_FIX_OP_LIST = [ - 'fused_elemwise_activation', 'bilinear_tensor_product', 'conv2d_transpose', - 'depthwise_conv2d_transpose', 'grid_sampler', 'lstmp', 'margin_rank_loss', - 'matmul', 'scatter', 'soft_relu', 'squared_l2_distance', 'tree_conv', 'cvm', - 'cudnn_lstm', 'rnn', 'multi_dot', 'index_add' + 'fused_elemwise_activation', + 'bilinear_tensor_product', + 'conv2d_transpose', + 'depthwise_conv2d_transpose', + 'grid_sampler', + 'lstmp', + 'margin_rank_loss', + 'matmul', + 'scatter', + 'soft_relu', + 'squared_l2_distance', + 'tree_conv', + 'cvm', + 'cudnn_lstm', + 'rnn', + 'multi_dot', + 'index_add', ] diff --git a/python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py b/python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py index ee8202aa9f3..a00c1a720aa 100644 --- a/python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py @@ -19,20 +19,20 @@ # reasons for skipping compile_vs_runtime test or be fixed later. COMPILE_RUN_OP_WHITE_LIST = [ - 'sequence_pool', \ - 'sequence_slice', \ - 'generate_proposals', \ - 'mine_hard_examples', \ - 'retinanet_detection_output', \ - 'ctc_align', \ - 'fusion_seqpool_cvm_concat', \ - 'gru', \ - 'rpn_target_assign', \ - 'retinanet_target_assign', \ - 'filter_by_instag', \ - 'im2sequence', \ - 'generate_proposal_labels', \ - 'detection_map', \ - 'locality_aware_nms', \ - 'var_conv_2d' + 'sequence_pool', + 'sequence_slice', + 'generate_proposals', + 'mine_hard_examples', + 'retinanet_detection_output', + 'ctc_align', + 'fusion_seqpool_cvm_concat', + 'gru', + 'rpn_target_assign', + 'retinanet_target_assign', + 'filter_by_instag', + 'im2sequence', + 'generate_proposal_labels', + 'detection_map', + 'locality_aware_nms', + 'var_conv_2d', ] diff --git a/python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py b/python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py index fb1cd35c453..33960cf4c64 100644 --- a/python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py @@ -17,7 +17,6 @@ NOT_CHECK_OP_LIST = ['deformable_conv', 'row_conv', 'kron'] # TODO(Shixiaowei02): Check if the items do not need fix. # no_grad_set has value in NEED_TO_FIX_OP_LIST -# yapf: disable NEED_TO_FIX_OP_LIST = [ 'affine_channel', 'affine_grid', @@ -72,4 +71,3 @@ NEED_TO_FIX_OP_LIST = [ 'spectral_norm', 'complex', ] -# yapf: enable diff --git a/python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py b/python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py index 9b57b0d8247..ba93c5bb874 100644 --- a/python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py @@ -14,76 +14,76 @@ # For op in NO_FP64_CHECK_GRAD_OP_LIST, the op test requires check_grad with fp64 precision NO_FP64_CHECK_GRAD_OP_LIST = [ - 'affine_grid', \ - 'clip', \ - 'conv2d', \ - 'conv2d_transpose', \ - 'conv3d', \ - 'conv3d_transpose', \ - 'conv_shift', \ - 'cos_sim', \ - 'cudnn_lstm', \ - 'cvm', \ - 'data_norm', \ - 'deformable_conv', \ - 'deformable_conv_v1', \ - 'deformable_psroi_pooling', \ - 'depthwise_conv2d', \ - 'depthwise_conv2d_transpose', \ - 'dropout', \ - 'fused_elemwise_activation', \ - 'hinge_loss', \ - 'huber_loss', \ - 'im2sequence', \ - 'increment', \ - 'l1_norm', \ - 'log_loss', \ - 'lrn', \ - 'margin_rank_loss', \ - 'match_matrix_tensor', \ - 'matmul', \ - 'max_pool2d_with_index', \ - 'max_pool3d_with_index', \ - 'minus', \ - 'modified_huber_loss', \ - 'nce', \ - 'pool2d', \ - 'pool3d', \ - 'prroi_pool', \ - 'rank_loss', \ - 'reduce_max', \ - 'reduce_min', \ - 'reshape2', \ - 'roi_perspective_transform', \ - 'row_conv', \ - 'scatter', \ - 'sequence_conv', \ - 'sequence_pool', \ - 'sequence_reverse', \ - 'sequence_slice', \ - 'sequence_topk_avg_pooling', \ - 'shuffle_channel', \ - 'sigmoid', \ - 'smooth_l1_loss', \ - 'softmax', \ - 'spectral_norm', \ - 'squared_l2_distance', \ - 'squared_l2_norm', \ - 'tanh', \ - 'mish', \ - 'transpose2', \ - 'trilinear_interp', \ - 'trilinear_interp_v2', \ - 'var_conv_2d', \ - 'warpctc', \ - 'bilateral_slice', \ - 'cast' + 'affine_grid', + 'clip', + 'conv2d', + 'conv2d_transpose', + 'conv3d', + 'conv3d_transpose', + 'conv_shift', + 'cos_sim', + 'cudnn_lstm', + 'cvm', + 'data_norm', + 'deformable_conv', + 'deformable_conv_v1', + 'deformable_psroi_pooling', + 'depthwise_conv2d', + 'depthwise_conv2d_transpose', + 'dropout', + 'fused_elemwise_activation', + 'hinge_loss', + 'huber_loss', + 'im2sequence', + 'increment', + 'l1_norm', + 'log_loss', + 'lrn', + 'margin_rank_loss', + 'match_matrix_tensor', + 'matmul', + 'max_pool2d_with_index', + 'max_pool3d_with_index', + 'minus', + 'modified_huber_loss', + 'nce', + 'pool2d', + 'pool3d', + 'prroi_pool', + 'rank_loss', + 'reduce_max', + 'reduce_min', + 'reshape2', + 'roi_perspective_transform', + 'row_conv', + 'scatter', + 'sequence_conv', + 'sequence_pool', + 'sequence_reverse', + 'sequence_slice', + 'sequence_topk_avg_pooling', + 'shuffle_channel', + 'sigmoid', + 'smooth_l1_loss', + 'softmax', + 'spectral_norm', + 'squared_l2_distance', + 'squared_l2_norm', + 'tanh', + 'mish', + 'transpose2', + 'trilinear_interp', + 'trilinear_interp_v2', + 'var_conv_2d', + 'warpctc', + 'bilateral_slice', + 'cast', ] NO_FP16_CHECK_GRAD_OP_LIST = [ - 'fused_elemwise_activation', \ - 'pool2d', \ - 'pool3d', \ - 'softmax',\ - 'conv2d_transpose' + 'fused_elemwise_activation', + 'pool2d', + 'pool3d', + 'softmax', + 'conv2d_transpose', ] diff --git a/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py b/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py index 91731c1dd0b..22bc42d9694 100644 --- a/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py @@ -13,47 +13,48 @@ # limitations under the License. NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST = [ - 'affine_channel', \ - 'bilinear_interp', \ - 'bilinear_interp_v2',\ - 'bilinear_tensor_product', \ - 'conv2d', \ - 'conv3d', \ - 'cross_entropy', \ - 'depthwise_conv2d_transpose', \ - 'grid_sampler', \ - 'group_norm', \ - 'gru', \ - 'gru_unit', \ - 'kldiv_loss', \ - 'lstm', \ - 'lstmp', \ - 'max_pool2d_with_index', \ - 'max_pool3d_with_index', \ - 'norm', \ - 'pool3d', \ - 'reduce_prod', \ - 'selu', \ - 'sigmoid_cross_entropy_with_logits', \ - 'soft_relu', \ - 'softmax_with_cross_entropy', \ - 'spp', \ - 'teacher_student_sigmoid_loss', \ - 'unpool', \ - 'yolov3_loss', \ - 'inverse', \ - 'bilateral_slice',\ - 'cudnn_lstm', \ - 'rnn', \ - 'lgamma', \ - 'sparse_attention', \ - 'svd', \ - 'matrix_power', \ - 'cholesky_solve', \ - 'solve', \ - 'qr', \ + 'affine_channel', + 'bilinear_interp', + 'bilinear_interp_v2', + 'bilinear_tensor_product', + 'conv2d', + 'conv3d', + 'cross_entropy', + 'depthwise_conv2d_transpose', + 'grid_sampler', + 'group_norm', + 'gru', + 'gru_unit', + 'kldiv_loss', + 'lstm', + 'lstmp', + 'max_pool2d_with_index', + 'max_pool3d_with_index', + 'norm', + 'pool3d', + 'reduce_prod', + 'selu', + 'sigmoid_cross_entropy_with_logits', + 'soft_relu', + 'softmax_with_cross_entropy', + 'spp', + 'teacher_student_sigmoid_loss', + 'unpool', + 'yolov3_loss', + 'inverse', + 'bilateral_slice', + 'cudnn_lstm', + 'rnn', + 'lgamma', + 'sparse_attention', + 'svd', + 'matrix_power', + 'cholesky_solve', + 'solve', + 'qr', ] -NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST = ['bilinear_interp',\ - 'bilinear_interp_v2' - ] +NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST = [ + 'bilinear_interp', + 'bilinear_interp_v2', +] diff --git a/python/paddle/fluid/tests/unittests/xpu/collective_allgather_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/collective_allgather_op_xpu.py index c2c2009a1f2..e42c5144a7b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/collective_allgather_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/collective_allgather_op_xpu.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveAllGather(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ class TestCollectiveAllGather(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allgather", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allgather", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/xpu/collective_allreduce_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/collective_allreduce_op_xpu.py index 07ea05d0ece..889eecf6327 100644 --- a/python/paddle/fluid/tests/unittests/xpu/collective_allreduce_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/collective_allreduce_op_xpu.py @@ -23,32 +23,36 @@ paddle.enable_static() class TestCollectiveAllReduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = 0 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allreduce_sum", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allreduce_sum", + inputs={'X': tindata}, + attrs={ + 'ring_id': ring_id, + }, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/xpu/collective_identity_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/collective_identity_op_xpu.py index 6737ab772d9..967d4fcc70a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/collective_identity_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/collective_identity_op_xpu.py @@ -22,7 +22,6 @@ paddle.enable_static() class TestCollectiveIdentity(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,22 +29,22 @@ class TestCollectiveIdentity(TestCollectiveRunnerBase): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_identity", - inputs={'X': tindata}, - outputs={'Out': toutdata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_identity", + inputs={'X': tindata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/xpu/get_test_cover_info.py b/python/paddle/fluid/tests/unittests/xpu/get_test_cover_info.py index c0129fabe28..f1276f765a8 100644 --- a/python/paddle/fluid/tests/unittests/xpu/get_test_cover_info.py +++ b/python/paddle/fluid/tests/unittests/xpu/get_test_cover_info.py @@ -99,7 +99,6 @@ xpu_test_device_op_type_white_list = [] class XPUOpTestWrapper(object): - def create_classes(self): base_class = None classes = [] @@ -110,7 +109,8 @@ def get_op_white_list(): op_white_list = xpu_test_op_white_list if os.getenv('XPU_TEST_OP_WHITE_LIST') is not None: op_white_list.extend( - os.getenv('XPU_TEST_OP_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_OP_WHITE_LIST').strip().split(',') + ) return list(set(op_white_list)) @@ -126,10 +126,13 @@ def get_type_white_list(): else: xpu2_type_white_list.append(t_type) - type_white_list = xpu1_type_white_list if version_str == "xpu1" else xpu2_type_white_list + type_white_list = ( + xpu1_type_white_list if version_str == "xpu1" else xpu2_type_white_list + ) if os.getenv('XPU_TEST_TYPE_WHITE_LIST') is not None: type_white_list.extend( - os.getenv('XPU_TEST_TYPE_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_TYPE_WHITE_LIST').strip().split(',') + ) return list(set(type_white_list)) @@ -137,7 +140,8 @@ def get_op_type_white_list(): op_type_white_list = xpu_test_op_type_white_list if os.getenv('XPU_TEST_OP_TYPE_WHITE_LIST') is not None: op_type_white_list.extend( - os.getenv('XPU_TEST_OP_TYPE_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_OP_TYPE_WHITE_LIST').strip().split(',') + ) return list(set(op_type_white_list)) @@ -145,7 +149,8 @@ def get_device_op_white_list(): device_op_white_list = xpu_test_device_op_white_list if os.getenv('XPU_TEST_DEVICE_OP_WHITE_LIST') is not None: device_op_white_list.extend( - os.getenv('XPU_TEST_DEVICE_OP_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_DEVICE_OP_WHITE_LIST').strip().split(',') + ) return list(set(device_op_white_list)) @@ -153,7 +158,8 @@ def get_device_op_type_white_list(): device_op_type_white_list = xpu_test_device_op_type_white_list if os.getenv('XPU_TEST_DEVICE_OP_TYPE_WHITE_LIST') is not None: device_op_type_white_list.extend( - os.getenv('XPU_TEST_DEVICE_OP_TYPE_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_DEVICE_OP_TYPE_WHITE_LIST').strip().split(',') + ) return list(set(device_op_type_white_list)) @@ -180,13 +186,15 @@ def make_xpu_op_list(xpu_version): if op_type == paddle.bfloat16: op_type = paddle.bfloat16 - if type_dict_paddle_to_str[ - op_type] in type_white_list or op_type not in type_dict_paddle_to_str.keys( - ): + if ( + type_dict_paddle_to_str[op_type] in type_white_list + or op_type not in type_dict_paddle_to_str.keys() + ): continue - device_op_type_name = device_op_name + '_' + type_dict_paddle_to_str[ - op_type] + device_op_type_name = ( + device_op_name + '_' + type_dict_paddle_to_str[op_type] + ) if device_op_type_name in device_op_type_white_list: continue @@ -201,12 +209,14 @@ def make_xpu_op_list(xpu_version): def get_xpu_op_support_types(op_name, dev_id=0): xpu_version = core.get_xpu_device_version(dev_id) support_type_list = core.get_xpu_device_op_support_types( - op_name, xpu_version) + op_name, xpu_version + ) support_type_str_list = [] for stype in support_type_list: if stype == paddle.bfloat16: support_type_str_list.append( - type_dict_paddle_to_str[paddle.bfloat16]) + type_dict_paddle_to_str[paddle.bfloat16] + ) else: support_type_str_list.append(type_dict_paddle_to_str[stype]) ops = make_xpu_op_list(xpu_version) @@ -243,12 +253,14 @@ def is_empty_grad_op_type(xpu_version, op, test_type): return False -def create_test_class(func_globals, - test_class, - test_type, - test_grad=True, - ignore_device_version=[], - test_device_version=[]): +def create_test_class( + func_globals, + test_class, + test_type, + test_grad=True, + ignore_device_version=[], + test_device_version=[], +): xpu_version = core.get_xpu_device_version(0) if xpu_version in ignore_device_version: return @@ -267,14 +279,19 @@ def create_test_class(func_globals, class_obj = test_class[1] cls_name = "{0}_{1}".format(test_class[0], str(test_type)) func_globals[cls_name] = type( - cls_name, (class_obj, ), { + cls_name, + (class_obj,), + { 'in_type': type_dict_str_to_numpy[test_type], 'in_type_str': test_type, - 'op_type_need_check_grad': True - }) - - if hasattr(test_class_obj, 'use_dynamic_create_class' - ) and test_class_obj.use_dynamic_create_class: + 'op_type_need_check_grad': True, + }, + ) + + if ( + hasattr(test_class_obj, 'use_dynamic_create_class') + and test_class_obj.use_dynamic_create_class + ): base_class, dynamic_classes = test_class_obj.dynamic_create_class() for dy_class in dynamic_classes: cls_name = "{0}_{1}".format(dy_class[0], str(test_type)) @@ -282,7 +299,7 @@ def create_test_class(func_globals, attr_dict['in_type'] = type_dict_str_to_numpy[test_type] attr_dict['in_type_str'] = test_type attr_dict['op_type_need_check_grad'] = True - func_globals[cls_name] = type(cls_name, (base_class, ), attr_dict) + func_globals[cls_name] = type(cls_name, (base_class,), attr_dict) record_op_test(op_name, test_type) if not no_grad: @@ -309,9 +326,12 @@ def get_test_cover_info(): total_len = len(set(xpu_op_list)) covered_len = len(set(xpu_op_covered)) print('{} test: {}/{}'.format(version_str, covered_len, total_len)) - if (len(diff_list) != 0): - print("These ops need to be tested on {0}! ops:{1}".format( - version_str, ','.join(diff_list))) + if len(diff_list) != 0: + print( + "These ops need to be tested on {0}! ops:{1}".format( + version_str, ','.join(diff_list) + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/xpu/test_accuracy_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_accuracy_op_xpu.py index 495f1d4fbb6..760a45055ad 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_accuracy_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_accuracy_op_xpu.py @@ -20,19 +20,21 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestAccuracyOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'accuracy' self.use_dynamic_create_class = False class TestXPUAccuracyOp(XPUOpTest): - def setUp(self): self.op_type = "accuracy" self.init_dtype() @@ -48,10 +50,11 @@ class XPUTestAccuracyOp(XPUOpTestWrapper): num_correct += 1 break self.outputs = { - 'Accuracy': - np.array([num_correct / float(n)]).astype(self.dtype), + 'Accuracy': np.array([num_correct / float(n)]).astype( + self.dtype + ), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } self.attrs = {'use_xpu': True} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_activation_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_activation_op_xpu.py index aac695e74c9..49b673133fc 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_activation_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_activation_op_xpu.py @@ -22,13 +22,16 @@ import paddle from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class TestActivationOPBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -57,13 +60,11 @@ class TestActivationOPBase(XPUOpTest): class XPUTestExpOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'exp' self.use_dynamic_create_class = False class XPUTestExp(TestActivationOPBase): - def set_case(self): self.op_type = 'exp' self.dtype = self.in_type @@ -81,13 +82,11 @@ for stype in support_types: class XPUTestSigmoidOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sigmoid' self.use_dynamic_create_class = False class XPUTestSigmoid(TestActivationOPBase): - def set_case(self): self.op_type = "sigmoid" self.dtype = self.in_type @@ -102,25 +101,22 @@ class XPUTestSigmoidOP(XPUOpTestWrapper): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestSigmoid2(XPUTestSigmoid): - def init_config(self): self.x = np.random.uniform(-2, 2, [100]).astype(self.dtype) class XPUTestSigmoid3(XPUTestSigmoid): - def init_config(self): self.x = np.random.uniform(-2, 2, [10, 12, 15]).astype(self.dtype) class XPUTestSigmoid4(XPUTestSigmoid): - def init_config(self): self.x = np.random.uniform(-2, 2, [19, 19]).astype(self.dtype) class XPUTestSigmoid5(XPUTestSigmoid): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [10, 20, 30, 40]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [10, 20, 30, 40]).astype( + self.dtype + ) support_types = get_xpu_op_support_types('sigmoid') @@ -129,13 +125,11 @@ for stype in support_types: class XPUTestTanhOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tanh' self.use_dynamic_create_class = False class XPUTestTanh(TestActivationOPBase): - def set_case(self): self.op_type = "tanh" self.dtype = self.in_type @@ -153,13 +147,11 @@ for stype in support_types: class XPUTestSqrtOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sqrt' self.use_dynamic_create_class = False class XPUTestSqrt(TestActivationOPBase): - def set_case(self): self.op_type = "sqrt" self.dtype = self.in_type @@ -178,13 +170,11 @@ for stype in support_types: class XPUTestAbsOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'abs' self.use_dynamic_create_class = False class XPUTestAbs(TestActivationOPBase): - def set_case(self): self.op_type = "abs" self.dtype = self.in_type @@ -208,13 +198,11 @@ for stype in support_types: class XPUTestReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'relu' self.use_dynamic_create_class = False class XPUTestRelu(TestActivationOPBase): - def set_case(self): self.op_type = "relu" self.dtype = self.in_type @@ -235,13 +223,11 @@ for stype in support_types: class XPUTestGeluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'gelu' self.use_dynamic_create_class = False class XPUTestGelu(TestActivationOPBase): - def set_case(self): self.op_type = "gelu" self.dtype = self.in_type @@ -262,22 +248,27 @@ for stype in support_types: def gelu(x, approximate): from scipy.special import erf + if approximate: - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * ( + 1.0 + + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))) + ) + ) else: y_ref = 0.5 * x * (1 + erf(x / np.sqrt(2))) return y_ref.astype(x.dtype) class XPUTestHardSwishOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'hard_swish' self.use_dynamic_create_class = False class XPUTestHardSwish(TestActivationOPBase): - def set_case(self): self.op_type = "hard_swish" self.dtype = self.in_type @@ -304,13 +295,11 @@ def hard_swish(x, offset, threshold, scale): class XPUTestLogOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'log' self.use_dynamic_create_class = False class XPUTestLog(TestActivationOPBase): - def set_case(self): self.op_type = "log" self.dtype = self.in_type @@ -322,22 +311,18 @@ class XPUTestLogOP(XPUOpTestWrapper): self.outputs = {'Out': out} class TestLogCase1(XPUTestLog): - def set_shape(self): self.shape = [1, 11, 17] class TestLogCase2(XPUTestLog): - def set_shape(self): self.shape = [2, 2, 2] class TestLogCase3(XPUTestLog): - def set_shape(self): self.shape = [2] class TestLogCase4(XPUTestLog): - def set_shape(self): self.shape = [1, 2, 3, 4] @@ -348,13 +333,11 @@ for stype in support_types: class XPUTestSquareOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'square' self.use_dynamic_create_class = False class XPUTestSquare(TestActivationOPBase): - def set_case(self): self.op_type = "square" self.dtype = self.in_type @@ -369,22 +352,18 @@ class XPUTestSquareOP(XPUOpTestWrapper): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestSquare2(XPUTestSquare): - def init_config(self): self.x = np.random.uniform(-2, 2, [100]).astype(self.dtype) class XPUTestSquare3(XPUTestSquare): - def init_config(self): self.x = np.random.uniform(-2, 2, [1, 15, 19]).astype(self.dtype) class XPUTestSquare4(XPUTestSquare): - def init_config(self): self.x = np.random.uniform(-2, 2, [100, 10]).astype(self.dtype) class XPUTestSquare5(XPUTestSquare): - def init_config(self): self.x = np.random.uniform(-2, 2, [1, 2, 5, 17]).astype(self.dtype) @@ -395,13 +374,11 @@ for stype in support_types: class XPUTestPowOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'pow' self.use_dynamic_create_class = False class XPUTestPowBase(TestActivationOPBase): - def set_case(self): self.op_type = "pow" self.dtype = self.in_type @@ -418,40 +395,37 @@ class XPUTestPowOP(XPUOpTestWrapper): self.factor = 3.0 class XPUTestPow1(XPUTestPowBase): - def init_config(self): self.x = np.random.uniform(-1, 1, [1024, 8]).astype(self.dtype) self.factor = 1 class XPUTestPow2(XPUTestPowBase): - def init_config(self): self.x = np.random.uniform(-1, 1, [1024, 8]).astype(self.dtype) self.factor = 2 class XPUTestPow3(XPUTestPowBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 512, 15, 15]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 512, 15, 15]).astype( + self.dtype + ) self.factor = 3 class XPUTestPow4(XPUTestPowBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 256, 22, 22]).astype( + self.dtype + ) self.factor = 4 class XPUTestPow5(XPUTestPowBase): - def init_config(self): - self.x = np.random.uniform(0, 1, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(0, 1, [4, 256, 22, 22]).astype( + self.dtype + ) self.factor = 1.2 class XPUTestPow6(XPUTestPowBase): - def init_config(self): self.x = np.random.uniform(0, 1, [1024, 8]).astype(self.dtype) self.factor = 3.2 @@ -463,13 +437,11 @@ for stype in support_types: class XPUTestLeakyReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'leaky_relu' self.use_dynamic_create_class = False class XPUTestLeakyRelu(TestActivationOPBase): - def set_case(self): self.op_type = "leaky_relu" self.dtype = self.in_type @@ -492,7 +464,7 @@ for stype in support_types: def leaky_relu(x, alpha): - if (alpha < 1): + if alpha < 1: y_ref = np.maximum(x, alpha * x) else: y_ref = np.minimum(x, alpha * x) @@ -500,13 +472,11 @@ def leaky_relu(x, alpha): class XPUTestReciprocalOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reciprocal' self.use_dynamic_create_class = False class XPUTestRecipocal(TestActivationOPBase): - def set_case(self): self.op_type = "reciprocal" self.dtype = self.in_type @@ -526,13 +496,11 @@ for stype in support_types: class XPUTestSoftPlusOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softplus' self.use_dynamic_create_class = False class XPUTestSoftPlusBase(TestActivationOPBase): - def set_case(self): self.op_type = "softplus" self.dtype = self.in_type @@ -550,21 +518,20 @@ class XPUTestSoftPlusOP(XPUOpTestWrapper): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestSoftPlus2(XPUTestSoftPlusBase): - def init_config(self): self.x = np.random.uniform(-2, 2, [1024, 8]).astype(self.dtype) class XPUTestSoftPlus3(XPUTestSoftPlusBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 512, 15, 15]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 512, 15, 15]).astype( + self.dtype + ) class XPUTestSoftPlus4(XPUTestSoftPlusBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 256, 22, 22]).astype( + self.dtype + ) support_types = get_xpu_op_support_types('softplus') @@ -574,20 +541,20 @@ for stype in support_types: def ref_softplus(x, beta=1, threshold=20): x_beta = beta * x - out = np.select([x_beta <= threshold, x_beta > threshold], - [np.log(1 + np.exp(x_beta)) / beta, x]) + out = np.select( + [x_beta <= threshold, x_beta > threshold], + [np.log(1 + np.exp(x_beta)) / beta, x], + ) return out # XPU_KP unittests, these ops can be found from xpu_op_kpfirst_list.h class XPUTestBReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'brelu' self.use_dynamic_create_class = False class XPUTestBRelu(TestActivationOPBase): - def set_case(self): self.op_type = "brelu" self.dtype = self.in_type @@ -614,13 +581,11 @@ for stype in support_types: class XPUTestCeilOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'ceil' self.use_dynamic_create_class = False class XPUTestCeil(TestActivationOPBase): - def set_case(self): self.op_type = "ceil" self.dtype = self.in_type @@ -640,13 +605,11 @@ for stype in support_types: class XPUTestCeluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'celu' self.use_dynamic_create_class = False class XPUTestCelu(TestActivationOPBase): - def set_case(self): self.op_type = "celu" self.dtype = self.in_type @@ -671,18 +634,16 @@ def ref_celu(x, alpha): class XPUTestEluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elu' self.use_dynamic_create_class = False class XPUTestElu(TestActivationOPBase): - def set_case(self): self.op_type = "elu" self.dtype = self.in_type - alpha = 1. + alpha = 1.0 x = np.random.uniform(-3, 3, [10, 12]).astype(self.dtype) out = ref_elu(x, alpha) @@ -702,13 +663,11 @@ def ref_elu(x, alpha): class XPUTestFloorOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'floor' self.use_dynamic_create_class = False class XPUTestFloor(TestActivationOPBase): - def set_case(self): self.op_type = "floor" self.dtype = self.in_type @@ -728,13 +687,11 @@ for stype in support_types: class XPUTestHardShrinkOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'hard_shrink' self.use_dynamic_create_class = False class XPUTestHardShrink(TestActivationOPBase): - def set_case(self): self.op_type = "hard_shrink" self.dtype = self.in_type @@ -762,13 +719,11 @@ def ref_hardshrink(x, threshold): class XPUTestHardSigmoidOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'hard_sigmoid' self.use_dynamic_create_class = False class XPUTestHardSigmoid(TestActivationOPBase): - def set_case(self): self.op_type = "hard_sigmoid" self.dtype = self.in_type @@ -777,7 +732,7 @@ class XPUTestHardSigmoidOP(XPUOpTestWrapper): x = np.random.uniform(-5, 5, [10, 12]).astype(self.dtype) lower_threshold = -self.offset / self.slope - upper_threshold = (1. - self.offset) / self.slope + upper_threshold = (1.0 - self.offset) / self.slope # Same reason as TestAbs delta = 0.005 @@ -789,7 +744,7 @@ class XPUTestHardSigmoidOP(XPUOpTestWrapper): self.attrs = { 'use_xpu': True, 'slope': self.slope, - 'offset': self.offset + 'offset': self.offset, } self.inputs = {'X': x} self.outputs = {'Out': out} @@ -801,17 +756,15 @@ for stype in support_types: def ref_hardsigmoid(x, slope=0.166666666666667, offset=0.5): - return np.maximum(np.minimum(x * slope + offset, 1.), 0.).astype(x.dtype) + return np.maximum(np.minimum(x * slope + offset, 1.0), 0.0).astype(x.dtype) class XPUTestLog1pOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'log1p' self.use_dynamic_create_class = False class XPUTestLog1p(TestActivationOPBase): - def set_case(self): self.op_type = "log1p" self.dtype = self.in_type @@ -831,13 +784,11 @@ for stype in support_types: class XPUTestLogsigmoidOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logsigmoid' self.use_dynamic_create_class = False class XPUTestLogsigmoid(TestActivationOPBase): - def set_case(self): self.op_type = "logsigmoid" self.dtype = self.in_type @@ -857,13 +808,11 @@ for stype in support_types: class XPUTestRelu6OP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'relu6' self.use_dynamic_create_class = False class XPUTestRelu6(TestActivationOPBase): - def set_case(self): self.op_type = "relu6" self.dtype = self.in_type @@ -891,13 +840,11 @@ def ref_relu6(x, threshold=6.0): class XPUTestSiluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'silu' self.use_dynamic_create_class = False class XPUTestSilu(TestActivationOPBase): - def set_case(self): self.op_type = "silu" self.dtype = self.in_type @@ -917,13 +864,11 @@ for stype in support_types: class XPUTestSoftReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'soft_relu' self.use_dynamic_create_class = False class XPUTestSoftRelu(TestActivationOPBase): - def set_case(self): self.op_type = "soft_relu" self.dtype = self.in_type @@ -950,13 +895,11 @@ for stype in support_types: class XPUTestSoftSignOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softsign' self.use_dynamic_create_class = False class XPUTestSoftSign(TestActivationOPBase): - def set_case(self): self.op_type = "softsign" self.dtype = self.in_type @@ -981,13 +924,11 @@ def ref_softsign(x): class XPUTestSoftshrinkOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softshrink' self.use_dynamic_create_class = False class XPUTestSoftshrink(TestActivationOPBase): - def set_case(self): self.op_type = "softshrink" self.dtype = self.in_type @@ -1010,18 +951,17 @@ for stype in support_types: def ref_softshrink(x, threshold=0.5): out = np.copy(x) out = (out < -threshold) * (out + threshold) + (out > threshold) * ( - out - threshold) + out - threshold + ) return out class XPUTestSwishOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'swish' self.use_dynamic_create_class = False class XPUTestSwishBase(TestActivationOPBase): - def set_case(self): self.op_type = "swish" self.dtype = self.in_type @@ -1037,21 +977,20 @@ class XPUTestSwishOP(XPUOpTestWrapper): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestSwish2(XPUTestSwishBase): - def init_config(self): self.x = np.random.uniform(-2, 2, [1024, 8]).astype(self.dtype) class XPUTestSwish3(XPUTestSwishBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 512, 15, 15]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 512, 15, 15]).astype( + self.dtype + ) class XPUTestSwish4(XPUTestSwishBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 256, 22, 22]).astype( + self.dtype + ) support_types = get_xpu_op_support_types('swish') @@ -1061,18 +1000,17 @@ for stype in support_types: def ref_swish(x): from scipy.special import expit + out = x * expit(x) return out class XPUTestThresholdedReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'thresholded_relu' self.use_dynamic_create_class = False class XPUTestThresholdedRelu(TestActivationOPBase): - def set_case(self): self.op_type = "thresholded_relu" self.dtype = self.in_type @@ -1099,13 +1037,11 @@ def ref_thresholded_relu(x, threshold=1.0): class XPUTestMishOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'mish' self.use_dynamic_create_class = False class XPUTestMishBase(TestActivationOPBase): - def set_case(self): self.op_type = "mish" self.dtype = self.in_type @@ -1122,21 +1058,20 @@ class XPUTestMishOP(XPUOpTestWrapper): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestMish2(XPUTestMishBase): - def init_config(self): self.x = np.random.uniform(-2, 2, [1024, 8]).astype(self.dtype) class XPUTestMish3(XPUTestMishBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 512, 15, 15]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 512, 15, 15]).astype( + self.dtype + ) class XPUTestMish4(XPUTestMishBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 256, 22, 22]).astype( + self.dtype + ) support_types = get_xpu_op_support_types('mish') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_adam_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_adam_op_xpu.py index 332fc3fd489..fb42d564577 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_adam_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_adam_op_xpu.py @@ -22,18 +22,20 @@ from paddle.fluid.op import Operator import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestAdamOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'adam' self.use_dynamic_create_class = False class TestAdamOp(XPUOpTest): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def setUp(self): self.init_dtype() @@ -45,20 +47,18 @@ class XPUTestAdamOp(XPUOpTestWrapper): self.set_shape() self.set_inputs() self.set_steps() - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step( + self.inputs, self.attrs + ) self.outputs = { - 'Moment1Out': - moment1_out, - 'Moment2Out': - moment2_out, - 'ParamOut': - param_out, - 'Beta1PowOut': - np.array([self.beta1_pow]).astype("float32") * self.beta1, - 'Beta2PowOut': - np.array([self.beta2_pow]).astype("float32") * self.beta2 + 'Moment1Out': moment1_out, + 'Moment2Out': moment2_out, + 'ParamOut': param_out, + 'Beta1PowOut': np.array([self.beta1_pow]).astype("float32") + * self.beta1, + 'Beta2PowOut': np.array([self.beta2_pow]).astype("float32") + * self.beta2, } def set_xpu(self): @@ -73,7 +73,7 @@ class XPUTestAdamOp(XPUOpTestWrapper): self.attrs = { 'epsilon': self.epsilon, 'beta1': self.beta1, - 'beta2': self.beta2 + 'beta2': self.beta2, } def set_data(self): @@ -103,18 +103,18 @@ class XPUTestAdamOp(XPUOpTestWrapper): 'Grad': grad, 'Moment1': moment1, 'Moment2': moment2, - 'LearningRate': - np.array([self.learning_rate]).astype("float32"), + 'LearningRate': np.array([self.learning_rate]).astype( + "float32" + ), 'Beta1Pow': np.array([self.beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([self.beta2_pow]).astype("float32") + 'Beta2Pow': np.array([self.beta2_pow]).astype("float32"), } def test_check_output(self): self.check_output_with_place(place=paddle.XPUPlace(0), atol=1e-2) class TestAdamOp2(TestAdamOp): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def set_data(self): self.beta1 = 0.9 @@ -123,29 +123,25 @@ class XPUTestAdamOp(XPUOpTestWrapper): self.epsilon = 1e-8 class TestAdamOp3(TestAdamOp2): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def set_shape(self): self.shape = (101, 47) class TestAdamOp4(TestAdamOp2): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def set_shape(self): self.shape = (512, 26) class TestAdamOp5(TestAdamOp2): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def set_shape(self): self.shape = (11, 1) class TestAdamOp6(TestAdamOp2): - '''Test Adam Op with beta as Variable - ''' + '''Test Adam Op with beta as Variable''' def set_shape(self): self.shape = (10, 10) @@ -157,8 +153,7 @@ class XPUTestAdamOp(XPUOpTestWrapper): self.epsilon = 1e-8 class TestAdamOp7(TestAdamOp): - '''Test Adam Op with float16 accuracy - ''' + '''Test Adam Op with float16 accuracy''' def setUp(self): self.init_dtype() @@ -170,20 +165,18 @@ class XPUTestAdamOp(XPUOpTestWrapper): self.set_shape() self.set_inputs() self.set_steps() - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step( + self.inputs, self.attrs + ) self.outputs = { - 'Moment1Out': - moment1_out, - 'Moment2Out': - moment2_out, - 'ParamOut': - param_out, - 'Beta1PowOut': - np.array([self.beta1_pow]).astype("float16") * self.beta1, - 'Beta2PowOut': - np.array([self.beta2_pow]).astype("float16") * self.beta2 + 'Moment1Out': moment1_out, + 'Moment2Out': moment2_out, + 'ParamOut': param_out, + 'Beta1PowOut': np.array([self.beta1_pow]).astype("float16") + * self.beta1, + 'Beta2PowOut': np.array([self.beta2_pow]).astype("float16") + * self.beta2, } def set_inputs(self): @@ -201,23 +194,24 @@ class XPUTestAdamOp(XPUOpTestWrapper): 'Grad': grad, 'Moment1': moment1, 'Moment2': moment2, - 'LearningRate': - np.array([self.learning_rate]).astype("float16"), + 'LearningRate': np.array([self.learning_rate]).astype( + "float16" + ), 'Beta1Pow': np.array([self.beta1_pow]).astype("float16"), - 'Beta2Pow': np.array([self.beta2_pow]).astype("float16") + 'Beta2Pow': np.array([self.beta2_pow]).astype("float16"), } class TestAdamOpMultipleSteps(TestAdamOp2): - '''Test Adam Operator with supplied attributes - ''' + '''Test Adam Operator with supplied attributes''' def set_steps(self): self.num_steps = 10 def test_check_output(self): for _ in range(self.num_steps): - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step( + self.inputs, self.attrs + ) beta1_pow_out = self.inputs['Beta1Pow'] * self.beta1 beta2_pow_out = self.inputs['Beta2Pow'] * self.beta2 @@ -226,12 +220,13 @@ class XPUTestAdamOp(XPUOpTestWrapper): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } # Verify output for this step - self.check_output_with_place(place=paddle.XPUPlace(0), - atol=1e-2) + self.check_output_with_place( + place=paddle.XPUPlace(0), atol=1e-2 + ) # Output of this step becomes input for next step self.inputs['Param'] = param_out @@ -244,7 +239,8 @@ class XPUTestAdamOp(XPUOpTestWrapper): # Randomize gradient for next step self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + -1, 1, (102, 105) + ).astype("float32") def adam_step(inputs, attributes): @@ -281,8 +277,9 @@ def adam_step(inputs, attributes): return param_out, moment1_out, moment2_out -def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, - lazy_mode): +def adam_step_sparse( + inputs, attributes, height, rows, row_numel, np_grad, lazy_mode +): ''' Simulate one step of the adam optimizer :param inputs: dict of inputs @@ -307,13 +304,16 @@ def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, param_out = np.zeros(shape=[height, row_numel]) def update_row(row_id, update_value): - moment1_out[row_id] = beta1 * moment1[row_id] + (1 - - beta1) * update_value - moment2_out[row_id] = beta2 * moment2[row_id] + ( - 1 - beta2) * np.square(update_value) + moment1_out[row_id] = ( + beta1 * moment1[row_id] + (1 - beta1) * update_value + ) + moment2_out[row_id] = beta2 * moment2[row_id] + (1 - beta2) * np.square( + update_value + ) lr_t = lr * np.sqrt(1 - beta2_pow) / (1 - beta1_pow) param_out[row_id] = param[row_id] - lr_t * ( - moment1_out[row_id] / (np.sqrt(moment2_out[row_id]) + epsilon)) + moment1_out[row_id] / (np.sqrt(moment2_out[row_id]) + epsilon) + ) if lazy_mode: for idx, row_id in enumerate(rows): @@ -329,7 +329,6 @@ def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, class TestSparseAdamOp(unittest.TestCase): - def setup(self, scope, place, lazy_mode): beta1 = 0.78 beta2 = 0.836 @@ -348,14 +347,14 @@ class TestSparseAdamOp(unittest.TestCase): "Moment2": np.full((height, row_numel), 5.0).astype("float32"), 'Beta1Pow': beta1_pow, 'Beta2Pow': beta2_pow, - "LearningRate": np.full((1), 2.0).astype("float32") + "LearningRate": np.full((1), 2.0).astype("float32"), } self.init_output = np.full((height, row_numel), 0.0).astype("float32") self.attrs = { 'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2, - 'min_row_size_to_use_multithread': 2 + 'min_row_size_to_use_multithread': 2, } grad_selected_rows = scope.var('Grad').get_selected_rows() @@ -370,15 +369,21 @@ class TestSparseAdamOp(unittest.TestCase): self.sparse_inputs = ["Grad"] - param_out, mom1, mom2 = adam_step_sparse(self.dense_inputs, self.attrs, - height, rows, row_numel, - np_array, lazy_mode) + param_out, mom1, mom2 = adam_step_sparse( + self.dense_inputs, + self.attrs, + height, + rows, + row_numel, + np_array, + lazy_mode, + ) self.outputs = { "ParamOut": param_out, "Moment1Out": mom1, "Moment2Out": mom2, 'Beta1PowOut': beta1_pow * beta1, - 'Beta2PowOut': beta2_pow * beta2 + 'Beta2PowOut': beta2_pow * beta2, } def check_with_place(self, place, lazy_mode): @@ -421,7 +426,6 @@ class TestSparseAdamOp(unittest.TestCase): class TestSparseAdamOp1(TestSparseAdamOp): - def setup(self, scope, place, lazy_mode): beta1 = 0.78 beta2 = 0.836 @@ -440,14 +444,14 @@ class TestSparseAdamOp1(TestSparseAdamOp): "Moment2": np.full((height, row_numel), 5.0).astype("float16"), 'Beta1Pow': beta1_pow, 'Beta2Pow': beta2_pow, - "LearningRate": np.full((1), 2.0).astype("float16") + "LearningRate": np.full((1), 2.0).astype("float16"), } self.init_output = np.full((height, row_numel), 0.0).astype("float16") self.attrs = { 'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2, - 'min_row_size_to_use_multithread': 2 + 'min_row_size_to_use_multithread': 2, } grad_selected_rows = scope.var('Grad').get_selected_rows() @@ -462,15 +466,21 @@ class TestSparseAdamOp1(TestSparseAdamOp): self.sparse_inputs = ["Grad"] - param_out, mom1, mom2 = adam_step_sparse(self.dense_inputs, self.attrs, - height, rows, row_numel, - np_array, lazy_mode) + param_out, mom1, mom2 = adam_step_sparse( + self.dense_inputs, + self.attrs, + height, + rows, + row_numel, + np_array, + lazy_mode, + ) self.outputs = { "ParamOut": param_out, "Moment1Out": mom1, "Moment2Out": mom2, 'Beta1PowOut': beta1_pow * beta1, - 'Beta2PowOut': beta2_pow * beta2 + 'Beta2PowOut': beta2_pow * beta2, } diff --git a/python/paddle/fluid/tests/unittests/xpu/test_adamw_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_adamw_op_xpu.py index bc4c9095e8f..7280258272b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_adamw_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_adamw_op_xpu.py @@ -23,7 +23,11 @@ import paddle.fluid as fluid from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def adamw_step(inputs, attributes): @@ -70,19 +74,17 @@ def simple_lr_setting(param, decay_rate, n_layers): else: depth = 0 - return decay_rate**(n_layers + 2 - depth) + return decay_rate ** (n_layers + 2 - depth) class XPUTestAdamwOp1(XPUOpTestWrapper): - def __init__(self): self.op_name = 'adamw' self.use_dynamic_create_class = False class TestAdamW(XPUOpTest): - def setUp(self): - #Test AdamW Op with supplied attributes + # Test AdamW Op with supplied attributes self.op_type = "adamw" self.init_shape() self.dtype = self.in_type_str @@ -106,7 +108,7 @@ class XPUTestAdamwOp1(XPUOpTestWrapper): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype(self.dtype), 'Beta1Pow': np.array([beta1_pow]).astype(self.dtype), - 'Beta2Pow': np.array([beta2_pow]).astype(self.dtype) + 'Beta2Pow': np.array([beta2_pow]).astype(self.dtype), } self.attrs = { @@ -114,18 +116,19 @@ class XPUTestAdamwOp1(XPUOpTestWrapper): 'beta1': beta1, 'beta2': beta2, "coeff": 0.5, - "with_decay": True + "with_decay": True, } - param_out, moment1_out, \ - moment2_out = adamw_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adamw_step( + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype(self.dtype) * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype(self.dtype) * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype(self.dtype) * beta2, } def init_shape(self): @@ -136,26 +139,22 @@ class XPUTestAdamwOp1(XPUOpTestWrapper): self.check_output_with_place(place=paddle.XPUPlace(0)) class TestAdamW2(TestAdamW): - def init_shape(self): self.shape = [ 1000, ] class TestAdamW3(TestAdamW): - def init_shape(self): self.shape = [200, 3000] class XPUTestAdamwOp2(XPUOpTestWrapper): - def __init__(self): self.op_name = 'adamw' self.use_dynamic_create_class = False class TestAdamWOp(unittest.TestCase): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype(self.in_type_str) @@ -165,7 +164,8 @@ class XPUTestAdamwOp2(XPUOpTestWrapper): learning_rate=0.01, parameters=linear.parameters(), apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + weight_decay=0.01, + ) for _ in range(2): out = linear(a) @@ -182,8 +182,9 @@ class XPUTestAdamwOp2(XPUOpTestWrapper): learning_rate=0.0, parameters=linear.parameters(), apply_decay_param_fun=lambda name: True, - weight_decay=0.01) - assert (adam.__str__() is not None) + weight_decay=0.01, + ) + assert adam.__str__() is not None def test_adamw_op(self): paddle.enable_static() @@ -202,25 +203,29 @@ class XPUTestAdamwOp2(XPUOpTestWrapper): shape=[1], value=0.85, dtype=self.in_type_str, - persistable=True) + persistable=True, + ) beta2 = fluid.layers.create_global_var( shape=[1], value=0.95, dtype=self.in_type_str, - persistable=True) + persistable=True, + ) betas = [beta1, beta2] - opt = paddle.optimizer.AdamW(learning_rate=1e-5, - beta1=beta1, - beta2=beta2, - weight_decay=0.01, - epsilon=1e-8) + opt = paddle.optimizer.AdamW( + learning_rate=1e-5, + beta1=beta1, + beta2=beta2, + weight_decay=0.01, + epsilon=1e-8, + ) opt.minimize(loss) exe.run(startup) data_np = np.random.random(shape).astype(self.in_type_str) - rets = exe.run(train_prog, - feed={"data": data_np}, - fetch_list=[loss]) + rets = exe.run( + train_prog, feed={"data": data_np}, fetch_list=[loss] + ) assert rets[0] is not None paddle.disable_static() @@ -228,20 +233,19 @@ class XPUTestAdamwOp2(XPUOpTestWrapper): paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - beta1=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, beta1=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - beta2=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, beta2=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, epsilon=-1, parameters=linear.parameters() + ) class TestAdamWOpGroup(TestAdamWOp): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype(self.in_type_str) @@ -250,14 +254,13 @@ class XPUTestAdamwOp2(XPUOpTestWrapper): linear_2 = paddle.nn.Linear(5, 3) adam = paddle.optimizer.AdamW( learning_rate=0.01, - parameters=[{ - 'params': linear_1.parameters() - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001 - }], + parameters=[ + {'params': linear_1.parameters()}, + {'params': linear_2.parameters(), 'weight_decay': 0.001}, + ], apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + weight_decay=0.01, + ) for _ in range(2): out = linear_1(a) @@ -267,7 +270,6 @@ class XPUTestAdamwOp2(XPUOpTestWrapper): adam.clear_gradients() class TestAdamWOpGroupWithLR(TestAdamWOp): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype(self.in_type_str) @@ -276,16 +278,21 @@ class XPUTestAdamwOp2(XPUOpTestWrapper): linear_2 = paddle.nn.Linear(5, 3) adam = paddle.optimizer.AdamW( learning_rate=paddle.optimizer.lr.PiecewiseDecay( - boundaries=[3, 6], values=[0.1, 0.2, 0.3]), - parameters=[{ - 'params': linear_1.parameters(), - 'learning_rate': 0.1, - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001, - }], + boundaries=[3, 6], values=[0.1, 0.2, 0.3] + ), + parameters=[ + { + 'params': linear_1.parameters(), + 'learning_rate': 0.1, + }, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + }, + ], apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + weight_decay=0.01, + ) for _ in range(2): out = linear_1(a) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_affine_channel_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_affine_channel_op_xpu.py index 447f18bd3c2..0718c040bca 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_affine_channel_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_affine_channel_op_xpu.py @@ -39,7 +39,6 @@ def affine_channel(x, scale, bias, layout): class TestAffineChannelOp(XPUOpTest): - def setUp(self): self.op_type = "affine_channel" self.init_test_case() @@ -70,17 +69,17 @@ class TestAffineChannelOp(XPUOpTest): if core.is_compiled_with_xpu(): paddle.enable_static() place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Scale', 'Bias'], - 'Out', - no_grad_set=set('X')) + self.check_grad_with_place( + place, ['Scale', 'Bias'], 'Out', no_grad_set=set('X') + ) def test_check_grad_stopgrad_dscale_dbias(self): if core.is_compiled_with_xpu(): paddle.enable_static() place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - no_grad_set=set(['Scale', 'Bias'])) + self.check_grad_with_place( + place, ['X'], 'Out', no_grad_set=set(['Scale', 'Bias']) + ) def init_test_case(self): self.shape = [2, 100, 3, 3] @@ -89,7 +88,6 @@ class TestAffineChannelOp(XPUOpTest): class TestAffineChannelOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): @@ -100,32 +98,31 @@ class TestAffineChannelOpError(unittest.TestCase): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[None, 1, 2, 2], - dtype='int32') + x2 = fluid.layers.data( + name='x2', shape=[None, 1, 2, 2], dtype='int32' + ) fluid.layers.affine_channel(x2) self.assertRaises(TypeError, test_x_dtype) def test_scale_type(): - x3 = fluid.layers.data(name='x3', - shape=[None, 1, 2, 2], - dtype='float32') + x3 = fluid.layers.data( + name='x3', shape=[None, 1, 2, 2], dtype='float32' + ) fluid.layers.affine_channel(x3, scale=1) self.assertRaises(TypeError, test_scale_type) def test_bias_type(): - x4 = fluid.layers.data(name='x4', - shape=[None, 1, 2, 2], - dtype='float32') + x4 = fluid.layers.data( + name='x4', shape=[None, 1, 2, 2], dtype='float32' + ) fluid.layers.affine_channel(x4, bias=1) self.assertRaises(TypeError, test_bias_type) class TestAffineChannelNHWC(TestAffineChannelOp): - def init_test_case(self): self.shape = [2, 3, 3, 100] self.C = 100 @@ -139,7 +136,6 @@ class TestAffineChannelNHWC(TestAffineChannelOp): class TestAffineChannel2D(TestAffineChannelOp): - def init_test_case(self): self.shape = [2, 100] self.C = 100 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py index e6bc61b895a..b6e95e25690 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py @@ -19,19 +19,21 @@ import paddle import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCheckFiniteAndUnscaleOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'check_finite_and_unscale' self.use_dynamic_create_class = False class TestCheckFiniteAndUnscaleOpNormal(XPUOpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() @@ -52,7 +54,6 @@ class XPUTestCheckFiniteAndUnscaleOp(XPUOpTestWrapper): self.check_output_with_place(place) class TestCheckFiniteAndUnscaleOpWithNan(XPUOpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() @@ -80,7 +81,6 @@ class XPUTestCheckFiniteAndUnscaleOp(XPUOpTestWrapper): self.check_output_with_place(place, no_check_set=['Out']) class TestCheckFiniteAndUnscaleOpWithInf(XPUOpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() @@ -108,7 +108,6 @@ class XPUTestCheckFiniteAndUnscaleOp(XPUOpTestWrapper): self.check_output_with_place(place, no_check_set=['Out']) class TestCheckFiniteAndUnscaleOpWithInfAndNan(XPUOpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_arg_max_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_arg_max_op_xpu.py index cdc16d45072..d041e859d98 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_arg_max_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_arg_max_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestArgMax(XPUOpTestWrapper): - def __init__(self): self.op_name = 'arg_max' class XPUBaseTestCase(XPUOpTest): - def initTestCase(self): self.dims = (3, 4) self.axis = 1 @@ -52,63 +54,53 @@ class XPUTestArgMax(XPUOpTestWrapper): self.check_output_with_place(place) class TestArgMaxCase1(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.axis = -1 class TestArgMaxCase2(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.axis = 0 class TestArgMaxCase3(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.axis = 1 class TestArgMaxCase4(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.axis = 2 class TestArgMaxCase5(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4) self.axis = -1 class TestArgMaxCase6(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4) self.axis = 0 class TestArgMaxCase7(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4) self.axis = 1 class TestArgMaxCase8(XPUBaseTestCase): - def initTestCase(self): - self.dims = (1, ) + self.dims = (1,) self.axis = 0 class TestArgMaxCase9(XPUBaseTestCase): - def initTestCase(self): - self.dims = (2, ) + self.dims = (2,) self.axis = 0 class TestArgMaxCase10(XPUBaseTestCase): - def initTestCase(self): - self.dims = (3, ) + self.dims = (3,) self.axis = 0 @@ -118,7 +110,6 @@ for stype in support_types: class TestArgMaxAPI(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -130,7 +121,6 @@ class TestArgMaxAPI(unittest.TestCase): self.place = [paddle.XPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -138,9 +128,9 @@ class TestArgMaxAPI(unittest.TestCase): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input, axis=self.axis) paddle_output = paddle.argmax(tensor_input, axis=self.axis) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() for place in self.place: @@ -148,7 +138,6 @@ class TestArgMaxAPI(unittest.TestCase): class TestArgMaxAPI_2(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -161,20 +150,20 @@ class TestArgMaxAPI_2(unittest.TestCase): self.place = [paddle.XPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) numpy_input = (np.random.random(self.dims)).astype(self.dtype) tensor_input = paddle.to_tensor(numpy_input) - numpy_output = np.argmax(numpy_input, - axis=self.axis).reshape(1, 4, 5) - paddle_output = paddle.argmax(tensor_input, - axis=self.axis, - keepdim=self.keep_dims) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + numpy_output = np.argmax(numpy_input, axis=self.axis).reshape( + 1, 4, 5 + ) + paddle_output = paddle.argmax( + tensor_input, axis=self.axis, keepdim=self.keep_dims + ) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_argsort_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_argsort_op_xpu.py index 039d3c2051e..8ee7716447f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_argsort_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_argsort_op_xpu.py @@ -20,13 +20,16 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestArgsortOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'argsort' self.use_dynamic_create_class = True @@ -36,14 +39,14 @@ class XPUTestArgsortOp(XPUOpTestWrapper): classes = [] for descending in [True, False]: for axis in [0, 1, 2, -1, -2]: - class_name = 'XPUTestArgsortOp_axis_' + str(axis) + '_' + str( - descending) + class_name = ( + 'XPUTestArgsortOp_axis_' + str(axis) + '_' + str(descending) + ) attr_dict = {'init_axis': axis, 'init_descending': descending} classes.append([class_name, attr_dict]) return base_class, classes class TestArgsortOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "argsort" @@ -51,16 +54,18 @@ class XPUTestArgsortOp(XPUOpTestWrapper): self.dtype = self.in_type self.input_shape = (2, 2, 2, 3, 3) self.axis = -1 if not hasattr(self, 'init_axis') else self.init_axis - self.descending = False if not hasattr( - self, 'init_descending') else self.init_descending + self.descending = ( + False + if not hasattr(self, 'init_descending') + else self.init_descending + ) if self.dtype == np.float32: self.x = np.random.random(self.input_shape).astype(self.dtype) else: - self.x = np.random.randint(low=-1000, - high=1000, - size=self.input_shape).astype( - self.dtype) + self.x = np.random.randint( + low=-1000, high=1000, size=self.input_shape + ).astype(self.dtype) self.inputs = {"X": self.x} self.attrs = {"axis": self.axis, "descending": self.descending} @@ -71,13 +76,15 @@ class XPUTestArgsortOp(XPUOpTestWrapper): if self.descending: self.indices = np.flip( np.argsort(self.x, kind='heapsort', axis=self.axis), - self.axis) + self.axis, + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.sort(self.x, kind='heapsort', axis=self.axis), self.axis + ) else: - self.indices = np.argsort(self.x, - kind='heapsort', - axis=self.axis) + self.indices = np.argsort( + self.x, kind='heapsort', axis=self.axis + ) self.sorted_x = np.sort(self.x, kind='heapsort', axis=self.axis) def set_xpu(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_assign_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_assign_op_xpu.py index 306ebf957c3..74f2a41a9ee 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_assign_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_assign_op_xpu.py @@ -17,6 +17,7 @@ import sys sys.path.append("..") import unittest import paddle + ''' class TestAssignOp(op_test.OpTest): def setUp(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_assign_value_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_assign_value_op_xpu.py index c06ef86b4fe..abb8dff9ecb 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_assign_value_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_assign_value_op_xpu.py @@ -21,20 +21,22 @@ import paddle.fluid as fluid import paddle.fluid.framework as framework import paddle.fluid.layers as layers from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestAssignValueOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'assign_value' self.use_dynamic_create_class = False class TestAssignValueOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -47,7 +49,8 @@ class XPUTestAssignValueOp(XPUOpTestWrapper): self.init_data() self.attrs["shape"] = self.value.shape self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( - self.value.dtype) + self.value.dtype + ) self.outputs = {"Out": self.value} def init_data(self): @@ -58,31 +61,29 @@ class XPUTestAssignValueOp(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestAssignValueOp2(TestAssignValueOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int32) self.attrs["int32_values"] = [int(v) for v in self.value.flat] class TestAssignValueOp3(TestAssignValueOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int64) self.attrs["int64_values"] = [int(v) for v in self.value.flat] class TestAssignValueOp4(TestAssignValueOp): - def init_data(self): - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) self.attrs["bool_values"] = [int(v) for v in self.value.flat] class TestAssignApi(unittest.TestCase): - def setUp(self): self.init_dtype() self.value = (-100 + 200 * np.random.random(size=(2, 5))).astype( - self.dtype) + self.dtype + ) self.place = fluid.XPUPlace(0) def init_dtype(self): @@ -101,23 +102,21 @@ class TestAssignApi(unittest.TestCase): class TestAssignApi2(TestAssignApi): - def init_dtype(self): self.dtype = "int32" class TestAssignApi3(TestAssignApi): - def init_dtype(self): self.dtype = "int64" class TestAssignApi4(TestAssignApi): - def setUp(self): self.init_dtype() - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) self.place = fluid.XPUPlace(0) def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_batch_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_batch_norm_op_xpu.py index 09d813d2376..7d818cc02c8 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_batch_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_batch_norm_op_xpu.py @@ -21,13 +21,18 @@ import paddle.fluid.core as core import paddle import paddle.fluid as fluid import paddle.nn.functional as F -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def ref_batch_norm_infer(x, scale, bias, mean, variance, momentum, epsilon, - data_layout): +def ref_batch_norm_infer( + x, scale, bias, mean, variance, momentum, epsilon, data_layout +): if data_layout == "NCHW": n, c, h, w = x.shape mean_tile = np.reshape(mean, (1, c, 1, 1)) @@ -46,12 +51,14 @@ def ref_batch_norm_infer(x, scale, bias, mean, variance, momentum, epsilon, else: raise ValueError( "Unsupported data layout! Only NCHW and NHWC is supported, but received " - + data_layout) + + data_layout + ) return y -def ref_batch_norm_train(x, y_grad, scale, bias, mean, variance, momentum, - epsilon, data_layout): +def ref_batch_norm_train( + x, y_grad, scale, bias, mean, variance, momentum, epsilon, data_layout +): # Forward if data_layout == "NCHW": n, c, h, w = x.shape @@ -65,8 +72,9 @@ def ref_batch_norm_train(x, y_grad, scale, bias, mean, variance, momentum, saved_mean_tile = np.tile(saved_mean_tile, (n, 1, h, w)) saved_variance_tile = np.reshape(saved_variance, (1, c, 1, 1)) saved_variance_tile = np.tile(saved_variance_tile, (n, 1, h, w)) - normalized_x = (x - saved_mean_tile) / np.sqrt(saved_variance_tile + - epsilon) + normalized_x = (x - saved_mean_tile) / np.sqrt( + saved_variance_tile + epsilon + ) scale_tile = np.reshape(scale, (1, c, 1, 1)) scale_tile = np.tile(scale_tile, (n, 1, h, w)) bias_tile = np.reshape(bias, (1, c, 1, 1)) @@ -84,10 +92,11 @@ def ref_batch_norm_train(x, y_grad, scale, bias, mean, variance, momentum, else: raise ValueError( "Unsupported data layout! Only NCHW and NHWC is supported, but received " - + data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = saved_variance * (1. - momentum) + momentum * variance - saved_inv_std = 1. / np.sqrt(saved_variance + epsilon) + + data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = saved_variance * (1.0 - momentum) + momentum * variance + saved_inv_std = 1.0 / np.sqrt(saved_variance + epsilon) # Backward # Use the following formulas to calculate gradients: # grad_scale = @@ -102,32 +111,48 @@ def ref_batch_norm_train(x, y_grad, scale, bias, mean, variance, momentum, if data_layout == "NCHW": x = np.transpose(x, (0, 2, 3, 1)) y_grad = np.transpose(y_grad, (0, 2, 3, 1)) - x_grad = scale * ( - y_grad - np.mean(y_grad, axis=(0, 1, 2)) - - (x - saved_mean) * np.mean(y_grad * (x - saved_mean), axis=(0, 1, 2)) / - (saved_variance + epsilon)) / np.sqrt(saved_variance + epsilon) - scale_grad = np.sum(y_grad * (x - saved_mean) / - np.sqrt(saved_variance + epsilon), - axis=(0, 1, 2)) + x_grad = ( + scale + * ( + y_grad + - np.mean(y_grad, axis=(0, 1, 2)) + - (x - saved_mean) + * np.mean(y_grad * (x - saved_mean), axis=(0, 1, 2)) + / (saved_variance + epsilon) + ) + / np.sqrt(saved_variance + epsilon) + ) + scale_grad = np.sum( + y_grad * (x - saved_mean) / np.sqrt(saved_variance + epsilon), + axis=(0, 1, 2), + ) bias_grad = np.sum(y_grad, axis=(0, 1, 2)) # Transfer back to N, C, H, W if data_layout == "NCHW": x_grad = np.transpose(x_grad, (0, 3, 1, 2)) x = np.transpose(x, (0, 3, 1, 2)) y_grad = np.transpose(y_grad, (0, 3, 1, 2)) - return y, mean_out, variance_out, saved_mean, saved_inv_std, x_grad, scale_grad, bias_grad + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_inv_std, + x_grad, + scale_grad, + bias_grad, + ) class XPUTestBatchNormOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'batch_norm' self.use_dynamic_create_class = False - @unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") + @unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" + ) class TestBatchNormOp(unittest.TestCase): - def setUp(self): self.op_type = "batch_norm" self.dtype = np.float32 @@ -146,13 +171,16 @@ class XPUTestBatchNormOp(XPUOpTestWrapper): else: raise ValueError( "Unsupported data layout! Only NCHW and NHWC is supported, but received " - + self.data_layout) + + self.data_layout + ) np.random.seed(1024) self.x_np = np.random.random_sample(self.shape).astype(self.dtype) - self.scale_np = np.random.random_sample([channel_size - ]).astype(self.dtype) - self.bias_np = np.random.random_sample([channel_size - ]).astype(self.dtype) + self.scale_np = np.random.random_sample([channel_size]).astype( + self.dtype + ) + self.bias_np = np.random.random_sample([channel_size]).astype( + self.dtype + ) self.mean_np = np.zeros([channel_size]).astype(self.dtype) self.variance_np = np.ones([channel_size]).astype(self.dtype) self.saved_mean_np = np.zeros([channel_size]).astype(self.dtype) @@ -173,43 +201,81 @@ class XPUTestBatchNormOp(XPUOpTestWrapper): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('X', self.x_np.shape, self.x_np.dtype) - scale = paddle.fluid.data('Scale', self.scale_np.shape, - self.scale_np.dtype) - bias = paddle.fluid.data('Bias', self.bias_np.shape, - self.bias_np.dtype) - mean = paddle.fluid.data('Mean', self.mean_np.shape, - self.mean_np.dtype) - variance = paddle.fluid.data('Variance', self.variance_np.shape, - self.variance_np.dtype) - y = F.batch_norm(x, mean, variance, scale, bias, False, - self.momentum, self.epsilon, self.data_layout) + scale = paddle.fluid.data( + 'Scale', self.scale_np.shape, self.scale_np.dtype + ) + bias = paddle.fluid.data( + 'Bias', self.bias_np.shape, self.bias_np.dtype + ) + mean = paddle.fluid.data( + 'Mean', self.mean_np.shape, self.mean_np.dtype + ) + variance = paddle.fluid.data( + 'Variance', self.variance_np.shape, self.variance_np.dtype + ) + y = F.batch_norm( + x, + mean, + variance, + scale, + bias, + False, + self.momentum, + self.epsilon, + self.data_layout, + ) exe = paddle.static.Executor(self.place) - [y_np] = exe.run(feed={ - 'X': self.x_np, - 'Scale': self.scale_np, - 'Bias': self.bias_np, - 'Mean': self.mean_np, - 'Variance': self.variance_np - }, - fetch_list=[y]) - y_np_ref = ref_batch_norm_infer(self.x_np, self.scale_np, - self.bias_np, self.mean_np, - self.variance_np, self.momentum, - self.epsilon, self.data_layout) + [y_np] = exe.run( + feed={ + 'X': self.x_np, + 'Scale': self.scale_np, + 'Bias': self.bias_np, + 'Mean': self.mean_np, + 'Variance': self.variance_np, + }, + fetch_list=[y], + ) + y_np_ref = ref_batch_norm_infer( + self.x_np, + self.scale_np, + self.bias_np, + self.mean_np, + self.variance_np, + self.momentum, + self.epsilon, + self.data_layout, + ) np.testing.assert_allclose(y_np_ref, y_np, rtol=1e-05) def test_train(self): y_grad_np = np.random.random_sample(self.shape).astype(self.dtype) - y_np, mean_out_np, variance_out_np, saved_mean_np, saved_variance_np, x_grad_np, scale_grad_np, bias_grad_np = ref_batch_norm_train( - self.x_np, y_grad_np, self.scale_np, self.bias_np, self.mean_np, - self.variance_np, self.momentum, self.epsilon, self.data_layout) + ( + y_np, + mean_out_np, + variance_out_np, + saved_mean_np, + saved_variance_np, + x_grad_np, + scale_grad_np, + bias_grad_np, + ) = ref_batch_norm_train( + self.x_np, + y_grad_np, + self.scale_np, + self.bias_np, + self.mean_np, + self.variance_np, + self.momentum, + self.epsilon, + self.data_layout, + ) inputs = { 'X': self.x_np, 'Scale': self.scale_np, 'Bias': self.bias_np, 'Mean': self.mean_np, 'Variance': self.variance_np, - 'Y@GRAD': y_grad_np + 'Y@GRAD': y_grad_np, } outputs = { 'Y': y_np, @@ -219,7 +285,7 @@ class XPUTestBatchNormOp(XPUOpTestWrapper): 'SavedVariance': saved_variance_np, 'X@GRAD': x_grad_np, 'Scale@GRAD': scale_grad_np, - 'Bias@GRAD': bias_grad_np + 'Bias@GRAD': bias_grad_np, } attrs = { 'momentum': self.momentum, @@ -240,9 +306,11 @@ class XPUTestBatchNormOp(XPUOpTestWrapper): arg_name = var_name np_value = inputs[var_name] if not block.has_var(var_name): - block.create_var(name=var_name, - shape=np_value.shape, - dtype=np_value.dtype) + block.create_var( + name=var_name, + shape=np_value.shape, + dtype=np_value.dtype, + ) input_vars[arg_name] = block.var(var_name) fetch_list = [] output_vars = {} @@ -250,22 +318,27 @@ class XPUTestBatchNormOp(XPUOpTestWrapper): arg_name = var_name np_value = outputs[var_name] if not block.has_var(var_name): - block.create_var(name=var_name, - shape=np_value.shape, - dtype=np_value.dtype) + block.create_var( + name=var_name, + shape=np_value.shape, + dtype=np_value.dtype, + ) if var_name == 'Mean': arg_name = 'MeanOut' # Share memory if var_name == 'Variance': arg_name = 'VarianceOut' # Share memory output_vars[arg_name] = block.var(var_name) fetch_list.append(var_name) - batch_norm_op = block.append_op(type="batch_norm", - inputs=input_vars, - outputs=output_vars, - attrs=attrs) + batch_norm_op = block.append_op( + type="batch_norm", + inputs=input_vars, + outputs=output_vars, + attrs=attrs, + ) # Generate the backward op_desc of batch_norm grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - batch_norm_op.desc, set(), []) + batch_norm_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -273,13 +346,11 @@ class XPUTestBatchNormOp(XPUOpTestWrapper): exe = paddle.static.Executor(self.place) outs = exe.run(program, feed=inputs, fetch_list=fetch_list) for id, name in enumerate(fetch_list): - np.testing.assert_allclose(outputs[name], - outs[id], - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + outputs[name], outs[id], rtol=1e-05, atol=1e-4 + ) class TestBatchNormOpUseGlobalStats(unittest.TestCase): - def setUp(self): self.places = [paddle.XPUPlace(0)] self.init_test() @@ -296,11 +367,14 @@ class XPUTestBatchNormOp(XPUOpTestWrapper): net1 = paddle.fluid.dygraph.BatchNorm( 6, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.0)), + initializer=fluid.initializer.Constant(1.0) + ), use_global_stats=self.use_global_stats, - trainable_statistics=self.trainable_statistics) + trainable_statistics=self.trainable_statistics, + ) net2 = paddle.nn.BatchNorm2D( - 6, use_global_stats=self.use_global_stats) + 6, use_global_stats=self.use_global_stats + ) net2.weight = net1.weight net2.bias = net1.bias if self.trainable_statistics == True: @@ -308,9 +382,9 @@ class XPUTestBatchNormOp(XPUOpTestWrapper): net2.training = False y1 = net1(x) y2 = net2(x) - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05 + ) class TestBatchNormOpUseGlobalStats1(TestBatchNormOpUseGlobalStats): ### test mode diff --git a/python/paddle/fluid/tests/unittests/xpu/test_bce_loss_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_bce_loss_op_xpu.py index e5fa996f16c..01256f1d9cc 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_bce_loss_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_bce_loss_op_xpu.py @@ -19,30 +19,33 @@ import paddle import numpy as np import unittest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() def bce_loss(input, label): - return -1 * (label * np.log(input) + (1. - label) * np.log(1. - input)) + return -1 * (label * np.log(input) + (1.0 - label) * np.log(1.0 - input)) class XPUTestBceLossOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bce_loss' self.use_dynamic_create_class = False class TestBceLossOp(XPUOpTest): - def setUp(self): self.op_type = "bce_loss" self.dtype = self.in_type self.place = paddle.XPUPlace(0) self.init_test_case() - input_np = np.random.uniform(0.1, 0.8, - self.shape).astype(self.dtype) + input_np = np.random.uniform(0.1, 0.8, self.shape).astype( + self.dtype + ) label_np = np.random.randint(0, 2, self.shape).astype(self.dtype) output_np = bce_loss(input_np, label_np) @@ -59,12 +62,10 @@ class XPUTestBceLossOp(XPUOpTestWrapper): self.shape = [10, 10] class TestBceLossOpCase1(TestBceLossOp): - def init_test_cast(self): self.shape = [2, 3, 4, 5] class TestBceLossOpCase2(TestBceLossOp): - def init_test_cast(self): self.shape = [2, 3, 20] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_bilinear_interp_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_bilinear_interp_v2_op_xpu.py index 811a6f6cb7c..842018ba8f4 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_bilinear_interp_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_bilinear_interp_v2_op_xpu.py @@ -20,21 +20,27 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest import unittest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def bilinear_interp_np(input, - out_h, - out_w, - scale_w=0, - scale_h=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): +def bilinear_interp_np( + input, + out_h, + out_w, + scale_w=0, + scale_h=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -48,7 +54,7 @@ def bilinear_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -56,7 +62,7 @@ def bilinear_interp_np(input, else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -67,37 +73,40 @@ def bilinear_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for i in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (i + 0.5) - 0.5) else: h = int(ratio_h * i) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (i + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: h1lambda = ratio_h * i - h h2lambda = 1.0 - h1lambda for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, i, j] = h2lambda*(w2lambda*input[:, :, h, w] + - w1lambda*input[:, :, h, w+wid]) + \ - h1lambda*(w2lambda*input[:, :, h+hid, w] + - w1lambda*input[:, :, h+hid, w+wid]) + out[:, :, i, j] = h2lambda * ( + w2lambda * input[:, :, h, w] + + w1lambda * input[:, :, h, w + wid] + ) + h1lambda * ( + w2lambda * input[:, :, h + hid, w] + + w1lambda * input[:, :, h + hid, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -106,13 +115,11 @@ def bilinear_interp_np(input, class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bilinear_interp_v2' self.use_dynamic_create_class = False class TestBilinearInterpOp(XPUOpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -133,7 +140,7 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -146,10 +153,18 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -162,11 +177,11 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -184,7 +199,7 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): self.input_shape = [2, 3, 5, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -193,76 +208,69 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): self.place = paddle.XPUPlace(0) class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 33]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase7(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -273,70 +281,62 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): self.align_mode = 1 class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -347,7 +347,6 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): self.align_mode = 1 class TestBilinearInterpScale4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -358,7 +357,6 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): self.align_mode = 1 class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -369,7 +367,6 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): self.align_mode = 0 class TestBilinearInterpOp_attr_tensor(XPUOpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -409,8 +406,9 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -422,9 +420,16 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -438,7 +443,7 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): self.input_shape = [2, 3, 5, 5] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True @@ -446,36 +451,36 @@ class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): self.place = paddle.XPUPlace(0) # out_size is a 1-D tensor - class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor - ): - + class TestBilinearInterp_attr_tensor_Case1( + TestBilinearInterpOp_attr_tensor + ): def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # scale is a 1-D tensor - class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor - ): - + class TestBilinearInterp_attr_tensor_Case2( + TestBilinearInterpOp_attr_tensor + ): def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True # scale is a 1-D tensor - class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor - ): - + class TestBilinearInterp_attr_tensor_Case3( + TestBilinearInterpOp_attr_tensor + ): def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_bitwise_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_bitwise_op_xpu.py index e7e81707376..2482709f345 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_bitwise_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_bitwise_op_xpu.py @@ -21,19 +21,21 @@ sys.path.append("..") import paddle from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() ################## TEST OP: BitwiseAnd ################## class XPUTestBitwiseAnd(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bitwise_and' class XPUTestBitwiseAndBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,20 +44,18 @@ class XPUTestBitwiseAnd(XPUOpTestWrapper): def set_case(self): self.op_type = 'bitwise_and' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_and(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -73,7 +73,6 @@ class XPUTestBitwiseAnd(XPUOpTestWrapper): pass class XPUTestBitwiseAndCase1(XPUTestBitwiseAndBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -82,7 +81,6 @@ class XPUTestBitwiseAnd(XPUOpTestWrapper): self.high = 100 class XPUTestBitwiseAndCase2(XPUTestBitwiseAndBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -91,7 +89,6 @@ class XPUTestBitwiseAnd(XPUOpTestWrapper): self.high = 100 class XPUTestBitwiseAndCase3(XPUTestBitwiseAndBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -107,12 +104,10 @@ for stype in support_types: ################## TEST OP: BitwiseOr ################## class XPUTestBitwiseOr(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bitwise_or' class XPUTestBitwiseOrBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -121,20 +116,18 @@ class XPUTestBitwiseOr(XPUOpTestWrapper): def set_case(self): self.op_type = 'bitwise_or' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_or(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -152,7 +145,6 @@ class XPUTestBitwiseOr(XPUOpTestWrapper): pass class XPUTestBitwiseOrCase1(XPUTestBitwiseOrBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -161,7 +153,6 @@ class XPUTestBitwiseOr(XPUOpTestWrapper): self.high = 100 class XPUTestBitwiseOrCase2(XPUTestBitwiseOrBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -170,7 +161,6 @@ class XPUTestBitwiseOr(XPUOpTestWrapper): self.high = 100 class XPUTestBitwiseOrCase3(XPUTestBitwiseOrBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -186,12 +176,10 @@ for stype in support_types: ################## TEST OP: BitwiseXor ################## class XPUTestBitwiseXor(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bitwise_xor' class XPUTestBitwiseXorBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -200,20 +188,18 @@ class XPUTestBitwiseXor(XPUOpTestWrapper): def set_case(self): self.op_type = 'bitwise_xor' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_xor(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -231,7 +217,6 @@ class XPUTestBitwiseXor(XPUOpTestWrapper): pass class XPUTestBitwiseXorCase1(XPUTestBitwiseXorBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -240,7 +225,6 @@ class XPUTestBitwiseXor(XPUOpTestWrapper): self.high = 100 class XPUTestBitwiseXorCase2(XPUTestBitwiseXorBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -249,7 +233,6 @@ class XPUTestBitwiseXor(XPUOpTestWrapper): self.high = 100 class XPUTestBitwiseXorCase3(XPUTestBitwiseXorBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -265,12 +248,10 @@ for stype in support_types: ################## TEST OP: BitwiseNot ################## class XPUTestBitwiseNot(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bitwise_not' class XPUTestBitwiseNotBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -279,10 +260,9 @@ class XPUTestBitwiseNot(XPUOpTestWrapper): def set_case(self): self.op_type = 'bitwise_not' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) out = np.bitwise_not(x) self.attrs = {'use_xpu': True} @@ -302,7 +282,6 @@ class XPUTestBitwiseNot(XPUOpTestWrapper): pass class XPUTestBitwiseNotBool(XPUTestBitwiseNotBase): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_bmm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_bmm_op_xpu.py index 840f88198cb..fbc0ca66921 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_bmm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_bmm_op_xpu.py @@ -18,7 +18,11 @@ import paddle import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -33,7 +37,6 @@ class XPUTestBmmOp(XPUOpTestWrapper): self.use_dynamic_create_class = False class TestBmmOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -66,31 +69,26 @@ class XPUTestBmmOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') class TestBmmOp1(TestBmmOp): - def set_shape(self): self.Xshape = (3, 3, 3) self.Yshape = (3, 3, 3) class TestBmmOp2(TestBmmOp): - def set_shape(self): self.Xshape = (128, 3, 16) self.Yshape = (128, 16, 3) class TestBmmOp3(TestBmmOp): - def set_shape(self): self.Xshape = (2048, 16, 27) self.Yshape = (2048, 27, 16) class TestBmmOp4(TestBmmOp): - def set_shape(self): self.Xshape = (2, 27, 27) self.Yshape = (2, 27, 27) class TestBmmOp5(TestBmmOp): - def set_shape(self): self.Xshape = (2, 1, 1) self.Yshape = (2, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_c_embedding_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_c_embedding_op_xpu.py index 7324fc100d5..854ec71cf73 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_c_embedding_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_c_embedding_op_xpu.py @@ -17,7 +17,11 @@ import sys sys.path.append("..") import paddle -from paddle.fluid.tests.unittests.c_embedding_op_base import TestCEmbeddingCPU, TestCEmbeddingOpBase, TestCEmbeddingOpFP32 +from paddle.fluid.tests.unittests.c_embedding_op_base import ( + TestCEmbeddingCPU, + TestCEmbeddingOpBase, + TestCEmbeddingOpFP32, +) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_cast_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_cast_op_xpu.py index 908be556fd1..a2e136dccaa 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_cast_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_cast_op_xpu.py @@ -23,7 +23,11 @@ import paddle.fluid as fluid from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) typeid_dict = { 'int32': int(core.VarDesc.VarType.INT32), @@ -36,7 +40,6 @@ typeid_dict = { class XPUTestCastOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'cast' self.use_dynamic_create_class = True @@ -51,12 +54,14 @@ class XPUTestCastOp(XPUOpTestWrapper): return base_class, classes class TestCastOp(XPUOpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) in_typename = self.in_type_str - out_typename = 'float32' if not hasattr( - self, 'out_typename') else self.out_typename + out_typename = ( + 'float32' + if not hasattr(self, 'out_typename') + else self.out_typename + ) self.inputs = {'X': ipt.astype(in_typename)} self.outputs = {'Out': ipt.astype(in_typename).astype(out_typename)} @@ -77,12 +82,12 @@ for stype in support_types: class TestCastOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of cast_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.XPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.XPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.cast, x1, 'int32') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_clip_by_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_clip_by_norm_op_xpu.py index 32838a21a5a..1bfa96b5975 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_clip_by_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_clip_by_norm_op_xpu.py @@ -22,7 +22,6 @@ import paddle class TestXPUClipByNormOp(XPUOpTest): - def setUp(self): self.op_type = "clip_by_norm" self.dtype = np.float32 @@ -50,26 +49,23 @@ class TestXPUClipByNormOp(XPUOpTest): self.check_output_with_place(place) def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1.0 class TestCase1(TestXPUClipByNormOp): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestCase2(TestXPUClipByNormOp): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestCase3(TestXPUClipByNormOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_clip_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_clip_op_xpu.py index 5b61bfa8d1b..cd6dd33b6fa 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_clip_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_clip_op_xpu.py @@ -23,17 +23,19 @@ import paddle from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestClipOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'clip' self.use_dynamic_create_class = False class TestClipOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -90,28 +92,24 @@ class XPUTestClipOp(XPUOpTestWrapper): paddle.disable_static() class TestClipOp1(TestClipOp): - def init_data(self): self.shape = (8, 16, 8) self.max = 0.7 self.min = 0.0 class TestClipOp2(TestClipOp): - def init_data(self): self.shape = (8, 16) self.max = 1.0 self.min = 0.0 class TestClipOp3(TestClipOp): - def init_data(self): self.shape = (4, 8, 16) self.max = 0.7 self.min = 0.2 class TestClipOp4(TestClipOp): - def init_data(self): self.shape = (4, 8, 8) self.max = 0.7 @@ -120,7 +118,6 @@ class XPUTestClipOp(XPUOpTestWrapper): self.inputs['Min'] = np.array([0.3]).astype('float32') class TestClipOp5(TestClipOp): - def init_data(self): self.shape = (4, 8, 16) self.max = 0.5 @@ -128,7 +125,6 @@ class XPUTestClipOp(XPUOpTestWrapper): class TestClipOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -148,7 +144,6 @@ class TestClipOpError(unittest.TestCase): class TestClipAPI(unittest.TestCase): - def _executed_api(self, x, min=None, max=None): return paddle.clip(x, min, max) @@ -160,8 +155,11 @@ class TestClipAPI(unittest.TestCase): min = fluid.data(name='min', shape=[1], dtype='float32') max = fluid.data(name='max', shape=[1], dtype='float32') - place = fluid.XPUPlace( - 0) if fluid.core.is_compiled_with_xpu() else fluid.CPUPlace() + place = ( + fluid.XPUPlace(0) + if fluid.core.is_compiled_with_xpu() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) out_1 = self._executed_api(images, min=min, max=max) @@ -170,7 +168,7 @@ class TestClipAPI(unittest.TestCase): out_4 = self._executed_api(images, max=0.7) out_5 = self._executed_api(images, min=min) out_6 = self._executed_api(images, max=max) - out_7 = self._executed_api(images, max=-1.) + out_7 = self._executed_api(images, max=-1.0) out_8 = self._executed_api(images) res1, res2, res3, res4, res5, res6, res7, res8 = exe.run( @@ -178,9 +176,10 @@ class TestClipAPI(unittest.TestCase): feed={ "image": data, "min": np.array([0.2]).astype('float32'), - "max": np.array([0.8]).astype('float32') + "max": np.array([0.8]).astype('float32'), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) np.testing.assert_allclose(res1, data.clip(0.2, 0.8)) np.testing.assert_allclose(res2, data.clip(0.2, 0.9)) @@ -194,8 +193,11 @@ class TestClipAPI(unittest.TestCase): def test_clip_dygraph(self): paddle.disable_static() - place = fluid.XPUPlace( - 0) if fluid.core.is_compiled_with_xpu() else fluid.CPUPlace() + place = ( + fluid.XPUPlace(0) + if fluid.core.is_compiled_with_xpu() + else fluid.CPUPlace() + ) paddle.disable_static(place) data_shape = [1, 9, 9, 4] data = np.random.random(data_shape).astype('float32') @@ -223,7 +225,6 @@ class TestClipAPI(unittest.TestCase): class TestInplaceClipAPI(TestClipAPI): - def _executed_api(self, x, min=None, max=None): return x.clip_(min, max) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_coalesce_tensor_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_coalesce_tensor_op_xpu.py index 9b295aa12cb..5370c1947a8 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_coalesce_tensor_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_coalesce_tensor_op_xpu.py @@ -22,19 +22,21 @@ sys.path.append("..") alignment = 256 import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCoalesceTensorOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'coalesce_tensor' self.use_dynamic_create_class = False class TestAllocContinuousSpace(XPUOpTest): - def setUp(self): self.op_type = "coalesce_tensor" self.use_xpu = True @@ -45,12 +47,13 @@ class XPUTestCoalesceTensorOp(XPUOpTestWrapper): self.set_constant = attrs["set_constant"] self.Inputs = self.init_input() self.Outputs, self.FusedOutput = self.init_output( - self.Inputs, self.set_constant, self.constant) + self.Inputs, self.set_constant, self.constant + ) self.inputs = {'Input': self.Inputs} self.attrs = attrs self.outputs = { 'Output': self.Outputs, - 'FusedOutput': self.FusedOutput + 'FusedOutput': self.FusedOutput, } def init_dtype(self): @@ -61,8 +64,9 @@ class XPUTestCoalesceTensorOp(XPUOpTestWrapper): inputs.append(("x1", np.random.random([20, 3]).astype(self.dtype))) inputs.append(("x2", np.random.random([20]).astype(self.dtype))) inputs.append(("x3", np.random.random([1]).astype(self.dtype))) - inputs.append(("x4", np.random.random([200, - 30]).astype(self.dtype))) + inputs.append( + ("x4", np.random.random([200, 30]).astype(self.dtype)) + ) inputs.append(("x5", np.random.random([30]).astype(self.dtype))) inputs.append(("x6", np.random.random([1]).astype(self.dtype))) return inputs @@ -72,7 +76,7 @@ class XPUTestCoalesceTensorOp(XPUOpTestWrapper): "copy_data": True, "set_constant": False, "constant": 0.0, - "dtype": self.fluid_dtype + "dtype": self.fluid_dtype, } def init_output(self, input_list, set_constant, constant): @@ -88,33 +92,37 @@ class XPUTestCoalesceTensorOp(XPUOpTestWrapper): coalesce_tensor_var = np.concatenate([input for input in inputs]) if set_constant: - coalesce_tensor_var = np.ones( - (len(coalesce_tensor_var))) * constant - outputs = [(out[0], - np.ones(out[1].shape).astype(self.dtype) * constant) - for out in outputs] + coalesce_tensor_var = ( + np.ones((len(coalesce_tensor_var))) * constant + ) + outputs = [ + ( + out[0], + np.ones(out[1].shape).astype(self.dtype) * constant, + ) + for out in outputs + ] return outputs, coalesce_tensor_var def test_check_output(self): - self.check_output_with_place(place=core.XPUPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=core.XPUPlace(0), no_check_set=["FusedOutput"], atol=1e-5 + ) class TestAllocContinuousSpace2(TestAllocContinuousSpace): - def init_attr(self): return { "copy_data": False, "set_constant": True, "constant": 0.5, "dtype": self.fluid_dtype, - "user_defined_size_of_dtype": 2 + "user_defined_size_of_dtype": 2, } def test_check_output(self): - self.check_output_with_place(place=core.XPUPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=core.XPUPlace(0), no_check_set=["FusedOutput"], atol=1e-5 + ) support_types = get_xpu_op_support_types('coalesce_tensor') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_collective_allgather_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_collective_allgather_xpu.py index eed54081115..568b3039a95 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_collective_allgather_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_collective_allgather_xpu.py @@ -22,33 +22,38 @@ import sys sys.path.append("..") -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCAllgatherOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'c_allgather' self.use_dynamic_create_class = False class TestCAllgatherOp(TestDistBase): - def _setup_config(self): pass def test_allgather(self): - self.check_with_place("collective_allgather_op_xpu.py", "allgather", - self.in_type_str) + self.check_with_place( + "collective_allgather_op_xpu.py", "allgather", self.in_type_str + ) support_types = get_xpu_op_support_types('c_allgather') for stype in support_types: - create_test_class(globals(), - XPUTestCAllgatherOP, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestCAllgatherOP, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_collective_allreduce_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_collective_allreduce_xpu.py index 57a96530c55..3fa7405d548 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_collective_allreduce_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_collective_allreduce_xpu.py @@ -22,33 +22,38 @@ import sys sys.path.append("..") -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCAllreduceOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'c_allreduce_sum' self.use_dynamic_create_class = False class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce(self): - self.check_with_place("collective_allreduce_op_xpu.py", "allreduce", - self.in_type_str) + self.check_with_place( + "collective_allreduce_op_xpu.py", "allreduce", self.in_type_str + ) support_types = get_xpu_op_support_types('c_allreduce_sum') for stype in support_types: - create_test_class(globals(), - XPUTestCAllreduceOP, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestCAllreduceOP, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_collective_base_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_collective_base_xpu.py index 74e72672369..1c00f3b94b8 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_collective_base_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_collective_base_xpu.py @@ -50,10 +50,10 @@ def DataTypeCast(date_type): class TestCollectiveRunnerBase(object): - def get_model(self, train_prog, startup_prog): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): while True: @@ -61,13 +61,15 @@ class TestCollectiveRunnerBase(object): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -75,44 +77,51 @@ class TestCollectiveRunnerBase(object): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - bkcl_id_var = block.create_var(name=nameGen.generate('bkcl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - - block.append_op(type='c_gen_bkcl_id', - inputs={}, - outputs={'Out': bkcl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - - block.append_op(type='c_comm_init', - inputs={'X': bkcl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': self.global_ring_id - }) + bkcl_id_var = block.create_var( + name=nameGen.generate('bkcl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + + block.append_op( + type='c_gen_bkcl_id', + inputs={}, + outputs={'Out': bkcl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + + block.append_op( + type='c_comm_init', + inputs={'X': bkcl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': self.global_ring_id, + }, + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -121,8 +130,9 @@ class TestCollectiveRunnerBase(object): rank = args["trainerid"] current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) self.rank = rank result = self.get_model(train_prog, startup_prog) device_id = int(os.getenv("FLAGS_selected_xpus", "0")) @@ -131,11 +141,12 @@ class TestCollectiveRunnerBase(object): exe.run(startup_prog) np.random.seed(os.getpid()) np_data_type = DataTypeCast(args["data_type"]) - indata = np.random.uniform(low=-10.0, high=10.0, - size=(10, 1000)).astype(np_data_type) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=[result.name]) + indata = np.random.uniform( + low=-10.0, high=10.0, size=(10, 1000) + ).astype(np_data_type) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=[result.name] + ) sys.stdout.buffer.write(pickle.dumps(out[0])) @@ -157,12 +168,13 @@ from contextlib import closing class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self.temp_dir = tempfile.TemporaryDirectory() @@ -171,10 +183,10 @@ class TestDistBase(unittest.TestCase): self.temp_dir.cleanup() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -192,7 +204,7 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -200,9 +212,9 @@ class TestDistBase(unittest.TestCase): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) tr_cmd = "%s %s" @@ -215,14 +227,16 @@ class TestDistBase(unittest.TestCase): tr0_proc = subprocess.Popen( tr0_cmd.strip().split(), stdout=subprocess.PIPE, - #stderr=tr0_pipe, - env=env0) + # stderr=tr0_pipe, + env=env0, + ) tr1_proc = subprocess.Popen( tr0_cmd.strip().split(), stdout=subprocess.PIPE, - #stderr=tr1_pipe, - env=env1) + # stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -231,15 +245,21 @@ class TestDistBase(unittest.TestCase): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - data_type, - check_error_log=False, - need_envs={}): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, + model_file, + col_type, + data_type, + check_error_log=False, + need_envs={}, + ): required_envs = { "FLAGS_eager_delete_tensor_gb": "0.0", "PATH": os.getenv("PATH"), @@ -254,14 +274,17 @@ class TestDistBase(unittest.TestCase): required_envs["GLOG_v"] = "3" required_envs["GLOG_logtostderr"] = "1" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) np_data_type = DataTypeCast(data_type) np.random.seed(pid0) - input1 = np.random.uniform(low=-10.0, high=10.0, - size=(10, 1000)).astype(np_data_type) + input1 = np.random.uniform( + low=-10.0, high=10.0, size=(10, 1000) + ).astype(np_data_type) np.random.seed(pid1) - input2 = np.random.uniform(low=-10.0, high=10.0, - size=(10, 1000)).astype(np_data_type) + input2 = np.random.uniform( + low=-10.0, high=10.0, size=(10, 1000) + ).astype(np_data_type) if col_type == "allgather": need_result = np.vstack((input1, input2)) np.testing.assert_allclose(tr0_out, need_result) @@ -275,38 +298,33 @@ class TestDistBase(unittest.TestCase): np.testing.assert_allclose(tr1_out, need_result) elif col_type == "scatter": need_result = input2 - need_result1 = need_result[0:need_result.shape[0] // 2] - need_result2 = need_result[need_result.shape[0] // 2:] + need_result1 = need_result[0 : need_result.shape[0] // 2] + need_result2 = need_result[need_result.shape[0] // 2 :] np.testing.assert_allclose(tr0_out, need_result1) np.testing.assert_allclose(tr1_out, need_result2) elif col_type == "allreduce": need_result = input1 + input2 - np.testing.assert_allclose(tr0_out, - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out, need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "reduce_scatter": tmp = input1 + input2 - need_result1 = tmp[0:tmp.shape[0] // 2] - need_result2 = tmp[tmp.shape[0] // 2:] - np.testing.assert_allclose(tr0_out, - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result2, - rtol=1e-05, - atol=1e-05) + need_result1 = tmp[0 : tmp.shape[0] // 2] + need_result2 = tmp[tmp.shape[0] // 2 :] + np.testing.assert_allclose( + tr0_out, need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv": need_result = input1 - np.testing.assert_allclose(tr1_out, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr1_out, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "identity": need_result1 = input1 need_result2 = input2 @@ -324,35 +342,29 @@ class TestDistBase(unittest.TestCase): np.testing.assert_allclose(tr1_out, need_result2) elif col_type == "concat": need_result = np.concatenate((input1, input2), axis=1) - np.testing.assert_allclose(tr0_out, - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out, need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "split": need_result1 = np.split(input1, 2, axis=1)[0] need_result2 = np.split(input2, 2, axis=1)[1] - np.testing.assert_allclose(tr0_out, - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out, need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv_array": need_result1 = np.array([[0, 1, 2]]) need_result2 = np.array([[3, 4, 5]]) - np.testing.assert_allclose(tr1_out[0][0], - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0][1], - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr1_out[0][0], need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0][1], need_result2, rtol=1e-05, atol=1e-05 + ) else: pass diff --git a/python/paddle/fluid/tests/unittests/xpu/test_collective_identity_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_collective_identity_xpu.py index 07c523629b6..4d84efebbe2 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_collective_identity_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_collective_identity_xpu.py @@ -22,33 +22,38 @@ import sys sys.path.append("..") -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCIdentityOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'c_identity' self.use_dynamic_create_class = False class TestCIdentityOp(TestDistBase): - def _setup_config(self): pass def test_identity(self): - self.check_with_place("collective_identity_op_xpu.py", "identity", - self.in_type_str) + self.check_with_place( + "collective_identity_op_xpu.py", "identity", self.in_type_str + ) support_types = get_xpu_op_support_types('c_identity') for stype in support_types: - create_test_class(globals(), - XPUTestCIdentityOP, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestCIdentityOP, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_compare_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_compare_op_xpu.py index cdaf767a1de..8e7fb2eb342 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_compare_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_compare_op_xpu.py @@ -24,7 +24,6 @@ from xpu.get_test_cover_info import XPUOpTestWrapper class TestCompareOpBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.config() @@ -33,10 +32,12 @@ class TestCompareOpBase(XPUOpTest): self.outputs = {'Out': self.result} def set_case(self): - self.x = np.random.uniform(self.lbound, self.hbound, - self.x_shape).astype(self.dtype) - self.y = np.random.uniform(self.lbound, self.hbound, - self.y_shape).astype(self.dtype) + self.x = np.random.uniform( + self.lbound, self.hbound, self.x_shape + ).astype(self.dtype) + self.y = np.random.uniform( + self.lbound, self.hbound, self.y_shape + ).astype(self.dtype) self.result = self.compute(self.x, self.y) def config(self): @@ -54,13 +55,11 @@ class TestCompareOpBase(XPUOpTest): class XPUTestLessThanOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'less_than' self.use_dynamic_create_class = False class LessThanOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'less_than' @@ -74,7 +73,6 @@ class XPUTestLessThanOP(XPUOpTestWrapper): self.y_shape = [11, 17] class LessThanOpTestCase2(LessThanOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -82,7 +80,6 @@ class XPUTestLessThanOP(XPUOpTestWrapper): self.y_shape = [1] class LessThanOpTestCase3(LessThanOpTestCase1): - def set_data(self): self.lbound = -300 self.hbound = 300 @@ -90,7 +87,6 @@ class XPUTestLessThanOP(XPUOpTestWrapper): self.y_shape = [1] class LessThanOpTestCase4(LessThanOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -98,7 +94,6 @@ class XPUTestLessThanOP(XPUOpTestWrapper): self.y_shape = [1] class LessThanOpTestCase5(LessThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -112,13 +107,11 @@ for stype in support_types: class XPUTestLessEqualOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'less_equal' self.use_dynamic_create_class = False class LessEqualOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'less_equal' @@ -132,7 +125,6 @@ class XPUTestLessEqualOp(XPUOpTestWrapper): self.y_shape = [11, 17] class LessEqualOpTestCase2(LessEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -140,7 +132,6 @@ class XPUTestLessEqualOp(XPUOpTestWrapper): self.y_shape = [11, 17, 255] class LessEqualOpTestCase3(LessEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -148,7 +139,6 @@ class XPUTestLessEqualOp(XPUOpTestWrapper): self.y_shape = [1] class LessEqualOpTestCase4(LessEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -156,7 +146,6 @@ class XPUTestLessEqualOp(XPUOpTestWrapper): self.y_shape = [1] class LessEqualOpTestCase5(LessEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -170,13 +159,11 @@ for stype in support_types: class XPUTestGreaterThanOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'greater_than' self.use_dynamic_create_class = False class GreaterThanOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'greater_than' @@ -190,7 +177,6 @@ class XPUTestGreaterThanOp(XPUOpTestWrapper): self.y_shape = [128, 128, 512] class GreaterThanOpTestCase2(GreaterThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -198,7 +184,6 @@ class XPUTestGreaterThanOp(XPUOpTestWrapper): self.y_shape = [1] class GreaterThanOpTestCase3(GreaterThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -206,7 +191,6 @@ class XPUTestGreaterThanOp(XPUOpTestWrapper): self.y_shape = [1] class GreaterThanOpTestCase4(GreaterThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -214,7 +198,6 @@ class XPUTestGreaterThanOp(XPUOpTestWrapper): self.y_shape = [11, 17] class GreaterThanOpTestCase5(GreaterThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -228,13 +211,11 @@ for stype in support_types: class XPUTestGreaterEqualOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'greater_equal' self.use_dynamic_create_class = False class GreaterEqualOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'greater_equal' @@ -248,7 +229,6 @@ class XPUTestGreaterEqualOp(XPUOpTestWrapper): self.y_shape = [10, 10, 20, 20] class GreaterEqualOpTestCase2(GreaterEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -256,7 +236,6 @@ class XPUTestGreaterEqualOp(XPUOpTestWrapper): self.y_shape = [10, 10] class GreaterEqualOpTestCase3(GreaterEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -264,7 +243,6 @@ class XPUTestGreaterEqualOp(XPUOpTestWrapper): self.y_shape = [1] class GreaterEqualOpTestCase4(GreaterEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -272,7 +250,6 @@ class XPUTestGreaterEqualOp(XPUOpTestWrapper): self.y_shape = [1] class GreaterEqualOpTestCase5(GreaterEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -286,13 +263,11 @@ for stype in support_types: class XPUTestEqualOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'equal' self.use_dynamic_create_class = False class EqualOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'equal' @@ -306,7 +281,6 @@ class XPUTestEqualOp(XPUOpTestWrapper): self.y_shape = [10, 30, 15] class EqualOpTestCase2(EqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -314,7 +288,6 @@ class XPUTestEqualOp(XPUOpTestWrapper): self.y_shape = [1] class EqualOpTestCase3(EqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -322,7 +295,6 @@ class XPUTestEqualOp(XPUOpTestWrapper): self.y_shape = [10, 30] class EqualOpTestCase4(EqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -330,7 +302,6 @@ class XPUTestEqualOp(XPUOpTestWrapper): self.y_shape = [256, 256, 10] class EqualOpTestCase5(EqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -344,13 +315,11 @@ for stype in support_types: class XPUTestNotEqualOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'not_equal' self.use_dynamic_create_class = False class NotEqualOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'not_equal' @@ -364,7 +333,6 @@ class XPUTestNotEqualOp(XPUOpTestWrapper): self.y_shape = [1] class NotEqualOpTestCase2(NotEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -372,7 +340,6 @@ class XPUTestNotEqualOp(XPUOpTestWrapper): self.y_shape = [11, 17] class NotEqualOpTestCase3(NotEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -380,7 +347,6 @@ class XPUTestNotEqualOp(XPUOpTestWrapper): self.y_shape = [1] class NotEqualOpTestCase4(NotEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -388,7 +354,6 @@ class XPUTestNotEqualOp(XPUOpTestWrapper): self.y_shape = [256, 256, 10] class NotEqualOpTestCase5(NotEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_concat_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_concat_op_xpu.py index 951eec301c0..e7e4ca073c0 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_concat_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_concat_op_xpu.py @@ -21,19 +21,21 @@ import numpy as np import paddle from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestConcatOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'concat' self.use_dynamic_create_class = False class TestConcatOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "concat" @@ -47,14 +49,16 @@ class XPUTestConcatOp(XPUOpTestWrapper): self.attrs = {'axis': self.axis} if self.axis < 0: self.actual_axis = self.axis + len(self.x0.shape) - self.actual_axis = self.actual_axis if self.actual_axis > 0 else 0 + self.actual_axis = ( + self.actual_axis if self.actual_axis > 0 else 0 + ) else: self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), - axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } def set_inputs(self): @@ -83,12 +87,10 @@ class XPUTestConcatOp(XPUOpTestWrapper): self.check_grad_with_place(place, ['x2'], 'Out') class TestConcatOpAxis0XPU(TestConcatOp): - def init_axis(self): self.axis = 0 class TestConcatOpAxis1XPU(TestConcatOp): - def set_inputs(self): self.x0 = np.random.random((5, 1, 4, 5)).astype(self.dtype) self.x1 = np.random.random((5, 2, 4, 5)).astype(self.dtype) @@ -98,34 +100,29 @@ class XPUTestConcatOp(XPUOpTestWrapper): self.axis = 1 class TestConcatOpAxis2XPU(TestConcatOp): - def init_axis(self): self.axis = 2 class TestConcatOpAxis3XPU(TestConcatOp): - def init_axis(self): self.axis = 3 class TestConcatOpAxisNeg1XPU(TestConcatOp): - def init_axis(self): self.axis = -1 class TestConcatOpAxisNeg2XPU(TestConcatOp): - def init_axis(self): self.axis = -2 class TestConcatOpAxisNeg3XPU(TestConcatOp): - def init_axis(self): self.axis = -3 @skip_check_grad_ci( - reason="The function 'check_grad' for large inputs is too slow.") + reason="The function 'check_grad' for large inputs is too slow." + ) class TestConcatOp3(TestConcatOp): - def set_inputs(self): self.x0 = np.random.random((1, 256, 170, 256)).astype(self.dtype) self.x1 = np.random.random((1, 128, 170, 256)).astype(self.dtype) @@ -136,11 +133,9 @@ class XPUTestConcatOp(XPUOpTestWrapper): pass @skip_check_grad_ci( - reason= - "This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." + reason="This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." ) class TestConcatOp4(TestConcatOp): - def set_inputs(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_conv2d_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_conv2d_op_xpu.py index b7924818177..973e2908c4e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_conv2d_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_conv2d_op_xpu.py @@ -21,25 +21,34 @@ import numpy as np import paddle.fluid.core as core from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper - - -def conv2d_forward_naive(input, - filter, - group, - conv_param, - padding_algorithm='EXPLICIT', - data_format='NCHW'): +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) + + +def conv2d_forward_naive( + input, + filter, + group, + conv_param, + padding_algorithm='EXPLICIT', + data_format='NCHW', +): if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Unknown Attr(data_format): '%s' ." - "It can only be 'NCHW' or 'NHWC'." % str(data_format)) + raise ValueError( + "Unknown Attr(data_format): '%s' ." + "It can only be 'NCHW' or 'NHWC'." % str(data_format) + ) - channel_last = (data_format == "NHWC") + channel_last = data_format == "NHWC" if channel_last: input = np.transpose(input, [0, 3, 1, 2]) @@ -52,17 +61,22 @@ def conv2d_forward_naive(input, sub_out_c = out_c // group sub_f_n = f_n // group - stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ - 'dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -82,39 +96,52 @@ def conv2d_forward_naive(input, if len(pad) == 4: pad_h_0, pad_h_1 = pad[0], pad[1] pad_w_0, pad_w_1 = pad[2], pad[3] - out_h = 1 + (in_h + pad_h_0 + pad_h_1 - (dilation[0] * - (f_h - 1) + 1)) // stride[0] - out_w = 1 + (in_w + pad_w_0 + pad_w_1 - (dilation[1] * - (f_w - 1) + 1)) // stride[1] + out_h = ( + 1 + + (in_h + pad_h_0 + pad_h_1 - (dilation[0] * (f_h - 1) + 1)) + // stride[0] + ) + out_w = ( + 1 + + (in_w + pad_w_0 + pad_w_1 - (dilation[1] * (f_w - 1) + 1)) + // stride[1] + ) out = np.zeros((out_n, out_c, out_h, out_w)) - d_bolck_h = (dilation[0] * (f_h - 1) + 1) - d_bolck_w = (dilation[1] * (f_w - 1) + 1) + d_bolck_h = dilation[0] * (f_h - 1) + 1 + d_bolck_w = dilation[1] * (f_w - 1) + 1 - input_pad = np.pad(input, - ((0, 0), (0, 0), (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), - mode='constant', - constant_values=0) + input_pad = np.pad( + input, + ((0, 0), (0, 0), (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), + mode='constant', + constant_values=0, + ) filter_dilation = np.zeros((f_n, f_c, d_bolck_h, d_bolck_w)) - filter_dilation[:, :, 0:d_bolck_h:dilation[0], - 0:d_bolck_w:dilation[1]] = filter + filter_dilation[ + :, :, 0 : d_bolck_h : dilation[0], 0 : d_bolck_w : dilation[1] + ] = filter for i in range(out_h): for j in range(out_w): for g in range(group): - input_pad_masked = \ - input_pad[:, g * f_c:(g + 1) * f_c, - i * stride[0]:i * stride[0] + d_bolck_h, - j * stride[1]:j * stride[1] + d_bolck_w] - - f_sub = filter_dilation[g * sub_f_n:(g + 1) * sub_f_n, :, :, :] + input_pad_masked = input_pad[ + :, + g * f_c : (g + 1) * f_c, + i * stride[0] : i * stride[0] + d_bolck_h, + j * stride[1] : j * stride[1] + d_bolck_w, + ] + + f_sub = filter_dilation[ + g * sub_f_n : (g + 1) * sub_f_n, :, :, : + ] # sub_f_n == sub_out_c for k in range(sub_out_c): # Multiplication of Corresponding Elements, then sum all - out[:, g * sub_out_c + k, i, j] = \ - np.sum(input_pad_masked * f_sub[k, :, :, :], - axis=(1, 2, 3)) + out[:, g * sub_out_c + k, i, j] = np.sum( + input_pad_masked * f_sub[k, :, :, :], axis=(1, 2, 3) + ) if channel_last: out = np.transpose(out, [0, 2, 3, 1]) @@ -123,9 +150,7 @@ def conv2d_forward_naive(input, def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -139,9 +164,7 @@ def create_test_channel_last_class(parent): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -152,9 +175,7 @@ def create_test_padding_SAME_class(parent): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -165,13 +186,11 @@ def create_test_padding_VALID_class(parent): class XPUTestConv2DOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'conv2d' self.use_dynamic_create_class = False class TestConv2DOp(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -190,7 +209,7 @@ class XPUTestConv2DOp(XPUOpTestWrapper): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } np.random.seed(100) @@ -205,16 +224,18 @@ class XPUTestConv2DOp(XPUOpTestWrapper): else: input2 = input np.random.seed(1) - filter = np.random.uniform(-1, 1, - self.filter_size).astype(self.dtype) + filter = np.random.uniform(-1, 1, self.filter_size).astype( + self.dtype + ) - output, _, _, _, _ = conv2d_forward_naive(input2, filter, - self.groups, conv2d_param) + output, _, _, _, _ = conv2d_forward_naive( + input2, filter, self.groups, conv2d_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': XPUOpTest.np_dtype_to_fluid_dtype(input), - 'Filter': XPUOpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': XPUOpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -224,15 +245,15 @@ class XPUTestConv2DOp(XPUOpTestWrapper): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu_before_depthwise_conv': - self.fuse_relu_before_depthwise_conv, - 'exhaustive_search': self.exhaustive_search + 'fuse_relu_before_depthwise_conv': self.fuse_relu_before_depthwise_conv, + 'exhaustive_search': self.exhaustive_search, } self.outputs = {'Output': output} def has_cuda(self): - return core.is_compiled_with_cuda() and (self.use_cudnn - or self.use_cuda) + return core.is_compiled_with_cuda() and ( + self.use_cudnn or self.use_cuda + ) def test_check_output(self): if core.is_compiled_with_xpu(): @@ -240,33 +261,40 @@ class XPUTestConv2DOp(XPUOpTestWrapper): self.check_output_with_place(self.place) def test_check_grad(self): - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output') + self.check_grad_with_place( + self.place, {'Input', 'Filter'}, 'Output' + ) def test_check_grad_no_filter(self): - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad_no_input(self): - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def init_test_case(self): self.pad = [0, 0] @@ -289,7 +317,6 @@ class XPUTestConv2DOp(XPUOpTestWrapper): pass class TestWithPad(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -299,7 +326,6 @@ class XPUTestConv2DOp(XPUOpTestWrapper): self.filter_size = [6, f_c, 3, 3] class TestWithStride(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -309,7 +335,6 @@ class XPUTestConv2DOp(XPUOpTestWrapper): self.filter_size = [6, f_c, 3, 3] class TestWith1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -324,13 +349,11 @@ class XPUTestConv2DOp(XPUOpTestWrapper): # ---- test asymmetric padding ---- class XPUTestConv2DOp_v2(XPUOpTestWrapper): - def __init__(self): self.op_name = 'conv2d' self.use_dynamic_create_class = False class TestConv2DOp_v2(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -351,7 +374,7 @@ class XPUTestConv2DOp_v2(XPUOpTestWrapper): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } np.random.seed(100) @@ -366,17 +389,22 @@ class XPUTestConv2DOp_v2(XPUOpTestWrapper): else: input2 = input np.random.seed(8) - filter = np.random.uniform(-1, 1, - self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input2, filter, - self.groups, conv2d_param, - self.padding_algorithm, - self.data_format) + filter = np.random.uniform(-1, 1, self.filter_size).astype( + self.dtype + ) + output, _, _, _, _ = conv2d_forward_naive( + input2, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': XPUOpTest.np_dtype_to_fluid_dtype(input), - 'Filter': XPUOpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': XPUOpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -387,15 +415,15 @@ class XPUTestConv2DOp_v2(XPUOpTestWrapper): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu_before_depthwise_conv': - self.fuse_relu_before_depthwise_conv, - 'exhaustive_search': self.exhaustive_search + 'fuse_relu_before_depthwise_conv': self.fuse_relu_before_depthwise_conv, + 'exhaustive_search': self.exhaustive_search, } self.outputs = {'Output': output} def has_cuda(self): - return core.is_compiled_with_cuda() and (self.use_cudnn - or self.use_cuda) + return core.is_compiled_with_cuda() and ( + self.use_cudnn or self.use_cuda + ) def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode @@ -405,35 +433,42 @@ class XPUTestConv2DOp_v2(XPUOpTestWrapper): def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output') + self.check_grad_with_place( + self.place, {'Input', 'Filter'}, 'Output' + ) def test_check_grad_no_filter(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad_no_input(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def init_test_case(self): self.pad = [0, 0] @@ -463,13 +498,11 @@ class XPUTestConv2DOp_v2(XPUOpTestWrapper): pass class TestConv2DOp_AsyPadding(TestConv2DOp_v2): - def init_paddings(self): self.pad = [0, 0, 0, 0] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -482,7 +515,6 @@ class XPUTestConv2DOp_v2(XPUOpTestWrapper): self.padding_algorithm = "EXPLICIT" class TestWithStride_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -496,14 +528,13 @@ class XPUTestConv2DOp_v2(XPUOpTestWrapper): class XPUTestConv2DOp_NHWC(XPUOpTestWrapper): - def __init__(self): self.op_name = 'conv2d' self.use_dynamic_create_class = False class TestConv2DOp_AsyPadding_NHWC( - XPUTestConv2DOp_v2.TestConv2DOp_AsyPadding): - + XPUTestConv2DOp_v2.TestConv2DOp_AsyPadding + ): def init_data_format(self): self.data_format = "NHWC" @@ -511,9 +542,9 @@ class XPUTestConv2DOp_NHWC(XPUOpTestWrapper): N, C, H, W = self.input_size self.input_size = [N, H, W, C] - class TestWithPad_AsyPadding_NHWC(XPUTestConv2DOp_v2.TestWithPad_AsyPadding - ): - + class TestWithPad_AsyPadding_NHWC( + XPUTestConv2DOp_v2.TestWithPad_AsyPadding + ): def init_data_format(self): self.data_format = "NHWC" @@ -526,19 +557,21 @@ support_types = get_xpu_op_support_types('conv2d') for stype in ['float32']: create_test_class(globals(), XPUTestConv2DOp, stype) create_test_class(globals(), XPUTestConv2DOp_v2, stype) - create_test_class(globals(), - XPUTestConv2DOp_NHWC, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) - -#---------- test SAME VALID ----------- -#create_test_padding_SAME_class(TestConv2DOp_AsyPadding) -#create_test_padding_SAME_class(TestWithPad_AsyPadding) -#create_test_padding_SAME_class(TestWithStride_AsyPadding) - -#create_test_padding_VALID_class(TestConv2DOp_AsyPadding) -#create_test_padding_VALID_class(TestWithPad_AsyPadding) -#create_test_padding_VALID_class(TestWithStride_AsyPadding) + create_test_class( + globals(), + XPUTestConv2DOp_NHWC, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) + +# ---------- test SAME VALID ----------- +# create_test_padding_SAME_class(TestConv2DOp_AsyPadding) +# create_test_padding_SAME_class(TestWithPad_AsyPadding) +# create_test_padding_SAME_class(TestWithStride_AsyPadding) + +# create_test_padding_VALID_class(TestConv2DOp_AsyPadding) +# create_test_padding_VALID_class(TestWithPad_AsyPadding) +# create_test_padding_VALID_class(TestWithStride_AsyPadding) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_conv2d_transpose_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_conv2d_transpose_op_xpu.py index 49011dd3618..47d13d04b3a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_conv2d_transpose_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_conv2d_transpose_op_xpu.py @@ -19,7 +19,11 @@ import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() @@ -28,9 +32,10 @@ paddle.enable_static() def conv2dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 3, 1, 2]) @@ -41,18 +46,22 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -86,43 +95,54 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): if 'output_padding' in attrs: out_pad_h = attrs['output_padding'][0] out_pad_w = attrs['output_padding'][1] - out = np.zeros((in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), - dtype=input_.dtype) + out = np.zeros( + (in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), dtype=input_.dtype + ) for n in range(in_n): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, g * sub_in_c:(g + 1) * sub_in_c, i, - j] # (c) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, i, j + ] # (c) input_masked = np.reshape(input_masked, (sub_in_c, 1, 1)) input_masked = np.tile(input_masked, (1, f_h, f_w)) for k in range(f_out_c): tmp_out = np.sum( - input_masked * - filter_[g * sub_in_c:(g + 1) * sub_in_c, k, :, :], - axis=0) + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, k, :, : + ], + axis=0, + ) i1, i2 = i * stride[0], i * stride[0] + d_bolck_h j1, j2 = j * stride[1], j * stride[1] + d_bolck_w - out[n, g * f_out_c + k, i1:i2:dilations[0], - j1:j2:dilations[1]] += tmp_out - - out = out[:, :, pad_h_0:out_h - pad_h_1 + out_pad_h, - pad_w_0:out_w - pad_w_1 + out_pad_w] + out[ + n, + g * f_out_c + k, + i1 : i2 : dilations[0], + j1 : j2 : dilations[1], + ] += tmp_out + + out = out[ + :, + :, + pad_h_0 : out_h - pad_h_1 + out_pad_h, + pad_w_0 : out_w - pad_w_1 + out_pad_w, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 1]) return out class XPUTestConv2DTransposeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'conv2d_transpose' self.use_dynamic_create_class = False class TestConv2DTransposeOp(XPUOpTest): - def setUp(self): # init as conv transpose self.need_check_grad = True @@ -151,7 +171,7 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): 'use_cudnn': self.use_cudnn, 'is_test': self.is_test, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } if self.output_size is not None: self.attrs['output_size'] = self.output_size @@ -160,7 +180,8 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): self.attrs['output_padding'] = self.output_padding output = conv2dtranspose_forward_naive( - input_, filter_, self.attrs).astype(self.dtype) + input_, filter_, self.attrs + ).astype(self.dtype) self.outputs = {'Output': output} @@ -169,20 +190,21 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): def test_check_grad_no_input(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def test_check_grad_no_filter(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad(self): if self.need_check_grad: - self.check_grad_with_place(self.place, set(['Input', 'Filter']), - 'Output') + self.check_grad_with_place( + self.place, set(['Input', 'Filter']), 'Output' + ) def init_test_case(self): self.pad = [0, 0] @@ -199,7 +221,6 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): self.op_type = "conv2d_transpose" class TestWithSymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -210,7 +231,6 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): self.filter_size = [f_c, 6, 3, 3] class TestWithAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -221,7 +241,6 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): self.filter_size = [f_c, 6, 3, 3] class TestWithSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [2, 1] self.dilations = [1, 2] @@ -232,7 +251,6 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): self.padding_algorithm = 'SAME' class TestWithVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -243,7 +261,6 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): self.padding_algorithm = 'VALID' class TestWithGroups(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -254,7 +271,6 @@ class XPUTestConv2DTransposeOp(XPUOpTestWrapper): self.filter_size = [f_c, 3, 3, 3] class TestWithStride(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_deformable_conv_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_deformable_conv_op_xpu.py index 85b05e8990b..678afafc08c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_deformable_conv_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_deformable_conv_op_xpu.py @@ -22,7 +22,11 @@ import paddle.fluid.core as core import paddle.fluid as fluid from op_test_xpu import OpTest, XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def dmc_bilinear(data_im, height, width, h, w): @@ -64,8 +68,11 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): assert f_c * group == in_c assert np.mod(out_c, group) == 0 - stride, pad, dilation = conv_param['stride'], conv_param['pad'],\ - conv_param['dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) // stride[0] out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) // stride[1] assert out_h == in_h @@ -78,31 +85,47 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): for w in range(out_w): for kh in range(f_h): for kw in range(f_w): - offset_h_table = \ - offset[n, ::2, h, w].reshape(f_h, f_w) - offset_w_table = \ - offset[n, 1::2, h, w].reshape(f_h, f_w) - mask_table = \ - mask[n, :, h, w].reshape(f_h, f_w) + offset_h_table = offset[n, ::2, h, w].reshape( + f_h, f_w + ) + offset_w_table = offset[n, 1::2, h, w].reshape( + f_h, f_w + ) + mask_table = mask[n, :, h, w].reshape(f_h, f_w) offset_h = offset_h_table[kh, kw] offset_w = offset_w_table[kh, kw] val = 0 - im_h = h * stride[0] + kh * dilation[0] \ - + offset_h - pad[0] - im_w = w * stride[0] + kw * dilation[0] \ - + offset_w - pad[1] - if im_h > -1 and im_w > -1 and \ - im_h < in_h and im_w < in_h: - val = dmc_bilinear(input[n, c], in_h, in_w, - im_h, im_w) + im_h = ( + h * stride[0] + + kh * dilation[0] + + offset_h + - pad[0] + ) + im_w = ( + w * stride[0] + + kw * dilation[0] + + offset_w + - pad[1] + ) + if ( + im_h > -1 + and im_w > -1 + and im_h < in_h + and im_w < in_h + ): + val = dmc_bilinear( + input[n, c], in_h, in_w, im_h, im_w + ) val_out = val * mask_table[kh, kw] - col_buffer[n, c * f_h * f_w + kh * f_w + kw, - h * in_w + w] = val_out + col_buffer[ + n, c * f_h * f_w + kh * f_w + kw, h * in_w + w + ] = val_out out = np.zeros((in_n, group, int(out_c // group), out_h * out_w)) weight = filter.reshape(group, int(out_c // group), f_c * f_h * f_w) col_buffer = col_buffer.reshape( - (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w)) + (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w) + ) for n in range(in_n): for g in range(group): out[n, g] = np.matmul(weight[g], col_buffer[n, g]) @@ -111,13 +134,11 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): class XPUTestModulatedDeformableConvOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'deformable_conv' self.use_dynamic_create_class = False class TestModulatedDeformableConvOp(XPUOpTest): - def setUp(self): self.op_type = "deformable_conv" self.dtype = self.in_type @@ -129,22 +150,23 @@ class XPUTestModulatedDeformableConvOp(XPUOpTestWrapper): conv_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) offset = 10 * np.random.random(self.offset_size).astype(self.dtype) mask = 10 * np.random.random(self.mask_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = dconv_im2col_gemm(input, offset, mask, filter, self.groups, - conv_param) + output = dconv_im2col_gemm( + input, offset, mask, filter, self.groups, conv_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Offset': OpTest.np_dtype_to_fluid_dtype(offset), 'Mask': OpTest.np_dtype_to_fluid_dtype(mask), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -165,9 +187,11 @@ class XPUTestModulatedDeformableConvOp(XPUOpTestWrapper): if core.is_compiled_with_xpu(): paddle.enable_static() self.check_grad_with_place( - self.place, {'Input', 'Offset', 'Mask', 'Filter'}, + self.place, + {'Input', 'Offset', 'Mask', 'Filter'}, 'Output', - max_relative_error=0.06) + max_relative_error=0.06, + ) def init_test_case(self): self.pad = [1, 1] @@ -179,17 +203,28 @@ class XPUTestModulatedDeformableConvOp(XPUOpTestWrapper): self.filter_size = [8, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -199,7 +234,6 @@ class XPUTestModulatedDeformableConvOp(XPUOpTestWrapper): self.groups = 1 class TestWithDilation(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [1, 1] @@ -209,24 +243,34 @@ class XPUTestModulatedDeformableConvOp(XPUOpTestWrapper): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): self.dilations = [2, 2] class TestWith3x3(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -236,56 +280,61 @@ class XPUTestModulatedDeformableConvOp(XPUOpTestWrapper): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestModulatedDeformableConvInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): paddle.enable_static() input = [1, 3, 32, 32] - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=1) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=1 + ) self.assertRaises(TypeError, test_invalid_input) def test_invalid_offset(): paddle.enable_static() - input = fluid.data(name='input', - shape=[None, 3, 32, 32], - dtype='int32') - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=1) + input = fluid.data( + name='input', shape=[None, 3, 32, 32], dtype='int32' + ) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=1 + ) self.assertRaises(TypeError, test_invalid_offset) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_depthwise_conv2d_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_depthwise_conv2d_op_xpu.py index a8bfd681634..75c6bd7b524 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_depthwise_conv2d_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_depthwise_conv2d_op_xpu.py @@ -22,17 +22,19 @@ import paddle paddle.enable_static() from test_conv2d_op_xpu import XPUTestConv2DOp, XPUTestConv2DOp_v2 -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestDepthwiseConv2DOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'depthwise_conv2d' self.use_dynamic_create_class = False class TestDepthwiseConv(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -45,7 +47,6 @@ class XPUTestDepthwiseConv2DOp(XPUOpTestWrapper): self.op_type = "depthwise_conv2d" class TestDepthwiseConv2(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -58,7 +59,6 @@ class XPUTestDepthwiseConv2DOp(XPUOpTestWrapper): self.op_type = "depthwise_conv2d" class TestDepthwiseConv3(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -71,7 +71,6 @@ class XPUTestDepthwiseConv2DOp(XPUOpTestWrapper): self.op_type = "depthwise_conv2d" class TestDepthwiseConvWithDilation(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -85,7 +84,6 @@ class XPUTestDepthwiseConv2DOp(XPUOpTestWrapper): self.op_type = "depthwise_conv2d" class TestDepthwiseConvWithDilation2(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -100,13 +98,11 @@ class XPUTestDepthwiseConv2DOp(XPUOpTestWrapper): class XPUTestDepthwiseConv2DOp_v2(XPUOpTestWrapper): - def __init__(self): self.op_name = 'depthwise_conv2d' self.use_dynamic_create_class = False class TestDepthwiseConv_AsyPadding(XPUTestConv2DOp_v2.TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = False self.stride = [2, 2] @@ -122,7 +118,6 @@ class XPUTestDepthwiseConv2DOp_v2(XPUOpTestWrapper): self.padding_algorithm = "EXPLICIT" class TestDepthwiseConv2_AsyPadding(XPUTestConv2DOp_v2.TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = False self.stride = [1, 1] @@ -138,7 +133,6 @@ class XPUTestDepthwiseConv2DOp_v2(XPUOpTestWrapper): self.padding_algorithm = "EXPLICIT" class TestDepthwiseConv3_AsyPadding(XPUTestConv2DOp_v2.TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = False self.stride = [1, 1] @@ -154,8 +148,8 @@ class XPUTestDepthwiseConv2DOp_v2(XPUOpTestWrapper): self.padding_algorithm = "EXPLICIT" class TestDepthwiseConvWithDilation_AsyPadding( - XPUTestConv2DOp_v2.TestConv2DOp_v2): - + XPUTestConv2DOp_v2.TestConv2DOp_v2 + ): def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -173,8 +167,8 @@ class XPUTestDepthwiseConv2DOp_v2(XPUOpTestWrapper): self.padding_algorithm = "EXPLICIT" class TestDepthwiseConvWithDilation2_AsyPadding( - XPUTestConv2DOp_v2.TestConv2DOp_v2): - + XPUTestConv2DOp_v2.TestConv2DOp_v2 + ): def init_test_case(self): self.use_cuda = True self.pad = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_device_guard_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_device_guard_xpu.py index 3fbd3c6c2fe..8c41cb8675c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_device_guard_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_device_guard_xpu.py @@ -44,17 +44,16 @@ def get_vaild_warning_num(warning, w): class TestDeviceGuard(unittest.TestCase): - def test_device_guard(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.full(shape=[1, 3, 8, 8], - fill_value=0.5, - dtype='float32') - data2 = paddle.full(shape=[1, 3, 5, 5], - fill_value=0.5, - dtype='float32') + data1 = paddle.full( + shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32' + ) + data2 = paddle.full( + shape=[1, 3, 5, 5], fill_value=0.5, dtype='float32' + ) shape = paddle.shape(data2) with paddle.static.device_guard("cpu"): shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4]) @@ -75,12 +74,12 @@ class TestDeviceGuard(unittest.TestCase): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.full(shape=[1, 3, 8, 8], - fill_value=0.5, - dtype='float32') - data2 = paddle.full(shape=[1, 3, 5, 5], - fill_value=0.5, - dtype='float32') + data1 = paddle.full( + shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32' + ) + data2 = paddle.full( + shape=[1, 3, 5, 5], fill_value=0.5, dtype='float32' + ) shape = paddle.shape(data2) with paddle.static.device_guard("cpu"): shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4]) @@ -101,32 +100,50 @@ class TestDeviceGuard(unittest.TestCase): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.full(shape=[2, 255, 13, 13], - fill_value=0.3, - dtype='float32') - gt_box = paddle.full(shape=[2, 6, 4], - fill_value=0.5, - dtype='float32') + x = paddle.full( + shape=[2, 255, 13, 13], fill_value=0.3, dtype='float32' + ) + gt_box = paddle.full( + shape=[2, 6, 4], fill_value=0.5, dtype='float32' + ) gt_label = paddle.full(shape=[2, 6], fill_value=1.0, dtype='int32') - gt_score = paddle.full(shape=[2, 6], - fill_value=0.5, - dtype='float32') + gt_score = paddle.full( + shape=[2, 6], fill_value=0.5, dtype='float32' + ) anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, - 198, 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] anchor_mask = [0, 1, 2] with paddle.static.device_guard("xpu"): # yolov3_loss only has cpu kernel, so its cpu kernel will be executed - loss = fluid.layers.yolov3_loss(x=x, - gt_box=gt_box, - gt_label=gt_label, - gt_score=gt_score, - anchors=anchors, - anchor_mask=anchor_mask, - class_num=80, - ignore_thresh=0.7, - downsample_ratio=32) + loss = fluid.layers.yolov3_loss( + x=x, + gt_box=gt_box, + gt_label=gt_label, + gt_score=gt_score, + anchors=anchors, + anchor_mask=anchor_mask, + class_num=80, + ignore_thresh=0.7, + downsample_ratio=32, + ) execute(main_program, startup_program) @@ -159,7 +176,6 @@ class TestDeviceGuard(unittest.TestCase): execute(main_program, startup_program) def test_error(self): - def device_attr(): with paddle.static.device_guard("cpu1"): out = paddle.full(shape=[1], fill_value=0.2, dtype='float32') @@ -176,17 +192,18 @@ class TestDeviceGuard(unittest.TestCase): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.static.data(name="data_1", - shape=[4, 2], - dtype="float32") - label = paddle.static.data(name="label", - shape=[4, 1], - dtype="int64") + data1 = paddle.static.data( + name="data_1", shape=[4, 2], dtype="float32" + ) + label = paddle.static.data( + name="label", shape=[4, 1], dtype="int64" + ) fc1 = paddle.static.nn.fc(x=data1, size=10) fc2 = paddle.static.nn.fc(x=fc1, size=10) with paddle.static.device_guard("xpu"): out = paddle.nn.functional.softmax_with_cross_entropy( - logits=fc1 + fc2, label=label) + logits=fc1 + fc2, label=label + ) loss = paddle.mean(out) opt = paddle.optimizer.SGD(0.1) opt.minimize(loss) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py index 7aff1acfdf7..794ff490d7e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py @@ -25,17 +25,19 @@ from op_test_xpu import XPUOpTest paddle.enable_static() -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestDropoutOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'dropout' self.use_dynamic_create_class = False class TestDropoutOp(XPUOpTest): - def setUp(self): self.init_inputs_shape() self.init_attrs() @@ -46,7 +48,7 @@ class XPUTestDropoutOp(XPUOpTestWrapper): 'dropout_prob': self.dropout_prob, 'fix_seed': self.fix_seed, 'is_test': self.is_test, - 'dropout_implementation': self.dropout_implementation + 'dropout_implementation': self.dropout_implementation, } out = self.inputs['X'] * (1.0 - self.dropout_prob) @@ -74,19 +76,19 @@ class XPUTestDropoutOp(XPUOpTestWrapper): self.check_output() def test_check_grad_normal(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return self.check_grad(['X'], 'Out') class TestDropoutOpInput1d(TestDropoutOp): - def init_inputs_shape(self): self.shape = [2000] class TestDropoutOp2(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64] @@ -97,12 +99,10 @@ class XPUTestDropoutOp(XPUOpTestWrapper): self.dropout_implementation = "upscale_in_train" class TestDropoutOp3(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64, 2] class TestDropoutOp4(TestDropoutOp): - def init_attrs(self): self.__class__.no_need_check_grad = True self.dropout_prob = 0.35 @@ -111,7 +111,6 @@ class XPUTestDropoutOp(XPUOpTestWrapper): self.dropout_implementation = "downgrade_in_infer" class TestDropoutOp5(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64, 3] @@ -123,15 +122,16 @@ class XPUTestDropoutOp(XPUOpTestWrapper): self.dropout_implementation = "downgrade_in_infer" class TestDropoutOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, - 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), + [[1, 1, 1, 1]], + fluid.CPUPlace(), + ) fluid.layers.dropout(x1, dropout_prob=0.5) self.assertRaises(TypeError, test_Variable) @@ -139,15 +139,14 @@ class XPUTestDropoutOp(XPUOpTestWrapper): def test_dtype(): # the input dtype of dropout must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.dropout(x2, dropout_prob=0.5) self.assertRaises(TypeError, test_dtype) class TestDropoutCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -159,13 +158,12 @@ class XPUTestDropoutOp(XPUOpTestWrapper): input_np = np.random.random([40, 40]).astype(self.in_type) result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.Dropout(p=0.) + m = paddle.nn.Dropout(p=0.0) m.eval() result = m(input) np.testing.assert_allclose(result.numpy(), result_np) class TestDropoutBackward(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -183,13 +181,15 @@ class XPUTestDropoutOp(XPUOpTestWrapper): input = paddle.uniform([40, 40], dtype=self.in_type) input.stop_gradient = False - out, mask = _legacy_C_ops.dropout(input, 'dropout_prob', - 0.5) + out, mask = _legacy_C_ops.dropout( + input, 'dropout_prob', 0.5 + ) out.backward() np.testing.assert_allclose( input.gradient(), - self.cal_grad_downscale_in_infer(mask.numpy())) + self.cal_grad_downscale_in_infer(mask.numpy()), + ) def test_backward_upscale_train(self): for place in self.places: @@ -198,15 +198,19 @@ class XPUTestDropoutOp(XPUOpTestWrapper): prob = 0.5 input = paddle.uniform([40, 40], dtype=self.in_type) input.stop_gradient = False - out, mask = _legacy_C_ops.dropout(input, 'dropout_prob', - prob, - "dropout_implementation", - "upscale_in_train") + out, mask = _legacy_C_ops.dropout( + input, + 'dropout_prob', + prob, + "dropout_implementation", + "upscale_in_train", + ) out.backward() np.testing.assert_allclose( input.gradient(), - self.cal_grad_upscale_train(mask.numpy(), prob)) + self.cal_grad_upscale_train(mask.numpy(), prob), + ) def test_backward_upscale_train_2(self): for place in self.places: @@ -215,15 +219,19 @@ class XPUTestDropoutOp(XPUOpTestWrapper): prob = 0.3 input = paddle.uniform([40, 40], dtype=self.in_type) input.stop_gradient = False - out, mask = _legacy_C_ops.dropout(input, 'dropout_prob', - prob, - "dropout_implementation", - "upscale_in_train") + out, mask = _legacy_C_ops.dropout( + input, + 'dropout_prob', + prob, + "dropout_implementation", + "upscale_in_train", + ) out.backward() np.testing.assert_allclose( input.gradient(), - self.cal_grad_upscale_train(mask.numpy(), prob)) + self.cal_grad_upscale_train(mask.numpy(), prob), + ) support_types = get_xpu_op_support_types('dropout') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu.py index 6ab52fcc836..2d04b3d7549 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu.py @@ -22,19 +22,21 @@ from op_test_xpu import XPUOpTest import unittest import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseAddOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_add' self.use_dynamic_create_class = False class TestElementwiseAddOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_add" self.init_dtype() @@ -43,7 +45,7 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.init_max_relative_error() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -57,27 +59,33 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X', 'Y'], + place, + ['X', 'Y'], 'Out', - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['Y'], + place, + ['Y'], 'Out', no_grad_set=set("X"), - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X'], + place, + ['X'], 'Out', no_grad_set=set("Y"), - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -94,32 +102,30 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.max_relative_error = 0.006 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) self.out = self.x + self.y @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." + ) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) self.out = self.x + self.y class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -129,7 +135,6 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = 0 class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -139,14 +144,12 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = 1 class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) self.out = self.x + self.y.reshape(1, 1, 100) class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -156,7 +159,6 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = 1 class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3, 4).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -166,28 +168,24 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = 0 class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) self.out = self.x + self.y class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) self.out = self.x + self.y class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) self.out = self.x + self.y class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -197,9 +195,9 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = 1 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -209,7 +207,6 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = 1 class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -219,7 +216,6 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = -1 class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -229,7 +225,6 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = -1 class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -239,7 +234,6 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = -1 class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 3, 10, 12).astype(self.dtype) @@ -249,30 +243,32 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): self.axis = 2 class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, - y1) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + self.assertRaises( + TypeError, fluid.layers.elementwise_add, x1, y1 + ) # the input dtype of elementwise_add must be float16 or float32 or float64 or int32 or int64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="uint8") - y2 = fluid.layers.data(name='y2', - shape=[3, 4, 5, 6], - dtype="uint8") - self.assertRaises(TypeError, fluid.layers.elementwise_add, x2, - y2) + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="uint8" + ) + y2 = fluid.layers.data( + name='y2', shape=[3, 4, 5, 6], dtype="uint8" + ) + self.assertRaises( + TypeError, fluid.layers.elementwise_add, x2, y2 + ) class TestAddOp(unittest.TestCase): - def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="float32") @@ -287,7 +283,7 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -297,7 +293,7 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): place = fluid.XPUPlace(0) exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -308,7 +304,7 @@ class XPUTestElementwiseAddOp(XPUOpTestWrapper): y = fluid.dygraph.to_variable(np_y) z = paddle.add(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu_kp.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu_kp.py index c969332c5ae..7f201d3487e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu_kp.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu_kp.py @@ -26,10 +26,10 @@ from paddle.fluid import Program, program_guard paddle.enable_static() -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_add" self.init_dtype() @@ -38,7 +38,7 @@ class TestElementwiseAddOp(XPUOpTest): self.init_max_relative_error() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -52,27 +52,33 @@ class TestElementwiseAddOp(XPUOpTest): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X', 'Y'], + place, + ['X', 'Y'], 'Out', - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['Y'], + place, + ['Y'], 'Out', no_grad_set=set("X"), - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X'], + place, + ['X'], 'Out', no_grad_set=set("Y"), - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -89,44 +95,46 @@ class TestElementwiseAddOp(XPUOpTest): self.max_relative_error = 0.006 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -136,10 +144,10 @@ class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): self.axis = 0 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -149,20 +157,20 @@ class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): self.axis = 1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) self.out = self.x + self.y.reshape(1, 1, 100) -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -172,10 +180,10 @@ class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): self.axis = 1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3, 4).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -185,40 +193,40 @@ class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): self.axis = 0 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -228,12 +236,13 @@ class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): self.axis = 1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -243,10 +252,10 @@ class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): self.axis = 1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -256,10 +265,10 @@ class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): self.axis = -1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -269,10 +278,10 @@ class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): self.axis = -1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -282,10 +291,10 @@ class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): self.axis = -1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 3, 10, 12).astype(self.dtype) @@ -295,17 +304,19 @@ class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): self.axis = 2 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, y1) # the input dtype of elementwise_add must be float16 or float32 or float64 or int32 or int64 @@ -315,10 +326,10 @@ class TestElementwiseAddOpError(unittest.TestCase): self.assertRaises(TypeError, fluid.layers.elementwise_add, x2, y2) -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestAddOp(unittest.TestCase): - def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="float32") @@ -333,7 +344,7 @@ class TestAddOp(unittest.TestCase): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -343,7 +354,7 @@ class TestAddOp(unittest.TestCase): place = fluid.XPUPlace(0) exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -354,7 +365,7 @@ class TestAddOp(unittest.TestCase): y = fluid.dygraph.to_variable(np_y) z = paddle.add(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_div_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_div_op_xpu.py index 956e71cb69e..4144a7068e0 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_div_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_div_op_xpu.py @@ -20,19 +20,21 @@ import paddle import paddle.fluid as fluid from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseDivOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_div' self.use_dynamic_create_class = False class ElementwiseDivOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_div" self.dtype = self.in_type @@ -48,7 +50,7 @@ class XPUTestElementwiseDivOp(XPUOpTestWrapper): def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) @@ -62,133 +64,138 @@ class XPUTestElementwiseDivOp(XPUOpTestWrapper): def test_check_grad_normal(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.05) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.05 + ) def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.05, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.05, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.05, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.05, + no_grad_set=set('Y'), + ) def init_dtype(self): pass @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseDivOp_scalar(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [20, 3, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} class TestElementwiseDivOp_Vector(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) } class TestElementwiseDivOp_broadcast_0(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [100, 3, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.attrs = {'axis': 0} self.outputs = { - 'Out': - np.divide(self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseDivOp_broadcast_1(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 100, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseDivOp_broadcast_2(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.outputs = { - 'Out': - np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseDivOp_broadcast_3(ElementwiseDivOp): - def init_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [2, 10, 12, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [10, 12]).astype(self.dtype) + 'X': np.random.uniform(0.1, 1, [2, 10, 12, 5]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [10, 12]).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.divide(self.inputs['X'], - self.inputs['Y'].reshape(1, 10, 12, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 10, 12, 1) + ) } class TestElementwiseDivOp_broadcast_4(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 50]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype(self.dtype), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) } class TestElementwiseDivOp_broadcast_5(ElementwiseDivOp): - def init_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [2, 3, 4, 20]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype(self.dtype) + 'X': np.random.uniform(0.1, 1, [2, 3, 4, 20]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype( + self.dtype + ), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) } class TestElementwiseDivOp_commonuse_1(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype(self.dtype), @@ -199,25 +206,26 @@ class XPUTestElementwiseDivOp(XPUOpTestWrapper): } class TestElementwiseDivOp_commonuse_2(ElementwiseDivOp): - def init_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [30, 3, 1, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, - [30, 1, 4, 1]).astype(self.dtype), + 'X': np.random.uniform(0.1, 1, [30, 3, 1, 5]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [30, 1, 4, 1]).astype( + self.dtype + ), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) } class TestElementwiseDivOp_xsize_lessthan_ysize(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [10, 12]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, - [2, 3, 10, 12]).astype(self.dtype), + 'Y': np.random.uniform(0.1, 1, [2, 3, 10, 12]).astype( + self.dtype + ), } self.attrs = {'axis': 2} @@ -227,18 +235,18 @@ class XPUTestElementwiseDivOp(XPUOpTestWrapper): } class TestElementwiseDivBroadcast(unittest.TestCase): - def test_shape_with_batch_sizes(self): with fluid.program_guard(fluid.Program()): - x_var = fluid.data(name='x', - dtype='float32', - shape=[None, 3, None, None]) - one = 2. + x_var = fluid.data( + name='x', dtype='float32', shape=[None, 3, None, None] + ) + one = 2.0 out = one / x_var exe = fluid.Executor(fluid.XPUPlace(0)) - x = np.random.uniform(0.1, 0.6, - (1, 3, 32, 32)).astype('float32') - out_result, = exe.run(feed={'x': x}, fetch_list=[out]) + x = np.random.uniform(0.1, 0.6, (1, 3, 32, 32)).astype( + 'float32' + ) + (out_result,) = exe.run(feed={'x': x}, fetch_list=[out]) self.assertEqual((out_result == (2 / x)).all(), True) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_floordiv_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_floordiv_op_xpu.py index d954e0d6ce6..f557221b488 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_floordiv_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_floordiv_op_xpu.py @@ -19,20 +19,22 @@ import numpy as np import paddle from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() import random class XPUTestElementwiseModOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_floordiv' self.use_dynamic_create_class = False class TestElementwiseModOp(XPUOpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -46,7 +48,7 @@ class XPUTestElementwiseModOp(XPUOpTestWrapper): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -65,7 +67,6 @@ class XPUTestElementwiseModOp(XPUOpTestWrapper): pass class TestElementwiseModOp_scalar(TestElementwiseModOp): - def init_input_output(self): scale_x = random.randint(0, 100000) scale_y = random.randint(1, 100000) @@ -74,7 +75,6 @@ class XPUTestElementwiseModOp(XPUOpTestWrapper): self.out = np.floor_divide(self.x, self.y) class TestElementwiseModOpInverse(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.uniform(0, 10000, [10]).astype(self.dtype) self.y = np.random.uniform(1, 1000, [10, 10]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_max_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_max_op_xpu.py index 582f73d525c..f2a05670f41 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_max_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_max_op_xpu.py @@ -19,19 +19,21 @@ import numpy as np from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseMaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_max' self.use_dynamic_create_class = False class TestElementwiseOp(XPUOpTest): - def setUp(self): self.use_xpu = True self.op_type = "elementwise_max" @@ -63,23 +65,29 @@ class XPUTestElementwiseMaxOp(XPUOpTestWrapper): def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.006, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.006, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.006, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.006, + no_grad_set=set('Y'), + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseMaxOp_scalar(TestElementwiseOp): - def init_input_output(self): x = np.random.random_integers(-5, 5, [2, 3, 20]).astype(self.dtype) y = np.array([0.5]).astype(self.dtype) @@ -89,86 +97,85 @@ class XPUTestElementwiseMaxOp(XPUOpTestWrapper): } class TestElementwiseMaxOp_Vector(TestElementwiseOp): - def init_input_output(self): - x = np.random.random((100, )).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x + sgn * np.random.uniform(0.1, 1, (100, )).astype(self.dtype) + x = np.random.random((100,)).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x + sgn * np.random.uniform(0.1, 1, (100,)).astype(self.dtype) self.inputs = {'X': x, 'Y': y} self.outputs = { 'Out': np.maximum(self.inputs['X'], self.inputs['Y']) } class TestElementwiseMaxOp_broadcast_0(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (100, 5, 2)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 0} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseMaxOp_broadcast_1(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseMaxOp_broadcast_2(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (1, 3, 100)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseMaxOp_broadcast_3(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 50, 2, 1)).astype(self.dtype) sgn = np.random.choice([-1, 1], (50, 2)).astype(self.dtype) - y = x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (50, 2)).astype(self.dtype) + y = x[0, :, :, 0] + sgn * np.random.uniform(1, 2, (50, 2)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 50, 2, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 50, 2, 1) + ) } class TestElementwiseMaxOp_broadcast_4(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 5)).astype(self.dtype) - y = x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 5)).astype(self.dtype) + y = x + sgn * np.random.uniform(1, 2, (2, 3, 1, 5)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.outputs = { diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_min_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_min_op_xpu.py index c7eb91e83be..498410d6dbb 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_min_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_min_op_xpu.py @@ -19,19 +19,21 @@ import numpy as np from op_test import skip_check_grad_ci import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseMinOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_min' self.use_dynamic_create_class = False class TestElementwiseOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_min" # If x and y have the same value, the min() is not differentiable. @@ -62,23 +64,29 @@ class XPUTestElementwiseMinOp(XPUOpTestWrapper): def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseMinOp_scalar(TestElementwiseOp): - def init_input_output(self): x = np.random.random_integers(-5, 5, [10, 3, 4]).astype(self.dtype) y = np.array([0.5]).astype(self.dtype) @@ -88,82 +96,81 @@ class XPUTestElementwiseMinOp(XPUOpTestWrapper): } class TestElementwiseMinOp_Vector(TestElementwiseOp): - def init_input_output(self): - x = np.random.random((100, )).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x + sgn * np.random.uniform(0.1, 1, (100, )).astype(self.dtype) + x = np.random.random((100,)).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x + sgn * np.random.uniform(0.1, 1, (100,)).astype(self.dtype) self.inputs = {'X': x, 'Y': y} self.outputs = { 'Out': np.minimum(self.inputs['X'], self.inputs['Y']) } class TestElementwiseMinOp_broadcast_0(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (100, 3, 2)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.attrs = {'axis': 0} self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseMinOp_broadcast_1(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.attrs = {'axis': 1} self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseMinOp_broadcast_2(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseMinOp_broadcast_3(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 25, 4, 1)).astype(self.dtype) sgn = np.random.choice([-1, 1], (25, 4)).astype(self.dtype) - y = x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (25, 4)).astype(self.dtype) + y = x[0, :, :, 0] + sgn * np.random.uniform(1, 2, (25, 4)).astype( + self.dtype + ) self.attrs = {'axis': 1} self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 25, 4, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 25, 4, 1) + ) } class TestElementwiseMinOp_broadcast_4(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 10, 2, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 10, 1, 5)).astype(self.dtype) - y = x + sgn * \ - np.random.uniform(1, 2, (2, 10, 1, 5)).astype(self.dtype) + y = x + sgn * np.random.uniform(1, 2, (2, 10, 1, 5)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.outputs = { 'Out': np.minimum(self.inputs['X'], self.inputs['Y']) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mod_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mod_op_xpu.py index f446fd1f2ad..334cd0794b4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mod_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mod_op_xpu.py @@ -21,19 +21,21 @@ import paddle.fluid as fluid import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseModOp(XPUOpTestWrapper): - def __init__(self) -> None: self.op_name = 'elementwise_mod' self.use_dynamic_create_class = False class ElementwiseModOp(XPUOpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -43,7 +45,7 @@ class XPUTestElementwiseModOp(XPUOpTestWrapper): self.out = np.mod(self.x, self.y) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} @@ -70,29 +72,26 @@ class XPUTestElementwiseModOp(XPUOpTestWrapper): self.check_output_with_place(place) class TestElementwiseModOp_broadcast_1(ElementwiseModOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 100, 3).astype(self.dtype), - 'Y': np.random.rand(2, 100, 3).astype(self.dtype) + 'Y': np.random.rand(2, 100, 3).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = {'Out': self.inputs['X'] % self.inputs['Y']} class TestElementwiseModOp_broadcast_2(ElementwiseModOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(22, 128, 3).astype(self.dtype), - 'Y': np.random.rand(22, 128, 3).astype(self.dtype) + 'Y': np.random.rand(22, 128, 3).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = {'Out': self.inputs['X'] % self.inputs['Y']} class TestRemainderOp(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): np_x = np.random.rand(22, 128, 3).astype('int64') @@ -105,7 +104,7 @@ class XPUTestElementwiseModOp(XPUOpTestWrapper): self.assertEqual((np_z == z_expected).all(), True) np_x = np.array([-3.3, 11.5, -2, 3.5]) - np_y = np.array([-1.2, 2., 3.3, -2.3]) + np_y = np.array([-1.2, 2.0, 3.3, -2.3]) x = paddle.to_tensor(np_x) y = paddle.to_tensor(np_y) z = x % y diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mul_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mul_op_xpu.py index 0e09eafbdb7..22ee95c07d4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mul_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mul_op_xpu.py @@ -21,19 +21,21 @@ import paddle.fluid as fluid from paddle.fluid import Program, program_guard import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseMulOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_mul' self.use_dynamic_create_class = False class ElementwiseMulOp(XPUOpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -56,27 +58,33 @@ class XPUTestElementwiseMulOp(XPUOpTestWrapper): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X', 'Y'], + place, + ['X', 'Y'], 'Out', - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['Y'], + place, + ['Y'], 'Out', no_grad_set=set("X"), - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X'], + place, + ['X'], 'Out', no_grad_set=set('Y'), - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -84,7 +92,7 @@ class XPUTestElementwiseMulOp(XPUOpTestWrapper): self.out = np.multiply(self.x, self.y) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} @@ -96,33 +104,31 @@ class XPUTestElementwiseMulOp(XPUOpTestWrapper): pass @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 3, 4).astype(self.dtype), - 'Y': np.random.rand(1).astype(self.dtype) + 'Y': np.random.rand(1).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def init_input_output(self): self.inputs = { - 'X': np.random.random((100, )).astype(self.dtype), - 'Y': np.random.random((100, )).astype(self.dtype) + 'X': np.random.random((100,)).astype(self.dtype), + 'Y': np.random.random((100,)).astype(self.dtype), } self.outputs = { 'Out': np.multiply(self.inputs['X'], self.inputs['Y']) } class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(100, 2, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.outputs = { 'Out': self.inputs['X'] * self.inputs['Y'].reshape(100, 1, 1) @@ -130,11 +136,10 @@ class XPUTestElementwiseMulOp(XPUOpTestWrapper): self.attrs = {'axis': 0} class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 100, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.attrs = {'axis': 1} @@ -143,11 +148,10 @@ class XPUTestElementwiseMulOp(XPUOpTestWrapper): } class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.outputs = { @@ -155,61 +159,54 @@ class XPUTestElementwiseMulOp(XPUOpTestWrapper): } class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(self.dtype), - 'Y': np.random.rand(10, 12).astype(self.dtype) + 'Y': np.random.rand(10, 12).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - self.inputs['X'] * self.inputs['Y'].reshape(1, 10, 12, 1) + 'Out': self.inputs['X'] * self.inputs['Y'].reshape(1, 10, 12, 1) } class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 2, 11).astype(self.dtype), - 'Y': np.random.rand(10, 1, 11).astype(self.dtype) + 'Y': np.random.rand(10, 1, 11).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 4, 2, 3).astype(self.dtype), - 'Y': np.random.rand(10, 4, 1, 3).astype(self.dtype) + 'Y': np.random.rand(10, 4, 1, 3).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(1, 1, 100).astype(self.dtype) + 'Y': np.random.rand(1, 1, 100).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(30, 3, 1, 5).astype(self.dtype), - 'Y': np.random.rand(30, 1, 4, 1).astype(self.dtype) + 'Y': np.random.rand(30, 1, 4, 1).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 10).astype(self.dtype), - 'Y': np.random.rand(2, 2, 10, 10).astype(self.dtype) + 'Y': np.random.rand(2, 2, 10, 10).astype(self.dtype), } self.attrs = {'axis': 2} @@ -219,26 +216,29 @@ class XPUTestElementwiseMulOp(XPUOpTestWrapper): } class TestElementwiseMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_mul must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - self.assertRaises(TypeError, fluid.layers.elementwise_mul, x1, - y1) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + self.assertRaises( + TypeError, fluid.layers.elementwise_mul, x1, y1 + ) # the input dtype of elementwise_mul must be float32 - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="uint8") - y2 = fluid.layers.data(name='y2', - shape=[3, 4, 5, 6], - dtype="uint8") - self.assertRaises(TypeError, fluid.layers.elementwise_mul, x2, - y2) + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="uint8" + ) + y2 = fluid.layers.data( + name='y2', shape=[3, 4, 5, 6], dtype="uint8" + ) + self.assertRaises( + TypeError, fluid.layers.elementwise_mul, x2, y2 + ) support_types = get_xpu_op_support_types('elementwise_mul') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_pow_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_pow_op_xpu.py index 6387346f4ec..95315d55878 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_pow_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_pow_op_xpu.py @@ -19,20 +19,22 @@ import numpy as np import paddle from op_test import OpTest, skip_check_grad_ci from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @skip_check_grad_ci(reason="XPU does not support grad op currently") class XPUTestElementwisePowOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_pow' self.use_dynamic_create_class = False class TestElementwisePowOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_pow" self.dtype = self.in_type @@ -42,7 +44,7 @@ class XPUTestElementwisePowOp(XPUOpTestWrapper): def compute_input_output(self): self.inputs = { 'X': np.random.uniform(1, 2, [20, 5]).astype(self.dtype), - 'Y': np.random.uniform(1, 2, [20, 5]).astype(self.dtype) + 'Y': np.random.uniform(1, 2, [20, 5]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @@ -52,110 +54,107 @@ class XPUTestElementwisePowOp(XPUOpTestWrapper): self.check_output_with_place(place) class TestElementwisePowOp_big_shape_1(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(1, 2, [10, 10]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [10, 10]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [10, 10]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_big_shape_2(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(1, 2, [10, 10]).astype(self.dtype), - 'Y': np.random.uniform(0.2, 2, [10, 10]).astype(self.dtype) + 'Y': np.random.uniform(0.2, 2, [10, 10]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwisePowOp_scalar(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [3, 3, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_tensor(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - 'Y': np.random.uniform(1, 3, [100]).astype(self.dtype) + 'Y': np.random.uniform(1, 3, [100]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_broadcast_0(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 1, 100]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_broadcast_1(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 100, 1]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': np.power(self.inputs['X'], - self.inputs['Y'].reshape(100, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(100, 1) + ) } class TestElementwisePowOp_broadcast_2(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [100, 3, 1]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.attrs = {'axis': 0} self.outputs = { - 'Out': - np.power(self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwisePowOp_broadcast_3(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [2, 20, 5, 1]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [20, 5]).astype(self.dtype) + 'X': np.random.uniform(0.1, 1, [2, 20, 5, 1]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [20, 5]).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.power(self.inputs['X'], - self.inputs['Y'].reshape(1, 20, 5, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(1, 20, 5, 1) + ) } class TestElementwisePowOp_broadcast_4(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [2, 10, 3, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype(self.dtype) + 'X': np.random.uniform(0.1, 1, [2, 10, 3, 5]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype( + self.dtype + ), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOpInt(OpTest): - def setUp(self): self.op_type = "elementwise_pow" self.inputs = { 'X': np.asarray([1, 3, 6]), - 'Y': np.asarray([1, 1, 1]) + 'Y': np.asarray([1, 1, 1]), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_sub_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_sub_op_xpu.py index 67f4d561f6e..927855f461d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_sub_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_sub_op_xpu.py @@ -20,19 +20,21 @@ import paddle from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest import unittest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseSubOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_sub' self.use_dynamic_create_class = False class TestElementwiseOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_sub" self.use_xpu = True @@ -42,7 +44,7 @@ class XPUTestElementwiseSubOp(XPUOpTestWrapper): def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} @@ -59,45 +61,49 @@ class XPUTestElementwiseSubOp(XPUOpTestWrapper): def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseSubOp_scalar(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 3, 4).astype(self.dtype), - 'Y': np.random.rand(1).astype(self.dtype) + 'Y': np.random.rand(1).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_Vector(TestElementwiseOp): - def init_input_output(self): self.inputs = { - 'X': np.random.random((100, )).astype(self.dtype), - 'Y': np.random.random((100, )).astype(self.dtype) + 'X': np.random.random((100,)).astype(self.dtype), + 'Y': np.random.random((100,)).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_broadcast_0(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(100, 3, 2).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.attrs = {'axis': 0} @@ -106,11 +112,10 @@ class XPUTestElementwiseSubOp(XPUOpTestWrapper): } class TestElementwiseSubOp_broadcast_1(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 100, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.attrs = {'axis': 1} @@ -119,11 +124,10 @@ class XPUTestElementwiseSubOp(XPUOpTestWrapper): } class TestElementwiseSubOp_broadcast_2(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.outputs = { @@ -131,52 +135,46 @@ class XPUTestElementwiseSubOp(XPUOpTestWrapper): } class TestElementwiseSubOp_broadcast_3(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(self.dtype), - 'Y': np.random.rand(10, 12).astype(self.dtype) + 'Y': np.random.rand(10, 12).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - self.inputs['X'] - self.inputs['Y'].reshape(1, 10, 12, 1) + 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 10, 12, 1) } class TestElementwiseSubOp_broadcast_4(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 5, 3, 12).astype(self.dtype), - 'Y': np.random.rand(2, 5, 1, 12).astype(self.dtype) + 'Y': np.random.rand(2, 5, 1, 12).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_1(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(1, 1, 100).astype(self.dtype) + 'Y': np.random.rand(1, 1, 100).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_2(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 3, 1, 4).astype(self.dtype), - 'Y': np.random.rand(10, 1, 12, 1).astype(self.dtype) + 'Y': np.random.rand(10, 1, 12, 1).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_xsize_lessthan_ysize(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 12).astype(self.dtype), - 'Y': np.random.rand(2, 3, 10, 12).astype(self.dtype) + 'Y': np.random.rand(2, 3, 10, 12).astype(self.dtype), } self.attrs = {'axis': 2} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_empty_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_empty_op_xpu.py index ec4a7e8b1cf..cb56e9b51f4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_empty_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_empty_op_xpu.py @@ -1,4 +1,4 @@ -#Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,20 +21,22 @@ import numpy as np import paddle from op_test_xpu import XPUOpTest from paddle.fluid.framework import convert_np_dtype_to_dtype_ -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestEmptyOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'empty' self.use_dynamic_create_class = False # Situation 1: Attr(shape) is a list(without tensor) class TestEmptyOp(XPUOpTest): - def setUp(self): self.op_type = "empty" self.init_dtype() @@ -50,24 +52,34 @@ class XPUTestEmptyOp(XPUOpTestWrapper): def verify_output(self, outs): data_type = outs[0].dtype if data_type in [ - 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'float16', 'int16' + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'float16', + 'int16', ]: max_value = np.nanmax(outs[0]) min_value = np.nanmin(outs[0]) always_full_zero = max_value == 0.0 and min_value == 0.0 always_non_full_zero = max_value >= min_value - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = outs[0].size true_num = np.sum(outs[0] == True) false_num = np.sum(outs[0] == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: - #pass + # pass self.assertTrue(False, 'invalid data type') def set_shape(self): @@ -90,34 +102,30 @@ class XPUTestEmptyOp(XPUOpTestWrapper): self.__class__.op_type = self.op_type class TestEmptyOpCase1(TestEmptyOp): - def set_shape(self): self.shape = [50] class TestEmptyOpCase2(TestEmptyOp): - def set_shape(self): self.shape = [1, 50, 3, 4] class TestEmptyOpCase3(TestEmptyOp): - def set_shape(self): self.shape = [5, 5, 5] # Situation 2: shape is a tensor class TestEmptyOp_ShapeTensor(TestEmptyOp): - def set_inputs(self): self.inputs = {"ShapeTensor": np.array(self.shape).astype("int32")} # Situation 3: Attr(shape) is a list(with tensor) class TestEmptyOp_ShapeTensorList(TestEmptyOp): - def set_inputs(self): shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_expand_as_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_expand_as_v2_op_xpu.py index 36b53de7004..246696be643 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_expand_as_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_expand_as_v2_op_xpu.py @@ -20,20 +20,22 @@ sys.path.append("..") from op_test_xpu import XPUOpTest import paddle import paddle.fluid as fluid -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) class XPUTestExpandAsV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'expand_as_v2' self.use_dynamic_create_class = False class TestExpandAsV2XPUOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -65,7 +67,6 @@ class XPUTestExpandAsV2Op(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestExpandAsOpRank2(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(10, 12).astype(self.dtype) self.inputs = {'X': x} @@ -78,7 +79,6 @@ class XPUTestExpandAsV2Op(XPUOpTestWrapper): self.outputs = {'Out': output} class TestExpandAsOpRank3(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(2, 3, 20).astype(self.dtype) self.inputs = {'X': x} @@ -91,7 +91,6 @@ class XPUTestExpandAsV2Op(XPUOpTestWrapper): self.outputs = {'Out': output} class TestExpandAsOpRank4(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(1, 1, 7, 16).astype(self.dtype) self.inputs = {'X': x} @@ -104,7 +103,6 @@ class XPUTestExpandAsV2Op(XPUOpTestWrapper): self.outputs = {'Out': output} class TestExpandAsOpRank5(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(1, 1, 7, 16, 1).astype(self.dtype) self.inputs = {'X': x} @@ -117,7 +115,6 @@ class XPUTestExpandAsV2Op(XPUOpTestWrapper): self.outputs = {'Out': output} class TestExpandAsOpRank6(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(1, 1, 7, 16, 1, 1).astype(self.dtype) self.inputs = {'X': x} @@ -132,29 +129,28 @@ class XPUTestExpandAsV2Op(XPUOpTestWrapper): # Test python API class TestExpandAsV2API(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([2, 12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[2, 12, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[2, 12, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = paddle.expand_as(x, y=y) exe = fluid.Executor(place=fluid.XPUPlace(0)) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (2, 1, 1))) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_expand_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_expand_v2_op_xpu.py index 545bebcb960..e5a30ad65bc 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_expand_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_expand_v2_op_xpu.py @@ -20,7 +20,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest import paddle.fluid as fluid import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) @@ -29,13 +33,11 @@ np.random.seed(10) # CANN Op Support X: float32, int32, int64 # Situation 1: shape is a list(without tensor) class XPUTestExpandV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'expand_v2' self.use_dynamic_create_class = False class TestExpandV2XPUOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -65,42 +67,36 @@ class XPUTestExpandV2Op(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestExpandV2OpRank2_DimExpanding(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] self.expand_times = [2, 1] class TestExpandV2OpRank2(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = [1, 140] self.shape = [12, 140] self.expand_times = [12, 1] class TestExpandV2OpRank3_Corner(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) self.expand_times = (1, 1, 1) class TestExpandV2OpRank4(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) self.expand_times = (1, 1, 1, 1) class TestExpandV2OpRank5(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = (2, 4, 1, 15) self.shape = (2, -1, 4, -1) self.expand_times = (1, 1, 4, 1) class TestExpandV2OpRank6(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = (4, 1, 30) self.shape = (2, -1, 4, 30) @@ -108,7 +104,6 @@ class XPUTestExpandV2Op(XPUOpTestWrapper): # Situation 2: shape is a list(with tensor) class TestExpandV2OpXPURank1_tensor_attr(TestExpandV2XPUOp): - def setUp(self): self.set_xpu() self.place = paddle.XPUPlace(0) @@ -117,8 +112,9 @@ class XPUTestExpandV2Op(XPUOpTestWrapper): self.dtype = np.float32 expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -135,8 +131,8 @@ class XPUTestExpandV2Op(XPUOpTestWrapper): self.infer_expand_shape = [-1] class TestExpandV2OpRank2_Corner_tensor_attr( - TestExpandV2OpXPURank1_tensor_attr): - + TestExpandV2OpXPURank1_tensor_attr + ): def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -145,7 +141,6 @@ class XPUTestExpandV2Op(XPUOpTestWrapper): # Situation 3: shape is a tensor class TestExpandV2XPUOp_tensor(TestExpandV2XPUOp): - def setUp(self): self.set_xpu() self.place = paddle.XPUPlace(0) @@ -170,7 +165,6 @@ class XPUTestExpandV2Op(XPUOpTestWrapper): # Situation 5: input x is int32 # skip grad check for int32 class TestExpandV2OpInteger(XPUOpTest): - def init_type(self): self.dtype = 'int32' @@ -198,20 +192,23 @@ class TestExpandV2OpInteger(XPUOpTest): # Test python API class TestExpandV2API(unittest.TestCase): - def test_static(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', + shape=[12, 14], + append_batch_size=False, + dtype="float32", + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.expand(x, shape=[12, 14]) out_2 = paddle.expand(x, shape=[positive_2, 14]) @@ -220,14 +217,14 @@ class TestExpandV2API(unittest.TestCase): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=paddle.XPUPlace(0)) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fill_any_like_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fill_any_like_op_xpu.py index 614891d174d..56b099b1e86 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fill_any_like_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fill_any_like_op_xpu.py @@ -20,19 +20,21 @@ import paddle import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestFillAnyLikeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'fill_any_like' self.use_dynamic_create_class = False class TestFillAnyLikeOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -60,22 +62,18 @@ class XPUTestFillAnyLikeOp(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestFillAnyLikeOp2(TestFillAnyLikeOp): - def set_value(self): self.value = -0.0 class TestFillAnyLikeOp3(TestFillAnyLikeOp): - def set_value(self): self.value = 1.0 class TestFillAnyLikeOp4(TestFillAnyLikeOp): - def init(self): self.value = 1e-9 class TestFillAnyLikeOp5(TestFillAnyLikeOp): - def set_value(self): if self.dtype == "float16": self.value = 0.05 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fill_constant_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fill_constant_op_xpu.py index 2494323d885..e09834075a7 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fill_constant_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fill_constant_op_xpu.py @@ -20,21 +20,22 @@ import paddle import numpy as np from op_test import convert_float_to_uint16 from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestFillConstantOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'fill_constant' self.use_dynamic_create_class = False # Situation 1: Attr(shape) is a list(without tensor) class TestFillConstantOp(XPUOpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.init_dtype() self.set_xpu() self.op_type = "fill_constant" @@ -105,7 +106,7 @@ class XPUTestFillConstantOp(XPUOpTestWrapper): self.attrs = { 'shape': self.shape, 'dtype': self.index, - 'value': self.value + 'value': self.value, } self.outputs = {'Out': np.full(self.shape, self.value)} @@ -113,58 +114,55 @@ class XPUTestFillConstantOp(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestFillConstantOp2(TestFillConstantOp): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' def set_shape(self): self.shape = [10, 10] class TestFillConstantOp3(TestFillConstantOp): - '''Test fill_constant op with specified int64 value - ''' + '''Test fill_constant op with specified int64 value''' def set_shape(self): self.shape = [123, 2, 1] class TestFillConstantOp4(TestFillConstantOp): - '''Test fill_constant op with specified int value - ''' + '''Test fill_constant op with specified int value''' def set_shape(self): self.shape = [123, 3, 2, 1] class TestFillConstantOp5(TestFillConstantOp): - '''Test fill_constant op with specified float value - ''' + '''Test fill_constant op with specified float value''' def set_shape(self): self.shape = [123] # Situation 2: Attr(shape) is a list(with tensor) class TestFillConstantOp1_ShapeTensorList(TestFillConstantOp): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' def set_data(self): shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = { 'shape': self.infer_shape, 'dtype': self.index, - 'value': self.value + 'value': self.value, } self.outputs = {'Out': np.full(self.shape, self.value)} if self.index == 22: self.outputs = { - 'Out': - np.full( + 'Out': np.full( self.shape, convert_float_to_uint16( - np.array([self.value]).astype("float32"))) + np.array([self.value]).astype("float32") + ), + ) } def set_shape(self): @@ -172,14 +170,14 @@ class XPUTestFillConstantOp(XPUOpTestWrapper): self.infer_shape = [123, 1] class TestFillConstantOp2_ShapeTensorList(TestFillConstantOp): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' def set_data(self): shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape, 'dtype': self.index} @@ -190,23 +188,22 @@ class XPUTestFillConstantOp(XPUOpTestWrapper): self.infer_shape = [1, 1, 1] class TestFillConstantOp3_ShapeTensorList( - TestFillConstantOp1_ShapeTensorList): - + TestFillConstantOp1_ShapeTensorList + ): def set_shape(self): self.shape = [123, 3, 2, 1] self.infer_shape = [123, 111, 11, 1] class TestFillConstantOp4_ShapeTensorList( - TestFillConstantOp1_ShapeTensorList): - + TestFillConstantOp1_ShapeTensorList + ): def set_shape(self): self.shape = [123] self.infer_shape = [1] # Situation 3: shape is a tensor class TestFillConstantOp1_ShapeTensor(TestFillConstantOp): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' def set_data(self): self.inputs = {"ShapeTensor": np.array(self.shape).astype("int32")} @@ -214,11 +211,12 @@ class XPUTestFillConstantOp(XPUOpTestWrapper): self.outputs = {'Out': np.full(self.shape, self.value)} if self.index == 22: self.outputs = { - 'Out': - np.full( + 'Out': np.full( self.shape, convert_float_to_uint16( - np.array([self.value]).astype("float32"))) + np.array([self.value]).astype("float32") + ), + ) } def set_shape(self): @@ -226,19 +224,18 @@ class XPUTestFillConstantOp(XPUOpTestWrapper): # Situation 4: value is a tensor class TestFillConstantOp1_ValueTensor(TestFillConstantOp): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' def set_data(self): self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype(self.dtype) + 'ValueTensor': np.array([self.value]).astype(self.dtype), } if self.index == 22: self.inputs = { - 'ValueTensor': - convert_float_to_uint16( - np.array([self.value]).astype("float32")) + 'ValueTensor': convert_float_to_uint16( + np.array([self.value]).astype("float32") + ) } self.attrs = {'value': self.value, 'dtype': self.index} self.outputs = {'Out': np.full(self.shape, self.value)} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_flatten2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_flatten2_op_xpu.py index 29d1cb3c25e..307889b4f26 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_flatten2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_flatten2_op_xpu.py @@ -19,19 +19,21 @@ sys.path.append("..") import numpy as np import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestFlatten2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'flatten2' self.use_dynamic_create_class = False class TestFlatten2Op(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "flatten2" @@ -44,7 +46,7 @@ class XPUTestFlatten2Op(XPUOpTestWrapper): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype(self.dtype) + "XShape": np.random.random(self.in_shape).astype(self.dtype), } def set_xpu(self): @@ -65,14 +67,12 @@ class XPUTestFlatten2Op(XPUOpTestWrapper): self.attrs = {"axis": self.axis} class TestFlatten2OpWithCornerAxis(TestFlatten2Op): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.axis = 0 self.new_shape = (1, 120) class TestFlatten2OpWithDefaultAxis(TestFlatten2Op): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -81,7 +81,6 @@ class XPUTestFlatten2Op(XPUOpTestWrapper): self.attrs = {} class TestFlatten2OpSixDims(TestFlatten2Op): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_flatten_contiguous_range_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_flatten_contiguous_range_op_xpu.py index 1296f5ce724..aa029301003 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_flatten_contiguous_range_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_flatten_contiguous_range_op_xpu.py @@ -22,19 +22,21 @@ import sys sys.path.append("..") from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestFlattenOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'flatten_contiguous_range' self.use_dynamic_create_class = False class TestFlattenOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "flatten_contiguous_range" @@ -52,7 +54,7 @@ class XPUTestFlattenOp(XPUOpTestWrapper): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype(self.dtype) + "XShape": np.random.random(self.in_shape).astype(self.dtype), } def set_xpu(self): @@ -68,7 +70,7 @@ class XPUTestFlattenOp(XPUOpTestWrapper): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 self.stop_axis = -1 - self.new_shape = (120) + self.new_shape = 120 def init_attrs(self): self.attrs = { @@ -78,7 +80,6 @@ class XPUTestFlattenOp(XPUOpTestWrapper): } class TestFlattenOp_1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 1 @@ -88,11 +89,10 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_2(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -102,11 +102,10 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_3(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -116,11 +115,10 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_4(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = -2 @@ -130,11 +128,10 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_5(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 2 @@ -144,11 +141,10 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.start_axis = 3 @@ -158,11 +154,10 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_Float32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -173,11 +168,10 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_int32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -189,14 +183,13 @@ class XPUTestFlattenOp(XPUOpTestWrapper): self.attrs = { "start_axis": self.start_axis, "stop_axis": self.stop_axis, - 'use_xpu': True + 'use_xpu': True, } def test_check_grad(self): pass class TestFlattenOp_int8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -207,14 +200,13 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): pass class TestFlattenOp_int64(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -225,7 +217,7 @@ class XPUTestFlattenOp(XPUOpTestWrapper): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -233,45 +225,58 @@ class XPUTestFlattenOp(XPUOpTestWrapper): class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_ValueError1(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) out = paddle.flatten(x_var, start_axis=2, stop_axis=1) self.assertRaises(ValueError, test_ValueError1) def test_ValueError2(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=10, stop_axis=1) self.assertRaises(ValueError, test_ValueError2) def test_ValueError3(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=2, stop_axis=10) self.assertRaises(ValueError, test_ValueError3) def test_type(): # dtype must be float32, float64, int8, int32, int64 - x2 = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x2 = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x2 = x2.astype('float16') - x2_var = paddle.fluid.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2_var = paddle.fluid.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) paddle.flatten(x2_var) self.assertRaises(TypeError, test_type) @@ -283,7 +288,6 @@ class TestFlatten2OpError(unittest.TestCase): class TestStaticFlattenPythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -293,9 +297,9 @@ class TestStaticFlattenPythonAPI(unittest.TestCase): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[2, 3, 4, 4], - dtype='float32') + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32' + ) out = self.execute_api(x, start_axis=-2, stop_axis=-1) exe = paddle.static.Executor(place=paddle.XPUPlace(0)) @@ -304,17 +308,22 @@ class TestStaticFlattenPythonAPI(unittest.TestCase): class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): - def execute_api(self, x, start_axis=0, stop_axis=-1): return x.flatten_(start_axis, stop_axis) class TestFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_InputError(): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_flatten_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_flatten_op_xpu.py index 5d93a5a5ac0..13efc51fa26 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_flatten_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_flatten_op_xpu.py @@ -19,19 +19,21 @@ sys.path.append("..") import numpy as np import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestFlattenOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'flatten' self.use_dynamic_create_class = False class TestFlattenOp(XPUOpTest): - def setUp(self): self.op_type = "flatten" self.use_xpu = True @@ -59,14 +61,12 @@ class XPUTestFlattenOp(XPUOpTestWrapper): self.attrs = {"axis": self.axis} class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 2, 10) self.axis = 0 self.new_shape = (1, 120) class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -75,7 +75,6 @@ class XPUTestFlattenOp(XPUOpTestWrapper): self.attrs = {} class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fleet_exe_dist_model_run_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fleet_exe_dist_model_run_xpu.py index 867379bf81e..8d5091d3235 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fleet_exe_dist_model_run_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fleet_exe_dist_model_run_xpu.py @@ -23,7 +23,6 @@ paddle.enable_static() class TestDistModelRun(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -34,8 +33,9 @@ class TestDistModelRun(unittest.TestCase): def test_dist_model_run(self): # step 0: declare folder to save the model and params - path_prefix = os.path.join(self.temp_dir.name, - "dist_model_run_test/inf") + path_prefix = os.path.join( + self.temp_dir.name, "dist_model_run_test/inf" + ) # step 1: saving the inference model and params x = paddle.static.data(name='x', shape=[28, 28], dtype='float32') @@ -47,12 +47,11 @@ class TestDistModelRun(unittest.TestCase): exe.run(paddle.static.default_startup_program()) x_data = np.random.randn(28, 28).astype('float32') y_data = np.random.randint(0, 9, size=[28, 1]).astype('int64') - exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[avg_loss], + ) paddle.static.save_inference_model(path_prefix, [x, y], [avg_loss], exe) print('save model to', path_prefix) @@ -74,14 +73,16 @@ class TestDistModelRun(unittest.TestCase): print("dist model rst:", dist_model_rst) # step 4: use framework's api to inference with fake data - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.load_inference_model(path_prefix, exe)) - results = exe.run(inference_program, - feed={ - 'x': x_tensor, - 'y': y_tensor - }, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(path_prefix, exe) + results = exe.run( + inference_program, + feed={'x': x_tensor, 'y': y_tensor}, + fetch_list=fetch_targets, + ) load_inference_model_rst = results[0] print("load inference model api rst:", load_inference_model_rst) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_grad_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_grad_op_xpu.py index 0c68982d140..435f44e4736 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_grad_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_grad_op_xpu.py @@ -22,7 +22,11 @@ import numpy as np import paddle import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def get_outputs(DOut, X, Y): @@ -34,13 +38,11 @@ def get_outputs(DOut, X, Y): class XPUTestFuseGemmGradOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'fused_gemm_epilogue_grad' self.use_dynamic_create_class = False class TestFuseGemmEpilogueGradOpDXYBias1(XPUOpTest): - def setUp(self): paddle.enable_static() self.op_type = "fused_gemm_epilogue_grad" @@ -53,13 +55,14 @@ class XPUTestFuseGemmGradOp(XPUOpTestWrapper): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - DX, DY, DBias = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + DX, DY, DBias = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DX': DX, 'DY': DY, 'DBias': DBias} def test_check_output(self): @@ -69,48 +72,48 @@ class XPUTestFuseGemmGradOp(XPUOpTestWrapper): self.check_output_with_place(core.XPUPlace(0), atol=self.atol) class TestFuseGemmEpilogueGradOpDXYBias2(XPUOpTest): - def init_data(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - _, DY, DBias = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + _, DY, DBias = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DY': DY, 'DBias': DBias} class TestFuseGemmEpilogueGradOpDXYBias3(XPUOpTest): - def init_data(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - _, DY, _ = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + _, DY, _ = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DY': DY} class TestFuseGemmEpilogueGradOpDXYBias4(XPUOpTest): - def init_data(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - DX, DY, _ = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + DX, DY, _ = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DX': DX, 'DY': DY} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_op_xpu.py index bb6cc6a03cb..c37d1bff5dd 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_op_xpu.py @@ -22,12 +22,19 @@ import paddle import paddle.fluid.core as core from paddle import _legacy_C_ops from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def gelu(x): - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * (1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + ) return y_ref.astype(x.dtype) @@ -82,13 +89,11 @@ def matmul_grad(x, y, bias, dz, trans_x, trans_y): class XPUTestFuseGemmOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'fused_gemm_epilogue' self.use_dynamic_create_class = False class TestFuseGemmBase(XPUOpTest): - def setUp(self): self.__class__.no_need_check_grad = True self.op_type = "fused_gemm_epilogue" @@ -97,13 +102,14 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype) - 0.5, 'Y': np.random.random(self.y_shape).astype(self.dtype) - 0.5, - 'Bias': - np.random.random(self.bias_shape).astype(self.dtype) - 0.5 + 'Bias': np.random.random(self.bias_shape).astype(self.dtype) + - 0.5, } if self.trans_x == True: - numpy_input_x = self.inputs['X'].reshape( - (self.x_shape[0], -1)).T + numpy_input_x = ( + self.inputs['X'].reshape((self.x_shape[0], -1)).T + ) else: numpy_input_x = self.inputs['X'].reshape((-1, self.x_shape[-1])) @@ -113,14 +119,17 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): numpy_input_y = self.inputs['Y'] self.outputs = { - 'Out': - get_output(numpy_input_x, numpy_input_y, self.inputs['Bias'], - self.activation).reshape(self.out_shape) + 'Out': get_output( + numpy_input_x, + numpy_input_y, + self.inputs['Bias'], + self.activation, + ).reshape(self.out_shape) } self.attrs = { "activation": self.activation, "trans_y": self.trans_y, - "trans_x": self.trans_x + "trans_x": self.trans_x, } def init_dtype_type(self): @@ -144,7 +153,6 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): self.check_output_with_place(core.XPUPlace(0), atol=self.atol) class TestFuseGemmEpilogueOp1(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [4, 8] self.y_shape = [4, 128] @@ -157,7 +165,6 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): self.trans_x = True class TestFuseGemmEpilogueOp2(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [8, 4] self.y_shape = [128, 4] @@ -170,7 +177,6 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): self.trans_x = False class TestFuseGemmEpilogueOp3(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [4, 8] self.y_shape = [128, 4] @@ -183,7 +189,6 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): self.trans_x = True class TestFuseGemmEpilogueOp4(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [2, 2, 8, 4] self.y_shape = [4, 128] @@ -196,7 +201,6 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): self.trans_x = False class TestFuseGemmEpilogueOp5(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [4, 2, 2, 8] self.y_shape = [4, 128] @@ -209,7 +213,6 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): self.trans_x = True class TestFuseGemmEpilogueOp6(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [8, 4] self.y_shape = [4, 128] @@ -222,7 +225,6 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): self.trans_x = False class TestFuseGemmEpilogueOp7(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [8, 4] self.y_shape = [4, 128] @@ -236,7 +238,6 @@ class XPUTestFuseGemmOp(XPUOpTestWrapper): class TestEagerFusedGemmEpilogue(unittest.TestCase): - def setUp(self): paddle.set_device('xpu') @@ -244,22 +245,22 @@ class TestEagerFusedGemmEpilogue(unittest.TestCase): paddle.disable_static() x_np = np.random.random((8, 4)).astype(np.float32) - 0.5 y_np = np.random.random((4, 128)).astype(np.float32) - 0.5 - bias_np = np.random.random((128, )).astype(np.float32) - 0.5 + bias_np = np.random.random((128,)).astype(np.float32) - 0.5 x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) bias = paddle.to_tensor(bias_np) x.stop_gradient = False y.stop_gradient = False - out1 = _legacy_C_ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, 'activation', - 'none') - out2 = _legacy_C_ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, 'activation', - 'relu') - out3 = _legacy_C_ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, 'activation', - 'gelu') + out1 = _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'none' + ) + out2 = _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'relu' + ) + out3 = _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'gelu' + ) out_np1 = get_output(x_np, y_np, bias_np, 'none') out_np2 = get_output(x_np, y_np, bias_np, 'relu') @@ -269,13 +270,16 @@ class TestEagerFusedGemmEpilogue(unittest.TestCase): np.testing.assert_allclose(out2, out_np2, atol=1e-04) np.testing.assert_allclose(out3, out_np3, atol=1e-03) - out_grad_np1 = np.random.randint(low=-20, high=20, - size=out_np1.shape).astype(np.float32) - paddle.autograd.backward(out1, - grad_tensors=[paddle.to_tensor(out_grad_np1)]) + out_grad_np1 = np.random.randint( + low=-20, high=20, size=out_np1.shape + ).astype(np.float32) + paddle.autograd.backward( + out1, grad_tensors=[paddle.to_tensor(out_grad_np1)] + ) x_grad_np, y_grad_np, bias_grad_np = matmul_grad( - x_np, y_np, bias_np, out_grad_np1, False, False) + x_np, y_np, bias_np, out_grad_np1, False, False + ) np.testing.assert_allclose(x.grad.numpy(), x_grad_np, atol=1e-02) self.assertEqual(y_grad_np.shape, y_np.shape) np.testing.assert_allclose(y.grad.numpy(), y_grad_np, atol=1e-03) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fused_resnet_basic_block_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fused_resnet_basic_block_op_xpu.py index 7913e7e0148..f45a4a135e3 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fused_resnet_basic_block_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fused_resnet_basic_block_op_xpu.py @@ -24,19 +24,21 @@ import paddle.nn as nn from paddle.fluid import core from paddle.incubate.xpu.resnet_block import ResNetBasicBlock from paddle.fluid.framework import default_main_program -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestResNetBasicBlockOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "resnet_basic_block" self.use_dynamic_create_class = False class TestResNetBasicBlockOp(OpTest): - def setUp(self): paddle.disable_static() self.dtype = self.in_type @@ -70,65 +72,86 @@ class XPUTestResNetBasicBlockOp(XPUOpTestWrapper): conv1_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) conv2_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) conv3_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - bn1_weight = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0)) - bn1_bias = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.0)) - bn2_weight = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0)) - bn2_bias = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.0)) - bn3_weight = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0)) - bn3_bias = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.0)) + learning_rate=0.001, + ) + bn1_weight = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ) + bn1_bias = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.0) + ) + bn2_weight = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ) + bn2_bias = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.0) + ) + bn3_weight = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ) + bn3_bias = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.0) + ) - self.conv1 = nn.Conv2D(in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=3, - stride=self.stride, - padding=1, - weight_attr=conv1_weight, - bias_attr=None, - data_format='NCHW') - self.bn1 = nn.BatchNorm(self.out_channels, - act='relu', - param_attr=bn1_weight, - bias_attr=bn1_bias, - data_layout='NCHW') - self.conv2 = nn.Conv2D(in_channels=self.out_channels, - out_channels=self.out_channels, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2_weight, - bias_attr=None, - data_format='NCHW') - self.bn2 = nn.BatchNorm(self.out_channels, - act=None, - param_attr=bn2_weight, - bias_attr=bn2_bias, - data_layout='NCHW') - self.conv3 = nn.Conv2D(in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=1, - stride=self.stride, - padding=0, - weight_attr=conv3_weight, - bias_attr=None, - data_format='NCHW') - self.bn3 = nn.BatchNorm(self.out_channels, - act=None, - param_attr=bn3_weight, - bias_attr=bn3_bias, - data_layout='NCHW') + self.conv1 = nn.Conv2D( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=3, + stride=self.stride, + padding=1, + weight_attr=conv1_weight, + bias_attr=None, + data_format='NCHW', + ) + self.bn1 = nn.BatchNorm( + self.out_channels, + act='relu', + param_attr=bn1_weight, + bias_attr=bn1_bias, + data_layout='NCHW', + ) + self.conv2 = nn.Conv2D( + in_channels=self.out_channels, + out_channels=self.out_channels, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2_weight, + bias_attr=None, + data_format='NCHW', + ) + self.bn2 = nn.BatchNorm( + self.out_channels, + act=None, + param_attr=bn2_weight, + bias_attr=bn2_bias, + data_layout='NCHW', + ) + self.conv3 = nn.Conv2D( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=1, + stride=self.stride, + padding=0, + weight_attr=conv3_weight, + bias_attr=None, + data_format='NCHW', + ) + self.bn3 = nn.BatchNorm( + self.out_channels, + act=None, + param_attr=bn3_weight, + bias_attr=bn3_bias, + data_layout='NCHW', + ) self.relu = nn.ReLU() tensor_src = paddle.to_tensor(self.src, stop_gradient=False) @@ -139,8 +162,9 @@ class XPUTestResNetBasicBlockOp(XPUOpTestWrapper): bn1_out = self.bn1(self.conv1(tensor_src)) bn2_out = self.bn2(self.conv2(bn1_out)) result = self.relu(bn2_out + z_out) - paddle.autograd.backward([result], [paddle.to_tensor(self.dout)], - True) + paddle.autograd.backward( + [result], [paddle.to_tensor(self.dout)], True + ) return result, tensor_src.grad def FusedResNetBasicBlock(self): @@ -148,25 +172,34 @@ class XPUTestResNetBasicBlockOp(XPUOpTestWrapper): fused_conv1_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) fused_conv2_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) fused_conv3_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) fused_bn1_weight = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0) + ) fused_bn1_bias = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0) + ) fused_bn2_weight = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0) + ) fused_bn2_bias = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0) + ) fused_bn3_weight = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0) + ) fused_bn3_bias = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0) + ) if self.has_shortcut: self.resnet_basic_block = ResNetBasicBlock( @@ -195,7 +228,8 @@ class XPUTestResNetBasicBlockOp(XPUOpTestWrapper): padding1=1, padding2=1, padding3=0, - has_shortcut=True) + has_shortcut=True, + ) else: self.resnet_basic_block = ResNetBasicBlock( num_channels1=self.in_channels, @@ -223,7 +257,8 @@ class XPUTestResNetBasicBlockOp(XPUOpTestWrapper): padding1=1, padding2=1, padding3=1, - has_shortcut=False) + has_shortcut=False, + ) x = paddle.to_tensor(self.src, stop_gradient=False) out = self.resnet_basic_block.forward(x) @@ -235,36 +270,46 @@ class XPUTestResNetBasicBlockOp(XPUOpTestWrapper): default_main_program().random_seed = 1 base_out, base_grad = self.Base() fused_out, fused_grad = self.FusedResNetBasicBlock() - np.testing.assert_allclose(base_out.numpy(), - fused_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(base_grad.numpy(), - fused_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + base_out.numpy(), + fused_out.numpy(), + rtol=self.rtol, + atol=self.atol, + ) + np.testing.assert_allclose( + base_grad.numpy(), + fused_grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) def test_out_and_grad(self): self.has_shortcut = False default_main_program().random_seed = 1 base_out, base_grad = self.Base() fused_out, fused_grad = self.FusedResNetBasicBlock() - np.testing.assert_allclose(base_out.numpy(), - fused_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(base_grad.numpy(), - fused_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + base_out.numpy(), + fused_out.numpy(), + rtol=self.rtol, + atol=self.atol, + ) + np.testing.assert_allclose( + base_grad.numpy(), + fused_grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) support_types = get_xpu_op_support_types('resnet_basic_block') for stype in support_types: - create_test_class(globals(), - XPUTestResNetBasicBlockOp, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestResNetBasicBlockOp, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gather_nd_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_gather_nd_op_xpu.py index 96e1ded83ae..ec4db1fd741 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gather_nd_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gather_nd_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestGatherNd(XPUOpTestWrapper): - def __init__(self): self.op_name = 'gather_nd' class XPUTestGatherNdBase(XPUOpTest): - def setUp(self): self.op_type = "gather_nd" self.dtype = self.in_type @@ -51,115 +53,104 @@ class XPUTestGatherNd(XPUOpTestWrapper): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([[], []]).astype("int32") self.output = np.vstack( - (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :])) + (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :]) + ) class XPUTestGatherNdOpWithEmptyIndex1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([[], []]).astype("int32") self.output = np.vstack( - (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :])) + (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :]) + ) class XPUTestGatherNdOpWithEmptyIndex2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([[], []]).astype("int64") self.output = np.vstack( - (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :])) + (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :]) + ) class XPUTestGatherNdOpWithIndex1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([1]).astype("int32") self.output = self.xnp[self.inp] class XPUTestGatherNdOpWithIndex2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([1]).astype("int64") self.output = self.xnp[self.inp] class XPUTestGatherNdOpWithLowIndex1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([[1], [2]]).astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithLowIndex2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([1, 2]).astype("int64") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithHighRankSame1(XPUTestGatherNdBase): - def init_data(self): shape = (5, 2, 3, 1, 10) self.xnp = np.random.rand(*shape).astype(self.in_type) - self.inp = np.vstack([ - np.random.randint(0, s, size=2) for s in shape - ]).T.astype("int32") + self.inp = np.vstack( + [np.random.randint(0, s, size=2) for s in shape] + ).T.astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithHighRankSame2(XPUTestGatherNdBase): - def init_data(self): shape = (5, 2, 3, 1, 10) self.xnp = np.random.rand(*shape).astype(self.in_type) - self.inp = np.vstack([ - np.random.randint(0, s, size=2) for s in shape - ]).T.astype("int64") + self.inp = np.vstack( + [np.random.randint(0, s, size=2) for s in shape] + ).T.astype("int64") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithHighRankDiff1(XPUTestGatherNdBase): - def init_data(self): shape = (2, 3, 4, 1, 10) self.xnp = np.random.rand(*shape).astype(self.in_type) - self.inp = np.vstack([ - np.random.randint(0, s, size=200) for s in shape - ]).T.astype("int32") + self.inp = np.vstack( + [np.random.randint(0, s, size=200) for s in shape] + ).T.astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithHighRankDiff2(XPUTestGatherNdBase): - def init_data(self): shape = (2, 3, 4, 1, 10) self.xnp = np.random.rand(*shape).astype(self.in_type) - self.inp = np.vstack([ - np.random.randint(0, s, size=200) for s in shape - ]).T.astype("int64") + self.inp = np.vstack( + [np.random.randint(0, s, size=200) for s in shape] + ).T.astype("int64") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithSameIndexAsX1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([[1, 1], [2, 1]]).astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithSameIndexAsX2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([[1, 1], [2, 1]]).astype("int64") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpIndex1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([1, 2]).astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpIndex2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([1, 2]).astype("int64") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py index 3a1cbdee77a..b1a776bd49b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py @@ -21,7 +21,11 @@ import numpy as np import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -34,12 +38,10 @@ def gather_numpy(x, index, axis): class XPUTestGather(XPUOpTestWrapper): - def __init__(self): self.op_name = 'gather' class TestXPUGatherOp(XPUOpTest): - def setUp(self): self.op_type = "gather" self.place = paddle.XPUPlace(0) @@ -49,7 +51,7 @@ class XPUTestGather(XPUOpTestWrapper): xnp = np.random.random(self.x_shape).astype(self.dtype) self.inputs = { 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) + 'Index': np.array(self.index).astype(self.index_type), } self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} @@ -67,28 +69,24 @@ class XPUTestGather(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class TestCase1(TestXPUGatherOp): - def init_config(self): - self.x_shape = (100) + self.x_shape = 100 self.index = [1, 3, 5] self.index_type = np.int32 class TestCase2(TestXPUGatherOp): - def init_config(self): - self.x_shape = (100) + self.x_shape = 100 self.index = [1, 3, 5] self.index_type = np.int64 class TestCase3(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.index = [1, 3, 5] self.index_type = np.int32 class TestCase4(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': False} @@ -96,7 +94,6 @@ class XPUTestGather(XPUOpTestWrapper): self.index_type = np.int32 class TestCase5(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': False} @@ -104,7 +101,6 @@ class XPUTestGather(XPUOpTestWrapper): self.index_type = np.int32 class TestCase6(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': True} @@ -112,7 +108,6 @@ class XPUTestGather(XPUOpTestWrapper): self.index_type = np.int32 class TestCase7(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': True} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gaussian_random_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_gaussian_random_op_xpu.py index d95f7b967d5..89725bb8b6d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gaussian_random_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gaussian_random_op_xpu.py @@ -20,20 +20,22 @@ import numpy as np import paddle import paddle.fluid as fluid from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestGaussianRandomOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'gaussian_random' self.use_dynamic_create_class = False class TestGaussianRandomOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -50,7 +52,7 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): "mean": self.mean, "std": self.std, "seed": 10, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } paddle.seed(10) @@ -58,11 +60,12 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def test_check_output(self): - self.check_output_with_place_customized(self.verify_output, - self.place) + self.check_output_with_place_customized( + self.verify_output, self.place + ) def verify_output(self, outs): self.assertEqual(outs[0].shape, (123, 92)) @@ -76,30 +79,28 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): np.testing.assert_allclose(hist, hist2, rtol=0, atol=0.01) class TestMeanStdAreInt(TestGaussianRandomOp): - def set_attrs(self): self.mean = 1 self.std = 2 # Situation 2: Attr(shape) is a list(with tensor) class TestGaussianRandomOp_ShapeTensorList(TestGaussianRandomOp): - def setUp(self): - '''Test gaussian_random op with specified value - ''' + '''Test gaussian_random op with specified value''' self.init() self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.attrs = { 'shape': self.infer_shape, 'mean': self.mean, 'std': self.std, 'seed': self.seed, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.inputs = {"ShapeTensorList": shape_tensor_list} @@ -114,12 +115,13 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): self.seed = 10 def test_check_output(self): - self.check_output_with_place_customized(self.verify_output, - self.place) + self.check_output_with_place_customized( + self.verify_output, self.place + ) class TestGaussianRandomOp2_ShapeTensorList( - TestGaussianRandomOp_ShapeTensorList): - + TestGaussianRandomOp_ShapeTensorList + ): def init_data(self): self.shape = [123, 92] self.infer_shape = [-1, -1] @@ -129,8 +131,8 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): self.seed = 10 class TestGaussianRandomOp3_ShapeTensorList( - TestGaussianRandomOp_ShapeTensorList): - + TestGaussianRandomOp_ShapeTensorList + ): def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -140,8 +142,8 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): self.seed = 10 class TestGaussianRandomOp4_ShapeTensorList( - TestGaussianRandomOp_ShapeTensorList): - + TestGaussianRandomOp_ShapeTensorList + ): def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -152,10 +154,8 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): # Situation 3: shape is a tensor class TestGaussianRandomOp1_ShapeTensor(TestGaussianRandomOp): - def setUp(self): - '''Test gaussian_random op with specified value - ''' + '''Test gaussian_random op with specified value''' self.init() self.init_data() self.use_mkldnn = False @@ -165,7 +165,7 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): 'mean': self.mean, 'std': self.std, 'seed': self.seed, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': np.zeros((123, 92), dtype=self.dtype)} @@ -179,54 +179,61 @@ class XPUTestGaussianRandomOp(XPUOpTestWrapper): # Test python API class TestGaussianRandomAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2000) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 500) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - - out_1 = fluid.layers.gaussian_random(shape=[2000, 500], - dtype="float32", - mean=0.0, - std=1.0, - seed=10) - - out_2 = fluid.layers.gaussian_random(shape=[2000, positive_2_int32], - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_3 = fluid.layers.gaussian_random(shape=[2000, positive_2_int64], - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_4 = fluid.layers.gaussian_random(shape=shape_tensor_int32, - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_5 = fluid.layers.gaussian_random(shape=shape_tensor_int64, - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_6 = fluid.layers.gaussian_random(shape=shape_tensor_int64, - dtype=np.float32, - mean=0., - std=1.0, - seed=10) + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + + out_1 = fluid.layers.gaussian_random( + shape=[2000, 500], dtype="float32", mean=0.0, std=1.0, seed=10 + ) + + out_2 = fluid.layers.gaussian_random( + shape=[2000, positive_2_int32], + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_3 = fluid.layers.gaussian_random( + shape=[2000, positive_2_int64], + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_4 = fluid.layers.gaussian_random( + shape=shape_tensor_int32, + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_5 = fluid.layers.gaussian_random( + shape=shape_tensor_int64, + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_6 = fluid.layers.gaussian_random( + shape=shape_tensor_int64, + dtype=np.float32, + mean=0.0, + std=1.0, + seed=10, + ) exe = fluid.Executor(place=fluid.XPUPlace(0)) res_1, res_2, res_3, res_4, res_5, res_6 = exe.run( @@ -235,20 +242,21 @@ class TestGaussianRandomAPI(unittest.TestCase): "shape_tensor_int32": np.array([2000, 500]).astype("int32"), "shape_tensor_int64": np.array([2000, 500]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6], + ) self.assertAlmostEqual(np.mean(res_1), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_1), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_1), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_2), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_2), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_2), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_3), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_3), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_3), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_4), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_5), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_5), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_5), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_5), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_5), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_6), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_6), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_6), 1.0, delta=0.1) def test_default_dtype(self): paddle.disable_static() @@ -276,7 +284,6 @@ class TestGaussianRandomAPI(unittest.TestCase): class TestStandardNormalDtype(unittest.TestCase): - def test_default_dtype(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gen_bkcl_id_op.py b/python/paddle/fluid/tests/unittests/xpu/test_gen_bkcl_id_op.py index e5ac263fb7d..ebecc1b8b71 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gen_bkcl_id_op.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gen_bkcl_id_op.py @@ -37,30 +37,35 @@ def run_gen_bkc_id(attr): with paddle.static.program_guard(main_program, startup_program): bkcl_id_var = startup_program.global_block().create_var( - name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) for i in range(1, bkcl_comm_num): startup_program.global_block().create_var( name="BKCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) if use_hallreduce: for i in range(0, bkcl_comm_num): startup_program.global_block().create_var( name="Hierarchical_inter_BKCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().create_var( name="Hierarchical_exter_BKCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_bkcl_id", inputs={}, outputs={"BKCLID": bkcl_id_var}, - attrs=attr) + attrs=attr, + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -68,7 +73,6 @@ def run_gen_bkc_id(attr): class TestGenBKCLIdOp(unittest.TestCase): - def setUp(self): try: self._dist_ut_port_0 = int(os.environ["PADDLE_DIST_UT_PORT"]) @@ -101,7 +105,7 @@ class TestGenBKCLIdOp(unittest.TestCase): for i in range(nranks): attr['trainer_id'] = i # NOTE: multiprocessing cannot be covered by coverage - p = Process(target=run_gen_bkc_id, args=(attr, )) + p = Process(target=run_gen_bkc_id, args=(attr,)) p.start() procs.append(p) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_generate_proposals_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_generate_proposals_v2_op_xpu.py index d9c5f90e9ee..d087e8bf796 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_generate_proposals_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_generate_proposals_v2_op_xpu.py @@ -24,7 +24,11 @@ import paddle.fluid.core as core import copy from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -49,30 +53,48 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): pred_bbox = np.zeros_like(bbox_deltas, dtype=np.float32) if variances is not None: for i in range(bbox_deltas.shape[0]): - pred_bbox[i, 0] = variances[i, 0] * bbox_deltas[i, 0] * anchor_loc[ - i, 0] + anchor_loc[i, 2] - pred_bbox[i, 1] = variances[i, 1] * bbox_deltas[i, 1] * anchor_loc[ - i, 1] + anchor_loc[i, 3] - pred_bbox[i, 2] = math.exp( - min(variances[i, 2] * bbox_deltas[i, 2], math.log( - 1000 / 16.0))) * anchor_loc[i, 0] - pred_bbox[i, 3] = math.exp( - min(variances[i, 3] * bbox_deltas[i, 3], math.log( - 1000 / 16.0))) * anchor_loc[i, 1] + pred_bbox[i, 0] = ( + variances[i, 0] * bbox_deltas[i, 0] * anchor_loc[i, 0] + + anchor_loc[i, 2] + ) + pred_bbox[i, 1] = ( + variances[i, 1] * bbox_deltas[i, 1] * anchor_loc[i, 1] + + anchor_loc[i, 3] + ) + pred_bbox[i, 2] = ( + math.exp( + min( + variances[i, 2] * bbox_deltas[i, 2], + math.log(1000 / 16.0), + ) + ) + * anchor_loc[i, 0] + ) + pred_bbox[i, 3] = ( + math.exp( + min( + variances[i, 3] * bbox_deltas[i, 3], + math.log(1000 / 16.0), + ) + ) + * anchor_loc[i, 1] + ) else: for i in range(bbox_deltas.shape[0]): - pred_bbox[i, - 0] = bbox_deltas[i, 0] * anchor_loc[i, 0] + anchor_loc[i, - 2] - pred_bbox[i, - 1] = bbox_deltas[i, 1] * anchor_loc[i, 1] + anchor_loc[i, - 3] - pred_bbox[i, 2] = math.exp( - min(bbox_deltas[i, 2], math.log(1000 / 16.0))) * anchor_loc[i, - 0] - pred_bbox[i, 3] = math.exp( - min(bbox_deltas[i, 3], math.log(1000 / 16.0))) * anchor_loc[i, - 1] + pred_bbox[i, 0] = ( + bbox_deltas[i, 0] * anchor_loc[i, 0] + anchor_loc[i, 2] + ) + pred_bbox[i, 1] = ( + bbox_deltas[i, 1] * anchor_loc[i, 1] + anchor_loc[i, 3] + ) + pred_bbox[i, 2] = ( + math.exp(min(bbox_deltas[i, 2], math.log(1000 / 16.0))) + * anchor_loc[i, 0] + ) + pred_bbox[i, 3] = ( + math.exp(min(bbox_deltas[i, 3], math.log(1000 / 16.0))) + * anchor_loc[i, 1] + ) proposals[:, 0] = pred_bbox[:, 0] - pred_bbox[:, 2] / 2 proposals[:, 1] = pred_bbox[:, 1] - pred_bbox[:, 3] / 2 proposals[:, 2] = pred_bbox[:, 0] + pred_bbox[:, 2] / 2 - offset @@ -84,43 +106,47 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): def clip_tiled_boxes(boxes, im_shape, pixel_offset=True): """Clip boxes to image boundaries. im_shape is [height, width] and boxes has shape (N, 4 * num_tiled_boxes).""" - assert boxes.shape[1] % 4 == 0, \ - 'boxes.shape[1] is {:d}, but must be divisible by 4.'.format( + assert ( + boxes.shape[1] % 4 == 0 + ), 'boxes.shape[1] is {:d}, but must be divisible by 4.'.format( boxes.shape[1] ) offset = 1 if pixel_offset else 0 # x1 >= 0 - boxes[:, - 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - offset), - 0) + boxes[:, 0::4] = np.maximum( + np.minimum(boxes[:, 0::4], im_shape[1] - offset), 0 + ) # y1 >= 0 - boxes[:, - 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - offset), - 0) + boxes[:, 1::4] = np.maximum( + np.minimum(boxes[:, 1::4], im_shape[0] - offset), 0 + ) # x2 < im_shape[1] - boxes[:, - 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - offset), - 0) + boxes[:, 2::4] = np.maximum( + np.minimum(boxes[:, 2::4], im_shape[1] - offset), 0 + ) # y2 < im_shape[0] - boxes[:, - 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - offset), - 0) + boxes[:, 3::4] = np.maximum( + np.minimum(boxes[:, 3::4], im_shape[0] - offset), 0 + ) return boxes def filter_boxes(boxes, min_size, im_shape, pixel_offset=True): - """Only keep boxes with both sides >= min_size and center within the image. - """ + """Only keep boxes with both sides >= min_size and center within the image.""" # Scale min_size to match image scale min_size = max(min_size, 1.0) offset = 1 if pixel_offset else 0 ws = boxes[:, 2] - boxes[:, 0] + offset hs = boxes[:, 3] - boxes[:, 1] + offset if pixel_offset: - x_ctr = boxes[:, 0] + ws / 2. - y_ctr = boxes[:, 1] + hs / 2. - keep = np.where((ws >= min_size) & (hs >= min_size) - & (x_ctr < im_shape[1]) & (y_ctr < im_shape[0]))[0] + x_ctr = boxes[:, 0] + ws / 2.0 + y_ctr = boxes[:, 1] + hs / 2.0 + keep = np.where( + (ws >= min_size) + & (hs >= min_size) + & (x_ctr < im_shape[1]) + & (y_ctr < im_shape[0]) + )[0] else: keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep @@ -182,9 +208,9 @@ def nms(boxes, scores, nms_threshold, eta=1.0, pixel_offset=True): for k in range(len(selected_indices)): if keep: kept_idx = selected_indices[k] - overlap = iou(boxes[idx], - boxes[kept_idx], - pixel_offset=pixel_offset) + overlap = iou( + boxes[idx], boxes[kept_idx], pixel_offset=pixel_offset + ) keep = True if overlap <= adaptive_threshold else False else: break @@ -195,9 +221,19 @@ def nms(boxes, scores, nms_threshold, eta=1.0, pixel_offset=True): return selected_indices -def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, - scores, pre_nms_topN, post_nms_topN, nms_thresh, - min_size, eta, pixel_offset): +def proposal_for_one_image( + im_shape, + all_anchors, + variances, + bbox_deltas, + scores, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, +): # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # - bbox deltas will be (4 * A, H, W) format from conv output @@ -250,11 +286,13 @@ def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, # take post_nms_topN (e.g. 1000) # return the top proposals if nms_thresh > 0: - keep = nms(boxes=proposals, - scores=scores, - nms_threshold=nms_thresh, - eta=eta, - pixel_offset=pixel_offset) + keep = nms( + boxes=proposals, + scores=scores, + nms_threshold=nms_thresh, + eta=eta, + pixel_offset=pixel_offset, + ) if post_nms_topN > 0 and post_nms_topN < len(keep): keep = keep[:post_nms_topN] proposals = proposals[keep, :] @@ -263,9 +301,19 @@ def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, return proposals, scores -def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, - variances, pre_nms_topN, post_nms_topN, - nms_thresh, min_size, eta, pixel_offset): +def generate_proposals_v2_in_python( + scores, + bbox_deltas, + im_shape, + anchors, + variances, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, +): all_anchors = anchors.reshape(-1, 4) rois = np.empty((0, 5), dtype=np.float32) roi_probs = np.empty((0, 1), dtype=np.float32) @@ -276,10 +324,18 @@ def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, num_images = scores.shape[0] for img_idx in range(num_images): img_i_boxes, img_i_probs = proposal_for_one_image( - im_shape[img_idx, :], all_anchors, variances, - bbox_deltas[img_idx, :, :, :], scores[img_idx, :, :, :], - pre_nms_topN, post_nms_topN, nms_thresh, min_size, eta, - pixel_offset) + im_shape[img_idx, :], + all_anchors, + variances, + bbox_deltas[img_idx, :, :, :], + scores[img_idx, :, :, :], + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, + ) rois_num.append(img_i_probs.shape[0]) rpn_rois.append(img_i_boxes) rpn_roi_probs.append(img_i_probs) @@ -287,8 +343,9 @@ def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, return rpn_rois, rpn_roi_probs, rois_num -def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, - variances, stride, offset): +def anchor_generator_in_python( + input_feat, anchor_sizes, aspect_ratios, variances, stride, offset +): num_anchors = len(aspect_ratios) * len(anchor_sizes) layer_h = input_feat.shape[2] layer_w = input_feat.shape[3] @@ -312,11 +369,12 @@ def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, scale_h = anchor_size / stride[1] w = scale_w * base_w h = scale_h * base_h - out_anchors[h_idx, w_idx, - idx, :] = [(x_ctr - 0.5 * (w - 1)), - (y_ctr - 0.5 * (h - 1)), - (x_ctr + 0.5 * (w - 1)), - (y_ctr + 0.5 * (h - 1))] + out_anchors[h_idx, w_idx, idx, :] = [ + (x_ctr - 0.5 * (w - 1)), + (y_ctr - 0.5 * (h - 1)), + (x_ctr + 0.5 * (w - 1)), + (y_ctr + 0.5 * (h - 1)), + ] idx += 1 # set the variance. @@ -327,13 +385,11 @@ def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, class XPUGenerateProposalsV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'generate_proposals_v2' self.use_dynamic_create_class = False class TestGenerateProposalsV2Op(XPUOpTest): - def set_data(self): self.init_input_shape() self.init_test_params() @@ -344,7 +400,7 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): 'BboxDeltas': self.bbox_deltas, 'ImShape': self.im_shape.astype(self.dtype), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -388,7 +444,7 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 3.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = True def init_test_input(self): @@ -396,30 +452,45 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): input_channels = self.input_feat_shape[1] layer_h = self.input_feat_shape[2] layer_w = self.input_feat_shape[3] - input_feat = np.random.random((batch_size, input_channels, layer_h, - layer_w)).astype(self.dtype) + input_feat = np.random.random( + (batch_size, input_channels, layer_h, layer_w) + ).astype(self.dtype) self.anchors, self.variances = anchor_generator_in_python( input_feat=input_feat, - anchor_sizes=[16., 32.], + anchor_sizes=[16.0, 32.0], aspect_ratios=[0.5, 1.0], variances=[1.0, 1.0, 1.0, 1.0], stride=[16.0, 16.0], - offset=0.5) + offset=0.5, + ) num_anchors = self.anchors.shape[2] self.scores = np.random.random( - (batch_size, num_anchors, layer_h, layer_w)).astype(self.dtype) + (batch_size, num_anchors, layer_h, layer_w) + ).astype(self.dtype) self.bbox_deltas = np.random.random( - (batch_size, num_anchors * 4, layer_h, - layer_w)).astype(self.dtype) + (batch_size, num_anchors * 4, layer_h, layer_w) + ).astype(self.dtype) def init_test_output(self): - self.rpn_rois, self.rpn_roi_probs, self.rois_num = generate_proposals_v2_in_python( - self.scores, self.bbox_deltas, self.im_shape, self.anchors, - self.variances, self.pre_nms_topN, self.post_nms_topN, - self.nms_thresh, self.min_size, self.eta, self.pixel_offset) + ( + self.rpn_rois, + self.rpn_roi_probs, + self.rois_num, + ) = generate_proposals_v2_in_python( + self.scores, + self.bbox_deltas, + self.im_shape, + self.anchors, + self.variances, + self.pre_nms_topN, + self.post_nms_topN, + self.nms_thresh, + self.min_size, + self.eta, + self.pixel_offset, + ) class TestGenerateProposalsV2OutLodOp(TestGenerateProposalsV2Op): - def set_data(self): self.init_input_shape() self.init_test_params() @@ -430,7 +501,7 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): 'BboxDeltas': self.bbox_deltas, 'ImShape': self.im_shape.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -440,37 +511,34 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): 'min_size': self.min_size, 'eta': self.eta, 'pixel_offset': self.pixel_offset, - 'return_rois_num': True + 'return_rois_num': True, } self.outputs = { 'RpnRois': (self.rpn_rois[0], [self.rois_num]), 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.rois_num]), - 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)) + 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)), } class TestGenerateProposalsV2OpNoBoxLeft(TestGenerateProposalsV2Op): - def init_test_params(self): self.pre_nms_topN = 12000 # train 12000, test 2000 self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 1000.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = True class TestGenerateProposalsV2OpNoOffset(TestGenerateProposalsV2Op): - def init_test_params(self): self.pre_nms_topN = 12000 # train 12000, test 2000 self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 3.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = False class TestGenerateProposalsV2OpMaskRcnn1XPU(TestGenerateProposalsV2Op): - def init_input_shape(self): self.input_feat_shape = (1, 20, 48, 64) # Another case is [768, 1024] @@ -481,7 +549,7 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): self.post_nms_topN = 2000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 0.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = False def init_test_input(self): @@ -489,21 +557,24 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): input_channels = self.input_feat_shape[1] layer_h = self.input_feat_shape[2] layer_w = self.input_feat_shape[3] - input_feat = np.random.random((batch_size, input_channels, layer_h, - layer_w)).astype(self.dtype) + input_feat = np.random.random( + (batch_size, input_channels, layer_h, layer_w) + ).astype(self.dtype) self.anchors, self.variances = anchor_generator_in_python( input_feat=input_feat, anchor_sizes=[32, 64, 128, 256, 512], aspect_ratios=[0.5, 1.0, 2.0], variances=[1.0, 1.0, 1.0, 1.0], stride=[16.0, 16.0], - offset=0.5) + offset=0.5, + ) num_anchors = self.anchors.shape[2] self.scores = np.random.random( - (batch_size, num_anchors, layer_h, layer_w)).astype(self.dtype) + (batch_size, num_anchors, layer_h, layer_w) + ).astype(self.dtype) self.bbox_deltas = np.random.random( - (batch_size, num_anchors * 4, layer_h, - layer_w)).astype(self.dtype) + (batch_size, num_anchors * 4, layer_h, layer_w) + ).astype(self.dtype) self.anchors = self.anchors.reshape(-1, 4) self.variances = self.variances.reshape(-1, 4) @@ -519,7 +590,7 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): 'BboxDeltas': self.bbox_deltas, 'ImShape': self.im_shape.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -529,23 +600,25 @@ class XPUGenerateProposalsV2Op(XPUOpTestWrapper): 'min_size': self.min_size, 'eta': self.eta, 'pixel_offset': self.pixel_offset, - 'return_rois_num': True + 'return_rois_num': True, } self.outputs = { 'RpnRois': (self.rpn_rois[0], [self.rois_num]), 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.rois_num]), - 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)) + 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)), } support_types = get_xpu_op_support_types('generate_proposals_v2') for stype in support_types: - create_test_class(globals(), - XPUGenerateProposalsV2Op, - stype, - test_grad=False, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUGenerateProposalsV2Op, + stype, + test_grad=False, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_grid_sampler_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_grid_sampler_op_xpu.py index fbd5c68ddd4..eb992cd4213 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_grid_sampler_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_grid_sampler_op_xpu.py @@ -21,7 +21,11 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -30,12 +34,15 @@ def AffineGrid(theta, grid_shape): n = grid_shape[0] h = grid_shape[1] w = grid_shape[2] - h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, - axis=0).T[:, :, np.newaxis] - w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, - axis=0)[:, :, np.newaxis] - grid = np.concatenate([w_idx, h_idx, np.ones([h, w, 1])], - axis=2) # h * w * 3 + h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, axis=0).T[ + :, :, np.newaxis + ] + w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, axis=0)[ + :, :, np.newaxis + ] + grid = np.concatenate( + [w_idx, h_idx, np.ones([h, w, 1])], axis=2 + ) # h * w * 3 grid = np.repeat(grid[np.newaxis, :], n, axis=0) # n * h * w *3 ret = np.zeros([n, h * w, 2]) @@ -55,13 +62,17 @@ def getGridPointValue(data, x, y): out_H = x.shape[1] out_W = x.shape[2] - #out = np.zeros(data_shape, dtype='float64') + # out = np.zeros(data_shape, dtype='float64') out = np.zeros([N, C, out_H, out_W], dtype='float64') for i in range(N): for j in range(out_H): for k in range(out_W): - if y[i, j, k] < 0 or y[i, j, k] > in_H - 1 or x[ - i, j, k] < 0 or x[i, j, k] > in_W - 1: + if ( + y[i, j, k] < 0 + or y[i, j, k] > in_H - 1 + or x[i, j, k] < 0 + or x[i, j, k] > in_W - 1 + ): out[i, :, j, k] = 0 else: out[i, :, j, k] = data[i, :, y[i, j, k], x[i, j, k]] @@ -77,27 +88,28 @@ def unnormalizeAndClip(grid_slice, max_val, align_corners, padding_mode): if align_corners: grid_slice = 0.5 * ((grid_slice.astype('float64') + 1.0) * max_val) else: - grid_slice = 0.5 * ((grid_slice.astype('float64') + 1.0) * - (max_val + 1)) - 0.5 + grid_slice = ( + 0.5 * ((grid_slice.astype('float64') + 1.0) * (max_val + 1)) - 0.5 + ) if padding_mode == "border": grid_slice = clip(grid_slice, 0, max_val) elif padding_mode == "reflection": double_range = 2 * max_val if align_corners else (max_val + 1) * 2 - grid_abs = np.abs(grid_slice) if align_corners else np.abs(grid_slice + - 0.5) + grid_abs = ( + np.abs(grid_slice) if align_corners else np.abs(grid_slice + 0.5) + ) extra = grid_abs - np.floor(grid_abs / double_range) * double_range grid_slice = np.minimum(extra, double_range - extra) - grid_slice = grid_slice if align_corners else clip( - grid_slice - 0.5, 0, max_val) + grid_slice = ( + grid_slice if align_corners else clip(grid_slice - 0.5, 0, max_val) + ) return grid_slice -def GridSampler(data, - grid, - align_corners=True, - mode="bilinear", - padding_mode="zeros"): +def GridSampler( + data, grid, align_corners=True, mode="bilinear", padding_mode="zeros" +): dims = data.shape N = dims[0] in_C = dims[1] @@ -121,14 +133,18 @@ def GridSampler(data, y0 = np.floor(y).astype('int32') y1 = y0 + 1 - wa = np.tile(((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wb = np.tile(((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wc = np.tile(((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wd = np.tile(((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) + wa = np.tile( + ((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wb = np.tile( + ((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wc = np.tile( + ((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wd = np.tile( + ((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) va = getGridPointValue(data, x0, y0) vb = getGridPointValue(data, x0, y1) @@ -144,13 +160,11 @@ def GridSampler(data, class XPUTestGridSamplerOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'grid_sampler' self.use_dynamic_create_class = False class TestXPUGridSamplerOp(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -180,9 +194,9 @@ class XPUTestGridSamplerOP(XPUOpTestWrapper): "mode": self.mode, } self.outputs = { - 'Output': - GridSampler(x, grid, self.align_corners, self.mode, - self.padding_mode) + 'Output': GridSampler( + x, grid, self.align_corners, self.mode, self.padding_mode + ) } def initTestCase(self): @@ -203,7 +217,6 @@ class XPUTestGridSamplerOP(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X', 'Grid'], 'Output') class TestGridSample1(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -213,7 +226,6 @@ class XPUTestGridSamplerOP(XPUOpTestWrapper): self.mode = "bilinear" class TestGridSample2(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -223,7 +235,6 @@ class XPUTestGridSamplerOP(XPUOpTestWrapper): self.mode = "bilinear" class TestGridSample3(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -233,7 +244,6 @@ class XPUTestGridSamplerOP(XPUOpTestWrapper): self.mode = "bilinear" class TestGridSample4(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -243,7 +253,6 @@ class XPUTestGridSamplerOP(XPUOpTestWrapper): self.mode = "bilinear" class TestGridSample5(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -253,7 +262,6 @@ class XPUTestGridSamplerOP(XPUOpTestWrapper): self.mode = "nearest" class TestGridSample6(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 128, 128) self.grid_shape = (2, 130, 130, 2) @@ -263,7 +271,6 @@ class XPUTestGridSamplerOP(XPUOpTestWrapper): self.mode = "bilinear" class TestGridSample7(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 128, 128) self.grid_shape = (2, 130, 130, 2) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_huber_loss_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_huber_loss_op_xpu.py index 1e7361035df..50d77fc1a3d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_huber_loss_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_huber_loss_op_xpu.py @@ -22,7 +22,11 @@ import paddle from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -36,13 +40,11 @@ def huber_loss_forward(val, delta): class XPUTestHuberLossOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'huber_loss' self.use_dynamic_create_class = False class TestHuberLossOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = 'huber_loss' @@ -55,11 +57,11 @@ class XPUTestHuberLossOp(XPUOpTestWrapper): def set_inputs(self): shape = self.set_shape() - x = np.random.uniform(0, 1., shape).astype(self.dtype) - y = np.random.uniform(0, 1., shape).astype(self.dtype) + x = np.random.uniform(0, 1.0, shape).astype(self.dtype) + y = np.random.uniform(0, 1.0, shape).astype(self.dtype) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } def set_attrs(self): @@ -69,8 +71,9 @@ class XPUTestHuberLossOp(XPUOpTestWrapper): delta = self.attrs['delta'] shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - delta).astype(self.dtype) + loss = np.vectorize(huber_loss_forward)(residual, delta).astype( + self.dtype + ) self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def set_shape(self): @@ -89,27 +92,24 @@ class XPUTestHuberLossOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("residual")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("residual") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('residual')) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set('residual') + ) class TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (640) + return 640 class TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (10, 10) class TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (10, 10, 1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_index_select_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_index_select_op_xpu.py index 3eb8ff49f7f..816e7ac7967 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_index_select_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_index_select_op_xpu.py @@ -23,33 +23,34 @@ import numpy as np import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestIndexSelect(XPUOpTestWrapper): - def __init__(self): self.op_name = 'index_select' class TestXPUIndexSelectOp(XPUOpTest): - def setUp(self): self.op_type = "index_select" self.place = paddle.XPUPlace(0) self.dtype = self.in_type self.init_dtype_type() - index_np = np.random.randint(low=0, - high=self.x_shape[self.dim], - size=self.index_size).astype( - self.index_type) + index_np = np.random.randint( + low=0, high=self.x_shape[self.dim], size=self.index_size + ).astype(self.index_type) x_np = np.random.random(self.x_shape).astype(self.dtype) self.inputs = {'X': x_np, 'Index': index_np} self.attrs = {'dim': self.dim} - outer_loop = np.prod(self.x_shape[:self.dim]) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + outer_loop = np.prod(self.x_shape[: self.dim]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] for i in range(outer_loop): @@ -77,7 +78,6 @@ class XPUTestIndexSelect(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class TestXPUIndexSelectOpCase2(TestXPUIndexSelectOp): - def init_dtype_type(self): self.index_type = np.int32 self.dim = -2 @@ -86,10 +86,14 @@ class XPUTestIndexSelect(XPUOpTestWrapper): class TestIndexSelectAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]]) + self.data_x = np.array( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] + ) self.data_index = np.array([0, 1, 1]).astype('int32') def test_index_select_api(self): @@ -98,39 +102,37 @@ class TestIndexSelectAPI(unittest.TestCase): # case 1: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='index', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='index', shape=[3], dtype='int32', append_batch_size=False + ) z = paddle.index_select(x, index, axis=1) exe = fluid.Executor(fluid.XPUPlace(0)) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 2: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='index', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='index', shape=[3], dtype='int32', append_batch_size=False + ) z = paddle.index_select(x, index) exe = fluid.Executor(fluid.XPUPlace(0)) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) def test_dygraph_api(self): @@ -141,8 +143,9 @@ class TestIndexSelectAPI(unittest.TestCase): index = fluid.dygraph.to_variable(self.data_index) z = paddle.index_select(x, index) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) # case 2: @@ -151,8 +154,9 @@ class TestIndexSelectAPI(unittest.TestCase): index = fluid.dygraph.to_variable(self.data_index) z = paddle.index_select(x, index, axis=1) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_instance_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_instance_norm_op_xpu.py index 55d942e0d72..8fc1ca75240 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_instance_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_instance_norm_op_xpu.py @@ -19,7 +19,11 @@ import unittest sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -53,13 +57,11 @@ def _cal_mean_variance(x, epsilon, mean_shape): class XPUTestInstanceNormOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'instance_norm' self.use_dynamic_create_class = False class XPUTestInstanceNormOp(XPUOpTest): - def setUp(self): self.op_type = "instance_norm" self.dtype = self.in_type @@ -79,8 +81,13 @@ class XPUTestInstanceNormOp(XPUOpTestWrapper): bias_np = np.random.random_sample(scale_shape).astype(np.float32) mean, variance = self.set_global_mean_var(mean_shape, x_np) - ref_y_np, ref_saved_mean, variance_tmp = _reference_instance_norm_naive( - x_np, scale_np, bias_np, epsilon, mean, variance) + ( + ref_y_np, + ref_saved_mean, + variance_tmp, + ) = _reference_instance_norm_naive( + x_np, scale_np, bias_np, epsilon, mean, variance + ) ref_saved_variance = 1 / np.sqrt(variance_tmp + epsilon) @@ -88,7 +95,7 @@ class XPUTestInstanceNormOp(XPUOpTestWrapper): self.outputs = { 'Y': ref_y_np, 'SavedMean': ref_saved_mean, - 'SavedVariance': ref_saved_variance + 'SavedVariance': ref_saved_variance, } self.attrs = {'epsilon': epsilon, 'use_xpu': True} @@ -106,27 +113,22 @@ class XPUTestInstanceNormOp(XPUOpTestWrapper): self.check_grad_with_place(paddle.XPUPlace(0), ['X'], 'Y') class TestXPUInstanceNormOp1(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [10, 12, 32, 32] class TestXPUInstanceNormOp2(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [4, 5, 6, 7] class TestXPUInstanceNormOp3(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [1, 8, 16, 16] class TestXPUInstanceNormOp4(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [4, 16, 256, 128] class TestXPUInstanceNormOp5(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [10, 3, 512, 1] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_iou_similarity_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_iou_similarity_op_xpu.py index 2aac4dc83fe..548976ae62e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_iou_similarity_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_iou_similarity_op_xpu.py @@ -18,20 +18,22 @@ import sys sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestIOUSimilarityOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'iou_similarity' self.use_dynamic_create_class = False class TestXPUIOUSimilarityOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -51,11 +53,13 @@ class XPUTestIOUSimilarityOp(XPUOpTestWrapper): self.inputs = {'X': self.boxes1, 'Y': self.boxes2} self.attrs = { "box_normalized": self.box_normalized, - 'use_xpu': True + 'use_xpu': True, } self.outputs = {'Out': self.output} - def _compute_iou(self, ): + def _compute_iou( + self, + ): for row in range(self.boxes1.shape[0]): for col in range(self.boxes2.shape[0]): xmin1, ymin1, xmax1, ymax1 = self.boxes1[row] @@ -84,7 +88,6 @@ class XPUTestIOUSimilarityOp(XPUOpTestWrapper): self.output[row, col] = sim_score class TestXPUIOUSimilarityOpWithLoD(TestXPUIOUSimilarityOp): - def test_check_output(self): self.check_output_with_place(self.place, check_dygraph=False) @@ -97,13 +100,12 @@ class XPUTestIOUSimilarityOp(XPUOpTestWrapper): self._compute_iou() self.inputs = { 'X': (self.boxes1, self.boxes1_lod), - 'Y': self.boxes2 + 'Y': self.boxes2, } self.attrs = {"box_normalized": self.box_normalized} self.outputs = {'Out': (self.output, self.output_lod)} class TestXPUIOUSimilarityOpWithBoxNormalized(TestXPUIOUSimilarityOp): - def test_check_output(self): self.check_output_with_place(self.place, check_dygraph=False) @@ -116,7 +118,7 @@ class XPUTestIOUSimilarityOp(XPUOpTestWrapper): self._compute_iou() self.inputs = { 'X': (self.boxes1, self.boxes1_lod), - 'Y': self.boxes2 + 'Y': self.boxes2, } self.attrs = {"box_normalized": self.box_normalized} self.outputs = {'Out': (self.output, self.output_lod)} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_label_smooth_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_label_smooth_op_xpu.py index 3384a0b0938..0a2c7b6b773 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_label_smooth_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_label_smooth_op_xpu.py @@ -19,13 +19,16 @@ import sys sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestLabelSmoothOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'label_smooth' self.use_dynamic_create_class = True @@ -37,15 +40,15 @@ class XPUTestLabelSmoothOp(XPUOpTestWrapper): label_dims = [1, 7, 12] for bs in batch_sizes: for label_dim in label_dims: - class_name = 'XPUTestLabelSmooth_' + \ - str(bs) + "_" + str(label_dim) + class_name = ( + 'XPUTestLabelSmooth_' + str(bs) + "_" + str(label_dim) + ) attr_dict = {'batch_size': bs, 'label_dim': label_dim} classes.append([class_name, attr_dict]) classes.append(['XPUTestLabelSmooth_3d', {'is_3d': True}]) return base_class, classes class TestLabelSmoothOp(XPUOpTest): - def setUp(self): self.op_type = "label_smooth" self.epsilon = 0.1 @@ -53,21 +56,26 @@ class XPUTestLabelSmoothOp(XPUOpTestWrapper): if not hasattr(self, 'batch_size'): self.batch_size = 10 self.label_dim = 12 - self.label = np.zeros( - (self.batch_size, self.label_dim)).astype("float32") - nonzero_index = np.random.randint(self.label_dim, - size=(self.batch_size)) + self.label = np.zeros((self.batch_size, self.label_dim)).astype( + "float32" + ) + nonzero_index = np.random.randint( + self.label_dim, size=(self.batch_size) + ) self.label[np.arange(self.batch_size), nonzero_index] = 1 smoothed_label = ( - 1 - self.epsilon) * self.label + self.epsilon / self.label_dim + 1 - self.epsilon + ) * self.label + self.epsilon / self.label_dim self.inputs = {'X': self.label} self.attrs = {'epsilon': self.epsilon} self.outputs = {'Out': smoothed_label} if hasattr(self, 'is_3d') and self.is_3d: self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) def test_check_output(self): if not paddle.is_compiled_with_xpu(): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_lamb_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_lamb_op_xpu.py index b86dae39426..0c5354c9575 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_lamb_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_lamb_op_xpu.py @@ -20,7 +20,11 @@ import numpy as np from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def lamb_step(inputs, attributes): @@ -51,14 +55,16 @@ def lamb_step(inputs, attributes): moment2_unbiased = moment2_out / (1 - beta2_pow) r_1 = np.linalg.norm(param) - r_2 = np.linalg.norm(moment1_unbiased / - (np.sqrt(moment2_unbiased) + epsilon) + - weight_decay * param) + r_2 = np.linalg.norm( + moment1_unbiased / (np.sqrt(moment2_unbiased) + epsilon) + + weight_decay * param + ) lr_t = lr * r_1 / r_2 - param_out = param - lr_t * (moment1_unbiased / - (np.sqrt(moment2_unbiased) + epsilon) + - weight_decay * param) + param_out = param - lr_t * ( + moment1_unbiased / (np.sqrt(moment2_unbiased) + epsilon) + + weight_decay * param + ) beta1_pow_out = beta1_pow * beta1 beta2_pow_out = beta2_pow * beta2 @@ -67,24 +73,21 @@ def lamb_step(inputs, attributes): class XPUTestLambOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'lamb' self.use_dynamic_create_class = False class TestLambOp1(XPUOpTest): - def set_attrs(self): self.attrs = { 'epsilon': 1e-4, 'beta1': 0.78, 'beta2': 0.836, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } def setUp(self): - '''Test Lamb Op with supplied attributes - ''' + '''Test Lamb Op with supplied attributes''' # self.op_type = self.op_name self.__class__.op_type = 'lamb' self.dtype = self.in_type @@ -105,56 +108,63 @@ class XPUTestLambOp(XPUOpTestWrapper): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } - - param_out, moment1_out, moment2_out, \ - beta1_pow_out, beta2_pow_out = lamb_step(self.inputs, self.attrs) + ( + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + ) = lamb_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } def test_check_output(self): self.check_output_with_place(paddle.XPUPlace(0)) class TestLambOp2(TestLambOp1): - def set_attrs(self): self.attrs = { 'epsilon': 1e-8, 'beta1': 0.9, 'beta2': 0.999, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } class TestLambOpMultipleSteps(TestLambOp1): - def set_attrs(self): self.attrs = { 'epsilon': 1e-8, 'beta1': 0.9, 'beta2': 0.999, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } self.num_steps = 10 def test_check_output(self): for i in range(self.num_steps): - param_out, moment1_out, moment2_out, \ - beta1_pow_out, beta2_pow_out = lamb_step(self.inputs, self.attrs) + ( + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + ) = lamb_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } # Verify output for this step @@ -171,7 +181,8 @@ class XPUTestLambOp(XPUOpTestWrapper): # Randomize gradient for next step self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + -1, 1, (102, 105) + ).astype("float32") support_types = get_xpu_op_support_types('lamb') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_layer_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_layer_norm_op_xpu.py index db0ac518932..3c377db4c0b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_layer_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_layer_norm_op_xpu.py @@ -21,7 +21,11 @@ from functools import reduce sys.path.append("..") from op_test_xpu import XPUOpTest from operator import mul -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -29,12 +33,13 @@ paddle.enable_static() def ref_layer_norm(x, scale, bias, epsilon, begin_norm_axis=1): x_shape = x.shape left = reduce(mul, x_shape[0:begin_norm_axis], 1) - right = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + right = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) x.shape = [left, right] mean = np.mean(x, axis=1) variance = np.var(x, axis=1) + epsilon - y = np.divide((x - mean.reshape([left, 1])), - (np.sqrt(variance)).reshape([left, 1])) + y = np.divide( + (x - mean.reshape([left, 1])), (np.sqrt(variance)).reshape([left, 1]) + ) if scale is not None: y = scale.reshape([1, right]) * y if bias is not None: @@ -44,13 +49,11 @@ def ref_layer_norm(x, scale, bias, epsilon, begin_norm_axis=1): class XPUTestLayerNormOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'layer_norm' self.use_dynamic_create_class = False class TestXPULayerNormOp(XPUOpTest): - def setUp(self): self.op_type = "layer_norm" self.dtype = self.in_type @@ -63,25 +66,27 @@ class XPUTestLayerNormOp(XPUOpTestWrapper): if self.dtype == np.float16: self.atol = 1e-2 - right = reduce(mul, - self.shape[self.begin_norm_axis:len(self.shape)], 1) + right = reduce( + mul, self.shape[self.begin_norm_axis : len(self.shape)], 1 + ) np.random.seed(10) x_np = np.random.uniform(0.1, 1, self.shape).astype(self.dtype) scale_np = np.random.uniform(0.1, 1, [right]).astype('float32') bias_np = np.random.uniform(0.1, 1, [right]).astype('float32') ref_y_np, ref_mean_np, ref_variance_np = ref_layer_norm( - x_np, scale_np, bias_np, self.epsilon, self.begin_norm_axis) + x_np, scale_np, bias_np, self.epsilon, self.begin_norm_axis + ) ref_y_np = ref_y_np.astype(self.dtype) self.inputs = {'X': x_np, 'Scale': scale_np, 'Bias': bias_np} self.outputs = { 'Y': ref_y_np, 'Mean': ref_mean_np, - 'Variance': ref_variance_np + 'Variance': ref_variance_np, } self.attrs = { 'begin_norm_axis': self.begin_norm_axis, - 'use_xpu': True + 'use_xpu': True, } def set_attrs(self): @@ -91,27 +96,23 @@ class XPUTestLayerNormOp(XPUOpTestWrapper): self.check_output_with_place(paddle.XPUPlace(0), atol=self.atol) def test_check_grad(self): - self.check_grad_with_place(paddle.XPUPlace(0), ['X'], - 'Y', - max_relative_error=self.atol) + self.check_grad_with_place( + paddle.XPUPlace(0), ['X'], 'Y', max_relative_error=self.atol + ) class TestXPULayerNormOpAxis2(TestXPULayerNormOp): - def set_attrs(self): self.begin_norm_axis = 2 class TestXPULayerNormOpAxis3(TestXPULayerNormOp): - def set_attrs(self): self.begin_norm_axis = 3 class TestXPULayerNormOp2D(TestXPULayerNormOp): - def set_attrs(self): self.shape = [10, 12] class TestXPULayerNormOp3D(TestXPULayerNormOp): - def set_attrs(self): self.shape = [4, 5, 6] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_log_loss_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_log_loss_op_xpu.py index 94297432e30..4d50fc2da7d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_log_loss_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_log_loss_op_xpu.py @@ -26,7 +26,6 @@ def sigmoid_array(x): class TestXPULogLossOp(OpTest): - def setUp(self): self.op_type = 'log_loss' samples_num = 100 @@ -41,8 +40,9 @@ class TestXPULogLossOp(OpTest): } self.attrs = {'epsilon': epsilon} - loss = -labels * np.log(predicted + epsilon) - ( - 1 - labels) * np.log(1 - predicted + epsilon) + loss = -labels * np.log(predicted + epsilon) - (1 - labels) * np.log( + 1 - predicted + epsilon + ) self.outputs = {'Loss': loss} def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_log_softmax_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_log_softmax_op_xpu.py index c13161b17cb..d961e46bd62 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_log_softmax_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_log_softmax_op_xpu.py @@ -22,14 +22,18 @@ import paddle import paddle.nn.functional as F from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) def ref_log_softmax(x): - shiftx = (x - np.max(x)) + shiftx = x - np.max(x) out = shiftx - np.log(np.exp(shiftx).sum()) return out @@ -39,14 +43,14 @@ def ref_log_softmax_grad(x, axis): axis += len(x.shape) out = np.apply_along_axis(ref_log_softmax, axis, x) axis_dim = x.shape[axis] - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) dx = dout - np.exp(out) * dout.copy().sum(axis=axis, keepdims=True).repeat( - axis_dim, axis=axis) + axis_dim, axis=axis + ) return dx class XPUTestLogSoftmaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'log_softmax' self.use_dynamic_create_class = True @@ -58,14 +62,12 @@ class XPUTestLogSoftmaxOp(XPUOpTestWrapper): shape_arr = [[2, 3, 4, 5], [12, 10], [2, 5], [7, 7], [3, 5, 7]] for axis in axis_arr: for shape in shape_arr: - class_name = 'XPUTestLogSoftmax_' + \ - str(axis) + "_" + str(shape) + class_name = 'XPUTestLogSoftmax_' + str(axis) + "_" + str(shape) attr_dict = {'axis': axis, 'shape': shape} classes.append([class_name, attr_dict]) return base_class, classes class TestXPULogSoftmaxOp(XPUOpTest): - def setUp(self): self.op_type = 'log_softmax' self.python_api = F.log_softmax @@ -76,7 +78,7 @@ class XPUTestLogSoftmaxOp(XPUOpTestWrapper): self.shape = [2, 3, 4, 5] self.axis = -1 - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -91,9 +93,12 @@ class XPUTestLogSoftmaxOp(XPUOpTestWrapper): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], ['Out'], - user_defined_grads=[self.x_grad], - check_eager=True) + self.check_grad( + ['X'], + ['Out'], + user_defined_grads=[self.x_grad], + check_eager=True, + ) support_types = get_xpu_op_support_types('log_softmax') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_logical_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_logical_op_xpu.py index ad0868d29bd..9b0d68ca084 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_logical_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_logical_op_xpu.py @@ -21,19 +21,21 @@ sys.path.append("..") import paddle from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() ################## TEST OP: logical_and ################## class XPUTestLogicalAnd(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logical_and' class XPUTestLogicalAndBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,20 +44,18 @@ class XPUTestLogicalAnd(XPUOpTestWrapper): def set_case(self): self.op_type = 'logical_and' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.logical_and(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -73,7 +73,6 @@ class XPUTestLogicalAnd(XPUOpTestWrapper): pass class XPUTestLogicalAndCase1(XPUTestLogicalAndBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -89,12 +88,10 @@ for stype in support_types: ################## TEST OP: logical_or ################## class XPUTestLogicalOr(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logical_or' class XPUTestLogicalOrBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -103,20 +100,18 @@ class XPUTestLogicalOr(XPUOpTestWrapper): def set_case(self): self.op_type = 'logical_or' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.logical_or(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -134,7 +129,6 @@ class XPUTestLogicalOr(XPUOpTestWrapper): pass class XPUTestLogicalOrCase1(XPUTestLogicalOrBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -150,12 +144,10 @@ for stype in support_types: ################## TEST OP: logical_xor ################## class XPUTestLogicalXor(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logical_xor' class XPUTestLogicalXorBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -164,20 +156,18 @@ class XPUTestLogicalXor(XPUOpTestWrapper): def set_case(self): self.op_type = 'logical_xor' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.logical_xor(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -195,7 +185,6 @@ class XPUTestLogicalXor(XPUOpTestWrapper): pass class XPUTestLogicalXorCase1(XPUTestLogicalXorBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -211,12 +200,10 @@ for stype in support_types: ################## TEST OP: LogicalNot ################## class XPUTestLogicalNot(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logical_not' class XPUTestLogicalNotBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -225,10 +212,9 @@ class XPUTestLogicalNot(XPUOpTestWrapper): def set_case(self): self.op_type = 'logical_not' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) out = np.logical_not(x) self.attrs = {'use_xpu': True} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py index db55aff3817..932507066b9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py @@ -25,7 +25,7 @@ paddle.enable_static() def ref_logsumexp(x, axis=None, keepdim=False, reduce_all=False): if isinstance(axis, int): - axis = (axis, ) + axis = (axis,) elif isinstance(axis, list): axis = tuple(axis) if reduce_all: @@ -35,7 +35,6 @@ def ref_logsumexp(x, axis=None, keepdim=False, reduce_all=False): class XPUTestLogsumexp(XPUOpTest): - def setUp(self): self.op_type = 'logsumexp' self.shape = [2, 3, 4, 5] @@ -54,7 +53,7 @@ class XPUTestLogsumexp(XPUOpTest): self.attrs = { 'axis': self.axis, 'keepdim': self.keepdim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } def set_attrs(self): @@ -70,31 +69,26 @@ class XPUTestLogsumexp(XPUOpTest): class TestLogsumexp_shape(XPUTestLogsumexp): - def set_attrs(self): self.shape = [4, 5, 6] class TestLogsumexp_axis(XPUTestLogsumexp): - def set_attrs(self): self.axis = [0, -1] class TestLogsumexp_axis_all(XPUTestLogsumexp): - def set_attrs(self): self.axis = [0, 1, 2, 3] class TestLogsumexp_keepdim(XPUTestLogsumexp): - def set_attrs(self): self.keepdim = True class TestLogsumexp_reduce_all(XPUTestLogsumexp): - def set_attrs(self): self.reduce_all = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_lookup_table_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_lookup_table_v2_op_xpu.py index 35df3c244bd..039effe0275 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_lookup_table_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_lookup_table_v2_op_xpu.py @@ -29,7 +29,6 @@ paddle.enable_static() class TestLookupTableOp(OpTest): - def setUp(self): self.op_type = "lookup_table_v2" table = np.random.random((17, 31)).astype("float64") @@ -42,15 +41,16 @@ class TestLookupTableOp(OpTest): def test_check_grad(self): - self.check_grad_with_place(inputs_to_check=['W'], - output_names='Out', - no_grad_set=set('Ids'), - place=paddle.XPUPlace(0), - in_place=True) + self.check_grad_with_place( + inputs_to_check=['W'], + output_names='Out', + no_grad_set=set('Ids'), + place=paddle.XPUPlace(0), + in_place=True, + ) class TestLookupTableOpWithTensorIds(OpTest): - def setUp(self): self.op_type = "lookup_table_v2" table = np.random.random((17, 31)).astype("float64") @@ -62,19 +62,21 @@ class TestLookupTableOpWithTensorIds(OpTest): self.check_output_with_place(place=paddle.XPUPlace(0)) def test_check_grad(self): - self.check_grad_with_place(inputs_to_check=['W'], - output_names='Out', - no_grad_set=set('Ids'), - place=paddle.XPUPlace(0), - in_place=True) + self.check_grad_with_place( + inputs_to_check=['W'], + output_names='Out', + no_grad_set=set('Ids'), + place=paddle.XPUPlace(0), + in_place=True, + ) @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithPadding(TestLookupTableOp): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -86,9 +88,9 @@ class TestLookupTableOpWithPadding(TestLookupTableOp): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -99,7 +101,6 @@ class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): class TestLookupTableWIsSelectedRows(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([0, 4, 3, 5]).astype("int64") @@ -147,13 +148,14 @@ class TestLookupTableWIsSelectedRows(unittest.TestCase): self.check_with_place(place) -class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows - ): - +class TestLookupTableWithTensorIdsWIsSelectedRows( + TestLookupTableWIsSelectedRows +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -163,7 +165,6 @@ class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows class TestLookupTableApi(unittest.TestCase): - def test_api(self): x = fluid.layers.data(name='x', shape=[20], dtype='int64') emb = fluid.embedding(input=x, size=[128, 64]) @@ -173,15 +174,16 @@ class TestLookupTableApi(unittest.TestCase): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_data, - }, - fetch_list=[emb], - return_numpy=False) + ret = exe.run( + feed={ + 'x': x_data, + }, + fetch_list=[emb], + return_numpy=False, + ) class TestEmbedOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.randint(0, 10, (4, 6)).astype("int64") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_masked_select_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_masked_select_op_xpu.py index d4ccbc18c34..6a2976ccbb5 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_masked_select_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_masked_select_op_xpu.py @@ -20,7 +20,11 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -34,12 +38,10 @@ def np_masked_select(x, mask): class XPUTestMaskedSelectOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'masked_select' class TestMaskedSelectOp(XPUOpTest): - def setUp(self): self.init() self.dtype = self.in_type @@ -60,14 +62,12 @@ class XPUTestMaskedSelectOp(XPUOpTestWrapper): self.shape = (50, 3) class TestMaskedSelectOp1(TestMaskedSelectOp): - def init(self): self.shape = (6, 8, 9, 18) class TestMaskedSelectOp2(TestMaskedSelectOp): - def init(self): - self.shape = (168, ) + self.shape = (168,) support_types = get_xpu_op_support_types('masked_select') @@ -76,7 +76,6 @@ for stype in support_types: class TestMaskedSelectAPI(unittest.TestCase): - def test_imperative_mode(self): paddle.disable_static(paddle.XPUPlace(0)) shape = (88, 6, 8) @@ -101,27 +100,26 @@ class TestMaskedSelectAPI(unittest.TestCase): exe = paddle.static.Executor(place=paddle.XPUPlace(0)) - res = exe.run(paddle.static.default_main_program(), - feed={ - "x": np_x, - "mask": np_mask - }, - fetch_list=[out]) + res = exe.run( + paddle.static.default_main_program(), + feed={"x": np_x, "mask": np_mask}, + fetch_list=[out], + ) self.assertEqual(np.allclose(res, np_out), True) class TestMaskedSelectError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') mask = paddle.fluid.data(shape=shape, dtype='bool', name='mask') - mask_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='mask_float') + mask_float = paddle.fluid.data( + shape=shape, dtype='float32', name='mask_float' + ) np_x = np.random.random(shape).astype('float32') np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_matmul_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_matmul_op_xpu.py index 0afcd1c5734..c4aab23a952 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_matmul_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_matmul_op_xpu.py @@ -22,7 +22,11 @@ import paddle import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): @@ -64,8 +68,9 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): return Out -def generate_compatible_shapes(dim_X, dim_Y, transpose_X, transpose_Y, - batch_size): +def generate_compatible_shapes( + dim_X, dim_Y, transpose_X, transpose_Y, batch_size +): BATCH_SIZE = 2 if batch_size != None: BATCH_SIZE = batch_size @@ -131,6 +136,7 @@ def generate_compatible_shapes_2(dim, transpose_X, transpose_Y): def generate_negative_dims(in_shape): from itertools import combinations + size = len(in_shape) indexs = list() shapes = list() @@ -138,7 +144,8 @@ def generate_negative_dims(in_shape): indexs.extend(list(combinations([j for j in range(size)], i + 1))) for idx in indexs: shapes.append( - [in_shape[i] if i not in idx else -1 for i in range(size)]) + [in_shape[i] if i not in idx else -1 for i in range(size)] + ) return shapes @@ -151,49 +158,48 @@ def test_negative_dims_program(obj): with program_guard(Program(), Program()): x = fluid.data(name='x', shape=shape_x, dtype=obj.in_type_str) y = fluid.data(name='y', shape=shape_y, dtype=obj.in_type_str) - output = fluid.layers.matmul(x, y, obj.transpose_X, - obj.transpose_Y) + output = fluid.layers.matmul( + x, y, obj.transpose_X, obj.transpose_Y + ) obj.assertEqual(len(Ref.shape), len(output.shape)) for idx in range(len(Ref.shape)): if output.shape[idx] != -1: obj.assertEqual(Ref.shape[idx], output.shape[idx]) exe = fluid.Executor(fluid.XPUPlace(0)) - res, = exe.run(fluid.default_main_program(), - feed={ - 'x': X, - 'y': Y - }, - fetch_list=[output]) + (res,) = exe.run( + fluid.default_main_program(), + feed={'x': X, 'y': Y}, + fetch_list=[output], + ) np.allclose(res, Ref, atol=1e-3) class XPUTestMatmulOpErr(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul" self.use_dynamic_create_class = False class TestMatmulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The inputs type of matmul_op must be Variable. input1 = 12 - self.assertRaises(TypeError, fluid.layers.matmul, input1, - input1) + self.assertRaises( + TypeError, fluid.layers.matmul, input1, input1 + ) # The inputs dtype of matmul_op must be float32, float16 - input2 = fluid.layers.data(name='input2', - shape=[10, 10], - dtype="int32") - self.assertRaises(TypeError, fluid.layers.matmul, input2, - input2) - input3 = fluid.layers.data(name='input3', - shape=[2, 2], - dtype="float16") + input2 = fluid.layers.data( + name='input2', shape=[10, 10], dtype="int32" + ) + self.assertRaises( + TypeError, fluid.layers.matmul, input2, input2 + ) + input3 = fluid.layers.data( + name='input3', shape=[2, 2], dtype="float16" + ) fluid.layers.matmul(input3, input3) class API_TestMm(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2], dtype=self.in_type) @@ -203,13 +209,12 @@ class XPUTestMatmulOpErr(XPUOpTestWrapper): exe = fluid.Executor(fluid.XPUPlace(0)) data1 = np.random.rand(2).astype(self.in_type) data2 = np.random.rand(2).astype(self.in_type) - np_res = exe.run(feed={ - 'x': data1, - 'y': data2 - }, - fetch_list=[result]) - expected_result = np.matmul(data1.reshape(1, 2), - data2.reshape(2, 1)) + np_res = exe.run( + feed={'x': data1, 'y': data2}, fetch_list=[result] + ) + expected_result = np.matmul( + data1.reshape(1, 2), data2.reshape(2, 1) + ) np.testing.assert_allclose(np_res, expected_result, atol=1e-3) @@ -222,89 +227,94 @@ class XPUTestMatmulOpErr(XPUOpTestWrapper): data2 = fluid.dygraph.to_variable(input_array2) out = paddle.mm(data1, data2) expected_result = np.matmul(input_array1, input_array2) - np.testing.assert_allclose(expected_result, - out.numpy(), - atol=1e-3) + np.testing.assert_allclose( + expected_result, out.numpy(), atol=1e-3 + ) class Test_API_Matmul(unittest.TestCase): - def test_dygraph_without_out(self): device = fluid.XPUPlace(0) with fluid.dygraph.guard(device): input_array1 = np.random.rand(3, 4).astype(self.in_type) input_array2 = np.random.rand(4, 3).astype(self.in_type) data1 = fluid.dygraph.to_variable(input_array1).astype( - self.in_type) + self.in_type + ) data2 = fluid.dygraph.to_variable(input_array2).astype( - self.in_type) + self.in_type + ) out = paddle.matmul(data1, data2) expected_result = np.matmul(input_array1, input_array2) - np.testing.assert_allclose(expected_result, - out.numpy(), - atol=1e-3) + np.testing.assert_allclose( + expected_result, out.numpy(), atol=1e-3 + ) class API_TestMmError(unittest.TestCase): - def test_errors(self): - def test_error1(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[3, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[3, 10], dtype="float32" + ) paddle.mm(data1, data2) self.assertRaises(ValueError, test_error1) def test_error2(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[-1, 10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 2, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 2, 10], dtype="float32" + ) paddle.mm(data1, data2) test_error2() def test_error3(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[10, 10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[3, 2, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[10, 10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[3, 2, 10], dtype="float32" + ) paddle.mm(data1, data2) self.assertRaises(ValueError, test_error3) class TestMatmulBaseGenerator(XPUOpTest): - def setUp(self): self.op_type = "matmul" - self.dtype = np.float32 if not hasattr(self, - 'in_type') else self.in_type + self.dtype = ( + np.float32 if not hasattr(self, 'in_type') else self.in_type + ) - self.__class__.no_need_check_grad = False if not hasattr( - self, 'no_need_check_grad') else self.no_need_check_grad + self.__class__.no_need_check_grad = ( + False + if not hasattr(self, 'no_need_check_grad') + else self.no_need_check_grad + ) shape_X = [4, 5] if not hasattr(self, 'shape_X') else self.shape_X shape_Y = [5, 6] if not hasattr(self, 'shape_Y') else self.shape_Y - transpose_X = False if not hasattr(self, - 'transpose_X') else self.transpose_X - transpose_Y = False if not hasattr(self, - 'transpose_Y') else self.transpose_Y + transpose_X = ( + False if not hasattr(self, 'transpose_X') else self.transpose_X + ) + transpose_Y = ( + False if not hasattr(self, 'transpose_Y') else self.transpose_Y + ) X = np.random.random(shape_X).astype(self.dtype) Y = np.random.random(shape_Y).astype(self.dtype) - Out = reference_matmul(X, Y, transpose_X, - transpose_Y).astype(self.dtype) + Out = reference_matmul(X, Y, transpose_X, transpose_Y).astype( + self.dtype + ) self.inputs = {'X': X, 'Y': Y} self.attrs = {'transpose_X': transpose_X, 'transpose_Y': transpose_Y} self.outputs = {'Out': Out} @@ -314,40 +324,43 @@ class TestMatmulBaseGenerator(XPUOpTest): self.check_output_with_place(place, atol=1e-3) def test_check_grad_normal(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=5e-2) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=5e-2 + ) def test_check_grad_ignore_x(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=5e-2, - no_grad_set=set("X")) + self.check_grad_with_place( + place, ['Y'], 'Out', max_relative_error=5e-2, no_grad_set=set("X") + ) def test_check_grad_ignore_y(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=5e-2, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=5e-2, no_grad_set=set('Y') + ) class XPUTestMatmulOp1(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul" self.use_dynamic_create_class = True @@ -366,19 +379,19 @@ class XPUTestMatmulOp1(XPUOpTestWrapper): no_need_check_grad = False if batch >= 5: no_need_check_grad = True - class_name = ( - 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}_batch_{}' - .format(dim_X, dim_Y, transose_x, transose_y, - batch)) + class_name = 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}_batch_{}'.format( + dim_X, dim_Y, transose_x, transose_y, batch + ) shape_x, shape_y = generate_compatible_shapes( - dim_X, dim_Y, transose_x, transose_y, batch) + dim_X, dim_Y, transose_x, transose_y, batch + ) attr_dict = { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': transose_x, 'transpose_Y': transose_y, 'no_need_check_grad': no_need_check_grad, - 'op_type': "matmul" + 'op_type': "matmul", } classes.append([class_name, attr_dict]) @@ -386,7 +399,6 @@ class XPUTestMatmulOp1(XPUOpTestWrapper): class XPUTestMatmulOp2(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul" self.use_dynamic_create_class = True @@ -402,12 +414,12 @@ class XPUTestMatmulOp2(XPUOpTestWrapper): for transose_x in [True, False]: for transose_y in [True, False]: for batch in batch_size: - class_name = ( - 'TestMatMulAPI_dimX_{}_dim_Y_{}_transX_{}_transY_{}_batch_{}' - .format(dim_X, dim_Y, transose_x, transose_y, - batch)) + class_name = 'TestMatMulAPI_dimX_{}_dim_Y_{}_transX_{}_transY_{}_batch_{}'.format( + dim_X, dim_Y, transose_x, transose_y, batch + ) shape_x, shape_y = generate_compatible_shapes( - dim_X, dim_Y, transose_x, transose_y, batch) + dim_X, dim_Y, transose_x, transose_y, batch + ) attr_dict = { 'shape_X': shape_x, 'shape_Y': shape_y, @@ -420,7 +432,6 @@ class XPUTestMatmulOp2(XPUOpTestWrapper): class XPUTestMatmulOp3(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul" self.use_dynamic_create_class = True @@ -431,17 +442,18 @@ class XPUTestMatmulOp3(XPUOpTestWrapper): for dim in [4]: for transpose_X in [False, True]: for transpose_Y in [False, True]: - class_name = ( - 'TestMatMulOp2_dimX_{}_dim_Y_{}_transX_{}_transY_{}'. - format(dim, dim, transpose_X, transpose_Y)) + class_name = 'TestMatMulOp2_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim, dim, transpose_X, transpose_Y + ) shape_X, shape_Y = generate_compatible_shapes_2( - dim, transpose_X, transpose_Y) + dim, transpose_X, transpose_Y + ) attr_dict = { 'shape_X': shape_X, 'shape_Y': shape_Y, 'transpose_X': transpose_X, 'transpose_Y': transpose_Y, - 'op_type': "matmul" + 'op_type': "matmul", } classes.append([class_name, attr_dict]) return base_class, classes diff --git a/python/paddle/fluid/tests/unittests/xpu/test_matmul_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_matmul_v2_op_xpu.py index 8473f9c506d..63354ac7607 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_matmul_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_matmul_v2_op_xpu.py @@ -21,7 +21,11 @@ from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): @@ -30,7 +34,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): # transpose X and Y appropriately. if transpose_X: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -39,7 +43,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -55,7 +59,6 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): class XPUTestMatmulV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul_v2" self.use_dynamic_create_class = False @@ -66,8 +69,8 @@ class XPUTestMatmulV2Op(XPUOpTestWrapper): """ def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -96,8 +99,10 @@ class XPUTestMatmulV2Op(XPUOpTestWrapper): self.check_output_with_place(place) def test_check_grad(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return place = paddle.XPUPlace(0) self.check_grad_with_place(place, ['X', 'Y'], 'Out') @@ -108,7 +113,7 @@ class XPUTestMatmulV2Op(XPUOpTestWrapper): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (100, 3) self.trans_x = False self.trans_y = False @@ -119,7 +124,7 @@ class XPUTestMatmulV2Op(XPUOpTestWrapper): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False @@ -142,7 +147,7 @@ class XPUTestMatmulV2Op(XPUOpTestWrapper): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False @@ -219,7 +224,7 @@ class XPUTestMatmulV2Op(XPUOpTestWrapper): def config(self): self.x_shape = (1, 20, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -274,7 +279,7 @@ class XPUTestMatmulV2Op(XPUOpTestWrapper): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False @@ -285,7 +290,7 @@ class XPUTestMatmulV2Op(XPUOpTestWrapper): def config(self): self.x_shape = (8, 111, 4, 17) - self.y_shape = (17) + self.y_shape = 17 self.trans_x = False self.trans_y = False diff --git a/python/paddle/fluid/tests/unittests/xpu/test_mean_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_mean_op_xpu.py index 2ec886796f0..c86aaeea6cf 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_mean_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_mean_op_xpu.py @@ -25,19 +25,21 @@ from paddle.fluid import Program, program_guard np.random.seed(10) from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestMeanOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'mean' self.use_dynamic_create_class = False class TestMeanOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -65,41 +67,36 @@ class XPUTestMeanOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class TestMeanOp1(TestMeanOp): - def set_shape(self): - self.shape = (5) + self.shape = 5 class TestMeanOp2(TestMeanOp): - def set_shape(self): self.shape = (5, 7, 8) class TestMeanOp3(TestMeanOp): - def set_shape(self): self.shape = (10, 5, 7, 8) class TestMeanOp4(TestMeanOp): - def set_shape(self): self.shape = (2, 2, 3, 3, 3) class TestMeanOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of mean_op must be Variable. input1 = 12 self.assertRaises(TypeError, paddle.mean, input1) # The input dtype of mean_op must be float16, float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, paddle.mean, input2) - input3 = fluid.layers.data(name='input3', - shape=[4], - dtype="float16") + input3 = fluid.layers.data( + name='input3', shape=[4], dtype="float16" + ) fluid.layers.softmax(input3) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu.py index 554508b877f..591cb325417 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu.py @@ -20,19 +20,21 @@ sys.path.append("..") import paddle from test_merged_momentum_op_xpu_base import TestMergedMomentumBase -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestMergedMomentumOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'merged_momentum' self.use_dynamic_create_class = False class TestMergedMomentumOp(TestMergedMomentumBase): - def setUp(self): super().setUp() self.set_case() @@ -46,22 +48,18 @@ class XPUTestMergedMomentumOP(XPUOpTestWrapper): self.check_with_place(self.place, self.in_type) class TestMergedMomentum1(TestMergedMomentumOp): - def set_case(self): self.shapes = [[3, 4], [2, 7], [5, 6, 8]] class TestMergedMomentum2(TestMergedMomentumOp): - def set_case(self): self.shapes = [[3, 4], [2, 7]] class TestMergedMomentum3(TestMergedMomentumOp): - def set_case(self): self.shapes = [[3, 4]] class TestMergedMomentum4(TestMergedMomentumOp): - def set_case(self): self.shapes = [[3, 4], [2, 7], [5, 6, 7], [9, 9], [10, 12]] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu_base.py b/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu_base.py index 64d1e5aa9b2..05044f11ee4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu_base.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu_base.py @@ -22,17 +22,19 @@ from paddle.fluid.layer_helper import LayerHelper from collections import OrderedDict -def run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False, - use_nesterov=True): +def run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, + use_nesterov=True, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -44,48 +46,70 @@ def run_momentum_op(params, helper = LayerHelper(op_type, **locals()) param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -95,8 +119,9 @@ def run_momentum_op(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -115,10 +140,9 @@ def run_momentum_op(params, 'regularization_method': 'l2_decay', 'regularization_coeff': 2.0, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -131,22 +155,18 @@ def run_momentum_op(params, inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars attrs = { - 'mu': - mu, - 'multi_precision': - multi_precision, - 'rescale_grad': - rescale_grad, - 'use_nesterov': - use_nesterov, - 'regularization_method': - ['l2_decay' for i in range(len(param_vars))], + 'mu': mu, + 'multi_precision': multi_precision, + 'rescale_grad': rescale_grad, + 'use_nesterov': use_nesterov, + 'regularization_method': [ + 'l2_decay' for i in range(len(param_vars)) + ], 'regularization_coeff': [2.0 for i in range(len(param_vars))], } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -155,7 +175,6 @@ def run_momentum_op(params, class TestMergedMomentumBase(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -179,22 +198,31 @@ class TestMergedMomentumBase(unittest.TestCase): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, dtype, multi_precision=False): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, dtype, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data( + self.shapes, multi_precision, self.seed, dtype, place + ) def run_op(use_nesterov, use_merged): # NPU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged, - use_nesterov=use_nesterov) + return run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + use_nesterov=use_nesterov, + ) outs1 = run_op(use_nesterov=True, use_merged=True) outs2 = run_op(use_nesterov=True, use_merged=False) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_momentum_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_momentum_op_xpu.py index 5cc9cb49e27..0ee2af0e202 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_momentum_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_momentum_op_xpu.py @@ -22,14 +22,25 @@ import paddle import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def calculate_momentum_by_numpy(param, grad, mu, velocity, use_nesterov, - learning_rate, regularization_method, - regularization_coeff): +def calculate_momentum_by_numpy( + param, + grad, + mu, + velocity, + use_nesterov, + learning_rate, + regularization_method, + regularization_coeff, +): if regularization_method == "l2_decay": grad = grad + regularization_coeff * param velocity_out = mu * velocity + grad @@ -40,21 +51,20 @@ def calculate_momentum_by_numpy(param, grad, mu, velocity, use_nesterov, else: velocity_out = mu * velocity + grad if use_nesterov: - param_out = param - grad * learning_rate - \ - velocity_out * mu * learning_rate + param_out = ( + param - grad * learning_rate - velocity_out * mu * learning_rate + ) else: param_out = param - learning_rate * velocity_out return param_out, velocity_out class XPUTestMomentumOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'momentum' self.use_dynamic_create_class = False class TestMomentumOPBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.xpu_version = core.get_xpu_device_version(0) @@ -65,12 +75,15 @@ class XPUTestMomentumOP(XPUOpTestWrapper): self.op_type = 'momentum' self.init_config() - self.param = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) - self.grad = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.param = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) + self.grad = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) self.velocity = np.random.uniform(-1, 1, self.input_shape).astype( - self.dtype) + self.dtype + ) param_out, velocity_out = calculate_momentum_by_numpy( param=self.param, grad=self.grad, @@ -79,7 +92,8 @@ class XPUTestMomentumOP(XPUOpTestWrapper): use_nesterov=self.use_nesterov, learning_rate=self.learning_rate, regularization_method=self.regularization_method, - regularization_coeff=self.regularization_coeff) + regularization_coeff=self.regularization_coeff, + ) param_out = param_out.astype(self.dtype) velocity_out = velocity_out.astype(self.dtype) self.inputs = { @@ -93,7 +107,7 @@ class XPUTestMomentumOP(XPUOpTestWrapper): 'mu': self.mu, 'use_nesterov': self.use_nesterov, 'regularization_method': self.regularization_method, - 'regularization_coeff': self.regularization_coeff + 'regularization_coeff': self.regularization_coeff, } self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -112,7 +126,6 @@ class XPUTestMomentumOP(XPUOpTestWrapper): self.regularization_coeff = 0 class XPUTestMomentum1(TestMomentumOPBase): - def init_config(self): self.input_shape = [2, 768] self.learning_rate = np.array([0.002]).astype(float) @@ -122,7 +135,6 @@ class XPUTestMomentumOP(XPUOpTestWrapper): self.regularization_coeff = 0 class XPUTestMomentum2(TestMomentumOPBase): - def init_config(self): self.input_shape = [3, 8, 4096] self.learning_rate = np.array([0.005]).astype(float) @@ -132,7 +144,6 @@ class XPUTestMomentumOP(XPUOpTestWrapper): self.regularization_coeff = 0 class XPUTestMomentum3(TestMomentumOPBase): - def init_config(self): self.input_shape = [1024] self.learning_rate = np.array([0.01]).astype(float) @@ -147,7 +158,6 @@ class XPUTestMomentumOP(XPUOpTestWrapper): self.regularization_coeff = 0 class XPUTestMomentum4(TestMomentumOPBase): - def init_config(self): self.input_shape = [2, 2, 255] self.learning_rate = np.array([0.0005]).astype(float) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_mul_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_mul_op_xpu.py index b8d4666761a..08b3b3e89a3 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_mul_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_mul_op_xpu.py @@ -24,18 +24,23 @@ from paddle.fluid import Program, program_guard paddle.enable_static() -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class TestMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of mul_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.XPUPlace(0)) - x2 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.XPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.XPUPlace(0) + ) + x2 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.XPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.mul, x1, x2) # The input dtype of mul_op must be float32. x3 = fluid.layers.data(name='x3', shape=[4], dtype="int32") @@ -44,26 +49,26 @@ class TestMulOpError(unittest.TestCase): class XPUTestMulOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'mul' self.use_dynamic_create_class = False class TestXPUMulOp1(XPUOpTest): - def setUp(self): self.op_type = "mul" self.dtype = self.in_type self.inputs = { 'X': np.random.random((3, 4, 2, 9)).astype(self.in_type_str), - 'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.in_type_str) + 'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.in_type_str), } self.attrs = { 'x_num_col_dims': 2, 'y_num_col_dims': 2, } - result = np.dot(self.inputs['X'].reshape(3 * 4, 2 * 9), - self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3)) + result = np.dot( + self.inputs['X'].reshape(3 * 4, 2 * 9), + self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3), + ) result = result.reshape(3, 4, 1, 2, 3) self.outputs = {'Out': result} @@ -75,35 +80,40 @@ class XPUTestMulOp(XPUOpTestWrapper): def test_check_grad_normal(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.1) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.1 + ) def test_check_grad_ingore_x(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.1, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.1, + no_grad_set=set("X"), + ) def test_check_grad_ignore_y(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.1, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.1, + no_grad_set=set('Y'), + ) class TestXPUMulOp2(XPUOpTest): - def setUp(self): self.op_type = "mul" self.use_xpu = True self.dtype = self.in_type self.inputs = { 'X': np.random.random((20, 5)).astype(self.in_type_str), - 'Y': np.random.random((5, 21)).astype(self.in_type_str) + 'Y': np.random.random((5, 21)).astype(self.in_type_str), } self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} @@ -115,25 +125,31 @@ class XPUTestMulOp(XPUOpTestWrapper): def test_check_grad_normal(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.1) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.1 + ) def test_check_grad_ingore_x(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.1, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.1, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.1, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.1, + no_grad_set=set('Y'), + ) support_types = get_xpu_op_support_types('mul') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_nearest_interp_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_nearest_interp_v2_op_xpu.py index 3ded60a3d80..0f2d4fd27ec 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_nearest_interp_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_nearest_interp_v2_op_xpu.py @@ -21,20 +21,26 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def nearest_neighbor_interp_np(X, - out_h, - out_w, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp_np( + X, + out_h, + out_w, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -47,16 +53,16 @@ def nearest_neighbor_interp_np(X, n, c, in_h, in_w = X.shape ratio_h = ratio_w = 0.0 - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -84,17 +90,19 @@ def nearest_neighbor_interp_np(X, return out.astype(X.dtype) -def nearest_neighbor_interp3d_np(X, - out_d, - out_h, - out_w, - scale_d=0, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp3d_np( + X, + out_d, + out_h, + out_w, + scale_d=0, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -109,24 +117,24 @@ def nearest_neighbor_interp3d_np(X, n, c, in_d, in_h, in_w = X.shape ratio_d = ratio_h = ratio_w = 0.0 - if (out_d > 1): - if (align_corners): + if out_d > 1: + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: if scale_d > 0: ratio_d = 1.0 / scale_d else: ratio_d = 1.0 * in_d / out_d - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -158,13 +166,11 @@ def nearest_neighbor_interp3d_np(X, class XPUNearestInterpOpWrapper(XPUOpTestWrapper): - def __init__(self): self.op_name = 'nearest_interp_v2' self.use_dynamic_create_class = False class TestNearestInterpOp(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -174,7 +180,7 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): self.data_layout = 'NCHW' self.interp_method = 'nearest' - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.init_test_case() @@ -233,13 +239,30 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): # output_np if len(self.input_shape) == 4: output_np = nearest_neighbor_interp_np( - input_np, out_h, out_w, scale_h, scale_w, self.out_size, - self.actual_shape, self.align_corners, self.data_layout) + input_np, + out_h, + out_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) elif len(self.input_shape) == 5: output_np = nearest_neighbor_interp3d_np( - input_np, out_d, out_h, out_w, scale_d, scale_h, scale_w, - self.out_size, self.actual_shape, self.align_corners, - self.data_layout) + input_np, + out_d, + out_h, + out_w, + scale_d, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.outputs = {'Out': output_np} self.inputs = {'X': input_np} @@ -255,7 +278,7 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } else: self.attrs = { @@ -263,7 +286,7 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: @@ -298,21 +321,18 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): """ class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [4, 1, 7, 8] self.out_h = 1 @@ -320,7 +340,6 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): self.out_size = np.array([2, 2]).astype("int32") class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 @@ -328,7 +347,6 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): self.out_size = np.array([11, 11]).astype("int32") class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -336,14 +354,12 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): self.out_size = np.array([65, 129]).astype("int32") class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 64 @@ -365,21 +381,18 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): """ class TestNearestInterpWithoutCorners(TestNearestInterpOp): - def set_align_corners(self): self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 2, 5, 7] self.out_h = 64 @@ -388,7 +401,6 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): self.out_size = np.array([66, 40]).astype("int32") class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 2, 7, 5] self.out_h = 64 @@ -411,7 +423,6 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): """ class TestNearestInterpOp_attr_tensor(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -420,7 +431,7 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): self.actual_shape = None self.interp_method = 'nearest' - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.init_test_case() @@ -457,8 +468,9 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -470,10 +482,16 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, - self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def init_dtype(self): @@ -493,7 +511,6 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 @@ -502,7 +519,6 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 64 @@ -512,7 +528,6 @@ class XPUNearestInterpOpWrapper(XPUOpTestWrapper): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 64 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_one_hot_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_one_hot_op_xpu.py index 579f8c1faf0..d3f1a300664 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_one_hot_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_one_hot_op_xpu.py @@ -20,19 +20,21 @@ sys.path.append("..") import paddle import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestOneHotOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'one_hot' self.use_dynamic_create_class = False class TestXPUOneHotOP(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -49,11 +51,15 @@ class XPUTestOneHotOP(XPUOpTestWrapper): np.random.randint(0, self.depth - 1) for i in range(sum(self.x_lod[0])) ] - self.x = np.array(self.x).astype(self.dtype).reshape( - [sum(self.x_lod[0]), 1]) - - self.out = np.zeros(shape=(np.product(self.x.shape[:-1]), - self.depth)).astype('float32') + self.x = ( + np.array(self.x) + .astype(self.dtype) + .reshape([sum(self.x_lod[0]), 1]) + ) + + self.out = np.zeros( + shape=(np.product(self.x.shape[:-1]), self.depth) + ).astype('float32') for i in range(np.product(self.x.shape)): self.out[i, self.x[i]] = 1.0 @@ -62,7 +68,7 @@ class XPUTestOneHotOP(XPUOpTestWrapper): def set_input(self): self.inputs = { 'X': (self.x, self.x_lod), - 'depth_tensor': self.depth_np + 'depth_tensor': self.depth_np, } self.attrs = {'dtype': int(core.VarDesc.VarType.FP32)} @@ -73,31 +79,27 @@ class XPUTestOneHotOP(XPUOpTestWrapper): self.dtype = self.in_type class TestXPUOneHotOP_attr(TestXPUOneHotOP): - def set_input(self): self.inputs = {'X': (self.x, self.x_lod)} self.attrs = { 'dtype': int(core.VarDesc.VarType.FP32), - 'depth': self.depth + 'depth': self.depth, } class TestXPUOneHotOP_default_dtype(TestXPUOneHotOP): - def set_input(self): self.inputs = { 'X': (self.x, self.x_lod), - 'depth_tensor': self.depth_np + 'depth_tensor': self.depth_np, } self.attrs = {} class TestXPUOneHotOP_default_dtype_attr(TestXPUOneHotOP): - def set_input(self): self.inputs = {'X': (self.x, self.x_lod)} self.attrs = {'depth': self.depth} class TestXPUOneHotOP_out_of_range(TestXPUOneHotOP): - def set_data(self): self.depth = 10 self.x_lod = [[4, 1, 3, 3]] @@ -105,11 +107,15 @@ class XPUTestOneHotOP(XPUOpTestWrapper): np.random.choice([-1, self.depth]) for i in range(sum(self.x_lod[0])) ] - self.x = np.array(self.x).astype(self.dtype).reshape( - [sum(self.x_lod[0]), 1]) - - self.out = np.zeros(shape=(np.product(self.x.shape[:-1]), - self.depth)).astype('float32') + self.x = ( + np.array(self.x) + .astype(self.dtype) + .reshape([sum(self.x_lod[0]), 1]) + ) + + self.out = np.zeros( + shape=(np.product(self.x.shape[:-1]), self.depth) + ).astype('float32') self.outputs = {'Out': (self.out, self.x_lod)} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_one_hot_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_one_hot_v2_op_xpu.py index 6e724173449..19228435ea4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_one_hot_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_one_hot_v2_op_xpu.py @@ -21,19 +21,21 @@ import sys sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestOneHotOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'one_hot_v2' self.use_dynamic_create_class = False class TestOneHotOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -48,8 +50,9 @@ class XPUTestOneHotOp(XPUOpTestWrapper): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0])]) - out = np.zeros(shape=(np.product(x.shape), - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape), depth)).astype( + self.dtype + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -62,7 +65,6 @@ class XPUTestOneHotOp(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestOneHotOp_attr(TestOneHotOp): - def setUp(self): self.init() depth = 10 @@ -71,8 +73,9 @@ class XPUTestOneHotOp(XPUOpTestWrapper): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + self.dtype + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -80,12 +83,11 @@ class XPUTestOneHotOp(XPUOpTestWrapper): self.inputs = {'X': (x, x_lod)} self.attrs = { 'dtype': int(core.VarDesc.VarType.FP32), - 'depth': depth + 'depth': depth, } self.outputs = {'Out': (out, x_lod)} class TestOneHotOp_default_dtype(TestOneHotOp): - def setUp(self): self.init() depth = 10 @@ -95,8 +97,9 @@ class XPUTestOneHotOp(XPUOpTestWrapper): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0])]) - out = np.zeros(shape=(np.product(x.shape), - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape), depth)).astype( + self.dtype + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -106,7 +109,6 @@ class XPUTestOneHotOp(XPUOpTestWrapper): self.outputs = {'Out': (out, x_lod)} class TestOneHotOp_default_dtype_attr(TestOneHotOp): - def setUp(self): self.init() depth = 10 @@ -115,8 +117,9 @@ class XPUTestOneHotOp(XPUOpTestWrapper): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + self.dtype + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -126,7 +129,6 @@ class XPUTestOneHotOp(XPUOpTestWrapper): self.outputs = {'Out': (out, x_lod)} class TestOneHotOp_out_of_range(TestOneHotOp): - def setUp(self): self.init() depth = 10 @@ -134,8 +136,9 @@ class XPUTestOneHotOp(XPUOpTestWrapper): x = [np.random.choice([-1, depth]) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0])]) - out = np.zeros(shape=(np.product(x.shape), - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape), depth)).astype( + self.dtype + ) self.inputs = {'X': (x, x_lod)} self.attrs = {'depth': depth, 'allow_out_of_range': True} @@ -143,7 +146,6 @@ class XPUTestOneHotOp(XPUOpTestWrapper): class TestOneHotOpApi(unittest.TestCase): - def test_api(self): depth = 10 self._run(depth) @@ -154,39 +156,45 @@ class TestOneHotOpApi(unittest.TestCase): def test_api_with_dygraph(self): depth = 10 - label = np.array([np.random.randint(0, depth - 1) - for i in range(6)]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, depth - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(): one_hot_label = fluid.one_hot( - input=fluid.dygraph.to_variable(label), depth=depth) + input=fluid.dygraph.to_variable(label), depth=depth + ) def _run(self, depth): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = fluid.one_hot(input=label, depth=depth) place = fluid.XPUPlace(0) - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) class BadInputTestOnehotV2(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="float32") + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="float32", + ) one_hot_label = fluid.one_hot(input=label, depth=4) self.assertRaises(TypeError, test_bad_x) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_p_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_p_norm_op_xpu.py index c99c8968018..a03a6e9939a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_p_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_p_norm_op_xpu.py @@ -19,7 +19,11 @@ import unittest sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -63,13 +67,11 @@ def ref_p_norm(x, axis, porder, keepdims=False, reduce_all=False): class XPUTestPNormOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'p_norm' self.use_dynamic_create_class = False class TestXPUPNormOp(XPUOpTest): - def setUp(self): self.op_type = "p_norm" self.dtype = self.in_type @@ -84,15 +86,16 @@ class XPUTestPNormOp(XPUOpTestWrapper): x_np = np.random.uniform(-10, 10, self.shape).astype(self.dtype) - ref_y_np = ref_p_norm(x_np, self.axis, self.porder, self.keepdims, - self.asvector) + ref_y_np = ref_p_norm( + x_np, self.axis, self.porder, self.keepdims, self.asvector + ) self.inputs = {'X': x_np} self.outputs = {'Out': ref_y_np} self.attrs = { 'epsilon': self.epsilon, 'axis': self.axis, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } def set_attrs(self): @@ -105,70 +108,60 @@ class XPUTestPNormOp(XPUOpTestWrapper): self.check_grad_with_place(paddle.XPUPlace(0), ['X'], 'Out') class TestPnormOp2(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = 2.0 class TestPnormOp3(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = np.inf class TestPnormOp4(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = -np.inf class TestPnormOp5(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = 0 class TestPnormOp6(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = -1 self.porder = 2 class TestPnormOp7(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3, 10] self.axis = 2 self.porder = 2.0 class TestPnormOp8(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = np.inf class TestPnormOp9(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3, 10] self.axis = 1 self.porder = -np.inf class TestPnormOp10(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3, 10] self.axis = 2 self.porder = 0 class TestPnormOp11(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3, 10] self.axis = -1 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_pool2d_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_pool2d_op_xpu.py index f630c21c94a..45c9f518cbd 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_pool2d_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_pool2d_op_xpu.py @@ -20,33 +20,43 @@ import numpy as np from op_test_xpu import XPUOpTest from test_pool2d_op import adaptive_start_index, adaptive_end_index -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() -def max_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_type=np.float64): +def max_pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_type=np.float64, +): N, C, H, W = x.shape if global_pool == 1: ksize = [H, W] if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - - 1) // strides[1] + 1 if ceil_mode else ( - W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + H_out = ( + (H - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + 2 * paddings[1] + strides[1] - 1) // strides[1] + 1 + if ceil_mode + else (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + ) out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -66,27 +76,33 @@ def max_pool2D_forward_naive(x, return out -def avg_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_type=np.float64): +def avg_pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_type=np.float64, +): N, C, H, W = x.shape if global_pool == 1: ksize = [H, W] if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - - 1) // strides[1] + 1 if ceil_mode else ( - W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + H_out = ( + (H - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + 2 * paddings[1] + strides[1] - 1) // strides[1] + 1 + if ceil_mode + else (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + ) out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -108,39 +124,44 @@ def avg_pool2D_forward_naive(x, x_masked = x[:, :, r_start:r_end, c_start:c_end] - if (exclusive or adaptive): + if exclusive or adaptive: field_size = (r_end - r_start) * (c_end - c_start) if data_type == np.int8 or data_type == np.uint8: - out[:, :, i, - j] = (np.rint(np.sum(x_masked, axis=(2, 3)) / - field_size)).astype(data_type) + out[:, :, i, j] = ( + np.rint(np.sum(x_masked, axis=(2, 3)) / field_size) + ).astype(data_type) else: - out[:, :, i, j] = (np.sum(x_masked, axis=(2, 3)) / - field_size).astype(data_type) + out[:, :, i, j] = ( + np.sum(x_masked, axis=(2, 3)) / field_size + ).astype(data_type) return out -def pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCHW', - pool_type="max", - padding_algorithm="EXPLICIT"): +def pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCHW', + pool_type="max", + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -150,9 +171,10 @@ def pool2D_forward_naive(x, if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0] @@ -160,7 +182,8 @@ def pool2D_forward_naive(x, raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCHW": @@ -173,8 +196,11 @@ def pool2D_forward_naive(x, is_sys = True if len(paddings) == 2 else False N = x.shape[0] - C, H, W = [x.shape[1], x.shape[2], x.shape[3]] if data_format == 'NCHW' \ + C, H, W = ( + [x.shape[1], x.shape[2], x.shape[3]] + if data_format == 'NCHW' else [x.shape[3], x.shape[1], x.shape[2]] + ) if global_pool == 1: ksize = [H, W] @@ -188,13 +214,26 @@ def pool2D_forward_naive(x, if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 - W_out = (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 - - out = np.zeros((N, C, H_out, W_out)) if data_format=='NCHW' \ + H_out = ( + (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + ) + + out = ( + np.zeros((N, C, H_out, W_out)) + if data_format == 'NCHW' else np.zeros((N, H_out, W_out, C)) + ) for i in range(H_out): if adaptive: in_h_start = adaptive_start_index(i, H, ksize[0]) @@ -222,21 +261,22 @@ def pool2D_forward_naive(x, if data_format == 'NCHW': x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) - + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) -# if (exclusive or adaptive) else (ksize[0] * ksize[1]) + # if (exclusive or adaptive) else (ksize[0] * ksize[1]) out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size elif pool_type == 'max': out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) elif data_format == 'NHWC': x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size elif pool_type == 'max': out[:, i, j, :] = np.max(x_masked, axis=(1, 2)) @@ -244,13 +284,11 @@ def pool2D_forward_naive(x, class XPUTestPool2D_Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'pool2d' self.use_dynamic_create_class = False class TestPool2D_Op(XPUOpTest): - def setUp(self): self.op_type = "pool2d" self.dtype = self.in_type @@ -272,10 +310,18 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): input = np.random.random(self.shape).astype(self.dtype) output = pool2D_forward_naive( - input, self.ksize, self.strides, self.paddings, - self.global_pool, self.ceil_mode, self.exclusive, self.adaptive, - self.data_format, self.pool_type, - self.padding_algorithm).astype(self.dtype) + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': XPUOpTest.np_dtype_to_fluid_dtype(input)} self.attrs = { @@ -290,7 +336,7 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): 'exclusive': self.exclusive, 'adaptive': self.adaptive, "padding_algorithm": self.padding_algorithm, - 'ceil_mode': self.ceil_mode + 'ceil_mode': self.ceil_mode, } self.outputs = {'Out': output} @@ -335,12 +381,10 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.adaptive = False class TestAvgPoolAdaptive(TestPool2D_Op): - def init_adaptive(self): self.adaptive = True class TestAvgPoolAdaptiveAsyOutSize(TestPool2D_Op): - def init_adaptive(self): self.adaptive = True @@ -353,7 +397,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.paddings = [0, 0, 0, 0] class TestCase1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -372,7 +415,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 7, 7] class TestCase2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -391,25 +433,21 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 7, 7] class TestCase3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestPool2D_AsyPadding(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -419,7 +457,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 5, 5] class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -429,7 +466,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 7, 7] class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -439,7 +475,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 7, 7] class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -449,7 +484,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 5, 5] class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -459,7 +493,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 7, 7] class TestCase5_AsyPadding(TestCase5): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -469,7 +502,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 7, 7] class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -482,7 +514,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.shape = [2, 3, 7, 7] class TestCaseCeil1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -504,7 +535,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.ceil_mode = True class TestCaseCeil2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -526,7 +556,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.ceil_mode = True class TestCaseCeil3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive @@ -535,7 +564,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.ceil_mode = True class TestCaseCeil4(TestCaseCeil1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive @@ -544,7 +572,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.ceil_mode = True class TestCaseCeil5(TestCaseCeil2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive @@ -553,7 +580,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.ceil_mode = True class TestCaseAdaptiveAvg(TestPool2D_Op): - def init_test_case(self): self.ksize = [2, 2] self.strides = [2, 2] @@ -575,7 +601,6 @@ class XPUTestPool2D_Op(XPUOpTestWrapper): self.adaptive = True class TestCaseAdaptiveMax(TestPool2D_Op): - def init_test_case(self): self.ksize = [2, 2] self.strides = [2, 2] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_pow2_decay_with_linear_warmup_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_pow2_decay_with_linear_warmup_op_xpu.py index 620cdac1811..246511c9f6c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_pow2_decay_with_linear_warmup_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_pow2_decay_with_linear_warmup_op_xpu.py @@ -28,8 +28,9 @@ def gen_pow2_warmup_op_lr(warmup_steps, total_steps, base_lr, end_lr, place): main = paddle.static.Program() startup = paddle.static.Program() with paddle.static.program_guard(main, startup): - lr = pow2_decay_with_linear_warmup(warmup_steps, total_steps, base_lr, - end_lr) + lr = pow2_decay_with_linear_warmup( + warmup_steps, total_steps, base_lr, end_lr + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): exe.run(startup) @@ -39,18 +40,21 @@ def gen_pow2_warmup_op_lr(warmup_steps, total_steps, base_lr, end_lr, place): class Pow2Warmup(LinearWarmup): - def __init__(self, warmup_steps, total_steps, base_lr, end_lr): assert total_steps > warmup_steps - lr_sch = PolynomialDecay(learning_rate=base_lr, - decay_steps=total_steps - warmup_steps, - end_lr=end_lr, - power=2) + lr_sch = PolynomialDecay( + learning_rate=base_lr, + decay_steps=total_steps - warmup_steps, + end_lr=end_lr, + power=2, + ) - super(Pow2Warmup, self).__init__(learning_rate=lr_sch, - warmup_steps=warmup_steps, - start_lr=0.0, - end_lr=base_lr) + super(Pow2Warmup, self).__init__( + learning_rate=lr_sch, + warmup_steps=warmup_steps, + start_lr=0.0, + end_lr=base_lr, + ) def gen_pow2_warmup_py_lr(warmup_steps, total_steps, base_lr, end_lr, place): @@ -62,7 +66,6 @@ def gen_pow2_warmup_py_lr(warmup_steps, total_steps, base_lr, end_lr, place): class TestPowWarmup(unittest.TestCase): - def setUp(self): paddle.enable_static() self.op_type = 'pow2_decay_with_linear_warmup' diff --git a/python/paddle/fluid/tests/unittests/xpu/test_prior_box_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_prior_box_op_xpu.py index 4561a54d65e..acc05963ee4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_prior_box_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_prior_box_op_xpu.py @@ -22,19 +22,21 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestPriorBoxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'prior_box' self.use_dynamic_create_class = False class TestPriorBoxOp(XPUOpTest): - def setUp(self): self.op_type = "prior_box" self.use_xpu = True @@ -56,7 +58,7 @@ class XPUTestPriorBoxOp(XPUOpTestWrapper): 'min_max_aspect_ratios_order': self.min_max_aspect_ratios_order, 'step_w': self.step_w, 'step_h': self.step_h, - 'offset': self.offset + 'offset': self.offset, } if len(self.max_sizes) > 0: self.attrs['max_sizes'] = self.max_sizes @@ -95,11 +97,13 @@ class XPUTestPriorBoxOp(XPUOpTestWrapper): self.flip = True self.set_min_max_aspect_ratios_order() self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] - self.aspect_ratios = np.array(self.aspect_ratios, - dtype=np.float64).flatten() + self.aspect_ratios = np.array( + self.aspect_ratios, dtype=np.float64 + ).flatten() self.variances = [0.1, 0.1, 0.2, 0.2] - self.variances = np.array(self.variances, - dtype=np.float64).flatten() + self.variances = np.array( + self.variances, dtype=np.float64 + ).flatten() self.clip = True self.num_priors = len(self.real_aspect_ratios) * len(self.min_sizes) @@ -109,12 +113,22 @@ class XPUTestPriorBoxOp(XPUOpTestWrapper): def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype(self.dtype) + ( + self.batch_size, + self.image_channels, + self.image_w, + self.image_h, + ) + ).astype(self.dtype) self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype(self.dtype) + ( + self.batch_size, + self.input_channels, + self.layer_w, + self.layer_h, + ) + ).astype(self.dtype) def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -135,72 +149,76 @@ class XPUTestPriorBoxOp(XPUOpTestWrapper): ar = self.real_aspect_ratios[r] c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 else: - c_w = c_h = min_size / 2. - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + c_w = c_h = min_size / 2.0 + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # rest of priors for r in range(len(self.real_aspect_ratios)): ar = self.real_aspect_ratios[r] - if abs(ar - 1.) < 1e-6: + if abs(ar - 1.0) < 1e-6: continue c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) # set the variance. - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype(self.dtype) self.out_var = out_var.astype(self.dtype) class TestPriorBoxOpWithoutMaxSize(TestPriorBoxOp): - def set_max_sizes(self): self.max_sizes = [] class TestPriorBoxOpWithSpecifiedOutOrder(TestPriorBoxOp): - def set_min_max_aspect_ratios_order(self): self.min_max_aspect_ratios_order = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_range_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_range_xpu.py index 3135255bea2..02893acc0e2 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_range_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_range_xpu.py @@ -19,19 +19,21 @@ import sys sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestRangeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "range" self.use_dynamic_create_class = False class TestRangeOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "range" @@ -40,13 +42,13 @@ class XPUTestRangeOp(XPUOpTestWrapper): self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange( + self.case[0], self.case[1], self.case[2] + ).astype(self.dtype) } def set_xpu(self): @@ -63,27 +65,22 @@ class XPUTestRangeOp(XPUOpTestWrapper): self.check_output_with_place(place, check_dygraph=False) class TestRangeOpCase0(TestRangeOp): - def init_config(self): self.case = (0, 5, 1) class TestRangeOpCase1(TestRangeOp): - def init_config(self): self.case = (0, 5, 2) class TestRangeOpCase2(TestRangeOp): - def init_config(self): self.case = (10, 1, -2) class TestRangeOpCase3(TestRangeOp): - def init_config(self): self.case = (-1, -10, -2) class TestRangeOpCase4(TestRangeOp): - def init_config(self): self.case = (10, -10, -11) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_all_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_all_op_xpu.py index 1d70b5fb3ca..df485b49ac1 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_all_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_all_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceAllOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_all' class XPUTestReduceAllBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.set_case() @@ -42,11 +44,12 @@ class XPUTestReduceAllOp(XPUOpTestWrapper): 'use_xpu': True, 'reduce_all': True, 'keep_dim': True, - 'dim': (3, 5, 4) + 'dim': (3, 5, 4), } self.inputs = { - 'X': - np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} @@ -57,14 +60,13 @@ class XPUTestReduceAllOp(XPUOpTestWrapper): pass class XPUTestReduceAllCase1(XPUTestReduceAllBase): - def set_case(self): self.op_type = 'reduce_all' self.attrs = { 'use_xpu': True, 'reduce_all': True, 'keep_dim': True, - 'dim': [1] + 'dim': [1], } self.inputs = { 'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool") @@ -72,23 +74,22 @@ class XPUTestReduceAllOp(XPUOpTestWrapper): self.outputs = {'Out': self.inputs['X'].all()} class XPUTestReduceAllCase2(XPUTestReduceAllBase): - def set_case(self): self.op_type = 'reduce_all' self.attrs = { 'use_xpu': True, 'reduce_all': True, 'keep_dim': False, - 'dim': (3, 6) + 'dim': (3, 6), } self.inputs = { - 'X': - np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} class XPUTestReduceAllCase3(XPUTestReduceAllBase): - def set_case(self): self.op_type = 'reduce_all' self.attrs = { diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_amax_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_amax_op_xpu.py index 05b9edf7d8a..4394340aa1c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_amax_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_amax_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceAmaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_amax' class XPUTestReduceAmaxBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.set_case() @@ -39,7 +41,7 @@ class XPUTestReduceAmaxOp(XPUOpTestWrapper): def set_case(self): self.op_type = 'reduce_amax' self.shape = (20, 10) - self.attrs = {'use_xpu': True, 'keep_dim': False, 'dim': (1, )} + self.attrs = {'use_xpu': True, 'keep_dim': False, 'dim': (1,)} self.inputs = { 'X': np.random.randint(0, 100, self.shape).astype("float32") @@ -47,10 +49,11 @@ class XPUTestReduceAmaxOp(XPUOpTestWrapper): expect_intput = self.inputs['X'] self.outputs = { - 'Out': - np.amax(expect_intput, - axis=self.attrs['dim'], - keepdims=self.attrs['keep_dim']) + 'Out': np.amax( + expect_intput, + axis=self.attrs['dim'], + keepdims=self.attrs['keep_dim'], + ) } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_amin_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_amin_op_xpu.py index 4e164840126..77c45ce0642 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_amin_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_amin_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceAmaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_amin' class XPUTestReduceAmaxBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.set_case() @@ -39,7 +41,7 @@ class XPUTestReduceAmaxOp(XPUOpTestWrapper): def set_case(self): self.op_type = 'reduce_amin' self.shape = (20, 10) - self.attrs = {'use_xpu': True, 'keep_dim': False, 'dim': (1, )} + self.attrs = {'use_xpu': True, 'keep_dim': False, 'dim': (1,)} self.inputs = { 'X': np.random.randint(0, 100, self.shape).astype("float32") @@ -47,10 +49,11 @@ class XPUTestReduceAmaxOp(XPUOpTestWrapper): expect_intput = self.inputs['X'] self.outputs = { - 'Out': - np.amin(expect_intput, - axis=self.attrs['dim'], - keepdims=self.attrs['keep_dim']) + 'Out': np.amin( + expect_intput, + axis=self.attrs['dim'], + keepdims=self.attrs['keep_dim'], + ) } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_any_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_any_op_xpu.py index 116eea5bb10..032d138558d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_any_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_any_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceAnyOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_any' class XPUTestReduceAnyBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.set_case() @@ -42,11 +44,12 @@ class XPUTestReduceAnyOp(XPUOpTestWrapper): 'use_xpu': True, 'reduce_all': True, 'keep_dim': True, - 'dim': (3, 5, 4) + 'dim': (3, 5, 4), } self.inputs = { - 'X': - np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -57,7 +60,6 @@ class XPUTestReduceAnyOp(XPUOpTestWrapper): pass class XPUTestReduceAnyCase1(XPUTestReduceAnyBase): - def set_case(self): self.op_type = 'reduce_any' self.attrs = { @@ -72,18 +74,18 @@ class XPUTestReduceAnyOp(XPUOpTestWrapper): self.outputs = {'Out': self.inputs['X'].any(axis=1)} class XPUTestReduceAnyCase2(XPUTestReduceAnyBase): - def set_case(self): self.op_type = 'reduce_any' self.attrs = { 'use_xpu': True, 'reduce_all': True, 'keep_dim': False, - 'dim': (3, 6) + 'dim': (3, 6), } self.inputs = { - 'X': - np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_max_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_max_op_xpu.py index fd6b96d7ed7..ee0922110e5 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_max_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_max_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceMaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_max' class XPUTestReduceMaxBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,21 +44,21 @@ class XPUTestReduceMaxOp(XPUOpTestWrapper): self.attrs = { 'use_xpu': True, 'reduce_all': self.reduce_all, - 'keep_dim': self.keep_dim + 'keep_dim': self.keep_dim, } self.inputs = {'X': np.random.random(self.shape).astype("float32")} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].max()} else: self.outputs = { - 'Out': - self.inputs['X'].max(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].max( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = False @@ -67,10 +69,9 @@ class XPUTestReduceMaxOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class XPUTestReduceMaxCase1(XPUTestReduceMaxBase): - def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_mean_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_mean_op_xpu.py index 0b1b9070025..41efec0c29b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_mean_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_mean_op_xpu.py @@ -18,20 +18,22 @@ import sys sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestMeanOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_mean' self.use_dynamic_create_class = False class TestMeanOp(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -47,7 +49,6 @@ class XPUTestMeanOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class TestMeanOp5D(TestMeanOp): - def setUp(self): super().setUp() self.inputs = { @@ -57,7 +58,6 @@ class XPUTestMeanOp(XPUOpTestWrapper): self.outputs = {'Out': self.inputs['X'].mean(axis=0)} class TestMeanOp6D(TestMeanOp): - def setUp(self): super().setUp() self.inputs = { @@ -67,25 +67,23 @@ class XPUTestMeanOp(XPUOpTestWrapper): self.outputs = {'Out': self.inputs['X'].mean(axis=0)} class TestMeanOp8D(TestMeanOp): - def setUp(self): super().setUp() self.inputs = { - 'X': np.random.random( - (1, 3, 1, 2, 1, 4, 3, 10)).astype(self.dtype) + 'X': np.random.random((1, 3, 1, 2, 1, 4, 3, 10)).astype( + self.dtype + ) } self.attrs = {'dim': (0, 3), 'use_xpu': True} self.outputs = {'Out': self.inputs['X'].mean(axis=(0, 3))} class XPUTestReduce(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_mean' self.use_dynamic_create_class = False class Test1DReduce(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -105,7 +103,6 @@ class XPUTestReduce(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class Test2DReduce0(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [0], 'use_xpu': True} @@ -113,7 +110,6 @@ class XPUTestReduce(XPUOpTestWrapper): self.outputs = {'Out': self.inputs['X'].mean(axis=0)} class Test2DReduce1(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [1], 'use_xpu': True} @@ -123,7 +119,6 @@ class XPUTestReduce(XPUOpTestWrapper): } class Test3DReduce0(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [1], 'use_xpu': True} @@ -133,7 +128,6 @@ class XPUTestReduce(XPUOpTestWrapper): } class Test3DReduce1(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [2], 'use_xpu': True} @@ -143,7 +137,6 @@ class XPUTestReduce(XPUOpTestWrapper): } class Test3DReduce2(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [-2], 'use_xpu': True} @@ -153,7 +146,6 @@ class XPUTestReduce(XPUOpTestWrapper): } class Test3DReduce3(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [1, 2], 'use_xpu': True} @@ -163,7 +155,6 @@ class XPUTestReduce(XPUOpTestWrapper): } class Test6DReduce(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [1, -1], 'use_xpu': True} @@ -175,30 +166,31 @@ class XPUTestReduce(XPUOpTestWrapper): } class TestKeepDimReduce(Test1DReduce): - def setUp(self): super().setUp() self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [1], 'keep_dim': True, 'use_xpu': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), + keepdims=self.attrs['keep_dim'], + ) } class TestKeepDim8DReduce(Test1DReduce): - def setUp(self): super().setUp() self.inputs = { - 'X': np.random.random( - (2, 5, 3, 2, 2, 3, 4, 2)).astype(self.dtype) + 'X': np.random.random((2, 5, 3, 2, 2, 3, 4, 2)).astype( + self.dtype + ) } self.attrs = {'dim': (3, 4, 5), 'keep_dim': True, 'use_xpu': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), + keepdims=self.attrs['keep_dim'], + ) } diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_min_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_min_op_xpu.py index 576f550f239..6639071b3b0 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_min_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_min_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceMinOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_min' class XPUTestReduceMinBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,21 +44,21 @@ class XPUTestReduceMinOp(XPUOpTestWrapper): self.attrs = { 'use_xpu': True, 'reduce_all': self.reduce_all, - 'keep_dim': self.keep_dim + 'keep_dim': self.keep_dim, } self.inputs = {'X': np.random.random(self.shape).astype("float32")} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].min()} else: self.outputs = { - 'Out': - self.inputs['X'].min(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].min( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = False @@ -67,10 +69,9 @@ class XPUTestReduceMinOp(XPUOpTestWrapper): pass class XPUTestReduceMinCase1(XPUTestReduceMinBase): - def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_prod_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_prod_op_xpu.py index 17bd2356eff..ad2cb5143b9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_prod_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_prod_op_xpu.py @@ -21,19 +21,21 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceProdOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_prod' self.use_dynamic_create_class = False class TestXPUReduceProdOp(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -45,21 +47,21 @@ class XPUTestReduceProdOP(XPUOpTestWrapper): self.attrs = { 'dim': self.axis, 'keep_dim': self.keep_dim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.inputs = {'X': np.random.random(self.shape).astype(self.dtype)} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].prod()} else: self.outputs = { - 'Out': - self.inputs['X'].prod(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].prod( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def initTestCase(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) def init_dtype(self): self.dtype = self.in_type @@ -71,84 +73,71 @@ class XPUTestReduceProdOP(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class TestProdOp5D(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (1, 2, 5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestProdOp6D(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (1, 1, 2, 5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestProdOp8D(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (1, 3, 1, 2, 1, 4, 3, 10) self.axis = (0, 3) class Test1DReduce(TestXPUReduceProdOp): - def initTestCase(self): self.shape = 120 - self.axis = (0, ) + self.axis = (0,) class Test2DReduce0(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (20, 10) - self.axis = (0, ) + self.axis = (0,) class Test2DReduce1(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (20, 10) - self.axis = (1, ) + self.axis = (1,) class Test3DReduce0(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (1, ) + self.axis = (1,) class Test3DReduce1(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (2, ) + self.axis = (2,) class Test3DReduce2(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (-2, ) + self.axis = (-2,) class Test3DReduce3(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 7) self.axis = (1, 2) class TestKeepDimReduce(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 10) - self.axis = (1, ) + self.axis = (1,) self.keep_dim = True class TestKeepDim8DReduce(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (2, 5, 3, 2, 2, 3, 4, 2) self.axis = (3, 4, 5) self.keep_dim = True class TestReduceAll(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 2, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_sum_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_sum_op_xpu.py index f3e0287d47a..dc377df7ab1 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_sum_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_sum_op_xpu.py @@ -20,18 +20,20 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceSumOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_sum' class XPUTestReduceSumBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,21 +44,21 @@ class XPUTestReduceSumOp(XPUOpTestWrapper): self.attrs = { 'use_xpu': True, 'reduce_all': self.reduce_all, - 'keep_dim': self.keep_dim + 'keep_dim': self.keep_dim, } self.inputs = {'X': np.random.random(self.shape).astype("float32")} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].sum()} else: self.outputs = { - 'Out': - self.inputs['X'].sum(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = False @@ -67,10 +69,9 @@ class XPUTestReduceSumOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class XPUTestReduceSumCase1(XPUTestReduceSumBase): - def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_refactor_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_refactor_op_xpu.py index a338f04091d..aa875797550 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_refactor_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_refactor_op_xpu.py @@ -23,7 +23,11 @@ from paddle.fluid import core from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -39,7 +43,6 @@ def huber_loss_forward(val, delta): # 1.动态生成不同参数的测试case，wrapper类中必须实现dynamic_create_class方法 # self.use_dynamic_create_class置为True class XPUTestArgsortOp1(XPUOpTestWrapper): - def __init__(self): self.op_name = 'argsort' self.use_dynamic_create_class = True @@ -49,14 +52,14 @@ class XPUTestArgsortOp1(XPUOpTestWrapper): classes = [] for descending in [True, False]: for axis in [0, 1, 2, -1, -2]: - class_name = 'XPUTestArgsortOp_axis_' + str(axis) + '_' + str( - descending) + class_name = ( + 'XPUTestArgsortOp_axis_' + str(axis) + '_' + str(descending) + ) attr_dict = {'init_axis': axis, 'init_descending': descending} classes.append([class_name, attr_dict]) return base_class, classes class TestArgsortOp(XPUOpTest): - def setUp(self): self.op_type = "argsort" self.place = paddle.XPUPlace(0) @@ -65,16 +68,18 @@ class XPUTestArgsortOp1(XPUOpTestWrapper): self.input_shape = (2, 2, 2, 3, 3) self.axis = -1 if not hasattr(self, 'init_axis') else self.init_axis - self.descending = False if not hasattr( - self, 'init_descending') else self.init_descending + self.descending = ( + False + if not hasattr(self, 'init_descending') + else self.init_descending + ) if self.in_type == np.float32: self.x = np.random.random(self.input_shape).astype(self.dtype) else: - self.x = np.random.randint(low=-1000, - high=1000, - size=self.input_shape).astype( - self.dtype) + self.x = np.random.randint( + low=-1000, high=1000, size=self.input_shape + ).astype(self.dtype) self.inputs = {"X": self.x} self.attrs = {"axis": self.axis, "descending": self.descending} self.get_output() @@ -84,13 +89,15 @@ class XPUTestArgsortOp1(XPUOpTestWrapper): if self.descending: self.indices = np.flip( np.argsort(self.x, kind='heapsort', axis=self.axis), - self.axis) + self.axis, + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.sort(self.x, kind='heapsort', axis=self.axis), self.axis + ) else: - self.indices = np.argsort(self.x, - kind='heapsort', - axis=self.axis) + self.indices = np.argsort( + self.x, kind='heapsort', axis=self.axis + ) self.sorted_x = np.sort(self.x, kind='heapsort', axis=self.axis) def test_check_output(self): @@ -99,13 +106,11 @@ class XPUTestArgsortOp1(XPUOpTestWrapper): # 2. 为不同参数的测试case定义一个测试类，self.use_dynamic_create_class需要置为False class XPUTestArgsortOp2(XPUOpTestWrapper): - def __init__(self): self.op_name = 'argsort' self.use_dynamic_create_class = False class TestArgsortOp(XPUOpTest): - def setUp(self): self.op_type = "argsort" self.place = paddle.XPUPlace(0) @@ -119,10 +124,9 @@ class XPUTestArgsortOp2(XPUOpTestWrapper): if self.in_type == np.float32: self.x = np.random.random(self.input_shape).astype(self.dtype) else: - self.x = np.random.randint(low=-1000, - high=1000, - size=self.input_shape).astype( - self.dtype) + self.x = np.random.randint( + low=-1000, high=1000, size=self.input_shape + ).astype(self.dtype) self.inputs = {"X": self.x} self.attrs = {"axis": self.axis, "descending": self.descending} self.get_output() @@ -132,13 +136,15 @@ class XPUTestArgsortOp2(XPUOpTestWrapper): if self.descending: self.indices = np.flip( np.argsort(self.x, kind='heapsort', axis=self.axis), - self.axis) + self.axis, + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.sort(self.x, kind='heapsort', axis=self.axis), self.axis + ) else: - self.indices = np.argsort(self.x, - kind='heapsort', - axis=self.axis) + self.indices = np.argsort( + self.x, kind='heapsort', axis=self.axis + ) self.sorted_x = np.sort(self.x, kind='heapsort', axis=self.axis) def init_inputshape(self): @@ -157,57 +163,46 @@ class XPUTestArgsortOp2(XPUOpTestWrapper): self.descending = False class TestArgsortOpAxis0XPU(TestArgsortOp): - def init_axis(self): self.axis = 0 class TestArgsortOpAxis1XPU(TestArgsortOp): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2XPU(TestArgsortOp): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1XPU(TestArgsortOp): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2XPU(TestArgsortOp): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisXPU(TestArgsortOp): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0XPU(TestArgsortOpAxis0XPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1XPU(TestArgsortOpAxis1XPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2XPU(TestArgsortOpAxis2XPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1XPU(TestArgsortOpAxisNeg1XPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2XPU(TestArgsortOpAxisNeg2XPU): - def init_direction(self): self.descending = True @@ -219,13 +214,11 @@ for stype in support_types: class XPUTestHuberLossOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'huber_loss' self.use_dynamic_create_class = False class TestHuberLossOp(XPUOpTest): - def setUp(self): self.op_type = 'huber_loss' self.place = paddle.XPUPlace(0) @@ -237,11 +230,11 @@ class XPUTestHuberLossOp(XPUOpTestWrapper): def set_inputs(self): shape = self.set_shape() - x = np.random.uniform(0, 1., shape).astype(self.dtype) - y = np.random.uniform(0, 1., shape).astype(self.dtype) + x = np.random.uniform(0, 1.0, shape).astype(self.dtype) + y = np.random.uniform(0, 1.0, shape).astype(self.dtype) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } def set_attrs(self): @@ -251,8 +244,9 @@ class XPUTestHuberLossOp(XPUOpTestWrapper): delta = self.attrs['delta'] shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - delta).astype(self.dtype) + loss = np.vectorize(huber_loss_forward)(residual, delta).astype( + self.dtype + ) self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def set_shape(self): @@ -265,27 +259,24 @@ class XPUTestHuberLossOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("residual")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("residual") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('residual')) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set('residual') + ) class TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (640) + return 640 class TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (10, 10) class TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (10, 10, 1) @@ -293,10 +284,12 @@ class XPUTestHuberLossOp(XPUOpTestWrapper): support_types = get_xpu_op_support_types('huber_loss') for stype in support_types: create_test_class(globals(), XPUTestHuberLossOp, stype) - create_test_class(globals(), - XPUTestHuberLossOp, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestHuberLossOp, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reshape2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reshape2_op_xpu.py index 534fd17bf8c..987aaf54cfa 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reshape2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reshape2_op_xpu.py @@ -21,20 +21,22 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReshapeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "reshape2" self.use_dynamic_create_class = False # situation 1: have shape( list, no tensor), no actual shape(Tensor) class TestReshapeOp(XPUOpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" @@ -55,7 +57,7 @@ class XPUTestReshapeOp(XPUOpTestWrapper): def init_test_output(self): self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype(self.dtype) + 'XShape': np.random.random(self.ori_shape).astype(self.dtype), } def init_attrs(self): @@ -72,14 +74,12 @@ class XPUTestReshapeOp(XPUOpTestWrapper): self.check_grad_with_place(place, ["X"], "Out") class TestReshapeOpDimInfer1(TestReshapeOp): - def init_data(self): self.ori_shape = (5, 25) self.new_shape = (5, -1, 5) self.infered_shape = (5, -1, 5) class TestReshapeOpDimInfer2(TestReshapeOp): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -87,7 +87,6 @@ class XPUTestReshapeOp(XPUOpTestWrapper): # situation 2: have shape(list, no tensor), have actual shape(Tensor) class TestReshapeOpWithInputShape(TestReshapeOp): - def init_data(self): self.ori_shape = (6, 20) self.new_shape = (0, -1, 20) @@ -96,18 +95,17 @@ class XPUTestReshapeOp(XPUOpTestWrapper): def init_test_input(self): self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - "Shape": np.array(self.actual_shape, dtype="int32") + "Shape": np.array(self.actual_shape, dtype="int32"), } def init_test_output(self): self.outputs = { "Out": self.inputs["X"].reshape(self.actual_shape), - 'XShape': np.random.random(self.ori_shape).astype(self.dtype) + 'XShape': np.random.random(self.ori_shape).astype(self.dtype), } # Situation 3: have shape(list, have tensor), no actual shape(Tensor) class TestReshapeOp_attr_ShapeTensor(TestReshapeOp): - def init_data(self): self.ori_shape = (4, 25) self.new_shape = (10, 10) @@ -117,29 +115,30 @@ class XPUTestReshapeOp(XPUOpTestWrapper): def init_test_input(self): shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - 'ShapeTensor': shape_tensor + 'ShapeTensor': shape_tensor, } def init_attrs(self): self.attrs = {'shape': self.shape, "use_xpu": True} - class TestReshapeOpDimInfer1_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor - ): - + class TestReshapeOpDimInfer1_attr_ShapeTensor( + TestReshapeOp_attr_ShapeTensor + ): def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 20) self.infered_shape = (5, -1, 20) self.shape = (5, -1, -1) - class TestReshapeOpDimInfer2_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor - ): - + class TestReshapeOpDimInfer2_attr_ShapeTensor( + TestReshapeOp_attr_ShapeTensor + ): def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -148,7 +147,6 @@ class XPUTestReshapeOp(XPUOpTestWrapper): # Situation 4: have shape(Tensor), no actual shape(Tensor) class TestReshapeOp_attr_OnlyShape(TestReshapeOp): - def init_data(self): self.ori_shape = (4, 25) self.new_shape = (10, 10) @@ -157,14 +155,13 @@ class XPUTestReshapeOp(XPUOpTestWrapper): def init_test_input(self): self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - "Shape": np.array(self.new_shape, dtype="int32") + "Shape": np.array(self.new_shape, dtype="int32"), } def init_attrs(self): self.attrs = {"use_xpu": True} class TestReshapeOpDimInfer1_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): - def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 10) @@ -172,7 +169,6 @@ class XPUTestReshapeOp(XPUOpTestWrapper): self.shape = (5, -1, -1) class TestReshapeOpDimInfer2_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_rmsprop_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_rmsprop_op_xpu.py index 0fb0a855bb0..29b980fa128 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_rmsprop_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_rmsprop_op_xpu.py @@ -22,28 +22,32 @@ import paddle import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def calculate_rmsprop_by_numpy(param, grad, mean_square, moment, learning_rate, - epsilon, decay, momentum): +def calculate_rmsprop_by_numpy( + param, grad, mean_square, moment, learning_rate, epsilon, decay, momentum +): mean_square_out = decay * mean_square + (1 - decay) * grad * grad moment_out = momentum * moment + learning_rate * grad / np.sqrt( - mean_square_out + epsilon) + mean_square_out + epsilon + ) param_out = param - moment_out return param_out, mean_square_out, moment_out class XPUTestRMSPropOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'rmsprop' self.use_dynamic_create_class = False class TestRMSPropOPBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.xpu_version = core.get_xpu_device_version(0) @@ -55,19 +59,25 @@ class XPUTestRMSPropOP(XPUOpTestWrapper): self.dtype = self.in_type self.init_config() - self.param = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) - self.grad = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.param = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) + self.grad = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) self.mean_square = np.random.uniform(0, 1, self.input_shape).astype( - self.dtype) - self.moment = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.dtype + ) + self.moment = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) self.mean_grad = np.random.uniform(-1, 1, self.input_shape).astype( - self.dtype) + self.dtype + ) self.mean_grad_out = np.random.uniform( - -1, 1, self.input_shape).astype(self.dtype) + -1, 1, self.input_shape + ).astype(self.dtype) param_out, mean_square_out, moment_out = calculate_rmsprop_by_numpy( param=self.param, @@ -77,7 +87,8 @@ class XPUTestRMSPropOP(XPUOpTestWrapper): learning_rate=self.learning_rate, epsilon=self.epsilon, decay=self.decay, - momentum=self.momentum) + momentum=self.momentum, + ) self.inputs = { 'Param': self.param, 'Grad': self.grad, @@ -92,22 +103,22 @@ class XPUTestRMSPropOP(XPUOpTestWrapper): 'epsilon': self.epsilon, 'decay': self.decay, 'momentum': self.momentum, - 'centered': - False, # TODO(houj04): when XDNN api supports 'center = True', add more test cases + 'centered': False, # TODO(houj04): when XDNN api supports 'center = True', add more test cases } self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, 'MeanSquareOut': mean_square_out, - 'MeanGradOut': self.mean_grad_out + 'MeanGradOut': self.mean_grad_out, } def init_dtype(self): self.dtype = np.float32 def test_check_output(self): - self.check_output_with_place(self.place, - no_check_set=['MeanGradOut']) + self.check_output_with_place( + self.place, no_check_set=['MeanGradOut'] + ) def init_config(self): self.input_shape = [864] @@ -117,7 +128,6 @@ class XPUTestRMSPropOP(XPUOpTestWrapper): self.momentum = 0.1 class XPUTestRMSProp1(TestRMSPropOPBase): - def init_config(self): self.input_shape = [2, 768] self.learning_rate = np.array([0.002]).astype(self.dtype) @@ -126,7 +136,6 @@ class XPUTestRMSPropOP(XPUOpTestWrapper): self.momentum = 0.1 class XPUTestRMSProp2(TestRMSPropOPBase): - def init_config(self): self.input_shape = [3, 8, 4096] self.learning_rate = np.array([0.005]).astype(self.dtype) @@ -135,7 +144,6 @@ class XPUTestRMSPropOP(XPUOpTestWrapper): self.momentum = 0 class XPUTestRMSProp3(TestRMSPropOPBase): - def init_config(self): self.input_shape = [1024] self.learning_rate = np.array([0.01]).astype(self.dtype) @@ -144,7 +152,6 @@ class XPUTestRMSPropOP(XPUOpTestWrapper): self.momentum = 0.02 class XPUTestRMSProp4(TestRMSPropOPBase): - def init_config(self): self.input_shape = [2, 2, 255] self.learning_rate = np.array([0.0005]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_rnn_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_rnn_op_xpu.py index 6012ca710a9..342123d0c84 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_rnn_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_rnn_op_xpu.py @@ -24,7 +24,11 @@ from op_test_xpu import XPUOpTest sys.path.append("../rnn") from rnn_numpy import LSTM from convert import get_params_for_net -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) random.seed(2) np.set_printoptions(threshold=np.inf) @@ -32,13 +36,11 @@ paddle.enable_static() class XPUTestRNNOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'rnn' self.use_dynamic_create_class = False class TestRNNOp(XPUOpTest): - def setUp(self): self.init_size() self.init_dtype() @@ -55,41 +57,52 @@ class XPUTestRNNOp(XPUOpTestWrapper): self.direction_num = 2 if self.is_bidirec else 1 direction = "bidirectional" if self.is_bidirec else "forward" - input = np.random.uniform(low=-0.1, - high=0.1, - size=(self.seq_length, self.batch_size, - self.input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, + high=0.1, + size=(self.seq_length, self.batch_size, self.input_size), + ).astype(self.dtype) input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 - rnn1 = LSTM(self.input_size, - self.hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - dtype=self.dtype) + rnn1 = LSTM( + self.input_size, + self.hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) - output, (last_hidden, - last_cell) = rnn1(input, - sequence_length=self.sequence_length) + output, (last_hidden, last_cell) = rnn1( + input, sequence_length=self.sequence_length + ) init_h = np.zeros( - (self.num_layers * self.direction_num, self.batch_size, - self.hidden_size)).astype(self.dtype) + ( + self.num_layers * self.direction_num, + self.batch_size, + self.hidden_size, + ) + ).astype(self.dtype) init_c = np.zeros( - (self.num_layers * self.direction_num, self.batch_size, - self.hidden_size)).astype(self.dtype) + ( + self.num_layers * self.direction_num, + self.batch_size, + self.hidden_size, + ) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") self.inputs = { 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h), ('init_c', init_c)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -104,14 +117,16 @@ class XPUTestRNNOp(XPUOpTestWrapper): 'hidden_size': self.hidden_size, 'num_layers': self.num_layers, 'mode': self.mode, - 'is_test': self.is_test + 'is_test': self.is_test, } self.outputs = { 'Out': output, - "State": [('last_hidden', last_hidden), - ('last_cell', last_cell)], + "State": [ + ('last_hidden', last_hidden), + ('last_cell', last_cell), + ], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def init_dtype(self): @@ -124,15 +139,19 @@ class XPUTestRNNOp(XPUOpTestWrapper): def test_check_output(self): self.check_output_with_place( - self.place, atol=0.01, no_check_set=['Reserve', 'DropoutState']) + self.place, atol=0.01, no_check_set=['Reserve', 'DropoutState'] + ) def test_grad(self): if not self.is_test: var_name_list = self.get_weight_names() grad_check_list = ['Input', 'init_h', 'init_c'] grad_check_list.extend(var_name_list) - self.check_grad_with_place(self.place, set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad_with_place( + self.place, + set(grad_check_list), + ['Out', 'last_hidden', 'last_cell'], + ) def init_size(self): self.seq_length = 12 @@ -154,43 +173,36 @@ class XPUTestRNNOp(XPUOpTestWrapper): pass class TestRNNOp1(TestRNNOp): - def set_attrs(self): self.sequence_length = None class TestRNNOp2(TestRNNOp): - def set_attrs(self): self.num_layers = 1 self.is_bidirec = True class TestRNNOp3(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = False class TestRNNOp4(TestRNNOp): - def set_attrs(self): self.num_layers = 3 self.is_bidirec = False class TestRNNOp5(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True class TestRNNOp6(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True self.sequence_length = None class TestRNNOp7(TestRNNOp): - def set_attrs(self): self.num_layers = 3 self.is_bidirec = True @@ -198,10 +210,12 @@ class XPUTestRNNOp(XPUOpTestWrapper): support_types = get_xpu_op_support_types('rnn') for stype in support_types: - create_test_class(globals(), - XPUTestRNNOp, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestRNNOp, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_roi_align_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_roi_align_op_xpu.py index ad63cbb5c0a..f4459632432 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_roi_align_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_roi_align_op_xpu.py @@ -21,19 +21,21 @@ import numpy as np import paddle.fluid.core as core from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestROIAlignOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'roi_align' self.use_dynamic_create_class = False class TestROIAlignOp(XPUOpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -48,7 +50,7 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.continuous_coordinate + 'aligned': self.continuous_coordinate, } self.outputs = {'Out': self.out_data} @@ -62,8 +64,12 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): self.xpu_version = core.get_xpu_device_version(0) # n, c, h, w - self.x_dim = (self.batch_size, self.channels, self.height, - self.width) + self.x_dim = ( + self.batch_size, + self.channels, + self.height, + self.width, + ) self.spatial_scale = 1.0 / 2.0 self.pooled_height = 2 @@ -75,25 +81,51 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): self.continuous_coordinate = bool(np.random.randint(2)) self.x = np.random.random(self.x_dim).astype(self.dtype) - def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, - roi_bin_grid_w, bin_size_h, bin_size_w): + def pre_calc( + self, + x_i, + roi_xmin, + roi_ymin, + roi_bin_grid_h, + roi_bin_grid_w, + bin_size_h, + bin_size_w, + ): count = roi_bin_grid_h * roi_bin_grid_w - bilinear_pos = np.zeros([ - self.channels, self.pooled_height, self.pooled_width, count, 4 - ], np.float32) + bilinear_pos = np.zeros( + [ + self.channels, + self.pooled_height, + self.pooled_width, + count, + 4, + ], + np.float32, + ) bilinear_w = np.zeros( - [self.pooled_height, self.pooled_width, count, 4], np.float32) + [self.pooled_height, self.pooled_width, count, 4], np.float32 + ) for ph in range(self.pooled_width): for pw in range(self.pooled_height): c = 0 for iy in range(roi_bin_grid_h): - y = roi_ymin + ph * bin_size_h + (iy + 0.5) * \ - bin_size_h / roi_bin_grid_h + y = ( + roi_ymin + + ph * bin_size_h + + (iy + 0.5) * bin_size_h / roi_bin_grid_h + ) for ix in range(roi_bin_grid_w): - x = roi_xmin + pw * bin_size_w + (ix + 0.5) * \ - bin_size_w / roi_bin_grid_w - if y < -1.0 or y > self.height or \ - x < -1.0 or x > self.width: + x = ( + roi_xmin + + pw * bin_size_w + + (ix + 0.5) * bin_size_w / roi_bin_grid_w + ) + if ( + y < -1.0 + or y > self.height + or x < -1.0 + or x > self.width + ): continue if y <= 0: y = 0 @@ -114,14 +146,18 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): hy = 1 - ly hx = 1 - lx for ch in range(self.channels): - bilinear_pos[ch, ph, pw, c, 0] = x_i[ch, y_low, - x_low] - bilinear_pos[ch, ph, pw, c, 1] = x_i[ch, y_low, - x_high] - bilinear_pos[ch, ph, pw, c, 2] = x_i[ch, y_high, - x_low] - bilinear_pos[ch, ph, pw, c, 3] = x_i[ch, y_high, - x_high] + bilinear_pos[ch, ph, pw, c, 0] = x_i[ + ch, y_low, x_low + ] + bilinear_pos[ch, ph, pw, c, 1] = x_i[ + ch, y_low, x_high + ] + bilinear_pos[ch, ph, pw, c, 2] = x_i[ + ch, y_high, x_low + ] + bilinear_pos[ch, ph, pw, c, 3] = x_i[ + ch, y_high, x_high + ] bilinear_w[ph, pw, c, 0] = hy * hx bilinear_w[ph, pw, c, 1] = hy * lx bilinear_w[ph, pw, c, 2] = ly * hx @@ -131,8 +167,13 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): def calc_roi_align(self): self.out_data = np.zeros( - (self.rois_num, self.channels, self.pooled_height, - self.pooled_width)).astype(self.dtype) + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ).astype(self.dtype) for i in range(self.rois_num): roi = self.rois[i] @@ -150,15 +191,27 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): roi_height = max(roi_height, 1) bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) - roi_bin_grid_h = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_height / self.pooled_height) - roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_width / self.pooled_width) + roi_bin_grid_h = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_height / self.pooled_height) + ) + roi_bin_grid_w = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_width / self.pooled_width) + ) count = int(roi_bin_grid_h * roi_bin_grid_w) pre_size = count * self.pooled_width * self.pooled_height bilinear_pos, bilinear_w = self.pre_calc( - x_i, roi_xmin, roi_ymin, int(roi_bin_grid_h), - int(roi_bin_grid_w), bin_size_h, bin_size_w) + x_i, + roi_xmin, + roi_ymin, + int(roi_bin_grid_h), + int(roi_bin_grid_w), + bin_size_h, + bin_size_w, + ) for ch in range(self.channels): align_per_bin = (bilinear_pos[ch] * bilinear_w).sum(axis=-1) output_val = align_per_bin.mean(axis=-1) @@ -171,17 +224,20 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.random_integers( 0, - self.height // self.spatial_scale - self.pooled_height) + self.height // self.spatial_scale - self.pooled_height, + ) x2 = np.random.random_integers( - x1 + self.pooled_width, - self.width // self.spatial_scale) + x1 + self.pooled_width, self.width // self.spatial_scale + ) y2 = np.random.random_integers( y1 + self.pooled_height, - self.height // self.spatial_scale) + self.height // self.spatial_scale, + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) @@ -205,7 +261,6 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, {'X'}, 'Out') class TestROIAlignInLodOp(TestROIAlignOp): - def set_data(self): self.init_test_case() self.make_rois() @@ -216,7 +271,7 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } self.attrs = { @@ -224,7 +279,7 @@ class XPUTestROIAlignOp(XPUOpTestWrapper): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.continuous_coordinate + 'aligned': self.continuous_coordinate, } self.outputs = {'Out': self.out_data} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sampling_id_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sampling_id_op_xpu.py index 75bdbff364d..d2e9aa48174 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sampling_id_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sampling_id_op_xpu.py @@ -23,7 +23,6 @@ import paddle class TestSamplingIdShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() x = fluid.layers.data(name='x', shape=[3], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_scale_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_scale_op_xpu.py index d41ba5e59d4..47c95c2158b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_scale_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_scale_op_xpu.py @@ -22,17 +22,19 @@ import paddle from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestScaleOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'scale' self.use_dynamic_create_class = False class TestScaleOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -77,33 +79,27 @@ class XPUTestScaleOp(XPUOpTestWrapper): self.check_output_with_place(place) class TestScaleOp1(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': 3.5} class TestScaleOp2(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': 6.77} class TestScaleOp3(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': -9.19} class TestScaleOp4(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': 0.0} class TestScaleOp5(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': -0.003} class TestScaleApiStatic(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -121,13 +117,11 @@ class TestScaleApiStatic(unittest.TestCase): class TestScaleInplaceApiStatic(TestScaleApiStatic): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) class TestScaleApiDygraph(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -141,7 +135,6 @@ class TestScaleApiDygraph(unittest.TestCase): class TestScaleInplaceApiDygraph(TestScaleApiDygraph): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_scatter_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_scatter_op_xpu.py index a7047014f75..4cd2de7e6a0 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_scatter_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_scatter_op_xpu.py @@ -20,13 +20,17 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper, type_dict_str_to_numpy +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, + type_dict_str_to_numpy, +) paddle.enable_static() class XPUTestScatterOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'scatter' self.use_dynamic_create_class = True @@ -47,7 +51,7 @@ class XPUTestScatterOp(XPUOpTestWrapper): 'init_index_np': index_np, 'init_updates_np': updates_np, 'init_output_np': output_np, - 'test_name': 'case1' + 'test_name': 'case1', } test_data_case.append(data_dict) @@ -62,7 +66,7 @@ class XPUTestScatterOp(XPUOpTestWrapper): 'init_index_np': index_np, 'init_updates_np': updates_np, 'init_output_np': output_np, - 'test_name': 'case2' + 'test_name': 'case2', } test_data_case.append(data_dict) @@ -80,16 +84,21 @@ class XPUTestScatterOp(XPUOpTestWrapper): 'init_index_np': index_np, 'init_updates_np': updates_np, 'init_output_np': output_np, - 'test_name': 'case3' + 'test_name': 'case3', } test_data_case.append(data_dict) for data_dict in test_data_case: for index_type in ['int32', 'int64']: for overwrite in [True, False]: - class_name = 'XPUTestScatterOp_index_type_' + data_dict[ - 'test_name'] + '_' + str(index_type) + '_' + str( - overwrite) + class_name = ( + 'XPUTestScatterOp_index_type_' + + data_dict['test_name'] + + '_' + + str(index_type) + + '_' + + str(overwrite) + ) attr_dict = data_dict attr_dict['index_type'] = type_dict_str_to_numpy[index_type] attr_dict['init_overwrite'] = overwrite @@ -97,13 +106,16 @@ class XPUTestScatterOp(XPUOpTestWrapper): return base_class, classes class TestScatterOp(XPUOpTest): - def setUp(self): self.init_config() - self.index_type = np.int32 if not hasattr( - self, 'index_type') else self.index_type - self.overwrite = True if not hasattr( - self, 'init_overwrite') else self.init_overwrite + self.index_type = ( + np.int32 if not hasattr(self, 'index_type') else self.index_type + ) + self.overwrite = ( + True + if not hasattr(self, 'init_overwrite') + else self.init_overwrite + ) if not hasattr(self, 'init_ref_np'): self.ref_np = np.ones((3, 50)).astype(self.dtype) @@ -120,7 +132,7 @@ class XPUTestScatterOp(XPUOpTestWrapper): self.inputs = { 'X': self.ref_np, 'Ids': self.index_np, - 'Updates': self.updates_np + 'Updates': self.updates_np, } self.attrs = {'overwrite': self.overwrite} self.outputs = {'Out': self.output_np} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sequence_conv_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sequence_conv_op_xpu.py index 77327c529cc..e15bfdbbe58 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_sequence_conv_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sequence_conv_op_xpu.py @@ -27,13 +27,15 @@ paddle.enable_static() np.set_printoptions(threshold=np.inf) -def seqconv(x, - lod, - filter, - context_length, - context_start, - padding_trainable=False, - padding_data=None): +def seqconv( + x, + lod, + filter, + context_length, + context_start, + padding_trainable=False, + padding_data=None, +): [T, M] = x.shape col = np.zeros((T, context_length * M)).astype('float32') offset = [0] @@ -48,67 +50,87 @@ def seqconv(x, out_end = offset[i + 1] if in_begin < offset[i]: pad_size = np.min( - [offset[i] - in_begin, offset[i + 1] - offset[i]]) + [offset[i] - in_begin, offset[i + 1] - offset[i]] + ) if padding_trainable: - sub_w = padding_data[j:j + pad_size, :] - col[offset[i]:offset[i] + pad_size, - j * M:(j + 1) * M] = sub_w + sub_w = padding_data[j : j + pad_size, :] + col[ + offset[i] : offset[i] + pad_size, j * M : (j + 1) * M + ] = sub_w out_begin = offset[i] + pad_size in_begin = offset[i] if in_end > offset[i + 1]: pad_size = np.min( - [in_end - offset[i + 1], offset[i + 1] - offset[i]]) + [in_end - offset[i + 1], offset[i + 1] - offset[i]] + ) if padding_trainable: - sub_w = padding_data[begin_pad + context_start + j - - pad_size:begin_pad + context_start + - j, :] - col[offset[i + 1] - pad_size:offset[i + 1], - j * M:(j + 1) * M] = sub_w + sub_w = padding_data[ + begin_pad + + context_start + + j + - pad_size : begin_pad + + context_start + + j, + :, + ] + col[ + offset[i + 1] - pad_size : offset[i + 1], + j * M : (j + 1) * M, + ] = sub_w in_end = offset[i + 1] out_end = offset[i + 1] - pad_size if in_end <= in_begin: continue in_sub = x[in_begin:in_end, :] - col[out_begin:out_end, j * M:(j + 1) * M] += in_sub + col[out_begin:out_end, j * M : (j + 1) * M] += in_sub return np.dot(col, filter) class XPUTestSequenceConv(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sequence_conv' class TestSeqProject(XPUOpTest): - def setUp(self): self.init_test_case() self.op_type = 'sequence_conv' self.dtype = self.in_type self.use_xpu = True - if self.context_length == 1 \ - and self.context_start == 0 \ - and self.padding_trainable: - print("If context_start is 0 " \ - "and context_length is 1," \ - " padding_trainable should be false.") + if ( + self.context_length == 1 + and self.context_start == 0 + and self.padding_trainable + ): + print( + "If context_start is 0 " + "and context_length is 1," + " padding_trainable should be false." + ) return # one level, batch size x = np.random.uniform( - -6.10907e-05, 0.000104218, - [self.input_size[0], self.input_size[1]]).astype(self.dtype) - w = np.random.uniform(-3.17068e-05, 0.000159822, [ - self.context_length * self.input_size[1], - self.output_represention - ]).astype(self.dtype) + -6.10907e-05, + 0.000104218, + [self.input_size[0], self.input_size[1]], + ).astype(self.dtype) + w = np.random.uniform( + -3.17068e-05, + 0.000159822, + [ + self.context_length * self.input_size[1], + self.output_represention, + ], + ).astype(self.dtype) begin_pad = np.max([0, -self.context_start]) end_pad = np.max([0, self.context_start + self.context_length - 1]) total_pad = begin_pad + end_pad padding_data = np.random.uniform( - 0, 0, [total_pad, self.input_size[1]]).astype(self.dtype) + 0, 0, [total_pad, self.input_size[1]] + ).astype(self.dtype) self.pad_data = padding_data self.inputs = { 'X': (x, self.lod), @@ -128,11 +150,17 @@ class XPUTestSequenceConv(XPUOpTestWrapper): 'contextStart': self.context_start, 'contextLength': self.context_length, 'paddingTrainable': self.padding_trainable, - 'contextStride': self.context_stride + 'contextStride': self.context_stride, } - out = seqconv(x, self.lod, w, self.context_length, - self.context_start, self.padding_trainable, - self.pad_data) + out = seqconv( + x, + self.lod, + w, + self.context_length, + self.context_start, + self.padding_trainable, + self.pad_data, + ) self.outputs = {'Out': out} def test_check_output(self): @@ -144,32 +172,32 @@ class XPUTestSequenceConv(XPUOpTestWrapper): def test_check_grad_padding_data(self): if self.padding_trainable: - self.check_grad(['PaddingData'], - 'Out', - no_grad_set=set(['X', 'Filter'])) + self.check_grad( + ['PaddingData'], 'Out', no_grad_set=set(['X', 'Filter']) + ) def test_check_grad_Filter(self): - self.check_grad(['Filter'], - 'Out', - no_grad_set=set(self.inputs_val_no_f)) + self.check_grad( + ['Filter'], 'Out', no_grad_set=set(self.inputs_val_no_f) + ) def test_check_grad_input_filter(self): if self.padding_trainable: - self.check_grad(['X', 'Filter'], - 'Out', - no_grad_set=set(['PaddingData'])) + self.check_grad( + ['X', 'Filter'], 'Out', no_grad_set=set(['PaddingData']) + ) def test_check_grad_padding_input(self): if self.padding_trainable: - self.check_grad(self.inputs_val_no_f, - 'Out', - no_grad_set=set(['Filter'])) + self.check_grad( + self.inputs_val_no_f, 'Out', no_grad_set=set(['Filter']) + ) def test_check_grad_padding_filter(self): if self.padding_trainable: - self.check_grad(self.inputs_val_no_x, - 'Out', - no_grad_set=set(['X'])) + self.check_grad( + self.inputs_val_no_x, 'Out', no_grad_set=set(['X']) + ) def init_test_case(self): self.input_row = 7 @@ -188,7 +216,6 @@ class XPUTestSequenceConv(XPUOpTestWrapper): self.output_represention = 8 # output feature size class TestSeqProjectCase1(TestSeqProject): - def init_test_case(self): self.input_row = 11 self.context_start = -2 @@ -205,7 +232,6 @@ class XPUTestSequenceConv(XPUOpTestWrapper): self.output_represention = 8 # output feature size class TestSeqProjectCase2Len0(TestSeqProject): - def init_test_case(self): self.input_row = 11 self.context_start = -2 @@ -222,7 +248,6 @@ class XPUTestSequenceConv(XPUOpTestWrapper): self.output_represention = 8 # output feature size class TestSeqProjectCase3(TestSeqProject): - def init_test_case(self): self.input_row = 25 self.context_start = -2 @@ -233,8 +258,11 @@ class XPUTestSequenceConv(XPUOpTestWrapper): self.input_size = [self.input_row, 25] idx = list(range(self.input_size[0])) del idx[0] - offset_lod = [[0] + np.sort(random.sample(idx, 8)).tolist() + - [self.input_size[0]]] + offset_lod = [ + [0] + + np.sort(random.sample(idx, 8)).tolist() + + [self.input_size[0]] + ] self.lod = [[]] # convert from offset-based lod to length-based lod for i in range(len(offset_lod[0]) - 1): @@ -242,7 +270,6 @@ class XPUTestSequenceConv(XPUOpTestWrapper): self.output_represention = 8 # output feature size class TestSeqProjectCase4(TestSeqProject): - def init_test_case(self): self.input_row = 7835 self.input_col = 128 @@ -252,21 +279,139 @@ class XPUTestSequenceConv(XPUOpTestWrapper): self.context_stride = 1 self.input_size = [self.input_row, self.input_col] - offset_lod = [[ - 0, 1, 2, 3, 131, 241, 242, 263, 264, 265, 266, 267, 268, 387, - 515, 516, 644, 645, 772, 794, 922, 923, 924, 944, 945, 1073, - 1074, 1202, 1330, 1458, 1556, 1557, 1558, 1686, 1748, 1876, - 1912, 1913, 1914, 2032, 2066, 2194, 2308, 2309, 2347, 2475, - 2476, 2477, 2478, 2606, 2607, 2735, 2736, 2737, 2738, 2838, - 2966, 2967, 2968, 2969, 3097, 3225, 3353, 3481, 3482, 3520, - 3642, 3643, 3754, 3882, 3883, 4010, 4011, 4012, 4140, 4219, - 4228, 4356, 4357, 4415, 4475, 4476, 4604, 4605, 4606, 4694, - 4695, 4808, 4936, 4961, 4962, 5004, 5132, 5260, 5312, 5440, - 5441, 5569, 5570, 5675, 5676, 5750, 5810, 5811, 5939, 6021, - 6149, 6277, 6278, 6364, 6425, 6519, 6647, 6648, 6739, 6867, - 6995, 6996, 7120, 7223, 7244, 7367, 7407, 7408, 7467, 7595, - 7699, 7827, 7835 - ]] + offset_lod = [ + [ + 0, + 1, + 2, + 3, + 131, + 241, + 242, + 263, + 264, + 265, + 266, + 267, + 268, + 387, + 515, + 516, + 644, + 645, + 772, + 794, + 922, + 923, + 924, + 944, + 945, + 1073, + 1074, + 1202, + 1330, + 1458, + 1556, + 1557, + 1558, + 1686, + 1748, + 1876, + 1912, + 1913, + 1914, + 2032, + 2066, + 2194, + 2308, + 2309, + 2347, + 2475, + 2476, + 2477, + 2478, + 2606, + 2607, + 2735, + 2736, + 2737, + 2738, + 2838, + 2966, + 2967, + 2968, + 2969, + 3097, + 3225, + 3353, + 3481, + 3482, + 3520, + 3642, + 3643, + 3754, + 3882, + 3883, + 4010, + 4011, + 4012, + 4140, + 4219, + 4228, + 4356, + 4357, + 4415, + 4475, + 4476, + 4604, + 4605, + 4606, + 4694, + 4695, + 4808, + 4936, + 4961, + 4962, + 5004, + 5132, + 5260, + 5312, + 5440, + 5441, + 5569, + 5570, + 5675, + 5676, + 5750, + 5810, + 5811, + 5939, + 6021, + 6149, + 6277, + 6278, + 6364, + 6425, + 6519, + 6647, + 6648, + 6739, + 6867, + 6995, + 6996, + 7120, + 7223, + 7244, + 7367, + 7407, + 7408, + 7467, + 7595, + 7699, + 7827, + 7835, + ] + ] self.lod = [[]] # convert from offset-based lod to length-based lod for i in range(len(offset_lod[0]) - 1): @@ -280,19 +425,18 @@ for stype in support_types: class TestSeqConvApi(unittest.TestCase): - def test_api(self): import paddle.fluid as fluid x = fluid.layers.data('x', shape=[32], lod_level=1) - y = fluid.layers.sequence_conv(input=x, - num_filters=2, - filter_size=3, - padding_start=None) + y = fluid.layers.sequence_conv( + input=x, num_filters=2, filter_size=3, padding_start=None + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(10, 32).astype("float32"), [[2, 3, 1, 4]], place) + np.random.rand(10, 32).astype("float32"), [[2, 3, 1, 4]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) ret = exe.run(feed={'x': x_tensor}, fetch_list=[y], return_numpy=False) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sequence_unpad_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sequence_unpad_op_xpu.py index 10979e21c62..d24c98ad6c9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sequence_unpad_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sequence_unpad_op_xpu.py @@ -23,19 +23,21 @@ import paddle.fluid as fluid import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSequenceUnpadOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sequence_unpad' self.use_dynamic_create_class = False class TestSequenceUnpadOp(XPUOpTest): - def setUp(self): self.init_dtype() self.initTestCase() @@ -63,49 +65,44 @@ class XPUTestSequenceUnpadOp(XPUOpTestWrapper): x = np.random.random(self.x_shape).astype(self.dtype) out_lod = [self.length] - out = x[0, 0:self.length[0]] + out = x[0, 0 : self.length[0]] for i in range(1, x.shape[0]): - out = np.append(out, x[i, 0:self.length[i]], axis=0) + out = np.append(out, x[i, 0 : self.length[i]], axis=0) - out_shape = (sum(self.length), ) + out_shape = (sum(self.length),) if len(self.x_shape) == 2: - out_shape = out_shape + (1, ) + out_shape = out_shape + (1,) else: out_shape = out_shape + self.x_shape[2:] self.inputs = { 'X': x, - 'Length': np.array(self.length).astype('int64') + 'Length': np.array(self.length).astype('int64'), } self.outputs = {'Out': (out.reshape(out_shape), out_lod)} class TestSequenceUnpadOp2(TestSequenceUnpadOp): - def initTestCase(self): self.length = [2, 3, 4] self.x_shape = (3, 5, 4, 3) class TestSequenceUnpadOp3(TestSequenceUnpadOp): - def initTestCase(self): self.length = [5, 2, 3, 4] self.x_shape = (4, 5, 3, 3, 6) class TestSequenceUnpadOp4(TestSequenceUnpadOp): - def initTestCase(self): self.length = [5, 5, 5, 5] self.x_shape = (4, 5, 3, 3, 6) class TestSequenceUnpadOp5(TestSequenceUnpadOp): - def initTestCase(self): self.length = [1, 4, 3, 1] self.x_shape = (4, 5, 3, 3, 6) class TestSequenceUnpadOpError(unittest.TestCase): - def test_error(self): """ The type of 'x' in fluid.layers.sequence_unpad must be , but received . diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sgd_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sgd_op_xpu.py index 07f5e524ccf..2c7bb941410 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sgd_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sgd_op_xpu.py @@ -21,17 +21,19 @@ import paddle import paddle.fluid as fluid from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestSgdOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sgd' self.use_dynamic_create_class = False class TestSGDOp(XPUOpTest): - def setUp(self): self.op_type = "sgd" self.dtype = self.in_type @@ -51,7 +53,6 @@ class XPUTestSgdOp(XPUOpTestWrapper): self.check_output_with_place(paddle.XPUPlace(0)) class TestSGDOpCase8X(TestSGDOp): - def conf(self): self.h = 10 self.w = 64 @@ -63,12 +64,11 @@ for stype in support_types: class TestSGDOpWithLargeInput(unittest.TestCase): - def runTest(self): data = fluid.layers.fill_constant(shape=[1], value=128, dtype='int64') - label = fluid.layers.fill_constant(shape=[1, 150], - value=0.5, - dtype='float32') + label = fluid.layers.fill_constant( + shape=[1, 150], value=0.5, dtype='float32' + ) emb = fluid.embedding(input=data, size=(10000, 150), dtype='float32') out = fluid.layers.l2_normalize(x=emb, axis=-1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_shape_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_shape_op_xpu.py index a4128064a7e..1a7a51b8314 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_shape_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_shape_op_xpu.py @@ -18,7 +18,11 @@ import sys sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle from paddle.fluid import core from paddle.fluid.op import Operator @@ -27,13 +31,11 @@ paddle.enable_static() class XPUTestShapeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "shape" self.use_dynamic_create_class = False class TestShapeOp(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.op_type = "shape" @@ -51,32 +53,26 @@ class XPUTestShapeOp(XPUOpTestWrapper): self.check_output_with_place(place) class TestShapeOp1(TestShapeOp): - def config(self): self.shape = [2] class TestShapeOp2(TestShapeOp): - def config(self): self.shape = [1, 2, 3] class TestShapeOp3(TestShapeOp): - def config(self): self.shape = [1, 2, 3, 4] class TestShapeOp4(TestShapeOp): - def config(self): self.shape = [1, 2, 3, 4, 1024] class TestShapeOp5(TestShapeOp): - def config(self): self.shape = [1, 2, 3, 4, 1, 201] class TestShapeWithSelectedRows(unittest.TestCase): - def setUp(self): self.dtype = self.in_type diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sigmoid_cross_entropy_with_logits_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sigmoid_cross_entropy_with_logits_op_xpu.py index e9c68dd1999..6221d4f608f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sigmoid_cross_entropy_with_logits_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sigmoid_cross_entropy_with_logits_op_xpu.py @@ -20,7 +20,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) from scipy.special import logit from scipy.special import expit @@ -29,15 +33,13 @@ paddle.enable_static() class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def __init__(self): self.op_name = "sigmoid_cross_entropy_with_logits" self.use_dynamic_create_class = False class TestSigmoidCrossEntropyWithLogitsOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "sigmoid_cross_entropy_with_logits" @@ -60,13 +62,14 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint( + 0, 2, (batch_size, num_classes) + ).astype(self.dtype), } self.attrs = {'num_classes': num_classes, 'batch_size': batch_size} @@ -85,9 +88,9 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): self.dtype = self.in_type class TestSigmoidCrossEntropyWithLogitsOp2( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = 64 @@ -95,13 +98,14 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): ignore_index = -1 self.ignore_index = ignore_index self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(-1, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint( + -1, 2, (batch_size, num_classes) + ).astype(self.dtype), } self.attrs = {'ignore_index': ignore_index} @@ -117,21 +121,22 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): self.outputs = {'Out': out} class TestSigmoidCrossEntropyWithLogitsOp3( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.uniform( + 0, 1, (batch_size, num_classes) + ).astype(self.dtype), } self.attrs = {'num_classes': num_classes, 'batch_size': batch_size} @@ -145,9 +150,9 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): self.outputs = {'Out': -term1 - term2} class TestSigmoidCrossEntropyWithLogitsOp4( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = 64 @@ -155,13 +160,14 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): ignore_index = -1 self.ignore_index = ignore_index self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(-1, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint( + -1, 2, (batch_size, num_classes) + ).astype(self.dtype), } self.attrs = {'ignore_index': ignore_index, 'normalize': True} @@ -176,26 +182,27 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): out[np.where(self.inputs['Label'] == self.ignore_index)] = 0 if self.attrs['normalize']: out = out / float( - np.where(self.inputs['Label'] != self.ignore_index)[0].size) + np.where(self.inputs['Label'] != self.ignore_index)[0].size + ) self.outputs = {'Out': out} class TestSigmoidCrossEntropyWithLogitsOp5( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, tuple(batch_size + - [num_classes])).astype(self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } self.attrs = {'num_classes': num_classes, 'batch_size': batch_size} @@ -209,22 +216,22 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): self.outputs = {'Out': -term1 - term2} class TestSigmoidCrossEntropyWithLogitsOp6( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def set_inputs(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, tuple(batch_size + - [num_classes])).astype(self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.randint( + 0, 2, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } self.attrs = {'num_classes': num_classes, 'batch_size': batch_size} @@ -238,9 +245,9 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): self.outputs = {'Out': -term1 - term2} class TestSigmoidCrossEntropyWithLogitsNorm( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = [10, 10] @@ -248,14 +255,14 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): ignore_index = -1 self.ignore_index = ignore_index self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.randint( - -1, 2, tuple(batch_size + [num_classes])).astype(self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.randint( + -1, 2, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } self.attrs = {'ignore_index': ignore_index, 'normalize': True} @@ -270,7 +277,8 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): out[np.where(self.inputs['Label'] == self.ignore_index)] = 0 if self.attrs['normalize']: out = out / float( - np.where(self.inputs['Label'] != self.ignore_index)[0].size) + np.where(self.inputs['Label'] != self.ignore_index)[0].size + ) self.outputs = {'Out': out} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sign_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sign_op_xpu.py index 97625292831..b498c5fc3a1 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sign_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sign_op_xpu.py @@ -21,19 +21,21 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSignOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sign' self.use_dynamic_create_class = False class TestSignOPBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -43,8 +45,9 @@ class XPUTestSignOP(XPUOpTestWrapper): self.op_type = 'sign' self.dtype = self.in_type self.init_config() - self.x = np.random.uniform(-10, 10, - self.input_shape).astype(self.dtype) + self.x = np.random.uniform(-10, 10, self.input_shape).astype( + self.dtype + ) self.inputs = {'X': self.x} self.outputs = {'Out': np.sign(self.x)} self.attrs = {'use_xpu': True} @@ -62,22 +65,18 @@ class XPUTestSignOP(XPUOpTestWrapper): self.input_shape = [864] class XPUTestSign1(TestSignOPBase): - def init_config(self): self.input_shape = [2, 768] class XPUTestSign2(TestSignOPBase): - def init_config(self): self.input_shape = [3, 8, 4096] class XPUTestSign3(TestSignOPBase): - def init_config(self): self.input_shape = [1024] class XPUTestSign4(TestSignOPBase): - def init_config(self): self.input_shape = [2, 2, 255] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_slice_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_slice_op_xpu.py index c22bb99b4a9..5a77d9cb51a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_slice_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_slice_op_xpu.py @@ -19,7 +19,11 @@ import unittest sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -27,13 +31,11 @@ paddle.enable_static() # Situation 1: starts(list, no tensor), ends(list, no tensor) # 1.1 without attr(decrease) class XPUTestSliceOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'slice' self.use_dynamic_create_class = False class TestSliceOp(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -46,7 +48,7 @@ class XPUTestSliceOp(XPUOpTestWrapper): 'starts': self.starts, 'ends': self.ends, 'infer_flags': self.infer_flags, - "use_xpu": True + "use_xpu": True, } def config(self): @@ -62,14 +64,16 @@ class XPUTestSliceOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['Input'], 'Out') else: user_defined_grad_outputs = np.random.random( - self.out.shape).astype(self.dtype) + self.out.shape + ).astype(self.dtype) self.check_grad_with_place( - self.place, ['Input'], + self.place, + ['Input'], 'Out', - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) class TestCase1(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-3, 0, 2] @@ -79,7 +83,6 @@ class XPUTestSliceOp(XPUOpTestWrapper): self.out = self.input[-3:3, 0:100, 2:-1, :] class TestCase2(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-3, 0, 2] @@ -91,13 +94,11 @@ class XPUTestSliceOp(XPUOpTestWrapper): # 1.2 with attr(decrease) class XPUTestSliceOp_decs_dim(XPUOpTestWrapper): - def __init__(self): self.op_name = 'slice' self.use_dynamic_create_class = False class TestSliceOp_decs_dim(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -111,7 +112,7 @@ class XPUTestSliceOp_decs_dim(XPUOpTestWrapper): 'ends': self.ends, 'infer_flags': self.infer_flags, 'decrease_axis': self.decrease_axis, - "use_xpu": True + "use_xpu": True, } def config(self): @@ -131,14 +132,16 @@ class XPUTestSliceOp_decs_dim(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['Input'], 'Out') else: user_defined_grad_outputs = np.random.random( - self.out.shape).astype(self.dtype) + self.out.shape + ).astype(self.dtype) self.check_grad_with_place( - self.place, ['Input'], + self.place, + ['Input'], 'Out', - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) class TestSliceOp_decs_dim_2(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [1, 0, 2] @@ -149,7 +152,6 @@ class XPUTestSliceOp_decs_dim(XPUOpTestWrapper): self.out = self.input[1, 0, 2:4, :] class TestSliceOp_decs_dim_3(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1, 0, 2] @@ -160,7 +162,6 @@ class XPUTestSliceOp_decs_dim(XPUOpTestWrapper): self.out = self.input[-1, 0, 2:4, :] class TestSliceOp_decs_dim_4(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype(self.dtype) self.starts = [0, 1, 2, 3] @@ -171,7 +172,6 @@ class XPUTestSliceOp_decs_dim(XPUOpTestWrapper): self.out = self.input[0, 1, 2, 3:4] class TestSliceOp_decs_dim_5(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1] @@ -182,7 +182,6 @@ class XPUTestSliceOp_decs_dim(XPUOpTestWrapper): self.out = self.input[:, :, :, -1] class TestSliceOp_decs_dim_6(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [0, 1, 2, 3] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_softmax_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_softmax_op_xpu.py index a4997c91ffb..736b3b7fbe6 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_softmax_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_softmax_op_xpu.py @@ -19,7 +19,11 @@ import unittest sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) @@ -29,7 +33,7 @@ def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -44,7 +48,6 @@ def ref_softmax(x, axis=None, dtype=None): class XPUTestSoftmaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softmax' self.use_dynamic_create_class = True @@ -56,14 +59,12 @@ class XPUTestSoftmaxOp(XPUOpTestWrapper): axis = [-1, 0, 1] for shape in shapes: for axi in axis: - class_name = 'XPUTestSoftmax_' + \ - str(shape) + "_" + str(axi) + class_name = 'XPUTestSoftmax_' + str(shape) + "_" + str(axi) attr_dict = {'shape': shape, 'axis': axi} classes.append([class_name, attr_dict]) return base_class, classes class TestSoftmaxOp(XPUOpTest): - def setUp(self): self.op_type = "softmax" if not hasattr(self, 'shape'): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_softmax_with_cross_entropy_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_softmax_with_cross_entropy_op_xpu.py index cb604af4f89..f4482e5edd7 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_softmax_with_cross_entropy_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_softmax_with_cross_entropy_op_xpu.py @@ -22,7 +22,11 @@ import paddle import unittest import numpy as np -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): @@ -33,7 +37,7 @@ def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): axis %= len(shape) n = int(np.prod(shape[:axis])) axis_dim = shape[axis] - remain = int(np.prod(shape[axis + 1:])) + remain = int(np.prod(shape[axis + 1 :])) softmax_reshape = softmax.reshape((n, axis_dim, remain)) label_reshape = label.reshape((n, 1, remain)) result = np.zeros_like(label_reshape, dtype=softmax.dtype) @@ -46,7 +50,6 @@ def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): class XPUTestSoftmaxWithCrossEntropyOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softmax_with_cross_entropy' self.use_dynamic_create_class = True @@ -54,26 +57,39 @@ class XPUTestSoftmaxWithCrossEntropyOp(XPUOpTestWrapper): def dynamic_create_class(self): base_class = self.TestSoftmaxWithCrossEntropyOp classes = [] - shapes = [[41, 37], [3, 5, 7, 11], [3, 5, 7, 1], [1023, 38512], - [1, 511]] + shapes = [ + [41, 37], + [3, 5, 7, 11], + [3, 5, 7, 1], + [1023, 38512], + [1, 511], + ] for soft_label in [True, False]: for numeric_stable_mode in [True, False]: for shape in shapes: for logits_type in [0, 1, 2]: for axis in range(len(shape)): - if (not numeric_stable_mode): + if not numeric_stable_mode: axis = -1 - class_name = 'XPUTestSoftmaxWithCrossEntropy_' + \ - str(soft_label) + "_" + \ - str(numeric_stable_mode) + "_" + \ - str(shape) + "_" + \ - str(logits_type) + "_" + \ - str(axis) - attr_dict = {'soft_label': soft_label, \ - 'numeric_stable_mode': numeric_stable_mode, \ - 'shape': shape, \ - 'logits_type': logits_type, - 'axis': axis} + class_name = ( + 'XPUTestSoftmaxWithCrossEntropy_' + + str(soft_label) + + "_" + + str(numeric_stable_mode) + + "_" + + str(shape) + + "_" + + str(logits_type) + + "_" + + str(axis) + ) + attr_dict = { + 'soft_label': soft_label, + 'numeric_stable_mode': numeric_stable_mode, + 'shape': shape, + 'logits_type': logits_type, + 'axis': axis, + } classes.append([class_name, attr_dict]) return base_class, classes @@ -95,8 +111,10 @@ class XPUTestSoftmaxWithCrossEntropyOp(XPUOpTestWrapper): self.soft_label = True self.axis = -1 logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) if self.logits_type == 1: self.logits = np.full(self.shape, -500.0).astype(self.dtype) elif self.logits_type == 2 and len(self.shape) == 4: @@ -105,24 +123,25 @@ class XPUTestSoftmaxWithCrossEntropyOp(XPUOpTestWrapper): softmax = np.apply_along_axis(stable_softmax, self.axis, logits) if self.soft_label: - labels = np.random.uniform(0.1, 1.0, - self.shape).astype(self.dtype) + labels = np.random.uniform(0.1, 1.0, self.shape).astype( + self.dtype + ) labels /= np.sum(labels, axis=self.axis, keepdims=True) else: axis_dim = self.shape[self.axis] self.shape[self.axis] = 1 - labels = np.random.randint(0, - axis_dim, - self.shape, - dtype="int64") + labels = np.random.randint( + 0, axis_dim, self.shape, dtype="int64" + ) - loss = cross_entropy(softmax, labels, self.soft_label, self.axis, - self.ignore_index) + loss = cross_entropy( + softmax, labels, self.soft_label, self.axis, self.ignore_index + ) self.inputs = {"Logits": logits, "Label": labels} self.outputs = { "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -143,9 +162,9 @@ class XPUTestSoftmaxWithCrossEntropyOp(XPUOpTestWrapper): if paddle.is_compiled_with_xpu(): paddle.enable_static() place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ["Logits"], - "Loss", - max_relative_error=0.2) + self.check_grad_with_place( + place, ["Logits"], "Loss", max_relative_error=0.2 + ) support_types = get_xpu_op_support_types('softmax_with_cross_entropy') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_split_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_split_op_xpu.py index 3d6ab9e0481..c7dff56cb62 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_split_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_split_op_xpu.py @@ -20,20 +20,22 @@ import numpy as np from op_test_xpu import XPUOpTest import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSplitOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'split' self.use_dynamic_create_class = False # test with attr(num) class TestSplitOp(XPUOpTest): - def setUp(self): self.init_dtype() self.__class__.use_xpu = True @@ -44,12 +46,13 @@ class XPUTestSplitOp(XPUOpTestWrapper): self.attrs = { 'axis': self.axis, 'sections': self.sections, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = { + 'Out': [('out%d' % i, out[i]) for i in range(len(out))] + } def init_dtype(self): self.dtype = self.in_type @@ -66,7 +69,6 @@ class XPUTestSplitOp(XPUOpTestWrapper): # unknown sections class TestSplitOp1(TestSplitOp): - def initParameters(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) self.axis = 2 @@ -76,7 +78,6 @@ class XPUTestSplitOp(XPUOpTestWrapper): # test with int32 class TestSplitOp2(TestSplitOp): - def initParameters(self): self.x = np.random.random((4, 5, 6)).astype(np.int32) self.axis = 2 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_squeeze2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_squeeze2_op_xpu.py index fdd81c0c138..1b2a52a6962 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_squeeze2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_squeeze2_op_xpu.py @@ -20,20 +20,22 @@ sys.path.append("..") import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestSqueeze2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = "squeeze2" self.use_dynamic_create_class = False class TestSqueeze2Op(XPUOpTest): - def setUp(self): self.op_type = "squeeze2" self.__class__.op_type = "squeeze2" @@ -45,7 +47,7 @@ class XPUTestSqueeze2Op(XPUOpTestWrapper): } self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype(self.dtype) + "XShape": np.random.random(self.ori_shape).astype(self.dtype), } self.init_attrs() @@ -73,7 +75,6 @@ class XPUTestSqueeze2Op(XPUOpTestWrapper): # Correct: There is mins axis. class TestSqueeze2Op1(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -81,7 +82,6 @@ class XPUTestSqueeze2Op(XPUOpTestWrapper): # Correct: No axes input. class TestSqueeze2Op2(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -89,7 +89,6 @@ class XPUTestSqueeze2Op(XPUOpTestWrapper): # Correct: Just part of axes be squeezed. class TestSqueeze2Op3(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_squeeze_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_squeeze_op_xpu.py index f1c6873823f..a87fe32d62e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_squeeze_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_squeeze_op_xpu.py @@ -23,20 +23,22 @@ import paddle import paddle.fluid as fluid from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSqueezeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "squeeze" self.use_dynamic_create_class = False # Correct: General. class TestSqueezeOp(XPUOpTest): - def setUp(self): self.op_type = "squeeze" self.__class__.op_type = "squeeze" @@ -75,7 +77,6 @@ class XPUTestSqueezeOp(XPUOpTestWrapper): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 3, 1, 40) self.axes = (0, -2) @@ -83,7 +84,6 @@ class XPUTestSqueezeOp(XPUOpTestWrapper): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -91,7 +91,6 @@ class XPUTestSqueezeOp(XPUOpTestWrapper): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -99,7 +98,6 @@ class XPUTestSqueezeOp(XPUOpTestWrapper): # Correct: The demension of axis is not of size 1 remains unchanged. class TestSqueezeOp4(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, 2) @@ -107,13 +105,13 @@ class XPUTestSqueezeOp(XPUOpTestWrapper): class TestSqueezeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input type of softmax_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.XPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.XPUPlace(0) + ) self.assertRaises(TypeError, paddle.squeeze, x1) # The input axes of squeeze must be list. x2 = paddle.static.data(name='x2', shape=[4], dtype="int32") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_stack_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_stack_op_xpu.py index 2102348de36..1f215acdb74 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_stack_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_stack_op_xpu.py @@ -21,20 +21,22 @@ from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestStackOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'stack' self.use_dynamic_create_class = False @skip_check_grad_ci(reason="There is no grad kernel for stack_xpu op.") class TestStackOp(XPUOpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -49,7 +51,8 @@ class XPUTestStackOp(XPUOpTestWrapper): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(self.dtype)) + np.random.random(size=self.input_dim).astype(self.dtype) + ) tmp = [] x_names = self.get_x_names() @@ -79,21 +82,19 @@ class XPUTestStackOp(XPUOpTestWrapper): if self.dtype == np.int32 or self.dtype == np.int64: pass else: - self.check_grad_with_place(paddle.XPUPlace(0), - self.get_x_names(), 'Y') + self.check_grad_with_place( + paddle.XPUPlace(0), self.get_x_names(), 'Y' + ) class TestStackOp1(TestStackOp): - def initParameters(self): self.num_inputs = 16 class TestStackOp2(TestStackOp): - def initParameters(self): self.num_inputs = 30 class TestStackOp3(TestStackOp): - def initParameters(self): self.axis = -1 @@ -101,7 +102,6 @@ class XPUTestStackOp(XPUOpTestWrapper): pass class TestStackOp4(TestStackOp): - def initParameters(self): self.axis = -4 @@ -109,17 +109,14 @@ class XPUTestStackOp(XPUOpTestWrapper): pass class TestStackOp5(TestStackOp): - def initParameters(self): self.axis = 1 class TestStackOp6(TestStackOp): - def initParameters(self): self.axis = 3 class TestStackOp7(TestStackOp): - def initParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -130,7 +127,6 @@ class XPUTestStackOp(XPUOpTestWrapper): pass class TestStackOp8(TestStackOp): - def initParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sum_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sum_op_xpu.py index 3d74b224cbb..778cb66b9a8 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sum_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sum_op_xpu.py @@ -22,19 +22,21 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSumOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sum' self.use_dynamic_create_class = False class TestSumOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -66,30 +68,24 @@ class XPUTestSumOp(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['x0'], 'Out') class TestSumOp1(TestSumOp): - def set_shape(self): - self.shape = (5) + self.shape = 5 class TestSumOp2(TestSumOp): - def set_shape(self): self.shape = (1, 1, 1, 1, 1) class TestSumOp3(TestSumOp): - def set_shape(self): self.shape = (10, 5, 7) class TestSumOp4(TestSumOp): - def set_shape(self): self.shape = (2, 2, 3, 3) def create_test_sum_fp16_class(parent): - class TestSumFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -105,15 +101,14 @@ def create_test_sum_fp16_class(parent): class API_Test_Add_n(unittest.TestCase): - def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - input0 = fluid.layers.fill_constant(shape=[2, 3], - dtype='int64', - value=5) - input1 = fluid.layers.fill_constant(shape=[2, 3], - dtype='int64', - value=3) + input0 = fluid.layers.fill_constant( + shape=[2, 3], dtype='int64', value=5 + ) + input1 = fluid.layers.fill_constant( + shape=[2, 3], dtype='int64', value=3 + ) expected_result = np.empty((2, 3)) expected_result.fill(8) sum_value = paddle.add_n([input0, input1]) @@ -132,9 +127,7 @@ class API_Test_Add_n(unittest.TestCase): class TestRaiseSumError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.sum([11, 22]) @@ -155,9 +148,7 @@ class TestRaiseSumError(unittest.TestCase): class TestRaiseSumsError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.sums([11, 22]) @@ -193,9 +184,7 @@ class TestRaiseSumsError(unittest.TestCase): class TestSumOpError(unittest.TestCase): - def test_errors(self): - def test_empty_list_input(): with fluid.dygraph.guard(): fluid._legacy_C_ops.sum([]) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_temporal_shift_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_temporal_shift_op_xpu.py index 0158284c019..5fce6052502 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_temporal_shift_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_temporal_shift_op_xpu.py @@ -22,7 +22,11 @@ import paddle import paddle.nn.functional as F from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) @@ -33,13 +37,14 @@ def temporal_shift(x, seg_num, shift_ratio, data_format): x = np.transpose(x, (0, 3, 1, 2)) shape = x.shape reshape_x = x.reshape((-1, seg_num, shape[1], shape[2], shape[3])) - pad_x = np.pad(reshape_x, ((0, 0), (1, 1), (0, 0), (0, 0), (0, 0)), - 'constant') + pad_x = np.pad( + reshape_x, ((0, 0), (1, 1), (0, 0), (0, 0), (0, 0)), 'constant' + ) c1 = int(shape[1] * shift_ratio) c2 = int(shape[1] * 2 * shift_ratio) slice1 = pad_x[:, :seg_num, :c1, :, :] - slice2 = pad_x[:, 2:seg_num + 2, c1:c2, :, :] - slice3 = pad_x[:, 1:seg_num + 1, c2:, :, :] + slice2 = pad_x[:, 2 : seg_num + 2, c1:c2, :, :] + slice3 = pad_x[:, 1 : seg_num + 1, c2:, :, :] concat_x = np.concatenate([slice1, slice2, slice3], axis=2) out = concat_x.reshape(shape) if data_format == "NHWC": @@ -48,13 +53,11 @@ def temporal_shift(x, seg_num, shift_ratio, data_format): class XPUTestTemporalShiftOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "temporal_shift" self.use_dynamic_create_class = False class TestXPUTemporalShift(XPUOpTest): - def setUp(self): self.initTestCase() self.op_type = 'temporal_shift' @@ -65,15 +68,16 @@ class XPUTestTemporalShiftOp(XPUOpTestWrapper): self.attrs = { "seg_num": self.seg_num, "shift_ratio": self.shift_ratio, - "data_format": self.data_format + "data_format": self.data_format, } self.inputs = { "X": x, } - output = temporal_shift(x, self.seg_num, self.shift_ratio, - self.data_format) + output = temporal_shift( + x, self.seg_num, self.shift_ratio, self.data_format + ) self.outputs = {"Out": output} self.python_out_sig = ["Out"] @@ -91,7 +95,6 @@ class XPUTestTemporalShiftOp(XPUOpTestWrapper): self.data_format = 'NCHW' class TestXPUTemporalShift2(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (1, 1, 1, 1) self.seg_num = 1 @@ -100,7 +103,6 @@ class XPUTestTemporalShiftOp(XPUOpTestWrapper): self.data_format = 'NCHW' class TestXPUTemporalShift3(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (4, 9, 1, 1) self.seg_num = 2 @@ -109,7 +111,6 @@ class XPUTestTemporalShiftOp(XPUOpTestWrapper): self.data_format = 'NCHW' class TestXPUTemporalShift4(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (4, 1, 10, 10) self.seg_num = 2 @@ -118,7 +119,6 @@ class XPUTestTemporalShiftOp(XPUOpTestWrapper): self.data_format = 'NCHW' class TestXPUTemporalShift5(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (1, 1, 1, 1) self.seg_num = 1 @@ -127,7 +127,6 @@ class XPUTestTemporalShiftOp(XPUOpTestWrapper): self.data_format = 'NHWC' class TestXPUTemporalShift6(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (6, 5, 5, 1) self.seg_num = 3 @@ -136,7 +135,6 @@ class XPUTestTemporalShiftOp(XPUOpTestWrapper): self.data_format = 'NHWC' class TestXPUTemporalShift7(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (9, 1, 1, 4) self.seg_num = 3 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_tile_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_tile_op_xpu.py index 1854e164386..ef6c65f71dd 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_tile_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_tile_op_xpu.py @@ -20,21 +20,23 @@ sys.path.append("..") from op_test_xpu import XPUOpTest import paddle import paddle.fluid as fluid -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) -#Situation 1: repeat_times is a list (without tensor) +# Situation 1: repeat_times is a list (without tensor) class XPUTestTileOpRank1(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tile' self.use_dynamic_create_class = False class TestTileOpRank1(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.__class__.no_need_check_grad = True @@ -55,39 +57,33 @@ class XPUTestTileOpRank1(XPUOpTestWrapper): def test_check_output(self): self.check_output_with_place(self.place) - #with dimension expanding + # with dimension expanding class TestTileOpRank2Expanding(TestTileOpRank1): - def init_data(self): self.ori_shape = [120] self.repeat_times = [2, 2] class TestTileOpRank2(TestTileOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] class TestTileOpRank3_Corner(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (1, 1, 1) class TestTileOpRank3_Corner2(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (2, 2) class TestTileOpRank3(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.repeat_times = (2, 1, 4) class TestTileOpRank4(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.repeat_times = (3, 2, 1, 2) @@ -95,13 +91,11 @@ class XPUTestTileOpRank1(XPUOpTestWrapper): # Situation 2: repeat_times is a list (with tensor) class XPUTestTileOpRank1_tensor_attr(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tile' self.use_dynamic_create_class = False class TestTileOpRank1_tensor_attr(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.__class__.no_need_check_grad = True @@ -110,8 +104,9 @@ class XPUTestTileOpRank1_tensor_attr(XPUOpTestWrapper): self.init_data() repeat_times_tensor = [] for index, ele in enumerate(self.repeat_times): - repeat_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + repeat_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -130,14 +125,12 @@ class XPUTestTileOpRank1_tensor_attr(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [1, 1] self.infer_repeat_times = [1, -1] class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -146,13 +139,11 @@ class XPUTestTileOpRank1_tensor_attr(XPUOpTestWrapper): # Situation 3: repeat_times is a tensor class XPUTestTileOpRank1_tensor(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tile' self.use_dynamic_create_class = False class TestTileOpRank1_tensor(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.__class__.no_need_check_grad = True @@ -176,7 +167,6 @@ class XPUTestTileOpRank1_tensor(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestTileOpRank2_tensor(TestTileOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -191,7 +181,6 @@ for stype in support_types: # Test python API class TestTileAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(paddle.XPUPlace(0)): np_x = np.random.random([12, 14]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_top_k_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_top_k_op_xpu.py index 5cd42c91267..fe251b320a6 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_top_k_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_top_k_op_xpu.py @@ -19,7 +19,11 @@ import sys sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -28,23 +32,24 @@ def random_unique_float(row, k, dtype): # create a random float array with 10x length arr = np.random.uniform(-10.0, 10.0, int(row * k * 10)).astype(dtype) arr = np.unique(arr) - assert arr.shape[ - 0] >= row * k, "failed to create enough unique values: %d vs %d" % ( - arr.shape[0], row * k) - arr = arr[:row * k] + assert ( + arr.shape[0] >= row * k + ), "failed to create enough unique values: %d vs %d" % ( + arr.shape[0], + row * k, + ) + arr = arr[: row * k] np.random.shuffle(arr) arr = arr.reshape(row, k) return arr class XPUTestTopkOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'top_k' self.use_dynamic_create_class = False class TestXPUTopkOP(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -85,28 +90,24 @@ class XPUTestTopkOP(XPUOpTestWrapper): self.check_grad_with_place(self.place, ['X'], 'Out') class TestTopk1(TestXPUTopkOP): - def set_case(self): self.variable_k = True self.row = 100 self.top_k = 1 class TestTopk2(TestXPUTopkOP): - def set_case(self): self.variable_k = False self.row = 16 self.top_k = 256 class TestTopk3(TestXPUTopkOP): - def set_case(self): self.variable_k = True self.row = 10 self.top_k = 512 class TestTopk4(TestXPUTopkOP): - def set_case(self): self.variable_k = False self.row = 5 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_top_k_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_top_k_v2_op_xpu.py index 4ed7619bf30..5393834c370 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_top_k_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_top_k_v2_op_xpu.py @@ -19,7 +19,11 @@ import sys sys.path.append("..") from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -41,13 +45,11 @@ def numpy_topk(x, k=1, axis=-1, largest=True): class XPUTestTopKV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'top_k_v2' self.use_dynamic_create_class = False class TestTopkOp(XPUOpTest): - def init_args(self): self.k = 3 self.axis = 1 @@ -62,12 +64,11 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.attrs = { 'k': self.k, 'axis': self.axis, - 'largest': self.largest + 'largest': self.largest, } - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -81,7 +82,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.check_grad(set(['X']), 'Out') class TestTopkOp1(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -89,7 +89,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(100, 155).astype(self.dtype) class TestTopkOp2(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -97,7 +96,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp3(TestTopkOp): - def init_args(self): self.k = 5 self.axis = 1 @@ -105,7 +103,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp4(TestTopkOp): - def init_args(self): self.k = 1 self.axis = 1 @@ -113,7 +110,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp5(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 2 @@ -121,7 +117,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp6(TestTopkOp): - def init_args(self): self.k = 5 self.axis = 1 @@ -129,7 +124,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(8, 32, 64).astype(self.dtype) class TestTopkOp7(TestTopkOp): - def init_args(self): self.k = 10 self.axis = 2 @@ -137,7 +131,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(8, 5, 10, 16).astype(self.dtype) class TestTopkOp8(TestTopkOp): - def init_args(self): self.k = 1 self.axis = 1 @@ -145,7 +138,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(8, 32, 64).astype(self.dtype) class TestTopkOp9(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -153,7 +145,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp10(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -161,7 +152,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp11(TestTopkOp): - def init_args(self): self.k = 5 self.axis = 1 @@ -169,7 +159,6 @@ class XPUTestTopKV2Op(XPUOpTestWrapper): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp12(TestTopkOp): - def init_args(self): self.k = 1 self.axis = 1 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_transpose_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_transpose_op_xpu.py index 3dd7531f350..1261487d393 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_transpose_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_transpose_op_xpu.py @@ -22,7 +22,6 @@ import paddle class TestXPUTransposeOp(XPUOpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -32,11 +31,11 @@ class TestXPUTransposeOp(XPUOpTest): self.attrs = { 'axis': list(self.axis), 'use_mkldnn': False, - 'use_xpu': True + 'use_xpu': True, } self.outputs = { 'XShape': np.random.random(self.shape).astype("float32"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -61,77 +60,66 @@ class TestXPUTransposeOp(XPUOpTest): class TestCase_ZeroDim(TestXPUTransposeOp): - def initTestCase(self): self.shape = () self.axis = () class TestCase0(TestXPUTransposeOp): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1(TestXPUTransposeOp): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase2(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) class TestCase5(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 16, 96) self.axis = (0, 2, 1) class TestCase6(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 10, 12, 16) self.axis = (3, 1, 2, 0) class TestCase7(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 10, 2, 16) self.axis = (0, 1, 3, 2) class TestCase8(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (0, 1, 3, 2, 4, 5, 6, 7) class TestCase9(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_tril_triu_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_tril_triu_op_xpu.py index 5acd5f22b60..a80d6adfb94 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_tril_triu_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_tril_triu_op_xpu.py @@ -20,19 +20,21 @@ import paddle.tensor as tensor import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestTrilTriuOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tril_triu' self.use_dynamic_create_class = False class TestTrilTriuOp(XPUOpTest): - def setUp(self): self.init_dtype() self.initTestCase() @@ -42,9 +44,9 @@ class XPUTestTrilTriuOp(XPUOpTestWrapper): self.op_type = "tril_triu" self.place = paddle.XPUPlace(0) if self.dtype == np.int32: - self.X = np.arange(1, - self.get_Xshape_prod() + 1, - dtype=self.dtype).reshape(self.Xshape) + self.X = np.arange( + 1, self.get_Xshape_prod() + 1, dtype=self.dtype + ).reshape(self.Xshape) else: self.X = np.random.random(self.Xshape).astype(dtype=self.dtype) self.inputs = {'X': self.X} @@ -53,9 +55,9 @@ class XPUTestTrilTriuOp(XPUOpTestWrapper): 'lower': True if self.real_op_type == 'tril' else False, } self.outputs = { - 'Out': - self.real_np_op(self.X, self.diagonal) - if self.diagonal else self.real_np_op(self.X) + 'Out': self.real_np_op(self.X, self.diagonal) + if self.diagonal + else self.real_np_op(self.X) } def init_dtype(self): @@ -78,11 +80,14 @@ class XPUTestTrilTriuOp(XPUOpTestWrapper): def test_check_grad_normal(self): if self.dtype == np.int32: user_defined_grad_outputs = np.random.random( - self.Xshape).astype('float32') + self.Xshape + ).astype('float32') self.check_grad_with_place( - self.place, ['X'], + self.place, + ['X'], 'Out', - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') @@ -91,74 +96,70 @@ class XPUTestTrilTriuOp(XPUOpTestWrapper): self.Xshape = (10, 10) class TestTrilTriuOp1(TestTrilTriuOp): - def initTestCase(self): self.diagonal = -3 self.Xshape = (5, 5) class TestTrilTriuOp2(TestTrilTriuOp): - def initTestCase(self): self.diagonal = 4 self.Xshape = (11, 17) class TestTrilTriuOp3(TestTrilTriuOp): - def initTestCase(self): self.diagonal = 10 self.Xshape = (2, 25, 25) class TestTrilTriuOp4(TestTrilTriuOp): - def initTestCase(self): self.diagonal = -10 self.Xshape = (1, 2, 33, 11) class TestTrilTriuOp5(TestTrilTriuOp): - def initTestCase(self): self.diagonal = 11 self.Xshape = (1, 1, 99) class TestTrilTriuOp6(TestTrilTriuOp): - def initTestCase(self): self.diagonal = 5 self.Xshape = (1, 2, 3, 5, 99) class TestTrilTriuOp7(TestTrilTriuOp): - def initTestCase(self): self.diagonal = -100 self.Xshape = (2, 2, 3, 4, 5) class TestTrilTriuOpError(unittest.TestCase): - def test_errors1(self): paddle.enable_static() data = fluid.data(shape=(20, 22), dtype='float32', name="data1") op_type = np.random.choice(['triu', 'tril']) errmsg = { - "diagonal: TypeError": - "diagonal in {} must be a python Int".format(op_type), + "diagonal: TypeError": "diagonal in {} must be a python Int".format( + op_type + ), } expected = list(errmsg.keys())[0] - with self.assertRaisesRegex(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegex( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal='2022') def test_errors2(self): paddle.enable_static() - data = fluid.data(shape=(200, ), dtype='float32', name="data2") + data = fluid.data(shape=(200,), dtype='float32', name="data2") op_type = np.random.choice(['triu', 'tril']) errmsg = { - "input: ValueError": - "x shape in {} must be at least 2-D".format(op_type), + "input: ValueError": "x shape in {} must be at least 2-D".format( + op_type + ), } expected = list(errmsg.keys())[0] - with self.assertRaisesRegex(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegex( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal=[None]) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_truncated_gaussian_random_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_truncated_gaussian_random_op_xpu.py index 649538aeab2..ea79e9078db 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_truncated_gaussian_random_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_truncated_gaussian_random_op_xpu.py @@ -21,19 +21,21 @@ import paddle import paddle.fluid as fluid from paddle.fluid.executor import Executor from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestTruncatedGaussianRandomOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'truncated_gaussian_random' self.use_dynamic_create_class = False class TestTruncatedGaussianRandomOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -64,9 +66,9 @@ class XPUTestTruncatedGaussianRandomOp(XPUOpTestWrapper): program = fluid.Program() block = program.global_block() vout = block.create_var(name="Out") - op = block.append_op(type=self.op_type, - outputs={"Out": vout}, - attrs=self.attrs) + op = block.append_op( + type=self.op_type, outputs={"Out": vout}, attrs=self.attrs + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) @@ -82,28 +84,24 @@ class XPUTestTruncatedGaussianRandomOp(XPUOpTestWrapper): np.testing.assert_allclose(np.var(tensor), 0.773, atol=0.05) class TestTruncatedGaussianRandomOp_1(TestTruncatedGaussianRandomOp): - def set_attrs(self): self.shape = [4096, 2] self.mean = 5.0 self.std = 1.0 class TestTruncatedGaussianRandomOp_2(TestTruncatedGaussianRandomOp): - def set_attrs(self): self.shape = [1024] self.mean = -2.0 self.std = 1.0 class TestTruncatedGaussianRandomOp_3(TestTruncatedGaussianRandomOp): - def set_attrs(self): self.shape = [11 * 13 * 17] self.mean = -1.0 self.std = 1.0 class TestTruncatedGaussianRandomOp_4(TestTruncatedGaussianRandomOp): - def set_attrs(self): self.shape = [2049] self.mean = 5.1234 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_uniform_random_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_uniform_random_op_xpu.py index 84f949a74bf..4847e9db781 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_uniform_random_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_uniform_random_op_xpu.py @@ -18,13 +18,15 @@ sys.path.append("..") import unittest import numpy as np import paddle -from test_uniform_random_op import TestUniformRandomOp, TestUniformRandomOpSelectedRows +from test_uniform_random_op import ( + TestUniformRandomOp, + TestUniformRandomOpSelectedRows, +) paddle.enable_static() class TestXPUUniformRandomOp(TestUniformRandomOp): - def test_check_output(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) @@ -35,7 +37,6 @@ class TestXPUUniformRandomOp(TestUniformRandomOp): class TestXPUUniformRandomOpSelectedRows(TestUniformRandomOpSelectedRows): - def test_check_output(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze2_op_xpu.py index 743a8189528..c745466fb24 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze2_op_xpu.py @@ -21,19 +21,21 @@ import numpy as np import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestUnsqueeze2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = "unsqueeze2" self.use_dynamic_create_class = False class TestUnsqueeze2Op(XPUOpTest): - def setUp(self): self.op_type = "unsqueeze2" self.__class__.op_type = "unsqueeze2" @@ -45,7 +47,7 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): } self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype(self.dtype) + "XShape": np.random.random(self.ori_shape).astype(self.dtype), } self.init_attrs() @@ -73,15 +75,13 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): # Correct: Single input index. class TestUnsqueeze2Op1(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueeze2Op2(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -89,7 +89,6 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): # Correct: There is duplicated axis. class TestUnsqueeze2Op3(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -97,7 +96,6 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): # Correct: Reversed axes. class TestUnsqueeze2Op4(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -105,7 +103,6 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): # axes is a list(with tensor) class TestUnsqueeze2Op_AxesTensorList(XPUOpTest): - def setUp(self): self.op_type = "unsqueeze2" self.__class__.op_type = "unsqueeze2" @@ -115,17 +112,18 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): axes_tensor_list = [] for index, ele in enumerate(self.axes): - axes_tensor_list.append(("axes" + str(index), np.ones( - (1)).astype('int32') * ele)) + axes_tensor_list.append( + ("axes" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - "AxesTensorList": axes_tensor_list + "AxesTensorList": axes_tensor_list, } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype(self.dtype) + "XShape": np.random.random(self.ori_shape).astype(self.dtype), } def init_dtype(self): @@ -151,28 +149,24 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): self.attrs = {} class TestUnsqueeze2Op1_AxesTensorList(TestUnsqueeze2Op_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueeze2Op2_AxesTensorList(TestUnsqueeze2Op_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) self.new_shape = (1, 20, 5, 1) class TestUnsqueeze2Op3_AxesTensorList(TestUnsqueeze2Op_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) self.new_shape = (1, 10, 2, 1, 1, 5) class TestUnsqueeze2Op4_AxesTensorList(TestUnsqueeze2Op_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -180,7 +174,6 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): # axes is a Tensor class TestUnsqueeze2Op_AxesTensor(XPUOpTest): - def setUp(self): self.op_type = "unsqueeze2" self.__class__.op_type = "unsqueeze2" @@ -190,12 +183,12 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - "AxesTensor": np.array(self.axes).astype("int32") + "AxesTensor": np.array(self.axes).astype("int32"), } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype(self.dtype) + "XShape": np.random.random(self.ori_shape).astype(self.dtype), } def init_dtype(self): @@ -221,28 +214,24 @@ class XPUTestUnsqueeze2Op(XPUOpTestWrapper): self.attrs = {} class TestUnsqueeze2Op1_AxesTensor(TestUnsqueeze2Op_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueeze2Op2_AxesTensor(TestUnsqueeze2Op_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) self.new_shape = (1, 20, 5, 1) class TestUnsqueeze2Op3_AxesTensor(TestUnsqueeze2Op_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) self.new_shape = (1, 10, 2, 1, 1, 5) class TestUnsqueeze2Op4_AxesTensor(TestUnsqueeze2Op_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze_op_xpu.py index 6340610a67d..1f0cb53500f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze_op_xpu.py @@ -21,20 +21,22 @@ import numpy as np import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() # Correct: General. class XPUTestUnsqueezeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "unsqueeze" self.use_dynamic_create_class = False class TestUnsqueezeOp(XPUOpTest): - def setUp(self): self.op_type = "unsqueeze" self.__class__.op_type = "unsqueeze" @@ -70,15 +72,13 @@ class XPUTestUnsqueezeOp(XPUOpTestWrapper): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -86,7 +86,6 @@ class XPUTestUnsqueezeOp(XPUOpTestWrapper): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -94,7 +93,6 @@ class XPUTestUnsqueezeOp(XPUOpTestWrapper): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py index 3b6136cd346..1c8715b93cd 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py @@ -23,19 +23,21 @@ import paddle.fluid as fluid import paddle.fluid.contrib.mixed_precision.amp_nn as amp_nn from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestUpdateLossScalingOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "update_loss_scaling" self.use_dynamic_create_class = False class TestUpdateLossScalingOp(XPUOpTest): - def setUp(self): self.op_type = "update_loss_scaling" self.init() @@ -47,14 +49,14 @@ class XPUTestUpdateLossScalingOp(XPUOpTestWrapper): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', x)], 'LossScaling': self.prev_loss_scaling * self.incr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def init(self): @@ -78,7 +80,6 @@ class XPUTestUpdateLossScalingOp(XPUOpTestWrapper): self.check_output_with_place(place, no_check_set=['Out']) class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): - def setUp(self): self.op_type = "update_loss_scaling" self.init() @@ -93,38 +94,37 @@ class XPUTestUpdateLossScalingOp(XPUOpTestWrapper): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', np.zeros_like(x))], 'LossScaling': self.prev_loss_scaling * self.decr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def test_check_output(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_output_with_place(place) - #self.check_output() + # self.check_output() class TestUpdateLossScalingLayer(unittest.TestCase): - def loss_scaling_check(self, scope=fluid.Scope()): a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -138,33 +138,41 @@ class XPUTestUpdateLossScalingOp(XPUOpTestWrapper): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = fluid.XPUPlace(0) exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], a_v) assert np.array_equal(result_v[1], b_v) assert np.array_equal(result_v[0], result_v[2]) @@ -179,15 +187,15 @@ class XPUTestUpdateLossScalingOp(XPUOpTestWrapper): b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -204,33 +212,41 @@ class XPUTestUpdateLossScalingOp(XPUOpTestWrapper): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = fluid.XPUPlace(0) exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], np.zeros_like(a_v)) assert np.array_equal(result_v[1], np.zeros_like(b_v)) assert np.array_equal(result_v[2], np.zeros_like(a_v)) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_where_index_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_where_index_xpu.py index 6c38d74ccea..8123bcd73f9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_where_index_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_where_index_xpu.py @@ -23,18 +23,20 @@ import paddle.fluid as fluid from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestWhereIndexOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'where_index' class TestWhereIndexOp(XPUOpTest): - def setUp(self): self.init_config() self.init_data() @@ -55,7 +57,6 @@ class XPUTestWhereIndexOp(XPUOpTestWrapper): self.__class__.no_need_check_grad = True class TestAllFalse(TestWhereIndexOp): - def init_data(self): self.inputs = { 'Condition': np.array([False, False, False]).astype(self.dtype), @@ -63,29 +64,31 @@ class XPUTestWhereIndexOp(XPUOpTestWrapper): self.outputs = {'Out': np.array([], dtype='int64')} class TestRank2(TestWhereIndexOp): - def init_data(self): self.inputs = { - 'Condition': - np.array([[True, False], [False, True]]).astype(self.dtype), + 'Condition': np.array([[True, False], [False, True]]).astype( + self.dtype + ), } self.outputs = {'Out': np.array([[0, 0], [1, 1]], dtype='int64')} class TestRank3(TestWhereIndexOp): - def init_data(self): self.inputs = { - 'Condition': - np.array([[[True, False], [False, True]], - [[False, True], [True, False]], - [[False, False], [False, True]]]).astype(self.dtype), + 'Condition': np.array( + [ + [[True, False], [False, True]], + [[False, True], [True, False]], + [[False, False], [False, True]], + ] + ).astype(self.dtype), } self.outputs = { - 'Out': - np.array( + 'Out': np.array( [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], - dtype='int64') + dtype='int64', + ) } @@ -95,7 +98,6 @@ for stype in support_types: class TestWhereOpError(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): cond = fluid.layers.data(name='cond', shape=[4], dtype='bool') @@ -108,9 +110,7 @@ class TestWhereOpError(unittest.TestCase): class TestWhereRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.where([10]) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_where_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_where_op_xpu.py index 018079109c9..18af22f3c64 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_where_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_where_op_xpu.py @@ -23,18 +23,20 @@ import paddle.fluid as fluid from paddle.fluid.backward import append_backward from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestWhereOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'where' class TestXPUWhereOp(XPUOpTest): - def setUp(self): self.init_config() self.init_data() @@ -56,19 +58,18 @@ class XPUTestWhereOp(XPUOpTestWrapper): self.check_output_with_place(self.place) class TestXPUWhereOp2(TestXPUWhereOp): - def init_data(self): self.x = np.random.uniform(-5, 5, (60, 2)).astype(self.dtype) self.y = np.random.uniform(-5, 5, (60, 2)).astype(self.dtype) self.cond = np.ones((60, 2)).astype("bool") class TestXPUWhereOp3(TestXPUWhereOp): - def init_data(self): self.x = np.random.uniform(-3, 5, (20, 2, 4)).astype(self.dtype) self.y = np.random.uniform(-3, 5, (20, 2, 4)).astype(self.dtype) - self.cond = np.array(np.random.randint(2, size=(20, 2, 4)), - dtype=bool) + self.cond = np.array( + np.random.randint(2, size=(20, 2, 4)), dtype=bool + ) support_types = get_xpu_op_support_types('where') @@ -77,7 +78,6 @@ for stype in support_types: class TestXPUWhereAPI(unittest.TestCase): - def setUp(self): self.__class__.use_xpu = True self.place = paddle.XPUPlace(0) @@ -102,9 +102,9 @@ class TestXPUWhereAPI(unittest.TestCase): train_prog = fluid.Program() startup = fluid.Program() with fluid.program_guard(train_prog, startup): - cond = fluid.data(name='cond', - shape=self.shape, - dtype='bool') + cond = fluid.data( + name='cond', shape=self.shape, dtype='bool' + ) x = fluid.data(name='x', shape=self.shape, dtype='float32') y = fluid.data(name='y', shape=self.shape, dtype='float32') @@ -122,24 +122,25 @@ class TestXPUWhereAPI(unittest.TestCase): fetch_list.append(x.grad_name) if y_stop_gradient is False: fetch_list.append(y.grad_name) - out = exe.run(train_prog, - feed={ - 'cond': self.cond, - 'x': self.x, - 'y': self.y - }, - fetch_list=fetch_list) + out = exe.run( + train_prog, + feed={'cond': self.cond, 'x': self.x, 'y': self.y}, + fetch_list=fetch_list, + ) assert np.array_equal(out[0], self.out) if x_stop_gradient is False: - assert np.array_equal(out[2], - self.ref_x_backward(out[1])) + assert np.array_equal( + out[2], self.ref_x_backward(out[1]) + ) if y.stop_gradient is False: - assert np.array_equal(out[3], - self.ref_y_backward(out[1])) + assert np.array_equal( + out[3], self.ref_y_backward(out[1]) + ) elif y.stop_gradient is False: - assert np.array_equal(out[2], - self.ref_y_backward(out[1])) + assert np.array_equal( + out[2], self.ref_y_backward(out[1]) + ) def test_api_broadcast(self, use_cuda=False): train_prog = fluid.Program() @@ -148,24 +149,21 @@ class TestXPUWhereAPI(unittest.TestCase): x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32') y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32') x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype("float32") - y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, - 1.0]]).astype("float32") + y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]).astype( + "float32" + ) result = paddle.where(x > 1, x=x, y=y) exe = fluid.Executor(self.place) exe.run(startup) - out = exe.run(train_prog, - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + train_prog, feed={'x': x_i, 'y': y_i}, fetch_list=[result] + ) assert np.array_equal(out[0], np.where(x_i > 1, x_i, y_i)) class TestWhereDygraphAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(paddle.XPUPlace(0)): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_while_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_while_op_xpu.py index 80fe5f8c144..c1fa366cc88 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_while_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_while_op_xpu.py @@ -24,20 +24,16 @@ paddle.enable_static() class TestWhileOp(unittest.TestCase): - def simple_net(self): - d0 = layers.data("d0", - shape=[10], - append_batch_size=False, - dtype='float32') - d1 = layers.data("d1", - shape=[10], - append_batch_size=False, - dtype='float32') - d2 = layers.data("d2", - shape=[10], - append_batch_size=False, - dtype='float32') + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32' + ) + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32' + ) + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32' + ) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = True init = layers.zeros(shape=[10], dtype='float32') @@ -95,12 +91,10 @@ class TestWhileOp(unittest.TestCase): for i in range(3): d.append(numpy.random.random(size=[10]).astype('float32')) - outs = exe.run(feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2] - }, - fetch_list=[sum_result]) + outs = exe.run( + feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, + fetch_list=[sum_result], + ) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) def test_simple_net_forward(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_xpu_place.py b/python/paddle/fluid/tests/unittests/xpu/test_xpu_place.py index 3576bbdaed3..75679047301 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_xpu_place.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_xpu_place.py @@ -21,7 +21,6 @@ import paddle.static as static class Test_XPU_Places(unittest.TestCase): - def assert_places_equal(self, places0, places1): self.assertEqual(len(places0), len(places1)) for place0, place1 in zip(places0, places1): diff --git a/python/paddle/fluid/trainer_desc.py b/python/paddle/fluid/trainer_desc.py index c4c17c7095a..f8151297c2d 100644 --- a/python/paddle/fluid/trainer_desc.py +++ b/python/paddle/fluid/trainer_desc.py @@ -17,8 +17,12 @@ import sys import os __all__ = [ - 'TrainerDesc', 'MultiTrainer', 'DistMultiTrainer', 'PipelineTrainer', - 'HeterXpuTrainer', 'HeterPipelineTrainer' + 'TrainerDesc', + 'MultiTrainer', + 'DistMultiTrainer', + 'PipelineTrainer', + 'HeterXpuTrainer', + 'HeterPipelineTrainer', ] @@ -43,8 +47,10 @@ class TrainerDesc(object): sys.path.append(cur_path + "/proto") from proto import trainer_desc_pb2 + self.proto_desc = trainer_desc_pb2.TrainerDesc() import multiprocessing as mp + # set default thread num == cpu count self.proto_desc.thread_num = mp.cpu_count() self._fleet_desc = None @@ -53,26 +59,26 @@ class TrainerDesc(object): self._infer = False def _set_heter_info(self, ret): - #ret = = fu.split_program_by_device(program) - #start_list, end_list, send_list, recv_list, program_list = fu.split_program_by_device(program) - #if len(start_list) != 3: + # ret = = fu.split_program_by_device(program) + # start_list, end_list, send_list, recv_list, program_list = fu.split_program_by_device(program) + # if len(start_list) != 3: # print("start_list len=", len(start_list), " will not set heter info") # return - #for i in start_list[0]: + # for i in start_list[0]: # self.proto_desc.op_run_start_idx.append(i) - #for i in end_list[0]: + # for i in end_list[0]: # self.proto_desc.op_run_end_idx.append(i) - #for i in send_list[0]: + # for i in send_list[0]: # self.proto_desc.op_run_send_list.append(i) - #for i in recv_list[0]: + # for i in recv_list[0]: # self.proto_desc.op_run_recv_list.append(i) if ret is None: return - #for i in ret[0]: # start_list[1]: + # for i in ret[0]: # start_list[1]: # self.proto_desc.xpu_start_idx.append(i) self.proto_desc.xpu_start_idx = ret[0] - #for i in ret[1]: #end_list[1]: + # for i in ret[1]: #end_list[1]: # self.proto_desc.o_end_idx.append(i) self.proto_desc.xpu_end_idx = ret[1] for i in ret[2]: # send_list[1]: @@ -80,13 +86,13 @@ class TrainerDesc(object): for i in ret[3]: # recv_list[1]: self.proto_desc.xpu_recv_list.append(i) - #for i in start_list[2]: + # for i in start_list[2]: # self.proto_desc.op_run_end_start_idx.append(i) - #for i in end_list[2]: + # for i in end_list[2]: # self.proto_desc.op_run_end_idx.append(i) - #for i in send_list[2]: + # for i in send_list[2]: # self.proto_desc.op_run_end_send_list.append(i) - #for i in recv_list[2]: + # for i in recv_list[2]: # self.proto_desc.op_run_end_recv_list.append(i) def _set_fetch_var_and_info(self, fetch_vars, fetch_info, print_period): @@ -95,7 +101,8 @@ class TrainerDesc(object): for i, v in enumerate(fetch_vars): self.proto_desc.fetch_config.fetch_var_names.extend([v.name]) self.proto_desc.fetch_config.fetch_var_str_format.extend( - [fetch_info[i]]) + [fetch_info[i]] + ) self.proto_desc.fetch_config.print_period = print_period def _set_debug(self, debug): @@ -114,6 +121,7 @@ class TrainerDesc(object): self._fleet_desc = fleet_desc ## serialize fleet_desc from google.protobuf import text_format + fleet_desc_str = text_format.MessageToString(fleet_desc) self.proto_desc.fleet_desc = fleet_desc_str @@ -137,8 +145,11 @@ class TrainerDesc(object): self.proto_desc.no_cvm = no_cvm def _set_scale_sparse_grad_with_batch_size( - self, scale_sparse_gradient_with_batch_size=True): - self.proto_desc.scale_sparse_gradient_with_batch_size = scale_sparse_gradient_with_batch_size + self, scale_sparse_gradient_with_batch_size=True + ): + self.proto_desc.scale_sparse_gradient_with_batch_size = ( + scale_sparse_gradient_with_batch_size + ) def _set_scale_datanorm(self, scale_datanorm=-1): self.proto_desc.scale_datanorm = scale_datanorm @@ -203,16 +214,21 @@ class TrainerDesc(object): self.proto_desc.loss_names.append(loss) def _set_adjust_ins_weight(self, config_dict): - self.proto_desc.adjust_ins_weight_config.need_adjust = \ - config_dict.get("need_adjust", False) - self.proto_desc.adjust_ins_weight_config.nid_slot = \ - config_dict.get("nid_slot", "") - self.proto_desc.adjust_ins_weight_config.nid_adjw_threshold = \ - config_dict.get("nid_adjw_threshold", 0.0) - self.proto_desc.adjust_ins_weight_config.nid_adjw_ratio = \ - config_dict.get("nid_adjw_ratio", 0.0) - self.proto_desc.adjust_ins_weight_config.ins_weight_slot = \ - config_dict.get("ins_weight_slot", "") + self.proto_desc.adjust_ins_weight_config.need_adjust = config_dict.get( + "need_adjust", False + ) + self.proto_desc.adjust_ins_weight_config.nid_slot = config_dict.get( + "nid_slot", "" + ) + self.proto_desc.adjust_ins_weight_config.nid_adjw_threshold = ( + config_dict.get("nid_adjw_threshold", 0.0) + ) + self.proto_desc.adjust_ins_weight_config.nid_adjw_ratio = ( + config_dict.get("nid_adjw_ratio", 0.0) + ) + self.proto_desc.adjust_ins_weight_config.ins_weight_slot = ( + config_dict.get("ins_weight_slot", "") + ) def _set_copy_table_config(self, config_dict): config = self.proto_desc.copy_table_config @@ -227,9 +243,9 @@ class TrainerDesc(object): dest_sparse_tables = [dest_sparse_tables] if len(src_sparse_tables) != len(dest_sparse_tables): raise ValueError( - "len(src_sparse_tables) != len(dest_sparse_tables)," \ - " %s vs %s" % (len(src_sparse_tables), \ - len(dest_sparse_tables))) + "len(src_sparse_tables) != len(dest_sparse_tables)," + " %s vs %s" % (len(src_sparse_tables), len(dest_sparse_tables)) + ) for i in src_sparse_tables: config.src_sparse_tables.append(i) for i in dest_sparse_tables: @@ -243,9 +259,9 @@ class TrainerDesc(object): dest_dense_tables = [dest_dense_tables] if len(src_dense_tables) != len(dest_dense_tables): raise ValueError( - "len(src_dense_tables) != len(dest_dense_tables)," \ - " %s vs %s" % (len(src_dense_tables), \ - len(dest_dense_tables))) + "len(src_dense_tables) != len(dest_dense_tables)," + " %s vs %s" % (len(src_dense_tables), len(dest_dense_tables)) + ) for i in src_dense_tables: config.src_dense_tables.append(i) for i in dest_dense_tables: @@ -261,8 +277,9 @@ class TrainerDesc(object): dest_var_list = [dest_var_list] if len(src_var_list) != len(dest_var_list): raise ValueError( - "len(src_var_list) != len(dest_var_list), %s vs" \ - " %s" % (len(src_var_list), len(dest_var_list))) + "len(src_var_list) != len(dest_var_list), %s vs" + " %s" % (len(src_var_list), len(dest_var_list)) + ) for i in src_var_list: config.src_var_list.append(i) for i in dest_var_list: @@ -279,19 +296,22 @@ class TrainerDesc(object): raise ValueError("dependency len %s != 1" % len(values)) for value in values: m.values.append(value) - config.dense_pull_after_copy = \ - config_dict.get("dense_pull_after_copy", True) - config.enable_dependency = \ - config_dict.get("enable_dependency", False) - config.sparse_copy_by_feasign = \ - config_dict.get("sparse_copy_by_feasign", True) + config.dense_pull_after_copy = config_dict.get( + "dense_pull_after_copy", True + ) + config.enable_dependency = config_dict.get("enable_dependency", False) + config.sparse_copy_by_feasign = config_dict.get( + "sparse_copy_by_feasign", True + ) def _desc(self): from google.protobuf import text_format + return self.proto_desc.SerializeToString() def __str__(self): from google.protobuf import text_format + return text_format.MessageToString(self.proto_desc) diff --git a/python/paddle/fluid/trainer_factory.py b/python/paddle/fluid/trainer_factory.py index 3ba9f9eea46..7ac367b38fd 100644 --- a/python/paddle/fluid/trainer_factory.py +++ b/python/paddle/fluid/trainer_factory.py @@ -19,12 +19,26 @@ import logging import numpy as np from paddle.fluid.log_helper import get_logger -local_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') - -from .trainer_desc import MultiTrainer, DistMultiTrainer, PipelineTrainer, HeterXpuTrainer, PSGPUTrainer, HeterPipelineTrainer -from .device_worker import Hogwild, DownpourSGD, DownpourLite, Section, DownpourSGDOPT, HeterSection +local_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) + +from .trainer_desc import ( + MultiTrainer, + DistMultiTrainer, + PipelineTrainer, + HeterXpuTrainer, + PSGPUTrainer, + HeterPipelineTrainer, +) +from .device_worker import ( + Hogwild, + DownpourSGD, + DownpourLite, + Section, + DownpourSGDOPT, + HeterSection, +) from .framework import Variable from multiprocessing import Process, Manager @@ -67,18 +81,24 @@ class TrainerFactory(object): trainer._set_mpi_rank(opt_info["mpi_rank"]) if opt_info.get("mpi_size") is not None: trainer._set_mpi_size(opt_info["mpi_size"]) - if opt_info.get("dump_fields") is not None and len( - opt_info.get("dump_fields")) != 0: + if ( + opt_info.get("dump_fields") is not None + and len(opt_info.get("dump_fields")) != 0 + ): trainer._set_dump_fields(opt_info["dump_fields"]) - if opt_info.get("dump_fields_path") is not None and len( - opt_info.get("dump_fields_path")) != 0: + if ( + opt_info.get("dump_fields_path") is not None + and len(opt_info.get("dump_fields_path")) != 0 + ): trainer._set_dump_fields_path(opt_info["dump_fields_path"]) if opt_info.get("dump_file_num") is not None: trainer._set_dump_file_num(opt_info["dump_file_num"]) if opt_info.get("dump_converter") is not None: trainer._set_dump_converter(opt_info["dump_converter"]) - if opt_info.get("dump_param") is not None and len( - opt_info.get("dump_param")) != 0: + if ( + opt_info.get("dump_param") is not None + and len(opt_info.get("dump_param")) != 0 + ): trainer._set_dump_param(opt_info["dump_param"]) if opt_info.get("worker_places") is not None: trainer._set_worker_places(opt_info["worker_places"]) @@ -86,15 +106,18 @@ class TrainerFactory(object): trainer._set_use_ps_gpu(opt_info["use_ps_gpu"]) if opt_info.get("is_dump_in_simple_mode") is not None: trainer._set_is_dump_in_simple_mode( - opt_info["is_dump_in_simple_mode"]) + opt_info["is_dump_in_simple_mode"] + ) if opt_info.get("enable_random_dump") is not None: trainer._set_enable_random_dump( - opt_info["enable_random_dump"]) + opt_info["enable_random_dump"] + ) if opt_info.get("dump_interval") is not None: trainer._set_dump_interval(opt_info["dump_interval"]) if opt_info.get("random_with_lineid") is not None: trainer._set_random_with_lineid( - opt_info["random_with_lineid"]) + opt_info["random_with_lineid"] + ) if "fleet_desc" in opt_info: device_worker._set_fleet_desc(opt_info["fleet_desc"]) @@ -103,20 +126,25 @@ class TrainerFactory(object): trainer._set_use_cvm(opt_info["use_cvm"]) if opt_info.get("no_cvm") is not None: trainer._set_no_cvm(opt_info["no_cvm"]) - if opt_info.get( - "scale_sparse_gradient_with_batch_size") is not None: + if ( + opt_info.get("scale_sparse_gradient_with_batch_size") + is not None + ): trainer._set_scale_sparse_grad_with_batch_size( - opt_info["scale_sparse_gradient_with_batch_size"]) + opt_info["scale_sparse_gradient_with_batch_size"] + ) if opt_info.get("scale_datanorm") is not None: trainer._set_scale_datanorm(opt_info["scale_datanorm"]) if opt_info.get("adjust_ins_weight") is not None: trainer._set_adjust_ins_weight( - opt_info["adjust_ins_weight"]) + opt_info["adjust_ins_weight"] + ) if opt_info.get("copy_table") is not None: trainer._set_copy_table_config(opt_info["copy_table"]) if opt_info.get("check_nan_var_names") is not None: trainer._set_check_nan_var_names( - opt_info["check_nan_var_names"]) + opt_info["check_nan_var_names"] + ) if opt_info.get("loss_names") is not None: trainer._set_loss_names(opt_info["loss_names"]) trainer._set_device_worker(device_worker) @@ -131,8 +159,9 @@ class FetchHandlerMonitor(object): def __init__(self, scope, handler): self.fetch_instance = handler - self.fetch_thread = threading.Thread(target=self.handler_launch_func, - args=(scope, self.fetch_instance)) + self.fetch_thread = threading.Thread( + target=self.handler_launch_func, args=(scope, self.fetch_instance) + ) self.running_lock = threading.Lock() self.running = False @@ -145,7 +174,8 @@ class FetchHandlerMonitor(object): var_name_to_key[fetch_instance.var_dict[key].name] = key else: local_logger.warning( - "the value of {} is not a Variable".format(key)) + "the value of {} is not a Variable".format(key) + ) var_name_to_key["None.var"] = key elapsed_secs = 0 while True: @@ -165,7 +195,9 @@ class FetchHandlerMonitor(object): if var == None: local_logger.warning( "{} value currently not available".format( - var_name_to_key[key])) + var_name_to_key[key] + ) + ) res_dict = {} for key in fetch_dict: user_name = var_name_to_key[key] @@ -177,12 +209,14 @@ class FetchHandlerMonitor(object): lod = res_dict[user_name].lod() if len(lod) > 0: - raise RuntimeError("Some of your fetched tensors \ + raise RuntimeError( + "Some of your fetched tensors \ hold LoD information. \ They can not be completely cast \ to Python ndarray. We can \ not return LoDTensor itself directly, \ - please choose another targets") + please choose another targets" + ) if res_dict[user_name]._is_initialized(): res_dict[user_name] = np.array(res_dict[user_name]) else: diff --git a/python/paddle/fluid/transpiler/__init__.py b/python/paddle/fluid/transpiler/__init__.py index 166b04c21ac..8da4210dba2 100644 --- a/python/paddle/fluid/transpiler/__init__.py +++ b/python/paddle/fluid/transpiler/__init__.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .distribute_transpiler import DistributeTranspiler, DistributeTranspilerConfig +from .distribute_transpiler import ( + DistributeTranspiler, + DistributeTranspilerConfig, +) from .memory_optimization_transpiler import memory_optimize, release_memory from .ps_dispatcher import HashName, RoundRobin diff --git a/python/paddle/fluid/transpiler/ascend_transpiler.py b/python/paddle/fluid/transpiler/ascend_transpiler.py index 69fb2b18336..60ab474e904 100644 --- a/python/paddle/fluid/transpiler/ascend_transpiler.py +++ b/python/paddle/fluid/transpiler/ascend_transpiler.py @@ -20,7 +20,6 @@ from paddle.distributed import fleet class AscendTranspiler(collective.Collective): - def __init__(self, startup_program, main_program): self.nrings = 1 super(AscendTranspiler, self).__init__(self.nrings) @@ -32,8 +31,10 @@ class AscendTranspiler(collective.Collective): ring_id = -1 grad = None for idx, op in reversed(list(enumerate(block.ops))): - if self._is_backward_op(op) and \ - self.op_role_var_key in op.attr_names: + if ( + self._is_backward_op(op) + and self.op_role_var_key in op.attr_names + ): op_role_var = op.all_attrs()[self.op_role_var_key] if len(op_role_var) == 0: @@ -50,22 +51,26 @@ class AscendTranspiler(collective.Collective): # As we search ops reversedly, we should insert c_allreduce_sum # op in the same way to keep the ring_id alternate ring_id = (ring_id + 1) % self.nrings - block._insert_op(offset + 1, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) - block._insert_op(offset + 2, - type='scale', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'scale': 1.0 / fleet.worker_num(), - self.op_role_key: OpRole.Backward - }) + block._insert_op( + offset + 1, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) + block._insert_op( + offset + 2, + type='scale', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'scale': 1.0 / fleet.worker_num(), + self.op_role_key: OpRole.Backward, + }, + ) if grad is None: return diff --git a/python/paddle/fluid/transpiler/collective.py b/python/paddle/fluid/transpiler/collective.py index 84912529cfb..aedbaeb43cd 100644 --- a/python/paddle/fluid/transpiler/collective.py +++ b/python/paddle/fluid/transpiler/collective.py @@ -32,8 +32,7 @@ OpRole = core.op_proto_and_checker_maker.OpRole class Collective(object): - ''' - ''' + ''' ''' def __init__(self, nrings): self.nrings = nrings @@ -48,8 +47,15 @@ class Collective(object): self.op_role_key = op_maker.kOpRoleAttrName() self.op_role_var_key = op_maker.kOpRoleVarAttrName() - def transpile(self, startup_program, main_program, rank, endpoints, - current_endpoint, wait_port): + def transpile( + self, + startup_program, + main_program, + rank, + endpoints, + current_endpoint, + wait_port, + ): # in case of '127.0.0.1:6700,127.0.0.1:6701,...' if isinstance(endpoints, str): endpoints = endpoints.split(',') @@ -63,7 +69,11 @@ class Collective(object): self.main_program = default_main_program() self.nranks = len(endpoints) - if self.nranks == 1 and self.mode != "single_process_multi_thread" and self.mode != "box": + if ( + self.nranks == 1 + and self.mode != "single_process_multi_thread" + and self.mode != "box" + ): raise ValueError('the number of endpoints must > 1') if rank < 0: @@ -71,8 +81,11 @@ class Collective(object): self.rank = rank if current_endpoint not in endpoints: - raise ValueError('current endpoint %s is not in %s', - current_endpoint, str(endpoints)) + raise ValueError( + 'current endpoint %s is not in %s', + current_endpoint, + str(endpoints), + ) self.endpoints = endpoints self.current_endpoint = current_endpoint @@ -96,19 +109,26 @@ class Collective(object): def _transpile_startup_program(self): for ring_id in range(self.nrings): - self._init_communicator(self.startup_program, self.current_endpoint, - self.endpoints, self.rank, ring_id, - self.wait_port) + self._init_communicator( + self.startup_program, + self.current_endpoint, + self.endpoints, + self.rank, + ring_id, + self.wait_port, + ) self._broadcast_params() - def _init_communicator(self, - program, - current_endpoint, - endpoints, - rank, - ring_id, - wait_port, - has_multitrainer=False): + def _init_communicator( + self, + program, + current_endpoint, + endpoints, + rank, + ring_id, + wait_port, + has_multitrainer=False, + ): nranks = len(endpoints) other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) @@ -119,63 +139,76 @@ class Collective(object): block = program.global_block() if core.is_compiled_with_npu(): - hccl_id_var = block.create_var(name=unique_name.generate('hccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) + hccl_id_var = block.create_var( + name=unique_name.generate('hccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) endpoint_to_index_map = {e: idx for idx, e in enumerate(endpoints)} - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': hccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - self.op_role_key: OpRole.Forward - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': hccl_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': ring_id, - 'device_id': - int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': hccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + self.op_role_key: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': hccl_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': ring_id, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + self.op_role_key: OpRole.Forward, + }, + ) else: - nccl_id_var = block.create_var(name=unique_name.generate('nccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - self.op_role_key: OpRole.Forward - }) + nccl_id_var = block.create_var( + name=unique_name.generate('nccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + self.op_role_key: OpRole.Forward, + }, + ) if not has_multitrainer: - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': ring_id, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': ring_id, + self.op_role_key: OpRole.Forward, + }, + ) else: - block.append_op(type='c_comm_init_multitrainer', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'ntrainers': nranks, - 'trainer_id': rank, - 'ring_id': ring_id, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_comm_init_multitrainer', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'ntrainers': nranks, + 'trainer_id': rank, + 'ring_id': ring_id, + self.op_role_key: OpRole.Forward, + }, + ) def _broadcast_params(self): block = self.startup_program.global_block() @@ -185,23 +218,24 @@ class Collective(object): continue ring_id = (ring_id + 1) % self.nrings - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + self.op_role_key: OpRole.Forward, + }, + ) for ring_id in range(self.nrings): - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, self.op_role_key: OpRole.Forward}, + ) def _is_loss_grad_op(self, op): if self.op_role_key not in op.attr_names: @@ -210,21 +244,25 @@ class Collective(object): return op_role & int(OpRole.Backward) and op_role & int(OpRole.Loss) def _is_backward_op(self, op): - return self.op_role_key in op.attr_names and \ - int(op.all_attrs()[self.op_role_key]) & int(OpRole.Backward) + return self.op_role_key in op.attr_names and int( + op.all_attrs()[self.op_role_key] + ) & int(OpRole.Backward) def _is_update_op(self, op): - return 'Param' in op.input_names and 'Grad' in op.input_names and \ - "LearningRate" in op.input_names + return ( + 'Param' in op.input_names + and 'Grad' in op.input_names + and "LearningRate" in op.input_names + ) def _is_optimizer_op(self, op): - return self.op_role_key in op.attr_names and \ - int(op.all_attrs()[self.op_role_key]) & int(OpRole.Optimize) + return self.op_role_key in op.attr_names and int( + op.all_attrs()[self.op_role_key] + ) & int(OpRole.Optimize) class GradAllReduce(Collective): - ''' - ''' + ''' ''' def __init__(self, nrings=2): Collective.__init__(self, nrings) @@ -243,22 +281,26 @@ class GradAllReduce(Collective): for idx, op in reversed(list(enumerate(block.ops))): if self._is_loss_grad_op(op): loss_grad_var = block.vars[op.output_arg_names[0]] - block._insert_op(idx + 1, - type='scale', - inputs={'X': loss_grad_var}, - outputs={'Out': loss_grad_var}, - attrs={ - 'scale': 1.0 / self.nranks, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / self.nranks, + self.op_role_key: OpRole.Backward, + }, + ) def _insert_allreduce_ops(self): block = self.main_program.global_block() ring_id = -1 grad = None for idx, op in reversed(list(enumerate(block.ops))): - if self._is_backward_op(op) and \ - self.op_role_var_key in op.attr_names: + if ( + self._is_backward_op(op) + and self.op_role_var_key in op.attr_names + ): op_role_var = op.all_attrs()[self.op_role_var_key] if len(op_role_var) == 0: @@ -279,20 +321,23 @@ class GradAllReduce(Collective): type='c_sync_calc_stream', inputs={'X': grad}, outputs={'Out': grad}, - attrs={self.op_role_key: OpRole.Backward}) + attrs={self.op_role_key: OpRole.Backward}, + ) offset += 1 # As we search ops reversedly, we should insert c_allreduce_sum # op in the same way to keep the ring_id alternate ring_id = (ring_id + 1) % self.nrings - block._insert_op(offset, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) if grad is None: return @@ -300,20 +345,21 @@ class GradAllReduce(Collective): for idx, op in enumerate(block.ops): if self._is_optimizer_op(op): for ring_id in range(self.nrings): - block._insert_op(idx + ring_id, - type='c_sync_comm_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx + ring_id, + type='c_sync_comm_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) break class LocalSGD(Collective): - ''' - ''' + ''' ''' def __init__(self, nrings=2): Collective.__init__(self, nrings) @@ -330,14 +376,18 @@ class LocalSGD(Collective): non_dist_params.append(param) for param in non_dist_params: - snapshot = block.create_var(name=self.snapshot_name(param.name), - shape=param.shape, - persistable=True, - stop_gradient=True) - block.append_op(type='assign', - inputs={'X': [param]}, - outputs={'Out': [snapshot]}, - attrs={self.op_role_key: OpRole.Forward}) + snapshot = block.create_var( + name=self.snapshot_name(param.name), + shape=param.shape, + persistable=True, + stop_gradient=True, + ) + block.append_op( + type='assign', + inputs={'X': [param]}, + outputs={'Out': [snapshot]}, + attrs={self.op_role_key: OpRole.Forward}, + ) def snapshot_name(self, param_name): return param_name + self.snapshot_key @@ -352,72 +402,78 @@ class LocalSGD(Collective): if param.is_distributed: continue - snapshot = block.create_var(name=self.snapshot_name(param.name), - shape=param.shape, - persistable=True, - stop_gradient=True, - dtype=param.dtype) - - block._insert_op(idx + 1, - type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={self.op_role_key: OpRole.Optimize}) - block._insert_op(idx + 2, - type='c_sync_calc_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={self.op_role_key: OpRole.Optimize}) + snapshot = block.create_var( + name=self.snapshot_name(param.name), + shape=param.shape, + persistable=True, + stop_gradient=True, + dtype=param.dtype, + ) + + block._insert_op( + idx + 1, + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={self.op_role_key: OpRole.Optimize}, + ) + block._insert_op( + idx + 2, + type='c_sync_calc_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={self.op_role_key: OpRole.Optimize}, + ) ring_id = (ring_id + 1) % self.nrings - block._insert_op(idx + 3, - type='c_allreduce_sum', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Optimize - }) + block._insert_op( + idx + 3, + type='c_allreduce_sum', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Optimize, + }, + ) ordered_param_snapshot.append((param, snapshot)) for ring_id in range(self.nrings): - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Optimize - }) + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, self.op_role_key: OpRole.Optimize}, + ) for param_snapshot in reversed(ordered_param_snapshot): param = param_snapshot[0] snapshot = param_snapshot[1] - block.append_op(type='scale', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'scale': 1.0 / self.nranks, - self.op_role_key: OpRole.Optimize - }) - block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={self.op_role_key: OpRole.Optimize}) - block.append_op(type='assign', - inputs={'X': [param]}, - outputs={'Out': [snapshot]}, - attrs={self.op_role_key: OpRole.Optimize}) + block.append_op( + type='scale', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'scale': 1.0 / self.nranks, + self.op_role_key: OpRole.Optimize, + }, + ) + block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={self.op_role_key: OpRole.Optimize}, + ) + block.append_op( + type='assign', + inputs={'X': [param]}, + outputs={'Out': [snapshot]}, + attrs={self.op_role_key: OpRole.Optimize}, + ) class SingleProcessMultiThread(GradAllReduce): - ''' - ''' + ''' ''' def __init__(self): GradAllReduce.__init__(self, 1) @@ -429,16 +485,16 @@ class SingleProcessMultiThread(GradAllReduce): class MultiThread(GradAllReduce): - ''' - ''' + ''' ''' def __init__(self, nrings=1, trans_mode="all_reduce"): GradAllReduce.__init__(self, nrings) self.mode = "box" self.trans_mode = trans_mode self.fuse_grad_size_in_num = 128 - gpu_nums = os.getenv("FLAGS_selected_gpus", - "0,1,2,3,4,5,6,7,8").split(",") + gpu_nums = os.getenv("FLAGS_selected_gpus", "0,1,2,3,4,5,6,7,8").split( + "," + ) self.gpu_num = len(gpu_nums) def _transpile_startup_program(self): @@ -448,10 +504,15 @@ class MultiThread(GradAllReduce): print("total endpoints: ", self.endpoints) print("rank: %d, ring_id: %d" % (self.rank, self.nrings)) for ring_id in range(self.nrings): - self._init_communicator(self.startup_program, - self.current_endpoint, self.endpoints, - self.rank, ring_id, self.wait_port, - True) + self._init_communicator( + self.startup_program, + self.current_endpoint, + self.endpoints, + self.rank, + ring_id, + self.wait_port, + True, + ) else: if "xpu" in self.trans_mode: @@ -462,13 +523,14 @@ class MultiThread(GradAllReduce): block.append_op( type='c_comm_init_all', attrs={ - 'devices': - list( - map(int, - os.getenv("FLAGS_selected_gpus").split(","))), - 'ring_id': - 0 - }) + 'devices': list( + map( + int, os.getenv("FLAGS_selected_gpus").split(",") + ) + ), + 'ring_id': 0, + }, + ) else: print("begin to _transpile_startup_program for single-node") block = self.startup_program.global_block() @@ -484,8 +546,10 @@ class MultiThread(GradAllReduce): elif self.trans_mode == "fuse_all_reduce": print("begin to transpile in fuse all-reduce mode") self._insert_fuse_allreduce_ops() - elif self.trans_mode == "all_reduce_xpu" and len( - os.getenv("FLAGS_selected_gpus").split(",")) == 1: + elif ( + self.trans_mode == "all_reduce_xpu" + and len(os.getenv("FLAGS_selected_gpus").split(",")) == 1 + ): print( "skip transpile in all-reduce-xpu mode when number of devices is only one" ) @@ -501,8 +565,10 @@ class MultiThread(GradAllReduce): ring_id = -1 grad = None for idx, op in reversed(list(enumerate(block.ops))): - if self._is_backward_op(op) and \ - self.op_role_var_key in op.attr_names: + if ( + self._is_backward_op(op) + and self.op_role_var_key in op.attr_names + ): op_role_var = op.all_attrs()[self.op_role_var_key] if len(op_role_var) == 0: continue @@ -516,7 +582,8 @@ class MultiThread(GradAllReduce): shape=[self.allgather_ranks] + list(param.shape), persistable=False, dtype=core.VarDesc.VarType.FP32, - stop_gradient=True) + stop_gradient=True, + ) grad = block.vars[op_role_var[i + 1]] if param.is_distributed: # no need to care: used in PLSC continue @@ -528,21 +595,24 @@ class MultiThread(GradAllReduce): type='c_sync_calc_stream', inputs={'X': grad}, outputs={'Out': grad}, - attrs={self.op_role_key: OpRole.Backward}) + attrs={self.op_role_key: OpRole.Backward}, + ) offset += 1 # As we search ops reversedly, we should insert c_allgather # op in the same way to keep the ring_id alternate ring_id = (ring_id + 1) % self.nrings - block._insert_op(offset, - type='c_allgather', - inputs={'X': grad}, - outputs={'Out': new_grad_var}, - attrs={ - 'nranks': self.allgather_ranks, - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + offset, + type='c_allgather', + inputs={'X': grad}, + outputs={'Out': new_grad_var}, + attrs={ + 'nranks': self.allgather_ranks, + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) if grad is None: return @@ -550,14 +620,16 @@ class MultiThread(GradAllReduce): for idx, op in enumerate(block.ops): if self._is_optimizer_op(op): for ring_id in range(self.nrings): - block._insert_op(idx + ring_id, - type='c_sync_comm_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx + ring_id, + type='c_sync_comm_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) break def _update_adam_ops(self): @@ -569,7 +641,9 @@ class MultiThread(GradAllReduce): for idx, op in reversed(list(enumerate(block.ops))): if self._is_optimizer_op(op): offset = idx - if op.type != 'adam' and op.type != 'lamb': # filter out scale op + if ( + op.type != 'adam' and op.type != 'lamb' + ): # filter out scale op continue param_name = op.input("Param")[0] inputs = { @@ -578,26 +652,23 @@ class MultiThread(GradAllReduce): "Moment1": block.vars[op.input("Moment1")[0]], "Moment2": block.vars[op.input("Moment2")[0]], "Beta1Pow": block.vars[op.input("Beta1Pow")[0]], - "Beta2Pow": block.vars[op.input("Beta2Pow")[0]] + "Beta2Pow": block.vars[op.input("Beta2Pow")[0]], } outputs = { "ParamOut": block.vars[op.output("ParamOut")[0]], "Moment1Out": block.vars[op.output("Moment1Out")[0]], "Moment2Out": block.vars[op.output("Moment2Out")[0]], "Beta1PowOut": block.vars[op.output("Beta1PowOut")[0]], - "Beta2PowOut": block.vars[op.output("Beta2PowOut")[0]] + "Beta2PowOut": block.vars[op.output("Beta2PowOut")[0]], } attrs = { - "epsilon": - op.attr('epsilon'), - "beta1": - op.attr('beta1'), - "beta2": - op.attr('beta2'), - "lazy_mode": - op.attr('lazy_mode'), - "min_row_size_to_use_multithread": - op.attr('min_row_size_to_use_multithread') + "epsilon": op.attr('epsilon'), + "beta1": op.attr('beta1'), + "beta2": op.attr('beta2'), + "lazy_mode": op.attr('lazy_mode'), + "min_row_size_to_use_multithread": op.attr( + 'min_row_size_to_use_multithread' + ), } split_vars = [ block.create_var( @@ -605,29 +676,30 @@ class MultiThread(GradAllReduce): shape=block.vars[op.input("Param")[0]].shape, persistable=False, dtype=core.VarDesc.VarType.FP32, - stop_gradient=True) for i in range(self.allgather_ranks) + stop_gradient=True, + ) + for i in range(self.allgather_ranks) ] - block._insert_op(offset, - type="split", - inputs={ - 'X': - block.vars[op.input("Param")[0] + - "_allgather"] - }, - outputs={'Out': split_vars}, - attrs={ - 'num': self.allgather_ranks, - 'axis': 0 - }) + block._insert_op( + offset, + type="split", + inputs={ + 'X': block.vars[op.input("Param")[0] + "_allgather"] + }, + outputs={'Out': split_vars}, + attrs={'num': self.allgather_ranks, 'axis': 0}, + ) offset += 1 for i in range(self.allgather_ranks): inputs["Grad"] = split_vars[i] - block._insert_op(offset, - type=op.type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + block._insert_op( + offset, + type=op.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) offset += 1 # remove the original adam op block._remove_op(offset) @@ -642,13 +714,17 @@ class MultiThread(GradAllReduce): param_grads = [] # find all grad params for op in reversed(block.ops): - if self._is_backward_op(op) and \ - self.op_role_var_key in op.attr_names: + if ( + self._is_backward_op(op) + and self.op_role_var_key in op.attr_names + ): op_role_var = op.all_attrs()[self.op_role_var_key] if len(op_role_var) == 0: continue - assert len(op_role_var) % 2 == 0, "vars need to be one param var followed by one grad var, " \ - "but got odd number of vars" + assert len(op_role_var) % 2 == 0, ( + "vars need to be one param var followed by one grad var, " + "but got odd number of vars" + ) for i in range(0, len(op_role_var), 2): param_name = op_role_var[i] param = block.var(param_name) @@ -664,9 +740,11 @@ class MultiThread(GradAllReduce): last_dtype = None # split the grad based on dtype and fused size for var in param_grads: - if len(segments) == 0 \ - or len(segments[-1]) == self.fuse_grad_size_in_num \ - or var.dtype != last_dtype: + if ( + len(segments) == 0 + or len(segments[-1]) == self.fuse_grad_size_in_num + or var.dtype != last_dtype + ): segments.append([var]) last_dtype = var.dtype else: @@ -677,45 +755,51 @@ class MultiThread(GradAllReduce): if self._is_optimizer_op(op): for segment in segments: # insert coalesce tensor - tmp_var = block.create_var(name=unique_name.generate( - 'FusedOutput_{}'.format(segment[0].name)), - dtype=segment[0].dtype, - persistable=False, - stop_gradient=True) + tmp_var = block.create_var( + name=unique_name.generate( + 'FusedOutput_{}'.format(segment[0].name) + ), + dtype=segment[0].dtype, + persistable=False, + stop_gradient=True, + ) fused_vars.append(tmp_var) - block._insert_op(idx, - type="coalesce_tensor", - inputs={"Input": segment}, - outputs={ - "Output": segment, - "FusedOutput": tmp_var - }, - attrs={ - "copy_data": True, - "use_align": True, - "dtype": segment[0].dtype, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx, + type="coalesce_tensor", + inputs={"Input": segment}, + outputs={"Output": segment, "FusedOutput": tmp_var}, + attrs={ + "copy_data": True, + "use_align": True, + "dtype": segment[0].dtype, + self.op_role_key: OpRole.Backward, + }, + ) break # insert the allreduce_sum op for idx, op in enumerate(block.ops): if self._is_optimizer_op(op): for fused_var in fused_vars: - block._insert_op(idx, - type='c_allreduce_sum', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': False, - self.op_role_key: OpRole.Backward - }) - block._insert_op(idx, - type='c_sync_calc_stream', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={self.op_role_key: OpRole.Backward}) + block._insert_op( + idx, + type='c_allreduce_sum', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': False, + self.op_role_key: OpRole.Backward, + }, + ) + block._insert_op( + idx, + type='c_sync_calc_stream', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={self.op_role_key: OpRole.Backward}, + ) break if len(fused_vars) == 0: @@ -725,13 +809,15 @@ class MultiThread(GradAllReduce): # insert the sync comm op for idx, op in enumerate(block.ops): if self._is_optimizer_op(op): - block._insert_op(idx, - type='c_sync_comm_stream', - inputs={'X': fused_vars[0]}, - outputs={'Out': fused_vars[0]}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx, + type='c_sync_comm_stream', + inputs={'X': fused_vars[0]}, + outputs={'Out': fused_vars[0]}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) break block._sync_with_cpp() diff --git a/python/paddle/fluid/transpiler/details/checkport.py b/python/paddle/fluid/transpiler/details/checkport.py index 71c9565a362..c0d8cdc9145 100644 --- a/python/paddle/fluid/transpiler/details/checkport.py +++ b/python/paddle/fluid/transpiler/details/checkport.py @@ -38,8 +38,9 @@ def wait_server_ready(endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): @@ -51,8 +52,9 @@ def wait_server_ready(endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + str(not_ready_endpoints) + - "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: diff --git a/python/paddle/fluid/transpiler/details/ufind.py b/python/paddle/fluid/transpiler/details/ufind.py index 0e30d0e3f9c..fef6f24570c 100644 --- a/python/paddle/fluid/transpiler/details/ufind.py +++ b/python/paddle/fluid/transpiler/details/ufind.py @@ -14,7 +14,7 @@ class UnionFind(object): - """ Union-find data structure. + """Union-find data structure. Union-find is a data structure that keeps track of a set of elements partitioned into a number of disjoint (non-overlapping) subsets. diff --git a/python/paddle/fluid/transpiler/details/vars_distributed.py b/python/paddle/fluid/transpiler/details/vars_distributed.py index f78559e6559..32a27ed33b0 100644 --- a/python/paddle/fluid/transpiler/details/vars_distributed.py +++ b/python/paddle/fluid/transpiler/details/vars_distributed.py @@ -35,14 +35,16 @@ class VarDistributed(object): the slice var's properties, such as type/shape/offset/endpoint. """ - def __init__(self, - origin_var, - slice_var, - is_slice=None, - block_id=None, - offset=None, - vtype=None, - endpoint=None): + def __init__( + self, + origin_var, + slice_var, + is_slice=None, + block_id=None, + offset=None, + vtype=None, + endpoint=None, + ): """ Args: origin_var(Variable|VarStruct): origin var properties @@ -85,8 +87,14 @@ class VarDistributed(object): @staticmethod def __create_var_struct(var): - return VarStruct(var.name, var.shape, var.dtype, var.type, - var.lod_level, var.persistable) + return VarStruct( + var.name, + var.shape, + var.dtype, + var.type, + var.lod_level, + var.persistable, + ) @staticmethod def equal(var1, var2): @@ -97,26 +105,45 @@ class VarDistributed(object): """ assert isinstance(var1, VarStruct) and isinstance(var2, VarStruct) - return var1.name == var2.name and \ - var1.type == var2.type and \ - var1.shape == var2.shape and \ - var1.dtype == var2.dtype and \ - var1.lod_level == var2.lod_level and \ - var1.persistable == var2.persistable + return ( + var1.name == var2.name + and var1.type == var2.type + and var1.shape == var2.shape + and var1.dtype == var2.dtype + and var1.lod_level == var2.lod_level + and var1.persistable == var2.persistable + ) def __str__(self): - origin_var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})". \ - format(i="{", e="}", name=self.origin.name, type=self.origin.type, - shape=self.origin.shape, dtype=self.origin.dtype) - - slice_var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})" \ - ".slice({is_slice}).block({block_id}).offset({offset})". \ - format(i="{", e="}", name=self.slice.name, type=self.slice.type, - shape=self.slice.shape, dtype=self.slice.dtype, - is_slice=self.is_slice, block_id=self.block_id, offset=self.offset) + origin_var_str = ( + "{name} : fluid.{type}.shape{shape}.astype({dtype})".format( + i="{", + e="}", + name=self.origin.name, + type=self.origin.type, + shape=self.origin.shape, + dtype=self.origin.dtype, + ) + ) + + slice_var_str = ( + "{name} : fluid.{type}.shape{shape}.astype({dtype})" + ".slice({is_slice}).block({block_id}).offset({offset})".format( + i="{", + e="}", + name=self.slice.name, + type=self.slice.type, + shape=self.slice.shape, + dtype=self.slice.dtype, + is_slice=self.is_slice, + block_id=self.block_id, + offset=self.offset, + ) + ) return "var owned: {}, origin var: ( {} ), slice var: ( {} ), endpoint: {} ".format( - self.vtype, origin_var_str, slice_var_str, self.endpoint) + self.vtype, origin_var_str, slice_var_str, self.endpoint + ) class VarsDistributed(object): @@ -130,14 +157,16 @@ class VarsDistributed(object): def __init__(self): self.distributed_vars = [] - def add_distributed_var(self, - origin_var, - slice_var, - is_slice=None, - block_id=None, - offset=None, - vtype=None, - endpoint=None): + def add_distributed_var( + self, + origin_var, + slice_var, + is_slice=None, + block_id=None, + offset=None, + vtype=None, + endpoint=None, + ): """ add distributed var in this. @@ -153,8 +182,16 @@ class VarsDistributed(object): None """ self.distributed_vars.append( - VarDistributed(origin_var, slice_var, is_slice, block_id, offset, - vtype, endpoint)) + VarDistributed( + origin_var, + slice_var, + is_slice, + block_id, + offset, + vtype, + endpoint, + ) + ) def get_distributed_var_by_slice(self, var_name): """ @@ -177,12 +214,14 @@ class VarsDistributed(object): Returns: bool: equal will return True else False """ - return var1.name == var2.name and \ - var1.type == var2.type and \ - var1.shape == var2.shape and \ - var1.dtype == var2.dtype and \ - var1.lod_level == var2.lod_level and \ - var1.persistable == var2.persistable + return ( + var1.name == var2.name + and var1.type == var2.type + and var1.shape == var2.shape + and var1.dtype == var2.dtype + and var1.lod_level == var2.lod_level + and var1.persistable == var2.persistable + ) def get_distributed_var_by_origin_and_ep(self, origin_var_name, endpoint): """ @@ -195,7 +234,10 @@ class VarsDistributed(object): VarDistributed: distributed var. """ for dist_var in self.distributed_vars: - if dist_var.origin.name == origin_var_name and dist_var.endpoint == endpoint: + if ( + dist_var.origin.name == origin_var_name + and dist_var.endpoint == endpoint + ): return dist_var return None diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py index 670f5431122..28dbf22153f 100644 --- a/python/paddle/fluid/transpiler/distribute_transpiler.py +++ b/python/paddle/fluid/transpiler/distribute_transpiler.py @@ -40,8 +40,14 @@ import numpy as np from .ps_dispatcher import RoundRobin, PSDispatcher from .. import core, framework, unique_name, initializer -from ..framework import Program, default_main_program, \ - default_startup_program, Block, Parameter, grad_var_name +from ..framework import ( + Program, + default_main_program, + default_startup_program, + Block, + Parameter, + grad_var_name, +) from .details import wait_server_ready, UnionFind, VarStruct, VarsDistributed from .details import delete_ops, find_op_by_output_arg from ..distribute_lookup_table import find_distributed_lookup_table @@ -52,8 +58,9 @@ LOOKUP_TABLE_GRAD_TYPE = ["lookup_table_grad", "lookup_table_v2_grad"] OP_NAME_SCOPE = "op_namescope" CLIP_OP_NAME_SCOPE = "@CLIP" OP_ROLE_VAR_ATTR_NAME = core.op_proto_and_checker_maker.kOpRoleVarAttrName() -RPC_OP_ROLE_ATTR_NAME = op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( -) +RPC_OP_ROLE_ATTR_NAME = ( + op_role_attr_name +) = core.op_proto_and_checker_maker.kOpRoleAttrName() OPT_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.Optimize RPC_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.RPC DIST_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.Dist @@ -75,7 +82,6 @@ def log(*args): class VarBlock: - def __init__(self, varname, offset, size): self.varname = varname # NOTE: real offset is offset * size @@ -129,8 +135,9 @@ def slice_variable(var_list, slice_count, min_block_size): # update split_count after aligning split_count = int(math.ceil(var_numel / float(block_size))) for block_id in range(split_count): - curr_block_size = min(block_size, - var_numel - ((block_id) * block_size)) + curr_block_size = min( + block_size, var_numel - ((block_id) * block_size) + ) block = VarBlock(var.name, block_id, curr_block_size) blocks.append(str(block)) return blocks @@ -242,14 +249,16 @@ class DistributeTranspilerConfig(object): class ServerRuntimeConfig(object): - def __init__(self): self._rpc_send_thread_num = int( - os.getenv("FLAGS_rpc_send_thread_num", "12")) + os.getenv("FLAGS_rpc_send_thread_num", "12") + ) self._rpc_get_thread_num = int( - os.getenv("FLAGS_rpc_get_thread_num", "12")) + os.getenv("FLAGS_rpc_get_thread_num", "12") + ) self._rpc_prefetch_thread_num = int( - os.getenv("FLAGS_rpc_prefetch_thread_num", "12")) + os.getenv("FLAGS_rpc_prefetch_thread_num", "12") + ) class DistributeTranspiler(object): @@ -337,8 +346,8 @@ class DistributeTranspiler(object): global PRINT_LOG if self.config.print_log: PRINT_LOG = True - assert (self.config.min_block_size >= 8192) - assert (self.config.split_method.__bases__[0] == PSDispatcher) + assert self.config.min_block_size >= 8192 + assert self.config.split_method.__bases__[0] == PSDispatcher self.counter_var = None def _set_server_config(self, server_config=None): @@ -351,12 +360,14 @@ class DistributeTranspiler(object): "In DistributeTranspiler, server_config must be an instance of ServerRuntimeConfig" ) - def _transpile_nccl2(self, - trainer_id, - trainers, - current_endpoint, - startup_program=None, - wait_port=True): + def _transpile_nccl2( + self, + trainer_id, + trainers, + current_endpoint, + startup_program=None, + wait_port=True, + ): if not startup_program: startup_program = default_startup_program() if trainer_id >= 0: @@ -367,53 +378,55 @@ class DistributeTranspiler(object): wait_server_ready(worker_endpoints) nccl_id_var = startup_program.global_block().create_var( - name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) for i in range(1, self.config.nccl_comm_num): startup_program.global_block().create_var( name="NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) if self.config.use_hierarchical_allreduce: for i in range(0, self.config.nccl_comm_num): startup_program.global_block().create_var( name="Hierarchical_inter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().create_var( name="Hierarchical_exter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_nccl_id", inputs={}, outputs={"NCCLID": nccl_id_var}, attrs={ - "trainers": - trainers.split(","), - "trainer_id": - trainer_id, - "nccl_comm_num": - self.config.nccl_comm_num, - "use_hierarchical_allreduce": - self.config.use_hierarchical_allreduce, - "hierarchical_allreduce_inter_nranks": - self.config.hierarchical_allreduce_inter_nranks - }) + "trainers": trainers.split(","), + "trainer_id": trainer_id, + "nccl_comm_num": self.config.nccl_comm_num, + "use_hierarchical_allreduce": self.config.use_hierarchical_allreduce, + "hierarchical_allreduce_inter_nranks": self.config.hierarchical_allreduce_inter_nranks, + }, + ) return nccl_id_var else: raise ValueError("must set trainer_id > 0") - def _transpile_collective(self, - collective_mode, - trainer_id, - trainers, - current_endpoint, - startup_program=None, - main_program=None, - wait_port=True): + def _transpile_collective( + self, + collective_mode, + trainer_id, + trainers, + current_endpoint, + startup_program=None, + main_program=None, + wait_port=True, + ): if isinstance(trainers, str): endpoints = trainers.split(",") elif isinstance(trainers, list): @@ -421,8 +434,10 @@ class DistributeTranspiler(object): elif collective_mode != "single_process_multi_thread": raise ValueError('invalid trainers config: ' + str(trainers)) - if len(endpoints - ) == 1 and collective_mode != "single_process_multi_thread": + if ( + len(endpoints) == 1 + and collective_mode != "single_process_multi_thread" + ): raise ValueError('invalid trainer number in distributed: 1') if startup_program is None: @@ -441,28 +456,34 @@ class DistributeTranspiler(object): else: raise ValueError('invalid collective_mode: %s' % collective_mode) - transpiler.transpile(startup_program=startup_program, - main_program=main_program, - rank=trainer_id, - endpoints=endpoints, - current_endpoint=current_endpoint, - wait_port=wait_port) + transpiler.transpile( + startup_program=startup_program, + main_program=main_program, + rank=trainer_id, + endpoints=endpoints, + current_endpoint=current_endpoint, + wait_port=wait_port, + ) def _get_all_remote_sparse_update_op(self, main_program): sparse_update_ops = [] sparse_update_op_types = ["lookup_table", "nce", "lookup_table_v2"] for op in main_program.global_block().ops: - if op.type in sparse_update_op_types and op.attr( - 'remote_prefetch') is True: + if ( + op.type in sparse_update_op_types + and op.attr('remote_prefetch') is True + ): sparse_update_ops.append(op) return sparse_update_ops - def _update_remote_sparse_update_op(self, program, - need_sparse_update_params): + def _update_remote_sparse_update_op( + self, program, need_sparse_update_params + ): for param_varname, attrs in need_sparse_update_params.items(): height_sections = self.sparse_param_to_height_sections[ - param_varname] + param_varname + ] endpoints = attrs[0] table_names = attrs[1] @@ -518,10 +539,7 @@ class DistributeTranspiler(object): program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Outputs": outputs}, attrs={ "table_names": table_names, @@ -529,8 +547,9 @@ class DistributeTranspiler(object): "endpoints": endpoints, "padding_idx": padding_idx, "trainer_id": self.trainer_id, - "lookup_table_version": op_type - }) + "lookup_table_version": op_type, + }, + ) else: raise ValueError( "something wrong with distribute_transpiler, submit a issue is recommended" @@ -545,14 +564,16 @@ class DistributeTranspiler(object): return True return False - def transpile(self, - trainer_id, - program=None, - pservers="127.0.0.1:6174", - trainers=1, - sync_mode=True, - startup_program=None, - current_endpoint="127.0.0.1:6174"): + def transpile( + self, + trainer_id, + program=None, + pservers="127.0.0.1:6174", + trainers=1, + sync_mode=True, + startup_program=None, + current_endpoint="127.0.0.1:6174", + ): """ Transpile the input program to distributed programs with config and arguments. @@ -604,34 +625,49 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler self.origin_startup_program = self.startup_program.clone() if self.config.mode == "nccl2": - assert (isinstance(trainers, str)) + assert isinstance(trainers, str) self.origin_program._trainers_endpoints = trainers.split(",") self.origin_program._nccl_comm_num = self.config.nccl_comm_num - self.origin_program._use_hierarchical_allreduce = self.config.use_hierarchical_allreduce + self.origin_program._use_hierarchical_allreduce = ( + self.config.use_hierarchical_allreduce + ) # check use_hierarchical_allreduce options if self.config.use_hierarchical_allreduce: trainers_num = len(self.origin_program._trainers_endpoints) # selected automaticly if self.config.hierarchical_allreduce_inter_nranks <= 1: - self.config.hierarchical_allreduce_inter_nranks = core.get_cuda_device_count( + self.config.hierarchical_allreduce_inter_nranks = ( + core.get_cuda_device_count() ) - assert trainers_num > self.config.hierarchical_allreduce_inter_nranks, \ - "trainers_num:{} < hierarchical_allreduce_inter_nranks:{}".format( - trainers_num, self.config.hierarchical_allreduce_inter_nranks) - - assert trainers_num % self.config.hierarchical_allreduce_inter_nranks == 0, \ - "trainers_num:{} mod hierarchical_allreduce_inter_nranks:{} != 0".format( - trainers_num, self.config.hierarchical_allreduce_inter_nranks) - - self.origin_program._hierarchical_allreduce_inter_nranks = \ - int(self.config.hierarchical_allreduce_inter_nranks) + assert ( + trainers_num + > self.config.hierarchical_allreduce_inter_nranks + ), "trainers_num:{} < hierarchical_allreduce_inter_nranks:{}".format( + trainers_num, + self.config.hierarchical_allreduce_inter_nranks, + ) + + assert ( + trainers_num + % self.config.hierarchical_allreduce_inter_nranks + == 0 + ), "trainers_num:{} mod hierarchical_allreduce_inter_nranks:{} != 0".format( + trainers_num, + self.config.hierarchical_allreduce_inter_nranks, + ) + + self.origin_program._hierarchical_allreduce_inter_nranks = int( + self.config.hierarchical_allreduce_inter_nranks + ) - self._transpile_nccl2(trainer_id, - trainers, - current_endpoint, - startup_program=startup_program, - wait_port=self.config.wait_port) + self._transpile_nccl2( + trainer_id, + trainers, + current_endpoint, + startup_program=startup_program, + wait_port=self.config.wait_port, + ) return if self.config.mode == "collective": @@ -642,7 +678,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler current_endpoint=current_endpoint, startup_program=startup_program, main_program=program, - wait_port=self.config.wait_port) + wait_port=self.config.wait_port, + ) return self.trainer_num = trainers @@ -664,7 +701,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # get all sparse update ops self.sparse_update_ops = self._get_all_remote_sparse_update_op( - self.origin_program) + self.origin_program + ) # use_sparse_update_param_name -> split_height_section self.sparse_param_to_height_sections = dict() self.need_delete_optimize_vars = [] @@ -674,7 +712,9 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler self.origin_program._endpoints = self.pserver_endpoints self.origin_program._ps_endpoint = current_endpoint self.origin_program._is_chief = self.trainer_id == 0 - self.origin_program._distributed_lookup_table = self.table_name if self.table_name else None + self.origin_program._distributed_lookup_table = ( + self.table_name if self.table_name else None + ) # split and create vars, then put split vars in dicts for later use. # step 1: split and create vars, then put split vars in dicts for later use. @@ -701,29 +741,31 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler eplist = ps_dispatcher.dispatch(splited_vars) if not self.config.slice_var_up: - assert (len(splited_vars) == 1) + assert len(splited_vars) == 1 splited_grad_varname = grad_varname if len(splited_vars) == 1: splited_grad_varname = splited_vars[0].name - index = find_op_by_output_arg(program.global_block(), - splited_grad_varname, - reverse=True) + index = find_op_by_output_arg( + program.global_block(), splited_grad_varname, reverse=True + ) elif len(splited_vars) > 1: orig_var = program.global_block().vars[splited_grad_varname] - index = find_op_by_output_arg(program.global_block(), - splited_grad_varname, - reverse=True) + index = find_op_by_output_arg( + program.global_block(), splited_grad_varname, reverse=True + ) if not self.config.runtime_split_send_recv: - self._insert_split_op(program, orig_var, index, - splited_vars) + self._insert_split_op( + program, orig_var, index, splited_vars + ) index += 1 else: AssertionError( - "Can not insert the send op by original " - "variable name :", splited_grad_varname) + "Can not insert the send op by original " "variable name :", + splited_grad_varname, + ) if splited_vars[0].type == core.VarDesc.VarType.SELECTED_ROWS: sparse_param_name = self.grad_name_to_param_name[grad_varname] @@ -733,7 +775,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler ] dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) self.grad_name_to_send_dummy_out[grad_varname] = dummy_output if self.config.runtime_split_send_recv: @@ -764,35 +807,36 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler inputs={"X": send_input_vars}, outputs={"Out": dummy_output}, attrs={ - "epmap": - eplist, - "sections": - sections, - "send_varnames": - send_varnames, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE, + "epmap": eplist, + "sections": sections, + "send_varnames": send_varnames, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, OP_ROLE_VAR_ATTR_NAME: [ self.grad_name_to_param_name[grad_varname], - splited_grad_varname - ] - }) + splited_grad_varname, + ], + }, + ) for _, var in enumerate(splited_vars): send_vars.append(var) send_barrier_out = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) if self.has_distributed_lookup_table: self.grad_name_to_send_dummy_out[ - self.table_name] = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + self.table_name + ] = program.global_block().create_var( + name=framework.generate_control_dev_var_name() + ) input_deps = list(self.grad_name_to_send_dummy_out.values()) if not self.sync_mode: lr_ops = self._get_lr_ops() if len(lr_ops) > 0 and self.counter_var: decay_dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) if self.config.runtime_split_send_recv: # async mode, using communicator to merge and send send_varnames = [self.counter_var.name] @@ -804,39 +848,34 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler inputs={"X": self.counter_var}, outputs={"Out": decay_dummy_output}, attrs={ - "epmap": - pserver_endpoints, - "sections": - sections, - "send_varnames": - send_varnames, - "merge_add": - True, - "use_send_handler": - False, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE, - OP_ROLE_VAR_ATTR_NAME: - [self.counter_var.name, self.counter_var.name] - }) + "epmap": pserver_endpoints, + "sections": sections, + "send_varnames": send_varnames, + "merge_add": True, + "use_send_handler": False, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + OP_ROLE_VAR_ATTR_NAME: [ + self.counter_var.name, + self.counter_var.name, + ], + }, + ) input_deps.append(decay_dummy_output) if self.sync_mode: fetch_barrier_input = [] - program.global_block().append_op(type="send_barrier", - inputs={"X": list(input_deps)}, - outputs={"Out": send_barrier_out}, - attrs={ - "endpoints": - pserver_endpoints, - "trainer_id": - self.trainer_id, - "half_async": - False, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + program.global_block().append_op( + type="send_barrier", + inputs={"X": list(input_deps)}, + outputs={"Out": send_barrier_out}, + attrs={ + "endpoints": pserver_endpoints, + "trainer_id": self.trainer_id, + "half_async": False, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) fetch_barrier_input.append(send_barrier_out) else: @@ -849,8 +888,9 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler "endpoints": pserver_endpoints, "trainer_id": self.trainer_id, "half_async": True, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) # step 3: insert recv op to receive parameters from parameter server recv_vars = [] @@ -864,7 +904,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler self.param_grad_ep_mapping[ep]["grads"].append(send_vars[i]) distributed_var = self.vars_overview.get_distributed_var_by_slice( - recv_vars[i].name) + recv_vars[i].name + ) distributed_var.endpoint = ep need_sparse_update_params = {} @@ -883,7 +924,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler else: # connect deps to send op in async mode recv_dep_in = self.grad_name_to_send_dummy_out[ - self.param_name_to_grad_name[param_varname]] + self.param_name_to_grad_name[param_varname] + ] # get recv op_role_var, if not split, the grad should have .trainer suffix # if split, grad should be the original grad var name. ParallelExecutor @@ -896,8 +938,11 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler if param_varname in self.sparse_param_to_height_sections: for table_name in table_names: - distributed_var = self.vars_overview.get_distributed_var_by_slice( - table_name) + distributed_var = ( + self.vars_overview.get_distributed_var_by_slice( + table_name + ) + ) distributed_var.vtype = "RemotePrefetch" need_sparse_update_params[param_varname] = (eps, table_names) @@ -914,33 +959,31 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler inputs={"X": [recv_dep_in]}, outputs={"Out": splited_var}, attrs={ - "epmap": - eps, - "recv_varnames": - recv_varnames, - "trainer_id": - self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE, - OP_ROLE_VAR_ATTR_NAME: - [param_varname, recv_op_role_var_name] - }) + "epmap": eps, + "recv_varnames": recv_varnames, + "trainer_id": self.trainer_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + OP_ROLE_VAR_ATTR_NAME: [ + param_varname, + recv_op_role_var_name, + ], + }, + ) self._update_remote_sparse_update_op(program, need_sparse_update_params) if self.sync_mode: # form a WAW dependency - program.global_block().append_op(type="fetch_barrier", - inputs={"X": fetch_barrier_input}, - outputs={"Out": all_recv_outputs}, - attrs={ - "endpoints": - pserver_endpoints, - "trainer_id": - self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + program.global_block().append_op( + type="fetch_barrier", + inputs={"X": fetch_barrier_input}, + outputs={"Out": all_recv_outputs}, + attrs={ + "endpoints": pserver_endpoints, + "trainer_id": self.trainer_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) for param_varname, splited_var in self.param_var_mapping.items(): if len(splited_var) <= 1: @@ -954,14 +997,16 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler outputs={"Out": [orig_param]}, attrs={ "axis": 0, - RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE, + }, + ) self._get_trainer_startup_program(recv_vars=recv_vars, eplist=eplist) if self.has_distributed_lookup_table: - self._replace_lookup_table_op_with_prefetch(program, - pserver_endpoints) + self._replace_lookup_table_op_with_prefetch( + program, pserver_endpoints + ) self._split_table_grad_and_add_send_vars(program, pserver_endpoints) self._get_distributed_optimizer_vars() @@ -972,8 +1017,10 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler sparse_table_names = [] for op in self.origin_program.global_block().ops: - if op.type in sparse_update_op_types and op.attr( - 'is_sparse') is True: + if ( + op.type in sparse_update_op_types + and op.attr('is_sparse') is True + ): sparse_table_names.append(op.input("W")[0]) if op.type == "distributed_lookup_table": sparse_table_names.append(op.input("W")[0]) @@ -993,14 +1040,16 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler table_param_init_op.append(op) init_op_num = len(table_param_init_op) if init_op_num != 1: - raise ValueError("table init op num should be 1, now is " + - str(init_op_num)) + raise ValueError( + "table init op num should be 1, now is " + str(init_op_num) + ) table_init_op = table_param_init_op[0] self.startup_program.global_block().append_op( type="fake_init", inputs={}, outputs={"Out": table_var}, - attrs={"shape": table_init_op.attr('shape')}) + attrs={"shape": table_init_op.attr('shape')}, + ) delete_ops(self.startup_program.global_block(), table_param_init_op) def _delete_trainer_optimizer(self, is_startup): @@ -1124,7 +1173,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler type=var.type, dtype=var.dtype, shape=var.shape, - lod_level=var.lod_level) + lod_level=var.lod_level, + ) op = startup_program.global_block().append_op( type="recv", @@ -1133,11 +1183,13 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler attrs={ "epmap": eps, "trainer_id": self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) fetch_barrier_out = startup_program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) startup_program.global_block().append_op( type="fetch_barrier", inputs={}, @@ -1145,8 +1197,9 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler attrs={ "endpoints": self.pserver_endpoints, "trainer_id": self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) for varname, splited_var in self.param_var_mapping.items(): if varname in sparse_table_names: @@ -1158,19 +1211,22 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler if varname in startup_program.global_block().vars: orig_param = startup_program.global_block().vars[varname] else: - origin_param_var = self.origin_program.global_block( - ).vars[varname] + origin_param_var = self.origin_program.global_block().vars[ + varname + ] orig_param = startup_program.global_block().create_var( name=varname, persistable=origin_param_var.persistable, type=origin_param_var.type, dtype=origin_param_var.dtype, - shape=origin_param_var.shape) + shape=origin_param_var.shape, + ) startup_program.global_block().append_op( type="concat", inputs={"X": splited_var}, outputs={"Out": [orig_param]}, - attrs={"axis": 0}) + attrs={"axis": 0}, + ) return startup_program @@ -1231,21 +1287,26 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler orig_var_name = v.name # NOTE: single_trainer_var must be created for multi-trainer # case to merge grads from multiple trainers - single_trainer_var = \ - pserver_program.global_block().create_var( - name=orig_var_name, - persistable=True, - type=v.type, - dtype=v.dtype, - shape=v.shape) - if self.sync_mode or self.config.completely_not_async and self.trainer_num > 1: + single_trainer_var = pserver_program.global_block().create_var( + name=orig_var_name, + persistable=True, + type=v.type, + dtype=v.dtype, + shape=v.shape, + ) + if ( + self.sync_mode + or self.config.completely_not_async + and self.trainer_num > 1 + ): for trainer_id in range(self.trainer_num): var = pserver_program.global_block().create_var( name="%s.trainer_%d" % (orig_var_name, trainer_id), persistable=False, type=v.type, dtype=v.dtype, - shape=v.shape) + shape=v.shape, + ) recv_inputs.append(var) else: recv_inputs.append(single_trainer_var) @@ -1261,12 +1322,13 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler opt_op_on_pserver = [] for _, op in enumerate(self.optimize_ops): if self._is_optimizer_op(op) and self._is_opt_op_on_pserver( - endpoint, op): + endpoint, op + ): opt_op_on_pserver.append(op) # step 3.3 # prepare if dc asgd is enabled if self.config.enable_dc_asgd == True: - assert (self.sync_mode == False) + assert self.sync_mode == False self.param_bak_list = [] # add param_bak for each trainer for p in self.param_grad_ep_mapping[endpoint]["params"]: @@ -1278,7 +1340,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler name=param_bak_name, type=p.type, shape=p.shape, - dtype=p.dtype) + dtype=p.dtype, + ) self.param_bak_list.append((p, tmpvar)) # step 3.4 @@ -1291,12 +1354,19 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # sparse grad name to param name sparse_grad_to_param = [] - def __append_optimize_op__(op, block, grad_to_block_id, merged_var, - lr_ops): + def __append_optimize_op__( + op, block, grad_to_block_id, merged_var, lr_ops + ): if self._is_optimizer_op(op): - self._append_pserver_ops(block, op, endpoint, grad_to_block_id, - self.origin_program, merged_var, - sparse_grad_to_param) + self._append_pserver_ops( + block, + op, + endpoint, + grad_to_block_id, + self.origin_program, + merged_var, + sparse_grad_to_param, + ) elif op not in lr_ops: self._append_pserver_non_opt_ops(block, op) @@ -1332,13 +1402,15 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler lr_decay_block_id = -1 if len(lr_ops) > 0: lr_decay_block = pserver_program._create_block( - pserver_program.num_blocks - 1) + pserver_program.num_blocks - 1 + ) optimize_blocks.append(lr_decay_block) for _, op in enumerate(lr_ops): cloned_op = self._append_pserver_non_opt_ops(lr_decay_block, op) # append sub blocks to pserver_program in lr_decay_op - __clone_lr_op_sub_block__(cloned_op, pserver_program, - lr_decay_block) + __clone_lr_op_sub_block__( + cloned_op, pserver_program, lr_decay_block + ) lr_decay_block_id = lr_decay_block.idx # append op to the current block @@ -1355,54 +1427,78 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # find the origin grad var before clipping/L2Decay, # merged_var should be the input var name of L2Decay grad_varname_for_block = op.attr(OP_ROLE_VAR_ATTR_NAME)[1] - if op.attr( - OP_ROLE_VAR_ATTR_NAME)[0] == optimize_target_param_name: + if ( + op.attr(OP_ROLE_VAR_ATTR_NAME)[0] + == optimize_target_param_name + ): merged_var = self._append_pserver_grad_merge_ops( - per_opt_block, grad_varname_for_block, endpoint, - grad_to_block_id, self.origin_program) + per_opt_block, + grad_varname_for_block, + endpoint, + grad_to_block_id, + self.origin_program, + ) if merged_var: break # append optimize op once then append other ops. if merged_var: for _, op in enumerate(self.optimize_ops): # optimizer is connected to itself - if op.attr(OP_ROLE_VAR_ATTR_NAME)[0] == optimize_target_param_name and \ - op not in global_ops: - log("append opt op: ", op.type, op.input_arg_names, - merged_var) - __append_optimize_op__(op, per_opt_block, - grad_to_block_id, merged_var, - lr_ops) + if ( + op.attr(OP_ROLE_VAR_ATTR_NAME)[0] + == optimize_target_param_name + and op not in global_ops + ): + log( + "append opt op: ", + op.type, + op.input_arg_names, + merged_var, + ) + __append_optimize_op__( + op, + per_opt_block, + grad_to_block_id, + merged_var, + lr_ops, + ) # dedup grad to ids list grad_to_block_id = list(set(grad_to_block_id)) # append global ops if global_ops: opt_state_block = pserver_program._create_block( - pserver_program.num_blocks - 1) + pserver_program.num_blocks - 1 + ) optimize_blocks.append(opt_state_block) for glb_op in global_ops: - __append_optimize_op__(glb_op, opt_state_block, - grad_to_block_id, None, lr_ops) + __append_optimize_op__( + glb_op, opt_state_block, grad_to_block_id, None, lr_ops + ) # process distributed lookup_table prefetch_var_name_to_block_id = [] if self.has_distributed_lookup_table: pserver_index = self.pserver_endpoints.index(endpoint) table_opt_block = self._create_table_optimize_block( - pserver_index, pserver_program, pre_block_idx, grad_to_block_id) + pserver_index, pserver_program, pre_block_idx, grad_to_block_id + ) optimize_blocks.append(table_opt_block) lookup_table_var_name_to_block_id = self._create_prefetch_block( - pserver_index, pserver_program, table_opt_block) + pserver_index, pserver_program, table_opt_block + ) checkpoint_block_id = self._create_checkpoint_save_block( - pserver_program, table_opt_block.idx) + pserver_program, table_opt_block.idx + ) pserver_program._distributed_lookup_table = self.table_name prefetch_var_name_to_block_id.extend( - lookup_table_var_name_to_block_id) + lookup_table_var_name_to_block_id + ) if len(optimize_blocks) == 0: - logging.warn("pserver [" + str(endpoint) + - "] has no optimize block!!") + logging.warn( + "pserver [" + str(endpoint) + "] has no optimize block!!" + ) pre_block_idx = pserver_program.num_blocks - 1 empty_block = pserver_program._create_block(pre_block_idx) optimize_blocks.append(empty_block) @@ -1420,8 +1516,7 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler "lr_decay_block_id": lr_decay_block_id, "rpc_get_thread_num": self.server_config._rpc_get_thread_num, "rpc_send_thread_num": self.server_config._rpc_send_thread_num, - "rpc_prefetch_thread_num": - self.server_config._rpc_prefetch_thread_num + "rpc_prefetch_thread_num": self.server_config._rpc_prefetch_thread_num, } if self.has_distributed_lookup_table: @@ -1431,13 +1526,16 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler if len(prefetch_var_name_to_block_id) > 0: attrs[ - 'prefetch_var_name_to_block_id'] = prefetch_var_name_to_block_id + 'prefetch_var_name_to_block_id' + ] = prefetch_var_name_to_block_id # step5 append the listen_and_serv op - pserver_program.global_block().append_op(type="listen_and_serv", - inputs={'X': recv_inputs}, - outputs={}, - attrs=attrs) + pserver_program.global_block().append_op( + type="listen_and_serv", + inputs={'X': recv_inputs}, + outputs={}, + attrs=attrs, + ) pserver_program._sync_with_cpp() # save pserver program to generate pserver side startup relatively. @@ -1471,14 +1569,14 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler pserver_program, pserver_startup_program = t.get_pserver_programs(current_endpoint) """ pserver_prog = self.get_pserver_program(endpoint) - pserver_startup = self.get_startup_program(endpoint, - pserver_program=pserver_prog) + pserver_startup = self.get_startup_program( + endpoint, pserver_program=pserver_prog + ) return pserver_prog, pserver_startup - def get_startup_program(self, - endpoint, - pserver_program=None, - startup_program=None): + def get_startup_program( + self, endpoint, pserver_program=None, startup_program=None + ): """ **Deprecated** @@ -1550,22 +1648,28 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler new_inputs = self._get_input_map_from_op(pserver_vars, op) if op.type in [ - "gaussian_random", "fill_constant", "uniform_random", - "truncated_gaussian_random" + "gaussian_random", + "fill_constant", + "uniform_random", + "truncated_gaussian_random", ]: op._set_attr("shape", list(new_outputs["Out"].shape)) - s_prog.global_block().append_op(type=op.type, - inputs=new_inputs, - outputs=new_outputs, - attrs=op.all_attrs()) + s_prog.global_block().append_op( + type=op.type, + inputs=new_inputs, + outputs=new_outputs, + attrs=op.all_attrs(), + ) if self.config.enable_dc_asgd: for p, p_bak in self.param_bak_list: startup_param_var = s_prog.global_block().vars[p.name] startup_tmpvar = s_prog.global_block().vars[p_bak.name] # copy init random value to param_bak - s_prog.global_block().append_op(type="assign", - inputs={"X": startup_param_var}, - outputs={"Out": startup_tmpvar}) + s_prog.global_block().append_op( + type="assign", + inputs={"X": startup_param_var}, + outputs={"Out": startup_tmpvar}, + ) return s_prog @@ -1588,8 +1692,9 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler orig_dim1_flatten = 1 if len(slice_vars[0].shape) >= 2: - orig_dim1_flatten = reduce(lambda x, y: x * y, - slice_vars[0].shape[1:]) + orig_dim1_flatten = reduce( + lambda x, y: x * y, slice_vars[0].shape[1:] + ) for slice_var in slice_vars[:block_idx]: skip_dim0 += slice_var.shape[0] @@ -1599,12 +1704,12 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler return is_slice, block_idx, offset def _get_distributed_optimizer_vars(self): - def _get_distributed_optimizer_var(endpoint): opt_op_on_pserver = [] for _, op in enumerate(self.optimize_ops): if self._is_optimizer_op(op) and self._is_opt_op_on_pserver( - endpoint, op): + endpoint, op + ): opt_op_on_pserver.append(op) for opt_op in opt_op_on_pserver: @@ -1613,20 +1718,25 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler if key == "Param": param_name = opt_op.input(key)[0] dist_var = self.vars_overview.get_distributed_var_by_origin_and_ep( - param_name, endpoint) + param_name, endpoint + ) break for key in opt_op.input_names: if key in [ - "Param", "Grad", "LearningRate", "Beta1Tensor", - "Beta2Tensor" + "Param", + "Grad", + "LearningRate", + "Beta1Tensor", + "Beta2Tensor", ]: continue origin_var = self.origin_program.global_block().vars[ - opt_op.input(key)[0]] + opt_op.input(key)[0] + ] # update accumulator variable shape new_shape = self._get_optimizer_input_shape( - opt_op.type, key, origin_var.shape, - dist_var.slice.shape) + opt_op.type, key, origin_var.shape, dist_var.slice.shape + ) if new_shape == dist_var.slice.shape: splited_var = VarStruct( @@ -1635,7 +1745,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler dtype=origin_var.dtype, type=origin_var.type, lod_level=origin_var.lod_level, - persistable=origin_var.persistable) + persistable=origin_var.persistable, + ) self.vars_overview.add_distributed_var( origin_var=origin_var, @@ -1644,7 +1755,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler block_id=dist_var.block_id, offset=dist_var.offset, vtype="Optimizer", - endpoint=endpoint) + endpoint=endpoint, + ) else: self.vars_overview.add_distributed_var( origin_var=origin_var, @@ -1653,13 +1765,15 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler block_id=0, offset=0, vtype="Optimizer", - endpoint=endpoint) + endpoint=endpoint, + ) for ep in self.pserver_endpoints: _get_distributed_optimizer_var(ep) - def _update_dist_lookup_table_vars(self, param_list, grad_list, - params_grads): + def _update_dist_lookup_table_vars( + self, param_list, grad_list, params_grads + ): # TODO(wuyi): put find a way to put dist lookup table stuff all together. # update self.table_param_grad and self.trainer_side_table_grad_list program = self.origin_program @@ -1668,22 +1782,25 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler param for param in param_list if param.name != self.table_name ] grad_list = [ - grad for grad in grad_list + grad + for grad in grad_list if grad.name != grad_var_name(self.table_name) ] self.table_param_grad = [ - param_grad for param_grad in params_grads + param_grad + for param_grad in params_grads if param_grad[0].name == self.table_name ][0] table_grad_var = self.table_param_grad[1] if self.sync_mode: self.trainer_side_table_grad_list = [ program.global_block().create_var( - name="%s.trainer_%d.pserver_%d" % - (table_grad_var.name, self.trainer_id, index), + name="%s.trainer_%d.pserver_%d" + % (table_grad_var.name, self.trainer_id, index), type=table_grad_var.type, shape=table_grad_var.shape, - dtype=table_grad_var.dtype) + dtype=table_grad_var.dtype, + ) for index in range(len(self.pserver_endpoints)) ] else: @@ -1692,7 +1809,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler name="%s.pserver_%d" % (table_grad_var.name, index), type=table_grad_var.type, shape=table_grad_var.shape, - dtype=table_grad_var.dtype) + dtype=table_grad_var.dtype, + ) for index in range(len(self.pserver_endpoints)) ] return param_list, grad_list @@ -1719,68 +1837,81 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler param_grad_set.add(g.name) param_list, grad_list = self._update_dist_lookup_table_vars( - param_list, grad_list, self.params_grads) + param_list, grad_list, self.params_grads + ) if self.config.slice_var_up: # when we slice var up into blocks, we will slice the var according to # pserver services' count. A pserver may have two or more listening ports. - grad_blocks = slice_variable(grad_list, len(self.pserver_endpoints), - self.config.min_block_size) - param_blocks = slice_variable(param_list, - len(self.pserver_endpoints), - self.config.min_block_size) + grad_blocks = slice_variable( + grad_list, + len(self.pserver_endpoints), + self.config.min_block_size, + ) + param_blocks = slice_variable( + param_list, + len(self.pserver_endpoints), + self.config.min_block_size, + ) else: # when we do NOT slice var up into blocks, we will always slice params # grads into one block. - grad_blocks = slice_variable(grad_list, 1, - self.config.min_block_size) - param_blocks = slice_variable(param_list, 1, - self.config.min_block_size) - assert (len(grad_blocks) == len(param_blocks)) + grad_blocks = slice_variable( + grad_list, 1, self.config.min_block_size + ) + param_blocks = slice_variable( + param_list, 1, self.config.min_block_size + ) + assert len(grad_blocks) == len(param_blocks) # origin_param_name -> [splited_param_vars] self.param_var_mapping = self._create_vars_from_blocklist( - self.origin_program, param_blocks) + self.origin_program, param_blocks + ) for orig_name, splited_vars in self.param_var_mapping.items(): orig_var = self.origin_program.global_block().var(orig_name) for splited_var in splited_vars: is_slice, block_id, offset = self._get_slice_var_info( - splited_var) - - self.vars_overview.add_distributed_var(origin_var=orig_var, - slice_var=splited_var, - block_id=block_id, - offset=offset, - is_slice=is_slice, - vtype="Param") + splited_var + ) + + self.vars_overview.add_distributed_var( + origin_var=orig_var, + slice_var=splited_var, + block_id=block_id, + offset=offset, + is_slice=is_slice, + vtype="Param", + ) # origin_grad_name -> [splited_grad_vars] self.grad_var_mapping = self._create_vars_from_blocklist( self.origin_program, grad_blocks, - add_trainer_suffix=self.trainer_num > 1) + add_trainer_suffix=self.trainer_num > 1, + ) # dict(grad_splited_var -> param_splited_var) self.grad_param_mapping = collections.OrderedDict() for g, p in zip(grad_blocks, param_blocks): g_name, g_bid, _ = g.split(":") p_name, p_bid, _ = p.split(":") - self.grad_param_mapping[self.grad_var_mapping[g_name][int(g_bid)]] = \ - self.param_var_mapping[p_name][int(p_bid)] + self.grad_param_mapping[ + self.grad_var_mapping[g_name][int(g_bid)] + ] = self.param_var_mapping[p_name][int(p_bid)] # create mapping of endpoint -> split var to create pserver side program self.param_grad_ep_mapping = collections.OrderedDict() [ - self.param_grad_ep_mapping.update({ep: { - "params": [], - "grads": [] - }}) for ep in self.pserver_endpoints + self.param_grad_ep_mapping.update({ep: {"params": [], "grads": []}}) + for ep in self.pserver_endpoints ] # transpiler function for dis lookup_table - def _replace_lookup_table_op_with_prefetch(self, program, - pserver_endpoints): + def _replace_lookup_table_op_with_prefetch( + self, program, pserver_endpoints + ): # 1. replace lookup_table_op with split_ids_op -> prefetch_op -> sum_op self.all_in_ids_vars = [] self.all_prefetch_input_vars = [] @@ -1793,16 +1924,22 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler continue_search_lookup_table_op = False all_ops = program.global_block().ops for op in all_ops: - if op.type == LOOKUP_TABLE_TYPE and self.table_name == op.input( - "W")[0]: + if ( + op.type == LOOKUP_TABLE_TYPE + and self.table_name == op.input("W")[0] + ): if not op.attr('is_distributed'): raise RuntimeError( "lookup_table_op that lookup an distributed embedding table" - "should set is_distributed to true") + "should set is_distributed to true" + ) continue_search_lookup_table_op = True - lookup_table_op_index = lookup_table_op_index if lookup_table_op_index != -1 else list( - all_ops).index(op) + lookup_table_op_index = ( + lookup_table_op_index + if lookup_table_op_index != -1 + else list(all_ops).index(op) + ) ids_name = op.input("Ids") out_name = op.output("Out") @@ -1822,14 +1959,16 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler name=str("prefetch_compress_in_tmp_" + str(index)), type=self.all_in_ids_vars[0].type, shape=self.all_in_ids_vars[0].shape, - dtype=self.all_in_ids_vars[0].dtype) + dtype=self.all_in_ids_vars[0].dtype, + ) self.all_prefetch_input_vars.append(in_var) out_var = program.global_block().create_var( name=str("prefetch_compress_out_tmp_" + str(index)), type=self.all_out_emb_vars[0].type, shape=self.all_out_emb_vars[0].shape, - dtype=self.all_out_emb_vars[0].dtype) + dtype=self.all_out_emb_vars[0].dtype, + ) self.all_prefetch_output_vars.append(out_var) # insert split_ids_op @@ -1837,7 +1976,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler index=lookup_table_op_index, type="split_ids", inputs={'Ids': self.all_in_ids_vars}, - outputs={"Out": self.all_prefetch_input_vars}) + outputs={"Out": self.all_prefetch_input_vars}, + ) # insert prefetch_op program.global_block()._insert_op( @@ -1850,7 +1990,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # FIXME(qiao) temporarily disable this config because prefetch # is not act as other rpc op, it's more like a forward op # RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + }, + ) # insert concat_op program.global_block()._insert_op( @@ -1859,9 +2000,10 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler inputs={ 'Ids': self.all_in_ids_vars, 'Rows': self.all_prefetch_input_vars, - 'X': self.all_prefetch_output_vars + 'X': self.all_prefetch_output_vars, }, - outputs={"Out": self.all_out_emb_vars}) + outputs={"Out": self.all_out_emb_vars}, + ) def _split_table_grad_and_add_send_vars(self, program, pserver_endpoints): # 2. add split_ids_op and send_op to send gradient to pservers @@ -1880,32 +2022,34 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler 'Ids': [program.global_block().vars[table_grad_name]] }, outputs={"Out": self.trainer_side_table_grad_list}, - attrs={RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE}) + attrs={RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE}, + ) program.global_block()._insert_op( index=op_index + 2, type="send", inputs={'X': self.trainer_side_table_grad_list}, outputs={ - 'Out': - [self.grad_name_to_send_dummy_out[self.table_name]] - if self.sync_mode else [] + 'Out': [ + self.grad_name_to_send_dummy_out[self.table_name] + ] + if self.sync_mode + else [] }, attrs={ - "epmap": - pserver_endpoints, - "trainer_id": - self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE, + "epmap": pserver_endpoints, + "trainer_id": self.trainer_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, OP_ROLE_VAR_ATTR_NAME: [ self.grad_name_to_param_name[table_grad_name], - table_grad_name - ] - }) + table_grad_name, + ], + }, + ) break - def _create_prefetch_block(self, pserver_index, pserver_program, - optimize_block): + def _create_prefetch_block( + self, pserver_index, pserver_program, optimize_block + ): # STEP: create prefetch block table_var = pserver_program.global_block().vars[self.table_name] prefetch_var_name_to_block_id = [] @@ -1915,46 +2059,53 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler name=trainer_ids.name, type=trainer_ids.type, shape=trainer_ids.shape, - dtype=trainer_ids.dtype) + dtype=trainer_ids.dtype, + ) trainer_out = self.all_prefetch_output_vars[pserver_index] pserver_out = pserver_program.global_block().create_var( name=trainer_out.name, type=trainer_out.type, shape=trainer_out.shape, - dtype=trainer_out.dtype) + dtype=trainer_out.dtype, + ) prefetch_block.append_op( type="lookup_sparse_table", - inputs={ - 'Ids': pserver_ids, - "W": table_var - }, + inputs={'Ids': pserver_ids, "W": table_var}, outputs={"Out": pserver_out}, attrs={ "is_sparse": True, # has no effect on lookup_table op "is_distributed": True, - "padding_idx": -1 - }) - prefetch_var_name_to_block_id.append(trainer_ids.name + ":" + - str(prefetch_block.idx)) + "padding_idx": -1, + }, + ) + prefetch_var_name_to_block_id.append( + trainer_ids.name + ":" + str(prefetch_block.idx) + ) return prefetch_var_name_to_block_id - def _create_table_optimize_block(self, pserver_index, pserver_program, - pre_block_idx, grad_to_block_id): + def _create_table_optimize_block( + self, pserver_index, pserver_program, pre_block_idx, grad_to_block_id + ): # STEP: create table optimize block table_opt_block = pserver_program._create_block(pre_block_idx) # create table param and grad var in pserver program # create table optimize block in pserver program table_opt_op = [ - op for op in self.optimize_ops if 'Param' in op.input_names + op + for op in self.optimize_ops + if 'Param' in op.input_names and op.input("Param")[0] == self.table_name ][0] origin_param_var = self.origin_program.global_block().vars[ - self.table_name] + self.table_name + ] zero_dim = int( - math.ceil(origin_param_var.shape[0] / - float(len(self.pserver_endpoints)))) + math.ceil( + origin_param_var.shape[0] / float(len(self.pserver_endpoints)) + ) + ) table_shape = list(origin_param_var.shape) table_shape[0] = zero_dim @@ -1963,28 +2114,34 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler shape=table_shape, dtype=origin_param_var.dtype, type=core.VarDesc.VarType.SELECTED_ROWS, - persistable=True) + persistable=True, + ) # parameter must be selected rows param_var.desc.set_type(core.VarDesc.VarType.SELECTED_ROWS) grad_var = pserver_program.global_block()._clone_variable( - self.origin_program.global_block().vars[grad_var_name( - self.table_name)]) + self.origin_program.global_block().vars[ + grad_var_name(self.table_name) + ] + ) lr_var = pserver_program.global_block()._clone_variable( - self.origin_program.global_block().vars[table_opt_op.input( - "LearningRate")[0]]) + self.origin_program.global_block().vars[ + table_opt_op.input("LearningRate")[0] + ] + ) if self.sync_mode: # create grad vars in pserver program table_grad_var = self.table_param_grad[1] pserver_side_table_grad_list = [ pserver_program.global_block().create_var( - name="%s.trainer_%d.pserver_%d" % - (table_grad_var.name, index, pserver_index), + name="%s.trainer_%d.pserver_%d" + % (table_grad_var.name, index, pserver_index), type=table_grad_var.type, shape=table_grad_var.shape, - dtype=table_grad_var.dtype) + dtype=table_grad_var.dtype, + ) for index in range(self.trainer_num) ] @@ -1993,28 +2150,36 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler type="sum", inputs={"X": pserver_side_table_grad_list}, outputs={"Out": [grad_var]}, - attrs={"use_mkldnn": False}) + attrs={"use_mkldnn": False}, + ) else: # in async_mode, for table gradient, it also need to be split to each parameter server origin_grad_name = grad_var.name splited_grad_name = self.trainer_side_table_grad_list[ - pserver_index].name + pserver_index + ].name if not splited_grad_name.startswith(origin_grad_name): - raise ValueError("origin_grad_var: " + splited_grad_name + - " grad_var:" + grad_var.name) + raise ValueError( + "origin_grad_var: " + + splited_grad_name + + " grad_var:" + + grad_var.name + ) grad_var = pserver_program.global_block()._rename_var( - origin_grad_name, splited_grad_name) + origin_grad_name, splited_grad_name + ) inputs = { "Param": [param_var], "Grad": [grad_var], - "LearningRate": [lr_var] + "LearningRate": [lr_var], } outputs = {"ParamOut": [param_var]} # only support sgd now logging.warn( "distribute lookup table only support sgd optimizer, change it's optimizer to sgd instead of " - + table_opt_op.type) + + table_opt_op.type + ) table_opt_block.append_op(type="sgd", inputs=inputs, outputs=outputs) # add table parameter gradient and it's block id to grad_to_block_id @@ -2027,23 +2192,26 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler create a new block to handle save checkpoint. """ - pserver_program.global_block().create_var(name="kLookupTablePath", - persistable=True, - type=core.VarDesc.VarType.RAW) + pserver_program.global_block().create_var( + name="kLookupTablePath", + persistable=True, + type=core.VarDesc.VarType.RAW, + ) checkpoint_save_block = pserver_program._create_block(pre_block_idx) # this 'file_path' do not be used in save lookup table variable - checkpoint_save_block.append_op(type='save', - inputs={'X': [self.table_name]}, - outputs={}, - attrs={'file_path': "none"}) + checkpoint_save_block.append_op( + type='save', + inputs={'X': [self.table_name]}, + outputs={}, + attrs={'file_path': "none"}, + ) return checkpoint_save_block.idx - def _create_vars_from_blocklist(self, - program, - block_list, - add_trainer_suffix=False): + def _create_vars_from_blocklist( + self, program, block_list, add_trainer_suffix=False + ): """ Create vars for each split. NOTE: only grads need to be named for different trainers, use @@ -2071,14 +2239,18 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler orig_var = program.global_block().var(varname) if len(split) == 1: if self.sync_mode and add_trainer_suffix: - new_var_name = "%s.trainer_%d" % \ - (orig_var.name, self.trainer_id) + new_var_name = "%s.trainer_%d" % ( + orig_var.name, + self.trainer_id, + ) program.global_block()._rename_var(varname, new_var_name) - var_mapping[varname] = \ - [program.global_block().var(new_var_name)] + var_mapping[varname] = [ + program.global_block().var(new_var_name) + ] else: - var_mapping[varname] = \ - [program.global_block().var(orig_var.name)] + var_mapping[varname] = [ + program.global_block().var(orig_var.name) + ] continue var_mapping[varname] = [] orig_shape = orig_var.shape @@ -2094,28 +2266,33 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler splited_shape.extend(orig_shape[1:]) new_var_name = "" if self.sync_mode and add_trainer_suffix: - new_var_name = "%s.block%d.trainer_%d" % \ - (varname, i, self.trainer_id) + new_var_name = "%s.block%d.trainer_%d" % ( + varname, + i, + self.trainer_id, + ) else: - new_var_name = "%s.block%d" % \ - (varname, i) + new_var_name = "%s.block%d" % (varname, i) var = program.global_block().create_var( name=new_var_name, persistable=False, dtype=orig_var.dtype, type=orig_var.type, - shape=splited_shape) # flattend split var + shape=splited_shape, + ) # flattend split var var_mapping[varname].append(var) program.global_block()._sync_with_cpp() return var_mapping def _clone_var(self, block, var, persistable=True): - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=persistable) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=persistable, + ) @staticmethod def _get_splited_var_sections(splited_vars): @@ -2131,34 +2308,37 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler sparse_param_name = self.grad_name_to_param_name[orig_var.name] if self._is_input_of_remote_sparse_update_op(sparse_param_name): self.sparse_param_to_height_sections[ - sparse_param_name] = height_sections - program.global_block()._insert_op(index=index + 1, - type="split_selected_rows", - inputs={"X": orig_var}, - outputs={"Out": splited_vars}, - attrs={ - "height_sections": - height_sections, - RPC_OP_ROLE_ATTR_NAME: - DIST_OP_ROLE_ATTR_VALUE - }) + sparse_param_name + ] = height_sections + program.global_block()._insert_op( + index=index + 1, + type="split_selected_rows", + inputs={"X": orig_var}, + outputs={"Out": splited_vars}, + attrs={ + "height_sections": height_sections, + RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE, + }, + ) elif orig_var.type == core.VarDesc.VarType.LOD_TENSOR: - program.global_block()._insert_op(index=index + 1, - type="split_byref", - inputs={"X": orig_var}, - outputs={"Out": splited_vars}, - attrs={ - "sections": - height_sections, - RPC_OP_ROLE_ATTR_NAME: - DIST_OP_ROLE_ATTR_VALUE - }) + program.global_block()._insert_op( + index=index + 1, + type="split_byref", + inputs={"X": orig_var}, + outputs={"Out": splited_vars}, + attrs={ + "sections": height_sections, + RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE, + }, + ) else: - AssertionError("Variable type should be in set " - "[LOD_TENSOR, SELECTED_ROWS]") + AssertionError( + "Variable type should be in set " "[LOD_TENSOR, SELECTED_ROWS]" + ) - def _get_optimizer_input_shape(self, op_type, varkey, orig_shape, - param_shape): + def _get_optimizer_input_shape( + self, op_type, varkey, orig_shape, param_shape + ): """ Returns the shape for optimizer inputs that need to be reshaped when Param and Grad is split to multiple servers. @@ -2190,8 +2370,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler pass else: raise ValueError( - "Not supported optimizer for distributed training: %s" % - op_type) + "Not supported optimizer for distributed training: %s" % op_type + ) return orig_shape def _get_varname_parts(self, varname): @@ -2201,30 +2381,36 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler block_part = "" trainer_idx = varname.find(".trainer_") if trainer_idx >= 0: - trainer_part = varname[trainer_idx + 1:] + trainer_part = varname[trainer_idx + 1 :] else: trainer_idx = len(varname) block_index = varname.find(".block") if block_index >= 0: - block_part = varname[block_index + 1:trainer_idx] + block_part = varname[block_index + 1 : trainer_idx] else: block_index = len(varname) - orig_var_name = varname[0:min(block_index, trainer_idx)] + orig_var_name = varname[0 : min(block_index, trainer_idx)] return orig_var_name, block_part, trainer_part def _orig_varname(self, varname): orig, _, _ = self._get_varname_parts(varname) return orig - def _append_pserver_grad_merge_ops(self, optimize_block, - grad_varname_for_block, endpoint, - grad_to_block_id, origin_program): + def _append_pserver_grad_merge_ops( + self, + optimize_block, + grad_varname_for_block, + endpoint, + grad_to_block_id, + origin_program, + ): program = optimize_block.program pserver_block = program.global_block() grad_block = None for g in self.param_grad_ep_mapping[endpoint]["grads"]: - if self._orig_varname(g.name) == \ - self._orig_varname(grad_varname_for_block): + if self._orig_varname(g.name) == self._orig_varname( + grad_varname_for_block + ): grad_block = g break if not grad_block: @@ -2232,7 +2418,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # is not dealing with this grad block return None orig_varname, block_name, trainer_name = self._get_varname_parts( - grad_block.name) + grad_block.name + ) if block_name: merged_var_name = '.'.join([orig_varname, block_name]) else: @@ -2240,90 +2427,104 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler merged_var = pserver_block.vars[merged_var_name] grad_to_block_id.append(merged_var.name + ":" + str(optimize_block.idx)) - if self.sync_mode or self.config.completely_not_async and self.trainer_num > 1: + if ( + self.sync_mode + or self.config.completely_not_async + and self.trainer_num > 1 + ): vars2merge = [] for i in range(self.trainer_num): - per_trainer_name = "%s.trainer_%d" % \ - (merged_var_name, i) + per_trainer_name = "%s.trainer_%d" % (merged_var_name, i) vars2merge.append(pserver_block.vars[per_trainer_name]) - optimize_block.append_op(type="sum", - inputs={"X": vars2merge}, - outputs={"Out": merged_var}, - attrs={"use_mkldnn": False}) + optimize_block.append_op( + type="sum", + inputs={"X": vars2merge}, + outputs={"Out": merged_var}, + attrs={"use_mkldnn": False}, + ) optimize_block.append_op( type="scale", inputs={"X": merged_var}, outputs={"Out": merged_var}, - attrs={"scale": 1.0 / float(self.trainer_num)}) + attrs={"scale": 1.0 / float(self.trainer_num)}, + ) return merged_var def _append_dc_asgd_ops(self, block, param_var, grad_var): # NOTE: can not use grammar candy here, should put ops in specific block - local_param_bak = block.create_var(name="%s.local_bak" % param_var.name, - shape=param_var.shape, - type=param_var.type, - dtype=param_var.dtype, - persistable=False) + local_param_bak = block.create_var( + name="%s.local_bak" % param_var.name, + shape=param_var.shape, + type=param_var.type, + dtype=param_var.dtype, + persistable=False, + ) # trainer_id_var is block local - trainer_id_var = block.create_var(name="@TRAINER_ID@", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype=core.VarDesc.VarType.INT64, - shape=[1], - persistable=False) + trainer_id_var = block.create_var( + name="@TRAINER_ID@", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype=core.VarDesc.VarType.INT64, + shape=[1], + persistable=False, + ) # ref_inputs = [x[1] for x in self.param_bak_list] ref_inputs = [] for p, p_bak in self.param_bak_list: if p.name == param_var.name: ref_inputs.append(p_bak) - block.append_op(type="ref_by_trainer_id", - inputs={ - "X": ref_inputs, - "TrainerId": trainer_id_var - }, - outputs={"Out": local_param_bak}) + block.append_op( + type="ref_by_trainer_id", + inputs={"X": ref_inputs, "TrainerId": trainer_id_var}, + outputs={"Out": local_param_bak}, + ) def __create_temp_var__(): - return block.create_var(name=unique_name.generate("tmp_dc_output"), - shape=param_var.shape, - type=param_var.type, - dtype=param_var.dtype, - persistable=False) + return block.create_var( + name=unique_name.generate("tmp_dc_output"), + shape=param_var.shape, + type=param_var.type, + dtype=param_var.dtype, + persistable=False, + ) o1 = __create_temp_var__() - block.append_op(type="elementwise_sub", - inputs={ - "X": param_var, - "Y": local_param_bak - }, - outputs={"Out": o1}) + block.append_op( + type="elementwise_sub", + inputs={"X": param_var, "Y": local_param_bak}, + outputs={"Out": o1}, + ) o2 = __create_temp_var__() - block.append_op(type="elementwise_mul", - inputs={ - "X": o1, - "Y": grad_var - }, - outputs={"Out": o2}) + block.append_op( + type="elementwise_mul", + inputs={"X": o1, "Y": grad_var}, + outputs={"Out": o2}, + ) o3 = __create_temp_var__() - block.append_op(type="elementwise_mul", - inputs={ - "X": o2, - "Y": grad_var - }, - outputs={"Out": o3}) + block.append_op( + type="elementwise_mul", + inputs={"X": o2, "Y": grad_var}, + outputs={"Out": o3}, + ) # TODO(typhoonzero): append scale o4 = __create_temp_var__() - block.append_op(type="elementwise_add", - inputs={ - "X": grad_var, - "Y": o3 - }, - outputs={"Out": o4}) + block.append_op( + type="elementwise_add", + inputs={"X": grad_var, "Y": o3}, + outputs={"Out": o4}, + ) return o4 - def _append_pserver_ops(self, optimize_block, opt_op, endpoint, - grad_to_block_id, origin_program, merged_var, - sparse_grad_to_param): + def _append_pserver_ops( + self, + optimize_block, + opt_op, + endpoint, + grad_to_block_id, + origin_program, + merged_var, + sparse_grad_to_param, + ): program = optimize_block.program pserver_block = program.global_block() new_inputs = collections.OrderedDict() @@ -2349,9 +2550,10 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # Note!! This is for l2decay on sparse gradient, because it will create a new tensor for # decayed gradient but not inplace modify the origin one origin_grad_name = opt_op.input(key)[0] - if core.kNewGradSuffix( - ) in origin_grad_name and pserver_block.has_var( - origin_grad_name): + if ( + core.kNewGradSuffix() in origin_grad_name + and pserver_block.has_var(origin_grad_name) + ): new_grad = pserver_block.var(origin_grad_name) new_inputs[key] = new_grad else: @@ -2360,10 +2562,12 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler param_block = _get_param_block(opt_op) if not param_block: return - tmpvar = pserver_block.create_var(name=param_block.name, - persistable=True, - dtype=param_block.dtype, - shape=param_block.shape) + tmpvar = pserver_block.create_var( + name=param_block.name, + persistable=True, + dtype=param_block.dtype, + shape=param_block.shape, + ) new_inputs[key] = tmpvar elif key == "LearningRate": # learning rate variable has already be created by non-optimize op, @@ -2377,42 +2581,53 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler name=origin_var.name, persistable=origin_var.persistable, dtype=origin_var.dtype, - shape=origin_var.shape) + shape=origin_var.shape, + ) new_inputs[key] = tmpvar for key in opt_op.input_names: new_shape = None if key in [ - "Param", "Grad", "LearningRate", "Beta1Tensor", - "Beta2Tensor" + "Param", + "Grad", + "LearningRate", + "Beta1Tensor", + "Beta2Tensor", ]: continue var = self.origin_program.global_block().vars[opt_op.input(key)[0]] param_var = new_inputs["Param"] # update accumulator variable shape - new_shape = self._get_optimizer_input_shape(opt_op.type, key, - var.shape, - param_var.shape) - tmpvar = pserver_block.create_var(name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=new_shape) + new_shape = self._get_optimizer_input_shape( + opt_op.type, key, var.shape, param_var.shape + ) + tmpvar = pserver_block.create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=new_shape, + ) new_inputs[key] = tmpvar # change output's ParamOut variable outputs = self._get_output_map_from_op( - self.origin_program.global_block().vars, opt_op) + self.origin_program.global_block().vars, opt_op + ) outputs["ParamOut"] = new_inputs["Param"] - optimize_block.append_op(type=opt_op.type, - inputs=new_inputs, - outputs=outputs, - attrs=opt_op.all_attrs()) + optimize_block.append_op( + type=opt_op.type, + inputs=new_inputs, + outputs=outputs, + attrs=opt_op.all_attrs(), + ) # record sparse grad to param name if new_inputs["Grad"].type == core.VarDesc.VarType.SELECTED_ROWS: sparse_grad_to_param.append( - str(new_inputs["Grad"].name) + ":" + - str(new_inputs["Param"].name)) + str(new_inputs["Grad"].name) + + ":" + + str(new_inputs["Param"].name) + ) def _get_pserver_grad_param_var(self, var, var_dict): """ @@ -2431,15 +2646,20 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # skip per trainer vars if g.name.find(".trainer_") == -1: # only param or grads have split blocks - if self._orig_varname(g.name) in self.grad_name_to_param_name or \ - self._orig_varname(g.name) in self.param_name_to_grad_name: + if ( + self._orig_varname(g.name) + in self.grad_name_to_param_name + or self._orig_varname(g.name) + in self.param_name_to_grad_name + ): grad_block = g break return grad_block def _clone_lr_op(self, program, block, op): inputs = self._get_input_map_from_op( - self.origin_program.global_block().vars, op) + self.origin_program.global_block().vars, op + ) for key, varlist in inputs.items(): if not isinstance(varlist, list): varlist = [varlist] @@ -2448,7 +2668,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler block._clone_variable(var) outputs = self._get_output_map_from_op( - self.origin_program.global_block().vars, op) + self.origin_program.global_block().vars, op + ) for key, varlist in outputs.items(): if not isinstance(varlist, list): varlist = [varlist] @@ -2456,16 +2677,16 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler if var not in program.global_block().vars: block._clone_variable(var) - return block.append_op(type=op.type, - inputs=inputs, - outputs=outputs, - attrs=op.all_attrs()) + return block.append_op( + type=op.type, inputs=inputs, outputs=outputs, attrs=op.all_attrs() + ) def _append_pserver_non_opt_ops(self, optimize_block, opt_op): program = optimize_block.program # Append the ops for parameters that do not need to be optimized/updated inputs = self._get_input_map_from_op( - self.origin_program.global_block().vars, opt_op) + self.origin_program.global_block().vars, opt_op + ) for key, varlist in inputs.items(): if not isinstance(varlist, list): varlist = [varlist] @@ -2474,8 +2695,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # for ops like clipping and weight decay, get the split var (xxx.block0) # for inputs/outputs grad_block = self._get_pserver_grad_param_var( - var, - program.global_block().vars) + var, program.global_block().vars + ) if grad_block: varlist[i] = grad_block elif var.name not in program.global_block().vars: @@ -2486,15 +2707,16 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler inputs[key] = varlist outputs = self._get_output_map_from_op( - self.origin_program.global_block().vars, opt_op) + self.origin_program.global_block().vars, opt_op + ) for key, varlist in outputs.items(): if not isinstance(varlist, list): varlist = [varlist] for i in range(len(varlist)): var = varlist[i] grad_block = self._get_pserver_grad_param_var( - var, - program.global_block().vars) + var, program.global_block().vars + ) if grad_block: varlist[i] = grad_block elif var.name not in program.global_block().vars: @@ -2504,17 +2726,20 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler varlist[i] = program.global_block().vars[var.name] outputs[key] = varlist - return optimize_block.append_op(type=opt_op.type, - inputs=inputs, - outputs=outputs, - attrs=opt_op.all_attrs()) + return optimize_block.append_op( + type=opt_op.type, + inputs=inputs, + outputs=outputs, + attrs=opt_op.all_attrs(), + ) def _is_op_connected(self, op1, op2): # If one op's input is another op's output or # one op's output is another op's input, we say # the two operator is connected. - if set(op1.desc.output_arg_names()) & set(op2.desc.input_arg_names()) or \ - set(op1.desc.input_arg_names()) & set(op2.desc.output_arg_names()): + if set(op1.desc.output_arg_names()) & set( + op2.desc.input_arg_names() + ) or set(op1.desc.input_arg_names()) & set(op2.desc.output_arg_names()): return True return False @@ -2530,8 +2755,7 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler return ufind def _is_optimizer_op(self, op): - if "Param" in op.input_names and \ - "LearningRate" in op.input_names: + if "Param" in op.input_names and "LearningRate" in op.input_names: return True return False @@ -2579,14 +2803,16 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler block = self.origin_program.global_block() for index, op in enumerate(block.ops): role_id = int(op.attr(RPC_OP_ROLE_ATTR_NAME)) - if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or \ - role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) | \ - int(OPT_OP_ROLE_ATTR_VALUE): + if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or role_id == int( + LR_SCHED_OP_ROLE_ATTR_VALUE + ) | int(OPT_OP_ROLE_ATTR_VALUE): if self.sync_mode == False and op.type == 'increment': inputs = self._get_input_map_from_op( - self.origin_program.global_block().vars, op) + self.origin_program.global_block().vars, op + ) outputs = self._get_output_map_from_op( - self.origin_program.global_block().vars, op) + self.origin_program.global_block().vars, op + ) for key in outputs: counter_var = outputs[key] all_trainer_counter_inputs = [ @@ -2595,21 +2821,29 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler type=counter_var.type, shape=counter_var.shape, dtype=counter_var.dtype, - persistable=counter_var.persistable) + persistable=counter_var.persistable, + ) for id_ in range(self.trainer_num) ] for i, op in enumerate( - self.startup_program.global_block().ops): + self.startup_program.global_block().ops + ): if op.type == 'fill_constant': for key in op.output_names: - if len(op.output(key)) == 1 and op.output( - key)[0] == counter_var.name: - self.startup_program.global_block( - ).ops[i]._set_attr( - 'value', float(0.0 - self.trainer_num)) + if ( + len(op.output(key)) == 1 + and op.output(key)[0] == counter_var.name + ): + self.startup_program.global_block().ops[ + i + ]._set_attr( + 'value', float(0.0 - self.trainer_num) + ) for var in all_trainer_counter_inputs: - if var.name == "%s.trainer_%d" % (counter_var.name, - self.trainer_id): + if var.name == "%s.trainer_%d" % ( + counter_var.name, + self.trainer_id, + ): self.counter_var = var self.startup_program.global_block().create_var( name=var.name, @@ -2617,8 +2851,10 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler dtype=var.dtype, shape=var.shape, persistable=var.persistable, - initializer=initializer.Constant(1)) - op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( + initializer=initializer.Constant(1), + ) + op_role_attr_name = ( + core.op_proto_and_checker_maker.kOpRoleAttrName() ) block._remove_op(index) op = block._insert_op( @@ -2626,7 +2862,8 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler type='sum', inputs={'X': all_trainer_counter_inputs}, outputs=outputs, - attrs={op_role_attr_name: LR_SCHED_OP_ROLE_ATTR_VALUE}) + attrs={op_role_attr_name: LR_SCHED_OP_ROLE_ATTR_VALUE}, + ) lr_ops.append(op) log("append lr op: ", op.type) return lr_ops @@ -2652,8 +2889,12 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler for op2 in block.ops: # NOTE: we need to skip all optimize ops, since it is connected # with forward/backward ops and lr ops, we only need the lr ops. - if op1 != op2 and self._is_op_connected(op1, op2) and \ - not self._is_optimizer_op(op1) and not self._is_optimizer_op(op2): + if ( + op1 != op2 + and self._is_op_connected(op1, op2) + and not self._is_optimizer_op(op1) + and not self._is_optimizer_op(op2) + ): ufind.union(op1, op2) # find all ops which is related with lr var for op1 in block.ops: @@ -2669,8 +2910,9 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler # optimize op_maker = core.op_proto_and_checker_maker optimize_role = core.op_proto_and_checker_maker.OpRole.Optimize - if op_maker.kOpRoleAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(optimize_role): + if op_maker.kOpRoleAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(optimize_role): return True return False @@ -2691,13 +2933,16 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler if self._is_opt_role_op(op): # Todo(chengmo): Whether clip related op belongs to Optimize guard should be discussed # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs( - ) and CLIP_OP_NAME_SCOPE in op.attr( - OP_NAME_SCOPE - ) and self.config.mode != "nccl2" and self.config.mode != "collective": + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + and self.config.mode != "nccl2" + and self.config.mode != "collective" + ): op._set_attr( "op_role", - int(core.op_proto_and_checker_maker.OpRole.Backward)) + int(core.op_proto_and_checker_maker.OpRole.Backward), + ) continue opt_ops.append(op) if op.attr(OP_ROLE_VAR_ATTR_NAME): @@ -2706,10 +2951,12 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler if not param_name in optimize_params: optimize_params.add(param_name) log("adding param_grad pair: ", param_name, grad_name) - params_grads.append([ - origin_var_dict[param_name], - origin_var_dict[grad_name] - ]) + params_grads.append( + [ + origin_var_dict[param_name], + origin_var_dict[grad_name], + ] + ) else: pass diff --git a/python/paddle/fluid/transpiler/geo_sgd_transpiler.py b/python/paddle/fluid/transpiler/geo_sgd_transpiler.py index fe018f611a3..0163d591cb4 100644 --- a/python/paddle/fluid/transpiler/geo_sgd_transpiler.py +++ b/python/paddle/fluid/transpiler/geo_sgd_transpiler.py @@ -30,21 +30,32 @@ import numpy as np from .ps_dispatcher import RoundRobin, PSDispatcher from .. import core, framework -from ..framework import Program, default_main_program, \ - default_startup_program, Block, Parameter +from ..framework import ( + Program, + default_main_program, + default_startup_program, + Block, + Parameter, +) from .details import wait_server_ready, VarsDistributed from .details import delete_ops from ..distribute_lookup_table import find_distributed_lookup_table -from .distribute_transpiler import DistributeTranspiler, DistributeTranspilerConfig, slice_variable, same_or_split_var, ServerRuntimeConfig +from .distribute_transpiler import ( + DistributeTranspiler, + DistributeTranspilerConfig, + slice_variable, + same_or_split_var, + ServerRuntimeConfig, +) from paddle.fluid.incubate.fleet.parameter_server.mode import DistributedMode -RPC_OP_ROLE_ATTR_NAME = op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( -) +RPC_OP_ROLE_ATTR_NAME = ( + op_role_attr_name +) = core.op_proto_and_checker_maker.kOpRoleAttrName() RPC_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.RPC class GeoSgdTranspiler(DistributeTranspiler): - def __init__(self, config=None): if config is not None: self.config = config @@ -55,17 +66,19 @@ class GeoSgdTranspiler(DistributeTranspiler): if self.config.split_method is None: self.config.split_method = RoundRobin - assert (self.config.min_block_size >= 8192) - assert (self.config.split_method.__bases__[0] == PSDispatcher) - - def transpile(self, - trainer_id, - program=None, - pservers="127.0.0.1:6174", - trainers=1, - sync_mode=False, - startup_program=None, - current_endpoint="127.0.0.1:6174"): + assert self.config.min_block_size >= 8192 + assert self.config.split_method.__bases__[0] == PSDispatcher + + def transpile( + self, + trainer_id, + program=None, + pservers="127.0.0.1:6174", + trainers=1, + sync_mode=False, + startup_program=None, + current_endpoint="127.0.0.1:6174", + ): if program is None: program = default_main_program() if startup_program is None: @@ -92,7 +105,9 @@ class GeoSgdTranspiler(DistributeTranspiler): # distribute lookup table self.table_name = find_distributed_lookup_table(self.origin_program) self.has_distributed_lookup_table = self.table_name != None - self.origin_program._distributed_lookup_table = self.table_name if self.table_name else None + self.origin_program._distributed_lookup_table = ( + self.table_name if self.table_name else None + ) # add distributed attrs to program self.origin_program._is_distributed = True @@ -127,7 +142,8 @@ class GeoSgdTranspiler(DistributeTranspiler): for i, ep in enumerate(eplist): self.param_opt_ep_mapping[ep]["params"].append(recv_vars[i]) distributed_var = self.vars_overview.get_distributed_var_by_slice( - recv_vars[i].name) + recv_vars[i].name + ) distributed_var.endpoint = ep origin_name = self.split_to_origin_mapping[recv_vars[i].name] self.vars_info[origin_name]["epmap"].append(ep) @@ -142,11 +158,14 @@ class GeoSgdTranspiler(DistributeTranspiler): if op.type == "lookup_table": op._set_attr('remote_prefetch', False) for input_var_name, sparse_var_name in zip( - op.input("Ids"), op.input("W")): + op.input("Ids"), op.input("W") + ): if sparse_var_name in self.sparse_var_list: if input_var_name in unique_sparse_var: - if unique_sparse_var[ - input_var_name] == sparse_var_name: + if ( + unique_sparse_var[input_var_name] + == sparse_var_name + ): continue input_var = program.global_block().var(input_var_name) self.sparse_var.append(input_var) @@ -155,32 +174,39 @@ class GeoSgdTranspiler(DistributeTranspiler): # batch training loop end flag dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) program.global_block().append_op( type="send", inputs={"X": self.sparse_var}, outputs={"Out": dummy_output}, - attrs={"send_varnames": self.sparse_tables}) + attrs={"send_varnames": self.sparse_tables}, + ) # add param_init flag in trainer startup program self.trainer_startup_program = self._get_trainer_startup_program( - recv_vars=recv_vars, eplist=eplist) + recv_vars=recv_vars, eplist=eplist + ) for delta_var in self.delta_vars_list: self.trainer_startup_program.global_block().create_var( name=delta_var.name, persistable=delta_var.persistable, dtype=delta_var.dtype, type=delta_var.type, - shape=delta_var.shape) + shape=delta_var.shape, + ) dummy_output = self.trainer_startup_program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) param_init = self.trainer_startup_program.global_block().create_var( - name="param_init") + name="param_init" + ) self.trainer_startup_program.global_block().append_op( type="send", inputs={"X": [param_init]}, outputs={"Out": dummy_output}, - attrs={"send_varnames": [param_init.name]}) + attrs={"send_varnames": [param_init.name]}, + ) def _get_vars_info(self): return self.vars_info @@ -193,8 +219,9 @@ class GeoSgdTranspiler(DistributeTranspiler): def get_pserver_programs(self, endpoint): pserver_prog = self.get_pserver_program(endpoint) self.param_grad_ep_mapping = self.param_opt_ep_mapping - pserver_startup = self.get_startup_program(endpoint, - pserver_program=pserver_prog) + pserver_startup = self.get_startup_program( + endpoint, pserver_program=pserver_prog + ) return pserver_prog, pserver_startup def get_pserver_program(self, endpoint): @@ -224,21 +251,27 @@ class GeoSgdTranspiler(DistributeTranspiler): delta_var_name = "%s.delta" % (param.name) if var.name in self.sparse_var_splited_list: delta_type = core.VarDesc.VarType.SELECTED_ROWS - sparse_grad_to_param.append(":".join( - [delta_var_name, param.name])) + sparse_grad_to_param.append( + ":".join([delta_var_name, param.name]) + ) else: delta_type = param.type - delta_var = pserver_block.create_var(name=delta_var_name, - persistable=False, - type=delta_type, - dtype=param.dtype, - shape=param.shape) - - per_opt_block.append_op(type="sum", - inputs={"X": [param, delta_var]}, - outputs={"Out": param}) - param_to_block_id.append(delta_var_name + ":" + - str(per_opt_block.idx)) + delta_var = pserver_block.create_var( + name=delta_var_name, + persistable=False, + type=delta_type, + dtype=param.dtype, + shape=param.shape, + ) + + per_opt_block.append_op( + type="sum", + inputs={"X": [param, delta_var]}, + outputs={"Out": param}, + ) + param_to_block_id.append( + delta_var_name + ":" + str(per_opt_block.idx) + ) attrs = { "optimize_blocks": optimize_block, @@ -249,15 +282,16 @@ class GeoSgdTranspiler(DistributeTranspiler): "sparse_grad_to_param": sparse_grad_to_param, "rpc_get_thread_num": self.server_config._rpc_get_thread_num, "rpc_send_thread_num": self.server_config._rpc_send_thread_num, - "rpc_prefetch_thread_num": - self.server_config._rpc_prefetch_thread_num + "rpc_prefetch_thread_num": self.server_config._rpc_prefetch_thread_num, } # step5 append the listen_and_serv op - pserver_program.global_block().append_op(type="listen_and_serv", - inputs={'X': recv_inputs}, - outputs={}, - attrs=attrs) + pserver_program.global_block().append_op( + type="listen_and_serv", + inputs={'X': recv_inputs}, + outputs={}, + attrs=attrs, + ) pserver_program._sync_with_cpp() # save pserver program to generate pserver side startup relatively. @@ -284,21 +318,28 @@ class GeoSgdTranspiler(DistributeTranspiler): # step 2. Slice vars into numbers of piece with block_size # when we slice var up into blocks, we will slice the var according to # pserver services' count. A pserver may have two or more listening ports. - param_blocks = slice_variable(param_list, len(self.pserver_endpoints), - self.config.min_block_size) + param_blocks = slice_variable( + param_list, len(self.pserver_endpoints), self.config.min_block_size + ) # step 3. Create split param from split blocks # origin_param_name -> [splited_param_vars] # Todo: update _create_vars_from_blocklist self.param_var_mapping = self._create_vars_from_blocklist( - self.origin_program, param_blocks) + self.origin_program, param_blocks + ) # step 4. Create mapping of endpoint -> split var to create pserver side program self.param_opt_ep_mapping = collections.OrderedDict() [ - self.param_opt_ep_mapping.update({ep: { - "params": [], - }}) for ep in self.pserver_endpoints + self.param_opt_ep_mapping.update( + { + ep: { + "params": [], + } + } + ) + for ep in self.pserver_endpoints ] # step 5. Create delta var of Geo-Sgd & record vars information @@ -325,28 +366,34 @@ class GeoSgdTranspiler(DistributeTranspiler): persistable=False, dtype=origin_var.dtype, type=delta_type, - shape=origin_var.shape) + shape=origin_var.shape, + ) self.delta_vars_list.append(delta_var) for splited_var in splited_vars: is_slice, block_id, offset = self._get_slice_var_info( - splited_var) - self.vars_overview.add_distributed_var(origin_var=origin_var, - slice_var=splited_var, - block_id=block_id, - offset=offset, - is_slice=is_slice, - vtype="Param") + splited_var + ) + self.vars_overview.add_distributed_var( + origin_var=origin_var, + slice_var=splited_var, + block_id=block_id, + offset=offset, + is_slice=is_slice, + vtype="Param", + ) self.split_to_origin_mapping[splited_var.name] = origin_name if origin_name in self.sparse_var_list: self.sparse_var_splited_list.append(splited_var.name) self.vars_info[origin_name]["var_names"].append( - splited_var.name) + splited_var.name + ) if len(splited_vars) != 1: self.origin_program.global_block().create_var( name=".".join([splited_var.name, "delta"]), persistable=False, dtype=splited_var.dtype, type=delta_type, - shape=splited_var.shape) + shape=splited_var.shape, + ) diff --git a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py index 0aa4a9d8f54..c71197fa0ad 100755 --- a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py +++ b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py @@ -15,13 +15,11 @@ import logging -def memory_optimize(input_program, - skip_opt_set=None, - print_log=False, - level=0, - skip_grads=True): +def memory_optimize( + input_program, skip_opt_set=None, print_log=False, level=0, skip_grads=True +): """ - :api_attr: Static Graph + :api_attr: Static Graph This API is deprecated since 1.6. Please do not use it. The better memory optimization strategies are enabled by default. @@ -38,15 +36,18 @@ def memory_optimize(input_program, 'default value).\n' ' 2. Inplace strategy, which is enabled by setting ' 'build_strategy.enable_inplace=True (True is the default value) ' - 'when using CompiledProgram or ParallelExecutor.\n') + 'when using CompiledProgram or ParallelExecutor.\n' + ) def release_memory(input_program, skip_opt_set=None): """ - :api_attr: Static Graph + :api_attr: Static Graph This API is deprecated since 1.6. Please do not use it. The better memory optimization strategies are enabled by default. """ - logging.warn('paddle.fluid.release_memory() is deprecated, it would not' - ' take any memory release on your program') + logging.warn( + 'paddle.fluid.release_memory() is deprecated, it would not' + ' take any memory release on your program' + ) diff --git a/python/paddle/fluid/transpiler/ps_dispatcher.py b/python/paddle/fluid/transpiler/ps_dispatcher.py index 0bf7442fdea..6271f8152c1 100644 --- a/python/paddle/fluid/transpiler/ps_dispatcher.py +++ b/python/paddle/fluid/transpiler/ps_dispatcher.py @@ -46,7 +46,7 @@ class PSDispatcher(object): class HashName(PSDispatcher): """ - :api_attr: Static Graph + :api_attr: Static Graph Hash variable names to several endpoints using python "hash()" function. @@ -88,7 +88,7 @@ class HashName(PSDispatcher): class RoundRobin(PSDispatcher): """ - :api_attr: Static Graph + :api_attr: Static Graph Distribute variables to several endpoints using RondRobin method. diff --git a/python/paddle/fluid/unique_name.py b/python/paddle/fluid/unique_name.py index 39270ec2dfe..1c0c28dae8b 100644 --- a/python/paddle/fluid/unique_name.py +++ b/python/paddle/fluid/unique_name.py @@ -121,6 +121,7 @@ def generate(key): # in order to keep name consistency. def generate_with_ignorable_key(key): from .framework import _non_static_mode, _dygraph_tracer + if _non_static_mode(): return _dygraph_tracer()._generate_unique_name() diff --git a/python/paddle/fluid/variable_index.py b/python/paddle/fluid/variable_index.py index 3d67cd7e230..f3c7fff38fa 100644 --- a/python/paddle/fluid/variable_index.py +++ b/python/paddle/fluid/variable_index.py @@ -22,7 +22,6 @@ MAX_INTEGER = 2**31 - 1 def is_list_tuple(index, contain_type): - def _is_list_tuple(item): if not (isinstance(item, (list, tuple)) or type(item) == contain_type): return False @@ -58,14 +57,13 @@ def get_list_index_shape(var_dims, index_dims): out_dims_shape = [1] * out_dims_size - out_dims_shape[:index_dims_size - 1] = index_dims[1:] + out_dims_shape[: index_dims_size - 1] = index_dims[1:] - out_dims_shape[index_dims_size - 1:] = var_dims[index_dims[0]:] + out_dims_shape[index_dims_size - 1 :] = var_dims[index_dims[0] :] return out_dims_shape class SliceInfo: - def __init__(self): self.pre_shape = None self.indexes = [] @@ -73,7 +71,8 @@ class SliceInfo: def update(self, index): if is_list_tuple(index, int) or isinstance( - index, (paddle.fluid.Variable, np.ndarray)): + index, (paddle.fluid.Variable, np.ndarray) + ): # convert index to Tensor if not isinstance(index, paddle.fluid.Variable): index = paddle.assign(index) @@ -83,8 +82,10 @@ class SliceInfo: else: if index.dtype != self.dtype: raise IndexError( - "Data type of Tensor/List index should be same. The current data type is {}, but the previous data type is {}." - .format(index.dtype, self.dtype)) + "Data type of Tensor/List index should be same. The current data type is {}, but the previous data type is {}.".format( + index.dtype, self.dtype + ) + ) self.indexes.append(index) @@ -93,16 +94,20 @@ class SliceInfo: else: if self.pre_shape != index.shape: # broadcast - cur_shape = paddle.broadcast_shape(self.pre_shape, - index.shape) + cur_shape = paddle.broadcast_shape( + self.pre_shape, index.shape + ) for i in range(len(self.indexes)): self.indexes[i] = paddle.broadcast_to( - self.indexes[i], cur_shape) + self.indexes[i], cur_shape + ) self.pre_shape = self.indexes[-1].shape else: raise ValueError( - "Index should be list/tuple of int or Tensor, but received {}.". - format(index)) + "Index should be list/tuple of int or Tensor, but received {}.".format( + index + ) + ) def shape_stride(self, shape): s = [1] * len(shape) @@ -119,19 +124,22 @@ class SliceInfo: if not isinstance(index, paddle.fluid.Variable): raise ValueError( "only support list/tensor index, but received {}.".format( - type(index))) + type(index) + ) + ) if len(self.indexes) <= len(tensor_shape) or len(self.indexes) == 1: shape = paddle.stack(self.indexes) - axes = list(range(1, - len(self.pre_shape) + 1)) + [ - 0, - ] + axes = list(range(1, len(self.pre_shape) + 1)) + [ + 0, + ] else: raise ValueError( - "too many indices for tensor: tensor is {}-dimensional, but {} were indexed" - .format(len(tensor_shape), self.pre_shape[0])) + "too many indices for tensor: tensor is {}-dimensional, but {} were indexed".format( + len(tensor_shape), self.pre_shape[0] + ) + ) shape_transpose = paddle.transpose(shape, axes) return shape_transpose @@ -148,7 +156,8 @@ class SliceInfo: tensor_type = None if tensor_origin.dtype in [ - core.VarDesc.VarType.FP32, core.VarDesc.VarType.FP64 + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP64, ]: tensor = tensor_origin else: @@ -161,36 +170,47 @@ class SliceInfo: shape_transpose = self.get_offset_stride(tensor_origin.shape) index = paddle.assign(shape_transpose) - gather_tensor_shape = get_list_index_shape(tensor.shape, [ - len(self.indexes), - ] + list(self.indexes[-1].shape)) + gather_tensor_shape = get_list_index_shape( + tensor.shape, + [ + len(self.indexes), + ] + + list(self.indexes[-1].shape), + ) value_dims_bd = [ 1, ] * len(gather_tensor_shape) - value_dims_bd[-len(value.shape):] = list(value.shape) + value_dims_bd[-len(value.shape) :] = list(value.shape) for i in range(len(gather_tensor_shape)): - if not (value_dims_bd[i] == gather_tensor_shape[i] - or value_dims_bd[i] == 1): - raise ValueError("{} can not broadcast into {}".format( - value.shape, gather_tensor_shape)) + if not ( + value_dims_bd[i] == gather_tensor_shape[i] + or value_dims_bd[i] == 1 + ): + raise ValueError( + "{} can not broadcast into {}".format( + value.shape, gather_tensor_shape + ) + ) value_broadcast = paddle.broadcast_to(value, gather_tensor_shape) value_1d = value_broadcast.reshape( - [-1] + gather_tensor_shape[len(index.shape) - 1:]) + [-1] + gather_tensor_shape[len(index.shape) - 1 :] + ) index_1d = index.reshape([-1, index.shape[-1]]) tensor_stride = paddle.assign( - self.shape_stride(tensor.shape[:index.shape[-1]])) + self.shape_stride(tensor.shape[: index.shape[-1]]) + ) inds = [] for i in range(index_1d.shape[0]): temp = (index_1d[i] * tensor_stride).sum() inds.append(temp) index_1d = paddle.stack(inds).reshape([-1]) - t_reshape = tensor.reshape([-1] + list(tensor.shape[index.shape[-1]:])) + t_reshape = tensor.reshape([-1] + list(tensor.shape[index.shape[-1] :])) out = paddle.scatter(t_reshape, index_1d, value_1d) if tensor_type is not None: out = out.astype(tensor_type) @@ -201,6 +221,7 @@ class SliceInfo: def replace_ellipsis(var, item): from .framework import Variable + # Use slice(None) to replace Ellipsis. # For var, var.shape = [3,4,5,6] # @@ -212,7 +233,8 @@ def replace_ellipsis(var, item): # Remove Variable to skip bug when counting Ellipsis item_remove_var = [ - ele for ele in item + ele + for ele in item if not isinstance(ele, (Variable, np.ndarray)) and ele is not None ] ell_count = item_remove_var.count(Ellipsis) @@ -226,9 +248,9 @@ def replace_ellipsis(var, item): if ell_idx == len(item) - 1: return item[:-1] else: - item[ell_idx:ell_idx + - 1] = [slice(None) - ] * (len(var.shape) - len(item) + item.count(None) + 1) + item[ell_idx : ell_idx + 1] = [slice(None)] * ( + len(var.shape) - len(item) + item.count(None) + 1 + ) return item @@ -256,6 +278,7 @@ def replace_none(item): def is_integer_or_scalar_tensor(ele): from .framework import Variable + if isinstance(ele, int): return True elif isinstance(ele, Variable): @@ -266,6 +289,7 @@ def is_integer_or_scalar_tensor(ele): def is_bool_tensor(ele): from .framework import Variable + if isinstance(ele, Variable) and ele.dtype == paddle.bool: return True return False @@ -276,8 +300,9 @@ def deal_attrs(attrs, attr, attr_name, tensor_attr_name, inputs, infer_flags): from .layers import utils if utils._contain_var(attr): - inputs[tensor_attr_name] = utils._convert_to_tensor_list(attr, - dtype="int64") + inputs[tensor_attr_name] = utils._convert_to_tensor_list( + attr, dtype="int64" + ) for i, dim in enumerate(attr): if isinstance(dim, Variable): attrs[attr_name].append(-1) @@ -291,16 +316,19 @@ def deal_attrs(attrs, attr, attr_name, tensor_attr_name, inputs, infer_flags): # the item is a tensor of bool def get_value_for_bool_tensor(var, item): if len(item.shape) > len(var.shape): - raise IndexError("The dims of bool index doesn't match indexed array, " - "the dims of bool index except to be equal or less " - "than {}, but received {}.".format( - len(var.shape), len(item.shape))) + raise IndexError( + "The dims of bool index doesn't match indexed array, " + "the dims of bool index except to be equal or less " + "than {}, but received {}.".format(len(var.shape), len(item.shape)) + ) for i, dim_len in enumerate(item.shape): if dim_len != var.shape[i]: raise IndexError( - "The dimension of bool index doesn't match indexed array along "\ - "dimension {}, the target dimension is {}, but received {}.". - format(i, var.shape[i], dim_len)) + "The dimension of bool index doesn't match indexed array along " + "dimension {}, the target dimension is {}, but received {}.".format( + i, var.shape[i], dim_len + ) + ) def idx_not_empty(var, item): from .layers.nn import where @@ -315,8 +343,10 @@ def get_value_for_bool_tensor(var, item): return paddle.empty(var_shape, dtype=var.dtype) from .layers.control_flow import cond - return cond(item.any(), lambda: idx_not_empty(var, item), - lambda: idx_empty(var)) + + return cond( + item.any(), lambda: idx_not_empty(var, item), lambda: idx_empty(var) + ) def _getitem_impl_(var, item): @@ -330,12 +360,13 @@ def _getitem_impl_(var, item): Sliced variable """ from .framework import default_main_program, Variable + if isinstance(item, list): if not is_one_dim_list(item, int): item = tuple(item) if not isinstance(item, tuple): - item = (item, ) + item = (item,) decrease_axes = [] axes = [] @@ -351,11 +382,15 @@ def _getitem_impl_(var, item): slice_info = SliceInfo() for dim, slice_item in enumerate(item): - if is_integer_or_scalar_tensor( - slice_item) and not is_bool_tensor(slice_item): - if isinstance(slice_item, - int) and var.shape[dim] is not None and var.shape[ - dim] >= 0 and slice_item >= var.shape[dim]: + if is_integer_or_scalar_tensor(slice_item) and not is_bool_tensor( + slice_item + ): + if ( + isinstance(slice_item, int) + and var.shape[dim] is not None + and var.shape[dim] >= 0 + and slice_item >= var.shape[dim] + ): # For python, if users write a, b = var, the __getitem__ # method will iterate through 0, 1, 2 ... until __getitem__ # throws an IndexError, then stop. The var[0], var[1] will @@ -365,7 +400,8 @@ def _getitem_impl_(var, item): # We raises IndexError here to support grammar like `a, b = var` raise IndexError( "slice_item %d at dim %d should be >= 0 and < var.shape[%d]: %d" - % (slice_item, dim, dim, var.shape[dim])) + % (slice_item, dim, dim, var.shape[dim]) + ) decrease_axes.append(dim) start = slice_item step = 1 @@ -385,9 +421,9 @@ def _getitem_impl_(var, item): start = 0 if step > 0 else MAX_INTEGER if end is None: if var.shape[dim] != -1 and ( - paddle.fluid.framework._non_static_mode() - or var.desc.type() != - core.VarDesc.VarType.LOD_TENSOR_ARRAY): + paddle.fluid.framework._non_static_mode() + or var.desc.type() != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): end = var.shape[dim] if step > 0 else -1 else: end = MAX_INTEGER if step > 0 else -1 @@ -407,15 +443,19 @@ def _getitem_impl_(var, item): if len(item) != 1: raise IndexError( - "When index contains a list, its length must be 1, but received {}." - .format(len(item))) + "When index contains a list, its length must be 1, but received {}.".format( + len(item) + ) + ) new_slice_item = [] if all_bool: if len(slice_item) != var.shape[0]: raise IndexError( - "The dimension of bool index doesn't match indexed array along "\ - "dimension 0, the target dimension is {}, but received {}.". - format(var.shape[0], len(slice_item))) + "The dimension of bool index doesn't match indexed array along " + "dimension 0, the target dimension is {}, but received {}.".format( + var.shape[0], len(slice_item) + ) + ) for idx, ele in enumerate(slice_item): if ele is True: new_slice_item.append(idx) @@ -455,8 +495,10 @@ def _getitem_impl_(var, item): else: raise IndexError( - "Valid index accept int or slice or ellipsis or list, but received {}." - .format(slice_item)) + "Valid index accept int or slice or ellipsis or list, but received {}.".format( + slice_item + ) + ) axes.append(dim) starts.append(start) @@ -467,8 +509,10 @@ def _getitem_impl_(var, item): if slice_info.indexes: if len(slice_info.indexes) != len(item): raise IndexError( - "Valid index accept int or slice or ellipsis or list, but received {}." - .format(item)) + "Valid index accept int or slice or ellipsis or list, but received {}.".format( + item + ) + ) return slice_info.get_item(var) inputs = {'Input': [var]} @@ -476,7 +520,7 @@ def _getitem_impl_(var, item): 'axes': axes, 'starts': [], 'ends': [], - 'decrease_axis': decrease_axes + 'decrease_axis': decrease_axes, } if use_strided_slice: attrs['strides'] = [] @@ -484,8 +528,9 @@ def _getitem_impl_(var, item): infer_flags = [1] * len(axes) deal_attrs(attrs, starts, "starts", "StartsTensorList", inputs, infer_flags) deal_attrs(attrs, ends, "ends", "EndsTensorList", inputs, infer_flags) - deal_attrs(attrs, steps, "strides", "StridesTensorList", inputs, - infer_flags) + deal_attrs( + attrs, steps, "strides", "StridesTensorList", inputs, infer_flags + ) attrs['infer_flags'] = infer_flags out = var @@ -500,23 +545,29 @@ def _getitem_impl_(var, item): end = inputs['EndsTensorList'] else: end = attrs['ends'] - out = paddle._C_ops.slice(var, axes, st, end, attrs['infer_flags'], - attrs['decrease_axis']) + out = paddle._C_ops.slice( + var, axes, st, end, attrs['infer_flags'], attrs['decrease_axis'] + ) else: target_block = default_main_program().current_block() slice_out_var = target_block.create_var( - name=unique_name.generate_with_ignorable_key(var.name + "_" + - op_type), - dtype=var.dtype) - target_block.append_op(type=op_type, - inputs=inputs, - outputs={'Out': [slice_out_var]}, - attrs=attrs) + name=unique_name.generate_with_ignorable_key( + var.name + "_" + op_type + ), + dtype=var.dtype, + ) + target_block.append_op( + type=op_type, + inputs=inputs, + outputs={'Out': [slice_out_var]}, + attrs=attrs, + ) out = slice_out_var if len(reverse_axes) > 0: from .layers.tensor import reverse + out = reverse(out, axis=reverse_axes) # Deal with cases when all axes are decreased. @@ -546,40 +597,49 @@ def _getitem_impl_(var, item): # out = x[0, 1, 1, None] # out.shape : (1) from ..tensor import unsqueeze + out = unsqueeze(out, axis=none_axes) return out def _setitem_for_tensor_array(var, item, value): - """ branches for tensor array setitem operation. - A item can be a: - (1) int/Variable, which is a simple number/variable such as [1], [-2] - (2) Slice, which is represented by bounds such as [2:-1] - (3) Tuple, which includes the above two cases such as [2:-1, 1] - If item is case (1), we perform paddle.tensor.array_write, - in other cases, we raise a NotImplementedError. + """branches for tensor array setitem operation. + A item can be a: + (1) int/Variable, which is a simple number/variable such as [1], [-2] + (2) Slice, which is represented by bounds such as [2:-1] + (3) Tuple, which includes the above two cases such as [2:-1, 1] + If item is case (1), we perform paddle.tensor.array_write, + in other cases, we raise a NotImplementedError. """ from ..framework import LayerHelper, core, _non_static_mode from .framework import Variable - assert not _non_static_mode( + + assert ( + not _non_static_mode() ), "setitem for tensor_array must be called in static graph mode." if isinstance(item, (Variable, int)): - from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable + from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + to_static_variable, + ) from paddle import cast from paddle.tensor import array_write + item = paddle.cast(to_static_variable(item), dtype='int64') value = to_static_variable(value) array_write(x=value, i=item, array=var) else: raise NotImplementedError( - "Only support __setitem__ by Int/Variable in tensor_array, but gets {}" - .format(type(item))) + "Only support __setitem__ by Int/Variable in tensor_array, but gets {}".format( + type(item) + ) + ) def _setitem_impl_(var, item, value): from .framework import default_main_program, Variable from paddle.fluid import core + if var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: return _setitem_for_tensor_array(var, item, value) @@ -589,7 +649,7 @@ def _setitem_impl_(var, item, value): item = tuple(item) # 1. Parse item if not isinstance(item, tuple): - item = (item, ) + item = (item,) decrease_axes = [] axes = [] @@ -603,8 +663,9 @@ def _setitem_impl_(var, item, value): slice_info = SliceInfo() dim = 0 for _, slice_item in enumerate(item): - if is_integer_or_scalar_tensor( - slice_item) and not is_bool_tensor(slice_item): + if is_integer_or_scalar_tensor(slice_item) and not is_bool_tensor( + slice_item + ): decrease_axes.append(dim) start = slice_item end = slice_item + 1 if slice_item != -1 else MAX_INTEGER @@ -624,7 +685,8 @@ def _setitem_impl_(var, item, value): if not isinstance(step, Variable) and step == 0: raise ValueError( "When assign a value to a paddle.Tensor, step can not be 0, " - "but received step is {}.".format(step)) + "but received step is {}.".format(step) + ) if isinstance(step, Variable) and (start is None or end is None): raise ValueError( @@ -644,15 +706,19 @@ def _setitem_impl_(var, item, value): for i in slice_item: if not isinstance(i, bool): - raise TypeError("Doesn't support {} in index list.".format( - type(i))) + raise TypeError( + "Doesn't support {} in index list.".format(type(i)) + ) if len(item) != 1: raise IndexError( - "When index contains a bool list, its length must be 1, but received {}." - .format(len(item))) + "When index contains a bool list, its length must be 1, but received {}.".format( + len(item) + ) + ) from .layers import assign + idx_tensor = assign(slice_item) return set_value_for_bool_tensor(var, idx_tensor, value) @@ -660,8 +726,10 @@ def _setitem_impl_(var, item, value): if slice_item.dtype == core.VarDesc.VarType.BOOL: if len(item) != 1: raise IndexError( - "When index contains a bool tensor, its length must be 1, but received {}." - .format(len(item))) + "When index contains a bool tensor, its length must be 1, but received {}.".format( + len(item) + ) + ) return set_value_for_bool_tensor(var, slice_item, value) else: slice_info.update(slice_item) @@ -669,7 +737,8 @@ def _setitem_impl_(var, item, value): else: raise IndexError( "Valid index accept int, slice, ellipsis, None, list of bool, Variable, " - "but received {}.".format(slice_item)) + "but received {}.".format(slice_item) + ) axes.append(dim) starts.append(start) @@ -680,8 +749,10 @@ def _setitem_impl_(var, item, value): if slice_info.indexes: if len(slice_info.indexes) != len(item): raise IndexError( - "Valid index accept int or slice or ellipsis or list, but received {}." - .format(item)) + "Valid index accept int or slice or ellipsis or list, but received {}.".format( + item + ) + ) return slice_info.set_item(var, value) attrs = { 'axes': axes, @@ -689,10 +760,11 @@ def _setitem_impl_(var, item, value): 'ends': ends, 'steps': steps, 'decrease_axes': decrease_axes, - 'none_axes': none_axes + 'none_axes': none_axes, } from .layers import utils + if utils._contain_var(starts): inputs['StartsTensorList'] = utils._convert_to_tensor_list(starts) del attrs['starts'] @@ -708,6 +780,7 @@ def _setitem_impl_(var, item, value): attrs['dtype'] = dtype from .data_feeder import convert_dtype + # 2.1 value is an integer of float if isinstance(value, (int, float)): value = np.array([value]).astype(convert_dtype(dtype)) @@ -737,7 +810,8 @@ def _setitem_impl_(var, item, value): raise TypeError( "When assign a numpy.ndarray, integer or float to a paddle.Tensor, " "the data type of the paddle.Tensor must be bool, float32, int32, int64 or float16, but " - "received %s." % convert_dtype(dtype)) + "received %s." % convert_dtype(dtype) + ) attrs[value_name] = values attrs["shape"] = shape @@ -747,17 +821,21 @@ def _setitem_impl_(var, item, value): raise TypeError( "Only support to assign an integer, float, numpy.ndarray or " "paddle.Tensor to a paddle.Tensor, but received {}".format( - type(value))) + type(value) + ) + ) if paddle.fluid.framework._non_static_mode(): var._bump_inplace_version() cur_block = default_main_program().current_block() - cur_block.append_op(type="set_value", - inputs=inputs, - outputs={'Out': var}, - attrs=attrs, - inplace_map={"Input": "Out"}) + cur_block.append_op( + type="set_value", + inputs=inputs, + outputs={'Out': var}, + attrs=attrs, + inplace_map={"Input": "Out"}, + ) return var @@ -765,16 +843,19 @@ def _setitem_impl_(var, item, value): # the item is a tensor of bool def set_value_for_bool_tensor(var, item, value): if len(item.shape) > len(var.shape): - raise IndexError("The dims of bool index doesn't match indexed array, " - "the dims of bool index except to be equal or less " - "than {}, but received {}.".format( - len(var.shape), len(item.shape))) + raise IndexError( + "The dims of bool index doesn't match indexed array, " + "the dims of bool index except to be equal or less " + "than {}, but received {}.".format(len(var.shape), len(item.shape)) + ) for i, dim_len in enumerate(item.shape): if dim_len != var.shape[i]: raise IndexError( "The dimension of bool index doesn't match indexed array along " - "dimension {}, the target dimension is {}, but received {}.". - format(i, var.shape[i], dim_len)) + "dimension {}, the target dimension is {}, but received {}.".format( + i, var.shape[i], dim_len + ) + ) def idx_not_empty(var, item, value): from .framework import Variable @@ -792,6 +873,7 @@ def set_value_for_bool_tensor(var, item, value): var[:] = out from .layers.control_flow import cond + # If all the bool index is False, just do nothing cond(item.any(), lambda: idx_not_empty(var, item, value)) diff --git a/python/paddle/fluid/wrapped_decorator.py b/python/paddle/fluid/wrapped_decorator.py index 5f837b57563..7e7dbff6561 100644 --- a/python/paddle/fluid/wrapped_decorator.py +++ b/python/paddle/fluid/wrapped_decorator.py @@ -19,7 +19,6 @@ __all__ = ['wrap_decorator', 'signature_safe_contextmanager'] def wrap_decorator(decorator_func): - @decorator.decorator def __impl__(func, *args, **kwargs): wrapped_func = decorator_func(func) diff --git a/python/paddle/framework/__init__.py b/python/paddle/framework/__init__.py index 34423e3f3ed..a6b79dd29e2 100644 --- a/python/paddle/framework/__init__.py +++ b/python/paddle/framework/__init__.py @@ -47,10 +47,19 @@ from ..fluid.framework import set_flags # noqa: F401 from ..fluid.dygraph.base import enable_dygraph as disable_static # noqa: F401 from ..fluid.dygraph.base import disable_dygraph as enable_static # noqa: F401 from ..fluid.framework import _non_static_mode as in_dynamic_mode # noqa: F401 -from ..fluid.framework import _non_static_mode # noqa: F401; temporary used for hackson -from ..fluid.framework import _current_expected_place, _get_paddle_place # noqa: F401 +from ..fluid.framework import ( + _non_static_mode, +) # noqa: F401; temporary used for hackson +from ..fluid.framework import ( + _current_expected_place, + _get_paddle_place, +) # noqa: F401 from ..fluid.framework import dygraph_only # noqa: F401 -from ..fluid.framework import convert_np_dtype_to_dtype_, _varbase_creator, OpProtoHolder # noqa: F401 +from ..fluid.framework import ( + convert_np_dtype_to_dtype_, + _varbase_creator, + OpProtoHolder, +) # noqa: F401 from ..fluid.framework import _dygraph_tracer # noqa: F401 from ..fluid.layer_helper import LayerHelper # noqa: F401 diff --git a/python/paddle/framework/framework.py b/python/paddle/framework/framework.py index 8da6a10a485..e366b13077d 100644 --- a/python/paddle/framework/framework.py +++ b/python/paddle/framework/framework.py @@ -45,11 +45,16 @@ def set_default_dtype(d): else: raise TypeError( "set_default_dtype only supports [float16, float32, float64] " - ", but received %s" % d.__name__) + ", but received %s" % d.__name__ + ) else: if d in [ - 'float16', 'float32', 'float64', u'float16', u'float32', - u'float64' + 'float16', + 'float32', + 'float64', + u'float16', + u'float32', + u'float64', ]: # this code is a little bit dangerous, since error could happen # when casting no-ascii code to str in python2. @@ -60,7 +65,8 @@ def set_default_dtype(d): else: raise TypeError( "set_default_dtype only supports [float16, float32, float64] " - ", but received %s" % str(d)) + ", but received %s" % str(d) + ) LayerHelperBase.set_default_dtype(d) diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 77e1fb47f88..0052c1fe655 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -24,14 +24,31 @@ import paddle # deprecated module import from paddle import fluid from paddle.fluid import core -from paddle.fluid.io import _unpack_saved_dict, _pack_loaded_dict, _pickle_loads_mac +from paddle.fluid.io import ( + _unpack_saved_dict, + _pack_loaded_dict, + _pickle_loads_mac, +) from paddle.fluid.io import _legacy_save as _legacy_static_save from paddle.fluid.io import _open_file_buffer, _is_file_path, _is_memory_buffer -from paddle.fluid.framework import Variable, _varbase_creator, _dygraph_tracer, _non_static_mode, ParamBase, EagerParamBase, _current_expected_place, Program +from paddle.fluid.framework import ( + Variable, + _varbase_creator, + _dygraph_tracer, + _non_static_mode, + ParamBase, + EagerParamBase, + _current_expected_place, + Program, +) from paddle.fluid.dygraph.jit import _SaveLoadConfig -from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers +from paddle.fluid.dygraph.io import ( + _construct_program_holders, + _construct_params_and_buffers, +) from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX + try: from collections.abc import Iterable except: @@ -68,7 +85,8 @@ def _load_state_dict_from_save_inference_model(model_path, config): # 2. load layer parameters & buffers with fluid.dygraph.guard(): persistable_var_dict = _construct_params_and_buffers( - model_path, programs, config.params_filename, append_suffix=False) + model_path, programs, config.params_filename, append_suffix=False + ) # 3. construct state_dict load_param_dict = dict() @@ -84,10 +102,15 @@ def _load_state_dict_from_save_inference_model(model_path, config): structured_para_dict = dict() for var_name in load_param_dict: structured_name = extra_var_info[var_name].get( - 'structured_name', None) - assert structured_name is not None, "Cannot find saved variable (%s)'s structured name in saved model." % var_name + 'structured_name', None + ) + assert structured_name is not None, ( + "Cannot find saved variable (%s)'s structured name in saved model." + % var_name + ) structured_para_dict[structured_name] = load_param_dict[ - var_name] + var_name + ] load_param_dict = structured_para_dict return load_param_dict @@ -115,7 +138,8 @@ def _load_state_dict_from_save_params(model_path): type='load', inputs={}, outputs={'Out': new_var}, - attrs={'file_path': os.path.join(model_path, name)}) + attrs={'file_path': os.path.join(model_path, name)}, + ) load_var_list.append(new_var) # 3. construct state_dict @@ -151,7 +175,8 @@ def _build_load_path_and_config(path, config): raise ValueError( "The %s.pdmodel and %s directory exist at the same time, " "don't know which one to load, please make sure that the specified target " - "of ``path`` is unique." % (path, path)) + "of ``path`` is unique." % (path, path) + ) elif not prefix_format_exist and not directory_format_exist: error_msg = "The ``path`` (%s) to load model not exists." # if current path is a prefix, and the path.pdparams or path.pdopt @@ -160,10 +185,12 @@ def _build_load_path_and_config(path, config): params_file_path = path + ".pdparams" opti_file_path = path + ".pdopt" if os.path.exists(params_file_path) or os.path.exists(opti_file_path): - error_msg += " If you want to load the results saved by `fluid.save_dygraph`, " \ - "please specify the full file name, not just the file name prefix. For " \ - "example, it should be written as `paddle.load('model.pdparams')` instead of " \ + error_msg += ( + " If you want to load the results saved by `fluid.save_dygraph`, " + "please specify the full file name, not just the file name prefix. For " + "example, it should be written as `paddle.load('model.pdparams')` instead of " "`paddle.load('model')`." + ) raise ValueError(error_msg % path) else: if prefix_format_exist: @@ -173,13 +200,15 @@ def _build_load_path_and_config(path, config): warnings.warn( "When loading the result saved with the " "specified file prefix, the ``model_filename`` config does " - "not take effect.") + "not take effect." + ) config.model_filename = file_prefix + INFER_MODEL_SUFFIX if config.params_filename is not None: warnings.warn( "When loading the result saved with the " "specified file prefix, the ``params_filename`` config does " - "not take effect.") + "not take effect." + ) config.params_filename = file_prefix + INFER_PARAMS_SUFFIX else: # Compatible with the old save_inference_model format @@ -190,7 +219,10 @@ def _build_load_path_and_config(path, config): def _parse_load_config(configs): supported_configs = [ - 'model_filename', 'params_filename', 'keep_name_table', 'return_numpy' + 'model_filename', + 'params_filename', + 'keep_name_table', + 'return_numpy', ] # input check @@ -198,7 +230,8 @@ def _parse_load_config(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.load` is not supported." - % key) + % key + ) # construct inner config inner_config = _SaveLoadConfig() @@ -218,7 +251,8 @@ def _parse_save_config(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.save` is not supported." - % key) + % key + ) # construct inner config inner_config = _SaveLoadConfig() @@ -231,19 +265,22 @@ def _parse_save_config(configs): def _pickle_save(obj, f, protocol): # TODO(weixin):add support for BytesIO. if not isinstance(protocol, int): - raise ValueError("The 'protocol' MUST be `int`, but received {}".format( - type(protocol))) + raise ValueError( + "The 'protocol' MUST be `int`, but received {}".format( + type(protocol) + ) + ) if protocol < 2 or protocol > 4: raise ValueError( - "Expected 1<'protocol'<5, but received protocol={}".format( - protocol)) + "Expected 1<'protocol'<5, but received protocol={}".format(protocol) + ) def reduce_varbase(self): data = self.numpy() name = self.name - return (tuple, ((name, data), )) + return (tuple, ((name, data),)) def reduce_LoDTensor(self): data = np.array(self) @@ -252,7 +289,8 @@ def _pickle_save(obj, f, protocol): def reduce_Layer(self): raise ValueError( - "paddle do not support saving `paddle.nn.Layer` object.") + "paddle do not support saving `paddle.nn.Layer` object." + ) dispatch_table_layer = dict() @@ -260,8 +298,9 @@ def _pickle_save(obj, f, protocol): dispatch_table_layer[layer.__class__] = reduce_Layer return layer - _parse_every_object(obj, lambda v: isinstance(v, fluid.Layer), - create_layer_dispatch_table) + _parse_every_object( + obj, lambda v: isinstance(v, fluid.Layer), create_layer_dispatch_table + ) def add_dispatch_table(): # This is not a good method, because the pickle module has been modified. @@ -289,7 +328,7 @@ def _pickle_save(obj, f, protocol): max_bytes = 2**30 for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i:i + max_bytes]) + f.write(pickle_bytes[i : i + max_bytes]) else: pickler = pickle.Pickler(f, protocol) pickler.dispatch_table = copyreg.dispatch_table.copy() @@ -306,7 +345,8 @@ def _pickle_save(obj, f, protocol): def _contain_x(obj, condition_func): if isinstance(obj, core.SelectedRows): raise NotImplementedError( - "`paddle.save` do not support saving 'SelectedRows'.") + "`paddle.save` do not support saving 'SelectedRows'." + ) if condition_func(obj): return True @@ -330,8 +370,16 @@ def _is_state_dict(obj): def condition(obj): return isinstance( - obj, (fluid.Layer, Program, core.VarBase, core.eager.Tensor, - core.LoDTensor, core.SelectedRows)) + obj, + ( + fluid.Layer, + Program, + core.VarBase, + core.eager.Tensor, + core.LoDTensor, + core.SelectedRows, + ), + ) # If the value of a dict is a core.VarBase/LoDTensor or a dict # that does not contain a paddle type(Layer, Program, VarBase, LoDTensor, SelectedRows), @@ -342,7 +390,8 @@ def _is_state_dict(obj): if _contain_x(v, condition): return False elif not isinstance( - value, (core.VarBase, core.eager.Tensor, core.LoDTensor)): + value, (core.VarBase, core.eager.Tensor, core.LoDTensor) + ): return False return True @@ -370,8 +419,10 @@ def _transformed_from_lodtensor(obj): def _to_LodTensor(ndarray): if not isinstance(ndarray, np.ndarray): raise TypeError( - 'Type of `ndarray` should be numpy.ndarray, but received {}.'. - format(type(ndarray))) + 'Type of `ndarray` should be numpy.ndarray, but received {}.'.format( + type(ndarray) + ) + ) t = core.LoDTensor() place = _current_expected_place() t.set(ndarray, place) @@ -418,26 +469,30 @@ def _parse_every_object(obj, condition_func, convert_func): if condition_func(obj[key]): obj[key] = convert_func(obj[key]) else: - obj[key] = _parse_every_object(obj[key], condition_func, - convert_func) + obj[key] = _parse_every_object( + obj[key], condition_func, convert_func + ) return obj elif type(obj) == tuple: return tuple( - _parse_every_object(list(obj), condition_func, convert_func)) + _parse_every_object(list(obj), condition_func, convert_func) + ) elif type(obj) == set: return set(_parse_every_object(list(obj), condition_func, convert_func)) else: if isinstance(obj, Iterable) and not isinstance( - obj, - (str, np.ndarray, core.VarBase, core.eager.Tensor, core.LoDTensor)): + obj, + (str, np.ndarray, core.VarBase, core.eager.Tensor, core.LoDTensor), + ): raise NotImplementedError( - "The iteratable objects supported are tuple, list, dict, OrderedDict, string. But received {}." - .format(type(obj))) + "The iteratable objects supported are tuple, list, dict, OrderedDict, string. But received {}.".format( + type(obj) + ) + ) return obj def _parse_load_result(obj, return_numpy): - def is_layer(obj): return isinstance(obj, fluid.Layer) @@ -463,13 +518,15 @@ def _parse_load_result(obj, return_numpy): # tuple(name, ndarry) was converted from varbase of paddle2.1, # and all tuple(name, ndarry) are converted to tensor. if _contain_x(obj, _transformed_from_varbase): - return _parse_every_object(obj, _transformed_from_varbase, - tuple_to_tensor) + return _parse_every_object( + obj, _transformed_from_varbase, tuple_to_tensor + ) # If there is no tuple(name, ndary), it is considered to be saved by paddle2.0 # or converted from LoDTensor, and all ndarrays are converted to tensor. else: - return _parse_every_object(obj, _transformed_from_lodtensor, - ndarray_to_tensor) + return _parse_every_object( + obj, _transformed_from_lodtensor, ndarray_to_tensor + ) def _save_lod_tensor(tensor, file_name): @@ -490,8 +547,10 @@ def _save_lod_tensor(tensor, file_name): else: raise NotImplementedError( - 'Only supports saving objects to file or BytesIO, but received {}'. - format(type(file_name))) + 'Only supports saving objects to file or BytesIO, but received {}'.format( + type(file_name) + ) + ) return _seek @@ -509,8 +568,10 @@ def _load_lod_tensor(file_name): else: raise NotImplementedError( - 'Only supports load objects from file or BytesIO, but received {}'. - format(type(file_name))) + 'Only supports load objects from file or BytesIO, but received {}'.format( + type(file_name) + ) + ) return temp_t, _seek @@ -529,8 +590,10 @@ def _save_selected_rows(selected_rows, file_name): _seek = f.tell() else: raise NotImplementedError( - 'Only supports saving objects to file or BytesIO, but received {}'. - format(type(file_name))) + 'Only supports saving objects to file or BytesIO, but received {}'.format( + type(file_name) + ) + ) return _seek @@ -544,13 +607,16 @@ def _load_selected_rows(file_name): with _open_file_buffer(file_name, 'rb') as f: selected_rows_bytes = f.read() paddle.fluid.core.load_selected_rows_from_memory( - temp_sr, selected_rows_bytes) + temp_sr, selected_rows_bytes + ) _seek = f.tell() else: raise NotImplementedError( - 'Only supports load objects from file or BytesIO, but received {}'. - format(type(file_name))) + 'Only supports load objects from file or BytesIO, but received {}'.format( + type(file_name) + ) + ) return temp_sr, _seek @@ -565,8 +631,10 @@ def _save_binary_var(obj, path): else: # Since the concept of 'Tensor' is only exposed to users, the error message can only contain tensor instead of 'LoDTensor' or 'SelectedRows' raise NotImplementedError( - "When use_binary_format = True, `paddle.save` expected Tensor, but received {}." - .format(type(obj))) + "When use_binary_format = True, `paddle.save` expected Tensor, but received {}.".format( + type(obj) + ) + ) def save(obj, path, protocol=4, **configs): @@ -694,7 +762,8 @@ def save(obj, path, protocol=4, **configs): raise ValueError( "The input path MUST be format of dirname/filename " "[dirname\\filename in Windows system], but received " - "filename is empty string.") + "filename is empty string." + ) # 2. save object dirname = os.path.dirname(path) @@ -702,15 +771,19 @@ def save(obj, path, protocol=4, **configs): os.makedirs(dirname) elif not _is_memory_buffer(path): raise ValueError( - "only supports saving objects to file and `BytesIO`, but got {}". - format(type(path))) + "only supports saving objects to file and `BytesIO`, but got {}".format( + type(path) + ) + ) config = _parse_save_config(configs) if not isinstance(config.use_binary_format, bool): raise TypeError( - "Type of `use_binary_format` should be bool, but received {}.". - format(type(config.use_binary_format))) + "Type of `use_binary_format` should be bool, but received {}.".format( + type(config.use_binary_format) + ) + ) if config.use_binary_format: _save_binary_var(obj, path) @@ -742,19 +815,23 @@ def _legacy_save(obj, path, protocol=2): if not isinstance(obj, dict): raise NotImplementedError( "Now only supports save state_dict of Layer or Optimizer, " - "expect dict, but received %s." % type(obj)) + "expect dict, but received %s." % type(obj) + ) if len(obj) == 0: warnings.warn("The input state dict is empty, no need to save.") if not isinstance(protocol, int): - raise ValueError("The 'protocol' MUST be `int`, but received {}".format( - type(protocol))) + raise ValueError( + "The 'protocol' MUST be `int`, but received {}".format( + type(protocol) + ) + ) if protocol < 2 or protocol > 4: raise ValueError( - "Expected 1<'protocol'<5, but received protocol={}".format( - protocol)) + "Expected 1<'protocol'<5, but received protocol={}".format(protocol) + ) if _is_file_path(path): filename = os.path.basename(path) @@ -762,7 +839,8 @@ def _legacy_save(obj, path, protocol=2): raise ValueError( "The input path MUST be format of dirname/filename " "[dirname\\filename in Windows system], but received " - "filename is empty string.") + "filename is empty string." + ) # 2. save object dirname = os.path.dirname(path) if dirname and not os.path.exists(dirname): @@ -774,13 +852,16 @@ def _legacy_save(obj, path, protocol=2): saved_obj = _unpack_saved_dict(saved_obj, protocol) # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if _is_file_path( - path) and sys.platform == 'darwin' and sys.version_info.major == 3: + if ( + _is_file_path(path) + and sys.platform == 'darwin' + and sys.version_info.major == 3 + ): pickle_bytes = pickle.dumps(saved_obj, protocol=protocol) with open(path, 'wb') as f: max_bytes = 2**30 for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i:i + max_bytes]) + f.write(pickle_bytes[i : i + max_bytes]) else: with _open_file_buffer(path, 'wb') as f: pickle.dump(saved_obj, f, protocol=protocol) @@ -947,9 +1028,11 @@ def load(path, **configs): try: with _open_file_buffer(path, 'rb') as f: # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if _is_file_path( - path - ) and sys.platform == 'darwin' and sys.version_info.major == 3: + if ( + _is_file_path(path) + and sys.platform == 'darwin' + and sys.version_info.major == 3 + ): load_result = _pickle_loads_mac(path, f) else: load_result = pickle.load(f, encoding='latin1') @@ -963,18 +1046,24 @@ def load(path, **configs): for key in load_result["StructuredToParameterName@@"]: if isinstance(load_result[key], np.ndarray): load_result[key] = _ndarray_to_tensor( - load_result[key], config.return_numpy) + load_result[key], config.return_numpy + ) - if not config.keep_name_table and "StructuredToParameterName@@" in load_result: + if ( + not config.keep_name_table + and "StructuredToParameterName@@" in load_result + ): del load_result["StructuredToParameterName@@"] else: # paddle2.1 static.save/load load_result = _parse_load_result( - load_result, config.return_numpy) + load_result, config.return_numpy + ) else: - load_result = _parse_load_result(load_result, - config.return_numpy) + load_result = _parse_load_result( + load_result, config.return_numpy + ) except exception_type as msg_pickle: try: @@ -994,12 +1083,15 @@ def load(path, **configs): with _open_file_buffer(path, "rb") as f: program_desc_str = f.read() program = Program.parse_from_string( - program_desc_str) + program_desc_str + ) return program except: raise ValueError( "`paddle.load` can not parse the file:{}.".format( - path)) + path + ) + ) else: load_result = _legacy_load(path, **configs) @@ -1016,7 +1108,10 @@ def _legacy_load(path, **configs): with _open_file_buffer(path, 'rb') as f: load_result = pickle.load(f, encoding='latin1') load_result = _pack_loaded_dict(load_result) - if not config.keep_name_table and "StructuredToParameterName@@" in load_result: + if ( + not config.keep_name_table + and "StructuredToParameterName@@" in load_result + ): del load_result["StructuredToParameterName@@"] else: # file prefix and directory are compatible cases @@ -1037,7 +1132,8 @@ def _legacy_load(path, **configs): # the user to configure the `use_structured_name` argument when `set_state_dict` # NOTE(chenweihang): `jit.save` doesn't save optimizer state load_result = _load_state_dict_from_save_inference_model( - model_path, config) + model_path, config + ) else: # load state dict by `io.save_params/persistables` save format # TODO(chenweihang): [ Now only supports loading parameters separately ] diff --git a/python/paddle/framework/random.py b/python/paddle/framework/random.py index 50235591cf2..6d7d704808d 100644 --- a/python/paddle/framework/random.py +++ b/python/paddle/framework/random.py @@ -37,7 +37,7 @@ def seed(seed): gen = paddle.seed(102) """ - #TODO(zhiqiu): 1. remove program.random_seed when all random-related op upgrade + # TODO(zhiqiu): 1. remove program.random_seed when all random-related op upgrade # 2. support gpu generator by global device seed = int(seed) diff --git a/python/paddle/geometric/math.py b/python/paddle/geometric/math.py index 186fe5c2d7d..a4a27093484 100644 --- a/python/paddle/geometric/math.py +++ b/python/paddle/geometric/math.py @@ -54,29 +54,30 @@ def segment_sum(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "SUM")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "SUM") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "SUM" + ) return out check_variable_and_dtype( - data, "X", ("float32", "float64", "int32", "int64", "float16"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + data, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "segment_pool", + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_sum", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "SUM"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "SUM"}, + ) return out @@ -116,29 +117,30 @@ def segment_mean(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "MEAN")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MEAN") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MEAN" + ) return out check_variable_and_dtype( - data, "X", ("float32", "float64", "int32", "int64", "float16"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + data, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "segment_pool", + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_mean", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MEAN"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MEAN"}, + ) return out @@ -177,29 +179,30 @@ def segment_min(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "MIN")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MIN") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MIN" + ) return out check_variable_and_dtype( - data, "X", ("float32", "float64", "int32", "int64", "float16"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + data, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "segment_pool", + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_min", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MIN"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MIN"}, + ) return out @@ -238,27 +241,28 @@ def segment_max(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "MAX")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MAX") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MAX" + ) return out check_variable_and_dtype( - data, "X", ("float32", "float64", "int32", "int64", "float16"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + data, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "segment_pool", + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_max", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MAX"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MAX"}, + ) return out diff --git a/python/paddle/geometric/message_passing/send_recv.py b/python/paddle/geometric/message_passing/send_recv.py index cf290b29bee..8a05ef10516 100644 --- a/python/paddle/geometric/message_passing/send_recv.py +++ b/python/paddle/geometric/message_passing/send_recv.py @@ -16,20 +16,25 @@ import numpy as np from paddle.fluid.layer_helper import LayerHelper from paddle.fluid.framework import _in_legacy_dygraph, in_dygraph_mode from paddle.fluid.framework import Variable -from paddle.fluid.data_feeder import check_dtype, check_type, check_variable_and_dtype +from paddle.fluid.data_feeder import ( + check_dtype, + check_type, + check_variable_and_dtype, +) from paddle import _C_ops, _legacy_C_ops -from .utils import convert_out_size_to_list, get_out_size_tensor_inputs, reshape_lhs_rhs +from .utils import ( + convert_out_size_to_list, + get_out_size_tensor_inputs, + reshape_lhs_rhs, +) __all__ = [] -def send_u_recv(x, - src_index, - dst_index, - reduce_op="sum", - out_size=None, - name=None): +def send_u_recv( + x, src_index, dst_index, reduce_op="sum", out_size=None, name=None +): """ Graph Learning message passing api. @@ -110,66 +115,85 @@ def send_u_recv(x, if reduce_op not in ["sum", "mean", "max", "min"]: raise ValueError( "reduce_op should be `sum`, `mean`, `max` or `min`, but received %s" - % reduce_op) + % reduce_op + ) # TODO(daisiming): Should we add judgement for out_size: max(dst_index) + 1. if _in_legacy_dygraph(): out_size = convert_out_size_to_list(out_size) - out, tmp = _legacy_C_ops.graph_send_recv(x, src_index, dst_index, - None, 'reduce_op', - reduce_op.upper(), 'out_size', - out_size) + out, tmp = _legacy_C_ops.graph_send_recv( + x, + src_index, + dst_index, + None, + 'reduce_op', + reduce_op.upper(), + 'out_size', + out_size, + ) return out if in_dygraph_mode(): out_size = convert_out_size_to_list(out_size) - return _C_ops.graph_send_recv(x, src_index, dst_index, - reduce_op.upper(), out_size) + return _C_ops.graph_send_recv( + x, src_index, dst_index, reduce_op.upper(), out_size + ) check_variable_and_dtype( - x, "X", ("float32", "float64", "int32", "int64", "float16"), - "graph_send_recv") - check_variable_and_dtype(src_index, "Src_index", ("int32", "int64"), - "graph_send_recv") - check_variable_and_dtype(dst_index, "Dst_index", ("int32", "int64"), - "graph_send_recv") + x, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "graph_send_recv", + ) + check_variable_and_dtype( + src_index, "Src_index", ("int32", "int64"), "graph_send_recv" + ) + check_variable_and_dtype( + dst_index, "Dst_index", ("int32", "int64"), "graph_send_recv" + ) if out_size: - check_type(out_size, 'out_size', (int, np.int32, np.int64, Variable), - 'graph_send_recv') + check_type( + out_size, + 'out_size', + (int, np.int32, np.int64, Variable), + 'graph_send_recv', + ) if isinstance(out_size, Variable): - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], - 'graph_send_recv') + check_dtype( + out_size.dtype, 'out_size', ['int32', 'int64'], 'graph_send_recv' + ) helper = LayerHelper("send_u_recv", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - dst_count = helper.create_variable_for_type_inference(dtype="int32", - stop_gradient=True) + dst_count = helper.create_variable_for_type_inference( + dtype="int32", stop_gradient=True + ) inputs = {"X": x, "Src_index": src_index, "Dst_index": dst_index} attrs = {"reduce_op": reduce_op.upper()} - get_out_size_tensor_inputs(inputs=inputs, - attrs=attrs, - out_size=out_size, - op_type='graph_send_recv') - - helper.append_op(type="graph_send_recv", - inputs=inputs, - outputs={ - "Out": out, - "Dst_count": dst_count - }, - attrs=attrs) + get_out_size_tensor_inputs( + inputs=inputs, attrs=attrs, out_size=out_size, op_type='graph_send_recv' + ) + + helper.append_op( + type="graph_send_recv", + inputs=inputs, + outputs={"Out": out, "Dst_count": dst_count}, + attrs=attrs, + ) return out -def send_ue_recv(x, - y, - src_index, - dst_index, - message_op="add", - reduce_op="sum", - out_size=None, - name=None): +def send_ue_recv( + x, + y, + src_index, + dst_index, + message_op="add", + reduce_op="sum", + out_size=None, + name=None, +): """ Graph Learning message passing api. @@ -259,13 +283,15 @@ def send_ue_recv(x, if message_op not in ["add", "sub", "mul", "div"]: raise ValueError( - "message_op should be `add`, `sub`, `mul`, `div`, but received %s" % - message_op) + "message_op should be `add`, `sub`, `mul`, `div`, but received %s" + % message_op + ) if reduce_op not in ["sum", "mean", "max", "min"]: raise ValueError( "reduce_op should be `sum`, `mean`, `max` or `min`, but received %s" - % reduce_op) + % reduce_op + ) x, y = reshape_lhs_rhs(x, y) @@ -274,61 +300,89 @@ def send_ue_recv(x, y = -y if message_op == "div": message_op = 'mul' - y = 1. / (y + 1e-12) + y = 1.0 / (y + 1e-12) # TODO(daisiming): Should we add judgement for out_size: max(dst_index) + 1. if _in_legacy_dygraph(): out_size = convert_out_size_to_list(out_size) - out, tmp = _legacy_C_ops.graph_send_ue_recv(x, y, src_index, dst_index, - None, 'message_op', - message_op.upper(), - 'reduce_op', - reduce_op.upper(), - 'out_size', out_size) + out, tmp = _legacy_C_ops.graph_send_ue_recv( + x, + y, + src_index, + dst_index, + None, + 'message_op', + message_op.upper(), + 'reduce_op', + reduce_op.upper(), + 'out_size', + out_size, + ) return out if in_dygraph_mode(): out_size = convert_out_size_to_list(out_size) - return _C_ops.graph_send_ue_recv(x, y, src_index, dst_index, - message_op.upper(), reduce_op.upper(), - out_size) + return _C_ops.graph_send_ue_recv( + x, + y, + src_index, + dst_index, + message_op.upper(), + reduce_op.upper(), + out_size, + ) check_variable_and_dtype( - x, "X", ("float32", "float64", "int32", "int64", "float16"), - "graph_send_ue_recv") + x, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "graph_send_ue_recv", + ) + check_variable_and_dtype( + y, + "Y", + ("float32", "float64", "int32", "int64", "float16"), + "graph_send_ue_recv", + ) check_variable_and_dtype( - y, "Y", ("float32", "float64", "int32", "int64", "float16"), - "graph_send_ue_recv") - check_variable_and_dtype(src_index, "Src_index", ("int32", "int64"), - "graph_send_ue_recv") - check_variable_and_dtype(dst_index, "Dst_index", ("int32", "int64"), - "graph_send_ue_recv") + src_index, "Src_index", ("int32", "int64"), "graph_send_ue_recv" + ) + check_variable_and_dtype( + dst_index, "Dst_index", ("int32", "int64"), "graph_send_ue_recv" + ) if out_size: - check_type(out_size, 'out_size', (int, np.int32, np.int64, Variable), - 'graph_send_ue_recv') + check_type( + out_size, + 'out_size', + (int, np.int32, np.int64, Variable), + 'graph_send_ue_recv', + ) if isinstance(out_size, Variable): - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], - 'graph_send_ue_recv') + check_dtype( + out_size.dtype, 'out_size', ['int32', 'int64'], 'graph_send_ue_recv' + ) helper = LayerHelper("send_ue_recv", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - dst_count = helper.create_variable_for_type_inference(dtype="int32", - stop_gradient=True) + dst_count = helper.create_variable_for_type_inference( + dtype="int32", stop_gradient=True + ) inputs = {"X": x, "Y": y, "Src_index": src_index, "Dst_index": dst_index} attrs = {"message_op": message_op.upper(), "reduce_op": reduce_op.upper()} - get_out_size_tensor_inputs(inputs=inputs, - attrs=attrs, - out_size=out_size, - op_type='graph_send_ue_recv') - - helper.append_op(type="graph_send_ue_recv", - inputs=inputs, - outputs={ - "Out": out, - "Dst_count": dst_count - }, - attrs=attrs) + get_out_size_tensor_inputs( + inputs=inputs, + attrs=attrs, + out_size=out_size, + op_type='graph_send_ue_recv', + ) + + helper.append_op( + type="graph_send_ue_recv", + inputs=inputs, + outputs={"Out": out, "Dst_count": dst_count}, + attrs=attrs, + ) return out @@ -397,8 +451,9 @@ def send_uv(x, y, src_index, dst_index, message_op="add", name=None): if message_op not in ['add', 'sub', 'mul', 'div']: raise ValueError( - "message_op should be `add`, `sub`, `mul`, `div`, but received %s" % - message_op) + "message_op should be `add`, `sub`, `mul`, `div`, but received %s" + % message_op + ) x, y = reshape_lhs_rhs(x, y) @@ -407,38 +462,50 @@ def send_uv(x, y, src_index, dst_index, message_op="add", name=None): y = -y if message_op == 'div': message_op = 'mul' - y = 1. / (y + 1e-12) + y = 1.0 / (y + 1e-12) if in_dygraph_mode(): - return _C_ops.graph_send_uv(x, y, src_index, dst_index, - message_op.upper()) + return _C_ops.graph_send_uv( + x, y, src_index, dst_index, message_op.upper() + ) else: if _in_legacy_dygraph(): - return _legacy_C_ops.graph_send_uv(x, y, src_index, dst_index, - "message_op", message_op.upper()) + return _legacy_C_ops.graph_send_uv( + x, y, src_index, dst_index, "message_op", message_op.upper() + ) else: helper = LayerHelper("send_uv", **locals()) check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float32', 'float64', 'float16'], - 'graph_send_uv') + x, + 'x', + ['int32', 'int64', 'float32', 'float64', 'float16'], + 'graph_send_uv', + ) + check_variable_and_dtype( + y, + 'y', + ['int32', 'int64', 'float32', 'float64', 'float16'], + 'graph_send_uv', + ) + check_variable_and_dtype( + src_index, 'src_index', ['int32', 'int64'], 'graph_send_uv' + ) check_variable_and_dtype( - y, 'y', ['int32', 'int64', 'float32', 'float64', 'float16'], - 'graph_send_uv') - check_variable_and_dtype(src_index, 'src_index', ['int32', 'int64'], - 'graph_send_uv') - check_variable_and_dtype(dst_index, 'dst_index', ['int32', 'int64'], - 'graph_send_uv') + dst_index, 'dst_index', ['int32', 'int64'], 'graph_send_uv' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) inputs = { 'x': x, 'y': y, 'src_index': src_index, - 'dst_index': dst_index + 'dst_index': dst_index, } attrs = {'message_op': message_op.upper()} - helper.append_op(type="graph_send_uv", - inputs=inputs, - attrs=attrs, - outputs={"out": out}) + helper.append_op( + type="graph_send_uv", + inputs=inputs, + attrs=attrs, + outputs={"out": out}, + ) return out diff --git a/python/paddle/geometric/message_passing/utils.py b/python/paddle/geometric/message_passing/utils.py index 51c08852298..12e2e52d6b4 100644 --- a/python/paddle/geometric/message_passing/utils.py +++ b/python/paddle/geometric/message_passing/utils.py @@ -44,9 +44,14 @@ def get_out_size_tensor_inputs(inputs, attrs, out_size, op_type): attrs['out_size'] = [out_size] elif isinstance(out_size, Variable): out_size.stop_gradient = True - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], 'op_type', - '(When type of out_size in' + op_type + ' is Variable.)') - if (convert_dtype(out_size.dtype) == 'int64'): + check_dtype( + out_size.dtype, + 'out_size', + ['int32', 'int64'], + 'op_type', + '(When type of out_size in' + op_type + ' is Variable.)', + ) + if convert_dtype(out_size.dtype) == 'int64': out_size = cast(out_size, 'int32') inputs["Out_size"] = out_size else: @@ -69,16 +74,26 @@ def reshape_lhs_rhs(x, y): max_ndims = max(len(x.shape), len(y.shape)) x_pad_ndims = max_ndims - len(x.shape) y_pad_ndims = max_ndims - len(y.shape) - new_x_shape = [ - x_shape[0], - ] + [ - 1, - ] * x_pad_ndims + list(x_shape[1:]) - new_y_shape = [ - y_shape[0], - ] + [ - 1, - ] * y_pad_ndims + list(y_shape[1:]) + new_x_shape = ( + [ + x_shape[0], + ] + + [ + 1, + ] + * x_pad_ndims + + list(x_shape[1:]) + ) + new_y_shape = ( + [ + y_shape[0], + ] + + [ + 1, + ] + * y_pad_ndims + + list(y_shape[1:]) + ) x = paddle.reshape(x, new_x_shape) y = paddle.reshape(y, new_y_shape) diff --git a/python/paddle/geometric/reindex.py b/python/paddle/geometric/reindex.py index 3c4cfb964ed..ade88222bbe 100644 --- a/python/paddle/geometric/reindex.py +++ b/python/paddle/geometric/reindex.py @@ -21,12 +21,9 @@ from paddle import _legacy_C_ops __all__ = [] -def reindex_graph(x, - neighbors, - count, - value_buffer=None, - index_buffer=None, - name=None): +def reindex_graph( + x, neighbors, count, value_buffer=None, index_buffer=None, name=None +): """ Reindex Graph API. @@ -93,58 +90,62 @@ def reindex_graph(x, # out_nodes: [0, 1, 2, 8, 9, 4, 7, 6] """ - use_buffer_hashtable = True if value_buffer is not None \ - and index_buffer is not None else False + use_buffer_hashtable = ( + True if value_buffer is not None and index_buffer is not None else False + ) if _non_static_mode(): - reindex_src, reindex_dst, out_nodes = \ - _legacy_C_ops.graph_reindex(x, neighbors, count, value_buffer, index_buffer, - "flag_buffer_hashtable", use_buffer_hashtable) + reindex_src, reindex_dst, out_nodes = _legacy_C_ops.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + "flag_buffer_hashtable", + use_buffer_hashtable, + ) return reindex_src, reindex_dst, out_nodes check_variable_and_dtype(x, "X", ("int32", "int64"), "graph_reindex") - check_variable_and_dtype(neighbors, "Neighbors", ("int32", "int64"), - "graph_reindex") + check_variable_and_dtype( + neighbors, "Neighbors", ("int32", "int64"), "graph_reindex" + ) check_variable_and_dtype(count, "Count", ("int32"), "graph_reindex") if use_buffer_hashtable: - check_variable_and_dtype(value_buffer, "HashTable_Value", ("int32"), - "graph_reindex") - check_variable_and_dtype(index_buffer, "HashTable_Index", ("int32"), - "graph_reindex") + check_variable_and_dtype( + value_buffer, "HashTable_Value", ("int32"), "graph_reindex" + ) + check_variable_and_dtype( + index_buffer, "HashTable_Index", ("int32"), "graph_reindex" + ) helper = LayerHelper("reindex_graph", **locals()) reindex_src = helper.create_variable_for_type_inference(dtype=x.dtype) reindex_dst = helper.create_variable_for_type_inference(dtype=x.dtype) out_nodes = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="graph_reindex", - inputs={ - "X": - x, - "Neighbors": - neighbors, - "Count": - count, - "HashTable_Value": - value_buffer if use_buffer_hashtable else None, - "HashTable_Index": - index_buffer if use_buffer_hashtable else None, - }, - outputs={ - "Reindex_Src": reindex_src, - "Reindex_Dst": reindex_dst, - "Out_Nodes": out_nodes - }, - attrs={"flag_buffer_hashtable": use_buffer_hashtable}) + helper.append_op( + type="graph_reindex", + inputs={ + "X": x, + "Neighbors": neighbors, + "Count": count, + "HashTable_Value": value_buffer if use_buffer_hashtable else None, + "HashTable_Index": index_buffer if use_buffer_hashtable else None, + }, + outputs={ + "Reindex_Src": reindex_src, + "Reindex_Dst": reindex_dst, + "Out_Nodes": out_nodes, + }, + attrs={"flag_buffer_hashtable": use_buffer_hashtable}, + ) return reindex_src, reindex_dst, out_nodes -def reindex_heter_graph(x, - neighbors, - count, - value_buffer=None, - index_buffer=None, - name=None): +def reindex_heter_graph( + x, neighbors, count, value_buffer=None, index_buffer=None, name=None +): """ Reindex HeterGraph API. @@ -220,15 +221,22 @@ def reindex_heter_graph(x, # out_nodes: [0, 1, 2, 8, 9, 4, 7, 6, 3, 5] """ - use_buffer_hashtable = True if value_buffer is not None \ - and index_buffer is not None else False + use_buffer_hashtable = ( + True if value_buffer is not None and index_buffer is not None else False + ) if _non_static_mode(): neighbors = paddle.concat(neighbors, axis=0) count = paddle.concat(count, axis=0) - reindex_src, reindex_dst, out_nodes = \ - _legacy_C_ops.graph_reindex(x, neighbors, count, value_buffer, index_buffer, - "flag_buffer_hashtable", use_buffer_hashtable) + reindex_src, reindex_dst, out_nodes = _legacy_C_ops.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + "flag_buffer_hashtable", + use_buffer_hashtable, + ) return reindex_src, reindex_dst, out_nodes if isinstance(neighbors, Variable): @@ -240,15 +248,18 @@ def reindex_heter_graph(x, count = paddle.concat(count, axis=0) check_variable_and_dtype(x, "X", ("int32", "int64"), "heter_graph_reindex") - check_variable_and_dtype(neighbors, "Neighbors", ("int32", "int64"), - "graph_reindex") + check_variable_and_dtype( + neighbors, "Neighbors", ("int32", "int64"), "graph_reindex" + ) check_variable_and_dtype(count, "Count", ("int32"), "graph_reindex") if use_buffer_hashtable: - check_variable_and_dtype(value_buffer, "HashTable_Value", ("int32"), - "graph_reindex") - check_variable_and_dtype(index_buffer, "HashTable_Index", ("int32"), - "graph_reindex") + check_variable_and_dtype( + value_buffer, "HashTable_Value", ("int32"), "graph_reindex" + ) + check_variable_and_dtype( + index_buffer, "HashTable_Index", ("int32"), "graph_reindex" + ) helper = LayerHelper("reindex_heter_graph", **locals()) reindex_src = helper.create_variable_for_type_inference(dtype=x.dtype) @@ -256,23 +267,20 @@ def reindex_heter_graph(x, out_nodes = helper.create_variable_for_type_inference(dtype=x.dtype) neighbors = paddle.concat(neighbors, axis=0) count = paddle.concat(count, axis=0) - helper.append_op(type="graph_reindex", - inputs={ - "X": - x, - "Neighbors": - neighbors, - "Count": - count, - "HashTable_Value": - value_buffer if use_buffer_hashtable else None, - "HashTable_Index": - index_buffer if use_buffer_hashtable else None, - }, - outputs={ - "Reindex_Src": reindex_src, - "Reindex_Dst": reindex_dst, - "Out_Nodes": out_nodes - }, - attrs={"flag_buffer_hashtable": use_buffer_hashtable}) + helper.append_op( + type="graph_reindex", + inputs={ + "X": x, + "Neighbors": neighbors, + "Count": count, + "HashTable_Value": value_buffer if use_buffer_hashtable else None, + "HashTable_Index": index_buffer if use_buffer_hashtable else None, + }, + outputs={ + "Reindex_Src": reindex_src, + "Reindex_Dst": reindex_dst, + "Out_Nodes": out_nodes, + }, + attrs={"flag_buffer_hashtable": use_buffer_hashtable}, + ) return reindex_src, reindex_dst, out_nodes diff --git a/python/paddle/geometric/sampling/neighbors.py b/python/paddle/geometric/sampling/neighbors.py index 63ec44b4f39..1353bd5814f 100644 --- a/python/paddle/geometric/sampling/neighbors.py +++ b/python/paddle/geometric/sampling/neighbors.py @@ -20,14 +20,16 @@ from paddle import _legacy_C_ops __all__ = [] -def sample_neighbors(row, - colptr, - input_nodes, - sample_size=-1, - eids=None, - return_eids=False, - perm_buffer=None, - name=None): +def sample_neighbors( + row, + colptr, + input_nodes, + sample_size=-1, + eids=None, + return_eids=False, + perm_buffer=None, + name=None, +): """ Graph Sample Neighbors API. @@ -90,54 +92,78 @@ def sample_neighbors(row, if return_eids: if eids is None: raise ValueError( - "`eids` should not be None if `return_eids` is True.") + "`eids` should not be None if `return_eids` is True." + ) use_perm_buffer = True if perm_buffer is not None else False if _non_static_mode(): - out_neighbors, out_count, out_eids = _legacy_C_ops.graph_sample_neighbors( - row, colptr, input_nodes, eids, perm_buffer, "sample_size", - sample_size, "return_eids", return_eids, "flag_perm_buffer", - use_perm_buffer) + ( + out_neighbors, + out_count, + out_eids, + ) = _legacy_C_ops.graph_sample_neighbors( + row, + colptr, + input_nodes, + eids, + perm_buffer, + "sample_size", + sample_size, + "return_eids", + return_eids, + "flag_perm_buffer", + use_perm_buffer, + ) if return_eids: return out_neighbors, out_count, out_eids return out_neighbors, out_count - check_variable_and_dtype(row, "Row", ("int32", "int64"), - "graph_sample_neighbors") - check_variable_and_dtype(colptr, "Col_Ptr", ("int32", "int64"), - "graph_sample_neighbors") - check_variable_and_dtype(input_nodes, "X", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + row, "Row", ("int32", "int64"), "graph_sample_neighbors" + ) + check_variable_and_dtype( + colptr, "Col_Ptr", ("int32", "int64"), "graph_sample_neighbors" + ) + check_variable_and_dtype( + input_nodes, "X", ("int32", "int64"), "graph_sample_neighbors" + ) if return_eids: - check_variable_and_dtype(eids, "Eids", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + eids, "Eids", ("int32", "int64"), "graph_sample_neighbors" + ) if use_perm_buffer: - check_variable_and_dtype(perm_buffer, "Perm_Buffer", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + perm_buffer, + "Perm_Buffer", + ("int32", "int64"), + "graph_sample_neighbors", + ) helper = LayerHelper("sample_neighbors", **locals()) out_neighbors = helper.create_variable_for_type_inference(dtype=row.dtype) out_count = helper.create_variable_for_type_inference(dtype=row.dtype) out_eids = helper.create_variable_for_type_inference(dtype=row.dtype) - helper.append_op(type="graph_sample_neighbors", - inputs={ - "Row": row, - "Col_Ptr": colptr, - "X": input_nodes, - "Eids": eids if return_eids else None, - "Perm_Buffer": perm_buffer if use_perm_buffer else None - }, - outputs={ - "Out": out_neighbors, - "Out_Count": out_count, - "Out_Eids": out_eids - }, - attrs={ - "sample_size": sample_size, - "return_eids": return_eids, - "flag_perm_buffer": use_perm_buffer - }) + helper.append_op( + type="graph_sample_neighbors", + inputs={ + "Row": row, + "Col_Ptr": colptr, + "X": input_nodes, + "Eids": eids if return_eids else None, + "Perm_Buffer": perm_buffer if use_perm_buffer else None, + }, + outputs={ + "Out": out_neighbors, + "Out_Count": out_count, + "Out_Eids": out_eids, + }, + attrs={ + "sample_size": sample_size, + "return_eids": return_eids, + "flag_perm_buffer": use_perm_buffer, + }, + ) if return_eids: return out_neighbors, out_count, out_eids return out_neighbors, out_count diff --git a/python/paddle/hapi/callbacks.py b/python/paddle/hapi/callbacks.py index 886a8180159..af53b181436 100644 --- a/python/paddle/hapi/callbacks.py +++ b/python/paddle/hapi/callbacks.py @@ -28,17 +28,19 @@ from .progressbar import ProgressBar __all__ = [] -def config_callbacks(callbacks=None, - model=None, - batch_size=None, - epochs=None, - steps=None, - log_freq=2, - verbose=2, - save_freq=1, - save_dir=None, - metrics=None, - mode='train'): +def config_callbacks( + callbacks=None, + model=None, + batch_size=None, + epochs=None, + steps=None, + log_freq=2, + verbose=2, + save_freq=1, + save_dir=None, + metrics=None, + mode='train', +): cbks = callbacks or [] cbks = cbks if isinstance(cbks, (list, tuple)) else [cbks] if not any(isinstance(k, ProgBarLogger) for k in cbks) and verbose: @@ -68,7 +70,6 @@ def config_callbacks(callbacks=None, class CallbackList(object): - def __init__(self, callbacks=None): # copy self.callbacks = [c for c in callbacks] @@ -95,8 +96,11 @@ class CallbackList(object): func(*args) def _check_mode(self, mode): - assert mode in ['train', 'eval', 'predict'], \ - 'mode should be train, eval or predict' + assert mode in [ + 'train', + 'eval', + 'predict', + ], 'mode should be train, eval or predict' def on_begin(self, mode, logs=None): self._check_mode(mode) @@ -167,8 +171,7 @@ class Callback(object): self.params = params def set_model(self, model): - """model is instance of paddle.Model. - """ + """model is instance of paddle.Model.""" self.model = model def on_train_begin(self, logs=None): @@ -391,31 +394,39 @@ class ProgBarLogger(Callback): cnt = timer['count'] if timer['count'] > 0 else 1.0 samples = timer['samples'] if timer['samples'] > 0 else 1.0 values.append( - ('avg_reader_cost', "%.5f sec" % (timer['data_time'] / cnt))) + ('avg_reader_cost', "%.5f sec" % (timer['data_time'] / cnt)) + ) values.append( - ('avg_batch_cost', "%.5f sec" % (timer['batch_time'] / cnt))) + ('avg_batch_cost', "%.5f sec" % (timer['batch_time'] / cnt)) + ) values.append( - ('ips', "%.5f samples/sec" % - (samples / (timer['data_time'] + timer['batch_time'])))) + ( + 'ips', + "%.5f samples/sec" + % (samples / (timer['data_time'] + timer['batch_time'])), + ) + ) timer['count'] = 0 timer['samples'] = 0 - timer['data_time'] = 0. - timer['batch_time'] = 0. + timer['data_time'] = 0.0 + timer['batch_time'] = 0.0 progbar.update(steps, values) def on_train_batch_begin(self, step, logs=None): self._train_timer['batch_data_end_time'] = time.time() self._train_timer['data_time'] += ( - self._train_timer['batch_data_end_time'] - - self._train_timer['batch_start_time']) + self._train_timer['batch_data_end_time'] + - self._train_timer['batch_start_time'] + ) def on_train_batch_end(self, step, logs=None): logs = logs or {} self.train_step += 1 self._train_timer['batch_time'] += ( - time.time() - self._train_timer['batch_data_end_time']) + time.time() - self._train_timer['batch_data_end_time'] + ) self._train_timer['count'] += 1 samples = logs.get('batch_size', 1) self._train_timer['samples'] += samples @@ -442,8 +453,9 @@ class ProgBarLogger(Callback): 'samples': 0, } - self.eval_progbar = ProgressBar(num=self.eval_steps, - verbose=self.verbose) + self.eval_progbar = ProgressBar( + num=self.eval_steps, verbose=self.verbose + ) if self._is_print(): print('Eval begin...') @@ -452,8 +464,9 @@ class ProgBarLogger(Callback): def on_eval_batch_begin(self, step, logs=None): self._eval_timer['batch_data_end_time'] = time.time() self._eval_timer['data_time'] += ( - self._eval_timer['batch_data_end_time'] - - self._eval_timer['batch_start_time']) + self._eval_timer['batch_data_end_time'] + - self._eval_timer['batch_start_time'] + ) def on_eval_batch_end(self, step, logs=None): logs = logs or {} @@ -462,7 +475,8 @@ class ProgBarLogger(Callback): self.evaled_samples += samples self._eval_timer['batch_time'] += ( - time.time() - self._eval_timer['batch_data_end_time']) + time.time() - self._eval_timer['batch_data_end_time'] + ) self._eval_timer['count'] += 1 samples = logs.get('batch_size', 1) self._eval_timer['samples'] += samples @@ -486,8 +500,9 @@ class ProgBarLogger(Callback): 'samples': 0, } - self.test_progbar = ProgressBar(num=self.test_steps, - verbose=self.verbose) + self.test_progbar = ProgressBar( + num=self.test_steps, verbose=self.verbose + ) if self._is_print(): print('Predict begin...') @@ -496,8 +511,9 @@ class ProgBarLogger(Callback): def on_predict_batch_begin(self, step, logs=None): self._test_timer['batch_data_end_time'] = time.time() self._test_timer['data_time'] += ( - self._test_timer['batch_data_end_time'] - - self._test_timer['batch_start_time']) + self._test_timer['batch_data_end_time'] + - self._test_timer['batch_start_time'] + ) def on_predict_batch_end(self, step, logs=None): logs = logs or {} @@ -506,7 +522,8 @@ class ProgBarLogger(Callback): self.tested_samples += samples self._test_timer['batch_time'] += ( - time.time() - self._test_timer['batch_data_end_time']) + time.time() - self._test_timer['batch_data_end_time'] + ) self._test_timer['count'] += 1 samples = logs.get('batch_size', 1) self._test_timer['samples'] += samples @@ -665,25 +682,34 @@ class LRScheduler(Callback): def __init__(self, by_step=True, by_epoch=False): if by_step and by_epoch: raise ValueError( - "by_step option is mutually exclusive with by_epoch") + "by_step option is mutually exclusive with by_epoch" + ) self.by_step = by_step self.by_epoch = by_epoch def on_epoch_end(self, epoch, logs=None): if self.by_epoch: - if self.model._optimizer and \ - hasattr(self.model._optimizer, '_learning_rate') and \ - isinstance(self.model._optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): + if ( + self.model._optimizer + and hasattr(self.model._optimizer, '_learning_rate') + and isinstance( + self.model._optimizer._learning_rate, + paddle.optimizer.lr.LRScheduler, + ) + ): self.model._optimizer._learning_rate.step() def on_train_batch_end(self, step, logs=None): if self.by_step: - if self.model._optimizer and \ - hasattr(self.model._optimizer, '_learning_rate') and \ - isinstance(self.model._optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): + if ( + self.model._optimizer + and hasattr(self.model._optimizer, '_learning_rate') + and isinstance( + self.model._optimizer._learning_rate, + paddle.optimizer.lr.LRScheduler, + ) + ): self.model._optimizer._learning_rate.step() @@ -761,14 +787,16 @@ class EarlyStopping(Callback): callbacks=[callbacks]) """ - def __init__(self, - monitor='loss', - mode='auto', - patience=0, - verbose=1, - min_delta=0, - baseline=None, - save_best_model=True): + def __init__( + self, + monitor='loss', + mode='auto', + patience=0, + verbose=1, + min_delta=0, + baseline=None, + save_best_model=True, + ): super(EarlyStopping, self).__init__() self.monitor = monitor self.patience = patience @@ -782,8 +810,10 @@ class EarlyStopping(Callback): # The value of `save_dir` is set in function `config_callbacks` self.save_dir = None if mode not in ['auto', 'min', 'max']: - warnings.warn('EarlyStopping mode %s is unknown, ' - 'fallback to auto mode.' % mode) + warnings.warn( + 'EarlyStopping mode %s is unknown, ' + 'fallback to auto mode.' % mode + ) mode = 'auto' if mode == 'min': self.monitor_op = np.less @@ -812,7 +842,8 @@ class EarlyStopping(Callback): def on_eval_end(self, logs=None): if logs is None or self.monitor not in logs: warnings.warn( - 'Monitor of EarlyStopping should be loss or metric name.') + 'Monitor of EarlyStopping should be loss or metric name.' + ) return current = logs[self.monitor] if isinstance(current, (list, tuple)): @@ -835,9 +866,14 @@ class EarlyStopping(Callback): if self.verbose > 0: print('Epoch %d: Early stopping.' % (self.stopped_epoch + 1)) if self.save_best_model and self.save_dir is not None: - print('Best checkpoint has been saved at %s' % - (os.path.abspath( - os.path.join(self.save_dir, 'best_model')))) + print( + 'Best checkpoint has been saved at %s' + % ( + os.path.abspath( + os.path.join(self.save_dir, 'best_model') + ) + ) + ) self.stopped_epoch += 1 @@ -926,9 +962,9 @@ class VisualDL(Callback): else: continue - self.writer.add_scalar(tag=temp_tag, - step=total_step, - value=temp_value) + self.writer.add_scalar( + tag=temp_tag, step=total_step, value=temp_value + ) def on_train_batch_end(self, step, logs=None): logs = logs or {} @@ -1022,21 +1058,24 @@ class ReduceLROnPlateau(Callback): """ - def __init__(self, - monitor='loss', - factor=0.1, - patience=10, - verbose=1, - mode='auto', - min_delta=1e-4, - cooldown=0, - min_lr=0): + def __init__( + self, + monitor='loss', + factor=0.1, + patience=10, + verbose=1, + mode='auto', + min_delta=1e-4, + cooldown=0, + min_lr=0, + ): super(ReduceLROnPlateau, self).__init__() self.monitor = monitor if factor >= 1.0: - raise ValueError('ReduceLROnPlateau ' - 'does not support a factor >= 1.0.') + raise ValueError( + 'ReduceLROnPlateau ' 'does not support a factor >= 1.0.' + ) self.factor = factor self.min_lr = min_lr @@ -1053,14 +1092,16 @@ class ReduceLROnPlateau(Callback): self._reset() def _reset(self): - """Resets wait counter and cooldown counter. - """ + """Resets wait counter and cooldown counter.""" if self.mode not in ['auto', 'min', 'max']: - warnings.warn('Learning rate reduction mode %s is unknown, ' - 'fallback to auto mode.' % self.mode) + warnings.warn( + 'Learning rate reduction mode %s is unknown, ' + 'fallback to auto mode.' % self.mode + ) self.mode = 'auto' - if (self.mode == 'min' - or (self.mode == 'auto' and 'acc' not in self.monitor)): + if self.mode == 'min' or ( + self.mode == 'auto' and 'acc' not in self.monitor + ): self.monitor_op = lambda a, b: np.less(a, b - self.min_delta) self.best = np.Inf else: @@ -1075,7 +1116,8 @@ class ReduceLROnPlateau(Callback): def on_eval_end(self, logs=None): if logs is None or self.monitor not in logs: warnings.warn( - 'Monitor of ReduceLROnPlateau should be loss or metric name.') + 'Monitor of ReduceLROnPlateau should be loss or metric name.' + ) return else: try: @@ -1083,12 +1125,16 @@ class ReduceLROnPlateau(Callback): if not isinstance(lr, float): warnings.warn( 'Expected learning_rate be float, bug got {}.'.format( - type(lr))) + type(lr) + ) + ) return except Exception as e: warnings.warn( - 'There are something wrong when get learning_rate from optimizer: {}.' - .format(e)) + 'There are something wrong when get learning_rate from optimizer: {}.'.format( + e + ) + ) return current = logs[self.monitor] @@ -1115,8 +1161,10 @@ class ReduceLROnPlateau(Callback): new_lr = max(new_lr, self.min_lr) self.model._optimizer._learning_rate = new_lr if self.verbose > 0 and ParallelEnv().local_rank == 0: - print('\nEpoch %d: ReduceLROnPlateau reducing learning ' - 'rate to %s.' % (self.epoch + 1, new_lr)) + print( + '\nEpoch %d: ReduceLROnPlateau reducing learning ' + 'rate to %s.' % (self.epoch + 1, new_lr) + ) self.cooldown_counter = self.cooldown self.wait = 0 self.epoch += 1 diff --git a/python/paddle/hapi/dynamic_flops.py b/python/paddle/hapi/dynamic_flops.py index 8e2bed5bc74..50284e93e3b 100644 --- a/python/paddle/hapi/dynamic_flops.py +++ b/python/paddle/hapi/dynamic_flops.py @@ -17,7 +17,9 @@ import warnings import paddle.nn as nn import numpy as np from .static_flops import static_flops, Table -from paddle.fluid.dygraph.dygraph_to_static.program_translator import unwrap_decorators +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + unwrap_decorators, +) __all__ = [] @@ -106,10 +108,9 @@ def flops(net, input_size, custom_ops=None, print_detail=False): _, net.forward = unwrap_decorators(net.forward) inputs = paddle.randn(input_size) - return dynamic_flops(net, - inputs=inputs, - custom_ops=custom_ops, - print_detail=print_detail) + return dynamic_flops( + net, inputs=inputs, custom_ops=custom_ops, print_detail=print_detail + ) elif isinstance(net, paddle.static.Program): return static_flops(net, print_detail=print_detail) else: @@ -123,8 +124,9 @@ def count_convNd(m, x, y): x = x[0] kernel_ops = np.product(m.weight.shape[2:]) bias_ops = 1 if m.bias is not None else 0 - total_ops = int( - y.numel()) * (x.shape[1] / m._groups * kernel_ops + bias_ops) + total_ops = int(y.numel()) * ( + x.shape[1] / m._groups * kernel_ops + bias_ops + ) m.total_ops += abs(int(total_ops)) @@ -205,7 +207,7 @@ register_hooks = { nn.AvgPool3D: count_avgpool, nn.AdaptiveAvgPool1D: count_adap_avgpool, nn.AdaptiveAvgPool2D: count_adap_avgpool, - nn.AdaptiveAvgPool3D: count_adap_avgpool + nn.AdaptiveAvgPool3D: count_adap_avgpool, } @@ -227,7 +229,8 @@ def dynamic_flops(model, inputs, custom_ops=None, print_detail=False): flops_fn = custom_ops[m_type] if m_type not in types_collection: print( - "Customize Function has been applied to {}".format(m_type)) + "Customize Function has been applied to {}".format(m_type) + ) elif m_type in register_hooks: flops_fn = register_hooks[m_type] if m_type not in types_collection: @@ -235,8 +238,10 @@ def dynamic_flops(model, inputs, custom_ops=None, print_detail=False): else: if m_type not in types_collection: print( - "Cannot find suitable count function for {}. Treat it as zero FLOPs." - .format(m_type)) + "Cannot find suitable count function for {}. Treat it as zero FLOPs.".format( + m_type + ) + ) if flops_fn is not None: flops_handler = m.register_forward_post_hook(flops_fn) @@ -260,8 +265,12 @@ def dynamic_flops(model, inputs, custom_ops=None, print_detail=False): for m in model.sublayers(): if len(list(m.children())) > 0: continue - if {'total_ops', 'total_params', 'input_shape', - 'output_shape'}.issubset(set(list(m._buffers.keys()))): + if { + 'total_ops', + 'total_params', + 'input_shape', + 'output_shape', + }.issubset(set(list(m._buffers.keys()))): total_ops += m.total_ops total_params += m.total_params @@ -271,26 +280,36 @@ def dynamic_flops(model, inputs, custom_ops=None, print_detail=False): handler.remove() table = Table( - ["Layer Name", "Input Shape", "Output Shape", "Params", "Flops"]) + ["Layer Name", "Input Shape", "Output Shape", "Params", "Flops"] + ) for n, m in model.named_sublayers(): if len(list(m.children())) > 0: continue - if {'total_ops', 'total_params', 'input_shape', - 'output_shape'}.issubset(set(list(m._buffers.keys()))): - table.add_row([ - m.full_name(), - list(m.input_shape.numpy()), - list(m.output_shape.numpy()), - int(m.total_params), - int(m.total_ops) - ]) + if { + 'total_ops', + 'total_params', + 'input_shape', + 'output_shape', + }.issubset(set(list(m._buffers.keys()))): + table.add_row( + [ + m.full_name(), + list(m.input_shape.numpy()), + list(m.output_shape.numpy()), + int(m.total_params), + int(m.total_ops), + ] + ) m._buffers.pop("total_ops") m._buffers.pop("total_params") m._buffers.pop('input_shape') m._buffers.pop('output_shape') if print_detail: table.print_table() - print('Total Flops: {} Total Params: {}'.format(int(total_ops), - int(total_params))) + print( + 'Total Flops: {} Total Params: {}'.format( + int(total_ops), int(total_params) + ) + ) return int(total_ops) diff --git a/python/paddle/hapi/hub.py b/python/paddle/hapi/hub.py index ccf03aa2db0..3d5cb3027c4 100644 --- a/python/paddle/hapi/hub.py +++ b/python/paddle/hapi/hub.py @@ -53,10 +53,12 @@ def _import_module(name, repo_dir): def _git_archive_link(repo_owner, repo_name, branch, source): if source == 'github': return 'https://github.com/{}/{}/archive/{}.zip'.format( - repo_owner, repo_name, branch) + repo_owner, repo_name, branch + ) elif source == 'gitee': return 'https://gitee.com/{}/{}/repository/archive/{}.zip'.format( - repo_owner, repo_name, branch) + repo_owner, repo_name, branch + ) def _parse_repo_info(repo, source): @@ -94,8 +96,9 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'): # We don't know the repo name before downloading the zip file # and inspect name from it. # To check if cached repo exists, we need to normalize folder names. - repo_dir = os.path.join(hub_dir, - '_'.join([repo_owner, repo_name, normalized_br])) + repo_dir = os.path.join( + hub_dir, '_'.join([repo_owner, repo_name, normalized_br]) + ) use_cache = (not force_reload) and os.path.exists(repo_dir) @@ -113,7 +116,8 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'): hub_dir, check_exist=not force_reload, decompress=False, - method=('wget' if source == 'gitee' else 'get')) + method=('wget' if source == 'gitee' else 'get'), + ) shutil.move(fpath, cached_file) with zipfile.ZipFile(cached_file) as cached_zipfile: @@ -132,11 +136,11 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'): def _load_entry_from_hubconf(m, name): - '''load entry from hubconf - ''' + '''load entry from hubconf''' if not isinstance(name, str): raise ValueError( - 'Invalid input: model should be a str of function name') + 'Invalid input: model should be a str of function name' + ) func = getattr(m, name, None) @@ -162,8 +166,9 @@ def _check_dependencies(m): pkg for pkg in dependencies if not _check_module_exists(pkg) ] if len(missing_deps): - raise RuntimeError('Missing dependencies: {}'.format( - ', '.join(missing_deps))) + raise RuntimeError( + 'Missing dependencies: {}'.format(', '.join(missing_deps)) + ) def list(repo_dir, source='github', force_reload=False): @@ -193,19 +198,21 @@ def list(repo_dir, source='github', force_reload=False): """ if source not in ('github', 'gitee', 'local'): raise ValueError( - 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".' - .format(source)) + 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".'.format( + source + ) + ) if source in ('github', 'gitee'): - repo_dir = _get_cache_or_reload(repo_dir, - force_reload, - True, - source=source) + repo_dir = _get_cache_or_reload( + repo_dir, force_reload, True, source=source + ) hub_module = _import_module(MODULE_HUBCONF.split('.')[0], repo_dir) entrypoints = [ - f for f in dir(hub_module) + f + for f in dir(hub_module) if callable(getattr(hub_module, f)) and not f.startswith('_') ] @@ -240,14 +247,15 @@ def help(repo_dir, model, source='github', force_reload=False): """ if source not in ('github', 'gitee', 'local'): raise ValueError( - 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".' - .format(source)) + 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".'.format( + source + ) + ) if source in ('github', 'gitee'): - repo_dir = _get_cache_or_reload(repo_dir, - force_reload, - True, - source=source) + repo_dir = _get_cache_or_reload( + repo_dir, force_reload, True, source=source + ) hub_module = _import_module(MODULE_HUBCONF.split('.')[0], repo_dir) @@ -283,14 +291,15 @@ def load(repo_dir, model, source='github', force_reload=False, **kwargs): """ if source not in ('github', 'gitee', 'local'): raise ValueError( - 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".' - .format(source)) + 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".'.format( + source + ) + ) if source in ('github', 'gitee'): - repo_dir = _get_cache_or_reload(repo_dir, - force_reload, - True, - source=source) + repo_dir = _get_cache_or_reload( + repo_dir, force_reload, True, source=source + ) hub_module = _import_module(MODULE_HUBCONF.split('.')[0], repo_dir) diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 788043477a2..a790375a742 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -64,8 +64,9 @@ def to_list(value): def to_numpy(var): - assert isinstance(var, (Variable, fluid.core.VarBase, - fluid.core.eager.Tensor)), "not a variable" + assert isinstance( + var, (Variable, fluid.core.VarBase, fluid.core.eager.Tensor) + ), "not a variable" if isinstance(var, (fluid.core.VarBase, fluid.core.eager.Tensor)): return var.numpy() t = global_scope().find_var(var.name).get_tensor() @@ -100,10 +101,9 @@ def extract_args(func): def _all_gather(x, nranks, ring_id=0, use_calc_stream=True): - return collective._c_allgather(x, - nranks, - ring_id=ring_id, - use_calc_stream=use_calc_stream) + return collective._c_allgather( + x, nranks, ring_id=ring_id, use_calc_stream=use_calc_stream + ) def wait_server_ready(endpoints): @@ -114,7 +114,8 @@ def wait_server_ready(endpoints): for ep in endpoints: ip_port = ep.split(":") with contextlib.closing( - socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -126,8 +127,9 @@ def wait_server_ready(endpoints): break -def init_communicator(program, rank, nranks, wait_port, current_endpoint, - endpoints): +def init_communicator( + program, rank, nranks, wait_port, current_endpoint, endpoints +): if nranks < 2: return other_endpoints = endpoints[:] @@ -139,53 +141,66 @@ def init_communicator(program, rank, nranks, wait_port, current_endpoint, nccl_id_var = block.create_var( name=fluid.unique_name.generate('nccl_id'), persistable=True, - type=fluid.core.VarDesc.VarType.RAW) - - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': 0, - }) + type=fluid.core.VarDesc.VarType.RAW, + ) + + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': 0, + }, + ) elif core.is_compiled_with_npu(): hccl_id_var = block.create_var( name=fluid.unique_name.generate('hccl_id'), persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': hccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': hccl_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': 0, - 'device_id': int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks - }) + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': hccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': hccl_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': 0, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + }, + ) def prepare_distributed_context(place=None): if place is None: - place = fluid.CUDAPlace(ParallelEnv().dev_id) if ParallelEnv().nranks > 1 \ + place = ( + fluid.CUDAPlace(ParallelEnv().dev_id) + if ParallelEnv().nranks > 1 else fluid.CUDAPlace(0) + ) place = _get_paddle_place(place) strategy = fluid.dygraph.parallel.ParallelStrategy() @@ -203,9 +218,14 @@ def prepare_distributed_context(place=None): def _init_context(): communicator_prog = fluid.Program() - init_communicator(communicator_prog, strategy.local_rank, - strategy.nranks, True, strategy.current_endpoint, - strategy.trainer_endpoints) + init_communicator( + communicator_prog, + strategy.local_rank, + strategy.nranks, + True, + strategy.current_endpoint, + strategy.trainer_endpoints, + ) exe = fluid.Executor(place) exe.run(communicator_prog) @@ -215,7 +235,7 @@ def prepare_distributed_context(place=None): fluid.enable_dygraph(place) else: - assert ("Only support CUDAPlace for now.") + assert "Only support CUDAPlace for now." _parallel_context_initialized = True return strategy @@ -264,7 +284,7 @@ class StaticGraphAdapter(object): 'eval_total': 0, 'test_total': 0, 'eval_batch': 0, - 'test_batch': 0 + 'test_batch': 0, } self._nranks = ParallelEnv().nranks @@ -284,10 +304,13 @@ class StaticGraphAdapter(object): self.model.mode = value def train_batch(self, inputs, labels=None, update=True): - assert self.model._optimizer, \ - "model not ready, please call `model.prepare()` first" + assert ( + self.model._optimizer + ), "model not ready, please call `model.prepare()` first" self.mode = 'train' - assert update is True, "Does not support `update == False` in static mode by now." + assert ( + update is True + ), "Does not support `update == False` in static mode by now." return self._run(inputs, labels) def eval_batch(self, inputs, labels=None): @@ -302,7 +325,6 @@ class StaticGraphAdapter(object): return self.model.network.parameters(*args, **kwargs) def save(self, path): - def _save(state, path): if not state: return @@ -326,8 +348,7 @@ class StaticGraphAdapter(object): # XXX `optimizer.state_dict()` only work in dygraph mode optim_path = path + ".pdopt" optim = { - p.name: p - for p in filter(is_belong_to_optimizer, prog.list_vars()) + p.name: p for p in filter(is_belong_to_optimizer, prog.list_vars()) } if not optim: return @@ -343,8 +364,10 @@ class StaticGraphAdapter(object): # restore parameter states fluid.core._create_loaded_parameter( - [param for param, state in param_state_pairs], global_scope(), - executor) + [param for param, state in param_state_pairs], + global_scope(), + executor, + ) for param, state in param_state_pairs: self._set_var(param, state) @@ -372,9 +395,10 @@ class StaticGraphAdapter(object): # static-graph, since the time of global_step to increase is # different. state_val = ( - np.array(converted_state.pop("global_step")) - 1 - ) if "global_step" in converted_state else converted_state.pop( - "@LR_DECAY_COUNTER@", None) + (np.array(converted_state.pop("global_step")) - 1) + if "global_step" in converted_state + else converted_state.pop("@LR_DECAY_COUNTER@", None) + ) if state_val is not None: converted_state[var.name] = state_val elif var.name.startswith("learning_rate_"): @@ -391,36 +415,61 @@ class StaticGraphAdapter(object): opt_cls_name = self.model._optimizer.__class__.__name__ opt_unq_name = None for name in self.model._optimizer._accumulators.keys(): - accum_name = name if opt_name is None else name[ - len(opt_name) + 1:] - for param_name, state_var in self.model._optimizer._accumulators[ - name].items(): + accum_name = ( + name + if opt_name is None + else name[len(opt_name) + 1 :] + ) + for ( + param_name, + state_var, + ) in self.model._optimizer._accumulators[name].items(): if opt_unq_name is None: # can not infer out the exact unique(opt_name), # thus try to extract rather than generate - for state_key in sorted(state.keys(), - key=lambda x: len(x), - reverse=True): - prefix = param_name + "_" + ( - opt_cls_name - if opt_name is None else opt_name) + "_" + for state_key in sorted( + state.keys(), + key=lambda x: len(x), + reverse=True, + ): + prefix = ( + param_name + + "_" + + ( + opt_cls_name + if opt_name is None + else opt_name + ) + + "_" + ) if state_key.startswith(prefix): - prefix_offset = state_key[len( - prefix):].find("_") + len(prefix) + prefix_offset = state_key[ + len(prefix) : + ].find("_") + len(prefix) opt_unq_name = state_key[ - len(param_name + "_"):prefix_offset] + len( + param_name + "_" + ) : prefix_offset + ] # TODO: assert # assert opt_unq_name is None # gen(param.name + "_" + gen(opt_name) + "_" + accum_name) # always end with "_0" since the unique optimizer._name - dy_state_name = (param_name + "_" + opt_unq_name + - "_" + accum_name + "_0") + dy_state_name = ( + param_name + + "_" + + opt_unq_name + + "_" + + accum_name + + "_0" + ) converted_state[ - state_var.name] = converted_state.pop( - dy_state_name) + state_var.name + ] = converted_state.pop(dy_state_name) - assert var.name in converted_state, \ - "variable [{}] is not in optimizer state file".format(var.name) + assert ( + var.name in converted_state + ), "variable [{}] is not in optimizer state file".format(var.name) self._set_var(var, converted_state[var.name]) def _set_var(self, var, ndarray): @@ -439,15 +488,17 @@ class StaticGraphAdapter(object): def _run(self, inputs, labels=None): compiled_prog = self._compiled_progs.get(self.mode, None) - assert compiled_prog, \ - "Model is not ready, please call `model.prepare()` first" + assert ( + compiled_prog + ), "Model is not ready, please call `model.prepare()` first" inputs = to_list(inputs) if labels is not None: labels = to_list(labels) - assert len(inputs) == len(self._input_vars[self.mode]), \ - "number of inputs" \ + assert len(inputs) == len(self._input_vars[self.mode]), ( + "number of inputs" + " does not match number of arguments of `forward` method" + ) feed = {} input_names = [v.name for v in self._input_vars[self.mode]] @@ -457,8 +508,10 @@ class StaticGraphAdapter(object): # train and test may take different arguments if inputs[idx] is not None: feed[n] = inputs[idx] - if self._amp_level == 'O2' and input_dtypes[ - idx] == core.VarDesc.VarType.FP16: + if ( + self._amp_level == 'O2' + and input_dtypes[idx] == core.VarDesc.VarType.FP16 + ): if isinstance(feed[n], core.LoDTensor): feed[n] = feed[n]._as_type(core.VarDesc.VarType.FP16) elif isinstance(feed[n], np.array): @@ -486,10 +539,12 @@ class StaticGraphAdapter(object): else: pruned_fetch_list.append(fetch_var) - rets = self._executor.run(compiled_prog, - feed=feed, - fetch_list=pruned_fetch_list, - return_numpy=False) + rets = self._executor.run( + compiled_prog, + feed=feed, + fetch_list=pruned_fetch_list, + return_numpy=False, + ) # restore pruned fetch_list Variable from feeds for i, name in enumerate(pruned_fetch_idx_name_map): @@ -505,20 +560,24 @@ class StaticGraphAdapter(object): metrics = [] for metric, state in zip(self.model._metrics, metric_states): # cut off padding size - if self.mode != 'train' and self.model._test_dataloader is not None \ - and isinstance(self.model._test_dataloader, DataLoader) \ - and self._nranks > 1: + if ( + self.mode != 'train' + and self.model._test_dataloader is not None + and isinstance(self.model._test_dataloader, DataLoader) + and self._nranks > 1 + ): total_size = len(self.model._test_dataloader.dataset) # TODO: fixme if have better way to get batch size samples = state[0].shape[0] current_count = self._merge_count.get(self.mode + '_total', 0) if current_count + samples >= total_size: state = [ - s[:int(total_size - current_count), ...] for s in state + s[: int(total_size - current_count), ...] for s in state ] self._merge_count[self.mode + '_total'] = 0 - self._merge_count[self.mode + '_batch'] = int(total_size - - current_count) + self._merge_count[self.mode + '_batch'] = int( + total_size - current_count + ) else: self._merge_count[self.mode + '_total'] += samples self._merge_count[self.mode + '_batch'] = samples @@ -550,8 +609,11 @@ class StaticGraphAdapter(object): if mode != 'train': for op in list(prog.global_block().ops): prog.global_block()._remove_op(0) - if mode == 'train' and self.model._optimizer \ - and self.model._optimizer._learning_rate_map: + if ( + mode == 'train' + and self.model._optimizer + and self.model._optimizer._learning_rate_map + ): # HACK workaround learning rate map issue lr_var = self.model._optimizer._learning_rate_map[self._orig_prog] new_lr_var = prog.global_block().vars[lr_var.name] @@ -589,20 +651,27 @@ class StaticGraphAdapter(object): dist_strategy.amp = True dist_strategy.amp_configs = self._amp_configs.copy() dist_strategy.amp_configs.update(self._amp_custom_lists) - dist_strategy.amp_configs[ - 'use_pure_fp16'] = self._amp_level == 'O2' + dist_strategy.amp_configs['use_pure_fp16'] = ( + self._amp_level == 'O2' + ) self.model._optimizer = fleet.distributed_optimizer( - self.model._optimizer, strategy=dist_strategy) + self.model._optimizer, strategy=dist_strategy + ) elif self._amp_level != "O0" and core.is_compiled_with_cuda: - amp_lists = paddle.static.amp.AutoMixedPrecisionLists( - **self._amp_custom_lists - ) if self._amp_custom_lists else None + amp_lists = ( + paddle.static.amp.AutoMixedPrecisionLists( + **self._amp_custom_lists + ) + if self._amp_custom_lists + else None + ) self.model._optimizer = paddle.static.amp.decorate( self.model._optimizer, amp_lists=amp_lists, use_pure_fp16=self._amp_level == "O2", use_fp16_guard=self._use_fp16_guard, - **self._amp_configs) + **self._amp_configs + ) self.model._optimizer.minimize(self._loss_endpoint) @@ -615,7 +684,7 @@ class StaticGraphAdapter(object): self._endpoints[mode] = { "output": outputs, "loss": to_list(losses), - "metric": metrics + "metric": metrics, } def _compile_and_initialize(self, prog, mode): @@ -623,8 +692,9 @@ class StaticGraphAdapter(object): if compiled_prog is not None: return compiled_prog - assert self.model._place is not None, \ - "device is not set, please call `model.prepare()` first" + assert ( + self.model._place is not None + ), "device is not set, please call `model.prepare()` first" place = self.model._place @@ -637,8 +707,11 @@ class StaticGraphAdapter(object): uninitialized = [] for var_py in self._startup_prog.list_vars(): var = fluid.global_scope().find_var(var_py.name) - if not var_py.name.startswith('nccl_id') and var and \ - var.get_tensor()._is_initialized(): + if ( + not var_py.name.startswith('nccl_id') + and var + and var.get_tensor()._is_initialized() + ): continue uninitialized.append(var_py) @@ -646,7 +719,10 @@ class StaticGraphAdapter(object): startup_prog = self._startup_prog._prune(uninitialized) self._executor.run(startup_prog) - if self._amp_level == "O2" and mode == 'train' and core.is_compiled_with_cuda( + if ( + self._amp_level == "O2" + and mode == 'train' + and core.is_compiled_with_cuda() ): self.model._optimizer.amp_init(place) @@ -659,7 +735,6 @@ class StaticGraphAdapter(object): class DynamicGraphAdapter(object): - def __init__(self, model): super(DynamicGraphAdapter, self).__init__() self.model = model @@ -669,7 +744,7 @@ class DynamicGraphAdapter(object): 'eval_total': 0, 'test_total': 0, 'eval_batch': 0, - 'test_batch': 0 + 'test_batch': 0, } self._input_info = None @@ -686,7 +761,8 @@ class DynamicGraphAdapter(object): stradegy.trainer_endpoints = ParallelEnv().trainer_endpoints stradegy.current_endpoint = ParallelEnv().current_endpoint self.ddp_model = fluid.dygraph.parallel.DataParallel( - self.model.network, stradegy) + self.model.network, stradegy + ) @property def mode(self): @@ -698,8 +774,9 @@ class DynamicGraphAdapter(object): # TODO multi device in dygraph mode not implemented at present time def train_batch(self, inputs, labels=None, update=True): - assert self.model._optimizer, \ - "model not ready, please call `model.prepare()` first" + assert ( + self.model._optimizer + ), "model not ready, please call `model.prepare()` first" self.model.network.train() self.mode = 'train' inputs = to_list(inputs) @@ -711,9 +788,11 @@ class DynamicGraphAdapter(object): if self._amp_level != "O0" and self.model._scaler is None: self.model._scaler = paddle.amp.GradScaler(**self._amp_configs) - with paddle.amp.auto_cast(enable=self._amp_level != 'O0', - **self._amp_custom_lists, - level=self._amp_level): + with paddle.amp.auto_cast( + enable=self._amp_level != 'O0', + **self._amp_custom_lists, + level=self._amp_level + ): if self._nranks > 1: outputs = self.ddp_model(*[to_variable(x) for x in inputs]) else: @@ -741,8 +820,11 @@ class DynamicGraphAdapter(object): m = metric.update(*[to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) - return ([to_numpy(l) for l in losses], metrics) \ - if len(metrics) > 0 else [to_numpy(l) for l in losses] + return ( + ([to_numpy(l) for l in losses], metrics) + if len(metrics) > 0 + else [to_numpy(l) for l in losses] + ) def eval_batch(self, inputs, labels=None): self.model.network.eval() @@ -772,21 +854,25 @@ class DynamicGraphAdapter(object): metrics = [] for metric in self.model._metrics: # cut off padding value. - if self.model._test_dataloader is not None and self._nranks > 1 \ - and isinstance(self.model._test_dataloader, DataLoader): + if ( + self.model._test_dataloader is not None + and self._nranks > 1 + and isinstance(self.model._test_dataloader, DataLoader) + ): total_size = len(self.model._test_dataloader.dataset) samples = outputs[0].shape[0] current_count = self._merge_count.get(self.mode + '_total', 0) if current_count + samples >= total_size: outputs = [ - o[:int(total_size - current_count)] for o in outputs + o[: int(total_size - current_count)] for o in outputs ] labels = [ - l[:int(total_size - current_count)] for l in labels + l[: int(total_size - current_count)] for l in labels ] self._merge_count[self.mode + '_total'] = 0 - self._merge_count[self.mode + '_batch'] = int(total_size - - current_count) + self._merge_count[self.mode + '_batch'] = int( + total_size - current_count + ) else: self._merge_count[self.mode + '_total'] += samples self._merge_count[self.mode + '_batch'] = samples @@ -853,38 +939,48 @@ class DynamicGraphAdapter(object): opt_unq_name = '' opt_cls_name = self.model._optimizer.__class__.__name__ - opt_name = opt_unq_name[:opt_unq_name.rfind("_")] # remove suffix idx + opt_name = opt_unq_name[: opt_unq_name.rfind("_")] # remove suffix idx param_names = [param.name for param in self.model.network.parameters()] - for var_name, state_var in sorted(optim_state.items(), - key=lambda x: len(x[0]), - reverse=True): + for var_name, state_var in sorted( + optim_state.items(), key=lambda x: len(x[0]), reverse=True + ): if var_name in ["@LR_DECAY_COUNTER@", "global_step"]: # NOTE: dygraph saved global_step is 1 larger than that in # static-graph, since the time of global_step to increase is # different. if var_name == "@LR_DECAY_COUNTER@": - converted_state["global_step"] = np.array( - converted_state.pop("@LR_DECAY_COUNTER@")) + 1 + converted_state["global_step"] = ( + np.array(converted_state.pop("@LR_DECAY_COUNTER@")) + 1 + ) else: # moment and other accumulators # extend state dict to include promising dygraph names for param_name in param_names: if var_name.startswith(param_name + "_" + opt_name): # when init optimizer with name - accum_name = var_name[len(param_name + "_" + opt_name + - "_"):] - elif var_name.startswith(param_name + - "_") and opt_name == opt_cls_name: + accum_name = var_name[ + len(param_name + "_" + opt_name + "_") : + ] + elif ( + var_name.startswith(param_name + "_") + and opt_name == opt_cls_name + ): # when init optimizer without name - accum_name = var_name[len(param_name + "_"):] + accum_name = var_name[len(param_name + "_") :] else: continue # remove suffix idx - accum_name = accum_name[:accum_name.rfind("_")] + accum_name = accum_name[: accum_name.rfind("_")] # state names always end with "_0" in dygraph because of the # unique optimizer._name - dy_state_name = (param_name + "_" + opt_unq_name + "_" + - accum_name + "_0") + dy_state_name = ( + param_name + + "_" + + opt_unq_name + + "_" + + accum_name + + "_0" + ) converted_state[dy_state_name] = state_var if not hasattr(self.model._optimizer, 'set_state_dict'): @@ -896,12 +992,16 @@ class DynamicGraphAdapter(object): self.model._optimizer.set_state_dict(converted_state) def prepare(self): - if self._amp_level == "O2" and self.model.mode == 'train' and core.is_compiled_with_cuda( + if ( + self._amp_level == "O2" + and self.model.mode == 'train' + and core.is_compiled_with_cuda() ): self.model.network, self.model._optimizer = paddle.amp.decorate( models=self.model.network, optimizers=self.model._optimizer, - level='O2') + level='O2', + ) if self._amp_level != "O0": self.model._scaler = None @@ -1336,17 +1436,24 @@ class Model(object): state = param_state.get(key, None) if state is None: raise ValueError( - "{} is not found in the providing file.".format(key)) + "{} is not found in the providing file.".format(key) + ) if list(state.shape) != list(param.shape): raise ValueError( - "{} receives a shape {}, but the expected shape is {}.". - format(key, list(state.shape), list(param.shape))) + "{} receives a shape {}, but the expected shape is {}.".format( + key, list(state.shape), list(param.shape) + ) + ) return param, state def _strip_postfix(path): path, ext = os.path.splitext(path) - assert ext in ['', '.pdparams', '.pdopt', '.pdmodel'], \ - "Unknown postfix {} from weights".format(ext) + assert ext in [ + '', + '.pdparams', + '.pdopt', + '.pdmodel', + ], "Unknown postfix {} from weights".format(ext) return path path = _strip_postfix(path) @@ -1360,15 +1467,17 @@ class Model(object): except ValueError as err: if skip_mismatch: warnings.warn( - ("Skip loading for {}. ".format(key) + str(err))) + ("Skip loading for {}. ".format(key) + str(err)) + ) # reset optimizer when mismatch happens reset_optimizer = True else: raise err matched_param_state.append(match_res) - optim_state = None if reset_optimizer else _load_state_from_path( - path + ".pdopt") + optim_state = ( + None if reset_optimizer else _load_state_from_path(path + ".pdopt") + ) # TODO: support save/load scaler state in static graph if _non_static_mode(): @@ -1377,8 +1486,9 @@ class Model(object): if os.path.exists(path + '.pdscaler'): scaler_state = paddle.load(path + '.pdscaler') - return self._adapter.load(matched_param_state, optim_state, - scaler_state) + return self._adapter.load( + matched_param_state, optim_state, scaler_state + ) else: return self._adapter.load(matched_param_state, optim_state) @@ -1410,13 +1520,14 @@ class Model(object): return self._adapter.parameters() def _prepare_amp(self, amp_configs): - def _check_pure_fp16_configs(): # pure float16 training has some restricts now if self._adapter._amp_level == "O2" and self._optimizer._grad_clip: # clip by value is not supported - assert isinstance(self._optimizer._grad_clip, (paddle.nn.ClipGradByGlobalNorm, paddle.nn.ClipGradByNorm)), \ - "Only GradientClipByNorm and GradientClipByGlobalNorm are supported in amp training with level=O2 currently." + assert isinstance( + self._optimizer._grad_clip, + (paddle.nn.ClipGradByGlobalNorm, paddle.nn.ClipGradByNorm), + ), "Only GradientClipByNorm and GradientClipByGlobalNorm are supported in amp training with level=O2 currently." self._adapter._amp_custom_lists = {} self._adapter._amp_configs = {} @@ -1428,7 +1539,8 @@ class Model(object): elif isinstance(amp_configs, str): if amp_configs not in ('O0', 'O1', 'O2'): raise ValueError( - "The level of amp_configs should be 'O0', 'O1' or 'O2'.") + "The level of amp_configs should be 'O0', 'O1' or 'O2'." + ) self._adapter._amp_level = amp_configs _check_pure_fp16_configs() return @@ -1437,7 +1549,8 @@ class Model(object): self._adapter._amp_level = 'O1' elif amp_configs['level'] not in ('O0', 'O1', 'O2'): raise ValueError( - "amp_configs['level'] should be 'O0', 'O1' or 'O2'.") + "amp_configs['level'] should be 'O0', 'O1' or 'O2'." + ) else: self._adapter._amp_level = amp_configs['level'] amp_config_key_set = set(amp_configs.keys()) - {'level'} @@ -1454,12 +1567,14 @@ class Model(object): # construct amp_custom_lists if self._adapter._amp_level != 'O0' and amp_config_key_set: for param_name in [ - 'custom_white_list', 'custom_black_list', - 'custom_black_varnames' + 'custom_white_list', + 'custom_black_list', + 'custom_black_varnames', ]: if param_name in amp_config_key_set: self._adapter._amp_custom_lists[param_name] = amp_configs[ - param_name] + param_name + ] amp_config_key_set -= {param_name} def _check_amp_configs(amp_config_key_set): @@ -1474,13 +1589,16 @@ class Model(object): } if amp_config_key_set - accepted_param_set: raise ValueError( - "Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, but {} could not be recognized." - .format(tuple(amp_config_key_set - accepted_param_set))) + "Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, but {} could not be recognized.".format( + tuple(amp_config_key_set - accepted_param_set) + ) + ) if 'use_fp16_guard' in amp_config_key_set: if _non_static_mode(): raise ValueError( - "'use_fp16_guard' is supported in static mode only.") + "'use_fp16_guard' is supported in static mode only." + ) self._adapter._use_fp16_guard = amp_configs['use_fp16_guard'] amp_config_key_set.remove('use_fp16_guard') @@ -1490,11 +1608,9 @@ class Model(object): for key in amp_configs_set: self._adapter._amp_configs[key] = amp_configs[key] - def prepare(self, - optimizer=None, - loss=None, - metrics=None, - amp_configs=None): + def prepare( + self, optimizer=None, loss=None, metrics=None, amp_configs=None + ): """ Configures the model before runing. @@ -1534,15 +1650,17 @@ class Model(object): if ParallelEnv().nranks > 1 and not _parallel_context_initialized: if fluid._non_static_mode(): main_prog_seed = fluid.default_main_program().random_seed - startup_prog_seed = fluid.default_startup_program( - ).random_seed + startup_prog_seed = ( + fluid.default_startup_program().random_seed + ) fluid.disable_dygraph() paddle.disable_static(self._place) # enable_dygraph would create and switch to a new program, # thus also copy seed to the new program fluid.default_main_program().random_seed = main_prog_seed - fluid.default_startup_program( - ).random_seed = startup_prog_seed + fluid.default_startup_program().random_seed = ( + startup_prog_seed + ) else: prepare_distributed_context(self._place) _parallel_context_initialized = True @@ -1557,30 +1675,32 @@ class Model(object): metrics = metrics or [] for metric in to_list(metrics): - assert isinstance(metric, Metric), \ - "{} is not sub class of Metric".format( - metric.__class__.__name__) + assert isinstance( + metric, Metric + ), "{} is not sub class of Metric".format(metric.__class__.__name__) self._metrics = to_list(metrics) self._prepare_amp(amp_configs) self._adapter.prepare() - def fit(self, - train_data=None, - eval_data=None, - batch_size=1, - epochs=1, - eval_freq=1, - log_freq=10, - save_dir=None, - save_freq=1, - verbose=2, - drop_last=False, - shuffle=True, - num_workers=0, - callbacks=None, - accumulate_grad_batches=1, - num_iters=None): + def fit( + self, + train_data=None, + eval_data=None, + batch_size=1, + epochs=1, + eval_freq=1, + log_freq=10, + save_dir=None, + save_freq=1, + verbose=2, + drop_last=False, + shuffle=True, + num_workers=0, + callbacks=None, + accumulate_grad_batches=1, + num_iters=None, + ): """ Trains the model for a fixed number of epochs. If `eval_data` is set, evaluation will be done at the end of each epoch. @@ -1714,30 +1834,36 @@ class Model(object): epochs=2, save_dir='mnist_checkpoint') """ - assert train_data is not None, \ - "train_data must be given!" + assert train_data is not None, "train_data must be given!" if isinstance(train_data, Dataset): - train_sampler = DistributedBatchSampler(train_data, - batch_size=batch_size, - shuffle=shuffle, - drop_last=drop_last) - train_loader = DataLoader(train_data, - batch_sampler=train_sampler, - places=self._place, - num_workers=num_workers, - return_list=True) + train_sampler = DistributedBatchSampler( + train_data, + batch_size=batch_size, + shuffle=shuffle, + drop_last=drop_last, + ) + train_loader = DataLoader( + train_data, + batch_sampler=train_sampler, + places=self._place, + num_workers=num_workers, + return_list=True, + ) else: train_loader = train_data if eval_data is not None and isinstance(eval_data, Dataset): - eval_sampler = DistributedBatchSampler(eval_data, - batch_size=batch_size) - eval_loader = DataLoader(eval_data, - batch_sampler=eval_sampler, - places=self._place, - num_workers=num_workers, - return_list=True) + eval_sampler = DistributedBatchSampler( + eval_data, batch_size=batch_size + ) + eval_loader = DataLoader( + eval_data, + batch_sampler=eval_sampler, + places=self._place, + num_workers=num_workers, + return_list=True, + ) elif eval_data is not None: eval_loader = eval_data else: @@ -1750,8 +1876,11 @@ class Model(object): steps = self._len_data_loader(train_loader) self.num_iters = num_iters - if num_iters is not None and isinstance(num_iters, int) and isinstance( - steps, int): + if ( + num_iters is not None + and isinstance(num_iters, int) + and isinstance(steps, int) + ): assert num_iters > 0, "num_iters must be greater than 0!" epochs = (num_iters // steps) + 1 steps = min(num_iters, steps) @@ -1779,10 +1908,10 @@ class Model(object): if do_eval and epoch % eval_freq == 0: eval_steps = self._len_data_loader(eval_loader) - cbks.on_begin('eval', { - 'steps': eval_steps, - 'metrics': self._metrics_name() - }) + cbks.on_begin( + 'eval', + {'steps': eval_steps, 'metrics': self._metrics_name()}, + ) eval_logs = self._run_one_epoch(eval_loader, cbks, 'eval') @@ -1793,14 +1922,16 @@ class Model(object): cbks.on_end('train', logs) self._test_dataloader = None - def evaluate(self, - eval_data, - batch_size=1, - log_freq=10, - verbose=2, - num_workers=0, - callbacks=None, - num_iters=None): + def evaluate( + self, + eval_data, + batch_size=1, + log_freq=10, + verbose=2, + num_workers=0, + callbacks=None, + num_iters=None, + ): """ Evaluate the loss and metrics of the model on input dataset. @@ -1854,13 +1985,16 @@ class Model(object): """ if eval_data is not None and isinstance(eval_data, Dataset): - eval_sampler = DistributedBatchSampler(eval_data, - batch_size=batch_size) - eval_loader = DataLoader(eval_data, - batch_sampler=eval_sampler, - places=self._place, - num_workers=num_workers, - return_list=True) + eval_sampler = DistributedBatchSampler( + eval_data, batch_size=batch_size + ) + eval_loader = DataLoader( + eval_data, + batch_sampler=eval_sampler, + places=self._place, + num_workers=num_workers, + return_list=True, + ) else: eval_loader = eval_data @@ -1876,15 +2010,17 @@ class Model(object): eval_steps = self._len_data_loader(eval_loader) self.num_iters = num_iters - if num_iters is not None and isinstance(num_iters, int) and isinstance( - eval_steps, int): + if ( + num_iters is not None + and isinstance(num_iters, int) + and isinstance(eval_steps, int) + ): assert num_iters > 0, "num_iters must be greater than 0!" eval_steps = min(num_iters, eval_steps) self.num_iters = eval_steps - cbks.on_begin('eval', { - 'steps': eval_steps, - 'metrics': self._metrics_name() - }) + cbks.on_begin( + 'eval', {'steps': eval_steps, 'metrics': self._metrics_name()} + ) logs = self._run_one_epoch(eval_loader, cbks, 'eval') @@ -1898,13 +2034,15 @@ class Model(object): return eval_result - def predict(self, - test_data, - batch_size=1, - num_workers=0, - stack_outputs=False, - verbose=1, - callbacks=None): + def predict( + self, + test_data, + batch_size=1, + num_workers=0, + stack_outputs=False, + verbose=1, + callbacks=None, + ): """ Compute the output predictions on testing data. @@ -1975,13 +2113,16 @@ class Model(object): """ if test_data is not None and isinstance(test_data, Dataset): - test_sampler = DistributedBatchSampler(test_data, - batch_size=batch_size) - test_loader = DataLoader(test_data, - batch_sampler=test_sampler, - places=self._place, - num_workers=num_workers, - return_list=True) + test_sampler = DistributedBatchSampler( + test_data, batch_size=batch_size + ) + test_loader = DataLoader( + test_data, + batch_sampler=test_sampler, + places=self._place, + num_workers=num_workers, + return_list=True, + ) else: test_loader = test_data @@ -2031,7 +2172,8 @@ class Model(object): if self._is_shape_inferred: warnings.warn( "'inputs' was not specified when Model initialization, so the input shape to be saved will be the shape derived from the user's actual inputs. The input shape to be saved is %s. For saving correct input shapes, please provide 'inputs' for Model initialization." - % self._input_info[0]) + % self._input_info[0] + ) paddle.jit.save(layer, path, input_spec=self._inputs) @@ -2042,7 +2184,8 @@ class Model(object): raise ValueError( "The input path MUST be format of dirname/file_prefix " "[dirname\\file_prefix in Windows system], but received " - "file_prefix is empty string.") + "file_prefix is empty string." + ) dirname = os.path.dirname(path) if dirname and not os.path.exists(dirname): @@ -2053,21 +2196,24 @@ class Model(object): params_filename = file_prefix + INFER_PARAMS_SUFFIX prog = self._adapter._progs.get('test', None) - assert prog, \ - "Model is not ready, please call `model.prepare()` first" + assert ( + prog + ), "Model is not ready, please call `model.prepare()` first" infer_prog = prog.clone(for_test=True) input_names = [v.name for v in self._adapter._input_vars['test']] endpoints = self._adapter._endpoints['test']['output'] - fluid.io.save_inference_model(model_path, - input_names, - endpoints, - self._adapter._executor, - main_program=infer_prog, - model_filename=model_filename, - params_filename=params_filename) + fluid.io.save_inference_model( + model_path, + input_names, + endpoints, + self._adapter._executor, + main_program=infer_prog, + model_filename=model_filename, + params_filename=params_filename, + ) def _run_one_epoch( self, @@ -2093,16 +2239,21 @@ class Model(object): # LoDTensor.shape is callable, where LoDTensor comes from # DataLoader in static graph - batch_size = data[0].shape()[0] if callable( - data[0].shape) else data[0].shape[0] + batch_size = ( + data[0].shape()[0] + if callable(data[0].shape) + else data[0].shape[0] + ) callbacks.on_batch_begin(mode, step, logs) if mode != 'predict': - _inputs = [data[:len(self._inputs)], data[len(self._inputs):]] + _inputs = [data[: len(self._inputs)], data[len(self._inputs) :]] if mode == 'train': - _inputs.append((step + 1) % self._accumulate == 0 - or step + 1 == len(data_loader)) + _inputs.append( + (step + 1) % self._accumulate == 0 + or step + 1 == len(data_loader) + ) outs = getattr(self, mode + '_batch')(*_inputs) @@ -2123,15 +2274,17 @@ class Model(object): logs[k] = v else: if self._inputs is not None: - outs = self.predict_batch(data[:len(self._inputs)]) + outs = self.predict_batch(data[: len(self._inputs)]) else: outs = self.predict_batch(data) outputs.append(outs) logs['step'] = step - if mode == 'train' or self._adapter._merge_count.get( - mode + '_batch', 0) <= 0: + if ( + mode == 'train' + or self._adapter._merge_count.get(mode + '_batch', 0) <= 0 + ): logs['batch_size'] = batch_size * ParallelEnv().nranks else: logs['batch_size'] = self._adapter._merge_count[mode + '_batch'] @@ -2185,8 +2338,9 @@ class Model(object): # {'total_params': 61610, 'trainable_params': 61610} """ - assert (input_size is not None or self._inputs - is not None), "'input_size' or 'self._input' must be set" + assert ( + input_size is not None or self._inputs is not None + ), "'input_size' or 'self._input' must be set" if input_size is not None: _input_size = input_size else: @@ -2203,7 +2357,10 @@ class Model(object): if is_input: arg_names = extract_args(self.network.forward)[1:] # While Saving inference model in dygraph, and providing inputs only in running. - if shapes is not None and dtypes is not None and fluid._non_static_mode( + if ( + shapes is not None + and dtypes is not None + and fluid._non_static_mode() ): out_specs = [ Input(name=n, dtype=dtypes[i], shape=shapes[i]) @@ -2216,7 +2373,8 @@ class Model(object): elif isinstance(specs, dict): assert is_input is False out_specs = [ - specs[n] for n in extract_args(self.network.forward) + specs[n] + for n in extract_args(self.network.forward) if n != 'self' ] else: @@ -2227,8 +2385,10 @@ class Model(object): assert isinstance(spec, Input) if spec.name is None: raise ValueError( - "Requires Input[{}].name != None, but receive `None` with {}." - .format(i, spec)) + "Requires Input[{}].name != None, but receive `None` with {}.".format( + i, spec + ) + ) return out_specs @@ -2253,6 +2413,7 @@ class Model(object): "Update self._inputs according to given inputs." self._input_info = self._adapter._input_info if self._input_info is not None and len(self._input_info) == 2: - self._inputs = self._verify_spec(None, self._input_info[0], - self._input_info[1], True) + self._inputs = self._verify_spec( + None, self._input_info[0], self._input_info[1], True + ) self._is_shape_inferred = True diff --git a/python/paddle/hapi/model_summary.py b/python/paddle/hapi/model_summary.py index de4ec449a06..d90c64b7621 100644 --- a/python/paddle/hapi/model_summary.py +++ b/python/paddle/hapi/model_summary.py @@ -160,18 +160,20 @@ def summary(net, input_size=None, dtypes=None, input=None): _input_size = [] for item in input_size: if isinstance(item, int): - item = (item, ) - assert isinstance(item, - (tuple, InputSpec)), 'When input_size is list, \ + item = (item,) + assert isinstance( + item, (tuple, InputSpec) + ), 'When input_size is list, \ expect item in input_size is a tuple or InputSpec, but got {}'.format( - type(item)) + type(item) + ) if isinstance(item, InputSpec): _input_size.append(tuple(item.shape)) else: _input_size.append(item) elif isinstance(input_size, int): - _input_size = (input_size, ) + _input_size = (input_size,) else: _input_size = input_size @@ -207,8 +209,10 @@ def summary(net, input_size=None, dtypes=None, input=None): elif isinstance(item, numbers.Number): if item <= 0: raise ValueError( - "Expected element in input size greater than zero, but got {}" - .format(item)) + "Expected element in input size greater than zero, but got {}".format( + item + ) + ) new_shape.append(item) return tuple(new_shape) @@ -231,7 +235,6 @@ def summary(net, input_size=None, dtypes=None, input=None): @no_grad() def summary_string(model, input_size=None, dtypes=None, input=None): - def _all_is_numper(items): for item in items: if not isinstance(item, numbers.Number): @@ -272,7 +275,6 @@ def summary_string(model, input_size=None, dtypes=None, input=None): return output_shape def register_hook(layer): - def hook(layer, input, output): class_name = str(layer.__class__).split(".")[-1].split("'")[0] @@ -310,7 +312,8 @@ def summary_string(model, input_size=None, dtypes=None, input=None): try: if (getattr(getattr(layer, k), 'trainable')) and ( - not getattr(getattr(layer, k), 'stop_gradient')): + not getattr(getattr(layer, k), 'stop_gradient') + ): summary[m_key]["trainable_params"] += np.prod(v.shape) summary[m_key]["trainable"] = True trainable_flag = True @@ -321,9 +324,11 @@ def summary_string(model, input_size=None, dtypes=None, input=None): summary[m_key]["nb_params"] = params - if (not isinstance(layer, nn.Sequential) - and not isinstance(layer, nn.LayerList) - and (not (layer == model) or depth < 1)): + if ( + not isinstance(layer, nn.Sequential) + and not isinstance(layer, nn.LayerList) + and (not (layer == model) or depth < 1) + ): hooks.append(layer.register_forward_post_hook(hook)) # For rnn, gru and lstm layer @@ -368,24 +373,30 @@ def summary_string(model, input_size=None, dtypes=None, input=None): 'input_shape_width': 20, 'output_shape_width': 20, 'params_width': 15, - 'table_width': 75 + 'table_width': 75, } for layer in summary: if head_length['output_shape_width'] < len( - str(summary[layer]["output_shape"])): + str(summary[layer]["output_shape"]) + ): head_length['output_shape_width'] = len( - str(summary[layer]["output_shape"])) + str(summary[layer]["output_shape"]) + ) if head_length['input_shape_width'] < len( - str(summary[layer]["input_shape"])): + str(summary[layer]["input_shape"]) + ): head_length['input_shape_width'] = len( - str(summary[layer]["input_shape"])) + str(summary[layer]["input_shape"]) + ) if head_length['layer_width'] < len(str(layer)): head_length['layer_width'] = len(str(layer)) if head_length['params_width'] < len( - str(summary[layer]["nb_params"])): + str(summary[layer]["nb_params"]) + ): head_length['params_width'] = len( - str(summary[layer]["nb_params"])) + str(summary[layer]["nb_params"]) + ) _temp_width = 0 for k, v in head_length.items(): @@ -401,10 +412,15 @@ def summary_string(model, input_size=None, dtypes=None, input=None): summary_str += "-" * table_width['table_width'] + "\n" line_new = "{:^{}} {:^{}} {:^{}} {:^{}}".format( - "Layer (type)", table_width['layer_width'], "Input Shape", - table_width['input_shape_width'], "Output Shape", - table_width['output_shape_width'], "Param #", - table_width['params_width']) + "Layer (type)", + table_width['layer_width'], + "Input Shape", + table_width['input_shape_width'], + "Output Shape", + table_width['output_shape_width'], + "Param #", + table_width['params_width'], + ) summary_str += line_new + "\n" summary_str += "=" * table_width['table_width'] + "\n" total_params = 0 @@ -414,17 +430,21 @@ def summary_string(model, input_size=None, dtypes=None, input=None): for layer in summary: # input_shape, output_shape, trainable, nb_params line_new = "{:^{}} {:^{}} {:^{}} {:^{}}".format( - layer, table_width['layer_width'], + layer, + table_width['layer_width'], str(summary[layer]["input_shape"]), table_width['input_shape_width'], str(summary[layer]["output_shape"]), - table_width['output_shape_width'], "{0:,}".format( - summary[layer]["nb_params"]), table_width['params_width']) + table_width['output_shape_width'], + "{0:,}".format(summary[layer]["nb_params"]), + table_width['params_width'], + ) total_params += summary[layer]["nb_params"] try: total_output += np.sum( - np.prod(summary[layer]["output_shape"], axis=-1)) + np.prod(summary[layer]["output_shape"], axis=-1) + ) except: for output_shape in summary[layer]["output_shape"]: total_output += np.sum(np.prod(output_shape, axis=-1)) @@ -436,26 +456,31 @@ def summary_string(model, input_size=None, dtypes=None, input=None): def _get_input_size(input_size, size): if isinstance(input_size, (list, tuple)) and _all_is_numper(input_size): - size = abs(np.prod(input_size) * 4. / (1024**2.)) + size = abs(np.prod(input_size) * 4.0 / (1024**2.0)) else: size = sum([_get_input_size(i, size) for i in input_size]) return size total_input_size = _get_input_size(input_size, 0) - total_output_size = abs(2. * total_output * 4. / - (1024**2.)) # x2 for gradients - total_params_size = abs(total_params * 4. / (1024**2.)) + total_output_size = abs( + 2.0 * total_output * 4.0 / (1024**2.0) + ) # x2 for gradients + total_params_size = abs(total_params * 4.0 / (1024**2.0)) total_size = total_params_size + total_output_size + total_input_size summary_str += "=" * table_width['table_width'] + "\n" summary_str += "Total params: {0:,}".format(total_params) + "\n" summary_str += "Trainable params: {0:,}".format(trainable_params) + "\n" - summary_str += "Non-trainable params: {0:,}".format(total_params - - trainable_params) + "\n" + summary_str += ( + "Non-trainable params: {0:,}".format(total_params - trainable_params) + + "\n" + ) summary_str += "-" * table_width['table_width'] + "\n" summary_str += "Input size (MB): %0.2f" % total_input_size + "\n" - summary_str += "Forward/backward pass size (MB): %0.2f" % total_output_size + "\n" + summary_str += ( + "Forward/backward pass size (MB): %0.2f" % total_output_size + "\n" + ) summary_str += "Params size (MB): %0.2f" % total_params_size + "\n" summary_str += "Estimated Total Size (MB): %0.2f" % total_size + "\n" summary_str += "-" * table_width['table_width'] + "\n" @@ -463,5 +488,5 @@ def summary_string(model, input_size=None, dtypes=None, input=None): # return summary return summary_str, { 'total_params': total_params, - 'trainable_params': trainable_params + 'trainable_params': trainable_params, } diff --git a/python/paddle/hapi/progressbar.py b/python/paddle/hapi/progressbar.py index 41e2b5422af..43b913a6fc1 100644 --- a/python/paddle/hapi/progressbar.py +++ b/python/paddle/hapi/progressbar.py @@ -23,15 +23,17 @@ __all__ = [] class ProgressBar(object): - """progress bar """ - - def __init__(self, - num=None, - width=30, - verbose=1, - start=True, - file=sys.stdout, - name='step'): + """progress bar""" + + def __init__( + self, + num=None, + width=30, + verbose=1, + start=True, + file=sys.stdout, + name='step', + ): self._num = num if isinstance(num, int) and num <= 0: raise TypeError('num should be None or integer (> 0)') @@ -47,11 +49,12 @@ class ProgressBar(object): self._last_update = 0 self.name = name - self._dynamic_display = ((hasattr(self.file, 'isatty') - and self.file.isatty()) - or 'ipykernel' in sys.modules - or 'posix' in sys.modules - or 'PYCHARM_HOSTED' in os.environ) + self._dynamic_display = ( + (hasattr(self.file, 'isatty') and self.file.isatty()) + or 'ipykernel' in sys.modules + or 'posix' in sys.modules + or 'PYCHARM_HOSTED' in os.environ + ) def _get_max_width(self): if sys.version_info > (3, 3): @@ -81,13 +84,17 @@ class ProgressBar(object): in_list = np.asarray(in_list) out = np.vectorize( lambda x: struct.unpack(' 0: - bar_chars += ('=' * (prog_width - 1)) + bar_chars += '=' * (prog_width - 1) if current_num < self._num: bar_chars += '>' else: bar_chars += '=' - bar_chars += ('.' * (self._width - prog_width)) + bar_chars += '.' * (self._width - prog_width) bar_chars += ']' else: bar_chars = self.name + ' %3d' % current_num @@ -150,8 +159,11 @@ class ProgressBar(object): if self._num is not None and current_num < self._num: eta = time_per_unit * (self._num - current_num) if eta > 3600: - eta_format = '%d:%02d:%02d' % (eta // 3600, - (eta % 3600) // 60, eta % 60) + eta_format = '%d:%02d:%02d' % ( + eta // 3600, + (eta % 3600) // 60, + eta % 60, + ) elif eta > 60: eta_format = '%d:%02d' % (eta // 60, eta % 60) else: @@ -162,7 +174,7 @@ class ProgressBar(object): info += fps self._total_width += len(info) if prev_total_width > self._total_width: - info += (' ' * (prev_total_width - self._total_width)) + info += ' ' * (prev_total_width - self._total_width) # newline for another epoch if self._num is not None and current_num >= self._num: @@ -176,8 +188,10 @@ class ProgressBar(object): elif self._verbose == 2 or self._verbose == 3: if self._num: numdigits = int(np.log10(self._num)) + 1 - count = (self.name + ' %' + str(numdigits) + - 'd/%d') % (current_num, self._num) + count = (self.name + ' %' + str(numdigits) + 'd/%d') % ( + current_num, + self._num, + ) else: count = self.name + ' %3d' % current_num info = count + info @@ -191,9 +205,11 @@ class ProgressBar(object): info += ' %.4f' % v else: info += ' %.4e' % v - elif isinstance(v, np.ndarray) and \ - v.size == 1 and \ - v.dtype in [np.float32, np.float64]: + elif ( + isinstance(v, np.ndarray) + and v.size == 1 + and v.dtype in [np.float32, np.float64] + ): if abs(v[0]) > 1e-3: info += ' %.4f' % v[0] else: diff --git a/python/paddle/hapi/static_flops.py b/python/paddle/hapi/static_flops.py index 8926bcb3552..d1effd27b46 100644 --- a/python/paddle/hapi/static_flops.py +++ b/python/paddle/hapi/static_flops.py @@ -20,7 +20,6 @@ __all__ = [] class VarWrapper(object): - def __init__(self, var, graph): assert isinstance(var, Variable) assert isinstance(graph, GraphWrapper) @@ -41,7 +40,6 @@ class VarWrapper(object): class OpWrapper(object): - def __init__(self, op, graph): assert isinstance(graph, GraphWrapper) self._op = op @@ -87,8 +85,7 @@ class GraphWrapper(object): """ def __init__(self, program=None, in_nodes=[], out_nodes=[]): - """ - """ + """ """ super(GraphWrapper, self).__init__() self.program = Program() if program is None else program self.persistables = {} @@ -212,7 +209,6 @@ def static_flops(program, print_detail=False): class Table(object): - def __init__(self, table_heads): self.table_heads = table_heads self.table_len = [] @@ -226,8 +222,10 @@ class Table(object): print('The row_str should be a list') if len(row_str) != self.col_num: print( - 'The length of row data should be equal the length of table heads, but the data: {} is not equal table heads {}' - .format(len(row_str), self.col_num)) + 'The length of row data should be equal the length of table heads, but the data: {} is not equal table heads {}'.format( + len(row_str), self.col_num + ) + ) for i in range(self.col_num): if len(str(row_str[i])) > self.table_len[i]: self.table_len[i] = len(str(row_str[i])) diff --git a/python/paddle/hub.py b/python/paddle/hub.py index acdb28cb6f0..e6f38d6ee11 100644 --- a/python/paddle/hub.py +++ b/python/paddle/hub.py @@ -16,6 +16,4 @@ from .hapi.hub import list # noqa: F401 from .hapi.hub import help # noqa: F401 from .hapi.hub import load # noqa: F401 -__all__ = [ #noqa - 'list', 'help', 'load' -] +__all__ = ['list', 'help', 'load'] # noqa diff --git a/python/paddle/incubate/__init__.py b/python/paddle/incubate/__init__.py index 68c7db05499..2730db97f0e 100644 --- a/python/paddle/incubate/__init__.py +++ b/python/paddle/incubate/__init__.py @@ -29,10 +29,10 @@ from .tensor import segment_max from .tensor import segment_min from .passes import fuse_resnet_unit_pass -from . import autograd #noqa: F401 -from . import autotune #noqa: F401 -from . import nn #noqa: F401 -from . import asp #noqa: F401 +from . import autograd # noqa: F401 +from . import autotune # noqa: F401 +from . import nn # noqa: F401 +from . import asp # noqa: F401 from ..fluid.layers.loss import identity_loss diff --git a/python/paddle/incubate/asp/__init__.py b/python/paddle/incubate/asp/__init__.py index d2a56fd117c..662e24900fc 100644 --- a/python/paddle/incubate/asp/__init__.py +++ b/python/paddle/incubate/asp/__init__.py @@ -13,13 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.contrib.sparsity import calculate_density #noqa: F401 -from ...fluid.contrib.sparsity import decorate #noqa: F401 -from ...fluid.contrib.sparsity import prune_model #noqa: F401 -from ...fluid.contrib.sparsity import set_excluded_layers #noqa: F401 -from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401 +from ...fluid.contrib.sparsity import calculate_density # noqa: F401 +from ...fluid.contrib.sparsity import decorate # noqa: F401 +from ...fluid.contrib.sparsity import prune_model # noqa: F401 +from ...fluid.contrib.sparsity import set_excluded_layers # noqa: F401 +from ...fluid.contrib.sparsity import reset_excluded_layers # noqa: F401 -__all__ = [ #noqa - 'calculate_density', 'decorate', 'prune_model', 'set_excluded_layers', - 'reset_excluded_layers' +__all__ = [ # noqa + 'calculate_density', + 'decorate', + 'prune_model', + 'set_excluded_layers', + 'reset_excluded_layers', ] diff --git a/python/paddle/incubate/autograd/__init__.py b/python/paddle/incubate/autograd/__init__.py index c5ff3b18d4d..d9b9e417819 100644 --- a/python/paddle/incubate/autograd/__init__.py +++ b/python/paddle/incubate/autograd/__init__.py @@ -17,6 +17,12 @@ from .primx import prim2orig from .utils import disable_prim, enable_prim, prim_enabled __all__ = [ # noqa - 'vjp', 'jvp', 'Jacobian', 'Hessian', 'enable_prim', 'disable_prim', - 'forward_grad', 'grad' + 'vjp', + 'jvp', + 'Jacobian', + 'Hessian', + 'enable_prim', + 'disable_prim', + 'forward_grad', + 'grad', ] diff --git a/python/paddle/incubate/autograd/functional.py b/python/paddle/incubate/autograd/functional.py index dce83b7505f..eafaaaea329 100644 --- a/python/paddle/incubate/autograd/functional.py +++ b/python/paddle/incubate/autograd/functional.py @@ -253,8 +253,7 @@ class Jacobian(object): @property def shape(self): - """The shape of flattened Jacobian matrix. - """ + """The shape of flattened Jacobian matrix.""" return self._jacobian.shape @@ -308,11 +307,11 @@ class Hessian(object): """ def __init__(self, func, xs, is_batched=False): - def _jac_func(*xs): jac = Jacobian(func, xs, is_batched=is_batched) - if (is_batched and jac.shape[1] != 1) or (not is_batched - and jac.shape[0] != 1): + if (is_batched and jac.shape[1] != 1) or ( + not is_batched and jac.shape[0] != 1 + ): raise RuntimeError( "The function given to Hessian shoud return as single element Tensor or batched single element Tensor." ) @@ -325,8 +324,7 @@ class Hessian(object): @property def shape(self): - """The shape of flattened Hessian matrix. - """ + """The shape of flattened Hessian matrix.""" return self.symbolic.shape @@ -369,32 +367,41 @@ class _Jacobian(object): @property def _lazy_axis(self): - """"The axis of lazily evaluated.""" + """ "The axis of lazily evaluated.""" raise NotImplementedError def _lazy_indexes(self, indexes): idx = indexes[self._lazy_axis] - return (idx, ) if isinstance(idx, int) else tuple( - range(idx.start, idx.stop, idx.step)) + return ( + (idx,) + if isinstance(idx, int) + else tuple(range(idx.start, idx.stop, idx.step)) + ) def _flatten(self, xs): raise NotImplementedError def _shifted_indexes(self, indexes, lazy_axis_size=0): idx = indexes[self._lazy_axis] - shifted_lazy_axis_idx = 0 if isinstance(idx, int) else slice( - 0, lazy_axis_size, 1) - return indexes[:self._lazy_axis] + ( - shifted_lazy_axis_idx, ) + indexes[self._lazy_axis + 1:] + shifted_lazy_axis_idx = ( + 0 if isinstance(idx, int) else slice(0, lazy_axis_size, 1) + ) + return ( + indexes[: self._lazy_axis] + + (shifted_lazy_axis_idx,) + + indexes[self._lazy_axis + 1 :] + ) def __getitem__(self, indexes): indexes = _multi_index(indexes, self.shape) if isinstance(indexes[self._lazy_axis], int): - other_indexes = indexes[:self._lazy_axis] + \ - indexes[self._lazy_axis+1:] - return self._cached_evaluate( - indexes[self._lazy_axis])[other_indexes] + other_indexes = ( + indexes[: self._lazy_axis] + indexes[self._lazy_axis + 1 :] + ) + return self._cached_evaluate(indexes[self._lazy_axis])[ + other_indexes + ] lazy_indexes = self._lazy_indexes(indexes) # Using concat and reshape to replace stack operator temporarily, as # it is not a primitive operator. @@ -402,7 +409,8 @@ class _Jacobian(object): shape[self._lazy_axis] = len(lazy_indexes) part_jac = paddle.concat( [self._cached_evaluate(i) for i in lazy_indexes], - axis=self._lazy_axis).reshape(shape) + axis=self._lazy_axis, + ).reshape(shape) return part_jac[self._shifted_indexes(indexes, len(lazy_indexes))] def _cached_evaluate(self, k): @@ -435,13 +443,15 @@ class _JacobianNoBatch(_Jacobian): return 0 def _flatten(self, xs): - return paddle.concat(tuple(x.reshape((-1, )) for x in xs)) + return paddle.concat(tuple(x.reshape((-1,)) for x in xs)) def _evaluate(self, row_index): - return self._flatten(_grad( - self._flatten_ys[row_index], - self._xs, - )) + return self._flatten( + _grad( + self._flatten_ys[row_index], + self._xs, + ) + ) class _JacobianBatchFirst(_Jacobian): @@ -455,8 +465,11 @@ class _JacobianBatchFirst(_Jacobian): @property def shape(self): - return (self._flatten_xs.shape[0], self._flatten_ys.shape[1], - self._flatten_xs.shape[1]) + return ( + self._flatten_xs.shape[0], + self._flatten_ys.shape[1], + self._flatten_xs.shape[1], + ) @property def _lazy_axis(self): @@ -464,7 +477,8 @@ class _JacobianBatchFirst(_Jacobian): def _flatten(self, xs): return paddle.concat( - tuple(x.reshape((x.shape[0], -1)) for x in utils.as_tensors(xs)), 1) + tuple(x.reshape((x.shape[0], -1)) for x in utils.as_tensors(xs)), 1 + ) def _evaluate(self, row_index): return self._flatten(_grad(self._flatten_ys[:, row_index], self._xs)) @@ -490,24 +504,27 @@ def _multi_index(indexes, shape): Returns: tuple: The standard format index as the above description. """ - indexes = indexes if isinstance(indexes, typing.Sequence) else (indexes, ) + indexes = indexes if isinstance(indexes, typing.Sequence) else (indexes,) if any(isinstance(i, type(Ellipsis)) for i in indexes): raise IndexError('Ellipsis index currently is not supported.') # Fill the right-most elements. - indexes = indexes + (slice(0, None, None), ) * (len(shape) - len(indexes)) + indexes = indexes + (slice(0, None, None),) * (len(shape) - len(indexes)) # Convert to positive index. positive_indexes = [] for i, index in enumerate(indexes): if isinstance(index, slice): - index = slice(index.start or 0, index.stop or shape[i], index.step - or 1) + index = slice( + index.start or 0, index.stop or shape[i], index.step or 1 + ) positive_indexes.append( slice( index.start + shape[i] if index.start < 0 else index.start, index.stop + shape[i] if index.stop < 0 else index.stop, # Negative step means index backward, no need to convert to # positive interger. - index.step)) + index.step, + ) + ) elif isinstance(index, int): positive_indexes.append(index + shape[i] if index < 0 else index) else: @@ -522,8 +539,8 @@ def _replace_none_with_zero_tensor(xs, refs): return xs elif isinstance(xs, typing.Sequence): return tuple( - _replace_none_with_zero_tensor(x, refs[i]) - for i, x in enumerate(xs)) + _replace_none_with_zero_tensor(x, refs[i]) for i, x in enumerate(xs) + ) else: return xs @@ -568,8 +585,11 @@ def _grad(ys, xs, v=None): # follow code snippet fixes the problem by return the first element of # xs_grad when the xs is a signle Tensor. xs_grad = paddle.grad(ys, xs, v, create_graph=True, allow_unused=True) - if isinstance(xs, paddle.fluid.framework.Variable) and isinstance( - xs_grad, typing.Sequence) and len(xs_grad) > 0: + if ( + isinstance(xs, paddle.fluid.framework.Variable) + and isinstance(xs_grad, typing.Sequence) + and len(xs_grad) > 0 + ): xs_grad = xs_grad[0] else: xs_grad = paddle.incubate.autograd.grad(ys, xs, v) @@ -645,18 +665,23 @@ def _check_inputs(func, xs, v=None): raise TypeError(f"Expected 'fun' is Callable, but got {type(func)}.") if not isinstance(xs, (framework.Variable, typing.Sequence)): - raise TypeError(f"Expected 'xs' is a Tensor|Sequence[Tensor]," - f"but got {type(xs)}.") + raise TypeError( + f"Expected 'xs' is a Tensor|Sequence[Tensor]," + f"but got {type(xs)}." + ) if isinstance(xs, typing.Sequence) and not all( - isinstance(x, framework.Variable) for x in xs): + isinstance(x, framework.Variable) for x in xs + ): raise TypeError("All elements of 'xs' shoule be Tensor.") if not isinstance(v, (framework.Variable, typing.Sequence, type(None))): raise TypeError( - f"Expected 'v' is Tensor|Sequence[Tensor]|None, but got {type(v)}.") + f"Expected 'v' is Tensor|Sequence[Tensor]|None, but got {type(v)}." + ) if isinstance(v, typing.Sequence) and not all( - isinstance(e, framework.Variable) for e in v): + isinstance(e, framework.Variable) for e in v + ): raise TypeError("All elements of 'xs' shoule be Tensor.") @@ -666,11 +691,14 @@ def _check_v_shape(v, refs): v, refs = utils.as_tensors(v), utils.as_tensors(refs) if len(refs) != len(v): - raise RuntimeError(f"The argument v is a tuple of invalid length:" - f"should be {len(refs)} but got {len(v)}.") + raise RuntimeError( + f"The argument v is a tuple of invalid length:" + f"should be {len(refs)} but got {len(v)}." + ) for index, (element_v, element_ref) in enumerate(zip(v, refs)): if element_v.shape != element_ref.shape: raise RuntimeError( f"The v[{index}] has invalid shape: should " - f"be {element_ref.shape} but got {element_v.shape}.") + f"be {element_ref.shape} but got {element_v.shape}." + ) diff --git a/python/paddle/incubate/autograd/primapi.py b/python/paddle/incubate/autograd/primapi.py index fa289e259cb..18a06af5dca 100644 --- a/python/paddle/incubate/autograd/primapi.py +++ b/python/paddle/incubate/autograd/primapi.py @@ -64,25 +64,35 @@ def forward_grad(outputs, inputs, grad_inputs=None): paddle.disable_static() """ if not utils.prim_enabled(): - raise RuntimeError('forward_grad must be running on primitive' - 'operators, use enable_prim to turn it on.') + raise RuntimeError( + 'forward_grad must be running on primitive' + 'operators, use enable_prim to turn it on.' + ) if not isinstance(outputs, (framework.Variable, typing.Sequence)): - raise TypeError(f'Expected outputs is Tensor|Sequence[Tesnor], ' - f'but got {type(outputs)}.') + raise TypeError( + f'Expected outputs is Tensor|Sequence[Tesnor], ' + f'but got {type(outputs)}.' + ) if not isinstance(inputs, (framework.Variable, typing.Sequence)): - raise TypeError(f'Expected inputs is Tensor|Sequence[Tesnor], ' - f'but got {type(inputs)}.') + raise TypeError( + f'Expected inputs is Tensor|Sequence[Tesnor], ' + f'but got {type(inputs)}.' + ) - ys, xs, xs_dot = utils.as_tensors(outputs), utils.as_tensors( - inputs), utils.as_tensors(grad_inputs) + ys, xs, xs_dot = ( + utils.as_tensors(outputs), + utils.as_tensors(inputs), + utils.as_tensors(grad_inputs), + ) block = framework.default_main_program().current_block() if any(x.block != block for x in xs + ys): raise RuntimeError( 'Variable in inputs and targets should exist in current block of ' - 'main program.') + 'main program.' + ) primx.orig2prim(block) ad = primx.Transform(ys[0].block) @@ -141,22 +151,32 @@ def grad(outputs, inputs, grad_outputs=None): # backward.gradients returns a list though the inputs is a signle Tensor. # The follow code snippet fixes the problem by return the first element # of grad_inputs when the inputs is a signle Tensor. - if isinstance(inputs, framework.Variable) and isinstance( - grad_inputs, typing.Sequence) and len(grad_inputs) > 0: + if ( + isinstance(inputs, framework.Variable) + and isinstance(grad_inputs, typing.Sequence) + and len(grad_inputs) > 0 + ): return grad_inputs[0] else: return grad_inputs if not isinstance(outputs, (framework.Variable, typing.Sequence)): - raise TypeError(f'Expected outputs is Tensor|Sequence[Tesnor], ' - f'but got {type(outputs)}.') + raise TypeError( + f'Expected outputs is Tensor|Sequence[Tesnor], ' + f'but got {type(outputs)}.' + ) if not isinstance(inputs, (framework.Variable, typing.Sequence)): - raise TypeError(f'Expected inputs is Tensor|Sequence[Tesnor], ' - f'but got {type(inputs)}.') + raise TypeError( + f'Expected inputs is Tensor|Sequence[Tesnor], ' + f'but got {type(inputs)}.' + ) - ys, xs, ys_bar = utils.as_tensors(outputs), utils.as_tensors( - inputs), utils.as_tensors(grad_outputs) + ys, xs, ys_bar = ( + utils.as_tensors(outputs), + utils.as_tensors(inputs), + utils.as_tensors(grad_outputs), + ) block = framework.default_main_program().current_block() if any((x is not None and x.block != block) for x in xs + ys): raise RuntimeError( diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index 502c10783e8..29a17dda1d4 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -37,13 +37,9 @@ def _simple_binop(helper): if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=optype, - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Z': out}, - attrs={}) + helper.append_op( + type=optype, inputs={'X': x, 'Y': y}, outputs={'Z': out}, attrs={} + ) return out @@ -53,16 +49,16 @@ def _manipulation_unop(helper): attrs = { k: helper.kwargs[k] - for k in ('shape', 'axis', 'index') if k in helper.kwargs + for k in ('shape', 'axis', 'index') + if k in helper.kwargs } if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=optype, - inputs={'X': x}, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=optype, inputs={'X': x}, outputs={'Y': out}, attrs=attrs + ) return out @@ -94,23 +90,23 @@ def set_value(x, y, axis, starts, ends, strides, out): assert x is out, "x and out should be the same Tensor in set_value" attrs = {'axes': axis, 'starts': starts, 'ends': ends, 'steps': strides} helper = LayerHelper('set_value', **locals()) - helper.append_op(type=helper.layer_type, - inputs={ - 'Input': x, - 'ValueTensor': y - }, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'Input': x, 'ValueTensor': y}, + outputs={'Out': out}, + attrs=attrs, + ) return out def mean(x, axis=None, keepdim=False): axes = axis or tuple(range(0, len(x.shape))) sum = reduce_sum(x, axis=axes, keepdim=keepdim) - norm = fill_const(shape=sum.shape, - value=functools.reduce(operator.mul, - [x.shape[axis] for axis in axes]), - dtype=sum.dtype) + norm = fill_const( + shape=sum.shape, + value=functools.reduce(operator.mul, [x.shape[axis] for axis in axes]), + dtype=sum.dtype, + ) return div(sum, norm) @@ -122,16 +118,18 @@ def zeros(shape, dtype): return fill_const(0, shape, dtype) -def batch_norm(x, - axis, - gamma, - beta, - run_mean, - run_var, - eps=1e-5, - momentum=0.9, - use_run_stat=False, - reserve_space=None): +def batch_norm( + x, + axis, + gamma, + beta, + run_mean, + run_var, + eps=1e-5, + momentum=0.9, + use_run_stat=False, + reserve_space=None, +): """batch normalizer. Args: @@ -150,40 +148,59 @@ def batch_norm(x, Defaults to False. """ reduce_axes = tuple(i for i in range(len(x.shape)) if i != axis) - stats_shape = tuple(1 if i in reduce_axes else s - for i, s in enumerate(x.shape)) + stats_shape = tuple( + 1 if i in reduce_axes else s for i, s in enumerate(x.shape) + ) batch_mean = zeros(run_mean.shape, run_mean.dtype) batch_var = zeros(run_var.shape, run_var.dtype) if not use_run_stat: batch_mean = mean(x, reduce_axes, keepdim=True) - batch_var = mean(square(sub(x, broadcast(batch_mean, x.shape))), - reduce_axes, - keepdim=True) + batch_var = mean( + square(sub(x, broadcast(batch_mean, x.shape))), + reduce_axes, + keepdim=True, + ) x_hat = div( sub(x, broadcast(batch_mean, x.shape)), sqrt( - add(broadcast(batch_var, x.shape), - fill_const(eps, x.shape, batch_var.dtype)))) + add( + broadcast(batch_var, x.shape), + fill_const(eps, x.shape, batch_var.dtype), + ) + ), + ) momentum = fill_const(momentum, run_mean.shape, run_mean.dtype) run_mean = add( mul(momentum, run_mean), - mul(sub(ones(run_mean.shape, run_mean.dtype), momentum), - reshape(batch_mean, run_mean.shape))) + mul( + sub(ones(run_mean.shape, run_mean.dtype), momentum), + reshape(batch_mean, run_mean.shape), + ), + ) run_var = add( mul(momentum, run_var), - mul(sub(ones(run_var.shape, run_var.dtype), momentum), - reshape(batch_var, run_var.shape))) + mul( + sub(ones(run_var.shape, run_var.dtype), momentum), + reshape(batch_var, run_var.shape), + ), + ) else: x_hat = div( sub(x, broadcast(reshape(run_mean, stats_shape), x.shape)), sqrt( - add(broadcast(reshape(run_var, stats_shape), x.shape), - fill_const(eps, x.shape, x.dtype)))) - y = add(mul(broadcast(reshape(gamma, stats_shape), x_hat.shape), x_hat), - broadcast(reshape(beta, stats_shape), x_hat.shape)) + add( + broadcast(reshape(run_var, stats_shape), x.shape), + fill_const(eps, x.shape, x.dtype), + ) + ), + ) + y = add( + mul(broadcast(reshape(gamma, stats_shape), x_hat.shape), x_hat), + broadcast(reshape(beta, stats_shape), x_hat.shape), + ) if reserve_space: return run_mean, reserve_space, batch_mean, batch_var, run_var, y @@ -192,7 +209,7 @@ def batch_norm(x, def square(x): - return pow(x, fill_const(2., x.shape, x.dtype)) + return pow(x, fill_const(2.0, x.shape, x.dtype)) @REGISTER_FN('add_p', 'X', 'Y', 'Z') @@ -279,10 +296,12 @@ def split(x, num_or_sections, axis=0, outs=None): helper.create_variable_for_type_inference(dtype=x.dtype) for i in range(n) ] - helper.append_op(type=helper.layer_type, - inputs={'X': x}, - outputs={'YS': outs}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'X': x}, + outputs={'YS': outs}, + attrs=attrs, + ) return outs @@ -294,17 +313,19 @@ def concat(xs, axis=0, out=None): helper = LayerHelper('concat_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=xs[0].dtype) - helper.append_op(type=helper.layer_type, - inputs={'XS': xs}, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'XS': xs}, + outputs={'Y': out}, + attrs=attrs, + ) return out @REGISTER_FN('reduce_sum_p', 'X', 'Y') def reduce_sum(x, axis=None, keepdim=False, out=None): axes = axis or tuple(range(0, len(x.shape))) - axes = (axes, ) if isinstance(axes, int) else axes + axes = (axes,) if isinstance(axes, int) else axes if not isinstance(axis, (tuple, list)): raise TypeError(f'axis must be tuple or list, but got {type(axis)}') if not isinstance(keepdim, bool): @@ -315,10 +336,9 @@ def reduce_sum(x, axis=None, keepdim=False, out=None): if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={'X': x}, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, inputs={'X': x}, outputs={'Y': out}, attrs=attrs + ) return out @@ -330,28 +350,33 @@ def matmul(x, y, out=None): @REGISTER_FN('slice_select_p', 'X', 'Y') def slice_select(x, axis, starts, ends, strides, out=None): if not isinstance(axis, (list, tuple)): - raise TypeError(f'Argument type error. `axis` is supposed to be list or' - f' tuple but found {type(axis)}.') + raise TypeError( + f'Argument type error. `axis` is supposed to be list or' + f' tuple but found {type(axis)}.' + ) if not isinstance(starts, (list, tuple)): raise TypeError( f'Argument type error. `starts` is supposed to be list or' - f' tuple but found {type(starts)}.') + f' tuple but found {type(starts)}.' + ) if not isinstance(ends, (list, tuple)): - raise TypeError(f'Argument type error. `ends` is supposed to be list or' - f' tuple but found {type(ends)}.') + raise TypeError( + f'Argument type error. `ends` is supposed to be list or' + f' tuple but found {type(ends)}.' + ) assert len(axis) == len(starts) == len(ends) == len(strides), ( f'len(axis), len(starts), len(ends) and len(strides) should be equal, ' f'but len(axis)={len(axis)}, len(starts)={len(starts)}, ' - f'len(ends)={len(ends)} and len(strides)={len(strides)}') + f'len(ends)={len(ends)} and len(strides)={len(strides)}' + ) attrs = {'axis': axis, 'starts': starts, 'ends': ends, 'strides': strides} helper = LayerHelper('slice_select_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={'X': x}, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, inputs={'X': x}, outputs={'Y': out}, attrs=attrs + ) return out @@ -360,22 +385,23 @@ def slice_assign(x, y, axis, starts, ends, strides, out=None): assert len(starts) == len(ends) == len(strides) == len(axis), ( f'len(starts), len(ends), len(strides) and len(axis) should be equal, ' f'but len(starts)={len(starts)}, len(ends)={len(ends)}, ' - f'len(strides)={len(strides)} and len(axis)={len(axis)}') + f'len(strides)={len(strides)} and len(axis)={len(axis)}' + ) assert len(y.shape) == len(x.shape), ( f'len(y.shape) should be equal to len(x.shape), ' - f'but len(y.shape)={len(y.shape)} and len(x.shape)={len(x.shape)}.') + f'but len(y.shape)={len(y.shape)} and len(x.shape)={len(x.shape)}.' + ) attrs = {'axis': axis, 'starts': starts, 'ends': ends, 'strides': strides} helper = LayerHelper('slice_assign_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Z': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'X': x, 'Y': y}, + outputs={'Z': out}, + attrs=attrs, + ) return out @@ -385,13 +411,12 @@ def gather(x, indextensor, axis, out=None): helper = LayerHelper('gather_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={ - 'X': x, - 'IndexTensor': indextensor - }, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'X': x, 'IndexTensor': indextensor}, + outputs={'Y': out}, + attrs=attrs, + ) return out @@ -399,26 +424,26 @@ def gather(x, indextensor, axis, out=None): def scatter_add(x, y, indextensor, axis, out=None): assert len(x.shape) == len(y.shape), ( f'len(x.shape) should be equal to len(y.shape), ' - f'but len(x.shape)={len(x.shape)} and len(y.shape)={len(y.shape)}.') - assert len( - indextensor.shape - ) == 1, f'len(indextensor.shape) must be equal to 1, but got {len(indextensor.shape)}.' + f'but len(x.shape)={len(x.shape)} and len(y.shape)={len(y.shape)}.' + ) + assert ( + len(indextensor.shape) == 1 + ), f'len(indextensor.shape) must be equal to 1, but got {len(indextensor.shape)}.' assert y.shape[axis] == indextensor.shape[0], ( f'y.shape[axis] should be equal to indextensor.shape[0], ' f'but y.shape[axis]={y.shape[axis]} and ' - f'indextensor.shape[0]={indextensor.shape[0]}.') + f'indextensor.shape[0]={indextensor.shape[0]}.' + ) attrs = {'axis': axis} helper = LayerHelper('scatter_add_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={ - 'X': x, - 'Y': y, - 'IndexTensor': indextensor - }, - outputs={'Z': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'X': x, 'Y': y, 'IndexTensor': indextensor}, + outputs={'Z': out}, + attrs=attrs, + ) return out @@ -431,24 +456,26 @@ def log(x, out=None): def select(cond, x, y, out=None): if len(cond.shape) != len(x.shape): raise ValueError( - "len(cond.shape) should be equal to len(x.shape), but len(cond.shape)={} and len(x.shape)={}." - .format(len(cond.shape), len(x.shape))) + "len(cond.shape) should be equal to len(x.shape), but len(cond.shape)={} and len(x.shape)={}.".format( + len(cond.shape), len(x.shape) + ) + ) if len(x.shape) != len(y.shape): raise ValueError( - "len(x.shape) should be equal to len(y.shape), but len(x.shape)={} and len(y.shape)={}." - .format(len(x.shape), len(y.shape))) + "len(x.shape) should be equal to len(y.shape), but len(x.shape)={} and len(y.shape)={}.".format( + len(x.shape), len(y.shape) + ) + ) helper = LayerHelper('select_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={ - 'Condition': cond, - 'X': x, - 'Y': y - }, - outputs={'Z': out}) + helper.append_op( + type=helper.layer_type, + inputs={'Condition': cond, 'X': x, 'Y': y}, + outputs={'Z': out}, + ) return out @@ -492,10 +519,12 @@ def cast(x, dtype, out=None): helper = LayerHelper('cast_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=helper.layer_type, - inputs={'X': x}, - outputs={'Y': out}, - attrs={'dtype': dtype}) + helper.append_op( + type=helper.layer_type, + inputs={'X': x}, + outputs={'Y': out}, + attrs={'dtype': dtype}, + ) return out @@ -511,7 +540,7 @@ def uniform_random(dtype, min_value, max_value, seed, shape=None, out=None): 'dtype': dtype, 'min': min_value, 'max': max_value, - 'seed': seed + 'seed': seed, } helper = LayerHelper('uniform_random_p', **locals()) if out is None: diff --git a/python/paddle/incubate/autograd/primreg.py b/python/paddle/incubate/autograd/primreg.py index 31f47f81963..7d81847f905 100644 --- a/python/paddle/incubate/autograd/primreg.py +++ b/python/paddle/incubate/autograd/primreg.py @@ -14,7 +14,8 @@ class Registry(object): - """ A general registry object. """ + """A general registry object.""" + __slots__ = ['name', 'tab'] def __init__(self, name): @@ -22,7 +23,9 @@ class Registry(object): self.tab = {} def register(self, name, value): - assert name not in self.tab, f'name "{name}" should not be registered before.' + assert ( + name not in self.tab + ), f'name "{name}" should not be registered before.' self.tab[name] = value def lookup(self, name): @@ -78,15 +81,17 @@ def op_position_inputs(op): """ args = _primop_position_argnames.lookup(op.type) - assert args is not None, f'args of {op.type} should not be None in op_position_inputs().' + assert ( + args is not None + ), f'args of {op.type} should not be None in op_position_inputs().' *input_names, _ = args inputs = [] for name in input_names: vars = list(map(op.block.var, op.input(name))) - assert len( - vars - ) >= 0, f'len(vars) should be greater than or equal to 0, but len(vars)={len(vars)}.' + assert ( + len(vars) >= 0 + ), f'len(vars) should be greater than or equal to 0, but len(vars)={len(vars)}.' if len(vars) > 1: inputs.append(vars) else: @@ -120,9 +125,9 @@ def op_position_output(op): *_, output_name = args outvars = list(map(op.block.var, op.output(output_name))) - assert len( - outvars - ) >= 0, f'len(outvars) should be greater than or equal to 0, but len(outvars)={len(outvars)}.' + assert ( + len(outvars) >= 0 + ), f'len(outvars) should be greater than or equal to 0, but len(outvars)={len(outvars)}.' if len(outvars) > 1: output = outvars else: @@ -184,9 +189,10 @@ def REGISTER_ORIG2PRIM(op_type): raise TypeError(f'op_type must be str, but got {type(op_type)}.') def wrapper(f): - def _lower(op, *args, **kwargs): - assert op.type == op_type, f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' + assert ( + op.type == op_type + ), f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' return f(op, *args, **kwargs) _orig2prim.register(op_type, _lower) @@ -216,9 +222,10 @@ def REGISTER_PRIM2ORIG(op_type): raise TypeError(f'op_type must be str, but got {type(op_type)}.') def wrapper(f): - def _lower(op, *args, **kwargs): - assert op.type == op_type, f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' + assert ( + op.type == op_type + ), f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' return f(op, *args, **kwargs) _prim2orig.register(op_type, _lower) @@ -247,9 +254,10 @@ def REGISTER_JVP(op_type): raise TypeError(f'op_type must be str, but got {type(op_type)}.') def wrapper(f): - def _jvp(op, *args, **kwargs): - assert op.type == op_type, f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' + assert ( + op.type == op_type + ), f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' return f(op, *args, **kwargs) _primop_jvp.register(op_type, _jvp) @@ -280,9 +288,10 @@ def REGISTER_TRANSPOSE(op_type): raise TypeError(f'op_type must be str, but got {type(op_type)}.') def wrapper(f): - def _transpose(op, dot_checker, *args, **kwargs): - assert op.type == op_type, f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' + assert ( + op.type == op_type + ), f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' return f(op, dot_checker, *args, **kwargs) _primop_transpose.register(op_type, _transpose) diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 8a2f94145cd..1d98d62cef3 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -19,15 +19,54 @@ import typing import paddle from . import primops -from .primops import (add, broadcast, concat, cos, div, eq, erf, exp, - fill_const, gather, ge, gt, log, matmul, mul, ne, neg, - reduce_sum, reshape, scatter_add, select, set_value, sin, - slice_assign, slice_select, split, sqrt, sub, tanh, - transpose, bernoulli, rsqrt, uniform_random) -from .primreg import (REGISTER_JVP, REGISTER_ORIG2PRIM, REGISTER_PRIM2ORIG, - REGISTER_TRANSPOSE, lookup_fn, lookup_jvp, - lookup_orig2prim, lookup_prim2orig, lookup_transpose, - op_position_inputs, op_position_output) +from .primops import ( + add, + broadcast, + concat, + cos, + div, + eq, + erf, + exp, + fill_const, + gather, + ge, + gt, + log, + matmul, + mul, + ne, + neg, + reduce_sum, + reshape, + scatter_add, + select, + set_value, + sin, + slice_assign, + slice_select, + split, + sqrt, + sub, + tanh, + transpose, + bernoulli, + rsqrt, + uniform_random, +) +from .primreg import ( + REGISTER_JVP, + REGISTER_ORIG2PRIM, + REGISTER_PRIM2ORIG, + REGISTER_TRANSPOSE, + lookup_fn, + lookup_jvp, + lookup_orig2prim, + lookup_prim2orig, + lookup_transpose, + op_position_inputs, + op_position_output, +) from .utils import INT_DTYPE_2_STRING, get_output_var_list @@ -94,20 +133,20 @@ def elementwise_add_orig2prim(op, x, y): if x.shape != y.shape: y = broadcast(y, shape=x.shape) if op.attr('Scale_x') - 1.0 > 1e-5: - scale_x = fill_const(shape=x.shape, - dtype=x.dtype, - value=op.attr('Scale_x')) + scale_x = fill_const( + shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x') + ) x = mul(x, scale_x) if op.attr('Scale_y') - 1.0 > 1e-5: - scale_y = fill_const(shape=y.shape, - dtype=y.dtype, - value=op.attr('Scale_y')) + scale_y = fill_const( + shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y') + ) y = mul(y, scale_y) z = add(x, y) if op.attr('Scale_out') - 1.0 > 1e-5: - scale_out = fill_const(shape=z.shape, - dtype=z.dtype, - value=op.attr('Scale_out')) + scale_out = fill_const( + shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out') + ) z = mul(z, scale_out) return z @@ -117,20 +156,20 @@ def elementwise_sub_orig2prim(op, x, y): if x.shape != y.shape: y = broadcast(y, shape=x.shape) if op.attr('Scale_x') - 1.0 > 1e-5: - scale_x = fill_const(shape=x.shape, - dtype=x.dtype, - value=op.attr('Scale_x')) + scale_x = fill_const( + shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x') + ) x = mul(x, scale_x) if op.attr('Scale_y') - 1.0 > 1e-5: - scale_y = fill_const(shape=y.shape, - dtype=y.dtype, - value=op.attr('Scale_y')) + scale_y = fill_const( + shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y') + ) y = mul(y, scale_y) z = sub(x, y) if op.attr('Scale_out') - 1.0 > 1e-5: - scale_out = fill_const(shape=z.shape, - dtype=z.dtype, - value=op.attr('Scale_out')) + scale_out = fill_const( + shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out') + ) z = mul(z, scale_out) return z @@ -140,20 +179,20 @@ def elementwise_mul_orig2prim(op, x, y): if x.shape != y.shape: y = broadcast(y, shape=x.shape) if op.attr('Scale_x') - 1.0 > 1e-5: - scale_x = fill_const(shape=x.shape, - dtype=x.dtype, - value=op.attr('Scale_x')) + scale_x = fill_const( + shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x') + ) x = mul(x, scale_x) if op.attr('Scale_y') - 1.0 > 1e-5: - scale_y = fill_const(shape=y.shape, - dtype=y.dtype, - value=op.attr('Scale_y')) + scale_y = fill_const( + shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y') + ) y = mul(y, scale_y) z = mul(x, y) if op.attr('Scale_out') - 1.0 > 1e-5: - scale_out = fill_const(shape=z.shape, - dtype=z.dtype, - value=op.attr('Scale_out')) + scale_out = fill_const( + shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out') + ) z = mul(z, scale_out) return z @@ -209,23 +248,26 @@ def fill_zeros_like_orig2prim(op, x): def fill_any_like_orig2prim(op, x): if op.attr('dtype') == -1: return fill_const(value=op.attr('value'), shape=x.shape, dtype=x.dtype) - return fill_const(value=op.attr('value'), - shape=x.shape, - dtype=paddle.dtype(op.attr('dtype'))) + return fill_const( + value=op.attr('value'), + shape=x.shape, + dtype=paddle.dtype(op.attr('dtype')), + ) @REGISTER_ORIG2PRIM('fill_constant') -def fill_const_orig2prim(op, - shape_tensor=None, - shape_tensor_list=None, - value_tensor=None): +def fill_const_orig2prim( + op, shape_tensor=None, shape_tensor_list=None, value_tensor=None +): if shape_tensor or shape_tensor_list or value_tensor: raise TypeError( 'fill_const_orig2prim currently not support Tensor input of shape and value.' ) - return fill_const(value=op.attr('value'), - shape=op.attr('shape'), - dtype=paddle.dtype(op.attr('dtype'))) + return fill_const( + value=op.attr('value'), + shape=op.attr('shape'), + dtype=paddle.dtype(op.attr('dtype')), + ) @REGISTER_ORIG2PRIM('sum') @@ -244,9 +286,9 @@ def index_select_orig2prim(op, index_t, x): @REGISTER_ORIG2PRIM('scale') def scale_orig2prim(op, scale_t, x): if scale_t is None: - scale_t = fill_const(shape=x.shape, - dtype=x.dtype, - value=op.attr('scale')) + scale_t = fill_const( + shape=x.shape, dtype=x.dtype, value=op.attr('scale') + ) bias_t = fill_const(shape=x.shape, dtype=x.dtype, value=op.attr('bias')) if op.attr('bias_after_scale'): return add(mul(x, scale_t), bias_t) @@ -272,14 +314,14 @@ def rsqrt_orig2prim(op, x): @REGISTER_ORIG2PRIM('matmul_v2') def matmul_v2_orig2prim(op, x, y): - def trans(shape): ret = [i for i in range(len(shape))] ret[-1], ret[-2] = ret[-2], ret[-1] return ret - assert len(x.shape) < 4 and len( - y.shape) < 4, 'Do not support multi batchsize dimensions currently.' + assert ( + len(x.shape) < 4 and len(y.shape) < 4 + ), 'Do not support multi batchsize dimensions currently.' if len(x.shape) == 1: x = broadcast(x, shape=[1, x.shape[0]]) @@ -295,12 +337,16 @@ def matmul_v2_orig2prim(op, x, y): ## NOTE(lml): The second output of reshape2 Xshape, which is only used in reshape2_grad, is meanlingless in new autograd mechanism, thus we use a zero tensor instead. @REGISTER_ORIG2PRIM('reshape2') def reshape2_orig2prim(op, shape_t, shape_tl, x): - assert shape_t is None, 'Can not lower reshape2 into prim ops with shapetensor.' - assert shape_tl is None, 'Can not lower reshape2 into prim ops with shapetensorlist.' + assert ( + shape_t is None + ), 'Can not lower reshape2 into prim ops with shapetensor.' + assert ( + shape_tl is None + ), 'Can not lower reshape2 into prim ops with shapetensorlist.' y, xshape = get_output_var_list(op) - return reshape(x, shape=y.shape), fill_const(shape=xshape.shape, - dtype=xshape.dtype, - value=0.0) + return reshape(x, shape=y.shape), fill_const( + shape=xshape.shape, dtype=xshape.dtype, value=0.0 + ) @REGISTER_ORIG2PRIM('concat') @@ -311,10 +357,16 @@ def concat_orig2prim(op, axis_t, xs): @REGISTER_ORIG2PRIM('slice') def slice_orig2prim(op, ends_t, ends_tl, x, starts_t, starts_tl): - assert starts_t is None, 'Can not lower concat into prim ops with startstensor.' + assert ( + starts_t is None + ), 'Can not lower concat into prim ops with startstensor.' assert ends_t is None, 'Can not lower concat into prim ops with endstensor.' - assert starts_tl is None, 'Can not lower concat into prim ops with startstensorlist.' - assert ends_tl is None, 'Can not lower concat into prim ops with endstensorlist.' + assert ( + starts_tl is None + ), 'Can not lower concat into prim ops with startstensorlist.' + assert ( + ends_tl is None + ), 'Can not lower concat into prim ops with endstensorlist.' starts = op.attr('starts') ends = op.attr('ends') strides = [1 for _ in starts] @@ -329,12 +381,12 @@ def slice_orig2prim(op, ends_t, ends_tl, x, starts_t, starts_tl): def sigmoid_orig2prim(op, x): return div( fill_const(value=1.0, shape=x.shape, dtype=x.dtype), - (add(fill_const(value=1.0, shape=x.shape, dtype=x.dtype), exp(neg(x))))) + (add(fill_const(value=1.0, shape=x.shape, dtype=x.dtype), exp(neg(x)))), + ) @REGISTER_ORIG2PRIM('p_norm') def p_norm_orig2prim(op, x): - def num_el(shape): n = 1 for s in shape: @@ -342,7 +394,8 @@ def p_norm_orig2prim(op, x): return n assert op.attr( - 'asvector'), 'Only support lower pnorm when asvector=True currently' + 'asvector' + ), 'Only support lower pnorm when asvector=True currently' if len(x.shape) > 1: x = reshape(x, shape=[num_el(x.shape)]) @@ -435,25 +488,38 @@ def gelu_orig2prim(op, x): x, mul( fill_const(0.044715, x.shape, x.dtype), - primops.pow(x, fill_const(3., x.shape, - x.dtype)))))))) + primops.pow( + x, fill_const(3.0, x.shape, x.dtype) + ), + ), + ), + ) + ), + ), + ) return mul(x, cdf) else: return mul( mul(fill_const(0.5, x.shape, x.dtype), x), - add(fill_const(1.0, x.shape, x.dtype), - erf(mul(x, fill_const(1 / math.sqrt(2.), x.shape, x.dtype))))) + add( + fill_const(1.0, x.shape, x.dtype), + erf(mul(x, fill_const(1 / math.sqrt(2.0), x.shape, x.dtype))), + ), + ) @REGISTER_ORIG2PRIM('dropout') def dropout_orig2prim(op, seed_t, x): - assert seed_t is None, 'Can not lower dropout into prim ops with seedtensor.' + assert ( + seed_t is None + ), 'Can not lower dropout into prim ops with seedtensor.' mask = bernoulli(shape=x.shape, dtype=x.dtype, p=op.attr('dropout_prob')) if op.attr('dropout_implementation') == 'upscale_in_train': if op.attr('is_test') == False: out = div( mul(x, mask), - fill_const(1.0 - op.attr('dropout_prob'), x.shape, x.dtype)) + fill_const(1.0 - op.attr('dropout_prob'), x.shape, x.dtype), + ) return primops.cast(mask, dtype=paddle.uint8), out else: return primops.cast(mask, dtype=paddle.uint8), x @@ -462,7 +528,8 @@ def dropout_orig2prim(op, seed_t, x): return primops.cast(mask, dtype=paddle.uint8), mul(x, mask) else: return primops.cast(mask, dtype=paddle.uint8), mul( - x, fill_const(1.0 - op.attr('dropout_prob'), x.shape, x.dtype)) + x, fill_const(1.0 - op.attr('dropout_prob'), x.shape, x.dtype) + ) else: raise RuntimeError( 'Unsupported dropout_implementation, only support upscale_in_train and downgrade_in_infer' @@ -485,50 +552,62 @@ def uniform_random_orig2prim(op, shape_t, shape_tl): @REGISTER_ORIG2PRIM('reduce_sum') def reduce_sum_orig2prim(op, x): - axes = tuple(range(0, len( - x.shape))) if op.attr('reduce_all') else op.attr('dim') + axes = ( + tuple(range(0, len(x.shape))) + if op.attr('reduce_all') + else op.attr('dim') + ) return reduce_sum(x, axis=axes, keepdim=op.attr('keep_dim')) @REGISTER_ORIG2PRIM('reduce_mean') def reduce_mean_orig2prim(op, x): - axes = tuple(range(0, len( - x.shape))) if op.attr('reduce_all') else op.attr('dim') + axes = ( + tuple(range(0, len(x.shape))) + if op.attr('reduce_all') + else op.attr('dim') + ) return primops.mean(x, axes, op.attr('keep_dim')) @REGISTER_ORIG2PRIM('batch_norm') -def batch_norm_orig2prim(op, bias, run_mean, momentum_tensor, scale, run_var, - x): +def batch_norm_orig2prim( + op, bias, run_mean, momentum_tensor, scale, run_var, x +): momentum = op.attr('momentum') eps = op.attr('epsilon') is_test = op.attr('is_test') data_layout = op.attr('data_layout') use_global_stats = op.attr('use_global_stats') trainable_statistics = op.attr('trainable_statistics') - reserve_space = None if len( - op.output_names) == 5 else get_output_var_list(op)[1] + reserve_space = ( + None if len(op.output_names) == 5 else get_output_var_list(op)[1] + ) - feature_axis = 1 if data_layout in ('NC', 'NCL', 'NCHW', - 'NCHWD') else len(x.shape) - 1 + feature_axis = ( + 1 if data_layout in ('NC', 'NCL', 'NCHW', 'NCHWD') else len(x.shape) - 1 + ) use_run_stat = (is_test and (not trainable_statistics)) or use_global_stats - return primops.batch_norm(x, - feature_axis, - scale, - bias, - run_mean, - run_var, - eps=eps, - momentum=momentum, - use_run_stat=use_run_stat, - reserve_space=reserve_space) + return primops.batch_norm( + x, + feature_axis, + scale, + bias, + run_mean, + run_var, + eps=eps, + momentum=momentum, + use_run_stat=use_run_stat, + reserve_space=reserve_space, + ) @REGISTER_ORIG2PRIM('size') def size_orig2prim(op, x): - return fill_const(functools.reduce(operator.mul, x.shape), (1, ), - paddle.int64) + return fill_const( + functools.reduce(operator.mul, x.shape), (1,), paddle.int64 + ) ## Register prim2orig lower rules @@ -617,9 +696,9 @@ def split_prim2orig(op, x): num_or_sections = op.attr('num_or_sections') if len(num_or_sections) == 1: num_or_sections = num_or_sections[0] - return paddle.split(x, - num_or_sections=num_or_sections, - axis=op.attr('axis')) + return paddle.split( + x, num_or_sections=num_or_sections, axis=op.attr('axis') + ) @REGISTER_PRIM2ORIG('concat_p') @@ -639,23 +718,27 @@ def matmul_prim2orig(op, x, y): @REGISTER_PRIM2ORIG('slice_select_p') def slice_select_prim2orig(op, x): - return paddle.strided_slice(x, - axes=op.attr('axis'), - starts=op.attr('starts'), - ends=op.attr('ends'), - strides=op.attr('strides')) + return paddle.strided_slice( + x, + axes=op.attr('axis'), + starts=op.attr('starts'), + ends=op.attr('ends'), + strides=op.attr('strides'), + ) @REGISTER_PRIM2ORIG('slice_assign_p') def slice_assign_prim2orig(op, x, y): x_copy = paddle.assign(x) - return set_value(x_copy, - y, - axis=op.attr('axis'), - starts=op.attr('starts'), - ends=op.attr('ends'), - strides=op.attr('strides'), - out=x_copy) + return set_value( + x_copy, + y, + axis=op.attr('axis'), + starts=op.attr('starts'), + ends=op.attr('ends'), + strides=op.attr('strides'), + out=x_copy, + ) @REGISTER_PRIM2ORIG('gather_p') @@ -673,26 +756,32 @@ def scatter_add_prim2orig(op, index_t, x, y): @REGISTER_PRIM2ORIG('fill_constant_p') def fill_constant_prim2orig(op): - return paddle.full(shape=op.attr('shape'), - fill_value=op.attr('value'), - dtype=INT_DTYPE_2_STRING[op.attr('dtype')]) + return paddle.full( + shape=op.attr('shape'), + fill_value=op.attr('value'), + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + ) @REGISTER_PRIM2ORIG('bernoulli_p') def bernoulli_prim2orig(op): - t = paddle.full(shape=op.attr('shape'), - fill_value=op.attr('p'), - dtype=INT_DTYPE_2_STRING[op.attr('dtype')]) + t = paddle.full( + shape=op.attr('shape'), + fill_value=op.attr('p'), + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + ) return paddle.bernoulli(t) @REGISTER_PRIM2ORIG('uniform_random_p') def uniform_random_prim2orig(op): - return paddle.uniform(shape=op.attr('shape'), - dtype=INT_DTYPE_2_STRING[op.attr('dtype')], - min=op.attr('min'), - max=op.attr('max'), - seed=op.attr('seed')) + return paddle.uniform( + shape=op.attr('shape'), + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + min=op.attr('min'), + max=op.attr('max'), + seed=op.attr('seed'), + ) @REGISTER_PRIM2ORIG('select_p') @@ -810,7 +899,7 @@ def tanh_jvp(op, x_dot): def sin_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return mul(x_dot, cos(x)) @@ -818,7 +907,7 @@ def sin_jvp(op, x_dot): def cos_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return mul(x_dot, neg(sin(x))) @@ -834,25 +923,26 @@ def exp_jvp(op, x_dot): def erf_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return mul( - fill_const(2. / math.sqrt(math.pi), x.shape, x.dtype), - mul(x_dot, exp(neg(primops.pow(x, fill_const(2., x.shape, x.dtype)))))) + fill_const(2.0 / math.sqrt(math.pi), x.shape, x.dtype), + mul(x_dot, exp(neg(primops.pow(x, fill_const(2.0, x.shape, x.dtype))))), + ) @REGISTER_JVP('abs_p') def abs_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) - return select(ge(x, fill_const(0., x.shape, x.dtype)), x_dot, neg(x_dot)) + (x,) = op_position_inputs(op) + return select(ge(x, fill_const(0.0, x.shape, x.dtype)), x_dot, neg(x_dot)) @REGISTER_JVP('log_p') def log_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return div(x_dot, x) @@ -929,12 +1019,9 @@ def slice_select_jvp(op, x_dot): starts = op.attr('starts') ends = op.attr('ends') strides = op.attr('strides') - return linear_jvp(op, - x_dot, - axis=axis, - starts=starts, - ends=ends, - strides=strides) + return linear_jvp( + op, x_dot, axis=axis, starts=starts, ends=ends, strides=strides + ) @REGISTER_JVP('slice_assign_p') @@ -948,13 +1035,9 @@ def slice_assign_jvp(op, x_dot, y_dot): starts = op.attr('starts') ends = op.attr('ends') strides = op.attr('strides') - return linear_jvp(op, - x_dot, - y_dot, - axis=axis, - starts=starts, - ends=ends, - strides=strides) + return linear_jvp( + op, x_dot, y_dot, axis=axis, starts=starts, ends=ends, strides=strides + ) @REGISTER_JVP('gather_p') @@ -993,7 +1076,7 @@ def eq_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: return None x, _ = op_position_inputs(op) - z_dot = fill_const(value=0., shape=x.shape, dtype=x.dtype) + z_dot = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) return z_dot @@ -1002,7 +1085,7 @@ def gt_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: return None x, _ = op_position_inputs(op) - z_dot = fill_const(value=0., shape=x.shape, dtype=x.dtype) + z_dot = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) return z_dot @@ -1011,7 +1094,7 @@ def ge_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: return None x, _ = op_position_inputs(op) - z_dot = fill_const(value=0., shape=x.shape, dtype=x.dtype) + z_dot = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) return z_dot @@ -1020,13 +1103,12 @@ def ne_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: return None x, _ = op_position_inputs(op) - z_dot = fill_const(value=0., shape=x.shape, dtype=x.dtype) + z_dot = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) return z_dot @REGISTER_JVP('pow_p') def pow_jvp(op, x_dot, y_dot): - def _compute_t1(x, y): zero_y = fill_const(value=0.0, shape=y.shape, dtype=y.dtype) one_y = fill_const(value=1.0, shape=y.shape, dtype=y.dtype) @@ -1096,7 +1178,8 @@ def add_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) or check_dot(y), ( f'(check_dot(x) or check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) x_bar = z_bar if check_dot(x) else None y_bar = z_bar if check_dot(y) else None return x_bar, y_bar @@ -1107,7 +1190,8 @@ def sub_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) or check_dot(y), ( f'(check_dot(x) or check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) x_bar = z_bar if check_dot(x) else None y_bar = neg(z_bar) if check_dot(y) else None return x_bar, y_bar @@ -1118,7 +1202,8 @@ def mul_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) ^ check_dot(y), ( f'(check_dot(x) ^ check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) if check_dot(x): return mul(z_bar, y), None else: @@ -1135,14 +1220,14 @@ def div_transpose(op, check_dot, z_bar): @REGISTER_TRANSPOSE('reshape_p') def reshape_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' return reshape(y_bar, shape=x.shape) @REGISTER_TRANSPOSE('broadcast_p') def broadcast_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' bat = len(y_bar.shape) - len(x.shape) axis = list(range(bat)) @@ -1155,7 +1240,7 @@ def broadcast_transpose(op, check_dot, y_bar): @REGISTER_TRANSPOSE('transpose_p') def transpose_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' axis = op.attr('axis') reordered = sorted((k, i) for i, k in enumerate(axis)) @@ -1165,14 +1250,14 @@ def transpose_transpose(op, check_dot, y_bar): @REGISTER_TRANSPOSE('split_p') def split_transpose(op, check_dot, ys_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' return concat(ys_bar, axis=op.attr('axis')) @REGISTER_TRANSPOSE('concat_p') def concat_transpose(op, check_dot, y_bar): - xs, = op_position_inputs(op) + (xs,) = op_position_inputs(op) if not isinstance(xs, typing.Sequence): xs = [xs] for x in xs: @@ -1186,7 +1271,7 @@ def concat_transpose(op, check_dot, y_bar): @REGISTER_TRANSPOSE('reduce_sum_p') def reduce_sum_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' axes = op.attr('axis') shape = tuple(1 if i in axes else size for i, size in enumerate(x.shape)) @@ -1199,7 +1284,8 @@ def matmul_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) ^ check_dot(y), ( f'(check_dot(x) ^ check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) # TODO: replace it. this is hacky axis = [1, 0] if len(x.shape) == 2 else [0, 2, 1] if check_dot(x): @@ -1210,19 +1296,16 @@ def matmul_transpose(op, check_dot, z_bar): @REGISTER_TRANSPOSE('slice_select_p') def slice_select_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' zeros = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) axis = op.attr('axis') starts = op.attr('starts') ends = op.attr('ends') strides = op.attr('strides') - return slice_assign(zeros, - y_bar, - axis=axis, - starts=starts, - ends=ends, - strides=strides) + return slice_assign( + zeros, y_bar, axis=axis, starts=starts, ends=ends, strides=strides + ) @REGISTER_TRANSPOSE('slice_assign_p') @@ -1230,23 +1313,19 @@ def slice_assign_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) and check_dot(y), ( f'(check_dot(x) and check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) zeros = fill_const(value=0.0, shape=y.shape, dtype=y.dtype) axis = op.attr('axis') starts = op.attr('starts') ends = op.attr('ends') strides = op.attr('strides') - x_bar = slice_assign(z_bar, - zeros, - axis=axis, - starts=starts, - ends=ends, - strides=strides) - y_bar = slice_select(z_bar, - axis=axis, - starts=starts, - ends=ends, - strides=strides) + x_bar = slice_assign( + z_bar, zeros, axis=axis, starts=starts, ends=ends, strides=strides + ) + y_bar = slice_select( + z_bar, axis=axis, starts=starts, ends=ends, strides=strides + ) return x_bar, y_bar @@ -1266,7 +1345,8 @@ def scatter_add_transpose(op, check_dot, z_bar): x, y, indextensor = op_position_inputs(op) assert check_dot(x) and check_dot(y), ( f'(check_dot(x) and check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) axis = op.attr('axis') zeros = fill_const(value=0.0, shape=y.shape, dtype=y.dtype) x_bar = scatter_add(z_bar, zeros, indextensor, axis=axis) @@ -1286,8 +1366,11 @@ def select_transpose(op, check_dot, z_bar): zeros_x = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) zeros_y = fill_const(value=0.0, shape=y.shape, dtype=y.dtype) - cond_bar = fill_const(value=0.0, shape=y.shape, - dtype=cond.dtype) if check_dot(cond) else None + cond_bar = ( + fill_const(value=0.0, shape=y.shape, dtype=cond.dtype) + if check_dot(cond) + else None + ) x_bar = select(cond, z_bar, zeros_x) if check_dot(x) else None y_bar = select(cond, zeros_y, z_bar) if check_dot(y) else None @@ -1296,5 +1379,5 @@ def select_transpose(op, check_dot, z_bar): @REGISTER_TRANSPOSE('cast_p') def cast_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return primops.cast(y_bar, x.dtype) diff --git a/python/paddle/incubate/autograd/primx.py b/python/paddle/incubate/autograd/primx.py index c57f651d344..cdf16e77b1f 100644 --- a/python/paddle/incubate/autograd/primx.py +++ b/python/paddle/incubate/autograd/primx.py @@ -20,15 +20,23 @@ from paddle.fluid.framework import Operator, default_main_program from paddle.incubate.autograd.utils import as_tensors from .primops import add, fill_const -from .primreg import (lookup_orig2prim, lookup_prim2orig, op_position_inputs, - op_position_output) +from .primreg import ( + lookup_orig2prim, + lookup_prim2orig, + op_position_inputs, + op_position_output, +) from .primrules import _jvp, _orig2prim, _prim2orig, _transpose -from .utils import (flatten, flatten_and_remove_none, get_input_var_list, - get_output_var_list) +from .utils import ( + flatten, + flatten_and_remove_none, + get_input_var_list, + get_output_var_list, +) def topo_path(xs, ys, block=None): - """ Returns the list of ops on the path from `xs` to `ys` in topological + """Returns the list of ops on the path from `xs` to `ys` in topological order. TODO(Tongxin): supporting control flow and nested blocks. @@ -50,13 +58,16 @@ def topo_path(xs, ys, block=None): # Initialize reached vars for x in xs: - assert x is None or x.block == block, 'x is not None and x.block != block' + assert ( + x is None or x.block == block + ), 'x is not None and x.block != block' reached_vars[id(x)] = x # Reaching test, returning whether an op is reached from the given input reaching = lambda op: any( id(v) in reached_vars - for v in flatten_and_remove_none(get_input_var_list(op))) + for v in flatten_and_remove_none(get_input_var_list(op)) + ) # block.ops are supposedly in the order that preserves correct data # dependence. @@ -70,7 +81,8 @@ def topo_path(xs, ys, block=None): used_vars = OrderedDict((id(y), y) for y in ys if id(y) in reached_vars) back_reaching = lambda op: any( id(out) in used_vars - for out in flatten_and_remove_none(get_output_var_list(op))) + for out in flatten_and_remove_none(get_output_var_list(op)) + ) # Backward pass to find all used variables for op in reversed(path): @@ -86,7 +98,7 @@ def topo_path(xs, ys, block=None): def output_vars_on_path(path): - """ Returns the output variables of all the ops on the path from `xs` + """Returns the output variables of all the ops on the path from `xs` to `ys`. Args: @@ -104,7 +116,7 @@ def output_vars_on_path(path): class VarMap(object): - """ A general map data structure for linking variables to variables. + """A general map data structure for linking variables to variables. An example is linking variables to their gradients. """ @@ -125,7 +137,8 @@ class VarMap(object): if isinstance(key_vars, paddle.fluid.framework.Variable): if not isinstance(value_vars, paddle.fluid.framework.Variable): raise TypeError( - f'value_vars must be Variable, but got {type(value_vars)}') + f'value_vars must be Variable, but got {type(value_vars)}' + ) self.tab[id(key_vars)] = id(value_vars) else: assert len(key_vars) == len(value_vars), ( @@ -168,11 +181,12 @@ class VarMap(object): # TODO(lml): supporting control flow, nested blocks, and block other than current block of main program. class Transform(object): - """ An object that maintains the state of transformations applied to a - primitve program. """ + """An object that maintains the state of transformations applied to a + primitve program.""" def __init__(self, block): - assert block == default_main_program().current_block( + assert ( + block == default_main_program().current_block() ), 'only support transform on current block of main program.' self.block = block self.vars = self.init_vars(block) @@ -224,7 +238,7 @@ class Transform(object): block._sync_with_cpp() def var2dot_rec(self, vars): - """ Lookup var2dot recursively.""" + """Lookup var2dot recursively.""" if isinstance(vars, paddle.fluid.framework.Variable): dot = self.var2dot.lookup(vars) return dot @@ -243,7 +257,7 @@ class Transform(object): return bars def linearize(self, xs, ys, xs_dot=None): - """ Performs the linearization transform, a.k.a, forward mode AD + """Performs the linearization transform, a.k.a, forward mode AD transform, on a primitive lowered program. Args: @@ -265,15 +279,18 @@ class Transform(object): else: assert len(xs) == len(xs_dot), ( f'len(xs) should be equal to len(xs_dot), ' - f'but len(xs)={len(xs)} and len(xs_dot)={len(xs_dot)}') + f'but len(xs)={len(xs)} and len(xs_dot)={len(xs_dot)}' + ) for x, dot in zip(xs, xs_dot): assert x.dtype == dot.dtype, ( f'x.dtype should be equal to dot.dtype, ' - f'but x.dtype={x.dtype} and dot.dtype={dot.dtype}') + f'but x.dtype={x.dtype} and dot.dtype={dot.dtype}' + ) assert x.shape == dot.shape, ( f'x.shape should be equal to dot.shape, ' - f'but x.shape={x.shape} and dot.shape={dot.shape}') + f'but x.shape={x.shape} and dot.shape={dot.shape}' + ) self.var2dot.add(x, dot) path, unused_xs, _ = topo_path(xs, ys, self.block) @@ -299,7 +316,7 @@ class Transform(object): return xs_dot, ys_dot def transpose(self, ys_dot, xs_dot, ys_bar=None, retain_fwd=False): - """ Performs the transpose transform, a.k.a, reverse mode AD + """Performs the transpose transform, a.k.a, reverse mode AD transform, on a linearized primitive program. Note, `transpose` is supposed to be used in couple with `linearize`. @@ -328,7 +345,8 @@ class Transform(object): else: assert len(ys_dot) == len(ys_bar), ( f'len(ys_dot) should be equal to len(ys_bar), ' - f'but len(ys_dot)={len(ys_dot)} and len(ys_bar)={len(ys_bar)}') + f'but len(ys_dot)={len(ys_dot)} and len(ys_bar)={len(ys_bar)}' + ) for y_dot, y_bar in zip(ys_dot, ys_bar): assert y_dot.shape == y_bar.shape, ( f'y_dot.shape should be equal to y_bar.shape, ' @@ -372,7 +390,8 @@ class Transform(object): ins = flatten(op_position_inputs(op)) assert len(ins) == len(ins_bar), ( f'len(ins) should be equal to len(ins_bar), ' - f'but len(ins)={len(ins)} and len(ins_bar)={len(ins_bar)}') + f'but len(ins)={len(ins)} and len(ins_bar)={len(ins_bar)}' + ) for dot, bar in zip(ins, ins_bar): if bar is not None: @@ -391,7 +410,8 @@ class Transform(object): vars_to_remove = set() for op in path: vars_to_remove.update( - flatten_and_remove_none(get_output_var_list(op))) + flatten_and_remove_none(get_output_var_list(op)) + ) op_indexes = [] @@ -459,10 +479,12 @@ def _lower(block, reverse, blacklist): bind(input_args, to_bind, value_table) for orig_out, new_out in zip( - expand_nested_list(get_output_var_list(op)), - expand_nested_list(as_tensors(lower_fn(op, *input_args)))): + expand_nested_list(get_output_var_list(op)), + expand_nested_list(as_tensors(lower_fn(op, *input_args))), + ): assert not (orig_out is None) ^ ( - new_out is None), "orig_out and new_out should match." + new_out is None + ), "orig_out and new_out should match." vars_to_remove.add(new_out.name) value_table[new_out.name] = new_out to_bind[orig_out.name] = new_out.name @@ -471,7 +493,8 @@ def _lower(block, reverse, blacklist): inputs = {} for i in range(len(op.input_names)): inputs[op.input_names[i]] = bind_name( - op.input(op.input_names[i]), to_bind) + op.input(op.input_names[i]), to_bind + ) outputs = {} for i in range(len(op.output_names)): @@ -481,14 +504,17 @@ def _lower(block, reverse, blacklist): for name in sorted(op.attr_names): attrs[name] = op.attr(name) from paddle.fluid.dygraph.base import param_guard + new_op_desc = block.desc.append_op() with param_guard(inputs), param_guard(outputs): - op = Operator(block=block, - desc=new_op_desc, - type=op.type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + op = Operator( + block=block, + desc=new_op_desc, + type=op.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) block.ops.append(op) # Step3: Do some post-processing work @@ -508,8 +534,9 @@ def _lower(block, reverse, blacklist): op._rename_output(out_name, to_bind_rev[out_name]) for var_name in sorted(vars_to_remove): - assert var_name in to_bind_rev, 'var_name "{}" is not in to_bind_rev.'.format( - var_name) + assert ( + var_name in to_bind_rev + ), 'var_name "{}" is not in to_bind_rev.'.format(var_name) if var_name != to_bind_rev[var_name]: block.desc._remove_var(var_name.encode()) del block.vars[var_name] @@ -535,7 +562,8 @@ def orig2prim(block=None): """ block = default_main_program().current_block() if block is None else block - assert block == default_main_program().current_block( + assert ( + block == default_main_program().current_block() ), 'block is neither None nor current block of main program' _lower(block, reverse=False, blacklist=[]) @@ -580,7 +608,8 @@ def prim2orig(block=None, blacklist=None): """ block = default_main_program().current_block() if block is None else block - assert block == default_main_program().current_block( + assert ( + block == default_main_program().current_block() ), 'block is neither None nor current block of main program' blacklist = [] if blacklist is None else blacklist _lower(block, reverse=True, blacklist=blacklist) diff --git a/python/paddle/incubate/autograd/utils.py b/python/paddle/incubate/autograd/utils.py index effb8c1abc6..2b8082bf48d 100644 --- a/python/paddle/incubate/autograd/utils.py +++ b/python/paddle/incubate/autograd/utils.py @@ -18,7 +18,6 @@ from paddle.fluid import framework as framework class PrimOption(object): - def __init__(self): self.enable_prim = False @@ -175,7 +174,7 @@ def flatten_and_remove_none(inp): def as_tensors(xs): if isinstance(xs, framework.Variable): - return (xs, ) + return (xs,) elif isinstance(xs, typing.Sequence): return tuple(xs) else: diff --git a/python/paddle/incubate/autotune.py b/python/paddle/incubate/autotune.py index 4c577cba3e7..daca4aee14e 100644 --- a/python/paddle/incubate/autotune.py +++ b/python/paddle/incubate/autotune.py @@ -147,7 +147,8 @@ def set_config(config=None): if "tuning_steps" in dataloader_config: if isinstance(dataloader_config['tuning_steps'], int): paddle.fluid.reader.set_autotune_config( - use_autoune, dataloader_config['tuning_steps']) + use_autoune, dataloader_config['tuning_steps'] + ) else: warnings.warn( "The auto-tuning configuration of the dataloader is incorrect." diff --git a/python/paddle/incubate/distributed/fleet/__init__.py b/python/paddle/incubate/distributed/fleet/__init__.py index 94e1a7c8bbe..f098e4c951c 100644 --- a/python/paddle/incubate/distributed/fleet/__init__.py +++ b/python/paddle/incubate/distributed/fleet/__init__.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.distributed.fleet.recompute import recompute_sequential, recompute_hybrid +from paddle.distributed.fleet.recompute import ( + recompute_sequential, + recompute_hybrid, +) __all__ = ["recompute_sequential", "recompute_hybrid"] diff --git a/python/paddle/incubate/distributed/models/moe/gate/base_gate.py b/python/paddle/incubate/distributed/models/moe/gate/base_gate.py index 9715f4b2a25..f1b4e74c241 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/base_gate.py +++ b/python/paddle/incubate/distributed/models/moe/gate/base_gate.py @@ -23,7 +23,6 @@ import paddle.nn as nn class BaseGate(nn.Layer): - def __init__(self, num_expert, world_size): super().__init__() self.world_size = world_size diff --git a/python/paddle/incubate/distributed/models/moe/gate/gshard_gate.py b/python/paddle/incubate/distributed/models/moe/gate/gshard_gate.py index 4152dee3a37..9587dfb346a 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/gshard_gate.py +++ b/python/paddle/incubate/distributed/models/moe/gate/gshard_gate.py @@ -27,15 +27,16 @@ from ..utils import limit_by_capacity class GShardGate(NaiveGate): - - def __init__(self, - d_model, - num_expert, - world_size, - topk=2, - capacity=(1.2, 2.4), - random_routing=True, - group=None): + def __init__( + self, + d_model, + num_expert, + world_size, + topk=2, + capacity=(1.2, 2.4), + random_routing=True, + group=None, + ): assert topk == 2, "topk should be 2 in gshard" super().__init__(d_model, num_expert, world_size) self.capacity = capacity @@ -43,29 +44,39 @@ class GShardGate(NaiveGate): self.group = group def forward(self, x): - topk_val, topk_idx, gate_score = super().forward(x, - return_all_scores=True) + topk_val, topk_idx, gate_score = super().forward( + x, return_all_scores=True + ) s = gate_score.shape[0] top1_idx = topk_idx.flatten() - c_e = paddle.scatter(paddle.zeros(shape=[self.tot_expert]), - top1_idx, - paddle.ones_like(top1_idx, dtype="float32"), - overwrite=False) / s + c_e = ( + paddle.scatter( + paddle.zeros(shape=[self.tot_expert]), + top1_idx, + paddle.ones_like(top1_idx, dtype="float32"), + overwrite=False, + ) + / s + ) m_e = paddle.mean(F.softmax(gate_score, axis=1), axis=0) loss = paddle.mean(c_e * m_e) * (self.num_expert**2) self.set_loss(loss) cap_rate = self.capacity[0 if self.training else 1] capacity = math.ceil(cap_rate * x.shape[0]) - _new_lec, _new_gec, topk_idx = limit_by_capacity(topk_idx, - self.num_expert, - self.world_size, - capacity, - group=self.group) + _new_lec, _new_gec, topk_idx = limit_by_capacity( + topk_idx, + self.num_expert, + self.world_size, + capacity, + group=self.group, + ) if self.random_routing: - rand_routing_prob = paddle.rand(shape=[gate_score.shape[0]], - dtype="float32") + rand_routing_prob = paddle.rand( + shape=[gate_score.shape[0]], dtype="float32" + ) topk_idx = paddle.distributed.models.moe.utils._random_routing( - topk_idx, topk_val, rand_routing_prob) + topk_idx, topk_val, rand_routing_prob + ) return topk_val, topk_idx diff --git a/python/paddle/incubate/distributed/models/moe/gate/naive_gate.py b/python/paddle/incubate/distributed/models/moe/gate/naive_gate.py index 20001377fd1..4781f6bba89 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/naive_gate.py +++ b/python/paddle/incubate/distributed/models/moe/gate/naive_gate.py @@ -26,7 +26,6 @@ import paddle.nn as nn class NaiveGate(BaseGate): - def __init__(self, d_model, num_expert, world_size, topk=2): super().__init__(num_expert, world_size) self.gate = nn.Linear(d_model, self.tot_expert) @@ -36,11 +35,9 @@ class NaiveGate(BaseGate): def forward(self, inp, return_all_scores=False): gate = self.gate(inp) - gate_top_k_val, gate_top_k_idx = paddle.topk(gate, - k=self.top_k, - axis=-1, - largest=True, - sorted=False) + gate_top_k_val, gate_top_k_idx = paddle.topk( + gate, k=self.top_k, axis=-1, largest=True, sorted=False + ) if return_all_scores: return gate_top_k_val, gate_top_k_idx, gate diff --git a/python/paddle/incubate/distributed/models/moe/gate/switch_gate.py b/python/paddle/incubate/distributed/models/moe/gate/switch_gate.py index 62e32e4db70..c9e2f1fca49 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/switch_gate.py +++ b/python/paddle/incubate/distributed/models/moe/gate/switch_gate.py @@ -27,15 +27,16 @@ from ..utils import limit_by_capacity class SwitchGate(NaiveGate): - - def __init__(self, - d_model, - num_expert, - world_size, - topk=1, - switch_eps=.1, - capacity=(1.2, 2.4), - group=None): + def __init__( + self, + d_model, + num_expert, + world_size, + topk=1, + switch_eps=0.1, + capacity=(1.2, 2.4), + group=None, + ): assert topk == 1, "topk should be 1 in switch" super().__init__(d_model, num_expert, world_size, topk=1) self.switch_eps = switch_eps @@ -55,19 +56,25 @@ class SwitchGate(NaiveGate): cap_rate = self.capacity[0 if self.training else 1] capacity = math.ceil(cap_rate * inp.shape[0]) - _new_lec, _new_gec, top1_idx = limit_by_capacity(top1_idx, - self.num_expert, - self.world_size, - capacity, - group=self.group) + _new_lec, _new_gec, top1_idx = limit_by_capacity( + top1_idx, + self.num_expert, + self.world_size, + capacity, + group=self.group, + ) valid_idx = top1_idx[top1_idx > -1] valid_idx_tmp = paddle.reshape(valid_idx, shape=[len(valid_idx), 1]) - fraction_expert = paddle.scatter_nd_add( - x=paddle.zeros(shape=[self.tot_expert]), - index=valid_idx_tmp, - updates=paddle.ones_like(valid_idx, dtype=paddle.float32).reshape( - shape=[len(valid_idx)]), - ) / valid_idx.numel() + fraction_expert = ( + paddle.scatter_nd_add( + x=paddle.zeros(shape=[self.tot_expert]), + index=valid_idx_tmp, + updates=paddle.ones_like( + valid_idx, dtype=paddle.float32 + ).reshape(shape=[len(valid_idx)]), + ) + / valid_idx.numel() + ) prob_expert = score.sum(axis=0) / valid_idx.numel() loss = (fraction_expert * prob_expert).sum() * self.tot_expert self.set_loss(loss) diff --git a/python/paddle/incubate/distributed/models/moe/grad_clip.py b/python/paddle/incubate/distributed/models/moe/grad_clip.py index 91de68e4bfa..aaeef878078 100644 --- a/python/paddle/incubate/distributed/models/moe/grad_clip.py +++ b/python/paddle/incubate/distributed/models/moe/grad_clip.py @@ -80,18 +80,21 @@ class ClipGradForMOEByGlobalNorm(ClipGradBase): sdg.step() """ - def __init__(self, - clip_norm, - is_expert_param_func=None, - moe_group=None, - group_name="default_moe_group"): + def __init__( + self, + clip_norm, + is_expert_param_func=None, + moe_group=None, + group_name="default_moe_group", + ): super(ClipGradForMOEByGlobalNorm, self).__init__() self.clip_norm = float(clip_norm) self.group_name = group_name self.moe_group = moe_group if moe_group is not None and moe_group.nranks > 1: - assert is_expert_param_func is not None, \ - "When moe group size > 1, a function for selecting expert params must be specified." + assert ( + is_expert_param_func is not None + ), "When moe group size > 1, a function for selecting expert params must be specified." self.is_expert_param_func = is_expert_param_func def __str__(self): @@ -120,11 +123,18 @@ class ClipGradForMOEByGlobalNorm(ClipGradBase): sum_square_list.append(sum_square) # all parameters have been filterd out - if len(sum_square_list) + len(sum_square_list_fp16) + len( - sum_square_list_fp32) == 0: + if ( + len(sum_square_list) + + len(sum_square_list_fp16) + + len(sum_square_list_fp32) + == 0 + ): return None, None - assert sum_dtype in ["float64", "float32", None], \ - "sum's type must be float64/ float32 / None" + assert sum_dtype in [ + "float64", + "float32", + None, + ], "sum's type must be float64/ float32 / None" if sum_dtype != "float64": sum_dtype = 'float64' if len(sum_square_list) > 0 else "float32" @@ -166,16 +176,20 @@ class ClipGradForMOEByGlobalNorm(ClipGradBase): # why to return sum_dtype? # we will call `get_l2_norm_pow` twice and the precisions may be different. # For convenience and simplification, we use sum_dtype directly instead of global_norm_var_normal.dtype - global_norm_var_normal, sum_dtype \ - = self.get_l2_norm_pow(normal_params_grads) + global_norm_var_normal, sum_dtype = self.get_l2_norm_pow( + normal_params_grads + ) global_norm_var_moe = None if len(moe_params_grads) > 0: - global_norm_var_moe, _ \ - = self.get_l2_norm_pow(moe_params_grads, sum_dtype) + global_norm_var_moe, _ = self.get_l2_norm_pow( + moe_params_grads, sum_dtype + ) if global_norm_var_moe is not None: - collective.all_reduce(global_norm_var_moe, - op=collective.ReduceOp.SUM, - group=self.moe_group) + collective.all_reduce( + global_norm_var_moe, + op=collective.ReduceOp.SUM, + group=self.moe_group, + ) if global_norm_var_normal is None and global_norm_var_moe is None: return params_grads @@ -187,19 +201,20 @@ class ClipGradForMOEByGlobalNorm(ClipGradBase): if global_norm_var_normal.dtype != global_norm_var_moe.dtype: # compared with normal norm, moe norm is the later one, # so its precision is no lower than normal norm - global_norm_var_normal = \ - global_norm_var_normal.astype(global_norm_var_moe.dtype) + global_norm_var_normal = global_norm_var_normal.astype( + global_norm_var_moe.dtype + ) global_norm_var = global_norm_var_normal + global_norm_var_moe params_and_grads = [] global_norm_var = layers.sqrt(global_norm_var) - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var, - y=max_global_norm)) + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var, y=max_global_norm), + ) for p, g in params_grads: if g is None: continue @@ -207,8 +222,11 @@ class ClipGradForMOEByGlobalNorm(ClipGradBase): params_and_grads.append((p, g)) continue # TODO(wangxi): use inplace elementwise_mul - clip_input = (clip_var.astype('float16') - if g.dtype == core.VarDesc.VarType.FP16 else clip_var) + clip_input = ( + clip_var.astype('float16') + if g.dtype == core.VarDesc.VarType.FP16 + else clip_var + ) new_grad = layers.elementwise_mul(x=g, y=clip_input) params_and_grads.append((p, new_grad)) return params_and_grads diff --git a/python/paddle/incubate/distributed/models/moe/moe_layer.py b/python/paddle/incubate/distributed/models/moe/moe_layer.py index b2d11e7a317..345a10eed9c 100644 --- a/python/paddle/incubate/distributed/models/moe/moe_layer.py +++ b/python/paddle/incubate/distributed/models/moe/moe_layer.py @@ -43,11 +43,14 @@ def _local_gather(inp, pos, out_batch_size, maybe_overlap=True): if pos.shape != [0]: origin_dtype = inp.dtype inp = paddle.cast(inp, dtype="float32") - inp_buf = paddle.scatter(paddle.zeros( - shape=[out_batch_size, inp.shape[-1]], dtype="float32"), - pos, - inp, - overwrite=True) + inp_buf = paddle.scatter( + paddle.zeros( + shape=[out_batch_size, inp.shape[-1]], dtype="float32" + ), + pos, + inp, + overwrite=True, + ) inp_buf = paddle.cast(inp_buf, dtype=origin_dtype) else: inp_buf = paddle.zeros([out_batch_size, inp.shape[-1]], dtype=inp.dtype) @@ -59,8 +62,11 @@ def _all_gather(tensor, group=None, use_calc_stream=True): return if in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) tensor_shape = list(tensor.shape) tensor_shape[0] *= group.nranks out = paddle.empty(tensor_shape, tensor.dtype) @@ -70,11 +76,20 @@ def _all_gather(tensor, group=None, use_calc_stream=True): return out else: ring_id = 0 if group is None else group.id - nranks = paddle.distributed.collective._get_global_group( - ).nranks if group is None else group.nranks - return paddle._legacy_C_ops.c_allgather(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'nranks', nranks) + nranks = ( + paddle.distributed.collective._get_global_group().nranks + if group is None + else group.nranks + ) + return paddle._legacy_C_ops.c_allgather( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + ) class MoEScatter(PyLayer): @@ -85,20 +100,24 @@ class MoEScatter(PyLayer): """ @staticmethod - def forward(ctx, - inp, - pos, - local_expert_count, - global_expert_count, - fwd_batch_size, - world_size, - group=None): + def forward( + ctx, + inp, + pos, + local_expert_count, + global_expert_count, + fwd_batch_size, + world_size, + group=None, + ): local_input_buf = _local_scatter(inp, pos) if world_size > 1: - global_input_buf = global_scatter(local_input_buf, - local_expert_count, - global_expert_count, - group=group) + global_input_buf = global_scatter( + local_input_buf, + local_expert_count, + global_expert_count, + group=group, + ) else: global_input_buf = local_input_buf @@ -114,10 +133,9 @@ class MoEScatter(PyLayer): (inp_batch_size, world_size, group) = ctx.moe_args if world_size > 1: - local_grad_in = global_gather(grad, - local_expert_count, - global_expert_count, - group=group) + local_grad_in = global_gather( + grad, local_expert_count, global_expert_count, group=group + ) else: local_grad_in = grad grad_in = _local_gather(local_grad_in, pos, inp_batch_size) @@ -131,25 +149,28 @@ class MoEGather(PyLayer): """ @staticmethod - def forward(ctx, + def forward( + ctx, + global_output_buf, + pos, + local_expert_count, + global_expert_count, + local_batch_size, + world_size, + group=None, + ): + if world_size > 1: + local_output_buf = global_gather( global_output_buf, - pos, local_expert_count, global_expert_count, - local_batch_size, - world_size, - group=None): - if world_size > 1: - local_output_buf = global_gather(global_output_buf, - local_expert_count, - global_expert_count, - group=group) + group=group, + ) else: local_output_buf = global_output_buf - output = _local_gather(local_output_buf, - pos, - local_batch_size, - maybe_overlap=False) + output = _local_gather( + local_output_buf, pos, local_batch_size, maybe_overlap=False + ) ctx.moe_args = (global_output_buf.shape[0], world_size, group) variables = (pos, local_expert_count, global_expert_count) @@ -162,10 +183,12 @@ class MoEGather(PyLayer): fwd_batch_size, world_size, group = ctx.moe_args grad_out_buf = _local_scatter(grad_out, pos) if world_size > 1: - global_grad_out_buf = global_scatter(grad_out_buf, - local_expert_count, - global_expert_count, - group=group) + global_grad_out_buf = global_scatter( + grad_out_buf, + local_expert_count, + global_expert_count, + group=group, + ) else: global_grad_out_buf = grad_out_buf return global_grad_out_buf, None, None, None @@ -187,10 +210,9 @@ class AllGather(PyLayer): @staticmethod def backward(ctx, grad_out): rank, dim0 = ctx.args - return paddle.slice(grad_out, - axes=[0], - starts=[rank * dim0], - ends=[(rank + 1) * dim0]) + return paddle.slice( + grad_out, axes=[0], starts=[rank * dim0], ends=[(rank + 1) * dim0] + ) class Slice(PyLayer): @@ -204,10 +226,9 @@ class Slice(PyLayer): local_batch_size = B // world_size batch_start = local_batch_size * rank batch_end = min(batch_start + local_batch_size, B) - inp = paddle.slice(inp, - axes=[0], - starts=[batch_start], - ends=[batch_end]) + inp = paddle.slice( + inp, axes=[0], starts=[batch_start], ends=[batch_end] + ) ctx.args = world_size, group return inp @@ -219,10 +240,12 @@ class Slice(PyLayer): def prepare_forward(gate, num_expert, world_size, moe_group): pos, local_expert_count, global_expert_count = count_by_gate( - gate, num_expert, world_size, group=moe_group) + gate, num_expert, world_size, group=moe_group + ) with paddle.no_grad(): fwd_expert_count = global_expert_count.reshape_( - [world_size, num_expert]).sum(axis=0) + [world_size, num_expert] + ).sum(axis=0) fwd_batch_size = int(fwd_expert_count.sum().item()) return ( pos, @@ -296,14 +319,16 @@ class MoELayer(nn.Layer): """ - def __init__(self, - d_model, - experts, - gate=None, - moe_group=None, - mp_group=None, - recompute_interval=0, - recompute_ctx=None): + def __init__( + self, + d_model, + experts, + gate=None, + moe_group=None, + mp_group=None, + recompute_interval=0, + recompute_ctx=None, + ): super(MoELayer, self).__init__() self.recompute_ctx = recompute_ctx @@ -311,8 +336,9 @@ class MoELayer(nn.Layer): if gate is None: gate = dict() - assert isinstance(gate, (dict, BaseGate)), \ - "gate config' type must be dict or an instance of BaseGate" + assert isinstance( + gate, (dict, BaseGate) + ), "gate config' type must be dict or an instance of BaseGate" # only support mp/dp self.group = moe_group @@ -330,26 +356,36 @@ class MoELayer(nn.Layer): self.top_k = gate.get("top_k", 2) gate = gate.get("type", "gshard") if gate == "naive" or gate is None: - gate = NaiveGate(self.d_model, - num_expert=len(experts), - world_size=self.world_size, - topk=self.top_k) + gate = NaiveGate( + self.d_model, + num_expert=len(experts), + world_size=self.world_size, + topk=self.top_k, + ) elif gate == "gshard": - gate = GShardGate(self.d_model, - num_expert=len(experts), - world_size=self.world_size, - topk=self.top_k, - group=self.group) + gate = GShardGate( + self.d_model, + num_expert=len(experts), + world_size=self.world_size, + topk=self.top_k, + group=self.group, + ) elif gate == "switch": - gate = SwitchGate(self.d_model, - num_expert=len(experts), - world_size=self.world_size, - topk=self.top_k, - group=self.group) + gate = SwitchGate( + self.d_model, + num_expert=len(experts), + world_size=self.world_size, + topk=self.top_k, + group=self.group, + ) else: - assert False, "We only support naive gate, \ + assert ( + False + ), "We only support naive gate, \ gshard gate and switch gate, \ - but you choose {} gate.".format(str(gate)) + but you choose {} gate.".format( + str(gate) + ) elif isinstance(gate, NaiveGate): self.top_k = gate.top_k elif isinstance(gate, BaseGate): @@ -391,9 +427,15 @@ class MoELayer(nn.Layer): temp_pos = pos assert topk == self.top_k - x = MoEScatter.apply(inp, temp_pos, local_expert_count, - global_expert_count, fwd_batch_size, - self.world_size, self.group) + x = MoEScatter.apply( + inp, + temp_pos, + local_expert_count, + global_expert_count, + fwd_batch_size, + self.world_size, + self.group, + ) d_model = self.d_model @@ -408,22 +450,36 @@ class MoELayer(nn.Layer): for idx, expert_count in enumerate(fwd_expert_count): if expert_count <= 0: continue - y.append(experts[idx](x[last_index:expert_count + last_index])) + y.append( + experts[idx](x[last_index : expert_count + last_index]) + ) last_index = expert_count + last_index return paddle.concat(y, axis=0) if self.recompute_interval <= 0 or x.shape[0] == 0: x = experts_fwd(x, fwd_expert_count.numpy(), self.experts) else: - x = recompute_hybrid(self.recompute_ctx, experts_fwd, x, - fwd_expert_count.numpy(), self.experts) + x = recompute_hybrid( + self.recompute_ctx, + experts_fwd, + x, + fwd_expert_count.numpy(), + self.experts, + ) out_batch_size = inp.shape[0] if len(gate.shape) == 2: out_batch_size *= gate.shape[1] - x = MoEGather.apply(x, pos, local_expert_count, global_expert_count, - out_batch_size, self.world_size, self.group) + x = MoEGather.apply( + x, + pos, + local_expert_count, + global_expert_count, + out_batch_size, + self.world_size, + self.group, + ) x = x.reshape([-1, self.top_k, d_model]) value = value.reshape([x.shape[0], 1, self.top_k]) diff --git a/python/paddle/incubate/distributed/models/moe/utils.py b/python/paddle/incubate/distributed/models/moe/utils.py index 10203a0cd18..65ab86ded6a 100644 --- a/python/paddle/incubate/distributed/models/moe/utils.py +++ b/python/paddle/incubate/distributed/models/moe/utils.py @@ -19,7 +19,12 @@ # Copyright 2021, Jiaao He. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"). -from paddle.distributed.models.moe.utils import _number_count, _limit_by_capacity, _prune_gate_by_capacity, _assign_pos +from paddle.distributed.models.moe.utils import ( + _number_count, + _limit_by_capacity, + _prune_gate_by_capacity, + _assign_pos, +) import paddle from paddle.fluid.framework import in_dygraph_mode @@ -29,17 +34,24 @@ def _alltoall(in_tensor_list, group=None, use_calc_stream=True): return if in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) out = paddle.empty(in_tensor_list.shape, in_tensor_list.dtype) task = group.process_group.alltoall(in_tensor_list, out) task.wait() return out else: ring_id = 0 if group is None else group.id - return paddle._legacy_C_ops.alltoall(in_tensor_list, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id) + return paddle._legacy_C_ops.alltoall( + in_tensor_list, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + ) def count_by_gate(gate, num_expert, world_size, require_pos=True, group=None): @@ -61,13 +73,12 @@ def count_by_gate(gate, num_expert, world_size, require_pos=True, group=None): def limit_by_capacity(topk_idx, num_expert, world_size, capacity, group=None): with paddle.no_grad(): - capacity = paddle.ones(shape=[num_expert], - dtype=paddle.int64) * capacity - pos, lec, gec = count_by_gate(topk_idx, - num_expert, - world_size, - require_pos=False, - group=group) + capacity = ( + paddle.ones(shape=[num_expert], dtype=paddle.int64) * capacity + ) + pos, lec, gec = count_by_gate( + topk_idx, num_expert, world_size, require_pos=False, group=group + ) new_gec = _limit_by_capacity(gec, capacity, world_size) if world_size > 1: assert group.nranks == world_size @@ -75,7 +86,8 @@ def limit_by_capacity(topk_idx, num_expert, world_size, capacity, group=None): else: new_lec = new_gec - topk_idx = _prune_gate_by_capacity(topk_idx, new_lec, num_expert, - world_size) + topk_idx = _prune_gate_by_capacity( + topk_idx, new_lec, num_expert, world_size + ) return new_lec, new_gec, topk_idx diff --git a/python/paddle/incubate/multiprocessing/reductions.py b/python/paddle/incubate/multiprocessing/reductions.py index c54626175bc..b16361971ed 100644 --- a/python/paddle/incubate/multiprocessing/reductions.py +++ b/python/paddle/incubate/multiprocessing/reductions.py @@ -35,16 +35,17 @@ def _supported_check(): return False if not sys.version_info >= (3, 4): - warnings.warn("Use `paddle.multiprocessing` to share paddle tensor " - "requires python version greater than 3.4 ." - " `paddle.multiprocessing` will not take any effect !!!") + warnings.warn( + "Use `paddle.multiprocessing` to share paddle tensor " + "requires python version greater than 3.4 ." + " `paddle.multiprocessing` will not take any effect !!!" + ) return False return True class LRUSharedCache(OrderedDict): - def __init__(self): self.limit = 128 self._after_fork() @@ -84,9 +85,9 @@ def cuda_from_cache(key): def rebuild_tensor(cls, lodtensor, metadata): if cls == paddle.fluid.framework.ParamBase: - tensor = paddle.fluid.framework.ParamBase(lodtensor.shape(), - lodtensor._dtype(), - **metadata) + tensor = paddle.fluid.framework.ParamBase( + lodtensor.shape(), lodtensor._dtype(), **metadata + ) tensor.value().get_tensor()._share_data_with(lodtensor) else: size, stop_gradient = metadata @@ -107,8 +108,11 @@ def reduce_tensor(tensor): "Refusing to serialize non-leaf tensor which not stop_gradient, you can detach it!" ) # TODO: add serializing name and hooks check - if tensor.place.is_cpu_place() or tensor.place.is_gpu_place( - ) or tensor.place.is_cuda_pinned_place(): + if ( + tensor.place.is_cpu_place() + or tensor.place.is_gpu_place() + or tensor.place.is_cuda_pinned_place() + ): if type(tensor) == paddle.fluid.framework.ParamBase: metadata = copy.deepcopy(tensor.__dict__) else: @@ -118,7 +122,8 @@ def reduce_tensor(tensor): else: raise ValueError( "Only support tensors of CPU/CUDA/CUDAPinned Place, Not support %s for now!" - % tensor.place) + % tensor.place + ) def rebuild_lodtensor_filename(cls, ipc_name, size, type_idx, dims, lod): @@ -127,12 +132,14 @@ def rebuild_lodtensor_filename(cls, ipc_name, size, type_idx, dims, lod): return lodtensor -def rebuild_cuda_tensor(cls, handle, offset_bytes, size, type_idx, dims, lod, - device_idx): +def rebuild_cuda_tensor( + cls, handle, offset_bytes, size, type_idx, dims, lod, device_idx +): cache_tensor = cuda_from_cache((handle, offset_bytes)) if cache_tensor is None: lodtensor = cls._new_shared_cuda( - (handle, offset_bytes, size, type_idx, dims, lod, device_idx)) + (handle, offset_bytes, size, type_idx, dims, lod, device_idx) + ) # We only cache cuda shared tensor here. # The opening cost of cudaIpcMemoryHandle is very high. # Since we cache the recived tensor directly, @@ -141,28 +148,32 @@ def rebuild_cuda_tensor(cls, handle, offset_bytes, size, type_idx, dims, lod, shared_cache[(handle, offset_bytes)] = lodtensor else: lodtensor = paddle.fluid.core.LoDTensor() - lodtensor._share_buffer_with(cache_tensor, - (size, type_idx, dims, lod, device_idx)) + lodtensor._share_buffer_with( + cache_tensor, (size, type_idx, dims, lod, device_idx) + ) return lodtensor def rebuild_lodtensor_empty(cls): - #TODO: check if tensor initialized - #TODO: handle the dtype of empty tensor + # TODO: check if tensor initialized + # TODO: handle the dtype of empty tensor return cls() def reduce_lodtensor(lodtensor): - if lodtensor._place().is_cpu_place() or lodtensor._place( - ).is_cuda_pinned_place(): + if ( + lodtensor._place().is_cpu_place() + or lodtensor._place().is_cuda_pinned_place() + ): for dim in lodtensor.shape(): if dim == 0: # Empty tensors have nothing be mmapped. - return (rebuild_lodtensor_empty, (type(lodtensor), )) + return (rebuild_lodtensor_empty, (type(lodtensor),)) # Default use share filename stratege - metadata = lodtensor._share_filename( + metadata = ( + lodtensor._share_filename() ) # ipc_name, size, type_idx, dims, lod rebuild = rebuild_lodtensor_filename lodtensor._shared_incref() @@ -174,7 +185,7 @@ def reduce_lodtensor(lodtensor): else: raise RuntimeError("We only support pass cpu/gpu lodtensor for now!") - return (rebuild, (type(lodtensor), ) + metadata) + return (rebuild, (type(lodtensor),) + metadata) def init_reductions(): diff --git a/python/paddle/incubate/nn/__init__.py b/python/paddle/incubate/nn/__init__.py index cf15ee7d8ff..62a48b783df 100644 --- a/python/paddle/incubate/nn/__init__.py +++ b/python/paddle/incubate/nn/__init__.py @@ -17,9 +17,11 @@ from .layer.fused_transformer import FusedFeedForward # noqa: F401 from .layer.fused_transformer import FusedTransformerEncoderLayer # noqa: F401 from .layer.fused_transformer import FusedMultiTransformer # noqa: F401 from .layer.fused_linear import FusedLinear # noqa: F401 -from .layer.fused_transformer import FusedBiasDropoutResidualLayerNorm # noqa: F401 +from .layer.fused_transformer import ( + FusedBiasDropoutResidualLayerNorm, +) # noqa: F401 -__all__ = [ #noqa +__all__ = [ # noqa 'FusedMultiHeadAttention', 'FusedFeedForward', 'FusedTransformerEncoderLayer', diff --git a/python/paddle/incubate/nn/functional/fused_matmul_bias.py b/python/paddle/incubate/nn/functional/fused_matmul_bias.py index 6c0722fc706..e7180abfe9b 100644 --- a/python/paddle/incubate/nn/functional/fused_matmul_bias.py +++ b/python/paddle/incubate/nn/functional/fused_matmul_bias.py @@ -18,12 +18,9 @@ from paddle.tensor.linalg import matmul from paddle import _legacy_C_ops -def fused_matmul_bias(x, - y, - bias=None, - transpose_x=False, - transpose_y=False, - name=None): +def fused_matmul_bias( + x, y, bias=None, transpose_x=False, transpose_y=False, name=None +): """ Applies matrix multiplication of two tensors and then bias addition if provided. This method requires CUDA version >= 11.6. @@ -57,23 +54,18 @@ def fused_matmul_bias(x, if bias is None: return matmul(x, y, transpose_x, transpose_y, name) if _non_static_mode(): - return _legacy_C_ops.fused_gemm_epilogue(x, y, bias, 'trans_x', - transpose_x, 'trans_y', - transpose_y) + return _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', transpose_x, 'trans_y', transpose_y + ) helper = LayerHelper('fused_matmul_bias', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='fused_gemm_epilogue', - inputs={ - 'X': x, - 'Y': y, - 'Bias': bias - }, - outputs={'Out': out}, - attrs={ - 'trans_x': transpose_x, - 'trans_y': transpose_y - }) + helper.append_op( + type='fused_gemm_epilogue', + inputs={'X': x, 'Y': y, 'Bias': bias}, + outputs={'Out': out}, + attrs={'trans_x': transpose_x, 'trans_y': transpose_y}, + ) return out diff --git a/python/paddle/incubate/nn/functional/fused_transformer.py b/python/paddle/incubate/nn/functional/fused_transformer.py index 2eaa7ceadc7..dffddb8b9ec 100644 --- a/python/paddle/incubate/nn/functional/fused_transformer.py +++ b/python/paddle/incubate/nn/functional/fused_transformer.py @@ -28,26 +28,28 @@ def _verify_dropout_rate(dropout_rate): raise ValueError("dropout_rate argument should between 0 and 1") -def fused_feedforward(x, - linear1_weight, - linear2_weight, - linear1_bias=None, - linear2_bias=None, - ln1_scale=None, - ln1_bias=None, - ln2_scale=None, - ln2_bias=None, - dropout1_rate=0.5, - dropout2_rate=0.5, - activation="relu", - ln1_epsilon=1e-5, - ln2_epsilon=1e-5, - pre_layer_norm=False, - training=True, - mode='upscale_in_train', - ring_id=-1, - add_residual=True, - name=None): +def fused_feedforward( + x, + linear1_weight, + linear2_weight, + linear1_bias=None, + linear2_bias=None, + ln1_scale=None, + ln1_bias=None, + ln2_scale=None, + ln2_bias=None, + dropout1_rate=0.5, + dropout2_rate=0.5, + activation="relu", + ln1_epsilon=1e-5, + ln2_epsilon=1e-5, + pre_layer_norm=False, + training=True, + mode='upscale_in_train', + ring_id=-1, + add_residual=True, + name=None, +): r""" This is a fusion operator to compute feed forward layer in transformer model architecture. This operator only supports running on GPU. The function of the operator is consistent with @@ -126,112 +128,161 @@ def fused_feedforward(x, raise ValueError( "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed out, _, _, _, _, _, _, _, _, _, _ = _legacy_C_ops.fused_feedforward( - x, None, None, linear1_weight, linear1_bias, linear2_weight, - linear2_bias, ln1_scale, ln1_bias, ln2_scale, ln2_bias, - 'pre_layer_norm', pre_layer_norm, 'ln1_epsilon', ln1_epsilon, - 'ln2_epsilon', ln2_epsilon, 'act_method', activation, - 'dropout1_rate', dropout1_rate, 'dropout2_rate', dropout2_rate, - "is_test", not training, "dropout1_fix_seed", seed is not None, - "dropout2_fix_seed", seed is not None, "dropout1_seed", - seed if seed is not None else 0, "dropout2_seed", - seed if seed is not None else 0, 'dropout1_implementation', mode, - 'dropout2_implementation', mode, 'add_residual', add_residual, - 'ring_id', ring_id) + x, + None, + None, + linear1_weight, + linear1_bias, + linear2_weight, + linear2_bias, + ln1_scale, + ln1_bias, + ln2_scale, + ln2_bias, + 'pre_layer_norm', + pre_layer_norm, + 'ln1_epsilon', + ln1_epsilon, + 'ln2_epsilon', + ln2_epsilon, + 'act_method', + activation, + 'dropout1_rate', + dropout1_rate, + 'dropout2_rate', + dropout2_rate, + "is_test", + not training, + "dropout1_fix_seed", + seed is not None, + "dropout2_fix_seed", + seed is not None, + "dropout1_seed", + seed if seed is not None else 0, + "dropout2_seed", + seed if seed is not None else 0, + 'dropout1_implementation', + mode, + 'dropout2_implementation', + mode, + 'add_residual', + add_residual, + 'ring_id', + ring_id, + ) return out helper = LayerHelper("fused_feedforward") dtype = x.dtype - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'fused_feedforward') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], - 'fused_feedforward') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'fused_feedforward' + ) + check_dtype( + dtype, 'dtype', ['float16', 'float32', 'float64'], 'fused_feedforward' + ) out = helper.create_variable_for_type_inference(x.dtype) dropout1_mask = helper.create_variable_for_type_inference( - 'uint8', stop_gradient=True) + 'uint8', stop_gradient=True + ) dropout2_mask = helper.create_variable_for_type_inference( - 'uint8', stop_gradient=True) - ln1_mean = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - ln1_variance = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - ln2_mean = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - ln2_variance = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - linear1_out = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - ln1_out = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - dropout1_out = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - dropout2_out = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) + 'uint8', stop_gradient=True + ) + ln1_mean = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + ln1_variance = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + ln2_mean = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + ln2_variance = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + linear1_out = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + ln1_out = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + dropout1_out = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + dropout2_out = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) if (seed is None or seed == 0) and helper.main_program.random_seed != 0: seed = helper.main_program.random_seed - helper.append_op(type='fused_feedforward', - inputs={ - 'X': x, - 'Linear1Weight': linear1_weight, - 'Linear1Bias': linear1_bias, - 'Linear2Weight': linear2_weight, - 'Linear2Bias': linear2_bias, - 'Ln1Scale': ln1_scale, - 'Ln1Bias': ln1_bias, - 'Ln2Scale': ln2_scale, - 'Ln2Bias': ln2_bias, - }, - outputs={ - 'Out': out, - 'Dropout1Mask': dropout1_mask, - 'Dropout2Mask': dropout2_mask, - 'Ln1Mean': ln1_mean, - 'Ln1Variance': ln1_variance, - 'Ln2Mean': ln2_mean, - 'Ln2Variance': ln2_variance, - 'Linear1Out': linear1_out, - 'Ln1Out': ln1_out, - 'Dropout1Out': dropout1_out, - 'Dropout2Out': dropout2_out, - }, - attrs={ - 'dropout1_rate': dropout1_rate, - 'dropout2_rate': dropout2_rate, - 'act_method': activation, - 'pre_layer_norm': pre_layer_norm, - 'ln1_epsilon': ln1_epsilon, - 'ln2_epsilon': ln2_epsilon, - 'is_test': not training, - 'dropout1_fix_seed': seed is not None, - 'dropout2_fix_seed': seed is not None, - 'dropout1_seed': seed if seed is not None else 0, - 'dropout2_seed': seed if seed is not None else 0, - 'dropout1_implementation': mode, - 'dropout2_implementation': mode, - 'add_residual': add_residual, - 'ring_id': ring_id, - }) + helper.append_op( + type='fused_feedforward', + inputs={ + 'X': x, + 'Linear1Weight': linear1_weight, + 'Linear1Bias': linear1_bias, + 'Linear2Weight': linear2_weight, + 'Linear2Bias': linear2_bias, + 'Ln1Scale': ln1_scale, + 'Ln1Bias': ln1_bias, + 'Ln2Scale': ln2_scale, + 'Ln2Bias': ln2_bias, + }, + outputs={ + 'Out': out, + 'Dropout1Mask': dropout1_mask, + 'Dropout2Mask': dropout2_mask, + 'Ln1Mean': ln1_mean, + 'Ln1Variance': ln1_variance, + 'Ln2Mean': ln2_mean, + 'Ln2Variance': ln2_variance, + 'Linear1Out': linear1_out, + 'Ln1Out': ln1_out, + 'Dropout1Out': dropout1_out, + 'Dropout2Out': dropout2_out, + }, + attrs={ + 'dropout1_rate': dropout1_rate, + 'dropout2_rate': dropout2_rate, + 'act_method': activation, + 'pre_layer_norm': pre_layer_norm, + 'ln1_epsilon': ln1_epsilon, + 'ln2_epsilon': ln2_epsilon, + 'is_test': not training, + 'dropout1_fix_seed': seed is not None, + 'dropout2_fix_seed': seed is not None, + 'dropout1_seed': seed if seed is not None else 0, + 'dropout2_seed': seed if seed is not None else 0, + 'dropout1_implementation': mode, + 'dropout2_implementation': mode, + 'add_residual': add_residual, + 'ring_id': ring_id, + }, + ) return out -def fused_bias_dropout_residual_layer_norm(x, - residual, - bias=None, - ln_scale=None, - ln_bias=None, - dropout_rate=0.5, - ln_epsilon=1e-5, - training=True, - mode='upscale_in_train', - name=None): +def fused_bias_dropout_residual_layer_norm( + x, + residual, + bias=None, + ln_scale=None, + ln_bias=None, + dropout_rate=0.5, + ln_epsilon=1e-5, + training=True, + mode='upscale_in_train', + name=None, +): r""" The fused_bias_dropout_residual_layer_norm operator. The pseudo code is as follows: @@ -291,37 +342,72 @@ def fused_bias_dropout_residual_layer_norm(x, raise ValueError( "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if ln_scale is not None: - assert len(ln_scale.shape - ) == 1, "The dims of the shape of ln_scale should be 1." - assert x.shape[len(x.shape) - 1] == ln_scale.shape[ - 0], "The dim of ln_scale must equal to the last dim of x." + assert ( + len(ln_scale.shape) == 1 + ), "The dims of the shape of ln_scale should be 1." + assert ( + x.shape[len(x.shape) - 1] == ln_scale.shape[0] + ), "The dim of ln_scale must equal to the last dim of x." if ln_bias is not None: - assert len( - ln_bias.shape) == 1, "The dims of the shape of ln_bias should be 1." - assert x.shape[len(x.shape) - 1] == ln_bias.shape[ - 0], "The dim of ln_bias must equal to the last dim of x." + assert ( + len(ln_bias.shape) == 1 + ), "The dims of the shape of ln_bias should be 1." + assert ( + x.shape[len(x.shape) - 1] == ln_bias.shape[0] + ), "The dim of ln_bias must equal to the last dim of x." if _non_static_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed - _, _, _, _, final_out = _legacy_C_ops.fused_bias_dropout_residual_layer_norm( - x, residual, bias, ln_scale, ln_bias, 'dropout_rate', dropout_rate, - 'ln_epsilon', ln_epsilon, 'is_test', not training, - 'dropout_fix_seed', seed is not None, 'dropout_seed', - seed if seed is not None else 0, 'dropout_implementation', mode) + ( + _, + _, + _, + _, + final_out, + ) = _legacy_C_ops.fused_bias_dropout_residual_layer_norm( + x, + residual, + bias, + ln_scale, + ln_bias, + 'dropout_rate', + dropout_rate, + 'ln_epsilon', + ln_epsilon, + 'is_test', + not training, + 'dropout_fix_seed', + seed is not None, + 'dropout_seed', + seed if seed is not None else 0, + 'dropout_implementation', + mode, + ) return final_out else: - helper = LayerHelper('fused_bias_dropout_residual_layer_norm', - **locals()) + helper = LayerHelper( + 'fused_bias_dropout_residual_layer_norm', **locals() + ) dtype = x.dtype # check dtypes - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'fused_bias_dropout_residual_layer_norm') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], - 'fused_bias_dropout_residual_layer_norm') + check_variable_and_dtype( + x, + 'x', + ['float16', 'float32', 'float64'], + 'fused_bias_dropout_residual_layer_norm', + ) + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'fused_bias_dropout_residual_layer_norm', + ) # set inputs inputs = dict() inputs['X'] = [x] @@ -345,50 +431,57 @@ def fused_bias_dropout_residual_layer_norm(x, } # set outputs dropout_mask_out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) ln_mean_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) ln_variance_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) bias_dropout_residual_out = helper.create_variable_for_type_inference( - dtype=dtype) + dtype=dtype + ) final_out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='fused_bias_dropout_residual_layer_norm', - inputs=inputs, - outputs={ - "BiasDropoutResidualOut": - bias_dropout_residual_out, - "DropoutMaskOut": dropout_mask_out, - "LnMean": ln_mean_out, - "LnVariance": ln_variance_out, - 'Y': final_out, - }, - attrs=attrs) + helper.append_op( + type='fused_bias_dropout_residual_layer_norm', + inputs=inputs, + outputs={ + "BiasDropoutResidualOut": bias_dropout_residual_out, + "DropoutMaskOut": dropout_mask_out, + "LnMean": ln_mean_out, + "LnVariance": ln_variance_out, + 'Y': final_out, + }, + attrs=attrs, + ) return final_out -def fused_multi_head_attention(x, - qkv_weight, - linear_weight, - pre_layer_norm=False, - pre_ln_scale=None, - pre_ln_bias=None, - ln_scale=None, - ln_bias=None, - pre_ln_epsilon=1e-05, - qkv_bias=None, - linear_bias=None, - cache_kv=None, - attn_mask=None, - dropout_rate=0.5, - attn_dropout_rate=0.5, - ln_epsilon=1e-05, - training=True, - mode='upscale_in_train', - ring_id=-1, - add_residual=True, - name=None): +def fused_multi_head_attention( + x, + qkv_weight, + linear_weight, + pre_layer_norm=False, + pre_ln_scale=None, + pre_ln_bias=None, + ln_scale=None, + ln_bias=None, + pre_ln_epsilon=1e-05, + qkv_bias=None, + linear_bias=None, + cache_kv=None, + attn_mask=None, + dropout_rate=0.5, + attn_dropout_rate=0.5, + ln_epsilon=1e-05, + training=True, + mode='upscale_in_train', + ring_id=-1, + add_residual=True, + name=None, +): r""" Attention mapps queries and a set of key-value pairs to outputs, and Multi-Head Attention performs multiple parallel attention to jointly attending @@ -512,7 +605,9 @@ def fused_multi_head_attention(x, raise ValueError( "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): if default_main_program().random_seed != 0: @@ -520,29 +615,83 @@ def fused_multi_head_attention(x, # pre_ln_mean, pre_ln_variance, pre_ln_out, qkv_out, qkv_bias_out, transpose_out, qk_out, # qktv_out, softmax_out, attn_dropout_mask_out, attn_dropout_out, attn_mask_out, fmha_out, # linear_out, dropout_mask_out, ln_mean_out, ln_var_out, bias_dropout_residual_out, final_out - assert len(qkv_weight.shape - ) == 4, "The dims of the shape of qkv_weight should be 4." - assert qkv_weight.shape[ - 0] == 3, "The shape of qkv_weight should be [3, num_head, head_dim, embed_dim]." - assert qkv_weight.shape[3] == x.shape[ - 2], "The 3rd dim of qkv_weight and 2nd dim of x should be the same, i.e., embed_dim." + assert ( + len(qkv_weight.shape) == 4 + ), "The dims of the shape of qkv_weight should be 4." + assert ( + qkv_weight.shape[0] == 3 + ), "The shape of qkv_weight should be [3, num_head, head_dim, embed_dim]." + assert ( + qkv_weight.shape[3] == x.shape[2] + ), "The 3rd dim of qkv_weight and 2nd dim of x should be the same, i.e., embed_dim." if ring_id == -1: # under mp, the num head will be split, this equation will not hold - assert qkv_weight.shape[1] * qkv_weight.shape[2] == qkv_weight.shape[ - 3], "embed_dim must be divisible by num_heads." - - _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, cache_kv_out, final_out = _legacy_C_ops.fused_attention( - x, pre_ln_scale, pre_ln_bias, qkv_weight, qkv_bias, cache_kv, - attn_mask, linear_weight, linear_bias, ln_scale, ln_bias, - 'pre_layer_norm', pre_layer_norm, 'epsilon', pre_ln_epsilon, - 'dropout_rate', dropout_rate, 'attn_dropout_rate', - attn_dropout_rate, 'ln_epsilon', ln_epsilon, 'is_test', - not training, 'attn_dropout_fix_seed', seed is not None, - 'dropout_fix_seed', seed is not None, 'attn_dropout_seed', - seed if seed is not None else 0, 'dropout_seed', - seed if seed is not None else 0, 'attn_dropout_implementation', - mode, 'dropout_implementation', mode, 'add_residual', add_residual, - 'ring_id', ring_id) + assert ( + qkv_weight.shape[1] * qkv_weight.shape[2] == qkv_weight.shape[3] + ), "embed_dim must be divisible by num_heads." + + ( + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + cache_kv_out, + final_out, + ) = _legacy_C_ops.fused_attention( + x, + pre_ln_scale, + pre_ln_bias, + qkv_weight, + qkv_bias, + cache_kv, + attn_mask, + linear_weight, + linear_bias, + ln_scale, + ln_bias, + 'pre_layer_norm', + pre_layer_norm, + 'epsilon', + pre_ln_epsilon, + 'dropout_rate', + dropout_rate, + 'attn_dropout_rate', + attn_dropout_rate, + 'ln_epsilon', + ln_epsilon, + 'is_test', + not training, + 'attn_dropout_fix_seed', + seed is not None, + 'dropout_fix_seed', + seed is not None, + 'attn_dropout_seed', + seed if seed is not None else 0, + 'dropout_seed', + seed if seed is not None else 0, + 'attn_dropout_implementation', + mode, + 'dropout_implementation', + mode, + 'add_residual', + add_residual, + 'ring_id', + ring_id, + ) if cache_kv is not None: return final_out, cache_kv_out return final_out @@ -550,10 +699,18 @@ def fused_multi_head_attention(x, helper = LayerHelper('fused_multi_head_attention', **locals()) dtype = x.dtype # check dtypes - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'fused_multihead_attention') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], - 'fused_multi_head_attention') + check_variable_and_dtype( + x, + 'x', + ['float16', 'float32', 'float64'], + 'fused_multihead_attention', + ) + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'fused_multi_head_attention', + ) # set inputs inputs = dict() @@ -573,7 +730,8 @@ def fused_multi_head_attention(x, inputs['Ln2Scale'] = [ln_scale] if ln_bias: inputs['Ln2Bias'] = [ln_bias] - if cache_kv: inputs['CacheKV'] = [cache_kv] + if cache_kv: + inputs['CacheKV'] = [cache_kv] if (seed is None or seed == 0) and helper.main_program.random_seed != 0: seed = helper.main_program.random_seed @@ -593,14 +751,16 @@ def fused_multi_head_attention(x, 'attn_dropout_implementation': mode, 'dropout_implementation': mode, 'add_residual': add_residual, - 'ring_id': ring_id + 'ring_id': ring_id, } # set outputs pre_ln_mean_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) pre_ln_variance_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) pre_ln_out = helper.create_variable_for_type_inference(dtype=dtype) qkv_out = helper.create_variable_for_type_inference(dtype=dtype) @@ -611,79 +771,88 @@ def fused_multi_head_attention(x, qktv_out = helper.create_variable_for_type_inference(dtype=dtype) softmax_out = helper.create_variable_for_type_inference(dtype=dtype) attn_dropout_mask_out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) attn_dropout_out = helper.create_variable_for_type_inference( - dtype=dtype) + dtype=dtype + ) attn_mask_out = helper.create_variable_for_type_inference(dtype=dtype) fmha_out = helper.create_variable_for_type_inference(dtype=dtype) out_linear_out = helper.create_variable_for_type_inference(dtype=dtype) dropout_mask_out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) ln_mean_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) ln_variance_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) bias_dropout_residual_out = helper.create_variable_for_type_inference( - dtype=dtype) + dtype=dtype + ) final_out = helper.create_variable_for_type_inference(dtype=dtype) cache_kv_out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='fused_attention', - inputs=inputs, - outputs={ - "LnMean": pre_ln_mean_out, - "LnVariance": pre_ln_variance_out, - "LnOut": pre_ln_out, - "QKVOut": qkv_out, - "QKVBiasOut": qkv_bias_out, - "TransposeOut2": transpose_out, - "QKOut": qk_out, - "QKTVOut": qktv_out, - "SoftmaxOut": softmax_out, - "AttnDropoutMaskOut": attn_dropout_mask_out, - "AttnDropoutOut": attn_dropout_out, - "SrcMaskOut": attn_mask_out, - "FMHAOut": fmha_out, - "OutLinearOut": out_linear_out, - "DropoutMaskOut": dropout_mask_out, - "Ln2Mean": ln_mean_out, - "Ln2Variance": ln_variance_out, - "BiasDropoutResidualOut": - bias_dropout_residual_out, - 'Y': final_out, - 'CacheKVOut': cache_kv_out - }, - attrs=attrs) + helper.append_op( + type='fused_attention', + inputs=inputs, + outputs={ + "LnMean": pre_ln_mean_out, + "LnVariance": pre_ln_variance_out, + "LnOut": pre_ln_out, + "QKVOut": qkv_out, + "QKVBiasOut": qkv_bias_out, + "TransposeOut2": transpose_out, + "QKOut": qk_out, + "QKTVOut": qktv_out, + "SoftmaxOut": softmax_out, + "AttnDropoutMaskOut": attn_dropout_mask_out, + "AttnDropoutOut": attn_dropout_out, + "SrcMaskOut": attn_mask_out, + "FMHAOut": fmha_out, + "OutLinearOut": out_linear_out, + "DropoutMaskOut": dropout_mask_out, + "Ln2Mean": ln_mean_out, + "Ln2Variance": ln_variance_out, + "BiasDropoutResidualOut": bias_dropout_residual_out, + 'Y': final_out, + 'CacheKVOut': cache_kv_out, + }, + attrs=attrs, + ) return (final_out, cache_kv_out) if cache_kv else final_out -def fused_multi_transformer(x, - ln_scales, - ln_biases, - qkv_weights, - qkv_biases, - linear_weights, - linear_biases, - ffn_ln_scales, - ffn_ln_biases, - ffn1_weights, - ffn1_biases, - ffn2_weights, - ffn2_biases, - pre_layer_norm=True, - epsilon=1e-05, - cache_kvs=None, - pre_caches=None, - time_step=None, - attn_mask=None, - dropout_rate=0.0, - activation="gelu", - training=False, - mode='upscale_in_train', - trans_qkvw=True, - ring_id=-1, - name=None): +def fused_multi_transformer( + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + linear_weights, + linear_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + pre_layer_norm=True, + epsilon=1e-05, + cache_kvs=None, + pre_caches=None, + time_step=None, + attn_mask=None, + dropout_rate=0.0, + activation="gelu", + training=False, + mode='upscale_in_train', + trans_qkvw=True, + ring_id=-1, + name=None, +): r""" This is a fusion operator to compute multi transformer layers in transformer model architecture. This operator only supports running on GPU. The function of the transformer layer is consistent @@ -823,18 +992,47 @@ def fused_multi_transformer(x, raise ValueError( "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): cache_kv_out, final_out = _legacy_C_ops.fused_multi_transformer( - x, ln_scales, ln_biases, qkv_weights, qkv_biases, cache_kvs, - pre_caches, time_step, attn_mask, linear_weights, linear_biases, - ffn_ln_scales, ffn_ln_biases, ffn1_weights, ffn1_biases, - ffn2_weights, ffn2_biases, cache_kvs, 'pre_layer_norm', - pre_layer_norm, 'epsilon', epsilon, 'dropout_rate', dropout_rate, - 'is_test', not training, 'dropout_implementation', mode, - 'act_method', activation, 'trans_qkvw', trans_qkvw, 'ring_id', - ring_id) + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + cache_kvs, + pre_caches, + time_step, + attn_mask, + linear_weights, + linear_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + cache_kvs, + 'pre_layer_norm', + pre_layer_norm, + 'epsilon', + epsilon, + 'dropout_rate', + dropout_rate, + 'is_test', + not training, + 'dropout_implementation', + mode, + 'act_method', + activation, + 'trans_qkvw', + trans_qkvw, + 'ring_id', + ring_id, + ) if cache_kvs is not None: return final_out, cache_kv_out return final_out @@ -842,10 +1040,12 @@ def fused_multi_transformer(x, helper = LayerHelper('fused_multi_transformer', **locals()) dtype = x.dtype # check dtypes - check_variable_and_dtype(x, 'x', ['float16', 'float32'], - 'fused_multi_transformer') - check_dtype(dtype, 'dtype', ['float16', 'float32'], - 'fused_multi_transformer') + check_variable_and_dtype( + x, 'x', ['float16', 'float32'], 'fused_multi_transformer' + ) + check_dtype( + dtype, 'dtype', ['float16', 'float32'], 'fused_multi_transformer' + ) # set inputs inputs = dict() @@ -885,7 +1085,7 @@ def fused_multi_transformer(x, 'dropout_implementation': mode, 'act_method': activation, 'trans_qkvw': trans_qkvw, - 'ring_id': ring_id + 'ring_id': ring_id, } outputs = dict() @@ -895,9 +1095,11 @@ def fused_multi_transformer(x, # NOTE: inplace outputs['CacheKVOut'] = cache_kvs - helper.append_op(type='fused_multi_transformer', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='fused_multi_transformer', + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) return (final_out, cache_kvs) if cache_kvs else final_out diff --git a/python/paddle/incubate/nn/layer/fused_linear.py b/python/paddle/incubate/nn/layer/fused_linear.py index 4a5f2c12105..70a18233f78 100644 --- a/python/paddle/incubate/nn/layer/fused_linear.py +++ b/python/paddle/incubate/nn/layer/fused_linear.py @@ -66,30 +66,31 @@ class FusedLinear(Layer): print(y.shape) # [3, 5] """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - bias_attr=None, - transpose_weight=False, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + bias_attr=None, + transpose_weight=False, + name=None, + ): super(FusedLinear, self).__init__() if transpose_weight: weight_shape = [out_features, in_features] else: weight_shape = [in_features, out_features] dtype = self._helper.get_default_dtype() - self.weight = self.create_parameter(shape=weight_shape, - attr=weight_attr, - dtype=dtype, - is_bias=False) - self.bias = self.create_parameter(shape=[out_features], - attr=bias_attr, - dtype=dtype, - is_bias=True) + self.weight = self.create_parameter( + shape=weight_shape, attr=weight_attr, dtype=dtype, is_bias=False + ) + self.bias = self.create_parameter( + shape=[out_features], attr=bias_attr, dtype=dtype, is_bias=True + ) self.transpose_weight = transpose_weight self.name = name def forward(self, input): - return F.fused_linear(input, self.weight, self.bias, - self.transpose_weight, self.name) + return F.fused_linear( + input, self.weight, self.bias, self.transpose_weight, self.name + ) diff --git a/python/paddle/incubate/nn/layer/fused_transformer.py b/python/paddle/incubate/nn/layer/fused_transformer.py index ca58f81e19d..0f9b225eb6d 100644 --- a/python/paddle/incubate/nn/layer/fused_transformer.py +++ b/python/paddle/incubate/nn/layer/fused_transformer.py @@ -14,7 +14,10 @@ from paddle.incubate.nn import functional as incubate_f from paddle.nn import Layer import paddle -from paddle.nn.layer.transformer import _convert_attention_mask, _convert_param_attr_to_list +from paddle.nn.layer.transformer import ( + _convert_attention_mask, + _convert_param_attr_to_list, +) from paddle.nn.initializer import Constant from paddle.fluid.dygraph import no_grad from paddle.fluid.framework import convert_np_dtype_to_dtype_, _non_static_mode @@ -49,7 +52,8 @@ def _to_dtype(t, dtype): if t.place.is_gpu_place(): size_dtype = core.size_of_dtype(dtype) waiting_alloc_memory = ( - (np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 + ((np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 + ) gpu_memory_available = core.gpu_memory_available() if gpu_memory_available < waiting_alloc_memory: t_used = t._copy_to(paddle.CPUPlace(), False) @@ -104,31 +108,38 @@ class FusedBiasDropoutResidualLayerNorm(Layer): output = fused_bias_dropout_residual_ln(x, residual) # [2, 4, 128] """ - def __init__(self, - embed_dim, - dropout_rate=0.5, - weight_attr=None, - bias_attr=None, - epsilon=1e-5, - name=None): + def __init__( + self, + embed_dim, + dropout_rate=0.5, + weight_attr=None, + bias_attr=None, + epsilon=1e-5, + name=None, + ): super(FusedBiasDropoutResidualLayerNorm, self).__init__() - assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " - "but recieved {}".format(embed_dim)) + assert embed_dim > 0, ( + "Expected embed_dim to be greater than 0, " + "but recieved {}".format(embed_dim) + ) self._dtype = self._helper.get_default_dtype() self._bias_attr = bias_attr self._weight_attr = weight_attr self.embed_dim = embed_dim - self.linear_bias = self.create_parameter(shape=[embed_dim], - attr=self._bias_attr, - dtype=self._dtype, - is_bias=True) + self.linear_bias = self.create_parameter( + shape=[embed_dim], + attr=self._bias_attr, + dtype=self._dtype, + is_bias=True, + ) self.ln_scale = self.create_parameter( attr=self._weight_attr, shape=[embed_dim], - default_initializer=Constant(value=1.0)) - self.ln_bias = self.create_parameter(attr=self._bias_attr, - shape=[embed_dim], - is_bias=True) + default_initializer=Constant(value=1.0), + ) + self.ln_bias = self.create_parameter( + attr=self._bias_attr, shape=[embed_dim], is_bias=True + ) self.dropout_rate = dropout_rate self._epsilon = epsilon @@ -161,14 +172,20 @@ class FusedBiasDropoutResidualLayerNorm(Layer): ln_epsilon=self._epsilon, training=self.training, mode='upscale_in_train', - name=self.name) + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'embed_dim={}, seq_len={}, dropout_rate={}, epsilon={}, dtype={}{}'.format( - self.embed_dim, self.seq_len, self.dropout_rate, self._epsilon, - self._dtype, name_str) + self.embed_dim, + self.seq_len, + self.dropout_rate, + self._epsilon, + self._dtype, + name_str, + ) class FusedMultiHeadAttention(Layer): @@ -244,33 +261,40 @@ class FusedMultiHeadAttention(Layer): output = multi_head_attn(query, None, None, attn_mask=attn_mask) # [2, 4, 128] """ - def __init__(self, - embed_dim, - num_heads, - dropout_rate=0.5, - attn_dropout_rate=0.5, - kdim=None, - vdim=None, - normalize_before=False, - need_weights=False, - qkv_weight_attr=None, - qkv_bias_attr=None, - linear_weight_attr=None, - linear_bias_attr=None, - pre_ln_scale_attr=None, - pre_ln_bias_attr=None, - ln_scale_attr=None, - ln_bias_attr=None, - epsilon=1e-5, - nranks=1, - ring_id=-1, - name=None): + def __init__( + self, + embed_dim, + num_heads, + dropout_rate=0.5, + attn_dropout_rate=0.5, + kdim=None, + vdim=None, + normalize_before=False, + need_weights=False, + qkv_weight_attr=None, + qkv_bias_attr=None, + linear_weight_attr=None, + linear_bias_attr=None, + pre_ln_scale_attr=None, + pre_ln_bias_attr=None, + ln_scale_attr=None, + ln_bias_attr=None, + epsilon=1e-5, + nranks=1, + ring_id=-1, + name=None, + ): super(FusedMultiHeadAttention, self).__init__() - assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " - "but received {}".format(embed_dim)) - assert num_heads > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(num_heads)) + assert embed_dim > 0, ( + "Expected embed_dim to be greater than 0, " + "but received {}".format(embed_dim) + ) + assert ( + num_heads > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + num_heads + ) self.normalize_before = normalize_before self._dtype = self._helper.get_default_dtype() @@ -283,7 +307,9 @@ class FusedMultiHeadAttention(Layer): self.kdim = kdim self.vdim = vdim self.need_weights = need_weights - assert self.head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == embed_dim + ), "embed_dim must be divisible by num_heads" assert need_weights is False, "Only support need_weight is False now." # tensor model parallel @@ -294,21 +320,26 @@ class FusedMultiHeadAttention(Layer): shape=[3, num_heads, self.head_dim, embed_dim], attr=qkv_weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) self.qkv_bias = self.create_parameter( shape=[3, num_heads, self.head_dim], attr=qkv_bias_attr, dtype=self._dtype, - is_bias=True) + is_bias=True, + ) self.linear_weight = self.create_parameter( shape=[num_heads * self.head_dim, embed_dim], attr=linear_weight_attr, dtype=self._dtype, - is_bias=False) - self.linear_bias = self.create_parameter(shape=[embed_dim], - attr=linear_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + self.linear_bias = self.create_parameter( + shape=[embed_dim], + attr=linear_bias_attr, + dtype=self._dtype, + is_bias=True, + ) # tensor model parallel if nranks > 1: @@ -323,10 +354,11 @@ class FusedMultiHeadAttention(Layer): self.pre_ln_scale = self.create_parameter( attr=pre_ln_scale_attr, shape=[embed_dim], - default_initializer=Constant(value=1.0)) - self.pre_ln_bias = self.create_parameter(attr=pre_ln_bias_attr, - shape=[embed_dim], - is_bias=True) + default_initializer=Constant(value=1.0), + ) + self.pre_ln_bias = self.create_parameter( + attr=pre_ln_bias_attr, shape=[embed_dim], is_bias=True + ) self.ln_scale = None self.ln_bias = None else: @@ -335,10 +367,11 @@ class FusedMultiHeadAttention(Layer): self.ln_scale = self.create_parameter( attr=ln_scale_attr, shape=[embed_dim], - default_initializer=Constant(value=1.0)) - self.ln_bias = self.create_parameter(attr=ln_bias_attr, - shape=[embed_dim], - is_bias=True) + default_initializer=Constant(value=1.0), + ) + self.ln_bias = self.create_parameter( + attr=ln_bias_attr, shape=[embed_dim], is_bias=True + ) self.dropout_rate = dropout_rate self.attn_dropout_rate = attn_dropout_rate @@ -402,15 +435,25 @@ class FusedMultiHeadAttention(Layer): ln_epsilon=self._epsilon, training=self.training, ring_id=self._ring_id, - name=self.name) + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'embed_dim={}, num_heads={}, dropout_rate={}, attn_dropout_rate={}, epsilon={}, kdim={}, vdim={}, normalize_before={}, need_weights={}, dtype={}{}'.format( - self.embed_dim, self.num_heads, self.dropout_rate, - self.attn_dropout_rate, self._epsilon, self.kdim, self.vdim, - self.normalize_before, self.need_weights, self._dtype, name_str) + self.embed_dim, + self.num_heads, + self.dropout_rate, + self.attn_dropout_rate, + self._epsilon, + self.kdim, + self.vdim, + self.normalize_before, + self.need_weights, + self._dtype, + name_str, + ) def _amp_decorate(self, dtype): # tmp fix for amp.decorator(O2) @@ -493,33 +536,39 @@ class FusedFeedForward(Layer): # (1, 8, 8) """ - def __init__(self, - d_model, - dim_feedforward, - dropout_rate=0.1, - epsilon=1e-05, - activation="relu", - act_dropout_rate=None, - normalize_before=False, - linear1_weight_attr=None, - linear1_bias_attr=None, - linear2_weight_attr=None, - linear2_bias_attr=None, - ln1_scale_attr=None, - ln1_bias_attr=None, - ln2_scale_attr=None, - ln2_bias_attr=None, - nranks=1, - ring_id=-1, - name=None): + def __init__( + self, + d_model, + dim_feedforward, + dropout_rate=0.1, + epsilon=1e-05, + activation="relu", + act_dropout_rate=None, + normalize_before=False, + linear1_weight_attr=None, + linear1_bias_attr=None, + linear2_weight_attr=None, + linear2_bias_attr=None, + ln1_scale_attr=None, + ln1_bias_attr=None, + ln2_scale_attr=None, + ln2_bias_attr=None, + nranks=1, + ring_id=-1, + name=None, + ): super(FusedFeedForward, self).__init__() - assert d_model > 0, ( - "Expected d_model to be greater than 0, but received {}".format( - d_model)) - assert dim_feedforward > 0, ( - "Expected dim_feedforward to be greater than 0, but received {}". - format(dim_feedforward)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, but received {}".format( + d_model + ) + assert ( + dim_feedforward > 0 + ), "Expected dim_feedforward to be greater than 0, but received {}".format( + dim_feedforward + ) self._dtype = self._helper.get_default_dtype() self._d_model = d_model @@ -528,7 +577,9 @@ class FusedFeedForward(Layer): dim_feedforward = dim_feedforward // nranks self._dim_feedforward = dim_feedforward self._dropout_rate = dropout_rate - self._act_dropout_rate = dropout_rate if act_dropout_rate is None else act_dropout_rate + self._act_dropout_rate = ( + dropout_rate if act_dropout_rate is None else act_dropout_rate + ) self._act_method = activation self._normalize_before = normalize_before self._epsilon = epsilon @@ -538,22 +589,28 @@ class FusedFeedForward(Layer): shape=[d_model, dim_feedforward], attr=linear1_weight_attr, dtype=self._dtype, - is_bias=False) - self._linear1_bias = self.create_parameter(shape=[dim_feedforward], - attr=linear1_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + self._linear1_bias = self.create_parameter( + shape=[dim_feedforward], + attr=linear1_bias_attr, + dtype=self._dtype, + is_bias=True, + ) self._linear2_weight = self.create_parameter( shape=[dim_feedforward, d_model], attr=linear2_weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) - self._linear2_bias = self.create_parameter(shape=[d_model], - attr=linear2_bias_attr, - dtype=self._dtype, - is_bias=True) + self._linear2_bias = self.create_parameter( + shape=[d_model], + attr=linear2_bias_attr, + dtype=self._dtype, + is_bias=True, + ) if nranks > 1: assert ring_id != -1 @@ -567,10 +624,11 @@ class FusedFeedForward(Layer): shape=[d_model], attr=ln1_scale_attr, is_bias=False, - default_initializer=Constant(1.0)) - self._ln1_bias = self.create_parameter(shape=[d_model], - attr=ln1_bias_attr, - is_bias=True) + default_initializer=Constant(1.0), + ) + self._ln1_bias = self.create_parameter( + shape=[d_model], attr=ln1_bias_attr, is_bias=True + ) self._ln2_scale = None self._ln2_bias = None else: @@ -580,10 +638,11 @@ class FusedFeedForward(Layer): shape=[d_model], attr=ln2_scale_attr, is_bias=False, - default_initializer=Constant(1.0)) - self._ln2_bias = self.create_parameter(shape=[d_model], - attr=ln2_bias_attr, - is_bias=True) + default_initializer=Constant(1.0), + ) + self._ln2_bias = self.create_parameter( + shape=[d_model], attr=ln2_bias_attr, is_bias=True + ) self.name = name @@ -606,15 +665,23 @@ class FusedFeedForward(Layer): pre_layer_norm=self._normalize_before, training=self.training, ring_id=self._ring_id, - name=self.name) + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'd_model={}, dim_feedforward={}, dropout_rate={}, epsilon={}, activation={}, act_dropout_rate={}, normalize_before={}, dtype={}{}'.format( - self._d_model, self._dim_feedforward, self._dropout_rate, - self._epsilon, self._act_method, self._act_dropout_rate, - self._normalize_before, self._dtype, name_str) + self._d_model, + self._dim_feedforward, + self._dropout_rate, + self._epsilon, + self._act_method, + self._act_dropout_rate, + self._normalize_before, + self._dtype, + name_str, + ) def _amp_decorate(self, dtype): # tmp fix for amp.decorator(O2) @@ -694,31 +761,44 @@ class FusedTransformerEncoderLayer(Layer): enc_output = encoder_layer(enc_input, attn_mask) # [2, 4, 128] """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout_rate=0.1, - activation="relu", - attn_dropout_rate=None, - act_dropout_rate=None, - normalize_before=False, - weight_attr=None, - bias_attr=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout_rate=0.1, + activation="relu", + attn_dropout_rate=None, + act_dropout_rate=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 super(FusedTransformerEncoderLayer, self).__init__() - assert d_model > 0, ("Expected d_model to be greater than 0, " - "but received {}".format(d_model)) - assert nhead > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(nhead)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, " "but received {}".format( + d_model + ) + assert ( + nhead > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + nhead + ) assert dim_feedforward > 0, ( "Expected dim_feedforward to be greater than 0, " - "but received {}".format(dim_feedforward)) - attn_dropout_rate = dropout_rate if attn_dropout_rate is None else attn_dropout_rate - act_dropout_rate = dropout_rate if act_dropout_rate is None else act_dropout_rate + "but received {}".format(dim_feedforward) + ) + attn_dropout_rate = ( + dropout_rate if attn_dropout_rate is None else attn_dropout_rate + ) + act_dropout_rate = ( + dropout_rate if act_dropout_rate is None else act_dropout_rate + ) self.normalize_before = normalize_before weight_attrs = _convert_param_attr_to_list(weight_attr, 2) @@ -737,18 +817,21 @@ class FusedTransformerEncoderLayer(Layer): pre_ln_scale_attr=weight_attrs[0], pre_ln_bias_attr=bias_attrs[0], ln_scale_attr=weight_attrs[0], - ln_bias_attr=bias_attrs[0]) - - self.ffn = FusedFeedForward(d_model, - dim_feedforward, - dropout_rate=dropout_rate, - activation=activation, - act_dropout_rate=act_dropout_rate, - normalize_before=self.normalize_before, - linear1_weight_attr=weight_attrs[1], - linear1_bias_attr=bias_attrs[1], - linear2_weight_attr=weight_attrs[1], - linear2_bias_attr=bias_attrs[1]) + ln_bias_attr=bias_attrs[0], + ) + + self.ffn = FusedFeedForward( + d_model, + dim_feedforward, + dropout_rate=dropout_rate, + activation=activation, + act_dropout_rate=act_dropout_rate, + normalize_before=self.normalize_before, + linear1_weight_attr=weight_attrs[1], + linear1_bias_attr=bias_attrs[1], + linear2_weight_attr=weight_attrs[1], + linear2_bias_attr=bias_attrs[1], + ) def forward(self, src, src_mask=None, cache=None): """ @@ -784,9 +867,9 @@ class FusedTransformerEncoderLayer(Layer): if cache is None: attn_out = self.fused_attn(src, attn_mask=src_mask) else: - attn_out, incremental_cache = self.fused_attn(src, - attn_mask=src_mask, - cache=cache) + attn_out, incremental_cache = self.fused_attn( + src, attn_mask=src_mask, cache=cache + ) ffn_out = self.ffn(attn_out) @@ -887,21 +970,23 @@ class FusedTransformer(Layer): cross_attn_mask) # [2, 6, 128] """ - def __init__(self, - d_model=512, - nhead=8, - num_encoder_layers=6, - num_decoder_layers=6, - dim_feedforward=2048, - dropout=0.1, - activation="relu", - attn_dropout=None, - act_dropout=None, - normalize_before=False, - weight_attr=None, - bias_attr=None, - custom_encoder=None, - custom_decoder=None): + def __init__( + self, + d_model=512, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + attn_dropout=None, + act_dropout=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + custom_encoder=None, + custom_decoder=None, + ): super(fusedTransformer, self).__init__() raise NotImplementedError() @@ -1069,40 +1154,49 @@ class FusedMultiTransformer(Layer): enc_output = encoder_layers(enc_input, attn_mask) # [2, 4, 128] """ - def __init__(self, - embed_dim, - num_heads, - dim_feedforward, - dropout_rate=0.0, - activation="gelu", - normalize_before=True, - ln_scale_attrs=None, - ln_bias_attrs=None, - qkv_weight_attrs=None, - qkv_bias_attrs=None, - linear_weight_attrs=None, - linear_bias_attrs=None, - ffn_ln_scale_attrs=None, - ffn_ln_bias_attrs=None, - ffn1_weight_attrs=None, - ffn1_bias_attrs=None, - ffn2_weight_attrs=None, - ffn2_bias_attrs=None, - epsilon=1e-5, - num_layers=-1, - nranks=1, - trans_qkvw=True, - ring_id=-1, - name=None): + def __init__( + self, + embed_dim, + num_heads, + dim_feedforward, + dropout_rate=0.0, + activation="gelu", + normalize_before=True, + ln_scale_attrs=None, + ln_bias_attrs=None, + qkv_weight_attrs=None, + qkv_bias_attrs=None, + linear_weight_attrs=None, + linear_bias_attrs=None, + ffn_ln_scale_attrs=None, + ffn_ln_bias_attrs=None, + ffn1_weight_attrs=None, + ffn1_bias_attrs=None, + ffn2_weight_attrs=None, + ffn2_bias_attrs=None, + epsilon=1e-5, + num_layers=-1, + nranks=1, + trans_qkvw=True, + ring_id=-1, + name=None, + ): super(FusedMultiTransformer, self).__init__() - assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " - "but received {}".format(embed_dim)) - assert num_heads > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(num_heads)) - assert dim_feedforward > 0, ( - "Expected dim_feedforward to be greater than 0, but received {}". - format(dim_feedforward)) + assert embed_dim > 0, ( + "Expected embed_dim to be greater than 0, " + "but received {}".format(embed_dim) + ) + assert ( + num_heads > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + num_heads + ) + assert ( + dim_feedforward > 0 + ), "Expected dim_feedforward to be greater than 0, but received {}".format( + dim_feedforward + ) self.normalize_before = normalize_before self._dtype = self._helper.get_default_dtype() @@ -1113,7 +1207,9 @@ class FusedMultiTransformer(Layer): self.embed_dim = embed_dim self.num_heads = num_heads self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == embed_dim + ), "embed_dim must be divisible by num_heads" # tensor model parallel if nranks > 1: @@ -1159,57 +1255,71 @@ class FusedMultiTransformer(Layer): ln_scale = self.create_parameter( attr=ln_scale_attr, shape=[embed_dim], - default_initializer=Constant(value=1.0)) - ln_bias = self.create_parameter(attr=ln_bias_attr, - shape=[embed_dim], - is_bias=True) + default_initializer=Constant(value=1.0), + ) + ln_bias = self.create_parameter( + attr=ln_bias_attr, shape=[embed_dim], is_bias=True + ) qkv_weight = self.create_parameter( shape=[3, num_heads, self.head_dim, embed_dim] - if trans_qkvw else [embed_dim, 3, num_heads, self.head_dim], + if trans_qkvw + else [embed_dim, 3, num_heads, self.head_dim], attr=qkv_weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) qkv_bias = self.create_parameter( shape=[3, num_heads, self.head_dim], attr=qkv_bias_attr, dtype=self._dtype, - is_bias=True) + is_bias=True, + ) linear_weight = self.create_parameter( shape=[num_heads * self.head_dim, embed_dim], attr=linear_weight_attr, dtype=self._dtype, - is_bias=False) - linear_bias = self.create_parameter(shape=[embed_dim], - attr=linear_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + linear_bias = self.create_parameter( + shape=[embed_dim], + attr=linear_bias_attr, + dtype=self._dtype, + is_bias=True, + ) ffn_ln_scale = self.create_parameter( shape=[embed_dim], attr=ffn_ln_scale_attr, is_bias=False, - default_initializer=Constant(1.0)) - ffn_ln_bias = self.create_parameter(shape=[embed_dim], - attr=ffn_ln_bias_attr, - is_bias=True) + default_initializer=Constant(1.0), + ) + ffn_ln_bias = self.create_parameter( + shape=[embed_dim], attr=ffn_ln_bias_attr, is_bias=True + ) ffn1_weight = self.create_parameter( shape=[embed_dim, dim_feedforward], attr=ffn1_weight_attr, dtype=self._dtype, - is_bias=False) - ffn1_bias = self.create_parameter(shape=[dim_feedforward], - attr=ffn1_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + ffn1_bias = self.create_parameter( + shape=[dim_feedforward], + attr=ffn1_bias_attr, + dtype=self._dtype, + is_bias=True, + ) ffn2_weight = self.create_parameter( shape=[dim_feedforward, embed_dim], attr=ffn2_weight_attr, dtype=self._dtype, - is_bias=False) - ffn2_bias = self.create_parameter(shape=[embed_dim], - attr=ffn2_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + ffn2_bias = self.create_parameter( + shape=[embed_dim], + attr=ffn2_bias_attr, + dtype=self._dtype, + is_bias=True, + ) # tensor model parallel if nranks > 1: @@ -1240,12 +1350,9 @@ class FusedMultiTransformer(Layer): self.activation = activation self.name = name - def forward(self, - src, - attn_mask=None, - caches=None, - pre_caches=None, - time_step=None): + def forward( + self, src, attn_mask=None, caches=None, pre_caches=None, time_step=None + ): r""" Applies multi transformer layers on the input. @@ -1306,5 +1413,6 @@ class FusedMultiTransformer(Layer): mode='upscale_in_train', trans_qkvw=self._trans_qkvw, ring_id=self._ring_id, - name=self.name) + name=self.name, + ) return out diff --git a/python/paddle/incubate/operators/__init__.py b/python/paddle/incubate/operators/__init__.py index bc4ba8c3890..eb105a12e1a 100644 --- a/python/paddle/incubate/operators/__init__.py +++ b/python/paddle/incubate/operators/__init__.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .softmax_mask_fuse_upper_triangle import softmax_mask_fuse_upper_triangle # noqa: F401 +from .softmax_mask_fuse_upper_triangle import ( + softmax_mask_fuse_upper_triangle, +) # noqa: F401 from .softmax_mask_fuse import softmax_mask_fuse # noqa: F401 -from .resnet_unit import ResNetUnit #noqa: F401 -from .graph_send_recv import graph_send_recv #noqa: F401 -from .graph_khop_sampler import graph_khop_sampler #noqa: F401 -from .graph_sample_neighbors import graph_sample_neighbors #noqa: F401 -from .graph_reindex import graph_reindex #noqa: F401 +from .resnet_unit import ResNetUnit # noqa: F401 +from .graph_send_recv import graph_send_recv # noqa: F401 +from .graph_khop_sampler import graph_khop_sampler # noqa: F401 +from .graph_sample_neighbors import graph_sample_neighbors # noqa: F401 +from .graph_reindex import graph_reindex # noqa: F401 diff --git a/python/paddle/incubate/operators/graph_khop_sampler.py b/python/paddle/incubate/operators/graph_khop_sampler.py index 21223833813..821c4b418ed 100644 --- a/python/paddle/incubate/operators/graph_khop_sampler.py +++ b/python/paddle/incubate/operators/graph_khop_sampler.py @@ -18,13 +18,15 @@ from paddle.fluid.data_feeder import check_variable_and_dtype from paddle import _legacy_C_ops -def graph_khop_sampler(row, - colptr, - input_nodes, - sample_sizes, - sorted_eids=None, - return_eids=False, - name=None): +def graph_khop_sampler( + row, + colptr, + input_nodes, + sample_sizes, + sorted_eids=None, + return_eids=False, + name=None, +): """ Graph Khop Sampler API. @@ -86,36 +88,64 @@ def graph_khop_sampler(row, if _non_static_mode(): if return_eids: if sorted_eids is None: - raise ValueError("`sorted_eid` should not be None " - "if return_eids is True.") - edge_src, edge_dst, sample_index, reindex_nodes, edge_eids = \ - _legacy_C_ops.graph_khop_sampler(row, sorted_eids, - colptr, input_nodes, - "sample_sizes", sample_sizes, - "return_eids", True) + raise ValueError( + "`sorted_eid` should not be None " "if return_eids is True." + ) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + edge_eids, + ) = _legacy_C_ops.graph_khop_sampler( + row, + sorted_eids, + colptr, + input_nodes, + "sample_sizes", + sample_sizes, + "return_eids", + True, + ) return edge_src, edge_dst, sample_index, reindex_nodes, edge_eids else: - edge_src, edge_dst, sample_index, reindex_nodes, _ = \ - _legacy_C_ops.graph_khop_sampler(row, None, - colptr, input_nodes, - "sample_sizes", sample_sizes, - "return_eids", False) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + _, + ) = _legacy_C_ops.graph_khop_sampler( + row, + None, + colptr, + input_nodes, + "sample_sizes", + sample_sizes, + "return_eids", + False, + ) return edge_src, edge_dst, sample_index, reindex_nodes - check_variable_and_dtype(row, "Row", ("int32", "int64"), - "graph_khop_sampler") + check_variable_and_dtype( + row, "Row", ("int32", "int64"), "graph_khop_sampler" + ) if return_eids: if sorted_eids is None: - raise ValueError("`sorted_eid` should not be None " - "if return_eids is True.") - check_variable_and_dtype(sorted_eids, "Eids", ("int32", "int64"), - "graph_khop_sampler") - - check_variable_and_dtype(colptr, "Col_Ptr", ("int32", "int64"), - "graph_khop_sampler") - check_variable_and_dtype(input_nodes, "X", ("int32", "int64"), - "graph_khop_sampler") + raise ValueError( + "`sorted_eid` should not be None " "if return_eids is True." + ) + check_variable_and_dtype( + sorted_eids, "Eids", ("int32", "int64"), "graph_khop_sampler" + ) + + check_variable_and_dtype( + colptr, "Col_Ptr", ("int32", "int64"), "graph_khop_sampler" + ) + check_variable_and_dtype( + input_nodes, "X", ("int32", "int64"), "graph_khop_sampler" + ) helper = LayerHelper("graph_khop_sampler", **locals()) edge_src = helper.create_variable_for_type_inference(dtype=row.dtype) @@ -123,24 +153,23 @@ def graph_khop_sampler(row, sample_index = helper.create_variable_for_type_inference(dtype=row.dtype) reindex_nodes = helper.create_variable_for_type_inference(dtype=row.dtype) edge_eids = helper.create_variable_for_type_inference(dtype=row.dtype) - helper.append_op(type="graph_khop_sampler", - inputs={ - "Row": row, - "Eids": sorted_eids, - "Col_Ptr": colptr, - "X": input_nodes - }, - outputs={ - "Out_Src": edge_src, - "Out_Dst": edge_dst, - "Sample_Index": sample_index, - "Reindex_X": reindex_nodes, - "Out_Eids": edge_eids - }, - attrs={ - "sample_sizes": sample_sizes, - "return_eids": return_eids - }) + helper.append_op( + type="graph_khop_sampler", + inputs={ + "Row": row, + "Eids": sorted_eids, + "Col_Ptr": colptr, + "X": input_nodes, + }, + outputs={ + "Out_Src": edge_src, + "Out_Dst": edge_dst, + "Sample_Index": sample_index, + "Reindex_X": reindex_nodes, + "Out_Eids": edge_eids, + }, + attrs={"sample_sizes": sample_sizes, "return_eids": return_eids}, + ) if return_eids: return edge_src, edge_dst, sample_index, reindex_nodes, edge_eids else: diff --git a/python/paddle/incubate/operators/graph_reindex.py b/python/paddle/incubate/operators/graph_reindex.py index d8f67741908..d721c9a002e 100644 --- a/python/paddle/incubate/operators/graph_reindex.py +++ b/python/paddle/incubate/operators/graph_reindex.py @@ -19,17 +19,21 @@ from paddle import _legacy_C_ops import paddle.utils.deprecated as deprecated -@deprecated(since="2.4.0", - update_to="paddle.geometric.reindex_graph", - level=1, - reason="paddle.incubate.graph_reindex will be removed in future") -def graph_reindex(x, - neighbors, - count, - value_buffer=None, - index_buffer=None, - flag_buffer_hashtable=False, - name=None): +@deprecated( + since="2.4.0", + update_to="paddle.geometric.reindex_graph", + level=1, + reason="paddle.incubate.graph_reindex will be removed in future", +) +def graph_reindex( + x, + neighbors, + count, + value_buffer=None, + index_buffer=None, + flag_buffer_hashtable=False, + name=None, +): """ Graph Reindex API. @@ -107,47 +111,55 @@ def graph_reindex(x, """ if flag_buffer_hashtable: if value_buffer is None or index_buffer is None: - raise ValueError("`value_buffer` and `index_buffer` should not" - "be None if `flag_buffer_hashtable` is True.") + raise ValueError( + "`value_buffer` and `index_buffer` should not" + "be None if `flag_buffer_hashtable` is True." + ) if _non_static_mode(): - reindex_src, reindex_dst, out_nodes = \ - _legacy_C_ops.graph_reindex(x, neighbors, count, value_buffer, index_buffer, - "flag_buffer_hashtable", flag_buffer_hashtable) + reindex_src, reindex_dst, out_nodes = _legacy_C_ops.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + "flag_buffer_hashtable", + flag_buffer_hashtable, + ) return reindex_src, reindex_dst, out_nodes check_variable_and_dtype(x, "X", ("int32", "int64"), "graph_reindex") - check_variable_and_dtype(neighbors, "Neighbors", ("int32", "int64"), - "graph_reindex") + check_variable_and_dtype( + neighbors, "Neighbors", ("int32", "int64"), "graph_reindex" + ) check_variable_and_dtype(count, "Count", ("int32"), "graph_reindex") if flag_buffer_hashtable: - check_variable_and_dtype(value_buffer, "HashTable_Value", ("int32"), - "graph_reindex") - check_variable_and_dtype(index_buffer, "HashTable_Index", ("int32"), - "graph_reindex") + check_variable_and_dtype( + value_buffer, "HashTable_Value", ("int32"), "graph_reindex" + ) + check_variable_and_dtype( + index_buffer, "HashTable_Index", ("int32"), "graph_reindex" + ) helper = LayerHelper("graph_reindex", **locals()) reindex_src = helper.create_variable_for_type_inference(dtype=x.dtype) reindex_dst = helper.create_variable_for_type_inference(dtype=x.dtype) out_nodes = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="graph_reindex", - inputs={ - "X": - x, - "Neighbors": - neighbors, - "Count": - count, - "HashTable_Value": - value_buffer if flag_buffer_hashtable else None, - "HashTable_Index": - index_buffer if flag_buffer_hashtable else None, - }, - outputs={ - "Reindex_Src": reindex_src, - "Reindex_Dst": reindex_dst, - "Out_Nodes": out_nodes - }, - attrs={"flag_buffer_hashtable": flag_buffer_hashtable}) + helper.append_op( + type="graph_reindex", + inputs={ + "X": x, + "Neighbors": neighbors, + "Count": count, + "HashTable_Value": value_buffer if flag_buffer_hashtable else None, + "HashTable_Index": index_buffer if flag_buffer_hashtable else None, + }, + outputs={ + "Reindex_Src": reindex_src, + "Reindex_Dst": reindex_dst, + "Out_Nodes": out_nodes, + }, + attrs={"flag_buffer_hashtable": flag_buffer_hashtable}, + ) return reindex_src, reindex_dst, out_nodes diff --git a/python/paddle/incubate/operators/graph_sample_neighbors.py b/python/paddle/incubate/operators/graph_sample_neighbors.py index b39910f9e11..48e52bc6914 100644 --- a/python/paddle/incubate/operators/graph_sample_neighbors.py +++ b/python/paddle/incubate/operators/graph_sample_neighbors.py @@ -23,16 +23,19 @@ import paddle.utils.deprecated as deprecated since="2.4.0", update_to="paddle.geometric.sample_neighbors", level=1, - reason="paddle.incubate.graph_sample_neighbors will be removed in future") -def graph_sample_neighbors(row, - colptr, - input_nodes, - eids=None, - perm_buffer=None, - sample_size=-1, - return_eids=False, - flag_perm_buffer=False, - name=None): + reason="paddle.incubate.graph_sample_neighbors will be removed in future", +) +def graph_sample_neighbors( + row, + colptr, + input_nodes, + eids=None, + perm_buffer=None, + sample_size=-1, + return_eids=False, + flag_perm_buffer=False, + name=None, +): """ Graph Sample Neighbors API. @@ -96,59 +99,83 @@ def graph_sample_neighbors(row, if return_eids: if eids is None: raise ValueError( - "`eids` should not be None if `return_eids` is True.") + "`eids` should not be None if `return_eids` is True." + ) if flag_perm_buffer: if perm_buffer is None: raise ValueError( "`perm_buffer` should not be None if `flag_perm_buffer`" - "is True.") + "is True." + ) if _non_static_mode(): - out_neighbors, out_count, out_eids = _legacy_C_ops.graph_sample_neighbors( - row, colptr, input_nodes, eids, perm_buffer, "sample_size", - sample_size, "return_eids", return_eids, "flag_perm_buffer", - flag_perm_buffer) + ( + out_neighbors, + out_count, + out_eids, + ) = _legacy_C_ops.graph_sample_neighbors( + row, + colptr, + input_nodes, + eids, + perm_buffer, + "sample_size", + sample_size, + "return_eids", + return_eids, + "flag_perm_buffer", + flag_perm_buffer, + ) if return_eids: return out_neighbors, out_count, out_eids return out_neighbors, out_count - check_variable_and_dtype(row, "Row", ("int32", "int64"), - "graph_sample_neighbors") - check_variable_and_dtype(colptr, "Col_Ptr", ("int32", "int64"), - "graph_sample_neighbors") - check_variable_and_dtype(input_nodes, "X", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + row, "Row", ("int32", "int64"), "graph_sample_neighbors" + ) + check_variable_and_dtype( + colptr, "Col_Ptr", ("int32", "int64"), "graph_sample_neighbors" + ) + check_variable_and_dtype( + input_nodes, "X", ("int32", "int64"), "graph_sample_neighbors" + ) if return_eids: - check_variable_and_dtype(eids, "Eids", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + eids, "Eids", ("int32", "int64"), "graph_sample_neighbors" + ) if flag_perm_buffer: - check_variable_and_dtype(perm_buffer, "Perm_Buffer", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + perm_buffer, + "Perm_Buffer", + ("int32", "int64"), + "graph_sample_neighbors", + ) helper = LayerHelper("graph_sample_neighbors", **locals()) out_neighbors = helper.create_variable_for_type_inference(dtype=row.dtype) out_count = helper.create_variable_for_type_inference(dtype=row.dtype) out_eids = helper.create_variable_for_type_inference(dtype=row.dtype) - helper.append_op(type="graph_sample_neighbors", - inputs={ - "Row": row, - "Col_Ptr": colptr, - "X": input_nodes, - "Eids": eids if return_eids else None, - "Perm_Buffer": - perm_buffer if flag_perm_buffer else None - }, - outputs={ - "Out": out_neighbors, - "Out_Count": out_count, - "Out_Eids": out_eids - }, - attrs={ - "sample_size": sample_size, - "return_eids": return_eids, - "flag_perm_buffer": flag_perm_buffer - }) + helper.append_op( + type="graph_sample_neighbors", + inputs={ + "Row": row, + "Col_Ptr": colptr, + "X": input_nodes, + "Eids": eids if return_eids else None, + "Perm_Buffer": perm_buffer if flag_perm_buffer else None, + }, + outputs={ + "Out": out_neighbors, + "Out_Count": out_count, + "Out_Eids": out_eids, + }, + attrs={ + "sample_size": sample_size, + "return_eids": return_eids, + "flag_perm_buffer": flag_perm_buffer, + }, + ) if return_eids: return out_neighbors, out_count, out_eids return out_neighbors, out_count diff --git a/python/paddle/incubate/operators/graph_send_recv.py b/python/paddle/incubate/operators/graph_send_recv.py index 939d8c343fd..88fc421f4a0 100644 --- a/python/paddle/incubate/operators/graph_send_recv.py +++ b/python/paddle/incubate/operators/graph_send_recv.py @@ -16,7 +16,12 @@ import numpy as np from paddle.fluid.layer_helper import LayerHelper from paddle.fluid.framework import _in_legacy_dygraph, in_dygraph_mode from paddle.fluid.framework import Variable -from paddle.fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from paddle.fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from paddle.fluid.layers.tensor import cast from paddle import _C_ops, _legacy_C_ops import paddle.utils.deprecated as deprecated @@ -26,13 +31,11 @@ import paddle.utils.deprecated as deprecated since="2.4.0", update_to="paddle.geometric.send_u_recv", level=1, - reason="graph_send_recv in paddle.incubate will be removed in future") -def graph_send_recv(x, - src_index, - dst_index, - pool_type="sum", - out_size=None, - name=None): + reason="graph_send_recv in paddle.incubate will be removed in future", +) +def graph_send_recv( + x, src_index, dst_index, pool_type="sum", out_size=None, name=None +): r""" Graph Learning Send_Recv combine operator. @@ -116,54 +119,69 @@ def graph_send_recv(x, if pool_type not in ["sum", "mean", "max", "min"]: raise ValueError( "pool_type should be `sum`, `mean`, `max` or `min`, but received %s" - % pool_type) + % pool_type + ) # TODO(daisiming): Should we add judgement for out_size: max(dst_index) + 1. if _in_legacy_dygraph(): out_size = convert_out_size_to_list(out_size) - out, tmp = _legacy_C_ops.graph_send_recv(x, src_index, dst_index, - None, 'reduce_op', - pool_type.upper(), 'out_size', - out_size) + out, tmp = _legacy_C_ops.graph_send_recv( + x, + src_index, + dst_index, + None, + 'reduce_op', + pool_type.upper(), + 'out_size', + out_size, + ) return out if in_dygraph_mode(): out_size = convert_out_size_to_list(out_size) - return _C_ops.graph_send_recv(x, src_index, dst_index, - pool_type.upper(), out_size) - - check_variable_and_dtype(x, "X", ("float32", "float64", "int32", "int64"), - "graph_send_recv") - check_variable_and_dtype(src_index, "Src_index", ("int32", "int64"), - "graph_send_recv") - check_variable_and_dtype(dst_index, "Dst_index", ("int32", "int64"), - "graph_send_recv") + return _C_ops.graph_send_recv( + x, src_index, dst_index, pool_type.upper(), out_size + ) + + check_variable_and_dtype( + x, "X", ("float32", "float64", "int32", "int64"), "graph_send_recv" + ) + check_variable_and_dtype( + src_index, "Src_index", ("int32", "int64"), "graph_send_recv" + ) + check_variable_and_dtype( + dst_index, "Dst_index", ("int32", "int64"), "graph_send_recv" + ) if out_size: - check_type(out_size, 'out_size', (int, np.int32, np.int64, Variable), - 'graph_send_recv') + check_type( + out_size, + 'out_size', + (int, np.int32, np.int64, Variable), + 'graph_send_recv', + ) if isinstance(out_size, Variable): - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], - 'graph_send_recv') + check_dtype( + out_size.dtype, 'out_size', ['int32', 'int64'], 'graph_send_recv' + ) helper = LayerHelper("graph_send_recv", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - dst_count = helper.create_variable_for_type_inference(dtype="int32", - stop_gradient=True) + dst_count = helper.create_variable_for_type_inference( + dtype="int32", stop_gradient=True + ) inputs = {"X": x, "Src_index": src_index, "Dst_index": dst_index} attrs = {"reduce_op": pool_type.upper()} - get_out_size_tensor_inputs(inputs=inputs, - attrs=attrs, - out_size=out_size, - op_type='graph_send_recv') - - helper.append_op(type="graph_send_recv", - inputs=inputs, - outputs={ - "Out": out, - "Dst_count": dst_count - }, - attrs=attrs) + get_out_size_tensor_inputs( + inputs=inputs, attrs=attrs, out_size=out_size, op_type='graph_send_recv' + ) + + helper.append_op( + type="graph_send_recv", + inputs=inputs, + outputs={"Out": out, "Dst_count": dst_count}, + attrs=attrs, + ) return out @@ -192,9 +210,14 @@ def get_out_size_tensor_inputs(inputs, attrs, out_size, op_type): attrs['out_size'] = [out_size] elif isinstance(out_size, Variable): out_size.stop_gradient = True - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], op_type, - '(When type of out_size in' + op_type + ' is Variable.)') - if (convert_dtype(out_size.dtype) == 'int64'): + check_dtype( + out_size.dtype, + 'out_size', + ['int32', 'int64'], + op_type, + '(When type of out_size in' + op_type + ' is Variable.)', + ) + if convert_dtype(out_size.dtype) == 'int64': out_size = cast(out_size, 'int32') inputs["Out_size"] = out_size else: diff --git a/python/paddle/incubate/operators/resnet_unit.py b/python/paddle/incubate/operators/resnet_unit.py index c8fb583bff4..601bbfd11de 100644 --- a/python/paddle/incubate/operators/resnet_unit.py +++ b/python/paddle/incubate/operators/resnet_unit.py @@ -21,37 +21,77 @@ from paddle.fluid.layer_helper import LayerHelper from paddle.fluid.param_attr import ParamAttr -def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, - scale_z, bias_z, mean_z, var_z, stride, stride_z, padding, - dilation, groups, momentum, eps, data_format, fuse_add, - has_shortcut, use_global_stats, is_test, act): +def resnet_unit( + x, + filter_x, + scale_x, + bias_x, + mean_x, + var_x, + z, + filter_z, + scale_z, + bias_z, + mean_z, + var_z, + stride, + stride_z, + padding, + dilation, + groups, + momentum, + eps, + data_format, + fuse_add, + has_shortcut, + use_global_stats, + is_test, + act, +): helper = LayerHelper('resnet_unit', **locals()) bn_param_dtype = fluid.core.VarDesc.VarType.FP32 bit_mask_dtype = fluid.core.VarDesc.VarType.INT32 out = helper.create_variable_for_type_inference(x.dtype) - bit_mask = helper.create_variable_for_type_inference(dtype=bit_mask_dtype, - stop_gradient=True) + bit_mask = helper.create_variable_for_type_inference( + dtype=bit_mask_dtype, stop_gradient=True + ) # intermediate_out for x - conv_x = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + conv_x = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean_x = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd_x = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) running_mean_x = mean_x running_var_x = var_x # intermediate_out for z - conv_z = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + conv_z = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean_z = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd_z = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) - running_mean_z = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if mean_z is None else mean_z - running_var_z = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if var_z is None else var_z + dtype=bn_param_dtype, stop_gradient=True + ) + running_mean_z = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if mean_z is None + else mean_z + ) + running_var_z = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if var_z is None + else var_z + ) inputs = { 'X': x, @@ -65,7 +105,7 @@ def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, 'ScaleZ': scale_z, 'BiasZ': bias_z, 'MeanZ': mean_z, - 'VarZ': var_z + 'VarZ': var_z, } attrs = { @@ -81,7 +121,7 @@ def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, 'has_shortcut': has_shortcut, 'use_global_stats': use_global_stats, 'is_test': is_test, - 'act_type': act + 'act_type': act, } outputs = { @@ -99,10 +139,9 @@ def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, 'RunningVarZ': running_var_z, } - helper.append_op(type='resnet_unit', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='resnet_unit', inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -113,31 +152,33 @@ class ResNetUnit(Layer): ResNetUnit is designed for optimize the performence by using cudnnv8 API. """ - def __init__(self, - num_channels_x, - num_filters, - filter_size, - stride=1, - momentum=0.9, - eps=1e-5, - data_format='NHWC', - act='relu', - fuse_add=False, - has_shortcut=False, - use_global_stats=False, - is_test=False, - filter_x_attr=None, - scale_x_attr=None, - bias_x_attr=None, - moving_mean_x_name=None, - moving_var_x_name=None, - num_channels_z=1, - stride_z=1, - filter_z_attr=None, - scale_z_attr=None, - bias_z_attr=None, - moving_mean_z_name=None, - moving_var_z_name=None): + def __init__( + self, + num_channels_x, + num_filters, + filter_size, + stride=1, + momentum=0.9, + eps=1e-5, + data_format='NHWC', + act='relu', + fuse_add=False, + has_shortcut=False, + use_global_stats=False, + is_test=False, + filter_x_attr=None, + scale_x_attr=None, + bias_x_attr=None, + moving_mean_x_name=None, + moving_var_x_name=None, + num_channels_z=1, + stride_z=1, + filter_z_attr=None, + scale_z_attr=None, + bias_z_attr=None, + moving_mean_z_name=None, + moving_var_z_name=None, + ): super(ResNetUnit, self).__init__() self._stride = stride self._stride_z = stride_z @@ -158,89 +199,124 @@ class ResNetUnit(Layer): valid_format = {'NHWC', 'NCHW'} if data_format not in valid_format: raise ValueError( - "conv_format must be one of {}, but got conv_format='{}'". - format(valid_format, data_format)) + "conv_format must be one of {}, but got conv_format='{}'".format( + valid_format, data_format + ) + ) def _get_default_param_initializer(channels): filter_elem_num = np.prod(self._kernel_size) * channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return I.Normal(0.0, std) - is_nchw = (data_format == 'NCHW') + is_nchw = data_format == 'NCHW' # initial filter bn_param_dtype = fluid.core.VarDesc.VarType.FP32 if not is_nchw: bn_param_shape = [1, 1, 1, num_filters] filter_x_shape = [ - num_filters, filter_size, filter_size, num_channels_x + num_filters, + filter_size, + filter_size, + num_channels_x, ] filter_z_shape = [ - num_filters, filter_size, filter_size, num_channels_z + num_filters, + filter_size, + filter_size, + num_channels_z, ] else: bn_param_shape = [1, num_filters, 1, 1] filter_x_shape = [ - num_filters, num_channels_x, filter_size, filter_size + num_filters, + num_channels_x, + filter_size, + filter_size, ] filter_z_shape = [ - num_filters, num_channels_z, filter_size, filter_size + num_filters, + num_channels_z, + filter_size, + filter_size, ] self.filter_x = self.create_parameter( shape=filter_x_shape, attr=filter_x_attr, - default_initializer=_get_default_param_initializer(num_channels_x)) + default_initializer=_get_default_param_initializer(num_channels_x), + ) self.scale_x = self.create_parameter( shape=bn_param_shape, attr=scale_x_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_x = self.create_parameter(shape=bn_param_shape, - attr=bias_x_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_x = self.create_parameter(attr=ParamAttr( - name=moving_mean_x_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_x = self.create_parameter( + shape=bn_param_shape, + attr=bias_x_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_x = self.create_parameter( + attr=ParamAttr( + name=moving_mean_x_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn_param_shape, + dtype=bn_param_dtype, + ) self.mean_x.stop_gradient = True - self.var_x = self.create_parameter(attr=ParamAttr( - name=moving_var_x_name, - initializer=I.Constant(1.0), - trainable=False), - shape=bn_param_shape, - dtype=bn_param_dtype) + self.var_x = self.create_parameter( + attr=ParamAttr( + name=moving_var_x_name, + initializer=I.Constant(1.0), + trainable=False, + ), + shape=bn_param_shape, + dtype=bn_param_dtype, + ) self.var_x.stop_gradient = True if has_shortcut: self.filter_z = self.create_parameter( shape=filter_z_shape, attr=filter_z_attr, default_initializer=_get_default_param_initializer( - num_channels_z)) + num_channels_z + ), + ) self.scale_z = self.create_parameter( shape=bn_param_shape, attr=scale_z_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_z = self.create_parameter(shape=bn_param_shape, - attr=bias_z_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_z = self.create_parameter(attr=ParamAttr( - name=moving_mean_z_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_z = self.create_parameter( + shape=bn_param_shape, + attr=bias_z_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_z = self.create_parameter( + attr=ParamAttr( + name=moving_mean_z_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn_param_shape, + dtype=bn_param_dtype, + ) self.mean_z.stop_gradient = True - self.var_z = self.create_parameter(attr=ParamAttr( - name=moving_var_z_name, - initializer=I.Constant(1.0), - trainable=False), - shape=bn_param_shape, - dtype=bn_param_dtype) + self.var_z = self.create_parameter( + attr=ParamAttr( + name=moving_var_z_name, + initializer=I.Constant(1.0), + trainable=False, + ), + shape=bn_param_shape, + dtype=bn_param_dtype, + ) self.var_z.stop_gradient = True else: self.filter_z = None @@ -253,12 +329,31 @@ class ResNetUnit(Layer): if self._fuse_add and z is None: raise ValueError("z can not be None") - out = resnet_unit(x, self.filter_x, self.scale_x, self.bias_x, - self.mean_x, self.var_x, z, self.filter_z, - self.scale_z, self.bias_z, self.mean_z, self.var_z, - self._stride, self._stride_z, self._padding, - self._dilation, self._groups, self._momentum, - self._eps, self._data_format, self._fuse_add, - self._has_shortcut, self._use_global_stats, - self._is_test, self._act) + out = resnet_unit( + x, + self.filter_x, + self.scale_x, + self.bias_x, + self.mean_x, + self.var_x, + z, + self.filter_z, + self.scale_z, + self.bias_z, + self.mean_z, + self.var_z, + self._stride, + self._stride_z, + self._padding, + self._dilation, + self._groups, + self._momentum, + self._eps, + self._data_format, + self._fuse_add, + self._has_shortcut, + self._use_global_stats, + self._is_test, + self._act, + ) return out diff --git a/python/paddle/incubate/operators/softmax_mask_fuse.py b/python/paddle/incubate/operators/softmax_mask_fuse.py index 27afabaa0db..b8081af826d 100644 --- a/python/paddle/incubate/operators/softmax_mask_fuse.py +++ b/python/paddle/incubate/operators/softmax_mask_fuse.py @@ -60,10 +60,9 @@ def softmax_mask_fuse(x, mask, name=None): return out helper = LayerHelper('fused_softmax_mask', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='fused_softmax_mask', - inputs={ - 'X': [x], - 'Mask': [mask] - }, - outputs={'Out': [out]}) + helper.append_op( + type='fused_softmax_mask', + inputs={'X': [x], 'Mask': [mask]}, + outputs={'Out': [out]}, + ) return out diff --git a/python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py b/python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py index d73b8ba6d65..aab40986bef 100644 --- a/python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py +++ b/python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py @@ -63,7 +63,9 @@ def softmax_mask_fuse_upper_triangle(x): out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='fused_softmax_mask_upper_triangle', - inputs={'X': [x]}, - outputs={'Out': [out]}) + helper.append_op( + type='fused_softmax_mask_upper_triangle', + inputs={'X': [x]}, + outputs={'Out': [out]}, + ) return out diff --git a/python/paddle/incubate/optimizer/distributed_fused_lamb.py b/python/paddle/incubate/optimizer/distributed_fused_lamb.py index 56a9a991347..ef1bacd0752 100644 --- a/python/paddle/incubate/optimizer/distributed_fused_lamb.py +++ b/python/paddle/incubate/optimizer/distributed_fused_lamb.py @@ -32,84 +32,88 @@ def init_communicator(block, rank, ranks, ring_id): local_rank = ranks.index(rank) comm_var_name = unique_name.generate('comm_id') - comm_id_var = block.create_var(name=comm_var_name, - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': comm_id_var}, - attrs={ - 'rank': local_rank, - 'endpoint': cur_ep, - 'other_endpoints': other_eps, - 'ring_id': ring_id - }) - block.append_op(type='c_comm_init', - inputs={'X': comm_id_var}, - outputs={}, - attrs={ - 'nranks': len(ranks), - 'rank': local_rank, - 'ring_id': ring_id - }) + comm_id_var = block.create_var( + name=comm_var_name, persistable=True, type=core.VarDesc.VarType.RAW + ) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': local_rank, + 'endpoint': cur_ep, + 'other_endpoints': other_eps, + 'ring_id': ring_id, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': comm_id_var}, + outputs={}, + attrs={'nranks': len(ranks), 'rank': local_rank, 'ring_id': ring_id}, + ) tmp_var = block.create_var(name=unique_name.generate('tmp')) - block.append_op(type='fill_constant', - outputs={'Out': tmp_var}, - attrs={'value': 1}) - block.append_op(type='c_allreduce_sum', - inputs={'X': tmp_var}, - outputs={'Out': tmp_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True - }) - block.append_op(type='c_sync_calc_stream', - inputs={'X': tmp_var}, - outputs={'Out': tmp_var}) + block.append_op( + type='fill_constant', outputs={'Out': tmp_var}, attrs={'value': 1} + ) + block.append_op( + type='c_allreduce_sum', + inputs={'X': tmp_var}, + outputs={'Out': tmp_var}, + attrs={'ring_id': ring_id, 'use_calc_stream': True}, + ) + block.append_op( + type='c_sync_calc_stream', + inputs={'X': tmp_var}, + outputs={'Out': tmp_var}, + ) return ring_id def broadcast_parameters(block, parameters, ring_id): for p in parameters: - block.append_op(type='c_broadcast', - inputs={'X': p}, - outputs={'Out': p}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True - }) + block.append_op( + type='c_broadcast', + inputs={'X': p}, + outputs={'Out': p}, + attrs={'ring_id': ring_id, 'use_calc_stream': True}, + ) class DistributedFusedLamb(Optimizer): - - def __init__(self, - learning_rate=0.001, - lamb_weight_decay=0.01, - beta1=0.9, - beta2=0.999, - epsilon=1e-6, - parameters=None, - grad_clip=None, - exclude_from_weight_decay_fn=None, - clip_after_allreduce=True, - is_grad_scaled_by_nranks=True, - alignment=128, - use_master_param_norm=True, - gradient_accumulation_steps=1, - use_master_acc_grad=True, - nproc_per_node=None, - use_hierarchical_allreduce=False, - name=None): - assert not framework._non_static_mode( + def __init__( + self, + learning_rate=0.001, + lamb_weight_decay=0.01, + beta1=0.9, + beta2=0.999, + epsilon=1e-6, + parameters=None, + grad_clip=None, + exclude_from_weight_decay_fn=None, + clip_after_allreduce=True, + is_grad_scaled_by_nranks=True, + alignment=128, + use_master_param_norm=True, + gradient_accumulation_steps=1, + use_master_acc_grad=True, + nproc_per_node=None, + use_hierarchical_allreduce=False, + name=None, + ): + assert ( + not framework._non_static_mode() ), "DistributedFusedLamb does not support dygraph mode" - super(DistributedFusedLamb, self).__init__(learning_rate=learning_rate, - grad_clip=None, - name=name) + super(DistributedFusedLamb, self).__init__( + learning_rate=learning_rate, grad_clip=None, name=name + ) self._beta1 = beta1 self._beta2 = beta2 self._epsilon = epsilon - self._weight_decay = lamb_weight_decay if lamb_weight_decay is not None else 0.0 + self._weight_decay = ( + lamb_weight_decay if lamb_weight_decay is not None else 0.0 + ) if grad_clip is not None: assert isinstance( grad_clip, ClipGradByGlobalNorm @@ -137,14 +141,16 @@ class DistributedFusedLamb(Optimizer): self._found_inf = main_block.create_var( name=unique_name.generate('found_inf'), shape=[1], - dtype=core.VarDesc.VarType.BOOL) + dtype=core.VarDesc.VarType.BOOL, + ) self._step = None if self._gradient_accumulation_steps > 1: self._stop_update = main_block.create_var( name=unique_name.generate('stop_update'), shape=[1], - dtype=core.VarDesc.VarType.BOOL) + dtype=core.VarDesc.VarType.BOOL, + ) else: self._stop_update = None @@ -169,11 +175,13 @@ class DistributedFusedLamb(Optimizer): def _create_scale_from_constant(self, value): name = unique_name.generate('global_scale') - return layers.create_global_var(name=name, - shape=[1], - dtype='float32', - value=float(value), - persistable=True) + return layers.create_global_var( + name=name, + shape=[1], + dtype='float32', + value=float(value), + persistable=True, + ) def _get_or_create_scale(self): if self._scale is None: @@ -184,17 +192,21 @@ class DistributedFusedLamb(Optimizer): startup_block = self.helper.startup_program.global_block() if name is not None: name = unique_name.generate(name) - startup_var = startup_block.create_var(name=name, - shape=shape, - dtype=dtype, - persistable=True, - stop_gradient=True) + startup_var = startup_block.create_var( + name=name, + shape=shape, + dtype=dtype, + persistable=True, + stop_gradient=True, + ) main_block = self.helper.main_program.global_block() - main_var = main_block.create_var(name=startup_var.name, - shape=startup_var.shape, - dtype=startup_var.dtype, - persistable=True, - stop_gradient=True) + main_var = main_block.create_var( + name=startup_var.name, + shape=startup_var.shape, + dtype=startup_var.dtype, + persistable=True, + stop_gradient=True, + ) return main_var def _get_parameter(self, name, scope=None): @@ -224,20 +236,25 @@ class DistributedFusedLamb(Optimizer): for p, g in params_grads: flattened.extend([p, g]) with flattened[0].block.program._optimized_guard(flattened), name_scope( - "optimizer"): + "optimizer" + ): self._apply_gradients_impl(params_grads) def _apply_gradients_impl(self, params_grads): for p, g in params_grads: - assert g.type == core.VarDesc.VarType.LOD_TENSOR, "Only support dense gradient" + assert ( + g.type == core.VarDesc.VarType.LOD_TENSOR + ), "Only support dense gradient" g.persistable = True # the gradient must be persistable for fusion fp32_fused_param = self._create_persistable_var('fp32_fused_param') fp32_fused_grad = self._create_persistable_var('fp32_fused_grad') - fp16_fused_param = self._create_persistable_var('fp16_fused_param', - dtype='float16') - fp16_fused_grad = self._create_persistable_var('fp16_fused_grad', - dtype='float16') + fp16_fused_param = self._create_persistable_var( + 'fp16_fused_param', dtype='float16' + ) + fp16_fused_grad = self._create_persistable_var( + 'fp16_fused_grad', dtype='float16' + ) master_params = [] for p, g in params_grads: @@ -255,15 +272,18 @@ class DistributedFusedLamb(Optimizer): param_info = self._create_persistable_var('param_info', dtype='int32') param_info.is_distributed = True - fused_offsets = self._create_persistable_var('fused_offsets', - dtype='int32') + fused_offsets = self._create_persistable_var( + 'fused_offsets', dtype='int32' + ) fp32_partial_fused_offsets = self._create_persistable_var( - 'fp32_partial_fused_offsets', dtype='int32') + 'fp32_partial_fused_offsets', dtype='int32' + ) fp32_partial_fused_offsets.is_distributed = True fp16_partial_fused_offsets = self._create_persistable_var( - 'fp16_partial_fused_offsets', dtype='int32') + 'fp16_partial_fused_offsets', dtype='int32' + ) fp16_partial_fused_offsets.is_distributed = True param_order = self._create_persistable_var('param_order', dtype='int32') @@ -274,8 +294,9 @@ class DistributedFusedLamb(Optimizer): self._create_persistable_var('fp32_acc_fused_grad') ] fp16_acc_fused_grad = [ - self._create_persistable_var('fp16_acc_fused_grad', - dtype='float16') + self._create_persistable_var( + 'fp16_acc_fused_grad', dtype='float16' + ) ] acc_step = [self._create_persistable_var('acc_step', dtype='int64')] else: @@ -291,33 +312,40 @@ class DistributedFusedLamb(Optimizer): nproc_per_node = nranks else: nproc_per_node = self._nproc_per_node - assert nranks % nproc_per_node == 0, "nranks should be exactly divided by nproc_per_node" + assert ( + nranks % nproc_per_node == 0 + ), "nranks should be exactly divided by nproc_per_node" - shard_inside_node = (nranks > nproc_per_node) + shard_inside_node = nranks > nproc_per_node local_rank = rank % nproc_per_node node_id = int(rank / nproc_per_node) node_num = int(nranks / nproc_per_node) ring_ids = [] startup_block = self.helper.startup_program.global_block() if nranks > 1: - ring_id = init_communicator(startup_block, rank, - list(range(nranks)), 0) + ring_id = init_communicator( + startup_block, rank, list(range(nranks)), 0 + ) ring_ids.append(ring_id) use_hierarchical_allreduce = False if node_num > 1 and len(ring_ids) <= 1 and shard_inside_node: local_group_ranks = list( - range(node_id * nproc_per_node, (node_id + 1) * nproc_per_node)) - ring_id = init_communicator(startup_block, rank, local_group_ranks, - 1) + range(node_id * nproc_per_node, (node_id + 1) * nproc_per_node) + ) + ring_id = init_communicator( + startup_block, rank, local_group_ranks, 1 + ) ring_ids.append(ring_id) if self._use_hierarchical_allreduce and nranks > nproc_per_node: use_hierarchical_allreduce = True outer_group_ranks = list( - range(rank % nproc_per_node, nranks, nproc_per_node)) - ring_id = init_communicator(startup_block, rank, - outer_group_ranks, ring_ids[-1] + 1) + range(rank % nproc_per_node, nranks, nproc_per_node) + ) + ring_id = init_communicator( + startup_block, rank, outer_group_ranks, ring_ids[-1] + 1 + ) ring_ids.append(ring_id) scale = self._get_or_create_scale() @@ -331,11 +359,13 @@ class DistributedFusedLamb(Optimizer): apply_weight_decay[i] = 0 for g in grads: - startup_block.create_var(name=g.name, - type=g.type, - dtype=g.dtype, - persistable=g.persistable, - shape=g.shape) + startup_block.create_var( + name=g.name, + type=g.type, + dtype=g.dtype, + persistable=g.persistable, + shape=g.shape, + ) if nranks > 1: broadcast_parameters(startup_block, params, ring_ids[0]) @@ -375,7 +405,8 @@ class DistributedFusedLamb(Optimizer): 'moment2': 0.0, 'beta1': self._beta1, 'beta2': self._beta2, - }) + }, + ) main_block = self.helper.main_program.global_block() self._create_global_learning_rate() @@ -418,19 +449,15 @@ class DistributedFusedLamb(Optimizer): 'Moment2Out': [moment2], 'Beta1PowOut': [beta1pow], 'Beta2PowOut': [beta2pow], - 'ParamOut': - params, - 'GradOut': - grads, + 'ParamOut': params, + 'GradOut': grads, 'FoundInf': [self._found_inf], - 'FP32AccFusedGrad': - fp32_acc_fused_grad, - 'FP16AccFusedGrad': - fp16_acc_fused_grad, - 'AccStep': - acc_step, - 'StopUpdate': - self._stop_update if self._stop_update is not None else [], + 'FP32AccFusedGrad': fp32_acc_fused_grad, + 'FP16AccFusedGrad': fp16_acc_fused_grad, + 'AccStep': acc_step, + 'StopUpdate': self._stop_update + if self._stop_update is not None + else [], 'Step': [step], }, attrs={ @@ -448,5 +475,6 @@ class DistributedFusedLamb(Optimizer): 'acc_steps': self._gradient_accumulation_steps, 'use_master_acc_grad': self._use_master_acc_grad, 'use_hierarchical_allreduce': use_hierarchical_allreduce, - }) + }, + ) return [lamb_op] diff --git a/python/paddle/incubate/optimizer/functional/bfgs.py b/python/paddle/incubate/optimizer/functional/bfgs.py index 58d647f4dd4..3349cd533e2 100644 --- a/python/paddle/incubate/optimizer/functional/bfgs.py +++ b/python/paddle/incubate/optimizer/functional/bfgs.py @@ -15,22 +15,28 @@ import numpy as np from .line_search import strong_wolfe -from .utils import _value_and_gradient, check_input_type, check_initial_inverse_hessian_estimate +from .utils import ( + _value_and_gradient, + check_input_type, + check_initial_inverse_hessian_estimate, +) import paddle -def minimize_bfgs(objective_func, - initial_position, - max_iters=50, - tolerance_grad=1e-7, - tolerance_change=1e-9, - initial_inverse_hessian_estimate=None, - line_search_fn='strong_wolfe', - max_line_search_iters=50, - initial_step_length=1.0, - dtype='float32', - name=None): +def minimize_bfgs( + objective_func, + initial_position, + max_iters=50, + tolerance_grad=1e-7, + tolerance_change=1e-9, + initial_inverse_hessian_estimate=None, + line_search_fn='strong_wolfe', + max_line_search_iters=50, + initial_step_length=1.0, + dtype='float32', + name=None, +): r""" Minimizes a differentiable function `func` using the BFGS method. The BFGS is a quasi-Newton method for solving an unconstrained optimization problem over a differentiable function. @@ -91,8 +97,10 @@ def minimize_bfgs(objective_func, if dtype not in ['float32', 'float64']: raise ValueError( - "The dtype must be 'float32' or 'float64', but the specified is {}." - .format(dtype)) + "The dtype must be 'float32' or 'float64', but the specified is {}.".format( + dtype + ) + ) op_name = 'minimize_bfgs' check_input_type(initial_position, 'initial_position', op_name) @@ -101,8 +109,11 @@ def minimize_bfgs(objective_func, if initial_inverse_hessian_estimate is None: initial_inverse_hessian_estimate = I else: - check_input_type(initial_inverse_hessian_estimate, - 'initial_inverse_hessian_estimate', op_name) + check_input_type( + initial_inverse_hessian_estimate, + 'initial_inverse_hessian_estimate', + op_name, + ) check_initial_inverse_hessian_estimate(initial_inverse_hessian_estimate) Hk = paddle.assign(initial_inverse_hessian_estimate) @@ -131,11 +142,14 @@ def minimize_bfgs(objective_func, xk=xk, pk=pk, initial_step_length=initial_step_length, - dtype=dtype) + dtype=dtype, + ) else: raise NotImplementedError( - "Currently only support line_search_fn = 'strong_wolfe', but the specified is '{}'" - .format(line_search_fn)) + "Currently only support line_search_fn = 'strong_wolfe', but the specified is '{}'".format( + line_search_fn + ) + ) num_func_calls += ls_func_calls ############# update Hk ############# @@ -150,14 +164,17 @@ def minimize_bfgs(objective_func, rhok_inv = paddle.dot(yk, sk) rhok = paddle.static.nn.cond( - rhok_inv == 0., + rhok_inv == 0.0, lambda: paddle.full(shape=[1], fill_value=1000.0, dtype=dtype), - lambda: 1. / rhok_inv) + lambda: 1.0 / rhok_inv, + ) Vk_transpose = I - rhok * sk * yk.t() Vk = I - rhok * yk * sk.t() - Hk = paddle.matmul(paddle.matmul(Vk_transpose, Hk), - Vk) + rhok * sk * sk.t() + Hk = ( + paddle.matmul(paddle.matmul(Vk_transpose, Hk), Vk) + + rhok * sk * sk.t() + ) k += 1 @@ -165,15 +182,16 @@ def minimize_bfgs(objective_func, gnorm = paddle.linalg.norm(g1, p=np.inf) pk_norm = paddle.linalg.norm(pk, p=np.inf) paddle.assign( - done | (gnorm < tolerance_grad) | (pk_norm < tolerance_change), - done) + done | (gnorm < tolerance_grad) | (pk_norm < tolerance_change), done + ) paddle.assign(done, is_converge) # when alpha=0, there is no chance to get xk change. - paddle.assign(done | (alpha == 0.), done) + paddle.assign(done | (alpha == 0.0), done) return [k, done, is_converge, num_func_calls, xk, value, g1, Hk] paddle.static.nn.while_loop( cond=cond, body=body, - loop_vars=[k, done, is_converge, num_func_calls, xk, value, g1, Hk]) + loop_vars=[k, done, is_converge, num_func_calls, xk, value, g1, Hk], + ) return is_converge, num_func_calls, xk, value, g1, Hk diff --git a/python/paddle/incubate/optimizer/functional/lbfgs.py b/python/paddle/incubate/optimizer/functional/lbfgs.py index 5925d47decd..a0cf07f8d4a 100644 --- a/python/paddle/incubate/optimizer/functional/lbfgs.py +++ b/python/paddle/incubate/optimizer/functional/lbfgs.py @@ -15,23 +15,29 @@ import numpy as np from .line_search import strong_wolfe -from .utils import _value_and_gradient, check_input_type, check_initial_inverse_hessian_estimate +from .utils import ( + _value_and_gradient, + check_input_type, + check_initial_inverse_hessian_estimate, +) import paddle -def minimize_lbfgs(objective_func, - initial_position, - history_size=100, - max_iters=50, - tolerance_grad=1e-8, - tolerance_change=1e-8, - initial_inverse_hessian_estimate=None, - line_search_fn='strong_wolfe', - max_line_search_iters=50, - initial_step_length=1.0, - dtype='float32', - name=None): +def minimize_lbfgs( + objective_func, + initial_position, + history_size=100, + max_iters=50, + tolerance_grad=1e-8, + tolerance_change=1e-8, + initial_inverse_hessian_estimate=None, + line_search_fn='strong_wolfe', + max_line_search_iters=50, + initial_step_length=1.0, + dtype='float32', + name=None, +): r""" Minimizes a differentiable function `func` using the L-BFGS method. The L-BFGS is a quasi-Newton method for solving an unconstrained optimization problem over a differentiable function. @@ -91,8 +97,10 @@ def minimize_lbfgs(objective_func, """ if dtype not in ['float32', 'float64']: raise ValueError( - "The dtype must be 'float32' or 'float64', but the specified is {}." - .format(dtype)) + "The dtype must be 'float32' or 'float64', but the specified is {}.".format( + dtype + ) + ) op_name = 'minimize_lbfgs' check_input_type(initial_position, 'initial_position', op_name) @@ -100,8 +108,11 @@ def minimize_lbfgs(objective_func, if initial_inverse_hessian_estimate is None: H0 = paddle.eye(initial_position.shape[0], dtype=dtype) else: - check_input_type(initial_inverse_hessian_estimate, - 'initial_inverse_hessian_estimate', op_name) + check_input_type( + initial_inverse_hessian_estimate, + 'initial_inverse_hessian_estimate', + op_name, + ) check_initial_inverse_hessian_estimate(initial_inverse_hessian_estimate) H0 = initial_inverse_hessian_estimate @@ -114,9 +125,9 @@ def minimize_lbfgs(objective_func, is_converge = paddle.full(shape=[1], fill_value=False, dtype='bool') num_func_calls = paddle.full(shape=[1], fill_value=1, dtype='int64') - history_size = paddle.full(shape=[1], - fill_value=history_size, - dtype='int64') + history_size = paddle.full( + shape=[1], fill_value=history_size, dtype='int64' + ) head = paddle.full(shape=[1], fill_value=1, dtype='int64') tail = paddle.full(shape=[1], fill_value=0, dtype='int64') @@ -130,20 +141,44 @@ def minimize_lbfgs(objective_func, rhok_vec = paddle.zeros((history_size + 1, 1), dtype=dtype) ai_vec = paddle.zeros((history_size + 1, 1), dtype=dtype) - def cond(k, done, is_converge, num_func_calls, value, xk, g1, sk_vec, - yk_vec, rhok_vec, head, tail): + def cond( + k, + done, + is_converge, + num_func_calls, + value, + xk, + g1, + sk_vec, + yk_vec, + rhok_vec, + head, + tail, + ): return (k < max_iters) & ~done - def body(k, done, is_converge, num_func_calls, value, xk, g1, sk_vec, - yk_vec, rhok_vec, head, tail): + def body( + k, + done, + is_converge, + num_func_calls, + value, + xk, + g1, + sk_vec, + yk_vec, + rhok_vec, + head, + tail, + ): # use assign to cut off the relevance between g1 and q, or they will change together. ############# compute p_k by two-loop recursion ############# q = paddle.assign(g1) # In a array circle, the index may out of range, so must use mod. - i = paddle.full(shape=[1], - fill_value=(head - 1).mod(history_size), - dtype='int64') + i = paddle.full( + shape=[1], fill_value=(head - 1).mod(history_size), dtype='int64' + ) def cond(i, q): return i != tail @@ -180,11 +215,14 @@ def minimize_lbfgs(objective_func, xk=xk, pk=pk, initial_step_length=initial_step_length, - dtype=dtype) + dtype=dtype, + ) else: raise NotImplementedError( - "Currently only support line_search_fn = 'strong_wolfe', but the specified is '{}'" - .format(line_search_fn)) + "Currently only support line_search_fn = 'strong_wolfe', but the specified is '{}'".format( + line_search_fn + ) + ) paddle.assign(num_func_calls + ls_func_calls, num_func_calls) ############# update sk_vec, yk_vec, rhok_vec ############# @@ -193,9 +231,10 @@ def minimize_lbfgs(objective_func, rhok_inv = paddle.dot(yk, sk) rhok = paddle.static.nn.cond( - rhok_inv == 0., + rhok_inv == 0.0, lambda: paddle.full(shape=[1], fill_value=1000.0, dtype=dtype), - lambda: 1. / rhok_inv) + lambda: 1.0 / rhok_inv, + ) sk_vec[head] = sk yk_vec[head] = yk @@ -216,21 +255,43 @@ def minimize_lbfgs(objective_func, gnorm = paddle.linalg.norm(g1, p=np.inf) pk_norm = paddle.linalg.norm(pk, p=np.inf) paddle.assign( - done | (gnorm < tolerance_grad) | (pk_norm < tolerance_change), - done) + done | (gnorm < tolerance_grad) | (pk_norm < tolerance_change), done + ) paddle.assign(done, is_converge) # when alpha=0, there is no chance to get xk change. - paddle.assign(done | (alpha == 0.), done) + paddle.assign(done | (alpha == 0.0), done) return [ - k, done, is_converge, num_func_calls, value, xk, g1, sk_vec, yk_vec, - rhok_vec, head, tail + k, + done, + is_converge, + num_func_calls, + value, + xk, + g1, + sk_vec, + yk_vec, + rhok_vec, + head, + tail, ] - paddle.static.nn.while_loop(cond=cond, - body=body, - loop_vars=[ - k, done, is_converge, num_func_calls, value, - xk, g1, sk_vec, yk_vec, rhok_vec, head, tail - ]) + paddle.static.nn.while_loop( + cond=cond, + body=body, + loop_vars=[ + k, + done, + is_converge, + num_func_calls, + value, + xk, + g1, + sk_vec, + yk_vec, + rhok_vec, + head, + tail, + ], + ) return is_converge, num_func_calls, xk, value, g1 diff --git a/python/paddle/incubate/optimizer/functional/line_search.py b/python/paddle/incubate/optimizer/functional/line_search.py index 375f86ddbe5..94400200f25 100644 --- a/python/paddle/incubate/optimizer/functional/line_search.py +++ b/python/paddle/incubate/optimizer/functional/line_search.py @@ -31,8 +31,9 @@ def cubic_interpolation_(x1, f1, g1, x2, f2, g2): Returns: min_pos: the minimun point between the specified points in the cubic curve. """ - xmin, xmax = paddle.static.nn.cond(x1 <= x2, lambda: (x1, x2), lambda: - (x2, x1)) + xmin, xmax = paddle.static.nn.cond( + x1 <= x2, lambda: (x1, x2), lambda: (x2, x1) + ) d1 = g1 + g2 - 3 * (f1 - f2) / (x1 - x2) d2_square = d1**2 - g1 * g2 @@ -51,22 +52,24 @@ def cubic_interpolation_(x1, f1, g1, x2, f2, g2): return paddle.minimum(paddle.maximum(min_pos, xmin), xmax) def false_func1(): - return (xmin + xmax) / 2. + return (xmin + xmax) / 2.0 - min_pos = paddle.static.nn.cond(d2_square >= 0., true_func1, false_func1) + min_pos = paddle.static.nn.cond(d2_square >= 0.0, true_func1, false_func1) return min_pos -def strong_wolfe(f, - xk, - pk, - max_iters=20, - tolerance_change=1e-8, - initial_step_length=1.0, - c1=1e-4, - c2=0.9, - alpha_max=10, - dtype='float32'): +def strong_wolfe( + f, + xk, + pk, + max_iters=20, + tolerance_change=1e-8, + initial_step_length=1.0, + c1=1e-4, + c2=0.9, + alpha_max=10, + dtype='float32', +): r"""Implements of line search algorithm that satisfies the strong Wolfe conditions using double zoom. Reference: @@ -143,36 +146,67 @@ def strong_wolfe(f, def phi_and_derphi(a): r"""Compute function value and derivative of phi at a. - phi = f(xk + a * pk) - phi'(a) = f'(xk + a * pk) * pk + phi = f(xk + a * pk) + phi'(a) = f'(xk + a * pk) * pk """ phi_value, f_grad = _value_and_gradient(f, xk + a * pk) phi_grad = paddle.dot(f_grad, pk) # return f_grad to be used in bfgs/l-bfgs to compute yk to avoid computint repeatly. return phi_value, f_grad, phi_grad - def zoom(a_lo, phi_lo, derphi_lo, derf_lo, a_hi, phi_hi, derphi_hi, phi_0, - derphi_0): + def zoom( + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, + phi_0, + derphi_0, + ): # find the exact a from the bracket [a_lo, a_hi] max_zoom_iters = max_iters j = paddle.full(shape=[1], fill_value=0, dtype='int64') done_zoom = paddle.full(shape=[1], fill_value=False, dtype='bool') - def cond_zoom(j, done_zoom, a_lo, phi_lo, derphi_lo, derf_lo, a_hi, - phi_hi, derphi_hi): + def cond_zoom( + j, + done_zoom, + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, + ): pred = paddle.abs(a_hi - a_lo) < tolerance_change paddle.assign(done_zoom | pred, done_zoom) return (j < max_zoom_iters) & ~done_zoom - def body_zoom(j, done_zoom, a_lo, phi_lo, derphi_lo, derf_lo, a_hi, - phi_hi, derphi_hi): - aj = cubic_interpolation_(a_lo, phi_lo, derphi_lo, a_hi, phi_hi, - derphi_hi) # 21 + def body_zoom( + j, + done_zoom, + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, + ): + aj = cubic_interpolation_( + a_lo, phi_lo, derphi_lo, a_hi, phi_hi, derphi_hi + ) # 21 min_change = 0.1 * paddle.abs(a_hi - a_lo) - pred = paddle.minimum(paddle.abs(aj - a_lo), - paddle.abs(aj - a_hi)) < min_change - aj = paddle.static.nn.cond(pred, lambda: 0.5 * (a_lo + a_hi), - lambda: aj) + pred = ( + paddle.minimum(paddle.abs(aj - a_lo), paddle.abs(aj - a_hi)) + < min_change + ) + aj = paddle.static.nn.cond( + pred, lambda: 0.5 * (a_lo + a_hi), lambda: aj + ) phi_j, derf_j, derphi_j = phi_and_derphi(aj) @@ -183,7 +217,7 @@ def strong_wolfe(f, paddle.assign(derphi_j, derphi_hi) def false_fn(a_lo, done_zoom): - pred3 = (paddle.abs(derphi_j) <= -c2 * derphi_0) + pred3 = paddle.abs(derphi_j) <= -c2 * derphi_0 paddle.assign(pred3, done_zoom) def true_fn(): @@ -200,26 +234,43 @@ def strong_wolfe(f, paddle.assign(derf_j, derf_lo) pred2 = (phi_j > phi_0 + c1 * aj * derphi_0) | (phi_j >= phi_lo) - paddle.static.nn.cond(pred2, true_fn, - lambda: false_fn(a_lo, done_zoom)) + paddle.static.nn.cond( + pred2, true_fn, lambda: false_fn(a_lo, done_zoom) + ) j = paddle.static.nn.cond(done_zoom, lambda: j, lambda: j + 1) return [ - j, done_zoom, a_lo, phi_lo, derphi_lo, derf_lo, a_hi, phi_hi, - derphi_hi + j, + done_zoom, + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, ] - paddle.static.nn.while_loop(cond=cond_zoom, - body=body_zoom, - loop_vars=[ - j, done_zoom, a_lo, phi_lo, derphi_lo, - derf_lo, a_hi, phi_hi, derphi_hi - ]) + paddle.static.nn.while_loop( + cond=cond_zoom, + body=body_zoom, + loop_vars=[ + j, + done_zoom, + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, + ], + ) # j is the number of object function called in zoom. return j alpha_max = paddle.full(shape=[1], fill_value=alpha_max, dtype=dtype) - a1 = paddle.full(shape=[1], fill_value=0., dtype=dtype) + a1 = paddle.full(shape=[1], fill_value=0.0, dtype=dtype) a2 = paddle.full(shape=[1], fill_value=initial_step_length, dtype=dtype) phi_1, derf_1, derphi_1 = phi_and_derphi(a1) @@ -245,15 +296,25 @@ def strong_wolfe(f, paddle.assign(done | paddle.any(paddle.isinf(phi_2)), done) def true_fn1(): - j = zoom(a1, phi_1, derphi_1, derf_1, a2, phi_2, derphi_2, phi_0, - derphi_0) + j = zoom( + a1, + phi_1, + derphi_1, + derf_1, + a2, + phi_2, + derphi_2, + phi_0, + derphi_0, + ) paddle.assign(a1, a_star) paddle.assign(phi_1, phi_star) paddle.assign(derf_1, derf_star) paddle.assign(ls_func_calls + j, ls_func_calls) - pred1 = ~done & ((phi_2 > phi_0 + c1 * a2 * derphi_0) | - ((phi_2 >= phi_0) & (i > 1))) + pred1 = ~done & ( + (phi_2 > phi_0 + c1 * a2 * derphi_0) | ((phi_2 >= phi_0) & (i > 1)) + ) paddle.assign(done | pred1, done) paddle.static.nn.cond(pred1, true_fn1, None) @@ -267,8 +328,17 @@ def strong_wolfe(f, paddle.static.nn.cond(pred2, true_fn2, None) def true_fn3(): - j = zoom(a2, phi_2, derphi_2, derf_2, a1, phi_1, derphi_1, phi_0, - derphi_0) + j = zoom( + a2, + phi_2, + derphi_2, + derf_2, + a1, + phi_1, + derphi_1, + phi_0, + derphi_0, + ) paddle.assign(a2, a_star) paddle.assign(phi_2, phi_star) paddle.assign(derf_2, derf_star) @@ -291,6 +361,7 @@ def strong_wolfe(f, paddle.static.nn.while_loop( cond=cond, body=body, - loop_vars=[i, ls_func_calls, a1, a2, phi_1, derf_1, done]) + loop_vars=[i, ls_func_calls, a1, a2, phi_1, derf_1, done], + ) return a_star, phi_star, derf_star, ls_func_calls diff --git a/python/paddle/incubate/optimizer/functional/utils.py b/python/paddle/incubate/optimizer/functional/utils.py index e6612878e71..f9d0dddb117 100644 --- a/python/paddle/incubate/optimizer/functional/utils.py +++ b/python/paddle/incubate/optimizer/functional/utils.py @@ -53,26 +53,29 @@ def check_initial_inverse_hessian_estimate(H0): else: def create_tmp_var(program, name, dtype, shape): - return program.current_block().create_var(name=name, - dtype=dtype, - shape=shape) + return program.current_block().create_var( + name=name, dtype=dtype, shape=shape + ) - out_var = create_tmp_var(paddle.static.default_main_program(), - name='output', - dtype='float32', - shape=[-1]) + out_var = create_tmp_var( + paddle.static.default_main_program(), + name='output', + dtype='float32', + shape=[-1], + ) def false_fn(): - paddle.static.nn.py_func(func=raise_func, - x=is_symmetric, - out=out_var) + paddle.static.nn.py_func( + func=raise_func, x=is_symmetric, out=out_var + ) paddle.static.nn.cond(is_symmetric, None, false_fn) # eigvals only support cpu paddle.set_device("cpu") eigvals = paddle.paddle.linalg.eigvals(H0) - is_positive = paddle.all(eigvals.real() > 0.) and paddle.all( - eigvals.imag() == 0.) + is_positive = paddle.all(eigvals.real() > 0.0) and paddle.all( + eigvals.imag() == 0.0 + ) paddle.static.nn.cond(is_positive, None, false_fn) diff --git a/python/paddle/incubate/optimizer/lookahead.py b/python/paddle/incubate/optimizer/lookahead.py index dcf50cb5224..99a3be4f8e4 100644 --- a/python/paddle/incubate/optimizer/lookahead.py +++ b/python/paddle/incubate/optimizer/lookahead.py @@ -115,24 +115,27 @@ class LookAhead(Optimizer): _slow_str = "slow" def __init__(self, inner_optimizer, alpha=0.5, k=5, name=None): - assert (inner_optimizer is not None), "inner optimizer can not be None" + assert inner_optimizer is not None, "inner optimizer can not be None" assert ( 0.0 <= alpha <= 1.0 ), "alpha should be larger or equal to 0.0, and less or equal than 1.0" - assert (isinstance(k, int) and k > 0), "k should be a positive integer" + assert isinstance(k, int) and k > 0, "k should be a positive integer" self.inner_optimizer = inner_optimizer if self.inner_optimizer._parameter_list is None: - parameters = framework.default_main_program().global_block( - ).all_parameters() + parameters = ( + framework.default_main_program().global_block().all_parameters() + ) else: parameters = self.inner_optimizer._parameter_list - super(LookAhead, self).__init__(learning_rate=alpha, - parameters=parameters, - weight_decay=None, - grad_clip=None, - name=name) + super(LookAhead, self).__init__( + learning_rate=alpha, + parameters=parameters, + weight_decay=None, + grad_clip=None, + name=name, + ) self.alpha = alpha self.k = k @@ -178,9 +181,9 @@ class LookAhead(Optimizer): grad_var = param._grad_ivar() params_grads.append((param, grad_var)) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -195,24 +198,28 @@ class LookAhead(Optimizer): shape=[1], value=0, dtype='int32', - persistable=True) + persistable=True, + ) - self.helper.append_op(type='increment', - inputs={'X': [self._global_step_var]}, - outputs={'Out': [self._global_step_var]}, - attrs={'step': 1.0}) + self.helper.append_op( + type='increment', + inputs={'X': [self._global_step_var]}, + outputs={'Out': [self._global_step_var]}, + attrs={'step': 1.0}, + ) def _append_optimize_op(self, block, param_and_grad): one_var = paddle.ones(shape=[1], dtype='int32', name='lookahead_ones') - zero_var = paddle.zeros(shape=[1], - dtype='int32', - name='lookahead_zeros') + zero_var = paddle.zeros( + shape=[1], dtype='int32', name='lookahead_zeros' + ) k_var = layers.create_global_var( name=unique_name.generate("lookahead_k"), shape=[1], value=self.k, dtype='int32', - persistable=True) + persistable=True, + ) mod = paddle.remainder(self._global_step_var, k_var) @@ -235,11 +242,9 @@ class LookAhead(Optimizer): paddle.assign(tmp_var_1, slow_var) @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): """ Add operations to minimize ``loss`` by updating ``parameters``. @@ -286,12 +291,13 @@ class LookAhead(Optimizer): loss, startup_program=startup_program, parameters=parameters, - no_grad_set=no_grad_set) + no_grad_set=no_grad_set, + ) self._increment_global_var() - _ = self._apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + _ = self._apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads diff --git a/python/paddle/incubate/optimizer/modelaverage.py b/python/paddle/incubate/optimizer/modelaverage.py index 5fa9dfb67bc..fc00f28cb44 100644 --- a/python/paddle/incubate/optimizer/modelaverage.py +++ b/python/paddle/incubate/optimizer/modelaverage.py @@ -162,17 +162,21 @@ class ModelAverage(Optimizer): """ - def __init__(self, - average_window_rate, - parameters=None, - min_average_window=10000, - max_average_window=10000, - name=None): - super(ModelAverage, self).__init__(learning_rate=0.0, - parameters=parameters, - weight_decay=None, - grad_clip=None, - name=name) + def __init__( + self, + average_window_rate, + parameters=None, + min_average_window=10000, + max_average_window=10000, + name=None, + ): + super(ModelAverage, self).__init__( + learning_rate=0.0, + parameters=parameters, + weight_decay=None, + grad_clip=None, + name=name, + ) self.helper = LayerHelper(self.__class__.__name__) self.average_window = average_window_rate @@ -182,7 +186,8 @@ class ModelAverage(Optimizer): if not framework._non_static_mode(): global_block = framework.default_main_program().global_block() - all_parameters = parameters if parameters else global_block.all_parameters( + all_parameters = ( + parameters if parameters else global_block.all_parameters() ) self._create_accumulators(global_block, all_parameters) @@ -207,18 +212,15 @@ class ModelAverage(Optimizer): self._add_accumulator('sum_2', param) self._add_accumulator('sum_3', param) self._add_accumulator('restore', param) - self._add_accumulator('num_accumulates', - param, - dtype='int64', - shape=[1]) - self._add_accumulator('old_num_accumulates', - param, - dtype='int64', - shape=[1]) - self._add_accumulator('num_updates', - param, - dtype='int64', - shape=[1]) + self._add_accumulator( + 'num_accumulates', param, dtype='int64', shape=[1] + ) + self._add_accumulator( + 'old_num_accumulates', param, dtype='int64', shape=[1] + ) + self._add_accumulator( + 'num_updates', param, dtype='int64', shape=[1] + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -226,26 +228,50 @@ class ModelAverage(Optimizer): sum_1 = self._get_accumulator('sum_1', param_and_grad[0]) sum_2 = self._get_accumulator('sum_2', param_and_grad[0]) sum_3 = self._get_accumulator('sum_3', param_and_grad[0]) - num_accumulates = self._get_accumulator('num_accumulates', - param_and_grad[0]) - old_num_accumulates = self._get_accumulator('old_num_accumulates', - param_and_grad[0]) + num_accumulates = self._get_accumulator( + 'num_accumulates', param_and_grad[0] + ) + old_num_accumulates = self._get_accumulator( + 'old_num_accumulates', param_and_grad[0] + ) num_updates = self._get_accumulator('num_updates', param_and_grad[0]) if in_dygraph_mode(): _, _, _, _, _, _ = _C_ops.average_accumulates_( - param_and_grad[0], sum_1, sum_2, sum_3, num_accumulates, - old_num_accumulates, num_updates, self.average_window, - self.max_average_window, self.min_average_window) + param_and_grad[0], + sum_1, + sum_2, + sum_3, + num_accumulates, + old_num_accumulates, + num_updates, + self.average_window, + self.max_average_window, + self.min_average_window, + ) return None elif framework._non_static_mode(): _, _, _, _, _, _ = _legacy_C_ops.average_accumulates( - param_and_grad[0], sum_1, sum_2, sum_3, num_accumulates, - old_num_accumulates, num_updates, sum_1, sum_2, sum_3, - num_accumulates, old_num_accumulates, num_updates, - 'average_window', self.average_window, 'min_average_window', - self.min_average_window, 'max_average_window', - self.max_average_window) + param_and_grad[0], + sum_1, + sum_2, + sum_3, + num_accumulates, + old_num_accumulates, + num_updates, + sum_1, + sum_2, + sum_3, + num_accumulates, + old_num_accumulates, + num_updates, + 'average_window', + self.average_window, + 'min_average_window', + self.min_average_window, + 'max_average_window', + self.max_average_window, + ) return None block = framework.default_main_program().global_block() @@ -262,7 +288,7 @@ class ModelAverage(Optimizer): "in_sum_3": sum_3, "in_num_accumulates": num_accumulates, "in_old_num_accumulates": old_num_accumulates, - "in_num_updates": num_updates + "in_num_updates": num_updates, } outputs = { @@ -274,20 +300,20 @@ class ModelAverage(Optimizer): "out_num_updates": num_updates, } - average_accumulates_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + average_accumulates_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return average_accumulates_op @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): """ Add operations to minimize ``loss`` by updating ``parameters``. @@ -423,10 +449,12 @@ class ModelAverage(Optimizer): """ if framework._non_static_mode(): for param in self._parameter_list: - num_accumulates = self._get_accumulator('num_accumulates', - param) + num_accumulates = self._get_accumulator( + 'num_accumulates', param + ) old_num_accumulates = self._get_accumulator( - 'old_num_accumulates', param) + 'old_num_accumulates', param + ) sum_1 = self._get_accumulator('sum_1', param) sum_2 = self._get_accumulator('sum_2', param) sum_3 = self._get_accumulator('sum_3', param) @@ -436,8 +464,9 @@ class ModelAverage(Optimizer): total_param = sum_1 + sum_2 + sum_3 total_accumulates = num_accumulates + old_num_accumulates total_param = paddle.cast(total_param, dtype='float32') - total_accumulates = paddle.cast(total_accumulates, - dtype='float32') + total_accumulates = paddle.cast( + total_accumulates, dtype='float32' + ) average_param = total_param / total_accumulates paddle.assign(average_param, param) try: @@ -448,7 +477,8 @@ class ModelAverage(Optimizer): return if executor is None: raise RuntimeError( - "Executor should not be None in static graph mode.") + "Executor should not be None in static graph mode." + ) executor.run(self.apply_program) try: yield @@ -504,7 +534,8 @@ class ModelAverage(Optimizer): return if executor is None: raise RuntimeError( - "Executor should not be None in static graph mode.") + "Executor should not be None in static graph mode." + ) executor.run(self.restore_program) def _add_average_apply_op(self, block, param): @@ -514,18 +545,22 @@ class ModelAverage(Optimizer): sum_2 = block._clone_variable(self._get_accumulator('sum_2', param)) sum_3 = block._clone_variable(self._get_accumulator('sum_3', param)) num_accumulates = block._clone_variable( - self._get_accumulator('num_accumulates', param)) + self._get_accumulator('num_accumulates', param) + ) old_num_accumulates = block._clone_variable( - self._get_accumulator('old_num_accumulates', param)) + self._get_accumulator('old_num_accumulates', param) + ) # backup param value to grad layers.assign(input=param, output=grad) # param = (sum_1 + sum_2 + sum_3) / (num_accumulates + old_num_accumulates) tmp = layers.sum(x=[num_accumulates, old_num_accumulates]) sum = layers.sum(x=[sum_1, sum_2, sum_3]) tmp = layers.cast( - x=tmp, dtype='float32' if self._dtype is None else self._dtype) + x=tmp, dtype='float32' if self._dtype is None else self._dtype + ) sum = layers.cast( - x=sum, dtype='float32' if self._dtype is None else self._dtype) + x=sum, dtype='float32' if self._dtype is None else self._dtype + ) layers.ops._elementwise_div(x=sum, y=tmp, out=param) def _add_average_restore_op(self, block, param): diff --git a/python/paddle/incubate/passes/fuse_resnet_unit_pass.py b/python/paddle/incubate/passes/fuse_resnet_unit_pass.py index fce745ffaf8..6441427f469 100644 --- a/python/paddle/incubate/passes/fuse_resnet_unit_pass.py +++ b/python/paddle/incubate/passes/fuse_resnet_unit_pass.py @@ -21,32 +21,35 @@ def set_resnet_unit_attrs(resnet_unit, has_shortcut): resnet_unit.SetAttr("has_shortcut", has_shortcut) resnet_unit.SetAttr("data_format", 'NHWC') resnet_unit.SetAttr("dilation", 1) - resnet_unit.Attr("stride").MappedPattern(op="conv2d", - name="strides", - element_index=0) - resnet_unit.Attr("padding").MappedPattern(op="conv2d", - name="paddings", - element_index=0) + resnet_unit.Attr("stride").MappedPattern( + op="conv2d", name="strides", element_index=0 + ) + resnet_unit.Attr("padding").MappedPattern( + op="conv2d", name="paddings", element_index=0 + ) resnet_unit.Attr("group").MappedPattern(op="conv2d", name="groups") resnet_unit.Attr("op_device").MappedPattern(op="conv2d", name="op_device") - resnet_unit.Attr("op_namescope").MappedPattern(op="conv2d", - name="op_namescope") + resnet_unit.Attr("op_namescope").MappedPattern( + op="conv2d", name="op_namescope" + ) resnet_unit.Attr("momentum").MappedPattern(op="batch_norm", name="momentum") resnet_unit.Attr("epsilon").MappedPattern(op="batch_norm", name="epsilon") - resnet_unit.Attr("use_global_stats").MappedPattern(op="batch_norm", - name="use_global_stats") + resnet_unit.Attr("use_global_stats").MappedPattern( + op="batch_norm", name="use_global_stats" + ) def set_resnet_unit_outputs(resnet_unit, meanX, varX, meanZ=None, varZ=None): - resnet_unit.SetOutputs(RunningMeanX=meanX, - RunningVarX=varX, - RunningMeanZ=meanZ, - RunningVarZ=varZ) + resnet_unit.SetOutputs( + RunningMeanX=meanX, + RunningVarX=varX, + RunningMeanZ=meanZ, + RunningVarZ=varZ, + ) @ir.RegisterPass def fuse_resnet_unit(): - def pattern_conv_bn(x, filter, scale, bias, mean, var): filter.Attr("shape")[0].Mod(32).EQ(0) filter.Attr("shape")[1].Mod(8).EQ(0) @@ -54,11 +57,9 @@ def fuse_resnet_unit(): filter.Attr("shape")[3].EQ(1) conv2d = ir.PassDesc.OP.conv2d(Input=x, Filter=filter) conv2d.SetAttr("data_format", 'NHWC') - bn = ir.PassDesc.OP.batch_norm(X=conv2d, - Bias=bias, - Mean=mean, - Scale=scale, - Variance=var) + bn = ir.PassDesc.OP.batch_norm( + X=conv2d, Bias=bias, Mean=mean, Scale=scale, Variance=var + ) return bn def pattern_one_input(x, filter, scale, bias, mean, var): @@ -67,42 +68,68 @@ def fuse_resnet_unit(): return relu def replace_one_input(x, filter, scale, bias, mean, var): - resnet_unit = ir.PassDesc.OP.resnet_unit(X=x, - FilterX=filter, - ScaleX=scale, - BiasX=bias, - MeanX=mean, - VarX=var) + resnet_unit = ir.PassDesc.OP.resnet_unit( + X=x, FilterX=filter, ScaleX=scale, BiasX=bias, MeanX=mean, VarX=var + ) set_resnet_unit_attrs(resnet_unit, False) set_resnet_unit_outputs(resnet_unit, mean, var) return resnet_unit.Output("Y") - def pattern_two_input(x, filterX, scaleX, biasX, meanX, varX, z, filterZ, - scaleZ, biasZ, meanZ, varZ): + def pattern_two_input( + x, + filterX, + scaleX, + biasX, + meanX, + varX, + z, + filterZ, + scaleZ, + biasZ, + meanZ, + varZ, + ): bnX = pattern_conv_bn(x, filterX, scaleX, biasX, meanX, varX) bnZ = pattern_conv_bn(x, filterZ, scaleZ, biasZ, meanZ, varZ) - ewadd = ir.PassDesc.OP.elementwise_add(X=bnX.Output("Y"), - Y=bnZ.Output("Y")) + ewadd = ir.PassDesc.OP.elementwise_add( + X=bnX.Output("Y"), Y=bnZ.Output("Y") + ) relu = ir.PassDesc.OP.relu(X=ewadd) return relu - def replace_two_input(x, filterX, scaleX, biasX, meanX, varX, z, filterZ, - scaleZ, biasZ, meanZ, varZ): - resnet_unit = ir.PassDesc.OP.resnet_unit(X=x, - FilterX=filterX, - ScaleX=scaleX, - BiasX=biasX, - MeanX=meanX, - VarX=varX, - Z=z, - FilterZ=filterZ, - ScaleZ=scaleZ, - BiasZ=biasZ, - MeanZ=meanZ, - VarZ=varZ) + def replace_two_input( + x, + filterX, + scaleX, + biasX, + meanX, + varX, + z, + filterZ, + scaleZ, + biasZ, + meanZ, + varZ, + ): + resnet_unit = ir.PassDesc.OP.resnet_unit( + X=x, + FilterX=filterX, + ScaleX=scaleX, + BiasX=biasX, + MeanX=meanX, + VarX=varX, + Z=z, + FilterZ=filterZ, + ScaleZ=scaleZ, + BiasZ=biasZ, + MeanZ=meanZ, + VarZ=varZ, + ) set_resnet_unit_attrs(resnet_unit, True) set_resnet_unit_outputs(resnet_unit, meanX, varX, meanZ, varZ) return resnet_unit.Output("Y") - return (pattern_one_input, replace_one_input), (pattern_two_input, - replace_two_input) + return (pattern_one_input, replace_one_input), ( + pattern_two_input, + replace_two_input, + ) diff --git a/python/paddle/incubate/tensor/math.py b/python/paddle/incubate/tensor/math.py index 005d2cee2dd..36f4944427f 100644 --- a/python/paddle/incubate/tensor/math.py +++ b/python/paddle/incubate/tensor/math.py @@ -21,10 +21,12 @@ import paddle.utils.deprecated as deprecated __all__ = [] -@deprecated(since="2.4.0", - update_to="paddle.geometric.segment_sum", - level=1, - reason="paddle.incubate.segment_sum will be removed in future") +@deprecated( + since="2.4.0", + update_to="paddle.geometric.segment_sum", + level=1, + reason="paddle.incubate.segment_sum will be removed in future", +) def segment_sum(data, segment_ids, name=None): r""" Segment Sum Operator. @@ -59,36 +61,36 @@ def segment_sum(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "SUM")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "SUM") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "SUM" + ) return out - check_variable_and_dtype(data, "X", - ("float32", "float64", "int32", "int64"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + check_variable_and_dtype( + data, "X", ("float32", "float64", "int32", "int64"), "segment_pool" + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_sum", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "SUM"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "SUM"}, + ) return out -@deprecated(since="2.4.0", - update_to="paddle.geometric.segment_mean", - level=1, - reason="paddle.incubate.segment_mean will be removed in future") +@deprecated( + since="2.4.0", + update_to="paddle.geometric.segment_mean", + level=1, + reason="paddle.incubate.segment_mean will be removed in future", +) def segment_mean(data, segment_ids, name=None): r""" Segment mean Operator. @@ -125,36 +127,36 @@ def segment_mean(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "MEAN")[0] if _non_static_mode(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MEAN") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MEAN" + ) return out - check_variable_and_dtype(data, "X", - ("float32", "float64", "int32", "int64"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + check_variable_and_dtype( + data, "X", ("float32", "float64", "int32", "int64"), "segment_pool" + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_mean", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MEAN"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MEAN"}, + ) return out -@deprecated(since="2.4.0", - update_to="paddle.geometric.segment_min", - level=1, - reason="paddle.incubate.segment_min will be removed in future") +@deprecated( + since="2.4.0", + update_to="paddle.geometric.segment_min", + level=1, + reason="paddle.incubate.segment_min will be removed in future", +) def segment_min(data, segment_ids, name=None): r""" Segment min operator. @@ -191,36 +193,36 @@ def segment_min(data, segment_ids, name=None): return _C_ops.segment_pool(data, segment_ids, "MIN")[0] if _non_static_mode(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MIN") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MIN" + ) return out - check_variable_and_dtype(data, "X", - ("float32", "float64", "int32", "int64"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + check_variable_and_dtype( + data, "X", ("float32", "float64", "int32", "int64"), "segment_pool" + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_min", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MIN"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MIN"}, + ) return out -@deprecated(since="2.4.0", - update_to="paddle.geometric.segment_max", - level=1, - reason="paddle.incubate.segment_max will be removed in future") +@deprecated( + since="2.4.0", + update_to="paddle.geometric.segment_max", + level=1, + reason="paddle.incubate.segment_max will be removed in future", +) def segment_max(data, segment_ids, name=None): r""" Segment max operator. @@ -258,27 +260,25 @@ def segment_max(data, segment_ids, name=None): return out if _non_static_mode(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MAX") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MAX" + ) return out - check_variable_and_dtype(data, "X", - ("float32", "float64", "int32", "int64"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + check_variable_and_dtype( + data, "X", ("float32", "float64", "int32", "int64"), "segment_pool" + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_max", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MAX"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MAX"}, + ) return out diff --git a/python/paddle/incubate/xpu/resnet_block.py b/python/paddle/incubate/xpu/resnet_block.py index 1eb3637fa58..148ea5a5e91 100644 --- a/python/paddle/incubate/xpu/resnet_block.py +++ b/python/paddle/incubate/xpu/resnet_block.py @@ -24,106 +24,232 @@ from paddle import _legacy_C_ops __all__ = ['resnet_basic_block', 'ResNetBasicBlock'] -def resnet_basic_block(x, - filter1, - scale1, - bias1, - mean1, - var1, - filter2, - scale2, - bias2, - mean2, - var2, - filter3, - scale3, - bias3, - mean3, - var3, - stride1, - stride2, - stride3, - padding1, - padding2, - padding3, - dilation1, - dilation2, - dilation3, - groups, - momentum, - eps, - data_format, - has_shortcut, - use_global_stats=None, - training=False, - trainable_statistics=False, - find_conv_max=True): +def resnet_basic_block( + x, + filter1, + scale1, + bias1, + mean1, + var1, + filter2, + scale2, + bias2, + mean2, + var2, + filter3, + scale3, + bias3, + mean3, + var3, + stride1, + stride2, + stride3, + padding1, + padding2, + padding3, + dilation1, + dilation2, + dilation3, + groups, + momentum, + eps, + data_format, + has_shortcut, + use_global_stats=None, + training=False, + trainable_statistics=False, + find_conv_max=True, +): if fluid.framework._non_static_mode(): - attrs = ('stride1', stride1, 'stride2', stride2, 'stride3', stride3, - 'padding1', padding1, 'padding2', padding2, 'padding3', - padding3, 'dilation1', dilation1, 'dilation2', dilation2, - 'dilation3', dilation3, 'group', groups, 'momentum', momentum, - 'epsilon', eps, 'data_format', data_format, 'has_shortcut', - has_shortcut, 'use_global_stats', use_global_stats, - "trainable_statistics", trainable_statistics, 'is_test', - not training, 'act_type', "relu", 'find_conv_input_max', - find_conv_max) + attrs = ( + 'stride1', + stride1, + 'stride2', + stride2, + 'stride3', + stride3, + 'padding1', + padding1, + 'padding2', + padding2, + 'padding3', + padding3, + 'dilation1', + dilation1, + 'dilation2', + dilation2, + 'dilation3', + dilation3, + 'group', + groups, + 'momentum', + momentum, + 'epsilon', + eps, + 'data_format', + data_format, + 'has_shortcut', + has_shortcut, + 'use_global_stats', + use_global_stats, + "trainable_statistics", + trainable_statistics, + 'is_test', + not training, + 'act_type', + "relu", + 'find_conv_input_max', + find_conv_max, + ) - out, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ = \ - _legacy_C_ops.resnet_basic_block(x, filter1, scale1, bias1, mean1, var1, filter2, scale2, bias2, mean2, var2, \ - filter3, scale3, bias3, mean3, var3, mean1, var1, mean2, var2, mean3, var3, *attrs) + ( + out, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + ) = _legacy_C_ops.resnet_basic_block( + x, + filter1, + scale1, + bias1, + mean1, + var1, + filter2, + scale2, + bias2, + mean2, + var2, + filter3, + scale3, + bias3, + mean3, + var3, + mean1, + var1, + mean2, + var2, + mean3, + var3, + *attrs + ) return out helper = LayerHelper('resnet_basic_block', **locals()) bn_param_dtype = fluid.core.VarDesc.VarType.FP32 max_dtype = fluid.core.VarDesc.VarType.FP32 - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - conv1 = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + conv1 = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean1 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd1 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) - running_mean1 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if mean1 is None else mean1 - running_var1 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if var1 is None else var1 - conv2 = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - conv2_input = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) + running_mean1 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if mean1 is None + else mean1 + ) + running_var1 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if var1 is None + else var1 + ) + conv2 = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + conv2_input = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean2 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd2 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) - running_mean2 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if mean2 is None else mean2 - running_var2 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if var2 is None else var2 - conv3 = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) + running_mean2 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if mean2 is None + else mean2 + ) + running_var2 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if var2 is None + else var2 + ) + conv3 = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean3 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd3 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) - running_mean3 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if mean3 is None else mean3 - running_var3 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if var3 is None else var3 + dtype=bn_param_dtype, stop_gradient=True + ) + running_mean3 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if mean3 is None + else mean3 + ) + running_var3 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if var3 is None + else var3 + ) conv1_input_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv1_filter_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv2_input_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv2_filter_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv3_input_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv3_filter_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) inputs = { 'X': x, @@ -163,7 +289,7 @@ def resnet_basic_block(x, "trainable_statistics": trainable_statistics, 'is_test': not training, 'act_type': "relu", - 'find_conv_input_max': find_conv_max + 'find_conv_input_max': find_conv_max, } outputs = { @@ -191,10 +317,9 @@ def resnet_basic_block(x, 'MaxInput3': conv3_input_max, 'MaxFilter3': conv3_filter_max, } - helper.append_op(type='resnet_basic_block', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='resnet_basic_block', inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -303,56 +428,60 @@ class ResNetBasicBlock(Layer): print(out.shape) # [2, 8, 16, 16] """ - def __init__(self, - num_channels1, - num_filter1, - filter1_size, - num_channels2, - num_filter2, - filter2_size, - num_channels3, - num_filter3, - filter3_size, - stride1=1, - stride2=1, - stride3=1, - act='relu', - momentum=0.9, - eps=1e-5, - data_format='NCHW', - has_shortcut=False, - use_global_stats=False, - is_test=False, - filter1_attr=None, - scale1_attr=None, - bias1_attr=None, - moving_mean1_name=None, - moving_var1_name=None, - filter2_attr=None, - scale2_attr=None, - bias2_attr=None, - moving_mean2_name=None, - moving_var2_name=None, - filter3_attr=None, - scale3_attr=None, - bias3_attr=None, - moving_mean3_name=None, - moving_var3_name=None, - padding1=0, - padding2=0, - padding3=0, - dilation1=1, - dilation2=1, - dilation3=1, - trainable_statistics=False, - find_conv_max=True): + def __init__( + self, + num_channels1, + num_filter1, + filter1_size, + num_channels2, + num_filter2, + filter2_size, + num_channels3, + num_filter3, + filter3_size, + stride1=1, + stride2=1, + stride3=1, + act='relu', + momentum=0.9, + eps=1e-5, + data_format='NCHW', + has_shortcut=False, + use_global_stats=False, + is_test=False, + filter1_attr=None, + scale1_attr=None, + bias1_attr=None, + moving_mean1_name=None, + moving_var1_name=None, + filter2_attr=None, + scale2_attr=None, + bias2_attr=None, + moving_mean2_name=None, + moving_var2_name=None, + filter3_attr=None, + scale3_attr=None, + bias3_attr=None, + moving_mean3_name=None, + moving_var3_name=None, + padding1=0, + padding2=0, + padding3=0, + dilation1=1, + dilation2=1, + dilation3=1, + trainable_statistics=False, + find_conv_max=True, + ): super(ResNetBasicBlock, self).__init__() self._stride1 = stride1 self._stride2 = stride2 - self._kernel1_size = utils.convert_to_list(filter1_size, 2, - 'filter1_size') - self._kernel2_size = utils.convert_to_list(filter2_size, 2, - 'filter2_size') + self._kernel1_size = utils.convert_to_list( + filter1_size, 2, 'filter1_size' + ) + self._kernel2_size = utils.convert_to_list( + filter2_size, 2, 'filter2_size' + ) self._dilation1 = dilation1 self._dilation2 = dilation2 self._padding1 = padding1 @@ -369,8 +498,9 @@ class ResNetBasicBlock(Layer): self._find_conv_max = find_conv_max if has_shortcut: - self._kernel3_size = utils.convert_to_list(filter3_size, 2, - 'filter3_size') + self._kernel3_size = utils.convert_to_list( + filter3_size, 2, 'filter3_size' + ) self._padding3 = padding3 self._stride3 = stride3 self._dilation3 = dilation3 @@ -385,11 +515,13 @@ class ResNetBasicBlock(Layer): if data_format not in valid_format: raise ValueError( "conv_format must be one of {}, but got conv_format={}".format( - valid_format, data_format)) + valid_format, data_format + ) + ) def _get_default_param_initializer(channels, kernel_size): filter_elem_num = np.prod(kernel_size) * channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return I.Normal(0.0, std) # init filter @@ -403,92 +535,128 @@ class ResNetBasicBlock(Layer): shape=filter1_shape, attr=filter1_attr, default_initializer=_get_default_param_initializer( - num_channels1, self._kernel1_size)) + num_channels1, self._kernel1_size + ), + ) self.scale_1 = self.create_parameter( shape=bn1_param_shape, attr=scale1_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_1 = self.create_parameter(shape=bn1_param_shape, - attr=bias1_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_1 = self.create_parameter(attr=ParamAttr( - name=moving_mean1_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn1_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_1 = self.create_parameter( + shape=bn1_param_shape, + attr=bias1_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_1 = self.create_parameter( + attr=ParamAttr( + name=moving_mean1_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn1_param_shape, + dtype=bn_param_dtype, + ) self.mean_1.stop_gradient = True self.var_1 = self.create_parameter( - attr=ParamAttr(name=moving_var1_name, - initializer=I.Constant(1.0), - trainable=False), + attr=ParamAttr( + name=moving_var1_name, + initializer=I.Constant(1.0), + trainable=False, + ), shape=bn1_param_shape, - dtype=bn_param_dtype) + dtype=bn_param_dtype, + ) self.var_1.stop_gradient = True self.filter_2 = self.create_parameter( shape=filter2_shape, attr=filter2_attr, default_initializer=_get_default_param_initializer( - num_channels2, self._kernel2_size)) + num_channels2, self._kernel2_size + ), + ) self.scale_2 = self.create_parameter( shape=bn2_param_shape, attr=scale2_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_2 = self.create_parameter(shape=bn2_param_shape, - attr=bias2_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_2 = self.create_parameter(attr=ParamAttr( - name=moving_mean2_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn2_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_2 = self.create_parameter( + shape=bn2_param_shape, + attr=bias2_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_2 = self.create_parameter( + attr=ParamAttr( + name=moving_mean2_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn2_param_shape, + dtype=bn_param_dtype, + ) self.mean_2.stop_gradient = True self.var_2 = self.create_parameter( - attr=ParamAttr(name=moving_var2_name, - initializer=I.Constant(1.0), - trainable=False), + attr=ParamAttr( + name=moving_var2_name, + initializer=I.Constant(1.0), + trainable=False, + ), shape=bn2_param_shape, - dtype=bn_param_dtype) + dtype=bn_param_dtype, + ) self.var_2.stop_gradient = True if has_shortcut: bn3_param_shape = [1, 1, num_filter3] filter3_shape = [ - num_filter3, num_channels3, filter3_size, filter3_size + num_filter3, + num_channels3, + filter3_size, + filter3_size, ] self.filter_3 = self.create_parameter( shape=filter3_shape, attr=filter3_attr, default_initializer=_get_default_param_initializer( - num_channels3, self._kernel3_size)) + num_channels3, self._kernel3_size + ), + ) self.scale_3 = self.create_parameter( shape=bn3_param_shape, attr=scale3_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_3 = self.create_parameter(shape=bn3_param_shape, - attr=bias3_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_3 = self.create_parameter(attr=ParamAttr( - name=moving_mean3_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn3_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_3 = self.create_parameter( + shape=bn3_param_shape, + attr=bias3_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_3 = self.create_parameter( + attr=ParamAttr( + name=moving_mean3_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn3_param_shape, + dtype=bn_param_dtype, + ) self.mean_3.stop_gradient = True - self.var_3 = self.create_parameter(attr=ParamAttr( - name=moving_var3_name, - initializer=I.Constant(1.0), - trainable=False), - shape=bn3_param_shape, - dtype=bn_param_dtype) + self.var_3 = self.create_parameter( + attr=ParamAttr( + name=moving_var3_name, + initializer=I.Constant(1.0), + trainable=False, + ), + shape=bn3_param_shape, + dtype=bn_param_dtype, + ) self.var_3.stop_gradient = True else: self.filter_3 = None @@ -532,5 +700,6 @@ class ResNetBasicBlock(Layer): use_global_stats=self._use_global_stats, training=self.training, trainable_statistics=self._trainable_statistics, - find_conv_max=self._find_conv_max) + find_conv_max=self._find_conv_max, + ) return out diff --git a/python/paddle/inference/__init__.py b/python/paddle/inference/__init__.py index 1b9a96f437b..359f5caccb0 100644 --- a/python/paddle/inference/__init__.py +++ b/python/paddle/inference/__init__.py @@ -28,8 +28,18 @@ from ..fluid.inference import get_num_bytes_of_data_type # noqa: F401 from ..fluid.inference import PredictorPool # noqa: F401 __all__ = [ # noqa - 'Config', 'DataType', 'PlaceType', 'PrecisionType', 'Tensor', 'Predictor', - 'create_predictor', 'get_version', '_get_phi_kernel_name', - 'get_trt_compile_version', 'convert_to_mixed_precision', - 'get_trt_runtime_version', 'get_num_bytes_of_data_type', 'PredictorPool' + 'Config', + 'DataType', + 'PlaceType', + 'PrecisionType', + 'Tensor', + 'Predictor', + 'create_predictor', + 'get_version', + '_get_phi_kernel_name', + 'get_trt_compile_version', + 'convert_to_mixed_precision', + 'get_trt_runtime_version', + 'get_num_bytes_of_data_type', + 'PredictorPool', ] diff --git a/python/paddle/io/__init__.py b/python/paddle/io/__init__.py index 87acda904b5..a9c0e9a2f2d 100755 --- a/python/paddle/io/__init__.py +++ b/python/paddle/io/__init__.py @@ -30,9 +30,20 @@ from ..fluid.dataloader import WeightedRandomSampler # noqa: F401 from ..fluid.dataloader import Subset # noqa: F401 from ..fluid.dataloader import random_split # noqa: F401 -__all__ = [ #noqa - 'Dataset', 'IterableDataset', 'TensorDataset', 'ComposeDataset', - 'ChainDataset', 'BatchSampler', 'DistributedBatchSampler', 'DataLoader', - 'get_worker_info', 'Sampler', 'SequenceSampler', 'RandomSampler', - 'WeightedRandomSampler', 'random_split', 'Subset' +__all__ = [ # noqa + 'Dataset', + 'IterableDataset', + 'TensorDataset', + 'ComposeDataset', + 'ChainDataset', + 'BatchSampler', + 'DistributedBatchSampler', + 'DataLoader', + 'get_worker_info', + 'Sampler', + 'SequenceSampler', + 'RandomSampler', + 'WeightedRandomSampler', + 'random_split', + 'Subset', ] diff --git a/python/paddle/jit/__init__.py b/python/paddle/jit/__init__.py index 857a5d8695a..a706d4d1211 100644 --- a/python/paddle/jit/__init__.py +++ b/python/paddle/jit/__init__.py @@ -26,6 +26,13 @@ from ..fluid.dygraph.io import TranslatedLayer # noqa: F401 from . import dy2static # noqa: F401 __all__ = [ # noqa - 'save', 'load', 'TracedLayer', 'to_static', 'ProgramTranslator', - 'TranslatedLayer', 'set_code_level', 'set_verbosity', 'not_to_static' + 'save', + 'load', + 'TracedLayer', + 'to_static', + 'ProgramTranslator', + 'TranslatedLayer', + 'set_code_level', + 'set_verbosity', + 'not_to_static', ] diff --git a/python/paddle/jit/dy2static/convert_call_func.py b/python/paddle/jit/dy2static/convert_call_func.py index 247efa8f4f1..17e33f53bca 100644 --- a/python/paddle/jit/dy2static/convert_call_func.py +++ b/python/paddle/jit/dy2static/convert_call_func.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.dygraph.dygraph_to_static.convert_call_func import convert_call # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_call_func import ( # noqa: F401 + convert_call, +) __all__ = [] diff --git a/python/paddle/jit/dy2static/convert_operators.py b/python/paddle/jit/dy2static/convert_operators.py index 3a941a1dff3..54210302c41 100644 --- a/python/paddle/jit/dy2static/convert_operators.py +++ b/python/paddle/jit/dy2static/convert_operators.py @@ -12,19 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.dygraph.dygraph_to_static.convert_operators import cast_bool_if_necessary # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_assert # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_ifelse # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_len # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_and # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_not # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_or # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_pop # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_print # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_shape_compare # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_var_dtype # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_shape # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_while_loop # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import unpack_by_structure, indexable, convert_attr # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import ( # noqa: F401 + cast_bool_if_necessary, + convert_assert, + convert_ifelse, + convert_len, + convert_logical_and, + convert_logical_not, + convert_logical_or, + convert_pop, + convert_print, + convert_shape_compare, + convert_var_dtype, + convert_shape, + convert_while_loop, + unpack_by_structure, + indexable, + convert_attr, +) __all__ = [] diff --git a/python/paddle/jit/dy2static/variable_trans_func.py b/python/paddle/jit/dy2static/variable_trans_func.py index 582e677a503..88f8cd3c2c4 100644 --- a/python/paddle/jit/dy2static/variable_trans_func.py +++ b/python/paddle/jit/dy2static/variable_trans_func.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import create_bool_as_type # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.variable_trans_func import ( # noqa: F401 + create_bool_as_type, + to_static_variable, +) __all__ = [] diff --git a/python/paddle/jit/layer.py b/python/paddle/jit/layer.py index 97b59894850..25d9ca5dbcc 100644 --- a/python/paddle/jit/layer.py +++ b/python/paddle/jit/layer.py @@ -18,7 +18,6 @@ from paddle.fluid.core import Load class Layer(object): - def __init__(self): self.cpp_layer = None # {name: Function} @@ -34,8 +33,7 @@ class Layer(object): setattr(self, name, self.functions[name]) -class Function(): - +class Function: def __init__(self, function, info): self.function = function self.info = FunctionInfo(info) @@ -44,8 +42,7 @@ class Function(): return core.eager.jit_function_call(self.function, args) -class FunctionInfo(): - +class FunctionInfo: def __init__(self, info): self.info = info diff --git a/python/paddle/linalg.py b/python/paddle/linalg.py index 834b631e5c5..9ea43479569 100644 --- a/python/paddle/linalg.py +++ b/python/paddle/linalg.py @@ -38,7 +38,7 @@ from .tensor.linalg import triangular_solve # noqa: F401 from .tensor.linalg import lstsq __all__ = [ - 'cholesky', #noqa + 'cholesky', # noqa 'norm', 'cond', 'cov', @@ -61,5 +61,5 @@ __all__ = [ 'solve', 'cholesky_solve', 'triangular_solve', - 'lstsq' + 'lstsq', ] diff --git a/python/paddle/metric/__init__.py b/python/paddle/metric/__init__.py index 70fe075e577..60dff58ec48 100644 --- a/python/paddle/metric/__init__.py +++ b/python/paddle/metric/__init__.py @@ -19,6 +19,11 @@ from .metrics import Recall # noqa: F401 from .metrics import Auc # noqa: F401 from .metrics import accuracy # noqa: F401 -__all__ = [ #noqa - 'Metric', 'Accuracy', 'Precision', 'Recall', 'Auc', 'accuracy' +__all__ = [ # noqa + 'Metric', + 'Accuracy', + 'Precision', + 'Recall', + 'Auc', + 'accuracy', ] diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 1404c7531b4..e8493b17b92 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -116,7 +116,9 @@ class Metric(metaclass=abc.ABCMeta): """ raise NotImplementedError( "function 'reset' not implemented in {}.".format( - self.__class__.__name__)) + self.__class__.__name__ + ) + ) @abc.abstractmethod def update(self, *args): @@ -132,7 +134,9 @@ class Metric(metaclass=abc.ABCMeta): """ raise NotImplementedError( "function 'update' not implemented in {}.".format( - self.__class__.__name__)) + self.__class__.__name__ + ) + ) @abc.abstractmethod def accumulate(self): @@ -141,7 +145,9 @@ class Metric(metaclass=abc.ABCMeta): """ raise NotImplementedError( "function 'accumulate' not implemented in {}.".format( - self.__class__.__name__)) + self.__class__.__name__ + ) + ) @abc.abstractmethod def name(self): @@ -150,7 +156,9 @@ class Metric(metaclass=abc.ABCMeta): """ raise NotImplementedError( "function 'name' not implemented in {}.".format( - self.__class__.__name__)) + self.__class__.__name__ + ) + ) def compute(self, *args): """ @@ -231,7 +239,7 @@ class Accuracy(Metric): """ - def __init__(self, topk=(1, ), name=None, *args, **kwargs): + def __init__(self, topk=(1,), name=None, *args, **kwargs): super(Accuracy, self).__init__(*args, **kwargs) self.topk = topk self.maxk = max(topk) @@ -253,12 +261,12 @@ class Accuracy(Metric): Tensor: Correct mask, a tensor with shape [batch_size, d0, ..., topk]. """ pred = paddle.argsort(pred, descending=True) - pred = paddle.slice(pred, - axes=[len(pred.shape) - 1], - starts=[0], - ends=[self.maxk]) - if (len(label.shape) == 1) or \ - (len(label.shape) == 2 and label.shape[-1] == 1): + pred = paddle.slice( + pred, axes=[len(pred.shape) - 1], starts=[0], ends=[self.maxk] + ) + if (len(label.shape) == 1) or ( + len(label.shape) == 2 and label.shape[-1] == 1 + ): # In static mode, the real label data shape may be different # from shape defined by paddle.static.InputSpec in model # building, reshape to the right shape. @@ -297,7 +305,7 @@ class Accuracy(Metric): """ Resets all of the metric state. """ - self.total = [0.] * len(self.topk) + self.total = [0.0] * len(self.topk) self.count = [0] * len(self.topk) def accumulate(self): @@ -306,7 +314,7 @@ class Accuracy(Metric): """ res = [] for t, c in zip(self.total, self.count): - r = float(t) / c if c > 0 else 0. + r = float(t) / c if c > 0 else 0.0 res.append(r) res = res[0] if len(self.topk) == 1 else res return res @@ -446,7 +454,7 @@ class Precision(Metric): A scaler float: results of the calculated precision. """ ap = self.tp + self.fp - return float(self.tp) / ap if ap != 0 else .0 + return float(self.tp) / ap if ap != 0 else 0.0 def name(self): """ @@ -572,7 +580,7 @@ class Recall(Metric): A scaler float: results of the calculated Recall. """ recall = self.tp + self.fn - return float(self.tp) / recall if recall != 0 else .0 + return float(self.tp) / recall if recall != 0 else 0.0 def reset(self): """ @@ -670,12 +678,9 @@ class Auc(Metric): model.fit(data, batch_size=16) """ - def __init__(self, - curve='ROC', - num_thresholds=4095, - name='auc', - *args, - **kwargs): + def __init__( + self, curve='ROC', num_thresholds=4095, name='auc', *args, **kwargs + ): super(Auc, self).__init__(*args, **kwargs) self._curve = curve self._num_thresholds = num_thresholds @@ -737,11 +742,14 @@ class Auc(Metric): tot_neg_prev = tot_neg tot_pos += self._stat_pos[idx] tot_neg += self._stat_neg[idx] - auc += self.trapezoid_area(tot_neg, tot_neg_prev, tot_pos, - tot_pos_prev) + auc += self.trapezoid_area( + tot_neg, tot_neg_prev, tot_pos, tot_pos_prev + ) idx -= 1 - return auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + return ( + auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + ) def reset(self): """ @@ -799,29 +807,29 @@ def accuracy(input, label, k=1, correct=None, total=None, name=None): total = _varbase_creator(dtype="int32") topk_out, topk_indices = paddle.topk(input, k=k) - _acc, _, _ = _legacy_C_ops.accuracy(topk_out, topk_indices, label, - correct, total) + _acc, _, _ = _legacy_C_ops.accuracy( + topk_out, topk_indices, label, correct, total + ) return _acc helper = LayerHelper("accuracy", **locals()) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'accuracy') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'accuracy' + ) topk_out, topk_indices = paddle.topk(input, k=k) acc_out = helper.create_variable_for_type_inference(dtype="float32") if correct is None: correct = helper.create_variable_for_type_inference(dtype="int32") if total is None: total = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type="accuracy", - inputs={ - "Out": [topk_out], - "Indices": [topk_indices], - "Label": [label] - }, - outputs={ - "Accuracy": [acc_out], - "Correct": [correct], - "Total": [total], - }) + helper.append_op( + type="accuracy", + inputs={"Out": [topk_out], "Indices": [topk_indices], "Label": [label]}, + outputs={ + "Accuracy": [acc_out], + "Correct": [correct], + "Total": [total], + }, + ) return acc_out diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index 331131d6e23..8e02cdffd5e 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -163,35 +163,41 @@ from . import quant # noqa: F401 import paddle.utils.deprecated as deprecated -@deprecated(since="2.0.0", - update_to="paddle.nn.funcitional.diag_embed", - level=1, - reason="diag_embed in paddle.nn will be removed in future") +@deprecated( + since="2.0.0", + update_to="paddle.nn.funcitional.diag_embed", + level=1, + reason="diag_embed in paddle.nn will be removed in future", +) def diag_embed(*args): ''' - alias name of paddle.nn.functional.diag_embed + alias name of paddle.nn.functional.diag_embed ''' return functional.diag_embed(*args) -@deprecated(since="2.0.0", - update_to="paddle.nn.utils.remove_weight_norm", - level=1, - reason="remove_weight_norm in paddle.nn will be removed in future") +@deprecated( + since="2.0.0", + update_to="paddle.nn.utils.remove_weight_norm", + level=1, + reason="remove_weight_norm in paddle.nn will be removed in future", +) def remove_weight_norm(*args): ''' - alias name of paddle.nn.utils.remove_weight_norm + alias name of paddle.nn.utils.remove_weight_norm ''' return utils.remove_weight_norm(*args) -@deprecated(since="2.0.0", - update_to="paddle.nn.utils.weight_norm", - level=1, - reason="weight_norm in paddle.nn will be removed in future") +@deprecated( + since="2.0.0", + update_to="paddle.nn.utils.weight_norm", + level=1, + reason="weight_norm in paddle.nn will be removed in future", +) def weight_norm(*args): ''' - alias name of paddle.nn.utils.weight_norm + alias name of paddle.nn.utils.weight_norm ''' return utils.weight_norm(*args) diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index 929fd243715..254ab7ff8a0 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -68,10 +68,12 @@ def celu(x, alpha=1.0, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'celu') helper = LayerHelper("celu", **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='celu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'alpha': alpha}) + helper.append_op( + type='celu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'alpha': alpha}, + ) return out @@ -118,10 +120,12 @@ def elu(x, alpha=1.0, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'elu') helper = LayerHelper("elu", **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='elu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'alpha': alpha}) + helper.append_op( + type='elu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'alpha': alpha}, + ) return out @@ -131,7 +135,7 @@ def elu_(x, alpha=1.0, name=None): Inplace version of ``elu`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_nn_cn_elu`. """ - assert alpha >= 0., "elu_ only support alpha >= 0, please use elu instead." + assert alpha >= 0.0, "elu_ only support alpha >= 0, please use elu instead." if in_dygraph_mode(): return _C_ops.elu_(x, alpha) return _legacy_C_ops.elu_(x, 'alpha', alpha) @@ -187,10 +191,12 @@ def gelu(x, approximate=False, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'gelu') helper = LayerHelper("gelu", **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='gelu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'approximate': approximate}) + helper.append_op( + type='gelu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'approximate': approximate}, + ) return out @@ -233,14 +239,17 @@ def hardshrink(x, threshold=0.5, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.hard_shrink(x, 'threshold', threshold) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hardshrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hardshrink' + ) helper = LayerHelper('hardshrink', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='hard_shrink', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='hard_shrink', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out @@ -284,18 +293,18 @@ def hardtanh(x, min=-1.0, max=1.0, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.brelu(x, 't_min', min, 't_max', max) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hardtanh') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hardtanh' + ) helper = LayerHelper('hardtanh', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='brelu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 't_min': min, - 't_max': max - }) + helper.append_op( + type='brelu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'t_min': min, 't_max': max}, + ) return out @@ -341,18 +350,18 @@ def hardsigmoid(x, slope=0.1666667, offset=0.5, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.hard_sigmoid(x, 'slope', slope, 'offset', offset) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hardsigmoid') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hardsigmoid' + ) helper = LayerHelper('hardsigmoid', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='hard_sigmoid', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'slope': slope, - 'offset': offset - }) + helper.append_op( + type='hard_sigmoid', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'slope': slope, 'offset': offset}, + ) return out @@ -395,8 +404,9 @@ def hardswish(x, name=None): if in_dygraph_mode(): return _C_ops.hardswish(x, 6, 6, 3) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hardswish') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hardswish' + ) helper = LayerHelper('hardswish', **locals()) out = helper.create_variable_for_type_inference(x.dtype) @@ -444,14 +454,17 @@ def leaky_relu(x, negative_slope=0.01, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.leaky_relu(x, 'alpha', negative_slope) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'leaky_relu') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'leaky_relu' + ) helper = LayerHelper('leaky_relu', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='leaky_relu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'alpha': negative_slope}) + helper.append_op( + type='leaky_relu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'alpha': negative_slope}, + ) return out @@ -498,60 +511,68 @@ def prelu(x, weight, data_format="NCHW", name=None): # [ 6. , 7. , 8. , 9. ]]]] """ check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'prelu') - check_variable_and_dtype(weight, 'weight', - ['float16', 'float32', 'float64'], 'prelu') + check_variable_and_dtype( + weight, 'weight', ['float16', 'float32', 'float64'], 'prelu' + ) - assert len(weight.shape - ) == 1, "The dim count of weight shape should be 1 in prelu()." + assert ( + len(weight.shape) == 1 + ), "The dim count of weight shape should be 1 in prelu()." mode = 'all' if weight.shape[0] > 1: true_data_format = [ - 'NC', 'NCL', 'NCHW', 'NCDHW', 'NLC', 'NHWC', 'NDHWC' + 'NC', + 'NCL', + 'NCHW', + 'NCDHW', + 'NLC', + 'NHWC', + 'NDHWC', ] if data_format not in true_data_format: raise ValueError( "data_format must be one of 'NC', 'NCL', 'NCHW', 'NCDHW', " - "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format)) + "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format) + ) data_format = 'NCHW' if data_format[1] == 'C' else 'NHWC' - assert len( - x.shape - ) > 1, "The dim count of x should be equal or larger than 2 in prelu() when weight shape is not [1]." + assert ( + len(x.shape) > 1 + ), "The dim count of x should be equal or larger than 2 in prelu() when weight shape is not [1]." - #NOTE(GuoxiaWang): support NHWC data format + # NOTE(GuoxiaWang): support NHWC data format if data_format == 'NHWC': - assert weight.shape[0] == x.shape[ - -1], "The weight size should be equal to x input channel in prelu() when weight shape is not [1]." + assert ( + weight.shape[0] == x.shape[-1] + ), "The weight size should be equal to x input channel in prelu() when weight shape is not [1]." else: - assert weight.shape[0] == x.shape[ - 1], "The weight size should be equal to x input channel in prelu() when weight shape is not [1]." + assert ( + weight.shape[0] == x.shape[1] + ), "The weight size should be equal to x input channel in prelu() when weight shape is not [1]." mode = 'channel' if in_dygraph_mode(): return _C_ops.prelu(x, weight, data_format, mode) if _in_legacy_dygraph(): - return _legacy_C_ops.prelu(x, weight, 'mode', mode, 'data_format', - data_format) + return _legacy_C_ops.prelu( + x, weight, 'mode', mode, 'data_format', data_format + ) helper = LayerHelper('prelu', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type="prelu", - inputs={ - "X": x, - "Alpha": weight - }, - outputs={"Out": out}, - attrs={ - "mode": mode, - "data_format": data_format - }) + helper.append_op( + type="prelu", + inputs={"X": x, "Alpha": weight}, + outputs={"Out": out}, + attrs={"mode": mode, "data_format": data_format}, + ) return out -def rrelu(x, lower=1. / 8., upper=1. / 3., training=True, name=None): +def rrelu(x, lower=1.0 / 8.0, upper=1.0 / 3.0, training=True, name=None): r""" rrelu activation. @@ -623,47 +644,56 @@ def rrelu(x, lower=1. / 8., upper=1. / 3., training=True, name=None): """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'X', ['float16', 'float32', 'float64'], - 'rrelu') + check_variable_and_dtype( + x, 'X', ['float16', 'float32', 'float64'], 'rrelu' + ) if not isinstance(lower, float) or not isinstance(upper, float): raise TypeError( - "The lower and upper values must be float type. Received: lower {}, upper {}." - .format(lower, upper)) + "The lower and upper values must be float type. Received: lower {}, upper {}.".format( + lower, upper + ) + ) if lower < 0 or lower > 1: raise ValueError( - "The lower value must be no less than zero or greater than one. Received: {}." - .format(lower)) + "The lower value must be no less than zero or greater than one. Received: {}.".format( + lower + ) + ) if upper < lower: raise ValueError( - "The upper value must be greater than lower value. Received: lower {}, upper {}." - .format(lower, upper)) + "The upper value must be greater than lower value. Received: lower {}, upper {}.".format( + lower, upper + ) + ) if upper > 1: raise ValueError( "The upper value must be no greater than one. Received: {}.".format( - upper)) + upper + ) + ) is_test = not training if _in_legacy_dygraph(): - out, noise = _legacy_C_ops.rrelu(x, 'lower', lower, 'upper', upper, - 'is_test', is_test) + out, noise = _legacy_C_ops.rrelu( + x, 'lower', lower, 'upper', upper, 'is_test', is_test + ) return out helper = LayerHelper('rrelu', **locals()) out = helper.create_variable_for_type_inference(x.dtype) noise = helper.create_variable_for_type_inference(dtype=x.dtype) attrs = {'lower': lower, 'upper': upper, 'is_test': is_test} - helper.append_op(type='rrelu', - inputs={"X": x}, - outputs={ - "Out": out, - "Noise": noise - }, - attrs=attrs) + helper.append_op( + type='rrelu', + inputs={"X": x}, + outputs={"Out": out, "Noise": noise}, + attrs=attrs, + ) return out @@ -748,8 +778,9 @@ def log_sigmoid(x, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.logsigmoid(x) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'log_sigmoid') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'log_sigmoid' + ) helper = LayerHelper("log_sigmoid", **locals()) out = helper.create_variable_for_type_inference(x.dtype) helper.append_op(type='logsigmoid', inputs={'X': x}, outputs={'Out': out}) @@ -818,19 +849,19 @@ def maxout(x, groups, axis=1, name=None): if axis not in [1, -1, 3]: raise ValueError( "Attr(axis) should be 1 when data format is NCHW, -1 or 3 when data format is NHWC. Received " - "Attr(axis): %s." % str(axis)) + "Attr(axis): %s." % str(axis) + ) if axis == -1: axis = 3 helper = LayerHelper('maxout', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='maxout', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'groups': groups, - 'axis': axis - }) + helper.append_op( + type='maxout', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'groups': groups, 'axis': axis}, + ) return out @@ -869,17 +900,21 @@ def relu6(x, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'relu6') helper = LayerHelper('relu6', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='relu6', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='relu6', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out -def selu(x, - scale=1.0507009873554804934193349852946, - alpha=1.6732632423543772848170429916717, - name=None): +def selu( + x, + scale=1.0507009873554804934193349852946, + alpha=1.6732632423543772848170429916717, + name=None, +): r""" selu activation @@ -915,11 +950,13 @@ def selu(x, """ if scale <= 1.0: raise ValueError( - "The scale must be greater than 1.0. Received: {}.".format(scale)) + "The scale must be greater than 1.0. Received: {}.".format(scale) + ) if alpha < 0: raise ValueError( - "The alpha must be no less than zero. Received: {}.".format(alpha)) + "The alpha must be no less than zero. Received: {}.".format(alpha) + ) if in_dygraph_mode(): return _C_ops.selu(x, scale, alpha) @@ -929,13 +966,12 @@ def selu(x, check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'selu') helper = LayerHelper('selu', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='selu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'scale': scale, - 'alpha': alpha - }) + helper.append_op( + type='selu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'scale': scale, 'alpha': alpha}, + ) return out @@ -1096,44 +1132,50 @@ def softmax(x, axis=-1, dtype=None, name=None): use_cudnn = True if in_dygraph_mode(): - outs_cast = x if dtype is None \ - else _C_ops.cast(x, dtype) + outs_cast = x if dtype is None else _C_ops.cast(x, dtype) return _C_ops.softmax(outs_cast, axis) if _in_legacy_dygraph(): - outs_cast = x if dtype is None \ + outs_cast = ( + x + if dtype is None else _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) - return _legacy_C_ops.softmax(outs_cast, 'axis', axis, 'use_cudnn', - use_cudnn) + ) + return _legacy_C_ops.softmax( + outs_cast, 'axis', axis, 'use_cudnn', use_cudnn + ) if dtype is None: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softmax') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softmax' + ) else: check_dtype( - dtype, 'dtype', ['float32', 'float64'], 'softmax', - 'If dtype is not None, it only support float32 or float64.') + dtype, + 'dtype', + ['float32', 'float64'], + 'softmax', + 'If dtype is not None, it only support float32 or float64.', + ) helper = LayerHelper("softmax", **locals()) outs_cast = x if dtype is not None: outs_cast = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='cast', - inputs={'X': x}, - outputs={'Out': outs_cast}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': dtype - }) + helper.append_op( + type='cast', + inputs={'X': x}, + outputs={'Out': outs_cast}, + attrs={'in_dtype': x.dtype, 'out_dtype': dtype}, + ) outs_softmax = helper.create_variable_for_type_inference(outs_cast.dtype) - helper.append_op(type='softmax', - inputs={'X': outs_cast}, - outputs={'Out': outs_softmax}, - attrs={ - 'axis': axis, - 'use_cudnn': use_cudnn - }) + helper.append_op( + type='softmax', + inputs={'X': outs_cast}, + outputs={'Out': outs_softmax}, + attrs={'axis': axis, 'use_cudnn': use_cudnn}, + ) return outs_softmax @@ -1149,15 +1191,22 @@ def softmax_(x, axis=-1, dtype=None, name=None): use_cudnn = True if in_dygraph_mode(): - outs_cast = x if dtype is None \ + outs_cast = ( + x + if dtype is None else _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) + ) return _C_ops.softmax_(outs_cast, axis) if _in_legacy_dygraph(): - outs_cast = x if dtype is None \ + outs_cast = ( + x + if dtype is None else _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) - return _legacy_C_ops.softmax_(outs_cast, 'axis', axis, 'use_cudnn', - use_cudnn) + ) + return _legacy_C_ops.softmax_( + outs_cast, 'axis', axis, 'use_cudnn', use_cudnn + ) def softplus(x, beta=1, threshold=20, name=None): @@ -1195,17 +1244,17 @@ def softplus(x, beta=1, threshold=20, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.softplus(x, 'beta', beta, 'threshold', threshold) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softplus') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softplus' + ) helper = LayerHelper('softplus', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='softplus', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'beta': beta, - 'threshold': threshold - }) + helper.append_op( + type='softplus', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'beta': beta, 'threshold': threshold}, + ) return out @@ -1245,21 +1294,26 @@ def softshrink(x, threshold=0.5, name=None): if threshold < 0: raise ValueError( "The threshold must be no less than zero. Received: {}.".format( - threshold)) + threshold + ) + ) if in_dygraph_mode(): return _C_ops.softshrink(x, threshold) if _in_legacy_dygraph(): return _legacy_C_ops.softshrink(x, 'lambda', threshold) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softshrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softshrink' + ) helper = LayerHelper('softshrink', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='softshrink', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'lambda': threshold}) + helper.append_op( + type='softshrink', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'lambda': threshold}, + ) return out @@ -1293,8 +1347,9 @@ def softsign(x, name=None): if in_dynamic_mode(): return _legacy_C_ops.softsign(x) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softsign') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softsign' + ) helper = LayerHelper('softsign', **locals()) out = helper.create_variable_for_type_inference(x.dtype) helper.append_op(type='softsign', inputs={'X': x}, outputs={'Out': out}) @@ -1334,10 +1389,9 @@ def swish(x, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'swish') helper = LayerHelper('swish', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='swish', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'beta': 1.0}) + helper.append_op( + type='swish', inputs={'X': x}, outputs={'Out': out}, attrs={'beta': 1.0} + ) return out @@ -1413,8 +1467,9 @@ def tanhshrink(x, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.tanh_shrink(x) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'tanhshrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'tanhshrink' + ) helper = LayerHelper('tanh_shrink', **locals()) out = helper.create_variable_for_type_inference(x.dtype) helper.append_op(type='tanh_shrink', inputs={'X': x}, outputs={'Out': out}) @@ -1461,14 +1516,17 @@ def thresholded_relu(x, threshold=1.0, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.thresholded_relu(x, 'threshold', threshold) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'thresholded_relu') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'thresholded_relu' + ) helper = LayerHelper('thresholded_relu', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='thresholded_relu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='thresholded_relu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out @@ -1541,30 +1599,36 @@ def log_softmax(x, axis=-1, dtype=None, name=None): return _legacy_C_ops.log_softmax(x, 'axis', axis) if dtype is None: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'log_softmax') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'log_softmax' + ) else: check_dtype( - dtype, 'dtype', ['float32', 'float64'], 'log_softmax', - 'If dtype is not None, it only support float32 or float64.') + dtype, + 'dtype', + ['float32', 'float64'], + 'log_softmax', + 'If dtype is not None, it only support float32 or float64.', + ) helper = LayerHelper("log_softmax", **locals()) out_cast = x if dtype is not None: out_cast = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='cast', - inputs={'X': x}, - outputs={'Out': out_cast}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': dtype - }) + helper.append_op( + type='cast', + inputs={'X': x}, + outputs={'Out': out_cast}, + attrs={'in_dtype': x.dtype, 'out_dtype': dtype}, + ) out = helper.create_variable_for_type_inference(out_cast.dtype) - helper.append_op(type='log_softmax', - inputs={'X': out_cast}, - outputs={'Out': out}, - attrs={'axis': axis}) + helper.append_op( + type='log_softmax', + inputs={'X': out_cast}, + outputs={'Out': out}, + attrs={'axis': axis}, + ) return out @@ -1607,8 +1671,9 @@ def glu(x, axis=-1, name=None): # [-1.0577879 , -0.46985325]], dtype=float32) """ - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - "glu") + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], "glu" + ) a, b = chunk(x, 2, axis=axis, name=name) gate = sigmoid(b, name=name) out = paddle.multiply(a, gate, name=name) @@ -1676,18 +1741,17 @@ def gumbel_softmax(x, temperature=1.0, hard=False, axis=-1, name=None): return _C_ops.gumbel_softmax(x, temperature, hard, axis) if in_dynamic_mode(): - return _legacy_C_ops.gumbel_softmax(x, 'temperature', temperature, - 'hard', hard, 'axis', axis) + return _legacy_C_ops.gumbel_softmax( + x, 'temperature', temperature, 'hard', hard, 'axis', axis + ) helper = LayerHelper("gumbel_softmax", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'gumbel_softmax') out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='gumbel_softmax', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'temperature': temperature, - 'hard': hard, - 'axis': axis - }) + helper.append_op( + type='gumbel_softmax', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'temperature': temperature, 'hard': hard, 'axis': axis}, + ) return out diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py index 2dc4db30265..b6936c5a90c 100644 --- a/python/paddle/nn/functional/common.py +++ b/python/paddle/nn/functional/common.py @@ -18,14 +18,23 @@ from paddle.fluid.layers.tensor import fill_constant from ...tensor import concat from ...tensor.creation import zeros from paddle.static import Variable + # TODO: define the common functions to build a neural network from ...tensor.manipulation import squeeze from ...tensor.manipulation import unsqueeze from ...tensor import clip from ...tensor import sum from ...tensor import sqrt -from ...fluid.data_feeder import check_variable_and_dtype, check_dtype, check_type -from ...fluid.framework import _in_legacy_dygraph, _non_static_mode, in_dygraph_mode +from ...fluid.data_feeder import ( + check_variable_and_dtype, + check_dtype, + check_type, +) +from ...fluid.framework import ( + _in_legacy_dygraph, + _non_static_mode, + in_dygraph_mode, +) from paddle import _C_ops, _legacy_C_ops from paddle.framework import in_dynamic_mode @@ -108,26 +117,28 @@ def unfold(x, kernel_sizes, strides=1, paddings=0, dilations=1, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'unfold') - assert len(x.shape) == 4, \ - "input should be the format of [N, C, H, W]" + assert len(x.shape) == 4, "input should be the format of [N, C, H, W]" if isinstance(kernel_sizes, int): kernel_sizes = [kernel_sizes, kernel_sizes] else: - assert isinstance(kernel_sizes, list) and (len(kernel_sizes) == 2), \ - "kernel_sizes should either be an integer or a list of two integers" + assert isinstance(kernel_sizes, list) and ( + len(kernel_sizes) == 2 + ), "kernel_sizes should either be an integer or a list of two integers" if isinstance(strides, int): strides = [strides, strides] else: - assert isinstance(strides, list) and (len(strides) == 2), \ - "strides should either be an integer or a list of two integers" + assert isinstance(strides, list) and ( + len(strides) == 2 + ), "strides should either be an integer or a list of two integers" if isinstance(dilations, int): dilations = [dilations, dilations] else: - assert isinstance(dilations, list) and (len(dilations) == 2), \ - "dilations should either be an integer or a list of two integers" + assert isinstance(dilations, list) and ( + len(dilations) == 2 + ), "dilations should either be an integer or a list of two integers" if isinstance(paddings, int): paddings = [paddings] * 4 @@ -143,32 +154,37 @@ def unfold(x, kernel_sizes, strides=1, paddings=0, dilations=1, name=None): else: raise ValueError( "Unexpected type of paddings, it should be either an integer or a list" - "of 2 or 4 integers") + "of 2 or 4 integers" + ) if in_dygraph_mode(): return _C_ops.unfold(x, kernel_sizes, strides, paddings, dilations) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="unfold", - inputs={"X": x}, - outputs={"Y": out}, - attrs={ - "kernel_sizes": kernel_sizes, - "strides": strides, - "paddings": paddings, - "dilations": dilations - }) + helper.append_op( + type="unfold", + inputs={"X": x}, + outputs={"Y": out}, + attrs={ + "kernel_sizes": kernel_sizes, + "strides": strides, + "paddings": paddings, + "dilations": dilations, + }, + ) return out -def interpolate(x, - size=None, - scale_factor=None, - mode='nearest', - align_corners=False, - align_mode=0, - data_format='NCHW', - name=None): +def interpolate( + x, + size=None, + scale_factor=None, + mode='nearest', + align_corners=False, + align_mode=0, + data_format='NCHW', + name=None, +): """ This API resizes a batch of images. @@ -376,7 +392,8 @@ def interpolate(x, if resample not in resample_methods: raise ValueError( "The 'resample' of image_resize can only be 'area', 'linear', 'bilinear', 'trilinear', " - " 'bicubic' or 'nearest' currently.") + " 'bicubic' or 'nearest' currently." + ) if resample in ['LINEAR'] and len(x.shape) != 3: raise ValueError("'linear' only support 3-D tensor.") @@ -403,8 +420,11 @@ def interpolate(x, ) if resample == 'AREA': - if isinstance(size, list) or isinstance(size, tuple) or isinstance( - size, Variable): + if ( + isinstance(size, list) + or isinstance(size, tuple) + or isinstance(size, Variable) + ): if len(size) == 0: raise ValueError("output size can not be empty") if len(x.shape) == 3: @@ -418,19 +438,25 @@ def interpolate(x, dtype = helper.input_dtype(input_param_name='x') if len(x.shape) == 3 and data_format not in ['NCW', 'NWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCW` or `NWC` supported for 3-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCW` or `NWC` supported for 3-D input." + ) elif len(x.shape) == 4 and data_format not in ['NCHW', 'NHWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCHW` or `NHWC` supported for 4-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCHW` or `NHWC` supported for 4-D input." + ) elif len(x.shape) == 5 and data_format not in ['NCDHW', 'NDHWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCDHW` or `NDHWC` supported for 5-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCDHW` or `NDHWC` supported for 5-D input." + ) def _is_list_or_turple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if data_format == 'NCHW' or data_format == 'NCDHW' or data_format == 'NCW': data_layout = 'NCHW' @@ -448,7 +474,7 @@ def interpolate(x, "interp_method": resample_type, "align_corners": align_corners, "align_mode": align_mode, - "data_layout": data_layout + "data_layout": data_layout, } out_shape = size @@ -476,9 +502,9 @@ def interpolate(x, if isinstance(dim_size, Variable): contain_var = True continue - assert dim_size > 0, ( - "Each dimension size given in out_shape must be greater than 0." - ) + assert ( + dim_size > 0 + ), "Each dimension size given in out_shape must be greater than 0." if contain_var: new_size_tensor = [] @@ -489,14 +515,13 @@ def interpolate(x, new_size_tensor.append(dim) size_list.append(-1) else: - assert (isinstance(dim, int)) + assert isinstance(dim, int) temp_out = helper.create_variable_for_type_inference( - 'int32') - fill_constant([1], - 'int32', - dim, - force_cpu=True, - out=temp_out) + 'int32' + ) + fill_constant( + [1], 'int32', dim, force_cpu=True, out=temp_out + ) new_size_tensor.append(temp_out) size_list.append(dim) inputs['SizeTensor'] = new_size_tensor @@ -504,7 +529,8 @@ def interpolate(x, if len(x.shape) == 3: if len(out_shape) != 1: raise ValueError( - "size length should be 2 for input 3-D tensor") + "size length should be 2 for input 3-D tensor" + ) if contain_var: attrs['out_w'] = size_list[0] else: @@ -512,8 +538,9 @@ def interpolate(x, attrs['out_w'] = out_shape[0] if len(x.shape) == 4: if len(out_shape) != 2: - raise ValueError("size length should be 2 for " - "input 4-D tensor.") + raise ValueError( + "size length should be 2 for " "input 4-D tensor." + ) if contain_var: attrs['out_h'] = size_list[0] attrs['out_w'] = size_list[1] @@ -523,8 +550,9 @@ def interpolate(x, attrs['out_w'] = out_shape[1] if len(x.shape) == 5: if len(out_shape) != 3: - raise ValueError("size length should be 3 for " - "input 5-D tensor.") + raise ValueError( + "size length should be 3 for " "input 5-D tensor." + ) if contain_var: attrs['out_d'] = size_list[0] attrs['out_h'] = size_list[1] @@ -550,9 +578,10 @@ def interpolate(x, attrs['scale'] = list(map(float, scale_list)) elif isinstance(scale, list) or isinstance(scale, tuple): if len(scale) != len(x.shape) - 2: - raise ValueError("scale_shape length should be {} for " - "input {}-D tensor.".format( - len(x.shape) - 2, len(x.shape))) + raise ValueError( + "scale_shape length should be {} for " + "input {}-D tensor.".format(len(x.shape) - 2, len(x.shape)) + ) for value in scale: if value <= 0: raise ValueError("Attr(scale) should be greater than zero.") @@ -572,80 +601,114 @@ def interpolate(x, if resample_type == "linear": if in_dygraph_mode(): out = _C_ops.linear_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.linear_interp_v2(x, *dy_attr) elif resample_type == "bilinear": if in_dygraph_mode(): out = _C_ops.bilinear_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.bilinear_interp_v2(x, *dy_attr) elif resample_type == "trilinear": if in_dygraph_mode(): out = _C_ops.trilinear_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.trilinear_interp_v2(x, *dy_attr) elif resample_type == "nearest": if in_dygraph_mode(): out = _C_ops.nearest_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.nearest_interp_v2(x, *dy_attr) elif resample_type == "bicubic": if in_dygraph_mode(): out = _C_ops.bicubic_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.bicubic_interp_v2(x, *dy_attr) return out out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='{}_interp_v2'.format(resample_type), - inputs=inputs, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type='{}_interp_v2'.format(resample_type), + inputs=inputs, + outputs={"Out": out}, + attrs=attrs, + ) return out -def upsample(x, - size=None, - scale_factor=None, - mode='nearest', - align_corners=False, - align_mode=0, - data_format='NCHW', - name=None): +def upsample( + x, + size=None, + scale_factor=None, + mode='nearest', + align_corners=False, + align_mode=0, + data_format='NCHW', + name=None, +): """ This API resizes a batch of images. @@ -827,8 +890,9 @@ def upsample(x, # [2L, 3L, 12L, 12L] """ - return interpolate(x, size, scale_factor, mode, align_corners, align_mode, - data_format) + return interpolate( + x, size, scale_factor, mode, align_corners, align_mode, data_format + ) def bilinear(x1, x2, weight, bias=None, name=None): @@ -879,19 +943,16 @@ def bilinear(x1, x2, weight, bias=None, name=None): helper = LayerHelper("bilinear", **locals()) out = helper.create_variable_for_type_inference(dtype=x1.dtype) - helper.append_op(type="bilinear_tensor_product", - inputs=inputs, - outputs={"Out": out}) + helper.append_op( + type="bilinear_tensor_product", inputs=inputs, outputs={"Out": out} + ) return out -def dropout(x, - p=0.5, - axis=None, - training=True, - mode="upscale_in_train", - name=None): +def dropout( + x, p=0.5, axis=None, training=True, mode="upscale_in_train", name=None +): """ Dropout is a regularization technique for reducing overfitting by preventing neuron co-adaption during training. The dropout operator randomly sets the @@ -1043,7 +1104,8 @@ def dropout(x, if isinstance(p, (int, float)): # fast return for p == 0 - if p == 0: return x + if p == 0: + return x elif p < 0 or p > 1: raise ValueError("p argument should between 0 and 1") if mode not in ('downscale_in_infer', 'upscale_in_train'): @@ -1055,41 +1117,63 @@ def dropout(x, if axis == None: # commonly used dropout seed = None - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed if in_dygraph_mode(): - out, mask = _C_ops.dropout( x, None, p, not training, mode, \ - seed if seed is not None else 0, seed is not None) + out, mask = _C_ops.dropout( + x, + None, + p, + not training, + mode, + seed if seed is not None else 0, + seed is not None, + ) return out - out, mask = _legacy_C_ops.dropout(x, 'dropout_prob', p, 'is_test', - not training, 'fix_seed', seed - is not None, 'seed', - seed if seed is not None else 0, - 'dropout_implementation', mode) + out, mask = _legacy_C_ops.dropout( + x, + 'dropout_prob', + p, + 'is_test', + not training, + 'fix_seed', + seed is not None, + 'seed', + seed if seed is not None else 0, + 'dropout_implementation', + mode, + ) return out helper = LayerHelper('dropout', **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'dropout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'dropout' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) mask = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) def get_attrs(prog, dropout_prob, is_test, seed): if (seed is None or seed == 0) and prog.random_seed != 0: seed = prog.random_seed - if isinstance(dropout_prob, - Variable) and not dropout_prob.shape != [1]: + if isinstance( + dropout_prob, Variable + ) and not dropout_prob.shape != [1]: raise TypeError( - "Required p.shape == [1] if type(p) is Variable, but received p.shape = {}" - .format(p.shape)) + "Required p.shape == [1] if type(p) is Variable, but received p.shape = {}".format( + p.shape + ) + ) attrs = { 'dropout_prob': dropout_prob, 'is_test': is_test, @@ -1101,38 +1185,45 @@ def dropout(x, attrs = get_attrs(helper.main_program, p, not training, seed) - helper.append_op(type='dropout', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs=attrs) + helper.append_op( + type='dropout', + inputs={'X': [x]}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs=attrs, + ) return out - else: #sometimes called dropout_nd #TODO: optimize with c++ + else: # sometimes called dropout_nd #TODO: optimize with c++ if not in_dynamic_mode(): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'dropout') dtype = x.dtype keep_prob = 1 - p if training: - if in_dynamic_mode() and p == 1.: - return paddle.scale(x, scale=0.) + if in_dynamic_mode() and p == 1.0: + return paddle.scale(x, scale=0.0) - scale_input = paddle.scale( - x, scale=1 / keep_prob) if mode == 'upscale_in_train' else x + scale_input = ( + paddle.scale(x, scale=1 / keep_prob) + if mode == 'upscale_in_train' + else x + ) - #get mask shape + # get mask shape input_shape = x.shape if not in_dynamic_mode(): input_shape_tensor = paddle.shape(x) drop_axes = [axis] if isinstance(axis, int) else list(axis) if min(drop_axes) < 0 or max(drop_axes) > len(input_shape) - 1: - raise ValueError("axis value should be greater than or equal to 0 and less than dimensions of x:{}, but get axis value:{} " \ - .format(len(input_shape), max(drop_axes))) + raise ValueError( + "axis value should be greater than or equal to 0 and less than dimensions of x:{}, but get axis value:{} ".format( + len(input_shape), max(drop_axes) + ) + ) if len(drop_axes) > len(input_shape): raise ValueError( - "length of axis should not be greater than dimensions of x:{}, but get length of axis: {}" - .format(len(input_shape), len(drop_axes))) + "length of axis should not be greater than dimensions of x:{}, but get length of axis: {}".format( + len(input_shape), len(drop_axes) + ) + ) mask_shape = [1] * len(input_shape) if not in_dynamic_mode(): for i in drop_axes: @@ -1141,11 +1232,10 @@ def dropout(x, for i in drop_axes: mask_shape[i] = input_shape[i] - #get mask - random_tensor = paddle.uniform(mask_shape, - dtype='float32', - min=0., - max=1.0) + # get mask + random_tensor = paddle.uniform( + mask_shape, dtype='float32', min=0.0, max=1.0 + ) p = full(shape=[1], fill_value=p, dtype='float32') keep_mask = paddle.greater_equal(random_tensor, p) @@ -1154,8 +1244,11 @@ def dropout(x, ret = paddle.multiply(scale_input, keep_mask, name=name) return ret else: # test - ret = paddle.scale( - x, scale=keep_prob) if mode == 'downscale_in_infer' else x + ret = ( + paddle.scale(x, scale=keep_prob) + if mode == 'downscale_in_infer' + else x + ) return ret @@ -1196,20 +1289,26 @@ def dropout2d(x, p=0.5, training=True, data_format='NCHW', name=None): """ input_shape = x.shape if len(input_shape) != 4: - raise ValueError("dimensions of x should be 4, but received {} != 4"\ - .format(len(input_shape))) + raise ValueError( + "dimensions of x should be 4, but received {} != 4".format( + len(input_shape) + ) + ) if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) - return dropout(x, - p=p, - axis=[0, 1] if data_format == 'NCHW' else [0, 3], - training=training, - mode="upscale_in_train", - name=name) + return dropout( + x, + p=p, + axis=[0, 1] if data_format == 'NCHW' else [0, 3], + training=training, + mode="upscale_in_train", + name=name, + ) def dropout3d(x, p=0.5, training=True, data_format='NCDHW', name=None): @@ -1248,20 +1347,26 @@ def dropout3d(x, p=0.5, training=True, data_format='NCDHW', name=None): input_shape = x.shape if len(input_shape) != 5: - raise ValueError("dimensions of x should be 5, but received {} != 5" \ - .format(len(input_shape))) + raise ValueError( + "dimensions of x should be 5, but received {} != 5".format( + len(input_shape) + ) + ) if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) - return dropout(x, - p=p, - axis=[0, 1] if data_format == 'NCDHW' else [0, 4], - training=training, - mode="upscale_in_train", - name=name) + return dropout( + x, + p=p, + axis=[0, 1] if data_format == 'NCDHW' else [0, 4], + training=training, + mode="upscale_in_train", + name=name, + ) def alpha_dropout(x, p=0.5, training=True, name=None): @@ -1303,37 +1408,40 @@ def alpha_dropout(x, p=0.5, training=True, name=None): raise ValueError("p argument should between 0 and 1") if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'alpha_dropout') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'alpha_dropout' + ) if training: if p == 1: - return paddle.scale(x, scale=0.) - #get transformation params + return paddle.scale(x, scale=0.0) + # get transformation params alpha = 1.6732632423543772848170429916717 scale = 1.0507009873554804934193349852946 alpha_p = -alpha * scale - a = ((1 - p) * (1 + p * alpha_p**2))**-0.5 + a = ((1 - p) * (1 + p * alpha_p**2)) ** -0.5 b = -a * alpha_p * p dtype = x.dtype input_shape = x.shape - #get mask - random_tensor = paddle.uniform(input_shape, - dtype='float32', - min=0., - max=1.0) + # get mask + random_tensor = paddle.uniform( + input_shape, dtype='float32', min=0.0, max=1.0 + ) p = full(shape=[1], fill_value=p, dtype='float32') keep_mask = paddle.greater_equal(random_tensor, p) keep_mask = paddle.cast(keep_mask, dtype) drop_mask = paddle.subtract( - full(shape=input_shape, fill_value=1., dtype=dtype), keep_mask) + full(shape=input_shape, fill_value=1.0, dtype=dtype), keep_mask + ) - #apply mask + # apply mask b = full(shape=[1], fill_value=b, dtype=dtype) - y = paddle.add(paddle.multiply(x, keep_mask), - paddle.scale(drop_mask, scale=alpha_p)) + y = paddle.add( + paddle.multiply(x, keep_mask), + paddle.scale(drop_mask, scale=alpha_p), + ) res = paddle.add(paddle.scale(y, scale=a), b, name=name) return res else: # test @@ -1452,18 +1560,28 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): # [6. 4. 5. 6. 4. 5.] # [3. 1. 2. 3. 1. 2.]]]] """ - assert mode in ['reflect', 'replicate', 'constant', 'circular'], \ - "mode should be one of constant, reflect, replicate, circular, but got {}.".format(mode) + assert mode in [ + 'reflect', + 'replicate', + 'constant', + 'circular', + ], "mode should be one of constant, reflect, replicate, circular, but got {}.".format( + mode + ) data_format = data_format.upper() - assert data_format in ["NCL", "NCHW", "NCDHW", "NLC", "NHWC", "NDHWC"], \ - "data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " \ + assert data_format in ["NCL", "NCHW", "NCDHW", "NLC", "NHWC", "NDHWC"], ( + "data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " "but got {}".format(data_format) + ) x_dim = len(x.shape) - if mode == "constant" and isinstance( - pad, (list, tuple)) and len(pad) == x_dim * 2: + if ( + mode == "constant" + and isinstance(pad, (list, tuple)) + and len(pad) == x_dim * 2 + ): paddings = pad pad_value = value @@ -1471,10 +1589,20 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): out = _C_ops.pad(x, paddings, float(pad_value)) return out - check_variable_and_dtype(x, 'x', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], "pad") + check_variable_and_dtype( + x, + 'x', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + "pad", + ) check_type(pad_value, 'pad_value', (float, int, Variable), 'pad') if isinstance(pad_value, int): @@ -1483,17 +1611,18 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): helper = LayerHelper('pad', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'paddings': paddings, - 'pad_value': pad_value - }) + helper.append_op( + type='pad', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'paddings': paddings, 'pad_value': pad_value}, + ) return out assert x_dim in [ - 3, 4, 5 + 3, + 4, + 5, ], "input tesor dimension must be in [3, 4, 5] but got {}".format(x_dim) supported_format_map = { @@ -1501,9 +1630,11 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): 4: ["NCHW", "NHWC"], 5: ["NCDHW", "NDHWC"], } - assert data_format in supported_format_map[x_dim], \ - "input tensor dimension is {}, it's data format should be in {} but got {}".format( - x_dim, supported_format_map[x_dim], data_format) + assert ( + data_format in supported_format_map[x_dim] + ), "input tensor dimension is {}, it's data format should be in {} but got {}".format( + x_dim, supported_format_map[x_dim], data_format + ) unsqueezed_dim = [] @@ -1511,21 +1642,21 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): if data_format in ["NCL", "NCHW", "NCDHW"]: data_format = "NCDHW" if x_dim == 3: - pad = concat([zeros((4, ), dtype="int32"), pad], axis=0) + pad = concat([zeros((4,), dtype="int32"), pad], axis=0) unsqueezed_dim = [3, 4] x = unsqueeze(x, axis=unsqueezed_dim) elif x_dim == 4: - pad = concat([pad, zeros((2, ), dtype="int32")], axis=0) + pad = concat([pad, zeros((2,), dtype="int32")], axis=0) unsqueezed_dim = [2] x = unsqueeze(x, axis=unsqueezed_dim) elif data_format in ["NLC", "NHWC", "NDHWC"]: data_format = "NDHWC" if x_dim == 3: - pad = concat([zeros((4, ), dtype="int32"), pad], axis=0) + pad = concat([zeros((4,), dtype="int32"), pad], axis=0) unsqueezed_dim = [2, 3] x = unsqueeze(x, axis=unsqueezed_dim) elif x_dim == 4: - pad = concat([pad, zeros((2, ), dtype="int32")], axis=0) + pad = concat([pad, zeros((2,), dtype="int32")], axis=0) unsqueezed_dim = [1] x = unsqueeze(x, axis=unsqueezed_dim) else: @@ -1559,9 +1690,19 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): if _in_legacy_dygraph(): if isinstance(pad, Variable): pad = pad.numpy().tolist() - out = _legacy_C_ops.pad3d(x, "paddings", pad, "mode", mode, "value", - value, "data_format", data_format, "name", - name) + out = _legacy_C_ops.pad3d( + x, + "paddings", + pad, + "mode", + mode, + "value", + value, + "data_format", + data_format, + "name", + name, + ) else: attrs = {'mode': mode, 'value': value, 'data_format': data_format} inputs = {'X': [x]} @@ -1575,10 +1716,9 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): dtype = helper.input_dtype(input_param_name='input') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad3d', - inputs=inputs, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type='pad3d', inputs=inputs, outputs={"Out": out}, attrs=attrs + ) if len(unsqueezed_dim) != 0: out = squeeze(out, axis=unsqueezed_dim) @@ -1619,12 +1759,14 @@ def zeropad2d(x, padding, data_format="NCHW", name=None): # [0. 0. 0. 0. 0. 0.]]]] """ - return pad(x, - pad=padding, - mode='constant', - value=0, - data_format=data_format, - name=name) + return pad( + x, + pad=padding, + mode='constant', + value=0, + data_format=data_format, + name=name, + ) def cosine_similarity(x1, x2, axis=1, eps=1e-8): @@ -1731,12 +1873,13 @@ def linear(x, weight, bias=None, name=None): # [2.1077576 2.1077576 2.1077576 2.1077576 ]] """ if in_dygraph_mode(): - #TODO(jiabin): using addmm for fast forward route + # TODO(jiabin): using addmm for fast forward route return _C_ops.linear(x, weight, bias) else: if _in_legacy_dygraph(): - pre_bias = _legacy_C_ops.matmul_v2(x, weight, 'trans_x', False, - 'trans_y', False) + pre_bias = _legacy_C_ops.matmul_v2( + x, weight, 'trans_x', False, 'trans_y', False + ) if bias is None: return pre_bias @@ -1746,27 +1889,30 @@ def linear(x, weight, bias=None, name=None): helper = LayerHelper('linear', **locals()) dtype = x.dtype - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'linear') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear' + ) inputs = {'X': [x], 'Y': [weight]} attrs = {'trans_x': False, 'trans_y': False} tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='matmul_v2', - inputs=inputs, - outputs={'Out': tmp}, - attrs=attrs) + helper.append_op( + type='matmul_v2', + inputs=inputs, + outputs={'Out': tmp}, + attrs=attrs, + ) if bias is not None: res = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_add', - inputs={ - 'X': [tmp], - 'Y': [bias] - }, - outputs={'Out': [res]}, - attrs={'axis': len(x.shape) - 1}) + helper.append_op( + type='elementwise_add', + inputs={'X': [tmp], 'Y': [bias]}, + outputs={'Out': [res]}, + attrs={'axis': len(x.shape) - 1}, + ) else: res = tmp return res @@ -1830,29 +1976,32 @@ def label_smooth(label, prior_dist=None, epsilon=0.1, name=None): #[[[0.03333334 0.93333334 0.03333334] # [0.93333334 0.03333334 0.93333334]]] """ - if epsilon > 1. or epsilon < 0.: + if epsilon > 1.0 or epsilon < 0.0: raise ValueError("The value of epsilon must be between 0 and 1.") if in_dygraph_mode(): return _C_ops.label_smooth(label, prior_dist, float(epsilon)) elif paddle.in_dynamic_mode(): - return _legacy_C_ops.label_smooth(label, prior_dist, 'epsilon', - float(epsilon)) + return _legacy_C_ops.label_smooth( + label, prior_dist, 'epsilon', float(epsilon) + ) - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'label_smooth') + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'label_smooth' + ) helper = LayerHelper("label_smooth", **locals()) label.stop_gradient = True smooth_label = helper.create_variable_for_type_inference(label.dtype) - helper.append_op(type="label_smooth", - inputs={ - "X": label, - "PriorDist": prior_dist - } if prior_dist else {"X": label}, - outputs={"Out": smooth_label}, - attrs={"epsilon": float(epsilon)}) + helper.append_op( + type="label_smooth", + inputs={"X": label, "PriorDist": prior_dist} + if prior_dist + else {"X": label}, + outputs={"Out": smooth_label}, + attrs={"epsilon": float(epsilon)}, + ) return smooth_label @@ -1963,7 +2112,10 @@ def class_center_sample(label, num_classes, num_samples, group=None): if not (group == False or group is None or hasattr(group, 'is_member')): raise ValueError( 'Expected group is False, None or instance of paddle.distributed.collective.Group \ - (got group: {})'.format(group)) + (got group: {})'.format( + group + ) + ) return if hasattr(group, 'is_member') and not group.is_member(): @@ -1976,76 +2128,112 @@ def class_center_sample(label, num_classes, num_samples, group=None): if core.is_compiled_with_dist(): parallel_env = paddle.distributed.ParallelEnv() global_rank = parallel_env.rank - rank = global_rank if group is None else group.get_group_rank( - global_rank) + rank = ( + global_rank + if group is None + else group.get_group_rank(global_rank) + ) nranks = parallel_env.world_size if group is None else group.nranks if num_samples > num_classes: raise ValueError( - 'Expected num_samples less than or equal to {}, got num_samples {}'. - format(num_classes, num_samples)) + 'Expected num_samples less than or equal to {}, got num_samples {}'.format( + num_classes, num_samples + ) + ) label_size = 1 for dim in list(label.shape): label_size *= dim if label_size != -1 and label_size < 1: - raise ValueError('Expected label_size > 0 \ - (got label_size: {})'.format(label_size)) + raise ValueError( + 'Expected label_size > 0 \ + (got label_size: {})'.format( + label_size + ) + ) label_dims = len(list(label.shape)) if label_dims != 1: - raise ValueError('Expected label_dims == 1 \ - (got label_dims: {})'.format(label_dims)) + raise ValueError( + 'Expected label_dims == 1 \ + (got label_dims: {})'.format( + label_dims + ) + ) seed = None if (seed is None or seed == 0) and default_main_program().random_seed != 0: seed = default_main_program().random_seed if in_dygraph_mode(): - return _C_ops.class_center_sample(label, num_classes, num_samples, - ring_id, rank, nranks, seed - is not None, - seed if seed is not None else 0) + return _C_ops.class_center_sample( + label, + num_classes, + num_samples, + ring_id, + rank, + nranks, + seed is not None, + seed if seed is not None else 0, + ) elif paddle.in_dynamic_mode(): - remapped_label, sampled_class_center = _legacy_C_ops.class_center_sample( - label, 'num_classes', num_classes, 'num_samples', num_samples, - 'ring_id', ring_id, 'nranks', nranks, 'rank', rank, 'fix_seed', seed - is not None, 'seed', seed if seed is not None else 0) + ( + remapped_label, + sampled_class_center, + ) = _legacy_C_ops.class_center_sample( + label, + 'num_classes', + num_classes, + 'num_samples', + num_samples, + 'ring_id', + ring_id, + 'nranks', + nranks, + 'rank', + rank, + 'fix_seed', + seed is not None, + 'seed', + seed if seed is not None else 0, + ) return remapped_label, sampled_class_center - check_variable_and_dtype(label, 'label', ['int64', 'int32'], - 'class_center_sample') + check_variable_and_dtype( + label, 'label', ['int64', 'int32'], 'class_center_sample' + ) op_type = 'class_center_sample' helper = LayerHelper(op_type, **locals()) remapped_label = helper.create_variable_for_type_inference( - dtype=label.dtype) + dtype=label.dtype + ) sampled_class_center = helper.create_variable_for_type_inference( - dtype=label.dtype) - helper.append_op(type=op_type, - inputs={'Label': label}, - outputs={ - 'RemappedLabel': remapped_label, - 'SampledLocalClassCenter': sampled_class_center - }, - attrs={ - 'num_classes': num_classes, - 'num_samples': num_samples, - 'ring_id': ring_id, - 'nranks': nranks, - 'rank': rank, - 'fix_seed': seed is not None, - 'seed': seed if seed is not None else 0 - }) + dtype=label.dtype + ) + helper.append_op( + type=op_type, + inputs={'Label': label}, + outputs={ + 'RemappedLabel': remapped_label, + 'SampledLocalClassCenter': sampled_class_center, + }, + attrs={ + 'num_classes': num_classes, + 'num_samples': num_samples, + 'ring_id': ring_id, + 'nranks': nranks, + 'rank': rank, + 'fix_seed': seed is not None, + 'seed': seed if seed is not None else 0, + }, + ) return remapped_label, sampled_class_center -def fold(x, - output_sizes, - kernel_sizes, - strides=1, - paddings=0, - dilations=1, - name=None): +def fold( + x, output_sizes, kernel_sizes, strides=1, paddings=0, dilations=1, name=None +): r""" Combines an array of sliding local blocks into a large containing @@ -2108,35 +2296,38 @@ def fold(x, check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'fold') - assert len(x.shape) == 3, \ - "input should be the format of [N, C, L]" + assert len(x.shape) == 3, "input should be the format of [N, C, L]" def _is_list_or_turple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if isinstance(output_sizes, int): output_sizes = [output_sizes, output_sizes] else: - assert _is_list_or_turple_(output_sizes) and (len(output_sizes) == 2), \ - "output_sizes should either be an integer or a list/tuple of two integers" + assert _is_list_or_turple_(output_sizes) and ( + len(output_sizes) == 2 + ), "output_sizes should either be an integer or a list/tuple of two integers" if isinstance(kernel_sizes, int): kernel_sizes = [kernel_sizes, kernel_sizes] else: - assert _is_list_or_turple_(kernel_sizes) and (len(kernel_sizes) == 2), \ - "kernel_sizes should either be an integer or a list/tuple of two integers" + assert _is_list_or_turple_(kernel_sizes) and ( + len(kernel_sizes) == 2 + ), "kernel_sizes should either be an integer or a list/tuple of two integers" if isinstance(strides, int): strides = [strides, strides] else: - assert _is_list_or_turple_(strides) and (len(strides) == 2), \ - "strides should either be an integer or a list/tuple of two integers" + assert _is_list_or_turple_(strides) and ( + len(strides) == 2 + ), "strides should either be an integer or a list/tuple of two integers" if isinstance(dilations, int): dilations = [dilations, dilations] else: - assert _is_list_or_turple_(dilations) and (len(dilations) == 2), \ - "dilations should either be an integer or a list/tuple of two integers" + assert _is_list_or_turple_(dilations) and ( + len(dilations) == 2 + ), "dilations should either be an integer or a list/tuple of two integers" if isinstance(paddings, int): paddings = [paddings] * 4 @@ -2152,26 +2343,39 @@ def fold(x, else: raise ValueError( "Unexpected type of paddings, it should be either an integer or a list" - "of 2 or 4 integers") + "of 2 or 4 integers" + ) if in_dygraph_mode(): - out = _C_ops.fold(x, output_sizes, kernel_sizes, strides, paddings, - dilations) + out = _C_ops.fold( + x, output_sizes, kernel_sizes, strides, paddings, dilations + ) elif in_dynamic_mode(): - out = _legacy_C_ops.fold(x, "output_sizes", output_sizes, - "kernel_sizes", kernel_sizes, "strides", - strides, "paddings", paddings, "dilations", - dilations) + out = _legacy_C_ops.fold( + x, + "output_sizes", + output_sizes, + "kernel_sizes", + kernel_sizes, + "strides", + strides, + "paddings", + paddings, + "dilations", + dilations, + ) else: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="fold", - inputs={"X": x}, - outputs={"Y": out}, - attrs={ - "output_sizes": output_sizes, - "kernel_sizes": kernel_sizes, - "strides": strides, - "paddings": paddings, - "dilations": dilations - }) + helper.append_op( + type="fold", + inputs={"X": x}, + outputs={"Y": out}, + attrs={ + "output_sizes": output_sizes, + "kernel_sizes": kernel_sizes, + "strides": strides, + "paddings": paddings, + "dilations": dilations, + }, + ) return out diff --git a/python/paddle/nn/functional/conv.py b/python/paddle/nn/functional/conv.py index 17eded8ac06..57359421617 100644 --- a/python/paddle/nn/functional/conv.py +++ b/python/paddle/nn/functional/conv.py @@ -14,7 +14,12 @@ from ...device import get_cudnn_version from ...static import Variable -from ...fluid.layers.utils import convert_to_list, _is_symmetric_padding, _contain_var, _convert_to_tensor_list +from ...fluid.layers.utils import ( + convert_to_list, + _is_symmetric_padding, + _contain_var, + _convert_to_tensor_list, +) from ...fluid.data_feeder import check_variable_and_dtype, check_dtype from ...fluid.layer_helper import LayerHelper from ...tensor.manipulation import unsqueeze, squeeze @@ -56,8 +61,10 @@ def _update_padding_nd(padding, channel_last, num_dims): padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '{}'. It can only be 'SAME' or 'VALID'.". - format(padding)) + "Unknown padding: '{}'. It can only be 'SAME' or 'VALID'.".format( + padding + ) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0] * num_dims @@ -72,10 +79,12 @@ def _update_padding_nd(padding, channel_last, num_dims): if not _zero_padding_in_batch_and_channel(padding, channel_last): raise ValueError( "Non-zero padding({}) in the batch or channel dimensions " - "is not supported.".format(padding)) + "is not supported.".format(padding) + ) padding_algorithm = "EXPLICIT" padding = _exclude_padding_in_batch_and_channel( - padding, channel_last) + padding, channel_last + ) if _is_symmetric_padding(padding, num_dims): padding = padding[0::2] # for padding like [pad_before, pad_after, pad_before, pad_after, ...] @@ -96,42 +105,60 @@ def _update_padding_nd(padding, channel_last, num_dims): padding = convert_to_list(padding, num_dims, 'padding') if not all([p >= 0 for p in padding]): raise ValueError( - "Invalid padding, all value should be larger than or equal to 0, but received: {}" - .format(padding)) + "Invalid padding, all value should be larger than or equal to 0, but received: {}".format( + padding + ) + ) return padding, padding_algorithm -def _conv_nd(x, - weight, - bias=None, - stride=1, - padding=0, - padding_algorithm=None, - dilation=1, - groups=1, - data_format="NCHW", - channel_dim=1, - op_type="conv2d", - use_cudnn=True, - use_mkldnn=False, - name=None): +def _conv_nd( + x, + weight, + bias=None, + stride=1, + padding=0, + padding_algorithm=None, + dilation=1, + groups=1, + data_format="NCHW", + channel_dim=1, + op_type="conv2d", + use_cudnn=True, + use_mkldnn=False, + name=None, +): # Due to the poor performance of NHWC, we transpose the input to NCHW. if in_dygraph_mode() and op_type == "conv2d": - pre_bias = _C_ops.conv2d(x, weight, stride, padding, padding_algorithm, - groups, dilation, data_format, False, -1, - False) + pre_bias = _C_ops.conv2d( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + data_format, + False, + -1, + False, + ) if bias is not None: - channel_dim = channel_dim + len( - x.shape) if channel_dim < 0 else channel_dim + channel_dim = ( + channel_dim + len(x.shape) if channel_dim < 0 else channel_dim + ) if isinstance(x, tuple): x = x[0] if isinstance(bias, tuple): bias = bias[0] if len(bias.shape) < len(x.shape): tmp_bias = _C_ops.reshape( - bias, [1 for i in range(channel_dim)] + bias.shape + - [1 for i in range(len(x.shape) - channel_dim - 1)]) + bias, + [1 for i in range(channel_dim)] + + bias.shape + + [1 for i in range(len(x.shape) - channel_dim - 1)], + ) return _C_ops.add(pre_bias, tmp_bias) else: return _C_ops.add(pre_bias, bias) @@ -139,40 +166,82 @@ def _conv_nd(x, return pre_bias if in_dygraph_mode() and op_type == "depthwise_conv2d": - pre_bias = _C_ops.depthwise_conv2d(x, weight, stride, padding, - padding_algorithm, groups, dilation, - data_format, False, -1, False, False, - use_cudnn) + pre_bias = _C_ops.depthwise_conv2d( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + data_format, + False, + -1, + False, + False, + use_cudnn, + ) if bias is not None: - channel_dim = channel_dim + len( - x.shape) if channel_dim < 0 else channel_dim + channel_dim = ( + channel_dim + len(x.shape) if channel_dim < 0 else channel_dim + ) tmp_bias = _C_ops.reshape( - bias, [1 for i in range(channel_dim)] + bias.shape + - [1 for i in range(len(x.shape) - channel_dim - 1)]) + bias, + [1 for i in range(channel_dim)] + + bias.shape + + [1 for i in range(len(x.shape) - channel_dim - 1)], + ) return _C_ops.add(pre_bias, tmp_bias) else: return pre_bias if in_dygraph_mode() and op_type == "conv3d": - pre_bias = _C_ops.conv3d(x, weight, stride, padding, padding_algorithm, - groups, dilation, data_format, False, -1, - False) + pre_bias = _C_ops.conv3d( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + data_format, + False, + -1, + False, + ) if bias is not None: - channel_dim = channel_dim + len( - x.shape) if channel_dim < 0 else channel_dim + channel_dim = ( + channel_dim + len(x.shape) if channel_dim < 0 else channel_dim + ) tmp_bias = _C_ops.reshape( bias, - bias.shape + [1 for i in range(len(x.shape) - channel_dim - 1)]) + bias.shape + [1 for i in range(len(x.shape) - channel_dim - 1)], + ) return _C_ops.add(pre_bias, tmp_bias) else: return pre_bias if in_dynamic_mode(): - attrs = ('strides', stride, 'paddings', padding, 'dilations', dilation, - 'groups', groups, 'use_cudnn', use_cudnn, 'use_mkldnn', - use_mkldnn, 'fuse_relu_before_depthwise_conv', False, - "padding_algorithm", padding_algorithm, "data_format", - data_format) + attrs = ( + 'strides', + stride, + 'paddings', + padding, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + 'use_mkldnn', + use_mkldnn, + 'fuse_relu_before_depthwise_conv', + False, + "padding_algorithm", + padding_algorithm, + "data_format", + data_format, + ) pre_bias = getattr(_legacy_C_ops, op_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) @@ -189,44 +258,42 @@ def _conv_nd(x, 'use_mkldnn': use_mkldnn, 'fuse_relu_before_depthwise_conv': False, "padding_algorithm": padding_algorithm, - "data_format": data_format + "data_format": data_format, } - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - op_type) + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], op_type + ) helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') pre_bias = helper.create_variable_for_type_inference(dtype) outputs = {"Output": [pre_bias]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [bias] - }, - outputs={'Out': [out]}, - attrs={ - 'axis': channel_dim, - 'use_mkldnn': use_mkldnn - }) + helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [bias]}, + outputs={'Out': [out]}, + attrs={'axis': channel_dim, 'use_mkldnn': use_mkldnn}, + ) else: out = pre_bias return out -def conv1d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format='NCL', - name=None): +def conv1d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format='NCL', + name=None, +): r""" The convolution1D layer calculates the output based on the input, filter and strides, paddings, dilations, groups parameters. Input and @@ -339,36 +406,45 @@ def conv1d(x, use_cudnn = False if data_format not in ["NCL", "NLC"]: - raise ValueError("Attr(data_format) should be 'NCL' or 'NLC'. " - "Received Attr(data_format): {}.".format(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCL' or 'NLC'. " + "Received Attr(data_format): {}.".format(data_format) + ) - channel_last = (data_format == "NLC") + channel_last = data_format == "NLC" channel_dim = -1 if channel_last else 1 conv2d_data_format = "NHWC" if channel_last else "NCHW" if len(x.shape) != 3: raise ValueError( - "Input x should be 3D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 3D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] num_filters = weight.shape[0] if num_channels < 0: - raise ValueError("The channel dimension of the input({}) " - "should be defined. Received: {}.".format( - x.shape, num_channels)) + raise ValueError( + "The channel dimension of the input({}) " + "should be defined. Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv1d should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv1d should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, x.shape, groups)) + ", the groups is {}".format(num_channels, x.shape, groups) + ) if num_filters % groups != 0: raise ValueError( "the number of filters must be divisible by groups," "received: the number of filters is {}, the shape of weight is {}" - ", the groups is {}".format(num_filters, weight.shape, groups)) + ", the groups is {}".format(num_filters, weight.shape, groups) + ) # update attrs padding, padding_algorithm = _update_padding_nd(padding, channel_last, 1) @@ -379,8 +455,10 @@ def conv1d(x, padding = [0] + padding else: raise ValueError( - "The size of padding's dimension should be 1 or 2. But got padding={}" - .format(padding)) + "The size of padding's dimension should be 1 or 2. But got padding={}".format( + padding + ) + ) stride = [1] + convert_to_list(stride, 1, 'stride') dilation = [1] + convert_to_list(dilation, 1, 'dilation') weight = unsqueeze(weight, axis=[-2]) @@ -388,14 +466,18 @@ def conv1d(x, l_type = "conv2d" # When "groups==num_channels and num_filters% num_channels == 0" using depthwise_conv2d has better performance - if (is_compiled_with_cuda() and num_channels == groups and num_channels != 1 - and num_filters % num_channels == 0): + if ( + is_compiled_with_cuda() + and num_channels == groups + and num_channels != 1 + and num_filters % num_channels == 0 + ): l_type = 'depthwise_conv2d' use_cudnn = False # NPU only supports depthwise_conv2d when "input_channel = output_channel = groups" if is_compiled_with_npu(): - if (num_channels == groups and num_channels == num_filters): + if num_channels == groups and num_channels == num_filters: l_type = 'depthwise_conv2d' else: l_type = 'conv2d' @@ -404,17 +486,44 @@ def conv1d(x, x = unsqueeze(x, axis=[squeeze_aixs]) if in_dygraph_mode(): - out = getattr(_C_ops, - l_type)(x, weight, stride, padding, padding_algorithm, - groups, dilation, conv2d_data_format, False, -1, - False, False, use_cudnn) + out = getattr(_C_ops, l_type)( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + conv2d_data_format, + False, + -1, + False, + False, + use_cudnn, + ) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) elif _in_legacy_dygraph(): - attrs = ('strides', stride, 'paddings', padding, 'dilations', dilation, - 'groups', groups, 'use_cudnn', use_cudnn, 'use_mkldnn', False, - 'fuse_relu_before_depthwise_conv', False, "padding_algorithm", - padding_algorithm, "data_format", conv2d_data_format) + attrs = ( + 'strides', + stride, + 'paddings', + padding, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + 'use_mkldnn', + False, + 'fuse_relu_before_depthwise_conv', + False, + "padding_algorithm", + padding_algorithm, + "data_format", + conv2d_data_format, + ) out = getattr(_legacy_C_ops, l_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) @@ -429,33 +538,35 @@ def conv1d(x, 'use_mkldnn': False, 'fuse_relu_before_depthwise_conv': False, "padding_algorithm": padding_algorithm, - "data_format": conv2d_data_format + "data_format": conv2d_data_format, } - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'conv2d') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'conv2d' + ) helper = LayerHelper(l_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) outputs = {"Output": [out]} - helper.append_op(type=l_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=l_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) out = squeeze(out, axis=[squeeze_aixs]) return out -def conv2d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NCHW", - name=None): +def conv2d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NCHW", + name=None, +): r""" The convolution2D layer calculates the output based on the input, filter @@ -564,40 +675,52 @@ def conv2d(x, """ # entry checks if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Attr(data_format) should be 'NCHW' or 'NHWC'. " - "Received Attr(data_format): {}.".format(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCHW' or 'NHWC'. " + "Received Attr(data_format): {}.".format(data_format) + ) - channel_last = (data_format == "NHWC") + channel_last = data_format == "NHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 4: raise ValueError( - "Input x should be 4D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 4D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] num_filters = weight.shape[0] if num_channels < 0: - raise ValueError("The channel dimension of the input({}) " - "should be defined. Received: {}.".format( - x.shape, num_channels)) + raise ValueError( + "The channel dimension of the input({}) " + "should be defined. Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv2d should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv2d should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, x.shape, groups)) + ", the groups is {}".format(num_channels, x.shape, groups) + ) if num_filters % groups != 0: raise ValueError( "the number of filters must be divisible by groups," "received: the number of filters is {}, the shape of weight is {}" - ", the groups is {}".format(num_filters, weight.shape, groups)) + ", the groups is {}".format(num_filters, weight.shape, groups) + ) cudnn_version = get_cudnn_version() - use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) # update attrs padding, padding_algorithm = _update_padding_nd(padding, channel_last, 2) @@ -605,8 +728,11 @@ def conv2d(x, dilation = convert_to_list(dilation, 2, 'dilation') l_type = "conv2d" - if (num_channels == groups and num_channels != 1 - and num_filters % num_channels == 0): + if ( + num_channels == groups + and num_channels != 1 + and num_filters % num_channels == 0 + ): l_type = 'depthwise_conv2d' if is_compiled_with_rocm(): use_cudnn = True @@ -614,9 +740,19 @@ def conv2d(x, use_cudnn = False else: if in_dygraph_mode(): - pre_bias = _C_ops.conv2d(x, weight, stride, padding, - padding_algorithm, groups, dilation, - data_format, False, -1, False) + pre_bias = _C_ops.conv2d( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + data_format, + False, + -1, + False, + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) return out @@ -627,31 +763,50 @@ def conv2d(x, # NPU only supports depthwise_conv2d when "input_channel = output_channel = groups" if is_compiled_with_npu(): - if (num_channels == groups and num_channels == num_filters): + if num_channels == groups and num_channels == num_filters: l_type = 'depthwise_conv2d' else: l_type = 'conv2d' - if (is_compiled_with_cuda() and get_flags("FLAGS_conv2d_disable_cudnn") - ["FLAGS_conv2d_disable_cudnn"]): + if ( + is_compiled_with_cuda() + and get_flags("FLAGS_conv2d_disable_cudnn")[ + "FLAGS_conv2d_disable_cudnn" + ] + ): use_cudnn = False - return _conv_nd(x, weight, bias, stride, padding, padding_algorithm, - dilation, groups, data_format, channel_dim, l_type, - use_cudnn, use_mkldnn, name) - - -def conv1d_transpose(x, - weight, - bias=None, - stride=1, - padding=0, - output_padding=0, - groups=1, - dilation=1, - output_size=None, - data_format="NCL", - name=None): + return _conv_nd( + x, + weight, + bias, + stride, + padding, + padding_algorithm, + dilation, + groups, + data_format, + channel_dim, + l_type, + use_cudnn, + use_mkldnn, + name, + ) + + +def conv1d_transpose( + x, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, + output_size=None, + data_format="NCL", + name=None, +): r""" The 1-D convolution transpose layer calculates the output based on the input, filter, and dilation, stride, padding. Input(Input) and output(Output) @@ -780,28 +935,36 @@ def conv1d_transpose(x, raise ValueError( "Attr(data_format) of conv2d_transpose got wrong value: " "received {}, but only 'NCL' or 'NLC' are supported.".format( - data_format)) - channel_last = (data_format == "NLC") + data_format + ) + ) + channel_last = data_format == "NLC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 3: raise ValueError( - "Input x should be 3D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 3D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] if num_channels < 0: - raise ValueError("The channel dimension of the input({}) " - "should be defined. Received: {}.".format( - x.shape, num_channels)) + raise ValueError( + "The channel dimension of the input({}) " + "should be defined. Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv1d_transpose should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv1d_transpose should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, x.shape, groups)) + ", the groups is {}".format(num_channels, x.shape, groups) + ) # update attrs padding, padding_algorithm = _update_padding_nd(padding, channel_last, 1) @@ -812,8 +975,10 @@ def conv1d_transpose(x, padding = padding + [0] else: raise ValueError( - "The size of padding's dimension should 1 or 2. But got padding={}". - format(padding)) + "The size of padding's dimension should 1 or 2. But got padding={}".format( + padding + ) + ) stride = convert_to_list(stride, 1, 'stride') + [1] dilation = convert_to_list(dilation, 1, 'dilation') + [1] @@ -822,30 +987,40 @@ def conv1d_transpose(x, output_size = [] else: if output_padding != 0: - raise ValueError('output_padding option is mutually exclusive with ' - 'output_size') + raise ValueError( + 'output_padding option is mutually exclusive with ' + 'output_size' + ) if isinstance(output_size, (list, tuple, int)): output_size = convert_to_list(output_size, 1, 'output_size') + [1] else: raise ValueError( - "output_size should be int, or list, tuple of ints") + "output_size should be int, or list, tuple of ints" + ) if output_padding == 0: output_padding = [] else: - output_padding = convert_to_list(output_padding, 1, - 'output_padding') + [0] + output_padding = convert_to_list( + output_padding, 1, 'output_padding' + ) + [0] if len(output_padding) > 0 and output_padding[0] > stride[0]: raise ValueError( "The size of output_padding should not be greater than stride." "But got output_padding={} and stride={}".format( - output_padding[0], stride[0])) + output_padding[0], stride[0] + ) + ) op_type = 'conv2d_transpose' num_filters = weight.shape[1] - if (num_channels == groups and num_channels != 1 and num_filters == 1 - and not use_cudnn): + if ( + num_channels == groups + and num_channels != 1 + and num_filters == 1 + and not use_cudnn + ): op_type = 'depthwise_conv2d_transpose' use_cudnn = False @@ -856,17 +1031,41 @@ def conv1d_transpose(x, weight = unsqueeze(weight, axis=[-1]) if in_dygraph_mode(): - out = getattr(_C_ops, - op_type)(x, weight, stride, padding, output_padding, - output_size, padding_algorithm, groups, dilation, - conv2d_data_format) + out = getattr(_C_ops, op_type)( + x, + weight, + stride, + padding, + output_padding, + output_size, + padding_algorithm, + groups, + dilation, + conv2d_data_format, + ) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) elif _in_legacy_dygraph(): - attrs = ('output_padding', output_padding, 'output_size', output_size, - 'strides', stride, 'paddings', padding, 'padding_algorithm', - padding_algorithm, 'dilations', dilation, 'groups', groups, - 'use_cudnn', use_cudnn, 'data_format', conv2d_data_format) + attrs = ( + 'output_padding', + output_padding, + 'output_size', + output_size, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + 'data_format', + conv2d_data_format, + ) out = getattr(_legacy_C_ops, op_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) @@ -881,18 +1080,18 @@ def conv1d_transpose(x, 'dilations': dilation, 'groups': groups, 'use_cudnn': use_cudnn, - 'data_format': conv2d_data_format + 'data_format': conv2d_data_format, } - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'conv2d_transpose') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'conv2d_transpose' + ) helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) outputs = {"Output": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) @@ -900,17 +1099,19 @@ def conv1d_transpose(x, return out -def conv2d_transpose(x, - weight, - bias=None, - stride=1, - padding=0, - output_padding=0, - dilation=1, - groups=1, - output_size=None, - data_format='NCHW', - name=None): +def conv2d_transpose( + x, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + dilation=1, + groups=1, + output_size=None, + data_format='NCHW', + name=None, +): r""" The convolution2D transpose layer calculates the output based on the input, @@ -1039,32 +1240,43 @@ def conv2d_transpose(x, raise ValueError( "Attr(data_format) of conv2d_transpose got wrong value: " "received {}, but only 'NCHW' or 'NHWC' are supported.".format( - data_format)) - channel_last = (data_format == "NHWC") + data_format + ) + ) + channel_last = data_format == "NHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 4: raise ValueError( - "Input x should be 4D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 4D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] if num_channels < 0: - raise ValueError("The channel dimension of the input({}) " - "should be defined. Received: {}.".format( - x.shape, num_channels)) + raise ValueError( + "The channel dimension of the input({}) " + "should be defined. Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv2d_transpose should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv2d_transpose should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, x.shape, groups)) + ", the groups is {}".format(num_channels, x.shape, groups) + ) cudnn_version = get_cudnn_version() - use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) # update attrs padding, padding_algorithm = _update_padding_nd(padding, channel_last, 2) @@ -1075,8 +1287,10 @@ def conv2d_transpose(x, output_size = [] else: if output_padding != 0: - raise ValueError('output_padding option is mutually exclusive with ' - 'output_size') + raise ValueError( + 'output_padding option is mutually exclusive with ' + 'output_size' + ) if isinstance(output_size, (list, tuple)): if _contain_var(output_size): output_size = _convert_to_tensor_list(output_size) @@ -1085,15 +1299,21 @@ def conv2d_transpose(x, elif isinstance(output_size, int): output_size = convert_to_list(output_size, 2, 'output_size') elif isinstance(output_size, Variable): - check_dtype(output_size.dtype, 'output_size', ['int32', 'int64'], - 'conv2d_transpose') - if len(output_size.shape) == 1 and (output_size.shape[0] == 1 - or output_size.shape[0] == 2): + check_dtype( + output_size.dtype, + 'output_size', + ['int32', 'int64'], + 'conv2d_transpose', + ) + if len(output_size.shape) == 1 and ( + output_size.shape[0] == 1 or output_size.shape[0] == 2 + ): if output_size.shape[0] == 1: output_size = [output_size, output_size] else: raise ValueError( - "output_size must contain one or two integers.") + "output_size must contain one or two integers." + ) else: raise ValueError( "output_size should be int or Tensor or list, tuple of ints or Tensor" @@ -1106,24 +1326,54 @@ def conv2d_transpose(x, op_type = 'conv2d_transpose' num_filters = weight.shape[1] - if (num_channels == groups and num_channels != 1 and num_filters == 1): + if num_channels == groups and num_channels != 1 and num_filters == 1: op_type = 'depthwise_conv2d_transpose' use_cudnn = False if in_dygraph_mode(): - op = _C_ops.conv2d_transpose if op_type == 'conv2d_transpose' else _C_ops.depthwise_conv2d_transpose - pre_bias = op(x, weight, stride, padding, output_padding, output_size, - padding_algorithm, groups, dilation, data_format) + op = ( + _C_ops.conv2d_transpose + if op_type == 'conv2d_transpose' + else _C_ops.depthwise_conv2d_transpose + ) + pre_bias = op( + x, + weight, + stride, + padding, + output_padding, + output_size, + padding_algorithm, + groups, + dilation, + data_format, + ) if bias is not None: return nn.elementwise_add(pre_bias, bias, axis=channel_dim) else: return pre_bias if _in_legacy_dygraph(): - attrs = ('output_padding', output_padding, 'output_size', output_size, - 'strides', stride, 'paddings', padding, 'padding_algorithm', - padding_algorithm, 'dilations', dilation, 'groups', groups, - 'use_cudnn', use_cudnn, 'data_format', data_format) + attrs = ( + 'output_padding', + output_padding, + 'output_size', + output_size, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + 'data_format', + data_format, + ) pre_bias = getattr(_legacy_C_ops, op_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) @@ -1140,17 +1390,17 @@ def conv2d_transpose(x, 'dilations': dilation, 'groups': groups, 'use_cudnn': use_cudnn, - 'data_format': data_format + 'data_format': data_format, } - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'conv2d_transpose') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'conv2d_transpose' + ) helper = LayerHelper(op_type, **locals()) pre_bias = helper.create_variable_for_type_inference(x.dtype) outputs = {"Output": [pre_bias]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) @@ -1160,15 +1410,17 @@ def conv2d_transpose(x, return out -def conv3d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NCDHW", - name=None): +def conv3d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NCDHW", + name=None, +): r""" The convolution3D layer calculates the output based on the input, filter @@ -1276,60 +1528,88 @@ def conv3d(x, if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): {}.".format(data_format)) + "Attr(data_format): {}.".format(data_format) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 5: raise ValueError( - "Input x should be 5D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 5D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] num_filters = weight.shape[0] if num_channels < 0: raise ValueError( "The channel dimension of the input({}) should be defined. " - "Received: {}.".format(x.shape, num_channels)) + "Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv3d should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv3d should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "The number of input channels must be divisible by Attr(groups). " "Received: number of channels({}), groups({}).".format( - num_channels, groups)) + num_channels, groups + ) + ) if num_filters % groups != 0: raise ValueError( "The number of filters must be divisible by Attr(groups). " "Received: number of filters({}), groups({}).".format( - num_filters, groups)) + num_filters, groups + ) + ) cudnn_version = get_cudnn_version() - use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) padding, padding_algorithm = _update_padding_nd(padding, channel_last, 3) stride = convert_to_list(stride, 3, 'stride') dilation = convert_to_list(dilation, 3, 'dilation') op_type = "conv3d" - return _conv_nd(x, weight, bias, stride, padding, padding_algorithm, - dilation, groups, data_format, channel_dim, op_type, - use_cudnn, False, name) - - -def conv3d_transpose(x, - weight, - bias=None, - stride=1, - padding=0, - output_padding=0, - groups=1, - dilation=1, - output_size=None, - data_format='NCDHW', - name=None): + return _conv_nd( + x, + weight, + bias, + stride, + padding, + padding_algorithm, + dilation, + groups, + data_format, + channel_dim, + op_type, + use_cudnn, + False, + name, + ) + + +def conv3d_transpose( + x, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, + output_size=None, + data_format='NCDHW', + name=None, +): r""" The convolution3d transpose layer calculates the output based on the input, filter, and dilations, strides, paddings. Input(Input) and output(Output) @@ -1463,29 +1743,37 @@ def conv3d_transpose(x, if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): {}.".format(data_format)) + "Attr(data_format): {}.".format(data_format) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 5: raise ValueError( - "Input x should be 5D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 5D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] num_filters = weight.shape[1] if num_channels < 0: raise ValueError( "The channel dimension of the input({}) should be defined. " - "Received: {}.".format(x.shape, num_channels)) + "Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv3d_transpose should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv3d_transpose should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "The number of input channels must be divisible by Attr(groups). " "Received: number of channels({}), groups({}).".format( - num_channels, groups)) + num_channels, groups + ) + ) padding, padding_algorithm = _update_padding_nd(padding, channel_last, 3) stride = convert_to_list(stride, 3, 'stride') @@ -1494,13 +1782,16 @@ def conv3d_transpose(x, output_size = [] else: if output_padding != 0: - raise ValueError('output_padding option is mutually exclusive with ' - 'output_size') + raise ValueError( + 'output_padding option is mutually exclusive with ' + 'output_size' + ) if isinstance(output_size, (list, tuple, int)): output_size = convert_to_list(output_size, 3, 'output_size') else: raise ValueError( - "output_size should be int, or list, tuple of ints") + "output_size should be int, or list, tuple of ints" + ) if output_padding == 0: output_padding = [] @@ -1509,28 +1800,55 @@ def conv3d_transpose(x, cudnn_version = get_cudnn_version() - #TODO(LielinJiang): whether to use cudnn according to the version of cudnn - use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + # TODO(LielinJiang): whether to use cudnn according to the version of cudnn + use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) op_type = 'conv3d_transpose' data_format_ = "NHWC" if channel_last else "NCHW" if in_dygraph_mode(): - pre_bias = _C_ops.conv3d_transpose(x, weight, stride, padding, - output_padding, output_size, - padding_algorithm, groups, dilation, - data_format_) + pre_bias = _C_ops.conv3d_transpose( + x, + weight, + stride, + padding, + output_padding, + output_size, + padding_algorithm, + groups, + dilation, + data_format_, + ) if bias is not None: return nn.elementwise_add(pre_bias, bias, axis=channel_dim) else: return pre_bias if _in_legacy_dygraph(): - attrs = ('output_padding', output_padding, 'output_size', output_size, - 'paddings', padding, "padding_algorithm", padding_algorithm, - 'strides', stride, 'dilations', dilation, 'groups', groups, - 'use_cudnn', use_cudnn, "data_format", data_format_) + attrs = ( + 'output_padding', + output_padding, + 'output_size', + output_size, + 'paddings', + padding, + "padding_algorithm", + padding_algorithm, + 'strides', + stride, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + "data_format", + data_format_, + ) pre_bias = getattr(_legacy_C_ops, op_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) @@ -1547,19 +1865,19 @@ def conv3d_transpose(x, 'dilations': dilation, 'groups': groups, 'use_cudnn': use_cudnn, - "data_format": data_format_ + "data_format": data_format_, } helper = LayerHelper(op_type, **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'conv3d') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'conv3d' + ) pre_bias = helper.create_variable_for_type_inference(x.dtype) outputs = {"Output": [pre_bias]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) else: diff --git a/python/paddle/nn/functional/distance.py b/python/paddle/nn/functional/distance.py index 1c29d509741..c1d40a83cfb 100644 --- a/python/paddle/nn/functional/distance.py +++ b/python/paddle/nn/functional/distance.py @@ -21,7 +21,7 @@ from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph __all__ = [] -def pairwise_distance(x, y, p=2., epsilon=1e-6, keepdim=False, name=None): +def pairwise_distance(x, y, p=2.0, epsilon=1e-6, keepdim=False, name=None): r""" It computes the pairwise distance between two vectors. The distance is calculated by p-oreder norm: @@ -70,40 +70,42 @@ def pairwise_distance(x, y, p=2., epsilon=1e-6, keepdim=False, name=None): sub = _C_ops.subtract(x, y) # p_norm op has not uesd epsilon, so change it to the following. if epsilon != 0.0: - epsilon = paddle.fluid.dygraph.base.to_variable([epsilon], - dtype=sub.dtype) + epsilon = paddle.fluid.dygraph.base.to_variable( + [epsilon], dtype=sub.dtype + ) sub = _C_ops.add(sub, epsilon) - return _C_ops.p_norm(sub, p, -1, 0., keepdim, False) + return _C_ops.p_norm(sub, p, -1, 0.0, keepdim, False) if _in_legacy_dygraph(): sub = _legacy_C_ops.elementwise_sub(x, y) if epsilon != 0.0: - epsilon = paddle.fluid.dygraph.base.to_variable([epsilon], - dtype=sub.dtype) + epsilon = paddle.fluid.dygraph.base.to_variable( + [epsilon], dtype=sub.dtype + ) sub = _legacy_C_ops.elementwise_add(sub, epsilon) - return _legacy_C_ops.p_norm(sub, 'axis', -1, 'porder', p, 'keepdim', - keepdim, 'epsilon', 0.) + return _legacy_C_ops.p_norm( + sub, 'axis', -1, 'porder', p, 'keepdim', keepdim, 'epsilon', 0.0 + ) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'PairwiseDistance') check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'PairwiseDistance') sub = paddle.subtract(x, y) if epsilon != 0.0: epsilon_var = sub.block.create_var(dtype=sub.dtype) - epsilon_var = paddle.full(shape=[1], - fill_value=epsilon, - dtype=sub.dtype) + epsilon_var = paddle.full( + shape=[1], fill_value=epsilon, dtype=sub.dtype + ) sub = paddle.add(sub, epsilon_var) helper = LayerHelper("PairwiseDistance", name=name) attrs = { 'axis': -1, 'porder': p, 'keepdim': keepdim, - 'epsilon': 0., + 'epsilon': 0.0, } out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='p_norm', - inputs={'X': sub}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='p_norm', inputs={'X': sub}, outputs={'Out': out}, attrs=attrs + ) return out diff --git a/python/paddle/nn/functional/extension.py b/python/paddle/nn/functional/extension.py index cc11472643c..1cc9ad6caf4 100644 --- a/python/paddle/nn/functional/extension.py +++ b/python/paddle/nn/functional/extension.py @@ -22,7 +22,11 @@ from ...tensor.creation import assign from ...tensor.layer_function_generator import templatedoc from paddle import in_dynamic_mode from paddle import _C_ops, _legacy_C_ops -from ...fluid.framework import _non_static_mode, _in_legacy_dygraph, in_dygraph_mode +from ...fluid.framework import ( + _non_static_mode, + _in_legacy_dygraph, + in_dygraph_mode, +) from ...fluid.data_feeder import check_variable_and_dtype, check_type from ...framework import core, convert_np_dtype_to_dtype_ @@ -102,50 +106,55 @@ def diag_embed(input, offset=0, dim1=-2, dim2=-1): if in_dygraph_mode(): return _C_ops.diag_embed(input, offset, dim1, dim2) elif in_dynamic_mode(): - return _legacy_C_ops.diag_embed(input, "offset", offset, "dim1", dim1, - "dim2", dim2) + return _legacy_C_ops.diag_embed( + input, "offset", offset, "dim1", dim1, "dim2", dim2 + ) inputs = {'Input': [input]} attrs = {'offset': offset, 'dim1': dim1, 'dim2': dim2} def __check_input(input, offset, dim1, dim2): - check_dtype(input.dtype, 'Input', - ['int32', 'int64', 'float16', 'float32', 'float64'], - 'diag_embed') + check_dtype( + input.dtype, + 'Input', + ['int32', 'int64', 'float16', 'float32', 'float64'], + 'diag_embed', + ) input_shape = list(input.shape) - assert len(input_shape) >= 1, \ - "Input must be at least 1-dimensional, " \ - "But received Input's dimensional: %s.\n" % \ - len(input_shape) + assert len(input_shape) >= 1, ( + "Input must be at least 1-dimensional, " + "But received Input's dimensional: %s.\n" % len(input_shape) + ) - assert np.abs(dim1) <= len(input_shape), \ - "Dim1 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert np.abs(dim1) <= len(input_shape), ( + "Dim1 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape) + 1), len(input_shape), dim1) + ) - assert np.abs(dim2) <= len(input_shape), \ - "Dim2 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert np.abs(dim2) <= len(input_shape), ( + "Dim2 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape) + 1), len(input_shape), dim2) + ) dim1_ = dim1 if dim1 >= 0 else len(input_shape) + dim1 + 1 dim2_ = dim2 if dim2 >= 0 else len(input_shape) + dim2 + 1 - assert dim1_ != dim2_, \ - "dim1 and dim2 cannot be the same dimension." \ - "But received dim1 = %d, dim2 = %d\n"%(dim1, dim2) + assert dim1_ != dim2_, ( + "dim1 and dim2 cannot be the same dimension." + "But received dim1 = %d, dim2 = %d\n" % (dim1, dim2) + ) __check_input(input, offset, dim1, dim2) helper = LayerHelper("diag_embed", **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='diag_embed', - inputs={'Input': [input]}, - attrs={ - 'offset': offset, - 'dim1': dim1, - 'dim2': dim2 - }, - outputs={'Out': [out]}) + helper.append_op( + type='diag_embed', + inputs={'Input': [input]}, + attrs={'offset': offset, 'dim1': dim1, 'dim2': dim2}, + outputs={'Out': [out]}, + ) out.stop_gradient = True return out @@ -233,10 +242,9 @@ def sequence_mask(x, maxlen=None, dtype='int64', name=None): else: attrs['maxlen'] = maxlen - helper.append_op(type='sequence_mask', - inputs=inputs, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type='sequence_mask', inputs=inputs, outputs={'Y': out}, attrs=attrs + ) out.stop_gradient = True return out @@ -317,18 +325,19 @@ def gather_tree(ids, parents): return _legacy_C_ops.gather_tree(ids, parents) else: helper = LayerHelper('gather_tree', **locals()) - check_variable_and_dtype(ids, 'ids', ['int32', 'int64'], - 'gather_tree') - check_variable_and_dtype(parents, 'parents', ['int32', 'int64'], - 'gather_tree') + check_variable_and_dtype( + ids, 'ids', ['int32', 'int64'], 'gather_tree' + ) + check_variable_and_dtype( + parents, 'parents', ['int32', 'int64'], 'gather_tree' + ) out = helper.create_variable_for_type_inference(dtype=ids.dtype) - helper.append_op(type="gather_tree", - inputs={ - "Ids": ids, - "Parents": parents - }, - outputs={"Out": out}) + helper.append_op( + type="gather_tree", + inputs={"Ids": ids, "Parents": parents}, + outputs={"Out": out}, + ) return out @@ -365,14 +374,22 @@ def temporal_shift(x, seg_num, shift_ratio=0.25, name=None, data_format="NCHW"): out = F.temporal_shift(x=input, seg_num=2, shift_ratio=0.2) """ if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Attr(data_format) should be 'NCHW' or 'NHWC'. " - "Received Attr(data_format): {}.".format(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCHW' or 'NHWC'. " + "Received Attr(data_format): {}.".format(data_format) + ) if in_dygraph_mode(): return _C_ops.temporal_shift(x, seg_num, shift_ratio, data_format) if _non_static_mode(): - return _legacy_C_ops.temporal_shift(x, 'seg_num', seg_num, - 'shift_ratio', shift_ratio, - 'data_format', data_format) + return _legacy_C_ops.temporal_shift( + x, + 'seg_num', + seg_num, + 'shift_ratio', + shift_ratio, + 'data_format', + data_format, + ) helper = LayerHelper("temporal_shift", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'temporal_shift') @@ -384,12 +401,14 @@ def temporal_shift(x, seg_num, shift_ratio=0.25, name=None, data_format="NCHW"): if not isinstance(seg_num, int): raise TypeError("seg_num must be int type.") - helper.append_op(type="temporal_shift", - inputs={"X": x}, - outputs={"Out": out}, - attrs={ - "seg_num": seg_num, - "shift_ratio": shift_ratio, - "data_format": data_format - }) + helper.append_op( + type="temporal_shift", + inputs={"X": x}, + outputs={"Out": out}, + attrs={ + "seg_num": seg_num, + "shift_ratio": shift_ratio, + "data_format": data_format, + }, + ) return out diff --git a/python/paddle/nn/functional/input.py b/python/paddle/nn/functional/input.py index 8df602e5c77..67448320820 100644 --- a/python/paddle/nn/functional/input.py +++ b/python/paddle/nn/functional/input.py @@ -88,15 +88,18 @@ def one_hot(x, num_classes, name=None): return _C_ops.one_hot(x, num_classes) else: if _in_legacy_dygraph(): - return _legacy_C_ops.one_hot_v2(x, 'depth', num_classes, - 'allow_out_of_range', False) + return _legacy_C_ops.one_hot_v2( + x, 'depth', num_classes, 'allow_out_of_range', False + ) else: - check_variable_and_dtype(x, 'input', ['int32', 'int64'], - 'one_hot_v2') + check_variable_and_dtype( + x, 'input', ['int32', 'int64'], 'one_hot_v2' + ) helper = LayerHelper("one_hot_v2", **locals()) one_hot_out = helper.create_variable_for_type_inference( - dtype='float32') + dtype='float32' + ) if not isinstance(num_classes, Variable): # user attribute inputs = {'X': x} @@ -105,11 +108,13 @@ def one_hot(x, num_classes, name=None): num_classes.stop_gradient = True inputs = {'X': x, 'depth_tensor': num_classes} attrs = {'allow_out_of_range': False} - helper.append_op(type="one_hot_v2", - inputs=inputs, - attrs=attrs, - outputs={'Out': one_hot_out}, - stop_gradient=True) + helper.append_op( + type="one_hot_v2", + inputs=inputs, + attrs=attrs, + outputs={'Out': one_hot_out}, + stop_gradient=True, + ) return one_hot_out @@ -189,43 +194,61 @@ def embedding(x, weight, padding_idx=None, sparse=False, name=None): x=x, weight=w, sparse=True, name="embedding") """ - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - weight.shape[0] + padding_idx) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (weight.shape[0] + padding_idx) + ) if padding_idx >= weight.shape[0] or padding_idx < -weight.shape[0]: - raise ValueError("padding_idx must be within [-{}, {})".format( - weight.shape[0], weight.shape[0])) + raise ValueError( + "padding_idx must be within [-{}, {})".format( + weight.shape[0], weight.shape[0] + ) + ) if in_dygraph_mode(): return _C_ops.embedding(x, weight, padding_idx, sparse) elif _in_legacy_dygraph(): - return _legacy_C_ops.lookup_table_v2(weight, x, 'is_sparse', sparse, - 'is_distributed', False, - 'remote_prefetch', False, - 'padding_idx', padding_idx) + return _legacy_C_ops.lookup_table_v2( + weight, + x, + 'is_sparse', + sparse, + 'is_distributed', + False, + 'remote_prefetch', + False, + 'padding_idx', + padding_idx, + ) else: helper = LayerHelper('embedding', **locals()) dtype = helper.input_dtype(input_param_name='weight') - check_variable_and_dtype(x, 'input', - ['uint8', 'int8', 'int16', 'int32', 'int64'], - 'embedding') + check_variable_and_dtype( + x, + 'input', + ['uint8', 'int8', 'int16', 'int32', 'int64'], + 'embedding', + ) is_distributed = False remote_prefetch = sparse and (not is_distributed) tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='lookup_table_v2', - inputs={ - 'Ids': x, - 'W': weight - }, - outputs={'Out': tmp}, - attrs={ - 'is_sparse': sparse, - 'is_distributed': is_distributed, - 'remote_prefetch': remote_prefetch, - 'padding_idx': padding_idx - }) + helper.append_op( + type='lookup_table_v2', + inputs={'Ids': x, 'W': weight}, + outputs={'Out': tmp}, + attrs={ + 'is_sparse': sparse, + 'is_distributed': is_distributed, + 'remote_prefetch': remote_prefetch, + 'padding_idx': padding_idx, + }, + ) return tmp diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index ed28bc21902..48cda9d0b4f 100755 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -28,7 +28,12 @@ from paddle.utils import deprecated from paddle import _C_ops, _legacy_C_ops from paddle import in_dynamic_mode from paddle.framework import core, _non_static_mode -from ...fluid.framework import _in_legacy_dygraph, in_dygraph_mode, _non_static_mode, _current_expected_place +from ...fluid.framework import ( + _in_legacy_dygraph, + in_dygraph_mode, + _non_static_mode, + _current_expected_place, +) __all__ = [] @@ -77,25 +82,32 @@ def dice_loss(input, label, epsilon=0.00001, name=None): """ assert input.dtype in (paddle.float32, paddle.float64) assert label.dtype in (paddle.int32, paddle.int64) - assert len(input.shape) >= 2, \ - "The rank of input should be greater than or equal to 2." - assert len(input.shape) == len( - label.shape), ("The rank of input and label should be equal, " - "but received input: %d, label: %d." % - (len(input.shape), len(label.shape))) - assert label.shape[-1] == 1, ("The last dimension of label should be 1, " - "but received %d." % label.shape[-1]) - assert input.shape[:-1] == label.shape[:-1], ( - "All dimensions should be equal except the last one.") - assert input.numel() > 0 and label.numel() > 0, \ - "Any dimension of input and label cannot be equal to 0." + assert ( + len(input.shape) >= 2 + ), "The rank of input should be greater than or equal to 2." + assert len(input.shape) == len(label.shape), ( + "The rank of input and label should be equal, " + "but received input: %d, label: %d." + % (len(input.shape), len(label.shape)) + ) + assert label.shape[-1] == 1, ( + "The last dimension of label should be 1, " + "but received %d." % label.shape[-1] + ) + assert ( + input.shape[:-1] == label.shape[:-1] + ), "All dimensions should be equal except the last one." + assert ( + input.numel() > 0 and label.numel() > 0 + ), "Any dimension of input and label cannot be equal to 0." label = paddle.squeeze(label, [-1]) label = paddle.nn.functional.one_hot(label, input.shape[-1]) reduce_dim = list(range(1, len(input.shape))) inse = paddle.sum(input * label, axis=reduce_dim) dice_denominator = paddle.sum(input, axis=reduce_dim) + paddle.sum( - label, axis=reduce_dim) + label, axis=reduce_dim + ) dice_score = 1 - inse * 2 / (dice_denominator + epsilon) return paddle.mean(dice_score) @@ -146,23 +158,24 @@ def log_loss(input, label, epsilon=1e-4, name=None): loss = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='log_loss', - inputs={ - 'Predicted': [input], - 'Labels': [label] - }, - outputs={'Loss': [loss]}, - attrs={'epsilon': epsilon}) + helper.append_op( + type='log_loss', + inputs={'Predicted': [input], 'Labels': [label]}, + outputs={'Loss': [loss]}, + attrs={'epsilon': epsilon}, + ) return loss -def fluid_softmax_with_cross_entropy(logits, - label, - soft_label=False, - ignore_index=-100, - numeric_stable_mode=True, - return_softmax=False, - axis=-1): +def fluid_softmax_with_cross_entropy( + logits, + label, + soft_label=False, + ignore_index=-100, + numeric_stable_mode=True, + return_softmax=False, + axis=-1, +): r""" This operator implements the cross entropy loss function with softmax. This function @@ -252,19 +265,41 @@ def fluid_softmax_with_cross_entropy(logits, if _non_static_mode(): if core.is_compiled_with_npu(): softmax, backprop, loss = _legacy_C_ops.softmax_with_cross_entropy( - logits, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', numeric_stable_mode, - 'axis', axis) + logits, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + numeric_stable_mode, + 'axis', + axis, + ) else: if in_dygraph_mode(): softmax, loss = _C_ops.cross_entropy_with_softmax( - logits, label, soft_label, True, numeric_stable_mode, - ignore_index, axis) + logits, + label, + soft_label, + True, + numeric_stable_mode, + ignore_index, + axis, + ) if _in_legacy_dygraph(): softmax, loss = _legacy_C_ops.softmax_with_cross_entropy( - logits, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', numeric_stable_mode, - 'axis', axis) + logits, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + numeric_stable_mode, + 'axis', + axis, + ) if not return_softmax: return loss else: @@ -274,7 +309,7 @@ def fluid_softmax_with_cross_entropy(logits, 'soft_label': soft_label, 'ignore_index': ignore_index, 'numeric_stable_mode': numeric_stable_mode, - 'axis': axis + 'axis': axis, } helper = LayerHelper('softmax_with_cross_entropy', **locals()) softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) @@ -284,13 +319,12 @@ def fluid_softmax_with_cross_entropy(logits, if core.is_compiled_with_npu() or core.is_compiled_with_mlu(): backprop = helper.create_variable_for_type_inference(dtype=logits.dtype) outputs['Backprop'] = backprop - helper.append_op(type='softmax_with_cross_entropy', - inputs={ - 'Logits': logits, - 'Label': label - }, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='softmax_with_cross_entropy', + inputs={'Logits': logits, 'Label': label}, + outputs=outputs, + attrs=attrs, + ) if return_softmax: return loss, softmax @@ -336,34 +370,37 @@ def npair_loss(anchor, positive, labels, l2_reg=0.002): print(npair_loss) """ - check_variable_and_dtype(anchor, 'anchor', ['float32', 'float64'], - 'npair_loss') - check_variable_and_dtype(positive, 'positive', ['float32', 'float64'], - 'positive') - check_variable_and_dtype(labels, 'labels', ['float32', 'float64', 'int64'], - 'labels') + check_variable_and_dtype( + anchor, 'anchor', ['float32', 'float64'], 'npair_loss' + ) + check_variable_and_dtype( + positive, 'positive', ['float32', 'float64'], 'positive' + ) + check_variable_and_dtype( + labels, 'labels', ['float32', 'float64', 'int64'], 'labels' + ) Beta = 0.25 batch_size = labels.shape[0] labels = paddle.reshape(labels, shape=[batch_size, 1]) labels = paddle.tile(labels, repeat_times=[1, batch_size]) - labels = paddle.equal(labels, paddle.transpose(labels, - perm=[1, - 0])).astype('float32') + labels = paddle.equal(labels, paddle.transpose(labels, perm=[1, 0])).astype( + 'float32' + ) labels = labels / paddle.sum(labels, axis=1, keepdim=True) - l2loss = paddle.mean(paddle.sum(paddle.square(anchor), 1)) \ - + paddle.mean(paddle.sum(paddle.square(positive), 1)) + l2loss = paddle.mean(paddle.sum(paddle.square(anchor), 1)) + paddle.mean( + paddle.sum(paddle.square(positive), 1) + ) l2loss = l2loss * Beta * l2_reg - similarity_matrix = paddle.matmul(anchor, - positive, - transpose_x=False, - transpose_y=True) - softmax_ce = fluid_softmax_with_cross_entropy(logits=similarity_matrix, - label=labels, - soft_label=True) + similarity_matrix = paddle.matmul( + anchor, positive, transpose_x=False, transpose_y=True + ) + softmax_ce = fluid_softmax_with_cross_entropy( + logits=similarity_matrix, label=labels, soft_label=True + ) cross_entropy = paddle.sum(labels * softmax_ce, 0) celoss = paddle.mean(cross_entropy) @@ -411,32 +448,35 @@ def square_error_cost(input, label): square_out = _legacy_C_ops.square(minus_out) return square_out - check_variable_and_dtype(input, "input", ['float32', 'float64'], - 'square_error_cost') - check_variable_and_dtype(label, "label", ['float32', 'float64'], - 'square_error_cost') + check_variable_and_dtype( + input, "input", ['float32', 'float64'], 'square_error_cost' + ) + check_variable_and_dtype( + label, "label", ['float32', 'float64'], 'square_error_cost' + ) helper = LayerHelper('square_error_cost', **locals()) minus_out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='elementwise_sub', - inputs={ - 'X': [input], - 'Y': [label] - }, - outputs={'Out': [minus_out]}) + helper.append_op( + type='elementwise_sub', + inputs={'X': [input], 'Y': [label]}, + outputs={'Out': [minus_out]}, + ) square_out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='square', - inputs={'X': [minus_out]}, - outputs={'Out': [square_out]}) + helper.append_op( + type='square', inputs={'X': [minus_out]}, outputs={'Out': [square_out]} + ) return square_out -def edit_distance(input, - label, - normalized=True, - ignored_tokens=None, - input_length=None, - label_length=None): +def edit_distance( + input, + label, + normalized=True, + ignored_tokens=None, + input_length=None, + label_length=None, +): """ This op computes the edit distances, also called Levenshtein distance, between a batch of hypothesis strings and their references. It measures how dissimilar two strings are by counting @@ -512,21 +552,26 @@ def edit_distance(input, erased_input = helper.create_variable_for_type_inference(dtype="int64") erased_label = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="sequence_erase", - inputs={"X": [input]}, - outputs={"Out": [erased_input]}, - attrs={"tokens": ignored_tokens}) + helper.append_op( + type="sequence_erase", + inputs={"X": [input]}, + outputs={"Out": [erased_input]}, + attrs={"tokens": ignored_tokens}, + ) input = erased_input - helper.append_op(type="sequence_erase", - inputs={"X": [label]}, - outputs={"Out": [erased_label]}, - attrs={"tokens": ignored_tokens}) + helper.append_op( + type="sequence_erase", + inputs={"X": [label]}, + outputs={"Out": [erased_label]}, + attrs={"tokens": ignored_tokens}, + ) label = erased_label if in_dygraph_mode(): - return _C_ops.edit_distance(input, label, input_length, label_length, - normalized) + return _C_ops.edit_distance( + input, label, input_length, label_length, normalized + ) this_inputs = {"Hyps": [input], "Refs": [label]} if input_length is not None and label_length is not None: @@ -536,22 +581,19 @@ def edit_distance(input, # edit distance op edit_distance_out = helper.create_variable_for_type_inference(dtype="int64") sequence_num = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="edit_distance", - inputs=this_inputs, - outputs={ - "Out": [edit_distance_out], - "SequenceNum": [sequence_num] - }, - attrs={"normalized": normalized}) + helper.append_op( + type="edit_distance", + inputs=this_inputs, + outputs={"Out": [edit_distance_out], "SequenceNum": [sequence_num]}, + attrs={"normalized": normalized}, + ) return edit_distance_out, sequence_num -def binary_cross_entropy(input, - label, - weight=None, - reduction='mean', - name=None): +def binary_cross_entropy( + input, label, weight=None, reduction='mean', name=None +): """ This op measures the binary_cross_entropy loss between input predictions ``input`` and target labels ``label`` . The binary_cross_entropy loss can be described as: @@ -619,8 +661,9 @@ def binary_cross_entropy(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in binary_cross_entropy should be 'sum', " - "'mean' or 'none', but received %s, which is not allowed." % - reduction) + "'mean' or 'none', but received %s, which is not allowed." + % reduction + ) if in_dygraph_mode(): out = _C_ops.bce_loss(input, label) @@ -640,27 +683,32 @@ def binary_cross_entropy(input, if weight is not None: out = _legacy_C_ops.elementwise_mul(out, weight, 'axis', -1) if reduction == 'sum': - return _legacy_C_ops.reduce_sum(out, 'dim', [0], 'keep_dim', - False, "reduce_all", True) + return _legacy_C_ops.reduce_sum( + out, 'dim', [0], 'keep_dim', False, "reduce_all", True + ) elif reduction == 'mean': return _legacy_C_ops.mean(out) else: return out else: - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'binary_cross_entropy') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'binary_cross_entropy') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'binary_cross_entropy' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'binary_cross_entropy' + ) sub_name = name if weight is None and reduction == 'none' else None helper = LayerHelper("binary_cross_entropy", name=sub_name) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='bce_loss', - inputs={ - 'X': [input], - 'Label': [label], - }, - outputs={'Out': [out]}) + helper.append_op( + type='bce_loss', + inputs={ + 'X': [input], + 'Label': [label], + }, + outputs={'Out': [out]}, + ) if weight is not None: if isinstance(weight, paddle.static.Variable): @@ -668,7 +716,8 @@ def binary_cross_entropy(input, out = paddle.multiply(out, weight, name=weight_name) else: raise ValueError( - "The weight is not a Tensor, please convert to Tensor.") + "The weight is not a Tensor, please convert to Tensor." + ) if reduction == 'sum': return paddle.sum(out, name=name) @@ -678,12 +727,9 @@ def binary_cross_entropy(input, return out -def binary_cross_entropy_with_logits(logit, - label, - weight=None, - reduction='mean', - pos_weight=None, - name=None): +def binary_cross_entropy_with_logits( + logit, label, weight=None, reduction='mean', pos_weight=None, name=None +): r""" This operator combines the sigmoid layer and the :ref:`api_nn_loss_BCELoss` layer. Also, we can see it as the combine of ``sigmoid_cross_entropy_with_logits`` @@ -765,16 +811,23 @@ def binary_cross_entropy_with_logits(logit, raise ValueError( "The value of 'reduction' in binary_cross_entropy_with_logits " "should be 'sum', 'mean' or 'none', but received %s, which is not allowed." - % reduction) + % reduction + ) if in_dygraph_mode(): - one = _C_ops.full([1], float(1.0), core.VarDesc.VarType.FP32, - _current_expected_place()) - out = _C_ops.sigmoid_cross_entropy_with_logits(logit, label, False, - -100) + one = _C_ops.full( + [1], + float(1.0), + core.VarDesc.VarType.FP32, + _current_expected_place(), + ) + out = _C_ops.sigmoid_cross_entropy_with_logits( + logit, label, False, -100 + ) if pos_weight is not None: log_weight = _C_ops.add( - _C_ops.multiply(label, _C_ops.subtract(pos_weight, one)), one) + _C_ops.multiply(label, _C_ops.subtract(pos_weight, one)), one + ) out = _C_ops.multiply(out, log_weight) if weight is not None: out = _C_ops.multiply(out, weight) @@ -787,14 +840,27 @@ def binary_cross_entropy_with_logits(logit, return out elif _in_legacy_dygraph(): one = _varbase_creator(dtype=logit.dtype) - _legacy_C_ops.fill_constant(one, 'value', float(1.0), 'force_cpu', - False, 'dtype', one.dtype, 'str_value', - '1.0', 'shape', [1]) + _legacy_C_ops.fill_constant( + one, + 'value', + float(1.0), + 'force_cpu', + False, + 'dtype', + one.dtype, + 'str_value', + '1.0', + 'shape', + [1], + ) out = _legacy_C_ops.sigmoid_cross_entropy_with_logits(logit, label) if pos_weight is not None: log_weight = _legacy_C_ops.elementwise_add( _legacy_C_ops.elementwise_mul( - label, _legacy_C_ops.elementwise_sub(pos_weight, one)), one) + label, _legacy_C_ops.elementwise_sub(pos_weight, one) + ), + one, + ) out = _legacy_C_ops.elementwise_mul(out, log_weight) if weight is not None: out = _legacy_C_ops.elementwise_mul(out, weight) @@ -806,30 +872,49 @@ def binary_cross_entropy_with_logits(logit, else: return out - check_variable_and_dtype(logit, 'logit', ['float32', 'float64'], - 'binary_cross_entropy_with_logits') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'binary_cross_entropy_with_logits') + check_variable_and_dtype( + logit, + 'logit', + ['float32', 'float64'], + 'binary_cross_entropy_with_logits', + ) + check_variable_and_dtype( + label, + 'label', + ['float32', 'float64'], + 'binary_cross_entropy_with_logits', + ) sigmoid_name = None if reduction == 'none' and pos_weight is None and weight is None: sigmoid_name = name out = paddle.fluid.layers.sigmoid_cross_entropy_with_logits( - logit, label, name=sigmoid_name) + logit, label, name=sigmoid_name + ) one = paddle.full(shape=[1], fill_value=1.0, dtype=logit.dtype) if pos_weight is not None: - check_variable_and_dtype(pos_weight, 'pos_weight', - ['float32', 'float64'], - 'binary_cross_entropy_with_logits') + check_variable_and_dtype( + pos_weight, + 'pos_weight', + ['float32', 'float64'], + 'binary_cross_entropy_with_logits', + ) log_weight = paddle.add( - paddle.multiply(label, paddle.subtract(pos_weight, one)), one) - pos_weight_name = name if reduction == 'none' and weight is None else None + paddle.multiply(label, paddle.subtract(pos_weight, one)), one + ) + pos_weight_name = ( + name if reduction == 'none' and weight is None else None + ) out = paddle.multiply(out, log_weight, name=pos_weight_name) if weight is not None: - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'binary_cross_entropy_with_logits') + check_variable_and_dtype( + weight, + 'weight', + ['float32', 'float64'], + 'binary_cross_entropy_with_logits', + ) weight_name = name if reduction == 'none' else None out = paddle.multiply(out, weight, name=weight_name) @@ -840,15 +925,17 @@ def binary_cross_entropy_with_logits(logit, return out -def hsigmoid_loss(input, - label, - num_classes, - weight, - bias=None, - path_table=None, - path_code=None, - is_sparse=False, - name=None): +def hsigmoid_loss( + input, + label, + num_classes, + weight, + bias=None, + path_table=None, + path_code=None, + is_sparse=False, + name=None, +): """ The hierarchical sigmoid organizes the classes into a complete binary tree to reduce the computational complexity and speed up the model training, especially the training of language model. @@ -930,36 +1017,63 @@ def hsigmoid_loss(input, # [1.92374969]] """ if in_dygraph_mode(): - out, _, _ = _C_ops.hierarchical_sigmoid(input, weight, label, - path_table, path_code, bias, - num_classes, is_sparse, 0, [], - [], [], is_sparse) + out, _, _ = _C_ops.hierarchical_sigmoid( + input, + weight, + label, + path_table, + path_code, + bias, + num_classes, + is_sparse, + 0, + [], + [], + [], + is_sparse, + ) return out elif _in_legacy_dygraph(): out, _, _ = _legacy_C_ops.hierarchical_sigmoid( - input, weight, label, path_table, path_code, bias, 'num_classes', - num_classes, 'is_sparse', is_sparse, 'remote_prefetch', is_sparse) + input, + weight, + label, + path_table, + path_code, + bias, + 'num_classes', + num_classes, + 'is_sparse', + is_sparse, + 'remote_prefetch', + is_sparse, + ) return out - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'hsigmoid_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'hsigmoid_loss' + ) check_variable_and_dtype(label, 'label', ['int64'], 'hsigmoid_loss') - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'hsigmoid_loss') + check_variable_and_dtype( + weight, 'weight', ['float32', 'float64'], 'hsigmoid_loss' + ) if bias is not None: - check_variable_and_dtype(bias, 'bias', ['float32', 'float64'], - 'hsigmoid_loss') + check_variable_and_dtype( + bias, 'bias', ['float32', 'float64'], 'hsigmoid_loss' + ) if path_table is not None: - check_variable_and_dtype(path_table, 'path_table', ['int64'], - 'hsigmoid_loss') + check_variable_and_dtype( + path_table, 'path_table', ['int64'], 'hsigmoid_loss' + ) if path_code is not None: - check_variable_and_dtype(path_code, 'path_code', ['int64'], - 'hsigmoid_loss') + check_variable_and_dtype( + path_code, 'path_code', ['int64'], 'hsigmoid_loss' + ) attrs = { "num_classes": num_classes, "is_sparse": is_sparse, - "remote_prefetch": is_sparse + "remote_prefetch": is_sparse, } inputs = { @@ -968,7 +1082,7 @@ def hsigmoid_loss(input, "Bias": bias, "PathTable": path_table, "PathCode": path_code, - "Label": label + "Label": label, } helper = LayerHelper('hsigmoid_loss', **locals()) @@ -976,10 +1090,9 @@ def hsigmoid_loss(input, pre_out = helper.create_variable_for_type_inference(input.dtype) outputs = {"Out": out, "PreOut": pre_out, "W_Out": weight} - helper.append_op(type="hierarchical_sigmoid", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="hierarchical_sigmoid", inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1036,34 +1149,35 @@ def smooth_l1_loss(input, label, reduction='mean', delta=1.0, name=None): print(output) # [0.068004] """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'smooth_l1_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'smooth_l1_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'smooth_l1_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'smooth_l1_loss' + ) if in_dygraph_mode(): out, residual = _C_ops.huber_loss(input, label, delta) else: helper = LayerHelper('huber_loss', **locals()) residual = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) - helper.append_op(type='huber_loss', - inputs={ - 'X': input, - 'Y': label - }, - outputs={ - 'Out': out, - 'Residual': residual - }, - attrs={'delta': delta}) + dtype=helper.input_dtype() + ) + helper.append_op( + type='huber_loss', + inputs={'X': input, 'Y': label}, + outputs={'Out': out, 'Residual': residual}, + attrs={'delta': delta}, + ) if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in smooth_l1_loss should be 'sum', 'mean' or" - " 'none', but received %s, which is not allowed." % reduction) + " 'none', but received %s, which is not allowed." % reduction + ) if reduction == 'none': return out elif reduction == 'mean': @@ -1072,12 +1186,9 @@ def smooth_l1_loss(input, label, reduction='mean', delta=1.0, name=None): return paddle.sum(out) -def margin_ranking_loss(input, - other, - label, - margin=0.0, - reduction='mean', - name=None): +def margin_ranking_loss( + input, other, label, margin=0.0, reduction='mean', name=None +): r""" Calcluate the margin rank loss between the input, other and label, use the math function as follows. @@ -1123,7 +1234,8 @@ def margin_ranking_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in MarginRankingLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) if in_dygraph_mode(): out = _C_ops.subtract(other, input) out = _C_ops.multiply(out, label) @@ -1150,12 +1262,15 @@ def margin_ranking_loss(input, return out helper = LayerHelper("margin_ranking_loss", **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'margin_rank_loss') - check_variable_and_dtype(other, 'other', ['float32', 'float64'], - 'margin_rank_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'margin_rank_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'margin_rank_loss' + ) + check_variable_and_dtype( + other, 'other', ['float32', 'float64'], 'margin_rank_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'margin_rank_loss' + ) out = paddle.subtract(input, other) neg_label = paddle.neg(label) @@ -1169,24 +1284,28 @@ def margin_ranking_loss(input, result_out = helper.create_variable_for_type_inference(input.dtype) if reduction == 'none': - helper.append_op(type="relu", - inputs={"X": out}, - outputs={"Out": result_out}) + helper.append_op( + type="relu", inputs={"X": out}, outputs={"Out": result_out} + ) return result_out elif reduction == 'sum': out = paddle.nn.functional.relu(out) attrs = {"dim": [0], "keep_dim": False, "reduce_all": True} - helper.append_op(type="reduce_sum", - inputs={"X": out}, - outputs={"Out": result_out}, - attrs=attrs) + helper.append_op( + type="reduce_sum", + inputs={"X": out}, + outputs={"Out": result_out}, + attrs=attrs, + ) return result_out elif reduction == 'mean': out = paddle.nn.functional.relu(out) - helper.append_op(type="mean", - inputs={"X": out}, - outputs={"Out": result_out}, - attrs={}) + helper.append_op( + type="mean", + inputs={"X": out}, + outputs={"Out": result_out}, + attrs={}, + ) return result_out @@ -1250,7 +1369,8 @@ def l1_loss(input, label, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in L1Loss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) if in_dygraph_mode(): unreduced = _C_ops.abs(_C_ops.subtract(input, label)) @@ -1262,25 +1382,24 @@ def l1_loss(input, label, reduction='mean', name=None): else: return unreduced elif _in_legacy_dygraph(): - unreduced = _elementwise_op_in_dygraph(input, - label, - axis=-1, - act='abs', - op_name='elementwise_sub') + unreduced = _elementwise_op_in_dygraph( + input, label, axis=-1, act='abs', op_name='elementwise_sub' + ) if reduction == 'mean': return _legacy_C_ops.mean(unreduced) elif reduction == 'sum': - return _legacy_C_ops.reduce_sum(unreduced, 'dim', [0], 'keep_dim', - False, 'reduce_all', True) + return _legacy_C_ops.reduce_sum( + unreduced, 'dim', [0], 'keep_dim', False, 'reduce_all', True + ) else: return unreduced - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'l1_loss') - check_variable_and_dtype(label, 'label', - ['float32', 'float64', 'int32', 'int64'], - 'l1_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64', 'int32', 'int64'], 'l1_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64', 'int32', 'int64'], 'l1_loss' + ) if reduction == 'sum': unreduced = paddle.fluid.layers.elementwise_sub(input, label, act='abs') @@ -1289,18 +1408,14 @@ def l1_loss(input, label, reduction='mean', name=None): unreduced = paddle.fluid.layers.elementwise_sub(input, label, act='abs') return paddle.mean(unreduced, name=name) else: - return paddle.fluid.layers.elementwise_sub(input, - label, - act='abs', - name=name) - - -def nll_loss(input, - label, - weight=None, - ignore_index=-100, - reduction='mean', - name=None): + return paddle.fluid.layers.elementwise_sub( + input, label, act='abs', name=name + ) + + +def nll_loss( + input, label, weight=None, ignore_index=-100, reduction='mean', name=None +): """ This api returns negative log likelihood. See more detail in :ref:`api_nn_loss_NLLLoss` . @@ -1349,13 +1464,15 @@ def nll_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in nll_loss should be 'sum', 'mean' or " - "'none', but received %s, which is not allowed." % reduction) + "'none', but received %s, which is not allowed." % reduction + ) input_shape = list(input.shape) input_dims = len(input_shape) if input_dims < 2: raise ValueError( - 'Expected 2 or more dimensions (got {})'.format(input_dims)) + 'Expected 2 or more dimensions (got {})'.format(input_dims) + ) n = input_shape[0] c = input_shape[1] if in_dygraph_mode(): @@ -1363,21 +1480,29 @@ def nll_loss(input, input = _C_ops.reshape(input, [n, c, 1, -1]) label = _C_ops.reshape(label, [n, 1, -1]) out_shape = [n] + input_shape[2:] - out, total_weight = _C_ops.nll_loss(input, label, weight, ignore_index, - reduction) + out, total_weight = _C_ops.nll_loss( + input, label, weight, ignore_index, reduction + ) if input_dims != 2 and input_dims != 4 and reduction == 'none': out = _C_ops.reshape(out, out_shape) return out elif _in_legacy_dygraph(): if input_dims != 2 and input_dims != 4: - input, _ = _legacy_C_ops.reshape2(input, None, 'shape', - [n, c, 1, -1]) + input, _ = _legacy_C_ops.reshape2( + input, None, 'shape', [n, c, 1, -1] + ) label, _ = _legacy_C_ops.reshape2(label, None, 'shape', [n, 1, -1]) out_shape = [n] + input_shape[2:] - out, total_weight = _legacy_C_ops.nll_loss(input, label, weight, - 'ignore_index', ignore_index, - 'reduction', reduction) + out, total_weight = _legacy_C_ops.nll_loss( + input, + label, + weight, + 'ignore_index', + ignore_index, + 'reduction', + reduction, + ) if input_dims != 2 and input_dims != 4 and reduction == 'none': out, _ = _legacy_C_ops.reshape2(out, None, 'shape', out_shape) return out @@ -1401,10 +1526,9 @@ def nll_loss(input, total_weight = helper.create_variable_for_type_inference(dtype=input.dtype) outputs = {'Out': out, 'Total_weight': total_weight} - helper.append_op(type='nll_loss', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='nll_loss', inputs=inputs, outputs=outputs, attrs=attrs + ) if input_dims != 2 and input_dims != 4 and reduction == 'none': out = reshape(out, shape=out_shape) @@ -1482,13 +1606,15 @@ def kl_div(input, label, reduction='mean', name=None): """ # ugly type promotion - if fluid.data_feeder.convert_dtype( - input.dtype) == 'float32' and fluid.data_feeder.convert_dtype( - label.dtype) == 'float64': + if ( + fluid.data_feeder.convert_dtype(input.dtype) == 'float32' + and fluid.data_feeder.convert_dtype(label.dtype) == 'float64' + ): input = paddle.cast(input, 'float64') - elif fluid.data_feeder.convert_dtype( - input.dtype) == 'float64' and fluid.data_feeder.convert_dtype( - label.dtype) == 'float32': + elif ( + fluid.data_feeder.convert_dtype(input.dtype) == 'float64' + and fluid.data_feeder.convert_dtype(label.dtype) == 'float32' + ): label = paddle.cast(label, 'float64') if in_dygraph_mode(): @@ -1521,13 +1647,12 @@ def kl_div(input, label, reduction='mean', name=None): fluid.data_feeder.check_type(reduction, 'reduction', str, 'kl_div') loss = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='kldiv_loss', - inputs={ - 'X': input, - 'Target': label - }, - outputs={'Loss': loss}, - attrs={'reduction': 'none'}) + helper.append_op( + type='kldiv_loss', + inputs={'X': input, 'Target': label}, + outputs={'Loss': loss}, + attrs={'reduction': 'none'}, + ) if reduction == 'mean': loss = paddle.mean(loss) @@ -1590,31 +1715,38 @@ def mse_loss(input, label, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'mse_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if not in_dynamic_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'mse_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'mse_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'mse_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'mse_loss' + ) if reduction == 'none': return paddle.square(paddle.subtract(input, label), name=name) elif reduction == 'mean': - return paddle.mean(paddle.square(paddle.subtract(input, label)), - name=name) + return paddle.mean( + paddle.square(paddle.subtract(input, label)), name=name + ) else: - return paddle.sum(paddle.square(paddle.subtract(input, label)), - name=name) + return paddle.sum( + paddle.square(paddle.subtract(input, label)), name=name + ) -def ctc_loss(log_probs, - labels, - input_lengths, - label_lengths, - blank=0, - reduction='mean', - norm_by_times=False): +def ctc_loss( + log_probs, + labels, + input_lengths, + label_lengths, + blank=0, + reduction='mean', + norm_by_times=False, +): """ An operator integrating the open source Warp-CTC library (https://github.com/baidu-research/warp-ctc) @@ -1693,8 +1825,9 @@ def ctc_loss(log_probs, """ - loss_out = fluid.layers.warpctc(log_probs, labels, blank, norm_by_times, - input_lengths, label_lengths) + loss_out = fluid.layers.warpctc( + log_probs, labels, blank, norm_by_times, input_lengths, label_lengths + ) loss_out = paddle.squeeze(loss_out, [-1]) assert reduction in ['mean', 'sum', 'none'] @@ -1705,15 +1838,17 @@ def ctc_loss(log_probs, return loss_out -def margin_cross_entropy(logits, - label, - margin1=1.0, - margin2=0.5, - margin3=0.0, - scale=64.0, - group=None, - return_softmax=False, - reduction='mean'): +def margin_cross_entropy( + logits, + label, + margin1=1.0, + margin2=0.5, + margin3=0.0, + scale=64.0, + group=None, + return_softmax=False, + reduction='mean', +): r""" .. math:: @@ -1901,7 +2036,10 @@ def margin_cross_entropy(logits, if not (group == False or group is None or hasattr(group, 'is_member')): raise ValueError( 'Expected group is False, None or instance of paddle.distributed.collective.Group \ - (got group: {})'.format(group)) + (got group: {})'.format( + group + ) + ) return if hasattr(group, 'is_member') and not group.is_member(): @@ -1915,8 +2053,11 @@ def margin_cross_entropy(logits, if core.is_compiled_with_dist(): parallel_env = paddle.distributed.ParallelEnv() global_rank = parallel_env.rank - rank = global_rank if group is None else group.get_group_rank( - global_rank) + rank = ( + global_rank + if group is None + else group.get_group_rank(global_rank) + ) nranks = parallel_env.world_size if group is None else group.nranks input_dims = len(list(logits.shape)) @@ -1924,15 +2065,26 @@ def margin_cross_entropy(logits, if input_dims - 1 != label_dims and input_dims != label_dims: raise ValueError( 'Expected input_dims - 1 = label_dims or input_dims == label_dims\ - (got nput_dims{}, label_dims{})'.format(input_dims, label_dims)) + (got nput_dims{}, label_dims{})'.format( + input_dims, label_dims + ) + ) if input_dims - 1 == label_dims: label = paddle.unsqueeze(label, axis=-1) if in_dygraph_mode(): - softmax, loss = _C_ops.margin_cross_entropy(logits, label, - return_softmax, ring_id, - rank, nranks, margin1, - margin2, margin3, scale) + softmax, loss = _C_ops.margin_cross_entropy( + logits, + label, + return_softmax, + ring_id, + rank, + nranks, + margin1, + margin2, + margin3, + scale, + ) if reduction == 'mean': loss = paddle.mean(loss) elif reduction == 'sum': @@ -1943,9 +2095,25 @@ def margin_cross_entropy(logits, return loss, softmax elif _in_legacy_dygraph(): softmax, loss = _legacy_C_ops.margin_cross_entropy( - logits, label, 'ring_id', ring_id, 'rank', rank, 'nranks', nranks, - 'margin1', margin1, 'margin2', margin2, 'margin3', margin3, 'scale', - scale, 'return_softmax', return_softmax) + logits, + label, + 'ring_id', + ring_id, + 'rank', + rank, + 'nranks', + nranks, + 'margin1', + margin1, + 'margin2', + margin2, + 'margin3', + margin3, + 'scale', + scale, + 'return_softmax', + return_softmax, + ) if reduction == 'mean': loss = paddle.mean(loss) elif reduction == 'sum': @@ -1960,31 +2128,31 @@ def margin_cross_entropy(logits, softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) loss = helper.create_variable_for_type_inference(dtype=logits.dtype) - check_variable_and_dtype(logits, 'logits', - ['float16', 'float32', 'float64'], - 'margin_cross_entropy') - check_variable_and_dtype(label, 'label', ['int32', 'int64'], - 'margin_cross_entropy') - - helper.append_op(type=op_type, - inputs={ - 'Logits': logits, - 'Label': label - }, - outputs={ - 'Softmax': softmax, - 'Loss': loss - }, - attrs={ - 'return_softmax': return_softmax, - 'ring_id': ring_id, - 'rank': rank, - 'nranks': nranks, - 'margin1': margin1, - 'margin2': margin2, - 'margin3': margin3, - 'scale': scale, - }) + check_variable_and_dtype( + logits, + 'logits', + ['float16', 'float32', 'float64'], + 'margin_cross_entropy', + ) + check_variable_and_dtype( + label, 'label', ['int32', 'int64'], 'margin_cross_entropy' + ) + + helper.append_op( + type=op_type, + inputs={'Logits': logits, 'Label': label}, + outputs={'Softmax': softmax, 'Loss': loss}, + attrs={ + 'return_softmax': return_softmax, + 'ring_id': ring_id, + 'rank': rank, + 'nranks': nranks, + 'margin1': margin1, + 'margin2': margin2, + 'margin3': margin3, + 'scale': scale, + }, + ) if reduction == 'mean': loss = paddle.mean(loss) @@ -2001,16 +2169,20 @@ def margin_cross_entropy(logits, since="2.0.0", update_to="paddle.nn.functional.cross_entropy", level=1, - reason= - ('Please notice that behavior of "paddle.nn.functional.softmax_with_cross_entropy" ' - 'and "paddle.nn.functional.cross_entropy" is different.')) -def softmax_with_cross_entropy(logits, - label, - soft_label=False, - ignore_index=-100, - numeric_stable_mode=True, - return_softmax=False, - axis=-1): + reason=( + 'Please notice that behavior of "paddle.nn.functional.softmax_with_cross_entropy" ' + 'and "paddle.nn.functional.cross_entropy" is different.' + ), +) +def softmax_with_cross_entropy( + logits, + label, + soft_label=False, + ignore_index=-100, + numeric_stable_mode=True, + return_softmax=False, + axis=-1, +): r""" This operator implements the cross entropy loss function with softmax. This function combines the calculation of the softmax operation and the cross entropy loss function @@ -2096,20 +2268,28 @@ def softmax_with_cross_entropy(logits, out = paddle.nn.functional.softmax_with_cross_entropy(logits=x, label=label) print(out) """ - return fluid_softmax_with_cross_entropy(logits, label, soft_label, - ignore_index, numeric_stable_mode, - return_softmax, axis) - - -def cross_entropy(input, - label, - weight=None, - ignore_index=-100, - reduction='mean', - soft_label=False, - axis=-1, - use_softmax=True, - name=None): + return fluid_softmax_with_cross_entropy( + logits, + label, + soft_label, + ignore_index, + numeric_stable_mode, + return_softmax, + axis, + ) + + +def cross_entropy( + input, + label, + weight=None, + ignore_index=-100, + reduction='mean', + soft_label=False, + axis=-1, + use_softmax=True, + name=None, +): r""" By default, this operator implements the cross entropy loss function with softmax. This function combines the calculation of the softmax operation and the cross entropy loss function @@ -2355,12 +2535,14 @@ def cross_entropy(input, raise ValueError( "The value of 'reduction' in softmax_cross_entropy" "should be 'sum', 'mean' or 'none', but received %s, which is not allowed." - % reduction) + % reduction + ) if ignore_index > 0 and soft_label == True: raise ValueError( "When soft_label == True, the value of 'ignore_index' in softmax_cross_entropy" - "should be '-100', but received %s, which is not allowed." % - ignore_index) + "should be '-100', but received %s, which is not allowed." + % ignore_index + ) input_dims = len(list(input.shape)) if input_dims == 0: @@ -2370,29 +2552,53 @@ def cross_entropy(input, if input_dims - 1 != label_dims and input_dims != label_dims: raise ValueError( 'Expected nput_dims - 1 = label_dims or input_dims == label_dims\ - (got nput_dims{}, label_dims{})'.format(input_dims, label_dims)) + (got nput_dims{}, label_dims{})'.format( + input_dims, label_dims + ) + ) if input_dims - 1 == label_dims: label = paddle.unsqueeze(label, axis=axis) if in_dygraph_mode(): if soft_label == False: - valid_label = paddle.cast(label != ignore_index, - dtype=label.dtype) * label + valid_label = ( + paddle.cast(label != ignore_index, dtype=label.dtype) * label + ) if core.is_compiled_with_npu() or core.is_compiled_with_mlu(): if soft_label == False: _, _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, valid_label, 'soft_label', soft_label, - 'ignore_index', ignore_index, 'numeric_stable_mode', True, - 'axis', axis, 'use_softmax', use_softmax) + input, + valid_label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) else: _, _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', True, 'axis', axis, - 'use_softmax', use_softmax) + input, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) else: - _, out = _C_ops.cross_entropy_with_softmax(input, label, soft_label, - use_softmax, True, - ignore_index, axis) + _, out = _C_ops.cross_entropy_with_softmax( + input, label, soft_label, use_softmax, True, ignore_index, axis + ) if weight is not None: @@ -2402,11 +2608,12 @@ def cross_entropy(input, # weight's shape is C, where C is class num. # for 1d case: label's shape is [N,C], weight_gather's shape is N. # for 2d case: label's shape is [N,H,W,C], weight_gather's shape is [N,H,W]. - weight_gather = paddle.matmul(x=paddle.cast( - label, weight.dtype), - y=weight, - transpose_x=False, - transpose_y=True) + weight_gather = paddle.matmul( + x=paddle.cast(label, weight.dtype), + y=weight, + transpose_x=False, + transpose_y=True, + ) out_shape = list(out.shape) weight_gather_reshape = reshape(weight_gather, shape=out_shape) out = paddle.cast(out, weight_gather_reshape.dtype) @@ -2417,29 +2624,44 @@ def cross_entropy(input, raise ValueError( "input's class_dimension({}) must equal to " "weight's class_dimension({}) " - "when weight is provided" \ - .format(input.shape[axis], weight.shape[-1])) - - ignore_weight_mask = paddle.cast((label != ignore_index), - out.dtype) - if ignore_weight_mask.ndim > 1 and ignore_weight_mask.shape[ - axis] == 1: + "when weight is provided".format( + input.shape[axis], weight.shape[-1] + ) + ) + + ignore_weight_mask = paddle.cast( + (label != ignore_index), out.dtype + ) + if ( + ignore_weight_mask.ndim > 1 + and ignore_weight_mask.shape[axis] == 1 + ): # TODO: Temporarily use squeeze instead of squeeze_ - ignore_weight_mask = paddle.squeeze(ignore_weight_mask, - axis) + ignore_weight_mask = paddle.squeeze( + ignore_weight_mask, axis + ) if axis != -1 and axis != valid_label.ndim - 1: - temp_perm = list(range(axis % valid_label.ndim)) \ - + list(range((axis % valid_label.ndim + 1), valid_label.ndim)) \ - + [axis % valid_label.ndim] + temp_perm = ( + list(range(axis % valid_label.ndim)) + + list( + range( + (axis % valid_label.ndim + 1), valid_label.ndim + ) + ) + + [axis % valid_label.ndim] + ) weight_gather = _C_ops.gather_nd( - weight, valid_label.transpose(temp_perm)) + weight, valid_label.transpose(temp_perm) + ) else: weight_gather = _C_ops.gather_nd(weight, valid_label) - weight_gather = _C_ops.multiply(weight_gather, - ignore_weight_mask) + weight_gather = _C_ops.multiply( + weight_gather, ignore_weight_mask + ) input_shape = list(label.shape) - weight_gather_reshape = reshape(weight_gather, - shape=input_shape) + weight_gather_reshape = reshape( + weight_gather, shape=input_shape + ) out = paddle.cast(out, weight_gather_reshape.dtype) out = _C_ops.multiply(out, weight_gather_reshape) @@ -2460,22 +2682,24 @@ def cross_entropy(input, # for each label[i],set 1 or 0, according to ignore_index # mask[i]=0, if label[i]==ignore_index # mask[i]=1, otherwise - mask = (label != ignore_index) + mask = label != ignore_index if weight is None: mask = paddle.cast(mask, dtype=out_sum.dtype) count = _C_ops.sum(mask, [], None, False) ret = out_sum / (count + (count == 0.0)) else: mask = paddle.cast(mask, weight_gather_reshape.dtype) - weight_ignored = _C_ops.multiply(mask, - weight_gather_reshape) + weight_ignored = _C_ops.multiply( + mask, weight_gather_reshape + ) weight_sum = _C_ops.sum(weight_ignored, [], None, False) ret = out_sum / (weight_sum + (weight_sum == 0.0)) return ret elif weight is not None: out_sum = _C_ops.sum(out, [], None, False) - total_weight = _C_ops.sum(weight_gather_reshape, [], None, - False) + total_weight = _C_ops.sum( + weight_gather_reshape, [], None, False + ) return out_sum / (total_weight + (total_weight == 0.0)) else: return _C_ops.mean_all(out) @@ -2487,32 +2711,65 @@ def cross_entropy(input, elif _in_legacy_dygraph(): if soft_label == False: - valid_label = paddle.cast(label != ignore_index, - dtype=label.dtype) * label + valid_label = ( + paddle.cast(label != ignore_index, dtype=label.dtype) * label + ) label_min = paddle.min(valid_label) label_max = paddle.max(valid_label) if label_min < 0: - raise ValueError("Target {} is out of lower bound.".format( - label_min.item())) + raise ValueError( + "Target {} is out of lower bound.".format(label_min.item()) + ) if label_max >= input.shape[axis]: - raise ValueError("Target {} is out of upper bound.".format( - label_max.item())) + raise ValueError( + "Target {} is out of upper bound.".format(label_max.item()) + ) if core.is_compiled_with_npu() or core.is_compiled_with_mlu(): if soft_label == False: _, _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, valid_label, 'soft_label', soft_label, - 'ignore_index', ignore_index, 'numeric_stable_mode', True, - 'axis', axis, 'use_softmax', use_softmax) + input, + valid_label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) else: _, _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', True, 'axis', axis, - 'use_softmax', use_softmax) + input, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) else: _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', True, 'axis', axis, - 'use_softmax', use_softmax) + input, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) if weight is not None: @@ -2522,11 +2779,12 @@ def cross_entropy(input, # weight's shape is C, where C is class num. # for 1d case: label's shape is [N,C], weight_gather's shape is N. # for 2d case: label's shape is [N,H,W,C], weight_gather's shape is [N,H,W]. - weight_gather = paddle.matmul(x=paddle.cast( - label, weight.dtype), - y=weight, - transpose_x=False, - transpose_y=True) + weight_gather = paddle.matmul( + x=paddle.cast(label, weight.dtype), + y=weight, + transpose_x=False, + transpose_y=True, + ) out_shape = list(out.shape) weight_gather_reshape = reshape(weight_gather, shape=out_shape) out = paddle.cast(out, weight_gather_reshape.dtype) @@ -2538,29 +2796,44 @@ def cross_entropy(input, raise ValueError( "input's class_dimension({}) must equal to " "weight's class_dimension({}) " - "when weight is provided" \ - .format(input.shape[axis], weight.shape[-1])) - - ignore_weight_mask = paddle.cast((label != ignore_index), - out.dtype) - if ignore_weight_mask.ndim > 1 and ignore_weight_mask.shape[ - axis] == 1: + "when weight is provided".format( + input.shape[axis], weight.shape[-1] + ) + ) + + ignore_weight_mask = paddle.cast( + (label != ignore_index), out.dtype + ) + if ( + ignore_weight_mask.ndim > 1 + and ignore_weight_mask.shape[axis] == 1 + ): # TODO: Temporarily use squeeze instead of squeeze_ - ignore_weight_mask = paddle.squeeze(ignore_weight_mask, - axis) + ignore_weight_mask = paddle.squeeze( + ignore_weight_mask, axis + ) if axis != -1 and axis != valid_label.ndim - 1: - temp_perm = list(range(axis % valid_label.ndim)) \ - + list(range((axis % valid_label.ndim + 1), valid_label.ndim)) \ - + [axis % valid_label.ndim] + temp_perm = ( + list(range(axis % valid_label.ndim)) + + list( + range( + (axis % valid_label.ndim + 1), valid_label.ndim + ) + ) + + [axis % valid_label.ndim] + ) weight_gather = _legacy_C_ops.gather_nd( - weight, valid_label.transpose(temp_perm)) + weight, valid_label.transpose(temp_perm) + ) else: weight_gather = _legacy_C_ops.gather_nd(weight, valid_label) weight_gather = _legacy_C_ops.elementwise_mul( - weight_gather, ignore_weight_mask) + weight_gather, ignore_weight_mask + ) input_shape = list(label.shape) - weight_gather_reshape = reshape(weight_gather, - shape=input_shape) + weight_gather_reshape = reshape( + weight_gather, shape=input_shape + ) out = paddle.cast(out, weight_gather_reshape.dtype) out = _legacy_C_ops.elementwise_mul(out, weight_gather_reshape) @@ -2581,7 +2854,7 @@ def cross_entropy(input, # for each label[i],set 1 or 0, according to ignore_index # mask[i]=0, if label[i]==ignore_index # mask[i]=1, otherwise - mask = (label != ignore_index) + mask = label != ignore_index if weight is None: mask = paddle.cast(mask, dtype=out_sum.dtype) count = _legacy_C_ops.reduce_sum(mask, 'reduce_all', True) @@ -2589,15 +2862,18 @@ def cross_entropy(input, else: mask = paddle.cast(mask, weight_gather_reshape.dtype) weight_ignored = _legacy_C_ops.elementwise_mul( - mask, weight_gather_reshape) + mask, weight_gather_reshape + ) weight_sum = _legacy_C_ops.reduce_sum( - weight_ignored, 'reduce_all', True) + weight_ignored, 'reduce_all', True + ) ret = out_sum / (weight_sum + (weight_sum == 0.0)) return ret elif weight is not None: out_sum = _legacy_C_ops.reduce_sum(out, 'reduce_all', True) - total_weight = _legacy_C_ops.reduce_sum(weight_gather_reshape, - 'reduce_all', True) + total_weight = _legacy_C_ops.reduce_sum( + weight_gather_reshape, 'reduce_all', True + ) return out_sum / (total_weight + (total_weight == 0.0)) else: return _legacy_C_ops.mean(out) @@ -2606,18 +2882,24 @@ def cross_entropy(input, out = paddle.squeeze(out, axis=axis) return out - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'softmax_cross_entropy') check_variable_and_dtype( - label, 'label', + input, + 'input', + ['float16', 'float32', 'float64'], + 'softmax_cross_entropy', + ) + check_variable_and_dtype( + label, + 'label', ['uint8', 'int8', 'int16', 'int32', 'int64', 'float32', 'float64'], - 'softmax_cross_entropy') + 'softmax_cross_entropy', + ) attrs = { 'soft_label': soft_label, 'ignore_index': ignore_index, 'numeric_stable_mode': True, 'axis': axis, - 'use_softmax': use_softmax + 'use_softmax': use_softmax, } helper = LayerHelper('softmax_with_cross_entropy', **locals()) softmax = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -2627,17 +2909,17 @@ def cross_entropy(input, if core.is_compiled_with_npu() or core.is_compiled_with_mlu(): backprop = helper.create_variable_for_type_inference(dtype=input.dtype) outputs['Backprop'] = backprop - helper.append_op(type='softmax_with_cross_entropy', - inputs={ - 'Logits': input, - 'Label': label - }, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='softmax_with_cross_entropy', + inputs={'Logits': input, 'Label': label}, + outputs=outputs, + attrs=attrs, + ) if weight is not None: - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'softmax_cross_entropy') + check_variable_and_dtype( + weight, 'weight', ['float32', 'float64'], 'softmax_cross_entropy' + ) weight_name = name if reduction == 'none' else None if soft_label == True: # chajchaj: @@ -2645,34 +2927,48 @@ def cross_entropy(input, # weight's shape is C, where C is class num. # for 1d case: label's shape is [N,C], weight_gather's shape is N. # for 2d case: label's shape is [N,H,W,C], weight_gather's shape is [N,H,W]. - weight_gather = paddle.matmul(x=paddle.cast(label, weight.dtype), - y=weight, - transpose_x=False, - transpose_y=True) + weight_gather = paddle.matmul( + x=paddle.cast(label, weight.dtype), + y=weight, + transpose_x=False, + transpose_y=True, + ) out_shape = list(out.shape) weight_gather_reshape = reshape(weight_gather, shape=out_shape) out = paddle.cast(out, weight_gather_reshape.dtype) else: if input.shape[axis] != weight.shape[-1]: - raise ValueError("input's class_dimension({}) must equal to " - "weight's class_dimension({}) " - "when weight is provided" \ - .format(input.shape[axis], weight.shape[-1])) + raise ValueError( + "input's class_dimension({}) must equal to " + "weight's class_dimension({}) " + "when weight is provided".format( + input.shape[axis], weight.shape[-1] + ) + ) valid_label = paddle.multiply( - paddle.cast(label != ignore_index, dtype=label.dtype), label) - ignore_weight_mask = paddle.cast((label != ignore_index), - input.dtype) - if ignore_weight_mask.ndim > 1 and ignore_weight_mask.shape[ - axis] == 1: + paddle.cast(label != ignore_index, dtype=label.dtype), label + ) + ignore_weight_mask = paddle.cast( + (label != ignore_index), input.dtype + ) + if ( + ignore_weight_mask.ndim > 1 + and ignore_weight_mask.shape[axis] == 1 + ): ignore_weight_mask = paddle.squeeze(ignore_weight_mask, axis) if axis != -1 and axis != valid_label.ndim - 1: - temp_perm = list(range(axis % valid_label.ndim)) \ - + list(range((axis % valid_label.ndim + 1), valid_label.ndim)) \ - + [axis % valid_label.ndim] + temp_perm = ( + list(range(axis % valid_label.ndim)) + + list( + range((axis % valid_label.ndim + 1), valid_label.ndim) + ) + + [axis % valid_label.ndim] + ) weight_gather = paddle.gather_nd( - weight, paddle.transpose(valid_label, temp_perm)) + weight, paddle.transpose(valid_label, temp_perm) + ) else: weight_gather = paddle.gather_nd(weight, valid_label) weight_gather = paddle.multiply(weight_gather, ignore_weight_mask) @@ -2689,8 +2985,8 @@ def cross_entropy(input, # for each label[i],set 1 or 0, according to ignore_index # mask[i]=0, if label[i]==ignore_index # mask[i]=1, otherwise - mask = (label != ignore_index) - if (weight is None): + mask = label != ignore_index + if weight is None: mask = paddle.cast(mask, dtype=out_sum.dtype) count = paddle.sum(mask, name=name) ret = out_sum / (count + (count == 0.0)) @@ -2714,13 +3010,15 @@ def cross_entropy(input, return out -def sigmoid_focal_loss(logit, - label, - normalizer=None, - alpha=0.25, - gamma=2.0, - reduction='sum', - name=None): +def sigmoid_focal_loss( + logit, + label, + normalizer=None, + alpha=0.25, + gamma=2.0, + reduction='sum', + name=None, +): r""" `Focal Loss `_ is proposed to address the foreground-background class imbalance for classification tasks. It down-weights @@ -2794,37 +3092,49 @@ def sigmoid_focal_loss(logit, raise ValueError( "The value of 'reduction' in sigmoid_focal_loss " "should be 'sum', 'mean' or 'none', but received %s, which is not allowed." - % reduction) + % reduction + ) if normalizer is not None: - check_variable_and_dtype(normalizer, 'normalizer', - ['float32', 'float64'], 'sigmoid_focal_loss') + check_variable_and_dtype( + normalizer, + 'normalizer', + ['float32', 'float64'], + 'sigmoid_focal_loss', + ) normalizer_shape = list(normalizer.shape) normalizer_dims = len(normalizer_shape) if normalizer_dims > 1: raise ValueError( - "Expected one dimension of normalizer in sigmoid_focal_loss but got {}." - .format(normalizer_dims)) + "Expected one dimension of normalizer in sigmoid_focal_loss but got {}.".format( + normalizer_dims + ) + ) if in_dygraph_mode(): place = _current_expected_place() one = _C_ops.full(logit.shape, float(1.0), logit.dtype, place) - loss = _C_ops.sigmoid_cross_entropy_with_logits(logit, label, False, - -100) + loss = _C_ops.sigmoid_cross_entropy_with_logits( + logit, label, False, -100 + ) pred = _C_ops.sigmoid(logit) p_t = _C_ops.add( _C_ops.multiply(pred, label), - _C_ops.multiply(_C_ops.subtract(one, pred), - _C_ops.subtract(one, label))) + _C_ops.multiply( + _C_ops.subtract(one, pred), _C_ops.subtract(one, label) + ), + ) alpha = fluid.dygraph.base.to_variable([alpha], dtype=loss.dtype) alpha_t = _C_ops.add( _C_ops.multiply(alpha, label), - _C_ops.multiply(_C_ops.subtract(one, alpha), - _C_ops.subtract(one, label))) + _C_ops.multiply( + _C_ops.subtract(one, alpha), _C_ops.subtract(one, label) + ), + ) loss = _C_ops.multiply(alpha_t, loss) gamma = fluid.dygraph.base.to_variable([gamma], dtype=loss.dtype) @@ -2843,9 +3153,19 @@ def sigmoid_focal_loss(logit, elif _in_legacy_dygraph(): one = _varbase_creator(dtype=logit.dtype) - _legacy_C_ops.fill_constant(one, 'value', float(1.0), 'force_cpu', - False, 'dtype', one.dtype, 'str_value', - '1.0', 'shape', logit.shape) + _legacy_C_ops.fill_constant( + one, + 'value', + float(1.0), + 'force_cpu', + False, + 'dtype', + one.dtype, + 'str_value', + '1.0', + 'shape', + logit.shape, + ) loss = _legacy_C_ops.sigmoid_cross_entropy_with_logits(logit, label) pred = _legacy_C_ops.sigmoid(logit) @@ -2854,19 +3174,24 @@ def sigmoid_focal_loss(logit, _legacy_C_ops.elementwise_mul(pred, label), _legacy_C_ops.elementwise_mul( _legacy_C_ops.elementwise_sub(one, pred), - _legacy_C_ops.elementwise_sub(one, label))) + _legacy_C_ops.elementwise_sub(one, label), + ), + ) alpha = fluid.dygraph.base.to_variable([alpha], dtype=loss.dtype) alpha_t = _legacy_C_ops.elementwise_add( _legacy_C_ops.elementwise_mul(alpha, label), _legacy_C_ops.elementwise_mul( _legacy_C_ops.elementwise_sub(one, alpha), - _legacy_C_ops.elementwise_sub(one, label))) + _legacy_C_ops.elementwise_sub(one, label), + ), + ) loss = _legacy_C_ops.elementwise_mul(alpha_t, loss) gamma = fluid.dygraph.base.to_variable([gamma], dtype=loss.dtype) gamma_t = _legacy_C_ops.elementwise_pow( - _legacy_C_ops.elementwise_sub(one, p_t), gamma) + _legacy_C_ops.elementwise_sub(one, p_t), gamma + ) loss = _legacy_C_ops.elementwise_mul(gamma_t, loss) if normalizer is not None: @@ -2879,16 +3204,19 @@ def sigmoid_focal_loss(logit, return loss - check_variable_and_dtype(logit, 'logit', ['float32', 'float64'], - 'sigmoid_focal_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'sigmoid_focal_loss') + check_variable_and_dtype( + logit, 'logit', ['float32', 'float64'], 'sigmoid_focal_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'sigmoid_focal_loss' + ) bce_name = None if reduction == 'none' and normalizer is None: bce_name = name loss = paddle.nn.functional.binary_cross_entropy_with_logits( - logit, label, reduction='none', name=bce_name) + logit, label, reduction='none', name=bce_name + ) pred = paddle.nn.functional.sigmoid(logit) p_t = pred * label + (1 - pred) * (1 - label) @@ -2911,11 +3239,9 @@ def sigmoid_focal_loss(logit, return loss -def multi_label_soft_margin_loss(input, - label, - weight=None, - reduction="mean", - name=None): +def multi_label_soft_margin_loss( + input, label, weight=None, reduction="mean", name=None +): r""" Calculate a multi-class multi-classification hinge loss (margin-based loss) between input :math:`x` (a 2D mini-batch `Tensor`) @@ -2973,25 +3299,42 @@ def multi_label_soft_margin_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'multi_label_soft_margin_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if not (input.shape == label.shape): - raise ValueError("The input and label should have same dimension," - "but received {}!={}".format(input.shape, label.shape)) + raise ValueError( + "The input and label should have same dimension," + "but received {}!={}".format(input.shape, label.shape) + ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'multilabel_soft_margin_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'multilabel_soft_margin_loss') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64'], + 'multilabel_soft_margin_loss', + ) + check_variable_and_dtype( + label, + 'label', + ['float32', 'float64'], + 'multilabel_soft_margin_loss', + ) - loss = -(label * paddle.nn.functional.log_sigmoid(input) + - (1 - label) * paddle.nn.functional.log_sigmoid(-input)) + loss = -( + label * paddle.nn.functional.log_sigmoid(input) + + (1 - label) * paddle.nn.functional.log_sigmoid(-input) + ) if weight is not None: if not _non_static_mode(): - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'multilabel_soft_margin_loss') + check_variable_and_dtype( + weight, + 'weight', + ['float32', 'float64'], + 'multilabel_soft_margin_loss', + ) loss = loss * weight loss = loss.mean(axis=-1) # only return N loss values @@ -3081,17 +3424,21 @@ def hinge_embedding_loss(input, label, margin=1.0, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'hinge_embedding_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'hinge_embedding_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'hinge_embedding_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'hinge_embedding_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'hinge_embedding_loss' + ) zero_ = paddle.zeros([1], dtype=input.dtype) - loss = paddle.where(label == 1., input, zero_) + \ - paddle.where(label == -1., paddle.nn.functional.relu(margin - input), zero_) + loss = paddle.where(label == 1.0, input, zero_) + paddle.where( + label == -1.0, paddle.nn.functional.relu(margin - input), zero_ + ) if reduction == 'mean': return paddle.mean(loss, name=name) @@ -3101,12 +3448,9 @@ def hinge_embedding_loss(input, label, margin=1.0, reduction='mean', name=None): return loss -def cosine_embedding_loss(input1, - input2, - label, - margin=0, - reduction='mean', - name=None): +def cosine_embedding_loss( + input1, input2, label, margin=0, reduction='mean', name=None +): r""" This operator computes the cosine embedding loss of Tensor ``input1``, ``input2`` and ``label`` as follows. @@ -3167,12 +3511,14 @@ def cosine_embedding_loss(input1, """ if len(label.shape) != 1: raise ValueError( - "1D target tensor expected, multi-target not supported") + "1D target tensor expected, multi-target not supported" + ) if input1.shape != input2.shape: raise ValueError( "the shape of input tensor 1 should be equal to input tensor 2, but found inputs with " - "different sizes") + "different sizes" + ) if len(input1.shape) > 2: raise ValueError( @@ -3181,9 +3527,13 @@ def cosine_embedding_loss(input1, if input1.dtype not in [paddle.float32, paddle.float64]: raise ValueError( - "The data type of input Variable must be 'float32' or 'float64'") + "The data type of input Variable must be 'float32' or 'float64'" + ) if label.dtype not in [ - paddle.int32, paddle.int64, paddle.float32, paddle.float64 + paddle.int32, + paddle.int64, + paddle.float32, + paddle.float64, ]: raise ValueError( "The data type of label Variable must be 'int32', 'int64', 'float32', 'float64'" @@ -3209,14 +3559,16 @@ def cosine_embedding_loss(input1, return paddle.sum(out, name=name) -def triplet_margin_with_distance_loss(input, - positive, - negative, - distance_function=None, - margin=1.0, - swap=False, - reduction='mean', - name=None): +def triplet_margin_with_distance_loss( + input, + positive, + negative, + distance_function=None, + margin=1.0, + swap=False, + reduction='mean', + name=None, +): r""" Measures the triplet loss given an input tensors :math:`x1`, :math:`x2`, :math:`x3` and a margin with a value greater than :math:`0`. @@ -3291,28 +3643,47 @@ def triplet_margin_with_distance_loss(input, """ if reduction not in ['sum', 'mean', 'none']: - raise ValueError("'reduction' in 'triplet_margin_with_distance_loss' " - "should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + raise ValueError( + "'reduction' in 'triplet_margin_with_distance_loss' " + "should be 'sum', 'mean' or 'none', " + "but received {}.".format(reduction) + ) if margin < 0: raise ValueError( "The margin between positive samples and negative samples should be greater than 0." ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'triplet_margin_with_distance_loss') - check_variable_and_dtype(positive, 'positive', ['float32', 'float64'], - 'triplet_margin_with_distance_loss') - check_variable_and_dtype(negative, 'negative', ['float32', 'float64'], - 'triplet_margin_with_distance_loss') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64'], + 'triplet_margin_with_distance_loss', + ) + check_variable_and_dtype( + positive, + 'positive', + ['float32', 'float64'], + 'triplet_margin_with_distance_loss', + ) + check_variable_and_dtype( + negative, + 'negative', + ['float32', 'float64'], + 'triplet_margin_with_distance_loss', + ) if not (input.shape == positive.shape == negative.shape): - raise ValueError("input's shape must equal to " - "positive's shape and " - "negative's shape") + raise ValueError( + "input's shape must equal to " + "positive's shape and " + "negative's shape" + ) - distance_function = distance_function if distance_function is not None \ + distance_function = ( + distance_function + if distance_function is not None else paddle.nn.PairwiseDistance(2) + ) positive_dist = distance_function(input, positive) negative_dist = distance_function(input, negative) @@ -3324,7 +3695,8 @@ def triplet_margin_with_distance_loss(input, if not paddle.all(positive_dist > 0) or not paddle.all(negative_dist > 0): raise ValueError( "The positive distance or negative distance should be greater than 0, " - "The distance functions should be checked.") + "The distance functions should be checked." + ) loss = paddle.clip(positive_dist - negative_dist + margin, min=0.0) @@ -3336,15 +3708,17 @@ def triplet_margin_with_distance_loss(input, return loss -def triplet_margin_loss(input, - positive, - negative, - margin=1.0, - p=2, - epsilon=1e-6, - swap=False, - reduction='mean', - name=None): +def triplet_margin_loss( + input, + positive, + negative, + margin=1.0, + p=2, + epsilon=1e-6, + swap=False, + reduction='mean', + name=None, +): r""" Measures the triplet loss given an input tensors :math:`x1`, :math:`x2`, :math:`x3` and a margin with a value greater than :math:`0`. @@ -3421,23 +3795,29 @@ def triplet_margin_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'triplet_margin_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if margin < 0: raise ValueError( "The margin between positive samples and negative samples should be greater than 0." ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'triplet_margin_loss') - check_variable_and_dtype(positive, 'positive', ['float32', 'float64'], - 'triplet_margin_loss') - check_variable_and_dtype(negative, 'negative', ['float32', 'float64'], - 'triplet_margin_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'triplet_margin_loss' + ) + check_variable_and_dtype( + positive, 'positive', ['float32', 'float64'], 'triplet_margin_loss' + ) + check_variable_and_dtype( + negative, 'negative', ['float32', 'float64'], 'triplet_margin_loss' + ) if not (input.shape == positive.shape == negative.shape): - raise ValueError("input's shape must equal to " - "positive's shape and " - "negative's shape") + raise ValueError( + "input's shape must equal to " + "positive's shape and " + "negative's shape" + ) distance_function = paddle.nn.PairwiseDistance(p, epsilon=epsilon) positive_dist = distance_function(input, positive) @@ -3457,13 +3837,15 @@ def triplet_margin_loss(input, return loss -def multi_margin_loss(input, - label, - p: int = 1, - margin: float = 1.0, - weight=None, - reduction='mean', - name=None): +def multi_margin_loss( + input, + label, + p: int = 1, + margin: float = 1.0, + weight=None, + reduction='mean', + name=None, +): r""" Measures a multi-class classification hinge loss between input :math:`input` and label :math:`label`: @@ -3526,39 +3908,55 @@ def multi_margin_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'multi_margin_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'multi_margin_loss') - check_variable_and_dtype(label, 'label', ['int32', 'int64'], - 'multi_margin_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'multi_margin_loss' + ) + check_variable_and_dtype( + label, 'label', ['int32', 'int64'], 'multi_margin_loss' + ) if not (input.shape[0] == label.shape[0]): raise ValueError( "The label's shape[0] should be equal to input's shape[0], " "but received input's shape[0] {} and label's shape[0]:{}. ".format( - input.shape[0], label.shape[0])) + input.shape[0], label.shape[0] + ) + ) label = label.reshape((-1, 1)) index_sample = paddle.index_sample(input, label) if weight is not None: if not _non_static_mode(): - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'multi_margin_loss') + check_variable_and_dtype( + weight, 'weight', ['float32', 'float64'], 'multi_margin_loss' + ) if not (input.shape[1] == weight.shape[0]): raise ValueError( "The weight's shape[0] should be equal to input's shape[1]" - "but received weight's shape[0]: {} and input's shape[1]: {}". - format(weight.shape[0], input.shape[1])) + "but received weight's shape[0]: {} and input's shape[1]: {}".format( + weight.shape[0], input.shape[1] + ) + ) weight = paddle.gather(weight, label, axis=0).reshape((-1, 1)) loss = paddle.mean( paddle.pow( - paddle.clip(weight * - (margin - index_sample + input), min=0.0), p), - axis=1) - weight * (margin**p / paddle.shape(input)[1]) + paddle.clip(weight * (margin - index_sample + input), min=0.0), + p, + ), + axis=1, + ) - weight * (margin**p / paddle.shape(input)[1]) else: - loss = paddle.mean(paddle.pow( - paddle.clip(margin - index_sample + input, min=0.0), p), - axis=1) - margin**p / paddle.shape(input)[1] + loss = ( + paddle.mean( + paddle.pow( + paddle.clip(margin - index_sample + input, min=0.0), p + ), + axis=1, + ) + - margin**p / paddle.shape(input)[1] + ) if reduction == 'mean': return paddle.mean(loss, name=name) @@ -3621,20 +4019,23 @@ def soft_margin_loss(input, label, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in soft_margin_loss should be 'sum', " - "'mean' or 'none', but received %s, which is not allowed." % - reduction) + "'mean' or 'none', but received %s, which is not allowed." + % reduction + ) if not _non_static_mode(): - fluid.data_feeder.check_variable_and_dtype(input, 'input', - ['float32', 'float64'], - 'soft_margin_loss') fluid.data_feeder.check_variable_and_dtype( - label, 'label', ['int32', 'int64', 'float32', 'float64'], - 'soft_margin_loss') + input, 'input', ['float32', 'float64'], 'soft_margin_loss' + ) + fluid.data_feeder.check_variable_and_dtype( + label, + 'label', + ['int32', 'int64', 'float32', 'float64'], + 'soft_margin_loss', + ) if not (input.shape == label.shape): - raise ValueError("input's shape must equal to " - "label's shape") + raise ValueError("input's shape must equal to " "label's shape") label = fluid.layers.cast(label, input.dtype) out = paddle.log(1 + paddle.exp(-label * input)) diff --git a/python/paddle/nn/functional/norm.py b/python/paddle/nn/functional/norm.py index d62dcdcfbf4..87d61e91809 100644 --- a/python/paddle/nn/functional/norm.py +++ b/python/paddle/nn/functional/norm.py @@ -83,18 +83,30 @@ def normalize(x, p=2, axis=1, epsilon=1e-12, name=None): if _in_legacy_dygraph(): eps = fluid.dygraph.base.to_variable([epsilon], dtype=x.dtype) - out = _legacy_C_ops.p_norm(x, 'axis', axis, 'porder', float(p), - 'keepdim', True, 'epsilon', epsilon) + out = _legacy_C_ops.p_norm( + x, + 'axis', + axis, + 'porder', + float(p), + 'keepdim', + True, + 'epsilon', + epsilon, + ) return x / _legacy_C_ops.elementwise_max(out, eps) check_type(p, 'p', (float, int), 'normalize') check_type(axis, 'axis', (int), 'normalize') - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'normalize') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'normalize' + ) if len(x.shape) == 1 and axis != 0 and axis != -1: raise ValueError( - "Axis must be 0 or -1 when x is a 1-D tensor, but received axis = {}" - .format(axis)) + "Axis must be 0 or -1 when x is a 1-D tensor, but received axis = {}".format( + axis + ) + ) attrs = { 'axis': axis, @@ -104,26 +116,27 @@ def normalize(x, p=2, axis=1, epsilon=1e-12, name=None): } helper = LayerHelper('p_norm', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='p_norm', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='p_norm', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) eps = out.block.create_var(dtype=out.dtype) eps = paddle.full(shape=[1], fill_value=epsilon, dtype=out.dtype) return paddle.divide(x, paddle.maximum(out, eps), name=name) -def batch_norm(x, - running_mean, - running_var, - weight, - bias, - training=False, - momentum=0.9, - epsilon=1e-05, - data_format="NCHW", - use_global_stats=None, - name=None): +def batch_norm( + x, + running_mean, + running_var, + weight, + bias, + training=False, + momentum=0.9, + epsilon=1e-05, + data_format="NCHW", + use_global_stats=None, + name=None, +): """ Applies Batch Normalization as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . @@ -175,7 +188,8 @@ def batch_norm(x, if data_format not in true_data_format: raise ValueError( "data_format must be one of 'NC', 'NCL', 'NCHW', 'NCDHW', " - "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format)) + "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format) + ) data_format = 'NCHW' if data_format[1] == 'C' else 'NHWC' @@ -187,29 +201,64 @@ def batch_norm(x, if in_dygraph_mode(): batch_norm_out, _, _, _, _, _ = _C_ops.batch_norm( - x, weight, bias, running_mean, running_var, momentum, epsilon, - data_format, not training, use_global_stats, trainable_statistics, - False) - - return dygraph_utils._append_activation_in_dygraph(batch_norm_out, - act=None) + x, + weight, + bias, + running_mean, + running_var, + momentum, + epsilon, + data_format, + not training, + use_global_stats, + trainable_statistics, + False, + ) + + return dygraph_utils._append_activation_in_dygraph( + batch_norm_out, act=None + ) elif _in_legacy_dygraph(): # for dygraph need tuple - attrs = ("momentum", momentum, "epsilon", epsilon, "is_test", - not training, "data_layout", data_format, "use_mkldnn", False, - "fuse_with_relu", False, "use_global_stats", use_global_stats, - "trainable_statistics", trainable_statistics) + attrs = ( + "momentum", + momentum, + "epsilon", + epsilon, + "is_test", + not training, + "data_layout", + data_format, + "use_mkldnn", + False, + "fuse_with_relu", + False, + "use_global_stats", + use_global_stats, + "trainable_statistics", + trainable_statistics, + ) batch_norm_out, _, _, _, _, _ = _legacy_C_ops.batch_norm( - x, weight, bias, running_mean, running_var, None, mean_out, - variance_out, *attrs) - - return dygraph_utils._append_activation_in_dygraph(batch_norm_out, - act=None) - - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'BatchNorm') + x, + weight, + bias, + running_mean, + running_var, + None, + mean_out, + variance_out, + *attrs + ) + + return dygraph_utils._append_activation_in_dygraph( + batch_norm_out, act=None + ) + + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'BatchNorm' + ) # for static need dict attrs = { @@ -228,16 +277,18 @@ def batch_norm(x, "Scale": [weight], "Bias": [bias], "Mean": [running_mean], - "Variance": [running_var] + "Variance": [running_var], } helper = LayerHelper('batch_norm', **locals()) param_dtype = x.dtype if x.dtype != 'float16' else 'float32' - saved_mean = helper.create_variable_for_type_inference(dtype=param_dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=param_dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=param_dtype, stop_gradient=True) + dtype=param_dtype, stop_gradient=True + ) batch_norm_out = helper.create_variable_for_type_inference(x.dtype) outputs = { @@ -245,29 +296,26 @@ def batch_norm(x, "MeanOut": [running_mean], "VarianceOut": [running_var], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } if training or trainable_statistics: # reserve_space is only used for training. reserve_space = helper.create_variable_for_type_inference( - dtype=x.dtype, stop_gradient=True) + dtype=x.dtype, stop_gradient=True + ) outputs["ReserveSpace"] = [reserve_space] - helper.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return helper.append_activation(batch_norm_out) -def layer_norm(x, - normalized_shape, - weight=None, - bias=None, - epsilon=1e-05, - name=None): +def layer_norm( + x, normalized_shape, weight=None, bias=None, epsilon=1e-05, name=None +): """ see more detail in paddle.nn.LayerNorm @@ -304,32 +352,49 @@ def layer_norm(x, normalized_shape = list(normalized_shape) elif not isinstance(normalized_shape, list): raise ValueError( - "`normalized_shape` should be int, list of ints or tuple of ints.") + "`normalized_shape` should be int, list of ints or tuple of ints." + ) normalized_ndim = len(normalized_shape) begin_norm_axis = input_ndim - normalized_ndim - if input_ndim < normalized_ndim or input_shape[ - begin_norm_axis:] != normalized_shape: + if ( + input_ndim < normalized_ndim + or input_shape[begin_norm_axis:] != normalized_shape + ): str_normalized_shape = str(normalized_shape) - raise ValueError('Given normalized_shape is ' + str_normalized_shape + - ', expected input with shape [*, ' + - str_normalized_shape[1:] + ', but got input shape ' + - str(input_shape)) + raise ValueError( + 'Given normalized_shape is ' + + str_normalized_shape + + ', expected input with shape [*, ' + + str_normalized_shape[1:] + + ', but got input shape ' + + str(input_shape) + ) if in_dygraph_mode(): - pre_act, _, _, = _C_ops.layer_norm(x, weight, bias, epsilon, - begin_norm_axis, False) + ( + pre_act, + _, + _, + ) = _C_ops.layer_norm(x, weight, bias, epsilon, begin_norm_axis, False) return dygraph_utils._append_activation_in_dygraph(pre_act, act=None) if _in_legacy_dygraph(): - pre_act, _, _ = _legacy_C_ops.layer_norm(x, weight, bias, 'epsilon', - epsilon, 'begin_norm_axis', - begin_norm_axis) + pre_act, _, _ = _legacy_C_ops.layer_norm( + x, + weight, + bias, + 'epsilon', + epsilon, + 'begin_norm_axis', + begin_norm_axis, + ) return dygraph_utils._append_activation_in_dygraph(pre_act, act=None) - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'LayerNorm') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'LayerNorm' + ) inputs = dict() inputs['X'] = [x] @@ -343,37 +408,40 @@ def layer_norm(x, helper = LayerHelper('layer_norm', **locals()) dtype = x.dtype - mean_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) - variance_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + mean_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) + variance_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) layer_norm_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="layer_norm", - inputs=inputs, - outputs={ - "Y": layer_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis - }) + helper.append_op( + type="layer_norm", + inputs=inputs, + outputs={ + "Y": layer_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}, + ) return helper.append_activation(layer_norm_out) -def instance_norm(x, - running_mean=None, - running_var=None, - weight=None, - bias=None, - use_input_stats=True, - momentum=0.9, - eps=1e-05, - data_format="NCHW", - name=None): +def instance_norm( + x, + running_mean=None, + running_var=None, + weight=None, + bias=None, + use_input_stats=True, + momentum=0.9, + eps=1e-05, + data_format="NCHW", + name=None, +): """ See more detail in nn.layer.InstanceNorm2D. @@ -410,9 +478,17 @@ def instance_norm(x, out = _C_ops.instance_norm(x, weight, bias, eps) return out if _in_legacy_dygraph(): - out, _, _ = _legacy_C_ops.instance_norm(x, weight, bias, "epsilon", eps, - "momentum", momentum, - "data_format", data_format) + out, _, _ = _legacy_C_ops.instance_norm( + x, + weight, + bias, + "epsilon", + eps, + "momentum", + momentum, + "data_format", + data_format, + ) return out check_variable_and_dtype(x, 'input', ['float32', 'float64'], "InstanceNorm") @@ -425,104 +501,106 @@ def instance_norm(x, inputs = {"X": [x]} helper = LayerHelper('instance_norm', **locals()) - saved_mean = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=x.dtype, stop_gradient=True) + dtype=x.dtype, stop_gradient=True + ) instance_norm_out = helper.create_variable_for_type_inference(x.dtype) outputs = { "Y": [instance_norm_out], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } - helper.append_op(type="instance_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="instance_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return instance_norm_out -def local_response_norm(x, - size, - alpha=1e-4, - beta=0.75, - k=1., - data_format="NCHW", - name=None): +def local_response_norm( + x, size, alpha=1e-4, beta=0.75, k=1.0, data_format="NCHW", name=None +): r""" - Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions. - For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks `_ - - The formula is as follows: - - .. math:: - - Output(i, x, y) = Input(i, x, y) / \left(k + \alpha \sum\limits^{\min(C-1, i + size/2)}_{j = \max(0, i - size/2)}(Input(j, x, y))^2\right)^{\beta} - - In the above equation: + Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions. + For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks `_ - - :math:`size` : The number of channels to sum over. - - :math:`k` : The offset (avoid being divided by 0). - - :math:`\\alpha` : The scaling parameter. - - :math:`\\beta` : The exponent parameter. + The formula is as follows: + .. math:: - Args: - x (Tensor): The input 3-D/4-D/5-D tensor. The data type is float32. - size (int): The number of channels to sum over. - alpha (float, optional): The scaling parameter, positive. Default:1e-4 - beta (float, optional): The exponent, positive. Default:0.75 - k (float, optional): An offset, positive. Default: 1.0 - data_format (str, optional): Specify the data format of the input, and the data format of the output - will be consistent with that of the input. An optional string from: - If x is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`, - the data is stored in the order of: `[batch_size, input_channels, feature_length]`. - If x is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`, - the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`. - If x is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`, - the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. - name (str, optional): Name for the operation (optional, default is None). For more information, - please refer to :ref:`api_guide_Name`. + Output(i, x, y) = Input(i, x, y) / \left(k + \alpha \sum\limits^{\min(C-1, i + size/2)}_{j = \max(0, i - size/2)}(Input(j, x, y))^2\right)^{\beta} + + In the above equation: + + - :math:`size` : The number of channels to sum over. + - :math:`k` : The offset (avoid being divided by 0). + - :math:`\\alpha` : The scaling parameter. + - :math:`\\beta` : The exponent parameter. + + + Args: + x (Tensor): The input 3-D/4-D/5-D tensor. The data type is float32. + size (int): The number of channels to sum over. + alpha (float, optional): The scaling parameter, positive. Default:1e-4 + beta (float, optional): The exponent, positive. Default:0.75 + k (float, optional): An offset, positive. Default: 1.0 + data_format (str, optional): Specify the data format of the input, and the data format of the output + will be consistent with that of the input. An optional string from: + If x is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`, + the data is stored in the order of: `[batch_size, input_channels, feature_length]`. + If x is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`, + the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`. + If x is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`, + the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. + name (str, optional): Name for the operation (optional, default is None). For more information, + please refer to :ref:`api_guide_Name`. - Returns: - A tensor storing the transformation result with the same shape and data type as input. + Returns: + A tensor storing the transformation result with the same shape and data type as input. - Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle + import paddle - x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32") - y = paddle.nn.functional.local_response_norm(x, size=5) - print(y.shape) # [3, 3, 112, 112] - """ + x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32") + y = paddle.nn.functional.local_response_norm(x, size=5) + print(y.shape) # [3, 3, 112, 112] + """ if not in_dynamic_mode(): check_variable_and_dtype(x, 'x', ['float32'], 'local_response_norm') if data_format not in ['NCL', 'NLC', 'NCHW', 'NHWC', 'NCDHW', 'NDHWC']: raise ValueError( - "data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " \ - "but got {}".format(data_format)) + "data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " + "but got {}".format(data_format) + ) sizes = x.shape dim = len(sizes) if dim < 3: raise ValueError( - 'Expected 3D or higher dimensionality input, but got {} dimensions'. - format(dim)) + 'Expected 3D or higher dimensionality input, but got {} dimensions'.format( + dim + ) + ) for i, sz in enumerate(sizes): if not sz > 0 and i > 0: - raise ValueError("Expected every dim's size to be larger than 0, " - "but the size of the {}-th dim is {}".format( - i, sz)) + raise ValueError( + "Expected every dim's size to be larger than 0, " + "but the size of the {}-th dim is {}".format(i, sz) + ) channel_last = True if data_format[-1] == "C" else False from functools import reduce + sum_sizes = reduce(lambda x, y: x * y, sizes[1:]) div = paddle.unsqueeze(paddle.multiply(x, x), axis=1) @@ -530,8 +608,11 @@ def local_response_norm(x, pad4d_shape = [0, 0, size // 2, (size - 1) // 2] pool2d_shape = (size, 1) reshape_shape = [ - sizes[0], 1, sizes[1], sizes[2], - int(sum_sizes / (sizes[1] * sizes[2])) + sizes[0], + 1, + sizes[1], + sizes[2], + int(sum_sizes / (sizes[1] * sizes[2])), ] pad5d_shape = [0, 0, 0, 0, size // 2, (size - 1) // 2] pool3d_shape = (size, 1, 1) @@ -539,26 +620,29 @@ def local_response_norm(x, pad4d_shape = [size // 2, (size - 1) // 2, 0, 0] pool2d_shape = (1, size) reshape_shape = [ - sizes[0], 1, sizes[1], - int(sum_sizes / (sizes[1] * sizes[-1])), sizes[-1] + sizes[0], + 1, + sizes[1], + int(sum_sizes / (sizes[1] * sizes[-1])), + sizes[-1], ] pad5d_shape = [size // 2, (size - 1) // 2, 0, 0, 0, 0] pool3d_shape = (1, 1, size) if dim == 3: div = paddle.nn.functional.pad(div, pad=pad4d_shape) - div = paddle.nn.functional.avg_pool2d(div, - kernel_size=pool2d_shape, - stride=1) + div = paddle.nn.functional.avg_pool2d( + div, kernel_size=pool2d_shape, stride=1 + ) div = paddle.squeeze(div, axis=1) else: div = paddle.reshape(div, shape=reshape_shape) - div = paddle.nn.functional.pad(div, - pad=pad5d_shape, - data_format='NCDHW') - div = paddle.nn.functional.avg_pool3d(div, - kernel_size=pool3d_shape, - stride=1) + div = paddle.nn.functional.pad( + div, pad=pad5d_shape, data_format='NCDHW' + ) + div = paddle.nn.functional.avg_pool3d( + div, kernel_size=pool3d_shape, stride=1 + ) div = paddle.reshape(paddle.squeeze(div, axis=1), sizes) div = paddle.scale(div, scale=alpha, bias=k) diff --git a/python/paddle/nn/functional/pooling.py b/python/paddle/nn/functional/pooling.py index 7e465b59748..f9ece56dc7e 100755 --- a/python/paddle/nn/functional/pooling.py +++ b/python/paddle/nn/functional/pooling.py @@ -32,7 +32,9 @@ def _check_input(x, dimension): if len(x.shape) != dimension: raise ValueError( "Excepted Input X is {}-D tensor, but received {}-D {}".format( - dimension, len(x.shape), type(x))) + dimension, len(x.shape), type(x) + ) + ) def _check_instance(x, x_name, types=(int, float)): @@ -40,16 +42,19 @@ def _check_instance(x, x_name, types=(int, float)): if not isinstance(x, types): raise ValueError( "Excepted {} type for {} but received type: {}. ".format( - types, x_name, type(x))) + types, x_name, type(x) + ) + ) def _check_value_limitation(x, x_name, min_limit=1e-3): - def _check_value(x, x_name, min_limit=1e-3): if isinstance(x, int) and min_limit is not None and x < min_limit: raise ValueError( - "Excepted the input {} to be greater than {} but received x: {}. " - .format(x_name, min_limit, x)) + "Excepted the input {} to be greater than {} but received x: {}. ".format( + x_name, min_limit, x + ) + ) for ele in x: _check_value(ele, x_name) @@ -73,21 +78,24 @@ def _channel_last(data_format, num_dims): if data_format not in ['NCL', 'NLC']: raise ValueError( "Attr(data_format) should be 'NCL' or 'NLC'. Received " - "Attr(data_format): %s" % str(data_format)) + "Attr(data_format): %s" % str(data_format) + ) else: return True if data_format == "NLC" else False if num_dims == 2: if data_format not in ['NCHW', 'NHWC']: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s" % str(data_format)) + "Attr(data_format): %s" % str(data_format) + ) else: return True if data_format == "NHWC" else False if num_dims == 3: if data_format not in ['NCDHW', 'NDHWC']: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s" % str(data_format)) + "Attr(data_format): %s" % str(data_format) + ) else: return True if data_format == "NDHWC" else False @@ -97,13 +105,16 @@ def _update_padding_nd(padding, num_dims, channel_last=False, ceil_mode=False): padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '{}'. It can only be 'SAME' or 'VALID'.". - format(padding)) + "Unknown padding: '{}'. It can only be 'SAME' or 'VALID'.".format( + padding + ) + ) if padding == "VALID": if ceil_mode != False: raise ValueError( "When Attr(padding) is \"VALID\", Attr(ceil_mode) must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) padding_algorithm = "VALID" padding = [0] * num_dims @@ -118,10 +129,12 @@ def _update_padding_nd(padding, num_dims, channel_last=False, ceil_mode=False): if not _zero_padding_in_batch_and_channel(padding, channel_last): raise ValueError( "Non-zero padding({}) in the batch or channel dimensions " - "is not supported.".format(padding)) + "is not supported.".format(padding) + ) padding_algorithm = "EXPLICIT" padding = _exclude_padding_in_batch_and_channel( - padding, channel_last) + padding, channel_last + ) if utils._is_symmetric_padding(padding, num_dims): padding = padding[0::2] # for padding like [pad_before, pad_after, pad_before, pad_after, ...] @@ -144,25 +157,29 @@ def _update_padding_nd(padding, num_dims, channel_last=False, ceil_mode=False): def _expand_low_nd_padding(padding): - #1d to 2d fake input + # 1d to 2d fake input if len(padding) == 2: padding = [0] * 2 + padding elif len(padding) == 1: padding = [0] + padding else: raise ValueError( - "The size of padding's dimmention should be 1 or 2. But got padding={}" - .format(padding)) + "The size of padding's dimmention should be 1 or 2. But got padding={}".format( + padding + ) + ) return padding -def avg_pool1d(x, - kernel_size, - stride=None, - padding=0, - exclusive=True, - ceil_mode=False, - name=None): +def avg_pool1d( + x, + kernel_size, + stride=None, + padding=0, + exclusive=True, + ceil_mode=False, + name=None, +): """ This API implements average pooling 1d operation, See more details in :ref:`api_nn_pooling_AvgPool1d` . @@ -221,28 +238,56 @@ def avg_pool1d(x, _check_value_limitation(stride, "stride", min_limit=1e-3) channel_last = _channel_last("NCL", 1) - padding, padding_algorithm = _update_padding_nd(padding, - 1, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 1, channel_last=channel_last, ceil_mode=ceil_mode + ) # use 2d to implenment 1d should expand padding in advance. padding = _expand_low_nd_padding(padding) if in_dygraph_mode(): - output = _C_ops.pool2d(x, kernel_size, stride, padding, ceil_mode, - exclusive, data_format, 'avg', False, False, - padding_algorithm, True) + output = _C_ops.pool2d( + x, + kernel_size, + stride, + padding, + ceil_mode, + exclusive, + data_format, + 'avg', + False, + False, + padding_algorithm, + True, + ) return squeeze(output, [2]) if _in_legacy_dygraph(): - output = _legacy_C_ops.pool2d(x, 'pooling_type', 'avg', 'ksize', - kernel_size, 'global_pooling', False, - 'strides', stride, 'paddings', padding, - 'padding_algorithm', padding_algorithm, - 'use_cudnn', True, 'ceil_mode', ceil_mode, - 'use_mkldnn', False, 'exclusive', - exclusive, 'data_format', data_format) + output = _legacy_C_ops.pool2d( + x, + 'pooling_type', + 'avg', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + exclusive, + 'data_format', + data_format, + ) return squeeze(output, [2]) op_type = 'pool2d' @@ -250,35 +295,39 @@ def avg_pool1d(x, dtype = helper.input_dtype(input_param_name='x') pool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={"X": x}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": 'avg', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": 'avg', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) return squeeze(pool_out, [2]) -def avg_pool2d(x, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - exclusive=True, - divisor_override=None, - data_format="NCHW", - name=None): +def avg_pool2d( + x, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + exclusive=True, + divisor_override=None, + data_format="NCHW", + name=None, +): """ This API implements average pooling 2d operation. See more details in :ref:`api_nn_pooling_AvgPool2d` . @@ -340,23 +389,52 @@ def avg_pool2d(x, _check_value_limitation(stride, "stride", min_limit=1e-3) channel_last = _channel_last(data_format, 2) - padding, padding_algorithm = _update_padding_nd(padding, - 2, - channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 2, channel_last, ceil_mode=ceil_mode + ) if _non_static_mode(): if in_dygraph_mode(): - output = _C_ops.pool2d(x, kernel_size, stride, padding, ceil_mode, - exclusive, data_format, 'avg', False, False, - padding_algorithm, True) + output = _C_ops.pool2d( + x, + kernel_size, + stride, + padding, + ceil_mode, + exclusive, + data_format, + 'avg', + False, + False, + padding_algorithm, + True, + ) else: output = _legacy_C_ops.pool2d( - x, 'pooling_type', 'avg', 'ksize', kernel_size, - 'global_pooling', False, 'padding_algorithm', padding_algorithm, - 'strides', stride, 'paddings', padding, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', - exclusive, 'data_format', data_format) + x, + 'pooling_type', + 'avg', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'strides', + stride, + 'paddings', + padding, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + exclusive, + 'data_format', + data_format, + ) if divisor_override is None: return output else: @@ -369,22 +447,24 @@ def avg_pool2d(x, dtype = helper.input_dtype(input_param_name='x') pool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={"X": x}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": "avg", - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": "avg", + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) if divisor_override is None: return pool_out @@ -393,15 +473,17 @@ def avg_pool2d(x, return pool_out * (kernel_size[0] * kernel_size[1]) / divisor_override -def avg_pool3d(x, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - exclusive=True, - divisor_override=None, - data_format="NCDHW", - name=None): +def avg_pool3d( + x, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + exclusive=True, + divisor_override=None, + data_format="NCDHW", + name=None, +): """ This API implements average pooling 3d operation. See more details in :ref:`api_nn_pooling_AvgPool3d` . @@ -459,25 +541,54 @@ def avg_pool3d(x, stride = utils.convert_to_list(stride, 3, 'pool_stride') channel_last = _channel_last(data_format, 3) - padding, padding_algorithm = _update_padding_nd(padding, - 3, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 3, channel_last=channel_last, ceil_mode=ceil_mode + ) _check_value_limitation(kernel_size, "kernel_size", min_limit=1e-3) _check_value_limitation(stride, "stride", min_limit=1e-3) if in_dygraph_mode(): - pool_out = _C_ops.pool3d(x, kernel_size, stride, padding, ceil_mode, - exclusive, data_format, 'avg', False, False, - padding_algorithm, True) + pool_out = _C_ops.pool3d( + x, + kernel_size, + stride, + padding, + ceil_mode, + exclusive, + data_format, + 'avg', + False, + False, + padding_algorithm, + True, + ) elif _in_legacy_dygraph(): pool_out = _legacy_C_ops.pool3d( - x, 'pooling_type', 'avg', 'ksize', kernel_size, 'strides', stride, - 'paddings', padding, 'global_pooling', False, 'padding_algorithm', - padding_algorithm, 'use_cudnn', True, 'ceil_mode', ceil_mode, - 'use_mkldnn', False, 'exclusive', exclusive, 'data_format', - data_format) + x, + 'pooling_type', + 'avg', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + exclusive, + 'data_format', + data_format, + ) else: op_type = "pool3d" helper = LayerHelper(op_type, **locals()) @@ -486,38 +597,45 @@ def avg_pool3d(x, pool_out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": pool_out} - helper.append_op(type=op_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'avg', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'avg', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) if divisor_override is None: return pool_out else: _check_instance(divisor_override, "divisor_override") - return pool_out * (kernel_size[0] * kernel_size[1] * - kernel_size[2]) / divisor_override + return ( + pool_out + * (kernel_size[0] * kernel_size[1] * kernel_size[2]) + / divisor_override + ) -def max_pool1d(x, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - name=None): +def max_pool1d( + x, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + name=None, +): """ This API implements max pooling 1d opereation. See more details in :ref:`api_nn_pooling_MaxPool1d` . @@ -570,44 +688,96 @@ def max_pool1d(x, else: stride = [1] + utils.convert_to_list(stride, 1, 'pool_stride') - padding, padding_algorithm = _update_padding_nd(padding, - 1, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 1, ceil_mode=ceil_mode + ) # use 2d to implenment 1d should expand padding in advance. padding = _expand_low_nd_padding(padding) if in_dygraph_mode(): if return_mask: - pool_out = _C_ops.max_pool2d_with_index(x, kernel_size, stride, - padding, False, False) - return (squeeze(pool_out[0], [2]), - squeeze(pool_out[1], [2])) if return_mask else squeeze( - pool_out[0], [2]) + pool_out = _C_ops.max_pool2d_with_index( + x, kernel_size, stride, padding, False, False + ) + return ( + (squeeze(pool_out[0], [2]), squeeze(pool_out[1], [2])) + if return_mask + else squeeze(pool_out[0], [2]) + ) else: - pool_out = _C_ops.pool2d(x, kernel_size, stride, padding, ceil_mode, - True, data_format, 'max', False, False, - padding_algorithm, True) + pool_out = _C_ops.pool2d( + x, + kernel_size, + stride, + padding, + ceil_mode, + True, + data_format, + 'max', + False, + False, + padding_algorithm, + True, + ) return squeeze(pool_out, [2]) if _in_legacy_dygraph(): if return_mask: pool_out = _legacy_C_ops.max_pool2d_with_index( - x, 'ksize', kernel_size, 'global_pooling', False, 'strides', - stride, 'paddings', padding, 'padding_algorithm', - padding_algorithm, 'use_cudnn', True, 'ceil_mode', ceil_mode, - 'use_mkldnn', False, 'exclusive', True, 'data_format', - data_format) - return (squeeze(pool_out[0], [2]), - squeeze(pool_out[1], [2])) if return_mask else squeeze( - pool_out[0], [2]) + x, + 'ksize', + kernel_size, + 'global_pooling', + False, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) + return ( + (squeeze(pool_out[0], [2]), squeeze(pool_out[1], [2])) + if return_mask + else squeeze(pool_out[0], [2]) + ) else: pool_out = _legacy_C_ops.pool2d( - x, 'pooling_type', 'max', 'ksize', kernel_size, - 'global_pooling', False, 'padding_algorithm', padding_algorithm, - 'strides', stride, 'paddings', padding, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', True, - 'data_format', data_format) + x, + 'pooling_type', + 'max', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'strides', + stride, + 'paddings', + padding, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return squeeze(pool_out, [2]) op_type = 'max_pool2d_with_index' if return_mask else "pool2d" @@ -617,36 +787,44 @@ def max_pool1d(x, mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=op_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": True, - "data_format": data_format, - }) - - return (squeeze(pool_out, [2]), - squeeze(mask, [2])) if return_mask else squeeze(pool_out, [2]) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": True, + "data_format": data_format, + }, + ) + + return ( + (squeeze(pool_out, [2]), squeeze(mask, [2])) + if return_mask + else squeeze(pool_out, [2]) + ) def _unpool_output_size(x, kernel_size, stride, padding, output_size): - assert output_size is None or isinstance( - output_size, (list, tuple) - ), "Required output_size is None|list|tuple, but received %s" % output_size + assert output_size is None or isinstance(output_size, (list, tuple)), ( + "Required output_size is None|list|tuple, but received %s" % output_size + ) input_size = x.shape default_size = [] for d in range(len(kernel_size)): - default_size.append((input_size[-len(kernel_size) + d] - 1) * - stride[d] + kernel_size[d] - 2 * padding[d]) + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) has_static_var = False if output_size is None: @@ -666,28 +844,33 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): raise ValueError( "output_size should be a sequence containing " "{} or {} elements, but it has a length of '{}'".format( - len(kernel_size), - len(kernel_size) + 2, len(output_size))) + len(kernel_size), len(kernel_size) + 2, len(output_size) + ) + ) if not has_static_var: for d in range(len(kernel_size)): min_size = default_size[d] - stride[d] max_size = default_size[d] + stride[d] if not (min_size < output_size[d] < max_size): raise ValueError( - 'invalid output_size "{}" (dim {} must be between {} and {})' - .format(output_size, d, min_size, max_size)) + 'invalid output_size "{}" (dim {} must be between {} and {})'.format( + output_size, d, min_size, max_size + ) + ) return output_size -def max_unpool1d(x, - indices, - kernel_size, - stride=None, - padding=0, - data_format="NCL", - output_size=None, - name=None): +def max_unpool1d( + x, + indices, + kernel_size, + stride=None, + padding=0, + data_format="NCL", + output_size=None, + name=None, +): r""" This API implements max unpooling 1d opereation. `max_unpool1d` accepts the output of `max_pool1d` as input, @@ -745,8 +928,10 @@ def max_unpool1d(x, """ """NCL to NCHW""" if data_format not in ["NCL"]: - raise ValueError("Attr(data_format) should be 'NCL'. Received " - "Attr(data_format): %s." % str(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCL'. Received " + "Attr(data_format): %s." % str(data_format) + ) data_format = "NCHW" x = unsqueeze(x, [2]) indices = unsqueeze(indices, [2]) @@ -759,18 +944,32 @@ def max_unpool1d(x, # use 2d to implenment 1d should expand padding in advance. padding = _expand_low_nd_padding(padding) - output_size = _unpool_output_size(x, kernel_size, stride, padding, - output_size) + output_size = _unpool_output_size( + x, kernel_size, stride, padding, output_size + ) if in_dygraph_mode(): - output = _C_ops.unpool(x, indices, kernel_size, stride, padding, - output_size, data_format) + output = _C_ops.unpool( + x, indices, kernel_size, stride, padding, output_size, data_format + ) return squeeze(output, [2]) elif in_dynamic_mode(): - output = _legacy_C_ops.unpool(x, indices, 'unpooling_type', 'max', - 'ksize', kernel_size, 'strides', stride, - 'paddings', padding, "output_size", - output_size, "data_format", data_format) + output = _legacy_C_ops.unpool( + x, + indices, + 'unpooling_type', + 'max', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + "output_size", + output_size, + "data_format", + data_format, + ) return squeeze(output, [2]) op_type = "unpool" @@ -778,30 +977,31 @@ def max_unpool1d(x, dtype = helper.input_dtype(input_param_name="x") unpool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={ - "X": x, - "Indices": indices - }, - outputs={"Out": unpool_out}, - attrs={ - "unpooling_type": "max", - "ksize": kernel_size, - "strides": stride, - "paddings": padding, - "output_size": output_size - }) + helper.append_op( + type=op_type, + inputs={"X": x, "Indices": indices}, + outputs={"Out": unpool_out}, + attrs={ + "unpooling_type": "max", + "ksize": kernel_size, + "strides": stride, + "paddings": padding, + "output_size": output_size, + }, + ) return squeeze(unpool_out, [2]) -def max_unpool2d(x, - indices, - kernel_size, - stride=None, - padding=0, - data_format="NCHW", - output_size=None, - name=None): +def max_unpool2d( + x, + indices, + kernel_size, + stride=None, + padding=0, + data_format="NCHW", + output_size=None, + name=None, +): r""" This API implements max unpooling 2d opereation. See more details in :ref:`api_nn_pooling_MaxUnPool2D` . @@ -875,21 +1075,37 @@ def max_unpool2d(x, padding = utils.convert_to_list(padding, 2, 'padding') if data_format not in ["NCHW"]: - raise ValueError("Attr(data_format) should be 'NCHW'. Received " - "Attr(data_format): %s." % str(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCHW'. Received " + "Attr(data_format): %s." % str(data_format) + ) - output_size = _unpool_output_size(x, kernel_size, stride, padding, - output_size) + output_size = _unpool_output_size( + x, kernel_size, stride, padding, output_size + ) if in_dygraph_mode(): - output = _C_ops.unpool(x, indices, kernel_size, stride, padding, - output_size, data_format) + output = _C_ops.unpool( + x, indices, kernel_size, stride, padding, output_size, data_format + ) return output elif in_dynamic_mode(): - output = _legacy_C_ops.unpool(x, indices, 'unpooling_type', 'max', - 'ksize', kernel_size, 'strides', stride, - 'paddings', padding, "output_size", - output_size, "data_format", data_format) + output = _legacy_C_ops.unpool( + x, + indices, + 'unpooling_type', + 'max', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + "output_size", + output_size, + "data_format", + data_format, + ) return output op_type = "unpool" @@ -897,30 +1113,31 @@ def max_unpool2d(x, dtype = helper.input_dtype(input_param_name="x") unpool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={ - "X": x, - "Indices": indices - }, - outputs={"Out": unpool_out}, - attrs={ - "unpooling_type": "max", - "ksize": kernel_size, - "strides": stride, - "paddings": padding, - "output_size": output_size - }) + helper.append_op( + type=op_type, + inputs={"X": x, "Indices": indices}, + outputs={"Out": unpool_out}, + attrs={ + "unpooling_type": "max", + "ksize": kernel_size, + "strides": stride, + "paddings": padding, + "output_size": output_size, + }, + ) return unpool_out -def max_unpool3d(x, - indices, - kernel_size, - stride=None, - padding=0, - data_format="NCDHW", - output_size=None, - name=None): +def max_unpool3d( + x, + indices, + kernel_size, + stride=None, + padding=0, + data_format="NCDHW", + output_size=None, + name=None, +): r""" This API implements max unpooling 3d opereation. `max_unpool3d` accepts the output of `max_pool3d` as input, @@ -992,21 +1209,37 @@ def max_unpool3d(x, padding = utils.convert_to_list(padding, 3, 'padding') if data_format not in ["NCDHW"]: - raise ValueError("Attr(data_format) should be 'NCDHW'. Received " - "Attr(data_format): %s." % str(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCDHW'. Received " + "Attr(data_format): %s." % str(data_format) + ) - output_size = _unpool_output_size(x, kernel_size, stride, padding, - output_size) + output_size = _unpool_output_size( + x, kernel_size, stride, padding, output_size + ) if in_dygraph_mode(): - output = _C_ops.unpool3d(x, indices, kernel_size, stride, padding, - output_size, data_format) + output = _C_ops.unpool3d( + x, indices, kernel_size, stride, padding, output_size, data_format + ) return output elif in_dynamic_mode(): - output = _legacy_C_ops.unpool3d(x, indices, 'unpooling_type', 'max', - 'ksize', kernel_size, 'strides', stride, - 'paddings', padding, "output_size", - output_size, "data_format", data_format) + output = _legacy_C_ops.unpool3d( + x, + indices, + 'unpooling_type', + 'max', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + "output_size", + output_size, + "data_format", + data_format, + ) return output op_type = "unpool3d" @@ -1014,30 +1247,31 @@ def max_unpool3d(x, dtype = helper.input_dtype(input_param_name="x") unpool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={ - "X": x, - "Indices": indices - }, - outputs={"Out": unpool_out}, - attrs={ - "unpooling_type": "max", - "ksize": kernel_size, - "strides": stride, - "paddings": padding, - "output_size": output_size - }) + helper.append_op( + type=op_type, + inputs={"X": x, "Indices": indices}, + outputs={"Out": unpool_out}, + attrs={ + "unpooling_type": "max", + "ksize": kernel_size, + "strides": stride, + "paddings": padding, + "output_size": output_size, + }, + ) return unpool_out -def max_pool2d(x, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NCHW", - name=None): +def max_pool2d( + x, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NCHW", + name=None, +): """ This API implements max pooling 2d operation. See more details in :ref:`api_nn_pooling_MaxPool2d` . @@ -1096,14 +1330,14 @@ def max_pool2d(x, if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) channel_last = True if data_format == "NHWC" else False - padding, padding_algorithm = _update_padding_nd(padding, - num_dims=2, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, num_dims=2, channel_last=channel_last, ceil_mode=ceil_mode + ) if data_format == "NHWC" and return_mask: raise ValueError( @@ -1112,69 +1346,122 @@ def max_pool2d(x, if in_dygraph_mode(): if return_mask: - output = _C_ops.max_pool2d_with_index(x, kernel_size, stride, - padding, False, False) + output = _C_ops.max_pool2d_with_index( + x, kernel_size, stride, padding, False, False + ) return output if return_mask else output[0] else: - return _C_ops.pool2d(x, kernel_size, stride, padding, ceil_mode, - True, data_format, 'max', False, False, - padding_algorithm, True) + return _C_ops.pool2d( + x, + kernel_size, + stride, + padding, + ceil_mode, + True, + data_format, + 'max', + False, + False, + padding_algorithm, + True, + ) if _in_legacy_dygraph(): if return_mask: output = _legacy_C_ops.max_pool2d_with_index( - x, 'ksize', kernel_size, 'global_pooling', False, 'strides', - stride, 'paddings', padding, 'padding_algorithm', - padding_algorithm, 'use_cudnn', True, 'ceil_mode', ceil_mode, - 'use_mkldnn', False, 'exclusive', True, 'data_format', - data_format) + x, + 'ksize', + kernel_size, + 'global_pooling', + False, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return output if return_mask else output[0] else: output = _legacy_C_ops.pool2d( - x, 'pooling_type', 'max', 'ksize', kernel_size, - 'global_pooling', False, 'padding_algorithm', padding_algorithm, - 'strides', stride, 'paddings', padding, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', True, - 'data_format', data_format) + x, + 'pooling_type', + 'max', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'strides', + stride, + 'paddings', + padding, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return output op_type = 'max_pool2d_with_index' if return_mask else "pool2d" helper = LayerHelper(op_type, **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'max_pool2d') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'max_pool2d' + ) dtype = helper.input_dtype(input_param_name='x') pool_out = helper.create_variable_for_type_inference(dtype) mask = helper.create_variable_for_type_inference("int32") outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=op_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": True, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": True, + "data_format": data_format, + }, + ) return (pool_out, mask) if return_mask else pool_out -def max_pool3d(x, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NCDHW", - name=None): +def max_pool3d( + x, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NCDHW", + name=None, +): """ This API implements max pooling 2d operation. See more details in :ref:`api_nn_pooling_MaxPool3d` . @@ -1239,10 +1526,9 @@ def max_pool3d(x, channel_last = _channel_last(data_format, 3) - padding, padding_algorithm = _update_padding_nd(padding, - 3, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 3, channel_last=channel_last, ceil_mode=ceil_mode + ) if data_format == "NDHWC" and return_mask: raise ValueError( @@ -1251,30 +1537,80 @@ def max_pool3d(x, if in_dygraph_mode(): if return_mask: - output = _C_ops.max_pool3d_with_index(x, kernel_size, stride, - padding, False, False) + output = _C_ops.max_pool3d_with_index( + x, kernel_size, stride, padding, False, False + ) return output if return_mask else output[0] else: - return _C_ops.pool3d(x, kernel_size, stride, padding, ceil_mode, - True, data_format, 'max', False, False, - padding_algorithm, True) + return _C_ops.pool3d( + x, + kernel_size, + stride, + padding, + ceil_mode, + True, + data_format, + 'max', + False, + False, + padding_algorithm, + True, + ) if _in_legacy_dygraph(): if return_mask: output = _legacy_C_ops.max_pool3d_with_index( - x, 'pooling_type', 'max', 'ksize', kernel_size, 'strides', - stride, 'paddings', padding, 'global_pooling', False, - 'padding_algorithm', padding_algorithm, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', True, - 'data_format', data_format) + x, + 'pooling_type', + 'max', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return output if return_mask else output[0] else: output = _legacy_C_ops.pool3d( - x, 'pooling_type', 'max', 'ksize', kernel_size, - 'global_pooling', False, 'padding_algorithm', padding_algorithm, - 'strides', stride, 'paddings', padding, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', True, - 'data_format', data_format) + x, + 'pooling_type', + 'max', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'strides', + stride, + 'paddings', + padding, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return output op_type = "max_pool3d_with_index" if return_mask else "pool3d" @@ -1285,22 +1621,24 @@ def max_pool3d(x, mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=op_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": False, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": False, + "data_format": data_format, + }, + ) return (pool_out, mask) if return_mask else pool_out @@ -1344,21 +1682,34 @@ def adaptive_avg_pool1d(x, output_size, name=None): """ pool_type = 'avg' if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'adaptive_pool2d') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'adaptive_pool2d' + ) check_type(output_size, 'pool_size', (int), 'adaptive_pool1d') _check_input(x, 3) pool_size = [1] + utils.convert_to_list(output_size, 1, 'pool_size') x = unsqueeze(x, [2]) if in_dygraph_mode(): - pool_out = _C_ops.pool2d(x, pool_size, [1, 1], [0, 0], False, True, - "NCHW", pool_type, False, True, "EXPLICIT", - False) + pool_out = _C_ops.pool2d( + x, + pool_size, + [1, 1], + [0, 0], + False, + True, + "NCHW", + pool_type, + False, + True, + "EXPLICIT", + False, + ) return squeeze(pool_out, [2]) if _in_legacy_dygraph(): - pool_out = _legacy_C_ops.pool2d(x, 'pooling_type', pool_type, 'ksize', - pool_size, 'adaptive', True) + pool_out = _legacy_C_ops.pool2d( + x, 'pooling_type', pool_type, 'ksize', pool_size, 'adaptive', True + ) return squeeze(pool_out, [2]) l_type = "pool2d" @@ -1368,14 +1719,16 @@ def adaptive_avg_pool1d(x, output_size, name=None): pool_out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": pool_out} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "adaptive": True, + }, + ) return squeeze(pool_out, [2]) @@ -1438,14 +1791,16 @@ def adaptive_avg_pool2d(x, output_size, data_format='NCHW', name=None): # out.shape is [2, 3, 3, 3] """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'adaptive_avg_pool2d') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'adaptive_avg_pool2d' + ) check_type(data_format, 'data_format', str, 'adaptive_avg_pool2d') if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) if data_format == "NCHW": in_h, in_w = x.shape[2:4] @@ -1471,14 +1826,35 @@ def adaptive_avg_pool2d(x, output_size, data_format='NCHW', name=None): output_size = utils._convert_to_tensor_list(output_size) if in_dygraph_mode(): - return _C_ops.pool2d(x, output_size, [1, 1], [0, 0], False, True, - data_format, 'avg', False, True, "EXPLICIT", False) + return _C_ops.pool2d( + x, + output_size, + [1, 1], + [0, 0], + False, + True, + data_format, + 'avg', + False, + True, + "EXPLICIT", + False, + ) if _in_legacy_dygraph(): - return _legacy_C_ops.pool2d(x, 'pooling_type', 'avg', 'ksize', - output_size, 'global_pooling', False, - 'adaptive', True, 'data_format', - data_format) + return _legacy_C_ops.pool2d( + x, + 'pooling_type', + 'avg', + 'ksize', + output_size, + 'global_pooling', + False, + 'adaptive', + True, + 'data_format', + data_format, + ) l_type = 'pool2d' @@ -1488,15 +1864,17 @@ def adaptive_avg_pool2d(x, output_size, data_format='NCHW', name=None): outputs = {"Out": pool_out} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": "avg", - "ksize": output_size, - "adaptive": True, - "data_format": data_format, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": "avg", + "ksize": output_size, + "adaptive": True, + "data_format": data_format, + }, + ) return pool_out @@ -1562,14 +1940,16 @@ def adaptive_avg_pool3d(x, output_size, data_format='NCDHW', name=None): # out.shape is [2, 3, 3, 3, 3] """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'adaptive_avg_pool3d') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'adaptive_avg_pool3d' + ) check_type(data_format, 'data_format', str, 'adaptive_avg_pool3d') if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) if data_format == "NCDHW": in_l, in_h, in_w = x.shape[2:5] @@ -1588,13 +1968,34 @@ def adaptive_avg_pool3d(x, output_size, data_format='NCDHW', name=None): output_size[2] = in_w if in_dygraph_mode(): - return _C_ops.pool3d(x, output_size, [1, 1, 1], [0, 0, 0], False, True, - data_format, 'avg', False, True, "EXPLICIT", False) + return _C_ops.pool3d( + x, + output_size, + [1, 1, 1], + [0, 0, 0], + False, + True, + data_format, + 'avg', + False, + True, + "EXPLICIT", + False, + ) elif _in_legacy_dygraph(): - return _legacy_C_ops.pool3d(x, 'pooling_type', 'avg', 'ksize', - output_size, 'global_pooling', False, - 'adaptive', True, 'data_format', - data_format) + return _legacy_C_ops.pool3d( + x, + 'pooling_type', + 'avg', + 'ksize', + output_size, + 'global_pooling', + False, + 'adaptive', + True, + 'data_format', + data_format, + ) l_type = 'pool3d' @@ -1603,15 +2004,17 @@ def adaptive_avg_pool3d(x, output_size, data_format='NCDHW', name=None): pool_out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": pool_out} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": "avg", - "ksize": output_size, - "adaptive": True, - "data_format": data_format, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": "avg", + "ksize": output_size, + "adaptive": True, + "data_format": data_format, + }, + ) return pool_out @@ -1662,8 +2065,9 @@ def adaptive_max_pool1d(x, output_size, return_mask=False, name=None): """ pool_type = 'max' if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'adaptive_max_pool1d') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'adaptive_max_pool1d' + ) check_type(output_size, 'pool_size', int, 'adaptive_max_pool1d') check_type(return_mask, 'return_mask', bool, 'adaptive_max_pool1d') _check_input(x, 3) @@ -1672,17 +2076,23 @@ def adaptive_max_pool1d(x, output_size, return_mask=False, name=None): x = unsqueeze(x, [2]) if in_dygraph_mode(): - pool_out = _C_ops.max_pool2d_with_index(x, pool_size, [1, 1], [0, 0], - False, True) - return (squeeze(pool_out[0], [2]), squeeze( - pool_out[1], [2])) if return_mask else squeeze(pool_out[0], [2]) + pool_out = _C_ops.max_pool2d_with_index( + x, pool_size, [1, 1], [0, 0], False, True + ) + return ( + (squeeze(pool_out[0], [2]), squeeze(pool_out[1], [2])) + if return_mask + else squeeze(pool_out[0], [2]) + ) if _in_legacy_dygraph(): - pool_out = _legacy_C_ops.max_pool2d_with_index(x, 'pooling_type', - pool_type, 'ksize', - pool_size, 'adaptive', - True) - return (squeeze(pool_out[0], [2]), squeeze( - pool_out[1], [2])) if return_mask else squeeze(pool_out[0], [2]) + pool_out = _legacy_C_ops.max_pool2d_with_index( + x, 'pooling_type', pool_type, 'ksize', pool_size, 'adaptive', True + ) + return ( + (squeeze(pool_out[0], [2]), squeeze(pool_out[1], [2])) + if return_mask + else squeeze(pool_out[0], [2]) + ) l_type = 'max_pool2d_with_index' @@ -1693,64 +2103,70 @@ def adaptive_max_pool1d(x, output_size, return_mask=False, name=None): mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "adaptive": True, + }, + ) - return (squeeze(pool_out, [2]), - squeeze(mask, [2])) if return_mask else squeeze(pool_out, [2]) + return ( + (squeeze(pool_out, [2]), squeeze(mask, [2])) + if return_mask + else squeeze(pool_out, [2]) + ) def adaptive_max_pool2d(x, output_size, return_mask=False, name=None): """ - This operation applies a 2D adaptive max pooling on input tensor. - See more details in :ref:`api_nn_pooling_AdaptiveMaxPool2d` . + This operation applies a 2D adaptive max pooling on input tensor. + See more details in :ref:`api_nn_pooling_AdaptiveMaxPool2d` . - Args: - x (Tensor): The input tensor of adaptive max pool2d operator, which is a 4-D tensor. The data type can be float16, float32, float64, int32 or int64. - output_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, it must contain two elements, (H, W). H and W can be either a int, or None which means the size will be the same as that of the input. - return_mask (bool): If true, the index of max pooling point will be returned along with outputs. Default False. - name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. + Args: + x (Tensor): The input tensor of adaptive max pool2d operator, which is a 4-D tensor. The data type can be float16, float32, float64, int32 or int64. + output_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, it must contain two elements, (H, W). H and W can be either a int, or None which means the size will be the same as that of the input. + return_mask (bool): If true, the index of max pooling point will be returned along with outputs. Default False. + name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. - Returns: - Tensor: The output tensor of adaptive max pool2d result. The data type is same as input tensor. + Returns: + Tensor: The output tensor of adaptive max pool2d result. The data type is same as input tensor. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - # max adaptive pool2d - # suppose input data in the shape of [N, C, H, W], `output_size` is [m, n] - # output shape is [N, C, m, n], adaptive pool divide H and W dimensions - # of input data into m*n grids averagely and performs poolings in each - # grid to get output. - # adaptive max pool performs calculations as follow: - # - # for i in range(m): - # for j in range(n): - # hstart = floor(i * H / m) - # hend = ceil((i + 1) * H / m) - # wstart = floor(i * W / n) - # wend = ceil((i + 1) * W / n) - # output[:, :, i, j] = max(input[:, :, hstart: hend, wstart: wend]) - # - import paddle + # max adaptive pool2d + # suppose input data in the shape of [N, C, H, W], `output_size` is [m, n] + # output shape is [N, C, m, n], adaptive pool divide H and W dimensions + # of input data into m*n grids averagely and performs poolings in each + # grid to get output. + # adaptive max pool performs calculations as follow: + # + # for i in range(m): + # for j in range(n): + # hstart = floor(i * H / m) + # hend = ceil((i + 1) * H / m) + # wstart = floor(i * W / n) + # wend = ceil((i + 1) * W / n) + # output[:, :, i, j] = max(input[:, :, hstart: hend, wstart: wend]) + # + import paddle - input_data = paddle.randn(shape=(2, 3, 32, 32)) - out = paddle.nn.functional.adaptive_max_pool2d( - x = input_data, - output_size=[3, 3]) - # out.shape is [2, 3, 3, 3] + input_data = paddle.randn(shape=(2, 3, 32, 32)) + out = paddle.nn.functional.adaptive_max_pool2d( + x = input_data, + output_size=[3, 3]) + # out.shape is [2, 3, 3, 3] """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'adaptive_max_pool2d') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'adaptive_max_pool2d' + ) check_type(return_mask, 'return_mask', bool, 'adaptive_max_pool2d') - #check_type(output_size, 'pool_size', (int), 'adaptive_max_pool2d') + # check_type(output_size, 'pool_size', (int), 'adaptive_max_pool2d') _check_input(x, 4) in_h, in_w = x.shape[2:4] @@ -1763,13 +2179,14 @@ def adaptive_max_pool2d(x, output_size, return_mask=False, name=None): if output_size[1] == None: output_size[1] = in_w if in_dygraph_mode(): - pool_out = _C_ops.max_pool2d_with_index(x, output_size, [1, 1], [0, 0], - False, True) + pool_out = _C_ops.max_pool2d_with_index( + x, output_size, [1, 1], [0, 0], False, True + ) return pool_out if return_mask else pool_out[0] if _in_legacy_dygraph(): - pool_out = _legacy_C_ops.max_pool2d_with_index(x, 'pooling_type', 'max', - 'ksize', output_size, - 'adaptive', True) + pool_out = _legacy_C_ops.max_pool2d_with_index( + x, 'pooling_type', 'max', 'ksize', output_size, 'adaptive', True + ) return pool_out if return_mask else pool_out[0] l_type = 'max_pool2d_with_index' @@ -1781,67 +2198,70 @@ def adaptive_max_pool2d(x, output_size, return_mask=False, name=None): mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": output_size, - "adaptive": True, - }) - #return (pool_out, mask) if return_mask else pool_out + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": output_size, + "adaptive": True, + }, + ) + # return (pool_out, mask) if return_mask else pool_out return pool_out def adaptive_max_pool3d(x, output_size, return_mask=False, name=None): """ - This operation applies a 3D adaptive max pooling on input tensor. - See more details in :ref:`api_nn_pooling_AdaptiveMaxPool3d` . + This operation applies a 3D adaptive max pooling on input tensor. + See more details in :ref:`api_nn_pooling_AdaptiveMaxPool3d` . - Args: - x (Tensor): The input tensor of adaptive max pool3d operator, which is a 5-D tensor. The data type can be float32, float64. - output_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, it must contain three elements, (D, H, W). D, H and W can be either a int, or None which means the size will be the same as that of the input. - return_mask (bool): If true, the index of max pooling point will be returned along with outputs. Default False. - name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. + Args: + x (Tensor): The input tensor of adaptive max pool3d operator, which is a 5-D tensor. The data type can be float32, float64. + output_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, it must contain three elements, (D, H, W). D, H and W can be either a int, or None which means the size will be the same as that of the input. + return_mask (bool): If true, the index of max pooling point will be returned along with outputs. Default False. + name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. - Returns: - Tensor: The output tensor of adaptive max pool3d result. The data type is same as input tensor. + Returns: + Tensor: The output tensor of adaptive max pool3d result. The data type is same as input tensor. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - # adaptive max pool3d - # suppose input data in the shape of [N, C, D, H, W], `output_size` is [l, m, n] - # output shape is [N, C, l, m, n], adaptive pool divide D, H and W dimensions - # of input data into m*n grids averagely and performs poolings in each - # grid to get output. - # adaptive max pool performs calculations as follow: - # - # for i in range(l): - # for j in range(m): - # for k in range(n): - # dstart = floor(i * D / l) - # dend = ceil((i + 1) * D / l) - # hstart = floor(i * H / m) - # hend = ceil((i + 1) * H / m) - # wstart = floor(i * W / n) - # wend = ceil((i + 1) * W / n) - # output[:, :, i, j, k] = max(input[:, :, dstart: dend, hstart: hend, wstart: wend]) - # - import paddle + # adaptive max pool3d + # suppose input data in the shape of [N, C, D, H, W], `output_size` is [l, m, n] + # output shape is [N, C, l, m, n], adaptive pool divide D, H and W dimensions + # of input data into m*n grids averagely and performs poolings in each + # grid to get output. + # adaptive max pool performs calculations as follow: + # + # for i in range(l): + # for j in range(m): + # for k in range(n): + # dstart = floor(i * D / l) + # dend = ceil((i + 1) * D / l) + # hstart = floor(i * H / m) + # hend = ceil((i + 1) * H / m) + # wstart = floor(i * W / n) + # wend = ceil((i + 1) * W / n) + # output[:, :, i, j, k] = max(input[:, :, dstart: dend, hstart: hend, wstart: wend]) + # + import paddle - input_data = paddle.randn(shape=(2, 3, 8, 32, 32)) - out = paddle.nn.functional.adaptive_max_pool3d( - x = input_data, - output_size=[3, 3, 3]) - # out.shape is [2, 3, 3, 3, 3] + input_data = paddle.randn(shape=(2, 3, 8, 32, 32)) + out = paddle.nn.functional.adaptive_max_pool3d( + x = input_data, + output_size=[3, 3, 3]) + # out.shape is [2, 3, 3, 3, 3] """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'adaptive_max_pool3d') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'adaptive_max_pool3d' + ) check_type(return_mask, 'return_mask', bool, 'adaptive_max_pool3d') - #check_type(output_size, 'pool_size', (int), 'adaptive_max_pool3d') + # check_type(output_size, 'pool_size', (int), 'adaptive_max_pool3d') _check_input(x, 5) in_l, in_h, in_w = x.shape[2:5] @@ -1859,12 +2279,13 @@ def adaptive_max_pool3d(x, output_size, return_mask=False, name=None): if in_dynamic_mode(): if in_dygraph_mode(): # By default, strides is [1,1,1] and paddings is [0, 0, 0] - pool_out = _C_ops.max_pool3d_with_index(x, output_size, [1, 1, 1], - [0, 0, 0], False, True) + pool_out = _C_ops.max_pool3d_with_index( + x, output_size, [1, 1, 1], [0, 0, 0], False, True + ) elif _in_legacy_dygraph(): pool_out = _legacy_C_ops.max_pool3d_with_index( - x, 'pooling_type', 'max', 'ksize', output_size, 'adaptive', - True) + x, 'pooling_type', 'max', 'ksize', output_size, 'adaptive', True + ) return pool_out if return_mask else pool_out[0] l_type = 'max_pool3d_with_index' @@ -1876,13 +2297,15 @@ def adaptive_max_pool3d(x, output_size, return_mask=False, name=None): mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": output_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": output_size, + "adaptive": True, + }, + ) return (pool_out, mask) if return_mask else pool_out diff --git a/python/paddle/nn/functional/sparse_attention.py b/python/paddle/nn/functional/sparse_attention.py index 87df72fe0ef..0dfb545aae8 100644 --- a/python/paddle/nn/functional/sparse_attention.py +++ b/python/paddle/nn/functional/sparse_attention.py @@ -17,14 +17,16 @@ from paddle import _legacy_C_ops from paddle import in_dynamic_mode -def sparse_attention(query, - key, - value, - sparse_csr_offset, - sparse_csr_columns, - key_padding_mask=None, - attn_mask=None, - name=None): +def sparse_attention( + query, + key, + value, + sparse_csr_offset, + sparse_csr_columns, + key_padding_mask=None, + attn_mask=None, + name=None, +): r""" This operator sparsify the Attention matrix in Transformer module to achieve the effect of reducing memory consumption and computation. @@ -141,9 +143,19 @@ def sparse_attention(query, # [1.99830270, 2.99830270]]]] """ if in_dynamic_mode(): - result_attention, result_sdd, result_softmax = _legacy_C_ops.sparse_attention( - query, key, value, sparse_csr_offset, sparse_csr_columns, - key_padding_mask, attn_mask) + ( + result_attention, + result_sdd, + result_softmax, + ) = _legacy_C_ops.sparse_attention( + query, + key, + value, + sparse_csr_offset, + sparse_csr_columns, + key_padding_mask, + attn_mask, + ) return result_attention helper = LayerHelper('sparse_attention', **locals()) @@ -163,7 +175,7 @@ def sparse_attention(query, outputs = { 'Out': out, 'SparseDotSdd': result_sdd, - 'Softmax': result_softmax + 'Softmax': result_softmax, } helper.append_op(type='sparse_attention', inputs=inputs, outputs=outputs) return out diff --git a/python/paddle/nn/functional/vision.py b/python/paddle/nn/functional/vision.py index af1656d6417..bc77dcefa45 100644 --- a/python/paddle/nn/functional/vision.py +++ b/python/paddle/nn/functional/vision.py @@ -82,45 +82,65 @@ def affine_grid(theta, out_shape, align_corners=True, name=None): if theta.shape[1] == 3: use_cudnn = False if is_compiled_with_rocm(): - use_cudnn = False # ROCM platform do not have MIOPEN kernel for affine_grid + use_cudnn = ( + False # ROCM platform do not have MIOPEN kernel for affine_grid + ) if in_dygraph_mode(): - _out_shape = out_shape.numpy().tolist() if isinstance( - out_shape, Variable) else out_shape + _out_shape = ( + out_shape.numpy().tolist() + if isinstance(out_shape, Variable) + else out_shape + ) return _C_ops.affine_grid(theta, _out_shape, use_cudnn, align_corners) elif in_dynamic_mode(): - _out_shape = out_shape.numpy().tolist() if isinstance( - out_shape, Variable) else out_shape - return _legacy_C_ops.affine_grid(theta, "output_shape", _out_shape, - "align_corners", align_corners, - "use_cudnn", use_cudnn) + _out_shape = ( + out_shape.numpy().tolist() + if isinstance(out_shape, Variable) + else out_shape + ) + return _legacy_C_ops.affine_grid( + theta, + "output_shape", + _out_shape, + "align_corners", + align_corners, + "use_cudnn", + use_cudnn, + ) helper = LayerHelper('affine_grid') - check_variable_and_dtype(theta, 'theta', ['float32', 'float64'], - 'affine_grid') + check_variable_and_dtype( + theta, 'theta', ['float32', 'float64'], 'affine_grid' + ) out = helper.create_variable_for_type_inference(theta.dtype) ipts = {'Theta': theta} attrs = {"align_corners": align_corners, "use_cudnn": use_cudnn} if isinstance(out_shape, Variable): ipts['OutputShape'] = out_shape - check_variable_and_dtype(out_shape, 'out_shape', ['int32'], - 'affine_grid') + check_variable_and_dtype( + out_shape, 'out_shape', ['int32'], 'affine_grid' + ) else: attrs['output_shape'] = out_shape - helper.append_op(type='affine_grid', - inputs=ipts, - outputs={'Output': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='affine_grid', + inputs=ipts, + outputs={'Output': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out -def grid_sample(x, - grid, - mode='bilinear', - padding_mode='zeros', - align_corners=True, - name=None): +def grid_sample( + x, + grid, + mode='bilinear', + padding_mode='zeros', + align_corners=True, + name=None, +): """ Sample input X by using bilinear interpolation or nearest interpolation based on flow field grid, which is usually @@ -253,22 +273,33 @@ def grid_sample(x, _padding_modes = ['zeros', 'reflection', 'border'] if mode not in _modes: raise ValueError( - "The mode of grid sample function should be in {}, but got: {}". - format(_modes, mode)) + "The mode of grid sample function should be in {}, but got: {}".format( + _modes, mode + ) + ) if padding_mode not in _padding_modes: raise ValueError( - "The padding mode of grid sample function should be in {}, but got: {}" - .format(_padding_modes, padding_mode)) + "The padding mode of grid sample function should be in {}, but got: {}".format( + _padding_modes, padding_mode + ) + ) if not isinstance(align_corners, bool): - raise ValueError("The align corners should be bool, but got: {}".format( - align_corners)) + raise ValueError( + "The align corners should be bool, but got: {}".format( + align_corners + ) + ) cudnn_version = get_cudnn_version() use_cudnn = False - if not is_compiled_with_rocm() and ( - cudnn_version is not None - ) and align_corners and mode == 'bilinear' and padding_mode == 'zeros': + if ( + not is_compiled_with_rocm() + and (cudnn_version is not None) + and align_corners + and mode == 'bilinear' + and padding_mode == 'zeros' + ): use_cudnn = True # CUDNN always computes gradients for all inputs x.stop_gradient = False @@ -280,26 +311,37 @@ def grid_sample(x, if in_dygraph_mode(): return _C_ops.grid_sample(x, grid, mode, padding_mode, align_corners) elif in_dynamic_mode(): - attrs = ('mode', mode, 'padding_mode', padding_mode, 'align_corners', - align_corners, 'use_cudnn', use_cudnn) + attrs = ( + 'mode', + mode, + 'padding_mode', + padding_mode, + 'align_corners', + align_corners, + 'use_cudnn', + use_cudnn, + ) out = getattr(_legacy_C_ops, 'grid_sampler')(x, grid, *attrs) else: helper = LayerHelper("grid_sample", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'grid_sample') - check_variable_and_dtype(grid, 'grid', ['float32', 'float64'], - 'grid_sample') + check_variable_and_dtype( + grid, 'grid', ['float32', 'float64'], 'grid_sample' + ) ipts = {'X': x, 'Grid': grid} attrs = { 'mode': mode, 'padding_mode': padding_mode, 'align_corners': align_corners, - 'use_cudnn': use_cudnn + 'use_cudnn': use_cudnn, } out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='grid_sampler', - inputs=ipts, - attrs=attrs, - outputs={'Output': out}) + helper.append_op( + type='grid_sampler', + inputs=ipts, + attrs=attrs, + outputs={'Output': out}, + ) return out @@ -336,24 +378,25 @@ def pixel_shuffle(x, upscale_factor, data_format="NCHW", name=None): if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'." - "But recevie Attr(data_format): {} ".format(data_format)) + "But recevie Attr(data_format): {} ".format(data_format) + ) if in_dygraph_mode(): return _C_ops.pixel_shuffle(x, upscale_factor, data_format) if _in_legacy_dygraph(): - return _legacy_C_ops.pixel_shuffle(x, "upscale_factor", upscale_factor, - "data_format", data_format) + return _legacy_C_ops.pixel_shuffle( + x, "upscale_factor", upscale_factor, "data_format", data_format + ) helper = LayerHelper("pixel_shuffle", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'pixel_shuffle') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="pixel_shuffle", - inputs={"X": x}, - outputs={"Out": out}, - attrs={ - "upscale_factor": upscale_factor, - "data_format": data_format - }) + helper.append_op( + type="pixel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"upscale_factor": upscale_factor, "data_format": data_format}, + ) return out @@ -383,8 +426,10 @@ def pixel_unshuffle(x, downscale_factor, data_format="NCHW", name=None): """ if len(x.shape) != 4: raise ValueError( - "Input x should be 4D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 4D tensor, but received x with the shape of {}".format( + x.shape + ) + ) if not isinstance(downscale_factor, int): raise TypeError("Downscale factor must be int type") @@ -395,23 +440,26 @@ def pixel_unshuffle(x, downscale_factor, data_format="NCHW", name=None): if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'." - "But recevie Attr(data_format): {} ".format(data_format)) + "But recevie Attr(data_format): {} ".format(data_format) + ) if _non_static_mode(): - return _legacy_C_ops.pixel_unshuffle(x, "downscale_factor", - downscale_factor, "data_format", - data_format) + return _legacy_C_ops.pixel_unshuffle( + x, "downscale_factor", downscale_factor, "data_format", data_format + ) helper = LayerHelper("pixel_unshuffle", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'pixel_unshuffle') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="pixel_unshuffle", - inputs={"X": x}, - outputs={"Out": out}, - attrs={ - "downscale_factor": downscale_factor, - "data_format": data_format - }) + helper.append_op( + type="pixel_unshuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={ + "downscale_factor": downscale_factor, + "data_format": data_format, + }, + ) return out @@ -452,8 +500,10 @@ def channel_shuffle(x, groups, data_format="NCHW", name=None): """ if len(x.shape) != 4: raise ValueError( - "Input x should be 4D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 4D tensor, but received x with the shape of {}".format( + x.shape + ) + ) if not isinstance(groups, int): raise TypeError("groups must be int type") @@ -464,20 +514,21 @@ def channel_shuffle(x, groups, data_format="NCHW", name=None): if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'." - "But recevie Attr(data_format): {} ".format(data_format)) + "But recevie Attr(data_format): {} ".format(data_format) + ) if _non_static_mode(): - return _legacy_C_ops.channel_shuffle(x, "groups", groups, "data_format", - data_format) + return _legacy_C_ops.channel_shuffle( + x, "groups", groups, "data_format", data_format + ) helper = LayerHelper("channel_shuffle", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'channel_shuffle') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="channel_shuffle", - inputs={"X": x}, - outputs={"Out": out}, - attrs={ - "groups": groups, - "data_format": data_format - }) + helper.append_op( + type="channel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"groups": groups, "data_format": data_format}, + ) return out diff --git a/python/paddle/nn/initializer/__init__.py b/python/paddle/nn/initializer/__init__.py index 530c52bf5f2..e078e19ed2b 100644 --- a/python/paddle/nn/initializer/__init__.py +++ b/python/paddle/nn/initializer/__init__.py @@ -36,8 +36,19 @@ from .orthogonal import Orthogonal # noqa: F401 from .dirac import Dirac # noqa: F401 -__all__ = [ #noqa - 'Bilinear', 'Constant', 'KaimingUniform', 'KaimingNormal', 'XavierNormal', - 'XavierUniform', 'Assign', 'Normal', 'TruncatedNormal', 'Uniform', - 'Orthogonal', 'Dirac', 'set_global_initializer', 'calculate_gain' +__all__ = [ # noqa + 'Bilinear', + 'Constant', + 'KaimingUniform', + 'KaimingNormal', + 'XavierNormal', + 'XavierUniform', + 'Assign', + 'Normal', + 'TruncatedNormal', + 'Uniform', + 'Orthogonal', + 'Dirac', + 'set_global_initializer', + 'calculate_gain', ] diff --git a/python/paddle/nn/initializer/assign.py b/python/paddle/nn/initializer/assign.py index 2cdd5fdf1aa..22560890fe9 100644 --- a/python/paddle/nn/initializer/assign.py +++ b/python/paddle/nn/initializer/assign.py @@ -83,15 +83,19 @@ class Assign(NumpyArrayInitializer): def __init__(self, value, name=None): import numpy - check_type(value, 'value', - (numpy.ndarray, list, tuple, paddle.static.Variable), - 'Assign') - if (isinstance(value, (list, tuple))): + check_type( + value, + 'value', + (numpy.ndarray, list, tuple, paddle.static.Variable), + 'Assign', + ) + + if isinstance(value, (list, tuple)): value = numpy.array(value) # TODO: value is already is a tensor, accounting efficiency maybe it does not need to convert tensor to numpy data and then initialized. - if (isinstance(value, paddle.static.Variable)): + if isinstance(value, paddle.static.Variable): value = value.numpy() super(Assign, self).__init__(value) diff --git a/python/paddle/nn/initializer/dirac.py b/python/paddle/nn/initializer/dirac.py index 74fc285c6bb..f95cd2d8274 100644 --- a/python/paddle/nn/initializer/dirac.py +++ b/python/paddle/nn/initializer/dirac.py @@ -90,7 +90,8 @@ class Dirac(Initializer): def __init__(self, groups=1, name=None): assert groups > 0 and isinstance( - groups, int), " 'groups' must be a positive integer. " + groups, int + ), " 'groups' must be a positive integer. " super(Dirac, self).__init__() self._groups = groups @@ -108,43 +109,49 @@ class Dirac(Initializer): block = self._check_block(block) assert isinstance(var, framework.Parameter) assert isinstance(block, framework.Block) - check_variable_and_dtype(var, "Out", - ['float16', 'bfloat16', 'float32', 'float64'], - 'Dirac') + check_variable_and_dtype( + var, "Out", ['float16', 'bfloat16', 'float32', 'float64'], 'Dirac' + ) assert len(var.shape) in [ - 3, 4, 5 + 3, + 4, + 5, ], "Only Tensor with 3/4/5 dimensions can be initialized by Dirac" assert ( - var.shape[0] % - self._groups) == 0, "Tensor 0-dimension must be divisible by groups" + var.shape[0] % self._groups + ) == 0, "Tensor 0-dimension must be divisible by groups" if var.dtype != VarDesc.VarType.FP32: - out_var = block.create_var(name=unique_name.generate(".".join( - ['dirac', var.name, 'tmp'])), - shape=var.shape, - dtype=VarDesc.VarType.FP32, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate(".".join(['dirac', var.name, 'tmp'])), + shape=var.shape, + dtype=VarDesc.VarType.FP32, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_var = var op = None if framework.in_dygraph_mode(): with fluid.dygraph.no_grad(): place = _current_expected_place() - _C_ops.full_(out_var, out_var.shape, str(float(0)), - out_var.dtype, place) + _C_ops.full_( + out_var, out_var.shape, str(float(0)), out_var.dtype, place + ) else: - block.append_op(type='fill_constant', - inputs={}, - outputs={'Out': out_var}, - attrs={ - 'value': float(0), - 'dtype': out_var.dtype, - 'shape': out_var.shape, - }, - stop_gradient=True) + block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': out_var}, + attrs={ + 'value': float(0), + 'dtype': out_var.dtype, + 'shape': out_var.shape, + }, + stop_gradient=True, + ) origin_shape = var.shape num_per_group = origin_shape[0] // self._groups @@ -162,9 +169,9 @@ class Dirac(Initializer): value_list.append(1.0) offset = 0 for (k, stride) in enumerate(strides): - if (k == 0): + if k == 0: offset += (j + i * num_per_group) * stride - elif (k == 1): + elif k == 1: offset += j * stride else: offset += origin_shape[k] // 2 * stride @@ -174,71 +181,86 @@ class Dirac(Initializer): tmp_out = _C_ops.reshape(out_var, [-1]) tmp_out._share_underline_tensor_to(out_var) else: - x_shape = block.create_var(name=unique_name.generate(".".join( - [out_var.name, "XShape"])), - dtype=out_var.dtype, - shape=out_var.shape, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False, - stop_gradient=True) - block.append_op(type="reshape2", - inputs={"X": out_var}, - attrs={'shape': [-1]}, - outputs={ - "Out": out_var, - "XShape": x_shape - }, - stop_gradient=True) + x_shape = block.create_var( + name=unique_name.generate(".".join([out_var.name, "XShape"])), + dtype=out_var.dtype, + shape=out_var.shape, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type="reshape2", + inputs={"X": out_var}, + attrs={'shape': [-1]}, + outputs={"Out": out_var, "XShape": x_shape}, + stop_gradient=True, + ) index_tensor = block.create_var( name=unique_name.generate('scatter_index'), persistable=False, - stop_gradient=True) + stop_gradient=True, + ) if framework.in_dygraph_mode(): with fluid.dygraph.no_grad(): tmp_tensor = framework._varbase_creator() - _C_ops.assign_value_(tmp_tensor, [len(idx_list)], - VarDesc.VarType.INT64, idx_list, - _current_expected_place()) + _C_ops.assign_value_( + tmp_tensor, + [len(idx_list)], + VarDesc.VarType.INT64, + idx_list, + _current_expected_place(), + ) tmp_tensor._share_underline_tensor_to(index_tensor) else: - block.append_op(type='assign_value', - outputs={'Out': index_tensor}, - attrs={ - 'dtype': VarDesc.VarType.INT64, - 'shape': [len(idx_list)], - 'int64_values': idx_list - }, - stop_gradient=True) + block.append_op( + type='assign_value', + outputs={'Out': index_tensor}, + attrs={ + 'dtype': VarDesc.VarType.INT64, + 'shape': [len(idx_list)], + 'int64_values': idx_list, + }, + stop_gradient=True, + ) value_tensor = block.create_var( name=unique_name.generate('scatter_value'), persistable=False, - stop_gradient=True) + stop_gradient=True, + ) if framework.in_dygraph_mode(): with fluid.dygraph.no_grad(): tmp_tensor = framework._varbase_creator() - _C_ops.assign_value_(tmp_tensor, [len(value_list)], - VarDesc.VarType.FP32, value_list, - _current_expected_place()) + _C_ops.assign_value_( + tmp_tensor, + [len(value_list)], + VarDesc.VarType.FP32, + value_list, + _current_expected_place(), + ) tmp_tensor._share_underline_tensor_to(value_tensor) else: - block.append_op(type='assign_value', - outputs={'Out': value_tensor}, - attrs={ - 'dtype': VarDesc.VarType.FP32, - 'shape': [len(value_list)], - 'fp32_values': value_list - }, - stop_gradient=True) + block.append_op( + type='assign_value', + outputs={'Out': value_tensor}, + attrs={ + 'dtype': VarDesc.VarType.FP32, + 'shape': [len(value_list)], + 'fp32_values': value_list, + }, + stop_gradient=True, + ) if framework.in_dygraph_mode(): with fluid.dygraph.no_grad(): - tmp_out = _C_ops.scatter(out_var, index_tensor, value_tensor, - True) + tmp_out = _C_ops.scatter( + out_var, index_tensor, value_tensor, True + ) tmp_out._share_underline_tensor_to(out_var) tmp_reshape_out = _C_ops.reshape(out_var, origin_shape) tmp_reshape_out._share_underline_tensor_to(out_var) @@ -246,39 +268,40 @@ class Dirac(Initializer): tmp_cast_out = _C_ops.cast(out_var, var.dtype) tmp_cast_out._share_underline_tensor_to(var) else: - op = block.append_op(type="scatter", - inputs={ - "X": out_var, - "Ids": index_tensor, - "Updates": value_tensor - }, - attrs={'overwrite': True}, - outputs={"Out": out_var}, - stop_gradient=True) - x_shape = block.create_var(name=unique_name.generate(".".join( - [out_var.name, "XShape"])), - dtype=out_var.dtype, - shape=out_var.shape, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False, - stop_gradient=True) - block.append_op(type="reshape2", - inputs={"X": out_var}, - attrs={'shape': origin_shape}, - outputs={ - "Out": out_var, - "XShape": x_shape - }, - stop_gradient=True) + op = block.append_op( + type="scatter", + inputs={ + "X": out_var, + "Ids": index_tensor, + "Updates": value_tensor, + }, + attrs={'overwrite': True}, + outputs={"Out": out_var}, + stop_gradient=True, + ) + x_shape = block.create_var( + name=unique_name.generate(".".join([out_var.name, "XShape"])), + dtype=out_var.dtype, + shape=out_var.shape, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type="reshape2", + inputs={"X": out_var}, + attrs={'shape': origin_shape}, + outputs={"Out": out_var, "XShape": x_shape}, + stop_gradient=True, + ) if var.dtype != VarDesc.VarType.FP32: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }, - stop_gradient=True) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + stop_gradient=True, + ) if not in_dynamic_mode(): var.op = op return op diff --git a/python/paddle/nn/initializer/kaiming.py b/python/paddle/nn/initializer/kaiming.py index ab4c90343fe..8449ce925fb 100644 --- a/python/paddle/nn/initializer/kaiming.py +++ b/python/paddle/nn/initializer/kaiming.py @@ -58,11 +58,13 @@ class KaimingNormal(MSRAInitializer): """ def __init__(self, fan_in=None, negative_slope=0.0, nonlinearity='relu'): - super(KaimingNormal, self).__init__(uniform=False, - fan_in=fan_in, - seed=0, - negative_slope=negative_slope, - nonlinearity=nonlinearity) + super(KaimingNormal, self).__init__( + uniform=False, + fan_in=fan_in, + seed=0, + negative_slope=negative_slope, + nonlinearity=nonlinearity, + ) class KaimingUniform(MSRAInitializer): @@ -104,8 +106,10 @@ class KaimingUniform(MSRAInitializer): """ def __init__(self, fan_in=None, negative_slope=0.0, nonlinearity='relu'): - super(KaimingUniform, self).__init__(uniform=True, - fan_in=fan_in, - seed=0, - negative_slope=negative_slope, - nonlinearity=nonlinearity) + super(KaimingUniform, self).__init__( + uniform=True, + fan_in=fan_in, + seed=0, + negative_slope=negative_slope, + nonlinearity=nonlinearity, + ) diff --git a/python/paddle/nn/initializer/orthogonal.py b/python/paddle/nn/initializer/orthogonal.py index 65871ebad0b..84fabba483d 100644 --- a/python/paddle/nn/initializer/orthogonal.py +++ b/python/paddle/nn/initializer/orthogonal.py @@ -85,15 +85,16 @@ class Orthogonal(Initializer): assert isinstance(var, framework.Parameter) assert isinstance(block, framework.Block) # 'qr' op only support float32/float64 now - check_variable_and_dtype(var, "Out", ["float32", "float64"], - "Orthogonal") + check_variable_and_dtype( + var, "Out", ["float32", "float64"], "Orthogonal" + ) self._seed = block.program.random_seed shape = var.shape - assert len( - shape - ) >= 2, "Only Tensor with 2 or more dimensions can be initialized by Orthogonal" + assert ( + len(shape) >= 2 + ), "Only Tensor with 2 or more dimensions can be initialized by Orthogonal" row = shape[0] col = 1 @@ -105,9 +106,9 @@ class Orthogonal(Initializer): if framework.in_dygraph_mode(): with no_grad(): place = framework._current_expected_place() - normal_var = _C_ops.gaussian_random(flatten_shape, 0.0, 1.0, - self._seed, var.dtype, - place) + normal_var = _C_ops.gaussian_random( + flatten_shape, 0.0, 1.0, self._seed, var.dtype, place + ) q, r = _C_ops.qr(normal_var, 'reduced') r_diag = _C_ops.diag(r, 0, 0) @@ -127,107 +128,114 @@ class Orthogonal(Initializer): return None - normal_var = block.create_var(name=unique_name.generate('.'.join( - ['gaussian_random', 'tmp'])), - dtype=var.dtype, - persistable=False, - stop_gradient=True) - block.append_op(type='gaussian_random', - inputs={}, - outputs={'Out': normal_var}, - attrs={ - 'mean': 0.0, - 'std': 1.0, - 'shape': flatten_shape, - 'seed': self._seed, - 'dtype': var.dtype - }, - stop_gradient=True) - - q = block.create_var(name=unique_name.generate('.'.join( - ['qr', 'q', 'tmp'])), - dtype=normal_var.dtype, - persistable=False, - stop_gradient=True) - r = block.create_var(name=unique_name.generate('.'.join( - ['qr', 'r', 'tmp'])), - dtype=normal_var.dtype, - persistable=False, - stop_gradient=True) - block.append_op(type='qr', - inputs={'X': [normal_var]}, - outputs={ - 'Q': q, - 'R': r, - }, - attrs={'mode': 'reduced'}, - stop_gradient=True) - - r_diag = block.create_var(name=unique_name.generate('.'.join( - ['diag', 'tmp'])), - dtype=r.dtype, - persistable=False, - stop_gradient=True) - block.append_op(type='diag_v2', - inputs={'X': r}, - outputs={'Out': r_diag}, - attrs={ - 'offset': 0, - 'padding_value': 0 - }, - stop_gradient=True) + normal_var = block.create_var( + name=unique_name.generate('.'.join(['gaussian_random', 'tmp'])), + dtype=var.dtype, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='gaussian_random', + inputs={}, + outputs={'Out': normal_var}, + attrs={ + 'mean': 0.0, + 'std': 1.0, + 'shape': flatten_shape, + 'seed': self._seed, + 'dtype': var.dtype, + }, + stop_gradient=True, + ) + + q = block.create_var( + name=unique_name.generate('.'.join(['qr', 'q', 'tmp'])), + dtype=normal_var.dtype, + persistable=False, + stop_gradient=True, + ) + r = block.create_var( + name=unique_name.generate('.'.join(['qr', 'r', 'tmp'])), + dtype=normal_var.dtype, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='qr', + inputs={'X': [normal_var]}, + outputs={ + 'Q': q, + 'R': r, + }, + attrs={'mode': 'reduced'}, + stop_gradient=True, + ) + + r_diag = block.create_var( + name=unique_name.generate('.'.join(['diag', 'tmp'])), + dtype=r.dtype, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='diag_v2', + inputs={'X': r}, + outputs={'Out': r_diag}, + attrs={'offset': 0, 'padding_value': 0}, + stop_gradient=True, + ) r_sign = r_diag - block.append_op(type='sign', - inputs={'X': [r_diag]}, - outputs={'Out': r_sign}, - stop_gradient=True) - - block.append_op(type='elementwise_mul', - inputs={ - 'X': q, - 'Y': r_sign - }, - outputs={'Out': q}, - attrs={}, - stop_gradient=True) - - x_shape = block.create_var(name=unique_name.generate('.'.join( - ['transpose', 'shape', 'tmp'])), - dtype=q.dtype, - persistable=False, - stop_gradient=True) + block.append_op( + type='sign', + inputs={'X': [r_diag]}, + outputs={'Out': r_sign}, + stop_gradient=True, + ) + + block.append_op( + type='elementwise_mul', + inputs={'X': q, 'Y': r_sign}, + outputs={'Out': q}, + attrs={}, + stop_gradient=True, + ) + + x_shape = block.create_var( + name=unique_name.generate('.'.join(['transpose', 'shape', 'tmp'])), + dtype=q.dtype, + persistable=False, + stop_gradient=True, + ) if row < col: - q_transpose = block.create_var(name=unique_name.generate('.'.join( - ['transpose', 'tmp'])), - dtype=q.dtype, - persistable=False, - stop_gradient=True) - block.append_op(type='transpose2', - inputs={'X': q}, - outputs={ - 'Out': q_transpose, - 'XShape': x_shape - }, - attrs={'axis': [1, 0]}, - stop_gradient=True) + q_transpose = block.create_var( + name=unique_name.generate('.'.join(['transpose', 'tmp'])), + dtype=q.dtype, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='transpose2', + inputs={'X': q}, + outputs={'Out': q_transpose, 'XShape': x_shape}, + attrs={'axis': [1, 0]}, + stop_gradient=True, + ) q = q_transpose - block.append_op(type='reshape2', - inputs={'X': q}, - outputs={ - 'Out': q, - "XShape": x_shape - }, - attrs={'shape': var.shape}, - stop_gradient=True) - - op = block.append_op(type='scale', - inputs={'X': q}, - outputs={'Out': var}, - attrs={ - 'scale': self._gain, - 'bias': 0.0 - }) + block.append_op( + type='reshape2', + inputs={'X': q}, + outputs={'Out': q, "XShape": x_shape}, + attrs={'shape': var.shape}, + stop_gradient=True, + ) + + op = block.append_op( + type='scale', + inputs={'X': q}, + outputs={'Out': var}, + attrs={'scale': self._gain, 'bias': 0.0}, + ) return op diff --git a/python/paddle/nn/initializer/uniform.py b/python/paddle/nn/initializer/uniform.py index 2c7a57b1195..578a4772478 100644 --- a/python/paddle/nn/initializer/uniform.py +++ b/python/paddle/nn/initializer/uniform.py @@ -55,9 +55,6 @@ class Uniform(UniformInitializer): assert low is not None, 'low should not be None' assert high is not None, 'high should not be None' assert high >= low, 'high should greater or equal than low' - super(Uniform, self).__init__(low=low, - high=high, - seed=0, - diag_num=0, - diag_step=0, - diag_val=1.0) + super(Uniform, self).__init__( + low=low, high=high, seed=0, diag_num=0, diag_step=0, diag_val=1.0 + ) diff --git a/python/paddle/nn/initializer/xavier.py b/python/paddle/nn/initializer/xavier.py index 2c6d60a6bb8..149b13df389 100644 --- a/python/paddle/nn/initializer/xavier.py +++ b/python/paddle/nn/initializer/xavier.py @@ -63,10 +63,9 @@ class XavierNormal(XavierInitializer): """ def __init__(self, fan_in=None, fan_out=None, name=None): - super(XavierNormal, self).__init__(uniform=False, - fan_in=fan_in, - fan_out=fan_out, - seed=0) + super(XavierNormal, self).__init__( + uniform=False, fan_in=fan_in, fan_out=fan_out, seed=0 + ) class XavierUniform(XavierInitializer): @@ -118,7 +117,6 @@ class XavierUniform(XavierInitializer): """ def __init__(self, fan_in=None, fan_out=None, name=None): - super(XavierUniform, self).__init__(uniform=True, - fan_in=fan_in, - fan_out=fan_out, - seed=0) + super(XavierUniform, self).__init__( + uniform=True, fan_in=fan_in, fan_out=fan_out, seed=0 + ) diff --git a/python/paddle/nn/layer/activation.py b/python/paddle/nn/layer/activation.py index 2382acbd0ca..e3fa71c0332 100644 --- a/python/paddle/nn/layer/activation.py +++ b/python/paddle/nn/layer/activation.py @@ -402,12 +402,14 @@ class PReLU(Layer): # [ 6. , 7. , 8. , 9. ]]]] """ - def __init__(self, - num_parameters=1, - init=0.25, - weight_attr=None, - data_format="NCHW", - name=None): + def __init__( + self, + num_parameters=1, + init=0.25, + weight_attr=None, + data_format="NCHW", + name=None, + ): super(PReLU, self).__init__() self._num_parameters = num_parameters self._init = init @@ -415,12 +417,13 @@ class PReLU(Layer): self._name = name self._data_format = data_format - self._weight = self.create_parameter(attr=self._weight_attr, - shape=[self._num_parameters], - dtype=get_default_dtype(), - is_bias=False, - default_initializer=Constant( - self._init)) + self._weight = self.create_parameter( + attr=self._weight_attr, + shape=[self._num_parameters], + dtype=get_default_dtype(), + is_bias=False, + default_initializer=Constant(self._init), + ) def forward(self, x): return F.prelu(x, self._weight, data_format=self._data_format) @@ -428,8 +431,12 @@ class PReLU(Layer): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'num_parameters={}, data_format={}, init={}, dtype={}{}'.format( - self._num_parameters, self._data_format, self._init, self._dtype, - name_str) + self._num_parameters, + self._data_format, + self._init, + self._dtype, + name_str, + ) class RReLU(Layer): @@ -503,22 +510,22 @@ class RReLU(Layer): # [ 6. 7. 8. 9. ]]]] """ - def __init__(self, lower=1. / 8., upper=1. / 3., name=None): + def __init__(self, lower=1.0 / 8.0, upper=1.0 / 3.0, name=None): super(RReLU, self).__init__() self._lower = lower self._upper = upper self._name = name def forward(self, x): - return F.rrelu(x, - lower=self._lower, - upper=self._upper, - training=self.training) + return F.rrelu( + x, lower=self._lower, upper=self._upper, training=self.training + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'lower={}, upper={}, training={}, dtype={}{}'.format( - self._lower, self._upper, self.training, self._dtype, name_str) + self._lower, self._upper, self.training, self._dtype, name_str + ) class ReLU(Layer): @@ -637,10 +644,12 @@ class SELU(Layer): # [[0, 1.050701],[2.101402, 3.152103]] """ - def __init__(self, - scale=1.0507009873554804934193349852946, - alpha=1.6732632423543772848170429916717, - name=None): + def __init__( + self, + scale=1.0507009873554804934193349852946, + alpha=1.6732632423543772848170429916717, + name=None, + ): super(SELU, self).__init__() self._scale = scale self._alpha = alpha @@ -651,8 +660,9 @@ class SELU(Layer): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' - return 'scale={:.16f}, alpha={:.16f}{}'.format(self._scale, self._alpha, - name_str) + return 'scale={:.16f}, alpha={:.16f}{}'.format( + self._scale, self._alpha, name_str + ) class LeakyReLU(Layer): @@ -835,8 +845,9 @@ class Softplus(Layer): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' - return 'beta={}, threshold={}{}'.format(self._beta, self._threshold, - name_str) + return 'beta={}, threshold={}{}'.format( + self._beta, self._threshold, name_str + ) class Softshrink(Layer): @@ -1469,8 +1480,11 @@ class Softmax2D(Layer): self._name = name def forward(self, x): - assert x.ndim == 3 or x.ndim == 4, "Softmax2D requires a 3D or 4D tensor as input. Received: {}D.".format( - x.ndim) + assert ( + x.ndim == 3 or x.ndim == 4 + ), "Softmax2D requires a 3D or 4D tensor as input. Received: {}D.".format( + x.ndim + ) return F.softmax(x, axis=-3, dtype=self._dtype, name=self._name) def extra_repr(self): diff --git a/python/paddle/nn/layer/common.py b/python/paddle/nn/layer/common.py index 6cc2056459f..215f395e1fb 100644 --- a/python/paddle/nn/layer/common.py +++ b/python/paddle/nn/layer/common.py @@ -144,37 +144,43 @@ class Linear(Layer): # [2.1077576 2.1077576 2.1077576 2.1077576 ]] """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - bias_attr=None, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + bias_attr=None, + name=None, + ): super(Linear, self).__init__() self._dtype = self._helper.get_default_dtype() self._weight_attr = weight_attr self._bias_attr = bias_attr - self.weight = self.create_parameter(shape=[in_features, out_features], - attr=self._weight_attr, - dtype=self._dtype, - is_bias=False) - self.bias = self.create_parameter(shape=[out_features], - attr=self._bias_attr, - dtype=self._dtype, - is_bias=True) + self.weight = self.create_parameter( + shape=[in_features, out_features], + attr=self._weight_attr, + dtype=self._dtype, + is_bias=False, + ) + self.bias = self.create_parameter( + shape=[out_features], + attr=self._bias_attr, + dtype=self._dtype, + is_bias=True, + ) self.name = name def forward(self, input): - out = F.linear(x=input, - weight=self.weight, - bias=self.bias, - name=self.name) + out = F.linear( + x=input, weight=self.weight, bias=self.bias, name=self.name + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'in_features={}, out_features={}, dtype={}{}'.format( - self.weight.shape[0], self.weight.shape[1], self._dtype, name_str) + self.weight.shape[0], self.weight.shape[1], self._dtype, name_str + ) class Upsample(Layer): @@ -371,14 +377,16 @@ class Upsample(Layer): """ - def __init__(self, - size=None, - scale_factor=None, - mode='nearest', - align_corners=False, - align_mode=0, - data_format='NCHW', - name=None): + def __init__( + self, + size=None, + scale_factor=None, + mode='nearest', + align_corners=False, + align_mode=0, + data_format='NCHW', + name=None, + ): super(Upsample, self).__init__() self.size = size self.scale_factor = scale_factor @@ -389,14 +397,16 @@ class Upsample(Layer): self.name = name def forward(self, x): - out = F.interpolate(x, - size=self.size, - scale_factor=self.scale_factor, - mode=self.mode, - align_corners=self.align_corners, - align_mode=self.align_mode, - data_format=self.data_format, - name=self.name) + out = F.interpolate( + x, + size=self.size, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners, + align_mode=self.align_mode, + data_format=self.data_format, + name=self.name, + ) return out @@ -407,8 +417,13 @@ class Upsample(Layer): main_str = 'size={}'.format(self.size) name_str = ', name={}'.format(self.name) if self.name else '' return '{}, mode={}, align_corners={}, align_mode={}, data_format={}{}'.format( - main_str, self.mode, self.align_corners, self.align_mode, - self.data_format, name_str) + main_str, + self.mode, + self.align_corners, + self.align_mode, + self.data_format, + name_str, + ) class UpsamplingNearest2D(Layer): @@ -462,11 +477,9 @@ class UpsamplingNearest2D(Layer): # [2L, 3L, 12L, 12L] """ - def __init__(self, - size=None, - scale_factor=None, - data_format='NCHW', - name=None): + def __init__( + self, size=None, scale_factor=None, data_format='NCHW', name=None + ): super(UpsamplingNearest2D, self).__init__() self.size = size self.scale_factor = scale_factor @@ -474,14 +487,16 @@ class UpsamplingNearest2D(Layer): self.name = name def forward(self, x): - out = F.interpolate(x, - size=self.size, - scale_factor=self.scale_factor, - mode='nearest', - align_corners=False, - align_mode=0, - data_format=self.data_format, - name=self.name) + out = F.interpolate( + x, + size=self.size, + scale_factor=self.scale_factor, + mode='nearest', + align_corners=False, + align_mode=0, + data_format=self.data_format, + name=self.name, + ) return out @@ -491,8 +506,9 @@ class UpsamplingNearest2D(Layer): else: main_str = 'size={}'.format(self.size) name_str = ', name={}'.format(self.name) if self.name else '' - return '{}, data_format={}{}'.format(main_str, self.data_format, - name_str) + return '{}, data_format={}{}'.format( + main_str, self.data_format, name_str + ) class UpsamplingBilinear2D(Layer): @@ -547,11 +563,9 @@ class UpsamplingBilinear2D(Layer): # [2L, 3L, 12L, 12L] """ - def __init__(self, - size=None, - scale_factor=None, - data_format='NCHW', - name=None): + def __init__( + self, size=None, scale_factor=None, data_format='NCHW', name=None + ): super(UpsamplingBilinear2D, self).__init__() self.size = size self.scale_factor = scale_factor @@ -559,14 +573,16 @@ class UpsamplingBilinear2D(Layer): self.name = name def forward(self, x): - out = F.interpolate(x, - size=self.size, - scale_factor=self.scale_factor, - mode='bilinear', - align_corners=True, - align_mode=0, - data_format=self.data_format, - name=self.name) + out = F.interpolate( + x, + size=self.size, + scale_factor=self.scale_factor, + mode='bilinear', + align_corners=True, + align_mode=0, + data_format=self.data_format, + name=self.name, + ) return out @@ -576,8 +592,9 @@ class UpsamplingBilinear2D(Layer): else: main_str = 'size={}'.format(self.size) name_str = ', name={}'.format(self.name) if self.name else '' - return '{}, data_format={}{}'.format(main_str, self.data_format, - name_str) + return '{}, data_format={}{}'.format( + main_str, self.data_format, name_str + ) class Bilinear(Layer): @@ -634,13 +651,15 @@ class Bilinear(Layer): """ - def __init__(self, - in1_features, - in2_features, - out_features, - weight_attr=None, - bias_attr=None, - name=None): + def __init__( + self, + in1_features, + in2_features, + out_features, + weight_attr=None, + bias_attr=None, + name=None, + ): super(Bilinear, self).__init__() self._weight_attr = weight_attr self._bias_attr = bias_attr @@ -651,17 +670,23 @@ class Bilinear(Layer): self._dtype = self._helper.get_default_dtype() weight_shape = [ - self._out_features, self._in1_features, self._in2_features + self._out_features, + self._in1_features, + self._in2_features, ] - self.weight = self.create_parameter(attr=self._weight_attr, - shape=weight_shape, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=weight_shape, + dtype=self._dtype, + is_bias=False, + ) bias_shape = [1, self._out_features] - self.bias = self.create_parameter(attr=self._bias_attr, - shape=bias_shape, - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=bias_shape, + dtype=self._dtype, + is_bias=True, + ) def forward(self, x1, x2): return F.bilinear(x1, x2, self.weight, self.bias, self._name) @@ -669,8 +694,12 @@ class Bilinear(Layer): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'in1_features={}, in2_features={}, out_features={}, dtype={}{}'.format( - self._in1_features, self._in2_features, self._out_features, - self._dtype, name_str) + self._in1_features, + self._in2_features, + self._out_features, + self._dtype, + name_str, + ) class Dropout(Layer): @@ -721,7 +750,7 @@ class Dropout(Layer): print(x) print(y_train) print(y_test) - """ + """ def __init__(self, p=0.5, axis=None, mode="upscale_in_train", name=None): super(Dropout, self).__init__() @@ -732,18 +761,21 @@ class Dropout(Layer): self.name = name def forward(self, input): - out = F.dropout(input, - p=self.p, - axis=self.axis, - training=self.training, - mode=self.mode, - name=self.name) + out = F.dropout( + input, + p=self.p, + axis=self.axis, + training=self.training, + mode=self.mode, + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'p={}, axis={}, mode={}{}'.format(self.p, self.axis, self.mode, - name_str) + return 'p={}, axis={}, mode={}{}'.format( + self.p, self.axis, self.mode, name_str + ) class Dropout2D(Layer): @@ -783,7 +815,7 @@ class Dropout2D(Layer): print(x) print(y_train) print(y_test) - """ + """ def __init__(self, p=0.5, data_format='NCHW', name=None): super(Dropout2D, self).__init__() @@ -793,17 +825,20 @@ class Dropout2D(Layer): self.name = name def forward(self, input): - out = F.dropout2d(input, - p=self.p, - training=self.training, - data_format=self.data_format, - name=self.name) + out = F.dropout2d( + input, + p=self.p, + training=self.training, + data_format=self.data_format, + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'p={}, data_format={}{}'.format(self.p, self.data_format, - name_str) + return 'p={}, data_format={}{}'.format( + self.p, self.data_format, name_str + ) class Dropout3D(Layer): @@ -843,7 +878,7 @@ class Dropout3D(Layer): print(x) print(y_train) print(y_test) - """ + """ def __init__(self, p=0.5, data_format='NCDHW', name=None): super(Dropout3D, self).__init__() @@ -853,17 +888,20 @@ class Dropout3D(Layer): self.name = name def forward(self, input): - out = F.dropout3d(input, - p=self.p, - training=self.training, - data_format=self.data_format, - name=self.name) + out = F.dropout3d( + input, + p=self.p, + training=self.training, + data_format=self.data_format, + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'p={}, data_format={}{}'.format(self.p, self.data_format, - name_str) + return 'p={}, data_format={}{}'.format( + self.p, self.data_format, name_str + ) class AlphaDropout(Layer): @@ -902,7 +940,7 @@ class AlphaDropout(Layer): print(y_train) # [[-0.10721093, 1.6655989 ], [-0.7791938, -0.7791938]] (randomly) print(y_test) - """ + """ def __init__(self, p=0.5, name=None): super(AlphaDropout, self).__init__() @@ -910,10 +948,9 @@ class AlphaDropout(Layer): self.name = name def forward(self, input): - out = F.alpha_dropout(input, - p=self.p, - training=self.training, - name=self.name) + out = F.alpha_dropout( + input, p=self.p, training=self.training, name=self.name + ) return out def extra_repr(self): @@ -963,12 +1000,9 @@ class Pad1D(Layer): # [0. 4. 5. 6. 0. 0.]]] """ - def __init__(self, - padding, - mode='constant', - value=0.0, - data_format="NCL", - name=None): + def __init__( + self, padding, mode='constant', value=0.0, data_format="NCL", name=None + ): super(Pad1D, self).__init__() self._pad = _npairs(padding, 1) self._mode = mode @@ -977,17 +1011,20 @@ class Pad1D(Layer): self._name = name def forward(self, x): - return F.pad(x, - pad=self._pad, - mode=self._mode, - value=self._value, - data_format=self._data_format, - name=self._name) + return F.pad( + x, + pad=self._pad, + mode=self._mode, + value=self._value, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'padding={}, mode={}, value={}, data_format={}{}'.format( - self._pad, self._mode, self._value, self._data_format, name_str) + self._pad, self._mode, self._value, self._data_format, name_str + ) class Pad2D(Layer): @@ -1036,12 +1073,9 @@ class Pad2D(Layer): # [0. 0. 0. 0.]]]] """ - def __init__(self, - padding, - mode='constant', - value=0.0, - data_format="NCHW", - name=None): + def __init__( + self, padding, mode='constant', value=0.0, data_format="NCHW", name=None + ): super(Pad2D, self).__init__() self._pad = _npairs(padding, 2) self._mode = mode @@ -1050,17 +1084,20 @@ class Pad2D(Layer): self._name = name def forward(self, x): - return F.pad(x, - pad=self._pad, - mode=self._mode, - value=self._value, - data_format=self._data_format, - name=self._name) + return F.pad( + x, + pad=self._pad, + mode=self._mode, + value=self._value, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'padding={}, mode={}, value={}, data_format={}{}'.format( - self._pad, self._mode, self._value, self._data_format, name_str) + self._pad, self._mode, self._value, self._data_format, name_str + ) class ZeroPad2D(Layer): @@ -1111,23 +1148,25 @@ class ZeroPad2D(Layer): super(ZeroPad2D, self).__init__() self._pad = _npairs(padding, 2) self._mode = 'constant' - self._value = 0. + self._value = 0.0 self._data_format = data_format self._name = name def forward(self, x): - return F.pad(x, - pad=self._pad, - mode=self._mode, - value=self._value, - data_format=self._data_format, - name=self._name) + return F.pad( + x, + pad=self._pad, + mode=self._mode, + value=self._value, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' - return 'padding={}, data_format={}{}'.format(self._pad, - self._data_format, - name_str) + return 'padding={}, data_format={}{}'.format( + self._pad, self._data_format, name_str + ) class Pad3D(Layer): @@ -1176,12 +1215,14 @@ class Pad3D(Layer): # [0. 0. 0. 0.]]]]] """ - def __init__(self, - padding, - mode='constant', - value=0.0, - data_format="NCDHW", - name=None): + def __init__( + self, + padding, + mode='constant', + value=0.0, + data_format="NCDHW", + name=None, + ): super(Pad3D, self).__init__() self._pad = _npairs(padding, 3) self._mode = mode @@ -1190,17 +1231,20 @@ class Pad3D(Layer): self._name = name def forward(self, x): - return F.pad(x, - pad=self._pad, - mode=self._mode, - value=self._value, - data_format=self._data_format, - name=self._name) + return F.pad( + x, + pad=self._pad, + mode=self._mode, + value=self._value, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'padding={}, mode={}, value={}, data_format={}{}'.format( - self._pad, self._mode, self._value, self._data_format, name_str) + self._pad, self._mode, self._value, self._data_format, name_str + ) class CosineSimilarity(Layer): @@ -1361,13 +1405,15 @@ class Embedding(Layer): """ - def __init__(self, - num_embeddings, - embedding_dim, - padding_idx=None, - sparse=False, - weight_attr=None, - name=None): + def __init__( + self, + num_embeddings, + embedding_dim, + padding_idx=None, + sparse=False, + weight_attr=None, + name=None, + ): super(Embedding, self).__init__() self._num_embeddings = num_embeddings self._embedding_dim = embedding_dim @@ -1381,12 +1427,20 @@ class Embedding(Layer): if self._embedding_dim <= 0: raise ValueError("embedding_dim must be gather than 0") - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - num_embeddings + padding_idx) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (num_embeddings + padding_idx) + ) if padding_idx >= num_embeddings or padding_idx < -num_embeddings: - raise ValueError("padding_idx must be within [-{}, {})".format( - num_embeddings, num_embeddings)) + raise ValueError( + "padding_idx must be within [-{}, {})".format( + num_embeddings, num_embeddings + ) + ) self._dtype = self._helper.get_default_dtype() self._size = [self._num_embeddings, self._embedding_dim] @@ -1394,21 +1448,25 @@ class Embedding(Layer): self._weight_attr = weight_attr self._remote_prefetch = False self._name = name - self.weight = self.create_parameter(attr=self._weight_attr, - shape=self._size, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False, + ) if in_dynamic_mode() and padding_idx != -1: with paddle.no_grad(): self.weight[padding_idx] = 0.0 def forward(self, x): - return F.embedding(x, - weight=self.weight, - padding_idx=self._padding_idx, - sparse=self._sparse, - name=self._name) + return F.embedding( + x, + weight=self.weight, + padding_idx=self._padding_idx, + sparse=self._sparse, + name=self._name, + ) def extra_repr(self): main_str = '{_num_embeddings}, {_embedding_dim}' @@ -1466,12 +1524,9 @@ class Unfold(Layer): print(result) """ - def __init__(self, - kernel_sizes, - dilations=1, - paddings=0, - strides=1, - name=None): + def __init__( + self, kernel_sizes, dilations=1, paddings=0, strides=1, name=None + ): super(Unfold, self).__init__() self.kernel_sizes = kernel_sizes @@ -1481,17 +1536,24 @@ class Unfold(Layer): self.name = name def forward(self, input): - return F.unfold(input, - kernel_sizes=self.kernel_sizes, - strides=self.strides, - paddings=self.paddings, - dilations=self.dilations, - name=self.name) + return F.unfold( + input, + kernel_sizes=self.kernel_sizes, + strides=self.strides, + paddings=self.paddings, + dilations=self.dilations, + name=self.name, + ) def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'kernel_size={}, dilation={}, padding={}, stride={}{}'.\ - format(self.kernel_sizes, self.dilations, self.paddings, self.strides, name_str) + return 'kernel_size={}, dilation={}, padding={}, stride={}{}'.format( + self.kernel_sizes, + self.dilations, + self.paddings, + self.strides, + name_str, + ) class Fold(Layer): @@ -1551,13 +1613,15 @@ class Fold(Layer): # y.shape = [2,3,4,5] """ - def __init__(self, - output_sizes, - kernel_sizes, - dilations=1, - paddings=0, - strides=1, - name=None): + def __init__( + self, + output_sizes, + kernel_sizes, + dilations=1, + paddings=0, + strides=1, + name=None, + ): super(Fold, self).__init__() self.output_sizes = output_sizes @@ -1568,15 +1632,22 @@ class Fold(Layer): self.name = name def forward(self, input): - return F.fold(input, - output_sizes=self.output_sizes, - kernel_sizes=self.kernel_sizes, - strides=self.strides, - paddings=self.paddings, - dilations=self.dilations, - name=self.name) + return F.fold( + input, + output_sizes=self.output_sizes, + kernel_sizes=self.kernel_sizes, + strides=self.strides, + paddings=self.paddings, + dilations=self.dilations, + name=self.name, + ) def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'kernel_size={}, dilation={}, padding={}, stride={}{}'.\ - format(self.kernel_sizes, self.dilations, self.paddings, self.strides, name_str) + return 'kernel_size={}, dilation={}, padding={}, stride={}{}'.format( + self.kernel_sizes, + self.dilations, + self.paddings, + self.strides, + name_str, + ) diff --git a/python/paddle/nn/layer/container.py b/python/paddle/nn/layer/container.py index fe2753a6141..a3ca6294e38 100644 --- a/python/paddle/nn/layer/container.py +++ b/python/paddle/nn/layer/container.py @@ -275,10 +275,10 @@ class LayerDict(Layer): """ - assert isinstance( - sublayers, Iterable - ), "The type of sublayers is not iterable of key/value pairs, the type of sublayers is " + type( - sublayers).__name__ + assert isinstance(sublayers, Iterable), ( + "The type of sublayers is not iterable of key/value pairs, the type of sublayers is " + + type(sublayers).__name__ + ) if isinstance(sublayers, (OrderedDict, LayerDict, Mapping)): for key, layer in sublayers.items(): @@ -287,7 +287,11 @@ class LayerDict(Layer): # handle this format [(key1, layer1), (key2, layer2)...] for i, kv in enumerate(sublayers): if len(kv) != 2: - raise ValueError("The length of the " + str(i) + - "'s element in sublayers is " + - str(len(kv)) + ", which must be 2.") + raise ValueError( + "The length of the " + + str(i) + + "'s element in sublayers is " + + str(len(kv)) + + ", which must be 2." + ) self.add_sublayer(kv[0], kv[1]) diff --git a/python/paddle/nn/layer/conv.py b/python/paddle/nn/layer/conv.py index fd5583d50ba..61f9044d03a 100644 --- a/python/paddle/nn/layer/conv.py +++ b/python/paddle/nn/layer/conv.py @@ -31,7 +31,7 @@ __all__ = [] def _get_default_param_initializer(num_channels, filter_size): filter_elem_num = num_channels * np.prod(filter_size) - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std) @@ -44,24 +44,27 @@ def _reverse_repeat_list(t, n): class _ConvNd(Layer): - - def __init__(self, - in_channels, - out_channels, - kernel_size, - transposed, - dims, - stride=1, - padding=0, - padding_mode='zeros', - output_padding=0, - dilation=1, - groups=1, - weight_attr=None, - bias_attr=None, - data_format="NCHW"): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + transposed, + dims, + stride=1, + padding=0, + padding_mode='zeros', + output_padding=0, + dilation=1, + groups=1, + weight_attr=None, + bias_attr=None, + data_format="NCHW", + ): super(_ConvNd, self).__init__() - assert weight_attr is not False, "weight_attr should not be False in Conv." + assert ( + weight_attr is not False + ), "weight_attr should not be False in Conv." self._param_attr = weight_attr self._bias_attr = bias_attr self._groups = groups @@ -72,11 +75,16 @@ class _ConvNd(Layer): valid_padding_modes = {'zeros', 'reflect', 'replicate', 'circular'} if padding_mode not in valid_padding_modes: raise ValueError( - "padding_mode must be one of {}, but got padding_mode='{}'". - format(valid_padding_modes, padding_mode)) + "padding_mode must be one of {}, but got padding_mode='{}'".format( + valid_padding_modes, padding_mode + ) + ) - if padding_mode in {'reflect', 'replicate', 'circular' - } and not isinstance(padding, int): + if padding_mode in { + 'reflect', + 'replicate', + 'circular', + } and not isinstance(padding, int): raise TypeError( "when padding_mode in ['reflect', 'replicate', 'circular'], type of padding must be int" ) @@ -84,12 +92,16 @@ class _ConvNd(Layer): valid_format = {'NHWC', 'NCHW', 'NDHWC', 'NCDHW', 'NLC', 'NCL'} if data_format not in valid_format: raise ValueError( - "data_format must be one of {}, but got data_format='{}'". - format(valid_format, data_format)) + "data_format must be one of {}, but got data_format='{}'".format( + valid_format, data_format + ) + ) - channel_last = (data_format == "NHWC") or (data_format - == "NDHWC") or (data_format - == "NLC") + channel_last = ( + (data_format == "NHWC") + or (data_format == "NDHWC") + or (data_format == "NLC") + ) if channel_last: self._channel_dim = len(data_format) - 1 else: @@ -97,66 +109,86 @@ class _ConvNd(Layer): self._stride = utils.convert_to_list(stride, dims, 'stride') self._dilation = utils.convert_to_list(dilation, dims, 'dilation') - self._kernel_size = utils.convert_to_list(kernel_size, dims, - 'kernel_size') + self._kernel_size = utils.convert_to_list( + kernel_size, dims, 'kernel_size' + ) self._padding = padding self._padding_mode = padding_mode self.output_padding = output_padding if dims != 1: self._updated_padding, self._padding_algorithm = _update_padding_nd( - padding, channel_last, dims) + padding, channel_last, dims + ) if transposed: - filter_shape = [self._in_channels, out_channels // groups - ] + self._kernel_size + filter_shape = [ + self._in_channels, + out_channels // groups, + ] + self._kernel_size else: if in_channels % groups != 0: raise ValueError("in_channels must be divisible by groups.") if padding_mode in {'reflect', 'replicate', 'circular'}: - _paired_padding = utils.convert_to_list(padding, dims, - 'padding') + _paired_padding = utils.convert_to_list( + padding, dims, 'padding' + ) self._reversed_padding_repeated_twice = _reverse_repeat_list( - _paired_padding, 2) + _paired_padding, 2 + ) - self._updated_padding, self._padding_algorithm = _update_padding_nd( - 0, channel_last, dims) + ( + self._updated_padding, + self._padding_algorithm, + ) = _update_padding_nd(0, channel_last, dims) - filter_shape = [out_channels, in_channels // groups - ] + self._kernel_size + filter_shape = [ + out_channels, + in_channels // groups, + ] + self._kernel_size def _get_default_param_initializer(): if transposed: return None filter_elem_num = np.prod(self._kernel_size) * self._in_channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std) self.weight = self.create_parameter( shape=filter_shape, attr=self._param_attr, - default_initializer=_get_default_param_initializer()) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._out_channels], - is_bias=True) + default_initializer=_get_default_param_initializer(), + ) + self.bias = self.create_parameter( + attr=self._bias_attr, shape=[self._out_channels], is_bias=True + ) cudnn_version = get_cudnn_version() - self._use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + self._use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) self._op_type = "conv" + str(dims) + 'd' - if self._op_type == 'conv2d' and (in_channels == groups - and in_channels != 1 - and out_channels % in_channels == 0): + if self._op_type == 'conv2d' and ( + in_channels == groups + and in_channels != 1 + and out_channels % in_channels == 0 + ): self._op_type = 'depthwise_conv2d' if is_compiled_with_rocm(): self._use_cudnn = True else: self._use_cudnn = False - if (is_compiled_with_cuda() and get_flags("FLAGS_conv2d_disable_cudnn") - ["FLAGS_conv2d_disable_cudnn"]): + if ( + is_compiled_with_cuda() + and get_flags("FLAGS_conv2d_disable_cudnn")[ + "FLAGS_conv2d_disable_cudnn" + ] + ): self._use_cudnn = False def extra_repr(self): @@ -299,50 +331,58 @@ class Conv1D(_ConvNd): # [160. 211.]]] """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NCL"): - super(Conv1D, self).__init__(in_channels, - out_channels, - kernel_size, - False, - 1, - stride=stride, - padding=padding, - padding_mode=padding_mode, - dilation=dilation, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NCL", + ): + super(Conv1D, self).__init__( + in_channels, + out_channels, + kernel_size, + False, + 1, + stride=stride, + padding=padding, + padding_mode=padding_mode, + dilation=dilation, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x): padding = 0 if self._padding_mode != "zeros": - x = F.pad(x, - self._reversed_padding_repeated_twice, - mode=self._padding_mode, - data_format=self._data_format) + x = F.pad( + x, + self._reversed_padding_repeated_twice, + mode=self._padding_mode, + data_format=self._data_format, + ) else: padding = self._padding - out = F.conv1d(x, - self.weight, - bias=self.bias, - padding=padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format) + out = F.conv1d( + x, + self.weight, + bias=self.bias, + padding=padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + ) return out @@ -474,43 +514,49 @@ class Conv1DTranspose(_ConvNd): # [[[60. 16. 99. 75. 4.]]] """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - output_padding=0, - groups=1, - dilation=1, - weight_attr=None, - bias_attr=None, - data_format="NCL"): - super(Conv1DTranspose, self).__init__(in_channels, - out_channels, - kernel_size, - True, - 1, - stride=stride, - padding=padding, - dilation=dilation, - output_padding=output_padding, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, + weight_attr=None, + bias_attr=None, + data_format="NCL", + ): + super(Conv1DTranspose, self).__init__( + in_channels, + out_channels, + kernel_size, + True, + 1, + stride=stride, + padding=padding, + dilation=dilation, + output_padding=output_padding, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x, output_size=None): - out = F.conv1d_transpose(x, - self.weight, - bias=self.bias, - output_size=output_size, - output_padding=self.output_padding, - padding=self._padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format) + out = F.conv1d_transpose( + x, + self.weight, + bias=self.bias, + output_size=output_size, + output_padding=self.output_padding, + padding=self._padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + ) return out @@ -625,51 +671,59 @@ class Conv2D(_ConvNd): # (2, 6, 6, 6) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NCHW"): - super(Conv2D, self).__init__(in_channels, - out_channels, - kernel_size, - False, - 2, - stride=stride, - padding=padding, - padding_mode=padding_mode, - dilation=dilation, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NCHW", + ): + super(Conv2D, self).__init__( + in_channels, + out_channels, + kernel_size, + False, + 2, + stride=stride, + padding=padding, + padding_mode=padding_mode, + dilation=dilation, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x): if self._padding_mode != 'zeros': - x = F.pad(x, - self._reversed_padding_repeated_twice, - mode=self._padding_mode, - data_format=self._data_format) - - out = F.conv._conv_nd(x, - self.weight, - bias=self.bias, - stride=self._stride, - padding=self._updated_padding, - padding_algorithm=self._padding_algorithm, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format, - channel_dim=self._channel_dim, - op_type=self._op_type, - use_cudnn=self._use_cudnn) + x = F.pad( + x, + self._reversed_padding_repeated_twice, + mode=self._padding_mode, + data_format=self._data_format, + ) + + out = F.conv._conv_nd( + x, + self.weight, + bias=self.bias, + stride=self._stride, + padding=self._updated_padding, + padding_algorithm=self._padding_algorithm, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + channel_dim=self._channel_dim, + op_type=self._op_type, + use_cudnn=self._use_cudnn, + ) return out @@ -790,31 +844,35 @@ class Conv2DTranspose(_ConvNd): # (2, 6, 10, 10) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - output_padding=0, - dilation=1, - groups=1, - weight_attr=None, - bias_attr=None, - data_format="NCHW"): - super(Conv2DTranspose, self).__init__(in_channels, - out_channels, - kernel_size, - True, - 2, - stride=stride, - padding=padding, - dilation=dilation, - output_padding=output_padding, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + output_padding=0, + dilation=1, + groups=1, + weight_attr=None, + bias_attr=None, + data_format="NCHW", + ): + super(Conv2DTranspose, self).__init__( + in_channels, + out_channels, + kernel_size, + True, + 2, + stride=stride, + padding=padding, + dilation=dilation, + output_padding=output_padding, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x, output_size=None): if output_size is None: @@ -822,16 +880,18 @@ class Conv2DTranspose(_ConvNd): else: output_padding = 0 - out = F.conv2d_transpose(x, - self.weight, - bias=self.bias, - padding=self._padding, - output_padding=output_padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - output_size=output_size, - data_format=self._data_format) + out = F.conv2d_transpose( + x, + self.weight, + bias=self.bias, + padding=self._padding, + output_padding=output_padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + output_size=output_size, + data_format=self._data_format, + ) return out @@ -942,51 +1002,59 @@ class Conv3D(_ConvNd): # (2, 6, 6, 6, 6) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NCDHW"): - super(Conv3D, self).__init__(in_channels, - out_channels, - kernel_size, - False, - 3, - stride=stride, - padding=padding, - padding_mode=padding_mode, - dilation=dilation, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NCDHW", + ): + super(Conv3D, self).__init__( + in_channels, + out_channels, + kernel_size, + False, + 3, + stride=stride, + padding=padding, + padding_mode=padding_mode, + dilation=dilation, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x): if self._padding_mode != 'zeros': - x = F.pad(x, - self._reversed_padding_repeated_twice, - mode=self._padding_mode, - data_format=self._data_format) - - out = F.conv._conv_nd(x, - self.weight, - bias=self.bias, - stride=self._stride, - padding=self._updated_padding, - padding_algorithm=self._padding_algorithm, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format, - channel_dim=self._channel_dim, - op_type=self._op_type, - use_cudnn=self._use_cudnn) + x = F.pad( + x, + self._reversed_padding_repeated_twice, + mode=self._padding_mode, + data_format=self._data_format, + ) + + out = F.conv._conv_nd( + x, + self.weight, + bias=self.bias, + stride=self._stride, + padding=self._updated_padding, + padding_algorithm=self._padding_algorithm, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + channel_dim=self._channel_dim, + op_type=self._op_type, + use_cudnn=self._use_cudnn, + ) return out @@ -1116,31 +1184,35 @@ class Conv3DTranspose(_ConvNd): # (2, 6, 10, 10, 10) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - output_padding=0, - dilation=1, - groups=1, - weight_attr=None, - bias_attr=None, - data_format="NCDHW"): - super(Conv3DTranspose, self).__init__(in_channels, - out_channels, - kernel_size, - True, - 3, - stride=stride, - padding=padding, - dilation=dilation, - output_padding=output_padding, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + output_padding=0, + dilation=1, + groups=1, + weight_attr=None, + bias_attr=None, + data_format="NCDHW", + ): + super(Conv3DTranspose, self).__init__( + in_channels, + out_channels, + kernel_size, + True, + 3, + stride=stride, + padding=padding, + dilation=dilation, + output_padding=output_padding, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x, output_size=None): if output_size is None: @@ -1148,14 +1220,16 @@ class Conv3DTranspose(_ConvNd): else: output_padding = 0 - out = F.conv3d_transpose(x, - self.weight, - bias=self.bias, - padding=self._padding, - output_padding=output_padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - output_size=output_size, - data_format=self._data_format) + out = F.conv3d_transpose( + x, + self.weight, + bias=self.bias, + padding=self._padding, + output_padding=output_padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + output_size=output_size, + data_format=self._data_format, + ) return out diff --git a/python/paddle/nn/layer/distance.py b/python/paddle/nn/layer/distance.py index a7a488c833d..344c1d482d4 100644 --- a/python/paddle/nn/layer/distance.py +++ b/python/paddle/nn/layer/distance.py @@ -59,7 +59,7 @@ class PairwiseDistance(Layer): """ - def __init__(self, p=2., epsilon=1e-6, keepdim=False, name=None): + def __init__(self, p=2.0, epsilon=1e-6, keepdim=False, name=None): super(PairwiseDistance, self).__init__() self.p = p self.epsilon = epsilon @@ -68,8 +68,9 @@ class PairwiseDistance(Layer): def forward(self, x, y): - return F.pairwise_distance(x, y, self.p, self.epsilon, self.keepdim, - self.name) + return F.pairwise_distance( + x, y, self.p, self.epsilon, self.keepdim, self.name + ) def extra_repr(self): main_str = 'p={p}' diff --git a/python/paddle/nn/layer/loss.py b/python/paddle/nn/layer/loss.py index 6de2717a061..cd1c8bcb470 100644 --- a/python/paddle/nn/layer/loss.py +++ b/python/paddle/nn/layer/loss.py @@ -105,15 +105,14 @@ class BCEWithLogitsLoss(Layer): """ - def __init__(self, - weight=None, - reduction='mean', - pos_weight=None, - name=None): + def __init__( + self, weight=None, reduction='mean', pos_weight=None, name=None + ): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in BCEWithLogitsLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(BCEWithLogitsLoss, self).__init__() self.weight = weight @@ -123,8 +122,13 @@ class BCEWithLogitsLoss(Layer): def forward(self, logit, label): out = paddle.nn.functional.binary_cross_entropy_with_logits( - logit, label, self.weight, self.reduction, self.pos_weight, - self.name) + logit, + label, + self.weight, + self.reduction, + self.pos_weight, + self.name, + ) return out @@ -374,14 +378,16 @@ class CrossEntropyLoss(Layer): """ - def __init__(self, - weight=None, - ignore_index=-100, - reduction='mean', - soft_label=False, - axis=-1, - use_softmax=True, - name=None): + def __init__( + self, + weight=None, + ignore_index=-100, + reduction='mean', + soft_label=False, + axis=-1, + use_softmax=True, + name=None, + ): super(CrossEntropyLoss, self).__init__() self.weight = weight self.reduction = reduction @@ -392,15 +398,17 @@ class CrossEntropyLoss(Layer): self.name = name def forward(self, input, label): - ret = paddle.nn.functional.cross_entropy(input, - label, - weight=self.weight, - ignore_index=self.ignore_index, - reduction=self.reduction, - soft_label=self.soft_label, - axis=self.axis, - use_softmax=self.use_softmax, - name=self.name) + ret = paddle.nn.functional.cross_entropy( + input, + label, + weight=self.weight, + ignore_index=self.ignore_index, + reduction=self.reduction, + soft_label=self.soft_label, + axis=self.axis, + use_softmax=self.use_softmax, + name=self.name, + ) return ret @@ -476,18 +484,21 @@ class HSigmoidLoss(Layer): # [2.34564662]] """ - def __init__(self, - feature_size, - num_classes, - weight_attr=None, - bias_attr=None, - is_custom=False, - is_sparse=False, - name=None): + def __init__( + self, + feature_size, + num_classes, + weight_attr=None, + bias_attr=None, + is_custom=False, + is_sparse=False, + name=None, + ): super(HSigmoidLoss, self).__init__() if (num_classes < 2) and (not is_custom): raise ValueError( - "num_classes must not be less than 2 with default tree") + "num_classes must not be less than 2 with default tree" + ) if (not is_custom) and (is_sparse): print("Sparse mode should not be used without custom tree") @@ -505,29 +516,34 @@ class HSigmoidLoss(Layer): self._dtype = paddle.get_default_dtype() remote_prefetch = is_sparse - print("With sparse mode, if your models has only" - " small parameter prefetch may cause speed down") + print( + "With sparse mode, if your models has only" + " small parameter prefetch may cause speed down" + ) C = self._num_classes if is_custom else self._num_classes - 1 - self.weight = self.create_parameter([C, self._feature_size], - attr=self._weight_attr, - is_bias=False, - dtype=self._dtype) - self.bias = self.create_parameter([C, 1], - attr=self._bias_attr, - is_bias=True, - dtype=self._dtype) + self.weight = self.create_parameter( + [C, self._feature_size], + attr=self._weight_attr, + is_bias=False, + dtype=self._dtype, + ) + self.bias = self.create_parameter( + [C, 1], attr=self._bias_attr, is_bias=True, dtype=self._dtype + ) def forward(self, input, label, path_table=None, path_code=None): - out = F.hsigmoid_loss(input, - label, - self._num_classes, - self.weight, - self.bias, - path_table=path_table, - path_code=path_code, - is_sparse=self._is_sparse, - name=self._name) + out = F.hsigmoid_loss( + input, + label, + self._num_classes, + self.weight, + self.bias, + path_table=path_table, + path_code=path_code, + is_sparse=self._is_sparse, + name=self._name, + ) return out @@ -584,17 +600,18 @@ class MSELoss(Layer): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'MSELoss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) self.reduction = reduction def forward(self, input, label): if not in_dynamic_mode(): - fluid.data_feeder.check_variable_and_dtype(input, 'input', - ['float32', 'float64'], - 'MSELoss') - fluid.data_feeder.check_variable_and_dtype(label, 'label', - ['float32', 'float64'], - 'MSELoss') + fluid.data_feeder.check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'MSELoss' + ) + fluid.data_feeder.check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'MSELoss' + ) if in_dygraph_mode(): square_out = paddle._C_ops.square(paddle.subtract(input, label)) @@ -676,16 +693,16 @@ class L1Loss(Layer): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in L1Loss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(L1Loss, self).__init__() self.reduction = reduction self.name = name def forward(self, input, label): - return paddle.nn.functional.l1_loss(input, - label, - self.reduction, - name=self.name) + return paddle.nn.functional.l1_loss( + input, label, self.reduction, name=self.name + ) class BCELoss(Layer): @@ -765,7 +782,8 @@ class BCELoss(Layer): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in bce_loss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(BCELoss, self).__init__() self.weight = weight @@ -773,10 +791,9 @@ class BCELoss(Layer): self.name = name def forward(self, input, label): - out = paddle.nn.functional.binary_cross_entropy(input, label, - self.weight, - self.reduction, - self.name) + out = paddle.nn.functional.binary_cross_entropy( + input, label, self.weight, self.reduction, self.name + ) return out @@ -866,15 +883,14 @@ class NLLLoss(Layer): """ - def __init__(self, - weight=None, - ignore_index=-100, - reduction='mean', - name=None): + def __init__( + self, weight=None, ignore_index=-100, reduction='mean', name=None + ): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in nll_loss should be 'sum', 'mean' or " - "'none', but received %s, which is not allowed." % reduction) + "'none', but received %s, which is not allowed." % reduction + ) super(NLLLoss, self).__init__() self._weight = weight self._ignore_index = ignore_index @@ -882,12 +898,14 @@ class NLLLoss(Layer): self._name = name def forward(self, input, label): - return F.nll_loss(input, - label, - weight=self._weight, - ignore_index=self._ignore_index, - reduction=self._reduction, - name=self._name) + return F.nll_loss( + input, + label, + weight=self._weight, + ignore_index=self._ignore_index, + reduction=self._reduction, + name=self._name, + ) class KLDivLoss(Layer): @@ -1019,17 +1037,17 @@ class MarginRankingLoss(Layer): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in MarginRankingLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(MarginRankingLoss, self).__init__() self.margin = margin self.reduction = reduction self.name = name def forward(self, input, other, label): - out = paddle.nn.functional.margin_ranking_loss(input, other, label, - self.margin, - self.reduction, - self.name) + out = paddle.nn.functional.margin_ranking_loss( + input, other, label, self.margin, self.reduction, self.name + ) return out @@ -1113,19 +1131,23 @@ class CTCLoss(Layer): self.blank = blank self.reduction = reduction - def forward(self, - log_probs, - labels, - input_lengths, - label_lengths, - norm_by_times=False): - return paddle.nn.functional.ctc_loss(log_probs, - labels, - input_lengths, - label_lengths, - self.blank, - self.reduction, - norm_by_times=norm_by_times) + def forward( + self, + log_probs, + labels, + input_lengths, + label_lengths, + norm_by_times=False, + ): + return paddle.nn.functional.ctc_loss( + log_probs, + labels, + input_lengths, + label_lengths, + self.blank, + self.reduction, + norm_by_times=norm_by_times, + ) class SmoothL1Loss(Layer): @@ -1192,11 +1214,13 @@ class SmoothL1Loss(Layer): self.name = name def forward(self, input, label): - return F.smooth_l1_loss(input, - label, - reduction=self.reduction, - delta=self.delta, - name=self.name) + return F.smooth_l1_loss( + input, + label, + reduction=self.reduction, + delta=self.delta, + name=self.name, + ) class MultiLabelSoftMarginLoss(Layer): @@ -1264,17 +1288,20 @@ class MultiLabelSoftMarginLoss(Layer): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'MultiLabelSoftMarginloss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) self.weight = weight self.reduction = reduction self.name = name def forward(self, input, label): - return F.multi_label_soft_margin_loss(input, - label, - weight=self.weight, - reduction=self.reduction, - name=self.name) + return F.multi_label_soft_margin_loss( + input, + label, + weight=self.weight, + reduction=self.reduction, + name=self.name, + ) class HingeEmbeddingLoss(Layer): @@ -1364,11 +1391,13 @@ class HingeEmbeddingLoss(Layer): self.name = name def forward(self, input, label): - return F.hinge_embedding_loss(input, - label, - reduction=self.reduction, - margin=self.margin, - name=self.name) + return F.hinge_embedding_loss( + input, + label, + reduction=self.reduction, + margin=self.margin, + name=self.name, + ) class CosineEmbeddingLoss(Layer): @@ -1442,23 +1471,27 @@ class CosineEmbeddingLoss(Layer): if margin > 1 or margin < -1: raise ValueError( "The value of 'margin' should be in the interval of [-1, 1], but received %f, which is not allowed." - % margin) + % margin + ) if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' should be 'sum', 'mean' or " - "'none', but received %s, which is not allowed." % reduction) + "'none', but received %s, which is not allowed." % reduction + ) super(CosineEmbeddingLoss, self).__init__() self.margin = margin self.reduction = reduction self.name = name def forward(self, input1, input2, label): - return F.cosine_embedding_loss(input1, - input2, - label, - margin=self.margin, - reduction=self.reduction, - name=self.name) + return F.cosine_embedding_loss( + input1, + input2, + label, + margin=self.margin, + reduction=self.reduction, + name=self.name, + ) class TripletMarginWithDistanceLoss(Layer): @@ -1540,18 +1573,21 @@ class TripletMarginWithDistanceLoss(Layer): """ - def __init__(self, - distance_function=None, - margin=1.0, - swap=False, - reduction: str = 'mean', - name=None): + def __init__( + self, + distance_function=None, + margin=1.0, + swap=False, + reduction: str = 'mean', + name=None, + ): super(TripletMarginWithDistanceLoss, self).__init__() if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in TripletMarginWithDistanceLoss " "should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) self.margin = margin self.swap = swap self.reduction = reduction @@ -1559,13 +1595,15 @@ class TripletMarginWithDistanceLoss(Layer): self.name = name def forward(self, input, positive, negative): - return F.triplet_margin_with_distance_loss(input, - positive, - negative, - margin=self.margin, - swap=self.swap, - reduction=self.reduction, - name=self.name) + return F.triplet_margin_with_distance_loss( + input, + positive, + negative, + margin=self.margin, + swap=self.swap, + reduction=self.reduction, + name=self.name, + ) class TripletMarginLoss(Layer): @@ -1643,18 +1681,21 @@ class TripletMarginLoss(Layer): """ - def __init__(self, - margin=1.0, - p=2., - epsilon=1e-6, - swap=False, - reduction='mean', - name=None): + def __init__( + self, + margin=1.0, + p=2.0, + epsilon=1e-6, + swap=False, + reduction='mean', + name=None, + ): super(TripletMarginLoss, self).__init__() if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in TripletMarginLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) self.margin = margin self.p = p self.epsilon = epsilon @@ -1663,98 +1704,103 @@ class TripletMarginLoss(Layer): self.name = name def forward(self, input, positive, negative): - return F.triplet_margin_loss(input, - positive, - negative, - margin=self.margin, - p=self.p, - epsilon=self.epsilon, - swap=self.swap, - reduction=self.reduction, - name=self.name) + return F.triplet_margin_loss( + input, + positive, + negative, + margin=self.margin, + p=self.p, + epsilon=self.epsilon, + swap=self.swap, + reduction=self.reduction, + name=self.name, + ) class MultiMarginLoss(Layer): r"""Creates a criterion that optimizes a multi-class classification hinge loss (margin-based loss) between - input :math:`input` and label :math:`label`: + input :math:`input` and label :math:`label`: - For i-th mini-batch sample, the loss in terms of the 1D input :math:`input_i` and scalar - output :math:`label_i` is: + For i-th mini-batch sample, the loss in terms of the 1D input :math:`input_i` and scalar + output :math:`label_i` is: - .. math:: - \text{loss}(input_i, label_i) = \frac{\sum_{j} \max(0, \text{margin} - input_i[label_i] + input_i[j])^p}{\text{C}} + .. math:: + \text{loss}(input_i, label_i) = \frac{\sum_{j} \max(0, \text{margin} - input_i[label_i] + input_i[j])^p}{\text{C}} - where :math:`0 \leq j \leq \text{C}-1`, :math:`0 \leq i \leq \text{N}-1` and :math:`j \neq label_i`. + where :math:`0 \leq j \leq \text{C}-1`, :math:`0 \leq i \leq \text{N}-1` and :math:`j \neq label_i`. - Optionally, you can give non-equal weighting on the classes by passing - a 1D :attr:`weight` tensor into the constructor. + Optionally, you can give non-equal weighting on the classes by passing + a 1D :attr:`weight` tensor into the constructor. - The loss function for i-th sample then becomes: + The loss function for i-th sample then becomes: - .. math:: - \text{loss}(input_i, label_i) = \frac{\sum_{j} \max(0, weight[label_i] * (\text{margin} - input_i[label_i] + input_i[j]))^p}{\text{C}} + .. math:: + \text{loss}(input_i, label_i) = \frac{\sum_{j} \max(0, weight[label_i] * (\text{margin} - input_i[label_i] + input_i[j]))^p}{\text{C}} - Parameters: + Parameters: - p (int, Optional):The norm degree for pairwise distance. Default: :math:`1`. + p (int, Optional):The norm degree for pairwise distance. Default: :math:`1`. - margin (float, Optional):Default: :math:`1`. + margin (float, Optional):Default: :math:`1`. - weight (Tensor,optional): a manual rescaling weight given to each class. - If given, has to be a Tensor of shape (C,) and the data type is float32, float64. - Default is ``'None'`` . + weight (Tensor,optional): a manual rescaling weight given to each class. + If given, has to be a Tensor of shape (C,) and the data type is float32, float64. + Default is ``'None'`` . - reduction (str, optional): Indicate how to calculate the loss by batch_size, - the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. - If :attr:`reduction` is ``'none'``, the unreduced loss is returned; - If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; - If :attr:`reduction` is ``'sum'``, the summed loss is returned. - Default: ``'mean'`` + reduction (str, optional): Indicate how to calculate the loss by batch_size, + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. + If :attr:`reduction` is ``'none'``, the unreduced loss is returned; + If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; + If :attr:`reduction` is ``'sum'``, the summed loss is returned. + Default: ``'mean'`` - name (str, optional): Name for the operation (optional, default is None). - For more information, please refer to :ref:`api_guide_Name`. + name (str, optional): Name for the operation (optional, default is None). + For more information, please refer to :ref:`api_guide_Name`. - Call parameters: - input (Tensor): Input tensor, the data type is float32 or float64. + Call parameters: + input (Tensor): Input tensor, the data type is float32 or float64. - label (Tensor): Label tensor, 0<= label < input.shape[1], the data type is int32 or int64. + label (Tensor): Label tensor, 0<= label < input.shape[1], the data type is int32 or int64. - Shape: - input: 2-D Tensor, the shape is [N, C], N is batch size and `C` means number of classes. + Shape: + input: 2-D Tensor, the shape is [N, C], N is batch size and `C` means number of classes. - label: 1-D Tensor, the shape is [N,]. + label: 1-D Tensor, the shape is [N,]. - output: scalar. If :attr:`reduction` is ``'none'``, then same shape as the label. + output: scalar. If :attr:`reduction` is ``'none'``, then same shape as the label. - Returns: - A callable object of MultiMarginLoss. + Returns: + A callable object of MultiMarginLoss. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - import paddle - import paddle.nn as nn + import paddle + import paddle.nn as nn - input = paddle.to_tensor([[1, -2, 3], [0, -1, 2], [1, 0, 1]], dtype=paddle.float32) - label = paddle.to_tensor([0, 1, 2], dtype=paddle.int32) + input = paddle.to_tensor([[1, -2, 3], [0, -1, 2], [1, 0, 1]], dtype=paddle.float32) + label = paddle.to_tensor([0, 1, 2], dtype=paddle.int32) - multi_margin_loss = nn.MultiMarginLoss(reduction='mean') - loss = multi_margin_loss(input, label) - print(loss) - """ + multi_margin_loss = nn.MultiMarginLoss(reduction='mean') + loss = multi_margin_loss(input, label) + print(loss) + """ - def __init__(self, - p: int = 1, - margin: float = 1.0, - weight=None, - reduction="mean", - name=None): + def __init__( + self, + p: int = 1, + margin: float = 1.0, + weight=None, + reduction="mean", + name=None, + ): super(MultiMarginLoss, self).__init__() if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'MultiMarginLoss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) self.p = p self.margin = margin self.weight = weight @@ -1762,13 +1808,15 @@ class MultiMarginLoss(Layer): self.name = name def forward(self, input, label): - return F.multi_margin_loss(input, - label, - p=self.p, - margin=self.margin, - weight=self.weight, - reduction=self.reduction, - name=self.name) + return F.multi_margin_loss( + input, + label, + p=self.p, + margin=self.margin, + weight=self.weight, + reduction=self.reduction, + name=self.name, + ) class SoftMarginLoss(Layer): @@ -1830,13 +1878,15 @@ class SoftMarginLoss(Layer): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in SoftMarginLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(SoftMarginLoss, self).__init__() self.reduction = reduction self.name = name def forward(self, input, label): - out = paddle.nn.functional.soft_margin_loss(input, label, - self.reduction, self.name) + out = paddle.nn.functional.soft_margin_loss( + input, label, self.reduction, self.name + ) return out diff --git a/python/paddle/nn/layer/norm.py b/python/paddle/nn/layer/norm.py index d011bfd34aa..1fca251c570 100644 --- a/python/paddle/nn/layer/norm.py +++ b/python/paddle/nn/layer/norm.py @@ -59,18 +59,22 @@ class _InstanceNormBase(Layer): See InstaceNorm1D, InstanceNorm2D or InstanceNorm3D for more details. """ - def __init__(self, - num_features, - epsilon=1e-5, - momentum=0.9, - weight_attr=None, - bias_attr=None, - data_format="NCHW", - name=None): + def __init__( + self, + num_features, + epsilon=1e-5, + momentum=0.9, + weight_attr=None, + bias_attr=None, + data_format="NCHW", + name=None, + ): super(_InstanceNormBase, self).__init__() if weight_attr == False or bias_attr == False: - assert weight_attr == bias_attr, "weight_attr and bias_attr must be set to Fasle at the same time in InstanceNorm" + assert ( + weight_attr == bias_attr + ), "weight_attr and bias_attr must be set to Fasle at the same time in InstanceNorm" self._epsilon = epsilon self._weight_attr = weight_attr self._bias_attr = bias_attr @@ -81,11 +85,14 @@ class _InstanceNormBase(Layer): attr=self._weight_attr, shape=[num_features], default_initializer=Constant(1.0), - is_bias=False) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[num_features], - default_initializer=Constant(0.0), - is_bias=True) + is_bias=False, + ) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[num_features], + default_initializer=Constant(0.0), + is_bias=True, + ) else: self.scale = None self.bias = None @@ -96,14 +103,14 @@ class _InstanceNormBase(Layer): def forward(self, input): self._check_input_dim(input) - return instance_norm(input, - weight=self.scale, - bias=self.bias, - eps=self._epsilon) + return instance_norm( + input, weight=self.scale, bias=self.bias, eps=self._epsilon + ) def extra_repr(self): - return 'num_features={}, epsilon={}'.format(self._num_features, - self._epsilon) + return 'num_features={}, epsilon={}'.format( + self._num_features, self._epsilon + ) class InstanceNorm1D(_InstanceNormBase): @@ -169,8 +176,11 @@ Where `H` means height of feature map, `W` means width of feature map. def _check_input_dim(self, input): if len(input.shape) != 2 and len(input.shape) != 3: - raise ValueError('expected 2D or 3D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 2D or 3D input (got {}D input)'.format( + len(input.shape) + ) + ) class InstanceNorm2D(_InstanceNormBase): @@ -235,8 +245,9 @@ Where `H` means height of feature map, `W` means width of feature map. def _check_input_dim(self, input): if len(input.shape) != 4: - raise ValueError('expected 4D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 4D input (got {}D input)'.format(len(input.shape)) + ) class InstanceNorm3D(_InstanceNormBase): @@ -301,8 +312,9 @@ Where `H` means height of feature map, `W` means width of feature map. def _check_input_dim(self, input): if len(input.shape) != 5: - raise ValueError('expected 5D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 5D input (got {}D input)'.format(len(input.shape)) + ) class GroupNorm(Layer): @@ -349,14 +361,16 @@ class GroupNorm(Layer): print(group_norm_out.numpy()) """ - def __init__(self, - num_groups, - num_channels, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCHW', - name=None): + def __init__( + self, + num_groups, + num_channels, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCHW', + name=None, + ): super(GroupNorm, self).__init__() self._weight_attr = weight_attr self._bias_attr = bias_attr @@ -370,39 +384,57 @@ class GroupNorm(Layer): if weight_attr == False: self.weight = self.create_parameter( - attr=None, shape=param_shape, default_initializer=Constant(1.0)) + attr=None, shape=param_shape, default_initializer=Constant(1.0) + ) self.weight.stop_gradient = True else: self.weight = self.create_parameter( attr=self._weight_attr, shape=param_shape, - default_initializer=Constant(1.0)) - self.weight.stop_gradient = self._weight_attr != None and self._weight_attr.learning_rate == 0. + default_initializer=Constant(1.0), + ) + self.weight.stop_gradient = ( + self._weight_attr != None + and self._weight_attr.learning_rate == 0.0 + ) if bias_attr == False: - self.bias = self.create_parameter(attr=None, - shape=param_shape, - default_initializer=Constant(0.0), - is_bias=True) + self.bias = self.create_parameter( + attr=None, + shape=param_shape, + default_initializer=Constant(0.0), + is_bias=True, + ) self.bias.stop_gradient = True else: - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - is_bias=True) - self.bias.stop_gradient = self._bias_attr != None and self._bias_attr.learning_rate == 0. + self.bias = self.create_parameter( + attr=self._bias_attr, shape=param_shape, is_bias=True + ) + self.bias.stop_gradient = ( + self._bias_attr != None and self._bias_attr.learning_rate == 0.0 + ) def forward(self, input): mean_out = self._helper.create_variable_for_type_inference( - dtype=input.dtype, stop_gradient=True) + dtype=input.dtype, stop_gradient=True + ) variance_out = self._helper.create_variable_for_type_inference( - dtype=input.dtype, stop_gradient=True) + dtype=input.dtype, stop_gradient=True + ) if in_dygraph_mode(): - pre_act = _C_ops.group_norm(input, self.weight, self.bias, - self._epsilon, self._num_groups, "NCHW") + pre_act = _C_ops.group_norm( + input, + self.weight, + self.bias, + self._epsilon, + self._num_groups, + "NCHW", + ) - return dygraph_utils._append_activation_in_dygraph(pre_act, - act=None) + return dygraph_utils._append_activation_in_dygraph( + pre_act, act=None + ) elif _in_legacy_dygraph(): pre_act, _, _ = _legacy_C_ops.group_norm( @@ -416,8 +448,9 @@ class GroupNorm(Layer): 'groups', self._num_groups, ) - return dygraph_utils._append_activation_in_dygraph(pre_act, - act=None) + return dygraph_utils._append_activation_in_dygraph( + pre_act, act=None + ) inputs = {'X': input} if self.bias is not None: @@ -427,25 +460,26 @@ class GroupNorm(Layer): # create output group_norm_out = self._helper.create_variable_for_type_inference( - dtype=input.dtype) - - self._helper.append_op(type="group_norm", - inputs=inputs, - outputs={ - "Y": group_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": self._epsilon, - "groups": self._num_groups - }) + dtype=input.dtype + ) + + self._helper.append_op( + type="group_norm", + inputs=inputs, + outputs={ + "Y": group_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": self._epsilon, "groups": self._num_groups}, + ) return self._helper.append_activation(group_norm_out, None) def extra_repr(self): return 'num_groups={}, num_channels={}, epsilon={}'.format( - self._num_groups, self._num_channels, self._epsilon) + self._num_groups, self._num_channels, self._epsilon + ) class LayerNorm(Layer): @@ -506,12 +540,14 @@ class LayerNorm(Layer): print(layer_norm_out) """ - def __init__(self, - normalized_shape, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - name=None): + def __init__( + self, + normalized_shape, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + name=None, + ): super(LayerNorm, self).__init__() if isinstance(normalized_shape, numbers.Integral): normalized_shape = [normalized_shape] @@ -528,25 +564,29 @@ class LayerNorm(Layer): self.weight = self.create_parameter( attr=self._weight_attr, shape=param_shape, - default_initializer=Constant(1.0)) + default_initializer=Constant(1.0), + ) if bias_attr is False: self.bias = None else: - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, shape=param_shape, is_bias=True + ) def forward(self, input): - return layer_norm(input, - normalized_shape=self._normalized_shape, - weight=self.weight, - bias=self.bias, - epsilon=self._epsilon) + return layer_norm( + input, + normalized_shape=self._normalized_shape, + weight=self.weight, + bias=self.bias, + epsilon=self._epsilon, + ) def extra_repr(self): - return 'normalized_shape={}, epsilon={}'.format(self._normalized_shape, - self._epsilon) + return 'normalized_shape={}, epsilon={}'.format( + self._normalized_shape, self._epsilon + ) class _BatchNormBase(Layer): @@ -554,15 +594,17 @@ class _BatchNormBase(Layer): BatchNorm base . """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCHW', - use_global_stats=None, - name=None): + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCHW', + use_global_stats=None, + name=None, + ): super(_BatchNormBase, self).__init__() self._num_features = num_features self._weight_attr = weight_attr @@ -582,29 +624,40 @@ class _BatchNormBase(Layer): attr=None, shape=param_shape, dtype=self._dtype, - default_initializer=Constant(1.0)) + default_initializer=Constant(1.0), + ) self.weight.stop_gradient = True else: self.weight = self.create_parameter( attr=self._weight_attr, shape=param_shape, dtype=self._dtype, - default_initializer=Constant(1.0)) - self.weight.stop_gradient = self._weight_attr != None and self._weight_attr.learning_rate == 0. + default_initializer=Constant(1.0), + ) + self.weight.stop_gradient = ( + self._weight_attr != None + and self._weight_attr.learning_rate == 0.0 + ) if bias_attr == False: - self.bias = self.create_parameter(attr=None, - shape=param_shape, - dtype=self._dtype, - default_initializer=Constant(0.0), - is_bias=True) + self.bias = self.create_parameter( + attr=None, + shape=param_shape, + dtype=self._dtype, + default_initializer=Constant(0.0), + is_bias=True, + ) self.bias.stop_gradient = True else: - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - dtype=self._dtype, - is_bias=True) - self.bias.stop_gradient = self._bias_attr != None and self._bias_attr.learning_rate == 0. + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=True, + ) + self.bias.stop_gradient = ( + self._bias_attr != None and self._bias_attr.learning_rate == 0.0 + ) moving_mean_name = None moving_variance_name = None @@ -613,22 +666,28 @@ class _BatchNormBase(Layer): moving_mean_name = name + "_mean" moving_variance_name = name + "_variance" - self._mean = self.create_parameter(dtype=self._dtype, - attr=ParamAttr( - name=moving_mean_name, - initializer=Constant(0.0), - trainable=False, - do_model_average=True), - shape=param_shape) + self._mean = self.create_parameter( + dtype=self._dtype, + attr=ParamAttr( + name=moving_mean_name, + initializer=Constant(0.0), + trainable=False, + do_model_average=True, + ), + shape=param_shape, + ) self._mean.stop_gradient = True - self._variance = self.create_parameter(dtype=self._dtype, - attr=ParamAttr( - name=moving_variance_name, - initializer=Constant(1.0), - trainable=False, - do_model_average=True), - shape=param_shape) + self._variance = self.create_parameter( + dtype=self._dtype, + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + do_model_average=True, + ), + shape=param_shape, + ) self._variance.stop_gradient = True self._data_format = data_format @@ -652,22 +711,26 @@ class _BatchNormBase(Layer): if self.training: warnings.warn( - "When training, we now always track global mean and variance.") - - return batch_norm(input, - self._mean, - self._variance, - weight=self.weight, - bias=self.bias, - training=self.training, - momentum=self._momentum, - epsilon=self._epsilon, - data_format=self._data_format, - use_global_stats=self._use_global_stats) + "When training, we now always track global mean and variance." + ) + + return batch_norm( + input, + self._mean, + self._variance, + weight=self.weight, + bias=self.bias, + training=self.training, + momentum=self._momentum, + epsilon=self._epsilon, + data_format=self._data_format, + use_global_stats=self._use_global_stats, + ) def extra_repr(self): main_str = 'num_features={}, momentum={}, epsilon={}'.format( - self._num_features, self._momentum, self._epsilon) + self._num_features, self._momentum, self._epsilon + ) if self._data_format != 'NCHW': main_str += ', data_format={}'.format(self._data_format) if self._name is not None: @@ -747,18 +810,27 @@ class BatchNorm1D(_BatchNormBase): print(batch_norm_out) """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCL', - use_global_stats=None, - name=None): - super(BatchNorm1D, - self).__init__(num_features, momentum, epsilon, weight_attr, - bias_attr, data_format, use_global_stats, name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCL', + use_global_stats=None, + name=None, + ): + super(BatchNorm1D, self).__init__( + num_features, + momentum, + epsilon, + weight_attr, + bias_attr, + data_format, + use_global_stats, + name, + ) def _check_data_format(self, input): if input == 'NCHW' or input == 'NC' or input == 'NCL': @@ -767,12 +839,16 @@ class BatchNorm1D(_BatchNormBase): self._data_format = "NHWC" else: raise ValueError( - 'expected NC , NCL, NLC or None for data_format input') + 'expected NC , NCL, NLC or None for data_format input' + ) def _check_input_dim(self, input): if len(input.shape) != 2 and len(input.shape) != 3: - raise ValueError('expected 2D or 3D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 2D or 3D input (got {}D input)'.format( + len(input.shape) + ) + ) class BatchNorm2D(_BatchNormBase): @@ -856,8 +932,9 @@ class BatchNorm2D(_BatchNormBase): def _check_input_dim(self, input): if len(input.shape) != 4: - raise ValueError('expected 4D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 4D input (got {}D input)'.format(len(input.shape)) + ) class BatchNorm3D(_BatchNormBase): @@ -931,18 +1008,27 @@ class BatchNorm3D(_BatchNormBase): print(batch_norm_out) """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCDHW', - use_global_stats=None, - name=None): - super(BatchNorm3D, - self).__init__(num_features, momentum, epsilon, weight_attr, - bias_attr, data_format, use_global_stats, name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCDHW', + use_global_stats=None, + name=None, + ): + super(BatchNorm3D, self).__init__( + num_features, + momentum, + epsilon, + weight_attr, + bias_attr, + data_format, + use_global_stats, + name, + ) def _check_data_format(self, input): if input == 'NCHW' or input == 'NCDHW': @@ -951,12 +1037,14 @@ class BatchNorm3D(_BatchNormBase): self._data_format = 'NHWC' else: raise ValueError( - 'expected NCDHW, NDHWC or None for data_format input') + 'expected NCDHW, NDHWC or None for data_format input' + ) def _check_input_dim(self, input): if len(input.shape) != 5: - raise ValueError('expected 5D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 5D input (got {}D input)'.format(len(input.shape)) + ) class SyncBatchNorm(_BatchNormBase): @@ -1046,17 +1134,26 @@ class SyncBatchNorm(_BatchNormBase): # [[[[0.26824948, 1.0936325],[0.26824948, -1.6301316]],[[ 0.8095662, -0.665287],[-1.2744656, 1.1301866 ]]]] """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCHW', - name=None): - super(SyncBatchNorm, - self).__init__(num_features, momentum, epsilon, weight_attr, - bias_attr, data_format, None, name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCHW', + name=None, + ): + super(SyncBatchNorm, self).__init__( + num_features, + momentum, + epsilon, + weight_attr, + bias_attr, + data_format, + None, + name, + ) def _check_data_format(self): if self._data_format in ['NCHW', 'NCDHW', 'NC', 'NCL']: @@ -1080,24 +1177,55 @@ class SyncBatchNorm(_BatchNormBase): ### use_global_stats only support False in sync_batch_norm if in_dygraph_mode(): sync_batch_norm_out, _, _, _, _, _ = _C_ops.sync_batch_norm_( - x, self.weight, self.bias, self._mean, self._variance, - self._momentum, self._epsilon, self._data_format, - not self.training, False, False, False) + x, + self.weight, + self.bias, + self._mean, + self._variance, + self._momentum, + self._epsilon, + self._data_format, + not self.training, + False, + False, + False, + ) return sync_batch_norm_out elif in_dynamic_mode(): - attrs = ("momentum", self._momentum, "epsilon", self._epsilon, - "is_test", not self.training, "data_layout", - self._data_format, "use_mkldnn", False, "fuse_with_relu", - False, "use_global_stats", False, 'trainable_statistics', - False) + attrs = ( + "momentum", + self._momentum, + "epsilon", + self._epsilon, + "is_test", + not self.training, + "data_layout", + self._data_format, + "use_mkldnn", + False, + "fuse_with_relu", + False, + "use_global_stats", + False, + 'trainable_statistics', + False, + ) sync_batch_norm_out, _, _, _, _, _ = _legacy_C_ops.sync_batch_norm( - x, self.weight, self.bias, self._mean, self._variance, mean_out, - variance_out, *attrs) + x, + self.weight, + self.bias, + self._mean, + self._variance, + mean_out, + variance_out, + *attrs + ) return sync_batch_norm_out - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'SyncBatchNorm') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'SyncBatchNorm' + ) attrs = { "momentum": self._momentum, @@ -1115,28 +1243,30 @@ class SyncBatchNorm(_BatchNormBase): "Scale": [self.weight], "Bias": [self.bias], "Mean": [self._mean], - "Variance": [self._variance] + "Variance": [self._variance], } saved_mean = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) saved_variance = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) sync_batch_norm_out = self._helper.create_variable_for_type_inference( - self._dtype) + self._dtype + ) outputs = { "Y": [sync_batch_norm_out], "MeanOut": [mean_out], "VarianceOut": [variance_out], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } - self._helper.append_op(type="sync_batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="sync_batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return sync_batch_norm_out @classmethod @@ -1162,18 +1292,28 @@ class SyncBatchNorm(_BatchNormBase): """ layer_output = layer if isinstance(layer, _BatchNormBase): - if layer._weight_attr != None and not isinstance( - layer._weight_attr, - bool) and layer._weight_attr.name != None: + if ( + layer._weight_attr != None + and not isinstance(layer._weight_attr, bool) + and layer._weight_attr.name != None + ): layer._weight_attr.name = layer._weight_attr.name + '_sync' - if layer._bias_attr != None and not isinstance( - layer._bias_attr, bool) and layer._bias_attr.name != None: + if ( + layer._bias_attr != None + and not isinstance(layer._bias_attr, bool) + and layer._bias_attr.name != None + ): layer._bias_attr.name = layer._bias_attr.name + '_sync' - layer_output = SyncBatchNorm(layer._num_features, layer._momentum, - layer._epsilon, layer._weight_attr, - layer._bias_attr, layer._data_format, - layer._name) + layer_output = SyncBatchNorm( + layer._num_features, + layer._momentum, + layer._epsilon, + layer._weight_attr, + layer._bias_attr, + layer._data_format, + layer._name, + ) if layer._weight_attr != False and layer._bias_attr != False: with no_grad(): @@ -1183,58 +1323,61 @@ class SyncBatchNorm(_BatchNormBase): layer_output._variance = layer._variance for name, sublayer in layer.named_children(): - layer_output.add_sublayer(name, - cls.convert_sync_batchnorm(sublayer)) + layer_output.add_sublayer( + name, cls.convert_sync_batchnorm(sublayer) + ) del layer return layer_output class LocalResponseNorm(Layer): """ - Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions. - For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks `_ + Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions. + For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks `_ - See more details in :ref:`api_paddle_nn_functional_local_response_norm` . + See more details in :ref:`api_paddle_nn_functional_local_response_norm` . - Parameters: - size (int): The number of channels to sum over. - alpha (float, optional): The scaling parameter, positive. Default:1e-4 - beta (float, optional): The exponent, positive. Default:0.75 - k (float, optional): An offset, positive. Default: 1.0 - data_format (str, optional): Specify the data format of the input, and the data format of the output - will be consistent with that of the input. An optional string from: - If input is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`, - the data is stored in the order of: `[batch_size, input_channels, feature_length]`. - If input is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`, - the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`. - If input is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`, - the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. - name (str, optional): Name for the operation (optional, default is None). For more information, - please refer to :ref:`api_guide_Name`. - - Shape: - - input: 3-D/4-D/5-D tensor. - - output: 3-D/4-D/5-D tensor, the same shape as input. + Parameters: + size (int): The number of channels to sum over. + alpha (float, optional): The scaling parameter, positive. Default:1e-4 + beta (float, optional): The exponent, positive. Default:0.75 + k (float, optional): An offset, positive. Default: 1.0 + data_format (str, optional): Specify the data format of the input, and the data format of the output + will be consistent with that of the input. An optional string from: + If input is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`, + the data is stored in the order of: `[batch_size, input_channels, feature_length]`. + If input is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`, + the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`. + If input is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`, + the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. + name (str, optional): Name for the operation (optional, default is None). For more information, + please refer to :ref:`api_guide_Name`. - Examples: + Shape: + - input: 3-D/4-D/5-D tensor. + - output: 3-D/4-D/5-D tensor, the same shape as input. - .. code-block:: python + Examples: - import paddle + .. code-block:: python - x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32") - m = paddle.nn.LocalResponseNorm(size=5) - y = m(x) - print(y.shape) # [3, 3, 112, 112] - """ + import paddle + + x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32") + m = paddle.nn.LocalResponseNorm(size=5) + y = m(x) + print(y.shape) # [3, 3, 112, 112] + """ - def __init__(self, - size, - alpha=0.0001, - beta=0.75, - k=1.0, - data_format="NCHW", - name=None): + def __init__( + self, + size, + alpha=0.0001, + beta=0.75, + k=1.0, + data_format="NCHW", + name=None, + ): super(LocalResponseNorm, self).__init__() self.size = size self.alpha = alpha @@ -1244,13 +1387,21 @@ class LocalResponseNorm(Layer): self.name = name def forward(self, input): - out = F.local_response_norm(input, self.size, self.alpha, self.beta, - self.k, self.data_format, self.name) + out = F.local_response_norm( + input, + self.size, + self.alpha, + self.beta, + self.k, + self.data_format, + self.name, + ) return out def extra_repr(self): main_str = 'size={}, alpha={}, beta={}, k={}'.format( - self.size, self.alpha, self.beta, self.k) + self.size, self.alpha, self.beta, self.k + ) if self.data_format != 'NCHW': main_str += ', data_format={}'.format(self.data_format) if self.name is not None: diff --git a/python/paddle/nn/layer/pooling.py b/python/paddle/nn/layer/pooling.py index 3e47275410d..a9b5af5199f 100755 --- a/python/paddle/nn/layer/pooling.py +++ b/python/paddle/nn/layer/pooling.py @@ -76,13 +76,15 @@ class AvgPool1D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - exclusive=True, - ceil_mode=False, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + exclusive=True, + ceil_mode=False, + name=None, + ): super(AvgPool1D, self).__init__() self.kernel_size = kernel_size self.stride = stride @@ -92,13 +94,21 @@ class AvgPool1D(Layer): self.name = name def forward(self, x): - out = F.avg_pool1d(x, self.kernel_size, self.stride, self.padding, - self.exclusive, self.ceil_mode, self.name) + out = F.avg_pool1d( + x, + self.kernel_size, + self.stride, + self.padding, + self.exclusive, + self.ceil_mode, + self.name, + ) return out def extra_repr(self): return 'kernel_size={kernel_size}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class AvgPool2D(Layer): @@ -173,15 +183,17 @@ class AvgPool2D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - exclusive=True, - divisor_override=None, - data_format="NCHW", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + exclusive=True, + divisor_override=None, + data_format="NCHW", + name=None, + ): super(AvgPool2D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -193,19 +205,22 @@ class AvgPool2D(Layer): self.name = name def forward(self, x): - return F.avg_pool2d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - ceil_mode=self.ceil_mode, - exclusive=self.exclusive, - divisor_override=self.divisor, - data_format=self.data_format, - name=self.name) + return F.avg_pool2d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + ceil_mode=self.ceil_mode, + exclusive=self.exclusive, + divisor_override=self.divisor, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class AvgPool3D(Layer): @@ -266,15 +281,17 @@ class AvgPool3D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - exclusive=True, - divisor_override=None, - data_format="NCDHW", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + exclusive=True, + divisor_override=None, + data_format="NCDHW", + name=None, + ): super(AvgPool3D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -286,19 +303,22 @@ class AvgPool3D(Layer): self.name = name def forward(self, x): - return F.avg_pool3d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - ceil_mode=self.ceil_mode, - exclusive=self.exclusive, - divisor_override=self.divisor, - data_format=self.data_format, - name=self.name) + return F.avg_pool3d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + ceil_mode=self.ceil_mode, + exclusive=self.exclusive, + divisor_override=self.divisor, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class MaxPool1D(Layer): @@ -362,13 +382,15 @@ class MaxPool1D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + name=None, + ): super(MaxPool1D, self).__init__() self.kernel_size = kernel_size self.stride = stride @@ -378,13 +400,21 @@ class MaxPool1D(Layer): self.name = name def forward(self, input): - out = F.max_pool1d(input, self.kernel_size, self.stride, self.padding, - self.return_mask, self.ceil_mode, self.name) + out = F.max_pool1d( + input, + self.kernel_size, + self.stride, + self.padding, + self.return_mask, + self.ceil_mode, + self.name, + ) return out def extra_repr(self): return 'kernel_size={kernel_size}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class MaxPool2D(Layer): @@ -460,14 +490,16 @@ class MaxPool2D(Layer): # output.shape [1, 3, 16, 16], max_indices.shape [1, 3, 16, 16], """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NCHW", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NCHW", + name=None, + ): super(MaxPool2D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -478,18 +510,21 @@ class MaxPool2D(Layer): self.name = name def forward(self, x): - return F.max_pool2d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - return_mask=self.return_mask, - ceil_mode=self.ceil_mode, - data_format=self.data_format, - name=self.name) + return F.max_pool2d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + return_mask=self.return_mask, + ceil_mode=self.ceil_mode, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class MaxPool3D(Layer): @@ -553,14 +588,16 @@ class MaxPool3D(Layer): # output.shape [1, 2, 3, 16, 16], max_indices.shape [1, 2, 3, 16, 16], """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NCDHW", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NCDHW", + name=None, + ): super(MaxPool3D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -571,18 +608,21 @@ class MaxPool3D(Layer): self.name = name def forward(self, x): - return F.max_pool3d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - return_mask=self.return_mask, - ceil_mode=self.ceil_mode, - data_format=self.data_format, - name=self.name) + return F.max_pool3d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + return_mask=self.return_mask, + ceil_mode=self.ceil_mode, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class AdaptiveAvgPool1D(Layer): @@ -724,10 +764,12 @@ class AdaptiveAvgPool2D(Layer): self._name = name def forward(self, x): - return F.adaptive_avg_pool2d(x, - output_size=self._output_size, - data_format=self._data_format, - name=self._name) + return F.adaptive_avg_pool2d( + x, + output_size=self._output_size, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): return 'output_size={}'.format(self._output_size) @@ -816,10 +858,12 @@ class AdaptiveAvgPool3D(Layer): self._name = name def forward(self, x): - return F.adaptive_avg_pool3d(x, - output_size=self._output_size, - data_format=self._data_format, - name=self._name) + return F.adaptive_avg_pool3d( + x, + output_size=self._output_size, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): return 'output_size={}'.format(self._output_size) @@ -898,12 +942,14 @@ class AdaptiveMaxPool1D(Layer): self.name = name def forward(self, input): - return F.adaptive_max_pool1d(input, self.output_size, self.return_mask, - self.name) + return F.adaptive_max_pool1d( + input, self.output_size, self.return_mask, self.name + ) def extra_repr(self): - return 'output_size={}, return_mask={}'.format(self.output_size, - self.return_mask) + return 'output_size={}, return_mask={}'.format( + self.output_size, self.return_mask + ) class AdaptiveMaxPool2D(Layer): @@ -976,14 +1022,17 @@ class AdaptiveMaxPool2D(Layer): self._name = name def forward(self, x): - return F.adaptive_max_pool2d(x, - output_size=self._output_size, - return_mask=self._return_mask, - name=self._name) + return F.adaptive_max_pool2d( + x, + output_size=self._output_size, + return_mask=self._return_mask, + name=self._name, + ) def extra_repr(self): - return 'output_size={}, return_mask={}'.format(self._output_size, - self._return_mask) + return 'output_size={}, return_mask={}'.format( + self._output_size, self._return_mask + ) class AdaptiveMaxPool3D(Layer): @@ -1068,14 +1117,17 @@ class AdaptiveMaxPool3D(Layer): self._name = name def forward(self, x): - return F.adaptive_max_pool3d(x, - output_size=self._output_size, - return_mask=self._return_mask, - name=self._name) + return F.adaptive_max_pool3d( + x, + output_size=self._output_size, + return_mask=self._return_mask, + name=self._name, + ) def extra_repr(self): - return 'output_size={}, return_mask={}'.format(self._output_size, - self._return_mask) + return 'output_size={}, return_mask={}'.format( + self._output_size, self._return_mask + ) class MaxUnPool1D(Layer): @@ -1130,13 +1182,15 @@ class MaxUnPool1D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - data_format="NCL", - output_size=None, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + data_format="NCL", + output_size=None, + name=None, + ): super(MaxUnPool1D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -1146,14 +1200,16 @@ class MaxUnPool1D(Layer): self.name = name def forward(self, x, indices): - return F.max_unpool1d(x, - indices, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - data_format=self.data_format, - output_size=self.output_size, - name=self.name) + return F.max_unpool1d( + x, + indices, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + data_format=self.data_format, + output_size=self.output_size, + name=self.name, + ) def extra_repr(self): return 'output_size={}'.format(self.output_size) @@ -1214,13 +1270,15 @@ class MaxUnPool2D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - data_format="NCHW", - output_size=None, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + data_format="NCHW", + output_size=None, + name=None, + ): super(MaxUnPool2D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -1230,14 +1288,16 @@ class MaxUnPool2D(Layer): self.name = name def forward(self, x, indices): - return F.max_unpool2d(x, - indices, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - data_format=self.data_format, - output_size=self.output_size, - name=self.name) + return F.max_unpool2d( + x, + indices, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + data_format=self.data_format, + output_size=self.output_size, + name=self.name, + ) def extra_repr(self): return 'output_size={}'.format(self.output_size) @@ -1302,13 +1362,15 @@ class MaxUnPool3D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - data_format="NCDHW", - output_size=None, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + data_format="NCDHW", + output_size=None, + name=None, + ): super(MaxUnPool3D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -1318,14 +1380,16 @@ class MaxUnPool3D(Layer): self.name = name def forward(self, x, indices): - return F.max_unpool3d(x, - indices, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - data_format=self.data_format, - output_size=self.output_size, - name=self.name) + return F.max_unpool3d( + x, + indices, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + data_format=self.data_format, + output_size=self.output_size, + name=self.name, + ) def extra_repr(self): return 'output_size={}'.format(self.output_size) diff --git a/python/paddle/nn/layer/rnn.py b/python/paddle/nn/layer/rnn.py index 2821277c8f8..7e0fa6d7d70 100644 --- a/python/paddle/nn/layer/rnn.py +++ b/python/paddle/nn/layer/rnn.py @@ -29,6 +29,7 @@ from paddle.fluid.framework import in_dygraph_mode from paddle.framework import core from paddle.static import default_startup_program from paddle.static import program_guard + try: from collections.abc import Sequence except: @@ -139,12 +140,9 @@ class RNNCellBase(Layer): and mostly used in RNN. """ - def get_initial_states(self, - batch_ref, - shape=None, - dtype=None, - init_value=0., - batch_dim_idx=0): + def get_initial_states( + self, batch_ref, shape=None, dtype=None, init_value=0.0, batch_dim_idx=0 + ): r""" Generate initialized states according to provided shape, data type and value. @@ -180,17 +178,17 @@ class RNNCellBase(Layer): def _is_shape_sequence(seq): """For shape, list/tuple of integer is the finest-grained objection""" - if (isinstance(seq, list) or isinstance(seq, tuple)): - if reduce(lambda flag, x: isinstance(x, int) and flag, seq, - True): + if isinstance(seq, list) or isinstance(seq, tuple): + if reduce( + lambda flag, x: isinstance(x, int) and flag, seq, True + ): return False # TODO: Add check for the illegal if isinstance(seq, dict): return True - return (isinstance(seq, Sequence) and not isinstance(seq, str)) + return isinstance(seq, Sequence) and not isinstance(seq, str) class Shape(object): - def __init__(self, shape): self.shape = shape if shape[0] == -1 else ([-1] + list(shape)) @@ -211,13 +209,16 @@ class RNNCellBase(Layer): states_dtypes = map_structure(lambda shape: dtype, states_shapes) init_states = map_structure( - lambda shape, dtype: paddle.fluid.layers. - fill_constant_batch_size_like(input=batch_ref, - shape=shape.shape, - dtype=dtype, - value=init_value, - input_dim_idx=batch_dim_idx), - states_shapes, states_dtypes) + lambda shape, dtype: paddle.fluid.layers.fill_constant_batch_size_like( + input=batch_ref, + shape=shape.shape, + dtype=dtype, + value=init_value, + input_dim_idx=batch_dim_idx, + ), + states_shapes, + states_dtypes, + ) return init_states @property @@ -233,7 +234,8 @@ class RNNCellBase(Layer): `get_initial_states`. """ raise NotImplementedError( - "Please add implementaion for `state_shape` in the used cell.") + "Please add implementaion for `state_shape` in the used cell." + ) @property def state_dtype(self): @@ -248,7 +250,8 @@ class RNNCellBase(Layer): `get_initial_states`. """ raise NotImplementedError( - "Please add implementaion for `state_dtype` in the used cell.") + "Please add implementaion for `state_dtype` in the used cell." + ) class SimpleRNNCell(RNNCellBase): @@ -318,50 +321,57 @@ class SimpleRNNCell(RNNCellBase): """ - def __init__(self, - input_size, - hidden_size, - activation="tanh", - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): + def __init__( + self, + input_size, + hidden_size, + activation="tanh", + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): super(SimpleRNNCell, self).__init__() if hidden_size <= 0: raise ValueError( - "hidden_size of {} must be greater than 0, but now equals to {}" - .format(self.__class__.__name__, hidden_size)) + "hidden_size of {} must be greater than 0, but now equals to {}".format( + self.__class__.__name__, hidden_size + ) + ) std = 1.0 / math.sqrt(hidden_size) self.weight_ih = self.create_parameter( (hidden_size, input_size), weight_ih_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.weight_hh = self.create_parameter( (hidden_size, hidden_size), weight_hh_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_ih = self.create_parameter( - (hidden_size, ), + (hidden_size,), bias_ih_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_hh = self.create_parameter( - (hidden_size, ), + (hidden_size,), bias_hh_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.input_size = input_size self.hidden_size = hidden_size if activation not in ["tanh", "relu"]: raise ValueError( "activation for SimpleRNNCell should be tanh or relu, " - "but get {}".format(activation)) + "but get {}".format(activation) + ) self.activation = activation - self._activation_fn = paddle.tanh \ - if activation == "tanh" \ - else F.relu + self._activation_fn = paddle.tanh if activation == "tanh" else F.relu def forward(self, inputs, states=None): if states is None: @@ -378,7 +388,7 @@ class SimpleRNNCell(RNNCellBase): @property def state_shape(self): - return (self.hidden_size, ) + return (self.hidden_size,) def extra_repr(self): s = '{input_size}, {hidden_size}' @@ -471,38 +481,46 @@ class LSTMCell(RNNCellBase): """ - def __init__(self, - input_size, - hidden_size, - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): + def __init__( + self, + input_size, + hidden_size, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): super(LSTMCell, self).__init__() if hidden_size <= 0: raise ValueError( - "hidden_size of {} must be greater than 0, but now equals to {}" - .format(self.__class__.__name__, hidden_size)) + "hidden_size of {} must be greater than 0, but now equals to {}".format( + self.__class__.__name__, hidden_size + ) + ) std = 1.0 / math.sqrt(hidden_size) self.weight_ih = self.create_parameter( (4 * hidden_size, input_size), weight_ih_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.weight_hh = self.create_parameter( (4 * hidden_size, hidden_size), weight_hh_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_ih = self.create_parameter( - (4 * hidden_size, ), + (4 * hidden_size,), bias_ih_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_hh = self.create_parameter( - (4 * hidden_size, ), + (4 * hidden_size,), bias_hh_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.hidden_size = hidden_size self.input_size = input_size @@ -538,7 +556,7 @@ class LSTMCell(RNNCellBase): automatically inserted into shape). These two shapes correspond to :math:`h_{t-1}` and :math:`c_{t-1}` separately. """ - return ((self.hidden_size, ), (self.hidden_size, )) + return ((self.hidden_size,), (self.hidden_size,)) def extra_repr(self): return '{input_size}, {hidden_size}'.format(**self.__dict__) @@ -622,38 +640,46 @@ class GRUCell(RNNCellBase): """ - def __init__(self, - input_size, - hidden_size, - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): + def __init__( + self, + input_size, + hidden_size, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): super(GRUCell, self).__init__() if hidden_size <= 0: raise ValueError( - "hidden_size of {} must be greater than 0, but now equals to {}" - .format(self.__class__.__name__, hidden_size)) + "hidden_size of {} must be greater than 0, but now equals to {}".format( + self.__class__.__name__, hidden_size + ) + ) std = 1.0 / math.sqrt(hidden_size) self.weight_ih = self.create_parameter( (3 * hidden_size, input_size), weight_ih_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.weight_hh = self.create_parameter( (3 * hidden_size, hidden_size), weight_hh_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_ih = self.create_parameter( - (3 * hidden_size, ), + (3 * hidden_size,), bias_ih_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_hh = self.create_parameter( - (3 * hidden_size, ), + (3 * hidden_size,), bias_hh_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.hidden_size = hidden_size self.input_size = input_size @@ -689,7 +715,7 @@ class GRUCell(RNNCellBase): size would be automatically inserted into shape). The shape corresponds to the shape of :math:`h_{t-1}`. """ - return (self.hidden_size, ) + return (self.hidden_size,) def extra_repr(self): return '{input_size}, {hidden_size}'.format(**self.__dict__) @@ -754,11 +780,9 @@ class RNN(Layer): self.is_reverse = is_reverse self.time_major = time_major - def forward(self, - inputs, - initial_states=None, - sequence_length=None, - **kwargs): + def forward( + self, inputs, initial_states=None, sequence_length=None, **kwargs + ): final_outputs, final_states = paddle.fluid.layers.rnn( self.cell, inputs, @@ -766,7 +790,8 @@ class RNN(Layer): sequence_length=sequence_length, time_major=self.time_major, is_reverse=self.is_reverse, - **kwargs) + **kwargs + ) return final_outputs, final_states @@ -825,27 +850,35 @@ class BiRNN(Layer): self.cell_fw = cell_fw self.cell_bw = cell_bw if cell_fw.input_size != cell_bw.input_size: - raise ValueError("input size of forward cell({}) does not equals" - "that of backward cell({})".format( - cell_fw.input_size, cell_bw.input_size)) + raise ValueError( + "input size of forward cell({}) does not equals" + "that of backward cell({})".format( + cell_fw.input_size, cell_bw.input_size + ) + ) for cell in [self.cell_fw, self.cell_bw]: if not hasattr(cell, "call"): # for non-dygraph mode, `rnn` api uses cell.call cell.call = cell.forward self.time_major = time_major - def forward(self, - inputs, - initial_states=None, - sequence_length=None, - **kwargs): + def forward( + self, inputs, initial_states=None, sequence_length=None, **kwargs + ): if isinstance(initial_states, (list, tuple)): - assert len(initial_states) == 2, \ - "length of initial_states should be 2 when it is a list/tuple" + assert ( + len(initial_states) == 2 + ), "length of initial_states should be 2 when it is a list/tuple" outputs, final_states = paddle.fluid.layers.birnn( - self.cell_fw, self.cell_bw, inputs, initial_states, sequence_length, - self.time_major, **kwargs) + self.cell_fw, + self.cell_bw, + inputs, + initial_states, + sequence_length, + self.time_major, + **kwargs + ) return outputs, final_states @@ -855,18 +888,20 @@ class RNNBase(LayerList): and other common methods for SimpleRNN, LSTM and GRU. """ - def __init__(self, - mode, - input_size, - hidden_size, - num_layers=1, - direction="forward", - time_major=False, - dropout=0., - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None): + def __init__( + self, + mode, + input_size, + hidden_size, + num_layers=1, + direction="forward", + time_major=False, + dropout=0.0, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + ): super(RNNBase, self).__init__() bidirectional_list = ["bidirectional", "bidirect"] self.mode = mode @@ -882,7 +917,7 @@ class RNNBase(LayerList): "weight_ih_attr": weight_ih_attr, "weight_hh_attr": weight_hh_attr, "bias_ih_attr": bias_ih_attr, - "bias_hh_attr": bias_hh_attr + "bias_hh_attr": bias_hh_attr, } if mode == "LSTM": @@ -911,11 +946,13 @@ class RNNBase(LayerList): else: raise ValueError( "direction should be forward or bidirect (or bidirectional), " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.could_use_cudnn = True self.could_use_cudnn &= len(self.parameters()) == num_layers * 4 * ( - 2 if direction in bidirectional_list else 1) + 2 if direction in bidirectional_list else 1 + ) # Expose params as RNN's attribute, which can make it compatible when # replacing small ops composed rnn with cpp rnn kernel. @@ -927,8 +964,10 @@ class RNNBase(LayerList): for direction in range(self.num_directions): suffix = '_reverse' if direction == 1 else '' param_names.extend(['weight_ih_l{}{}', 'weight_hh_l{}{}']) - if bias_ih_attr != False: param_names.append('bias_ih_l{}{}') - if bias_hh_attr != False: param_names.append('bias_hh_l{}{}') + if bias_ih_attr != False: + param_names.append('bias_ih_l{}{}') + if bias_hh_attr != False: + param_names.append('bias_hh_l{}{}') param_names = [x.format(layer, suffix) for x in param_names] for name, param in zip(param_names, self.parameters()): setattr(self, name, param) @@ -949,8 +988,11 @@ class RNNBase(LayerList): shape = [np.prod(param.shape) for param in params] self._all_weights = [None] * len(params) for i, param in enumerate(params): - offset = 0 if i % 4 < 2 else (2 * self.num_layers * - self.num_directions) + offset = ( + 0 + if i % 4 < 2 + else (2 * self.num_layers * self.num_directions) + ) layer_idx = i // 4 self._all_weights[offset + layer_idx * 2 + i % 2] = param # Wrap using a list to avoid registed into params and saving, maybe @@ -958,38 +1000,49 @@ class RNNBase(LayerList): # add both to main_program and startup_program for static-graph. # Use Constant initializer to avoid make effect on random generator. self._flat_weight = [ - self.create_parameter(shape=[np.sum(shape)], - dtype=params[0].dtype, - default_initializer=I.Constant(0.0)) + self.create_parameter( + shape=[np.sum(shape)], + dtype=params[0].dtype, + default_initializer=I.Constant(0.0), + ) ] # dropout state may also can be hided and avoid saving # should dropout state be persistable for static-graph self._dropout_state = self.create_variable( - dtype=core.VarDesc.VarType.UINT8) + dtype=core.VarDesc.VarType.UINT8 + ) if in_dynamic_mode(): with paddle.no_grad(): - _legacy_C_ops.coalesce_tensor(self._all_weights, - self._all_weights, - self._flat_weight[0], - "copy_data", True, - "use_align", False, "dtype", - params[0].dtype) + _legacy_C_ops.coalesce_tensor( + self._all_weights, + self._all_weights, + self._flat_weight[0], + "copy_data", + True, + "use_align", + False, + "dtype", + params[0].dtype, + ) return # for static-graph, append coalesce_tensor into startup program - with program_guard(default_startup_program(), - default_startup_program()): + with program_guard( + default_startup_program(), default_startup_program() + ): with paddle.no_grad(): - self._helper.append_op(type="coalesce_tensor", - inputs={"Input": self._all_weights}, - outputs={ - "Output": self._all_weights, - "FusedOutput": self._flat_weight - }, - attrs={ - "copy_data": True, - "use_align": False, - "dtype": params[0].dtype - }) + self._helper.append_op( + type="coalesce_tensor", + inputs={"Input": self._all_weights}, + outputs={ + "Output": self._all_weights, + "FusedOutput": self._flat_weight, + }, + attrs={ + "copy_data": True, + "use_align": False, + "dtype": params[0].dtype, + }, + ) def _cudnn_impl(self, inputs, initial_states, sequence_length): if not self.time_major: @@ -997,18 +1050,43 @@ class RNNBase(LayerList): if in_dygraph_mode(): out, _, state = _C_ops.rnn( - inputs, initial_states, self._all_weights, sequence_length, - self._dropout_state, self.dropout, self.num_directions == 2, - self.input_size, self.hidden_size, self.num_layers, self.mode, - 0, not self.training) + inputs, + initial_states, + self._all_weights, + sequence_length, + self._dropout_state, + self.dropout, + self.num_directions == 2, + self.input_size, + self.hidden_size, + self.num_layers, + self.mode, + 0, + not self.training, + ) elif in_dynamic_mode(): _, _, out, state = _legacy_C_ops.rnn( - inputs, initial_states, self._all_weights, sequence_length, - self._dropout_state, self.state_components, 'dropout_prob', - self.dropout, 'is_bidirec', self.num_directions == 2, - 'input_size', self.input_size, 'hidden_size', self.hidden_size, - 'num_layers', self.num_layers, 'mode', self.mode, 'is_test', - not self.training) + inputs, + initial_states, + self._all_weights, + sequence_length, + self._dropout_state, + self.state_components, + 'dropout_prob', + self.dropout, + 'is_bidirec', + self.num_directions == 2, + 'input_size', + self.input_size, + 'hidden_size', + self.hidden_size, + 'num_layers', + self.num_layers, + 'mode', + self.mode, + 'is_test', + not self.training, + ) else: out = self._helper.create_variable_for_type_inference(inputs.dtype) state = [ @@ -1016,13 +1094,14 @@ class RNNBase(LayerList): for i in range(self.state_components) ] reserve = self._helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) inputs = { 'Input': inputs, 'WeightList': self._all_weights, 'PreState': initial_states, - 'SequenceLength': sequence_length + 'SequenceLength': sequence_length, } attrs = { 'dropout_prob': self.dropout, @@ -1031,7 +1110,7 @@ class RNNBase(LayerList): 'hidden_size': self.hidden_size, 'num_layers': self.num_layers, 'mode': self.mode, - 'is_test': not self.training + 'is_test': not self.training, } outputs = { @@ -1041,51 +1120,67 @@ class RNNBase(LayerList): 'DropoutState': self._dropout_state, } - self._helper.append_op(type="rnn", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="rnn", inputs=inputs, outputs=outputs, attrs=attrs + ) - out = paddle.tensor.transpose(out, - [1, 0, 2]) if not self.time_major else out + out = ( + paddle.tensor.transpose(out, [1, 0, 2]) + if not self.time_major + else out + ) return out, tuple(state) if len(state) > 1 else state[0] def forward(self, inputs, initial_states=None, sequence_length=None): batch_index = 1 if self.time_major else 0 dtype = inputs.dtype if initial_states is None: - state_shape = (self.num_layers * self.num_directions, -1, - self.hidden_size) - initial_states = tuple([ - paddle.fluid.layers.fill_constant_batch_size_like( - inputs, state_shape, dtype, 0, batch_index, 1) - for _ in range(self.state_components) - ]) + state_shape = ( + self.num_layers * self.num_directions, + -1, + self.hidden_size, + ) + initial_states = tuple( + [ + paddle.fluid.layers.fill_constant_batch_size_like( + inputs, state_shape, dtype, 0, batch_index, 1 + ) + for _ in range(self.state_components) + ] + ) else: - initial_states = [initial_states] if isinstance( - initial_states, paddle.static.Variable) else initial_states - - if self.could_use_cudnn and (not paddle.device.is_compiled_with_rocm() - or sequence_length is None): + initial_states = ( + [initial_states] + if isinstance(initial_states, paddle.static.Variable) + else initial_states + ) + + if self.could_use_cudnn and ( + not paddle.device.is_compiled_with_rocm() or sequence_length is None + ): # Add CPU kernel and dispatch in backend later return self._cudnn_impl(inputs, initial_states, sequence_length) - states = split_states(initial_states, self.num_directions == 2, - self.state_components) + states = split_states( + initial_states, self.num_directions == 2, self.state_components + ) final_states = [] for i, rnn_layer in enumerate(self): if i > 0: - inputs = F.dropout(inputs, - self.dropout, - training=self.training, - mode="upscale_in_train") + inputs = F.dropout( + inputs, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) outputs, final_state = rnn_layer(inputs, states[i], sequence_length) final_states.append(final_state) inputs = outputs - final_states = concat_states(final_states, self.num_directions == 2, - self.state_components) + final_states = concat_states( + final_states, self.num_directions == 2, self.state_components + ) return outputs, final_states def extra_repr(self): @@ -1184,19 +1279,21 @@ class SimpleRNN(RNNBase): """ - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - time_major=False, - dropout=0., - activation="tanh", - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + time_major=False, + dropout=0.0, + activation="tanh", + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): if activation == "tanh": mode = "RNN_TANH" elif activation == "relu": @@ -1204,10 +1301,19 @@ class SimpleRNN(RNNBase): else: raise ValueError("Unknown activation '{}'".format(activation)) self.activation = activation - super(SimpleRNN, - self).__init__(mode, input_size, hidden_size, num_layers, - direction, time_major, dropout, weight_ih_attr, - weight_hh_attr, bias_ih_attr, bias_hh_attr) + super(SimpleRNN, self).__init__( + mode, + input_size, + hidden_size, + num_layers, + direction, + time_major, + dropout, + weight_ih_attr, + weight_hh_attr, + bias_ih_attr, + bias_hh_attr, + ) class LSTM(RNNBase): @@ -1307,22 +1413,33 @@ class LSTM(RNNBase): """ - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - time_major=False, - dropout=0., - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): - super(LSTM, - self).__init__("LSTM", input_size, hidden_size, num_layers, - direction, time_major, dropout, weight_ih_attr, - weight_hh_attr, bias_ih_attr, bias_hh_attr) + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + time_major=False, + dropout=0.0, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): + super(LSTM, self).__init__( + "LSTM", + input_size, + hidden_size, + num_layers, + direction, + time_major, + dropout, + weight_ih_attr, + weight_hh_attr, + bias_ih_attr, + bias_hh_attr, + ) class GRU(RNNBase): @@ -1415,19 +1532,30 @@ class GRU(RNNBase): """ - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - time_major=False, - dropout=0., - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): - super(GRU, - self).__init__("GRU", input_size, hidden_size, num_layers, - direction, time_major, dropout, weight_ih_attr, - weight_hh_attr, bias_ih_attr, bias_hh_attr) + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + time_major=False, + dropout=0.0, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): + super(GRU, self).__init__( + "GRU", + input_size, + hidden_size, + num_layers, + direction, + time_major, + dropout, + weight_ih_attr, + weight_hh_attr, + bias_ih_attr, + bias_hh_attr, + ) diff --git a/python/paddle/nn/layer/transformer.py b/python/paddle/nn/layer/transformer.py index 0ea83b3d84c..4115cbc7a80 100644 --- a/python/paddle/nn/layer/transformer.py +++ b/python/paddle/nn/layer/transformer.py @@ -49,7 +49,8 @@ def _convert_param_attr_to_list(param_attr, n): """ if isinstance(param_attr, (list, tuple)): assert len(param_attr) == n, ( - "length of param_attr should be %d when it is a list/tuple" % n) + "length of param_attr should be %d when it is a list/tuple" % n + ) param_attrs = [] for attr in param_attr: if isinstance(attr, bool): @@ -151,21 +152,27 @@ class MultiHeadAttention(Layer): Cache = collections.namedtuple("Cache", ["k", "v"]) StaticCache = collections.namedtuple("StaticCache", ["k", "v"]) - def __init__(self, - embed_dim, - num_heads, - dropout=0., - kdim=None, - vdim=None, - need_weights=False, - weight_attr=None, - bias_attr=None): + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + kdim=None, + vdim=None, + need_weights=False, + weight_attr=None, + bias_attr=None, + ): super(MultiHeadAttention, self).__init__() - assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " - "but received {}".format(embed_dim)) - assert num_heads > 0, ("Expected num_heads to be greater than 0, " - "but received {}".format(num_heads)) + assert embed_dim > 0, ( + "Expected embed_dim to be greater than 0, " + "but received {}".format(embed_dim) + ) + assert num_heads > 0, ( + "Expected num_heads to be greater than 0, " + "but received {}".format(num_heads) + ) self.embed_dim = embed_dim self.kdim = kdim if kdim is not None else embed_dim @@ -175,24 +182,22 @@ class MultiHeadAttention(Layer): self.need_weights = need_weights self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" - - self.q_proj = Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = Linear(self.kdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = Linear(self.vdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" + + self.q_proj = Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = Linear( + self.kdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = Linear( + self.vdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) def _prepare_qkv(self, query, key, value, cache=None): r""" @@ -338,12 +343,14 @@ class MultiHeadAttention(Layer): input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) v = layers.fill_constant_batch_size_like( input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) return self.Cache(k, v) else: # incremental_state with initial value, mainly for usage like UniLM @@ -410,19 +417,21 @@ class MultiHeadAttention(Layer): q, k, v, cache = self._prepare_qkv(query, key, value, cache) # scale dot product attention - product = paddle.matmul(x=q * (self.head_dim**-0.5), - y=k, - transpose_y=True) + product = paddle.matmul( + x=q * (self.head_dim**-0.5), y=k, transpose_y=True + ) if attn_mask is not None: # Support bool or int mask attn_mask = _convert_attention_mask(attn_mask, product.dtype) product = product + attn_mask weights = F.softmax(product) if self.dropout: - weights = F.dropout(weights, - self.dropout, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -498,30 +507,39 @@ class TransformerEncoderLayer(Layer): enc_output = encoder_layer(enc_input, attn_mask) # [2, 4, 128] """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="relu", - attn_dropout=None, - act_dropout=None, - normalize_before=False, - weight_attr=None, - bias_attr=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="relu", + attn_dropout=None, + act_dropout=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 super(TransformerEncoderLayer, self).__init__() - assert d_model > 0, ("Expected d_model to be greater than 0, " - "but received {}".format(d_model)) - assert nhead > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(nhead)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, " "but received {}".format( + d_model + ) + assert ( + nhead > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + nhead + ) assert dim_feedforward > 0, ( "Expected dim_feedforward to be greater than 0, " - "but received {}".format(dim_feedforward)) + "but received {}".format(dim_feedforward) + ) attn_dropout = dropout if attn_dropout is None else attn_dropout act_dropout = dropout if act_dropout is None else act_dropout @@ -530,20 +548,20 @@ class TransformerEncoderLayer(Layer): weight_attrs = _convert_param_attr_to_list(weight_attr, 2) bias_attrs = _convert_param_attr_to_list(bias_attr, 2) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0]) - self.linear1 = Linear(d_model, - dim_feedforward, - weight_attrs[1], - bias_attr=bias_attrs[1]) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + ) + self.linear1 = Linear( + d_model, dim_feedforward, weight_attrs[1], bias_attr=bias_attrs[1] + ) self.dropout = Dropout(act_dropout, mode="upscale_in_train") - self.linear2 = Linear(dim_feedforward, - d_model, - weight_attrs[1], - bias_attr=bias_attrs[1]) + self.linear2 = Linear( + dim_feedforward, d_model, weight_attrs[1], bias_attr=bias_attrs[1] + ) self.norm1 = LayerNorm(d_model) self.norm2 = LayerNorm(d_model) self.dropout1 = Dropout(dropout, mode="upscale_in_train") @@ -591,8 +609,9 @@ class TransformerEncoderLayer(Layer): if cache is None: src = self.self_attn(src, src, src, src_mask) else: - src, incremental_cache = self.self_attn(src, src, src, src_mask, - cache) + src, incremental_cache = self.self_attn( + src, src, src, src_mask, cache + ) src = residual + self.dropout1(src) if not self.normalize_before: @@ -624,8 +643,9 @@ class TransformerEncoderLayer(Layer): `MultiHeadAttention.gen_cache` and `MultiHeadAttention.forward` \ for more details. """ - incremental_cache = self.self_attn.gen_cache(src, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + src, type=self.self_attn.Cache + ) return incremental_cache @@ -659,10 +679,16 @@ class TransformerEncoder(Layer): def __init__(self, encoder_layer, num_layers, norm=None): super(TransformerEncoder, self).__init__() - self.layers = LayerList([ - (encoder_layer if i == 0 else type(encoder_layer)( - **encoder_layer._config)) for i in range(num_layers) - ]) + self.layers = LayerList( + [ + ( + encoder_layer + if i == 0 + else type(encoder_layer)(**encoder_layer._config) + ) + for i in range(num_layers) + ] + ) self.num_layers = num_layers self.norm = norm @@ -708,9 +734,9 @@ class TransformerEncoder(Layer): if cache is None: output = mod(output, src_mask=src_mask) else: - output, new_cache = mod(output, - src_mask=src_mask, - cache=cache[i]) + output, new_cache = mod( + output, src_mask=src_mask, cache=cache[i] + ) new_caches.append(new_cache) if self.norm is not None: @@ -807,30 +833,39 @@ class TransformerDecoderLayer(Layer): cross_attn_mask) # [2, 4, 128] """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="relu", - attn_dropout=None, - act_dropout=None, - normalize_before=False, - weight_attr=None, - bias_attr=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="relu", + attn_dropout=None, + act_dropout=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 super(TransformerDecoderLayer, self).__init__() - assert d_model > 0, ("Expected d_model to be greater than 0, " - "but received {}".format(d_model)) - assert nhead > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(nhead)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, " "but received {}".format( + d_model + ) + assert ( + nhead > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + nhead + ) assert dim_feedforward > 0, ( "Expected dim_feedforward to be greater than 0, " - "but received {}".format(dim_feedforward)) + "but received {}".format(dim_feedforward) + ) attn_dropout = dropout if attn_dropout is None else attn_dropout act_dropout = dropout if act_dropout is None else act_dropout @@ -839,25 +874,27 @@ class TransformerDecoderLayer(Layer): weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0]) - self.cross_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[1], - bias_attr=bias_attrs[1]) - self.linear1 = Linear(d_model, - dim_feedforward, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + ) + self.cross_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[1], + bias_attr=bias_attrs[1], + ) + self.linear1 = Linear( + d_model, dim_feedforward, weight_attrs[2], bias_attr=bias_attrs[2] + ) self.dropout = Dropout(act_dropout, mode="upscale_in_train") - self.linear2 = Linear(dim_feedforward, - d_model, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.linear2 = Linear( + dim_feedforward, d_model, weight_attrs[2], bias_attr=bias_attrs[2] + ) self.norm1 = LayerNorm(d_model) self.norm2 = LayerNorm(d_model) self.norm3 = LayerNorm(d_model) @@ -922,8 +959,9 @@ class TransformerDecoderLayer(Layer): if cache is None: tgt = self.self_attn(tgt, tgt, tgt, tgt_mask, None) else: - tgt, incremental_cache = self.self_attn(tgt, tgt, tgt, tgt_mask, - cache[0]) + tgt, incremental_cache = self.self_attn( + tgt, tgt, tgt, tgt_mask, cache[0] + ) tgt = residual + self.dropout1(tgt) if not self.normalize_before: tgt = self.norm1(tgt) @@ -934,8 +972,9 @@ class TransformerDecoderLayer(Layer): if cache is None: tgt = self.cross_attn(tgt, memory, memory, memory_mask, None) else: - tgt, static_cache = self.cross_attn(tgt, memory, memory, - memory_mask, cache[1]) + tgt, static_cache = self.cross_attn( + tgt, memory, memory, memory_mask, cache[1] + ) tgt = residual + self.dropout2(tgt) if not self.normalize_before: tgt = self.norm2(tgt) @@ -947,8 +986,9 @@ class TransformerDecoderLayer(Layer): tgt = residual + self.dropout3(tgt) if not self.normalize_before: tgt = self.norm3(tgt) - return tgt if cache is None else (tgt, (incremental_cache, - static_cache)) + return ( + tgt if cache is None else (tgt, (incremental_cache, static_cache)) + ) def gen_cache(self, memory): r""" @@ -972,10 +1012,12 @@ class TransformerDecoderLayer(Layer): See `MultiHeadAttention.gen_cache` and `MultiHeadAttention.forward` \ for more details. """ - incremental_cache = self.self_attn.gen_cache(memory, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + memory, type=self.self_attn.Cache + ) static_cache = self.cross_attn.gen_cache( - memory, memory, type=self.cross_attn.StaticCache) + memory, memory, type=self.cross_attn.StaticCache + ) return incremental_cache, static_cache @@ -1016,10 +1058,16 @@ class TransformerDecoder(Layer): def __init__(self, decoder_layer, num_layers, norm=None): super(TransformerDecoder, self).__init__() - self.layers = LayerList([ - (decoder_layer if i == 0 else type(decoder_layer)( - **decoder_layer._config)) for i in range(num_layers) - ]) + self.layers = LayerList( + [ + ( + decoder_layer + if i == 0 + else type(decoder_layer)(**decoder_layer._config) + ) + for i in range(num_layers) + ] + ) self.num_layers = num_layers self.norm = norm @@ -1077,17 +1125,21 @@ class TransformerDecoder(Layer): new_caches = [] for i, mod in enumerate(self.layers): if cache is None: - output = mod(output, - memory, - tgt_mask=tgt_mask, - memory_mask=memory_mask, - cache=None) + output = mod( + output, + memory, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + cache=None, + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - memory_mask=memory_mask, - cache=cache[i]) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + cache=cache[i], + ) new_caches.append(new_cache) if self.norm is not None: @@ -1217,30 +1269,39 @@ class Transformer(Layer): cross_attn_mask) # [2, 6, 128] """ - def __init__(self, - d_model=512, - nhead=8, - num_encoder_layers=6, - num_decoder_layers=6, - dim_feedforward=2048, - dropout=0.1, - activation="relu", - attn_dropout=None, - act_dropout=None, - normalize_before=False, - weight_attr=None, - bias_attr=None, - custom_encoder=None, - custom_decoder=None): + def __init__( + self, + d_model=512, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + attn_dropout=None, + act_dropout=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + custom_encoder=None, + custom_decoder=None, + ): super(Transformer, self).__init__() - assert d_model > 0, ("Expected d_model to be greater than 0, " - "but received {}".format(d_model)) - assert nhead > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(nhead)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, " "but received {}".format( + d_model + ) + assert ( + nhead > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + nhead + ) assert dim_feedforward > 0, ( "Expected dim_feedforward to be greater than 0, " - "but received {}".format(dim_feedforward)) + "but received {}".format(dim_feedforward) + ) if isinstance(bias_attr, (list, tuple)): if len(bias_attr) == 1: @@ -1253,9 +1314,9 @@ class Transformer(Layer): encoder_bias_attr = [bias_attr[0], bias_attr[-1]] decoder_bias_attr = bias_attr else: - assert False, ( - "length of bias_attr should be 1 or 2 or 3 when it is a list/tuple" - ) + assert ( + False + ), "length of bias_attr should be 1 or 2 or 3 when it is a list/tuple" else: encoder_bias_attr = bias_attr decoder_bias_attr = bias_attr @@ -1267,15 +1328,17 @@ class Transformer(Layer): elif len(weight_attr) == 2: encoder_weight_attr = weight_attr decoder_weight_attr = [ - weight_attr[0], weight_attr[0], weight_attr[-1] + weight_attr[0], + weight_attr[0], + weight_attr[-1], ] elif len(weight_attr) == 3: encoder_weight_attr = [weight_attr[0], weight_attr[-1]] decoder_weight_attr = weight_attr else: - assert False, ( - "length of weight_attr should be 1 or 2 or 3 when it is a list/tuple" - ) + assert ( + False + ), "length of weight_attr should be 1 or 2 or 3 when it is a list/tuple" else: encoder_weight_attr = weight_attr decoder_weight_attr = weight_attr @@ -1284,23 +1347,41 @@ class Transformer(Layer): self.encoder = custom_encoder else: encoder_layer = TransformerEncoderLayer( - d_model, nhead, dim_feedforward, dropout, activation, - attn_dropout, act_dropout, normalize_before, - encoder_weight_attr, encoder_bias_attr) + d_model, + nhead, + dim_feedforward, + dropout, + activation, + attn_dropout, + act_dropout, + normalize_before, + encoder_weight_attr, + encoder_bias_attr, + ) encoder_norm = LayerNorm(d_model) - self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, - encoder_norm) + self.encoder = TransformerEncoder( + encoder_layer, num_encoder_layers, encoder_norm + ) if custom_decoder is not None: self.decoder = custom_decoder else: decoder_layer = TransformerDecoderLayer( - d_model, nhead, dim_feedforward, dropout, activation, - attn_dropout, act_dropout, normalize_before, - decoder_weight_attr, decoder_bias_attr) + d_model, + nhead, + dim_feedforward, + dropout, + activation, + attn_dropout, + act_dropout, + normalize_before, + decoder_weight_attr, + decoder_bias_attr, + ) decoder_norm = LayerNorm(d_model) - self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers, - decoder_norm) + self.decoder = TransformerDecoder( + decoder_layer, num_decoder_layers, decoder_norm + ) self.d_model = d_model self.nhead = nhead @@ -1359,10 +1440,9 @@ class Transformer(Layer): tgt_mask = _convert_attention_mask(tgt_mask, tgt.dtype) memory_mask = _convert_attention_mask(memory_mask, memory.dtype) - output = self.decoder(tgt, - memory, - tgt_mask=tgt_mask, - memory_mask=memory_mask) + output = self.decoder( + tgt, memory, tgt_mask=tgt_mask, memory_mask=memory_mask + ) return output def generate_square_subsequent_mask(self, length): @@ -1397,6 +1477,10 @@ class Transformer(Layer): """ return paddle.tensor.triu( - paddle.full(shape=[length, length], - fill_value=-np.inf, - dtype=paddle.get_default_dtype()), 1) + paddle.full( + shape=[length, length], + fill_value=-np.inf, + dtype=paddle.get_default_dtype(), + ), + 1, + ) diff --git a/python/paddle/nn/layer/vision.py b/python/paddle/nn/layer/vision.py index fe367828815..704baeb98f9 100644 --- a/python/paddle/nn/layer/vision.py +++ b/python/paddle/nn/layer/vision.py @@ -67,16 +67,19 @@ class PixelShuffle(Layer): raise TypeError("upscale factor must be int type") if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Data format should be 'NCHW' or 'NHWC'." - "But recevie data format: {}".format(data_format)) + raise ValueError( + "Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format) + ) self._upscale_factor = upscale_factor self._data_format = data_format self._name = name def forward(self, x): - return functional.pixel_shuffle(x, self._upscale_factor, - self._data_format, self._name) + return functional.pixel_shuffle( + x, self._upscale_factor, self._data_format, self._name + ) def extra_repr(self): main_str = 'upscale_factor={}'.format(self._upscale_factor) @@ -130,16 +133,19 @@ class PixelUnshuffle(Layer): raise ValueError("Downscale factor must be positive") if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Data format should be 'NCHW' or 'NHWC'." - "But recevie data format: {}".format(data_format)) + raise ValueError( + "Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format) + ) self._downscale_factor = downscale_factor self._data_format = data_format self._name = name def forward(self, x): - return functional.pixel_unshuffle(x, self._downscale_factor, - self._data_format, self._name) + return functional.pixel_unshuffle( + x, self._downscale_factor, self._data_format, self._name + ) def extra_repr(self): main_str = 'downscale_factor={}'.format(self._downscale_factor) @@ -202,16 +208,19 @@ class ChannelShuffle(Layer): raise ValueError("groups must be positive") if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Data format should be 'NCHW' or 'NHWC'." - "But recevie data format: {}".format(data_format)) + raise ValueError( + "Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format) + ) self._groups = groups self._data_format = data_format self._name = name def forward(self, x): - return functional.channel_shuffle(x, self._groups, self._data_format, - self._name) + return functional.channel_shuffle( + x, self._groups, self._data_format, self._name + ) def extra_repr(self): main_str = 'groups={}'.format(self._groups) diff --git a/python/paddle/nn/quant/functional_layers.py b/python/paddle/nn/quant/functional_layers.py index ca1eb5f4fb3..2c0eb88e087 100644 --- a/python/paddle/nn/quant/functional_layers.py +++ b/python/paddle/nn/quant/functional_layers.py @@ -19,13 +19,11 @@ __all__ = [] class FloatFunctionalLayer(Layer): - def __init__(self): super(FloatFunctionalLayer, self).__init__() class add(FloatFunctionalLayer): - def __init__(self): super(add, self).__init__() @@ -34,7 +32,6 @@ class add(FloatFunctionalLayer): class subtract(FloatFunctionalLayer): - def __init__(self): super(subtract, self).__init__() @@ -43,7 +40,6 @@ class subtract(FloatFunctionalLayer): class multiply(FloatFunctionalLayer): - def __init__(self): super(multiply, self).__init__() @@ -52,7 +48,6 @@ class multiply(FloatFunctionalLayer): class divide(FloatFunctionalLayer): - def __init__(self): super(divide, self).__init__() @@ -61,7 +56,6 @@ class divide(FloatFunctionalLayer): class reshape(FloatFunctionalLayer): - def __init__(self): super(reshape, self).__init__() @@ -70,7 +64,6 @@ class reshape(FloatFunctionalLayer): class transpose(FloatFunctionalLayer): - def __init__(self): super(transpose, self).__init__() @@ -79,7 +72,6 @@ class transpose(FloatFunctionalLayer): class concat(FloatFunctionalLayer): - def __init__(self): super(concat, self).__init__() @@ -88,7 +80,6 @@ class concat(FloatFunctionalLayer): class flatten(FloatFunctionalLayer): - def __init__(self): super(flatten, self).__init__() diff --git a/python/paddle/nn/quant/lsq.py b/python/paddle/nn/quant/lsq.py index 6ed6e78a6d9..ff2c3d12807 100644 --- a/python/paddle/nn/quant/lsq.py +++ b/python/paddle/nn/quant/lsq.py @@ -28,7 +28,6 @@ def round(x): class LsqFunc(PyLayer): - @staticmethod def forward(ctx, weight, alpha, g, Qn, Qp, per_channel=False, quant_axis=0): ctx.save_for_backward(weight, alpha) @@ -65,21 +64,39 @@ class LsqFunc(PyLayer): upper_flag = paddle.cast((q_w > Qp), 'float32') middle_flag = 1.0 - lower_flag - upper_flag if per_channel: - grad_alpha = ((lower_flag * Qn + upper_flag * Qp + - middle_flag * round(q_w) - middle_flag * q_w) * - grad_weight * g) + grad_alpha = ( + ( + lower_flag * Qn + + upper_flag * Qp + + middle_flag * round(q_w) + - middle_flag * q_w + ) + * grad_weight + * g + ) grad_alpha = grad_alpha.reshape( - (grad_alpha.shape[quant_axis], -1)).sum(axis=1) + (grad_alpha.shape[quant_axis], -1) + ).sum(axis=1) else: - grad_alpha = ((lower_flag * Qn + upper_flag * Qp + - middle_flag * round(q_w) - middle_flag * q_w) * - grad_weight * g).sum().unsqueeze(axis=0)[0] + grad_alpha = ( + ( + ( + lower_flag * Qn + + upper_flag * Qp + + middle_flag * round(q_w) + - middle_flag * q_w + ) + * grad_weight + * g + ) + .sum() + .unsqueeze(axis=0)[0] + ) grad_weight = middle_flag * grad_weight return grad_weight, grad_alpha class LsqPlusActFunc(PyLayer): - @staticmethod def forward(ctx, x, alpha, beta, g, Qn, Qp): ctx.save_for_backward(x, alpha, beta) @@ -95,25 +112,38 @@ class LsqPlusActFunc(PyLayer): lower_flag = paddle.cast((q_x < Qn), 'float32') upper_flag = paddle.cast((q_x > Qp), 'float32') middle_flag = 1.0 - lower_flag - upper_flag - grad_alpha = ((lower_flag * Qn + upper_flag * Qp + - middle_flag * round(q_x) - middle_flag * q_x) * grad_x * - g).sum().unsqueeze(axis=0)[0] - grad_beta = ((lower_flag + upper_flag) * grad_x * - g).sum().unsqueeze(axis=0)[0] + grad_alpha = ( + ( + ( + lower_flag * Qn + + upper_flag * Qp + + middle_flag * round(q_x) + - middle_flag * q_x + ) + * grad_x + * g + ) + .sum() + .unsqueeze(axis=0)[0] + ) + grad_beta = ( + ((lower_flag + upper_flag) * grad_x * g).sum().unsqueeze(axis=0)[0] + ) grad_x = middle_flag * grad_x return grad_x, grad_alpha, grad_beta class FakeQuantActLSQPlus(Layer): - - def __init__(self, - quant_bits, - all_postive=False, - symmetric=False, - batch_init=20, - dtype='float32', - name=None, - reduce_type=None): + def __init__( + self, + quant_bits, + all_postive=False, + symmetric=False, + batch_init=20, + dtype='float32', + name=None, + reduce_type=None, + ): super(FakeQuantActLSQPlus, self).__init__() ''' Args: @@ -138,46 +168,51 @@ class FakeQuantActLSQPlus(Layer): self.Qp = 2**self.bits - 1 else: # signed activation - self.Qn = -2**(self.bits - 1) - self.Qp = 2**(self.bits - 1) - 1 + self.Qn = -(2 ** (self.bits - 1)) + self.Qp = 2 ** (self.bits - 1) - 1 - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) self._scale_name = unique_name.generate(scale_prefix) - s_attr = ParamAttr(name=self._scale_name, - initializer=Constant(1.0), - trainable=True) + s_attr = ParamAttr( + name=self._scale_name, initializer=Constant(1.0), trainable=True + ) self.s = self.create_parameter(shape=[1], attr=s_attr, dtype='float32') self.s.stop_gradient = False if not self.symmetric: - beta_prefix = "{}.beta".format( - name) if name else 'quant_dequant.beta' + beta_prefix = ( + "{}.beta".format(name) if name else 'quant_dequant.beta' + ) self._beta_name = unique_name.generate(beta_prefix) - beta_attr = ParamAttr(name=self._beta_name, - initializer=Constant(0.0), - trainable=True) - self.beta = self.create_parameter(shape=[1], - attr=beta_attr, - dtype='float32') + beta_attr = ParamAttr( + name=self._beta_name, initializer=Constant(0.0), trainable=True + ) + self.beta = self.create_parameter( + shape=[1], attr=beta_attr, dtype='float32' + ) self.beta.stop_gradient = False self.init_state = 0 def forward(self, activation): if self.reduce_type == "max": - paddle.distributed.all_reduce(self.s, - op=paddle.distributed.ReduceOp.MAX) + paddle.distributed.all_reduce( + self.s, op=paddle.distributed.ReduceOp.MAX + ) if not self.symmetric and self.reduce_type == "max": - paddle.distributed.all_reduce(self.beta, - op=paddle.distributed.ReduceOp.MAX) + paddle.distributed.all_reduce( + self.beta, op=paddle.distributed.ReduceOp.MAX + ) if self.init_state == 0: - self.g = paddle.to_tensor(1.0 / - math.sqrt(activation.numel() * self.Qp)) + self.g = paddle.to_tensor( + 1.0 / math.sqrt(activation.numel() * self.Qp) + ) min_a = paddle.min(activation.detach()) max_a = paddle.max(activation.detach()) self.s.set_value((max_a - min_a) / (self.Qp - self.Qn)) @@ -187,40 +222,41 @@ class FakeQuantActLSQPlus(Layer): elif self.init_state < self.batch_init: min_a = paddle.min(activation.detach()) max_a = paddle.max(activation.detach()) - self.s.set_value(self.s * 0.9 + 0.1 * (max_a - min_a) / - (self.Qp - self.Qn)) + self.s.set_value( + self.s * 0.9 + 0.1 * (max_a - min_a) / (self.Qp - self.Qn) + ) if not self.symmetric: - self.beta.set_value(self.s * 0.9 + 0.1 * - (min_a - self.s * self.Qn)) + self.beta.set_value( + self.s * 0.9 + 0.1 * (min_a - self.s * self.Qn) + ) self.init_state += 1 else: self.init_state += 1 activation.stop_gradient = False if not self.symmetric: - q_a = LsqPlusActFunc.apply(activation, self.s, self.beta, self.g, - self.Qn, self.Qp) + q_a = LsqPlusActFunc.apply( + activation, self.s, self.beta, self.g, self.Qn, self.Qp + ) else: - q_a = LsqFunc.apply(activation, - self.s, - self.g, - self.Qn, - self.Qp, - per_channel=False) + q_a = LsqFunc.apply( + activation, self.s, self.g, self.Qn, self.Qp, per_channel=False + ) return q_a class FakeQuantWeightLSQPlus(Layer): - - def __init__(self, - quant_bits, - all_postive=False, - per_channel=False, - batch_init=20, - channel_num=None, - quant_linear=False, - dtype='float32', - name=None, - reduce_type=None): + def __init__( + self, + quant_bits, + all_postive=False, + per_channel=False, + batch_init=20, + channel_num=None, + quant_linear=False, + dtype='float32', + name=None, + reduce_type=None, + ): super(FakeQuantWeightLSQPlus, self).__init__() ''' Args: @@ -251,25 +287,27 @@ class FakeQuantWeightLSQPlus(Layer): self.Qp = 2**self.bits - 1 else: # signed weight - self.Qn = -2**(self.bits - 1) - self.Qp = 2**(self.bits - 1) - 1 + self.Qn = -(2 ** (self.bits - 1)) + self.Qp = 2 ** (self.bits - 1) - 1 self.init_state = 0 - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) self._scale_name = unique_name.generate(scale_prefix) - s_attr = ParamAttr(name=self._scale_name, - initializer=Constant(1.0), - trainable=True) - self.s = self.create_parameter(shape=[channel_num], - attr=s_attr, - dtype=dtype) + s_attr = ParamAttr( + name=self._scale_name, initializer=Constant(1.0), trainable=True + ) + self.s = self.create_parameter( + shape=[channel_num], attr=s_attr, dtype=dtype + ) self.s.stop_gradient = False def forward(self, weight): if self.reduce_type == "max": - paddle.distributed.all_reduce(self.s, - op=paddle.distributed.ReduceOp.MAX) + paddle.distributed.all_reduce( + self.s, op=paddle.distributed.ReduceOp.MAX + ) if self.init_state == 0: self.g = paddle.to_tensor(1.0 / math.sqrt(weight.numel() * self.Qp)) @@ -278,19 +316,22 @@ class FakeQuantWeightLSQPlus(Layer): weight_tmp = weight.detach().reshape((weight.shape[0], -1)) mean = paddle.mean(weight_tmp, axis=self.collect_axis) std = paddle.std(weight_tmp, axis=self.collect_axis) - s = paddle.max(paddle.stack( - [paddle.abs(mean - 3 * std), - paddle.abs(mean + 3 * std)]), - axis=0) + s = paddle.max( + paddle.stack( + [paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)] + ), + axis=0, + ) self.s.set_value(s / self.div) else: mean = paddle.mean(weight.detach()) std = paddle.std(weight.detach()) self.s.set_value( - max([ - paddle.abs(mean - 3 * std), - paddle.abs(mean + 3 * std) - ]) / self.div) + max( + [paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)] + ) + / self.div + ) self.init_state += 1 elif self.init_state < self.batch_init: self.div = 2**self.bits - 1 @@ -298,22 +339,36 @@ class FakeQuantWeightLSQPlus(Layer): weight_tmp = weight.detach().reshape((weight.shape[0], -1)) mean = paddle.mean(weight_tmp, axis=self.collect_axis) std = paddle.std(weight_tmp, axis=self.collect_axis) - s = paddle.max(paddle.stack( - [paddle.abs(mean - 3 * std), - paddle.abs(mean + 3 * std)]), - axis=0) + s = paddle.max( + paddle.stack( + [paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)] + ), + axis=0, + ) self.s.set_value(s * 0.9 + 0.1 * s / self.div) else: mean = paddle.mean(weight.detach()) std = paddle.std(weight.detach()) - self.s.set_value(self.s * 0.9 + 0.1 * max( - [paddle.abs(mean - 3 * std), - paddle.abs(mean + 3 * std)]) / self.div) + self.s.set_value( + self.s * 0.9 + + 0.1 + * max( + [paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)] + ) + / self.div + ) self.init_state += 1 elif self.init_state == self.batch_init: self.init_state += 1 weight.stop_gradient = False - w_q = LsqFunc.apply(weight, self.s, self.g, self.Qn, self.Qp, - self.per_channel, self.quant_axis) + w_q = LsqFunc.apply( + weight, + self.s, + self.g, + self.Qn, + self.Qp, + self.per_channel, + self.quant_axis, + ) return w_q diff --git a/python/paddle/nn/quant/quant_layers.py b/python/paddle/nn/quant/quant_layers.py index affa180bd30..6eeaed7f86f 100644 --- a/python/paddle/nn/quant/quant_layers.py +++ b/python/paddle/nn/quant/quant_layers.py @@ -42,9 +42,9 @@ __all__ = [ 'QuantizedColumnParallelLinear', ] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class FakeQuantAbsMax(Layer): @@ -57,26 +57,31 @@ class FakeQuantAbsMax(Layer): :math:`Out = round(X / scale * range) * scale / range` """ - def __init__(self, - name=None, - quant_bits=8, - dtype='float32', - quant_on_weight=False, - reduce_type=None): + def __init__( + self, + name=None, + quant_bits=8, + dtype='float32', + quant_on_weight=False, + reduce_type=None, + ): super(FakeQuantAbsMax, self).__init__() self._quant_bits = quant_bits self._name = name self._reduce_type = reduce_type - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) self._scale_name = unique_name.generate(scale_prefix) if quant_on_weight: - scale_attr = ParamAttr(name=self._scale_name, - initializer=Constant(0.001), - trainable=False) - self._scale = self.create_parameter(shape=[1], - attr=scale_attr, - dtype=self._dtype) + scale_attr = ParamAttr( + name=self._scale_name, + initializer=Constant(0.001), + trainable=False, + ) + self._scale = self.create_parameter( + shape=[1], attr=scale_attr, dtype=self._dtype + ) self._scale.stop_gradient = True else: self._scale = None @@ -84,16 +89,18 @@ class FakeQuantAbsMax(Layer): def forward(self, input): if in_dynamic_mode(): attrs = ('bit_length', self._quant_bits) - quant_out = _varbase_creator(type=input.type, - name="{}.quantized.dequantized".format( - input.name), - shape=input.shape, - dtype=input.dtype, - persistable=False) + quant_out = _varbase_creator( + type=input.type, + name="{}.quantized.dequantized".format(input.name), + shape=input.shape, + dtype=input.dtype, + persistable=False, + ) out_scale = self._scale if self._reduce_type == "max": paddle.distributed.all_reduce( - out_scale, op=paddle.distributed.ReduceOp.MAX) + out_scale, op=paddle.distributed.ReduceOp.MAX + ) if not out_scale: out_scale = _varbase_creator( @@ -101,10 +108,12 @@ class FakeQuantAbsMax(Layer): name=self._scale_name, shape=[1], dtype=self._dtype, - persistable=False) + persistable=False, + ) out_scale.stop_gradient = True out, _, = _legacy_C_ops.fake_quantize_dequantize_abs_max( - input, quant_out, out_scale, *attrs) + input, quant_out, out_scale, *attrs + ) return out check_variable_and_dtype(input, 'input', ['float32'], "FakeQuantAbsMax") @@ -115,7 +124,8 @@ class FakeQuantAbsMax(Layer): dtype=input.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) out_scale = self._scale if not out_scale: out_scale = self._helper.create_variable( @@ -123,13 +133,16 @@ class FakeQuantAbsMax(Layer): dtype=self._dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) outputs = {"Out": [quant_out], "OutScale": [out_scale]} - self._helper.append_op(type="fake_quantize_dequantize_abs_max", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="fake_quantize_dequantize_abs_max", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) return quant_out @@ -144,75 +157,108 @@ class FakeQuantMovingAverageAbsMax(Layer): :math:`Out = round(X / scale * range) * scale / range` """ - def __init__(self, - name=None, - moving_rate=0.9, - quant_bits=8, - dtype='float32', - reduce_type=None): + def __init__( + self, + name=None, + moving_rate=0.9, + quant_bits=8, + dtype='float32', + reduce_type=None, + ): super(FakeQuantMovingAverageAbsMax, self).__init__() self._moving_rate = moving_rate self._quant_bits = quant_bits self._reduce_type = reduce_type - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' - scale_attr = ParamAttr(name=unique_name.generate(scale_prefix), - initializer=Constant(0.001), - trainable=False) - self._scale = self.create_parameter(shape=[1], - attr=scale_attr, - dtype=dtype) + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) + scale_attr = ParamAttr( + name=unique_name.generate(scale_prefix), + initializer=Constant(0.001), + trainable=False, + ) + self._scale = self.create_parameter( + shape=[1], attr=scale_attr, dtype=dtype + ) self._scale.stop_gradient = True - state_prefix = "{}.state".format( - name) if name else 'quant_dequant.state' - state_attr = ParamAttr(name=unique_name.generate(state_prefix), - initializer=Constant(1), - trainable=False) - self._state = self.create_parameter(shape=[1], - attr=state_attr, - dtype=dtype) + state_prefix = ( + "{}.state".format(name) if name else 'quant_dequant.state' + ) + state_attr = ParamAttr( + name=unique_name.generate(state_prefix), + initializer=Constant(1), + trainable=False, + ) + self._state = self.create_parameter( + shape=[1], attr=state_attr, dtype=dtype + ) self._state.stop_gradient = True - accum_prefix = "{}.accum".format( - name) if name else 'quant_dequant.accum' - accum_attr = ParamAttr(name=unique_name.generate(accum_prefix), - initializer=Constant(1), - trainable=False) - self._accum = self.create_parameter(shape=[1], - attr=accum_attr, - dtype=dtype) + accum_prefix = ( + "{}.accum".format(name) if name else 'quant_dequant.accum' + ) + accum_attr = ParamAttr( + name=unique_name.generate(accum_prefix), + initializer=Constant(1), + trainable=False, + ) + self._accum = self.create_parameter( + shape=[1], attr=accum_attr, dtype=dtype + ) self._accum.stop_gradient = True def forward(self, input): if in_dynamic_mode(): - attrs = ('moving_rate', self._moving_rate, 'bit_length', - self._quant_bits, 'is_test', not self.training) - quant_out = _varbase_creator(type=input.type, - name="{}.quantized.dequantized".format( - input.name), - shape=input.shape, - dtype=input.dtype, - persistable=False) + attrs = ( + 'moving_rate', + self._moving_rate, + 'bit_length', + self._quant_bits, + 'is_test', + not self.training, + ) + quant_out = _varbase_creator( + type=input.type, + name="{}.quantized.dequantized".format(input.name), + shape=input.shape, + dtype=input.dtype, + persistable=False, + ) if self._reduce_type == "max": paddle.distributed.all_reduce( - self._scale, op=paddle.distributed.ReduceOp.MAX) + self._scale, op=paddle.distributed.ReduceOp.MAX + ) state = self._state if self.training else None accum = self._accum if self.training else None - out, _, _, _ = _legacy_C_ops.fake_quantize_dequantize_moving_average_abs_max( - input, self._scale, accum, state, quant_out, self._scale, state, - accum, *attrs) + ( + out, + _, + _, + _, + ) = _legacy_C_ops.fake_quantize_dequantize_moving_average_abs_max( + input, + self._scale, + accum, + state, + quant_out, + self._scale, + state, + accum, + *attrs + ) return out - check_variable_and_dtype(input, 'input', ['float32'], - "FakeQuantMovingAverageAbsMax") + check_variable_and_dtype( + input, 'input', ['float32'], "FakeQuantMovingAverageAbsMax" + ) attrs = { 'moving_rate': self._moving_rate, 'bit_length': self._quant_bits, - 'is_test': not self.training + 'is_test': not self.training, } inputs = {"X": [input], "InScale": [self._scale]} quant_out = self._helper.create_variable( @@ -220,7 +266,8 @@ class FakeQuantMovingAverageAbsMax(Layer): dtype=input.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) outputs = {"Out": [quant_out], "OutScale": [self._scale]} if self.training: @@ -233,22 +280,26 @@ class FakeQuantMovingAverageAbsMax(Layer): type="fake_quantize_dequantize_moving_average_abs_max", inputs=inputs, outputs=outputs, - attrs=attrs) + attrs=attrs, + ) return quant_out class FakeQuantChannelWiseAbsMax(Layer): - - def __init__(self, - name=None, - channel_num=None, - quant_bits=8, - quant_axis=0, - dtype='float32', - quant_on_weight=False, - reduce_type=None): - assert quant_on_weight == True, "Channel_wise only can be used on weight quantization." + def __init__( + self, + name=None, + channel_num=None, + quant_bits=8, + quant_axis=0, + dtype='float32', + quant_on_weight=False, + reduce_type=None, + ): + assert ( + quant_on_weight == True + ), "Channel_wise only can be used on weight quantization." super(FakeQuantChannelWiseAbsMax, self).__init__() self._quant_bits = quant_bits self._quant_axis = quant_axis @@ -256,50 +307,65 @@ class FakeQuantChannelWiseAbsMax(Layer): self._name = name self._channel_num = channel_num self._reduce_type = reduce_type - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) self._scale_name = unique_name.generate(scale_prefix) if quant_on_weight: - scale_attr = ParamAttr(name=self._scale_name, - initializer=Constant(0.0), - trainable=False) - self._scale = self.create_parameter(shape=[self._channel_num], - attr=scale_attr, - dtype=self._dtype) + scale_attr = ParamAttr( + name=self._scale_name, + initializer=Constant(0.0), + trainable=False, + ) + self._scale = self.create_parameter( + shape=[self._channel_num], attr=scale_attr, dtype=self._dtype + ) self._scale.stop_gradient = True else: self._scale = None def forward(self, input): if in_dynamic_mode(): - attrs = ('bit_length', self._quant_bits, 'quant_axis', - self._quant_axis) - quant_out = _varbase_creator(type=input.type, - name="{}.quantized.dequantized".format( - input.name), - shape=input.shape, - dtype=input.dtype, - persistable=False) + attrs = ( + 'bit_length', + self._quant_bits, + 'quant_axis', + self._quant_axis, + ) + quant_out = _varbase_creator( + type=input.type, + name="{}.quantized.dequantized".format(input.name), + shape=input.shape, + dtype=input.dtype, + persistable=False, + ) out_scale = self._scale if self._reduce_type == "max": paddle.distributed.all_reduce( - out_scale, op=paddle.distributed.ReduceOp.MAX) + out_scale, op=paddle.distributed.ReduceOp.MAX + ) if out_scale is None: out_scale = _varbase_creator( type=core.VarDesc.VarType.LOD_TENSOR, name=self._scale_name, shape=[self._channel_num], dtype=self._dtype, - persistable=False) + persistable=False, + ) out_scale.stop_gradient = True - out, _, = _legacy_C_ops.fake_channel_wise_quantize_dequantize_abs_max( - input, quant_out, out_scale, *attrs) + ( + out, + _, + ) = _legacy_C_ops.fake_channel_wise_quantize_dequantize_abs_max( + input, quant_out, out_scale, *attrs + ) return out - check_variable_and_dtype(input, 'input', ['float32'], - "FakeQuantChannelWiseAbsMax") + check_variable_and_dtype( + input, 'input', ['float32'], "FakeQuantChannelWiseAbsMax" + ) attrs = {'bit_length': self._quant_bits, 'quant_axis': self._quant_axis} inputs = {"X": [input]} quant_out = self._helper.create_variable( @@ -307,7 +373,8 @@ class FakeQuantChannelWiseAbsMax(Layer): dtype=input.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) out_scale = self._scale if not out_scale: out_scale = self._helper.create_variable( @@ -315,25 +382,24 @@ class FakeQuantChannelWiseAbsMax(Layer): dtype=self._dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) outputs = {"Out": [quant_out], "OutScale": [out_scale]} self._helper.append_op( type="fake_channel_wise_quantize_dequantize_abs_max", inputs=inputs, outputs=outputs, - attrs=attrs) + attrs=attrs, + ) return quant_out class MovingAverageAbsMaxScale(Layer): - - def __init__(self, - name=None, - moving_rate=0.9, - dtype='float32', - reduce_type=None): + def __init__( + self, name=None, moving_rate=0.9, dtype='float32', reduce_type=None + ): r""" MovingAverageMaxScale layer is used to calculating the output quantization scale of Layer. Its computational formula is described as below: @@ -346,56 +412,75 @@ class MovingAverageAbsMaxScale(Layer): self._reduce_type = reduce_type scale_prefix = '{}.scale'.format(name) if name else 'outscale.scale' scale_name = unique_name.generate(scale_prefix) - scale_attr = ParamAttr(name=scale_name, - initializer=Constant(0), - trainable=False) - self._scale = self.create_parameter(shape=[1], - attr=scale_attr, - dtype=dtype) + scale_attr = ParamAttr( + name=scale_name, initializer=Constant(0), trainable=False + ) + self._scale = self.create_parameter( + shape=[1], attr=scale_attr, dtype=dtype + ) self._scale.stop_gradient = True state_prefix = "{}.state".format(name) if name else 'outscale.state' - state_attr = ParamAttr(name=unique_name.generate(state_prefix), - initializer=Constant(0), - trainable=False) - self._state = self.create_parameter(shape=[1], - attr=state_attr, - dtype=dtype) + state_attr = ParamAttr( + name=unique_name.generate(state_prefix), + initializer=Constant(0), + trainable=False, + ) + self._state = self.create_parameter( + shape=[1], attr=state_attr, dtype=dtype + ) self._state.stop_gradient = True accum_prefix = "{}.accum".format(name) if name else 'outscale.accum' - accum_attr = ParamAttr(name=unique_name.generate(accum_prefix), - initializer=Constant(0), - trainable=False) - self._accum = self.create_parameter(shape=[1], - attr=accum_attr, - dtype=dtype) + accum_attr = ParamAttr( + name=unique_name.generate(accum_prefix), + initializer=Constant(0), + trainable=False, + ) + self._accum = self.create_parameter( + shape=[1], attr=accum_attr, dtype=dtype + ) self._accum.stop_gradient = True def forward(self, input): if in_dynamic_mode(): - attrs = ('moving_rate', self._moving_rate, 'is_test', - not self.training) - - quant_out = _varbase_creator(type=input.type, - name="{}.tmp".format(input.name), - shape=input.shape, - dtype=input.dtype, - persistable=False) + attrs = ( + 'moving_rate', + self._moving_rate, + 'is_test', + not self.training, + ) + + quant_out = _varbase_creator( + type=input.type, + name="{}.tmp".format(input.name), + shape=input.shape, + dtype=input.dtype, + persistable=False, + ) if self._reduce_type == "max": paddle.distributed.all_reduce( - self._scale, op=paddle.distributed.ReduceOp.MAX) + self._scale, op=paddle.distributed.ReduceOp.MAX + ) state = self._state if self.training else None accum = self._accum if self.training else None out, _, _, _ = _legacy_C_ops.moving_average_abs_max_scale( - input, accum, state, quant_out, self._scale, state, accum, - *attrs) + input, + accum, + state, + quant_out, + self._scale, + state, + accum, + *attrs + ) return out - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'MovingAverageAbsMaxScale') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'MovingAverageAbsMaxScale' + ) attrs = {'moving_rate': self._moving_rate, 'is_test': not self.training} inputs = {"X": [input]} @@ -404,7 +489,8 @@ class MovingAverageAbsMaxScale(Layer): dtype=input.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) outputs = {"Out": [quant_out], "OutScale": [self._scale]} if self.training: @@ -413,10 +499,12 @@ class MovingAverageAbsMaxScale(Layer): outputs['OutState'] = [self._state] outputs['OutAccum'] = [self._accum] - self._helper.append_op(type="moving_average_abs_max_scale", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="moving_average_abs_max_scale", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) return quant_out @@ -430,17 +518,19 @@ class QuantizedConv2D(Layer): The only difference is that its inputs are all fake quantized. """ - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): super(QuantizedConv2D, self).__init__() # For Conv2D self._groups = getattr(layer, '_groups') @@ -449,7 +539,8 @@ class QuantizedConv2D(Layer): self._padding_mode = getattr(layer, '_padding_mode') if self._padding_mode != 'zeros': self._reversed_padding_repeated_twice = getattr( - layer, '_reversed_padding_repeated_twice') + layer, '_reversed_padding_repeated_twice' + ) self._dilation = getattr(layer, '_dilation') self._data_format = getattr(layer, '_data_format') self.weight = getattr(layer, 'weight') @@ -468,7 +559,8 @@ class QuantizedConv2D(Layer): dtype=self._dtype, quant_on_weight=True, channel_num=self.weight.shape[self._conv2d_quant_axis], - quant_axis=self._conv2d_quant_axis) + quant_axis=self._conv2d_quant_axis, + ) if act_quant_layer is not None: self._fake_quant_input = act_quant_layer() else: @@ -478,12 +570,15 @@ class QuantizedConv2D(Layer): moving_rate=moving_rate, quant_bits=activation_bits, dtype=self._dtype, - quant_on_weight=False) + quant_on_weight=False, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input): if self._act_preprocess is not None: @@ -496,20 +591,24 @@ class QuantizedConv2D(Layer): quant_weight = self._fake_quant_weight(weight) if self._padding_mode != 'zeros': - quant_input = F.pad(quant_input, - self._reversed_padding_repeated_twice, - mode=self._padding_mode, - data_format=self._data_format) + quant_input = F.pad( + quant_input, + self._reversed_padding_repeated_twice, + mode=self._padding_mode, + data_format=self._data_format, + ) self._padding = 0 - return F.conv2d(quant_input, - quant_weight, - bias=self.bias, - padding=self._padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format) + return F.conv2d( + quant_input, + quant_weight, + bias=self.bias, + padding=self._padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + ) class QuantizedConv2DTranspose(Layer): @@ -533,17 +632,19 @@ class QuantizedConv2DTranspose(Layer): # (2, 6, 10, 10), (2, 6, 10, 10) """ - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): r""" Constructor. @@ -572,8 +673,10 @@ class QuantizedConv2DTranspose(Layer): dtype=self._dtype, quant_on_weight=True, channel_num=self.weight.shape[ - self._conv2d_transpose_quant_axis], - quant_axis=self._conv2d_transpose_quant_axis) + self._conv2d_transpose_quant_axis + ], + quant_axis=self._conv2d_transpose_quant_axis, + ) if act_quant_layer is not None: self._fake_quant_input = act_quant_layer() else: @@ -583,12 +686,15 @@ class QuantizedConv2DTranspose(Layer): moving_rate=moving_rate, quant_bits=activation_bits, dtype=self._dtype, - quant_on_weight=False) + quant_on_weight=False, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input, output_size=None): if self._act_preprocess is not None: @@ -605,16 +711,18 @@ class QuantizedConv2DTranspose(Layer): else: output_padding = 0 - return F.conv2d_transpose(quant_input, - quant_weight, - bias=self.bias, - padding=self._padding, - output_padding=output_padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - output_size=output_size, - data_format=self._data_format) + return F.conv2d_transpose( + quant_input, + quant_weight, + bias=self.bias, + padding=self._padding, + output_padding=output_padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + output_size=output_size, + data_format=self._data_format, + ) class QuantizedLinear(Layer): @@ -623,17 +731,19 @@ class QuantizedLinear(Layer): The only difference is that its inputs are all fake quantized. """ - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): super(QuantizedLinear, self).__init__() # For Linear self.weight = getattr(layer, 'weight') @@ -654,7 +764,8 @@ class QuantizedLinear(Layer): quant_on_weight=True, channel_num=self.weight.shape[self._linear_quant_axis], quant_axis=self._linear_quant_axis, - quant_linear=True) + quant_linear=True, + ) if act_quant_layer is not None: self._fake_quant_input = act_quant_layer() @@ -665,12 +776,15 @@ class QuantizedLinear(Layer): moving_rate=moving_rate, quant_bits=activation_bits, dtype=self._dtype, - quant_on_weight=False) + quant_on_weight=False, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input): if self._act_preprocess is not None: @@ -682,32 +796,36 @@ class QuantizedLinear(Layer): weight = self._weight_preprocess(self.weight) quant_weight = self._fake_quant_weight(weight) - out = F.linear(x=quant_input, - weight=quant_weight, - bias=self.bias, - name=self.name) + out = F.linear( + x=quant_input, weight=quant_weight, bias=self.bias, name=self.name + ) return out class QuantizedColumnParallelLinear(Layer): - - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): super(QuantizedColumnParallelLinear, self).__init__() ''' ''' - assert weight_quant_layer is None, "When quantizing ColumnParallelLinear, weight_quant_layer should be None." - assert act_quant_layer is None, "When quantizing ColumnParallelLinear, act_quant_layer should be None." + assert ( + weight_quant_layer is None + ), "When quantizing ColumnParallelLinear, weight_quant_layer should be None." + assert ( + act_quant_layer is None + ), "When quantizing ColumnParallelLinear, act_quant_layer should be None." self.weight = getattr(layer, 'weight') self.bias = getattr(layer, 'bias') @@ -729,7 +847,9 @@ class QuantizedColumnParallelLinear(Layer): channel_num=self.weight.shape[self._linear_quant_axis], quant_axis=self._linear_quant_axis, reduce_type='max' - if paddle.distributed.get_world_size() > 1 else None) + if paddle.distributed.get_world_size() > 1 + else None, + ) self._fake_quant_input = _get_fake_quant_type( activation_quantize_type, @@ -738,17 +858,21 @@ class QuantizedColumnParallelLinear(Layer): quant_bits=activation_bits, dtype=self._dtype, quant_on_weight=False, - reduce_type=None) + reduce_type=None, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input): if self.is_mp: input_parallel = paddle.distributed.collective._c_identity( - input, group=self.model_parallel_group) + input, group=self.model_parallel_group + ) else: input_parallel = input @@ -761,35 +885,40 @@ class QuantizedColumnParallelLinear(Layer): weight = self._weight_preprocess(self.weight) quant_weight = self._fake_quant_weight(weight) - output_parallel = F.linear(x=quant_input, - weight=quant_weight, - bias=self.bias, - name=self.name) + output_parallel = F.linear( + x=quant_input, weight=quant_weight, bias=self.bias, name=self.name + ) if self.gather_output and self.is_mp: output = paddle.distributed.collective._c_concat( - output_parallel, group=self.model_parallel_group) + output_parallel, group=self.model_parallel_group + ) else: output = output_parallel return output class QuantizedRowParallelLinear(Layer): - - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): super(QuantizedRowParallelLinear, self).__init__() - assert weight_quant_layer is None, "When quantizing RowParallelLinear, weight_quant_layer cannot defined by yourself." - assert act_quant_layer is None, "When quantizing RowParallelLinear, act_quant_layer cannot defined by yourself." + assert ( + weight_quant_layer is None + ), "When quantizing RowParallelLinear, weight_quant_layer cannot defined by yourself." + assert ( + act_quant_layer is None + ), "When quantizing RowParallelLinear, act_quant_layer cannot defined by yourself." # For Linear self.weight = getattr(layer, 'weight') @@ -812,7 +941,9 @@ class QuantizedRowParallelLinear(Layer): channel_num=self.weight.shape[self._linear_quant_axis], quant_axis=self._linear_quant_axis, reduce_type='max' - if paddle.distributed.get_world_size() > 1 else None) + if paddle.distributed.get_world_size() > 1 + else None, + ) self._fake_quant_input = _get_fake_quant_type( activation_quantize_type, @@ -822,12 +953,16 @@ class QuantizedRowParallelLinear(Layer): dtype=self._dtype, quant_on_weight=False, reduce_type='max' - if paddle.distributed.get_world_size() > 1 else None) + if paddle.distributed.get_world_size() > 1 + else None, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input): if self.input_is_parallel or (not self.is_mp): @@ -835,7 +970,8 @@ class QuantizedRowParallelLinear(Layer): else: # split last dim input_parallel = paddle.distributed.collective._c_split( - input, group=self.model_parallel_group) + input, group=self.model_parallel_group + ) if self._act_preprocess is not None: input_parallel = self._act_preprocess(input_parallel) @@ -846,15 +982,16 @@ class QuantizedRowParallelLinear(Layer): weight = self._weight_preprocess(self.weight) quant_weight = self._fake_quant_weight(weight) - output_parallel = F.linear(x=quant_input, - weight=quant_weight, - name=self.name) + output_parallel = F.linear( + x=quant_input, weight=quant_weight, name=self.name + ) if self.is_mp: output_ = paddle.distributed.collective._mp_allreduce( output_parallel, group=self.model_parallel_group, use_calc_stream=True, - use_model_parallel=True) + use_model_parallel=True, + ) else: output_ = output_parallel output = output_ + self.bias if self.bias is not None else output_ @@ -867,12 +1004,14 @@ class MAOutputScaleLayer(Layer): Calculate the scale (moving average abs max) for the output of the input layer. """ - def __init__(self, - layer=None, - moving_rate=0.9, - name=None, - dtype='float32', - reduce_type=None): + def __init__( + self, + layer=None, + moving_rate=0.9, + name=None, + dtype='float32', + reduce_type=None, + ): r""" Construct """ @@ -880,14 +1019,18 @@ class MAOutputScaleLayer(Layer): self._layer = layer if name is None: name = layer.full_name() - self._ma_output_scale = \ - MovingAverageAbsMaxScale(name, moving_rate, dtype, reduce_type) + self._ma_output_scale = MovingAverageAbsMaxScale( + name, moving_rate, dtype, reduce_type + ) def forward(self, *inputs, **kwargs): out = self._layer(*inputs, **kwargs) # TODO (jc): support the ops of several outputs - if (isinstance(out, list) or isinstance(out, tuple) - or isinstance(out, dict)): + if ( + isinstance(out, list) + or isinstance(out, tuple) + or isinstance(out, dict) + ): return out else: return self._ma_output_scale(out) @@ -898,15 +1041,17 @@ class FakeQuantMAOutputScaleLayer(Layer): Add FakeQuantMovingAverageAbsMax layer to the behind of the input layer. """ - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - name=None, - reduce_type=None, - *args, - **kwargs): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + name=None, + reduce_type=None, + *args, + **kwargs + ): super(FakeQuantMAOutputScaleLayer, self).__init__() self._layer = layer @@ -917,7 +1062,8 @@ class FakeQuantMAOutputScaleLayer(Layer): quant_bits=activation_bits, dtype=self._dtype, quant_on_weight=False, - reduce_type=reduce_type) + reduce_type=reduce_type, + ) def forward(self, *inputs, **kwargs): out = self._layer(*inputs, **kwargs) @@ -933,7 +1079,7 @@ def _get_fake_quant_type(quant_type, **kwargs): "name": kwargs.get("name", None), "quant_bits": kwargs.get("quant_bits", 8), "dtype": kwargs.get("dtype", "float32"), - "reduce_type": kwargs.get("reduce_type", None) + "reduce_type": kwargs.get("reduce_type", None), } if quant_type == 'abs_max': @@ -946,7 +1092,8 @@ def _get_fake_quant_type(quant_type, **kwargs): call_args["quant_axis"] = kwargs.get("quant_axis", 0) assert call_args["channel_num"] is not None, ( "You need to input channel_num" - "when you use channel_wise_abs_max strategy.") + "when you use channel_wise_abs_max strategy." + ) elif quant_type == 'lsq_weight': call_args["all_postive"] = kwargs.get("all_postive", False) call_args["per_channel"] = False @@ -960,7 +1107,8 @@ def _get_fake_quant_type(quant_type, **kwargs): call_args["quant_linear"] = kwargs.get("quant_linear", False) assert call_args["channel_num"] is not None, ( "You need to input channel_num" - "when you use channel_wise_abs_max strategy.") + "when you use channel_wise_abs_max strategy." + ) elif quant_type == 'lsq_act': call_args["all_postive"] = kwargs.get("all_postive", False) call_args["symmetric"] = kwargs.get("symmetric", True) @@ -969,7 +1117,7 @@ def _get_fake_quant_type(quant_type, **kwargs): 'moving_average_abs_max': FakeQuantMovingAverageAbsMax, 'channel_wise_abs_max': FakeQuantChannelWiseAbsMax, 'lsq_weight': FakeQuantWeightLSQPlus, - 'lsq_act': FakeQuantActLSQPlus + 'lsq_act': FakeQuantActLSQPlus, } return fake_quant_map[quant_type](**call_args) diff --git a/python/paddle/nn/utils/__init__.py b/python/paddle/nn/utils/__init__.py index 5afdaa8d848..23e1e233cc0 100644 --- a/python/paddle/nn/utils/__init__.py +++ b/python/paddle/nn/utils/__init__.py @@ -14,9 +14,16 @@ from .spectral_norm_hook import spectral_norm from .weight_norm_hook import weight_norm, remove_weight_norm # noqa: F401 -from .transform_parameters import parameters_to_vector, vector_to_parameters, _stride_column # noqa: F401 +from .transform_parameters import ( + parameters_to_vector, + vector_to_parameters, + _stride_column, +) # noqa: F401 -__all__ = [ #noqa - 'weight_norm', 'remove_weight_norm', 'spectral_norm', - 'parameters_to_vector', 'vector_to_parameters' +__all__ = [ # noqa + 'weight_norm', + 'remove_weight_norm', + 'spectral_norm', + 'parameters_to_vector', + 'vector_to_parameters', ] diff --git a/python/paddle/nn/utils/spectral_norm_hook.py b/python/paddle/nn/utils/spectral_norm_hook.py index a2a93d3241e..288e5ea59c9 100644 --- a/python/paddle/nn/utils/spectral_norm_hook.py +++ b/python/paddle/nn/utils/spectral_norm_hook.py @@ -20,21 +20,21 @@ from .. import functional as F __all__ = [] -def normal_(x, mean=0., std=1.): +def normal_(x, mean=0.0, std=1.0): temp_value = paddle.normal(mean, std, shape=x.shape) x.set_value(temp_value) return x class SpectralNorm(object): - def __init__(self, name='weight', n_power_iterations=1, dim=0, eps=1e-12): self.name = name self.dim = dim if n_power_iterations <= 0: raise ValueError( 'Expected n_power_iterations to be positive, but ' - 'got n_power_iterations={}'.format(n_power_iterations)) + 'got n_power_iterations={}'.format(n_power_iterations) + ) self.n_power_iterations = n_power_iterations self.eps = eps @@ -43,8 +43,9 @@ class SpectralNorm(object): if self.dim != 0: # transpose dim to front weight_mat = weight_mat.transpose( - [self.dim] + - [d for d in range(weight_mat.dim()) if d != self.dim]) + [self.dim] + + [d for d in range(weight_mat.dim()) if d != self.dim] + ) height = weight_mat.shape[0] @@ -61,20 +62,24 @@ class SpectralNorm(object): for _ in range(self.n_power_iterations): v.set_value( F.normalize( - paddle.matmul(weight_mat, - u, - transpose_x=True, - transpose_y=False), + paddle.matmul( + weight_mat, + u, + transpose_x=True, + transpose_y=False, + ), axis=0, epsilon=self.eps, - )) + ) + ) u.set_value( F.normalize( paddle.matmul(weight_mat, v), axis=0, epsilon=self.eps, - )) + ) + ) if self.n_power_iterations > 0: u = u.clone() v = v.clone() @@ -84,15 +89,20 @@ class SpectralNorm(object): return weight def __call__(self, layer, inputs): - setattr(layer, self.name, - self.compute_weight(layer, do_power_iteration=layer.training)) + setattr( + layer, + self.name, + self.compute_weight(layer, do_power_iteration=layer.training), + ) @staticmethod def apply(layer, name, n_power_iterations, dim, eps): for k, hook in layer._forward_pre_hooks.items(): if isinstance(hook, SpectralNorm) and hook.name == name: - raise RuntimeError("Cannot register two spectral_norm hooks on " - "the same parameter {}".format(name)) + raise RuntimeError( + "Cannot register two spectral_norm hooks on " + "the same parameter {}".format(name) + ) fn = SpectralNorm(name, n_power_iterations, dim, eps) weight = layer._parameters[name] @@ -103,9 +113,9 @@ class SpectralNorm(object): # randomly initialize u and v u = layer.create_parameter([h]) - u = normal_(u, 0., 1.) + u = normal_(u, 0.0, 1.0) v = layer.create_parameter([w]) - v = normal_(v, 0., 1.) + v = normal_(v, 0.0, 1.0) u = F.normalize(u, axis=0, epsilon=fn.eps) v = F.normalize(v, axis=0, epsilon=fn.eps) @@ -124,11 +134,9 @@ class SpectralNorm(object): return fn -def spectral_norm(layer, - name='weight', - n_power_iterations=1, - eps=1e-12, - dim=None): +def spectral_norm( + layer, name='weight', n_power_iterations=1, eps=1e-12, dim=None +): r""" Applies spectral normalization to a parameter according to the following Calculation: @@ -198,8 +206,8 @@ def spectral_norm(layer, if dim is None: if isinstance( - layer, - (Conv1DTranspose, Conv2DTranspose, Conv3DTranspose, Linear)): + layer, (Conv1DTranspose, Conv2DTranspose, Conv3DTranspose, Linear) + ): dim = 1 else: dim = 0 diff --git a/python/paddle/nn/utils/transform_parameters.py b/python/paddle/nn/utils/transform_parameters.py index 63ddd097444..4076b808dda 100644 --- a/python/paddle/nn/utils/transform_parameters.py +++ b/python/paddle/nn/utils/transform_parameters.py @@ -15,11 +15,16 @@ from functools import reduce import paddle -from paddle.fluid.framework import dygraph_only, _dygraph_tracer, _varbase_creator, in_dygraph_mode +from paddle.fluid.framework import ( + dygraph_only, + _dygraph_tracer, + _varbase_creator, + in_dygraph_mode, +) from paddle import _C_ops -#input==output, inplace strategy of reshape has no cost almostly +# input==output, inplace strategy of reshape has no cost almostly def _inplace_reshape_dygraph(x, shape): x_shape = _varbase_creator(dtype='int64') if in_dygraph_mode(): @@ -27,14 +32,13 @@ def _inplace_reshape_dygraph(x, shape): tmp_out = _C_ops.reshape(x, shape) tmp_out._share_underline_tensor_to(x) else: - _dygraph_tracer().trace_op(type="reshape2", - inputs={'X': x}, - outputs={ - 'Out': x, - 'XShape': x_shape - }, - attrs={'shape': shape}, - stop_gradient=True) + _dygraph_tracer().trace_op( + type="reshape2", + inputs={'X': x}, + outputs={'Out': x, 'XShape': x_shape}, + attrs={'shape': shape}, + stop_gradient=True, + ) @dygraph_only @@ -106,11 +110,13 @@ def parameters_to_vector(parameters, name=None): tmp = _C_ops.concat(parameters, 0) tmp._share_underline_tensor_to(out) else: - _dygraph_tracer().trace_op(type='concat', - inputs={'X': parameters}, - outputs={'Out': [out]}, - attrs={'axis': 0}, - stop_gradient=True) + _dygraph_tracer().trace_op( + type='concat', + inputs={'X': parameters}, + outputs={'Out': [out]}, + attrs={'axis': 0}, + stop_gradient=True, + ) for i, param in enumerate(parameters): _inplace_reshape_dygraph(param, origin_shapes[i]) return out @@ -161,14 +167,13 @@ def vector_to_parameters(vec, parameters, name=None): for i in range(0, len(parameters)): res[i]._share_underline_tensor_to(parameters[i]) else: - _dygraph_tracer().trace_op(type='split', - inputs={'X': [vec]}, - outputs={'Out': parameters}, - attrs={ - 'axis': 0, - 'sections': sections - }, - stop_gradient=True) + _dygraph_tracer().trace_op( + type='split', + inputs={'X': [vec]}, + outputs={'Out': parameters}, + attrs={'axis': 0, 'sections': sections}, + stop_gradient=True, + ) for i, param in enumerate(parameters): _inplace_reshape_dygraph(param, origin_shapes[i]) diff --git a/python/paddle/nn/utils/weight_norm_hook.py b/python/paddle/nn/utils/weight_norm_hook.py index b836ad28439..cdcb97aa9a0 100755 --- a/python/paddle/nn/utils/weight_norm_hook.py +++ b/python/paddle/nn/utils/weight_norm_hook.py @@ -34,16 +34,15 @@ def l2_norm(x, axis, epsilon=1e-12, name=None): helper = LayerHelper("l2_normalize", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) norm = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="norm", - inputs={"X": x}, - outputs={ - "Out": out, - "Norm": norm - }, - attrs={ - "axis": 1 if axis is None else axis, - "epsilon": epsilon, - }) + helper.append_op( + type="norm", + inputs={"X": x}, + outputs={"Out": out, "Norm": norm}, + attrs={ + "axis": 1 if axis is None else axis, + "epsilon": epsilon, + }, + ) return paddle.squeeze(norm, axis=[axis]) @@ -90,14 +89,13 @@ def _weight_norm(v, g, dim): v_normalized = F.l2_normalize(p_matrix, axis=1) v_normalized = paddle.reshape(v_normalized, transposed_shape) v_normalized = paddle.transpose(v_normalized, perm) - weight = F.elementwise_mul(v_normalized, - g, - axis=dim if dim is not None else -1) + weight = F.elementwise_mul( + v_normalized, g, axis=dim if dim is not None else -1 + ) return weight class WeightNorm(object): - def __init__(self, name, dim): if dim is None: dim = -1 @@ -113,8 +111,10 @@ class WeightNorm(object): def apply(layer, name, dim): for k, hook in layer._forward_pre_hooks.items(): if isinstance(hook, WeightNorm) and hook.name == name: - raise RuntimeError("Cannot register two weight_norm hooks on " - "the same parameter {}".format(name)) + raise RuntimeError( + "Cannot register two weight_norm hooks on " + "the same parameter {}".format(name) + ) if dim is None: dim = -1 diff --git a/python/paddle/onnx/export.py b/python/paddle/onnx/export.py index 5cdf0b06eec..7123f485bf8 100644 --- a/python/paddle/onnx/export.py +++ b/python/paddle/onnx/export.py @@ -94,11 +94,14 @@ def export(layer, path, input_spec=None, opset_version=9, **configs): raise ValueError( "The input path MUST be format of dirname/file_prefix " "[dirname\\file_prefix in Windows system], but " - "the file_prefix is empty in received path: {}".format(path)) + "the file_prefix is empty in received path: {}".format(path) + ) save_file = path + '.onnx' - p2o.dygraph2onnx(layer, - save_file, - input_spec=input_spec, - opset_version=opset_version, - **configs) + p2o.dygraph2onnx( + layer, + save_file, + input_spec=input_spec, + opset_version=opset_version, + **configs + ) diff --git a/python/paddle/optimizer/__init__.py b/python/paddle/optimizer/__init__.py index cd75fd4906e..cef51897b20 100644 --- a/python/paddle/optimizer/__init__.py +++ b/python/paddle/optimizer/__init__.py @@ -24,7 +24,15 @@ from .momentum import Momentum # noqa: F401 from .lamb import Lamb # noqa: F401 from . import lr # noqa: F401 -__all__ = [ #noqa - 'Optimizer', 'Adagrad', 'Adam', 'AdamW', 'Adamax', 'RMSProp', 'Adadelta', - 'SGD', 'Momentum', 'Lamb' +__all__ = [ # noqa + 'Optimizer', + 'Adagrad', + 'Adam', + 'AdamW', + 'Adamax', + 'RMSProp', + 'Adadelta', + 'SGD', + 'Momentum', + 'Lamb', ] diff --git a/python/paddle/optimizer/adadelta.py b/python/paddle/optimizer/adadelta.py index 203ad8f617c..9587f4dd8ee 100644 --- a/python/paddle/optimizer/adadelta.py +++ b/python/paddle/optimizer/adadelta.py @@ -107,25 +107,29 @@ class Adadelta(Optimizer): _avg_squared_grad_acc_str = "_avg_squared_grad" _avg_squared_update_acc_str = "_avg_squared_update" - def __init__(self, - learning_rate=0.001, - epsilon=1.0e-6, - rho=0.95, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate=0.001, + epsilon=1.0e-6, + rho=0.95, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set.") if epsilon is None: raise ValueError("epsilon is not set.") if rho is None: raise ValueError("rho is not set.") - super(Adadelta, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(Adadelta, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "adadelta" self._epsilon = epsilon self._rho = rho @@ -149,43 +153,44 @@ class Adadelta(Optimizer): param_and_grad = self._update_param_group(param_and_grad) avg_squared_grad_acc = self._get_accumulator( - self._avg_squared_grad_acc_str, param_and_grad[0]) + self._avg_squared_grad_acc_str, param_and_grad[0] + ) avg_squared_update_acc = self._get_accumulator( - self._avg_squared_update_acc_str, param_and_grad[0]) + self._avg_squared_update_acc_str, param_and_grad[0] + ) if in_dygraph_mode(): with no_grad(): - _C_ops.adadelta_(param_and_grad[0], param_and_grad[1], - avg_squared_grad_acc, avg_squared_update_acc, - self._rho, self._epsilon) + _C_ops.adadelta_( + param_and_grad[0], + param_and_grad[1], + avg_squared_grad_acc, + avg_squared_update_acc, + self._rho, + self._epsilon, + ) return None if not isinstance(block, framework.Block): raise TypeError("block is not instance of framework.Block.") # Create the adadelta optimizer op - adadelta_op = block.append_op(type=self.type, - inputs={ - "Param": param_and_grad[0], - "Grad": param_and_grad[1], - "AvgSquaredGrad": - avg_squared_grad_acc, - "AvgSquaredUpdate": - avg_squared_update_acc - }, - outputs={ - "ParamOut": - param_and_grad[0], - "AvgSquaredGradOut": - avg_squared_grad_acc, - "AvgSquaredUpdateOut": - avg_squared_update_acc - }, - attrs={ - "epsilon": self._epsilon, - "rho": self._rho - }, - stop_gradient=True) + adadelta_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "AvgSquaredGrad": avg_squared_grad_acc, + "AvgSquaredUpdate": avg_squared_update_acc, + }, + outputs={ + "ParamOut": param_and_grad[0], + "AvgSquaredGradOut": avg_squared_grad_acc, + "AvgSquaredUpdateOut": avg_squared_update_acc, + }, + attrs={"epsilon": self._epsilon, "rho": self._rho}, + stop_gradient=True, + ) return adadelta_op diff --git a/python/paddle/optimizer/adagrad.py b/python/paddle/optimizer/adagrad.py index e0296a1cb0a..a4d9416e93b 100644 --- a/python/paddle/optimizer/adagrad.py +++ b/python/paddle/optimizer/adagrad.py @@ -106,21 +106,25 @@ class Adagrad(Optimizer): """ _moment_acc_str = "moment" - def __init__(self, - learning_rate, - epsilon=1.0e-6, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None, - initial_accumulator_value=0.0): + def __init__( + self, + learning_rate, + epsilon=1.0e-6, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + initial_accumulator_value=0.0, + ): assert learning_rate is not None assert epsilon is not None - super(Adagrad, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(Adagrad, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "adagrad" self._epsilon = epsilon self.initial_accumulator_value = initial_accumulator_value @@ -136,9 +140,11 @@ class Adagrad(Optimizer): parameters = self._update_param_group(parameters) for p in parameters: - self._add_accumulator(self._moment_acc_str, - p, - fill_value=self.initial_accumulator_value) + self._add_accumulator( + self._moment_acc_str, + p, + fill_value=self.initial_accumulator_value, + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -146,26 +152,22 @@ class Adagrad(Optimizer): if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - moment_acc = self._get_accumulator(self._moment_acc_str, - param_and_grad[0]) + moment_acc = self._get_accumulator( + self._moment_acc_str, param_and_grad[0] + ) # Create the adagrad optimizer op - adagrad_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "Moment": - moment_acc, - "LearningRate": - self._create_param_lr(param_and_grad) - }, - outputs={ - "ParamOut": param_and_grad[0], - "MomentOut": moment_acc - }, - attrs={"epsilon": self._epsilon}, - stop_gradient=True) + adagrad_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "Moment": moment_acc, + "LearningRate": self._create_param_lr(param_and_grad), + }, + outputs={"ParamOut": param_and_grad[0], "MomentOut": moment_acc}, + attrs={"epsilon": self._epsilon}, + stop_gradient=True, + ) return adagrad_op @@ -173,6 +175,7 @@ class Adagrad(Optimizer): self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) self.initial_accumulator_value = parameters.get( 'initial_accumulator_value', - self._default_dict['initial_accumulator_value']) + self._default_dict['initial_accumulator_value'], + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/adam.py b/python/paddle/optimizer/adam.py index a49af1a4589..c6e03ccf64c 100644 --- a/python/paddle/optimizer/adam.py +++ b/python/paddle/optimizer/adam.py @@ -161,18 +161,20 @@ class Adam(Optimizer): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameters=None, - weight_decay=None, - grad_clip=None, - lazy_mode=False, - multi_precision=False, - use_multi_tensor=False, - name=None): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameters=None, + weight_decay=None, + grad_clip=None, + lazy_mode=False, + multi_precision=False, + use_multi_tensor=False, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None @@ -180,20 +182,25 @@ class Adam(Optimizer): if not isinstance(beta1, Variable): if not 0 <= beta1 < 1: raise ValueError( - "Invaild value of beta1, expect beta1 in [0,1).") + "Invaild value of beta1, expect beta1 in [0,1)." + ) if not isinstance(beta2, Variable): if not 0 <= beta2 < 1: raise ValueError( - "Invaild value of beta2, expect beta2 in [0,1).") + "Invaild value of beta2, expect beta2 in [0,1)." + ) if not isinstance(epsilon, Variable): if not 0 <= epsilon: raise ValueError( - "Invaild value of epsilon, expect epsilon >= 0.") - super(Adam, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + "Invaild value of epsilon, expect epsilon >= 0." + ) + super(Adam, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "adam" self._beta1 = beta1 self._beta2 = beta2 @@ -215,15 +222,15 @@ class Adam(Optimizer): self._moment2_dict = {'FP32_LODTensor': [], 'FP16_LODTensor': []} self._beta1_pow_acc_dict = { 'FP32_LODTensor': [], - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } self._beta2_pow_acc_dict = { 'FP32_LODTensor': [], - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } self._master_weight_dict = { 'FP32_LODTensor': None, - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } def _create_master_weight(self, param): @@ -234,19 +241,23 @@ class Adam(Optimizer): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -260,20 +271,30 @@ class Adam(Optimizer): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _add_moments_pows(self, p): acc_dtype = p.dtype - if acc_dtype == core.VarDesc.VarType.FP16 or acc_dtype == core.VarDesc.VarType.BF16: + if ( + acc_dtype == core.VarDesc.VarType.FP16 + or acc_dtype == core.VarDesc.VarType.BF16 + ): acc_dtype = core.VarDesc.VarType.FP32 self._add_accumulator(self._moment1_acc_str, p, dtype=acc_dtype) self._add_accumulator(self._moment2_acc_str, p, dtype=acc_dtype) @@ -281,18 +302,24 @@ class Adam(Optimizer): name=self._beta1_pow_acc_str, param=p, dtype=acc_dtype, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_accumulator( name=self._beta2_pow_acc_str, param=p, dtype=acc_dtype, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -305,7 +332,10 @@ class Adam(Optimizer): master_p = self._create_master_weight(p) self._add_moments_pows(master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Adam optimizer." @@ -317,50 +347,105 @@ class Adam(Optimizer): if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) lr = self._create_param_lr(param_and_grad) # create the adam optimize op if framework.in_dygraph_mode(): found_inf = self._get_auxiliary_var('found_inf') - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) _, _, _, _, _, _ = _C_ops.adam_( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, found_inf, _beta1, - _beta2, self._epsilon, self._lazy_mode, 1000, find_master, - False) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + found_inf, + _beta1, + _beta2, + self._epsilon, + self._lazy_mode, + 1000, + find_master, + False, + ) return None if framework._in_legacy_dygraph(): - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) _, _, _, _, _, _ = _legacy_C_ops.adam( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, param_and_grad[0], - moment1, moment2, beta1_pow_acc, beta2_pow_acc, master_weight, - 'epsilon', self._epsilon, 'lazy_mode', self._lazy_mode, - 'min_row_size_to_use_multithread', 1000, 'beta1', _beta1, - 'beta2', _beta2, 'multi_precision', find_master) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'epsilon', + self._epsilon, + 'lazy_mode', + self._lazy_mode, + 'min_row_size_to_use_multithread', + 1000, + 'beta1', + _beta1, + 'beta2', + _beta2, + 'multi_precision', + find_master, + ) return None @@ -371,7 +456,7 @@ class Adam(Optimizer): "Moment1": [moment1], "Moment2": [moment2], "Beta1Pow": [beta1_pow_acc], - "Beta2Pow": [beta2_pow_acc] + "Beta2Pow": [beta2_pow_acc], } outputs = { "ParamOut": [param_and_grad[0]], @@ -383,7 +468,7 @@ class Adam(Optimizer): attrs = { "lazy_mode": self._lazy_mode, "min_row_size_to_use_multithread": 1000, - "multi_precision": find_master + "multi_precision": find_master, } if isinstance(self._beta1, Variable): @@ -403,11 +488,13 @@ class Adam(Optimizer): inputs["MasterParam"] = master_weight outputs["MasterParamOut"] = master_weight - adam_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + adam_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return adam_op @@ -443,24 +530,28 @@ class Adam(Optimizer): if param._grad_ivar() is not None: grad_var = param._grad_ivar() if in_dygraph_mode(): - if hasattr(grad_var, "is_selected_rows" - ) and grad_var.is_selected_rows( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "is_selected_rows") + and grad_var.is_selected_rows() + and self.regularization is not None + ): raise RuntimeError( "Adam don't support weight_decay with sparse parameters, please set it to None." ) else: - if hasattr( - grad_var, "_is_sparse") and grad_var._is_sparse( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "_is_sparse") + and grad_var._is_sparse() + and self.regularization is not None + ): raise RuntimeError( "Adam don't support weight_decay with sparse parameters, please set it to None." ) params_grads.append((param, grad_var)) - optimize_ops = self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + optimize_ops = self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) else: # optimize parameters in groups for param_group in self._param_groups: @@ -472,11 +563,11 @@ class Adam(Optimizer): grad_var = param._grad_ivar() params_grads['params'].append((param, grad_var)) params_grads.update( - {k: v - for k, v in param_group.items() if k != 'params'}) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + {k: v for k, v in param_group.items() if k != 'params'} + ) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) def _multi_tensor_init(self, target_block, parameters): """ @@ -490,10 +581,12 @@ class Adam(Optimizer): for param in parameters: moment1 = self._get_accumulator(self._moment1_acc_str, param) moment2 = self._get_accumulator(self._moment2_acc_str, param) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param + ) if param.dtype == paddle.float32: self._param_dict['FP32_LODTensor'].append(param) @@ -509,7 +602,8 @@ class Adam(Optimizer): self._beta2_pow_acc_dict['FP16_LODTensor'].append(beta2_pow_acc) if self._multi_precision: self._master_weight_dict['FP16_LODTensor'].append( - self._master_weights[param.name]) + self._master_weights[param.name] + ) else: self._master_weight_dict['FP16_LODTensor'] = None else: @@ -517,8 +611,9 @@ class Adam(Optimizer): "Now multi_tensor_momentum only support fp32 and fp16 parameters and grad is LOD_TENSOR." ) - def _append_optimize_multi_tensor_op(self, target_block, - parameters_and_grads): + def _append_optimize_multi_tensor_op( + self, target_block, parameters_and_grads + ): """ For Multi Tensor, append optimize merged_operator to block. """ @@ -532,15 +627,19 @@ class Adam(Optimizer): if param_and_grad[1] is None: continue if param_and_grad[0].stop_gradient is False: - if param_and_grad[ - 0].dtype == paddle.float32 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + if ( + param_and_grad[0].dtype == paddle.float32 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP32_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP32_LODTensor'].append(lr) - elif param_and_grad[ - 0].dtype == paddle.float16 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + elif ( + param_and_grad[0].dtype == paddle.float16 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP16_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP16_LODTensor'].append(lr) @@ -551,21 +650,27 @@ class Adam(Optimizer): if param_and_grad[0].stop_gradient is False: param_grad_dict = dict() param_grad_dict['params'] = param_and_grad - param_grad_dict.update({ - k: v - for k, v in parameters_and_grads.items() - if k != 'params' - }) + param_grad_dict.update( + { + k: v + for k, v in parameters_and_grads.items() + if k != 'params' + } + ) param_and_grad = self._update_param_group(param_grad_dict) - if param_and_grad[ - 0].dtype == paddle.float32 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + if ( + param_and_grad[0].dtype == paddle.float32 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP32_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP32_LODTensor'].append(lr) - elif param_and_grad[ - 0].dtype == paddle.float16 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + elif ( + param_and_grad[0].dtype == paddle.float16 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP16_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP16_LODTensor'].append(lr) @@ -575,34 +680,59 @@ class Adam(Optimizer): if len(self._param_dict[key]) > 0: find_master = self._multi_precision and key == 'FP16_LODTensor' - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) if framework._non_static_mode(): if in_dygraph_mode(): _, _, _, _, _, _ = _C_ops.merged_adam_( - self._param_dict[key], grad_dict[key], lr_dict[key], - self._moment1_dict[key], self._moment2_dict[key], + self._param_dict[key], + grad_dict[key], + lr_dict[key], + self._moment1_dict[key], + self._moment2_dict[key], self._beta1_pow_acc_dict[key], self._beta2_pow_acc_dict[key], - self._master_weight_dict[key], _beta1, _beta2, - self._epsilon, find_master, False) + self._master_weight_dict[key], + _beta1, + _beta2, + self._epsilon, + find_master, + False, + ) else: _, _, _, _, _, _ = _legacy_C_ops.merged_adam( - self._param_dict[key], grad_dict[key], lr_dict[key], - self._moment1_dict[key], self._moment2_dict[key], + self._param_dict[key], + grad_dict[key], + lr_dict[key], + self._moment1_dict[key], + self._moment2_dict[key], self._beta1_pow_acc_dict[key], self._beta2_pow_acc_dict[key], self._master_weight_dict[key], - self._param_dict[key], self._moment1_dict[key], + self._param_dict[key], + self._moment1_dict[key], self._moment2_dict[key], self._beta1_pow_acc_dict[key], self._beta2_pow_acc_dict[key], - self._master_weight_dict[key], 'epsilon', - self._epsilon, 'beta1', _beta1, 'beta2', _beta2, - 'multi_precision', find_master) + self._master_weight_dict[key], + 'epsilon', + self._epsilon, + 'beta1', + _beta1, + 'beta2', + _beta2, + 'multi_precision', + find_master, + ) else: inputs = { "Param": self._param_dict[key], @@ -611,37 +741,41 @@ class Adam(Optimizer): "Moment1": self._moment1_dict[key], "Moment2": self._moment2_dict[key], "Beta1Pow": self._beta1_pow_acc_dict[key], - "Beta2Pow": self._beta2_pow_acc_dict[key] + "Beta2Pow": self._beta2_pow_acc_dict[key], } outputs = { "ParamOut": self._param_dict[key], "Moment1Out": self._moment1_dict[key], "Moment2Out": self._moment2_dict[key], "Beta1PowOut": self._beta1_pow_acc_dict[key], - "Beta2PowOut": self._beta2_pow_acc_dict[key] + "Beta2PowOut": self._beta2_pow_acc_dict[key], } attrs = { "epsilon": self._epsilon, "beta1": _beta1, - "beta2": _beta2 + "beta2": _beta2, } if find_master: inputs["MasterParam"] = self._master_weight_dict[key] outputs["MasterParamOut"] = self._master_weight_dict[ - key] + key + ] attrs["multi_precision"] = find_master - target_block.append_op(type="merged_adam", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + target_block.append_op( + type="merged_adam", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return None def _update_param_group(self, parameters): self._beta1 = parameters.get('beta1', self._default_dict['beta1']) self._beta2 = parameters.get('beta2', self._default_dict['beta2']) self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) - self._lazy_mode = parameters.get('lazy_mode', - self._default_dict['lazy_mode']) + self._lazy_mode = parameters.get( + 'lazy_mode', self._default_dict['lazy_mode'] + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/adamax.py b/python/paddle/optimizer/adamax.py index 81ae8531f0e..0a809e6acf8 100644 --- a/python/paddle/optimizer/adamax.py +++ b/python/paddle/optimizer/adamax.py @@ -132,15 +132,17 @@ class Adamax(Optimizer): _inf_norm_acc_str = "inf_norm" _beta1_pow_acc_str = "beta1_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None @@ -151,11 +153,13 @@ class Adamax(Optimizer): raise ValueError("Invaild value of beta2, expect beta2 in [0,1).") if not 0 <= epsilon: raise ValueError("Invaild value of epsilon, expect epsilon >= 0.") - super(Adamax, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(Adamax, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "adamax" self._beta1 = beta1 self._beta2 = beta2 @@ -163,7 +167,7 @@ class Adamax(Optimizer): self._default_dict = { 'beta1': beta1, 'beta2': beta2, - 'epsilon': epsilon + 'epsilon': epsilon, } def _create_accumulators(self, block, parameters): @@ -174,10 +178,12 @@ class Adamax(Optimizer): for p in parameters: self._add_accumulator(self._moment_acc_str, p) self._add_accumulator(self._inf_norm_acc_str, p) - self._add_accumulator(name=self._beta1_pow_acc_str, - param=p, - fill_value=self._beta1, - shape=[1]) + self._add_accumulator( + name=self._beta1_pow_acc_str, + param=p, + fill_value=self._beta1, + shape=[1], + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -185,22 +191,43 @@ class Adamax(Optimizer): param_and_grad = self._update_param_group(param_and_grad) moment = self._get_accumulator(self._moment_acc_str, param_and_grad[0]) - inf_norm = self._get_accumulator(self._inf_norm_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) + inf_norm = self._get_accumulator( + self._inf_norm_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) if framework.in_dygraph_mode(): - _C_ops.adamax_(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), moment, - inf_norm, beta1_pow_acc, self._beta1, self._beta2, - self._epsilon) + _C_ops.adamax_( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + moment, + inf_norm, + beta1_pow_acc, + self._beta1, + self._beta2, + self._epsilon, + ) elif framework._in_legacy_dygraph(): - _legacy_C_ops.adamax(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), moment, - inf_norm, beta1_pow_acc, param_and_grad[0], - moment, inf_norm, "beta1", self._beta1, - "beta2", self._beta2, "epsilon", self._epsilon) + _legacy_C_ops.adamax( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + moment, + inf_norm, + beta1_pow_acc, + param_and_grad[0], + moment, + inf_norm, + "beta1", + self._beta1, + "beta2", + self._beta2, + "epsilon", + self._epsilon, + ) else: # create the adamax optimize op adamax_op = block.append_op( @@ -211,25 +238,25 @@ class Adamax(Optimizer): "LearningRate": self._create_param_lr(param_and_grad), "Moment": moment, "InfNorm": inf_norm, - "Beta1Pow": beta1_pow_acc + "Beta1Pow": beta1_pow_acc, }, outputs={ "ParamOut": param_and_grad[0], "MomentOut": moment, - "InfNormOut": inf_norm + "InfNormOut": inf_norm, }, attrs={ "beta1": self._beta1, "beta2": self._beta2, - "epsilon": self._epsilon + "epsilon": self._epsilon, }, - stop_gradient=True) + stop_gradient=True, + ) return adamax_op def _finish_update(self, block, parameters_and_grads): - """Update Beta1 Power accumulator - """ + """Update Beta1 Power accumulator""" assert isinstance(block, framework.Block) if isinstance(parameters_and_grads, list): for param, grad in parameters_and_grads: @@ -237,47 +264,61 @@ class Adamax(Optimizer): continue if framework.in_dygraph_mode(): beta1_pow_acc = self._get_accumulator( - self._beta1_pow_acc_str, param) + self._beta1_pow_acc_str, param + ) with no_grad(): - tmp = _C_ops.scale(beta1_pow_acc, self._beta1, 0.0, - True) + tmp = _C_ops.scale( + beta1_pow_acc, self._beta1, 0.0, True + ) beta1_pow_acc.copy_(tmp, False) continue with param.block.program._optimized_guard( - [param, grad]), name_scope('adamax'): + [param, grad] + ), name_scope('adamax'): beta1_pow_acc = self._get_accumulator( - self._beta1_pow_acc_str, param) - block.append_op(type="scale", - inputs={"X": beta1_pow_acc}, - outputs={"Out": beta1_pow_acc}, - attrs={"scale": self._beta1}, - stop_gradient=True) + self._beta1_pow_acc_str, param + ) + block.append_op( + type="scale", + inputs={"X": beta1_pow_acc}, + outputs={"Out": beta1_pow_acc}, + attrs={"scale": self._beta1}, + stop_gradient=True, + ) else: for param, grad in parameters_and_grads['params']: if grad is None or param.stop_gradient is True: continue if framework.in_dygraph_mode(): beta1_pow_acc = self._get_accumulator( - self._beta1_pow_acc_str, param) + self._beta1_pow_acc_str, param + ) self._beta1 = parameters_and_grads.get( - 'beta1', self._default_dict['beta1']) + 'beta1', self._default_dict['beta1'] + ) with no_grad(): - tmp = _C_ops.scale(beta1_pow_acc, self._beta1, 0.0, - True) + tmp = _C_ops.scale( + beta1_pow_acc, self._beta1, 0.0, True + ) beta1_pow_acc.copy_(tmp, False) continue with param.block.program._optimized_guard( - [param, grad]), name_scope('adamax'): + [param, grad] + ), name_scope('adamax'): beta1_pow_acc = self._get_accumulator( - self._beta1_pow_acc_str, param) + self._beta1_pow_acc_str, param + ) self._beta1 = parameters_and_grads.get( - 'beta1', self._default_dict['beta1']) - block.append_op(type="scale", - inputs={"X": beta1_pow_acc}, - outputs={"Out": beta1_pow_acc}, - attrs={"scale": self._beta1}, - stop_gradient=True) + 'beta1', self._default_dict['beta1'] + ) + block.append_op( + type="scale", + inputs={"X": beta1_pow_acc}, + outputs={"Out": beta1_pow_acc}, + attrs={"scale": self._beta1}, + stop_gradient=True, + ) def _update_param_group(self, parameters): self._beta1 = parameters.get('beta1', self._default_dict['beta1']) diff --git a/python/paddle/optimizer/adamw.py b/python/paddle/optimizer/adamw.py index 1c0dbb3134c..dca844b6682 100644 --- a/python/paddle/optimizer/adamw.py +++ b/python/paddle/optimizer/adamw.py @@ -148,19 +148,21 @@ class AdamW(Optimizer): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameters=None, - weight_decay=0.01, - lr_ratio=None, - apply_decay_param_fun=None, - grad_clip=None, - lazy_mode=False, - multi_precision=False, - name=None): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameters=None, + weight_decay=0.01, + lr_ratio=None, + apply_decay_param_fun=None, + grad_clip=None, + lazy_mode=False, + multi_precision=False, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None @@ -171,14 +173,16 @@ class AdamW(Optimizer): raise ValueError("Invaild value of beta2, expect beta2 in [0,1).") if not 0 <= epsilon: raise ValueError("Invaild value of epsilon, expect epsilon >= 0.") - if not isinstance(weight_decay, float) and \ - not isinstance(weight_decay, framework.Variable): + if not isinstance(weight_decay, float) and not isinstance( + weight_decay, framework.Variable + ): raise TypeError("weight_decay should be float or Tensor.") if lr_ratio is not None: assert isinstance(lr_ratio, Callable) if not core.is_compiled_with_cuda(): raise NotImplementedError( - "'lr_ratio' is unimplemented in CPU, XPU and NPU") + "'lr_ratio' is unimplemented in CPU, XPU and NPU" + ) if parameters is not None: # paddle.Tensor is also iterable, so here we don't check whether @@ -187,13 +191,16 @@ class AdamW(Optimizer): if isinstance(parameters, (paddle.Tensor, core.eager.Tensor)): raise TypeError( "`parameters` argument given to the optimizer should be " - "an iterable of paddle Tensors, but got argument type is `{}`." - .format(type(parameters))) + "an iterable of paddle Tensors, but got argument type is `{}`.".format( + type(parameters) + ) + ) if isinstance(parameters, dict): raise TypeError( "`parameters` argument should not get dict type, " "if parameter groups is needed, please set `parameters`" - " as list of dict") + " as list of dict" + ) self._parameter_list = list(parameters) else: self._parameter_list = None @@ -207,8 +214,9 @@ class AdamW(Optimizer): if not isinstance(learning_rate, (float, LRScheduler)): raise TypeError( - "learning rate should be float or LRScheduler, got %s here" % - type(learning_rate)) + "learning rate should be float or LRScheduler, got %s here" + % type(learning_rate) + ) if grad_clip is not None: if not isinstance(grad_clip, GradientClipBase): raise TypeError( @@ -220,8 +228,9 @@ class AdamW(Optimizer): if self._parameter_list: if isinstance(self._parameter_list[0], dict): for param_group in self._parameter_list: - assert 'params' in param_group, \ - 'params should be set in parameters if parameter groups are optimized in different options' + assert ( + 'params' in param_group + ), 'params should be set in parameters if parameter groups are optimized in different options' self._dtype = self._parameter_list[0]['params'][0].dtype else: self._dtype = self._parameter_list[0].dtype @@ -260,7 +269,7 @@ class AdamW(Optimizer): 'beta2': beta2, 'epsilon': epsilon, 'lazy_mode': lazy_mode, - 'grad_clip': grad_clip + 'grad_clip': grad_clip, } self._param_groups = [] @@ -297,7 +306,8 @@ class AdamW(Optimizer): elif isinstance(params, set): raise TypeError( "optimizer parameters should be in ordered collections," - "but received set, please use list instead.") + "but received set, please use list instead." + ) else: param_group['params'] = list(params) @@ -311,11 +321,13 @@ class AdamW(Optimizer): if not param_set.isdisjoint(set(param_group['params'])): raise ValueError( - "some parameters appear in more than one parameter group") + "some parameters appear in more than one parameter group" + ) for param in param_group['params']: param.optimize_attr['learning_rate'] = param_group.get( - 'learning_rate', 1.) + 'learning_rate', 1.0 + ) self._param_groups.append(param_group) @@ -327,19 +339,23 @@ class AdamW(Optimizer): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -353,15 +369,22 @@ class AdamW(Optimizer): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _add_moments_pows(self, p): @@ -374,18 +397,24 @@ class AdamW(Optimizer): name=self._beta1_pow_acc_str, param=p, dtype=acc_dtype, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_accumulator( name=self._beta2_pow_acc_str, param=p, dtype=acc_dtype, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -398,7 +427,10 @@ class AdamW(Optimizer): master_p = self._create_master_weight(p) self._add_moments_pows(master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Adam optimizer." @@ -413,53 +445,112 @@ class AdamW(Optimizer): # Whether we should do weight decay for the parameter. with_decay = True - if self._apply_decay_param_fun is not None \ - and not self._apply_decay_param_fun(param.name): + if ( + self._apply_decay_param_fun is not None + and not self._apply_decay_param_fun(param.name) + ): with_decay = False - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) lr = self._create_param_lr(param_and_grad) # create the adamw optimize op if framework._non_static_mode(): - lr_ratio_ = 1. if self._lr_ratio is None else self._lr_ratio( - param_and_grad[0]) - - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + lr_ratio_ = ( + 1.0 + if self._lr_ratio is None + else self._lr_ratio(param_and_grad[0]) + ) + + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) if framework.in_dygraph_mode(): found_inf = self._get_auxiliary_var('found_inf') _, _, _, _, _, _ = _C_ops.adamw_( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, found_inf, - _beta1, _beta2, self._epsilon, lr_ratio_, - self._weight_decay, with_decay, self._lazy_mode, 1000, - find_master, False) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + found_inf, + _beta1, + _beta2, + self._epsilon, + lr_ratio_, + self._weight_decay, + with_decay, + self._lazy_mode, + 1000, + find_master, + False, + ) else: _, _, _, _, _, _ = _legacy_C_ops.adamw( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, - param_and_grad[0], moment1, moment2, beta1_pow_acc, - beta2_pow_acc, master_weight, 'epsilon', self._epsilon, - 'lazy_mode', self._lazy_mode, - 'min_row_size_to_use_multithread', 1000, 'beta1', _beta1, - 'beta2', _beta2, "with_decay", with_decay, 'coeff', - self._weight_decay, 'multi_precision', find_master, - 'lr_ratio', lr_ratio_) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'epsilon', + self._epsilon, + 'lazy_mode', + self._lazy_mode, + 'min_row_size_to_use_multithread', + 1000, + 'beta1', + _beta1, + 'beta2', + _beta2, + "with_decay", + with_decay, + 'coeff', + self._weight_decay, + 'multi_precision', + find_master, + 'lr_ratio', + lr_ratio_, + ) return None inputs = { @@ -486,18 +577,14 @@ class AdamW(Optimizer): "Beta2PowOut": [beta2_pow_acc], } attrs = { - "lazy_mode": - self._lazy_mode, - "min_row_size_to_use_multithread": - 1000, - "multi_precision": - find_master, - "with_decay": - with_decay, - "coeff": - self._weight_decay, - "lr_ratio": - 1. if self._lr_ratio is None else self._lr_ratio(param_and_grad[0]) + "lazy_mode": self._lazy_mode, + "min_row_size_to_use_multithread": 1000, + "multi_precision": find_master, + "with_decay": with_decay, + "coeff": self._weight_decay, + "lr_ratio": 1.0 + if self._lr_ratio is None + else self._lr_ratio(param_and_grad[0]), } if isinstance(self._beta1, Variable): @@ -517,11 +604,13 @@ class AdamW(Optimizer): inputs["MasterParam"] = master_weight outputs["MasterParamOut"] = master_weight - adamw_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + adamw_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return adamw_op @@ -560,24 +649,28 @@ class AdamW(Optimizer): if param._grad_ivar() is not None: grad_var = param._grad_ivar() if framework.in_dygraph_mode(): - if hasattr(grad_var, "is_selected_rows" - ) and grad_var.is_selected_rows( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "is_selected_rows") + and grad_var.is_selected_rows() + and self.regularization is not None + ): raise RuntimeError( "AdamW don't support weight_decay with sparse parameters, please set it to None." ) else: - if hasattr( - grad_var, "_is_sparse") and grad_var._is_sparse( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "_is_sparse") + and grad_var._is_sparse() + and self.regularization is not None + ): raise RuntimeError( "AdamW don't support weight_decay with sparse parameters, please set it to None." ) params_grads.append((param, grad_var)) - optimize_ops = self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + optimize_ops = self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) else: # optimize parameters in groups for param_group in self._param_groups: @@ -588,35 +681,41 @@ class AdamW(Optimizer): if param._grad_ivar() is not None: grad_var = param._grad_ivar() if framework.in_dygraph_mode(): - if hasattr(grad_var, "is_selected_rows" - ) and grad_var.is_selected_rows( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "is_selected_rows") + and grad_var.is_selected_rows() + and self.regularization is not None + ): raise RuntimeError( "AdamW don't support weight_decay with sparse parameters, please set it to None." ) else: - if hasattr(grad_var, - "_is_sparse") and grad_var._is_sparse( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "_is_sparse") + and grad_var._is_sparse() + and self.regularization is not None + ): raise RuntimeError( "AdamW don't support weight_decay with sparse parameters, please set it to None." ) params_grads['params'].append((param, grad_var)) params_grads.update( - {k: v - for k, v in param_group.items() if k != 'params'}) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + {k: v for k, v in param_group.items() if k != 'params'} + ) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) def _update_param_group(self, parameters): self._beta1 = parameters.get('beta1', self._default_dict['beta1']) self._beta2 = parameters.get('beta2', self._default_dict['beta2']) self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) - self._lazy_mode = parameters.get('lazy_mode', - self._default_dict['lazy_mode']) - self._weight_decay = parameters.get('weight_decay', - self._default_dict['weight_decay']) + self._lazy_mode = parameters.get( + 'lazy_mode', self._default_dict['lazy_mode'] + ) + self._weight_decay = parameters.get( + 'weight_decay', self._default_dict['weight_decay'] + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/lamb.py b/python/paddle/optimizer/lamb.py index 6a683c6ef91..4fc46ad1052 100644 --- a/python/paddle/optimizer/lamb.py +++ b/python/paddle/optimizer/lamb.py @@ -98,26 +98,30 @@ class Lamb(Optimizer): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - lamb_weight_decay=0.01, - beta1=0.9, - beta2=0.999, - epsilon=1e-6, - parameters=None, - grad_clip=None, - exclude_from_weight_decay_fn=None, - multi_precision=False, - name=None): + def __init__( + self, + learning_rate=0.001, + lamb_weight_decay=0.01, + beta1=0.9, + beta2=0.999, + epsilon=1e-6, + parameters=None, + grad_clip=None, + exclude_from_weight_decay_fn=None, + multi_precision=False, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(Lamb, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=None, - grad_clip=grad_clip, - name=name) + super(Lamb, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=None, + grad_clip=grad_clip, + name=name, + ) self.type = "lamb" self._beta1 = beta1 self._beta2 = beta2 @@ -160,19 +164,23 @@ class Lamb(Optimizer): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -199,15 +207,22 @@ class Lamb(Optimizer): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _add_moments_pows(self, p): @@ -218,21 +233,27 @@ class Lamb(Optimizer): self._add_accumulator(self._moment1_acc_str, p, dtype=acc_dtype) self._add_accumulator(self._moment2_acc_str, p, dtype=acc_dtype) self._add_accumulator( - name=self._beta1_pow_acc_str, - param=p, - dtype=acc_dtype, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, - shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + name=self._beta1_pow_acc_str, + param=p, + dtype=acc_dtype, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, + shape=[1], + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_accumulator( - name=self._beta2_pow_acc_str, - param=p, - dtype=acc_dtype, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, - shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + name=self._beta2_pow_acc_str, + param=p, + dtype=acc_dtype, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, + shape=[1], + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -241,24 +262,32 @@ class Lamb(Optimizer): block.program._use_lamb = True - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - - if self._exclude_from_weight_decay_fn is not None \ - and self._exclude_from_weight_decay_fn(param_and_grad[0]): + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + + if ( + self._exclude_from_weight_decay_fn is not None + and self._exclude_from_weight_decay_fn(param_and_grad[0]) + ): weight_decay = 0.0 else: weight_decay = self._lamb_weight_decay lr = self._create_param_lr(param_and_grad) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) p_name = param_and_grad[0].name if find_master: master_weight = self._master_weights[p_name] @@ -268,20 +297,50 @@ class Lamb(Optimizer): found_inf = self._get_auxiliary_var('found_inf') if framework.in_dygraph_mode(): - _C_ops.lamb_(param_and_grad[0], param_and_grad[1], lr, moment1, - moment2, beta1_pow_acc, beta2_pow_acc, master_weight, - found_inf, weight_decay, self._beta1, self._beta2, - self._epsilon, find_master) + _C_ops.lamb_( + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + found_inf, + weight_decay, + self._beta1, + self._beta2, + self._epsilon, + find_master, + ) return None if framework._non_static_mode(): - _legacy_C_ops.lamb(param_and_grad[0], param_and_grad[1], lr, - moment1, moment2, beta1_pow_acc, beta2_pow_acc, - master_weight, param_and_grad[0], moment1, - moment2, beta1_pow_acc, beta2_pow_acc, - master_weight, 'beta1', self._beta1, 'beta2', - self._beta2, 'epsilon', self._epsilon, - 'weight_decay', weight_decay, 'multi_precision', - find_master) + _legacy_C_ops.lamb( + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'beta1', + self._beta1, + 'beta2', + self._beta2, + 'epsilon', + self._epsilon, + 'weight_decay', + weight_decay, + 'multi_precision', + find_master, + ) return None # create the lamb optimize op @@ -292,14 +351,14 @@ class Lamb(Optimizer): "Moment1": moment1, "Moment2": moment2, "Beta1Pow": beta1_pow_acc, - "Beta2Pow": beta2_pow_acc + "Beta2Pow": beta2_pow_acc, } outputs = { "ParamOut": param_and_grad[0], "Moment1Out": moment1, "Moment2Out": moment2, "Beta1PowOut": beta1_pow_acc, - "Beta2PowOut": beta2_pow_acc + "Beta2PowOut": beta2_pow_acc, } attrs = { "beta1": self._beta1, @@ -316,11 +375,13 @@ class Lamb(Optimizer): if found_inf: inputs["SkipUpdate"] = found_inf - lamb_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + lamb_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return lamb_op @@ -329,9 +390,11 @@ class Lamb(Optimizer): self._beta2 = parameters.get('beta2', self._default_dict['beta2']) self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) self._lamb_weight_decay = parameters.get( - 'lamb_weight_decay', self._default_dict['lamb_weight_decay']) + 'lamb_weight_decay', self._default_dict['lamb_weight_decay'] + ) self._exclude_from_weight_decay_fn = parameters.get( 'exclude_from_weight_decay_fn', - self._default_dict['exclude_from_weight_decay_fn']) + self._default_dict['exclude_from_weight_decay_fn'], + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/lr.py b/python/paddle/optimizer/lr.py index 4dc451d6334..4e2dbafcabb 100644 --- a/python/paddle/optimizer/lr.py +++ b/python/paddle/optimizer/lr.py @@ -93,8 +93,10 @@ class LRScheduler(object): def __init__(self, learning_rate=0.1, last_epoch=-1, verbose=False): if not isinstance(learning_rate, (float, int)): raise TypeError( - "The type of learning rate must be float, but received {}". - format(type(learning_rate))) + "The type of learning rate must be float, but received {}".format( + type(learning_rate) + ) + ) self.base_lr = float(learning_rate) self.last_lr = float(learning_rate) self.last_epoch = last_epoch @@ -133,8 +135,11 @@ class LRScheduler(object): self.last_lr = self.get_lr() if self.verbose: - print('Epoch {}: {} set learning rate to {}.'.format( - self.last_epoch, self.__class__.__name__, self.last_lr)) + print( + 'Epoch {}: {} set learning rate to {}.'.format( + self.last_epoch, self.__class__.__name__, self.last_lr + ) + ) def state_dict(self): """ @@ -153,7 +158,8 @@ class LRScheduler(object): assert value.shape == [ 1 ], "shape of Tensor in state_dict must be [1] {}".format( - value.shape) + value.shape + ) value = value.numpy()[0] state_dict[key] = value @@ -184,8 +190,10 @@ class LRScheduler(object): self.__dict__[key] = state_dict[key] else: raise RuntimeError( - "Please check whether state_dict is correct for optimizer. Can't find [ {} ] in state_dict" - .format(key)) + "Please check whether state_dict is correct for optimizer. Can't find [ {} ] in state_dict".format( + key + ) + ) if len(state_dict) > len(self.keys): warnings.warn( "There are some unused values in state_dict. Maybe the optimizer have different 'LearningRateDecay' when invoking state_dict and set_dict" @@ -279,12 +287,14 @@ class NoamDecay(LRScheduler): """ - def __init__(self, - d_model, - warmup_steps, - learning_rate=1.0, - last_epoch=-1, - verbose=False): + def __init__( + self, + d_model, + warmup_steps, + learning_rate=1.0, + last_epoch=-1, + verbose=False, + ): self.d_model = d_model self.warmup_steps = warmup_steps super(NoamDecay, self).__init__(learning_rate, last_epoch, verbose) @@ -379,8 +389,9 @@ class PiecewiseDecay(LRScheduler): def __init__(self, boundaries, values, last_epoch=-1, verbose=False): self.boundaries = boundaries self.values = values - super(PiecewiseDecay, self).__init__(last_epoch=last_epoch, - verbose=verbose) + super(PiecewiseDecay, self).__init__( + last_epoch=last_epoch, verbose=verbose + ) def get_lr(self): for i in range(len(self.boundaries)): @@ -460,10 +471,13 @@ class NaturalExpDecay(LRScheduler): """ def __init__(self, learning_rate, gamma, last_epoch=-1, verbose=False): - assert gamma > 0.0, " 'gamma' must be a positive number so that the learning rate will decay." + assert ( + gamma > 0.0 + ), " 'gamma' must be a positive number so that the learning rate will decay." self.gamma = gamma - super(NaturalExpDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(NaturalExpDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): return self.base_lr * math.exp(-1 * self.gamma * self.last_epoch) @@ -543,8 +557,9 @@ class InverseTimeDecay(LRScheduler): def __init__(self, learning_rate, gamma, last_epoch=-1, verbose=False): self.gamma = gamma - super(InverseTimeDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(InverseTimeDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): return self.base_lr / (1 + self.gamma * self.last_epoch) @@ -637,30 +652,37 @@ class PolynomialDecay(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - decay_steps, - end_lr=0.0001, - power=1.0, - cycle=False, - last_epoch=-1, - verbose=False): + def __init__( + self, + learning_rate, + decay_steps, + end_lr=0.0001, + power=1.0, + cycle=False, + last_epoch=-1, + verbose=False, + ): assert decay_steps > 0 and isinstance( - decay_steps, int), " 'decay_steps' must be a positive integer." + decay_steps, int + ), " 'decay_steps' must be a positive integer." self.decay_steps = decay_steps self.end_lr = end_lr - assert power > 0.0, " 'power' must be greater than 0.0 so that the learning rate will decay." + assert ( + power > 0.0 + ), " 'power' must be greater than 0.0 so that the learning rate will decay." self.power = power self.cycle = cycle - super(PolynomialDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(PolynomialDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): tmp_epoch_num = self.last_epoch tmp_decay_steps = self.decay_steps if self.cycle: div_res = math.ceil( - float(self.last_epoch) / float(self.decay_steps)) + float(self.last_epoch) / float(self.decay_steps) + ) if self.last_epoch == 0: div_res = 1 @@ -669,8 +691,8 @@ class PolynomialDecay(LRScheduler): tmp_epoch_num = min(self.last_epoch, self.decay_steps) return (self.base_lr - self.end_lr) * ( - (1 - float(tmp_epoch_num) / float(tmp_decay_steps))** - self.power) + self.end_lr + (1 - float(tmp_epoch_num) / float(tmp_decay_steps)) ** self.power + ) + self.end_lr class LinearWarmup(LRScheduler): @@ -758,27 +780,36 @@ class LinearWarmup(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - warmup_steps, - start_lr, - end_lr, - last_epoch=-1, - verbose=False): - type_check = isinstance(learning_rate, float) or isinstance( - learning_rate, int) or isinstance(learning_rate, LRScheduler) + def __init__( + self, + learning_rate, + warmup_steps, + start_lr, + end_lr, + last_epoch=-1, + verbose=False, + ): + type_check = ( + isinstance(learning_rate, float) + or isinstance(learning_rate, int) + or isinstance(learning_rate, LRScheduler) + ) if not type_check: raise TypeError( - "the type of learning_rate should be [int, float or LRScheduler], the current type is {}" - .format(learning_rate)) + "the type of learning_rate should be [int, float or LRScheduler], the current type is {}".format( + learning_rate + ) + ) self.learning_rate = learning_rate assert warmup_steps > 0 and isinstance( - warmup_steps, int), " 'warmup_steps' must be a positive integer." + warmup_steps, int + ), " 'warmup_steps' must be a positive integer." self.warmup_steps = warmup_steps self.start_lr = start_lr self.end_lr = end_lr - assert end_lr > start_lr, "end_lr {} must be greater than start_lr {}".format( - end_lr, start_lr) + assert ( + end_lr > start_lr + ), "end_lr {} must be greater than start_lr {}".format(end_lr, start_lr) super(LinearWarmup, self).__init__(start_lr, last_epoch, verbose) def state_dict(self): @@ -803,7 +834,8 @@ class LinearWarmup(LRScheduler): def get_lr(self): if self.last_epoch < self.warmup_steps: return (self.end_lr - self.start_lr) * float( - self.last_epoch) / float(self.warmup_steps) + self.start_lr + self.last_epoch + ) / float(self.warmup_steps) + self.start_lr else: if isinstance(self.learning_rate, LRScheduler): self.learning_rate.step(self.last_epoch - self.warmup_steps) @@ -884,10 +916,13 @@ class ExponentialDecay(LRScheduler): """ def __init__(self, learning_rate, gamma, last_epoch=-1, verbose=False): - assert gamma > 0.0 and gamma < 1.0, " 'gamma' must be in interval (0.0, 1.0) so that the learning rate will decay." + assert ( + gamma > 0.0 and gamma < 1.0 + ), " 'gamma' must be in interval (0.0, 1.0) so that the learning rate will decay." self.gamma = gamma - super(ExponentialDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(ExponentialDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): return self.base_lr * (self.gamma**self.last_epoch) @@ -973,21 +1008,21 @@ class MultiStepDecay(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - milestones, - gamma=0.1, - last_epoch=-1, - verbose=False): + def __init__( + self, learning_rate, milestones, gamma=0.1, last_epoch=-1, verbose=False + ): if not isinstance(milestones, (tuple, list)): raise TypeError( "The type of 'milestones' in 'MultiStepDecay' must be 'tuple, list', but received %s." - % type(milestones)) + % type(milestones) + ) - if not all([ + if not all( + [ milestones[i] < milestones[i + 1] for i in range(len(milestones) - 1) - ]): + ] + ): raise ValueError('The elements of milestones must be incremented') if gamma >= 1.0: raise ValueError('gamma should be < 1.0.') @@ -1000,7 +1035,7 @@ class MultiStepDecay(LRScheduler): for i in range(len(self.milestones)): if self.last_epoch < self.milestones[i]: return self.base_lr * (self.gamma**i) - return self.base_lr * (self.gamma**len(self.milestones)) + return self.base_lr * (self.gamma ** len(self.milestones)) class StepDecay(LRScheduler): @@ -1082,21 +1117,20 @@ class StepDecay(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - step_size, - gamma=0.1, - last_epoch=-1, - verbose=False): + def __init__( + self, learning_rate, step_size, gamma=0.1, last_epoch=-1, verbose=False + ): if not isinstance(step_size, int): raise TypeError( - "The type of 'step_size' must be 'int', but received %s." % - type(step_size)) + "The type of 'step_size' must be 'int', but received %s." + % type(step_size) + ) if gamma >= 1.0: raise ValueError('gamma should be < 1.0.') assert step_size > 0 and isinstance( - step_size, int), " 'step_size' must be a positive integer." + step_size, int + ), " 'step_size' must be a positive integer." self.step_size = step_size self.gamma = gamma super(StepDecay, self).__init__(learning_rate, last_epoch, verbose) @@ -1185,7 +1219,8 @@ class LambdaDecay(LRScheduler): if not callable(lr_lambda): raise TypeError( "The type of 'lr_lambda' in 'LambdaDecay' must be 'function', but received %s." - % type(lr_lambda)) + % type(lr_lambda) + ) self.lr_lambda = lr_lambda super(LambdaDecay, self).__init__(learning_rate, last_epoch, verbose) @@ -1281,17 +1316,19 @@ class ReduceOnPlateau(LRScheduler): """ - def __init__(self, - learning_rate, - mode='min', - factor=0.1, - patience=10, - threshold=1e-4, - threshold_mode='rel', - cooldown=0, - min_lr=0, - epsilon=1e-8, - verbose=False): + def __init__( + self, + learning_rate, + mode='min', + factor=0.1, + patience=10, + threshold=1e-4, + threshold_mode='rel', + cooldown=0, + min_lr=0, + epsilon=1e-8, + verbose=False, + ): mode = mode.lower() if mode not in ['min', 'max']: raise ValueError('mode: ' + mode + ' is unknown!') @@ -1299,18 +1336,21 @@ class ReduceOnPlateau(LRScheduler): if factor >= 1.0: raise ValueError( - 'new_lr = origin_lr * gamma and gamma should be < 1.0.') + 'new_lr = origin_lr * gamma and gamma should be < 1.0.' + ) self.factor = factor threshold_mode = threshold_mode.lower() if threshold_mode not in ['rel', 'abs']: - raise ValueError('threshold mode: ' + threshold_mode + - ' is unknown!') + raise ValueError( + 'threshold mode: ' + threshold_mode + ' is unknown!' + ) self.threshold_mode = threshold_mode if not isinstance(learning_rate, (float, int)): raise TypeError( "The type of 'learning_rate' in 'ReduceOnPlateau' must be 'float', but received %s." - % type(learning_rate)) + % type(learning_rate) + ) self.patience = patience self.threshold = threshold @@ -1333,8 +1373,11 @@ class ReduceOnPlateau(LRScheduler): # "cooldown_counter / best / num_bad_epochs / last_epoch / last_lr" will be stored. def state_keys(self): self.keys = [ - 'cooldown_counter', 'best', 'num_bad_epochs', 'last_epoch', - 'last_lr' + 'cooldown_counter', + 'best', + 'num_bad_epochs', + 'last_epoch', + 'last_lr', ] def step(self, metrics, epoch=None): @@ -1364,18 +1407,25 @@ class ReduceOnPlateau(LRScheduler): else: # need to declarate explicitly from paddle.framework import VarBase as Tensor + tmp = Tensor # loss must be float, numpy.ndarray or 1-D Tensor with shape [1] if isinstance(metrics, (tmp, numpy.ndarray)): - assert len(metrics.shape) == 1 and metrics.shape[0] == 1, "the metrics.shape " \ - "should be (1L,), but the current metrics.shape is {}. Maybe that " \ - "you should call paddle.mean to process it first.".format( - metrics.shape) - elif not isinstance(metrics, - (int, float, numpy.float32, numpy.float64)): + assert len(metrics.shape) == 1 and metrics.shape[0] == 1, ( + "the metrics.shape " + "should be (1L,), but the current metrics.shape is {}. Maybe that " + "you should call paddle.mean to process it first.".format( + metrics.shape + ) + ) + elif not isinstance( + metrics, (int, float, numpy.float32, numpy.float64) + ): raise TypeError( - "metrics must be 'int', 'float', 'np.float', 'numpy.ndarray' or 'paddle.Tensor', but receive {}" - .format(type(metrics))) + "metrics must be 'int', 'float', 'np.float', 'numpy.ndarray' or 'paddle.Tensor', but receive {}".format( + type(metrics) + ) + ) if self.cooldown_counter > 0: self.cooldown_counter -= 1 @@ -1393,9 +1443,13 @@ class ReduceOnPlateau(LRScheduler): if self.last_lr - new_lr > self.epsilon: self.last_lr = new_lr if self.verbose: - print('Epoch {}: {} set learning rate to {}.'.format( - self.last_epoch, self.__class__.__name__, - self.last_lr)) + print( + 'Epoch {}: {} set learning rate to {}.'.format( + self.last_epoch, + self.__class__.__name__, + self.last_lr, + ) + ) def _is_better(self, current, best): if self.mode == 'min' and self.threshold_mode == 'rel': @@ -1493,41 +1547,50 @@ class CosineAnnealingDecay(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - T_max, - eta_min=0, - last_epoch=-1, - verbose=False): + def __init__( + self, learning_rate, T_max, eta_min=0, last_epoch=-1, verbose=False + ): if not isinstance(T_max, int): raise TypeError( "The type of 'T_max' in 'CosineAnnealingDecay' must be 'int', but received %s." - % type(T_max)) + % type(T_max) + ) if not isinstance(eta_min, (float, int)): raise TypeError( "The type of 'eta_min' in 'CosineAnnealingDecay' must be 'float, int', but received %s." - % type(eta_min)) + % type(eta_min) + ) assert T_max > 0 and isinstance( - T_max, int), " 'T_max' must be a positive integer." + T_max, int + ), " 'T_max' must be a positive integer." self.T_max = T_max self.eta_min = float(eta_min) - super(CosineAnnealingDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(CosineAnnealingDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): if self.last_epoch == 0: return self.base_lr elif (self.last_epoch - 1 - self.T_max) % (2 * self.T_max) == 0: - return self.last_lr + (self.base_lr - self.eta_min) * ( - 1 - math.cos(math.pi / self.T_max)) / 2 + return ( + self.last_lr + + (self.base_lr - self.eta_min) + * (1 - math.cos(math.pi / self.T_max)) + / 2 + ) return (1 + math.cos(math.pi * self.last_epoch / self.T_max)) / ( - 1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max)) * ( - self.last_lr - self.eta_min) + self.eta_min + 1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max) + ) * (self.last_lr - self.eta_min) + self.eta_min def _get_closed_form_lr(self): - return self.eta_min + (self.base_lr - self.eta_min) * ( - 1 + math.cos(math.pi * self.last_epoch / self.T_max)) / 2 + return ( + self.eta_min + + (self.base_lr - self.eta_min) + * (1 + math.cos(math.pi * self.last_epoch / self.T_max)) + / 2 + ) class MultiplicativeDecay(LRScheduler): @@ -1581,11 +1644,13 @@ class MultiplicativeDecay(LRScheduler): if not callable(lr_lambda): raise TypeError( "The type of 'lr_lambda' in 'MultiplicativeDecay' must be 'function', but received %s." - % type(lr_lambda)) + % type(lr_lambda) + ) self.lr_lambda = lr_lambda - super(MultiplicativeDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(MultiplicativeDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): cur_lr = self.base_lr @@ -1678,29 +1743,35 @@ class OneCycleLR(LRScheduler): scheduler.step() # You should update learning rate each step """ - def __init__(self, - max_learning_rate, - total_steps, - divide_factor=25., - end_learning_rate=0.0001, - phase_pct=0.3, - anneal_strategy='cos', - three_phase=False, - last_epoch=-1, - verbose=False): + def __init__( + self, + max_learning_rate, + total_steps, + divide_factor=25.0, + end_learning_rate=0.0001, + phase_pct=0.3, + anneal_strategy='cos', + three_phase=False, + last_epoch=-1, + verbose=False, + ): # Check type and value of max_learning_rate if not isinstance(max_learning_rate, (float, int)): raise TypeError( - "'max_learning_rate' must be 'float' or 'int', but received {}". - format(type(max_learning_rate))) + "'max_learning_rate' must be 'float' or 'int', but received {}".format( + type(max_learning_rate) + ) + ) if max_learning_rate < 0: raise ValueError("'max_learning_rate' must be a positive integer.") # Check type and value of end_learning_rate if not isinstance(end_learning_rate, (float, int)): raise TypeError( - "'end_learning_rate' must be 'float' or 'int', but received {}". - format(type(end_learning_rate))) + "'end_learning_rate' must be 'float' or 'int', but received {}".format( + type(end_learning_rate) + ) + ) if end_learning_rate < 0: raise ValueError("'end_learning_rate' must be a positive integer.") @@ -1708,7 +1779,9 @@ class OneCycleLR(LRScheduler): if not isinstance(total_steps, int): raise TypeError( "'total_step' must be 'int', but received {}".format( - type(total_steps))) + type(total_steps) + ) + ) if total_steps <= 0: raise ValueError("'total_step' must be a positive integer.") self.total_steps = total_steps @@ -1717,17 +1790,23 @@ class OneCycleLR(LRScheduler): if not isinstance(phase_pct, float): raise TypeError( "'phase_pct' must be 'float', but received {}".format( - type(phase_pct))) + type(phase_pct) + ) + ) if phase_pct < 0 or phase_pct > 1: raise ValueError( "'phase_pct' must be between 0 and 1, but received {}".format( - phase_pct)) + phase_pct + ) + ) # Check type and value of divide_factor if not isinstance(divide_factor, (float, int)): raise TypeError( - "'divide_factor' must be 'float' or 'int', but received {}". - format(type(divide_factor))) + "'divide_factor' must be 'float' or 'int', but received {}".format( + type(divide_factor) + ) + ) initial_lr = max_learning_rate / float(divide_factor) min_lr = float(end_learning_rate) @@ -1750,17 +1829,22 @@ class OneCycleLR(LRScheduler): self._step_config[1] - self._step_config[0], self._step_config[2] - self._step_config[1], self._step_config[3] - self._step_config[2], - self._step_config[3] - - self._step_config[2], # for the last step. + self._step_config[3] + - self._step_config[2], # for the last step. ] # start lr and end lr of each phase. self._lr_config = [ - initial_lr, max_learning_rate, initial_lr, min_lr + initial_lr, + max_learning_rate, + initial_lr, + min_lr, ] else: self._step_config = [ - 0, phase_pct * self.total_steps - 1, self.total_steps - 1, - self.total_steps - 1 + 0, + phase_pct * self.total_steps - 1, + self.total_steps - 1, + self.total_steps - 1, ] self._steps_size = [ self._step_config[1] - self._step_config[0], @@ -1776,8 +1860,10 @@ class OneCycleLR(LRScheduler): self.anneal_func = self._linear_annealing else: raise ValueError( - "'anneal_strategy' must by one of 'cos' or 'linear', but received {}" - .format(anneal_strategy)) + "'anneal_strategy' must by one of 'cos' or 'linear', but received {}".format( + anneal_strategy + ) + ) super(OneCycleLR, self).__init__(initial_lr, last_epoch, verbose) def _cos_annealing(self, start_lr, end_lr, pct): @@ -1792,17 +1878,21 @@ class OneCycleLR(LRScheduler): if current_step > self.total_steps: raise ValueError( - "Tried to step {} times. However the number of total steps is {}" - .format(current_step, self.total_steps)) + "Tried to step {} times. However the number of total steps is {}".format( + current_step, self.total_steps + ) + ) for (i, (end_step, step_size)) in enumerate( - zip(self._step_config[1:], self._steps_size)): + zip(self._step_config[1:], self._steps_size) + ): # i == len(self._lr_config) - 2 catch the last step, otherwise it will return None. if current_step <= end_step or i == len(self._lr_config) - 2: # self._step_config[i] means start step of a phase. percentage = (current_step - self._step_config[i]) / step_size - return self.anneal_func(self._lr_config[i], - self._lr_config[i + 1], percentage) + return self.anneal_func( + self._lr_config[i], self._lr_config[i + 1], percentage + ) class CyclicLR(LRScheduler): @@ -1896,71 +1986,93 @@ class CyclicLR(LRScheduler): scheduler.step() # You should update learning rate each step """ - def __init__(self, - base_learning_rate, - max_learning_rate, - step_size_up, - step_size_down=None, - mode='triangular', - exp_gamma=1., - scale_fn=None, - scale_mode='cycle', - last_epoch=-1, - verbose=False): + def __init__( + self, + base_learning_rate, + max_learning_rate, + step_size_up, + step_size_down=None, + mode='triangular', + exp_gamma=1.0, + scale_fn=None, + scale_mode='cycle', + last_epoch=-1, + verbose=False, + ): # check type and value of max_learning_rate if not isinstance(max_learning_rate, (float, int)): raise TypeError( - "'max_learning_rate' must be 'float' or 'int', but received {}". - format(type(max_learning_rate))) + "'max_learning_rate' must be 'float' or 'int', but received {}".format( + type(max_learning_rate) + ) + ) if max_learning_rate < 0: raise ValueError( - "'max_learning_rate' must be a positive integer, but received {}" - .format(max_learning_rate)) + "'max_learning_rate' must be a positive integer, but received {}".format( + max_learning_rate + ) + ) # check type and value of step_size_up if not isinstance(step_size_up, int): raise TypeError( - "The type of 'step_size_up' must be int, but received {}". - format(type(step_size_up))) + "The type of 'step_size_up' must be int, but received {}".format( + type(step_size_up) + ) + ) if step_size_up <= 0: raise ValueError( - "'step_size_up' must be a positive integer, but received {}". - format(step_size_up)) + "'step_size_up' must be a positive integer, but received {}".format( + step_size_up + ) + ) # check type and value of step_size_down if step_size_down is not None: if not isinstance(step_size_down, int): raise TypeError( - "The type of 'step_size_down' must be int, but received {}". - format(type(step_size_down))) + "The type of 'step_size_down' must be int, but received {}".format( + type(step_size_down) + ) + ) if step_size_down <= 0: raise ValueError( - "'step_size_down' must be a positive integer, but received {}" - .format(step_size_down)) + "'step_size_down' must be a positive integer, but received {}".format( + step_size_down + ) + ) # check type of exp_gamma if not isinstance(exp_gamma, float): raise TypeError( "The type of 'exp_gamma' must be float, but received {}".format( - type(exp_gamma))) + type(exp_gamma) + ) + ) step_size_up = float(step_size_up) - step_size_down = float( - step_size_down) if step_size_down is not None else step_size_up + step_size_down = ( + float(step_size_down) + if step_size_down is not None + else step_size_up + ) self.cycle_size = step_size_up + step_size_down self.step_up_pct = step_size_up / self.cycle_size self.max_lr = float(max_learning_rate) self.amplitude = self.max_lr - base_learning_rate - if mode not in ['triangular', 'triangular2', 'exp_range' - ] and scale_fn is None: + if ( + mode not in ['triangular', 'triangular2', 'exp_range'] + and scale_fn is None + ): raise ValueError( "'mode' is invalid and 'scale_fn' is not specified, make sure one of 'mode' or 'scale_fn' is valid" ) if scale_mode not in ['cycle', 'iterations']: raise ValueError( - "'scale_mode' must be one of 'cycle' or 'iterations") + "'scale_mode' must be one of 'cycle' or 'iterations" + ) self.mode = mode self.gamma = exp_gamma # only for exp_range mode @@ -1981,10 +2093,10 @@ class CyclicLR(LRScheduler): super().__init__(base_learning_rate, last_epoch, verbose) def _triangular_scale_fn(self, x): - return 1. + return 1.0 def _triangular2_scale_fn(self, x): - return 1 / (2.**(x - 1)) + return 1 / (2.0 ** (x - 1)) def _exp_range_scale_fn(self, x): return self.gamma**x @@ -1993,7 +2105,7 @@ class CyclicLR(LRScheduler): iterations = self.last_epoch cycle = 1 + iterations // self.cycle_size - pct_per_cycle = 1. + iterations / self.cycle_size - cycle + pct_per_cycle = 1.0 + iterations / self.cycle_size - cycle if pct_per_cycle <= self.step_up_pct: scale_factor = pct_per_cycle / self.step_up_pct diff --git a/python/paddle/optimizer/momentum.py b/python/paddle/optimizer/momentum.py index 5ec696f691c..8cca050625a 100644 --- a/python/paddle/optimizer/momentum.py +++ b/python/paddle/optimizer/momentum.py @@ -121,29 +121,35 @@ class Momentum(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate=0.001, - momentum=0.9, - parameters=None, - use_nesterov=False, - weight_decay=None, - grad_clip=None, - multi_precision=False, - rescale_grad=1.0, - use_multi_tensor=False, - name=None): + def __init__( + self, + learning_rate=0.001, + momentum=0.9, + parameters=None, + use_nesterov=False, + weight_decay=None, + grad_clip=None, + multi_precision=False, + rescale_grad=1.0, + use_multi_tensor=False, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set") if momentum is None: raise ValueError("momentum is not set") - predicate = lambda regular: isinstance(regular, - (L2DecayRegularizer, float)) + predicate = lambda regular: isinstance( + regular, (L2DecayRegularizer, float) + ) if isinstance(parameters, list): if isinstance(parameters[0], dict): for param_group in parameters: - decay = param_group[ - 'weight_decay'] if 'weight_decay' in param_group else weight_decay + decay = ( + param_group['weight_decay'] + if 'weight_decay' in param_group + else weight_decay + ) reg_method, reg_coeff = self._update_regularization(decay) param_group['regularization_method'] = reg_method param_group['regularization_coeff'] = reg_coeff @@ -151,16 +157,20 @@ class Momentum(Optimizer): param_group['weight_decay'] = py_regular py_regular = None if predicate(weight_decay) else weight_decay - super(Momentum, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=py_regular, - grad_clip=grad_clip, - name=name) + super(Momentum, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=py_regular, + grad_clip=grad_clip, + name=name, + ) self.type = "momentum" self._momentum = momentum self._use_nesterov = bool(use_nesterov) - self._regularization_method, self._regularization_coeff = self._update_regularization( - weight_decay) + ( + self._regularization_method, + self._regularization_coeff, + ) = self._update_regularization(weight_decay) self._multi_precision = multi_precision self._rescale_grad = rescale_grad self._master_weights = {} @@ -178,25 +188,25 @@ class Momentum(Optimizer): self._velocity_dict = {'FP32_LODTensor': [], 'FP16_LODTensor': []} self._master_weight_dict = { 'FP32_LODTensor': None, - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } self._regularization_method_dict = { 'FP32_LODTensor': [], - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } self._regularization_coeff_dict = { 'FP32_LODTensor': [], - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } def _update_regularization(self, weight_decay): reg_method = "" reg_coeff = 0.0 - if (isinstance(weight_decay, L2DecayRegularizer)): + if isinstance(weight_decay, L2DecayRegularizer): reg_method = "l2_decay" reg_coeff = weight_decay._regularization_coeff - if (isinstance(weight_decay, float)): + if isinstance(weight_decay, float): reg_method = "l2_decay" reg_coeff = weight_decay return reg_method, reg_coeff @@ -209,19 +219,23 @@ class Momentum(Optimizer): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -237,15 +251,22 @@ class Momentum(Optimizer): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _create_accumulators(self, block, parameters): @@ -263,7 +284,10 @@ class Momentum(Optimizer): master_p = self._create_master_weight(p) self._add_accumulator(self._velocity_acc_str, master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Momentum optimizer." @@ -271,25 +295,28 @@ class Momentum(Optimizer): self._add_accumulator(self._velocity_acc_str, p) def _create_regularization_of_grad(self, param, grad, regularization=None): - """ Create and add backward regularization Operators + """Create and add backward regularization Operators Function helper of append_regularization_ops. """ # If ParamAttr is set to L2Decay, we skip doing regularization here. And then we fused # L2Decay with momentum which can refer to _append_optimize_op below. - if hasattr(param, 'regularizer') and isinstance(param.regularizer, - L2DecayRegularizer): + if hasattr(param, 'regularizer') and isinstance( + param.regularizer, L2DecayRegularizer + ): return grad return super(Momentum, self)._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - velocity_acc = self._get_accumulator(self._velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._velocity_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) # For fusion of momentum and l2decay @@ -306,30 +333,56 @@ class Momentum(Optimizer): regularization_method = "" regularization_coeff = 0.0 - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) if _in_legacy_dygraph(): if isinstance(param_and_grad, dict): self._update_regularization(param_and_grad['weight_decay']) _, _, _ = _legacy_C_ops.momentum( - param_and_grad[0], param_and_grad[1], velocity_acc, lr, - master_weight, param_and_grad[0], velocity_acc, master_weight, - 'mu', self._momentum, 'use_nesterov', self._use_nesterov, - 'regularization_method', regularization_method, - 'regularization_coeff', regularization_coeff, 'multi_precision', - find_master) + param_and_grad[0], + param_and_grad[1], + velocity_acc, + lr, + master_weight, + param_and_grad[0], + velocity_acc, + master_weight, + 'mu', + self._momentum, + 'use_nesterov', + self._use_nesterov, + 'regularization_method', + regularization_method, + 'regularization_coeff', + regularization_coeff, + 'multi_precision', + find_master, + ) return None if in_dygraph_mode(): if isinstance(param_and_grad, dict): self._update_regularization(param_and_grad['weight_decay']) - return _C_ops.momentum_(param_and_grad[0], param_and_grad[1], - velocity_acc, lr, master_weight, - self._momentum, self._use_nesterov, - regularization_method, regularization_coeff, - find_master, self._rescale_grad) + return _C_ops.momentum_( + param_and_grad[0], + param_and_grad[1], + velocity_acc, + lr, + master_weight, + self._momentum, + self._use_nesterov, + regularization_method, + regularization_coeff, + find_master, + self._rescale_grad, + ) attrs = { "mu": self._momentum, @@ -337,19 +390,19 @@ class Momentum(Optimizer): "regularization_method": regularization_method, "regularization_coeff": regularization_coeff, "multi_precision": find_master, - "rescale_grad": self._rescale_grad + "rescale_grad": self._rescale_grad, } inputs = { "Param": [param_and_grad[0]], "Grad": [param_and_grad[1]], "Velocity": [velocity_acc], - "LearningRate": [lr] + "LearningRate": [lr], } outputs = { "ParamOut": [param_and_grad[0]], - "VelocityOut": [velocity_acc] + "VelocityOut": [velocity_acc], } if find_master: @@ -357,11 +410,13 @@ class Momentum(Optimizer): outputs["MasterParamOut"] = master_weight # create the momentum optimize op - momentum_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + momentum_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return momentum_op @@ -383,7 +438,9 @@ class Momentum(Optimizer): # we skip param's l2decay before, so fuse it with momentum here. if isinstance(param.regularizer, L2DecayRegularizer): regularization_method = "l2_decay" - regularization_coeff = param.regularizer._regularization_coeff + regularization_coeff = ( + param.regularizer._regularization_coeff + ) elif param.regularizer is not None: regularization_method = "" regularization_coeff = 0.0 @@ -392,28 +449,34 @@ class Momentum(Optimizer): self._velocity_dict['FP32_LODTensor'].append(velocity_acc) # fp32 no master weight self._regularization_method_dict['FP32_LODTensor'].append( - regularization_method) + regularization_method + ) self._regularization_coeff_dict['FP32_LODTensor'].append( - regularization_coeff) + regularization_coeff + ) elif param.dtype == paddle.float16: self._param_dict['FP16_LODTensor'].append(param) self._velocity_dict['FP16_LODTensor'].append(velocity_acc) if self._multi_precision: self._master_weight_dict['FP16_LODTensor'].append( - self._master_weights[param.name]) + self._master_weights[param.name] + ) else: self._master_weight_dict['FP16_LODTensor'] = None self._regularization_method_dict['FP16_LODTensor'].append( - regularization_method) + regularization_method + ) self._regularization_coeff_dict['FP16_LODTensor'].append( - regularization_coeff) + regularization_coeff + ) else: raise ValueError( "Now multi_tensor_momentum only support fp32 and fp16 parameters and grad is LOD_TENSOR." ) - def _append_optimize_multi_tensor_op(self, target_block, - parameters_and_grads): + def _append_optimize_multi_tensor_op( + self, target_block, parameters_and_grads + ): """ For Multi Tensor, append optimize merged_operator to block. """ @@ -427,15 +490,19 @@ class Momentum(Optimizer): if param_and_grad[1] is None: continue if param_and_grad[0].stop_gradient is False: - if param_and_grad[ - 0].dtype == paddle.float32 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + if ( + param_and_grad[0].dtype == paddle.float32 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP32_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP32_LODTensor'].append(lr) - elif param_and_grad[ - 0].dtype == paddle.float16 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + elif ( + param_and_grad[0].dtype == paddle.float16 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP16_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP16_LODTensor'].append(lr) @@ -446,21 +513,27 @@ class Momentum(Optimizer): if param_and_grad[0].stop_gradient is False: param_grad_dict = dict() param_grad_dict['params'] = param_and_grad - param_grad_dict.update({ - k: v - for k, v in parameters_and_grads.items() - if k != 'params' - }) + param_grad_dict.update( + { + k: v + for k, v in parameters_and_grads.items() + if k != 'params' + } + ) param_and_grad = self._update_param_group(param_grad_dict) - if param_and_grad[ - 0].dtype == paddle.float32 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + if ( + param_and_grad[0].dtype == paddle.float32 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP32_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP32_LODTensor'].append(lr) - elif param_and_grad[ - 0].dtype == paddle.float16 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + elif ( + param_and_grad[0].dtype == paddle.float16 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP16_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP16_LODTensor'].append(lr) @@ -473,26 +546,39 @@ class Momentum(Optimizer): if framework._non_static_mode(): if in_dygraph_mode(): _, _, _ = _C_ops.merged_momentum_( - self._param_dict[key], grad_dict[key], - self._velocity_dict[key], lr_dict[key], - self._master_weight_dict[key], self._momentum, + self._param_dict[key], + grad_dict[key], + self._velocity_dict[key], + lr_dict[key], + self._master_weight_dict[key], + self._momentum, self._use_nesterov, self._regularization_method_dict[key], - self._regularization_coeff_dict[key], find_master, - self._rescale_grad) + self._regularization_coeff_dict[key], + find_master, + self._rescale_grad, + ) else: _, _, _ = _legacy_C_ops.merged_momentum( - self._param_dict[key], grad_dict[key], - self._velocity_dict[key], lr_dict[key], + self._param_dict[key], + grad_dict[key], + self._velocity_dict[key], + lr_dict[key], self._master_weight_dict[key], - self._param_dict[key], self._velocity_dict[key], - self._master_weight_dict[key], 'mu', self._momentum, - 'use_nesterov', self._use_nesterov, + self._param_dict[key], + self._velocity_dict[key], + self._master_weight_dict[key], + 'mu', + self._momentum, + 'use_nesterov', + self._use_nesterov, 'regularization_method', self._regularization_method_dict[key], 'regularization_coeff', self._regularization_coeff_dict[key], - 'multi_precision', find_master) + 'multi_precision', + find_master, + ) else: inputs = { "Param": self._param_dict[key], @@ -505,38 +591,45 @@ class Momentum(Optimizer): "VelocityOut": self._velocity_dict[key], } attrs = { - "mu": - self._momentum, - "use_nesterov": - self._use_nesterov, - "regularization_method": - self._regularization_method_dict[key], - "regularization_coeff": - self._regularization_coeff_dict[key], + "mu": self._momentum, + "use_nesterov": self._use_nesterov, + "regularization_method": self._regularization_method_dict[ + key + ], + "regularization_coeff": self._regularization_coeff_dict[ + key + ], } if find_master: inputs["MasterParam"] = self._master_weight_dict[key] outputs["MasterParamOut"] = self._master_weight_dict[ - key] + key + ] attrs["multi_precision"] = find_master - target_block.append_op(type="merged_momentum", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + target_block.append_op( + type="merged_momentum", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return None def _update_param_group(self, parameters): - self._momentum = parameters.get('momentum', - self._default_dict['momentum']) - self._use_nesterov = parameters.get('use_nesterov', - self._default_dict['use_nesterov']) - self._rescale_grad = parameters.get('rescale_grad', - self._default_dict['rescale_grad']) + self._momentum = parameters.get( + 'momentum', self._default_dict['momentum'] + ) + self._use_nesterov = parameters.get( + 'use_nesterov', self._default_dict['use_nesterov'] + ) + self._rescale_grad = parameters.get( + 'rescale_grad', self._default_dict['rescale_grad'] + ) self._regularization_method = parameters.get( - 'regularization_method', - self._default_dict['regularization_method']) + 'regularization_method', self._default_dict['regularization_method'] + ) self._regularization_coeff = parameters.get( - 'regularization_coeff', self._default_dict['regularization_coeff']) + 'regularization_coeff', self._default_dict['regularization_coeff'] + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index b90bd5ae3ff..54ac0db5536 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -17,13 +17,22 @@ import logging from collections import defaultdict import paddle -from paddle.fluid.framework import Variable, default_main_program, device_guard, name_scope +from paddle.fluid.framework import ( + Variable, + default_main_program, + device_guard, + name_scope, +) from ..fluid import framework from ..fluid import layers from ..fluid import unique_name from ..fluid.backward import _get_no_grad_set_name, append_backward -from ..fluid.clip import GradientClipBase, append_gradient_clip_ops, error_clip_callback +from ..fluid.clip import ( + GradientClipBase, + append_gradient_clip_ops, + error_clip_callback, +) from ..fluid.framework import program_guard, Parameter from ..fluid.initializer import Constant from ..fluid.layer_helper import LayerHelper @@ -31,24 +40,36 @@ from ..fluid.dygraph import base as imperative_base from paddle.fluid import core from .lr import LRScheduler from paddle import _C_ops, _legacy_C_ops -from paddle.fluid.framework import _in_legacy_dygraph, _in_eager_without_dygraph_check, _current_expected_place, in_dygraph_mode +from paddle.fluid.framework import ( + _in_legacy_dygraph, + _in_eager_without_dygraph_check, + _current_expected_place, + in_dygraph_mode, +) __all__ = [] @framework.static_only -def append_backward_new(loss_list, - parameter_list=None, - no_grad_set=None, - callbacks=None, - checkpoints=None, - distop_context=None): +def append_backward_new( + loss_list, + parameter_list=None, + no_grad_set=None, + callbacks=None, + checkpoints=None, + distop_context=None, +): from paddle.incubate.autograd.primx import orig2prim, Transform + program = default_main_program() - assert program.num_blocks == 1, "The append_backward_new interface is designed to process only one block." + assert ( + program.num_blocks == 1 + ), "The append_backward_new interface is designed to process only one block." block = program.current_block() for el in loss_list: - assert el.block == block, 'variable in loss_list should be in current block of main program' + assert ( + el.block == block + ), 'variable in loss_list should be in current block of main program' orig2prim(block) ad = Transform(block) @@ -152,12 +173,14 @@ class Optimizer(object): """ @imperative_base.no_grad - def __init__(self, - learning_rate, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + ): if parameters is not None: # paddle.Tensor is also iterable, so here we don't check whether @@ -166,13 +189,16 @@ class Optimizer(object): if isinstance(parameters, (paddle.Tensor, core.eager.Tensor)): raise TypeError( "`parameters` argument given to the optimizer should be " - "an iterable of paddle Tensors, but got argument type is `{}`." - .format(type(parameters))) + "an iterable of paddle Tensors, but got argument type is `{}`.".format( + type(parameters) + ) + ) if isinstance(parameters, dict): raise TypeError( "`parameters` argument should not get dict type, " "if parameter groups is needed, please set `parameters`" - " as list of dict") + " as list of dict" + ) self._parameter_list = list(parameters) else: self._parameter_list = None @@ -186,18 +212,22 @@ class Optimizer(object): if weight_decay is not None: if not isinstance(self._parameter_list[0], dict): for param in self._parameter_list: - if hasattr(param, 'regularizer' - ) and param.regularizer is not None: + if ( + hasattr(param, 'regularizer') + and param.regularizer is not None + ): logging.info( "If regularizer of a Parameter has been set by 'paddle.ParamAttr' or 'static.WeightNormParamAttr' already. " "The weight_decay[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % weight_decay.__str__()) + % weight_decay.__str__() + ) break if not isinstance(learning_rate, (float, LRScheduler)): raise TypeError( - "learning rate should be float or LRScheduler, got %s here" % - type(learning_rate)) + "learning rate should be float or LRScheduler, got %s here" + % type(learning_rate) + ) if grad_clip is not None: if not isinstance(grad_clip, GradientClipBase): raise TypeError( @@ -205,6 +235,7 @@ class Optimizer(object): ) if isinstance(weight_decay, float): from ..fluid.regularizer import L2Decay + self.regularization = L2Decay(weight_decay) else: self.regularization = weight_decay @@ -216,8 +247,9 @@ class Optimizer(object): if self._parameter_list: if isinstance(self._parameter_list[0], dict): for param_group in self._parameter_list: - assert 'params' in param_group, \ - 'params should be set in parameters if parameter groups are optimized in different options' + assert ( + 'params' in param_group + ), 'params should be set in parameters if parameter groups are optimized in different options' self._dtype = self._parameter_list[0]['params'][0].dtype else: self._dtype = self._parameter_list[0].dtype @@ -237,7 +269,7 @@ class Optimizer(object): self.clear_gradients = self.clear_grad self._default_dict = { 'weight_decay': self.regularization, - 'grad_clip': self._grad_clip + 'grad_clip': self._grad_clip, } self._param_groups = [] @@ -342,8 +374,9 @@ class Optimizer(object): self._accumulators_holder = state_dict for k, v in self._accumulators.items(): for para_name, var_tmp in v.items(): - assert var_tmp.name in state_dict, \ - "optimizer Tensor {} not found".format( var_tmp.name ) + assert ( + var_tmp.name in state_dict + ), "optimizer Tensor {} not found".format(var_tmp.name) var = var_tmp.value() tensor = var.get_tensor() model_np = np.array(tensor) @@ -357,16 +390,23 @@ class Optimizer(object): elif isinstance(load_para, np.ndarray): load_para_np = load_para else: - raise RuntimeError("State dict type {} not supprt".format( - str(type(load_para)))) - - assert model_np.shape == load_para_np.shape, \ - "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( - model_np.name, model_np.shape, load_para_np.shape) + raise RuntimeError( + "State dict type {} not supprt".format( + str(type(load_para)) + ) + ) + + assert ( + model_np.shape == load_para_np.shape + ), "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( + model_np.name, model_np.shape, load_para_np.shape + ) - assert model_np.dtype == load_para_np.dtype, \ - "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( - model_np.name, model_np.dtype, load_para_np.dtype) + assert ( + model_np.dtype == load_para_np.dtype + ), "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( + model_np.name, model_np.dtype, load_para_np.dtype + ) tensor.set(load_para_np, framework._current_expected_place()) @@ -375,45 +415,57 @@ class Optimizer(object): def _create_global_learning_rate(self): # lr var can't be float16, for pure fp16 training, should extra handle the dtype for lr - _lr_dtype = paddle.get_default_dtype( - ) if self._dtype is None else self._dtype - _lr_dtype = paddle.float32 if ( - paddle.get_default_dtype() != "float16" - and _lr_dtype == paddle.float16) else _lr_dtype + _lr_dtype = ( + paddle.get_default_dtype() if self._dtype is None else self._dtype + ) + _lr_dtype = ( + paddle.float32 + if ( + paddle.get_default_dtype() != "float16" + and _lr_dtype == paddle.float16 + ) + else _lr_dtype + ) if isinstance(self._learning_rate, LRScheduler): lr_var = self._global_learning_rate() # only create global lr_var once if not isinstance(lr_var, framework.Variable): lr_name = unique_name.generate('learning_rate') self._learning_rate._var_name = lr_name - lr_var = self.helper.create_global_variable(name=lr_name, - shape=[1], - persistable=True, - stop_gradient=True, - dtype=_lr_dtype) + lr_var = self.helper.create_global_variable( + name=lr_name, + shape=[1], + persistable=True, + stop_gradient=True, + dtype=_lr_dtype, + ) main_prog = framework.default_main_program() main_prog.lr_sheduler = self._learning_rate main_prog.lr_var = lr_var self._learning_rate_map[ - framework.default_main_program()] = lr_var + framework.default_main_program() + ] = lr_var lr_value = float(self._learning_rate()) self.helper.set_variable_initializer( - lr_var, initializer=Constant(value=lr_value)) + lr_var, initializer=Constant(value=lr_value) + ) elif isinstance(self._learning_rate, float): # only create global lr_var once lr = self._global_learning_rate() if isinstance(lr, framework.Variable): return else: - self._learning_rate_map[framework.default_main_program( - )] = layers.create_global_var( + self._learning_rate_map[ + framework.default_main_program() + ] = layers.create_global_var( name=unique_name.generate("learning_rate"), shape=[1], value=float(self._learning_rate), dtype=_lr_dtype, - persistable=True) + persistable=True, + ) @framework.dygraph_only def set_lr(self, value): @@ -454,7 +506,8 @@ class Optimizer(object): if not isinstance(value, (int, float)): raise TypeError( "The type of 'value' in optimizer.set_lr must be float, but received %s." - % (type(value))) + % (type(value)) + ) if isinstance(self._learning_rate, LRScheduler): raise RuntimeError( "optimizer's learning rate can't be LRScheduler when invoke this API, because this will lead to conflict." @@ -464,23 +517,36 @@ class Optimizer(object): if current_lr is not None: if in_dygraph_mode(): place = _current_expected_place() - _C_ops.full_(current_lr, list(current_lr.shape), float(value), - current_lr.dtype, place) + _C_ops.full_( + current_lr, + list(current_lr.shape), + float(value), + current_lr.dtype, + place, + ) elif _in_legacy_dygraph(): - _legacy_C_ops.fill_constant(current_lr, 'value', float(value), - 'dtype', current_lr.dtype, 'shape', - list(current_lr.shape)) + _legacy_C_ops.fill_constant( + current_lr, + 'value', + float(value), + 'dtype', + current_lr.dtype, + 'shape', + list(current_lr.shape), + ) else: global_block = framework.default_main_program().global_block() - global_block.append_op(type='fill_constant', - outputs={'Out': [current_lr]}, - attrs={ - 'dtype': current_lr.dtype, - 'shape': list(current_lr.shape), - 'value': float(value) - }, - stop_gradient=True) + global_block.append_op( + type='fill_constant', + outputs={'Out': [current_lr]}, + attrs={ + 'dtype': current_lr.dtype, + 'shape': list(current_lr.shape), + 'value': float(value), + }, + stop_gradient=True, + ) def get_lr(self): """ @@ -554,8 +620,7 @@ class Optimizer(object): return self._learning_rate_map.get(program, None) def _append_optimize_op(self, block, param_and_grad): - """ append optimize operator to block and return all the added optimize_op - """ + """append optimize operator to block and return all the added optimize_op""" raise NotImplementedError( "Class \"Optimizer\" connot be used directly as an optimizer, please use its subclasses such as \"Adam\"" ) @@ -572,8 +637,8 @@ class Optimizer(object): return self._global_learning_rate() else: with default_main_program()._lr_schedule_guard( - is_with_opt=True), framework.name_scope( - 'scale_with_param_lr'): + is_with_opt=True + ), framework.name_scope('scale_with_param_lr'): return self._global_learning_rate() * param_lr else: return self._global_learning_rate() @@ -600,14 +665,16 @@ class Optimizer(object): """ pass - def _add_accumulator(self, - name, - param, - dtype=None, - fill_value=0.0, - shape=None, - type=None, - device=None): + def _add_accumulator( + self, + name, + param, + dtype=None, + fill_value=0.0, + shape=None, + type=None, + device=None, + ): """Utility function to add an accumulator for a parameter Args: @@ -619,13 +686,17 @@ class Optimizer(object): """ if self._name is not None: name = self._name + "_" + name - if (name in self._accumulators - and param.name in self._accumulators[name]): + if ( + name in self._accumulators + and param.name in self._accumulators[name] + ): if framework._non_static_mode(): return self._accumulators[name][param.name] raise Exception( "Accumulator {} already exists for parameter {}".format( - name, param.name)) + name, param.name + ) + ) if shape == None: shape = param.shape assert isinstance(self.helper, LayerHelper) @@ -639,20 +710,25 @@ class Optimizer(object): persistable=True, dtype=dtype or param.dtype, type=core.VarDesc.VarType.LOD_TENSOR - if framework._in_eager_without_dygraph_check() else - (param.type if type is None else type), + if framework._in_eager_without_dygraph_check() + else (param.type if type is None else type), shape=shape, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) if device is None: device = self._get_device_for_param(param.name) with device_guard(device): self.helper.set_variable_initializer( - var, initializer=Constant(value=float(fill_value))) + var, initializer=Constant(value=float(fill_value)) + ) if framework._non_static_mode(): if len(self._accumulators_holder) > 0: - assert var_name in self._accumulators_holder, \ - "Optimizer set error, {} should in state dict".format( var_name ) + assert ( + var_name in self._accumulators_holder + ), "Optimizer set error, {} should in state dict".format( + var_name + ) var.set_value(self._accumulators_holder[var_name]) self._accumulators[name][param.name] = var @@ -670,11 +746,15 @@ class Optimizer(object): """ if self._name is not None: name = self._name + "_" + name - if (name not in self._accumulators - or param.name not in self._accumulators[name]): + if ( + name not in self._accumulators + or param.name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, param.name)) + name, param.name + ) + ) return self._accumulators[name][param.name] def _update_param_device_map(self, parameters_and_grads, target_block): @@ -682,13 +762,15 @@ class Optimizer(object): if param_and_grad[0].stop_gradient is False: param_name = param_and_grad[0].name ops = target_block.ops - device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) for op in ops: input_arg_names = op.input_arg_names if param_name in input_arg_names: self._param_device_map[param_name] = op.attr( - device_attr_name) + device_attr_name + ) break def _get_device_for_param(self, param_name): @@ -725,10 +807,12 @@ class Optimizer(object): target_block = global_block current_block = framework.default_main_program().current_block() if current_block.idx != global_block.idx: - assert current_block.backward_block_idx != -1, \ - "current block is not global_block, but it doesn't have backward block." + assert ( + current_block.backward_block_idx != -1 + ), "current block is not global_block, but it doesn't have backward block." target_block = framework.default_main_program().blocks[ - current_block.backward_block_idx] + current_block.backward_block_idx + ] start = len(target_block.ops) self.helper = LayerHelper(self.__class__.__name__) @@ -737,57 +821,83 @@ class Optimizer(object): # NOTE: Multi Tensor support [ Momentum, Adam ] for dygraph mode if self._use_multi_tensor and self.__class__.__name__ in [ - 'Momentum', 'Adam' + 'Momentum', + 'Adam', ]: - if len(self._param_dict['FP32_LODTensor']) == 0 and len( - self._param_dict['FP16_LODTensor']) == 0: + if ( + len(self._param_dict['FP32_LODTensor']) == 0 + and len(self._param_dict['FP16_LODTensor']) == 0 + ): if isinstance(parameters_and_grads, list): - self._multi_tensor_init(target_block, [ - p[0] - for p in parameters_and_grads if not p[0].stop_gradient - ]) + self._multi_tensor_init( + target_block, + [ + p[0] + for p in parameters_and_grads + if not p[0].stop_gradient + ], + ) else: self._update_param_group(parameters_and_grads) - self._multi_tensor_init(target_block, [ - p[0] for p in parameters_and_grads['params'] - if not p[0].stop_gradient - ]) + self._multi_tensor_init( + target_block, + [ + p[0] + for p in parameters_and_grads['params'] + if not p[0].stop_gradient + ], + ) if framework._non_static_mode(): - self._append_optimize_multi_tensor_op(target_block, - parameters_and_grads) + self._append_optimize_multi_tensor_op( + target_block, parameters_and_grads + ) else: - self._update_param_device_map(parameters_and_grads, - target_block) + self._update_param_device_map( + parameters_and_grads, target_block + ) # NOTE: Multi Tensor requires all parameters to be in the same device and program. # param_grad_list = [p_0,g_0,p_1,g_1,....] param_grad_list = [] for param_and_grad in parameters_and_grads: - if not param_and_grad[0].stop_gradient and param_and_grad[ - 1] is not None: + if ( + not param_and_grad[0].stop_gradient + and param_and_grad[1] is not None + ): param_grad_list.append(param_and_grad[0]) param_grad_list.append(param_and_grad[1]) with param_grad_list[0].block.program._optimized_guard( - param_grad_list), name_scope("optimizer"): + param_grad_list + ), name_scope("optimizer"): device = self._get_device_for_param(param_grad_list[0].name) with device_guard(device): self._append_optimize_multi_tensor_op( - target_block, parameters_and_grads) + target_block, parameters_and_grads + ) else: if not framework._non_static_mode(): - params_grads_device_map = parameters_and_grads[ - 'params'] if isinstance(parameters_and_grads, - dict) else parameters_and_grads - self._update_param_device_map(params_grads_device_map, - target_block) + params_grads_device_map = ( + parameters_and_grads['params'] + if isinstance(parameters_and_grads, dict) + else parameters_and_grads + ) + self._update_param_device_map( + params_grads_device_map, target_block + ) if isinstance(parameters_and_grads, list): - self._create_accumulators(target_block, [ - p[0] for p in parameters_and_grads if not p[0].stop_gradient - ]) + self._create_accumulators( + target_block, + [ + p[0] + for p in parameters_and_grads + if not p[0].stop_gradient + ], + ) else: params_acc_dict = parameters_and_grads.copy() params_acc_dict['params'] = [ - p[0] for p in params_acc_dict['params'] + p[0] + for p in params_acc_dict['params'] if not p[0].stop_gradient ] self._create_accumulators(target_block, params_acc_dict) @@ -798,8 +908,9 @@ class Optimizer(object): if param_and_grad[1] is None: continue if param_and_grad[0].stop_gradient is False: - self._append_optimize_op(target_block, - param_and_grad) + self._append_optimize_op( + target_block, param_and_grad + ) else: for param_and_grad in parameters_and_grads['params']: if param_and_grad[1] is None: @@ -807,25 +918,31 @@ class Optimizer(object): if param_and_grad[0].stop_gradient is False: param_grad_dict = dict() param_grad_dict['params'] = param_and_grad - param_grad_dict.update({ - k: v - for k, v in parameters_and_grads.items() - if k != 'params' - }) - self._append_optimize_op(target_block, - param_grad_dict) + param_grad_dict.update( + { + k: v + for k, v in parameters_and_grads.items() + if k != 'params' + } + ) + self._append_optimize_op( + target_block, param_grad_dict + ) else: for param_and_grad in parameters_and_grads: if param_and_grad[1] is None: continue with param_and_grad[0].block.program._optimized_guard( - param_and_grad), name_scope("optimizer"): + param_and_grad + ), name_scope("optimizer"): if param_and_grad[0].stop_gradient is False: device = self._get_device_for_param( - param_and_grad[0].name) + param_and_grad[0].name + ) with device_guard(device): optimize_op = self._append_optimize_op( - target_block, param_and_grad) + target_block, param_and_grad + ) # Get custom finish ops for subclasses # FIXME: Need to fix this once we figure out how to handle dependencies @@ -837,12 +954,14 @@ class Optimizer(object): def _append_dgc_ops(self, param_and_grad): pass - def backward(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameters=None, + no_grad_set=None, + callbacks=None, + ): """ The first part of ``minimize``, do auto-diff to append backward operations for the current program. @@ -891,8 +1010,7 @@ class Optimizer(object): self._dtype = loss.dtype if framework._non_static_mode(): - parameter_list = parameters if parameters \ - else self._parameter_list + parameter_list = parameters if parameters else self._parameter_list params_grads = [] for param in parameter_list: @@ -906,23 +1024,26 @@ class Optimizer(object): if callbacks is None: callbacks = [error_clip_callback] else: - assert (isinstance(callbacks, list)) + assert isinstance(callbacks, list) program = loss.block.program - assert len(loss.shape) == 1 and loss.shape[0] == 1, \ - "The loss.shape should be (1L,), but the current loss.shape is {}. " \ + assert len(loss.shape) == 1 and loss.shape[0] == 1, ( + "The loss.shape should be (1L,), but the current loss.shape is {}. " "Maybe that you should call paddle.mean to process the current loss.".format( - loss.shape) - parameter_list = parameters if parameters \ - else self._parameter_list + loss.shape + ) + ) + parameter_list = parameters if parameters else self._parameter_list with program_guard(program, startup_program): from paddle.incubate.autograd.utils import prim_enabled + if prim_enabled(): - params_grads = append_backward_new([loss], parameter_list, - act_no_grad_set, - callbacks) + params_grads = append_backward_new( + [loss], parameter_list, act_no_grad_set, callbacks + ) else: - params_grads = append_backward(loss, parameter_list, - act_no_grad_set, callbacks) + params_grads = append_backward( + loss, parameter_list, act_no_grad_set, callbacks + ) # Note: since we can't use all_reduce_op now, # dgc_op should be the last op of one grad. self._append_dgc_ops(params_grads) @@ -967,8 +1088,9 @@ class Optimizer(object): params_grads = append_gradient_clip_ops(params_grads) # Add regularization if any - params_grads = self.append_regularization_ops(params_grads, - self.regularization) + params_grads = self.append_regularization_ops( + params_grads, self.regularization + ) optimize_ops = self._create_optimization_pass(params_grads) return optimize_ops @@ -986,21 +1108,26 @@ class Optimizer(object): list: A list of operators appended to the current program. """ if framework._non_static_mode(): - with program_guard(framework.default_main_program(), - framework.default_startup_program()): + with program_guard( + framework.default_main_program(), + framework.default_startup_program(), + ): if isinstance(params_grads, list): if self._grad_clip is not None: params_grads = self._grad_clip(params_grads) params_grads = self.append_regularization_ops( - params_grads, self.regularization) + params_grads, self.regularization + ) else: grad_clip = params_grads['grad_clip'] if grad_clip is not None: params_grads['params'] = grad_clip( - params_grads['params']) + params_grads['params'] + ) params_grads['params'] = self.append_regularization_ops( - params_grads['params'], self.regularization) + params_grads['params'], self.regularization + ) optimize_ops = self._create_optimization_pass(params_grads) else: program = loss.block.program @@ -1009,15 +1136,18 @@ class Optimizer(object): return optimize_ops def _create_regularization_of_grad(self, param, grad, regularization=None): - """ Create and add backward regularization Operators + """Create and add backward regularization Operators Function helper of append_regularization_ops. """ # If no gradient or no regularization is specified, then we don't need to do anything if grad is None or ( - (not hasattr(param, 'regularizer') or - (hasattr(param, 'regularizer') and param.regularizer is None)) - and regularization is None): + ( + not hasattr(param, 'regularizer') + or (hasattr(param, 'regularizer') and param.regularizer is None) + ) + and regularization is None + ): return grad regularization_term = None if hasattr(param, 'regularizer') and param.regularizer is not None: @@ -1044,7 +1174,8 @@ class Optimizer(object): dtype=param.dtype, shape=param.shape, lod_level=param.lod_level, - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) inputs = {"X": [grad, regularization_term]} outputs = {"Out": [new_grad]} @@ -1052,9 +1183,9 @@ class Optimizer(object): return new_grad - def append_regularization_ops(self, - parameters_and_grads, - regularization=None): + def append_regularization_ops( + self, parameters_and_grads, regularization=None + ): r"""Create and add backward regularization Operators Creates and adds backward regularization operators in the BlockDesc. @@ -1079,21 +1210,28 @@ class Optimizer(object): if framework._non_static_mode(): for param, grad in parameters_and_grads: new_grad = self._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) params_and_grads.append((param, new_grad)) else: repeate_regularizer = False with framework.name_scope('regularization'): for param, grad in parameters_and_grads: - if not repeate_regularizer and param.regularizer is not None and regularization is not None: + if ( + not repeate_regularizer + and param.regularizer is not None + and regularization is not None + ): repeate_regularizer = True logging.info( "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % regularization.__str__()) + % regularization.__str__() + ) with param.block.program._optimized_guard([param, grad]): new_grad = self._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) params_and_grads.append((param, new_grad)) return params_and_grads @@ -1101,7 +1239,8 @@ class Optimizer(object): no_grad_set = _get_no_grad_set_name(no_grad_set) parameters = loss.block.program.global_block().all_parameters() param_no_trainable = set( - [param.name for param in parameters if param.stop_gradient is True]) + [param.name for param in parameters if param.stop_gradient is True] + ) # If the parameter is no trainable, it should not have a gradient. no_grad_set.update(param_no_trainable) @@ -1142,7 +1281,8 @@ class Optimizer(object): """ param_list = [] if self._parameter_list is None or not isinstance( - self._parameter_list[0], dict): + self._parameter_list[0], dict + ): for p in self._parameter_list: if not p.stop_gradient: param_list.append(p) @@ -1159,11 +1299,9 @@ class Optimizer(object): core.clear_gradients(param_list, set_to_zero) @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): """ Add operations to minimize ``loss`` by updating ``parameters``. @@ -1208,17 +1346,18 @@ class Optimizer(object): """ assert isinstance(loss, Variable), "The loss should be an Tensor." - parameter_list = parameters if parameters \ - else self._parameter_list + parameter_list = parameters if parameters else self._parameter_list - params_grads = self.backward(loss, - startup_program=startup_program, - parameters=parameter_list, - no_grad_set=no_grad_set) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameters=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self._apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + optimize_ops = self._apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads @@ -1258,9 +1397,9 @@ class Optimizer(object): grad_var = param._grad_ivar() params_grads.append((param, grad_var)) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) else: # optimize parameters in groups @@ -1273,11 +1412,11 @@ class Optimizer(object): grad_var = param._grad_ivar() params_grads['params'].append((param, grad_var)) params_grads.update( - {k: v - for k, v in param_group.items() if k != 'params'}) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + {k: v for k, v in param_group.items() if k != 'params'} + ) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) def _add_param_group(self, param_group): """ @@ -1293,7 +1432,8 @@ class Optimizer(object): elif isinstance(params, set): raise TypeError( "optimizer parameters should be in ordered collections," - "but received set, please use list instead.") + "but received set, please use list instead." + ) else: param_group['params'] = list(params) @@ -1307,18 +1447,21 @@ class Optimizer(object): if not param_set.isdisjoint(set(param_group['params'])): raise ValueError( - "some parameters appear in more than one parameter group") + "some parameters appear in more than one parameter group" + ) for param in param_group['params']: weight_decay = param_group['weight_decay'] if isinstance(weight_decay, float): from ..fluid.regularizer import L2Decay + regularization = L2Decay(weight_decay) else: regularization = weight_decay param.regularizer = regularization param.optimize_attr['learning_rate'] = param_group.get( - 'learning_rate', 1.) + 'learning_rate', 1.0 + ) self._param_groups.append(param_group) @@ -1344,8 +1487,9 @@ class Optimizer(object): pass @framework.dygraph_only - def _append_optimize_multi_tensor_op(self, target_block, - parameters_and_grads): + def _append_optimize_multi_tensor_op( + self, target_block, parameters_and_grads + ): """ For Multi Tensor, append optimize merged_operator to block. """ diff --git a/python/paddle/optimizer/rmsprop.py b/python/paddle/optimizer/rmsprop.py index 5ae5e1468be..06d7c58fb39 100644 --- a/python/paddle/optimizer/rmsprop.py +++ b/python/paddle/optimizer/rmsprop.py @@ -141,16 +141,18 @@ class RMSProp(Optimizer): _mean_square_acc_str = "mean_square" _mean_grad_acc_str = "mean_grad" - def __init__(self, - learning_rate, - rho=0.95, - epsilon=1.0e-6, - momentum=0.0, - centered=False, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + rho=0.95, + epsilon=1.0e-6, + momentum=0.0, + centered=False, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set.") if rho is None: @@ -166,11 +168,13 @@ class RMSProp(Optimizer): if not 0.0 <= rho: raise ValueError("Invalid value of rho, expect rho >= 0.") - super(RMSProp, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(RMSProp, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "rmsprop" self._rho = rho @@ -203,49 +207,50 @@ class RMSProp(Optimizer): if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - momentum_acc = self._get_accumulator(self._momentum_acc_str, - param_and_grad[0]) - mean_square_acc = self._get_accumulator(self._mean_square_acc_str, - param_and_grad[0]) - mean_grad_acc = self._get_accumulator(self._mean_grad_acc_str, - param_and_grad[0]) - rmsprop_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "Moment": - momentum_acc, - "MeanSquare": - mean_square_acc, - "MeanGrad": - mean_grad_acc, - "LearningRate": - self._create_param_lr(param_and_grad), - }, - outputs={ - "ParamOut": param_and_grad[0], - "MomentOut": momentum_acc, - "MeanSquareOut": mean_square_acc, - "MeanGradOut": mean_grad_acc - }, - attrs={ - "epsilon": self._epsilon, - "decay": self._rho, - "momentum": self._momentum, - "centered": self._centered - }, - stop_gradient=True) + momentum_acc = self._get_accumulator( + self._momentum_acc_str, param_and_grad[0] + ) + mean_square_acc = self._get_accumulator( + self._mean_square_acc_str, param_and_grad[0] + ) + mean_grad_acc = self._get_accumulator( + self._mean_grad_acc_str, param_and_grad[0] + ) + rmsprop_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "Moment": momentum_acc, + "MeanSquare": mean_square_acc, + "MeanGrad": mean_grad_acc, + "LearningRate": self._create_param_lr(param_and_grad), + }, + outputs={ + "ParamOut": param_and_grad[0], + "MomentOut": momentum_acc, + "MeanSquareOut": mean_square_acc, + "MeanGradOut": mean_grad_acc, + }, + attrs={ + "epsilon": self._epsilon, + "decay": self._rho, + "momentum": self._momentum, + "centered": self._centered, + }, + stop_gradient=True, + ) return rmsprop_op def _update_param_group(self, parameters): self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) self._rho = parameters.get('rho', self._default_dict['rho']) - self._momentum = parameters.get('momentum', - self._default_dict['momentum']) - self._centered = parameters.get('centered', - self._default_dict['centered']) + self._momentum = parameters.get( + 'momentum', self._default_dict['momentum'] + ) + self._centered = parameters.get( + 'centered', self._default_dict['centered'] + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/sgd.py b/python/paddle/optimizer/sgd.py index 40db1d361eb..b8a9331b422 100644 --- a/python/paddle/optimizer/sgd.py +++ b/python/paddle/optimizer/sgd.py @@ -72,20 +72,24 @@ class SGD(Optimizer): """ - def __init__(self, - learning_rate=0.001, - parameters=None, - weight_decay=None, - grad_clip=None, - multi_precision=False, - name=None): + def __init__( + self, + learning_rate=0.001, + parameters=None, + weight_decay=None, + grad_clip=None, + multi_precision=False, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set") - super(SGD, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(SGD, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "sgd" self._multi_precision = multi_precision self._master_weights = {} @@ -98,19 +102,23 @@ class SGD(Optimizer): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -124,7 +132,10 @@ class SGD(Optimizer): if self._multi_precision and p.dtype == core.VarDesc.VarType.FP16: master_p = self._create_master_weight(p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Adam optimizer." @@ -135,19 +146,35 @@ class SGD(Optimizer): if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) lr = self._create_param_lr(param_and_grad) if in_dygraph_mode(): - _C_ops.sgd_(param_and_grad[0], lr, param_and_grad[1], master_weight, - find_master) + _C_ops.sgd_( + param_and_grad[0], + lr, + param_and_grad[1], + master_weight, + find_master, + ) return None if _in_legacy_dygraph(): - _legacy_C_ops.sgd(param_and_grad[0], lr, param_and_grad[1], - master_weight, param_and_grad[0], master_weight) + _legacy_C_ops.sgd( + param_and_grad[0], + lr, + param_and_grad[1], + master_weight, + param_and_grad[0], + master_weight, + ) return None assert isinstance(block, framework.Block) @@ -155,7 +182,7 @@ class SGD(Optimizer): inputs = { "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": lr + "LearningRate": lr, } outputs = {"ParamOut": param_and_grad[0]} @@ -166,11 +193,13 @@ class SGD(Optimizer): inputs["MasterParam"] = master_weight outputs["MasterParamOut"] = master_weight - sgd_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + sgd_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return sgd_op diff --git a/python/paddle/profiler/__init__.py b/python/paddle/profiler/__init__.py index 76108cf2953..a59f8f14a54 100644 --- a/python/paddle/profiler/__init__.py +++ b/python/paddle/profiler/__init__.py @@ -21,7 +21,14 @@ from .utils import RecordEvent, load_profiler_result from .profiler_statistic import SortedKeys __all__ = [ - 'ProfilerState', 'ProfilerTarget', 'make_scheduler', - 'export_chrome_tracing', 'export_protobuf', 'Profiler', 'RecordEvent', - 'load_profiler_result', 'SortedKeys', 'SummaryView' + 'ProfilerState', + 'ProfilerTarget', + 'make_scheduler', + 'export_chrome_tracing', + 'export_protobuf', + 'Profiler', + 'RecordEvent', + 'load_profiler_result', + 'SortedKeys', + 'SummaryView', ] diff --git a/python/paddle/profiler/profiler.py b/python/paddle/profiler/profiler.py index e4c32db2e9f..bcd87d9c826 100644 --- a/python/paddle/profiler/profiler.py +++ b/python/paddle/profiler/profiler.py @@ -22,11 +22,15 @@ import importlib import json import paddle -from paddle.fluid.core import (_Profiler, ProfilerOptions, TracerEventType, - enable_memory_recorder, - enable_input_shape_recorder, - disable_memory_recorder, - disable_input_shape_recorder) +from paddle.fluid.core import ( + _Profiler, + ProfilerOptions, + TracerEventType, + enable_memory_recorder, + enable_input_shape_recorder, + disable_memory_recorder, + disable_input_shape_recorder, +) from .utils import RecordEvent, wrap_optimizers from .profiler_statistic import StatisticData, _build_table, SortedKeys @@ -105,12 +109,14 @@ class ProfilerTarget(Enum): CUSTOM_DEVICE = 3 -def make_scheduler(*, - closed: int, - ready: int, - record: int, - repeat: int = 0, - skip_first: int = 0) -> Callable: +def make_scheduler( + *, + closed: int, + ready: int, + record: int, + repeat: int = 0, + skip_first: int = 0 +) -> Callable: r""" Return a scheduler function, which scheduler the :ref:`state ` according to the setting. The state transform confirms to: @@ -164,7 +170,9 @@ def make_scheduler(*, step = step - skip_first period_steps = closed + ready + record has_repeated = step // period_steps - if repeat > 0 and has_repeated >= repeat: # the period has repeated repeat times, return CLOSED state + if ( + repeat > 0 and has_repeated >= repeat + ): # the period has repeated repeat times, return CLOSED state return ProfilerState.CLOSED mod_step = step % period_steps if mod_step < closed: @@ -176,12 +184,19 @@ def make_scheduler(*, return ProfilerState.RECORD else: return ProfilerState.RECORD_AND_RETURN - assert closed >= 0 and ready >= 0 and record > 0 and \ - repeat >= 0 and skip_first >= 0, "Invalid profiler scheduler arguments" + + assert ( + closed >= 0 + and ready >= 0 + and record > 0 + and repeat >= 0 + and skip_first >= 0 + ), "Invalid profiler scheduler arguments" if ready == 0: - warn("Profiler will record data after enabling profiler immediately, \ + warn( + "Profiler will record data after enabling profiler immediately, \ some data collected at the beginning of profiling may be 'noisy' because of overhead." - ) + ) return getScheduleState @@ -192,8 +207,9 @@ def _default_state_scheduler(step: int): return ProfilerState.RECORD -def export_chrome_tracing(dir_name: str, - worker_name: Optional[str] = None) -> Callable: +def export_chrome_tracing( + dir_name: str, worker_name: Optional[str] = None +) -> Callable: r""" Return a callable, used for outputing tracing data to chrome tracing format file. The output file will be saved in directory ``dir_name``, and file name will be set as `worker_name`. @@ -226,24 +242,29 @@ def export_chrome_tracing(dir_name: str, os.makedirs(dir_name, exist_ok=True) except Exception: raise RuntimeError( - "Can not create directory '{}' for saving profiling results.". - format(dir_name)) + "Can not create directory '{}' for saving profiling results.".format( + dir_name + ) + ) def handle_fn(prof): nonlocal worker_name if not worker_name: - worker_name = "host_{}pid_{}".format(socket.gethostname(), - str(os.getpid())) + worker_name = "host_{}pid_{}".format( + socket.gethostname(), str(os.getpid()) + ) now = datetime.datetime.now() filename = '{}_time_{}.paddle_trace.json'.format( - worker_name, now.strftime('%Y_%m_%d_%H_%M_%S_%f')) + worker_name, now.strftime('%Y_%m_%d_%H_%M_%S_%f') + ) prof.export(os.path.join(dir_name, filename), "json") return handle_fn -def export_protobuf(dir_name: str, - worker_name: Optional[str] = None) -> Callable: +def export_protobuf( + dir_name: str, worker_name: Optional[str] = None +) -> Callable: r""" Return a callable, used for outputing tracing data to protobuf file. The output file will be saved in directory ``dir_name``, and file name will be set as ``worker_name``. @@ -276,17 +297,21 @@ def export_protobuf(dir_name: str, os.makedirs(dir_name, exist_ok=True) except Exception: raise RuntimeError( - "Can not create directory '{}' for saving profiling results.". - format(dir_name)) + "Can not create directory '{}' for saving profiling results.".format( + dir_name + ) + ) def handle_fn(prof): nonlocal worker_name if not worker_name: - worker_name = "host_{}pid_{}".format(socket.gethostname(), - str(os.getpid())) + worker_name = "host_{}pid_{}".format( + socket.gethostname(), str(os.getpid()) + ) now = datetime.datetime.now() filename = '{}_time_{}.paddle_trace.pb'.format( - worker_name, now.strftime('%Y_%m_%d_%H_%M_%S_%f')) + worker_name, now.strftime('%Y_%m_%d_%H_%M_%S_%f') + ) prof.export(os.path.join(dir_name, filename), "pb") return handle_fn @@ -298,11 +323,15 @@ def _get_supported_targets() -> Iterable[ProfilerTarget]: """ if _Profiler.is_cupti_supported(): return [ - ProfilerTarget.CPU, ProfilerTarget.GPU, ProfilerTarget.CUSTOM_DEVICE + ProfilerTarget.CPU, + ProfilerTarget.GPU, + ProfilerTarget.CUSTOM_DEVICE, ] if _Profiler.is_cnpapi_supported(): return [ - ProfilerTarget.CPU, ProfilerTarget.MLU, ProfilerTarget.CUSTOM_DEVICE + ProfilerTarget.CPU, + ProfilerTarget.MLU, + ProfilerTarget.CUSTOM_DEVICE, ] return [ProfilerTarget.CPU, ProfilerTarget.CUSTOM_DEVICE] @@ -432,36 +461,40 @@ class Profiler: # | ips | 1086.42904 | 1227.30604 | 959.92796 | """ - def __init__(self, - *, - targets: Optional[Iterable[ProfilerTarget]] = None, - scheduler: Union[Callable[[int], ProfilerState], tuple, - None] = None, - on_trace_ready: Optional[Callable[..., Any]] = None, - record_shapes: Optional[bool] = False, - profile_memory=False, - timer_only: Optional[bool] = False, - emit_nvtx: Optional[bool] = False, - custom_device_types: Optional[list] = []): + def __init__( + self, + *, + targets: Optional[Iterable[ProfilerTarget]] = None, + scheduler: Union[Callable[[int], ProfilerState], tuple, None] = None, + on_trace_ready: Optional[Callable[..., Any]] = None, + record_shapes: Optional[bool] = False, + profile_memory=False, + timer_only: Optional[bool] = False, + emit_nvtx: Optional[bool] = False, + custom_device_types: Optional[list] = [] + ): supported_targets = _get_supported_targets() if targets: self.targets = set(targets) for target in targets: if target not in supported_targets: self.targets.remove(target) - warn("Profiling {} is not supported in current context.". - format(target)) + warn( + "Profiling {} is not supported in current context.".format( + target + ) + ) else: self.targets = supported_targets profileoption = ProfilerOptions() if ProfilerTarget.CPU in self.targets: profileoption.trace_switch |= 1 if ProfilerTarget.GPU in self.targets: - profileoption.trace_switch |= (1 << 1) + profileoption.trace_switch |= 1 << 1 if ProfilerTarget.MLU in self.targets: - profileoption.trace_switch |= (1 << 2) + profileoption.trace_switch |= 1 << 2 if ProfilerTarget.CUSTOM_DEVICE in self.targets: - profileoption.trace_switch |= (1 << 3) + profileoption.trace_switch |= 1 << 3 if not custom_device_types: custom_device_types = paddle.device.get_all_custom_device_type() wrap_optimizers() @@ -473,17 +506,19 @@ class Profiler: start_batch, end_batch = scheduler start_batch = max(start_batch, 0) if start_batch >= 1: - self.scheduler = make_scheduler(closed=max(start_batch - 1, 0), - ready=1, - record=(end_batch - - start_batch), - repeat=1) + self.scheduler = make_scheduler( + closed=max(start_batch - 1, 0), + ready=1, + record=(end_batch - start_batch), + repeat=1, + ) else: - self.scheduler = make_scheduler(closed=0, - ready=0, - record=(end_batch - - start_batch), - repeat=1) + self.scheduler = make_scheduler( + closed=0, + ready=0, + record=(end_batch - start_batch), + repeat=1, + ) else: self.scheduler = _default_state_scheduler @@ -549,9 +584,10 @@ class Profiler: elif self.current_state == ProfilerState.RECORD_AND_RETURN: self.profiler.prepare() self.profiler.start() - self.record_event = RecordEvent(name="ProfileStep#{}".format( - self.step_num), - event_type=TracerEventType.ProfileStep) + self.record_event = RecordEvent( + name="ProfileStep#{}".format(self.step_num), + event_type=TracerEventType.ProfileStep, + ) self.record_event.begin() def stop(self): @@ -593,7 +629,10 @@ class Profiler: ) self.profiler.start() self.profiler.stop() - if self.current_state == ProfilerState.RECORD or self.current_state == ProfilerState.RECORD_AND_RETURN: + if ( + self.current_state == ProfilerState.RECORD + or self.current_state == ProfilerState.RECORD_AND_RETURN + ): self.profiler_result = self.profiler.stop() if self.on_trace_ready: self.on_trace_ready(self) @@ -635,9 +674,10 @@ class Profiler: self.step_num += 1 self.current_state = self.scheduler(self.step_num) self._trigger_action() - self.record_event = RecordEvent(name="ProfileStep#{}".format( - self.step_num), - event_type=TracerEventType.ProfileStep) + self.record_event = RecordEvent( + name="ProfileStep#{}".format(self.step_num), + event_type=TracerEventType.ProfileStep, + ) self.record_event.begin() def step_info(self, unit=None): @@ -694,7 +734,9 @@ class Profiler: if self.current_state == ProfilerState.RECORD: # CLOSED -> RECORD self.profiler.prepare() self.profiler.start() - if self.current_state == ProfilerState.RECORD_AND_RETURN: # CLOSED -> RECORD_AND_RETURN + if ( + self.current_state == ProfilerState.RECORD_AND_RETURN + ): # CLOSED -> RECORD_AND_RETURN self.profiler.prepare() self.profiler.start() @@ -707,7 +749,9 @@ class Profiler: self.profiler.stop() if self.current_state == ProfilerState.RECORD: # READY -> RECORD self.profiler.start() - if self.current_state == ProfilerState.RECORD_AND_RETURN: # READY -> RECORD_AND_RETURN + if ( + self.current_state == ProfilerState.RECORD_AND_RETURN + ): # READY -> RECORD_AND_RETURN self.profiler.start() elif self.previous_state == ProfilerState.RECORD: @@ -723,21 +767,31 @@ class Profiler: ) self.profiler.stop() self.profiler.prepare() - if self.current_state == ProfilerState.RECORD_AND_RETURN: # RECORD -> RECORD_AND_RETURN + if ( + self.current_state == ProfilerState.RECORD_AND_RETURN + ): # RECORD -> RECORD_AND_RETURN pass else: assert self.previous_state == ProfilerState.RECORD_AND_RETURN - if self.current_state == ProfilerState.CLOSED: # RECORD_AND_RETURN -> CLOSED + if ( + self.current_state == ProfilerState.CLOSED + ): # RECORD_AND_RETURN -> CLOSED self.profiler_result = self.profiler.stop() - if self.current_state == ProfilerState.READY: # RECORD_AND_RETURN -> READY + if ( + self.current_state == ProfilerState.READY + ): # RECORD_AND_RETURN -> READY self.profiler_result = self.profiler.stop() self.profiler.prepare() - if self.current_state == ProfilerState.RECORD: # RECORD_AND_RETURN -> RECORD + if ( + self.current_state == ProfilerState.RECORD + ): # RECORD_AND_RETURN -> RECORD self.profiler_result = self.profiler.stop() self.profiler.prepare() self.profiler.start() - if self.current_state == ProfilerState.RECORD_AND_RETURN: # RECORD_AND_RETURN -> RECORD_AND_RETURN + if ( + self.current_state == ProfilerState.RECORD_AND_RETURN + ): # RECORD_AND_RETURN -> RECORD_AND_RETURN self.profiler_result = self.profiler.stop() self.profiler.prepare() self.profiler.start() @@ -772,12 +826,14 @@ class Profiler: if self.profiler_result: self.profiler_result.save(path, format) - def summary(self, - sorted_by=SortedKeys.CPUTotal, - op_detail=True, - thread_sep=False, - time_unit='ms', - views=None): + def summary( + self, + sorted_by=SortedKeys.CPUTotal, + op_detail=True, + thread_sep=False, + time_unit='ms', + views=None, + ): r""" Print the Summary table. Currently support overview, model, distributed, operator, memory manipulation and userdefined summary. @@ -811,14 +867,18 @@ class Profiler: if self.profiler_result: statistic_data = StatisticData( self.profiler_result.get_data(), - self.profiler_result.get_extra_info()) + self.profiler_result.get_extra_info(), + ) print( - _build_table(statistic_data, - sorted_by=sorted_by, - op_detail=op_detail, - thread_sep=thread_sep, - time_unit=time_unit, - views=views)) + _build_table( + statistic_data, + sorted_by=sorted_by, + op_detail=op_detail, + thread_sep=thread_sep, + time_unit=time_unit, + views=views, + ) + ) def get_profiler(config_path): @@ -851,17 +911,20 @@ def get_profiler(config_path): method = getattr(module, key) if not use_direct: translated_config_dict['scheduler'] = method( - *value['args'], **value['kwargs']) + *value['args'], **value['kwargs'] + ) else: translated_config_dict['scheduler'] = method else: translated_config_dict['scheduler'] = [ - config_dict['scheduler'][0], config_dict['scheduler'][1] + config_dict['scheduler'][0], + config_dict['scheduler'][1], ] except: print( - 'Set scheduler parameter error, use default parameter instead.') + 'Set scheduler parameter error, use default parameter instead.' + ) translated_config_dict['scheduler'] = None if "on_trace_ready" in config_dict: try: @@ -873,7 +936,8 @@ def get_profiler(config_path): method = getattr(module, key) if not use_direct: translated_config_dict['on_trace_ready'] = method( - *value['args'], **value['kwargs']) + *value['args'], **value['kwargs'] + ) else: translated_config_dict['on_trace_ready'] = method except: diff --git a/python/paddle/profiler/profiler_statistic.py b/python/paddle/profiler/profiler_statistic.py index 20465f1ae85..c383323d51f 100755 --- a/python/paddle/profiler/profiler_statistic.py +++ b/python/paddle/profiler/profiler_statistic.py @@ -17,16 +17,29 @@ import re from paddle.fluid.core import TracerEventType, TracerMemEventType -from .statistic_helper import intersection_ranges, merge_ranges, merge_self_ranges, sum_ranges +from .statistic_helper import ( + intersection_ranges, + merge_ranges, + merge_self_ranges, + sum_ranges, +) _AllTracerEventType = [ - TracerEventType.Operator, TracerEventType.Dataloader, - TracerEventType.ProfileStep, TracerEventType.CudaRuntime, - TracerEventType.Kernel, TracerEventType.Memcpy, TracerEventType.Memset, - TracerEventType.UserDefined, TracerEventType.OperatorInner, - TracerEventType.Forward, TracerEventType.Backward, - TracerEventType.Optimization, TracerEventType.Communication, - TracerEventType.PythonOp, TracerEventType.PythonUserDefined + TracerEventType.Operator, + TracerEventType.Dataloader, + TracerEventType.ProfileStep, + TracerEventType.CudaRuntime, + TracerEventType.Kernel, + TracerEventType.Memcpy, + TracerEventType.Memset, + TracerEventType.UserDefined, + TracerEventType.OperatorInner, + TracerEventType.Forward, + TracerEventType.Backward, + TracerEventType.Optimization, + TracerEventType.Communication, + TracerEventType.PythonOp, + TracerEventType.PythonUserDefined, ] _CommunicationOpName = ['allreduce', 'broadcast', 'rpc'] @@ -90,19 +103,19 @@ class HostStatisticNode: for child in self.children_node: self.gpu_time += child.gpu_time self.general_gpu_time += child.general_gpu_time - self.self_cpu_time -= (child.end_ns - child.start_ns) + self.self_cpu_time -= child.end_ns - child.start_ns for rt in self.runtime_node: - self.self_cpu_time -= (rt.end_ns - rt.start_ns) + self.self_cpu_time -= rt.end_ns - rt.start_ns self.gpu_time += rt.gpu_time self.self_gpu_time += rt.gpu_time self.general_gpu_time += rt.general_gpu_time self.self_general_gpu_time += rt.general_gpu_time for device in self.hostnode.device_node: if device.type == TracerEventType.Kernel: - self.gpu_time += (device.end_ns - device.start_ns) - self.self_gpu_time += (device.end_ns - device.start_ns) - self.general_gpu_time += (device.end_ns - device.start_ns) - self.self_general_gpu_time += (device.end_ns - device.start_ns) + self.gpu_time += device.end_ns - device.start_ns + self.self_gpu_time += device.end_ns - device.start_ns + self.general_gpu_time += device.end_ns - device.start_ns + self.self_general_gpu_time += device.end_ns - device.start_ns @property def end_ns(self): @@ -172,12 +185,14 @@ def wrap_tree(nodetrees): stack.append(childnode) child_statistic_node = HostStatisticNode(childnode) current_statistic_node.children_node.append( - child_statistic_node) + child_statistic_node + ) newstack.append(child_statistic_node) for runtimenode in current_node.runtime_node: runtime_statistic_node = HostStatisticNode(runtimenode) current_statistic_node.runtime_node.append( - runtime_statistic_node) + runtime_statistic_node + ) # recursive calculate node statistic values for thread_id, root_statistic_node in node_statistic_tree.items(): root_statistic_node.cal_statistic() @@ -193,11 +208,12 @@ class TimeRangeSummary: def __init__(self): self.CPUTimeRange = collections.defaultdict(list) self.GPUTimeRange = collections.defaultdict( - lambda: collections.defaultdict( - list)) # GPU events should be divided into different devices + lambda: collections.defaultdict(list) + ) # GPU events should be divided into different devices self.CPUTimeRangeSum = collections.defaultdict(int) self.GPUTimeRangeSum = collections.defaultdict( - lambda: collections.defaultdict(int)) + lambda: collections.defaultdict(int) + ) self.call_times = collections.defaultdict(int) def parse(self, nodetrees): @@ -208,42 +224,50 @@ class TimeRangeSummary: for threadid, hostnodes in thread2hostnodes.items(): CPUTimeRange = collections.defaultdict(list) GPUTimeRange = collections.defaultdict( - lambda: collections.defaultdict(lambda: collections.defaultdict( - list))) # device_id/type/stream_id - for hostnode in hostnodes[1:]: #skip root node + lambda: collections.defaultdict( + lambda: collections.defaultdict(list) + ) + ) # device_id/type/stream_id + for hostnode in hostnodes[1:]: # skip root node CPUTimeRange[hostnode.type].append( - (hostnode.start_ns, hostnode.end_ns)) + (hostnode.start_ns, hostnode.end_ns) + ) self.call_times[hostnode.type] += 1 for runtimenode in hostnode.runtime_node: CPUTimeRange[runtimenode.type].append( - (runtimenode.start_ns, runtimenode.end_ns)) + (runtimenode.start_ns, runtimenode.end_ns) + ) self.call_times[runtimenode.type] += 1 for devicenode in runtimenode.device_node: GPUTimeRange[devicenode.device_id][devicenode.type][ - devicenode.stream_id].append( - (devicenode.start_ns, devicenode.end_ns)) + devicenode.stream_id + ].append((devicenode.start_ns, devicenode.end_ns)) self.call_times[devicenode.type] += 1 for event_type, time_ranges in CPUTimeRange.items(): time_ranges = merge_self_ranges(time_ranges, is_sorted=False) self.CPUTimeRange[event_type] = merge_ranges( - self.CPUTimeRange[event_type], time_ranges, is_sorted=True) + self.CPUTimeRange[event_type], time_ranges, is_sorted=True + ) for device_id, device_time_ranges in GPUTimeRange.items(): for event_type, event_time_ranges in device_time_ranges.items(): for stream_id, time_ranges in event_time_ranges.items(): - time_ranges = merge_self_ranges(time_ranges, - is_sorted=False) + time_ranges = merge_self_ranges( + time_ranges, is_sorted=False + ) self.GPUTimeRange[device_id][event_type] = merge_ranges( self.GPUTimeRange[device_id][event_type], time_ranges, - is_sorted=True) + is_sorted=True, + ) for event_type, time_ranges in self.CPUTimeRange.items(): self.CPUTimeRangeSum[event_type] = sum_ranges(time_ranges) for device_id, device_time_ranges in self.GPUTimeRange.items(): for event_type, time_ranges in device_time_ranges.items(): self.GPUTimeRangeSum[device_id][event_type] = sum_ranges( - time_ranges) + time_ranges + ) def get_gpu_devices(self): return self.GPUTimeRange.keys() @@ -276,57 +300,68 @@ class DistributedSummary: ''' thread2hostnodes = traverse_tree(nodetrees) for threadid, hostnodes in thread2hostnodes.items(): - for hostnode in hostnodes[1:]: #skip root node + for hostnode in hostnodes[1:]: # skip root node # case 1: TracerEventType is Communication if hostnode.type == TracerEventType.Communication: self.cpu_communication_range.append( - (hostnode.start_ns, hostnode.end_ns)) + (hostnode.start_ns, hostnode.end_ns) + ) device_nodes = get_device_nodes(hostnode) for device_node in device_nodes: if device_node.type == TracerEventType.Kernel: self.gpu_communication_range.append( - (device_node.start_ns, device_node.end_ns)) + (device_node.start_ns, device_node.end_ns) + ) - #case 2: TracerEventType is Operator but is communication op - elif hostnode.type == TracerEventType.Operator and any([ + # case 2: TracerEventType is Operator but is communication op + elif hostnode.type == TracerEventType.Operator and any( + [ name in hostnode.name.lower() for name in _CommunicationOpName - ]): + ] + ): self.cpu_communication_range.append( - (hostnode.start_ns, hostnode.end_ns)) + (hostnode.start_ns, hostnode.end_ns) + ) device_nodes = get_device_nodes(hostnode) for device_node in device_nodes: if device_node.type == TracerEventType.Kernel: self.gpu_communication_range.append( - (device_node.start_ns, device_node.end_ns)) + (device_node.start_ns, device_node.end_ns) + ) - #case 3: Others, filter kernels named with nccl + # case 3: Others, filter kernels named with nccl else: for runtimenode in hostnode.runtime_node: for devicenode in runtimenode.device_node: if devicenode.type == TracerEventType.Kernel: if 'nccl' in devicenode.name.lower(): self.gpu_communication_range.append( - (devicenode.start_ns, - devicenode.end_ns)) + (devicenode.start_ns, devicenode.end_ns) + ) else: self.computation_range.append( - (devicenode.start_ns, - devicenode.end_ns)) + (devicenode.start_ns, devicenode.end_ns) + ) self.cpu_calls = len(set(self.cpu_communication_range)) self.gpu_calls = len(set(self.gpu_communication_range)) self.cpu_communication_range = merge_self_ranges( - self.cpu_communication_range, is_sorted=False) + self.cpu_communication_range, is_sorted=False + ) self.gpu_communication_range = merge_self_ranges( - self.gpu_communication_range, is_sorted=False) - self.communication_range = merge_ranges(self.cpu_communication_range, - self.gpu_communication_range, - is_sorted=True) - self.computation_range = merge_self_ranges(self.computation_range, - is_sorted=False) - self.overlap_range = intersection_ranges(self.communication_range, - self.computation_range, - is_sorted=True) + self.gpu_communication_range, is_sorted=False + ) + self.communication_range = merge_ranges( + self.cpu_communication_range, + self.gpu_communication_range, + is_sorted=True, + ) + self.computation_range = merge_self_ranges( + self.computation_range, is_sorted=False + ) + self.overlap_range = intersection_ranges( + self.communication_range, self.computation_range, is_sorted=True + ) class EventSummary: @@ -335,7 +370,6 @@ class EventSummary: """ class DeviceItem: - def __init__(self, name): self.name = name self.call = 0 @@ -359,7 +393,6 @@ class EventSummary: self.add_gpu_time(node.end_ns - node.start_ns) class OperatorItem: - def __init__(self, name): self.name = name self.call = 0 @@ -420,7 +453,8 @@ class EventSummary: if child.type != TracerEventType.Operator: if child.name not in self.operator_inners: self.operator_inners[ - child.name] = EventSummary.OperatorItem(child.name) + child.name + ] = EventSummary.OperatorItem(child.name) self.operator_inners[child.name].add_item(child) for runtimenode in node.runtime_node: @@ -431,7 +465,6 @@ class EventSummary: self.devices[name].add_item(devicenode) class GeneralItem: - def __init__(self, name): self.name = name self.call = 0 @@ -490,10 +523,12 @@ class EventSummary: def __init__(self): self.items = {} # for operator summary self.thread_items = collections.defaultdict( - dict) # for operator summary + dict + ) # for operator summary self.userdefined_items = {} # for userdefined summary self.userdefined_thread_items = collections.defaultdict( - dict) # for userdefined summary + dict + ) # for userdefined summary self.model_perspective_items = {} # for model summary self.memory_manipulation_items = {} # for memory manipulation summary self.kernel_items = {} # for kernel summary @@ -503,19 +538,31 @@ class EventSummary: Analysis operator event in the nodetress. """ node_statistic_trees, thread2host_statistic_nodes = wrap_tree(nodetrees) - for threadid, host_statistic_nodes in thread2host_statistic_nodes.items( - ): + for ( + threadid, + host_statistic_nodes, + ) in thread2host_statistic_nodes.items(): for host_statistic_node in host_statistic_nodes[ - 1:]: #skip root node + 1: + ]: # skip root node if host_statistic_node.type == TracerEventType.Operator: self.add_operator_item(host_statistic_node) - if host_statistic_node.type == TracerEventType.UserDefined\ - or host_statistic_node.type == TracerEventType.PythonUserDefined: - if 'memcpy' in host_statistic_node.name.lower() or 'memorycopy' in host_statistic_node.name.lower()\ - or 'memset' in host_statistic_node.name.lower(): + if ( + host_statistic_node.type == TracerEventType.UserDefined + or host_statistic_node.type + == TracerEventType.PythonUserDefined + ): + if ( + 'memcpy' in host_statistic_node.name.lower() + or 'memorycopy' in host_statistic_node.name.lower() + or 'memset' in host_statistic_node.name.lower() + ): self.add_memory_manipulation_item(host_statistic_node) else: - if host_statistic_node.type == TracerEventType.PythonUserDefined: + if ( + host_statistic_node.type + == TracerEventType.PythonUserDefined + ): self.add_userdefined_item(host_statistic_node) self.add_kernel_item(host_statistic_nodes[0]) @@ -525,10 +572,15 @@ class EventSummary: while deque: current_node = deque.popleft() for child in current_node.children_node: - if child.type == TracerEventType.Forward or child.type == TracerEventType.Dataloader\ - or child.type == TracerEventType.Backward or child.type == TracerEventType.Optimization: + if ( + child.type == TracerEventType.Forward + or child.type == TracerEventType.Dataloader + or child.type == TracerEventType.Backward + or child.type == TracerEventType.Optimization + ): self.add_model_perspective_item( - child) #find first model perspective node + child + ) # find first model perspective node else: if child.type == TracerEventType.ProfileStep: self.add_model_perspective_item(child) @@ -537,40 +589,46 @@ class EventSummary: def add_operator_item(self, operator_node): if operator_node.name not in self.items: self.items[operator_node.name] = EventSummary.OperatorItem( - operator_node.name) + operator_node.name + ) self.items[operator_node.name].add_item(operator_node) if operator_node.name not in self.thread_items[operator_node.thread_id]: self.thread_items[operator_node.thread_id][ - operator_node.name] = EventSummary.OperatorItem( - operator_node.name) + operator_node.name + ] = EventSummary.OperatorItem(operator_node.name) self.thread_items[operator_node.thread_id][operator_node.name].add_item( - operator_node) + operator_node + ) def add_userdefined_item(self, userdefined_node): if userdefined_node.name not in self.userdefined_items: self.userdefined_items[ - userdefined_node.name] = EventSummary.GeneralItem( - userdefined_node.name) + userdefined_node.name + ] = EventSummary.GeneralItem(userdefined_node.name) self.userdefined_items[userdefined_node.name].add_item(userdefined_node) - if userdefined_node.name not in self.userdefined_thread_items[ - userdefined_node.thread_id]: + if ( + userdefined_node.name + not in self.userdefined_thread_items[userdefined_node.thread_id] + ): self.userdefined_thread_items[userdefined_node.thread_id][ - userdefined_node.name] = EventSummary.GeneralItem( - userdefined_node.name) + userdefined_node.name + ] = EventSummary.GeneralItem(userdefined_node.name) self.userdefined_thread_items[userdefined_node.thread_id][ - userdefined_node.name].add_item(userdefined_node) + userdefined_node.name + ].add_item(userdefined_node) def add_memory_manipulation_item(self, memory_manipulation_node): if memory_manipulation_node.name not in self.memory_manipulation_items: self.memory_manipulation_items[ - memory_manipulation_node.name] = EventSummary.GeneralItem( - memory_manipulation_node.name) + memory_manipulation_node.name + ] = EventSummary.GeneralItem(memory_manipulation_node.name) self.memory_manipulation_items[memory_manipulation_node.name].add_item( - memory_manipulation_node) + memory_manipulation_node + ) def add_model_perspective_item(self, model_perspective_node): if model_perspective_node.type == TracerEventType.Forward: @@ -605,7 +663,6 @@ class MemorySummary: """ class MemoryItem: - def __init__(self, event_name, place, memory_type='Allocated'): self.event_name = event_name self.place = place @@ -617,11 +674,17 @@ class MemorySummary: self.memory_type = memory_type def add_memory_record(self, size, allocation_type): - if allocation_type == TracerMemEventType.Allocate or allocation_type == TracerMemEventType.ReservedAllocate: + if ( + allocation_type == TracerMemEventType.Allocate + or allocation_type == TracerMemEventType.ReservedAllocate + ): self.allocation_count += 1 self.allocation_size += size - elif allocation_type == TracerMemEventType.Free or allocation_type == TracerMemEventType.ReservedFree: + elif ( + allocation_type == TracerMemEventType.Free + or allocation_type == TracerMemEventType.ReservedFree + ): self.free_count += 1 self.free_size -= size # size is sign(-) when free. @@ -631,35 +694,49 @@ class MemorySummary: def __init__(self): self.allocated_items = collections.defaultdict( - dict) # for memory summary, device type: event + dict + ) # for memory summary, device type: event self.reserved_items = collections.defaultdict( - dict) # for memory summary, device type: event + dict + ) # for memory summary, device type: event self.peak_allocation_values = collections.defaultdict(int) self.peak_reserved_values = collections.defaultdict(int) def _analyse_node_memory(self, event_name, node): for memnode in node.mem_node: # self mem node - if memnode.type == TracerMemEventType.Allocate or memnode.type == TracerMemEventType.Free: + if ( + memnode.type == TracerMemEventType.Allocate + or memnode.type == TracerMemEventType.Free + ): if event_name not in self.allocated_items[memnode.place]: - self.allocated_items[ - memnode.place][event_name] = MemorySummary.MemoryItem( - event_name, memnode.place, 'Allocated') - self.allocated_items[ - memnode.place][event_name].add_memory_record( - memnode.increase_bytes, memnode.type) - elif memnode.type == TracerMemEventType.ReservedAllocate or memnode.type == TracerMemEventType.ReservedFree: + self.allocated_items[memnode.place][ + event_name + ] = MemorySummary.MemoryItem( + event_name, memnode.place, 'Allocated' + ) + self.allocated_items[memnode.place][ + event_name + ].add_memory_record(memnode.increase_bytes, memnode.type) + elif ( + memnode.type == TracerMemEventType.ReservedAllocate + or memnode.type == TracerMemEventType.ReservedFree + ): if event_name not in self.reserved_items[memnode.place]: - self.reserved_items[ - memnode.place][event_name] = MemorySummary.MemoryItem( - event_name, memnode.place, 'Reserved') - self.reserved_items[ - memnode.place][event_name].add_memory_record( - memnode.increase_bytes, memnode.type) + self.reserved_items[memnode.place][ + event_name + ] = MemorySummary.MemoryItem( + event_name, memnode.place, 'Reserved' + ) + self.reserved_items[memnode.place][ + event_name + ].add_memory_record(memnode.increase_bytes, memnode.type) self.peak_allocation_values[memnode.place] = max( self.peak_allocation_values[memnode.place], - memnode.peak_allocated) + memnode.peak_allocated, + ) self.peak_reserved_values[memnode.place] = max( - self.peak_reserved_values[memnode.place], memnode.peak_reserved) + self.peak_reserved_values[memnode.place], memnode.peak_reserved + ) def parse(self, nodetrees): r""" @@ -667,7 +744,7 @@ class MemorySummary: """ thread2hostnodes = traverse_tree(nodetrees) for threadid, host_nodes in thread2hostnodes.items(): - for host_node in host_nodes[1:]: #skip root node + for host_node in host_nodes[1:]: # skip root node if host_node.type == TracerEventType.OperatorInner: continue if host_node.type == TracerEventType.Operator: @@ -694,16 +771,19 @@ class StatisticData: self.memory_summary.parse(node_trees) -def _build_table(statistic_data, - sorted_by=SortedKeys.CPUTotal, - op_detail=True, - thread_sep=False, - time_unit='ms', - row_limit=100, - max_src_column_width=75, - views=None): +def _build_table( + statistic_data, + sorted_by=SortedKeys.CPUTotal, + op_detail=True, + thread_sep=False, + time_unit='ms', + row_limit=100, + max_src_column_width=75, + views=None, +): from .profiler import SummaryView + """Prints a summary of events.""" # format table row SPACING_SIZE = 2 @@ -712,8 +792,9 @@ def _build_table(statistic_data, line_length_list = [-SPACING_SIZE] def add_column(padding, text_dir='<'): - row_format_list[0] += '{: ' + text_dir + str(padding) + '}' + ( - ' ' * SPACING_SIZE) + row_format_list[0] += ( + '{: ' + text_dir + str(padding) + '}' + (' ' * SPACING_SIZE) + ) header_sep_list[0] += '-' * padding + (' ' * SPACING_SIZE) line_length_list[0] += padding + SPACING_SIZE @@ -751,7 +832,8 @@ def _build_table(statistic_data, return '{}{:.2f}'.format(' ' * indent, ratio * 100) total_time = statistic_data.time_range_summary.get_cpu_range_sum( - TracerEventType.ProfileStep) + TracerEventType.ProfileStep + ) if views is None or SummaryView.DeviceView in views: @@ -776,19 +858,23 @@ def _build_table(statistic_data, row_values = [ 'CPU(Process)', format_ratio( - float(statistic_data.extra_info['Process Cpu Utilization'])) + float(statistic_data.extra_info['Process Cpu Utilization']) + ), ] append(row_format.format(*row_values)) row_values = [ 'CPU(System)', format_ratio( - float(statistic_data.extra_info['System Cpu Utilization'])) + float(statistic_data.extra_info['System Cpu Utilization']) + ), ] append(row_format.format(*row_values)) for gpu_name in statistic_data.time_range_summary.get_gpu_devices(): gpu_time = float( statistic_data.time_range_summary.get_gpu_range_sum( - gpu_name, TracerEventType.Kernel)) + gpu_name, TracerEventType.Kernel + ) + ) utilization = gpu_time / total_time row_values = ['GPU{}'.format(gpu_name), format_ratio(utilization)] append(row_format.format(*row_values)) @@ -797,7 +883,8 @@ def _build_table(statistic_data, append( "Note:\nCPU(Process) Utilization = Current process CPU time over all cpu cores / elapsed time, so max utilization can be reached 100% * number of cpu cores.\n" "CPU(System) Utilization = All processes CPU time over all cpu cores(busy time) / (busy time + idle time).\n" - "GPU Utilization = Current process GPU time / elapsed time.") + "GPU Utilization = Current process GPU time / elapsed time." + ) append('-' * line_length) append('') append('') @@ -833,54 +920,71 @@ def _build_table(statistic_data, cpu_call_times.update(statistic_data.time_range_summary.call_times) gpu_call_times.update(statistic_data.time_range_summary.call_times) - for event_type, value in statistic_data.time_range_summary.CPUTimeRangeSum.items( - ): + for ( + event_type, + value, + ) in statistic_data.time_range_summary.CPUTimeRangeSum.items(): if event_type != TracerEventType.Communication: cpu_type_time[event_type] = value if statistic_data.distributed_summary.cpu_communication_range: cpu_type_time[TracerEventType.Communication] = sum_ranges( - statistic_data.distributed_summary.cpu_communication_range) + statistic_data.distributed_summary.cpu_communication_range + ) cpu_call_times[ - TracerEventType. - Communication] = statistic_data.distributed_summary.cpu_calls + TracerEventType.Communication + ] = statistic_data.distributed_summary.cpu_calls for event_type in [ - TracerEventType.Dataloader, TracerEventType.Forward, - TracerEventType.Backward, TracerEventType.Optimization + TracerEventType.Dataloader, + TracerEventType.Forward, + TracerEventType.Backward, + TracerEventType.Optimization, ]: event_type_name = str(event_type).split('.')[1] - if event_type in cpu_call_times and event_type_name in statistic_data.event_summary.model_perspective_items: + if ( + event_type in cpu_call_times + and event_type_name + in statistic_data.event_summary.model_perspective_items + ): cpu_call_times[ - event_type] = statistic_data.event_summary.model_perspective_items[ - event_type_name].call + event_type + ] = statistic_data.event_summary.model_perspective_items[ + event_type_name + ].call cpu_type_time[ - event_type] = statistic_data.event_summary.model_perspective_items[ - event_type_name].cpu_time + event_type + ] = statistic_data.event_summary.model_perspective_items[ + event_type_name + ].cpu_time gpu_time_range = collections.defaultdict(list) - for device_id, device_time_ranges in statistic_data.time_range_summary.GPUTimeRange.items( - ): + for ( + device_id, + device_time_ranges, + ) in statistic_data.time_range_summary.GPUTimeRange.items(): for event_type, time_range in device_time_ranges.items(): gpu_time_range[event_type] = merge_ranges( - gpu_time_range[event_type], time_range, is_sorted=True) + gpu_time_range[event_type], time_range, is_sorted=True + ) for event_type, time_range in gpu_time_range.items(): gpu_type_time[event_type] = sum_ranges(time_range) if statistic_data.distributed_summary.gpu_communication_range: gpu_type_time[TracerEventType.Communication] = sum_ranges( - statistic_data.distributed_summary.gpu_communication_range) + statistic_data.distributed_summary.gpu_communication_range + ) gpu_call_times[ - TracerEventType. - Communication] = statistic_data.distributed_summary.gpu_calls + TracerEventType.Communication + ] = statistic_data.distributed_summary.gpu_calls - sorted_items = sorted(cpu_type_time.items(), - key=lambda x: x[1], - reverse=True) + sorted_items = sorted( + cpu_type_time.items(), key=lambda x: x[1], reverse=True + ) event_type, time = sorted_items[0] row_values = [ '{}'.format(str(event_type).split('.')[1]), cpu_call_times[event_type], format_time(time, unit=time_unit), - format_ratio(float(time) / total_time) + format_ratio(float(time) / total_time), ] append(row_format.format(*row_values)) for event_type, time in sorted_items[1:]: @@ -888,7 +992,7 @@ def _build_table(statistic_data, ' {}'.format(str(event_type).split('.')[1]), cpu_call_times[event_type], format_time(time, unit=time_unit), - format_ratio(float(time) / total_time) + format_ratio(float(time) / total_time), ] append(row_format.format(*row_values)) append(header_sep) @@ -900,7 +1004,7 @@ def _build_table(statistic_data, ' {}'.format(str(event_type).split('.')[1]), gpu_call_times[event_type], format_time(time, unit=time_unit), - format_ratio(float(time) / total_time) + format_ratio(float(time) / total_time), ] append(row_format.format(*row_values)) @@ -916,7 +1020,8 @@ def _build_table(statistic_data, "Thread 2:\n" " Operator: |____________| |___|\n" "After merged:\n" - " Result: |______________| |__________|\n") + " Result: |______________| |__________|\n" + ) append('-' * line_length) append('') append('') @@ -924,16 +1029,24 @@ def _build_table(statistic_data, if views is None or SummaryView.ModelView in views: ###### Print Model Summary Report ###### - model_perspective_items = statistic_data.event_summary.model_perspective_items + model_perspective_items = ( + statistic_data.event_summary.model_perspective_items + ) if len(model_perspective_items) > 1: all_row_values = [] accmulation_time = 0 gpu_accmulation_time = 0 - gpu_total_time = statistic_data.event_summary.model_perspective_items[ - 'ProfileStep'].gpu_time + gpu_total_time = ( + statistic_data.event_summary.model_perspective_items[ + 'ProfileStep' + ].gpu_time + ) for name in [ - 'ProfileStep', 'Dataloader', 'Forward', 'Backward', - 'Optimization' + 'ProfileStep', + 'Dataloader', + 'Forward', + 'Backward', + 'Optimization', ]: if name in model_perspective_items: item = model_perspective_items[name] @@ -941,22 +1054,28 @@ def _build_table(statistic_data, gpu_ratio = 0 else: gpu_ratio = float(item.gpu_time) / gpu_total_time - name = '{}'.format( - name) if 'ProfileStep' in name else ' {}'.format(name) + name = ( + '{}'.format(name) + if 'ProfileStep' in name + else ' {}'.format(name) + ) row_values = [ - '{}'.format(name), item.call, + '{}'.format(name), + item.call, '{} / {} / {} / {} / {}'.format( format_time(item.cpu_time, unit=time_unit), format_time(item.avg_cpu_time, unit=time_unit), format_time(item.max_cpu_time, unit=time_unit), format_time(item.min_cpu_time, unit=time_unit), - format_ratio(float(item.cpu_time) / total_time)), + format_ratio(float(item.cpu_time) / total_time), + ), '{} / {} / {} / {} / {}'.format( format_time(item.gpu_time, unit=time_unit), format_time(item.avg_gpu_time, unit=time_unit), format_time(item.max_gpu_time, unit=time_unit), format_time(item.min_gpu_time, unit=time_unit), - format_ratio(gpu_ratio)) + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) if 'ProfileStep' not in name: @@ -970,12 +1089,16 @@ def _build_table(statistic_data, else: gpu_ratio = float(other_gpu_time) / gpu_total_time row_values = [ - ' Others', '-', '{} / - / - / - / {}'.format( + ' Others', + '-', + '{} / - / - / - / {}'.format( format_time(other_time, unit=time_unit), - format_ratio(float(other_time) / total_time)), + format_ratio(float(other_time) / total_time), + ), '{} / - / - / - / {}'.format( format_time(other_gpu_time, unit=time_unit), - format_ratio(gpu_ratio)) + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) # Calculate the column width @@ -983,16 +1106,20 @@ def _build_table(statistic_data, cpu_data_description_width = 40 gpu_data_description_width = 40 for row_values in all_row_values: - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > cpu_data_description_width: cpu_data_description_width = len(row_values[2]) if len(row_values[3]) > gpu_data_description_width: gpu_data_description_width = len(row_values[3]) headers = [ - 'Name', 'Calls', 'CPU Total / Avg / Max / Min / Ratio(%)', - 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'CPU Total / Avg / Max / Min / Ratio(%)', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] row_format_list = [""] header_sep_list = [""] @@ -1052,35 +1179,38 @@ def _build_table(statistic_data, append(row_format.format(*headers)) append(header_sep) communication_time = sum_ranges( - statistic_data.distributed_summary.communication_range) + statistic_data.distributed_summary.communication_range + ) computation_time = sum_ranges( - statistic_data.distributed_summary.computation_range) + statistic_data.distributed_summary.computation_range + ) overlap_time = sum_ranges( - statistic_data.distributed_summary.overlap_range) + statistic_data.distributed_summary.overlap_range + ) row_values = [ 'ProfileStep', format_time(total_time, unit=time_unit), - format_ratio(float(total_time) / total_time) + format_ratio(float(total_time) / total_time), ] append(row_format.format(*row_values)) row_values = [ ' Communication', format_time(communication_time, unit=time_unit), - format_ratio(float(communication_time) / total_time) + format_ratio(float(communication_time) / total_time), ] append(row_format.format(*row_values)) row_values = [ ' Computation', format_time(computation_time, unit=time_unit), - format_ratio(float(computation_time) / total_time) + format_ratio(float(computation_time) / total_time), ] append(row_format.format(*row_values)) row_values = [ ' Overlap', format_time(overlap_time, unit=time_unit), - format_ratio(float(overlap_time) / total_time) + format_ratio(float(overlap_time) / total_time), ] append(row_format.format(*row_values)) append(header_sep) @@ -1095,7 +1225,8 @@ def _build_table(statistic_data, " Total: |_________________| |______________|\n" "Computation time(Kernel):\n" " GPU: |________________|\n" - "Overlap time: |___________|\n") + "Overlap time: |___________|\n" + ) append('-' * line_length) append('') append('') @@ -1115,37 +1246,47 @@ def _build_table(statistic_data, for thread_id, items in thread_items.items(): all_row_values.append("Thread: {}".format(thread_id)) if sorted_by == SortedKeys.CPUTotal: - sorted_items = sorted(items.items(), - key=lambda x: x[1].cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), key=lambda x: x[1].cpu_time, reverse=True + ) elif sorted_by == SortedKeys.CPUAvg: - sorted_items = sorted(items.items(), - key=lambda x: x[1].avg_cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].avg_cpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.CPUMax: - sorted_items = sorted(items.items(), - key=lambda x: x[1].max_cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].max_cpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.CPUMin: - sorted_items = sorted(items.items(), - key=lambda x: x[1].min_cpu_time) + sorted_items = sorted( + items.items(), key=lambda x: x[1].min_cpu_time + ) elif sorted_by == SortedKeys.GPUTotal: - sorted_items = sorted(items.items(), - key=lambda x: x[1].general_gpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].general_gpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.GPUAvg: sorted_items = sorted( items.items(), key=lambda x: x[1].avg_general_gpu_time, - reverse=True) + reverse=True, + ) elif sorted_by == SortedKeys.GPUMax: sorted_items = sorted( items.items(), key=lambda x: x[1].max_general_gpu_time, - reverse=True) + reverse=True, + ) elif sorted_by == SortedKeys.GPUMin: sorted_items = sorted( - items.items(), key=lambda x: x[1].min_general_gpu_time) + items.items(), key=lambda x: x[1].min_general_gpu_time + ) total_op_cpu_time = 0 total_op_gpu_time = 0 @@ -1161,124 +1302,177 @@ def _build_table(statistic_data, if total_op_gpu_time == 0: gpu_ratio = 0 else: - gpu_ratio = float( - item.general_gpu_time) / total_op_gpu_time + gpu_ratio = ( + float(item.general_gpu_time) / total_op_gpu_time + ) row_values = [ - name, item.call, '{} / {} / {} / {} / {}'.format( + name, + item.call, + '{} / {} / {} / {} / {}'.format( format_time(item.cpu_time, unit=time_unit), format_time(item.avg_cpu_time, unit=time_unit), format_time(item.max_cpu_time, unit=time_unit), format_time(item.min_cpu_time, unit=time_unit), - format_ratio(cpu_ratio)), + format_ratio(cpu_ratio), + ), '{} / {} / {} / {} / {}'.format( format_time(item.general_gpu_time, unit=time_unit), - format_time(item.avg_general_gpu_time, - unit=time_unit), - format_time(item.max_general_gpu_time, - unit=time_unit), - format_time(item.min_general_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)) + format_time( + item.avg_general_gpu_time, unit=time_unit + ), + format_time( + item.max_general_gpu_time, unit=time_unit + ), + format_time( + item.min_general_gpu_time, unit=time_unit + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) if op_detail: - for innerop_name, innerop_node in item.operator_inners.items( - ): + for ( + innerop_name, + innerop_node, + ) in item.operator_inners.items(): if item.cpu_time == 0: cpu_ratio = 0 else: - cpu_ratio = float( - innerop_node.cpu_time) / item.cpu_time + cpu_ratio = ( + float(innerop_node.cpu_time) / item.cpu_time + ) if item.general_gpu_time == 0: gpu_ratio = 0 else: - gpu_ratio = float(innerop_node.general_gpu_time - ) / item.general_gpu_time + gpu_ratio = ( + float(innerop_node.general_gpu_time) + / item.general_gpu_time + ) if len(innerop_name) + 2 > name_column_width: - innerop_name = innerop_name[:name_column_width - - 5] + innerop_name = innerop_name[ + : name_column_width - 5 + ] innerop_name += "..." row_values = [ - ' {}'.format(innerop_name), innerop_node.call, + ' {}'.format(innerop_name), + innerop_node.call, '{} / {} / {} / {} / {}'.format( - format_time(innerop_node.cpu_time, - unit=time_unit), - format_time(innerop_node.avg_cpu_time, - unit=time_unit), - format_time(innerop_node.max_cpu_time, - unit=time_unit), - format_time(innerop_node.min_cpu_time, - unit=time_unit), - format_ratio(cpu_ratio)), + format_time( + innerop_node.cpu_time, unit=time_unit + ), + format_time( + innerop_node.avg_cpu_time, + unit=time_unit, + ), + format_time( + innerop_node.max_cpu_time, + unit=time_unit, + ), + format_time( + innerop_node.min_cpu_time, + unit=time_unit, + ), + format_ratio(cpu_ratio), + ), '{} / {} / {} / {} / {}'.format( - format_time(innerop_node.general_gpu_time, - unit=time_unit), + format_time( + innerop_node.general_gpu_time, + unit=time_unit, + ), format_time( innerop_node.avg_general_gpu_time, - unit=time_unit), + unit=time_unit, + ), format_time( innerop_node.max_general_gpu_time, - unit=time_unit), + unit=time_unit, + ), format_time( innerop_node.min_general_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)) + unit=time_unit, + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) - for device_node_name, device_node in innerop_node.devices.items( - ): + for ( + device_node_name, + device_node, + ) in innerop_node.devices.items(): if innerop_node.general_gpu_time == 0: gpu_ratio = 0 else: - gpu_ratio = float( - device_node.gpu_time - ) / innerop_node.general_gpu_time - if len(device_node_name - ) + 4 > name_column_width: - device_node_name = device_node_name[: - name_column_width - - 7] + gpu_ratio = ( + float(device_node.gpu_time) + / innerop_node.general_gpu_time + ) + if ( + len(device_node_name) + 4 + > name_column_width + ): + device_node_name = device_node_name[ + : name_column_width - 7 + ] device_node_name += "..." row_values = [ ' {}'.format(device_node_name), - device_node.call, '- / - / - / - / -', + device_node.call, + '- / - / - / - / -', '{} / {} / {} / {} / {}'.format( - format_time(device_node.gpu_time, - unit=time_unit), - format_time(device_node.avg_gpu_time, - unit=time_unit), - format_time(device_node.max_gpu_time, - unit=time_unit), - format_time(device_node.min_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)) + format_time( + device_node.gpu_time, unit=time_unit + ), + format_time( + device_node.avg_gpu_time, + unit=time_unit, + ), + format_time( + device_node.max_gpu_time, + unit=time_unit, + ), + format_time( + device_node.min_gpu_time, + unit=time_unit, + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) - for device_node_name, device_node in item.devices.items( - ): + for ( + device_node_name, + device_node, + ) in item.devices.items(): if item.general_gpu_time == 0: gpu_ratio = 0 else: - gpu_ratio = float(device_node.gpu_time - ) / item.general_gpu_time + gpu_ratio = ( + float(device_node.gpu_time) + / item.general_gpu_time + ) if len(device_node_name) + 2 > name_column_width: - device_node_name = device_node_name[: - name_column_width - - 5] + device_node_name = device_node_name[ + : name_column_width - 5 + ] device_node_name += "..." row_values = [ ' {}'.format(device_node_name), - device_node.call, '- / - / - / - / -', + device_node.call, + '- / - / - / - / -', '{} / {} / {} / {} / {}'.format( - format_time(device_node.gpu_time, - unit=time_unit), - format_time(device_node.avg_gpu_time, - unit=time_unit), - format_time(device_node.max_gpu_time, - unit=time_unit), - format_time(device_node.min_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)) + format_time( + device_node.gpu_time, unit=time_unit + ), + format_time( + device_node.avg_gpu_time, unit=time_unit + ), + format_time( + device_node.max_gpu_time, unit=time_unit + ), + format_time( + device_node.min_gpu_time, unit=time_unit + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) # Calculate the column width @@ -1288,16 +1482,20 @@ def _build_table(statistic_data, for row_values in all_row_values: if isinstance(row_values, str): continue - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > cpu_data_description_width: cpu_data_description_width = len(row_values[2]) if len(row_values[3]) > gpu_data_description_width: gpu_data_description_width = len(row_values[3]) headers = [ - 'Name', 'Calls', 'CPU Total / Avg / Max / Min / Ratio(%)', - 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'CPU Total / Avg / Max / Min / Ratio(%)', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] row_format_list = [""] header_sep_list = [""] @@ -1333,20 +1531,27 @@ def _build_table(statistic_data, all_row_values = [] kernel_items = statistic_data.event_summary.kernel_items if sorted_by == SortedKeys.GPUAvg: - sorted_items = sorted(kernel_items.items(), - key=lambda x: x[1].avg_gpu_time, - reverse=True) + sorted_items = sorted( + kernel_items.items(), + key=lambda x: x[1].avg_gpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.GPUMax: - sorted_items = sorted(kernel_items.items(), - key=lambda x: x[1].max_gpu_time, - reverse=True) + sorted_items = sorted( + kernel_items.items(), + key=lambda x: x[1].max_gpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.GPUMin: - sorted_items = sorted(kernel_items.items(), - key=lambda x: x[1].min_gpu_time) + sorted_items = sorted( + kernel_items.items(), key=lambda x: x[1].min_gpu_time + ) else: - sorted_items = sorted(kernel_items.items(), - key=lambda x: x[1].gpu_time, - reverse=True) + sorted_items = sorted( + kernel_items.items(), + key=lambda x: x[1].gpu_time, + reverse=True, + ) total_kernel_gpu_time = 0 for name, item in sorted_items: @@ -1364,20 +1569,25 @@ def _build_table(statistic_data, format_time(item.avg_gpu_time, unit=time_unit), format_time(item.max_gpu_time, unit=time_unit), format_time(item.min_gpu_time, unit=time_unit), - format_ratio(gpu_ratio)), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) headers = [ - 'Name', 'Calls', 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] # Calculate the column width name_column_width = 90 calltime_width = 6 gpu_data_description_width = 40 for row_values in all_row_values: - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > gpu_data_description_width: gpu_data_description_width = len(row_values[2]) @@ -1407,7 +1617,7 @@ def _build_table(statistic_data, else: name = row_values[0] if len(name) > name_column_width: - row_values[0] = name[:name_column_width - 3] + '...' + row_values[0] = name[: name_column_width - 3] + '...' else: row_values[0] = name append(row_format.format(*row_values)) @@ -1420,9 +1630,14 @@ def _build_table(statistic_data, ###### Print Memory Manipulation Summary Report ###### if statistic_data.event_summary.memory_manipulation_items: all_row_values = [] - memory_manipulation_items = statistic_data.event_summary.memory_manipulation_items - gpu_total_time = statistic_data.event_summary.model_perspective_items[ - 'ProfileStep'].general_gpu_time + memory_manipulation_items = ( + statistic_data.event_summary.memory_manipulation_items + ) + gpu_total_time = ( + statistic_data.event_summary.model_perspective_items[ + 'ProfileStep' + ].general_gpu_time + ) for name, item in memory_manipulation_items.items(): if gpu_total_time == 0: gpu_ratio = 0 @@ -1436,19 +1651,23 @@ def _build_table(statistic_data, format_time(item.avg_cpu_time, unit=time_unit), format_time(item.max_cpu_time, unit=time_unit), format_time(item.min_cpu_time, unit=time_unit), - format_ratio(float(item.cpu_time) / total_time)), + format_ratio(float(item.cpu_time) / total_time), + ), '{} / {} / {} / {} / {}'.format( format_time(item.general_gpu_time, unit=time_unit), format_time(item.avg_general_gpu_time, unit=time_unit), format_time(item.max_general_gpu_time, unit=time_unit), format_time(item.min_general_gpu_time, unit=time_unit), - format_ratio(gpu_ratio)), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) headers = [ - 'Name', 'Calls', 'CPU Total / Avg / Max / Min / Ratio(%)', - 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'CPU Total / Avg / Max / Min / Ratio(%)', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] # Calculate the column width name_column_width = 0 @@ -1458,8 +1677,10 @@ def _build_table(statistic_data, for row_values in all_row_values: if len(row_values[0]) > name_column_width: name_column_width = len(row_values[0]) - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > cpu_data_description_width: cpu_data_description_width = len(row_values[2]) @@ -1495,56 +1716,71 @@ def _build_table(statistic_data, ###### Print UserDefined Summary Report ###### if statistic_data.event_summary.userdefined_items: all_row_values = [] - gpu_total_time = statistic_data.event_summary.model_perspective_items[ - 'ProfileStep'].general_gpu_time + gpu_total_time = ( + statistic_data.event_summary.model_perspective_items[ + 'ProfileStep' + ].general_gpu_time + ) if thread_sep == True: - userdefined_thread_items = statistic_data.event_summary.userdefined_thread_items + userdefined_thread_items = ( + statistic_data.event_summary.userdefined_thread_items + ) else: userdefined_thread_items = { - 'All threads merged': - statistic_data.event_summary.userdefined_items + 'All threads merged': statistic_data.event_summary.userdefined_items } for thread_id, items in userdefined_thread_items.items(): all_row_values.append("Thread: {}".format(thread_id)) if sorted_by == SortedKeys.CPUTotal: - sorted_items = sorted(items.items(), - key=lambda x: x[1].cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), key=lambda x: x[1].cpu_time, reverse=True + ) elif sorted_by == SortedKeys.CPUAvg: - sorted_items = sorted(items.items(), - key=lambda x: x[1].avg_cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].avg_cpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.CPUMax: - sorted_items = sorted(items.items(), - key=lambda x: x[1].max_cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].max_cpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.CPUMin: - sorted_items = sorted(items.items(), - key=lambda x: x[1].min_cpu_time) + sorted_items = sorted( + items.items(), key=lambda x: x[1].min_cpu_time + ) elif sorted_by == SortedKeys.GPUTotal: - sorted_items = sorted(items.items(), - key=lambda x: x[1].general_gpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].general_gpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.GPUAvg: sorted_items = sorted( items.items(), key=lambda x: x[1].avg_general_gpu_time, - reverse=True) + reverse=True, + ) elif sorted_by == SortedKeys.GPUMax: sorted_items = sorted( items.items(), key=lambda x: x[1].max_general_gpu_time, - reverse=True) + reverse=True, + ) elif sorted_by == SortedKeys.GPUMin: sorted_items = sorted( - items.items(), key=lambda x: x[1].min_general_gpu_time) + items.items(), key=lambda x: x[1].min_general_gpu_time + ) for name, item in sorted_items: if gpu_total_time == 0: gpu_ratio = 0 else: - gpu_ratio = float( - item.general_gpu_time) / gpu_total_time + gpu_ratio = ( + float(item.general_gpu_time) / gpu_total_time + ) row_values = [ name, item.call, @@ -1553,16 +1789,21 @@ def _build_table(statistic_data, format_time(item.avg_cpu_time, unit=time_unit), format_time(item.max_cpu_time, unit=time_unit), format_time(item.min_cpu_time, unit=time_unit), - format_ratio(float(item.cpu_time) / total_time)), + format_ratio(float(item.cpu_time) / total_time), + ), '{} / {} / {} / {} / {}'.format( format_time(item.general_gpu_time, unit=time_unit), - format_time(item.avg_general_gpu_time, - unit=time_unit), - format_time(item.max_general_gpu_time, - unit=time_unit), - format_time(item.min_general_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)), + format_time( + item.avg_general_gpu_time, unit=time_unit + ), + format_time( + item.max_general_gpu_time, unit=time_unit + ), + format_time( + item.min_general_gpu_time, unit=time_unit + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) @@ -1576,8 +1817,10 @@ def _build_table(statistic_data, continue if len(row_values[0]) > name_column_width: name_column_width = len(row_values[0]) - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > cpu_data_description_width: cpu_data_description_width = len(row_values[2]) @@ -1585,8 +1828,10 @@ def _build_table(statistic_data, gpu_data_description_width = len(row_values[3]) headers = [ - 'Name', 'Calls', 'CPU Total / Avg / Max / Min / Ratio(%)', - 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'CPU Total / Avg / Max / Min / Ratio(%)', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] row_format_list = [""] header_sep_list = [""] @@ -1618,39 +1863,61 @@ def _build_table(statistic_data, if views is None or SummaryView.MemoryView in views: ###### Print Memory Summary Report ###### - if statistic_data.memory_summary.allocated_items or statistic_data.memory_summary.reserved_items: - for device_type, memory_events in statistic_data.memory_summary.allocated_items.items( - ): + if ( + statistic_data.memory_summary.allocated_items + or statistic_data.memory_summary.reserved_items + ): + for ( + device_type, + memory_events, + ) in statistic_data.memory_summary.allocated_items.items(): all_row_values = [] - sorted_items = sorted(memory_events.items(), - key=lambda x: x[1].increase_size, - reverse=True) + sorted_items = sorted( + memory_events.items(), + key=lambda x: x[1].increase_size, + reverse=True, + ) for event_name, item in sorted_items: row_values = [ - event_name, item.memory_type, item.allocation_count, - item.free_count, item.allocation_size, item.free_size, - item.increase_size + event_name, + item.memory_type, + item.allocation_count, + item.free_count, + item.allocation_size, + item.free_size, + item.increase_size, ] all_row_values.append(row_values) sorted_reserved_items = sorted( - statistic_data.memory_summary.reserved_items[device_type]. - items(), + statistic_data.memory_summary.reserved_items[ + device_type + ].items(), key=lambda x: x[1].increase_size, - reverse=True) + reverse=True, + ) for event_name, item in sorted_reserved_items: row_values = [ - event_name, item.memory_type, item.allocation_count, - item.free_count, item.allocation_size, item.free_size, - item.increase_size + event_name, + item.memory_type, + item.allocation_count, + item.free_count, + item.allocation_size, + item.free_size, + item.increase_size, ] all_row_values.append(row_values) # Calculate the column width headers = [ - 'Name', 'Type', 'Allocation Count', 'Free Count', - 'Allocation Size', 'Free Size', 'Increased Size' + 'Name', + 'Type', + 'Allocation Count', + 'Free Count', + 'Allocation Size', + 'Free Size', + 'Increased Size', ] row_format_list = [""] header_sep_list = [""] @@ -1671,14 +1938,24 @@ def _build_table(statistic_data, # construct table string append( - add_title(line_length, - "Memory Summary - {}".format(device_type))) - append('Peak Allocated Memory: {}'.format( - statistic_data.memory_summary. - peak_allocation_values[device_type])) - append('Peak Reserved Memory: {}'.format( - statistic_data.memory_summary. - peak_reserved_values[device_type])) + add_title( + line_length, "Memory Summary - {}".format(device_type) + ) + ) + append( + 'Peak Allocated Memory: {}'.format( + statistic_data.memory_summary.peak_allocation_values[ + device_type + ] + ) + ) + append( + 'Peak Reserved Memory: {}'.format( + statistic_data.memory_summary.peak_reserved_values[ + device_type + ] + ) + ) append(header_sep) append(row_format.format(*headers)) append(header_sep) diff --git a/python/paddle/profiler/statistic_helper.py b/python/paddle/profiler/statistic_helper.py index b6925a2b774..13ce42d7d73 100644 --- a/python/paddle/profiler/statistic_helper.py +++ b/python/paddle/profiler/statistic_helper.py @@ -16,7 +16,7 @@ def sum_ranges(ranges): result = 0 for time_range in ranges: - result += (time_range[1] - time_range[0]) + result += time_range[1] - time_range[0] return result @@ -30,11 +30,14 @@ def merge_self_ranges(src_ranges, is_sorted=False): for cur_indx in range(1, len(src_ranges)): if src_ranges[cur_indx][1] > merged_ranges[-1][1]: if src_ranges[cur_indx][0] <= merged_ranges[-1][1]: - merged_ranges[-1] = (merged_ranges[-1][0], - src_ranges[cur_indx][1]) + merged_ranges[-1] = ( + merged_ranges[-1][0], + src_ranges[cur_indx][1], + ) else: merged_ranges.append( - (src_ranges[cur_indx][0], src_ranges[cur_indx][1])) + (src_ranges[cur_indx][0], src_ranges[cur_indx][1]) + ) return merged_ranges @@ -129,7 +132,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range2 = range_list2[indx2] elif range2[0] <= range1[0] and range2[1] < range1[1]: - assert (range2[1] > range1[0]) + assert range2[1] > range1[0] result_range.append((range1[0], range2[1])) range1 = (range2[1], range1[1]) indx2 += 1 @@ -138,7 +141,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range2 = range_list2[indx2] elif range2[0] <= range1[0]: - assert (range2[1] >= range1[1]) + assert range2[1] >= range1[1] result_range.append(range1) range2 = (range1[1], range2[1]) indx1 += 1 @@ -147,7 +150,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range1 = range_list1[indx1] elif range2[1] < range1[1]: - assert (range2[0] > range1[0]) + assert range2[0] > range1[0] result_range.append(range2) range1 = (range2[1], range1[1]) indx2 += 1 @@ -156,7 +159,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range2 = range_list2[indx2] elif range2[0] < range1[1]: - assert (range2[1] >= range1[1]) + assert range2[1] >= range1[1] result_range.append((range2[0], range1[1])) range2 = (range1[1], range2[1]) indx1 += 1 @@ -165,7 +168,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range1 = range_list1[indx1] else: - assert (range2[0] >= range1[1]) + assert range2[0] >= range1[1] indx1 += 1 if indx1 == len1: break @@ -207,17 +210,17 @@ def subtract_ranges(range_list1, range_list2, is_sorted=False): if indx2 != len2: range2 = range_list2[indx2] elif range2[0] <= range1[0]: - assert (range2[1] >= range1[1]) + assert range2[1] >= range1[1] range2 = (range1[1], range2[1]) indx1 += 1 if indx1 != len1: range1 = range_list1[indx1] elif range2[0] < range1[1]: - assert (range2[0] > range1[0]) + assert range2[0] > range1[0] result_range.append((range1[0], range2[0])) range1 = (range2[0], range1[1]) else: - assert (range2[0] >= range1[1]) + assert range2[0] >= range1[1] result_range.append(range1) indx1 += 1 if indx1 != len1: diff --git a/python/paddle/profiler/timer.py b/python/paddle/profiler/timer.py index 1f0fb6acde6..311fc373c5f 100644 --- a/python/paddle/profiler/timer.py +++ b/python/paddle/profiler/timer.py @@ -126,20 +126,28 @@ class Event(object): else: speed_avg = float(self.total_iters) / self.batch_records['total'] - reader_summary = dict(max=self.reader_records['max'], - min=self.reader_records['min'], - avg=reader_avg) - batch_summary = dict(max=self.batch_records['max'], - min=self.batch_records['min'], - avg=batch_avg) - ips_summary = dict(max=self.speed_records['max'], - min=self.speed_records['min'], - avg=speed_avg) + reader_summary = dict( + max=self.reader_records['max'], + min=self.reader_records['min'], + avg=reader_avg, + ) + batch_summary = dict( + max=self.batch_records['max'], + min=self.batch_records['min'], + avg=batch_avg, + ) + ips_summary = dict( + max=self.speed_records['max'], + min=self.speed_records['min'], + avg=speed_avg, + ) reader_ratio = (reader_avg / batch_avg) * 100 - summary = dict(reader_summary=reader_summary, - batch_summary=batch_summary, - ips_summary=ips_summary, - reader_ratio=reader_ratio) + summary = dict( + reader_summary=reader_summary, + batch_summary=batch_summary, + ips_summary=ips_summary, + reader_ratio=reader_ratio, + ) return summary @@ -205,8 +213,11 @@ class TimerHook(Hook): """ reader_cost = timeit.default_timer() - self.start_reader - if (benchmark.current_event is None) or ( - not benchmark.current_event.need_record) or (reader_cost == 0): + if ( + (benchmark.current_event is None) + or (not benchmark.current_event.need_record) + or (reader_cost == 0) + ): return benchmark.current_event.record_reader(reader_cost) @@ -220,8 +231,9 @@ class TimerHook(Hook): """ - if (benchmark.current_event is - None) or (not benchmark.current_event.need_record): + if (benchmark.current_event is None) or ( + not benchmark.current_event.need_record + ): return batch_cost = timeit.default_timer() - self.start_time benchmark.current_event.record_batch(batch_cost, benchmark.num_samples) @@ -250,10 +262,20 @@ class TimerHook(Hook): print('Perf Summary'.center(100, '=')) if summary['reader_ratio'] != 0: print('Reader Ratio: ' + '%.3f' % (summary['reader_ratio']) + '%') - print('Time Unit: s, IPS Unit: %s' % - (benchmark.current_event.speed_unit)) - print('|', ''.center(15), '|', 'avg'.center(15), '|', 'max'.center(15), - '|', 'min'.center(15), '|') + print( + 'Time Unit: s, IPS Unit: %s' % (benchmark.current_event.speed_unit) + ) + print( + '|', + ''.center(15), + '|', + 'avg'.center(15), + '|', + 'max'.center(15), + '|', + 'min'.center(15), + '|', + ) # if DataLoader is not called, reader_summary is unnecessary. if summary['reader_summary']['avg'] != 0: self._print_stats('reader_cost', summary['reader_summary']) @@ -264,8 +286,17 @@ class TimerHook(Hook): avg_str = '%.5f' % (message_dict['avg']) max_str = '%.5f' % (message_dict['max']) min_str = '%.5f' % (message_dict['min']) - print('|', item.center(15), '|', avg_str.center(15), '|', - max_str.center(15), '|', min_str.center(15), '|') + print( + '|', + item.center(15), + '|', + avg_str.center(15), + '|', + max_str.center(15), + '|', + min_str.center(15), + '|', + ) class TimeAverager(object): @@ -357,8 +388,10 @@ class Benchmark(object): message += ' %s: %.5f s' % ('batch_cost', batch_average) speed_average = self.current_event.speed_average() if speed_average: - message += ' ips: %.3f %s' % (speed_average, - self.current_event.speed_unit) + message += ' ips: %.3f %s' % ( + speed_average, + self.current_event.speed_unit, + ) self.current_event.reset() return message @@ -389,8 +422,10 @@ class Benchmark(object): # set reader for the current event at the first iter if self.current_event.reader is None: self.current_event.reader = reader - elif self.current_event.reader.__dict__[ - '_dataset'] != reader.__dict__['_dataset']: + elif ( + self.current_event.reader.__dict__['_dataset'] + != reader.__dict__['_dataset'] + ): # enter a new task but not calling beign() to record it. # we pause the timer until the end of new task, so that # the cost of new task is not added to the current event. @@ -398,8 +433,10 @@ class Benchmark(object): self.current_event.need_record = False else: # when the new task exits, continue timing for the current event. - if self.current_event.reader.__dict__[ - '_dataset'] == reader.__dict__['_dataset']: + if ( + self.current_event.reader.__dict__['_dataset'] + == reader.__dict__['_dataset'] + ): self.current_event.need_record = True self.hooks['timer_hook'].start_time = timeit.default_timer() diff --git a/python/paddle/profiler/utils.py b/python/paddle/profiler/utils.py index fe05aaeb81f..efe3975f144 100644 --- a/python/paddle/profiler/utils.py +++ b/python/paddle/profiler/utils.py @@ -18,16 +18,19 @@ import functools from contextlib import ContextDecorator from paddle.fluid import core -from paddle.fluid.core import (_RecordEvent, TracerEventType) +from paddle.fluid.core import _RecordEvent, TracerEventType _is_profiler_used = False _has_optimizer_wrapped = False _AllowedEventTypeList = [ - TracerEventType.Dataloader, TracerEventType.ProfileStep, - TracerEventType.Forward, TracerEventType.Backward, - TracerEventType.Optimization, TracerEventType.PythonOp, - TracerEventType.PythonUserDefined + TracerEventType.Dataloader, + TracerEventType.ProfileStep, + TracerEventType.Forward, + TracerEventType.Backward, + TracerEventType.Optimization, + TracerEventType.PythonOp, + TracerEventType.PythonUserDefined, ] @@ -65,9 +68,10 @@ class RecordEvent(ContextDecorator): """ def __init__( - self, - name: str, - event_type: TracerEventType = TracerEventType.PythonUserDefined): + self, + name: str, + event_type: TracerEventType = TracerEventType.PythonUserDefined, + ): self.name = name self.event_type = event_type self.event = None @@ -100,8 +104,12 @@ class RecordEvent(ContextDecorator): if not _is_profiler_used: return if self.event_type not in _AllowedEventTypeList: - warn("Only TracerEvent Type in [{}, {}, {}, {}, {}, {},{}]\ - can be recorded.".format(*_AllowedEventTypeList)) + warn( + "Only TracerEvent Type in [{}, {}, {}, {}, {}, {},{}]\ + can be recorded.".format( + *_AllowedEventTypeList + ) + ) self.event = None else: self.event = _RecordEvent(self.name, self.event_type) @@ -160,14 +168,13 @@ def in_profiler_mode(): def wrap_optimizers(): - def optimizer_warpper(func): - @functools.wraps(func) def warpper(*args, **kwargs): if in_profiler_mode(): - with RecordEvent('Optimization Step', - event_type=TracerEventType.Optimization): + with RecordEvent( + 'Optimization Step', event_type=TracerEventType.Optimization + ): return func(*args, **kwargs) else: return func(*args, **kwargs) @@ -178,6 +185,7 @@ def wrap_optimizers(): if _has_optimizer_wrapped == True: return import paddle.optimizer as optimizer + for classname in optimizer.__all__: if classname != 'Optimizer': classobject = getattr(optimizer, classname) diff --git a/python/paddle/quantization/__init__.py b/python/paddle/quantization/__init__.py index 77da408d8e0..66d2354d480 100644 --- a/python/paddle/quantization/__init__.py +++ b/python/paddle/quantization/__init__.py @@ -12,14 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.contrib.slim.quantization.imperative.ptq_config import PTQConfig, default_ptq_config -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import BaseQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import AbsmaxQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import PerChannelAbsmaxQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import KLQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import HistQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import SUPPORT_ACT_QUANTIZERS -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import SUPPORT_WT_QUANTIZERS -from ...fluid.contrib.slim.quantization.imperative.ptq_registry import PTQRegistry +from ...fluid.contrib.slim.quantization.imperative.ptq_config import ( + PTQConfig, + default_ptq_config, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + BaseQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + AbsmaxQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + PerChannelAbsmaxQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + KLQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + HistQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + SUPPORT_ACT_QUANTIZERS, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + SUPPORT_WT_QUANTIZERS, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_registry import ( + PTQRegistry, +) from ...fluid.contrib.slim.quantization.imperative.ptq import ImperativePTQ -from ...fluid.contrib.slim.quantization.imperative.qat import ImperativeQuantAware +from ...fluid.contrib.slim.quantization.imperative.qat import ( + ImperativeQuantAware, +) diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index f7ba974fc0f..b2aa88cc810 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -277,7 +277,7 @@ def compose(*readers, **kwargs): if isinstance(x, tuple): return x else: - return (x, ) + return (x,) def reader(): rs = [] @@ -292,7 +292,8 @@ def compose(*readers, **kwargs): if o is None: # None will be not be present if compose is aligned raise ComposeNotAligned( - "outputs of readers are not aligned.") + "outputs of readers are not aligned." + ) yield sum(list(map(make_tuple, outputs)), ()) return reader @@ -330,7 +331,7 @@ def buffered(reader, size): print(i) """ - class EndSignal(): + class EndSignal: pass end = EndSignal() @@ -343,10 +344,13 @@ def buffered(reader, size): def data_reader(): r = reader() q = Queue(maxsize=size) - t = Thread(target=read_worker, args=( - r, - q, - )) + t = Thread( + target=read_worker, + args=( + r, + q, + ), + ) t.daemon = True t.start() e = q.get() @@ -398,7 +402,7 @@ def firstn(reader, n): return firstn_reader -class XmapEndSignal(): +class XmapEndSignal: pass @@ -470,8 +474,11 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False): t.start() # start several handle_workers target = order_handle_worker if order else handle_worker - args = (in_queue, out_queue, mapper, - out_order) if order else (in_queue, out_queue, mapper) + args = ( + (in_queue, out_queue, mapper, out_order) + if order + else (in_queue, out_queue, mapper) + ) workers = [] for i in range(process_num): worker = Thread(target=target, args=args) @@ -579,7 +586,8 @@ def multiprocess_reader(readers, use_pipe=True, queue_size=1000): if sys.platform == 'win32': raise NotImplementedError( - "The multiprocess_reader method is not supported on windows.") + "The multiprocess_reader method is not supported on windows." + ) # ujson is ultra fast json encoder and decoder written in pure C with bindings for Python 3.6+. try: @@ -587,11 +595,13 @@ def multiprocess_reader(readers, use_pipe=True, queue_size=1000): except Exception as e: warnings.warn( "The `ujson` module is not found, use the `json` module, `ujson` encodes and decodes faster, " - "you can install `ujson` through `pip install ujson`.") + "you can install `ujson` through `pip install ujson`." + ) import json - assert isinstance(readers, (list, tuple)) and len(readers) > 0, ( - "`readers` must be list or tuple.") + assert ( + isinstance(readers, (list, tuple)) and len(readers) > 0 + ), "`readers` must be list or tuple." def _read_into_queue(reader, queue): try: @@ -607,8 +617,9 @@ def multiprocess_reader(readers, use_pipe=True, queue_size=1000): def queue_reader(): queue = fork_context.Queue(queue_size) for reader in readers: - p = fork_context.Process(target=_read_into_queue, - args=(reader, queue)) + p = fork_context.Process( + target=_read_into_queue, args=(reader, queue) + ) p.start() reader_num = len(readers) @@ -649,8 +660,9 @@ def multiprocess_reader(readers, use_pipe=True, queue_size=1000): for reader in readers: parent_conn, child_conn = fork_context.Pipe() conns.append(parent_conn) - p = fork_context.Process(target=_read_into_pipe, - args=(reader, child_conn)) + p = fork_context.Process( + target=_read_into_pipe, args=(reader, child_conn) + ) p.start() reader_num = len(readers) diff --git a/python/paddle/reader/tests/decorator_test.py b/python/paddle/reader/tests/decorator_test.py index 902a8cbe06c..92d257e065a 100644 --- a/python/paddle/reader/tests/decorator_test.py +++ b/python/paddle/reader/tests/decorator_test.py @@ -23,7 +23,6 @@ __all__ = [] def reader_creator_10(dur): - def reader(): for i in range(10): # this invocation helps testing paddle.reader.buffer @@ -34,7 +33,6 @@ def reader_creator_10(dur): class TestMap(unittest.TestCase): - def test_map(self): d = {"h": 0, "i": 1} @@ -51,7 +49,6 @@ class TestMap(unittest.TestCase): class TestBuffered(unittest.TestCase): - def test_read(self): for size in range(20): b = paddle.reader.buffered(reader_creator_10(0), size) @@ -76,10 +73,10 @@ class TestBuffered(unittest.TestCase): class TestCompose(unittest.TestCase): - def test_compse(self): - reader = paddle.reader.compose(reader_creator_10(0), - reader_creator_10(0)) + reader = paddle.reader.compose( + reader_creator_10(0), reader_creator_10(0) + ) for idx, e in enumerate(reader()): self.assertEqual(e, (idx, idx)) @@ -87,7 +84,8 @@ class TestCompose(unittest.TestCase): total = 0 reader = paddle.reader.compose( paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)), - reader_creator_10(0)) + reader_creator_10(0), + ) with self.assertRaises(paddle.reader.ComposeNotAligned): for e in reader(): total += 1 @@ -96,10 +94,11 @@ class TestCompose(unittest.TestCase): def test_compose_not_aligned_no_check(self): total = 0 - reader = paddle.reader.compose(paddle.reader.chain( - reader_creator_10(0), reader_creator_10(0)), - reader_creator_10(0), - check_alignment=False) + reader = paddle.reader.compose( + paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)), + reader_creator_10(0), + check_alignment=False, + ) for e in reader(): total += 1 # expecting 10, not 20 @@ -107,7 +106,6 @@ class TestCompose(unittest.TestCase): class TestChain(unittest.TestCase): - def test_chain(self): c = paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)) idx = 0 @@ -118,7 +116,6 @@ class TestChain(unittest.TestCase): class TestShuffle(unittest.TestCase): - def test_shuffle(self): case = [(0, True), (1, True), (10, False), (100, False)] a = reader_creator_10(0) @@ -133,11 +130,9 @@ class TestShuffle(unittest.TestCase): class TestXmap(unittest.TestCase): - def test_xmap(self): - def mapper(x): - return (x + 1) + return x + 1 orders = (True, False) thread_nums = (1, 2, 4, 8, 16) @@ -145,9 +140,9 @@ class TestXmap(unittest.TestCase): for order in orders: for tNum in thread_nums: for size in buffered_size: - reader = paddle.reader.xmap_readers(mapper, - reader_creator_10(0), - tNum, size, order) + reader = paddle.reader.xmap_readers( + mapper, reader_creator_10(0), tNum, size, order + ) for n in range(3): result = [] for i in reader(): @@ -159,7 +154,6 @@ class TestXmap(unittest.TestCase): class TestMultiProcessReader(unittest.TestCase): - def setup(self): self.samples = [] for i in range(1000): @@ -178,7 +172,8 @@ class TestMultiProcessReader(unittest.TestCase): self.setup() results = [] for data in paddle.reader.multiprocess_reader( - [self.reader0, self.reader1, self.reader2], 100, use_pipe)(): + [self.reader0, self.reader1, self.reader2], 100, use_pipe + )(): results.append(data) self.assertEqual(sorted(self.samples), sorted(results)) diff --git a/python/paddle/signal.py b/python/paddle/signal.py index 535a388d0e0..82d46b81967 100644 --- a/python/paddle/signal.py +++ b/python/paddle/signal.py @@ -122,7 +122,8 @@ def frame(x, frame_length, hop_length, axis=-1, name=None): if frame_length > x.shape[axis]: raise ValueError( f'Attribute frame_length should be less equal than sequence length, ' - f'but got ({frame_length}) > ({x.shape[axis]}).') + f'but got ({frame_length}) > ({x.shape[axis]}).' + ) op_type = 'frame' @@ -130,25 +131,33 @@ def frame(x, frame_length, hop_length, axis=-1, name=None): return _C_ops.frame(x, frame_length, hop_length, axis) if _in_legacy_dygraph(): - attrs = ('frame_length', frame_length, 'hop_length', hop_length, 'axis', - axis) + attrs = ( + 'frame_length', + frame_length, + 'hop_length', + hop_length, + 'axis', + axis, + ) op = getattr(_legacy_C_ops, op_type) out = op(x, *attrs) else: check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], - op_type) + x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], op_type + ) helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type=op_type, - inputs={'X': x}, - attrs={ - 'frame_length': frame_length, - 'hop_length': hop_length, - 'axis': axis - }, - outputs={'Out': out}) + helper.append_op( + type=op_type, + inputs={'X': x}, + attrs={ + 'frame_length': frame_length, + 'hop_length': hop_length, + 'axis': axis, + }, + outputs={'Out': out}, + ) return out @@ -223,31 +232,32 @@ def overlap_add(x, hop_length, axis=-1, name=None): out = op(x, *attrs) else: check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], - op_type) + x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], op_type + ) helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type=op_type, - inputs={'X': x}, - attrs={ - 'hop_length': hop_length, - 'axis': axis - }, - outputs={'Out': out}) + helper.append_op( + type=op_type, + inputs={'X': x}, + attrs={'hop_length': hop_length, 'axis': axis}, + outputs={'Out': out}, + ) return out -def stft(x, - n_fft, - hop_length=None, - win_length=None, - window=None, - center=True, - pad_mode='reflect', - normalized=False, - onesided=True, - name=None): +def stft( + x, + n_fft, + hop_length=None, + win_length=None, + window=None, + center=True, + pad_mode='reflect', + normalized=False, + onesided=True, + name=None, +): r""" Short-time Fourier transform (STFT). @@ -310,13 +320,15 @@ def stft(x, paddle.randn([8, 48000], dtype=paddle.float64)*1j # [8, 48000] complex128 y1 = stft(x, n_fft=512, center=False, onesided=False) # [8, 512, 372] """ - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'complex64', 'complex128'], - 'stft') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'complex64', 'complex128'], 'stft' + ) x_rank = len(x.shape) - assert x_rank in [1, 2], \ - f'x should be a 1D or 2D real tensor, but got rank of x is {x_rank}' + assert x_rank in [ + 1, + 2, + ], f'x should be a 1D or 2D real tensor, but got rank of x is {x_rank}' if x_rank == 1: # (batch, seq_length) x = x.unsqueeze(0) @@ -324,69 +336,77 @@ def stft(x, if hop_length is None: hop_length = int(n_fft // 4) - assert hop_length > 0, \ - f'hop_length should be > 0, but got {hop_length}.' + assert hop_length > 0, f'hop_length should be > 0, but got {hop_length}.' if win_length is None: win_length = n_fft if _non_static_mode(): - assert 0 < n_fft <= x.shape[-1], \ - f'n_fft should be in (0, seq_length({x.shape[-1]})], but got {n_fft}.' + assert ( + 0 < n_fft <= x.shape[-1] + ), f'n_fft should be in (0, seq_length({x.shape[-1]})], but got {n_fft}.' - assert 0 < win_length <= n_fft, \ - f'win_length should be in (0, n_fft({n_fft})], but got {win_length}.' + assert ( + 0 < win_length <= n_fft + ), f'win_length should be in (0, n_fft({n_fft})], but got {win_length}.' if window is not None: - assert len(window.shape) == 1 and len(window) == win_length, \ - f'expected a 1D window tensor of size equal to win_length({win_length}), but got window with shape {window.shape}.' + assert ( + len(window.shape) == 1 and len(window) == win_length + ), f'expected a 1D window tensor of size equal to win_length({win_length}), but got window with shape {window.shape}.' else: - window = paddle.ones(shape=(win_length, ), dtype=x.dtype) + window = paddle.ones(shape=(win_length,), dtype=x.dtype) if win_length < n_fft: pad_left = (n_fft - win_length) // 2 pad_right = n_fft - win_length - pad_left - window = paddle.nn.functional.pad(window, - pad=[pad_left, pad_right], - mode='constant') + window = paddle.nn.functional.pad( + window, pad=[pad_left, pad_right], mode='constant' + ) if center: - assert pad_mode in ['constant', 'reflect'], \ - 'pad_mode should be "reflect" or "constant", but got "{}".'.format(pad_mode) + assert pad_mode in [ + 'constant', + 'reflect', + ], 'pad_mode should be "reflect" or "constant", but got "{}".'.format( + pad_mode + ) pad_length = n_fft // 2 # FIXME: Input `x` can be a complex tensor but pad does not supprt complex input. - x = paddle.nn.functional.pad(x.unsqueeze(-1), - pad=[pad_length, pad_length], - mode=pad_mode, - data_format="NLC").squeeze(-1) + x = paddle.nn.functional.pad( + x.unsqueeze(-1), + pad=[pad_length, pad_length], + mode=pad_mode, + data_format="NLC", + ).squeeze(-1) x_frames = frame(x=x, frame_length=n_fft, hop_length=hop_length, axis=-1) x_frames = x_frames.transpose( - perm=[0, 2, - 1]) # switch n_fft to last dim, egs: (batch, num_frames, n_fft) + perm=[0, 2, 1] + ) # switch n_fft to last dim, egs: (batch, num_frames, n_fft) x_frames = paddle.multiply(x_frames, window) norm = 'ortho' if normalized else 'backward' if is_complex(x_frames): - assert not onesided, \ - 'onesided should be False when input or window is a complex Tensor.' + assert ( + not onesided + ), 'onesided should be False when input or window is a complex Tensor.' if not is_complex(x): - out = fft_r2c(x=x_frames, - n=None, - axis=-1, - norm=norm, - forward=True, - onesided=onesided, - name=name) + out = fft_r2c( + x=x_frames, + n=None, + axis=-1, + norm=norm, + forward=True, + onesided=onesided, + name=name, + ) else: - out = fft_c2c(x=x_frames, - n=None, - axis=-1, - norm=norm, - forward=True, - name=name) + out = fft_c2c( + x=x_frames, n=None, axis=-1, norm=norm, forward=True, name=name + ) out = out.transpose(perm=[0, 2, 1]) # (batch, n_fft, num_frames) @@ -396,17 +416,19 @@ def stft(x, return out -def istft(x, - n_fft, - hop_length=None, - win_length=None, - window=None, - center=True, - normalized=False, - onesided=True, - length=None, - return_complex=False, - name=None): +def istft( + x, + n_fft, + hop_length=None, + win_length=None, + window=None, + center=True, + normalized=False, + onesided=True, + length=None, + return_complex=False, + name=None, +): r""" Inverse short-time Fourier transform (ISTFT). @@ -482,8 +504,12 @@ def istft(x, check_variable_and_dtype(x, 'x', ['complex64', 'complex128'], 'istft') x_rank = len(x.shape) - assert x_rank in [2, 3], \ - 'x should be a 2D or 3D complex tensor, but got rank of x is {}'.format(x_rank) + assert x_rank in [ + 2, + 3, + ], 'x should be a 2D or 3D complex tensor, but got rank of x is {}'.format( + x_rank + ) if x_rank == 2: # (batch, n_fft, n_frames) x = x.unsqueeze(0) @@ -495,83 +521,107 @@ def istft(x, win_length = n_fft # Assure no gaps between frames. - assert 0 < hop_length <= win_length, \ - 'hop_length should be in (0, win_length({})], but got {}.'.format(win_length, hop_length) - - assert 0 < win_length <= n_fft, \ - 'win_length should be in (0, n_fft({})], but got {}.'.format(n_fft, win_length) + assert ( + 0 < hop_length <= win_length + ), 'hop_length should be in (0, win_length({})], but got {}.'.format( + win_length, hop_length + ) + + assert ( + 0 < win_length <= n_fft + ), 'win_length should be in (0, n_fft({})], but got {}.'.format( + n_fft, win_length + ) n_frames = x.shape[-1] fft_size = x.shape[-2] if _non_static_mode(): if onesided: - assert (fft_size == n_fft // 2 + 1), \ - 'fft_size should be equal to n_fft // 2 + 1({}) when onesided is True, but got {}.'.format(n_fft // 2 + 1, fft_size) + assert ( + fft_size == n_fft // 2 + 1 + ), 'fft_size should be equal to n_fft // 2 + 1({}) when onesided is True, but got {}.'.format( + n_fft // 2 + 1, fft_size + ) else: - assert (fft_size == n_fft), \ - 'fft_size should be equal to n_fft({}) when onesided is False, but got {}.'.format(n_fft, fft_size) + assert ( + fft_size == n_fft + ), 'fft_size should be equal to n_fft({}) when onesided is False, but got {}.'.format( + n_fft, fft_size + ) if window is not None: - assert len(window.shape) == 1 and len(window) == win_length, \ - 'expected a 1D window tensor of size equal to win_length({}), but got window with shape {}.'.format(win_length, window.shape) + assert ( + len(window.shape) == 1 and len(window) == win_length + ), 'expected a 1D window tensor of size equal to win_length({}), but got window with shape {}.'.format( + win_length, window.shape + ) else: - window_dtype = paddle.float32 if x.dtype in [ - paddle.float32, paddle.complex64 - ] else paddle.float64 - window = paddle.ones(shape=(win_length, ), dtype=window_dtype) + window_dtype = ( + paddle.float32 + if x.dtype in [paddle.float32, paddle.complex64] + else paddle.float64 + ) + window = paddle.ones(shape=(win_length,), dtype=window_dtype) if win_length < n_fft: pad_left = (n_fft - win_length) // 2 pad_right = n_fft - win_length - pad_left # FIXME: Input `window` can be a complex tensor but pad does not supprt complex input. - window = paddle.nn.functional.pad(window, - pad=[pad_left, pad_right], - mode='constant') + window = paddle.nn.functional.pad( + window, pad=[pad_left, pad_right], mode='constant' + ) x = x.transpose( - perm=[0, 2, - 1]) # switch n_fft to last dim, egs: (batch, num_frames, n_fft) + perm=[0, 2, 1] + ) # switch n_fft to last dim, egs: (batch, num_frames, n_fft) norm = 'ortho' if normalized else 'backward' if return_complex: - assert not onesided, \ - 'onesided should be False when input(output of istft) or window is a complex Tensor.' + assert ( + not onesided + ), 'onesided should be False when input(output of istft) or window is a complex Tensor.' out = fft_c2c(x=x, n=None, axis=-1, norm=norm, forward=False, name=None) else: - assert not is_complex(window), \ - 'Data type of window should not be complex when return_complex is False.' + assert not is_complex( + window + ), 'Data type of window should not be complex when return_complex is False.' if onesided is False: - x = x[:, :, :n_fft // 2 + 1] + x = x[:, :, : n_fft // 2 + 1] out = fft_c2r(x=x, n=None, axis=-1, norm=norm, forward=False, name=None) out = paddle.multiply(out, window).transpose( - perm=[0, 2, 1]) # (batch, n_fft, num_frames) - out = overlap_add(x=out, hop_length=hop_length, - axis=-1) # (batch, seq_length) + perm=[0, 2, 1] + ) # (batch, n_fft, num_frames) + out = overlap_add( + x=out, hop_length=hop_length, axis=-1 + ) # (batch, seq_length) window_envelop = overlap_add( x=paddle.tile( x=paddle.multiply(window, window).unsqueeze(0), - repeat_times=[n_frames, - 1]).transpose(perm=[1, 0]), # (n_fft, num_frames) + repeat_times=[n_frames, 1], + ).transpose( + perm=[1, 0] + ), # (n_fft, num_frames) hop_length=hop_length, - axis=-1) # (seq_length, ) + axis=-1, + ) # (seq_length, ) if length is None: if center: - out = out[:, (n_fft // 2):-(n_fft // 2)] - window_envelop = window_envelop[(n_fft // 2):-(n_fft // 2)] + out = out[:, (n_fft // 2) : -(n_fft // 2)] + window_envelop = window_envelop[(n_fft // 2) : -(n_fft // 2)] else: if center: start = n_fft // 2 else: start = 0 - out = out[:, start:start + length] - window_envelop = window_envelop[start:start + length] + out = out[:, start : start + length] + window_envelop = window_envelop[start : start + length] # Check whether the Nonzero Overlap Add (NOLA) constraint is met. if _non_static_mode() and window_envelop.abs().min().item() < 1e-11: diff --git a/python/paddle/sparse/binary.py b/python/paddle/sparse/binary.py index 1e7fcfe7634..45c39e874a5 100644 --- a/python/paddle/sparse/binary.py +++ b/python/paddle/sparse/binary.py @@ -266,10 +266,9 @@ def add(x, y, name=None): inputs = {'x': x, 'y': y} helper = LayerHelper(op_type) out = helper.create_sparse_variable_for_type_inference(x.dtype) - helper.append_op(type=op_type, - inputs=inputs, - outputs={'out': out}, - attrs={}) + helper.append_op( + type=op_type, inputs=inputs, outputs={'out': out}, attrs={} + ) return out diff --git a/python/paddle/sparse/creation.py b/python/paddle/sparse/creation.py index 5ca050be08a..684d449af71 100644 --- a/python/paddle/sparse/creation.py +++ b/python/paddle/sparse/creation.py @@ -51,8 +51,8 @@ def _get_place(place): if place is None: place = _current_expected_place() elif not isinstance( - place, - (core.Place, core.CPUPlace, core.CUDAPinnedPlace, core.CUDAPlace)): + place, (core.Place, core.CPUPlace, core.CUDAPinnedPlace, core.CUDAPlace) + ): raise ValueError( "'place' must be any of paddle.Place, paddle.CPUPlace, paddle.CUDAPinnedPlace, paddle.CUDAPlace" ) @@ -66,12 +66,9 @@ def _check_indices_dtype(dtype): ) -def sparse_coo_tensor(indices, - values, - shape=None, - dtype=None, - place=None, - stop_gradient=True): +def sparse_coo_tensor( + indices, values, shape=None, dtype=None, place=None, stop_gradient=True +): r""" Constructs a sparse ``paddle.Tensor`` in coordinate format according to the indices and values of the specified non-zero elements. @@ -119,10 +116,9 @@ def sparse_coo_tensor(indices, place = _get_place(place) if not isinstance(indices, core.eager.Tensor): - indices = to_tensor(indices, - dtype=None, - place=place, - stop_gradient=True) + indices = to_tensor( + indices, dtype=None, place=place, stop_gradient=True + ) if not isinstance(values, core.eager.Tensor): values = to_tensor(values, dtype, place, stop_gradient) if len(indices.shape) != 2: @@ -135,8 +131,10 @@ def sparse_coo_tensor(indices, if nnz != values.shape[0]: raise ValueError( - "the indices and values must have same number of non-zero, but get {} and {}" - .format(nnz, values.shape[0])) + "the indices and values must have same number of non-zero, but get {} and {}".format( + nnz, values.shape[0] + ) + ) dense_dim = len(values.shape) - 1 @@ -156,11 +154,15 @@ def sparse_coo_tensor(indices, if shape < min_shape: raise ValueError( "the minimun shape required is {}, but get {}".format( - min_shape, shape)) + min_shape, shape + ) + ) if len(shape) != sparse_dim + dense_dim: raise ValueError( - "the number of dimensions(len(shape) must be sparse_dim({}) + dense_dim({}), but get {}" - .format(sparse_dim, dense_dim, len(shape))) + "the number of dimensions(len(shape) must be sparse_dim({}) + dense_dim({}), but get {}".format( + sparse_dim, dense_dim, len(shape) + ) + ) return _C_ops.sparse_sparse_coo_tensor(values, indices, shape) @@ -172,22 +174,17 @@ def sparse_coo_tensor(indices, attrs = {'dense_shape': shape} helper = LayerHelper(op_type) out = helper.create_sparse_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs=inputs, - outputs={'out': out}, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs={'out': out}, attrs=attrs + ) return out -#TODO: need to support shape is None +# TODO: need to support shape is None @dygraph_only -def sparse_csr_tensor(crows, - cols, - values, - shape, - dtype=None, - place=None, - stop_gradient=True): +def sparse_csr_tensor( + crows, cols, values, shape, dtype=None, place=None, stop_gradient=True +): r""" Constructs a sparse ``paddle.Tensor`` in CSR(Compressed Sparse Row) format according to the ``crows``, ``cols`` and ``values``. @@ -250,8 +247,10 @@ def sparse_csr_tensor(crows, if len(shape) != 2 and len(shape) != 3: raise ValueError( - "SparseCsrTensor only support 2-D or 3-D matrix. but get shape {}". - format(shape)) + "SparseCsrTensor only support 2-D or 3-D matrix. but get shape {}".format( + shape + ) + ) rows = shape[len(shape) - 2] if not crows.place._equals(place): @@ -268,26 +267,32 @@ def sparse_csr_tensor(crows, if len(crows.shape) != 1 or len(cols.shape) != 1 or len(values.shape) != 1: raise ValueError("The 'crows', 'cols' and 'values' must be 1-D.") - if (len(cols) != len(values)): + if len(cols) != len(values): raise ValueError("the length of cols must be same as length of values") if len(shape) == 2: if crows.shape[0] != rows + 1: raise ValueError( - "The length({}) of crows must be equal to the rows({})+1 of matrix." - .format(crows.shape[0], rows)) + "The length({}) of crows must be equal to the rows({})+1 of matrix.".format( + crows.shape[0], rows + ) + ) if crows[0] != 0: raise ValueError("the 0th value of crows must be 0") if crows[-1] != values.shape[0]: raise ValueError( - "the last value of crows must be equal the number of non-zero") + "the last value of crows must be equal the number of non-zero" + ) else: if crows.shape[0] % (rows + 1) != 0: raise ValueError( - "The length({}) of crows must be divisible the rows({})+1 of matrix." - .format(crows.shape[0], rows)) + "The length({}) of crows must be divisible the rows({})+1 of matrix.".format( + crows.shape[0], rows + ) + ) # TODO(zkh2016): check whether the value in crows and cols is legal - return core.eager.sparse_csr_tensor(crows, cols, values, shape, - stop_gradient) + return core.eager.sparse_csr_tensor( + crows, cols, values, shape, stop_gradient + ) diff --git a/python/paddle/sparse/nn/functional/activation.py b/python/paddle/sparse/nn/functional/activation.py index 516cfc5cd58..a50a64f5164 100644 --- a/python/paddle/sparse/nn/functional/activation.py +++ b/python/paddle/sparse/nn/functional/activation.py @@ -52,10 +52,9 @@ def relu(x, name=None): op_type = 'sparse_relu' helper = LayerHelper(op_type) out = helper.create_sparse_variable_for_type_inference(x.dtype) - helper.append_op(type=op_type, - inputs={'x': x}, - outputs={'out': out}, - attrs={}) + helper.append_op( + type=op_type, inputs={'x': x}, outputs={'out': out}, attrs={} + ) return out diff --git a/python/paddle/sparse/nn/functional/conv.py b/python/paddle/sparse/nn/functional/conv.py index 03b0f2e9be6..ec1907a7770 100644 --- a/python/paddle/sparse/nn/functional/conv.py +++ b/python/paddle/sparse/nn/functional/conv.py @@ -21,50 +21,66 @@ from paddle.nn.functional.conv import _update_padding_nd from paddle.fluid.layer_helper import LayerHelper -def _conv3d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - subm=False, - key=None, - data_format="NDHWC", - name=None): +def _conv3d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + subm=False, + key=None, + data_format="NDHWC", + name=None, +): assert groups == 1, "Currently, only support groups=1" dims = 3 # Currently, only support 'NDHWC' if data_format not in ["NDHWC"]: - raise ValueError("Attr(data_format) should be 'NDHWC'. Received " - "Attr(data_format): {}.".format(data_format)) + raise ValueError( + "Attr(data_format) should be 'NDHWC'. Received " + "Attr(data_format): {}.".format(data_format) + ) if len(x.shape) != 5: raise ValueError( - "Input x should be 5D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 5D tensor, but received x with the shape of {}".format( + x.shape + ) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 5: raise ValueError( - "Input x should be 5D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 5D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] if num_channels < 0: raise ValueError( "The channel dimension of the input({}) should be defined. " - "Received: {}.".format(x.shape, num_channels)) + "Received: {}.".format(x.shape, num_channels) + ) padding, padding_algorithm = _update_padding_nd(padding, channel_last, dims) stride = convert_to_list(stride, dims, 'stride') dilation = convert_to_list(dilation, dims, 'dilation') if in_dynamic_mode(): - pre_bias = _C_ops.sparse_conv3d(x, weight, padding, dilation, stride, - groups, subm, - key if key is not None else "") + pre_bias = _C_ops.sparse_conv3d( + x, + weight, + padding, + dilation, + stride, + groups, + subm, + key if key is not None else "", + ) if bias is not None: return add(pre_bias, bias) else: @@ -77,35 +93,38 @@ def _conv3d(x, 'strides': stride, 'groups': groups, 'subm': subm, - 'key': key + 'key': key, } op_type = 'sparse_conv3d' helper = LayerHelper(op_type, **locals()) - rulebook = helper.create_variable_for_type_inference(dtype='int32', - stop_gradient=True) - counter = helper.create_variable_for_type_inference(dtype='int32', - stop_gradient=True) + rulebook = helper.create_variable_for_type_inference( + dtype='int32', stop_gradient=True + ) + counter = helper.create_variable_for_type_inference( + dtype='int32', stop_gradient=True + ) pre_bias = helper.create_sparse_variable_for_type_inference(x.dtype) outputs = {"out": pre_bias, "rulebook": rulebook, "counter": counter} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: return add(pre_bias, bias) else: return pre_bias -def conv3d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NDHWC", - name=None): +def conv3d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NDHWC", + name=None, +): r""" The sparse convolution3d functional calculates the output based on the input, filter @@ -207,20 +226,33 @@ def conv3d(x, print(y.shape) # (1, 1, 1, 2, 1) """ - return _conv3d(x, weight, bias, stride, padding, dilation, groups, False, - None, data_format, name) - - -def subm_conv3d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NDHWC", - key=None, - name=None): + return _conv3d( + x, + weight, + bias, + stride, + padding, + dilation, + groups, + False, + None, + data_format, + name, + ) + + +def subm_conv3d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NDHWC", + key=None, + name=None, +): r""" The sparse submanifold convolution3d functional calculates the output based on the input, filter @@ -327,5 +359,16 @@ def subm_conv3d(x, print(y.shape) #(1, 1, 3, 4, 1) """ - return _conv3d(x, weight, bias, stride, padding, dilation, groups, True, - key, data_format, name) + return _conv3d( + x, + weight, + bias, + stride, + padding, + dilation, + groups, + True, + key, + data_format, + name, + ) diff --git a/python/paddle/sparse/nn/functional/pooling.py b/python/paddle/sparse/nn/functional/pooling.py index d86ac58e922..740324ea397 100644 --- a/python/paddle/sparse/nn/functional/pooling.py +++ b/python/paddle/sparse/nn/functional/pooling.py @@ -19,13 +19,15 @@ from paddle.nn.functional.pooling import _update_padding_nd __all__ = [] -def max_pool3d(x, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - data_format="NDHWC", - name=None): +def max_pool3d( + x, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + data_format="NDHWC", + name=None, +): """ Implements sparse max pooling 3d operation. See more details in :ref:`api_sparse_pooling_MaxPool3d` . @@ -75,9 +77,12 @@ def max_pool3d(x, """ assert in_dynamic_mode(), "Currently, Sparse API only support dynamic mode" - assert x.is_sparse_coo( + assert ( + x.is_sparse_coo() ), "Currently, sparse.relu only support the input of SparseCooTensor" - assert data_format == 'NDHWC', "Currently, sparse.max_pool3d only support data format of 'NDHWC'" + assert ( + data_format == 'NDHWC' + ), "Currently, sparse.max_pool3d only support data format of 'NDHWC'" kernel_size = utils.convert_to_list(kernel_size, 3, 'pool_size') if stride is None: @@ -87,12 +92,11 @@ def max_pool3d(x, channel_last = True - padding, padding_algorithm = _update_padding_nd(padding, - 3, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 3, channel_last=channel_last, ceil_mode=ceil_mode + ) - #TODO(zkh2016): remove the dependency on dilation from the backend + # TODO(zkh2016): remove the dependency on dilation from the backend dilation = [1, 1, 1] return _C_ops.sparse_maxpool(x, kernel_size, padding, dilation, stride) diff --git a/python/paddle/sparse/nn/functional/transformer.py b/python/paddle/sparse/nn/functional/transformer.py index f17d728078a..38118ba359b 100644 --- a/python/paddle/sparse/nn/functional/transformer.py +++ b/python/paddle/sparse/nn/functional/transformer.py @@ -19,14 +19,16 @@ from paddle.fluid.framework import dygraph_only @dygraph_only -def attention(query, - key, - value, - sparse_mask, - key_padding_mask=None, - attn_mask=None, - name=None): - """ +def attention( + query, + key, + value, + sparse_mask, + key_padding_mask=None, + attn_mask=None, + name=None, +): + r""" Note: This API is only used from ``CUDA 11.7`` . @@ -37,7 +39,7 @@ def attention(query, .. math:: - result = softmax(\\frac{ Q * K^T }{\\sqrt{d}}) * V + result = softmax(\frac{ Q * K^T }{\sqrt{d}}) * V where : ``Q``, ``K``, and ``V`` represent the three input parameters of the attention module. The shape of the three parameters are: `[batch_size, num_heads, seq_len, head_dim]`, and @@ -89,5 +91,6 @@ def attention(query, output = paddle.sparse.nn.functional.attention(query, key, value, sp_mask, kp_mask, attn_mask) output.backward() """ - return _C_ops.sparse_fused_attention(query, key, value, sparse_mask, - key_padding_mask, attn_mask) + return _C_ops.sparse_fused_attention( + query, key, value, sparse_mask, key_padding_mask, attn_mask + ) diff --git a/python/paddle/sparse/nn/layer/conv.py b/python/paddle/sparse/nn/layer/conv.py index 8368eeeb03c..599fb2d7b2e 100644 --- a/python/paddle/sparse/nn/layer/conv.py +++ b/python/paddle/sparse/nn/layer/conv.py @@ -23,23 +23,26 @@ __all__ = [] class _Conv3D(Layer): - - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - subm=False, - key=None, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NDHWC"): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + subm=False, + key=None, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NDHWC", + ): super(_Conv3D, self).__init__() - assert weight_attr is not False, "weight_attr should not be False in Conv." + assert ( + weight_attr is not False + ), "weight_attr should not be False in Conv." self._param_attr = weight_attr self._bias_attr = bias_attr self._groups = groups @@ -49,56 +52,66 @@ class _Conv3D(Layer): self._subm = subm self._key = key - assert padding_mode == 'zeros', "Currently, only support padding_mode='zeros'" + assert ( + padding_mode == 'zeros' + ), "Currently, only support padding_mode='zeros'" assert groups == 1, "Currently, only support groups=1" valid_format = {'NDHWC'} if data_format not in valid_format: raise ValueError( - "data_format must be one of {}, but got data_format='{}'". - format(valid_format, data_format)) + "data_format must be one of {}, but got data_format='{}'".format( + valid_format, data_format + ) + ) channel_last = data_format == "NDHWC" dims = 3 self._stride = utils.convert_to_list(stride, dims, 'stride') self._dilation = utils.convert_to_list(dilation, dims, 'dilation') - self._kernel_size = utils.convert_to_list(kernel_size, dims, - 'kernel_size') + self._kernel_size = utils.convert_to_list( + kernel_size, dims, 'kernel_size' + ) self._padding = padding self._padding_mode = padding_mode self._updated_padding, self._padding_algorithm = _update_padding_nd( - padding, channel_last, dims) + padding, channel_last, dims + ) # the sparse conv restricts the shape is [D, H, W, in_channels, out_channels] filter_shape = self._kernel_size + [ - self._in_channels, self._out_channels + self._in_channels, + self._out_channels, ] def _get_default_param_initializer(): filter_elem_num = np.prod(self._kernel_size) * self._in_channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std) self.weight = self.create_parameter( shape=filter_shape, attr=self._param_attr, - default_initializer=_get_default_param_initializer()) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._out_channels], - is_bias=True) + default_initializer=_get_default_param_initializer(), + ) + self.bias = self.create_parameter( + attr=self._bias_attr, shape=[self._out_channels], is_bias=True + ) def forward(self, x): - out = F.conv._conv3d(x, - self.weight, - bias=self.bias, - stride=self._stride, - padding=self._updated_padding, - dilation=self._dilation, - groups=self._groups, - subm=self._subm, - key=self._key, - data_format=self._data_format) + out = F.conv._conv3d( + x, + self.weight, + bias=self.bias, + stride=self._stride, + padding=self._updated_padding, + dilation=self._dilation, + groups=self._groups, + subm=self._subm, + key=self._key, + data_format=self._data_format, + ) return out def extra_repr(self): @@ -223,31 +236,35 @@ class Conv3D(_Conv3D): # (1, 1, 1, 2, 1) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NDHWC"): - super(Conv3D, self).__init__(in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation, - groups=groups, - subm=False, - key=None, - padding_mode=padding_mode, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NDHWC", + ): + super(Conv3D, self).__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + subm=False, + key=None, + padding_mode=padding_mode, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) class SubmConv3D(_Conv3D): @@ -360,29 +377,33 @@ class SubmConv3D(_Conv3D): # (1, 1, 3, 4, 1) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - key=None, - weight_attr=None, - bias_attr=None, - data_format="NDHWC"): - super(SubmConv3D, self).__init__(in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation, - groups=groups, - subm=True, - key=key, - padding_mode=padding_mode, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + key=None, + weight_attr=None, + bias_attr=None, + data_format="NDHWC", + ): + super(SubmConv3D, self).__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + subm=True, + key=key, + padding_mode=padding_mode, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) diff --git a/python/paddle/sparse/nn/layer/norm.py b/python/paddle/sparse/nn/layer/norm.py index b36a85f6b78..99b96fb530a 100644 --- a/python/paddle/sparse/nn/layer/norm.py +++ b/python/paddle/sparse/nn/layer/norm.py @@ -97,23 +97,27 @@ class BatchNorm(paddle.nn.BatchNorm1D): # [1, 6, 6, 6, 3] """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NDHWC', - use_global_stats=None, - name=None): - super(BatchNorm, self).__init__(num_features, - momentum=momentum, - epsilon=epsilon, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format, - use_global_stats=use_global_stats, - name=name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NDHWC', + use_global_stats=None, + name=None, + ): + super(BatchNorm, self).__init__( + num_features, + momentum=momentum, + epsilon=epsilon, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + use_global_stats=use_global_stats, + name=name, + ) def _check_data_format(self, input): if input != "NDHWC": @@ -124,7 +128,8 @@ class BatchNorm(paddle.nn.BatchNorm1D): if self.training: warnings.warn( - "When training, we now always track global mean and variance.") + "When training, we now always track global mean and variance." + ) if self._use_global_stats == None: self._use_global_stats = not self.training @@ -136,9 +141,19 @@ class BatchNorm(paddle.nn.BatchNorm1D): if in_dynamic_mode(): batch_norm_out, _, _, _, _, _ = _C_ops.sparse_batch_norm( - input, self.weight, self.bias, self._mean, self._variance, - self._momentum, self._epsilon, data_format, not self.training, - self._use_global_stats, trainable_statistics, False) + input, + self.weight, + self.bias, + self._mean, + self._variance, + self._momentum, + self._epsilon, + data_format, + not self.training, + self._use_global_stats, + trainable_statistics, + False, + ) return batch_norm_out else: inputs = { @@ -146,7 +161,7 @@ class BatchNorm(paddle.nn.BatchNorm1D): 'scale': self.weight, 'bias': self.bias, 'mean': self._mean, - 'variance': self._variance + 'variance': self._variance, } attrs = { 'momentum': self._momentum, @@ -155,21 +170,26 @@ class BatchNorm(paddle.nn.BatchNorm1D): 'is_test': not self.training, 'use_global_stats': self._use_global_stats, 'trainable_statistics': trainable_statistics, - 'fuse_with_relu': False + 'fuse_with_relu': False, } op_type = 'sparse_batch_norm' helper = LayerHelper(op_type) dtype = input.dtype mean_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) variance_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) saved_mean = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) reserve_space = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) out = helper.create_sparse_variable_for_type_inference(dtype) outputs = { "out": out, @@ -177,12 +197,11 @@ class BatchNorm(paddle.nn.BatchNorm1D): "variance_out": variance_out, "saved_mean": saved_mean, "saved_variance": saved_variance, - "reserve_space": reserve_space + "reserve_space": reserve_space, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -283,24 +302,42 @@ class SyncBatchNorm(paddle.nn.SyncBatchNorm): # [-0.88415730, 1.57439375]]) """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCHW', - name=None): - super(SyncBatchNorm, - self).__init__(num_features, momentum, epsilon, weight_attr, - bias_attr, data_format, name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCHW', + name=None, + ): + super(SyncBatchNorm, self).__init__( + num_features, + momentum, + epsilon, + weight_attr, + bias_attr, + data_format, + name, + ) def forward(self, x): self._check_data_format() sync_batch_norm_out, _, _, _, _, _ = _C_ops.sparse_sync_batch_norm_( - x, self.weight, self.bias, self._mean, self._variance, - self._momentum, self._epsilon, self._data_format, not self.training, - False, False, False) + x, + self.weight, + self.bias, + self._mean, + self._variance, + self._momentum, + self._epsilon, + self._data_format, + not self.training, + False, + False, + False, + ) return sync_batch_norm_out @classmethod @@ -327,27 +364,41 @@ class SyncBatchNorm(paddle.nn.SyncBatchNorm): layer_output = layer if isinstance(layer, _BatchNormBase): - if layer._weight_attr != None and not isinstance( - layer._weight_attr, - bool) and layer._weight_attr.name != None: + if ( + layer._weight_attr != None + and not isinstance(layer._weight_attr, bool) + and layer._weight_attr.name != None + ): layer._weight_attr.name = layer._weight_attr.name + '_sync' - if layer._bias_attr != None and not isinstance( - layer._bias_attr, bool) and layer._bias_attr.name != None: + if ( + layer._bias_attr != None + and not isinstance(layer._bias_attr, bool) + and layer._bias_attr.name != None + ): layer._bias_attr.name = layer._bias_attr.name + '_sync' - #convert sparse BatchNorm + # convert sparse BatchNorm if isinstance(layer, BatchNorm): - layer_output = SyncBatchNorm(layer._num_features, - layer._momentum, layer._epsilon, - layer._weight_attr, - layer._bias_attr, - layer._data_format, layer._name) - #convert dense BatchNorm + layer_output = SyncBatchNorm( + layer._num_features, + layer._momentum, + layer._epsilon, + layer._weight_attr, + layer._bias_attr, + layer._data_format, + layer._name, + ) + # convert dense BatchNorm else: layer_output = paddle.nn.SyncBatchNorm( - layer._num_features, layer._momentum, layer._epsilon, - layer._weight_attr, layer._bias_attr, layer._data_format, - layer._name) + layer._num_features, + layer._momentum, + layer._epsilon, + layer._weight_attr, + layer._bias_attr, + layer._data_format, + layer._name, + ) if layer._weight_attr != False and layer._bias_attr != False: with no_grad(): @@ -357,7 +408,8 @@ class SyncBatchNorm(paddle.nn.SyncBatchNorm): layer_output._variance = layer._variance for name, sublayer in layer.named_children(): - layer_output.add_sublayer(name, - cls.convert_sync_batchnorm(sublayer)) + layer_output.add_sublayer( + name, cls.convert_sync_batchnorm(sublayer) + ) del layer return layer_output diff --git a/python/paddle/sparse/nn/layer/pooling.py b/python/paddle/sparse/nn/layer/pooling.py index ee15bf6f793..7d6141bab0f 100644 --- a/python/paddle/sparse/nn/layer/pooling.py +++ b/python/paddle/sparse/nn/layer/pooling.py @@ -73,14 +73,16 @@ class MaxPool3D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NDHWC", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NDHWC", + name=None, + ): super(MaxPool3D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -91,14 +93,17 @@ class MaxPool3D(Layer): self.name = name def forward(self, x): - return F.max_pool3d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - ceil_mode=self.ceil_mode, - data_format=self.data_format, - name=self.name) + return F.max_pool3d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + ceil_mode=self.ceil_mode, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) diff --git a/python/paddle/sparse/unary.py b/python/paddle/sparse/unary.py index c26ec35d3b7..14a97d6cb03 100644 --- a/python/paddle/sparse/unary.py +++ b/python/paddle/sparse/unary.py @@ -15,7 +15,11 @@ import numpy as np from paddle import _C_ops -from paddle.fluid.framework import dygraph_only, core, convert_np_dtype_to_dtype_ +from paddle.fluid.framework import ( + dygraph_only, + core, + convert_np_dtype_to_dtype_, +) __all__ = [] diff --git a/python/paddle/static/__init__.py b/python/paddle/static/__init__.py index 44ab1336061..78ad1cfabc3 100644 --- a/python/paddle/static/__init__.py +++ b/python/paddle/static/__init__.py @@ -73,18 +73,55 @@ from ..fluid.layers.metric_op import accuracy # noqa: F401 from ..fluid.contrib.layers import ctr_metric_bundle # noqa: F401 from ..fluid.layers import exponential_decay # noqa: F401 -__all__ = [ #noqa - 'append_backward', 'gradients', 'Executor', 'global_scope', 'scope_guard', - 'BuildStrategy', 'CompiledProgram', 'ipu_shard_guard', 'IpuCompiledProgram', - 'IpuStrategy', 'Print', 'py_func', 'ExecutionStrategy', 'name_scope', - 'ParallelExecutor', 'program_guard', 'WeightNormParamAttr', - 'ExponentialMovingAverage', 'default_main_program', - 'default_startup_program', 'Program', 'data', 'InputSpec', 'save', 'load', - 'save_inference_model', 'load_inference_model', 'serialize_program', - 'serialize_persistables', 'save_to_file', 'deserialize_program', - 'deserialize_persistables', 'load_from_file', 'normalize_program', - 'load_program_state', 'set_program_state', 'cpu_places', 'cuda_places', - 'xpu_places', 'npu_places', 'mlu_places', 'Variable', 'create_global_var', - 'accuracy', 'auc', 'device_guard', 'create_parameter', 'set_ipu_shard', - 'ctr_metric_bundle', 'exponential_decay' +__all__ = [ # noqa + 'append_backward', + 'gradients', + 'Executor', + 'global_scope', + 'scope_guard', + 'BuildStrategy', + 'CompiledProgram', + 'ipu_shard_guard', + 'IpuCompiledProgram', + 'IpuStrategy', + 'Print', + 'py_func', + 'ExecutionStrategy', + 'name_scope', + 'ParallelExecutor', + 'program_guard', + 'WeightNormParamAttr', + 'ExponentialMovingAverage', + 'default_main_program', + 'default_startup_program', + 'Program', + 'data', + 'InputSpec', + 'save', + 'load', + 'save_inference_model', + 'load_inference_model', + 'serialize_program', + 'serialize_persistables', + 'save_to_file', + 'deserialize_program', + 'deserialize_persistables', + 'load_from_file', + 'normalize_program', + 'load_program_state', + 'set_program_state', + 'cpu_places', + 'cuda_places', + 'xpu_places', + 'npu_places', + 'mlu_places', + 'Variable', + 'create_global_var', + 'accuracy', + 'auc', + 'device_guard', + 'create_parameter', + 'set_ipu_shard', + 'ctr_metric_bundle', + 'exponential_decay', ] diff --git a/python/paddle/static/amp/__init__.py b/python/paddle/static/amp/__init__.py index 8ee3225057d..ffcfa404cd3 100644 --- a/python/paddle/static/amp/__init__.py +++ b/python/paddle/static/amp/__init__.py @@ -14,8 +14,12 @@ from ...fluid.contrib.mixed_precision import decorate # noqa: F401 from ...fluid.contrib.mixed_precision import CustomOpLists # noqa: F401 -from ...fluid.contrib.mixed_precision import AutoMixedPrecisionLists # noqa: F401 +from ...fluid.contrib.mixed_precision import ( + AutoMixedPrecisionLists, +) # noqa: F401 from ...fluid.contrib.mixed_precision import fp16_guard # noqa: F401 from ...fluid.contrib.mixed_precision import cast_model_to_fp16 # noqa: F401 -from ...fluid.contrib.mixed_precision import cast_parameters_to_fp16 # noqa: F401 +from ...fluid.contrib.mixed_precision import ( + cast_parameters_to_fp16, +) # noqa: F401 from ...fluid.contrib.mixed_precision import bf16 # noqa: F401 diff --git a/python/paddle/static/input.py b/python/paddle/static/input.py index 6cd57007641..8a398b42ded 100644 --- a/python/paddle/static/input.py +++ b/python/paddle/static/input.py @@ -105,7 +105,8 @@ def data(name, shape, dtype=None, lod_level=0): stop_gradient=True, lod_level=lod_level, is_data=True, - need_check_feed=True) + need_check_feed=True, + ) else: return helper.create_global_variable( name=name, @@ -115,7 +116,8 @@ def data(name, shape, dtype=None, lod_level=0): stop_gradient=True, lod_level=lod_level, is_data=True, - need_check_feed=True) + need_check_feed=True, + ) class InputSpec(object): @@ -164,7 +166,8 @@ class InputSpec(object): def __repr__(self): return '{}(shape={}, dtype={}, name={})'.format( - type(self).__name__, self.shape, self.dtype, self.name) + type(self).__name__, self.shape, self.dtype, self.name + ) @classmethod def from_tensor(cls, tensor, name=None): @@ -196,7 +199,9 @@ class InputSpec(object): else: raise ValueError( "Input `tensor` should be a Tensor, but received {}.".format( - type(tensor).__name__)) + type(tensor).__name__ + ) + ) @classmethod def from_numpy(cls, ndarray, name=None): @@ -245,13 +250,17 @@ class InputSpec(object): if isinstance(batch_size, (list, tuple)): if len(batch_size) != 1: raise ValueError( - "Length of batch_size: {} shall be 1, but received {}.". - format(batch_size, len(batch_size))) + "Length of batch_size: {} shall be 1, but received {}.".format( + batch_size, len(batch_size) + ) + ) batch_size = batch_size[1] elif not isinstance(batch_size, int): raise TypeError( "type(batch_size) shall be `int`, but received {}.".format( - type(batch_size).__name__)) + type(batch_size).__name__ + ) + ) new_shape = [batch_size] + list(self.shape) self.shape = tuple(new_shape) @@ -277,7 +286,8 @@ class InputSpec(object): """ if len(self.shape) == 0: raise ValueError( - "Not support to unbatch a InputSpec when len(shape) == 0.") + "Not support to unbatch a InputSpec when len(shape) == 0." + ) self.shape = self._verify(self.shape[1:]) return self @@ -288,20 +298,25 @@ class InputSpec(object): """ if not isinstance(shape, (list, tuple)): raise TypeError( - "Type of `shape` in InputSpec should be one of (tuple, list), but received {}." - .format(type(shape).__name__)) + "Type of `shape` in InputSpec should be one of (tuple, list), but received {}.".format( + type(shape).__name__ + ) + ) if len(shape) == 0: raise ValueError( - "`shape` in InputSpec should contain at least 1 element, but received {}." - .format(shape)) + "`shape` in InputSpec should contain at least 1 element, but received {}.".format( + shape + ) + ) for i, ele in enumerate(shape): if ele is not None: if not isinstance(ele, int): raise ValueError( - "shape[{}] should be an `int`, but received `{}`:{}.". - format(i, - type(ele).__name__, ele)) + "shape[{}] should be an `int`, but received `{}`:{}.".format( + i, type(ele).__name__, ele + ) + ) if ele is None or ele < -1: shape[i] = -1 @@ -326,8 +341,9 @@ class InputSpec(object): def __eq__(self, other): slots = ['shape', 'dtype', 'name'] - return (type(self) is type(other) and all( - getattr(self, attr) == getattr(other, attr) for attr in slots)) + return type(self) is type(other) and all( + getattr(self, attr) == getattr(other, attr) for attr in slots + ) def __ne__(self, other): return not self == other diff --git a/python/paddle/static/io.py b/python/paddle/static/io.py index 279fc60a669..ac598328352 100644 --- a/python/paddle/static/io.py +++ b/python/paddle/static/io.py @@ -37,9 +37,9 @@ from paddle.fluid.log_helper import get_logger __all__ = [] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def _check_args(caller, args, supported_args=None, deprecated_args=None): @@ -48,12 +48,16 @@ def _check_args(caller, args, supported_args=None, deprecated_args=None): for arg in args: if arg in deprecated_args: raise ValueError( - "argument '{}' in function '{}' is deprecated, only {} are supported." - .format(arg, caller, supported_args)) + "argument '{}' in function '{}' is deprecated, only {} are supported.".format( + arg, caller, supported_args + ) + ) elif arg not in supported_args: raise ValueError( - "function '{}' doesn't support argument '{}',\n only {} are supported." - .format(caller, arg, supported_args)) + "function '{}' doesn't support argument '{}',\n only {} are supported.".format( + caller, arg, supported_args + ) + ) def _check_vars(name, var_list): @@ -61,7 +65,8 @@ def _check_vars(name, var_list): var_list = [var_list] if not var_list or not all([isinstance(var, Variable) for var in var_list]): raise ValueError( - "'{}' should be a Variable or a list of Variable.".format(name)) + "'{}' should be a Variable or a list of Variable.".format(name) + ) def _normalize_path_prefix(path_prefix): @@ -90,29 +95,35 @@ def _get_valid_program(program=None): "The type of input program is invalid, expected tyep is Program, but received None" ) warnings.warn( - "The input is a CompiledProgram, this is not recommended.") + "The input is a CompiledProgram, this is not recommended." + ) if not isinstance(program, Program): raise TypeError( "The type of input program is invalid, expected type is fluid.Program, but received %s" - % type(program)) + % type(program) + ) return program def _clone_var_in_block(block, var): assert isinstance(var, Variable) if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR: - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True, + ) else: - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + persistable=True, + ) def normalize_program(program, feed_vars, fetch_vars): @@ -155,18 +166,21 @@ def normalize_program(program, feed_vars, fetch_vars): """ if not isinstance(program, Program): raise TypeError( - "program type must be `fluid.Program`, but received `%s`" % - type(program)) + "program type must be `fluid.Program`, but received `%s`" + % type(program) + ) if not isinstance(feed_vars, list): feed_vars = [feed_vars] if not all(isinstance(v, Variable) for v in feed_vars): raise TypeError( - "feed_vars type must be a Variable or a list of Variable.") + "feed_vars type must be a Variable or a list of Variable." + ) if not isinstance(fetch_vars, list): fetch_vars = [fetch_vars] if not all(isinstance(v, Variable) for v in fetch_vars): raise TypeError( - "fetch_vars type must be a Variable or a list of Variable.") + "fetch_vars type must be a Variable or a list of Variable." + ) # remind users to set auc_states to 0 if auc op were found. for op in program.global_block().ops: @@ -174,8 +188,10 @@ def normalize_program(program, feed_vars, fetch_vars): device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName() op._set_attr(device_attr_name, "") if op.type == 'auc': - warnings.warn("Be sure that you have set auc states to 0 " - "before saving inference model.") + warnings.warn( + "Be sure that you have set auc states to 0 " + "before saving inference model." + ) break # fix the bug that the activation op's output as target will be pruned. @@ -185,9 +201,9 @@ def normalize_program(program, feed_vars, fetch_vars): uniq_fetch_vars = [] for i, var in enumerate(fetch_vars): if var.dtype != paddle.bool: - var = layers.scale(var, - 1., - name="save_infer_model/scale_{}".format(i)) + var = layers.scale( + var, 1.0, name="save_infer_model/scale_{}".format(i) + ) uniq_fetch_vars.append(var) fetch_vars = uniq_fetch_vars @@ -205,7 +221,8 @@ def normalize_program(program, feed_vars, fetch_vars): feed_var_names = [var.name for var in feed_vars] copy_program = copy_program._prune_with_input( - feeded_var_names=feed_var_names, targets=fetch_vars) + feeded_var_names=feed_var_names, targets=fetch_vars + ) copy_program = copy_program._inference_optimize(prune_read_op=True) fetch_var_names = [var.name for var in fetch_vars] prepend_feed_ops(copy_program, feed_var_names) @@ -235,9 +252,11 @@ def is_persistable(var): param = fluid.default_main_program().global_block().var('fc.b') res = fluid.io.is_persistable(param) """ - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -361,8 +380,10 @@ def _serialize_persistables(program, executor): vars_ = list(filter(is_persistable, program.list_vars())) # warn if no variable found in model if len(vars_) == 0: - warnings.warn("no variable in your model, please ensure there are any " - "variables in your model to save") + warnings.warn( + "no variable in your model, please ensure there are any " + "variables in your model to save" + ) return None # create a new program and clone persitable vars to it save_program = Program() @@ -379,16 +400,16 @@ def _serialize_persistables(program, executor): in_vars.append(save_var_map[name]) out_var_name = unique_name.generate("out_var") - out_var = save_block.create_var(type=core.VarDesc.VarType.RAW, - name=out_var_name) + out_var = save_block.create_var( + type=core.VarDesc.VarType.RAW, name=out_var_name + ) out_var.desc.set_persistable(True) - save_block.append_op(type='save_combine', - inputs={'X': in_vars}, - outputs={'Y': out_var}, - attrs={ - 'file_path': '', - 'save_to_memory': True - }) + save_block.append_op( + type='save_combine', + inputs={'X': in_vars}, + outputs={'Y': out_var}, + attrs={'file_path': '', 'save_to_memory': True}, + ) # run save_program to save vars # NOTE(zhiqiu): save op will add variable kLookupTablePath to save_program.desc, # which leads to diff between save_program and its desc. Call _sync_with_cpp @@ -435,8 +456,9 @@ def save_to_file(path, content): @static_only -def save_inference_model(path_prefix, feed_vars, fetch_vars, executor, - **kwargs): +def save_inference_model( + path_prefix, feed_vars, fetch_vars, executor, **kwargs +): """ Save current model and its parameters to given path. i.e. Given path_prefix = "/path/to/modelname", after invoking @@ -516,7 +538,8 @@ def save_inference_model(path_prefix, feed_vars, fetch_vars, executor, program = normalize_program(program, feed_vars, fetch_vars) # serialize and save program program_bytes = _serialize_program( - program._remove_training_info(clip_extra=clip_extra)) + program._remove_training_info(clip_extra=clip_extra) + ) save_to_file(model_path, program_bytes) # serialize and save params params_bytes = _serialize_persistables(program, executor) @@ -566,8 +589,9 @@ def deserialize_program(data): """ program = Program.parse_from_string(data) if not core._is_program_version_supported(program._version()): - raise ValueError("Unsupported program version: %d\n" % - program._version()) + raise ValueError( + "Unsupported program version: %d\n" % program._version() + ) return program @@ -616,8 +640,9 @@ def deserialize_persistables(program, data, executor): """ if not isinstance(program, Program): raise TypeError( - "program type must be `fluid.Program`, but received `%s`" % - type(program)) + "program type must be `fluid.Program`, but received `%s`" + % type(program) + ) # load params to a tmp program load_program = Program() load_block = load_program.global_block() @@ -641,9 +666,9 @@ def deserialize_persistables(program, data, executor): load_var_map[var_copy.name] = var_copy if data is None: - assert len( - origin_shape_map - ) == 0, "Required 'data' shall be not None if program contains parameter, but received 'data' is None." + assert ( + len(origin_shape_map) == 0 + ), "Required 'data' shall be not None if program contains parameter, but received 'data' is None." return # append load_combine op to load parameters, @@ -655,10 +680,8 @@ def deserialize_persistables(program, data, executor): inputs={}, outputs={"Out": load_var_list}, # if load from memory, file_path is data - attrs={ - 'file_path': data, - 'model_from_memory': True - }) + attrs={'file_path': data, 'model_from_memory': True}, + ) executor.run(load_program) # check var shape for var in check_vars: @@ -673,7 +696,9 @@ def deserialize_persistables(program, data, executor): raise RuntimeError( "Shape mismatch, program needs a parameter with shape ({}), " "but the loaded parameter ('{}') has a shape of ({}).".format( - origin_shape, var.name, new_shape)) + origin_shape, var.name, new_shape + ) + ) def load_from_file(path): @@ -778,7 +803,7 @@ def load_inference_model(path_prefix, executor, **kwargs): """ # check kwargs supported_args = ('model_filename', 'params_filename') - deprecated_args = ('pserver_endpoints', ) + deprecated_args = ('pserver_endpoints',) caller = inspect.currentframe().f_code.co_name _check_args(caller, kwargs, supported_args, deprecated_args) @@ -789,7 +814,8 @@ def load_inference_model(path_prefix, executor, **kwargs): params_filename = kwargs.get('params_filename', None) if params_filename is None: raise ValueError( - "params_filename cannot be None when path_prefix is None.") + "params_filename cannot be None when path_prefix is None." + ) load_dirname = '' program_bytes = model_filename params_bytes = params_filename @@ -812,21 +838,26 @@ def load_inference_model(path_prefix, executor, **kwargs): if model_filename is None: model_path = os.path.join(path_prefix, "__model__") else: - model_path = os.path.join(path_prefix, - model_filename + ".pdmodel") + model_path = os.path.join( + path_prefix, model_filename + ".pdmodel" + ) if not os.path.exists(model_path): model_path = os.path.join(path_prefix, model_filename) # set params_path if params_filename is None: params_path = os.path.join(path_prefix, "") else: - params_path = os.path.join(path_prefix, - params_filename + ".pdiparams") + params_path = os.path.join( + path_prefix, params_filename + ".pdiparams" + ) if not os.path.exists(params_path): params_path = os.path.join(path_prefix, params_filename) - _logger.warning("The old way to load inference model is deprecated." - " model path: {}, params path: {}".format( - model_path, params_path)) + _logger.warning( + "The old way to load inference model is deprecated." + " model path: {}, params path: {}".format( + model_path, params_path + ) + ) program_bytes = load_from_file(model_path) load_dirname = os.path.dirname(params_path) params_filename = os.path.basename(params_path) diff --git a/python/paddle/static/nn/__init__.py b/python/paddle/static/nn/__init__.py index 41f8c99b526..64e079ae375 100755 --- a/python/paddle/static/nn/__init__.py +++ b/python/paddle/static/nn/__init__.py @@ -59,7 +59,7 @@ from ...fluid.layers.sequence_lod import sequence_scatter # noqa: F401 from ...fluid.layers.sequence_lod import sequence_enumerate # noqa: F401 from ...fluid.layers.sequence_lod import sequence_reverse # noqa: F401 -__all__ = [ #noqa +__all__ = [ # noqa 'fc', 'batch_norm', 'embedding', diff --git a/python/paddle/static/nn/common.py b/python/paddle/static/nn/common.py index baeb39e8cf6..98c5e81a2e8 100755 --- a/python/paddle/static/nn/common.py +++ b/python/paddle/static/nn/common.py @@ -19,13 +19,15 @@ __all__ = [] @static_only -def fc(x, - size, - num_flatten_dims=1, - weight_attr=None, - bias_attr=None, - activation=None, - name=None): +def fc( + x, + size, + num_flatten_dims=1, + weight_attr=None, + bias_attr=None, + activation=None, + name=None, +): r""" Fully-Connected layer can take a tensor or a list of tensor as its inputs. @@ -154,30 +156,34 @@ def fc(x, bias_attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Constant(value=1.0))) # out: [[1.8 1.8]] """ - return paddle.fluid.layers.fc(input=x, - size=size, - num_flatten_dims=num_flatten_dims, - param_attr=weight_attr, - bias_attr=bias_attr, - act=activation, - name=name) + return paddle.fluid.layers.fc( + input=x, + size=size, + num_flatten_dims=num_flatten_dims, + param_attr=weight_attr, + bias_attr=bias_attr, + act=activation, + name=name, + ) @static_only -def deform_conv2d(x, - offset, - mask, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=1, - deformable_groups=1, - im2col_step=1, - weight_attr=None, - bias_attr=None, - name=None): +def deform_conv2d( + x, + offset, + mask, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + im2col_step=1, + weight_attr=None, + bias_attr=None, + name=None, +): r""" Compute 2-D deformable convolution on 4-D input. @@ -318,7 +324,8 @@ def deform_conv2d(x, param_attr=weight_attr, bias_attr=bias_attr, modulated=False, - name=name) + name=name, + ) else: return paddle.fluid.layers.deformable_conv( input=x, @@ -335,4 +342,5 @@ def deform_conv2d(x, param_attr=weight_attr, bias_attr=bias_attr, modulated=True, - name=name) + name=name, + ) diff --git a/python/paddle/static/quantization/__init__.py b/python/paddle/static/quantization/__init__.py index 325fee5624b..3a1a7549f83 100644 --- a/python/paddle/static/quantization/__init__.py +++ b/python/paddle/static/quantization/__init__.py @@ -12,20 +12,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.contrib.slim.quantization.quantization_pass import QuantizationTransformPass -from ...fluid.contrib.slim.quantization.quantization_pass import QuantizationFreezePass -from ...fluid.contrib.slim.quantization.quantization_pass import ConvertToInt8Pass -from ...fluid.contrib.slim.quantization.quantization_pass import TransformForMobilePass -from ...fluid.contrib.slim.quantization.quantization_pass import OutScaleForTrainingPass -from ...fluid.contrib.slim.quantization.quantization_pass import OutScaleForInferencePass -from ...fluid.contrib.slim.quantization.quantization_pass import AddQuantDequantPass -from ...fluid.contrib.slim.quantization.quantization_pass import ReplaceFakeQuantDequantPass +from ...fluid.contrib.slim.quantization.quantization_pass import ( + QuantizationTransformPass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + QuantizationFreezePass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + ConvertToInt8Pass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + TransformForMobilePass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + OutScaleForTrainingPass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + OutScaleForInferencePass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + AddQuantDequantPass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + ReplaceFakeQuantDequantPass, +) from ...fluid.contrib.slim.quantization.quantization_pass import QuantWeightPass -from ...fluid.contrib.slim.quantization.quantization_pass import QuantizationTransformPassV2 -from ...fluid.contrib.slim.quantization.quantization_pass import AddQuantDequantPassV2 -from ...fluid.contrib.slim.quantization.quant_int8_mkldnn_pass import QuantInt8MkldnnPass -from ...fluid.contrib.slim.quantization.quant2_int8_mkldnn_pass import Quant2Int8MkldnnPass +from ...fluid.contrib.slim.quantization.quantization_pass import ( + QuantizationTransformPassV2, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + AddQuantDequantPassV2, +) +from ...fluid.contrib.slim.quantization.quant_int8_mkldnn_pass import ( + QuantInt8MkldnnPass, +) +from ...fluid.contrib.slim.quantization.quant2_int8_mkldnn_pass import ( + Quant2Int8MkldnnPass, +) -from ...fluid.contrib.slim.quantization.post_training_quantization import PostTrainingQuantization -from ...fluid.contrib.slim.quantization.post_training_quantization import PostTrainingQuantizationProgram -from ...fluid.contrib.slim.quantization.post_training_quantization import WeightQuantization +from ...fluid.contrib.slim.quantization.post_training_quantization import ( + PostTrainingQuantization, +) +from ...fluid.contrib.slim.quantization.post_training_quantization import ( + PostTrainingQuantizationProgram, +) +from ...fluid.contrib.slim.quantization.post_training_quantization import ( + WeightQuantization, +) diff --git a/python/paddle/static/sparsity/__init__.py b/python/paddle/static/sparsity/__init__.py index 8d3166b19a0..37b48132bda 100644 --- a/python/paddle/static/sparsity/__init__.py +++ b/python/paddle/static/sparsity/__init__.py @@ -13,20 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.contrib.sparsity import calculate_density #noqa: F401 -from ...fluid.contrib.sparsity import decorate #noqa: F401 -from ...fluid.contrib.sparsity import prune_model #noqa: F401 -from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401 -from ...fluid.contrib.sparsity import add_supported_layer #noqa: F401 -from ...fluid.contrib import sparsity #noqa: F401 +from ...fluid.contrib.sparsity import calculate_density # noqa: F401 +from ...fluid.contrib.sparsity import decorate # noqa: F401 +from ...fluid.contrib.sparsity import prune_model # noqa: F401 +from ...fluid.contrib.sparsity import reset_excluded_layers # noqa: F401 +from ...fluid.contrib.sparsity import add_supported_layer # noqa: F401 +from ...fluid.contrib import sparsity # noqa: F401 def set_excluded_layers(main_program, param_names): - sparsity.set_excluded_layers(param_names=param_names, - main_program=main_program) + sparsity.set_excluded_layers( + param_names=param_names, main_program=main_program + ) -__all__ = [ #noqa - 'calculate_density', 'decorate', 'prune_model', 'set_excluded_layers', - 'reset_excluded_layers', 'add_supported_layer' +__all__ = [ # noqa + 'calculate_density', + 'decorate', + 'prune_model', + 'set_excluded_layers', + 'reset_excluded_layers', + 'add_supported_layer', ] diff --git a/python/paddle/sysconfig.py b/python/paddle/sysconfig.py index 2ce327c7696..9a839f199a7 100644 --- a/python/paddle/sysconfig.py +++ b/python/paddle/sysconfig.py @@ -31,6 +31,7 @@ def get_include(): """ import paddle + return os.path.join(os.path.dirname(paddle.__file__), 'include') @@ -48,4 +49,5 @@ def get_lib(): """ import paddle + return os.path.join(os.path.dirname(paddle.__file__), 'libs') diff --git a/python/paddle/tensor/array.py b/python/paddle/tensor/array.py index 527cf692525..382c6f524b4 100644 --- a/python/paddle/tensor/array.py +++ b/python/paddle/tensor/array.py @@ -48,22 +48,24 @@ def array_length(array): """ if _non_static_mode(): assert isinstance( - array, - list), "The 'array' in array_write must be a list in dygraph mode" + array, list + ), "The 'array' in array_write must be a list in dygraph mode" return len(array) - if not isinstance( - array, - Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError( - "array should be tensor array vairable in array_length Op") + "array should be tensor array vairable in array_length Op" + ) helper = LayerHelper('array_length', **locals()) tmp = helper.create_variable_for_type_inference(dtype='int64') tmp.stop_gradient = True - helper.append_op(type='lod_array_length', - inputs={'X': [array]}, - outputs={'Out': [tmp]}) + helper.append_op( + type='lod_array_length', inputs={'X': [array]}, outputs={'Out': [tmp]} + ) return tmp @@ -108,8 +110,8 @@ def array_read(array, i): """ if _non_static_mode(): assert isinstance( - array, - list), "The 'array' in array_read must be list in dygraph mode" + array, list + ), "The 'array' in array_read must be list in dygraph mode" assert isinstance( i, Variable ), "The index 'i' in array_read must be Variable in dygraph mode" @@ -121,17 +123,17 @@ def array_read(array, i): check_variable_and_dtype(i, 'i', ['int64'], 'array_read') helper = LayerHelper('array_read', **locals()) - if not isinstance( - array, - Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError("array should be tensor array vairable") out = helper.create_variable_for_type_inference(dtype=array.dtype) - helper.append_op(type='read_from_array', - inputs={ - 'X': [array], - 'I': [i] - }, - outputs={'Out': [out]}) + helper.append_op( + type='read_from_array', + inputs={'X': [array], 'I': [i]}, + outputs={'Out': [out]}, + ) return out @@ -180,8 +182,8 @@ def array_write(x, i, array=None): if array is None: array = create_array(x.dtype) assert isinstance( - array, - list), "The 'array' in array_write must be a list in dygraph mode" + array, list + ), "The 'array' in array_write must be a list in dygraph mode" assert i <= len( array ), "The index 'i' should not be greater than the length of 'array' in dygraph mode" @@ -195,22 +197,24 @@ def array_write(x, i, array=None): check_type(x, 'x', (Variable), 'array_write') helper = LayerHelper('array_write', **locals()) if array is not None: - if not isinstance( - array, Variable - ) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError( - "array should be tensor array vairable in array_write Op") + "array should be tensor array vairable in array_write Op" + ) if array is None: array = helper.create_variable( name="{0}.out".format(helper.name), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=x.dtype) - helper.append_op(type='write_to_array', - inputs={ - 'X': [x], - 'I': [i] - }, - outputs={'Out': [array]}) + dtype=x.dtype, + ) + helper.append_op( + type='write_to_array', + inputs={'X': [x], 'I': [i]}, + outputs={'Out': [array]}, + ) return array @@ -247,16 +251,20 @@ def create_array(dtype, initialized_list=None): if initialized_list is not None: if not isinstance(initialized_list, (list, tuple)): raise TypeError( - "Require type(initialized_list) should be list/tuple, but received {}" - .format(type(initialized_list))) + "Require type(initialized_list) should be list/tuple, but received {}".format( + type(initialized_list) + ) + ) array = list(initialized_list) # NOTE: Only support plain list like [x, y,...], not support nested list in static mode. for val in array: if not isinstance(val, Variable): raise TypeError( - "All values in `initialized_list` should be Variable, but recevied {}." - .format(type(val))) + "All values in `initialized_list` should be Variable, but recevied {}.".format( + type(val) + ) + ) if _non_static_mode(): return array @@ -265,7 +273,8 @@ def create_array(dtype, initialized_list=None): tensor_array = helper.create_variable( name="{0}.out".format(helper.name), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=dtype) + dtype=dtype, + ) for val in array: array_write(x=val, i=array_length(tensor_array), array=tensor_array) diff --git a/python/paddle/tensor/attribute.py b/python/paddle/tensor/attribute.py index d68eebf1364..e0db6a6e8b4 100644 --- a/python/paddle/tensor/attribute.py +++ b/python/paddle/tensor/attribute.py @@ -115,16 +115,29 @@ def shape(input): out.stop_gradient = True return out - check_variable_and_dtype(input, 'input', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'shape') + check_variable_and_dtype( + input, + 'input', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'shape', + ) helper = LayerHelper('shape', **locals()) out = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type='shape', - inputs={'Input': input}, - outputs={'Out': out}, - stop_gradient=True) + helper.append_op( + type='shape', + inputs={'Input': input}, + outputs={'Out': out}, + stop_gradient=True, + ) return out @@ -156,11 +169,14 @@ def is_complex(x): # False """ if not isinstance(x, (paddle.Tensor, paddle.static.Variable)): - raise TypeError("Expected Tensor, but received type of x: {}".format( - type(x))) + raise TypeError( + "Expected Tensor, but received type of x: {}".format(type(x)) + ) dtype = x.dtype - is_complex_dtype = (dtype == core.VarDesc.VarType.COMPLEX64 - or dtype == core.VarDesc.VarType.COMPLEX128) + is_complex_dtype = ( + dtype == core.VarDesc.VarType.COMPLEX64 + or dtype == core.VarDesc.VarType.COMPLEX128 + ) return is_complex_dtype @@ -187,13 +203,16 @@ def is_floating_point(x): # False """ if not isinstance(x, (paddle.Tensor, paddle.static.Variable)): - raise TypeError("Expected Tensor, but received type of x: {}".format( - type(x))) + raise TypeError( + "Expected Tensor, but received type of x: {}".format(type(x)) + ) dtype = x.dtype - is_fp_dtype = (dtype == core.VarDesc.VarType.FP32 - or dtype == core.VarDesc.VarType.FP64 - or dtype == core.VarDesc.VarType.FP16 - or dtype == core.VarDesc.VarType.BF16) + is_fp_dtype = ( + dtype == core.VarDesc.VarType.FP32 + or dtype == core.VarDesc.VarType.FP64 + or dtype == core.VarDesc.VarType.FP16 + or dtype == core.VarDesc.VarType.BF16 + ) return is_fp_dtype @@ -224,14 +243,17 @@ def is_integer(x): # True """ if not isinstance(x, (paddle.Tensor, paddle.static.Variable)): - raise TypeError("Expected Tensor, but received type of x: {}".format( - type(x))) + raise TypeError( + "Expected Tensor, but received type of x: {}".format(type(x)) + ) dtype = x.dtype - is_int_dtype = (dtype == core.VarDesc.VarType.UINT8 - or dtype == core.VarDesc.VarType.INT8 - or dtype == core.VarDesc.VarType.INT16 - or dtype == core.VarDesc.VarType.INT32 - or dtype == core.VarDesc.VarType.INT64) + is_int_dtype = ( + dtype == core.VarDesc.VarType.UINT8 + or dtype == core.VarDesc.VarType.INT8 + or dtype == core.VarDesc.VarType.INT16 + or dtype == core.VarDesc.VarType.INT32 + or dtype == core.VarDesc.VarType.INT64 + ) return is_int_dtype @@ -276,7 +298,8 @@ def real(x, name=None): check_variable_and_dtype(x, 'x', ['complex64', 'complex128'], 'real') helper = LayerHelper('real', **locals()) out = helper.create_variable_for_type_inference( - dtype=_complex_to_real_dtype(helper.input_dtype())) + dtype=_complex_to_real_dtype(helper.input_dtype()) + ) helper.append_op(type='real', inputs={'X': x}, outputs={'Out': out}) return out @@ -322,6 +345,7 @@ def imag(x, name=None): check_variable_and_dtype(x, 'x', ['complex64', 'complex128'], 'imag') helper = LayerHelper('imag', **locals()) out = helper.create_variable_for_type_inference( - dtype=_complex_to_real_dtype(helper.input_dtype())) + dtype=_complex_to_real_dtype(helper.input_dtype()) + ) helper.append_op(type='imag', inputs={'X': x}, outputs={'Out': out}) return out diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py index 675c186d47f..8f44f3ffe16 100644 --- a/python/paddle/tensor/creation.py +++ b/python/paddle/tensor/creation.py @@ -22,12 +22,21 @@ from ..framework import _current_expected_place, _get_paddle_place from ..framework import core from ..framework import in_dygraph_mode, _non_static_mode from ..framework import LayerHelper -from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from ..fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from ..framework import convert_np_dtype_to_dtype_ + # TODO: define functions to get create a tensor import paddle from paddle import _C_ops, _legacy_C_ops -from ..fluid.framework import _in_legacy_dygraph, _in_eager_without_dygraph_check +from ..fluid.framework import ( + _in_legacy_dygraph, + _in_eager_without_dygraph_check, +) import warnings __all__ = [] @@ -98,11 +107,17 @@ def linspace(start, stop, num, dtype=None, name=None): with device_guard("cpu"): tensor_num = fill_constant([1], 'int32', num, force_cpu=True) if in_dygraph_mode(): - return _C_ops.linspace(tensor_start, tensor_stop, tensor_num, dtype, - _current_expected_place()) + return _C_ops.linspace( + tensor_start, + tensor_stop, + tensor_num, + dtype, + _current_expected_place(), + ) if _in_legacy_dygraph(): - return _legacy_C_ops.linspace(tensor_start, tensor_stop, tensor_num, - 'dtype', dtype) + return _legacy_C_ops.linspace( + tensor_start, tensor_stop, tensor_num, 'dtype', dtype + ) helper = LayerHelper("linspace", **locals()) @@ -110,41 +125,53 @@ def linspace(start, stop, num, dtype=None, name=None): stop_dtype = convert_dtype(tensor_stop.dtype) out_dtype = convert_dtype(dtype) if isinstance(start, Variable): - check_dtype(start.dtype, 'start', - ['float32', 'float64', 'int32', 'int64'], 'linspace') + check_dtype( + start.dtype, + 'start', + ['float32', 'float64', 'int32', 'int64'], + 'linspace', + ) else: check_type(start, 'start', (int, float), 'linspace') if isinstance(stop, Variable): - check_dtype(stop.dtype, 'stop', - ['float32', 'float64', 'int32', 'int64'], 'linspace') + check_dtype( + stop.dtype, + 'stop', + ['float32', 'float64', 'int32', 'int64'], + 'linspace', + ) else: check_type(stop, 'stop', (int, float), 'linspace') if isinstance(num, Variable): check_dtype(num.dtype, 'num', ['int32'], 'linspace') - check_dtype(dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], - 'linspace') - if ((stop_dtype == "float64" or start_dtype == "float64") - and out_dtype in ["float32", "int32"]) or ( - (stop_dtype == "int64" or start_dtype == "int64") - and out_dtype == "int32"): + check_dtype( + dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], 'linspace' + ) + if ( + (stop_dtype == "float64" or start_dtype == "float64") + and out_dtype in ["float32", "int32"] + ) or ( + (stop_dtype == "int64" or start_dtype == "int64") + and out_dtype == "int32" + ): raise ValueError( "The dtype of start/stop is {}/{} but the attr(dtype) of linspace is {}, " - "which may cause data type overflows. Please reset attr(dtype) of linspace." - .format(start_dtype, stop_dtype, dtype)) + "which may cause data type overflows. Please reset attr(dtype) of linspace.".format( + start_dtype, stop_dtype, dtype + ) + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='linspace', - inputs={ - 'Start': tensor_start, - 'Stop': tensor_stop, - 'Num': tensor_num - }, - attrs={'dtype': dtype}, - outputs={'Out': [out]}) + helper.append_op( + type='linspace', + inputs={'Start': tensor_start, 'Stop': tensor_stop, 'Num': tensor_num}, + attrs={'dtype': dtype}, + outputs={'Out': [out]}, + ) if isinstance(num, int): - out.desc.set_shape((num, )) + out.desc.set_shape((num,)) return out @@ -210,8 +237,9 @@ def logspace(start, stop, num, base=10.0, dtype=None, name=None): with device_guard("cpu"): tensor_base = fill_constant([1], dtype, base) if _non_static_mode(): - return _legacy_C_ops.logspace(tensor_start, tensor_stop, tensor_num, - tensor_base, 'dtype', dtype) + return _legacy_C_ops.logspace( + tensor_start, tensor_stop, tensor_num, tensor_base, 'dtype', dtype + ) helper = LayerHelper("logspace", **locals()) @@ -220,14 +248,22 @@ def logspace(start, stop, num, base=10.0, dtype=None, name=None): base_dtype = convert_dtype(tensor_base.dtype) out_dtype = convert_dtype(dtype) if isinstance(start, Variable): - check_dtype(start.dtype, 'start', - ['float32', 'float64', 'int32', 'int64'], 'logspace') + check_dtype( + start.dtype, + 'start', + ['float32', 'float64', 'int32', 'int64'], + 'logspace', + ) else: check_type(start, 'start', (int, float), 'logspace') if isinstance(stop, Variable): - check_dtype(stop.dtype, 'stop', - ['float32', 'float64', 'int32', 'int64'], 'logspace') + check_dtype( + stop.dtype, + 'stop', + ['float32', 'float64', 'int32', 'int64'], + 'logspace', + ) else: check_type(stop, 'stop', (int, float), 'logspace') @@ -235,37 +271,55 @@ def logspace(start, stop, num, base=10.0, dtype=None, name=None): check_dtype(num.dtype, 'num', ['int32'], 'logspace') if isinstance(base, Variable): - check_dtype(base.dtype, 'base', - ['float32', 'float64', 'int32', 'int64'], 'logspace') + check_dtype( + base.dtype, + 'base', + ['float32', 'float64', 'int32', 'int64'], + 'logspace', + ) else: check_type(base, 'base', (int, float), 'logspace') - check_dtype(dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], - 'logspace') - if ((stop_dtype == "float64" or start_dtype == "float64" - or base_dtype == "float64") - and out_dtype in ["float32", "int32"]) or \ - ((stop_dtype == "int64" or start_dtype == "int64" - or base_dtype == "int64") - and out_dtype == "int32"): + check_dtype( + dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], 'logspace' + ) + if ( + ( + stop_dtype == "float64" + or start_dtype == "float64" + or base_dtype == "float64" + ) + and out_dtype in ["float32", "int32"] + ) or ( + ( + stop_dtype == "int64" + or start_dtype == "int64" + or base_dtype == "int64" + ) + and out_dtype == "int32" + ): raise ValueError( "The dtype of start/stop/base is {}/{}/{} but the attr(dtype) of logspace is {}, " - "which may cause data type overflows. Please reset attr(dtype) of logspace." - .format(start_dtype, stop_dtype, base_dtype, dtype)) + "which may cause data type overflows. Please reset attr(dtype) of logspace.".format( + start_dtype, stop_dtype, base_dtype, dtype + ) + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='logspace', - inputs={ - 'Start': tensor_start, - 'Stop': tensor_stop, - 'Num': tensor_num, - 'Base': tensor_base - }, - attrs={'dtype': dtype}, - outputs={'Out': [out]}) + helper.append_op( + type='logspace', + inputs={ + 'Start': tensor_start, + 'Stop': tensor_stop, + 'Num': tensor_num, + 'Base': tensor_base, + }, + attrs={'dtype': dtype}, + outputs={'Out': [out]}, + ) if isinstance(num, int): - out.desc.set_shape((num, )) + out.desc.set_shape((num,)) return out @@ -313,17 +367,25 @@ def _to_tensor_non_static(data, dtype=None, place=None, stop_gradient=True): return data else: raise TypeError( - "Can't constructs a 'paddle.Tensor' with data type {}, data type must be scalar|list|tuple|np.ndarray|paddle.Tensor" - .format(type(data))) + "Can't constructs a 'paddle.Tensor' with data type {}, data type must be scalar|list|tuple|np.ndarray|paddle.Tensor".format( + type(data) + ) + ) if not dtype: if data.dtype in [ - 'float16', 'float32', 'float64', 'complex64', 'complex128' + 'float16', + 'float32', + 'float64', + 'complex64', + 'complex128', ]: default_type = paddle.get_default_dtype() if np.iscomplexobj(data): - default_type = 'complex64' if default_type in [ - 'float16', 'float32' - ] else 'complex128' + default_type = ( + 'complex64' + if default_type in ['float16', 'float32'] + else 'complex128' + ) data = data.astype(default_type) # Windows default type is 'int32', while Linux/Mac is 'int64'. Unify they. if data.dtype in ['int32']: @@ -334,18 +396,22 @@ def _to_tensor_non_static(data, dtype=None, place=None, stop_gradient=True): data = data.astype(convert_dtype(dtype)) if _in_eager_without_dygraph_check() and isinstance(data, np.ndarray): - return core.eager.Tensor(value=data, - place=place, - persistable=False, - zero_copy=False, - name=None, - stop_gradient=stop_gradient) + return core.eager.Tensor( + value=data, + place=place, + persistable=False, + zero_copy=False, + name=None, + stop_gradient=stop_gradient, + ) else: - return paddle.Tensor(value=data, - place=place, - persistable=False, - zero_copy=False, - stop_gradient=stop_gradient) + return paddle.Tensor( + value=data, + place=place, + persistable=False, + zero_copy=False, + stop_gradient=stop_gradient, + ) def _to_tensor_static(data, dtype=None, stop_gradient=None): @@ -360,8 +426,11 @@ def _to_tensor_static(data, dtype=None, stop_gradient=None): elif isinstance(data, (list, tuple)): data = np.array(data) - if isinstance(data, - np.ndarray) and not dtype and data.dtype != 'object': + if ( + isinstance(data, np.ndarray) + and not dtype + and data.dtype != 'object' + ): if data.dtype in ['float16', 'float32', 'float64']: data = data.astype(paddle.get_default_dtype()) elif data.dtype in ['int32']: @@ -376,10 +445,14 @@ def _to_tensor_static(data, dtype=None, stop_gradient=None): target_dtype = convert_dtype(target_dtype) - if isinstance(data, np.ndarray) and len(data.shape) > 0 and any( - isinstance(x, Variable) for x in data): + if ( + isinstance(data, np.ndarray) + and len(data.shape) > 0 + and any(isinstance(x, Variable) for x in data) + ): if not all( - [x.shape == (1, ) for x in data if isinstance(x, Variable)]): + [x.shape == (1,) for x in data if isinstance(x, Variable)] + ): raise TypeError( "Unsupport paddle.to_tensor([Variable, Variable...]) with non-scalar variable." ) @@ -513,27 +586,31 @@ def full_like(x, fill_value, dtype=None, name=None): return _C_ops.full_like(x, fill_value, dtype, x.place) if _in_legacy_dygraph(): - return _legacy_C_ops.fill_any_like(x, 'value', fill_value, 'dtype', - dtype) + return _legacy_C_ops.fill_any_like( + x, 'value', fill_value, 'dtype', dtype + ) helper = LayerHelper("full_like", **locals()) check_variable_and_dtype( - x, 'x', + x, + 'x', ['bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64'], - 'full_like') + 'full_like', + ) check_dtype( - dtype, 'dtype', + dtype, + 'dtype', ['bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64'], - 'full_like/zeros_like/ones_like') + 'full_like/zeros_like/ones_like', + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='fill_any_like', - inputs={'X': [x]}, - attrs={ - 'value': fill_value, - "dtype": dtype - }, - outputs={'Out': [out]}) + helper.append_op( + type='fill_any_like', + inputs={'X': [x]}, + attrs={'value': fill_value, "dtype": dtype}, + outputs={'Out': [out]}, + ) out.stop_gradient = True return out @@ -728,26 +805,34 @@ def eye(num_rows, num_columns=None, dtype=None, name=None): if _non_static_mode(): if in_dygraph_mode(): - out = _C_ops.eye(num_rows, num_columns, dtype, - _current_expected_place()) + out = _C_ops.eye( + num_rows, num_columns, dtype, _current_expected_place() + ) elif _in_legacy_dygraph(): - out = _legacy_C_ops.eye('dtype', dtype, 'num_rows', num_rows, - 'num_columns', num_columns) + out = _legacy_C_ops.eye( + 'dtype', dtype, 'num_rows', num_rows, 'num_columns', num_columns + ) else: helper = LayerHelper("eye", **locals()) - check_dtype(dtype, 'dtype', - ['float16', 'float32', 'float64', 'int32', 'int64'], 'eye') + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'eye', + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='eye', - inputs={}, - outputs={'Out': [out]}, - attrs={ - 'num_rows': num_rows, - 'num_columns': num_columns, - 'dtype': dtype - }, - stop_gradient=True) + helper.append_op( + type='eye', + inputs={}, + outputs={'Out': [out]}, + attrs={ + 'num_rows': num_rows, + 'num_columns': num_columns, + 'dtype': dtype, + }, + stop_gradient=True, + ) out.stop_gradient = True return out @@ -866,8 +951,11 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): start = 0 out_shape = None - if not isinstance(start, Variable) and not isinstance( - end, Variable) and not isinstance(step, Variable): + if ( + not isinstance(start, Variable) + and not isinstance(end, Variable) + and not isinstance(step, Variable) + ): out_shape = [int(math.ceil((end - start) / step))] if not isinstance(dtype, core.VarDesc.VarType): @@ -899,17 +987,16 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): out.stop_gradient = True return out - check_dtype(dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], - 'range/arange') + check_dtype( + dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], 'range/arange' + ) helper = LayerHelper('range', **locals()) out = helper.create_variable_for_type_inference(dtype, shape=out_shape) - helper.append_op(type='range', - inputs={ - 'Start': start, - 'End': end, - 'Step': step - }, - outputs={'Out': out}) + helper.append_op( + type='range', + inputs={'Start': start, 'End': end, 'Step': step}, + outputs={'Out': out}, + ) out.stop_gradient = True if out_shape is not None: out.desc.set_shape(out_shape) @@ -917,28 +1004,30 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): def _tril_triu_op(helper): - """Base op of tril_op and triu_op - """ + """Base op of tril_op and triu_op""" op_type = helper.layer_type x = helper.kwargs.get('x', None) assert x is not None, 'x cannot be None in {}'.format(op_type) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - op_type) + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + op_type, + ) if len(x.shape) < 2: raise ValueError("x shape in {} must be at least 2-D".format(op_type)) diagonal = helper.kwargs.get('diagonal', 0) - if not isinstance(diagonal, (int, )): + if not isinstance(diagonal, (int,)): raise TypeError("diagonal in {} must be a python Int".format(op_type)) name = helper.kwargs.get('name', None) if name is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: - out = helper.create_variable(name=name, - dtype=x.dtype, - persistable=False) + out = helper.create_variable( + name=name, dtype=x.dtype, persistable=False + ) helper.append_op( type="tril_triu", @@ -1131,18 +1220,21 @@ def meshgrid(*args, **kwargs): raise TypeError("The type of input args in meshgrid should be list.") for id, input_ in enumerate(args): - check_dtype(input_.dtype, 'create data type', - ['float16', 'float32', 'float64', 'int32', 'int64'], - 'meshgrid') + check_dtype( + input_.dtype, + 'create data type', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'meshgrid', + ) num = len(args) out = [ helper.create_variable_for_type_inference(dtype=args[i].dtype) for i in range(num) ] - helper.append_op(type='meshgrid', - inputs={'X': list(args)}, - outputs={'Out': out}) + helper.append_op( + type='meshgrid', inputs={'X': list(args)}, outputs={'Out': out} + ) return out @@ -1242,17 +1334,21 @@ def diagflat(x, offset=0, name=None): if _in_legacy_dygraph(): if len(x.shape) == 1: - return _legacy_C_ops.diag_v2(x, "offset", offset, "padding_value", - padding_value) + return _legacy_C_ops.diag_v2( + x, "offset", offset, "padding_value", padding_value + ) else: y, _ = _legacy_C_ops.flatten_contiguous_range( - x, "start_axis", 0, "stop_axis", -1) - return _legacy_C_ops.diag_v2(y, "offset", offset, "padding_value", - padding_value) + x, "start_axis", 0, "stop_axis", -1 + ) + return _legacy_C_ops.diag_v2( + y, "offset", offset, "padding_value", padding_value + ) check_type(x, 'x', (Variable), 'diagflat') - check_dtype(x.dtype, 'x', ['float32', 'float64', 'int32', 'int64'], - 'diagflat') + check_dtype( + x.dtype, 'x', ['float32', 'float64', 'int32', 'int64'], 'diagflat' + ) check_type(offset, 'offset', (int), 'diagflat') helper = LayerHelper("diagflat", **locals()) @@ -1261,33 +1357,27 @@ def diagflat(x, offset=0, name=None): out2 = helper.create_variable_for_type_inference(dtype=x.dtype) if len(x.shape) == 1: - helper.append_op(type='diag_v2', - inputs={'X': x}, - outputs={'Out': out2}, - attrs={ - 'offset': offset, - 'padding_value': padding_value - }) + helper.append_op( + type='diag_v2', + inputs={'X': x}, + outputs={'Out': out2}, + attrs={'offset': offset, 'padding_value': padding_value}, + ) else: - helper.append_op(type='flatten_contiguous_range', - inputs={'X': x}, - outputs={ - 'Out': out1, - 'XShape': out1_shape - }, - attrs={ - 'start_axis': 0, - 'stop_axis': -1 - }) + helper.append_op( + type='flatten_contiguous_range', + inputs={'X': x}, + outputs={'Out': out1, 'XShape': out1_shape}, + attrs={'start_axis': 0, 'stop_axis': -1}, + ) out1.stop_gradient = True - helper.append_op(type='diag_v2', - inputs={'X': out1}, - outputs={'Out': out2}, - attrs={ - 'offset': offset, - 'padding_value': padding_value - }) + helper.append_op( + type='diag_v2', + inputs={'X': out1}, + outputs={'Out': out2}, + attrs={'offset': offset, 'padding_value': padding_value}, + ) out2.stop_gradient = True return out2 @@ -1371,30 +1461,36 @@ def diag(x, offset=0, padding_value=0, name=None): return _C_ops.diag(x, offset, padding_value) else: if _in_legacy_dygraph(): - return _legacy_C_ops.diag_v2(x, "offset", offset, "padding_value", - padding_value) + return _legacy_C_ops.diag_v2( + x, "offset", offset, "padding_value", padding_value + ) else: check_type(x, 'x', (Variable), 'diag_v2') - check_dtype(x.dtype, 'x', ['float32', 'float64', 'int32', 'int64'], - 'diag_v2') + check_dtype( + x.dtype, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'diag_v2', + ) check_type(offset, 'offset', (int), 'diag_v2') check_type(padding_value, 'padding_value', (int, float), 'diag_v2') if len(x.shape) != 1 and len(x.shape) != 2: raise ValueError( - "The dimension of input x must be either 1 or 2, but received {}" - .format(len(x.shape))) + "The dimension of input x must be either 1 or 2, but received {}".format( + len(x.shape) + ) + ) helper = LayerHelper("diag_v2", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='diag_v2', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'offset': offset, - 'padding_value': padding_value - }) + helper.append_op( + type='diag_v2', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'offset': offset, 'padding_value': padding_value}, + ) out.stop_gradient = True return out @@ -1456,42 +1552,48 @@ def empty(shape, dtype=None, name=None): if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) - out = _C_ops.empty(shape, convert_np_dtype_to_dtype_(dtype), - _current_expected_place()) + out = _C_ops.empty( + shape, convert_np_dtype_to_dtype_(dtype), _current_expected_place() + ) out.stop_gradient = True return out if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - out = _legacy_C_ops.empty('shape', shape, 'dtype', - convert_np_dtype_to_dtype_(dtype)) + out = _legacy_C_ops.empty( + 'shape', shape, 'dtype', convert_np_dtype_to_dtype_(dtype) + ) out.stop_gradient = True return out helper = LayerHelper("empty", **locals()) inputs = {} - check_dtype(dtype, 'dtype', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'empty') + check_dtype( + dtype, + 'dtype', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'empty', + ) check_type(shape, 'shape', (Variable, list, tuple), 'empty') if isinstance(shape, Variable): check_dtype(shape.dtype, 'shape', ['int32', 'int64'], 'empty') attrs = {} - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='empty') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='empty' + ) out = helper.create_variable_for_type_inference(dtype=dtype) attrs['dtype'] = convert_np_dtype_to_dtype_(dtype) - helper.append_op(type='empty', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs, - stop_gradient=True) + helper.append_op( + type='empty', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + stop_gradient=True, + ) out.stop_gradient = True return out @@ -1529,40 +1631,51 @@ def empty_like(x, dtype=None, name=None): dtype = convert_dtype(dtype) if in_dygraph_mode(): - out = _C_ops.empty(x.shape, convert_np_dtype_to_dtype_(dtype), - _current_expected_place()) + out = _C_ops.empty( + x.shape, + convert_np_dtype_to_dtype_(dtype), + _current_expected_place(), + ) out.stop_gradient = True return out if _in_legacy_dygraph(): - out = _legacy_C_ops.empty('shape', x.shape, 'dtype', - convert_np_dtype_to_dtype_(dtype)) + out = _legacy_C_ops.empty( + 'shape', x.shape, 'dtype', convert_np_dtype_to_dtype_(dtype) + ) out.stop_gradient = True return out helper = LayerHelper("empty_like", **locals()) check_variable_and_dtype( - x, 'x', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'empty_like') - check_dtype(dtype, 'dtype', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'empty_like') + x, + 'x', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'empty_like', + ) + check_dtype( + dtype, + 'dtype', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'empty_like', + ) out = helper.create_variable_for_type_inference(dtype=dtype) inputs = {} attrs = {} attrs['dtype'] = convert_np_dtype_to_dtype_(dtype) shape = paddle.shape(x) - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='empty_like') - - helper.append_op(type='empty', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs, - stop_gradient=True) + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='empty_like' + ) + + helper.append_op( + type='empty', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + stop_gradient=True, + ) out.stop_gradient = True return out @@ -1597,8 +1710,12 @@ def assign(x, output=None): """ input = x helper = LayerHelper('assign', **locals()) - check_type(input, 'input', - (Variable, np.ndarray, list, tuple, float, int, bool), 'assign') + check_type( + input, + 'input', + (Variable, np.ndarray, list, tuple, float, int, bool), + 'assign', + ) is_inplace = True if output is not None else False if np.isscalar(input) and not isinstance(input, str): @@ -1621,24 +1738,40 @@ def assign(x, output=None): output = core.VarBase() _legacy_C_ops.assign(input, output) else: - check_dtype(input.dtype, 'input', [ - 'float16', 'uint16', 'float32', 'float64', 'int32', 'int64', - 'uint8', 'bool' - ], 'assign', '(When the type of input in assign is Variable.)') + check_dtype( + input.dtype, + 'input', + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'bool', + ], + 'assign', + '(When the type of input in assign is Variable.)', + ) if output is None: output = helper.create_variable_for_type_inference( - dtype=input.dtype) - helper.append_op(type='assign', - inputs={'X': [input]}, - outputs={'Out': [output]}) + dtype=input.dtype + ) + helper.append_op( + type='assign', inputs={'X': [input]}, outputs={'Out': [output]} + ) elif isinstance(input, np.ndarray): # We now support the form of [var, VAR...] if the Var.shape=[1,] if len(input.shape) > 0 and any(isinstance(x, Variable) for x in input): # We only deal with the case where the list is nested one level, convert all scalars into variables, and then use stack to process. It is necessary to ensure the consistency of types. - if not all([ - x.shape == (1, ) for x in input + if not all( + [ + x.shape == (1,) + for x in input if isinstance(x, (Variable, core.eager.Tensor)) - ]): + ] + ): raise TypeError( "Unsupport paddle.assign([Variable, Variable...]) with non-scalar variable." ) @@ -1654,8 +1787,7 @@ def assign(x, output=None): return ret if input.dtype == 'object': - """ may be this form [[Var], [Var], [3], [4]], we reject them. - """ + """may be this form [[Var], [Var], [3], [4]], we reject them.""" raise TypeError( "The type of received input == `object`, it is not supported to convert to tensor, such as [[Var], [Var], [3], [4]]" ) @@ -1667,7 +1799,8 @@ def assign(x, output=None): warnings.warn( "paddle.assign doesn't support float64 input now due " "to current platform protobuf data limitation, we convert " - "it to float32") + "it to float32" + ) dtype = core.VarDesc.VarType.FP32 if dtype == core.VarDesc.VarType.BOOL: value_name = "bool_values" @@ -1685,31 +1818,49 @@ def assign(x, output=None): raise TypeError( "When the type of 'input' in assign is numpy.ndarray, " "the data type of 'input' must be bool, float32, int32 or int64, but " - "received %s." % convert_dtype(dtype)) + "received %s." % convert_dtype(dtype) + ) if input.size > 1024 * 1024: - raise ValueError("The size of input is too big. Please consider " - "saving it to file and 'load_op' to load it") + raise ValueError( + "The size of input is too big. Please consider " + "saving it to file and 'load_op' to load it" + ) if in_dygraph_mode(): if output is None: output = zeros(list(input.shape), dtype) - _C_ops.assign_value_(output, list(input.shape), dtype, values, - _current_expected_place()) + _C_ops.assign_value_( + output, + list(input.shape), + dtype, + values, + _current_expected_place(), + ) elif _in_legacy_dygraph(): if output is None: output = core.VarBase() - _legacy_C_ops.assign_value(output, 'shape', list(input.shape), - 'dtype', dtype, value_name, values) + _legacy_C_ops.assign_value( + output, + 'shape', + list(input.shape), + 'dtype', + dtype, + value_name, + values, + ) else: if output is None: output = helper.create_variable_for_type_inference( - dtype=input.dtype) - helper.append_op(type='assign_value', - outputs={'Out': [output]}, - attrs={ - 'dtype': dtype, - 'shape': list(input.shape), - value_name: values - }) + dtype=input.dtype + ) + helper.append_op( + type='assign_value', + outputs={'Out': [output]}, + attrs={ + 'dtype': dtype, + 'shape': list(input.shape), + value_name: values, + }, + ) if is_inplace and _in_legacy_dygraph(): output._bump_inplace_version() @@ -1747,7 +1898,7 @@ def clone(x, name=None): return x.clone() -#NOTE(zhiqiu): not public +# NOTE(zhiqiu): not public def _memcpy(input, place=None, output=None): """ @@ -1775,10 +1926,22 @@ def _memcpy(input, place=None, output=None): check_type(input, 'input', (Variable), 'memcpy') if isinstance(input, (Variable, core.VarBase)): - check_dtype(input.dtype, 'input', [ - 'float16', 'uint16', 'float32', 'float64', 'int32', 'int64', - 'uint8', 'bool' - ], 'memcpy', '(When the type of input in memcpy is Variable.)') + check_dtype( + input.dtype, + 'input', + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'bool', + ], + 'memcpy', + '(When the type of input in memcpy is Variable.)', + ) if output is None: output = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -1800,10 +1963,12 @@ def _memcpy(input, place=None, output=None): dst_place_type = 4 attrs = {'dst_place_type': dst_place_type} - helper.append_op(type='memcpy', - inputs={'X': [input]}, - outputs={'Out': [output]}, - attrs=attrs) + helper.append_op( + type='memcpy', + inputs={'X': [input]}, + outputs={'Out': [output]}, + attrs=attrs, + ) return output @@ -1846,7 +2011,8 @@ def complex(real, imag, name=None): helper = LayerHelper(op_type, **locals()) inputs = {"X": real, "Y": imag} out = helper.create_variable_for_type_inference( - dtype=_real_to_complex_dtype(real.dtype)) + dtype=_real_to_complex_dtype(real.dtype) + ) outputs = {"Out": out} attrs = {} helper.append_op(type=op_type, inputs=inputs, attrs=attrs, outputs=outputs) @@ -1914,13 +2080,15 @@ def tril_indices(row, col, offset=0, dtype='int64'): dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): - out = _C_ops.tril_indices(row, col, offset, dtype, - _current_expected_place()) + out = _C_ops.tril_indices( + row, col, offset, dtype, _current_expected_place() + ) return out if _in_legacy_dygraph(): - out = _legacy_C_ops.tril_indices('rows', row, 'cols', col, 'offset', - offset, "dtype", dtype) + out = _legacy_C_ops.tril_indices( + 'rows', row, 'cols', col, 'offset', offset, "dtype", dtype + ) return out else: @@ -1928,15 +2096,12 @@ def tril_indices(row, col, offset=0, dtype='int64'): out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='tril_indices', - inputs={}, - outputs={'out': [out]}, - attrs={ - 'rows': row, - 'cols': col, - 'offset': offset, - 'dtype': dtype - }) + helper.append_op( + type='tril_indices', + inputs={}, + outputs={'out': [out]}, + attrs={'rows': row, 'cols': col, 'offset': offset, 'dtype': dtype}, + ) return out @@ -1999,13 +2164,15 @@ def triu_indices(row, col=None, offset=0, dtype='int64'): dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): - out = _C_ops.triu_indices(row, col, offset, dtype, - _current_expected_place()) + out = _C_ops.triu_indices( + row, col, offset, dtype, _current_expected_place() + ) return out if _in_legacy_dygraph(): - out = _legacy_C_ops.triu_indices('row', row, 'col', col, 'offset', - offset, "dtype", dtype) + out = _legacy_C_ops.triu_indices( + 'row', row, 'col', col, 'offset', offset, "dtype", dtype + ) return out else: @@ -2013,13 +2180,10 @@ def triu_indices(row, col=None, offset=0, dtype='int64'): out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='triu_indices', - inputs={}, - outputs={'out': [out]}, - attrs={ - 'row': row, - 'col': col, - 'offset': offset, - 'dtype': dtype - }) + helper.append_op( + type='triu_indices', + inputs={}, + outputs={'out': [out]}, + attrs={'row': row, 'col': col, 'offset': offset, 'dtype': dtype}, + ) return out diff --git a/python/paddle/tensor/einsum.py b/python/paddle/tensor/einsum.py index f8b014afb61..5c792f8fe0d 100644 --- a/python/paddle/tensor/einsum.py +++ b/python/paddle/tensor/einsum.py @@ -50,12 +50,13 @@ def parse_op_labels(labelstr, operand): ''' # Sanity checks for c in labelstr.replace('.', ''): - assert c.isalpha(), ( - f"Invalid equation: {c} is not a valid label, which should be letters." - ) + assert ( + c.isalpha() + ), f"Invalid equation: {c} is not a valid label, which should be letters." - assert labelstr.replace('...', '', 1).find('.') == -1, ( - "Invalid equation: `.` is found outside of an ellipsis.") + assert ( + labelstr.replace('...', '', 1).find('.') == -1 + ), "Invalid equation: `.` is found outside of an ellipsis." # Check shape. Note, in Paddle a tensor rank is always nonzero ndims = len(operand.shape) @@ -63,8 +64,9 @@ def parse_op_labels(labelstr, operand): full_labelstr = labelstr.replace('...', '.' * (ndims - len(labelstr) + 3)) - assert len(full_labelstr) == ndims, ( - f"Invalid equation: the label string '{labelstr}' misses dimensions.") + assert ( + len(full_labelstr) == ndims + ), f"Invalid equation: the label string '{labelstr}' misses dimensions." return full_labelstr @@ -88,7 +90,8 @@ def parse_labels(labelstr, operands): nop_labels = labelstr.split(',') assert len(nop_labels) == len(operands), ( f"Invalid equation: the number of operands is {len(operands)}, " - f"but found {len(nop_labels)} segments in the label equation.") + f"but found {len(nop_labels)} segments in the label equation." + ) return list(map(parse_op_labels, nop_labels, operands)) @@ -99,8 +102,9 @@ def validate_rhs(rhs, input_labels, n_bcast_dims): ''' # Sanity check. if n_bcast_dims > 0: - assert '...' in rhs, ( - "Invalid equation: missing ellipsis in output labels.") + assert ( + '...' in rhs + ), "Invalid equation: missing ellipsis in output labels." rhs = rhs.replace('...', '') rhs_set = set(rhs) @@ -112,10 +116,12 @@ def validate_rhs(rhs, input_labels, n_bcast_dims): non_input_labels = rhs_set.difference(input_labels) assert not non_input_labels, ( f"Invalid equation: " - f"output label {sorted(non_input_labels)} not used by any input.") + f"output label {sorted(non_input_labels)} not used by any input." + ) # Verify that output labels are not duplicate - assert len(rhs) == len(rhs_set), ( - "Invalid equation: duplicate output labels are found.") + assert len(rhs) == len( + rhs_set + ), "Invalid equation: duplicate output labels are found." def build_view(in_labels, out_labels): @@ -157,8 +163,8 @@ def build_view(in_labels, out_labels): # fill the broadcast dimension indices from right to left. if s: for ax, dim in zip( - range(start, end)[::-1], - range(s.start(), s.end())[::-1]): + range(start, end)[::-1], range(s.start(), s.end())[::-1] + ): inv_map[ax] = dim # Now work on non-broadcast dimensions @@ -217,7 +223,8 @@ def build_global_view(nop_labels, rhs, n_bcast_dims): g_labels_out = rhs.replace('...', '.' * n_bcast_dims) else: g_labels_out = '.' * n_bcast_dims + ''.join( - l for l, c in zip(labels, count) if c == 1) + l for l, c in zip(labels, count) if c == 1 + ) for i in range(len(count))[::-1]: if labels[i] in g_labels_out: @@ -265,12 +272,14 @@ def build_global_shape(g_view, g_labels, op_shapes): assert not non_bcastable, ( f"Invalid operands: label {g_labels[non_bcastable[0]]} " - f"corresponds to non-broadcastable dimensions.") + f"corresponds to non-broadcastable dimensions." + ) g_shape = [sizes.pop() if len(sizes) > 0 else 1 for sizes in g_shape] - g_masks = [[s > 1 or s == -1 for s in view_shape] - for view_shape in view_shapes] + g_masks = [ + [s > 1 or s == -1 for s in view_shape] for view_shape in view_shapes + ] return g_shape, g_masks @@ -295,8 +304,9 @@ def diagonalize(labels, operand): -------- 'ijj...i' would be merged into 'ij...' ''' - assert not has_duplicated_labels(labels), ( - 'Duplicate labels are not supported.') + assert not has_duplicated_labels( + labels + ), 'Duplicate labels are not supported.' return labels, operand @@ -356,12 +366,21 @@ def plan_matmul(plan, g_view, op1, op2, g_supports, g_shape, I, J1, J2, K): plan.add_step(step) # Check if conditions hold for turnning the operation into a matmul - if j1 + j2 > 0 and k > 0 and -1 not in np.concatenate( - (op1_vshape, op2_vshape)): - op1_shape = list(op1_vshape[I]) + [np.prod(op1_vshape[J1]) - ] + [np.prod(op1_vshape[K])] - op2_shape = list(op2_vshape[I]) + [np.prod(op2_vshape[J2]) - ] + [np.prod(op2_vshape[K])] + if ( + j1 + j2 > 0 + and k > 0 + and -1 not in np.concatenate((op1_vshape, op2_vshape)) + ): + op1_shape = ( + list(op1_vshape[I]) + + [np.prod(op1_vshape[J1])] + + [np.prod(op1_vshape[K])] + ) + op2_shape = ( + list(op2_vshape[I]) + + [np.prod(op2_vshape[J2])] + + [np.prod(op2_vshape[K])] + ) # Merge J dims and K dims by reshaping step = reshape, [var1], var1, op1_shape @@ -410,15 +429,22 @@ def plan_matmul(plan, g_view, op1, op2, g_supports, g_shape, I, J1, J2, K): step = squeeze, [var2], var2, [-1, -2] plan.add_step(step) elif j1 + j2 == 0 and -1 not in np.concatenate( - (op1_vshape[K], op2_vshape[K])): + (op1_vshape[K], op2_vshape[K]) + ): assert all(op1_vshape[K] == op2_vshape[K]) - step = reshape, [ - var1 - ], var1, list(op1_vshape[I]) + [1] + [np.prod(op1_vshape[K])] + step = ( + reshape, + [var1], + var1, + list(op1_vshape[I]) + [1] + [np.prod(op1_vshape[K])], + ) plan.add_step(step) - step = reshape, [ - var2 - ], var2, list(op2_vshape[I]) + [1] + [np.prod(op2_vshape[K])] + step = ( + reshape, + [var2], + var2, + list(op2_vshape[I]) + [1] + [np.prod(op2_vshape[K])], + ) plan.add_step(step) step = matmul, [var1, var2], var2, False, True plan.add_step(step) @@ -447,8 +473,9 @@ def plan_matmul(plan, g_view, op1, op2, g_supports, g_shape, I, J1, J2, K): g_view[op2] = list(op2_view) -def plan_summation(plan, g_view, op1, op2, g_supports, g_shape, g_count, - n_bcast): +def plan_summation( + plan, g_view, op1, op2, g_supports, g_shape, g_count, n_bcast +): ''' Plan various kinds of summation ''' @@ -462,8 +489,9 @@ def plan_summation(plan, g_view, op1, op2, g_supports, g_shape, g_count, I, K, J1, J2 = list(range(n_bcast)), [], [], [] - for ax, dim1, dim2 in zip(range(n_bcast, ndim), op1_view[n_bcast:], - op2_view[n_bcast:]): + for ax, dim1, dim2 in zip( + range(n_bcast, ndim), op1_view[n_bcast:], op2_view[n_bcast:] + ): if (dim1 != -1) != (dim2 != -1): if dim1 != -1: @@ -529,7 +557,6 @@ def plan_broadcast(plan, operands, nop_axes): class Plan: - def __init__(self): self.env = {} self.steps = [] @@ -633,8 +660,9 @@ def plan_einsum(operands, g_view, g_shape, g_supports, g_count, n_bcast): # op1 is a one element tensor. plan_scalar_prod(plan, i - 1, i) else: - plan_summation(plan, g_view, i - 1, i, g_supports, g_shape, g_count, - n_bcast) + plan_summation( + plan, g_view, i - 1, i, g_supports, g_shape, g_count, n_bcast + ) # for ax, dim in enumerate(g_view[nop-1][:nout]): # assert dim == ax @@ -676,7 +704,9 @@ def preprocess(equation, *operands): """ equation = equation.replace(" ", "") nop = len(operands) - assert nop > 0, "Required at least one operand in Einsum API, but received %s " % nop + assert nop > 0, ( + "Required at least one operand in Einsum API, but received %s " % nop + ) # Part the equation to left hand side and right hand side lhs, *rhs = equation.lower().split('->') @@ -690,16 +720,20 @@ def preprocess(equation, *operands): assert len(lhs.split(',')) == len(operands), ( f"Invalid equation: the number of operands is {len(operands)}, " - f"but found {len(lhs.split(','))} segments in the label equation.") + f"but found {len(lhs.split(','))} segments in the label equation." + ) - assert not ('...' in lhs and '...' not in rhs - ), 'Invalid equation: missing ellipsis in output labels.' + assert not ( + '...' in lhs and '...' not in rhs + ), 'Invalid equation: missing ellipsis in output labels.' - assert not (len(list(filter(has_duplicated_labels, lhs.split(',')))) > - 0), 'Duplicate labels are not supported.' + assert not ( + len(list(filter(has_duplicated_labels, lhs.split(',')))) > 0 + ), 'Duplicate labels are not supported.' assert not has_duplicated_labels( - rhs), 'Invalid equation: duplicate output labels are found.' + rhs + ), 'Invalid equation: duplicate output labels are found.' return lhs, rhs, labels @@ -717,10 +751,10 @@ def parse_fake_shape(equation, operands, labels): shaped = collections.namedtuple('shaped', ['shape']) def fake_shape(label, op): - assert len(op.shape) == len( - label - ), "length of shape and length of label must be the same, but received %d != %d" % ( - len(op.shape), len(label)) + assert len(op.shape) == len(label), ( + "length of shape and length of label must be the same, but received %d != %d" + % (len(op.shape), len(label)) + ) fakes = [s for i, (l, s) in enumerate(zip(label, op.shape)) if l != '.'] fakes = list(map(abs, fakes)) # make -1 -> 1 if '.' in label: @@ -732,7 +766,6 @@ def parse_fake_shape(equation, operands, labels): def rhs_inference(lhs): - def is_free(key): return cnt.get(key) == 1 and key not in ['.', ','] @@ -751,7 +784,8 @@ def gen_equation_for_opteinsum(lhs, rhs): def get_used_label(counter): used = set(counter.elements()) for c in string.ascii_lowercase: - if c not in used: return c + if c not in used: + return c raise ValueError( "You have used all `a` - `z`, there can't find a unused for einsum optimization" ) @@ -784,14 +818,15 @@ def einsum_v2(equation, *operands): var_list = list(operands) for path in cons: (a, b), _, eq, *__ = path - assert a > b, "Assume the first var_idx is smaller than the second_idx. opt_einsum can guarantee it." + assert ( + a > b + ), "Assume the first var_idx is smaller than the second_idx. opt_einsum can guarantee it." var_s = [var_list.pop(a), var_list.pop(b)] eq = eq.replace(broadcast_label, "...") var_list.append(gen_einsum_op(eq, *var_s)) - assert len( - var_list - ) == 1, "There must be one elements in list, but received %d." % len( - var_list) + assert ( + len(var_list) == 1 + ), "There must be one elements in list, but received %d." % len(var_list) return var_list[0] @@ -805,8 +840,9 @@ def gen_einsum_op(equation, *operands): if _in_legacy_dygraph(): # dygraph - return _legacy_C_ops.einsum(operands, len(operands), len(operands), - 'equation', equation)[0] + return _legacy_C_ops.einsum( + operands, len(operands), len(operands), 'equation', equation + )[0] for inp in operands: check_variable_and_dtype(inp, 'dtype', ['float32', 'float64'], 'einsum') @@ -823,14 +859,12 @@ def gen_einsum_op(equation, *operands): helper.create_variable_for_type_inference(dtype=operands[0].dtype) for i in range(len(operands)) ] - helper.append_op(type='einsum', - inputs={'Operands': operands}, - outputs={ - 'Out': out, - "InnerCache": caches, - "XShape": xshape - }, - attrs=attrs) + helper.append_op( + type='einsum', + inputs={'Operands': operands}, + outputs={'Out': out, "InnerCache": caches, "XShape": xshape}, + attrs=attrs, + ) return out @@ -992,6 +1026,7 @@ def einsum(equation, *operands): # [0.51476848, 0.23367381, 0.39229113]]]) """ import os + if int(os.environ.get('FLAGS_new_einsum', "1")): return einsum_v2(equation, *operands) @@ -1037,9 +1072,11 @@ def einsum(equation, *operands): # Counting how many non-trivial dimensions remain for each ax g_labels, g_view, g_nout, g_count = build_global_view( - nop_labels, rhs, n_bcast_dims) - g_shape, g_supports = build_global_shape(g_view, g_labels, - [op.shape for op in operands]) + nop_labels, rhs, n_bcast_dims + ) + g_shape, g_supports = build_global_shape( + g_view, g_labels, [op.shape for op in operands] + ) # Now we're ready to build up an execution plan args = operands, g_view, g_shape, g_supports, g_count, n_bcast_dims diff --git a/python/paddle/tensor/layer_function_generator.py b/python/paddle/tensor/layer_function_generator.py index e4b36ba5e14..fb2a611257e 100644 --- a/python/paddle/tensor/layer_function_generator.py +++ b/python/paddle/tensor/layer_function_generator.py @@ -19,7 +19,13 @@ import string from io import StringIO from ..static import Variable from ..fluid.proto import framework_pb2 -from ..framework import OpProtoHolder, _non_static_mode, convert_np_dtype_to_dtype_, core, in_dygraph_mode +from ..framework import ( + OpProtoHolder, + _non_static_mode, + convert_np_dtype_to_dtype_, + core, + in_dygraph_mode, +) from ..framework import LayerHelper from ..fluid.data_feeder import check_variable_and_dtype from paddle import _C_ops, _legacy_C_ops @@ -53,16 +59,16 @@ _two_bang_pattern_ = re.compile(r"!!([^!]+)!!") def escape_math(text): - #return _two_bang_pattern_.sub( + # return _two_bang_pattern_.sub( # r'$$\1$$', # _single_dollar_pattern_.sub(r':math:\n`\1`', # _two_dollar_pattern_.sub(r"!!\1!!", text))) return _two_dollar_pattern_.sub(r':math:`\1`', text) -def _generate_doc_string_(op_proto, - additional_args_lines=None, - skip_attrs_set=None): +def _generate_doc_string_( + op_proto, additional_args_lines=None, skip_attrs_set=None +): """ Generate docstring by OpProto @@ -142,23 +148,30 @@ def generate_layer_fn(op_type): """ op_proto = OpProtoHolder.instance().get_op_proto(op_type) - not_intermediate_outputs = \ - [output for output in op_proto.outputs if not output.intermediate] - intermediate_outputs = \ - [output for output in op_proto.outputs if output.intermediate] + not_intermediate_outputs = [ + output for output in op_proto.outputs if not output.intermediate + ] + intermediate_outputs = [ + output for output in op_proto.outputs if output.intermediate + ] if len(not_intermediate_outputs) != 1: - raise ValueError("Only one non intermediate output operator can be", - "automatically generated. {0}".format(op_type)) + raise ValueError( + "Only one non intermediate output operator can be", + "automatically generated. {0}".format(op_type), + ) if not_intermediate_outputs[0].duplicable: raise ValueError( - "Only non duplicable op can be automatically generated.") + "Only non duplicable op can be automatically generated." + ) for output in intermediate_outputs: if output.duplicable: - raise ValueError("The op can be automatically generated only when ", - "all intermediate ops are not duplicable.") + raise ValueError( + "The op can be automatically generated only when ", + "all intermediate ops are not duplicable.", + ) o_name = not_intermediate_outputs[0].name intermediate_output_names = [output.name for output in intermediate_outputs] @@ -183,14 +196,17 @@ def generate_layer_fn(op_type): for each in val: if not isinstance(each, Variable): raise ValueError( - "input of {0} must be variable".format(op_type)) + "input of {0} must be variable".format(op_type) + ) if dtype is None: dtype = each.dtype elif dtype != each.dtype: raise ValueError( "operator {0} must input same dtype. {1} vs {2}".format( - op_type, dtype, each.dtype)) + op_type, dtype, each.dtype + ) + ) if dtype is None: arg_dtype = kwargs.get("dtype") @@ -222,8 +238,11 @@ def generate_layer_fn(op_type): outputs = dict() out = kwargs.pop(_convert_(o_name), []) if out: - out_var = out[0] if (isinstance(out, list) - or isinstance(out, tuple)) else out + out_var = ( + out[0] + if (isinstance(out, list) or isinstance(out, tuple)) + else out + ) else: out_var = helper.create_variable_for_type_inference(dtype=dtype) outputs[o_name] = [out_var] @@ -231,10 +250,9 @@ def generate_layer_fn(op_type): outputs[name] = [ helper.create_variable_for_type_inference(dtype=dtype) ] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=kwargs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=kwargs + ) return helper.append_activation(out_var) func.__name__ = op_type @@ -265,14 +283,25 @@ def generate_activation_fn(op_type): return op(x) if op_type not in ["abs", "exp", "square"]: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - op_type) + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], op_type + ) else: # abs exp square ops support dtype(int32, int64, float16, float32, float64) - check_variable_and_dtype(x, 'x', [ - 'int32', 'int64', 'float16', 'float32', 'float64', 'complex64', - 'complex128' - ], op_type) + check_variable_and_dtype( + x, + 'x', + [ + 'int32', + 'int64', + 'float16', + 'float32', + 'float64', + 'complex64', + 'complex128', + ], + op_type, + ) helper = LayerHelper(op_type, **locals()) @@ -285,7 +314,8 @@ def generate_activation_fn(op_type): op_proto, additional_args_lines=[ "name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`." - ]) + ], + ) return func @@ -308,15 +338,19 @@ def generate_inplace_fn(inplace_op_type): op = getattr(_legacy_C_ops, inplace_op_type) return op(x) warnings.warn( - "In static mode, {}() is the same as {}() and does not perform inplace operation." - .format(inplace_op_type, origin_op_type)) + "In static mode, {}() is the same as {}() and does not perform inplace operation.".format( + inplace_op_type, origin_op_type + ) + ) return generate_activation_fn(origin_op_type)(x, name) func.__name__ = inplace_op_type func.__doc__ = """ Inplace version of ``{0}`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_fluid_layers_{1}`. -""".format(origin_op_type, origin_op_type) +""".format( + origin_op_type, origin_op_type + ) return func @@ -360,18 +394,21 @@ def templatedoc(op_type=None): for each_input in op_proto.inputs: input_name = _convert_(each_input.name) args["{0}_comment".format(input_name)] = trim_ending_dot( - each_input.comment) + each_input.comment + ) args["{0}_type".format(input_name)] = "Variable" for each_attr in op_proto.attrs: input_name = _convert_(each_attr.name) args["{0}_comment".format(input_name)] = trim_ending_dot( - each_attr.comment) + each_attr.comment + ) args["{0}_type".format(input_name)] = _type_to_str_(each_attr.type) for each_opt in op_proto.outputs: output_name = _convert_(each_opt.name) args["{0}_comment".format(output_name)] = trim_ending_dot( - each_opt.comment) + each_opt.comment + ) args["{0}_type".format(output_name)] = "Variable" func.__doc__ = tmpl.substitute(args) return func diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index d341deabbd2..3ed56a35dfa 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -15,7 +15,11 @@ import numpy as np from ..framework import LayerHelper from ..framework import _non_static_mode, in_dygraph_mode -from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype +from ..fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, +) from ..static import Variable from ..fluid.framework import _in_legacy_dygraph from .manipulation import cast @@ -90,10 +94,21 @@ def transpose(x, perm, name=None): out, _ = _legacy_C_ops.transpose2(x, 'axis', perm) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'transpose') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'transpose', + ) check_type(perm, 'perm', (list, tuple), 'transpose') if isinstance(perm, tuple): perm = list(perm) @@ -102,24 +117,25 @@ def transpose(x, perm, name=None): "Input(perm) is the permutation of dimensions of Input(x), " "its length should be equal to dimensions of Input(x), " "but received dimension of Input(x) is %s, " - "the length of Input(perm) is %s." % (len(x.shape), len(perm))) + "the length of Input(perm) is %s." % (len(x.shape), len(perm)) + ) for idx, dim in enumerate(perm): if dim >= len(x.shape): raise ValueError( "Each element in Input(perm) should be less than Input(x)'s dimension, " "but %d-th element in Input(perm) is %d which exceeds Input(x)'s " - "dimension %d." % (idx, perm[idx], len(x.shape))) + "dimension %d." % (idx, perm[idx], len(x.shape)) + ) helper = LayerHelper('transpose', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='transpose2', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'XShape': [x_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [x]}, + outputs={'Out': [out], 'XShape': [x_shape]}, + attrs={'axis': perm}, + ) return out @@ -234,21 +250,22 @@ def matmul(x, y, transpose_x=False, transpose_y=False, name=None): var_names = {'x': x, 'y': y} for name, val in var_names.items(): check_variable_and_dtype( - val, name, + val, + name, ['float16', 'float32', 'float64', 'complex64', 'complex128'], - 'matmul') + 'matmul', + ) __check_input(x, y) helper = LayerHelper('matmul_v2', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='matmul_v2', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='matmul_v2', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs=attrs, + ) return out @@ -358,32 +375,35 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): return _C_ops.frobenius_norm(input, dim, keepdim, False) if _in_legacy_dygraph(): if dim is None: - return _legacy_C_ops.frobenius_norm(input, 'keep_dim', keepdim, - 'reduce_all', True) - return _legacy_C_ops.frobenius_norm(input, 'dim', dim, 'keep_dim', - keepdim, 'reduce_all', False) + return _legacy_C_ops.frobenius_norm( + input, 'keep_dim', keepdim, 'reduce_all', True + ) + return _legacy_C_ops.frobenius_norm( + input, 'dim', dim, 'keep_dim', keepdim, 'reduce_all', False + ) attrs = {'dim': dim, 'keep_dim': keepdim, 'reduce_all': False} if dim is None: attrs['reduce_all'] = True - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'frobenius_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'frobenius_norm' + ) helper = LayerHelper('frobenius_norm', **locals()) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) - helper.append_op(type='frobenius_norm', - inputs={'X': input}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='frobenius_norm', + inputs={'X': input}, + outputs={'Out': out}, + attrs=attrs, + ) return out - def vector_norm(input, - porder=None, - axis=None, - keepdim=False, - asvector=False, - name=None): + def vector_norm( + input, porder=None, axis=None, keepdim=False, asvector=False, name=None + ): """ Calculate the p-order vector norm for certain dimension of Tensor `input`. Args: @@ -393,21 +413,32 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): keepdim (bool, optional): Whether keep the dimensions as the `input`, Default False. """ if in_dygraph_mode(): - if axis is None: axis = -1 + if axis is None: + axis = -1 return _C_ops.p_norm(input, porder, axis, 1e-12, keepdim, asvector) if _in_legacy_dygraph(): - if axis is None: axis = -1 - return _legacy_C_ops.p_norm(input, 'porder', porder, 'axis', axis, - 'keepdim', keepdim, 'asvector', - asvector) + if axis is None: + axis = -1 + return _legacy_C_ops.p_norm( + input, + 'porder', + porder, + 'axis', + axis, + 'keepdim', + keepdim, + 'asvector', + asvector, + ) if porder is not None: check_type(porder, 'porder', (float, int), 'p_norm') if axis is not None: check_type(axis, 'axis', (int), 'p_norm') - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'p_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'p_norm' + ) attrs = { 'axis': axis if axis is not None else -1, @@ -418,23 +449,27 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): } helper = LayerHelper('p_norm', **locals()) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) - helper.append_op(type='p_norm', - inputs={'X': input}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='p_norm', + inputs={'X': input}, + outputs={'Out': out}, + attrs=attrs, + ) return out - def inf_norm(input, - porder=None, - axis=axis, - keepdim=False, - asvector=False, - name=None): + def inf_norm( + input, porder=None, axis=axis, keepdim=False, asvector=False, name=None + ): if in_dygraph_mode(): out = _C_ops.abs(input) - reduce_all = True if axis == None or axis == [] or asvector == True else False + reduce_all = ( + True + if axis == None or axis == [] or asvector == True + else False + ) axis = axis if axis != None and axis != [] else [0] if reduce_all: assert (axis == []) or (axis is None) @@ -445,28 +480,31 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): helper = LayerHelper('inf_norm', **locals()) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) helper.append_op(type='abs', inputs={'X': input}, outputs={'Out': out}) reduce_out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) - reduce_all = True if axis == None or axis == [] or asvector == True else False + reduce_all = ( + True if axis == None or axis == [] or asvector == True else False + ) axis = axis if axis != None and axis != [] else [0] - reduce_type = 'reduce_max' if porder == np.float64( - 'inf') else 'reduce_min' - helper.append_op(type=reduce_type, - inputs={'X': out}, - outputs={'Out': reduce_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + reduce_type = ( + 'reduce_max' if porder == np.float64('inf') else 'reduce_min' + ) + helper.append_op( + type=reduce_type, + inputs={'X': out}, + outputs={'Out': reduce_out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return reduce_out - def p_matrix_norm(input, porder=1., axis=axis, keepdim=False, name=None): + def p_matrix_norm(input, porder=1.0, axis=axis, keepdim=False, name=None): """ NOTE: This function actually treats the matrix as flattened vector to calculate vector norm instead of matrix norm. @@ -475,38 +513,48 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): abs_out = _C_ops.abs(input) pow_out = _C_ops.pow(abs_out, porder) sum_out = _C_ops.sum(pow_out, axis, None, keepdim) - out = _C_ops.pow(sum_out, float(1. / porder)) + out = _C_ops.pow(sum_out, float(1.0 / porder)) return out block = LayerHelper('norm', **locals()) out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) abs_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='abs', - inputs={'X': input}, - outputs={'Out': abs_out}) + dtype=block.input_dtype() + ) + block.append_op( + type='abs', inputs={'X': input}, outputs={'Out': abs_out} + ) pow_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) - block.append_op(type='pow', - inputs={'X': abs_out}, - outputs={'Out': pow_out}, - attrs={'factor': porder}) + block.append_op( + type='pow', + inputs={'X': abs_out}, + outputs={'Out': pow_out}, + attrs={'factor': porder}, + ) sum_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='reduce_sum', - inputs={'X': pow_out}, - outputs={'Out': sum_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': True if axis is None else False - }) - block.append_op(type='pow', - inputs={'X': sum_out}, - outputs={'Out': out}, - attrs={'factor': float(1. / porder)}) + dtype=block.input_dtype() + ) + block.append_op( + type='reduce_sum', + inputs={'X': pow_out}, + outputs={'Out': sum_out}, + attrs={ + 'dim': axis, + 'keep_dim': keepdim, + 'reduce_all': True if axis is None else False, + }, + ) + block.append_op( + type='pow', + inputs={'X': sum_out}, + outputs={'Out': out}, + attrs={'factor': float(1.0 / porder)}, + ) return out if axis is None and p is not None: @@ -515,50 +563,60 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): return frobenius_norm(x, dim=axis, keepdim=keepdim, name=name) else: raise ValueError( - "only valid string values are 'fro', found {}".format(p)) + "only valid string values are 'fro', found {}".format(p) + ) elif isinstance(p, (int, float)): - return vector_norm(x, - porder=p, - axis=axis, - keepdim=keepdim, - asvector=True, - name=name) + return vector_norm( + x, + porder=p, + axis=axis, + keepdim=keepdim, + asvector=True, + name=name, + ) else: raise ValueError( - "only valid p type is string or float, found {}".format( - type(p))) + "only valid p type is string or float, found {}".format(type(p)) + ) if isinstance(axis, tuple): axis = list(axis) if isinstance(axis, list) and len(axis) == 1: axis = axis[0] - #calculate vector norm, where axis is int or list with only one integer + # calculate vector norm, where axis is int or list with only one integer if isinstance(axis, int): if isinstance(p, str): if p == "fro": - return vector_norm(x, - porder=2, - axis=axis, - keepdim=keepdim, - asvector=False, - name=name) + return vector_norm( + x, + porder=2, + axis=axis, + keepdim=keepdim, + asvector=False, + name=name, + ) else: raise ValueError( - "only valid string values are 'fro', found {}".format(p)) + "only valid string values are 'fro', found {}".format(p) + ) elif isinstance(p, (int, float)): - return vector_norm(x, - axis=axis, - porder=p, - keepdim=keepdim, - asvector=False, - name=name) + return vector_norm( + x, + axis=axis, + porder=p, + keepdim=keepdim, + asvector=False, + name=name, + ) else: raise ValueError( - "unspport p for p-order vector norm. except float, found {}". - format(p)) - #calculate matrix norm, where axis is list with two integers + "unspport p for p-order vector norm. except float, found {}".format( + p + ) + ) + # calculate matrix norm, where axis is list with two integers elif isinstance(axis, list) and len(axis) == 2: if p == "fro": return frobenius_norm(x, dim=axis, keepdim=keepdim, name=name) @@ -566,18 +624,20 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): return inf_norm(x, porder=p, axis=axis, keepdim=keepdim, name=name) elif p == 0: raise ValueError( - "just suport axis type int or list (length of list <=1) if p = 0, found {}" - .format(axis)) + "just suport axis type int or list (length of list <=1) if p = 0, found {}".format( + axis + ) + ) else: - return p_matrix_norm(x, - porder=p, - axis=axis, - keepdim=keepdim, - name=name) + return p_matrix_norm( + x, porder=p, axis=axis, keepdim=keepdim, name=name + ) else: raise ValueError( - "except axis type int or list (length of list <=2), found {}". - format(axis)) + "except axis type int or list (length of list <=2), found {}".format( + axis + ) + ) def dist(x, y, p=2, name=None): @@ -674,10 +734,9 @@ def dist(x, y, p=2, name=None): inputs = {"X": [x], "Y": [y]} outputs = {'Out': [out]} attrs = {"p": float(p)} - helper.append_op(type='dist', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='dist', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -768,7 +827,7 @@ def cond(x, p=None, name=None): """ - def mat_norm(input, porder=1., axis=None): + def mat_norm(input, porder=1.0, axis=None): """ NOTE: Calculate the matrix norm of a square matrix or batches of square matrices, @@ -789,54 +848,81 @@ def cond(x, p=None, name=None): elif _in_legacy_dygraph(): abs_out = _legacy_C_ops.abs(input) - sum_out = _legacy_C_ops.reduce_sum(abs_out, 'dim', axis, 'keepdim', - keepdim, 'reduce_all', - reduce_all) + sum_out = _legacy_C_ops.reduce_sum( + abs_out, + 'dim', + axis, + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) if porder == 1 or porder == np.inf: - return _legacy_C_ops.reduce_max(sum_out, 'dim', [-1], 'keepdim', - keepdim, 'reduce_all', - reduce_all) + return _legacy_C_ops.reduce_max( + sum_out, + 'dim', + [-1], + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) if porder == -1 or porder == -np.inf: - return _legacy_C_ops.reduce_min(sum_out, 'dim', [-1], 'keepdim', - keepdim, 'reduce_all', - reduce_all) + return _legacy_C_ops.reduce_min( + sum_out, + 'dim', + [-1], + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) else: block = LayerHelper('norm', **locals()) abs_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) sum_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='abs', - inputs={'X': input}, - outputs={'Out': abs_out}) - block.append_op(type='reduce_sum', - inputs={'X': abs_out}, - outputs={'Out': sum_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + dtype=block.input_dtype() + ) + block.append_op( + type='abs', inputs={'X': input}, outputs={'Out': abs_out} + ) + block.append_op( + type='reduce_sum', + inputs={'X': abs_out}, + outputs={'Out': sum_out}, + attrs={ + 'dim': axis, + 'keep_dim': keepdim, + 'reduce_all': reduce_all, + }, + ) if porder == 1 or porder == np.inf: - block.append_op(type='reduce_max', - inputs={'X': sum_out}, - outputs={'Out': out}, - attrs={ - 'dim': [-1], - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + block.append_op( + type='reduce_max', + inputs={'X': sum_out}, + outputs={'Out': out}, + attrs={ + 'dim': [-1], + 'keep_dim': keepdim, + 'reduce_all': reduce_all, + }, + ) if porder == -1 or porder == -np.inf: - block.append_op(type='reduce_min', - inputs={'X': sum_out}, - outputs={'Out': out}, - attrs={ - 'dim': [-1], - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + block.append_op( + type='reduce_min', + inputs={'X': sum_out}, + outputs={'Out': out}, + attrs={ + 'dim': [-1], + 'keep_dim': keepdim, + 'reduce_all': reduce_all, + }, + ) return out def fro_norm(input, porder=2, axis=[-1]): @@ -851,50 +937,66 @@ def cond(x, p=None, name=None): pow_out = _C_ops.pow(input, porder) sum_out_1 = _C_ops.sum(pow_out, axis, None, keepdim) sum_out_2 = _C_ops.sum(sum_out_1, axis, None, keepdim) - return _C_ops.pow(sum_out_2, float(1. / porder)) + return _C_ops.pow(sum_out_2, float(1.0 / porder)) elif paddle.in_dynamic_mode(): pow_out = _legacy_C_ops.pow(input, 'factor', porder) - sum_out_1 = _legacy_C_ops.reduce_sum(pow_out, 'dim', axis, - 'keepdim', keepdim, - 'reduce_all', reduce_all) - sum_out_2 = _legacy_C_ops.reduce_sum(sum_out_1, 'dim', axis, - 'keepdim', keepdim, - 'reduce_all', reduce_all) - return _legacy_C_ops.pow(sum_out_2, 'factor', float(1. / porder)) + sum_out_1 = _legacy_C_ops.reduce_sum( + pow_out, + 'dim', + axis, + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) + sum_out_2 = _legacy_C_ops.reduce_sum( + sum_out_1, + 'dim', + axis, + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) + return _legacy_C_ops.pow(sum_out_2, 'factor', float(1.0 / porder)) block = LayerHelper('norm', **locals()) pow_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) sum_out_1 = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) sum_out_2 = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='pow', - inputs={'X': input}, - outputs={'Out': pow_out}, - attrs={'factor': porder}) - block.append_op(type='reduce_sum', - inputs={'X': pow_out}, - outputs={'Out': sum_out_1}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) - block.append_op(type='reduce_sum', - inputs={'X': sum_out_1}, - outputs={'Out': sum_out_2}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) - block.append_op(type='pow', - inputs={'X': sum_out_2}, - outputs={'Out': out}, - attrs={'factor': float(1. / porder)}) + dtype=block.input_dtype() + ) + block.append_op( + type='pow', + inputs={'X': input}, + outputs={'Out': pow_out}, + attrs={'factor': porder}, + ) + block.append_op( + type='reduce_sum', + inputs={'X': pow_out}, + outputs={'Out': sum_out_1}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) + block.append_op( + type='reduce_sum', + inputs={'X': sum_out_1}, + outputs={'Out': sum_out_2}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) + block.append_op( + type='pow', + inputs={'X': sum_out_2}, + outputs={'Out': out}, + attrs={'factor': float(1.0 / porder)}, + ) return out def svd_norm(input, porder, axis=[-1]): @@ -913,9 +1015,15 @@ def cond(x, p=None, name=None): if in_dygraph_mode(): return _C_ops.sum(s, axis, None, keepdim) else: - return _legacy_C_ops.reduce_sum(s, 'dim', axis, 'keepdim', - keepdim, 'reduce_all', - reduce_all) + return _legacy_C_ops.reduce_sum( + s, + 'dim', + axis, + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) if in_dygraph_mode(): max_out = _C_ops.max(s, axis, keepdim) min_out = _C_ops.min(s, axis, keepdim) @@ -925,75 +1033,70 @@ def cond(x, p=None, name=None): return _C_ops.divide(min_out, max_out) else: - max_out = _legacy_C_ops.reduce_max(s, 'dim', axis, 'keepdim', - keepdim, 'reduce_all', - reduce_all) - min_out = _legacy_C_ops.reduce_min(s, 'dim', axis, 'keepdim', - keepdim, 'reduce_all', - reduce_all) + max_out = _legacy_C_ops.reduce_max( + s, 'dim', axis, 'keepdim', keepdim, 'reduce_all', reduce_all + ) + min_out = _legacy_C_ops.reduce_min( + s, 'dim', axis, 'keepdim', keepdim, 'reduce_all', reduce_all + ) if porder == 2: return _legacy_C_ops.elementwise_div( - max_out, min_out, 'aixs', axis, 'use_mkldnn', False) + max_out, min_out, 'aixs', axis, 'use_mkldnn', False + ) if porder == -2: return _legacy_C_ops.elementwise_div( - min_out, max_out, 'aixs', axis, 'use_mkldnn', False) + min_out, max_out, 'aixs', axis, 'use_mkldnn', False + ) block = LayerHelper('norm', **locals()) out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) if porder == "nuc": - block.append_op(type='reduce_sum', - inputs={'X': s}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + block.append_op( + type='reduce_sum', + inputs={'X': s}, + outputs={'Out': out}, + attrs={ + 'dim': axis, + 'keep_dim': keepdim, + 'reduce_all': reduce_all, + }, + ) return out max_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) min_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='reduce_max', - inputs={'X': s}, - outputs={'Out': max_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) - block.append_op(type='reduce_min', - inputs={'X': s}, - outputs={'Out': min_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + dtype=block.input_dtype() + ) + block.append_op( + type='reduce_max', + inputs={'X': s}, + outputs={'Out': max_out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) + block.append_op( + type='reduce_min', + inputs={'X': s}, + outputs={'Out': min_out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) if porder == 2: - block.append_op(type='elementwise_div', - inputs={ - 'X': max_out, - 'Y': min_out - }, - outputs={'Out': out}, - attrs={ - 'aixs': axis, - 'use_mkldnn': False - }) + block.append_op( + type='elementwise_div', + inputs={'X': max_out, 'Y': min_out}, + outputs={'Out': out}, + attrs={'aixs': axis, 'use_mkldnn': False}, + ) return out if porder == -2: - block.append_op(type='elementwise_div', - inputs={ - 'X': min_out, - 'Y': max_out - }, - outputs={'Out': out}, - attrs={ - 'aixs': axis, - 'use_mkldnn': False - }) + block.append_op( + type='elementwise_div', + inputs={'X': min_out, 'Y': max_out}, + outputs={'Out': out}, + attrs={'aixs': axis, 'use_mkldnn': False}, + ) return out def empty_tensor(input, shape): @@ -1004,8 +1107,9 @@ def cond(x, p=None, name=None): x_shape = list(x.shape) if not len(x_shape) >= 2: raise ValueError( - "input should be a matrix or batches of matrices, " + - "but the dimention of received input is {}".format(len(x_shape))) + "input should be a matrix or batches of matrices, " + + "but the dimention of received input is {}".format(len(x_shape)) + ) if p == None: p = 2 x_size = 0 if (0 in x_shape) else 1 @@ -1020,21 +1124,26 @@ def cond(x, p=None, name=None): return svd_norm(x, p) * svd_norm(x_inv, p) if p in (1, -1): return mat_norm(x, porder=p, axis=[-2]) * mat_norm( - x_inv, porder=p, axis=[-2]) + x_inv, porder=p, axis=[-2] + ) if p in (np.inf, -np.inf): return mat_norm(x, porder=p, axis=[-1]) * mat_norm( - x_inv, porder=p, axis=[-1]) + x_inv, porder=p, axis=[-1] + ) else: - raise ValueError("only support p is {} when input is a ".format(p) + - "square matrix or batches of square matrices") + raise ValueError( + "only support p is {} when input is a ".format(p) + + "square matrix or batches of square matrices" + ) elif p in (2, -2): if x_size == 0: return empty_tensor(x, x_shape[:-2]) return svd_norm(x, porder=p) else: raise ValueError( - "unsupported {} for p, only supporting ('fro', 'nuc', ".format(p) + - "1, -1, 2, -2, inf, -inf) or none") + "unsupported {} for p, only supporting ('fro', 'nuc', ".format(p) + + "1, -1, 2, -2, inf, -inf) or none" + ) def dot(x, y, name=None): @@ -1082,25 +1191,23 @@ def dot(x, y, name=None): assert x is not None, 'x cannot be None in {}'.format(op_type) assert y is not None, 'y cannot be None in {}'.format(op_type) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - op_type) - check_variable_and_dtype(y, 'y', ['float32', 'float64', 'int32', 'int64'], - op_type) + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], op_type + ) + check_variable_and_dtype( + y, 'y', ['float32', 'float64', 'int32', 'int64'], op_type + ) helper = LayerHelper(op_type, **locals()) if name is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: - out = helper.create_variable(name=name, - dtype=x.dtype, - persistable=False) - helper.append_op(type="dot", - inputs={ - 'X': x, - 'Y': y - }, - attrs={}, - outputs={"Out": out}) + out = helper.create_variable( + name=name, dtype=x.dtype, persistable=False + ) + helper.append_op( + type="dot", inputs={'X': x, 'Y': y}, attrs={}, outputs={"Out": out} + ) return out @@ -1143,7 +1250,8 @@ def cov(x, rowvar=True, ddof=True, fweights=None, aweights=None, name=None): if len(x.shape) > 2 or len(x.shape) < 1: raise ValueError( "Input(x) only support N-D (1<=N<=2) tensor in cov, but received " - "length of Input(input) is %s." % len(x.shape)) + "length of Input(input) is %s." % len(x.shape) + ) check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'cov') nx = x if len(x.shape) == 1: @@ -1157,16 +1265,20 @@ def cov(x, rowvar=True, ddof=True, fweights=None, aweights=None, name=None): if len(w.shape) > 1: raise ValueError( "Input(fweights) only support N-D (N<=1) tensor in cov, but received " - "shape of Input(input) is %s." % len(fweights.shape)) + "shape of Input(input) is %s." % len(fweights.shape) + ) if fweights.shape[0] != observation_num: raise ValueError( "The number of Input(fweights) should equal to x's dim[1]: {}, but received " - "size of Input(fweights) is {}.".format(observation_num, - fweights.shape[0])) + "size of Input(fweights) is {}.".format( + observation_num, fweights.shape[0] + ) + ) if fweights.min() < 0: raise ValueError( "The value of Input(fweights) cannot be negtive, but received " - "min of Input(fweights) is {}.".format(fweights.min())) + "min of Input(fweights) is {}.".format(fweights.min()) + ) if not paddle.all(fweights == paddle.round(fweights.astype('float64'))): raise ValueError("Input(fweights) must be integer ") @@ -1175,18 +1287,23 @@ def cov(x, rowvar=True, ddof=True, fweights=None, aweights=None, name=None): if len(aw.shape) > 1: raise ValueError( "Input(aweights) only support N-D (N<=1) tensor in cov, but received " - "length of Input(input) is %s." % len(aweights.shape)) - check_variable_and_dtype(aweights, 'dtype', ['float32', 'float64'], - 'cov') + "length of Input(input) is %s." % len(aweights.shape) + ) + check_variable_and_dtype( + aweights, 'dtype', ['float32', 'float64'], 'cov' + ) if aweights.shape[0] != observation_num: raise ValueError( "The number of Input(aweights) should equal to x's dim[1]: {}, but received " - "size of Input(aweights) is {}.".format(observation_num, - aweights.shape[0])) + "size of Input(aweights) is {}.".format( + observation_num, aweights.shape[0] + ) + ) if aweights.min() < 0: raise ValueError( "The value of Input(aweights) cannot be negtive, but received " - "min of Input(aweights) is {}.".format(aweights.min())) + "min of Input(aweights) is {}.".format(aweights.min()) + ) if w is not None: w = w * aw else: @@ -1260,7 +1377,8 @@ def t(input, name=None): raise ValueError( "Input(input) only support N-D (N<=2) tensor, but received " "length of Input(input) is %s. Perhaps you can use paddle." - "tensor.transpose() instead." % len(input.shape)) + "tensor.transpose() instead." % len(input.shape) + ) if in_dygraph_mode(): if len(input.shape) == 1: return input @@ -1278,8 +1396,11 @@ def t(input, name=None): return out check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'transpose') + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'transpose', + ) helper = LayerHelper('t', **locals()) out = helper.create_variable_for_type_inference(input.dtype) @@ -1287,13 +1408,12 @@ def t(input, name=None): if len(input.shape) == 1: out = input else: - helper.append_op(type='transpose2', - inputs={'X': [input]}, - outputs={ - 'Out': [out], - 'XShape': [input_shape] - }, - attrs={'axis': [1, 0]}) + helper.append_op( + type='transpose2', + inputs={'X': [input]}, + outputs={'Out': [out], 'XShape': [input_shape]}, + attrs={'axis': [1, 0]}, + ) return out @@ -1350,13 +1470,12 @@ def cross(x, y, axis=9, name=None): attrs = dict() attrs['dim'] = axis - helper.append_op(type='cross', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='cross', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs=attrs, + ) return out @@ -1411,10 +1530,12 @@ def cholesky(x, upper=False, name=None): check_type(upper, 'upper', bool, 'cholesky') helper = LayerHelper('cholesky', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='cholesky', - inputs={'X': [x]}, - outputs={'Out': out}, - attrs={'upper': upper}) + helper.append_op( + type='cholesky', + inputs={'X': [x]}, + outputs={'Out': out}, + attrs={'upper': upper}, + ) return out @@ -1464,8 +1585,9 @@ def matrix_rank(x, tol=None, hermitian=False, name=None): else: tol_tensor = tol use_default_tol = False - return _C_ops.matrix_rank_tol(x, tol_tensor, use_default_tol, - hermitian) + return _C_ops.matrix_rank_tol( + x, tol_tensor, use_default_tol, hermitian + ) if tol is None: tol_attr = 0.0 @@ -1491,9 +1613,16 @@ def matrix_rank(x, tol=None, hermitian=False, name=None): tol_tensor = None tol_attr = float(tol) use_default_tol = False - return _legacy_C_ops.matrix_rank(x, tol_tensor, "tol", tol_attr, - 'hermitian', hermitian, - 'use_default_tol', use_default_tol) + return _legacy_C_ops.matrix_rank( + x, + tol_tensor, + "tol", + tol_attr, + 'hermitian', + hermitian, + 'use_default_tol', + use_default_tol, + ) inputs = {} attrs = {} @@ -1516,10 +1645,9 @@ def matrix_rank(x, tol=None, hermitian=False, name=None): helper = LayerHelper('matrix_rank', **locals()) out = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type='matrix_rank', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='matrix_rank', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -1566,16 +1694,22 @@ def bmm(x, y, name=None): y_shape = y.shape if not len(x_shape) == len(y_shape) == 3: raise ValueError( - "x and y should be 3-dimensional. But received x's dimention: {}, y's dimention: {}" - .format(x_shape, y_shape)) + "x and y should be 3-dimensional. But received x's dimention: {}, y's dimention: {}".format( + x_shape, y_shape + ) + ) if x_shape[2] != y_shape[1]: raise ValueError( - "x's width must be equal with y's height. But received x's shape: {}, y's shape: {}" - .format(x_shape, y_shape)) + "x's width must be equal with y's height. But received x's shape: {}, y's shape: {}".format( + x_shape, y_shape + ) + ) if x_shape[0] != y_shape[0]: raise ValueError( - "x's batch (shape[0]) must be equal with y's batch (shape[0]). But received x's shape: {}, y's shape: {}" - .format(x_shape, y_shape)) + "x's batch (shape[0]) must be equal with y's batch (shape[0]). But received x's shape: {}, y's shape: {}".format( + x_shape, y_shape + ) + ) if in_dygraph_mode(): return _C_ops.bmm(x, y) @@ -1618,22 +1752,21 @@ def histogram(input, bins=100, min=0, max=0, name=None): return _C_ops.histogram(input, bins, min, max) if _in_legacy_dygraph(): - return _legacy_C_ops.histogram(input, "bins", bins, "min", min, "max", - max) + return _legacy_C_ops.histogram( + input, "bins", bins, "min", min, "max", max + ) helper = LayerHelper('histogram', **locals()) - check_variable_and_dtype(input, 'X', - ['int32', 'int64', 'float32', 'float64'], - 'histogram') + check_variable_and_dtype( + input, 'X', ['int32', 'int64', 'float32', 'float64'], 'histogram' + ) out = helper.create_variable_for_type_inference(VarDesc.VarType.INT64) - helper.append_op(type='histogram', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'bins': bins, - 'min': min, - 'max': max - }) + helper.append_op( + type='histogram', + inputs={'X': input}, + outputs={'Out': out}, + attrs={'bins': bins, 'min': min, 'max': max}, + ) return out @@ -1677,19 +1810,21 @@ def bincount(x, weights=None, minlength=0, name=None): check_variable_and_dtype(x, 'X', ['int32', 'int64'], 'bincount') if weights is not None: - check_variable_and_dtype(weights, 'Weights', - ['int32', 'int64', 'float32', 'float64'], - 'bincount') + check_variable_and_dtype( + weights, + 'Weights', + ['int32', 'int64', 'float32', 'float64'], + 'bincount', + ) out = helper.create_variable_for_type_inference(dtype=weights.dtype) else: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='bincount', - inputs={ - 'X': x, - 'Weights': weights - }, - outputs={'Out': out}, - attrs={'minlength': minlength}) + helper.append_op( + type='bincount', + inputs={'X': x, 'Weights': weights}, + outputs={'Out': out}, + attrs={'minlength': minlength}, + ) return out @@ -1734,29 +1869,31 @@ def mv(x, vec, name=None): def __check_input(x, vec): var_names = {'x': x, 'vec': vec} for name, val in var_names.items(): - check_variable_and_dtype(val, name, ['float32', 'float64'], - 'mv') + check_variable_and_dtype( + val, name, ['float32', 'float64'], 'mv' + ) x_shape = list(x.shape) vec_shape = list(vec.shape) if len(x_shape) != 2: raise ValueError( - "x should be 2-dimensional. But received x's dimention: {}" - .format(x_shape)) + "x should be 2-dimensional. But received x's dimention: {}".format( + x_shape + ) + ) if len(vec_shape) != 1: raise ValueError( - "vec should be 1-dimensional. But received vec's dimention: {}" - .format(vec_shape)) + "vec should be 1-dimensional. But received vec's dimention: {}".format( + vec_shape + ) + ) __check_input(x, vec) helper = LayerHelper('mv', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='mv', - inputs={ - 'X': x, - 'Vec': vec - }, - outputs={'Out': out}) + helper.append_op( + type='mv', inputs={'X': x, 'Vec': vec}, outputs={'Out': out} + ) return out @@ -1796,22 +1933,23 @@ def det(x, name=None): check_dtype(x.dtype, 'Input', ['float32', 'float64'], 'det') input_shape = list(x.shape) - assert len(input_shape) >= 2, \ - "The x must be at least 2-dimensional, " \ - "but received Input x's dimensional: %s.\n" % \ - len(input_shape) - - assert (input_shape[-1] == input_shape[-2]), \ - "Expect squared input," \ - "but received %s by %s matrix.\n" \ - %(input_shape[-2], input_shape[-1]) \ + assert len(input_shape) >= 2, ( + "The x must be at least 2-dimensional, " + "but received Input x's dimensional: %s.\n" % len(input_shape) + ) + assert ( + input_shape[-1] == input_shape[-2] + ), "Expect squared input," "but received %s by %s matrix.\n" % ( + input_shape[-2], + input_shape[-1], + ) helper = LayerHelper('determinant', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='determinant', - inputs={'Input': [x]}, - outputs={'Out': [out]}) + helper.append_op( + type='determinant', inputs={'Input': [x]}, outputs={'Out': [out]} + ) return out @@ -1855,22 +1993,23 @@ def slogdet(x, name=None): check_dtype(x.dtype, 'Input', ['float32', 'float64'], 'slogdet') input_shape = list(x.shape) - assert len(input_shape) >= 2, \ - "The x must be at least 2-dimensional, " \ - "but received Input x's dimensional: %s.\n" % \ - len(input_shape) - - assert (input_shape[-1] == input_shape[-2]), \ - "Expect squared input," \ - "but received %s by %s matrix.\n" \ - %(input_shape[-2], input_shape[-1]) \ + assert len(input_shape) >= 2, ( + "The x must be at least 2-dimensional, " + "but received Input x's dimensional: %s.\n" % len(input_shape) + ) + assert ( + input_shape[-1] == input_shape[-2] + ), "Expect squared input," "but received %s by %s matrix.\n" % ( + input_shape[-2], + input_shape[-1], + ) helper = LayerHelper('slogdeterminant', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='slogdeterminant', - inputs={'Input': [x]}, - outputs={'Out': [out]}) + helper.append_op( + type='slogdeterminant', inputs={'Input': [x]}, outputs={'Out': [out]} + ) return out @@ -1937,11 +2076,7 @@ def svd(x, full_matrices=False, name=None): helper.append_op( type='svd', inputs={'X': [x]}, - outputs={ - 'U': u, - 'VH': vh, - 'S': s - }, + outputs={'U': u, 'VH': vh, 'S': s}, attrs=attrs, ) return u, s, vh @@ -2010,10 +2145,12 @@ def matrix_power(x, n, name=None): check_type(n, 'n', int, 'matrix_power') helper = LayerHelper('matrix_power', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='matrix_power', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'n': n}) + helper.append_op( + type='matrix_power', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'n': n}, + ) return out @@ -2078,13 +2215,9 @@ def qr(x, mode="reduced", name=None): r = helper.create_variable_for_type_inference(dtype=x.dtype) attrs = dict() attrs['mode'] = mode - helper.append_op(type='qr', - inputs={'X': [x]}, - outputs={ - 'Q': q, - 'R': r - }, - attrs=attrs) + helper.append_op( + type='qr', inputs={'X': [x]}, outputs={'Q': q, 'R': r}, attrs=attrs + ) if mode == "r": return r else: @@ -2183,14 +2316,12 @@ def lu(x, pivot=True, get_infos=False, name=None): info = helper.create_variable_for_type_inference(dtype='int') attrs = dict() attrs['pivot'] = pivot - helper.append_op(type='lu', - inputs={'X': x}, - outputs={ - 'Out': lu, - 'Pivots': p, - 'Infos': info - }, - attrs=attrs) + helper.append_op( + type='lu', + inputs={'X': x}, + outputs={'Out': lu, 'Pivots': p, 'Infos': info}, + attrs=attrs, + ) if get_infos: return lu, p, info else: @@ -2275,8 +2406,9 @@ def lu_unpack(x, y, unpack_ludata=True, unpack_pivots=True, name=None): return P, L, U if paddle.in_dynamic_mode(): - P, L, U = _legacy_C_ops.lu_unpack(x, y, 'unpack_ludata', unpack_ludata, - 'unpack_pivots', unpack_pivots) + P, L, U = _legacy_C_ops.lu_unpack( + x, y, 'unpack_ludata', unpack_ludata, 'unpack_pivots', unpack_pivots + ) return P, L, U check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'lu_unpack') @@ -2288,17 +2420,12 @@ def lu_unpack(x, y, unpack_ludata=True, unpack_pivots=True, name=None): attrs = dict() attrs['unpack_ludata'] = unpack_ludata attrs['unpack_pivots'] = unpack_pivots - helper.append_op(type='lu_unpack', - inputs={ - 'X': x, - 'Pivots': y - }, - outputs={ - 'Pmat': p, - 'L': l, - 'U': u - }, - attrs=attrs) + helper.append_op( + type='lu_unpack', + inputs={'X': x, 'Pivots': y}, + outputs={'Pmat': p, 'L': l, 'U': u}, + attrs=attrs, + ) return p, l, u @@ -2354,9 +2481,9 @@ def eig(x, name=None): w, v = _legacy_C_ops.eig(x) return w, v - check_variable_and_dtype(x, 'X', - ['float32', 'float64', 'complex64', 'complex128'], - 'eig') + check_variable_and_dtype( + x, 'X', ['float32', 'float64', 'complex64', 'complex128'], 'eig' + ) helper = LayerHelper('eig', **locals()) w = helper.create_variable_for_type_inference(x.dtype) @@ -2405,20 +2532,24 @@ def eigvals(x, name=None): # [(-0.27078833542132674+0j), (0.29962280156230725+0j), (0.8824477020120244+0j)] #complex128 """ - check_variable_and_dtype(x, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'eigvals') + check_variable_and_dtype( + x, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], 'eigvals' + ) x_shape = list(x.shape) if len(x_shape) < 2: raise ValueError( - "The dimension of Input(x) should be at least 2, but received x's dimention = {}, x's shape = {}" - .format(len(x_shape), x_shape)) + "The dimension of Input(x) should be at least 2, but received x's dimention = {}, x's shape = {}".format( + len(x_shape), x_shape + ) + ) if x_shape[-1] != x_shape[-2]: raise ValueError( - "The last two dimensions of Input(x) should be equal, but received x's shape = {}" - .format(x_shape)) + "The last two dimensions of Input(x) should be equal, but received x's shape = {}".format( + x_shape + ) + ) if in_dygraph_mode(): return _C_ops.eigvals(x) @@ -2494,11 +2625,16 @@ def multi_dot(x, name=None): check_type(x, 'x', (list, tuple), 'multi_dot') for id, item in enumerate(x): - check_variable_and_dtype(item, 'x[' + str(id) + ']', - ['float16', 'float32', 'float64'], 'multi_dot') + check_variable_and_dtype( + item, + 'x[' + str(id) + ']', + ['float16', 'float32', 'float64'], + 'multi_dot', + ) if item.dtype != x[0].dtype: raise TypeError( - "All the Tensors in the input must have the same data type.") + "All the Tensors in the input must have the same data type." + ) helper = LayerHelper('multi_dot', **locals()) dtype = helper.input_dtype(input_param_name='x') @@ -2551,32 +2687,35 @@ def eigh(x, UPLO='L', name=None): if len(x.shape) < 2: raise ValueError( "Input(input) only support >=2 tensor, but received " - "length of Input(input) is %s." % len(x.shape)) + "length of Input(input) is %s." % len(x.shape) + ) if x_shape[-1] != x_shape[-2]: raise ValueError( - "The input matrix must be batches of square matrices. But received x's dimention: {}" - .format(x_shape)) + "The input matrix must be batches of square matrices. But received x's dimention: {}".format( + x_shape + ) + ) if UPLO != 'L' and UPLO != 'U': raise ValueError( - "UPLO must be L or U. But received UPLO is: {}".format(UPLO)) + "UPLO must be L or U. But received UPLO is: {}".format(UPLO) + ) __check_input(x, UPLO) helper = LayerHelper('eigh', **locals()) - check_variable_and_dtype(x, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'eigh') + check_variable_and_dtype( + x, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], 'eigh' + ) out_value = helper.create_variable_for_type_inference(dtype=x.dtype) out_vector = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='eigh', - inputs={'X': x}, - outputs={ - 'Eigenvalues': out_value, - 'Eigenvectors': out_vector - }, - attrs={'UPLO': UPLO}) + helper.append_op( + type='eigh', + inputs={'X': x}, + outputs={'Eigenvalues': out_value, 'Eigenvectors': out_vector}, + attrs={'UPLO': UPLO}, + ) return out_value, out_vector @@ -2695,8 +2834,9 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): if not hermitian: # combine svd and matmul op u, s, vt = _legacy_C_ops.svd(x, 'full_matrices', False) - max_singular_val = _legacy_C_ops.reduce_max(s, 'dim', [-1], 'keep_dim', True, \ - 'reduce_all', False) + max_singular_val = _legacy_C_ops.reduce_max( + s, 'dim', [-1], 'keep_dim', True, 'reduce_all', False + ) rcond = paddle.to_tensor(rcond, dtype=x.dtype) cutoff = rcond * max_singular_val y = float('inf') @@ -2718,15 +2858,17 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): if in_dygraph_mode(): out_2 = _C_ops.matmul(out_1, u, False, True) else: - out_2 = _legacy_C_ops.matmul_v2(out_1, u, 'trans_x', False, - 'trans_y', True) + out_2 = _legacy_C_ops.matmul_v2( + out_1, u, 'trans_x', False, 'trans_y', True + ) return out_2 else: # combine eigh and matmul op s, u = _legacy_C_ops.eigh(x, 'UPLO', 'L') s_abs = paddle.abs(s) - max_singular_val = _legacy_C_ops.reduce_max(s_abs, 'dim', [-1], 'keep_dim', True, \ - 'reduce_all', False) + max_singular_val = _legacy_C_ops.reduce_max( + s_abs, 'dim', [-1], 'keep_dim', True, 'reduce_all', False + ) rcond = paddle.to_tensor(rcond, dtype=s.dtype) cutoff = rcond * max_singular_val y = float('inf') @@ -2745,8 +2887,9 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): if in_dygraph_mode(): out_2 = _C_ops.matmul(out_1, u_conj, False, True) else: - out_2 = _legacy_C_ops.matmul_v2(out_1, u_conj, 'trans_x', False, - 'trans_y', True) + out_2 = _legacy_C_ops.matmul_v2( + out_1, u_conj, 'trans_x', False, 'trans_y', True + ) return out_2 else: if not hermitian: @@ -2760,23 +2903,17 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): helper.append_op( type='svd', inputs={'X': [x]}, - outputs={ - 'U': u, - 'VH': vt, - 'S': s - }, + outputs={'U': u, 'VH': vt, 'S': s}, attrs={'full_matrices': False}, ) max_singular_val = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='reduce_max', - inputs={'X': s}, - outputs={'Out': max_singular_val}, - attrs={ - 'dim': [-1], - 'keep_dim': True, - 'reduce_all': False - }) + helper.append_op( + type='reduce_max', + inputs={'X': s}, + outputs={'Out': max_singular_val}, + attrs={'dim': [-1], 'keep_dim': True, 'reduce_all': False}, + ) rcond = full(shape=[1], fill_value=rcond, dtype=dtype) cutoff = rcond * max_singular_val @@ -2792,59 +2929,50 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): st = helper.create_variable_for_type_inference(dtype=dtype) st_shape = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='unsqueeze2', - inputs={'X': singular}, - attrs={'axes': [-2]}, - outputs={ - 'Out': st, - 'XShape': st_shape - }) + helper.append_op( + type='unsqueeze2', + inputs={'X': singular}, + attrs={'axes': [-2]}, + outputs={'Out': st, 'XShape': st_shape}, + ) dims = list(range(len(vt.shape))) perm = dims[:-2] + [dims[-1]] + [dims[-2]] v = helper.create_variable_for_type_inference(dtype) v_shape = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='transpose2', - inputs={'X': [vt]}, - outputs={ - 'Out': [v], - 'XShape': [v_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [vt]}, + outputs={'Out': [v], 'XShape': [v_shape]}, + attrs={'axis': perm}, + ) out_1 = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_mul', - inputs={ - 'X': v, - 'Y': st - }, - outputs={'Out': out_1}, - attrs={ - 'axis': -1, - 'use_mkldnn': False - }) + helper.append_op( + type='elementwise_mul', + inputs={'X': v, 'Y': st}, + outputs={'Out': out_1}, + attrs={'axis': -1, 'use_mkldnn': False}, + ) out_1 = helper.append_activation(out_1) out_2 = helper.create_variable_for_type_inference(dtype) helper.append_op( type='matmul_v2', - inputs={ - 'X': out_1, - 'Y': u - }, + inputs={'X': out_1, 'Y': u}, outputs={'Out': out_2}, - attrs={ - 'trans_x': False, - 'trans_y': True - }, + attrs={'trans_x': False, 'trans_y': True}, ) return out_2 else: helper = LayerHelper('pinv', **locals()) dtype = x.dtype check_variable_and_dtype( - x, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], - 'pinv') + x, + 'dtype', + ['float32', 'float64', 'complex64', 'complex128'], + 'pinv', + ) if dtype == paddle.complex128: s_type = 'float64' @@ -2855,26 +2983,23 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): u = helper.create_variable_for_type_inference(dtype) s = helper.create_variable_for_type_inference(s_type) - helper.append_op(type='eigh', - inputs={'X': x}, - outputs={ - 'Eigenvalues': s, - 'Eigenvectors': u - }, - attrs={'UPLO': 'L'}) + helper.append_op( + type='eigh', + inputs={'X': x}, + outputs={'Eigenvalues': s, 'Eigenvectors': u}, + attrs={'UPLO': 'L'}, + ) s_abs = helper.create_variable_for_type_inference(s_type) - helper.append_op(type='abs', - inputs={'X': s}, - outputs={'Out': s_abs}) + helper.append_op( + type='abs', inputs={'X': s}, outputs={'Out': s_abs} + ) max_singular_val = helper.create_variable_for_type_inference(s_type) - helper.append_op(type='reduce_max', - inputs={'X': s_abs}, - outputs={'Out': max_singular_val}, - attrs={ - 'dim': [-1], - 'keep_dim': True, - 'reduce_all': False - }) + helper.append_op( + type='reduce_max', + inputs={'X': s_abs}, + outputs={'Out': max_singular_val}, + attrs={'dim': [-1], 'keep_dim': True, 'reduce_all': False}, + ) rcond = full(shape=[1], fill_value=rcond, dtype=s_type) cutoff = rcond * max_singular_val @@ -2890,44 +3015,33 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): st = helper.create_variable_for_type_inference(dtype=s_type) st_shape = helper.create_variable_for_type_inference(dtype=s_type) - helper.append_op(type='unsqueeze2', - inputs={'X': singular}, - attrs={'axes': [-2]}, - outputs={ - 'Out': st, - 'XShape': st_shape - }) + helper.append_op( + type='unsqueeze2', + inputs={'X': singular}, + attrs={'axes': [-2]}, + outputs={'Out': st, 'XShape': st_shape}, + ) out_1 = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_mul', - inputs={ - 'X': u, - 'Y': st - }, - outputs={'Out': out_1}, - attrs={ - 'axis': -1, - 'use_mkldnn': False - }) + helper.append_op( + type='elementwise_mul', + inputs={'X': u, 'Y': st}, + outputs={'Out': out_1}, + attrs={'axis': -1, 'use_mkldnn': False}, + ) out_1 = helper.append_activation(out_1) u_conj = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='conj', - inputs={'X': u}, - outputs={'Out': [u_conj]}) + helper.append_op( + type='conj', inputs={'X': u}, outputs={'Out': [u_conj]} + ) out_2 = helper.create_variable_for_type_inference(dtype) helper.append_op( type='matmul_v2', - inputs={ - 'X': out_1, - 'Y': u_conj - }, + inputs={'X': out_1, 'Y': u_conj}, outputs={'Out': out_2}, - attrs={ - 'trans_x': False, - 'trans_y': True - }, + attrs={'trans_x': False, 'trans_y': True}, ) return out_2 @@ -2984,21 +3098,15 @@ def solve(x, y, name=None): check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'solve') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="solve", - inputs={ - "X": x, - "Y": y - }, - outputs={"Out": out}) + helper.append_op( + type="solve", inputs={"X": x, "Y": y}, outputs={"Out": out} + ) return out -def triangular_solve(x, - y, - upper=True, - transpose=False, - unitriangular=False, - name=None): +def triangular_solve( + x, y, upper=True, transpose=False, unitriangular=False, name=None +): r""" Computes the solution of a system of equations with a triangular coefficient. `x` is coefficient matrix `y` is multiple right-hand sides of equations. @@ -3054,9 +3162,16 @@ def triangular_solve(x, return _C_ops.triangular_solve(x, y, upper, transpose, unitriangular) if paddle.in_dynamic_mode(): - return _legacy_C_ops.triangular_solve(x, y, 'upper', upper, 'transpose', - transpose, 'unitriangular', - unitriangular) + return _legacy_C_ops.triangular_solve( + x, + y, + 'upper', + upper, + 'transpose', + transpose, + 'unitriangular', + unitriangular, + ) inputs = {"X": [x], "Y": [y]} helper = LayerHelper("triangular_solve", **locals()) @@ -3064,17 +3179,16 @@ def triangular_solve(x, check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'triangular_solve') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='triangular_solve', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={ - 'upper': upper, - 'transpose': transpose, - 'unitriangular': unitriangular - }) + helper.append_op( + type='triangular_solve', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={ + 'upper': upper, + 'transpose': transpose, + 'unitriangular': unitriangular, + }, + ) return out @@ -3122,13 +3236,12 @@ def cholesky_solve(x, y, upper=False, name=None): check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'cholesky_solve') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='cholesky_solve', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={'upper': upper}) + helper.append_op( + type='cholesky_solve', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={'upper': upper}, + ) return out @@ -3172,36 +3285,39 @@ def eigvalsh(x, UPLO='L', name=None): if len(x.shape) < 2: raise ValueError( "Input(input) only support >=2 tensor, but received " - "length of Input(input) is %s." % len(x.shape)) + "length of Input(input) is %s." % len(x.shape) + ) if x_shape[-1] != x_shape[-2]: raise ValueError( - "The input matrix must be batches of square matrices. But received x's dimention: {}" - .format(x_shape)) + "The input matrix must be batches of square matrices. But received x's dimention: {}".format( + x_shape + ) + ) if UPLO != 'L' and UPLO != 'U': raise ValueError( - "UPLO must be L or U. But received UPLO is: {}".format(UPLO)) + "UPLO must be L or U. But received UPLO is: {}".format(UPLO) + ) __check_input(x, UPLO) helper = LayerHelper('eigvalsh', **locals()) - check_variable_and_dtype(x, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'eigvalsh') + check_variable_and_dtype( + x, + 'dtype', + ['float32', 'float64', 'complex64', 'complex128'], + 'eigvalsh', + ) out_value = helper.create_variable_for_type_inference(dtype=x.dtype) out_vector = helper.create_variable_for_type_inference(dtype=x.dtype) is_test = x.stop_gradient - helper.append_op(type='eigvalsh', - inputs={'X': x}, - outputs={ - 'Eigenvalues': out_value, - 'Eigenvectors': out_vector - }, - attrs={ - 'UPLO': UPLO, - 'is_test': is_test - }) + helper.append_op( + type='eigvalsh', + inputs={'X': x}, + outputs={'Eigenvalues': out_value, 'Eigenvectors': out_vector}, + attrs={'UPLO': UPLO, 'is_test': is_test}, + ) return out_value @@ -3268,14 +3384,18 @@ def lstsq(x, y, rcond=None, driver=None, name=None): if device == "cpu": if driver not in (None, "gels", "gelss", "gelsd", "gelsy"): raise ValueError( - "Only support valid driver is 'gels', 'gelss', 'gelsd', 'gelsy' or None for CPU inputs. But got {}" - .format(driver)) + "Only support valid driver is 'gels', 'gelss', 'gelsd', 'gelsy' or None for CPU inputs. But got {}".format( + driver + ) + ) driver = "gelsy" if driver is None else driver elif "gpu" in device: if driver not in (None, "gels"): raise ValueError( - "Only support valid driver is 'gels' or None for CUDA inputs. But got {}" - .format(driver)) + "Only support valid driver is 'gels' or None for CUDA inputs. But got {}".format( + driver + ) + ) driver = "gels" if driver is None else driver else: raise RuntimeError("Only support lstsq api for CPU or CUDA device.") @@ -3296,10 +3416,12 @@ def lstsq(x, y, rcond=None, driver=None, name=None): if _non_static_mode(): if in_dygraph_mode(): solution, residuals, rank, singular_values = _C_ops.lstsq( - x, y, rcond, driver) + x, y, rcond, driver + ) else: solution, residuals, rank, singular_values = _legacy_C_ops.lstsq( - x, y, 'rcond', rcond, 'driver', driver) + x, y, 'rcond', rcond, 'driver', driver + ) if driver == "gels": rank = paddle.empty(shape=[0], dtype=paddle.int32) @@ -3310,33 +3432,29 @@ def lstsq(x, y, rcond=None, driver=None, name=None): return solution, residuals, rank, singular_values helper = LayerHelper('lstsq', **locals()) - check_variable_and_dtype(x, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'lstsq') - check_variable_and_dtype(y, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'lstsq') + check_variable_and_dtype( + x, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], 'lstsq' + ) + check_variable_and_dtype( + y, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], 'lstsq' + ) solution = helper.create_variable_for_type_inference(dtype=x.dtype) residuals = helper.create_variable_for_type_inference(dtype=x.dtype) rank = helper.create_variable_for_type_inference(dtype=paddle.int32) singular_values = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='lstsq', - inputs={ - 'X': x, - 'Y': y - }, - outputs={ - 'Solution': solution, - 'Residuals': residuals, - 'Rank': rank, - 'SingularValues': singular_values - }, - attrs={ - 'rcond': rcond, - 'driver': driver - }) + helper.append_op( + type='lstsq', + inputs={'X': x, 'Y': y}, + outputs={ + 'Solution': solution, + 'Residuals': residuals, + 'Rank': rank, + 'SingularValues': singular_values, + }, + attrs={'rcond': rcond, 'driver': driver}, + ) if driver == "gels": rank = paddle.static.data(name='rank', shape=[0]) @@ -3388,11 +3506,12 @@ def corrcoef(x, rowvar=True, name=None): if len(x.shape) > 2 or len(x.shape) < 1: raise ValueError( "Input(x) only support N-D (1<=N<=2) tensor in corrcoef, but received " - "length of Input(input) is %s." % len(x.shape)) + "length of Input(input) is %s." % len(x.shape) + ) check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'corrcoef') c = cov(x, rowvar) - if (c.ndim == 0): + if c.ndim == 0: # scalar covariance # nan if incorrect value (nan, inf, 0), 1 otherwise return c / c @@ -3407,8 +3526,9 @@ def corrcoef(x, rowvar=True, name=None): # Clip to [-1, 1]. This does not guarantee if paddle.is_complex(c): - return paddle.complex(paddle.clip(c.real(), -1, 1), - paddle.clip(c.imag(), -1, 1)) + return paddle.complex( + paddle.clip(c.real(), -1, 1), paddle.clip(c.imag(), -1, 1) + ) else: c = paddle.clip(c, -1, 1) diff --git a/python/paddle/tensor/logic.py b/python/paddle/tensor/logic.py index 6945ab3fd3f..bd7052153f6 100755 --- a/python/paddle/tensor/logic.py +++ b/python/paddle/tensor/logic.py @@ -16,8 +16,10 @@ import paddle from ..fluid.data_feeder import check_type, check_variable_and_dtype from .layer_function_generator import templatedoc from ..static import Variable + # TODO: define logic functions of a tensor from ..fluid.framework import _in_eager_mode_ + if _in_eager_mode_: Tensor = paddle.fluid.framework.core.eager.Tensor else: @@ -26,6 +28,7 @@ else: from ..framework import in_dygraph_mode from ..framework import LayerHelper from ..fluid.framework import _in_legacy_dygraph + # TODO: define logic functions of a tensor from paddle import _C_ops, _legacy_C_ops from paddle.tensor.creation import full @@ -47,14 +50,18 @@ def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): else: return op(x) check_variable_and_dtype( - x, "x", + x, + "x", ["bool", "int8", "int16", "int32", "int64", "float32", "float64"], - op_name) + op_name, + ) if y is not None: check_variable_and_dtype( - y, "y", + y, + "y", ["bool", "int8", "int16", "int32", "int64", "float32", "float64"], - op_name) + op_name, + ) if out is not None: check_type(out, "out", Variable, op_name) @@ -63,18 +70,16 @@ def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): if binary_op and x.dtype != y.dtype: raise ValueError( "(InvalidArgument) The DataType of %s Op's Variable must be consistent, but received %s and %s." - % (op_name, x.dtype, y.dtype)) + % (op_name, x.dtype, y.dtype) + ) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) if binary_op: - helper.append_op(type=op_name, - inputs={ - "X": x, - "Y": y - }, - outputs={"Out": out}) + helper.append_op( + type=op_name, inputs={"X": x, "Y": y}, outputs={"Out": out} + ) else: helper.append_op(type=op_name, inputs={"X": x}, outputs={"Out": out}) @@ -116,12 +121,9 @@ def logical_and(x, y, out=None, name=None): if in_dygraph_mode(): return _C_ops.logical_and(x, y) - return _logical_op(op_name="logical_and", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_and", x=x, y=y, name=name, out=out, binary_op=True + ) def logical_or(x, y, out=None, name=None): @@ -161,12 +163,9 @@ def logical_or(x, y, out=None, name=None): """ if in_dygraph_mode(): return _C_ops.logical_or(x, y) - return _logical_op(op_name="logical_or", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_or", x=x, y=y, name=name, out=out, binary_op=True + ) def logical_xor(x, y, out=None, name=None): @@ -207,12 +206,9 @@ def logical_xor(x, y, out=None, name=None): if in_dygraph_mode(): return _C_ops.logical_xor(x, y) - return _logical_op(op_name="logical_xor", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_xor", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -245,12 +241,9 @@ def logical_not(x, out=None, name=None): """ if in_dygraph_mode(): return _C_ops.logical_not(x) - return _logical_op(op_name="logical_not", - x=x, - y=None, - name=name, - out=out, - binary_op=False) + return _logical_op( + op_name="logical_not", x=x, y=None, name=name, out=out, binary_op=False + ) def is_empty(x, name=None): @@ -288,16 +281,17 @@ def is_empty(x, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.is_empty(x) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'is_empty') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'is_empty' + ) check_type(name, "name", (str, type(None)), "is_empty") helper = LayerHelper("is_empty", **locals()) cond = helper.create_variable_for_type_inference(dtype='bool') cond.stop_gradient = True - helper.append_op(type='is_empty', - inputs={'X': [x]}, - outputs={'Out': [cond]}) + helper.append_op( + type='is_empty', inputs={'X': [x]}, outputs={'Out': [cond]} + ) return cond @@ -338,12 +332,9 @@ def equal_all(x, y, name=None): helper = LayerHelper("equal_all", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') - helper.append_op(type='equal_all', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='equal_all', inputs={'X': [x], 'Y': [y]}, outputs={'Out': [out]} + ) return out @@ -393,8 +384,9 @@ def allclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None): if in_dygraph_mode(): return _C_ops.allclose(x, y, rtol, atol, equal_nan) if _in_legacy_dygraph(): - return _legacy_C_ops.allclose(x, y, 'rtol', str(rtol), 'atol', - str(atol), 'equal_nan', equal_nan) + return _legacy_C_ops.allclose( + x, y, 'rtol', str(rtol), 'atol', str(atol), 'equal_nan', equal_nan + ) check_variable_and_dtype(x, "input", ['float32', 'float64'], 'allclose') check_variable_and_dtype(y, "input", ['float32', 'float64'], 'allclose') check_type(rtol, 'rtol', float, 'allclose') @@ -407,10 +399,9 @@ def allclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None): inputs = {'Input': x, 'Other': y} outputs = {'Out': out} attrs = {'rtol': str(rtol), 'atol': str(atol), 'equal_nan': equal_nan} - helper.append_op(type='allclose', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='allclose', inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -446,8 +437,10 @@ def equal(x, y, name=None): """ if not isinstance(y, (int, bool, float, Variable)): raise TypeError( - "Type of input args must be float, bool, int or Tensor, but received type {}" - .format(type(y))) + "Type of input args must be float, bool, int or Tensor, but received type {}".format( + type(y) + ) + ) if not isinstance(y, Variable): y = full(shape=[1], dtype=x.dtype, fill_value=y) @@ -459,21 +452,26 @@ def equal(x, y, name=None): return _legacy_C_ops.equal(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "equal") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "equal", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "equal") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "equal", + ) helper = LayerHelper("equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -511,21 +509,26 @@ def greater_equal(x, y, name=None): return _legacy_C_ops.greater_equal(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "greater_equal") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "greater_equal", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "greater_equal") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "greater_equal", + ) helper = LayerHelper("greater_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='greater_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='greater_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -562,21 +565,26 @@ def greater_than(x, y, name=None): return _legacy_C_ops.greater_than(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "greater_than") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "greater_than", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "greater_than") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "greater_than", + ) helper = LayerHelper("greater_than", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='greater_than', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='greater_than', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -615,21 +623,26 @@ def less_equal(x, y, name=None): return _legacy_C_ops.less_equal(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "less_equal") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "less_equal", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "less_equal") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "less_equal", + ) helper = LayerHelper("less_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='less_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='less_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -668,21 +681,26 @@ def less_than(x, y, name=None): return _legacy_C_ops.less_than(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "less_than") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "less_than", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "less_than") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "less_than", + ) helper = LayerHelper("less_than", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='less_than', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='less_than', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -721,21 +739,26 @@ def not_equal(x, y, name=None): return _legacy_C_ops.not_equal(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "not_equal") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "not_equal", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "not_equal") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "not_equal", + ) helper = LayerHelper("not_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='not_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='not_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -782,11 +805,15 @@ def _bitwise_op(op_name, x, y, out=None, name=None, binary_op=True): return op(x) check_variable_and_dtype( - x, "x", ["bool", "uint8", "int8", "int16", "int32", "int64"], op_name) + x, "x", ["bool", "uint8", "int8", "int16", "int32", "int64"], op_name + ) if y is not None: check_variable_and_dtype( - y, "y", ["bool", "uint8", "int8", "int16", "int32", "int64"], - op_name) + y, + "y", + ["bool", "uint8", "int8", "int16", "int32", "int64"], + op_name, + ) if out is not None: check_type(out, "out", Variable, op_name) @@ -798,12 +825,9 @@ def _bitwise_op(op_name, x, y, out=None, name=None, binary_op=True): out = helper.create_variable_for_type_inference(dtype=x.dtype) if binary_op: - helper.append_op(type=op_name, - inputs={ - "X": x, - "Y": y - }, - outputs={"Out": out}) + helper.append_op( + type=op_name, inputs={"X": x, "Y": y}, outputs={"Out": out} + ) else: helper.append_op(type=op_name, inputs={"X": x}, outputs={"Out": out}) @@ -834,12 +858,9 @@ def bitwise_and(x, y, out=None, name=None): """ if in_dygraph_mode() and out is None: return _C_ops.bitwise_and(x, y) - return _bitwise_op(op_name="bitwise_and", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _bitwise_op( + op_name="bitwise_and", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -867,12 +888,9 @@ def bitwise_or(x, y, out=None, name=None): if in_dygraph_mode() and out is None: return _C_ops.bitwise_or(x, y) - return _bitwise_op(op_name="bitwise_or", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _bitwise_op( + op_name="bitwise_or", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -899,12 +917,9 @@ def bitwise_xor(x, y, out=None, name=None): """ if in_dygraph_mode() and out is None: return _C_ops.bitwise_xor(x, y) - return _bitwise_op(op_name="bitwise_xor", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _bitwise_op( + op_name="bitwise_xor", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -930,12 +945,9 @@ def bitwise_not(x, out=None, name=None): if in_dygraph_mode() and out is None: return _C_ops.bitwise_not(x) - return _bitwise_op(op_name="bitwise_not", - x=x, - y=None, - name=name, - out=out, - binary_op=False) + return _bitwise_op( + op_name="bitwise_not", x=x, y=None, name=name, out=out, binary_op=False + ) @templatedoc() @@ -982,8 +994,9 @@ def isclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None): if in_dygraph_mode(): return _C_ops.isclose(x, y, rtol, atol, equal_nan) if _in_legacy_dygraph(): - return _legacy_C_ops.isclose(x, y, 'rtol', str(rtol), 'atol', str(atol), - 'equal_nan', equal_nan) + return _legacy_C_ops.isclose( + x, y, 'rtol', str(rtol), 'atol', str(atol), 'equal_nan', equal_nan + ) check_variable_and_dtype(x, "input", ['float32', 'float64'], 'isclose') check_variable_and_dtype(y, "input", ['float32', 'float64'], 'isclose') @@ -997,8 +1010,7 @@ def isclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None): inputs = {'Input': x, 'Other': y} outputs = {'Out': out} attrs = {'rtol': str(rtol), 'atol': str(atol), 'equal_nan': equal_nan} - helper.append_op(type='isclose', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='isclose', inputs=inputs, outputs=outputs, attrs=attrs + ) return out diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 485e18f5064..27e40992334 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -19,9 +19,15 @@ from ..framework import core, in_dygraph_mode from ..fluid.framework import _in_legacy_dygraph, _non_static_mode from ..framework import LayerHelper from ..framework import convert_np_dtype_to_dtype_, dygraph_only -from ..fluid.data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..fluid.data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from ..fluid.layers import utils import numpy as np + # TODO: define functions to manipulate a tensor from ..fluid.dygraph.inplace_utils import inplace_apis_in_dygraph_only import paddle @@ -70,25 +76,50 @@ def cast(x, dtype): out = _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', - 'uint8', 'uint16' - ], 'cast') - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8', 'uint16' - ], 'cast') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'cast', + ) + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'cast', + ) helper = LayerHelper('cast', **locals()) out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=x.stop_gradient) - helper.append_op(type='cast', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': out.dtype - }) + dtype=dtype, stop_gradient=x.stop_gradient + ) + helper.append_op( + type='cast', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'in_dtype': x.dtype, 'out_dtype': out.dtype}, + ) return out @@ -169,7 +200,8 @@ def slice(input, axes, starts, ends): axes = list(axes) if len(axes) == 0: raise ValueError( - "Input axes should not be an empty list/tuple.") + "Input axes should not be an empty list/tuple." + ) for i in range(len(axes)): if axes[i] < 0: axes[i] = max(0, axes[i] + len(input.shape)) @@ -178,8 +210,10 @@ def slice(input, axes, starts, ends): else: raise ValueError( - "Input axes must be a python list or tuple, but reveived {}". - format(type(axes))) + "Input axes must be a python list or tuple, but reveived {}".format( + type(axes) + ) + ) infer_flags = list(1 for i in range(len(axes))) @@ -188,7 +222,8 @@ def slice(input, axes, starts, ends): if isinstance(starts, (list, tuple)): starts = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in starts ] elif isinstance(starts, tmp_tensor_type): @@ -199,7 +234,9 @@ def slice(input, axes, starts, ends): if isinstance(ends, (list, tuple)): ends = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item for item in ends + if isinstance(item, tmp_tensor_type) + else item + for item in ends ] elif isinstance(ends, tmp_tensor_type): tensor_t = ends.numpy() @@ -217,7 +254,8 @@ def slice(input, axes, starts, ends): axes = list(axes) if len(axes) == 0: raise ValueError( - "Input axes should not be an empty list/tuple.") + "Input axes should not be an empty list/tuple." + ) for i in range(len(axes)): if axes[i] < 0: axes[i] = max(0, axes[i] + len(input.shape)) @@ -226,8 +264,10 @@ def slice(input, axes, starts, ends): else: raise ValueError( - "Input axes must be a python list or tuple, but reveived {}" - .format(type(axes))) + "Input axes must be a python list or tuple, but reveived {}".format( + type(axes) + ) + ) infer_flags = list(1 for i in range(len(axes))) @@ -236,7 +276,8 @@ def slice(input, axes, starts, ends): if isinstance(starts, (list, tuple)): starts = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in starts ] attrs += ('starts', starts) @@ -248,7 +289,8 @@ def slice(input, axes, starts, ends): if isinstance(ends, (list, tuple)): ends = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in ends ] attrs += ('ends', ends) @@ -257,16 +299,27 @@ def slice(input, axes, starts, ends): ends_tensor.stop_gradient = True infer_flags = list(-1 for i in range(len(axes))) - return _legacy_C_ops.slice(input, starts_tensor, ends_tensor, None, - None, 'axes', axes, 'infer_flags', - infer_flags, *attrs) + return _legacy_C_ops.slice( + input, + starts_tensor, + ends_tensor, + None, + None, + 'axes', + axes, + 'infer_flags', + infer_flags, + *attrs, + ) if not isinstance(starts, (list, tuple, Variable)): raise ValueError( - "Input starts must be an Variable, python list or tuple.") + "Input starts must be an Variable, python list or tuple." + ) if not isinstance(ends, (list, tuple, Variable)): raise ValueError( - "Input ends must be an Variable, python list or tuple.") + "Input ends must be an Variable, python list or tuple." + ) helper = LayerHelper('slice', **locals()) @@ -313,11 +366,11 @@ def slice(input, axes, starts, ends): # infer_flags attrs['infer_flags'] = infer_flags out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('input')) - helper.append_op(type='slice', - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + dtype=helper.input_dtype('input') + ) + helper.append_op( + type='slice', inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out @@ -379,10 +432,21 @@ def transpose(x, perm, name=None): out, _ = _legacy_C_ops.transpose2(x, 'axis', perm) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'transpose') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'transpose', + ) check_type(perm, 'perm', (list, tuple), 'transpose') if isinstance(perm, tuple): perm = list(perm) @@ -391,24 +455,25 @@ def transpose(x, perm, name=None): "Input(perm) is the permutation of dimensions of Input(x), " "its length should be equal to dimensions of Input(x), " "but received dimension of Input(x) is %s, " - "the length of Input(perm) is %s." % (len(x.shape), len(perm))) + "the length of Input(perm) is %s." % (len(x.shape), len(perm)) + ) for idx, dim in enumerate(perm): if dim >= len(x.shape): raise ValueError( "Each element in Input(perm) should be less than Input(x)'s dimension, " "but %d-th element in Input(perm) is %d which exceeds Input(x)'s " - "dimension %d." % (idx, perm[idx], len(x.shape))) + "dimension %d." % (idx, perm[idx], len(x.shape)) + ) helper = LayerHelper('transpose', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='transpose2', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'XShape': [x_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [x]}, + outputs={'Out': [out], 'XShape': [x_shape]}, + attrs={'axis': perm}, + ) return out @@ -462,13 +527,12 @@ def unstack(x, axis=0, num=None): for _ in range(num): outs.append(helper.create_variable_for_type_inference(x.dtype)) - helper.append_op(type='unstack', - inputs={'X': [x]}, - outputs={'Y': outs}, - attrs={ - 'axis': axis, - 'num': num - }) + helper.append_op( + type='unstack', + inputs={'X': [x]}, + outputs={'Y': outs}, + attrs={'axis': axis, 'num': num}, + ) return outs @@ -519,27 +583,31 @@ def shard_index(input, index_num, nshards, shard_id, ignore_value=-1): # [[-1], [1]] """ if in_dygraph_mode(): - return _C_ops.shard_index(input, index_num, nshards, shard_id, - ignore_value) + return _C_ops.shard_index( + input, index_num, nshards, shard_id, ignore_value + ) check_variable_and_dtype(input, 'input', ['int64', 'int32'], 'shard_index') op_type = 'shard_index' helper = LayerHelper(op_type, **locals()) if shard_id < 0 or shard_id >= nshards: - raise ValueError('The shard_id(%d) should be in [0, %d)' % - (shard_id, nshards)) + raise ValueError( + 'The shard_id(%d) should be in [0, %d)' % (shard_id, nshards) + ) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type=op_type, - inputs={'X': [input]}, - outputs={'Out': out}, - attrs={ - 'index_num': index_num, - 'nshards': nshards, - 'shard_id': shard_id, - 'ignore_value': ignore_value - }, - stop_gradient=True) + helper.append_op( + type=op_type, + inputs={'X': [input]}, + outputs={'Out': out}, + attrs={ + 'index_num': index_num, + 'nshards': nshards, + 'shard_id': shard_id, + 'ignore_value': ignore_value, + }, + stop_gradient=True, + ) return out @@ -631,12 +699,15 @@ def crop(x, shape=None, offsets=None, name=None): """ helper = LayerHelper('crop_tensor', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'crop_tensor') - check_type(shape, 'shape', (list, tuple, Variable, type(None)), - 'crop_tensor') - check_type(offsets, 'offsets', (list, tuple, Variable, type(None)), - 'crop_tensor') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'crop_tensor' + ) + check_type( + shape, 'shape', (list, tuple, Variable, type(None)), 'crop_tensor' + ) + check_type( + offsets, 'offsets', (list, tuple, Variable, type(None)), 'crop_tensor' + ) if offsets is None: offsets = [0] * len(x.shape) @@ -655,25 +726,30 @@ def crop(x, shape=None, offsets=None, name=None): if not isinstance(shape_val, int): raise TypeError( "Attr(shape)'s dtype of Op(crop_tensor) should be int32, but received: %s." - % type(shape_val)) + % type(shape_val) + ) if shape_val == 0: raise ValueError( "Attr(shape) of Op(crop_tensor) should not be zero, but received: %s." - % str(shape_val)) + % str(shape_val) + ) if shape_val < -1: raise ValueError( "When the element in Attr(shape) of Op(crop_tensor) is negative, only -1 is supported, but received: %s." - % str(shape_val)) + % str(shape_val) + ) def _attr_offsets_check(offset_val): if not isinstance(offset_val, int): raise TypeError( "Attr(offsets)'s dtype of Op(crop_tensor) should be int32, but received: %s." - % type(offset_val)) + % type(offset_val) + ) if offset_val < 0: raise ValueError( "Attr(offsets) of Op(crop_tensor) should be greater or equal to zero, but received: %s." - % str(offset_val)) + % str(offset_val) + ) if isinstance(offsets, Variable): offsets.stop_gradient = True @@ -714,11 +790,9 @@ def crop(x, shape=None, offsets=None, name=None): else: _attr_shape_check(dim_size) temp_out = helper.create_variable_for_type_inference('int32') - fill_constant([1], - 'int32', - dim_size, - force_cpu=True, - out=temp_out) + fill_constant( + [1], 'int32', dim_size, force_cpu=True, out=temp_out + ) new_shape_tensor.append(temp_out) shape_attr.append(dim_size) ipts['ShapeTensor'] = new_shape_tensor @@ -728,10 +802,12 @@ def crop(x, shape=None, offsets=None, name=None): _attr_shape_check(dim_size) attrs['shape'] = shape - helper.append_op(type='crop_tensor', - inputs=ipts, - outputs={'Out': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='crop_tensor', + inputs=ipts, + outputs={'Out': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out @@ -763,13 +839,15 @@ def fill_(x, value): """ if not isinstance(value, (float, int)): raise TypeError( - "The type of 'value' must be int or float, but received %s." % - (type(value))) + "The type of 'value' must be int or float, but received %s." + % (type(value)) + ) if in_dygraph_mode(): return _C_ops.fill_(x, value) else: - return _legacy_C_ops.fill_any_(x, "value_float", float(value), - "value_int", int(value)) + return _legacy_C_ops.fill_any_( + x, "value_float", float(value), "value_int", int(value) + ) @dygraph_only @@ -798,10 +876,11 @@ def zero_(x): """ if in_dygraph_mode(): - return _C_ops.fill_(x, 0.) + return _C_ops.fill_(x, 0.0) else: - return _legacy_C_ops.fill_any_(x, "value_float", 0., "value_int", - int(0)) + return _legacy_C_ops.fill_any_( + x, "value_float", 0.0, "value_int", int(0) + ) @dygraph_only @@ -833,39 +912,45 @@ def fill_diagonal_(x, value, offset=0, wrap=False, name=None): helper = LayerHelper("fill_diagonal_", **locals()) check_type(x, 'X', (Variable), 'fill_diagonal_') dtype = helper.input_dtype('x') - check_dtype(dtype, 'X', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'fill_diagonal_') + check_dtype( + dtype, + 'X', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'fill_diagonal_', + ) check_type(value, 'value', (bool, int, float), 'fill_diagonal_') check_type(wrap, 'wrap', (bool), 'fill_diagonal_') inshape = x.shape inshapeset = set(inshape) - assert len(inshape) >= 2, ('Tensor dims should >= 2 in fill_diagonal_ API') + assert len(inshape) >= 2, 'Tensor dims should >= 2 in fill_diagonal_ API' if len(inshape) > 2: - assert len(inshapeset) == 1, ( - 'Tensor dims should be equal while input dims > 2 in fill_diagonal_ API' - ) + assert ( + len(inshapeset) == 1 + ), 'Tensor dims should be equal while input dims > 2 in fill_diagonal_ API' if in_dygraph_mode(): if len(inshape) == 2: return _C_ops.fill_diagonal_(x, value, offset, wrap) return _C_ops.fill_diagonal_(x, value, offset, True) if len(inshape) == 2: - return _legacy_C_ops.fill_diagonal_(x, 'value', value, 'offset', offset, - 'wrap', wrap) - return _legacy_C_ops.fill_diagonal_(x, 'value', value, 'offset', offset, - 'wrap', True) + return _legacy_C_ops.fill_diagonal_( + x, 'value', value, 'offset', offset, 'wrap', wrap + ) + return _legacy_C_ops.fill_diagonal_( + x, 'value', value, 'offset', offset, 'wrap', True + ) def _fill_diagonal_tensor_impl(x, y, offset=0, dim1=0, dim2=1, inplace=False): inshape = x.shape - assert dim1 < len(inshape) and dim1 >= -len(inshape), ( - 'dim1 should between [-rank,rank) in fill_diagonal_tensor_') - assert dim2 < len(inshape) and dim2 >= -len(inshape), ( - 'dim2 should between [-rank,rank) in fill_diagonal_tensor_') - assert len(inshape) >= 2, ( - 'Tensor dims should >= 2 in fill_diagonal_tensor_') + assert dim1 < len(inshape) and dim1 >= -len( + inshape + ), 'dim1 should between [-rank,rank) in fill_diagonal_tensor_' + assert dim2 < len(inshape) and dim2 >= -len( + inshape + ), 'dim2 should between [-rank,rank) in fill_diagonal_tensor_' + assert len(inshape) >= 2, 'Tensor dims should >= 2 in fill_diagonal_tensor_' dim1 %= len(inshape) dim2 %= len(inshape) @@ -873,11 +958,14 @@ def _fill_diagonal_tensor_impl(x, y, offset=0, dim1=0, dim2=1, inplace=False): for i in range(len(inshape)): if i != dim1 and i != dim2: predshape.append(inshape[i]) - diaglen = min(min(inshape[dim1], inshape[dim1] + offset), - min(inshape[dim2], inshape[dim2] - offset)) + diaglen = min( + min(inshape[dim1], inshape[dim1] + offset), + min(inshape[dim2], inshape[dim2] - offset), + ) predshape.append(diaglen) assert tuple(predshape) == tuple( - y.shape), ("the y shape should be {}".format(predshape)) + y.shape + ), "the y shape should be {}".format(predshape) if len(y.shape) == 1: y = y.reshape([1, -1]) @@ -885,14 +973,15 @@ def _fill_diagonal_tensor_impl(x, y, offset=0, dim1=0, dim2=1, inplace=False): if in_dygraph_mode(): return _C_ops.fill_diagonal_tensor_(x, y, offset, dim1, dim2) else: - return _legacy_C_ops.fill_diagonal_tensor_(x, y, 'offset', offset, - 'dim1', dim1, 'dim2', - dim2) + return _legacy_C_ops.fill_diagonal_tensor_( + x, y, 'offset', offset, 'dim1', dim1, 'dim2', dim2 + ) if in_dygraph_mode(): return _C_ops.fill_diagonal_tensor(x, y, offset, dim1, dim2) else: - return _legacy_C_ops.fill_diagonal_tensor(x, y, 'offset', offset, - 'dim1', dim1, 'dim2', dim2) + return _legacy_C_ops.fill_diagonal_tensor( + x, y, 'offset', offset, 'dim1', dim1, 'dim2', dim2 + ) def fill_diagonal_tensor_(x, y, offset=0, dim1=0, dim2=1, name=None): @@ -924,12 +1013,9 @@ def fill_diagonal_tensor_(x, y, offset=0, dim1=0, dim2=1, name=None): print(x.tolist()) #[[1.0, 2.0, 2.0], [2.0, 1.0, 2.0], [2.0, 2.0, 1.0], [2.0, 2.0, 2.0]] """ - return _fill_diagonal_tensor_impl(x, - y, - offset=offset, - dim1=dim1, - dim2=dim2, - inplace=True) + return _fill_diagonal_tensor_impl( + x, y, offset=offset, dim1=dim1, dim2=dim2, inplace=True + ) def fill_diagonal_tensor(x, y, offset=0, dim1=0, dim2=1, name=None): @@ -958,12 +1044,9 @@ def fill_diagonal_tensor(x, y, offset=0, dim1=0, dim2=1, name=None): print(nx.tolist()) #[[1.0, 2.0, 2.0], [2.0, 1.0, 2.0], [2.0, 2.0, 1.0], [2.0, 2.0, 2.0]] """ - return _fill_diagonal_tensor_impl(x, - y, - offset=offset, - dim1=dim1, - dim2=dim2, - inplace=False) + return _fill_diagonal_tensor_impl( + x, y, offset=offset, dim1=dim1, dim2=dim2, inplace=False + ) @dygraph_only @@ -1062,10 +1145,21 @@ def concat(x, axis=0, name=None): check_type(input, 'input', (list, tuple, Variable), 'concat') if not isinstance(input, Variable): for id, x in enumerate(input): - check_variable_and_dtype(x, 'input[' + str(id) + ']', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', - 'int8', 'unit8' - ], 'concat') + check_variable_and_dtype( + x, + 'input[' + str(id) + ']', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'unit8', + ], + 'concat', + ) if x.dtype != input[0].dtype: raise TypeError( "All the Tensors in the input must have the same data type." @@ -1076,8 +1170,11 @@ def concat(x, axis=0, name=None): if isinstance(axis, Variable): check_dtype( - axis.dtype, 'axis', ['int32', 'int64'], 'concat', - "The data type of axis must be int32 or int64 when axis is a Tensor" + axis.dtype, + 'axis', + ['int32', 'int64'], + 'concat', + "The data type of axis must be int32 or int64 when axis is a Tensor", ) helper = LayerHelper('concat', **locals()) @@ -1088,19 +1185,17 @@ def concat(x, axis=0, name=None): # This feature is supported for Dynamic-to-Static, because after transformed, the type of inputs[0] # is LOD_TENSOR_ARRAY in some scenarios. And this feature can be used in static mode. - assert len(input) == 1, "If the elements of 'input' in concat are Variable(LoDTensorArray), " \ - "number of the elements must be 1, but received %s." % len(input) + assert len(input) == 1, ( + "If the elements of 'input' in concat are Variable(LoDTensorArray), " + "number of the elements must be 1, but received %s." % len(input) + ) out_index = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': input[0]}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': False - }) + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': input[0]}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': False}, + ) else: inputs = {'X': input} attrs = {} @@ -1110,10 +1205,9 @@ def concat(x, axis=0, name=None): else: attrs['axis'] = axis - helper.append_op(type='concat', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='concat', inputs=inputs, outputs={'Out': [out]}, attrs=attrs + ) return out @@ -1155,17 +1249,21 @@ def broadcast_tensors(input, name=None): check_type(input, 'input', (list, tuple), 'broadcast_tensors') if num_inputs < 1: raise TypeError( - "At least 1 tensor is needed to perform broadcast_tensors") + "At least 1 tensor is needed to perform broadcast_tensors" + ) # Check input types for id, x in enumerate(input): check_variable_and_dtype( - x, 'input[' + str(id) + ']', + x, + 'input[' + str(id) + ']', ['bool', 'float32', 'float64', 'int32', 'int64'], - 'broadcast_tensors') + 'broadcast_tensors', + ) if x.dtype != input[0].dtype: raise TypeError( - "All the Tensors in the input must have the same data type.") + "All the Tensors in the input must have the same data type." + ) # Check bcast semantics output_shape_r_last_tensor_index = [] @@ -1183,8 +1281,11 @@ def broadcast_tensors(input, name=None): output_shape_r.append(shape[i]) output_shape_r_last_tensor_index.append(j) else: - invalid = (output_shape_r[i] != shape[i] - and output_shape_r[i] != 1 and shape[i] != 1) + invalid = ( + output_shape_r[i] != shape[i] + and output_shape_r[i] != 1 + and shape[i] != 1 + ) if invalid: last_index = output_shape_r_last_tensor_index[i] raise TypeError( @@ -1203,14 +1304,15 @@ def broadcast_tensors(input, name=None): while i < num_inputs: out.append( helper.create_variable_for_type_inference( - dtype=helper.input_dtype())) + dtype=helper.input_dtype() + ) + ) i += 1 inputs = {'X': input} - helper.append_op(type='broadcast_tensors', - inputs=inputs, - outputs={'Out': out}, - attrs={}) + helper.append_op( + type='broadcast_tensors', inputs=inputs, outputs={'Out': out}, attrs={} + ) return out @@ -1253,19 +1355,21 @@ def flip(x, axis, name=None): helper = LayerHelper("flip", **locals()) check_type(x, 'X', (Variable), 'flip') dtype = helper.input_dtype('x') - check_dtype(dtype, 'X', - ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - 'flip') + check_dtype( + dtype, + 'X', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + 'flip', + ) check_type(axis, 'axis', (list, tuple), 'flip') if name is None: out = helper.create_variable_for_type_inference(dtype) else: out = helper.create_variable(name=name, dtype=dtype, persistable=False) - helper.append_op(type="flip", - inputs={"X": x}, - outputs={"Out": out}, - attrs={"axis": axis}) + helper.append_op( + type="flip", inputs={"X": x}, outputs={"Out": out}, attrs={"axis": axis} + ) return out @@ -1324,9 +1428,12 @@ def rot90(x, k=1, axes=[0, 1], name=None): helper = LayerHelper("rot90", **locals()) check_type(x, 'X', (Variable), 'rot90') dtype = helper.input_dtype('x') - check_dtype(dtype, 'X', - ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - 'rot90') + check_dtype( + dtype, + 'X', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + 'rot90', + ) check_type(axes, 'axes', (list, tuple), 'rot90') input_total_dims = len(x.shape) @@ -1334,23 +1441,31 @@ def rot90(x, k=1, axes=[0, 1], name=None): if total_rot_dims != 2: raise ValueError( "expected total rotation axes == 2, but got axes = {}".format( - total_rot_dims)) + total_rot_dims + ) + ) if input_total_dims < 2: raise ValueError( "expected total dims >= 2, but got total dims = {}".format( - input_total_dims)) + input_total_dims + ) + ) if not (axes[0] != axes[1] and abs(axes[0] - axes[1]) != input_total_dims): raise ValueError( - "expected rotation axes to be different, but got axis0 = {}, and axis1 = {}" - .format(axes[0], axes[1])) + "expected rotation axes to be different, but got axis0 = {}, and axis1 = {}".format( + axes[0], axes[1] + ) + ) if not (axes[0] < input_total_dims and axes[0] >= -input_total_dims): - raise ValueError("Rotation axis0 out of range, axis0 = {}".format( - axes[0])) + raise ValueError( + "Rotation axis0 out of range, axis0 = {}".format(axes[0]) + ) if not (axes[1] < input_total_dims and axes[1] >= -input_total_dims): - raise ValueError("Rotation axis1 out of range, axis1 = {}".format( - axes[1])) + raise ValueError( + "Rotation axis1 out of range, axis1 = {}".format(axes[1]) + ) k %= 4 if k == 0: @@ -1359,8 +1474,10 @@ def rot90(x, k=1, axes=[0, 1], name=None): return flip(flip(x, axes[0]), axes[1]) axes_list = list(range(0, input_total_dims)) - (axes_list[axes[0]], axes_list[axes[1]]) = (axes_list[axes[1]], - axes_list[axes[0]]) + (axes_list[axes[0]], axes_list[axes[1]]) = ( + axes_list[axes[1]], + axes_list[axes[0]], + ) if k == 1: return transpose(flip(x, axes[1]), axes_list) else: @@ -1439,19 +1556,29 @@ def flatten(x, start_axis=0, stop_axis=-1, name=None): if not paddle.in_dynamic_mode(): check_variable_and_dtype( - x, 'x', + x, + 'x', ['float32', 'float64', 'int8', 'int16', 'int32', 'int64', 'uint8'], - 'flatten') + 'flatten', + ) x_dim = len(x.shape) - if not (isinstance(start_axis, - int)) or (start_axis > x_dim - 1) or start_axis < -x_dim: + if ( + not (isinstance(start_axis, int)) + or (start_axis > x_dim - 1) + or start_axis < -x_dim + ): raise ValueError( - "The start_axis should be a int, and in range [-rank(x), rank(x))") - if not (isinstance(stop_axis, - int)) or (stop_axis > x_dim - 1) or stop_axis < -x_dim: + "The start_axis should be a int, and in range [-rank(x), rank(x))" + ) + if ( + not (isinstance(stop_axis, int)) + or (stop_axis > x_dim - 1) + or stop_axis < -x_dim + ): raise ValueError( - "The stop_axis should be a int, and in range [-rank(x), rank(x))") + "The stop_axis should be a int, and in range [-rank(x), rank(x))" + ) if start_axis < 0: start_axis = start_axis + x_dim if stop_axis < 0: @@ -1464,22 +1591,19 @@ def flatten(x, start_axis=0, stop_axis=-1, name=None): if _in_legacy_dygraph(): dy_out, _ = _legacy_C_ops.flatten_contiguous_range( - x, 'start_axis', start_axis, 'stop_axis', stop_axis) + x, 'start_axis', start_axis, 'stop_axis', stop_axis + ) return dy_out helper = LayerHelper('flatten', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='flatten_contiguous_range', - inputs={"X": x}, - outputs={ - 'Out': out, - 'XShape': x_shape - }, - attrs={ - "start_axis": start_axis, - "stop_axis": stop_axis - }) + helper.append_op( + type='flatten_contiguous_range', + inputs={"X": x}, + outputs={'Out': out, 'XShape': x_shape}, + attrs={"start_axis": start_axis, "stop_axis": stop_axis}, + ) return out @@ -1493,14 +1617,22 @@ def flatten_(x, start_axis=0, stop_axis=-1, name=None): raise ValueError("The input x should be a Tensor") x_dim = len(x.shape) - if not (isinstance(start_axis, - int)) or (start_axis > x_dim - 1) or start_axis < -x_dim: + if ( + not (isinstance(start_axis, int)) + or (start_axis > x_dim - 1) + or start_axis < -x_dim + ): raise ValueError( - "The start_axis should be a int, and in range [-rank(x), rank(x))") - if not (isinstance(stop_axis, - int)) or (stop_axis > x_dim - 1) or stop_axis < -x_dim: + "The start_axis should be a int, and in range [-rank(x), rank(x))" + ) + if ( + not (isinstance(stop_axis, int)) + or (stop_axis > x_dim - 1) + or stop_axis < -x_dim + ): raise ValueError( - "The stop_axis should be a int, and in range [-rank(x), rank(x))") + "The stop_axis should be a int, and in range [-rank(x), rank(x))" + ) if start_axis < 0: start_axis = start_axis + x_dim if stop_axis < 0: @@ -1513,7 +1645,8 @@ def flatten_(x, start_axis=0, stop_axis=-1, name=None): if _in_legacy_dygraph(): dy_out, _ = _legacy_C_ops.flatten_contiguous_range_( - x, 'start_axis', start_axis, 'stop_axis', stop_axis) + x, 'start_axis', start_axis, 'stop_axis', stop_axis + ) return dy_out @@ -1571,8 +1704,10 @@ def roll(x, shifts, axis=None, name=None): for i in range(len(axis)): if axis[i] >= len_origin_shape or axis[i] < -len_origin_shape: raise ValueError( - "axis is out of range, it should be in range [{}, {}), but received {}" - .format(-len_origin_shape, len_origin_shape, axis)) + "axis is out of range, it should be in range [{}, {}), but received {}".format( + -len_origin_shape, len_origin_shape, axis + ) + ) else: axis = [] @@ -1588,22 +1723,20 @@ def roll(x, shifts, axis=None, name=None): out = helper.create_variable_for_type_inference(x.dtype) if isinstance(shifts, Variable): - helper.append_op(type='roll', - inputs={ - 'X': x, - "ShiftsTensor": shifts - }, - outputs={'Out': out}, - attrs={'axis': axis}) + helper.append_op( + type='roll', + inputs={'X': x, "ShiftsTensor": shifts}, + outputs={'Out': out}, + attrs={'axis': axis}, + ) else: check_type(shifts, 'shifts', (list, tuple), 'roll') - helper.append_op(type='roll', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'axis': axis, - 'shifts': shifts - }) + helper.append_op( + type='roll', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'axis': axis, 'shifts': shifts}, + ) return out @@ -1704,42 +1837,53 @@ def stack(x, axis=0, name=None): if not isinstance(x, list) and not isinstance(x, tuple): # NOTE:(zhiqiu) Only support Variable as input if the Variable is a LOD_TENSOR_ARRAY create by create_array, array_write, array_read, etc. # In that case, Variable is array of tensors indeed. - if isinstance(x, Variable) and x.desc.type( - ) == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + isinstance(x, Variable) + and x.desc.type() == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): x = [x] else: raise TypeError( - "The type of '%s' in %s must be %s, but received %s" % - ('x', 'stack', 'list[Tensor], tuple[Tensor] or TensorArray', - type(x))) + "The type of '%s' in %s must be %s, but received %s" + % ( + 'x', + 'stack', + 'list[Tensor], tuple[Tensor] or TensorArray', + type(x), + ) + ) helper = LayerHelper('stack', **locals()) out = helper.create_variable_for_type_inference(x[0].dtype) if x[0].desc.type() == core.VarDesc.VarType.LOD_TENSOR_ARRAY: - assert len(x) == 1, "If the elements of 'x' in stack are Variable(LoDTensorArray), " \ - "number of the elements must be 1, but received %s." % len(x) + assert len(x) == 1, ( + "If the elements of 'x' in stack are Variable(LoDTensorArray), " + "number of the elements must be 1, but received %s." % len(x) + ) out_index = helper.create_variable_for_type_inference(dtype="int32") for i in x: - check_variable_and_dtype(i, 'x', \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'stack') - - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': x[0]}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': True - }) + check_variable_and_dtype( + i, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'stack', + ) + + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': x[0]}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': True}, + ) else: - helper.append_op(type='stack', - inputs={'X': x}, - outputs={'Y': out}, - attrs={'axis': axis}) + helper.append_op( + type='stack', + inputs={'X': x}, + outputs={'Y': out}, + attrs={'axis': axis}, + ) return out @@ -1813,15 +1957,17 @@ def split(x, num_or_sections, axis=0, name=None): if utils._contain_var(num_or_sections): for index, item in enumerate(num_or_sections): if isinstance(item, Variable): - num_or_sections[index] = num_or_sections[index].numpy( - )[0] + num_or_sections[index] = num_or_sections[index].numpy()[ + 0 + ] attrs += ('sections', list(num_or_sections)) else: attrs += ('sections', list(num_or_sections)) else: raise TypeError( "The type of 'num_or_sections' in split must be int, list or tuple in imperative mode, but " - "received %s." % (type(num_or_sections))) + "received %s." % (type(num_or_sections)) + ) if in_dygraph_mode(): if isinstance(num_or_sections, int): return _C_ops.split_with_num(input, num_or_sections, dim) @@ -1832,10 +1978,21 @@ def split(x, num_or_sections, axis=0, name=None): _legacy_C_ops.split(input, out, *attrs) return out - check_variable_and_dtype(input, 'input', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'uint8', - 'int8' - ], 'split') + check_variable_and_dtype( + input, + 'input', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'int8', + ], + 'split', + ) check_type(num_or_sections, 'num_or_sections', (list, int, tuple), 'split') check_type(dim, 'dim', (int, Variable), 'split') if isinstance(dim, Variable): @@ -1855,19 +2012,18 @@ def split(x, num_or_sections, axis=0, name=None): dim_size.stop_gradient = True tensor_list.append(dim_size) else: - assert (isinstance(dim_size, int)) + assert isinstance(dim_size, int) if dim_size == -1: assert unk_dim_idx == -1, ( "Only one value of 'num_or_section' in split can " - "be -1. But received num_or_section[%d] is also -1." % - idx) + "be -1. But received num_or_section[%d] is also -1." + % idx + ) unk_dim_idx = idx temp_out = helper.create_variable_for_type_inference('int32') - fill_constant([1], - 'int32', - dim_size, - force_cpu=True, - out=temp_out) + fill_constant( + [1], 'int32', dim_size, force_cpu=True, out=temp_out + ) tensor_list.append(temp_out) return tensor_list @@ -1882,31 +2038,37 @@ def split(x, num_or_sections, axis=0, name=None): if isinstance(num_or_sections, int): assert num_or_sections > 1, 'num_or_sections must be more than 1.' if isinstance(dim, int) and input_shape[dim] > 0: - assert input_shape[dim] % num_or_sections ==0, \ - "The input's size along the split dimension " \ - "must be evenly divisible by Attr(num_or_sections). " \ - "But %d is not evenly divisible by %d. " % (num_or_sections,input_shape[dim]) + assert input_shape[dim] % num_or_sections == 0, ( + "The input's size along the split dimension " + "must be evenly divisible by Attr(num_or_sections). " + "But %d is not evenly divisible by %d. " + % (num_or_sections, input_shape[dim]) + ) num = num_or_sections else: if isinstance(dim, int) and input_shape[dim] > 0: - assert len(num_or_sections) <= input_shape[ - dim], 'len(num_or_sections) must not be more than input.shape[dim].' + assert ( + len(num_or_sections) <= input_shape[dim] + ), 'len(num_or_sections) must not be more than input.shape[dim].' num = len(num_or_sections) attrs['sections'] = list( - map(lambda ele: -1 - if isinstance(ele, Variable) else ele, num_or_sections)) + map( + lambda ele: -1 if isinstance(ele, Variable) else ele, + num_or_sections, + ) + ) if utils._contain_var(num_or_sections): inputs['SectionsTensorList'] = _get_SectionsTensorList( - num_or_sections) + num_or_sections + ) outs = [ helper.create_variable_for_type_inference(dtype=helper.input_dtype()) for i in range(num) ] - helper.append_op(type='split', - inputs=inputs, - outputs={'Out': outs}, - attrs=attrs) + helper.append_op( + type='split', inputs=inputs, outputs={'Out': outs}, attrs=attrs + ) return outs @@ -1947,8 +2109,10 @@ def vsplit(x, num_or_sections, name=None): """ if x.ndim < 2: raise ValueError( - "The input tensor's dimension must be greater than 1, but got {}". - format(x.ndim)) + "The input tensor's dimension must be greater than 1, but got {}".format( + x.ndim + ) + ) return split(x, num_or_sections, axis=0, name=name) @@ -2041,10 +2205,22 @@ def squeeze(x, axis=None, name=None): return out helper = LayerHelper("squeeze", **locals()) - check_variable_and_dtype(input, 'input', [ - 'float16', 'float32', 'float64', 'bool', 'int8', 'int32', 'int64', - 'complex64', 'complex128' - ], 'squeeze') + check_variable_and_dtype( + input, + 'input', + [ + 'float16', + 'float32', + 'float64', + 'bool', + 'int8', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'squeeze', + ) check_type(axes, 'axis/axes', (int, list, tuple, Variable), 'squeeze') attrs = {} @@ -2059,13 +2235,12 @@ def squeeze(x, axis=None, name=None): out = helper.create_variable_for_type_inference(dtype=input.dtype) x_shape = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="squeeze2", - inputs={"X": input}, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="squeeze2", + inputs={"X": input}, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return out @@ -2092,12 +2267,14 @@ def squeeze_(x, axis=None, name=None): return out -def unique_consecutive(x, - return_inverse=False, - return_counts=False, - axis=None, - dtype="int64", - name=None): +def unique_consecutive( + x, + return_inverse=False, + return_counts=False, + axis=None, + dtype="int64", + name=None, +): r""" Eliminates all but the first element from every consecutive group of equivalent elements. @@ -2164,7 +2341,8 @@ def unique_consecutive(x, attr_dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): out, inverse, counts = _C_ops.unique_consecutive( - x, return_inverse, return_counts, axis, attr_dtype) + x, return_inverse, return_counts, axis, attr_dtype + ) outs = [out] if return_inverse: outs.append(inverse) @@ -2175,8 +2353,16 @@ def unique_consecutive(x, return tuple(outs) elif paddle.in_dynamic_mode(): out, inverse, counts = _legacy_C_ops.unique_consecutive( - x, 'dtype', attr_dtype, 'return_inverse', return_inverse, - 'return_counts', return_counts, 'axis', axis) + x, + 'dtype', + attr_dtype, + 'return_inverse', + return_inverse, + 'return_counts', + return_counts, + 'axis', + axis, + ) outs = [out] if return_inverse: outs.append(inverse) @@ -2185,9 +2371,12 @@ def unique_consecutive(x, if len(outs) == 1: return outs[0] return tuple(outs) - check_variable_and_dtype(x, "input", - ['float32', 'float64', 'int32', 'int64'], - 'unique_consecutive') + check_variable_and_dtype( + x, + "input", + ['float32', 'float64', 'int32', 'int64'], + 'unique_consecutive', + ) check_type(return_inverse, 'return_inverse', bool, 'unique_consecutive') check_type(return_counts, 'return_counts', bool, 'unique_consecutive') check_dtype(dtype, 'dtype', ['int32', 'int64'], 'unique_consecutive') @@ -2200,34 +2389,38 @@ def unique_consecutive(x, "return_counts": return_counts, "axis": axis, } - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - inverse = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) - counts = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + inverse = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) + counts = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) outputs = {"Out": out, "Index": inverse, "Counts": counts} outs = [out] if return_inverse: outs.append(inverse) if return_counts: outs.append(counts) - helper.append_op(type="unique_consecutive", - inputs={"X": x}, - attrs=attrs, - outputs=outputs) + helper.append_op( + type="unique_consecutive", inputs={"X": x}, attrs=attrs, outputs=outputs + ) if len(outs) == 1: return outs[0] return tuple(outs) -def unique(x, - return_index=False, - return_inverse=False, - return_counts=False, - axis=None, - dtype="int64", - name=None): +def unique( + x, + return_index=False, + return_inverse=False, + return_counts=False, + axis=None, + dtype="int64", + name=None, +): r""" Returns the unique elements of `x` in ascending order. @@ -2289,13 +2482,24 @@ def unique(x, if _non_static_mode(): if in_dygraph_mode(): out, indices, inverse, counts = _C_ops.unique( - x, return_index, return_inverse, return_counts, axis, - attr_dtype) + x, return_index, return_inverse, return_counts, axis, attr_dtype + ) if _in_legacy_dygraph(): out, inverse, indices, counts = _legacy_C_ops.unique( - x, 'dtype', attr_dtype, 'return_index', return_index, - 'return_inverse', return_inverse, 'return_counts', - return_counts, 'axis', axis, "is_sorted", True) + x, + 'dtype', + attr_dtype, + 'return_index', + return_index, + 'return_inverse', + return_inverse, + 'return_counts', + return_counts, + 'axis', + axis, + "is_sorted", + True, + ) outs = [out] if return_index: outs.append(indices) @@ -2309,8 +2513,9 @@ def unique(x, return tuple(outs) - check_variable_and_dtype(x, "input", - ['float32', 'float64', 'int32', 'int64'], 'unique') + check_variable_and_dtype( + x, "input", ['float32', 'float64', 'int32', 'int64'], 'unique' + ) check_type(return_index, 'return_index', bool, 'unique') check_type(return_inverse, 'return_inverse', bool, 'unique') check_type(return_counts, 'return_counts', bool, 'unique') @@ -2325,21 +2530,25 @@ def unique(x, "return_inverse": return_inverse, "return_counts": return_counts, "axis": axis, - "is_sorted": True + "is_sorted": True, } - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - indices = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) - inverse = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) - counts = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + indices = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) + inverse = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) + counts = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) outputs = { "Out": out, "Indices": indices, "Index": inverse, - "Counts": counts + "Counts": counts, } outs = [out] if return_index: @@ -2349,10 +2558,9 @@ def unique(x, if return_counts: outs.append(counts) - helper.append_op(type="unique", - inputs={"X": x}, - attrs=attrs, - outputs=outputs) + helper.append_op( + type="unique", inputs={"X": x}, attrs=attrs, outputs=outputs + ) if len(outs) == 1: return outs[0] @@ -2424,18 +2632,23 @@ def unsqueeze(x, axis, name=None): return _C_ops.unsqueeze(input, axes) check_type(axes, 'axis/axes', (int, list, tuple, Variable), 'unsqueeze') - check_variable_and_dtype(input, 'input', [ - 'float16', - 'float32', - 'float64', - 'bool', - 'int8', - 'int16', - 'int32', - 'int64', - 'complex64', - 'complex128', - ], 'unsqueeze') + check_variable_and_dtype( + input, + 'input', + [ + 'float16', + 'float32', + 'float64', + 'bool', + 'int8', + 'int16', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'unsqueeze', + ) helper = LayerHelper("unsqueeze2", **locals()) inputs = {"X": input} attrs = {} @@ -2453,13 +2666,12 @@ def unsqueeze(x, axis, name=None): out = helper.create_variable_for_type_inference(dtype=input.dtype) x_shape = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="unsqueeze2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="unsqueeze2", + inputs=inputs, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return out @@ -2539,13 +2751,16 @@ def gather(x, index, axis=None, name=None): return _C_ops.gather(x, index, axis) if _in_legacy_dygraph(): axis = axis.item() if isinstance(axis, paddle.Tensor) else axis - return _legacy_C_ops.gather(x, index, None, "axis", axis, "overwrite", - False) + return _legacy_C_ops.gather( + x, index, None, "axis", axis, "overwrite", False + ) check_variable_and_dtype( - x, 'x', + x, + 'x', ['float16', 'float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], - 'gather') + 'gather', + ) check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather') if isinstance(axis, Variable): @@ -2555,25 +2770,19 @@ def gather(x, index, axis=None, name=None): dtype = helper.input_dtype('x') out = helper.create_variable_for_type_inference(dtype) if not isinstance(axis, Variable): - helper.append_op(type="gather", - inputs={ - "X": x, - "Index": index - }, - attrs={ - 'axis': axis, - 'overwrite': False - }, - outputs={"Out": out}) + helper.append_op( + type="gather", + inputs={"X": x, "Index": index}, + attrs={'axis': axis, 'overwrite': False}, + outputs={"Out": out}, + ) else: - helper.append_op(type="gather", - inputs={ - "X": x, - "Index": index, - "Axis": axis - }, - attrs={"overwrite": False}, - outputs={"Out": out}) + helper.append_op( + type="gather", + inputs={"X": x, "Index": index, "Axis": axis}, + attrs={"overwrite": False}, + outputs={"Out": out}, + ) return out @@ -2613,8 +2822,9 @@ def unbind(input, axis=0): return _C_ops.unbind(input, axis) if not isinstance(axis, (int)): - raise TypeError("The type of 'axis' must be int, but received %s." % - (type(axis))) + raise TypeError( + "The type of 'axis' must be int, but received %s." % (type(axis)) + ) if isinstance(axis, np.generic): axis = np.asscalar(axis) input_shape = input.shape @@ -2626,16 +2836,19 @@ def unbind(input, axis=0): helper = LayerHelper("unbind", **locals()) check_type(input, 'input', (Variable), 'unbind') dtype = helper.input_dtype() - check_dtype(dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], - 'unbind') + check_dtype( + dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], 'unbind' + ) outs = [ helper.create_variable_for_type_inference(dtype=helper.input_dtype()) for i in range(num) ] - helper.append_op(type="unbind", - inputs={"X": input}, - outputs={"Out": outs}, - attrs={"axis": axis}) + helper.append_op( + type="unbind", + inputs={"X": input}, + outputs={"Out": outs}, + attrs={"axis": axis}, + ) return outs @@ -2717,23 +2930,25 @@ def scatter(x, index, updates, overwrite=True, name=None): return _C_ops.scatter(x, index, updates, overwrite) else: if _in_legacy_dygraph(): - return _legacy_C_ops.scatter(x, index, updates, 'overwrite', - overwrite) + return _legacy_C_ops.scatter( + x, index, updates, 'overwrite', overwrite + ) else: check_variable_and_dtype( - x, 'dtype', ['float32', 'float64', 'float16', 'int32', 'int64'], - 'scatter') + x, + 'dtype', + ['float32', 'float64', 'float16', 'int32', 'int64'], + 'scatter', + ) check_type(overwrite, 'overwrite', bool, 'scatter') helper = LayerHelper('scatter', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type="scatter", - inputs={ - "X": x, - "Ids": index, - "Updates": updates - }, - attrs={'overwrite': overwrite}, - outputs={"Out": out}) + helper.append_op( + type="scatter", + inputs={"X": x, "Ids": index, "Updates": updates}, + attrs={'overwrite': overwrite}, + outputs={"Out": out}, + ) return out @@ -2830,13 +3045,11 @@ def scatter_nd_add(x, index, updates, name=None): helper = LayerHelper('scatter_nd_add', **locals()) dtype = helper.input_dtype(input_param_name='x') output = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="scatter_nd_add", - inputs={ - "X": x, - "Index": index, - "Updates": updates - }, - outputs={"Out": output}) + helper.append_op( + type="scatter_nd_add", + inputs={"X": x, "Index": index, "Updates": updates}, + outputs={"Out": output}, + ) return output @@ -2963,7 +3176,9 @@ def tile(x, repeat_times, name=None): """ if in_dygraph_mode(): if isinstance(repeat_times, core.eager.Tensor): - assert repeat_times.ndim == 1, "Only support ndim == 1 while repeat_times is a Tensor." + assert ( + repeat_times.ndim == 1 + ), "Only support ndim == 1 while repeat_times is a Tensor." repeat_times = repeat_times.numpy().tolist() return _C_ops.tile(x, repeat_times) @@ -2973,26 +3188,30 @@ def tile(x, repeat_times, name=None): check_type(repeat_times, 'repeat_times', (list, tuple, Variable), 'tile') if isinstance(repeat_times, Variable): - assert len( - repeat_times.shape) == 1, ('repeat_times must be an 1-D Tensor.') + assert ( + len(repeat_times.shape) == 1 + ), 'repeat_times must be an 1-D Tensor.' else: for elem in repeat_times: if isinstance(elem, Variable): - assert len(elem.shape) == 1, ( - 'Elements in repeat_times must be 1-D Tensors or integers.') + assert ( + len(elem.shape) == 1 + ), 'Elements in repeat_times must be 1-D Tensors or integers.' else: type_tuple = (int, np.int32, np.int64) - assert isinstance(elem, type_tuple), ( - 'Elements in repeat_times must be 1-D Tensors or integers.') + assert isinstance( + elem, type_tuple + ), 'Elements in repeat_times must be 1-D Tensors or integers.' - check_variable_and_dtype(x, 'x', - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'tile') + check_variable_and_dtype( + x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'tile' + ) if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: raise ValueError( "When the date type is bool for the input 'x' of tile op, you " "must set its stop_gradient to be True by " - "some_var.stop_gradient == True supporting some_var is the input.") + "some_var.stop_gradient == True supporting some_var is the input." + ) helper = LayerHelper('tile', **locals()) @@ -3006,8 +3225,9 @@ def tile(x, repeat_times, name=None): attrs_repeat_times.append(-1) else: attrs_repeat_times.append(times) - assert times > 0, ( - "All elements in repeat_times must be positive for tile.") + assert ( + times > 0 + ), "All elements in repeat_times must be positive for tile." return attrs_repeat_times if isinstance(repeat_times, Variable): @@ -3018,14 +3238,14 @@ def tile(x, repeat_times, name=None): attrs['repeat_times'] = get_attr_repeat_times(repeat_times) if utils._contain_var(repeat_times): inputs['repeat_times_tensor'] = utils._convert_to_tensor_list( - repeat_times) + repeat_times + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='tile', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='tile', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -3063,9 +3283,9 @@ def expand_as(x, y, name=None): if _non_static_mode(): return _legacy_C_ops.expand_as_v2(x, 'target_shape', y.shape) - check_variable_and_dtype(x, 'x', - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'expand_as') + check_variable_and_dtype( + x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'expand_as' + ) check_type(y, 'y', Variable, 'expand_as') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: @@ -3073,16 +3293,19 @@ def expand_as(x, y, name=None): "When the data type of input 'x' for expand_as is bool, " "you must set its stop_gradient to be False by " "some_var.stop_gradient = True, supporting " - "some_var as the input 'x'.") + "some_var as the input 'x'." + ) inputs = {"X": [x], "Y": [y]} helper = LayerHelper('expand_as', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='expand_as_v2', - inputs=inputs, - attrs={'target_shape': y.shape}, - outputs={'Out': out}) + helper.append_op( + type='expand_as_v2', + inputs=inputs, + attrs={'target_shape': y.shape}, + outputs={'Out': out}, + ) return out @@ -3119,27 +3342,30 @@ def broadcast_to(x, shape, name=None): return _legacy_C_ops.expand_v2(x, 'shape', shape) if isinstance(shape, Variable): - assert len(shape.shape) == 1, ('shape must be an 1-D Tensor.') + assert len(shape.shape) == 1, 'shape must be an 1-D Tensor.' else: for elem in shape: if isinstance(elem, Variable): - assert len(elem.shape) == 1, ( - 'Elements in shape must be 1-D Tensors or integers.') + assert ( + len(elem.shape) == 1 + ), 'Elements in shape must be 1-D Tensors or integers.' else: type_tuple = (int, np.int32, np.int64) - assert isinstance(elem, type_tuple), ( - 'Elements in shape must be 1-D Tensors or integers.') + assert isinstance( + elem, type_tuple + ), 'Elements in shape must be 1-D Tensors or integers.' - check_variable_and_dtype(x, 'x', - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'broadcast_to') + check_variable_and_dtype( + x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'broadcast_to' + ) check_type(shape, 'shape', (list, tuple, Variable), 'broadcast_to') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: raise ValueError( "When the data type of input 'x' for broadcast_to is bool, " "you must set its stop_gradient to be False by " "some_var.stop_gradient = True, supporting " - "some_var as the input.") + "some_var as the input." + ) inputs = {"X": [x]} attrs = {} @@ -3153,9 +3379,9 @@ def broadcast_to(x, shape, name=None): attrs_expand_shape.append(-1) else: attrs_expand_shape.append(shape) - assert shape > 0 or shape == -1, ( - "All elements in shape of broadcast_to must be positive or -1." - ) + assert ( + shape > 0 or shape == -1 + ), "All elements in shape of broadcast_to must be positive or -1." return attrs_expand_shape if isinstance(shape, Variable): @@ -3165,14 +3391,14 @@ def broadcast_to(x, shape, name=None): attrs['shape'] = get_attr_expand_shape(shape) if utils._contain_var(shape): inputs['expand_shapes_tensor'] = utils._convert_to_tensor_list( - shape) + shape + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='expand_v2', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='expand_v2', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -3211,26 +3437,33 @@ def expand(x, shape, name=None): return _legacy_C_ops.expand_v2(x, 'shape', shape) if isinstance(shape, Variable): - assert len(shape.shape) == 1, ('shape must be an 1-D Tensor.') + assert len(shape.shape) == 1, 'shape must be an 1-D Tensor.' else: for elem in shape: if isinstance(elem, Variable): - assert len(elem.shape) == 1, ( - 'Elements in shape must be 1-D Tensors or integers.') + assert ( + len(elem.shape) == 1 + ), 'Elements in shape must be 1-D Tensors or integers.' else: type_tuple = (int, np.int32, np.int64) - assert isinstance(elem, type_tuple), ( - 'Elements in shape must be 1-D Tensors or integers.') + assert isinstance( + elem, type_tuple + ), 'Elements in shape must be 1-D Tensors or integers.' check_variable_and_dtype( - x, 'x', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'expand') + x, + 'x', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'expand', + ) check_type(shape, 'shape', (list, tuple, Variable), 'expand') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: - raise ValueError("When the data type of input 'x' for expand is bool, " - "you must set its stop_gradient to be False by " - "some_var.stop_gradient = True, supporting " - "some_var as the input.") + raise ValueError( + "When the data type of input 'x' for expand is bool, " + "you must set its stop_gradient to be False by " + "some_var.stop_gradient = True, supporting " + "some_var as the input." + ) inputs = {"X": [x]} attrs = {} @@ -3244,8 +3477,9 @@ def expand(x, shape, name=None): attrs_expand_shape.append(-2) else: attrs_expand_shape.append(shape) - assert shape > 0 or shape == -1, ( - "All elements in shape of expand must be positive or -1.") + assert ( + shape > 0 or shape == -1 + ), "All elements in shape of expand must be positive or -1." return attrs_expand_shape if isinstance(shape, Variable): @@ -3255,14 +3489,14 @@ def expand(x, shape, name=None): attrs['shape'] = get_attr_expand_shape(shape) if utils._contain_var(shape): inputs['expand_shapes_tensor'] = utils._convert_to_tensor_list( - shape) + shape + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='expand_v2', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='expand_v2', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -3331,7 +3565,7 @@ def reshape(x, shape, name=None): if in_dygraph_mode(): tmp_tensor_type = core.eager.Tensor - #TODO(zhiqiu): enable inplace in dygraph mode. + # TODO(zhiqiu): enable inplace in dygraph mode. if inplace: warnings.warn( "Inplace on reshape is not allowed and will be discarded in dygraph mode currently." @@ -3339,7 +3573,9 @@ def reshape(x, shape, name=None): if isinstance(shape, (list, tuple)): shape = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item for item in shape + if isinstance(item, tmp_tensor_type) + else item + for item in shape ] out = _C_ops.reshape(x, shape) elif isinstance(shape, tmp_tensor_type): @@ -3348,7 +3584,8 @@ def reshape(x, shape, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return dygraph_utils._append_activation_in_dygraph(out, act) else: @@ -3370,14 +3607,26 @@ def reshape(x, shape, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return dygraph_utils._append_activation_in_dygraph(out, act) - check_variable_and_dtype(x, 'x', [ - 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', 'bool', - 'uint16' - ], 'reshape') + check_variable_and_dtype( + x, + 'x', + [ + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'bool', + 'uint16', + ], + 'reshape', + ) check_type(shape, 'shape', (list, tuple, Variable), 'reshape') check_type(actual_shape, 'actual_shape', (Variable, type(None)), 'reshape') @@ -3401,20 +3650,23 @@ def reshape(x, shape, name=None): "\t# z.shape is [-1, -1, 4]\n\n" " If your target shape in Reshape represents dynamic shape, " "please turn it into a Tensor under @to_static. See above example for details." - % dim_idx) + % dim_idx + ) unk_dim_idx = dim_idx elif dim_size == 0: assert dim_idx < len(x.shape), ( "The index of 0 in `shape` must be less than " "the input tensor X's dimensions. " - "But received shape[%d] = 0, X's dimensions = %d." % - (dim_idx, len(x.shape))) + "But received shape[%d] = 0, X's dimensions = %d." + % (dim_idx, len(x.shape)) + ) else: assert dim_size > 0, ( "Each dimension value of 'shape' in reshape must not " "be negative except one unknown dimension. " - "But received shape[%d] = %s." % - (dim_idx, str(dim_size))) + "But received shape[%d] = %s." + % (dim_idx, str(dim_size)) + ) return attrs_shape inputs = {"X": x} @@ -3423,8 +3675,10 @@ def reshape(x, shape, name=None): shape.stop_gradient = True inputs["Shape"] = shape elif isinstance(shape, (list, tuple)): - assert len(shape) > 0, ("The size of 'shape' in reshape can't be zero, " - "but received %s." % len(shape)) + assert len(shape) > 0, ( + "The size of 'shape' in reshape can't be zero, " + "but received %s." % len(shape) + ) attrs["shape"] = get_attr_shape(shape) if utils._contain_var(shape): inputs['ShapeTensor'] = utils._convert_to_tensor_list(shape) @@ -3432,16 +3686,18 @@ def reshape(x, shape, name=None): actual_shape.stop_gradient = True inputs["Shape"] = actual_shape - out = x if inplace else helper.create_variable_for_type_inference( - dtype=x.dtype) + out = ( + x + if inplace + else helper.create_variable_for_type_inference(dtype=x.dtype) + ) x_shape = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="reshape2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="reshape2", + inputs=inputs, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return helper.append_activation(out) @@ -3457,7 +3713,9 @@ def reshape_(x, shape, name=None): if isinstance(shape, (list, tuple)): shape = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item for item in shape + if isinstance(item, tmp_tensor_type) + else item + for item in shape ] out = _C_ops.reshape_(x, shape) elif isinstance(shape, tmp_tensor_type): @@ -3466,7 +3724,8 @@ def reshape_(x, shape, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return out else: @@ -3567,18 +3826,20 @@ def gather_nd(x, index, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.gather_nd(x, index) check_variable_and_dtype( - x, 'x', ['bool', 'float32', 'float64', 'int16', 'int32', 'int64'], - 'gather_np') + x, + 'x', + ['bool', 'float32', 'float64', 'int16', 'int32', 'int64'], + 'gather_np', + ) check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather_np') helper = LayerHelper('gather_nd', **locals()) dtype = helper.input_dtype() output = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="gather_nd", - inputs={ - "X": x, - "Index": index - }, - outputs={"Out": output}) + helper.append_op( + type="gather_nd", + inputs={"X": x, "Index": index}, + outputs={"Out": output}, + ) return output @@ -3670,8 +3931,11 @@ def strided_slice(x, axes, starts, ends, strides, name=None): helper = LayerHelper('strided_slice', **locals()) check_variable_and_dtype( - x, 'x', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'strided_slice') + x, + 'x', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'strided_slice', + ) check_type(axes, 'axes', (list, tuple), 'strided_slice') check_type(starts, 'starts', (list, tuple, Variable), 'strided_slice') check_type(ends, 'ends', (list, tuple, Variable), 'strided_slice') @@ -3679,8 +3943,9 @@ def strided_slice(x, axes, starts, ends, strides, name=None): def check_list_elements_dtype(list_input, input_name): if isinstance(list_input, Variable): - check_dtype(list_input.dtype, input_name, ['int32'], - 'strided_slice') + check_dtype( + list_input.dtype, input_name, ['int32'], 'strided_slice' + ) else: for i, var in enumerate(list_input): var_name = input_name + '[' + str(i) + ']' @@ -3699,7 +3964,7 @@ def strided_slice(x, axes, starts, ends, strides, name=None): dim.stop_gradient = True new_list_tensor.append(dim) else: - assert (isinstance(dim, int)) + assert isinstance(dim, int) temp_out = helper.create_variable_for_type_inference('int32') fill_constant([1], 'int32', dim, force_cpu=True, out=temp_out) new_list_tensor.append(temp_out) @@ -3716,7 +3981,7 @@ def strided_slice(x, axes, starts, ends, strides, name=None): 'starts': starts, 'ends': ends, 'strides': strides, - 'infer_flags': infer_flags + 'infer_flags': infer_flags, } else: # starts @@ -3771,11 +4036,11 @@ def strided_slice(x, axes, starts, ends, strides, name=None): attrs['strides'] = strides attrs['infer_flags'] = infer_flags out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('x')) - helper.append_op(type='strided_slice', - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + dtype=helper.input_dtype('x') + ) + helper.append_op( + type='strided_slice', inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out @@ -3903,8 +4168,9 @@ def tensordot(x, y, axes=2, name=None): if paddle.in_dynamic_mode(): return tolist(var) raise TypeError( - "The 'axes' with type 'Tensor' in " + op_type + - " is not available in static graph mode, " + "The 'axes' with type 'Tensor' in " + + op_type + + " is not available in static graph mode, " "please convert its type to int|Tuple|List, or use dynamic graph mode." ) @@ -3912,8 +4178,10 @@ def tensordot(x, y, axes=2, name=None): axes_y = [] if np.issubdtype(type(axes), np.integer): assert axes >= 0, ( - "The 'axes' in " + op_type + - f" should not be negative, but received axes={axes}.") + "The 'axes' in " + + op_type + + f" should not be negative, but received axes={axes}." + ) axes_x = range(x.ndim - axes, x.ndim) axes_y = range(axes) else: @@ -3953,7 +4221,11 @@ def tensordot(x, y, axes=2, name=None): shape_x[dim_x] = 1 x = x.sum(dim_x).reshape(shape_x) else: - assert sx == sy, "The dimensional size for 'x' and 'y' in " + op_type + f" should match each other, but 'x' has size {sx} in dim {dim_x} while 'y' has size {sy} in dim {dim_y}." + assert sx == sy, ( + "The dimensional size for 'x' and 'y' in " + + op_type + + f" should match each other, but 'x' has size {sx} in dim {dim_x} while 'y' has size {sy} in dim {dim_y}." + ) need_contracted_dim_x[dim_x] = True need_contracted_dim_y[dim_y] = True @@ -3981,9 +4253,11 @@ def tensordot(x, y, axes=2, name=None): shape_out = [1] x = x.transpose(perm=perm_x).reshape( - [not_contraction_size_x, contraction_size]) + [not_contraction_size_x, contraction_size] + ) y = y.transpose(perm=perm_y).reshape( - [contraction_size, not_contraction_size_y]) + [contraction_size, not_contraction_size_y] + ) out = x.matmul(y).reshape(shape_out) return out @@ -4027,7 +4301,8 @@ def as_complex(x, name=None): helper = LayerHelper(op_type, **locals()) inputs = {"X": x} out = helper.create_variable_for_type_inference( - dtype=_real_to_complex_dtype(x.dtype)) + dtype=_real_to_complex_dtype(x.dtype) + ) outputs = {"Out": out} attrs = {} helper.append_op(type=op_type, inputs=inputs, attrs=attrs, outputs=outputs) @@ -4079,7 +4354,8 @@ def as_real(x, name=None): helper = LayerHelper(op_type, **locals()) inputs = {"X": x} out = helper.create_variable_for_type_inference( - dtype=_complex_to_real_dtype(x.dtype)) + dtype=_complex_to_real_dtype(x.dtype) + ) outputs = {"Out": out} helper.append_op(type=op_type, inputs=inputs, outputs=outputs) return out @@ -4131,23 +4407,27 @@ def repeat_interleave(x, repeats, axis=None, name=None): return _C_ops.repeat_interleave(x, repeats, axis) helper = LayerHelper("repeat_interleave", **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.manipulation.repeat_interleave') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.manipulation.repeat_interleave', + ) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='repeat_interleave', - inputs={ - 'X': - x, - 'RepeatsTensor': - repeats if isinstance(repeats, Variable) else None - }, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'Repeats': repeats if isinstance(repeats, int) else 0 - }) + helper.append_op( + type='repeat_interleave', + inputs={ + 'X': x, + 'RepeatsTensor': repeats if isinstance(repeats, Variable) else None, + }, + outputs={'Out': out}, + attrs={ + 'dim': axis, + 'Repeats': repeats if isinstance(repeats, int) else 0, + }, + ) return out @@ -4184,7 +4464,8 @@ def moveaxis(x, source, destination, name=None): dst = [destination] if isinstance(destination, int) else destination assert len(src) == len( - dst), "'source' must have the same number with 'destination'" + dst + ), "'source' must have the same number with 'destination'" count = Counter(src).most_common(1) if count[0][1] > 1: @@ -4201,29 +4482,31 @@ def moveaxis(x, source, destination, name=None): dst_dims = list(range(ndim)) for i, axis in enumerate(zip(src, dst)): - assert isinstance(axis[0], - int), "Each elemment of 'source' must be integer." + assert isinstance( + axis[0], int + ), "Each elemment of 'source' must be integer." if axis[0] < 0: - assert axis[ - 0] >= -ndim, "'source' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis[0] >= -ndim + ), "'source' must be in the range of [-{0}, {0})".format(ndim) src[i] += ndim else: - assert axis[ - 0] < ndim, "'source' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis[0] < ndim + ), "'source' must be in the range of [-{0}, {0})".format(ndim) - assert isinstance(axis[1], - int), "Each elemment of 'source' must be integer." + assert isinstance( + axis[1], int + ), "Each elemment of 'source' must be integer." if axis[1] < 0: - assert axis[ - 1] >= -ndim, "'source' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis[1] >= -ndim + ), "'source' must be in the range of [-{0}, {0})".format(ndim) dst[i] += ndim else: - assert axis[ - 1] < ndim, "'source' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis[1] < ndim + ), "'source' must be in the range of [-{0}, {0})".format(ndim) perm[dst[i]] = src[i] src_dims.remove(src[i]) dst_dims.remove(dst[i]) @@ -4239,32 +4522,44 @@ def moveaxis(x, source, destination, name=None): out, _ = _legacy_C_ops.transpose2(x, 'axis', perm) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'moveaxis') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'moveaxis', + ) helper = LayerHelper('moveaxis', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='transpose2', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'XShape': [x_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [x]}, + outputs={'Out': [out], 'XShape': [x_shape]}, + attrs={'axis': perm}, + ) return out def non_negative_axis(arr, axis): ndim = len(arr.shape) if axis >= 0: - assert axis < ndim, "'axis' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis < ndim + ), "'axis' must be in the range of [-{0}, {0})".format(ndim) else: - assert axis >= -ndim, "'axis' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis >= -ndim + ), "'axis' must be in the range of [-{0}, {0})".format(ndim) axis += ndim return axis @@ -4307,9 +4602,10 @@ def take_along_axis(arr, indices, axis): print(result) # [[1, 2, 3]] """ - if (len(arr.shape) != len(indices.shape)): + if len(arr.shape) != len(indices.shape): raise ValueError( - "`indices` and `arr` must have the same number of dimensions!") + "`indices` and `arr` must have the same number of dimensions!" + ) axis = non_negative_axis(arr, axis) broadcast_shape = infer_broadcast_shape(arr, indices, axis) if not broadcast_shape: @@ -4325,10 +4621,14 @@ def take_along_axis(arr, indices, axis): return _C_ops.take_along_axis(arr, indices, axis) return _legacy_C_ops.take_along_axis(arr, indices, 'Axis', axis) check_variable_and_dtype( - arr, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], - 'take_along_axis') - check_variable_and_dtype(indices, 'index', ['int32', 'int64'], - 'take_along_axis') + arr, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], + 'take_along_axis', + ) + check_variable_and_dtype( + indices, 'index', ['int32', 'int64'], 'take_along_axis' + ) indices = paddle.broadcast_to(indices, broadcast_shape) broadcast_shape_list = list(broadcast_shape) broadcast_shape_list[axis] = list(arr.shape)[axis] @@ -4337,13 +4637,12 @@ def take_along_axis(arr, indices, axis): helper = LayerHelper('take_along_axis', **locals()) dtype = helper.input_dtype() result = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="take_along_axis", - inputs={ - "Input": arr, - "Index": indices - }, - attrs={"Axis": axis}, - outputs={"Result": result}) + helper.append_op( + type="take_along_axis", + inputs={"Input": arr, "Index": indices}, + attrs={"Axis": axis}, + outputs={"Result": result}, + ) return result @@ -4376,44 +4675,48 @@ def put_along_axis(arr, indices, values, axis, reduce='assign'): # [60, 40, 50]] """ - if (len(arr.shape) != len(indices.shape)): + if len(arr.shape) != len(indices.shape): raise ValueError( - "`indices` and `arr` must have the same number of dimensions!") + "`indices` and `arr` must have the same number of dimensions!" + ) axis = non_negative_axis(arr, axis) broadcast_shape = infer_broadcast_shape(arr, indices, axis) if _non_static_mode(): - values = paddle.to_tensor(values) if not isinstance( - values, paddle.Tensor) else values + values = ( + paddle.to_tensor(values) + if not isinstance(values, paddle.Tensor) + else values + ) if broadcast_shape: indices = paddle.broadcast_to(indices, broadcast_shape) values = paddle.broadcast_to(values, indices.shape) if in_dygraph_mode(): return _C_ops.put_along_axis(arr, indices, values, axis, reduce) - return _legacy_C_ops.put_along_axis(arr, indices, values, "Axis", axis, - "Reduce", reduce) + return _legacy_C_ops.put_along_axis( + arr, indices, values, "Axis", axis, "Reduce", reduce + ) check_variable_and_dtype( - arr, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], - 'put_along_axis') - check_variable_and_dtype(indices, 'index', ['int32', 'int64'], - 'put_along_axis') + arr, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], + 'put_along_axis', + ) + check_variable_and_dtype( + indices, 'index', ['int32', 'int64'], 'put_along_axis' + ) if broadcast_shape: indices = paddle.broadcast_to(indices, broadcast_shape) values = paddle.broadcast_to(values, indices.shape) helper = LayerHelper('put_along_axis', **locals()) dtype = helper.input_dtype() result = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="put_along_axis", - inputs={ - "Input": arr, - "Index": indices, - "Value": values - }, - attrs={ - "Axis": axis, - "Reduce": reduce - }, - outputs={"Result": result}) + helper.append_op( + type="put_along_axis", + inputs={"Input": arr, "Index": indices, "Value": values}, + attrs={"Axis": axis, "Reduce": reduce}, + outputs={"Result": result}, + ) return result @@ -4423,20 +4726,25 @@ def put_along_axis_(arr, indices, values, axis, reduce='assign'): Inplace version of ``put_along_axis`` API, the output Tensor will be inplaced with input ``arr``. Please refer to :ref:`api_tensor_put_along_axis`. """ - if (len(arr.shape) != len(indices.shape)): + if len(arr.shape) != len(indices.shape): raise ValueError( - "`indices` and `arr` must have the same number of dimensions!") + "`indices` and `arr` must have the same number of dimensions!" + ) axis = non_negative_axis(arr, axis) broadcast_shape = infer_broadcast_shape(arr, indices, axis) - values = paddle.to_tensor(values) if not isinstance( - values, paddle.Tensor) else values + values = ( + paddle.to_tensor(values) + if not isinstance(values, paddle.Tensor) + else values + ) if broadcast_shape: indices = paddle.broadcast_to(indices, broadcast_shape) values = paddle.broadcast_to(values, indices.shape) if in_dygraph_mode(): return _C_ops.put_along_axis_(arr, indices, values, axis, reduce) - return _legacy_C_ops.put_along_axis_(arr, indices, values, "Axis", axis, - "Reduce", reduce) + return _legacy_C_ops.put_along_axis_( + arr, indices, values, "Axis", axis, "Reduce", reduce + ) def index_add(x, index, axis, value, name=None): @@ -4475,24 +4783,36 @@ def index_add(x, index, axis, value, name=None): helper = LayerHelper("index_add", **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.manipulation.index_add') - check_variable_and_dtype(index, 'index', ['int32', 'int64'], - 'paddle.tensor.manipulation.index_add') + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.manipulation.index_add', + ) + check_variable_and_dtype( + index, + 'index', + ['int32', 'int64'], + 'paddle.tensor.manipulation.index_add', + ) check_variable_and_dtype( - value, 'add_value', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.manipulation.index_add') + value, + 'add_value', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.manipulation.index_add', + ) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='index_add', - inputs={ - 'X': x, - 'Index': index, - 'AddValue': value, - }, - outputs={'Out': out}, - attrs={'axis': axis}) + helper.append_op( + type='index_add', + inputs={ + 'X': x, + 'Index': index, + 'AddValue': value, + }, + outputs={'Out': out}, + attrs={'axis': axis}, + ) return out @@ -4528,7 +4848,7 @@ __METHODS = { 'fill_diagonal_': fill_diagonal_, 'fill_diagonal_tensor_': fill_diagonal_tensor_, "fill_diagonal_tensor": fill_diagonal_tensor, - 'tolist': tolist + 'tolist': tolist, } for name, func in __METHODS.items(): setattr(core.VarBase, name, func) diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index a2720ec48f2..cd8dbd08b94 100644 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -27,10 +27,21 @@ from .layer_function_generator import generate_layer_fn import paddle from ..static import Variable -from ..framework import core, in_dygraph_mode, _non_static_mode, LayerHelper, _in_legacy_dygraph +from ..framework import ( + core, + in_dygraph_mode, + _non_static_mode, + LayerHelper, + _in_legacy_dygraph, +) from ..fluid.framework import _in_legacy_dygraph from ..framework import _varbase_creator, convert_np_dtype_to_dtype_ -from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from ..fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from ..fluid.dygraph.inplace_utils import inplace_apis_in_dygraph_only from ..fluid.layers import utils @@ -177,15 +188,33 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): return dygraph_utils._append_activation_in_dygraph(out, act) elif _in_legacy_dygraph(): _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale - out = _legacy_C_ops.scale(x, 'scale', float(_scale), 'bias', - float(bias), 'bias_after_scale', - bias_after_scale) + out = _legacy_C_ops.scale( + x, + 'scale', + float(_scale), + 'bias', + float(bias), + 'bias_after_scale', + bias_after_scale, + ) return dygraph_utils._append_activation_in_dygraph(out, act) - check_variable_and_dtype(x, "x", [ - 'float16', 'uint16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8' - ], "scale") + check_variable_and_dtype( + x, + "x", + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + ], + "scale", + ) inputs = {'X': [x]} attrs = { 'bias': float(bias), @@ -198,10 +227,9 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): helper = LayerHelper('scale', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='scale', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='scale', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return helper.append_activation(out) @@ -240,13 +268,12 @@ def stanh(x, scale_a=0.67, scale_b=1.7159, name=None): helper = LayerHelper('stanh', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='stanh', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'scale_a': scale_a, - 'scale_b': scale_b - }) + helper.append_op( + type='stanh', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'scale_a': scale_a, 'scale_b': scale_b}, + ) return out @@ -310,20 +337,23 @@ def multiplex(inputs, index, name=None): check_type(inputs, 'inputs', (list), 'multiplex') if len(inputs) < 2: raise ValueError( - "inputs should be a list object with at least 2 elements.") + "inputs should be a list object with at least 2 elements." + ) for id, x in enumerate(inputs): - check_variable_and_dtype(x, 'input[' + str(id) + ']', - ['float32', 'float64', 'int32', 'int64'], - 'multiplex') + check_variable_and_dtype( + x, + 'input[' + str(id) + ']', + ['float32', 'float64', 'int32', 'int64'], + 'multiplex', + ) check_variable_and_dtype(index, "index", ['int32', 'int64'], 'multiplex') out = helper.create_variable_for_type_inference(inputs[0].dtype) - helper.append_op(type='multiplex', - inputs={ - 'X': inputs, - 'Ids': index - }, - outputs={'Out': [out]}) + helper.append_op( + type='multiplex', + inputs={'X': inputs, 'Ids': index}, + outputs={'Out': [out]}, + ) return out @@ -337,9 +367,15 @@ def scale_(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): return _C_ops.scale_(x, scale, float(bias), bias_after_scale) if _in_legacy_dygraph(): _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale - return _legacy_C_ops.scale_(x, 'scale', float(_scale), 'bias', - float(bias), 'bias_after_scale', - bias_after_scale) + return _legacy_C_ops.scale_( + x, + 'scale', + float(_scale), + 'bias', + float(bias), + 'bias_after_scale', + bias_after_scale, + ) def pow(x, y, name=None): @@ -395,31 +431,28 @@ def pow(x, y, name=None): return _C_ops.elementwise_pow(x, y) else: raise TypeError( - 'y must be scalar or tensor type, but received: %s ' % - (y.dtype)) + 'y must be scalar or tensor type, but received: %s ' % (y.dtype) + ) if _in_legacy_dygraph(): if isinstance(y, (int, float)): return _legacy_C_ops.pow(x, 'factor', y) elif isinstance(y, (paddle.Tensor, Variable)): - return _elementwise_op_in_dygraph(x, - y, - axis=-1, - act=None, - op_name='elementwise_pow') + return _elementwise_op_in_dygraph( + x, y, axis=-1, act=None, op_name='elementwise_pow' + ) else: raise TypeError( - 'y must be scalar or tensor type, but received: %s ' % - (y.dtype)) + 'y must be scalar or tensor type, but received: %s ' % (y.dtype) + ) # in static graph mode if isinstance(y, (int, float)): helper = LayerHelper('pow', **locals()) inputs = {'X': x} attrs = {'factor': y} out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='pow', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='pow', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out elif isinstance(y, (paddle.Tensor, Variable)): # TODO A potential speed improvement is supporting different types in C++ and removing the cast ops here @@ -427,8 +460,9 @@ def pow(x, y, name=None): out = helper.create_variable_for_type_inference(dtype=x.dtype) return _elementwise_op(LayerHelper('elementwise_pow', **locals())) else: - raise TypeError('y must be scalar or tensor type, but received: %s ' % - (type(y))) + raise TypeError( + 'y must be scalar or tensor type, but received: %s ' % (type(y)) + ) OP_NAMEMAPPING = { @@ -445,13 +479,9 @@ OP_NAMEMAPPING = { @dygraph_only -def _elementwise_op_in_dygraph(x, - y, - axis=-1, - act=None, - use_mkldnn=False, - op_name=None): - +def _elementwise_op_in_dygraph( + x, y, axis=-1, act=None, use_mkldnn=False, op_name=None +): def is_inplace(op_name): return op_name[-1] == "_" @@ -462,16 +492,17 @@ def _elementwise_op_in_dygraph(x, if in_dygraph_mode(): op = getattr( _C_ops, - OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name) + OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name, + ) out = op(x, y) if _in_legacy_dygraph(): op = getattr(_legacy_C_ops, op_name) out = op(x, y, 'axis', axis, 'use_mkldnn', use_mkldnn) - return dygraph_utils._append_activation_in_dygraph(out, - act, - use_mkldnn=use_mkldnn) + return dygraph_utils._append_activation_in_dygraph( + out, act, use_mkldnn=use_mkldnn + ) def _elementwise_op(helper): @@ -485,11 +516,17 @@ def _elementwise_op(helper): assert x is not None, 'x cannot be None in {}'.format(original_op_type) assert y is not None, 'y cannot be None in {}'.format(original_op_type) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - original_op_type) + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + original_op_type, + ) check_variable_and_dtype( - y, 'y', ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - original_op_type) + y, + 'y', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + original_op_type, + ) axis = helper.kwargs.get('axis', -1) use_mkldnn = helper.kwargs.get('use_mkldnn', False) @@ -499,20 +536,16 @@ def _elementwise_op(helper): if name is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: - out = helper.create_variable(name=name, - dtype=x.dtype, - persistable=False) - - helper.append_op(type=op_type, - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={ - 'axis': axis, - 'use_mkldnn': use_mkldnn - }) + out = helper.create_variable( + name=name, dtype=x.dtype, persistable=False + ) + + helper.append_op( + type=op_type, + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={'axis': axis, 'use_mkldnn': use_mkldnn}, + ) return helper.append_activation(out) @@ -592,8 +625,10 @@ def add_(x, y, name=None): out_shape = broadcast_shape(x.shape, y.shape) if out_shape != x.shape: raise ValueError( - "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation." - .format(out_shape, x.shape)) + "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format( + out_shape, x.shape + ) + ) if in_dygraph_mode(): return _C_ops.add_(x, y) @@ -663,11 +698,9 @@ def subtract(x, y, name=None): return _C_ops.subtract(x, y) else: if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) else: return _elementwise_op(LayerHelper(op_type, **locals())) @@ -684,17 +717,17 @@ def subtract_(x, y, name=None): out_shape = broadcast_shape(x.shape, y.shape) if out_shape != x.shape: raise ValueError( - "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation." - .format(out_shape, x.shape)) + "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format( + out_shape, x.shape + ) + ) if in_dygraph_mode(): return _C_ops.subtract_(x, y) else: - out = _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_sub_') + out = _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_sub_' + ) return out @@ -735,11 +768,9 @@ def divide(x, y, name=None): return _C_ops.divide(x, y) else: if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) else: return _elementwise_op(LayerHelper(op_type, **locals())) @@ -839,8 +870,10 @@ def remainder_(x, y, name=None): out_shape = broadcast_shape(x.shape, y.shape) if out_shape != x.shape: raise ValueError( - "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation." - .format(out_shape, x.shape)) + "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format( + out_shape, x.shape + ) + ) return _elementwise_op_in_dygraph(x, y, axis=axis, op_name=op_type) @@ -892,16 +925,15 @@ def multiply(x, y, name=None): return _C_ops.multiply(x, y) else: if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) else: if x.dtype != y.dtype: raise TypeError( 'Input tensors must be same type, but received type of x: %s, type of y: %s ' - % (x.dtype, y.dtype)) + % (x.dtype, y.dtype) + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -966,11 +998,9 @@ def maximum(x, y, name=None): if in_dygraph_mode(): return _C_ops.maximum(x, y) elif _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1034,11 +1064,9 @@ def minimum(x, y, name=None): if in_dygraph_mode(): return _C_ops.minimum(x, y) elif _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1104,11 +1132,9 @@ def fmax(x, y, name=None): if in_dygraph_mode(): return _C_ops.fmax(x, y, axis) if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1174,11 +1200,9 @@ def fmin(x, y, name=None): if in_dygraph_mode(): return _C_ops.fmin(x, y, axis) if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1273,24 +1297,59 @@ def sum(x, axis=None, dtype=None, keepdim=False, name=None): if _in_legacy_dygraph(): if dtype_flag: - return _legacy_C_ops.reduce_sum(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all_flag, - 'in_dtype', x.dtype, 'out_dtype', - dtype) + return _legacy_C_ops.reduce_sum( + x, + 'dim', + axis, + 'keep_dim', + keepdim, + 'reduce_all', + reduce_all_flag, + 'in_dtype', + x.dtype, + 'out_dtype', + dtype, + ) else: - return _legacy_C_ops.reduce_sum(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all_flag) + return _legacy_C_ops.reduce_sum( + x, + 'dim', + axis, + 'keep_dim', + keepdim, + 'reduce_all', + reduce_all_flag, + ) attrs = {'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all_flag} if dtype_flag: attrs.update({'in_dtype': x.dtype, 'out_dtype': dtype}) - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', - 'complex64', 'complex128', u'bool', u'float16', u'float32', u'float64', - u'int32', u'int64', u'complex64', u'complex128' - ], 'sum') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'complex64', + 'complex128', + u'bool', + u'float16', + u'float32', + u'float64', + u'int32', + u'int64', + u'complex64', + u'complex128', + ], + 'sum', + ) check_type(axis, 'axis', (int, list, tuple, type(None), Variable), 'sum') @@ -1299,10 +1358,9 @@ def sum(x, axis=None, dtype=None, keepdim=False, name=None): out = helper.create_variable_for_type_inference(dtype=dtype) else: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_sum', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_sum', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -1353,8 +1411,9 @@ def nansum(x, axis=None, dtype=None, keepdim=False, name=None): out5 = paddle.nansum(y, axis=[1, 2]) # [8, 19] out6 = paddle.nansum(y, axis=[0, 1]) # [9, 18] """ - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'nansum') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'nansum' + ) check_type(axis, 'axis', (int, list, tuple, type(None)), 'nansum') zero_tensor = paddle.zeros_like(x) @@ -1419,16 +1478,17 @@ def nanmean(x, axis=None, keepdim=False, name=None): """ if isinstance(axis, int): axis = [axis] - check_variable_and_dtype(x, 'x/input', - ['uint16', 'float16', 'float32', 'float64'], - 'nanmean') + check_variable_and_dtype( + x, 'x/input', ['uint16', 'float16', 'float32', 'float64'], 'nanmean' + ) if axis is not None: check_type(axis, 'axis/dim', (int, list, tuple), 'nanmean') cnt = paddle.sum(~paddle.isnan(x), axis=axis, keepdim=keepdim) return paddle.divide( paddle.nansum(x, axis=axis, keepdim=keepdim, name=name), - cnt.astype(x.dtype)) + cnt.astype(x.dtype), + ) def count_nonzero(x, axis=None, keepdim=False, name=None): @@ -1485,8 +1545,9 @@ def count_nonzero(x, axis=None, keepdim=False, name=None): axis = [axis] dims = len(x.shape) for i in range(len(axis)): - if not isinstance(axis[i], int) or not (axis[i] < dims - and axis[i] >= -dims): + if not isinstance(axis[i], int) or not ( + axis[i] < dims and axis[i] >= -dims + ): raise ValueError( "Axis should be None, int, or a list, element should in range [-rank(x), rank(x))." ) @@ -1568,18 +1629,29 @@ def add_n(inputs, name=None): if isinstance(inputs, list) or isinstance(inputs, tuple): if len(inputs) > 0: for input in inputs: - check_variable_and_dtype(input, "inputs", \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'add_n') + check_variable_and_dtype( + input, + "inputs", + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'add_n', + ) else: - check_variable_and_dtype(inputs, "inputs", \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'add_n') + check_variable_and_dtype( + inputs, + "inputs", + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'add_n', + ) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('inputs')) - helper.append_op(type='sum', - inputs={'X': inputs}, - outputs={'Out': out}, - attrs={'use_mkldnn': False}) + dtype=helper.input_dtype('inputs') + ) + helper.append_op( + type='sum', + inputs={'X': inputs}, + outputs={'Out': out}, + attrs={'use_mkldnn': False}, + ) return out @@ -1622,15 +1694,14 @@ def trunc(input, name=None): attrs = {} helper = LayerHelper("trunc", **locals()) - check_variable_and_dtype(input, 'X', - ['int32', 'int64', 'float32', 'float64'], - 'trunc') + check_variable_and_dtype( + input, 'X', ['int32', 'int64', 'float32', 'float64'], 'trunc' + ) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="trunc", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="trunc", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1709,8 +1780,9 @@ def mm(input, mat2, name=None): def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): - check_variable_and_dtype(val, name, - ['float16', 'float32', 'float64'], 'mm') + check_variable_and_dtype( + val, name, ['float16', 'float32', 'float64'], 'mm' + ) x_shape = list(x.shape) y_shape = list(y.shape) if len(x_shape) == 1: @@ -1725,7 +1797,8 @@ def mm(input, mat2, name=None): "After performing an optional transpose, Input X's width should be " "equal to Y's width for multiplication " "prerequisites. But received X's shape: %s, Y's shape: %s\n" - % (x_shape, y_shape)) + % (x_shape, y_shape) + ) if len(y_shape) > 2 and len(x_shape) > 2: for i, dim_x in enumerate(x_shape[:-2]): @@ -1737,18 +1810,16 @@ def mm(input, mat2, name=None): "When the matrix is larger than 2 dimensions, the higher " "dimensional values of the two matrices need to be equal. " "But received x_shape[%d] != y_shape[%d]. X's shape: %s, " - "Y's shape: %s.\n" % (i, i, x_shape, y_shape)) + "Y's shape: %s.\n" % (i, i, x_shape, y_shape) + ) __check_input(input, mat2) helper = LayerHelper('mm', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='matmul_v2', - inputs={ - 'X': input, - 'Y': mat2 - }, - outputs={'Out': out}) + helper.append_op( + type='matmul_v2', inputs={'X': input, 'Y': mat2}, outputs={'Out': out} + ) return out @@ -1796,36 +1867,50 @@ def addmm(input, x, y, beta=1.0, alpha=1.0, name=None): y_shape = y.shape if not len(x_shape) == len(y_shape) == 2: raise ValueError( - "The dimention of x, y should be 2 but receive x's shape: {}, y's shape: {}" - .format(x_shape, y_shape)) + "The dimention of x, y should be 2 but receive x's shape: {}, y's shape: {}".format( + x_shape, y_shape + ) + ) if x_shape[1] != y_shape[0]: raise ValueError( - "The input Variable x's width must be equal with Variable y' height. But received x's shape = {}, y's shape = {}." - .format(x_shape, y_shape)) + "The input Variable x's width must be equal with Variable y' height. But received x's shape = {}, y's shape = {}.".format( + x_shape, y_shape + ) + ) if len(input_shape) == 2: if input_shape[0] != x_shape[0]: if input_shape[0] != 1: raise ValueError( - "When x's dimension[0] is not equal with input's dimension[0], input's dimension[0] must be 1 but got {}" - .format(input_shape[0])) + "When x's dimension[0] is not equal with input's dimension[0], input's dimension[0] must be 1 but got {}".format( + input_shape[0] + ) + ) if input_shape[1] != y_shape[1] and input_shape[1] != 1: raise ValueError( - "When y's dimension[1] is not equal with input's dimension[1], input's dimension[1] must be 1 but got {}" - .format(input_shape[1])) + "When y's dimension[1] is not equal with input's dimension[1], input's dimension[1] must be 1 but got {}".format( + input_shape[1] + ) + ) if input_shape[1] != y_shape[1]: if input_shape[1] != 1: raise ValueError( - "When y's dimension[1] is not equal with input's dimension[1], input's dimension[1] must be 1 but got {}" - .format(input_shape[1])) + "When y's dimension[1] is not equal with input's dimension[1], input's dimension[1] must be 1 but got {}".format( + input_shape[1] + ) + ) elif len(input_shape) == 1: if input_shape[0] not in (y_shape[1], 1): raise ValueError( - "The input's shape: {} is not broadcastable with [x.shape[0], y.shape[1]]: [{},{}]" - .format(input_shape, x_shape[0], y_shape[1])) + "The input's shape: {} is not broadcastable with [x.shape[0], y.shape[1]]: [{},{}]".format( + input_shape, x_shape[0], y_shape[1] + ) + ) else: raise ValueError( - "The dimention of input should be 2 or 1 but receive input's shape: {}" - .format(input_shape)) + "The dimention of input should be 2 or 1 but receive input's shape: {}".format( + input_shape + ) + ) if in_dygraph_mode(): return _C_ops.addmm(input, x, y, alpha, beta) @@ -1838,16 +1923,16 @@ def addmm(input, x, y, beta=1.0, alpha=1.0, name=None): attrs = {'Alpha': alpha, 'Beta': beta} helper = LayerHelper("addmm", **locals()) - check_variable_and_dtype(input, 'Input', ['float32', 'float64'], - 'addmm') + check_variable_and_dtype( + input, 'Input', ['float32', 'float64'], 'addmm' + ) check_variable_and_dtype(x, 'X', ['float32', 'float64'], 'addmm') check_variable_and_dtype(y, 'Y', ['float32', 'float64'], 'addmm') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="addmm", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="addmm", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1890,19 +1975,24 @@ def renorm(x, p, axis, max_norm): if not axis < len(input_shape): raise ValueError( "the axis:{} should be less then the shape's size {}:{}".format( - axis, len(input_shape), input_shape)) + axis, len(input_shape), input_shape + ) + ) if not axis >= 0: if not axis >= -1 * len(input_shape): raise ValueError( - "the axis:{} should not be less than -1 * length of input_shape:{}" - .format(axis, -1 * len(input_shape))) + "the axis:{} should not be less than -1 * length of input_shape:{}".format( + axis, -1 * len(input_shape) + ) + ) axis = axis + len(input_shape) if in_dygraph_mode(): out = _C_ops.renorm(x, p, axis, max_norm) return out elif _in_legacy_dygraph(): - out = _legacy_C_ops.renorm(x, 'p', p, 'axis', axis, 'max_norm', - max_norm) + out = _legacy_C_ops.renorm( + x, 'p', p, 'axis', axis, 'max_norm', max_norm + ) return out inputs = {'X': x} @@ -1911,10 +2001,9 @@ def renorm(x, p, axis, max_norm): helper = LayerHelper("renorm", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="renorm", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="renorm", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1965,9 +2054,9 @@ def inner(x, y, name=None): def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): - check_variable_and_dtype(val, name, - ['float16', 'float32', 'float64'], - 'inner') + check_variable_and_dtype( + val, name, ['float16', 'float32', 'float64'], 'inner' + ) x_shape = list(xshape) y_shape = list(yshape) @@ -1978,18 +2067,16 @@ def inner(x, y, name=None): "After performing an optional transpose, Input X's last dim should be " "equal to Y's last dim for multiplication " "prerequisites. But received X's shape: %s, Y's shape: %s\n" - % (x_shape, y_shape)) + % (x_shape, y_shape) + ) __check_input(nx, ny) helper = LayerHelper('inner', **locals()) out = helper.create_variable_for_type_inference(dtype=nx.dtype) - helper.append_op(type='matmul_v2', - inputs={ - 'X': nx, - 'Y': ny.T - }, - outputs={'Out': out}) + helper.append_op( + type='matmul_v2', inputs={'X': nx, 'Y': ny.T}, outputs={'Out': out} + ) return out.reshape(dstshape) @@ -2033,19 +2120,17 @@ def outer(x, y, name=None): def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): - check_variable_and_dtype(val, name, - ['float16', 'float32', 'float64'], 'inner') + check_variable_and_dtype( + val, name, ['float16', 'float32', 'float64'], 'inner' + ) __check_input(nx, ny) helper = LayerHelper('outer', **locals()) out = helper.create_variable_for_type_inference(dtype=nx.dtype) - helper.append_op(type='matmul_v2', - inputs={ - 'X': nx, - 'Y': ny - }, - outputs={'Out': out}) + helper.append_op( + type='matmul_v2', inputs={'X': nx, 'Y': ny}, outputs={'Out': out} + ) return out @@ -2093,9 +2178,11 @@ def logsumexp(x, axis=None, keepdim=False, name=None): """ if isinstance(axis, int): axis = [axis] - reduce_all = True if axis is None \ - or len(axis)==0 \ - or len(axis) == len(x.shape) else False + reduce_all = ( + True + if axis is None or len(axis) == 0 or len(axis) == len(x.shape) + else False + ) if axis is None or len(axis) == 0: axis = [0] @@ -2104,18 +2191,18 @@ def logsumexp(x, axis=None, keepdim=False, name=None): axis = range(len(x.shape)) return _C_ops.logsumexp(x, axis, keepdim, reduce_all) if _in_legacy_dygraph(): - return _legacy_C_ops.logsumexp(x, 'axis', axis, 'keepdim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.logsumexp( + x, 'axis', axis, 'keepdim', keepdim, 'reduce_all', reduce_all + ) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'logsumexp') helper = LayerHelper('logsumexp', **locals()) attrs = {'axis': axis, 'keepdim': keepdim, 'reduce_all': reduce_all} out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='logsumexp', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='logsumexp', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -2157,14 +2244,15 @@ def inverse(x, name=None): raise ValueError( "The input of inverse is expected to be a Tensor whose number " "of dimensions is no less than 2. But reviced: %d, " - "x's shape: %s." % (len(x.shape), x.shape)) + "x's shape: %s." % (len(x.shape), x.shape) + ) _check_input(x) helper = LayerHelper('inverse', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='inverse', - inputs={'Input': [x]}, - outputs={'Output': [out]}) + helper.append_op( + type='inverse', inputs={'Input': [x]}, outputs={'Output': [out]} + ) return out @@ -2180,8 +2268,10 @@ def _get_reduce_axis(axis): axis = [axis] else: raise TypeError( - "The type of axis must be int, list or tuple, but received {}". - format(type(axis))) + "The type of axis must be int, list or tuple, but received {}".format( + type(axis) + ) + ) reduce_all = True if axis == None or axis == [] else False if axis == None: axis = [] @@ -2206,8 +2296,10 @@ def _get_reduce_all_value(axis): axis = [axis] else: raise TypeError( - "The type of axis must be int, list or tuple, but received {}". - format(type(axis))) + "The type of axis must be int, list or tuple, but received {}".format( + type(axis) + ) + ) reduce_all = True if axis == None or axis == [] else False axis = axis if axis != None and axis != [] else [0] @@ -2296,24 +2388,24 @@ def max(x, axis=None, keepdim=False, name=None): if in_dygraph_mode(): return _C_ops.max(x, axis, keepdim) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_max(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_max( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('max', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'max') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'max' + ) if not isinstance(axis, Variable) and utils._contain_var(axis): axis = utils._convert_to_tensor_list(axis) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_max', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_max', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -2399,24 +2491,24 @@ def min(x, axis=None, keepdim=False, name=None): return _C_ops.min(x, axis, keepdim) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_min(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_min( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('min', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'min') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'min' + ) if not isinstance(axis, Variable) and utils._contain_var(axis): axis = utils._convert_to_tensor_list(axis) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_min', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_min', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -2514,22 +2606,22 @@ def amax(x, axis=None, keepdim=False, name=None): if in_dygraph_mode(): return _C_ops.amax(x, axis, keepdim) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_amax(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_amax( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('amax', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'amax') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'amax' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_amax', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_amax', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -2628,21 +2720,21 @@ def amin(x, axis=None, keepdim=False, name=None): if in_dygraph_mode(): return _C_ops.amin(x, axis, keepdim) elif _in_legacy_dygraph(): - return _legacy_C_ops.reduce_amin(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_amin( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('amin', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'amin') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'amin' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_amin', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_amin', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -2856,16 +2948,27 @@ def clip(x, min=None, max=None, name=None): if min is not None: check_type(min, 'min', (float, int, Variable), 'clip') if isinstance(min, Variable): - check_dtype(min.dtype, 'min', ['float32', 'float64', 'int32'], - 'clip', '(When the type of min in clip is Variable.)') + check_dtype( + min.dtype, + 'min', + ['float32', 'float64', 'int32'], + 'clip', + '(When the type of min in clip is Variable.)', + ) if max is not None: check_type(max, 'max', (float, int, Variable), 'clip') if isinstance(max, Variable): - check_dtype(max.dtype, 'max', ['float32', 'float64', 'int32'], - 'clip', '(When the type of max in clip is Variable.)') + check_dtype( + max.dtype, + 'max', + ['float32', 'float64', 'int32'], + 'clip', + '(When the type of max in clip is Variable.)', + ) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'clip') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'clip' + ) inputs = {'X': x} attrs = {'min': min_, 'max': max_} @@ -2884,11 +2987,11 @@ def clip(x, min=None, max=None, name=None): helper = LayerHelper('clip', **locals()) output = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('x')) - helper.append_op(type='clip', - inputs=inputs, - outputs={'Out': [output]}, - attrs=attrs) + dtype=helper.input_dtype('x') + ) + helper.append_op( + type='clip', inputs=inputs, outputs={'Out': [output]}, attrs=attrs + ) return output @@ -2957,52 +3060,56 @@ def trace(x, offset=0, axis1=0, axis2=1, name=None): """ def __check_input(x, offset, axis1, axis2): - check_dtype(x.dtype, 'Input', - ['int32', 'int64', 'float16', 'float32', 'float64'], - 'trace') + check_dtype( + x.dtype, + 'Input', + ['int32', 'int64', 'float16', 'float32', 'float64'], + 'trace', + ) input_shape = list(x.shape) - assert len(input_shape) >= 2, \ - "The x must be at least 2-dimensional, " \ - "But received Input x's dimensional: %s.\n" % \ - len(input_shape) + assert len(input_shape) >= 2, ( + "The x must be at least 2-dimensional, " + "But received Input x's dimensional: %s.\n" % len(input_shape) + ) axis1_ = axis1 if axis1 >= 0 else len(input_shape) + axis1 axis2_ = axis2 if axis2 >= 0 else len(input_shape) + axis2 - assert ((0 <= axis1_) and (axis1_ < len(input_shape))), \ - "The argument axis1 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert (0 <= axis1_) and (axis1_ < len(input_shape)), ( + "The argument axis1 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape)), len(input_shape) - 1, axis1) + ) - assert ((0 <= axis2_) and (axis2_ < len(input_shape))), \ - "The argument axis2 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert (0 <= axis2_) and (axis2_ < len(input_shape)), ( + "The argument axis2 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape)), len(input_shape) - 1, axis2) + ) - - assert axis1_ != axis2_, \ - "axis1 and axis2 cannot be the same axis." \ - "But received axis1 = %d, axis2 = %d\n"%(axis1, axis2) + assert axis1_ != axis2_, ( + "axis1 and axis2 cannot be the same axis." + "But received axis1 = %d, axis2 = %d\n" % (axis1, axis2) + ) if in_dygraph_mode(): return _C_ops.trace(x, offset, axis1, axis2) if _in_legacy_dygraph(): - return _legacy_C_ops.trace(x, 'offset', offset, 'axis1', axis1, 'axis2', - axis2) + return _legacy_C_ops.trace( + x, 'offset', offset, 'axis1', axis1, 'axis2', axis2 + ) __check_input(x, offset, axis1, axis2) helper = LayerHelper('trace', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='trace', - inputs={'Input': [x]}, - attrs={ - 'offset': offset, - 'axis1': axis1, - 'axis2': axis2 - }, - outputs={'Out': [out]}) + helper.append_op( + type='trace', + inputs={'Input': [x]}, + attrs={'offset': offset, 'axis1': axis1, 'axis2': axis2}, + outputs={'Out': [out]}, + ) return out @@ -3075,47 +3182,52 @@ def diagonal(x, offset=0, axis1=0, axis2=1, name=None): return _C_ops.diagonal(x, offset, axis1, axis2) else: if _in_legacy_dygraph(): - return _legacy_C_ops.diagonal(x, 'offset', offset, 'axis1', axis1, - 'axis2', axis2) + return _legacy_C_ops.diagonal( + x, 'offset', offset, 'axis1', axis1, 'axis2', axis2 + ) def __check_input(x, offset, axis1, axis2): - check_dtype(x.dtype, 'Input', - ['bool', 'int32', 'int64', 'float16', 'float32', 'float64'], - 'diagonal') + check_dtype( + x.dtype, + 'Input', + ['bool', 'int32', 'int64', 'float16', 'float32', 'float64'], + 'diagonal', + ) input_shape = list(x.shape) - assert len(input_shape) >= 2, \ - "The x must be at least 2-dimensional, " \ - "But received Input x's dimensional: %s.\n" % \ - len(input_shape) + assert len(input_shape) >= 2, ( + "The x must be at least 2-dimensional, " + "But received Input x's dimensional: %s.\n" % len(input_shape) + ) axis1_ = axis1 if axis1 >= 0 else len(input_shape) + axis1 axis2_ = axis2 if axis2 >= 0 else len(input_shape) + axis2 - assert axis1_ < len(input_shape), \ - "The argument axis1 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert axis1_ < len(input_shape), ( + "The argument axis1 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape)), len(input_shape) - 1, axis1) + ) - assert axis2_ < len(input_shape), \ - "The argument axis2 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert axis2_ < len(input_shape), ( + "The argument axis2 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape)), len(input_shape) - 1, axis2) + ) - assert axis1_ != axis2_, \ - "axis1 and axis2 cannot be the same axis." \ - "But received axis1 = %d, axis2 = %d\n"%(axis1, axis2) + assert axis1_ != axis2_, ( + "axis1 and axis2 cannot be the same axis." + "But received axis1 = %d, axis2 = %d\n" % (axis1, axis2) + ) __check_input(x, offset, axis1, axis2) helper = LayerHelper('diagonal', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='diagonal', - inputs={'Input': [x]}, - attrs={ - 'offset': offset, - 'axis1': axis1, - 'axis2': axis2 - }, - outputs={'Out': [out]}) + helper.append_op( + type='diagonal', + inputs={'Input': [x]}, + attrs={'offset': offset, 'axis1': axis1, 'axis2': axis2}, + outputs={'Out': [out]}, + ) return out @@ -3154,9 +3266,11 @@ def kron(x, y, name=None): return _C_ops.kron(x, y) helper = LayerHelper('kron', **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'kron') + x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'kron' + ) check_variable_and_dtype( - y, 'y', ['float16', 'float32', 'float64', 'int32', 'int64'], 'kron') + y, 'y', ['float16', 'float32', 'float64', 'int32', 'int64'], 'kron' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) helper.append_op(type="kron", inputs={"X": x, "Y": y}, outputs={"Out": out}) @@ -3212,7 +3326,8 @@ def cumsum(x, axis=None, dtype=None, name=None): x = cast(x, dtype) if in_dygraph_mode(): - if axis is None: axis = -1 + if axis is None: + axis = -1 return _C_ops.cumsum(x, axis, flatten, False, False) if _in_legacy_dygraph(): if axis is None: @@ -3286,27 +3401,29 @@ def logcumsumexp(x, axis=None, dtype=None, name=None): x = cast(x, dtype) if in_dygraph_mode(): - if axis is None: axis = -1 + if axis is None: + axis = -1 return _C_ops.logcumsumexp(x, axis, flatten, False, False) if _in_legacy_dygraph(): if axis is None: return _legacy_C_ops.logcumsumexp(x, 'flatten', flatten) else: - return _legacy_C_ops.logcumsumexp(x, 'axis', axis, 'flatten', - flatten) + return _legacy_C_ops.logcumsumexp( + x, 'axis', axis, 'flatten', flatten + ) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - "logcumsumexp") + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], "logcumsumexp" + ) helper = LayerHelper('logcumsumexp', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='logcumsumexp', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'axis': axis, - 'flatten': flatten - }) + helper.append_op( + type='logcumsumexp', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'axis': axis, 'flatten': flatten}, + ) return out @@ -3366,17 +3483,21 @@ def cumprod(x, dim=None, dtype=None, name=None): return _legacy_C_ops.cumprod(x, 'dim', dim) check_variable_and_dtype( - x, "x", + x, + "x", ['complex64', 'complex128', 'float32', 'float64', 'int32', 'int64'], - 'cumprod') + 'cumprod', + ) check_type(dim, 'dim', int, 'cumprod') helper = LayerHelper('cumprod', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='cumprod', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'dim': dim}) + helper.append_op( + type='cumprod', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': dim}, + ) return out @@ -3407,7 +3528,8 @@ def isfinite(x, name=None): return _legacy_C_ops.isfinite_v2(x) helper = LayerHelper("isfinite_v2", **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isfinite') + x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isfinite' + ) out = helper.create_variable_for_type_inference('bool') helper.append_op(type="isfinite_v2", inputs={"X": x}, outputs={"Out": out}) return out @@ -3440,7 +3562,8 @@ def isinf(x, name=None): return _legacy_C_ops.isinf_v2(x) helper = LayerHelper("isinf_v2", **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isinf') + x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isinf' + ) out = helper.create_variable_for_type_inference(dtype='bool') helper.append_op(type="isinf_v2", inputs={"X": x}, outputs={"Out": out}) return out @@ -3474,7 +3597,8 @@ def isnan(x, name=None): return _legacy_C_ops.isnan_v2(x) helper = LayerHelper("isnan_v2", **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isnan') + x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isnan' + ) out = helper.create_variable_for_type_inference(dtype='bool') helper.append_op(type="isnan_v2", inputs={"X": x}, outputs={"Out": out}) return out @@ -3535,8 +3659,9 @@ def prod(x, axis=None, keepdim=False, dtype=None, name=None): """ if dtype is not None: - check_dtype(dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], - 'prod') + check_dtype( + dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], 'prod' + ) if x.dtype != convert_np_dtype_to_dtype_(dtype): x = cast(x, dtype) @@ -3551,35 +3676,39 @@ def prod(x, axis=None, keepdim=False, dtype=None, name=None): dim = [dim] else: raise TypeError( - "The type of axis must be int, list or tuple, but received {}" - .format(type(dim))) + "The type of axis must be int, list or tuple, but received {}".format( + type(dim) + ) + ) - reduce_all = True if dim is None or len(dim) == 0 or len(dim) == len( - x.shape) else False + reduce_all = ( + True + if dim is None or len(dim) == 0 or len(dim) == len(x.shape) + else False + ) if dim is None or len(dim) == 0: dim = [0] if in_dygraph_mode(): return _C_ops.reduce_prod(x, dim, keepdim, reduce_all) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_prod(x, 'dim', dim, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_prod( + x, 'dim', dim, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('reduce_prod', **locals()) - check_variable_and_dtype(x, 'x/input', - ['float32', 'float64', 'int32', 'int64'], - 'reduce_prod') + check_variable_and_dtype( + x, 'x/input', ['float32', 'float64', 'int32', 'int64'], 'reduce_prod' + ) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) if not isinstance(dim, Variable) and utils._contain_var(dim): dim = utils._convert_to_tensor_list(dim) - helper.append_op(type='reduce_prod', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': dim, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_prod', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': dim, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -3697,13 +3826,16 @@ def increment(x, value=1.0, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.increment(x, 'step', value) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'increment') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'increment' + ) helper = LayerHelper("increment", **locals()) - helper.append_op(type='increment', - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={'step': float(value)}) + helper.append_op( + type='increment', + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={'step': float(value)}, + ) return x @@ -3774,13 +3906,14 @@ def all(x, axis=None, keepdim=False, name=None): if _in_legacy_dygraph(): axis = axis if axis != None and axis != [] else [0] - return _legacy_C_ops.reduce_all(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all_flag) + return _legacy_C_ops.reduce_all( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all_flag + ) attrs = { 'dim': axis if axis != None and axis != [] and axis != () else [0], 'keep_dim': keepdim, - 'reduce_all': reduce_all_flag + 'reduce_all': reduce_all_flag, } check_variable_and_dtype(x, 'x', ['bool'], 'all') @@ -3788,10 +3921,9 @@ def all(x, axis=None, keepdim=False, name=None): helper = LayerHelper('all', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_all', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_all', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -3863,13 +3995,14 @@ def any(x, axis=None, keepdim=False, name=None): if _in_legacy_dygraph(): axis = axis if axis != None and axis != [] else [0] - return _legacy_C_ops.reduce_any(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all_flag) + return _legacy_C_ops.reduce_any( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all_flag + ) attrs = { 'dim': axis if axis != None and axis != [] and axis != () else [0], 'keep_dim': keepdim, - 'reduce_all': reduce_all_flag + 'reduce_all': reduce_all_flag, } check_variable_and_dtype(x, 'x', ['bool'], 'any') @@ -3878,10 +4011,9 @@ def any(x, axis=None, keepdim=False, name=None): helper = LayerHelper('any', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_any', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_any', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -3948,9 +4080,11 @@ def conj(x, name=None): return _legacy_C_ops.conj(x) check_variable_and_dtype( - x, "x", + x, + "x", ['complex64', 'complex128', 'float32', 'float64', 'int32', 'int64'], - 'conj') + 'conj', + ) helper = LayerHelper('conj', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) @@ -4057,12 +4191,9 @@ def neg(x, name=None): # [0.4 0.2 -0.1 -0.3] """ - return scale(x, - scale=-1.0, - bias=0.0, - bias_after_scale=True, - act=None, - name=name) + return scale( + x, scale=-1.0, bias=0.0, bias_after_scale=True, act=None, name=name + ) def atan2(x, y, name=None): @@ -4115,11 +4246,17 @@ def atan2(x, y, name=None): return _legacy_C_ops.atan2(x, y) else: check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], - 'atan2') + x, + 'x', + ['int32', 'int64', 'float16', 'float32', 'float64'], + 'atan2', + ) check_variable_and_dtype( - y, 'y', ['int32', 'int64', 'float16', 'float32', 'float64'], - 'atan2') + y, + 'y', + ['int32', 'int64', 'float16', 'float32', 'float64'], + 'atan2', + ) helper = LayerHelper('atan2', **locals()) inputs = {'X1': x, 'X2': y} @@ -4178,10 +4315,9 @@ def logit(x, eps=None, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'logit') helper = LayerHelper("logit", **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='logit', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'eps': eps}) + helper.append_op( + type='logit', inputs={'X': x}, outputs={'Out': out}, attrs={'eps': eps} + ) return out @@ -4254,8 +4390,10 @@ def lerp_(x, y, weight, name=None): out_shape = broadcast_shape(out_shape, weight.shape) if out_shape != x.shape: raise ValueError( - "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation." - .format(out_shape, x.shape)) + "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format( + out_shape, x.shape + ) + ) if in_dygraph_mode(): return _C_ops.lerp_(x, y, weight) return _legacy_C_ops.lerp_(x, y, weight) @@ -4363,26 +4501,28 @@ def rad2deg(x, name=None): x = cast(x, dtype="float32") return _legacy_C_ops.scale(x, 'scale', rad2deg_scale) else: - check_variable_and_dtype(x, 'x', - ['int32', 'int64', 'float32', 'float64'], - 'rad2deg') + check_variable_and_dtype( + x, 'x', ['int32', 'int64', 'float32', 'float64'], 'rad2deg' + ) helper = LayerHelper('rad2deg', **locals()) out_cast = x if convert_dtype(x.dtype) in ['int32', 'int64']: out_cast = helper.create_variable_for_type_inference( - dtype=paddle.float32) - helper.append_op(type='cast', - inputs={'X': x}, - outputs={'Out': out_cast}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': paddle.float32 - }) + dtype=paddle.float32 + ) + helper.append_op( + type='cast', + inputs={'X': x}, + outputs={'Out': out_cast}, + attrs={'in_dtype': x.dtype, 'out_dtype': paddle.float32}, + ) out = helper.create_variable_for_type_inference(dtype=out_cast.dtype) - helper.append_op(type='scale', - inputs={'X': out_cast}, - outputs={'Out': out}, - attrs={'scale': rad2deg_scale}) + helper.append_op( + type='scale', + inputs={'X': out_cast}, + outputs={'Out': out}, + attrs={'scale': rad2deg_scale}, + ) return out @@ -4429,26 +4569,28 @@ def deg2rad(x, name=None): x = cast(x, dtype="float32") return _legacy_C_ops.scale(x, 'scale', deg2rad_scale) else: - check_variable_and_dtype(x, 'x', - ['int32', 'int64', 'float32', 'float64'], - 'deg2rad') + check_variable_and_dtype( + x, 'x', ['int32', 'int64', 'float32', 'float64'], 'deg2rad' + ) helper = LayerHelper('deg2rad', **locals()) out_cast = x if convert_dtype(x.dtype) in ['int32', 'int64']: out_cast = helper.create_variable_for_type_inference( - dtype=paddle.float32) - helper.append_op(type='cast', - inputs={'X': x}, - outputs={'Out': out_cast}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': paddle.float32 - }) + dtype=paddle.float32 + ) + helper.append_op( + type='cast', + inputs={'X': x}, + outputs={'Out': out_cast}, + attrs={'in_dtype': x.dtype, 'out_dtype': paddle.float32}, + ) out = helper.create_variable_for_type_inference(dtype=out_cast.dtype) - helper.append_op(type='scale', - inputs={'X': out_cast}, - outputs={'Out': out}, - attrs={'scale': deg2rad_scale}) + helper.append_op( + type='scale', + inputs={'X': out_cast}, + outputs={'Out': out}, + attrs={'scale': deg2rad_scale}, + ) return out @@ -4513,11 +4655,16 @@ def gcd(x, y, name=None): # paddle.mod will raise an error when any element of y is 0. To avoid # that, we change those zeros to ones. Their values don't matter because # they won't be used. - y_not_equal_0 = (y != 0) + y_not_equal_0 = y != 0 y_safe = paddle.where(y_not_equal_0, y, paddle.ones(y.shape, y.dtype)) - x, y = (paddle.where(y_not_equal_0, y, x), - paddle.where(y_not_equal_0, paddle.mod(x, y_safe), - paddle.zeros(y.shape, y.dtype))) + x, y = ( + paddle.where(y_not_equal_0, y, x), + paddle.where( + y_not_equal_0, + paddle.mod(x, y_safe), + paddle.zeros(y.shape, y.dtype), + ), + ) return (paddle.where(x < y, y, x), paddle.where(x < y, x, y)) if paddle.in_dynamic_mode(): @@ -4586,8 +4733,9 @@ def lcm(x, y, name=None): # they won't be used. d_equal_0 = paddle.equal(d, 0) d_safe = paddle.where(d_equal_0, paddle.ones(d.shape, d.dtype), d) - out = paddle.where(d_equal_0, paddle.zeros(d.shape, d.dtype), - paddle.abs(x * y) // d_safe) + out = paddle.where( + d_equal_0, paddle.zeros(d.shape, d.dtype), paddle.abs(x * y) // d_safe + ) return out @@ -4682,14 +4830,16 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): attrs_1 += ('starts', starts_1) ends_1 = [dim_len - 1] attrs_1 += ('ends', ends_1) - input_front = _C_ops.slice(new_input, axes, starts_1, ends_1, - infer_flags, []) + input_front = _C_ops.slice( + new_input, axes, starts_1, ends_1, infer_flags, [] + ) starts_2 = [1] attrs_2 += ('starts', starts_2) ends_2 = [dim_len] attrs_2 += ('ends', ends_2) - input_back = _C_ops.slice(new_input, axes, starts_2, ends_2, - infer_flags, []) + input_back = _C_ops.slice( + new_input, axes, starts_2, ends_2, infer_flags, [] + ) if x.dtype == paddle.bool: return _C_ops.logical_xor(input_back, input_front) @@ -4722,14 +4872,34 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): attrs_1 += ('starts', starts_1) ends_1 = [dim_len - 1] attrs_1 += ('ends', ends_1) - input_front = _legacy_C_ops.slice(new_input, None, None, None, None, 'axes', axes, \ - 'infer_flags', infer_flags, *attrs_1) + input_front = _legacy_C_ops.slice( + new_input, + None, + None, + None, + None, + 'axes', + axes, + 'infer_flags', + infer_flags, + *attrs_1 + ) starts_2 = [1] attrs_2 += ('starts', starts_2) ends_2 = [dim_len] attrs_2 += ('ends', ends_2) - input_back = _legacy_C_ops.slice(new_input, None, None, None, None, 'axes', axes, \ - 'infer_flags', infer_flags, *attrs_2) + input_back = _legacy_C_ops.slice( + new_input, + None, + None, + None, + None, + 'axes', + axes, + 'infer_flags', + infer_flags, + *attrs_2 + ) if x.dtype == paddle.bool: return _legacy_C_ops.logical_xor(input_back, input_front) @@ -4737,7 +4907,8 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): return elementwise_sub(input_back, input_front, axis=axis) else: check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'bool', 'int32', 'int64'], 'diff') + x, 'x', ['float32', 'float64', 'bool', 'int32', 'int64'], 'diff' + ) check_type(axis, 'axis', (int), 'diff') helper = LayerHelper('diff', **locals()) has_pend = False @@ -4754,10 +4925,12 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): if has_pend: new_input = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='concat', - inputs={'X': input_list}, - outputs={'Out': [new_input]}, - attrs={'axis': axis}) + helper.append_op( + type='concat', + inputs={'X': input_list}, + outputs={'Out': [new_input]}, + attrs={'axis': axis}, + ) else: new_input = x @@ -4768,29 +4941,32 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): attrs_1['starts'] = starts_1 attrs_1['ends'] = ends_1 input_front = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='slice', - inputs={'Input': new_input}, - attrs=attrs_1, - outputs={'Out': input_front}) + helper.append_op( + type='slice', + inputs={'Input': new_input}, + attrs=attrs_1, + outputs={'Out': input_front}, + ) attrs_2 = {'axes': axes} starts_2 = [1] ends_2 = [dim_len] attrs_2['starts'] = starts_2 attrs_2['ends'] = ends_2 input_back = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='slice', - inputs={'Input': new_input}, - attrs=attrs_2, - outputs={'Out': input_back}) + helper.append_op( + type='slice', + inputs={'Input': new_input}, + attrs=attrs_2, + outputs={'Out': input_back}, + ) if dtype == paddle.bool: out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='logical_xor', - inputs={ - "X": input_back, - "Y": input_front - }, - outputs={"Out": out}) + helper.append_op( + type='logical_xor', + inputs={"X": input_back, "Y": input_front}, + outputs={"Out": out}, + ) else: out = elementwise_sub(input_back, input_front, axis=axis) @@ -4843,14 +5019,15 @@ def angle(x, name=None): elif paddle.in_dynamic_mode(): return _legacy_C_ops.angle(x) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'complex64', 'complex128'], - 'angle') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'complex64', 'complex128'], 'angle' + ) op_type = "angle" helper = LayerHelper(op_type, **locals()) inputs = {"X": x} out = helper.create_variable_for_type_inference( - dtype=_complex_to_real_dtype(x.dtype)) + dtype=_complex_to_real_dtype(x.dtype) + ) outputs = {"Out": out} helper.append_op(type=op_type, inputs=inputs, outputs=outputs) return out @@ -4899,11 +5076,9 @@ def heaviside(x, y, name=None): axis = -1 act = None if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -4935,35 +5110,37 @@ def frac(x, name=None): axis = -1 act = None if x.dtype not in [ - paddle.int32, paddle.int64, paddle.float32, paddle.float64 + paddle.int32, + paddle.int64, + paddle.float32, + paddle.float64, ]: raise TypeError( - "The data type of input must be one of ['int32', 'int64', 'float32', 'float64'], but got {}" - .format(x.dtype)) + "The data type of input must be one of ['int32', 'int64', 'float32', 'float64'], but got {}".format( + x.dtype + ) + ) if in_dygraph_mode(): y = _C_ops.trunc(x) return _C_ops.subtract(x, y) else: if _in_legacy_dygraph(): y = _legacy_C_ops.trunc(x) - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) else: inputs = {"X": x} attrs = {} helper = LayerHelper("trunc", **locals()) - check_variable_and_dtype(x, "X", - ['int32', 'int64', 'float32', 'float64'], - 'trunc') + check_variable_and_dtype( + x, "X", ['int32', 'int64', 'float32', 'float64'], 'trunc' + ) y = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="trunc", - inputs=inputs, - attrs=attrs, - outputs={"Out": y}) + helper.append_op( + type="trunc", inputs=inputs, attrs=attrs, outputs={"Out": y} + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -4993,12 +5170,17 @@ def sgn(x, name=None): """ if x.dtype not in [ - paddle.float16, paddle.float32, paddle.float64, paddle.complex64, - paddle.complex128 + paddle.float16, + paddle.float32, + paddle.float64, + paddle.complex64, + paddle.complex128, ]: raise TypeError( - "The data type of input must be one of ['float16', 'float32', 'float64', 'complex64', 'complex128'], but got {}" - .format(x.dtype)) + "The data type of input must be one of ['float16', 'float32', 'float64', 'complex64', 'complex128'], but got {}".format( + x.dtype + ) + ) if paddle.is_complex(x): expand_x = paddle.as_real(x) x_abs = paddle.abs(x) @@ -5079,18 +5261,24 @@ def take(x, index, mode='raise', name=None): """ if mode not in ['raise', 'wrap', 'clip']: raise ValueError( - "'mode' in 'take' should be 'raise', 'wrap', 'clip', but received {}." - .format(mode)) + "'mode' in 'take' should be 'raise', 'wrap', 'clip', but received {}.".format( + mode + ) + ) if paddle.in_dynamic_mode(): if not isinstance(index, (paddle.Tensor, Variable)): raise TypeError( "The type of 'index' must be Tensor, but got {}".format( - type(index))) + type(index) + ) + ) if index.dtype not in [paddle.int32, paddle.int64]: raise TypeError( - "The data type of 'index' must be one of ['int32', 'int64'], but got {}" - .format(index.dtype)) + "The data type of 'index' must be one of ['int32', 'int64'], but got {}".format( + index.dtype + ) + ) else: check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'take') @@ -5105,8 +5293,9 @@ def take(x, index, mode='raise', name=None): elif mode == 'wrap': # The out of range indices are constrained by taking the remainder. index_1d = paddle.where(index_1d < 0, index_1d % max_index, index_1d) - index_1d = paddle.where(index_1d >= max_index, index_1d % max_index, - index_1d) + index_1d = paddle.where( + index_1d >= max_index, index_1d % max_index, index_1d + ) elif mode == 'clip': # 'clip' mode disables indexing with negative numbers. index_1d = clip(index_1d, 0, max_index - 1) @@ -5140,26 +5329,32 @@ def frexp(x, name=None): print(paddle.tensor.math.frexp(x)) # (Tensor(shape=[1, 4], dtype=float32, place=Place(cpu), stop_gradient=True,[[0.50000000, 0.50000000, 0.75000000, 0.50000000]]), # Tensor(shape=[1, 4], dtype=float32, place=Place(cpu), stop_gradient=True,[[1., 2., 2., 3.]])) - """ + """ if x.dtype not in [paddle.float32, paddle.float64]: raise TypeError( - "The data type of input must be one of ['float32', 'float64'], but got {}" - .format(x.dtype)) + "The data type of input must be one of ['float32', 'float64'], but got {}".format( + x.dtype + ) + ) input_x = paddle.abs(x) exponent = paddle.floor(paddle.log2(input_x)) - exponent = paddle.where(paddle.isinf(exponent), - paddle.full_like(exponent, 0), exponent) + exponent = paddle.where( + paddle.isinf(exponent), paddle.full_like(exponent, 0), exponent + ) # 0填充 mantissa = paddle.divide(input_x, 2**exponent) # 计算exponent - exponent = paddle.where((mantissa >= 1), - paddle.add(exponent, paddle.ones_like(exponent)), - exponent) - mantissa = paddle.where((mantissa >= 1), - paddle.divide(mantissa, - 2**paddle.ones_like(exponent)), - mantissa) + exponent = paddle.where( + (mantissa >= 1), + paddle.add(exponent, paddle.ones_like(exponent)), + exponent, + ) + mantissa = paddle.where( + (mantissa >= 1), + paddle.divide(mantissa, 2 ** paddle.ones_like(exponent)), + mantissa, + ) mantissa = paddle.where((x < 0), mantissa * -1, mantissa) return mantissa, exponent diff --git a/python/paddle/tensor/ops.py b/python/paddle/tensor/ops.py index 46648f06d82..b3f88e9fcdf 100644 --- a/python/paddle/tensor/ops.py +++ b/python/paddle/tensor/ops.py @@ -12,13 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .layer_function_generator import generate_layer_fn, generate_activation_fn, generate_inplace_fn, add_sample_code +from .layer_function_generator import ( + generate_layer_fn, + generate_activation_fn, + generate_inplace_fn, + add_sample_code, +) from ..fluid.framework import in_dygraph_mode from .. import _C_ops __deprecated_func_name__ = { 'tanh_shrink': 'tanhshrink', - 'logsigmoid': 'log_sigmoid' + 'logsigmoid': 'log_sigmoid', } __activations_noattr__ = [ @@ -103,7 +108,8 @@ for _OP in set(__inplace_unary_func__): globals()[_OP] = _func add_sample_code( - globals()["sigmoid"], r""" + globals()["sigmoid"], + r""" Examples: .. code-block:: python @@ -115,10 +121,12 @@ Examples: print(out) # [0.40131234 0.450166 0.52497919 0.57444252] -""") +""", +) add_sample_code( - globals()["silu"], r""" + globals()["silu"], + r""" Examples: .. code-block:: python @@ -130,10 +138,12 @@ Examples: print(out) # [ 0.7310586 1.7615942 2.8577224, 3.9280552 ] -""") +""", +) add_sample_code( - globals()["logsigmoid"], r""" + globals()["logsigmoid"], + r""" Examples: .. code-block:: python @@ -145,10 +155,12 @@ Examples: print(out) # [-0.91301525 -0.79813887 -0.64439666 -0.55435524] -""") +""", +) add_sample_code( - globals()["exp"], r""" + globals()["exp"], + r""" Examples: .. code-block:: python @@ -159,10 +171,12 @@ Examples: print(out) # [0.67032005 0.81873075 1.10517092 1.34985881] -""") +""", +) add_sample_code( - globals()["expm1"], r""" + globals()["expm1"], + r""" Examples: .. code-block:: python @@ -173,10 +187,12 @@ Examples: print(out) # [-0.32967997, -0.18126924, 0.10517092, 0.34985882] -""") +""", +) add_sample_code( - globals()["tanh"], r""" + globals()["tanh"], + r""" Examples: .. code-block:: python @@ -187,10 +203,12 @@ Examples: print(out) # [-0.37994896 -0.19737532 0.09966799 0.29131261] -""") +""", +) add_sample_code( - globals()["atan"], r""" + globals()["atan"], + r""" Examples: .. code-block:: python @@ -201,10 +219,12 @@ Examples: print(out) # [-0.38050638 -0.19739556 0.09966865 0.29145679] -""") +""", +) add_sample_code( - globals()["tanh_shrink"], r""" + globals()["tanh_shrink"], + r""" Examples: .. code-block:: python @@ -216,10 +236,12 @@ Examples: print(out) # [-0.020051, -0.00262468, 0.000332005, 0.00868739] -""") +""", +) add_sample_code( - globals()["sqrt"], r""" + globals()["sqrt"], + r""" Examples: .. code-block:: python @@ -230,10 +252,12 @@ Examples: print(out) # [0.31622777 0.4472136 0.54772256 0.63245553] -""") +""", +) add_sample_code( - globals()["rsqrt"], r""" + globals()["rsqrt"], + r""" Examples: .. code-block:: python @@ -244,10 +268,12 @@ Examples: print(out) # [3.16227766 2.23606798 1.82574186 1.58113883] -""") +""", +) add_sample_code( - globals()["abs"], r""" + globals()["abs"], + r""" Examples: .. code-block:: python @@ -258,10 +284,12 @@ Examples: print(out) # [0.4 0.2 0.1 0.3] -""") +""", +) add_sample_code( - globals()["ceil"], r""" + globals()["ceil"], + r""" Examples: .. code-block:: python @@ -272,10 +300,12 @@ Examples: print(out) # [-0. -0. 1. 1.] -""") +""", +) add_sample_code( - globals()["floor"], r""" + globals()["floor"], + r""" Examples: .. code-block:: python @@ -286,10 +316,12 @@ Examples: print(out) # [-1. -1. 0. 0.] -""") +""", +) add_sample_code( - globals()["cos"], r""" + globals()["cos"], + r""" Examples: .. code-block:: python @@ -300,10 +332,12 @@ Examples: print(out) # [0.92106099 0.98006658 0.99500417 0.95533649] -""") +""", +) add_sample_code( - globals()["tan"], r""" + globals()["tan"], + r""" Examples: .. code-block:: python @@ -314,10 +348,12 @@ Examples: print(out) # [-0.42279324, -0.20271005, 0.10033467, 0.30933627] -""") +""", +) add_sample_code( - globals()["acos"], r""" + globals()["acos"], + r""" Examples: .. code-block:: python @@ -328,10 +364,12 @@ Examples: print(out) # [1.98231317 1.77215425 1.47062891 1.26610367] -""") +""", +) add_sample_code( - globals()["sin"], r""" + globals()["sin"], + r""" Examples: .. code-block:: python @@ -342,10 +380,12 @@ Examples: print(out) # [-0.38941834 -0.19866933 0.09983342 0.29552021] -""") +""", +) add_sample_code( - globals()["asin"], r""" + globals()["asin"], + r""" Examples: .. code-block:: python @@ -356,10 +396,12 @@ Examples: print(out) # [-0.41151685 -0.20135792 0.10016742 0.30469265] -""") +""", +) add_sample_code( - globals()["cosh"], r""" + globals()["cosh"], + r""" Examples: .. code-block:: python @@ -370,10 +412,12 @@ Examples: print(out) # [1.08107237 1.02006676 1.00500417 1.04533851] -""") +""", +) add_sample_code( - globals()["sinh"], r""" + globals()["sinh"], + r""" Examples: .. code-block:: python @@ -384,10 +428,12 @@ Examples: print(out) # [-0.41075233 -0.201336 0.10016675 0.30452029] -""") +""", +) add_sample_code( - globals()["asinh"], r""" + globals()["asinh"], + r""" Examples: .. code-block:: python @@ -398,10 +444,12 @@ Examples: print(out) # [-0.39003533, -0.19869010, 0.09983408, 0.29567307] -""") +""", +) add_sample_code( - globals()["acosh"], r""" + globals()["acosh"], + r""" Examples: .. code-block:: python @@ -412,10 +460,12 @@ Examples: print(out) # [0. , 1.76274729, 2.06343699, 2.29243159] -""") +""", +) add_sample_code( - globals()["atanh"], r""" + globals()["atanh"], + r""" Examples: .. code-block:: python @@ -426,10 +476,12 @@ Examples: print(out) # [-0.42364895, -0.20273256, 0.10033535, 0.30951962] -""") +""", +) add_sample_code( - globals()["round"], r""" + globals()["round"], + r""" Examples: .. code-block:: python @@ -440,10 +492,12 @@ Examples: print(out) # [-1. -0. 1. 2.] -""") +""", +) add_sample_code( - globals()["reciprocal"], r""" + globals()["reciprocal"], + r""" Examples: .. code-block:: python @@ -454,10 +508,12 @@ Examples: print(out) # [-2.5 -5. 10. 3.33333333] -""") +""", +) add_sample_code( - globals()["square"], r""" + globals()["square"], + r""" Examples: .. code-block:: python @@ -468,10 +524,12 @@ Examples: print(out) # [0.16 0.04 0.01 0.09] -""") +""", +) add_sample_code( - globals()["softplus"], r""" + globals()["softplus"], + r""" Examples: .. code-block:: python @@ -483,10 +541,12 @@ Examples: print(out) # [0.513015, 0.598139, 0.744397, 0.854355] -""") +""", +) add_sample_code( - globals()["softsign"], r""" + globals()["softsign"], + r""" Examples: .. code-block:: python @@ -498,7 +558,8 @@ Examples: print(out) # [-0.285714, -0.166667, 0.0909091, 0.230769] -""") +""", +) __all__ += ['erf'] diff --git a/python/paddle/tensor/random.py b/python/paddle/tensor/random.py index 208139b1abb..2449f9c3194 100644 --- a/python/paddle/tensor/random.py +++ b/python/paddle/tensor/random.py @@ -17,12 +17,21 @@ from ..framework import core from ..framework import convert_np_dtype_to_dtype_, dygraph_only from ..framework import LayerHelper -from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, check_shape +from ..fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + check_shape, +) from ..fluid.layers import utils import paddle from paddle import _C_ops, _legacy_C_ops from paddle.static import Variable -from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph, _current_expected_place +from paddle.fluid.framework import ( + in_dygraph_mode, + _in_legacy_dygraph, + _current_expected_place, +) __all__ = [] @@ -75,11 +84,11 @@ def bernoulli(x, name=None): helper = LayerHelper("randint", **locals()) out = helper.create_variable_for_type_inference( - dtype=x.dtype) # maybe set out to int32 ? - helper.append_op(type='bernoulli', - inputs={"X": x}, - outputs={'Out': out}, - attrs={}) + dtype=x.dtype + ) # maybe set out to int32 ? + helper.append_op( + type='bernoulli', inputs={"X": x}, outputs={'Out': out}, attrs={} + ) out.stop_gradient = True return out @@ -124,10 +133,9 @@ def poisson(x, name=None): helper = LayerHelper("poisson", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='poisson', - inputs={'X': x}, - outputs={'Out': out}, - attrs={}) + helper.append_op( + type='poisson', inputs={'X': x}, outputs={'Out': out}, attrs={} + ) return out @@ -179,28 +187,30 @@ def multinomial(x, num_samples=1, replacement=False, name=None): """ - assert core.is_compiled_with_rocm() == False, ( - "multinomial op is not supported on ROCM yet.") + assert ( + core.is_compiled_with_rocm() == False + ), "multinomial op is not supported on ROCM yet." if in_dygraph_mode(): return _C_ops.multinomial(x, num_samples, replacement) if _in_legacy_dygraph(): - return _legacy_C_ops.multinomial(x, 'num_samples', num_samples, - 'replacement', replacement) + return _legacy_C_ops.multinomial( + x, 'num_samples', num_samples, 'replacement', replacement + ) check_variable_and_dtype(x, "x", ["float32", "float64"], "multinomial") helper = LayerHelper("multinomial", **locals()) out = helper.create_variable_for_type_inference( - dtype=convert_np_dtype_to_dtype_('int64')) - helper.append_op(type='multinomial', - inputs={"X": x}, - outputs={'Out': out}, - attrs={ - 'num_samples': num_samples, - 'replacement': replacement - }) + dtype=convert_np_dtype_to_dtype_('int64') + ) + helper.append_op( + type='multinomial', + inputs={"X": x}, + outputs={'Out': out}, + attrs={'num_samples': num_samples, 'replacement': replacement}, + ) out.stop_gradient = True return out @@ -237,23 +247,34 @@ def gaussian(shape, mean=0.0, std=1.0, dtype=None, name=None): dtype = paddle.framework.get_default_dtype() if dtype not in ['float32', 'float64']: raise TypeError( - "{} only supports [float32, float64], but the default dtype is {}" - .format(op_type_for_check, dtype)) + "{} only supports [float32, float64], but the default dtype is {}".format( + op_type_for_check, dtype + ) + ) if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) place = _current_expected_place() - return _C_ops.gaussian_random(shape, float(mean), float(std), seed, - dtype, place) + return _C_ops.gaussian_random( + shape, float(mean), float(std), seed, dtype, place + ) if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.gaussian_random('shape', shape, - 'mean', float(mean), 'std', - float(std), 'seed', seed, 'dtype', - dtype) + return _legacy_C_ops.gaussian_random( + 'shape', + shape, + 'mean', + float(mean), + 'std', + float(std), + 'seed', + seed, + 'dtype', + dtype, + ) check_shape(shape, op_type_for_check) check_dtype(dtype, 'dtype', ['float32', 'float64'], op_type_for_check) @@ -264,19 +285,17 @@ def gaussian(shape, mean=0.0, std=1.0, dtype=None, name=None): 'std': std, 'seed': seed, 'dtype': dtype, - 'use_mkldnn': False + 'use_mkldnn': False, } - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type=op_type_for_check) + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type=op_type_for_check + ) helper = LayerHelper('gaussian', **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='gaussian_random', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='gaussian_random', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) out.stop_gradient = True return out @@ -446,13 +465,19 @@ def normal(mean=0.0, std=1.0, shape=None, name=None): check_type(std, 'std', (int, float, Variable), 'normal') if isinstance(mean, Variable): check_dtype( - mean.dtype, 'mean', ['float32', 'float64'], 'normal', - "If mean is Tensor, it's data type only support float32, float64." + mean.dtype, + 'mean', + ['float32', 'float64'], + 'normal', + "If mean is Tensor, it's data type only support float32, float64.", ) if isinstance(std, Variable): check_dtype( - std.dtype, 'std', ['float32', 'float64'], 'normal', - "If std is Tensor, it's data type only support float32, float64." + std.dtype, + 'std', + ['float32', 'float64'], + 'normal', + "If std is Tensor, it's data type only support float32, float64.", ) if shape is not None: check_shape(shape, 'normal') @@ -549,23 +574,39 @@ def uniform(shape, dtype=None, min=-1.0, max=1.0, seed=0, name=None): dtype = paddle.framework.get_default_dtype() if dtype not in ['float32', 'float64']: raise TypeError( - "uniform/rand only supports [float32, float64], but the default dtype is {}" - .format(dtype)) + "uniform/rand only supports [float32, float64], but the default dtype is {}".format( + dtype + ) + ) if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) - return _C_ops.uniform_random(shape, dtype, float(min), float(max), seed, - _current_expected_place()) + return _C_ops.uniform_random( + shape, + dtype, + float(min), + float(max), + seed, + _current_expected_place(), + ) if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.uniform_random('shape', - shape, 'min', float(min), 'max', - float(max), 'seed', seed, 'dtype', - dtype) + return _legacy_C_ops.uniform_random( + 'shape', + shape, + 'min', + float(min), + 'max', + float(max), + 'seed', + seed, + 'dtype', + dtype, + ) check_type(shape, 'shape', (list, tuple, Variable), 'uniform/rand') check_dtype(dtype, 'dtype', ('float32', 'float64'), 'uniform/rand') @@ -574,17 +615,15 @@ def uniform(shape, dtype=None, min=-1.0, max=1.0, seed=0, name=None): inputs = dict() attrs = {'seed': seed, 'min': min, 'max': max, 'dtype': dtype} - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='uniform/rand') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='uniform/rand' + ) helper = LayerHelper("uniform", **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="uniform_random", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="uniform_random", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) out.stop_gradient = True return out @@ -627,8 +666,9 @@ def uniform_(x, min=-1.0, max=1.0, seed=0, name=None): if in_dygraph_mode(): return _C_ops.uniform_random_inplace_(x, min, max, seed, 0, 0, 1.0) else: - return _legacy_C_ops.uniform_random_inplace_(x, 'min', min, 'max', max, - 'seed', seed) + return _legacy_C_ops.uniform_random_inplace_( + x, 'min', min, 'max', max, 'seed', seed + ) def randint(low=0, high=None, shape=[1], dtype=None, name=None): @@ -699,8 +739,10 @@ def randint(low=0, high=None, shape=[1], dtype=None, name=None): if high is None: if low <= 0: raise ValueError( - "If high is None, low must be greater than 0, but received low = {0}." - .format(low)) + "If high is None, low must be greater than 0, but received low = {0}.".format( + low + ) + ) high = low low = 0 if dtype is None: @@ -714,29 +756,29 @@ def randint(low=0, high=None, shape=[1], dtype=None, name=None): return _C_ops.randint(low, high, shape, dtype, place) if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.randint('shape', shape, 'low', low, 'high', high, - 'seed', 0, 'dtype', dtype) + return _legacy_C_ops.randint( + 'shape', shape, 'low', low, 'high', high, 'seed', 0, 'dtype', dtype + ) check_shape(shape, 'randint') check_dtype(dtype, 'dtype', ['int32', 'int64'], 'randint') if low >= high: raise ValueError( "randint's low must less then high, but received low = {0}, " - "high = {1}".format(low, high)) + "high = {1}".format(low, high) + ) inputs = dict() attrs = {'low': low, 'high': high, 'seed': 0, 'dtype': dtype} - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='randint') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='randint' + ) helper = LayerHelper("randint", **locals()) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='randint', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='randint', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) out.stop_gradient = True return out @@ -868,8 +910,10 @@ def randint_like(x, low=0, high=None, dtype=None, name=None): if high is None: if low <= 0: raise ValueError( - "If high is None, low must be greater than 0, but received low = {0}." - .format(low)) + "If high is None, low must be greater than 0, but received low = {0}.".format( + low + ) + ) high = low low = 0 if dtype is None: @@ -881,36 +925,49 @@ def randint_like(x, low=0, high=None, dtype=None, name=None): if low >= high: raise ValueError( "randint_like's low must less then high, but received low = {0}, " - "high = {1}".format(low, high)) + "high = {1}".format(low, high) + ) if paddle.in_dynamic_mode(): shape = utils.convert_shape_to_list(shape) - out = _legacy_C_ops.randint('shape', shape, 'low', low, 'high', high, - 'seed', 0, 'dtype', - core.VarDesc.VarType.INT64) + out = _legacy_C_ops.randint( + 'shape', + shape, + 'low', + low, + 'high', + high, + 'seed', + 0, + 'dtype', + core.VarDesc.VarType.INT64, + ) out = paddle.cast(out, dtype) return out check_shape(shape, 'randint_like') - check_dtype(dtype, 'dtype', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'randint_like') + check_dtype( + dtype, + 'dtype', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'randint_like', + ) inputs = {"ShapeTensor": shape} attrs = { 'low': low, 'high': high, 'seed': 0, - 'dtype': core.VarDesc.VarType.INT64 + 'dtype': core.VarDesc.VarType.INT64, } helper = LayerHelper("randint", **locals()) out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) - helper.append_op(type='randint', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + dtype=core.VarDesc.VarType.INT64 + ) + helper.append_op( + type='randint', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) out.stop_gradient = True out = paddle.cast(out, dtype) return out @@ -956,16 +1013,16 @@ def randperm(n, dtype="int64", name=None): if n < 1: raise ValueError("The input n should be greater than 0 in randperm op.") - check_dtype(dtype, 'dtype', ['int64', 'int32', 'float32', 'float64'], - 'randperm') + check_dtype( + dtype, 'dtype', ['int64', 'int32', 'float32', 'float64'], 'randperm' + ) helper = LayerHelper("randperm", **locals()) out = helper.create_variable_for_type_inference(dtype) attrs = {'n': n, 'dtype': dtype, 'seed': 0} - helper.append_op(type='randperm', - inputs={}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='randperm', inputs={}, outputs={'Out': out}, attrs=attrs + ) out.stop_gradient = True return out @@ -1064,8 +1121,10 @@ def exponential_(x, lam=1.0, name=None): check_variable_and_dtype(x, "x", ["float32", "float64"], "exponential") helper = LayerHelper("exponential", **locals()) - helper.append_op(type='exponential', - inputs={"X": x}, - outputs={'Out': x}, - attrs={"lambda": lam}) + helper.append_op( + type='exponential', + inputs={"X": x}, + outputs={'Out': x}, + attrs={"lambda": lam}, + ) return x diff --git a/python/paddle/tensor/search.py b/python/paddle/tensor/search.py index ffb6573c9eb..165809b39fb 100644 --- a/python/paddle/tensor/search.py +++ b/python/paddle/tensor/search.py @@ -93,28 +93,30 @@ def argsort(x, axis=-1, descending=False, name=None): return ids if _in_legacy_dygraph(): - _, ids = _legacy_C_ops.argsort(x, 'axis', axis, 'descending', - descending) + _, ids = _legacy_C_ops.argsort( + x, 'axis', axis, 'descending', descending + ) return ids check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], - 'argsort') + x, + 'x', + ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], + 'argsort', + ) helper = LayerHelper("argsort", **locals()) - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - ids = helper.create_variable_for_type_inference(VarDesc.VarType.INT64, - stop_gradient=True) - helper.append_op(type='argsort', - inputs={'X': x}, - outputs={ - 'Out': out, - 'Indices': ids - }, - attrs={ - 'axis': axis, - 'descending': descending - }) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + ids = helper.create_variable_for_type_inference( + VarDesc.VarType.INT64, stop_gradient=True + ) + helper.append_op( + type='argsort', + inputs={'X': x}, + outputs={'Out': out, 'Indices': ids}, + attrs={'axis': axis, 'descending': descending}, + ) return ids @@ -161,7 +163,8 @@ def argmax(x, axis=None, keepdim=False, dtype="int64", name=None): if axis is not None and not isinstance(axis, (int, Variable)): raise TypeError( "The type of 'axis' must be int or Tensor or None in argmax, but received %s." - % (type(axis))) + % (type(axis)) + ) if dtype is None: raise ValueError( @@ -177,14 +180,26 @@ def argmax(x, axis=None, keepdim=False, dtype="int64", name=None): if in_dygraph_mode(): return _C_ops.argmax(x, axis, keepdim, flatten, var_dtype) if _in_legacy_dygraph(): - out = _legacy_C_ops.arg_max(x, 'axis', axis, 'dtype', var_dtype, - 'keepdims', keepdim, 'flatten', flatten) + out = _legacy_C_ops.arg_max( + x, + 'axis', + axis, + 'dtype', + var_dtype, + 'keepdims', + keepdim, + 'flatten', + flatten, + ) return out helper = LayerHelper("argmax", **locals()) check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], - 'paddle.argmax') + x, + 'x', + ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], + 'paddle.argmax', + ) check_dtype(var_dtype, 'dtype', ['int32', 'int64'], 'argmin') attrs = {} out = helper.create_variable_for_type_inference(var_dtype) @@ -192,10 +207,9 @@ def argmax(x, axis=None, keepdim=False, dtype="int64", name=None): attrs['axis'] = axis attrs['flatten'] = flatten attrs['dtype'] = var_dtype - helper.append_op(type='arg_max', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='arg_max', inputs={'X': x}, outputs={'Out': [out]}, attrs=attrs + ) out.stop_gradient = True return out @@ -243,7 +257,8 @@ def argmin(x, axis=None, keepdim=False, dtype="int64", name=None): if axis is not None and not isinstance(axis, (int, Variable)): raise TypeError( "The type of 'axis' must be int or Tensor or None in argmin, but received %s." - % (type(axis))) + % (type(axis)) + ) if dtype is None: raise ValueError( @@ -259,14 +274,26 @@ def argmin(x, axis=None, keepdim=False, dtype="int64", name=None): if in_dygraph_mode(): return _C_ops.argmin(x, axis, keepdim, flatten, var_dtype) if _in_legacy_dygraph(): - out = _legacy_C_ops.arg_min(x, 'axis', axis, 'dtype', var_dtype, - 'keepdims', keepdim, 'flatten', flatten) + out = _legacy_C_ops.arg_min( + x, + 'axis', + axis, + 'dtype', + var_dtype, + 'keepdims', + keepdim, + 'flatten', + flatten, + ) return out helper = LayerHelper("argmin", **locals()) check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], - 'paddle.argmin') + x, + 'x', + ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], + 'paddle.argmin', + ) check_dtype(var_dtype, 'dtype', ['int32', 'int64'], 'argmin') out = helper.create_variable_for_type_inference(var_dtype) attrs = {} @@ -274,10 +301,9 @@ def argmin(x, axis=None, keepdim=False, dtype="int64", name=None): attrs['axis'] = axis attrs['flatten'] = flatten attrs['dtype'] = var_dtype - helper.append_op(type='arg_min', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='arg_min', inputs={'X': x}, outputs={'Out': [out]}, attrs=attrs + ) out.stop_gradient = True return out @@ -325,20 +351,24 @@ def index_select(x, index, axis=0, name=None): return _legacy_C_ops.index_select(x, index, 'dim', axis) helper = LayerHelper("index_select", **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.search.index_select') - check_variable_and_dtype(index, 'index', ['int32', 'int64'], - 'paddle.tensor.search.index_select') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.search.index_select', + ) + check_variable_and_dtype( + index, 'index', ['int32', 'int64'], 'paddle.tensor.search.index_select' + ) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='index_select', - inputs={ - 'X': x, - 'Index': index - }, - outputs={'Out': out}, - attrs={'dim': axis}) + helper.append_op( + type='index_select', + inputs={'X': x, 'Index': index}, + outputs={'Out': out}, + attrs={'dim': axis}, + ) return out @@ -406,11 +436,12 @@ def nonzero(x, as_tuple=False): helper = LayerHelper("where_index", **locals()) outs = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) + dtype=core.VarDesc.VarType.INT64 + ) - helper.append_op(type='where_index', - inputs={'Condition': x}, - outputs={'Out': [outs]}) + helper.append_op( + type='where_index', inputs={'Condition': x}, outputs={'Out': [outs]} + ) if not as_tuple: return outs @@ -419,7 +450,8 @@ def nonzero(x, as_tuple=False): else: for i in range(rank): list_out.append( - paddle.slice(outs, axes=[1], starts=[i], ends=[i + 1])) + paddle.slice(outs, axes=[1], starts=[i], ends=[i + 1]) + ) return tuple(list_out) @@ -484,24 +516,23 @@ def sort(x, axis=-1, descending=False, name=None): return outs if _in_legacy_dygraph(): - outs, _ = _legacy_C_ops.argsort(x, 'axis', axis, 'descending', - descending) + outs, _ = _legacy_C_ops.argsort( + x, 'axis', axis, 'descending', descending + ) return outs helper = LayerHelper("sort", **locals()) - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=False) - ids = helper.create_variable_for_type_inference(VarDesc.VarType.INT64, - stop_gradient=True) - helper.append_op(type='argsort', - inputs={'X': x}, - outputs={ - 'Out': out, - 'Indices': ids - }, - attrs={ - 'axis': axis, - 'descending': descending - }) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=False + ) + ids = helper.create_variable_for_type_inference( + VarDesc.VarType.INT64, stop_gradient=True + ) + helper.append_op( + type='argsort', + inputs={'X': x}, + outputs={'Out': out, 'Indices': ids}, + attrs={'axis': axis, 'descending': descending}, + ) return out @@ -550,13 +581,12 @@ def mode(x, axis=-1, keepdim=False, name=None): values = helper.create_variable_for_type_inference(dtype=x.dtype) indices = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="mode", - inputs=inputs, - outputs={ - "Out": [values], - "Indices": [indices] - }, - attrs=attrs) + helper.append_op( + type="mode", + inputs=inputs, + outputs={"Out": [values], "Indices": [indices]}, + attrs=attrs, + ) indices.stop_gradient = True return values, indices @@ -618,12 +648,12 @@ def where(condition, x=None, y=None, name=None): if not paddle.in_dynamic_mode(): check_variable_and_dtype(condition, 'condition', ['bool'], 'where') - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'int32', 'int64'], - 'where') - check_variable_and_dtype(y, 'y', - ['float32', 'float64', 'int32', 'int64'], - 'where') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'where' + ) + check_variable_and_dtype( + y, 'y', ['float32', 'float64', 'int32', 'int64'], 'where' + ) condition_shape = list(condition.shape) x_shape = list(x.shape) @@ -651,19 +681,22 @@ def where(condition, x=None, y=None, name=None): return _C_ops.where(broadcast_condition, broadcast_x, broadcast_y) else: if _in_legacy_dygraph(): - return _legacy_C_ops.where(broadcast_condition, broadcast_x, - broadcast_y) + return _legacy_C_ops.where( + broadcast_condition, broadcast_x, broadcast_y + ) else: helper = LayerHelper("where", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='where', - inputs={ - 'Condition': broadcast_condition, - 'X': broadcast_x, - 'Y': broadcast_y - }, - outputs={'Out': [out]}) + helper.append_op( + type='where', + inputs={ + 'Condition': broadcast_condition, + 'X': broadcast_x, + 'Y': broadcast_y, + }, + outputs={'Out': [out]}, + ) return out @@ -748,19 +781,25 @@ def index_sample(x, index): return _legacy_C_ops.index_sample(x, index) else: helper = LayerHelper("index_sample", **locals()) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.search.index_sample') - check_variable_and_dtype(index, 'index', ['int32', 'int64'], - 'paddle.tensor.search.index_sample') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.search.index_sample', + ) + check_variable_and_dtype( + index, + 'index', + ['int32', 'int64'], + 'paddle.tensor.search.index_sample', + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='index_sample', - inputs={ - 'X': x, - 'Index': index - }, - outputs={'Out': out}) + helper.append_op( + type='index_sample', + inputs={'X': x, 'Index': index}, + outputs={'Out': out}, + ) return out @@ -800,17 +839,19 @@ def masked_select(x, mask, name=None): return _legacy_C_ops.masked_select(x, mask) helper = LayerHelper("masked_select", **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.search.mask_select') - check_variable_and_dtype(mask, 'mask', ['bool'], - 'paddle.tensor.search.masked_select') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.search.mask_select', + ) + check_variable_and_dtype( + mask, 'mask', ['bool'], 'paddle.tensor.search.masked_select' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='masked_select', - inputs={ - 'X': x, - 'Mask': mask - }, - outputs={'Y': out}) + helper.append_op( + type='masked_select', inputs={'X': x, 'Mask': mask}, outputs={'Y': out} + ) return out @@ -870,12 +911,21 @@ def topk(x, k, axis=None, largest=True, sorted=True, name=None): if _non_static_mode(): if axis is None: - out, indices = _legacy_C_ops.top_k_v2(x, 'k', int(k), 'largest', - largest, 'sorted', sorted) + out, indices = _legacy_C_ops.top_k_v2( + x, 'k', int(k), 'largest', largest, 'sorted', sorted + ) else: - out, indices = _legacy_C_ops.top_k_v2(x, 'k', int(k), 'axis', axis, - 'largest', largest, 'sorted', - sorted) + out, indices = _legacy_C_ops.top_k_v2( + x, + 'k', + int(k), + 'axis', + axis, + 'largest', + largest, + 'sorted', + sorted, + ) return out, indices helper = LayerHelper("top_k_v2", **locals()) @@ -893,13 +943,12 @@ def topk(x, k, axis=None, largest=True, sorted=True, name=None): values = helper.create_variable_for_type_inference(dtype=x.dtype) indices = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="top_k_v2", - inputs=inputs, - outputs={ - "Out": [values], - "Indices": [indices] - }, - attrs=attrs) + helper.append_op( + type="top_k_v2", + inputs=inputs, + outputs={"Out": [values], "Indices": [indices]}, + attrs=attrs, + ) indices.stop_gradient = True return values, indices @@ -949,9 +998,12 @@ def bucketize(x, sorted_sequence, out_int32=False, right=False, name=None): # [0, 1, 3, 2]]) """ - check_variable_and_dtype(sorted_sequence, 'SortedSequence', - ['float32', 'float64', 'int32', 'int64'], - 'paddle.searchsorted') + check_variable_and_dtype( + sorted_sequence, + 'SortedSequence', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.searchsorted', + ) if sorted_sequence.dim() != 1: raise ValueError( f"sorted_sequence tensor must be 1 dimension, but got dim {sorted_sequence.dim()}" @@ -959,11 +1011,9 @@ def bucketize(x, sorted_sequence, out_int32=False, right=False, name=None): return searchsorted(sorted_sequence, x, out_int32, right, name) -def searchsorted(sorted_sequence, - values, - out_int32=False, - right=False, - name=None): +def searchsorted( + sorted_sequence, values, out_int32=False, right=False, name=None +): """ Find the index of the corresponding `sorted_sequence` in the innermost dimension based on the given `values`. @@ -1009,29 +1059,32 @@ def searchsorted(sorted_sequence, return _C_ops.searchsorted(sorted_sequence, values, out_int32, right) if _in_legacy_dygraph(): - return _legacy_C_ops.searchsorted(sorted_sequence, values, "out_int32", - out_int32, "right", right) + return _legacy_C_ops.searchsorted( + sorted_sequence, values, "out_int32", out_int32, "right", right + ) - check_variable_and_dtype(sorted_sequence, 'SortedSequence', - ['float32', 'float64', 'int32', 'int64'], - 'paddle.searchsorted') - check_variable_and_dtype(values, 'Values', - ['float32', 'float64', 'int32', 'int64'], - 'paddle.searchsorted') + check_variable_and_dtype( + sorted_sequence, + 'SortedSequence', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.searchsorted', + ) + check_variable_and_dtype( + values, + 'Values', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.searchsorted', + ) helper = LayerHelper('searchsorted', **locals()) out_type = 'int32' if out_int32 else 'int64' out = helper.create_variable_for_type_inference(dtype=out_type) - helper.append_op(type='searchsorted', - inputs={ - 'SortedSequence': sorted_sequence, - "Values": values - }, - outputs={'Out': out}, - attrs={ - "out_int32": out_int32, - "right": right - }) + helper.append_op( + type='searchsorted', + inputs={'SortedSequence': sorted_sequence, "Values": values}, + outputs={'Out': out}, + attrs={"out_int32": out_int32, "right": right}, + ) return out @@ -1077,8 +1130,9 @@ def kthvalue(x, k, axis=None, keepdim=False, name=None): if _non_static_mode(): if axis is not None: if _in_legacy_dygraph(): - return _legacy_C_ops.kthvalue(x, 'k', k, "axis", axis, - "keepdim", keepdim) + return _legacy_C_ops.kthvalue( + x, 'k', k, "axis", axis, "keepdim", keepdim + ) return _C_ops.kthvalue(x, k, axis, keepdim) else: if _in_legacy_dygraph(): @@ -1093,12 +1147,11 @@ def kthvalue(x, k, axis=None, keepdim=False, name=None): values = helper.create_variable_for_type_inference(dtype=x.dtype) indices = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="kthvalue", - inputs=inputs, - outputs={ - "Out": [values], - "Indices": [indices] - }, - attrs=attrs) + helper.append_op( + type="kthvalue", + inputs=inputs, + outputs={"Out": [values], "Indices": [indices]}, + attrs=attrs, + ) indices.stop_gradient = True return values, indices diff --git a/python/paddle/tensor/stat.py b/python/paddle/tensor/stat.py index bbe98c6ab7e..371e3fafd05 100644 --- a/python/paddle/tensor/stat.py +++ b/python/paddle/tensor/stat.py @@ -85,9 +85,11 @@ def mean(x, axis=None, keepdim=False, name=None): else: if isinstance(axis, int): axis = [axis] - reduce_all = True if axis is None \ - or len(axis)==0 \ - or len(axis) == len(x.shape) else False + reduce_all = ( + True + if axis is None or len(axis) == 0 or len(axis) == len(x.shape) + else False + ) if axis is None or len(axis) == 0: axis = [0] @@ -96,18 +98,27 @@ def mean(x, axis=None, keepdim=False, name=None): axis = list(range(len(x.shape))) return _C_ops.mean(x, axis, keepdim) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_mean(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) - - check_variable_and_dtype(x, 'x/input', - ['uint16', 'float16', 'float32', 'float64'], - 'mean/reduce_mean') - check_type(axis, 'axis/dim', (int, list, tuple, Variable), - 'mean/reduce_mean') + return _legacy_C_ops.reduce_mean( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) + + check_variable_and_dtype( + x, + 'x/input', + ['uint16', 'float16', 'float32', 'float64'], + 'mean/reduce_mean', + ) + check_type( + axis, 'axis/dim', (int, list, tuple, Variable), 'mean/reduce_mean' + ) if isinstance(axis, (list, tuple)): for item in axis: - check_type(item, 'elements of axis/dim', (int, Variable), - 'mean/reduce_mean') + check_type( + item, + 'elements of axis/dim', + (int, Variable), + 'mean/reduce_mean', + ) helper = LayerHelper('mean', **locals()) @@ -115,10 +126,9 @@ def mean(x, axis=None, keepdim=False, name=None): axis = utils._convert_to_tensor_list(axis) attrs = {'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all} out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='reduce_mean', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_mean', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -156,15 +166,16 @@ def var(x, axis=None, unbiased=True, keepdim=False, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'var') u = mean(x, axis, True, name) - out = paddle.sum((x - u)**2, axis, keepdim=keepdim, name=name) + out = paddle.sum((x - u) ** 2, axis, keepdim=keepdim, name=name) dtype = x.dtype - n = paddle.cast(paddle.numel(x), paddle.int64) \ - / paddle.cast(paddle.numel(out), paddle.int64) + n = paddle.cast(paddle.numel(x), paddle.int64) / paddle.cast( + paddle.numel(out), paddle.int64 + ) n = n.astype(dtype) if unbiased: one_const = paddle.ones([1], x.dtype) - n = where(n > one_const, n - 1., one_const) + n = where(n > one_const, n - 1.0, one_const) out /= n return out @@ -252,7 +263,8 @@ def numel(x, name=None): raise TypeError("x must be a Tensor in numel") helper = LayerHelper('numel', **locals()) out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) + dtype=core.VarDesc.VarType.INT64 + ) helper.append_op(type='size', inputs={'Input': x}, outputs={'Out': out}) return out @@ -320,8 +332,9 @@ def nanmedian(x, axis=None, keepdim=True, name=None): ) for i in range(len(axis)): - if not isinstance(axis[i], int) or not (axis[i] < dims - and axis[i] >= -dims): + if not isinstance(axis[i], int) or not ( + axis[i] < dims and axis[i] >= -dims + ): raise ValueError( "Axis should be None, int, or a list, element should in range [-rank(x), rank(x))." ) @@ -332,25 +345,25 @@ def nanmedian(x, axis=None, keepdim=True, name=None): raise ValueError("Axis has duplicated elements.") if _in_legacy_dygraph(): - median_index, out = _legacy_C_ops.nanmedian(x, 'axis', axis, 'keepdim', - keepdim) + median_index, out = _legacy_C_ops.nanmedian( + x, 'axis', axis, 'keepdim', keepdim + ) return out check_variable_and_dtype( - x, 'X', ['int32', 'int64', 'float16', 'float32', 'float64'], - 'nanmedian') + x, 'X', ['int32', 'int64', 'float16', 'float32', 'float64'], 'nanmedian' + ) helper = LayerHelper('nanmedian', **locals()) attrs = {'axis': axis, 'keepdim': keepdim} out = helper.create_variable_for_type_inference(x.dtype) medians = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='nanmedian', - inputs={'X': x}, - outputs={ - 'Out': out, - 'MedianIndex': medians - }, - attrs=attrs) + helper.append_op( + type='nanmedian', + inputs={'X': x}, + outputs={'Out': out, 'MedianIndex': medians}, + attrs=attrs, + ) return out @@ -423,21 +436,22 @@ def median(x, axis=None, keepdim=False, name=None): dtype = 'float64' if x.dtype == core.VarDesc.VarType.FP64 else 'float32' if sz & 1 == 0: out_tensor = paddle.slice( - tensor_topk, axes=[axis], starts=[kth - 1], - ends=[kth]) + paddle.slice( - tensor_topk, axes=[axis], starts=[kth], ends=[kth + 1]) + tensor_topk, axes=[axis], starts=[kth - 1], ends=[kth] + ) + paddle.slice(tensor_topk, axes=[axis], starts=[kth], ends=[kth + 1]) out_tensor = paddle.cast(out_tensor, dtype=dtype) / 2 else: - out_tensor = paddle.cast(paddle.slice(tensor_topk, - axes=[axis], - starts=[kth], - ends=[kth + 1]), - dtype=dtype) + out_tensor = paddle.cast( + paddle.slice( + tensor_topk, axes=[axis], starts=[kth], ends=[kth + 1] + ), + dtype=dtype, + ) out_tensor = out_tensor + paddle.sum( - paddle.cast(paddle.isnan(x), dtype=dtype) * x, axis=axis, keepdim=True) + paddle.cast(paddle.isnan(x), dtype=dtype) * x, axis=axis, keepdim=True + ) if not keepdim or is_flatten: if not is_flatten: - newshape = x.shape[:axis] + x.shape[axis + 1:] + newshape = x.shape[:axis] + x.shape[axis + 1 :] elif not keepdim: newshape = [1] else: @@ -501,7 +515,8 @@ def _compute_quantile(x, q, axis=None, keepdim=False, ignore_nan=False): axis_src, axis_dst = [], [] for axis_single in axis: if not isinstance(axis_single, int) or not ( - axis_single < dims and axis_single >= -dims): + axis_single < dims and axis_single >= -dims + ): raise ValueError( "Axis should be None, int, or a list, element should in range [-rank(x), rank(x))." ) @@ -523,9 +538,9 @@ def _compute_quantile(x, q, axis=None, keepdim=False, ignore_nan=False): out_shape[axis] = 1 mask = x.isnan() - valid_counts = mask.logical_not().sum(axis=axis, - keepdim=True, - dtype='float64') + valid_counts = mask.logical_not().sum( + axis=axis, keepdim=True, dtype='float64' + ) indices = [] @@ -552,15 +567,18 @@ def _compute_quantile(x, q, axis=None, keepdim=False, ignore_nan=False): for index in indices: indices_below = paddle.floor(index).astype(paddle.int32) indices_upper = paddle.ceil(index).astype(paddle.int32) - tensor_upper = paddle.take_along_axis(sorted_tensor, - indices_upper, - axis=axis) - tensor_below = paddle.take_along_axis(sorted_tensor, - indices_below, - axis=axis) - weights = (index - indices_below.astype('float64')) - out = paddle.lerp(tensor_below.astype('float64'), - tensor_upper.astype('float64'), weights) + tensor_upper = paddle.take_along_axis( + sorted_tensor, indices_upper, axis=axis + ) + tensor_below = paddle.take_along_axis( + sorted_tensor, indices_below, axis=axis + ) + weights = index - indices_below.astype('float64') + out = paddle.lerp( + tensor_below.astype('float64'), + tensor_upper.astype('float64'), + weights, + ) if not keepdim: out = paddle.squeeze(out, axis=axis) else: diff --git a/python/paddle/tensor/to_string.py b/python/paddle/tensor/to_string.py index 76f5c603f0f..4739628f717 100644 --- a/python/paddle/tensor/to_string.py +++ b/python/paddle/tensor/to_string.py @@ -30,11 +30,13 @@ class PrintOptions(object): DEFAULT_PRINT_OPTIONS = PrintOptions() -def set_printoptions(precision=None, - threshold=None, - edgeitems=None, - sci_mode=None, - linewidth=None): +def set_printoptions( + precision=None, + threshold=None, + edgeitems=None, + sci_mode=None, + linewidth=None, +): """Set the printing options for Tensor. Args: @@ -105,29 +107,35 @@ def _to_summary(var): return var elif len(var.shape) == 1: if var.shape[0] > 2 * edgeitems: - return np.concatenate([var[:edgeitems], var[(-1 * edgeitems):]]) + return np.concatenate([var[:edgeitems], var[(-1 * edgeitems) :]]) else: return var else: # recursively handle all dimensions if var.shape[0] > 2 * edgeitems: begin = [x for x in var[:edgeitems]] - end = [x for x in var[(-1 * edgeitems):]] + end = [x for x in var[(-1 * edgeitems) :]] return np.stack([_to_summary(x) for x in (begin + end)]) else: return np.stack([_to_summary(x) for x in var]) def _format_item(np_var, max_width=0, signed=False): - if np_var.dtype == np.float32 or np_var.dtype == np.float64 or np_var.dtype == np.float16: + if ( + np_var.dtype == np.float32 + or np_var.dtype == np.float64 + or np_var.dtype == np.float16 + ): if DEFAULT_PRINT_OPTIONS.sci_mode: item_str = '{{:.{}e}}'.format( - DEFAULT_PRINT_OPTIONS.precision).format(np_var) + DEFAULT_PRINT_OPTIONS.precision + ).format(np_var) elif np.ceil(np_var) == np_var: item_str = '{:.0f}.'.format(np_var) else: item_str = '{{:.{}f}}'.format( - DEFAULT_PRINT_OPTIONS.precision).format(np_var) + DEFAULT_PRINT_OPTIONS.precision + ).format(np_var) else: item_str = '{}'.format(np_var) @@ -180,42 +188,56 @@ def _format_tensor(var, summary, indent=0, max_width=0, signed=False): items_per_line = max(1, items_per_line) if summary and var.shape[0] > 2 * edgeitems: - items = [ - _format_item(item, max_width, signed) - for item in list(var)[:edgeitems] - ] + ['...'] + [ - _format_item(item, max_width, signed) - for item in list(var)[(-1 * edgeitems):] - ] + items = ( + [ + _format_item(item, max_width, signed) + for item in list(var)[:edgeitems] + ] + + ['...'] + + [ + _format_item(item, max_width, signed) + for item in list(var)[(-1 * edgeitems) :] + ] + ) else: items = [ _format_item(item, max_width, signed) for item in list(var) ] lines = [ - items[i:i + items_per_line] + items[i : i + items_per_line] for i in range(0, len(items), items_per_line) ] s = (',\n' + ' ' * (indent + 1)).join( - [', '.join(line) for line in lines]) + [', '.join(line) for line in lines] + ) return '[' + s + ']' else: # recursively handle all dimensions if summary and var.shape[0] > 2 * edgeitems: - vars = [ - _format_tensor(x, summary, indent + 1, max_width, signed) - for x in var[:edgeitems] - ] + ['...'] + [ - _format_tensor(x, summary, indent + 1, max_width, signed) - for x in var[(-1 * edgeitems):] - ] + vars = ( + [ + _format_tensor(x, summary, indent + 1, max_width, signed) + for x in var[:edgeitems] + ] + + ['...'] + + [ + _format_tensor(x, summary, indent + 1, max_width, signed) + for x in var[(-1 * edgeitems) :] + ] + ) else: vars = [ _format_tensor(x, summary, indent + 1, max_width, signed) for x in var ] - return '[' + (',' + '\n' * (len(var.shape) - 1) + ' ' * - (indent + 1)).join(vars) + ']' + return ( + '[' + + (',' + '\n' * (len(var.shape) - 1) + ' ' * (indent + 1)).join( + vars + ) + + ']' + ) def to_string(var, prefix='Tensor'): @@ -248,19 +270,19 @@ def to_string(var, prefix='Tensor'): max_width, signed = _get_max_width(_to_summary(np_var)) - data = _format_tensor(np_var, - summary, - indent=indent, - max_width=max_width, - signed=signed) + data = _format_tensor( + np_var, summary, indent=indent, max_width=max_width, signed=signed + ) - return _template.format(prefix=prefix, - shape=var.shape, - dtype=dtype, - place=var._place_str, - stop_gradient=var.stop_gradient, - indent=' ' * indent, - data=data) + return _template.format( + prefix=prefix, + shape=var.shape, + dtype=dtype, + place=var._place_str, + stop_gradient=var.stop_gradient, + indent=' ' * indent, + data=data, + ) def _format_dense_tensor(tensor, indent): @@ -282,11 +304,9 @@ def _format_dense_tensor(tensor, indent): max_width, signed = _get_max_width(_to_summary(np_tensor)) - data = _format_tensor(np_tensor, - sumary, - indent=indent, - max_width=max_width, - signed=signed) + data = _format_tensor( + np_tensor, sumary, indent=indent, max_width=max_width, signed=signed + ) return data @@ -297,38 +317,47 @@ def sparse_tensor_to_string(tensor, prefix='Tensor'): indices_tensor = tensor.indices() values_tensor = tensor.values() indices_data = 'indices=' + _format_dense_tensor( - indices_tensor, indent + len('indices=')) - values_data = 'values=' + _format_dense_tensor(values_tensor, - indent + len('values=')) - return _template.format(prefix=prefix, - shape=tensor.shape, - dtype=tensor.dtype, - place=tensor._place_str, - stop_gradient=tensor.stop_gradient, - indent=' ' * indent, - indices=indices_data, - values=values_data) + indices_tensor, indent + len('indices=') + ) + values_data = 'values=' + _format_dense_tensor( + values_tensor, indent + len('values=') + ) + return _template.format( + prefix=prefix, + shape=tensor.shape, + dtype=tensor.dtype, + place=tensor._place_str, + stop_gradient=tensor.stop_gradient, + indent=' ' * indent, + indices=indices_data, + values=values_data, + ) else: _template = "{prefix}(shape={shape}, dtype={dtype}, place={place}, stop_gradient={stop_gradient}, \n{indent}{crows}, \n{indent}{cols}, \n{indent}{values})" crows_tensor = tensor.crows() cols_tensor = tensor.cols() elements_tensor = tensor.values() - crows_data = 'crows=' + _format_dense_tensor(crows_tensor, - indent + len('crows=')) - cols_data = 'cols=' + _format_dense_tensor(cols_tensor, - indent + len('cols=')) - values_data = 'values=' + _format_dense_tensor(elements_tensor, - indent + len('values=')) - - return _template.format(prefix=prefix, - shape=tensor.shape, - dtype=tensor.dtype, - place=tensor._place_str, - stop_gradient=tensor.stop_gradient, - indent=' ' * indent, - crows=crows_data, - cols=cols_data, - values=values_data) + crows_data = 'crows=' + _format_dense_tensor( + crows_tensor, indent + len('crows=') + ) + cols_data = 'cols=' + _format_dense_tensor( + cols_tensor, indent + len('cols=') + ) + values_data = 'values=' + _format_dense_tensor( + elements_tensor, indent + len('values=') + ) + + return _template.format( + prefix=prefix, + shape=tensor.shape, + dtype=tensor.dtype, + place=tensor._place_str, + stop_gradient=tensor.stop_gradient, + indent=' ' * indent, + crows=crows_data, + cols=cols_data, + values=values_data, + ) def tensor_to_string(tensor, prefix='Tensor'): @@ -347,10 +376,12 @@ def tensor_to_string(tensor, prefix='Tensor'): return "Tensor(Not initialized)" else: data = _format_dense_tensor(tensor, indent) - return _template.format(prefix=prefix, - shape=tensor.shape, - dtype=dtype, - place=tensor._place_str, - stop_gradient=tensor.stop_gradient, - indent=' ' * indent, - data=data) + return _template.format( + prefix=prefix, + shape=tensor.shape, + dtype=dtype, + place=tensor._place_str, + stop_gradient=tensor.stop_gradient, + indent=' ' * indent, + data=data, + ) diff --git a/python/paddle/tests/dist_hapi_mnist_dynamic.py b/python/paddle/tests/dist_hapi_mnist_dynamic.py index 71c63eb62b4..50e77976da4 100644 --- a/python/paddle/tests/dist_hapi_mnist_dynamic.py +++ b/python/paddle/tests/dist_hapi_mnist_dynamic.py @@ -28,7 +28,6 @@ from paddle.vision.datasets import MNIST class MnistDataset(MNIST): - def __init__(self, mode, return_label=True): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -37,7 +36,7 @@ class MnistDataset(MNIST): img = np.reshape(self.images[idx], [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) @@ -52,10 +51,10 @@ def compute_accuracy(pred, gt): return np.sum(correct) / correct.shape[0] -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestDistTraning(unittest.TestCase): - def test_dynamic_multiple_gpus(self): device = set_device('gpu') @@ -66,9 +65,9 @@ class TestDistTraning(unittest.TestCase): labels = [Input([None, 1], 'int64', 'label')] model = Model(LeNet(), inputs, labels) - optim = fluid.optimizer.Momentum(learning_rate=0.001, - momentum=.9, - parameter_list=model.parameters()) + optim = fluid.optimizer.Momentum( + learning_rate=0.001, momentum=0.9, parameter_list=model.parameters() + ) model.prepare(optim, CrossEntropyLoss(), Accuracy()) train_dataset = MnistDataset(mode='train') @@ -76,17 +75,19 @@ class TestDistTraning(unittest.TestCase): test_dataset = MnistDataset(mode='test', return_label=False) cbk = paddle.callbacks.ProgBarLogger(50) - model.fit(train_dataset, - val_dataset, - epochs=2, - batch_size=batch_size, - callbacks=cbk) + model.fit( + train_dataset, + val_dataset, + epochs=2, + batch_size=batch_size, + callbacks=cbk, + ) eval_result = model.evaluate(val_dataset, batch_size=batch_size) - output = model.predict(test_dataset, - batch_size=batch_size, - stack_outputs=True) + output = model.predict( + test_dataset, batch_size=batch_size, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(test_dataset)) diff --git a/python/paddle/tests/dist_hapi_mnist_static.py b/python/paddle/tests/dist_hapi_mnist_static.py index cc3b6572501..ce7e5c18a6a 100644 --- a/python/paddle/tests/dist_hapi_mnist_static.py +++ b/python/paddle/tests/dist_hapi_mnist_static.py @@ -28,7 +28,6 @@ from paddle.vision.datasets import MNIST class MnistDataset(MNIST): - def __init__(self, mode, return_label=True): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -37,7 +36,7 @@ class MnistDataset(MNIST): img = np.reshape(self.images[idx], [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) @@ -52,10 +51,10 @@ def compute_accuracy(pred, gt): return np.sum(correct) / correct.shape[0] -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestDistTraning(unittest.TestCase): - def test_static_multiple_gpus(self): paddle.enable_static() device = set_device('gpu') @@ -67,9 +66,9 @@ class TestDistTraning(unittest.TestCase): labels = [Input([None, 1], 'int64', 'label')] model = Model(LeNet(), inputs, labels) - optim = fluid.optimizer.Momentum(learning_rate=0.001, - momentum=.9, - parameter_list=model.parameters()) + optim = fluid.optimizer.Momentum( + learning_rate=0.001, momentum=0.9, parameter_list=model.parameters() + ) model.prepare(optim, CrossEntropyLoss(), Accuracy()) train_dataset = MnistDataset(mode='train') @@ -77,17 +76,19 @@ class TestDistTraning(unittest.TestCase): test_dataset = MnistDataset(mode='test', return_label=False) cbk = paddle.callbacks.ProgBarLogger(50) - model.fit(train_dataset, - val_dataset, - epochs=2, - batch_size=batch_size, - callbacks=cbk) + model.fit( + train_dataset, + val_dataset, + epochs=2, + batch_size=batch_size, + callbacks=cbk, + ) eval_result = model.evaluate(val_dataset, batch_size=batch_size) - output = model.predict(test_dataset, - batch_size=batch_size, - stack_outputs=True) + output = model.predict( + test_dataset, batch_size=batch_size, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(test_dataset)) diff --git a/python/paddle/tests/dist_hapi_pure_fp16_static.py b/python/paddle/tests/dist_hapi_pure_fp16_static.py index 0d7fa491fc1..6be15ec44be 100644 --- a/python/paddle/tests/dist_hapi_pure_fp16_static.py +++ b/python/paddle/tests/dist_hapi_pure_fp16_static.py @@ -25,10 +25,10 @@ from paddle.nn.layer.loss import CrossEntropyLoss from paddle.vision.models import LeNet -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestDistTraningWithPureFP16(unittest.TestCase): - def test_amp_training_purefp16(self): if not fluid.is_compiled_with_cuda(): self.skipTest('module not tested when ONLY_CPU compling') @@ -42,13 +42,17 @@ class TestDistTraningWithPureFP16(unittest.TestCase): inputs = InputSpec([None, 1, 28, 28], "float32", 'x') labels = InputSpec([None, 1], "int64", "y") model = Model(net, inputs, labels) - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters(), - multi_precision=True) + optim = paddle.optimizer.Adam( + learning_rate=0.001, + parameters=model.parameters(), + multi_precision=True, + ) amp_configs = {"level": amp_level, "use_fp16_guard": False} - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum"), - amp_configs=amp_configs) + model.prepare( + optimizer=optim, + loss=CrossEntropyLoss(reduction="sum"), + amp_configs=amp_configs, + ) model.train_batch([data], [label]) diff --git a/python/paddle/tests/hapi_mnist_bf16_static.py b/python/paddle/tests/hapi_mnist_bf16_static.py index 55db397be49..489215301ed 100644 --- a/python/paddle/tests/hapi_mnist_bf16_static.py +++ b/python/paddle/tests/hapi_mnist_bf16_static.py @@ -37,17 +37,18 @@ set_device('cpu') def parse_args(): parser = argparse.ArgumentParser("Lenet BF16 train static script") - parser.add_argument('-bf16', - '--bf16', - type=ast.literal_eval, - default=False, - help="whether use bf16") + parser.add_argument( + '-bf16', + '--bf16', + type=ast.literal_eval, + default=False, + help="whether use bf16", + ) args = parser.parse_args() return args class MnistDataset(MNIST): - def __init__(self, mode, return_label=True): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -56,7 +57,7 @@ class MnistDataset(MNIST): img = np.reshape(self.images[idx], [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) @@ -73,8 +74,12 @@ def compute_accuracy(pred, gt): def main(args): print('download training data and load training data') - train_dataset = MnistDataset(mode='train', ) - val_dataset = MnistDataset(mode='test', ) + train_dataset = MnistDataset( + mode='train', + ) + val_dataset = MnistDataset( + mode='test', + ) test_dataset = MnistDataset(mode='test', return_label=False) im_shape = (-1, 1, 28, 28) @@ -88,10 +93,20 @@ def main(args): if args.bf16: optim = amp.bf16.decorate_bf16( optim, - amp_lists=amp.bf16.AutoMixedPrecisionListsBF16(custom_bf16_list={ - 'matmul_v2', 'pool2d', 'relu', 'scale', 'elementwise_add', - 'reshape2', 'slice', 'reduce_mean', 'conv2d' - }, )) + amp_lists=amp.bf16.AutoMixedPrecisionListsBF16( + custom_bf16_list={ + 'matmul_v2', + 'pool2d', + 'relu', + 'scale', + 'elementwise_add', + 'reshape2', + 'slice', + 'reduce_mean', + 'conv2d', + }, + ), + ) # Configuration model model.prepare(optim, paddle.nn.CrossEntropyLoss(), Accuracy()) @@ -103,9 +118,9 @@ def main(args): model.fit(train_dataset, epochs=2, batch_size=batch_size, verbose=1) eval_result = model.evaluate(val_dataset, batch_size=batch_size, verbose=1) - output = model.predict(test_dataset, - batch_size=batch_size, - stack_outputs=True) + output = model.predict( + test_dataset, batch_size=batch_size, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(test_dataset)) diff --git a/python/paddle/tests/hubconf.py b/python/paddle/tests/hubconf.py index d730dcff51e..92194898331 100644 --- a/python/paddle/tests/hubconf.py +++ b/python/paddle/tests/hubconf.py @@ -19,5 +19,5 @@ from test_hapi_hub_model import MM as _MM def MM(out_channels=8, pretrained=False): '''This is a test demo for paddle hub - ''' + ''' # fmt: skip return _MM(out_channels) diff --git a/python/paddle/tests/test_async_read_write.py b/python/paddle/tests/test_async_read_write.py index a235b786480..682f72422d6 100644 --- a/python/paddle/tests/test_async_read_write.py +++ b/python/paddle/tests/test_async_read_write.py @@ -22,60 +22,78 @@ from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph class TestAsyncRead(unittest.TestCase): - def func_setUp(self): - self.empty = paddle.to_tensor(np.array([], dtype="int64"), - place=paddle.CPUPlace()) + self.empty = paddle.to_tensor( + np.array([], dtype="int64"), place=paddle.CPUPlace() + ) data = np.random.randn(100, 50, 50).astype("float32") self.src = paddle.to_tensor(data, place=paddle.CUDAPinnedPlace()) self.dst = paddle.empty(shape=[100, 50, 50], dtype="float32") - self.index = paddle.to_tensor(np.array([1, 3, 5, 7, 9], - dtype="int64")).cpu() - self.buffer = paddle.empty(shape=[50, 50, 50], - dtype="float32").pin_memory() + self.index = paddle.to_tensor( + np.array([1, 3, 5, 7, 9], dtype="int64") + ).cpu() + self.buffer = paddle.empty( + shape=[50, 50, 50], dtype="float32" + ).pin_memory() self.stream = cuda.Stream() def func_test_async_read_empty_offset_and_count(self): with cuda.stream_guard(self.stream): if _in_legacy_dygraph(): - core.async_read(self.src, self.dst, self.index, self.buffer, - self.empty, self.empty) + core.async_read( + self.src, + self.dst, + self.index, + self.buffer, + self.empty, + self.empty, + ) else: - core.eager.async_read(self.src, self.dst, self.index, - self.buffer, self.empty, self.empty) + core.eager.async_read( + self.src, + self.dst, + self.index, + self.buffer, + self.empty, + self.empty, + ) array1 = paddle.gather(self.src, self.index) - array2 = self.dst[:len(self.index)] + array2 = self.dst[: len(self.index)] np.testing.assert_allclose(array1.numpy(), array2.numpy(), rtol=1e-05) def func_test_async_read_success(self): - offset = paddle.to_tensor(np.array([10, 20], dtype="int64"), - place=paddle.CPUPlace()) - count = paddle.to_tensor(np.array([5, 10], dtype="int64"), - place=paddle.CPUPlace()) + offset = paddle.to_tensor( + np.array([10, 20], dtype="int64"), place=paddle.CPUPlace() + ) + count = paddle.to_tensor( + np.array([5, 10], dtype="int64"), place=paddle.CPUPlace() + ) with cuda.stream_guard(self.stream): if _in_legacy_dygraph(): - core.async_read(self.src, self.dst, self.index, self.buffer, - offset, count) + core.async_read( + self.src, self.dst, self.index, self.buffer, offset, count + ) else: - core.eager.async_read(self.src, self.dst, self.index, - self.buffer, offset, count) + core.eager.async_read( + self.src, self.dst, self.index, self.buffer, offset, count + ) # index data index_array1 = paddle.gather(self.src, self.index) count_numel = paddle.sum(count).numpy()[0] - index_array2 = self.dst[count_numel:count_numel + len(self.index)] - np.testing.assert_allclose(index_array1.numpy(), - index_array2.numpy(), - rtol=1e-05) + index_array2 = self.dst[count_numel : count_numel + len(self.index)] + np.testing.assert_allclose( + index_array1.numpy(), index_array2.numpy(), rtol=1e-05 + ) # offset, count offset_a = paddle.gather(self.src, paddle.to_tensor(np.arange(10, 15))) offset_b = paddle.gather(self.src, paddle.to_tensor(np.arange(20, 30))) offset_array1 = paddle.concat([offset_a, offset_b], axis=0) offset_array2 = self.dst[:count_numel] - np.testing.assert_allclose(offset_array1.numpy(), - offset_array2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + offset_array1.numpy(), offset_array2.numpy(), rtol=1e-05 + ) def func_test_async_read_only_1dim(self): src = paddle.rand([40], dtype="float32").pin_memory() @@ -83,13 +101,15 @@ class TestAsyncRead(unittest.TestCase): buffer_ = paddle.empty([20]).pin_memory() with cuda.stream_guard(self.stream): if _in_legacy_dygraph(): - core.async_read(src, dst, self.index, buffer_, self.empty, - self.empty) + core.async_read( + src, dst, self.index, buffer_, self.empty, self.empty + ) else: - core.eager.async_read(src, dst, self.index, buffer_, self.empty, - self.empty) + core.eager.async_read( + src, dst, self.index, buffer_, self.empty, self.empty + ) array1 = paddle.gather(src, self.index) - array2 = dst[:len(self.index)] + array2 = dst[: len(self.index)] np.testing.assert_allclose(array1.numpy(), array2.numpy(), rtol=1e-05) def test_main(self): @@ -109,18 +129,20 @@ class TestAsyncRead(unittest.TestCase): class TestAsyncWrite(unittest.TestCase): - def func_setUp(self): self.src = paddle.rand(shape=[100, 50, 50, 5], dtype="float32") - self.dst = paddle.empty(shape=[200, 50, 50, 5], - dtype="float32").pin_memory() + self.dst = paddle.empty( + shape=[200, 50, 50, 5], dtype="float32" + ).pin_memory() self.stream = cuda.Stream() def func_test_async_write_success(self): - offset = paddle.to_tensor(np.array([0, 60], dtype="int64"), - place=paddle.CPUPlace()) - count = paddle.to_tensor(np.array([40, 60], dtype="int64"), - place=paddle.CPUPlace()) + offset = paddle.to_tensor( + np.array([0, 60], dtype="int64"), place=paddle.CPUPlace() + ) + count = paddle.to_tensor( + np.array([40, 60], dtype="int64"), place=paddle.CPUPlace() + ) with cuda.stream_guard(self.stream): if _in_legacy_dygraph(): core.async_write(self.src, self.dst, offset, count) @@ -130,9 +152,9 @@ class TestAsyncWrite(unittest.TestCase): offset_a = paddle.gather(self.dst, paddle.to_tensor(np.arange(0, 40))) offset_b = paddle.gather(self.dst, paddle.to_tensor(np.arange(60, 120))) offset_array = paddle.concat([offset_a, offset_b], axis=0) - np.testing.assert_allclose(self.src.numpy(), - offset_array.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + self.src.numpy(), offset_array.numpy(), rtol=1e-05 + ) def test_async_write_success(self): with _test_eager_guard(): diff --git a/python/paddle/tests/test_audio_backend.py b/python/paddle/tests/test_audio_backend.py index 79e793e2dc8..fb583d96e1f 100644 --- a/python/paddle/tests/test_audio_backend.py +++ b/python/paddle/tests/test_audio_backend.py @@ -20,12 +20,10 @@ import paddle.audio class TestAudioBackends(unittest.TestCase): - def setUp(self): self.initParmas() def initParmas(self): - def get_wav_data(dtype: str, num_channels: int, num_frames: int): dtype_ = getattr(paddle, dtype) base = paddle.linspace(-1.0, 1.0, num_frames, dtype=dtype_) * 0.1 @@ -37,19 +35,21 @@ class TestAudioBackends(unittest.TestCase): self.sr = 16000 self.dtype = "float32" self.window_size = 1024 - waveform_tensor = get_wav_data(self.dtype, - self.num_channels, - num_frames=self.duration * self.sr) + waveform_tensor = get_wav_data( + self.dtype, self.num_channels, num_frames=self.duration * self.sr + ) # shape (1, 8000) self.waveform = waveform_tensor.numpy() def test_backend(self): base_dir = os.getcwd() wave_wav_path = os.path.join(base_dir, "wave_test.wav") - paddle.audio.save(wave_wav_path, - paddle.to_tensor(self.waveform), - self.sr, - channels_first=True) + paddle.audio.save( + wave_wav_path, + paddle.to_tensor(self.waveform), + self.sr, + channels_first=True, + ) # test backends(wave)(wave_backend) info wav_info = paddle.audio.info(wave_wav_path) @@ -74,9 +74,9 @@ class TestAudioBackends(unittest.TestCase): np.testing.assert_array_almost_equal(wav_data, waveform) with open(wave_wav_path, 'rb') as file_: - wav_data, sr = paddle.audio.load(file_, - normalize=False, - num_frames=10000) + wav_data, sr = paddle.audio.load( + file_, normalize=False, num_frames=10000 + ) with soundfile.SoundFile(wave_wav_path, "r") as file_: dtype = "int16" frames = file_._prepare_read(0, None, -1) @@ -101,6 +101,7 @@ class TestAudioBackends(unittest.TestCase): try: import paddleaudio + backends = paddle.audio.backends.list_available_backends() for backend in backends: self.assertTrue(backend in ["wave_backend", "soundfile"]) @@ -119,18 +120,22 @@ class TestAudioBackends(unittest.TestCase): pass try: - paddle.audio.save(wave_wav_path, - paddle.to_tensor(self.waveform), - self.sr, - bits_per_sample=24, - channels_first=True) + paddle.audio.save( + wave_wav_path, + paddle.to_tensor(self.waveform), + self.sr, + bits_per_sample=24, + channels_first=True, + ) except ValueError: pass try: - paddle.audio.save(wave_wav_path, - paddle.to_tensor(self.waveform).unsqueeze(0), - self.sr) + paddle.audio.save( + wave_wav_path, + paddle.to_tensor(self.waveform).unsqueeze(0), + self.sr, + ) except AssertionError: pass diff --git a/python/paddle/tests/test_audio_datasets.py b/python/paddle/tests/test_audio_datasets.py index 59ba1d543bd..4db1730ae41 100644 --- a/python/paddle/tests/test_audio_datasets.py +++ b/python/paddle/tests/test_audio_datasets.py @@ -24,7 +24,6 @@ def parameterize(*params): class TestAudioDatasets(unittest.TestCase): - @parameterize(["dev", "train"], [40, 64]) def test_tess_dataset(self, mode: str, params: int): """ @@ -34,36 +33,34 @@ class TestAudioDatasets(unittest.TestCase): https://doi.org/10.5683/SP2/E8H2MF """ archive = { - 'url': - 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set_lite.zip', + 'url': 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set_lite.zip', 'md5': '9ffb5e3adf28d4d6b787fa94bd59b975', } # small part of TESS dataset for test. - tess_dataset = paddle.audio.datasets.TESS(mode=mode, - feat_type='mfcc', - n_mfcc=params, - archive=archive) + tess_dataset = paddle.audio.datasets.TESS( + mode=mode, feat_type='mfcc', n_mfcc=params, archive=archive + ) idx = np.random.randint(0, 30) elem = tess_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 6) - tess_dataset = paddle.audio.datasets.TESS(mode=mode, - feat_type='spectrogram', - n_fft=params) + tess_dataset = paddle.audio.datasets.TESS( + mode=mode, feat_type='spectrogram', n_fft=params + ) elem = tess_dataset[idx] self.assertTrue(elem[0].shape[0] == (params // 2 + 1)) self.assertTrue(0 <= elem[1] <= 6) - tess_dataset = paddle.audio.datasets.TESS(mode="dev", - feat_type='logmelspectrogram', - n_mels=params) + tess_dataset = paddle.audio.datasets.TESS( + mode="dev", feat_type='logmelspectrogram', n_mels=params + ) elem = tess_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 6) - tess_dataset = paddle.audio.datasets.TESS(mode="dev", - feat_type='melspectrogram', - n_mels=params) + tess_dataset = paddle.audio.datasets.TESS( + mode="dev", feat_type='melspectrogram', n_mels=params + ) elem = tess_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 6) @@ -77,43 +74,42 @@ class TestAudioDatasets(unittest.TestCase): http://dx.doi.org/10.1145/2733373.2806390 """ archive = { - 'url': - 'https://bj.bcebos.com/paddleaudio/datasets/ESC-50-master-lite.zip', + 'url': 'https://bj.bcebos.com/paddleaudio/datasets/ESC-50-master-lite.zip', 'md5': '1e9ba53265143df5b2804a743f2d1956', } # small part of ESC50 dataset for test. - esc50_dataset = paddle.audio.datasets.ESC50(mode=mode, - feat_type='raw', - archive=archive) + esc50_dataset = paddle.audio.datasets.ESC50( + mode=mode, feat_type='raw', archive=archive + ) idx = np.random.randint(0, 6) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == 220500) self.assertTrue(0 <= elem[1] <= 2) - esc50_dataset = paddle.audio.datasets.ESC50(mode=mode, - feat_type='mfcc', - n_mfcc=params, - archive=archive) + esc50_dataset = paddle.audio.datasets.ESC50( + mode=mode, feat_type='mfcc', n_mfcc=params, archive=archive + ) idx = np.random.randint(0, 6) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 2) - esc50_dataset = paddle.audio.datasets.ESC50(mode=mode, - feat_type='spectrogram', - n_fft=params) + esc50_dataset = paddle.audio.datasets.ESC50( + mode=mode, feat_type='spectrogram', n_fft=params + ) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == (params // 2 + 1)) self.assertTrue(0 <= elem[1] <= 2) esc50_dataset = paddle.audio.datasets.ESC50( - mode=mode, feat_type='logmelspectrogram', n_mels=params) + mode=mode, feat_type='logmelspectrogram', n_mels=params + ) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 2) - esc50_dataset = paddle.audio.datasets.ESC50(mode=mode, - feat_type='melspectrogram', - n_mels=params) + esc50_dataset = paddle.audio.datasets.ESC50( + mode=mode, feat_type='melspectrogram', n_mels=params + ) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 2) diff --git a/python/paddle/tests/test_audio_functions.py b/python/paddle/tests/test_audio_functions.py index 5766299f243..5542b4483b6 100644 --- a/python/paddle/tests/test_audio_functions.py +++ b/python/paddle/tests/test_audio_functions.py @@ -28,12 +28,10 @@ def parameterize(*params): class TestAudioFuncitons(unittest.TestCase): - def setUp(self): self.initParmas() def initParmas(self): - def get_wav_data(dtype: str, num_channels: int, num_frames: int): dtype_ = getattr(paddle, dtype) base = paddle.linspace(-1.0, 1.0, num_frames, dtype=dtype_) * 0.1 @@ -53,49 +51,56 @@ class TestAudioFuncitons(unittest.TestCase): self.sr = 16000 self.dtype = "float32" self.window_size = 1024 - waveform_tensor = get_wav_data(self.dtype, - self.num_channels, - num_frames=self.duration * self.sr) + waveform_tensor = get_wav_data( + self.dtype, self.num_channels, num_frames=self.duration * self.sr + ) self.waveform = waveform_tensor.numpy() @parameterize([1.0, 3.0, 9.0, 25.0], [True, False]) def test_audio_function(self, val: float, htk_flag: bool): mel_paddle = paddle.audio.functional.hz_to_mel(val, htk_flag) mel_paddle_tensor = paddle.audio.functional.hz_to_mel( - paddle.to_tensor(val), htk_flag) + paddle.to_tensor(val), htk_flag + ) mel_librosa = librosa.hz_to_mel(val, htk_flag) np.testing.assert_almost_equal(mel_paddle, mel_librosa, decimal=5) - np.testing.assert_almost_equal(mel_paddle_tensor.numpy(), - mel_librosa, - decimal=4) + np.testing.assert_almost_equal( + mel_paddle_tensor.numpy(), mel_librosa, decimal=4 + ) hz_paddle = paddle.audio.functional.mel_to_hz(val, htk_flag) hz_paddle_tensor = paddle.audio.functional.mel_to_hz( - paddle.to_tensor(val), htk_flag) + paddle.to_tensor(val), htk_flag + ) hz_librosa = librosa.mel_to_hz(val, htk_flag) np.testing.assert_almost_equal(hz_paddle, hz_librosa, decimal=4) - np.testing.assert_almost_equal(hz_paddle_tensor.numpy(), - hz_librosa, - decimal=4) + np.testing.assert_almost_equal( + hz_paddle_tensor.numpy(), hz_librosa, decimal=4 + ) decibel_paddle = paddle.audio.functional.power_to_db( - paddle.to_tensor(val)) + paddle.to_tensor(val) + ) decibel_librosa = librosa.power_to_db(val) - np.testing.assert_almost_equal(decibel_paddle.numpy(), - decibel_paddle, - decimal=5) - - @parameterize([64, 128, 256], [0.0, 0.5, 1.0], [10000, 11025], - [False, True]) - def test_audio_function_mel(self, n_mels: int, f_min: float, f_max: float, - htk_flag: bool): - librosa_mel_freq = librosa.mel_frequencies(n_mels, f_min, f_max, - htk_flag) + np.testing.assert_almost_equal( + decibel_paddle.numpy(), decibel_paddle, decimal=5 + ) + + @parameterize( + [64, 128, 256], [0.0, 0.5, 1.0], [10000, 11025], [False, True] + ) + def test_audio_function_mel( + self, n_mels: int, f_min: float, f_max: float, htk_flag: bool + ): + librosa_mel_freq = librosa.mel_frequencies( + n_mels, f_min, f_max, htk_flag + ) paddle_mel_freq = paddle.audio.functional.mel_frequencies( - n_mels, f_min, f_max, htk_flag, 'float64') - np.testing.assert_almost_equal(paddle_mel_freq, - librosa_mel_freq, - decimal=3) + n_mels, f_min, f_max, htk_flag, 'float64' + ) + np.testing.assert_almost_equal( + paddle_mel_freq, librosa_mel_freq, decimal=3 + ) @parameterize([8000, 16000], [64, 128, 256]) def test_audio_function_fft(self, sr: int, n_fft: int): @@ -106,60 +111,77 @@ class TestAudioFuncitons(unittest.TestCase): @parameterize([1.0, 3.0, 9.0]) def test_audio_function_exception(self, spect: float): try: - paddle.audio.functional.power_to_db(paddle.to_tensor([spect]), - amin=0) + paddle.audio.functional.power_to_db( + paddle.to_tensor([spect]), amin=0 + ) except Exception: pass try: - paddle.audio.functional.power_to_db(paddle.to_tensor([spect]), - ref_value=0) + paddle.audio.functional.power_to_db( + paddle.to_tensor([spect]), ref_value=0 + ) except Exception: pass try: - paddle.audio.functional.power_to_db(paddle.to_tensor([spect]), - top_db=-1) + paddle.audio.functional.power_to_db( + paddle.to_tensor([spect]), top_db=-1 + ) except Exception: pass - @parameterize([ - "hamming", "hann", "triang", "bohman", "blackman", "cosine", "tukey", - "taylor" - ], [1, 512]) + @parameterize( + [ + "hamming", + "hann", + "triang", + "bohman", + "blackman", + "cosine", + "tukey", + "taylor", + ], + [1, 512], + ) def test_window(self, window_type: str, n_fft: int): window_scipy = signal.get_window(window_type, n_fft) window_paddle = paddle.audio.functional.get_window(window_type, n_fft) - np.testing.assert_array_almost_equal(window_scipy, - window_paddle.numpy(), - decimal=5) + np.testing.assert_array_almost_equal( + window_scipy, window_paddle.numpy(), decimal=5 + ) @parameterize([1, 512]) def test_gaussian_window_and_exception(self, n_fft: int): window_scipy_gaussain = signal.windows.gaussian(n_fft, std=7) window_paddle_gaussian = paddle.audio.functional.get_window( - ('gaussian', 7), n_fft, False) - np.testing.assert_array_almost_equal(window_scipy_gaussain, - window_paddle_gaussian.numpy(), - decimal=5) + ('gaussian', 7), n_fft, False + ) + np.testing.assert_array_almost_equal( + window_scipy_gaussain, window_paddle_gaussian.numpy(), decimal=5 + ) window_scipy_general_gaussain = signal.windows.general_gaussian( - n_fft, 1, 7) + n_fft, 1, 7 + ) window_paddle_general_gaussian = paddle.audio.functional.get_window( - ('general_gaussian', 1, 7), n_fft, False) - np.testing.assert_array_almost_equal(window_scipy_gaussain, - window_paddle_gaussian.numpy(), - decimal=5) + ('general_gaussian', 1, 7), n_fft, False + ) + np.testing.assert_array_almost_equal( + window_scipy_gaussain, window_paddle_gaussian.numpy(), decimal=5 + ) window_scipy_exp = signal.windows.exponential(n_fft) window_paddle_exp = paddle.audio.functional.get_window( - ('exponential', None, 1), n_fft, False) - np.testing.assert_array_almost_equal(window_scipy_exp, - window_paddle_exp.numpy(), - decimal=5) + ('exponential', None, 1), n_fft, False + ) + np.testing.assert_array_almost_equal( + window_scipy_exp, window_paddle_exp.numpy(), decimal=5 + ) try: - window_paddle = paddle.audio.functional.get_window(("kaiser", 1.0), - self.n_fft) + window_paddle = paddle.audio.functional.get_window( + ("kaiser", 1.0), self.n_fft + ) except NotImplementedError: pass @@ -170,7 +192,8 @@ class TestAudioFuncitons(unittest.TestCase): try: window_paddle = paddle.audio.functional.get_window( - "fake_window", self.n_fft) + "fake_window", self.n_fft + ) except ValueError: pass @@ -181,7 +204,6 @@ class TestAudioFuncitons(unittest.TestCase): @parameterize([5, 13, 23], [257, 513, 1025]) def test_create_dct(self, n_mfcc: int, n_mels: int): - def dct(n_filters, n_input): basis = np.empty((n_filters, n_input)) basis[0, :] = 1.0 / np.sqrt(n_input) @@ -195,14 +217,17 @@ class TestAudioFuncitons(unittest.TestCase): paddle_dct = paddle.audio.functional.create_dct(n_mfcc, n_mels) np.testing.assert_array_almost_equal(librosa_dct, paddle_dct, decimal=5) - @parameterize([128, 256, 512], ["hamming", "hann", "triang", "bohman"], - [True, False]) - def test_stft_and_spect(self, n_fft: int, window_str: str, - center_flag: bool): + @parameterize( + [128, 256, 512], ["hamming", "hann", "triang", "bohman"], [True, False] + ) + def test_stft_and_spect( + self, n_fft: int, window_str: str, center_flag: bool + ): hop_length = int(n_fft / 4) if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram feature_librosa = librosa.core.stft( y=self.waveform, n_fft=n_fft, @@ -214,9 +239,9 @@ class TestAudioFuncitons(unittest.TestCase): pad_mode=self.pad_mode, ) x = paddle.to_tensor(self.waveform).unsqueeze(0) - window = paddle.audio.functional.get_window(window_str, - n_fft, - dtype=x.dtype) + window = paddle.audio.functional.get_window( + window_str, n_fft, dtype=x.dtype + ) feature_paddle = paddle.signal.stft( x=x, n_fft=n_fft, @@ -228,9 +253,9 @@ class TestAudioFuncitons(unittest.TestCase): normalized=False, onesided=True, ).squeeze(0) - np.testing.assert_array_almost_equal(feature_librosa, - feature_paddle, - decimal=5) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddle, decimal=5 + ) feature_bg = np.power(np.abs(feature_librosa), 2.0) feature_extractor = paddle.audio.features.Spectrogram( @@ -243,16 +268,18 @@ class TestAudioFuncitons(unittest.TestCase): pad_mode=self.pad_mode, ) feature_layer = feature_extractor(x).squeeze(0) - np.testing.assert_array_almost_equal(feature_layer, - feature_bg, - decimal=3) + np.testing.assert_array_almost_equal( + feature_layer, feature_bg, decimal=3 + ) - @parameterize([128, 256, 512], [64, 82], - ["hamming", "hann", "triang", "bohman"]) + @parameterize( + [128, 256, 512], [64, 82], ["hamming", "hann", "triang", "bohman"] + ) def test_istft(self, n_fft: int, hop_length: int, window_str: str): if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram # librosa # Get stft result from librosa. stft_matrix = librosa.core.stft( @@ -274,10 +301,9 @@ class TestAudioFuncitons(unittest.TestCase): length=None, ) x = paddle.to_tensor(stft_matrix).unsqueeze(0) - window = paddle.audio.functional.get_window(window_str, - n_fft, - dtype=paddle.to_tensor( - self.waveform).dtype) + window = paddle.audio.functional.get_window( + window_str, n_fft, dtype=paddle.to_tensor(self.waveform).dtype + ) feature_paddle = paddle.signal.istft( x=x, n_fft=n_fft, @@ -291,9 +317,9 @@ class TestAudioFuncitons(unittest.TestCase): return_complex=False, ).squeeze(0) - np.testing.assert_array_almost_equal(feature_librosa, - feature_paddle, - decimal=5) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddle, decimal=5 + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_audio_logmel_feature.py b/python/paddle/tests/test_audio_logmel_feature.py index 413f4053c9c..a3a39cd79ba 100644 --- a/python/paddle/tests/test_audio_logmel_feature.py +++ b/python/paddle/tests/test_audio_logmel_feature.py @@ -28,12 +28,10 @@ def parameterize(*params): class TestFeatures(unittest.TestCase): - def setUp(self): self.initParmas() def initParmas(self): - def get_wav_data(dtype: str, num_channels: int, num_frames: int): dtype_ = getattr(paddle, dtype) base = paddle.linspace(-1.0, 1.0, num_frames, dtype=dtype_) * 0.1 @@ -46,32 +44,44 @@ class TestFeatures(unittest.TestCase): self.num_channels = 1 self.sr = 16000 self.dtype = "float32" - waveform_tensor = get_wav_data(self.dtype, - self.num_channels, - num_frames=self.duration * self.sr) + waveform_tensor = get_wav_data( + self.dtype, self.num_channels, num_frames=self.duration * self.sr + ) self.waveform = waveform_tensor.numpy() - @parameterize([16000], ["hamming", "bohman"], [128], [128, 64], [64, 32], - [0.0, 50.0]) - def test_log_melspect(self, sr: int, window_str: str, n_fft: int, - hop_length: int, n_mels: int, fmin: float): + @parameterize( + [16000], ["hamming", "bohman"], [128], [128, 64], [64, 32], [0.0, 50.0] + ) + def test_log_melspect( + self, + sr: int, + window_str: str, + n_fft: int, + hop_length: int, + n_mels: int, + fmin: float, + ): if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram # librosa: - feature_librosa = librosa.feature.melspectrogram(y=self.waveform, - sr=sr, - n_fft=n_fft, - hop_length=hop_length, - window=window_str, - n_mels=n_mels, - center=True, - fmin=fmin, - pad_mode='reflect') + feature_librosa = librosa.feature.melspectrogram( + y=self.waveform, + sr=sr, + n_fft=n_fft, + hop_length=hop_length, + window=window_str, + n_mels=n_mels, + center=True, + fmin=fmin, + pad_mode='reflect', + ) feature_librosa = librosa.power_to_db(feature_librosa, top_db=None) x = paddle.to_tensor(self.waveform, dtype=paddle.float64).unsqueeze( - 0) # Add batch dim. + 0 + ) # Add batch dim. feature_extractor = paddle.audio.features.LogMelSpectrogram( sr=sr, n_fft=n_fft, @@ -81,54 +91,63 @@ class TestFeatures(unittest.TestCase): n_mels=n_mels, f_min=fmin, top_db=None, - dtype=x.dtype) + dtype=x.dtype, + ) feature_layer = feature_extractor(x).squeeze(0).numpy() - np.testing.assert_array_almost_equal(feature_librosa, - feature_layer, - decimal=2) + np.testing.assert_array_almost_equal( + feature_librosa, feature_layer, decimal=2 + ) # relative difference np.testing.assert_allclose(feature_librosa, feature_layer, rtol=1e-4) - @parameterize([16000], [256, 128], [40, 64], [64, 128], - ['float32', 'float64']) - def test_mfcc(self, sr: int, n_fft: int, n_mfcc: int, n_mels: int, - dtype: str): + @parameterize( + [16000], [256, 128], [40, 64], [64, 128], ['float32', 'float64'] + ) + def test_mfcc( + self, sr: int, n_fft: int, n_mfcc: int, n_mels: int, dtype: str + ): if paddle.version.cuda() != 'False': if float(paddle.version.cuda()) >= 11.0: return if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram # librosa: np_dtype = getattr(np, dtype) - feature_librosa = librosa.feature.mfcc(y=self.waveform, - sr=sr, - S=None, - n_mfcc=n_mfcc, - dct_type=2, - lifter=0, - n_fft=n_fft, - hop_length=64, - n_mels=n_mels, - fmin=50.0, - dtype=np_dtype) + feature_librosa = librosa.feature.mfcc( + y=self.waveform, + sr=sr, + S=None, + n_mfcc=n_mfcc, + dct_type=2, + lifter=0, + n_fft=n_fft, + hop_length=64, + n_mels=n_mels, + fmin=50.0, + dtype=np_dtype, + ) # paddlespeech.audio.features.layer - x = paddle.to_tensor(self.waveform, - dtype=dtype).unsqueeze(0) # Add batch dim. - feature_extractor = paddle.audio.features.MFCC(sr=sr, - n_mfcc=n_mfcc, - n_fft=n_fft, - hop_length=64, - n_mels=n_mels, - top_db=self.top_db, - dtype=x.dtype) + x = paddle.to_tensor(self.waveform, dtype=dtype).unsqueeze( + 0 + ) # Add batch dim. + feature_extractor = paddle.audio.features.MFCC( + sr=sr, + n_mfcc=n_mfcc, + n_fft=n_fft, + hop_length=64, + n_mels=n_mels, + top_db=self.top_db, + dtype=x.dtype, + ) feature_layer = feature_extractor(x).squeeze(0).numpy() - np.testing.assert_array_almost_equal(feature_librosa, - feature_layer, - decimal=3) + np.testing.assert_array_almost_equal( + feature_librosa, feature_layer, decimal=3 + ) np.testing.assert_allclose(feature_librosa, feature_layer, rtol=1e-1) @@ -142,19 +161,19 @@ class TestFeatures(unittest.TestCase): center=True, pad_mode='reflect', top_db=self.top_db, - dtype=x.dtype) + dtype=x.dtype, + ) feature_layer_logmel = feature_extractor(x).squeeze(0).numpy() - feature_layer_mfcc = scipy.fftpack.dct(feature_layer_logmel, - axis=0, - type=2, - norm="ortho")[:n_mfcc] - np.testing.assert_array_almost_equal(feature_layer_mfcc, - feature_librosa, - decimal=3) - np.testing.assert_allclose(feature_layer_mfcc, - feature_librosa, - rtol=1e-1) + feature_layer_mfcc = scipy.fftpack.dct( + feature_layer_logmel, axis=0, type=2, norm="ortho" + )[:n_mfcc] + np.testing.assert_array_almost_equal( + feature_layer_mfcc, feature_librosa, decimal=3 + ) + np.testing.assert_allclose( + feature_layer_mfcc, feature_librosa, rtol=1e-1 + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_audio_mel_feature.py b/python/paddle/tests/test_audio_mel_feature.py index 7ddfc9ac487..ec7b843caf7 100644 --- a/python/paddle/tests/test_audio_mel_feature.py +++ b/python/paddle/tests/test_audio_mel_feature.py @@ -27,12 +27,10 @@ def parameterize(*params): class TestFeatures(unittest.TestCase): - def setUp(self): self.initParmas() def initParmas(self): - def get_wav_data(dtype: str, num_channels: int, num_frames: int): dtype_ = getattr(paddle, dtype) base = paddle.linspace(-1.0, 1.0, num_frames, dtype=dtype_) * 0.1 @@ -44,15 +42,17 @@ class TestFeatures(unittest.TestCase): self.num_channels = 1 self.sr = 16000 self.dtype = "float32" - waveform_tensor = get_wav_data(self.dtype, - self.num_channels, - num_frames=self.duration * self.sr) + waveform_tensor = get_wav_data( + self.dtype, self.num_channels, num_frames=self.duration * self.sr + ) self.waveform = waveform_tensor.numpy() - @parameterize([8000], [128, 256], [64, 32], [0.0, 1.0], - ['float32', 'float64']) - def test_mel(self, sr: int, n_fft: int, n_mels: int, fmin: float, - dtype: str): + @parameterize( + [8000], [128, 256], [64, 32], [0.0, 1.0], ['float32', 'float64'] + ) + def test_mel( + self, sr: int, n_fft: int, n_mels: int, fmin: float, dtype: str + ): feature_librosa = librosa.filters.mel( sr=sr, n_fft=n_fft, @@ -75,40 +75,47 @@ class TestFeatures(unittest.TestCase): dtype=paddle_dtype, ) - np.testing.assert_array_almost_equal(feature_librosa, - feature_functional) + np.testing.assert_array_almost_equal( + feature_librosa, feature_functional + ) @parameterize([8000, 16000], [128, 256], [64, 82], [40, 80], [False, True]) - def test_melspect(self, sr: int, n_fft: int, hop_length: int, n_mels: int, - htk: bool): + def test_melspect( + self, sr: int, n_fft: int, hop_length: int, n_mels: int, htk: bool + ): if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram # librosa: - feature_librosa = librosa.feature.melspectrogram(y=self.waveform, - sr=sr, - n_fft=n_fft, - hop_length=hop_length, - n_mels=n_mels, - htk=htk, - fmin=50.0) + feature_librosa = librosa.feature.melspectrogram( + y=self.waveform, + sr=sr, + n_fft=n_fft, + hop_length=hop_length, + n_mels=n_mels, + htk=htk, + fmin=50.0, + ) # paddle.audio.features.layer x = paddle.to_tensor(self.waveform, dtype=paddle.float64).unsqueeze( - 0) # Add batch dim. + 0 + ) # Add batch dim. feature_extractor = paddle.audio.features.MelSpectrogram( sr=sr, n_fft=n_fft, hop_length=hop_length, n_mels=n_mels, htk=htk, - dtype=x.dtype) + dtype=x.dtype, + ) feature_layer = feature_extractor(x).squeeze(0).numpy() - np.testing.assert_array_almost_equal(feature_librosa, - feature_layer, - decimal=5) + np.testing.assert_array_almost_equal( + feature_librosa, feature_layer, decimal=5 + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_callback_early_stop.py b/python/paddle/tests/test_callback_early_stop.py index 79a62654201..612c25a6d60 100644 --- a/python/paddle/tests/test_callback_early_stop.py +++ b/python/paddle/tests/test_callback_early_stop.py @@ -27,7 +27,6 @@ from paddle.nn.layer.loss import CrossEntropyLoss class MnistDataset(MNIST): - def __init__(self, mode, return_label=True, sample_num=None): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -40,14 +39,13 @@ class MnistDataset(MNIST): img = np.reshape(img, [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) class TestCallbacks(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() @@ -64,44 +62,55 @@ class TestCallbacks(unittest.TestCase): val_dataset = MnistDataset(mode='test', sample_num=sample_num) net = LeNet() - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=net.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=net.parameters() + ) inputs = [InputSpec([None, 1, 28, 28], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(net, inputs=inputs, labels=labels) - model.prepare(optim, - loss=CrossEntropyLoss(reduction="sum"), - metrics=[Accuracy()]) - callbacks_0 = paddle.callbacks.EarlyStopping('loss', - mode='min', - patience=1, - verbose=1, - min_delta=0, - baseline=None, - save_best_model=True) - callbacks_1 = paddle.callbacks.EarlyStopping('acc', - mode='auto', - patience=1, - verbose=1, - min_delta=0, - baseline=0, - save_best_model=True) - callbacks_2 = paddle.callbacks.EarlyStopping('loss', - mode='auto_', - patience=1, - verbose=1, - min_delta=0, - baseline=None, - save_best_model=True) - callbacks_3 = paddle.callbacks.EarlyStopping('acc_', - mode='max', - patience=1, - verbose=1, - min_delta=0, - baseline=0, - save_best_model=True) + model.prepare( + optim, + loss=CrossEntropyLoss(reduction="sum"), + metrics=[Accuracy()], + ) + callbacks_0 = paddle.callbacks.EarlyStopping( + 'loss', + mode='min', + patience=1, + verbose=1, + min_delta=0, + baseline=None, + save_best_model=True, + ) + callbacks_1 = paddle.callbacks.EarlyStopping( + 'acc', + mode='auto', + patience=1, + verbose=1, + min_delta=0, + baseline=0, + save_best_model=True, + ) + callbacks_2 = paddle.callbacks.EarlyStopping( + 'loss', + mode='auto_', + patience=1, + verbose=1, + min_delta=0, + baseline=None, + save_best_model=True, + ) + callbacks_3 = paddle.callbacks.EarlyStopping( + 'acc_', + mode='max', + patience=1, + verbose=1, + min_delta=0, + baseline=0, + save_best_model=True, + ) model.fit( train_dataset, val_dataset, @@ -110,15 +119,18 @@ class TestCallbacks(unittest.TestCase): save_dir=self.save_dir, epochs=10, verbose=0, - callbacks=[callbacks_0, callbacks_1, callbacks_2, callbacks_3]) + callbacks=[callbacks_0, callbacks_1, callbacks_2, callbacks_3], + ) # Test for no val_loader - model.fit(train_dataset, - batch_size=64, - save_freq=10, - save_dir=self.save_dir, - epochs=10, - verbose=0, - callbacks=[callbacks_0]) + model.fit( + train_dataset, + batch_size=64, + save_freq=10, + save_dir=self.save_dir, + epochs=10, + verbose=0, + callbacks=[callbacks_0], + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_callback_reduce_lr_on_plateau.py b/python/paddle/tests/test_callback_reduce_lr_on_plateau.py index 02ac3245b7f..06ffb5888b8 100644 --- a/python/paddle/tests/test_callback_reduce_lr_on_plateau.py +++ b/python/paddle/tests/test_callback_reduce_lr_on_plateau.py @@ -27,34 +27,35 @@ from paddle.fluid.framework import _test_eager_guard # Accelerate unittest class CustomMnist(MNIST): - def __len__(self): return 8 class TestReduceLROnPlateau(unittest.TestCase): - def func_reduce_lr_on_plateau(self): transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) train_dataset = CustomMnist(mode='train', transform=transform) val_dataset = CustomMnist(mode='test', transform=transform) net = LeNet() - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=net.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=net.parameters() + ) inputs = [InputSpec([None, 1, 28, 28], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(net, inputs=inputs, labels=labels) model.prepare(optim, loss=CrossEntropyLoss(), metrics=[Accuracy()]) - callbacks = paddle.callbacks.ReduceLROnPlateau(patience=1, - verbose=1, - cooldown=1) - model.fit(train_dataset, - val_dataset, - batch_size=8, - log_freq=1, - save_freq=10, - epochs=10, - callbacks=[callbacks]) + callbacks = paddle.callbacks.ReduceLROnPlateau( + patience=1, verbose=1, cooldown=1 + ) + model.fit( + train_dataset, + val_dataset, + batch_size=8, + log_freq=1, + save_freq=10, + epochs=10, + callbacks=[callbacks], + ) def test_reduce_lr_on_plateau(self): with _test_eager_guard(): @@ -71,41 +72,46 @@ class TestReduceLROnPlateau(unittest.TestCase): train_dataset = CustomMnist(mode='train', transform=transform) val_dataset = CustomMnist(mode='test', transform=transform) net = LeNet() - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=net.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=net.parameters() + ) inputs = [InputSpec([None, 1, 28, 28], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(net, inputs=inputs, labels=labels) model.prepare(optim, loss=CrossEntropyLoss(), metrics=[Accuracy()]) - callbacks = paddle.callbacks.ReduceLROnPlateau(monitor='miou', - patience=3, - verbose=1) - model.fit(train_dataset, - val_dataset, - batch_size=8, - log_freq=1, - save_freq=10, - epochs=1, - callbacks=[callbacks]) + callbacks = paddle.callbacks.ReduceLROnPlateau( + monitor='miou', patience=3, verbose=1 + ) + model.fit( + train_dataset, + val_dataset, + batch_size=8, + log_freq=1, + save_freq=10, + epochs=1, + callbacks=[callbacks], + ) optim = paddle.optimizer.Adam( - learning_rate=paddle.optimizer.lr.PiecewiseDecay([0.001, 0.0001], - [5, 10]), - parameters=net.parameters()) + learning_rate=paddle.optimizer.lr.PiecewiseDecay( + [0.001, 0.0001], [5, 10] + ), + parameters=net.parameters(), + ) model.prepare(optim, loss=CrossEntropyLoss(), metrics=[Accuracy()]) - callbacks = paddle.callbacks.ReduceLROnPlateau(monitor='acc', - mode='max', - patience=3, - verbose=1, - cooldown=1) - model.fit(train_dataset, - val_dataset, - batch_size=8, - log_freq=1, - save_freq=10, - epochs=3, - callbacks=[callbacks]) + callbacks = paddle.callbacks.ReduceLROnPlateau( + monitor='acc', mode='max', patience=3, verbose=1, cooldown=1 + ) + model.fit( + train_dataset, + val_dataset, + batch_size=8, + log_freq=1, + save_freq=10, + epochs=3, + callbacks=[callbacks], + ) def test_warn_or_error(self): with _test_eager_guard(): diff --git a/python/paddle/tests/test_callback_visualdl.py b/python/paddle/tests/test_callback_visualdl.py index a1fc1cbadad..aa6616f6730 100644 --- a/python/paddle/tests/test_callback_visualdl.py +++ b/python/paddle/tests/test_callback_visualdl.py @@ -24,13 +24,11 @@ from paddle.fluid.framework import _test_eager_guard class MnistDataset(MNIST): - def __len__(self): return 512 class TestCallbacks(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() @@ -49,15 +47,16 @@ class TestCallbacks(unittest.TestCase): model = paddle.Model(net, inputs, labels) optim = paddle.optimizer.Adam(0.001, parameters=net.parameters()) - model.prepare(optimizer=optim, - loss=paddle.nn.CrossEntropyLoss(), - metrics=paddle.metric.Accuracy()) + model.prepare( + optimizer=optim, + loss=paddle.nn.CrossEntropyLoss(), + metrics=paddle.metric.Accuracy(), + ) callback = paddle.callbacks.VisualDL(log_dir='visualdl_log_dir') - model.fit(train_dataset, - eval_dataset, - batch_size=64, - callbacks=callback) + model.fit( + train_dataset, eval_dataset, batch_size=64, callbacks=callback + ) def test_visualdl_callback(self): with _test_eager_guard(): diff --git a/python/paddle/tests/test_callbacks.py b/python/paddle/tests/test_callbacks.py index cc43f82c5c2..70dda874dac 100644 --- a/python/paddle/tests/test_callbacks.py +++ b/python/paddle/tests/test_callbacks.py @@ -27,7 +27,6 @@ from paddle.vision.datasets import MNIST class MnistDataset(MNIST): - def __init__(self, mode, return_label=True, sample_num=None): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -40,14 +39,13 @@ class MnistDataset(MNIST): img = np.reshape(img, [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) class TestCallbacks(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() @@ -64,14 +62,16 @@ class TestCallbacks(unittest.TestCase): lenet = Model(LeNet(), inputs) lenet.prepare() - cbks = config_callbacks(model=lenet, - batch_size=128, - epochs=epochs, - steps=steps, - log_freq=freq, - verbose=self.verbose, - metrics=['loss', 'acc'], - save_dir=self.save_dir) + cbks = config_callbacks( + model=lenet, + batch_size=128, + epochs=epochs, + steps=steps, + log_freq=freq, + verbose=self.verbose, + metrics=['loss', 'acc'], + save_dir=self.save_dir, + ) cbks.on_begin('train') logs = {'loss': 50.341673, 'acc': 0.00256} diff --git a/python/paddle/tests/test_dataset_cifar.py b/python/paddle/tests/test_dataset_cifar.py index 95bf21c4da1..abf79fb1e39 100644 --- a/python/paddle/tests/test_dataset_cifar.py +++ b/python/paddle/tests/test_dataset_cifar.py @@ -19,7 +19,6 @@ from paddle.vision.datasets import Cifar10, Cifar100 class TestCifar10Train(unittest.TestCase): - def test_main(self): cifar = Cifar10(mode='train') self.assertTrue(len(cifar) == 50000) @@ -37,7 +36,6 @@ class TestCifar10Train(unittest.TestCase): class TestCifar10Test(unittest.TestCase): - def test_main(self): cifar = Cifar10(mode='test') self.assertTrue(len(cifar) == 10000) @@ -72,7 +70,6 @@ class TestCifar10Test(unittest.TestCase): class TestCifar100Train(unittest.TestCase): - def test_main(self): cifar = Cifar100(mode='train') self.assertTrue(len(cifar) == 50000) @@ -90,7 +87,6 @@ class TestCifar100Train(unittest.TestCase): class TestCifar100Test(unittest.TestCase): - def test_main(self): cifar = Cifar100(mode='test') self.assertTrue(len(cifar) == 10000) diff --git a/python/paddle/tests/test_dataset_conll05.py b/python/paddle/tests/test_dataset_conll05.py index 8dc1f56779f..9eb0036718b 100644 --- a/python/paddle/tests/test_dataset_conll05.py +++ b/python/paddle/tests/test_dataset_conll05.py @@ -20,7 +20,6 @@ from paddle.text.datasets import Conll05st class TestConll05st(unittest.TestCase): - def test_main(self): conll05st = Conll05st() self.assertTrue(len(conll05st) == 5267) diff --git a/python/paddle/tests/test_dataset_imdb.py b/python/paddle/tests/test_dataset_imdb.py index c70b7fedf36..aed8c387409 100644 --- a/python/paddle/tests/test_dataset_imdb.py +++ b/python/paddle/tests/test_dataset_imdb.py @@ -19,7 +19,6 @@ from paddle.text.datasets import Imdb class TestImdbTrain(unittest.TestCase): - def test_main(self): imdb = Imdb(mode='train') self.assertTrue(len(imdb) == 25000) @@ -34,7 +33,6 @@ class TestImdbTrain(unittest.TestCase): class TestImdbTest(unittest.TestCase): - def test_main(self): imdb = Imdb(mode='test') self.assertTrue(len(imdb) == 25000) diff --git a/python/paddle/tests/test_dataset_imikolov.py b/python/paddle/tests/test_dataset_imikolov.py index 6379ed11e5d..6ffeeda73c3 100644 --- a/python/paddle/tests/test_dataset_imikolov.py +++ b/python/paddle/tests/test_dataset_imikolov.py @@ -19,7 +19,6 @@ from paddle.text.datasets import Imikolov class TestImikolovTrain(unittest.TestCase): - def test_main(self): imikolov = Imikolov(mode='train', data_type='NGRAM', window_size=2) self.assertTrue(len(imikolov) == 929589) @@ -32,7 +31,6 @@ class TestImikolovTrain(unittest.TestCase): class TestImikolovTest(unittest.TestCase): - def test_main(self): imikolov = Imikolov(mode='test', data_type='NGRAM', window_size=2) self.assertTrue(len(imikolov) == 82430) diff --git a/python/paddle/tests/test_dataset_movielens.py b/python/paddle/tests/test_dataset_movielens.py index 78a62116476..e5c6d8376ee 100644 --- a/python/paddle/tests/test_dataset_movielens.py +++ b/python/paddle/tests/test_dataset_movielens.py @@ -19,7 +19,6 @@ from paddle.text.datasets import Movielens class TestMovielensTrain(unittest.TestCase): - def test_main(self): movielens = Movielens(mode='train') # movielens dataset random split train/test @@ -37,7 +36,6 @@ class TestMovielensTrain(unittest.TestCase): class TestMovielensTest(unittest.TestCase): - def test_main(self): movielens = Movielens(mode='test') # movielens dataset random split train/test diff --git a/python/paddle/tests/test_dataset_uci_housing.py b/python/paddle/tests/test_dataset_uci_housing.py index 563dd62efa3..fe85e7683dd 100644 --- a/python/paddle/tests/test_dataset_uci_housing.py +++ b/python/paddle/tests/test_dataset_uci_housing.py @@ -19,7 +19,6 @@ from paddle.text.datasets import UCIHousing, WMT14 class TestUCIHousingTrain(unittest.TestCase): - def test_main(self): uci_housing = UCIHousing(mode='train') self.assertTrue(len(uci_housing) == 404) @@ -36,7 +35,6 @@ class TestUCIHousingTrain(unittest.TestCase): class TestUCIHousingTest(unittest.TestCase): - def test_main(self): uci_housing = UCIHousing(mode='test') self.assertTrue(len(uci_housing) == 102) @@ -53,7 +51,6 @@ class TestUCIHousingTest(unittest.TestCase): class TestWMT14Train(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='train', dict_size=50) self.assertTrue(len(wmt14) == 191155) @@ -69,7 +66,6 @@ class TestWMT14Train(unittest.TestCase): class TestWMT14Test(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='test', dict_size=50) self.assertTrue(len(wmt14) == 5957) @@ -85,7 +81,6 @@ class TestWMT14Test(unittest.TestCase): class TestWMT14Gen(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='gen', dict_size=50) self.assertTrue(len(wmt14) == 3001) diff --git a/python/paddle/tests/test_dataset_voc.py b/python/paddle/tests/test_dataset_voc.py index 4d67ba9c10a..459068c2df2 100644 --- a/python/paddle/tests/test_dataset_voc.py +++ b/python/paddle/tests/test_dataset_voc.py @@ -23,7 +23,6 @@ voc2012.VOC_MD5 = '34cb1fe5bdc139a5454b25b16118fff8' class TestVOC2012Train(unittest.TestCase): - def test_main(self): voc2012 = VOC2012(mode='train') self.assertTrue(len(voc2012) == 3) @@ -40,7 +39,6 @@ class TestVOC2012Train(unittest.TestCase): class TestVOC2012Valid(unittest.TestCase): - def test_main(self): voc2012 = VOC2012(mode='valid') self.assertTrue(len(voc2012) == 1) @@ -57,7 +55,6 @@ class TestVOC2012Valid(unittest.TestCase): class TestVOC2012Test(unittest.TestCase): - def test_main(self): voc2012 = VOC2012(mode='test') self.assertTrue(len(voc2012) == 2) diff --git a/python/paddle/tests/test_dataset_wmt.py b/python/paddle/tests/test_dataset_wmt.py index 48186ab1864..43663945e20 100644 --- a/python/paddle/tests/test_dataset_wmt.py +++ b/python/paddle/tests/test_dataset_wmt.py @@ -19,7 +19,6 @@ from paddle.text.datasets import WMT14, WMT16 class TestWMT14Train(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='train', dict_size=50) self.assertTrue(len(wmt14) == 191155) @@ -35,7 +34,6 @@ class TestWMT14Train(unittest.TestCase): class TestWMT14Test(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='test', dict_size=50) self.assertTrue(len(wmt14) == 5957) @@ -51,7 +49,6 @@ class TestWMT14Test(unittest.TestCase): class TestWMT14Gen(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='gen', dict_size=50) self.assertTrue(len(wmt14) == 3001) @@ -67,12 +64,10 @@ class TestWMT14Gen(unittest.TestCase): class TestWMT16Train(unittest.TestCase): - def test_main(self): - wmt16 = WMT16(mode='train', - src_dict_size=50, - trg_dict_size=50, - lang='en') + wmt16 = WMT16( + mode='train', src_dict_size=50, trg_dict_size=50, lang='en' + ) self.assertTrue(len(wmt16) == 29000) # traversal whole dataset may cost a @@ -86,12 +81,10 @@ class TestWMT16Train(unittest.TestCase): class TestWMT16Test(unittest.TestCase): - def test_main(self): - wmt16 = WMT16(mode='test', - src_dict_size=50, - trg_dict_size=50, - lang='en') + wmt16 = WMT16( + mode='test', src_dict_size=50, trg_dict_size=50, lang='en' + ) self.assertTrue(len(wmt16) == 1000) # traversal whole dataset may cost a @@ -105,7 +98,6 @@ class TestWMT16Test(unittest.TestCase): class TestWMT16Val(unittest.TestCase): - def test_main(self): wmt16 = WMT16(mode='val', src_dict_size=50, trg_dict_size=50, lang='en') self.assertTrue(len(wmt16) == 1014) diff --git a/python/paddle/tests/test_datasets.py b/python/paddle/tests/test_datasets.py index 0f6396ec051..172d9007de5 100644 --- a/python/paddle/tests/test_datasets.py +++ b/python/paddle/tests/test_datasets.py @@ -20,13 +20,18 @@ import shutil import cv2 import paddle.vision.transforms as T -from paddle.vision.datasets import DatasetFolder, ImageFolder, MNIST, FashionMNIST, Flowers +from paddle.vision.datasets import ( + DatasetFolder, + ImageFolder, + MNIST, + FashionMNIST, + Flowers, +) from paddle.dataset.common import _check_exists_and_download from paddle.fluid.framework import _test_eager_guard class TestFolderDatasets(unittest.TestCase): - def setUp(self): self.data_dir = tempfile.mkdtemp() self.empty_dir = tempfile.mkdtemp() @@ -77,7 +82,6 @@ class TestFolderDatasets(unittest.TestCase): self.func_test_folder() def func_test_transform(self): - def fake_transform(img): return img @@ -112,7 +116,6 @@ class TestFolderDatasets(unittest.TestCase): class TestMNISTTest(unittest.TestCase): - def func_test_main(self): transform = T.Transpose() mnist = MNIST(mode='test', transform=transform) @@ -133,7 +136,6 @@ class TestMNISTTest(unittest.TestCase): class TestMNISTTrain(unittest.TestCase): - def func_test_main(self): transform = T.Transpose() mnist = MNIST(mode='train', transform=transform) @@ -170,7 +172,6 @@ class TestMNISTTrain(unittest.TestCase): class TestFASHIONMNISTTest(unittest.TestCase): - def func_test_main(self): transform = T.Transpose() mnist = FashionMNIST(mode='test', transform=transform) @@ -191,7 +192,6 @@ class TestFASHIONMNISTTest(unittest.TestCase): class TestFASHIONMNISTTrain(unittest.TestCase): - def func_test_main(self): transform = T.Transpose() mnist = FashionMNIST(mode='train', transform=transform) @@ -240,7 +240,6 @@ class TestFASHIONMNISTTrain(unittest.TestCase): class TestFlowersTrain(unittest.TestCase): - def func_test_main(self): flowers = Flowers(mode='train') self.assertTrue(len(flowers) == 6149) @@ -261,7 +260,6 @@ class TestFlowersTrain(unittest.TestCase): class TestFlowersValid(unittest.TestCase): - def func_test_main(self): flowers = Flowers(mode='valid') self.assertTrue(len(flowers) == 1020) @@ -282,7 +280,6 @@ class TestFlowersValid(unittest.TestCase): class TestFlowersTest(unittest.TestCase): - def func_test_main(self): flowers = Flowers(mode='test') self.assertTrue(len(flowers) == 1020) diff --git a/python/paddle/tests/test_dist_hapi_model.py b/python/paddle/tests/test_dist_hapi_model.py index 852572c3c7d..0ec5cc4f845 100644 --- a/python/paddle/tests/test_dist_hapi_model.py +++ b/python/paddle/tests/test_dist_hapi_model.py @@ -19,7 +19,12 @@ import copy import subprocess import paddle.fluid as fluid -from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc +from paddle.distributed.utils.launch_utils import ( + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, +) def get_cluster_from_args(selected_gpus): @@ -47,17 +52,19 @@ def get_gpus(selected_gpus): return selected_gpus -def start_local_trainers(cluster, - pod, - training_script, - eager_mode, - training_script_args, - log_dir=None): +def start_local_trainers( + cluster, + pod, + training_script, + eager_mode, + training_script_args, + log_dir=None, +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -68,7 +75,7 @@ def start_local_trainers(cluster, "PADDLE_TRAINER_ID": "%d" % t.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } if not eager_mode: @@ -101,7 +108,6 @@ def start_local_trainers(cluster, class TestMultipleGpus(unittest.TestCase): - def run_mnist_2gpu(self, target_file_name, eager_mode=True): if fluid.core.get_cuda_device_count() == 0: return @@ -112,11 +118,13 @@ class TestMultipleGpus(unittest.TestCase): cluster, pod = get_cluster_from_args(selected_gpus) - procs = start_local_trainers(cluster, - pod, - eager_mode=eager_mode, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + eager_mode=eager_mode, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_nranks()) diff --git a/python/paddle/tests/test_dlpack.py b/python/paddle/tests/test_dlpack.py index 256ca41ffff..5e4b8d2ef20 100644 --- a/python/paddle/tests/test_dlpack.py +++ b/python/paddle/tests/test_dlpack.py @@ -22,7 +22,6 @@ from paddle.fluid.framework import _test_eager_guard class TestDLPack(unittest.TestCase): - def func_test_dlpack_dygraph(self): paddle.disable_static() tensor = paddle.to_tensor(np.array([1, 2, 3, 4]).astype('int')) @@ -30,11 +29,13 @@ class TestDLPack(unittest.TestCase): out_from_dlpack = paddle.utils.dlpack.from_dlpack(dlpack) if paddle.fluid.framework.in_dygraph_mode(): self.assertTrue( - isinstance(out_from_dlpack, paddle.fluid.core.eager.Tensor)) + isinstance(out_from_dlpack, paddle.fluid.core.eager.Tensor) + ) else: self.assertTrue(isinstance(out_from_dlpack, paddle.Tensor)) - np.testing.assert_array_equal(np.array(out_from_dlpack), - np.array([1, 2, 3, 4]).astype('int')) + np.testing.assert_array_equal( + np.array(out_from_dlpack), np.array([1, 2, 3, 4]).astype('int') + ) def test_dlpack_dygraph(self): with _test_eager_guard(): @@ -58,26 +59,32 @@ class TestDLPack(unittest.TestCase): def test_dlpack_static(self): paddle.enable_static() tensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CPUPlace()) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CPUPlace(), + ) dlpack = paddle.utils.dlpack.to_dlpack(tensor) out_from_dlpack = paddle.utils.dlpack.from_dlpack(dlpack) self.assertTrue(isinstance(out_from_dlpack, fluid.core.Tensor)) np.testing.assert_array_equal( np.array(out_from_dlpack), - np.array([[1], [2], [3], [4]]).astype('int')) + np.array([[1], [2], [3], [4]]).astype('int'), + ) # when build with cuda if core.is_compiled_with_cuda(): gtensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CUDAPlace(0)) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CUDAPlace(0), + ) gdlpack = paddle.utils.dlpack.to_dlpack(gtensor) gout_from_dlpack = paddle.utils.dlpack.from_dlpack(gdlpack) self.assertTrue(isinstance(gout_from_dlpack, fluid.core.Tensor)) np.testing.assert_array_equal( np.array(gout_from_dlpack), - np.array([[1], [2], [3], [4]]).astype('int')) + np.array([[1], [2], [3], [4]]).astype('int'), + ) def func_test_dlpack_dtype_conversion(self): paddle.disable_static() @@ -104,7 +111,8 @@ class TestDLPack(unittest.TestCase): for dtype in complex_dtypes: x = paddle.to_tensor( [[1 + 6j, 2 + 5j, 3 + 4j], [4 + 3j, 5 + 2j, 6 + 1j]], - dtype=dtype) + dtype=dtype, + ) dlpack = paddle.utils.dlpack.to_dlpack(x) o = paddle.utils.dlpack.from_dlpack(dlpack) self.assertEqual(x.dtype, o.dtype) @@ -117,10 +125,10 @@ class TestDLPack(unittest.TestCase): class TestRaiseError(unittest.TestCase): - def func_test_from_dlpack_raise_type_error(self): - self.assertRaises(TypeError, paddle.utils.dlpack.from_dlpack, - np.zeros(5)) + self.assertRaises( + TypeError, paddle.utils.dlpack.from_dlpack, np.zeros(5) + ) def test_from_dlpack_raise_type_error(self): with _test_eager_guard(): diff --git a/python/paddle/tests/test_download.py b/python/paddle/tests/test_download.py index 3e6fcc54297..0978acbf56a 100644 --- a/python/paddle/tests/test_download.py +++ b/python/paddle/tests/test_download.py @@ -20,7 +20,6 @@ from paddle.utils.download import get_path_from_url class TestDownload(unittest.TestCase): - def download(self, url, md5sum): get_weights_path_from_url(url, md5sum) @@ -75,8 +74,9 @@ class TestDownload(unittest.TestCase): def test_uncompress_result(self): results = [ [ - "files/single_dir/file1", "files/single_dir/file2", - "files/single_file.pdparams" + "files/single_dir/file1", + "files/single_dir/file2", + "files/single_file.pdparams", ], ["single_dir/file1", "single_dir/file2"], ["single_file.pdparams"], @@ -89,8 +89,14 @@ class TestDownload(unittest.TestCase): for url, uncompressd_res in zip(tar_urls, results): uncompressed_path = get_path_from_url(url, root_dir='./test_tar') - self.assertTrue(all([os.path.exists(os.path.join("./test_tar", filepath)) \ - for filepath in uncompressd_res])) + self.assertTrue( + all( + [ + os.path.exists(os.path.join("./test_tar", filepath)) + for filepath in uncompressd_res + ] + ) + ) zip_urls = [ "https://paddle-hapi.bj.bcebos.com/unittest/files.zip", @@ -99,23 +105,37 @@ class TestDownload(unittest.TestCase): ] for url, uncompressd_res in zip(zip_urls, results): uncompressed_path = get_path_from_url(url, root_dir='./test_zip') - self.assertTrue(all([os.path.exists(os.path.join("./test_zip", filepath)) \ - for filepath in uncompressd_res])) + self.assertTrue( + all( + [ + os.path.exists(os.path.join("./test_zip", filepath)) + for filepath in uncompressd_res + ] + ) + ) - def test_retry_exception(self, ): + def test_retry_exception( + self, + ): with self.assertRaises(RuntimeError): from paddle.utils.download import _download + _download( 'www.baidu.com', './test', ) - def test_wget_download_error(self, ): + def test_wget_download_error( + self, + ): with self.assertRaises(RuntimeError): from paddle.utils.download import _download + _download('www.baidu', './test', method='wget') - def test_download_methods(self, ): + def test_download_methods( + self, + ): urls = [ "https://paddle-hapi.bj.bcebos.com/unittest/files.tar", "https://paddle-hapi.bj.bcebos.com/unittest/files.zip", @@ -123,6 +143,7 @@ class TestDownload(unittest.TestCase): import sys from paddle.utils.download import _download + if sys.platform == 'linux': methods = ['wget', 'get'] else: diff --git a/python/paddle/tests/test_hapi_amp.py b/python/paddle/tests/test_hapi_amp.py index 537dab34a68..e5be62a9e48 100644 --- a/python/paddle/tests/test_hapi_amp.py +++ b/python/paddle/tests/test_hapi_amp.py @@ -31,30 +31,31 @@ from paddle.vision.datasets import MNIST import paddle.vision.transforms as T -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestHapiWithAmp(unittest.TestCase): - def get_model(self, amp_config): net = LeNet() inputs = InputSpec([None, 1, 28, 28], "float32", 'x') labels = InputSpec([None, 1], "int64", "y") model = Model(net, inputs, labels) - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum"), - amp_configs=amp_config) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) + model.prepare( + optimizer=optim, + loss=CrossEntropyLoss(reduction="sum"), + amp_configs=amp_config, + ) return model def run_model(self, model): transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) train_dataset = MNIST(mode='train', transform=transform) - model.fit(train_dataset, - epochs=1, - batch_size=64, - num_iters=2, - log_freq=1) + model.fit( + train_dataset, epochs=1, batch_size=64, num_iters=2, log_freq=1 + ) def run_amp(self, amp_level): for dynamic in [True, False]: @@ -93,11 +94,9 @@ class TestHapiWithAmp(unittest.TestCase): model = self.get_model(amp_level) transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) train_dataset = MNIST(mode='train', transform=transform) - model.fit(train_dataset, - epochs=1, - batch_size=64, - num_iters=2, - log_freq=1) + model.fit( + train_dataset, epochs=1, batch_size=64, num_iters=2, log_freq=1 + ) temp_dir = tempfile.TemporaryDirectory() lenet_amp_path = os.path.join(temp_dir.name, './lenet_amp') model.save(lenet_amp_path) @@ -106,42 +105,43 @@ class TestHapiWithAmp(unittest.TestCase): paddle.seed(2021) new_model = self.get_model(amp_level) train_dataset = MNIST(mode='train', transform=transform) - new_model.fit(train_dataset, - epochs=1, - batch_size=64, - num_iters=1, - log_freq=1) + new_model.fit( + train_dataset, epochs=1, batch_size=64, num_iters=1, log_freq=1 + ) # not equal before load - self.assertNotEqual(new_model._scaler.state_dict()['incr_count'], - model._scaler.state_dict()['incr_count']) - print((new_model._scaler.state_dict()['incr_count'], - model._scaler.state_dict()['incr_count'])) + self.assertNotEqual( + new_model._scaler.state_dict()['incr_count'], + model._scaler.state_dict()['incr_count'], + ) + print( + ( + new_model._scaler.state_dict()['incr_count'], + model._scaler.state_dict()['incr_count'], + ) + ) # equal after load new_model.load(lenet_amp_path) temp_dir.cleanup() - self.assertEqual(new_model._scaler.state_dict()['incr_count'], - model._scaler.state_dict()['incr_count']) - self.assertEqual(new_model._scaler.state_dict()['decr_count'], - model._scaler.state_dict()['decr_count']) + self.assertEqual( + new_model._scaler.state_dict()['incr_count'], + model._scaler.state_dict()['incr_count'], + ) + self.assertEqual( + new_model._scaler.state_dict()['decr_count'], + model._scaler.state_dict()['decr_count'], + ) np.testing.assert_array_equal( new_model._optimizer.state_dict()['conv2d_1.w_0_moment1_0'].numpy(), - model._optimizer.state_dict()['conv2d_1.w_0_moment1_0'].numpy()) + model._optimizer.state_dict()['conv2d_1.w_0_moment1_0'].numpy(), + ) def test_dynamic_check_input(self): paddle.disable_static() amp_configs_list = [ - { - "level": "O3" - }, - { - "level": "O1", - "test": 0 - }, - { - "level": "O1", - "use_fp16_guard": True - }, + {"level": "O3"}, + {"level": "O1", "test": 0}, + {"level": "O1", "use_fp16_guard": True}, "O3", ] if not fluid.is_compiled_with_cuda(): @@ -149,21 +149,24 @@ class TestHapiWithAmp(unittest.TestCase): paddle.set_device('gpu') net = LeNet() model = Model(net) - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) loss = CrossEntropyLoss(reduction="sum") with self.assertRaises(ValueError): for amp_configs in amp_configs_list: - model.prepare(optimizer=optim, - loss=loss, - amp_configs=amp_configs) + model.prepare( + optimizer=optim, loss=loss, amp_configs=amp_configs + ) model.prepare(optimizer=optim, loss=loss, amp_configs="O2") - model.prepare(optimizer=optim, - loss=loss, - amp_configs={ - "custom_white_list": {"matmul"}, - "init_loss_scaling": 1.0 - }) + model.prepare( + optimizer=optim, + loss=loss, + amp_configs={ + "custom_white_list": {"matmul"}, + "init_loss_scaling": 1.0, + }, + ) def test_static_check_input(self): paddle.enable_static() @@ -177,8 +180,9 @@ class TestHapiWithAmp(unittest.TestCase): labels = InputSpec([None, 1], "int64", "y") model = Model(net, inputs, labels) - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) loss = CrossEntropyLoss(reduction="sum") with self.assertRaises(ValueError): model.prepare(optimizer=optim, loss=loss, amp_configs=amp_configs) diff --git a/python/paddle/tests/test_hapi_hub.py b/python/paddle/tests/test_hapi_hub.py index 526b35eddff..2a383311e1e 100644 --- a/python/paddle/tests/test_hapi_hub.py +++ b/python/paddle/tests/test_hapi_hub.py @@ -22,72 +22,90 @@ import numpy as np class TestHub(unittest.TestCase): - - def setUp(self, ): + def setUp( + self, + ): self.local_repo = os.path.dirname(os.path.abspath(__file__)) self.github_repo = 'lyuwenyu/paddlehub_demo:main' - def testLoad(self, ): - model = hub.load(self.local_repo, - model='MM', - source='local', - out_channels=8) + def testLoad( + self, + ): + model = hub.load( + self.local_repo, model='MM', source='local', out_channels=8 + ) data = paddle.rand((1, 3, 100, 100)) out = model(data) np.testing.assert_equal(out.shape, [1, 8, 50, 50]) - model = hub.load(self.github_repo, - model='MM', - source='github', - force_reload=True) - - model = hub.load(self.github_repo, - model='MM', - source='github', - force_reload=False, - pretrained=False) - - model = hub.load(self.github_repo.split(':')[0], - model='MM', - source='github', - force_reload=False, - pretrained=False) - - model = hub.load(self.github_repo, - model='MM', - source='github', - force_reload=False, - pretrained=True, - out_channels=8) + model = hub.load( + self.github_repo, model='MM', source='github', force_reload=True + ) + + model = hub.load( + self.github_repo, + model='MM', + source='github', + force_reload=False, + pretrained=False, + ) + + model = hub.load( + self.github_repo.split(':')[0], + model='MM', + source='github', + force_reload=False, + pretrained=False, + ) + + model = hub.load( + self.github_repo, + model='MM', + source='github', + force_reload=False, + pretrained=True, + out_channels=8, + ) data = paddle.ones((1, 3, 2, 2)) out = model(data) - gt = np.array([ - 1.53965068, 0., 0., 1.39455748, 0.72066200, 0.19773030, 2.09201908, - 0.37345418 - ]) + gt = np.array( + [ + 1.53965068, + 0.0, + 0.0, + 1.39455748, + 0.72066200, + 0.19773030, + 2.09201908, + 0.37345418, + ] + ) np.testing.assert_equal(out.shape, [1, 8, 1, 1]) - np.testing.assert_almost_equal(out.numpy(), - gt.reshape(1, 8, 1, 1), - decimal=5) + np.testing.assert_almost_equal( + out.numpy(), gt.reshape(1, 8, 1, 1), decimal=5 + ) - def testHelp(self, ): + def testHelp( + self, + ): docs1 = hub.help( self.local_repo, model='MM', source='local', ) - docs2 = hub.help(self.github_repo, - model='MM', - source='github', - force_reload=False) + docs2 = hub.help( + self.github_repo, model='MM', source='github', force_reload=False + ) assert docs1 == docs2 == 'This is a test demo for paddle hub\n ', '' - def testList(self, ): + def testList( + self, + ): models1 = hub.list( self.local_repo, source='local', @@ -102,35 +120,39 @@ class TestHub(unittest.TestCase): assert models1 == models2 == ['MM'], '' - def testExcept(self, ): + def testExcept( + self, + ): with self.assertRaises(ValueError): - _ = hub.help(self.github_repo, - model='MM', - source='github-test', - force_reload=False) + _ = hub.help( + self.github_repo, + model='MM', + source='github-test', + force_reload=False, + ) with self.assertRaises(ValueError): - _ = hub.load(self.github_repo, - model='MM', - source='github-test', - force_reload=False) + _ = hub.load( + self.github_repo, + model='MM', + source='github-test', + force_reload=False, + ) with self.assertRaises(ValueError): - _ = hub.list(self.github_repo, - source='github-test', - force_reload=False) + _ = hub.list( + self.github_repo, source='github-test', force_reload=False + ) with self.assertRaises(ValueError): - _ = hub.load(self.local_repo, - model=123, - source='local', - force_reload=False) + _ = hub.load( + self.local_repo, model=123, source='local', force_reload=False + ) with self.assertRaises(RuntimeError): - _ = hub.load(self.local_repo, - model='123', - source='local', - force_reload=False) + _ = hub.load( + self.local_repo, model='123', source='local', force_reload=False + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_hapi_hub_model.py b/python/paddle/tests/test_hapi_hub_model.py index 6170409fbe0..f7e2b716a1d 100644 --- a/python/paddle/tests/test_hapi_hub_model.py +++ b/python/paddle/tests/test_hapi_hub_model.py @@ -17,7 +17,6 @@ import paddle.nn.functional as F class MM(nn.Layer): - def __init__(self, out_channels): super(MM, self).__init__() self.conv = nn.Conv2D(3, out_channels, 3, 2, 1) diff --git a/python/paddle/tests/test_logger.py b/python/paddle/tests/test_logger.py index 4566a7ff693..e63b4e2cab2 100644 --- a/python/paddle/tests/test_logger.py +++ b/python/paddle/tests/test_logger.py @@ -21,7 +21,6 @@ from paddle.hapi.logger import setup_logger class TestSetupLogger(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() self.save_file = os.path.join(self.save_dir, 'logger.txt') diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index 61b20251e04..b3bfb1eb9c8 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -27,7 +27,7 @@ def one_hot(x, n_class): return res -def accuracy(pred, label, topk=(1, )): +def accuracy(pred, label, topk=(1,)): maxk = max(topk) pred = np.argsort(pred)[..., ::-1][..., :maxk] if len(label.shape) == 1: @@ -35,7 +35,7 @@ def accuracy(pred, label, topk=(1, )): elif label.shape[-1] != 1: label = np.argmax(label, axis=-1) label = label[..., np.newaxis] - correct = (pred == np.repeat(label, maxk, -1)) + correct = pred == np.repeat(label, maxk, -1) total = np.prod(np.array(label.shape[:-1])) @@ -50,16 +50,22 @@ def convert_to_one_hot(y, C): oh = np.random.choice(np.arange(C), C, replace=False).astype('float32') / C oh = np.tile(oh[np.newaxis, :], (y.shape[0], 1)) for i in range(y.shape[0]): - oh[i, int(y[i])] = 1. + oh[i, int(y[i])] = 1.0 return oh class TestAccuracy(unittest.TestCase): - def test_acc(self, squeeze_y=False): x = paddle.to_tensor( - np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.4, 0.3, 0.2], - [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) + np.array( + [ + [0.1, 0.2, 0.3, 0.4], + [0.1, 0.4, 0.3, 0.2], + [0.1, 0.2, 0.4, 0.3], + [0.1, 0.2, 0.3, 0.4], + ] + ) + ) y = np.array([[0], [1], [2], [3]]) if squeeze_y: @@ -79,8 +85,15 @@ class TestAccuracy(unittest.TestCase): self.assertEqual(m.accumulate(), 0.75) x = paddle.to_tensor( - np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.3, 0.4, 0.2], - [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) + np.array( + [ + [0.1, 0.2, 0.3, 0.4], + [0.1, 0.3, 0.4, 0.2], + [0.1, 0.2, 0.4, 0.3], + [0.1, 0.2, 0.3, 0.4], + ] + ) + ) y = paddle.to_tensor(np.array([[0], [1], [2], [3]])) correct = m.compute(x, y) # check results @@ -95,7 +108,7 @@ class TestAccuracy(unittest.TestCase): def test_1d_label(self): self.test_acc(True) - def compare(self, x_np, y_np, k=(1, )): + def compare(self, x_np, y_np, k=(1,)): x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) @@ -123,19 +136,20 @@ class TestAccuracy(unittest.TestCase): class TestAccuracyDynamic(unittest.TestCase): - def setUp(self): - self.topk = (1, ) + self.topk = (1,) self.class_num = 5 self.sample_num = 1000 self.name = None self.squeeze_label = False def random_pred_label(self): - label = np.random.randint(0, self.class_num, - (self.sample_num, 1)).astype('int64') - pred = np.random.randint(0, self.class_num, - (self.sample_num, 1)).astype('int32') + label = np.random.randint( + 0, self.class_num, (self.sample_num, 1) + ).astype('int64') + pred = np.random.randint( + 0, self.class_num, (self.sample_num, 1) + ).astype('int32') if self.squeeze_label: label = label.squeeze() pred_one_hot = convert_to_one_hot(pred, self.class_num) @@ -154,16 +168,19 @@ class TestAccuracyDynamic(unittest.TestCase): acc.update(*[s.numpy() for s in state]) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) - assert np.all(np.isclose(np.array(res_m, dtype='float64'), - np.array(res_f, dtype='float64'), rtol=1e-3)), \ - "Accuracy precision error: {} != {}".format(res_m, res_f) + assert np.all( + np.isclose( + np.array(res_m, dtype='float64'), + np.array(res_f, dtype='float64'), + rtol=1e-3, + ) + ), "Accuracy precision error: {} != {}".format(res_m, res_f) acc.reset() assert np.sum(acc.total) == 0 assert np.sum(acc.count) == 0 class TestAccuracyDynamicMultiTopk(TestAccuracyDynamic): - def setUp(self): self.topk = (1, 5) self.class_num = 10 @@ -173,9 +190,8 @@ class TestAccuracyDynamicMultiTopk(TestAccuracyDynamic): class TestAccuracyStatic(TestAccuracyDynamic): - def setUp(self): - self.topk = (1, ) + self.topk = (1,) self.class_num = 5 self.sample_num = 1000 self.name = None @@ -189,9 +205,9 @@ class TestAccuracyStatic(TestAccuracyDynamic): main_prog.random_seed = 1024 startup_prog.random_seed = 1024 with fluid.program_guard(main_prog, startup_prog): - pred = fluid.data(name='pred', - shape=[None, self.class_num], - dtype='float32') + pred = fluid.data( + name='pred', shape=[None, self.class_num], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') acc = paddle.metric.Accuracy(topk=self.topk, name=self.name) state = acc.compute(pred, label) @@ -201,18 +217,18 @@ class TestAccuracyStatic(TestAccuracyDynamic): for _ in range(10): label, pred = self.random_pred_label() - state_ret = exe.run(compiled_main_prog, - feed={ - 'pred': pred, - 'label': label - }, - fetch_list=[s.name for s in to_list(state)], - return_numpy=True) + state_ret = exe.run( + compiled_main_prog, + feed={'pred': pred, 'label': label}, + fetch_list=[s.name for s in to_list(state)], + return_numpy=True, + ) acc.update(*state_ret) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) - assert np.all(np.isclose(np.array(res_m), np.array(res_f), rtol=1e-3)), \ - "Accuracy precision error: {} != {}".format(res_m, res_f) + assert np.all( + np.isclose(np.array(res_m), np.array(res_f), rtol=1e-3) + ), "Accuracy precision error: {} != {}".format(res_m, res_f) acc.reset() assert np.sum(acc.total) == 0 assert np.sum(acc.count) == 0 @@ -221,7 +237,6 @@ class TestAccuracyStatic(TestAccuracyDynamic): class TestAccuracyStaticMultiTopk(TestAccuracyStatic): - def setUp(self): self.topk = (1, 5) self.class_num = 10 @@ -231,7 +246,6 @@ class TestAccuracyStaticMultiTopk(TestAccuracyStatic): class TestPrecision(unittest.TestCase): - def test_1d(self): x = np.array([0.1, 0.5, 0.6, 0.7]) @@ -240,13 +254,13 @@ class TestPrecision(unittest.TestCase): m = paddle.metric.Precision() m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 2. / 3.) + self.assertAlmostEqual(r, 2.0 / 3.0) x = paddle.to_tensor(np.array([0.1, 0.5, 0.6, 0.7, 0.2])) y = paddle.to_tensor(np.array([1, 0, 1, 1, 1])) m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 4. / 6.) + self.assertAlmostEqual(r, 4.0 / 6.0) def test_2d(self): x = np.array([0.1, 0.5, 0.6, 0.7]).reshape(-1, 1) @@ -255,13 +269,13 @@ class TestPrecision(unittest.TestCase): m = paddle.metric.Precision() m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 2. / 3.) + self.assertAlmostEqual(r, 2.0 / 3.0) x = np.array([0.1, 0.5, 0.6, 0.7, 0.2]).reshape(-1, 1) y = np.array([1, 0, 1, 1, 1]).reshape(-1, 1) m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 4. / 6.) + self.assertAlmostEqual(r, 4.0 / 6.0) # check reset m.reset() @@ -271,7 +285,6 @@ class TestPrecision(unittest.TestCase): class TestRecall(unittest.TestCase): - def test_1d(self): x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) @@ -279,13 +292,13 @@ class TestRecall(unittest.TestCase): m = paddle.metric.Recall() m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 2. / 3.) + self.assertAlmostEqual(r, 2.0 / 3.0) x = paddle.to_tensor(np.array([0.1, 0.5, 0.6, 0.7])) y = paddle.to_tensor(np.array([1, 0, 0, 1])) m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 3. / 5.) + self.assertAlmostEqual(r, 3.0 / 5.0) # check reset m.reset() @@ -295,10 +308,19 @@ class TestRecall(unittest.TestCase): class TestAuc(unittest.TestCase): - def test_auc_numpy(self): - x = np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], - [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]]) + x = np.array( + [ + [0.78, 0.22], + [0.62, 0.38], + [0.55, 0.45], + [0.30, 0.70], + [0.14, 0.86], + [0.59, 0.41], + [0.91, 0.08], + [0.16, 0.84], + ] + ) y = np.array([[0], [1], [1], [0], [1], [0], [0], [1]]) m = paddle.metric.Auc() m.update(x, y) @@ -310,8 +332,19 @@ class TestAuc(unittest.TestCase): def test_auc_tensor(self): x = paddle.to_tensor( - np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], - [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]])) + np.array( + [ + [0.78, 0.22], + [0.62, 0.38], + [0.55, 0.45], + [0.30, 0.70], + [0.14, 0.86], + [0.59, 0.41], + [0.91, 0.08], + [0.16, 0.84], + ] + ) + ) y = paddle.to_tensor(np.array([[0], [1], [1], [0], [1], [0], [0], [1]])) m = paddle.metric.Auc() m.update(x, y) diff --git a/python/paddle/tests/test_model.py b/python/paddle/tests/test_model.py index 5bb47b0ddfb..212939932cb 100644 --- a/python/paddle/tests/test_model.py +++ b/python/paddle/tests/test_model.py @@ -34,23 +34,28 @@ import paddle.vision.models as models import paddle.fluid.dygraph.jit as jit from paddle.io import DistributedBatchSampler, Dataset from paddle.hapi.model import prepare_distributed_context -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) class LeNetDygraph(paddle.nn.Layer): - def __init__(self, num_classes=10): super(LeNetDygraph, self).__init__() self.num_classes = num_classes - self.features = Sequential(Conv2D(1, 6, 3, stride=1, padding=1), ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2), - Conv2D(6, 16, 5, stride=1, padding=0), - ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2)) + self.features = Sequential( + Conv2D(1, 6, 3, stride=1, padding=1), + ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + Conv2D(6, 16, 5, stride=1, padding=0), + ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + ) if num_classes > 0: - self.fc = Sequential(Linear(400, 120), Linear(120, 84), - Linear(84, 10)) + self.fc = Sequential( + Linear(400, 120), Linear(120, 84), Linear(84, 10) + ) def forward(self, inputs): x = self.features(inputs) @@ -62,7 +67,6 @@ class LeNetDygraph(paddle.nn.Layer): class ModelInner(paddle.nn.Layer): - def __init__(self): super(ModelInner, self).__init__() self.fc = paddle.nn.Linear(3, 4) @@ -73,7 +77,6 @@ class ModelInner(paddle.nn.Layer): class ModelOutter(paddle.nn.Layer): - def __init__(self): super(ModelOutter, self).__init__() self.module1 = ModelInner() @@ -86,22 +89,25 @@ class ModelOutter(paddle.nn.Layer): class LeNetListInput(paddle.nn.Layer): - def __init__(self, num_classes=10): super(LeNetListInput, self).__init__() self.num_classes = num_classes self.cov = Conv2D(1, 6, 3, stride=1, padding=1) for param in self.cov.parameters(): param.trainable = False - self.features = Sequential(self.cov, ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2), - Conv2D(6, 16, 5, stride=1, padding=0), - ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2)) + self.features = Sequential( + self.cov, + ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + Conv2D(6, 16, 5, stride=1, padding=0), + ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + ) if num_classes > 0: - self.fc = Sequential(Linear(400, 120), Linear(120, 84), - Linear(84, 10)) + self.fc = Sequential( + Linear(400, 120), Linear(120, 84), Linear(84, 10) + ) def forward(self, inputs): x = inputs[0] @@ -114,7 +120,6 @@ class LeNetListInput(paddle.nn.Layer): class LeNetDictInput(LeNetDygraph): - def forward(self, inputs): x = self.features(inputs['x1']) @@ -125,7 +130,6 @@ class LeNetDictInput(LeNetDygraph): class MnistDataset(MNIST): - def __init__(self, mode, return_label=True, sample_num=None): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -138,7 +142,7 @@ class MnistDataset(MNIST): img = np.reshape(img, [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) @@ -152,8 +156,9 @@ def compute_acc(pred, label): def dynamic_train(model, dataloader): - optim = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=model.parameters()) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters() + ) model.train() for inputs, labels in dataloader: outputs = model(inputs) @@ -171,17 +176,22 @@ def dynamic_evaluate(model, dataloader): for inputs, labels in dataloader: outputs = model(inputs) - cnt += (np.argmax( - outputs.numpy(), - -1)[:, np.newaxis] == labels.numpy()).astype('int').sum() + cnt += ( + ( + np.argmax(outputs.numpy(), -1)[:, np.newaxis] + == labels.numpy() + ) + .astype('int') + .sum() + ) return cnt / len(dataloader.dataset) -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestModel(unittest.TestCase): - @classmethod def setUpClass(cls): if not fluid.is_compiled_with_cuda(): @@ -192,19 +202,19 @@ class TestModel(unittest.TestCase): sp_num = 1280 cls.train_dataset = MnistDataset(mode='train', sample_num=sp_num) cls.val_dataset = MnistDataset(mode='test', sample_num=sp_num) - cls.test_dataset = MnistDataset(mode='test', - return_label=False, - sample_num=sp_num) - - cls.train_loader = fluid.io.DataLoader(cls.train_dataset, - places=cls.device, - batch_size=64) - cls.val_loader = fluid.io.DataLoader(cls.val_dataset, - places=cls.device, - batch_size=64) - cls.test_loader = fluid.io.DataLoader(cls.test_dataset, - places=cls.device, - batch_size=64) + cls.test_dataset = MnistDataset( + mode='test', return_label=False, sample_num=sp_num + ) + + cls.train_loader = fluid.io.DataLoader( + cls.train_dataset, places=cls.device, batch_size=64 + ) + cls.val_loader = fluid.io.DataLoader( + cls.val_dataset, places=cls.device, batch_size=64 + ) + cls.test_loader = fluid.io.DataLoader( + cls.test_dataset, places=cls.device, batch_size=64 + ) seed = 333 paddle.seed(seed) @@ -277,46 +287,59 @@ class TestModel(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) net = LeNet() - optim_new = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=net.parameters()) + optim_new = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs=self.inputs, labels=self.labels) - model.prepare(optim_new, - loss=CrossEntropyLoss(reduction="sum"), - metrics=Accuracy()) + model.prepare( + optim_new, + loss=CrossEntropyLoss(reduction="sum"), + metrics=Accuracy(), + ) model.fit(self.train_dataset, batch_size=64, shuffle=False) result = model.evaluate(self.val_dataset, batch_size=64) np.testing.assert_allclose(result['acc'], self.acc1) - model.fit(self.train_dataset, - batch_size=64, - shuffle=False, - num_iters=num_iters) - - result = model.evaluate(self.val_dataset, - batch_size=64, - num_iters=num_iters) - - train_sampler = DistributedBatchSampler(self.train_dataset, - batch_size=64, - shuffle=False, - num_replicas=num_replicas, - rank=rank) - val_sampler = DistributedBatchSampler(self.val_dataset, - batch_size=64, - shuffle=False, - num_replicas=num_replicas, - rank=rank) - - train_loader = fluid.io.DataLoader(self.train_dataset, - batch_sampler=train_sampler, - places=self.device, - return_list=True) - - val_loader = fluid.io.DataLoader(self.val_dataset, - batch_sampler=val_sampler, - places=self.device, - return_list=True) + model.fit( + self.train_dataset, + batch_size=64, + shuffle=False, + num_iters=num_iters, + ) + + result = model.evaluate( + self.val_dataset, batch_size=64, num_iters=num_iters + ) + + train_sampler = DistributedBatchSampler( + self.train_dataset, + batch_size=64, + shuffle=False, + num_replicas=num_replicas, + rank=rank, + ) + val_sampler = DistributedBatchSampler( + self.val_dataset, + batch_size=64, + shuffle=False, + num_replicas=num_replicas, + rank=rank, + ) + + train_loader = fluid.io.DataLoader( + self.train_dataset, + batch_sampler=train_sampler, + places=self.device, + return_list=True, + ) + + val_loader = fluid.io.DataLoader( + self.val_dataset, + batch_sampler=val_sampler, + places=self.device, + return_list=True, + ) model.fit(train_loader, val_loader) fluid.disable_dygraph() if dynamic else None @@ -328,37 +351,48 @@ class TestModel(unittest.TestCase): paddle.framework.random._manual_program_seed(seed) net = LeNet() - optim_new = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=net.parameters()) + optim_new = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs=tuple(self.inputs), labels=tuple(self.labels)) - model.prepare(optim_new, - loss=CrossEntropyLoss(reduction="sum"), - metrics=Accuracy()) + model.prepare( + optim_new, + loss=CrossEntropyLoss(reduction="sum"), + metrics=Accuracy(), + ) model.fit(self.train_dataset, batch_size=64, shuffle=False) result = model.evaluate(self.val_dataset, batch_size=64) np.testing.assert_allclose(result['acc'], self.acc1) - train_sampler = DistributedBatchSampler(self.train_dataset, - batch_size=64, - shuffle=False, - num_replicas=num_replicas, - rank=rank) - val_sampler = DistributedBatchSampler(self.val_dataset, - batch_size=64, - shuffle=False, - num_replicas=num_replicas, - rank=rank) - - train_loader = fluid.io.DataLoader(self.train_dataset, - batch_sampler=train_sampler, - places=self.device, - return_list=True) - - val_loader = fluid.io.DataLoader(self.val_dataset, - batch_sampler=val_sampler, - places=self.device, - return_list=True) + train_sampler = DistributedBatchSampler( + self.train_dataset, + batch_size=64, + shuffle=False, + num_replicas=num_replicas, + rank=rank, + ) + val_sampler = DistributedBatchSampler( + self.val_dataset, + batch_size=64, + shuffle=False, + num_replicas=num_replicas, + rank=rank, + ) + + train_loader = fluid.io.DataLoader( + self.train_dataset, + batch_sampler=train_sampler, + places=self.device, + return_list=True, + ) + + val_loader = fluid.io.DataLoader( + self.val_dataset, + batch_sampler=val_sampler, + places=self.device, + return_list=True, + ) model.fit(train_loader, val_loader) fluid.disable_dygraph() if dynamic else None @@ -371,14 +405,16 @@ class TestModel(unittest.TestCase): result = model.evaluate(self.val_dataset, batch_size=64) np.testing.assert_allclose(result['acc'], self.acc1) - sampler = DistributedBatchSampler(self.val_dataset, - batch_size=64, - shuffle=False) + sampler = DistributedBatchSampler( + self.val_dataset, batch_size=64, shuffle=False + ) - val_loader = fluid.io.DataLoader(self.val_dataset, - batch_sampler=sampler, - places=self.device, - return_list=True) + val_loader = fluid.io.DataLoader( + self.val_dataset, + batch_sampler=sampler, + places=self.device, + return_list=True, + ) model.evaluate(val_loader) @@ -389,22 +425,24 @@ class TestModel(unittest.TestCase): model = Model(LeNet(), self.inputs) model.prepare() model.load(self.weight_path) - output = model.predict(self.test_dataset, - batch_size=64, - stack_outputs=True) + output = model.predict( + self.test_dataset, batch_size=64, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(self.test_dataset)) acc = compute_acc(output[0], self.val_dataset.labels) np.testing.assert_allclose(acc, self.acc1) - sampler = DistributedBatchSampler(self.test_dataset, - batch_size=64, - shuffle=False) + sampler = DistributedBatchSampler( + self.test_dataset, batch_size=64, shuffle=False + ) - test_loader = fluid.io.DataLoader(self.test_dataset, - batch_sampler=sampler, - places=self.device, - return_list=True) + test_loader = fluid.io.DataLoader( + self.test_dataset, + batch_sampler=sampler, + places=self.device, + return_list=True, + ) model.evaluate(test_loader) @@ -416,21 +454,21 @@ class TestModel(unittest.TestCase): model.prepare() model.load(self.weight_path) model._inputs = None - output = model.predict(self.test_dataset, - batch_size=64, - stack_outputs=True) + output = model.predict( + self.test_dataset, batch_size=64, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(self.test_dataset)) fluid.disable_dygraph() def test_summary_gpu(self): paddle.disable_static(self.device) rnn = paddle.nn.LSTM(16, 32, 2) - params_info = paddle.summary(rnn, [(-1, 23, 16), - ((2, None, 32), (2, -1, 32))]) + params_info = paddle.summary( + rnn, [(-1, 23, 16), ((2, None, 32), (2, -1, 32))] + ) class MyModel(paddle.nn.Layer): - def __init__(self): super(MyModel, self).__init__() self._fc = Linear(20, 10) @@ -441,17 +479,16 @@ class MyModel(paddle.nn.Layer): class MyDataset(Dataset): - def __getitem__(self, idx): - return np.random.random(size=(20,)).astype(np.float32), \ - np.random.randint(0, 10, size=(1,)).astype(np.int64) + return np.random.random(size=(20,)).astype( + np.float32 + ), np.random.randint(0, 10, size=(1,)).astype(np.int64) def __len__(self): return 40 class TestModelFunction(unittest.TestCase): - def set_seed(self, seed=1024): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) @@ -465,8 +502,9 @@ class TestModelFunction(unittest.TestCase): fluid.enable_dygraph(fluid.CPUPlace()) self.set_seed() m = MyModel() - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=m.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=m.parameters() + ) m.train() output = m(to_tensor(data)) loss = CrossEntropyLoss(reduction='sum')(output, to_tensor(label)) @@ -484,14 +522,15 @@ class TestModelFunction(unittest.TestCase): self.set_seed() net = MyModel() - optim2 = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim2 = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) inputs = [InputSpec([None, dim], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(net, inputs, labels) model.prepare(optim2, loss=CrossEntropyLoss(reduction="sum")) - loss, = model.train_batch([data], [label]) + (loss,) = model.train_batch([data], [label]) np.testing.assert_allclose(loss.flatten(), ref.flatten()) fluid.disable_dygraph() if dynamic else None @@ -517,7 +556,7 @@ class TestModelFunction(unittest.TestCase): inputs = [InputSpec([None, dim], 'float32', 'x')] model = Model(net, inputs) model.prepare() - out, = model.predict_batch([data]) + (out,) = model.predict_batch([data]) np.testing.assert_allclose(out, ref, rtol=1e-6) fluid.disable_dygraph() if dynamic else None @@ -532,11 +571,13 @@ class TestModelFunction(unittest.TestCase): net = MyModel() inputs = [InputSpec([None, 20], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs, labels) - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum")) + model.prepare( + optimizer=optim, loss=CrossEntropyLoss(reduction="sum") + ) model.save(path) model.load(path) fluid.disable_dygraph() if dynamic else None @@ -555,14 +596,17 @@ class TestModelFunction(unittest.TestCase): inputs = [InputSpec([None, 1, 28, 28], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] if new_optimizer: - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=net.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=net.parameters() + ) else: - optim = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs, labels) - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum")) + model.prepare( + optimizer=optim, loss=CrossEntropyLoss(reduction="sum") + ) model.fit(mnist_data, batch_size=64, verbose=0) model.save(path) model.load(path) @@ -570,16 +614,18 @@ class TestModelFunction(unittest.TestCase): shutil.rmtree(path) def test_dynamic_save_static_load(self): - path = os.path.join(tempfile.mkdtemp(), - '.cache_dynamic_save_static_load') + path = os.path.join( + tempfile.mkdtemp(), '.cache_dynamic_save_static_load' + ) if not os.path.exists(path): os.makedirs(path) # dynamic saving device = paddle.set_device('cpu') fluid.enable_dygraph(device) model = Model(MyModel()) - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=model.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=model.parameters() + ) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(path) fluid.disable_dygraph() @@ -587,34 +633,38 @@ class TestModelFunction(unittest.TestCase): inputs = [InputSpec([None, 20], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(MyModel(), inputs, labels) - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=model.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=model.parameters() + ) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.load(path) shutil.rmtree(path) def test_static_save_dynamic_load(self): - path = os.path.join(tempfile.mkdtemp(), - '.cache_test_static_save_dynamic_load') + path = os.path.join( + tempfile.mkdtemp(), '.cache_test_static_save_dynamic_load' + ) if not os.path.exists(path): os.makedirs(path) net = MyModel() inputs = [InputSpec([None, 20], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs, labels) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(path) device = paddle.set_device('cpu') - fluid.enable_dygraph(device) #if dynamic else None + fluid.enable_dygraph(device) # if dynamic else None net = MyModel() inputs = [InputSpec([None, 20], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs, labels) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.load(path) @@ -635,7 +685,6 @@ class TestModelFunction(unittest.TestCase): fluid.disable_dygraph() if dynamic else None def test_summary(self): - def _get_param_from_state_dict(state_dict): params = 0 for k, v in state_dict.items(): @@ -663,22 +712,21 @@ class TestModelFunction(unittest.TestCase): paddle.summary(ModelOutter(), input_size=(-1, 3)) def test_summary_nlp(self): - def _get_param_from_state_dict(state_dict): params = 0 for k, v in state_dict.items(): params += np.prod(v.numpy().shape) return params - nlp_net = paddle.nn.GRU(input_size=2, - hidden_size=3, - num_layers=3, - direction="bidirectional") + nlp_net = paddle.nn.GRU( + input_size=2, hidden_size=3, num_layers=3, direction="bidirectional" + ) paddle.summary(nlp_net, (1, 1, 2)) rnn = paddle.nn.LSTM(16, 32, 2) - params_info = paddle.summary(rnn, [(-1, 23, 16), - ((2, None, 32), (2, -1, 32))]) + params_info = paddle.summary( + rnn, [(-1, 23, 16), ((2, None, 32), (2, -1, 32))] + ) gt_params = _get_param_from_state_dict(rnn.state_dict()) np.testing.assert_allclose(params_info['total_params'], gt_params / 2.0) @@ -710,7 +758,7 @@ class TestModelFunction(unittest.TestCase): lenet_dict_input = LeNetDictInput() input_data = { 'x1': paddle.rand([1, 1, 28, 28]), - 'x2': paddle.rand([1, 400]) + 'x2': paddle.rand([1, 400]), } paddle.summary(lenet_dict_input, input=input_data) @@ -747,29 +795,35 @@ class TestModelFunction(unittest.TestCase): def customize_dropout(m, x, y): m.total_ops += 0 - paddle.flops(net, [1, 3, 224, 224], - custom_ops={paddle.nn.Dropout: customize_dropout}, - print_detail=True) + paddle.flops( + net, + [1, 3, 224, 224], + custom_ops={paddle.nn.Dropout: customize_dropout}, + print_detail=True, + ) def test_dynamic_flops_with_multiple_outputs(self): - net = paddle.nn.MaxPool2D(kernel_size=2, - stride=2, - padding=0, - return_mask=True) + net = paddle.nn.MaxPool2D( + kernel_size=2, stride=2, padding=0, return_mask=True + ) def customize_dropout(m, x, y): m.total_ops += 0 - paddle.flops(net, [1, 2, 32, 32], - custom_ops={paddle.nn.Dropout: customize_dropout}, - print_detail=True) + paddle.flops( + net, + [1, 2, 32, 32], + custom_ops={paddle.nn.Dropout: customize_dropout}, + print_detail=True, + ) def test_export_deploy_model(self): self.set_seed() np.random.seed(201) - save_dir = os.path.join(tempfile.mkdtemp(), - '.cache_test_export_deploy_model') + save_dir = os.path.join( + tempfile.mkdtemp(), '.cache_test_export_deploy_model' + ) if not os.path.exists(save_dir): os.makedirs(save_dir) @@ -782,28 +836,37 @@ class TestModelFunction(unittest.TestCase): model = Model(net, inputs) model.prepare() - tensor_img = np.array(np.random.random((1, 1, 28, 28)), - dtype=np.float32) + tensor_img = np.array( + np.random.random((1, 1, 28, 28)), dtype=np.float32 + ) model.save(save_dir, training=False) ori_results = model.predict_batch(tensor_img) fluid.disable_dygraph() if dynamic else None - place = fluid.CPUPlace( - ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not fluid.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) new_scope = fluid.Scope() with fluid.scope_guard(new_scope): exe = fluid.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.io.load_inference_model( - path_prefix=save_dir, executor=exe)) - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) - np.testing.assert_allclose(results, - ori_results, - rtol=1e-5, - atol=1e-6) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.io.load_inference_model( + path_prefix=save_dir, executor=exe + ) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) + np.testing.assert_allclose( + results, ori_results, rtol=1e-5, atol=1e-6 + ) paddle.enable_static() @@ -815,22 +878,26 @@ class TestModelFunction(unittest.TestCase): mnist_data = MnistDataset(mode='train') paddle.disable_static() # without inputs - save_dir = os.path.join(tempfile.mkdtemp(), - '.cache_test_dygraph_export_deploy') + save_dir = os.path.join( + tempfile.mkdtemp(), '.cache_test_dygraph_export_deploy' + ) if not os.path.exists(save_dir): os.makedirs(save_dir) for initial in ["fit", "train_batch", "eval_batch", "predict_batch"]: net = LeNet() model = Model(net) - optim = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=model.parameters()) - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum")) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters() + ) + model.prepare( + optimizer=optim, loss=CrossEntropyLoss(reduction="sum") + ) if initial == "fit": model.fit(mnist_data, batch_size=64, verbose=0) else: - img = np.array(np.random.random((1, 1, 28, 28)), - dtype=np.float32) + img = np.array( + np.random.random((1, 1, 28, 28)), dtype=np.float32 + ) label = np.array(np.random.rand(1, 1), dtype=np.int64) if initial == "train_batch": model.train_batch([img], [label]) @@ -842,52 +909,58 @@ class TestModelFunction(unittest.TestCase): model.save(save_dir, training=False) shutil.rmtree(save_dir) # with inputs, and the type of inputs is InputSpec - save_dir = os.path.join(tempfile.mkdtemp(), - '.cache_test_dygraph_export_deploy_2') + save_dir = os.path.join( + tempfile.mkdtemp(), '.cache_test_dygraph_export_deploy_2' + ) if not os.path.exists(save_dir): os.makedirs(save_dir) net = LeNet() inputs = InputSpec([None, 1, 28, 28], 'float32', 'x') model = Model(net, inputs) - optim = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=model.parameters()) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters() + ) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(save_dir, training=False) shutil.rmtree(save_dir) - def test_accumulate(self, ): + def test_accumulate( + self, + ): dim = 20 data = np.random.random(size=(4, dim)).astype(np.float32) label = np.random.randint(0, 10, size=(4, 1)).astype(np.int64) net = MyModel() - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) inputs = [InputSpec([None, dim], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] for amp_cfg in [None, 'O1']: model = Model(net, inputs, labels) - model.prepare(optim, - loss=CrossEntropyLoss(reduction="sum"), - amp_configs=amp_cfg) + model.prepare( + optim, + loss=CrossEntropyLoss(reduction="sum"), + amp_configs=amp_cfg, + ) losses, grads = [], [] for stat in [False, False, True]: - loss, = model.train_batch([data], [label], update=stat) + (loss,) = model.train_batch([data], [label], update=stat) losses.append(loss) grads.append([p.grad.numpy() for p in net.parameters()]) for grad1, grad2, grad3 in zip(*grads): np.testing.assert_almost_equal(grad1 * 2, grad2, decimal=4) - np.testing.assert_almost_equal(grad3, - np.zeros_like(grad3), - decimal=4) + np.testing.assert_almost_equal( + grad3, np.zeros_like(grad3), decimal=4 + ) np.testing.assert_almost_equal(losses[0], losses[1], decimal=4) np.testing.assert_almost_equal(losses[0], losses[2], decimal=4) class TestModelWithLRScheduler(unittest.TestCase): - def test_fit_by_step(self): base_lr = 1e-3 boundaries = [5, 8] @@ -897,17 +970,21 @@ class TestModelWithLRScheduler(unittest.TestCase): weight_decay = 5e-4 values = [base_lr * (0.1**i) for i in range(len(boundaries) + 1)] learning_rate = paddle.optimizer.lr.PiecewiseDecay( - boundaries=boundaries, values=values) + boundaries=boundaries, values=values + ) learning_rate = paddle.optimizer.lr.LinearWarmup( learning_rate=learning_rate, warmup_steps=4, - start_lr=base_lr / 5., + start_lr=base_lr / 5.0, end_lr=base_lr, - verbose=True) - optimizer = paddle.optimizer.Momentum(learning_rate=learning_rate, - weight_decay=weight_decay, - momentum=momentum, - parameters=parameters) + verbose=True, + ) + optimizer = paddle.optimizer.Momentum( + learning_rate=learning_rate, + weight_decay=weight_decay, + momentum=momentum, + parameters=parameters, + ) return optimizer # dynamic test @@ -923,8 +1000,10 @@ class TestModelWithLRScheduler(unittest.TestCase): dataset = MyDataset() model.fit(dataset, dataset, batch_size=4, epochs=10, num_workers=0) - np.testing.assert_allclose(model._optimizer._learning_rate.last_lr, - base_lr * (0.1**len(boundaries))) + np.testing.assert_allclose( + model._optimizer._learning_rate.last_lr, + base_lr * (0.1 ** len(boundaries)), + ) # static test paddle.enable_static() @@ -938,8 +1017,10 @@ class TestModelWithLRScheduler(unittest.TestCase): dataset = MyDataset() model.fit(dataset, dataset, batch_size=4, epochs=10, num_workers=0) - np.testing.assert_allclose(model._optimizer._learning_rate.last_lr, - base_lr * (0.1**len(boundaries))) + np.testing.assert_allclose( + model._optimizer._learning_rate.last_lr, + base_lr * (0.1 ** len(boundaries)), + ) def test_fit_by_epoch(self): base_lr = 1e-3 @@ -952,17 +1033,21 @@ class TestModelWithLRScheduler(unittest.TestCase): weight_decay = 5e-4 values = [base_lr * (0.1**i) for i in range(len(boundaries) + 1)] learning_rate = paddle.optimizer.lr.PiecewiseDecay( - boundaries=boundaries, values=values) + boundaries=boundaries, values=values + ) learning_rate = paddle.optimizer.lr.LinearWarmup( learning_rate=learning_rate, warmup_steps=wamup_epochs, - start_lr=base_lr / 5., + start_lr=base_lr / 5.0, end_lr=base_lr, - verbose=True) - optimizer = paddle.optimizer.Momentum(learning_rate=learning_rate, - weight_decay=weight_decay, - momentum=momentum, - parameters=parameters) + verbose=True, + ) + optimizer = paddle.optimizer.Momentum( + learning_rate=learning_rate, + weight_decay=weight_decay, + momentum=momentum, + parameters=parameters, + ) return optimizer # dynamic test @@ -977,23 +1062,27 @@ class TestModelWithLRScheduler(unittest.TestCase): dataset = MyDataset() - lr_scheduler_callback = paddle.callbacks.LRScheduler(by_step=False, - by_epoch=True) + lr_scheduler_callback = paddle.callbacks.LRScheduler( + by_step=False, by_epoch=True + ) - model.fit(dataset, - dataset, - batch_size=4, - epochs=epochs, - num_workers=0, - callbacks=lr_scheduler_callback) + model.fit( + dataset, + dataset, + batch_size=4, + epochs=epochs, + num_workers=0, + callbacks=lr_scheduler_callback, + ) cnt = 0 for b in boundaries: if b + wamup_epochs <= epochs: cnt += 1 - np.testing.assert_allclose(model._optimizer._learning_rate.last_lr, - base_lr * (0.1**cnt)) + np.testing.assert_allclose( + model._optimizer._learning_rate.last_lr, base_lr * (0.1**cnt) + ) # static test paddle.enable_static() @@ -1006,27 +1095,30 @@ class TestModelWithLRScheduler(unittest.TestCase): dataset = MyDataset() - lr_scheduler_callback = paddle.callbacks.LRScheduler(by_step=False, - by_epoch=True) + lr_scheduler_callback = paddle.callbacks.LRScheduler( + by_step=False, by_epoch=True + ) - model.fit(dataset, - dataset, - batch_size=4, - epochs=epochs, - num_workers=0, - callbacks=lr_scheduler_callback) + model.fit( + dataset, + dataset, + batch_size=4, + epochs=epochs, + num_workers=0, + callbacks=lr_scheduler_callback, + ) cnt = 0 for b in boundaries: if b + wamup_epochs <= epochs: cnt += 1 - np.testing.assert_allclose(model._optimizer._learning_rate.last_lr, - base_lr * (0.1**cnt)) + np.testing.assert_allclose( + model._optimizer._learning_rate.last_lr, base_lr * (0.1**cnt) + ) class TestRaiseError(unittest.TestCase): - def test_input_without_name(self): net = MyModel() inputs = [InputSpec([None, 10], 'float32')] @@ -1059,8 +1151,9 @@ class TestRaiseError(unittest.TestCase): model = Model(net, inputs) model.prepare() path = "" - tensor_img = np.array(np.random.random((1, 1, 28, 28)), - dtype=np.float32) + tensor_img = np.array( + np.random.random((1, 1, 28, 28)), dtype=np.float32 + ) with self.assertRaises(ValueError): model.save(path, training=False) diff --git a/python/paddle/tests/test_ops_roi_align.py b/python/paddle/tests/test_ops_roi_align.py index 145f77e846b..05c221e83a1 100644 --- a/python/paddle/tests/test_ops_roi_align.py +++ b/python/paddle/tests/test_ops_roi_align.py @@ -20,7 +20,6 @@ from paddle.vision.ops import roi_align, RoIAlign class TestRoIAlign(unittest.TestCase): - def setUp(self): self.data = np.random.rand(1, 256, 32, 32).astype('float32') boxes = np.random.rand(3, 4) @@ -40,38 +39,40 @@ class TestRoIAlign(unittest.TestCase): boxes = paddle.to_tensor(self.boxes) boxes_num = paddle.to_tensor(self.boxes_num) - align_out = roi_align(data, - boxes, - boxes_num=boxes_num, - output_size=output_size) + align_out = roi_align( + data, boxes, boxes_num=boxes_num, output_size=output_size + ) np.testing.assert_equal(align_out.shape, output_shape) else: - data = paddle.static.data(shape=self.data.shape, - dtype=self.data.dtype, - name='data') - boxes = paddle.static.data(shape=self.boxes.shape, - dtype=self.boxes.dtype, - name='boxes') - boxes_num = paddle.static.data(shape=self.boxes_num.shape, - dtype=self.boxes_num.dtype, - name='boxes_num') - - align_out = roi_align(data, - boxes, - boxes_num=boxes_num, - output_size=output_size) + data = paddle.static.data( + shape=self.data.shape, dtype=self.data.dtype, name='data' + ) + boxes = paddle.static.data( + shape=self.boxes.shape, dtype=self.boxes.dtype, name='boxes' + ) + boxes_num = paddle.static.data( + shape=self.boxes_num.shape, + dtype=self.boxes_num.dtype, + name='boxes_num', + ) + + align_out = roi_align( + data, boxes, boxes_num=boxes_num, output_size=output_size + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) - align_out = exe.run(paddle.static.default_main_program(), - feed={ - 'data': self.data, - 'boxes': self.boxes, - 'boxes_num': self.boxes_num - }, - fetch_list=[align_out]) + align_out = exe.run( + paddle.static.default_main_program(), + feed={ + 'data': self.data, + 'boxes': self.boxes, + 'boxes_num': self.boxes_num, + }, + fetch_list=[align_out], + ) np.testing.assert_equal(align_out[0].shape, output_shape) @@ -93,13 +94,19 @@ class TestRoIAlign(unittest.TestCase): align_out = roi_align_c(data, boxes, boxes_num) np.testing.assert_equal(align_out.shape, (3, 256, 4, 3)) - def test_value(self, ): - data = np.array([i for i in range(1, 17)]).reshape(1, 1, 4, - 4).astype(np.float32) - boxes = np.array([[1., 1., 2., 2.], [1.5, 1.5, 3., - 3.]]).astype(np.float32) + def test_value( + self, + ): + data = ( + np.array([i for i in range(1, 17)]) + .reshape(1, 1, 4, 4) + .astype(np.float32) + ) + boxes = np.array([[1.0, 1.0, 2.0, 2.0], [1.5, 1.5, 3.0, 3.0]]).astype( + np.float32 + ) boxes_num = np.array([2]).astype(np.int32) - output = np.array([[[[6.]]], [[[9.75]]]], dtype=np.float32) + output = np.array([[[[6.0]]], [[[9.75]]]], dtype=np.float32) data = paddle.to_tensor(data) boxes = paddle.to_tensor(boxes) diff --git a/python/paddle/tests/test_ops_roi_pool.py b/python/paddle/tests/test_ops_roi_pool.py index eaeb785df7d..923ec01503c 100644 --- a/python/paddle/tests/test_ops_roi_pool.py +++ b/python/paddle/tests/test_ops_roi_pool.py @@ -20,7 +20,6 @@ from paddle.vision.ops import roi_pool, RoIPool class TestRoIPool(unittest.TestCase): - def setUp(self): self.data = np.random.rand(1, 256, 32, 32).astype('float32') boxes = np.random.rand(3, 4) @@ -41,38 +40,40 @@ class TestRoIPool(unittest.TestCase): boxes = paddle.to_tensor(self.boxes) boxes_num = paddle.to_tensor(self.boxes_num) - pool_out = roi_pool(data, - boxes, - boxes_num=boxes_num, - output_size=output_size) + pool_out = roi_pool( + data, boxes, boxes_num=boxes_num, output_size=output_size + ) np.testing.assert_equal(pool_out.shape, output_shape) else: - data = paddle.static.data(shape=self.data.shape, - dtype=self.data.dtype, - name='data') - boxes = paddle.static.data(shape=self.boxes.shape, - dtype=self.boxes.dtype, - name='boxes') - boxes_num = paddle.static.data(shape=self.boxes_num.shape, - dtype=self.boxes_num.dtype, - name='boxes_num') - - pool_out = roi_pool(data, - boxes, - boxes_num=boxes_num, - output_size=output_size) + data = paddle.static.data( + shape=self.data.shape, dtype=self.data.dtype, name='data' + ) + boxes = paddle.static.data( + shape=self.boxes.shape, dtype=self.boxes.dtype, name='boxes' + ) + boxes_num = paddle.static.data( + shape=self.boxes_num.shape, + dtype=self.boxes_num.dtype, + name='boxes_num', + ) + + pool_out = roi_pool( + data, boxes, boxes_num=boxes_num, output_size=output_size + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) - pool_out = exe.run(paddle.static.default_main_program(), - feed={ - 'data': self.data, - 'boxes': self.boxes, - 'boxes_num': self.boxes_num - }, - fetch_list=[pool_out]) + pool_out = exe.run( + paddle.static.default_main_program(), + feed={ + 'data': self.data, + 'boxes': self.boxes, + 'boxes_num': self.boxes_num, + }, + fetch_list=[pool_out], + ) np.testing.assert_equal(pool_out[0].shape, output_shape) @@ -94,13 +95,19 @@ class TestRoIPool(unittest.TestCase): pool_out = roi_pool_c(data, boxes, boxes_num) np.testing.assert_equal(pool_out.shape, (3, 256, 4, 3)) - def test_value(self, ): - data = np.array([i for i in range(1, 17)]).reshape(1, 1, 4, - 4).astype(np.float32) - boxes = np.array([[1., 1., 2., 2.], [1.5, 1.5, 3., - 3.]]).astype(np.float32) + def test_value( + self, + ): + data = ( + np.array([i for i in range(1, 17)]) + .reshape(1, 1, 4, 4) + .astype(np.float32) + ) + boxes = np.array([[1.0, 1.0, 2.0, 2.0], [1.5, 1.5, 3.0, 3.0]]).astype( + np.float32 + ) boxes_num = np.array([2]).astype(np.int32) - output = np.array([[[[11.]]], [[[16.]]]], dtype=np.float32) + output = np.array([[[[11.0]]], [[[16.0]]]], dtype=np.float32) data = paddle.to_tensor(data) boxes = paddle.to_tensor(boxes) diff --git a/python/paddle/tests/test_pretrained_model.py b/python/paddle/tests/test_pretrained_model.py index 4c655ce3c1f..81df3adf8c7 100644 --- a/python/paddle/tests/test_pretrained_model.py +++ b/python/paddle/tests/test_pretrained_model.py @@ -26,7 +26,6 @@ import paddle.vision.models as models # test the predicted resutls of static graph and dynamic graph are equal # when used pretrained model class TestPretrainedModel(unittest.TestCase): - def infer(self, arch): path = os.path.join(tempfile.mkdtemp(), '.cache_test_pretrained_model') if not os.path.exists(path): diff --git a/python/paddle/tests/test_progressbar.py b/python/paddle/tests/test_progressbar.py index c42f1e4db0f..09340a9c247 100644 --- a/python/paddle/tests/test_progressbar.py +++ b/python/paddle/tests/test_progressbar.py @@ -21,7 +21,6 @@ from paddle.hapi.progressbar import ProgressBar class TestProgressBar(unittest.TestCase): - def prog_bar(self, num, epoch, width, verbose=1): for epoch in range(epoch): progbar = ProgressBar(num, verbose=verbose) @@ -40,7 +39,7 @@ class TestProgressBar(unittest.TestCase): progbar.update(1, [['loss', int(1)]]) progbar.update(1, [['loss', 'INF']]) progbar.update(1, [['loss', 1e-4]]) - progbar.update(1, [['loss', np.array([1.])]]) + progbar.update(1, [['loss', np.array([1.0])]]) progbar.update(1, [['loss', np.array([1e-4])]]) progbar.update(1, [['loss', np.array([1]).astype(np.uint16)]]) progbar.start() diff --git a/python/paddle/tests/test_read_file.py b/python/paddle/tests/test_read_file.py index ce387f039c7..d13e901b322 100644 --- a/python/paddle/tests/test_read_file.py +++ b/python/paddle/tests/test_read_file.py @@ -23,7 +23,6 @@ from paddle.vision.ops import read_file, decode_jpeg class TestReadFile(unittest.TestCase): - def setUp(self): fake_img = (np.random.random((400, 300, 3)) * 255).astype('uint8') self.temp_dir = tempfile.TemporaryDirectory() @@ -51,11 +50,13 @@ class TestReadFile(unittest.TestCase): place = paddle.CUDAPlace(0) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) - out = exe.run(paddle.static.default_main_program(), - fetch_list=[img]) + out = exe.run( + paddle.static.default_main_program(), fetch_list=[img] + ) - np.testing.assert_equal(out[0].shape, - img_cv2.transpose(2, 0, 1).shape) + np.testing.assert_equal( + out[0].shape, img_cv2.transpose(2, 0, 1).shape + ) def test_read_file_decode_jpeg_dynamic(self): self.read_file_decode_jpeg() diff --git a/python/paddle/tests/test_transforms.py b/python/paddle/tests/test_transforms.py index f182cbd799c..9e2cad57c37 100644 --- a/python/paddle/tests/test_transforms.py +++ b/python/paddle/tests/test_transforms.py @@ -28,7 +28,6 @@ import paddle.vision.transforms.functional as F class TestTransformsCV2(unittest.TestCase): - def setUp(self): self.backend = self.get_backend() set_image_backend(self.backend) @@ -39,11 +38,13 @@ class TestTransformsCV2(unittest.TestCase): os.makedirs(sub_dir) for j in range(2): if j == 0: - fake_img = (np.random.random( - (280, 350, 3)) * 255).astype('uint8') + fake_img = (np.random.random((280, 350, 3)) * 255).astype( + 'uint8' + ) else: - fake_img = (np.random.random( - (400, 300, 3)) * 255).astype('uint8') + fake_img = (np.random.random((400, 300, 3)) * 255).astype( + 'uint8' + ) cv2.imwrite(os.path.join(sub_dir, str(j) + '.jpg'), fake_img) def get_backend(self): @@ -54,7 +55,8 @@ class TestTransformsCV2(unittest.TestCase): return (np.random.rand(*shape) * 255).astype('uint8') elif self.backend == 'pil': return Image.fromarray( - (np.random.rand(*shape) * 255).astype('uint8')) + (np.random.rand(*shape) * 255).astype('uint8') + ) def get_shape(self, img): if isinstance(img, paddle.Tensor): @@ -79,16 +81,17 @@ class TestTransformsCV2(unittest.TestCase): mean=[123.675, 116.28, 103.53], std=[58.395, 57.120, 57.375], ) - trans = transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.ColorJitter(brightness=0.4, - contrast=0.4, - saturation=0.4, - hue=0.4), - transforms.RandomHorizontalFlip(), - transforms.Transpose(), - normalize, - ]) + trans = transforms.Compose( + [ + transforms.RandomResizedCrop(224), + transforms.ColorJitter( + brightness=0.4, contrast=0.4, saturation=0.4, hue=0.4 + ), + transforms.RandomHorizontalFlip(), + transforms.Transpose(), + normalize, + ] + ) self.do_transform(trans) @@ -98,83 +101,104 @@ class TestTransformsCV2(unittest.TestCase): self.do_transform(trans) def test_trans_resize(self): - trans = transforms.Compose([ - transforms.Resize(300), - transforms.RandomResizedCrop((280, 280)), - transforms.Resize(280), - transforms.Resize((256, 200)), - transforms.Resize((180, 160)), - transforms.CenterCrop(128), - transforms.CenterCrop((128, 128)), - ]) + trans = transforms.Compose( + [ + transforms.Resize(300), + transforms.RandomResizedCrop((280, 280)), + transforms.Resize(280), + transforms.Resize((256, 200)), + transforms.Resize((180, 160)), + transforms.CenterCrop(128), + transforms.CenterCrop((128, 128)), + ] + ) self.do_transform(trans) def test_flip(self): - trans = transforms.Compose([ - transforms.RandomHorizontalFlip(1.0), - transforms.RandomHorizontalFlip(0.0), - transforms.RandomVerticalFlip(0.0), - transforms.RandomVerticalFlip(1.0), - ]) + trans = transforms.Compose( + [ + transforms.RandomHorizontalFlip(1.0), + transforms.RandomHorizontalFlip(0.0), + transforms.RandomVerticalFlip(0.0), + transforms.RandomVerticalFlip(1.0), + ] + ) self.do_transform(trans) def test_color_jitter(self): - trans = transforms.Compose([ - transforms.BrightnessTransform(0.0), - transforms.HueTransform(0.0), - transforms.SaturationTransform(0.0), - transforms.ContrastTransform(0.0), - ]) + trans = transforms.Compose( + [ + transforms.BrightnessTransform(0.0), + transforms.HueTransform(0.0), + transforms.SaturationTransform(0.0), + transforms.ContrastTransform(0.0), + ] + ) self.do_transform(trans) def test_affine(self): - trans = transforms.Compose([ - transforms.RandomAffine(90), - transforms.RandomAffine([-10, 10], translate=[0.1, 0.3]), - transforms.RandomAffine(45, translate=[0.2, 0.2], scale=[0.2, 0.5]), - transforms.RandomAffine(10, - translate=[0.2, 0.2], - scale=[0.5, 0.5], - shear=[-10, 10]), - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40]), - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40], - interpolation='bilinear'), - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40], - interpolation='bilinear', - fill=114), - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40], - interpolation='bilinear', - fill=114, - center=(60, 80)), - ]) + trans = transforms.Compose( + [ + transforms.RandomAffine(90), + transforms.RandomAffine([-10, 10], translate=[0.1, 0.3]), + transforms.RandomAffine( + 45, translate=[0.2, 0.2], scale=[0.2, 0.5] + ), + transforms.RandomAffine( + 10, translate=[0.2, 0.2], scale=[0.5, 0.5], shear=[-10, 10] + ), + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + ), + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + interpolation='bilinear', + ), + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + interpolation='bilinear', + fill=114, + ), + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + interpolation='bilinear', + fill=114, + center=(60, 80), + ), + ] + ) self.do_transform(trans) def test_rotate(self): - trans = transforms.Compose([ - transforms.RandomRotation(90), - transforms.RandomRotation([-10, 10]), - transforms.RandomRotation(45, expand=True), - transforms.RandomRotation(10, expand=True, center=(60, 80)), - ]) + trans = transforms.Compose( + [ + transforms.RandomRotation(90), + transforms.RandomRotation([-10, 10]), + transforms.RandomRotation(45, expand=True), + transforms.RandomRotation(10, expand=True, center=(60, 80)), + ] + ) self.do_transform(trans) def test_perspective(self): - trans = transforms.Compose([ - transforms.RandomPerspective(prob=1.0), - transforms.RandomPerspective(prob=1.0, distortion_scale=0.9), - ]) + trans = transforms.Compose( + [ + transforms.RandomPerspective(prob=1.0), + transforms.RandomPerspective(prob=1.0, distortion_scale=0.9), + ] + ) self.do_transform(trans) def test_pad(self): @@ -191,10 +215,12 @@ class TestTransformsCV2(unittest.TestCase): img = trans_pad2(img) def test_random_crop(self): - trans = transforms.Compose([ - transforms.RandomCrop(200), - transforms.RandomCrop((140, 160)), - ]) + trans = transforms.Compose( + [ + transforms.RandomCrop(200), + transforms.RandomCrop((140, 160)), + ] + ) self.do_transform(trans) trans_random_crop1 = transforms.RandomCrop(224) @@ -211,18 +237,21 @@ class TestTransformsCV2(unittest.TestCase): trans_random_crop_same = transforms.RandomCrop((140, 160)) img = trans_random_crop_same(fake_img_crop2) - trans_random_crop_bigger = transforms.RandomCrop((180, 200), - pad_if_needed=True) + trans_random_crop_bigger = transforms.RandomCrop( + (180, 200), pad_if_needed=True + ) img = trans_random_crop_bigger(img) trans_random_crop_pad = transforms.RandomCrop((224, 256), 2, True) img = trans_random_crop_pad(img) def test_erase(self): - trans = transforms.Compose([ - transforms.RandomErasing(), - transforms.RandomErasing(value="random") - ]) + trans = transforms.Compose( + [ + transforms.RandomErasing(), + transforms.RandomErasing(value="random"), + ] + ) self.do_transform(trans) def test_grayscale(self): @@ -261,7 +290,7 @@ class TestTransformsCV2(unittest.TestCase): def test_keys(self): fake_img1 = self.create_image((200, 150, 3)) fake_img2 = self.create_image((200, 150, 3)) - trans_pad = transforms.Pad(10, keys=("image", )) + trans_pad = transforms.Pad(10, keys=("image",)) fake_img_padded = trans_pad((fake_img1, fake_img2)) def test_exception(self): @@ -332,24 +361,27 @@ class TestTransformsCV2(unittest.TestCase): transforms.RandomAffine(10, translate=[0.2, 0.2], scale=[1, 2, 3]), with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.2, 0.2], - scale=[0.5, 0.5], - shear=[1, 2, 3]), + transforms.RandomAffine( + 10, translate=[0.2, 0.2], scale=[0.5, 0.5], shear=[1, 2, 3] + ), with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 0, 20, 40]) + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 0, 20, 40], + ) with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40], - fill=114, - center=(1, 2, 3)) + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + fill=114, + center=(1, 2, 3), + ) with self.assertRaises(ValueError): transforms.RandomRotation(-2) @@ -402,19 +434,18 @@ class TestTransformsCV2(unittest.TestCase): class TestTransformsPIL(TestTransformsCV2): - def get_backend(self): return 'pil' class TestTransformsTensor(TestTransformsCV2): - def get_backend(self): return 'tensor' def create_image(self, shape): return paddle.to_tensor(np.random.rand(*shape)).transpose( - (2, 0, 1)) # hwc->chw + (2, 0, 1) + ) # hwc->chw def do_transform(self, trans): trans.transforms.insert(0, transforms.ToTensor(data_format='CHW')) @@ -428,11 +459,13 @@ class TestTransformsTensor(TestTransformsCV2): mean=[123.675, 116.28, 103.53], std=[58.395, 57.120, 57.375], ) - trans = transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - normalize, - ]) + trans = transforms.Compose( + [ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + normalize, + ] + ) self.do_transform(trans) def test_grayscale(self): @@ -489,10 +522,12 @@ class TestTransformsTensor(TestTransformsCV2): img = trans_pad4(img) def test_random_crop(self): - trans = transforms.Compose([ - transforms.RandomCrop(200), - transforms.RandomCrop((140, 160)), - ]) + trans = transforms.Compose( + [ + transforms.RandomCrop(200), + transforms.RandomCrop((140, 160)), + ] + ) self.do_transform(trans) trans_random_crop1 = transforms.RandomCrop(224) @@ -509,18 +544,21 @@ class TestTransformsTensor(TestTransformsCV2): trans_random_crop_same = transforms.RandomCrop((140, 160)) img = trans_random_crop_same(fake_img_crop2) - trans_random_crop_bigger = transforms.RandomCrop((180, 200), - pad_if_needed=True) + trans_random_crop_bigger = transforms.RandomCrop( + (180, 200), pad_if_needed=True + ) img = trans_random_crop_bigger(img) trans_random_crop_pad = transforms.RandomCrop((224, 256), 2, True) img = trans_random_crop_pad(img) def test_erase(self): - trans = transforms.Compose([ - transforms.RandomErasing(value=(0.5, )), - transforms.RandomErasing(value="random") - ]) + trans = transforms.Compose( + [ + transforms.RandomErasing(value=(0.5,)), + transforms.RandomErasing(value="random"), + ] + ) self.do_transform(trans) erase_trans = transforms.RandomErasing(value=(0.5, 0.2, 0.01)) @@ -586,16 +624,17 @@ class TestTransformsTensor(TestTransformsCV2): transforms.RandomAffine(10, translate=[0.2, 0.2], scale=[1, 2, 3]), with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.2, 0.2], - scale=[0.5, 0.5], - shear=[1, 2, 3]), + transforms.RandomAffine( + 10, translate=[0.2, 0.2], scale=[0.5, 0.5], shear=[1, 2, 3] + ), with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 0, 20, 40]) + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 0, 20, 40], + ) with self.assertRaises(ValueError): transforms.RandomRotation(-2) @@ -616,14 +655,14 @@ class TestTransformsTensor(TestTransformsCV2): class TestFunctional(unittest.TestCase): - def test_errors(self): with self.assertRaises(TypeError): F.to_tensor(1) with self.assertRaises(ValueError): fake_img = Image.fromarray( - (np.random.rand(28, 28, 3) * 255).astype('uint8')) + (np.random.rand(28, 28, 3) * 255).astype('uint8') + ) F.to_tensor(fake_img, data_format=1) with self.assertRaises(ValueError): @@ -636,7 +675,8 @@ class TestFunctional(unittest.TestCase): with self.assertRaises(TypeError): fake_img = Image.fromarray( - (np.random.rand(28, 28, 3) * 255).astype('uint8')) + (np.random.rand(28, 28, 3) * 255).astype('uint8') + ) F.resize(fake_img, '1') with self.assertRaises(TypeError): @@ -685,18 +725,18 @@ class TestFunctional(unittest.TestCase): F.affine(45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 0, 10]) with self.assertRaises(TypeError): - F.affine(45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10], - interpolation=2) + F.affine( + 45, + translate=[0.2, 0.2], + scale=0.5, + shear=[-10, 10], + interpolation=2, + ) with self.assertRaises(TypeError): - F.affine(45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10], - center=0) + F.affine( + 45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 10], center=0 + ) with self.assertRaises(TypeError): F.rotate(1, 0.1) @@ -720,23 +760,21 @@ class TestFunctional(unittest.TestCase): std = [0.5, 0.5, 0.5] normalized_img = F.normalize(tensor_img, mean, std) - normalized_img_tensor = F.normalize(tensor_img_hwc, - mean, - std, - data_format='HWC') + normalized_img_tensor = F.normalize( + tensor_img_hwc, mean, std, data_format='HWC' + ) normalized_img_pil = F.normalize(pil_img, mean, std, data_format='HWC') - normalized_img_np = F.normalize(np_img, - mean, - std, - data_format='HWC', - to_rgb=False) - - np.testing.assert_almost_equal(np.array(normalized_img_pil), - normalized_img_np) - np.testing.assert_almost_equal(normalized_img_tensor.numpy(), - normalized_img_np, - decimal=4) + normalized_img_np = F.normalize( + np_img, mean, std, data_format='HWC', to_rgb=False + ) + + np.testing.assert_almost_equal( + np.array(normalized_img_pil), normalized_img_np + ) + np.testing.assert_almost_equal( + normalized_img_tensor.numpy(), normalized_img_np, decimal=4 + ) def test_center_crop(self): np_img = (np.random.rand(28, 24, 3) * 255).astype('uint8') @@ -747,12 +785,14 @@ class TestFunctional(unittest.TestCase): pil_cropped_img = F.center_crop(pil_img, 4) tensor_cropped_img = F.center_crop(tensor_img, 4) - np.testing.assert_almost_equal(np_cropped_img, - np.array(pil_cropped_img)) - np.testing.assert_almost_equal(np_cropped_img, - tensor_cropped_img.numpy().transpose( - (1, 2, 0)), - decimal=4) + np.testing.assert_almost_equal( + np_cropped_img, np.array(pil_cropped_img) + ) + np.testing.assert_almost_equal( + np_cropped_img, + tensor_cropped_img.numpy().transpose((1, 2, 0)), + decimal=4, + ) def test_color_jitter_sub_function(self): np.random.seed(555) @@ -771,8 +811,9 @@ class TestFunctional(unittest.TestCase): def test_adjust_brightness(np_img, tensor_img): result_cv2 = np.array(F.adjust_brightness(np_img, 1.2)) result_tensor = F.adjust_brightness(tensor_img, 1.2).numpy() - result_tensor = np.transpose(result_tensor * 255, - (1, 2, 0)).astype('uint8') + result_tensor = np.transpose(result_tensor * 255, (1, 2, 0)).astype( + 'uint8' + ) np.testing.assert_equal(result_cv2, result_tensor) # For adjust_contrast / adjust_saturation / adjust_hue the implement is kind @@ -788,7 +829,7 @@ class TestFunctional(unittest.TestCase): def test_adjust_saturation(np_img, tensor_img): result_pil = np.array(F.adjust_saturation(np_img, 1.0)) result_tensor = F.adjust_saturation(tensor_img, 1.0).numpy() - result_tensor = np.transpose(result_tensor * 255., (1, 2, 0)) + result_tensor = np.transpose(result_tensor * 255.0, (1, 2, 0)) diff = np.max(np.abs(result_tensor - result_pil)) self.assertTrue(diff < 1.1) @@ -817,14 +858,16 @@ class TestFunctional(unittest.TestCase): tensor_padded_img = F.pad(tensor_img, [1, 2], padding_mode='reflect') np.testing.assert_almost_equal(np_padded_img, np.array(pil_padded_img)) - np.testing.assert_almost_equal(np_padded_img, - tensor_padded_img.numpy().transpose( - (1, 2, 0)), - decimal=3) + np.testing.assert_almost_equal( + np_padded_img, + tensor_padded_img.numpy().transpose((1, 2, 0)), + decimal=3, + ) tensor_padded_img = F.pad(tensor_img, 1, padding_mode='reflect') - tensor_padded_img = F.pad(tensor_img, [1, 2, 1, 2], - padding_mode='reflect') + tensor_padded_img = F.pad( + tensor_img, [1, 2, 1, 2], padding_mode='reflect' + ) pil_p_img = pil_img.convert('P') pil_padded_img = F.pad(pil_p_img, [1, 2]) @@ -840,16 +883,19 @@ class TestFunctional(unittest.TestCase): tensor_reseized_img = F.resize(tensor_img, 40) tensor_reseized_img2 = F.resize(tensor_img, (46, 40)) - np.testing.assert_almost_equal(np_reseized_img, - np.array(pil_reseized_img)) - np.testing.assert_almost_equal(np_reseized_img, - tensor_reseized_img.numpy().transpose( - (1, 2, 0)), - decimal=3) - np.testing.assert_almost_equal(np_reseized_img, - tensor_reseized_img2.numpy().transpose( - (1, 2, 0)), - decimal=3) + np.testing.assert_almost_equal( + np_reseized_img, np.array(pil_reseized_img) + ) + np.testing.assert_almost_equal( + np_reseized_img, + tensor_reseized_img.numpy().transpose((1, 2, 0)), + decimal=3, + ) + np.testing.assert_almost_equal( + np_reseized_img, + tensor_reseized_img2.numpy().transpose((1, 2, 0)), + decimal=3, + ) gray_img = (np.zeros([28, 32])).astype('uint8') gray_resize_img = F.resize(gray_img, 40) @@ -905,26 +951,30 @@ class TestFunctional(unittest.TestCase): expected_tensor = tensor_img.clone() expected_tensor[:, 10:15, 10:15] = paddle.to_tensor([0.88]) - tensor_result = F.erase(tensor_img, 10, 10, 5, 5, - paddle.to_tensor([0.88])) - np.testing.assert_equal(tensor_result.numpy(), - expected_tensor.numpy()) + tensor_result = F.erase( + tensor_img, 10, 10, 5, 5, paddle.to_tensor([0.88]) + ) + np.testing.assert_equal( + tensor_result.numpy(), expected_tensor.numpy() + ) def test_erase_backward(self): img = paddle.randn((3, 14, 14), dtype=np.float32) img.stop_gradient = False - erased = F.erase(img, 3, 3, 5, 5, paddle.ones((1, 1, 1), - dtype='float32')) + erased = F.erase( + img, 3, 3, 5, 5, paddle.ones((1, 1, 1), dtype='float32') + ) loss = erased.sum() loss.backward() expected_grad = np.ones((3, 14, 14), dtype=np.float32) - expected_grad[:, 3:8, 3:8] = 0. + expected_grad[:, 3:8, 3:8] = 0.0 np.testing.assert_equal(img.grad.numpy(), expected_grad) def test_image_load(self): - fake_img = Image.fromarray((np.random.random( - (32, 32, 3)) * 255).astype('uint8')) + fake_img = Image.fromarray( + (np.random.random((32, 32, 3)) * 255).astype('uint8') + ) temp_dir = tempfile.TemporaryDirectory() path = os.path.join(temp_dir.name, 'temp.jpg') @@ -947,35 +997,32 @@ class TestFunctional(unittest.TestCase): pil_img = Image.fromarray(np_img).convert('RGB') tensor_img = F.to_tensor(pil_img, data_format='CHW') * 255 - np.testing.assert_almost_equal(np_img, - tensor_img.transpose((1, 2, 0)), - decimal=4) - - np_affined_img = F.affine(np_img, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) - pil_affined_img = F.affine(pil_img, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) - tensor_affined_img = F.affine(tensor_img, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) - - np.testing.assert_equal(np_affined_img.shape, - np.array(pil_affined_img).shape) - np.testing.assert_equal(np_affined_img.shape, - tensor_affined_img.transpose((1, 2, 0)).shape) - - np.testing.assert_almost_equal(np.array(pil_affined_img), - tensor_affined_img.numpy().transpose( - (1, 2, 0)), - decimal=4) + np.testing.assert_almost_equal( + np_img, tensor_img.transpose((1, 2, 0)), decimal=4 + ) + + np_affined_img = F.affine( + np_img, 45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 10] + ) + pil_affined_img = F.affine( + pil_img, 45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 10] + ) + tensor_affined_img = F.affine( + tensor_img, 45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 10] + ) + + np.testing.assert_equal( + np_affined_img.shape, np.array(pil_affined_img).shape + ) + np.testing.assert_equal( + np_affined_img.shape, tensor_affined_img.transpose((1, 2, 0)).shape + ) + + np.testing.assert_almost_equal( + np.array(pil_affined_img), + tensor_affined_img.numpy().transpose((1, 2, 0)), + decimal=4, + ) def test_rotate(self): np_img = (np.random.rand(28, 28, 3) * 255).astype('uint8') @@ -987,64 +1034,70 @@ class TestFunctional(unittest.TestCase): rotated_tensor_img1 = F.rotate(tensor_img, 80, expand=True) - rotated_tensor_img2 = F.rotate(tensor_img, - 80, - interpolation='bilinear', - center=(10, 10), - expand=False) + rotated_tensor_img2 = F.rotate( + tensor_img, + 80, + interpolation='bilinear', + center=(10, 10), + expand=False, + ) - np.testing.assert_equal(rotated_np_img.shape, - np.array(rotated_pil_img).shape) - np.testing.assert_equal(rotated_np_img.shape, - rotated_tensor_img1.transpose((1, 2, 0)).shape) + np.testing.assert_equal( + rotated_np_img.shape, np.array(rotated_pil_img).shape + ) + np.testing.assert_equal( + rotated_np_img.shape, rotated_tensor_img1.transpose((1, 2, 0)).shape + ) def test_rotate1(self): np_img = (np.random.rand(28, 28, 3) * 255).astype('uint8') pil_img = Image.fromarray(np_img).convert('RGB') - rotated_np_img = F.rotate(np_img, - 80, - expand=True, - center=[0, 0], - fill=[0, 0, 0]) - rotated_pil_img = F.rotate(pil_img, - 80, - expand=True, - center=[0, 0], - fill=[0, 0, 0]) + rotated_np_img = F.rotate( + np_img, 80, expand=True, center=[0, 0], fill=[0, 0, 0] + ) + rotated_pil_img = F.rotate( + pil_img, 80, expand=True, center=[0, 0], fill=[0, 0, 0] + ) - np.testing.assert_equal(rotated_np_img.shape, - np.array(rotated_pil_img).shape) + np.testing.assert_equal( + rotated_np_img.shape, np.array(rotated_pil_img).shape + ) def test_perspective(self): np_img = (np.random.rand(32, 26, 3) * 255).astype('uint8') pil_img = Image.fromarray(np_img).convert('RGB') tensor_img = F.to_tensor(pil_img, data_format='CHW') * 255 - np.testing.assert_almost_equal(np_img, - tensor_img.transpose((1, 2, 0)), - decimal=4) + np.testing.assert_almost_equal( + np_img, tensor_img.transpose((1, 2, 0)), decimal=4 + ) startpoints = [[0, 0], [13, 0], [13, 15], [0, 15]] endpoints = [[3, 2], [12, 3], [10, 14], [2, 15]] np_perspectived_img = F.perspective(np_img, startpoints, endpoints) pil_perspectived_img = F.perspective(pil_img, startpoints, endpoints) - tensor_perspectived_img = F.perspective(tensor_img, startpoints, - endpoints) + tensor_perspectived_img = F.perspective( + tensor_img, startpoints, endpoints + ) - np.testing.assert_equal(np_perspectived_img.shape, - np.array(pil_perspectived_img).shape) + np.testing.assert_equal( + np_perspectived_img.shape, np.array(pil_perspectived_img).shape + ) np.testing.assert_equal( np_perspectived_img.shape, - tensor_perspectived_img.transpose((1, 2, 0)).shape) + tensor_perspectived_img.transpose((1, 2, 0)).shape, + ) result_pil = np.array(pil_perspectived_img) - result_tensor = tensor_perspectived_img.numpy().transpose( - (1, 2, 0)).astype('uint8') + result_tensor = ( + tensor_perspectived_img.numpy().transpose((1, 2, 0)).astype('uint8') + ) num_diff_pixels = (result_pil != result_tensor).sum() / 3.0 - ratio_diff_pixels = num_diff_pixels / result_tensor.shape[ - 0] / result_tensor.shape[1] + ratio_diff_pixels = ( + num_diff_pixels / result_tensor.shape[0] / result_tensor.shape[1] + ) # Tolerance : less than 6% of different pixels assert ratio_diff_pixels < 0.06 @@ -1054,10 +1107,12 @@ class TestFunctional(unittest.TestCase): def test_erase(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.erase(input1, 1, 1, 2, 2, 0.5), - F.erase(input2, 1, 1, 2, 2, 0.5) - ]) + target_result = paddle.stack( + [ + F.erase(input1, 1, 1, 2, 2, 0.5), + F.erase(input2, 1, 1, 2, 2, 0.5), + ] + ) batch_result = F.erase(batch_tensor, 1, 1, 2, 2, 0.5) @@ -1067,23 +1122,31 @@ class TestFunctional(unittest.TestCase): def test_affine(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.affine(input1, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]), - F.affine(input2, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) - ]) - batch_result = F.affine(batch_tensor, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) + target_result = paddle.stack( + [ + F.affine( + input1, + 45, + translate=[0.2, 0.2], + scale=0.5, + shear=[-10, 10], + ), + F.affine( + input2, + 45, + translate=[0.2, 0.2], + scale=0.5, + shear=[-10, 10], + ), + ] + ) + batch_result = F.affine( + batch_tensor, + 45, + translate=[0.2, 0.2], + scale=0.5, + shear=[-10, 10], + ) return paddle.allclose(batch_result, target_result) @@ -1093,10 +1156,12 @@ class TestFunctional(unittest.TestCase): input1, input2 = paddle.unbind(batch_tensor, axis=0) startpoints = [[0, 0], [3, 0], [4, 5], [6, 7]] endpoints = [[0, 1], [3, 1], [4, 4], [5, 7]] - target_result = paddle.stack([ - F.perspective(input1, startpoints, endpoints), - F.perspective(input2, startpoints, endpoints) - ]) + target_result = paddle.stack( + [ + F.perspective(input1, startpoints, endpoints), + F.perspective(input2, startpoints, endpoints), + ] + ) batch_result = F.perspective(batch_tensor, startpoints, endpoints) @@ -1106,10 +1171,12 @@ class TestFunctional(unittest.TestCase): def test_adjust_brightness(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.adjust_brightness(input1, 2.1), - F.adjust_brightness(input2, 2.1) - ]) + target_result = paddle.stack( + [ + F.adjust_brightness(input1, 2.1), + F.adjust_brightness(input2, 2.1), + ] + ) batch_result = F.adjust_brightness(batch_tensor, 2.1) @@ -1119,10 +1186,9 @@ class TestFunctional(unittest.TestCase): def test_adjust_contrast(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.adjust_contrast(input1, 0.3), - F.adjust_contrast(input2, 0.3) - ]) + target_result = paddle.stack( + [F.adjust_contrast(input1, 0.3), F.adjust_contrast(input2, 0.3)] + ) batch_result = F.adjust_contrast(batch_tensor, 0.3) @@ -1132,10 +1198,12 @@ class TestFunctional(unittest.TestCase): def test_adjust_saturation(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.adjust_saturation(input1, 1.1), - F.adjust_saturation(input2, 1.1) - ]) + target_result = paddle.stack( + [ + F.adjust_saturation(input1, 1.1), + F.adjust_saturation(input2, 1.1), + ] + ) batch_result = F.adjust_saturation(batch_tensor, 1.1) @@ -1146,8 +1214,8 @@ class TestFunctional(unittest.TestCase): def test_adjust_hue(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) target_result = paddle.stack( - [F.adjust_hue(input1, -0.2), - F.adjust_hue(input2, -0.2)]) + [F.adjust_hue(input1, -0.2), F.adjust_hue(input2, -0.2)] + ) batch_result = F.adjust_hue(batch_tensor, -0.2) diff --git a/python/paddle/tests/test_vision_models.py b/python/paddle/tests/test_vision_models.py index 1f53060beb0..dc98fc3219b 100644 --- a/python/paddle/tests/test_vision_models.py +++ b/python/paddle/tests/test_vision_models.py @@ -20,7 +20,6 @@ import paddle.vision.models as models class TestVisonModels(unittest.TestCase): - def models_infer(self, arch, pretrained=False, batch_norm=False): x = np.array(np.random.random((2, 3, 224, 224)), dtype=np.float32) diff --git a/python/paddle/text/__init__.py b/python/paddle/text/__init__.py index 5775a247858..fbfa0c3fe2e 100644 --- a/python/paddle/text/__init__.py +++ b/python/paddle/text/__init__.py @@ -21,7 +21,14 @@ from .datasets import UCIHousing # noqa: F401 from .datasets import WMT14 # noqa: F401 from .datasets import WMT16 # noqa: F401 -__all__ = [ #noqa - 'Conll05st', 'Imdb', 'Imikolov', 'Movielens', 'UCIHousing', 'WMT14', - 'WMT16', 'ViterbiDecoder', 'viterbi_decode' +__all__ = [ # noqa + 'Conll05st', + 'Imdb', + 'Imikolov', + 'Movielens', + 'UCIHousing', + 'WMT14', + 'WMT16', + 'ViterbiDecoder', + 'viterbi_decode', ] diff --git a/python/paddle/text/datasets/conll05.py b/python/paddle/text/datasets/conll05.py index ee475e7fc5d..807c8c3fbeb 100644 --- a/python/paddle/text/datasets/conll05.py +++ b/python/paddle/text/datasets/conll05.py @@ -91,47 +91,71 @@ class Conll05st(Dataset): """ - def __init__(self, - data_file=None, - word_dict_file=None, - verb_dict_file=None, - target_dict_file=None, - emb_file=None, - download=True): + def __init__( + self, + data_file=None, + word_dict_file=None, + verb_dict_file=None, + target_dict_file=None, + emb_file=None, + download=True, + ): self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, DATA_URL, - DATA_MD5, 'conll05st', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, DATA_URL, DATA_MD5, 'conll05st', download + ) self.word_dict_file = word_dict_file if self.word_dict_file is None: - assert download, "word_dict_file is not set and downloading automatically is disabled" + assert ( + download + ), "word_dict_file is not set and downloading automatically is disabled" self.word_dict_file = _check_exists_and_download( - word_dict_file, WORDDICT_URL, WORDDICT_MD5, 'conll05st', - download) + word_dict_file, + WORDDICT_URL, + WORDDICT_MD5, + 'conll05st', + download, + ) self.verb_dict_file = verb_dict_file if self.verb_dict_file is None: - assert download, "verb_dict_file is not set and downloading automatically is disabled" + assert ( + download + ), "verb_dict_file is not set and downloading automatically is disabled" self.verb_dict_file = _check_exists_and_download( - verb_dict_file, VERBDICT_URL, VERBDICT_MD5, 'conll05st', - download) + verb_dict_file, + VERBDICT_URL, + VERBDICT_MD5, + 'conll05st', + download, + ) self.target_dict_file = target_dict_file if self.target_dict_file is None: - assert download, "target_dict_file is not set and downloading automatically is disabled" + assert ( + download + ), "target_dict_file is not set and downloading automatically is disabled" self.target_dict_file = _check_exists_and_download( - target_dict_file, TRGDICT_URL, TRGDICT_MD5, 'conll05st', - download) + target_dict_file, + TRGDICT_URL, + TRGDICT_MD5, + 'conll05st', + download, + ) self.emb_file = emb_file if self.emb_file is None: - assert download, "emb_file is not set and downloading automatically is disabled" - self.emb_file = _check_exists_and_download(emb_file, EMB_URL, - EMB_MD5, 'conll05st', - download) + assert ( + download + ), "emb_file is not set and downloading automatically is disabled" + self.emb_file = _check_exists_and_download( + emb_file, EMB_URL, EMB_MD5, 'conll05st', download + ) self.word_dict = self._load_dict(self.word_dict_file) self.predicate_dict = self._load_dict(self.verb_dict_file) @@ -169,14 +193,17 @@ class Conll05st(Dataset): def _load_anno(self): tf = tarfile.open(self.data_file) wf = tf.extractfile( - "conll05st-release/test.wsj/words/test.wsj.words.gz") + "conll05st-release/test.wsj/words/test.wsj.words.gz" + ) pf = tf.extractfile( - "conll05st-release/test.wsj/props/test.wsj.props.gz") + "conll05st-release/test.wsj/props/test.wsj.props.gz" + ) self.sentences = [] self.predicates = [] self.labels = [] with gzip.GzipFile(fileobj=wf) as words_file, gzip.GzipFile( - fileobj=pf) as props_file: + fileobj=pf + ) as props_file: sentences = [] labels = [] one_seg = [] @@ -209,16 +236,17 @@ class Conll05st(Dataset): lbl_seq.append('I-' + cur_tag) is_in_bracket = False elif l.find('(') != -1 and l.find(')') != -1: - cur_tag = l[1:l.find('*')] + cur_tag = l[1 : l.find('*')] lbl_seq.append('B-' + cur_tag) is_in_bracket = False elif l.find('(') != -1 and l.find(')') == -1: - cur_tag = l[1:l.find('*')] + cur_tag = l[1 : l.find('*')] lbl_seq.append('B-' + cur_tag) is_in_bracket = True else: - raise RuntimeError('Unexpected label: %s' % - l) + raise RuntimeError( + 'Unexpected label: %s' % l + ) self.sentences.append(sentences) self.predicates.append(verb_list[i]) @@ -282,9 +310,17 @@ class Conll05st(Dataset): pred_idx = [self.predicate_dict.get(predicate)] * sen_len label_idx = [self.label_dict.get(w) for w in labels] - return (np.array(word_idx), np.array(ctx_n2_idx), np.array(ctx_n1_idx), - np.array(ctx_0_idx), np.array(ctx_p1_idx), np.array(ctx_p2_idx), - np.array(pred_idx), np.array(mark), np.array(label_idx)) + return ( + np.array(word_idx), + np.array(ctx_n2_idx), + np.array(ctx_n1_idx), + np.array(ctx_0_idx), + np.array(ctx_p1_idx), + np.array(ctx_p2_idx), + np.array(pred_idx), + np.array(mark), + np.array(label_idx), + ) def __len__(self): return len(self.sentences) diff --git a/python/paddle/text/datasets/imdb.py b/python/paddle/text/datasets/imdb.py index 80b650574b4..d71d23ff692 100644 --- a/python/paddle/text/datasets/imdb.py +++ b/python/paddle/text/datasets/imdb.py @@ -71,15 +71,20 @@ class Imdb(Dataset): """ def __init__(self, data_file=None, mode='train', cutoff=150, download=True): - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train', 'test', but got {}".format(mode) + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train', 'test', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, URL, MD5, - 'imdb', download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, URL, MD5, 'imdb', download + ) # Build a word dictionary from the corpus self.word_idx = self._build_work_dict(cutoff) @@ -111,9 +116,13 @@ class Imdb(Dataset): if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. data.append( - tarf.extractfile(tf).read().rstrip(b'\n\r').translate( - None, string.punctuation.encode( - 'latin-1')).lower().split()) + tarf.extractfile(tf) + .read() + .rstrip(b'\n\r') + .translate(None, string.punctuation.encode('latin-1')) + .lower() + .split() + ) tf = tarf.next() return data diff --git a/python/paddle/text/datasets/imikolov.py b/python/paddle/text/datasets/imikolov.py index 03d6560e9bc..d1081f4dda6 100644 --- a/python/paddle/text/datasets/imikolov.py +++ b/python/paddle/text/datasets/imikolov.py @@ -70,19 +70,25 @@ class Imikolov(Dataset): """ - def __init__(self, - data_file=None, - data_type='NGRAM', - window_size=-1, - mode='train', - min_word_freq=50, - download=True): - assert data_type.upper() in ['NGRAM', 'SEQ'], \ - "data type should be 'NGRAM', 'SEQ', but got {}".format(data_type) + def __init__( + self, + data_file=None, + data_type='NGRAM', + window_size=-1, + mode='train', + min_word_freq=50, + download=True, + ): + assert data_type.upper() in [ + 'NGRAM', + 'SEQ', + ], "data type should be 'NGRAM', 'SEQ', but got {}".format(data_type) self.data_type = data_type.upper() - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train', 'test', but got {}".format(mode) + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train', 'test', but got {}".format(mode) self.mode = mode.lower() self.window_size = window_size @@ -90,9 +96,12 @@ class Imikolov(Dataset): self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically disabled" - self.data_file = _check_exists_and_download(data_file, URL, MD5, - 'imikolov', download) + assert ( + download + ), "data_file is not set and downloading automatically disabled" + self.data_file = _check_exists_and_download( + data_file, URL, MD5, 'imikolov', download + ) # Build a word dictionary from the corpus self.word_idx = self._build_work_dict(min_word_freq) @@ -148,7 +157,7 @@ class Imikolov(Dataset): if len(l) >= self.window_size: l = [self.word_idx.get(w, UNK) for w in l] for i in range(self.window_size, len(l) + 1): - self.data.append(tuple(l[i - self.window_size:i])) + self.data.append(tuple(l[i - self.window_size : i])) elif self.data_type == 'SEQ': l = l.strip().split() l = [self.word_idx.get(w, UNK) for w in l] diff --git a/python/paddle/text/datasets/movielens.py b/python/paddle/text/datasets/movielens.py index f735d22f91f..cf23eae1d84 100644 --- a/python/paddle/text/datasets/movielens.py +++ b/python/paddle/text/datasets/movielens.py @@ -41,12 +41,18 @@ class MovieInfo(object): """ Get information from a movie. """ - return [[self.index], [categories_dict[c] for c in self.categories], - [movie_title_dict[w.lower()] for w in self.title.split()]] + return [ + [self.index], + [categories_dict[c] for c in self.categories], + [movie_title_dict[w.lower()] for w in self.title.split()], + ] def __str__(self): return "" % ( - self.index, self.title, self.categories) + self.index, + self.title, + self.categories, + ) def __repr__(self): return self.__str__() @@ -67,13 +73,20 @@ class UserInfo(object): """ Get information from a user. """ - return [[self.index], [0 if self.is_male else 1], [self.age], - [self.job_id]] + return [ + [self.index], + [0 if self.is_male else 1], + [self.age], + [self.job_id], + ] def __str__(self): return "" % ( - self.index, "M" if self.is_male else "F", age_table[self.age], - self.job_id) + self.index, + "M" if self.is_male else "F", + age_table[self.age], + self.job_id, + ) def __repr__(self): return str(self) @@ -124,21 +137,28 @@ class Movielens(Dataset): """ - def __init__(self, - data_file=None, - mode='train', - test_ratio=0.1, - rand_seed=0, - download=True): - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train', 'test', but got {}".format(mode) + def __init__( + self, + data_file=None, + mode='train', + test_ratio=0.1, + rand_seed=0, + download=True, + ): + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train', 'test', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, URL, MD5, - 'sentiment', download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, URL, MD5, 'sentiment', download + ) self.test_ratio = test_ratio self.rand_seed = rand_seed @@ -167,7 +187,8 @@ class Movielens(Dataset): categories_set.add(c) title = pattern.match(title).group(1) self.movie_info[int(movie_id)] = MovieInfo( - index=movie_id, categories=categories, title=title) + index=movie_id, categories=categories, title=title + ) for w in title.split(): title_word_set.add(w.lower()) @@ -181,10 +202,9 @@ class Movielens(Dataset): for line in user_file: line = line.decode(encoding='latin') uid, gender, age, job, _ = line.strip().split("::") - self.user_info[int(uid)] = UserInfo(index=uid, - gender=gender, - age=age, - job_id=job) + self.user_info[int(uid)] = UserInfo( + index=uid, gender=gender, age=age, job_id=job + ) def _load_data(self): self.data = [] @@ -201,9 +221,13 @@ class Movielens(Dataset): mov = self.movie_info[mov_id] usr = self.user_info[uid] - self.data.append(usr.value() + \ - mov.value(self.categories_dict, self.movie_title_dict) + \ - [[rating]]) + self.data.append( + usr.value() + + mov.value( + self.categories_dict, self.movie_title_dict + ) + + [[rating]] + ) def __getitem__(self, idx): data = self.data[idx] diff --git a/python/paddle/text/datasets/uci_housing.py b/python/paddle/text/datasets/uci_housing.py index 0bf677e271a..bf43332dd39 100644 --- a/python/paddle/text/datasets/uci_housing.py +++ b/python/paddle/text/datasets/uci_housing.py @@ -23,8 +23,19 @@ __all__ = [] URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data' MD5 = 'd4accdce7a25600298819f8e28e8d593' feature_names = [ - 'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', - 'PTRATIO', 'B', 'LSTAT' + 'CRIM', + 'ZN', + 'INDUS', + 'CHAS', + 'NOX', + 'RM', + 'AGE', + 'DIS', + 'RAD', + 'TAX', + 'PTRATIO', + 'B', + 'LSTAT', ] @@ -73,15 +84,20 @@ class UCIHousing(Dataset): """ def __init__(self, data_file=None, mode='train', download=True): - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train' or 'test', but got {}".format(mode) + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train' or 'test', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, URL, MD5, - 'uci_housing', download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, URL, MD5, 'uci_housing', download + ) # read dataset into memory self._load_data() @@ -91,8 +107,11 @@ class UCIHousing(Dataset): def _load_data(self, feature_num=14, ratio=0.8): data = np.fromfile(self.data_file, sep=' ') data = data.reshape(data.shape[0] // feature_num, feature_num) - maximums, minimums, avgs = data.max(axis=0), data.min( - axis=0), data.sum(axis=0) / data.shape[0] + maximums, minimums, avgs = ( + data.max(axis=0), + data.min(axis=0), + data.sum(axis=0) / data.shape[0], + ) for i in range(feature_num - 1): data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i]) offset = int(data.shape[0] * ratio) @@ -103,8 +122,9 @@ class UCIHousing(Dataset): def __getitem__(self, idx): data = self.data[idx] - return np.array(data[:-1]).astype(self.dtype), \ - np.array(data[-1:]).astype(self.dtype) + return np.array(data[:-1]).astype(self.dtype), np.array( + data[-1:] + ).astype(self.dtype) def __len__(self): return len(self.data) diff --git a/python/paddle/text/datasets/wmt14.py b/python/paddle/text/datasets/wmt14.py index 33c92c875e8..71f5c7af0f4 100644 --- a/python/paddle/text/datasets/wmt14.py +++ b/python/paddle/text/datasets/wmt14.py @@ -20,12 +20,14 @@ from paddle.dataset.common import _check_exists_and_download __all__ = [] -URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/' - 'cslm_joint_paper/data/dev+test.tgz') +URL_DEV_TEST = ( + 'http://www-lium.univ-lemans.fr/~schwenk/' + 'cslm_joint_paper/data/dev+test.tgz' +) MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' # this is a small set of data for test. The original data is too large and # will be add later. -URL_TRAIN = ('http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz') +URL_TRAIN = 'http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz' MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c' START = "" @@ -82,21 +84,24 @@ class WMT14(Dataset): """ - def __init__(self, - data_file=None, - mode='train', - dict_size=-1, - download=True): - assert mode.lower() in ['train', 'test', 'gen'], \ - "mode should be 'train', 'test' or 'gen', but got {}".format(mode) + def __init__( + self, data_file=None, mode='train', dict_size=-1, download=True + ): + assert mode.lower() in [ + 'train', + 'test', + 'gen', + ], "mode should be 'train', 'test' or 'gen', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, URL_TRAIN, - MD5_TRAIN, 'wmt14', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, URL_TRAIN, MD5_TRAIN, 'wmt14', download + ) # read dataset into memory assert dict_size > 0, "dict_size should be set as positive number" @@ -104,7 +109,6 @@ class WMT14(Dataset): self._load_data() def _load_data(self): - def __to_dict(fd, size): out_dict = dict() for line_count, line in enumerate(fd): @@ -119,13 +123,15 @@ class WMT14(Dataset): self.trg_ids_next = [] with tarfile.open(self.data_file, mode='r') as f: names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith("src.dict") ] assert len(names) == 1 self.src_dict = __to_dict(f.extractfile(names[0]), self.dict_size) names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith("trg.dict") ] assert len(names) == 1 @@ -133,7 +139,8 @@ class WMT14(Dataset): file_name = "{}/{}".format(self.mode, self.mode) names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith(file_name) ] for name in names: @@ -164,8 +171,11 @@ class WMT14(Dataset): self.trg_ids_next.append(trg_ids_next) def __getitem__(self, idx): - return (np.array(self.src_ids[idx]), np.array(self.trg_ids[idx]), - np.array(self.trg_ids_next[idx])) + return ( + np.array(self.src_ids[idx]), + np.array(self.trg_ids[idx]), + np.array(self.trg_ids_next[idx]), + ) def __len__(self): return len(self.src_ids) diff --git a/python/paddle/text/datasets/wmt16.py b/python/paddle/text/datasets/wmt16.py index 4538f4b8a6a..3048160acff 100644 --- a/python/paddle/text/datasets/wmt16.py +++ b/python/paddle/text/datasets/wmt16.py @@ -25,7 +25,7 @@ from paddle.dataset.common import _check_exists_and_download __all__ = [] -DATA_URL = ("http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz") +DATA_URL = "http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz" DATA_MD5 = "0c38be43600334966403524a40dcd81e" TOTAL_EN_WORDS = 11250 @@ -102,43 +102,55 @@ class WMT16(Dataset): """ - def __init__(self, - data_file=None, - mode='train', - src_dict_size=-1, - trg_dict_size=-1, - lang='en', - download=True): - assert mode.lower() in ['train', 'test', 'val'], \ - "mode should be 'train', 'test' or 'val', but got {}".format(mode) + def __init__( + self, + data_file=None, + mode='train', + src_dict_size=-1, + trg_dict_size=-1, + lang='en', + download=True, + ): + assert mode.lower() in [ + 'train', + 'test', + 'val', + ], "mode should be 'train', 'test' or 'val', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, DATA_URL, - DATA_MD5, 'wmt16', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, DATA_URL, DATA_MD5, 'wmt16', download + ) self.lang = lang assert src_dict_size > 0, "dict_size should be set as positive number" assert trg_dict_size > 0, "dict_size should be set as positive number" self.src_dict_size = min( - src_dict_size, (TOTAL_EN_WORDS if lang == "en" else TOTAL_DE_WORDS)) + src_dict_size, (TOTAL_EN_WORDS if lang == "en" else TOTAL_DE_WORDS) + ) self.trg_dict_size = min( - trg_dict_size, (TOTAL_DE_WORDS if lang == "en" else TOTAL_EN_WORDS)) + trg_dict_size, (TOTAL_DE_WORDS if lang == "en" else TOTAL_EN_WORDS) + ) # load source and target word dict self.src_dict = self._load_dict(lang, src_dict_size) - self.trg_dict = self._load_dict("de" if lang == "en" else "en", - trg_dict_size) + self.trg_dict = self._load_dict( + "de" if lang == "en" else "en", trg_dict_size + ) # load data self.data = self._load_data() def _load_dict(self, lang, dict_size, reverse=False): - dict_path = os.path.join(paddle.dataset.common.DATA_HOME, - "wmt16/%s_%d.dict" % (lang, dict_size)) + dict_path = os.path.join( + paddle.dataset.common.DATA_HOME, + "wmt16/%s_%d.dict" % (lang, dict_size), + ) dict_found = False if os.path.exists(dict_path): with open(dict_path, "rb") as d: @@ -161,18 +173,21 @@ class WMT16(Dataset): for line in f.extractfile("wmt16/train"): line = line.decode() line_split = line.strip().split("\t") - if len(line_split) != 2: continue + if len(line_split) != 2: + continue sen = line_split[0] if self.lang == "en" else line_split[1] for w in sen.split(): word_dict[w] += 1 with open(dict_path, "wb") as fout: fout.write( - ("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)).encode()) + ("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)).encode() + ) for idx, word in enumerate( - sorted(word_dict.items(), key=lambda x: x[1], - reverse=True)): - if idx + 3 == dict_size: break + sorted(word_dict.items(), key=lambda x: x[1], reverse=True) + ): + if idx + 3 == dict_size: + break fout.write(word[0].encode()) fout.write(b'\n') @@ -197,9 +212,11 @@ class WMT16(Dataset): if len(line_split) != 2: continue src_words = line_split[src_col].split() - src_ids = [start_id] + [ - self.src_dict.get(w, unk_id) for w in src_words - ] + [end_id] + src_ids = ( + [start_id] + + [self.src_dict.get(w, unk_id) for w in src_words] + + [end_id] + ) trg_words = line_split[trg_col].split() trg_ids = [self.trg_dict.get(w, unk_id) for w in trg_words] @@ -212,8 +229,11 @@ class WMT16(Dataset): self.trg_ids_next.append(trg_ids_next) def __getitem__(self, idx): - return (np.array(self.src_ids[idx]), np.array(self.trg_ids[idx]), - np.array(self.trg_ids_next[idx])) + return ( + np.array(self.src_ids[idx]), + np.array(self.trg_ids[idx]), + np.array(self.trg_ids_next[idx]), + ) def __len__(self): return len(self.src_ids) @@ -243,10 +263,14 @@ class WMT16(Dataset): en_dict = wmt16.get_dict('en') """ - dict_size = self.src_dict_size if lang == self.lang else self.trg_dict_size - - dict_path = os.path.join(paddle.dataset.common.DATA_HOME, - "wmt16/%s_%d.dict" % (lang, dict_size)) + dict_size = ( + self.src_dict_size if lang == self.lang else self.trg_dict_size + ) + + dict_path = os.path.join( + paddle.dataset.common.DATA_HOME, + "wmt16/%s_%d.dict" % (lang, dict_size), + ) assert os.path.exists(dict_path), "Word dictionary does not exist. " "Please invoke paddle.dataset.wmt16.train/test/validation first " "to build the dictionary." diff --git a/python/paddle/text/viterbi_decode.py b/python/paddle/text/viterbi_decode.py index a2a92da9472..07d4515cab8 100644 --- a/python/paddle/text/viterbi_decode.py +++ b/python/paddle/text/viterbi_decode.py @@ -21,11 +21,9 @@ from paddle import _C_ops, _legacy_C_ops __all__ = ['viterbi_decode', 'ViterbiDecoder'] -def viterbi_decode(potentials, - transition_params, - lengths, - include_bos_eos_tag=True, - name=None): +def viterbi_decode( + potentials, transition_params, lengths, include_bos_eos_tag=True, name=None +): """ Decode the highest scoring sequence of tags computed by transitions and potentials and get the viterbi path. @@ -59,34 +57,43 @@ def viterbi_decode(potentials, scores, path = paddle.text.viterbi_decode(emission, transition, length, False) # scores: [3.37089300, 1.56825531], path: [[1, 0, 0], [1, 1, 0]] """ if in_dygraph_mode(): - return _C_ops.viterbi_decode(potentials, transition_params, lengths, - include_bos_eos_tag) + return _C_ops.viterbi_decode( + potentials, transition_params, lengths, include_bos_eos_tag + ) if _non_static_mode(): - return _legacy_C_ops.viterbi_decode(potentials, transition_params, - lengths, 'include_bos_eos_tag', - include_bos_eos_tag) - check_variable_and_dtype(potentials, 'input', ['float32', 'float64'], - 'viterbi_decode') - check_variable_and_dtype(transition_params, 'transitions', - ['float32', 'float64'], 'viterbi_decode') + return _legacy_C_ops.viterbi_decode( + potentials, + transition_params, + lengths, + 'include_bos_eos_tag', + include_bos_eos_tag, + ) + check_variable_and_dtype( + potentials, 'input', ['float32', 'float64'], 'viterbi_decode' + ) + check_variable_and_dtype( + transition_params, + 'transitions', + ['float32', 'float64'], + 'viterbi_decode', + ) check_variable_and_dtype(lengths, 'length', 'int64', 'viterbi_decode') check_type(include_bos_eos_tag, 'include_tag', bool, 'viterbi_decode') helper = LayerHelper('viterbi_decode', **locals()) attrs = {'include_bos_eos_tag': include_bos_eos_tag} scores = helper.create_variable_for_type_inference(potentials.dtype) path = helper.create_variable_for_type_inference('int64') - helper.append_op(type='viterbi_decode', - inputs={ - 'Input': potentials, - 'Transition': transition_params, - 'Length': lengths - }, - outputs={ - 'Scores': scores, - 'Path': path - }, - attrs=attrs) + helper.append_op( + type='viterbi_decode', + inputs={ + 'Input': potentials, + 'Transition': transition_params, + 'Length': lengths, + }, + outputs={'Scores': scores, 'Path': path}, + attrs=attrs, + ) return scores, path @@ -134,5 +141,10 @@ class ViterbiDecoder(Layer): self.name = name def forward(self, potentials, lengths): - return viterbi_decode(potentials, self.transitions, lengths, - self.include_bos_eos_tag, self.name) + return viterbi_decode( + potentials, + self.transitions, + lengths, + self.include_bos_eos_tag, + self.name, + ) diff --git a/python/paddle/utils/__init__.py b/python/paddle/utils/__init__.py index 6994fd139b6..b51bf4b3753 100644 --- a/python/paddle/utils/__init__.py +++ b/python/paddle/utils/__init__.py @@ -28,6 +28,4 @@ from . import image_util # noqa: F401 from . import cpp_extension # noqa: F401 from . import dlpack -__all__ = [ #noqa - 'deprecated', 'run_check', 'require_version', 'try_import' -] +__all__ = ['deprecated', 'run_check', 'require_version', 'try_import'] # noqa diff --git a/python/paddle/utils/cpp_extension/__init__.py b/python/paddle/utils/cpp_extension/__init__.py index 843f78d5c80..9ad431f00a6 100644 --- a/python/paddle/utils/cpp_extension/__init__.py +++ b/python/paddle/utils/cpp_extension/__init__.py @@ -22,6 +22,10 @@ from .extension_utils import parse_op_info # noqa: F401 from .extension_utils import get_build_directory # noqa: F401 from .extension_utils import load_op_meta_info_and_register_op # noqa: F401 -__all__ = [ #noqa - 'CppExtension', 'CUDAExtension', 'load', 'setup', 'get_build_directory' +__all__ = [ # noqa + 'CppExtension', + 'CUDAExtension', + 'load', + 'setup', + 'get_build_directory', ] diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index 5ff1d1cc292..9dfa2f103da 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -21,14 +21,41 @@ from setuptools.command.easy_install import easy_install from setuptools.command.build_ext import build_ext from distutils.command.build import build -from .extension_utils import add_compile_flag, find_cuda_home, find_rocm_home, normalize_extension_kwargs -from .extension_utils import is_cuda_file, prepare_unix_cudaflags, prepare_win_cudaflags -from .extension_utils import _import_module_from_library, _write_setup_file, _jit_compile -from .extension_utils import check_abi_compatibility, log_v, CustomOpInfo, parse_op_name_from +from .extension_utils import ( + add_compile_flag, + find_cuda_home, + find_rocm_home, + normalize_extension_kwargs, +) +from .extension_utils import ( + is_cuda_file, + prepare_unix_cudaflags, + prepare_win_cudaflags, +) +from .extension_utils import ( + _import_module_from_library, + _write_setup_file, + _jit_compile, +) +from .extension_utils import ( + check_abi_compatibility, + log_v, + CustomOpInfo, + parse_op_name_from, +) from .extension_utils import _reset_so_rpath, clean_object_if_change_cflags -from .extension_utils import bootstrap_context, get_build_directory, add_std_without_repeat - -from .extension_utils import IS_WINDOWS, OS_NAME, MSVC_COMPILE_FLAGS, MSVC_COMPILE_FLAGS +from .extension_utils import ( + bootstrap_context, + get_build_directory, + add_std_without_repeat, +) + +from .extension_utils import ( + IS_WINDOWS, + OS_NAME, + MSVC_COMPILE_FLAGS, + MSVC_COMPILE_FLAGS, +) from .extension_utils import CLANG_COMPILE_FLAGS, CLANG_LINK_FLAGS from ...fluid import core @@ -39,6 +66,7 @@ from ...fluid import core if IS_WINDOWS: from distutils.command.build_ext import build_ext as _du_build_ext from unittest.mock import Mock + _du_build_ext.get_export_symbols = Mock(return_value=None) CUDA_HOME = find_cuda_home() @@ -147,7 +175,8 @@ def setup(**attr): # if not specific cmdclass in setup, add it automatically. if 'build_ext' not in cmdclass: cmdclass['build_ext'] = BuildExtension.with_options( - no_python_abi_suffix=True) + no_python_abi_suffix=True + ) attr['cmdclass'] = cmdclass error_msg = """ @@ -167,17 +196,19 @@ def setup(**attr): if 'name' not in attr: raise ValueError(error_msg) - assert not attr['name'].endswith('module'), \ - "Please don't use 'module' as suffix in `name` argument, " + assert not attr['name'].endswith( + 'module' + ), "Please don't use 'module' as suffix in `name` argument, " "it will be stripped in setuptools.bdist_egg and cause import error." ext_modules = attr.get('ext_modules', []) if not isinstance(ext_modules, list): ext_modules = [ext_modules] - assert len( - ext_modules - ) == 1, "Required only one Extension, but received {}. If you want to compile multi operators, you can include all necessary source files in one Extension.".format( - len(ext_modules)) + assert ( + len(ext_modules) == 1 + ), "Required only one Extension, but received {}. If you want to compile multi operators, you can include all necessary source files in one Extension.".format( + len(ext_modules) + ) # replace Extension.name with attr['name] to keep consistant with Package name. for ext_module in ext_modules: ext_module.name = attr['name'] @@ -335,7 +366,6 @@ class BuildExtension(build_ext, object): """ class cls_with_options(cls): - def __init__(self, *args, **kwargs): kwargs.update(options) cls.__init__(self, *args, **kwargs) @@ -380,8 +410,9 @@ class BuildExtension(build_ext, object): # cflags have changed and delete the built shared library to re-compile the source # even though source file content keep unchanged. so_name = self.get_ext_fullpath(self.extensions[0].name) - clean_object_if_change_cflags(os.path.abspath(so_name), - self.extensions[0]) + clean_object_if_change_cflags( + os.path.abspath(so_name), self.extensions[0] + ) # Consider .cu, .cu.cc as valid source extensions. self.compiler.src_extensions += ['.cu', '.cu.cc'] @@ -393,8 +424,9 @@ class BuildExtension(build_ext, object): else: original_compile = self.compiler._compile - def unix_custom_single_compiler(obj, src, ext, cc_args, extra_postargs, - pp_opts): + def unix_custom_single_compiler( + obj, src, ext, cc_args, extra_postargs, pp_opts + ): """ Monkey patch machanism to replace inner compiler to custom complie process on Unix platform. """ @@ -407,7 +439,9 @@ class BuildExtension(build_ext, object): # nvcc or hipcc compile CUDA source if is_cuda_file(src): if core.is_compiled_with_rocm(): - assert ROCM_HOME is not None, "Not found ROCM runtime, \ + assert ( + ROCM_HOME is not None + ), "Not found ROCM runtime, \ please use `export ROCM_PATH= XXX` to specify it." hipcc_cmd = os.path.join(ROCM_HOME, 'bin', 'hipcc') @@ -416,7 +450,9 @@ class BuildExtension(build_ext, object): if isinstance(cflags, dict): cflags = cflags['hipcc'] else: - assert CUDA_HOME is not None, "Not found CUDA runtime, \ + assert ( + CUDA_HOME is not None + ), "Not found CUDA runtime, \ please use `export CUDA_HOME= XXX` to specify it." nvcc_cmd = os.path.join(CUDA_HOME, 'bin', 'nvcc') @@ -435,7 +471,8 @@ class BuildExtension(build_ext, object): cflags.append('-D__HIP_PLATFORM_HCC__') cflags.append('-D__HIP_NO_HALF_CONVERSIONS__=1') cflags.append( - '-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP') + '-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP' + ) # NOTE(Aurelius84): Since Paddle 2.0, we require gcc version > 5.x, # so we add this flag to ensure the symbol names from user compiled @@ -449,22 +486,24 @@ class BuildExtension(build_ext, object): else: cflags.append('-DPADDLE_WITH_CUDA') - add_std_without_repeat(cflags, - self.compiler.compiler_type, - use_std14=True) + add_std_without_repeat( + cflags, self.compiler.compiler_type, use_std14=True + ) original_compile(obj, src, ext, cc_args, cflags, pp_opts) finally: # restore original_compiler self.compiler.set_executable('compiler_so', original_compiler) - def win_custom_single_compiler(sources, - output_dir=None, - macros=None, - include_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - depends=None): + def win_custom_single_compiler( + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None, + ): self.cflags = copy.deepcopy(extra_postargs) extra_postargs = None @@ -481,27 +520,32 @@ class BuildExtension(build_ext, object): # Using regex to match src, obj and include files src_regex = re.compile('/T(p|c)(.*)') src_list = [ - m.group(2) for m in (src_regex.match(elem) for elem in cmd) + m.group(2) + for m in (src_regex.match(elem) for elem in cmd) if m ] obj_regex = re.compile('/Fo(.*)') obj_list = [ - m.group(1) for m in (obj_regex.match(elem) for elem in cmd) + m.group(1) + for m in (obj_regex.match(elem) for elem in cmd) if m ] include_regex = re.compile(r'((\-|\/)I.*)') include_list = [ m.group(1) - for m in (include_regex.match(elem) for elem in cmd) if m + for m in (include_regex.match(elem) for elem in cmd) + if m ] assert len(src_list) == 1 and len(obj_list) == 1 src = src_list[0] obj = obj_list[0] if is_cuda_file(src): - assert CUDA_HOME is not None, "Not found CUDA runtime, \ + assert ( + CUDA_HOME is not None + ), "Not found CUDA runtime, \ please use `export CUDA_HOME= XXX` to specify it." nvcc_cmd = os.path.join(CUDA_HOME, 'bin', 'nvcc') @@ -515,8 +559,9 @@ class BuildExtension(build_ext, object): cflags = prepare_win_cudaflags(cflags) + ['--use-local-env'] for flag in MSVC_COMPILE_FLAGS: cflags = ['-Xcompiler', flag] + cflags - cmd = [nvcc_cmd, '-c', src, '-o', obj - ] + include_list + cflags + cmd = ( + [nvcc_cmd, '-c', src, '-o', obj] + include_list + cflags + ) elif isinstance(self.cflags, dict): cflags = MSVC_COMPILE_FLAGS + self.cflags['cxx'] cmd += cflags @@ -531,9 +576,16 @@ class BuildExtension(build_ext, object): try: self.compiler.spawn = win_custom_spawn - return original_compile(sources, output_dir, macros, - include_dirs, debug, extra_preargs, - extra_postargs, depends) + return original_compile( + sources, + output_dir, + macros, + include_dirs, + debug, + extra_preargs, + extra_postargs, + depends, + ) finally: self.compiler.spawn = original_spawn @@ -546,8 +598,9 @@ class BuildExtension(build_ext, object): def wrapper(source_filenames, strip_dir=0, output_dir=''): try: - objects = origina_func(source_filenames, strip_dir, - output_dir) + objects = origina_func( + source_filenames, strip_dir, output_dir + ) for i, source in enumerate(source_filenames): # modify xx.o -> xx.cu.o/xx.cu.obj if is_cuda_file(source): @@ -578,7 +631,8 @@ class BuildExtension(build_ext, object): self.compiler._compile = unix_custom_single_compiler self.compiler.object_filenames = object_filenames_with_cuda( - self.compiler.object_filenames, self.build_lib) + self.compiler.object_filenames, self.build_lib + ) self._record_op_info() print("Compiling user custom op, it will cost a few seconds.....") @@ -594,10 +648,11 @@ class BuildExtension(build_ext, object): split_str = '.' name_items = ext_name.split(split_str) if self.no_python_abi_suffix: - assert len( - name_items - ) > 2, "Expected len(name_items) > 2, but received {}".format( - len(name_items)) + assert ( + len(name_items) > 2 + ), "Expected len(name_items) > 2, but received {}".format( + len(name_items) + ) name_items.pop(-2) ext_name = split_str.join(name_items) @@ -613,11 +668,13 @@ class BuildExtension(build_ext, object): """ compiler_infos = ['clang'] + CLANG_COMPILE_FLAGS linker_infos = ['clang'] + CLANG_LINK_FLAGS - self.compiler.set_executables(compiler=compiler_infos, - compiler_so=compiler_infos, - compiler_cxx=['clang'], - linker_exe=['clang'], - linker_so=linker_infos) + self.compiler.set_executables( + compiler=compiler_infos, + compiler_so=compiler_infos, + compiler_cxx=['clang'], + linker_exe=['clang'], + linker_so=linker_infos, + ) def _check_abi(self): """ @@ -632,11 +689,16 @@ class BuildExtension(build_ext, object): check_abi_compatibility(compiler) # Warn user if VC env is activated but `DISTUTILS_USE_SDK` is not set. - if IS_WINDOWS and 'VSCMD_ARG_TGT_ARCH' in os.environ and 'DISTUTILS_USE_SDK' not in os.environ: + if ( + IS_WINDOWS + and 'VSCMD_ARG_TGT_ARCH' in os.environ + and 'DISTUTILS_USE_SDK' not in os.environ + ): msg = ( 'It seems that the VC environment is activated but DISTUTILS_USE_SDK is not set.' 'This may lead to multiple activations of the VC env.' - 'Please run `set DISTUTILS_USE_SDK=1` and try again.') + 'Please run `set DISTUTILS_USE_SDK=1` and try again.' + ) raise UserWarning(msg) def _record_op_info(self): @@ -657,9 +719,9 @@ class BuildExtension(build_ext, object): op_names = parse_op_name_from(sources) for op_name in op_names: - CustomOpInfo.instance().add(op_name, - so_name=so_name, - so_path=so_path) + CustomOpInfo.instance().add( + op_name, so_name=so_name, so_path=so_path + ) class EasyInstallCommand(easy_install, object): @@ -712,7 +774,6 @@ class BuildCommand(build, object): """ class cls_with_options(cls): - def __init__(self, *args, **kwargs): kwargs.update(options) cls.__init__(self, *args, **kwargs) @@ -735,14 +796,16 @@ class BuildCommand(build, object): self.build_base = self._specified_build_base -def load(name, - sources, - extra_cxx_cflags=None, - extra_cuda_cflags=None, - extra_ldflags=None, - extra_include_paths=None, - build_directory=None, - verbose=False): +def load( + name, + sources, + extra_cxx_cflags=None, + extra_cuda_cflags=None, + extra_ldflags=None, + extra_include_paths=None, + build_directory=None, + verbose=False, +): """ An Interface to automatically compile C++/CUDA source files Just-In-Time and return callable python function as other Paddle layers API. It will @@ -836,27 +899,42 @@ def load(name, file_path = os.path.join(build_directory, "{}_setup.py".format(name)) sources = [os.path.abspath(source) for source in sources] - if extra_cxx_cflags is None: extra_cxx_cflags = [] - if extra_cuda_cflags is None: extra_cuda_cflags = [] + if extra_cxx_cflags is None: + extra_cxx_cflags = [] + if extra_cuda_cflags is None: + extra_cuda_cflags = [] assert isinstance( extra_cxx_cflags, list ), "Required type(extra_cxx_cflags) == list[str], but received {}".format( - extra_cxx_cflags) + extra_cxx_cflags + ) assert isinstance( extra_cuda_cflags, list ), "Required type(extra_cuda_cflags) == list[str], but received {}".format( - extra_cuda_cflags) + extra_cuda_cflags + ) log_v( "additional extra_cxx_cflags: [{}], extra_cuda_cflags: [{}]".format( - ' '.join(extra_cxx_cflags), ' '.join(extra_cuda_cflags)), verbose) + ' '.join(extra_cxx_cflags), ' '.join(extra_cuda_cflags) + ), + verbose, + ) # write setup.py file and compile it build_base_dir = os.path.join(build_directory, name) - _write_setup_file(name, sources, file_path, build_base_dir, - extra_include_paths, extra_cxx_cflags, extra_cuda_cflags, - extra_ldflags, verbose) + _write_setup_file( + name, + sources, + file_path, + build_base_dir, + extra_include_paths, + extra_cxx_cflags, + extra_cuda_cflags, + extra_ldflags, + verbose, + ) _jit_compile(file_path, verbose) # import as callable python api diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index 8b07abe7c85..57a1ae1f0f5 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -50,23 +50,46 @@ OS_NAME = sys.platform IS_WINDOWS = OS_NAME.startswith('win') MSVC_COMPILE_FLAGS = [ - '/MT', '/wd4819', '/wd4251', '/wd4244', '/wd4267', '/wd4275', '/wd4018', - '/wd4190', '/EHsc', '/w', '/DGOOGLE_GLOG_DLL_DECL', - '/DBOOST_HAS_STATIC_ASSERT', '/DNDEBUG', '/DPADDLE_USE_DSO' + '/MT', + '/wd4819', + '/wd4251', + '/wd4244', + '/wd4267', + '/wd4275', + '/wd4018', + '/wd4190', + '/EHsc', + '/w', + '/DGOOGLE_GLOG_DLL_DECL', + '/DBOOST_HAS_STATIC_ASSERT', + '/DNDEBUG', + '/DPADDLE_USE_DSO', ] CLANG_COMPILE_FLAGS = [ - '-fno-common', '-dynamic', '-DNDEBUG', '-g', '-fwrapv', '-O3', '-arch', - 'x86_64' + '-fno-common', + '-dynamic', + '-DNDEBUG', + '-g', + '-fwrapv', + '-O3', + '-arch', + 'x86_64', ] CLANG_LINK_FLAGS = [ - '-dynamiclib', '-undefined', 'dynamic_lookup', '-arch', 'x86_64' + '-dynamiclib', + '-undefined', + 'dynamic_lookup', + '-arch', + 'x86_64', ] MSVC_LINK_FLAGS = ['/MACHINE:X64'] if core.is_compiled_with_rocm(): COMMON_HIPCC_FLAGS = [ - '-DPADDLE_WITH_HIP', '-DEIGEN_USE_GPU', '-DEIGEN_USE_HIP' + '-DPADDLE_WITH_HIP', + '-DEIGEN_USE_GPU', + '-DEIGEN_USE_HIP', ] else: COMMON_NVCC_FLAGS = ['-DPADDLE_WITH_CUDA', '-DEIGEN_USE_GPU'] @@ -114,7 +137,7 @@ DEFAULT_OP_ATTR_NAMES = [ core.op_proto_and_checker_maker.kOpNameScopeAttrName(), core.op_proto_and_checker_maker.kOpCreationCallstackAttrName(), core.op_proto_and_checker_maker.kOpDeviceAttrName(), - core.op_proto_and_checker_maker.kOpWithQuantAttrName() + core.op_proto_and_checker_maker.kOpWithQuantAttrName(), ] @@ -140,7 +163,8 @@ def custom_write_stub(resource, pyfile): Customized write_stub function to allow us to inject generated python api codes into egg python file. """ - _stub_template = textwrap.dedent(""" + _stub_template = textwrap.dedent( + """ import os import sys import types @@ -170,17 +194,18 @@ def custom_write_stub(resource, pyfile): {custom_api} - """).lstrip() + """ + ).lstrip() # Parse registerring op information _, op_info = CustomOpInfo.instance().last() so_path = op_info.so_path new_custom_ops = load_op_meta_info_and_register_op(so_path) - assert len( - new_custom_ops - ) > 0, "Required at least one custom operators, but received len(custom_op) = %d" % len( - new_custom_ops) + assert len(new_custom_ops) > 0, ( + "Required at least one custom operators, but received len(custom_op) = %d" + % len(new_custom_ops) + ) # NOTE: To avoid importing .so file instead of python file because they have same name, # we rename .so shared library to another name, see EasyInstallCommand. @@ -193,8 +218,10 @@ def custom_write_stub(resource, pyfile): with open(pyfile, 'w') as f: f.write( - _stub_template.format(resource=resource, - custom_api='\n\n'.join(api_content))) + _stub_template.format( + resource=resource, custom_api='\n\n'.join(api_content) + ) + ) OpInfo = collections.namedtuple('OpInfo', ['so_name', 'so_path']) @@ -213,8 +240,8 @@ class CustomOpInfo: def __init__(self): assert not hasattr( - self.__class__, - '_instance'), 'Please use `instance()` to get CustomOpInfo object!' + self.__class__, '_instance' + ), 'Please use `instance()` to get CustomOpInfo object!' # NOTE(Aurelius84): Use OrderedDict to save more order information self.op_info_map = collections.OrderedDict() @@ -229,20 +256,22 @@ class CustomOpInfo: return next(reversed(self.op_info_map.items())) -VersionFields = collections.namedtuple('VersionFields', [ - 'sources', - 'extra_compile_args', - 'extra_link_args', - 'library_dirs', - 'runtime_library_dirs', - 'include_dirs', - 'define_macros', - 'undef_macros', -]) +VersionFields = collections.namedtuple( + 'VersionFields', + [ + 'sources', + 'extra_compile_args', + 'extra_link_args', + 'library_dirs', + 'runtime_library_dirs', + 'include_dirs', + 'define_macros', + 'undef_macros', + ], +) class VersionManager: - def __init__(self, version_field): self.version_field = version_field self.version = self.hasher(version_field) @@ -253,14 +282,17 @@ class VersionManager: md5 = hashlib.md5() for field in version_field._fields: elem = getattr(version_field, field) - if not elem: continue + if not elem: + continue if isinstance(elem, (list, tuple, dict)): flat_elem = flatten(elem) md5 = combine_hash(md5, tuple(flat_elem)) else: raise RuntimeError( - "Support types with list, tuple and dict, but received {} with {}." - .format(type(elem), elem)) + "Support types with list, tuple and dict, but received {} with {}.".format( + type(elem), elem + ) + ) return md5.hexdigest() @@ -314,8 +346,10 @@ def clean_object_if_change_cflags(so_path, extension): # delete shared library file if version is changed to re-compile it. if so_version is not None and so_version != versioner.version: log_v( - "Re-Compiling {}, because specified cflags have been changed. New signature {} has been saved into {}." - .format(so_name, versioner.version, version_file)) + "Re-Compiling {}, because specified cflags have been changed. New signature {} has been saved into {}.".format( + so_name, versioner.version, version_file + ) + ) os.remove(so_path) # update new version information new_version_info = versioner.details @@ -335,13 +369,26 @@ def prepare_unix_cudaflags(cflags): Prepare all necessary compiled flags for nvcc compiling CUDA files. """ if core.is_compiled_with_rocm(): - cflags = COMMON_HIPCC_FLAGS + ['-Xcompiler', '-fPIC' - ] + cflags + get_rocm_arch_flags(cflags) + cflags = ( + COMMON_HIPCC_FLAGS + + ['-Xcompiler', '-fPIC'] + + cflags + + get_rocm_arch_flags(cflags) + ) else: - cflags = COMMON_NVCC_FLAGS + [ - '-ccbin', 'cc', '-Xcompiler', '-fPIC', '--expt-relaxed-constexpr', - '-DNVCC' - ] + cflags + get_cuda_arch_flags(cflags) + cflags = ( + COMMON_NVCC_FLAGS + + [ + '-ccbin', + 'cc', + '-Xcompiler', + '-fPIC', + '--expt-relaxed-constexpr', + '-DNVCC', + ] + + cflags + + get_cuda_arch_flags(cflags) + ) return cflags @@ -390,6 +437,7 @@ def _get_fluid_path(): Return installed fluid dir path. """ import paddle + return os.path.join(os.path.dirname(paddle.__file__), 'fluid') @@ -434,7 +482,8 @@ def _reset_so_rpath(so_path): origin_runtime_path = "@loader_path/../libs/" rpath = "@rpath/{}".format(_get_core_name()) cmd = 'install_name_tool -change {} {} {}'.format( - origin_runtime_path, rpath, so_path) + origin_runtime_path, rpath, so_path + ) run_cmd(cmd) @@ -448,7 +497,8 @@ def _get_include_dirs_when_compiling(compile_dir): path = os.path.abspath(compile_dir) include_dirs_file = os.path.join(path, include_dirs_file) assert os.path.isfile(include_dirs_file), "File {} does not exist".format( - include_dirs_file) + include_dirs_file + ) with open(include_dirs_file, 'r') as f: include_dirs = [line.strip() for line in f.readlines() if line.strip()] @@ -566,9 +616,13 @@ def create_sym_link_if_not_exist(): except Exception: warnings.warn( "Failed to create soft symbol link for {}.\n You can run prompt as administrator and execute the " - "following command manually: `mklink {} {}`. Now it will create hard link for {} trickly." - .format(raw_core_name, new_dll_core_path, core_path, - raw_core_name)) + "following command manually: `mklink {} {}`. Now it will create hard link for {} trickly.".format( + raw_core_name, + new_dll_core_path, + core_path, + raw_core_name, + ) + ) run_cmd('mklink /H {} {}'.format(new_dll_core_path, core_path)) # libpaddle with lib suffix assert os.path.exists(new_dll_core_path) @@ -583,8 +637,10 @@ def create_sym_link_if_not_exist(): assert os.path.exists(new_lib_core_path) except Exception: raise RuntimeError( - "Failed to create soft symbol link for {}.\n Please execute the following command manually: `ln -s {} {}`" - .format(raw_core_name, core_path, new_lib_core_path)) + "Failed to create soft symbol link for {}.\n Please execute the following command manually: `ln -s {} {}`".format( + raw_core_name, core_path, new_lib_core_path + ) + ) # libpaddle without suffix return raw_core_name[:-3] @@ -602,8 +658,9 @@ def find_cuda_home(): which_cmd = 'where' if IS_WINDOWS else 'which' try: with open(os.devnull, 'w') as devnull: - nvcc_path = subprocess.check_output([which_cmd, 'nvcc'], - stderr=devnull) + nvcc_path = subprocess.check_output( + [which_cmd, 'nvcc'], stderr=devnull + ) nvcc_path = nvcc_path.decode() # Multi CUDA, select the first nvcc_path = nvcc_path.split('\r\n')[0] @@ -621,8 +678,11 @@ def find_cuda_home(): else: cuda_home = "/usr/local/cuda" # step 3. check whether path is valid - if cuda_home and not os.path.exists( - cuda_home) and core.is_compiled_with_cuda(): + if ( + cuda_home + and not os.path.exists(cuda_home) + and core.is_compiled_with_cuda() + ): cuda_home = None return cuda_home @@ -640,8 +700,9 @@ def find_rocm_home(): which_cmd = 'where' if IS_WINDOWS else 'which' try: with open(os.devnull, 'w') as devnull: - hipcc_path = subprocess.check_output([which_cmd, 'hipcc'], - stderr=devnull) + hipcc_path = subprocess.check_output( + [which_cmd, 'hipcc'], stderr=devnull + ) hipcc_path = hipcc_path.decode() hipcc_path = hipcc_path.rstrip('\r\n') @@ -650,8 +711,11 @@ def find_rocm_home(): except: rocm_home = "/opt/rocm" # step 3. check whether path is valid - if rocm_home and not os.path.exists( - rocm_home) and core.is_compiled_with_rocm(): + if ( + rocm_home + and not os.path.exists(rocm_home) + and core.is_compiled_with_rocm() + ): rocm_home = None return rocm_home @@ -719,8 +783,9 @@ def find_clang_cpp_include(compiler='clang'): if "InstalledDir" in info: v1_path = info.split(':')[-1].strip() if v1_path and os.path.exists(v1_path): - std_v1_includes = os.path.join(os.path.dirname(v1_path), - 'include/c++/v1') + std_v1_includes = os.path.join( + os.path.dirname(v1_path), 'include/c++/v1' + ) except Exception: # Just raise warnings because the include dir is not required. warnings.warn( @@ -820,15 +885,20 @@ def get_build_directory(verbose=False): root_extensions_directory = os.environ.get('PADDLE_EXTENSION_DIR') if root_extensions_directory is None: dir_name = "paddle_extensions" - root_extensions_directory = os.path.join(os.path.expanduser('~/.cache'), - dir_name) + root_extensions_directory = os.path.join( + os.path.expanduser('~/.cache'), dir_name + ) if IS_WINDOWS: root_extensions_directory = os.path.normpath( - root_extensions_directory) + root_extensions_directory + ) log_v( - "$PADDLE_EXTENSION_DIR is not set, using path: {} by default.". - format(root_extensions_directory), verbose) + "$PADDLE_EXTENSION_DIR is not set, using path: {} by default.".format( + root_extensions_directory + ), + verbose, + ) if not os.path.exists(root_extensions_directory): os.makedirs(root_extensions_directory) @@ -843,8 +913,10 @@ def parse_op_info(op_name): """ if op_name not in OpProtoHolder.instance().op_proto_map: raise ValueError( - "Please load {} shared library file firstly by `paddle.utils.cpp_extension.load_op_meta_info_and_register_op(...)`" - .format(op_name)) + "Please load {} shared library file firstly by `paddle.utils.cpp_extension.load_op_meta_info_and_register_op(...)`".format( + op_name + ) + ) op_proto = OpProtoHolder.instance().get_op_proto(op_name) in_names = [x.name for x in op_proto.inputs] @@ -869,21 +941,22 @@ def _import_module_from_library(module_name, build_directory, verbose=False): ext_path = os.path.join(build_directory, module_name + dynamic_suffix) if not os.path.exists(ext_path): raise FileNotFoundError( - "Extension path: {} does not exist.".format(ext_path)) + "Extension path: {} does not exist.".format(ext_path) + ) # load custom op_info and kernels from .so shared library log_v('loading shared library from: {}'.format(ext_path), verbose) op_names = load_op_meta_info_and_register_op(ext_path) # generate Python api in ext_path - return _generate_python_module(module_name, op_names, build_directory, - verbose) + return _generate_python_module( + module_name, op_names, build_directory, verbose + ) -def _generate_python_module(module_name, - op_names, - build_directory, - verbose=False): +def _generate_python_module( + module_name, op_names, build_directory, verbose=False +): """ Automatically generate python file to allow import or load into as module """ @@ -895,8 +968,9 @@ def _generate_python_module(module_name, # NOTE: Use unique id as suffix to avoid write same file at same time in # both multi-thread and multi-process. thread_id = str(threading.currentThread().ident) - api_file = os.path.join(build_directory, - module_name + '_' + thread_id + '.py') + api_file = os.path.join( + build_directory, module_name + '_' + thread_id + '.py' + ) log_v("generate api file: {}".format(api_file), verbose) # delete the temp file before exit python process @@ -913,10 +987,17 @@ def _generate_python_module(module_name, def _custom_api_content(op_name): - params_str, ins_str, attrs_str, outs_str, in_names, attrs_names = _get_api_inputs_str( - op_name) + ( + params_str, + ins_str, + attrs_str, + outs_str, + in_names, + attrs_names, + ) = _get_api_inputs_str(op_name) lower_in_names = [p.split("@")[0].lower() for p in in_names] - API_TEMPLATE = textwrap.dedent(""" + API_TEMPLATE = textwrap.dedent( + """ import paddle.fluid.core as core from paddle.fluid.core import VarBase, CustomOpKernelContext from paddle.fluid.framework import _non_static_mode, _dygraph_tracer, _in_legacy_dygraph, in_dygraph_mode @@ -956,7 +1037,8 @@ def _custom_api_content(op_name): res = [outs[out_name] for out_name in out_names] return res[0] if len(res)==1 else res - """).lstrip() + """ + ).lstrip() # generate python api file api_content = API_TEMPLATE.format( @@ -967,7 +1049,8 @@ def _custom_api_content(op_name): # "[x, y, z]"" in_names="[" + ",".join(lower_in_names) + "]", attr_names="[" + ",".join(attrs_names) + "]", - out_names=outs_str) + out_names=outs_str, + ) return api_content @@ -978,7 +1061,8 @@ def _load_module_from_file(api_file_path, module_name, verbose=False): """ if not os.path.exists(api_file_path): raise FileNotFoundError( - "File : {} does not exist.".format(api_file_path)) + "File : {} does not exist.".format(api_file_path) + ) # Unique readable module name to place custom api. log_v('import module from file: {}'.format(api_file_path), verbose) @@ -1003,35 +1087,40 @@ def _get_api_inputs_str(op_name): # input name by `@`, and only use first substr as argument params_str = ','.join([p.split("@")[0].lower() for p in param_names]) # e.g: {'X': x, 'Y': y, 'Z': z} - ins_str = "{%s}" % ','.join([ - "'{}' : {}".format(in_name, - in_name.split("@")[0].lower()) - for in_name in in_names - ]) + ins_str = "{%s}" % ','.join( + [ + "'{}' : {}".format(in_name, in_name.split("@")[0].lower()) + for in_name in in_names + ] + ) # e.g: {'num': n} - attrs_str = "{%s}" % ",".join([ - "'{}' : {}".format(attr_name, - attr_name.split("@")[0].lower()) - for attr_name in attr_names - ]) + attrs_str = "{%s}" % ",".join( + [ + "'{}' : {}".format(attr_name, attr_name.split("@")[0].lower()) + for attr_name in attr_names + ] + ) # e.g: ['Out', 'Index'] outs_str = "[%s]" % ','.join(["'{}'".format(name) for name in out_names]) return params_str, ins_str, attrs_str, outs_str, in_names, attr_names -def _write_setup_file(name, - sources, - file_path, - build_dir, - include_dirs, - extra_cxx_cflags, - extra_cuda_cflags, - link_args, - verbose=False): +def _write_setup_file( + name, + sources, + file_path, + build_dir, + include_dirs, + extra_cxx_cflags, + extra_cuda_cflags, + link_args, + verbose=False, +): """ Automatically generate setup.py and write it into build directory. """ - template = textwrap.dedent(""" + template = textwrap.dedent( + """ import os from paddle.utils.cpp_extension import CppExtension, CUDAExtension, BuildExtension, setup from paddle.utils.cpp_extension import get_build_directory @@ -1048,21 +1137,24 @@ def _write_setup_file(name, cmdclass={{"build_ext" : BuildExtension.with_options( output_dir=r'{build_dir}', no_python_abi_suffix=True) - }})""").lstrip() + }})""" + ).lstrip() with_cuda = False if any([is_cuda_file(source) for source in sources]): with_cuda = True log_v("with_cuda: {}".format(with_cuda), verbose) - content = template.format(name=name, - prefix='CUDA' if with_cuda else 'Cpp', - sources=list2str(sources), - include_dirs=list2str(include_dirs), - extra_cxx_cflags=list2str(extra_cxx_cflags), - extra_cuda_cflags=list2str(extra_cuda_cflags), - extra_link_args=list2str(link_args), - build_dir=build_dir) + content = template.format( + name=name, + prefix='CUDA' if with_cuda else 'Cpp', + sources=list2str(sources), + include_dirs=list2str(include_dirs), + extra_cxx_cflags=list2str(extra_cxx_cflags), + extra_cuda_cflags=list2str(extra_cuda_cflags), + extra_link_args=list2str(link_args), + build_dir=build_dir, + ) log_v('write setup.py into {}'.format(file_path), verbose) with open(file_path, 'w') as f: @@ -1073,7 +1165,8 @@ def list2str(args): """ Convert list[str] into string. For example: ['x', 'y'] -> "['x', 'y']" """ - if args is None: return '[]' + if args is None: + return '[]' assert isinstance(args, (list, tuple)) args = ["{}".format(arg) for arg in args] return repr(args) @@ -1094,19 +1187,26 @@ def _jit_compile(file_path, verbose=False): py_version = py_version.decode() log_v( "Using Python interpreter: {}, version: {}".format( - interpreter, py_version.strip()), verbose) + interpreter, py_version.strip() + ), + verbose, + ) except Exception: _, error, _ = sys.exc_info() raise RuntimeError( 'Failed to check Python interpreter with `{}`, errors: {}'.format( - interpreter, error)) + interpreter, error + ) + ) if IS_WINDOWS: - compile_cmd = 'cd /d {} && {} {} build'.format(ext_dir, interpreter, - setup_file) + compile_cmd = 'cd /d {} && {} {} build'.format( + ext_dir, interpreter, setup_file + ) else: - compile_cmd = 'cd {} && {} {} build'.format(ext_dir, interpreter, - setup_file) + compile_cmd = 'cd {} && {} {} build'.format( + ext_dir, interpreter, setup_file + ) print("Compiling user custom op, it will cost a few seconds.....") run_cmd(compile_cmd, verbose) @@ -1144,15 +1244,16 @@ def run_cmd(command, verbose=False): # execute command try: if verbose: - return subprocess.check_call(command, - shell=True, - stderr=subprocess.STDOUT) + return subprocess.check_call( + command, shell=True, stderr=subprocess.STDOUT + ) else: return subprocess.check_call(command, shell=True, stdout=DEVNULL) except Exception: _, error, _ = sys.exc_info() - raise RuntimeError("Failed to run command: {}, errors: {}".format( - compile, error)) + raise RuntimeError( + "Failed to run command: {}, errors: {}".format(compile, error) + ) def check_abi_compatibility(compiler, verbose=False): @@ -1164,17 +1265,22 @@ def check_abi_compatibility(compiler, verbose=False): return True if not IS_WINDOWS: - cmd_out = subprocess.check_output(['which', compiler], - stderr=subprocess.STDOUT) + cmd_out = subprocess.check_output( + ['which', compiler], stderr=subprocess.STDOUT + ) compiler_path = os.path.realpath(cmd_out.decode()).strip() # if not found any suitable compiler, raise warning - if not any(name in compiler_path - for name in _expected_compiler_current_platform()): + if not any( + name in compiler_path + for name in _expected_compiler_current_platform() + ): warnings.warn( WRONG_COMPILER_WARNING.format( user_compiler=compiler, paddle_compiler=_expected_compiler_current_platform()[0], - platform=OS_NAME)) + platform=OS_NAME, + ) + ) return False version = (0, 0, 0) @@ -1185,13 +1291,15 @@ def check_abi_compatibility(compiler, verbose=False): if OS_NAME.startswith('linux'): mini_required_version = GCC_MINI_VERSION version_info = subprocess.check_output( - [compiler, '-dumpfullversion', '-dumpversion']) + [compiler, '-dumpfullversion', '-dumpversion'] + ) version_info = version_info.decode() version = version_info.strip().split('.') elif IS_WINDOWS: mini_required_version = MSVC_MINI_VERSION - compiler_info = subprocess.check_output(compiler, - stderr=subprocess.STDOUT) + compiler_info = subprocess.check_output( + compiler, stderr=subprocess.STDOUT + ) try: compiler_info = compiler_info.decode('UTF-8') except UnicodeDecodeError: @@ -1202,8 +1310,11 @@ def check_abi_compatibility(compiler, verbose=False): except Exception: # check compiler version failed _, error, _ = sys.exc_info() - warnings.warn('Failed to check compiler version for {}: {}'.format( - compiler, error)) + warnings.warn( + 'Failed to check compiler version for {}: {}'.format( + compiler, error + ) + ) return False # check version compatibility @@ -1211,8 +1322,10 @@ def check_abi_compatibility(compiler, verbose=False): if tuple(map(int, version)) >= mini_required_version: return True warnings.warn( - ABI_INCOMPATIBILITY_WARNING.format(user_compiler=compiler, - version='.'.join(version))) + ABI_INCOMPATIBILITY_WARNING.format( + user_compiler=compiler, version='.'.join(version) + ) + ) return False diff --git a/python/paddle/utils/deprecated.py b/python/paddle/utils/deprecated.py index 00a38cb1368..b4c17de3f17 100755 --- a/python/paddle/utils/deprecated.py +++ b/python/paddle/utils/deprecated.py @@ -34,23 +34,23 @@ warnings.simplefilter('default', DeprecationWarning) def deprecated(update_to="", since="", reason="", level=0): """Decorate a function to signify its deprecation. - This function wraps a method that will soon be removed and does two things: - - The docstring of the API will be modified to include a notice - about deprecation." - - Raises a :class:`~exceptions.DeprecatedWarning` when old API is called. - - Args: - since(str, optional): The version at which the decorated method is considered deprecated. - update_to(str, optional): The new API users should use. - reason(str, optional): The reason why the API is deprecated. - level(int, optional): The deprecated warning log level. It must be - an Integer and must be one of 0, 1, 2. - If `level == 0`, the warning message will not be showed. - If `level == 1`, the warning message will be showed normally. - If `level == 2`, it will raise `RuntimeError`. - - Returns: - decorator: decorated function or class. + This function wraps a method that will soon be removed and does two things: + - The docstring of the API will be modified to include a notice + about deprecation." + - Raises a :class:`~exceptions.DeprecatedWarning` when old API is called. + + Args: + since(str, optional): The version at which the decorated method is considered deprecated. + update_to(str, optional): The new API users should use. + reason(str, optional): The reason why the API is deprecated. + level(int, optional): The deprecated warning log level. It must be + an Integer and must be one of 0, 1, 2. + If `level == 0`, the warning message will not be showed. + If `level == 1`, the warning message will be showed normally. + If `level == 2`, it will raise `RuntimeError`. + + Returns: + decorator: decorated function or class. """ def decorator(func): @@ -62,7 +62,8 @@ def deprecated(update_to="", since="", reason="", level=0): assert isinstance(reason, str), 'type of "reason" must be str.' assert isinstance(level, int) and level >= 0 and level < 3, ( 'type of "level" must be int and must be one of 0, 1, 2. But ' - 'received: {}.'.format(level)) + 'received: {}.'.format(level) + ) _since = since.strip() _update_to = update_to.strip() @@ -77,7 +78,8 @@ def deprecated(update_to="", since="", reason="", level=0): assert _update_to.startswith( "paddle." ), 'Argument update_to must start with "paddle.", your value is "{}"'.format( - update_to) + update_to + ) msg += ' Please use "{}" instead.'.format(_update_to) if len(_reason) > 0: msg += "\nreason: {}".format(_reason) @@ -90,14 +92,17 @@ def deprecated(update_to="", since="", reason="", level=0): @functools.wraps(func) def wrapper(*args, **kwargs): """deprecated warning should be fired in 3 circumstances: - 1. current version is develop version, i.e. "0.0.0", because we assume develop version is always the latest version. - 2. since version is empty, in this case, API is deprecated in all versions. - 3. current version is newer than since version. + 1. current version is develop version, i.e. "0.0.0", because we assume develop version is always the latest version. + 2. since version is empty, in this case, API is deprecated in all versions. + 3. current version is newer than since version. """ if level == 2: - raise RuntimeError('API "{}.{}" has been deprecated.'.format( - func.__module__, func.__name__)) + raise RuntimeError( + 'API "{}.{}" has been deprecated.'.format( + func.__module__, func.__name__ + ) + ) warningmsg = "\033[93m\nWarning:\n%s \033[0m" % (msg) # ensure ANSI escape sequences print correctly in cmd and powershell @@ -108,10 +113,14 @@ def deprecated(update_to="", since="", reason="", level=0): v_current += [0] * (4 - len(v_current)) v_since = [int(i) for i in _since.split(".")] v_since += [0] * (4 - len(v_since)) - if paddle.__version__ == "0.0.0" or _since == "" or v_current >= v_since: - warnings.warn(warningmsg, - category=DeprecationWarning, - stacklevel=2) + if ( + paddle.__version__ == "0.0.0" + or _since == "" + or v_current >= v_since + ): + warnings.warn( + warningmsg, category=DeprecationWarning, stacklevel=2 + ) return func(*args, **kwargs) diff --git a/python/paddle/utils/dlpack.py b/python/paddle/utils/dlpack.py index bc05cd9c299..c0449cdcae9 100644 --- a/python/paddle/utils/dlpack.py +++ b/python/paddle/utils/dlpack.py @@ -51,7 +51,8 @@ def to_dlpack(x): if not isinstance(x, (paddle.Tensor, paddle.fluid.core.eager.Tensor)): raise TypeError( "The type of 'x' in to_dlpack must be paddle.Tensor," - " but received {}.".format(type(x))) + " but received {}.".format(type(x)) + ) return x.value().get_tensor()._to_dlpack() @@ -87,11 +88,12 @@ def from_dlpack(dlpack): """ t = type(dlpack) - dlpack_flag = (t.__module__ == 'builtins' and t.__name__ == 'PyCapsule') + dlpack_flag = t.__module__ == 'builtins' and t.__name__ == 'PyCapsule' if not dlpack_flag: raise TypeError( "The type of 'dlpack' in from_dlpack must be PyCapsule object," - " but received {}.".format(type(dlpack))) + " but received {}.".format(type(dlpack)) + ) if _non_static_mode(): out = paddle.fluid.core.from_dlpack(dlpack) diff --git a/python/paddle/utils/download.py b/python/paddle/utils/download.py index 270ad51c66f..a77d7b60dea 100644 --- a/python/paddle/utils/download.py +++ b/python/paddle/utils/download.py @@ -28,7 +28,6 @@ try: except: class tqdm(object): - def __init__(self, total=None): self.total = total self.n = 0 @@ -38,8 +37,9 @@ except: if self.total is None: sys.stderr.write("\r{0:.1f} bytes".format(self.n)) else: - sys.stderr.write("\r{0:.1f}%".format(100 * self.n / - float(self.total))) + sys.stderr.write( + "\r{0:.1f}%".format(100 * self.n / float(self.total)) + ) sys.stderr.flush() def __enter__(self): @@ -115,13 +115,10 @@ def _get_unique_endpoints(trainer_endpoints): return unique_endpoints -def get_path_from_url(url, - root_dir, - md5sum=None, - check_exist=True, - decompress=True, - method='get'): - """ Download from given url to root_dir. +def get_path_from_url( + url, root_dir, md5sum=None, check_exist=True, decompress=True, method='get' +): + """Download from given url to root_dir. if file or directory specified by url is exists under root_dir, return the path directly, otherwise download from url and decompress it, return the path. @@ -157,8 +154,9 @@ def get_path_from_url(url, time.sleep(1) if ParallelEnv().current_endpoint in unique_endpoints: - if decompress and (tarfile.is_tarfile(fullpath) - or zipfile.is_zipfile(fullpath)): + if decompress and ( + tarfile.is_tarfile(fullpath) or zipfile.is_zipfile(fullpath) + ): fullpath = _decompress(fullpath) return fullpath @@ -170,13 +168,18 @@ def _get_download(url, fullname): try: req = requests.get(url, stream=True) except Exception as e: # requests.exceptions.ConnectionError - logger.info("Downloading {} from {} failed with exception {}".format( - fname, url, str(e))) + logger.info( + "Downloading {} from {} failed with exception {}".format( + fname, url, str(e) + ) + ) return False if req.status_code != 200: - raise RuntimeError("Downloading from {} failed with code " - "{}!".format(url, req.status_code)) + raise RuntimeError( + "Downloading from {} failed with code " + "{}!".format(url, req.status_code) + ) # For protecting download interupted, download to # tmp_fullname firstly, move tmp_fullname to fullname @@ -202,18 +205,20 @@ def _wget_download(url, fullname): # using wget to download url tmp_fullname = fullname + "_tmp" # –user-agent - command = 'wget -O {} -t {} {}'.format(tmp_fullname, DOWNLOAD_RETRY_LIMIT, - url) - subprc = subprocess.Popen(command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + command = 'wget -O {} -t {} {}'.format( + tmp_fullname, DOWNLOAD_RETRY_LIMIT, url + ) + subprc = subprocess.Popen( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) _ = subprc.communicate() if subprc.returncode != 0: raise RuntimeError( - '{} failed. Please make sure `wget` is installed or {} exists'. - format(command, url)) + '{} failed. Please make sure `wget` is installed or {} exists'.format( + command, url + ) + ) shutil.move(tmp_fullname, fullname) @@ -237,7 +242,8 @@ def _download(url, path, md5sum=None, method='get'): """ assert method in _download_methods, 'make sure `{}` implemented'.format( - method) + method + ) if not osp.exists(path): os.makedirs(path) @@ -251,8 +257,9 @@ def _download(url, path, md5sum=None, method='get'): if retry_cnt < DOWNLOAD_RETRY_LIMIT: retry_cnt += 1 else: - raise RuntimeError("Download from {} failed. " - "Retry limit reached".format(url)) + raise RuntimeError( + "Download from {} failed. " "Retry limit reached".format(url) + ) if not _download_methods[method](url, fullname): time.sleep(1) @@ -273,8 +280,10 @@ def _md5check(fullname, md5sum=None): calc_md5sum = md5.hexdigest() if calc_md5sum != md5sum: - logger.info("File {} md5 check failed, {}(calc) != " - "{}(base)".format(fullname, calc_md5sum, md5sum)) + logger.info( + "File {} md5 check failed, {}(calc) != " + "{}(base)".format(fullname, calc_md5sum, md5sum) + ) return False return True @@ -314,7 +323,8 @@ def _uncompress_file_zip(filepath): elif _is_a_single_dir(file_list): # `strip(os.sep)` to remove `os.sep` in the tail of path rootpath = os.path.splitext(file_list[0].strip(os.sep))[0].split( - os.sep)[-1] + os.sep + )[-1] uncompressed_path = os.path.join(file_dir, rootpath) files.extractall(file_dir) @@ -340,7 +350,8 @@ def _uncompress_file_tar(filepath, mode="r:*"): files.extractall(file_dir) elif _is_a_single_dir(file_list): rootpath = os.path.splitext(file_list[0].strip(os.sep))[0].split( - os.sep)[-1] + os.sep + )[-1] uncompressed_path = os.path.join(file_dir, rootpath) files.extractall(file_dir) else: diff --git a/python/paddle/utils/gast/ast3.py b/python/paddle/utils/gast/ast3.py index 4696c1ba497..e8c9ce7b01a 100644 --- a/python/paddle/utils/gast/ast3.py +++ b/python/paddle/utils/gast/ast3.py @@ -49,10 +49,7 @@ class Ast3ToGAst(AstToGAst): if sys.version_info.minor < 8: def visit_Module(self, node): - new_node = gast.Module( - self._visit(node.body), - [] # type_ignores - ) + new_node = gast.Module(self._visit(node.body), []) # type_ignores return new_node def visit_Num(self, node): @@ -231,7 +228,8 @@ class Ast3ToGAst(AstToGAst): new_node = gast.ExceptHandler( self._visit(node.type), gast.Name(node.name, gast.Store(), None, None), - self._visit(node.body)) + self._visit(node.body), + ) ast.copy_location(new_node, node) return new_node else: @@ -253,7 +251,6 @@ class GAstToAst3(GAstToAst): if sys.version_info.minor < 9: def visit_Subscript(self, node): - def adjust_slice(s): if isinstance(s, ast.Slice): return s @@ -263,7 +260,8 @@ class GAstToAst3(GAstToAst): if isinstance(node.slice, gast.Tuple): if any(isinstance(elt, gast.slice) for elt in node.slice.elts): new_slice = ast.ExtSlice( - [adjust_slice(x) for x in self._visit(node.slice.elts)]) + [adjust_slice(x) for x in self._visit(node.slice.elts)] + ) else: value = ast.Tuple(self._visit(node.slice.elts), ast.Load()) ast.copy_location(value, node.slice) @@ -309,10 +307,11 @@ class GAstToAst3(GAstToAst): if sys.version_info.minor < 8: extra_args = tuple() else: - extra_args = self._visit(node.type_comment), + extra_args = (self._visit(node.type_comment),) - new_node = ast.arg(self._visit(node.id), self._visit(node.annotation), - *extra_args) + new_node = ast.arg( + self._visit(node.id), self._visit(node.annotation), *extra_args + ) return ast.copy_location(new_node, node) def visit_Name(self, node): @@ -325,8 +324,9 @@ class GAstToAst3(GAstToAst): def visit_ExceptHandler(self, node): if node.name: - new_node = ast.ExceptHandler(self._visit(node.type), node.name.id, - self._visit(node.body)) + new_node = ast.ExceptHandler( + self._visit(node.type), node.name.id, self._visit(node.body) + ) return ast.copy_location(new_node, node) else: return self.generic_visit(node) @@ -452,10 +452,13 @@ class GAstToAst3(GAstToAst): if sys.version_info.minor >= 8: new_node = ast.arguments( [self._make_arg(arg) for arg in node.posonlyargs], - [self._make_arg(n) for n in node.args], *extra_args) + [self._make_arg(n) for n in node.args], + *extra_args + ) else: - new_node = ast.arguments([self._make_arg(n) for n in node.args], - *extra_args) + new_node = ast.arguments( + [self._make_arg(n) for n in node.args], *extra_args + ) return new_node diff --git a/python/paddle/utils/gast/astn.py b/python/paddle/utils/gast/astn.py index eb45bd4e450..bd88ba5efc5 100644 --- a/python/paddle/utils/gast/astn.py +++ b/python/paddle/utils/gast/astn.py @@ -34,9 +34,7 @@ from . import gast def _generate_translators(to): - class Translator(ast.NodeTransformer): - def _visit(self, node): if isinstance(node, list): return [self._visit(n) for n in node] diff --git a/python/paddle/utils/gast/gast.py b/python/paddle/utils/gast/gast.py index 1248434fe35..34ada012440 100644 --- a/python/paddle/utils/gast/gast.py +++ b/python/paddle/utils/gast/gast.py @@ -44,557 +44,991 @@ except ImportError: def _make_node(Name, Fields, Attributes, Bases): - def create_node(self, *args, **kwargs): nbparam = len(args) + len(kwargs) - assert nbparam in (0, len(Fields)), \ - "Bad argument number for {}: {}, expecting {}".\ - format(Name, nbparam, len(Fields)) + assert nbparam in ( + 0, + len(Fields), + ), "Bad argument number for {}: {}, expecting {}".format( + Name, nbparam, len(Fields) + ) self._fields = Fields self._attributes = Attributes for argname, argval in zip(self._fields, args): setattr(self, argname, argval) for argname, argval in kwargs.items(): - assert argname in Fields, \ - "Invalid Keyword argument for {}: {}".format(Name, argname) + assert ( + argname in Fields + ), "Invalid Keyword argument for {}: {}".format(Name, argname) setattr(self, argname, argval) - setattr(_sys.modules[__name__], Name, - type(Name, Bases, {'__init__': create_node})) + setattr( + _sys.modules[__name__], + Name, + type(Name, Bases, {'__init__': create_node}), + ) _nodes = ( # mod - ('Module', (('body', 'type_ignores'), (), (mod, ))), - ('Interactive', (('body', ), (), (mod, ))), - ('Expression', (('body', ), (), (mod, ))), - ('FunctionType', (('argtypes', 'returns'), (), (mod, ))), - ('Suite', (('body', ), (), (mod, ))), - + ('Module', (('body', 'type_ignores'), (), (mod,))), + ('Interactive', (('body',), (), (mod,))), + ('Expression', (('body',), (), (mod,))), + ('FunctionType', (('argtypes', 'returns'), (), (mod,))), + ('Suite', (('body',), (), (mod,))), # stmt - ('FunctionDef', (('name', 'args', 'body', 'decorator_list', 'returns', - 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AsyncFunctionDef', (('name', 'args', 'body', 'decorator_list', 'returns', - 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('ClassDef', (( - 'name', - 'bases', - 'keywords', - 'body', - 'decorator_list', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Return', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Delete', (('targets', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Assign', (( - 'targets', - 'value', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AugAssign', (( - 'target', - 'op', - 'value', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AnnAssign', (( - 'target', - 'annotation', - 'value', - 'simple', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Print', (( - 'dest', - 'values', - 'nl', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('For', (('target', 'iter', 'body', 'orelse', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AsyncFor', (('target', 'iter', 'body', 'orelse', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('While', (( - 'test', - 'body', - 'orelse', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('If', (( - 'test', - 'body', - 'orelse', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('With', (('items', 'body', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AsyncWith', (('items', 'body', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Raise', (( - 'exc', - 'cause', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Try', (( - 'body', - 'handlers', - 'orelse', - 'finalbody', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Assert', (( - 'test', - 'msg', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Import', (('names', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('ImportFrom', (( - 'module', - 'names', - 'level', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Exec', (( - 'body', - 'globals', - 'locals', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Global', (('names', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Nonlocal', (('names', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Expr', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Pass', ((), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Break', ((), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Continue', ((), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - + ( + 'FunctionDef', + ( + ( + 'name', + 'args', + 'body', + 'decorator_list', + 'returns', + 'type_comment', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AsyncFunctionDef', + ( + ( + 'name', + 'args', + 'body', + 'decorator_list', + 'returns', + 'type_comment', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'ClassDef', + ( + ( + 'name', + 'bases', + 'keywords', + 'body', + 'decorator_list', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Return', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Delete', + ( + ('targets',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Assign', + ( + ( + 'targets', + 'value', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AugAssign', + ( + ( + 'target', + 'op', + 'value', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AnnAssign', + ( + ( + 'target', + 'annotation', + 'value', + 'simple', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Print', + ( + ( + 'dest', + 'values', + 'nl', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'For', + ( + ('target', 'iter', 'body', 'orelse', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AsyncFor', + ( + ('target', 'iter', 'body', 'orelse', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'While', + ( + ( + 'test', + 'body', + 'orelse', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'If', + ( + ( + 'test', + 'body', + 'orelse', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'With', + ( + ('items', 'body', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AsyncWith', + ( + ('items', 'body', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Raise', + ( + ( + 'exc', + 'cause', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Try', + ( + ( + 'body', + 'handlers', + 'orelse', + 'finalbody', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Assert', + ( + ( + 'test', + 'msg', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Import', + ( + ('names',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'ImportFrom', + ( + ( + 'module', + 'names', + 'level', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Exec', + ( + ( + 'body', + 'globals', + 'locals', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Global', + ( + ('names',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Nonlocal', + ( + ('names',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Expr', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Pass', + ( + (), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Break', + ( + (), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Continue', + ( + (), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), # expr - ('BoolOp', (( - 'op', - 'values', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('BinOp', (( - 'left', - 'op', - 'right', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('UnaryOp', (( - 'op', - 'operand', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Lambda', (( - 'args', - 'body', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('IfExp', (( - 'test', - 'body', - 'orelse', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Dict', (( - 'keys', - 'values', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Set', (('elts', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('ListComp', (( - 'elt', - 'generators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('SetComp', (( - 'elt', - 'generators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('DictComp', (( - 'key', - 'value', - 'generators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('GeneratorExp', (( - 'elt', - 'generators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Await', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Yield', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('YieldFrom', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Compare', (( - 'left', - 'ops', - 'comparators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Call', (( - 'func', - 'args', - 'keywords', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Repr', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('FormattedValue', (( - 'value', - 'conversion', - 'format_spec', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('JoinedStr', (('values', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Constant', (('value', 'kind'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Attribute', (( - 'value', - 'attr', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Subscript', (( - 'value', - 'slice', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Starred', (( - 'value', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Name', (('id', 'ctx', 'annotation', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('List', (( - 'elts', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Tuple', (( - 'elts', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - + ( + 'BoolOp', + ( + ( + 'op', + 'values', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'BinOp', + ( + ( + 'left', + 'op', + 'right', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'UnaryOp', + ( + ( + 'op', + 'operand', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Lambda', + ( + ( + 'args', + 'body', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'IfExp', + ( + ( + 'test', + 'body', + 'orelse', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Dict', + ( + ( + 'keys', + 'values', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Set', + ( + ('elts',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'ListComp', + ( + ( + 'elt', + 'generators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'SetComp', + ( + ( + 'elt', + 'generators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'DictComp', + ( + ( + 'key', + 'value', + 'generators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'GeneratorExp', + ( + ( + 'elt', + 'generators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Await', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Yield', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'YieldFrom', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Compare', + ( + ( + 'left', + 'ops', + 'comparators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Call', + ( + ( + 'func', + 'args', + 'keywords', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Repr', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'FormattedValue', + ( + ( + 'value', + 'conversion', + 'format_spec', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'JoinedStr', + ( + ('values',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Constant', + ( + ('value', 'kind'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Attribute', + ( + ( + 'value', + 'attr', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Subscript', + ( + ( + 'value', + 'slice', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Starred', + ( + ( + 'value', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Name', + ( + ('id', 'ctx', 'annotation', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'List', + ( + ( + 'elts', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Tuple', + ( + ( + 'elts', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), # expr_context - ('Load', ((), (), (expr_context, ))), - ('Store', ((), (), (expr_context, ))), - ('Del', ((), (), (expr_context, ))), - ('AugLoad', ((), (), (expr_context, ))), - ('AugStore', ((), (), (expr_context, ))), - ('Param', ((), (), (expr_context, ))), - + ('Load', ((), (), (expr_context,))), + ('Store', ((), (), (expr_context,))), + ('Del', ((), (), (expr_context,))), + ('AugLoad', ((), (), (expr_context,))), + ('AugStore', ((), (), (expr_context,))), + ('Param', ((), (), (expr_context,))), # slice - ('Slice', (('lower', 'upper', 'step'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (slice, ))), - + ( + 'Slice', + ( + ('lower', 'upper', 'step'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (slice,), + ), + ), # boolop - ('And', ((), (), (boolop, ))), - ('Or', ((), (), (boolop, ))), - + ('And', ((), (), (boolop,))), + ('Or', ((), (), (boolop,))), # operator - ('Add', ((), (), (operator, ))), - ('Sub', ((), (), (operator, ))), - ('Mult', ((), (), (operator, ))), - ('MatMult', ((), (), (operator, ))), - ('Div', ((), (), (operator, ))), - ('Mod', ((), (), (operator, ))), - ('Pow', ((), (), (operator, ))), - ('LShift', ((), (), (operator, ))), - ('RShift', ((), (), (operator, ))), - ('BitOr', ((), (), (operator, ))), - ('BitXor', ((), (), (operator, ))), - ('BitAnd', ((), (), (operator, ))), - ('FloorDiv', ((), (), (operator, ))), - + ('Add', ((), (), (operator,))), + ('Sub', ((), (), (operator,))), + ('Mult', ((), (), (operator,))), + ('MatMult', ((), (), (operator,))), + ('Div', ((), (), (operator,))), + ('Mod', ((), (), (operator,))), + ('Pow', ((), (), (operator,))), + ('LShift', ((), (), (operator,))), + ('RShift', ((), (), (operator,))), + ('BitOr', ((), (), (operator,))), + ('BitXor', ((), (), (operator,))), + ('BitAnd', ((), (), (operator,))), + ('FloorDiv', ((), (), (operator,))), # unaryop - ('Invert', ((), (), ( - unaryop, - AST, - ))), - ('Not', ((), (), ( - unaryop, - AST, - ))), - ('UAdd', ((), (), ( - unaryop, - AST, - ))), - ('USub', ((), (), ( - unaryop, - AST, - ))), - + ( + 'Invert', + ( + (), + (), + ( + unaryop, + AST, + ), + ), + ), + ( + 'Not', + ( + (), + (), + ( + unaryop, + AST, + ), + ), + ), + ( + 'UAdd', + ( + (), + (), + ( + unaryop, + AST, + ), + ), + ), + ( + 'USub', + ( + (), + (), + ( + unaryop, + AST, + ), + ), + ), # cmpop - ('Eq', ((), (), (cmpop, ))), - ('NotEq', ((), (), (cmpop, ))), - ('Lt', ((), (), (cmpop, ))), - ('LtE', ((), (), (cmpop, ))), - ('Gt', ((), (), (cmpop, ))), - ('GtE', ((), (), (cmpop, ))), - ('Is', ((), (), (cmpop, ))), - ('IsNot', ((), (), (cmpop, ))), - ('In', ((), (), (cmpop, ))), - ('NotIn', ((), (), (cmpop, ))), - + ('Eq', ((), (), (cmpop,))), + ('NotEq', ((), (), (cmpop,))), + ('Lt', ((), (), (cmpop,))), + ('LtE', ((), (), (cmpop,))), + ('Gt', ((), (), (cmpop,))), + ('GtE', ((), (), (cmpop,))), + ('Is', ((), (), (cmpop,))), + ('IsNot', ((), (), (cmpop,))), + ('In', ((), (), (cmpop,))), + ('NotIn', ((), (), (cmpop,))), # comprehension - ('comprehension', (('target', 'iter', 'ifs', 'is_async'), (), (AST, ))), - + ('comprehension', (('target', 'iter', 'ifs', 'is_async'), (), (AST,))), # excepthandler - ('ExceptHandler', (('type', 'name', 'body'), - ('lineno', 'col_offset', 'end_lineno', - 'end_col_offset'), (excepthandler, ))), - + ( + 'ExceptHandler', + ( + ('type', 'name', 'body'), + ('lineno', 'col_offset', 'end_lineno', 'end_col_offset'), + (excepthandler,), + ), + ), # arguments - ('arguments', (('args', 'posonlyargs', 'vararg', 'kwonlyargs', - 'kw_defaults', 'kwarg', 'defaults'), (), (AST, ))), - + ( + 'arguments', + ( + ( + 'args', + 'posonlyargs', + 'vararg', + 'kwonlyargs', + 'kw_defaults', + 'kwarg', + 'defaults', + ), + (), + (AST,), + ), + ), # keyword - ('keyword', (('arg', 'value'), ('lineno', 'col_offset', 'end_lineno', - 'end_col_offset'), (AST, ))), - + ( + 'keyword', + ( + ('arg', 'value'), + ('lineno', 'col_offset', 'end_lineno', 'end_col_offset'), + (AST,), + ), + ), # alias - ('alias', (('name', 'asname'), (), (AST, ))), - + ('alias', (('name', 'asname'), (), (AST,))), # withitem - ('withitem', (('context_expr', 'optional_vars'), (), (AST, ))), - + ('withitem', (('context_expr', 'optional_vars'), (), (AST,))), # type_ignore - ('type_ignore', ((), ('lineno', 'tag'), (TypeIgnore, ))), + ('type_ignore', ((), ('lineno', 'tag'), (TypeIgnore,))), ) for name, descr in _nodes: @@ -603,8 +1037,10 @@ for name, descr in _nodes: py_version = _sys.version_info.major if py_version != 3: raise RuntimeError( - 'Required Python version >= 3, but received Python version == {}'. - format(py_version)) + 'Required Python version >= 3, but received Python version == {}'.format( + py_version + ) + ) from .ast3 import ast_to_gast, gast_to_ast @@ -622,10 +1058,14 @@ def literal_eval(node_or_string): def get_docstring(node, clean=True): if not isinstance(node, (FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Constant): + if ( + node.body + and isinstance(node.body[0], Expr) + and isinstance(node.body[0].value, Constant) + ): if clean: import inspect + holder = node.body[0].value return inspect.cleandoc(getattr(holder, holder._fields[0])) return node.body[0].value.s @@ -641,8 +1081,11 @@ def copy_location(new_node, old_node): and return *new_node*. """ for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset': - if attr in old_node._attributes and attr in new_node._attributes \ - and hasattr(old_node, attr): + if ( + attr in old_node._attributes + and attr in new_node._attributes + and hasattr(old_node, attr) + ): setattr(new_node, attr, getattr(old_node, attr)) return new_node diff --git a/python/paddle/utils/image_util.py b/python/paddle/utils/image_util.py index a694d32c93b..107ee828af4 100644 --- a/python/paddle/utils/image_util.py +++ b/python/paddle/utils/image_util.py @@ -25,9 +25,10 @@ def resize_image(img, target_size): img: the input image to be resized. target_size: the target resized image size. """ - percent = (target_size / float(min(img.size[0], img.size[1]))) + percent = target_size / float(min(img.size[0], img.size[1])) resized_size = int(round(img.size[0] * percent)), int( - round(img.size[1] * percent)) + round(img.size[1] * percent) + ) img = img.resize(resized_size, Image.ANTIALIAS) return img @@ -56,8 +57,9 @@ def crop_img(im, inner_size, color=True, test=True): If True, crop the center of images. """ if color: - height, width = max(inner_size, - im.shape[1]), max(inner_size, im.shape[2]) + height, width = max(inner_size, im.shape[1]), max( + inner_size, im.shape[2] + ) padded_im = np.zeros((3, height, width)) startY = (height - im.shape[1]) / 2 startX = (width - im.shape[2]) / 2 @@ -65,8 +67,9 @@ def crop_img(im, inner_size, color=True, test=True): padded_im[:, startY:endY, startX:endX] = im else: im = im.astype('float32') - height, width = max(inner_size, - im.shape[0]), max(inner_size, im.shape[1]) + height, width = max(inner_size, im.shape[0]), max( + inner_size, im.shape[1] + ) padded_im = np.zeros((height, width)) startY = (height - im.shape[0]) / 2 startX = (width - im.shape[1]) / 2 @@ -120,15 +123,17 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True): mean = np.load(meta_path)['data_mean'] border = (mean_img_size - crop_size) / 2 if color: - assert (mean_img_size * mean_img_size * 3 == mean.shape[0]) + assert mean_img_size * mean_img_size * 3 == mean.shape[0] mean = mean.reshape(3, mean_img_size, mean_img_size) - mean = mean[:, border:border + crop_size, - border:border + crop_size].astype('float32') + mean = mean[ + :, border : border + crop_size, border : border + crop_size + ].astype('float32') else: - assert (mean_img_size * mean_img_size == mean.shape[0]) + assert mean_img_size * mean_img_size == mean.shape[0] mean = mean.reshape(mean_img_size, mean_img_size) - mean = mean[border:border + crop_size, - border:border + crop_size].astype('float32') + mean = mean[ + border : border + crop_size, border : border + crop_size + ].astype('float32') return mean @@ -166,28 +171,28 @@ def oversample(img, crop_dims): crops_ix[curr] = (i, j, i + crop_dims[0], j + crop_dims[1]) curr += 1 crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate( - [-crop_dims / 2.0, crop_dims / 2.0]) + [-crop_dims / 2.0, crop_dims / 2.0] + ) crops_ix = np.tile(crops_ix, (2, 1)) # Extract crops - crops = np.empty((10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), - dtype=np.float32) + crops = np.empty( + (10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), + dtype=np.float32, + ) ix = 0 for im in img: for crop in crops_ix: - crops[ix] = im[crop[0]:crop[2], crop[1]:crop[3], :] + crops[ix] = im[crop[0] : crop[2], crop[1] : crop[3], :] ix += 1 - crops[ix - 5:ix] = crops[ix - 5:ix, :, ::-1, :] # flip for mirrors + crops[ix - 5 : ix] = crops[ix - 5 : ix, :, ::-1, :] # flip for mirrors return crops class ImageTransformer: - - def __init__(self, - transpose=None, - channel_swap=None, - mean=None, - is_color=True): + def __init__( + self, transpose=None, channel_swap=None, mean=None, is_color=True + ): self.is_color = is_color self.set_transpose(transpose) self.set_channel_swap(channel_swap) diff --git a/python/paddle/utils/install_check.py b/python/paddle/utils/install_check.py index f8fd9d71df5..c9cc1bb7a49 100644 --- a/python/paddle/utils/install_check.py +++ b/python/paddle/utils/install_check.py @@ -24,13 +24,14 @@ def _simple_network(): """ Define a simple network composed by a single linear layer. """ - input = paddle.static.data(name="input", - shape=[None, 2, 2], - dtype="float32") + input = paddle.static.data( + name="input", shape=[None, 2, 2], dtype="float32" + ) weight = paddle.create_parameter( shape=[2, 3], dtype="float32", - attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Constant(0.1))) + attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Constant(0.1)), + ) bias = paddle.create_parameter(shape=[3], dtype="float32") linear_out = paddle.nn.functional.linear(x=input, weight=weight, bias=bias) out = paddle.tensor.sum(linear_out) @@ -68,7 +69,8 @@ def _is_cuda_available(): logging.warning( "You are using GPU version PaddlePaddle, but there is no GPU " "detected on your machine. Maybe CUDA devices is not set properly." - "\n Original Error is {}".format(e)) + "\n Original Error is {}".format(e) + ) return False @@ -83,7 +85,8 @@ def _is_npu_available(): logging.warning( "You are using NPU version PaddlePaddle, but there is no NPU " "detected on your machine. Maybe NPU devices is not set properly." - "\n Original Error is {}".format(e)) + "\n Original Error is {}".format(e) + ) return False @@ -98,7 +101,8 @@ def _is_xpu_available(): logging.warning( "You are using XPU version PaddlePaddle, but there is no XPU " "detected on your machine. Maybe XPU devices is not set properly." - "\n Original Error is {}".format(e)) + "\n Original Error is {}".format(e) + ) return False @@ -121,20 +125,22 @@ def _run_dygraph_single(use_cuda, use_xpu, use_npu): else: paddle.set_device('cpu') weight_attr = paddle.ParamAttr( - name="weight", initializer=paddle.nn.initializer.Constant(value=0.5)) + name="weight", initializer=paddle.nn.initializer.Constant(value=0.5) + ) bias_attr = paddle.ParamAttr( - name="bias", initializer=paddle.nn.initializer.Constant(value=1.0)) - linear = paddle.nn.Linear(2, - 4, - weight_attr=weight_attr, - bias_attr=bias_attr) + name="bias", initializer=paddle.nn.initializer.Constant(value=1.0) + ) + linear = paddle.nn.Linear( + 2, 4, weight_attr=weight_attr, bias_attr=bias_attr + ) input_np = _prepare_data(1) input_tensor = paddle.to_tensor(input_np) linear_out = linear(input_tensor) out = paddle.tensor.sum(linear_out) out.backward() - opt = paddle.optimizer.Adam(learning_rate=0.001, - parameters=linear.parameters()) + opt = paddle.optimizer.Adam( + learning_rate=0.001, parameters=linear.parameters() + ) opt.step() @@ -155,7 +161,8 @@ def _run_static_single(use_cuda, use_xpu, use_npu): with paddle.static.program_guard(train_prog, startup_prog): input, out, weight = _simple_network() param_grads = paddle.static.append_backward( - out, parameter_list=[weight.name])[0] + out, parameter_list=[weight.name] + )[0] if use_cuda: place = paddle.CUDAPlace(0) @@ -168,9 +175,11 @@ def _run_static_single(use_cuda, use_xpu, use_npu): exe = paddle.static.Executor(place) exe.run(startup_prog) - exe.run(train_prog, - feed={input.name: _prepare_data(1)}, - fetch_list=[out.name, param_grads[1].name]) + exe.run( + train_prog, + feed={input.name: _prepare_data(1)}, + fetch_list=[out.name, param_grads[1].name], + ) paddle.disable_static() @@ -195,8 +204,8 @@ def _run_static_parallel(use_cuda, use_xpu, use_npu, device_list): paddle.optimizer.SGD(learning_rate=0.01).minimize(loss) compiled_prog = paddle.static.CompiledProgram( - train_prog).with_data_parallel(loss_name=loss.name, - places=device_list) + train_prog + ).with_data_parallel(loss_name=loss.name, places=device_list) if use_cuda: place = paddle.CUDAPlace(0) @@ -211,9 +220,11 @@ def _run_static_parallel(use_cuda, use_xpu, use_npu, device_list): exe = paddle.static.Executor(place) exe.run(startup_prog) - exe.run(compiled_prog, - feed={input.name: _prepare_data(len(device_list))}, - fetch_list=[loss.name]) + exe.run( + compiled_prog, + feed={input.name: _prepare_data(len(device_list))}, + fetch_list=[loss.name], + ) paddle.disable_static() @@ -269,8 +280,11 @@ def run_check(): try: _run_static_parallel(use_cuda, use_xpu, use_npu, device_list) - print("PaddlePaddle works well on {} {}s.".format( - device_count, device_str)) + print( + "PaddlePaddle works well on {} {}s.".format( + device_count, device_str + ) + ) print( "PaddlePaddle is installed successfully! Let's start deep learning with PaddlePaddle now." ) @@ -280,10 +294,15 @@ def run_check(): "\n 1. There is not enough GPUs visible on your system" "\n 2. Some GPUs are occupied by other process now" "\n 3. NVIDIA-NCCL2 is not installed correctly on your system. Please follow instruction on https://github.com/NVIDIA/nccl-tests " - "\n to test your NCCL, or reinstall it following https://docs.nvidia.com/deeplearning/sdk/nccl-install-guide/index.html" - .format(device_count, device_str)) + "\n to test your NCCL, or reinstall it following https://docs.nvidia.com/deeplearning/sdk/nccl-install-guide/index.html".format( + device_count, device_str + ) + ) logging.warning("\n Original Error is: {}".format(e)) - print("PaddlePaddle is installed successfully ONLY for single {}! " - "Let's start deep learning with PaddlePaddle now.".format( - device_str)) + print( + "PaddlePaddle is installed successfully ONLY for single {}! " + "Let's start deep learning with PaddlePaddle now.".format( + device_str + ) + ) diff --git a/python/paddle/utils/lazy_import.py b/python/paddle/utils/lazy_import.py index d9146422819..e5ca0cdd5a8 100644 --- a/python/paddle/utils/lazy_import.py +++ b/python/paddle/utils/lazy_import.py @@ -35,6 +35,6 @@ def try_import(module_name): err_msg = ( "Failed importing {}. This likely means that some paddle modules " "require additional dependencies that have to be " - "manually installed (usually with `pip install {}`). ").format( - module_name, install_name) + "manually installed (usually with `pip install {}`). " + ).format(module_name, install_name) raise ImportError(err_msg) diff --git a/python/paddle/utils/op_version.py b/python/paddle/utils/op_version.py index 575e5f40772..9f9ae4d73c7 100644 --- a/python/paddle/utils/op_version.py +++ b/python/paddle/utils/op_version.py @@ -29,7 +29,6 @@ def Singleton(cls): class OpUpdateInfoHelper(object): - def __init__(self, info): self._info = info @@ -49,7 +48,6 @@ class OpUpdateInfoHelper(object): @Singleton class OpLastCheckpointChecker(object): - def __init__(self): self.raw_version_map = core.get_op_version_map() self.checkpoints_map = {} @@ -65,8 +63,9 @@ class OpLastCheckpointChecker(object): updates = [] if op_name in self.checkpoints_map: for update in self.checkpoints_map[op_name]: - if (update.type() == type) or (type - == core.OpUpdateType.kInvalid): + if (update.type() == type) or ( + type == core.OpUpdateType.kInvalid + ): if OpUpdateInfoHelper(update.info()).verify_key_value(key): updates.append(update.info()) return updates diff --git a/python/paddle/utils/profiler.py b/python/paddle/utils/profiler.py index e0bad20a327..27803cfa442 100644 --- a/python/paddle/utils/profiler.py +++ b/python/paddle/utils/profiler.py @@ -22,14 +22,19 @@ from ..fluid.profiler import profiler # noqa: F401 from ..fluid.profiler import stop_profiler from ..fluid.profiler import reset_profiler -__all__ = [ #noqa - 'Profiler', 'get_profiler', 'ProfilerOptions', 'cuda_profiler', - 'start_profiler', 'profiler', 'stop_profiler', 'reset_profiler' +__all__ = [ # noqa + 'Profiler', + 'get_profiler', + 'ProfilerOptions', + 'cuda_profiler', + 'start_profiler', + 'profiler', + 'stop_profiler', + 'reset_profiler', ] class ProfilerOptions(object): - def __init__(self, options=None): self.options = { 'state': 'All', @@ -39,7 +44,7 @@ class ProfilerOptions(object): 'output_thread_detail': False, 'profile_path': 'none', 'timeline_path': 'none', - 'op_summary_path': 'none' + 'op_summary_path': 'none', } if options is not None: for key in self.options.keys(): @@ -54,10 +59,13 @@ class ProfilerOptions(object): def __getitem__(self, name): if self.options.get(name, None) is None: raise ValueError( - "ProfilerOptions does not have an option named %s." % name) + "ProfilerOptions does not have an option named %s." % name + ) else: - if isinstance(self.options[name], - str) and self.options[name] == 'none': + if ( + isinstance(self.options[name], str) + and self.options[name] == 'none' + ): return None else: return self.options[name] @@ -67,7 +75,6 @@ _current_profiler = None class Profiler(object): - def __init__(self, enabled=True, options=None): if options is not None: self.profiler_options = options @@ -99,22 +106,28 @@ class Profiler(object): try: start_profiler( state=self.profiler_options['state'], - tracer_option=self.profiler_options['tracer_level']) + tracer_option=self.profiler_options['tracer_level'], + ) except Exception as e: warnings.warn( - "Profiler is not enabled becuase following exception:\n{}". - format(e)) + "Profiler is not enabled becuase following exception:\n{}".format( + e + ) + ) def stop(self): if self.enabled: try: stop_profiler( sorted_key=self.profiler_options['sorted_key'], - profile_path=self.profiler_options['profile_path']) + profile_path=self.profiler_options['profile_path'], + ) except Exception as e: warnings.warn( - "Profiler is not disabled becuase following exception:\n{}". - format(e)) + "Profiler is not disabled becuase following exception:\n{}".format( + e + ) + ) def reset(self): if self.enabled and core.is_profiler_enabled(): diff --git a/python/paddle/utils/unique_name.py b/python/paddle/utils/unique_name.py index d0d487c933d..77ecebcc45b 100644 --- a/python/paddle/utils/unique_name.py +++ b/python/paddle/utils/unique_name.py @@ -16,6 +16,4 @@ from ..fluid.unique_name import generate # noqa: F401 from ..fluid.unique_name import switch # noqa: F401 from ..fluid.unique_name import guard # noqa: F401 -__all__ = [ #noqa - 'generate', 'switch', 'guard' -] +__all__ = ['generate', 'switch', 'guard'] # noqa diff --git a/python/paddle/vision/__init__.py b/python/paddle/vision/__init__.py index 2f0052537e2..3e9b68ef8fd 100644 --- a/python/paddle/vision/__init__.py +++ b/python/paddle/vision/__init__.py @@ -112,6 +112,4 @@ from .transforms import adjust_contrast # noqa: F401 from .transforms import adjust_hue # noqa: F401 from .transforms import normalize # noqa: F401 -__all__ = [ #noqa - 'set_image_backend', 'get_image_backend', 'image_load' -] +__all__ = ['set_image_backend', 'get_image_backend', 'image_load'] # noqa diff --git a/python/paddle/vision/datasets/__init__.py b/python/paddle/vision/datasets/__init__.py index 10666b7c719..970c8cfcae8 100644 --- a/python/paddle/vision/datasets/__init__.py +++ b/python/paddle/vision/datasets/__init__.py @@ -21,7 +21,13 @@ from .cifar import Cifar10 # noqa: F401 from .cifar import Cifar100 # noqa: F401 from .voc2012 import VOC2012 # noqa: F401 -__all__ = [ #noqa - 'DatasetFolder', 'ImageFolder', 'MNIST', 'FashionMNIST', 'Flowers', - 'Cifar10', 'Cifar100', 'VOC2012' +__all__ = [ # noqa + 'DatasetFolder', + 'ImageFolder', + 'MNIST', + 'FashionMNIST', + 'Flowers', + 'Cifar10', + 'Cifar100', + 'VOC2012', ] diff --git a/python/paddle/vision/datasets/cifar.py b/python/paddle/vision/datasets/cifar.py index 1e3eb6515c8..5610b237379 100644 --- a/python/paddle/vision/datasets/cifar.py +++ b/python/paddle/vision/datasets/cifar.py @@ -33,7 +33,7 @@ MODE_FLAG_MAP = { 'train10': 'data_batch', 'test10': 'test_batch', 'train100': 'train', - 'test100': 'test' + 'test100': 'test', } @@ -102,14 +102,18 @@ class Cifar10(Dataset): # [3, 64, 64] 3 """ - def __init__(self, - data_file=None, - mode='train', - transform=None, - download=True, - backend=None): - assert mode.lower() in ['train', 'test'], \ - "mode.lower() should be 'train' or 'test', but got {}".format(mode) + def __init__( + self, + data_file=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + assert mode.lower() in [ + 'train', + 'test', + ], "mode.lower() should be 'train' or 'test', but got {}".format(mode) self.mode = mode.lower() if backend is None: @@ -117,18 +121,21 @@ class Cifar10(Dataset): if backend not in ['pil', 'cv2']: raise ValueError( "Expected backend are one of ['pil', 'cv2'], but got {}".format( - backend)) + backend + ) + ) self.backend = backend self._init_url_md5_flag() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, - self.data_url, - self.data_md5, 'cifar', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, self.data_url, self.data_md5, 'cifar', download + ) self.transform = transform @@ -145,8 +152,9 @@ class Cifar10(Dataset): def _load_data(self): self.data = [] with tarfile.open(self.data_file, mode='r') as f: - names = (each_item.name for each_item in f - if self.flag in each_item.name) + names = ( + each_item.name for each_item in f if self.flag in each_item.name + ) names = sorted(list(names)) @@ -243,14 +251,17 @@ class Cifar100(Cifar10): # [3, 64, 64] 49 """ - def __init__(self, - data_file=None, - mode='train', - transform=None, - download=True, - backend=None): - super(Cifar100, self).__init__(data_file, mode, transform, download, - backend) + def __init__( + self, + data_file=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + super(Cifar100, self).__init__( + data_file, mode, transform, download, backend + ) def _init_url_md5_flag(self): self.data_url = CIFAR100_URL diff --git a/python/paddle/vision/datasets/flowers.py b/python/paddle/vision/datasets/flowers.py index 963d639ed11..1b1c556407b 100644 --- a/python/paddle/vision/datasets/flowers.py +++ b/python/paddle/vision/datasets/flowers.py @@ -106,44 +106,57 @@ class Flowers(Dataset): # [3, 64, 96] [1] """ - def __init__(self, - data_file=None, - label_file=None, - setid_file=None, - mode='train', - transform=None, - download=True, - backend=None): - assert mode.lower() in ['train', 'valid', 'test'], \ - "mode should be 'train', 'valid' or 'test', but got {}".format(mode) + def __init__( + self, + data_file=None, + label_file=None, + setid_file=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + assert mode.lower() in [ + 'train', + 'valid', + 'test', + ], "mode should be 'train', 'valid' or 'test', but got {}".format(mode) if backend is None: backend = paddle.vision.get_image_backend() if backend not in ['pil', 'cv2']: raise ValueError( "Expected backend are one of ['pil', 'cv2'], but got {}".format( - backend)) + backend + ) + ) self.backend = backend flag = MODE_FLAG_MAP[mode.lower()] if not data_file: - assert download, "data_file is not set and downloading automatically is disabled" - data_file = _check_exists_and_download(data_file, DATA_URL, - DATA_MD5, 'flowers', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + data_file = _check_exists_and_download( + data_file, DATA_URL, DATA_MD5, 'flowers', download + ) if not label_file: - assert download, "label_file is not set and downloading automatically is disabled" - label_file = _check_exists_and_download(label_file, LABEL_URL, - LABEL_MD5, 'flowers', - download) + assert ( + download + ), "label_file is not set and downloading automatically is disabled" + label_file = _check_exists_and_download( + label_file, LABEL_URL, LABEL_MD5, 'flowers', download + ) if not setid_file: - assert download, "setid_file is not set and downloading automatically is disabled" - setid_file = _check_exists_and_download(setid_file, SETID_URL, - SETID_MD5, 'flowers', - download) + assert ( + download + ), "setid_file is not set and downloading automatically is disabled" + setid_file = _check_exists_and_download( + setid_file, SETID_URL, SETID_MD5, 'flowers', download + ) self.transform = transform diff --git a/python/paddle/vision/datasets/folder.py b/python/paddle/vision/datasets/folder.py index 4a024fa2779..6ac0c4ca917 100644 --- a/python/paddle/vision/datasets/folder.py +++ b/python/paddle/vision/datasets/folder.py @@ -33,8 +33,9 @@ def has_valid_extension(filename, extensions): Returns: bool: True if the filename ends with one of given extensions """ - assert isinstance(extensions, - (list, tuple)), ("`extensions` must be list or tuple.") + assert isinstance( + extensions, (list, tuple) + ), "`extensions` must be list or tuple." extensions = tuple([x.lower() for x in extensions]) return filename.lower().endswith(extensions) @@ -199,24 +200,29 @@ class DatasetFolder(Dataset): shutil.rmtree(fake_data_dir) """ - def __init__(self, - root, - loader=None, - extensions=None, - transform=None, - is_valid_file=None): + def __init__( + self, + root, + loader=None, + extensions=None, + transform=None, + is_valid_file=None, + ): self.root = root self.transform = transform if extensions is None: extensions = IMG_EXTENSIONS classes, class_to_idx = self._find_classes(self.root) - samples = make_dataset(self.root, class_to_idx, extensions, - is_valid_file) + samples = make_dataset( + self.root, class_to_idx, extensions, is_valid_file + ) if len(samples) == 0: - raise (RuntimeError("Found 0 directories in subfolders of: " + - self.root + "\n" - "Supported extensions are: " + - ",".join(extensions))) + raise ( + RuntimeError( + "Found 0 directories in subfolders of: " + self.root + "\n" + "Supported extensions are: " + ",".join(extensions) + ) + ) self.loader = default_loader if loader is None else loader self.extensions = extensions @@ -245,7 +251,8 @@ class DatasetFolder(Dataset): classes = [d.name for d in os.scandir(dir) if d.is_dir()] else: classes = [ - d for d in os.listdir(dir) + d + for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d)) ] classes.sort() @@ -271,8 +278,17 @@ class DatasetFolder(Dataset): return len(self.samples) -IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', - '.tiff', '.webp') +IMG_EXTENSIONS = ( + '.jpg', + '.jpeg', + '.png', + '.ppm', + '.bmp', + '.pgm', + '.tif', + '.tiff', + '.webp', +) def pil_loader(path): @@ -288,6 +304,7 @@ def cv2_loader(path): def default_loader(path): from paddle.vision import get_image_backend + if get_image_backend() == 'cv2': return cv2_loader(path) else: @@ -412,14 +429,16 @@ class ImageFolder(Dataset): # [3, 64, 64] shutil.rmtree(fake_data_dir) - """ - - def __init__(self, - root, - loader=None, - extensions=None, - transform=None, - is_valid_file=None): + """ + + def __init__( + self, + root, + loader=None, + extensions=None, + transform=None, + is_valid_file=None, + ): self.root = root if extensions is None: extensions = IMG_EXTENSIONS @@ -439,10 +458,12 @@ class ImageFolder(Dataset): samples.append(f) if len(samples) == 0: - raise (RuntimeError("Found 0 files in subfolders of: " + self.root + - "\n" - "Supported extensions are: " + - ",".join(extensions))) + raise ( + RuntimeError( + "Found 0 files in subfolders of: " + self.root + "\n" + "Supported extensions are: " + ",".join(extensions) + ) + ) self.loader = default_loader if loader is None else loader self.extensions = extensions diff --git a/python/paddle/vision/datasets/mnist.py b/python/paddle/vision/datasets/mnist.py index 5b819fb3bb8..c273d41d4a6 100644 --- a/python/paddle/vision/datasets/mnist.py +++ b/python/paddle/vision/datasets/mnist.py @@ -88,6 +88,7 @@ class MNIST(Dataset): print(type(img), img.shape, label) # [1, 28, 28] [7] """ + NAME = 'mnist' URL_PREFIX = 'https://dataset.bj.bcebos.com/mnist/' TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz' @@ -99,40 +100,64 @@ class MNIST(Dataset): TRAIN_LABEL_URL = URL_PREFIX + 'train-labels-idx1-ubyte.gz' TRAIN_LABEL_MD5 = 'd53e105ee54ea40749a09fcbcd1e9432' - def __init__(self, - image_path=None, - label_path=None, - mode='train', - transform=None, - download=True, - backend=None): - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train' or 'test', but got {}".format(mode) + def __init__( + self, + image_path=None, + label_path=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train' or 'test', but got {}".format(mode) if backend is None: backend = paddle.vision.get_image_backend() if backend not in ['pil', 'cv2']: raise ValueError( "Expected backend are one of ['pil', 'cv2'], but got {}".format( - backend)) + backend + ) + ) self.backend = backend self.mode = mode.lower() self.image_path = image_path if self.image_path is None: - assert download, "image_path is not set and downloading automatically is disabled" - image_url = self.TRAIN_IMAGE_URL if mode == 'train' else self.TEST_IMAGE_URL - image_md5 = self.TRAIN_IMAGE_MD5 if mode == 'train' else self.TEST_IMAGE_MD5 + assert ( + download + ), "image_path is not set and downloading automatically is disabled" + image_url = ( + self.TRAIN_IMAGE_URL if mode == 'train' else self.TEST_IMAGE_URL + ) + image_md5 = ( + self.TRAIN_IMAGE_MD5 if mode == 'train' else self.TEST_IMAGE_MD5 + ) self.image_path = _check_exists_and_download( - image_path, image_url, image_md5, self.NAME, download) + image_path, image_url, image_md5, self.NAME, download + ) self.label_path = label_path if self.label_path is None: - assert download, "label_path is not set and downloading automatically is disabled" - label_url = self.TRAIN_LABEL_URL if self.mode == 'train' else self.TEST_LABEL_URL - label_md5 = self.TRAIN_LABEL_MD5 if self.mode == 'train' else self.TEST_LABEL_MD5 + assert ( + download + ), "label_path is not set and downloading automatically is disabled" + label_url = ( + self.TRAIN_LABEL_URL + if self.mode == 'train' + else self.TEST_LABEL_URL + ) + label_md5 = ( + self.TRAIN_LABEL_MD5 + if self.mode == 'train' + else self.TEST_LABEL_MD5 + ) self.label_path = _check_exists_and_download( - label_path, label_url, label_md5, self.NAME, download) + label_path, label_url, label_md5, self.NAME, download + ) self.transform = transform @@ -156,14 +181,16 @@ class MNIST(Dataset): # image file : 16B magic_byte_img = '>IIII' magic_img, image_num, rows, cols = struct.unpack_from( - magic_byte_img, img_buf, offset_img) + magic_byte_img, img_buf, offset_img + ) offset_img += struct.calcsize(magic_byte_img) offset_lab = 0 # label file : 8B magic_byte_lab = '>II' magic_lab, label_num = struct.unpack_from( - magic_byte_lab, lab_buf, offset_lab) + magic_byte_lab, lab_buf, offset_lab + ) offset_lab += struct.calcsize(magic_byte_lab) while True: @@ -175,17 +202,19 @@ class MNIST(Dataset): step_label += buffer_size fmt_images = '>' + str(buffer_size * rows * cols) + 'B' - images_temp = struct.unpack_from(fmt_images, img_buf, - offset_img) + images_temp = struct.unpack_from( + fmt_images, img_buf, offset_img + ) images = np.reshape( - images_temp, - (buffer_size, rows * cols)).astype('float32') + images_temp, (buffer_size, rows * cols) + ).astype('float32') offset_img += struct.calcsize(fmt_images) for i in range(buffer_size): self.images.append(images[i, :]) self.labels.append( - np.array([labels[i]]).astype('int64')) + np.array([labels[i]]).astype('int64') + ) def __getitem__(self, idx): image, label = self.images[idx], self.labels[idx] diff --git a/python/paddle/vision/datasets/voc2012.py b/python/paddle/vision/datasets/voc2012.py index 8931b0f86cc..81478378ae4 100644 --- a/python/paddle/vision/datasets/voc2012.py +++ b/python/paddle/vision/datasets/voc2012.py @@ -102,31 +102,40 @@ class VOC2012(Dataset): # (281, 500) """ - def __init__(self, - data_file=None, - mode='train', - transform=None, - download=True, - backend=None): - assert mode.lower() in ['train', 'valid', 'test'], \ - "mode should be 'train', 'valid' or 'test', but got {}".format(mode) + def __init__( + self, + data_file=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + assert mode.lower() in [ + 'train', + 'valid', + 'test', + ], "mode should be 'train', 'valid' or 'test', but got {}".format(mode) if backend is None: backend = paddle.vision.get_image_backend() if backend not in ['pil', 'cv2']: raise ValueError( "Expected backend are one of ['pil', 'cv2'], but got {}".format( - backend)) + backend + ) + ) self.backend = backend self.flag = MODE_FLAG_MAP[mode.lower()] self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, VOC_URL, - VOC_MD5, CACHE_DIR, - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, VOC_URL, VOC_MD5, CACHE_DIR, download + ) self.transform = transform # read dataset into memory diff --git a/python/paddle/vision/image.py b/python/paddle/vision/image.py index c30b99799c0..eda914d939c 100644 --- a/python/paddle/vision/image.py +++ b/python/paddle/vision/image.py @@ -82,8 +82,10 @@ def set_image_backend(backend): global _image_backend if backend not in ['pil', 'cv2', 'tensor']: raise ValueError( - "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}". - format(backend)) + "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}".format( + backend + ) + ) _image_backend = backend @@ -152,8 +154,10 @@ def image_load(path, backend=None): backend = _image_backend if backend not in ['pil', 'cv2', 'tensor']: raise ValueError( - "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}". - format(backend)) + "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}".format( + backend + ) + ) if backend == 'pil': return Image.open(path) diff --git a/python/paddle/vision/models/__init__.py b/python/paddle/vision/models/__init__.py index 72bb6ee8e8d..08f559bd440 100644 --- a/python/paddle/vision/models/__init__.py +++ b/python/paddle/vision/models/__init__.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from .resnet import ResNet # noqa: F401 from .resnet import resnet18 # noqa: F401 @@ -64,18 +64,56 @@ from .shufflenetv2 import shufflenet_v2_x1_5 # noqa: F401 from .shufflenetv2 import shufflenet_v2_x2_0 # noqa: F401 from .shufflenetv2 import shufflenet_v2_swish # noqa: F401 -__all__ = [ #noqa - 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', - 'resnext50_32x4d', 'resnext50_64x4d', 'resnext101_32x4d', - 'resnext101_64x4d', 'resnext152_32x4d', 'resnext152_64x4d', - 'wide_resnet50_2', 'wide_resnet101_2', 'VGG', 'vgg11', 'vgg13', 'vgg16', - 'vgg19', 'MobileNetV1', 'mobilenet_v1', 'MobileNetV2', 'mobilenet_v2', - 'MobileNetV3Small', 'MobileNetV3Large', 'mobilenet_v3_small', - 'mobilenet_v3_large', 'LeNet', 'DenseNet', 'densenet121', 'densenet161', - 'densenet169', 'densenet201', 'densenet264', 'AlexNet', 'alexnet', - 'InceptionV3', 'inception_v3', 'SqueezeNet', 'squeezenet1_0', - 'squeezenet1_1', 'GoogLeNet', 'googlenet', 'ShuffleNetV2', - 'shufflenet_v2_x0_25', 'shufflenet_v2_x0_33', 'shufflenet_v2_x0_5', - 'shufflenet_v2_x1_0', 'shufflenet_v2_x1_5', 'shufflenet_v2_x2_0', - 'shufflenet_v2_swish' +__all__ = [ # noqa + 'ResNet', + 'resnet18', + 'resnet34', + 'resnet50', + 'resnet101', + 'resnet152', + 'resnext50_32x4d', + 'resnext50_64x4d', + 'resnext101_32x4d', + 'resnext101_64x4d', + 'resnext152_32x4d', + 'resnext152_64x4d', + 'wide_resnet50_2', + 'wide_resnet101_2', + 'VGG', + 'vgg11', + 'vgg13', + 'vgg16', + 'vgg19', + 'MobileNetV1', + 'mobilenet_v1', + 'MobileNetV2', + 'mobilenet_v2', + 'MobileNetV3Small', + 'MobileNetV3Large', + 'mobilenet_v3_small', + 'mobilenet_v3_large', + 'LeNet', + 'DenseNet', + 'densenet121', + 'densenet161', + 'densenet169', + 'densenet201', + 'densenet264', + 'AlexNet', + 'alexnet', + 'InceptionV3', + 'inception_v3', + 'SqueezeNet', + 'squeezenet1_0', + 'squeezenet1_1', + 'GoogLeNet', + 'googlenet', + 'ShuffleNetV2', + 'shufflenet_v2_x0_25', + 'shufflenet_v2_x0_33', + 'shufflenet_v2_x0_5', + 'shufflenet_v2_x1_0', + 'shufflenet_v2_x1_5', + 'shufflenet_v2_x2_0', + 'shufflenet_v2_swish', ] diff --git a/python/paddle/vision/models/alexnet.py b/python/paddle/vision/models/alexnet.py index 634125ba612..c0d1e029938 100644 --- a/python/paddle/vision/models/alexnet.py +++ b/python/paddle/vision/models/alexnet.py @@ -34,16 +34,17 @@ __all__ = [] class ConvPoolLayer(nn.Layer): - - def __init__(self, - input_channels, - output_channels, - filter_size, - stride, - padding, - stdv, - groups=1, - act=None): + def __init__( + self, + input_channels, + output_channels, + filter_size, + stride, + padding, + stdv, + groups=1, + act=None, + ): super(ConvPoolLayer, self).__init__() self.relu = ReLU() if act == "relu" else None @@ -56,7 +57,8 @@ class ConvPoolLayer(nn.Layer): padding=padding, groups=groups, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) self._pool = MaxPool2D(kernel_size=3, stride=2, padding=0) def forward(self, inputs): @@ -109,7 +111,8 @@ class AlexNet(nn.Layer): stride=1, padding=1, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) stdv = 1.0 / math.sqrt(384 * 3 * 3) self._conv4 = Conv2D( 384, @@ -118,7 +121,8 @@ class AlexNet(nn.Layer): stride=1, padding=1, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) stdv = 1.0 / math.sqrt(256 * 3 * 3) self._conv5 = ConvPoolLayer(256, 256, 3, 1, 1, stdv, act="relu") @@ -129,19 +133,22 @@ class AlexNet(nn.Layer): in_features=256 * 6 * 6, out_features=4096, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) self._drop2 = Dropout(p=0.5, mode="downscale_in_infer") self._fc7 = Linear( in_features=4096, out_features=4096, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) self._fc8 = Linear( in_features=4096, out_features=num_classes, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) def forward(self, inputs): x = self._conv1(inputs) @@ -169,10 +176,14 @@ def _alexnet(arch, pretrained, **kwargs): model = AlexNet(**kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.load_dict(param) diff --git a/python/paddle/vision/models/densenet.py b/python/paddle/vision/models/densenet.py index 309f5424ceb..89716701443 100644 --- a/python/paddle/vision/models/densenet.py +++ b/python/paddle/vision/models/densenet.py @@ -25,45 +25,53 @@ from paddle.utils.download import get_weights_path_from_url __all__ = [] model_urls = { - 'densenet121': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet121_pretrained.pdparams', - 'db1b239ed80a905290fd8b01d3af08e4'), - 'densenet161': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet161_pretrained.pdparams', - '62158869cb315098bd25ddbfd308a853'), - 'densenet169': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet169_pretrained.pdparams', - '82cc7c635c3f19098c748850efb2d796'), - 'densenet201': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet201_pretrained.pdparams', - '16ca29565a7712329cf9e36e02caaf58'), - 'densenet264': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet264_pretrained.pdparams', - '3270ce516b85370bba88cfdd9f60bff4'), + 'densenet121': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet121_pretrained.pdparams', + 'db1b239ed80a905290fd8b01d3af08e4', + ), + 'densenet161': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet161_pretrained.pdparams', + '62158869cb315098bd25ddbfd308a853', + ), + 'densenet169': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet169_pretrained.pdparams', + '82cc7c635c3f19098c748850efb2d796', + ), + 'densenet201': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet201_pretrained.pdparams', + '16ca29565a7712329cf9e36e02caaf58', + ), + 'densenet264': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet264_pretrained.pdparams', + '3270ce516b85370bba88cfdd9f60bff4', + ), } class BNACConvLayer(nn.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - pad=0, - groups=1, - act="relu"): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + pad=0, + groups=1, + act="relu", + ): super(BNACConvLayer, self).__init__() self._batch_norm = BatchNorm(num_channels, act=act) - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=pad, - groups=groups, - weight_attr=ParamAttr(), - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=pad, + groups=groups, + weight_attr=ParamAttr(), + bias_attr=False, + ) def forward(self, input): y = self._batch_norm(input) @@ -72,22 +80,25 @@ class BNACConvLayer(nn.Layer): class DenseLayer(nn.Layer): - def __init__(self, num_channels, growth_rate, bn_size, dropout): super(DenseLayer, self).__init__() self.dropout = dropout - self.bn_ac_func1 = BNACConvLayer(num_channels=num_channels, - num_filters=bn_size * growth_rate, - filter_size=1, - pad=0, - stride=1) - - self.bn_ac_func2 = BNACConvLayer(num_channels=bn_size * growth_rate, - num_filters=growth_rate, - filter_size=3, - pad=1, - stride=1) + self.bn_ac_func1 = BNACConvLayer( + num_channels=num_channels, + num_filters=bn_size * growth_rate, + filter_size=1, + pad=0, + stride=1, + ) + + self.bn_ac_func2 = BNACConvLayer( + num_channels=bn_size * growth_rate, + num_filters=growth_rate, + filter_size=3, + pad=1, + stride=1, + ) if dropout: self.dropout_func = Dropout(p=dropout, mode="downscale_in_infer") @@ -102,14 +113,9 @@ class DenseLayer(nn.Layer): class DenseBlock(nn.Layer): - - def __init__(self, - num_channels, - num_layers, - bn_size, - growth_rate, - dropout, - name=None): + def __init__( + self, num_channels, num_layers, bn_size, growth_rate, dropout, name=None + ): super(DenseBlock, self).__init__() self.dropout = dropout self.dense_layer_func = [] @@ -119,10 +125,14 @@ class DenseBlock(nn.Layer): self.dense_layer_func.append( self.add_sublayer( "{}_{}".format(name, layer + 1), - DenseLayer(num_channels=pre_channel, - growth_rate=growth_rate, - bn_size=bn_size, - dropout=dropout))) + DenseLayer( + num_channels=pre_channel, + growth_rate=growth_rate, + bn_size=bn_size, + dropout=dropout, + ), + ) + ) pre_channel = pre_channel + growth_rate def forward(self, input): @@ -133,15 +143,16 @@ class DenseBlock(nn.Layer): class TransitionLayer(nn.Layer): - def __init__(self, num_channels, num_output_features): super(TransitionLayer, self).__init__() - self.conv_ac_func = BNACConvLayer(num_channels=num_channels, - num_filters=num_output_features, - filter_size=1, - pad=0, - stride=1) + self.conv_ac_func = BNACConvLayer( + num_channels=num_channels, + num_filters=num_output_features, + filter_size=1, + pad=0, + stride=1, + ) self.pool2d_avg = AvgPool2D(kernel_size=2, stride=2, padding=0) @@ -152,25 +163,28 @@ class TransitionLayer(nn.Layer): class ConvBNLayer(nn.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - pad=0, - groups=1, - act="relu"): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + pad=0, + groups=1, + act="relu", + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=pad, - groups=groups, - weight_attr=ParamAttr(), - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=pad, + groups=groups, + weight_attr=ParamAttr(), + bias_attr=False, + ) self._batch_norm = BatchNorm(num_filters, act=act) def forward(self, input): @@ -210,34 +224,40 @@ class DenseNet(nn.Layer): # [1, 1000] """ - def __init__(self, - layers=121, - bn_size=4, - dropout=0., - num_classes=1000, - with_pool=True): + def __init__( + self, + layers=121, + bn_size=4, + dropout=0.0, + num_classes=1000, + with_pool=True, + ): super(DenseNet, self).__init__() self.num_classes = num_classes self.with_pool = with_pool supported_layers = [121, 161, 169, 201, 264] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format( - supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) densenet_spec = { 121: (64, 32, [6, 12, 24, 16]), 161: (96, 48, [6, 12, 36, 24]), 169: (64, 32, [6, 12, 32, 32]), 201: (64, 32, [6, 12, 48, 32]), - 264: (64, 32, [6, 12, 64, 48]) + 264: (64, 32, [6, 12, 64, 48]), } num_init_features, growth_rate, block_config = densenet_spec[layers] - self.conv1_func = ConvBNLayer(num_channels=3, - num_filters=num_init_features, - filter_size=7, - stride=2, - pad=3, - act='relu') + self.conv1_func = ConvBNLayer( + num_channels=3, + num_filters=num_init_features, + filter_size=7, + stride=2, + pad=3, + act='relu', + ) self.pool2d_max = MaxPool2D(kernel_size=3, stride=2, padding=1) self.block_config = block_config self.dense_block_func_list = [] @@ -248,12 +268,16 @@ class DenseNet(nn.Layer): self.dense_block_func_list.append( self.add_sublayer( "db_conv_{}".format(i + 2), - DenseBlock(num_channels=pre_num_channels, - num_layers=num_layers, - bn_size=bn_size, - growth_rate=growth_rate, - dropout=dropout, - name='conv' + str(i + 2)))) + DenseBlock( + num_channels=pre_num_channels, + num_layers=num_layers, + bn_size=bn_size, + growth_rate=growth_rate, + dropout=dropout, + name='conv' + str(i + 2), + ), + ) + ) num_features = num_features + num_layers * growth_rate pre_num_channels = num_features @@ -262,8 +286,12 @@ class DenseNet(nn.Layer): self.transition_func_list.append( self.add_sublayer( "tr_conv{}_blk".format(i + 2), - TransitionLayer(num_channels=pre_num_channels, - num_output_features=num_features // 2))) + TransitionLayer( + num_channels=pre_num_channels, + num_output_features=num_features // 2, + ), + ) + ) pre_num_channels = num_features // 2 num_features = num_features // 2 @@ -277,7 +305,8 @@ class DenseNet(nn.Layer): num_features, num_classes, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr()) + bias_attr=ParamAttr(), + ) def forward(self, input): conv = self.conv1_func(input) @@ -303,10 +332,14 @@ class DenseNet(nn.Layer): def _densenet(arch, layers, pretrained, **kwargs): model = DenseNet(layers=layers, **kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) diff --git a/python/paddle/vision/models/googlenet.py b/python/paddle/vision/models/googlenet.py index 4eb02f1de3c..8d6409dc42f 100644 --- a/python/paddle/vision/models/googlenet.py +++ b/python/paddle/vision/models/googlenet.py @@ -25,35 +25,34 @@ from paddle.utils.download import get_weights_path_from_url __all__ = [] model_urls = { - "googlenet": - ("https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/GoogLeNet_pretrained.pdparams", - "80c06f038e905c53ab32c40eca6e26ae") + "googlenet": ( + "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/GoogLeNet_pretrained.pdparams", + "80c06f038e905c53ab32c40eca6e26ae", + ) } def xavier(channels, filter_size): - stdv = (3.0 / (filter_size**2 * channels))**0.5 + stdv = (3.0 / (filter_size**2 * channels)) ** 0.5 param_attr = ParamAttr(initializer=Uniform(-stdv, stdv)) return param_attr class ConvLayer(nn.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1): + def __init__( + self, num_channels, num_filters, filter_size, stride=1, groups=1 + ): super(ConvLayer, self).__init__() - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) def forward(self, inputs): y = self._conv(inputs) @@ -61,9 +60,17 @@ class ConvLayer(nn.Layer): class Inception(nn.Layer): - - def __init__(self, input_channels, output_channels, filter1, filter3R, - filter3, filter5R, filter5, proj): + def __init__( + self, + input_channels, + output_channels, + filter1, + filter3R, + filter3, + filter5R, + filter5, + proj, + ): super(Inception, self).__init__() self._conv1 = ConvLayer(input_channels, filter1, 1) @@ -153,9 +160,9 @@ class GoogLeNet(nn.Layer): if num_classes > 0: # out self._drop = Dropout(p=0.4, mode="downscale_in_infer") - self._fc_out = Linear(1024, - num_classes, - weight_attr=xavier(1024, 1)) + self._fc_out = Linear( + 1024, num_classes, weight_attr=xavier(1024, 1) + ) # out1 self._conv_o1 = ConvLayer(512, 128, 1) @@ -254,9 +261,11 @@ def googlenet(pretrained=False, **kwargs): assert ( arch in model_urls ), "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) diff --git a/python/paddle/vision/models/inceptionv3.py b/python/paddle/vision/models/inceptionv3.py index a4baa2e9a77..24c55bdd757 100644 --- a/python/paddle/vision/models/inceptionv3.py +++ b/python/paddle/vision/models/inceptionv3.py @@ -26,45 +26,55 @@ from ..ops import ConvNormActivation __all__ = [] model_urls = { - "inception_v3": - ("https://paddle-hapi.bj.bcebos.com/models/inception_v3.pdparams", - "649a4547c3243e8b59c656f41fe330b8") + "inception_v3": ( + "https://paddle-hapi.bj.bcebos.com/models/inception_v3.pdparams", + "649a4547c3243e8b59c656f41fe330b8", + ) } class InceptionStem(nn.Layer): - def __init__(self): super().__init__() - self.conv_1a_3x3 = ConvNormActivation(in_channels=3, - out_channels=32, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) - self.conv_2a_3x3 = ConvNormActivation(in_channels=32, - out_channels=32, - kernel_size=3, - stride=1, - padding=0, - activation_layer=nn.ReLU) - self.conv_2b_3x3 = ConvNormActivation(in_channels=32, - out_channels=64, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) + self.conv_1a_3x3 = ConvNormActivation( + in_channels=3, + out_channels=32, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) + self.conv_2a_3x3 = ConvNormActivation( + in_channels=32, + out_channels=32, + kernel_size=3, + stride=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.conv_2b_3x3 = ConvNormActivation( + in_channels=32, + out_channels=64, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) self.max_pool = MaxPool2D(kernel_size=3, stride=2, padding=0) - self.conv_3b_1x1 = ConvNormActivation(in_channels=64, - out_channels=80, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.conv_4a_3x3 = ConvNormActivation(in_channels=80, - out_channels=192, - kernel_size=3, - padding=0, - activation_layer=nn.ReLU) + self.conv_3b_1x1 = ConvNormActivation( + in_channels=64, + out_channels=80, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.conv_4a_3x3 = ConvNormActivation( + in_channels=80, + out_channels=192, + kernel_size=3, + padding=0, + activation_layer=nn.ReLU, + ) def forward(self, x): x = self.conv_1a_3x3(x) @@ -78,51 +88,63 @@ class InceptionStem(nn.Layer): class InceptionA(nn.Layer): - def __init__(self, num_channels, pool_features): super().__init__() - self.branch1x1 = ConvNormActivation(in_channels=num_channels, - out_channels=64, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - - self.branch5x5_1 = ConvNormActivation(in_channels=num_channels, - out_channels=48, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch5x5_2 = ConvNormActivation(in_channels=48, - out_channels=64, - kernel_size=5, - padding=2, - activation_layer=nn.ReLU) - - self.branch3x3dbl_1 = ConvNormActivation(in_channels=num_channels, - out_channels=64, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3dbl_2 = ConvNormActivation(in_channels=64, - out_channels=96, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) - self.branch3x3dbl_3 = ConvNormActivation(in_channels=96, - out_channels=96, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) - - self.branch_pool = AvgPool2D(kernel_size=3, - stride=1, - padding=1, - exclusive=False) - self.branch_pool_conv = ConvNormActivation(in_channels=num_channels, - out_channels=pool_features, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + + self.branch5x5_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=48, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch5x5_2 = ConvNormActivation( + in_channels=48, + out_channels=64, + kernel_size=5, + padding=2, + activation_layer=nn.ReLU, + ) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=64, + out_channels=96, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_3 = ConvNormActivation( + in_channels=96, + out_channels=96, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) + + self.branch_pool = AvgPool2D( + kernel_size=3, stride=1, padding=1, exclusive=False + ) + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=pool_features, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -135,38 +157,46 @@ class InceptionA(nn.Layer): branch_pool = self.branch_pool(x) branch_pool = self.branch_pool_conv(branch_pool) - x = paddle.concat([branch1x1, branch5x5, branch3x3dbl, branch_pool], - axis=1) + x = paddle.concat( + [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=1 + ) return x class InceptionB(nn.Layer): - def __init__(self, num_channels): super().__init__() - self.branch3x3 = ConvNormActivation(in_channels=num_channels, - out_channels=384, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) - - self.branch3x3dbl_1 = ConvNormActivation(in_channels=num_channels, - out_channels=64, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3dbl_2 = ConvNormActivation(in_channels=64, - out_channels=96, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) - self.branch3x3dbl_3 = ConvNormActivation(in_channels=96, - out_channels=96, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) + self.branch3x3 = ConvNormActivation( + in_channels=num_channels, + out_channels=384, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=64, + out_channels=96, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_3 = ConvNormActivation( + in_channels=96, + out_channels=96, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) self.branch_pool = MaxPool2D(kernel_size=3, stride=2) @@ -185,69 +215,87 @@ class InceptionB(nn.Layer): class InceptionC(nn.Layer): - def __init__(self, num_channels, channels_7x7): super().__init__() - self.branch1x1 = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - - self.branch7x7_1 = ConvNormActivation(in_channels=num_channels, - out_channels=channels_7x7, - kernel_size=1, - stride=1, - padding=0, - activation_layer=nn.ReLU) - self.branch7x7_2 = ConvNormActivation(in_channels=channels_7x7, - out_channels=channels_7x7, - kernel_size=(1, 7), - stride=1, - padding=(0, 3), - activation_layer=nn.ReLU) - self.branch7x7_3 = ConvNormActivation(in_channels=channels_7x7, - out_channels=192, - kernel_size=(7, 1), - stride=1, - padding=(3, 0), - activation_layer=nn.ReLU) - - self.branch7x7dbl_1 = ConvNormActivation(in_channels=num_channels, - out_channels=channels_7x7, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch7x7dbl_2 = ConvNormActivation(in_channels=channels_7x7, - out_channels=channels_7x7, - kernel_size=(7, 1), - padding=(3, 0), - activation_layer=nn.ReLU) - self.branch7x7dbl_3 = ConvNormActivation(in_channels=channels_7x7, - out_channels=channels_7x7, - kernel_size=(1, 7), - padding=(0, 3), - activation_layer=nn.ReLU) - self.branch7x7dbl_4 = ConvNormActivation(in_channels=channels_7x7, - out_channels=channels_7x7, - kernel_size=(7, 1), - padding=(3, 0), - activation_layer=nn.ReLU) - self.branch7x7dbl_5 = ConvNormActivation(in_channels=channels_7x7, - out_channels=192, - kernel_size=(1, 7), - padding=(0, 3), - activation_layer=nn.ReLU) - - self.branch_pool = AvgPool2D(kernel_size=3, - stride=1, - padding=1, - exclusive=False) - self.branch_pool_conv = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + + self.branch7x7_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=channels_7x7, + kernel_size=1, + stride=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch7x7_2 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(1, 7), + stride=1, + padding=(0, 3), + activation_layer=nn.ReLU, + ) + self.branch7x7_3 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=192, + kernel_size=(7, 1), + stride=1, + padding=(3, 0), + activation_layer=nn.ReLU, + ) + + self.branch7x7dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=channels_7x7, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch7x7dbl_2 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(7, 1), + padding=(3, 0), + activation_layer=nn.ReLU, + ) + self.branch7x7dbl_3 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(1, 7), + padding=(0, 3), + activation_layer=nn.ReLU, + ) + self.branch7x7dbl_4 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(7, 1), + padding=(3, 0), + activation_layer=nn.ReLU, + ) + self.branch7x7dbl_5 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=192, + kernel_size=(1, 7), + padding=(0, 3), + activation_layer=nn.ReLU, + ) + + self.branch_pool = AvgPool2D( + kernel_size=3, stride=1, padding=1, exclusive=False + ) + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -265,49 +313,61 @@ class InceptionC(nn.Layer): branch_pool = self.branch_pool(x) branch_pool = self.branch_pool_conv(branch_pool) - x = paddle.concat([branch1x1, branch7x7, branch7x7dbl, branch_pool], - axis=1) + x = paddle.concat( + [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=1 + ) return x class InceptionD(nn.Layer): - def __init__(self, num_channels): super().__init__() - self.branch3x3_1 = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3_2 = ConvNormActivation(in_channels=192, - out_channels=320, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) - - self.branch7x7x3_1 = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch7x7x3_2 = ConvNormActivation(in_channels=192, - out_channels=192, - kernel_size=(1, 7), - padding=(0, 3), - activation_layer=nn.ReLU) - self.branch7x7x3_3 = ConvNormActivation(in_channels=192, - out_channels=192, - kernel_size=(7, 1), - padding=(3, 0), - activation_layer=nn.ReLU) - self.branch7x7x3_4 = ConvNormActivation(in_channels=192, - out_channels=192, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) + self.branch3x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3_2 = ConvNormActivation( + in_channels=192, + out_channels=320, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) + + self.branch7x7x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch7x7x3_2 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=(1, 7), + padding=(0, 3), + activation_layer=nn.ReLU, + ) + self.branch7x7x3_3 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=(7, 1), + padding=(3, 0), + activation_layer=nn.ReLU, + ) + self.branch7x7x3_4 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) self.branch_pool = MaxPool2D(kernel_size=3, stride=2) @@ -327,60 +387,76 @@ class InceptionD(nn.Layer): class InceptionE(nn.Layer): - def __init__(self, num_channels): super().__init__() - self.branch1x1 = ConvNormActivation(in_channels=num_channels, - out_channels=320, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3_1 = ConvNormActivation(in_channels=num_channels, - out_channels=384, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3_2a = ConvNormActivation(in_channels=384, - out_channels=384, - kernel_size=(1, 3), - padding=(0, 1), - activation_layer=nn.ReLU) - self.branch3x3_2b = ConvNormActivation(in_channels=384, - out_channels=384, - kernel_size=(3, 1), - padding=(1, 0), - activation_layer=nn.ReLU) - - self.branch3x3dbl_1 = ConvNormActivation(in_channels=num_channels, - out_channels=448, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3dbl_2 = ConvNormActivation(in_channels=448, - out_channels=384, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) - self.branch3x3dbl_3a = ConvNormActivation(in_channels=384, - out_channels=384, - kernel_size=(1, 3), - padding=(0, 1), - activation_layer=nn.ReLU) - self.branch3x3dbl_3b = ConvNormActivation(in_channels=384, - out_channels=384, - kernel_size=(3, 1), - padding=(1, 0), - activation_layer=nn.ReLU) - - self.branch_pool = AvgPool2D(kernel_size=3, - stride=1, - padding=1, - exclusive=False) - self.branch_pool_conv = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=320, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=384, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3_2a = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(1, 3), + padding=(0, 1), + activation_layer=nn.ReLU, + ) + self.branch3x3_2b = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(3, 1), + padding=(1, 0), + activation_layer=nn.ReLU, + ) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=448, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=448, + out_channels=384, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_3a = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(1, 3), + padding=(0, 1), + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_3b = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(3, 1), + padding=(1, 0), + activation_layer=nn.ReLU, + ) + + self.branch_pool = AvgPool2D( + kernel_size=3, stride=1, padding=1, exclusive=False + ) + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -403,8 +479,9 @@ class InceptionE(nn.Layer): branch_pool = self.branch_pool(x) branch_pool = self.branch_pool_conv(branch_pool) - x = paddle.concat([branch1x1, branch3x3, branch3x3dbl, branch_pool], - axis=1) + x = paddle.concat( + [branch1x1, branch3x3, branch3x3dbl, branch_pool], axis=1 + ) return x @@ -444,7 +521,7 @@ class InceptionV3(nn.Layer): "inception_b": [288], "inception_c": [[768, 768, 768, 768], [128, 160, 160, 192]], "inception_d": [768], - "inception_e": [1280, 2048] + "inception_e": [1280, 2048], } inception_a_list = self.layers_config["inception_a"] @@ -457,8 +534,9 @@ class InceptionV3(nn.Layer): self.inception_block_list = nn.LayerList() for i in range(len(inception_a_list[0])): - inception_a = InceptionA(inception_a_list[0][i], - inception_a_list[1][i]) + inception_a = InceptionA( + inception_a_list[0][i], inception_a_list[1][i] + ) self.inception_block_list.append(inception_a) for i in range(len(inception_b_list)): @@ -466,8 +544,9 @@ class InceptionV3(nn.Layer): self.inception_block_list.append(inception_b) for i in range(len(inception_c_list[0])): - inception_c = InceptionC(inception_c_list[0][i], - inception_c_list[1][i]) + inception_c = InceptionC( + inception_c_list[0][i], inception_c_list[1][i] + ) self.inception_block_list.append(inception_c) for i in range(len(inception_d_list)): @@ -488,7 +567,8 @@ class InceptionV3(nn.Layer): 2048, num_classes, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr()) + bias_attr=ParamAttr(), + ) def forward(self, x): x = self.inception_stem(x) @@ -538,10 +618,14 @@ def inception_v3(pretrained=False, **kwargs): model = InceptionV3(**kwargs) arch = "inception_v3" if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) diff --git a/python/paddle/vision/models/lenet.py b/python/paddle/vision/models/lenet.py index 44cfed2ef30..49c9bd2d68a 100644 --- a/python/paddle/vision/models/lenet.py +++ b/python/paddle/vision/models/lenet.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import paddle import paddle.nn as nn @@ -47,14 +47,21 @@ class LeNet(nn.Layer): def __init__(self, num_classes=10): super(LeNet, self).__init__() self.num_classes = num_classes - self.features = nn.Sequential(nn.Conv2D(1, 6, 3, stride=1, padding=1), - nn.ReLU(), nn.MaxPool2D(2, 2), - nn.Conv2D(6, 16, 5, stride=1, padding=0), - nn.ReLU(), nn.MaxPool2D(2, 2)) + self.features = nn.Sequential( + nn.Conv2D(1, 6, 3, stride=1, padding=1), + nn.ReLU(), + nn.MaxPool2D(2, 2), + nn.Conv2D(6, 16, 5, stride=1, padding=0), + nn.ReLU(), + nn.MaxPool2D(2, 2), + ) if num_classes > 0: - self.fc = nn.Sequential(nn.Linear(400, 120), nn.Linear(120, 84), - nn.Linear(84, num_classes)) + self.fc = nn.Sequential( + nn.Linear(400, 120), + nn.Linear(120, 84), + nn.Linear(84, num_classes), + ) def forward(self, inputs): x = self.features(inputs) diff --git a/python/paddle/vision/models/mobilenetv1.py b/python/paddle/vision/models/mobilenetv1.py index 22be953f2c1..78e83304db7 100644 --- a/python/paddle/vision/models/mobilenetv1.py +++ b/python/paddle/vision/models/mobilenetv1.py @@ -21,31 +21,41 @@ from ..ops import ConvNormActivation __all__ = [] model_urls = { - 'mobilenetv1_1.0': - ('https://paddle-hapi.bj.bcebos.com/models/mobilenetv1_1.0.pdparams', - '3033ab1975b1670bef51545feb65fc45') + 'mobilenetv1_1.0': ( + 'https://paddle-hapi.bj.bcebos.com/models/mobilenetv1_1.0.pdparams', + '3033ab1975b1670bef51545feb65fc45', + ) } class DepthwiseSeparable(nn.Layer): - - def __init__(self, in_channels, out_channels1, out_channels2, num_groups, - stride, scale): + def __init__( + self, + in_channels, + out_channels1, + out_channels2, + num_groups, + stride, + scale, + ): super(DepthwiseSeparable, self).__init__() - self._depthwise_conv = ConvNormActivation(in_channels, - int(out_channels1 * scale), - kernel_size=3, - stride=stride, - padding=1, - groups=int(num_groups * - scale)) - - self._pointwise_conv = ConvNormActivation(int(out_channels1 * scale), - int(out_channels2 * scale), - kernel_size=1, - stride=1, - padding=0) + self._depthwise_conv = ConvNormActivation( + in_channels, + int(out_channels1 * scale), + kernel_size=3, + stride=stride, + padding=1, + groups=int(num_groups * scale), + ) + + self._pointwise_conv = ConvNormActivation( + int(out_channels1 * scale), + int(out_channels2 * scale), + kernel_size=1, + stride=1, + padding=0, + ) def forward(self, x): x = self._depthwise_conv(x) @@ -88,101 +98,130 @@ class MobileNetV1(nn.Layer): self.num_classes = num_classes self.with_pool = with_pool - self.conv1 = ConvNormActivation(in_channels=3, - out_channels=int(32 * scale), - kernel_size=3, - stride=2, - padding=1) - - dws21 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 32 * scale), - out_channels1=32, - out_channels2=64, - num_groups=32, - stride=1, - scale=scale), - name="conv2_1") + self.conv1 = ConvNormActivation( + in_channels=3, + out_channels=int(32 * scale), + kernel_size=3, + stride=2, + padding=1, + ) + + dws21 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(32 * scale), + out_channels1=32, + out_channels2=64, + num_groups=32, + stride=1, + scale=scale, + ), + name="conv2_1", + ) self.dwsl.append(dws21) - dws22 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 64 * scale), - out_channels1=64, - out_channels2=128, - num_groups=64, - stride=2, - scale=scale), - name="conv2_2") + dws22 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(64 * scale), + out_channels1=64, + out_channels2=128, + num_groups=64, + stride=2, + scale=scale, + ), + name="conv2_2", + ) self.dwsl.append(dws22) - dws31 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 128 * scale), - out_channels1=128, - out_channels2=128, - num_groups=128, - stride=1, - scale=scale), - name="conv3_1") + dws31 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(128 * scale), + out_channels1=128, + out_channels2=128, + num_groups=128, + stride=1, + scale=scale, + ), + name="conv3_1", + ) self.dwsl.append(dws31) - dws32 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 128 * scale), - out_channels1=128, - out_channels2=256, - num_groups=128, - stride=2, - scale=scale), - name="conv3_2") + dws32 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(128 * scale), + out_channels1=128, + out_channels2=256, + num_groups=128, + stride=2, + scale=scale, + ), + name="conv3_2", + ) self.dwsl.append(dws32) - dws41 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 256 * scale), - out_channels1=256, - out_channels2=256, - num_groups=256, - stride=1, - scale=scale), - name="conv4_1") + dws41 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(256 * scale), + out_channels1=256, + out_channels2=256, + num_groups=256, + stride=1, + scale=scale, + ), + name="conv4_1", + ) self.dwsl.append(dws41) - dws42 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 256 * scale), - out_channels1=256, - out_channels2=512, - num_groups=256, - stride=2, - scale=scale), - name="conv4_2") + dws42 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(256 * scale), + out_channels1=256, + out_channels2=512, + num_groups=256, + stride=2, + scale=scale, + ), + name="conv4_2", + ) self.dwsl.append(dws42) for i in range(5): - tmp = self.add_sublayer(sublayer=DepthwiseSeparable( + tmp = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(512 * scale), + out_channels1=512, + out_channels2=512, + num_groups=512, + stride=1, + scale=scale, + ), + name="conv5_" + str(i + 1), + ) + self.dwsl.append(tmp) + + dws56 = self.add_sublayer( + sublayer=DepthwiseSeparable( in_channels=int(512 * scale), out_channels1=512, - out_channels2=512, + out_channels2=1024, num_groups=512, - stride=1, - scale=scale), - name="conv5_" + str(i + 1)) - self.dwsl.append(tmp) - - dws56 = self.add_sublayer(sublayer=DepthwiseSeparable( - in_channels=int(512 * scale), - out_channels1=512, - out_channels2=1024, - num_groups=512, - stride=2, - scale=scale), - name="conv5_6") + stride=2, + scale=scale, + ), + name="conv5_6", + ) self.dwsl.append(dws56) - dws6 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 1024 * scale), - out_channels1=1024, - out_channels2=1024, - num_groups=1024, - stride=1, - scale=scale), - name="conv6") + dws6 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(1024 * scale), + out_channels1=1024, + out_channels2=1024, + num_groups=1024, + stride=1, + scale=scale, + ), + name="conv6", + ) self.dwsl.append(dws6) if with_pool: @@ -208,10 +247,14 @@ class MobileNetV1(nn.Layer): def _mobilenet(arch, pretrained=False, **kwargs): model = MobileNetV1(**kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.load_dict(param) @@ -253,8 +296,7 @@ def mobilenet_v1(pretrained=False, scale=1.0, **kwargs): print(out.shape) # [1, 1000] """ - model = _mobilenet('mobilenetv1_' + str(scale), - pretrained, - scale=scale, - **kwargs) + model = _mobilenet( + 'mobilenetv1_' + str(scale), pretrained, scale=scale, **kwargs + ) return model diff --git a/python/paddle/vision/models/mobilenetv2.py b/python/paddle/vision/models/mobilenetv2.py index f67ea54fa63..43fade9e6dd 100644 --- a/python/paddle/vision/models/mobilenetv2.py +++ b/python/paddle/vision/models/mobilenetv2.py @@ -22,20 +22,17 @@ from ..ops import ConvNormActivation __all__ = [] model_urls = { - 'mobilenetv2_1.0': - ('https://paddle-hapi.bj.bcebos.com/models/mobilenet_v2_x1.0.pdparams', - '0340af0a901346c8d46f4529882fb63d') + 'mobilenetv2_1.0': ( + 'https://paddle-hapi.bj.bcebos.com/models/mobilenet_v2_x1.0.pdparams', + '0340af0a901346c8d46f4529882fb63d', + ) } class InvertedResidual(nn.Layer): - - def __init__(self, - inp, - oup, - stride, - expand_ratio, - norm_layer=nn.BatchNorm2D): + def __init__( + self, inp, oup, stride, expand_ratio, norm_layer=nn.BatchNorm2D + ): super(InvertedResidual, self).__init__() self.stride = stride assert stride in [1, 2] @@ -46,21 +43,28 @@ class InvertedResidual(nn.Layer): layers = [] if expand_ratio != 1: layers.append( - ConvNormActivation(inp, - hidden_dim, - kernel_size=1, - norm_layer=norm_layer, - activation_layer=nn.ReLU6)) - layers.extend([ - ConvNormActivation(hidden_dim, - hidden_dim, - stride=stride, - groups=hidden_dim, - norm_layer=norm_layer, - activation_layer=nn.ReLU6), - nn.Conv2D(hidden_dim, oup, 1, 1, 0, bias_attr=False), - norm_layer(oup), - ]) + ConvNormActivation( + inp, + hidden_dim, + kernel_size=1, + norm_layer=norm_layer, + activation_layer=nn.ReLU6, + ) + ) + layers.extend( + [ + ConvNormActivation( + hidden_dim, + hidden_dim, + stride=stride, + groups=hidden_dim, + norm_layer=norm_layer, + activation_layer=nn.ReLU6, + ), + nn.Conv2D(hidden_dim, oup, 1, 1, 0, bias_attr=False), + norm_layer(oup), + ] + ) self.conv = nn.Sequential(*layers) def forward(self, x): @@ -119,14 +123,17 @@ class MobileNetV2(nn.Layer): ] input_channel = _make_divisible(input_channel * scale, round_nearest) - self.last_channel = _make_divisible(last_channel * max(1.0, scale), - round_nearest) + self.last_channel = _make_divisible( + last_channel * max(1.0, scale), round_nearest + ) features = [ - ConvNormActivation(3, - input_channel, - stride=2, - norm_layer=norm_layer, - activation_layer=nn.ReLU6) + ConvNormActivation( + 3, + input_channel, + stride=2, + norm_layer=norm_layer, + activation_layer=nn.ReLU6, + ) ] for t, c, n, s in inverted_residual_setting: @@ -134,19 +141,25 @@ class MobileNetV2(nn.Layer): for i in range(n): stride = s if i == 0 else 1 features.append( - block(input_channel, - output_channel, - stride, - expand_ratio=t, - norm_layer=norm_layer)) + block( + input_channel, + output_channel, + stride, + expand_ratio=t, + norm_layer=norm_layer, + ) + ) input_channel = output_channel features.append( - ConvNormActivation(input_channel, - self.last_channel, - kernel_size=1, - norm_layer=norm_layer, - activation_layer=nn.ReLU6)) + ConvNormActivation( + input_channel, + self.last_channel, + kernel_size=1, + norm_layer=norm_layer, + activation_layer=nn.ReLU6, + ) + ) self.features = nn.Sequential(*features) @@ -155,7 +168,8 @@ class MobileNetV2(nn.Layer): if self.num_classes > 0: self.classifier = nn.Sequential( - nn.Dropout(0.2), nn.Linear(self.last_channel, num_classes)) + nn.Dropout(0.2), nn.Linear(self.last_channel, num_classes) + ) def forward(self, x): x = self.features(x) @@ -172,10 +186,14 @@ class MobileNetV2(nn.Layer): def _mobilenet(arch, pretrained=False, **kwargs): model = MobileNetV2(**kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.load_dict(param) @@ -217,8 +235,7 @@ def mobilenet_v2(pretrained=False, scale=1.0, **kwargs): print(out.shape) # [1, 1000] """ - model = _mobilenet('mobilenetv2_' + str(scale), - pretrained, - scale=scale, - **kwargs) + model = _mobilenet( + 'mobilenetv2_' + str(scale), pretrained, scale=scale, **kwargs + ) return model diff --git a/python/paddle/vision/models/mobilenetv3.py b/python/paddle/vision/models/mobilenetv3.py index f1bf6d8ae37..865f8d0dd32 100644 --- a/python/paddle/vision/models/mobilenetv3.py +++ b/python/paddle/vision/models/mobilenetv3.py @@ -23,12 +23,14 @@ from ..ops import ConvNormActivation __all__ = [] model_urls = { - "mobilenet_v3_small_x1.0": - ("https://paddle-hapi.bj.bcebos.com/models/mobilenet_v3_small_x1.0.pdparams", - "34fe0e7c1f8b00b2b056ad6788d0590c"), - "mobilenet_v3_large_x1.0": - ("https://paddle-hapi.bj.bcebos.com/models/mobilenet_v3_large_x1.0.pdparams", - "118db5792b4e183b925d8e8e334db3df"), + "mobilenet_v3_small_x1.0": ( + "https://paddle-hapi.bj.bcebos.com/models/mobilenet_v3_small_x1.0.pdparams", + "34fe0e7c1f8b00b2b056ad6788d0590c", + ), + "mobilenet_v3_large_x1.0": ( + "https://paddle-hapi.bj.bcebos.com/models/mobilenet_v3_large_x1.0.pdparams", + "118db5792b4e183b925d8e8e334db3df", + ), } @@ -45,11 +47,13 @@ class SqueezeExcitation(nn.Layer): scale_activation (Callable[..., paddle.nn.Layer]): ``sigma`` activation. Default: ``paddle.nn.Sigmoid`` """ - def __init__(self, - input_channels, - squeeze_channels, - activation=nn.ReLU, - scale_activation=nn.Sigmoid): + def __init__( + self, + input_channels, + squeeze_channels, + activation=nn.ReLU, + scale_activation=nn.Sigmoid, + ): super().__init__() self.avgpool = nn.AdaptiveAvgPool2D(1) self.fc1 = nn.Conv2D(input_channels, squeeze_channels, 1) @@ -70,20 +74,22 @@ class SqueezeExcitation(nn.Layer): class InvertedResidualConfig: - - def __init__(self, - in_channels, - kernel, - expanded_channels, - out_channels, - use_se, - activation, - stride, - scale=1.0): + def __init__( + self, + in_channels, + kernel, + expanded_channels, + out_channels, + use_se, + activation, + stride, + scale=1.0, + ): self.in_channels = self.adjust_channels(in_channels, scale=scale) self.kernel = kernel - self.expanded_channels = self.adjust_channels(expanded_channels, - scale=scale) + self.expanded_channels = self.adjust_channels( + expanded_channels, scale=scale + ) self.out_channels = self.adjust_channels(out_channels, scale=scale) self.use_se = use_se if activation is None: @@ -95,7 +101,9 @@ class InvertedResidualConfig: else: raise RuntimeError( "The activation function is not supported: {}".format( - activation)) + activation + ) + ) self.stride = stride @staticmethod @@ -104,9 +112,17 @@ class InvertedResidualConfig: class InvertedResidual(nn.Layer): - - def __init__(self, in_channels, expanded_channels, out_channels, - filter_size, stride, use_se, activation_layer, norm_layer): + def __init__( + self, + in_channels, + expanded_channels, + out_channels, + filter_size, + stride, + use_se, + activation_layer, + norm_layer, + ): super().__init__() self.use_res_connect = stride == 1 and in_channels == out_channels self.use_se = use_se @@ -120,7 +136,8 @@ class InvertedResidual(nn.Layer): stride=1, padding=0, norm_layer=norm_layer, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) self.bottleneck_conv = ConvNormActivation( in_channels=expanded_channels, @@ -130,21 +147,25 @@ class InvertedResidual(nn.Layer): padding=int((filter_size - 1) // 2), groups=expanded_channels, norm_layer=norm_layer, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) if self.use_se: - self.mid_se = SqueezeExcitation(expanded_channels, - _make_divisible(expanded_channels // - 4), - scale_activation=nn.Hardsigmoid) - - self.linear_conv = ConvNormActivation(in_channels=expanded_channels, - out_channels=out_channels, - kernel_size=1, - stride=1, - padding=0, - norm_layer=norm_layer, - activation_layer=None) + self.mid_se = SqueezeExcitation( + expanded_channels, + _make_divisible(expanded_channels // 4), + scale_activation=nn.Hardsigmoid, + ) + + self.linear_conv = ConvNormActivation( + in_channels=expanded_channels, + out_channels=out_channels, + kernel_size=1, + stride=1, + padding=0, + norm_layer=norm_layer, + activation_layer=None, + ) def forward(self, x): identity = x @@ -172,12 +193,9 @@ class MobileNetV3(nn.Layer): with_pool (bool, optional): Use pool before the last fc layer or not. Default: True. """ - def __init__(self, - config, - last_channel, - scale=1.0, - num_classes=1000, - with_pool=True): + def __init__( + self, config, last_channel, scale=1.0, num_classes=1000, with_pool=True + ): super().__init__() self.config = config @@ -190,25 +208,32 @@ class MobileNetV3(nn.Layer): self.lastconv_out_channels = self.lastconv_in_channels * 6 norm_layer = partial(nn.BatchNorm2D, epsilon=0.001, momentum=0.99) - self.conv = ConvNormActivation(in_channels=3, - out_channels=self.firstconv_in_channels, - kernel_size=3, - stride=2, - padding=1, - groups=1, - activation_layer=nn.Hardswish, - norm_layer=norm_layer) - - self.blocks = nn.Sequential(*[ - InvertedResidual(in_channels=cfg.in_channels, - expanded_channels=cfg.expanded_channels, - out_channels=cfg.out_channels, - filter_size=cfg.kernel, - stride=cfg.stride, - use_se=cfg.use_se, - activation_layer=cfg.activation_layer, - norm_layer=norm_layer) for cfg in self.config - ]) + self.conv = ConvNormActivation( + in_channels=3, + out_channels=self.firstconv_in_channels, + kernel_size=3, + stride=2, + padding=1, + groups=1, + activation_layer=nn.Hardswish, + norm_layer=norm_layer, + ) + + self.blocks = nn.Sequential( + *[ + InvertedResidual( + in_channels=cfg.in_channels, + expanded_channels=cfg.expanded_channels, + out_channels=cfg.out_channels, + filter_size=cfg.kernel, + stride=cfg.stride, + use_se=cfg.use_se, + activation_layer=cfg.activation_layer, + norm_layer=norm_layer, + ) + for cfg in self.config + ] + ) self.lastconv = ConvNormActivation( in_channels=self.lastconv_in_channels, @@ -218,7 +243,8 @@ class MobileNetV3(nn.Layer): padding=0, groups=1, norm_layer=norm_layer, - activation_layer=nn.Hardswish) + activation_layer=nn.Hardswish, + ) if with_pool: self.avgpool = nn.AdaptiveAvgPool2D(1) @@ -226,8 +252,10 @@ class MobileNetV3(nn.Layer): if num_classes > 0: self.classifier = nn.Sequential( nn.Linear(self.lastconv_out_channels, self.last_channel), - nn.Hardswish(), nn.Dropout(p=0.2), - nn.Linear(self.last_channel, num_classes)) + nn.Hardswish(), + nn.Dropout(p=0.2), + nn.Linear(self.last_channel, num_classes), + ) def forward(self, x): x = self.conv(x) @@ -288,11 +316,13 @@ class MobileNetV3Small(MobileNetV3): InvertedResidualConfig(96, 5, 576, 96, True, "hardswish", 1, scale), ] last_channel = _make_divisible(1024 * scale, 8) - super().__init__(config, - last_channel=last_channel, - scale=scale, - with_pool=with_pool, - num_classes=num_classes) + super().__init__( + config, + last_channel=last_channel, + scale=scale, + with_pool=with_pool, + num_classes=num_classes, + ) class MobileNetV3Large(MobileNetV3): @@ -332,31 +362,42 @@ class MobileNetV3Large(MobileNetV3): InvertedResidualConfig(24, 5, 72, 40, True, "relu", 2, scale), InvertedResidualConfig(40, 5, 120, 40, True, "relu", 1, scale), InvertedResidualConfig(40, 5, 120, 40, True, "relu", 1, scale), - InvertedResidualConfig(40, 3, 240, 80, False, "hardswish", 2, - scale), - InvertedResidualConfig(80, 3, 200, 80, False, "hardswish", 1, - scale), - InvertedResidualConfig(80, 3, 184, 80, False, "hardswish", 1, - scale), - InvertedResidualConfig(80, 3, 184, 80, False, "hardswish", 1, - scale), - InvertedResidualConfig(80, 3, 480, 112, True, "hardswish", 1, - scale), - InvertedResidualConfig(112, 3, 672, 112, True, "hardswish", 1, - scale), - InvertedResidualConfig(112, 5, 672, 160, True, "hardswish", 2, - scale), - InvertedResidualConfig(160, 5, 960, 160, True, "hardswish", 1, - scale), - InvertedResidualConfig(160, 5, 960, 160, True, "hardswish", 1, - scale), + InvertedResidualConfig( + 40, 3, 240, 80, False, "hardswish", 2, scale + ), + InvertedResidualConfig( + 80, 3, 200, 80, False, "hardswish", 1, scale + ), + InvertedResidualConfig( + 80, 3, 184, 80, False, "hardswish", 1, scale + ), + InvertedResidualConfig( + 80, 3, 184, 80, False, "hardswish", 1, scale + ), + InvertedResidualConfig( + 80, 3, 480, 112, True, "hardswish", 1, scale + ), + InvertedResidualConfig( + 112, 3, 672, 112, True, "hardswish", 1, scale + ), + InvertedResidualConfig( + 112, 5, 672, 160, True, "hardswish", 2, scale + ), + InvertedResidualConfig( + 160, 5, 960, 160, True, "hardswish", 1, scale + ), + InvertedResidualConfig( + 160, 5, 960, 160, True, "hardswish", 1, scale + ), ] last_channel = _make_divisible(1280 * scale, 8) - super().__init__(config, - last_channel=last_channel, - scale=scale, - with_pool=with_pool, - num_classes=num_classes) + super().__init__( + config, + last_channel=last_channel, + scale=scale, + with_pool=with_pool, + num_classes=num_classes, + ) def _mobilenet_v3(arch, pretrained=False, scale=1.0, **kwargs): @@ -369,9 +410,11 @@ def _mobilenet_v3(arch, pretrained=False, scale=1.0, **kwargs): assert ( arch in model_urls ), "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) @@ -412,10 +455,9 @@ def mobilenet_v3_small(pretrained=False, scale=1.0, **kwargs): print(out.shape) # [1, 1000] """ - model = _mobilenet_v3("mobilenet_v3_small", - scale=scale, - pretrained=pretrained, - **kwargs) + model = _mobilenet_v3( + "mobilenet_v3_small", scale=scale, pretrained=pretrained, **kwargs + ) return model @@ -453,8 +495,7 @@ def mobilenet_v3_large(pretrained=False, scale=1.0, **kwargs): print(out.shape) # [1, 1000] """ - model = _mobilenet_v3("mobilenet_v3_large", - scale=scale, - pretrained=pretrained, - **kwargs) + model = _mobilenet_v3( + "mobilenet_v3_large", scale=scale, pretrained=pretrained, **kwargs + ) return model diff --git a/python/paddle/vision/models/resnet.py b/python/paddle/vision/models/resnet.py index 4293e119582..f43bdb9bc1a 100644 --- a/python/paddle/vision/models/resnet.py +++ b/python/paddle/vision/models/resnet.py @@ -20,69 +20,87 @@ from paddle.utils.download import get_weights_path_from_url __all__ = [] model_urls = { - 'resnet18': ('https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams', - 'cf548f46534aa3560945be4b95cd11c4'), - 'resnet34': ('https://paddle-hapi.bj.bcebos.com/models/resnet34.pdparams', - '8d2275cf8706028345f78ac0e1d31969'), - 'resnet50': ('https://paddle-hapi.bj.bcebos.com/models/resnet50.pdparams', - 'ca6f485ee1ab0492d38f323885b0ad80'), - 'resnet101': ('https://paddle-hapi.bj.bcebos.com/models/resnet101.pdparams', - '02f35f034ca3858e1e54d4036443c92d'), - 'resnet152': ('https://paddle-hapi.bj.bcebos.com/models/resnet152.pdparams', - '7ad16a2f1e7333859ff986138630fd7a'), - 'resnext50_32x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext50_32x4d.pdparams', - 'dc47483169be7d6f018fcbb7baf8775d'), - "resnext50_64x4d": - ('https://paddle-hapi.bj.bcebos.com/models/resnext50_64x4d.pdparams', - '063d4b483e12b06388529450ad7576db'), - 'resnext101_32x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext101_32x4d.pdparams', - '967b090039f9de2c8d06fe994fb9095f'), - 'resnext101_64x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext101_64x4d.pdparams', - '98e04e7ca616a066699230d769d03008'), - 'resnext152_32x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext152_32x4d.pdparams', - '18ff0beee21f2efc99c4b31786107121'), - 'resnext152_64x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext152_64x4d.pdparams', - '77c4af00ca42c405fa7f841841959379'), - 'wide_resnet50_2': - ('https://paddle-hapi.bj.bcebos.com/models/wide_resnet50_2.pdparams', - '0282f804d73debdab289bd9fea3fa6dc'), - 'wide_resnet101_2': - ('https://paddle-hapi.bj.bcebos.com/models/wide_resnet101_2.pdparams', - 'd4360a2d23657f059216f5d5a1a9ac93'), + 'resnet18': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams', + 'cf548f46534aa3560945be4b95cd11c4', + ), + 'resnet34': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet34.pdparams', + '8d2275cf8706028345f78ac0e1d31969', + ), + 'resnet50': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet50.pdparams', + 'ca6f485ee1ab0492d38f323885b0ad80', + ), + 'resnet101': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet101.pdparams', + '02f35f034ca3858e1e54d4036443c92d', + ), + 'resnet152': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet152.pdparams', + '7ad16a2f1e7333859ff986138630fd7a', + ), + 'resnext50_32x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext50_32x4d.pdparams', + 'dc47483169be7d6f018fcbb7baf8775d', + ), + "resnext50_64x4d": ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext50_64x4d.pdparams', + '063d4b483e12b06388529450ad7576db', + ), + 'resnext101_32x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext101_32x4d.pdparams', + '967b090039f9de2c8d06fe994fb9095f', + ), + 'resnext101_64x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext101_64x4d.pdparams', + '98e04e7ca616a066699230d769d03008', + ), + 'resnext152_32x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext152_32x4d.pdparams', + '18ff0beee21f2efc99c4b31786107121', + ), + 'resnext152_64x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext152_64x4d.pdparams', + '77c4af00ca42c405fa7f841841959379', + ), + 'wide_resnet50_2': ( + 'https://paddle-hapi.bj.bcebos.com/models/wide_resnet50_2.pdparams', + '0282f804d73debdab289bd9fea3fa6dc', + ), + 'wide_resnet101_2': ( + 'https://paddle-hapi.bj.bcebos.com/models/wide_resnet101_2.pdparams', + 'd4360a2d23657f059216f5d5a1a9ac93', + ), } class BasicBlock(nn.Layer): expansion = 1 - def __init__(self, - inplanes, - planes, - stride=1, - downsample=None, - groups=1, - base_width=64, - dilation=1, - norm_layer=None): + def __init__( + self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None, + ): super(BasicBlock, self).__init__() if norm_layer is None: norm_layer = nn.BatchNorm2D if dilation > 1: raise NotImplementedError( - "Dilation > 1 not supported in BasicBlock") - - self.conv1 = nn.Conv2D(inplanes, - planes, - 3, - padding=1, - stride=stride, - bias_attr=False) + "Dilation > 1 not supported in BasicBlock" + ) + + self.conv1 = nn.Conv2D( + inplanes, planes, 3, padding=1, stride=stride, bias_attr=False + ) self.bn1 = norm_layer(planes) self.relu = nn.ReLU() self.conv2 = nn.Conv2D(planes, planes, 3, padding=1, bias_attr=False) @@ -113,37 +131,40 @@ class BottleneckBlock(nn.Layer): expansion = 4 - def __init__(self, - inplanes, - planes, - stride=1, - downsample=None, - groups=1, - base_width=64, - dilation=1, - norm_layer=None): + def __init__( + self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None, + ): super(BottleneckBlock, self).__init__() if norm_layer is None: norm_layer = nn.BatchNorm2D - width = int(planes * (base_width / 64.)) * groups + width = int(planes * (base_width / 64.0)) * groups self.conv1 = nn.Conv2D(inplanes, width, 1, bias_attr=False) self.bn1 = norm_layer(width) - self.conv2 = nn.Conv2D(width, - width, - 3, - padding=dilation, - stride=stride, - groups=groups, - dilation=dilation, - bias_attr=False) + self.conv2 = nn.Conv2D( + width, + width, + 3, + padding=dilation, + stride=stride, + groups=groups, + dilation=dilation, + bias_attr=False, + ) self.bn2 = norm_layer(width) - self.conv3 = nn.Conv2D(width, - planes * self.expansion, - 1, - bias_attr=False) + self.conv3 = nn.Conv2D( + width, planes * self.expansion, 1, bias_attr=False + ) self.bn3 = norm_layer(planes * self.expansion) self.relu = nn.ReLU() self.downsample = downsample @@ -214,20 +235,22 @@ class ResNet(nn.Layer): # [1, 1000] """ - def __init__(self, - block, - depth=50, - width=64, - num_classes=1000, - with_pool=True, - groups=1): + def __init__( + self, + block, + depth=50, + width=64, + num_classes=1000, + with_pool=True, + groups=1, + ): super(ResNet, self).__init__() layer_cfg = { 18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], - 152: [3, 8, 36, 3] + 152: [3, 8, 36, 3], } layers = layer_cfg[depth] self.groups = groups @@ -239,12 +262,14 @@ class ResNet(nn.Layer): self.inplanes = 64 self.dilation = 1 - self.conv1 = nn.Conv2D(3, - self.inplanes, - kernel_size=7, - stride=2, - padding=3, - bias_attr=False) + self.conv1 = nn.Conv2D( + 3, + self.inplanes, + kernel_size=7, + stride=2, + padding=3, + bias_attr=False, + ) self.bn1 = self._norm_layer(self.inplanes) self.relu = nn.ReLU() self.maxpool = nn.MaxPool2D(kernel_size=3, stride=2, padding=1) @@ -267,26 +292,40 @@ class ResNet(nn.Layer): stride = 1 if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( - nn.Conv2D(self.inplanes, - planes * block.expansion, - 1, - stride=stride, - bias_attr=False), + nn.Conv2D( + self.inplanes, + planes * block.expansion, + 1, + stride=stride, + bias_attr=False, + ), norm_layer(planes * block.expansion), ) layers = [] layers.append( - block(self.inplanes, planes, stride, downsample, self.groups, - self.base_width, previous_dilation, norm_layer)) + block( + self.inplanes, + planes, + stride, + downsample, + self.groups, + self.base_width, + previous_dilation, + norm_layer, + ) + ) self.inplanes = planes * block.expansion for _ in range(1, blocks): layers.append( - block(self.inplanes, - planes, - groups=self.groups, - base_width=self.base_width, - norm_layer=norm_layer)) + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + norm_layer=norm_layer, + ) + ) return nn.Sequential(*layers) @@ -313,10 +352,14 @@ class ResNet(nn.Layer): def _resnet(arch, Block, depth, pretrained, **kwargs): model = ResNet(Block, depth, **kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) @@ -591,8 +634,9 @@ def resnext101_32x4d(pretrained=False, **kwargs): """ kwargs['groups'] = 32 kwargs['width'] = 4 - return _resnet('resnext101_32x4d', BottleneckBlock, 101, pretrained, - **kwargs) + return _resnet( + 'resnext101_32x4d', BottleneckBlock, 101, pretrained, **kwargs + ) def resnext101_64x4d(pretrained=False, **kwargs): @@ -627,8 +671,9 @@ def resnext101_64x4d(pretrained=False, **kwargs): """ kwargs['groups'] = 64 kwargs['width'] = 4 - return _resnet('resnext101_64x4d', BottleneckBlock, 101, pretrained, - **kwargs) + return _resnet( + 'resnext101_64x4d', BottleneckBlock, 101, pretrained, **kwargs + ) def resnext152_32x4d(pretrained=False, **kwargs): @@ -663,8 +708,9 @@ def resnext152_32x4d(pretrained=False, **kwargs): """ kwargs['groups'] = 32 kwargs['width'] = 4 - return _resnet('resnext152_32x4d', BottleneckBlock, 152, pretrained, - **kwargs) + return _resnet( + 'resnext152_32x4d', BottleneckBlock, 152, pretrained, **kwargs + ) def resnext152_64x4d(pretrained=False, **kwargs): @@ -699,8 +745,9 @@ def resnext152_64x4d(pretrained=False, **kwargs): """ kwargs['groups'] = 64 kwargs['width'] = 4 - return _resnet('resnext152_64x4d', BottleneckBlock, 152, pretrained, - **kwargs) + return _resnet( + 'resnext152_64x4d', BottleneckBlock, 152, pretrained, **kwargs + ) def wide_resnet50_2(pretrained=False, **kwargs): @@ -768,5 +815,6 @@ def wide_resnet101_2(pretrained=False, **kwargs): # [1, 1000] """ kwargs['width'] = 64 * 2 - return _resnet('wide_resnet101_2', BottleneckBlock, 101, pretrained, - **kwargs) + return _resnet( + 'wide_resnet101_2', BottleneckBlock, 101, pretrained, **kwargs + ) diff --git a/python/paddle/vision/models/shufflenetv2.py b/python/paddle/vision/models/shufflenetv2.py index e6cadaed1fd..c443fa5a3ee 100644 --- a/python/paddle/vision/models/shufflenetv2.py +++ b/python/paddle/vision/models/shufflenetv2.py @@ -62,7 +62,8 @@ def create_activation_layer(act): return None else: raise RuntimeError( - "The activation function is not supported: {}".format(act)) + "The activation function is not supported: {}".format(act) + ) def channel_shuffle(x, groups): @@ -71,7 +72,8 @@ def channel_shuffle(x, groups): # reshape x = paddle.reshape( - x, shape=[batch_size, groups, channels_per_group, height, width]) + x, shape=[batch_size, groups, channels_per_group, height, width] + ) # transpose x = paddle.transpose(x, perm=[0, 2, 1, 3, 4]) @@ -82,27 +84,28 @@ def channel_shuffle(x, groups): class InvertedResidual(nn.Layer): - - def __init__(self, - in_channels, - out_channels, - stride, - activation_layer=nn.ReLU): + def __init__( + self, in_channels, out_channels, stride, activation_layer=nn.ReLU + ): super(InvertedResidual, self).__init__() - self._conv_pw = ConvNormActivation(in_channels=in_channels // 2, - out_channels=out_channels // 2, - kernel_size=1, - stride=1, - padding=0, - groups=1, - activation_layer=activation_layer) - self._conv_dw = ConvNormActivation(in_channels=out_channels // 2, - out_channels=out_channels // 2, - kernel_size=3, - stride=stride, - padding=1, - groups=out_channels // 2, - activation_layer=None) + self._conv_pw = ConvNormActivation( + in_channels=in_channels // 2, + out_channels=out_channels // 2, + kernel_size=1, + stride=1, + padding=0, + groups=1, + activation_layer=activation_layer, + ) + self._conv_dw = ConvNormActivation( + in_channels=out_channels // 2, + out_channels=out_channels // 2, + kernel_size=3, + stride=stride, + padding=1, + groups=out_channels // 2, + activation_layer=None, + ) self._conv_linear = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, @@ -110,13 +113,15 @@ class InvertedResidual(nn.Layer): stride=1, padding=0, groups=1, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) def forward(self, inputs): x1, x2 = paddle.split( inputs, num_or_sections=[inputs.shape[1] // 2, inputs.shape[1] // 2], - axis=1) + axis=1, + ) x2 = self._conv_pw(x2) x2 = self._conv_dw(x2) x2 = self._conv_linear(x2) @@ -125,22 +130,21 @@ class InvertedResidual(nn.Layer): class InvertedResidualDS(nn.Layer): - - def __init__(self, - in_channels, - out_channels, - stride, - activation_layer=nn.ReLU): + def __init__( + self, in_channels, out_channels, stride, activation_layer=nn.ReLU + ): super(InvertedResidualDS, self).__init__() # branch1 - self._conv_dw_1 = ConvNormActivation(in_channels=in_channels, - out_channels=in_channels, - kernel_size=3, - stride=stride, - padding=1, - groups=in_channels, - activation_layer=None) + self._conv_dw_1 = ConvNormActivation( + in_channels=in_channels, + out_channels=in_channels, + kernel_size=3, + stride=stride, + padding=1, + groups=in_channels, + activation_layer=None, + ) self._conv_linear_1 = ConvNormActivation( in_channels=in_channels, out_channels=out_channels // 2, @@ -148,22 +152,27 @@ class InvertedResidualDS(nn.Layer): stride=1, padding=0, groups=1, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) # branch2 - self._conv_pw_2 = ConvNormActivation(in_channels=in_channels, - out_channels=out_channels // 2, - kernel_size=1, - stride=1, - padding=0, - groups=1, - activation_layer=activation_layer) - self._conv_dw_2 = ConvNormActivation(in_channels=out_channels // 2, - out_channels=out_channels // 2, - kernel_size=3, - stride=stride, - padding=1, - groups=out_channels // 2, - activation_layer=None) + self._conv_pw_2 = ConvNormActivation( + in_channels=in_channels, + out_channels=out_channels // 2, + kernel_size=1, + stride=1, + padding=0, + groups=1, + activation_layer=activation_layer, + ) + self._conv_dw_2 = ConvNormActivation( + in_channels=out_channels // 2, + out_channels=out_channels // 2, + kernel_size=3, + stride=stride, + padding=1, + groups=out_channels // 2, + activation_layer=None, + ) self._conv_linear_2 = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, @@ -171,7 +180,8 @@ class InvertedResidualDS(nn.Layer): stride=1, padding=0, groups=1, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) def forward(self, inputs): x1 = self._conv_dw_1(inputs) @@ -232,15 +242,18 @@ class ShuffleNetV2(nn.Layer): elif scale == 2.0: stage_out_channels = [-1, 24, 224, 488, 976, 2048] else: - raise NotImplementedError("This scale size:[" + str(scale) + - "] is not implemented!") + raise NotImplementedError( + "This scale size:[" + str(scale) + "] is not implemented!" + ) # 1. conv1 - self._conv1 = ConvNormActivation(in_channels=3, - out_channels=stage_out_channels[1], - kernel_size=3, - stride=2, - padding=1, - activation_layer=activation_layer) + self._conv1 = ConvNormActivation( + in_channels=3, + out_channels=stage_out_channels[1], + kernel_size=3, + stride=2, + padding=1, + activation_layer=activation_layer, + ) self._max_pool = MaxPool2D(kernel_size=3, stride=2, padding=1) # 2. bottleneck sequences @@ -248,21 +261,25 @@ class ShuffleNetV2(nn.Layer): for stage_id, num_repeat in enumerate(stage_repeats): for i in range(num_repeat): if i == 0: - block = self.add_sublayer(sublayer=InvertedResidualDS( - in_channels=stage_out_channels[stage_id + 1], - out_channels=stage_out_channels[stage_id + 2], - stride=2, - activation_layer=activation_layer), - name=str(stage_id + 2) + "_" + - str(i + 1)) + block = self.add_sublayer( + sublayer=InvertedResidualDS( + in_channels=stage_out_channels[stage_id + 1], + out_channels=stage_out_channels[stage_id + 2], + stride=2, + activation_layer=activation_layer, + ), + name=str(stage_id + 2) + "_" + str(i + 1), + ) else: - block = self.add_sublayer(sublayer=InvertedResidual( - in_channels=stage_out_channels[stage_id + 2], - out_channels=stage_out_channels[stage_id + 2], - stride=1, - activation_layer=activation_layer), - name=str(stage_id + 2) + "_" + - str(i + 1)) + block = self.add_sublayer( + sublayer=InvertedResidual( + in_channels=stage_out_channels[stage_id + 2], + out_channels=stage_out_channels[stage_id + 2], + stride=1, + activation_layer=activation_layer, + ), + name=str(stage_id + 2) + "_" + str(i + 1), + ) self._block_list.append(block) # 3. last_conv self._last_conv = ConvNormActivation( @@ -271,7 +288,8 @@ class ShuffleNetV2(nn.Layer): kernel_size=1, stride=1, padding=0, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) # 4. pool if with_pool: self._pool2d_avg = AdaptiveAvgPool2D(1) @@ -303,9 +321,11 @@ def _shufflenet_v2(arch, pretrained=False, **kwargs): assert ( arch in model_urls ), "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) @@ -342,10 +362,9 @@ def shufflenet_v2_x0_25(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x0_25", - scale=0.25, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x0_25", scale=0.25, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x0_33(pretrained=False, **kwargs): @@ -378,10 +397,9 @@ def shufflenet_v2_x0_33(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x0_33", - scale=0.33, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x0_33", scale=0.33, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x0_5(pretrained=False, **kwargs): @@ -414,10 +432,9 @@ def shufflenet_v2_x0_5(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x0_5", - scale=0.5, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x0_5", scale=0.5, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x1_0(pretrained=False, **kwargs): @@ -450,10 +467,9 @@ def shufflenet_v2_x1_0(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x1_0", - scale=1.0, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x1_0", scale=1.0, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x1_5(pretrained=False, **kwargs): @@ -486,10 +502,9 @@ def shufflenet_v2_x1_5(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x1_5", - scale=1.5, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x1_5", scale=1.5, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x2_0(pretrained=False, **kwargs): @@ -522,10 +537,9 @@ def shufflenet_v2_x2_0(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x2_0", - scale=2.0, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x2_0", scale=2.0, pretrained=pretrained, **kwargs + ) def shufflenet_v2_swish(pretrained=False, **kwargs): @@ -558,8 +572,10 @@ def shufflenet_v2_swish(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_swish", - scale=1.0, - act="swish", - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_swish", + scale=1.0, + act="swish", + pretrained=pretrained, + **kwargs + ) diff --git a/python/paddle/vision/models/squeezenet.py b/python/paddle/vision/models/squeezenet.py index e2e7dc4b455..efb69b5da3a 100644 --- a/python/paddle/vision/models/squeezenet.py +++ b/python/paddle/vision/models/squeezenet.py @@ -24,25 +24,28 @@ from paddle.utils.download import get_weights_path_from_url __all__ = [] model_urls = { - 'squeezenet1_0': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/SqueezeNet1_0_pretrained.pdparams', - '30b95af60a2178f03cf9b66cd77e1db1'), - 'squeezenet1_1': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/SqueezeNet1_1_pretrained.pdparams', - 'a11250d3a1f91d7131fd095ebbf09eee'), + 'squeezenet1_0': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/SqueezeNet1_0_pretrained.pdparams', + '30b95af60a2178f03cf9b66cd77e1db1', + ), + 'squeezenet1_1': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/SqueezeNet1_1_pretrained.pdparams', + 'a11250d3a1f91d7131fd095ebbf09eee', + ), } class MakeFireConv(nn.Layer): - def __init__(self, input_channels, output_channels, filter_size, padding=0): super(MakeFireConv, self).__init__() - self._conv = Conv2D(input_channels, - output_channels, - filter_size, - padding=padding, - weight_attr=ParamAttr(), - bias_attr=ParamAttr()) + self._conv = Conv2D( + input_channels, + output_channels, + filter_size, + padding=padding, + weight_attr=ParamAttr(), + bias_attr=ParamAttr(), + ) def forward(self, x): x = self._conv(x) @@ -51,16 +54,19 @@ class MakeFireConv(nn.Layer): class MakeFire(nn.Layer): - - def __init__(self, input_channels, squeeze_channels, expand1x1_channels, - expand3x3_channels): + def __init__( + self, + input_channels, + squeeze_channels, + expand1x1_channels, + expand3x3_channels, + ): super(MakeFire, self).__init__() self._conv = MakeFireConv(input_channels, squeeze_channels, 1) self._conv_path1 = MakeFireConv(squeeze_channels, expand1x1_channels, 1) - self._conv_path2 = MakeFireConv(squeeze_channels, - expand3x3_channels, - 3, - padding=1) + self._conv_path2 = MakeFireConv( + squeeze_channels, expand3x3_channels, 3, padding=1 + ) def forward(self, inputs): x = self._conv(inputs) @@ -109,17 +115,21 @@ class SqueezeNet(nn.Layer): self.with_pool = with_pool supported_versions = ['1.0', '1.1'] - assert version in supported_versions, \ - "supported versions are {} but input version is {}".format( - supported_versions, version) + assert ( + version in supported_versions + ), "supported versions are {} but input version is {}".format( + supported_versions, version + ) if self.version == "1.0": - self._conv = Conv2D(3, - 96, - 7, - stride=2, - weight_attr=ParamAttr(), - bias_attr=ParamAttr()) + self._conv = Conv2D( + 3, + 96, + 7, + stride=2, + weight_attr=ParamAttr(), + bias_attr=ParamAttr(), + ) self._pool = MaxPool2D(kernel_size=3, stride=2, padding=0) self._conv1 = MakeFire(96, 16, 64, 64) self._conv2 = MakeFire(128, 16, 64, 64) @@ -130,13 +140,15 @@ class SqueezeNet(nn.Layer): self._conv7 = MakeFire(384, 64, 256, 256) self._conv8 = MakeFire(512, 64, 256, 256) else: - self._conv = Conv2D(3, - 64, - 3, - stride=2, - padding=1, - weight_attr=ParamAttr(), - bias_attr=ParamAttr()) + self._conv = Conv2D( + 3, + 64, + 3, + stride=2, + padding=1, + weight_attr=ParamAttr(), + bias_attr=ParamAttr(), + ) self._pool = MaxPool2D(kernel_size=3, stride=2, padding=0) self._conv1 = MakeFire(64, 16, 64, 64) self._conv2 = MakeFire(128, 16, 64, 64) @@ -148,11 +160,9 @@ class SqueezeNet(nn.Layer): self._conv8 = MakeFire(512, 64, 256, 256) self._drop = Dropout(p=0.5, mode="downscale_in_infer") - self._conv9 = Conv2D(512, - num_classes, - 1, - weight_attr=ParamAttr(), - bias_attr=ParamAttr()) + self._conv9 = Conv2D( + 512, num_classes, 1, weight_attr=ParamAttr(), bias_attr=ParamAttr() + ) self._avg_pool = AdaptiveAvgPool2D(1) def forward(self, inputs): @@ -195,10 +205,14 @@ class SqueezeNet(nn.Layer): def _squeezenet(arch, version, pretrained, **kwargs): model = SqueezeNet(version, **kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) diff --git a/python/paddle/vision/models/vgg.py b/python/paddle/vision/models/vgg.py index c9558c9c382..1f996439274 100644 --- a/python/paddle/vision/models/vgg.py +++ b/python/paddle/vision/models/vgg.py @@ -20,10 +20,14 @@ from paddle.utils.download import get_weights_path_from_url __all__ = [] model_urls = { - 'vgg16': ('https://paddle-hapi.bj.bcebos.com/models/vgg16.pdparams', - '89bbffc0f87d260be9b8cdc169c991c4'), - 'vgg19': ('https://paddle-hapi.bj.bcebos.com/models/vgg19.pdparams', - '23b18bb13d8894f60f54e642be79a0dd') + 'vgg16': ( + 'https://paddle-hapi.bj.bcebos.com/models/vgg16.pdparams', + '89bbffc0f87d260be9b8cdc169c991c4', + ), + 'vgg19': ( + 'https://paddle-hapi.bj.bcebos.com/models/vgg19.pdparams', + '23b18bb13d8894f60f54e642be79a0dd', + ), } @@ -111,15 +115,65 @@ def make_layers(cfg, batch_norm=False): cfgs = { 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], - 'B': - [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], + 'B': [ + 64, + 64, + 'M', + 128, + 128, + 'M', + 256, + 256, + 'M', + 512, + 512, + 'M', + 512, + 512, + 'M', + ], 'D': [ - 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, - 512, 512, 'M' + 64, + 64, + 'M', + 128, + 128, + 'M', + 256, + 256, + 256, + 'M', + 512, + 512, + 512, + 'M', + 512, + 512, + 512, + 'M', ], 'E': [ - 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, - 'M', 512, 512, 512, 512, 'M' + 64, + 64, + 'M', + 128, + 128, + 'M', + 256, + 256, + 256, + 256, + 'M', + 512, + 512, + 512, + 512, + 'M', + 512, + 512, + 512, + 512, + 'M', ], } @@ -128,10 +182,14 @@ def _vgg(arch, cfg, batch_norm, pretrained, **kwargs): model = VGG(make_layers(cfgs[cfg], batch_norm=batch_norm), **kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.load_dict(param) @@ -172,7 +230,7 @@ def vgg11(pretrained=False, batch_norm=False, **kwargs): """ model_name = 'vgg11' if batch_norm: - model_name += ('_bn') + model_name += '_bn' return _vgg(model_name, 'A', batch_norm, pretrained, **kwargs) @@ -209,7 +267,7 @@ def vgg13(pretrained=False, batch_norm=False, **kwargs): """ model_name = 'vgg13' if batch_norm: - model_name += ('_bn') + model_name += '_bn' return _vgg(model_name, 'B', batch_norm, pretrained, **kwargs) @@ -246,7 +304,7 @@ def vgg16(pretrained=False, batch_norm=False, **kwargs): """ model_name = 'vgg16' if batch_norm: - model_name += ('_bn') + model_name += '_bn' return _vgg(model_name, 'D', batch_norm, pretrained, **kwargs) @@ -283,5 +341,5 @@ def vgg19(pretrained=False, batch_norm=False, **kwargs): """ model_name = 'vgg19' if batch_norm: - model_name += ('_bn') + model_name += '_bn' return _vgg(model_name, 'E', batch_norm, pretrained, **kwargs) diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index b66db2aa737..e2ed14fa78d 100755 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -18,30 +18,48 @@ from ..fluid.data_feeder import check_type, check_variable_and_dtype from ..fluid.layers import nn, utils from ..nn import Layer, Conv2D, Sequential, ReLU, BatchNorm2D from ..fluid.initializer import Normal -from ..fluid.framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph +from ..fluid.framework import ( + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, +) from paddle import _C_ops, _legacy_C_ops from ..framework import _current_expected_place -__all__ = [ #noqa - 'yolo_loss', 'yolo_box', 'deform_conv2d', 'DeformConv2D', - 'distribute_fpn_proposals', 'generate_proposals', 'read_file', - 'decode_jpeg', 'roi_pool', 'RoIPool', 'psroi_pool', 'PSRoIPool', - 'roi_align', 'RoIAlign', 'nms', 'matrix_nms' +__all__ = [ # noqa + 'yolo_loss', + 'yolo_box', + 'deform_conv2d', + 'DeformConv2D', + 'distribute_fpn_proposals', + 'generate_proposals', + 'read_file', + 'decode_jpeg', + 'roi_pool', + 'RoIPool', + 'psroi_pool', + 'PSRoIPool', + 'roi_align', + 'RoIAlign', + 'nms', + 'matrix_nms', ] -def yolo_loss(x, - gt_box, - gt_label, - anchors, - anchor_mask, - class_num, - ignore_thresh, - downsample_ratio, - gt_score=None, - use_label_smooth=True, - name=None, - scale_x_y=1.): +def yolo_loss( + x, + gt_box, + gt_label, + anchors, + anchor_mask, + class_num, + ignore_thresh, + downsample_ratio, + gt_score=None, + use_label_smooth=True, + name=None, + scale_x_y=1.0, +): r""" This operator generates YOLOv3 loss based on given predict result and ground @@ -176,25 +194,50 @@ def yolo_loss(x, """ if in_dygraph_mode(): - loss, _, _ = _C_ops.yolov3_loss(x, gt_box, gt_label, gt_score, anchors, - anchor_mask, class_num, ignore_thresh, - downsample_ratio, use_label_smooth, - scale_x_y) + loss, _, _ = _C_ops.yolov3_loss( + x, + gt_box, + gt_label, + gt_score, + anchors, + anchor_mask, + class_num, + ignore_thresh, + downsample_ratio, + use_label_smooth, + scale_x_y, + ) return loss if _non_static_mode(): loss, _, _ = _legacy_C_ops.yolov3_loss( - x, gt_box, gt_label, gt_score, 'anchors', anchors, 'anchor_mask', - anchor_mask, 'class_num', class_num, 'ignore_thresh', ignore_thresh, - 'downsample_ratio', downsample_ratio, 'use_label_smooth', - use_label_smooth, 'scale_x_y', scale_x_y) + x, + gt_box, + gt_label, + gt_score, + 'anchors', + anchors, + 'anchor_mask', + anchor_mask, + 'class_num', + class_num, + 'ignore_thresh', + ignore_thresh, + 'downsample_ratio', + downsample_ratio, + 'use_label_smooth', + use_label_smooth, + 'scale_x_y', + scale_x_y, + ) return loss helper = LayerHelper('yolov3_loss', **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'yolo_loss') - check_variable_and_dtype(gt_box, 'gt_box', ['float32', 'float64'], - 'yolo_loss') + check_variable_and_dtype( + gt_box, 'gt_box', ['float32', 'float64'], 'yolo_loss' + ) check_variable_and_dtype(gt_label, 'gt_label', 'int32', 'yolo_loss') check_type(anchors, 'anchors', (list, tuple), 'yolo_loss') check_type(anchor_mask, 'anchor_mask', (list, tuple), 'yolo_loss') @@ -225,28 +268,32 @@ def yolo_loss(x, "scale_x_y": scale_x_y, } - helper.append_op(type='yolov3_loss', - inputs=inputs, - outputs={ - 'Loss': loss, - 'ObjectnessMask': objectness_mask, - 'GTMatchMask': gt_match_mask - }, - attrs=attrs) + helper.append_op( + type='yolov3_loss', + inputs=inputs, + outputs={ + 'Loss': loss, + 'ObjectnessMask': objectness_mask, + 'GTMatchMask': gt_match_mask, + }, + attrs=attrs, + ) return loss -def yolo_box(x, - img_size, - anchors, - class_num, - conf_thresh, - downsample_ratio, - clip_bbox=True, - name=None, - scale_x_y=1., - iou_aware=False, - iou_aware_factor=0.5): +def yolo_box( + x, + img_size, + anchors, + class_num, + conf_thresh, + downsample_ratio, + clip_bbox=True, + name=None, + scale_x_y=1.0, + iou_aware=False, + iou_aware_factor=0.5, +): r""" This operator generates YOLO detection boxes from output of YOLOv3 network. @@ -359,18 +406,41 @@ def yolo_box(x, scale_x_y=1.) """ if in_dygraph_mode(): - boxes, scores = _C_ops.yolo_box(x, img_size, anchors, class_num, - conf_thresh, downsample_ratio, - clip_bbox, scale_x_y, iou_aware, - iou_aware_factor) + boxes, scores = _C_ops.yolo_box( + x, + img_size, + anchors, + class_num, + conf_thresh, + downsample_ratio, + clip_bbox, + scale_x_y, + iou_aware, + iou_aware_factor, + ) return boxes, scores if _non_static_mode(): boxes, scores = _legacy_C_ops.yolo_box( - x, img_size, 'anchors', anchors, 'class_num', class_num, - 'conf_thresh', conf_thresh, 'downsample_ratio', downsample_ratio, - 'clip_bbox', clip_bbox, 'scale_x_y', scale_x_y, 'iou_aware', - iou_aware, 'iou_aware_factor', iou_aware_factor) + x, + img_size, + 'anchors', + anchors, + 'class_num', + class_num, + 'conf_thresh', + conf_thresh, + 'downsample_ratio', + downsample_ratio, + 'clip_bbox', + clip_bbox, + 'scale_x_y', + scale_x_y, + 'iou_aware', + iou_aware, + 'iou_aware_factor', + iou_aware_factor, + ) return boxes, scores helper = LayerHelper('yolo_box', **locals()) @@ -391,33 +461,37 @@ def yolo_box(x, "clip_bbox": clip_bbox, "scale_x_y": scale_x_y, "iou_aware": iou_aware, - "iou_aware_factor": iou_aware_factor + "iou_aware_factor": iou_aware_factor, } - helper.append_op(type='yolo_box', - inputs={ - "X": x, - "ImgSize": img_size, - }, - outputs={ - 'Boxes': boxes, - 'Scores': scores, - }, - attrs=attrs) + helper.append_op( + type='yolo_box', + inputs={ + "X": x, + "ImgSize": img_size, + }, + outputs={ + 'Boxes': boxes, + 'Scores': scores, + }, + attrs=attrs, + ) return boxes, scores -def deform_conv2d(x, - offset, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - deformable_groups=1, - groups=1, - mask=None, - name=None): +def deform_conv2d( + x, + offset, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + deformable_groups=1, + groups=1, + mask=None, + name=None, +): r""" Compute 2-D deformable convolution on 4-D input. Given input image x, output feature map y, the deformable convolution operation can be expressed as follow: @@ -537,34 +611,58 @@ def deform_conv2d(x, use_deform_conv2d_v1 = True if mask is None else False if in_dygraph_mode(): - pre_bias = _C_ops.deformable_conv(x, offset, weight, mask, stride, - padding, dilation, deformable_groups, - groups, 1) + pre_bias = _C_ops.deformable_conv( + x, + offset, + weight, + mask, + stride, + padding, + dilation, + deformable_groups, + groups, + 1, + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=1) else: out = pre_bias elif _in_legacy_dygraph(): - attrs = ('strides', stride, 'paddings', padding, 'dilations', dilation, - 'deformable_groups', deformable_groups, 'groups', groups, - 'im2col_step', 1) + attrs = ( + 'strides', + stride, + 'paddings', + padding, + 'dilations', + dilation, + 'deformable_groups', + deformable_groups, + 'groups', + groups, + 'im2col_step', + 1, + ) if use_deform_conv2d_v1: op_type = 'deformable_conv_v1' - pre_bias = getattr(_legacy_C_ops, op_type)(x, offset, weight, - *attrs) + pre_bias = getattr(_legacy_C_ops, op_type)( + x, offset, weight, *attrs + ) else: op_type = 'deformable_conv' - pre_bias = getattr(_legacy_C_ops, op_type)(x, offset, mask, weight, - *attrs) + pre_bias = getattr(_legacy_C_ops, op_type)( + x, offset, mask, weight, *attrs + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=1) else: out = pre_bias else: - check_variable_and_dtype(x, "x", ['float32', 'float64'], - 'deform_conv2d') - check_variable_and_dtype(offset, "offset", ['float32', 'float64'], - 'deform_conv2d') + check_variable_and_dtype( + x, "x", ['float32', 'float64'], 'deform_conv2d' + ) + check_variable_and_dtype( + offset, "offset", ['float32', 'float64'], 'deform_conv2d' + ) num_channels = x.shape[1] @@ -602,20 +700,18 @@ def deform_conv2d(x, 'deformable_groups': deformable_groups, 'im2col_step': 1, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [bias] - }, - outputs={'Out': [out]}, - attrs={'axis': 1}) + helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [bias]}, + outputs={'Out': [out]}, + attrs={'axis': 1}, + ) else: out = pre_bias return out @@ -755,19 +851,23 @@ class DeformConv2D(Layer): [8, 16, 26, 26] """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - deformable_groups=1, - groups=1, - weight_attr=None, - bias_attr=None): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + deformable_groups=1, + groups=1, + weight_attr=None, + bias_attr=None, + ): super(DeformConv2D, self).__init__() - assert weight_attr is not False, "weight_attr should not be False in Conv." + assert ( + weight_attr is not False + ), "weight_attr should not be False in Conv." self._weight_attr = weight_attr self._bias_attr = bias_attr self._deformable_groups = deformable_groups @@ -789,39 +889,44 @@ class DeformConv2D(Layer): def _get_default_param_initializer(): filter_elem_num = np.prod(self._kernel_size) * self._in_channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) self.weight = self.create_parameter( shape=filter_shape, attr=self._weight_attr, - default_initializer=_get_default_param_initializer()) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._out_channels], - is_bias=True) + default_initializer=_get_default_param_initializer(), + ) + self.bias = self.create_parameter( + attr=self._bias_attr, shape=[self._out_channels], is_bias=True + ) def forward(self, x, offset, mask=None): - out = deform_conv2d(x=x, - offset=offset, - weight=self.weight, - bias=self.bias, - stride=self._stride, - padding=self._padding, - dilation=self._dilation, - deformable_groups=self._deformable_groups, - groups=self._groups, - mask=mask) + out = deform_conv2d( + x=x, + offset=offset, + weight=self.weight, + bias=self.bias, + stride=self._stride, + padding=self._padding, + dilation=self._dilation, + deformable_groups=self._deformable_groups, + groups=self._groups, + mask=mask, + ) return out -def distribute_fpn_proposals(fpn_rois, - min_level, - max_level, - refer_level, - refer_scale, - pixel_offset=False, - rois_num=None, - name=None): +def distribute_fpn_proposals( + fpn_rois, + min_level, + max_level, + refer_level, + refer_scale, + pixel_offset=False, + rois_num=None, + name=None, +): r""" In Feature Pyramid Networks (FPN) models, it is needed to distribute all proposals into different FPN level, with respect to scale of the proposals, @@ -882,24 +987,56 @@ def distribute_fpn_proposals(fpn_rois, num_lvl = max_level - min_level + 1 if in_dygraph_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." - multi_rois, rois_num_per_level, restore_ind = _C_ops.distribute_fpn_proposals( - fpn_rois, rois_num, min_level, max_level, refer_level, refer_scale, - pixel_offset) + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." + ( + multi_rois, + rois_num_per_level, + restore_ind, + ) = _C_ops.distribute_fpn_proposals( + fpn_rois, + rois_num, + min_level, + max_level, + refer_level, + refer_scale, + pixel_offset, + ) return multi_rois, restore_ind, rois_num_per_level if _non_static_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." - attrs = ('min_level', min_level, 'max_level', max_level, 'refer_level', - refer_level, 'refer_scale', refer_scale, 'pixel_offset', - pixel_offset) - multi_rois, restore_ind, rois_num_per_level = _legacy_C_ops.distribute_fpn_proposals( - fpn_rois, rois_num, num_lvl, num_lvl, *attrs) + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." + attrs = ( + 'min_level', + min_level, + 'max_level', + max_level, + 'refer_level', + refer_level, + 'refer_scale', + refer_scale, + 'pixel_offset', + pixel_offset, + ) + ( + multi_rois, + restore_ind, + rois_num_per_level, + ) = _legacy_C_ops.distribute_fpn_proposals( + fpn_rois, rois_num, num_lvl, num_lvl, *attrs + ) return multi_rois, restore_ind, rois_num_per_level else: - check_variable_and_dtype(fpn_rois, 'fpn_rois', ['float32', 'float64'], - 'distribute_fpn_proposals') + check_variable_and_dtype( + fpn_rois, + 'fpn_rois', + ['float32', 'float64'], + 'distribute_fpn_proposals', + ) helper = LayerHelper('distribute_fpn_proposals', **locals()) dtype = helper.input_dtype('fpn_rois') multi_rois = [ @@ -925,16 +1062,18 @@ def distribute_fpn_proposals(fpn_rois, else: rois_num_per_level = None - helper.append_op(type='distribute_fpn_proposals', - inputs=inputs, - outputs=outputs, - attrs={ - 'min_level': min_level, - 'max_level': max_level, - 'refer_level': refer_level, - 'refer_scale': refer_scale, - 'pixel_offset': pixel_offset - }) + helper.append_op( + type='distribute_fpn_proposals', + inputs=inputs, + outputs=outputs, + attrs={ + 'min_level': min_level, + 'max_level': max_level, + 'refer_level': refer_level, + 'refer_scale': refer_scale, + 'pixel_offset': pixel_offset, + }, + ) return multi_rois, restore_ind, rois_num_per_level @@ -976,10 +1115,9 @@ def read_file(filename, name=None): helper = LayerHelper("read_file", **locals()) out = helper.create_variable_for_type_inference('uint8') - helper.append_op(type="read_file", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="read_file", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1029,10 +1167,9 @@ def decode_jpeg(x, mode='unchanged', name=None): helper = LayerHelper("decode_jpeg", **locals()) out = helper.create_variable_for_type_inference('uint8') - helper.append_op(type="decode_jpeg", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="decode_jpeg", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1080,34 +1217,47 @@ def psroi_pool(x, boxes, boxes_num, output_size, spatial_scale=1.0, name=None): if isinstance(output_size, int): output_size = (output_size, output_size) pooled_height, pooled_width = output_size - assert len(x.shape) == 4, \ - "Input features with shape should be (N, C, H, W)" + assert len(x.shape) == 4, "Input features with shape should be (N, C, H, W)" output_channels = int(x.shape[1] / (pooled_height * pooled_width)) if in_dygraph_mode(): - return _C_ops.psroi_pool(x, boxes, boxes_num, pooled_height, - pooled_width, output_channels, spatial_scale) + return _C_ops.psroi_pool( + x, + boxes, + boxes_num, + pooled_height, + pooled_width, + output_channels, + spatial_scale, + ) if _in_legacy_dygraph(): - return _legacy_C_ops.psroi_pool(x, boxes, boxes_num, "output_channels", - output_channels, "spatial_scale", - spatial_scale, "pooled_height", - pooled_height, "pooled_width", - pooled_width) + return _legacy_C_ops.psroi_pool( + x, + boxes, + boxes_num, + "output_channels", + output_channels, + "spatial_scale", + spatial_scale, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + ) helper = LayerHelper('psroi_pool', **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='psroi_pool', - inputs={ - 'X': x, - 'ROIs': boxes - }, - outputs={'Out': out}, - attrs={ - 'output_channels': output_channels, - 'spatial_scale': spatial_scale, - 'pooled_height': pooled_height, - 'pooled_width': pooled_width - }) + helper.append_op( + type='psroi_pool', + inputs={'X': x, 'ROIs': boxes}, + outputs={'Out': out}, + attrs={ + 'output_channels': output_channels, + 'spatial_scale': spatial_scale, + 'pooled_height': pooled_height, + 'pooled_width': pooled_width, + }, + ) return out @@ -1151,8 +1301,9 @@ class PSRoIPool(Layer): self.spatial_scale = spatial_scale def forward(self, x, boxes, boxes_num): - return psroi_pool(x, boxes, boxes_num, self.output_size, - self.spatial_scale) + return psroi_pool( + x, boxes, boxes_num, self.output_size, self.spatial_scale + ) def roi_pool(x, boxes, boxes_num, output_size, spatial_scale=1.0, name=None): @@ -1199,14 +1350,27 @@ def roi_pool(x, boxes, boxes_num, output_size, spatial_scale=1.0, name=None): pooled_height, pooled_width = output_size if in_dygraph_mode(): - assert boxes_num is not None, "boxes_num should not be None in dygraph mode." - return _C_ops.roi_pool(x, boxes, boxes_num, pooled_height, pooled_width, - spatial_scale) + assert ( + boxes_num is not None + ), "boxes_num should not be None in dygraph mode." + return _C_ops.roi_pool( + x, boxes, boxes_num, pooled_height, pooled_width, spatial_scale + ) if _in_legacy_dygraph(): - assert boxes_num is not None, "boxes_num should not be None in dygraph mode." + assert ( + boxes_num is not None + ), "boxes_num should not be None in dygraph mode." pool_out, argmaxes = _legacy_C_ops.roi_pool( - x, boxes, boxes_num, "pooled_height", pooled_height, "pooled_width", - pooled_width, "spatial_scale", spatial_scale) + x, + boxes, + boxes_num, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + "spatial_scale", + spatial_scale, + ) return pool_out else: @@ -1223,17 +1387,16 @@ def roi_pool(x, boxes, boxes_num, output_size, spatial_scale=1.0, name=None): } if boxes_num is not None: inputs['RoisNum'] = boxes_num - helper.append_op(type="roi_pool", - inputs=inputs, - outputs={ - "Out": pool_out, - "Argmax": argmaxes - }, - attrs={ - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "spatial_scale": spatial_scale - }) + helper.append_op( + type="roi_pool", + inputs=inputs, + outputs={"Out": pool_out, "Argmax": argmaxes}, + attrs={ + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "spatial_scale": spatial_scale, + }, + ) return pool_out @@ -1271,25 +1434,29 @@ class RoIPool(Layer): self._spatial_scale = spatial_scale def forward(self, x, boxes, boxes_num): - return roi_pool(x=x, - boxes=boxes, - boxes_num=boxes_num, - output_size=self._output_size, - spatial_scale=self._spatial_scale) + return roi_pool( + x=x, + boxes=boxes, + boxes_num=boxes_num, + output_size=self._output_size, + spatial_scale=self._spatial_scale, + ) def extra_repr(self): main_str = 'output_size={_output_size}, spatial_scale={_spatial_scale}' return main_str.format(**self.__dict__) -def roi_align(x, - boxes, - boxes_num, - output_size, - spatial_scale=1.0, - sampling_ratio=-1, - aligned=True, - name=None): +def roi_align( + x, + boxes, + boxes_num, + output_size, + spatial_scale=1.0, + sampling_ratio=-1, + aligned=True, + name=None, +): """ Implementing the roi_align layer. Region of Interest (RoI) Align operator (also known as RoI Align) is to @@ -1358,24 +1525,45 @@ def roi_align(x, pooled_height, pooled_width = output_size if in_dygraph_mode(): - assert boxes_num is not None, "boxes_num should not be None in dygraph mode." - return _C_ops.roi_align(x, boxes, boxes_num, pooled_height, - pooled_width, spatial_scale, sampling_ratio, - aligned) + assert ( + boxes_num is not None + ), "boxes_num should not be None in dygraph mode." + return _C_ops.roi_align( + x, + boxes, + boxes_num, + pooled_height, + pooled_width, + spatial_scale, + sampling_ratio, + aligned, + ) if _in_legacy_dygraph(): - assert boxes_num is not None, "boxes_num should not be None in dygraph mode." - align_out = _legacy_C_ops.roi_align(x, boxes, boxes_num, - "pooled_height", pooled_height, - "pooled_width", pooled_width, - "spatial_scale", spatial_scale, - "sampling_ratio", sampling_ratio, - "aligned", aligned) + assert ( + boxes_num is not None + ), "boxes_num should not be None in dygraph mode." + align_out = _legacy_C_ops.roi_align( + x, + boxes, + boxes_num, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + "spatial_scale", + spatial_scale, + "sampling_ratio", + sampling_ratio, + "aligned", + aligned, + ) return align_out else: check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'roi_align') - check_variable_and_dtype(boxes, 'boxes', ['float32', 'float64'], - 'roi_align') + check_variable_and_dtype( + boxes, 'boxes', ['float32', 'float64'], 'roi_align' + ) helper = LayerHelper('roi_align', **locals()) dtype = helper.input_dtype() align_out = helper.create_variable_for_type_inference(dtype) @@ -1385,16 +1573,18 @@ def roi_align(x, } if boxes_num is not None: inputs['RoisNum'] = boxes_num - helper.append_op(type="roi_align", - inputs=inputs, - outputs={"Out": align_out}, - attrs={ - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "spatial_scale": spatial_scale, - "sampling_ratio": sampling_ratio, - "aligned": aligned, - }) + helper.append_op( + type="roi_align", + inputs=inputs, + outputs={"Out": align_out}, + attrs={ + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "spatial_scale": spatial_scale, + "sampling_ratio": sampling_ratio, + "aligned": aligned, + }, + ) return align_out @@ -1436,12 +1626,14 @@ class RoIAlign(Layer): self._spatial_scale = spatial_scale def forward(self, x, boxes, boxes_num, aligned=True): - return roi_align(x=x, - boxes=boxes, - boxes_num=boxes_num, - output_size=self._output_size, - spatial_scale=self._spatial_scale, - aligned=aligned) + return roi_align( + x=x, + boxes=boxes, + boxes_num=boxes_num, + output_size=self._output_size, + spatial_scale=self._spatial_scale, + aligned=aligned, + ) class ConvNormActivation(Sequential): @@ -1465,30 +1657,34 @@ class ConvNormActivation(Sequential): bias (bool, optional): Whether to use bias in the convolution layer. By default, biases are included if ``norm_layer is None``. """ - def __init__(self, - in_channels, - out_channels, - kernel_size=3, - stride=1, - padding=None, - groups=1, - norm_layer=BatchNorm2D, - activation_layer=ReLU, - dilation=1, - bias=None): + def __init__( + self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=None, + groups=1, + norm_layer=BatchNorm2D, + activation_layer=ReLU, + dilation=1, + bias=None, + ): if padding is None: padding = (kernel_size - 1) // 2 * dilation if bias is None: bias = norm_layer is None layers = [ - Conv2D(in_channels, - out_channels, - kernel_size, - stride, - padding, - dilation=dilation, - groups=groups, - bias_attr=bias) + Conv2D( + in_channels, + out_channels, + kernel_size, + stride, + padding, + dilation=dilation, + groups=groups, + bias_attr=bias, + ) ] if norm_layer is not None: layers.append(norm_layer(out_channels)) @@ -1497,12 +1693,14 @@ class ConvNormActivation(Sequential): super().__init__(*layers) -def nms(boxes, - iou_threshold=0.3, - scores=None, - category_idxs=None, - categories=None, - top_k=None): +def nms( + boxes, + iou_threshold=0.3, + scores=None, + category_idxs=None, + categories=None, + top_k=None, +): r""" This operator implements non-maximum suppression. Non-maximum suppression (NMS) is used to select one bounding box out of many overlapping bounding boxes in object detection. @@ -1582,34 +1780,42 @@ def nms(boxes, helper = LayerHelper('nms', **locals()) out = helper.create_variable_for_type_inference('int64') - helper.append_op(type='nms', - inputs={'Boxes': boxes}, - outputs={'KeepBoxesIdxs': out}, - attrs={'iou_threshold': iou_threshold}) + helper.append_op( + type='nms', + inputs={'Boxes': boxes}, + outputs={'KeepBoxesIdxs': out}, + attrs={'iou_threshold': iou_threshold}, + ) return out if scores is None: return _nms(boxes, iou_threshold) import paddle + if category_idxs is None: sorted_global_indices = paddle.argsort(scores, descending=True) - sorted_keep_boxes_indices = _nms(boxes[sorted_global_indices], - iou_threshold) + sorted_keep_boxes_indices = _nms( + boxes[sorted_global_indices], iou_threshold + ) return sorted_global_indices[sorted_keep_boxes_indices] if top_k is not None: - assert top_k <= scores.shape[ - 0], "top_k should be smaller equal than the number of boxes" - assert categories is not None, "if category_idxs is given, categories which is a list of unique id of all categories is necessary" + assert ( + top_k <= scores.shape[0] + ), "top_k should be smaller equal than the number of boxes" + assert ( + categories is not None + ), "if category_idxs is given, categories which is a list of unique id of all categories is necessary" mask = paddle.zeros_like(scores, dtype=paddle.int32) for category_id in categories: cur_category_boxes_idxs = paddle.where(category_idxs == category_id)[0] shape = cur_category_boxes_idxs.shape[0] - cur_category_boxes_idxs = paddle.reshape(cur_category_boxes_idxs, - [shape]) + cur_category_boxes_idxs = paddle.reshape( + cur_category_boxes_idxs, [shape] + ) if shape == 0: continue elif shape == 1: @@ -1617,27 +1823,33 @@ def nms(boxes, continue cur_category_boxes = boxes[cur_category_boxes_idxs] cur_category_scores = scores[cur_category_boxes_idxs] - cur_category_sorted_indices = paddle.argsort(cur_category_scores, - descending=True) + cur_category_sorted_indices = paddle.argsort( + cur_category_scores, descending=True + ) cur_category_sorted_boxes = cur_category_boxes[ - cur_category_sorted_indices] + cur_category_sorted_indices + ] - cur_category_keep_boxes_sub_idxs = cur_category_sorted_indices[_nms( - cur_category_sorted_boxes, iou_threshold)] + cur_category_keep_boxes_sub_idxs = cur_category_sorted_indices[ + _nms(cur_category_sorted_boxes, iou_threshold) + ] updates = paddle.ones_like( cur_category_boxes_idxs[cur_category_keep_boxes_sub_idxs], - dtype=paddle.int32) + dtype=paddle.int32, + ) mask = paddle.scatter( mask, cur_category_boxes_idxs[cur_category_keep_boxes_sub_idxs], updates, - overwrite=True) + overwrite=True, + ) keep_boxes_idxs = paddle.where(mask)[0] shape = keep_boxes_idxs.shape[0] keep_boxes_idxs = paddle.reshape(keep_boxes_idxs, [shape]) - sorted_sub_indices = paddle.argsort(scores[keep_boxes_idxs], - descending=True) + sorted_sub_indices = paddle.argsort( + scores[keep_boxes_idxs], descending=True + ) if top_k is None: return keep_boxes_idxs[sorted_sub_indices] @@ -1650,19 +1862,21 @@ def nms(boxes, return keep_boxes_idxs[sorted_sub_indices][:top_k] -def generate_proposals(scores, - bbox_deltas, - img_size, - anchors, - variances, - pre_nms_top_n=6000, - post_nms_top_n=1000, - nms_thresh=0.5, - min_size=0.1, - eta=1.0, - pixel_offset=False, - return_rois_num=False, - name=None): +def generate_proposals( + scores, + bbox_deltas, + img_size, + anchors, + variances, + pre_nms_top_n=6000, + post_nms_top_n=1000, + nms_thresh=0.5, + min_size=0.1, + eta=1.0, + pixel_offset=False, + return_rois_num=False, + name=None, +): """ This operation proposes RoIs according to each box with their probability to be a foreground object. And @@ -1733,40 +1947,74 @@ def generate_proposals(scores, """ if in_dygraph_mode(): - assert return_rois_num, "return_rois_num should be True in dygraph mode." - attrs = (pre_nms_top_n, post_nms_top_n, nms_thresh, min_size, eta, - pixel_offset) + assert ( + return_rois_num + ), "return_rois_num should be True in dygraph mode." + attrs = ( + pre_nms_top_n, + post_nms_top_n, + nms_thresh, + min_size, + eta, + pixel_offset, + ) rpn_rois, rpn_roi_probs, rpn_rois_num = _C_ops.generate_proposals( - scores, bbox_deltas, img_size, anchors, variances, *attrs) + scores, bbox_deltas, img_size, anchors, variances, *attrs + ) return rpn_rois, rpn_roi_probs, rpn_rois_num elif _non_static_mode(): - assert return_rois_num, "return_rois_num should be True in dygraph mode." - attrs = ('pre_nms_topN', pre_nms_top_n, 'post_nms_topN', post_nms_top_n, - 'nms_thresh', nms_thresh, 'min_size', min_size, 'eta', eta, - 'pixel_offset', pixel_offset) - rpn_rois, rpn_roi_probs, rpn_rois_num = _legacy_C_ops.generate_proposals_v2( - scores, bbox_deltas, img_size, anchors, variances, *attrs) + assert ( + return_rois_num + ), "return_rois_num should be True in dygraph mode." + attrs = ( + 'pre_nms_topN', + pre_nms_top_n, + 'post_nms_topN', + post_nms_top_n, + 'nms_thresh', + nms_thresh, + 'min_size', + min_size, + 'eta', + eta, + 'pixel_offset', + pixel_offset, + ) + ( + rpn_rois, + rpn_roi_probs, + rpn_rois_num, + ) = _legacy_C_ops.generate_proposals_v2( + scores, bbox_deltas, img_size, anchors, variances, *attrs + ) return rpn_rois, rpn_roi_probs, rpn_rois_num helper = LayerHelper('generate_proposals_v2', **locals()) - check_variable_and_dtype(scores, 'scores', ['float32'], - 'generate_proposals_v2') - check_variable_and_dtype(bbox_deltas, 'bbox_deltas', ['float32'], - 'generate_proposals_v2') - check_variable_and_dtype(img_size, 'img_size', ['float32', 'float64'], - 'generate_proposals_v2') - check_variable_and_dtype(anchors, 'anchors', ['float32'], - 'generate_proposals_v2') - check_variable_and_dtype(variances, 'variances', ['float32'], - 'generate_proposals_v2') + check_variable_and_dtype( + scores, 'scores', ['float32'], 'generate_proposals_v2' + ) + check_variable_and_dtype( + bbox_deltas, 'bbox_deltas', ['float32'], 'generate_proposals_v2' + ) + check_variable_and_dtype( + img_size, 'img_size', ['float32', 'float64'], 'generate_proposals_v2' + ) + check_variable_and_dtype( + anchors, 'anchors', ['float32'], 'generate_proposals_v2' + ) + check_variable_and_dtype( + variances, 'variances', ['float32'], 'generate_proposals_v2' + ) rpn_rois = helper.create_variable_for_type_inference( - dtype=bbox_deltas.dtype) + dtype=bbox_deltas.dtype + ) rpn_roi_probs = helper.create_variable_for_type_inference( - dtype=scores.dtype) + dtype=scores.dtype + ) outputs = { 'RpnRois': rpn_rois, 'RpnRoiProbs': rpn_roi_probs, @@ -1776,23 +2024,25 @@ def generate_proposals(scores, rpn_rois_num.stop_gradient = True outputs['RpnRoisNum'] = rpn_rois_num - helper.append_op(type="generate_proposals_v2", - inputs={ - 'Scores': scores, - 'BboxDeltas': bbox_deltas, - 'ImShape': img_size, - 'Anchors': anchors, - 'Variances': variances - }, - attrs={ - 'pre_nms_topN': pre_nms_top_n, - 'post_nms_topN': post_nms_top_n, - 'nms_thresh': nms_thresh, - 'min_size': min_size, - 'eta': eta, - 'pixel_offset': pixel_offset - }, - outputs=outputs) + helper.append_op( + type="generate_proposals_v2", + inputs={ + 'Scores': scores, + 'BboxDeltas': bbox_deltas, + 'ImShape': img_size, + 'Anchors': anchors, + 'Variances': variances, + }, + attrs={ + 'pre_nms_topN': pre_nms_top_n, + 'post_nms_topN': post_nms_top_n, + 'nms_thresh': nms_thresh, + 'min_size': min_size, + 'eta': eta, + 'pixel_offset': pixel_offset, + }, + outputs=outputs, + ) rpn_rois.stop_gradient = True rpn_roi_probs.stop_gradient = True if not return_rois_num: @@ -1801,19 +2051,21 @@ def generate_proposals(scores, return rpn_rois, rpn_roi_probs, rpn_rois_num -def matrix_nms(bboxes, - scores, - score_threshold, - post_threshold, - nms_top_k, - keep_top_k, - use_gaussian=False, - gaussian_sigma=2., - background_label=0, - normalized=True, - return_index=False, - return_rois_num=True, - name=None): +def matrix_nms( + bboxes, + scores, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + use_gaussian=False, + gaussian_sigma=2.0, + background_label=0, + normalized=True, + return_index=False, + return_rois_num=True, + name=None, +): """ This operator does matrix non maximum suppression (NMS). First selects a subset of candidate bounding boxes that have higher scores @@ -1876,10 +2128,12 @@ def matrix_nms(bboxes, score_threshold=0.5, post_threshold=0.1, nms_top_k=400, keep_top_k=200, normalized=False) """ - check_variable_and_dtype(bboxes, 'BBoxes', ['float32', 'float64'], - 'matrix_nms') - check_variable_and_dtype(scores, 'Scores', ['float32', 'float64'], - 'matrix_nms') + check_variable_and_dtype( + bboxes, 'BBoxes', ['float32', 'float64'], 'matrix_nms' + ) + check_variable_and_dtype( + scores, 'Scores', ['float32', 'float64'], 'matrix_nms' + ) check_type(score_threshold, 'score_threshold', float, 'matrix_nms') check_type(post_threshold, 'post_threshold', float, 'matrix_nms') check_type(nms_top_k, 'nums_top_k', int, 'matrix_nms') @@ -1890,22 +2144,42 @@ def matrix_nms(bboxes, check_type(background_label, 'background_label', int, 'matrix_nms') if in_dygraph_mode(): - out, index, rois_num = _C_ops.matrix_nms(bboxes, scores, - score_threshold, nms_top_k, - keep_top_k, post_threshold, - use_gaussian, gaussian_sigma, - background_label, normalized) + out, index, rois_num = _C_ops.matrix_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + post_threshold, + use_gaussian, + gaussian_sigma, + background_label, + normalized, + ) if not return_index: index = None if not return_rois_num: rois_num = None return out, rois_num, index elif _in_legacy_dygraph(): - attrs = ('background_label', background_label, 'score_threshold', - score_threshold, 'post_threshold', post_threshold, 'nms_top_k', - nms_top_k, 'gaussian_sigma', gaussian_sigma, 'use_gaussian', - use_gaussian, 'keep_top_k', keep_top_k, 'normalized', - normalized) + attrs = ( + 'background_label', + background_label, + 'score_threshold', + score_threshold, + 'post_threshold', + post_threshold, + 'nms_top_k', + nms_top_k, + 'gaussian_sigma', + gaussian_sigma, + 'use_gaussian', + use_gaussian, + 'keep_top_k', + keep_top_k, + 'normalized', + normalized, + ) out, index, rois_num = _legacy_C_ops.matrix_nms(bboxes, scores, *attrs) if not return_index: index = None @@ -1921,22 +2195,21 @@ def matrix_nms(bboxes, rois_num = helper.create_variable_for_type_inference(dtype='int32') outputs['RoisNum'] = rois_num - helper.append_op(type="matrix_nms", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'post_threshold': post_threshold, - 'nms_top_k': nms_top_k, - 'gaussian_sigma': gaussian_sigma, - 'use_gaussian': use_gaussian, - 'keep_top_k': keep_top_k, - 'normalized': normalized - }, - outputs=outputs) + helper.append_op( + type="matrix_nms", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'post_threshold': post_threshold, + 'nms_top_k': nms_top_k, + 'gaussian_sigma': gaussian_sigma, + 'use_gaussian': use_gaussian, + 'keep_top_k': keep_top_k, + 'normalized': normalized, + }, + outputs=outputs, + ) output.stop_gradient = True if not return_index: diff --git a/python/paddle/vision/transforms/__init__.py b/python/paddle/vision/transforms/__init__.py index d615598bf2b..890e4b89827 100644 --- a/python/paddle/vision/transforms/__init__.py +++ b/python/paddle/vision/transforms/__init__.py @@ -51,7 +51,7 @@ from .functional import adjust_hue # noqa: F401 from .functional import normalize # noqa: F401 from .functional import erase # noqa: F401 -__all__ = [ #noqa +__all__ = [ # noqa 'BaseTransform', 'Compose', 'Resize', diff --git a/python/paddle/vision/transforms/functional.py b/python/paddle/vision/transforms/functional.py index 99d27a60ae3..45e50712c9e 100644 --- a/python/paddle/vision/transforms/functional.py +++ b/python/paddle/vision/transforms/functional.py @@ -66,11 +66,14 @@ def to_tensor(pic, data_format='CHW'): print(tensor.shape) """ - if not (_is_pil_image(pic) or _is_numpy_image(pic) - or _is_tensor_image(pic)): + if not ( + _is_pil_image(pic) or _is_numpy_image(pic) or _is_tensor_image(pic) + ): raise TypeError( - 'pic should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(pic))) + 'pic should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(pic) + ) + ) if _is_pil_image(pic): return F_pil.to_tensor(pic, data_format) @@ -124,11 +127,14 @@ def resize(img, size, interpolation='bilinear'): print(converted_img.size) # (150, 200) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.resize(img, size, interpolation) @@ -188,11 +194,14 @@ def pad(img, padding, fill=0, padding_mode='constant'): padded_img = F.pad(fake_img, padding=(2, 1)) print(padded_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.pad(img, padding, fill, padding_mode) @@ -231,11 +240,14 @@ def crop(img, top, left, height, width): print(cropped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.crop(img, top, left, height, width) @@ -248,33 +260,36 @@ def crop(img, top, left, height, width): def center_crop(img, output_size): """Crops the given Image and resize it to desired size. - Args: - img (PIL.Image|np.array): Image to be cropped. (0,0) denotes the top left corner of the image. - output_size (sequence or int): (height, width) of the crop box. If int, - it is used for both directions + Args: + img (PIL.Image|np.array): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions - Returns: - PIL.Image or np.array: Cropped image. + Returns: + PIL.Image or np.array: Cropped image. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - import numpy as np - from PIL import Image - from paddle.vision.transforms import functional as F + import numpy as np + from PIL import Image + from paddle.vision.transforms import functional as F - fake_img = (np.random.rand(256, 300, 3) * 255.).astype('uint8') + fake_img = (np.random.rand(256, 300, 3) * 255.).astype('uint8') - fake_img = Image.fromarray(fake_img) + fake_img = Image.fromarray(fake_img) - cropped_img = F.center_crop(fake_img, (150, 100)) - print(cropped_img.size) - """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + cropped_img = F.center_crop(fake_img, (150, 100)) + print(cropped_img.size) + """ + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.center_crop(img, output_size) @@ -308,11 +323,14 @@ def hflip(img): print(flpped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.hflip(img) @@ -346,11 +364,14 @@ def vflip(img): print(flpped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.vflip(img) @@ -391,11 +412,14 @@ def adjust_brightness(img, brightness_factor): """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.adjust_brightness(img, brightness_factor) @@ -431,11 +455,14 @@ def adjust_contrast(img, contrast_factor): converted_img = F.adjust_contrast(fake_img, 0.4) print(converted_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.adjust_contrast(img, contrast_factor) @@ -472,11 +499,14 @@ def adjust_saturation(img, saturation_factor): print(converted_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.adjust_saturation(img, saturation_factor) @@ -522,11 +552,14 @@ def adjust_hue(img, hue_factor): print(converted_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.adjust_hue(img, hue_factor) @@ -567,14 +600,16 @@ def _get_affine_matrix(center, angle, translate, scale, shear): return matrix -def affine(img, - angle, - translate, - scale, - shear, - interpolation="nearest", - fill=0, - center=None): +def affine( + img, + angle, + translate, + scale, + shear, + interpolation="nearest", + fill=0, + center=None, +): """Apply affine transformation on the image. Args: @@ -615,11 +650,14 @@ def affine(img, print(affined_img.shape) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if not isinstance(angle, (int, float)): raise TypeError("Argument angle should be int or float") @@ -635,7 +673,8 @@ def affine(img, if not isinstance(shear, (numbers.Number, (list, tuple))): raise TypeError( - "Shear should be either a single value or a sequence of two values") + "Shear should be either a single value or a sequence of two values" + ) if not isinstance(interpolation, str): raise TypeError("Argument interpolation should be a string") @@ -657,7 +696,8 @@ def affine(img, if len(shear) != 2: raise ValueError( - f"Shear should be a sequence containing two values. Got {shear}") + f"Shear should be a sequence containing two values. Got {shear}" + ) if center is not None and not isinstance(center, (list, tuple)): raise TypeError("Argument center should be a sequence") @@ -680,8 +720,9 @@ def affine(img, # otherwise image rotated by 90 degrees is shifted vs output image of F_t.affine if center is None: center = (width * 0.5, height * 0.5) - return F_cv2.affine(img, angle, translate, scale, shear, interpolation, - fill, center) + return F_cv2.affine( + img, angle, translate, scale, shear, interpolation, fill, center + ) if _is_tensor_image(img): center_f = [0.0, 0.0] @@ -696,12 +737,9 @@ def affine(img, return F_t.affine(img, matrix, interpolation, fill) -def rotate(img, - angle, - interpolation="nearest", - expand=False, - center=None, - fill=0): +def rotate( + img, angle, interpolation="nearest", expand=False, center=None, fill=0 +): """Rotates the image by angle. @@ -747,11 +785,14 @@ def rotate(img, print(rotated_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if isinstance(center, list): center = tuple(center) @@ -784,10 +825,24 @@ def _get_perspective_coeffs(startpoints, endpoints): for i, (p1, p2) in enumerate(zip(endpoints, startpoints)): a_matrix[2 * i, :] = [ - p1[0], p1[1], 1, 0, 0, 0, -p2[0] * p1[0], -p2[0] * p1[1] + p1[0], + p1[1], + 1, + 0, + 0, + 0, + -p2[0] * p1[0], + -p2[0] * p1[1], ] a_matrix[2 * i + 1, :] = [ - 0, 0, 0, p1[0], p1[1], 1, -p2[1] * p1[0], -p2[1] * p1[1] + 0, + 0, + 0, + p1[0], + p1[1], + 1, + -p2[1] * p1[0], + -p2[1] * p1[1], ] b_matrix = np.array(startpoints).reshape([8]) @@ -838,11 +893,14 @@ def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): print(perspectived_img.shape) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): coeffs = _get_perspective_coeffs(startpoints, endpoints) @@ -851,8 +909,9 @@ def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): coeffs = _get_perspective_coeffs(startpoints, endpoints) return F_t.perspective(img, coeffs, interpolation, fill) else: - return F_cv2.perspective(img, startpoints, endpoints, interpolation, - fill) + return F_cv2.perspective( + img, startpoints, endpoints, interpolation, fill + ) def to_grayscale(img, num_output_channels=1): @@ -882,11 +941,14 @@ def to_grayscale(img, num_output_channels=1): print(gray_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.to_grayscale(img, num_output_channels) @@ -942,53 +1004,53 @@ def normalize(img, mean, std, data_format='CHW', to_rgb=False): def erase(img, i, j, h, w, v, inplace=False): """Erase the pixels of selected area in input image with given value. - Args: - img (paddle.Tensor | np.array | PIL.Image): input Tensor image. - For Tensor input, the shape should be (C, H, W). For np.array input, - the shape should be (H, W, C). - i (int): y coordinate of the top-left point of erased region. - j (int): x coordinate of the top-left point of erased region. - h (int): Height of the erased region. - w (int): Width of the erased region. - v (paddle.Tensor | np.array): value used to replace the pixels in erased region. It - should be np.array when img is np.array or PIL.Image. - inplace (bool, optional): Whether this transform is inplace. Default: False. - - Returns: - paddle.Tensor | np.array | PIL.Image: Erased image. The type is same with input image. - - Examples: - .. code-block:: python - - import paddle - - fake_img = paddle.randn((3, 2, 4)).astype(paddle.float32) - print(fake_img) - - #Tensor(shape=[3, 2, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True, - # [[[ 0.02169025, -0.97859967, -1.39175487, -1.07478464], - # [ 0.20654772, 1.74624777, 0.32268861, -0.13857445]], - # - # [[-0.14993843, 1.10793507, -0.40056887, -1.94395220], - # [ 0.41686651, 0.44551995, -0.09356714, -0.60898107]], - # - # [[-0.24998808, -1.47699273, -0.88838995, 0.42629015], - # [ 0.56948012, -0.96200180, 0.53355658, 3.20450878]]]) - - values = paddle.zeros((1,1,1), dtype=paddle.float32) - result = paddle.vision.transforms.erase(fake_img, 0, 1, 1, 2, values) - - print(result) - - #Tensor(shape=[3, 2, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True, - # [[[ 0.02169025, 0. , 0. , -1.07478464], - # [ 0.20654772, 1.74624777, 0.32268861, -0.13857445]], - # - # [[-0.14993843, 0. , 0. , -1.94395220], - # [ 0.41686651, 0.44551995, -0.09356714, -0.60898107]], - # - # [[-0.24998808, 0. , 0. , 0.42629015], - # [ 0.56948012, -0.96200180, 0.53355658, 3.20450878]]]) + Args: + img (paddle.Tensor | np.array | PIL.Image): input Tensor image. + For Tensor input, the shape should be (C, H, W). For np.array input, + the shape should be (H, W, C). + i (int): y coordinate of the top-left point of erased region. + j (int): x coordinate of the top-left point of erased region. + h (int): Height of the erased region. + w (int): Width of the erased region. + v (paddle.Tensor | np.array): value used to replace the pixels in erased region. It + should be np.array when img is np.array or PIL.Image. + inplace (bool, optional): Whether this transform is inplace. Default: False. + + Returns: + paddle.Tensor | np.array | PIL.Image: Erased image. The type is same with input image. + + Examples: + .. code-block:: python + + import paddle + + fake_img = paddle.randn((3, 2, 4)).astype(paddle.float32) + print(fake_img) + + #Tensor(shape=[3, 2, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True, + # [[[ 0.02169025, -0.97859967, -1.39175487, -1.07478464], + # [ 0.20654772, 1.74624777, 0.32268861, -0.13857445]], + # + # [[-0.14993843, 1.10793507, -0.40056887, -1.94395220], + # [ 0.41686651, 0.44551995, -0.09356714, -0.60898107]], + # + # [[-0.24998808, -1.47699273, -0.88838995, 0.42629015], + # [ 0.56948012, -0.96200180, 0.53355658, 3.20450878]]]) + + values = paddle.zeros((1,1,1), dtype=paddle.float32) + result = paddle.vision.transforms.erase(fake_img, 0, 1, 1, 2, values) + + print(result) + + #Tensor(shape=[3, 2, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True, + # [[[ 0.02169025, 0. , 0. , -1.07478464], + # [ 0.20654772, 1.74624777, 0.32268861, -0.13857445]], + # + # [[-0.14993843, 0. , 0. , -1.94395220], + # [ 0.41686651, 0.44551995, -0.09356714, -0.60898107]], + # + # [[-0.24998808, 0. , 0. , 0.42629015], + # [ 0.56948012, -0.96200180, 0.53355658, 3.20450878]]]) """ if _is_tensor_image(img): diff --git a/python/paddle/vision/transforms/functional_cv2.py b/python/paddle/vision/transforms/functional_cv2.py index ccd638089af..52609b786fe 100644 --- a/python/paddle/vision/transforms/functional_cv2.py +++ b/python/paddle/vision/transforms/functional_cv2.py @@ -49,7 +49,8 @@ def to_tensor(pic, data_format='CHW'): if data_format not in ['CHW', 'HWC']: raise ValueError( - 'data_format should be CHW or HWC. Got {}'.format(data_format)) + 'data_format should be CHW or HWC. Got {}'.format(data_format) + ) if pic.ndim == 2: pic = pic[:, :, None] @@ -60,7 +61,7 @@ def to_tensor(pic, data_format='CHW'): img = paddle.to_tensor(pic) if paddle.fluid.data_feeder.convert_dtype(img.dtype) == 'uint8': - return paddle.cast(img, np.float32) / 255. + return paddle.cast(img, np.float32) / 255.0 else: return img @@ -90,11 +91,12 @@ def resize(img, size, interpolation='bilinear'): 'bilinear': cv2.INTER_LINEAR, 'area': cv2.INTER_AREA, 'bicubic': cv2.INTER_CUBIC, - 'lanczos': cv2.INTER_LANCZOS4 + 'lanczos': cv2.INTER_LANCZOS4, } - if not (isinstance(size, int) or - (isinstance(size, Iterable) and len(size) == 2)): + if not ( + isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2) + ): raise TypeError('Got inappropriate size arg: {}'.format(size)) h, w = img.shape[:2] @@ -108,18 +110,22 @@ def resize(img, size, interpolation='bilinear'): output = cv2.resize( img, dsize=(ow, oh), - interpolation=_cv2_interp_from_str[interpolation]) + interpolation=_cv2_interp_from_str[interpolation], + ) else: oh = size ow = int(size * w / h) output = cv2.resize( img, dsize=(ow, oh), - interpolation=_cv2_interp_from_str[interpolation]) + interpolation=_cv2_interp_from_str[interpolation], + ) else: - output = cv2.resize(img, - dsize=(size[1], size[0]), - interpolation=_cv2_interp_from_str[interpolation]) + output = cv2.resize( + img, + dsize=(size[1], size[0]), + interpolation=_cv2_interp_from_str[interpolation], + ) if len(img.shape) == 3 and img.shape[2] == 1: return output[:, :, np.newaxis] else: @@ -165,7 +171,7 @@ def pad(img, padding, fill=0, padding_mode='constant'): 'constant': cv2.BORDER_CONSTANT, 'edge': cv2.BORDER_REPLICATE, 'reflect': cv2.BORDER_REFLECT_101, - 'symmetric': cv2.BORDER_REFLECT + 'symmetric': cv2.BORDER_REFLECT, } if not isinstance(padding, (numbers.Number, list, tuple)): @@ -177,11 +183,16 @@ def pad(img, padding, fill=0, padding_mode='constant'): if isinstance(padding, Sequence) and len(padding) not in [2, 4]: raise ValueError( - "Padding must be an int or a 2, or 4 element tuple, not a " + - "{} element tuple".format(len(padding))) + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding)) + ) - assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \ - 'Padding mode should be either constant, edge, reflect or symmetric' + assert padding_mode in [ + 'constant', + 'edge', + 'reflect', + 'symmetric', + ], 'Padding mode should be either constant, edge, reflect or symmetric' if isinstance(padding, list): padding = tuple(padding) @@ -197,21 +208,25 @@ def pad(img, padding, fill=0, padding_mode='constant'): pad_bottom = padding[3] if len(img.shape) == 3 and img.shape[2] == 1: - return cv2.copyMakeBorder(img, - top=pad_top, - bottom=pad_bottom, - left=pad_left, - right=pad_right, - borderType=_cv2_pad_from_str[padding_mode], - value=fill)[:, :, np.newaxis] + return cv2.copyMakeBorder( + img, + top=pad_top, + bottom=pad_bottom, + left=pad_left, + right=pad_right, + borderType=_cv2_pad_from_str[padding_mode], + value=fill, + )[:, :, np.newaxis] else: - return cv2.copyMakeBorder(img, - top=pad_top, - bottom=pad_bottom, - left=pad_left, - right=pad_right, - borderType=_cv2_pad_from_str[padding_mode], - value=fill) + return cv2.copyMakeBorder( + img, + top=pad_top, + bottom=pad_bottom, + left=pad_left, + right=pad_right, + borderType=_cv2_pad_from_str[padding_mode], + value=fill, + ) def crop(img, top, left, height, width): @@ -230,30 +245,30 @@ def crop(img, top, left, height, width): """ - return img[top:top + height, left:left + width, :] + return img[top : top + height, left : left + width, :] def center_crop(img, output_size): """Crops the given image and resize it to desired size. - Args: - img (np.array): Image to be cropped. (0,0) denotes the top left corner of the image. - output_size (sequence or int): (height, width) of the crop box. If int, - it is used for both directions - backend (str, optional): The image proccess backend type. Options are `pil`, `cv2`. Default: 'pil'. + Args: + img (np.array): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions + backend (str, optional): The image proccess backend type. Options are `pil`, `cv2`. Default: 'pil'. - Returns: - np.array: Cropped image. + Returns: + np.array: Cropped image. - """ + """ if isinstance(output_size, numbers.Number): output_size = (int(output_size), int(output_size)) h, w = img.shape[0:2] th, tw = output_size - i = int(round((h - th) / 2.)) - j = int(round((w - tw) / 2.)) + i = int(round((h - th) / 2.0)) + j = int(round((w - tw) / 2.0)) return crop(img, i, j, th, tw) @@ -305,8 +320,11 @@ def adjust_brightness(img, brightness_factor): """ cv2 = try_import('cv2') - table = np.array([i * brightness_factor - for i in range(0, 256)]).clip(0, 255).astype('uint8') + table = ( + np.array([i * brightness_factor for i in range(0, 256)]) + .clip(0, 255) + .astype('uint8') + ) if len(img.shape) == 3 and img.shape[2] == 1: return cv2.LUT(img, table)[:, :, np.newaxis] @@ -329,8 +347,11 @@ def adjust_contrast(img, contrast_factor): """ cv2 = try_import('cv2') - table = np.array([(i - 74) * contrast_factor + 74 - for i in range(0, 256)]).clip(0, 255).astype('uint8') + table = ( + np.array([(i - 74) * contrast_factor + 74 for i in range(0, 256)]) + .clip(0, 255) + .astype('uint8') + ) if len(img.shape) == 3 and img.shape[2] == 1: return cv2.LUT(img, table)[:, :, np.newaxis] else: @@ -354,8 +375,9 @@ def adjust_saturation(img, saturation_factor): dtype = img.dtype img = img.astype(np.float32) - alpha = np.random.uniform(max(0, 1 - saturation_factor), - 1 + saturation_factor) + alpha = np.random.uniform( + max(0, 1 - saturation_factor), 1 + saturation_factor + ) gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray_img = gray_img[..., np.newaxis] img = img * alpha + gray_img * (1 - alpha) @@ -388,7 +410,8 @@ def adjust_hue(img, hue_factor): if not (-0.5 <= hue_factor <= 0.5): raise ValueError( - 'hue_factor:{} is not in [-0.5, 0.5].'.format(hue_factor)) + 'hue_factor:{} is not in [-0.5, 0.5].'.format(hue_factor) + ) dtype = img.dtype img = img.astype(np.uint8) @@ -404,14 +427,16 @@ def adjust_hue(img, hue_factor): return cv2.cvtColor(hsv_img, cv2.COLOR_HSV2BGR_FULL).astype(dtype) -def affine(img, - angle, - translate, - scale, - shear, - interpolation='nearest', - fill=0, - center=None): +def affine( + img, + angle, + translate, + scale, + shear, + interpolation='nearest', + fill=0, + center=None, +): """Affine the image by matrix. Args: @@ -442,7 +467,7 @@ def affine(img, 'bilinear': cv2.INTER_LINEAR, 'area': cv2.INTER_AREA, 'bicubic': cv2.INTER_CUBIC, - 'lanczos': cv2.INTER_LANCZOS4 + 'lanczos': cv2.INTER_LANCZOS4, } h, w = img.shape[0:2] @@ -469,25 +494,26 @@ def affine(img, M[1, 2] = ty if len(img.shape) == 3 and img.shape[2] == 1: - return cv2.warpAffine(img, - M, - dsize=(w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill)[:, :, np.newaxis] + return cv2.warpAffine( + img, + M, + dsize=(w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + )[:, :, np.newaxis] else: - return cv2.warpAffine(img, - M, - dsize=(w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill) - - -def rotate(img, - angle, - interpolation='nearest', - expand=False, - center=None, - fill=0): + return cv2.warpAffine( + img, + M, + dsize=(w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + ) + + +def rotate( + img, angle, interpolation='nearest', expand=False, center=None, fill=0 +): """Rotates the image by angle. Args: @@ -519,7 +545,7 @@ def rotate(img, 'bilinear': cv2.INTER_LINEAR, 'area': cv2.INTER_AREA, 'bicubic': cv2.INTER_CUBIC, - 'lanczos': cv2.INTER_LANCZOS4 + 'lanczos': cv2.INTER_LANCZOS4, } h, w = img.shape[0:2] @@ -549,8 +575,10 @@ def rotate(img, post_trans = (0, 0) expand_matrix[2], expand_matrix[5] = transform( - -center[0] - post_trans[0], -center[1] - post_trans[1], - expand_matrix) + -center[0] - post_trans[0], + -center[1] - post_trans[1], + expand_matrix, + ) expand_matrix[2] += center[0] expand_matrix[5] += center[1] @@ -567,15 +595,21 @@ def rotate(img, w, h = int(nw), int(nh) if len(img.shape) == 3 and img.shape[2] == 1: - return cv2.warpAffine(img, - M, (w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill)[:, :, np.newaxis] + return cv2.warpAffine( + img, + M, + (w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + )[:, :, np.newaxis] else: - return cv2.warpAffine(img, - M, (w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill) + return cv2.warpAffine( + img, + M, + (w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + ) def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): @@ -604,7 +638,7 @@ def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): 'bilinear': cv2.INTER_LINEAR, 'area': cv2.INTER_AREA, 'bicubic': cv2.INTER_CUBIC, - 'lanczos': cv2.INTER_LANCZOS4 + 'lanczos': cv2.INTER_LANCZOS4, } h, w = img.shape[0:2] @@ -613,17 +647,21 @@ def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): matrix = cv2.getPerspectiveTransform(startpoints, endpoints) if len(img.shape) == 3 and img.shape[2] == 1: - return cv2.warpPerspective(img, - matrix, - dsize=(w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill)[:, :, np.newaxis] + return cv2.warpPerspective( + img, + matrix, + dsize=(w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + )[:, :, np.newaxis] else: - return cv2.warpPerspective(img, - matrix, - dsize=(w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill) + return cv2.warpPerspective( + img, + matrix, + dsize=(w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + ) def to_grayscale(img, num_output_channels=1): @@ -646,7 +684,8 @@ def to_grayscale(img, num_output_channels=1): elif num_output_channels == 3: # much faster than doing cvtColor to go back to gray img = np.broadcast_to( - cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)[:, :, np.newaxis], img.shape) + cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)[:, :, np.newaxis], img.shape + ) else: raise ValueError('num_output_channels should be either 1 or 3') @@ -686,21 +725,21 @@ def normalize(img, mean, std, data_format='CHW', to_rgb=False): def erase(img, i, j, h, w, v, inplace=False): """Erase the pixels of selected area in input image array with given value. - Args: - img (np.array): input image array, which shape is (H, W, C). - i (int): y coordinate of the top-left point of erased region. - j (int): x coordinate of the top-left point of erased region. - h (int): Height of the erased region. - w (int): Width of the erased region. - v (np.array): value used to replace the pixels in erased region. - inplace (bool, optional): Whether this transform is inplace. Default: False. + Args: + img (np.array): input image array, which shape is (H, W, C). + i (int): y coordinate of the top-left point of erased region. + j (int): x coordinate of the top-left point of erased region. + h (int): Height of the erased region. + w (int): Width of the erased region. + v (np.array): value used to replace the pixels in erased region. + inplace (bool, optional): Whether this transform is inplace. Default: False. - Returns: - np.array: Erased image. + Returns: + np.array: Erased image. """ if not inplace: img = img.copy() - img[i:i + h, j:j + w, ...] = v + img[i : i + h, j : j + w, ...] = v return img diff --git a/python/paddle/vision/transforms/functional_pil.py b/python/paddle/vision/transforms/functional_pil.py index cd4556effb7..595d92a84b1 100644 --- a/python/paddle/vision/transforms/functional_pil.py +++ b/python/paddle/vision/transforms/functional_pil.py @@ -35,7 +35,7 @@ try: 'bicubic': Image.Resampling.BICUBIC, 'box': Image.Resampling.BOX, 'lanczos': Image.Resampling.LANCZOS, - 'hamming': Image.Resampling.HAMMING + 'hamming': Image.Resampling.HAMMING, } except: _pil_interp_from_str = { @@ -44,7 +44,7 @@ except: 'bicubic': Image.BICUBIC, 'box': Image.BOX, 'lanczos': Image.LANCZOS, - 'hamming': Image.HAMMING + 'hamming': Image.HAMMING, } __all__ = [] @@ -67,7 +67,8 @@ def to_tensor(pic, data_format='CHW'): if data_format not in ['CHW', 'HWC']: raise ValueError( - 'data_format should be CHW or HWC. Got {}'.format(data_format)) + 'data_format should be CHW or HWC. Got {}'.format(data_format) + ) # PIL Image if pic.mode == 'I': @@ -91,7 +92,7 @@ def to_tensor(pic, data_format='CHW'): dtype = paddle.fluid.data_feeder.convert_dtype(img.dtype) if dtype == 'uint8': - img = paddle.cast(img, np.float32) / 255. + img = paddle.cast(img, np.float32) / 255.0 img = img.reshape([pic.size[1], pic.size[0], nchannel]) @@ -122,8 +123,9 @@ def resize(img, size, interpolation='bilinear'): """ - if not (isinstance(size, int) or - (isinstance(size, Iterable) and len(size) == 2)): + if not ( + isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2) + ): raise TypeError('Got inappropriate size arg: {}'.format(size)) if isinstance(size, int): @@ -186,11 +188,16 @@ def pad(img, padding, fill=0, padding_mode='constant'): if isinstance(padding, Sequence) and len(padding) not in [2, 4]: raise ValueError( - "Padding must be an int or a 2, or 4 element tuple, not a " + - "{} element tuple".format(len(padding))) + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding)) + ) - assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \ - 'Padding mode should be either constant, edge, reflect or symmetric' + assert padding_mode in [ + 'constant', + 'edge', + 'reflect', + 'symmetric', + ], 'Padding mode should be either constant, edge, reflect or symmetric' if isinstance(padding, list): padding = tuple(padding) @@ -217,8 +224,11 @@ def pad(img, padding, fill=0, padding_mode='constant'): if img.mode == 'P': palette = img.getpalette() img = np.asarray(img) - img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right)), - padding_mode) + img = np.pad( + img, + ((pad_top, pad_bottom), (pad_left, pad_right)), + padding_mode, + ) img = Image.fromarray(img) img.putpalette(palette) return img @@ -226,13 +236,18 @@ def pad(img, padding, fill=0, padding_mode='constant'): img = np.asarray(img) # RGB image if len(img.shape) == 3: - img = np.pad(img, - ((pad_top, pad_bottom), (pad_left, pad_right), (0, 0)), - padding_mode) + img = np.pad( + img, + ((pad_top, pad_bottom), (pad_left, pad_right), (0, 0)), + padding_mode, + ) # Grayscale image if len(img.shape) == 2: - img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right)), - padding_mode) + img = np.pad( + img, + ((pad_top, pad_bottom), (pad_left, pad_right)), + padding_mode, + ) return Image.fromarray(img) @@ -258,24 +273,24 @@ def crop(img, top, left, height, width): def center_crop(img, output_size): """Crops the given PIL Image and resize it to desired size. - Args: - img (PIL.Image): Image to be cropped. (0,0) denotes the top left corner of the image. - output_size (sequence or int): (height, width) of the crop box. If int, - it is used for both directions - backend (str, optional): The image proccess backend type. Options are `pil`, `cv2`. Default: 'pil'. + Args: + img (PIL.Image): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions + backend (str, optional): The image proccess backend type. Options are `pil`, `cv2`. Default: 'pil'. - Returns: - PIL.Image: Cropped image. + Returns: + PIL.Image: Cropped image. - """ + """ if isinstance(output_size, numbers.Number): output_size = (int(output_size), int(output_size)) image_width, image_height = img.size crop_height, crop_width = output_size - crop_top = int(round((image_height - crop_height) / 2.)) - crop_left = int(round((image_width - crop_width) / 2.)) + crop_top = int(round((image_height - crop_height) / 2.0)) + crop_left = int(round((image_width - crop_width) / 2.0)) return crop(img, crop_top, crop_left, crop_height, crop_width) @@ -388,7 +403,8 @@ def adjust_hue(img, hue_factor): """ if not (-0.5 <= hue_factor <= 0.5): raise ValueError( - 'hue_factor:{} is not in [-0.5, 0.5].'.format(hue_factor)) + 'hue_factor:{} is not in [-0.5, 0.5].'.format(hue_factor) + ) input_mode = img.mode if input_mode in {'L', '1', 'I', 'F'}: @@ -428,16 +444,18 @@ def affine(img, matrix, interpolation="nearest", fill=0): if isinstance(fill, int): fill = tuple([fill] * 3) - return img.transform(img.size, Image.AFFINE, matrix, - _pil_interp_from_str[interpolation], fill) + return img.transform( + img.size, + Image.AFFINE, + matrix, + _pil_interp_from_str[interpolation], + fill, + ) -def rotate(img, - angle, - interpolation="nearest", - expand=False, - center=None, - fill=0): +def rotate( + img, angle, interpolation="nearest", expand=False, center=None, fill=0 +): """Rotates the image by angle. Args: @@ -467,11 +485,13 @@ def rotate(img, if isinstance(fill, int): fill = tuple([fill] * 3) - return img.rotate(angle, - _pil_interp_from_str[interpolation], - expand, - center, - fillcolor=fill) + return img.rotate( + angle, + _pil_interp_from_str[interpolation], + expand, + center, + fillcolor=fill, + ) def perspective(img, coeffs, interpolation="nearest", fill=0): @@ -497,8 +517,13 @@ def perspective(img, coeffs, interpolation="nearest", fill=0): if isinstance(fill, int): fill = tuple([fill] * 3) - return img.transform(img.size, Image.PERSPECTIVE, coeffs, - _pil_interp_from_str[interpolation], fill) + return img.transform( + img.size, + Image.PERSPECTIVE, + coeffs, + _pil_interp_from_str[interpolation], + fill, + ) def to_grayscale(img, num_output_channels=1): @@ -532,22 +557,22 @@ def to_grayscale(img, num_output_channels=1): def erase(img, i, j, h, w, v, inplace=False): """Erase the pixels of selected area in input image with given value. PIL format is - not support inplace. + not support inplace. - Args: - img (PIL.Image): input image, which shape is (C, H, W). - i (int): y coordinate of the top-left point of erased region. - j (int): x coordinate of the top-left point of erased region. - h (int): Height of the erased region. - w (int): Width of the erased region. - v (np.array): value used to replace the pixels in erased region. - inplace (bool, optional): Whether this transform is inplace. Default: False. + Args: + img (PIL.Image): input image, which shape is (C, H, W). + i (int): y coordinate of the top-left point of erased region. + j (int): x coordinate of the top-left point of erased region. + h (int): Height of the erased region. + w (int): Width of the erased region. + v (np.array): value used to replace the pixels in erased region. + inplace (bool, optional): Whether this transform is inplace. Default: False. - Returns: - PIL.Image: Erased image. + Returns: + PIL.Image: Erased image. """ np_img = np.array(img, dtype=np.uint8) - np_img[i:i + h, j:j + w, ...] = v + np_img[i : i + h, j : j + w, ...] = v img = Image.fromarray(np_img, 'RGB') return img diff --git a/python/paddle/vision/transforms/functional_tensor.py b/python/paddle/vision/transforms/functional_tensor.py index 911f7e2d260..d18fdfc51c9 100644 --- a/python/paddle/vision/transforms/functional_tensor.py +++ b/python/paddle/vision/transforms/functional_tensor.py @@ -22,13 +22,17 @@ __all__ = [] def _assert_image_tensor(img, data_format): - if not isinstance( - img, paddle.Tensor - ) or img.ndim < 3 or img.ndim > 4 or not data_format.lower() in ('chw', - 'hwc'): + if ( + not isinstance(img, paddle.Tensor) + or img.ndim < 3 + or img.ndim > 4 + or not data_format.lower() in ('chw', 'hwc') + ): raise RuntimeError( - 'not support [type={}, ndim={}, data_format={}] paddle image'. - format(type(img), img.ndim, data_format)) + 'not support [type={}, ndim={}, data_format={}] paddle image'.format( + type(img), img.ndim, data_format + ) + ) def _get_image_h_axis(data_format): @@ -78,13 +82,15 @@ def _get_image_num_channels(img, data_format): def _get_image_size(img, data_format): - return img.shape[_get_image_w_axis(data_format)], img.shape[ - _get_image_h_axis(data_format)] + return ( + img.shape[_get_image_w_axis(data_format)], + img.shape[_get_image_h_axis(data_format)], + ) def _rgb_to_hsv(img): """Convert a image Tensor from RGB to HSV. This implementation is based on Pillow ( - https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Convert.c) + https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Convert.c) """ maxc = img.max(axis=-3) minc = img.min(axis=-3) @@ -111,8 +117,7 @@ def _rgb_to_hsv(img): def _hsv_to_rgb(img): - """Convert a image Tensor from HSV to RGB. - """ + """Convert a image Tensor from HSV to RGB.""" h, s, v = img.unbind(axis=-3) f = h * 6.0 i = paddle.floor(f) @@ -123,22 +128,28 @@ def _hsv_to_rgb(img): q = paddle.clip(v * (1.0 - s * f), 0.0, 1.0) t = paddle.clip(v * (1.0 - s * (1.0 - f)), 0.0, 1.0) - mask = paddle.equal(i.unsqueeze(axis=-3), - paddle.arange(6, dtype=i.dtype).reshape( - (-1, 1, 1))).astype(img.dtype) - matrix = paddle.stack([ - paddle.stack([v, q, p, p, t, v], axis=-3), - paddle.stack([t, v, v, q, p, p], axis=-3), - paddle.stack([p, p, t, v, v, q], axis=-3) - ], - axis=-4) + mask = paddle.equal( + i.unsqueeze(axis=-3), + paddle.arange(6, dtype=i.dtype).reshape((-1, 1, 1)), + ).astype(img.dtype) + matrix = paddle.stack( + [ + paddle.stack([v, q, p, p, t, v], axis=-3), + paddle.stack([t, v, v, q, p, p], axis=-3), + paddle.stack([p, p, t, v, v, q], axis=-3), + ], + axis=-4, + ) return paddle.einsum("...ijk, ...xijk -> ...xjk", mask, matrix) def _blend_images(img1, img2, ratio): max_value = 1.0 if paddle.is_floating_point(img1) else 255.0 - return paddle.lerp(img2, img1, - float(ratio)).clip(0, max_value).astype(img1.dtype) + return ( + paddle.lerp(img2, img1, float(ratio)) + .clip(0, max_value) + .astype(img1.dtype) + ) def normalize(img, mean, std, data_format='CHW'): @@ -186,8 +197,9 @@ def to_grayscale(img, num_output_channels=1, data_format='CHW'): if num_output_channels not in (1, 3): raise ValueError('num_output_channels should be either 1 or 3') - rgb_weights = paddle.to_tensor([0.2989, 0.5870, 0.1140], - place=img.place).astype(img.dtype) + rgb_weights = paddle.to_tensor( + [0.2989, 0.5870, 0.1140], place=img.place + ).astype(img.dtype) if _is_channel_first(data_format): rgb_weights = rgb_weights.reshape((-1, 1, 1)) @@ -210,8 +222,9 @@ def _affine_grid(theta, w, h, ow, oh): y_grid = paddle.linspace(-oh * 0.5 + d, oh * 0.5 + d - 1, oh).unsqueeze_(-1) base_grid[..., 1] = y_grid - scaled_theta = theta.transpose( - (0, 2, 1)) / paddle.to_tensor([0.5 * w, 0.5 * h]) + scaled_theta = theta.transpose((0, 2, 1)) / paddle.to_tensor( + [0.5 * w, 0.5 * h] + ) output_grid = base_grid.reshape((1, oh * ow, 3)).bmm(scaled_theta) return output_grid.reshape((1, oh, ow, 2)) @@ -220,18 +233,18 @@ def _affine_grid(theta, w, h, ow, oh): def _grid_transform(img, grid, mode, fill): if img.shape[0] > 1: grid = grid.expand( - shape=[img.shape[0], grid.shape[1], grid.shape[2], grid.shape[3]]) + shape=[img.shape[0], grid.shape[1], grid.shape[2], grid.shape[3]] + ) if fill is not None: - dummy = paddle.ones((img.shape[0], 1, img.shape[2], img.shape[3]), - dtype=img.dtype) + dummy = paddle.ones( + (img.shape[0], 1, img.shape[2], img.shape[3]), dtype=img.dtype + ) img = paddle.concat((img, dummy), axis=1) - img = F.grid_sample(img, - grid, - mode=mode, - padding_mode="zeros", - align_corners=False) + img = F.grid_sample( + img, grid, mode=mode, padding_mode="zeros", align_corners=False + ) # Fill with required color if fill is not None: @@ -239,12 +252,13 @@ def _grid_transform(img, grid, mode, fill): img = img[:, :-1, :, :] # n c h w mask = mask.expand_as(img) len_fill = len(fill) if isinstance(fill, (tuple, list)) else 1 - fill_img = paddle.to_tensor(fill).reshape( - (1, len_fill, 1, 1)).expand_as(img) + fill_img = ( + paddle.to_tensor(fill).reshape((1, len_fill, 1, 1)).expand_as(img) + ) if mode == 'nearest': mask = paddle.cast(mask < 0.5, img.dtype) - img = img * (1. - mask) + mask * fill_img + img = img * (1.0 - mask) + mask * fill_img else: # 'bilinear' img = img * mask + (1.0 - mask) * fill_img @@ -282,11 +296,9 @@ def affine(img, matrix, interpolation="nearest", fill=None, data_format='CHW'): matrix = matrix.reshape((1, 2, 3)) shape = img.shape - grid = _affine_grid(matrix, - w=shape[-1], - h=shape[-2], - ow=shape[-1], - oh=shape[-2]) + grid = _affine_grid( + matrix, w=shape[-1], h=shape[-2], ow=shape[-1], oh=shape[-2] + ) if isinstance(fill, int): fill = tuple([fill] * 3) @@ -299,13 +311,15 @@ def affine(img, matrix, interpolation="nearest", fill=None, data_format='CHW'): return out -def rotate(img, - angle, - interpolation='nearest', - expand=False, - center=None, - fill=None, - data_format='CHW'): +def rotate( + img, + angle, + interpolation='nearest', + expand=False, + center=None, + fill=None, + data_format='CHW', +): """Rotates the image by angle. Args: @@ -358,9 +372,11 @@ def rotate(img, ] matrix[2] += matrix[0] * (-rotn_center[0] - post_trans[0]) + matrix[1] * ( - -rotn_center[1] - post_trans[1]) + -rotn_center[1] - post_trans[1] + ) matrix[5] += matrix[3] * (-rotn_center[0] - post_trans[0]) + matrix[4] * ( - -rotn_center[1] - post_trans[1]) + -rotn_center[1] - post_trans[1] + ) matrix[2] += rotn_center[0] matrix[5] += rotn_center[1] @@ -371,12 +387,20 @@ def rotate(img, if expand: # calculate output size corners = paddle.to_tensor( - [[-0.5 * w, -0.5 * h, 1.0], [-0.5 * w, 0.5 * h, 1.0], - [0.5 * w, 0.5 * h, 1.0], [0.5 * w, -0.5 * h, 1.0]], - place=matrix.place).astype(matrix.dtype) - - _pos = corners.reshape((1, -1, 3)).bmm(matrix.transpose( - (0, 2, 1))).reshape((1, -1, 2)) + [ + [-0.5 * w, -0.5 * h, 1.0], + [-0.5 * w, 0.5 * h, 1.0], + [0.5 * w, 0.5 * h, 1.0], + [0.5 * w, -0.5 * h, 1.0], + ], + place=matrix.place, + ).astype(matrix.dtype) + + _pos = ( + corners.reshape((1, -1, 3)) + .bmm(matrix.transpose((0, 2, 1))) + .reshape((1, -1, 2)) + ) _min = _pos.min(axis=-2).floor() _max = _pos.max(axis=-2).ceil() @@ -412,21 +436,21 @@ def _perspective_grid(img, coeffs, ow, oh, dtype): y_grid = paddle.linspace(d, oh * 1.0 + d - 1.0, oh).unsqueeze_(-1) base_grid[..., 1] = y_grid - scaled_theta1 = theta1.transpose( - (0, 2, 1)) / paddle.to_tensor([0.5 * ow, 0.5 * oh]) + scaled_theta1 = theta1.transpose((0, 2, 1)) / paddle.to_tensor( + [0.5 * ow, 0.5 * oh] + ) output_grid1 = base_grid.reshape((1, oh * ow, 3)).bmm(scaled_theta1) - output_grid2 = base_grid.reshape( - (1, oh * ow, 3)).bmm(theta2.transpose((0, 2, 1))) + output_grid2 = base_grid.reshape((1, oh * ow, 3)).bmm( + theta2.transpose((0, 2, 1)) + ) output_grid = output_grid1 / output_grid2 - 1.0 return output_grid.reshape((1, oh, ow, 2)) -def perspective(img, - coeffs, - interpolation="nearest", - fill=None, - data_format='CHW'): +def perspective( + img, coeffs, interpolation="nearest", fill=None, data_format='CHW' +): """Perspective the image. Args: @@ -521,48 +545,48 @@ def crop(img, top, left, height, width, data_format='CHW'): _assert_image_tensor(img, data_format) if _is_channel_first(data_format): - return img[:, top:top + height, left:left + width] + return img[:, top : top + height, left : left + width] else: - return img[top:top + height, left:left + width, :] + return img[top : top + height, left : left + width, :] def erase(img, i, j, h, w, v, inplace=False): """Erase the pixels of selected area in input Tensor image with given value. - Args: - img (paddle.Tensor): input Tensor image. - i (int): y coordinate of the top-left point of erased region. - j (int): x coordinate of the top-left point of erased region. - h (int): Height of the erased region. - w (int): Width of the erased region. - v (paddle.Tensor): value used to replace the pixels in erased region. - inplace (bool, optional): Whether this transform is inplace. Default: False. + Args: + img (paddle.Tensor): input Tensor image. + i (int): y coordinate of the top-left point of erased region. + j (int): x coordinate of the top-left point of erased region. + h (int): Height of the erased region. + w (int): Width of the erased region. + v (paddle.Tensor): value used to replace the pixels in erased region. + inplace (bool, optional): Whether this transform is inplace. Default: False. - Returns: - paddle.Tensor: Erased image. + Returns: + paddle.Tensor: Erased image. """ _assert_image_tensor(img, 'CHW') if not inplace: img = img.clone() - img[..., i:i + h, j:j + w] = v + img[..., i : i + h, j : j + w] = v return img def center_crop(img, output_size, data_format='CHW'): """Crops the given paddle.Tensor Image and resize it to desired size. - Args: - img (paddle.Tensor): Image to be cropped. (0,0) denotes the top left corner of the image. - output_size (sequence or int): (height, width) of the crop box. If int, - it is used for both directions - data_format (str, optional): Data format of img, should be 'HWC' or - 'CHW'. Default: 'CHW'. - Returns: - paddle.Tensor: Cropped image. + Args: + img (paddle.Tensor): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions + data_format (str, optional): Data format of img, should be 'HWC' or + 'CHW'. Default: 'CHW'. + Returns: + paddle.Tensor: Cropped image. - """ + """ _assert_image_tensor(img, data_format) if isinstance(output_size, numbers.Number): @@ -570,14 +594,16 @@ def center_crop(img, output_size, data_format='CHW'): image_width, image_height = _get_image_size(img, data_format) crop_height, crop_width = output_size - crop_top = int(round((image_height - crop_height) / 2.)) - crop_left = int(round((image_width - crop_width) / 2.)) - return crop(img, - crop_top, - crop_left, - crop_height, - crop_width, - data_format=data_format) + crop_top = int(round((image_height - crop_height) / 2.0)) + crop_left = int(round((image_width - crop_width) / 2.0)) + return crop( + img, + crop_top, + crop_left, + crop_height, + crop_width, + data_format=data_format, + ) def pad(img, padding, fill=0, padding_mode='constant', data_format='CHW'): @@ -625,11 +651,16 @@ def pad(img, padding, fill=0, padding_mode='constant', data_format='CHW'): if isinstance(padding, (list, tuple)) and len(padding) not in [2, 4]: raise ValueError( - "Padding must be an int or a 2, or 4 element tuple, not a " + - "{} element tuple".format(len(padding))) + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding)) + ) - assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \ - 'Padding mode should be either constant, edge, reflect or symmetric' + assert padding_mode in [ + 'constant', + 'edge', + 'reflect', + 'symmetric', + ], 'Padding mode should be either constant, edge, reflect or symmetric' if isinstance(padding, int): pad_left = pad_right = pad_top = pad_bottom = padding @@ -651,11 +682,13 @@ def pad(img, padding, fill=0, padding_mode='constant', data_format='CHW'): img = img.unsqueeze(0) # 'constant', 'reflect', 'replicate', 'circular' - img = F.pad(img, - pad=padding, - mode=padding_mode, - value=float(fill), - data_format='N' + data_format) + img = F.pad( + img, + pad=padding, + mode=padding_mode, + value=float(fill), + data_format='N' + data_format, + ) return img.squeeze(0) @@ -684,8 +717,10 @@ def resize(img, size, interpolation='bilinear', data_format='CHW'): """ _assert_image_tensor(img, data_format) - if not (isinstance(size, int) or - (isinstance(size, (tuple, list)) and len(size) == 2)): + if not ( + isinstance(size, int) + or (isinstance(size, (tuple, list)) and len(size) == 2) + ): raise TypeError('Got inappropriate size arg: {}'.format(size)) if isinstance(size, int): @@ -702,10 +737,12 @@ def resize(img, size, interpolation='bilinear', data_format='CHW'): oh, ow = size img = img.unsqueeze(0) - img = F.interpolate(img, - size=(oh, ow), - mode=interpolation.lower(), - data_format='N' + data_format.upper()) + img = F.interpolate( + img, + size=(oh, ow), + mode=interpolation.lower(), + data_format='N' + data_format.upper(), + ) return img.squeeze(0) @@ -725,8 +762,10 @@ def adjust_brightness(img, brightness_factor): """ _assert_image_tensor(img, 'CHW') assert brightness_factor >= 0, "brightness_factor should be non-negative." - assert _get_image_num_channels( - img, 'CHW') in [1, 3], "channels of input should be either 1 or 3." + assert _get_image_num_channels(img, 'CHW') in [ + 1, + 3, + ], "channels of input should be either 1 or 3." extreme_target = paddle.zeros_like(img, img.dtype) return _blend_images(img, extreme_target, brightness_factor) @@ -751,13 +790,13 @@ def adjust_contrast(img, contrast_factor): channels = _get_image_num_channels(img, 'CHW') dtype = img.dtype if paddle.is_floating_point(img) else paddle.float32 if channels == 1: - extreme_target = paddle.mean(img.astype(dtype), - axis=(-3, -2, -1), - keepdim=True) + extreme_target = paddle.mean( + img.astype(dtype), axis=(-3, -2, -1), keepdim=True + ) elif channels == 3: - extreme_target = paddle.mean(to_grayscale(img).astype(dtype), - axis=(-3, -2, -1), - keepdim=True) + extreme_target = paddle.mean( + to_grayscale(img).astype(dtype), axis=(-3, -2, -1), keepdim=True + ) else: raise ValueError("channels of input should be either 1 or 3.") @@ -813,7 +852,9 @@ def adjust_hue(img, hue_factor): """ _assert_image_tensor(img, 'CHW') - assert hue_factor >= -0.5 and hue_factor <= 0.5, "hue_factor should be in range [-0.5, 0.5]" + assert ( + hue_factor >= -0.5 and hue_factor <= 0.5 + ), "hue_factor should be in range [-0.5, 0.5]" channels = _get_image_num_channels(img, 'CHW') if channels == 1: return img @@ -824,7 +865,7 @@ def adjust_hue(img, hue_factor): img_hsv = _rgb_to_hsv(img) h, s, v = img_hsv.unbind(axis=-3) - h = (h + hue_factor) + h = h + hue_factor h = h - h.floor() img_adjusted = _hsv_to_rgb(paddle.stack([h, s, v], axis=-3)) diff --git a/python/paddle/vision/transforms/transforms.py b/python/paddle/vision/transforms/transforms.py index 5472df2fd9e..37270c97b28 100644 --- a/python/paddle/vision/transforms/transforms.py +++ b/python/paddle/vision/transforms/transforms.py @@ -46,33 +46,38 @@ def _get_image_size(img): return img.shape[2:][::-1] # nchw -> wh else: raise ValueError( - "The dim for input Tensor should be 3-D or 4-D, but received {}" - .format(len(img.shape))) + "The dim for input Tensor should be 3-D or 4-D, but received {}".format( + len(img.shape) + ) + ) else: raise TypeError("Unexpected type {}".format(type(img))) -def _check_input(value, - name, - center=1, - bound=(0, float('inf')), - clip_first_on_zero=True): +def _check_input( + value, name, center=1, bound=(0, float('inf')), clip_first_on_zero=True +): if isinstance(value, numbers.Number): if value < 0: raise ValueError( "If {} is a single number, it must be non negative.".format( - name)) + name + ) + ) value = [center - value, center + value] if clip_first_on_zero: value[0] = max(value[0], 0) elif isinstance(value, (tuple, list)) and len(value) == 2: if not bound[0] <= value[0] <= value[1] <= bound[1]: - raise ValueError("{} values should be between {}".format( - name, bound)) + raise ValueError( + "{} values should be between {}".format(name, bound) + ) else: raise TypeError( - "{} should be a single number or a list/tuple with lenght 2.". - format(name)) + "{} should be a single number or a list/tuple with lenght 2.".format( + name + ) + ) if value[0] == value[1] == center: value = None @@ -116,8 +121,10 @@ class Compose(object): data = f(data) except Exception as e: stack_info = traceback.format_exc() - print("fail to perform transform [{}] with error: " - "{} and stack:\n{}".format(f, e, str(stack_info))) + print( + "fail to perform transform [{}] with error: " + "{} and stack:\n{}".format(f, e, str(stack_info)) + ) raise e return data @@ -240,14 +247,16 @@ class BaseTransform(object): def __init__(self, keys=None): if keys is None: - keys = ("image", ) + keys = ("image",) elif not isinstance(keys, Sequence): raise ValueError( - "keys should be a sequence, but got keys={}".format(keys)) + "keys should be a sequence, but got keys={}".format(keys) + ) for k in keys: if self._get_apply(k) is None: raise NotImplementedError( - "{} is unsupported data structure".format(k)) + "{} is unsupported data structure".format(k) + ) self.keys = keys # storage some params get from function get_params() @@ -259,7 +268,7 @@ class BaseTransform(object): def __call__(self, inputs): """Apply transform on single input data""" if not isinstance(inputs, tuple): - inputs = (inputs, ) + inputs = (inputs,) self.params = self._get_params(inputs) @@ -271,7 +280,7 @@ class BaseTransform(object): else: outputs.append(apply_func(inputs[i])) if len(inputs) > len(self.keys): - outputs.extend(inputs[len(self.keys):]) + outputs.extend(inputs[len(self.keys) :]) if len(outputs) == 1: outputs = outputs[0] @@ -413,8 +422,9 @@ class Resize(BaseTransform): def __init__(self, size, interpolation='bilinear', keys=None): super(Resize, self).__init__(keys) - assert isinstance(size, int) or (isinstance(size, Iterable) - and len(size) == 2) + assert isinstance(size, int) or ( + isinstance(size, Iterable) and len(size) == 2 + ) self.size = size self.interpolation = interpolation @@ -473,19 +483,21 @@ class RandomResizedCrop(BaseTransform): """ - def __init__(self, - size, - scale=(0.08, 1.0), - ratio=(3. / 4, 4. / 3), - interpolation='bilinear', - keys=None): + def __init__( + self, + size, + scale=(0.08, 1.0), + ratio=(3.0 / 4, 4.0 / 3), + interpolation='bilinear', + keys=None, + ): super(RandomResizedCrop, self).__init__(keys) if isinstance(size, int): self.size = (size, size) else: self.size = size - assert (scale[0] <= scale[1]), "scale should be of kind (min, max)" - assert (ratio[0] <= ratio[1]), "ratio should be of kind (min, max)" + assert scale[0] <= scale[1], "scale should be of kind (min, max)" + assert ratio[0] <= ratio[1], "ratio should be of kind (min, max)" self.scale = scale self.ratio = ratio self.interpolation = interpolation @@ -696,12 +708,9 @@ class Normalize(BaseTransform): """ - def __init__(self, - mean=0.0, - std=1.0, - data_format='CHW', - to_rgb=False, - keys=None): + def __init__( + self, mean=0.0, std=1.0, data_format='CHW', to_rgb=False, keys=None + ): super(Normalize, self).__init__(keys) if isinstance(mean, numbers.Number): mean = [mean, mean, mean] @@ -715,8 +724,9 @@ class Normalize(BaseTransform): self.to_rgb = to_rgb def _apply_image(self, img): - return F.normalize(img, self.mean, self.std, self.data_format, - self.to_rgb) + return F.normalize( + img, self.mean, self.std, self.data_format, self.to_rgb + ) class Transpose(BaseTransform): @@ -934,11 +944,9 @@ class HueTransform(BaseTransform): def __init__(self, value, keys=None): super(HueTransform, self).__init__(keys) - self.value = _check_input(value, - 'hue', - center=0, - bound=(-0.5, 0.5), - clip_first_on_zero=False) + self.value = _check_input( + value, 'hue', center=0, bound=(-0.5, 0.5), clip_first_on_zero=False + ) def _apply_image(self, img): if self.value is None: @@ -985,12 +993,9 @@ class ColorJitter(BaseTransform): """ - def __init__(self, - brightness=0, - contrast=0, - saturation=0, - hue=0, - keys=None): + def __init__( + self, brightness=0, contrast=0, saturation=0, hue=0, keys=None + ): super(ColorJitter, self).__init__(keys) self.brightness = brightness self.contrast = contrast @@ -1033,8 +1038,9 @@ class ColorJitter(BaseTransform): Returns: PIL Image: Color jittered image. """ - transform = self._get_param(self.brightness, self.contrast, - self.saturation, self.hue) + transform = self._get_param( + self.brightness, self.contrast, self.saturation, self.hue + ) return transform(img) @@ -1093,13 +1099,15 @@ class RandomCrop(BaseTransform): print(crop_img.shape) # [3, 224, 224] """ - def __init__(self, - size, - padding=None, - pad_if_needed=False, - fill=0, - padding_mode='constant', - keys=None): + def __init__( + self, + size, + padding=None, + pad_if_needed=False, + fill=0, + padding_mode='constant', + keys=None, + ): super(RandomCrop, self).__init__(keys) if isinstance(size, numbers.Number): self.size = (int(size), int(size)) @@ -1144,12 +1152,14 @@ class RandomCrop(BaseTransform): # pad the width if needed if self.pad_if_needed and w < self.size[1]: - img = F.pad(img, (self.size[1] - w, 0), self.fill, - self.padding_mode) + img = F.pad( + img, (self.size[1] - w, 0), self.fill, self.padding_mode + ) # pad the height if needed if self.pad_if_needed and h < self.size[0]: - img = F.pad(img, (0, self.size[0] - h), self.fill, - self.padding_mode) + img = F.pad( + img, (0, self.size[0] - h), self.fill, self.padding_mode + ) i, j, h, w = self._get_param(img, self.size) @@ -1214,8 +1224,9 @@ class Pad(BaseTransform): if isinstance(padding, Sequence) and len(padding) not in [2, 4]: raise ValueError( - "Padding must be an int or a 2, or 4 element tuple, not a " + - "{} element tuple".format(len(padding))) + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding)) + ) super(Pad, self).__init__(keys) self.padding = padding @@ -1234,19 +1245,23 @@ class Pad(BaseTransform): def _check_sequence_input(x, name, req_sizes): - msg = req_sizes[0] if len(req_sizes) < 2 else " or ".join( - [str(s) for s in req_sizes]) + msg = ( + req_sizes[0] + if len(req_sizes) < 2 + else " or ".join([str(s) for s in req_sizes]) + ) if not isinstance(x, Sequence): raise TypeError(f"{name} should be a sequence of length {msg}.") if len(x) not in req_sizes: raise ValueError(f"{name} should be sequence of length {msg}.") -def _setup_angle(x, name, req_sizes=(2, )): +def _setup_angle(x, name, req_sizes=(2,)): if isinstance(x, numbers.Number): if x < 0: raise ValueError( - f"If {name} is a single number, it must be positive.") + f"If {name} is a single number, it must be positive." + ) x = [-x, x] else: _check_sequence_input(x, name, req_sizes) @@ -1312,31 +1327,34 @@ class RandomAffine(BaseTransform): print(fake_img.shape) """ - def __init__(self, - degrees, - translate=None, - scale=None, - shear=None, - interpolation='nearest', - fill=0, - center=None, - keys=None): - self.degrees = _setup_angle(degrees, name="degrees", req_sizes=(2, )) + def __init__( + self, + degrees, + translate=None, + scale=None, + shear=None, + interpolation='nearest', + fill=0, + center=None, + keys=None, + ): + self.degrees = _setup_angle(degrees, name="degrees", req_sizes=(2,)) super(RandomAffine, self).__init__(keys) assert interpolation in ['nearest', 'bilinear', 'bicubic'] self.interpolation = interpolation if translate is not None: - _check_sequence_input(translate, "translate", req_sizes=(2, )) + _check_sequence_input(translate, "translate", req_sizes=(2,)) for t in translate: if not (0.0 <= t <= 1.0): raise ValueError( - "translation values should be between 0 and 1") + "translation values should be between 0 and 1" + ) self.translate = translate if scale is not None: - _check_sequence_input(scale, "scale", req_sizes=(2, )) + _check_sequence_input(scale, "scale", req_sizes=(2,)) for s in scale: if s <= 0: raise ValueError("scale values should be positive") @@ -1354,15 +1372,12 @@ class RandomAffine(BaseTransform): self.fill = fill if center is not None: - _check_sequence_input(center, "center", req_sizes=(2, )) + _check_sequence_input(center, "center", req_sizes=(2,)) self.center = center - def _get_param(self, - img_size, - degrees, - translate=None, - scale_ranges=None, - shears=None): + def _get_param( + self, img_size, degrees, translate=None, scale_ranges=None, shears=None + ): """Get parameters for affine transformation Returns: @@ -1405,14 +1420,17 @@ class RandomAffine(BaseTransform): w, h = _get_image_size(img) img_size = [w, h] - ret = self._get_param(img_size, self.degrees, self.translate, - self.scale, self.shear) + ret = self._get_param( + img_size, self.degrees, self.translate, self.scale, self.shear + ) - return F.affine(img, - *ret, - interpolation=self.interpolation, - fill=self.fill, - center=self.center) + return F.affine( + img, + *ret, + interpolation=self.interpolation, + fill=self.fill, + center=self.center, + ) class RandomRotation(BaseTransform): @@ -1464,22 +1482,26 @@ class RandomRotation(BaseTransform): print(fake_img.size) """ - def __init__(self, - degrees, - interpolation='nearest', - expand=False, - center=None, - fill=0, - keys=None): + def __init__( + self, + degrees, + interpolation='nearest', + expand=False, + center=None, + fill=0, + keys=None, + ): if isinstance(degrees, numbers.Number): if degrees < 0: raise ValueError( - "If degrees is a single number, it must be positive.") + "If degrees is a single number, it must be positive." + ) self.degrees = (-degrees, degrees) else: if len(degrees) != 2: raise ValueError( - "If degrees is a sequence, it must be of len 2.") + "If degrees is a sequence, it must be of len 2." + ) self.degrees = degrees super(RandomRotation, self).__init__(keys) @@ -1504,8 +1526,9 @@ class RandomRotation(BaseTransform): angle = self._get_param(self.degrees) - return F.rotate(img, angle, self.interpolation, self.expand, - self.center, self.fill) + return F.rotate( + img, angle, self.interpolation, self.expand, self.center, self.fill + ) class RandomPerspective(BaseTransform): @@ -1553,15 +1576,19 @@ class RandomPerspective(BaseTransform): print(fake_img.shape) """ - def __init__(self, - prob=0.5, - distortion_scale=0.5, - interpolation='nearest', - fill=0, - keys=None): + def __init__( + self, + prob=0.5, + distortion_scale=0.5, + interpolation='nearest', + fill=0, + keys=None, + ): super(RandomPerspective, self).__init__(keys) assert 0 <= prob <= 1, "probability must be between 0 and 1" - assert 0 <= distortion_scale <= 1, "distortion_scale must be between 0 and 1" + assert ( + 0 <= distortion_scale <= 1 + ), "distortion_scale must be between 0 and 1" assert interpolation in ['nearest', 'bilinear', 'bicubic'] assert isinstance(fill, (numbers.Number, str, list, tuple)) @@ -1579,35 +1606,43 @@ class RandomPerspective(BaseTransform): half_height = height // 2 half_width = width // 2 topleft = [ - int(random.uniform(0, - int(distortion_scale * half_width) + 1)), - int(random.uniform(0, - int(distortion_scale * half_height) + 1)), + int(random.uniform(0, int(distortion_scale * half_width) + 1)), + int(random.uniform(0, int(distortion_scale * half_height) + 1)), ] topright = [ int( - random.uniform(width - int(distortion_scale * half_width) - 1, - width)), - int(random.uniform(0, - int(distortion_scale * half_height) + 1)), + random.uniform( + width - int(distortion_scale * half_width) - 1, width + ) + ), + int(random.uniform(0, int(distortion_scale * half_height) + 1)), ] botright = [ int( - random.uniform(width - int(distortion_scale * half_width) - 1, - width)), + random.uniform( + width - int(distortion_scale * half_width) - 1, width + ) + ), int( - random.uniform(height - int(distortion_scale * half_height) - 1, - height)), + random.uniform( + height - int(distortion_scale * half_height) - 1, height + ) + ), ] botleft = [ - int(random.uniform(0, - int(distortion_scale * half_width) + 1)), + int(random.uniform(0, int(distortion_scale * half_width) + 1)), int( - random.uniform(height - int(distortion_scale * half_height) - 1, - height)), + random.uniform( + height - int(distortion_scale * half_height) - 1, height + ) + ), + ] + startpoints = [ + [0, 0], + [width - 1, 0], + [width - 1, height - 1], + [0, height - 1], ] - startpoints = [[0, 0], [width - 1, 0], [width - 1, height - 1], - [0, height - 1]] endpoints = [topleft, topright, botright, botleft] return startpoints, endpoints @@ -1624,10 +1659,12 @@ class RandomPerspective(BaseTransform): width, height = _get_image_size(img) if random.random() < self.prob: - startpoints, endpoints = self.get_params(width, height, - self.distortion_scale) - return F.perspective(img, startpoints, endpoints, - self.interpolation, self.fill) + startpoints, endpoints = self.get_params( + width, height, self.distortion_scale + ) + return F.perspective( + img, startpoints, endpoints, self.interpolation, self.fill + ) return img @@ -1715,27 +1752,34 @@ class RandomErasing(BaseTransform): print(result) """ - def __init__(self, - prob=0.5, - scale=(0.02, 0.33), - ratio=(0.3, 3.3), - value=0, - inplace=False, - keys=None): + def __init__( + self, + prob=0.5, + scale=(0.02, 0.33), + ratio=(0.3, 3.3), + value=0, + inplace=False, + keys=None, + ): super(RandomErasing, self).__init__(keys) - assert isinstance(scale, - (tuple, list)), "scale should be a tuple or list" - assert (scale[0] >= 0 and scale[1] <= 1 and scale[0] <= scale[1] - ), "scale should be of kind (min, max) and in range [0, 1]" - assert isinstance(ratio, - (tuple, list)), "ratio should be a tuple or list" - assert (ratio[0] >= 0 - and ratio[0] <= ratio[1]), "ratio should be of kind (min, max)" - assert (prob >= 0 - and prob <= 1), "The probability should be in range [0, 1]" assert isinstance( - value, (numbers.Number, str, tuple, - list)), "value should be a number, tuple, list or str" + scale, (tuple, list) + ), "scale should be a tuple or list" + assert ( + scale[0] >= 0 and scale[1] <= 1 and scale[0] <= scale[1] + ), "scale should be of kind (min, max) and in range [0, 1]" + assert isinstance( + ratio, (tuple, list) + ), "ratio should be a tuple or list" + assert ( + ratio[0] >= 0 and ratio[0] <= ratio[1] + ), "ratio should be of kind (min, max)" + assert ( + prob >= 0 and prob <= 1 + ), "The probability should be in range [0, 1]" + assert isinstance( + value, (numbers.Number, str, tuple, list) + ), "value should be a number, tuple, list or str" if isinstance(value, str) and value != "random": raise ValueError("value must be 'random' when type is str") @@ -1779,7 +1823,8 @@ class RandomErasing(BaseTransform): if F._is_tensor_image(img): if value is None: v = paddle.normal(shape=[c, erase_h, erase_w]).astype( - img.dtype) + img.dtype + ) else: v = paddle.to_tensor(value, dtype=img.dtype)[:, None, None] else: @@ -1815,6 +1860,7 @@ class RandomErasing(BaseTransform): "Value should be a single number or a sequence with length equals to image's channel." ) top, left, erase_h, erase_w, v = self._get_param( - img, self.scale, self.ratio, value) + img, self.scale, self.ratio, value + ) return F.erase(img, top, left, erase_h, erase_w, v, self.inplace) return img diff --git a/r/example/mobilenet.py b/r/example/mobilenet.py index 96deb0aaf5f..082a1dea7de 100755 --- a/r/example/mobilenet.py +++ b/r/example/mobilenet.py @@ -51,7 +51,7 @@ def set_config(): def parse_data(): - """ parse input and output data """ + """parse input and output data""" with open('data/data.txt', 'r') as fr: data = np.array([float(_) for _ in fr.read().split()]) diff --git a/tools/CrossStackProfiler/CspChromeTraceFormatter.py b/tools/CrossStackProfiler/CspChromeTraceFormatter.py index ec5dba54b00..1fa8efe9880 100755 --- a/tools/CrossStackProfiler/CspChromeTraceFormatter.py +++ b/tools/CrossStackProfiler/CspChromeTraceFormatter.py @@ -16,7 +16,6 @@ import json class ChromeTraceFormatter(object): - def __init__(self): self._events = [] self._metadata = [] diff --git a/tools/CrossStackProfiler/CspFileReader.py b/tools/CrossStackProfiler/CspFileReader.py index 0825842fa1a..11dd052283c 100755 --- a/tools/CrossStackProfiler/CspFileReader.py +++ b/tools/CrossStackProfiler/CspFileReader.py @@ -18,6 +18,7 @@ import json import glob import logging from multiprocessing import Lock + """ Some terms to clarify the code in most case, one or more paremeters may be set as input args for a class or a function in form of single variable or k-v dict @@ -42,10 +43,17 @@ dcgmMetricParameterMap = { "02_gpuUtility": [("GPUTL", "GPUTL"), ("GRACT", "GRACT")], "03_smUtility": [("SMACT", "SMACT"), ("SMOCC", "SMOCC")], "04_memUtility": [("FB_USED_RATIO", "FB_USED_RATIO"), ("DRAMA", "DRAMA")], - "05_txUtility": [("NVLTX", "NVLTX"), ("NVLRX", "NVLRX"), ("PCITX", "PCITX"), - ("PCIRX", "PCIRX")], - "06_calUtility": [("FP32A", "FP32A"), ("FP16A", "FP16A"), - ("TENSO", "TENSO")] + "05_txUtility": [ + ("NVLTX", "NVLTX"), + ("NVLRX", "NVLRX"), + ("PCITX", "PCITX"), + ("PCIRX", "PCIRX"), + ], + "06_calUtility": [ + ("FP32A", "FP32A"), + ("FP16A", "FP16A"), + ("TENSO", "TENSO"), + ], } DCGMINFO_TRACE_NUM = len(dcgmMetricParameterMap.keys()) NETINFO_TRACE_NUM = 2 @@ -59,13 +67,13 @@ FILEORGANIZEFORM_BYRANK = "byRank" FILEORGANIZEFORM_BYTRAINER = "byTrainer" FILEORGANIZEFORM_BYOTHER = "other" FILEORGANIZEFORM = [ - FILEORGANIZEFORM_BYRANK, FILEORGANIZEFORM_BYTRAINER, - FILEORGANIZEFORM_BYOTHER + FILEORGANIZEFORM_BYRANK, + FILEORGANIZEFORM_BYTRAINER, + FILEORGANIZEFORM_BYOTHER, ] class FileReader(object): - def __init__(self, logger, args): self._logger = logger self._args = args @@ -104,8 +112,9 @@ class FileReader(object): if not isinstance(self._args[key], type): raise TypeError( - "Invalid type of key [%s] in args dict, it should be a %s!" % - (key, type)) + "Invalid type of key [%s] in args dict, it should be a %s!" + % (key, type) + ) exec("self._%s = self._args[\"%s\"]" % (key, key)) @@ -117,18 +126,22 @@ class FileReader(object): raise TypeError("Invalid type of args, it should be a dict!") self._checkArgsKey("organizeForm", str) - if self._organizeForm not in FILEORGANIZEFORM or \ - self._organizeForm == FILEORGANIZEFORM_BYOTHER: + if ( + self._organizeForm not in FILEORGANIZEFORM + or self._organizeForm == FILEORGANIZEFORM_BYOTHER + ): raise NotImplementedError( - "we have not known how to process this form of file [%s]!" % - self._organizeForm) + "we have not known how to process this form of file [%s]!" + % self._organizeForm + ) self._checkArgsKey("gpuPerTrainer", int) self._checkArgsKey("dataPath", str) if not os.path.exists(self._dataPath): - raise IOError("input data path [%s] not existed!" % - (self._dataPath)) + raise IOError( + "input data path [%s] not existed!" % (self._dataPath) + ) self._checkArgsKey("groupSize", int) self._checkArgsKey("displaySize", int) @@ -155,7 +168,8 @@ class FileReader(object): def _cmp(self, x, y): return self._getId(x, self._organizeForm) - self._getId( - y, self._organizeForm) + y, self._organizeForm + ) def _getFileList(self): self._fileList = glob.glob(os.path.join(self._dataPath, "*.*")) @@ -170,12 +184,14 @@ class FileReader(object): newFileList.append(file) else: raise NotImplementedError( - "[%s] is repeated by id, we don not how to process it!" % - file) + "[%s] is repeated by id, we don not how to process it!" + % file + ) if not self._fileList: - if (self._getId(self._fileList[-1]) - - self._getId(self._fileList[0])) != len(self._fileList) - 1: + if ( + self._getId(self._fileList[-1]) - self._getId(self._fileList[0]) + ) != len(self._fileList) - 1: raise Exception("The file id should be countious!") # sort @@ -185,19 +201,23 @@ class FileReader(object): self._fileList.sort(key=_sortBySuffix) if not self._fileList: - self._logger.warning("we can not find any file in dir [%s]!" % - self._dataPath) + self._logger.warning( + "we can not find any file in dir [%s]!" % self._dataPath + ) else: - self._logger.info("file list in dir [%s] is : %s !" % - (self._dataPath, ', '.join(self._fileList))) + self._logger.info( + "file list in dir [%s] is : %s !" + % (self._dataPath, ', '.join(self._fileList)) + ) return self._fileList def _getId(self, fileName, organizeForm, sed="."): if self._organizeForm != organizeForm: raise TypeError( - "Can not get rank id when organizer form is not %s!" % - organizeForm) + "Can not get rank id when organizer form is not %s!" + % organizeForm + ) if not os.path.isfile(fileName): raise IOError("[%s] is not a valid file!" % (fileName)) @@ -213,8 +233,9 @@ class FileReader(object): except IndexError as e: print(e) raise TypeError( - "invalid fileName [%s], the prefix should be a number!" % - fileName) + "invalid fileName [%s], the prefix should be a number!" + % fileName + ) def getRankId(self, fileName, sed="."): return self._getId(fileName, FILEORGANIZEFORM_BYRANK, sed) @@ -251,7 +272,7 @@ class FileReader(object): end = 0 for i in range(0, (n) * j, j): if i < len(ls) and (i + j) < len(ls): - ls_return.append(ls[i:i + j]) + ls_return.append(ls[i : i + j]) end = i + j ls_return.append(ls[end:]) return ls_return @@ -284,40 +305,29 @@ class FileReader(object): try: data = json.load(rf) except Exception: - self._logger.error("read [%s] error. not a json file!" % - (fileName)) - raise TypeError("read [%s] error. not a json file!" % - (fileName)) + self._logger.error( + "read [%s] error. not a json file!" % (fileName) + ) + raise TypeError( + "read [%s] error. not a json file!" % (fileName) + ) return data - def dumpOpInfoDict(self, - data, - groupId, - gpuId, - pretty=False, - tmpPath="./tmp"): - return self.dumpDict(data, - "opinfo", - groupId, - gpuId, - pretty=False, - tmpPath="./tmp") + def dumpOpInfoDict( + self, data, groupId, gpuId, pretty=False, tmpPath="./tmp" + ): + return self.dumpDict( + data, "opinfo", groupId, gpuId, pretty=False, tmpPath="./tmp" + ) def dumpDCGMDict(self, data, groupId, gpuId, pretty=False, tmpPath="./tmp"): - return self.dumpDict(data, - "dcgm", - groupId, - gpuId, - pretty=False, - tmpPath="./tmp") - - def dumpDict(self, - data, - name, - groupId, - gpuId, - pretty=False, - tmpPath="./tmp"): + return self.dumpDict( + data, "dcgm", groupId, gpuId, pretty=False, tmpPath="./tmp" + ) + + def dumpDict( + self, data, name, groupId, gpuId, pretty=False, tmpPath="./tmp" + ): self._lock.acquire() if not os.path.exists(tmpPath): os.makedirs(tmpPath) diff --git a/tools/CrossStackProfiler/CspReporter.py b/tools/CrossStackProfiler/CspReporter.py index 2e1e1f2c638..052ffd6fca1 100755 --- a/tools/CrossStackProfiler/CspReporter.py +++ b/tools/CrossStackProfiler/CspReporter.py @@ -28,41 +28,43 @@ from CspFileReader import FILEORGANIZEFORM_BYRANK, FILEORGANIZEFORM_BYTRAINER def get_argparse(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--profile_path', - type=str, - default='.', - help='Working path that store the monitor data.') - - parser.add_argument('--timeline_path', - type=str, - default='.', - help='Output timeline file name.') - - parser.add_argument('--gpuPerTrainer', - type=int, - default=8, - help='Gpus per trainer.') - - parser.add_argument('--trainerNum', - type=int, - default=4, - help='Num of trainer.') - - parser.add_argument('--groupSize', - type=int, - default=8, - help='Num of trainer in a group.') - - parser.add_argument('--displaySize', - type=int, - default=2, - help='Num of line need to display in a group.') + parser.add_argument( + '--profile_path', + type=str, + default='.', + help='Working path that store the monitor data.', + ) + + parser.add_argument( + '--timeline_path', + type=str, + default='.', + help='Output timeline file name.', + ) + + parser.add_argument( + '--gpuPerTrainer', type=int, default=8, help='Gpus per trainer.' + ) + + parser.add_argument( + '--trainerNum', type=int, default=4, help='Num of trainer.' + ) + + parser.add_argument( + '--groupSize', type=int, default=8, help='Num of trainer in a group.' + ) + + parser.add_argument( + '--displaySize', + type=int, + default=2, + help='Num of line need to display in a group.', + ) return parser.parse_args() class CspReporter(object): - def __init__(self, args): self._args = args print(self._args) @@ -83,8 +85,9 @@ class CspReporter(object): def _checkArgs(self): if self._trainerNum % self._groupSize != 0: raise Exception( - "Input args error: trainerNum[%d] %% groupSize[%d] != 0" % - (self._trainerNum, self._groupSize)) + "Input args error: trainerNum[%d] %% groupSize[%d] != 0" + % (self._trainerNum, self._groupSize) + ) def _init_logger(self): self._logger = getLogger() @@ -121,10 +124,12 @@ class CspReporter(object): "organizeForm": FILEORGANIZEFORM_BYRANK, } - self._dcgmFileReader = dcgmFileReader(self._logger, - self._dcgmFileReaderArgs) - self._profileFileReader = profileFileReader(self._logger, - self._profileFileReaderArgs) + self._dcgmFileReader = dcgmFileReader( + self._logger, self._dcgmFileReaderArgs + ) + self._profileFileReader = profileFileReader( + self._logger, self._profileFileReaderArgs + ) def _init_timeInfo(self): self._timePath = os.path.join(self._workPath, TIME_PATH) @@ -134,7 +139,8 @@ class CspReporter(object): def _set_timeInfo(self, timeFileNamePrefix="time.txt", sed="."): timeFileNameList = glob.glob( - os.path.join(self._timePath, timeFileNamePrefix, sed, "*")) + os.path.join(self._timePath, timeFileNamePrefix, sed, "*") + ) for timeFileName in timeFileNameList: trainerId = int(timeFileName.split(sed)[-1]) gpuId = int(timeFileName.split(sed)[-2]) @@ -143,33 +149,41 @@ class CspReporter(object): for line in rf: if line.startswith("start time:"): info["start_time"] = int( - float(line.split(":")[-1]) * 1e9) + float(line.split(":")[-1]) * 1e9 + ) - self._minTimeStamp = min(self._minTimeStamp, - info["start_time"]) + self._minTimeStamp = min( + self._minTimeStamp, info["start_time"] + ) if line.startswith("end time:"): info["end_time"] = int(float(line.split(":")[-1]) * 1e9) if not info: self._timeInfo[gpuId * trainerId] = info - def _generateTraceFileByGroupAndGpuId(self, pipileInfo, netInfo, groupId, - gpuId): + def _generateTraceFileByGroupAndGpuId( + self, pipileInfo, netInfo, groupId, gpuId + ): dcgmInfoDict = self._dcgmFileReader.getDcgmInfoDict(groupId, gpuId) opInfoDict = self._profileFileReader.getOpInfoDict(groupId, gpuId) traceObj = {} - traceObj["traceEvents"] = pipileInfo[str( - gpuId)] + opInfoDict["traceEvents"] + dcgmInfoDict[ - "traceEvents"] + netInfo["traceEvents"] + traceObj["traceEvents"] = ( + pipileInfo[str(gpuId)] + + opInfoDict["traceEvents"] + + dcgmInfoDict["traceEvents"] + + netInfo["traceEvents"] + ) - self._profileFileReader.dumpDict(traceObj, "traceFile", groupId, gpuId, - False, self._saveFilePath) + self._profileFileReader.dumpDict( + traceObj, "traceFile", groupId, gpuId, False, self._saveFilePath + ) def _generateTraceFileByGroup(self, groupId, processNum): # first we need to generate pipeline info pipileInfo = self._profileFileReader.getPipeLineInfo( - groupId, processNum) + groupId, processNum + ) # second we need to generate dcgm info dcgmInfo = self._dcgmFileReader.getDCGMTraceInfo(groupId, processNum) @@ -186,48 +200,56 @@ class CspReporter(object): pidList = [] for gpuId in range(self._gpuPerTrainer): - subproc = Process(target=self._generateTraceFileByGroupAndGpuId, - args=( - pipileInfo, - netInfo, - groupId, - gpuId, - )) + subproc = Process( + target=self._generateTraceFileByGroupAndGpuId, + args=( + pipileInfo, + netInfo, + groupId, + gpuId, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[traceFile]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, 1)) + % (subproc.pid, 1) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( - "[traceFile]: process [%d] has exited! remained %d process!" % - (t.pid, len(pidList))) + "[traceFile]: process [%d] has exited! remained %d process!" + % (t.pid, len(pidList)) + ) def generateTraceFile(self, processNum=8): processPool = [] pidList = [] for groupId in range(self._trainerNum / self._groupSize): - subproc = Process(target=self._generateTraceFileByGroup, - args=( - groupId, - processNum, - )) + subproc = Process( + target=self._generateTraceFileByGroup, + args=( + groupId, + processNum, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[GroupTraceFile]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, 1)) + % (subproc.pid, 1) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( "[GroupTraceFile]: process [%d] has exited! remained %d process!" - % (t.pid, len(pidList))) + % (t.pid, len(pidList)) + ) if __name__ == '__main__': diff --git a/tools/CrossStackProfiler/DCGMFileReader.py b/tools/CrossStackProfiler/DCGMFileReader.py index 1285ce388c1..86868c8734e 100755 --- a/tools/CrossStackProfiler/DCGMFileReader.py +++ b/tools/CrossStackProfiler/DCGMFileReader.py @@ -28,7 +28,6 @@ from CspFileReader import FILEORGANIZEFORM_BYTRAINER class dcgmFileReader(FileReader): - def parseFileByGroup(self, groupId, processNum=8): fileFist = self.getFileListByGroup(groupId) displaySize = min(self._displaySize, len(fileFist)) @@ -38,8 +37,9 @@ class dcgmFileReader(FileReader): return self._parseTask(fileFist) else: - self._logger.info("using [%d] process to do this work!" % - processNum) + self._logger.info( + "using [%d] process to do this work!" % processNum + ) processPool = [] pidList = [] @@ -48,23 +48,28 @@ class dcgmFileReader(FileReader): taskList = self._splitTaskListForMultiProcess(fileFist, processNum) for task in taskList: - subproc = Process(target=self._parseTask, args=( - task, - q, - )) + subproc = Process( + target=self._parseTask, + args=( + task, + q, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[DCGM reader]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, len(processPool))) + % (subproc.pid, len(processPool)) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( "[DCGM reader]: process [%d] has exited! remained %d process!" - % (t.pid, len(pidList))) + % (t.pid, len(pidList)) + ) isFistProcess = True for t in processPool: @@ -72,9 +77,9 @@ class dcgmFileReader(FileReader): isFistProcess = False dcgm_data = q.get() else: - dcgm_data = pd.concat([dcgm_data, q.get()], - axis=0, - join='outer') + dcgm_data = pd.concat( + [dcgm_data, q.get()], axis=0, join='outer' + ) return dcgm_data @@ -90,9 +95,9 @@ class dcgmFileReader(FileReader): is_first = False dcgm_data = tmp_data else: - dcgm_data = pd.concat([dcgm_data, tmp_data], - axis=0, - join='outer') + dcgm_data = pd.concat( + [dcgm_data, tmp_data], axis=0, join='outer' + ) dcgm_data = dcgm_data.dropna() if q is not None: q.put(dcgm_data) @@ -117,18 +122,23 @@ class dcgmFileReader(FileReader): for line in fp: # skip `nvidia-dcgm-dmon.sh` init and fini info lines - if 'nv-hostengine' in line or 'dmon' in line or 'Host Engine Listener Started' in line: + if ( + 'nv-hostengine' in line + or 'dmon' in line + or 'Host Engine Listener Started' in line + ): continue if not line.strip().startswith( - "GPU") and not line.strip().startswith("# Entity"): + "GPU" + ) and not line.strip().startswith("# Entity"): continue # skip non-needed headers (only the header in 1th line was needed) if line.strip().startswith("# Entity"): line = line.strip()[2:] - if 'Entity' == line[0:len('Entity')]: + if 'Entity' == line[0 : len('Entity')]: if has_header: continue else: @@ -153,15 +163,13 @@ class dcgmFileReader(FileReader): return dcgm - def _getDCGMTraceInfoByGpuId(self, - groupId, - gpuId, - dcgm_data, - pid_map, - q=None): + def _getDCGMTraceInfoByGpuId( + self, groupId, gpuId, dcgm_data, pid_map, q=None + ): self._logger.info( - "Begin to generate dcgm info, groupId = %d, gpuID = %d ..." % - (groupId, gpuId)) + "Begin to generate dcgm info, groupId = %d, gpuID = %d ..." + % (groupId, gpuId) + ) gpuDcgmData = dcgm_data[dcgm_data['Entity'].isin([gpuId])] @@ -220,27 +228,31 @@ class dcgmFileReader(FileReader): pidList = [] for gpuId in range(self._gpuPerTrainer): - subproc = Process(target=self._getDCGMTraceInfoByGpuId, - args=( - groupId, - gpuId, - dcgm_data, - pid_map, - q, - )) + subproc = Process( + target=self._getDCGMTraceInfoByGpuId, + args=( + groupId, + gpuId, + dcgm_data, + pid_map, + q, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[DCGM info]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, 1)) + % (subproc.pid, 1) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( - "[DCGM info]: process [%d] has exited! remained %d process!" % - (t.pid, len(pidList))) + "[DCGM info]: process [%d] has exited! remained %d process!" + % (t.pid, len(pidList)) + ) dcgmInfo = {} diff --git a/tools/CrossStackProfiler/NetFileReader.py b/tools/CrossStackProfiler/NetFileReader.py index 199e3ba8d4d..ed9cfdb1fc8 100755 --- a/tools/CrossStackProfiler/NetFileReader.py +++ b/tools/CrossStackProfiler/NetFileReader.py @@ -24,7 +24,6 @@ from CspFileReader import FILEORGANIZEFORM_BYTRAINER class netFileReader(FileReader): - def _parseSingleFile(self, fileNameList, tx_pid, rx_pid, q=None): traceInfo = {} @@ -53,8 +52,9 @@ class netFileReader(FileReader): for line in rf: try: event_str = json.loads(line.strip()) - event_str["pid"] = tx_pid if event_str[ - "name"] == "tx" else rx_pid + event_str["pid"] = ( + tx_pid if event_str["name"] == "tx" else rx_pid + ) # the unit of net is ms, we need ns event_str["ts"] = self._align_ts(event_str["ts"] * 1e6) event_str["id"] = trainerId @@ -62,8 +62,9 @@ class netFileReader(FileReader): except Exception: self._logger.warning( - "invalid record [%s] in [%s]. skip it!" % - (line[:-1], fileName)) + "invalid record [%s] in [%s]. skip it!" + % (line[:-1], fileName) + ) traceInfo["traceEvents"] = traceEventList if q is not None: @@ -73,7 +74,7 @@ class netFileReader(FileReader): def parseFileByGroup(self, groupId, processNum=8): fileFist = self.getFileListByGroup(groupId) - fileFist = fileFist[:min(self._displaySize, len(fileFist))] + fileFist = fileFist[: min(self._displaySize, len(fileFist))] manager = multiprocessing.Manager() q = manager.Queue() @@ -85,26 +86,30 @@ class netFileReader(FileReader): taskList = self._splitTaskListForMultiProcess(fileFist, processNum) for task in taskList: - subproc = Process(target=self._parseSingleFile, - args=( - task, - tx_pid, - rx_pid, - q, - )) + subproc = Process( + target=self._parseSingleFile, + args=( + task, + tx_pid, + rx_pid, + q, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[Net info]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, len(processPool))) + % (subproc.pid, len(processPool)) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( - "[Net info]: process [%d] has exited! remained %d process!" % - (t.pid, len(pidList))) + "[Net info]: process [%d] has exited! remained %d process!" + % (t.pid, len(pidList)) + ) traceInfo = {} isFistProcess = True diff --git a/tools/CrossStackProfiler/ProfileFileReader.py b/tools/CrossStackProfiler/ProfileFileReader.py index 282c2f1f254..e7a07c40820 100755 --- a/tools/CrossStackProfiler/ProfileFileReader.py +++ b/tools/CrossStackProfiler/ProfileFileReader.py @@ -22,12 +22,15 @@ from CspChromeTraceFormatter import ChromeTraceFormatter from CspFileReader import FileReader from CspFileReader import getLogger -from CspFileReader import NETINFO_TRACE_NUM, DCGMINFO_TRACE_NUM, PIPELINEINFO_TRACE_NUM +from CspFileReader import ( + NETINFO_TRACE_NUM, + DCGMINFO_TRACE_NUM, + PIPELINEINFO_TRACE_NUM, +) from CspFileReader import FILEORGANIZEFORM_BYRANK class profileFileReader(FileReader): - def _parseSingleFile(self, profile): with open(profile, 'rb') as f: profile_s = f.read() @@ -41,8 +44,9 @@ class profileFileReader(FileReader): for fileName in taskList: rankId = self.getRankId(fileName) - profile_dict["trainerRank.%03d" % - (rankId)] = self._parseSingleFile(fileName) + profile_dict["trainerRank.%03d" % (rankId)] = self._parseSingleFile( + fileName + ) self._logger.info("I finish processing %s!" % fileName) if q is not None: @@ -56,15 +60,16 @@ class profileFileReader(FileReader): if isinstance(items["args"], dict): args = items["args"] if "detail_info" in args: - if args["detail_info"] == "marker_forward_B" or \ - args["detail_info"] == "marker_forward_E" or \ - args["detail_info"] == "marker_backward_B" or \ - args["detail_info"] == "marker_backward_E": + if ( + args["detail_info"] == "marker_forward_B" + or args["detail_info"] == "marker_forward_E" + or args["detail_info"] == "marker_backward_B" + or args["detail_info"] == "marker_backward_E" + ): return True return False def _allocate_forwardBackwardInfo(self, restList, pid, tid): - def _cmp_ele(items): return items["ts"] @@ -79,7 +84,7 @@ class profileFileReader(FileReader): else: lastEle["dur"] = items["ts"] - lastEle["ts"] name = lastEle["args"]["detail_info"] - name = name[:name.rfind('_')] + name = name[: name.rfind('_')] name = name.split('_')[1] lastEle["name"] = name lastEle["args"]["detail_info"] = name @@ -130,7 +135,8 @@ class profileFileReader(FileReader): traceEventList.append(traceEvent) pipeLineList = self._allocate_forwardBackwardInfo( - traceEventList, pid, tid) + traceEventList, pid, tid + ) res[str(rankId)] = pipeLineList @@ -143,8 +149,9 @@ class profileFileReader(FileReader): fileFist = self.getFileListByGroup(groupId) self._logger.info( - "using [%d] process to do this work, total task num is %d!" % - (processNum, len(fileFist))) + "using [%d] process to do this work, total task num is %d!" + % (processNum, len(fileFist)) + ) processPool = [] pidList = [] @@ -153,23 +160,28 @@ class profileFileReader(FileReader): taskList = self._splitTaskListForMultiProcess(fileFist, processNum) for task in taskList: - subproc = Process(target=self._getPipeLineInfo, args=( - task, - q, - )) + subproc = Process( + target=self._getPipeLineInfo, + args=( + task, + q, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[pipeline info]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, len(task))) + % (subproc.pid, len(task)) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( "[pipeline info]: process [%d] has exited! remained %d process!" - % (t.pid, len(pidList))) + % (t.pid, len(pidList)) + ) pipeLineInfo = {} @@ -210,12 +222,15 @@ class profileFileReader(FileReader): # -1 device id represents CUDA API(RunTime) call.(e.g. cudaLaunch, cudaMemcpy) if event.device_id == -1: chrome_trace.emit_pid( - "%02d_%s:cuda_api" % (lineNum, k), pid) + "%02d_%s:cuda_api" % (lineNum, k), pid + ) lineNum = lineNum + 1 else: chrome_trace.emit_pid( - "%02d_%s:cpu:block:%d" % - (lineNum, k, event.device_id), pid) + "%02d_%s:cpu:block:%d" + % (lineNum, k, event.device_id), + pid, + ) lineNum = lineNum + 1 elif event.type == profiler_pb2.Event.GPUKernel: if (k, event.device_id, "GPUKernel") not in devices: @@ -225,8 +240,10 @@ class profileFileReader(FileReader): devices[(k, event.device_id, "GPUKernel")] = pid chrome_trace.emit_pid( - "%02d_%s:gpu:%d" % - (lineNum, k, event.device_id), pid) + "%02d_%s:gpu:%d" + % (lineNum, k, event.device_id), + pid, + ) lineNum = lineNum + 1 if not hasattr(profile_pb, "mem_events"): @@ -240,8 +257,10 @@ class profileFileReader(FileReader): mem_devices[(k, mevent.device_id, "GPU")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:gpu:%d" % - (lineNum, k, mevent.device_id), pid) + "%02d_memory usage on %s:gpu:%d" + % (lineNum, k, mevent.device_id), + pid, + ) lineNum = lineNum + 1 elif mevent.place == profiler_pb2.MemEvent.CPUPlace: if (k, mevent.device_id, "CPU") not in mem_devices: @@ -250,21 +269,29 @@ class profileFileReader(FileReader): mem_devices[(k, mevent.device_id, "CPU")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:cpu:%d" % - (lineNum, k, mevent.device_id), pid) + "%02d_memory usage on %s:cpu:%d" + % (lineNum, k, mevent.device_id), + pid, + ) lineNum = lineNum + 1 elif mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace: - if (k, mevent.device_id, - "CUDAPinnedPlace") not in mem_devices: + if ( + k, + mevent.device_id, + "CUDAPinnedPlace", + ) not in mem_devices: if gpuId == mevent.device_id: pid = initPid initPid = initPid + 1 - mem_devices[(k, mevent.device_id, - "CUDAPinnedPlace")] = pid + mem_devices[ + (k, mevent.device_id, "CUDAPinnedPlace") + ] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:cudapinnedplace:%d" % - (lineNum, k, mevent.device_id), pid) + "%02d_memory usage on %s:cudapinnedplace:%d" + % (lineNum, k, mevent.device_id), + pid, + ) lineNum = lineNum + 1 if (k, 0, "CPU") not in mem_devices: pid = initPid @@ -272,7 +299,8 @@ class profileFileReader(FileReader): mem_devices[(k, 0, "CPU")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:cpu:%d" % (lineNum, k, 0), pid) + "%02d_memory usage on %s:cpu:%d" % (lineNum, k, 0), pid + ) lineNum = lineNum + 1 if (k, 0, "GPU") not in mem_devices: # if gpuId == mevent.device_id: @@ -281,7 +309,8 @@ class profileFileReader(FileReader): mem_devices[(k, 0, "GPU")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:gpu:%d" % (lineNum, k, 0), pid) + "%02d_memory usage on %s:gpu:%d" % (lineNum, k, 0), pid + ) lineNum = lineNum + 1 if (k, 0, "CUDAPinnedPlace") not in mem_devices: pid = initPid @@ -289,8 +318,10 @@ class profileFileReader(FileReader): mem_devices[(k, 0, "CUDAPinnedPlace")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:cudapinnedplace:%d" % - (lineNum, k, 0), pid) + "%02d_memory usage on %s:cudapinnedplace:%d" + % (lineNum, k, 0), + pid, + ) lineNum = lineNum + 1 i = i + 1 return chrome_trace, devices, mem_devices @@ -307,7 +338,11 @@ class profileFileReader(FileReader): elif event.type == profiler_pb2.Event.GPUKernel: type = "GPUKernel" - if event.type == profiler_pb2.Event.GPUKernel and event.device_id != gpuId and rankId % self._gpuPerTrainer != gpuId: + if ( + event.type == profiler_pb2.Event.GPUKernel + and event.device_id != gpuId + and rankId % self._gpuPerTrainer != gpuId + ): continue pid = devices[(k, event.device_id, type)] @@ -318,10 +353,15 @@ class profileFileReader(FileReader): args['detail_info'] = event.detail_info # TODO(panyx0718): Chrome tracing only handles ms. However, some # ops takes micro-seconds. Hence, we keep the ns here. - chrome_trace.emit_region(self._align_ts(event.start_ns), - (event.end_ns - event.start_ns) / 1.0, - pid, event.sub_device_id, 'Op', - event.name, args) + chrome_trace.emit_region( + self._align_ts(event.start_ns), + (event.end_ns - event.start_ns) / 1.0, + pid, + event.sub_device_id, + 'Op', + event.name, + args, + ) return chrome_trace def _allocate_memory_event(self, profile_dict, mem_devices, gpuId): @@ -331,7 +371,7 @@ class profileFileReader(FileReader): place_to_str = { profiler_pb2.MemEvent.CPUPlace: "CPU", profiler_pb2.MemEvent.CUDAPlace: "GPU", - profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace" + profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace", } for k, profile_pb in profile_dict.items(): rankId = int(k.split(".")[-1]) @@ -352,9 +392,10 @@ class profileFileReader(FileReader): else: place = "UnDefine" - if (mevent.place == profiler_pb2.MemEvent.CUDAPlace - or mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace - ) and mevent.device_id != gpuId: + if ( + mevent.place == profiler_pb2.MemEvent.CUDAPlace + or mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace + ) and mevent.device_id != gpuId: continue crt_info['place'] = place @@ -377,15 +418,21 @@ class profileFileReader(FileReader): total_size = 0 while i < len(mem_list): total_size += mem_list[i]['size'] - while i < len(mem_list) - 1 and mem_list[i]['time'] == mem_list[ - i + 1]['time']: + while ( + i < len(mem_list) - 1 + and mem_list[i]['time'] == mem_list[i + 1]['time'] + ): total_size += mem_list[i + 1]['size'] i += 1 - chrome_trace.emit_counter("Memory", "Memory", - mem_list[i]['pid'], - self._align_ts(mem_list[i]['time']), - 0, total_size) + chrome_trace.emit_counter( + "Memory", + "Memory", + mem_list[i]['pid'], + self._align_ts(mem_list[i]['time']), + 0, + total_size, + ) i += 1 return chrome_trace @@ -395,21 +442,29 @@ class profileFileReader(FileReader): for file in fileFist: rankId = self.getRankId(file) localRank = rankId % self._gpuPerTrainer - if localRank == gpuId and (rankId / self._gpuPerTrainer - ) % self._groupSize < self._displaySize: + if ( + localRank == gpuId + and (rankId / self._gpuPerTrainer) % self._groupSize + < self._displaySize + ): newFileList.append(file) profile_dict = self._parseTask(newFileList) - initPid = PIPELINEINFO_TRACE_NUM + DCGMINFO_TRACE_NUM + NETINFO_TRACE_NUM + initPid = ( + PIPELINEINFO_TRACE_NUM + DCGMINFO_TRACE_NUM + NETINFO_TRACE_NUM + ) metaTrace, devicesPid, mem_devicesPid = self._allocate_pids( - profile_dict, gpuId, initPid) + profile_dict, gpuId, initPid + ) eventsTrace = self._allocate_events(profile_dict, devicesPid, gpuId) - memEventsTrace = self._allocate_memory_event(profile_dict, - mem_devicesPid, gpuId) + memEventsTrace = self._allocate_memory_event( + profile_dict, mem_devicesPid, gpuId + ) trace = {} - trace[ - 'traceEvents'] = metaTrace._metadata + eventsTrace._events + memEventsTrace._events + trace['traceEvents'] = ( + metaTrace._metadata + eventsTrace._events + memEventsTrace._events + ) self.dumpOpInfoDict(trace, groupId, gpuId, True) return trace @@ -421,24 +476,28 @@ class profileFileReader(FileReader): pidList = [] for gpuId in range(self._gpuPerTrainer): - subproc = Process(target=self._getOPTraceInfoByGpuId, - args=( - groupId, - gpuId, - )) + subproc = Process( + target=self._getOPTraceInfoByGpuId, + args=( + groupId, + gpuId, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[op info]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, 1)) + % (subproc.pid, 1) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( - "[op info]: process [%d] has exited! remained %d process!" % - (t.pid, len(pidList))) + "[op info]: process [%d] has exited! remained %d process!" + % (t.pid, len(pidList)) + ) opInfo = {} diff --git a/tools/analysisPyXml.py b/tools/analysisPyXml.py index b9280124aed..ee110d6ce7f 100644 --- a/tools/analysisPyXml.py +++ b/tools/analysisPyXml.py @@ -21,10 +21,16 @@ import sys def analysisPyXml(rootPath, ut): xml_path = '%s/build/pytest/%s/python-coverage.xml' % (rootPath, ut) - related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % (rootPath, ut, - ut) - notrelated_ut_map_file = '%s/build/ut_map/%s/notrelated_%s.txt' % (rootPath, - ut, ut) + related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % ( + rootPath, + ut, + ut, + ) + notrelated_ut_map_file = '%s/build/ut_map/%s/notrelated_%s.txt' % ( + rootPath, + ut, + ut, + ) tree = ElementTree.parse(xml_path) root = tree.getroot() error_files = [] @@ -40,18 +46,36 @@ def analysisPyXml(rootPath, ut): command = 'sed -n %sp %s' % (line_number, clazz_filename) _code, output = commands.getstatusoutput(command) if _code == 0: - if output.strip().startswith( - ('from', 'import', '__all__', 'def', 'class', '"""', - '@', '\'\'\'', 'logger', '_logger', 'logging', 'r"""', - 'pass', 'try', 'except', - 'if __name__ == "__main__"')) == False: - pattern = r"""(.*) = ('*')|(.*) = ("*")|(.*) = (\d)|(.*) = (-\d)|(.*) = (None)|(.*) = (True)|(.*) = (False)|(.*) = (URL_PREFIX*)|(.*) = (\[)|(.*) = (\{)|(.*) = (\()""" #a='b'/a="b"/a=0 + if ( + output.strip().startswith( + ( + 'from', + 'import', + '__all__', + 'def', + 'class', + '"""', + '@', + '\'\'\'', + 'logger', + '_logger', + 'logging', + 'r"""', + 'pass', + 'try', + 'except', + 'if __name__ == "__main__"', + ) + ) + == False + ): + pattern = r"""(.*) = ('*')|(.*) = ("*")|(.*) = (\d)|(.*) = (-\d)|(.*) = (None)|(.*) = (True)|(.*) = (False)|(.*) = (URL_PREFIX*)|(.*) = (\[)|(.*) = (\{)|(.*) = (\()""" # a='b'/a="b"/a=0 if re.match(pattern, output.strip()) == None: pyCov_file.append(clazz_filename) coverageMessage = 'RELATED' break else: - coverageMessage = 'FILTER' #hit filter logic + coverageMessage = 'FILTER' # hit filter logic else: coverageMessage = 'FILTER' else: @@ -61,8 +85,9 @@ def analysisPyXml(rootPath, ut): else: coverageMessage = 'NOT_RELATED' if coverageMessage in ['NOT_RELATED', 'ERROR', 'FILTER']: - os.system('echo %s >> %s' % - (clazz_filename, notrelated_ut_map_file)) + os.system( + 'echo %s >> %s' % (clazz_filename, notrelated_ut_map_file) + ) elif coverageMessage == 'RELATED': os.system('echo %s >> %s' % (clazz_filename, related_ut_map_file)) diff --git a/tools/check_api_compatible.py b/tools/check_api_compatible.py index 18fb4d7ecdf..ea08a4494bb 100644 --- a/tools/check_api_compatible.py +++ b/tools/check_api_compatible.py @@ -27,7 +27,9 @@ else: logger.addHandler(console) console.setFormatter( logging.Formatter( - "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s")) + "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s" + ) +) def _check_compatible(args_o, args_n, defaults_o, defaults_n): @@ -38,21 +40,30 @@ def _check_compatible(args_o, args_n, defaults_o, defaults_n): # 参数改名了，也要提醒关注 for idx in range(min(len(args_o), len(args_n))): if args_o[idx] != args_n[idx]: - logger.debug("args's %d parameter diff with previous: %s vs %s", - idx, args_o, args_n) + logger.debug( + "args's %d parameter diff with previous: %s vs %s", + idx, + args_o, + args_n, + ) return False # 新增加了参数，必须提供默认值。以及不能减少默认值数量 if (len(args_n) - len(defaults_n)) > (len(args_o) - len(defaults_o)): - logger.debug("defaults num less then previous: %s vs %s", defaults_o, - defaults_n) + logger.debug( + "defaults num less then previous: %s vs %s", defaults_o, defaults_n + ) return False # 默认值必须相等 for idx in range(min(len(defaults_o), len(defaults_n))): nidx_o = -1 - idx nidx_n = -1 - idx - (len(args_n) - len(args_o)) - if (defaults_o[nidx_o] != defaults_n[nidx_n]): - logger.debug("defaults's %d value diff with previous: %s vs %s", - nidx_n, defaults_o, defaults_n) + if defaults_o[nidx_o] != defaults_n[nidx_n]: + logger.debug( + "defaults's %d value diff with previous: %s vs %s", + nidx_n, + defaults_o, + defaults_n, + ) return False return True @@ -61,21 +72,26 @@ def check_compatible(old_api_spec, new_api_spec): """ check compatible, FullArgSpec """ - if not (isinstance(old_api_spec, inspect.FullArgSpec) - and isinstance(new_api_spec, inspect.FullArgSpec)): + if not ( + isinstance(old_api_spec, inspect.FullArgSpec) + and isinstance(new_api_spec, inspect.FullArgSpec) + ): logger.warning( "new_api_spec or old_api_spec is not instance of inspect.FullArgSpec" ) return False return _check_compatible( - old_api_spec.args, new_api_spec.args, + old_api_spec.args, + new_api_spec.args, [] if old_api_spec.defaults is None else old_api_spec.defaults, - [] if new_api_spec.defaults is None else new_api_spec.defaults) + [] if new_api_spec.defaults is None else new_api_spec.defaults, + ) def check_compatible_str(old_api_spec_str, new_api_spec_str): patArgSpec = re.compile( - r'args=(.*), varargs=.*defaults=(None|\((.*)\)), kwonlyargs=.*') + r'args=(.*), varargs=.*defaults=(None|\((.*)\)), kwonlyargs=.*' + ) mo_o = patArgSpec.search(old_api_spec_str) mo_n = patArgSpec.search(new_api_spec_str) if not (mo_o and mo_n): @@ -99,7 +115,8 @@ def read_argspec_from_file(specfile): """ res_dict = {} patArgSpec = re.compile( - r'^(paddle[^,]+)\s+\((ArgSpec.*),\s\(\'document\W*([0-9a-z]{32})') + r'^(paddle[^,]+)\s+\((ArgSpec.*),\s\(\'document\W*([0-9a-z]{32})' + ) fullargspec_prefix = 'inspect.Full' for line in specfile.readlines(): mo = patArgSpec.search(line) @@ -123,21 +140,23 @@ def parse_args(): """ global arguments parser = argparse.ArgumentParser( - description='check api compatible across versions') + description='check api compatible across versions' + ) parser.add_argument('--debug', dest='debug', action="store_true") parser.add_argument( 'prev', type=argparse.FileType('r'), - help='the previous version (the version from develop branch)') - parser.add_argument('post', - type=argparse.FileType('r'), - help='the post version (the version from PullRequest)') + help='the previous version (the version from develop branch)', + ) + parser.add_argument( + 'post', + type=argparse.FileType('r'), + help='the post version (the version from PullRequest)', + ) for item in arguments: - parser.add_argument(item[0], - dest=item[1], - help=item[4], - type=item[2], - default=item[3]) + parser.add_argument( + item[0], dest=item[1], help=item[4], type=item[2], default=item[3] + ) if len(sys.argv) < 2: parser.print_help() @@ -162,10 +181,12 @@ if __name__ == '__main__': if as_prev is None: # the api is deleted continue if isinstance(as_prev, str) or isinstance(as_post, str): - as_prev_str = as_prev if isinstance(as_prev, - str) else repr(as_prev) - as_post_str = as_post if isinstance(as_post, - str) else repr(as_post) + as_prev_str = ( + as_prev if isinstance(as_prev, str) else repr(as_prev) + ) + as_post_str = ( + as_post if isinstance(as_post, str) else repr(as_post) + ) if not check_compatible_str(as_prev_str, as_post_str): diff_api_names.append(as_post_name) else: diff --git a/tools/check_api_source_without_core_ops.py b/tools/check_api_source_without_core_ops.py index 797934a5a35..bf7027be5cb 100644 --- a/tools/check_api_source_without_core_ops.py +++ b/tools/check_api_source_without_core_ops.py @@ -27,8 +27,10 @@ with open(sys.argv[2], 'r') as f: differ = difflib.Differ() result = differ.compare(origin, new) -api_with_ops, api_without_ops = count_api_without_core_ops.get_apis_with_and_without_core_ops( - ['paddle']) +( + api_with_ops, + api_without_ops, +) = count_api_without_core_ops.get_apis_with_and_without_core_ops(['paddle']) error = False # get all diff apis diff --git a/tools/check_ctest_hung.py b/tools/check_ctest_hung.py index 094d051b03e..11e927eb681 100644 --- a/tools/check_ctest_hung.py +++ b/tools/check_ctest_hung.py @@ -38,8 +38,7 @@ Diff: set(['test_parallel_executor_crf']) passed = set() with open(logfile, "r") as fn: for l in fn.readlines(): - if l.find("Test ") != -1 and \ - l.find("Passed") != -1: + if l.find("Test ") != -1 and l.find("Passed") != -1: m = re.search(r"Test\s+#[0-9]*\:\s([a-z0-9_]+)", escape(l)) passed.add(m.group(1)) if l.find("Start ") != -1: diff --git a/tools/check_op_benchmark_result.py b/tools/check_op_benchmark_result.py index aaf194ff95e..8fce5081022 100644 --- a/tools/check_op_benchmark_result.py +++ b/tools/check_op_benchmark_result.py @@ -19,14 +19,12 @@ import argparse def check_path_exists(path): - """Assert whether file/directory exists. - """ + """Assert whether file/directory exists.""" assert os.path.exists(path), "%s does not exist." % path def parse_case_name(log_file_name): - """Parse case name. - """ + """Parse case name.""" case_id, case_info = log_file_name.split("-") direction = case_info.split(".")[0].split("_")[-1] @@ -34,8 +32,7 @@ def parse_case_name(log_file_name): def parse_log_file(log_file): - """Load one case result from log file. - """ + """Load one case result from log file.""" check_path_exists(log_file) result = None @@ -56,20 +53,20 @@ def parse_log_file(log_file): def load_benchmark_result_from_logs_dir(logs_dir): - """Load benchmark result from logs directory. - """ + """Load benchmark result from logs directory.""" check_path_exists(logs_dir) log_file_path = lambda log_file: os.path.join(logs_dir, log_file) - result_lambda = lambda log_file: (log_file, - parse_log_file(log_file_path(log_file))) + result_lambda = lambda log_file: ( + log_file, + parse_log_file(log_file_path(log_file)), + ) return dict(map(result_lambda, os.listdir(logs_dir))) def check_speed_result(case_name, develop_data, pr_data, pr_result): - """Check speed differences between develop and pr. - """ + """Check speed differences between develop and pr.""" pr_gpu_time = pr_data.get("gpu_time") develop_gpu_time = develop_data.get("gpu_time") if develop_gpu_time != 0.0: @@ -84,10 +81,14 @@ def check_speed_result(case_name, develop_data, pr_data, pr_result): total_time_diff = (pr_total_time - develop_total_time) / develop_total_time logging.info("------ OP: %s ------" % case_name) - logging.info("GPU time change: %s (develop: %.7f -> PR: %.7f)" % - (gpu_time_diff_str, develop_gpu_time, pr_gpu_time)) - logging.info("Total time change: %.5f%% (develop: %.7f -> PR: %.7f)" % - (total_time_diff * 100, develop_total_time, pr_total_time)) + logging.info( + "GPU time change: %s (develop: %.7f -> PR: %.7f)" + % (gpu_time_diff_str, develop_gpu_time, pr_gpu_time) + ) + logging.info( + "Total time change: %.5f%% (develop: %.7f -> PR: %.7f)" + % (total_time_diff * 100, develop_total_time, pr_total_time) + ) logging.info("backward: %s" % pr_result.get("backward")) logging.info("parameters:") for line in pr_result.get("parameters").strip().split("\n"): @@ -97,8 +98,7 @@ def check_speed_result(case_name, develop_data, pr_data, pr_result): def check_accuracy_result(case_name, pr_result): - """Check accuracy result. - """ + """Check accuracy result.""" logging.info("------ OP: %s ------" % case_name) logging.info("Accuracy diff: %s" % pr_result.get("diff")) logging.info("backward: %s" % pr_result.get("backward")) @@ -109,15 +109,16 @@ def check_accuracy_result(case_name, pr_result): return not pr_result.get("consistent") -def compare_benchmark_result(case_name, develop_result, pr_result, - check_results): - """Compare the differences between develop and pr. - """ +def compare_benchmark_result( + case_name, develop_result, pr_result, check_results +): + """Compare the differences between develop and pr.""" develop_speed = develop_result.get("speed") pr_speed = pr_result.get("speed") assert type(develop_speed) == type( - pr_speed), "The types of comparison results need to be consistent." + pr_speed + ), "The types of comparison results need to be consistent." if isinstance(develop_speed, dict) and isinstance(pr_speed, dict): if check_speed_result(case_name, develop_speed, pr_speed, pr_result): @@ -128,8 +129,7 @@ def compare_benchmark_result(case_name, develop_result, pr_result, def update_api_info_file(fail_case_list, api_info_file): - """Update api info file to auto retry benchmark test. - """ + """Update api info file to auto retry benchmark test.""" check_path_exists(api_info_file) # set of case names for performance check failures @@ -153,14 +153,14 @@ def update_api_info_file(fail_case_list, api_info_file): def summary_results(check_results, api_info_file): - """Summary results and return exit code. - """ + """Summary results and return exit code.""" for case_name in check_results["speed"]: logging.error("Check speed result with case \"%s\" failed." % case_name) for case_name in check_results["accuracy"]: - logging.error("Check accuracy result with case \"%s\" failed." % - case_name) + logging.error( + "Check accuracy result with case \"%s\" failed." % case_name + ) if len(check_results["speed"]) and api_info_file: update_api_info_file(check_results["speed"], api_info_file) @@ -172,33 +172,38 @@ def summary_results(check_results, api_info_file): if __name__ == "__main__": - """Load result from log directories and compare the differences. - """ + """Load result from log directories and compare the differences.""" logging.basicConfig( level=logging.INFO, - format="[%(filename)s:%(lineno)d] [%(levelname)s] %(message)s") + format="[%(filename)s:%(lineno)d] [%(levelname)s] %(message)s", + ) parser = argparse.ArgumentParser() parser.add_argument( "--develop_logs_dir", type=str, required=True, - help="Specify the benchmark result directory of develop branch.") + help="Specify the benchmark result directory of develop branch.", + ) parser.add_argument( "--pr_logs_dir", type=str, required=True, - help="Specify the benchmark result directory of PR branch.") - parser.add_argument("--api_info_file", - type=str, - required=False, - help="Specify the api info to run benchmark test.") + help="Specify the benchmark result directory of PR branch.", + ) + parser.add_argument( + "--api_info_file", + type=str, + required=False, + help="Specify the api info to run benchmark test.", + ) args = parser.parse_args() check_results = dict(accuracy=list(), speed=list()) develop_result_dict = load_benchmark_result_from_logs_dir( - args.develop_logs_dir) + args.develop_logs_dir + ) check_path_exists(args.pr_logs_dir) pr_log_files = os.listdir(args.pr_logs_dir) @@ -208,7 +213,8 @@ if __name__ == "__main__": if develop_result is None or pr_result is None: continue case_name = parse_case_name(log_file) - compare_benchmark_result(case_name, develop_result, pr_result, - check_results) + compare_benchmark_result( + case_name, develop_result, pr_result, check_results + ) exit(summary_results(check_results, args.api_info_file)) diff --git a/tools/check_op_desc.py b/tools/check_op_desc.py index 7367b88d5b8..1c1ff14f7b6 100644 --- a/tools/check_op_desc.py +++ b/tools/check_op_desc.py @@ -88,7 +88,9 @@ def diff_vars(origin_vars, new_vars): if var_name not in var_changed_error_massage.keys(): var_changed_error_massage[var_name] = {} var_changed_error_massage[var_name][arg_name] = ( - origin_arg_value, new_arg_value) + origin_arg_value, + new_arg_value, + ) for var_name in vars_name_only_in_origin: error, var_error = True, True @@ -111,7 +113,8 @@ def diff_vars(origin_vars, new_vars): # if added var is def, inference needs to review, needs to register. if not new_vars.get(var_name).get(EXTRA) and not new_vars.get( - var_name).get(QUANT): + var_name + ).get(QUANT): error, var_error = True, True var_add_def_message.append(var_name) @@ -160,7 +163,9 @@ def diff_attr(ori_attrs, new_attrs): if attr_name not in attr_changed_error_massage.keys(): attr_changed_error_massage[attr_name] = {} attr_changed_error_massage[attr_name][arg_name] = ( - origin_arg_value, new_arg_value) + origin_arg_value, + new_arg_value, + ) for attr_name in attrs_only_in_origin: error, attr_error = True, True @@ -179,7 +184,8 @@ def diff_attr(ori_attrs, new_attrs): # if added attr is def, inference needs to review, needs to register if not new_attrs.get(attr_name).get(EXTRA) and not new_attrs.get( - attr_name).get(QUANT): + attr_name + ).get(QUANT): error, var_error = True, True attr_added_define_message.append(attr_name) @@ -206,7 +212,8 @@ def check_io_registry(io_type, op, diff): for update_type in [ADD]: for item in diff.get(update_type, []): infos = checker.filter_updates( - op, version_update_map[io_type][update_type], item) + op, version_update_map[io_type][update_type], item + ) if not infos: if update_type not in results.keys(): results[update_type] = [] @@ -228,7 +235,8 @@ def check_attr_registry(op, diff, origin_attrs): for update_type in [ADD, CHANGE]: for item in diff.get(update_type, {}): infos = checker.filter_updates( - op, version_update_map[ATTRS][update_type], item) + op, version_update_map[ATTRS][update_type], item + ) if not infos: if update_type == ADD: if update_type not in results.keys(): @@ -279,8 +287,9 @@ def compare_op_desc(origin_op_desc, new_op_desc): origin_attrs = origin_info.get(ATTRS, {}) new_attrs = new_info.get(ATTRS, {}) attrs_error, attrs_diff = diff_attr(origin_attrs, new_attrs) - attrs_version_errors = check_attr_registry(op_type, attrs_diff, - origin_attrs) + attrs_version_errors = check_attr_registry( + op_type, attrs_diff, origin_attrs + ) if ins_diff: desc_error_message.setdefault(op_type, {})[INPUTS] = ins_diff @@ -290,21 +299,26 @@ def compare_op_desc(origin_op_desc, new_op_desc): desc_error_message.setdefault(op_type, {})[ATTRS] = attrs_diff if ins_version_errors: - version_error_message.setdefault(op_type, - {})[INPUTS] = ins_version_errors + version_error_message.setdefault(op_type, {})[ + INPUTS + ] = ins_version_errors if outs_version_errors: - version_error_message.setdefault(op_type, - {})[OUTPUTS] = outs_version_errors + version_error_message.setdefault(op_type, {})[ + OUTPUTS + ] = outs_version_errors if attrs_version_errors: - version_error_message.setdefault(op_type, - {})[ATTRS] = attrs_version_errors + version_error_message.setdefault(op_type, {})[ + ATTRS + ] = attrs_version_errors return desc_error_message, version_error_message def print_desc_error_message(error_message): - print("\n======================= \n" - "Op desc error for the changes of Inputs/Outputs/Attrs of OPs:\n") + print( + "\n======================= \n" + "Op desc error for the changes of Inputs/Outputs/Attrs of OPs:\n" + ) for op_name in error_message: print("For OP '{}':".format(op_name)) @@ -321,16 +335,24 @@ def print_desc_error_message(error_message): for arg in changed_args: ori_value, new_value = changed_args.get(arg) print( - " * The arg '{}' of Input '{}' is changed: from '{}' to '{}'." - .format(arg, name, ori_value, new_value)) + " * The arg '{}' of Input '{}' is changed: from '{}' to '{}'.".format( + arg, name, ori_value, new_value + ) + ) for name in Inputs_error.get(QUANT, {}): - print(" * The added Input '{}' is `quant`, need slim to review.". - format(name)) + print( + " * The added Input '{}' is `quant`, need slim to review.".format( + name + ) + ) for name in Inputs_error.get(DEF, {}): - print(" * The added Input '{}' is `def`, need inference to review.". - format(name)) + print( + " * The added Input '{}' is `def`, need inference to review.".format( + name + ) + ) # 2. print outputs error message Outputs_error = error_message.get(op_name, {}).get(OUTPUTS, {}) @@ -345,23 +367,31 @@ def print_desc_error_message(error_message): for arg in changed_args: ori_value, new_value = changed_args.get(arg) print( - " * The arg '{}' of Output '{}' is changed: from '{}' to '{}'." - .format(arg, name, ori_value, new_value)) + " * The arg '{}' of Output '{}' is changed: from '{}' to '{}'.".format( + arg, name, ori_value, new_value + ) + ) for name in Outputs_error.get(QUANT, {}): - print(" * The added Output '{}' is `quant`, need slim to review.". - format(name)) + print( + " * The added Output '{}' is `quant`, need slim to review.".format( + name + ) + ) for name in Outputs_error.get(DEF, {}): print( - " * The added Output '{}' is `def`, need inference to review.". - format(name)) + " * The added Output '{}' is `def`, need inference to review.".format( + name + ) + ) # 3. print attrs error message attrs_error = error_message.get(op_name, {}).get(ATTRS, {}) for name in attrs_error.get(ADD_WITH_DEFAULT, {}): - print(" * The added attr '{}' doesn't set default value.".format( - name)) + print( + " * The added attr '{}' doesn't set default value.".format(name) + ) for name in attrs_error.get(DELETE, {}): print(" * The attr '{}' is deleted.".format(name)) @@ -371,18 +401,26 @@ def print_desc_error_message(error_message): for arg in changed_args: ori_value, new_value = changed_args.get(arg) print( - " * The arg '{}' of attr '{}' is changed: from '{}' to '{}'." - .format(arg, name, ori_value, new_value)) + " * The arg '{}' of attr '{}' is changed: from '{}' to '{}'.".format( + arg, name, ori_value, new_value + ) + ) for name in attrs_error.get(QUANT, {}): # TODO(Wilber): - print(" * The added attr '{}' is `quant`, need slim to review.". - format(name)) + print( + " * The added attr '{}' is `quant`, need slim to review.".format( + name + ) + ) for name in attrs_error.get(DEF, {}): # TODO(Wilber): - print(" * The added attr '{}' is `def`, need inference to review.". - format(name)) + print( + " * The added attr '{}' is `def`, need inference to review.".format( + name + ) + ) def print_version_error_message(error_message): @@ -398,31 +436,42 @@ def print_version_error_message(error_message): error_list = inputs_error.get(ADD, []) if error_list: for tup in error_list: - print(" * The added input '{}' is not yet registered.".format( - tup[1])) + print( + " * The added input '{}' is not yet registered.".format( + tup[1] + ) + ) # 2. print outputs error message outputs_error = error_message.get(op_name, {}).get(OUTPUTS, {}) error_list = outputs_error.get(ADD, []) if error_list: for tup in error_list: - print(" * The added output '{}' is not yet registered.".format( - tup[1])) + print( + " * The added output '{}' is not yet registered.".format( + tup[1] + ) + ) - #3. print attrs error message + # 3. print attrs error message attrs_error = error_message.get(op_name, {}).get(ATTRS, {}) error_list = attrs_error.get(ADD, []) if error_list: for tup in error_list: print( " * The added attribute '{}' is not yet registered.".format( - tup[1])) - error_dic = error_message.get(op_name, {}).get(ATTRS, - {}).get(CHANGE, {}) + tup[1] + ) + ) + error_dic = ( + error_message.get(op_name, {}).get(ATTRS, {}).get(CHANGE, {}) + ) for key, val in error_dic.items(): print( " * The change of attribute '{}' is not yet registered.".format( - key)) + key + ) + ) def print_repeat_process(): @@ -449,7 +498,8 @@ if len(sys.argv) == 3: new_op_desc = f.read() desc_error_message, version_error_message = compare_op_desc( - origin_op_desc, new_op_desc) + origin_op_desc, new_op_desc + ) if error: print("-" * 30) print_desc_error_message(desc_error_message) diff --git a/tools/check_op_kernel_same_dtypes.py b/tools/check_op_kernel_same_dtypes.py index b5b3e81c89f..13d0119f83b 100644 --- a/tools/check_op_kernel_same_dtypes.py +++ b/tools/check_op_kernel_same_dtypes.py @@ -36,13 +36,15 @@ def get_all_kernels(): pattern = re.compile(r'data_type\[([^\]]+)\]') for op_info in op_infos: infos = pattern.findall(op_info) - if infos is None or len(infos) == 0: continue + if infos is None or len(infos) == 0: + continue register_type = infos[0].split(":")[-1] op_kernel_types[op_type].append(register_type.lower()) - for (op_type, op_kernels) in sorted(op_kernel_types.items(), - key=lambda x: x[0]): + for (op_type, op_kernels) in sorted( + op_kernel_types.items(), key=lambda x: x[0] + ): print(op_type, " ".join(sorted(op_kernels))) @@ -56,12 +58,18 @@ def read_file(file_path): def print_diff(op_type, op_kernel_dtype_set, grad_op_kernel_dtype_set): if len(op_kernel_dtype_set) > len(grad_op_kernel_dtype_set): lack_dtypes = list(op_kernel_dtype_set - grad_op_kernel_dtype_set) - print("{} supports [{}] now, but its grad op kernel not supported.". - format(op_type, " ".join(lack_dtypes))) + print( + "{} supports [{}] now, but its grad op kernel not supported.".format( + op_type, " ".join(lack_dtypes) + ) + ) else: lack_dtypes = list(grad_op_kernel_dtype_set - op_kernel_dtype_set) - print("{} supports [{}] now, but its forward op kernel not supported.". - format(op_type + "_grad", " ".join(lack_dtypes))) + print( + "{} supports [{}] now, but its forward op kernel not supported.".format( + op_type + "_grad", " ".join(lack_dtypes) + ) + ) def contain_current_op(op_type, op_info_dict): @@ -91,7 +99,8 @@ def check_change_or_add_op_kernel_dtypes_valid(): origin_dtype_set = origin_all_kernel_dtype_dict[op_type] # op kernel changed if origin_dtype_set != dtype_set and not contain_current_op( - op_type, added_or_changed_op_info): + op_type, added_or_changed_op_info + ): added_or_changed_op_info[op_type] = dtype_set else: # do nothing @@ -111,7 +120,8 @@ def check_change_or_add_op_kernel_dtypes_valid(): grad_op_type = op_type + "_grad" if grad_op_type in new_all_kernel_dtype_dict: grad_op_kernel_dtype_set = set( - new_all_kernel_dtype_dict[grad_op_type]) + new_all_kernel_dtype_dict[grad_op_type] + ) if dtype_set != grad_op_kernel_dtype_set: print_diff(op_type, dtype_set, grad_op_kernel_dtype_set) # if changed grad op @@ -119,7 +129,8 @@ def check_change_or_add_op_kernel_dtypes_valid(): forward_op_type = op_type.rstrip("_grad") if forward_op_type in new_all_kernel_dtype_dict: op_kernel_dtype_set = set( - new_all_kernel_dtype_dict[forward_op_type]) + new_all_kernel_dtype_dict[forward_op_type] + ) if op_kernel_dtype_set != dtype_set: print_diff(forward_op_type, op_kernel_dtype_set, dtype_set) @@ -129,6 +140,8 @@ if len(sys.argv) == 1: elif len(sys.argv) == 3: check_change_or_add_op_kernel_dtypes_valid() else: - print("Usage:\n" \ - "\tpython check_op_kernel_same_dtypes.py > all_kernels.txt\n" \ - "\tpython check_op_kernel_same_dtypes.py OP_KERNEL_DTYPE_DEV.spec OP_KERNEL_DTYPE_PR.spec > diff") + print( + "Usage:\n" + "\tpython check_op_kernel_same_dtypes.py > all_kernels.txt\n" + "\tpython check_op_kernel_same_dtypes.py OP_KERNEL_DTYPE_DEV.spec OP_KERNEL_DTYPE_PR.spec > diff" + ) diff --git a/tools/check_op_register_type.py b/tools/check_op_register_type.py index 9db2fc0bf51..af7f3ee42a8 100644 --- a/tools/check_op_register_type.py +++ b/tools/check_op_register_type.py @@ -34,18 +34,21 @@ def get_all_kernels(): op_kernel_types = collections.defaultdict(list) for op_type, op_infos in all_kernels_info.items(): is_grad_op = op_type.endswith("_grad") - if is_grad_op: continue + if is_grad_op: + continue pattern = re.compile(r'data_type\[([^\]]+)\]') for op_info in op_infos: infos = pattern.findall(op_info) - if infos is None or len(infos) == 0: continue + if infos is None or len(infos) == 0: + continue register_type = infos[0].split(":")[-1] op_kernel_types[op_type].append(register_type.lower()) - for (op_type, op_kernels) in sorted(op_kernel_types.items(), - key=lambda x: x[0]): + for (op_type, op_kernels) in sorted( + op_kernel_types.items(), key=lambda x: x[0] + ): print(op_type, " ".join(sorted(op_kernels))) @@ -63,8 +66,11 @@ def print_diff(op_type, register_types): if len(FLOATS - register_types) == 1: lack_types |= FLOATS - register_types - print("{} only supports [{}] now, but lacks [{}].".format( - op_type, " ".join(register_types), " ".join(lack_types))) + print( + "{} only supports [{}] now, but lacks [{}].".format( + op_type, " ".join(register_types), " ".join(lack_types) + ) + ) def check_add_op_valid(): @@ -77,10 +83,13 @@ def check_add_op_valid(): for each_diff in result: if each_diff[0] in ['+'] and len(each_diff) > 2: # if change or add op op_info = each_diff[1:].split() - if len(op_info) < 2: continue + if len(op_info) < 2: + continue register_types = set(op_info[1:]) - if len(FLOATS - register_types) == 1 or \ - len(INTS - register_types) == 1: + if ( + len(FLOATS - register_types) == 1 + or len(INTS - register_types) == 1 + ): print_diff(op_info[0], register_types) @@ -89,6 +98,8 @@ if len(sys.argv) == 1: elif len(sys.argv) == 3: check_add_op_valid() else: - print("Usage:\n" \ - "\tpython check_op_register_type.py > all_kernels.txt\n" \ - "\tpython check_op_register_type.py OP_TYPE_DEV.spec OP_TYPE_PR.spec > diff") + print( + "Usage:\n" + "\tpython check_op_register_type.py > all_kernels.txt\n" + "\tpython check_op_register_type.py OP_TYPE_DEV.spec OP_TYPE_PR.spec > diff" + ) diff --git a/tools/check_pr_approval.py b/tools/check_pr_approval.py index c242afd06e7..7e7e7720bf8 100644 --- a/tools/check_pr_approval.py +++ b/tools/check_pr_approval.py @@ -39,8 +39,11 @@ def check_approval(count, required_reviewers): else: required_reviewers_login.add(rr) - if len(set(approved_user_ids) & required_reviewers_int) + len( - approved_user_logins & required_reviewers_login) >= count: + if ( + len(set(approved_user_ids) & required_reviewers_int) + + len(approved_user_logins & required_reviewers_login) + >= count + ): print("TRUE") else: print("FALSE") diff --git a/tools/check_ut.py b/tools/check_ut.py index 0b1211f1e65..e08c358531f 100644 --- a/tools/check_ut.py +++ b/tools/check_ut.py @@ -20,7 +20,7 @@ from github import Github class PRChecker(object): - """ PR Checker. """ + """PR Checker.""" def __init__(self): self.github = Github(os.getenv('GITHUB_API_TOKEN'), timeout=60) diff --git a/tools/codestyle/docstring_checker.py b/tools/codestyle/docstring_checker.py index 18083d1fab1..0d163c20bbf 100644 --- a/tools/codestyle/docstring_checker.py +++ b/tools/codestyle/docstring_checker.py @@ -28,11 +28,10 @@ def register(linter): class Docstring(object): - """Docstring class holds the parsed doc string elements. - """ + """Docstring class holds the parsed doc string elements.""" def __init__(self): - self.d = defaultdict(list) #name->[] + self.d = defaultdict(list) # name->[] self.clear() def clear(self): @@ -40,7 +39,7 @@ class Docstring(object): self.d['Examples'] = [] self.d['Returns'] = [] self.d['Raises'] = [] - self.args = {} #arg_name->arg_type + self.args = {} # arg_name->arg_type def get_level(self, string, indent=' '): level = 0 @@ -111,7 +110,8 @@ class DocstringChecker(BaseChecker): """DosstringChecker is pylint checker to check docstring style. """ - __implements__ = (IAstroidChecker, ) + + __implements__ = (IAstroidChecker,) POSITIONAL_MESSAGE_ID = 'str-used-on-positional-format-argument' KEYWORD_MESSAGE_ID = 'str-used-on-keyword-format-argument' @@ -120,25 +120,41 @@ class DocstringChecker(BaseChecker): symbol = "doc-string" priority = -1 msgs = { - 'W9001': ('One line doc string on > 1 lines', symbol + "-one-line", - 'Used when a short doc string is on multiple lines'), - 'W9002': - ('Doc string does not end with "." period', symbol + "-end-with", - 'Used when a doc string does not end with a period'), - 'W9003': - ('All args with their types must be mentioned in doc string %s', - symbol + "-with-all-args", - 'Used when not all arguments are in the doc string '), - 'W9005': ('Missing docstring or docstring is too short', - symbol + "-missing", 'Add docstring longer >=10'), - 'W9006': ('Docstring indent error, use 4 space for indent', - symbol + "-indent-error", 'Use 4 space for indent'), - 'W9007': - ('You should add `Returns` in comments', symbol + "-with-returns", - 'There should be a `Returns` section in comments'), - 'W9008': - ('You should add `Raises` section in comments', symbol + "-with-raises", - 'There should be a `Raises` section in comments'), + 'W9001': ( + 'One line doc string on > 1 lines', + symbol + "-one-line", + 'Used when a short doc string is on multiple lines', + ), + 'W9002': ( + 'Doc string does not end with "." period', + symbol + "-end-with", + 'Used when a doc string does not end with a period', + ), + 'W9003': ( + 'All args with their types must be mentioned in doc string %s', + symbol + "-with-all-args", + 'Used when not all arguments are in the doc string ', + ), + 'W9005': ( + 'Missing docstring or docstring is too short', + symbol + "-missing", + 'Add docstring longer >=10', + ), + 'W9006': ( + 'Docstring indent error, use 4 space for indent', + symbol + "-indent-error", + 'Use 4 space for indent', + ), + 'W9007': ( + 'You should add `Returns` in comments', + symbol + "-with-returns", + 'There should be a `Returns` section in comments', + ), + 'W9008': ( + 'You should add `Raises` section in comments', + symbol + "-with-raises", + 'There should be a `Raises` section in comments', + ), } options = () @@ -321,8 +337,7 @@ class DocstringChecker(BaseChecker): return True args = [] for arg in node.args.get_children(): - if (not isinstance(arg, astroid.AssignName)) \ - or arg.name == "self": + if (not isinstance(arg, astroid.AssignName)) or arg.name == "self": continue args.append(arg.name) @@ -332,20 +347,24 @@ class DocstringChecker(BaseChecker): parsed_args = doc.args args_not_documented = set(args) - set(parsed_args) if len(args) > 0 and len(parsed_args) <= 0: - self.add_message('W9003', - node=node, - line=node.fromlineno, - args=list(args_not_documented)) + self.add_message( + 'W9003', + node=node, + line=node.fromlineno, + args=list(args_not_documented), + ) return False for t in args: if t not in parsed_args: - self.add_message('W9003', - node=node, - line=node.fromlineno, - args=[ - t, - ]) + self.add_message( + 'W9003', + node=node, + line=node.fromlineno, + args=[ + t, + ], + ) return False return True diff --git a/tools/codestyle/test_docstring_checker.py b/tools/codestyle/test_docstring_checker.py index 61b236b6b7d..09617a0f2f7 100644 --- a/tools/codestyle/test_docstring_checker.py +++ b/tools/codestyle/test_docstring_checker.py @@ -21,7 +21,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): CHECKER_CLASS = docstring_checker.DocstringChecker def test_one_line(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """get news. @@ -29,7 +30,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): if True: return 5 return 5 - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -37,13 +39,15 @@ class TestDocstring(pylint.testutils.CheckerTestCase): assert 'W9001' == got[0][0] def test_one_line_1(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """get news""" if True: return 5 return 5 - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -51,7 +55,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): assert 'W9002' == got[0][0] def test_args(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(scale, mean): """get news. Args: @@ -64,7 +69,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): mean=scale mean=scale mean=scale - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -72,7 +78,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): assert 'W9003' == got[0][0] def test_missing(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): mean=scale mean=scale @@ -85,7 +92,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): mean=scale mean=scale mean=scale - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -93,13 +101,15 @@ class TestDocstring(pylint.testutils.CheckerTestCase): assert 'W9005' == got[0][0] def test_indent(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """ get get get get get get get get get get get get get get get get. """ pass - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -107,7 +117,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): assert 'W9006' == got[0][0] def test_with_resturns(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """get news. Args: @@ -125,7 +136,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): mean=scale mean=scale return mean - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -133,7 +145,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): assert 'W9007' == got[0][0] def test_with_raises(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """get news. Args: @@ -151,7 +164,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): mean=scale mean=scale raise ValueError('A very specific bad thing happened.') - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() diff --git a/tools/continuous_integration/bisect.py b/tools/continuous_integration/bisect.py index afffc60a449..b2a2aa1f9b6 100644 --- a/tools/continuous_integration/bisect.py +++ b/tools/continuous_integration/bisect.py @@ -27,43 +27,42 @@ import subprocess import sys parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument('--git_dir', - type=str, - default='', - help='git repo root directory.') -parser.add_argument('--build_dir', - type=str, - default='', - help='build directory.') -parser.add_argument('--good_commit', - type=str, - default='', - help='The old commit known to be good.') -parser.add_argument('--bad_commit', - type=str, - default='', - help='The new commit known to be bad.') -parser.add_argument('--test_target', - type=str, - default='', - help='The test target to evaluate.') +parser.add_argument( + '--git_dir', type=str, default='', help='git repo root directory.' +) +parser.add_argument( + '--build_dir', type=str, default='', help='build directory.' +) +parser.add_argument( + '--good_commit', + type=str, + default='', + help='The old commit known to be good.', +) +parser.add_argument( + '--bad_commit', type=str, default='', help='The new commit known to be bad.' +) +parser.add_argument( + '--test_target', type=str, default='', help='The test target to evaluate.' +) parser.add_argument( '--bisect_branch', type=str, default='develop', - help='The mainline branch to bisect (feature branch ignored.') -parser.add_argument('--log_file', - type=str, - default='', - help='The file use to log outputs.') -parser.add_argument('--test_times', - type=int, - default=10, - help="Number of times to run the test target.") -parser.add_argument('--build_parallel', - type=int, - default=32, - help="make parallelism.") + help='The mainline branch to bisect (feature branch ignored.', +) +parser.add_argument( + '--log_file', type=str, default='', help='The file use to log outputs.' +) +parser.add_argument( + '--test_times', + type=int, + default=10, + help="Number of times to run the test target.", +) +parser.add_argument( + '--build_parallel', type=int, default=32, help="make parallelism." +) args = parser.parse_args() if not args.log_file: @@ -81,10 +80,13 @@ print_arguments() # List the commits in mainline branch. os.chdir(args.git_dir) -ret = subprocess.check_output([ - 'git rev-list --first-parent %s...%s' % (args.good_commit, args.bad_commit) -], - shell=True) +ret = subprocess.check_output( + [ + 'git rev-list --first-parent %s...%s' + % (args.good_commit, args.bad_commit) + ], + shell=True, +) sys.stdout.write('commits found:\n%s\n' % ret) commits = ret.strip().split('\n') os.chdir(args.build_dir) @@ -95,11 +97,13 @@ last_culprit = '' while True: # Get to the mainline branch and clean up os.chdir(args.git_dir) - subprocess.check_output([ - 'git checkout %s && git clean -fd && git checkout .' % - args.bisect_branch - ], - shell=True) + subprocess.check_output( + [ + 'git checkout %s && git clean -fd && git checkout .' + % args.bisect_branch + ], + shell=True, + ) if not commits: sys.stdout.write('no commits to bisect\n') @@ -115,9 +119,11 @@ while True: os.chdir(args.build_dir) sys.stdout.write('eval commit %d/%d: %s\n' % (pick_idx, len(commits), pick)) # Link error can happen without complete clean up. - cmd = ('rm -rf * && ' - 'cmake -DWITH_TESTING=ON %s >> %s && make -j%s >> %s' % - (args.git_dir, args.log_file, args.build_parallel, args.log_file)) + cmd = ( + 'rm -rf * && ' + 'cmake -DWITH_TESTING=ON %s >> %s && make -j%s >> %s' + % (args.git_dir, args.log_file, args.build_parallel, args.log_file) + ) sys.stdout.write('cmd: %s\n' % cmd) try: subprocess.check_output([cmd], shell=True) @@ -127,8 +133,11 @@ while True: # test the selected branch. passed = True try: - cmd = ('ctest --repeat-until-fail %s -R %s >> %s' % - (args.test_times, args.test_target, args.log_file)) + cmd = 'ctest --repeat-until-fail %s -R %s >> %s' % ( + args.test_times, + args.test_target, + args.log_file, + ) sys.stdout.write('cmd: %s\n' % cmd) subprocess.check_output([cmd], shell=True) except subprocess.CalledProcessError as e: @@ -136,10 +145,12 @@ while True: last_culprit = pick sys.stdout.write('eval %s passed: %s\n' % (pick, passed)) if passed: - if pick_idx == 0: break + if pick_idx == 0: + break commits = commits[:pick_idx] else: - if pick_idx + 1 >= len(commits): break - commits = commits[pick_idx + 1:] + if pick_idx + 1 >= len(commits): + break + commits = commits[pick_idx + 1 :] sys.stdout.write('Culprit commit: %s\n' % last_culprit) diff --git a/tools/count_api_without_core_ops.py b/tools/count_api_without_core_ops.py index d75578d610d..96a7c33dd95 100644 --- a/tools/count_api_without_core_ops.py +++ b/tools/count_api_without_core_ops.py @@ -37,9 +37,12 @@ def md5(doc): md5sum = hashinst.hexdigest() except UnicodeDecodeError as e: md5sum = None - print("Error({}) occurred when `md5({})`, discard it.".format( - str(e), doc), - file=sys.stderr) + print( + "Error({}) occurred when `md5({})`, discard it.".format( + str(e), doc + ), + file=sys.stderr, + ) return md5sum @@ -50,11 +53,13 @@ def split_with_and_without_core_ops(member, cur_name): if member.__doc__.find(':api_attr: Static Graph') != -1: return - if cur_name.find('ParamBase') != -1 or cur_name.find( - 'Parameter') != -1 or cur_name.find( - 'Variable') != -1 or cur_name.find( - 'control_flow') != -1 or cur_name.find( - 'contrib.mixed_precision') != -1: + if ( + cur_name.find('ParamBase') != -1 + or cur_name.find('Parameter') != -1 + or cur_name.find('Variable') != -1 + or cur_name.find('control_flow') != -1 + or cur_name.find('contrib.mixed_precision') != -1 + ): return if inspect.isclass(member): @@ -94,8 +99,9 @@ def visit_member(parent_name, member, func): if inspect.isclass(member): func(member, cur_name) for name, value in inspect.getmembers(member): - if hasattr(value, '__name__') and (not name.startswith("_") - or name == "__init__"): + if hasattr(value, '__name__') and ( + not name.startswith("_") or name == "__init__" + ): visit_member(cur_name, value, func) elif inspect.ismethoddescriptor(member): return @@ -106,11 +112,13 @@ def visit_member(parent_name, member, func): else: raise RuntimeError( "Unsupported generate signature of member, type {0}".format( - str(type(member)))) + str(type(member)) + ) + ) def is_primitive(instance): - int_types = (int, ) + int_types = (int,) pritimitive_types = int_types + (float, str) if isinstance(instance, pritimitive_types): return True @@ -171,8 +179,9 @@ def get_apis_with_and_without_core_ops(modules): api_with_ops = [] api_without_ops = [] for m in modules: - visit_all_module(importlib.import_module(m), - split_with_and_without_core_ops) + visit_all_module( + importlib.import_module(m), split_with_and_without_core_ops + ) return api_with_ops, api_without_ops @@ -189,7 +198,8 @@ if __name__ == "__main__": modules = sys.argv[2].split(",") if sys.argv[1] == '-c': api_with_ops, api_without_ops = get_apis_with_and_without_core_ops( - modules) + modules + ) print('api_with_ops:', len(api_with_ops)) print('\n'.join(api_with_ops)) @@ -203,9 +213,11 @@ if __name__ == "__main__": print(name, func_dict[name]) else: - print("""Usage: + print( + """Usage: 1. Count and list all operator-raleated APIs that contains append_op but not _legacy_C_ops.xx. python ./count_api_without_core_ops.py -c paddle 2. Print api and the md5 of source code of the api. python ./count_api_without_core_ops.py -p paddle - """) + """ + ) diff --git a/tools/coverage/coverage_diff.py b/tools/coverage/coverage_diff.py index 713afe1ebc3..980b2cd3363 100644 --- a/tools/coverage/coverage_diff.py +++ b/tools/coverage/coverage_diff.py @@ -86,7 +86,7 @@ def get_info_file_lines(info_file, diff_file): current_file = line.lstrip('SF:') if current_file.startswith('/paddle/'): - current_file = current_file[len('/paddle/'):] + current_file = current_file[len('/paddle/') :] current_lines = diff_file_lines.get(current_file, []) diff --git a/tools/coverage/coverage_diff_list.py b/tools/coverage/coverage_diff_list.py index c57c1e9d964..1fee1d3ae15 100644 --- a/tools/coverage/coverage_diff_list.py +++ b/tools/coverage/coverage_diff_list.py @@ -41,7 +41,7 @@ def filter_by(list_file, max_rate): name = split[0].strip() if name.startswith('/paddle/'): - name = name[len('/paddle/'):] + name = name[len('/paddle/') :] # rate diff --git a/tools/coverage/coverage_lines.py b/tools/coverage/coverage_lines.py index 3c5df9d88e8..10a0520c08c 100644 --- a/tools/coverage/coverage_lines.py +++ b/tools/coverage/coverage_lines.py @@ -29,8 +29,8 @@ def get_lines(info_file): Returns: float: Coverage rate. """ - hits = .0 - total = .0 + hits = 0.0 + total = 0.0 with open(info_file) as info_file: for line in info_file: @@ -68,10 +68,16 @@ if __name__ == '__main__': actual = round(actual, 3) if actual < expected: - print('expected >= {} %, actual {} %, failed'.format( - round(expected * 100, 1), round(actual * 100, 1))) + print( + 'expected >= {} %, actual {} %, failed'.format( + round(expected * 100, 1), round(actual * 100, 1) + ) + ) exit(1) - print('expected >= {} %, actual {} %, passed'.format( - round(expected * 100, 1), round(actual * 100, 1))) + print( + 'expected >= {} %, actual {} %, passed'.format( + round(expected * 100, 1), round(actual * 100, 1) + ) + ) diff --git a/tools/coverage/cuda_clean.py b/tools/coverage/cuda_clean.py index 82bb6a553c9..fea2326413d 100644 --- a/tools/coverage/cuda_clean.py +++ b/tools/coverage/cuda_clean.py @@ -65,7 +65,7 @@ def clean(pull_id): changed = [] for file in get_files(pull_id): - #changed.append('/paddle/build/{}.gcda'.format(file)) + # changed.append('/paddle/build/{}.gcda'.format(file)) changed.append(file) for parent, dirs, files in os.walk('/paddle/build/'): @@ -81,7 +81,7 @@ def clean(pull_id): if src_name not in changed: unused_file = parent + '/' + gcda - #print unused_file + # print unused_file os.remove(gcda) else: print(src_name) diff --git a/tools/coverage/gcda_clean.py b/tools/coverage/gcda_clean.py index 286f50eae9a..ab8ae043238 100644 --- a/tools/coverage/gcda_clean.py +++ b/tools/coverage/gcda_clean.py @@ -40,8 +40,11 @@ def get_pull(pull_id): repo = github.get_repo('PaddlePaddle/Paddle') except Exception as e: print(e) - print("get_repo error, retry {} times after {} secs.".format( - idx, idx * 10)) + print( + "get_repo error, retry {} times after {} secs.".format( + idx, idx * 10 + ) + ) else: break idx += 1 diff --git a/tools/coverage/python_coverage.py b/tools/coverage/python_coverage.py index d45fb4d58c5..53784a08abe 100644 --- a/tools/coverage/python_coverage.py +++ b/tools/coverage/python_coverage.py @@ -33,8 +33,9 @@ for clazz in root.findall('packages/package/classes/class'): clazz_filename = path.join(source, clazz_filename) if clazz_filename.startswith('/paddle/build/python/'): - clazz_filename = '/paddle/python/' + clazz_filename[ - len('/paddle/build/python/'):] + clazz_filename = ( + '/paddle/python/' + clazz_filename[len('/paddle/build/python/') :] + ) if not path.exists(clazz_filename): continue @@ -61,14 +62,20 @@ for clazz in root.findall('packages/package/classes/class'): taken = int(taken) for _ in range(taken): - print('BRDA:{},{},{},{}'.format(line_number, 0, branch_index, - line_hits)) + print( + 'BRDA:{},{},{},{}'.format( + line_number, 0, branch_index, line_hits + ) + ) branch_index += 1 if line_missing_branches: for missing_branch in line_missing_branches.split(','): - print('BRDA:{},{},{},{}'.format(line_number, 0, - branch_index, 0)) + print( + 'BRDA:{},{},{},{}'.format( + line_number, 0, branch_index, 0 + ) + ) branch_index += 1 print('DA:{},{}'.format(line_number, line_hits)) diff --git a/tools/externalError/spider.py b/tools/externalError/spider.py index d0f9f48c89f..d6062e5d50a 100644 --- a/tools/externalError/spider.py +++ b/tools/externalError/spider.py @@ -22,8 +22,8 @@ from html.parser import HTMLParser def parsing(externalErrorDesc): - #*********************************************************************************************# - #*********************************** CUDA Error Message **************************************# + # *********************************************************************************************# + # *********************************** CUDA Error Message **************************************# print("start crawling errorMessage for nvidia CUDA API--->") url = 'https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1g3f51e3575c2178246db0a94a430e0038' @@ -51,7 +51,8 @@ def parsing(externalErrorDesc): m_message = m_message.replace(list_a[idx], list_shape[idx]) m_message = m_message.replace( - '
Deprecated
', '') + '
Deprecated
', '' + ) res_span = r'()' res_span_detail = r'(.*?)' @@ -81,8 +82,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (m_type[0], m_message) print("End crawling errorMessage for nvidia CUDA API!\n") - #***********************************************************************************************# - #*********************************** CURAND Error Message **************************************# + # ***********************************************************************************************# + # *********************************** CURAND Error Message **************************************# print("start crawling errorMessage for nvidia CURAND API--->") url = 'https://docs.nvidia.com/cuda/curand/group__HOST.html#group__HOST_1gb94a31d5c165858c96b6c18b70644437' @@ -112,8 +113,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (m_type[0], m_message) print("End crawling errorMessage for nvidia CURAND API!\n") - #**************************************************************************************************# - #*********************************** CUDNN Error Message ******************************************# + # **************************************************************************************************# + # *********************************** CUDNN Error Message ******************************************# cudnnStatus_t = { "CUDNN_STATUS_SUCCESS": 0, "CUDNN_STATUS_NOT_INITIALIZED": 1, @@ -155,8 +156,9 @@ def parsing(externalErrorDesc): list_class_detail = re.findall(res_class_detail, m_message, re.S | re.M) assert len(list_class) == len(list_class_detail) for idx in range(len(list_class)): - m_message = m_message.replace(list_class[idx], - list_class_detail[idx]) + m_message = m_message.replace( + list_class[idx], list_class_detail[idx] + ) res_a = r'()' res_shape = r'(.*?)' @@ -189,8 +191,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (error[0], m_message) print("End crawling errorMessage for nvidia CUDNN API!\n") - #*************************************************************************************************# - #*********************************** CUBLAS Error Message ****************************************# + # *************************************************************************************************# + # *********************************** CUBLAS Error Message ****************************************# cublasStatus_t = { "CUBLAS_STATUS_SUCCESS": 0, "CUBLAS_STATUS_NOT_INITIALIZED": 1, @@ -201,7 +203,7 @@ def parsing(externalErrorDesc): "CUBLAS_STATUS_EXECUTION_FAILED": 13, "CUBLAS_STATUS_INTERNAL_ERROR": 14, "CUBLAS_STATUS_NOT_SUPPORTED": 15, - "CUBLAS_STATUS_LICENSE_ERROR": 16 + "CUBLAS_STATUS_LICENSE_ERROR": 16, } print("start crawling errorMessage for nvidia CUBLAS API--->") @@ -243,8 +245,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (error[0], m_message) print("End crawling errorMessage for nvidia CUBLAS API!\n") - #*************************************************************************************************# - #*********************************** CUSOLVER Error Message **************************************# + # *************************************************************************************************# + # *********************************** CUSOLVER Error Message **************************************# cusolverStatus_t = { "CUSOLVER_STATUS_SUCCESS": 0, "CUSOLVER_STATUS_NOT_INITIALIZED": 1, @@ -264,7 +266,7 @@ def parsing(externalErrorDesc): "CUSOLVER_STATUS_IRS_NOT_SUPPORTED": 15, "CUSOLVER_STATUS_IRS_OUT_OF_RANGE": 16, "CUSOLVER_STATUS_IRS_NRHS_NOT_SUPPORTED_FOR_REFINE_GMRES": 17, - "CUSOLVER_STATUS_IRS_INFOS_NOT_INITIALIZED": 18 + "CUSOLVER_STATUS_IRS_INFOS_NOT_INITIALIZED": 18, } print("start crawling errorMessage for nvidia CUSOLVER API--->") url = 'https://docs.nvidia.com/cuda/cusolver/index.html#cuSolverSPstatus' @@ -277,7 +279,9 @@ def parsing(externalErrorDesc): res_div = r'This is a status type returned by the library functions and.*?
(.*?)
' m_div = re.findall(res_div, html, re.S | re.M)[0] - res_dt = r'(.*?).*?colspan="1">(.*?)' + res_dt = ( + r'(.*?).*?colspan="1">(.*?)' + ) m_dt = re.findall(res_dt, m_div, re.S | re.M) for error in m_dt: @@ -304,20 +308,22 @@ def parsing(externalErrorDesc): res_strong = r'.*?' res_strong_detail = r'(.*?)' list_strong = re.findall(res_strong, m_message, re.S | re.M) - list_strong_detail = re.findall(res_strong_detail, m_message, - re.S | re.M) + list_strong_detail = re.findall( + res_strong_detail, m_message, re.S | re.M + ) assert len(list_strong) == len(list_strong_detail) for idx in range(len(list_strong)): - m_message = m_message.replace(list_strong[idx], - list_strong_detail[idx]) + m_message = m_message.replace( + list_strong[idx], list_strong_detail[idx] + ) _Messages = allMessageDesc.messages.add() _Messages.code = int(cusolverStatus_t[error[0]]) _Messages.message = "'%s'. %s" % (error[0], m_message) print("End crawling errorMessage for nvidia CUSOLVER API!\n") - #**********************************************************************************************# - #*************************************** NCCL error *******************************************# + # **********************************************************************************************# + # *************************************** NCCL error *******************************************# print("start crawling errorMessage for nvidia NCCL API--->") url = 'https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/types.html#ncclresult-t' allMessageDesc = externalErrorDesc.errors.add() @@ -335,8 +341,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (error[0], m_message) print("End crawling errorMessage for nvidia NCCL API!\n") - #*************************************************************************************************# - #*********************************** CUFFT Error Message **************************************# + # *************************************************************************************************# + # *********************************** CUFFT Error Message **************************************# print("start crawling errorMessage for nvidia CUFFT API--->") url = 'https://docs.nvidia.com/cuda/cufft/index.html#cufftresult' @@ -346,8 +352,7 @@ def parsing(externalErrorDesc): html = urllib.request.urlopen(url).read().decode('utf-8') class CUFFTHTMLParser(HTMLParser): - '''CUFFTHTML Parser - ''' + '''CUFFTHTML Parser''' def handle_data(self, data): if 'typedef enum cufftResult_t' in data: @@ -355,8 +360,10 @@ def parsing(externalErrorDesc): status, code, desc = re.split('=|//', line.strip()) _Messages = allMessageDesc.messages.add() _Messages.code = int(code.strip(' ,')) - _Messages.message = "'%s'. %s" % (status.strip(), - desc.strip()) + _Messages.message = "'%s'. %s" % ( + status.strip(), + desc.strip(), + ) CUFFTHTMLParser().feed(html) diff --git a/tools/final_ut_parallel_rule.py b/tools/final_ut_parallel_rule.py index a178bc2bbd7..4d98eee41fb 100644 --- a/tools/final_ut_parallel_rule.py +++ b/tools/final_ut_parallel_rule.py @@ -23,28 +23,44 @@ def classify_cases_by_mem(rootPath): """classify cases by mem""" case_filename = '%s/build/classify_case_by_cardNum.txt' % rootPath case_exec_100 = [ - 'test_conv_eltwiseadd_bn_fuse_pass', 'test_trt_convert_pool2d', - 'test_fc_fuse_pass', 'test_trt_convert_depthwise_conv2d', + 'test_conv_eltwiseadd_bn_fuse_pass', + 'test_trt_convert_pool2d', + 'test_fc_fuse_pass', + 'test_trt_convert_depthwise_conv2d', 'test_quant2_int8_resnet50_mkldnn', - 'test_conv_elementwise_add_act_fuse_pass', 'test_trt_convert_conv2d', - 'test_paddle_save_load', 'test_logical_op', 'test_nearest_interp_op', - 'test_pool2d_op', 'test_conv3d_transpose_op', 'test_lstmp_op', - 'test_cross_entropy2_op', 'test_sgd_op', 'test_imperative_ptq', - 'test_model', 'test_custom_relu_op_setup', 'test_dropout_op', - 'test_concat_op' - ] #木桶原理 70s-100s之间的case + 'test_conv_elementwise_add_act_fuse_pass', + 'test_trt_convert_conv2d', + 'test_paddle_save_load', + 'test_logical_op', + 'test_nearest_interp_op', + 'test_pool2d_op', + 'test_conv3d_transpose_op', + 'test_lstmp_op', + 'test_cross_entropy2_op', + 'test_sgd_op', + 'test_imperative_ptq', + 'test_model', + 'test_custom_relu_op_setup', + 'test_dropout_op', + 'test_concat_op', + ] # 木桶原理 70s-100s之间的case case_exec_200 = [ 'test_post_training_quantization_mnist', 'test_imperative_auto_mixed_precision', 'test_trt_dynamic_shape_ernie_fp16_ser_deser', - 'test_trt_dynamic_shape_ernie', 'test_layer_norm_op', - 'trt_quant_int8_yolov3_r50_test', 'test_gru_op', - 'test_post_training_quantization_while', 'test_mkldnn_log_softmax_op', - 'test_mkldnn_matmulv2_op', 'test_mkldnn_shape_op', + 'test_trt_dynamic_shape_ernie', + 'test_layer_norm_op', + 'trt_quant_int8_yolov3_r50_test', + 'test_gru_op', + 'test_post_training_quantization_while', + 'test_mkldnn_log_softmax_op', + 'test_mkldnn_matmulv2_op', + 'test_mkldnn_shape_op', 'interceptor_pipeline_short_path_test', - 'interceptor_pipeline_long_path_test', 'test_cpuonly_spawn' - ] #木桶原理 110s-200s之间的case 以及容易timeout + 'interceptor_pipeline_long_path_test', + 'test_cpuonly_spawn', + ] # 木桶原理 110s-200s之间的case 以及容易timeout case_always_timeout = [ 'test_quant2_int8_resnet50_channelwise_mkldnn', @@ -104,10 +120,10 @@ def classify_cases_by_mem(rootPath): if case not in new_lastest_mem: continue - #mem = 0 + # mem = 0 if new_lastest_mem[case]["mem_nvidia"] == 0: case_mem_0 = case_mem_0 + '|^' + case + '$' - #mem != 0 + # mem != 0 else: case_mem_1[case] = new_lastest_mem[case]["mem_nvidia"] diff --git a/tools/gen_ut_cmakelists.py b/tools/gen_ut_cmakelists.py index 62deef8c21b..14f8e37626e 100644 --- a/tools/gen_ut_cmakelists.py +++ b/tools/gen_ut_cmakelists.py @@ -44,9 +44,10 @@ def _process_envs(envs): processed_envs = [] for p in envs_parts: - assert " " not in p and \ - re.compile("^[a-zA-Z_][0-9a-zA-Z_]*=").search(p) is not None, \ - f"""The environment option format is wrong. The env variable name can only contains'a-z', 'A-Z', '0-9' and '_', + assert ( + " " not in p + and re.compile("^[a-zA-Z_][0-9a-zA-Z_]*=").search(p) is not None + ), f"""The environment option format is wrong. The env variable name can only contains'a-z', 'A-Z', '0-9' and '_', and the var can not contain space in either env names or values. However the var's format is '{p}'.""" @@ -95,8 +96,13 @@ def _proccess_archs(arch): arch = arch.upper().strip() if len(arch) > 0: for a in arch.split(";"): - assert a in ["GPU", "ROCM", "ASCEND", "ASCEND_CL", "XPU"], \ - f"""Supported arhc options are "GPU", "ROCM", "ASCEND" and "ASCEND_CL", "XPU", but the options is {a}""" + assert a in [ + "GPU", + "ROCM", + "ASCEND", + "ASCEND_CL", + "XPU", + ], f"""Supported arhc options are "GPU", "ROCM", "ASCEND" and "ASCEND_CL", "XPU", but the options is {a}""" archs += "WITH_" + a.upper() + " OR " arch = "(" + archs[:-4] + ")" else: @@ -121,7 +127,9 @@ def _process_os(os_): os_ = os_.upper() for p in os_.split(';'): assert p in [ - "WIN32", "APPLE", "LINUX" + "WIN32", + "APPLE", + "LINUX", ], f"""Supported os options are 'WIN32', 'APPLE' and 'LINUX', but the options is {p}""" os_ = os_.replace(";", " OR ") os_ = "(" + os_ + ")" @@ -133,8 +141,11 @@ def _process_os(os_): # check whether run_serial is 0, 1 or empty def _process_run_serial(run_serial): rs = run_serial.strip() - assert rs in ["1", "0", ""], \ - f"""the value of run_serial must be one of 0, 1 or empty. But this value is {rs}""" + assert rs in [ + "1", + "0", + "", + ], f"""the value of run_serial must be one of 0, 1 or empty. But this value is {rs}""" if rs == "": return "" return rs @@ -157,13 +168,15 @@ def _process_name(name, curdir): check whether name is with a legal format and check whther the test file exists. """ name = name.strip() - assert re.compile("^test_[0-9a-zA-Z_]+").search(name), \ - """If line is not the header of table, the test name must begin with "test_" """ \ + assert re.compile("^test_[0-9a-zA-Z_]+").search(name), ( + """If line is not the header of table, the test name must begin with "test_" """ """and the following substring must include at least one char of "0-9", "a-z", "A-Z" or "_".""" + ) filepath_prefix = os.path.join(curdir, name) suffix = [".py", ".sh"] - assert _file_with_extension(filepath_prefix, suffix), \ - f""" Please ensure the test file with the prefix '{filepath_prefix}' and one of the suffix {suffix} exists, because you specified a unittest named '{name}'""" + assert _file_with_extension( + filepath_prefix, suffix + ), f""" Please ensure the test file with the prefix '{filepath_prefix}' and one of the suffix {suffix} exists, because you specified a unittest named '{name}'""" return name @@ -182,14 +195,16 @@ def _norm_dirs(dirs): def _process_run_type(run_type): rt = run_type.strip() # completely match one of the strings: 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY' - assert re.compile("^(NIGHTLY|EXCLUSIVE|CINN|DIST|GPUPS|INFER|EXCLUSIVE:NIGHTLY|DIST:NIGHTLY)$").search(rt), \ - f""" run_type must be one of 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'""" \ + assert re.compile( + "^(NIGHTLY|EXCLUSIVE|CINN|DIST|GPUPS|INFER|EXCLUSIVE:NIGHTLY|DIST:NIGHTLY)$" + ).search(rt), ( + f""" run_type must be one of 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'""" f"""but the run_type is {rt}""" + ) return rt -class DistUTPortManager(): - +class DistUTPortManager: def __init__(self, ignore_dirs=[]): self.dist_ut_port = 21200 self.assigned_ports = dict() @@ -212,13 +227,17 @@ class DistUTPortManager(): ''' if test_name not in self.assigned_ports: self.assigned_ports[test_name] = port - self.dist_ut_port = max(self.dist_ut_port, - self.assigned_ports[test_name]) + self.dist_ut_port = max( + self.dist_ut_port, self.assigned_ports[test_name] + ) return self.assigned_ports[test_name] def process_dist_port_num(self, port_num): - assert re.compile("^[0-9]+$").search(port_num) and int(port_num) > 0 or port_num.strip()=="", \ - f"""port_num must be foramt as a positive integer or empty, but this port_num is '{port_num}'""" + assert ( + re.compile("^[0-9]+$").search(port_num) + and int(port_num) > 0 + or port_num.strip() == "" + ), f"""port_num must be foramt as a positive integer or empty, but this port_num is '{port_num}'""" port_num = port_num.strip() if len(port_num) == 0: return 0 @@ -242,7 +261,7 @@ class DistUTPortManager(): if matched is None: continue p = matched.span() - port = int(line[p[0]:p[1]].split("=")[-1]) + port = int(line[p[0] : p[1]].split("=")[-1]) # find the test name which the port belongs to for k in range(idx, 0, -1): @@ -252,8 +271,9 @@ class DistUTPortManager(): # matcg right tets name format, the name must start with 'test_' follwed bu at least one cahr of # '0-9'. 'a-z'. 'A-Z' or '_' - assert re.compile("^test_[0-9a-zA-Z_]+").search(name), \ - f'''we found a test for initial the latest dist_port but the test name '{name}' seems to be wrong + assert re.compile("^test_[0-9a-zA-Z_]+").search( + name + ), f'''we found a test for initial the latest dist_port but the test name '{name}' seems to be wrong at line {k-1}, in file {cmake_file_name} ''' self.gset_port(name, port) @@ -302,19 +322,35 @@ class DistUTPortManager(): # if such a drectory exists # step 1 - if len(self.last_test_name) > 0 and len( - self.last_test_cmake_file) > 0: + if ( + len(self.last_test_name) > 0 + and len(self.last_test_cmake_file) > 0 + ): with open( - self.last_test_cmake_file.replace( - "CMakeLists.txt", "testslist.csv")) as csv_file: + self.last_test_cmake_file.replace( + "CMakeLists.txt", "testslist.csv" + ) + ) as csv_file: found = False for line in csv_file.readlines(): - name, _, _, _, _, launcher, num_port, _, _, _ = line.strip( - ).split(",") + ( + name, + _, + _, + _, + _, + launcher, + num_port, + _, + _, + _, + ) = line.strip().split(",") if name == self.last_test_name: found = True break - assert found, f"no such test named '{self.last_test_name}' in file '{self.last_test_cmake_file}'" + assert ( + found + ), f"no such test named '{self.last_test_name}' in file '{self.last_test_cmake_file}'" if launcher[-2:] == ".sh": self.process_dist_port_num(num_port) @@ -336,8 +372,7 @@ class DistUTPortManager(): assert len(self.no_cmake_dirs) == 0, err_msg -class CMakeGenerator(): - +class CMakeGenerator: def __init__(self, current_dirs, ignore_dirs): self.processed_dirs = set() self.port_manager = DistUTPortManager(ignore_dirs) @@ -393,8 +428,18 @@ class CMakeGenerator(): endif()" """ - name, os_, archs, timeout, run_type, launcher, num_port, run_serial, envs, conditions = line.strip( - ).split(",") + ( + name, + os_, + archs, + timeout, + run_type, + launcher, + num_port, + run_serial, + envs, + conditions, + ) = line.strip().split(",") # name == "name" means the line being parsed is the header of the table # we should skip this line and return empty here. @@ -439,12 +484,15 @@ class CMakeGenerator(): "{envs}")%s endif() ''' - run_type_str = "" if len( - run_type) == 0 else f' LABELS "RUN_TYPE={run_type}"' - time_out_str = f' TIMEOUT "{timeout}"' if len( - timeout.strip()) > 0 else '' - run_serial_str = f' RUN_SERIAL {run_serial}' if len( - run_serial) > 0 else '' + run_type_str = ( + "" if len(run_type) == 0 else f' LABELS "RUN_TYPE={run_type}"' + ) + time_out_str = ( + f' TIMEOUT "{timeout}"' if len(timeout.strip()) > 0 else '' + ) + run_serial_str = ( + f' RUN_SERIAL {run_serial}' if len(run_serial) > 0 else '' + ) if len(time_out_str) > 0 or len(run_serial_str) > 0: set_properties = f''' set_tests_properties({name} PROPERTIES{time_out_str}{run_serial_str}{run_type_str})''' @@ -471,10 +519,14 @@ class CMakeGenerator(): if not os.path.isdir(c_path): continue self.processed_dirs.add(c_path) - if os.path.isfile(os.path.join(current_work_dir, c, "testslist.csv")) \ - or os.path.isfile(os.path.join(current_work_dir, c, "CMakeLists.txt")): - self._gen_cmakelists(os.path.join(current_work_dir, c), - depth + 1) + if os.path.isfile( + os.path.join(current_work_dir, c, "testslist.csv") + ) or os.path.isfile( + os.path.join(current_work_dir, c, "CMakeLists.txt") + ): + self._gen_cmakelists( + os.path.join(current_work_dir, c), depth + 1 + ) sub_dirs.append(c) if not os.path.isfile(os.path.join(current_work_dir, "testslist.csv")): @@ -506,10 +558,13 @@ class CMakeGenerator(): char_seq = "".join(char_seq) if char_seq != "".join(cmds.split()): - assert f"{current_work_dir}/CMakeLists.txt" not in self.modified_or_created_files, \ - f"the file {current_work_dir}/CMakeLists.txt are modified twice, which may cause some error" + assert ( + f"{current_work_dir}/CMakeLists.txt" + not in self.modified_or_created_files + ), f"the file {current_work_dir}/CMakeLists.txt are modified twice, which may cause some error" self.modified_or_created_files.append( - f"{current_work_dir}/CMakeLists.txt") + f"{current_work_dir}/CMakeLists.txt" + ) with open(f"{current_work_dir}/CMakeLists.txt", "w") as cmake_file: print(cmds, end="", file=cmake_file) @@ -523,8 +578,7 @@ if __name__ == "__main__": required=False, default=[], nargs="+", - help= - "Input a list of files named testslist.csv and output files named CmakeLists.txt in the same directories as the csv files respectly" + help="Input a list of files named testslist.csv and output files named CmakeLists.txt in the same directories as the csv files respectly", ) parser.add_argument( "--dirpaths", @@ -533,8 +587,7 @@ if __name__ == "__main__": required=False, default=[], nargs="+", - help= - "Input a list of dir paths including files named testslist.csv and output CmakeLists.txt in these directories respectly" + help="Input a list of dir paths including files named testslist.csv and output CmakeLists.txt in these directories respectly", ) parser.add_argument( "--ignore-cmake-dirs", @@ -543,18 +596,19 @@ if __name__ == "__main__": required=False, default=[], nargs='*', - help= - "To keep dist ports the same with old version cmake, old cmakelists.txt files are needed to parse dist_ports. If a directories are newly created and there is no cmakelists.txt file, the directory path must be specified by this option. The dirs are not recursive." + help="To keep dist ports the same with old version cmake, old cmakelists.txt files are needed to parse dist_ports. If a directories are newly created and there is no cmakelists.txt file, the directory path must be specified by this option. The dirs are not recursive.", ) args = parser.parse_args() - assert not (len(args.files) == 0 and len(args.dirpaths) - == 0), "You must provide at leate one file or dirpath" + assert not ( + len(args.files) == 0 and len(args.dirpaths) == 0 + ), "You must provide at leate one file or dirpath" current_work_dirs = [] if len(args.files) >= 1: for p in args.files: - assert os.path.basename( - p) == "testslist.csv", "you must input file named testslist.csv" + assert ( + os.path.basename(p) == "testslist.csv" + ), "you must input file named testslist.csv" current_work_dirs = current_work_dirs + [ os.path.dirname(file) for file in args.files ] diff --git a/tools/get_op_list.py b/tools/get_op_list.py index 05e666d2270..a6d726dc309 100644 --- a/tools/get_op_list.py +++ b/tools/get_op_list.py @@ -24,10 +24,12 @@ paddle.enable_static() def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--model_dir', - type=str, - default="", - help='Directory of the inference models.') + parser.add_argument( + '--model_dir', + type=str, + default="", + help='Directory of the inference models.', + ) return parser.parse_args() @@ -58,7 +60,7 @@ if __name__ == '__main__': args = parse_args() for root, dirs, files in os.walk(args.model_dir, topdown=True): for name in files: - if (re.match(r'.*pdmodel', name)): + if re.match(r'.*pdmodel', name): ops_set = get_model_ops(os.path.join(root, name)) phi_set = get_model_phi_kernels(ops_set) ops = ";".join(ops_set) diff --git a/tools/get_pr_ut.py b/tools/get_pr_ut.py index efe21b06391..ba8dbeb1da5 100644 --- a/tools/get_pr_ut.py +++ b/tools/get_pr_ut.py @@ -31,7 +31,7 @@ ssl._create_default_https_context = ssl._create_unverified_context class PRChecker(object): - """ PR Checker. """ + """PR Checker.""" def __init__(self): self.github = Github(os.getenv('GITHUB_API_TOKEN'), timeout=60) @@ -47,7 +47,7 @@ class PRChecker(object): self.full_case = False def init(self): - """ Get pull request. """ + """Get pull request.""" pr_id = os.getenv('GIT_PR_ID') if not pr_id: print('PREC No PR ID') @@ -72,7 +72,7 @@ class PRChecker(object): print('PREC test=allcase is set') self.full_case = True - #todo: exception + # todo: exception def __wget_with_retry(self, url): ix = 1 proxy = '--no-proxy' @@ -86,12 +86,15 @@ class PRChecker(object): proxy = '--no-proxy' code = subprocess.call( 'wget -q {} --no-check-certificate {}'.format(proxy, url), - shell=True) + shell=True, + ) if code == 0: return True print( - 'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]' - .format(url, ix, ix * 10, proxy)) + 'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]'.format( + url, ix, ix * 10, proxy + ) + ) time.sleep(ix * 10) ix += 1 return False @@ -105,16 +108,19 @@ class PRChecker(object): cur_proxy = urllib.request.ProxyHandler(without_proxy) else: cur_proxy = urllib.request.ProxyHandler(with_proxy) - opener = urllib.request.build_opener(cur_proxy, - urllib.request.HTTPHandler) + opener = urllib.request.build_opener( + cur_proxy, urllib.request.HTTPHandler + ) urllib.request.install_opener(opener) try: urllib.request.urlretrieve(url, filename) except Exception as e: print(e) print( - 'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]' - .format(url, ix, ix * 10, cur_proxy)) + 'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]'.format( + url, ix, ix * 10, cur_proxy + ) + ) continue else: return True @@ -124,7 +130,7 @@ class PRChecker(object): return False def get_pr_files(self): - """ Get files in pull request. """ + """Get files in pull request.""" page = 0 file_dict = {} file_count = 0 @@ -135,20 +141,23 @@ class PRChecker(object): for f in files: file_dict[PADDLE_ROOT + f.filename] = f.status file_count += 1 - if file_count == 30: #if pr file count = 31, nend to run all case + if file_count == 30: # if pr file count = 31, nend to run all case break page += 1 print("pr modify files: %s" % file_dict) return file_dict def get_is_white_file(self, filename): - """ judge is white file in pr's files. """ + """judge is white file in pr's files.""" isWhiteFile = False - not_white_files = (PADDLE_ROOT + 'cmake/', PADDLE_ROOT + 'patches/', - PADDLE_ROOT + 'tools/dockerfile/', - PADDLE_ROOT + 'tools/windows/', - PADDLE_ROOT + 'tools/test_runner.py', - PADDLE_ROOT + 'tools/parallel_UT_rule.py') + not_white_files = ( + PADDLE_ROOT + 'cmake/', + PADDLE_ROOT + 'patches/', + PADDLE_ROOT + 'tools/dockerfile/', + PADDLE_ROOT + 'tools/windows/', + PADDLE_ROOT + 'tools/test_runner.py', + PADDLE_ROOT + 'tools/parallel_UT_rule.py', + ) if 'cmakelist' in filename.lower(): isWhiteFile = False elif filename.startswith((not_white_files)): @@ -162,13 +171,16 @@ class PRChecker(object): if filetype == 'py': result = self.__get_comment_by_prog(content, self.py_prog_oneline) result.extend( - self.__get_comment_by_prog(content, self.py_prog_multiline_a)) + self.__get_comment_by_prog(content, self.py_prog_multiline_a) + ) result.extend( - self.__get_comment_by_prog(content, self.py_prog_multiline_b)) + self.__get_comment_by_prog(content, self.py_prog_multiline_b) + ) if filetype == 'cc': result = self.__get_comment_by_prog(content, self.cc_prog_oneline) result.extend( - self.__get_comment_by_prog(content, self.cc_prog_multiline)) + self.__get_comment_by_prog(content, self.cc_prog_multiline) + ) return result def __get_comment_by_prog(self, content, prog): @@ -181,15 +193,15 @@ class PRChecker(object): return result def get_comment_of_file(self, f): - #content = self.repo.get_contents(f.replace(PADDLE_ROOT, ''), 'pull/').decoded_content - #todo: get file from github + # content = self.repo.get_contents(f.replace(PADDLE_ROOT, ''), 'pull/').decoded_content + # todo: get file from github with open(f, encoding="utf-8") as fd: lines = fd.readlines() lineno = 1 inputs = '' for line in lines: - #for line in content.split('\n'): - #input += str(lineno) + '|' + line + '\n' + # for line in content.split('\n'): + # input += str(lineno) + '|' + line + '\n' inputs += str(lineno) + '|' + line lineno += 1 fietype = '' @@ -227,8 +239,9 @@ class PRChecker(object): end += 1 if data[ix][0] == '+': line_list = file_to_diff_lines.get(filename) - line = '{}{}'.format(lineno, - data[ix].replace('+', '|', 1)) + line = '{}{}'.format( + lineno, data[ix].replace('+', '|', 1) + ) if line_list: line_list.append(line) else: @@ -254,9 +267,11 @@ class PRChecker(object): return True def get_all_count(self): - p = subprocess.Popen("cd {}build && ctest -N".format(PADDLE_ROOT), - shell=True, - stdout=subprocess.PIPE) + p = subprocess.Popen( + "cd {}build && ctest -N".format(PADDLE_ROOT), + shell=True, + stdout=subprocess.PIPE, + ) out, err = p.communicate() for line in out.splitlines(): if 'Total Tests:' in str(line): @@ -264,12 +279,13 @@ class PRChecker(object): return int(all_counts) def file_is_unnit_test(self, filename): - #get all testcases by ctest-N + # get all testcases by ctest-N all_ut_file = '%s/build/all_ut_file' % PADDLE_ROOT os.system( "cd %s/build && ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d' > %s" - % (PADDLE_ROOT, all_ut_file)) - #determine whether filename is in all_ut_case + % (PADDLE_ROOT, all_ut_file) + ) + # determine whether filename is in all_ut_case with open(all_ut_file, 'r') as f: (filepath, tempfilename) = os.path.split(filename) for f_file in f: @@ -279,7 +295,7 @@ class PRChecker(object): return False def get_pr_ut(self): - """ Get unit tests in pull request. """ + """Get unit tests in pull request.""" if self.full_case: return '' check_added_ut = False @@ -288,7 +304,8 @@ class PRChecker(object): ret = self.__urlretrieve( 'https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json', - 'ut_file_map.json') + 'ut_file_map.json', + ) if not ret: print('PREC download file_ut.json failed') exit(1) @@ -303,19 +320,23 @@ class PRChecker(object): filterFiles = [] file_list = [] file_dict = self.get_pr_files() - if len(file_dict) == 30: #if pr file count = 31, nend to run all case + if len(file_dict) == 30: # if pr file count = 31, nend to run all case return '' for filename in file_dict: if filename.startswith(PADDLE_ROOT + 'python/'): file_list.append(filename) elif filename.startswith(PADDLE_ROOT + 'paddle/'): - if filename.startswith((PADDLE_ROOT + 'paddle/infrt', - PADDLE_ROOT + 'paddle/utils')): + if filename.startswith( + (PADDLE_ROOT + 'paddle/infrt', PADDLE_ROOT + 'paddle/utils') + ): filterFiles.append(filename) elif filename.startswith(PADDLE_ROOT + 'paddle/scripts'): if filename.startswith( - (PADDLE_ROOT + 'paddle/scripts/paddle_build.sh', - PADDLE_ROOT + 'paddle/scripts/paddle_build.bat')): + ( + PADDLE_ROOT + 'paddle/scripts/paddle_build.sh', + PADDLE_ROOT + 'paddle/scripts/paddle_build.bat', + ) + ): file_list.append(filename) else: filterFiles.append(filename) @@ -334,7 +355,8 @@ class PRChecker(object): ut_list.append('filterfiles_placeholder') ret = self.__urlretrieve( 'https://paddle-docker-tar.bj.bcebos.com/pre_test/prec_delta', - 'prec_delta') + 'prec_delta', + ) if ret: with open('prec_delta') as delta: for ut in delta: @@ -343,17 +365,25 @@ class PRChecker(object): print('PREC download prec_delta failed') exit(1) PRECISION_TEST_Cases_ratio = format( - float(len(ut_list)) / float(self.get_all_count()), '.2f') + float(len(ut_list)) / float(self.get_all_count()), '.2f' + ) print("filterFiles: %s" % filterFiles) print("ipipe_log_param_PRECISION_TEST: true") - print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" % - len(ut_list)) - print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" % - PRECISION_TEST_Cases_ratio) + print( + "ipipe_log_param_PRECISION_TEST_Cases_count: %s" % len(ut_list) + ) + print( + "ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" + % PRECISION_TEST_Cases_ratio + ) return '\n'.join(ut_list) else: for f in file_list: - if current_system == "Darwin" or current_system == "Windows" or self.suffix == ".py3": + if ( + current_system == "Darwin" + or current_system == "Windows" + or self.suffix == ".py3" + ): f_judge = f.replace(PADDLE_ROOT, '/paddle/', 1) f_judge = f_judge.replace('//', '/') else: @@ -362,25 +392,34 @@ class PRChecker(object): if f_judge.endswith('.md'): ut_list.append('md_placeholder') onlyCommentsFilesOrXpu.append(f_judge) - elif 'tests/unittests/xpu' in f_judge or 'tests/unittests/npu' in f_judge or 'op_npu.cc' in f_judge: + elif ( + 'tests/unittests/xpu' in f_judge + or 'tests/unittests/npu' in f_judge + or 'op_npu.cc' in f_judge + ): ut_list.append('xpu_npu_placeholder') onlyCommentsFilesOrXpu.append(f_judge) elif f_judge.endswith(('.h', '.cu', '.cc', '.py')): - #determine whether the new added file is a member of added_ut + # determine whether the new added file is a member of added_ut if file_dict[f] in ['added']: f_judge_in_added_ut = False - with open('{}/added_ut'.format( - PADDLE_ROOT)) as utfile: - (filepath, - tempfilename) = os.path.split(f_judge) + with open( + '{}/added_ut'.format(PADDLE_ROOT) + ) as utfile: + (filepath, tempfilename) = os.path.split( + f_judge + ) for f_file in utfile: - if f_file.strip('\n') == tempfilename.split( - ".")[0]: + if ( + f_file.strip('\n') + == tempfilename.split(".")[0] + ): f_judge_in_added_ut = True if f_judge_in_added_ut == True: print( "Adding new unit tests not hit mapFiles: %s" - % f_judge) + % f_judge + ) else: notHitMapFiles.append(f_judge) elif file_dict[f] in ['removed']: @@ -394,9 +433,11 @@ class PRChecker(object): else: notHitMapFiles.append(f_judge) else: - notHitMapFiles.append( - f_judge) if file_dict[f] != 'removed' else print( - "remove file not hit mapFiles: %s" % f_judge) + notHitMapFiles.append(f_judge) if file_dict[ + f + ] != 'removed' else print( + "remove file not hit mapFiles: %s" % f_judge + ) else: if file_dict[f] not in ['removed']: if self.is_only_comment(f): @@ -420,7 +461,8 @@ class PRChecker(object): if ut_list: ret = self.__urlretrieve( 'https://paddle-docker-tar.bj.bcebos.com/pre_test/prec_delta', - 'prec_delta') + 'prec_delta', + ) if ret: with open('prec_delta') as delta: for ut in delta: @@ -430,13 +472,17 @@ class PRChecker(object): exit(1) print("hitMapFiles: %s" % hitMapFiles) print("ipipe_log_param_PRECISION_TEST: true") - print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" % - len(ut_list)) + print( + "ipipe_log_param_PRECISION_TEST_Cases_count: %s" + % len(ut_list) + ) PRECISION_TEST_Cases_ratio = format( - float(len(ut_list)) / float(self.get_all_count()), - '.2f') - print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" % - PRECISION_TEST_Cases_ratio) + float(len(ut_list)) / float(self.get_all_count()), '.2f' + ) + print( + "ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" + % PRECISION_TEST_Cases_ratio + ) if len(filterFiles) != 0: print("filterFiles: %s" % filterFiles) return '\n'.join(ut_list) diff --git a/tools/get_single_test_cov.py b/tools/get_single_test_cov.py index 9fe6554ee6e..293fed5e4b3 100644 --- a/tools/get_single_test_cov.py +++ b/tools/get_single_test_cov.py @@ -35,10 +35,16 @@ def getFNDAFile(rootPath, test): def analysisFNDAFile(rootPath, test): - related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % (rootPath, test, - test) + related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % ( + rootPath, + test, + test, + ) notrelated_ut_map_file = '%s/build/ut_map/%s/notrelated_%s.txt' % ( - rootPath, test, test) + rootPath, + test, + test, + ) os.system('touch %s' % related_ut_map_file) os.system('touch %s' % notrelated_ut_map_file) fn_filename = '%s/build/ut_map/%s/fnda.tmp' % (rootPath, test) @@ -60,22 +66,30 @@ def analysisFNDAFile(rootPath, test): fn = message_list[i] matchObj = re.match( r'(.*)Maker(.*)|(.*)Touch(.*)Regist(.*)|(.*)Touch(.*)JitKernel(.*)|(.*)converterC2Ev(.*)', - fn, re.I) + fn, + re.I, + ) if matchObj == None: OP_REGIST = False break if OP_REGIST == False: related_file_list.append(clazz_filename) - os.system('echo %s >> %s' % - (clazz_filename, related_ut_map_file)) + os.system( + 'echo %s >> %s' % (clazz_filename, related_ut_map_file) + ) else: - os.system('echo %s >> %s' % - (clazz_filename, notrelated_ut_map_file)) + os.system( + 'echo %s >> %s' % (clazz_filename, notrelated_ut_map_file) + ) else: if clazz_filename != '': - if clazz_filename not in related_file_list: # xx.pb.cc in RELATED xx.pb.h not in RELATED - os.system('echo %s >> %s' % - (clazz_filename, notrelated_ut_map_file)) + if ( + clazz_filename not in related_file_list + ): # xx.pb.cc in RELATED xx.pb.h not in RELATED + os.system( + 'echo %s >> %s' + % (clazz_filename, notrelated_ut_map_file) + ) f.close() @@ -83,10 +97,12 @@ def getCovinfo(rootPath, test): ut_map_path = '%s/build/ut_map/%s' % (rootPath, test) os.system( 'cd %s && lcov --capture -d . -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1' - % ut_map_path) + % ut_map_path + ) os.system( "cd %s && lcov --extract coverage.info '/paddle/paddle/fluid/framework/*' '/paddle/paddle/fluid/imperative/*' '/paddle/paddle/fluid/inference/*' '/paddle/paddle/fluid/memory/*' '/paddle/paddle/fluid/operators/*' '/paddle/paddle/fluid/string/*' '/paddle/paddle/fluid/distributed/*' '/paddle/paddle/fluid/platform/*' '/paddle/paddle/fluid/pybind/*' '/paddle/build/*' -o coverage.info.tmp --rc lcov_branch_coverage=0 > /dev/null 2>&1" - % ut_map_path) + % ut_map_path + ) os.system('rm -rf %s/paddle' % ut_map_path) os.system('rm -rf %s/coverage.info' % ut_map_path) getFNDAFile(rootPath, test) diff --git a/tools/get_ut_file_map.py b/tools/get_ut_file_map.py index 5afc6b03f2f..2bd777cc827 100644 --- a/tools/get_ut_file_map.py +++ b/tools/get_ut_file_map.py @@ -35,7 +35,8 @@ def get_all_uts(rootPath): all_uts_paddle = '%s/build/all_uts_paddle' % rootPath os.system( r'cd %s/build && ctest -N -V | grep -Ei "Test[ \t]+#" | grep -oEi "\w+$" > %s' - % (rootPath, all_uts_paddle)) + % (rootPath, all_uts_paddle) + ) def remove_useless_file(rootPath): @@ -75,10 +76,11 @@ def handle_ut_file_map(rootPath): continue elif line.startswith('/paddle/build'): source_file = line.replace('/build', '') - #source_file = re.sub('.pb.*', '.proto', source_file) + # source_file = re.sub('.pb.*', '.proto', source_file) elif 'precise test map fileeee:' in line: - source_file = line.split( - 'precise test map fileeee:')[1].strip() + source_file = line.split('precise test map fileeee:')[ + 1 + ].strip() else: source_file = line if source_file not in ut_file_map: @@ -193,7 +195,8 @@ def ut_file_map_supplement(rootPath): if not os.path.exists(filename) and ut not in prec_delta_new_list: prec_delta_new_list.append(ut) prec_delta_new_list.append( - 'test_py_reader_error_msg') #add a python case for pycoverage + 'test_py_reader_error_msg' + ) # add a python case for pycoverage prec_delta_file = open("/pre_test/prec_delta", 'w') for ut in prec_delta_new_list: prec_delta_file.write(ut + '\n') diff --git a/tools/get_ut_mem_map.py b/tools/get_ut_mem_map.py index dd1511a7cf2..703f0ab98ac 100644 --- a/tools/get_ut_mem_map.py +++ b/tools/get_ut_mem_map.py @@ -39,20 +39,30 @@ def get_ut_mem(rootPath): mem_reserved = round( float( line.split(' : Reserved = ')[1].split( - ', Allocated = ')[0]), 2) + ', Allocated = ' + )[0] + ), + 2, + ) if mem_reserved > mem_reserved1: mem_reserved1 = mem_reserved if 'MAX_GPU_MEMORY_USE=' in line: mem_nvidia = round( float( - line.split('MAX_GPU_MEMORY_USE=')[1].split('\\n') - [0].strip()), 2) + line.split('MAX_GPU_MEMORY_USE=')[1] + .split('\\n')[0] + .strip() + ), + 2, + ) if mem_nvidia > mem_nvidia1: mem_nvidia1 = mem_nvidia if 'Total Test time (real)' in line: caseTime = float( - line.split('Total Test time (real) =')[1].split('sec') - [0].strip()) + line.split('Total Test time (real) =')[1] + .split('sec')[0] + .strip() + ) if mem_reserved1 != -1: case_dic[ut]['mem_reserved'] = mem_reserved1 if mem_nvidia1 != -1: diff --git a/tools/group_case_for_parallel.py b/tools/group_case_for_parallel.py index e2c8eb9ff04..8dd5f6f4ca2 100644 --- a/tools/group_case_for_parallel.py +++ b/tools/group_case_for_parallel.py @@ -19,17 +19,22 @@ import sys def group_case_for_parallel(rootPath): """group cases""" - #wget file + # wget file for filename in [ - 'nightly_case', 'single_card_tests', 'single_card_tests_mem0', - 'multiple_card_tests', 'multiple_card_tests_mem0', - 'exclusive_card_tests', 'exclusive_card_tests_mem0' + 'nightly_case', + 'single_card_tests', + 'single_card_tests_mem0', + 'multiple_card_tests', + 'multiple_card_tests_mem0', + 'exclusive_card_tests', + 'exclusive_card_tests_mem0', ]: os.system( 'cd %s/tools && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test_bak/%s --no-check-certificate' - % (rootPath, filename)) + % (rootPath, filename) + ) - #get nightly tests + # get nightly tests nightly_tests_file = open('%s/tools/nightly_case' % rootPath, 'r') nightly_tests = nightly_tests_file.read().strip().split('\n') nightly_tests_file.close() @@ -40,7 +45,7 @@ def group_case_for_parallel(rootPath): '%s/tools/multiple_card_tests_mem0' % rootPath, '%s/tools/multiple_card_tests' % rootPath, '%s/tools/exclusive_card_tests_mem0' % rootPath, - '%s/tools/exclusive_card_tests' % rootPath + '%s/tools/exclusive_card_tests' % rootPath, ] case_file = '%s/build/ut_list' % rootPath if os.path.exists(case_file): @@ -67,12 +72,14 @@ def group_case_for_parallel(rootPath): for line in lines: case_line_list = line.replace('^', '').replace('|', '').split('$') new_case_line_list = list( - set(all_need_run_cases).intersection(set(case_line_list))) + set(all_need_run_cases).intersection(set(case_line_list)) + ) if len(new_case_line_list) != 0: new_case_file_list.append(new_case_line_list) all_group_case += new_case_line_list all_need_run_cases = list( - set(all_need_run_cases).difference(set(all_group_case))) + set(all_need_run_cases).difference(set(all_group_case)) + ) for line in new_case_file_list: cases = '$|^'.join(case for case in line) @@ -81,7 +88,7 @@ def group_case_for_parallel(rootPath): fi.close() new_f.close() - #no parallel cases + # no parallel cases cases = '^job' if len(all_need_run_cases) != 0: for case in all_need_run_cases: diff --git a/tools/handle_h_cu_file.py b/tools/handle_h_cu_file.py index 2dfa014cf95..84fba9f2820 100644 --- a/tools/handle_h_cu_file.py +++ b/tools/handle_h_cu_file.py @@ -31,9 +31,12 @@ def worker(fun): def threadPool(threadPoolNum): threadPool = [] for i in range(threadPoolNum): - thread = threading.Thread(target=worker, args={ - doFun, - }) + thread = threading.Thread( + target=worker, + args={ + doFun, + }, + ) thread.daemon = True threadPool.append(thread) return threadPool @@ -60,10 +63,12 @@ def insert_pile_to_h_file(rootPath): os.system('echo "\n#include \n" >> %s' % line) os.system( 'echo "__attribute__((constructor)) static void calledFirst%s()\n{" >> %s' - % (func, line)) + % (func, line) + ) os.system( 'echo \' printf("precise test map fileeee: %%s\\\\n", __FILE__);\n}\' >> %s' - % line) + % line + ) os.system('echo "\n#endif" >> %s' % line) @@ -87,7 +92,8 @@ def get_h_cu_file(file_path): ut = filename.replace('^', '').replace('$', '').replace('.log', '') os.system( "cat %s/%s | grep 'precise test map fileeee:'| uniq >> %s/build/ut_map/%s/related_%s.txt" - % (dir_path, filename, rootPath, ut, ut)) + % (dir_path, filename, rootPath, ut, ut) + ) def doFun(file_path): diff --git a/tools/infrt/fake_models/multi_fc.py b/tools/infrt/fake_models/multi_fc.py index 31b6efd8dcb..29b941d5437 100644 --- a/tools/infrt/fake_models/multi_fc.py +++ b/tools/infrt/fake_models/multi_fc.py @@ -24,18 +24,22 @@ paddle.enable_static() a = fluid.layers.data(name="A", shape=[-1, size], dtype='float32') label = fluid.layers.data(name="label", shape=[size], dtype='float32') -fc_out = fluid.layers.fc(input=a, - size=size, - act="relu", - bias_attr=fluid.ParamAttr(name="fc_bias"), - num_flatten_dims=1) +fc_out = fluid.layers.fc( + input=a, + size=size, + act="relu", + bias_attr=fluid.ParamAttr(name="fc_bias"), + num_flatten_dims=1, +) for i in range(num_layers - 1): - fc_out = fluid.layers.fc(input=fc_out, - size=size, - act="relu", - bias_attr=fluid.ParamAttr(name="fc_bias"), - num_flatten_dims=1) + fc_out = fluid.layers.fc( + input=fc_out, + size=size, + act="relu", + bias_attr=fluid.ParamAttr(name="fc_bias"), + num_flatten_dims=1, + ) cost = fluid.layers.square_error_cost(fc_out, label) avg_cost = fluid.layers.mean(cost) @@ -49,7 +53,14 @@ loss = exe = fluid.Executor(cpu) exe.run(fluid.default_startup_program()) fluid.io.save_inference_model("./multi_fc_model", [a.name], [fc_out], exe) -fluid.io.save_inference_model("./multi_fc_model", [a.name], [fc_out], exe, None, - "fc.pdmodel", "fc.pdiparams") +fluid.io.save_inference_model( + "./multi_fc_model", + [a.name], + [fc_out], + exe, + None, + "fc.pdmodel", + "fc.pdiparams", +) print('output name', fc_out.name) diff --git a/tools/infrt/generate_pd_op_dialect_from_paddle_op_maker.py b/tools/infrt/generate_pd_op_dialect_from_paddle_op_maker.py index 7b97a06619e..313d844d6f6 100644 --- a/tools/infrt/generate_pd_op_dialect_from_paddle_op_maker.py +++ b/tools/infrt/generate_pd_op_dialect_from_paddle_op_maker.py @@ -82,8 +82,11 @@ def get_attrs_info(op_proto, op_attrs_proto): attrs_info[attr_name] = {} attrs_info[attr_name][TYPE] = attr_proto.type attrs_info[attr_name][GENERATED] = attr_proto.generated - attrs_info[attr_name][DEFAULT_VALUE] = attrs_default_values[ - attr_name] if attr_name in attrs_default_values else None + attrs_info[attr_name][DEFAULT_VALUE] = ( + attrs_default_values[attr_name] + if attr_name in attrs_default_values + else None + ) attrs_info[attr_name][EXTRA] = attr_proto.extra attrs_info[attr_name][QUANT] = attr_proto.quant return attrs_info @@ -118,20 +121,26 @@ def generate_all_ops_inputs_outputs_map(op_descs): inputs = list() outpus = list() for input_ in op_proto[INPUTS]: - if op_proto[INPUTS][input_][EXTRA] != True and op_proto[INPUTS][ - input_][INTERMEDIATE] != True: + if ( + op_proto[INPUTS][input_][EXTRA] != True + and op_proto[INPUTS][input_][INTERMEDIATE] != True + ): inputs.append(input_) for output_ in op_proto[OUTPUTS]: - if op_proto[OUTPUTS][output_][EXTRA] != True and op_proto[OUTPUTS][ - output_][INTERMEDIATE] != True: + if ( + op_proto[OUTPUTS][output_][EXTRA] != True + and op_proto[OUTPUTS][output_][INTERMEDIATE] != True + ): outpus.append(output_) ops_inputs_map[op_type] = inputs ops_outputs_map[op_type] = outpus # 2. Generate Cpp style map str cpp_style_ops_inputs_map_str = "" - start_ = "#include \n#include \n#include \n" + \ - "const std::unordered_map> pd_dialect_inputs_info_map_ = {\n" + start_ = ( + "#include \n#include \n#include \n" + + "const std::unordered_map> pd_dialect_inputs_info_map_ = {\n" + ) ops_inputs_str = "" for ele in ops_inputs_map.items(): op_name = ele[0] @@ -139,11 +148,14 @@ def generate_all_ops_inputs_outputs_map(op_descs): op_inputs_str = "{" input_idx = 0 for op_input in op_inputs: - op_input_str = '{left_brace}"{op_input}", {input_idx}{right_brace}, '.format( - left_brace="{", - op_input=op_input, - input_idx=input_idx, - right_brace="}") + op_input_str = ( + '{left_brace}"{op_input}", {input_idx}{right_brace}, '.format( + left_brace="{", + op_input=op_input, + input_idx=input_idx, + right_brace="}", + ) + ) input_idx = input_idx + 1 op_inputs_str = op_inputs_str + op_input_str op_inputs_str = op_inputs_str[:-2] + "}" @@ -151,7 +163,8 @@ def generate_all_ops_inputs_outputs_map(op_descs): left_brace="{", op_name=op_name, op_inputs=op_inputs_str, - right_brace="}") + right_brace="}", + ) ops_inputs_str = ops_inputs_str + " " + pair ops_inputs_str = ops_inputs_str[:-2] cpp_style_ops_inputs_map_str = start_ + ops_inputs_str + "\n};" @@ -165,11 +178,14 @@ def generate_all_ops_inputs_outputs_map(op_descs): op_outputs_str = "{" output_idx = 0 for op_output in op_outputs: - op_output_str = '{left_brace}"{op_output}", {output_idx}{right_brace}, '.format( - left_brace="{", - op_output=op_output, - output_idx=output_idx, - right_brace="}") + op_output_str = ( + '{left_brace}"{op_output}", {output_idx}{right_brace}, '.format( + left_brace="{", + op_output=op_output, + output_idx=output_idx, + right_brace="}", + ) + ) output_idx = output_idx + 1 op_outputs_str = op_outputs_str + op_output_str op_outputs_str = op_outputs_str[:-2] + "}" @@ -177,7 +193,8 @@ def generate_all_ops_inputs_outputs_map(op_descs): left_brace="{", op_name=op_name, op_outputs=op_outputs_str, - right_brace="}") + right_brace="}", + ) ops_outputs_str = ops_outputs_str + " " + pair ops_outputs_str = ops_outputs_str[:-2] cpp_style_ops_outputs_map_str = start_ + ops_outputs_str + "\n};" @@ -196,9 +213,11 @@ def get_constraint(op_type, op_proto): optional_input_num_ = 0 for input_ in op_proto[INPUTS]: - if op_proto[INPUTS][input_][EXTRA] != True and op_proto[INPUTS][input_][ - INTERMEDIATE] != True and op_proto[INPUTS][input_][ - DISPENSABLE] == True: + if ( + op_proto[INPUTS][input_][EXTRA] != True + and op_proto[INPUTS][input_][INTERMEDIATE] != True + and op_proto[INPUTS][input_][DISPENSABLE] == True + ): optional_input_num_ += 1 if optional_input_num_ > 1: constraint += ", AttrSizedOperandSegments" @@ -237,11 +256,21 @@ def convert_op_proto_into_mlir(op_descs): # 2. Op dialect # skip list ( ops whose dialect can not be generated automatically will be recorded here) skipped_op_list = [ - "cos_sim", "fused_embedding_seq_pool", "cosh", "kron", "recurrent", - "while", "conditional_block", "set_value", "run_program" + "cos_sim", + "fused_embedding_seq_pool", + "cosh", + "kron", + "recurrent", + "while", + "conditional_block", + "set_value", + "run_program", ] skipped_attr_list = [ - "trainable_statistics", "use_global_stats", "is_test", "use_quantizer" + "trainable_statistics", + "use_global_stats", + "is_test", + "use_quantizer", ] original_ops_ = get_original_ops() @@ -256,7 +285,8 @@ def convert_op_proto_into_mlir(op_descs): op_type_capitalize=op_type.capitalize(), constraint=constraint_, op_type=op_type, - left_brace="{") + left_brace="{", + ) SUMMARY = ' let summary = "{} op";\n'.format(op_type) # 2.2 Description @@ -265,27 +295,44 @@ def convert_op_proto_into_mlir(op_descs): for line_ in origin_contents: contents = contents + " {}\n".format(line_) DESCRIPTION = " let description = [{left_brace}\n{description} {right_brace}];\n".format( - left_brace="{", description=contents, right_brace="}") + left_brace="{", description=contents, right_brace="}" + ) # 2.3 arguments info ARGUMENTS = "" - if (len(op_proto[INPUTS]) > 0 or len(op_proto[ATTRS]) > 0): + if len(op_proto[INPUTS]) > 0 or len(op_proto[ATTRS]) > 0: ARGUMENTS = " let arguments = (ins " # 2.3.1 inputs for input_ in op_proto[INPUTS]: - if op_proto[INPUTS][input_][EXTRA] != True and op_proto[INPUTS][ - input_][INTERMEDIATE] != True: + if ( + op_proto[INPUTS][input_][EXTRA] != True + and op_proto[INPUTS][input_][INTERMEDIATE] != True + ): if op_proto[INPUTS][input_][DISPENSABLE] != True: if op_proto[INPUTS][input_][DUPLICABLE] != True: - ARGUMENTS = ARGUMENTS + " PD_Tensor:$" + input_ + "," + ARGUMENTS = ( + ARGUMENTS + " PD_Tensor:$" + input_ + "," + ) else: - ARGUMENTS = ARGUMENTS + " PD_Tensor_Array:$" + input_ + "," + ARGUMENTS = ( + ARGUMENTS + " PD_Tensor_Array:$" + input_ + "," + ) else: if op_proto[INPUTS][input_][DUPLICABLE] != True: - ARGUMENTS = ARGUMENTS + " Optional:$" + input_ + "," + ARGUMENTS = ( + ARGUMENTS + + " Optional:$" + + input_ + + "," + ) else: - ARGUMENTS = ARGUMENTS + " Optional:$" + input_ + "," + ARGUMENTS = ( + ARGUMENTS + + " Optional:$" + + input_ + + "," + ) # unsupported: BLOCK = 8; BLOCKS = 10; attr_mlir_converter = { @@ -298,70 +345,104 @@ def convert_op_proto_into_mlir(op_descs): 6: 'BoolAttr', 7: 'BoolArrayAttr', 9: 'SI64Attr', - 11: 'I64ArrayAttr' + 11: 'I64ArrayAttr', } # 2.3.2 attributes for attr in op_proto[ATTRS]: - if (op_proto[ATTRS][attr][EXTRA] - == True) or (attr in skipped_attr_list): + if (op_proto[ATTRS][attr][EXTRA] == True) or ( + attr in skipped_attr_list + ): continue if op_proto[ATTRS][attr][DEFAULT_VALUE] != None: if op_proto[ATTRS][attr][TYPE] in attr_mlir_converter: default_value = str( - op_proto[ATTRS][attr][DEFAULT_VALUE]) - if (attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] - in [ - 'I32ArrayAttr', 'F32ArrayAttr', - 'StrArrayAttr', 'BoolArrayAttr', - 'I64ArrayAttr' - ]): - default_value = default_value.replace('[', - '{').replace( - ']', '}') - if (attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] - in ['BoolAttr', 'BoolArrayAttr']): + op_proto[ATTRS][attr][DEFAULT_VALUE] + ) + if attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] in [ + 'I32ArrayAttr', + 'F32ArrayAttr', + 'StrArrayAttr', + 'BoolArrayAttr', + 'I64ArrayAttr', + ]: + default_value = default_value.replace( + '[', '{' + ).replace(']', '}') + if attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] in [ + 'BoolAttr', + 'BoolArrayAttr', + ]: default_value = default_value.lower() - elif (attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] - in ['StrAttr', 'StrArrayAttr']): + elif attr_mlir_converter[ + op_proto[ATTRS][attr][TYPE] + ] in ['StrAttr', 'StrArrayAttr']: default_value = default_value.replace('\'', '\\\"') - if attr_mlir_converter[op_proto[ATTRS][attr] - [TYPE]] == "StrAttr": + if ( + attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] + == "StrAttr" + ): default_value = '\\\"' + default_value + '\\\"' - attr_list = " DefaultValuedAttr<" + attr_mlir_converter[ - op_proto[ATTRS][attr] - [TYPE]] + ", \"" + default_value + "\">:$" + attr + "," + attr_list = ( + " DefaultValuedAttr<" + + attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] + + ", \"" + + default_value + + "\">:$" + + attr + + "," + ) ARGUMENTS += attr_list else: - print("Error:" + op_type + ":" + attr + ":" + - str(op_proto[ATTRS][attr][TYPE])) + print( + "Error:" + + op_type + + ":" + + attr + + ":" + + str(op_proto[ATTRS][attr][TYPE]) + ) else: if op_proto[ATTRS][attr][TYPE] in attr_mlir_converter: - attr_type_ = attr_mlir_converter[op_proto[ATTRS][attr] - [TYPE]] - if (attr_type_ in [ - 'StrAttr', 'I32ArrayAttr', 'F32ArrayAttr', - 'StrArrayAttr', 'BoolArrayAttr', 'I64ArrayAttr' - ]): + attr_type_ = attr_mlir_converter[ + op_proto[ATTRS][attr][TYPE] + ] + if attr_type_ in [ + 'StrAttr', + 'I32ArrayAttr', + 'F32ArrayAttr', + 'StrArrayAttr', + 'BoolArrayAttr', + 'I64ArrayAttr', + ]: attr_list = attr_type_ + ":$" + attr + "," ARGUMENTS += attr_list else: - print(" ouch Error:" + op_type + ":" + attr + ":" + - str(op_proto[ATTRS][attr][TYPE])) + print( + " ouch Error:" + + op_type + + ":" + + attr + + ":" + + str(op_proto[ATTRS][attr][TYPE]) + ) ARGUMENTS = ARGUMENTS[:-1] + ");\n" # 2.4 results info RESULTS = "" - if (len(op_proto[OUTPUTS]) > 0): + if len(op_proto[OUTPUTS]) > 0: outputs = "" for output_ in op_proto[OUTPUTS]: - if op_proto[OUTPUTS][output_][EXTRA] != True and op_proto[ - OUTPUTS][output_][INTERMEDIATE] != True: + if ( + op_proto[OUTPUTS][output_][EXTRA] != True + and op_proto[OUTPUTS][output_][INTERMEDIATE] != True + ): if op_proto[OUTPUTS][output_][DUPLICABLE] != True: outputs = outputs + "PD_Tensor:${},".format(output_) else: outputs = outputs + "PD_Tensor_Array:${},".format( - output_) + output_ + ) RESULTS = "\n let results = (outs {});\n".format(outputs[:-1]) with open(dst_dialect_file, 'a') as ops_mlir_file: @@ -376,8 +457,10 @@ def convert_op_proto_into_mlir(op_descs): ops_mlir_file.write("\n#endif // PD_OPS") print("Skipped ops num: " + str(len(skipped_op_list))) - print("Automatically generated op dialects num: " + - str(len(automatically_generated_op_dialect))) + print( + "Automatically generated op dialects num: " + + str(len(automatically_generated_op_dialect)) + ) if __name__ == "__main__": diff --git a/tools/infrt/generate_phi_kernel_dialect.py b/tools/infrt/generate_phi_kernel_dialect.py index 686a5e12a6c..a5d1003dbb9 100644 --- a/tools/infrt/generate_phi_kernel_dialect.py +++ b/tools/infrt/generate_phi_kernel_dialect.py @@ -17,21 +17,21 @@ import yaml import os from get_compat_kernel_signature import get_compat_kernels_info -#TODO @DannyIsFunny: more attr types need to be supported. +# TODO @DannyIsFunny: more attr types need to be supported. attr_type_converter = { "int": 'SI32Attr', "bool": 'BoolAttr', "int64_t": 'SI64Attr', "float": 'F32Attr', "string": 'StrAttr', - "vector": 'I32ArrayAttr' + "vector": 'I32ArrayAttr', } target_type_converter = {"CPU": "CPU", "GPU": "GPU", "Undefined": "UNK"} layout_type_converter = { "NCHW": "NCHW", "NHWC": "NHWC", - "Undefined(AnyLayout)": "ANY" + "Undefined(AnyLayout)": "ANY", } precision_type_converter = { "uint8": "UINT8", @@ -46,7 +46,7 @@ precision_type_converter = { "complex64": "COMPLEX64", "complex128": "COMPLEX128", "bool": "BOOL", - "Undefined": "UNK" + "Undefined": "UNK", } kernel_types_info_file = "./kernels.json" @@ -88,16 +88,19 @@ def generate_kernel_name(op_name, place_str): layout_ = layout_type_converter[layout_.strip()] precision_ = precision_type_converter[precision_.strip()] class_name_ = "{}{}".format( - op_name.replace("_", "").title(), "".join([ - target_.strip().title(), - precision_.strip(), - layout_.strip().title().title() - ])) + op_name.replace("_", "").title(), + "".join( + [ + target_.strip().title(), + precision_.strip(), + layout_.strip().title().title(), + ] + ), + ) alias_ = "{}.{}".format( op_name, - ".".join([target_.strip(), - precision_.strip(), - layout_.strip()])) + ".".join([target_.strip(), precision_.strip(), layout_.strip()]), + ) return alias_, class_name_ @@ -111,8 +114,9 @@ def generate_attrs_info(op_name, attrs_info): for index in range(len(attrs_info)): attr_name = kernel_attrs_names[op_name]["attrs"][index] attr_type = attr_type_converter[attrs_info[index]] - attrs_args_ += '{type_}:${name_},'.format(type_=attr_type, - name_=attr_name) + attrs_args_ += '{type_}:${name_},'.format( + type_=attr_type, name_=attr_name + ) return attrs_args_[:-1] @@ -125,7 +129,8 @@ def generate_inputs_info(input_info): layout_ = layout_type_converter[layout_.strip()] precision_ = precision_type_converter[precision_.strip()] input_args_ += " DenseTensor<\"{}\",\"{}\",\"{}\">:$in{},".format( - target_.strip(), precision_.strip(), layout_.strip(), str(index)) + target_.strip(), precision_.strip(), layout_.strip(), str(index) + ) input_args_ = input_args_[:-1] return input_args_ @@ -134,12 +139,13 @@ def generate_arguments_info(op_name, input_info, attr_info): input_args = generate_inputs_info(input_info) attr_args = generate_attrs_info(op_name, attr_info) context_args = "Context:$dev_ctx" - argument_list = [context_args - ] + input_args.split(",") + attr_args.split(",") - while ("" in argument_list): + argument_list = ( + [context_args] + input_args.split(",") + attr_args.split(",") + ) + while "" in argument_list: argument_list.remove("") argument_ = ",".join(argument_list) - return (("let arguments = (ins {});".format(argument_.strip(",")))) + return "let arguments = (ins {});".format(argument_.strip(",")) def generate_results_info(output_info): @@ -151,8 +157,9 @@ def generate_results_info(output_info): layout_ = layout_type_converter[layout_.strip()] precision_ = precision_type_converter[precision_.strip()] output_args_ += " DenseTensor<\"{}\",\"{}\",\"{}\">:$out{},".format( - target_.strip(), precision_.strip(), layout_.strip(), str(index)) - return ("{});".format(output_args_[:-1])) + target_.strip(), precision_.strip(), layout_.strip(), str(index) + ) + return "{});".format(output_args_[:-1]) def generate_supported_kernel_list(load_dict): @@ -205,11 +212,13 @@ def generate_cpu_kernel_dialect(op_name, kernel_alias_, kernel_info): alias, class_name = generate_kernel_name(op_name, kernel_alias_) summary = 'let summary = "{name}";'.format(name=alias) dialect_name = alias.split(".") - dialect_name = dialect_name[0] + "." + dialect_name[2] + "." + dialect_name[ - 3] + dialect_name = ( + dialect_name[0] + "." + dialect_name[2] + "." + dialect_name[3] + ) header = 'def {kernel_name} : PDTCPU_Kernel<"{name}",[NoSideEffect]> {left_brace}'.format( - kernel_name=class_name, name=dialect_name.lower(), left_brace="{") + kernel_name=class_name, name=dialect_name.lower(), left_brace="{" + ) inputs_ = kernel_info["input"] attributes = kernel_info["attribute"] @@ -223,7 +232,8 @@ def generate_cpu_kernel_dialect(op_name, kernel_alias_, kernel_info): summary_=summary, arguments_=arguments, results_=results, - right_brace="}") + right_brace="}", + ) return kernel_dialect @@ -232,11 +242,13 @@ def generate_gpu_kernel_dialect(op_name, kernel_alias_, kernel_info): alias, class_name = generate_kernel_name(op_name, kernel_alias_) summary = 'let summary = "{name}";'.format(name=alias) dialect_name = alias.split(".") - dialect_name = dialect_name[0] + "." + dialect_name[2] + "." + dialect_name[ - 3] + dialect_name = ( + dialect_name[0] + "." + dialect_name[2] + "." + dialect_name[3] + ) header = 'def {kernel_name} : PDTGPU_Kernel<"{name}",[NoSideEffect]> {left_brace}'.format( - kernel_name=class_name, name=dialect_name.lower(), left_brace="{") + kernel_name=class_name, name=dialect_name.lower(), left_brace="{" + ) inputs_ = kernel_info["input"] attributes = kernel_info["attribute"] arguments = generate_arguments_info(op_name, inputs_, attributes) @@ -249,7 +261,8 @@ def generate_gpu_kernel_dialect(op_name, kernel_alias_, kernel_info): summary_=summary, arguments_=arguments, results_=results, - right_brace="}") + right_brace="}", + ) return kernel_dialect @@ -270,7 +283,7 @@ include \"mlir/Interfaces/LoopLikeInterface.td\"\n\ include \"mlir/IR/OpBase.td\"\n\ include \"paddle/infrt/dialect/phi/ir/infrt_phi_kernel.td\"" - return (comment_ + includes_) + return comment_ + includes_ def get_kernel_target(kernel_alias_): @@ -298,33 +311,52 @@ def main(): for kernel_alias_ in kernel_info: if get_kernel_target(kernel_alias_) == "CPU": kernel_registry = generate_cpu_kernel_dialect( - op_name, kernel_alias_, kernel_info[kernel_alias_]) + op_name, kernel_alias_, kernel_info[kernel_alias_] + ) cpu_registry_ += kernel_registry elif get_kernel_target(kernel_alias_) == "GPU": kernel_registry = generate_gpu_kernel_dialect( - op_name, kernel_alias_, kernel_info[kernel_alias_]) + op_name, kernel_alias_, kernel_info[kernel_alias_] + ) gpu_registry_ += kernel_registry else: - print("Unsupported backend:" + - get_kernel_target(kernel_alias_)) + print( + "Unsupported backend:" + + get_kernel_target(kernel_alias_) + ) end = "#endif // PTEN_KERNELS" - with open("../../paddle/infrt/dialect/phi/ir/phi_cpu_kernels.td", - "w") as dst: - dst.write('{start_}\n{dialect_}\n{end_}'.format( - start_=head, dialect_=cpu_registry_, end_=end)) - with open("../../paddle/infrt/dialect/phi/ir/phi_gpu_kernels.td", - "w") as dst: - dst.write('{start_}\n{dialect_}\n{end_}'.format( - start_=head, dialect_=gpu_registry_, end_=end)) + with open( + "../../paddle/infrt/dialect/phi/ir/phi_cpu_kernels.td", "w" + ) as dst: + dst.write( + '{start_}\n{dialect_}\n{end_}'.format( + start_=head, dialect_=cpu_registry_, end_=end + ) + ) + with open( + "../../paddle/infrt/dialect/phi/ir/phi_gpu_kernels.td", "w" + ) as dst: + dst.write( + '{start_}\n{dialect_}\n{end_}'.format( + start_=head, dialect_=gpu_registry_, end_=end + ) + ) if __name__ == '__main__': if not os.path.exists(kernel_types_info_file): - print("Error: '{file_name}' not exist!".format( - file_name=kernel_types_info_file)) + print( + "Error: '{file_name}' not exist!".format( + file_name=kernel_types_info_file + ) + ) if not os.path.exists(kernel_signature_info_file): - print("Error: '{file_name}' not exist!".format( - file_name=kernel_signature_info_file)) + print( + "Error: '{file_name}' not exist!".format( + file_name=kernel_signature_info_file + ) + ) if os.path.exists(kernel_types_info_file) and os.path.exists( - kernel_signature_info_file): + kernel_signature_info_file + ): main() diff --git a/tools/infrt/get_compat_kernel_signature.py b/tools/infrt/get_compat_kernel_signature.py index 104d3ae30c2..f5f9afdafd1 100644 --- a/tools/infrt/get_compat_kernel_signature.py +++ b/tools/infrt/get_compat_kernel_signature.py @@ -56,16 +56,21 @@ def get_compat_kernels_info(): content = "" registry = False for line in txt: - if ("KernelSignature(" in line): + if "KernelSignature(" in line: content = "" registry = True - if (registry): + if registry: content += line - if (registry and ";" in line): - data = content.replace("\n", "").replace( - " ", - "").strip("return").strip("KernelSignature(").strip( - r"\);").replace("\"", "").replace("\\", "") + if registry and ";" in line: + data = ( + content.replace("\n", "") + .replace(" ", "") + .strip("return") + .strip("KernelSignature(") + .strip(r"\);") + .replace("\"", "") + .replace("\\", "") + ) registry = False if is_grad_kernel(data): continue @@ -74,14 +79,23 @@ def get_compat_kernels_info(): if name in kernels_info: cur_reg = kernels_info[name] kernels_info[name]["inputs"] = list( - set(registry_info["inputs"] + - kernels_info[name]["inputs"])) + set( + registry_info["inputs"] + + kernels_info[name]["inputs"] + ) + ) kernels_info[name]["attrs"] = list( - set(registry_info["attrs"] + - kernels_info[name]["attrs"])) + set( + registry_info["attrs"] + + kernels_info[name]["attrs"] + ) + ) kernels_info[name]["outputs"] = list( - set(registry_info["outputs"] + - kernels_info[name]["outputs"])) + set( + registry_info["outputs"] + + kernels_info[name]["outputs"] + ) + ) else: kernels_info[name] = registry_info diff --git a/tools/infrt/get_phi_kernel_info.py b/tools/infrt/get_phi_kernel_info.py index af73432a797..9eeee88276d 100644 --- a/tools/infrt/get_phi_kernel_info.py +++ b/tools/infrt/get_phi_kernel_info.py @@ -47,30 +47,34 @@ def get_skipped_kernel_list(): def parse_args(): parser = argparse.ArgumentParser("gather phi kernel and infermate info") - parser.add_argument("--paddle_root_path", - type=str, - required=True, - help="root path of paddle src[WORK_PATH/Paddle].") + parser.add_argument( + "--paddle_root_path", + type=str, + required=True, + help="root path of paddle src[WORK_PATH/Paddle].", + ) parser.add_argument( "--kernel_info_file", type=str, required=True, - help="kernel info file generated by get_phi_kernel_function.sh.") - parser.add_argument("--infermeta_wrap_file", - type=str, - required=True, - help="inferMeta wrap info file.") - parser.add_argument("--attr_info_file", - type=str, - required=True, - help="attr info file.") + help="kernel info file generated by get_phi_kernel_function.sh.", + ) + parser.add_argument( + "--infermeta_wrap_file", + type=str, + required=True, + help="inferMeta wrap info file.", + ) + parser.add_argument( + "--attr_info_file", type=str, required=True, help="attr info file." + ) parser.add_argument( "--generate_file", type=str, required=True, - default= - "../paddle/infrt/kernel/phi/infershaped/infershaped_kernel_launchers.cc", - help="generated file.") + default="../paddle/infrt/kernel/phi/infershaped/infershaped_kernel_launchers.cc", + help="generated file.", + ) args = parser.parse_args() return args @@ -167,15 +171,18 @@ def gen_include_headers(): def gen_namespace(): - return (""" + return ( + """ namespace infrt { namespace kernel { -""", """ +""", + """ } // namespace kernel } // namespace infrt -""") +""", + ) def gen_context(val): @@ -203,7 +210,7 @@ def gen_kernel_func(val, ctx_name, dtype_name): st = val.index('<') ed = val.index('>') func_name = val[:st] - template_name = val[st + 1:ed] + template_name = val[st + 1 : ed] if '::phi::' in template_name: return "&::phi::" + val else: @@ -273,7 +280,7 @@ def gen_register_code_info(item: List[str], attr_data: Dict[str, List[str]]): attr_data: {'phi_cpu.arg_min.float32.any': ['axisBool', 'keepdimsBool', 'flatten', 'dtype']} """ ctx_name, ir_ctx_name = gen_context(item[1]) - if (ctx_name == ""): + if ctx_name == "": return "" item[2] = gen_layout(item[2]) ir_dtypes, origin_dtypes = gen_dtype(item[4:-1]) @@ -290,11 +297,19 @@ def gen_register_code_info(item: List[str], attr_data: Dict[str, List[str]]): kernel_func = gen_kernel_func(item[3], ctx_name, origin_dtype) if item[0].lower() in skipped_kernel_list: continue - ir_name = ir_ctx_name + '.' + item[0].lower( - ) + '.' + ir_dtype + '.' + item[2].lower() + ir_name = ( + ir_ctx_name + + '.' + + item[0].lower() + + '.' + + ir_dtype + + '.' + + item[2].lower() + ) if ir_name in attr_data.keys() and attr_data[ir_name] is not None: attr_names = ', '.join( - ["\"" + a + "\"" for a in attr_data[ir_name]]) + ["\"" + a + "\"" for a in attr_data[ir_name]] + ) res += f""" registry->AddKernel("{ir_name}",""" @@ -320,8 +335,9 @@ registry->AddKernel("{ir_name}",""" return res -def gen_register_info(resources: List[List[str]], attr_data: Dict[str, - List[str]]): +def gen_register_info( + resources: List[List[str]], attr_data: Dict[str, List[str]] +): """ resources: [['add', 'CPU', 'ALL_LAYOUT', 'AddKernel', 'float', 'double', '...'(varaidic types), 'ElementwiseInferMeta'], ...] attr_data: {'phi_cpu.arg_min.float32.any': ['axisBool', 'keepdimsBool', 'flatten', 'dtype']} @@ -335,7 +351,7 @@ def gen_register_info(resources: List[List[str]], attr_data: Dict[str, if update_item[1] != "CPU": continue code = gen_register_code_info(item, attr_data) - if (code == ""): + if code == "": continue res += code @@ -347,7 +363,7 @@ def gen_register_info(resources: List[List[str]], attr_data: Dict[str, if update_item[1] != "GPU": continue code = gen_register_code_info(item, attr_data) - if (code == ""): + if code == "": continue res += code res += "#endif // INFRT_WITH_GPU" @@ -356,9 +372,11 @@ def gen_register_info(resources: List[List[str]], attr_data: Dict[str, return res -def gen_phi_kernel_register_code(resources: List[List[str]], - attr_data: Dict[str, List[str]], - src_file_path: str): +def gen_phi_kernel_register_code( + resources: List[List[str]], + attr_data: Dict[str, List[str]], + src_file_path: str, +): source_file = open(src_file_path, 'w') source_file.write(gen_warn_info()) source_file.write(gen_include_headers()) @@ -371,7 +389,9 @@ def gen_phi_kernel_register_code(resources: List[List[str]], if __name__ == "__main__": args = parse_args() - skipped_phi_api_list_file = args.paddle_root_path + skipped_phi_api_list_file + skipped_phi_api_list_file = ( + args.paddle_root_path + skipped_phi_api_list_file + ) api_yaml_file = args.paddle_root_path + api_yaml_file legacy_api_yaml_file = args.paddle_root_path + legacy_api_yaml_file infer_meta_data = [] diff --git a/tools/infrt/print_kernel_pass_info.py b/tools/infrt/print_kernel_pass_info.py index 1e84f3de86f..1a48708b71b 100644 --- a/tools/infrt/print_kernel_pass_info.py +++ b/tools/infrt/print_kernel_pass_info.py @@ -45,7 +45,8 @@ def get_compat_kernels_info(register): is_macro_defination = False for line in txt: if line.strip().startswith( - "#define") and line.strip().endswith("\\"): + "#define" + ) and line.strip().endswith("\\"): is_macro_defination = True continue if is_macro_defination: @@ -53,14 +54,18 @@ def get_compat_kernels_info(register): is_macro_defination = False continue - if (register in line): + if register in line: content = "" registry = True - if (registry): + if registry: content += line - if (registry and ";" in line): - kernel_name = content.replace("\n", "").replace( - " ", "").strip(register).split(",") + if registry and ";" in line: + kernel_name = ( + content.replace("\n", "") + .replace(" ", "") + .strip(register) + .split(",") + ) registry = False kernel_names.append(kernel_name[0]) return remove_grad_kernel(kernel_names) @@ -68,8 +73,9 @@ def get_compat_kernels_info(register): def show_kernel_statistics(backend, kernels): print("=== kernels statistics === ") - print("the number of " + backend + " kernels is: " + str(len(kernels)) + - "\n") + print( + "the number of " + backend + " kernels is: " + str(len(kernels)) + "\n" + ) print(kernels) print("\n") @@ -92,7 +98,7 @@ def get_passes_info(register): continue if register in line: registry_fun_found = True - if (registry_fun_found): + if registry_fun_found: pass_registry_func += line if registry_fun_found: for char in line: diff --git a/tools/jetson_infer_op.py b/tools/jetson_infer_op.py index e132a14373e..664f2b6616e 100644 --- a/tools/jetson_infer_op.py +++ b/tools/jetson_infer_op.py @@ -37,7 +37,7 @@ black_list = [ 'test_sync_batch_norm_op', # case too large 'test_reduce_op', - 'test_transpose_op' + 'test_transpose_op', ] op_diff_list = [ @@ -51,14 +51,18 @@ def parse_arguments(): :return: """ parser = argparse.ArgumentParser() - parser.add_argument('--shell_name', - type=str, - default='get_op_list.sh', - help='please input right name') - parser.add_argument('--op_list_file', - type=str, - default='list_op.txt', - help='please input right name') + parser.add_argument( + '--shell_name', + type=str, + default='get_op_list.sh', + help='please input right name', + ) + parser.add_argument( + '--op_list_file', + type=str, + default='list_op.txt', + help='please input right name', + ) return parser.parse_args() @@ -85,7 +89,7 @@ def get_prefix(line, end_char='d'): """ i = 0 prefix = '' - while (line[i] != end_char): + while line[i] != end_char: prefix += line[i] i += 1 return prefix @@ -116,7 +120,9 @@ def add_import_skip_return(file, pattern_import, pattern_skip, pattern_return): # add @skip_check_grad_ci match_obj = pattern_2.search(line) if match_obj is not None: - file_data += "@skip_check_grad_ci(reason='jetson do n0t neeed this !')\n" + file_data += ( + "@skip_check_grad_ci(reason='jetson do n0t neeed this !')\n" + ) print("### add @skip_check_grad_ci ####") # delete test_grad_output @@ -155,8 +161,14 @@ def set_diff_value(file, atol="1e-5", inplace_atol="1e-7"): :param inplace_atol: :return: """ - os.system(r"sed -i 's/self.check_output(/self\.check_output\(atol=" + atol + - ",inplace_atol=" + inplace_atol + ",/g\' " + file) + os.system( + r"sed -i 's/self.check_output(/self\.check_output\(atol=" + + atol + + ",inplace_atol=" + + inplace_atol + + ",/g\' " + + file + ) def change_op_file(start=0, end=0, op_list_file='list_op.txt', path='.'): @@ -182,8 +194,9 @@ def change_op_file(start=0, end=0, op_list_file='list_op.txt', path='.'): pattern_skip = r"^class .*\(OpTest\):$" pattern_return = r"def test.*grad.*\):$" # change file - add_import_skip_return(file_with_path, pattern_import, pattern_skip, - pattern_return) + add_import_skip_return( + file_with_path, pattern_import, pattern_skip, pattern_return + ) # op_diff if item in op_diff_list: set_diff_value(file_with_path) @@ -240,7 +253,7 @@ def run_file_change(op_list_file): :param op_list_file: :return: """ - if (os.path.exists("flag_change_file.txt")): + if os.path.exists("flag_change_file.txt"): print( "-----maybe op_file has changed, so don't need to change again------" ) @@ -269,9 +282,12 @@ def run_test_second(): "sed -n '/(Failed)$/p' test_op_log.txt | awk '{print $3}' >& rerun_op.txt" ) rerun_list = get_op_list('rerun_op.txt') - if (len(rerun_list)): - print("-------there are " + str(len(rerun_list)) + - " op(s) need to rerun!!!-------") + if len(rerun_list): + print( + "-------there are " + + str(len(rerun_list)) + + " op(s) need to rerun!!!-------" + ) for failed_op in rerun_list: os.system("ctest -R \"(" + failed_op + ")\" ") else: diff --git a/tools/parallel_UT_rule.py b/tools/parallel_UT_rule.py index ea36b778536..fa5a6c81315 100755 --- a/tools/parallel_UT_rule.py +++ b/tools/parallel_UT_rule.py @@ -669,7 +669,7 @@ HIGH_PARALLEL_JOB_NEW = [ 'test_trt_convert_reduce_sum', 'save_quant2_model_lstm', 'test_trt_convert_slice', - 'test_quant2_int8_lstm_mkldnn' + 'test_quant2_int8_lstm_mkldnn', ] # mem=0 but always timeout or failed : It run 15 job each time in Single cases; @@ -734,315 +734,805 @@ SECONDARY_HIGH_PARALLEL_JOB_NEW = [ # mem=0 but always timeout or failed : It run 12 job each time in Single cases; THIRD_HIGH_PARALLEL_JOB_NEW = [ - 'test_api_impl', 'test_analyzer_seq_pool1_fuse_compare_zero_copy', - 'test_analyzer_seq_pool1_profile', 'test_analyzer_mobilenet_transpose', - 'test_analyzer_resnet50', 'test_analyzer_int8_resnet50', - 'test_analyzer_int8_mobilenetv2', 'test_analyzer_bfloat16_resnet50', - 'test_analyzer_bfloat16_mobilenetv1', 'test_analyzer_int8_mobilenet_ssd', - 'test_dataset_cifar', 'test_dataset_imdb', 'test_dataset_movielens', - 'test_datasets', 'test_allgather', 'test_c_concat', 'test_c_split', - 'test_collective_reduce', 'test_cyclic_cifar_dataset', 'test_dyn_rnn', - 'test_multiclass_nms_op', 'test_communicator_geo', - 'test_quant_int8_mobilenetv2_mkldnn', 'test_analyzer_seq_pool1', - 'test_analyzer_transformer', 'test_analyzer_transformer_profile', - 'test_analyzer_int8_mobilenetv1', 'test_analyzer_bfloat16_googlenet', - 'test_analyzer_quant_performance_benchmark', 'test_dataset_wmt', - 'test_allreduce', 'test_broadcast', 'test_c_identity', - 'test_collective_sendrecv_api', 'test_fleet_utils', - 'test_fused_elemwise_activation_op', 'test_group_norm_op', - 'test_fleet_launch_nproc', 'test_quant_int8_resnet50_mkldnn', - 'test_quant2_int8_ernie_mkldnn', 'convert_model2dot_ernie' + 'test_api_impl', + 'test_analyzer_seq_pool1_fuse_compare_zero_copy', + 'test_analyzer_seq_pool1_profile', + 'test_analyzer_mobilenet_transpose', + 'test_analyzer_resnet50', + 'test_analyzer_int8_resnet50', + 'test_analyzer_int8_mobilenetv2', + 'test_analyzer_bfloat16_resnet50', + 'test_analyzer_bfloat16_mobilenetv1', + 'test_analyzer_int8_mobilenet_ssd', + 'test_dataset_cifar', + 'test_dataset_imdb', + 'test_dataset_movielens', + 'test_datasets', + 'test_allgather', + 'test_c_concat', + 'test_c_split', + 'test_collective_reduce', + 'test_cyclic_cifar_dataset', + 'test_dyn_rnn', + 'test_multiclass_nms_op', + 'test_communicator_geo', + 'test_quant_int8_mobilenetv2_mkldnn', + 'test_analyzer_seq_pool1', + 'test_analyzer_transformer', + 'test_analyzer_transformer_profile', + 'test_analyzer_int8_mobilenetv1', + 'test_analyzer_bfloat16_googlenet', + 'test_analyzer_quant_performance_benchmark', + 'test_dataset_wmt', + 'test_allreduce', + 'test_broadcast', + 'test_c_identity', + 'test_collective_sendrecv_api', + 'test_fleet_utils', + 'test_fused_elemwise_activation_op', + 'test_group_norm_op', + 'test_fleet_launch_nproc', + 'test_quant_int8_resnet50_mkldnn', + 'test_quant2_int8_ernie_mkldnn', + 'convert_model2dot_ernie', ] # mem != 0: It run 7 job each time in Single cases; 4 job each time in Multi cases; 3 job each time in exclusive cases FOURTH_HIGH_PARALLEL_JOB_NEW = [ - 'test_meshgrid_op', 'test_word2vec', 'test_analyzer_ner', - 'test_fetch_lod_tensor_array', 'test_adagrad_op_v2', - 'test_conv2d_fusion_op', 'test_hapi_amp', 'test_metrics', - 'test_clip_by_norm_op', 'test_lr_scheduler', 'test_generate_proposals_op', - 'test_masked_select_op', 'test_trt_anchor_generator_op', - 'test_imperative_ocr_attention_model', 'test_sentiment', 'test_chunk_op', - 'test_memcpy_op', 'test_warpctc_op', 'test_row_conv_op', - 'test_grid_sample_function', 'test_rnn_nets', 'test_pad3d_op', - 'test_imperative_mnist_sorted_gradient', 'tensor_test', - 'test_tensorrt_engine_op', 'test_dot_op', 'test_real_imag_op', - 'test_adam_optimizer_fp32_fp64', 'test_reduce_op', - 'test_density_prior_box_op', 'test_top_k_op', 'test_grid_generator', - 'test_randn_op', 'test_activation_mkldnn_op', 'test_lac', 'test_pad_op', - 'test_lstmp_op', 'test_loop', 'test_pylayer_op', - 'data_device_transform_test', 'test_trt_roi_align_op', - 'test_nn_functional_hot_op', 'test_top_k_v2_op', 'test_crop_op', - 'test_conv_bn_fuse_pass', 'test_beam_search_decode_op', 'test_auc_op', - 'test_pool2d_op', 'test_gaussian_random_op', 'test_maximum_op', - 'test_rnn_cell_api', 'device_code_test', 'test_ir_inplace_pass', - 'test_cos_sim_op', 'test_lite_tensor_utils', 'test_fit_a_line', - 'test_mish_op', 'test_transpose_op', 'test_mean_iou', - 'test_conv3d_transpose_op', 'test_jit_save_load', 'test_unsqueeze2_op', - 'test_eager_deletion_while_op', 'test_zeros_like_op', 'test_c_embedding_op', - 'test_regularizer', 'zero_copy_tensor_test', 'test_tensor_shape', - 'test_resnet', 'test_resnet_amp', 'test_dygraph_weight_norm', 'test_tracer', - 'test_list', 'test_sequence_concat', 'test_adaptive_avg_pool1d', - 'test_elementwise_div_op', 'test_conv1d_transpose_layer', 'test_adamw_op', - 'trt_fc_prelu_test', 'test_temporal_shift_op', - 'test_naive_best_fit_gpu_memory_limit', 'dlpack_tensor_test', - 'test_elementwise_max_op', 'test_typing', 'test_asp_pruning_2d_greedy', - 'test_fake_dequantize_op', 'test_crop_tensor_op', - 'test_imperative_load_static_param', 'test_imperative_qat_user_defined', - 'test_anchor_generator_op', 'test_if_else_op', 'test_prepare_op', - 'test_conj_op', 'test_imperative_hook_for_layer', 'test_roi_pool_op', - 'test_strided_slice_op', 'test_norm_all', 'test_weight_decay', - 'test_functional_conv2d', 'test_functional_conv3d_transpose', - 'test_imperative_layer_trainable', 'test_imperative_data_parallel', - 'test_digamma_op', 'test_distribution', 'test_box_clip_op', - 'custom_tensor_test', 'test_marker_op', 'test_dataloader_early_reset', - 'test_gather_nd_op', 'test_tensor_register_hook', 'test_retain_graph', - 'test_network_with_dtype', 'test_basic_api_transformation', 'test_diag', - 'test_lod_array_length_op', 'test_reinforcement_learning', - 'test_softmax_op', 'test_fc_fuse_pass', 'test_adaptive_max_pool2d', - 'test_inverse_op', 'test_declarative', 'test_imperative_double_grad', - 'test_tensor_methods', 'test_pool1d_api', 'system_allocator_test', - 'test_print', 'test_tensor_type_promotion', 'test_bce_with_logits_loss', - 'test_tensor', 'test_cross_op', 'concat_test', 'test_ast_util', - 'test_proximal_adagrad_op', 'test_pairwise_distance', - 'test_imperative_mnist', 'test_beam_search_decoder', - 'test_build_strategy_fusion_group_pass', 'test_dygraph_spectral_norm', - 'test_scale_mkldnn_op', 'test_load_state_dict_from_old_format', - 'test_margin_rank_loss_op', 'test_lookup_table_v2_op', - 'test_mix_precision_all_reduce_fuse', 'test_spp_op', 'test_op_converter', - 'mixed_vector_test', 'test_roi_align_op', 'test_pad_constant_like', - 'test_mul_op', 'test_spectral_norm_op', 'test_transformer', - 'test_for_enumerate', 'test_variable_trans_func', - 'test_squared_l2_distance_op', 'test_quantize_transpiler_v2', - 'test_im2sequence_op', 'test_reader_reset', 'test_one_hot_op', - 'test_adaptive_max_pool1d', 'test_label_smooth_op', - 'test_parallel_executor_fetch_feed', 'test_cast', - 'test_parallel_dygraph_sync_batch_norm', 'test_collect_fpn_proposals_op', - 'test_expand_as_v2_op', 'test_device', 'test_code_generator', - 'test_asp_pruning_2d_best', 'test_fleet_with_asp', 'test_pool2d_api', - 'test_mean_op', 'test_is_tensor', 'test_run_program_op', - 'test_cuda_random_seed', 'test_linear_interp_op', - 'test_fuse_all_reduce_pass', 'tensor_util_test', 'test_median', - 'test_nanmedian', 'test_linear', 'test_imperative_qat_amp', - 'test_truncated_gaussian_random_op', 'test_lstm_cudnn_op', - 'copy_same_tensor_test', 'test_squeeze2_op', - 'naive_best_fit_allocator_test', 'test_model', 'test_py_reader_combination', - 'test_prior_box_op', 'test_matmul_v2_mkldnn_op', 'test_sum_op', - 'test_paddle_imperative_double_grad', 'test_norm_op', 'test_pool3d_api', - 'test_imperative_gan', 'test_sequence_softmax_op', 'test_rand_op', - 'test_expand_v2_op', 'test_word2vec_book', 'test_histogram_op', - 'test_min_op', 'test_mse_loss', 'test_sign_op', - 'selected_rows_functor_gpu_test', 'test_fleet_base', 'test_logsumexp', - 'test_detection', 'test_image_classification_fp16', 'test_random_seed', - 'test_op_function_generator', 'test_unique_with_counts', - 'test_complex_elementwise_layers', 'test_array_read_write_op', - 'test_fusion_group_op', 'test_imperative_layer_apply', + 'test_meshgrid_op', + 'test_word2vec', + 'test_analyzer_ner', + 'test_fetch_lod_tensor_array', + 'test_adagrad_op_v2', + 'test_conv2d_fusion_op', + 'test_hapi_amp', + 'test_metrics', + 'test_clip_by_norm_op', + 'test_lr_scheduler', + 'test_generate_proposals_op', + 'test_masked_select_op', + 'test_trt_anchor_generator_op', + 'test_imperative_ocr_attention_model', + 'test_sentiment', + 'test_chunk_op', + 'test_memcpy_op', + 'test_warpctc_op', + 'test_row_conv_op', + 'test_grid_sample_function', + 'test_rnn_nets', + 'test_pad3d_op', + 'test_imperative_mnist_sorted_gradient', + 'tensor_test', + 'test_tensorrt_engine_op', + 'test_dot_op', + 'test_real_imag_op', + 'test_adam_optimizer_fp32_fp64', + 'test_reduce_op', + 'test_density_prior_box_op', + 'test_top_k_op', + 'test_grid_generator', + 'test_randn_op', + 'test_activation_mkldnn_op', + 'test_lac', + 'test_pad_op', + 'test_lstmp_op', + 'test_loop', + 'test_pylayer_op', + 'data_device_transform_test', + 'test_trt_roi_align_op', + 'test_nn_functional_hot_op', + 'test_top_k_v2_op', + 'test_crop_op', + 'test_conv_bn_fuse_pass', + 'test_beam_search_decode_op', + 'test_auc_op', + 'test_pool2d_op', + 'test_gaussian_random_op', + 'test_maximum_op', + 'test_rnn_cell_api', + 'device_code_test', + 'test_ir_inplace_pass', + 'test_cos_sim_op', + 'test_lite_tensor_utils', + 'test_fit_a_line', + 'test_mish_op', + 'test_transpose_op', + 'test_mean_iou', + 'test_conv3d_transpose_op', + 'test_jit_save_load', + 'test_unsqueeze2_op', + 'test_eager_deletion_while_op', + 'test_zeros_like_op', + 'test_c_embedding_op', + 'test_regularizer', + 'zero_copy_tensor_test', + 'test_tensor_shape', + 'test_resnet', + 'test_resnet_amp', + 'test_dygraph_weight_norm', + 'test_tracer', + 'test_list', + 'test_sequence_concat', + 'test_adaptive_avg_pool1d', + 'test_elementwise_div_op', + 'test_conv1d_transpose_layer', + 'test_adamw_op', + 'trt_fc_prelu_test', + 'test_temporal_shift_op', + 'test_naive_best_fit_gpu_memory_limit', + 'dlpack_tensor_test', + 'test_elementwise_max_op', + 'test_typing', + 'test_asp_pruning_2d_greedy', + 'test_fake_dequantize_op', + 'test_crop_tensor_op', + 'test_imperative_load_static_param', + 'test_imperative_qat_user_defined', + 'test_anchor_generator_op', + 'test_if_else_op', + 'test_prepare_op', + 'test_conj_op', + 'test_imperative_hook_for_layer', + 'test_roi_pool_op', + 'test_strided_slice_op', + 'test_norm_all', + 'test_weight_decay', + 'test_functional_conv2d', + 'test_functional_conv3d_transpose', + 'test_imperative_layer_trainable', + 'test_imperative_data_parallel', + 'test_digamma_op', + 'test_distribution', + 'test_box_clip_op', + 'custom_tensor_test', + 'test_marker_op', + 'test_dataloader_early_reset', + 'test_gather_nd_op', + 'test_tensor_register_hook', + 'test_retain_graph', + 'test_network_with_dtype', + 'test_basic_api_transformation', + 'test_diag', + 'test_lod_array_length_op', + 'test_reinforcement_learning', + 'test_softmax_op', + 'test_fc_fuse_pass', + 'test_adaptive_max_pool2d', + 'test_inverse_op', + 'test_declarative', + 'test_imperative_double_grad', + 'test_tensor_methods', + 'test_pool1d_api', + 'system_allocator_test', + 'test_print', + 'test_tensor_type_promotion', + 'test_bce_with_logits_loss', + 'test_tensor', + 'test_cross_op', + 'concat_test', + 'test_ast_util', + 'test_proximal_adagrad_op', + 'test_pairwise_distance', + 'test_imperative_mnist', + 'test_beam_search_decoder', + 'test_build_strategy_fusion_group_pass', + 'test_dygraph_spectral_norm', + 'test_scale_mkldnn_op', + 'test_load_state_dict_from_old_format', + 'test_margin_rank_loss_op', + 'test_lookup_table_v2_op', + 'test_mix_precision_all_reduce_fuse', + 'test_spp_op', + 'test_op_converter', + 'mixed_vector_test', + 'test_roi_align_op', + 'test_pad_constant_like', + 'test_mul_op', + 'test_spectral_norm_op', + 'test_transformer', + 'test_for_enumerate', + 'test_variable_trans_func', + 'test_squared_l2_distance_op', + 'test_quantize_transpiler_v2', + 'test_im2sequence_op', + 'test_reader_reset', + 'test_one_hot_op', + 'test_adaptive_max_pool1d', + 'test_label_smooth_op', + 'test_parallel_executor_fetch_feed', + 'test_cast', + 'test_parallel_dygraph_sync_batch_norm', + 'test_collect_fpn_proposals_op', + 'test_expand_as_v2_op', + 'test_device', + 'test_code_generator', + 'test_asp_pruning_2d_best', + 'test_fleet_with_asp', + 'test_pool2d_api', + 'test_mean_op', + 'test_is_tensor', + 'test_run_program_op', + 'test_cuda_random_seed', + 'test_linear_interp_op', + 'test_fuse_all_reduce_pass', + 'tensor_util_test', + 'test_median', + 'test_nanmedian', + 'test_linear', + 'test_imperative_qat_amp', + 'test_truncated_gaussian_random_op', + 'test_lstm_cudnn_op', + 'copy_same_tensor_test', + 'test_squeeze2_op', + 'naive_best_fit_allocator_test', + 'test_model', + 'test_py_reader_combination', + 'test_prior_box_op', + 'test_matmul_v2_mkldnn_op', + 'test_sum_op', + 'test_paddle_imperative_double_grad', + 'test_norm_op', + 'test_pool3d_api', + 'test_imperative_gan', + 'test_sequence_softmax_op', + 'test_rand_op', + 'test_expand_v2_op', + 'test_word2vec_book', + 'test_histogram_op', + 'test_min_op', + 'test_mse_loss', + 'test_sign_op', + 'selected_rows_functor_gpu_test', + 'test_fleet_base', + 'test_logsumexp', + 'test_detection', + 'test_image_classification_fp16', + 'test_random_seed', + 'test_op_function_generator', + 'test_unique_with_counts', + 'test_complex_elementwise_layers', + 'test_array_read_write_op', + 'test_fusion_group_op', + 'test_imperative_layer_apply', 'test_executor_return_tensor_not_overwriting', - 'test_optimizer_in_control_flow', 'test_lookup_table_op', 'test_randint_op', - 'test_randint_like', 'test_convert_call', - 'test_sigmoid_cross_entropy_with_logits_op', 'copy_cross_scope_test', - 'test_normalization_wrapper', 'test_flip', 'test_cosine_similarity_api', - 'test_cumsum_op', 'test_range', 'test_log_loss_op', 'test_where_index', - 'test_tril_triu_op', 'test_lod_reset_op', 'test_lod_tensor', - 'test_addmm_op', 'test_index_select_op', 'test_index_add_op', 'test_nvprof', - 'test_index_sample_op', 'test_unstack_op', 'test_increment', - 'strided_memcpy_test', 'test_target_assign_op', + 'test_optimizer_in_control_flow', + 'test_lookup_table_op', + 'test_randint_op', + 'test_randint_like', + 'test_convert_call', + 'test_sigmoid_cross_entropy_with_logits_op', + 'copy_cross_scope_test', + 'test_normalization_wrapper', + 'test_flip', + 'test_cosine_similarity_api', + 'test_cumsum_op', + 'test_range', + 'test_log_loss_op', + 'test_where_index', + 'test_tril_triu_op', + 'test_lod_reset_op', + 'test_lod_tensor', + 'test_addmm_op', + 'test_index_select_op', + 'test_index_add_op', + 'test_nvprof', + 'test_index_sample_op', + 'test_unstack_op', + 'test_increment', + 'strided_memcpy_test', + 'test_target_assign_op', 'test_trt_dynamic_shape_transformer_prune', - 'test_box_decoder_and_assign_op', 'test_trt_dynamic_shape', 'test_mnist', - 'test_convert_operators', 'test_fill_any_like_op', 'test_fill_constant_op', - 'test_callback_reduce_lr_on_plateau', 'test_tile_op', 'test_logical', - 'test_deformable_conv_op', 'test_elementwise_add_grad_grad', - 'test_simple_rnn_op', 'test_bicubic_interp_op', 'test_batch_norm_op_v2', - 'test_custom_relu_op_jit', 'test_math_op_patch_var_base', 'test_se_resnet', - 'test_device_guard', 'test_elementwise_div_grad_grad', 'test_minus_op', - 'test_shard_index_op', 'test_dygraph_recompute', 'test_momentum_op', - 'test_modelaverage', 'test_compare_reduce_op', 'test_affine_grid_op', - 'test_allclose_layer', 'test_elementwise_pow_op', 'test_trt_subgraph_pass', - 'test_adaptive_avg_pool2d', 'test_functional_conv3d', - 'test_executor_and_mul', 'test_kron_op', 'test_cast_mkldnn_op', - 'test_imperative_auto_prune', 'allocator_facade_frac_flags_test', - 'test_fill_zeros_like_op', 'test_gather_tree_op', 'test_elementwise_mul_op', - 'test_cycle_gan', 'test_parallel_executor_transformer_auto_growth', - 'test_bitwise_op', 'test_uniform_random_op', 'trt_split_converter_test', - 'test_huber_loss_op', 'test_slice', 'test_label_smooth_functional', - 'test_conv_shift_op', 'test_imperative_optimizer_v2', 'test_len', - 'test_imperative_named_members', 'test_sequence_reshape', - 'test_elementwise_min_op', 'test_flatten2_op', 'test_param_guard', - 'test_imperative_ptb_rnn', 'test_batch_fc_op', 'test_Tensor_type', - 'test_complex_getitem', 'lod_tensor_gpu_test', 'im2col_test', - 'test_unbind_op', 'test_imperative_ptq', 'test_auc_single_pred_op', - 'test_imperative_reinforcement', 'test_tf32_cublas', 'test_return', - 'test_py_reader_push_pop', 'test_lstm', 'test_dygraph_mnist_fp16', - 'test_shuffle_channel_op', 'test_partial_concat_op', - 'test_fill_zeros_like2_op', 'test_deformable_conv_v1_op', - 'test_complex_grad_accumulated', 'test_sequence_mask', 'test_fill_op', - 'test_imperative_deepcf', 'test_reorder_lod_tensor', 'test_multiply', - 'test_partial_program', 'test_fetch_feed', 'test_group', - 'test_trt_reduce_sum_op', 'data_type_transform_test', 'test_gru_rnn_op', - 'test_cudnn_grucell', 'test_argsort_op', 'test_batch_norm_op', - 'test_inplace', 'test_deprecated_decorator', 'test_complex_cast', - 'test_diag_v2', 'test_iou_similarity_op', - 'test_inplace_auto_generated_apis', 'test_dataset', 'test_bilinear_api', - 'test_empty_like_op', 'test_imperative_layer_children', 'nccl_op_test', - 'test_tree_conv_op', 'test_share_data_op', - 'test_ir_memory_optimize_transformer', 'test_lod_append_op', - 'test_math_op_patch', 'test_base_layer', 'test_dequantize_log_op', - 'test_complex_matmul', 'test_prelu_op', 'test_l1_norm_op', - 'test_rmsprop_op', 'test_fuse_bn_act_pass', 'test_inplace_addto_strategy', - 'test_ptb_lm_v2', 'test_paddle_save_load', 'test_prelu_mkldnn_op', - 'test_box_coder_op', 'test_atan2_op', 'test_unsqueeze_op', 'test_profiler', - 'test_affine_channel_op', 'test_leaky_relu_grad_grad_functor', - 'test_ctc_align', 'test_fuse_relu_depthwise_conv_pass', 'test_complex_kron', - 'test_imperative_skip_op', 'test_dgc_op', 'test_regularizer_api', - 'test_nll_loss', 'test_imperative_layers', 'test_rnn_decode_api', - 'test_imperative_partitial_backward', 'test_where_op', 'test_std_layer', - 'test_ir_embedding_eltwise_layernorm_fuse_pass', 'test_multihead_attention', - 'test_grid_sampler_op', 'test_initializer_nn', 'test_var_base', - 'test_fuse_elewise_add_act_pass', 'test_select_input_output_op', - 'test_lstm_op', 'test_break_continue', - 'test_imperative_parallel_coalesce_split', 'test_expand_as_op', - 'test_user_defined_quantization', 'test_tensor_to_list', - 'test_limit_gpu_memory', 'test_adamax_api', - 'test_softmax_mask_fuse_upper_triangle_op', 'test_fake_quantize_op', - 'vol2col_test', 'test_cast_op', 'test_proximal_gd_op', 'test_mul_nn_grad', - 'test_full_like_op', 'test_imperative_static_runner_while', - 'trt_instance_norm_test', 'test_elementwise_mod_op', - 'test_grad_clip_minimize', 'test_one_hot_v2_op', 'test_complex_sum_layer', - 'test_isfinite_v2_op', 'test_is_empty_op', 'test_simnet_v2', - 'beam_search_test', 'test_randperm_op', 'test_elementwise_add_op_inplace', - 'test_imperative_selected_rows', 'test_py_reader_using_executor', - 'test_activation_op', 'test_nn_functional_embedding_dygraph', - 'test_reshape_op', 'test_maxout_op', 'test_cudnn_lstmcell', - 'test_sigmoid_focal_loss', 'test_manual_seed', 'test_lrn_op', - 'test_ir_memory_optimize_nlp', 'test_dataset_dataloader', - 'test_complex_variable', 'test_lite_engine', 'test_neg_op', - 'test_view_op_reuse_allocation', 'test_split_op', 'test_ptb_lm', - 'test_elementwise_sub_op', 'test_compare_op', 'test_simnet', - 'test_label_semantic_roles', 'test_normal', - 'test_tensor_scalar_type_promotion_static', 'test_trt_group_norm_op', - 'test_learning_rate_scheduler', 'test_numel_op', 'test_adaptive_max_pool3d', - 'test_sequential', 'test_imperative_optimizer', 'test_subtract_op', - 'test_conv_transpose_nn_grad', 'test_sigmoid_focal_loss_op', - 'test_cuda_stream_event', 'test_sequence_pad_op', 'test_rnn_cells', - 'test_partial_sum_op', 'test_rnn_nets_static', 'test_max_op', - 'test_logical_op', 'test_squared_l2_norm_op', 'test_center_loss', - 'test_quantization_pass', 'test_imperative_gnn', - 'test_conv_elementwise_add_act_fuse_pass', 'test_roll_op', - 'test_imperative_container_layerdict', 'test_shape_op', 'test_bmm_op', - 'test_matmul_v2_op', 'test_hinge_loss_op', 'test_imperative_qat', - 'test_add_position_encoding_op', 'test_rnn_op', 'test_gradient_clip', - 'test_py_reader_pin_memory', 'test_concat_op', 'test_weight_decay_extend', - 'test_accuracy_op', 'test_cond', 'test_resnet_v2', 'test_adagrad_op', - 'test_mv_op', 'test_print_op', 'test_grad', 'test_square_error_cost', - 'test_rnn_cells_static', 'test_mkldnn_batch_norm_act_fuse_pass', - 'test_input_spec', 'test_adam_op', 'test_elementwise_floordiv_op', - 'test_eager_deletion_gru_net', 'test_diagonal_op', - 'test_imperative_static_runner_mnist', 'test_nearest_interp_op', - 'test_diag_embed', 'test_imperative_basic', 'test_merge_selectedrows_op', - 'test_feed_data_check_shape_type', 'test_complex_trace_layer', - 'test_slice_op', 'test_bmn', 'test_nn_quant_functional_layers', - 'test_broadcast_tensors_op', 'test_selu_op', 'test_group_norm_op_v2', - 'test_tensor_to_numpy', 'test_queue', 'test_rank_loss_op', 'test_trace_op', - 'test_case', 'test_prroi_pool_op', 'test_op_name_conflict', - 'test_psroi_pool_op', 'test_set_value_op', 'test_ones_like', - 'test_assign_value_op', 'test_ema', 'test_lamb_op', 'test_dgc_momentum_op', - 'test_custom_grad_input', 'test_trunc_op', 'test_bernoulli_op', - 'test_custom_relu_model', 'test_backward', 'test_conv3d_transpose_part2_op', - 'test_complex_transpose', 'test_memory_reuse_exclude_feed_var', - 'test_polygon_box_transform', 'math_function_gpu_test', - 'test_program_prune_backward', 'test_ema_fleet', 'test_fleet_amp_init', - 'test_normalize', 'test_correlation', + 'test_box_decoder_and_assign_op', + 'test_trt_dynamic_shape', + 'test_mnist', + 'test_convert_operators', + 'test_fill_any_like_op', + 'test_fill_constant_op', + 'test_callback_reduce_lr_on_plateau', + 'test_tile_op', + 'test_logical', + 'test_deformable_conv_op', + 'test_elementwise_add_grad_grad', + 'test_simple_rnn_op', + 'test_bicubic_interp_op', + 'test_batch_norm_op_v2', + 'test_custom_relu_op_jit', + 'test_math_op_patch_var_base', + 'test_se_resnet', + 'test_device_guard', + 'test_elementwise_div_grad_grad', + 'test_minus_op', + 'test_shard_index_op', + 'test_dygraph_recompute', + 'test_momentum_op', + 'test_modelaverage', + 'test_compare_reduce_op', + 'test_affine_grid_op', + 'test_allclose_layer', + 'test_elementwise_pow_op', + 'test_trt_subgraph_pass', + 'test_adaptive_avg_pool2d', + 'test_functional_conv3d', + 'test_executor_and_mul', + 'test_kron_op', + 'test_cast_mkldnn_op', + 'test_imperative_auto_prune', + 'allocator_facade_frac_flags_test', + 'test_fill_zeros_like_op', + 'test_gather_tree_op', + 'test_elementwise_mul_op', + 'test_cycle_gan', + 'test_parallel_executor_transformer_auto_growth', + 'test_bitwise_op', + 'test_uniform_random_op', + 'trt_split_converter_test', + 'test_huber_loss_op', + 'test_slice', + 'test_label_smooth_functional', + 'test_conv_shift_op', + 'test_imperative_optimizer_v2', + 'test_len', + 'test_imperative_named_members', + 'test_sequence_reshape', + 'test_elementwise_min_op', + 'test_flatten2_op', + 'test_param_guard', + 'test_imperative_ptb_rnn', + 'test_batch_fc_op', + 'test_Tensor_type', + 'test_complex_getitem', + 'lod_tensor_gpu_test', + 'im2col_test', + 'test_unbind_op', + 'test_imperative_ptq', + 'test_auc_single_pred_op', + 'test_imperative_reinforcement', + 'test_tf32_cublas', + 'test_return', + 'test_py_reader_push_pop', + 'test_lstm', + 'test_dygraph_mnist_fp16', + 'test_shuffle_channel_op', + 'test_partial_concat_op', + 'test_fill_zeros_like2_op', + 'test_deformable_conv_v1_op', + 'test_complex_grad_accumulated', + 'test_sequence_mask', + 'test_fill_op', + 'test_imperative_deepcf', + 'test_reorder_lod_tensor', + 'test_multiply', + 'test_partial_program', + 'test_fetch_feed', + 'test_group', + 'test_trt_reduce_sum_op', + 'data_type_transform_test', + 'test_gru_rnn_op', + 'test_cudnn_grucell', + 'test_argsort_op', + 'test_batch_norm_op', + 'test_inplace', + 'test_deprecated_decorator', + 'test_complex_cast', + 'test_diag_v2', + 'test_iou_similarity_op', + 'test_inplace_auto_generated_apis', + 'test_dataset', + 'test_bilinear_api', + 'test_empty_like_op', + 'test_imperative_layer_children', + 'nccl_op_test', + 'test_tree_conv_op', + 'test_share_data_op', + 'test_ir_memory_optimize_transformer', + 'test_lod_append_op', + 'test_math_op_patch', + 'test_base_layer', + 'test_dequantize_log_op', + 'test_complex_matmul', + 'test_prelu_op', + 'test_l1_norm_op', + 'test_rmsprop_op', + 'test_fuse_bn_act_pass', + 'test_inplace_addto_strategy', + 'test_ptb_lm_v2', + 'test_paddle_save_load', + 'test_prelu_mkldnn_op', + 'test_box_coder_op', + 'test_atan2_op', + 'test_unsqueeze_op', + 'test_profiler', + 'test_affine_channel_op', + 'test_leaky_relu_grad_grad_functor', + 'test_ctc_align', + 'test_fuse_relu_depthwise_conv_pass', + 'test_complex_kron', + 'test_imperative_skip_op', + 'test_dgc_op', + 'test_regularizer_api', + 'test_nll_loss', + 'test_imperative_layers', + 'test_rnn_decode_api', + 'test_imperative_partitial_backward', + 'test_where_op', + 'test_std_layer', + 'test_ir_embedding_eltwise_layernorm_fuse_pass', + 'test_multihead_attention', + 'test_grid_sampler_op', + 'test_initializer_nn', + 'test_var_base', + 'test_fuse_elewise_add_act_pass', + 'test_select_input_output_op', + 'test_lstm_op', + 'test_break_continue', + 'test_imperative_parallel_coalesce_split', + 'test_expand_as_op', + 'test_user_defined_quantization', + 'test_tensor_to_list', + 'test_limit_gpu_memory', + 'test_adamax_api', + 'test_softmax_mask_fuse_upper_triangle_op', + 'test_fake_quantize_op', + 'vol2col_test', + 'test_cast_op', + 'test_proximal_gd_op', + 'test_mul_nn_grad', + 'test_full_like_op', + 'test_imperative_static_runner_while', + 'trt_instance_norm_test', + 'test_elementwise_mod_op', + 'test_grad_clip_minimize', + 'test_one_hot_v2_op', + 'test_complex_sum_layer', + 'test_isfinite_v2_op', + 'test_is_empty_op', + 'test_simnet_v2', + 'beam_search_test', + 'test_randperm_op', + 'test_elementwise_add_op_inplace', + 'test_imperative_selected_rows', + 'test_py_reader_using_executor', + 'test_activation_op', + 'test_nn_functional_embedding_dygraph', + 'test_reshape_op', + 'test_maxout_op', + 'test_cudnn_lstmcell', + 'test_sigmoid_focal_loss', + 'test_manual_seed', + 'test_lrn_op', + 'test_ir_memory_optimize_nlp', + 'test_dataset_dataloader', + 'test_complex_variable', + 'test_lite_engine', + 'test_neg_op', + 'test_view_op_reuse_allocation', + 'test_split_op', + 'test_ptb_lm', + 'test_elementwise_sub_op', + 'test_compare_op', + 'test_simnet', + 'test_label_semantic_roles', + 'test_normal', + 'test_tensor_scalar_type_promotion_static', + 'test_trt_group_norm_op', + 'test_learning_rate_scheduler', + 'test_numel_op', + 'test_adaptive_max_pool3d', + 'test_sequential', + 'test_imperative_optimizer', + 'test_subtract_op', + 'test_conv_transpose_nn_grad', + 'test_sigmoid_focal_loss_op', + 'test_cuda_stream_event', + 'test_sequence_pad_op', + 'test_rnn_cells', + 'test_partial_sum_op', + 'test_rnn_nets_static', + 'test_max_op', + 'test_logical_op', + 'test_squared_l2_norm_op', + 'test_center_loss', + 'test_quantization_pass', + 'test_imperative_gnn', + 'test_conv_elementwise_add_act_fuse_pass', + 'test_roll_op', + 'test_imperative_container_layerdict', + 'test_shape_op', + 'test_bmm_op', + 'test_matmul_v2_op', + 'test_hinge_loss_op', + 'test_imperative_qat', + 'test_add_position_encoding_op', + 'test_rnn_op', + 'test_gradient_clip', + 'test_py_reader_pin_memory', + 'test_concat_op', + 'test_weight_decay_extend', + 'test_accuracy_op', + 'test_cond', + 'test_resnet_v2', + 'test_adagrad_op', + 'test_mv_op', + 'test_print_op', + 'test_grad', + 'test_square_error_cost', + 'test_rnn_cells_static', + 'test_mkldnn_batch_norm_act_fuse_pass', + 'test_input_spec', + 'test_adam_op', + 'test_elementwise_floordiv_op', + 'test_eager_deletion_gru_net', + 'test_diagonal_op', + 'test_imperative_static_runner_mnist', + 'test_nearest_interp_op', + 'test_diag_embed', + 'test_imperative_basic', + 'test_merge_selectedrows_op', + 'test_feed_data_check_shape_type', + 'test_complex_trace_layer', + 'test_slice_op', + 'test_bmn', + 'test_nn_quant_functional_layers', + 'test_broadcast_tensors_op', + 'test_selu_op', + 'test_group_norm_op_v2', + 'test_tensor_to_numpy', + 'test_queue', + 'test_rank_loss_op', + 'test_trace_op', + 'test_case', + 'test_prroi_pool_op', + 'test_op_name_conflict', + 'test_psroi_pool_op', + 'test_set_value_op', + 'test_ones_like', + 'test_assign_value_op', + 'test_ema', + 'test_lamb_op', + 'test_dgc_momentum_op', + 'test_custom_grad_input', + 'test_trunc_op', + 'test_bernoulli_op', + 'test_custom_relu_model', + 'test_backward', + 'test_conv3d_transpose_part2_op', + 'test_complex_transpose', + 'test_memory_reuse_exclude_feed_var', + 'test_polygon_box_transform', + 'math_function_gpu_test', + 'test_program_prune_backward', + 'test_ema_fleet', + 'test_fleet_amp_init', + 'test_normalize', + 'test_correlation', 'test_conv_elementwise_add2_act_fuse_pass', - 'test_imperative_container_layerlist', 'test_dequantize_abs_max_op', - 'test_fuse_optimizer_pass', 'test_optimizer', - 'test_dynamic_rnn_stop_gradient', 'test_raw_program_optimizer', 'test_pow', - 'test_inplace_softmax_with_cross_entropy', 'test_transforms', - 'test_unfold_op', 'test_assign_op', 'test_isinstance', - 'auto_growth_best_fit_allocator_facade_test', 'test_cholesky_op', - 'test_adaptive_avg_pool3d', 'test_paddle_save_load_binary', - 'test_fused_fc_elementwise_layernorm_op', 'test_sequence_enumerate_op', - 'test_lgamma_op', 'test_modified_huber_loss_op', 'trt_quant_int8_test', - 'test_callback_visualdl', 'test_linspace', 'test_update_loss_scaling_op', - 'test_arg_min_max_op', 'test_bce_loss', 'test_nn_margin_rank_loss', - 'test_arg_min_max_v2_op', 'test_variance_layer', - 'test_quantization_scale_pass', 'test_segment_ops', 'test_layers', - 'test_isfinite_op', 'test_imperative_qat_channelwise', 'test_eye_op', - 'test_imperative_framework', 'test_l1_loss', 'test_ifelse', - 'test_cache_program', 'test_ir_fc_fuse_pass', 'test_kldiv_loss_op', - 'test_switch_case', 'test_unique', 'test_prod_op', 'test_edit_distance_op', - 'test_sequence_expand_as', 'test_full_name_usage', 'test_glu', - 'test_pad2d_op', 'test_read_file', 'test_erf_op', 'test_sequence_unpad_op', - 'test_sequence_conv', 'allocator_facade_abs_flags_test', 'test_detach', - 'test_cross_entropy_op', 'test_wrappers', 'test_fleet_base_single', - 'test_conv_elementwise_add_fuse_pass', 'test_auto_growth_gpu_memory_limit', - 'test_sequence_reverse', 'test_fc_op', 'test_diagflat', 'test_adamax_op', - 'test_op_attr', 'paddle_infer_api_test', 'test_mixed_precision', - 'lite_mul_model_test', 'test_sort_op', 'test_imperative_out_scale', - 'test_vision_models', 'test_rnn_encoder_decoder', 'test_fleet_with_asp_amp', + 'test_imperative_container_layerlist', + 'test_dequantize_abs_max_op', + 'test_fuse_optimizer_pass', + 'test_optimizer', + 'test_dynamic_rnn_stop_gradient', + 'test_raw_program_optimizer', + 'test_pow', + 'test_inplace_softmax_with_cross_entropy', + 'test_transforms', + 'test_unfold_op', + 'test_assign_op', + 'test_isinstance', + 'auto_growth_best_fit_allocator_facade_test', + 'test_cholesky_op', + 'test_adaptive_avg_pool3d', + 'test_paddle_save_load_binary', + 'test_fused_fc_elementwise_layernorm_op', + 'test_sequence_enumerate_op', + 'test_lgamma_op', + 'test_modified_huber_loss_op', + 'trt_quant_int8_test', + 'test_callback_visualdl', + 'test_linspace', + 'test_update_loss_scaling_op', + 'test_arg_min_max_op', + 'test_bce_loss', + 'test_nn_margin_rank_loss', + 'test_arg_min_max_v2_op', + 'test_variance_layer', + 'test_quantization_scale_pass', + 'test_segment_ops', + 'test_layers', + 'test_isfinite_op', + 'test_imperative_qat_channelwise', + 'test_eye_op', + 'test_imperative_framework', + 'test_l1_loss', + 'test_ifelse', + 'test_cache_program', + 'test_ir_fc_fuse_pass', + 'test_kldiv_loss_op', + 'test_switch_case', + 'test_unique', + 'test_prod_op', + 'test_edit_distance_op', + 'test_sequence_expand_as', + 'test_full_name_usage', + 'test_glu', + 'test_pad2d_op', + 'test_read_file', + 'test_erf_op', + 'test_sequence_unpad_op', + 'test_sequence_conv', + 'allocator_facade_abs_flags_test', + 'test_detach', + 'test_cross_entropy_op', + 'test_wrappers', + 'test_fleet_base_single', + 'test_conv_elementwise_add_fuse_pass', + 'test_auto_growth_gpu_memory_limit', + 'test_sequence_reverse', + 'test_fc_op', + 'test_diagflat', + 'test_adamax_op', + 'test_op_attr', + 'paddle_infer_api_test', + 'test_mixed_precision', + 'lite_mul_model_test', + 'test_sort_op', + 'test_imperative_out_scale', + 'test_vision_models', + 'test_rnn_encoder_decoder', + 'test_fleet_with_asp_amp', 'test_partial_eager_deletion_transformer', - 'test_imperative_star_gan_with_gradient_penalty', 'test_stack_op', - 'test_shuffle_batch_op', 'test_clip_op', 'test_py_func_op', - 'test_pool_max_op', 'test_log_softmax', - 'test_imperative_container_parameterlist', 'test_multiplex_op', + 'test_imperative_star_gan_with_gradient_penalty', + 'test_stack_op', + 'test_shuffle_batch_op', + 'test_clip_op', + 'test_py_func_op', + 'test_pool_max_op', + 'test_log_softmax', + 'test_imperative_container_parameterlist', + 'test_multiplex_op', 'test_trt_transpose_flatten_concat_fuse_pass', - 'test_seqconv_eltadd_relu_fuse_pass', 'test_assert_op', - 'test_scatter_nd_op', 'test_sequence_expand', 'test_arange', - 'test_translated_layer', 'test_decoupled_py_reader_data_check', - 'test_analyzer_ernie_large', 'test_tensor_array_to_tensor', - 'test_functional_conv2d_transpose', 'test_error', 'test_callbacks', - 'test_imperative_recurrent_usage', 'test_deform_conv2d', - 'test_coalesce_tensor_op', 'test_tsm', 'test_fused_multihead_matmul_op', - 'test_softmax_mask_fuse_op', 'test_optimizer_grad', 'test_complex_abs', - 'test_gradient_accmulator', 'test_instance_norm_op_v2', - 'test_random_crop_op', 'test_mobile_net', + 'test_seqconv_eltadd_relu_fuse_pass', + 'test_assert_op', + 'test_scatter_nd_op', + 'test_sequence_expand', + 'test_arange', + 'test_translated_layer', + 'test_decoupled_py_reader_data_check', + 'test_analyzer_ernie_large', + 'test_tensor_array_to_tensor', + 'test_functional_conv2d_transpose', + 'test_error', + 'test_callbacks', + 'test_imperative_recurrent_usage', + 'test_deform_conv2d', + 'test_coalesce_tensor_op', + 'test_tsm', + 'test_fused_multihead_matmul_op', + 'test_softmax_mask_fuse_op', + 'test_optimizer_grad', + 'test_complex_abs', + 'test_gradient_accmulator', + 'test_instance_norm_op_v2', + 'test_random_crop_op', + 'test_mobile_net', 'test_parallel_executor_transformer', 'test_tensor_scalar_type_promotion_dynamic', - 'test_eager_deletion_delete_vars', 'test_asp_pruning_1d', - 'test_imperative_using_non_zero_gpu', 'test_machine_translation', - 'test_flatten_op', 'test_onnx_export', 'test_optimizer_for_varbase', - 'test_fusion_transpose_flatten_concat_op', 'best_fit_allocator_test', - 'test_ir_fusion_group_pass', 'test_trt_quant_conv2d_dequant_fuse_pass', - 'test_allclose_op', 'test_ftrl_op', 'test_elementwise_add_op', - 'test_instance_norm_op', 'test_lambv2_op', 'test_yolo_box_op', - 'test_parallel_executor_drop_scope', 'test_generator_dataloader', - 'test_conv2d_transpose_op_depthwise_conv', 'test_imperative_save_load_v2', - 'test_lookahead', 'test_moving_average_abs_max_scale_op', - 'test_roi_perspective_transform_op', 'test_tensorrt_engine', - 'test_affine_grid_function', 'test_nonzero_api', - 'test_ir_memory_optimize_pass', 'test_reduce_mkldnn_op', - 'test_bilinear_interp_op', 'test_cvm_op', 'test_scale_op', 'test_matmul_op', - 'test_sequence_pool', 'test_complex_simplenet', 'test_complex_reshape', - 'test_flatten_contiguous_range_op', 'test_python_operator_overriding', - 'lite_resnet50_test', 'test_sequence_erase_op', - 'test_deformable_psroi_pooling', 'test_multi_precision_fp16_train', - 'test_adam_op_multi_thread', 'test_decoupled_py_reader', - 'test_distribute_fpn_proposals_op', 'transform_test', 'test_nan_inf', - 'test_fuse_bn_add_act_pass', 'test_unpool_op', - 'test_parallel_executor_dry_run', 'test_layer_norm_op_v2', - 'test_embedding_id_stop_gradient', 'test_mkldnn_fc_act_fuse_pass', - 'sequence_pooling_test', 'test_get_tensor_from_selected_rows_op', - 'test_imperative_ptb_rnn_sorted_gradient', 'test_hapi_hub', - 'test_reverse_op', 'test_compiled_program', 'test_lambda', - 'test_adadelta_op', 'test_nn_sigmoid_op', 'test_nearest_interp_v2_op', - 'test_sequence_slice_op', 'test_program_translator', - 'test_eager_deletion_lstm_net', 'malloc_test', 'test_size_op', - 'test_analysis_predictor', 'test_recognize_digits', 'test_parameter', + 'test_eager_deletion_delete_vars', + 'test_asp_pruning_1d', + 'test_imperative_using_non_zero_gpu', + 'test_machine_translation', + 'test_flatten_op', + 'test_onnx_export', + 'test_optimizer_for_varbase', + 'test_fusion_transpose_flatten_concat_op', + 'best_fit_allocator_test', + 'test_ir_fusion_group_pass', + 'test_trt_quant_conv2d_dequant_fuse_pass', + 'test_allclose_op', + 'test_ftrl_op', + 'test_elementwise_add_op', + 'test_instance_norm_op', + 'test_lambv2_op', + 'test_yolo_box_op', + 'test_parallel_executor_drop_scope', + 'test_generator_dataloader', + 'test_conv2d_transpose_op_depthwise_conv', + 'test_imperative_save_load_v2', + 'test_lookahead', + 'test_moving_average_abs_max_scale_op', + 'test_roi_perspective_transform_op', + 'test_tensorrt_engine', + 'test_affine_grid_function', + 'test_nonzero_api', + 'test_ir_memory_optimize_pass', + 'test_reduce_mkldnn_op', + 'test_bilinear_interp_op', + 'test_cvm_op', + 'test_scale_op', + 'test_matmul_op', + 'test_sequence_pool', + 'test_complex_simplenet', + 'test_complex_reshape', + 'test_flatten_contiguous_range_op', + 'test_python_operator_overriding', + 'lite_resnet50_test', + 'test_sequence_erase_op', + 'test_deformable_psroi_pooling', + 'test_multi_precision_fp16_train', + 'test_adam_op_multi_thread', + 'test_decoupled_py_reader', + 'test_distribute_fpn_proposals_op', + 'transform_test', + 'test_nan_inf', + 'test_fuse_bn_add_act_pass', + 'test_unpool_op', + 'test_parallel_executor_dry_run', + 'test_layer_norm_op_v2', + 'test_embedding_id_stop_gradient', + 'test_mkldnn_fc_act_fuse_pass', + 'sequence_pooling_test', + 'test_get_tensor_from_selected_rows_op', + 'test_imperative_ptb_rnn_sorted_gradient', + 'test_hapi_hub', + 'test_reverse_op', + 'test_compiled_program', + 'test_lambda', + 'test_adadelta_op', + 'test_nn_sigmoid_op', + 'test_nearest_interp_v2_op', + 'test_sequence_slice_op', + 'test_program_translator', + 'test_eager_deletion_lstm_net', + 'malloc_test', + 'test_size_op', + 'test_analysis_predictor', + 'test_recognize_digits', + 'test_parameter', 'test_transpose_flatten_concat_fuse_pass', - 'test_imperative_trace_non_persistable_inputs', 'test_pass_builder', - 'thread_local_allocator_test', 'test_variable', 'test_fsp_op', - 'test_elementwise_gradient_op', 'test_multinomial_op', - 'test_trt_shuffle_channel_detect_pass', 'test_generate_proposals_v2_op', - 'test_graph', 'test_gelu_op', 'test_sample_logits_op', - 'test_weight_normalization', 'test_activation_bf16_mkldnn_op', - 'trt_dynamic_shape_test', 'test_traced_layer_err_msg', 'test_conv1d_layer', - 'test_asp_optimize', 'test_imperative_container_sequential', 'test_bert', - 'test_transformer_api', 'test_linear_interp_v2_op', 'test_pixel_shuffle', - 'test_expand_op', 'test_save_load', 'test_dropout_op', 'test_while_loop_op', - 'float16_gpu_test', 'test_dict', 'test_bilinear_tensor_product_op', - 'test_parallel_executor_pg', 'test_assert', 'test_smooth_l1_loss_op', - 'sequence_padding_test', 'test_analyzer_ernie', 'test_minimum_op', - 'test_yolov3_loss_op', 'test_decayed_adagrad_op', 'test_split_mkldnn_op', - 'test_squeeze_op', 'test_save_inference_model', 'test_smooth_l1_loss', - 'test_bilateral_slice_op', 'test_inplace_abn_op', 'test_fetch_unmerged', + 'test_imperative_trace_non_persistable_inputs', + 'test_pass_builder', + 'thread_local_allocator_test', + 'test_variable', + 'test_fsp_op', + 'test_elementwise_gradient_op', + 'test_multinomial_op', + 'test_trt_shuffle_channel_detect_pass', + 'test_generate_proposals_v2_op', + 'test_graph', + 'test_gelu_op', + 'test_sample_logits_op', + 'test_weight_normalization', + 'test_activation_bf16_mkldnn_op', + 'trt_dynamic_shape_test', + 'test_traced_layer_err_msg', + 'test_conv1d_layer', + 'test_asp_optimize', + 'test_imperative_container_sequential', + 'test_bert', + 'test_transformer_api', + 'test_linear_interp_v2_op', + 'test_pixel_shuffle', + 'test_expand_op', + 'test_save_load', + 'test_dropout_op', + 'test_while_loop_op', + 'float16_gpu_test', + 'test_dict', + 'test_bilinear_tensor_product_op', + 'test_parallel_executor_pg', + 'test_assert', + 'test_smooth_l1_loss_op', + 'sequence_padding_test', + 'test_analyzer_ernie', + 'test_minimum_op', + 'test_yolov3_loss_op', + 'test_decayed_adagrad_op', + 'test_split_mkldnn_op', + 'test_squeeze_op', + 'test_save_inference_model', + 'test_smooth_l1_loss', + 'test_bilateral_slice_op', + 'test_inplace_abn_op', + 'test_fetch_unmerged', 'test_parallel_executor_feed_persistable_var', 'test_parallel_executor_fetch_isolated_var', 'test_parallel_executor_inference_feed_partial_data', @@ -1050,37 +1540,66 @@ FOURTH_HIGH_PARALLEL_JOB_NEW = [ 'test_parallel_executor_test_while_train', 'test_parallel_executor_seresnext_with_fuse_all_reduce_gpu', 'test_parallel_ssa_graph_inference_feed_partial_data', - 'test_parallel_executor_seresnext_with_reduce_gpu', 'test_data_norm_op', - 'test_install_check', 'graph_node_test', 'trt_mobilenet_test', - 'trt_cascade_rcnn_test', 'trt_resnext_test', 'test_activation_nn_grad', - 'test_trt_dynamic_shape_ernie_fp16_ser_deser', 'test_cross_entropy2_op', - 'test_layer_norm_op', 'test_pool3d_op', 'test_static_save_load', - 'test_trt_flatten_op', 'test_trt_yolo_box_op', 'test_trt_reshape_op', - 'test_trt_elementwise_op', 'test_trt_affine_channel_op', 'test_trt_matmul', - 'test_trt_fc_fuse_pass', 'test_trt_pad_op', 'trt_resnet50_test', - 'test_imperative_lod_tensor_to_selected_rows', 'test_gru_unit_op', + 'test_parallel_executor_seresnext_with_reduce_gpu', + 'test_data_norm_op', + 'test_install_check', + 'graph_node_test', + 'trt_mobilenet_test', + 'trt_cascade_rcnn_test', + 'trt_resnext_test', + 'test_activation_nn_grad', + 'test_trt_dynamic_shape_ernie_fp16_ser_deser', + 'test_cross_entropy2_op', + 'test_layer_norm_op', + 'test_pool3d_op', + 'test_static_save_load', + 'test_trt_flatten_op', + 'test_trt_yolo_box_op', + 'test_trt_reshape_op', + 'test_trt_elementwise_op', + 'test_trt_affine_channel_op', + 'test_trt_matmul', + 'test_trt_fc_fuse_pass', + 'test_trt_pad_op', + 'trt_resnet50_test', + 'test_imperative_lod_tensor_to_selected_rows', + 'test_gru_unit_op', 'test_amp_check_finite_and_scale_op', - 'test_imperative_selected_rows_to_lod_tensor', 'test_imperative_save_load', - 'test_add_reader_dependency', 'test_imperative_transformer_sorted_gradient', - 'test_bicubic_interp_v2_op', 'test_rank_attention_op', - 'test_space_to_depth_op', 'test_image_classification', - 'test_custom_relu_op_setup', 'test_sgd_op' + 'test_imperative_selected_rows_to_lod_tensor', + 'test_imperative_save_load', + 'test_add_reader_dependency', + 'test_imperative_transformer_sorted_gradient', + 'test_bicubic_interp_v2_op', + 'test_rank_attention_op', + 'test_space_to_depth_op', + 'test_image_classification', + 'test_custom_relu_op_setup', + 'test_sgd_op', ] # mem != 0 : It run 7 job each time in Single cases; 3 job each time in exclusive cases FIFTH_PARALLEL_JOB_NEW = [ 'test_buffer_shared_memory_reuse_pass', 'test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass', - 'test_parallel_executor_crf', 'test_multiprocess_reader_exception', - 'buddy_allocator_test', 'test_multiprocess_dataloader_dataset', + 'test_parallel_executor_crf', + 'test_multiprocess_reader_exception', + 'buddy_allocator_test', + 'test_multiprocess_dataloader_dataset', 'test_multiprocess_dataloader_dynamic', - 'test_multiprocess_dataloader_static', 'test_imperative_resnet', - 'test_nn_grad', 'test_conv2d_op_depthwise_conv', 'test_yolov3', - 'test_conv_nn_grad', 'test_imperative_data_loader_fds_clear', - 'test_conv2d_op', 'test_imperative_data_loader_base', + 'test_multiprocess_dataloader_static', + 'test_imperative_resnet', + 'test_nn_grad', + 'test_conv2d_op_depthwise_conv', + 'test_yolov3', + 'test_conv_nn_grad', + 'test_imperative_data_loader_fds_clear', + 'test_conv2d_op', + 'test_imperative_data_loader_base', 'test_imperative_resnet_sorted_gradient', 'test_multiprocess_dataloader_iterable_dataset_dynamic', - 'test_imperative_se_resnext', 'test_norm_nn_grad', 'test_conv2d_api' + 'test_imperative_se_resnext', + 'test_norm_nn_grad', + 'test_conv2d_api', ] SIXTH_PARALLEL_JOB_NEW = [ @@ -2824,46 +3343,71 @@ def main(): if platform.system() != 'Windows': for unittest in SECONDARY_HIGH_PARALLEL_JOB_NEW: if unittest in test_cases: - secondary_high_parallel_job = secondary_high_parallel_job + '|^' + unittest + '$' + secondary_high_parallel_job = ( + secondary_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in THIRD_HIGH_PARALLEL_JOB_NEW: if unittest in test_cases: - third_high_parallel_job = third_high_parallel_job + '|^' + unittest + '$' + third_high_parallel_job = ( + third_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in SIXTH_PARALLEL_JOB_NEW: if unittest in test_cases: - sixth_high_parallel_job = sixth_high_parallel_job + '|^' + unittest + '$' + sixth_high_parallel_job = ( + sixth_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in LOWEST_PARALLEL_JOB_NEW: if unittest in test_cases: - lowest_high_parallel_job = lowest_high_parallel_job + '|^' + unittest + '$' + lowest_high_parallel_job = ( + lowest_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in fourth_high_parallel_job_list: if unittest in test_cases: - fourth_high_parallel_job = fourth_high_parallel_job + '|^' + unittest + '$' + fourth_high_parallel_job = ( + fourth_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in fifth_high_parallel_job_list: if unittest in test_cases: - fifth_high_parallel_job = fifth_high_parallel_job + '|^' + unittest + '$' + fifth_high_parallel_job = ( + fifth_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in test_cases: non_parallel_job = non_parallel_job + '|^' + unittest + '$' if platform.system() == 'Windows': - print("{};{};{};{}".format(high_parallel_job, fourth_high_parallel_job, - fifth_high_parallel_job, non_parallel_job)) + print( + "{};{};{};{}".format( + high_parallel_job, + fourth_high_parallel_job, + fifth_high_parallel_job, + non_parallel_job, + ) + ) else: - print("{};{};{};{};{};{};{};{}".format( - high_parallel_job, secondary_high_parallel_job, - third_high_parallel_job, fourth_high_parallel_job, - fifth_high_parallel_job, sixth_high_parallel_job, - lowest_high_parallel_job, non_parallel_job)) + print( + "{};{};{};{};{};{};{};{}".format( + high_parallel_job, + secondary_high_parallel_job, + third_high_parallel_job, + fourth_high_parallel_job, + fifth_high_parallel_job, + sixth_high_parallel_job, + lowest_high_parallel_job, + non_parallel_job, + ) + ) if __name__ == '__main__': diff --git a/tools/print_op_desc.py b/tools/print_op_desc.py index cf46849add4..c0ce5789b7b 100644 --- a/tools/print_op_desc.py +++ b/tools/print_op_desc.py @@ -87,8 +87,11 @@ def get_attrs_info(op_proto, op_attrs_proto): attrs_info[attr_name] = {} attrs_info[attr_name][TYPE] = attr_proto.type attrs_info[attr_name][GENERATED] = attr_proto.generated - attrs_info[attr_name][DEFAULT_VALUE] = attrs_default_values[ - attr_name] if attr_name in attrs_default_values else None + attrs_info[attr_name][DEFAULT_VALUE] = ( + attrs_default_values[attr_name] + if attr_name in attrs_default_values + else None + ) attrs_info[attr_name][EXTRA] = attr_proto.extra attrs_info[attr_name][QUANT] = attr_proto.quant return attrs_info diff --git a/tools/print_signatures.py b/tools/print_signatures.py index 4c35fdf1c4a..4c85ba123be 100644 --- a/tools/print_signatures.py +++ b/tools/print_signatures.py @@ -40,7 +40,9 @@ else: logger.addHandler(console) console.setFormatter( logging.Formatter( - "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s")) + "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s" + ) +) def md5(doc): @@ -50,15 +52,18 @@ def md5(doc): md5sum = hashinst.hexdigest() except UnicodeDecodeError as e: md5sum = None - print("Error({}) occurred when `md5({})`, discard it.".format( - str(e), doc), - file=sys.stderr) + print( + "Error({}) occurred when `md5({})`, discard it.".format( + str(e), doc + ), + file=sys.stderr, + ) return md5sum def is_primitive(instance): - int_types = (int, ) + int_types = (int,) pritimitive_types = int_types + (float, str) if isinstance(instance, pritimitive_types): return True @@ -107,12 +112,16 @@ def visit_all_module(mod): if instance_id in IdSet: continue IdSet.add(instance_id) - if hasattr(instance, - '__name__') and member_name != instance.__name__: + if ( + hasattr(instance, '__name__') + and member_name != instance.__name__ + ): print( - "Found alias API, alias name is: {}, original name is: {}" - .format(member_name, instance.__name__), - file=sys.stderr) + "Found alias API, alias name is: {}, original name is: {}".format( + member_name, instance.__name__ + ), + file=sys.stderr, + ) except: if cur_name not in ErrorSet and cur_name not in skiplist: ErrorSet.add(cur_name) @@ -130,7 +139,8 @@ def get_all_api(root_path='paddle', attr="__all__"): global api_info_dict api_counter = 0 for filefinder, name, ispkg in pkgutil.walk_packages( - path=paddle.__path__, prefix=paddle.__name__ + '.'): + path=paddle.__path__, prefix=paddle.__name__ + '.' + ): try: if name in sys.modules: m = sys.modules[name] @@ -145,11 +155,17 @@ def get_all_api(root_path='paddle', attr="__all__"): api_counter += process_module(paddle, attr) - logger.info('%s: collected %d apis, %d distinct apis.', attr, api_counter, - len(api_info_dict)) + logger.info( + '%s: collected %d apis, %d distinct apis.', + attr, + api_counter, + len(api_info_dict), + ) - return [(sorted(list(api_info['all_names']))[0], md5(api_info['docstring'])) - for api_info in api_info_dict.values()] + return [ + (sorted(list(api_info['all_names']))[0], md5(api_info['docstring'])) + for api_info in api_info_dict.values() + ] def insert_api_into_dict(full_name, gen_doc_anno=None): @@ -165,8 +181,9 @@ def insert_api_into_dict(full_name, gen_doc_anno=None): logger.warning("AttributeError occurred when `id(eval(%s))`", full_name) return None except Exception as e: - logger.warning("Exception(%s) occurred when `id(eval(%s))`", str(e), - full_name) + logger.warning( + "Exception(%s) occurred when `id(eval(%s))`", str(e), full_name + ) return None else: logger.debug("adding %s to api_info_dict.", full_name) @@ -187,8 +204,8 @@ def insert_api_into_dict(full_name, gen_doc_anno=None): api_info_dict[fc_id]["gen_doc_anno"] = gen_doc_anno if inspect.isfunction(obj): api_info_dict[fc_id]["signature"] = repr( - inspect.getfullargspec(obj)).replace( - 'FullArgSpec', 'ArgSpec', 1) + inspect.getfullargspec(obj) + ).replace('FullArgSpec', 'ArgSpec', 1) return api_info_dict[fc_id] @@ -198,9 +215,11 @@ def process_module(m, attr="__all__"): if hasattr(m, attr): # may have duplication of api for api in set(getattr(m, attr)): - if api[0] == '_': continue + if api[0] == '_': + continue # Exception occurred when `id(eval(paddle.dataset.conll05.test, get_dict))` - if ',' in api: continue + if ',' in api: + continue # api's fullname full_name = m.__name__ + "." + api @@ -210,17 +229,21 @@ def process_module(m, attr="__all__"): if inspect.isclass(api_info['object']): for name, value in inspect.getmembers(api_info['object']): if (not name.startswith("_")) and hasattr( - value, '__name__'): - method_full_name = full_name + '.' + name # value.__name__ + value, '__name__' + ): + method_full_name = ( + full_name + '.' + name + ) # value.__name__ method_api_info = insert_api_into_dict( - method_full_name, 'class_method') + method_full_name, 'class_method' + ) if method_api_info is not None: api_counter += 1 return api_counter def check_public_api(): - modulelist = [ #npqa + modulelist = [ # npqa paddle, paddle.amp, paddle.nn, @@ -297,7 +320,8 @@ def check_public_api(): instance = eval(cur_name) doc_md5 = md5(instance.__doc__) member_dict[cur_name] = "({}, ('document', '{}'))".format( - cur_name, doc_md5) + cur_name, doc_md5 + ) def check_allmodule_callable(): @@ -314,18 +338,23 @@ def parse_args(): """ parser = argparse.ArgumentParser(description='Print Apis Signatures') parser.add_argument('--debug', dest='debug', action="store_true") - parser.add_argument('--method', - dest='method', - type=str, - default='get_all_api', - help="using get_all_api or from_modulelist") - parser.add_argument('module', type=str, help='module', - default='paddle') # not used - parser.add_argument('--skipped', - dest='skipped', - type=str, - help='Skip Checking submodules', - default='paddle.fluid.libpaddle.eager.ops') + parser.add_argument( + '--method', + dest='method', + type=str, + default='get_all_api', + help="using get_all_api or from_modulelist", + ) + parser.add_argument( + 'module', type=str, help='module', default='paddle' + ) # not used + parser.add_argument( + '--skipped', + dest='skipped', + type=str, + help='Skip Checking submodules', + default='paddle.fluid.libpaddle.eager.ops', + ) if len(sys.argv) == 1: args = parser.parse_args(['paddle']) @@ -357,14 +386,22 @@ if __name__ == '__main__': if args.skipped != '' and api_name.find(args.skipped) >= 0: continue api_info = api_info_dict[all_api_names_to_k[api_name]] - print("{0} ({2}, ('document', '{1}'))".format( - api_name, md5(api_info['docstring']), api_info['signature'] - if 'signature' in api_info else 'ArgSpec()')) + print( + "{0} ({2}, ('document', '{1}'))".format( + api_name, + md5(api_info['docstring']), + api_info['signature'] + if 'signature' in api_info + else 'ArgSpec()', + ) + ) if len(ErrorSet) == 0: sys.exit(0) else: for erroritem in ErrorSet: - print("Error, new function {} is unreachable".format(erroritem), - file=sys.stderr) + print( + "Error, new function {} is unreachable".format(erroritem), + file=sys.stderr, + ) sys.exit(1) diff --git a/tools/prune_for_jetson.py b/tools/prune_for_jetson.py index e87268a9fe2..8bfe9ebf9f8 100644 --- a/tools/prune_for_jetson.py +++ b/tools/prune_for_jetson.py @@ -53,16 +53,20 @@ def find_kernel(content, pattern): def prune_phi_kernels(): tool_dir = os.path.dirname(os.path.abspath(__file__)) - all_op = glob.glob(os.path.join(tool_dir, '../paddle/phi/kernels/**/*.cc'), - recursive=True) - all_op += glob.glob(os.path.join(tool_dir, '../paddle/phi/kernels/**/*.cu'), - recursive=True) + all_op = glob.glob( + os.path.join(tool_dir, '../paddle/phi/kernels/**/*.cc'), recursive=True + ) + all_op += glob.glob( + os.path.join(tool_dir, '../paddle/phi/kernels/**/*.cu'), recursive=True + ) register_op_count = 0 for op_file in all_op: need_continue = False file_blacklist = [ - "kernels/empty_kernel.cc", "/cast_kernel.c", "/batch_norm_kernel.c" + "kernels/empty_kernel.cc", + "/cast_kernel.c", + "/batch_norm_kernel.c", ] for bname in file_blacklist: if op_file.find(bname) >= 0: @@ -99,19 +103,25 @@ def apply_patches(): ret = os.system( "cd %s && rm -f paddle/fluid/inference/api/tensorrt_predictor.* " " && rm -f paddle/fluid/inference/api/paddle_tensorrt_predictor.h " - " && git apply tools/infer_prune_patches/*.patch && cd -" % work_path) + " && git apply tools/infer_prune_patches/*.patch && cd -" % work_path + ) return ret == 0 def append_fluid_kernels(): op_white_list = ["load", "load_combine"] - #1. add to makefile - file_name = os.path.dirname(os.path.abspath(__file__)) \ - + "/../paddle/fluid/inference/tensorrt/CMakeLists.txt" + # 1. add to makefile + file_name = ( + os.path.dirname(os.path.abspath(__file__)) + + "/../paddle/fluid/inference/tensorrt/CMakeLists.txt" + ) append_str = "\nfile(APPEND ${pybind_file} \"USE_NO_KERNEL_OP__(tensorrt_engine);\\n\")\n" for op in op_white_list: - append_str = append_str + "file(APPEND ${pybind_file} \"USE_OP__(%s);\\n\")\n" % op + append_str = ( + append_str + + "file(APPEND ${pybind_file} \"USE_OP__(%s);\\n\")\n" % op + ) with open(file_name, 'r', encoding='utf-8') as f: content = ''.join(f.readlines()) @@ -120,22 +130,26 @@ def append_fluid_kernels(): new_content = content.replace(location_str, location_str + append_str) if new_content == content: - print("ERROR: can not find \"%s\" in file \"%s\"" % - (location_str, file_name)) + print( + "ERROR: can not find \"%s\" in file \"%s\"" + % (location_str, file_name) + ) return False with open(file_name, 'w', encoding='utf-8') as f: f.write(u'{}'.format(new_content)) - #2. add op and kernel register + # 2. add op and kernel register op_white_list.append("tensorrt_engine") tool_dir = os.path.dirname(os.path.abspath(__file__)) - all_op = glob.glob(os.path.join(tool_dir, - '../paddle/fluid/operators/**/*.cc'), - recursive=True) - all_op += glob.glob(os.path.join(tool_dir, - '../paddle/fluid/operators/**/*.cu'), - recursive=True) + all_op = glob.glob( + os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cc'), + recursive=True, + ) + all_op += glob.glob( + os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cu'), + recursive=True, + ) for op_file in all_op: with open(op_file, 'r', encoding='utf-8') as f: @@ -143,18 +157,18 @@ def append_fluid_kernels(): for op in op_white_list: patterns = { - "REGISTER_OPERATOR": - r"REGISTER_OPERATOR\(\s*%s\s*," % op, - "REGISTER_OP_CPU_KERNEL": - r"REGISTER_OP_CPU_KERNEL\(\s*%s\s*," % op, - "REGISTER_OP_CUDA_KERNEL": - r"REGISTER_OP_CUDA_KERNEL\(\s*%s\s*," % op + "REGISTER_OPERATOR": r"REGISTER_OPERATOR\(\s*%s\s*," % op, + "REGISTER_OP_CPU_KERNEL": r"REGISTER_OP_CPU_KERNEL\(\s*%s\s*," + % op, + "REGISTER_OP_CUDA_KERNEL": r"REGISTER_OP_CUDA_KERNEL\(\s*%s\s*," + % op, } for k, p in patterns.items(): matches = re.findall(p, content, flags=re.DOTALL) if len(matches) > 0: - content = content.replace(matches[0], - matches[0].replace(k, k + "__")) + content = content.replace( + matches[0], matches[0].replace(k, k + "__") + ) with open(op_file, 'w', encoding='utf-8') as f: f.write(u'{}'.format(content)) @@ -164,13 +178,13 @@ def append_fluid_kernels(): if __name__ == '__main__': print("================ step 1: apply patches =======================") - assert (apply_patches()) + assert apply_patches() print("==============================================================\n") print("================ step 2: append fluid op/kernels==============") - assert (append_fluid_kernels()) + assert append_fluid_kernels() print("==============================================================\n") print("================ step 3:prune phi kernels ====================") - assert (prune_phi_kernels()) + assert prune_phi_kernels() print("==============================================================\n") diff --git a/tools/pyCov_multithreading.py b/tools/pyCov_multithreading.py index fb4cef6805f..e808dcb55e0 100644 --- a/tools/pyCov_multithreading.py +++ b/tools/pyCov_multithreading.py @@ -32,9 +32,12 @@ def worker(fun): def threadPool(threadPoolNum): threadPool = [] for i in range(threadPoolNum): - thread = threading.Thread(target=worker, args={ - doFun, - }) + thread = threading.Thread( + target=worker, + args={ + doFun, + }, + ) thread.daemon = True threadPool.append(thread) return threadPool @@ -49,8 +52,9 @@ def getPyCovResult(params): os.system('cd %s && coverage combine `ls python-coverage.data.*`' % path) os.system('cd %s && pwd && coverage xml -i -o python-coverage.xml' % path) xml_path = '%s/python-coverage.xml' % path - os.system("python2.7 %s/tools/analysisPyXml.py %s %s" % - (rootPath, rootPath, ut)) + os.system( + "python2.7 %s/tools/analysisPyXml.py %s %s" % (rootPath, rootPath, ut) + ) endTime = int(time.time()) print('pyCov Time: %s' % (endTime - startTime)) diff --git a/tools/remove_grad_op_and_kernel.py b/tools/remove_grad_op_and_kernel.py index 19778c27dbe..8aa456bdb67 100644 --- a/tools/remove_grad_op_and_kernel.py +++ b/tools/remove_grad_op_and_kernel.py @@ -54,8 +54,11 @@ def update_operator_cmake(cmake_file): match = re.findall(pat2, content, flags=re.DOTALL) content = content.replace( - match[0], code2 + '\n' + - match[0].replace('py_func_op', 'py_func_op ${LOSS_OPS}')) + match[0], + code2 + + '\n' + + match[0].replace('py_func_op', 'py_func_op ${LOSS_OPS}'), + ) with open(cmake_file, 'w') as f: f.write(content) @@ -65,16 +68,23 @@ if __name__ == '__main__': tool_dir = os.path.dirname(os.path.abspath(__file__)) - all_op = glob.glob(os.path.join(tool_dir, - '../paddle/fluid/operators/**/*.cc'), - recursive=True) - all_op += glob.glob(os.path.join(tool_dir, - '../paddle/fluid/operators/**/*.cu'), - recursive=True) + all_op = glob.glob( + os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cc'), + recursive=True, + ) + all_op += glob.glob( + os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cu'), + recursive=True, + ) spec_ops = ['activation_op.cc'] - register_op_count, register_op_cpu_kernel_count, register_op_cuda_kernel_count, register_op_xpu_kernel_count = 0, 0, 0, 0 + ( + register_op_count, + register_op_cpu_kernel_count, + register_op_cuda_kernel_count, + register_op_xpu_kernel_count, + ) = (0, 0, 0, 0) register_op_kernel_count, register_op_kernel_with_custom_type_count = 0, 0 # 1. remove all grad op and kernel @@ -100,7 +110,9 @@ if __name__ == '__main__': op_kernel_pattern2 = r'REGISTER_OP_KERNEL\(.*?_grad,.*?\);?' custom_pattern1 = r'REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE\(.*?\);?' - custom_pattern2 = r'REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE\(.*?_grad,.*?\);?' + custom_pattern2 = ( + r'REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE\(.*?_grad,.*?\);?' + ) op_name = os.path.split(op_file)[1] if op_name in spec_ops: @@ -121,18 +133,24 @@ if __name__ == '__main__': with open(op_file, 'r', encoding='utf-8') as f: content = ''.join(f.readlines()) - op, op_count = remove_grad_op_and_kernel(content, op_pattern1, - op_pattern2) + op, op_count = remove_grad_op_and_kernel( + content, op_pattern1, op_pattern2 + ) cpu_kernel, cpu_kernel_count = remove_grad_op_and_kernel( - content, cpu_kernel_pattern1, cpu_kernel_pattern2) + content, cpu_kernel_pattern1, cpu_kernel_pattern2 + ) gpu_kernel, gpu_kernel_count = remove_grad_op_and_kernel( - content, gpu_kernel_pattern1, gpu_kernel_pattern2) + content, gpu_kernel_pattern1, gpu_kernel_pattern2 + ) xpu_kernel, xpu_kernel_count = remove_grad_op_and_kernel( - content, xpu_kernel_pattern1, xpu_kernel_pattern2) + content, xpu_kernel_pattern1, xpu_kernel_pattern2 + ) op_kernel, op_kernel_count = remove_grad_op_and_kernel( - content, op_kernel_pattern1, op_kernel_pattern2) + content, op_kernel_pattern1, op_kernel_pattern2 + ) custom_kernel, custom_kernel_count = remove_grad_op_and_kernel( - content, custom_pattern1, custom_pattern2) + content, custom_pattern1, custom_pattern2 + ) register_op_count += op_count register_op_cpu_kernel_count += cpu_kernel_count @@ -155,17 +173,24 @@ if __name__ == '__main__': f.write(u'{}'.format(content)) # 2. update operators/CMakeLists.txt - cmake_file = os.path.join(tool_dir, - '../paddle/fluid/operators/CMakeLists.txt') + cmake_file = os.path.join( + tool_dir, '../paddle/fluid/operators/CMakeLists.txt' + ) update_operator_cmake(cmake_file) print('We erase all grad op and kernel for Paddle-Inference lib.') print('%50s%10s' % ('type', 'count')) print('%50s%10s' % ('REGISTER_OPERATOR', register_op_count)) print('%50s%10s' % ('REGISTER_OP_CPU_KERNEL', register_op_cpu_kernel_count)) - print('%50s%10s' % - ('REGISTER_OP_CUDA_KERNEL', register_op_cuda_kernel_count)) + print( + '%50s%10s' % ('REGISTER_OP_CUDA_KERNEL', register_op_cuda_kernel_count) + ) print('%50s%10s' % ('REGISTER_OP_XPU_KERNEL', register_op_xpu_kernel_count)) print('%50s%10s' % ('REGISTER_OP_KERNEL', register_op_kernel_count)) - print('%50s%10s' % ('REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE', - register_op_kernel_with_custom_type_count)) + print( + '%50s%10s' + % ( + 'REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE', + register_op_kernel_with_custom_type_count, + ) + ) diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py index 7e624652ad5..24af09893e3 100644 --- a/tools/sampcd_processor.py +++ b/tools/sampcd_processor.py @@ -36,7 +36,8 @@ import time logger = logging.getLogger() if logger.handlers: console = logger.handlers[ - 0] # we assume the first handler is the one we want to configure + 0 + ] # we assume the first handler is the one we want to configure else: console = logging.StreamHandler(stream=sys.stderr) logger.addHandler(console) @@ -76,7 +77,7 @@ def find_all(srcstr, substr): """ indices = [] gotone = srcstr.find(substr) - while (gotone != -1): + while gotone != -1: indices.append(gotone) gotone = srcstr.find(substr, gotone + 1) return indices @@ -124,7 +125,7 @@ def extract_code_blocks_from_docstr(docstr): mo = re.search(r"Examples:", docstr) if mo is None: return code_blocks - ds_list = docstr[mo.start():].replace("\t", ' ').split("\n") + ds_list = docstr[mo.start() :].replace("\t", ' ').split("\n") lastlineindex = len(ds_list) - 1 cb_start_pat = re.compile(r"code-block::\s*python") @@ -148,16 +149,14 @@ def extract_code_blocks_from_docstr(docstr): def _append_code_block(): # nonlocal code_blocks, cb_cur, cb_cur_name, cb_cur_seq_id, cb_required - code_blocks.append({ - 'codes': - inspect.cleandoc("\n".join(cb_info['cb_cur'])), - 'name': - cb_info['cb_cur_name'], - 'id': - cb_info['cb_cur_seq_id'], - 'required': - cb_info['cb_required'], - }) + code_blocks.append( + { + 'codes': inspect.cleandoc("\n".join(cb_info['cb_cur'])), + 'name': cb_info['cb_cur_name'], + 'id': cb_info['cb_cur_seq_id'], + 'required': cb_info['cb_required'], + } + ) for lineno, linecont in enumerate(ds_list): if re.search(cb_start_pat, linecont): @@ -186,7 +185,9 @@ def extract_code_blocks_from_docstr(docstr): # docstring end if lineno == lastlineindex: mo = re.search(r"\S", linecont) - if mo is not None and cb_info['cb_cur_indent'] <= mo.start( + if ( + mo is not None + and cb_info['cb_cur_indent'] <= mo.start() ): cb_info['cb_cur'].append(linecont) if len(cb_info['cb_cur']): @@ -262,14 +263,21 @@ def is_required_match(requirestr, cbtitle='not-specified'): logger.info('%s: skipped', cbtitle) return None - if all([ - k in SAMPLE_CODE_TEST_CAPACITY for k in requires + if all( + [ + k in SAMPLE_CODE_TEST_CAPACITY + for k in requires if k not in ['skip', 'skiptest'] - ]): + ] + ): return True - logger.info('%s: the equipments [%s] not match the required [%s].', cbtitle, - ','.join(SAMPLE_CODE_TEST_CAPACITY), ','.join(requires)) + logger.info( + '%s: the equipments [%s] not match the required [%s].', + cbtitle, + ','.join(SAMPLE_CODE_TEST_CAPACITY), + ','.join(requires), + ) return False @@ -280,13 +288,18 @@ def insert_codes_into_codeblock(codeblock, apiname='not-specified'): global ENV_KEY_CODES_FRONTEND, GPU_ID, RUN_ON_DEVICE # readonly inserted_codes_f = '' inserted_codes_b = '' - if ENV_KEY_CODES_FRONTEND in os.environ and os.environ[ - ENV_KEY_CODES_FRONTEND]: + if ( + ENV_KEY_CODES_FRONTEND in os.environ + and os.environ[ENV_KEY_CODES_FRONTEND] + ): inserted_codes_f = os.environ[ENV_KEY_CODES_FRONTEND] else: cpu_str = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = ""\n' - gpu_str = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = "{}"\n'.format( - GPU_ID) + gpu_str = ( + '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = "{}"\n'.format( + GPU_ID + ) + ) if 'required' in codeblock and codeblock['required']: if codeblock['required'] == 'cpu': inserted_codes_f = cpu_str @@ -298,13 +311,18 @@ def insert_codes_into_codeblock(codeblock, apiname='not-specified'): elif RUN_ON_DEVICE == "gpu": inserted_codes_f = gpu_str inserted_codes_b = '\nprint("{}\'s sample code (name:{}, id:{}) is executed successfully!")'.format( - apiname, codeblock['name'], codeblock['id']) + apiname, codeblock['name'], codeblock['id'] + ) cb = codeblock['codes'] last_future_line_end = find_last_future_line_end(cb) if last_future_line_end: - return cb[:last_future_line_end] + inserted_codes_f + cb[ - last_future_line_end:] + inserted_codes_b + return ( + cb[:last_future_line_end] + + inserted_codes_f + + cb[last_future_line_end:] + + inserted_codes_b + ) else: return inserted_codes_f + cb + inserted_codes_b @@ -335,12 +353,14 @@ def sampcd_extract_to_file(srccom, name, htype="def", hname=""): if srccom.find("Examples:") != -1: logger.info("----example code check----") if srccom.find(">>>") != -1: - logger.warning(r"""Deprecated sample code style: + logger.warning( + r"""Deprecated sample code style: Examples: >>>codeline >>>codeline -Please use '.. code-block:: python' to format the sample code.""") +Please use '.. code-block:: python' to format the sample code.""" + ) return [] else: logger.error( @@ -358,22 +378,33 @@ Please use '.. code-block:: python' to format the sample code.""") # so, the following conditional statements are intentionally arranged. if matched == True: tfname = os.path.join( - SAMPLECODE_TEMPDIR, '{}_example{}'.format( + SAMPLECODE_TEMPDIR, + '{}_example{}'.format( name, - '.py' if len(codeblocks) == 1 else '_{}.py'.format(y + 1))) + '.py' if len(codeblocks) == 1 else '_{}.py'.format(y + 1), + ), + ) with open(tfname, 'w') as tempf: sampcd = insert_codes_into_codeblock(cb, name) tempf.write(sampcd) sample_code_filenames.append(tfname) elif matched is None: - logger.info('{}\' code block (name:{}, id:{}) is skipped.'.format( - name, cb['name'], cb['id'])) + logger.info( + '{}\' code block (name:{}, id:{}) is skipped.'.format( + name, cb['name'], cb['id'] + ) + ) SUMMARY_INFO['skiptest'].append("{}-{}".format(name, cb['id'])) elif matched == False: logger.info( - '{}\' code block (name:{}, id:{}) required({}) not match capacity({}).' - .format(name, cb['name'], cb['id'], cb['required'], - SAMPLE_CODE_TEST_CAPACITY)) + '{}\' code block (name:{}, id:{}) required({}) not match capacity({}).'.format( + name, + cb['name'], + cb['id'], + cb['required'], + SAMPLE_CODE_TEST_CAPACITY, + ) + ) if cb['required'] not in SUMMARY_INFO: SUMMARY_INFO[cb['required']] = [] SUMMARY_INFO[cb['required']].append("{}-{}".format(name, cb['id'])) @@ -406,9 +437,9 @@ def execute_samplecode(tfname): logger.info("----example code check----") logger.info("executing sample code: %s", tfname) start_time = time.time() - subprc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) output, error = subprc.communicate() msg = "".join(output.decode(encoding='utf-8')) err = "".join(error.decode(encoding='utf-8')) @@ -425,7 +456,13 @@ subprocess return code: %d Error Raised from Sample Code: stderr: %s stdout: %s -""", tfname, f.read(), subprc.returncode, err, msg) +""", + tfname, + f.read(), + subprc.returncode, + err, + msg, + ) logger.info("----example code check failed----") result = False else: @@ -450,6 +487,7 @@ def get_filenames(full_test=False): global whl_error import paddle # noqa: F401 import paddle.fluid.contrib.slim.quantization # noqa: F401 + whl_error = [] if full_test: get_full_api_from_pr_spec() @@ -470,7 +508,8 @@ def get_filenames(full_test=False): continue if hasattr(api_obj, '__doc__') and api_obj.__doc__: sample_code_filenames = sampcd_extract_to_file( - api_obj.__doc__, api) + api_obj.__doc__, api + ) for tfname in sample_code_filenames: all_sample_code_filenames[tfname] = api return all_sample_code_filenames @@ -492,7 +531,8 @@ def get_api_md5(path): return api_md5 pat = re.compile(r'\((paddle[^,]+)\W*document\W*([0-9a-z]{32})') patArgSpec = re.compile( - r'^(paddle[^,]+)\s+\(ArgSpec.*document\W*([0-9a-z]{32})') + r'^(paddle[^,]+)\s+\(ArgSpec.*document\W*([0-9a-z]{32})' + ) with open(API_spec) as f: for line in f.readlines(): mo = pat.search(line) @@ -509,6 +549,7 @@ def get_full_api(): """ global API_DIFF_SPEC_FN ## readonly from print_signatures import get_all_api_from_modulelist + member_dict = get_all_api_from_modulelist() with open(API_DIFF_SPEC_FN, 'w') as f: f.write("\n".join(member_dict.keys())) @@ -520,6 +561,7 @@ def get_full_api_by_walk(): """ global API_DIFF_SPEC_FN ## readonly from print_signatures import get_all_api + apilist = get_all_api() with open(API_DIFF_SPEC_FN, 'w') as f: f.write("\n".join([ai[0] for ai in apilist])) @@ -549,8 +591,12 @@ def get_incrementapi(): for key in pr_api: if key in dev_api: if dev_api[key] != pr_api[key]: - logger.debug("%s in dev is %s, different from pr's %s", key, - dev_api[key], pr_api[key]) + logger.debug( + "%s in dev is %s, different from pr's %s", + key, + dev_api[key], + pr_api[key], + ) f.write(key) f.write('\n') else: @@ -564,9 +610,9 @@ def exec_gen_doc(): cmd = ["bash", "document_preview.sh"] logger.info("----exec gen_doc----") start_time = time.time() - subprc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) output, error = subprc.communicate() msg = "".join(output.decode(encoding='utf-8')) err = "".join(error.decode(encoding='utf-8')) @@ -585,7 +631,8 @@ def exec_gen_doc(): logger.info("----exec gen_doc success----") for fn in [ - '/docs/en/develop/index_en.html', '/docs/zh/develop/index_cn.html' + '/docs/en/develop/index_en.html', + '/docs/zh/develop/index_cn.html', ]: if os.path.exists(fn): logger.info('%s exists.', fn) @@ -616,16 +663,16 @@ def parse_args(): parser.add_argument('--debug', dest='debug', action="store_true") parser.add_argument('--full-test', dest='full_test', action="store_true") parser.add_argument('mode', type=str, help='run on device', default='cpu') - parser.add_argument('--build-doc', - dest='build_doc', - action='store_true', - help='build doc if need.') + parser.add_argument( + '--build-doc', + dest='build_doc', + action='store_true', + help='build doc if need.', + ) for item in arguments: - parser.add_argument(item[0], - dest=item[1], - help=item[4], - type=item[2], - default=item[3]) + parser.add_argument( + item[0], dest=item[1], help=item[4], type=item[2], default=item[3] + ) if len(sys.argv) == 1: args = parser.parse_args(['cpu']) @@ -648,21 +695,24 @@ if __name__ == '__main__': logfHandler.setFormatter( logging.Formatter( "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s" - )) + ) + ) logger.addHandler(logfHandler) if args.mode == "gpu": GPU_ID = args.gpu_id logger.info("using GPU_ID %d", GPU_ID) elif args.mode != "cpu": - logger.error("Unrecognized argument:%s, 'cpu' or 'gpu' is desired.", - args.mode) + logger.error( + "Unrecognized argument:%s, 'cpu' or 'gpu' is desired.", args.mode + ) sys.exit("Invalid arguments") RUN_ON_DEVICE = args.mode get_test_capacity() logger.info("API check -- Example Code") - logger.info("sample_test running under python %s", - platform.python_version()) + logger.info( + "sample_test running under python %s", platform.python_version() + ) if os.path.exists(SAMPLECODE_TEMPDIR): if not os.path.isdir(SAMPLECODE_TEMPDIR): @@ -706,16 +756,18 @@ if __name__ == '__main__': ) for temp in result: if not temp[0]: - logger.info("In addition, mistakes found in sample codes: %s", - temp[1]) + logger.info( + "In addition, mistakes found in sample codes: %s", temp[1] + ) logger.info("----------------------------------------------------") exit(1) else: timeovered_test = {} for temp in result: if not temp[0]: - logger.info("In addition, mistakes found in sample codes: %s", - temp[1]) + logger.info( + "In addition, mistakes found in sample codes: %s", temp[1] + ) SUMMARY_INFO['failed'].append(temp[1]) else: SUMMARY_INFO['success'].append(temp[1]) @@ -723,31 +775,37 @@ if __name__ == '__main__': timeovered_test[temp[1]] = temp[3] if len(timeovered_test): - logger.info("%d sample codes ran time over 10s", - len(timeovered_test)) + logger.info( + "%d sample codes ran time over 10s", len(timeovered_test) + ) if args.debug: for k, v in timeovered_test.items(): logger.info('{} - {}s'.format(k, v)) if len(SUMMARY_INFO['success']): - logger.info("%d sample codes ran success", - len(SUMMARY_INFO['success'])) + logger.info( + "%d sample codes ran success", len(SUMMARY_INFO['success']) + ) for k, v in SUMMARY_INFO.items(): if k not in ['success', 'failed', 'skiptest', 'nocodes']: - logger.info("%d sample codes required not match for %s", len(v), - k) + logger.info( + "%d sample codes required not match for %s", len(v), k + ) if len(SUMMARY_INFO['skiptest']): - logger.info("%d sample codes skipped", - len(SUMMARY_INFO['skiptest'])) + logger.info( + "%d sample codes skipped", len(SUMMARY_INFO['skiptest']) + ) if args.debug: logger.info('\n'.join(SUMMARY_INFO['skiptest'])) if len(SUMMARY_INFO['nocodes']): - logger.info("%d apis don't have sample codes", - len(SUMMARY_INFO['nocodes'])) + logger.info( + "%d apis don't have sample codes", len(SUMMARY_INFO['nocodes']) + ) if args.debug: logger.info('\n'.join(SUMMARY_INFO['nocodes'])) if len(SUMMARY_INFO['failed']): - logger.info("%d sample codes ran failed", - len(SUMMARY_INFO['failed'])) + logger.info( + "%d sample codes ran failed", len(SUMMARY_INFO['failed']) + ) logger.info('\n'.join(SUMMARY_INFO['failed'])) logger.info( "Mistakes found in sample codes. Please recheck the sample codes." diff --git a/tools/summary_env.py b/tools/summary_env.py index 2abee2821c0..14fb512ca22 100644 --- a/tools/summary_env.py +++ b/tools/summary_env.py @@ -34,6 +34,7 @@ envs = {} def get_paddle_info(): try: import paddle + envs['paddle_version'] = paddle.__version__ envs['paddle_with_cuda'] = paddle.fluid.core.is_compiled_with_cuda() except: @@ -63,10 +64,9 @@ def get_python_info(): def run_shell_command(cmd): - out, err = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True).communicate() + out, err = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ).communicate() if err: return None else: @@ -82,7 +82,6 @@ def get_cuda_info(): def get_cudnn_info(): - def _get_cudnn_ver(cmd): out = run_shell_command(cmd) if out: @@ -93,15 +92,17 @@ def get_cudnn_info(): if platform.system() == "Windows": cudnn_dll_path = run_shell_command('where cudnn*') if cudnn_dll_path: - cudnn_header_path = cudnn_dll_path.split( - 'bin')[0] + r'include\cudnn.h' + cudnn_header_path = ( + cudnn_dll_path.split('bin')[0] + r'include\cudnn.h' + ) cmd = 'type "{0}" | findstr "{1}" | findstr /v "CUDNN_VERSION"' else: envs['cudnn_version'] = None return else: cudnn_header_path = run_shell_command( - 'whereis "cudnn.h" | awk \'{print $2}\'') + 'whereis "cudnn.h" | awk \'{print $2}\'' + ) if cudnn_header_path: cudnn_header_path = cudnn_header_path.strip() cmd = 'cat "{0}" | grep "{1}" | grep -v "CUDNN_VERSION"' @@ -112,7 +113,8 @@ def get_cudnn_info(): major = _get_cudnn_ver(cmd.format(cudnn_header_path, 'CUDNN_MAJOR')) minor = _get_cudnn_ver(cmd.format(cudnn_header_path, 'CUDNN_MINOR')) patch_level = _get_cudnn_ver( - cmd.format(cudnn_header_path, 'CUDNN_PATCHLEVEL')) + cmd.format(cudnn_header_path, 'CUDNN_PATCHLEVEL') + ) envs['cudnn_version'] = "{0}.{1}.{2}".format(major, minor, patch_level) @@ -120,8 +122,9 @@ def get_cudnn_info(): def get_driver_info(): driver_ver = run_shell_command('nvidia-smi') if driver_ver: - driver_ver = driver_ver.split('Driver Version:')[1].strip().split( - ' ')[0] + driver_ver = ( + driver_ver.split('Driver Version:')[1].strip().split(' ')[0] + ) else: driver_ver = None envs['nvidia_driver_version'] = driver_ver diff --git a/tools/test_check_api_compatible.py b/tools/test_check_api_compatible.py index 1bed3951740..d7c9a0b195d 100644 --- a/tools/test_check_api_compatible.py +++ b/tools/test_check_api_compatible.py @@ -25,10 +25,12 @@ from check_api_compatible import check_compatible_str class Test_check_compatible(unittest.TestCase): - def setUp(self) -> None: self.fullargspec_prefix = 'inspect.Full' - self.argspec_str_o = self.fullargspec_prefix + '''ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={})''' + self.argspec_str_o = ( + self.fullargspec_prefix + + '''ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={})''' + ) return super().setUp() def test_normal_not_changed(self): @@ -68,11 +70,13 @@ class Test_check_compatible(unittest.TestCase): class Test_check_compatible_str(unittest.TestCase): - def setUp(self) -> None: self.fullargspec_prefix = 'inspect.Full' # paddle.fluid.layer_helper_base.LayerHelperBase.create_parameter - self.argspec_str_o = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + self.argspec_str_o = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) return super().setUp() def test_normal_not_changed(self): @@ -81,26 +85,44 @@ class Test_check_compatible_str(unittest.TestCase): self.assertTrue(check_compatible_str(argspec_o, argspec_n)) def test_args_added(self): - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) argspec_o = self.argspec_str_o self.assertFalse(check_compatible_str(argspec_o, argspec_n)) - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR, argadded), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR, argadded), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) self.assertTrue(check_compatible_str(argspec_o, argspec_n)) - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR, argadded, 1), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR, argadded, 1), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) self.assertFalse(check_compatible_str(argspec_o, argspec_n)) - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(1, None, False, None, False, VarType.LOD_TENSOR, argadded), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(1, None, False, None, False, VarType.LOD_TENSOR, argadded), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) self.assertTrue(check_compatible_str(argspec_o, argspec_n)) def test_args_places_exchanged(self): - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'type', 'stop_gradient'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'type', 'stop_gradient'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) argspec_o = self.argspec_str_o self.assertFalse(check_compatible_str(argspec_o, argspec_n)) def test_args_reduced(self): - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) argspec_o = self.argspec_str_o self.assertFalse(check_compatible_str(argspec_o, argspec_n)) @@ -111,17 +133,23 @@ class Test_check_compatible_str(unittest.TestCase): class Test_read_argspec_from_file(unittest.TestCase): - def setUp(self) -> None: self.fullargspec_prefix = 'inspect.Full' - self.argspec_str_o = self.fullargspec_prefix + '''ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={})''' + self.argspec_str_o = ( + self.fullargspec_prefix + + '''ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={})''' + ) self.api_spec_file = tempfile.TemporaryFile('w+t') if self.api_spec_file: - self.api_spec_file.write("\n".join([ - """paddle.ones (ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={}), ('document', '50a3b3a77fa13bb2ae4337d8f9d091b7'))""", - # """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", - """paddle.five_plus_five (ArgSpec(), ('document', 'ff0f188c95030158cc6398d2a6c5five'))""", - ])) + self.api_spec_file.write( + "\n".join( + [ + """paddle.ones (ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={}), ('document', '50a3b3a77fa13bb2ae4337d8f9d091b7'))""", + # """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", + """paddle.five_plus_five (ArgSpec(), ('document', 'ff0f188c95030158cc6398d2a6c5five'))""", + ] + ) + ) self.api_spec_file.seek(0) return super().setUp() @@ -134,9 +162,11 @@ class Test_read_argspec_from_file(unittest.TestCase): api_argspec_dict = read_argspec_from_file(self.api_spec_file) argspec = eval(self.argspec_str_o) self.assertEqual( - api_argspec_dict.get('paddle.ones').args, argspec.args) + api_argspec_dict.get('paddle.ones').args, argspec.args + ) self.assertEqual( - api_argspec_dict.get('paddle.ones').defaults, argspec.defaults) + api_argspec_dict.get('paddle.ones').defaults, argspec.defaults + ) self.assertIsNone(api_argspec_dict.get('paddle.five_plus_five')) else: self.fail('api_spec_file error') diff --git a/tools/test_check_pr_approval.py b/tools/test_check_pr_approval.py index 5f3c7ca11cc..c126a749f1b 100644 --- a/tools/test_check_pr_approval.py +++ b/tools/test_check_pr_approval.py @@ -22,7 +22,6 @@ import sys class Test_check_approval(unittest.TestCase): - def setUp(self): self.codeset = 'UTF-8' # only key info in it @@ -68,47 +67,64 @@ class Test_check_approval(unittest.TestCase): "author_association": "CONTRIBUTOR" } ] -""".encode(self.codeset) +""".encode( + self.codeset + ) def test_ids(self): cmd = [sys.executable, 'check_pr_approval.py', '1', '26408901'] - subprc = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output, error = subprc.communicate(input=self.jsonstr) self.assertEqual('TRUE', output.decode(self.codeset).rstrip()) def test_logins(self): cmd = [sys.executable, 'check_pr_approval.py', '1', 'pangyoki'] - subprc = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output, error = subprc.communicate(input=self.jsonstr) self.assertEqual('TRUE', output.decode(self.codeset).rstrip()) def test_ids_and_logins(self): cmd = [ - sys.executable, 'check_pr_approval.py', '2', 'pangyoki', '13469016' + sys.executable, + 'check_pr_approval.py', + '2', + 'pangyoki', + '13469016', ] - subprc = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output, error = subprc.communicate(input=self.jsonstr) - #self.assertEqual('', error.rstrip()) + # self.assertEqual('', error.rstrip()) self.assertEqual('TRUE', output.decode(self.codeset).rstrip()) def test_check_with_required_reviewer_not_approved(self): cmd = [ - sys.executable, 'check_pr_approval.py', '2', 'wadefelix', - ' 13469016' + sys.executable, + 'check_pr_approval.py', + '2', + 'wadefelix', + ' 13469016', ] - subprc = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output, error = subprc.communicate(input=self.jsonstr) self.assertEqual('FALSE', output.decode(self.codeset).rstrip()) diff --git a/tools/test_print_signatures.py b/tools/test_print_signatures.py index 14275b6b7ae..c757fcce967 100644 --- a/tools/test_print_signatures.py +++ b/tools/test_print_signatures.py @@ -42,7 +42,7 @@ def func_example_2(func=functools.partial(func_example, 1)): pass -class ClassExample(): +class ClassExample: """ example Class """ @@ -55,7 +55,6 @@ class ClassExample(): class Test_all_in_print_signatures(unittest.TestCase): - def test_md5(self): algo = hashlib.md5() algo.update(func_example.__doc__.encode('utf-8')) @@ -64,13 +63,13 @@ class Test_all_in_print_signatures(unittest.TestCase): class Test_is_primitive(unittest.TestCase): - def test_single(self): self.assertTrue(is_primitive(2)) self.assertTrue(is_primitive(2.1)) self.assertTrue(is_primitive("2.1.1")) - self.assertFalse(is_primitive( - "hello paddle".encode('UTF-8'))) # True for python2 + self.assertFalse( + is_primitive("hello paddle".encode('UTF-8')) + ) # True for python2 self.assertFalse(is_primitive(1j)) self.assertTrue(is_primitive(True)) diff --git a/tools/test_runner.py b/tools/test_runner.py index bb1e623ce04..65da72b539d 100644 --- a/tools/test_runner.py +++ b/tools/test_runner.py @@ -28,7 +28,7 @@ import static_mode_white_list def main(): sys.path.append(os.getcwd()) if core.is_compiled_with_cuda() or core.is_compiled_with_rocm(): - if (os.getenv('FLAGS_enable_gpu_memory_usage_log') == None): + if os.getenv('FLAGS_enable_gpu_memory_usage_log') == None: os.environ['FLAGS_enable_gpu_memory_usage_log'] = 'true' os.environ['FLAGS_enable_gpu_memory_usage_log_mb'] = 'false' @@ -52,10 +52,12 @@ def main(): if not res.wasSuccessful(): some_test_failed = True - print(module_name, - 'failed\n', - buffer.getvalue(), - file=sys.stderr) + print( + module_name, + 'failed\n', + buffer.getvalue(), + file=sys.stderr, + ) if flag_need_static_mode: paddle.disable_static() diff --git a/tools/test_sampcd_processor.py b/tools/test_sampcd_processor.py index 42f5d96e71c..d389b2dbc87 100644 --- a/tools/test_sampcd_processor.py +++ b/tools/test_sampcd_processor.py @@ -32,7 +32,6 @@ from sampcd_processor import is_required_match class Test_find_all(unittest.TestCase): - def test_find_none(self): self.assertEqual(0, len(find_all('hello', 'world'))) @@ -40,12 +39,12 @@ class Test_find_all(unittest.TestCase): self.assertListEqual([0], find_all('hello', 'hello')) def test_find_two(self): - self.assertListEqual([1, 15], - find_all(' hello, world; hello paddle!', 'hello')) + self.assertListEqual( + [1, 15], find_all(' hello, world; hello paddle!', 'hello') + ) class Test_find_last_future_line_end(unittest.TestCase): - def test_no_instant(self): samplecodes = """ print(10//3) @@ -60,8 +59,9 @@ class Test_find_last_future_line_end(unittest.TestCase): """ mo = re.search("print_function\n", samplecodes) self.assertIsNotNone(mo) - self.assertGreaterEqual(find_last_future_line_end(samplecodes), - mo.end()) + self.assertGreaterEqual( + find_last_future_line_end(samplecodes), mo.end() + ) def test_2_instant(self): samplecodes = """ @@ -72,12 +72,12 @@ class Test_find_last_future_line_end(unittest.TestCase): """ mo = re.search("division\n", samplecodes) self.assertIsNotNone(mo) - self.assertGreaterEqual(find_last_future_line_end(samplecodes), - mo.end()) + self.assertGreaterEqual( + find_last_future_line_end(samplecodes), mo.end() + ) class Test_extract_code_blocks_from_docstr(unittest.TestCase): - def test_no_samplecode(self): docstr = """ placeholder @@ -103,12 +103,17 @@ class Test_extract_code_blocks_from_docstr(unittest.TestCase): print(1+1) """ codeblocks = extract_code_blocks_from_docstr(docstr) - self.assertListEqual(codeblocks, [{ - 'codes': """print(1+1)""", - 'name': None, - 'id': 1, - 'required': None, - }]) + self.assertListEqual( + codeblocks, + [ + { + 'codes': """print(1+1)""", + 'name': None, + 'id': 1, + 'required': None, + } + ], + ) def test_2_samplecodes(self): docstr = """ @@ -126,22 +131,27 @@ class Test_extract_code_blocks_from_docstr(unittest.TestCase): print(1+1) """ codeblocks = extract_code_blocks_from_docstr(docstr) - self.assertListEqual(codeblocks, [{ - 'codes': """print(1/0)""", - 'name': None, - 'id': 1, - 'required': None, - }, { - 'codes': """# required: gpu + self.assertListEqual( + codeblocks, + [ + { + 'codes': """print(1/0)""", + 'name': None, + 'id': 1, + 'required': None, + }, + { + 'codes': """# required: gpu print(1+1)""", - 'name': 'one_plus_one', - 'id': 2, - 'required': 'gpu', - }]) + 'name': 'one_plus_one', + 'id': 2, + 'required': 'gpu', + }, + ], + ) class Test_insert_codes_into_codeblock(unittest.TestCase): - def test_required_None(self): codeblock = { 'codes': """print(1/0)""", @@ -155,7 +165,8 @@ import os os.environ["CUDA_VISIBLE_DEVICES"] = "" print(1/0) print("not-specified's sample code (name:None, id:1) is executed successfully!")""", - insert_codes_into_codeblock(codeblock)) + insert_codes_into_codeblock(codeblock), + ) def test_required_gpu(self): codeblock = { @@ -172,7 +183,8 @@ os.environ["CUDA_VISIBLE_DEVICES"] = "0" # required: gpu print(1+1) print("not-specified's sample code (name:None, id:1) is executed successfully!")""", - insert_codes_into_codeblock(codeblock)) + insert_codes_into_codeblock(codeblock), + ) def test_from_future(self): codeblock = { @@ -193,7 +205,8 @@ import os os.environ["CUDA_VISIBLE_DEVICES"] = "" print(10//3) print("not-specified's sample code (name:future, id:1) is executed successfully!")""", - insert_codes_into_codeblock(codeblock)) + insert_codes_into_codeblock(codeblock), + ) def clear_capacity(): @@ -204,7 +217,6 @@ def clear_capacity(): class Test_get_test_capacity(unittest.TestCase): - def setUp(self): clear_capacity() get_test_capacity() @@ -216,34 +228,40 @@ class Test_get_test_capacity(unittest.TestCase): def test_NoEnvVar(self): clear_capacity() get_test_capacity() - self.assertCountEqual([ - 'cpu', - ], sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + self.assertCountEqual( + [ + 'cpu', + ], + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY, + ) def test_NoEnvVar_RUN_ON_DEVICE_gpu(self): clear_capacity() sampcd_processor.RUN_ON_DEVICE = 'gpu' get_test_capacity() - self.assertCountEqual(['cpu', 'gpu'], - sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + self.assertCountEqual( + ['cpu', 'gpu'], sampcd_processor.SAMPLE_CODE_TEST_CAPACITY + ) def test_EnvVar_gpu(self): clear_capacity() os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu' get_test_capacity() - self.assertCountEqual(['cpu', 'gpu'], - sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + self.assertCountEqual( + ['cpu', 'gpu'], sampcd_processor.SAMPLE_CODE_TEST_CAPACITY + ) def test_EnvVar_gpu_and_distributed(self): clear_capacity() os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu,distributed' get_test_capacity() - self.assertCountEqual(['cpu', 'gpu', 'distributed'], - sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + self.assertCountEqual( + ['cpu', 'gpu', 'distributed'], + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY, + ) class Test_is_required_match(unittest.TestCase): - def setUp(self): clear_capacity() @@ -284,16 +302,17 @@ class Test_is_required_match(unittest.TestCase): class Test_execute_samplecode(unittest.TestCase): - def setUp(self): if not os.path.exists(sampcd_processor.SAMPLECODE_TEMPDIR): os.mkdir(sampcd_processor.SAMPLECODE_TEMPDIR) self.successSampleCodeFile = os.path.join( - sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_success.py') + sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_success.py' + ) with open(self.successSampleCodeFile, 'w') as f: f.write('print(1+1)') self.failedSampleCodeFile = os.path.join( - sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_failed.py') + sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_failed.py' + ) with open(self.failedSampleCodeFile, 'w') as f: f.write('print(1/0)') @@ -303,7 +322,8 @@ class Test_execute_samplecode(unittest.TestCase): def test_run_success(self): result, tfname, msg, exec_time = execute_samplecode( - self.successSampleCodeFile) + self.successSampleCodeFile + ) self.assertTrue(result) self.assertEqual(self.successSampleCodeFile, tfname) self.assertIsNotNone(msg) @@ -312,7 +332,8 @@ class Test_execute_samplecode(unittest.TestCase): def test_run_failed(self): result, tfname, msg, exec_time = execute_samplecode( - self.failedSampleCodeFile) + self.failedSampleCodeFile + ) self.assertFalse(result) self.assertEqual(self.failedSampleCodeFile, tfname) self.assertIsNotNone(msg) @@ -326,7 +347,6 @@ def clear_summary_info(): class Test_sampcd_extract_to_file(unittest.TestCase): - def setUp(self): if not os.path.exists(sampcd_processor.SAMPLECODE_TEMPDIR): os.mkdir(sampcd_processor.SAMPLECODE_TEMPDIR) @@ -348,10 +368,15 @@ class Test_sampcd_extract_to_file(unittest.TestCase): """ funcname = 'one_plus_one' sample_code_filenames = sampcd_extract_to_file(comments, funcname) - self.assertCountEqual([ - os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, - funcname + '_example.py') - ], sample_code_filenames) + self.assertCountEqual( + [ + os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example.py', + ) + ], + sample_code_filenames, + ) def test_no_samplecode(self): comments = """ @@ -375,12 +400,19 @@ class Test_sampcd_extract_to_file(unittest.TestCase): """ funcname = 'one_plus_one' sample_code_filenames = sampcd_extract_to_file(comments, funcname) - self.assertCountEqual([ - os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, - funcname + '_example_1.py'), - os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, - funcname + '_example_2.py') - ], sample_code_filenames) + self.assertCountEqual( + [ + os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_1.py', + ), + os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_2.py', + ), + ], + sample_code_filenames, + ) def test_2_samplecodes_has_skipped(self): comments = """ @@ -421,71 +453,100 @@ class Test_sampcd_extract_to_file(unittest.TestCase): get_test_capacity() sample_code_filenames = sampcd_extract_to_file(comments, funcname) - self.assertCountEqual([ - os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, - funcname + '_example_2.py') - ], sample_code_filenames) - self.assertCountEqual(sampcd_processor.SUMMARY_INFO['skiptest'], - [funcname + '-1']) - self.assertCountEqual(sampcd_processor.SUMMARY_INFO['gpu'], - [funcname + '-3', funcname + '-6']) - self.assertCountEqual(sampcd_processor.SUMMARY_INFO['xpu'], - [funcname + '-4']) - self.assertCountEqual(sampcd_processor.SUMMARY_INFO['distributed'], - [funcname + '-5']) + self.assertCountEqual( + [ + os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_2.py', + ) + ], + sample_code_filenames, + ) + self.assertCountEqual( + sampcd_processor.SUMMARY_INFO['skiptest'], [funcname + '-1'] + ) + self.assertCountEqual( + sampcd_processor.SUMMARY_INFO['gpu'], + [funcname + '-3', funcname + '-6'], + ) + self.assertCountEqual( + sampcd_processor.SUMMARY_INFO['xpu'], [funcname + '-4'] + ) + self.assertCountEqual( + sampcd_processor.SUMMARY_INFO['distributed'], [funcname + '-5'] + ) class Test_get_api_md5(unittest.TestCase): - def setUp(self): self.api_pr_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec')) + os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec') + ) with open(self.api_pr_spec_filename, 'w') as f: - f.write("\n".join([ - """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", - """paddle.two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55two'))""", - """paddle.three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6cthree'))""", - """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", - """paddle.five_plus_five (ArgSpec(), ('document', 'ff0f188c95030158cc6398d2a6c5five'))""", - ])) + f.write( + "\n".join( + [ + """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", + """paddle.two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55two'))""", + """paddle.three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6cthree'))""", + """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", + """paddle.five_plus_five (ArgSpec(), ('document', 'ff0f188c95030158cc6398d2a6c5five'))""", + ] + ) + ) def tearDown(self): os.remove(self.api_pr_spec_filename) def test_get_api_md5(self): res = get_api_md5('paddle/fluid/API_PR.spec') - self.assertEqual("ff0f188c95030158cc6398d2a6c55one", - res['paddle.one_plus_one']) - self.assertEqual("ff0f188c95030158cc6398d2a6c55two", - res['paddle.two_plus_two']) - self.assertEqual("ff0f188c95030158cc6398d2a6cthree", - res['paddle.three_plus_three']) - self.assertEqual("ff0f188c95030158cc6398d2a6c5four", - res['paddle.four_plus_four']) - self.assertEqual("ff0f188c95030158cc6398d2a6c5five", - res['paddle.five_plus_five']) + self.assertEqual( + "ff0f188c95030158cc6398d2a6c55one", res['paddle.one_plus_one'] + ) + self.assertEqual( + "ff0f188c95030158cc6398d2a6c55two", res['paddle.two_plus_two'] + ) + self.assertEqual( + "ff0f188c95030158cc6398d2a6cthree", res['paddle.three_plus_three'] + ) + self.assertEqual( + "ff0f188c95030158cc6398d2a6c5four", res['paddle.four_plus_four'] + ) + self.assertEqual( + "ff0f188c95030158cc6398d2a6c5five", res['paddle.five_plus_five'] + ) class Test_get_incrementapi(unittest.TestCase): - def setUp(self): self.api_pr_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec')) + os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec') + ) with open(self.api_pr_spec_filename, 'w') as f: - f.write("\n".join([ - """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", - """paddle.two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55two'))""", - """paddle.three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6cthree'))""", - """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", - ])) + f.write( + "\n".join( + [ + """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", + """paddle.two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55two'))""", + """paddle.three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6cthree'))""", + """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", + ] + ) + ) self.api_dev_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "..", 'paddle/fluid/API_DEV.spec')) + os.path.join(os.getcwd(), "..", 'paddle/fluid/API_DEV.spec') + ) with open(self.api_dev_spec_filename, 'w') as f: - f.write("\n".join([ - """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", - ])) + f.write( + "\n".join( + [ + """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", + ] + ) + ) self.api_diff_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "dev_pr_diff_api.spec")) + os.path.join(os.getcwd(), "dev_pr_diff_api.spec") + ) def tearDown(self): os.remove(self.api_pr_spec_filename) @@ -496,10 +557,14 @@ class Test_get_incrementapi(unittest.TestCase): get_incrementapi() with open(self.api_diff_spec_filename, 'r') as f: lines = f.readlines() - self.assertCountEqual([ - "paddle.two_plus_two\n", "paddle.three_plus_three\n", - "paddle.four_plus_four\n" - ], lines) + self.assertCountEqual( + [ + "paddle.two_plus_two\n", + "paddle.three_plus_three\n", + "paddle.four_plus_four\n", + ], + lines, + ) # https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/fluid/layers/ops.py diff --git a/tools/timeline.py b/tools/timeline.py index e3de686f829..5323ea0f907 100644 --- a/tools/timeline.py +++ b/tools/timeline.py @@ -23,16 +23,15 @@ parser.add_argument( type=str, default='', help='Input profile file name. If there are multiple file, the format ' - 'should be trainer1=file1,trainer2=file2,ps=file3') -parser.add_argument('--timeline_path', - type=str, - default='', - help='Output timeline file name.') + 'should be trainer1=file1,trainer2=file2,ps=file3', +) +parser.add_argument( + '--timeline_path', type=str, default='', help='Output timeline file name.' +) args = parser.parse_args() class _ChromeTraceFormatter(object): - def __init__(self): self._events = [] self._metadata = [] @@ -128,7 +127,6 @@ class _ChromeTraceFormatter(object): class Timeline(object): - def __init__(self, profile_dict): self._profile_dict = profile_dict self._pid = 0 @@ -153,13 +151,15 @@ class Timeline(object): self._chrome_trace.emit_pid("%s:cuda_api" % k, pid) else: self._chrome_trace.emit_pid( - "%s:cpu:block:%d" % (k, event.device_id), pid) + "%s:cpu:block:%d" % (k, event.device_id), pid + ) elif event.type == profiler_pb2.Event.GPUKernel: if (k, event.device_id, "GPUKernel") not in self._devices: pid = self._allocate_pid() self._devices[(k, event.device_id, "GPUKernel")] = pid self._chrome_trace.emit_pid( - "%s:gpu:%d" % (k, event.device_id), pid) + "%s:gpu:%d" % (k, event.device_id), pid + ) if not hasattr(profile_pb, "mem_events"): continue for mevent in profile_pb.mem_events: @@ -169,50 +169,63 @@ class Timeline(object): self._mem_devices[(k, mevent.device_id, "GPU")] = pid self._chrome_trace.emit_pid( "memory usage on %s:gpu:%d" % (k, mevent.device_id), - pid) + pid, + ) elif mevent.place == profiler_pb2.MemEvent.CPUPlace: if (k, mevent.device_id, "CPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, mevent.device_id, "CPU")] = pid self._chrome_trace.emit_pid( "memory usage on %s:cpu:%d" % (k, mevent.device_id), - pid) + pid, + ) elif mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace: - if (k, mevent.device_id, - "CUDAPinnedPlace") not in self._mem_devices: + if ( + k, + mevent.device_id, + "CUDAPinnedPlace", + ) not in self._mem_devices: pid = self._allocate_pid() - self._mem_devices[(k, mevent.device_id, - "CUDAPinnedPlace")] = pid + self._mem_devices[ + (k, mevent.device_id, "CUDAPinnedPlace") + ] = pid self._chrome_trace.emit_pid( - "memory usage on %s:cudapinnedplace:%d" % - (k, mevent.device_id), pid) + "memory usage on %s:cudapinnedplace:%d" + % (k, mevent.device_id), + pid, + ) elif mevent.place == profiler_pb2.MemEvent.NPUPlace: if (k, mevent.device_id, "NPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, mevent.device_id, "NPU")] = pid self._chrome_trace.emit_pid( "memory usage on %s:npu:%d" % (k, mevent.device_id), - pid) + pid, + ) if (k, 0, "CPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, 0, "CPU")] = pid self._chrome_trace.emit_pid( - "memory usage on %s:cpu:%d" % (k, 0), pid) + "memory usage on %s:cpu:%d" % (k, 0), pid + ) if (k, 0, "GPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, 0, "GPU")] = pid self._chrome_trace.emit_pid( - "memory usage on %s:gpu:%d" % (k, 0), pid) + "memory usage on %s:gpu:%d" % (k, 0), pid + ) if (k, 0, "CUDAPinnedPlace") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, 0, "CUDAPinnedPlace")] = pid self._chrome_trace.emit_pid( - "memory usage on %s:cudapinnedplace:%d" % (k, 0), pid) + "memory usage on %s:cudapinnedplace:%d" % (k, 0), pid + ) if (k, 0, "NPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, 0, "NPU")] = pid self._chrome_trace.emit_pid( - "memory usage on %s:npu:%d" % (k, 0), pid) + "memory usage on %s:npu:%d" % (k, 0), pid + ) def _allocate_events(self): for k, profile_pb in self._profile_dict.items(): @@ -230,8 +243,14 @@ class Timeline(object): # TODO(panyx0718): Chrome tracing only handles ms. However, some # ops takes micro-seconds. Hence, we keep the ns here. self._chrome_trace.emit_region( - event.start_ns, (event.end_ns - event.start_ns) / 1.0, pid, - event.sub_device_id, 'Op', event.name, args) + event.start_ns, + (event.end_ns - event.start_ns) / 1.0, + pid, + event.sub_device_id, + 'Op', + event.name, + args, + ) def _allocate_memory_event(self): if not hasattr(profiler_pb2, "MemEvent"): @@ -240,7 +259,7 @@ class Timeline(object): profiler_pb2.MemEvent.CPUPlace: "CPU", profiler_pb2.MemEvent.CUDAPlace: "GPU", profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace", - profiler_pb2.MemEvent.NPUPlace: "NPU" + profiler_pb2.MemEvent.NPUPlace: "NPU", } for k, profile_pb in self._profile_dict.items(): mem_list = [] @@ -273,15 +292,21 @@ class Timeline(object): total_size = 0 while i < len(mem_list): total_size += mem_list[i]['size'] - while i < len(mem_list) - 1 and mem_list[i]['time'] == mem_list[ - i + 1]['time']: + while ( + i < len(mem_list) - 1 + and mem_list[i]['time'] == mem_list[i + 1]['time'] + ): total_size += mem_list[i + 1]['size'] i += 1 - self._chrome_trace.emit_counter("Memory", "Memory", - mem_list[i]['pid'], - mem_list[i]['time'], 0, - total_size) + self._chrome_trace.emit_counter( + "Memory", + "Memory", + mem_list[i]['pid'], + mem_list[i]['time'], + 0, + total_size, + ) i += 1 def generate_chrome_trace(self): diff --git a/tools/windows/get_prec_ut_list.py b/tools/windows/get_prec_ut_list.py index 4564c75910e..b5bb6a23673 100644 --- a/tools/windows/get_prec_ut_list.py +++ b/tools/windows/get_prec_ut_list.py @@ -47,5 +47,5 @@ if __name__ == '__main__': file_path = os.path.join(BUILD_DIR, 'all_ut_list') with open(file_path, 'r') as f: all_test_cases = f.read() - #prec_test_cases = sys.argv[2] + # prec_test_cases = sys.argv[2] get_prec_ut_list(all_test_cases, prec_test_cases) -- GitLab